diff --git a/.gitignore b/.gitignore index ef13cda3c36ace94e053843989bfeb0443cd408d..13f4d24877b4dab0aa7a99132c896ae2636e32e7 100644 --- a/.gitignore +++ b/.gitignore @@ -31,3 +31,14 @@ next-env.d.ts .cache/ pipeline_revised.py .local-data/ + +# Local data dumps. The runtime reads from .cache/hf-data/duckdb parquets; +# anything under /data/ is leftover from offline scripts and should not +# ship in git. The survey schema (loaded statically by app/survey) is the +# one exception. +/data/* +!/data/survey/ +/data/survey/* +!/data/survey/eval-schema-fields.json +mock_design/ +shoot.mjs diff --git a/app/developers/[id]/page.tsx b/app/developers/[id]/page.tsx index fc8024c7bddfc576bf41830ef4a0b2907f877c4d..fa92b11f80f3d60e23f0ae9703834a366c8a696d 100644 --- a/app/developers/[id]/page.tsx +++ b/app/developers/[id]/page.tsx @@ -9,7 +9,6 @@ import { InfiniteScrollSentinel } from "@/components/infinite-scroll" import { ModelTable } from "@/components/model-table" import { Navigation } from "@/components/navigation" import type { BenchmarkCard } from "@/lib/benchmark-schema" -import { lookupBenchmarkCard } from "@/lib/benchmark-metadata-utils" import { fetchDeveloperSummary, fetchBenchmarkMetadata } from "@/lib/dashboard-data-client" const PAGE_SIZE = 40 @@ -59,24 +58,6 @@ export default function DeveloperDetailPage() { [models] ) - // Collect unique domains from benchmarks this developer's models are evaluated on - const domainCoverage = useMemo(() => { - const domainMap = new Map>() - for (const model of models) { - for (const { benchmark } of model.top_scores) { - const card = lookupBenchmarkCard(benchmarkCards, benchmark) - for (const domain of card?.benchmark_details?.domains ?? []) { - const existing = domainMap.get(domain) ?? new Set() - existing.add(benchmark) - domainMap.set(domain, existing) - } - } - } - return Array.from(domainMap.entries()) - .map(([domain, benchmarks]) => ({ domain, count: benchmarks.size })) - .sort((a, b) => b.count - a.count) - }, [models, benchmarkCards]) - const filteredModels = useMemo(() => { const query = searchQuery.trim().toLowerCase() const filtered = query @@ -207,12 +188,6 @@ export default function DeveloperDetailPage() { · {totalResults.toLocaleString()} results - {domainCoverage.length > 0 && ( - - · {domainCoverage.length} - domains - - )} @@ -239,27 +214,6 @@ export default function DeveloperDetailPage() { - {/* DOMAIN COVERAGE — hairline tag row ---------------------- */} - {domainCoverage.length > 0 && ( -
-
Benchmark domain coverage
-
- {domainCoverage.map(({ domain, count }) => ( - - {domain} - - {count} - - - ))} -
-
- )} - {/* TABLE ---------------------------------------------------- */} {filteredModels.length === 0 ? (
diff --git a/app/evals/[id]/page.tsx b/app/evals/[id]/page.tsx index 45c6ac13da37372e6a56139d3722204555f5127e..c0a061c8c34af1efb25fd0f6ff6231fc7bd1353e 100644 --- a/app/evals/[id]/page.tsx +++ b/app/evals/[id]/page.tsx @@ -6,27 +6,11 @@ import Link from "next/link" import { ArrowLeft, ArrowUpRight, BarChart3, Grid3X3, Search } from "lucide-react" import { Navigation } from "@/components/navigation" import { EvalDetail } from "@/components/eval-detail" +import { ParamRangePicker } from "@/components/param-range-picker" import { useAudienceMode } from "@/components/audience-mode-provider" import type { BenchmarkEvalSummary } from "@/lib/eval-processing" import { fetchEvalSummary } from "@/lib/dashboard-data-client" - -const PARAM_RANGE_VALUES = [1, 2, 3, 4, 6, 8, 10, 12, 16, 24, 32, 48, 64, 96, 128, 192, 256, 384, 500] as const -const PARAM_RANGE_MARKERS = [ - { label: "< 1B", step: 0 }, - { label: "6B", step: PARAM_RANGE_VALUES.indexOf(6) }, - { label: "12B", step: PARAM_RANGE_VALUES.indexOf(12) }, - { label: "32B", step: PARAM_RANGE_VALUES.indexOf(32) }, - { label: "128B", step: PARAM_RANGE_VALUES.indexOf(128) }, - { label: "> 500B", step: PARAM_RANGE_VALUES.length - 1 }, -] as const - -function formatParamBoundLabel(step: number, bound: "min" | "max") { - const maxStepIndex = PARAM_RANGE_VALUES.length - 1 - if (bound === "min" && step <= 0) return "< 1B" - if (bound === "max" && step >= maxStepIndex) return "> 500B" - const value = PARAM_RANGE_VALUES[step] - return value != null ? `${value}B` : "Not reported" -} +import { PARAM_RANGE_MAX_INDEX, parseParamsBillionsFromModelName, paramStepToNumeric } from "@/lib/param-range" export default function EvalDetailPage() { const params = useParams() @@ -192,10 +176,7 @@ function CompositeEvalView({
{/* HERO ------------------------------------------------------------- */}
-
- {isPolicy ? "Benchmark suite" : "Composite · §3.2"} -
-

{summary.evaluation_name}

+

{summary.evaluation_name}

>(new Set()) const [minParamStep, setMinParamStep] = useState(0) - const [maxParamStep, setMaxParamStep] = useState(PARAM_RANGE_VALUES.length - 1) + const [maxParamStep, setMaxParamStep] = useState(PARAM_RANGE_MAX_INDEX) const PAGE_SIZE = 50 const metricDirection = useMemo(() => { @@ -470,9 +451,9 @@ function MatrixLeaderboard({ const avg = validScores.length > 0 ? validScores.reduce((a, b) => a + b, 0) / validScores.length : 0 - let sizeB: number | null = null - const sizeMatch = (data.name + " " + id).match(/\b(\d+(?:\.\d+)?)\s*[bB]\b/) - if (sizeMatch) sizeB = parseFloat(sizeMatch[1]) + const sizeB = + parseParamsBillionsFromModelName(data.name) ?? + parseParamsBillionsFromModelName(id) return { id, name: data.name, developer: data.developer, avg, scores: data.scores, sizeB } }) @@ -496,9 +477,11 @@ function MatrixLeaderboard({ }) }, [models, sortCol, sortAsc]) - const maxStepIndex = PARAM_RANGE_VALUES.length - 1 - const numericMinParams = minParamStep <= 0 ? null : (PARAM_RANGE_VALUES[minParamStep] ?? null) - const numericMaxParams = maxParamStep >= maxStepIndex ? null : (PARAM_RANGE_VALUES[maxParamStep] ?? null) + const numericMinParams = paramStepToNumeric(minParamStep, "min") + const numericMaxParams = paramStepToNumeric(maxParamStep, "max") + const [showUnknownSize, setShowUnknownSize] = useState(true) + + const hasParameterData = useMemo(() => models.some((m) => m.sizeB != null), [models]) const query = search.trim().toLowerCase() const filteredModels = sortedModels.filter((m) => { @@ -507,8 +490,9 @@ function MatrixLeaderboard({ m.developer.toLowerCase().includes(query) || m.id.toLowerCase().includes(query) )) return false - if (numericMinParams != null && (m.sizeB == null || m.sizeB < numericMinParams)) return false - if (numericMaxParams != null && (m.sizeB == null || m.sizeB > numericMaxParams)) return false + if (m.sizeB == null) return showUnknownSize + if (numericMinParams != null && m.sizeB < numericMinParams) return false + if (numericMaxParams != null && m.sizeB > numericMaxParams) return false return true }) @@ -594,99 +578,23 @@ function MatrixLeaderboard({ />
- {/* Param slider */} -
- - Params - -
-
- {PARAM_RANGE_MARKERS.map((marker) => ( - - {marker.label} - - ))} -
- -
-
-
-
-
- -
- {PARAM_RANGE_VALUES.map((_, stepIndex) => ( -
- - { - const v = Number(e.target.value) - setMinParamStep(Math.min(v, maxParamStep)) - }} - className="param-range-input" - aria-label="Minimum parameter filter" - /> - { - const v = Number(e.target.value) - setMaxParamStep(Math.max(v, minParamStep)) - }} - className="param-range-input" - aria-label="Maximum parameter filter" - /> -
-
- - - {formatParamBoundLabel(minParamStep, "min")} – {formatParamBoundLabel(maxParamStep, "max")} - -
+ {hasParameterData && ( + { + setMinParamStep(0) + setMaxParamStep(PARAM_RANGE_MAX_INDEX) + }} + showUnknownSize={showUnknownSize} + onShowUnknownSizeChange={setShowUnknownSize} + className="min-w-[260px] flex-1 sm:max-w-[420px]" + /> + )}
(null) const [totalModels, setTotalModels] = useState(0) + const [evalItems, setEvalItems] = useState>(new Map()) + const [benchmarkCards, setBenchmarkCards] = useState>({}) const [loading, setLoading] = useState(true) const [searchQuery, setSearchQuery] = useState("") const [sortBy, setSortBy] = useState("results") const [visibleCount, setVisibleCount] = useState(PAGE_SIZE) + const [domainPanelOpen, setDomainPanelOpen] = useState(false) + const [domainFilter, setDomainFilter] = useState>(new Set()) + const [selectedCategories, setSelectedCategories] = useState([]) const deferredSearchQuery = useDeferredValue(searchQuery) useEffect(() => { - Promise.all([fetchEvalHierarchy(), fetchEvalList()]) - .then(([h, list]) => { + Promise.all([fetchEvalHierarchy(), fetchEvalList(), fetchBenchmarkMetadata()]) + .then(([h, list, metadata]) => { setHierarchy(h) setTotalModels(list.totalModels) + const map = new Map() + for (const item of list.evals) map.set(item.evaluation_id, item) + setEvalItems(map) + setBenchmarkCards(metadata) }) .catch(console.error) .finally(() => setLoading(false)) @@ -72,6 +83,86 @@ export default function EvalsPage() { const families = hierarchy?.families ?? [] + // Build a domain → family-count map. The lite eval list doesn't carry + // benchmark cards, so we read domains from `benchmark-metadata.json` + // (keyed by benchmark / leaf / family key). For each family we union + // the domains across the family key itself and every leaf key, then + // count one bump per family per distinct domain. + const familyDomains = useMemo(() => { + const out = new Map>() + const lookupDomains = (key: string | null | undefined): string[] => { + if (!key) return [] + const card = benchmarkCards[key] + const domains = card?.benchmark_details?.domains + return Array.isArray(domains) ? domains : [] + } + for (const fam of families) { + const seen = new Set() + for (const d of lookupDomains(fam.key)) seen.add(d.trim().toLowerCase()) + for (const leaf of fam.leaves ?? []) { + for (const d of leaf.tags?.domains ?? []) seen.add(d.trim().toLowerCase()) + for (const d of lookupDomains(leaf.key)) seen.add(d.trim().toLowerCase()) + } + for (const id of fam.eval_summary_ids ?? []) { + for (const d of lookupDomains(id)) seen.add(d.trim().toLowerCase()) + } + seen.delete("") + out.set(fam.key, seen) + } + return out + }, [families, benchmarkCards]) + + // Domain → display label (from the first non-empty card occurrence) + + // count of families touching that domain. Sorted descending by count. + const domainCounts = useMemo(() => { + const counts = new Map() + const labels = new Map() + const recordLabel = (raw: string) => { + const key = raw.trim().toLowerCase() + if (!key || labels.has(key)) return + labels.set(key, raw.trim()) + } + for (const card of Object.values(benchmarkCards)) { + for (const d of card?.benchmark_details?.domains ?? []) recordLabel(d) + } + for (const fam of families) { + for (const leaf of fam.leaves ?? []) { + for (const d of leaf.tags?.domains ?? []) recordLabel(d) + } + } + for (const set of familyDomains.values()) { + for (const key of set) counts.set(key, (counts.get(key) ?? 0) + 1) + } + return Array.from(counts.entries()) + .map(([key, count]) => ({ domain: labels.get(key) ?? key, count, key })) + .sort((a, b) => b.count - a.count || a.domain.localeCompare(b.domain)) + }, [familyDomains, families, benchmarkCards]) + + const toggleDomain = useCallback((domain: string) => { + setDomainFilter((current) => { + const next = new Set(current) + if (next.has(domain)) next.delete(domain) + else next.add(domain) + return next + }) + }, []) + + const clearDomainFilter = useCallback(() => setDomainFilter(new Set()), []) + + // Categories present on the family list — drives the pill selector + // below the toolbar. Sort them by descending family count so the most + // common ones surface first. + const availableCategories = useMemo(() => { + const counts = new Map() + for (const fam of families) { + const cat = fam.category ?? "General" + counts.set(cat, (counts.get(cat) ?? 0) + 1) + } + return Array.from(counts.entries()) + .sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])) + .map(([category]) => category) + }, [families]) + const filteredFamilies = useMemo(() => { const query = deferredSearchQuery.trim().toLowerCase() let list = families @@ -85,6 +176,20 @@ export default function EvalsPage() { ) } + if (selectedCategories.length > 0) { + const set = new Set(selectedCategories) + list = list.filter((fam) => set.has(fam.category ?? "General")) + } + + if (domainFilter.size > 0) { + list = list.filter((fam) => { + const set = familyDomains.get(fam.key) + if (!set) return false + for (const key of set) if (domainFilter.has(key)) return true + return false + }) + } + return list.slice().sort((a, b) => { switch (sortBy) { case "name": @@ -98,11 +203,11 @@ export default function EvalsPage() { return familyEvalsCount(b) - familyEvalsCount(a) } }) - }, [families, deferredSearchQuery, sortBy]) + }, [families, deferredSearchQuery, sortBy, domainFilter, selectedCategories, familyDomains]) useEffect(() => { setVisibleCount(PAGE_SIZE) - }, [deferredSearchQuery, sortBy]) + }, [deferredSearchQuery, sortBy, domainFilter, selectedCategories]) const visibleFamilies = useMemo( () => filteredFamilies.slice(0, visibleCount), @@ -126,8 +231,7 @@ export default function EvalsPage() {

Evaluations are grouped into families. A family holds one or more benchmarks; each benchmark has one or more slices; each slice reports one or more - metrics. Metrics are not commensurable across rows — compare within a cell, not - across cells. + metrics.

{/* META ROW ------------------------------------------------- */} @@ -180,6 +284,41 @@ export default function EvalsPage() {
+ {domainCounts.length > 0 && ( + + )} + - setMinParamStep(Math.min(Number(event.target.value), maxParamStep)) - } - className="param-range-input" - aria-label="Minimum parameter filter" - /> - - setMaxParamStep(Math.max(Number(event.target.value), minParamStep)) - } - className="param-range-input" - aria-label="Maximum parameter filter" - /> -
- - {formatParamBoundLabel(minParamStep, "min")} – {formatParamBoundLabel(maxParamStep, "max")} - - {(minParamStep > 0 || maxParamStep < maxParamStepIndex) && ( - - )} -
- - )} - { - const nextMin = Number(event.target.value) - setMinParamStep(Math.min(nextMin, maxParamStep)) - }} - className="param-range-input" - aria-label="Minimum parameter filter" - /> - - { - const nextMax = Number(event.target.value) - setMaxParamStep(Math.max(nextMax, minParamStep)) - }} - className="param-range-input" - aria-label="Maximum parameter filter" - /> -
-
- - - {formatParamBoundLabel(minParamStep, "min")} to {formatParamBoundLabel(maxParamStep, "max")} - -
-
- - )} + showUnknownSize={showUnknownSize} + onShowUnknownSizeChange={setShowUnknownSize} + /> + + )} +
@@ -1204,7 +1020,6 @@ export function EvalDetail({ summary }: EvalDetailProps) { Avg of {modelResult.aggregate_components.length} )} - @@ -1375,10 +1190,6 @@ export function EvalDetail({ summary }: EvalDetailProps) { )} - {!isResearchView && ( - - )} - ("coverage") + // Default sort: the first root-scope metric (the benchmark's overall + // score), falling back to the first metric overall, then to model name. + // We don't sort by metric coverage by default — coverage tells you how + // many slices reported, not how the model performed. + const [sortKey, setSortKey] = useState(() => { + const metrics = summary.leaderboard_metrics ?? [] + const root = metrics.find((m) => m.scope === "root") + return root?.column_key ?? metrics[0]?.column_key ?? "model" + }) const [sortDirection, setSortDirection] = useState<"asc" | "desc">("desc") const [activeSubtaskTab, setActiveSubtaskTab] = useState("all") const [minParamStep, setMinParamStep] = useState(0) - const [maxParamStep, setMaxParamStep] = useState(PARAM_RANGE_VALUES.length - 1) + const [maxParamStep, setMaxParamStep] = useState(PARAM_RANGE_MAX_INDEX) const [expandedRows, setExpandedRows] = useState>({}) // Index ModelResultForBenchmark entries by model_info.id so we can power the @@ -1678,7 +1497,6 @@ function MultiMetricLeaderboard({ [allMetricKeys] ) const [visibleMetricKeys, setVisibleMetricKeys] = useState(() => defaultVisibleMetricKeys) - const maxParamStepIndex = PARAM_RANGE_VALUES.length - 1 const leaderboardMetricMap = useMemo( () => new Map(leaderboardMetrics.map((metric) => [metric.column_key, metric])), [leaderboardMetrics] @@ -1725,38 +1543,26 @@ function MultiMetricLeaderboard({ [visibleMetrics] ) - const numericMinParams = useMemo(() => { - if (minParamStep <= 0) { - return null - } - - return PARAM_RANGE_VALUES[minParamStep] ?? null - }, [minParamStep]) + const numericMinParams = useMemo(() => paramStepToNumeric(minParamStep, "min"), [minParamStep]) + const numericMaxParams = useMemo(() => paramStepToNumeric(maxParamStep, "max"), [maxParamStep]) + const [showUnknownSize, setShowUnknownSize] = useState(true) - const numericMaxParams = useMemo(() => { - if (maxParamStep >= PARAM_RANGE_VALUES.length - 1) { - return null - } - - return PARAM_RANGE_VALUES[maxParamStep] ?? null - }, [maxParamStep]) + const hasParameterData = useMemo( + () => leaderboardRows.some((row) => getParamsBillionsFromModelInfo(row.model_info) != null), + [leaderboardRows] + ) const filteredRows = useMemo(() => { - return leaderboardRows - .filter((row) => { - const paramsBillions = getParamsBillionsFromModelInfo(row.model_info) + return leaderboardRows.filter((row) => { + const paramsBillions = getParamsBillionsFromModelInfo(row.model_info) - if (numericMinParams != null && (paramsBillions == null || paramsBillions < numericMinParams)) { - return false - } - - if (numericMaxParams != null && (paramsBillions == null || paramsBillions > numericMaxParams)) { - return false - } + if (paramsBillions == null) return showUnknownSize - return true - }) - }, [leaderboardRows, numericMaxParams, numericMinParams]) + if (numericMinParams != null && paramsBillions < numericMinParams) return false + if (numericMaxParams != null && paramsBillions > numericMaxParams) return false + return true + }) + }, [leaderboardRows, numericMaxParams, numericMinParams, showUnknownSize]) const sortedRows = useMemo(() => { const rows = [...filteredRows] @@ -1789,11 +1595,6 @@ function MultiMetricLeaderboard({ return sortDirection === "asc" ? comparison : -comparison } - if (sortKey === "coverage") { - const comparison = left.metrics_present - right.metrics_present || compareNames(left, right) - return sortDirection === "asc" ? comparison : -comparison - } - if (sortKey === "updated") { const comparison = compareTimestamps(left.evaluation_timestamp, right.evaluation_timestamp) || compareNames(left, right) return sortDirection === "asc" ? comparison : -comparison @@ -1836,10 +1637,19 @@ function MultiMetricLeaderboard({ useEffect(() => { if (leaderboardMetricMap.has(sortKey) && !visibleMetricColumnKeySet.has(sortKey)) { - setSortKey("coverage") + // The currently-sorted metric was hidden — fall back to the first + // visible root-scope metric, then the first visible metric overall, + // then to the model name. + const visibleRoot = leaderboardMetrics.find( + (m) => m.scope === "root" && visibleMetricColumnKeySet.has(m.column_key), + ) + const fallback = visibleRoot?.column_key + ?? leaderboardMetrics.find((m) => visibleMetricColumnKeySet.has(m.column_key))?.column_key + ?? "model" + setSortKey(fallback) setSortDirection("desc") } - }, [leaderboardMetricMap, sortKey, visibleMetricColumnKeySet]) + }, [leaderboardMetricMap, leaderboardMetrics, sortKey, visibleMetricColumnKeySet]) useEffect(() => { if (!hasSubtaskTabs) { @@ -1858,11 +1668,6 @@ function MultiMetricLeaderboard({ } }, [activeSubtaskTab, hasSubtaskTabs, singleMetricSubtaskTabs]) - const hasParameterData = useMemo( - () => leaderboardRows.some((row) => getParamsBillionsFromModelInfo(row.model_info) != null), - [leaderboardRows] - ) - const pagedRows = useMemo( () => sortedRows.slice(0, page * 50), [page, sortedRows] @@ -1883,18 +1688,12 @@ function MultiMetricLeaderboard({ }) } - const getVisibleMetricCount = (row: LeaderboardMatrixRow) => - visibleMetrics.reduce( - (count, metric) => count + (isNumericScore(row.values[metric.column_key]) ? 1 : 0), - 0 - ) - const getDefaultSortDirection = (key: string): "asc" | "desc" => { if (key === "model" || key === "developer") { return "asc" } - if (key === "updated" || key === "coverage") { + if (key === "updated") { return "desc" } @@ -2033,84 +1832,21 @@ function MultiMetricLeaderboard({ {hasParameterData && (
-
-
-
- Parameter range -
-
- Narrow the matrix to comparable model sizes. -
-
- -
-
-
- {PARAM_RANGE_MARKERS.map((marker) => ( - - {marker.label} - - ))} -
- -
-
-
-
-
- -
- {PARAM_RANGE_VALUES.map((_, stepIndex) => ( -
- - { - const nextMin = Number(event.target.value) - setMinParamStep(Math.min(nextMin, maxParamStep)) - }} - className="param-range-input" - aria-label="Minimum parameter filter" - /> - - { - const nextMax = Number(event.target.value) - setMaxParamStep(Math.max(nextMax, minParamStep)) - }} - className="param-range-input" - aria-label="Maximum parameter filter" - /> -
-
- - - {formatParamBoundLabel(minParamStep, "min")} to {formatParamBoundLabel(maxParamStep, "max")} - -
-
+ { + setMinParamStep(0) + setMaxParamStep(PARAM_RANGE_MAX_INDEX) + }} + showUnknownSize={showUnknownSize} + onShowUnknownSizeChange={setShowUnknownSize} + />
)} @@ -2135,13 +1871,6 @@ function MultiMetricLeaderboard({ {isResearchView ? "Developer" : "Provider"} {getSortIndicator("developer")} -
{visibleMetrics.map((metric) => { const showSubtaskTopline = !hasSubtaskTabs && @@ -2248,10 +1977,6 @@ function MultiMetricLeaderboard({ > {row.model_info.developer ?? "Unknown developer"} - @@ -2262,11 +1987,6 @@ function MultiMetricLeaderboard({ - - {visibleMetrics.map((metric) => { const score = row.values[metric.column_key] const annotations = row.annotations_by_metric?.[metric.column_key] @@ -2561,23 +2281,40 @@ function BenchmarkCardPanel({ const license = ethical.data_licensing ?? "" const shortLicense = license && license !== "Not specified" ? license : null + // The outer collapsible trigger names the panel; the prominent top + // strip surfaces what readers most often want at a glance — domain + // and language tags, license, and any flagged/missing-field badge. + const hasChipStrip = + domains.length > 0 || + languages.length > 0 || + Boolean(shortLicense) || + flaggedFields.length > 0 || + missingFields.length > 0 + return (
-
-
- - - Benchmark Card - - {shortLicense && ( - {shortLicense} - )} + {hasChipStrip && ( +
+ {domains.map((d) => ( + + + {d} + + ))} + {languages.map((l) => ( + + + {l} + + ))} + {shortLicense && {shortLicense}} {(flaggedFields.length > 0 || missingFields.length > 0) && ( )}
-
- Structured metadata about this benchmark: what it measures, how it was built, and known limitations. -
-
+ )}
{knownIssues.length > 0 && } - {/* Overview + domains */} -
-

{details.overview}

- -
- {domains.map((d) => ( - - - {d} - - ))} - {languages.map((l) => ( - - - {l} - - ))} -
-
-
{/* Goal */}
= { - General: "bg-sky-400", - Reasoning: "bg-violet-400", - Agentic: "bg-amber-400", - Safety: "bg-rose-400", - Code: "bg-emerald-400", - Math: "bg-indigo-400", - Multilingual: "bg-teal-400", -} +import type { BenchmarkCard, CategoryType } from "@/lib/benchmark-schema" +import type { BenchmarkEvalListItem } from "@/lib/eval-processing" + +/** + * Per-category chip colour. Uses oklch tokens so the chip reads against + * both light and dark backgrounds; the saturation is held low to stay in + * the editorial palette (no candy-bright accents). + */ +// Categories use the neutral chip styling — colour-coded chips read as +// noise against the editorial palette. const LEAVES_INLINE_MIN = 2 const LEAVES_INLINE_MAX = 50 @@ -23,89 +22,172 @@ const LEAVES_INLINE_MAX = 50 interface FamilyTableProps { families: HierarchyFamily[] totalModels: number + evalItems?: Map + /** Optional benchmark-metadata index (keyed by benchmark / leaf / family + * key). Used to look up per-leaf domains when the hierarchy doesn't + * carry `leaf.tags.domains`, so the domain filter works on data that + * only ships domains via the metadata file. */ + benchmarkCards?: Record + /** Lower-cased domain slugs to filter the listing. When non-empty, every + * expandable family is auto-expanded and its leaves are restricted to + * those that touch one of the selected domains. Single-benchmark + * families are kept only when their domains intersect the filter. + * Pass `null`/`undefined` to disable filtering. */ + domainFilter?: Set | null } function slugify(value: string | null | undefined): string { return (value ?? "").toLowerCase().replace(/[^a-z0-9]+/g, "") } +/** Render a family key as a human-readable title — used as a fallback when + * the backend `display_name` is misleading (e.g. names a single leaf instead + * of the family). Common acronyms stay uppercase; everything else is title + * case. */ +const FAMILY_KEY_ACRONYMS = new Set([ + "llm", "llms", "aa", "hf", "api", "cli", "sql", "gpt", "qa", "ai", "ml", + "nlp", "rl", "vqa", "vlm", "mt", "cv", +]) +function humanizeFamilyKey(key: string): string { + return key + .split(/[_\-\s]+/) + .filter(Boolean) + .map((word) => { + if (FAMILY_KEY_ACRONYMS.has(word.toLowerCase())) return word.toUpperCase() + return word.charAt(0).toUpperCase() + word.slice(1).toLowerCase() + }) + .join("-") +} + interface LeafEntry { id: string leafKey: string leafName: string evalsCount: number + domains: string[] } -function collectLeafEntries(fam: HierarchyFamily): LeafEntry[] { +function collectLeafEntries( + fam: HierarchyFamily, + benchmarkCards?: Record, +): LeafEntry[] { const out: LeafEntry[] = [] for (const leaf of fam.leaves ?? []) { const ids = leaf.eval_summary_ids ?? [] if (ids.length === 0) continue + // Domain sources, in order of trust: + // (1) hierarchy `leaf.tags.domains` — sometimes absent + // (2) benchmark-metadata keyed by leaf.key + // (3) benchmark-metadata keyed by the leaf's eval_summary_id + const collected = new Set() + for (const d of leaf.tags?.domains ?? []) collected.add(d.toLowerCase()) + const cardByLeaf = benchmarkCards?.[leaf.key] + for (const d of cardByLeaf?.benchmark_details?.domains ?? []) collected.add(d.toLowerCase()) + for (const id of ids) { + const cardById = benchmarkCards?.[id] + for (const d of cardById?.benchmark_details?.domains ?? []) collected.add(d.toLowerCase()) + } out.push({ id: ids[0], leafKey: leaf.key, leafName: leaf.display_name || leaf.key, evalsCount: leaf.evals_count ?? ids.length, + domains: Array.from(collected), }) } return out } /** - * Pick the eval_summary_id that best matches a family's stated display_name. + * Pick the eval_summary_id to navigate to when the user clicks the family + * row. Returns null when the family has no genuine family-level summary — + * in that case the row click should expand the leaf list instead of + * opening one arbitrary child. * - * Backend hierarchy data sometimes has a family whose display_name names one - * specific leaf (e.g. family `llm_stats`, display_name "HumanEval", with 471 - * leaves). The legacy "directIds[0]" pick navigates to whichever leaf was - * processed first (often `aa_index`) — wrong. This helper: + * Some backend families flatten their leaf eval_summary_ids into the + * family's own `eval_summary_ids` array (e.g. family `llm_stats` whose + * direct ids are `llm_stats_aa_index`, `llm_stats_humaneval`, ... — each + * a leaf summary). Those are NOT family-level composites; treating them + * as such is what made clicking "LLM-Stats" land on AA Index. * - * 1. If there is a leaf whose slug matches the family's display_name slug, - * prefer that leaf's id. (`HumanEval` → leaf `humaneval`.) - * 2. Else if there is a direct family-level id whose slug equals the family - * key slug, prefer that (genuine family-level page). - * 3. Otherwise fall back to the first available id. + * We filter direct ids down to those that are NOT also leaf ids. Whatever + * remains is a real family-level summary. Then we apply slug-based + * priority among those. */ function pickFamilyNavId(fam: HierarchyFamily, leafEntries: LeafEntry[]): string | null { const directIds = fam.eval_summary_ids ?? [] - const all: Array<{ id: string; source: "direct" | "leaf"; leafKey?: string; leafName?: string }> = [ - ...directIds.map((id) => ({ id, source: "direct" as const })), - ...leafEntries.map((l) => ({ id: l.id, source: "leaf" as const, leafKey: l.leafKey, leafName: l.leafName })), - ] - if (all.length === 0) return null - if (all.length === 1) return all[0].id + const leafIdSet = new Set(leafEntries.map((l) => l.id)) + + // Real family-level summaries: direct ids that aren't actually leaf ids + // pulled up to the family. These resolve to is_aggregated/composite + // summaries on the detail page. + const compositeDirectIds = directIds.filter((id) => !leafIdSet.has(id)) + + if (compositeDirectIds.length === 0) { + // No genuine family-level composite. If there's exactly one leaf, the + // family is just that leaf in disguise — open it. Otherwise return + // null and let the caller expand the list. + if (leafEntries.length === 1) return leafEntries[0].id + return null + } + + if (compositeDirectIds.length === 1) return compositeDirectIds[0] const famNameSlug = slugify(fam.display_name) const famKeySlug = slugify(fam.key) - // 1. Leaf slug matches family display_name: e.g. display "HumanEval" → leaf "humaneval" - if (famNameSlug && famNameSlug !== famKeySlug) { - for (const entry of all) { - if (entry.source !== "leaf") continue - if (slugify(entry.leafKey) === famNameSlug || slugify(entry.leafName) === famNameSlug) { - return entry.id - } + // 1. Direct composite whose slug equals the family display_name slug + if (famNameSlug) { + for (const id of compositeDirectIds) { + if (slugify(id) === famNameSlug) return id } } - // 2. Direct family-level id: id slug equals family key slug - for (const entry of all) { - if (entry.source !== "direct") continue - if (slugify(entry.id) === famKeySlug) return entry.id + // 2. Direct composite whose slug equals the family key slug + for (const id of compositeDirectIds) { + if (slugify(id) === famKeySlug) return id } - // 3. Direct id starting with the family key only (a true family-level summary) - for (const entry of all) { - if (entry.source !== "direct") continue - const idSlug = slugify(entry.id) - if (idSlug.startsWith(famKeySlug) && idSlug.length === famKeySlug.length) { - return entry.id - } - } + // 3. First direct composite + return compositeDirectIds[0] +} + +/** Returns a one-line description for the family — but only when the + * description applies to the whole family. Specifically: we only use the + * benchmark_card overview attached to the family's *own* navigation + * target (a family-level/composite eval). We don't borrow descriptions + * from individual leaves, because a leaf's description describes that + * one benchmark, not the family as a whole. */ +function pickFamilyDescription( + navId: string | null, + leafEntries: LeafEntry[], + evalItems: Map | undefined, +): string | null { + if (!evalItems || !navId) return null + // If navId resolved to a leaf (single-benchmark family), the leaf's + // description IS the family's description — that case is fine. + // If navId resolved to a composite, ditto. The only case we exclude is + // navId === null (no family-level summary), which the early return + // covers. + void leafEntries + const overview = evalItems.get(navId)?.benchmark_card?.benchmark_details?.overview + if (!overview) return null + return overview.length > 140 ? overview.slice(0, 137) + "…" : overview +} - // 4. Fall back: leaves first, then direct - const leafFallback = all.find((e) => e.source === "leaf") - if (leafFallback) return leafFallback.id - return all[0].id +/** Detects whether the family's `display_name` is misleading: backend data + * sometimes labels a family after one of its leaves (e.g. family + * `llm_stats` with display_name "HumanEval"). When that's the case the + * row should be titled with the humanized key instead, so the user can + * see they're looking at a *family* rather than a single benchmark. */ +function isFamilyDisplayNameMisleading(fam: HierarchyFamily, leafEntries: LeafEntry[]): boolean { + const nameSlug = slugify(fam.display_name) + if (!nameSlug) return false + if (nameSlug === slugify(fam.key)) return false + if (leafEntries.length < 2) return false + return leafEntries.some( + (l) => slugify(l.leafKey) === nameSlug || slugify(l.leafName) === nameSlug, + ) } interface RowData { @@ -114,23 +196,63 @@ interface RowData { name: string keySlug: string category: CategoryType - composites: number benchmarks: number - slices: number - metrics: number evalsCount: number leaves: LeafEntry[] /** True when the family has many leaves with no clean family-level summary — * we open it expanded so the user picks a leaf directly. */ isAggregator: boolean + description: string | null } -export function FamilyTable({ families, totalModels }: FamilyTableProps) { +export function FamilyTable({ + families, + totalModels, + evalItems, + benchmarkCards, + domainFilter, +}: FamilyTableProps) { const router = useRouter() const [expanded, setExpanded] = useState>({}) + const filterActive = Boolean(domainFilter && domainFilter.size > 0) + + function leafMatchesFilter(leaf: LeafEntry): boolean { + if (!filterActive || !domainFilter) return true + return leaf.domains.some((d) => domainFilter.has(d)) + } + + function familyMatchesFilter( + fam: HierarchyFamily, + navId: string | null, + leafEntries: LeafEntry[], + ): boolean { + if (!filterActive || !domainFilter) return true + if (leafEntries.some(leafMatchesFilter)) return true + const candidates: BenchmarkCard | undefined = (() => { + if (navId) { + const fromList = evalItems?.get(navId)?.benchmark_card + if (fromList) return fromList + } + return undefined + })() + const sources: Array = [] + if (candidates) sources.push(candidates.benchmark_details?.domains ?? []) + sources.push(benchmarkCards?.[fam.key]?.benchmark_details?.domains ?? []) + for (const id of fam.eval_summary_ids ?? []) { + sources.push(benchmarkCards?.[id]?.benchmark_details?.domains ?? []) + } + for (const list of sources) { + for (const d of list) { + if (domainFilter.has(d.trim().toLowerCase())) return true + } + } + return false + } + const rows = useMemo(() => { - return families.map((fam) => { + const out: RowData[] = [] + for (const fam of families) { const composites = fam.composites ?? [] const standalone = fam.standalone_benchmarks ?? [] const benchmarks = fam.benchmarks ?? [] @@ -141,12 +263,6 @@ export function FamilyTable({ families, totalModels }: FamilyTableProps) { ...benchmarks, ...composites.flatMap((c) => c.benchmarks ?? []), ] - const sliceCount = - fam.slices?.length ?? - allBenchmarks.reduce( - (sum, b) => sum + ((b as { slices?: unknown[] }).slices?.length ?? 0), - 0, - ) const metricCount = (fam.metrics?.length ?? 0) + allBenchmarks.reduce( @@ -156,51 +272,70 @@ export function FamilyTable({ families, totalModels }: FamilyTableProps) { const benchmarkCount = allBenchmarks.length > 0 ? allBenchmarks.length : leaves.length - const leafEntries = collectLeafEntries(fam) + const leafEntries = collectLeafEntries(fam, benchmarkCards) const navId = pickFamilyNavId(fam, leafEntries) - // An "aggregator" family is one whose display_name doesn't really - // describe a single benchmark (its leaves are heterogeneous). We - // detect this by counting leaves and, when there are many, prefer - // showing the leaf list rather than relying on the family-level id. - const isAggregator = leafEntries.length >= LEAVES_INLINE_MIN + // An "aggregator" family has heterogeneous leaves; we expand it + // inline so the user can pick a benchmark directly. When the family + // has no real composite summary (navId === null) it's necessarily + // an aggregator — clicking the row toggles expand instead of + // navigating. + const isAggregator = leafEntries.length >= LEAVES_INLINE_MIN || navId == null + + const displayName = isFamilyDisplayNameMisleading(fam, leafEntries) + ? humanizeFamilyKey(fam.key) + : fam.display_name + + // Description sourcing: prefer the eval item the row navigates to; + // when there's no navId or its eval item carries no overview, walk + // the leaves until we find one whose benchmark_card has one. That + // way an aggregator family ("HELM", "BFCL") whose family-level row + // doesn't directly link to a single eval still surfaces a one-line + // description from any of its component benchmarks. + const description = pickFamilyDescription(navId, leafEntries, evalItems) - return { + if (!familyMatchesFilter(fam, navId, leafEntries)) continue + const visibleLeafEntries = filterActive + ? leafEntries.filter(leafMatchesFilter) + : leafEntries + + out.push({ key: fam.key, navId, - name: fam.display_name, + name: displayName, keySlug: fam.key, category: (fam.category ?? "General") as CategoryType, - composites: composites.length, benchmarks: benchmarkCount, - slices: sliceCount, - metrics: metricCount, evalsCount: fam.evals_count ?? metricCount, - leaves: leafEntries, + leaves: visibleLeafEntries, isAggregator, - } - }) - }, [families]) + description, + }) + } + return out + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [families, evalItems, benchmarkCards, domainFilter]) return (
handleSort("coverage")} - > - Coverage{getSortIndicator("coverage")} - - {getVisibleMetricCount(row)} - /{visibleMetrics.length} -
- + - - - {rows.map((row) => { - const dotClass = CATEGORY_DOT[row.category] ?? "bg-stone-400" - const isExpanded = expanded[row.key] ?? false + // When a domain filter is active we auto-expand every aggregator + // so the matching leaves are immediately visible, but still let + // the user collapse a row manually via the chevron. + const isExpanded = filterActive + ? expanded[row.key] ?? true + : expanded[row.key] ?? false const expandable = row.isAggregator const visibleLeaves = isExpanded ? row.leaves.slice(0, LEAVES_INLINE_MAX) @@ -213,12 +348,16 @@ export function FamilyTable({ families, totalModels }: FamilyTableProps) { { - // Allow chevron click without navigating + // Allow chevron click without double-handling const target = event.target as HTMLElement if (target.closest("[data-row-toggle]")) return - if (row.navId) router.push(`/evals/${encodeURIComponent(row.navId)}`) + if (row.navId) { + router.push(`/evals/${encodeURIComponent(row.navId)}`) + } else if (expandable) { + setExpanded((current) => ({ ...current, [row.key]: !isExpanded })) + } }} - style={{ cursor: row.navId ? "pointer" : "default" }} + style={{ cursor: row.navId || expandable ? "pointer" : "default" }} > - - - -
FamilyFamily CategorySuites BenchmarksSlicesMetrics Reported results
@@ -240,14 +379,18 @@ export function FamilyTable({ families, totalModels }: FamilyTableProps) { ) : ( )} -
{row.name}
+ {row.description && ( +
+ {row.description} +
+ )}
{row.keySlug} {expandable && ( @@ -264,18 +407,9 @@ export function FamilyTable({ families, totalModels }: FamilyTableProps) { {row.category}
- {row.composites > 0 ? row.composites.toLocaleString() : "—"} - {row.benchmarks.toLocaleString()} - {row.slices > 0 ? row.slices.toLocaleString() : "—"} - - {row.metrics > 0 ? row.metrics.toLocaleString() : "—"} - {row.evalsCount.toLocaleString()} {totalModels > 0 && ( @@ -292,7 +426,7 @@ export function FamilyTable({ families, totalModels }: FamilyTableProps) { {isExpanded && visibleLeaves.length > 0 && (
+
void + onMaxChange: (next: number) => void + /** + * `default` — Variant A: bracketed range with a labelled rail and a boxed + * mono readout, suitable for use as the headline call-out at the top of a + * leaderboard. + * + * `inline` — Variant B: a single-line picker with no boxed readout, sized + * to drop into a hairline toolbar alongside Sort and Filter pickers. + * + * `promo` — Variant C: warm-background framed slider with a left accent + * rule. Use when the slider actively reframes a chart/matrix below it. + */ + variant?: ParamRangeVariant + /** Headline shown to the left of the slider (default & promo variants). */ + headline?: string + /** Sub-text shown under the headline (default & promo variants). */ + subline?: string + /** Callback to reset both handles to the open range. When provided, a + * Reset affordance is rendered next to the readout while the slider + * is constrained. */ + onReset?: () => void + /** When defined, renders a small "Show models without known size" pill + * next to the readout. The toggle is independent of the slider — when + * off, models with no detected size are filtered out regardless of + * where the handles are. */ + showUnknownSize?: boolean + onShowUnknownSizeChange?: (next: boolean) => void + className?: string +} + +/** + * Themed dual-handle parameter-range picker. Shape and colour come from the + * design system: hairline rail, square outline thumbs, mono uppercase tick + * labels above the rail, and a boxed mono readout for the explicit bounds. + * + * The two `` elements provide native dragging + arrow-key + * a11y. The visual rail/fill/ticks/thumbs are absolutely-positioned overlays; + * the inputs themselves are kept transparent except for their thumbs (see + * `.param-range-input` in globals.css). + */ +export function ParamRangePicker({ + minStep, + maxStep, + onMinChange, + onMaxChange, + variant = "default", + headline = "Parameter range", + subline = "Narrow the matrix to comparable model sizes.", + onReset, + showUnknownSize, + onShowUnknownSizeChange, + className, +}: ParamRangePickerProps) { + const minId = useId() + const maxId = useId() + + const isInline = variant === "inline" + const isPromo = variant === "promo" + + const minPercent = (minStep / PARAM_RANGE_MAX_INDEX) * 100 + const maxPercent = (maxStep / PARAM_RANGE_MAX_INDEX) * 100 + const isConstrained = minStep > 0 || maxStep < PARAM_RANGE_MAX_INDEX + + const track = ( +
+
+ {PARAM_RANGE_MARKERS.map((marker, idx) => { + const isFirst = idx === 0 + const isLast = idx === PARAM_RANGE_MARKERS.length - 1 + const active = marker.step === minStep || marker.step === maxStep + return ( +
+ + {marker.label} + +
+ ) + })} +
+ +
+
+ + {/* Hidden inter-bucket micro-ticks to give the rail a metered feel */} +
+ {PARAM_RANGE_VALUES.map((_, stepIndex) => ( + + ))} +
+ + {/* Native inputs provide a11y + drag; we hide them visually and rely + on the .param-range-input thumb styling for the visible handles. */} + { + const next = Number(event.target.value) + onMinChange(Math.min(next, maxStep)) + }} + className="param-range-input" + aria-label={`Minimum ${headline.toLowerCase()}`} + /> + { + const next = Number(event.target.value) + onMaxChange(Math.max(next, minStep)) + }} + className="param-range-input" + aria-label={`Maximum ${headline.toLowerCase()}`} + /> +
+ ) + + const resetBtn = onReset && isConstrained ? ( + + ) : null + + const unknownToggle = + onShowUnknownSizeChange != null ? ( + + ) : null + + const readout = ( +
+
+ {formatParamBoundLabel(minStep, "min")} + {isInline ? "–" : "→"} + {formatParamBoundLabel(maxStep, "max")} +
+ {unknownToggle} + {resetBtn} +
+ ) + + if (isInline) { + return ( +
+ + {headline} + + {track} + {readout} +
+ ) + } + + if (isPromo) { + return ( +
+
+ {headline} +

{subline}

+
+
+ {track} + {readout} +
+
+ ) + } + + // Default (Variant A) + return ( +
+
+ {headline} + {subline} +
+ {track} + {readout} +
+ ) +} diff --git a/components/signals/benchmark-signals-strip.tsx b/components/signals/benchmark-signals-strip.tsx new file mode 100644 index 0000000000000000000000000000000000000000..3dbf713d9c623e7f830aea5ae71a3d2fcadb31c9 --- /dev/null +++ b/components/signals/benchmark-signals-strip.tsx @@ -0,0 +1,626 @@ +"use client" + +import type { BenchmarkEvalSummary } from "@/lib/eval-processing" +import type { ModelResultForBenchmark } from "@/lib/eval-processing" + +type SignalId = "reproducibility" | "completeness" | "provenance" | "comparability" + +const SIGNAL_GLYPHS: Record = { + reproducibility: "R", + completeness: "C", + provenance: "P", + comparability: "X", +} + +const SIGNAL_NAMES: Record = { + reproducibility: "Reproducibility", + completeness: "Completeness", + provenance: "Provenance", + comparability: "Comparability", +} + +const SIGNAL_ASKS: Record = { + reproducibility: "Could someone re-run this benchmark with what's documented?", + completeness: "How much of the benchmark card is filled in?", + provenance: "Who reported these scores and how many parties have replicated?", + comparability: "Where multiple reports exist, do they agree?", +} + +/** + * Reproducibility — paper §4.2.1, signal spec §3. + * + * The spec lists `temperature, top_p, max_tokens, prompt_template` as the + * base required fields. In the live EEE corpus only `temperature` and + * `max_tokens` are reliably populated, so we restrict the check to those + * two for now (per maintainer guidance). Agentic benchmarks additionally + * require `eval_plan` and `eval_limits` — the spec's classification rule + * is followed verbatim. + */ +const BASE_REQUIRED_FIELDS = ["temperature", "max_tokens"] as const +const AGENTIC_REQUIRED_FIELDS = ["eval_plan", "eval_limits"] as const + +const FIELD_LABELS: Record = { + temperature: "temperature", + top_p: "top-p", + max_tokens: "max tokens", + prompt_template: "prompt template", + eval_plan: "eval plan", + eval_limits: "eval limits", +} + +/** Setup fields compared to detect variant divergence (spec §6.1.2). */ +const COMPARABILITY_COMPARE_FIELDS = [ + "temperature", + "top_p", + "top_k", + "max_tokens", + "prompt_template", + "reasoning", +] as const + +/** + * Benchmark-level rollup of the four interpretive signals (paper §4.2.1, + * spec v1.0 §§3-6). Mirrors `CorpusSignalsStrip` but operates over a + * single `BenchmarkEvalSummary`. + * + * Each tile reports one headline statistic that reads "higher is better, + * more documentation = better", so the four are visually comparable. + */ +export function BenchmarkSignalsStrip({ summary }: { summary: BenchmarkEvalSummary }) { + const repro = deriveReproducibility(summary) + const comp = deriveCompleteness(summary) + const prov = deriveProvenance(summary) + const cmp = deriveComparability(summary) + + return ( +
+ + + + +
+ ) +} + +interface DerivedSignal { + statValue: string + statUnit: string + headline: string + detail: string +} + +// ────────────────────────────────────────────────────────────────────────── +// Reproducibility (spec §3) +// ────────────────────────────────────────────────────────────────────────── + +function isAgenticBenchmark(summary: BenchmarkEvalSummary): boolean { + const tasks = summary.benchmark_card?.purpose_and_intended_users?.tasks + if (Array.isArray(tasks)) { + const set = new Set(tasks.map((t) => String(t).toLowerCase())) + if (set.has("agentic") || set.has("tool_use") || set.has("multi_step_agent")) return true + } + for (const r of summary.model_results ?? []) { + const args = getGenerationArgs(r) + if (args && args.agentic_eval_config != null) return true + } + return false +} + +function getGenerationArgs(result: ModelResultForBenchmark): Record | null { + const gc = (result.result as { generation_config?: { generation_args?: Record } } | undefined) + ?.generation_config + if (!gc) return null + const args = gc.generation_args + return args && typeof args === "object" ? args : null +} + +function deriveReproducibility(summary: BenchmarkEvalSummary): DerivedSignal { + const triples = summary.model_results ?? [] + const agentic = isAgenticBenchmark(summary) + const required: string[] = agentic + ? [...BASE_REQUIRED_FIELDS, ...AGENTIC_REQUIRED_FIELDS] + : [...BASE_REQUIRED_FIELDS] + + if (triples.length === 0) { + return { + statValue: "—", + statUnit: "", + headline: "Reproducibility doesn't apply — no reported scores.", + detail: "", + } + } + + const fieldMissing = new Map(required.map((f) => [f, 0])) + let triplesWithoutGap = 0 + + for (const triple of triples) { + const args = getGenerationArgs(triple) ?? {} + let allPresent = true + for (const f of required) { + if (!isPopulated(args[f])) { + fieldMissing.set(f, (fieldMissing.get(f) ?? 0) + 1) + allPresent = false + } + } + if (allPresent) triplesWithoutGap++ + } + + const total = triples.length + const score = triplesWithoutGap / total + + const topMissing = Array.from(fieldMissing.entries()) + .filter(([, n]) => n > 0) + .sort((a, b) => b[1] - a[1]) + .slice(0, 2) + .map(([f, n]) => `${FIELD_LABELS[f] ?? f} (${formatPct(n / total)})`) + .join(", ") + + const headline = + score === 1 + ? "Every reported score has a complete generation config." + : score === 0 + ? "No reported score has all required setup fields." + : `${triplesWithoutGap} of ${total} triples document the full setup.` + + const detail = topMissing + ? `Most often missing: ${topMissing}.` + : `Required: ${required.map((f) => FIELD_LABELS[f] ?? f).join(", ")}.` + + return { statValue: pctNum(score), statUnit: "%", headline, detail } +} + +// ────────────────────────────────────────────────────────────────────────── +// Completeness (spec §4) +// ────────────────────────────────────────────────────────────────────────── + +interface CompletenessField { + path: string + label: string + coverage: "full" | "partial" | "reserved" + /** For partial: list of sub-item names whose presence is checked. */ + subitems?: readonly string[] +} + +const COMPLETENESS_FIELD_SET: readonly CompletenessField[] = [ + { path: "benchmark_details.overview", label: "overview", coverage: "full" }, + { path: "benchmark_details.data_type", label: "data type", coverage: "full" }, + { + path: "benchmark_details", + label: "domains / languages / resources", + coverage: "partial", + subitems: ["domains", "languages", "resources"], + }, + { + path: "purpose_and_intended_users", + label: "purpose", + coverage: "partial", + subitems: ["goal", "audience", "tasks", "limitations"], + }, + { + path: "data", + label: "data", + coverage: "partial", + subitems: ["source", "size", "format", "annotation"], + }, + { + path: "methodology", + label: "methodology", + coverage: "partial", + subitems: ["methods", "metrics", "calculation", "interpretation", "baseline_results", "validation"], + }, + { + path: "ethical_and_legal_considerations", + label: "ethical & legal", + coverage: "partial", + subitems: ["privacy_and_anonymity", "data_licensing", "consent_procedures", "compliance_with_regulations"], + }, + // Reserved — counted in the denominator even when unset (spec §4.2). + { path: "evalcards.lifecycle_status", label: "lifecycle status", coverage: "reserved" }, +] as const + +function deriveCompleteness(summary: BenchmarkEvalSummary): DerivedSignal { + const card = summary.benchmark_card + + const fieldScores: { path: string; label: string; coverage: CompletenessField["coverage"]; score: number }[] = [] + + for (const field of COMPLETENESS_FIELD_SET) { + let score = 0 + if (field.coverage === "reserved") { + // The eval-summary payload doesn't currently carry an + // evalcards.lifecycle_status section, so this scores 0 for now. + // It still occupies a denominator slot per spec. + score = 0 + } else if (field.coverage === "full") { + const value = card ? readCardPath(card, field.path) : undefined + score = isPopulated(value) ? 1 : 0 + } else { + // partial + const parent = card ? (readCardPath(card, field.path) as Record | undefined) : undefined + const subs = field.subitems ?? [] + if (!parent || subs.length === 0) { + score = 0 + } else { + let populated = 0 + for (const key of subs) if (isPopulated(parent[key])) populated++ + score = populated / subs.length + } + } + fieldScores.push({ path: field.path, label: field.label, coverage: field.coverage, score }) + } + + const total = fieldScores.length + const sumScore = fieldScores.reduce((acc, f) => acc + f.score, 0) + const completeness = total > 0 ? sumScore / total : null + + const populatedCount = fieldScores.reduce((acc, f) => acc + (f.score === 1 ? 1 : 0), 0) + const partialCount = fieldScores.filter((f) => f.score > 0 && f.score < 1).length + const missingCount = fieldScores.filter((f) => f.score === 0).length + + const topMissing = fieldScores + .filter((f) => f.score === 0 && f.coverage !== "reserved") + .slice(0, 2) + .map((f) => f.label) + .join(", ") + + const headline = !card + ? "No benchmark card has been authored yet." + : completeness === 1 + ? "Every documented field is populated." + : completeness != null && completeness >= 0.6 + ? "Most documented fields are populated." + : "Several documented fields are still empty." + + const detail = !card + ? "Reading context will lean on whatever the leaderboard JSON provides." + : `${populatedCount} full · ${partialCount} partial · ${missingCount} missing of ${total}${ + topMissing ? ` · gaps: ${topMissing}` : "" + }` + + return { statValue: pctNum(completeness), statUnit: "%", headline, detail } +} + +function readCardPath(card: unknown, path: string): unknown { + if (!card || typeof card !== "object") return undefined + let cur: unknown = card + for (const segment of path.split(".")) { + if (cur == null || typeof cur !== "object") return undefined + cur = (cur as Record)[segment] + } + return cur +} + +// ────────────────────────────────────────────────────────────────────────── +// Provenance (spec §5) +// ────────────────────────────────────────────────────────────────────────── + +type ProvenanceSourceType = "first_party" | "third_party" | "collaborative" | "unspecified" + +function readSourceType(result: ModelResultForBenchmark): ProvenanceSourceType { + const sm = result.source_metadata as { evaluator_relationship?: string } | undefined + const rel = sm?.evaluator_relationship + if (rel === "first_party" || rel === "third_party" || rel === "collaborative") return rel + return "unspecified" +} + +function readSourceOrg(result: ModelResultForBenchmark): string | null { + const sm = result.source_metadata as { source_organization_name?: string } | undefined + const org = sm?.source_organization_name + if (typeof org === "string" && org.trim().length > 0) return org.trim() + return null +} + +function metricKeyForResult(result: ModelResultForBenchmark): string { + const r = result.result as { metric_summary_id?: string; metric_key?: string; evaluation_name?: string } | undefined + return r?.metric_summary_id ?? r?.metric_key ?? r?.evaluation_name ?? "" +} + +function modelKeyForResult(result: ModelResultForBenchmark): string { + return result.model_info?.id ?? result.model_info?.name ?? "" +} + +function deriveProvenance(summary: BenchmarkEvalSummary): DerivedSignal { + const triples = summary.model_results ?? [] + if (triples.length === 0) { + return { + statValue: "—", + statUnit: "", + headline: "No reported scores yet.", + detail: "", + } + } + + const counts: Record = { + first_party: 0, + third_party: 0, + collaborative: 0, + unspecified: 0, + } + const distinctOrgs = new Set() + const orgsByGroup = new Map>() + + for (const t of triples) { + counts[readSourceType(t)]++ + const org = readSourceOrg(t) + if (org) distinctOrgs.add(org) + const groupKey = `${modelKeyForResult(t)}::${metricKeyForResult(t)}` + if (org) { + const existing = orgsByGroup.get(groupKey) + if (existing) existing.add(org) + else orgsByGroup.set(groupKey, new Set([org])) + } + } + + const total = triples.length + const attributed = total - counts.unspecified + const score = attributed / total + + const multiSourceGroups = Array.from(orgsByGroup.values()).filter((s) => s.size > 1).length + const eligibleGroups = orgsByGroup.size + const multiRate = eligibleGroups > 0 ? multiSourceGroups / eligibleGroups : null + + const headline = + counts.unspecified === total + ? "No triple carries an attribution." + : multiSourceGroups > 0 + ? `${multiSourceGroups} of ${eligibleGroups} (model, metric) groups have reports from more than one party.` + : `Single-source benchmark: ${distinctOrgs.size} reporting org${distinctOrgs.size === 1 ? "" : "s"}.` + + const dist: string[] = [] + if (counts.first_party > 0) dist.push(`${formatPct(counts.first_party / total)} first-party`) + if (counts.third_party > 0) dist.push(`${formatPct(counts.third_party / total)} third-party`) + if (counts.collaborative > 0) dist.push(`${formatPct(counts.collaborative / total)} collaborative`) + if (counts.unspecified > 0) dist.push(`${formatPct(counts.unspecified / total)} unspecified`) + + const detailBits = [dist.join(" · ")] + if (multiRate != null) detailBits.push(`${formatPct(multiRate)} multi-source`) + + return { statValue: pctNum(score), statUnit: "%", headline, detail: detailBits.join(" · ") } +} + +// ────────────────────────────────────────────────────────────────────────── +// Comparability (spec §6) +// ────────────────────────────────────────────────────────────────────────── + +function computeThreshold(metricConfig: BenchmarkEvalSummary["metric_config"]): number { + if (!metricConfig) return 0.05 + const unit = (metricConfig as { unit?: string; metric_unit?: string }).unit + ?? (metricConfig as { metric_unit?: string }).metric_unit + const scoreType = (metricConfig as { score_type?: string }).score_type + if (unit === "proportion" || scoreType === "continuous_normalized") return 0.05 + if (unit === "percent") return 5.0 + const min = metricConfig.min_score + const max = metricConfig.max_score + if (typeof min === "number" && typeof max === "number" && max > min) return 0.05 * (max - min) + return 0.05 +} + +function median(values: number[]): number { + if (values.length === 0) return Number.NaN + const sorted = [...values].sort((a, b) => a - b) + const mid = Math.floor(sorted.length / 2) + return sorted.length % 2 === 1 ? sorted[mid] : (sorted[mid - 1] + sorted[mid]) / 2 +} + +function deriveComparability(summary: BenchmarkEvalSummary): DerivedSignal { + const triples = summary.model_results ?? [] + if (triples.length === 0) { + return { statValue: "—", statUnit: "", headline: "No reported scores yet.", detail: "" } + } + + const threshold = computeThreshold(summary.metric_config) + + // Group triples by (model_id, metric_path). + const groups = new Map< + string, + Array<{ score: number; args: Record; org: string | null }> + >() + for (const t of triples) { + const score = t.score_details?.score + if (typeof score !== "number" || !Number.isFinite(score)) continue + const key = `${modelKeyForResult(t)}::${metricKeyForResult(t)}` + const args = getGenerationArgs(t) ?? {} + const entry = { score, args, org: readSourceOrg(t) } + const list = groups.get(key) + if (list) list.push(entry) + else groups.set(key, [entry]) + } + + let variantEligible = 0 + let variantDivergent = 0 + let crossPartyEligible = 0 + let crossPartyDivergent = 0 + + for (const list of groups.values()) { + if (list.length < 2) continue + + // Variant divergence — same group, different setups (spec §6.1). + const setupValueSets = new Map>() + for (const entry of list) { + for (const f of COMPARABILITY_COMPARE_FIELDS) { + const valKey = JSON.stringify(entry.args[f] ?? null) + let set = setupValueSets.get(f) + if (!set) { + set = new Set() + setupValueSets.set(f, set) + } + set.add(valKey) + } + } + const setupsDiffer = Array.from(setupValueSets.values()).some((s) => s.size > 1) + if (setupsDiffer) { + variantEligible++ + const scores = list.map((e) => e.score) + const divergence = Math.max(...scores) - Math.min(...scores) + if (divergence > threshold) variantDivergent++ + } + + // Cross-party divergence — same group, different orgs (spec §6.2). + const byOrg = new Map() + for (const entry of list) { + if (!entry.org) continue + const arr = byOrg.get(entry.org) + if (arr) arr.push(entry.score) + else byOrg.set(entry.org, [entry.score]) + } + if (byOrg.size >= 2) { + crossPartyEligible++ + const orgScores = Array.from(byOrg.values()).map((s) => median(s)) + const divergence = Math.max(...orgScores) - Math.min(...orgScores) + if (divergence > threshold) crossPartyDivergent++ + } + } + + const totalEligible = variantEligible + crossPartyEligible + if (totalEligible === 0) { + return { + statValue: "—", + statUnit: "", + headline: "Not enough overlapping reports to compare.", + detail: `${groups.size} (model, metric) groups · 0 multi-report`, + } + } + + const totalDivergent = variantDivergent + crossPartyDivergent + const agreementRate = (totalEligible - totalDivergent) / totalEligible + + const detailBits: string[] = [] + if (variantEligible > 0) { + detailBits.push( + `variant ${variantEligible - variantDivergent}/${variantEligible} agree`, + ) + } + if (crossPartyEligible > 0) { + detailBits.push( + `cross-party ${crossPartyEligible - crossPartyDivergent}/${crossPartyEligible} agree`, + ) + } + detailBits.push(`threshold ±${formatNumber(threshold)}`) + + const headline = + totalDivergent === 0 + ? "Reports that are directly comparable agree within threshold." + : totalDivergent === totalEligible + ? "Every comparable report disagrees beyond threshold." + : `${totalEligible - totalDivergent} of ${totalEligible} comparable reports agree.` + + return { + statValue: pctNum(agreementRate), + statUnit: "%", + headline, + detail: detailBits.join(" · "), + } +} + +// ────────────────────────────────────────────────────────────────────────── +// Helpers +// ────────────────────────────────────────────────────────────────────────── + +function isPopulated(value: unknown): boolean { + if (value == null) return false + if (typeof value === "string") return value.trim().length > 0 + if (Array.isArray(value)) return value.length > 0 + if (typeof value === "object") return Object.keys(value as Record).length > 0 + return Boolean(value) +} + +function pctNum(value: number | null | undefined): string { + if (value == null || !Number.isFinite(value)) return "—" + if (value <= 0) return "0" + if (value < 0.01) return "<1" + return `${Math.round(value * 100)}` +} + +function formatPct(value: number | null | undefined): string { + if (value == null || !Number.isFinite(value)) return "—" + if (value === 0) return "0%" + if (value < 0.01) return "<1%" + return `${Math.round(value * 100)}%` +} + +function formatNumber(value: number): string { + if (!Number.isFinite(value)) return "—" + if (value >= 100) return value.toFixed(0) + if (value >= 1) return value.toFixed(2) + return value.toFixed(3).replace(/0+$/g, "").replace(/\.$/, "") +} + +/** + * Compact one-row layout per signal — meant to drop in alongside the Card + * Quality Notes box, not dominate the page like the corpus dashboard's + * full tile grid. Glyph + name + percent live on one line; one short + * sentence summarising the score lives below. The "Asks" prompt is moved + * to the title attribute so it stays discoverable on hover but doesn't + * eat vertical space. + */ +function SignalRow({ + id, + statValue, + statUnit, + headline, + detail, +}: { + id: SignalId +} & DerivedSignal) { + return ( +
+
+ + {SIGNAL_GLYPHS[id]} + + + {SIGNAL_NAMES[id]} + + + {statValue} + {statUnit && ( + + {statUnit} + + )} + +
+
+ {headline} + {detail && ( + + {" · "} + {detail} + + )} +
+
+ ) +} diff --git a/data/benchmarks.json b/data/benchmarks.json deleted file mode 100644 index 93a8d3f748ae8d207b97c1815173ec0c437db16e..0000000000000000000000000000000000000000 --- a/data/benchmarks.json +++ /dev/null @@ -1,90 +0,0 @@ -[ - { - "benchmark": "ace", - "model_count": 12 - }, - { - "benchmark": "apex-agents", - "model_count": 20 - }, - { - "benchmark": "apex-v1", - "model_count": 10 - }, - { - "benchmark": "appworld_test_normal", - "model_count": 3 - }, - { - "benchmark": "bfcl", - "model_count": 109 - }, - { - "benchmark": "browsecompplus", - "model_count": 3 - }, - { - "benchmark": "global-mmlu-lite", - "model_count": 27 - }, - { - "benchmark": "helm_capabilities", - "model_count": 61 - }, - { - "benchmark": "helm_classic", - "model_count": 67 - }, - { - "benchmark": "helm_instruct", - "model_count": 4 - }, - { - "benchmark": "helm_lite", - "model_count": 91 - }, - { - "benchmark": "helm_mmlu", - "model_count": 79 - }, - { - "benchmark": "hfopenllm_v2", - "model_count": 4493 - }, - { - "benchmark": "la_leaderboard", - "model_count": 5 - }, - { - "benchmark": "livecodebenchpro", - "model_count": 27 - }, - { - "benchmark": "reward-bench", - "model_count": 328 - }, - { - "benchmark": "swe-bench", - "model_count": 3 - }, - { - "benchmark": "tau-bench-2_airline", - "model_count": 3 - }, - { - "benchmark": "tau-bench-2_retail", - "model_count": 3 - }, - { - "benchmark": "tau-bench-2_telecom", - "model_count": 3 - }, - { - "benchmark": "terminal-bench-2.0", - "model_count": 37 - }, - { - "benchmark": "theory_of_mind", - "model_count": 1 - } -] \ No newline at end of file diff --git a/data/developers.json b/data/developers.json deleted file mode 100644 index 8e7f213a796ee1b66924498ef53eea23ba6f5171..0000000000000000000000000000000000000000 --- a/data/developers.json +++ /dev/null @@ -1,3150 +0,0 @@ -[ - { - "developer": "0-hero", - "model_count": 3 - }, - { - "developer": "01-ai", - "model_count": 20 - }, - { - "developer": "1-800-LLMs", - "model_count": 2 - }, - { - "developer": "1024m", - "model_count": 2 - }, - { - "developer": "152334H", - "model_count": 1 - }, - { - "developer": "1TuanPham", - "model_count": 2 - }, - { - "developer": "3rd-Degree-Burn", - "model_count": 4 - }, - { - "developer": "4season", - "model_count": 1 - }, - { - "developer": "aaditya", - "model_count": 1 - }, - { - "developer": "AALF", - "model_count": 4 - }, - { - "developer": "Aashraf995", - "model_count": 4 - }, - { - "developer": "abacusai", - "model_count": 10 - }, - { - "developer": "AbacusResearch", - "model_count": 1 - }, - { - "developer": "abhishek", - "model_count": 5 - }, - { - "developer": "abideen", - "model_count": 1 - }, - { - "developer": "adamo1139", - "model_count": 1 - }, - { - "developer": "adriszmar", - "model_count": 1 - }, - { - "developer": "AELLM", - "model_count": 2 - }, - { - "developer": "aevalone", - "model_count": 1 - }, - { - "developer": "agentlans", - "model_count": 9 - }, - { - "developer": "AGI-0", - "model_count": 3 - }, - { - "developer": "Ahdoot", - "model_count": 2 - }, - { - "developer": "Ahjeong", - "model_count": 2 - }, - { - "developer": "ahmeda335", - "model_count": 1 - }, - { - "developer": "AI-MO", - "model_count": 2 - }, - { - "developer": "AI-Sweden-Models", - "model_count": 2 - }, - { - "developer": "AI2", - "model_count": 7 - }, - { - "developer": "ai21", - "model_count": 12 - }, - { - "developer": "ai21labs", - "model_count": 1 - }, - { - "developer": "ai4bharat", - "model_count": 1 - }, - { - "developer": "AI4free", - "model_count": 2 - }, - { - "developer": "AicoresSecurity", - "model_count": 4 - }, - { - "developer": "AIDC-AI", - "model_count": 1 - }, - { - "developer": "aixonlab", - "model_count": 3 - }, - { - "developer": "akhadangi", - "model_count": 5 - }, - { - "developer": "akjindal53244", - "model_count": 1 - }, - { - "developer": "alcholjung", - "model_count": 1 - }, - { - "developer": "Alepach", - "model_count": 3 - }, - { - "developer": "aleph-alpha", - "model_count": 3 - }, - { - "developer": "AlephAlpha", - "model_count": 3 - }, - { - "developer": "Alibaba", - "model_count": 6 - }, - { - "developer": "Alibaba-NLP", - "model_count": 1 - }, - { - "developer": "aliyun", - "model_count": 1 - }, - { - "developer": "allenai", - "model_count": 162 - }, - { - "developer": "allknowingroger", - "model_count": 88 - }, - { - "developer": "allura-org", - "model_count": 9 - }, - { - "developer": "aloobun", - "model_count": 2 - }, - { - "developer": "alpindale", - "model_count": 2 - }, - { - "developer": "Alsebay", - "model_count": 1 - }, - { - "developer": "altomek", - "model_count": 1 - }, - { - "developer": "Amaorynho", - "model_count": 4 - }, - { - "developer": "amazon", - "model_count": 8 - }, - { - "developer": "amd", - "model_count": 1 - }, - { - "developer": "Amu", - "model_count": 2 - }, - { - "developer": "anakin87", - "model_count": 1 - }, - { - "developer": "anthracite-org", - "model_count": 12 - }, - { - "developer": "Anthropic", - "model_count": 34 - }, - { - "developer": "apple", - "model_count": 1 - }, - { - "developer": "applied-compute", - "model_count": 1 - }, - { - "developer": "appvoid", - "model_count": 2 - }, - { - "developer": "arcee-ai", - "model_count": 11 - }, - { - "developer": "argilla", - "model_count": 2 - }, - { - "developer": "argilla-warehouse", - "model_count": 1 - }, - { - "developer": "arisin", - "model_count": 1 - }, - { - "developer": "ark", - "model_count": 1 - }, - { - "developer": "ArliAI", - "model_count": 2 - }, - { - "developer": "arshiaafshani", - "model_count": 1 - }, - { - "developer": "Arthur-LAGACHERIE", - "model_count": 1 - }, - { - "developer": "Artples", - "model_count": 2 - }, - { - "developer": "Aryanne", - "model_count": 3 - }, - { - "developer": "asharsha30", - "model_count": 1 - }, - { - "developer": "ashercn97", - "model_count": 2 - }, - { - "developer": "assskelad", - "model_count": 1 - }, - { - "developer": "AtAndDev", - "model_count": 1 - }, - { - "developer": "Ateron", - "model_count": 3 - }, - { - "developer": "athirdpath", - "model_count": 1 - }, - { - "developer": "AtlaAI", - "model_count": 2 - }, - { - "developer": "AuraIndustries", - "model_count": 4 - }, - { - "developer": "Aurel9", - "model_count": 1 - }, - { - "developer": "automerger", - "model_count": 1 - }, - { - "developer": "avemio", - "model_count": 1 - }, - { - "developer": "awnr", - "model_count": 5 - }, - { - "developer": "aws-prototyping", - "model_count": 1 - }, - { - "developer": "axolotl-ai-co", - "model_count": 1 - }, - { - "developer": "Ayush-Singh", - "model_count": 1 - }, - { - "developer": "Azure99", - "model_count": 6 - }, - { - "developer": "Ba2han", - "model_count": 1 - }, - { - "developer": "BAAI", - "model_count": 14 - }, - { - "developer": "baconnier", - "model_count": 2 - }, - { - "developer": "baebee", - "model_count": 3 - }, - { - "developer": "bamec66557", - "model_count": 27 - }, - { - "developer": "Baptiste-HUVELLE-10", - "model_count": 1 - }, - { - "developer": "BEE-spoke-data", - "model_count": 9 - }, - { - "developer": "belztjti", - "model_count": 2 - }, - { - "developer": "BenevolenceMessiah", - "model_count": 2 - }, - { - "developer": "benhaotang", - "model_count": 1 - }, - { - "developer": "beomi", - "model_count": 1 - }, - { - "developer": "beowolx", - "model_count": 1 - }, - { - "developer": "berkeley-nest", - "model_count": 2 - }, - { - "developer": "bfuzzy1", - "model_count": 7 - }, - { - "developer": "bhuvneshsaini", - "model_count": 1 - }, - { - "developer": "bigcode", - "model_count": 3 - }, - { - "developer": "bigscience", - "model_count": 7 - }, - { - "developer": "bittensor", - "model_count": 1 - }, - { - "developer": "BlackBeenie", - "model_count": 9 - }, - { - "developer": "Bllossom", - "model_count": 1 - }, - { - "developer": "bluuwhale", - "model_count": 1 - }, - { - "developer": "BoltMonkey", - "model_count": 3 - }, - { - "developer": "bond005", - "model_count": 1 - }, - { - "developer": "bosonai", - "model_count": 1 - }, - { - "developer": "braindao", - "model_count": 17 - }, - { - "developer": "BrainWave-ML", - "model_count": 1 - }, - { - "developer": "BramVanroy", - "model_count": 4 - }, - { - "developer": "brgx53", - "model_count": 6 - }, - { - "developer": "BSC-LT", - "model_count": 2 - }, - { - "developer": "bunnycore", - "model_count": 85 - }, - { - "developer": "byroneverson", - "model_count": 3 - }, - { - "developer": "ByteDance", - "model_count": 1 - }, - { - "developer": "c10x", - "model_count": 2 - }, - { - "developer": "CarrotAI", - "model_count": 2 - }, - { - "developer": "carsenk", - "model_count": 2 - }, - { - "developer": "Casual-Autopsy", - "model_count": 1 - }, - { - "developer": "cat-searcher", - "model_count": 2 - }, - { - "developer": "CausalLM", - "model_count": 3 - }, - { - "developer": "cckm", - "model_count": 1 - }, - { - "developer": "cgato", - "model_count": 1 - }, - { - "developer": "Changgil", - "model_count": 2 - }, - { - "developer": "chargoddard", - "model_count": 1 - }, - { - "developer": "chujiezheng", - "model_count": 2 - }, - { - "developer": "CIR-AMS", - "model_count": 1 - }, - { - "developer": "cjvt", - "model_count": 1 - }, - { - "developer": "ClaudioItaly", - "model_count": 4 - }, - { - "developer": "cloudyu", - "model_count": 7 - }, - { - "developer": "cluebbers", - "model_count": 3 - }, - { - "developer": "cognitivecomputations", - "model_count": 17 - }, - { - "developer": "cohere", - "model_count": 18 - }, - { - "developer": "CohereForAI", - "model_count": 8 - }, - { - "developer": "collaiborateorg", - "model_count": 1 - }, - { - "developer": "Columbia-NLP", - "model_count": 6 - }, - { - "developer": "CombinHorizon", - "model_count": 6 - }, - { - "developer": "ContactDoctor", - "model_count": 2 - }, - { - "developer": "ContextualAI", - "model_count": 16 - }, - { - "developer": "CoolSpring", - "model_count": 3 - }, - { - "developer": "Corianas", - "model_count": 3 - }, - { - "developer": "CortexLM", - "model_count": 1 - }, - { - "developer": "cpayne1303", - "model_count": 4 - }, - { - "developer": "Cran-May", - "model_count": 7 - }, - { - "developer": "CreitinGameplays", - "model_count": 1 - }, - { - "developer": "crestf411", - "model_count": 1 - }, - { - "developer": "cstr", - "model_count": 1 - }, - { - "developer": "CultriX", - "model_count": 32 - }, - { - "developer": "cyberagent", - "model_count": 1 - }, - { - "developer": "CYFRAGOVPL", - "model_count": 6 - }, - { - "developer": "Daemontatox", - "model_count": 32 - }, - { - "developer": "Dampfinchen", - "model_count": 1 - }, - { - "developer": "Danielbrdz", - "model_count": 7 - }, - { - "developer": "Dans-DiscountModels", - "model_count": 9 - }, - { - "developer": "darkc0de", - "model_count": 3 - }, - { - "developer": "Darkknight535", - "model_count": 1 - }, - { - "developer": "databricks", - "model_count": 6 - }, - { - "developer": "Databricks-Mosaic-Research", - "model_count": 1 - }, - { - "developer": "DavidAU", - "model_count": 25 - }, - { - "developer": "davidkim205", - "model_count": 2 - }, - { - "developer": "Davidsv", - "model_count": 1 - }, - { - "developer": "DavieLion", - "model_count": 5 - }, - { - "developer": "DebateLabKIT", - "model_count": 1 - }, - { - "developer": "Deci", - "model_count": 2 - }, - { - "developer": "DeepAutoAI", - "model_count": 12 - }, - { - "developer": "DeepMount00", - "model_count": 13 - }, - { - "developer": "DeepSeek", - "model_count": 9 - }, - { - "developer": "deepseek-ai", - "model_count": 13 - }, - { - "developer": "Delta-Vector", - "model_count": 7 - }, - { - "developer": "DevQuasar", - "model_count": 1 - }, - { - "developer": "dfurman", - "model_count": 4 - }, - { - "developer": "dicta-il", - "model_count": 2 - }, - { - "developer": "distilbert", - "model_count": 1 - }, - { - "developer": "divyanshukunwar", - "model_count": 1 - }, - { - "developer": "djuna", - "model_count": 15 - }, - { - "developer": "djuna-test-lab", - "model_count": 2 - }, - { - "developer": "dnhkng", - "model_count": 10 - }, - { - "developer": "Dongwei", - "model_count": 1 - }, - { - "developer": "DoppelReflEx", - "model_count": 29 - }, - { - "developer": "DreadPoor", - "model_count": 119 - }, - { - "developer": "dreamgen", - "model_count": 1 - }, - { - "developer": "DRXD1000", - "model_count": 2 - }, - { - "developer": "DUAL-GPO", - "model_count": 1 - }, - { - "developer": "dustinwloring1988", - "model_count": 7 - }, - { - "developer": "duyhv1411", - "model_count": 2 - }, - { - "developer": "dwikitheduck", - "model_count": 6 - }, - { - "developer": "dzakwan", - "model_count": 1 - }, - { - "developer": "DZgas", - "model_count": 1 - }, - { - "developer": "ECE-ILAB-PRYMMAL", - "model_count": 1 - }, - { - "developer": "Edgerunners", - "model_count": 1 - }, - { - "developer": "ehristoforu", - "model_count": 36 - }, - { - "developer": "EleutherAI", - "model_count": 12 - }, - { - "developer": "elinas", - "model_count": 1 - }, - { - "developer": "ell44ot", - "model_count": 1 - }, - { - "developer": "Enno-Ai", - "model_count": 4 - }, - { - "developer": "EnnoAi", - "model_count": 2 - }, - { - "developer": "Epiculous", - "model_count": 4 - }, - { - "developer": "EpistemeAI", - "model_count": 47 - }, - { - "developer": "EpistemeAI2", - "model_count": 15 - }, - { - "developer": "Eric111", - "model_count": 2 - }, - { - "developer": "Etherll", - "model_count": 8 - }, - { - "developer": "euclaise", - "model_count": 1 - }, - { - "developer": "Eurdem", - "model_count": 1 - }, - { - "developer": "EVA-UNIT-01", - "model_count": 2 - }, - { - "developer": "eworojoshua", - "model_count": 1 - }, - { - "developer": "ewre324", - "model_count": 4 - }, - { - "developer": "experiment-llm", - "model_count": 1 - }, - { - "developer": "facebook", - "model_count": 4 - }, - { - "developer": "failspy", - "model_count": 6 - }, - { - "developer": "FallenMerick", - "model_count": 1 - }, - { - "developer": "fblgit", - "model_count": 11 - }, - { - "developer": "Felladrin", - "model_count": 2 - }, - { - "developer": "fhai50032", - "model_count": 2 - }, - { - "developer": "FINGU-AI", - "model_count": 7 - }, - { - "developer": "flammenai", - "model_count": 6 - }, - { - "developer": "FlofloB", - "model_count": 27 - }, - { - "developer": "fluently-lm", - "model_count": 3 - }, - { - "developer": "fluently-sets", - "model_count": 2 - }, - { - "developer": "formulae", - "model_count": 10 - }, - { - "developer": "frameai", - "model_count": 1 - }, - { - "developer": "freewheelin", - "model_count": 4 - }, - { - "developer": "FuJhen", - "model_count": 4 - }, - { - "developer": "fulim", - "model_count": 1 - }, - { - "developer": "FuseAI", - "model_count": 4 - }, - { - "developer": "gabrielmbmb", - "model_count": 1 - }, - { - "developer": "GalrionSoftworks", - "model_count": 2 - }, - { - "developer": "gaverfraxz", - "model_count": 2 - }, - { - "developer": "gbueno86", - "model_count": 2 - }, - { - "developer": "general-preference", - "model_count": 2 - }, - { - "developer": "GenVRadmin", - "model_count": 4 - }, - { - "developer": "ghost-x", - "model_count": 1 - }, - { - "developer": "glaiveai", - "model_count": 1 - }, - { - "developer": "gmonsoon", - "model_count": 5 - }, - { - "developer": "godlikehhd", - "model_count": 26 - }, - { - "developer": "Goekdeniz-Guelmez", - "model_count": 10 - }, - { - "developer": "Google", - "model_count": 78 - }, - { - "developer": "GoToCompany", - "model_count": 2 - }, - { - "developer": "goulue5", - "model_count": 1 - }, - { - "developer": "gradientai", - "model_count": 1 - }, - { - "developer": "GreenNode", - "model_count": 1 - }, - { - "developer": "grimjim", - "model_count": 25 - }, - { - "developer": "GritLM", - "model_count": 2 - }, - { - "developer": "Groq", - "model_count": 1 - }, - { - "developer": "Gryphe", - "model_count": 5 - }, - { - "developer": "GuilhermeNaturaUmana", - "model_count": 1 - }, - { - "developer": "Gunulhona", - "model_count": 2 - }, - { - "developer": "gupta-tanish", - "model_count": 1 - }, - { - "developer": "gz987", - "model_count": 4 - }, - { - "developer": "h2oai", - "model_count": 5 - }, - { - "developer": "haoranxu", - "model_count": 3 - }, - { - "developer": "HarbingerX", - "model_count": 2 - }, - { - "developer": "Hastagaras", - "model_count": 3 - }, - { - "developer": "hatemmahmoud", - "model_count": 1 - }, - { - "developer": "HelpingAI", - "model_count": 4 - }, - { - "developer": "hendrydong", - "model_count": 1 - }, - { - "developer": "HeraiHench", - "model_count": 4 - }, - { - "developer": "HFXM", - "model_count": 1 - }, - { - "developer": "HiroseKoichi", - "model_count": 1 - }, - { - "developer": "HoangHa", - "model_count": 1 - }, - { - "developer": "hon9kon9ize", - "model_count": 2 - }, - { - "developer": "hongbai12", - "model_count": 1 - }, - { - "developer": "hotmailuser", - "model_count": 34 - }, - { - "developer": "HPAI-BSC", - "model_count": 3 - }, - { - "developer": "huawei-noah-ustc", - "model_count": 1 - }, - { - "developer": "HuggingFaceH4", - "model_count": 5 - }, - { - "developer": "HuggingFaceTB", - "model_count": 12 - }, - { - "developer": "huggyllama", - "model_count": 3 - }, - { - "developer": "huihui-ai", - "model_count": 8 - }, - { - "developer": "HumanLLMs", - "model_count": 3 - }, - { - "developer": "huu-ontocord", - "model_count": 1 - }, - { - "developer": "ibivibiv", - "model_count": 2 - }, - { - "developer": "ibm", - "model_count": 8 - }, - { - "developer": "ibm-granite", - "model_count": 20 - }, - { - "developer": "icefog72", - "model_count": 62 - }, - { - "developer": "IDEA-CCNL", - "model_count": 2 - }, - { - "developer": "ifable", - "model_count": 1 - }, - { - "developer": "iFaz", - "model_count": 8 - }, - { - "developer": "ilsp", - "model_count": 1 - }, - { - "developer": "IlyaGusev", - "model_count": 2 - }, - { - "developer": "Infinirc", - "model_count": 1 - }, - { - "developer": "inflatebot", - "model_count": 1 - }, - { - "developer": "infly", - "model_count": 1 - }, - { - "developer": "informatiker", - "model_count": 1 - }, - { - "developer": "INSAIT-Institute", - "model_count": 1 - }, - { - "developer": "insightfactory", - "model_count": 1 - }, - { - "developer": "instruction-pretrain", - "model_count": 1 - }, - { - "developer": "Intel", - "model_count": 4 - }, - { - "developer": "internlm", - "model_count": 9 - }, - { - "developer": "intervitens", - "model_count": 1 - }, - { - "developer": "IntervitensInc", - "model_count": 1 - }, - { - "developer": "inumulaisk", - "model_count": 1 - }, - { - "developer": "invalid-coder", - "model_count": 1 - }, - { - "developer": "Invalid-Null", - "model_count": 2 - }, - { - "developer": "invisietch", - "model_count": 4 - }, - { - "developer": "irahulpandey", - "model_count": 1 - }, - { - "developer": "iRyanBell", - "model_count": 2 - }, - { - "developer": "Isaak-Carter", - "model_count": 3 - }, - { - "developer": "J-LAB", - "model_count": 1 - }, - { - "developer": "JackFram", - "model_count": 2 - }, - { - "developer": "Jacoby746", - "model_count": 7 - }, - { - "developer": "jaredjoss", - "model_count": 1 - }, - { - "developer": "jaspionjader", - "model_count": 196 - }, - { - "developer": "jayasuryajsk", - "model_count": 1 - }, - { - "developer": "JayHyeon", - "model_count": 174 - }, - { - "developer": "jeanmichela", - "model_count": 1 - }, - { - "developer": "jebcarter", - "model_count": 1 - }, - { - "developer": "jebish7", - "model_count": 9 - }, - { - "developer": "jeffmeloy", - "model_count": 18 - }, - { - "developer": "jeonsworld", - "model_count": 1 - }, - { - "developer": "jiangxinyang-shanda", - "model_count": 1 - }, - { - "developer": "jieliu", - "model_count": 1 - }, - { - "developer": "Jimmy19991222", - "model_count": 8 - }, - { - "developer": "jiviai", - "model_count": 1 - }, - { - "developer": "jlzhou", - "model_count": 1 - }, - { - "developer": "johnsutor", - "model_count": 31 - }, - { - "developer": "jondurbin", - "model_count": 1 - }, - { - "developer": "Joseph717171", - "model_count": 2 - }, - { - "developer": "Josephgflowers", - "model_count": 7 - }, - { - "developer": "jpacifico", - "model_count": 18 - }, - { - "developer": "jsfs11", - "model_count": 3 - }, - { - "developer": "JungZoona", - "model_count": 2 - }, - { - "developer": "Junhoee", - "model_count": 1 - }, - { - "developer": "kaist-ai", - "model_count": 4 - }, - { - "developer": "katanemo", - "model_count": 3 - }, - { - "developer": "kavonalds", - "model_count": 3 - }, - { - "developer": "kayfour", - "model_count": 1 - }, - { - "developer": "keeeeenw", - "model_count": 1 - }, - { - "developer": "kekmodel", - "model_count": 1 - }, - { - "developer": "kevin009", - "model_count": 1 - }, - { - "developer": "Khetterman", - "model_count": 2 - }, - { - "developer": "khoantap", - "model_count": 9 - }, - { - "developer": "khulaifi95", - "model_count": 1 - }, - { - "developer": "Kimargin", - "model_count": 1 - }, - { - "developer": "Kimi", - "model_count": 1 - }, - { - "developer": "KingNish", - "model_count": 7 - }, - { - "developer": "kms7530", - "model_count": 4 - }, - { - "developer": "kno10", - "model_count": 2 - }, - { - "developer": "Kquant03", - "model_count": 2 - }, - { - "developer": "Krystalan", - "model_count": 2 - }, - { - "developer": "KSU-HW-SEC", - "model_count": 4 - }, - { - "developer": "Kuaishou", - "model_count": 1 - }, - { - "developer": "Kukedlc", - "model_count": 7 - }, - { - "developer": "Kumar955", - "model_count": 1 - }, - { - "developer": "kyutai", - "model_count": 1 - }, - { - "developer": "kz919", - "model_count": 1 - }, - { - "developer": "L-RAGE", - "model_count": 1 - }, - { - "developer": "ladydaina", - "model_count": 1 - }, - { - "developer": "laislemke", - "model_count": 1 - }, - { - "developer": "lalainy", - "model_count": 7 - }, - { - "developer": "Lambent", - "model_count": 1 - }, - { - "developer": "Langboat", - "model_count": 1 - }, - { - "developer": "langgptai", - "model_count": 2 - }, - { - "developer": "lars1234", - "model_count": 1 - }, - { - "developer": "Lawnakk", - "model_count": 10 - }, - { - "developer": "leafspark", - "model_count": 1 - }, - { - "developer": "LEESM", - "model_count": 4 - }, - { - "developer": "lemon07r", - "model_count": 17 - }, - { - "developer": "LenguajeNaturalAI", - "model_count": 2 - }, - { - "developer": "LeroyDyer", - "model_count": 58 - }, - { - "developer": "lesubra", - "model_count": 8 - }, - { - "developer": "LGAI-EXAONE", - "model_count": 4 - }, - { - "developer": "lightblue", - "model_count": 5 - }, - { - "developer": "LightningRodLabs", - "model_count": 3 - }, - { - "developer": "Lil-R", - "model_count": 8 - }, - { - "developer": "LilRg", - "model_count": 10 - }, - { - "developer": "LimYeri", - "model_count": 5 - }, - { - "developer": "lkoenig", - "model_count": 11 - }, - { - "developer": "llm-blender", - "model_count": 1 - }, - { - "developer": "LLM360", - "model_count": 2 - }, - { - "developer": "LLM4Binary", - "model_count": 1 - }, - { - "developer": "llmat", - "model_count": 1 - }, - { - "developer": "llnYou", - "model_count": 5 - }, - { - "developer": "lmsys", - "model_count": 5 - }, - { - "developer": "Locutusque", - "model_count": 6 - }, - { - "developer": "lodrick-the-lafted", - "model_count": 1 - }, - { - "developer": "lordjia", - "model_count": 2 - }, - { - "developer": "lt-asset", - "model_count": 1 - }, - { - "developer": "lunahr", - "model_count": 2 - }, - { - "developer": "Luni", - "model_count": 2 - }, - { - "developer": "Lunzima", - "model_count": 18 - }, - { - "developer": "LxzGordon", - "model_count": 2 - }, - { - "developer": "Lyte", - "model_count": 3 - }, - { - "developer": "M4-ai", - "model_count": 1 - }, - { - "developer": "m42-health", - "model_count": 1 - }, - { - "developer": "macadeliccc", - "model_count": 3 - }, - { - "developer": "madeagents", - "model_count": 4 - }, - { - "developer": "magnifi", - "model_count": 1 - }, - { - "developer": "Magpie-Align", - "model_count": 8 - }, - { - "developer": "MagusCorp", - "model_count": 1 - }, - { - "developer": "maldv", - "model_count": 7 - }, - { - "developer": "ManoloPueblo", - "model_count": 3 - }, - { - "developer": "marcuscedricridia", - "model_count": 40 - }, - { - "developer": "marin-community", - "model_count": 1 - }, - { - "developer": "MarinaraSpaghetti", - "model_count": 2 - }, - { - "developer": "Marsouuu", - "model_count": 8 - }, - { - "developer": "matouLeLoup", - "model_count": 5 - }, - { - "developer": "mattshumer", - "model_count": 3 - }, - { - "developer": "maywell", - "model_count": 1 - }, - { - "developer": "MaziyarPanahi", - "model_count": 44 - }, - { - "developer": "meditsolutions", - "model_count": 12 - }, - { - "developer": "meetkai", - "model_count": 1 - }, - { - "developer": "meraGPT", - "model_count": 1 - }, - { - "developer": "mergekit-community", - "model_count": 11 - }, - { - "developer": "MEscriva", - "model_count": 1 - }, - { - "developer": "Meta", - "model_count": 26 - }, - { - "developer": "meta-llama", - "model_count": 23 - }, - { - "developer": "meta-metrics", - "model_count": 1 - }, - { - "developer": "mhl1", - "model_count": 1 - }, - { - "developer": "microsoft", - "model_count": 19 - }, - { - "developer": "mightbe", - "model_count": 1 - }, - { - "developer": "migtissera", - "model_count": 8 - }, - { - "developer": "Minami-su", - "model_count": 5 - }, - { - "developer": "mindw96", - "model_count": 1 - }, - { - "developer": "minghaowu", - "model_count": 1 - }, - { - "developer": "MiniMax", - "model_count": 4 - }, - { - "developer": "ministral", - "model_count": 1 - }, - { - "developer": "mistral-community", - "model_count": 3 - }, - { - "developer": "mistralai", - "model_count": 36 - }, - { - "developer": "mixtao", - "model_count": 1 - }, - { - "developer": "mkurman", - "model_count": 3 - }, - { - "developer": "mkxu", - "model_count": 2 - }, - { - "developer": "mlabonne", - "model_count": 14 - }, - { - "developer": "MLP-KTLim", - "model_count": 1 - }, - { - "developer": "mlx-community", - "model_count": 2 - }, - { - "developer": "mmnga", - "model_count": 1 - }, - { - "developer": "mobiuslabsgmbh", - "model_count": 2 - }, - { - "developer": "ModelCloud", - "model_count": 1 - }, - { - "developer": "ModelSpace", - "model_count": 1 - }, - { - "developer": "moeru-ai", - "model_count": 3 - }, - { - "developer": "monsterapi", - "model_count": 2 - }, - { - "developer": "MoonRide", - "model_count": 1 - }, - { - "developer": "moonshot", - "model_count": 2 - }, - { - "developer": "Moonshot AI", - "model_count": 2 - }, - { - "developer": "moonshotai", - "model_count": 2 - }, - { - "developer": "mosaicml", - "model_count": 3 - }, - { - "developer": "mosama", - "model_count": 1 - }, - { - "developer": "Mostafa8Mehrabi", - "model_count": 1 - }, - { - "developer": "mrdayl", - "model_count": 5 - }, - { - "developer": "mrm8488", - "model_count": 2 - }, - { - "developer": "MrRobotoAI", - "model_count": 2 - }, - { - "developer": "MTSAIR", - "model_count": 2 - }, - { - "developer": "mukaj", - "model_count": 1 - }, - { - "developer": "Multiple", - "model_count": 1 - }, - { - "developer": "MultivexAI", - "model_count": 5 - }, - { - "developer": "Mxode", - "model_count": 5 - }, - { - "developer": "my_model", - "model_count": 1 - }, - { - "developer": "nanbeige", - "model_count": 2 - }, - { - "developer": "NAPS-ai", - "model_count": 7 - }, - { - "developer": "natong19", - "model_count": 2 - }, - { - "developer": "Naveenpoliasetty", - "model_count": 1 - }, - { - "developer": "nazimali", - "model_count": 2 - }, - { - "developer": "NbAiLab", - "model_count": 2 - }, - { - "developer": "nbeerbower", - "model_count": 51 - }, - { - "developer": "nbrahme", - "model_count": 1 - }, - { - "developer": "NCSOFT", - "model_count": 3 - }, - { - "developer": "necva", - "model_count": 2 - }, - { - "developer": "Nekochu", - "model_count": 4 - }, - { - "developer": "neopolita", - "model_count": 11 - }, - { - "developer": "netcat420", - "model_count": 48 - }, - { - "developer": "netease-youdao", - "model_count": 1 - }, - { - "developer": "NeverSleep", - "model_count": 2 - }, - { - "developer": "newsbang", - "model_count": 7 - }, - { - "developer": "Nexesenex", - "model_count": 44 - }, - { - "developer": "Nexusflow", - "model_count": 2 - }, - { - "developer": "nguyentd", - "model_count": 1 - }, - { - "developer": "ngxson", - "model_count": 2 - }, - { - "developer": "nhyha", - "model_count": 5 - }, - { - "developer": "nicolinho", - "model_count": 4 - }, - { - "developer": "nidum", - "model_count": 1 - }, - { - "developer": "NikolaSigmoid", - "model_count": 7 - }, - { - "developer": "nisten", - "model_count": 2 - }, - { - "developer": "Nitral-AI", - "model_count": 8 - }, - { - "developer": "NJS26", - "model_count": 1 - }, - { - "developer": "NLPark", - "model_count": 3 - }, - { - "developer": "nlpguy", - "model_count": 9 - }, - { - "developer": "Nohobby", - "model_count": 2 - }, - { - "developer": "noname0202", - "model_count": 8 - }, - { - "developer": "Norquinal", - "model_count": 8 - }, - { - "developer": "NotASI", - "model_count": 4 - }, - { - "developer": "notbdq", - "model_count": 1 - }, - { - "developer": "nothingiisreal", - "model_count": 3 - }, - { - "developer": "NousResearch", - "model_count": 19 - }, - { - "developer": "Novaciano", - "model_count": 11 - }, - { - "developer": "NTQAI", - "model_count": 2 - }, - { - "developer": "NucleusAI", - "model_count": 1 - }, - { - "developer": "nvidia", - "model_count": 22 - }, - { - "developer": "nxmwxm", - "model_count": 1 - }, - { - "developer": "NYTK", - "model_count": 2 - }, - { - "developer": "NyxKrage", - "model_count": 1 - }, - { - "developer": "occiglot", - "model_count": 1 - }, - { - "developer": "odyssey-labs", - "model_count": 1 - }, - { - "developer": "OEvortex", - "model_count": 5 - }, - { - "developer": "olabs-ai", - "model_count": 1 - }, - { - "developer": "OliveiraJLT", - "model_count": 1 - }, - { - "developer": "Omkar1102", - "model_count": 1 - }, - { - "developer": "OmnicromsBrain", - "model_count": 1 - }, - { - "developer": "OnlyCheeini", - "model_count": 1 - }, - { - "developer": "ontocord", - "model_count": 32 - }, - { - "developer": "oobabooga", - "model_count": 1 - }, - { - "developer": "oopere", - "model_count": 9 - }, - { - "developer": "open-atlas", - "model_count": 2 - }, - { - "developer": "open-neo", - "model_count": 2 - }, - { - "developer": "Open-Orca", - "model_count": 1 - }, - { - "developer": "open-thoughts", - "model_count": 1 - }, - { - "developer": "OpenAI", - "model_count": 75 - }, - { - "developer": "openai-community", - "model_count": 4 - }, - { - "developer": "OpenAssistant", - "model_count": 4 - }, - { - "developer": "openbmb", - "model_count": 7 - }, - { - "developer": "OpenBuddy", - "model_count": 22 - }, - { - "developer": "openchat", - "model_count": 6 - }, - { - "developer": "opencompass", - "model_count": 4 - }, - { - "developer": "OpenGenerativeAI", - "model_count": 2 - }, - { - "developer": "OpenLeecher", - "model_count": 1 - }, - { - "developer": "OpenLLM-France", - "model_count": 4 - }, - { - "developer": "OpenScholar", - "model_count": 1 - }, - { - "developer": "orai-nlp", - "model_count": 1 - }, - { - "developer": "Orenguteng", - "model_count": 2 - }, - { - "developer": "Orion-zhen", - "model_count": 2 - }, - { - "developer": "oxyapi", - "model_count": 1 - }, - { - "developer": "ozone-ai", - "model_count": 1 - }, - { - "developer": "ozone-research", - "model_count": 1 - }, - { - "developer": "P0x0", - "model_count": 1 - }, - { - "developer": "paloalma", - "model_count": 5 - }, - { - "developer": "pankajmathur", - "model_count": 29 - }, - { - "developer": "Parissa3", - "model_count": 1 - }, - { - "developer": "paulml", - "model_count": 1 - }, - { - "developer": "phronetic-ai", - "model_count": 1 - }, - { - "developer": "Pinkstack", - "model_count": 4 - }, - { - "developer": "pints-ai", - "model_count": 2 - }, - { - "developer": "piotr25691", - "model_count": 3 - }, - { - "developer": "PJMixers", - "model_count": 1 - }, - { - "developer": "PJMixers-Dev", - "model_count": 9 - }, - { - "developer": "PKU-Alignment", - "model_count": 4 - }, - { - "developer": "PocketDoc", - "model_count": 5 - }, - { - "developer": "PoLL", - "model_count": 1 - }, - { - "developer": "postbot", - "model_count": 1 - }, - { - "developer": "PowerInfer", - "model_count": 1 - }, - { - "developer": "PranavHarshan", - "model_count": 2 - }, - { - "developer": "Pretergeek", - "model_count": 9 - }, - { - "developer": "PrimeIntellect", - "model_count": 2 - }, - { - "developer": "prince-canuma", - "model_count": 1 - }, - { - "developer": "princeton-nlp", - "model_count": 51 - }, - { - "developer": "prithivMLmods", - "model_count": 110 - }, - { - "developer": "prometheus-eval", - "model_count": 2 - }, - { - "developer": "pszemraj", - "model_count": 2 - }, - { - "developer": "PuxAI", - "model_count": 1 - }, - { - "developer": "PygmalionAI", - "model_count": 1 - }, - { - "developer": "Q-bert", - "model_count": 1 - }, - { - "developer": "qingy2019", - "model_count": 7 - }, - { - "developer": "qingy2024", - "model_count": 17 - }, - { - "developer": "qq8933", - "model_count": 1 - }, - { - "developer": "Quazim0t0", - "model_count": 69 - }, - { - "developer": "Qwen", - "model_count": 80 - }, - { - "developer": "R-I-S-E", - "model_count": 2 - }, - { - "developer": "Rakuten", - "model_count": 3 - }, - { - "developer": "raphgg", - "model_count": 1 - }, - { - "developer": "rasyosef", - "model_count": 4 - }, - { - "developer": "Ray2333", - "model_count": 10 - }, - { - "developer": "RDson", - "model_count": 1 - }, - { - "developer": "realtreetune", - "model_count": 1 - }, - { - "developer": "recoilme", - "model_count": 6 - }, - { - "developer": "redrix", - "model_count": 2 - }, - { - "developer": "refuelai", - "model_count": 1 - }, - { - "developer": "Replete-AI", - "model_count": 9 - }, - { - "developer": "RESMPDEV", - "model_count": 2 - }, - { - "developer": "RezVortex", - "model_count": 2 - }, - { - "developer": "rhplus0831", - "model_count": 1 - }, - { - "developer": "rhymes-ai", - "model_count": 1 - }, - { - "developer": "rhysjones", - "model_count": 1 - }, - { - "developer": "riaz", - "model_count": 1 - }, - { - "developer": "RLHFlow", - "model_count": 4 - }, - { - "developer": "rmdhirr", - "model_count": 1 - }, - { - "developer": "Ro-xe", - "model_count": 4 - }, - { - "developer": "Rombo-Org", - "model_count": 1 - }, - { - "developer": "rombodawg", - "model_count": 14 - }, - { - "developer": "rootxhacker", - "model_count": 3 - }, - { - "developer": "rsh345", - "model_count": 1 - }, - { - "developer": "rubenroy", - "model_count": 3 - }, - { - "developer": "RubielLabarta", - "model_count": 1 - }, - { - "developer": "ruizhe1217", - "model_count": 1 - }, - { - "developer": "rwitz", - "model_count": 1 - }, - { - "developer": "RWKV", - "model_count": 1 - }, - { - "developer": "sabersaleh", - "model_count": 7 - }, - { - "developer": "sabersalehk", - "model_count": 4 - }, - { - "developer": "SaisExperiments", - "model_count": 6 - }, - { - "developer": "saishf", - "model_count": 2 - }, - { - "developer": "saishshinde15", - "model_count": 3 - }, - { - "developer": "sakaltcommunity", - "model_count": 2 - }, - { - "developer": "Sakalti", - "model_count": 66 - }, - { - "developer": "sakhan10", - "model_count": 1 - }, - { - "developer": "salesforce", - "model_count": 9 - }, - { - "developer": "saltlux", - "model_count": 2 - }, - { - "developer": "sam-paech", - "model_count": 3 - }, - { - "developer": "SanjiWatsuki", - "model_count": 2 - }, - { - "developer": "Sao10K", - "model_count": 8 - }, - { - "developer": "sarvamai", - "model_count": 1 - }, - { - "developer": "Saxo", - "model_count": 11 - }, - { - "developer": "schnapss", - "model_count": 1 - }, - { - "developer": "Schrieffer", - "model_count": 1 - }, - { - "developer": "sci-m-wang", - "model_count": 3 - }, - { - "developer": "SeaLLMs", - "model_count": 3 - }, - { - "developer": "securin", - "model_count": 1 - }, - { - "developer": "senseable", - "model_count": 1 - }, - { - "developer": "SenseLLM", - "model_count": 2 - }, - { - "developer": "SentientAGI", - "model_count": 2 - }, - { - "developer": "SeppeV", - "model_count": 1 - }, - { - "developer": "sequelbox", - "model_count": 6 - }, - { - "developer": "sethuiyer", - "model_count": 6 - }, - { - "developer": "SF-Foundation", - "model_count": 2 - }, - { - "developer": "sfairXC", - "model_count": 1 - }, - { - "developer": "shadowml", - "model_count": 2 - }, - { - "developer": "Sharathhebbar24", - "model_count": 2 - }, - { - "developer": "shastraai", - "model_count": 1 - }, - { - "developer": "ShikaiChen", - "model_count": 1 - }, - { - "developer": "shivam9980", - "model_count": 2 - }, - { - "developer": "shivank21", - "model_count": 1 - }, - { - "developer": "Shreyash2010", - "model_count": 1 - }, - { - "developer": "shuttleai", - "model_count": 1 - }, - { - "developer": "shyamieee", - "model_count": 1 - }, - { - "developer": "Sicarius-Prototyping", - "model_count": 3 - }, - { - "developer": "SicariusSicariiStuff", - "model_count": 19 - }, - { - "developer": "silma-ai", - "model_count": 2 - }, - { - "developer": "siqi00", - "model_count": 2 - }, - { - "developer": "skumar9", - "model_count": 1 - }, - { - "developer": "skymizer", - "model_count": 1 - }, - { - "developer": "SkyOrbis", - "model_count": 12 - }, - { - "developer": "Skywork", - "model_count": 15 - }, - { - "developer": "snowflake", - "model_count": 1 - }, - { - "developer": "Solshine", - "model_count": 2 - }, - { - "developer": "someon98", - "model_count": 1 - }, - { - "developer": "sometimesanotion", - "model_count": 58 - }, - { - "developer": "sonthenguyen", - "model_count": 6 - }, - { - "developer": "sophosympatheia", - "model_count": 1 - }, - { - "developer": "Sorawiz", - "model_count": 2 - }, - { - "developer": "Sourjayon", - "model_count": 2 - }, - { - "developer": "SpaceYL", - "model_count": 1 - }, - { - "developer": "speakleash", - "model_count": 5 - }, - { - "developer": "speakleash-ack-cyfronet-agh", - "model_count": 1 - }, - { - "developer": "Spestly", - "model_count": 3 - }, - { - "developer": "spmurrayzzz", - "model_count": 1 - }, - { - "developer": "spow12", - "model_count": 4 - }, - { - "developer": "ssmits", - "model_count": 1 - }, - { - "developer": "stabilityai", - "model_count": 9 - }, - { - "developer": "stanford", - "model_count": 1 - }, - { - "developer": "stanfordnlp", - "model_count": 2 - }, - { - "developer": "Stark2008", - "model_count": 3 - }, - { - "developer": "Steelskull", - "model_count": 2 - }, - { - "developer": "StelleX", - "model_count": 2 - }, - { - "developer": "sthenno", - "model_count": 9 - }, - { - "developer": "sthenno-com", - "model_count": 4 - }, - { - "developer": "streamerbtw1002", - "model_count": 1 - }, - { - "developer": "stupidity-ai", - "model_count": 1 - }, - { - "developer": "suayptalha", - "model_count": 12 - }, - { - "developer": "SultanR", - "model_count": 4 - }, - { - "developer": "sumink", - "model_count": 22 - }, - { - "developer": "sunbaby", - "model_count": 1 - }, - { - "developer": "Supichi", - "model_count": 11 - }, - { - "developer": "Svak", - "model_count": 2 - }, - { - "developer": "swap-uniba", - "model_count": 1 - }, - { - "developer": "Syed-Hasan-8503", - "model_count": 1 - }, - { - "developer": "synergetic", - "model_count": 1 - }, - { - "developer": "T145", - "model_count": 51 - }, - { - "developer": "talha2001", - "model_count": 1 - }, - { - "developer": "tangledgroup", - "model_count": 2 - }, - { - "developer": "tanliboy", - "model_count": 3 - }, - { - "developer": "tannedbum", - "model_count": 4 - }, - { - "developer": "Tarek07", - "model_count": 2 - }, - { - "developer": "TeeZee", - "model_count": 1 - }, - { - "developer": "teknium", - "model_count": 5 - }, - { - "developer": "Telugu-LLM-Labs", - "model_count": 2 - }, - { - "developer": "TencentARC", - "model_count": 4 - }, - { - "developer": "tensopolis", - "model_count": 15 - }, - { - "developer": "tensoropera", - "model_count": 1 - }, - { - "developer": "tenyx", - "model_count": 1 - }, - { - "developer": "TheDrummer", - "model_count": 9 - }, - { - "developer": "TheDrunkenSnail", - "model_count": 3 - }, - { - "developer": "TheHierophant", - "model_count": 1 - }, - { - "developer": "theo77186", - "model_count": 1 - }, - { - "developer": "theprint", - "model_count": 18 - }, - { - "developer": "TheTsar1209", - "model_count": 7 - }, - { - "developer": "thinkcoder", - "model_count": 1 - }, - { - "developer": "thirdeyeai", - "model_count": 1 - }, - { - "developer": "thomas-yanxin", - "model_count": 4 - }, - { - "developer": "THUDM", - "model_count": 5 - }, - { - "developer": "tianyil1", - "model_count": 1 - }, - { - "developer": "TIGER-Lab", - "model_count": 6 - }, - { - "developer": "tii-uae", - "model_count": 4 - }, - { - "developer": "tiiuae", - "model_count": 18 - }, - { - "developer": "Tijmen2", - "model_count": 1 - }, - { - "developer": "tinycompany", - "model_count": 15 - }, - { - "developer": "TinyLlama", - "model_count": 6 - }, - { - "developer": "tklohj", - "model_count": 1 - }, - { - "developer": "ToastyPigeon", - "model_count": 1 - }, - { - "developer": "together", - "model_count": 4 - }, - { - "developer": "togethercomputer", - "model_count": 10 - }, - { - "developer": "tokyotech-llm", - "model_count": 1 - }, - { - "developer": "tomasmcm", - "model_count": 1 - }, - { - "developer": "Trappu", - "model_count": 2 - }, - { - "developer": "Tremontaine", - "model_count": 1 - }, - { - "developer": "Triangle104", - "model_count": 61 - }, - { - "developer": "trthminh1112", - "model_count": 1 - }, - { - "developer": "Tsunami-th", - "model_count": 4 - }, - { - "developer": "TTTXXX01", - "model_count": 1 - }, - { - "developer": "tugstugi", - "model_count": 1 - }, - { - "developer": "UCLA-AGI", - "model_count": 10 - }, - { - "developer": "uiuc-oumi", - "model_count": 2 - }, - { - "developer": "UKzExecution", - "model_count": 1 - }, - { - "developer": "Unbabel", - "model_count": 1 - }, - { - "developer": "Undi95", - "model_count": 2 - }, - { - "developer": "universalml", - "model_count": 1 - }, - { - "developer": "unknown", - "model_count": 10 - }, - { - "developer": "unsloth", - "model_count": 6 - }, - { - "developer": "upstage", - "model_count": 4 - }, - { - "developer": "utkmst", - "model_count": 1 - }, - { - "developer": "uukuguy", - "model_count": 7 - }, - { - "developer": "v000000", - "model_count": 6 - }, - { - "developer": "V3N0M", - "model_count": 1 - }, - { - "developer": "VAGOsolutions", - "model_count": 17 - }, - { - "developer": "ValiantLabs", - "model_count": 11 - }, - { - "developer": "vhab10", - "model_count": 3 - }, - { - "developer": "vicgalle", - "model_count": 12 - }, - { - "developer": "viettelsecurity-ai", - "model_count": 1 - }, - { - "developer": "vihangd", - "model_count": 1 - }, - { - "developer": "Vikhrmodels", - "model_count": 2 - }, - { - "developer": "VIRNECT", - "model_count": 2 - }, - { - "developer": "voidful", - "model_count": 1 - }, - { - "developer": "vonjack", - "model_count": 7 - }, - { - "developer": "w4r10ck", - "model_count": 1 - }, - { - "developer": "wanlige", - "model_count": 3 - }, - { - "developer": "wannaphong", - "model_count": 1 - }, - { - "developer": "waqasali1707", - "model_count": 1 - }, - { - "developer": "wave-on-discord", - "model_count": 1 - }, - { - "developer": "weathermanj", - "model_count": 4 - }, - { - "developer": "wenbopan", - "model_count": 1 - }, - { - "developer": "weqweasdas", - "model_count": 5 - }, - { - "developer": "Weyaxi", - "model_count": 8 - }, - { - "developer": "win10", - "model_count": 9 - }, - { - "developer": "winglian", - "model_count": 2 - }, - { - "developer": "WizardLMTeam", - "model_count": 3 - }, - { - "developer": "Wladastic", - "model_count": 1 - }, - { - "developer": "writer", - "model_count": 8 - }, - { - "developer": "wzhouad", - "model_count": 1 - }, - { - "developer": "x0000001", - "model_count": 1 - }, - { - "developer": "xAI", - "model_count": 11 - }, - { - "developer": "Xclbr7", - "model_count": 4 - }, - { - "developer": "Xiaojian9992024", - "model_count": 12 - }, - { - "developer": "xinchen9", - "model_count": 5 - }, - { - "developer": "Xkev", - "model_count": 1 - }, - { - "developer": "xkp24", - "model_count": 8 - }, - { - "developer": "xMaulana", - "model_count": 1 - }, - { - "developer": "xukp20", - "model_count": 8 - }, - { - "developer": "xwen-team", - "model_count": 1 - }, - { - "developer": "xxx777xxxASD", - "model_count": 1 - }, - { - "developer": "yam-peleg", - "model_count": 3 - }, - { - "developer": "yandex", - "model_count": 1 - }, - { - "developer": "yanng1242", - "model_count": 1 - }, - { - "developer": "Yash21", - "model_count": 1 - }, - { - "developer": "yasserrmd", - "model_count": 2 - }, - { - "developer": "ycros", - "model_count": 1 - }, - { - "developer": "yfzp", - "model_count": 8 - }, - { - "developer": "yifAI", - "model_count": 1 - }, - { - "developer": "ylalain", - "model_count": 1 - }, - { - "developer": "ymcki", - "model_count": 11 - }, - { - "developer": "Youlln", - "model_count": 19 - }, - { - "developer": "YoungPanda", - "model_count": 1 - }, - { - "developer": "YOYO-AI", - "model_count": 21 - }, - { - "developer": "yuchenxie", - "model_count": 2 - }, - { - "developer": "Yuma42", - "model_count": 3 - }, - { - "developer": "yuvraj17", - "model_count": 3 - }, - { - "developer": "Z-AI", - "model_count": 2 - }, - { - "developer": "Z.ai", - "model_count": 2 - }, - { - "developer": "Z1-Coder", - "model_count": 1 - }, - { - "developer": "zai-org", - "model_count": 1 - }, - { - "developer": "zake7749", - "model_count": 2 - }, - { - "developer": "zelk12", - "model_count": 78 - }, - { - "developer": "ZeroXClem", - "model_count": 11 - }, - { - "developer": "zetasepic", - "model_count": 2 - }, - { - "developer": "ZeusLabs", - "model_count": 1 - }, - { - "developer": "ZhangShenao", - "model_count": 1 - }, - { - "developer": "zhengr", - "model_count": 1 - }, - { - "developer": "zhipu", - "model_count": 3 - }, - { - "developer": "zhipu-ai", - "model_count": 1 - }, - { - "developer": "ZHLiu627", - "model_count": 2 - }, - { - "developer": "ZiyiYe", - "model_count": 1 - } -] \ No newline at end of file diff --git a/data/developers/0-hero.json b/data/developers/0-hero.json deleted file mode 100644 index 6befeb911b4cd279e2f4c9e399610c237252dbb0..0000000000000000000000000000000000000000 --- a/data/developers/0-hero.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "developer": "0-hero", - "models": [ - { - "id": "0-hero/Matter-0.1-7B-boost-DPO-preview", - "name": "0-hero/Matter-0.1-7B-boost-DPO-preview", - "developer": "0-hero", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7448, - "reward-bench/Chat": 0.9106, - "reward-bench/Chat Hard": 0.6096, - "reward-bench/Safety": 0.7135, - "reward-bench/Reasoning": 0.8395, - "reward-bench/Prior Sets (0.5 weight)": 0.5566 - } - }, - { - "id": "0-hero/Matter-0.1-7B-DPO-preview", - "name": "0-hero/Matter-0.1-7B-DPO-preview", - "developer": "0-hero", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7247, - "reward-bench/Chat": 0.8939, - "reward-bench/Chat Hard": 0.5768, - "reward-bench/Safety": 0.6378, - "reward-bench/Reasoning": 0.8854, - "reward-bench/Prior Sets (0.5 weight)": 0.5348 - } - }, - { - "id": "0-hero/Matter-0.2-7B-DPO", - "name": "Matter-0.2-7B-DPO", - "developer": "0-hero", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3303, - "hfopenllm_v2/BBH": 0.3596, - "hfopenllm_v2/MATH Level 5": 0.0144, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3814, - "hfopenllm_v2/MMLU-PRO": 0.1164 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/01-ai.json b/data/developers/01-ai.json deleted file mode 100644 index 31a39d5a4bb68b5bca1fea86da4b9a81339329e9..0000000000000000000000000000000000000000 --- a/data/developers/01-ai.json +++ /dev/null @@ -1,417 +0,0 @@ -{ - "developer": "01-ai", - "models": [ - { - "id": "01-ai/Yi-1.5-34B", - "name": "Yi-1.5-34B", - "developer": "01-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2841, - "hfopenllm_v2/BBH": 0.5976, - "hfopenllm_v2/MATH Level 5": 0.1533, - "hfopenllm_v2/GPQA": 0.3658, - "hfopenllm_v2/MUSR": 0.4236, - "hfopenllm_v2/MMLU-PRO": 0.4666 - } - }, - { - "id": "01-ai/Yi-1.5-34B-32K", - "name": "Yi-1.5-34B-32K", - "developer": "01-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3119, - "hfopenllm_v2/BBH": 0.6016, - "hfopenllm_v2/MATH Level 5": 0.1541, - "hfopenllm_v2/GPQA": 0.3633, - "hfopenllm_v2/MUSR": 0.4398, - "hfopenllm_v2/MMLU-PRO": 0.4709 - } - }, - { - "id": "01-ai/Yi-1.5-34B-Chat", - "name": "Yi-1.5-34B-Chat", - "developer": "01-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6067, - "hfopenllm_v2/BBH": 0.6084, - "hfopenllm_v2/MATH Level 5": 0.2772, - "hfopenllm_v2/GPQA": 0.3649, - "hfopenllm_v2/MUSR": 0.4282, - "hfopenllm_v2/MMLU-PRO": 0.452 - } - }, - { - "id": "01-ai/Yi-1.5-34B-Chat-16K", - "name": "Yi-1.5-34B-Chat-16K", - "developer": "01-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4564, - "hfopenllm_v2/BBH": 0.61, - "hfopenllm_v2/MATH Level 5": 0.2137, - "hfopenllm_v2/GPQA": 0.3381, - "hfopenllm_v2/MUSR": 0.4398, - "hfopenllm_v2/MMLU-PRO": 0.4545 - } - }, - { - "id": "01-ai/Yi-1.5-6B", - "name": "Yi-1.5-6B", - "developer": "01-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2617, - "hfopenllm_v2/BBH": 0.4493, - "hfopenllm_v2/MATH Level 5": 0.0665, - "hfopenllm_v2/GPQA": 0.3138, - "hfopenllm_v2/MUSR": 0.4374, - "hfopenllm_v2/MMLU-PRO": 0.3144 - } - }, - { - "id": "01-ai/Yi-1.5-6B-Chat", - "name": "Yi-1.5-6B-Chat", - "developer": "01-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5145, - "hfopenllm_v2/BBH": 0.4571, - "hfopenllm_v2/MATH Level 5": 0.1624, - "hfopenllm_v2/GPQA": 0.302, - "hfopenllm_v2/MUSR": 0.4392, - "hfopenllm_v2/MMLU-PRO": 0.3193 - } - }, - { - "id": "01-ai/Yi-1.5-9B", - "name": "Yi-1.5-9B", - "developer": "01-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2936, - "hfopenllm_v2/BBH": 0.5143, - "hfopenllm_v2/MATH Level 5": 0.114, - "hfopenllm_v2/GPQA": 0.3792, - "hfopenllm_v2/MUSR": 0.4328, - "hfopenllm_v2/MMLU-PRO": 0.3916 - } - }, - { - "id": "01-ai/Yi-1.5-9B-32K", - "name": "Yi-1.5-9B-32K", - "developer": "01-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2303, - "hfopenllm_v2/BBH": 0.4963, - "hfopenllm_v2/MATH Level 5": 0.108, - "hfopenllm_v2/GPQA": 0.3591, - "hfopenllm_v2/MUSR": 0.4186, - "hfopenllm_v2/MMLU-PRO": 0.3765 - } - }, - { - "id": "01-ai/Yi-1.5-9B-Chat", - "name": "Yi-1.5-9B-Chat", - "developer": "01-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6046, - "hfopenllm_v2/BBH": 0.5559, - "hfopenllm_v2/MATH Level 5": 0.2258, - "hfopenllm_v2/GPQA": 0.3347, - "hfopenllm_v2/MUSR": 0.4259, - "hfopenllm_v2/MMLU-PRO": 0.3975 - } - }, - { - "id": "01-ai/Yi-1.5-9B-Chat-16K", - "name": "Yi-1.5-9B-Chat-16K", - "developer": "01-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4214, - "hfopenllm_v2/BBH": 0.5153, - "hfopenllm_v2/MATH Level 5": 0.1782, - "hfopenllm_v2/GPQA": 0.3087, - "hfopenllm_v2/MUSR": 0.4099, - "hfopenllm_v2/MMLU-PRO": 0.3994 - } - }, - { - "id": "01-ai/yi-34b", - "name": "Yi 34B", - "developer": "01-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.57, - "helm_lite/NarrativeQA": 0.782, - "helm_lite/NaturalQuestions (closed-book)": 0.443, - "helm_lite/OpenbookQA": 0.92, - "helm_lite/MMLU": 0.65, - "helm_lite/MATH": 0.375, - "helm_lite/GSM8K": 0.648, - "helm_lite/LegalBench": 0.618, - "helm_lite/MedQA": 0.656, - "helm_lite/WMT 2014": 0.172, - "helm_mmlu/MMLU All Subjects": 0.762, - "helm_mmlu/Abstract Algebra": 0.4, - "helm_mmlu/Anatomy": 0.748, - "helm_mmlu/College Physics": 0.5, - "helm_mmlu/Computer Security": 0.83, - "helm_mmlu/Econometrics": 0.588, - "helm_mmlu/Global Facts": 0.53, - "helm_mmlu/Jurisprudence": 0.898, - "helm_mmlu/Philosophy": 0.82, - "helm_mmlu/Professional Psychology": 0.835, - "helm_mmlu/Us Foreign Policy": 0.91, - "helm_mmlu/Astronomy": 0.901, - "helm_mmlu/Business Ethics": 0.75, - "helm_mmlu/Clinical Knowledge": 0.8, - "helm_mmlu/Conceptual Physics": 0.77, - "helm_mmlu/Electrical Engineering": 0.779, - "helm_mmlu/Elementary Mathematics": 0.656, - "helm_mmlu/Formal Logic": 0.548, - "helm_mmlu/High School World History": 0.907, - "helm_mmlu/Human Sexuality": 0.87, - "helm_mmlu/International Law": 0.909, - "helm_mmlu/Logical Fallacies": 0.883, - "helm_mmlu/Machine Learning": 0.58, - "helm_mmlu/Management": 0.893, - "helm_mmlu/Marketing": 0.936, - "helm_mmlu/Medical Genetics": 0.87, - "helm_mmlu/Miscellaneous": 0.902, - "helm_mmlu/Moral Scenarios": 0.606, - "helm_mmlu/Nutrition": 0.869, - "helm_mmlu/Prehistory": 0.877, - "helm_mmlu/Public Relations": 0.745, - "helm_mmlu/Security Studies": 0.833, - "helm_mmlu/Sociology": 0.9, - "helm_mmlu/Virology": 0.572, - "helm_mmlu/World Religions": 0.877, - "helm_mmlu/Mean win rate": 0.315, - "hfopenllm_v2/IFEval": 0.3046, - "hfopenllm_v2/BBH": 0.5457, - "hfopenllm_v2/MATH Level 5": 0.0514, - "hfopenllm_v2/GPQA": 0.3666, - "hfopenllm_v2/MUSR": 0.4119, - "hfopenllm_v2/MMLU-PRO": 0.4412 - } - }, - { - "id": "01-ai/Yi-34B-200K", - "name": "Yi-34B-200K", - "developer": "01-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1542, - "hfopenllm_v2/BBH": 0.5442, - "hfopenllm_v2/MATH Level 5": 0.0574, - "hfopenllm_v2/GPQA": 0.3565, - "hfopenllm_v2/MUSR": 0.3817, - "hfopenllm_v2/MMLU-PRO": 0.4535 - } - }, - { - "id": "01-ai/Yi-34B-Chat", - "name": "Yi-34B-Chat", - "developer": "01-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4699, - "hfopenllm_v2/BBH": 0.5561, - "hfopenllm_v2/MATH Level 5": 0.0627, - "hfopenllm_v2/GPQA": 0.3381, - "hfopenllm_v2/MUSR": 0.3978, - "hfopenllm_v2/MMLU-PRO": 0.4093 - } - }, - { - "id": "01-ai/yi-6b", - "name": "Yi 6B", - "developer": "01-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.253, - "helm_lite/NarrativeQA": 0.702, - "helm_lite/NaturalQuestions (closed-book)": 0.31, - "helm_lite/OpenbookQA": 0.8, - "helm_lite/MMLU": 0.53, - "helm_lite/MATH": 0.126, - "helm_lite/GSM8K": 0.375, - "helm_lite/LegalBench": 0.519, - "helm_lite/MedQA": 0.497, - "helm_lite/WMT 2014": 0.117, - "helm_mmlu/MMLU All Subjects": 0.64, - "helm_mmlu/Abstract Algebra": 0.3, - "helm_mmlu/Anatomy": 0.6, - "helm_mmlu/College Physics": 0.422, - "helm_mmlu/Computer Security": 0.73, - "helm_mmlu/Econometrics": 0.351, - "helm_mmlu/Global Facts": 0.43, - "helm_mmlu/Jurisprudence": 0.796, - "helm_mmlu/Philosophy": 0.678, - "helm_mmlu/Professional Psychology": 0.668, - "helm_mmlu/Us Foreign Policy": 0.87, - "helm_mmlu/Astronomy": 0.684, - "helm_mmlu/Business Ethics": 0.67, - "helm_mmlu/Clinical Knowledge": 0.66, - "helm_mmlu/Conceptual Physics": 0.621, - "helm_mmlu/Electrical Engineering": 0.662, - "helm_mmlu/Elementary Mathematics": 0.452, - "helm_mmlu/Formal Logic": 0.452, - "helm_mmlu/High School World History": 0.785, - "helm_mmlu/Human Sexuality": 0.763, - "helm_mmlu/International Law": 0.769, - "helm_mmlu/Logical Fallacies": 0.779, - "helm_mmlu/Machine Learning": 0.411, - "helm_mmlu/Management": 0.806, - "helm_mmlu/Marketing": 0.893, - "helm_mmlu/Medical Genetics": 0.77, - "helm_mmlu/Miscellaneous": 0.796, - "helm_mmlu/Moral Scenarios": 0.335, - "helm_mmlu/Nutrition": 0.739, - "helm_mmlu/Prehistory": 0.713, - "helm_mmlu/Public Relations": 0.718, - "helm_mmlu/Security Studies": 0.735, - "helm_mmlu/Sociology": 0.831, - "helm_mmlu/Virology": 0.452, - "helm_mmlu/World Religions": 0.836, - "helm_mmlu/Mean win rate": 0.651, - "hfopenllm_v2/IFEval": 0.2893, - "hfopenllm_v2/BBH": 0.4309, - "hfopenllm_v2/MATH Level 5": 0.0159, - "hfopenllm_v2/GPQA": 0.2693, - "hfopenllm_v2/MUSR": 0.3937, - "hfopenllm_v2/MMLU-PRO": 0.2991 - } - }, - { - "id": "01-ai/Yi-6B-200K", - "name": "Yi-6B-200K", - "developer": "01-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0843, - "hfopenllm_v2/BBH": 0.4289, - "hfopenllm_v2/MATH Level 5": 0.0181, - "hfopenllm_v2/GPQA": 0.2819, - "hfopenllm_v2/MUSR": 0.4587, - "hfopenllm_v2/MMLU-PRO": 0.2844 - } - }, - { - "id": "01-ai/Yi-6B-Chat", - "name": "Yi-6B-Chat", - "developer": "01-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3395, - "hfopenllm_v2/BBH": 0.4133, - "hfopenllm_v2/MATH Level 5": 0.0136, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.3688, - "hfopenllm_v2/MMLU-PRO": 0.3061 - } - }, - { - "id": "01-ai/Yi-9B", - "name": "Yi-9B", - "developer": "01-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2709, - "hfopenllm_v2/BBH": 0.494, - "hfopenllm_v2/MATH Level 5": 0.0559, - "hfopenllm_v2/GPQA": 0.318, - "hfopenllm_v2/MUSR": 0.4054, - "hfopenllm_v2/MMLU-PRO": 0.3574 - } - }, - { - "id": "01-ai/Yi-9B-200K", - "name": "Yi-9B-200K", - "developer": "01-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2327, - "hfopenllm_v2/BBH": 0.4793, - "hfopenllm_v2/MATH Level 5": 0.0665, - "hfopenllm_v2/GPQA": 0.3154, - "hfopenllm_v2/MUSR": 0.4294, - "hfopenllm_v2/MMLU-PRO": 0.3622 - } - }, - { - "id": "01-ai/Yi-Coder-9B-Chat", - "name": "Yi-Coder-9B-Chat", - "developer": "01-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4817, - "hfopenllm_v2/BBH": 0.4814, - "hfopenllm_v2/MATH Level 5": 0.04, - "hfopenllm_v2/GPQA": 0.2475, - "hfopenllm_v2/MUSR": 0.3992, - "hfopenllm_v2/MMLU-PRO": 0.2425 - } - }, - { - "id": "01-ai/yi-large-preview", - "name": "Yi Large Preview", - "developer": "01-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.471, - "helm_lite/NarrativeQA": 0.373, - "helm_lite/NaturalQuestions (closed-book)": 0.428, - "helm_lite/OpenbookQA": 0.946, - "helm_lite/MMLU": 0.712, - "helm_lite/MATH": 0.712, - "helm_lite/GSM8K": 0.69, - "helm_lite/LegalBench": 0.519, - "helm_lite/MedQA": 0.66, - "helm_lite/WMT 2014": 0.176, - "helm_mmlu/MMLU All Subjects": 0.793, - "helm_mmlu/Abstract Algebra": 0.6, - "helm_mmlu/Anatomy": 0.83, - "helm_mmlu/College Physics": 0.569, - "helm_mmlu/Computer Security": 0.86, - "helm_mmlu/Econometrics": 0.728, - "helm_mmlu/Global Facts": 0.52, - "helm_mmlu/Jurisprudence": 0.852, - "helm_mmlu/Philosophy": 0.842, - "helm_mmlu/Professional Psychology": 0.853, - "helm_mmlu/Us Foreign Policy": 0.85, - "helm_mmlu/Astronomy": 0.914, - "helm_mmlu/Business Ethics": 0.8, - "helm_mmlu/Clinical Knowledge": 0.857, - "helm_mmlu/Conceptual Physics": 0.864, - "helm_mmlu/Electrical Engineering": 0.779, - "helm_mmlu/Elementary Mathematics": 0.685, - "helm_mmlu/Formal Logic": 0.603, - "helm_mmlu/High School World History": 0.928, - "helm_mmlu/Human Sexuality": 0.901, - "helm_mmlu/International Law": 0.917, - "helm_mmlu/Logical Fallacies": 0.865, - "helm_mmlu/Machine Learning": 0.616, - "helm_mmlu/Management": 0.903, - "helm_mmlu/Marketing": 0.927, - "helm_mmlu/Medical Genetics": 0.83, - "helm_mmlu/Miscellaneous": 0.916, - "helm_mmlu/Moral Scenarios": 0.831, - "helm_mmlu/Nutrition": 0.846, - "helm_mmlu/Prehistory": 0.892, - "helm_mmlu/Public Relations": 0.827, - "helm_mmlu/Security Studies": 0.82, - "helm_mmlu/Sociology": 0.881, - "helm_mmlu/Virology": 0.59, - "helm_mmlu/World Religions": 0.871, - "helm_mmlu/Mean win rate": 0.258 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/1-800-llms.json b/data/developers/1-800-llms.json deleted file mode 100644 index 885526469de5418eddf3a50c6f5f70664911b03b..0000000000000000000000000000000000000000 --- a/data/developers/1-800-llms.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "1-800-LLMs", - "models": [ - { - "id": "1-800-LLMs/Qwen-2.5-14B-Hindi", - "name": "Qwen-2.5-14B-Hindi", - "developer": "1-800-LLMs", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5826, - "hfopenllm_v2/BBH": 0.6524, - "hfopenllm_v2/MATH Level 5": 0.3331, - "hfopenllm_v2/GPQA": 0.3624, - "hfopenllm_v2/MUSR": 0.4489, - "hfopenllm_v2/MMLU-PRO": 0.5263 - } - }, - { - "id": "1-800-LLMs/Qwen-2.5-14B-Hindi-Custom-Instruct", - "name": "Qwen-2.5-14B-Hindi-Custom-Instruct", - "developer": "1-800-LLMs", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3077, - "hfopenllm_v2/BBH": 0.6284, - "hfopenllm_v2/MATH Level 5": 0.3112, - "hfopenllm_v2/GPQA": 0.37, - "hfopenllm_v2/MUSR": 0.4491, - "hfopenllm_v2/MMLU-PRO": 0.5164 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/1024m.json b/data/developers/1024m.json deleted file mode 100644 index 761a59c34e642cf9a31ba6bedb3ad4c747679080..0000000000000000000000000000000000000000 --- a/data/developers/1024m.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "1024m", - "models": [ - { - "id": "1024m/PHI-4-Hindi", - "name": "PHI-4-Hindi", - "developer": "1024m", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0082, - "hfopenllm_v2/BBH": 0.671, - "hfopenllm_v2/MATH Level 5": 0.2334, - "hfopenllm_v2/GPQA": 0.3977, - "hfopenllm_v2/MUSR": 0.4914, - "hfopenllm_v2/MMLU-PRO": 0.5239 - } - }, - { - "id": "1024m/QWEN-14B-B100", - "name": "QWEN-14B-B100", - "developer": "1024m", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7762, - "hfopenllm_v2/BBH": 0.6533, - "hfopenllm_v2/MATH Level 5": 0.5438, - "hfopenllm_v2/GPQA": 0.3507, - "hfopenllm_v2/MUSR": 0.41, - "hfopenllm_v2/MMLU-PRO": 0.5179 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/152334h.json b/data/developers/152334h.json deleted file mode 100644 index 8f79212374b64adcfe1585d7cfb364de175eb6c3..0000000000000000000000000000000000000000 --- a/data/developers/152334h.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "152334H", - "models": [ - { - "id": "152334H/miqu-1-70b-sf", - "name": "miqu-1-70b-sf", - "developer": "152334H", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5182, - "hfopenllm_v2/BBH": 0.6102, - "hfopenllm_v2/MATH Level 5": 0.1246, - "hfopenllm_v2/GPQA": 0.3507, - "hfopenllm_v2/MUSR": 0.4582, - "hfopenllm_v2/MMLU-PRO": 0.4228 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/1tuanpham.json b/data/developers/1tuanpham.json deleted file mode 100644 index f278fb7ce517d014ec8ccbf051550b592383476a..0000000000000000000000000000000000000000 --- a/data/developers/1tuanpham.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "1TuanPham", - "models": [ - { - "id": "1TuanPham/T-VisStar-7B-v0.1", - "name": "T-VisStar-7B-v0.1", - "developer": "1TuanPham", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3607, - "hfopenllm_v2/BBH": 0.5052, - "hfopenllm_v2/MATH Level 5": 0.0574, - "hfopenllm_v2/GPQA": 0.2852, - "hfopenllm_v2/MUSR": 0.4375, - "hfopenllm_v2/MMLU-PRO": 0.3211 - } - }, - { - "id": "1TuanPham/T-VisStar-v0.1", - "name": "T-VisStar-v0.1", - "developer": "1TuanPham", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3607, - "hfopenllm_v2/BBH": 0.5052, - "hfopenllm_v2/MATH Level 5": 0.0574, - "hfopenllm_v2/GPQA": 0.2852, - "hfopenllm_v2/MUSR": 0.4375, - "hfopenllm_v2/MMLU-PRO": 0.3211 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/3rd-degree-burn.json b/data/developers/3rd-degree-burn.json deleted file mode 100644 index a9c30dd95da988409fe72636331540de646e02b4..0000000000000000000000000000000000000000 --- a/data/developers/3rd-degree-burn.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "developer": "3rd-Degree-Burn", - "models": [ - { - "id": "3rd-Degree-Burn/L-3.1-Science-Writer-8B", - "name": "L-3.1-Science-Writer-8B", - "developer": "3rd-Degree-Burn", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4263, - "hfopenllm_v2/BBH": 0.5041, - "hfopenllm_v2/MATH Level 5": 0.1035, - "hfopenllm_v2/GPQA": 0.2743, - "hfopenllm_v2/MUSR": 0.3959, - "hfopenllm_v2/MMLU-PRO": 0.3649 - } - }, - { - "id": "3rd-Degree-Burn/Llama-3.1-8B-Squareroot", - "name": "Llama-3.1-8B-Squareroot", - "developer": "3rd-Degree-Burn", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2213, - "hfopenllm_v2/BBH": 0.3461, - "hfopenllm_v2/MATH Level 5": 0.2659, - "hfopenllm_v2/GPQA": 0.2567, - "hfopenllm_v2/MUSR": 0.3089, - "hfopenllm_v2/MMLU-PRO": 0.175 - } - }, - { - "id": "3rd-Degree-Burn/Llama-3.1-8B-Squareroot-v1", - "name": "Llama-3.1-8B-Squareroot-v1", - "developer": "3rd-Degree-Burn", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2892, - "hfopenllm_v2/BBH": 0.3343, - "hfopenllm_v2/MATH Level 5": 0.0884, - "hfopenllm_v2/GPQA": 0.2559, - "hfopenllm_v2/MUSR": 0.3341, - "hfopenllm_v2/MMLU-PRO": 0.1127 - } - }, - { - "id": "3rd-Degree-Burn/Llama-Squared-8B", - "name": "Llama-Squared-8B", - "developer": "3rd-Degree-Burn", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2755, - "hfopenllm_v2/BBH": 0.4431, - "hfopenllm_v2/MATH Level 5": 0.0574, - "hfopenllm_v2/GPQA": 0.2718, - "hfopenllm_v2/MUSR": 0.3089, - "hfopenllm_v2/MMLU-PRO": 0.2366 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/4season.json b/data/developers/4season.json deleted file mode 100644 index f089a9c6a4b809bc200bc8fd4f2e397ac7b658d0..0000000000000000000000000000000000000000 --- a/data/developers/4season.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "4season", - "models": [ - { - "id": "4season/final_model_test_v2", - "name": "final_model_test_v2", - "developer": "4season", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3191, - "hfopenllm_v2/BBH": 0.6342, - "hfopenllm_v2/MATH Level 5": 0.0838, - "hfopenllm_v2/GPQA": 0.3272, - "hfopenllm_v2/MUSR": 0.4314, - "hfopenllm_v2/MMLU-PRO": 0.3528 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/aaditya.json b/data/developers/aaditya.json deleted file mode 100644 index 0c8f7b6e99b4d65bac2e439e583dab6c4397f20f..0000000000000000000000000000000000000000 --- a/data/developers/aaditya.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "aaditya", - "models": [ - { - "id": "aaditya/Llama3-OpenBioLLM-70B", - "name": "Llama3-OpenBioLLM-70B", - "developer": "aaditya", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7597, - "hfopenllm_v2/BBH": 0.6399, - "hfopenllm_v2/MATH Level 5": 0.1971, - "hfopenllm_v2/GPQA": 0.323, - "hfopenllm_v2/MUSR": 0.4417, - "hfopenllm_v2/MMLU-PRO": 0.4867 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/aalf.json b/data/developers/aalf.json deleted file mode 100644 index 83290565071a0a55a04ca2c414334445d97f9e10..0000000000000000000000000000000000000000 --- a/data/developers/aalf.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "developer": "AALF", - "models": [ - { - "id": "AALF/FuseChat-Llama-3.1-8B-Instruct-preview", - "name": "FuseChat-Llama-3.1-8B-Instruct-preview", - "developer": "AALF", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.719, - "hfopenllm_v2/BBH": 0.512, - "hfopenllm_v2/MATH Level 5": 0.2477, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.382, - "hfopenllm_v2/MMLU-PRO": 0.3733 - } - }, - { - "id": "AALF/FuseChat-Llama-3.1-8B-SFT-preview", - "name": "FuseChat-Llama-3.1-8B-SFT-preview", - "developer": "AALF", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7281, - "hfopenllm_v2/BBH": 0.524, - "hfopenllm_v2/MATH Level 5": 0.2251, - "hfopenllm_v2/GPQA": 0.3045, - "hfopenllm_v2/MUSR": 0.402, - "hfopenllm_v2/MMLU-PRO": 0.3743 - } - }, - { - "id": "AALF/gemma-2-27b-it-SimPO-37K", - "name": "gemma-2-27b-it-SimPO-37K", - "developer": "AALF", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2407, - "hfopenllm_v2/BBH": 0.3911, - "hfopenllm_v2/MATH Level 5": 0.0128, - "hfopenllm_v2/GPQA": 0.2802, - "hfopenllm_v2/MUSR": 0.3488, - "hfopenllm_v2/MMLU-PRO": 0.1971 - } - }, - { - "id": "AALF/gemma-2-27b-it-SimPO-37K-100steps", - "name": "gemma-2-27b-it-SimPO-37K-100steps", - "developer": "AALF", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2568, - "hfopenllm_v2/BBH": 0.3931, - "hfopenllm_v2/MATH Level 5": 0.0211, - "hfopenllm_v2/GPQA": 0.2886, - "hfopenllm_v2/MUSR": 0.3329, - "hfopenllm_v2/MMLU-PRO": 0.2125 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/aashraf995.json b/data/developers/aashraf995.json deleted file mode 100644 index 9c3d5bcd4f4262d4967db225c4d78902d29a0c24..0000000000000000000000000000000000000000 --- a/data/developers/aashraf995.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "developer": "Aashraf995", - "models": [ - { - "id": "Aashraf995/Creative-7B-nerd", - "name": "Creative-7B-nerd", - "developer": "Aashraf995", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4722, - "hfopenllm_v2/BBH": 0.5607, - "hfopenllm_v2/MATH Level 5": 0.3165, - "hfopenllm_v2/GPQA": 0.3263, - "hfopenllm_v2/MUSR": 0.4515, - "hfopenllm_v2/MMLU-PRO": 0.4492 - } - }, - { - "id": "Aashraf995/Gemma-Evo-10B", - "name": "Gemma-Evo-10B", - "developer": "Aashraf995", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7332, - "hfopenllm_v2/BBH": 0.6044, - "hfopenllm_v2/MATH Level 5": 0.2228, - "hfopenllm_v2/GPQA": 0.354, - "hfopenllm_v2/MUSR": 0.4595, - "hfopenllm_v2/MMLU-PRO": 0.4275 - } - }, - { - "id": "Aashraf995/Qwen-Evo-7B", - "name": "Qwen-Evo-7B", - "developer": "Aashraf995", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4757, - "hfopenllm_v2/BBH": 0.5709, - "hfopenllm_v2/MATH Level 5": 0.3142, - "hfopenllm_v2/GPQA": 0.3255, - "hfopenllm_v2/MUSR": 0.4541, - "hfopenllm_v2/MMLU-PRO": 0.4462 - } - }, - { - "id": "Aashraf995/QwenStock-14B", - "name": "QwenStock-14B", - "developer": "Aashraf995", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5009, - "hfopenllm_v2/BBH": 0.655, - "hfopenllm_v2/MATH Level 5": 0.3573, - "hfopenllm_v2/GPQA": 0.3893, - "hfopenllm_v2/MUSR": 0.4793, - "hfopenllm_v2/MMLU-PRO": 0.5382 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/abacusai.json b/data/developers/abacusai.json deleted file mode 100644 index 6a16c27bd82b7b735273906a5ef96bdeb921af4f..0000000000000000000000000000000000000000 --- a/data/developers/abacusai.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "developer": "abacusai", - "models": [ - { - "id": "abacusai/bigstral-12b-32k", - "name": "bigstral-12b-32k", - "developer": "abacusai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4194, - "hfopenllm_v2/BBH": 0.47, - "hfopenllm_v2/MATH Level 5": 0.0151, - "hfopenllm_v2/GPQA": 0.2928, - "hfopenllm_v2/MUSR": 0.456, - "hfopenllm_v2/MMLU-PRO": 0.2641 - } - }, - { - "id": "abacusai/bigyi-15b", - "name": "bigyi-15b", - "developer": "abacusai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2094, - "hfopenllm_v2/BBH": 0.4345, - "hfopenllm_v2/MATH Level 5": 0.0295, - "hfopenllm_v2/GPQA": 0.3096, - "hfopenllm_v2/MUSR": 0.3538, - "hfopenllm_v2/MMLU-PRO": 0.3003 - } - }, - { - "id": "abacusai/Dracarys-72B-Instruct", - "name": "Dracarys-72B-Instruct", - "developer": "abacusai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7856, - "hfopenllm_v2/BBH": 0.6944, - "hfopenllm_v2/MATH Level 5": 0.3965, - "hfopenllm_v2/GPQA": 0.3909, - "hfopenllm_v2/MUSR": 0.4558, - "hfopenllm_v2/MMLU-PRO": 0.5456 - } - }, - { - "id": "abacusai/Liberated-Qwen1.5-14B", - "name": "Liberated-Qwen1.5-14B", - "developer": "abacusai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3631, - "hfopenllm_v2/BBH": 0.4948, - "hfopenllm_v2/MATH Level 5": 0.1601, - "hfopenllm_v2/GPQA": 0.2836, - "hfopenllm_v2/MUSR": 0.4175, - "hfopenllm_v2/MMLU-PRO": 0.3512 - } - }, - { - "id": "abacusai/Llama-3-Smaug-8B", - "name": "Llama-3-Smaug-8B", - "developer": "abacusai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4867, - "hfopenllm_v2/BBH": 0.4931, - "hfopenllm_v2/MATH Level 5": 0.0853, - "hfopenllm_v2/GPQA": 0.2483, - "hfopenllm_v2/MUSR": 0.3622, - "hfopenllm_v2/MMLU-PRO": 0.3185 - } - }, - { - "id": "abacusai/Smaug-34B-v0.1", - "name": "Smaug-34B-v0.1", - "developer": "abacusai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5016, - "hfopenllm_v2/BBH": 0.5358, - "hfopenllm_v2/MATH Level 5": 0.0718, - "hfopenllm_v2/GPQA": 0.3297, - "hfopenllm_v2/MUSR": 0.3979, - "hfopenllm_v2/MMLU-PRO": 0.4543 - } - }, - { - "id": "abacusai/Smaug-72B-v0.1", - "name": "Smaug-72B-v0.1", - "developer": "abacusai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5167, - "hfopenllm_v2/BBH": 0.5996, - "hfopenllm_v2/MATH Level 5": 0.1911, - "hfopenllm_v2/GPQA": 0.3238, - "hfopenllm_v2/MUSR": 0.4473, - "hfopenllm_v2/MMLU-PRO": 0.4624 - } - }, - { - "id": "abacusai/Smaug-Llama-3-70B-Instruct-32K", - "name": "Smaug-Llama-3-70B-Instruct-32K", - "developer": "abacusai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7761, - "hfopenllm_v2/BBH": 0.6493, - "hfopenllm_v2/MATH Level 5": 0.2749, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.4208, - "hfopenllm_v2/MMLU-PRO": 0.4765 - } - }, - { - "id": "abacusai/Smaug-Mixtral-v0.1", - "name": "Smaug-Mixtral-v0.1", - "developer": "abacusai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5554, - "hfopenllm_v2/BBH": 0.5162, - "hfopenllm_v2/MATH Level 5": 0.0952, - "hfopenllm_v2/GPQA": 0.3012, - "hfopenllm_v2/MUSR": 0.4298, - "hfopenllm_v2/MMLU-PRO": 0.3352 - } - }, - { - "id": "abacusai/Smaug-Qwen2-72B-Instruct", - "name": "Smaug-Qwen2-72B-Instruct", - "developer": "abacusai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7825, - "hfopenllm_v2/BBH": 0.691, - "hfopenllm_v2/MATH Level 5": 0.4131, - "hfopenllm_v2/GPQA": 0.3616, - "hfopenllm_v2/MUSR": 0.4401, - "hfopenllm_v2/MMLU-PRO": 0.519 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/abacusresearch.json b/data/developers/abacusresearch.json deleted file mode 100644 index 221269693ea740084ef1ec9cc41801e075802fce..0000000000000000000000000000000000000000 --- a/data/developers/abacusresearch.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "AbacusResearch", - "models": [ - { - "id": "AbacusResearch/Jallabi-34B", - "name": "Jallabi-34B", - "developer": "AbacusResearch", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3529, - "hfopenllm_v2/BBH": 0.6023, - "hfopenllm_v2/MATH Level 5": 0.0521, - "hfopenllm_v2/GPQA": 0.3389, - "hfopenllm_v2/MUSR": 0.4822, - "hfopenllm_v2/MMLU-PRO": 0.4682 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/abhishek.json b/data/developers/abhishek.json deleted file mode 100644 index 4a990668962b400e2663f06544d4986dc1420d7b..0000000000000000000000000000000000000000 --- a/data/developers/abhishek.json +++ /dev/null @@ -1,75 +0,0 @@ -{ - "developer": "abhishek", - "models": [ - { - "id": "abhishek/autotrain-0tmgq-5tpbg", - "name": "autotrain-0tmgq-5tpbg", - "developer": "abhishek", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1952, - "hfopenllm_v2/BBH": 0.3127, - "hfopenllm_v2/MATH Level 5": 0.0128, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3584, - "hfopenllm_v2/MMLU-PRO": 0.1144 - } - }, - { - "id": "abhishek/autotrain-llama3-70b-orpo-v1", - "name": "autotrain-llama3-70b-orpo-v1", - "developer": "abhishek", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4233, - "hfopenllm_v2/BBH": 0.5998, - "hfopenllm_v2/MATH Level 5": 0.0106, - "hfopenllm_v2/GPQA": 0.2441, - "hfopenllm_v2/MUSR": 0.3579, - "hfopenllm_v2/MMLU-PRO": 0.1122 - } - }, - { - "id": "abhishek/autotrain-llama3-70b-orpo-v2", - "name": "autotrain-llama3-70b-orpo-v2", - "developer": "abhishek", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5406, - "hfopenllm_v2/BBH": 0.5899, - "hfopenllm_v2/MATH Level 5": 0.2107, - "hfopenllm_v2/GPQA": 0.2936, - "hfopenllm_v2/MUSR": 0.4113, - "hfopenllm_v2/MMLU-PRO": 0.4818 - } - }, - { - "id": "abhishek/autotrain-llama3-orpo-v2", - "name": "autotrain-llama3-orpo-v2", - "developer": "abhishek", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4372, - "hfopenllm_v2/BBH": 0.3159, - "hfopenllm_v2/MATH Level 5": 0.0468, - "hfopenllm_v2/GPQA": 0.2668, - "hfopenllm_v2/MUSR": 0.3792, - "hfopenllm_v2/MMLU-PRO": 0.2218 - } - }, - { - "id": "abhishek/autotrain-vr4a1-e5mms", - "name": "autotrain-vr4a1-e5mms", - "developer": "abhishek", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2142, - "hfopenllm_v2/BBH": 0.5001, - "hfopenllm_v2/MATH Level 5": 0.1412, - "hfopenllm_v2/GPQA": 0.3196, - "hfopenllm_v2/MUSR": 0.3891, - "hfopenllm_v2/MMLU-PRO": 0.3667 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/abideen.json b/data/developers/abideen.json deleted file mode 100644 index 2dc881b02a2a8f6f895b89ec57324cd2908d459f..0000000000000000000000000000000000000000 --- a/data/developers/abideen.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "abideen", - "models": [ - { - "id": "abideen/MedPhi-4-14B-v1", - "name": "MedPhi-4-14B-v1", - "developer": "abideen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6277, - "hfopenllm_v2/BBH": 0.6897, - "hfopenllm_v2/MATH Level 5": 0.2931, - "hfopenllm_v2/GPQA": 0.344, - "hfopenllm_v2/MUSR": 0.4155, - "hfopenllm_v2/MMLU-PRO": 0.5338 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/adamo1139.json b/data/developers/adamo1139.json deleted file mode 100644 index 762c7a68f0531c4d31557e4e771d8ce5da0d6b9d..0000000000000000000000000000000000000000 --- a/data/developers/adamo1139.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "adamo1139", - "models": [ - { - "id": "adamo1139/Yi-34B-200K-AEZAKMI-v2", - "name": "Yi-34B-200K-AEZAKMI-v2", - "developer": "adamo1139", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4555, - "hfopenllm_v2/BBH": 0.5384, - "hfopenllm_v2/MATH Level 5": 0.0566, - "hfopenllm_v2/GPQA": 0.3322, - "hfopenllm_v2/MUSR": 0.3886, - "hfopenllm_v2/MMLU-PRO": 0.4513 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/adriszmar.json b/data/developers/adriszmar.json deleted file mode 100644 index 1f1d39916960942963a9c3c265196aea3657be38..0000000000000000000000000000000000000000 --- a/data/developers/adriszmar.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "adriszmar", - "models": [ - { - "id": "adriszmar/QAIMath-Qwen2.5-7B-TIES", - "name": "QAIMath-Qwen2.5-7B-TIES", - "developer": "adriszmar", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1746, - "hfopenllm_v2/BBH": 0.3126, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.245, - "hfopenllm_v2/MUSR": 0.4096, - "hfopenllm_v2/MMLU-PRO": 0.1087 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/aellm.json b/data/developers/aellm.json deleted file mode 100644 index 9e995289b108488f3ccc1b3679d0a21d92b604db..0000000000000000000000000000000000000000 --- a/data/developers/aellm.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "AELLM", - "models": [ - { - "id": "AELLM/gemma-2-aeria-infinity-9b", - "name": "gemma-2-aeria-infinity-9b", - "developer": "AELLM", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7594, - "hfopenllm_v2/BBH": 0.5983, - "hfopenllm_v2/MATH Level 5": 0.2145, - "hfopenllm_v2/GPQA": 0.3339, - "hfopenllm_v2/MUSR": 0.402, - "hfopenllm_v2/MMLU-PRO": 0.3862 - } - }, - { - "id": "AELLM/gemma-2-lyco-infinity-9b", - "name": "gemma-2-lyco-infinity-9b", - "developer": "AELLM", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7316, - "hfopenllm_v2/BBH": 0.584, - "hfopenllm_v2/MATH Level 5": 0.1707, - "hfopenllm_v2/GPQA": 0.328, - "hfopenllm_v2/MUSR": 0.4006, - "hfopenllm_v2/MMLU-PRO": 0.3787 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/aevalone.json b/data/developers/aevalone.json deleted file mode 100644 index 55082929513d655b314911f3243f2292ef947361..0000000000000000000000000000000000000000 --- a/data/developers/aevalone.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "aevalone", - "models": [ - { - "id": "aevalone/distill_qw_test", - "name": "distill_qw_test", - "developer": "aevalone", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7409, - "hfopenllm_v2/BBH": 0.5246, - "hfopenllm_v2/MATH Level 5": 0.4781, - "hfopenllm_v2/GPQA": 0.3003, - "hfopenllm_v2/MUSR": 0.386, - "hfopenllm_v2/MMLU-PRO": 0.4092 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/agentlans.json b/data/developers/agentlans.json deleted file mode 100644 index 98f9740882541b697b43a695b6b4d1ee3bdf493a..0000000000000000000000000000000000000000 --- a/data/developers/agentlans.json +++ /dev/null @@ -1,131 +0,0 @@ -{ - "developer": "agentlans", - "models": [ - { - "id": "agentlans/Gemma2-9B-AdvancedFuse", - "name": "Gemma2-9B-AdvancedFuse", - "developer": "agentlans", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1543, - "hfopenllm_v2/BBH": 0.5859, - "hfopenllm_v2/MATH Level 5": 0.1005, - "hfopenllm_v2/GPQA": 0.3347, - "hfopenllm_v2/MUSR": 0.4231, - "hfopenllm_v2/MMLU-PRO": 0.4 - } - }, - { - "id": "agentlans/Llama-3.2-1B-Instruct-CrashCourse12K", - "name": "Llama-3.2-1B-Instruct-CrashCourse12K", - "developer": "agentlans", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5395, - "hfopenllm_v2/BBH": 0.3548, - "hfopenllm_v2/MATH Level 5": 0.071, - "hfopenllm_v2/GPQA": 0.2408, - "hfopenllm_v2/MUSR": 0.321, - "hfopenllm_v2/MMLU-PRO": 0.1809 - } - }, - { - "id": "agentlans/Llama3.1-8B-drill", - "name": "Llama3.1-8B-drill", - "developer": "agentlans", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7652, - "hfopenllm_v2/BBH": 0.5016, - "hfopenllm_v2/MATH Level 5": 0.1715, - "hfopenllm_v2/GPQA": 0.2676, - "hfopenllm_v2/MUSR": 0.3672, - "hfopenllm_v2/MMLU-PRO": 0.3776 - } - }, - { - "id": "agentlans/Llama3.1-Daredevilish", - "name": "Llama3.1-Daredevilish", - "developer": "agentlans", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6292, - "hfopenllm_v2/BBH": 0.5013, - "hfopenllm_v2/MATH Level 5": 0.1292, - "hfopenllm_v2/GPQA": 0.3012, - "hfopenllm_v2/MUSR": 0.4091, - "hfopenllm_v2/MMLU-PRO": 0.3697 - } - }, - { - "id": "agentlans/Llama3.1-Daredevilish-Instruct", - "name": "Llama3.1-Daredevilish-Instruct", - "developer": "agentlans", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7926, - "hfopenllm_v2/BBH": 0.5235, - "hfopenllm_v2/MATH Level 5": 0.1722, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.3911, - "hfopenllm_v2/MMLU-PRO": 0.3877 - } - }, - { - "id": "agentlans/Llama3.1-LexiHermes-SuperStorm", - "name": "Llama3.1-LexiHermes-SuperStorm", - "developer": "agentlans", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7835, - "hfopenllm_v2/BBH": 0.5266, - "hfopenllm_v2/MATH Level 5": 0.1616, - "hfopenllm_v2/GPQA": 0.323, - "hfopenllm_v2/MUSR": 0.3963, - "hfopenllm_v2/MMLU-PRO": 0.3844 - } - }, - { - "id": "agentlans/Llama3.1-SuperDeepFuse", - "name": "Llama3.1-SuperDeepFuse", - "developer": "agentlans", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7762, - "hfopenllm_v2/BBH": 0.5049, - "hfopenllm_v2/MATH Level 5": 0.1828, - "hfopenllm_v2/GPQA": 0.2743, - "hfopenllm_v2/MUSR": 0.3699, - "hfopenllm_v2/MMLU-PRO": 0.3775 - } - }, - { - "id": "agentlans/Llama3.1-SuperDeepFuse-CrashCourse12K", - "name": "Llama3.1-SuperDeepFuse-CrashCourse12K", - "developer": "agentlans", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7187, - "hfopenllm_v2/BBH": 0.5216, - "hfopenllm_v2/MATH Level 5": 0.1805, - "hfopenllm_v2/GPQA": 0.3129, - "hfopenllm_v2/MUSR": 0.4026, - "hfopenllm_v2/MMLU-PRO": 0.3631 - } - }, - { - "id": "agentlans/Qwen2.5-0.5B-Instruct-CrashCourse-dropout", - "name": "Qwen2.5-0.5B-Instruct-CrashCourse-dropout", - "developer": "agentlans", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2949, - "hfopenllm_v2/BBH": 0.3312, - "hfopenllm_v2/MATH Level 5": 0.0423, - "hfopenllm_v2/GPQA": 0.2634, - "hfopenllm_v2/MUSR": 0.3342, - "hfopenllm_v2/MMLU-PRO": 0.1608 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/agi-0.json b/data/developers/agi-0.json deleted file mode 100644 index c95120768c3cd74c99ec36ec0c05db50bb8deaf5..0000000000000000000000000000000000000000 --- a/data/developers/agi-0.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "developer": "AGI-0", - "models": [ - { - "id": "AGI-0/Art-v0-3B", - "name": "Art-v0-3B", - "developer": "AGI-0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3192, - "hfopenllm_v2/BBH": 0.3401, - "hfopenllm_v2/MATH Level 5": 0.2462, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3768, - "hfopenllm_v2/MMLU-PRO": 0.1179 - } - }, - { - "id": "AGI-0/Artificium-llama3.1-8B-001", - "name": "Artificium-llama3.1-8B-001", - "developer": "AGI-0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5248, - "hfopenllm_v2/BBH": 0.4256, - "hfopenllm_v2/MATH Level 5": 0.136, - "hfopenllm_v2/GPQA": 0.2659, - "hfopenllm_v2/MUSR": 0.3795, - "hfopenllm_v2/MMLU-PRO": 0.3182 - } - }, - { - "id": "AGI-0/smartllama3.1-8B-001", - "name": "smartllama3.1-8B-001", - "developer": "AGI-0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3518, - "hfopenllm_v2/BBH": 0.467, - "hfopenllm_v2/MATH Level 5": 0.1299, - "hfopenllm_v2/GPQA": 0.3062, - "hfopenllm_v2/MUSR": 0.4386, - "hfopenllm_v2/MMLU-PRO": 0.3487 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/ahdoot.json b/data/developers/ahdoot.json deleted file mode 100644 index a449cc05cb76624902adfae4bc27a73846de58bf..0000000000000000000000000000000000000000 --- a/data/developers/ahdoot.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "Ahdoot", - "models": [ - { - "id": "Ahdoot/StructuredThinker-v0.3-MoreStructure", - "name": "StructuredThinker-v0.3-MoreStructure", - "developer": "Ahdoot", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4193, - "hfopenllm_v2/BBH": 0.4838, - "hfopenllm_v2/MATH Level 5": 0.2908, - "hfopenllm_v2/GPQA": 0.297, - "hfopenllm_v2/MUSR": 0.4158, - "hfopenllm_v2/MMLU-PRO": 0.361 - } - }, - { - "id": "Ahdoot/Test_StealthThinker", - "name": "Test_StealthThinker", - "developer": "Ahdoot", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.422, - "hfopenllm_v2/BBH": 0.4647, - "hfopenllm_v2/MATH Level 5": 0.179, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.428, - "hfopenllm_v2/MMLU-PRO": 0.3597 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/ahjeong.json b/data/developers/ahjeong.json deleted file mode 100644 index 8bad04354c12c186962719731a334cb95ca3b61a..0000000000000000000000000000000000000000 --- a/data/developers/ahjeong.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "Ahjeong", - "models": [ - { - "id": "Ahjeong/MMPO_Gemma_7b", - "name": "Ahjeong/MMPO_Gemma_7b", - "developer": "Ahjeong", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7587, - "reward-bench/Chat": 0.9693, - "reward-bench/Chat Hard": 0.614, - "reward-bench/Safety": 0.7135, - "reward-bench/Reasoning": 0.7756, - "reward-bench/Prior Sets (0.5 weight)": 0.6831 - } - }, - { - "id": "Ahjeong/MMPO_Gemma_7b_gamma1.1_epoch3", - "name": "Ahjeong/MMPO_Gemma_7b_gamma1.1_epoch3", - "developer": "Ahjeong", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7652, - "reward-bench/Chat": 0.9721, - "reward-bench/Chat Hard": 0.6338, - "reward-bench/Safety": 0.7635, - "reward-bench/Reasoning": 0.7284, - "reward-bench/Prior Sets (0.5 weight)": 0.6913 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/ahmeda335.json b/data/developers/ahmeda335.json deleted file mode 100644 index b2e9d41dc695bef0c1a41cfdf0f2550bf69de595..0000000000000000000000000000000000000000 --- a/data/developers/ahmeda335.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "ahmeda335", - "models": [ - { - "id": "ahmeda335/13_outOf_32_pruned_layers_llama3.1-8b", - "name": "13_outOf_32_pruned_layers_llama3.1-8b", - "developer": "ahmeda335", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1748, - "hfopenllm_v2/BBH": 0.2883, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3803, - "hfopenllm_v2/MMLU-PRO": 0.1129 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/ai-mo.json b/data/developers/ai-mo.json deleted file mode 100644 index 7ef790ea54a6088c6a20cda8c7a1caabf46cd638..0000000000000000000000000000000000000000 --- a/data/developers/ai-mo.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "AI-MO", - "models": [ - { - "id": "AI-MO/NuminaMath-7B-CoT", - "name": "NuminaMath-7B-CoT", - "developer": "AI-MO", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2689, - "hfopenllm_v2/BBH": 0.4314, - "hfopenllm_v2/MATH Level 5": 0.2696, - "hfopenllm_v2/GPQA": 0.2659, - "hfopenllm_v2/MUSR": 0.3303, - "hfopenllm_v2/MMLU-PRO": 0.2868 - } - }, - { - "id": "AI-MO/NuminaMath-7B-TIR", - "name": "NuminaMath-7B-TIR", - "developer": "AI-MO", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2756, - "hfopenllm_v2/BBH": 0.4144, - "hfopenllm_v2/MATH Level 5": 0.1609, - "hfopenllm_v2/GPQA": 0.2584, - "hfopenllm_v2/MUSR": 0.3509, - "hfopenllm_v2/MMLU-PRO": 0.2733 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/ai-sweden-models.json b/data/developers/ai-sweden-models.json deleted file mode 100644 index 77a3d2cbac7fc01cec16655dc8c04fa574bbfb21..0000000000000000000000000000000000000000 --- a/data/developers/ai-sweden-models.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "AI-Sweden-Models", - "models": [ - { - "id": "AI-Sweden-Models/gpt-sw3-40b", - "name": "gpt-sw3-40b", - "developer": "AI-Sweden-Models", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.147, - "hfopenllm_v2/BBH": 0.3268, - "hfopenllm_v2/MATH Level 5": 0.0174, - "hfopenllm_v2/GPQA": 0.2349, - "hfopenllm_v2/MUSR": 0.3632, - "hfopenllm_v2/MMLU-PRO": 0.1276 - } - }, - { - "id": "AI-Sweden-Models/Llama-3-8B-instruct", - "name": "Llama-3-8B-instruct", - "developer": "AI-Sweden-Models", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2401, - "hfopenllm_v2/BBH": 0.4173, - "hfopenllm_v2/MATH Level 5": 0.0385, - "hfopenllm_v2/GPQA": 0.2659, - "hfopenllm_v2/MUSR": 0.4771, - "hfopenllm_v2/MMLU-PRO": 0.2597 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/ai2.json b/data/developers/ai2.json deleted file mode 100644 index 4934c11b7806e647da8c3821dcfccba2566bb947..0000000000000000000000000000000000000000 --- a/data/developers/ai2.json +++ /dev/null @@ -1,89 +0,0 @@ -{ - "developer": "AI2", - "models": [ - { - "id": "ai2/llama-2-chat-7b-nectar-3.8m.json", - "name": "ai2/llama-2-chat-7b-nectar-3.8m.json", - "developer": "AI2", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5843, - "reward-bench/Chat": 0.8631, - "reward-bench/Chat Hard": 0.2654, - "reward-bench/Safety": 0.6243 - } - }, - { - "id": "ai2/llama-2-chat-nectar-180k.json", - "name": "ai2/llama-2-chat-nectar-180k.json", - "developer": "AI2", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5235, - "reward-bench/Chat": 0.8827, - "reward-bench/Chat Hard": 0.2851, - "reward-bench/Safety": 0.4027 - } - }, - { - "id": "ai2/llama-2-chat-ultrafeedback-60k.jsonl", - "name": "ai2/llama-2-chat-ultrafeedback-60k.jsonl", - "developer": "AI2", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.644, - "reward-bench/Chat": 0.9441, - "reward-bench/Chat Hard": 0.4539, - "reward-bench/Safety": 0.5338 - } - }, - { - "id": "ai2/tulu-2-7b-rm-v0-nectar-binarized-3.8m-check...", - "name": "ai2/tulu-2-7b-rm-v0-nectar-binarized-3.8m-check...", - "developer": "AI2", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7008, - "reward-bench/Chat": 0.9385, - "reward-bench/Chat Hard": 0.3882, - "reward-bench/Safety": 0.7757 - } - }, - { - "id": "ai2/tulu-2-7b-rm-v0-nectar-binarized-700k.json", - "name": "ai2/tulu-2-7b-rm-v0-nectar-binarized-700k.json", - "developer": "AI2", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7127, - "reward-bench/Chat": 0.9358, - "reward-bench/Chat Hard": 0.4079, - "reward-bench/Safety": 0.7946 - } - }, - { - "id": "ai2/tulu-2-7b-rm-v0-nectar-binarized.json", - "name": "ai2/tulu-2-7b-rm-v0-nectar-binarized.json", - "developer": "AI2", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6756, - "reward-bench/Chat": 0.9134, - "reward-bench/Chat Hard": 0.3904, - "reward-bench/Safety": 0.723 - } - }, - { - "id": "ai2/tulu-2-7b-rm-v0.json", - "name": "ai2/tulu-2-7b-rm-v0.json", - "developer": "AI2", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6655, - "reward-bench/Chat": 0.933, - "reward-bench/Chat Hard": 0.4539, - "reward-bench/Safety": 0.6095 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/ai21.json b/data/developers/ai21.json deleted file mode 100644 index 1dd30a59b06772680c55206a64349513e184b4bf..0000000000000000000000000000000000000000 --- a/data/developers/ai21.json +++ /dev/null @@ -1,364 +0,0 @@ -{ - "developer": "ai21", - "models": [ - { - "id": "ai21/J1-Grande-v1-17B", - "name": "J1-Grande v1 17B", - "developer": "ai21", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.433, - "helm_classic/MMLU": 0.27, - "helm_classic/BoolQ": 0.722, - "helm_classic/NarrativeQA": 0.672, - "helm_classic/NaturalQuestions (open-book)": 0.578, - "helm_classic/QuAC": 0.362, - "helm_classic/HellaSwag": 0.739, - "helm_classic/OpenbookQA": 0.52, - "helm_classic/TruthfulQA": 0.193, - "helm_classic/MS MARCO (TREC)": 0.341, - "helm_classic/CNN/DailyMail": 0.143, - "helm_classic/XSUM": 0.122, - "helm_classic/IMDB": 0.953, - "helm_classic/CivilComments": 0.529, - "helm_classic/RAFT": 0.658 - } - }, - { - "id": "ai21/J1-Grande-v2-beta-17B", - "name": "J1-Grande v2 beta 17B", - "developer": "ai21", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.706, - "helm_classic/MMLU": 0.445, - "helm_classic/BoolQ": 0.812, - "helm_classic/NarrativeQA": 0.725, - "helm_classic/NaturalQuestions (open-book)": 0.625, - "helm_classic/QuAC": 0.392, - "helm_classic/HellaSwag": 0.764, - "helm_classic/OpenbookQA": 0.56, - "helm_classic/TruthfulQA": 0.306, - "helm_classic/MS MARCO (TREC)": 0.46, - "helm_classic/CNN/DailyMail": 0.146, - "helm_classic/XSUM": 0.152, - "helm_classic/IMDB": 0.957, - "helm_classic/CivilComments": 0.546, - "helm_classic/RAFT": 0.679 - } - }, - { - "id": "ai21/J1-Jumbo-v1-178B", - "name": "J1-Jumbo v1 178B", - "developer": "ai21", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.517, - "helm_classic/MMLU": 0.259, - "helm_classic/BoolQ": 0.776, - "helm_classic/NarrativeQA": 0.695, - "helm_classic/NaturalQuestions (open-book)": 0.595, - "helm_classic/QuAC": 0.358, - "helm_classic/HellaSwag": 0.765, - "helm_classic/OpenbookQA": 0.534, - "helm_classic/TruthfulQA": 0.175, - "helm_classic/MS MARCO (TREC)": 0.363, - "helm_classic/CNN/DailyMail": 0.144, - "helm_classic/XSUM": 0.129, - "helm_classic/IMDB": 0.943, - "helm_classic/CivilComments": 0.553, - "helm_classic/RAFT": 0.681 - } - }, - { - "id": "ai21/J1-Large-v1-7.5B", - "name": "J1-Large v1 7.5B", - "developer": "ai21", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.285, - "helm_classic/MMLU": 0.241, - "helm_classic/BoolQ": 0.683, - "helm_classic/NarrativeQA": 0.623, - "helm_classic/NaturalQuestions (open-book)": 0.532, - "helm_classic/QuAC": 0.328, - "helm_classic/HellaSwag": 0.7, - "helm_classic/OpenbookQA": 0.514, - "helm_classic/TruthfulQA": 0.197, - "helm_classic/MS MARCO (TREC)": 0.292, - "helm_classic/CNN/DailyMail": 0.134, - "helm_classic/XSUM": 0.102, - "helm_classic/IMDB": 0.956, - "helm_classic/CivilComments": 0.532, - "helm_classic/RAFT": 0.545 - } - }, - { - "id": "ai21/j2-grande", - "name": "Jurassic-2 Grande 17B", - "developer": "ai21", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.172, - "helm_lite/NarrativeQA": 0.744, - "helm_lite/NaturalQuestions (closed-book)": 0.35, - "helm_lite/OpenbookQA": 0.614, - "helm_lite/MMLU": 0.471, - "helm_lite/MATH": 0.064, - "helm_lite/GSM8K": 0.159, - "helm_lite/LegalBench": 0.468, - "helm_lite/MedQA": 0.39, - "helm_lite/WMT 2014": 0.102 - } - }, - { - "id": "ai21/j2-jumbo", - "name": "Jurassic-2 Jumbo 178B", - "developer": "ai21", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.215, - "helm_lite/NarrativeQA": 0.728, - "helm_lite/NaturalQuestions (closed-book)": 0.385, - "helm_lite/OpenbookQA": 0.688, - "helm_lite/MMLU": 0.483, - "helm_lite/MATH": 0.103, - "helm_lite/GSM8K": 0.239, - "helm_lite/LegalBench": 0.533, - "helm_lite/MedQA": 0.431, - "helm_lite/WMT 2014": 0.114 - } - }, - { - "id": "ai21/jamba-1.5-large", - "name": "Jamba 1.5 Large", - "developer": "ai21", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.637, - "helm_lite/NarrativeQA": 0.664, - "helm_lite/NaturalQuestions (closed-book)": 0.394, - "helm_lite/OpenbookQA": 0.948, - "helm_lite/MMLU": 0.683, - "helm_lite/MATH": 0.692, - "helm_lite/GSM8K": 0.846, - "helm_lite/LegalBench": 0.675, - "helm_lite/MedQA": 0.698, - "helm_lite/WMT 2014": 0.203, - "helm_mmlu/MMLU All Subjects": 0.782, - "helm_mmlu/Abstract Algebra": 0.53, - "helm_mmlu/Anatomy": 0.793, - "helm_mmlu/College Physics": 0.51, - "helm_mmlu/Computer Security": 0.8, - "helm_mmlu/Econometrics": 0.614, - "helm_mmlu/Global Facts": 0.54, - "helm_mmlu/Jurisprudence": 0.87, - "helm_mmlu/Philosophy": 0.849, - "helm_mmlu/Professional Psychology": 0.842, - "helm_mmlu/Us Foreign Policy": 0.92, - "helm_mmlu/Astronomy": 0.882, - "helm_mmlu/Business Ethics": 0.77, - "helm_mmlu/Clinical Knowledge": 0.849, - "helm_mmlu/Conceptual Physics": 0.779, - "helm_mmlu/Electrical Engineering": 0.793, - "helm_mmlu/Elementary Mathematics": 0.656, - "helm_mmlu/Formal Logic": 0.619, - "helm_mmlu/High School World History": 0.911, - "helm_mmlu/Human Sexuality": 0.832, - "helm_mmlu/International Law": 0.884, - "helm_mmlu/Logical Fallacies": 0.859, - "helm_mmlu/Machine Learning": 0.688, - "helm_mmlu/Management": 0.864, - "helm_mmlu/Marketing": 0.94, - "helm_mmlu/Medical Genetics": 0.89, - "helm_mmlu/Miscellaneous": 0.931, - "helm_mmlu/Moral Scenarios": 0.686, - "helm_mmlu/Nutrition": 0.869, - "helm_mmlu/Prehistory": 0.892, - "helm_mmlu/Public Relations": 0.755, - "helm_mmlu/Security Studies": 0.771, - "helm_mmlu/Sociology": 0.93, - "helm_mmlu/Virology": 0.554, - "helm_mmlu/World Religions": 0.865, - "helm_mmlu/Mean win rate": 0.147 - } - }, - { - "id": "ai21/jamba-1.5-mini", - "name": "Jamba 1.5 Mini", - "developer": "ai21", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.414, - "helm_lite/NarrativeQA": 0.746, - "helm_lite/NaturalQuestions (closed-book)": 0.388, - "helm_lite/OpenbookQA": 0.89, - "helm_lite/MMLU": 0.582, - "helm_lite/MATH": 0.318, - "helm_lite/GSM8K": 0.691, - "helm_lite/LegalBench": 0.503, - "helm_lite/MedQA": 0.632, - "helm_lite/WMT 2014": 0.179, - "helm_mmlu/MMLU All Subjects": 0.699, - "helm_mmlu/Abstract Algebra": 0.33, - "helm_mmlu/Anatomy": 0.711, - "helm_mmlu/College Physics": 0.48, - "helm_mmlu/Computer Security": 0.73, - "helm_mmlu/Econometrics": 0.491, - "helm_mmlu/Global Facts": 0.43, - "helm_mmlu/Jurisprudence": 0.88, - "helm_mmlu/Philosophy": 0.752, - "helm_mmlu/Professional Psychology": 0.76, - "helm_mmlu/Us Foreign Policy": 0.9, - "helm_mmlu/Astronomy": 0.822, - "helm_mmlu/Business Ethics": 0.76, - "helm_mmlu/Clinical Knowledge": 0.74, - "helm_mmlu/Conceptual Physics": 0.677, - "helm_mmlu/Electrical Engineering": 0.683, - "helm_mmlu/Elementary Mathematics": 0.553, - "helm_mmlu/Formal Logic": 0.452, - "helm_mmlu/High School World History": 0.84, - "helm_mmlu/Human Sexuality": 0.809, - "helm_mmlu/International Law": 0.893, - "helm_mmlu/Logical Fallacies": 0.81, - "helm_mmlu/Machine Learning": 0.509, - "helm_mmlu/Management": 0.825, - "helm_mmlu/Marketing": 0.915, - "helm_mmlu/Medical Genetics": 0.69, - "helm_mmlu/Miscellaneous": 0.902, - "helm_mmlu/Moral Scenarios": 0.269, - "helm_mmlu/Nutrition": 0.801, - "helm_mmlu/Prehistory": 0.824, - "helm_mmlu/Public Relations": 0.727, - "helm_mmlu/Security Studies": 0.755, - "helm_mmlu/Sociology": 0.876, - "helm_mmlu/Virology": 0.578, - "helm_mmlu/World Religions": 0.842, - "helm_mmlu/Mean win rate": 0.206 - } - }, - { - "id": "ai21/jamba-instruct", - "name": "Jamba Instruct", - "developer": "ai21", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.287, - "helm_lite/NarrativeQA": 0.658, - "helm_lite/NaturalQuestions (closed-book)": 0.384, - "helm_lite/OpenbookQA": 0.796, - "helm_lite/MMLU": 0.582, - "helm_lite/MATH": 0.38, - "helm_lite/GSM8K": 0.67, - "helm_lite/LegalBench": 0.54, - "helm_lite/MedQA": 0.519, - "helm_lite/WMT 2014": 0.164, - "helm_mmlu/MMLU All Subjects": 0.659, - "helm_mmlu/Abstract Algebra": 0.36, - "helm_mmlu/Anatomy": 0.615, - "helm_mmlu/College Physics": 0.422, - "helm_mmlu/Computer Security": 0.76, - "helm_mmlu/Econometrics": 0.439, - "helm_mmlu/Global Facts": 0.4, - "helm_mmlu/Jurisprudence": 0.796, - "helm_mmlu/Philosophy": 0.749, - "helm_mmlu/Professional Psychology": 0.716, - "helm_mmlu/Us Foreign Policy": 0.91, - "helm_mmlu/Astronomy": 0.73, - "helm_mmlu/Business Ethics": 0.6, - "helm_mmlu/Clinical Knowledge": 0.702, - "helm_mmlu/Conceptual Physics": 0.677, - "helm_mmlu/Electrical Engineering": 0.621, - "helm_mmlu/Elementary Mathematics": 0.497, - "helm_mmlu/Formal Logic": 0.444, - "helm_mmlu/High School World History": 0.797, - "helm_mmlu/Human Sexuality": 0.794, - "helm_mmlu/International Law": 0.835, - "helm_mmlu/Logical Fallacies": 0.706, - "helm_mmlu/Machine Learning": 0.536, - "helm_mmlu/Management": 0.786, - "helm_mmlu/Marketing": 0.885, - "helm_mmlu/Medical Genetics": 0.67, - "helm_mmlu/Miscellaneous": 0.865, - "helm_mmlu/Moral Scenarios": 0.465, - "helm_mmlu/Nutrition": 0.745, - "helm_mmlu/Prehistory": 0.796, - "helm_mmlu/Public Relations": 0.682, - "helm_mmlu/Security Studies": 0.743, - "helm_mmlu/Sociology": 0.891, - "helm_mmlu/Virology": 0.53, - "helm_mmlu/World Religions": 0.813, - "helm_mmlu/Mean win rate": 0.887 - } - }, - { - "id": "ai21/Jurassic-2-Grande-17B", - "name": "Jurassic-2 Grande 17B", - "developer": "ai21", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.743, - "helm_classic/MMLU": 0.475, - "helm_classic/BoolQ": 0.826, - "helm_classic/NarrativeQA": 0.737, - "helm_classic/NaturalQuestions (open-book)": 0.639, - "helm_classic/QuAC": 0.418, - "helm_classic/HellaSwag": 0.781, - "helm_classic/OpenbookQA": 0.542, - "helm_classic/TruthfulQA": 0.348, - "helm_classic/MS MARCO (TREC)": 0.514, - "helm_classic/CNN/DailyMail": 0.144, - "helm_classic/XSUM": 0.167, - "helm_classic/IMDB": 0.938, - "helm_classic/CivilComments": 0.547, - "helm_classic/RAFT": 0.712 - } - }, - { - "id": "ai21/Jurassic-2-Jumbo-178B", - "name": "Jurassic-2 Jumbo 178B", - "developer": "ai21", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.824, - "helm_classic/MMLU": 0.48, - "helm_classic/BoolQ": 0.829, - "helm_classic/NarrativeQA": 0.733, - "helm_classic/NaturalQuestions (open-book)": 0.669, - "helm_classic/QuAC": 0.435, - "helm_classic/HellaSwag": 0.788, - "helm_classic/OpenbookQA": 0.558, - "helm_classic/TruthfulQA": 0.437, - "helm_classic/MS MARCO (TREC)": 0.661, - "helm_classic/CNN/DailyMail": 0.149, - "helm_classic/XSUM": 0.182, - "helm_classic/IMDB": 0.938, - "helm_classic/CivilComments": 0.57, - "helm_classic/RAFT": 0.746 - } - }, - { - "id": "ai21/Jurassic-2-Large-7.5B", - "name": "Jurassic-2 Large 7.5B", - "developer": "ai21", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.553, - "helm_classic/MMLU": 0.339, - "helm_classic/BoolQ": 0.742, - "helm_classic/NarrativeQA": -1.0, - "helm_classic/NaturalQuestions (open-book)": 0.589, - "helm_classic/QuAC": -1.0, - "helm_classic/HellaSwag": 0.729, - "helm_classic/OpenbookQA": 0.53, - "helm_classic/TruthfulQA": 0.245, - "helm_classic/MS MARCO (TREC)": 0.464, - "helm_classic/CNN/DailyMail": 0.136, - "helm_classic/XSUM": 0.142, - "helm_classic/IMDB": 0.956, - "helm_classic/CivilComments": 0.57, - "helm_classic/RAFT": 0.622 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/ai21labs.json b/data/developers/ai21labs.json deleted file mode 100644 index a4677a41b6989649ba4cf65097296ceb40de8f1b..0000000000000000000000000000000000000000 --- a/data/developers/ai21labs.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "ai21labs", - "models": [ - { - "id": "ai21labs/Jamba-v0.1", - "name": "Jamba-v0.1", - "developer": "ai21labs", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2026, - "hfopenllm_v2/BBH": 0.3602, - "hfopenllm_v2/MATH Level 5": 0.0159, - "hfopenllm_v2/GPQA": 0.2685, - "hfopenllm_v2/MUSR": 0.359, - "hfopenllm_v2/MMLU-PRO": 0.2492 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/ai4bharat.json b/data/developers/ai4bharat.json deleted file mode 100644 index b71d1440943c4bf07cedce605684de3dd4f777d7..0000000000000000000000000000000000000000 --- a/data/developers/ai4bharat.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "ai4bharat", - "models": [ - { - "id": "ai4bharat/Airavata", - "name": "Airavata", - "developer": "ai4bharat", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0559, - "hfopenllm_v2/BBH": 0.3628, - "hfopenllm_v2/MATH Level 5": 0.0181, - "hfopenllm_v2/GPQA": 0.2743, - "hfopenllm_v2/MUSR": 0.3763, - "hfopenllm_v2/MMLU-PRO": 0.1635 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/ai4free.json b/data/developers/ai4free.json deleted file mode 100644 index c31b5f213a67e63a45fc560b57d6f186f5284b23..0000000000000000000000000000000000000000 --- a/data/developers/ai4free.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "AI4free", - "models": [ - { - "id": "AI4free/Dhanishtha", - "name": "Dhanishtha", - "developer": "AI4free", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2451, - "hfopenllm_v2/BBH": 0.3404, - "hfopenllm_v2/MATH Level 5": 0.256, - "hfopenllm_v2/GPQA": 0.2525, - "hfopenllm_v2/MUSR": 0.3569, - "hfopenllm_v2/MMLU-PRO": 0.1643 - } - }, - { - "id": "AI4free/t2", - "name": "t2", - "developer": "AI4free", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3867, - "hfopenllm_v2/BBH": 0.291, - "hfopenllm_v2/MATH Level 5": 0.1896, - "hfopenllm_v2/GPQA": 0.2576, - "hfopenllm_v2/MUSR": 0.3846, - "hfopenllm_v2/MMLU-PRO": 0.1144 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/aicoressecurity.json b/data/developers/aicoressecurity.json deleted file mode 100644 index 812ec81d59db5c76d60bd5a09b361da8eea79f99..0000000000000000000000000000000000000000 --- a/data/developers/aicoressecurity.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "developer": "AicoresSecurity", - "models": [ - { - "id": "AicoresSecurity/Cybernet-Sec-3B-R1-V0", - "name": "Cybernet-Sec-3B-R1-V0", - "developer": "AicoresSecurity", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6358, - "hfopenllm_v2/BBH": 0.4497, - "hfopenllm_v2/MATH Level 5": 0.1156, - "hfopenllm_v2/GPQA": 0.2634, - "hfopenllm_v2/MUSR": 0.3314, - "hfopenllm_v2/MMLU-PRO": 0.301 - } - }, - { - "id": "AicoresSecurity/Cybernet-Sec-3B-R1-V0-Coder", - "name": "Cybernet-Sec-3B-R1-V0-Coder", - "developer": "AicoresSecurity", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7098, - "hfopenllm_v2/BBH": 0.4478, - "hfopenllm_v2/MATH Level 5": 0.1488, - "hfopenllm_v2/GPQA": 0.2718, - "hfopenllm_v2/MUSR": 0.3408, - "hfopenllm_v2/MMLU-PRO": 0.3178 - } - }, - { - "id": "AicoresSecurity/Cybernet-Sec-3B-R1-V1", - "name": "Cybernet-Sec-3B-R1-V1", - "developer": "AicoresSecurity", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6146, - "hfopenllm_v2/BBH": 0.4282, - "hfopenllm_v2/MATH Level 5": 0.1518, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.3287, - "hfopenllm_v2/MMLU-PRO": 0.2876 - } - }, - { - "id": "AicoresSecurity/Cybernet-Sec-3B-R1-V1.1", - "name": "Cybernet-Sec-3B-R1-V1.1", - "developer": "AicoresSecurity", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.673, - "hfopenllm_v2/BBH": 0.4392, - "hfopenllm_v2/MATH Level 5": 0.176, - "hfopenllm_v2/GPQA": 0.271, - "hfopenllm_v2/MUSR": 0.3541, - "hfopenllm_v2/MMLU-PRO": 0.3088 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/aidc-ai.json b/data/developers/aidc-ai.json deleted file mode 100644 index 7016f4a16bdd0d8604b746144073520c40d16c45..0000000000000000000000000000000000000000 --- a/data/developers/aidc-ai.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "AIDC-AI", - "models": [ - { - "id": "AIDC-AI/Marco-o1", - "name": "Marco-o1", - "developer": "AIDC-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4771, - "hfopenllm_v2/BBH": 0.5364, - "hfopenllm_v2/MATH Level 5": 0.3746, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.4138, - "hfopenllm_v2/MMLU-PRO": 0.4117 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/aixonlab.json b/data/developers/aixonlab.json deleted file mode 100644 index 030865828370dbc407c177eed1de281d34ad8d0d..0000000000000000000000000000000000000000 --- a/data/developers/aixonlab.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "developer": "aixonlab", - "models": [ - { - "id": "aixonlab/Aether-12b", - "name": "Aether-12b", - "developer": "aixonlab", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2347, - "hfopenllm_v2/BBH": 0.5179, - "hfopenllm_v2/MATH Level 5": 0.1065, - "hfopenllm_v2/GPQA": 0.3163, - "hfopenllm_v2/MUSR": 0.3829, - "hfopenllm_v2/MMLU-PRO": 0.341 - } - }, - { - "id": "aixonlab/Grey-12b", - "name": "Grey-12b", - "developer": "aixonlab", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3968, - "hfopenllm_v2/BBH": 0.5699, - "hfopenllm_v2/MATH Level 5": 0.0982, - "hfopenllm_v2/GPQA": 0.3003, - "hfopenllm_v2/MUSR": 0.4516, - "hfopenllm_v2/MMLU-PRO": 0.3779 - } - }, - { - "id": "aixonlab/Zara-14b-v1.2", - "name": "Zara-14b-v1.2", - "developer": "aixonlab", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6197, - "hfopenllm_v2/BBH": 0.6405, - "hfopenllm_v2/MATH Level 5": 0.3535, - "hfopenllm_v2/GPQA": 0.3817, - "hfopenllm_v2/MUSR": 0.4675, - "hfopenllm_v2/MMLU-PRO": 0.5263 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/akhadangi.json b/data/developers/akhadangi.json deleted file mode 100644 index a45a05965f0b7c3c570fed151d6ee3e5cc9b7be0..0000000000000000000000000000000000000000 --- a/data/developers/akhadangi.json +++ /dev/null @@ -1,75 +0,0 @@ -{ - "developer": "akhadangi", - "models": [ - { - "id": "akhadangi/Llama3.2.1B.0.01-First", - "name": "Llama3.2.1B.0.01-First", - "developer": "akhadangi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0814, - "hfopenllm_v2/BBH": 0.3189, - "hfopenllm_v2/MATH Level 5": 0.0181, - "hfopenllm_v2/GPQA": 0.2483, - "hfopenllm_v2/MUSR": 0.3194, - "hfopenllm_v2/MMLU-PRO": 0.1197 - } - }, - { - "id": "akhadangi/Llama3.2.1B.0.01-Last", - "name": "Llama3.2.1B.0.01-Last", - "developer": "akhadangi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0917, - "hfopenllm_v2/BBH": 0.3159, - "hfopenllm_v2/MATH Level 5": 0.0136, - "hfopenllm_v2/GPQA": 0.2433, - "hfopenllm_v2/MUSR": 0.3206, - "hfopenllm_v2/MMLU-PRO": 0.1227 - } - }, - { - "id": "akhadangi/Llama3.2.1B.0.1-First", - "name": "Llama3.2.1B.0.1-First", - "developer": "akhadangi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1001, - "hfopenllm_v2/BBH": 0.312, - "hfopenllm_v2/MATH Level 5": 0.0211, - "hfopenllm_v2/GPQA": 0.245, - "hfopenllm_v2/MUSR": 0.3301, - "hfopenllm_v2/MMLU-PRO": 0.1169 - } - }, - { - "id": "akhadangi/Llama3.2.1B.0.1-Last", - "name": "Llama3.2.1B.0.1-Last", - "developer": "akhadangi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.095, - "hfopenllm_v2/BBH": 0.3164, - "hfopenllm_v2/MATH Level 5": 0.0211, - "hfopenllm_v2/GPQA": 0.2383, - "hfopenllm_v2/MUSR": 0.3341, - "hfopenllm_v2/MMLU-PRO": 0.1178 - } - }, - { - "id": "akhadangi/Llama3.2.1B.BaseFiT", - "name": "Llama3.2.1B.BaseFiT", - "developer": "akhadangi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0883, - "hfopenllm_v2/BBH": 0.3175, - "hfopenllm_v2/MATH Level 5": 0.0242, - "hfopenllm_v2/GPQA": 0.2534, - "hfopenllm_v2/MUSR": 0.3221, - "hfopenllm_v2/MMLU-PRO": 0.1172 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/akjindal53244.json b/data/developers/akjindal53244.json deleted file mode 100644 index 237ea0357d953fdc2d416f7c27241c406836e723..0000000000000000000000000000000000000000 --- a/data/developers/akjindal53244.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "akjindal53244", - "models": [ - { - "id": "akjindal53244/Llama-3.1-Storm-8B", - "name": "Llama-3.1-Storm-8B", - "developer": "akjindal53244", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8051, - "hfopenllm_v2/BBH": 0.5189, - "hfopenllm_v2/MATH Level 5": 0.1722, - "hfopenllm_v2/GPQA": 0.3263, - "hfopenllm_v2/MUSR": 0.4028, - "hfopenllm_v2/MMLU-PRO": 0.3803 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/alcholjung.json b/data/developers/alcholjung.json deleted file mode 100644 index 6e1e3376cbc24e2a296860982d2dd301a89de9d8..0000000000000000000000000000000000000000 --- a/data/developers/alcholjung.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "alcholjung", - "models": [ - { - "id": "alcholjung/llama3_medical_tuned", - "name": "llama3_medical_tuned", - "developer": "alcholjung", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0106, - "hfopenllm_v2/BBH": 0.4513, - "hfopenllm_v2/MATH Level 5": 0.0468, - "hfopenllm_v2/GPQA": 0.2861, - "hfopenllm_v2/MUSR": 0.466, - "hfopenllm_v2/MMLU-PRO": 0.2946 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/alepach.json b/data/developers/alepach.json deleted file mode 100644 index 781d99e668f919b1a6e78c381f192426b1911d1e..0000000000000000000000000000000000000000 --- a/data/developers/alepach.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "developer": "Alepach", - "models": [ - { - "id": "Alepach/notHumpback-M0", - "name": "notHumpback-M0", - "developer": "Alepach", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.235, - "hfopenllm_v2/BBH": 0.2785, - "hfopenllm_v2/MATH Level 5": 0.0189, - "hfopenllm_v2/GPQA": 0.2492, - "hfopenllm_v2/MUSR": 0.3552, - "hfopenllm_v2/MMLU-PRO": 0.1119 - } - }, - { - "id": "Alepach/notHumpback-M1", - "name": "notHumpback-M1", - "developer": "Alepach", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2207, - "hfopenllm_v2/BBH": 0.2882, - "hfopenllm_v2/MATH Level 5": 0.0159, - "hfopenllm_v2/GPQA": 0.2374, - "hfopenllm_v2/MUSR": 0.342, - "hfopenllm_v2/MMLU-PRO": 0.1091 - } - }, - { - "id": "Alepach/notHumpback-M1-v2", - "name": "notHumpback-M1-v2", - "developer": "Alepach", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2277, - "hfopenllm_v2/BBH": 0.2776, - "hfopenllm_v2/MATH Level 5": 0.0219, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3473, - "hfopenllm_v2/MMLU-PRO": 0.1119 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/aleph-alpha.json b/data/developers/aleph-alpha.json deleted file mode 100644 index 5431a53fbdc57b70164fd78532e9e2a2758e28be..0000000000000000000000000000000000000000 --- a/data/developers/aleph-alpha.json +++ /dev/null @@ -1,74 +0,0 @@ -{ - "developer": "aleph-alpha", - "models": [ - { - "id": "aleph-alpha/Luminous-Base-13B", - "name": "Luminous Base 13B", - "developer": "aleph-alpha", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.315, - "helm_classic/MMLU": 0.27, - "helm_classic/BoolQ": 0.719, - "helm_classic/NarrativeQA": 0.605, - "helm_classic/NaturalQuestions (open-book)": 0.568, - "helm_classic/QuAC": 0.334, - "helm_classic/HellaSwag": -1.0, - "helm_classic/OpenbookQA": -1.0, - "helm_classic/TruthfulQA": 0.182, - "helm_classic/MS MARCO (TREC)": -1.0, - "helm_classic/CNN/DailyMail": 0.11, - "helm_classic/XSUM": 0.105, - "helm_classic/IMDB": 0.939, - "helm_classic/CivilComments": 0.544, - "helm_classic/RAFT": 0.473 - } - }, - { - "id": "aleph-alpha/Luminous-Extended-30B", - "name": "Luminous Extended 30B", - "developer": "aleph-alpha", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.485, - "helm_classic/MMLU": 0.321, - "helm_classic/BoolQ": 0.767, - "helm_classic/NarrativeQA": 0.665, - "helm_classic/NaturalQuestions (open-book)": 0.609, - "helm_classic/QuAC": 0.349, - "helm_classic/HellaSwag": -1.0, - "helm_classic/OpenbookQA": -1.0, - "helm_classic/TruthfulQA": 0.221, - "helm_classic/MS MARCO (TREC)": -1.0, - "helm_classic/CNN/DailyMail": 0.139, - "helm_classic/XSUM": 0.124, - "helm_classic/IMDB": 0.947, - "helm_classic/CivilComments": 0.524, - "helm_classic/RAFT": 0.523 - } - }, - { - "id": "aleph-alpha/Luminous-Supreme-70B", - "name": "Luminous Supreme 70B", - "developer": "aleph-alpha", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.662, - "helm_classic/MMLU": 0.38, - "helm_classic/BoolQ": 0.775, - "helm_classic/NarrativeQA": 0.711, - "helm_classic/NaturalQuestions (open-book)": 0.649, - "helm_classic/QuAC": 0.37, - "helm_classic/HellaSwag": -1.0, - "helm_classic/OpenbookQA": -1.0, - "helm_classic/TruthfulQA": 0.222, - "helm_classic/MS MARCO (TREC)": -1.0, - "helm_classic/CNN/DailyMail": 0.15, - "helm_classic/XSUM": 0.136, - "helm_classic/IMDB": 0.959, - "helm_classic/CivilComments": 0.562, - "helm_classic/RAFT": 0.653 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/alephalpha.json b/data/developers/alephalpha.json deleted file mode 100644 index 6443dc032cb73772b33472d727d995f58333d00b..0000000000000000000000000000000000000000 --- a/data/developers/alephalpha.json +++ /dev/null @@ -1,59 +0,0 @@ -{ - "developer": "AlephAlpha", - "models": [ - { - "id": "AlephAlpha/luminous-base", - "name": "Luminous Base 13B", - "developer": "AlephAlpha", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.041, - "helm_lite/NarrativeQA": 0.633, - "helm_lite/NaturalQuestions (closed-book)": 0.197, - "helm_lite/OpenbookQA": 0.286, - "helm_lite/MMLU": 0.243, - "helm_lite/MATH": 0.026, - "helm_lite/GSM8K": 0.028, - "helm_lite/LegalBench": 0.332, - "helm_lite/MedQA": 0.26, - "helm_lite/WMT 2014": 0.066 - } - }, - { - "id": "AlephAlpha/luminous-extended", - "name": "Luminous Extended 30B", - "developer": "AlephAlpha", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.078, - "helm_lite/NarrativeQA": 0.684, - "helm_lite/NaturalQuestions (closed-book)": 0.253, - "helm_lite/OpenbookQA": 0.272, - "helm_lite/MMLU": 0.248, - "helm_lite/MATH": 0.04, - "helm_lite/GSM8K": 0.075, - "helm_lite/LegalBench": 0.421, - "helm_lite/MedQA": 0.276, - "helm_lite/WMT 2014": 0.083 - } - }, - { - "id": "AlephAlpha/luminous-supreme", - "name": "Luminous Supreme 70B", - "developer": "AlephAlpha", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.145, - "helm_lite/NarrativeQA": 0.743, - "helm_lite/NaturalQuestions (closed-book)": 0.299, - "helm_lite/OpenbookQA": 0.284, - "helm_lite/MMLU": 0.316, - "helm_lite/MATH": 0.078, - "helm_lite/GSM8K": 0.137, - "helm_lite/LegalBench": 0.452, - "helm_lite/MedQA": 0.276, - "helm_lite/WMT 2014": 0.102 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/alibaba-nlp.json b/data/developers/alibaba-nlp.json deleted file mode 100644 index 5f6d0b2ffd32c20f9bfaf3ede70baae22cb89e6d..0000000000000000000000000000000000000000 --- a/data/developers/alibaba-nlp.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "Alibaba-NLP", - "models": [ - { - "id": "Alibaba-NLP/gte-Qwen2-7B-instruct", - "name": "gte-Qwen2-7B-instruct", - "developer": "Alibaba-NLP", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2255, - "hfopenllm_v2/BBH": 0.4495, - "hfopenllm_v2/MATH Level 5": 0.0642, - "hfopenllm_v2/GPQA": 0.245, - "hfopenllm_v2/MUSR": 0.3559, - "hfopenllm_v2/MMLU-PRO": 0.3321 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/alibaba.json b/data/developers/alibaba.json deleted file mode 100644 index 759f3a2b29b65da87e85bd7df67845381cdc3145..0000000000000000000000000000000000000000 --- a/data/developers/alibaba.json +++ /dev/null @@ -1,85 +0,0 @@ -{ - "developer": "Alibaba", - "models": [ - { - "id": "alibaba/qwen-3-coder-480b", - "name": "Qwen 3 Coder 480B", - "developer": "Alibaba", - "evaluator_relationship": null, - "benchmark_scores": { - "terminal-bench-2.0/terminal-bench-2.0": 23.9 - } - }, - { - "id": "alibaba/qwen3-235b-a22b-instruct-2507", - "name": "qwen3-235b-a22b-instruct-2507", - "developer": "alibaba", - "evaluator_relationship": null, - "benchmark_scores": { - "global-mmlu-lite/Global MMLU Lite": 0.8798, - "global-mmlu-lite/Culturally Sensitive": 0.8522, - "global-mmlu-lite/Culturally Agnostic": 0.9075, - "global-mmlu-lite/Arabic": 0.88, - "global-mmlu-lite/English": 0.89, - "global-mmlu-lite/Bengali": 0.8875, - "global-mmlu-lite/German": 0.885, - "global-mmlu-lite/French": 0.88, - "global-mmlu-lite/Hindi": 0.8775, - "global-mmlu-lite/Indonesian": 0.88, - "global-mmlu-lite/Italian": 0.88, - "global-mmlu-lite/Japanese": 0.88, - "global-mmlu-lite/Korean": 0.875, - "global-mmlu-lite/Portuguese": 0.8875, - "global-mmlu-lite/Spanish": 0.875, - "global-mmlu-lite/Swahili": 0.87, - "global-mmlu-lite/Yoruba": 0.8725, - "global-mmlu-lite/Chinese": 0.8775, - "global-mmlu-lite/Burmese": 0.88 - } - }, - { - "id": "alibaba/qwen3-235b-a22b-thinking-2507", - "name": "qwen3-235b-a22b-thinking-2507", - "developer": "Alibaba", - "evaluator_relationship": null, - "benchmark_scores": { - "livecodebenchpro/Hard Problems": 0.0, - "livecodebenchpro/Medium Problems": 0.1267605633802817, - "livecodebenchpro/Easy Problems": 0.7605633802816901 - } - }, - { - "id": "alibaba/qwen3-30b-a3b", - "name": "qwen3-30b-a3b", - "developer": "Alibaba", - "evaluator_relationship": null, - "benchmark_scores": { - "livecodebenchpro/Hard Problems": 0.0, - "livecodebenchpro/Medium Problems": 0.028169014084507043, - "livecodebenchpro/Easy Problems": 0.5774647887323944 - } - }, - { - "id": "alibaba/qwen3-max", - "name": "alibaba/qwen3-max", - "developer": "Alibaba", - "evaluator_relationship": null, - "benchmark_scores": { - "livecodebenchpro/Hard Problems": 0.0, - "livecodebenchpro/Medium Problems": 0.04225352112676056, - "livecodebenchpro/Easy Problems": 0.36619718309859156 - } - }, - { - "id": "alibaba/qwen3-next-80b-a3b-thinking", - "name": "qwen3-next-80b-a3b-thinking", - "developer": "Alibaba", - "evaluator_relationship": null, - "benchmark_scores": { - "livecodebenchpro/Hard Problems": 0.0, - "livecodebenchpro/Medium Problems": 0.14084507042253522, - "livecodebenchpro/Easy Problems": 0.7464788732394366 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/aliyun.json b/data/developers/aliyun.json deleted file mode 100644 index 07bfc64c9c2fb6834fa3708114878f5fe38b0cb1..0000000000000000000000000000000000000000 --- a/data/developers/aliyun.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "developer": "aliyun", - "models": [ - { - "id": "aliyun/qwen3-next-80b-a3b-thinking", - "name": "qwen3-next-80b-a3b-thinking", - "developer": "aliyun", - "evaluator_relationship": null, - "benchmark_scores": { - "livecodebenchpro/Hard Problems": 0.0, - "livecodebenchpro/Medium Problems": 0.0704, - "livecodebenchpro/Easy Problems": 0.6901 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/allenai.json b/data/developers/allenai.json deleted file mode 100644 index 057e47fa0ab57c7a4a183a0d516044f548cc0365..0000000000000000000000000000000000000000 --- a/data/developers/allenai.json +++ /dev/null @@ -1,2521 +0,0 @@ -{ - "developer": "allenai", - "models": [ - { - "id": "allenai/llama-3-tulu-2-70b-uf-mean-rm", - "name": "allenai/llama-3-tulu-2-70b-uf-mean-rm", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7019, - "reward-bench/Chat": 0.8631, - "reward-bench/Chat Hard": 0.5614, - "reward-bench/Safety": 0.6095, - "reward-bench/Reasoning": 0.8268, - "reward-bench/Prior Sets (0.5 weight)": 0.5957 - } - }, - { - "id": "allenai/llama-3-tulu-2-8b-uf-mean-rm", - "name": "allenai/llama-3-tulu-2-8b-uf-mean-rm", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7342, - "reward-bench/Chat": 0.9525, - "reward-bench/Chat Hard": 0.5921, - "reward-bench/Safety": 0.6162, - "reward-bench/Reasoning": 0.8212, - "reward-bench/Prior Sets (0.5 weight)": 0.6434 - } - }, - { - "id": "allenai/llama-3-tulu-2-dpo-70b", - "name": "allenai/llama-3-tulu-2-dpo-70b", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7496, - "reward-bench/Chat": 0.9637, - "reward-bench/Chat Hard": 0.5746, - "reward-bench/Safety": 0.7486, - "reward-bench/Reasoning": 0.802, - "reward-bench/Prior Sets (0.5 weight)": 0.5687 - } - }, - { - "id": "allenai/llama-3-tulu-2-dpo-8b", - "name": "allenai/llama-3-tulu-2-dpo-8b", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7275, - "reward-bench/Chat": 0.9525, - "reward-bench/Chat Hard": 0.5351, - "reward-bench/Safety": 0.6649, - "reward-bench/Reasoning": 0.8663, - "reward-bench/Prior Sets (0.5 weight)": 0.5097 - } - }, - { - "id": "allenai/Llama-3.1-70B-Instruct-RM-RB2", - "name": "allenai/Llama-3.1-70B-Instruct-RM-RB2", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.9021, - "reward-bench/Factuality": 0.8126, - "reward-bench/Precise IF": 0.4188, - "reward-bench/Math": 0.6995, - "reward-bench/Safety": 0.9095, - "reward-bench/Focus": 0.8646, - "reward-bench/Ties": 0.8835, - "reward-bench/Chat": 0.9665, - "reward-bench/Chat Hard": 0.8355, - "reward-bench/Reasoning": 0.8969, - "reward-bench/Prior Sets (0.5 weight)": 0.0 - } - }, - { - "id": "allenai/Llama-3.1-8B-Base-RM-RB2", - "name": "allenai/Llama-3.1-8B-Base-RM-RB2", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.649, - "reward-bench/Chat": 0.933, - "reward-bench/Chat Hard": 0.7785, - "reward-bench/Safety": 0.8267, - "reward-bench/Reasoning": 0.7886, - "reward-bench/Prior Sets (0.5 weight)": 0.0, - "reward-bench/Factuality": 0.72, - "reward-bench/Precise IF": 0.3625, - "reward-bench/Math": 0.612, - "reward-bench/Focus": 0.8323, - "reward-bench/Ties": 0.5406 - } - }, - { - "id": "allenai/Llama-3.1-8B-Instruct-RM-RB2", - "name": "allenai/Llama-3.1-8B-Instruct-RM-RB2", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.8885, - "reward-bench/Factuality": 0.7432, - "reward-bench/Precise IF": 0.4437, - "reward-bench/Math": 0.6175, - "reward-bench/Safety": 0.8932, - "reward-bench/Focus": 0.9071, - "reward-bench/Ties": 0.7638, - "reward-bench/Chat": 0.9581, - "reward-bench/Chat Hard": 0.8158, - "reward-bench/Reasoning": 0.887, - "reward-bench/Prior Sets (0.5 weight)": 0.0 - } - }, - { - "id": "allenai/Llama-3.1-Tulu-3-70B", - "name": "Llama-3.1-Tulu-3-70B", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8379, - "hfopenllm_v2/BBH": 0.6157, - "hfopenllm_v2/MATH Level 5": 0.3829, - "hfopenllm_v2/GPQA": 0.3733, - "hfopenllm_v2/MUSR": 0.4988, - "hfopenllm_v2/MMLU-PRO": 0.4656 - } - }, - { - "id": "allenai/Llama-3.1-Tulu-3-70B-DPO", - "name": "Llama-3.1-Tulu-3-70B-DPO", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8282, - "hfopenllm_v2/BBH": 0.6146, - "hfopenllm_v2/MATH Level 5": 0.4494, - "hfopenllm_v2/GPQA": 0.3758, - "hfopenllm_v2/MUSR": 0.4923, - "hfopenllm_v2/MMLU-PRO": 0.4633 - } - }, - { - "id": "allenai/Llama-3.1-Tulu-3-70B-SFT", - "name": "Llama-3.1-Tulu-3-70B-SFT", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8051, - "hfopenllm_v2/BBH": 0.5951, - "hfopenllm_v2/MATH Level 5": 0.3316, - "hfopenllm_v2/GPQA": 0.3448, - "hfopenllm_v2/MUSR": 0.5026, - "hfopenllm_v2/MMLU-PRO": 0.4624 - } - }, - { - "id": "allenai/Llama-3.1-Tulu-3-70B-SFT-RM-RB2", - "name": "allenai/Llama-3.1-Tulu-3-70B-SFT-RM-RB2", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.722, - "reward-bench/Chat": 0.9693, - "reward-bench/Chat Hard": 0.8268, - "reward-bench/Safety": 0.8689, - "reward-bench/Reasoning": 0.8583, - "reward-bench/Prior Sets (0.5 weight)": 0.0, - "reward-bench/Factuality": 0.8084, - "reward-bench/Precise IF": 0.3688, - "reward-bench/Math": 0.6776, - "reward-bench/Focus": 0.7778, - "reward-bench/Ties": 0.8308 - } - }, - { - "id": "allenai/Llama-3.1-Tulu-3-8B", - "name": "Llama-3.1-Tulu-3-8B", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8267, - "hfopenllm_v2/BBH": 0.405, - "hfopenllm_v2/MATH Level 5": 0.1964, - "hfopenllm_v2/GPQA": 0.2987, - "hfopenllm_v2/MUSR": 0.4175, - "hfopenllm_v2/MMLU-PRO": 0.2827 - } - }, - { - "id": "allenai/Llama-3.1-Tulu-3-8B-DPO", - "name": "Llama-3.1-Tulu-3-8B-DPO", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8029, - "hfopenllm_v2/BBH": 0.4079, - "hfopenllm_v2/MATH Level 5": 0.2364, - "hfopenllm_v2/GPQA": 0.2936, - "hfopenllm_v2/MUSR": 0.4161, - "hfopenllm_v2/MMLU-PRO": 0.2898 - } - }, - { - "id": "allenai/Llama-3.1-Tulu-3-8B-DPO-RM-RB2", - "name": "allenai/Llama-3.1-Tulu-3-8B-DPO-RM-RB2", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.8431, - "reward-bench/Factuality": 0.7516, - "reward-bench/Precise IF": 0.3875, - "reward-bench/Math": 0.6284, - "reward-bench/Safety": 0.8662, - "reward-bench/Focus": 0.8545, - "reward-bench/Ties": 0.6397, - "reward-bench/Chat": 0.9553, - "reward-bench/Chat Hard": 0.761, - "reward-bench/Reasoning": 0.7898, - "reward-bench/Prior Sets (0.5 weight)": 0.0 - } - }, - { - "id": "allenai/Llama-3.1-Tulu-3-8B-RL-RM-RB2", - "name": "allenai/Llama-3.1-Tulu-3-8B-RL-RM-RB2", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6871, - "reward-bench/Chat": 0.9469, - "reward-bench/Chat Hard": 0.7588, - "reward-bench/Safety": 0.8644, - "reward-bench/Reasoning": 0.7715, - "reward-bench/Prior Sets (0.5 weight)": 0.0, - "reward-bench/Factuality": 0.7642, - "reward-bench/Precise IF": 0.4, - "reward-bench/Math": 0.6175, - "reward-bench/Focus": 0.8485, - "reward-bench/Ties": 0.6281 - } - }, - { - "id": "allenai/Llama-3.1-Tulu-3-8B-RM", - "name": "Llama-3.1-Tulu-3-8B-RM", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.167, - "hfopenllm_v2/BBH": 0.295, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2567, - "hfopenllm_v2/MUSR": 0.3764, - "hfopenllm_v2/MMLU-PRO": 0.1082, - "reward-bench/Score": 0.59, - "reward-bench/Factuality": 0.7453, - "reward-bench/Precise IF": 0.3469, - "reward-bench/Math": 0.6448, - "reward-bench/Safety": 0.7422, - "reward-bench/Focus": 0.5364, - "reward-bench/Ties": 0.5243 - } - }, - { - "id": "allenai/Llama-3.1-Tulu-3-8B-SFT", - "name": "Llama-3.1-Tulu-3-8B-SFT", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7403, - "hfopenllm_v2/BBH": 0.3872, - "hfopenllm_v2/MATH Level 5": 0.1178, - "hfopenllm_v2/GPQA": 0.2777, - "hfopenllm_v2/MUSR": 0.4268, - "hfopenllm_v2/MMLU-PRO": 0.2812 - } - }, - { - "id": "allenai/Llama-3.1-Tulu-3-8B-SFT-RM-RB2", - "name": "allenai/Llama-3.1-Tulu-3-8B-SFT-RM-RB2", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6821, - "reward-bench/Chat": 0.9497, - "reward-bench/Chat Hard": 0.7917, - "reward-bench/Safety": 0.8978, - "reward-bench/Reasoning": 0.8005, - "reward-bench/Prior Sets (0.5 weight)": 0.0, - "reward-bench/Factuality": 0.7326, - "reward-bench/Precise IF": 0.3875, - "reward-bench/Math": 0.5792, - "reward-bench/Focus": 0.8889, - "reward-bench/Ties": 0.6063 - } - }, - { - "id": "allenai/olmo-1.7-7b", - "name": "OLMo 1.7 7B", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_mmlu/MMLU All Subjects": 0.538, - "helm_mmlu/Abstract Algebra": 0.33, - "helm_mmlu/Anatomy": 0.496, - "helm_mmlu/College Physics": 0.333, - "helm_mmlu/Computer Security": 0.65, - "helm_mmlu/Econometrics": 0.404, - "helm_mmlu/Global Facts": 0.34, - "helm_mmlu/Jurisprudence": 0.565, - "helm_mmlu/Philosophy": 0.592, - "helm_mmlu/Professional Psychology": 0.526, - "helm_mmlu/Us Foreign Policy": 0.76, - "helm_mmlu/Astronomy": 0.526, - "helm_mmlu/Business Ethics": 0.59, - "helm_mmlu/Clinical Knowledge": 0.57, - "helm_mmlu/Conceptual Physics": 0.434, - "helm_mmlu/Electrical Engineering": 0.517, - "helm_mmlu/Elementary Mathematics": 0.307, - "helm_mmlu/Formal Logic": 0.325, - "helm_mmlu/High School World History": 0.713, - "helm_mmlu/Human Sexuality": 0.595, - "helm_mmlu/International Law": 0.612, - "helm_mmlu/Logical Fallacies": 0.607, - "helm_mmlu/Machine Learning": 0.375, - "helm_mmlu/Management": 0.689, - "helm_mmlu/Marketing": 0.769, - "helm_mmlu/Medical Genetics": 0.56, - "helm_mmlu/Miscellaneous": 0.734, - "helm_mmlu/Moral Scenarios": 0.335, - "helm_mmlu/Nutrition": 0.608, - "helm_mmlu/Prehistory": 0.593, - "helm_mmlu/Public Relations": 0.6, - "helm_mmlu/Security Studies": 0.522, - "helm_mmlu/Sociology": 0.751, - "helm_mmlu/Virology": 0.452, - "helm_mmlu/World Religions": 0.731, - "helm_mmlu/Mean win rate": 0.196 - } - }, - { - "id": "allenai/OLMo-1.7-7B-hf", - "name": "OLMo-1.7-7B-hf", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1569, - "hfopenllm_v2/BBH": 0.3014, - "hfopenllm_v2/MATH Level 5": 0.0023, - "hfopenllm_v2/GPQA": 0.255, - "hfopenllm_v2/MUSR": 0.3475, - "hfopenllm_v2/MMLU-PRO": 0.1124 - } - }, - { - "id": "allenai/OLMo-1B-hf", - "name": "OLMo-1B-hf", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2182, - "hfopenllm_v2/BBH": 0.3052, - "hfopenllm_v2/MATH Level 5": 0.0174, - "hfopenllm_v2/GPQA": 0.2617, - "hfopenllm_v2/MUSR": 0.4098, - "hfopenllm_v2/MMLU-PRO": 0.1174 - } - }, - { - "id": "allenai/olmo-2-0325-32b-instruct", - "name": "OLMo 2 32B Instruct March 2025", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.475, - "helm_capabilities/MMLU-Pro": 0.414, - "helm_capabilities/GPQA": 0.287, - "helm_capabilities/IFEval": 0.78, - "helm_capabilities/WildBench": 0.734, - "helm_capabilities/Omni-MATH": 0.161 - } - }, - { - "id": "allenai/olmo-2-1124-13b-instruct", - "name": "OLMo 2 13B Instruct November 2024", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.44, - "helm_capabilities/MMLU-Pro": 0.31, - "helm_capabilities/GPQA": 0.316, - "helm_capabilities/IFEval": 0.73, - "helm_capabilities/WildBench": 0.689, - "helm_capabilities/Omni-MATH": 0.156 - } - }, - { - "id": "allenai/OLMo-2-1124-7B-Instruct", - "name": "OLMo 2 7B Instruct November 2024", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.405, - "helm_capabilities/MMLU-Pro": 0.292, - "helm_capabilities/GPQA": 0.296, - "helm_capabilities/IFEval": 0.693, - "helm_capabilities/WildBench": 0.628, - "helm_capabilities/Omni-MATH": 0.116, - "hfopenllm_v2/IFEval": 0.7244, - "hfopenllm_v2/BBH": 0.4022, - "hfopenllm_v2/MATH Level 5": 0.1488, - "hfopenllm_v2/GPQA": 0.2785, - "hfopenllm_v2/MUSR": 0.3508, - "hfopenllm_v2/MMLU-PRO": 0.2672 - } - }, - { - "id": "allenai/olmo-7b", - "name": "OLMo 7B", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.052, - "helm_lite/NarrativeQA": 0.597, - "helm_lite/NaturalQuestions (closed-book)": 0.259, - "helm_lite/OpenbookQA": 0.222, - "helm_lite/MMLU": 0.305, - "helm_lite/MATH": 0.029, - "helm_lite/GSM8K": 0.044, - "helm_lite/LegalBench": 0.341, - "helm_lite/MedQA": 0.229, - "helm_lite/WMT 2014": 0.097, - "helm_mmlu/MMLU All Subjects": 0.295, - "helm_mmlu/Abstract Algebra": 0.26, - "helm_mmlu/Anatomy": 0.222, - "helm_mmlu/College Physics": 0.294, - "helm_mmlu/Computer Security": 0.3, - "helm_mmlu/Econometrics": 0.325, - "helm_mmlu/Global Facts": 0.32, - "helm_mmlu/Jurisprudence": 0.25, - "helm_mmlu/Philosophy": 0.325, - "helm_mmlu/Professional Psychology": 0.232, - "helm_mmlu/Us Foreign Policy": 0.26, - "helm_mmlu/Astronomy": 0.342, - "helm_mmlu/Business Ethics": 0.24, - "helm_mmlu/Clinical Knowledge": 0.26, - "helm_mmlu/Conceptual Physics": 0.319, - "helm_mmlu/Electrical Engineering": 0.29, - "helm_mmlu/Elementary Mathematics": 0.254, - "helm_mmlu/Formal Logic": 0.278, - "helm_mmlu/High School World History": 0.253, - "helm_mmlu/Human Sexuality": 0.267, - "helm_mmlu/International Law": 0.306, - "helm_mmlu/Logical Fallacies": 0.264, - "helm_mmlu/Machine Learning": 0.286, - "helm_mmlu/Management": 0.272, - "helm_mmlu/Marketing": 0.269, - "helm_mmlu/Medical Genetics": 0.28, - "helm_mmlu/Miscellaneous": 0.292, - "helm_mmlu/Moral Scenarios": 0.265, - "helm_mmlu/Nutrition": 0.34, - "helm_mmlu/Prehistory": 0.318, - "helm_mmlu/Public Relations": 0.345, - "helm_mmlu/Security Studies": 0.408, - "helm_mmlu/Sociology": 0.383, - "helm_mmlu/Virology": 0.416, - "helm_mmlu/World Religions": 0.234, - "helm_mmlu/Mean win rate": 0.68 - } - }, - { - "id": "allenai/OLMo-7B-hf", - "name": "OLMo-7B-hf", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2719, - "hfopenllm_v2/BBH": 0.3279, - "hfopenllm_v2/MATH Level 5": 0.0121, - "hfopenllm_v2/GPQA": 0.2727, - "hfopenllm_v2/MUSR": 0.3487, - "hfopenllm_v2/MMLU-PRO": 0.1173 - } - }, - { - "id": "allenai/OLMo-7B-Instruct", - "name": "allenai/OLMo-7B-Instruct", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6727, - "reward-bench/Chat": 0.8966, - "reward-bench/Chat Hard": 0.5066, - "reward-bench/Safety": 0.6486, - "reward-bench/Reasoning": 0.7168, - "reward-bench/Prior Sets (0.5 weight)": 0.5173 - } - }, - { - "id": "allenai/OLMo-7B-Instruct-hf", - "name": "OLMo-7B-Instruct-hf", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3473, - "hfopenllm_v2/BBH": 0.3706, - "hfopenllm_v2/MATH Level 5": 0.0136, - "hfopenllm_v2/GPQA": 0.271, - "hfopenllm_v2/MUSR": 0.3765, - "hfopenllm_v2/MMLU-PRO": 0.1785 - } - }, - { - "id": "allenai/OLMoE-1B-7B-0125-Instruct", - "name": "OLMoE 1B-7B Instruct January 2025", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.332, - "helm_capabilities/MMLU-Pro": 0.169, - "helm_capabilities/GPQA": 0.22, - "helm_capabilities/IFEval": 0.628, - "helm_capabilities/WildBench": 0.551, - "helm_capabilities/Omni-MATH": 0.093, - "hfopenllm_v2/IFEval": 0.6757, - "hfopenllm_v2/BBH": 0.3825, - "hfopenllm_v2/MATH Level 5": 0.0899, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3636, - "hfopenllm_v2/MMLU-PRO": 0.1915 - } - }, - { - "id": "allenai/OLMoE-1B-7B-0924", - "name": "OLMoE-1B-7B-0924", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2185, - "hfopenllm_v2/BBH": 0.3393, - "hfopenllm_v2/MATH Level 5": 0.0166, - "hfopenllm_v2/GPQA": 0.2475, - "hfopenllm_v2/MUSR": 0.3488, - "hfopenllm_v2/MMLU-PRO": 0.174 - } - }, - { - "id": "allenai/OLMoE-1B-7B-0924-Instruct", - "name": "OLMoE-1B-7B-0924-Instruct", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4667, - "hfopenllm_v2/BBH": 0.3902, - "hfopenllm_v2/MATH Level 5": 0.0279, - "hfopenllm_v2/GPQA": 0.2676, - "hfopenllm_v2/MUSR": 0.3848, - "hfopenllm_v2/MMLU-PRO": 0.1876 - } - }, - { - "id": "allenai/open_instruct_dev-reward_modeling__1__1739590997", - "name": "allenai/open_instruct_dev-reward_modeling__1__1739590997", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6004, - "reward-bench/Factuality": 0.7032, - "reward-bench/Precise IF": 0.375, - "reward-bench/Math": 0.623, - "reward-bench/Safety": 0.7867, - "reward-bench/Focus": 0.598, - "reward-bench/Ties": 0.5165 - } - }, - { - "id": "allenai/open_instruct_dev-reward_modeling__1__1739871066", - "name": "allenai/open_instruct_dev-reward_modeling__1__1739871066", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6012, - "reward-bench/Factuality": 0.6989, - "reward-bench/Precise IF": 0.425, - "reward-bench/Math": 0.6284, - "reward-bench/Safety": 0.7978, - "reward-bench/Focus": 0.604, - "reward-bench/Ties": 0.4527 - } - }, - { - "id": "allenai/open_instruct_dev-reward_modeling__1__1739925892", - "name": "allenai/open_instruct_dev-reward_modeling__1__1739925892", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6345, - "reward-bench/Factuality": 0.7432, - "reward-bench/Precise IF": 0.3563, - "reward-bench/Math": 0.623, - "reward-bench/Safety": 0.8111, - "reward-bench/Focus": 0.7131, - "reward-bench/Ties": 0.5606 - } - }, - { - "id": "allenai/open_instruct_dev-reward_modeling__1__1739943850", - "name": "allenai/open_instruct_dev-reward_modeling__1__1739943850", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.4978, - "reward-bench/Factuality": 0.5726, - "reward-bench/Precise IF": 0.3125, - "reward-bench/Math": 0.5191, - "reward-bench/Safety": 0.6489, - "reward-bench/Focus": 0.6222, - "reward-bench/Ties": 0.3114 - } - }, - { - "id": "allenai/open_instruct_dev-reward_modeling__1__1739943881", - "name": "allenai/open_instruct_dev-reward_modeling__1__1739943881", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5998, - "reward-bench/Factuality": 0.7032, - "reward-bench/Precise IF": 0.3187, - "reward-bench/Math": 0.5792, - "reward-bench/Safety": 0.8222, - "reward-bench/Focus": 0.6727, - "reward-bench/Ties": 0.5025 - } - }, - { - "id": "allenai/open_instruct_dev-reward_modeling__1__1739943972", - "name": "allenai/open_instruct_dev-reward_modeling__1__1739943972", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5289, - "reward-bench/Factuality": 0.6168, - "reward-bench/Precise IF": 0.375, - "reward-bench/Math": 0.5738, - "reward-bench/Safety": 0.6844, - "reward-bench/Focus": 0.5657, - "reward-bench/Ties": 0.3577 - } - }, - { - "id": "allenai/open_instruct_dev-reward_modeling__1__1739957701", - "name": "allenai/open_instruct_dev-reward_modeling__1__1739957701", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6194, - "reward-bench/Factuality": 0.6779, - "reward-bench/Precise IF": 0.3563, - "reward-bench/Math": 0.6011, - "reward-bench/Safety": 0.8022, - "reward-bench/Focus": 0.697, - "reward-bench/Ties": 0.5822 - } - }, - { - "id": "allenai/open_instruct_dev-reward_modeling__1__1739971507", - "name": "allenai/open_instruct_dev-reward_modeling__1__1739971507", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5717, - "reward-bench/Factuality": 0.68, - "reward-bench/Precise IF": 0.375, - "reward-bench/Math": 0.6066, - "reward-bench/Safety": 0.7667, - "reward-bench/Focus": 0.5475, - "reward-bench/Ties": 0.4545 - } - }, - { - "id": "allenai/open_instruct_dev-reward_modeling__1__1739971529", - "name": "allenai/open_instruct_dev-reward_modeling__1__1739971529", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5564, - "reward-bench/Factuality": 0.6568, - "reward-bench/Precise IF": 0.3563, - "reward-bench/Math": 0.5956, - "reward-bench/Safety": 0.7533, - "reward-bench/Focus": 0.5737, - "reward-bench/Ties": 0.4027 - } - }, - { - "id": "allenai/open_instruct_dev-reward_modeling__1__1739998765", - "name": "allenai/open_instruct_dev-reward_modeling__1__1739998765", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6008, - "reward-bench/Factuality": 0.7095, - "reward-bench/Precise IF": 0.4125, - "reward-bench/Math": 0.6066, - "reward-bench/Safety": 0.8022, - "reward-bench/Focus": 0.5859, - "reward-bench/Ties": 0.4883 - } - }, - { - "id": "allenai/open_instruct_dev-reward_modeling__1__1740005072", - "name": "allenai/open_instruct_dev-reward_modeling__1__1740005072", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6097, - "reward-bench/Factuality": 0.7137, - "reward-bench/Precise IF": 0.3937, - "reward-bench/Math": 0.6339, - "reward-bench/Safety": 0.7778, - "reward-bench/Focus": 0.6343, - "reward-bench/Ties": 0.5047 - } - }, - { - "id": "allenai/open_instruct_dev-reward_modeling__1__1740129284", - "name": "allenai/open_instruct_dev-reward_modeling__1__1740129284", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6129, - "reward-bench/Factuality": 0.7116, - "reward-bench/Precise IF": 0.4437, - "reward-bench/Math": 0.6448, - "reward-bench/Safety": 0.8022, - "reward-bench/Focus": 0.6101, - "reward-bench/Ties": 0.4652 - } - }, - { - "id": "allenai/open_instruct_dev-reward_modeling__1__1741286813", - "name": "allenai/open_instruct_dev-reward_modeling__1__1741286813", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6557, - "reward-bench/Factuality": 0.6295, - "reward-bench/Precise IF": 0.4188, - "reward-bench/Math": 0.612, - "reward-bench/Safety": 0.9111, - "reward-bench/Focus": 0.8263, - "reward-bench/Ties": 0.5365 - } - }, - { - "id": "allenai/open_instruct_dev-reward_modeling__1__1741287363", - "name": "allenai/open_instruct_dev-reward_modeling__1__1741287363", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6672, - "reward-bench/Factuality": 0.6295, - "reward-bench/Precise IF": 0.375, - "reward-bench/Math": 0.6066, - "reward-bench/Safety": 0.88, - "reward-bench/Focus": 0.9374, - "reward-bench/Ties": 0.5748 - } - }, - { - "id": "allenai/open_instruct_dev-reward_modeling__1__1741292911", - "name": "allenai/open_instruct_dev-reward_modeling__1__1741292911", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6607, - "reward-bench/Factuality": 0.6589, - "reward-bench/Precise IF": 0.4, - "reward-bench/Math": 0.6066, - "reward-bench/Safety": 0.9089, - "reward-bench/Focus": 0.8869, - "reward-bench/Ties": 0.5028 - } - }, - { - "id": "allenai/open_instruct_dev-reward_modeling__1__1742338142", - "name": "allenai/open_instruct_dev-reward_modeling__1__1742338142", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6344, - "reward-bench/Factuality": 0.7326, - "reward-bench/Precise IF": 0.3812, - "reward-bench/Math": 0.7049, - "reward-bench/Safety": 0.88, - "reward-bench/Focus": 0.6323, - "reward-bench/Ties": 0.475 - } - }, - { - "id": "allenai/open_instruct_dev-reward_modeling__1__1742519610", - "name": "allenai/open_instruct_dev-reward_modeling__1__1742519610", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6361, - "reward-bench/Factuality": 0.7074, - "reward-bench/Precise IF": 0.3812, - "reward-bench/Math": 0.6721, - "reward-bench/Safety": 0.82, - "reward-bench/Focus": 0.6444, - "reward-bench/Ties": 0.5915 - } - }, - { - "id": "allenai/open_instruct_dev-reward_modeling__1__1742519628", - "name": "allenai/open_instruct_dev-reward_modeling__1__1742519628", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5609, - "reward-bench/Factuality": 0.5179, - "reward-bench/Precise IF": 0.3563, - "reward-bench/Math": 0.623, - "reward-bench/Safety": 0.8356, - "reward-bench/Focus": 0.5071, - "reward-bench/Ties": 0.5254 - } - }, - { - "id": "allenai/open_instruct_dev-rm_1e-6_1_100pctflipped__1__1744241455", - "name": "allenai/open_instruct_dev-rm_1e-6_1_100pctflipped__1__1744241455", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.0576, - "reward-bench/Factuality": 0.04, - "reward-bench/Precise IF": 0.1313, - "reward-bench/Math": 0.0546, - "reward-bench/Safety": 0.0489, - "reward-bench/Focus": 0.0808, - "reward-bench/Ties": -0.01 - } - }, - { - "id": "allenai/open_instruct_dev-rm_1e-6_1_10pctflipped__1__1743295511", - "name": "allenai/open_instruct_dev-rm_1e-6_1_10pctflipped__1__1743295511", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5499, - "reward-bench/Factuality": 0.6821, - "reward-bench/Precise IF": 0.3937, - "reward-bench/Math": 0.5956, - "reward-bench/Safety": 0.7356, - "reward-bench/Focus": 0.5212, - "reward-bench/Ties": 0.3711 - } - }, - { - "id": "allenai/open_instruct_dev-rm_1e-6_1_20pctflipped__1__1743295406", - "name": "allenai/open_instruct_dev-rm_1e-6_1_20pctflipped__1__1743295406", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5054, - "reward-bench/Factuality": 0.6358, - "reward-bench/Precise IF": 0.3688, - "reward-bench/Math": 0.6066, - "reward-bench/Safety": 0.6867, - "reward-bench/Focus": 0.4424, - "reward-bench/Ties": 0.2922 - } - }, - { - "id": "allenai/open_instruct_dev-rm_1e-6_1_30pctflipped__1__1743325136", - "name": "allenai/open_instruct_dev-rm_1e-6_1_30pctflipped__1__1743325136", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.478, - "reward-bench/Factuality": 0.6442, - "reward-bench/Precise IF": 0.3563, - "reward-bench/Math": 0.612, - "reward-bench/Safety": 0.6356, - "reward-bench/Focus": 0.2707, - "reward-bench/Ties": 0.3496 - } - }, - { - "id": "allenai/open_instruct_dev-rm_1e-6_1_50pctflipped__1__1744241398", - "name": "allenai/open_instruct_dev-rm_1e-6_1_50pctflipped__1__1744241398", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.219, - "reward-bench/Factuality": 0.2484, - "reward-bench/Precise IF": 0.2812, - "reward-bench/Math": 0.2623, - "reward-bench/Safety": 0.3422, - "reward-bench/Focus": 0.1717, - "reward-bench/Ties": 0.008 - } - }, - { - "id": "allenai/open_instruct_dev-rm_1e-6_1_5pctflipped__1__1743444535", - "name": "allenai/open_instruct_dev-rm_1e-6_1_5pctflipped__1__1743444535", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5625, - "reward-bench/Factuality": 0.6821, - "reward-bench/Precise IF": 0.4062, - "reward-bench/Math": 0.6011, - "reward-bench/Safety": 0.7511, - "reward-bench/Focus": 0.5313, - "reward-bench/Ties": 0.403 - } - }, - { - "id": "allenai/open_instruct_dev-rm_1e-6_1_dpo__1__1743550054", - "name": "allenai/open_instruct_dev-rm_1e-6_1_dpo__1__1743550054", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5759, - "reward-bench/Factuality": 0.7074, - "reward-bench/Precise IF": 0.375, - "reward-bench/Math": 0.623, - "reward-bench/Safety": 0.7578, - "reward-bench/Focus": 0.5333, - "reward-bench/Ties": 0.459 - } - }, - { - "id": "allenai/open_instruct_dev-rm_1e-6_1_dpo_skyworks__1__1744530271", - "name": "allenai/open_instruct_dev-rm_1e-6_1_dpo_skyworks__1__1744530271", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6057, - "reward-bench/Factuality": 0.5053, - "reward-bench/Precise IF": 0.375, - "reward-bench/Math": 0.5902, - "reward-bench/Safety": 0.8422, - "reward-bench/Focus": 0.7798, - "reward-bench/Ties": 0.5419 - } - }, - { - "id": "allenai/open_instruct_dev-rm_1e-6_1_dpo_skyworkstulufull__1__1743550181", - "name": "allenai/open_instruct_dev-rm_1e-6_1_dpo_skyworkstulufull__1__1743550181", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6535, - "reward-bench/Factuality": 0.7137, - "reward-bench/Precise IF": 0.3812, - "reward-bench/Math": 0.6175, - "reward-bench/Safety": 0.8244, - "reward-bench/Focus": 0.7737, - "reward-bench/Ties": 0.6101 - } - }, - { - "id": "allenai/open_instruct_dev-rm_1e-6_1_rl__1__1743551221", - "name": "allenai/open_instruct_dev-rm_1e-6_1_rl__1__1743551221", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5799, - "reward-bench/Factuality": 0.7116, - "reward-bench/Precise IF": 0.3812, - "reward-bench/Math": 0.6284, - "reward-bench/Safety": 0.76, - "reward-bench/Focus": 0.5374, - "reward-bench/Ties": 0.461 - } - }, - { - "id": "allenai/open_instruct_dev-rm_1e-6_1_rl_skyworks__1__1744530262", - "name": "allenai/open_instruct_dev-rm_1e-6_1_rl_skyworks__1__1744530262", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5903, - "reward-bench/Factuality": 0.4863, - "reward-bench/Precise IF": 0.3625, - "reward-bench/Math": 0.5738, - "reward-bench/Safety": 0.8489, - "reward-bench/Focus": 0.7778, - "reward-bench/Ties": 0.4926 - } - }, - { - "id": "allenai/open_instruct_dev-rm_1e-6_1_rl_skyworkstulufull__1__1743551523", - "name": "allenai/open_instruct_dev-rm_1e-6_1_rl_skyworkstulufull__1__1743551523", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6483, - "reward-bench/Factuality": 0.7074, - "reward-bench/Precise IF": 0.3625, - "reward-bench/Math": 0.6175, - "reward-bench/Safety": 0.8222, - "reward-bench/Focus": 0.7758, - "reward-bench/Ties": 0.6044 - } - }, - { - "id": "allenai/open_instruct_dev-rm_1e-6_1_skyworkstulumix__1__1743205750", - "name": "allenai/open_instruct_dev-rm_1e-6_1_skyworkstulumix__1__1743205750", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5157, - "reward-bench/Factuality": 0.6084, - "reward-bench/Precise IF": 0.3688, - "reward-bench/Math": 0.6066, - "reward-bench/Safety": 0.7089, - "reward-bench/Focus": 0.4222, - "reward-bench/Ties": 0.3791 - } - }, - { - "id": "allenai/open_instruct_dev-rm_1e-6_2_10pctflipped__1__1743295427", - "name": "allenai/open_instruct_dev-rm_1e-6_2_10pctflipped__1__1743295427", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6009, - "reward-bench/Factuality": 0.7263, - "reward-bench/Precise IF": 0.375, - "reward-bench/Math": 0.5902, - "reward-bench/Safety": 0.7933, - "reward-bench/Focus": 0.7273, - "reward-bench/Ties": 0.3931 - } - }, - { - "id": "allenai/open_instruct_dev-rm_1e-6_2_20pctflipped__1__1743295446", - "name": "allenai/open_instruct_dev-rm_1e-6_2_20pctflipped__1__1743295446", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5716, - "reward-bench/Factuality": 0.6779, - "reward-bench/Precise IF": 0.3937, - "reward-bench/Math": 0.5464, - "reward-bench/Safety": 0.7533, - "reward-bench/Focus": 0.7051, - "reward-bench/Ties": 0.3534 - } - }, - { - "id": "allenai/open_instruct_dev-rm_1e-6_2_30pctflipped__1__1743325094", - "name": "allenai/open_instruct_dev-rm_1e-6_2_30pctflipped__1__1743325094", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5151, - "reward-bench/Factuality": 0.6484, - "reward-bench/Precise IF": 0.3312, - "reward-bench/Math": 0.5574, - "reward-bench/Safety": 0.7289, - "reward-bench/Focus": 0.4889, - "reward-bench/Ties": 0.3357 - } - }, - { - "id": "allenai/open_instruct_dev-rm_1e-6_2_5pctflipped__1__1743444636", - "name": "allenai/open_instruct_dev-rm_1e-6_2_5pctflipped__1__1743444636", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6119, - "reward-bench/Factuality": 0.72, - "reward-bench/Precise IF": 0.4062, - "reward-bench/Math": 0.6284, - "reward-bench/Safety": 0.8067, - "reward-bench/Focus": 0.6889, - "reward-bench/Ties": 0.421 - } - }, - { - "id": "allenai/open_instruct_dev-rm_1e-6_2_dpo__1__1743549325", - "name": "allenai/open_instruct_dev-rm_1e-6_2_dpo__1__1743549325", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6008, - "reward-bench/Factuality": 0.7179, - "reward-bench/Precise IF": 0.35, - "reward-bench/Math": 0.5956, - "reward-bench/Safety": 0.8, - "reward-bench/Focus": 0.6707, - "reward-bench/Ties": 0.4707 - } - }, - { - "id": "allenai/open_instruct_dev-rm_1e-6_2_rl__1__1743551238", - "name": "allenai/open_instruct_dev-rm_1e-6_2_rl__1__1743551238", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5965, - "reward-bench/Factuality": 0.7095, - "reward-bench/Precise IF": 0.3438, - "reward-bench/Math": 0.612, - "reward-bench/Safety": 0.8044, - "reward-bench/Focus": 0.6566, - "reward-bench/Ties": 0.453 - } - }, - { - "id": "allenai/open_instruct_dev-rm_1e-6_2_skyworkstulumix__1__1743205906", - "name": "allenai/open_instruct_dev-rm_1e-6_2_skyworkstulumix__1__1743205906", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5574, - "reward-bench/Factuality": 0.6526, - "reward-bench/Precise IF": 0.3937, - "reward-bench/Math": 0.6011, - "reward-bench/Safety": 0.7711, - "reward-bench/Focus": 0.5051, - "reward-bench/Ties": 0.4208 - } - }, - { - "id": "allenai/open_instruct_dev-rm_2e-5_1_100pctflipped__1__1744241529", - "name": "allenai/open_instruct_dev-rm_2e-5_1_100pctflipped__1__1744241529", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.0719, - "reward-bench/Factuality": 0.0421, - "reward-bench/Precise IF": 0.2062, - "reward-bench/Math": 0.0601, - "reward-bench/Safety": 0.0378, - "reward-bench/Focus": 0.0949, - "reward-bench/Ties": -0.01 - } - }, - { - "id": "allenai/open_instruct_dev-rm_2e-5_1_10pctflipped__1__1743295305", - "name": "allenai/open_instruct_dev-rm_2e-5_1_10pctflipped__1__1743295305", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.553, - "reward-bench/Factuality": 0.6674, - "reward-bench/Precise IF": 0.3563, - "reward-bench/Math": 0.6284, - "reward-bench/Safety": 0.6733, - "reward-bench/Focus": 0.5697, - "reward-bench/Ties": 0.4227 - } - }, - { - "id": "allenai/open_instruct_dev-rm_2e-5_1_20pctflipped__1__1743324778", - "name": "allenai/open_instruct_dev-rm_2e-5_1_20pctflipped__1__1743324778", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.4955, - "reward-bench/Factuality": 0.6189, - "reward-bench/Precise IF": 0.325, - "reward-bench/Math": 0.5792, - "reward-bench/Safety": 0.6378, - "reward-bench/Focus": 0.5657, - "reward-bench/Ties": 0.2466 - } - }, - { - "id": "allenai/open_instruct_dev-rm_2e-5_1_30pctflipped__1__1743326459", - "name": "allenai/open_instruct_dev-rm_2e-5_1_30pctflipped__1__1743326459", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.4198, - "reward-bench/Factuality": 0.5747, - "reward-bench/Precise IF": 0.3375, - "reward-bench/Math": 0.5464, - "reward-bench/Safety": 0.4933, - "reward-bench/Focus": 0.3596, - "reward-bench/Ties": 0.2073 - } - }, - { - "id": "allenai/open_instruct_dev-rm_2e-5_1_5pctflipped__1__1743443747", - "name": "allenai/open_instruct_dev-rm_2e-5_1_5pctflipped__1__1743443747", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5465, - "reward-bench/Factuality": 0.6821, - "reward-bench/Precise IF": 0.375, - "reward-bench/Math": 0.612, - "reward-bench/Safety": 0.7333, - "reward-bench/Focus": 0.5051, - "reward-bench/Ties": 0.3713 - } - }, - { - "id": "allenai/open_instruct_dev-rm_2e-5_1_skyworkstulumix__1__1743205935", - "name": "allenai/open_instruct_dev-rm_2e-5_1_skyworkstulumix__1__1743205935", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5197, - "reward-bench/Factuality": 0.6126, - "reward-bench/Precise IF": 0.3375, - "reward-bench/Math": 0.5847, - "reward-bench/Safety": 0.7333, - "reward-bench/Focus": 0.4646, - "reward-bench/Ties": 0.3855 - } - }, - { - "id": "allenai/open_instruct_dev-rm_2e-5_2_10pctflipped__1__1743295360", - "name": "allenai/open_instruct_dev-rm_2e-5_2_10pctflipped__1__1743295360", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.4555, - "reward-bench/Factuality": 0.5495, - "reward-bench/Precise IF": 0.3063, - "reward-bench/Math": 0.4262, - "reward-bench/Safety": 0.5711, - "reward-bench/Focus": 0.6101, - "reward-bench/Ties": 0.2696 - } - }, - { - "id": "allenai/open_instruct_dev-rm_2e-5_2_20pctflipped__1__1743295366", - "name": "allenai/open_instruct_dev-rm_2e-5_2_20pctflipped__1__1743295366", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.4422, - "reward-bench/Factuality": 0.5053, - "reward-bench/Precise IF": 0.3375, - "reward-bench/Math": 0.4044, - "reward-bench/Safety": 0.5422, - "reward-bench/Focus": 0.6646, - "reward-bench/Ties": 0.1991 - } - }, - { - "id": "allenai/open_instruct_dev-rm_2e-5_2_30pctflipped__1__1743326352", - "name": "allenai/open_instruct_dev-rm_2e-5_2_30pctflipped__1__1743326352", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.341, - "reward-bench/Factuality": 0.4674, - "reward-bench/Precise IF": 0.2875, - "reward-bench/Math": 0.3333, - "reward-bench/Safety": 0.3711, - "reward-bench/Focus": 0.3919, - "reward-bench/Ties": 0.195 - } - }, - { - "id": "allenai/open_instruct_dev-rm_2e-5_2_5pctflipped__1__1743444634", - "name": "allenai/open_instruct_dev-rm_2e-5_2_5pctflipped__1__1743444634", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.4698, - "reward-bench/Factuality": 0.5853, - "reward-bench/Precise IF": 0.2562, - "reward-bench/Math": 0.5027, - "reward-bench/Safety": 0.6489, - "reward-bench/Focus": 0.5697, - "reward-bench/Ties": 0.2562 - } - }, - { - "id": "allenai/open_instruct_dev-rm_2e-5_2_skyworkstulumix__1__1743205988", - "name": "allenai/open_instruct_dev-rm_2e-5_2_skyworkstulumix__1__1743205988", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.4791, - "reward-bench/Factuality": 0.6421, - "reward-bench/Precise IF": 0.3125, - "reward-bench/Math": 0.541, - "reward-bench/Safety": 0.6911, - "reward-bench/Focus": 0.4182, - "reward-bench/Ties": 0.27 - } - }, - { - "id": "allenai/open_instruct_dev-rm_3e-6_1_100pctflipped__1__1744242103", - "name": "allenai/open_instruct_dev-rm_3e-6_1_100pctflipped__1__1744242103", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.0607, - "reward-bench/Factuality": 0.0274, - "reward-bench/Precise IF": 0.1625, - "reward-bench/Math": 0.0656, - "reward-bench/Safety": 0.04, - "reward-bench/Focus": 0.0788, - "reward-bench/Ties": -0.01 - } - }, - { - "id": "allenai/open_instruct_dev-rm_3e-6_1_10pctflipped__1__1743324835", - "name": "allenai/open_instruct_dev-rm_3e-6_1_10pctflipped__1__1743324835", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6089, - "reward-bench/Factuality": 0.7284, - "reward-bench/Precise IF": 0.4375, - "reward-bench/Math": 0.612, - "reward-bench/Safety": 0.7622, - "reward-bench/Focus": 0.6444, - "reward-bench/Ties": 0.4686 - } - }, - { - "id": "allenai/open_instruct_dev-rm_3e-6_1_1pctflipped__1__1743445221", - "name": "allenai/open_instruct_dev-rm_3e-6_1_1pctflipped__1__1743445221", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6032, - "reward-bench/Factuality": 0.7158, - "reward-bench/Precise IF": 0.4062, - "reward-bench/Math": 0.6284, - "reward-bench/Safety": 0.7778, - "reward-bench/Focus": 0.5859, - "reward-bench/Ties": 0.5051 - } - }, - { - "id": "allenai/open_instruct_dev-rm_3e-6_1_20pctflipped__1__1743324826", - "name": "allenai/open_instruct_dev-rm_3e-6_1_20pctflipped__1__1743324826", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5831, - "reward-bench/Factuality": 0.6947, - "reward-bench/Precise IF": 0.4188, - "reward-bench/Math": 0.623, - "reward-bench/Safety": 0.74, - "reward-bench/Focus": 0.5758, - "reward-bench/Ties": 0.4465 - } - }, - { - "id": "allenai/open_instruct_dev-rm_3e-6_1_30pctflipped__1__1743326363", - "name": "allenai/open_instruct_dev-rm_3e-6_1_30pctflipped__1__1743326363", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5268, - "reward-bench/Factuality": 0.68, - "reward-bench/Precise IF": 0.3688, - "reward-bench/Math": 0.5792, - "reward-bench/Safety": 0.7178, - "reward-bench/Focus": 0.4343, - "reward-bench/Ties": 0.3809 - } - }, - { - "id": "allenai/open_instruct_dev-rm_3e-6_1_5pctflipped__1__1743444498", - "name": "allenai/open_instruct_dev-rm_3e-6_1_5pctflipped__1__1743444498", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6093, - "reward-bench/Factuality": 0.7326, - "reward-bench/Precise IF": 0.4313, - "reward-bench/Math": 0.6339, - "reward-bench/Safety": 0.7578, - "reward-bench/Focus": 0.5859, - "reward-bench/Ties": 0.5143 - } - }, - { - "id": "allenai/open_instruct_dev-rm_3e-6_1__2__1743897475", - "name": "allenai/open_instruct_dev-rm_3e-6_1__2__1743897475", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6122, - "reward-bench/Factuality": 0.7368, - "reward-bench/Precise IF": 0.4, - "reward-bench/Math": 0.623, - "reward-bench/Safety": 0.8044, - "reward-bench/Focus": 0.602, - "reward-bench/Ties": 0.5071 - } - }, - { - "id": "allenai/open_instruct_dev-rm_3e-6_1__3__1744311421", - "name": "allenai/open_instruct_dev-rm_3e-6_1__3__1744311421", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5995, - "reward-bench/Factuality": 0.7179, - "reward-bench/Precise IF": 0.3375, - "reward-bench/Math": 0.6066, - "reward-bench/Safety": 0.8, - "reward-bench/Focus": 0.6323, - "reward-bench/Ties": 0.503 - } - }, - { - "id": "allenai/open_instruct_dev-rm_3e-6_1_dpo__1__1743549903", - "name": "allenai/open_instruct_dev-rm_3e-6_1_dpo__1__1743549903", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6154, - "reward-bench/Factuality": 0.7326, - "reward-bench/Precise IF": 0.4375, - "reward-bench/Math": 0.6339, - "reward-bench/Safety": 0.7778, - "reward-bench/Focus": 0.6061, - "reward-bench/Ties": 0.5043 - } - }, - { - "id": "allenai/open_instruct_dev-rm_3e-6_1_dpo_skyworks__1__1744530368", - "name": "allenai/open_instruct_dev-rm_3e-6_1_dpo_skyworks__1__1744530368", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6604, - "reward-bench/Factuality": 0.6316, - "reward-bench/Precise IF": 0.3937, - "reward-bench/Math": 0.5792, - "reward-bench/Safety": 0.9044, - "reward-bench/Focus": 0.8929, - "reward-bench/Ties": 0.5604 - } - }, - { - "id": "allenai/open_instruct_dev-rm_3e-6_1_dpo_skyworkstulufull__1__1743550182", - "name": "allenai/open_instruct_dev-rm_3e-6_1_dpo_skyworkstulufull__1__1743550182", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6783, - "reward-bench/Factuality": 0.7705, - "reward-bench/Precise IF": 0.4, - "reward-bench/Math": 0.6066, - "reward-bench/Safety": 0.84, - "reward-bench/Focus": 0.8101, - "reward-bench/Ties": 0.6427 - } - }, - { - "id": "allenai/open_instruct_dev-rm_3e-6_1_no_if__2__1744316012", - "name": "allenai/open_instruct_dev-rm_3e-6_1_no_if__2__1744316012", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5911, - "reward-bench/Factuality": 0.7347, - "reward-bench/Precise IF": 0.4, - "reward-bench/Math": 0.6284, - "reward-bench/Safety": 0.74, - "reward-bench/Focus": 0.604, - "reward-bench/Ties": 0.4392 - } - }, - { - "id": "allenai/open_instruct_dev-rm_3e-6_1_no_if__3__1744315765", - "name": "allenai/open_instruct_dev-rm_3e-6_1_no_if__3__1744315765", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5926, - "reward-bench/Factuality": 0.7263, - "reward-bench/Precise IF": 0.3563, - "reward-bench/Math": 0.623, - "reward-bench/Safety": 0.7889, - "reward-bench/Focus": 0.5879, - "reward-bench/Ties": 0.4733 - } - }, - { - "id": "allenai/open_instruct_dev-rm_3e-6_1_rl__1__1743551527", - "name": "allenai/open_instruct_dev-rm_3e-6_1_rl__1__1743551527", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6126, - "reward-bench/Factuality": 0.7411, - "reward-bench/Precise IF": 0.425, - "reward-bench/Math": 0.623, - "reward-bench/Safety": 0.7822, - "reward-bench/Focus": 0.5939, - "reward-bench/Ties": 0.5104 - } - }, - { - "id": "allenai/open_instruct_dev-rm_3e-6_1_rl_skyworks__1__1744530236", - "name": "allenai/open_instruct_dev-rm_3e-6_1_rl_skyworks__1__1744530236", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6525, - "reward-bench/Factuality": 0.6021, - "reward-bench/Precise IF": 0.3875, - "reward-bench/Math": 0.5792, - "reward-bench/Safety": 0.8933, - "reward-bench/Focus": 0.8626, - "reward-bench/Ties": 0.59 - } - }, - { - "id": "allenai/open_instruct_dev-rm_3e-6_1_rl_skyworkstulufull__1__1743551530", - "name": "allenai/open_instruct_dev-rm_3e-6_1_rl_skyworkstulufull__1__1743551530", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6849, - "reward-bench/Factuality": 0.7453, - "reward-bench/Precise IF": 0.3812, - "reward-bench/Math": 0.612, - "reward-bench/Safety": 0.8422, - "reward-bench/Focus": 0.8404, - "reward-bench/Ties": 0.6885 - } - }, - { - "id": "allenai/open_instruct_dev-rm_3e-6_1_skyworkstulu75__1__1743534417", - "name": "allenai/open_instruct_dev-rm_3e-6_1_skyworkstulu75__1__1743534417", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.586, - "reward-bench/Factuality": 0.6632, - "reward-bench/Precise IF": 0.425, - "reward-bench/Math": 0.6557, - "reward-bench/Safety": 0.7778, - "reward-bench/Focus": 0.5172, - "reward-bench/Ties": 0.477 - } - }, - { - "id": "allenai/open_instruct_dev-rm_3e-6_1_skyworkstulufull__1__1743446486", - "name": "allenai/open_instruct_dev-rm_3e-6_1_skyworkstulufull__1__1743446486", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6773, - "reward-bench/Factuality": 0.7432, - "reward-bench/Precise IF": 0.4, - "reward-bench/Math": 0.612, - "reward-bench/Safety": 0.8422, - "reward-bench/Focus": 0.804, - "reward-bench/Ties": 0.6626 - } - }, - { - "id": "allenai/open_instruct_dev-rm_3e-6_1_skyworkstulufull__2__1744314745", - "name": "allenai/open_instruct_dev-rm_3e-6_1_skyworkstulufull__2__1744314745", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6793, - "reward-bench/Factuality": 0.7558, - "reward-bench/Precise IF": 0.4062, - "reward-bench/Math": 0.6284, - "reward-bench/Safety": 0.8311, - "reward-bench/Focus": 0.8061, - "reward-bench/Ties": 0.6485 - } - }, - { - "id": "allenai/open_instruct_dev-rm_3e-6_1_skyworkstulufull__3__1744311661", - "name": "allenai/open_instruct_dev-rm_3e-6_1_skyworkstulufull__3__1744311661", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6611, - "reward-bench/Factuality": 0.72, - "reward-bench/Precise IF": 0.3563, - "reward-bench/Math": 0.6393, - "reward-bench/Safety": 0.8444, - "reward-bench/Focus": 0.7636, - "reward-bench/Ties": 0.6428 - } - }, - { - "id": "allenai/open_instruct_dev-rm_3e-6_1_skyworkstulumix__1__1743204472", - "name": "allenai/open_instruct_dev-rm_3e-6_1_skyworkstulumix__1__1743204472", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5778, - "reward-bench/Factuality": 0.6674, - "reward-bench/Precise IF": 0.3875, - "reward-bench/Math": 0.6011, - "reward-bench/Safety": 0.7933, - "reward-bench/Focus": 0.5172, - "reward-bench/Ties": 0.5003 - } - }, - { - "id": "allenai/open_instruct_dev-rm_3e-6_2_10pctflipped__1__1743295267", - "name": "allenai/open_instruct_dev-rm_3e-6_2_10pctflipped__1__1743295267", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5746, - "reward-bench/Factuality": 0.6505, - "reward-bench/Precise IF": 0.35, - "reward-bench/Math": 0.5082, - "reward-bench/Safety": 0.7844, - "reward-bench/Focus": 0.7414, - "reward-bench/Ties": 0.4128 - } - }, - { - "id": "allenai/open_instruct_dev-rm_3e-6_2_1pctflipped__1__1743445759", - "name": "allenai/open_instruct_dev-rm_3e-6_2_1pctflipped__1__1743445759", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6065, - "reward-bench/Factuality": 0.7116, - "reward-bench/Precise IF": 0.35, - "reward-bench/Math": 0.5792, - "reward-bench/Safety": 0.8178, - "reward-bench/Focus": 0.7152, - "reward-bench/Ties": 0.465 - } - }, - { - "id": "allenai/open_instruct_dev-rm_3e-6_2_20pctflipped__1__1743324905", - "name": "allenai/open_instruct_dev-rm_3e-6_2_20pctflipped__1__1743324905", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5305, - "reward-bench/Factuality": 0.5832, - "reward-bench/Precise IF": 0.3312, - "reward-bench/Math": 0.459, - "reward-bench/Safety": 0.7178, - "reward-bench/Focus": 0.7071, - "reward-bench/Ties": 0.3849 - } - }, - { - "id": "allenai/open_instruct_dev-rm_3e-6_2_30pctflipped__1__1743326363", - "name": "allenai/open_instruct_dev-rm_3e-6_2_30pctflipped__1__1743326363", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.4436, - "reward-bench/Factuality": 0.5411, - "reward-bench/Precise IF": 0.3312, - "reward-bench/Math": 0.3115, - "reward-bench/Safety": 0.6267, - "reward-bench/Focus": 0.5414, - "reward-bench/Ties": 0.31 - } - }, - { - "id": "allenai/open_instruct_dev-rm_3e-6_2_5pctflipped__1__1743444505", - "name": "allenai/open_instruct_dev-rm_3e-6_2_5pctflipped__1__1743444505", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5925, - "reward-bench/Factuality": 0.68, - "reward-bench/Precise IF": 0.3688, - "reward-bench/Math": 0.5519, - "reward-bench/Safety": 0.78, - "reward-bench/Focus": 0.7434, - "reward-bench/Ties": 0.431 - } - }, - { - "id": "allenai/open_instruct_dev-rm_3e-6_2_dpo__1__1743550180", - "name": "allenai/open_instruct_dev-rm_3e-6_2_dpo__1__1743550180", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6198, - "reward-bench/Factuality": 0.7263, - "reward-bench/Precise IF": 0.3312, - "reward-bench/Math": 0.6339, - "reward-bench/Safety": 0.8133, - "reward-bench/Focus": 0.7232, - "reward-bench/Ties": 0.4908 - } - }, - { - "id": "allenai/open_instruct_dev-rm_3e-6_2_dpo_skyworkstulufull__1__1743550187", - "name": "allenai/open_instruct_dev-rm_3e-6_2_dpo_skyworkstulufull__1__1743550187", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6763, - "reward-bench/Factuality": 0.7411, - "reward-bench/Precise IF": 0.375, - "reward-bench/Math": 0.612, - "reward-bench/Safety": 0.8844, - "reward-bench/Focus": 0.8545, - "reward-bench/Ties": 0.5908 - } - }, - { - "id": "allenai/open_instruct_dev-rm_3e-6_2_rl__1__1743551509", - "name": "allenai/open_instruct_dev-rm_3e-6_2_rl__1__1743551509", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6245, - "reward-bench/Factuality": 0.7242, - "reward-bench/Precise IF": 0.35, - "reward-bench/Math": 0.6175, - "reward-bench/Safety": 0.8178, - "reward-bench/Focus": 0.7253, - "reward-bench/Ties": 0.5124 - } - }, - { - "id": "allenai/open_instruct_dev-rm_3e-6_2_rl_skyworkstulufull__1__1743551498", - "name": "allenai/open_instruct_dev-rm_3e-6_2_rl_skyworkstulufull__1__1743551498", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6673, - "reward-bench/Factuality": 0.7326, - "reward-bench/Precise IF": 0.3438, - "reward-bench/Math": 0.6175, - "reward-bench/Safety": 0.8622, - "reward-bench/Focus": 0.8566, - "reward-bench/Ties": 0.5911 - } - }, - { - "id": "allenai/open_instruct_dev-rm_3e-6_2_skyworkstulu75__1__1743548926", - "name": "allenai/open_instruct_dev-rm_3e-6_2_skyworkstulu75__1__1743548926", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5863, - "reward-bench/Factuality": 0.6674, - "reward-bench/Precise IF": 0.3937, - "reward-bench/Math": 0.6284, - "reward-bench/Safety": 0.8, - "reward-bench/Focus": 0.5515, - "reward-bench/Ties": 0.4768 - } - }, - { - "id": "allenai/open_instruct_dev-rm_3e-6_2_skyworkstulumix__1__1743205661", - "name": "allenai/open_instruct_dev-rm_3e-6_2_skyworkstulumix__1__1743205661", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.589, - "reward-bench/Factuality": 0.6842, - "reward-bench/Precise IF": 0.3688, - "reward-bench/Math": 0.6393, - "reward-bench/Safety": 0.7867, - "reward-bench/Focus": 0.6081, - "reward-bench/Ties": 0.447 - } - }, - { - "id": "allenai/open_instruct_dev-rm_llama70b_skyworkstulufull__4__1747266598", - "name": "allenai/open_instruct_dev-rm_llama70b_skyworkstulufull__4__1747266598", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7306, - "reward-bench/Factuality": 0.7474, - "reward-bench/Precise IF": 0.375, - "reward-bench/Math": 0.694, - "reward-bench/Safety": 0.8622, - "reward-bench/Focus": 0.8061, - "reward-bench/Ties": 0.8992 - } - }, - { - "id": "allenai/open_instruct_dev-rm_llama70b_skyworkstulufull__8__1745387923", - "name": "allenai/open_instruct_dev-rm_llama70b_skyworkstulufull__8__1745387923", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7573, - "reward-bench/Factuality": 0.8168, - "reward-bench/Precise IF": 0.4125, - "reward-bench/Math": 0.7049, - "reward-bench/Safety": 0.8733, - "reward-bench/Focus": 0.8545, - "reward-bench/Ties": 0.8814 - } - }, - { - "id": "allenai/open_instruct_dev-rm_llama_1e-6_1__1__1743896628", - "name": "allenai/open_instruct_dev-rm_llama_1e-6_1__1__1743896628", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6637, - "reward-bench/Factuality": 0.6947, - "reward-bench/Precise IF": 0.4062, - "reward-bench/Math": 0.6284, - "reward-bench/Safety": 0.8422, - "reward-bench/Focus": 0.7273, - "reward-bench/Ties": 0.6834 - } - }, - { - "id": "allenai/open_instruct_dev-rm_llama_1e-6_1_skyworks__1__1744062999", - "name": "allenai/open_instruct_dev-rm_llama_1e-6_1_skyworks__1__1744062999", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6665, - "reward-bench/Factuality": 0.5979, - "reward-bench/Precise IF": 0.3688, - "reward-bench/Math": 0.6339, - "reward-bench/Safety": 0.8956, - "reward-bench/Focus": 0.8606, - "reward-bench/Ties": 0.6422 - } - }, - { - "id": "allenai/open_instruct_dev-rm_llama_1e-6_1_skyworkstulufull__1__1743712777", - "name": "allenai/open_instruct_dev-rm_llama_1e-6_1_skyworkstulufull__1__1743712777", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7038, - "reward-bench/Factuality": 0.6947, - "reward-bench/Precise IF": 0.3937, - "reward-bench/Math": 0.6557, - "reward-bench/Safety": 0.8867, - "reward-bench/Focus": 0.8586, - "reward-bench/Ties": 0.7331 - } - }, - { - "id": "allenai/open_instruct_dev-rm_llama_1e-6_2__1__1743896638", - "name": "allenai/open_instruct_dev-rm_llama_1e-6_2__1__1743896638", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6754, - "reward-bench/Factuality": 0.6716, - "reward-bench/Precise IF": 0.4, - "reward-bench/Math": 0.6339, - "reward-bench/Safety": 0.8756, - "reward-bench/Focus": 0.7737, - "reward-bench/Ties": 0.6976 - } - }, - { - "id": "allenai/open_instruct_dev-rm_llama_1e-6_2_skyworkstulufull__1__1743800938", - "name": "allenai/open_instruct_dev-rm_llama_1e-6_2_skyworkstulufull__1__1743800938", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7241, - "reward-bench/Factuality": 0.7305, - "reward-bench/Precise IF": 0.4, - "reward-bench/Math": 0.6667, - "reward-bench/Safety": 0.9422, - "reward-bench/Focus": 0.9414, - "reward-bench/Ties": 0.6635 - } - }, - { - "id": "allenai/open_instruct_dev-rm_llama_2e-5_1_skyworkstulufull__1__1743712885", - "name": "allenai/open_instruct_dev-rm_llama_2e-5_1_skyworkstulufull__1__1743712885", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6716, - "reward-bench/Factuality": 0.6632, - "reward-bench/Precise IF": 0.3688, - "reward-bench/Math": 0.6284, - "reward-bench/Safety": 0.82, - "reward-bench/Focus": 0.8303, - "reward-bench/Ties": 0.719 - } - }, - { - "id": "allenai/open_instruct_dev-rm_llama_2e-5_2_skyworkstulufull__1__1743800773", - "name": "allenai/open_instruct_dev-rm_llama_2e-5_2_skyworkstulufull__1__1743800773", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6207, - "reward-bench/Factuality": 0.6358, - "reward-bench/Precise IF": 0.375, - "reward-bench/Math": 0.5902, - "reward-bench/Safety": 0.8267, - "reward-bench/Focus": 0.802, - "reward-bench/Ties": 0.4948 - } - }, - { - "id": "allenai/open_instruct_dev-rm_llama_2e-6_1_skyworkstulufull__1__1743893867", - "name": "allenai/open_instruct_dev-rm_llama_2e-6_1_skyworkstulufull__1__1743893867", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.719, - "reward-bench/Factuality": 0.7263, - "reward-bench/Precise IF": 0.3875, - "reward-bench/Math": 0.6393, - "reward-bench/Safety": 0.8956, - "reward-bench/Focus": 0.9273, - "reward-bench/Ties": 0.738 - } - }, - { - "id": "allenai/open_instruct_dev-rm_llama_3e-6_1__1__1743929424", - "name": "allenai/open_instruct_dev-rm_llama_3e-6_1__1__1743929424", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6572, - "reward-bench/Factuality": 0.7305, - "reward-bench/Precise IF": 0.3688, - "reward-bench/Math": 0.6284, - "reward-bench/Safety": 0.8289, - "reward-bench/Focus": 0.703, - "reward-bench/Ties": 0.6837 - } - }, - { - "id": "allenai/open_instruct_dev-rm_llama_3e-6_1__2__1744311395", - "name": "allenai/open_instruct_dev-rm_llama_3e-6_1__2__1744311395", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6938, - "reward-bench/Factuality": 0.7537, - "reward-bench/Precise IF": 0.45, - "reward-bench/Math": 0.6393, - "reward-bench/Safety": 0.8667, - "reward-bench/Focus": 0.7616, - "reward-bench/Ties": 0.6913 - } - }, - { - "id": "allenai/open_instruct_dev-rm_llama_3e-6_1__3__1744311491", - "name": "allenai/open_instruct_dev-rm_llama_3e-6_1__3__1744311491", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6754, - "reward-bench/Factuality": 0.7242, - "reward-bench/Precise IF": 0.4062, - "reward-bench/Math": 0.6284, - "reward-bench/Safety": 0.8422, - "reward-bench/Focus": 0.7535, - "reward-bench/Ties": 0.6976 - } - }, - { - "id": "allenai/open_instruct_dev-rm_llama_3e-6_1_skyworks__1__1744062787", - "name": "allenai/open_instruct_dev-rm_llama_3e-6_1_skyworks__1__1744062787", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7045, - "reward-bench/Factuality": 0.6253, - "reward-bench/Precise IF": 0.3812, - "reward-bench/Math": 0.6667, - "reward-bench/Safety": 0.92, - "reward-bench/Focus": 0.9232, - "reward-bench/Ties": 0.7109 - } - }, - { - "id": "allenai/open_instruct_dev-rm_llama_3e-6_1_skyworkstulufull__2__1744311461", - "name": "allenai/open_instruct_dev-rm_llama_3e-6_1_skyworkstulufull__2__1744311461", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7189, - "reward-bench/Factuality": 0.7305, - "reward-bench/Precise IF": 0.3937, - "reward-bench/Math": 0.6066, - "reward-bench/Safety": 0.8978, - "reward-bench/Focus": 0.9374, - "reward-bench/Ties": 0.7475 - } - }, - { - "id": "allenai/open_instruct_dev-rm_llama_3e-6_1_skyworkstulufull__3__1744311780", - "name": "allenai/open_instruct_dev-rm_llama_3e-6_1_skyworkstulufull__3__1744311780", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7172, - "reward-bench/Factuality": 0.7242, - "reward-bench/Precise IF": 0.4313, - "reward-bench/Math": 0.6175, - "reward-bench/Safety": 0.8778, - "reward-bench/Focus": 0.897, - "reward-bench/Ties": 0.7555 - } - }, - { - "id": "allenai/open_instruct_dev-rm_llama_3e-6_2__1__1743896489", - "name": "allenai/open_instruct_dev-rm_llama_3e-6_2__1__1743896489", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6813, - "reward-bench/Factuality": 0.7137, - "reward-bench/Precise IF": 0.4437, - "reward-bench/Math": 0.6284, - "reward-bench/Safety": 0.8644, - "reward-bench/Focus": 0.7596, - "reward-bench/Ties": 0.6781 - } - }, - { - "id": "allenai/open_instruct_dev-rm_llama_3e-6_2_skyworkstulufull__1__1743800713", - "name": "allenai/open_instruct_dev-rm_llama_3e-6_2_skyworkstulufull__1__1743800713", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7209, - "reward-bench/Factuality": 0.7116, - "reward-bench/Precise IF": 0.3875, - "reward-bench/Math": 0.6612, - "reward-bench/Safety": 0.9067, - "reward-bench/Focus": 0.9172, - "reward-bench/Ties": 0.7414 - } - }, - { - "id": "allenai/open_instruct_dev-rm_llama_4e-6_1_skyworkstulufull__1__1743893911", - "name": "allenai/open_instruct_dev-rm_llama_4e-6_1_skyworkstulufull__1__1743893911", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7266, - "reward-bench/Factuality": 0.7347, - "reward-bench/Precise IF": 0.4313, - "reward-bench/Math": 0.6339, - "reward-bench/Safety": 0.8933, - "reward-bench/Focus": 0.897, - "reward-bench/Ties": 0.7697 - } - }, - { - "id": "allenai/open_instruct_dev-rm_llamabase_1e-6_1_skyworkstulufull__1__1745386412", - "name": "allenai/open_instruct_dev-rm_llamabase_1e-6_1_skyworkstulufull__1__1745386412", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5342, - "reward-bench/Factuality": 0.6042, - "reward-bench/Precise IF": 0.275, - "reward-bench/Math": 0.6284, - "reward-bench/Safety": 0.7222, - "reward-bench/Focus": 0.5818, - "reward-bench/Ties": 0.3935 - } - }, - { - "id": "allenai/open_instruct_dev-rm_llamabase_1e-6_2_skyworkstulufull__1__1745441922", - "name": "allenai/open_instruct_dev-rm_llamabase_1e-6_2_skyworkstulufull__1__1745441922", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6111, - "reward-bench/Factuality": 0.6884, - "reward-bench/Precise IF": 0.3063, - "reward-bench/Math": 0.623, - "reward-bench/Safety": 0.8289, - "reward-bench/Focus": 0.7576, - "reward-bench/Ties": 0.4628 - } - }, - { - "id": "allenai/open_instruct_dev-rm_llamabase_2e-5_1_skyworkstulufull__1__1745386495", - "name": "allenai/open_instruct_dev-rm_llamabase_2e-5_1_skyworkstulufull__1__1745386495", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5825, - "reward-bench/Factuality": 0.6379, - "reward-bench/Precise IF": 0.325, - "reward-bench/Math": 0.5355, - "reward-bench/Safety": 0.8222, - "reward-bench/Focus": 0.7051, - "reward-bench/Ties": 0.4691 - } - }, - { - "id": "allenai/open_instruct_dev-rm_llamabase_2e-5_2_skyworkstulufull__1__1745386507", - "name": "allenai/open_instruct_dev-rm_llamabase_2e-5_2_skyworkstulufull__1__1745386507", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5598, - "reward-bench/Factuality": 0.5495, - "reward-bench/Precise IF": 0.3563, - "reward-bench/Math": 0.5902, - "reward-bench/Safety": 0.76, - "reward-bench/Focus": 0.7273, - "reward-bench/Ties": 0.3754 - } - }, - { - "id": "allenai/open_instruct_dev-rm_llamabase_3e-6_1_skyworkstulufull__1__1745386507", - "name": "allenai/open_instruct_dev-rm_llamabase_3e-6_1_skyworkstulufull__1__1745386507", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6101, - "reward-bench/Factuality": 0.6632, - "reward-bench/Precise IF": 0.35, - "reward-bench/Math": 0.6175, - "reward-bench/Safety": 0.7778, - "reward-bench/Focus": 0.7111, - "reward-bench/Ties": 0.5408 - } - }, - { - "id": "allenai/open_instruct_dev-rm_qwen32b_1e-6_skyworkstulufull__8__1748235917", - "name": "allenai/open_instruct_dev-rm_qwen32b_1e-6_skyworkstulufull__8__1748235917", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7185, - "reward-bench/Factuality": 0.7305, - "reward-bench/Precise IF": 0.4125, - "reward-bench/Math": 0.7158, - "reward-bench/Safety": 0.7933, - "reward-bench/Focus": 0.8545, - "reward-bench/Ties": 0.804 - } - }, - { - "id": "allenai/open_instruct_dev-rm_qwen32b_3e-6_skyworkstulufull__8__1748288961", - "name": "allenai/open_instruct_dev-rm_qwen32b_3e-6_skyworkstulufull__8__1748288961", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7325, - "reward-bench/Factuality": 0.7474, - "reward-bench/Precise IF": 0.4437, - "reward-bench/Math": 0.7158, - "reward-bench/Safety": 0.7978, - "reward-bench/Focus": 0.8141, - "reward-bench/Ties": 0.8763 - } - }, - { - "id": "allenai/open_instruct_dev-rm_qwen_1e-6_1_skyworks__1__1744062830", - "name": "allenai/open_instruct_dev-rm_qwen_1e-6_1_skyworks__1__1744062830", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6022, - "reward-bench/Factuality": 0.5284, - "reward-bench/Precise IF": 0.325, - "reward-bench/Math": 0.694, - "reward-bench/Safety": 0.7556, - "reward-bench/Focus": 0.7616, - "reward-bench/Ties": 0.5486 - } - }, - { - "id": "allenai/open_instruct_dev-rm_qwen_1e-6_1_skyworks__2__1744576024", - "name": "allenai/open_instruct_dev-rm_qwen_1e-6_1_skyworks__2__1744576024", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5948, - "reward-bench/Factuality": 0.5579, - "reward-bench/Precise IF": 0.2875, - "reward-bench/Math": 0.6776, - "reward-bench/Safety": 0.72, - "reward-bench/Focus": 0.7394, - "reward-bench/Ties": 0.5863 - } - }, - { - "id": "allenai/open_instruct_dev-rm_qwen_1e-6_1_skyworkstulufull__1__1743712914", - "name": "allenai/open_instruct_dev-rm_qwen_1e-6_1_skyworkstulufull__1__1743712914", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6492, - "reward-bench/Factuality": 0.6084, - "reward-bench/Precise IF": 0.35, - "reward-bench/Math": 0.6776, - "reward-bench/Safety": 0.76, - "reward-bench/Focus": 0.8, - "reward-bench/Ties": 0.699 - } - }, - { - "id": "allenai/open_instruct_dev-rm_qwen_2e-5_1_skyworkstulufull__1__1743713091", - "name": "allenai/open_instruct_dev-rm_qwen_2e-5_1_skyworkstulufull__1__1743713091", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6764, - "reward-bench/Factuality": 0.7074, - "reward-bench/Precise IF": 0.3, - "reward-bench/Math": 0.6885, - "reward-bench/Safety": 0.8622, - "reward-bench/Focus": 0.802, - "reward-bench/Ties": 0.6984 - } - }, - { - "id": "allenai/open_instruct_dev-rm_qwen_3e-6_1_skyworks__1__1744062829", - "name": "allenai/open_instruct_dev-rm_qwen_3e-6_1_skyworks__1__1744062829", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6408, - "reward-bench/Factuality": 0.6337, - "reward-bench/Precise IF": 0.3063, - "reward-bench/Math": 0.6831, - "reward-bench/Safety": 0.8467, - "reward-bench/Focus": 0.8222, - "reward-bench/Ties": 0.5529 - } - }, - { - "id": "allenai/open_instruct_dev-rm_qwen_3e-6_1_skyworks__2__1744576050", - "name": "allenai/open_instruct_dev-rm_qwen_3e-6_1_skyworks__2__1744576050", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6452, - "reward-bench/Factuality": 0.6063, - "reward-bench/Precise IF": 0.3187, - "reward-bench/Math": 0.7158, - "reward-bench/Safety": 0.8356, - "reward-bench/Focus": 0.8343, - "reward-bench/Ties": 0.5603 - } - }, - { - "id": "allenai/open_instruct_dev-rm_qwen_3e-6_1_skyworkstulufull__1__1743712916", - "name": "allenai/open_instruct_dev-rm_qwen_3e-6_1_skyworkstulufull__1__1743712916", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7013, - "reward-bench/Factuality": 0.7263, - "reward-bench/Precise IF": 0.3438, - "reward-bench/Math": 0.6995, - "reward-bench/Safety": 0.8222, - "reward-bench/Focus": 0.8444, - "reward-bench/Ties": 0.7714 - } - }, - { - "id": "allenai/open_instruct_dev-rm_qwen_3e-6_2__1__1743023576", - "name": "allenai/open_instruct_dev-rm_qwen_3e-6_2__1__1743023576", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6369, - "reward-bench/Factuality": 0.6905, - "reward-bench/Precise IF": 0.3187, - "reward-bench/Math": 0.6448, - "reward-bench/Safety": 0.7844, - "reward-bench/Focus": 0.7596, - "reward-bench/Ties": 0.6236 - } - }, - { - "id": "allenai/open_instruct_dev-rm_qwen_3e-6_3__1__1743023619", - "name": "allenai/open_instruct_dev-rm_qwen_3e-6_3__1__1743023619", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6221, - "reward-bench/Factuality": 0.6674, - "reward-bench/Precise IF": 0.325, - "reward-bench/Math": 0.612, - "reward-bench/Safety": 0.7978, - "reward-bench/Focus": 0.7455, - "reward-bench/Ties": 0.5852 - } - }, - { - "id": "allenai/open_instruct_dev-rm_qwenbase_1e-6_1_skyworkstulufull__1__1745388583", - "name": "allenai/open_instruct_dev-rm_qwenbase_1e-6_1_skyworkstulufull__1__1745388583", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5735, - "reward-bench/Factuality": 0.5895, - "reward-bench/Precise IF": 0.2625, - "reward-bench/Math": 0.6448, - "reward-bench/Safety": 0.6889, - "reward-bench/Focus": 0.6727, - "reward-bench/Ties": 0.5823 - } - }, - { - "id": "allenai/open_instruct_dev-rm_qwenbase_1e-6_2_skyworkstulufull__1__1745388604", - "name": "allenai/open_instruct_dev-rm_qwenbase_1e-6_2_skyworkstulufull__1__1745388604", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6336, - "reward-bench/Factuality": 0.6337, - "reward-bench/Precise IF": 0.3063, - "reward-bench/Math": 0.6885, - "reward-bench/Safety": 0.7244, - "reward-bench/Focus": 0.802, - "reward-bench/Ties": 0.6465 - } - }, - { - "id": "allenai/open_instruct_dev-rm_qwenbase_2e-5_1_skyworkstulufull__1__1745388738", - "name": "allenai/open_instruct_dev-rm_qwenbase_2e-5_1_skyworkstulufull__1__1745388738", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6824, - "reward-bench/Factuality": 0.6989, - "reward-bench/Precise IF": 0.3625, - "reward-bench/Math": 0.6831, - "reward-bench/Safety": 0.8311, - "reward-bench/Focus": 0.8081, - "reward-bench/Ties": 0.7107 - } - }, - { - "id": "allenai/open_instruct_dev-rm_qwenbase_2e-5_2_skyworkstulufull__1__1745388191", - "name": "allenai/open_instruct_dev-rm_qwenbase_2e-5_2_skyworkstulufull__1__1745388191", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6392, - "reward-bench/Factuality": 0.6589, - "reward-bench/Precise IF": 0.3312, - "reward-bench/Math": 0.6995, - "reward-bench/Safety": 0.7933, - "reward-bench/Focus": 0.7717, - "reward-bench/Ties": 0.5804 - } - }, - { - "id": "allenai/open_instruct_dev-rm_qwenbase_3e-6_1_skyworkstulufull__1__1745388737", - "name": "allenai/open_instruct_dev-rm_qwenbase_3e-6_1_skyworkstulufull__1__1745388737", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.664, - "reward-bench/Factuality": 0.6821, - "reward-bench/Precise IF": 0.3312, - "reward-bench/Math": 0.6448, - "reward-bench/Safety": 0.8133, - "reward-bench/Focus": 0.8061, - "reward-bench/Ties": 0.7066 - } - }, - { - "id": "allenai/open_instruct_dev-rm_qwenbase_3e-6_2_skyworkstulufull__1__1745388138", - "name": "allenai/open_instruct_dev-rm_qwenbase_3e-6_2_skyworkstulufull__1__1745388138", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6678, - "reward-bench/Factuality": 0.6505, - "reward-bench/Precise IF": 0.3312, - "reward-bench/Math": 0.6831, - "reward-bench/Safety": 0.7978, - "reward-bench/Focus": 0.8808, - "reward-bench/Ties": 0.6632 - } - }, - { - "id": "allenai/open_instruct_dev-rm_tulu3_70b_1__8__1742924455", - "name": "allenai/open_instruct_dev-rm_tulu3_70b_1__8__1742924455", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6618, - "reward-bench/Factuality": 0.7958, - "reward-bench/Precise IF": 0.325, - "reward-bench/Math": 0.6557, - "reward-bench/Safety": 0.8311, - "reward-bench/Focus": 0.6323, - "reward-bench/Ties": 0.7311 - } - }, - { - "id": "allenai/open_instruct_dev-rm_tulu3_70b_2__8__1742982964", - "name": "allenai/open_instruct_dev-rm_tulu3_70b_2__8__1742982964", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6605, - "reward-bench/Factuality": 0.7789, - "reward-bench/Precise IF": 0.3688, - "reward-bench/Math": 0.6448, - "reward-bench/Safety": 0.8844, - "reward-bench/Focus": 0.6667, - "reward-bench/Ties": 0.6195 - } - }, - { - "id": "allenai/tulu-2-dpo-13b", - "name": "allenai/tulu-2-dpo-13b", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7368, - "reward-bench/Chat": 0.9581, - "reward-bench/Chat Hard": 0.5833, - "reward-bench/Safety": 0.7946, - "reward-bench/Reasoning": 0.7323, - "reward-bench/Prior Sets (0.5 weight)": 0.4947 - } - }, - { - "id": "allenai/tulu-2-dpo-70b", - "name": "allenai/tulu-2-dpo-70b", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7621, - "reward-bench/Chat": 0.9749, - "reward-bench/Chat Hard": 0.6053, - "reward-bench/Safety": 0.8446, - "reward-bench/Reasoning": 0.7407, - "reward-bench/Prior Sets (0.5 weight)": 0.5278 - } - }, - { - "id": "allenai/tulu-2-dpo-7b", - "name": "allenai/tulu-2-dpo-7b", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7212, - "reward-bench/Chat": 0.9749, - "reward-bench/Chat Hard": 0.5614, - "reward-bench/Safety": 0.7527, - "reward-bench/Reasoning": 0.7176, - "reward-bench/Prior Sets (0.5 weight)": 0.4774 - } - }, - { - "id": "allenai/tulu-v2.5-13b-preference-mix-rm", - "name": "allenai/tulu-v2.5-13b-preference-mix-rm", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.8027, - "reward-bench/Chat": 0.9358, - "reward-bench/Chat Hard": 0.682, - "reward-bench/Safety": 0.773, - "reward-bench/Reasoning": 0.885, - "reward-bench/Prior Sets (0.5 weight)": 0.6724 - } - }, - { - "id": "allenai/tulu-v2.5-13b-uf-rm", - "name": "allenai/tulu-v2.5-13b-uf-rm", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.4806, - "reward-bench/Chat": 0.3939, - "reward-bench/Chat Hard": 0.4232, - "reward-bench/Safety": 0.5554, - "reward-bench/Reasoning": 0.4737, - "reward-bench/Prior Sets (0.5 weight)": 0.6326 - } - }, - { - "id": "allenai/tulu-v2.5-70b-preference-mix-rm", - "name": "allenai/tulu-v2.5-70b-preference-mix-rm", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6516, - "reward-bench/Chat": 0.7737, - "reward-bench/Chat Hard": 0.5921, - "reward-bench/Safety": 0.8486, - "reward-bench/Reasoning": 0.4138, - "reward-bench/Prior Sets (0.5 weight)": 0.6079 - } - }, - { - "id": "allenai/tulu-v2.5-70b-uf-rm", - "name": "allenai/tulu-v2.5-70b-uf-rm", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7398, - "reward-bench/Chat": 0.8659, - "reward-bench/Chat Hard": 0.7171, - "reward-bench/Safety": 0.7014, - "reward-bench/Reasoning": 0.757, - "reward-bench/Prior Sets (0.5 weight)": 0.5757 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/allknowingroger.json b/data/developers/allknowingroger.json deleted file mode 100644 index f7b5ce9aeb956a93784580621b818746c43f55b4..0000000000000000000000000000000000000000 --- a/data/developers/allknowingroger.json +++ /dev/null @@ -1,1237 +0,0 @@ -{ - "developer": "allknowingroger", - "models": [ - { - "id": "allknowingroger/Chocolatine-24B", - "name": "Chocolatine-24B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1958, - "hfopenllm_v2/BBH": 0.6191, - "hfopenllm_v2/MATH Level 5": 0.0008, - "hfopenllm_v2/GPQA": 0.3255, - "hfopenllm_v2/MUSR": 0.4323, - "hfopenllm_v2/MMLU-PRO": 0.4566 - } - }, - { - "id": "allknowingroger/Gemma2Slerp1-2.6B", - "name": "Gemma2Slerp1-2.6B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5354, - "hfopenllm_v2/BBH": 0.4343, - "hfopenllm_v2/MATH Level 5": 0.1065, - "hfopenllm_v2/GPQA": 0.2836, - "hfopenllm_v2/MUSR": 0.4562, - "hfopenllm_v2/MMLU-PRO": 0.2689 - } - }, - { - "id": "allknowingroger/Gemma2Slerp1-27B", - "name": "Gemma2Slerp1-27B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7186, - "hfopenllm_v2/BBH": 0.6399, - "hfopenllm_v2/MATH Level 5": 0.2583, - "hfopenllm_v2/GPQA": 0.3641, - "hfopenllm_v2/MUSR": 0.4767, - "hfopenllm_v2/MMLU-PRO": 0.4456 - } - }, - { - "id": "allknowingroger/Gemma2Slerp2-2.6B", - "name": "Gemma2Slerp2-2.6B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5747, - "hfopenllm_v2/BBH": 0.4308, - "hfopenllm_v2/MATH Level 5": 0.0906, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.4468, - "hfopenllm_v2/MMLU-PRO": 0.2696 - } - }, - { - "id": "allknowingroger/Gemma2Slerp2-27B", - "name": "Gemma2Slerp2-27B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7546, - "hfopenllm_v2/BBH": 0.6557, - "hfopenllm_v2/MATH Level 5": 0.2787, - "hfopenllm_v2/GPQA": 0.37, - "hfopenllm_v2/MUSR": 0.4621, - "hfopenllm_v2/MMLU-PRO": 0.4623 - } - }, - { - "id": "allknowingroger/Gemma2Slerp3-27B", - "name": "Gemma2Slerp3-27B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7426, - "hfopenllm_v2/BBH": 0.65, - "hfopenllm_v2/MATH Level 5": 0.2742, - "hfopenllm_v2/GPQA": 0.3549, - "hfopenllm_v2/MUSR": 0.474, - "hfopenllm_v2/MMLU-PRO": 0.4641 - } - }, - { - "id": "allknowingroger/Gemma2Slerp4-27B", - "name": "Gemma2Slerp4-27B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7497, - "hfopenllm_v2/BBH": 0.653, - "hfopenllm_v2/MATH Level 5": 0.2719, - "hfopenllm_v2/GPQA": 0.3666, - "hfopenllm_v2/MUSR": 0.4502, - "hfopenllm_v2/MMLU-PRO": 0.4649 - } - }, - { - "id": "allknowingroger/GemmaSlerp-9B", - "name": "GemmaSlerp-9B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7043, - "hfopenllm_v2/BBH": 0.5921, - "hfopenllm_v2/MATH Level 5": 0.216, - "hfopenllm_v2/GPQA": 0.344, - "hfopenllm_v2/MUSR": 0.4673, - "hfopenllm_v2/MMLU-PRO": 0.4161 - } - }, - { - "id": "allknowingroger/GemmaSlerp2-9B", - "name": "GemmaSlerp2-9B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7281, - "hfopenllm_v2/BBH": 0.5983, - "hfopenllm_v2/MATH Level 5": 0.2107, - "hfopenllm_v2/GPQA": 0.3523, - "hfopenllm_v2/MUSR": 0.4767, - "hfopenllm_v2/MMLU-PRO": 0.4239 - } - }, - { - "id": "allknowingroger/GemmaSlerp4-10B", - "name": "GemmaSlerp4-10B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7326, - "hfopenllm_v2/BBH": 0.6028, - "hfopenllm_v2/MATH Level 5": 0.2243, - "hfopenllm_v2/GPQA": 0.3532, - "hfopenllm_v2/MUSR": 0.454, - "hfopenllm_v2/MMLU-PRO": 0.425 - } - }, - { - "id": "allknowingroger/GemmaSlerp5-10B", - "name": "GemmaSlerp5-10B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7353, - "hfopenllm_v2/BBH": 0.6054, - "hfopenllm_v2/MATH Level 5": 0.2183, - "hfopenllm_v2/GPQA": 0.3523, - "hfopenllm_v2/MUSR": 0.4608, - "hfopenllm_v2/MMLU-PRO": 0.4328 - } - }, - { - "id": "allknowingroger/GemmaStock1-27B", - "name": "GemmaStock1-27B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7509, - "hfopenllm_v2/BBH": 0.6566, - "hfopenllm_v2/MATH Level 5": 0.2636, - "hfopenllm_v2/GPQA": 0.3641, - "hfopenllm_v2/MUSR": 0.4527, - "hfopenllm_v2/MMLU-PRO": 0.473 - } - }, - { - "id": "allknowingroger/HomerSlerp1-7B", - "name": "HomerSlerp1-7B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4621, - "hfopenllm_v2/BBH": 0.5518, - "hfopenllm_v2/MATH Level 5": 0.2719, - "hfopenllm_v2/GPQA": 0.318, - "hfopenllm_v2/MUSR": 0.4359, - "hfopenllm_v2/MMLU-PRO": 0.4504 - } - }, - { - "id": "allknowingroger/HomerSlerp2-7B", - "name": "HomerSlerp2-7B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4487, - "hfopenllm_v2/BBH": 0.5649, - "hfopenllm_v2/MATH Level 5": 0.2968, - "hfopenllm_v2/GPQA": 0.3196, - "hfopenllm_v2/MUSR": 0.4356, - "hfopenllm_v2/MMLU-PRO": 0.4515 - } - }, - { - "id": "allknowingroger/HomerSlerp3-7B", - "name": "HomerSlerp3-7B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4363, - "hfopenllm_v2/BBH": 0.5598, - "hfopenllm_v2/MATH Level 5": 0.3021, - "hfopenllm_v2/GPQA": 0.3171, - "hfopenllm_v2/MUSR": 0.4462, - "hfopenllm_v2/MMLU-PRO": 0.4535 - } - }, - { - "id": "allknowingroger/HomerSlerp4-7B", - "name": "HomerSlerp4-7B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4374, - "hfopenllm_v2/BBH": 0.5571, - "hfopenllm_v2/MATH Level 5": 0.327, - "hfopenllm_v2/GPQA": 0.3196, - "hfopenllm_v2/MUSR": 0.4408, - "hfopenllm_v2/MMLU-PRO": 0.4472 - } - }, - { - "id": "allknowingroger/limyClown-7B-slerp", - "name": "limyClown-7B-slerp", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4017, - "hfopenllm_v2/BBH": 0.5148, - "hfopenllm_v2/MATH Level 5": 0.0687, - "hfopenllm_v2/GPQA": 0.281, - "hfopenllm_v2/MUSR": 0.4293, - "hfopenllm_v2/MMLU-PRO": 0.3038 - } - }, - { - "id": "allknowingroger/LimyQstar-7B-slerp", - "name": "LimyQstar-7B-slerp", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3491, - "hfopenllm_v2/BBH": 0.5024, - "hfopenllm_v2/MATH Level 5": 0.0687, - "hfopenllm_v2/GPQA": 0.2987, - "hfopenllm_v2/MUSR": 0.4146, - "hfopenllm_v2/MMLU-PRO": 0.3103 - } - }, - { - "id": "allknowingroger/llama3-Jallabi-40B-s", - "name": "llama3-Jallabi-40B-s", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1921, - "hfopenllm_v2/BBH": 0.3252, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2374, - "hfopenllm_v2/MUSR": 0.375, - "hfopenllm_v2/MMLU-PRO": 0.1088 - } - }, - { - "id": "allknowingroger/Llama3.1-60B", - "name": "Llama3.1-60B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1815, - "hfopenllm_v2/BBH": 0.3242, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.3596, - "hfopenllm_v2/MMLU-PRO": 0.331 - } - }, - { - "id": "allknowingroger/llama3AnFeng-40B", - "name": "llama3AnFeng-40B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1742, - "hfopenllm_v2/BBH": 0.3794, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.3062, - "hfopenllm_v2/MUSR": 0.394, - "hfopenllm_v2/MMLU-PRO": 0.198 - } - }, - { - "id": "allknowingroger/Marco-01-slerp1-7B", - "name": "Marco-01-slerp1-7B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4681, - "hfopenllm_v2/BBH": 0.5541, - "hfopenllm_v2/MATH Level 5": 0.3157, - "hfopenllm_v2/GPQA": 0.3171, - "hfopenllm_v2/MUSR": 0.4452, - "hfopenllm_v2/MMLU-PRO": 0.4483 - } - }, - { - "id": "allknowingroger/Meme-7B-slerp", - "name": "Meme-7B-slerp", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5164, - "hfopenllm_v2/BBH": 0.4661, - "hfopenllm_v2/MATH Level 5": 0.0438, - "hfopenllm_v2/GPQA": 0.2861, - "hfopenllm_v2/MUSR": 0.4223, - "hfopenllm_v2/MMLU-PRO": 0.281 - } - }, - { - "id": "allknowingroger/Ministral-8B-slerp", - "name": "Ministral-8B-slerp", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1961, - "hfopenllm_v2/BBH": 0.4686, - "hfopenllm_v2/MATH Level 5": 0.0038, - "hfopenllm_v2/GPQA": 0.3121, - "hfopenllm_v2/MUSR": 0.4285, - "hfopenllm_v2/MMLU-PRO": 0.3119 - } - }, - { - "id": "allknowingroger/Mistralmash1-7B-s", - "name": "Mistralmash1-7B-s", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3961, - "hfopenllm_v2/BBH": 0.5277, - "hfopenllm_v2/MATH Level 5": 0.0921, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.4267, - "hfopenllm_v2/MMLU-PRO": 0.3293 - } - }, - { - "id": "allknowingroger/Mistralmash2-7B-s", - "name": "Mistralmash2-7B-s", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4102, - "hfopenllm_v2/BBH": 0.5305, - "hfopenllm_v2/MATH Level 5": 0.0793, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.4372, - "hfopenllm_v2/MMLU-PRO": 0.3345 - } - }, - { - "id": "allknowingroger/MistralPhi3-11B", - "name": "MistralPhi3-11B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1943, - "hfopenllm_v2/BBH": 0.6234, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.3322, - "hfopenllm_v2/MUSR": 0.4267, - "hfopenllm_v2/MMLU-PRO": 0.4688 - } - }, - { - "id": "allknowingroger/MixTAO-19B-pass", - "name": "MixTAO-19B-pass", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3814, - "hfopenllm_v2/BBH": 0.5128, - "hfopenllm_v2/MATH Level 5": 0.0612, - "hfopenllm_v2/GPQA": 0.2844, - "hfopenllm_v2/MUSR": 0.4783, - "hfopenllm_v2/MMLU-PRO": 0.3105 - } - }, - { - "id": "allknowingroger/MixTaoTruthful-13B-slerp", - "name": "MixTaoTruthful-13B-slerp", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4139, - "hfopenllm_v2/BBH": 0.5207, - "hfopenllm_v2/MATH Level 5": 0.0665, - "hfopenllm_v2/GPQA": 0.2844, - "hfopenllm_v2/MUSR": 0.4292, - "hfopenllm_v2/MMLU-PRO": 0.31 - } - }, - { - "id": "allknowingroger/MultiCalm-7B-slerp", - "name": "MultiCalm-7B-slerp", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3927, - "hfopenllm_v2/BBH": 0.5122, - "hfopenllm_v2/MATH Level 5": 0.0619, - "hfopenllm_v2/GPQA": 0.2827, - "hfopenllm_v2/MUSR": 0.4319, - "hfopenllm_v2/MMLU-PRO": 0.3033 - } - }, - { - "id": "allknowingroger/MultiMash-12B-slerp", - "name": "MultiMash-12B-slerp", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3974, - "hfopenllm_v2/BBH": 0.5142, - "hfopenllm_v2/MATH Level 5": 0.0808, - "hfopenllm_v2/GPQA": 0.2768, - "hfopenllm_v2/MUSR": 0.4438, - "hfopenllm_v2/MMLU-PRO": 0.3068 - } - }, - { - "id": "allknowingroger/MultiMash10-13B-slerp", - "name": "MultiMash10-13B-slerp", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4163, - "hfopenllm_v2/BBH": 0.5186, - "hfopenllm_v2/MATH Level 5": 0.0718, - "hfopenllm_v2/GPQA": 0.2861, - "hfopenllm_v2/MUSR": 0.4318, - "hfopenllm_v2/MMLU-PRO": 0.3117 - } - }, - { - "id": "allknowingroger/MultiMash11-13B-slerp", - "name": "MultiMash11-13B-slerp", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4251, - "hfopenllm_v2/BBH": 0.5194, - "hfopenllm_v2/MATH Level 5": 0.0702, - "hfopenllm_v2/GPQA": 0.2827, - "hfopenllm_v2/MUSR": 0.4373, - "hfopenllm_v2/MMLU-PRO": 0.3085 - } - }, - { - "id": "allknowingroger/MultiMash2-12B-slerp", - "name": "MultiMash2-12B-slerp", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4261, - "hfopenllm_v2/BBH": 0.5134, - "hfopenllm_v2/MATH Level 5": 0.0642, - "hfopenllm_v2/GPQA": 0.2794, - "hfopenllm_v2/MUSR": 0.4228, - "hfopenllm_v2/MMLU-PRO": 0.3043 - } - }, - { - "id": "allknowingroger/Multimash3-12B-slerp", - "name": "Multimash3-12B-slerp", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4437, - "hfopenllm_v2/BBH": 0.5177, - "hfopenllm_v2/MATH Level 5": 0.0627, - "hfopenllm_v2/GPQA": 0.2802, - "hfopenllm_v2/MUSR": 0.4344, - "hfopenllm_v2/MMLU-PRO": 0.3068 - } - }, - { - "id": "allknowingroger/MultiMash5-12B-slerp", - "name": "MultiMash5-12B-slerp", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4142, - "hfopenllm_v2/BBH": 0.5145, - "hfopenllm_v2/MATH Level 5": 0.0634, - "hfopenllm_v2/GPQA": 0.2777, - "hfopenllm_v2/MUSR": 0.4203, - "hfopenllm_v2/MMLU-PRO": 0.3028 - } - }, - { - "id": "allknowingroger/MultiMash6-12B-slerp", - "name": "MultiMash6-12B-slerp", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.43, - "hfopenllm_v2/BBH": 0.5196, - "hfopenllm_v2/MATH Level 5": 0.0725, - "hfopenllm_v2/GPQA": 0.2743, - "hfopenllm_v2/MUSR": 0.4306, - "hfopenllm_v2/MMLU-PRO": 0.3091 - } - }, - { - "id": "allknowingroger/MultiMash7-12B-slerp", - "name": "MultiMash7-12B-slerp", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4213, - "hfopenllm_v2/BBH": 0.5111, - "hfopenllm_v2/MATH Level 5": 0.0695, - "hfopenllm_v2/GPQA": 0.2785, - "hfopenllm_v2/MUSR": 0.4279, - "hfopenllm_v2/MMLU-PRO": 0.3029 - } - }, - { - "id": "allknowingroger/MultiMash8-13B-slerp", - "name": "MultiMash8-13B-slerp", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4321, - "hfopenllm_v2/BBH": 0.5178, - "hfopenllm_v2/MATH Level 5": 0.077, - "hfopenllm_v2/GPQA": 0.2886, - "hfopenllm_v2/MUSR": 0.4424, - "hfopenllm_v2/MMLU-PRO": 0.3126 - } - }, - { - "id": "allknowingroger/MultiMash9-13B-slerp", - "name": "MultiMash9-13B-slerp", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4188, - "hfopenllm_v2/BBH": 0.5194, - "hfopenllm_v2/MATH Level 5": 0.0785, - "hfopenllm_v2/GPQA": 0.2802, - "hfopenllm_v2/MUSR": 0.4398, - "hfopenllm_v2/MMLU-PRO": 0.31 - } - }, - { - "id": "allknowingroger/Multimerge-19B-pass", - "name": "Multimerge-19B-pass", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1773, - "hfopenllm_v2/BBH": 0.2892, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.343, - "hfopenllm_v2/MMLU-PRO": 0.1169 - } - }, - { - "id": "allknowingroger/MultiMerge-7B-slerp", - "name": "MultiMerge-7B-slerp", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3948, - "hfopenllm_v2/BBH": 0.514, - "hfopenllm_v2/MATH Level 5": 0.0665, - "hfopenllm_v2/GPQA": 0.2827, - "hfopenllm_v2/MUSR": 0.428, - "hfopenllm_v2/MMLU-PRO": 0.3037 - } - }, - { - "id": "allknowingroger/MultiverseEx26-7B-slerp", - "name": "MultiverseEx26-7B-slerp", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3939, - "hfopenllm_v2/BBH": 0.5134, - "hfopenllm_v2/MATH Level 5": 0.0755, - "hfopenllm_v2/GPQA": 0.2827, - "hfopenllm_v2/MUSR": 0.4293, - "hfopenllm_v2/MMLU-PRO": 0.3035 - } - }, - { - "id": "allknowingroger/Neuralcoven-7B-slerp", - "name": "Neuralcoven-7B-slerp", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3859, - "hfopenllm_v2/BBH": 0.5303, - "hfopenllm_v2/MATH Level 5": 0.0785, - "hfopenllm_v2/GPQA": 0.2852, - "hfopenllm_v2/MUSR": 0.429, - "hfopenllm_v2/MMLU-PRO": 0.3294 - } - }, - { - "id": "allknowingroger/Neuralmultiverse-7B-slerp", - "name": "Neuralmultiverse-7B-slerp", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3769, - "hfopenllm_v2/BBH": 0.5166, - "hfopenllm_v2/MATH Level 5": 0.065, - "hfopenllm_v2/GPQA": 0.2844, - "hfopenllm_v2/MUSR": 0.428, - "hfopenllm_v2/MMLU-PRO": 0.3042 - } - }, - { - "id": "allknowingroger/NeuralWestSeverus-7B-slerp", - "name": "NeuralWestSeverus-7B-slerp", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4136, - "hfopenllm_v2/BBH": 0.5244, - "hfopenllm_v2/MATH Level 5": 0.0733, - "hfopenllm_v2/GPQA": 0.271, - "hfopenllm_v2/MUSR": 0.4529, - "hfopenllm_v2/MMLU-PRO": 0.3137 - } - }, - { - "id": "allknowingroger/Ph3della5-14B", - "name": "Ph3della5-14B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4799, - "hfopenllm_v2/BBH": 0.6332, - "hfopenllm_v2/MATH Level 5": 0.1767, - "hfopenllm_v2/GPQA": 0.3423, - "hfopenllm_v2/MUSR": 0.4386, - "hfopenllm_v2/MMLU-PRO": 0.4787 - } - }, - { - "id": "allknowingroger/Ph3merge-14B", - "name": "Ph3merge-14B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2701, - "hfopenllm_v2/BBH": 0.6381, - "hfopenllm_v2/MATH Level 5": 0.0106, - "hfopenllm_v2/GPQA": 0.3381, - "hfopenllm_v2/MUSR": 0.4334, - "hfopenllm_v2/MMLU-PRO": 0.4611 - } - }, - { - "id": "allknowingroger/Ph3merge2-14B", - "name": "Ph3merge2-14B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1706, - "hfopenllm_v2/BBH": 0.3607, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2911, - "hfopenllm_v2/MUSR": 0.3911, - "hfopenllm_v2/MMLU-PRO": 0.1723 - } - }, - { - "id": "allknowingroger/Ph3merge3-14B", - "name": "Ph3merge3-14B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1645, - "hfopenllm_v2/BBH": 0.3597, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2852, - "hfopenllm_v2/MUSR": 0.4082, - "hfopenllm_v2/MMLU-PRO": 0.1647 - } - }, - { - "id": "allknowingroger/Ph3task1-14B", - "name": "Ph3task1-14B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4695, - "hfopenllm_v2/BBH": 0.6318, - "hfopenllm_v2/MATH Level 5": 0.1669, - "hfopenllm_v2/GPQA": 0.3507, - "hfopenllm_v2/MUSR": 0.4508, - "hfopenllm_v2/MMLU-PRO": 0.4734 - } - }, - { - "id": "allknowingroger/Ph3task2-14B", - "name": "Ph3task2-14B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4713, - "hfopenllm_v2/BBH": 0.6098, - "hfopenllm_v2/MATH Level 5": 0.1465, - "hfopenllm_v2/GPQA": 0.3305, - "hfopenllm_v2/MUSR": 0.4535, - "hfopenllm_v2/MMLU-PRO": 0.446 - } - }, - { - "id": "allknowingroger/Ph3task3-14B", - "name": "Ph3task3-14B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4962, - "hfopenllm_v2/BBH": 0.6298, - "hfopenllm_v2/MATH Level 5": 0.176, - "hfopenllm_v2/GPQA": 0.3414, - "hfopenllm_v2/MUSR": 0.4426, - "hfopenllm_v2/MMLU-PRO": 0.4771 - } - }, - { - "id": "allknowingroger/Ph3unsloth-3B-slerp", - "name": "Ph3unsloth-3B-slerp", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1894, - "hfopenllm_v2/BBH": 0.5468, - "hfopenllm_v2/MATH Level 5": 0.1012, - "hfopenllm_v2/GPQA": 0.3247, - "hfopenllm_v2/MUSR": 0.4528, - "hfopenllm_v2/MMLU-PRO": 0.3701 - } - }, - { - "id": "allknowingroger/Phi3mash1-17B-pass", - "name": "Phi3mash1-17B-pass", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1884, - "hfopenllm_v2/BBH": 0.6129, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.3196, - "hfopenllm_v2/MUSR": 0.4451, - "hfopenllm_v2/MMLU-PRO": 0.4589 - } - }, - { - "id": "allknowingroger/Quen2-65B", - "name": "Quen2-65B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1758, - "hfopenllm_v2/BBH": 0.2757, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2357, - "hfopenllm_v2/MUSR": 0.3209, - "hfopenllm_v2/MMLU-PRO": 0.1114 - } - }, - { - "id": "allknowingroger/Qwen2.5-42B-AGI", - "name": "Qwen2.5-42B-AGI", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1913, - "hfopenllm_v2/BBH": 0.2942, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.362, - "hfopenllm_v2/MMLU-PRO": 0.1168 - } - }, - { - "id": "allknowingroger/Qwen2.5-7B-task2", - "name": "Qwen2.5-7B-task2", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4527, - "hfopenllm_v2/BBH": 0.5626, - "hfopenllm_v2/MATH Level 5": 0.355, - "hfopenllm_v2/GPQA": 0.3163, - "hfopenllm_v2/MUSR": 0.437, - "hfopenllm_v2/MMLU-PRO": 0.4517 - } - }, - { - "id": "allknowingroger/Qwen2.5-7B-task3", - "name": "Qwen2.5-7B-task3", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5129, - "hfopenllm_v2/BBH": 0.5398, - "hfopenllm_v2/MATH Level 5": 0.2606, - "hfopenllm_v2/GPQA": 0.3171, - "hfopenllm_v2/MUSR": 0.4356, - "hfopenllm_v2/MMLU-PRO": 0.4501 - } - }, - { - "id": "allknowingroger/Qwen2.5-7B-task4", - "name": "Qwen2.5-7B-task4", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5005, - "hfopenllm_v2/BBH": 0.5583, - "hfopenllm_v2/MATH Level 5": 0.3112, - "hfopenllm_v2/GPQA": 0.3205, - "hfopenllm_v2/MUSR": 0.4395, - "hfopenllm_v2/MMLU-PRO": 0.4561 - } - }, - { - "id": "allknowingroger/Qwen2.5-7B-task7", - "name": "Qwen2.5-7B-task7", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4284, - "hfopenllm_v2/BBH": 0.5552, - "hfopenllm_v2/MATH Level 5": 0.065, - "hfopenllm_v2/GPQA": 0.3205, - "hfopenllm_v2/MUSR": 0.4326, - "hfopenllm_v2/MMLU-PRO": 0.4133 - } - }, - { - "id": "allknowingroger/Qwen2.5-7B-task8", - "name": "Qwen2.5-7B-task8", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4645, - "hfopenllm_v2/BBH": 0.5525, - "hfopenllm_v2/MATH Level 5": 0.3527, - "hfopenllm_v2/GPQA": 0.3205, - "hfopenllm_v2/MUSR": 0.4514, - "hfopenllm_v2/MMLU-PRO": 0.4433 - } - }, - { - "id": "allknowingroger/Qwen2.5-slerp-14B", - "name": "Qwen2.5-slerp-14B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4928, - "hfopenllm_v2/BBH": 0.6512, - "hfopenllm_v2/MATH Level 5": 0.4622, - "hfopenllm_v2/GPQA": 0.3674, - "hfopenllm_v2/MUSR": 0.4744, - "hfopenllm_v2/MMLU-PRO": 0.5379 - } - }, - { - "id": "allknowingroger/QwenSlerp12-7B", - "name": "QwenSlerp12-7B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5076, - "hfopenllm_v2/BBH": 0.5556, - "hfopenllm_v2/MATH Level 5": 0.2946, - "hfopenllm_v2/GPQA": 0.3154, - "hfopenllm_v2/MUSR": 0.4595, - "hfopenllm_v2/MMLU-PRO": 0.4461 - } - }, - { - "id": "allknowingroger/Qwenslerp2-14B", - "name": "Qwenslerp2-14B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5007, - "hfopenllm_v2/BBH": 0.6555, - "hfopenllm_v2/MATH Level 5": 0.4456, - "hfopenllm_v2/GPQA": 0.3683, - "hfopenllm_v2/MUSR": 0.4729, - "hfopenllm_v2/MMLU-PRO": 0.5403 - } - }, - { - "id": "allknowingroger/Qwenslerp2-7B", - "name": "Qwenslerp2-7B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5294, - "hfopenllm_v2/BBH": 0.5609, - "hfopenllm_v2/MATH Level 5": 0.3421, - "hfopenllm_v2/GPQA": 0.3129, - "hfopenllm_v2/MUSR": 0.4356, - "hfopenllm_v2/MMLU-PRO": 0.4515 - } - }, - { - "id": "allknowingroger/Qwenslerp3-14B", - "name": "Qwenslerp3-14B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5052, - "hfopenllm_v2/BBH": 0.6521, - "hfopenllm_v2/MATH Level 5": 0.4464, - "hfopenllm_v2/GPQA": 0.375, - "hfopenllm_v2/MUSR": 0.4676, - "hfopenllm_v2/MMLU-PRO": 0.5395 - } - }, - { - "id": "allknowingroger/Qwenslerp3-7B", - "name": "Qwenslerp3-7B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5018, - "hfopenllm_v2/BBH": 0.558, - "hfopenllm_v2/MATH Level 5": 0.3218, - "hfopenllm_v2/GPQA": 0.3247, - "hfopenllm_v2/MUSR": 0.4515, - "hfopenllm_v2/MMLU-PRO": 0.4542 - } - }, - { - "id": "allknowingroger/QwenSlerp4-14B", - "name": "QwenSlerp4-14B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6328, - "hfopenllm_v2/BBH": 0.6483, - "hfopenllm_v2/MATH Level 5": 0.3693, - "hfopenllm_v2/GPQA": 0.3725, - "hfopenllm_v2/MUSR": 0.465, - "hfopenllm_v2/MMLU-PRO": 0.5436 - } - }, - { - "id": "allknowingroger/QwenSlerp5-14B", - "name": "QwenSlerp5-14B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7119, - "hfopenllm_v2/BBH": 0.6357, - "hfopenllm_v2/MATH Level 5": 0.3565, - "hfopenllm_v2/GPQA": 0.3649, - "hfopenllm_v2/MUSR": 0.4675, - "hfopenllm_v2/MMLU-PRO": 0.5391 - } - }, - { - "id": "allknowingroger/QwenSlerp6-14B", - "name": "QwenSlerp6-14B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6867, - "hfopenllm_v2/BBH": 0.6384, - "hfopenllm_v2/MATH Level 5": 0.3724, - "hfopenllm_v2/GPQA": 0.3733, - "hfopenllm_v2/MUSR": 0.469, - "hfopenllm_v2/MMLU-PRO": 0.5406 - } - }, - { - "id": "allknowingroger/QwenStock1-14B", - "name": "QwenStock1-14B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5634, - "hfopenllm_v2/BBH": 0.6528, - "hfopenllm_v2/MATH Level 5": 0.3769, - "hfopenllm_v2/GPQA": 0.3767, - "hfopenllm_v2/MUSR": 0.473, - "hfopenllm_v2/MMLU-PRO": 0.5418 - } - }, - { - "id": "allknowingroger/QwenStock2-14B", - "name": "QwenStock2-14B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5563, - "hfopenllm_v2/BBH": 0.6569, - "hfopenllm_v2/MATH Level 5": 0.3882, - "hfopenllm_v2/GPQA": 0.3792, - "hfopenllm_v2/MUSR": 0.4756, - "hfopenllm_v2/MMLU-PRO": 0.5406 - } - }, - { - "id": "allknowingroger/QwenStock3-14B", - "name": "QwenStock3-14B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5615, - "hfopenllm_v2/BBH": 0.6565, - "hfopenllm_v2/MATH Level 5": 0.3776, - "hfopenllm_v2/GPQA": 0.3784, - "hfopenllm_v2/MUSR": 0.4756, - "hfopenllm_v2/MMLU-PRO": 0.5428 - } - }, - { - "id": "allknowingroger/RogerMerge-7B-slerp", - "name": "RogerMerge-7B-slerp", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3933, - "hfopenllm_v2/BBH": 0.516, - "hfopenllm_v2/MATH Level 5": 0.0687, - "hfopenllm_v2/GPQA": 0.2802, - "hfopenllm_v2/MUSR": 0.432, - "hfopenllm_v2/MMLU-PRO": 0.303 - } - }, - { - "id": "allknowingroger/ROGERphi-7B-slerp", - "name": "ROGERphi-7B-slerp", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3861, - "hfopenllm_v2/BBH": 0.5196, - "hfopenllm_v2/MATH Level 5": 0.0733, - "hfopenllm_v2/GPQA": 0.2886, - "hfopenllm_v2/MUSR": 0.4685, - "hfopenllm_v2/MMLU-PRO": 0.3053 - } - }, - { - "id": "allknowingroger/Rombos-LLM-V2.5-Qwen-42b", - "name": "Rombos-LLM-V2.5-Qwen-42b", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1879, - "hfopenllm_v2/BBH": 0.2969, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2626, - "hfopenllm_v2/MUSR": 0.3633, - "hfopenllm_v2/MMLU-PRO": 0.1168 - } - }, - { - "id": "allknowingroger/Strangecoven-7B-slerp", - "name": "Strangecoven-7B-slerp", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3746, - "hfopenllm_v2/BBH": 0.5368, - "hfopenllm_v2/MATH Level 5": 0.0763, - "hfopenllm_v2/GPQA": 0.2894, - "hfopenllm_v2/MUSR": 0.4199, - "hfopenllm_v2/MMLU-PRO": 0.3364 - } - }, - { - "id": "allknowingroger/Weirdslerp2-25B", - "name": "Weirdslerp2-25B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1754, - "hfopenllm_v2/BBH": 0.2874, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2492, - "hfopenllm_v2/MUSR": 0.3524, - "hfopenllm_v2/MMLU-PRO": 0.1128 - } - }, - { - "id": "allknowingroger/WestlakeMaziyar-7B-slerp", - "name": "WestlakeMaziyar-7B-slerp", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4838, - "hfopenllm_v2/BBH": 0.5245, - "hfopenllm_v2/MATH Level 5": 0.0665, - "hfopenllm_v2/GPQA": 0.3037, - "hfopenllm_v2/MUSR": 0.4474, - "hfopenllm_v2/MMLU-PRO": 0.3078 - } - }, - { - "id": "allknowingroger/YamMaths-7B-slerp", - "name": "YamMaths-7B-slerp", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4148, - "hfopenllm_v2/BBH": 0.5156, - "hfopenllm_v2/MATH Level 5": 0.0853, - "hfopenllm_v2/GPQA": 0.2802, - "hfopenllm_v2/MUSR": 0.4384, - "hfopenllm_v2/MMLU-PRO": 0.3131 - } - }, - { - "id": "allknowingroger/Yi-1.5-34B", - "name": "Yi-1.5-34B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1639, - "hfopenllm_v2/BBH": 0.2827, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2584, - "hfopenllm_v2/MUSR": 0.3857, - "hfopenllm_v2/MMLU-PRO": 0.1095 - } - }, - { - "id": "allknowingroger/Yi-blossom-40B", - "name": "Yi-blossom-40B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2009, - "hfopenllm_v2/BBH": 0.3215, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2743, - "hfopenllm_v2/MUSR": 0.3843, - "hfopenllm_v2/MMLU-PRO": 0.108 - } - }, - { - "id": "allknowingroger/Yibuddy-35B", - "name": "Yibuddy-35B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4235, - "hfopenllm_v2/BBH": 0.5916, - "hfopenllm_v2/MATH Level 5": 0.1571, - "hfopenllm_v2/GPQA": 0.3557, - "hfopenllm_v2/MUSR": 0.4505, - "hfopenllm_v2/MMLU-PRO": 0.4489 - } - }, - { - "id": "allknowingroger/Yillama-40B", - "name": "Yillama-40B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1697, - "hfopenllm_v2/BBH": 0.4063, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2827, - "hfopenllm_v2/MUSR": 0.3501, - "hfopenllm_v2/MMLU-PRO": 0.1981 - } - }, - { - "id": "allknowingroger/Yislerp-34B", - "name": "Yislerp-34B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3692, - "hfopenllm_v2/BBH": 0.6159, - "hfopenllm_v2/MATH Level 5": 0.216, - "hfopenllm_v2/GPQA": 0.3582, - "hfopenllm_v2/MUSR": 0.4566, - "hfopenllm_v2/MMLU-PRO": 0.4751 - } - }, - { - "id": "allknowingroger/Yislerp2-34B", - "name": "Yislerp2-34B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3999, - "hfopenllm_v2/BBH": 0.6246, - "hfopenllm_v2/MATH Level 5": 0.2296, - "hfopenllm_v2/GPQA": 0.3641, - "hfopenllm_v2/MUSR": 0.453, - "hfopenllm_v2/MMLU-PRO": 0.4724 - } - }, - { - "id": "allknowingroger/Yunconglong-13B-slerp", - "name": "Yunconglong-13B-slerp", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4242, - "hfopenllm_v2/BBH": 0.5166, - "hfopenllm_v2/MATH Level 5": 0.0544, - "hfopenllm_v2/GPQA": 0.281, - "hfopenllm_v2/MUSR": 0.4161, - "hfopenllm_v2/MMLU-PRO": 0.3036 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/allura-org.json b/data/developers/allura-org.json deleted file mode 100644 index 0e7deaf589722fd85c613eed5d6dac33a2bc38af..0000000000000000000000000000000000000000 --- a/data/developers/allura-org.json +++ /dev/null @@ -1,131 +0,0 @@ -{ - "developer": "allura-org", - "models": [ - { - "id": "allura-org/L3.1-8b-RP-Ink", - "name": "L3.1-8b-RP-Ink", - "developer": "allura-org", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7811, - "hfopenllm_v2/BBH": 0.4828, - "hfopenllm_v2/MATH Level 5": 0.148, - "hfopenllm_v2/GPQA": 0.2643, - "hfopenllm_v2/MUSR": 0.3608, - "hfopenllm_v2/MMLU-PRO": 0.3428 - } - }, - { - "id": "allura-org/Mistral-Small-24b-Sertraline-0304", - "name": "Mistral-Small-24b-Sertraline-0304", - "developer": "allura-org", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.68, - "hfopenllm_v2/BBH": 0.6525, - "hfopenllm_v2/MATH Level 5": 0.2228, - "hfopenllm_v2/GPQA": 0.3515, - "hfopenllm_v2/MUSR": 0.4395, - "hfopenllm_v2/MMLU-PRO": 0.5106 - } - }, - { - "id": "allura-org/Mistral-Small-Sisyphus-24b-2503", - "name": "Mistral-Small-Sisyphus-24b-2503", - "developer": "allura-org", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6848, - "hfopenllm_v2/BBH": 0.627, - "hfopenllm_v2/MATH Level 5": 0.25, - "hfopenllm_v2/GPQA": 0.2626, - "hfopenllm_v2/MUSR": 0.3977, - "hfopenllm_v2/MMLU-PRO": 0.5127 - } - }, - { - "id": "allura-org/MN-12b-RP-Ink", - "name": "MN-12b-RP-Ink", - "developer": "allura-org", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7186, - "hfopenllm_v2/BBH": 0.4834, - "hfopenllm_v2/MATH Level 5": 0.1186, - "hfopenllm_v2/GPQA": 0.2852, - "hfopenllm_v2/MUSR": 0.3818, - "hfopenllm_v2/MMLU-PRO": 0.3514 - } - }, - { - "id": "allura-org/MoE-Girl-1BA-7BT", - "name": "MoE-Girl-1BA-7BT", - "developer": "allura-org", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2705, - "hfopenllm_v2/BBH": 0.3139, - "hfopenllm_v2/MATH Level 5": 0.0151, - "hfopenllm_v2/GPQA": 0.2584, - "hfopenllm_v2/MUSR": 0.3436, - "hfopenllm_v2/MMLU-PRO": 0.1218 - } - }, - { - "id": "allura-org/MS-Meadowlark-22B", - "name": "MS-Meadowlark-22B", - "developer": "allura-org", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6697, - "hfopenllm_v2/BBH": 0.5163, - "hfopenllm_v2/MATH Level 5": 0.1835, - "hfopenllm_v2/GPQA": 0.3255, - "hfopenllm_v2/MUSR": 0.3843, - "hfopenllm_v2/MMLU-PRO": 0.3823 - } - }, - { - "id": "allura-org/Teleut-7b", - "name": "Teleut-7b", - "developer": "allura-org", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6379, - "hfopenllm_v2/BBH": 0.5141, - "hfopenllm_v2/MATH Level 5": 0.2409, - "hfopenllm_v2/GPQA": 0.3263, - "hfopenllm_v2/MUSR": 0.464, - "hfopenllm_v2/MMLU-PRO": 0.4131 - } - }, - { - "id": "allura-org/TQ2.5-14B-Aletheia-v1", - "name": "TQ2.5-14B-Aletheia-v1", - "developer": "allura-org", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.753, - "hfopenllm_v2/BBH": 0.6585, - "hfopenllm_v2/MATH Level 5": 0.3399, - "hfopenllm_v2/GPQA": 0.3624, - "hfopenllm_v2/MUSR": 0.4452, - "hfopenllm_v2/MMLU-PRO": 0.5241 - } - }, - { - "id": "allura-org/TQ2.5-14B-Neon-v1", - "name": "TQ2.5-14B-Neon-v1", - "developer": "allura-org", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6754, - "hfopenllm_v2/BBH": 0.6553, - "hfopenllm_v2/MATH Level 5": 0.3603, - "hfopenllm_v2/GPQA": 0.3716, - "hfopenllm_v2/MUSR": 0.461, - "hfopenllm_v2/MMLU-PRO": 0.5253 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/aloobun.json b/data/developers/aloobun.json deleted file mode 100644 index 234facc2bcdd2373c6bdd61bffca95724f2875db..0000000000000000000000000000000000000000 --- a/data/developers/aloobun.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "aloobun", - "models": [ - { - "id": "aloobun/d-SmolLM2-360M", - "name": "d-SmolLM2-360M", - "developer": "aloobun", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2097, - "hfopenllm_v2/BBH": 0.3196, - "hfopenllm_v2/MATH Level 5": 0.0128, - "hfopenllm_v2/GPQA": 0.2534, - "hfopenllm_v2/MUSR": 0.3981, - "hfopenllm_v2/MMLU-PRO": 0.1169 - } - }, - { - "id": "aloobun/Meta-Llama-3-7B-28Layers", - "name": "Meta-Llama-3-7B-28Layers", - "developer": "aloobun", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1964, - "hfopenllm_v2/BBH": 0.4437, - "hfopenllm_v2/MATH Level 5": 0.0279, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.3589, - "hfopenllm_v2/MMLU-PRO": 0.316 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/alpindale.json b/data/developers/alpindale.json deleted file mode 100644 index 7b1ddfd18f47ab373ad2bed61e1a4ed64086d492..0000000000000000000000000000000000000000 --- a/data/developers/alpindale.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "alpindale", - "models": [ - { - "id": "alpindale/magnum-72b-v1", - "name": "magnum-72b-v1", - "developer": "alpindale", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7606, - "hfopenllm_v2/BBH": 0.6982, - "hfopenllm_v2/MATH Level 5": 0.398, - "hfopenllm_v2/GPQA": 0.3909, - "hfopenllm_v2/MUSR": 0.4489, - "hfopenllm_v2/MMLU-PRO": 0.5468 - } - }, - { - "id": "alpindale/WizardLM-2-8x22B", - "name": "WizardLM-2-8x22B", - "developer": "alpindale", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5272, - "hfopenllm_v2/BBH": 0.6377, - "hfopenllm_v2/MATH Level 5": 0.25, - "hfopenllm_v2/GPQA": 0.3817, - "hfopenllm_v2/MUSR": 0.4387, - "hfopenllm_v2/MMLU-PRO": 0.4596 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/alsebay.json b/data/developers/alsebay.json deleted file mode 100644 index 1fe37a631a47cb31729718905ced5476eddcc6e9..0000000000000000000000000000000000000000 --- a/data/developers/alsebay.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "Alsebay", - "models": [ - { - "id": "Alsebay/Qwen2.5-7B-test-novelist", - "name": "Qwen2.5-7B-test-novelist", - "developer": "Alsebay", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5352, - "hfopenllm_v2/BBH": 0.5151, - "hfopenllm_v2/MATH Level 5": 0.2349, - "hfopenllm_v2/GPQA": 0.2911, - "hfopenllm_v2/MUSR": 0.4749, - "hfopenllm_v2/MMLU-PRO": 0.3866 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/altomek.json b/data/developers/altomek.json deleted file mode 100644 index a03cfa0efb1312f43ad7d237a6113f60efb95585..0000000000000000000000000000000000000000 --- a/data/developers/altomek.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "altomek", - "models": [ - { - "id": "altomek/YiSM-34B-0rn", - "name": "YiSM-34B-0rn", - "developer": "altomek", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4284, - "hfopenllm_v2/BBH": 0.614, - "hfopenllm_v2/MATH Level 5": 0.2281, - "hfopenllm_v2/GPQA": 0.3716, - "hfopenllm_v2/MUSR": 0.445, - "hfopenllm_v2/MMLU-PRO": 0.4696 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/amaorynho.json b/data/developers/amaorynho.json deleted file mode 100644 index 97e19559e8a14c6dbbae392c04a87041ddb2006a..0000000000000000000000000000000000000000 --- a/data/developers/amaorynho.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "developer": "Amaorynho", - "models": [ - { - "id": "Amaorynho/BBAI2006", - "name": "BBAI2006", - "developer": "Amaorynho", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1467, - "hfopenllm_v2/BBH": 0.2704, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2525, - "hfopenllm_v2/MUSR": 0.3605, - "hfopenllm_v2/MMLU-PRO": 0.1123 - } - }, - { - "id": "Amaorynho/BBAI270V4", - "name": "BBAI270V4", - "developer": "Amaorynho", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.199, - "hfopenllm_v2/BBH": 0.3071, - "hfopenllm_v2/MATH Level 5": 0.0083, - "hfopenllm_v2/GPQA": 0.2458, - "hfopenllm_v2/MUSR": 0.3314, - "hfopenllm_v2/MMLU-PRO": 0.1114 - } - }, - { - "id": "Amaorynho/BBAI_375", - "name": "BBAI_375", - "developer": "Amaorynho", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1467, - "hfopenllm_v2/BBH": 0.2704, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2525, - "hfopenllm_v2/MUSR": 0.3605, - "hfopenllm_v2/MMLU-PRO": 0.1123 - } - }, - { - "id": "Amaorynho/BBAIIFEV1", - "name": "BBAIIFEV1", - "developer": "Amaorynho", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8047, - "hfopenllm_v2/BBH": 0.5292, - "hfopenllm_v2/MATH Level 5": 0.1934, - "hfopenllm_v2/GPQA": 0.3104, - "hfopenllm_v2/MUSR": 0.4185, - "hfopenllm_v2/MMLU-PRO": 0.3857 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/amazon.json b/data/developers/amazon.json deleted file mode 100644 index 270f2ddd3eb7fdb318cb42628dd795524bd860f5..0000000000000000000000000000000000000000 --- a/data/developers/amazon.json +++ /dev/null @@ -1,327 +0,0 @@ -{ - "developer": "amazon", - "models": [ - { - "id": "amazon/amazon-nova-2-lite-v1-0-fc", - "name": "Amazon-Nova-2-Lite-v1:0 (FC)", - "developer": "amazon", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 80.0, - "bfcl/bfcl.overall.overall_accuracy": 27.1, - "bfcl/bfcl.overall.total_cost_usd": 78.19, - "bfcl/bfcl.overall.latency_mean_s": 8.55, - "bfcl/bfcl.overall.latency_std_s": 9.85, - "bfcl/bfcl.overall.latency_p95_s": 27.62, - "bfcl/bfcl.non_live.ast_accuracy": 86.96, - "bfcl/bfcl.non_live.simple_ast_accuracy": 76.33, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 94.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 91.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 86.0, - "bfcl/bfcl.live.live_accuracy": 80.83, - "bfcl/bfcl.live.live_simple_ast_accuracy": 83.33, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 80.15, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 87.5, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 79.17, - "bfcl/bfcl.multi_turn.accuracy": 2.12, - "bfcl/bfcl.multi_turn.base_accuracy": 2.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 1.5, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 2.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 2.5, - "bfcl/bfcl.web_search.accuracy": 5.0, - "bfcl/bfcl.web_search.base_accuracy": 4.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 6.0, - "bfcl/bfcl.memory.accuracy": 2.37, - "bfcl/bfcl.memory.kv_accuracy": 1.94, - "bfcl/bfcl.memory.vector_accuracy": 3.23, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 1.94, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 75.0, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 82.11 - } - }, - { - "id": "amazon/amazon-nova-micro-v1-0-fc", - "name": "Amazon-Nova-Micro-v1:0 (FC)", - "developer": "amazon", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 95.0, - "bfcl/bfcl.overall.overall_accuracy": 22.29, - "bfcl/bfcl.overall.total_cost_usd": 1.81, - "bfcl/bfcl.overall.latency_mean_s": 1.12, - "bfcl/bfcl.overall.latency_std_s": 0.45, - "bfcl/bfcl.overall.latency_p95_s": 1.79, - "bfcl/bfcl.non_live.ast_accuracy": 74.1, - "bfcl/bfcl.non_live.simple_ast_accuracy": 70.92, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 87.5, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 75.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 62.5, - "bfcl/bfcl.live.live_accuracy": 66.32, - "bfcl/bfcl.live.live_simple_ast_accuracy": 72.09, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 64.96, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 81.25, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 54.17, - "bfcl/bfcl.multi_turn.accuracy": 1.38, - "bfcl/bfcl.multi_turn.base_accuracy": 1.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 1.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 2.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 1.0, - "bfcl/bfcl.web_search.accuracy": 1.5, - "bfcl/bfcl.web_search.base_accuracy": 3.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 0.0, - "bfcl/bfcl.memory.accuracy": 2.37, - "bfcl/bfcl.memory.kv_accuracy": 1.94, - "bfcl/bfcl.memory.vector_accuracy": 2.58, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 2.58, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 81.25, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 70.65 - } - }, - { - "id": "amazon/amazon-nova-pro-v1-0-fc", - "name": "Amazon-Nova-Pro-v1:0 (FC)", - "developer": "amazon", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 88.0, - "bfcl/bfcl.overall.overall_accuracy": 24.97, - "bfcl/bfcl.overall.total_cost_usd": 48.44, - "bfcl/bfcl.overall.latency_mean_s": 2.25, - "bfcl/bfcl.overall.latency_std_s": 1.91, - "bfcl/bfcl.overall.latency_p95_s": 3.29, - "bfcl/bfcl.non_live.ast_accuracy": 86.58, - "bfcl/bfcl.non_live.simple_ast_accuracy": 75.83, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 93.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 93.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 84.0, - "bfcl/bfcl.live.live_accuracy": 78.53, - "bfcl/bfcl.live.live_simple_ast_accuracy": 81.4, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 77.97, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 81.25, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 70.83, - "bfcl/bfcl.multi_turn.accuracy": 1.88, - "bfcl/bfcl.multi_turn.base_accuracy": 1.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 0.5, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 2.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 3.0, - "bfcl/bfcl.web_search.accuracy": 2.5, - "bfcl/bfcl.web_search.base_accuracy": 4.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 1.0, - "bfcl/bfcl.memory.accuracy": 1.94, - "bfcl/bfcl.memory.kv_accuracy": 2.58, - "bfcl/bfcl.memory.vector_accuracy": 1.29, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 1.94, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 93.75, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 70.06 - } - }, - { - "id": "amazon/MegaBeam-Mistral-7B-300k", - "name": "MegaBeam-Mistral-7B-300k", - "developer": "amazon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5203, - "hfopenllm_v2/BBH": 0.4228, - "hfopenllm_v2/MATH Level 5": 0.0211, - "hfopenllm_v2/GPQA": 0.2735, - "hfopenllm_v2/MUSR": 0.398, - "hfopenllm_v2/MMLU-PRO": 0.2549 - } - }, - { - "id": "amazon/nova-lite-v1:0", - "name": "Amazon Nova Lite", - "developer": "amazon", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.551, - "helm_capabilities/MMLU-Pro": 0.6, - "helm_capabilities/GPQA": 0.397, - "helm_capabilities/IFEval": 0.776, - "helm_capabilities/WildBench": 0.75, - "helm_capabilities/Omni-MATH": 0.233, - "helm_lite/Mean win rate": 0.708, - "helm_lite/NarrativeQA": 0.768, - "helm_lite/NaturalQuestions (closed-book)": 0.352, - "helm_lite/OpenbookQA": 0.928, - "helm_lite/MMLU": 0.693, - "helm_lite/MATH": 0.779, - "helm_lite/GSM8K": 0.829, - "helm_lite/LegalBench": 0.659, - "helm_lite/MedQA": 0.696, - "helm_lite/WMT 2014": 0.204, - "helm_mmlu/MMLU All Subjects": 0.77, - "helm_mmlu/Abstract Algebra": 0.52, - "helm_mmlu/Anatomy": 0.719, - "helm_mmlu/College Physics": 0.608, - "helm_mmlu/Computer Security": 0.79, - "helm_mmlu/Econometrics": 0.675, - "helm_mmlu/Global Facts": 0.55, - "helm_mmlu/Jurisprudence": 0.852, - "helm_mmlu/Philosophy": 0.817, - "helm_mmlu/Professional Psychology": 0.812, - "helm_mmlu/Us Foreign Policy": 0.92, - "helm_mmlu/Astronomy": 0.862, - "helm_mmlu/Business Ethics": 0.73, - "helm_mmlu/Clinical Knowledge": 0.8, - "helm_mmlu/Conceptual Physics": 0.796, - "helm_mmlu/Electrical Engineering": 0.779, - "helm_mmlu/Elementary Mathematics": 0.757, - "helm_mmlu/Formal Logic": 0.643, - "helm_mmlu/High School World History": 0.886, - "helm_mmlu/Human Sexuality": 0.84, - "helm_mmlu/International Law": 0.843, - "helm_mmlu/Logical Fallacies": 0.81, - "helm_mmlu/Machine Learning": 0.509, - "helm_mmlu/Management": 0.864, - "helm_mmlu/Marketing": 0.889, - "helm_mmlu/Medical Genetics": 0.9, - "helm_mmlu/Miscellaneous": 0.872, - "helm_mmlu/Moral Scenarios": 0.694, - "helm_mmlu/Nutrition": 0.788, - "helm_mmlu/Prehistory": 0.849, - "helm_mmlu/Public Relations": 0.682, - "helm_mmlu/Security Studies": 0.788, - "helm_mmlu/Sociology": 0.896, - "helm_mmlu/Virology": 0.542, - "helm_mmlu/World Religions": 0.871, - "helm_mmlu/Mean win rate": 0.987 - } - }, - { - "id": "amazon/nova-micro-v1:0", - "name": "Amazon Nova Micro", - "developer": "amazon", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.522, - "helm_capabilities/MMLU-Pro": 0.511, - "helm_capabilities/GPQA": 0.383, - "helm_capabilities/IFEval": 0.76, - "helm_capabilities/WildBench": 0.743, - "helm_capabilities/Omni-MATH": 0.214, - "helm_lite/Mean win rate": 0.524, - "helm_lite/NarrativeQA": 0.744, - "helm_lite/NaturalQuestions (closed-book)": 0.285, - "helm_lite/OpenbookQA": 0.888, - "helm_lite/MMLU": 0.64, - "helm_lite/MATH": 0.76, - "helm_lite/GSM8K": 0.794, - "helm_lite/LegalBench": 0.615, - "helm_lite/MedQA": 0.608, - "helm_lite/WMT 2014": 0.192, - "helm_mmlu/MMLU All Subjects": 0.708, - "helm_mmlu/Abstract Algebra": 0.42, - "helm_mmlu/Anatomy": 0.726, - "helm_mmlu/College Physics": 0.5, - "helm_mmlu/Computer Security": 0.77, - "helm_mmlu/Econometrics": 0.57, - "helm_mmlu/Global Facts": 0.44, - "helm_mmlu/Jurisprudence": 0.815, - "helm_mmlu/Philosophy": 0.733, - "helm_mmlu/Professional Psychology": 0.739, - "helm_mmlu/Us Foreign Policy": 0.9, - "helm_mmlu/Astronomy": 0.822, - "helm_mmlu/Business Ethics": 0.71, - "helm_mmlu/Clinical Knowledge": 0.751, - "helm_mmlu/Conceptual Physics": 0.706, - "helm_mmlu/Electrical Engineering": 0.683, - "helm_mmlu/Elementary Mathematics": 0.55, - "helm_mmlu/Formal Logic": 0.508, - "helm_mmlu/High School World History": 0.84, - "helm_mmlu/Human Sexuality": 0.824, - "helm_mmlu/International Law": 0.843, - "helm_mmlu/Logical Fallacies": 0.798, - "helm_mmlu/Machine Learning": 0.562, - "helm_mmlu/Management": 0.816, - "helm_mmlu/Marketing": 0.91, - "helm_mmlu/Medical Genetics": 0.82, - "helm_mmlu/Miscellaneous": 0.83, - "helm_mmlu/Moral Scenarios": 0.464, - "helm_mmlu/Nutrition": 0.778, - "helm_mmlu/Prehistory": 0.787, - "helm_mmlu/Public Relations": 0.673, - "helm_mmlu/Security Studies": 0.718, - "helm_mmlu/Sociology": 0.846, - "helm_mmlu/Virology": 0.524, - "helm_mmlu/World Religions": 0.825, - "helm_mmlu/Mean win rate": 1.0 - } - }, - { - "id": "amazon/nova-premier-v1:0", - "name": "Amazon Nova Premier", - "developer": "amazon", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.637, - "helm_capabilities/MMLU-Pro": 0.726, - "helm_capabilities/GPQA": 0.518, - "helm_capabilities/IFEval": 0.803, - "helm_capabilities/WildBench": 0.788, - "helm_capabilities/Omni-MATH": 0.35 - } - }, - { - "id": "amazon/nova-pro-v1:0", - "name": "Amazon Nova Pro", - "developer": "amazon", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.591, - "helm_capabilities/MMLU-Pro": 0.673, - "helm_capabilities/GPQA": 0.446, - "helm_capabilities/IFEval": 0.815, - "helm_capabilities/WildBench": 0.777, - "helm_capabilities/Omni-MATH": 0.242, - "helm_lite/Mean win rate": 0.885, - "helm_lite/NarrativeQA": 0.791, - "helm_lite/NaturalQuestions (closed-book)": 0.405, - "helm_lite/OpenbookQA": 0.96, - "helm_lite/MMLU": 0.758, - "helm_lite/MATH": 0.821, - "helm_lite/GSM8K": 0.87, - "helm_lite/LegalBench": 0.736, - "helm_lite/MedQA": 0.811, - "helm_lite/WMT 2014": 0.229, - "helm_mmlu/MMLU All Subjects": 0.82, - "helm_mmlu/Abstract Algebra": 0.69, - "helm_mmlu/Anatomy": 0.807, - "helm_mmlu/College Physics": 0.647, - "helm_mmlu/Computer Security": 0.84, - "helm_mmlu/Econometrics": 0.702, - "helm_mmlu/Global Facts": 0.54, - "helm_mmlu/Jurisprudence": 0.861, - "helm_mmlu/Philosophy": 0.826, - "helm_mmlu/Professional Psychology": 0.864, - "helm_mmlu/Us Foreign Policy": 0.93, - "helm_mmlu/Astronomy": 0.895, - "helm_mmlu/Business Ethics": 0.81, - "helm_mmlu/Clinical Knowledge": 0.875, - "helm_mmlu/Conceptual Physics": 0.851, - "helm_mmlu/Electrical Engineering": 0.8, - "helm_mmlu/Elementary Mathematics": 0.831, - "helm_mmlu/Formal Logic": 0.714, - "helm_mmlu/High School World History": 0.928, - "helm_mmlu/Human Sexuality": 0.885, - "helm_mmlu/International Law": 0.901, - "helm_mmlu/Logical Fallacies": 0.871, - "helm_mmlu/Machine Learning": 0.625, - "helm_mmlu/Management": 0.922, - "helm_mmlu/Marketing": 0.923, - "helm_mmlu/Medical Genetics": 0.87, - "helm_mmlu/Miscellaneous": 0.912, - "helm_mmlu/Moral Scenarios": 0.76, - "helm_mmlu/Nutrition": 0.866, - "helm_mmlu/Prehistory": 0.926, - "helm_mmlu/Public Relations": 0.8, - "helm_mmlu/Security Studies": 0.849, - "helm_mmlu/Sociology": 0.905, - "helm_mmlu/Virology": 0.59, - "helm_mmlu/World Religions": 0.877, - "helm_mmlu/Mean win rate": 0.975 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/amd.json b/data/developers/amd.json deleted file mode 100644 index ec85a4364a761e30f1999f359d0d247d8857e139..0000000000000000000000000000000000000000 --- a/data/developers/amd.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "amd", - "models": [ - { - "id": "amd/AMD-Llama-135m", - "name": "AMD-Llama-135m", - "developer": "amd", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1842, - "hfopenllm_v2/BBH": 0.2974, - "hfopenllm_v2/MATH Level 5": 0.0053, - "hfopenllm_v2/GPQA": 0.2525, - "hfopenllm_v2/MUSR": 0.378, - "hfopenllm_v2/MMLU-PRO": 0.1169 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/amu.json b/data/developers/amu.json deleted file mode 100644 index cdcb9b2bf1d0155e992dd2cb1e040b235ebb7c33..0000000000000000000000000000000000000000 --- a/data/developers/amu.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "Amu", - "models": [ - { - "id": "Amu/t1-1.5B", - "name": "t1-1.5B", - "developer": "Amu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3394, - "hfopenllm_v2/BBH": 0.4008, - "hfopenllm_v2/MATH Level 5": 0.0514, - "hfopenllm_v2/GPQA": 0.2433, - "hfopenllm_v2/MUSR": 0.3517, - "hfopenllm_v2/MMLU-PRO": 0.2566 - } - }, - { - "id": "Amu/t1-3B", - "name": "t1-3B", - "developer": "Amu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3328, - "hfopenllm_v2/BBH": 0.3999, - "hfopenllm_v2/MATH Level 5": 0.1375, - "hfopenllm_v2/GPQA": 0.2408, - "hfopenllm_v2/MUSR": 0.3435, - "hfopenllm_v2/MMLU-PRO": 0.1284 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/anakin87.json b/data/developers/anakin87.json deleted file mode 100644 index 451701c03c05f06014d98ab731d7b6819d4a5185..0000000000000000000000000000000000000000 --- a/data/developers/anakin87.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "anakin87", - "models": [ - { - "id": "anakin87/gemma-2b-orpo", - "name": "gemma-2b-orpo", - "developer": "anakin87", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2478, - "hfopenllm_v2/BBH": 0.3426, - "hfopenllm_v2/MATH Level 5": 0.0189, - "hfopenllm_v2/GPQA": 0.2617, - "hfopenllm_v2/MUSR": 0.3728, - "hfopenllm_v2/MMLU-PRO": 0.1306 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/anthracite-org.json b/data/developers/anthracite-org.json deleted file mode 100644 index ea1ebc2ec69855fa07e44ad5f7f899ef47b408dd..0000000000000000000000000000000000000000 --- a/data/developers/anthracite-org.json +++ /dev/null @@ -1,173 +0,0 @@ -{ - "developer": "anthracite-org", - "models": [ - { - "id": "anthracite-org/magnum-v1-72b", - "name": "magnum-v1-72b", - "developer": "anthracite-org", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7606, - "hfopenllm_v2/BBH": 0.6982, - "hfopenllm_v2/MATH Level 5": 0.398, - "hfopenllm_v2/GPQA": 0.3909, - "hfopenllm_v2/MUSR": 0.4489, - "hfopenllm_v2/MMLU-PRO": 0.5486 - } - }, - { - "id": "anthracite-org/magnum-v2-12b", - "name": "magnum-v2-12b", - "developer": "anthracite-org", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3762, - "hfopenllm_v2/BBH": 0.5021, - "hfopenllm_v2/MATH Level 5": 0.0544, - "hfopenllm_v2/GPQA": 0.2911, - "hfopenllm_v2/MUSR": 0.4179, - "hfopenllm_v2/MMLU-PRO": 0.3167 - } - }, - { - "id": "anthracite-org/magnum-v2-72b", - "name": "magnum-v2-72b", - "developer": "anthracite-org", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.756, - "hfopenllm_v2/BBH": 0.7005, - "hfopenllm_v2/MATH Level 5": 0.3542, - "hfopenllm_v2/GPQA": 0.3859, - "hfopenllm_v2/MUSR": 0.4372, - "hfopenllm_v2/MMLU-PRO": 0.5456 - } - }, - { - "id": "anthracite-org/magnum-v2.5-12b-kto", - "name": "magnum-v2.5-12b-kto", - "developer": "anthracite-org", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3866, - "hfopenllm_v2/BBH": 0.5077, - "hfopenllm_v2/MATH Level 5": 0.0521, - "hfopenllm_v2/GPQA": 0.2936, - "hfopenllm_v2/MUSR": 0.4086, - "hfopenllm_v2/MMLU-PRO": 0.3215 - } - }, - { - "id": "anthracite-org/magnum-v3-27b-kto", - "name": "magnum-v3-27b-kto", - "developer": "anthracite-org", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5675, - "hfopenllm_v2/BBH": 0.586, - "hfopenllm_v2/MATH Level 5": 0.1813, - "hfopenllm_v2/GPQA": 0.3557, - "hfopenllm_v2/MUSR": 0.3855, - "hfopenllm_v2/MMLU-PRO": 0.4238 - } - }, - { - "id": "anthracite-org/magnum-v3-34b", - "name": "magnum-v3-34b", - "developer": "anthracite-org", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5115, - "hfopenllm_v2/BBH": 0.6088, - "hfopenllm_v2/MATH Level 5": 0.1949, - "hfopenllm_v2/GPQA": 0.3607, - "hfopenllm_v2/MUSR": 0.3872, - "hfopenllm_v2/MMLU-PRO": 0.4752 - } - }, - { - "id": "anthracite-org/magnum-v3-9b-chatml", - "name": "magnum-v3-9b-chatml", - "developer": "anthracite-org", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1275, - "hfopenllm_v2/BBH": 0.5428, - "hfopenllm_v2/MATH Level 5": 0.0695, - "hfopenllm_v2/GPQA": 0.3456, - "hfopenllm_v2/MUSR": 0.4432, - "hfopenllm_v2/MMLU-PRO": 0.4242 - } - }, - { - "id": "anthracite-org/magnum-v3-9b-customgemma2", - "name": "magnum-v3-9b-customgemma2", - "developer": "anthracite-org", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1273, - "hfopenllm_v2/BBH": 0.534, - "hfopenllm_v2/MATH Level 5": 0.0718, - "hfopenllm_v2/GPQA": 0.3289, - "hfopenllm_v2/MUSR": 0.4565, - "hfopenllm_v2/MMLU-PRO": 0.4205 - } - }, - { - "id": "anthracite-org/magnum-v4-12b", - "name": "magnum-v4-12b", - "developer": "anthracite-org", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3393, - "hfopenllm_v2/BBH": 0.5177, - "hfopenllm_v2/MATH Level 5": 0.1178, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.4093, - "hfopenllm_v2/MMLU-PRO": 0.3604 - } - }, - { - "id": "anthracite-org/magnum-v4-22b", - "name": "magnum-v4-22b", - "developer": "anthracite-org", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5629, - "hfopenllm_v2/BBH": 0.5486, - "hfopenllm_v2/MATH Level 5": 0.2002, - "hfopenllm_v2/GPQA": 0.328, - "hfopenllm_v2/MUSR": 0.4408, - "hfopenllm_v2/MMLU-PRO": 0.383 - } - }, - { - "id": "anthracite-org/magnum-v4-27b", - "name": "magnum-v4-27b", - "developer": "anthracite-org", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3454, - "hfopenllm_v2/BBH": 0.5867, - "hfopenllm_v2/MATH Level 5": 0.1798, - "hfopenllm_v2/GPQA": 0.37, - "hfopenllm_v2/MUSR": 0.438, - "hfopenllm_v2/MMLU-PRO": 0.4376 - } - }, - { - "id": "anthracite-org/magnum-v4-9b", - "name": "magnum-v4-9b", - "developer": "anthracite-org", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3503, - "hfopenllm_v2/BBH": 0.5336, - "hfopenllm_v2/MATH Level 5": 0.1307, - "hfopenllm_v2/GPQA": 0.3473, - "hfopenllm_v2/MUSR": 0.4516, - "hfopenllm_v2/MMLU-PRO": 0.3953 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/anthropic.json b/data/developers/anthropic.json deleted file mode 100644 index 78de20ed67b0f7683021c0fe9f2000b1dad9b2e2..0000000000000000000000000000000000000000 --- a/data/developers/anthropic.json +++ /dev/null @@ -1,1070 +0,0 @@ -{ - "developer": "Anthropic", - "models": [ - { - "id": "anthropic/claude-2.0", - "name": "Claude 2.0", - "developer": "Anthropic", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.489, - "helm_lite/NarrativeQA": 0.718, - "helm_lite/NaturalQuestions (closed-book)": 0.428, - "helm_lite/OpenbookQA": 0.862, - "helm_lite/MMLU": 0.639, - "helm_lite/MATH": 0.603, - "helm_lite/GSM8K": 0.583, - "helm_lite/LegalBench": 0.643, - "helm_lite/MedQA": 0.652, - "helm_lite/WMT 2014": 0.219 - } - }, - { - "id": "anthropic/claude-2.1", - "name": "Claude 2.1", - "developer": "Anthropic", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.437, - "helm_lite/NarrativeQA": 0.677, - "helm_lite/NaturalQuestions (closed-book)": 0.375, - "helm_lite/OpenbookQA": 0.872, - "helm_lite/MMLU": 0.643, - "helm_lite/MATH": 0.632, - "helm_lite/GSM8K": 0.604, - "helm_lite/LegalBench": 0.643, - "helm_lite/MedQA": 0.644, - "helm_lite/WMT 2014": 0.204, - "helm_mmlu/MMLU All Subjects": 0.735, - "helm_mmlu/Abstract Algebra": 0.4, - "helm_mmlu/Anatomy": 0.726, - "helm_mmlu/College Physics": 0.5, - "helm_mmlu/Computer Security": 0.81, - "helm_mmlu/Econometrics": 0.596, - "helm_mmlu/Global Facts": 0.55, - "helm_mmlu/Jurisprudence": 0.87, - "helm_mmlu/Philosophy": 0.794, - "helm_mmlu/Professional Psychology": 0.797, - "helm_mmlu/Us Foreign Policy": 0.92, - "helm_mmlu/Astronomy": 0.855, - "helm_mmlu/Business Ethics": 0.73, - "helm_mmlu/Clinical Knowledge": 0.785, - "helm_mmlu/Conceptual Physics": 0.766, - "helm_mmlu/Electrical Engineering": 0.724, - "helm_mmlu/Elementary Mathematics": 0.521, - "helm_mmlu/Formal Logic": 0.5, - "helm_mmlu/High School World History": 0.903, - "helm_mmlu/Human Sexuality": 0.847, - "helm_mmlu/International Law": 0.901, - "helm_mmlu/Logical Fallacies": 0.834, - "helm_mmlu/Machine Learning": 0.482, - "helm_mmlu/Management": 0.825, - "helm_mmlu/Marketing": 0.923, - "helm_mmlu/Medical Genetics": 0.81, - "helm_mmlu/Miscellaneous": 0.88, - "helm_mmlu/Moral Scenarios": 0.52, - "helm_mmlu/Nutrition": 0.781, - "helm_mmlu/Prehistory": 0.821, - "helm_mmlu/Public Relations": 0.773, - "helm_mmlu/Security Studies": 0.812, - "helm_mmlu/Sociology": 0.886, - "helm_mmlu/Virology": 0.554, - "helm_mmlu/World Religions": 0.854, - "helm_mmlu/Mean win rate": 0.048 - } - }, - { - "id": "anthropic/claude-3-5-haiku-20241022", - "name": "Claude 3.5 Haiku 20241022", - "developer": "Anthropic", - "evaluator_relationship": null, - "benchmark_scores": { - "global-mmlu-lite/Global MMLU Lite": 0.6114, - "global-mmlu-lite/Culturally Sensitive": 0.5834, - "global-mmlu-lite/Culturally Agnostic": 0.6394, - "global-mmlu-lite/Arabic": 0.695, - "global-mmlu-lite/English": 0.485, - "global-mmlu-lite/Bengali": 0.675, - "global-mmlu-lite/German": 0.565, - "global-mmlu-lite/French": 0.61, - "global-mmlu-lite/Hindi": 0.6575, - "global-mmlu-lite/Indonesian": 0.5475, - "global-mmlu-lite/Italian": 0.48, - "global-mmlu-lite/Japanese": 0.655, - "global-mmlu-lite/Korean": 0.6575, - "global-mmlu-lite/Portuguese": 0.5225, - "global-mmlu-lite/Spanish": 0.485, - "global-mmlu-lite/Swahili": 0.69, - "global-mmlu-lite/Yoruba": 0.6675, - "global-mmlu-lite/Chinese": 0.69, - "global-mmlu-lite/Burmese": 0.7, - "helm_capabilities/Mean score": 0.549, - "helm_capabilities/MMLU-Pro": 0.605, - "helm_capabilities/GPQA": 0.363, - "helm_capabilities/IFEval": 0.792, - "helm_capabilities/WildBench": 0.76, - "helm_capabilities/Omni-MATH": 0.224, - "helm_lite/Mean win rate": 0.531, - "helm_lite/NarrativeQA": 0.763, - "helm_lite/NaturalQuestions (closed-book)": 0.344, - "helm_lite/OpenbookQA": 0.854, - "helm_lite/MMLU": 0.671, - "helm_lite/MATH": 0.872, - "helm_lite/GSM8K": 0.815, - "helm_lite/LegalBench": 0.631, - "helm_lite/MedQA": 0.722, - "helm_lite/WMT 2014": 0.135, - "helm_mmlu/MMLU All Subjects": 0.743, - "helm_mmlu/Abstract Algebra": 0.47, - "helm_mmlu/Anatomy": 0.793, - "helm_mmlu/College Physics": 0.52, - "helm_mmlu/Computer Security": 0.84, - "helm_mmlu/Econometrics": 0.596, - "helm_mmlu/Global Facts": 0.5, - "helm_mmlu/Jurisprudence": 0.861, - "helm_mmlu/Philosophy": 0.823, - "helm_mmlu/Professional Psychology": 0.825, - "helm_mmlu/Us Foreign Policy": 0.94, - "helm_mmlu/Astronomy": 0.829, - "helm_mmlu/Business Ethics": 0.8, - "helm_mmlu/Clinical Knowledge": 0.823, - "helm_mmlu/Conceptual Physics": 0.723, - "helm_mmlu/Electrical Engineering": 0.717, - "helm_mmlu/Elementary Mathematics": 0.561, - "helm_mmlu/Formal Logic": 0.619, - "helm_mmlu/High School World History": 0.882, - "helm_mmlu/Human Sexuality": 0.885, - "helm_mmlu/International Law": 0.884, - "helm_mmlu/Logical Fallacies": 0.822, - "helm_mmlu/Machine Learning": 0.518, - "helm_mmlu/Management": 0.845, - "helm_mmlu/Marketing": 0.897, - "helm_mmlu/Medical Genetics": 0.83, - "helm_mmlu/Miscellaneous": 0.905, - "helm_mmlu/Moral Scenarios": 0.476, - "helm_mmlu/Nutrition": 0.846, - "helm_mmlu/Prehistory": 0.877, - "helm_mmlu/Public Relations": 0.727, - "helm_mmlu/Security Studies": 0.792, - "helm_mmlu/Sociology": 0.905, - "helm_mmlu/Virology": 0.566, - "helm_mmlu/World Religions": 0.865, - "helm_mmlu/Mean win rate": 0.128 - } - }, - { - "id": "anthropic/claude-3-5-sonnet-20240620", - "name": "Claude 3.5 Sonnet 20240620", - "developer": "Anthropic", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.885, - "helm_lite/NarrativeQA": 0.746, - "helm_lite/NaturalQuestions (closed-book)": 0.502, - "helm_lite/OpenbookQA": 0.972, - "helm_lite/MMLU": 0.799, - "helm_lite/MATH": 0.813, - "helm_lite/GSM8K": 0.949, - "helm_lite/LegalBench": 0.707, - "helm_lite/MedQA": 0.825, - "helm_lite/WMT 2014": 0.229, - "helm_mmlu/MMLU All Subjects": 0.865, - "helm_mmlu/Abstract Algebra": 0.75, - "helm_mmlu/Anatomy": 0.844, - "helm_mmlu/College Physics": 0.696, - "helm_mmlu/Computer Security": 0.89, - "helm_mmlu/Econometrics": 0.807, - "helm_mmlu/Global Facts": 0.72, - "helm_mmlu/Jurisprudence": 0.889, - "helm_mmlu/Philosophy": 0.891, - "helm_mmlu/Professional Psychology": 0.922, - "helm_mmlu/Us Foreign Policy": 0.96, - "helm_mmlu/Astronomy": 0.961, - "helm_mmlu/Business Ethics": 0.85, - "helm_mmlu/Clinical Knowledge": 0.913, - "helm_mmlu/Conceptual Physics": 0.885, - "helm_mmlu/Electrical Engineering": 0.828, - "helm_mmlu/Elementary Mathematics": 0.892, - "helm_mmlu/Formal Logic": 0.698, - "helm_mmlu/High School World History": 0.954, - "helm_mmlu/Human Sexuality": 0.939, - "helm_mmlu/International Law": 0.959, - "helm_mmlu/Logical Fallacies": 0.926, - "helm_mmlu/Machine Learning": 0.786, - "helm_mmlu/Management": 0.942, - "helm_mmlu/Marketing": 0.949, - "helm_mmlu/Medical Genetics": 0.98, - "helm_mmlu/Miscellaneous": 0.962, - "helm_mmlu/Moral Scenarios": 0.882, - "helm_mmlu/Nutrition": 0.912, - "helm_mmlu/Prehistory": 0.951, - "helm_mmlu/Public Relations": 0.855, - "helm_mmlu/Security Studies": 0.878, - "helm_mmlu/Sociology": 0.96, - "helm_mmlu/Virology": 0.602, - "helm_mmlu/World Religions": 0.924, - "helm_mmlu/Mean win rate": 0.17, - "reward-bench/Score": 0.6466, - "reward-bench/Chat": 0.9637, - "reward-bench/Chat Hard": 0.7401, - "reward-bench/Safety": 0.8519, - "reward-bench/Reasoning": 0.8469, - "reward-bench/Factuality": 0.5284, - "reward-bench/Precise IF": 0.3875, - "reward-bench/Math": 0.5683, - "reward-bench/Focus": 0.8697, - "reward-bench/Ties": 0.674 - } - }, - { - "id": "anthropic/claude-3-5-sonnet-20241022", - "name": "Claude 3.5 Sonnet 20241022", - "developer": "Anthropic", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.653, - "helm_capabilities/MMLU-Pro": 0.777, - "helm_capabilities/GPQA": 0.565, - "helm_capabilities/IFEval": 0.856, - "helm_capabilities/WildBench": 0.792, - "helm_capabilities/Omni-MATH": 0.276, - "helm_lite/Mean win rate": 0.846, - "helm_lite/NarrativeQA": 0.77, - "helm_lite/NaturalQuestions (closed-book)": 0.467, - "helm_lite/OpenbookQA": 0.966, - "helm_lite/MMLU": 0.809, - "helm_lite/MATH": 0.904, - "helm_lite/GSM8K": 0.956, - "helm_lite/LegalBench": 0.647, - "helm_lite/MedQA": 0.859, - "helm_lite/WMT 2014": 0.226, - "helm_mmlu/MMLU All Subjects": 0.873, - "helm_mmlu/Abstract Algebra": 0.78, - "helm_mmlu/Anatomy": 0.859, - "helm_mmlu/College Physics": 0.775, - "helm_mmlu/Computer Security": 0.87, - "helm_mmlu/Econometrics": 0.807, - "helm_mmlu/Global Facts": 0.8, - "helm_mmlu/Jurisprudence": 0.898, - "helm_mmlu/Philosophy": 0.891, - "helm_mmlu/Professional Psychology": 0.922, - "helm_mmlu/Us Foreign Policy": 0.96, - "helm_mmlu/Astronomy": 0.974, - "helm_mmlu/Business Ethics": 0.83, - "helm_mmlu/Clinical Knowledge": 0.928, - "helm_mmlu/Conceptual Physics": 0.906, - "helm_mmlu/Electrical Engineering": 0.848, - "helm_mmlu/Elementary Mathematics": 0.918, - "helm_mmlu/Formal Logic": 0.786, - "helm_mmlu/High School World History": 0.958, - "helm_mmlu/Human Sexuality": 0.939, - "helm_mmlu/International Law": 0.959, - "helm_mmlu/Logical Fallacies": 0.914, - "helm_mmlu/Machine Learning": 0.839, - "helm_mmlu/Management": 0.932, - "helm_mmlu/Marketing": 0.953, - "helm_mmlu/Medical Genetics": 0.96, - "helm_mmlu/Miscellaneous": 0.964, - "helm_mmlu/Moral Scenarios": 0.888, - "helm_mmlu/Nutrition": 0.922, - "helm_mmlu/Prehistory": 0.941, - "helm_mmlu/Public Relations": 0.8, - "helm_mmlu/Security Studies": 0.882, - "helm_mmlu/Sociology": 0.955, - "helm_mmlu/Virology": 0.584, - "helm_mmlu/World Religions": 0.901, - "helm_mmlu/Mean win rate": 0.311 - } - }, - { - "id": "anthropic/claude-3-7-sonnet-20250219", - "name": "claude-3-7-sonnet-20250219", - "developer": "Anthropic", - "evaluator_relationship": null, - "benchmark_scores": { - "global-mmlu-lite/Global MMLU Lite": 0.8078, - "global-mmlu-lite/Culturally Sensitive": 0.7794, - "global-mmlu-lite/Culturally Agnostic": 0.8362, - "global-mmlu-lite/Arabic": 0.7925, - "global-mmlu-lite/English": 0.7625, - "global-mmlu-lite/Bengali": 0.825, - "global-mmlu-lite/German": 0.8125, - "global-mmlu-lite/French": 0.7675, - "global-mmlu-lite/Hindi": 0.805, - "global-mmlu-lite/Indonesian": 0.8175, - "global-mmlu-lite/Italian": 0.8225, - "global-mmlu-lite/Japanese": 0.8425, - "global-mmlu-lite/Korean": 0.83, - "global-mmlu-lite/Portuguese": 0.77, - "global-mmlu-lite/Spanish": 0.8075, - "global-mmlu-lite/Swahili": 0.8125, - "global-mmlu-lite/Yoruba": 0.81, - "global-mmlu-lite/Chinese": 0.835, - "global-mmlu-lite/Burmese": 0.8125, - "helm_capabilities/Mean score": 0.674, - "helm_capabilities/MMLU-Pro": 0.784, - "helm_capabilities/GPQA": 0.608, - "helm_capabilities/IFEval": 0.834, - "helm_capabilities/WildBench": 0.814, - "helm_capabilities/Omni-MATH": 0.33, - "livecodebenchpro/Hard Problems": 0.0, - "livecodebenchpro/Medium Problems": 0.0, - "livecodebenchpro/Easy Problems": 0.28169014084507044, - "reward-bench/Score": 0.7539, - "reward-bench/Factuality": 0.7326, - "reward-bench/Precise IF": 0.5437, - "reward-bench/Math": 0.75, - "reward-bench/Safety": 0.9033, - "reward-bench/Focus": 0.9212, - "reward-bench/Ties": 0.6723 - } - }, - { - "id": "anthropic/claude-3-haiku-20240307", - "name": "Claude 3 Haiku 20240307", - "developer": "Anthropic", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.263, - "helm_lite/NarrativeQA": 0.244, - "helm_lite/NaturalQuestions (closed-book)": 0.144, - "helm_lite/OpenbookQA": 0.838, - "helm_lite/MMLU": 0.662, - "helm_lite/MATH": 0.131, - "helm_lite/GSM8K": 0.699, - "helm_lite/LegalBench": 0.46, - "helm_lite/MedQA": 0.702, - "helm_lite/WMT 2014": 0.148, - "helm_mmlu/MMLU All Subjects": 0.738, - "helm_mmlu/Abstract Algebra": 0.42, - "helm_mmlu/Anatomy": 0.711, - "helm_mmlu/College Physics": 0.48, - "helm_mmlu/Computer Security": 0.79, - "helm_mmlu/Econometrics": 0.632, - "helm_mmlu/Global Facts": 0.47, - "helm_mmlu/Jurisprudence": 0.861, - "helm_mmlu/Philosophy": 0.814, - "helm_mmlu/Professional Psychology": 0.802, - "helm_mmlu/Us Foreign Policy": 0.95, - "helm_mmlu/Astronomy": 0.901, - "helm_mmlu/Business Ethics": 0.78, - "helm_mmlu/Clinical Knowledge": 0.789, - "helm_mmlu/Conceptual Physics": 0.715, - "helm_mmlu/Electrical Engineering": 0.69, - "helm_mmlu/Elementary Mathematics": 0.558, - "helm_mmlu/Formal Logic": 0.579, - "helm_mmlu/High School World History": 0.878, - "helm_mmlu/Human Sexuality": 0.824, - "helm_mmlu/International Law": 0.901, - "helm_mmlu/Logical Fallacies": 0.791, - "helm_mmlu/Machine Learning": 0.589, - "helm_mmlu/Management": 0.874, - "helm_mmlu/Marketing": 0.91, - "helm_mmlu/Medical Genetics": 0.8, - "helm_mmlu/Miscellaneous": 0.893, - "helm_mmlu/Moral Scenarios": 0.502, - "helm_mmlu/Nutrition": 0.83, - "helm_mmlu/Prehistory": 0.824, - "helm_mmlu/Public Relations": 0.755, - "helm_mmlu/Security Studies": 0.808, - "helm_mmlu/Sociology": 0.9, - "helm_mmlu/Virology": 0.542, - "helm_mmlu/World Religions": 0.871, - "helm_mmlu/Mean win rate": 0.28, - "reward-bench/Score": 0.3711, - "reward-bench/Chat": 0.9274, - "reward-bench/Chat Hard": 0.5197, - "reward-bench/Safety": 0.595, - "reward-bench/Reasoning": 0.706, - "reward-bench/Prior Sets (0.5 weight)": 0.6635, - "reward-bench/Factuality": 0.4042, - "reward-bench/Precise IF": 0.2812, - "reward-bench/Math": 0.3552, - "reward-bench/Focus": 0.501, - "reward-bench/Ties": 0.0899 - } - }, - { - "id": "anthropic/claude-3-opus-20240229", - "name": "Claude 3 Opus 20240229", - "developer": "Anthropic", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.683, - "helm_lite/NarrativeQA": 0.351, - "helm_lite/NaturalQuestions (closed-book)": 0.441, - "helm_lite/OpenbookQA": 0.956, - "helm_lite/MMLU": 0.768, - "helm_lite/MATH": 0.76, - "helm_lite/GSM8K": 0.924, - "helm_lite/LegalBench": 0.662, - "helm_lite/MedQA": 0.775, - "helm_lite/WMT 2014": 0.24, - "helm_mmlu/MMLU All Subjects": 0.846, - "helm_mmlu/Abstract Algebra": 0.64, - "helm_mmlu/Anatomy": 0.8, - "helm_mmlu/College Physics": 0.716, - "helm_mmlu/Computer Security": 0.85, - "helm_mmlu/Econometrics": 0.789, - "helm_mmlu/Global Facts": 0.66, - "helm_mmlu/Jurisprudence": 0.88, - "helm_mmlu/Philosophy": 0.9, - "helm_mmlu/Professional Psychology": 0.904, - "helm_mmlu/Us Foreign Policy": 0.96, - "helm_mmlu/Astronomy": 0.967, - "helm_mmlu/Business Ethics": 0.86, - "helm_mmlu/Clinical Knowledge": 0.879, - "helm_mmlu/Conceptual Physics": 0.881, - "helm_mmlu/Electrical Engineering": 0.814, - "helm_mmlu/Elementary Mathematics": 0.862, - "helm_mmlu/Formal Logic": 0.698, - "helm_mmlu/High School World History": 0.941, - "helm_mmlu/Human Sexuality": 0.908, - "helm_mmlu/International Law": 0.901, - "helm_mmlu/Logical Fallacies": 0.896, - "helm_mmlu/Machine Learning": 0.741, - "helm_mmlu/Management": 0.942, - "helm_mmlu/Marketing": 0.944, - "helm_mmlu/Medical Genetics": 0.93, - "helm_mmlu/Miscellaneous": 0.951, - "helm_mmlu/Moral Scenarios": 0.826, - "helm_mmlu/Nutrition": 0.925, - "helm_mmlu/Prehistory": 0.941, - "helm_mmlu/Public Relations": 0.827, - "helm_mmlu/Security Studies": 0.886, - "helm_mmlu/Sociology": 0.94, - "helm_mmlu/Virology": 0.578, - "helm_mmlu/World Religions": 0.901, - "helm_mmlu/Mean win rate": 0.014, - "reward-bench/Score": 0.5744, - "reward-bench/Chat": 0.9469, - "reward-bench/Chat Hard": 0.6031, - "reward-bench/Safety": 0.8378, - "reward-bench/Reasoning": 0.7868, - "reward-bench/Factuality": 0.5389, - "reward-bench/Precise IF": 0.3312, - "reward-bench/Math": 0.5137, - "reward-bench/Focus": 0.6646, - "reward-bench/Ties": 0.5601 - } - }, - { - "id": "anthropic/claude-3-sonnet-20240229", - "name": "Claude 3 Sonnet 20240229", - "developer": "Anthropic", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.377, - "helm_lite/NarrativeQA": 0.111, - "helm_lite/NaturalQuestions (closed-book)": 0.028, - "helm_lite/OpenbookQA": 0.918, - "helm_lite/MMLU": 0.652, - "helm_lite/MATH": 0.084, - "helm_lite/GSM8K": 0.907, - "helm_lite/LegalBench": 0.49, - "helm_lite/MedQA": 0.684, - "helm_lite/WMT 2014": 0.218, - "helm_mmlu/MMLU All Subjects": 0.759, - "helm_mmlu/Abstract Algebra": 0.39, - "helm_mmlu/Anatomy": 0.711, - "helm_mmlu/College Physics": 0.559, - "helm_mmlu/Computer Security": 0.79, - "helm_mmlu/Econometrics": 0.64, - "helm_mmlu/Global Facts": 0.53, - "helm_mmlu/Jurisprudence": 0.861, - "helm_mmlu/Philosophy": 0.852, - "helm_mmlu/Professional Psychology": 0.814, - "helm_mmlu/Us Foreign Policy": 0.94, - "helm_mmlu/Astronomy": 0.855, - "helm_mmlu/Business Ethics": 0.82, - "helm_mmlu/Clinical Knowledge": 0.804, - "helm_mmlu/Conceptual Physics": 0.774, - "helm_mmlu/Electrical Engineering": 0.703, - "helm_mmlu/Elementary Mathematics": 0.635, - "helm_mmlu/Formal Logic": 0.579, - "helm_mmlu/High School World History": 0.895, - "helm_mmlu/Human Sexuality": 0.809, - "helm_mmlu/International Law": 0.909, - "helm_mmlu/Logical Fallacies": 0.853, - "helm_mmlu/Machine Learning": 0.643, - "helm_mmlu/Management": 0.922, - "helm_mmlu/Marketing": 0.85, - "helm_mmlu/Medical Genetics": 0.79, - "helm_mmlu/Miscellaneous": 0.872, - "helm_mmlu/Moral Scenarios": 0.626, - "helm_mmlu/Nutrition": 0.82, - "helm_mmlu/Prehistory": 0.864, - "helm_mmlu/Public Relations": 0.782, - "helm_mmlu/Security Studies": 0.865, - "helm_mmlu/Sociology": 0.905, - "helm_mmlu/Virology": 0.578, - "helm_mmlu/World Religions": 0.871, - "helm_mmlu/Mean win rate": 0.082, - "reward-bench/Score": 0.7458, - "reward-bench/Chat": 0.9344, - "reward-bench/Chat Hard": 0.5658, - "reward-bench/Safety": 0.8169, - "reward-bench/Reasoning": 0.6907, - "reward-bench/Prior Sets (0.5 weight)": 0.6963 - } - }, - { - "id": "anthropic/claude-3.7-sonnet", - "name": "anthropic/claude-3.7-sonnet", - "developer": "Anthropic", - "evaluator_relationship": null, - "benchmark_scores": { - "livecodebenchpro/Hard Problems": 0.0, - "livecodebenchpro/Medium Problems": 0.014084507042253521, - "livecodebenchpro/Easy Problems": 0.15492957746478872 - } - }, - { - "id": "anthropic/claude-haiku-4-5-20251001-fc", - "name": "Claude-Haiku-4-5-20251001 (FC)", - "developer": "Anthropic", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 6.0, - "bfcl/bfcl.overall.overall_accuracy": 68.7, - "bfcl/bfcl.overall.total_cost_usd": 14.23, - "bfcl/bfcl.overall.latency_mean_s": 1.68, - "bfcl/bfcl.overall.latency_std_s": 3.92, - "bfcl/bfcl.overall.latency_p95_s": 3.15, - "bfcl/bfcl.non_live.ast_accuracy": 86.5, - "bfcl/bfcl.non_live.simple_ast_accuracy": 71.0, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 94.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 92.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 88.5, - "bfcl/bfcl.live.live_accuracy": 78.68, - "bfcl/bfcl.live.live_simple_ast_accuracy": 83.72, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 77.59, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 75.0, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 75.0, - "bfcl/bfcl.multi_turn.accuracy": 53.62, - "bfcl/bfcl.multi_turn.base_accuracy": 63.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 42.5, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 52.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 56.0, - "bfcl/bfcl.web_search.accuracy": 83.5, - "bfcl/bfcl.web_search.base_accuracy": 86.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 81.0, - "bfcl/bfcl.memory.accuracy": 54.41, - "bfcl/bfcl.memory.kv_accuracy": 51.61, - "bfcl/bfcl.memory.vector_accuracy": 55.48, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 56.13, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 62.5, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 85.11 - } - }, - { - "id": "anthropic/claude-haiku-4-5-20251001-prompt", - "name": "Claude-Haiku-4-5-20251001 (Prompt)", - "developer": "Anthropic", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 87.0, - "bfcl/bfcl.overall.overall_accuracy": 25.26, - "bfcl/bfcl.overall.total_cost_usd": 45.13, - "bfcl/bfcl.overall.latency_mean_s": 3.75, - "bfcl/bfcl.overall.latency_std_s": 19.96, - "bfcl/bfcl.overall.latency_p95_s": 3.77, - "bfcl/bfcl.non_live.ast_accuracy": 55.42, - "bfcl/bfcl.non_live.simple_ast_accuracy": 55.67, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 84.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 38.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 44.0, - "bfcl/bfcl.live.live_accuracy": 52.48, - "bfcl/bfcl.live.live_simple_ast_accuracy": 66.67, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 49.76, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 56.25, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 16.67, - "bfcl/bfcl.multi_turn.accuracy": 1.75, - "bfcl/bfcl.multi_turn.base_accuracy": 1.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 0.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 4.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 1.5, - "bfcl/bfcl.web_search.accuracy": 19.5, - "bfcl/bfcl.web_search.base_accuracy": 20.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 19.0, - "bfcl/bfcl.memory.accuracy": 2.58, - "bfcl/bfcl.memory.kv_accuracy": 2.58, - "bfcl/bfcl.memory.vector_accuracy": 1.94, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 3.23, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 31.25, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 95.29, - "bfcl/bfcl.format_sensitivity.max_delta": 67.5, - "bfcl/bfcl.format_sensitivity.stddev": 20.07 - } - }, - { - "id": "anthropic/claude-haiku-4.5", - "name": "Claude Haiku 4.5", - "developer": "Anthropic", - "evaluator_relationship": null, - "benchmark_scores": { - "terminal-bench-2.0/terminal-bench-2.0": 35.5 - } - }, - { - "id": "anthropic/claude-instant-1.2", - "name": "Claude Instant 1.2", - "developer": "Anthropic", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.399, - "helm_lite/NarrativeQA": 0.616, - "helm_lite/NaturalQuestions (closed-book)": 0.343, - "helm_lite/OpenbookQA": 0.844, - "helm_lite/MMLU": 0.631, - "helm_lite/MATH": 0.499, - "helm_lite/GSM8K": 0.721, - "helm_lite/LegalBench": 0.586, - "helm_lite/MedQA": 0.559, - "helm_lite/WMT 2014": 0.194, - "helm_mmlu/MMLU All Subjects": 0.688, - "helm_mmlu/Abstract Algebra": 0.37, - "helm_mmlu/Anatomy": 0.637, - "helm_mmlu/College Physics": 0.49, - "helm_mmlu/Computer Security": 0.76, - "helm_mmlu/Econometrics": 0.614, - "helm_mmlu/Global Facts": 0.38, - "helm_mmlu/Jurisprudence": 0.833, - "helm_mmlu/Philosophy": 0.756, - "helm_mmlu/Professional Psychology": 0.724, - "helm_mmlu/Us Foreign Policy": 0.9, - "helm_mmlu/Astronomy": 0.743, - "helm_mmlu/Business Ethics": 0.7, - "helm_mmlu/Clinical Knowledge": 0.709, - "helm_mmlu/Conceptual Physics": 0.613, - "helm_mmlu/Electrical Engineering": 0.641, - "helm_mmlu/Elementary Mathematics": 0.45, - "helm_mmlu/Formal Logic": 0.444, - "helm_mmlu/High School World History": 0.878, - "helm_mmlu/Human Sexuality": 0.794, - "helm_mmlu/International Law": 0.851, - "helm_mmlu/Logical Fallacies": 0.81, - "helm_mmlu/Machine Learning": 0.67, - "helm_mmlu/Management": 0.835, - "helm_mmlu/Marketing": 0.885, - "helm_mmlu/Medical Genetics": 0.71, - "helm_mmlu/Miscellaneous": 0.828, - "helm_mmlu/Moral Scenarios": 0.488, - "helm_mmlu/Nutrition": 0.735, - "helm_mmlu/Prehistory": 0.762, - "helm_mmlu/Public Relations": 0.627, - "helm_mmlu/Security Studies": 0.784, - "helm_mmlu/Sociology": 0.841, - "helm_mmlu/Virology": 0.548, - "helm_mmlu/World Religions": 0.784, - "helm_mmlu/Mean win rate": 0.186 - } - }, - { - "id": "anthropic/claude-opus-4-1-20250805", - "name": "claude-opus-4-1-20250805", - "developer": "Anthropic", - "evaluator_relationship": null, - "benchmark_scores": { - "global-mmlu-lite/Global MMLU Lite": 0.943, - "global-mmlu-lite/Culturally Sensitive": 0.9331, - "global-mmlu-lite/Culturally Agnostic": 0.9528, - "global-mmlu-lite/Arabic": 0.945, - "global-mmlu-lite/English": 0.9475, - "global-mmlu-lite/Bengali": 0.9425, - "global-mmlu-lite/German": 0.94, - "global-mmlu-lite/French": 0.945, - "global-mmlu-lite/Hindi": 0.9475, - "global-mmlu-lite/Indonesian": 0.9425, - "global-mmlu-lite/Italian": 0.94, - "global-mmlu-lite/Japanese": 0.94, - "global-mmlu-lite/Korean": 0.95, - "global-mmlu-lite/Portuguese": 0.945, - "global-mmlu-lite/Spanish": 0.945, - "global-mmlu-lite/Swahili": 0.93, - "global-mmlu-lite/Yoruba": 0.9375, - "global-mmlu-lite/Chinese": 0.945, - "global-mmlu-lite/Burmese": 0.945 - } - }, - { - "id": "anthropic/claude-opus-4-20250514", - "name": "Claude 4 Opus 20250514", - "developer": "Anthropic", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.757, - "helm_capabilities/MMLU-Pro": 0.859, - "helm_capabilities/GPQA": 0.666, - "helm_capabilities/IFEval": 0.918, - "helm_capabilities/WildBench": 0.833, - "helm_capabilities/Omni-MATH": 0.511, - "reward-bench/Score": 0.7648, - "reward-bench/Factuality": 0.8267, - "reward-bench/Precise IF": 0.4188, - "reward-bench/Math": 0.7491, - "reward-bench/Safety": 0.8954, - "reward-bench/Focus": 0.8616, - "reward-bench/Ties": 0.8375 - } - }, - { - "id": "anthropic/claude-opus-4-20250514-thinking-10k", - "name": "Claude 4 Opus 20250514, extended thinking", - "developer": "Anthropic", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.78, - "helm_capabilities/MMLU-Pro": 0.875, - "helm_capabilities/GPQA": 0.709, - "helm_capabilities/IFEval": 0.849, - "helm_capabilities/WildBench": 0.852, - "helm_capabilities/Omni-MATH": 0.616 - } - }, - { - "id": "anthropic/claude-opus-4-5", - "name": "claude-opus-4-5", - "developer": "Anthropic", - "evaluator_relationship": null, - "benchmark_scores": { - "appworld_test_normal/appworld/test_normal": 0.68, - "browsecompplus/browsecompplus": 0.61, - "swe-bench/swe-bench": 0.65, - "tau-bench-2_airline/tau-bench-2/airline": 0.66, - "tau-bench-2_retail/tau-bench-2/retail": 0.78, - "tau-bench-2_telecom/tau-bench-2/telecom": 0.84 - } - }, - { - "id": "anthropic/claude-opus-4-5-20251101-fc", - "name": "Claude-Opus-4-5-20251101 (FC)", - "developer": "Anthropic", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 1.0, - "bfcl/bfcl.overall.overall_accuracy": 77.47, - "bfcl/bfcl.overall.total_cost_usd": 86.55, - "bfcl/bfcl.overall.latency_mean_s": 4.38, - "bfcl/bfcl.overall.latency_std_s": 3.13, - "bfcl/bfcl.overall.latency_p95_s": 7.56, - "bfcl/bfcl.non_live.ast_accuracy": 88.58, - "bfcl/bfcl.non_live.simple_ast_accuracy": 76.83, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 95.5, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 93.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 88.5, - "bfcl/bfcl.live.live_accuracy": 79.79, - "bfcl/bfcl.live.live_simple_ast_accuracy": 86.43, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 78.16, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 87.5, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 75.0, - "bfcl/bfcl.multi_turn.accuracy": 68.38, - "bfcl/bfcl.multi_turn.base_accuracy": 81.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 64.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 58.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 70.5, - "bfcl/bfcl.web_search.accuracy": 84.5, - "bfcl/bfcl.web_search.base_accuracy": 84.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 85.0, - "bfcl/bfcl.memory.accuracy": 73.76, - "bfcl/bfcl.memory.kv_accuracy": 70.97, - "bfcl/bfcl.memory.vector_accuracy": 72.9, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 77.42, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 62.5, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 84.72 - } - }, - { - "id": "anthropic/claude-opus-4-5-20251101-prompt", - "name": "Claude-Opus-4-5-20251101 (Prompt)", - "developer": "Anthropic", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 57.0, - "bfcl/bfcl.overall.overall_accuracy": 33.47, - "bfcl/bfcl.overall.total_cost_usd": 88.33, - "bfcl/bfcl.overall.latency_mean_s": 3.76, - "bfcl/bfcl.overall.latency_std_s": 13.19, - "bfcl/bfcl.overall.latency_p95_s": 5.52, - "bfcl/bfcl.non_live.ast_accuracy": 89.65, - "bfcl/bfcl.non_live.simple_ast_accuracy": 79.58, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 93.5, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 93.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 92.5, - "bfcl/bfcl.live.live_accuracy": 76.02, - "bfcl/bfcl.live.live_simple_ast_accuracy": 84.5, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 74.17, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 81.25, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 62.5, - "bfcl/bfcl.multi_turn.accuracy": 16.12, - "bfcl/bfcl.multi_turn.base_accuracy": 20.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 9.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 21.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 13.5, - "bfcl/bfcl.web_search.accuracy": 13.0, - "bfcl/bfcl.web_search.base_accuracy": 13.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 13.0, - "bfcl/bfcl.memory.accuracy": 1.94, - "bfcl/bfcl.memory.kv_accuracy": 1.29, - "bfcl/bfcl.memory.vector_accuracy": 1.94, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 2.58, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 68.75, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 90.75, - "bfcl/bfcl.format_sensitivity.max_delta": 13.0, - "bfcl/bfcl.format_sensitivity.stddev": 3.65 - } - }, - { - "id": "anthropic/claude-opus-4.1", - "name": "Claude Opus 4.1", - "developer": "Anthropic", - "evaluator_relationship": null, - "benchmark_scores": { - "terminal-bench-2.0/terminal-bench-2.0": 35.1 - } - }, - { - "id": "anthropic/claude-opus-4.5", - "name": "Claude Opus 4.5", - "developer": "Anthropic", - "evaluator_relationship": null, - "benchmark_scores": { - "terminal-bench-2.0/terminal-bench-2.0": 63.1 - } - }, - { - "id": "anthropic/claude-opus-4.6", - "name": "Claude Opus 4.6", - "developer": "Anthropic", - "evaluator_relationship": null, - "benchmark_scores": { - "terminal-bench-2.0/terminal-bench-2.0": 74.7 - } - }, - { - "id": "anthropic/claude-sonnet-4-20250514", - "name": "claude-sonnet-4-20250514", - "developer": "Anthropic", - "evaluator_relationship": null, - "benchmark_scores": { - "global-mmlu-lite/Global MMLU Lite": 0.9058, - "global-mmlu-lite/Culturally Sensitive": 0.8913, - "global-mmlu-lite/Culturally Agnostic": 0.9203, - "global-mmlu-lite/Arabic": 0.9125, - "global-mmlu-lite/English": 0.905, - "global-mmlu-lite/Bengali": 0.9075, - "global-mmlu-lite/German": 0.9125, - "global-mmlu-lite/French": 0.91, - "global-mmlu-lite/Hindi": 0.9, - "global-mmlu-lite/Indonesian": 0.9025, - "global-mmlu-lite/Italian": 0.9075, - "global-mmlu-lite/Japanese": 0.9, - "global-mmlu-lite/Korean": 0.9125, - "global-mmlu-lite/Portuguese": 0.91, - "global-mmlu-lite/Spanish": 0.9075, - "global-mmlu-lite/Swahili": 0.8975, - "global-mmlu-lite/Yoruba": 0.8975, - "global-mmlu-lite/Chinese": 0.9175, - "global-mmlu-lite/Burmese": 0.8925, - "helm_capabilities/Mean score": 0.733, - "helm_capabilities/MMLU-Pro": 0.843, - "helm_capabilities/GPQA": 0.643, - "helm_capabilities/IFEval": 0.839, - "helm_capabilities/WildBench": 0.825, - "helm_capabilities/Omni-MATH": 0.512, - "reward-bench/Score": 0.7117, - "reward-bench/Factuality": 0.7612, - "reward-bench/Precise IF": 0.3594, - "reward-bench/Math": 0.7049, - "reward-bench/Safety": 0.8909, - "reward-bench/Focus": 0.7596, - "reward-bench/Ties": 0.7939 - } - }, - { - "id": "anthropic/claude-sonnet-4-20250514-thinking-10k", - "name": "Claude 4 Sonnet 20250514, extended thinking", - "developer": "Anthropic", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.766, - "helm_capabilities/MMLU-Pro": 0.843, - "helm_capabilities/GPQA": 0.706, - "helm_capabilities/IFEval": 0.84, - "helm_capabilities/WildBench": 0.838, - "helm_capabilities/Omni-MATH": 0.602 - } - }, - { - "id": "anthropic/claude-sonnet-4-5-20250929", - "name": "claude-sonnet-4-5-20250929", - "developer": "Anthropic", - "evaluator_relationship": null, - "benchmark_scores": { - "livecodebenchpro/Hard Problems": 0.0, - "livecodebenchpro/Medium Problems": 0.0, - "livecodebenchpro/Easy Problems": 0.5352 - } - }, - { - "id": "anthropic/claude-sonnet-4-5-20250929-fc", - "name": "Claude-Sonnet-4-5-20250929 (FC)", - "developer": "Anthropic", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 2.0, - "bfcl/bfcl.overall.overall_accuracy": 73.24, - "bfcl/bfcl.overall.total_cost_usd": 43.73, - "bfcl/bfcl.overall.latency_mean_s": 4.31, - "bfcl/bfcl.overall.latency_std_s": 4.43, - "bfcl/bfcl.overall.latency_p95_s": 7.27, - "bfcl/bfcl.non_live.ast_accuracy": 88.65, - "bfcl/bfcl.non_live.simple_ast_accuracy": 72.58, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 95.5, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 94.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 92.0, - "bfcl/bfcl.live.live_accuracy": 81.13, - "bfcl/bfcl.live.live_simple_ast_accuracy": 89.53, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 78.92, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 87.5, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 83.33, - "bfcl/bfcl.multi_turn.accuracy": 61.37, - "bfcl/bfcl.multi_turn.base_accuracy": 69.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 65.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 52.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 59.0, - "bfcl/bfcl.web_search.accuracy": 81.0, - "bfcl/bfcl.web_search.base_accuracy": 82.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 80.0, - "bfcl/bfcl.memory.accuracy": 64.95, - "bfcl/bfcl.memory.kv_accuracy": 54.19, - "bfcl/bfcl.memory.vector_accuracy": 57.42, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 83.23, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 68.75, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 86.61 - } - }, - { - "id": "anthropic/claude-sonnet-4-5-20250929-prompt", - "name": "Claude-Sonnet-4-5-20250929 (Prompt)", - "developer": "Anthropic", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 89.0, - "bfcl/bfcl.overall.overall_accuracy": 24.9, - "bfcl/bfcl.overall.total_cost_usd": 47.82, - "bfcl/bfcl.overall.latency_mean_s": 3.84, - "bfcl/bfcl.overall.latency_std_s": 1.53, - "bfcl/bfcl.overall.latency_p95_s": 6.66, - "bfcl/bfcl.non_live.ast_accuracy": 59.81, - "bfcl/bfcl.non_live.simple_ast_accuracy": 47.25, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 79.5, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 53.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 59.0, - "bfcl/bfcl.live.live_accuracy": 46.56, - "bfcl/bfcl.live.live_simple_ast_accuracy": 73.26, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 40.17, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 56.25, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 33.33, - "bfcl/bfcl.multi_turn.accuracy": 1.62, - "bfcl/bfcl.multi_turn.base_accuracy": 2.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 0.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 3.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 1.5, - "bfcl/bfcl.web_search.accuracy": 16.0, - "bfcl/bfcl.web_search.base_accuracy": 16.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 16.0, - "bfcl/bfcl.memory.accuracy": 5.38, - "bfcl/bfcl.memory.kv_accuracy": 4.52, - "bfcl/bfcl.memory.vector_accuracy": 9.68, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 1.94, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 37.5, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 95.03, - "bfcl/bfcl.format_sensitivity.max_delta": 37.5, - "bfcl/bfcl.format_sensitivity.stddev": 10.07 - } - }, - { - "id": "anthropic/claude-sonnet-4.5", - "name": "Claude Sonnet 4.5", - "developer": "Anthropic", - "evaluator_relationship": null, - "benchmark_scores": { - "terminal-bench-2.0/terminal-bench-2.0": 42.5 - } - }, - { - "id": "anthropic/claude-v1.3", - "name": "Anthropic Claude v1.3", - "developer": "Anthropic", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_instruct/Mean win rate": 0.611, - "helm_instruct/Anthropic RLHF dataset": 4.965, - "helm_instruct/Best ChatGPT Prompts": 4.995, - "helm_instruct/Koala test dataset": 4.981, - "helm_instruct/Open Assistant": 4.975, - "helm_instruct/Self Instruct": 4.992, - "helm_instruct/Vicuna": 4.989, - "helm_lite/Mean win rate": 0.518, - "helm_lite/NarrativeQA": 0.723, - "helm_lite/NaturalQuestions (closed-book)": 0.409, - "helm_lite/OpenbookQA": 0.908, - "helm_lite/MMLU": 0.631, - "helm_lite/MATH": 0.54, - "helm_lite/GSM8K": 0.784, - "helm_lite/LegalBench": 0.629, - "helm_lite/MedQA": 0.618, - "helm_lite/WMT 2014": 0.219 - } - }, - { - "id": "anthropic/Opus 4.1", - "name": "Opus 4.1", - "developer": "Anthropic", - "evaluator_relationship": null, - "benchmark_scores": { - "ace/Overall Score": 0.4, - "ace/Gaming Score": 0.318 - } - }, - { - "id": "anthropic/Opus 4.5", - "name": "Opus 4.5", - "developer": "Anthropic", - "evaluator_relationship": null, - "benchmark_scores": { - "ace/Overall Score": 0.478, - "ace/Gaming Score": 0.391, - "apex-agents/Overall Pass@1": 0.184, - "apex-agents/Overall Pass@8": 0.34, - "apex-agents/Overall Mean Score": 0.348, - "apex-agents/Investment Banking Pass@1": 0.216, - "apex-agents/Management Consulting Pass@1": 0.132, - "apex-agents/Corporate Law Pass@1": 0.202, - "apex-agents/Corporate Lawyer Mean Score": 0.471, - "apex-v1/Medicine (MD) Score": 0.65 - } - }, - { - "id": "anthropic/Opus 4.6", - "name": "Opus 4.6", - "developer": "Anthropic", - "evaluator_relationship": null, - "benchmark_scores": { - "apex-agents/Overall Pass@1": 0.298, - "apex-agents/Corporate Lawyer Mean Score": 0.502 - } - }, - { - "id": "anthropic/Sonnet 4.5", - "name": "Sonnet 4.5", - "developer": "Anthropic", - "evaluator_relationship": null, - "benchmark_scores": { - "ace/Overall Score": 0.44, - "ace/Gaming Score": 0.373 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/apple.json b/data/developers/apple.json deleted file mode 100644 index bfdde752f718aeb716116ad152a8ca94d3a244e2..0000000000000000000000000000000000000000 --- a/data/developers/apple.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "apple", - "models": [ - { - "id": "apple/DCLM-7B", - "name": "DCLM-7B", - "developer": "apple", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2173, - "hfopenllm_v2/BBH": 0.4232, - "hfopenllm_v2/MATH Level 5": 0.037, - "hfopenllm_v2/GPQA": 0.3154, - "hfopenllm_v2/MUSR": 0.3921, - "hfopenllm_v2/MMLU-PRO": 0.3111 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/applied-compute.json b/data/developers/applied-compute.json deleted file mode 100644 index 45897ff6f4efc8574ee8e8aa6faebd24a5bd9681..0000000000000000000000000000000000000000 --- a/data/developers/applied-compute.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "developer": "applied-compute", - "models": [ - { - "id": "applied-compute/Applied Compute: Small", - "name": "Applied Compute: Small", - "developer": "applied-compute", - "evaluator_relationship": null, - "benchmark_scores": { - "apex-agents/Overall Pass@1": 0.23, - "apex-agents/Overall Mean Score": 0.401, - "apex-agents/Corporate Law Pass@1": 0.266, - "apex-agents/Corporate Lawyer Mean Score": 0.548 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/appvoid.json b/data/developers/appvoid.json deleted file mode 100644 index cd81f56e4efd66ed71d5b9dc55a6e2b36aa44396..0000000000000000000000000000000000000000 --- a/data/developers/appvoid.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "appvoid", - "models": [ - { - "id": "appvoid/arco-2", - "name": "arco-2", - "developer": "appvoid", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1991, - "hfopenllm_v2/BBH": 0.3146, - "hfopenllm_v2/MATH Level 5": 0.0136, - "hfopenllm_v2/GPQA": 0.2391, - "hfopenllm_v2/MUSR": 0.3536, - "hfopenllm_v2/MMLU-PRO": 0.1116 - } - }, - { - "id": "appvoid/arco-2-instruct", - "name": "arco-2-instruct", - "developer": "appvoid", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2164, - "hfopenllm_v2/BBH": 0.3133, - "hfopenllm_v2/MATH Level 5": 0.0128, - "hfopenllm_v2/GPQA": 0.2383, - "hfopenllm_v2/MUSR": 0.3496, - "hfopenllm_v2/MMLU-PRO": 0.1113 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/arcee-ai.json b/data/developers/arcee-ai.json deleted file mode 100644 index a67ed73a3aff7f3d10c24e6ce76a817fd626a3c8..0000000000000000000000000000000000000000 --- a/data/developers/arcee-ai.json +++ /dev/null @@ -1,159 +0,0 @@ -{ - "developer": "arcee-ai", - "models": [ - { - "id": "arcee-ai/Arcee-Blitz", - "name": "Arcee-Blitz", - "developer": "arcee-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5543, - "hfopenllm_v2/BBH": 0.6607, - "hfopenllm_v2/MATH Level 5": 0.3482, - "hfopenllm_v2/GPQA": 0.3851, - "hfopenllm_v2/MUSR": 0.5047, - "hfopenllm_v2/MMLU-PRO": 0.6154 - } - }, - { - "id": "arcee-ai/Arcee-Maestro-7B-Preview", - "name": "Arcee-Maestro-7B-Preview", - "developer": "arcee-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.275, - "hfopenllm_v2/BBH": 0.4648, - "hfopenllm_v2/MATH Level 5": 0.4992, - "hfopenllm_v2/GPQA": 0.3322, - "hfopenllm_v2/MUSR": 0.3885, - "hfopenllm_v2/MMLU-PRO": 0.3039 - } - }, - { - "id": "arcee-ai/Arcee-Nova", - "name": "Arcee-Nova", - "developer": "arcee-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7907, - "hfopenllm_v2/BBH": 0.6942, - "hfopenllm_v2/MATH Level 5": 0.4381, - "hfopenllm_v2/GPQA": 0.3851, - "hfopenllm_v2/MUSR": 0.4562, - "hfopenllm_v2/MMLU-PRO": 0.5452 - } - }, - { - "id": "arcee-ai/Arcee-Spark", - "name": "Arcee-Spark", - "developer": "arcee-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5718, - "hfopenllm_v2/BBH": 0.5481, - "hfopenllm_v2/MATH Level 5": 0.114, - "hfopenllm_v2/GPQA": 0.3062, - "hfopenllm_v2/MUSR": 0.4008, - "hfopenllm_v2/MMLU-PRO": 0.3813 - } - }, - { - "id": "arcee-ai/Llama-3.1-SuperNova-Lite", - "name": "Llama-3.1-SuperNova-Lite", - "developer": "arcee-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8017, - "hfopenllm_v2/BBH": 0.5152, - "hfopenllm_v2/MATH Level 5": 0.1828, - "hfopenllm_v2/GPQA": 0.3062, - "hfopenllm_v2/MUSR": 0.4163, - "hfopenllm_v2/MMLU-PRO": 0.3877 - } - }, - { - "id": "arcee-ai/Llama-Spark", - "name": "Llama-Spark", - "developer": "arcee-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7911, - "hfopenllm_v2/BBH": 0.5054, - "hfopenllm_v2/MATH Level 5": 0.139, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.3593, - "hfopenllm_v2/MMLU-PRO": 0.3721 - } - }, - { - "id": "arcee-ai/raspberry-3B", - "name": "raspberry-3B", - "developer": "arcee-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3154, - "hfopenllm_v2/BBH": 0.4269, - "hfopenllm_v2/MATH Level 5": 0.1035, - "hfopenllm_v2/GPQA": 0.2777, - "hfopenllm_v2/MUSR": 0.4123, - "hfopenllm_v2/MMLU-PRO": 0.2854 - } - }, - { - "id": "arcee-ai/SuperNova-Medius", - "name": "SuperNova-Medius", - "developer": "arcee-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7184, - "hfopenllm_v2/BBH": 0.6377, - "hfopenllm_v2/MATH Level 5": 0.469, - "hfopenllm_v2/GPQA": 0.3331, - "hfopenllm_v2/MUSR": 0.4233, - "hfopenllm_v2/MMLU-PRO": 0.5035 - } - }, - { - "id": "arcee-ai/Virtuoso-Lite", - "name": "Virtuoso-Lite", - "developer": "arcee-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.81, - "hfopenllm_v2/BBH": 0.6099, - "hfopenllm_v2/MATH Level 5": 0.253, - "hfopenllm_v2/GPQA": 0.344, - "hfopenllm_v2/MUSR": 0.4595, - "hfopenllm_v2/MMLU-PRO": 0.4441 - } - }, - { - "id": "arcee-ai/Virtuoso-Small", - "name": "Virtuoso-Small", - "developer": "arcee-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7935, - "hfopenllm_v2/BBH": 0.6518, - "hfopenllm_v2/MATH Level 5": 0.4094, - "hfopenllm_v2/GPQA": 0.3364, - "hfopenllm_v2/MUSR": 0.4339, - "hfopenllm_v2/MMLU-PRO": 0.5191 - } - }, - { - "id": "arcee-ai/Virtuoso-Small-v2", - "name": "Virtuoso-Small-v2", - "developer": "arcee-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8273, - "hfopenllm_v2/BBH": 0.6554, - "hfopenllm_v2/MATH Level 5": 0.466, - "hfopenllm_v2/GPQA": 0.3532, - "hfopenllm_v2/MUSR": 0.4313, - "hfopenllm_v2/MMLU-PRO": 0.5188 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/argilla-warehouse.json b/data/developers/argilla-warehouse.json deleted file mode 100644 index cafd84ecedb50e04d94c0f7000f6f1b99795d5c2..0000000000000000000000000000000000000000 --- a/data/developers/argilla-warehouse.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "argilla-warehouse", - "models": [ - { - "id": "argilla-warehouse/Llama-3.1-8B-MagPie-Ultra", - "name": "Llama-3.1-8B-MagPie-Ultra", - "developer": "argilla-warehouse", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5757, - "hfopenllm_v2/BBH": 0.462, - "hfopenllm_v2/MATH Level 5": 0.077, - "hfopenllm_v2/GPQA": 0.2668, - "hfopenllm_v2/MUSR": 0.3543, - "hfopenllm_v2/MMLU-PRO": 0.3144 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/argilla.json b/data/developers/argilla.json deleted file mode 100644 index 8008c2c0602464001b097978abfb5c652a714ab1..0000000000000000000000000000000000000000 --- a/data/developers/argilla.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "argilla", - "models": [ - { - "id": "argilla/notus-7b-v1", - "name": "notus-7b-v1", - "developer": "argilla", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5082, - "hfopenllm_v2/BBH": 0.4512, - "hfopenllm_v2/MATH Level 5": 0.0317, - "hfopenllm_v2/GPQA": 0.2894, - "hfopenllm_v2/MUSR": 0.3364, - "hfopenllm_v2/MMLU-PRO": 0.3004 - } - }, - { - "id": "argilla/notux-8x7b-v1", - "name": "notux-8x7b-v1", - "developer": "argilla", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5422, - "hfopenllm_v2/BBH": 0.5363, - "hfopenllm_v2/MATH Level 5": 0.0997, - "hfopenllm_v2/GPQA": 0.3087, - "hfopenllm_v2/MUSR": 0.4176, - "hfopenllm_v2/MMLU-PRO": 0.366 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/arisin.json b/data/developers/arisin.json deleted file mode 100644 index e516001be2b227004b679fcc822dc1d6ea19ea0f..0000000000000000000000000000000000000000 --- a/data/developers/arisin.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "arisin", - "models": [ - { - "id": "arisin/orca-platypus-13B-slerp", - "name": "orca-platypus-13B-slerp", - "developer": "arisin", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2672, - "hfopenllm_v2/BBH": 0.4631, - "hfopenllm_v2/MATH Level 5": 0.0159, - "hfopenllm_v2/GPQA": 0.2987, - "hfopenllm_v2/MUSR": 0.4253, - "hfopenllm_v2/MMLU-PRO": 0.2592 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/ark.json b/data/developers/ark.json deleted file mode 100644 index a7167e1b286fbdc55350fc0f8dc71e54f3b61330..0000000000000000000000000000000000000000 --- a/data/developers/ark.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "developer": "ark", - "models": [ - { - "id": "ark/ep-20250603132404-cgpjm", - "name": "ep-20250603132404-cgpjm", - "developer": "ark", - "evaluator_relationship": null, - "benchmark_scores": { - "livecodebenchpro/Hard Problems": 0.0, - "livecodebenchpro/Medium Problems": 0.0141, - "livecodebenchpro/Easy Problems": 0.507 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/arliai.json b/data/developers/arliai.json deleted file mode 100644 index e1d87d6549acf88234ad4a9341b69cff2439d761..0000000000000000000000000000000000000000 --- a/data/developers/arliai.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "ArliAI", - "models": [ - { - "id": "ArliAI/ArliAI-RPMax-12B-v1.1", - "name": "ArliAI-RPMax-12B-v1.1", - "developer": "ArliAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5349, - "hfopenllm_v2/BBH": 0.4752, - "hfopenllm_v2/MATH Level 5": 0.1125, - "hfopenllm_v2/GPQA": 0.2819, - "hfopenllm_v2/MUSR": 0.3618, - "hfopenllm_v2/MMLU-PRO": 0.3384 - } - }, - { - "id": "ArliAI/Llama-3.1-8B-ArliAI-RPMax-v1.1", - "name": "Llama-3.1-8B-ArliAI-RPMax-v1.1", - "developer": "ArliAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6359, - "hfopenllm_v2/BBH": 0.5016, - "hfopenllm_v2/MATH Level 5": 0.1314, - "hfopenllm_v2/GPQA": 0.2836, - "hfopenllm_v2/MUSR": 0.3577, - "hfopenllm_v2/MMLU-PRO": 0.3551 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/arshiaafshani.json b/data/developers/arshiaafshani.json deleted file mode 100644 index 1f3469e3647e1bc24484a7e92c7b1d2973239522..0000000000000000000000000000000000000000 --- a/data/developers/arshiaafshani.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "arshiaafshani", - "models": [ - { - "id": "arshiaafshani/Arsh-V1", - "name": "Arsh-V1", - "developer": "arshiaafshani", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6043, - "hfopenllm_v2/BBH": 0.674, - "hfopenllm_v2/MATH Level 5": 0.2621, - "hfopenllm_v2/GPQA": 0.3733, - "hfopenllm_v2/MUSR": 0.4899, - "hfopenllm_v2/MMLU-PRO": 0.5257 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/arthur-lagacherie.json b/data/developers/arthur-lagacherie.json deleted file mode 100644 index 194d8f751fe5b0c657ff9d2dbffc72d638e8c613..0000000000000000000000000000000000000000 --- a/data/developers/arthur-lagacherie.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "Arthur-LAGACHERIE", - "models": [ - { - "id": "Arthur-LAGACHERIE/Precis-1B-Instruct", - "name": "Precis-1B-Instruct", - "developer": "Arthur-LAGACHERIE", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3671, - "hfopenllm_v2/BBH": 0.3224, - "hfopenllm_v2/MATH Level 5": 0.0038, - "hfopenllm_v2/GPQA": 0.2659, - "hfopenllm_v2/MUSR": 0.3436, - "hfopenllm_v2/MMLU-PRO": 0.1426 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/artples.json b/data/developers/artples.json deleted file mode 100644 index ee7386fde874dc3f1d6529d96da80a827af37dff..0000000000000000000000000000000000000000 --- a/data/developers/artples.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "Artples", - "models": [ - { - "id": "Artples/L-MChat-7b", - "name": "L-MChat-7b", - "developer": "Artples", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5297, - "hfopenllm_v2/BBH": 0.46, - "hfopenllm_v2/MATH Level 5": 0.0921, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.4029, - "hfopenllm_v2/MMLU-PRO": 0.3299 - } - }, - { - "id": "Artples/L-MChat-Small", - "name": "L-MChat-Small", - "developer": "Artples", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3287, - "hfopenllm_v2/BBH": 0.4823, - "hfopenllm_v2/MATH Level 5": 0.0378, - "hfopenllm_v2/GPQA": 0.2676, - "hfopenllm_v2/MUSR": 0.3696, - "hfopenllm_v2/MMLU-PRO": 0.2464 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/aryanne.json b/data/developers/aryanne.json deleted file mode 100644 index 17738c5975a89f8da20b14683880dff2f2671b71..0000000000000000000000000000000000000000 --- a/data/developers/aryanne.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "developer": "Aryanne", - "models": [ - { - "id": "Aryanne/QwentileSwap", - "name": "QwentileSwap", - "developer": "Aryanne", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7378, - "hfopenllm_v2/BBH": 0.7008, - "hfopenllm_v2/MATH Level 5": 0.4222, - "hfopenllm_v2/GPQA": 0.3674, - "hfopenllm_v2/MUSR": 0.464, - "hfopenllm_v2/MMLU-PRO": 0.5946 - } - }, - { - "id": "Aryanne/SHBA", - "name": "SHBA", - "developer": "Aryanne", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7817, - "hfopenllm_v2/BBH": 0.5233, - "hfopenllm_v2/MATH Level 5": 0.1798, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.4161, - "hfopenllm_v2/MMLU-PRO": 0.3892 - } - }, - { - "id": "Aryanne/SuperHeart", - "name": "SuperHeart", - "developer": "Aryanne", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5192, - "hfopenllm_v2/BBH": 0.5215, - "hfopenllm_v2/MATH Level 5": 0.1563, - "hfopenllm_v2/GPQA": 0.3012, - "hfopenllm_v2/MUSR": 0.4436, - "hfopenllm_v2/MMLU-PRO": 0.3912 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/asharsha30.json b/data/developers/asharsha30.json deleted file mode 100644 index 22896ed7153814cf1efa535557809de4c2229894..0000000000000000000000000000000000000000 --- a/data/developers/asharsha30.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "asharsha30", - "models": [ - { - "id": "asharsha30/LLAMA_Harsha_8_B_ORDP_10k", - "name": "LLAMA_Harsha_8_B_ORDP_10k", - "developer": "asharsha30", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3464, - "hfopenllm_v2/BBH": 0.4669, - "hfopenllm_v2/MATH Level 5": 0.0665, - "hfopenllm_v2/GPQA": 0.2735, - "hfopenllm_v2/MUSR": 0.3697, - "hfopenllm_v2/MMLU-PRO": 0.281 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/ashercn97.json b/data/developers/ashercn97.json deleted file mode 100644 index 968df5c6a4a4a29fa6d509d913b24dd3850d9f23..0000000000000000000000000000000000000000 --- a/data/developers/ashercn97.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "ashercn97", - "models": [ - { - "id": "ashercn97/a1-v0.0.1", - "name": "a1-v0.0.1", - "developer": "ashercn97", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2198, - "hfopenllm_v2/BBH": 0.5188, - "hfopenllm_v2/MATH Level 5": 0.2145, - "hfopenllm_v2/GPQA": 0.3112, - "hfopenllm_v2/MUSR": 0.412, - "hfopenllm_v2/MMLU-PRO": 0.4165 - } - }, - { - "id": "ashercn97/a1-v002", - "name": "a1-v002", - "developer": "ashercn97", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2585, - "hfopenllm_v2/BBH": 0.5261, - "hfopenllm_v2/MATH Level 5": 0.2341, - "hfopenllm_v2/GPQA": 0.3188, - "hfopenllm_v2/MUSR": 0.4159, - "hfopenllm_v2/MMLU-PRO": 0.4175 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/assskelad.json b/data/developers/assskelad.json deleted file mode 100644 index 62961e2dd41dec4193a73dc4e3fd21d1abe21a10..0000000000000000000000000000000000000000 --- a/data/developers/assskelad.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "assskelad", - "models": [ - { - "id": "assskelad/smollm2-360M-sft_SmallThoughts", - "name": "smollm2-360M-sft_SmallThoughts", - "developer": "assskelad", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2007, - "hfopenllm_v2/BBH": 0.315, - "hfopenllm_v2/MATH Level 5": 0.0166, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3395, - "hfopenllm_v2/MMLU-PRO": 0.1182 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/atanddev.json b/data/developers/atanddev.json deleted file mode 100644 index d269c1fbba37e9ea4df30635a880656764997806..0000000000000000000000000000000000000000 --- a/data/developers/atanddev.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "AtAndDev", - "models": [ - { - "id": "AtAndDev/Qwen2.5-1.5B-continuous-learnt", - "name": "Qwen2.5-1.5B-continuous-learnt", - "developer": "AtAndDev", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4511, - "hfopenllm_v2/BBH": 0.4275, - "hfopenllm_v2/MATH Level 5": 0.1473, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.3623, - "hfopenllm_v2/MMLU-PRO": 0.2806 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/ateron.json b/data/developers/ateron.json deleted file mode 100644 index 14b39e12249b44d8c29773d550919aadc86d0a24..0000000000000000000000000000000000000000 --- a/data/developers/ateron.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "developer": "Ateron", - "models": [ - { - "id": "Ateron/Glowing-Forest-12B", - "name": "Glowing-Forest-12B", - "developer": "Ateron", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3592, - "hfopenllm_v2/BBH": 0.5492, - "hfopenllm_v2/MATH Level 5": 0.0778, - "hfopenllm_v2/GPQA": 0.3331, - "hfopenllm_v2/MUSR": 0.4449, - "hfopenllm_v2/MMLU-PRO": 0.3718 - } - }, - { - "id": "Ateron/Lotus-Magpic", - "name": "Lotus-Magpic", - "developer": "Ateron", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6286, - "hfopenllm_v2/BBH": 0.5254, - "hfopenllm_v2/MATH Level 5": 0.0997, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.4332, - "hfopenllm_v2/MMLU-PRO": 0.3491 - } - }, - { - "id": "Ateron/Way_of_MagPicaro", - "name": "Way_of_MagPicaro", - "developer": "Ateron", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2637, - "hfopenllm_v2/BBH": 0.5427, - "hfopenllm_v2/MATH Level 5": 0.0589, - "hfopenllm_v2/GPQA": 0.3339, - "hfopenllm_v2/MUSR": 0.4649, - "hfopenllm_v2/MMLU-PRO": 0.3536 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/athirdpath.json b/data/developers/athirdpath.json deleted file mode 100644 index 71f45173dabd165d55432713c1b8b7bd1b5cda60..0000000000000000000000000000000000000000 --- a/data/developers/athirdpath.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "athirdpath", - "models": [ - { - "id": "athirdpath/Llama-3.1-Instruct_NSFW-pretrained_e1-plus_reddit", - "name": "Llama-3.1-Instruct_NSFW-pretrained_e1-plus_reddit", - "developer": "athirdpath", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4521, - "hfopenllm_v2/BBH": 0.4939, - "hfopenllm_v2/MATH Level 5": 0.102, - "hfopenllm_v2/GPQA": 0.2919, - "hfopenllm_v2/MUSR": 0.3864, - "hfopenllm_v2/MMLU-PRO": 0.3565 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/atlaai.json b/data/developers/atlaai.json deleted file mode 100644 index 338a8d19ff5e81d140afc67a6e00cabdf7cb10b8..0000000000000000000000000000000000000000 --- a/data/developers/atlaai.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "developer": "AtlaAI", - "models": [ - { - "id": "AtlaAI/Selene-1", - "name": "AtlaAI/Selene-1", - "developer": "AtlaAI", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.9241, - "reward-bench/Chat": 0.9777, - "reward-bench/Chat Hard": 0.8399, - "reward-bench/Safety": 0.9216, - "reward-bench/Reasoning": 0.9572 - } - }, - { - "id": "AtlaAI/Selene-1-Mini-Llama-3.1-8B", - "name": "AtlaAI/Selene-1-Mini-Llama-3.1-8B", - "developer": "AtlaAI", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.8913, - "reward-bench/Chat": 0.9358, - "reward-bench/Chat Hard": 0.7939, - "reward-bench/Safety": 0.8926, - "reward-bench/Reasoning": 0.9429 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/auraindustries.json b/data/developers/auraindustries.json deleted file mode 100644 index 8e7fe10b080a62af2b3a06d2506e02c3caac5d9d..0000000000000000000000000000000000000000 --- a/data/developers/auraindustries.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "developer": "AuraIndustries", - "models": [ - { - "id": "AuraIndustries/Aura-4B", - "name": "Aura-4B", - "developer": "AuraIndustries", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3816, - "hfopenllm_v2/BBH": 0.449, - "hfopenllm_v2/MATH Level 5": 0.0423, - "hfopenllm_v2/GPQA": 0.2878, - "hfopenllm_v2/MUSR": 0.3938, - "hfopenllm_v2/MMLU-PRO": 0.2706 - } - }, - { - "id": "AuraIndustries/Aura-8B", - "name": "Aura-8B", - "developer": "AuraIndustries", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7205, - "hfopenllm_v2/BBH": 0.5131, - "hfopenllm_v2/MATH Level 5": 0.1518, - "hfopenllm_v2/GPQA": 0.2861, - "hfopenllm_v2/MUSR": 0.4004, - "hfopenllm_v2/MMLU-PRO": 0.3874 - } - }, - { - "id": "AuraIndustries/Aura-MoE-2x4B", - "name": "Aura-MoE-2x4B", - "developer": "AuraIndustries", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4601, - "hfopenllm_v2/BBH": 0.4339, - "hfopenllm_v2/MATH Level 5": 0.031, - "hfopenllm_v2/GPQA": 0.2718, - "hfopenllm_v2/MUSR": 0.4085, - "hfopenllm_v2/MMLU-PRO": 0.265 - } - }, - { - "id": "AuraIndustries/Aura-MoE-2x4B-v2", - "name": "Aura-MoE-2x4B-v2", - "developer": "AuraIndustries", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4778, - "hfopenllm_v2/BBH": 0.4315, - "hfopenllm_v2/MATH Level 5": 0.0317, - "hfopenllm_v2/GPQA": 0.2878, - "hfopenllm_v2/MUSR": 0.4101, - "hfopenllm_v2/MMLU-PRO": 0.261 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/aurel9.json b/data/developers/aurel9.json deleted file mode 100644 index 5e0fcfe0146d9a08ebc2129738d2c199dbac5334..0000000000000000000000000000000000000000 --- a/data/developers/aurel9.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "Aurel9", - "models": [ - { - "id": "Aurel9/testmerge-7b", - "name": "testmerge-7b", - "developer": "Aurel9", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.398, - "hfopenllm_v2/BBH": 0.519, - "hfopenllm_v2/MATH Level 5": 0.0657, - "hfopenllm_v2/GPQA": 0.3003, - "hfopenllm_v2/MUSR": 0.4659, - "hfopenllm_v2/MMLU-PRO": 0.3053 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/automerger.json b/data/developers/automerger.json deleted file mode 100644 index 59ef7711446d3e502098944b67b8cdece7088cb8..0000000000000000000000000000000000000000 --- a/data/developers/automerger.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "automerger", - "models": [ - { - "id": "automerger/YamshadowExperiment28-7B", - "name": "YamshadowExperiment28-7B", - "developer": "automerger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.407, - "hfopenllm_v2/BBH": 0.515, - "hfopenllm_v2/MATH Level 5": 0.0612, - "hfopenllm_v2/GPQA": 0.2869, - "hfopenllm_v2/MUSR": 0.4306, - "hfopenllm_v2/MMLU-PRO": 0.306 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/avemio.json b/data/developers/avemio.json deleted file mode 100644 index 8f56cb959cd209d06e114ab7f3b1732cb4178d36..0000000000000000000000000000000000000000 --- a/data/developers/avemio.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "avemio", - "models": [ - { - "id": "avemio/GRAG-NEMO-12B-ORPO-HESSIAN-AI", - "name": "GRAG-NEMO-12B-ORPO-HESSIAN-AI", - "developer": "avemio", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0, - "hfopenllm_v2/BBH": 0.2607, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3447, - "hfopenllm_v2/MMLU-PRO": 0.1061 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/awnr.json b/data/developers/awnr.json deleted file mode 100644 index 8c06c6f8375ba1d9602d9190503bb9a8a4e2be1c..0000000000000000000000000000000000000000 --- a/data/developers/awnr.json +++ /dev/null @@ -1,75 +0,0 @@ -{ - "developer": "awnr", - "models": [ - { - "id": "awnr/Mistral-7B-v0.1-signtensors-1-over-2", - "name": "Mistral-7B-v0.1-signtensors-1-over-2", - "developer": "awnr", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2179, - "hfopenllm_v2/BBH": 0.4423, - "hfopenllm_v2/MATH Level 5": 0.034, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.4006, - "hfopenllm_v2/MMLU-PRO": 0.3 - } - }, - { - "id": "awnr/Mistral-7B-v0.1-signtensors-1-over-4", - "name": "Mistral-7B-v0.1-signtensors-1-over-4", - "developer": "awnr", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2133, - "hfopenllm_v2/BBH": 0.3507, - "hfopenllm_v2/MATH Level 5": 0.0249, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.346, - "hfopenllm_v2/MMLU-PRO": 0.2311 - } - }, - { - "id": "awnr/Mistral-7B-v0.1-signtensors-3-over-8", - "name": "Mistral-7B-v0.1-signtensors-3-over-8", - "developer": "awnr", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2394, - "hfopenllm_v2/BBH": 0.43, - "hfopenllm_v2/MATH Level 5": 0.0332, - "hfopenllm_v2/GPQA": 0.3037, - "hfopenllm_v2/MUSR": 0.3818, - "hfopenllm_v2/MMLU-PRO": 0.3001 - } - }, - { - "id": "awnr/Mistral-7B-v0.1-signtensors-5-over-16", - "name": "Mistral-7B-v0.1-signtensors-5-over-16", - "developer": "awnr", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2118, - "hfopenllm_v2/BBH": 0.4124, - "hfopenllm_v2/MATH Level 5": 0.0295, - "hfopenllm_v2/GPQA": 0.281, - "hfopenllm_v2/MUSR": 0.3686, - "hfopenllm_v2/MMLU-PRO": 0.2958 - } - }, - { - "id": "awnr/Mistral-7B-v0.1-signtensors-7-over-16", - "name": "Mistral-7B-v0.1-signtensors-7-over-16", - "developer": "awnr", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2294, - "hfopenllm_v2/BBH": 0.4316, - "hfopenllm_v2/MATH Level 5": 0.0385, - "hfopenllm_v2/GPQA": 0.3037, - "hfopenllm_v2/MUSR": 0.3952, - "hfopenllm_v2/MMLU-PRO": 0.303 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/aws-prototyping.json b/data/developers/aws-prototyping.json deleted file mode 100644 index f6b88d0b982c2c112edab66cdfd44e4db0d9da5f..0000000000000000000000000000000000000000 --- a/data/developers/aws-prototyping.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "aws-prototyping", - "models": [ - { - "id": "aws-prototyping/MegaBeam-Mistral-7B-512k", - "name": "MegaBeam-Mistral-7B-512k", - "developer": "aws-prototyping", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5973, - "hfopenllm_v2/BBH": 0.3662, - "hfopenllm_v2/MATH Level 5": 0.0287, - "hfopenllm_v2/GPQA": 0.2827, - "hfopenllm_v2/MUSR": 0.3994, - "hfopenllm_v2/MMLU-PRO": 0.2589 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/axolotl-ai-co.json b/data/developers/axolotl-ai-co.json deleted file mode 100644 index 9ec75b3ec427478e4061d744c4239ca43009e1be..0000000000000000000000000000000000000000 --- a/data/developers/axolotl-ai-co.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "axolotl-ai-co", - "models": [ - { - "id": "axolotl-ai-co/romulus-mistral-nemo-12b-simpo", - "name": "romulus-mistral-nemo-12b-simpo", - "developer": "axolotl-ai-co", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6079, - "hfopenllm_v2/BBH": 0.5395, - "hfopenllm_v2/MATH Level 5": 0.114, - "hfopenllm_v2/GPQA": 0.2785, - "hfopenllm_v2/MUSR": 0.4233, - "hfopenllm_v2/MMLU-PRO": 0.3469 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/ayush-singh.json b/data/developers/ayush-singh.json deleted file mode 100644 index 19ff6c763c83495b47b9f84520d63cbf1a709765..0000000000000000000000000000000000000000 --- a/data/developers/ayush-singh.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "Ayush-Singh", - "models": [ - { - "id": "Ayush-Singh/Llama1B-sft-2", - "name": "Llama1B-sft-2", - "developer": "Ayush-Singh", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1374, - "hfopenllm_v2/BBH": 0.2834, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2458, - "hfopenllm_v2/MUSR": 0.3552, - "hfopenllm_v2/MMLU-PRO": 0.1117 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/azure99.json b/data/developers/azure99.json deleted file mode 100644 index 17b4e4f54c300e654b72150dff036be150cc2427..0000000000000000000000000000000000000000 --- a/data/developers/azure99.json +++ /dev/null @@ -1,89 +0,0 @@ -{ - "developer": "Azure99", - "models": [ - { - "id": "Azure99/blossom-v5-32b", - "name": "blossom-v5-32b", - "developer": "Azure99", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5235, - "hfopenllm_v2/BBH": 0.5955, - "hfopenllm_v2/MATH Level 5": 0.1866, - "hfopenllm_v2/GPQA": 0.3112, - "hfopenllm_v2/MUSR": 0.402, - "hfopenllm_v2/MMLU-PRO": 0.4235 - } - }, - { - "id": "Azure99/blossom-v5-llama3-8b", - "name": "blossom-v5-llama3-8b", - "developer": "Azure99", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4343, - "hfopenllm_v2/BBH": 0.4185, - "hfopenllm_v2/MATH Level 5": 0.0514, - "hfopenllm_v2/GPQA": 0.2651, - "hfopenllm_v2/MUSR": 0.367, - "hfopenllm_v2/MMLU-PRO": 0.2206 - } - }, - { - "id": "Azure99/blossom-v5.1-34b", - "name": "blossom-v5.1-34b", - "developer": "Azure99", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5697, - "hfopenllm_v2/BBH": 0.6109, - "hfopenllm_v2/MATH Level 5": 0.2591, - "hfopenllm_v2/GPQA": 0.3096, - "hfopenllm_v2/MUSR": 0.3928, - "hfopenllm_v2/MMLU-PRO": 0.4558 - } - }, - { - "id": "Azure99/blossom-v5.1-9b", - "name": "blossom-v5.1-9b", - "developer": "Azure99", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5086, - "hfopenllm_v2/BBH": 0.5343, - "hfopenllm_v2/MATH Level 5": 0.2122, - "hfopenllm_v2/GPQA": 0.3356, - "hfopenllm_v2/MUSR": 0.3994, - "hfopenllm_v2/MMLU-PRO": 0.3979 - } - }, - { - "id": "Azure99/Blossom-V6-14B", - "name": "Blossom-V6-14B", - "developer": "Azure99", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6395, - "hfopenllm_v2/BBH": 0.5069, - "hfopenllm_v2/MATH Level 5": 0.5257, - "hfopenllm_v2/GPQA": 0.2626, - "hfopenllm_v2/MUSR": 0.4035, - "hfopenllm_v2/MMLU-PRO": 0.4544 - } - }, - { - "id": "Azure99/Blossom-V6-7B", - "name": "Blossom-V6-7B", - "developer": "Azure99", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5538, - "hfopenllm_v2/BBH": 0.4974, - "hfopenllm_v2/MATH Level 5": 0.4585, - "hfopenllm_v2/GPQA": 0.3045, - "hfopenllm_v2/MUSR": 0.4301, - "hfopenllm_v2/MMLU-PRO": 0.4144 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/ba2han.json b/data/developers/ba2han.json deleted file mode 100644 index 09d3c8ab2b17ba42025a7a080c5b86be0e1095b4..0000000000000000000000000000000000000000 --- a/data/developers/ba2han.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "Ba2han", - "models": [ - { - "id": "Ba2han/Llama-Phi-3_DoRA", - "name": "Llama-Phi-3_DoRA", - "developer": "Ba2han", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5131, - "hfopenllm_v2/BBH": 0.5515, - "hfopenllm_v2/MATH Level 5": 0.1216, - "hfopenllm_v2/GPQA": 0.3263, - "hfopenllm_v2/MUSR": 0.4069, - "hfopenllm_v2/MMLU-PRO": 0.3915 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/baai.json b/data/developers/baai.json deleted file mode 100644 index a1ddaa9d361ef4811d7a7b68f818f5fc67e4ede0..0000000000000000000000000000000000000000 --- a/data/developers/baai.json +++ /dev/null @@ -1,201 +0,0 @@ -{ - "developer": "BAAI", - "models": [ - { - "id": "BAAI/Gemma2-9B-IT-Simpo-Infinity-Preference", - "name": "Gemma2-9B-IT-Simpo-Infinity-Preference", - "developer": "BAAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3176, - "hfopenllm_v2/BBH": 0.5979, - "hfopenllm_v2/MATH Level 5": 0.0974, - "hfopenllm_v2/GPQA": 0.3398, - "hfopenllm_v2/MUSR": 0.3966, - "hfopenllm_v2/MMLU-PRO": 0.3869 - } - }, - { - "id": "BAAI/Infinity-Instruct-3M-0613-Llama3-70B", - "name": "Infinity-Instruct-3M-0613-Llama3-70B", - "developer": "BAAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6821, - "hfopenllm_v2/BBH": 0.6642, - "hfopenllm_v2/MATH Level 5": 0.2153, - "hfopenllm_v2/GPQA": 0.3582, - "hfopenllm_v2/MUSR": 0.4523, - "hfopenllm_v2/MMLU-PRO": 0.473 - } - }, - { - "id": "BAAI/Infinity-Instruct-3M-0613-Mistral-7B", - "name": "Infinity-Instruct-3M-0613-Mistral-7B", - "developer": "BAAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.532, - "hfopenllm_v2/BBH": 0.4958, - "hfopenllm_v2/MATH Level 5": 0.0816, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.4351, - "hfopenllm_v2/MMLU-PRO": 0.3161 - } - }, - { - "id": "BAAI/Infinity-Instruct-3M-0625-Llama3-70B", - "name": "Infinity-Instruct-3M-0625-Llama3-70B", - "developer": "BAAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7442, - "hfopenllm_v2/BBH": 0.667, - "hfopenllm_v2/MATH Level 5": 0.2251, - "hfopenllm_v2/GPQA": 0.3574, - "hfopenllm_v2/MUSR": 0.4617, - "hfopenllm_v2/MMLU-PRO": 0.4586 - } - }, - { - "id": "BAAI/Infinity-Instruct-3M-0625-Llama3-8B", - "name": "Infinity-Instruct-3M-0625-Llama3-8B", - "developer": "BAAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.605, - "hfopenllm_v2/BBH": 0.4955, - "hfopenllm_v2/MATH Level 5": 0.0884, - "hfopenllm_v2/GPQA": 0.2752, - "hfopenllm_v2/MUSR": 0.3712, - "hfopenllm_v2/MMLU-PRO": 0.3252 - } - }, - { - "id": "BAAI/Infinity-Instruct-3M-0625-Mistral-7B", - "name": "Infinity-Instruct-3M-0625-Mistral-7B", - "developer": "BAAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5867, - "hfopenllm_v2/BBH": 0.494, - "hfopenllm_v2/MATH Level 5": 0.0763, - "hfopenllm_v2/GPQA": 0.2869, - "hfopenllm_v2/MUSR": 0.4272, - "hfopenllm_v2/MMLU-PRO": 0.323 - } - }, - { - "id": "BAAI/Infinity-Instruct-3M-0625-Qwen2-7B", - "name": "Infinity-Instruct-3M-0625-Qwen2-7B", - "developer": "BAAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5554, - "hfopenllm_v2/BBH": 0.5346, - "hfopenllm_v2/MATH Level 5": 0.1926, - "hfopenllm_v2/GPQA": 0.3129, - "hfopenllm_v2/MUSR": 0.3888, - "hfopenllm_v2/MMLU-PRO": 0.396 - } - }, - { - "id": "BAAI/Infinity-Instruct-3M-0625-Yi-1.5-9B", - "name": "Infinity-Instruct-3M-0625-Yi-1.5-9B", - "developer": "BAAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5186, - "hfopenllm_v2/BBH": 0.5509, - "hfopenllm_v2/MATH Level 5": 0.1639, - "hfopenllm_v2/GPQA": 0.354, - "hfopenllm_v2/MUSR": 0.4575, - "hfopenllm_v2/MMLU-PRO": 0.4118 - } - }, - { - "id": "BAAI/Infinity-Instruct-7M-0729-Llama3_1-8B", - "name": "Infinity-Instruct-7M-0729-Llama3_1-8B", - "developer": "BAAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6132, - "hfopenllm_v2/BBH": 0.5077, - "hfopenllm_v2/MATH Level 5": 0.1276, - "hfopenllm_v2/GPQA": 0.2928, - "hfopenllm_v2/MUSR": 0.3578, - "hfopenllm_v2/MMLU-PRO": 0.3224 - } - }, - { - "id": "BAAI/Infinity-Instruct-7M-0729-mistral-7B", - "name": "Infinity-Instruct-7M-0729-mistral-7B", - "developer": "BAAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6162, - "hfopenllm_v2/BBH": 0.4964, - "hfopenllm_v2/MATH Level 5": 0.0831, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.4062, - "hfopenllm_v2/MMLU-PRO": 0.3274 - } - }, - { - "id": "BAAI/Infinity-Instruct-7M-Gen-Llama3_1-70B", - "name": "Infinity-Instruct-7M-Gen-Llama3_1-70B", - "developer": "BAAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7335, - "hfopenllm_v2/BBH": 0.6695, - "hfopenllm_v2/MATH Level 5": 0.2523, - "hfopenllm_v2/GPQA": 0.3758, - "hfopenllm_v2/MUSR": 0.4539, - "hfopenllm_v2/MMLU-PRO": 0.4607 - } - }, - { - "id": "BAAI/Infinity-Instruct-7M-Gen-Llama3_1-8B", - "name": "Infinity-Instruct-7M-Gen-Llama3_1-8B", - "developer": "BAAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6132, - "hfopenllm_v2/BBH": 0.5077, - "hfopenllm_v2/MATH Level 5": 0.1276, - "hfopenllm_v2/GPQA": 0.2928, - "hfopenllm_v2/MUSR": 0.3578, - "hfopenllm_v2/MMLU-PRO": 0.3224 - } - }, - { - "id": "BAAI/Infinity-Instruct-7M-Gen-mistral-7B", - "name": "Infinity-Instruct-7M-Gen-mistral-7B", - "developer": "BAAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6147, - "hfopenllm_v2/BBH": 0.4964, - "hfopenllm_v2/MATH Level 5": 0.0831, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.4062, - "hfopenllm_v2/MMLU-PRO": 0.3274 - } - }, - { - "id": "BAAI/OPI-Llama-3.1-8B-Instruct", - "name": "OPI-Llama-3.1-8B-Instruct", - "developer": "BAAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2075, - "hfopenllm_v2/BBH": 0.3551, - "hfopenllm_v2/MATH Level 5": 0.0136, - "hfopenllm_v2/GPQA": 0.2743, - "hfopenllm_v2/MUSR": 0.3233, - "hfopenllm_v2/MMLU-PRO": 0.2124 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/baconnier.json b/data/developers/baconnier.json deleted file mode 100644 index e71ba7e1addf9b831d19fc8062eac0270044a36b..0000000000000000000000000000000000000000 --- a/data/developers/baconnier.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "baconnier", - "models": [ - { - "id": "baconnier/Napoleon_24B_V0.0", - "name": "Napoleon_24B_V0.0", - "developer": "baconnier", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1801, - "hfopenllm_v2/BBH": 0.6367, - "hfopenllm_v2/MATH Level 5": 0.2273, - "hfopenllm_v2/GPQA": 0.3792, - "hfopenllm_v2/MUSR": 0.442, - "hfopenllm_v2/MMLU-PRO": 0.504 - } - }, - { - "id": "baconnier/Napoleon_24B_V0.2", - "name": "Napoleon_24B_V0.2", - "developer": "baconnier", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2527, - "hfopenllm_v2/BBH": 0.5911, - "hfopenllm_v2/MATH Level 5": 0.1435, - "hfopenllm_v2/GPQA": 0.3381, - "hfopenllm_v2/MUSR": 0.446, - "hfopenllm_v2/MMLU-PRO": 0.4357 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/baebee.json b/data/developers/baebee.json deleted file mode 100644 index e4726f4d0644277ef595905d47aad6feb287568e..0000000000000000000000000000000000000000 --- a/data/developers/baebee.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "developer": "baebee", - "models": [ - { - "id": "baebee/7B-Cetacea", - "name": "7B-Cetacea", - "developer": "baebee", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5279, - "hfopenllm_v2/BBH": 0.4757, - "hfopenllm_v2/MATH Level 5": 0.0468, - "hfopenllm_v2/GPQA": 0.2861, - "hfopenllm_v2/MUSR": 0.4136, - "hfopenllm_v2/MMLU-PRO": 0.2955 - } - }, - { - "id": "baebee/mergekit-model_stock-nzjnheg", - "name": "mergekit-model_stock-nzjnheg", - "developer": "baebee", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4844, - "hfopenllm_v2/BBH": 0.5287, - "hfopenllm_v2/MATH Level 5": 0.1677, - "hfopenllm_v2/GPQA": 0.2802, - "hfopenllm_v2/MUSR": 0.3847, - "hfopenllm_v2/MMLU-PRO": 0.3699 - } - }, - { - "id": "baebee/mergekit-ties-fnjenli", - "name": "mergekit-ties-fnjenli", - "developer": "baebee", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1988, - "hfopenllm_v2/BBH": 0.3024, - "hfopenllm_v2/MATH Level 5": 0.0023, - "hfopenllm_v2/GPQA": 0.245, - "hfopenllm_v2/MUSR": 0.4019, - "hfopenllm_v2/MMLU-PRO": 0.1129 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/bamec66557.json b/data/developers/bamec66557.json deleted file mode 100644 index 6ee85cfaa2353754fb9ba639f4b4d697e5714b91..0000000000000000000000000000000000000000 --- a/data/developers/bamec66557.json +++ /dev/null @@ -1,383 +0,0 @@ -{ - "developer": "bamec66557", - "models": [ - { - "id": "bamec66557/mergekit-model_stock-zdaysvi", - "name": "mergekit-model_stock-zdaysvi", - "developer": "bamec66557", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6426, - "hfopenllm_v2/BBH": 0.5063, - "hfopenllm_v2/MATH Level 5": 0.1352, - "hfopenllm_v2/GPQA": 0.3138, - "hfopenllm_v2/MUSR": 0.4124, - "hfopenllm_v2/MMLU-PRO": 0.3688 - } - }, - { - "id": "bamec66557/mergekit-ties-sinbkow", - "name": "mergekit-ties-sinbkow", - "developer": "bamec66557", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6432, - "hfopenllm_v2/BBH": 0.5092, - "hfopenllm_v2/MATH Level 5": 0.145, - "hfopenllm_v2/GPQA": 0.3196, - "hfopenllm_v2/MUSR": 0.4045, - "hfopenllm_v2/MMLU-PRO": 0.3603 - } - }, - { - "id": "bamec66557/MISCHIEVOUS-12B", - "name": "MISCHIEVOUS-12B", - "developer": "bamec66557", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3852, - "hfopenllm_v2/BBH": 0.5405, - "hfopenllm_v2/MATH Level 5": 0.1276, - "hfopenllm_v2/GPQA": 0.3205, - "hfopenllm_v2/MUSR": 0.4145, - "hfopenllm_v2/MMLU-PRO": 0.3672 - } - }, - { - "id": "bamec66557/MISCHIEVOUS-12B-Mix_0.1v", - "name": "MISCHIEVOUS-12B-Mix_0.1v", - "developer": "bamec66557", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3636, - "hfopenllm_v2/BBH": 0.5436, - "hfopenllm_v2/MATH Level 5": 0.1329, - "hfopenllm_v2/GPQA": 0.328, - "hfopenllm_v2/MUSR": 0.4132, - "hfopenllm_v2/MMLU-PRO": 0.3674 - } - }, - { - "id": "bamec66557/MISCHIEVOUS-12B-Mix_0.2v", - "name": "MISCHIEVOUS-12B-Mix_0.2v", - "developer": "bamec66557", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3624, - "hfopenllm_v2/BBH": 0.5434, - "hfopenllm_v2/MATH Level 5": 0.1261, - "hfopenllm_v2/GPQA": 0.3255, - "hfopenllm_v2/MUSR": 0.4158, - "hfopenllm_v2/MMLU-PRO": 0.3663 - } - }, - { - "id": "bamec66557/MISCHIEVOUS-12B-Mix_0.3v", - "name": "MISCHIEVOUS-12B-Mix_0.3v", - "developer": "bamec66557", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.387, - "hfopenllm_v2/BBH": 0.5431, - "hfopenllm_v2/MATH Level 5": 0.1337, - "hfopenllm_v2/GPQA": 0.3196, - "hfopenllm_v2/MUSR": 0.4131, - "hfopenllm_v2/MMLU-PRO": 0.3664 - } - }, - { - "id": "bamec66557/MISCHIEVOUS-12B-Mix_0.4v", - "name": "MISCHIEVOUS-12B-Mix_0.4v", - "developer": "bamec66557", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6508, - "hfopenllm_v2/BBH": 0.5094, - "hfopenllm_v2/MATH Level 5": 0.1352, - "hfopenllm_v2/GPQA": 0.3171, - "hfopenllm_v2/MUSR": 0.4176, - "hfopenllm_v2/MMLU-PRO": 0.3683 - } - }, - { - "id": "bamec66557/MISCHIEVOUS-12B-Mix_0.5v", - "name": "MISCHIEVOUS-12B-Mix_0.5v", - "developer": "bamec66557", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3746, - "hfopenllm_v2/BBH": 0.5422, - "hfopenllm_v2/MATH Level 5": 0.1367, - "hfopenllm_v2/GPQA": 0.3205, - "hfopenllm_v2/MUSR": 0.4132, - "hfopenllm_v2/MMLU-PRO": 0.3661 - } - }, - { - "id": "bamec66557/MISCHIEVOUS-12B-Mix_0.6v", - "name": "MISCHIEVOUS-12B-Mix_0.6v", - "developer": "bamec66557", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4366, - "hfopenllm_v2/BBH": 0.5449, - "hfopenllm_v2/MATH Level 5": 0.1254, - "hfopenllm_v2/GPQA": 0.328, - "hfopenllm_v2/MUSR": 0.4185, - "hfopenllm_v2/MMLU-PRO": 0.3662 - } - }, - { - "id": "bamec66557/MISCHIEVOUS-12B-Mix_III_ex_V", - "name": "MISCHIEVOUS-12B-Mix_III_ex_V", - "developer": "bamec66557", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4316, - "hfopenllm_v2/BBH": 0.5449, - "hfopenllm_v2/MATH Level 5": 0.1322, - "hfopenllm_v2/GPQA": 0.3205, - "hfopenllm_v2/MUSR": 0.4198, - "hfopenllm_v2/MMLU-PRO": 0.3649 - } - }, - { - "id": "bamec66557/MISCHIEVOUS-12B-Mix_III_IV_V", - "name": "MISCHIEVOUS-12B-Mix_III_IV_V", - "developer": "bamec66557", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4031, - "hfopenllm_v2/BBH": 0.5465, - "hfopenllm_v2/MATH Level 5": 0.1292, - "hfopenllm_v2/GPQA": 0.3205, - "hfopenllm_v2/MUSR": 0.4198, - "hfopenllm_v2/MMLU-PRO": 0.3664 - } - }, - { - "id": "bamec66557/MISCHIEVOUS-12B-Mix_Neo", - "name": "MISCHIEVOUS-12B-Mix_Neo", - "developer": "bamec66557", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.625, - "hfopenllm_v2/BBH": 0.5078, - "hfopenllm_v2/MATH Level 5": 0.1329, - "hfopenllm_v2/GPQA": 0.3163, - "hfopenllm_v2/MUSR": 0.415, - "hfopenllm_v2/MMLU-PRO": 0.3685 - } - }, - { - "id": "bamec66557/Mistral-Nemo-VICIOUS_MESH-12B-2407", - "name": "Mistral-Nemo-VICIOUS_MESH-12B-2407", - "developer": "bamec66557", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6706, - "hfopenllm_v2/BBH": 0.5156, - "hfopenllm_v2/MATH Level 5": 0.1367, - "hfopenllm_v2/GPQA": 0.3154, - "hfopenllm_v2/MUSR": 0.431, - "hfopenllm_v2/MMLU-PRO": 0.3677 - } - }, - { - "id": "bamec66557/NameLess-12B-prob", - "name": "NameLess-12B-prob", - "developer": "bamec66557", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6602, - "hfopenllm_v2/BBH": 0.5158, - "hfopenllm_v2/MATH Level 5": 0.1261, - "hfopenllm_v2/GPQA": 0.3146, - "hfopenllm_v2/MUSR": 0.4336, - "hfopenllm_v2/MMLU-PRO": 0.3684 - } - }, - { - "id": "bamec66557/VICIOUS_MESH-12B", - "name": "VICIOUS_MESH-12B", - "developer": "bamec66557", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3716, - "hfopenllm_v2/BBH": 0.5436, - "hfopenllm_v2/MATH Level 5": 0.1344, - "hfopenllm_v2/GPQA": 0.328, - "hfopenllm_v2/MUSR": 0.4105, - "hfopenllm_v2/MMLU-PRO": 0.3679 - } - }, - { - "id": "bamec66557/VICIOUS_MESH-12B-0.1v", - "name": "VICIOUS_MESH-12B-0.1v", - "developer": "bamec66557", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3657, - "hfopenllm_v2/BBH": 0.5412, - "hfopenllm_v2/MATH Level 5": 0.1322, - "hfopenllm_v2/GPQA": 0.3247, - "hfopenllm_v2/MUSR": 0.4158, - "hfopenllm_v2/MMLU-PRO": 0.3683 - } - }, - { - "id": "bamec66557/VICIOUS_MESH-12B-0.X.ver", - "name": "VICIOUS_MESH-12B-0.X.ver", - "developer": "bamec66557", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3776, - "hfopenllm_v2/BBH": 0.5416, - "hfopenllm_v2/MATH Level 5": 0.1201, - "hfopenllm_v2/GPQA": 0.3213, - "hfopenllm_v2/MUSR": 0.4198, - "hfopenllm_v2/MMLU-PRO": 0.3671 - } - }, - { - "id": "bamec66557/VICIOUS_MESH-12B-ALPHA", - "name": "VICIOUS_MESH-12B-ALPHA", - "developer": "bamec66557", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6365, - "hfopenllm_v2/BBH": 0.5094, - "hfopenllm_v2/MATH Level 5": 0.1367, - "hfopenllm_v2/GPQA": 0.3138, - "hfopenllm_v2/MUSR": 0.4203, - "hfopenllm_v2/MMLU-PRO": 0.3697 - } - }, - { - "id": "bamec66557/VICIOUS_MESH-12B-BETA", - "name": "VICIOUS_MESH-12B-BETA", - "developer": "bamec66557", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6721, - "hfopenllm_v2/BBH": 0.5156, - "hfopenllm_v2/MATH Level 5": 0.1329, - "hfopenllm_v2/GPQA": 0.3163, - "hfopenllm_v2/MUSR": 0.431, - "hfopenllm_v2/MMLU-PRO": 0.3679 - } - }, - { - "id": "bamec66557/VICIOUS_MESH-12B-DELTA", - "name": "VICIOUS_MESH-12B-DELTA", - "developer": "bamec66557", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6469, - "hfopenllm_v2/BBH": 0.5055, - "hfopenllm_v2/MATH Level 5": 0.1375, - "hfopenllm_v2/GPQA": 0.3121, - "hfopenllm_v2/MUSR": 0.4057, - "hfopenllm_v2/MMLU-PRO": 0.3651 - } - }, - { - "id": "bamec66557/VICIOUS_MESH-12B-DIGAMMA", - "name": "VICIOUS_MESH-12B-DIGAMMA", - "developer": "bamec66557", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6429, - "hfopenllm_v2/BBH": 0.5061, - "hfopenllm_v2/MATH Level 5": 0.1337, - "hfopenllm_v2/GPQA": 0.3129, - "hfopenllm_v2/MUSR": 0.4097, - "hfopenllm_v2/MMLU-PRO": 0.3659 - } - }, - { - "id": "bamec66557/VICIOUS_MESH-12B-EPSILON", - "name": "VICIOUS_MESH-12B-EPSILON", - "developer": "bamec66557", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6305, - "hfopenllm_v2/BBH": 0.5038, - "hfopenllm_v2/MATH Level 5": 0.1261, - "hfopenllm_v2/GPQA": 0.3146, - "hfopenllm_v2/MUSR": 0.407, - "hfopenllm_v2/MMLU-PRO": 0.3648 - } - }, - { - "id": "bamec66557/VICIOUS_MESH-12B-GAMMA", - "name": "VICIOUS_MESH-12B-GAMMA", - "developer": "bamec66557", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6362, - "hfopenllm_v2/BBH": 0.5182, - "hfopenllm_v2/MATH Level 5": 0.1307, - "hfopenllm_v2/GPQA": 0.3138, - "hfopenllm_v2/MUSR": 0.4363, - "hfopenllm_v2/MMLU-PRO": 0.3666 - } - }, - { - "id": "bamec66557/VICIOUS_MESH-12B-NEMO", - "name": "VICIOUS_MESH-12B-NEMO", - "developer": "bamec66557", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4022, - "hfopenllm_v2/BBH": 0.5442, - "hfopenllm_v2/MATH Level 5": 0.1269, - "hfopenllm_v2/GPQA": 0.3238, - "hfopenllm_v2/MUSR": 0.4251, - "hfopenllm_v2/MMLU-PRO": 0.3716 - } - }, - { - "id": "bamec66557/VICIOUS_MESH-12B-OMEGA", - "name": "VICIOUS_MESH-12B-OMEGA", - "developer": "bamec66557", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.67, - "hfopenllm_v2/BBH": 0.5166, - "hfopenllm_v2/MATH Level 5": 0.1344, - "hfopenllm_v2/GPQA": 0.3154, - "hfopenllm_v2/MUSR": 0.4323, - "hfopenllm_v2/MMLU-PRO": 0.3677 - } - }, - { - "id": "bamec66557/VICIOUS_MESH-12B-UNION", - "name": "VICIOUS_MESH-12B-UNION", - "developer": "bamec66557", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6429, - "hfopenllm_v2/BBH": 0.5107, - "hfopenllm_v2/MATH Level 5": 0.139, - "hfopenllm_v2/GPQA": 0.3121, - "hfopenllm_v2/MUSR": 0.4257, - "hfopenllm_v2/MMLU-PRO": 0.3672 - } - }, - { - "id": "bamec66557/VICIOUS_MESH-12B_Razor", - "name": "VICIOUS_MESH-12B_Razor", - "developer": "bamec66557", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3736, - "hfopenllm_v2/BBH": 0.5447, - "hfopenllm_v2/MATH Level 5": 0.1299, - "hfopenllm_v2/GPQA": 0.323, - "hfopenllm_v2/MUSR": 0.4092, - "hfopenllm_v2/MMLU-PRO": 0.3669 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/baptiste-huvelle-10.json b/data/developers/baptiste-huvelle-10.json deleted file mode 100644 index 525db3466746c4a31d8f8e15019e7928407e064a..0000000000000000000000000000000000000000 --- a/data/developers/baptiste-huvelle-10.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "Baptiste-HUVELLE-10", - "models": [ - { - "id": "Baptiste-HUVELLE-10/LeTriomphant2.2_ECE_iLAB", - "name": "LeTriomphant2.2_ECE_iLAB", - "developer": "Baptiste-HUVELLE-10", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5076, - "hfopenllm_v2/BBH": 0.7256, - "hfopenllm_v2/MATH Level 5": 0.4449, - "hfopenllm_v2/GPQA": 0.3993, - "hfopenllm_v2/MUSR": 0.4626, - "hfopenllm_v2/MMLU-PRO": 0.5851 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/bee-spoke-data.json b/data/developers/bee-spoke-data.json deleted file mode 100644 index fc03c3478ee30f4c9b29cc59141c814c84a9837c..0000000000000000000000000000000000000000 --- a/data/developers/bee-spoke-data.json +++ /dev/null @@ -1,131 +0,0 @@ -{ - "developer": "BEE-spoke-data", - "models": [ - { - "id": "BEE-spoke-data/Meta-Llama-3-8Bee", - "name": "Meta-Llama-3-8Bee", - "developer": "BEE-spoke-data", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1951, - "hfopenllm_v2/BBH": 0.4626, - "hfopenllm_v2/MATH Level 5": 0.0483, - "hfopenllm_v2/GPQA": 0.3138, - "hfopenllm_v2/MUSR": 0.3654, - "hfopenllm_v2/MMLU-PRO": 0.322 - } - }, - { - "id": "BEE-spoke-data/smol_llama-101M-GQA", - "name": "smol_llama-101M-GQA", - "developer": "BEE-spoke-data", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1384, - "hfopenllm_v2/BBH": 0.3018, - "hfopenllm_v2/MATH Level 5": 0.006, - "hfopenllm_v2/GPQA": 0.2576, - "hfopenllm_v2/MUSR": 0.3713, - "hfopenllm_v2/MMLU-PRO": 0.1107 - } - }, - { - "id": "BEE-spoke-data/smol_llama-220M-GQA", - "name": "smol_llama-220M-GQA", - "developer": "BEE-spoke-data", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2386, - "hfopenllm_v2/BBH": 0.3032, - "hfopenllm_v2/MATH Level 5": 0.0106, - "hfopenllm_v2/GPQA": 0.2559, - "hfopenllm_v2/MUSR": 0.4059, - "hfopenllm_v2/MMLU-PRO": 0.1149 - } - }, - { - "id": "BEE-spoke-data/smol_llama-220M-GQA-fineweb_edu", - "name": "smol_llama-220M-GQA-fineweb_edu", - "developer": "BEE-spoke-data", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1988, - "hfopenllm_v2/BBH": 0.2929, - "hfopenllm_v2/MATH Level 5": 0.0068, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.4368, - "hfopenllm_v2/MMLU-PRO": 0.1127 - } - }, - { - "id": "BEE-spoke-data/smol_llama-220M-openhermes", - "name": "smol_llama-220M-openhermes", - "developer": "BEE-spoke-data", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1555, - "hfopenllm_v2/BBH": 0.3028, - "hfopenllm_v2/MATH Level 5": 0.0106, - "hfopenllm_v2/GPQA": 0.2676, - "hfopenllm_v2/MUSR": 0.3847, - "hfopenllm_v2/MMLU-PRO": 0.112 - } - }, - { - "id": "BEE-spoke-data/tFINE-900m-e16-d32-flan", - "name": "tFINE-900m-e16-d32-flan", - "developer": "BEE-spoke-data", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1506, - "hfopenllm_v2/BBH": 0.3028, - "hfopenllm_v2/MATH Level 5": 0.0098, - "hfopenllm_v2/GPQA": 0.2332, - "hfopenllm_v2/MUSR": 0.3724, - "hfopenllm_v2/MMLU-PRO": 0.1307 - } - }, - { - "id": "BEE-spoke-data/tFINE-900m-e16-d32-flan-infinity-instruct-7m-T2T_en-1024", - "name": "tFINE-900m-e16-d32-flan-infinity-instruct-7m-T2T_en-1024", - "developer": "BEE-spoke-data", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1321, - "hfopenllm_v2/BBH": 0.3138, - "hfopenllm_v2/MATH Level 5": 0.0106, - "hfopenllm_v2/GPQA": 0.2542, - "hfopenllm_v2/MUSR": 0.4393, - "hfopenllm_v2/MMLU-PRO": 0.1237 - } - }, - { - "id": "BEE-spoke-data/tFINE-900m-e16-d32-instruct_2e", - "name": "tFINE-900m-e16-d32-instruct_2e", - "developer": "BEE-spoke-data", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1403, - "hfopenllm_v2/BBH": 0.3135, - "hfopenllm_v2/MATH Level 5": 0.0136, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.4207, - "hfopenllm_v2/MMLU-PRO": 0.1237 - } - }, - { - "id": "BEE-spoke-data/tFINE-900m-instruct-orpo", - "name": "tFINE-900m-instruct-orpo", - "developer": "BEE-spoke-data", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.133, - "hfopenllm_v2/BBH": 0.3022, - "hfopenllm_v2/MATH Level 5": 0.0159, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3409, - "hfopenllm_v2/MMLU-PRO": 0.1152 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/belztjti.json b/data/developers/belztjti.json deleted file mode 100644 index c079f10ae86fc88cfd453883e20491bd3134d711..0000000000000000000000000000000000000000 --- a/data/developers/belztjti.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "belztjti", - "models": [ - { - "id": "belztjti/dffghgjh", - "name": "dffghgjh", - "developer": "belztjti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5784, - "hfopenllm_v2/BBH": 0.3582, - "hfopenllm_v2/MATH Level 5": 0.0234, - "hfopenllm_v2/GPQA": 0.2634, - "hfopenllm_v2/MUSR": 0.3475, - "hfopenllm_v2/MMLU-PRO": 0.3422 - } - }, - { - "id": "belztjti/dtfgv", - "name": "dtfgv", - "developer": "belztjti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3345, - "hfopenllm_v2/BBH": 0.3282, - "hfopenllm_v2/MATH Level 5": 0.0181, - "hfopenllm_v2/GPQA": 0.2693, - "hfopenllm_v2/MUSR": 0.3794, - "hfopenllm_v2/MMLU-PRO": 0.1504 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/benevolencemessiah.json b/data/developers/benevolencemessiah.json deleted file mode 100644 index 1cb1df580e8e3ee414b19dafb8df7ce1e3771378..0000000000000000000000000000000000000000 --- a/data/developers/benevolencemessiah.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "BenevolenceMessiah", - "models": [ - { - "id": "BenevolenceMessiah/Qwen2.5-72B-2x-Instruct-TIES-v1.0", - "name": "Qwen2.5-72B-2x-Instruct-TIES-v1.0", - "developer": "BenevolenceMessiah", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5473, - "hfopenllm_v2/BBH": 0.7273, - "hfopenllm_v2/MATH Level 5": 0.5785, - "hfopenllm_v2/GPQA": 0.3674, - "hfopenllm_v2/MUSR": 0.4207, - "hfopenllm_v2/MMLU-PRO": 0.5628 - } - }, - { - "id": "BenevolenceMessiah/Yi-Coder-9B-Chat-Instruct-TIES-MoE-v1.0", - "name": "Yi-Coder-9B-Chat-Instruct-TIES-MoE-v1.0", - "developer": "BenevolenceMessiah", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3012, - "hfopenllm_v2/BBH": 0.4909, - "hfopenllm_v2/MATH Level 5": 0.0415, - "hfopenllm_v2/GPQA": 0.2626, - "hfopenllm_v2/MUSR": 0.408, - "hfopenllm_v2/MMLU-PRO": 0.268 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/benhaotang.json b/data/developers/benhaotang.json deleted file mode 100644 index 3cefd0491456319d63ecbc33ab43d51d9c5f6854..0000000000000000000000000000000000000000 --- a/data/developers/benhaotang.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "benhaotang", - "models": [ - { - "id": "benhaotang/phi4-qwq-sky-t1", - "name": "phi4-qwq-sky-t1", - "developer": "benhaotang", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.046, - "hfopenllm_v2/BBH": 0.6711, - "hfopenllm_v2/MATH Level 5": 0.4101, - "hfopenllm_v2/GPQA": 0.3951, - "hfopenllm_v2/MUSR": 0.49, - "hfopenllm_v2/MMLU-PRO": 0.5244 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/beomi.json b/data/developers/beomi.json deleted file mode 100644 index 48772acd42eb341b4f10f2f3869c9336f1fc1969..0000000000000000000000000000000000000000 --- a/data/developers/beomi.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "beomi", - "models": [ - { - "id": "beomi/gemma-mling-7b", - "name": "gemma-mling-7b", - "developer": "beomi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2029, - "hfopenllm_v2/BBH": 0.4068, - "hfopenllm_v2/MATH Level 5": 0.0544, - "hfopenllm_v2/GPQA": 0.25, - "hfopenllm_v2/MUSR": 0.3759, - "hfopenllm_v2/MMLU-PRO": 0.2633 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/beowolx.json b/data/developers/beowolx.json deleted file mode 100644 index b4783772930dc66317186f829aae454ef8aa3927..0000000000000000000000000000000000000000 --- a/data/developers/beowolx.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "beowolx", - "models": [ - { - "id": "beowolx/CodeNinja-1.0-OpenChat-7B", - "name": "CodeNinja-1.0-OpenChat-7B", - "developer": "beowolx", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5447, - "hfopenllm_v2/BBH": 0.4441, - "hfopenllm_v2/MATH Level 5": 0.0672, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.4243, - "hfopenllm_v2/MMLU-PRO": 0.3015 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/berkeley-nest.json b/data/developers/berkeley-nest.json deleted file mode 100644 index d8136759c2d7820038932fbb3bc27e3af2e373c3..0000000000000000000000000000000000000000 --- a/data/developers/berkeley-nest.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "berkeley-nest", - "models": [ - { - "id": "berkeley-nest/Starling-LM-7B-alpha", - "name": "Starling-LM-7B-alpha", - "developer": "berkeley-nest", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.548, - "hfopenllm_v2/BBH": 0.444, - "hfopenllm_v2/MATH Level 5": 0.0838, - "hfopenllm_v2/GPQA": 0.297, - "hfopenllm_v2/MUSR": 0.412, - "hfopenllm_v2/MMLU-PRO": 0.3172 - } - }, - { - "id": "berkeley-nest/Starling-RM-7B-alpha", - "name": "berkeley-nest/Starling-RM-7B-alpha", - "developer": "berkeley-nest", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7113, - "reward-bench/Chat": 0.9804, - "reward-bench/Chat Hard": 0.4561, - "reward-bench/Safety": 0.8446, - "reward-bench/Reasoning": 0.58, - "reward-bench/Prior Sets (0.5 weight)": 0.6794 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/bfuzzy1.json b/data/developers/bfuzzy1.json deleted file mode 100644 index 46500df679589fb1e95ba7283b442a10a752a892..0000000000000000000000000000000000000000 --- a/data/developers/bfuzzy1.json +++ /dev/null @@ -1,103 +0,0 @@ -{ - "developer": "bfuzzy1", - "models": [ - { - "id": "bfuzzy1/acheron", - "name": "acheron", - "developer": "bfuzzy1", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1983, - "hfopenllm_v2/BBH": 0.3108, - "hfopenllm_v2/MATH Level 5": 0.0166, - "hfopenllm_v2/GPQA": 0.2391, - "hfopenllm_v2/MUSR": 0.3511, - "hfopenllm_v2/MMLU-PRO": 0.1096 - } - }, - { - "id": "bfuzzy1/acheron-c", - "name": "acheron-c", - "developer": "bfuzzy1", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1929, - "hfopenllm_v2/BBH": 0.3026, - "hfopenllm_v2/MATH Level 5": 0.003, - "hfopenllm_v2/GPQA": 0.2475, - "hfopenllm_v2/MUSR": 0.3382, - "hfopenllm_v2/MMLU-PRO": 0.1172 - } - }, - { - "id": "bfuzzy1/acheron-d", - "name": "acheron-d", - "developer": "bfuzzy1", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1925, - "hfopenllm_v2/BBH": 0.314, - "hfopenllm_v2/MATH Level 5": 0.0151, - "hfopenllm_v2/GPQA": 0.2366, - "hfopenllm_v2/MUSR": 0.3497, - "hfopenllm_v2/MMLU-PRO": 0.1134 - } - }, - { - "id": "bfuzzy1/acheron-m", - "name": "acheron-m", - "developer": "bfuzzy1", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1758, - "hfopenllm_v2/BBH": 0.2928, - "hfopenllm_v2/MATH Level 5": 0.0091, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3487, - "hfopenllm_v2/MMLU-PRO": 0.1113 - } - }, - { - "id": "bfuzzy1/acheron-m1a-llama", - "name": "acheron-m1a-llama", - "developer": "bfuzzy1", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1125, - "hfopenllm_v2/BBH": 0.2956, - "hfopenllm_v2/MATH Level 5": 0.0076, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3633, - "hfopenllm_v2/MMLU-PRO": 0.1146 - } - }, - { - "id": "bfuzzy1/Gunny", - "name": "Gunny", - "developer": "bfuzzy1", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7129, - "hfopenllm_v2/BBH": 0.4546, - "hfopenllm_v2/MATH Level 5": 0.173, - "hfopenllm_v2/GPQA": 0.2785, - "hfopenllm_v2/MUSR": 0.3583, - "hfopenllm_v2/MMLU-PRO": 0.3039 - } - }, - { - "id": "bfuzzy1/llambses-1", - "name": "llambses-1", - "developer": "bfuzzy1", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3554, - "hfopenllm_v2/BBH": 0.5047, - "hfopenllm_v2/MATH Level 5": 0.0687, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.4529, - "hfopenllm_v2/MMLU-PRO": 0.314 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/bhuvneshsaini.json b/data/developers/bhuvneshsaini.json deleted file mode 100644 index db93b0aa253c65665bb6c422aba8d2220cea00cc..0000000000000000000000000000000000000000 --- a/data/developers/bhuvneshsaini.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "bhuvneshsaini", - "models": [ - { - "id": "bhuvneshsaini/merged_model", - "name": "merged_model", - "developer": "bhuvneshsaini", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1813, - "hfopenllm_v2/BBH": 0.336, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.25, - "hfopenllm_v2/MUSR": 0.3497, - "hfopenllm_v2/MMLU-PRO": 0.1445 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/bigcode.json b/data/developers/bigcode.json deleted file mode 100644 index 49c489c55c2d18098aa824060d42d3ff9fd39a20..0000000000000000000000000000000000000000 --- a/data/developers/bigcode.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "developer": "bigcode", - "models": [ - { - "id": "bigcode/starcoder2-15b", - "name": "starcoder2-15b", - "developer": "bigcode", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.278, - "hfopenllm_v2/BBH": 0.4448, - "hfopenllm_v2/MATH Level 5": 0.0597, - "hfopenllm_v2/GPQA": 0.2735, - "hfopenllm_v2/MUSR": 0.3501, - "hfopenllm_v2/MMLU-PRO": 0.2353 - } - }, - { - "id": "bigcode/starcoder2-3b", - "name": "starcoder2-3b", - "developer": "bigcode", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2037, - "hfopenllm_v2/BBH": 0.3509, - "hfopenllm_v2/MATH Level 5": 0.0151, - "hfopenllm_v2/GPQA": 0.2441, - "hfopenllm_v2/MUSR": 0.3435, - "hfopenllm_v2/MMLU-PRO": 0.1636 - } - }, - { - "id": "bigcode/starcoder2-7b", - "name": "starcoder2-7b", - "developer": "bigcode", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2209, - "hfopenllm_v2/BBH": 0.3661, - "hfopenllm_v2/MATH Level 5": 0.031, - "hfopenllm_v2/GPQA": 0.2517, - "hfopenllm_v2/MUSR": 0.3793, - "hfopenllm_v2/MMLU-PRO": 0.1642 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/bigscience.json b/data/developers/bigscience.json deleted file mode 100644 index 988f8f7f8c610e750ec9a03223415b194ffc1e2a..0000000000000000000000000000000000000000 --- a/data/developers/bigscience.json +++ /dev/null @@ -1,121 +0,0 @@ -{ - "developer": "bigscience", - "models": [ - { - "id": "bigscience/BLOOM-176B", - "name": "BLOOM 176B", - "developer": "bigscience", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.446, - "helm_classic/MMLU": 0.299, - "helm_classic/BoolQ": 0.704, - "helm_classic/NarrativeQA": 0.662, - "helm_classic/NaturalQuestions (open-book)": 0.621, - "helm_classic/QuAC": 0.361, - "helm_classic/HellaSwag": 0.744, - "helm_classic/OpenbookQA": 0.534, - "helm_classic/TruthfulQA": 0.205, - "helm_classic/MS MARCO (TREC)": 0.386, - "helm_classic/CNN/DailyMail": 0.08, - "helm_classic/XSUM": 0.03, - "helm_classic/IMDB": 0.945, - "helm_classic/CivilComments": 0.62, - "helm_classic/RAFT": 0.592 - } - }, - { - "id": "bigscience/bloom-1b1", - "name": "bloom-1b1", - "developer": "bigscience", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1373, - "hfopenllm_v2/BBH": 0.3107, - "hfopenllm_v2/MATH Level 5": 0.0053, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.37, - "hfopenllm_v2/MMLU-PRO": 0.1108 - } - }, - { - "id": "bigscience/bloom-1b7", - "name": "bloom-1b7", - "developer": "bigscience", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1044, - "hfopenllm_v2/BBH": 0.3141, - "hfopenllm_v2/MATH Level 5": 0.0053, - "hfopenllm_v2/GPQA": 0.2584, - "hfopenllm_v2/MUSR": 0.3886, - "hfopenllm_v2/MMLU-PRO": 0.1086 - } - }, - { - "id": "bigscience/bloom-3b", - "name": "bloom-3b", - "developer": "bigscience", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1271, - "hfopenllm_v2/BBH": 0.3063, - "hfopenllm_v2/MATH Level 5": 0.0083, - "hfopenllm_v2/GPQA": 0.2399, - "hfopenllm_v2/MUSR": 0.3981, - "hfopenllm_v2/MMLU-PRO": 0.1133 - } - }, - { - "id": "bigscience/bloom-560m", - "name": "bloom-560m", - "developer": "bigscience", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.062, - "hfopenllm_v2/BBH": 0.3026, - "hfopenllm_v2/MATH Level 5": 0.0038, - "hfopenllm_v2/GPQA": 0.2617, - "hfopenllm_v2/MUSR": 0.4031, - "hfopenllm_v2/MMLU-PRO": 0.1164 - } - }, - { - "id": "bigscience/bloom-7b1", - "name": "bloom-7b1", - "developer": "bigscience", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1322, - "hfopenllm_v2/BBH": 0.3114, - "hfopenllm_v2/MATH Level 5": 0.0053, - "hfopenllm_v2/GPQA": 0.2643, - "hfopenllm_v2/MUSR": 0.3487, - "hfopenllm_v2/MMLU-PRO": 0.1105 - } - }, - { - "id": "bigscience/T0pp-11B", - "name": "T0pp 11B", - "developer": "bigscience", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.197, - "helm_classic/MMLU": 0.407, - "helm_classic/BoolQ": 0.0, - "helm_classic/NarrativeQA": 0.151, - "helm_classic/NaturalQuestions (open-book)": 0.19, - "helm_classic/QuAC": 0.121, - "helm_classic/HellaSwag": -1.0, - "helm_classic/OpenbookQA": -1.0, - "helm_classic/TruthfulQA": 0.377, - "helm_classic/MS MARCO (TREC)": -1.0, - "helm_classic/CNN/DailyMail": 0.122, - "helm_classic/XSUM": 0.09, - "helm_classic/IMDB": 0.207, - "helm_classic/CivilComments": 0.234, - "helm_classic/RAFT": 0.118 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/bittensor.json b/data/developers/bittensor.json deleted file mode 100644 index 978db9995aa093c5d6a14e1c3bbe579dbc2bf39f..0000000000000000000000000000000000000000 --- a/data/developers/bittensor.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "developer": "bittensor", - "models": [ - { - "id": "bittensor/bitagent-bounty-8b", - "name": "BitAgent-Bounty-8B", - "developer": "bittensor", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 36.0, - "bfcl/bfcl.overall.overall_accuracy": 46.23, - "bfcl/bfcl.overall.total_cost_usd": 18.02, - "bfcl/bfcl.overall.latency_mean_s": 16.52, - "bfcl/bfcl.overall.latency_std_s": 30.73, - "bfcl/bfcl.overall.latency_p95_s": 77.12, - "bfcl/bfcl.non_live.ast_accuracy": 81.6, - "bfcl/bfcl.non_live.simple_ast_accuracy": 72.42, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 93.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 83.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 78.0, - "bfcl/bfcl.live.live_accuracy": 93.12, - "bfcl/bfcl.live.live_simple_ast_accuracy": 90.31, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 94.02, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 75.0, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 95.83, - "bfcl/bfcl.multi_turn.accuracy": 62.38, - "bfcl/bfcl.multi_turn.base_accuracy": 75.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 49.5, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 68.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 57.0, - "bfcl/bfcl.web_search.accuracy": 0.0, - "bfcl/bfcl.web_search.base_accuracy": 0.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 0.0, - "bfcl/bfcl.memory.accuracy": 1.51, - "bfcl/bfcl.memory.kv_accuracy": 1.29, - "bfcl/bfcl.memory.vector_accuracy": 1.29, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 1.94, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 68.75, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 97.48 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/blackbeenie.json b/data/developers/blackbeenie.json deleted file mode 100644 index 9211639ebdf1c8151c1ae744c6de05a42adbccc3..0000000000000000000000000000000000000000 --- a/data/developers/blackbeenie.json +++ /dev/null @@ -1,131 +0,0 @@ -{ - "developer": "BlackBeenie", - "models": [ - { - "id": "BlackBeenie/Bloslain-8B-v0.2", - "name": "Bloslain-8B-v0.2", - "developer": "BlackBeenie", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5023, - "hfopenllm_v2/BBH": 0.5111, - "hfopenllm_v2/MATH Level 5": 0.145, - "hfopenllm_v2/GPQA": 0.3062, - "hfopenllm_v2/MUSR": 0.4076, - "hfopenllm_v2/MMLU-PRO": 0.3654 - } - }, - { - "id": "BlackBeenie/llama-3-luminous-merged", - "name": "llama-3-luminous-merged", - "developer": "BlackBeenie", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4323, - "hfopenllm_v2/BBH": 0.5154, - "hfopenllm_v2/MATH Level 5": 0.0869, - "hfopenllm_v2/GPQA": 0.2928, - "hfopenllm_v2/MUSR": 0.4149, - "hfopenllm_v2/MMLU-PRO": 0.3773 - } - }, - { - "id": "BlackBeenie/llama-3.1-8B-Galore-openassistant-guanaco", - "name": "llama-3.1-8B-Galore-openassistant-guanaco", - "developer": "BlackBeenie", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2635, - "hfopenllm_v2/BBH": 0.5213, - "hfopenllm_v2/MATH Level 5": 0.0665, - "hfopenllm_v2/GPQA": 0.3003, - "hfopenllm_v2/MUSR": 0.4406, - "hfopenllm_v2/MMLU-PRO": 0.3206 - } - }, - { - "id": "BlackBeenie/Llama-3.1-8B-OpenO1-SFT-v0.1", - "name": "Llama-3.1-8B-OpenO1-SFT-v0.1", - "developer": "BlackBeenie", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5124, - "hfopenllm_v2/BBH": 0.4787, - "hfopenllm_v2/MATH Level 5": 0.1526, - "hfopenllm_v2/GPQA": 0.2685, - "hfopenllm_v2/MUSR": 0.3618, - "hfopenllm_v2/MMLU-PRO": 0.3492 - } - }, - { - "id": "BlackBeenie/Llama-3.1-8B-pythonic-passthrough-merge", - "name": "Llama-3.1-8B-pythonic-passthrough-merge", - "developer": "BlackBeenie", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2316, - "hfopenllm_v2/BBH": 0.3454, - "hfopenllm_v2/MATH Level 5": 0.0113, - "hfopenllm_v2/GPQA": 0.2685, - "hfopenllm_v2/MUSR": 0.3778, - "hfopenllm_v2/MMLU-PRO": 0.1332 - } - }, - { - "id": "BlackBeenie/Neos-Gemma-2-9b", - "name": "Neos-Gemma-2-9b", - "developer": "BlackBeenie", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5876, - "hfopenllm_v2/BBH": 0.5503, - "hfopenllm_v2/MATH Level 5": 0.0982, - "hfopenllm_v2/GPQA": 0.323, - "hfopenllm_v2/MUSR": 0.3618, - "hfopenllm_v2/MMLU-PRO": 0.3981 - } - }, - { - "id": "BlackBeenie/Neos-Llama-3.1-8B", - "name": "Neos-Llama-3.1-8B", - "developer": "BlackBeenie", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4944, - "hfopenllm_v2/BBH": 0.4425, - "hfopenllm_v2/MATH Level 5": 0.1322, - "hfopenllm_v2/GPQA": 0.2685, - "hfopenllm_v2/MUSR": 0.375, - "hfopenllm_v2/MMLU-PRO": 0.3262 - } - }, - { - "id": "BlackBeenie/Neos-Llama-3.1-base", - "name": "Neos-Llama-3.1-base", - "developer": "BlackBeenie", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1751, - "hfopenllm_v2/BBH": 0.293, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2374, - "hfopenllm_v2/MUSR": 0.3499, - "hfopenllm_v2/MMLU-PRO": 0.1112 - } - }, - { - "id": "BlackBeenie/Neos-Phi-3-14B-v0.1", - "name": "Neos-Phi-3-14B-v0.1", - "developer": "BlackBeenie", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4022, - "hfopenllm_v2/BBH": 0.6212, - "hfopenllm_v2/MATH Level 5": 0.1782, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.4125, - "hfopenllm_v2/MMLU-PRO": 0.4564 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/bllossom.json b/data/developers/bllossom.json deleted file mode 100644 index 118b7d9252f14b9680c259a3ee44ddf57c525ca4..0000000000000000000000000000000000000000 --- a/data/developers/bllossom.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "Bllossom", - "models": [ - { - "id": "Bllossom/llama-3.2-Korean-Bllossom-AICA-5B", - "name": "llama-3.2-Korean-Bllossom-AICA-5B", - "developer": "Bllossom", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5172, - "hfopenllm_v2/BBH": 0.4293, - "hfopenllm_v2/MATH Level 5": 0.1239, - "hfopenllm_v2/GPQA": 0.2987, - "hfopenllm_v2/MUSR": 0.3834, - "hfopenllm_v2/MMLU-PRO": 0.271 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/bluuwhale.json b/data/developers/bluuwhale.json deleted file mode 100644 index b1b3fddc18204487786da7b6ea3631d8c4a75e1b..0000000000000000000000000000000000000000 --- a/data/developers/bluuwhale.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "bluuwhale", - "models": [ - { - "id": "bluuwhale/L3-SthenoMaid-8B-V1", - "name": "L3-SthenoMaid-8B-V1", - "developer": "bluuwhale", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7345, - "hfopenllm_v2/BBH": 0.5219, - "hfopenllm_v2/MATH Level 5": 0.108, - "hfopenllm_v2/GPQA": 0.2802, - "hfopenllm_v2/MUSR": 0.3687, - "hfopenllm_v2/MMLU-PRO": 0.3656 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/boltmonkey.json b/data/developers/boltmonkey.json deleted file mode 100644 index 93a1093c0cbb01b9a84ae22cf6063fc50d8793cb..0000000000000000000000000000000000000000 --- a/data/developers/boltmonkey.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "developer": "BoltMonkey", - "models": [ - { - "id": "BoltMonkey/DreadMix", - "name": "DreadMix", - "developer": "BoltMonkey", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7095, - "hfopenllm_v2/BBH": 0.5435, - "hfopenllm_v2/MATH Level 5": 0.1556, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.4212, - "hfopenllm_v2/MMLU-PRO": 0.379 - } - }, - { - "id": "BoltMonkey/NeuralDaredevil-SuperNova-Lite-7B-DARETIES-abliterated", - "name": "NeuralDaredevil-SuperNova-Lite-7B-DARETIES-abliterated", - "developer": "BoltMonkey", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7999, - "hfopenllm_v2/BBH": 0.5152, - "hfopenllm_v2/MATH Level 5": 0.1193, - "hfopenllm_v2/GPQA": 0.281, - "hfopenllm_v2/MUSR": 0.4019, - "hfopenllm_v2/MMLU-PRO": 0.3733 - } - }, - { - "id": "BoltMonkey/SuperNeuralDreadDevil-8b", - "name": "SuperNeuralDreadDevil-8b", - "developer": "BoltMonkey", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.771, - "hfopenllm_v2/BBH": 0.5286, - "hfopenllm_v2/MATH Level 5": 0.0929, - "hfopenllm_v2/GPQA": 0.2919, - "hfopenllm_v2/MUSR": 0.3977, - "hfopenllm_v2/MMLU-PRO": 0.3679 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/bond005.json b/data/developers/bond005.json deleted file mode 100644 index 63cb95c6e1355d070fbb69654eba15c592a3a05b..0000000000000000000000000000000000000000 --- a/data/developers/bond005.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "bond005", - "models": [ - { - "id": "bond005/meno-tiny-0.1", - "name": "meno-tiny-0.1", - "developer": "bond005", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.455, - "hfopenllm_v2/BBH": 0.4263, - "hfopenllm_v2/MATH Level 5": 0.139, - "hfopenllm_v2/GPQA": 0.2819, - "hfopenllm_v2/MUSR": 0.4185, - "hfopenllm_v2/MMLU-PRO": 0.2786 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/bosonai.json b/data/developers/bosonai.json deleted file mode 100644 index d7ccb9b0aaf5da4648f1403012b652cc4ff6f92e..0000000000000000000000000000000000000000 --- a/data/developers/bosonai.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "bosonai", - "models": [ - { - "id": "bosonai/Higgs-Llama-3-70B", - "name": "Higgs-Llama-3-70B", - "developer": "bosonai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5561, - "hfopenllm_v2/BBH": 0.6258, - "hfopenllm_v2/MATH Level 5": 0.2523, - "hfopenllm_v2/GPQA": 0.3666, - "hfopenllm_v2/MUSR": 0.4471, - "hfopenllm_v2/MMLU-PRO": 0.4902 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/braindao.json b/data/developers/braindao.json deleted file mode 100644 index 4266783f2b4aa241839aa12aaf190e95f551f44c..0000000000000000000000000000000000000000 --- a/data/developers/braindao.json +++ /dev/null @@ -1,243 +0,0 @@ -{ - "developer": "braindao", - "models": [ - { - "id": "braindao/DeepSeek-R1-Distill-Qwen-1.5B-Blunt", - "name": "DeepSeek-R1-Distill-Qwen-1.5B-Blunt", - "developer": "braindao", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2611, - "hfopenllm_v2/BBH": 0.2774, - "hfopenllm_v2/MATH Level 5": 0.1382, - "hfopenllm_v2/GPQA": 0.2475, - "hfopenllm_v2/MUSR": 0.3595, - "hfopenllm_v2/MMLU-PRO": 0.1184 - } - }, - { - "id": "braindao/DeepSeek-R1-Distill-Qwen-1.5B-Reflective", - "name": "DeepSeek-R1-Distill-Qwen-1.5B-Reflective", - "developer": "braindao", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3033, - "hfopenllm_v2/BBH": 0.2908, - "hfopenllm_v2/MATH Level 5": 0.1631, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.3356, - "hfopenllm_v2/MMLU-PRO": 0.113 - } - }, - { - "id": "braindao/DeepSeek-R1-Distill-Qwen-14B", - "name": "DeepSeek-R1-Distill-Qwen-14B", - "developer": "braindao", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4172, - "hfopenllm_v2/BBH": 0.3033, - "hfopenllm_v2/MATH Level 5": 0.176, - "hfopenllm_v2/GPQA": 0.2802, - "hfopenllm_v2/MUSR": 0.4488, - "hfopenllm_v2/MMLU-PRO": 0.1127 - } - }, - { - "id": "braindao/DeepSeek-R1-Distill-Qwen-14B-ABUB-ST", - "name": "DeepSeek-R1-Distill-Qwen-14B-ABUB-ST", - "developer": "braindao", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3752, - "hfopenllm_v2/BBH": 0.4927, - "hfopenllm_v2/MATH Level 5": 0.5015, - "hfopenllm_v2/GPQA": 0.3448, - "hfopenllm_v2/MUSR": 0.4221, - "hfopenllm_v2/MMLU-PRO": 0.4243 - } - }, - { - "id": "braindao/DeepSeek-R1-Distill-Qwen-14B-Blunt", - "name": "DeepSeek-R1-Distill-Qwen-14B-Blunt", - "developer": "braindao", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5612, - "hfopenllm_v2/BBH": 0.3283, - "hfopenllm_v2/MATH Level 5": 0.1639, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.4554, - "hfopenllm_v2/MMLU-PRO": 0.1447 - } - }, - { - "id": "braindao/DeepSeek-R1-Distill-Qwen-14B-Blunt-Uncensored", - "name": "DeepSeek-R1-Distill-Qwen-14B-Blunt-Uncensored", - "developer": "braindao", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5422, - "hfopenllm_v2/BBH": 0.317, - "hfopenllm_v2/MATH Level 5": 0.1631, - "hfopenllm_v2/GPQA": 0.2827, - "hfopenllm_v2/MUSR": 0.4487, - "hfopenllm_v2/MMLU-PRO": 0.1431 - } - }, - { - "id": "braindao/DeepSeek-R1-Distill-Qwen-14B-Blunt-Uncensored-Blunt", - "name": "DeepSeek-R1-Distill-Qwen-14B-Blunt-Uncensored-Blunt", - "developer": "braindao", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5221, - "hfopenllm_v2/BBH": 0.3199, - "hfopenllm_v2/MATH Level 5": 0.2508, - "hfopenllm_v2/GPQA": 0.2785, - "hfopenllm_v2/MUSR": 0.4527, - "hfopenllm_v2/MMLU-PRO": 0.1484 - } - }, - { - "id": "braindao/DeepSeek-R1-Distill-Qwen-14B-Blunt-Uncensored-Blunt-Reflective", - "name": "DeepSeek-R1-Distill-Qwen-14B-Blunt-Uncensored-Blunt-Reflective", - "developer": "braindao", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.554, - "hfopenllm_v2/BBH": 0.3371, - "hfopenllm_v2/MATH Level 5": 0.2372, - "hfopenllm_v2/GPQA": 0.2777, - "hfopenllm_v2/MUSR": 0.4248, - "hfopenllm_v2/MMLU-PRO": 0.1504 - } - }, - { - "id": "braindao/DeepSeek-R1-Distill-Qwen-14B-Blunt-Uncensored-Reflective", - "name": "DeepSeek-R1-Distill-Qwen-14B-Blunt-Uncensored-Reflective", - "developer": "braindao", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5139, - "hfopenllm_v2/BBH": 0.3013, - "hfopenllm_v2/MATH Level 5": 0.1473, - "hfopenllm_v2/GPQA": 0.2878, - "hfopenllm_v2/MUSR": 0.4433, - "hfopenllm_v2/MMLU-PRO": 0.1289 - } - }, - { - "id": "braindao/DeepSeek-R1-Distill-Qwen-14B-Reflective", - "name": "DeepSeek-R1-Distill-Qwen-14B-Reflective", - "developer": "braindao", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.429, - "hfopenllm_v2/BBH": 0.3012, - "hfopenllm_v2/MATH Level 5": 0.1918, - "hfopenllm_v2/GPQA": 0.2727, - "hfopenllm_v2/MUSR": 0.4554, - "hfopenllm_v2/MMLU-PRO": 0.1129 - } - }, - { - "id": "braindao/DeepSeek-R1-Distill-Qwen-7B", - "name": "DeepSeek-R1-Distill-Qwen-7B", - "developer": "braindao", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3968, - "hfopenllm_v2/BBH": 0.2887, - "hfopenllm_v2/MATH Level 5": 0.1918, - "hfopenllm_v2/GPQA": 0.2617, - "hfopenllm_v2/MUSR": 0.3767, - "hfopenllm_v2/MMLU-PRO": 0.1141 - } - }, - { - "id": "braindao/DeepSeek-R1-Distill-Qwen-7B-Blunt", - "name": "DeepSeek-R1-Distill-Qwen-7B-Blunt", - "developer": "braindao", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4266, - "hfopenllm_v2/BBH": 0.2902, - "hfopenllm_v2/MATH Level 5": 0.2145, - "hfopenllm_v2/GPQA": 0.271, - "hfopenllm_v2/MUSR": 0.3885, - "hfopenllm_v2/MMLU-PRO": 0.1169 - } - }, - { - "id": "braindao/DeepSeek-R1-Distill-Qwen-7B-ORPO-Uncensored", - "name": "DeepSeek-R1-Distill-Qwen-7B-ORPO-Uncensored", - "developer": "braindao", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3655, - "hfopenllm_v2/BBH": 0.2958, - "hfopenllm_v2/MATH Level 5": 0.1737, - "hfopenllm_v2/GPQA": 0.2534, - "hfopenllm_v2/MUSR": 0.3846, - "hfopenllm_v2/MMLU-PRO": 0.1133 - } - }, - { - "id": "braindao/DeepSeek-R1-Distill-Qwen-7B-Reflective", - "name": "DeepSeek-R1-Distill-Qwen-7B-Reflective", - "developer": "braindao", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3922, - "hfopenllm_v2/BBH": 0.2907, - "hfopenllm_v2/MATH Level 5": 0.2024, - "hfopenllm_v2/GPQA": 0.2542, - "hfopenllm_v2/MUSR": 0.39, - "hfopenllm_v2/MMLU-PRO": 0.1155 - } - }, - { - "id": "braindao/iq-code-evmind-0.5b", - "name": "iq-code-evmind-0.5b", - "developer": "braindao", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3216, - "hfopenllm_v2/BBH": 0.3164, - "hfopenllm_v2/MATH Level 5": 0.0242, - "hfopenllm_v2/GPQA": 0.2416, - "hfopenllm_v2/MUSR": 0.3304, - "hfopenllm_v2/MMLU-PRO": 0.1189 - } - }, - { - "id": "braindao/Qwen2.5-14B", - "name": "Qwen2.5-14B", - "developer": "braindao", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5409, - "hfopenllm_v2/BBH": 0.5853, - "hfopenllm_v2/MATH Level 5": 0.2923, - "hfopenllm_v2/GPQA": 0.3733, - "hfopenllm_v2/MUSR": 0.4124, - "hfopenllm_v2/MMLU-PRO": 0.4884 - } - }, - { - "id": "braindao/Qwen2.5-14B-Instruct", - "name": "Qwen2.5-14B-Instruct", - "developer": "braindao", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8143, - "hfopenllm_v2/BBH": 0.6404, - "hfopenllm_v2/MATH Level 5": 0.5529, - "hfopenllm_v2/GPQA": 0.3289, - "hfopenllm_v2/MUSR": 0.414, - "hfopenllm_v2/MMLU-PRO": 0.4889 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/brainwave-ml.json b/data/developers/brainwave-ml.json deleted file mode 100644 index 6b4fca8d6496e23c5bb9f54ca7bf1ad8b86b317a..0000000000000000000000000000000000000000 --- a/data/developers/brainwave-ml.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "BrainWave-ML", - "models": [ - { - "id": "BrainWave-ML/llama3.2-3B-maths-orpo", - "name": "llama3.2-3B-maths-orpo", - "developer": "BrainWave-ML", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2049, - "hfopenllm_v2/BBH": 0.2912, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3575, - "hfopenllm_v2/MMLU-PRO": 0.1168 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/bramvanroy.json b/data/developers/bramvanroy.json deleted file mode 100644 index 975c408ef82651e82f25f7e44d551c2a79925ac1..0000000000000000000000000000000000000000 --- a/data/developers/bramvanroy.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "developer": "BramVanroy", - "models": [ - { - "id": "BramVanroy/fietje-2", - "name": "fietje-2", - "developer": "BramVanroy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2098, - "hfopenllm_v2/BBH": 0.4036, - "hfopenllm_v2/MATH Level 5": 0.0159, - "hfopenllm_v2/GPQA": 0.2542, - "hfopenllm_v2/MUSR": 0.3696, - "hfopenllm_v2/MMLU-PRO": 0.1986 - } - }, - { - "id": "BramVanroy/fietje-2-chat", - "name": "fietje-2-chat", - "developer": "BramVanroy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2917, - "hfopenllm_v2/BBH": 0.415, - "hfopenllm_v2/MATH Level 5": 0.0189, - "hfopenllm_v2/GPQA": 0.2399, - "hfopenllm_v2/MUSR": 0.3528, - "hfopenllm_v2/MMLU-PRO": 0.2055 - } - }, - { - "id": "BramVanroy/fietje-2-instruct", - "name": "fietje-2-instruct", - "developer": "BramVanroy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.279, - "hfopenllm_v2/BBH": 0.4136, - "hfopenllm_v2/MATH Level 5": 0.0227, - "hfopenllm_v2/GPQA": 0.2332, - "hfopenllm_v2/MUSR": 0.3369, - "hfopenllm_v2/MMLU-PRO": 0.2104 - } - }, - { - "id": "BramVanroy/GEITje-7B-ultra", - "name": "GEITje-7B-ultra", - "developer": "BramVanroy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3723, - "hfopenllm_v2/BBH": 0.3776, - "hfopenllm_v2/MATH Level 5": 0.0159, - "hfopenllm_v2/GPQA": 0.2626, - "hfopenllm_v2/MUSR": 0.329, - "hfopenllm_v2/MMLU-PRO": 0.2011 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/brgx53.json b/data/developers/brgx53.json deleted file mode 100644 index 900b8b3bb64d152e8f11cf2a4b13ce2ea0c9b2a2..0000000000000000000000000000000000000000 --- a/data/developers/brgx53.json +++ /dev/null @@ -1,89 +0,0 @@ -{ - "developer": "brgx53", - "models": [ - { - "id": "brgx53/3Bgeneral-ECE-PRYMMAL-Martial", - "name": "3Bgeneral-ECE-PRYMMAL-Martial", - "developer": "brgx53", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3289, - "hfopenllm_v2/BBH": 0.5458, - "hfopenllm_v2/MATH Level 5": 0.1314, - "hfopenllm_v2/GPQA": 0.3247, - "hfopenllm_v2/MUSR": 0.4373, - "hfopenllm_v2/MMLU-PRO": 0.3934 - } - }, - { - "id": "brgx53/3Bgeneralv2-ECE-PRYMMAL-Martial", - "name": "3Bgeneralv2-ECE-PRYMMAL-Martial", - "developer": "brgx53", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5677, - "hfopenllm_v2/BBH": 0.5607, - "hfopenllm_v2/MATH Level 5": 0.3497, - "hfopenllm_v2/GPQA": 0.3112, - "hfopenllm_v2/MUSR": 0.4356, - "hfopenllm_v2/MMLU-PRO": 0.4505 - } - }, - { - "id": "brgx53/3Blareneg-ECE-PRYMMAL-Martial", - "name": "3Blareneg-ECE-PRYMMAL-Martial", - "developer": "brgx53", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2876, - "hfopenllm_v2/BBH": 0.5358, - "hfopenllm_v2/MATH Level 5": 0.1208, - "hfopenllm_v2/GPQA": 0.3347, - "hfopenllm_v2/MUSR": 0.4429, - "hfopenllm_v2/MMLU-PRO": 0.4016 - } - }, - { - "id": "brgx53/3Blarenegv2-ECE-PRYMMAL-Martial", - "name": "3Blarenegv2-ECE-PRYMMAL-Martial", - "developer": "brgx53", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5662, - "hfopenllm_v2/BBH": 0.5607, - "hfopenllm_v2/MATH Level 5": 0.3497, - "hfopenllm_v2/GPQA": 0.3112, - "hfopenllm_v2/MUSR": 0.4356, - "hfopenllm_v2/MMLU-PRO": 0.4505 - } - }, - { - "id": "brgx53/Barracuda-PRYMMAL-ECE-TW3", - "name": "Barracuda-PRYMMAL-ECE-TW3", - "developer": "brgx53", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.164, - "hfopenllm_v2/BBH": 0.3002, - "hfopenllm_v2/MATH Level 5": 0.0023, - "hfopenllm_v2/GPQA": 0.2534, - "hfopenllm_v2/MUSR": 0.3609, - "hfopenllm_v2/MMLU-PRO": 0.1093 - } - }, - { - "id": "brgx53/LaConfiance-PRYMMAL-ECE-TW3", - "name": "LaConfiance-PRYMMAL-ECE-TW3", - "developer": "brgx53", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1579, - "hfopenllm_v2/BBH": 0.2962, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2517, - "hfopenllm_v2/MUSR": 0.3846, - "hfopenllm_v2/MMLU-PRO": 0.1146 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/bsc-lt.json b/data/developers/bsc-lt.json deleted file mode 100644 index a8b8e66d18cd4161749d5209a0996487090ec0c7..0000000000000000000000000000000000000000 --- a/data/developers/bsc-lt.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "BSC-LT", - "models": [ - { - "id": "BSC-LT/salamandra-7b", - "name": "salamandra-7b", - "developer": "BSC-LT", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1367, - "hfopenllm_v2/BBH": 0.3517, - "hfopenllm_v2/MATH Level 5": 0.0038, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.3501, - "hfopenllm_v2/MMLU-PRO": 0.1493 - } - }, - { - "id": "BSC-LT/salamandra-7b-instruct", - "name": "salamandra-7b-instruct", - "developer": "BSC-LT", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2451, - "hfopenllm_v2/BBH": 0.3851, - "hfopenllm_v2/MATH Level 5": 0.0083, - "hfopenllm_v2/GPQA": 0.2643, - "hfopenllm_v2/MUSR": 0.4134, - "hfopenllm_v2/MMLU-PRO": 0.1805 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/bunnycore.json b/data/developers/bunnycore.json deleted file mode 100644 index 069b8a9f0214975725e28f414b82b07e73072c0f..0000000000000000000000000000000000000000 --- a/data/developers/bunnycore.json +++ /dev/null @@ -1,1195 +0,0 @@ -{ - "developer": "bunnycore", - "models": [ - { - "id": "bunnycore/Best-Mix-Llama-3.1-8B", - "name": "Best-Mix-Llama-3.1-8B", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2067, - "hfopenllm_v2/BBH": 0.3432, - "hfopenllm_v2/MATH Level 5": 0.2054, - "hfopenllm_v2/GPQA": 0.2651, - "hfopenllm_v2/MUSR": 0.2929, - "hfopenllm_v2/MMLU-PRO": 0.1565 - } - }, - { - "id": "bunnycore/Blabbertron-1.0", - "name": "Blabbertron-1.0", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7433, - "hfopenllm_v2/BBH": 0.5497, - "hfopenllm_v2/MATH Level 5": 0.4924, - "hfopenllm_v2/GPQA": 0.302, - "hfopenllm_v2/MUSR": 0.4337, - "hfopenllm_v2/MMLU-PRO": 0.4354 - } - }, - { - "id": "bunnycore/Blabbertron-1.1", - "name": "Blabbertron-1.1", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7265, - "hfopenllm_v2/BBH": 0.5534, - "hfopenllm_v2/MATH Level 5": 0.4804, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.4416, - "hfopenllm_v2/MMLU-PRO": 0.4431 - } - }, - { - "id": "bunnycore/CyberCore-Qwen-2.1-7B", - "name": "CyberCore-Qwen-2.1-7B", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5766, - "hfopenllm_v2/BBH": 0.5572, - "hfopenllm_v2/MATH Level 5": 0.3588, - "hfopenllm_v2/GPQA": 0.3079, - "hfopenllm_v2/MUSR": 0.4145, - "hfopenllm_v2/MMLU-PRO": 0.4445 - } - }, - { - "id": "bunnycore/DeepQwen-3B-LCoT-SCE", - "name": "DeepQwen-3B-LCoT-SCE", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.449, - "hfopenllm_v2/BBH": 0.4512, - "hfopenllm_v2/MATH Level 5": 0.247, - "hfopenllm_v2/GPQA": 0.2626, - "hfopenllm_v2/MUSR": 0.3514, - "hfopenllm_v2/MMLU-PRO": 0.329 - } - }, - { - "id": "bunnycore/DeepSeek-R1-Distill-Qwen-7B-RRP-Ex", - "name": "DeepSeek-R1-Distill-Qwen-7B-RRP-Ex", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3901, - "hfopenllm_v2/BBH": 0.3494, - "hfopenllm_v2/MATH Level 5": 0.1654, - "hfopenllm_v2/GPQA": 0.2785, - "hfopenllm_v2/MUSR": 0.3663, - "hfopenllm_v2/MMLU-PRO": 0.2508 - } - }, - { - "id": "bunnycore/DeepThinker-7B-Sce-v1", - "name": "DeepThinker-7B-Sce-v1", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1218, - "hfopenllm_v2/BBH": 0.3018, - "hfopenllm_v2/MATH Level 5": 0.0098, - "hfopenllm_v2/GPQA": 0.2517, - "hfopenllm_v2/MUSR": 0.4194, - "hfopenllm_v2/MMLU-PRO": 0.1123 - } - }, - { - "id": "bunnycore/DeepThinker-7B-Sce-v2", - "name": "DeepThinker-7B-Sce-v2", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1631, - "hfopenllm_v2/BBH": 0.3057, - "hfopenllm_v2/MATH Level 5": 0.0113, - "hfopenllm_v2/GPQA": 0.2584, - "hfopenllm_v2/MUSR": 0.4101, - "hfopenllm_v2/MMLU-PRO": 0.1146 - } - }, - { - "id": "bunnycore/FuseCyberMix-Qwen-2.5-7B-Instruct", - "name": "FuseCyberMix-Qwen-2.5-7B-Instruct", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7019, - "hfopenllm_v2/BBH": 0.5518, - "hfopenllm_v2/MATH Level 5": 0.4841, - "hfopenllm_v2/GPQA": 0.297, - "hfopenllm_v2/MUSR": 0.402, - "hfopenllm_v2/MMLU-PRO": 0.4337 - } - }, - { - "id": "bunnycore/FuseQwQen-7B", - "name": "FuseQwQen-7B", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7275, - "hfopenllm_v2/BBH": 0.5504, - "hfopenllm_v2/MATH Level 5": 0.4366, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.4217, - "hfopenllm_v2/MMLU-PRO": 0.4407 - } - }, - { - "id": "bunnycore/FwF-Qwen-7B-0.1", - "name": "FwF-Qwen-7B-0.1", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3005, - "hfopenllm_v2/BBH": 0.5019, - "hfopenllm_v2/MATH Level 5": 0.2764, - "hfopenllm_v2/GPQA": 0.271, - "hfopenllm_v2/MUSR": 0.3952, - "hfopenllm_v2/MMLU-PRO": 0.4061 - } - }, - { - "id": "bunnycore/FwF-Qwen-7B-0.2", - "name": "FwF-Qwen-7B-0.2", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4479, - "hfopenllm_v2/BBH": 0.5596, - "hfopenllm_v2/MATH Level 5": 0.426, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.4218, - "hfopenllm_v2/MMLU-PRO": 0.4382 - } - }, - { - "id": "bunnycore/Gemma-2-2B-Smart", - "name": "Gemma-2-2B-Smart", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1321, - "hfopenllm_v2/BBH": 0.3974, - "hfopenllm_v2/MATH Level 5": 0.0332, - "hfopenllm_v2/GPQA": 0.2827, - "hfopenllm_v2/MUSR": 0.4249, - "hfopenllm_v2/MMLU-PRO": 0.2426 - } - }, - { - "id": "bunnycore/Gemma2-9B-TitanFusion", - "name": "Gemma2-9B-TitanFusion", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1618, - "hfopenllm_v2/BBH": 0.5712, - "hfopenllm_v2/MATH Level 5": 0.077, - "hfopenllm_v2/GPQA": 0.3322, - "hfopenllm_v2/MUSR": 0.4136, - "hfopenllm_v2/MMLU-PRO": 0.396 - } - }, - { - "id": "bunnycore/HyperLlama-3.1-8B", - "name": "HyperLlama-3.1-8B", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7883, - "hfopenllm_v2/BBH": 0.5103, - "hfopenllm_v2/MATH Level 5": 0.1828, - "hfopenllm_v2/GPQA": 0.2869, - "hfopenllm_v2/MUSR": 0.3829, - "hfopenllm_v2/MMLU-PRO": 0.3783 - } - }, - { - "id": "bunnycore/Llama-3.1-8B-TitanFusion-Mix", - "name": "Llama-3.1-8B-TitanFusion-Mix", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4925, - "hfopenllm_v2/BBH": 0.5756, - "hfopenllm_v2/MATH Level 5": 0.1284, - "hfopenllm_v2/GPQA": 0.2953, - "hfopenllm_v2/MUSR": 0.4317, - "hfopenllm_v2/MMLU-PRO": 0.3695 - } - }, - { - "id": "bunnycore/Llama-3.1-8B-TitanFusion-v3", - "name": "Llama-3.1-8B-TitanFusion-v3", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.481, - "hfopenllm_v2/BBH": 0.5262, - "hfopenllm_v2/MATH Level 5": 0.142, - "hfopenllm_v2/GPQA": 0.3087, - "hfopenllm_v2/MUSR": 0.4302, - "hfopenllm_v2/MMLU-PRO": 0.3806 - } - }, - { - "id": "bunnycore/Llama-3.2-3B-All-Mix", - "name": "Llama-3.2-3B-All-Mix", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7226, - "hfopenllm_v2/BBH": 0.4508, - "hfopenllm_v2/MATH Level 5": 0.1503, - "hfopenllm_v2/GPQA": 0.2626, - "hfopenllm_v2/MUSR": 0.3287, - "hfopenllm_v2/MMLU-PRO": 0.316 - } - }, - { - "id": "bunnycore/Llama-3.2-3B-Bespoke-Thought", - "name": "Llama-3.2-3B-Bespoke-Thought", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4113, - "hfopenllm_v2/BBH": 0.4522, - "hfopenllm_v2/MATH Level 5": 0.1647, - "hfopenllm_v2/GPQA": 0.2659, - "hfopenllm_v2/MUSR": 0.3302, - "hfopenllm_v2/MMLU-PRO": 0.311 - } - }, - { - "id": "bunnycore/Llama-3.2-3B-Booval", - "name": "Llama-3.2-3B-Booval", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6669, - "hfopenllm_v2/BBH": 0.4514, - "hfopenllm_v2/MATH Level 5": 0.1269, - "hfopenllm_v2/GPQA": 0.2668, - "hfopenllm_v2/MUSR": 0.3394, - "hfopenllm_v2/MMLU-PRO": 0.3058 - } - }, - { - "id": "bunnycore/Llama-3.2-3B-Deep-Test", - "name": "Llama-3.2-3B-Deep-Test", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4652, - "hfopenllm_v2/BBH": 0.4531, - "hfopenllm_v2/MATH Level 5": 0.1284, - "hfopenllm_v2/GPQA": 0.2643, - "hfopenllm_v2/MUSR": 0.3394, - "hfopenllm_v2/MMLU-PRO": 0.3152 - } - }, - { - "id": "bunnycore/Llama-3.2-3B-Della", - "name": "Llama-3.2-3B-Della", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3561, - "hfopenllm_v2/BBH": 0.3683, - "hfopenllm_v2/MATH Level 5": 0.0302, - "hfopenllm_v2/GPQA": 0.276, - "hfopenllm_v2/MUSR": 0.3902, - "hfopenllm_v2/MMLU-PRO": 0.2128 - } - }, - { - "id": "bunnycore/Llama-3.2-3B-Long-Think", - "name": "Llama-3.2-3B-Long-Think", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5473, - "hfopenllm_v2/BBH": 0.461, - "hfopenllm_v2/MATH Level 5": 0.1458, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.3396, - "hfopenllm_v2/MMLU-PRO": 0.3048 - } - }, - { - "id": "bunnycore/Llama-3.2-3B-Mix-Skill", - "name": "Llama-3.2-3B-Mix-Skill", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6404, - "hfopenllm_v2/BBH": 0.4582, - "hfopenllm_v2/MATH Level 5": 0.1473, - "hfopenllm_v2/GPQA": 0.2617, - "hfopenllm_v2/MUSR": 0.3396, - "hfopenllm_v2/MMLU-PRO": 0.3121 - } - }, - { - "id": "bunnycore/Llama-3.2-3B-ProdigyPlus", - "name": "Llama-3.2-3B-ProdigyPlus", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4015, - "hfopenllm_v2/BBH": 0.4392, - "hfopenllm_v2/MATH Level 5": 0.1156, - "hfopenllm_v2/GPQA": 0.2685, - "hfopenllm_v2/MUSR": 0.358, - "hfopenllm_v2/MMLU-PRO": 0.2817 - } - }, - { - "id": "bunnycore/Llama-3.2-3B-ProdigyPlusPlus", - "name": "Llama-3.2-3B-ProdigyPlusPlus", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1645, - "hfopenllm_v2/BBH": 0.369, - "hfopenllm_v2/MATH Level 5": 0.0453, - "hfopenllm_v2/GPQA": 0.2534, - "hfopenllm_v2/MUSR": 0.3541, - "hfopenllm_v2/MMLU-PRO": 0.15 - } - }, - { - "id": "bunnycore/Llama-3.2-3B-RP-DeepThink", - "name": "Llama-3.2-3B-RP-DeepThink", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7144, - "hfopenllm_v2/BBH": 0.4563, - "hfopenllm_v2/MATH Level 5": 0.1609, - "hfopenllm_v2/GPQA": 0.2659, - "hfopenllm_v2/MUSR": 0.3302, - "hfopenllm_v2/MMLU-PRO": 0.3242 - } - }, - { - "id": "bunnycore/Llama-3.2-3b-RP-Toxic-Fuse", - "name": "Llama-3.2-3b-RP-Toxic-Fuse", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6834, - "hfopenllm_v2/BBH": 0.465, - "hfopenllm_v2/MATH Level 5": 0.2402, - "hfopenllm_v2/GPQA": 0.2777, - "hfopenllm_v2/MUSR": 0.3954, - "hfopenllm_v2/MMLU-PRO": 0.3106 - } - }, - { - "id": "bunnycore/Llama-3.2-3B-RRStock", - "name": "Llama-3.2-3B-RRStock", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6657, - "hfopenllm_v2/BBH": 0.4568, - "hfopenllm_v2/MATH Level 5": 0.1699, - "hfopenllm_v2/GPQA": 0.2659, - "hfopenllm_v2/MUSR": 0.3314, - "hfopenllm_v2/MMLU-PRO": 0.3236 - } - }, - { - "id": "bunnycore/Llama-3.2-3B-ToxicKod", - "name": "Llama-3.2-3B-ToxicKod", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6319, - "hfopenllm_v2/BBH": 0.4525, - "hfopenllm_v2/MATH Level 5": 0.1699, - "hfopenllm_v2/GPQA": 0.2659, - "hfopenllm_v2/MUSR": 0.3475, - "hfopenllm_v2/MMLU-PRO": 0.288 - } - }, - { - "id": "bunnycore/Maestro-S1k-7B-Sce", - "name": "Maestro-S1k-7B-Sce", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2523, - "hfopenllm_v2/BBH": 0.3104, - "hfopenllm_v2/MATH Level 5": 0.0279, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.3768, - "hfopenllm_v2/MMLU-PRO": 0.117 - } - }, - { - "id": "bunnycore/Phi-3.5-mini-TitanFusion-0.1", - "name": "Phi-3.5-mini-TitanFusion-0.1", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5228, - "hfopenllm_v2/BBH": 0.5374, - "hfopenllm_v2/MATH Level 5": 0.1186, - "hfopenllm_v2/GPQA": 0.3314, - "hfopenllm_v2/MUSR": 0.4453, - "hfopenllm_v2/MMLU-PRO": 0.3807 - } - }, - { - "id": "bunnycore/Phi-4-Model-Stock", - "name": "Phi-4-Model-Stock", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6879, - "hfopenllm_v2/BBH": 0.689, - "hfopenllm_v2/MATH Level 5": 0.4298, - "hfopenllm_v2/GPQA": 0.3549, - "hfopenllm_v2/MUSR": 0.4441, - "hfopenllm_v2/MMLU-PRO": 0.5368 - } - }, - { - "id": "bunnycore/Phi-4-Model-Stock-v2", - "name": "Phi-4-Model-Stock-v2", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6375, - "hfopenllm_v2/BBH": 0.6825, - "hfopenllm_v2/MATH Level 5": 0.3754, - "hfopenllm_v2/GPQA": 0.349, - "hfopenllm_v2/MUSR": 0.4662, - "hfopenllm_v2/MMLU-PRO": 0.5331 - } - }, - { - "id": "bunnycore/Phi-4-Model-Stock-v3", - "name": "Phi-4-Model-Stock-v3", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5912, - "hfopenllm_v2/BBH": 0.6726, - "hfopenllm_v2/MATH Level 5": 0.4902, - "hfopenllm_v2/GPQA": 0.2894, - "hfopenllm_v2/MUSR": 0.4166, - "hfopenllm_v2/MMLU-PRO": 0.5381 - } - }, - { - "id": "bunnycore/Phi-4-Model-Stock-v4", - "name": "Phi-4-Model-Stock-v4", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.711, - "hfopenllm_v2/BBH": 0.6924, - "hfopenllm_v2/MATH Level 5": 0.3829, - "hfopenllm_v2/GPQA": 0.3691, - "hfopenllm_v2/MUSR": 0.4611, - "hfopenllm_v2/MMLU-PRO": 0.5394 - } - }, - { - "id": "bunnycore/Phi-4-ReasoningRP", - "name": "Phi-4-ReasoningRP", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6736, - "hfopenllm_v2/BBH": 0.6922, - "hfopenllm_v2/MATH Level 5": 0.4569, - "hfopenllm_v2/GPQA": 0.344, - "hfopenllm_v2/MUSR": 0.4491, - "hfopenllm_v2/MMLU-PRO": 0.5421 - } - }, - { - "id": "bunnycore/Phi-4-RP-v0", - "name": "Phi-4-RP-v0", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6827, - "hfopenllm_v2/BBH": 0.6856, - "hfopenllm_v2/MATH Level 5": 0.3316, - "hfopenllm_v2/GPQA": 0.3523, - "hfopenllm_v2/MUSR": 0.4141, - "hfopenllm_v2/MMLU-PRO": 0.5364 - } - }, - { - "id": "bunnycore/Phi-4-RR-Shoup", - "name": "Phi-4-RR-Shoup", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6587, - "hfopenllm_v2/BBH": 0.6947, - "hfopenllm_v2/MATH Level 5": 0.4992, - "hfopenllm_v2/GPQA": 0.3372, - "hfopenllm_v2/MUSR": 0.444, - "hfopenllm_v2/MMLU-PRO": 0.5429 - } - }, - { - "id": "bunnycore/Phi-4-RStock-v0.1", - "name": "Phi-4-RStock-v0.1", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7019, - "hfopenllm_v2/BBH": 0.6928, - "hfopenllm_v2/MATH Level 5": 0.395, - "hfopenllm_v2/GPQA": 0.3649, - "hfopenllm_v2/MUSR": 0.4584, - "hfopenllm_v2/MMLU-PRO": 0.5401 - } - }, - { - "id": "bunnycore/Phi-4-Sce-exp-v0.1", - "name": "Phi-4-Sce-exp-v0.1", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6595, - "hfopenllm_v2/BBH": 0.6943, - "hfopenllm_v2/MATH Level 5": 0.503, - "hfopenllm_v2/GPQA": 0.3356, - "hfopenllm_v2/MUSR": 0.4441, - "hfopenllm_v2/MMLU-PRO": 0.5423 - } - }, - { - "id": "bunnycore/Phi-4-Stock-Ex", - "name": "Phi-4-Stock-Ex", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6575, - "hfopenllm_v2/BBH": 0.6864, - "hfopenllm_v2/MATH Level 5": 0.4086, - "hfopenllm_v2/GPQA": 0.3507, - "hfopenllm_v2/MUSR": 0.4624, - "hfopenllm_v2/MMLU-PRO": 0.5375 - } - }, - { - "id": "bunnycore/Phi-4-Stock-RP", - "name": "Phi-4-Stock-RP", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6399, - "hfopenllm_v2/BBH": 0.686, - "hfopenllm_v2/MATH Level 5": 0.3414, - "hfopenllm_v2/GPQA": 0.3582, - "hfopenllm_v2/MUSR": 0.4715, - "hfopenllm_v2/MMLU-PRO": 0.5317 - } - }, - { - "id": "bunnycore/Phi-4-Trim-Exp1", - "name": "Phi-4-Trim-Exp1", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1219, - "hfopenllm_v2/BBH": 0.2852, - "hfopenllm_v2/MATH Level 5": 0.0053, - "hfopenllm_v2/GPQA": 0.255, - "hfopenllm_v2/MUSR": 0.4177, - "hfopenllm_v2/MMLU-PRO": 0.1147 - } - }, - { - "id": "bunnycore/Phi-Seek-4-Sce-V1", - "name": "Phi-Seek-4-Sce-V1", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2935, - "hfopenllm_v2/BBH": 0.6459, - "hfopenllm_v2/MATH Level 5": 0.2145, - "hfopenllm_v2/GPQA": 0.276, - "hfopenllm_v2/MUSR": 0.3982, - "hfopenllm_v2/MMLU-PRO": 0.5123 - } - }, - { - "id": "bunnycore/Qandora-2.5-7B-Creative", - "name": "Qandora-2.5-7B-Creative", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6803, - "hfopenllm_v2/BBH": 0.5542, - "hfopenllm_v2/MATH Level 5": 0.3059, - "hfopenllm_v2/GPQA": 0.3104, - "hfopenllm_v2/MUSR": 0.4212, - "hfopenllm_v2/MMLU-PRO": 0.448 - } - }, - { - "id": "bunnycore/QandoraExp-7B", - "name": "QandoraExp-7B", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7509, - "hfopenllm_v2/BBH": 0.5478, - "hfopenllm_v2/MATH Level 5": 0.4743, - "hfopenllm_v2/GPQA": 0.3104, - "hfopenllm_v2/MUSR": 0.4312, - "hfopenllm_v2/MMLU-PRO": 0.441 - } - }, - { - "id": "bunnycore/QandoraExp-7B-Persona", - "name": "QandoraExp-7B-Persona", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6247, - "hfopenllm_v2/BBH": 0.5558, - "hfopenllm_v2/MATH Level 5": 0.3104, - "hfopenllm_v2/GPQA": 0.3146, - "hfopenllm_v2/MUSR": 0.4372, - "hfopenllm_v2/MMLU-PRO": 0.4407 - } - }, - { - "id": "bunnycore/QandoraExp-7B-v2", - "name": "QandoraExp-7B-v2", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5607, - "hfopenllm_v2/BBH": 0.5445, - "hfopenllm_v2/MATH Level 5": 0.4713, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.4045, - "hfopenllm_v2/MMLU-PRO": 0.3909 - } - }, - { - "id": "bunnycore/Qwen-2.5-7B-Deep-Sky-T1", - "name": "Qwen-2.5-7B-Deep-Sky-T1", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4208, - "hfopenllm_v2/BBH": 0.414, - "hfopenllm_v2/MATH Level 5": 0.0551, - "hfopenllm_v2/GPQA": 0.281, - "hfopenllm_v2/MUSR": 0.4018, - "hfopenllm_v2/MMLU-PRO": 0.2104 - } - }, - { - "id": "bunnycore/Qwen-2.5-7B-Deep-Stock-v1", - "name": "Qwen-2.5-7B-Deep-Stock-v1", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5695, - "hfopenllm_v2/BBH": 0.5361, - "hfopenllm_v2/MATH Level 5": 0.2644, - "hfopenllm_v2/GPQA": 0.2777, - "hfopenllm_v2/MUSR": 0.4109, - "hfopenllm_v2/MMLU-PRO": 0.4066 - } - }, - { - "id": "bunnycore/Qwen-2.5-7B-Deep-Stock-v4", - "name": "Qwen-2.5-7B-Deep-Stock-v4", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7753, - "hfopenllm_v2/BBH": 0.5453, - "hfopenllm_v2/MATH Level 5": 0.4894, - "hfopenllm_v2/GPQA": 0.3003, - "hfopenllm_v2/MUSR": 0.4127, - "hfopenllm_v2/MMLU-PRO": 0.4342 - } - }, - { - "id": "bunnycore/Qwen-2.5-7B-Deep-Stock-v5", - "name": "Qwen-2.5-7B-Deep-Stock-v5", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4509, - "hfopenllm_v2/BBH": 0.4672, - "hfopenllm_v2/MATH Level 5": 0.1473, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.3648, - "hfopenllm_v2/MMLU-PRO": 0.2832 - } - }, - { - "id": "bunnycore/Qwen-2.5-7B-Exp-Sce", - "name": "Qwen-2.5-7B-Exp-Sce", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7652, - "hfopenllm_v2/BBH": 0.5506, - "hfopenllm_v2/MATH Level 5": 0.3255, - "hfopenllm_v2/GPQA": 0.2987, - "hfopenllm_v2/MUSR": 0.443, - "hfopenllm_v2/MMLU-PRO": 0.4259 - } - }, - { - "id": "bunnycore/Qwen-2.5-7B-R1-Stock", - "name": "Qwen-2.5-7B-R1-Stock", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7573, - "hfopenllm_v2/BBH": 0.5393, - "hfopenllm_v2/MATH Level 5": 0.5008, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.3994, - "hfopenllm_v2/MMLU-PRO": 0.4294 - } - }, - { - "id": "bunnycore/Qwen-2.5-7b-S1k", - "name": "Qwen-2.5-7b-S1k", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7162, - "hfopenllm_v2/BBH": 0.5563, - "hfopenllm_v2/MATH Level 5": 0.4781, - "hfopenllm_v2/GPQA": 0.2844, - "hfopenllm_v2/MUSR": 0.4071, - "hfopenllm_v2/MMLU-PRO": 0.4382 - } - }, - { - "id": "bunnycore/Qwen-2.5-7B-Stock-Deep-Bespoke", - "name": "Qwen-2.5-7B-Stock-Deep-Bespoke", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5206, - "hfopenllm_v2/BBH": 0.492, - "hfopenllm_v2/MATH Level 5": 0.1888, - "hfopenllm_v2/GPQA": 0.281, - "hfopenllm_v2/MUSR": 0.4068, - "hfopenllm_v2/MMLU-PRO": 0.358 - } - }, - { - "id": "bunnycore/Qwen2.5-1.5B-Model-Stock", - "name": "Qwen2.5-1.5B-Model-Stock", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1829, - "hfopenllm_v2/BBH": 0.2874, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3674, - "hfopenllm_v2/MMLU-PRO": 0.11 - } - }, - { - "id": "bunnycore/Qwen2.5-3B-Model-Stock", - "name": "Qwen2.5-3B-Model-Stock", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6381, - "hfopenllm_v2/BBH": 0.4712, - "hfopenllm_v2/MATH Level 5": 0.3799, - "hfopenllm_v2/GPQA": 0.2886, - "hfopenllm_v2/MUSR": 0.3942, - "hfopenllm_v2/MMLU-PRO": 0.325 - } - }, - { - "id": "bunnycore/Qwen2.5-3B-Model-Stock-v2", - "name": "Qwen2.5-3B-Model-Stock-v2", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.649, - "hfopenllm_v2/BBH": 0.4677, - "hfopenllm_v2/MATH Level 5": 0.3867, - "hfopenllm_v2/GPQA": 0.2869, - "hfopenllm_v2/MUSR": 0.3915, - "hfopenllm_v2/MMLU-PRO": 0.327 - } - }, - { - "id": "bunnycore/Qwen2.5-3B-Model-Stock-v3.1", - "name": "Qwen2.5-3B-Model-Stock-v3.1", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6481, - "hfopenllm_v2/BBH": 0.4737, - "hfopenllm_v2/MATH Level 5": 0.3897, - "hfopenllm_v2/GPQA": 0.2844, - "hfopenllm_v2/MUSR": 0.3968, - "hfopenllm_v2/MMLU-PRO": 0.329 - } - }, - { - "id": "bunnycore/Qwen2.5-3B-Model-Stock-v3.2", - "name": "Qwen2.5-3B-Model-Stock-v3.2", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6353, - "hfopenllm_v2/BBH": 0.4727, - "hfopenllm_v2/MATH Level 5": 0.3754, - "hfopenllm_v2/GPQA": 0.2836, - "hfopenllm_v2/MUSR": 0.3928, - "hfopenllm_v2/MMLU-PRO": 0.3294 - } - }, - { - "id": "bunnycore/Qwen2.5-3B-Model-Stock-v4.1", - "name": "Qwen2.5-3B-Model-Stock-v4.1", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6381, - "hfopenllm_v2/BBH": 0.482, - "hfopenllm_v2/MATH Level 5": 0.3769, - "hfopenllm_v2/GPQA": 0.2794, - "hfopenllm_v2/MUSR": 0.3941, - "hfopenllm_v2/MMLU-PRO": 0.3387 - } - }, - { - "id": "bunnycore/Qwen2.5-3B-RP-Mix", - "name": "Qwen2.5-3B-RP-Mix", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5721, - "hfopenllm_v2/BBH": 0.4894, - "hfopenllm_v2/MATH Level 5": 0.2153, - "hfopenllm_v2/GPQA": 0.2735, - "hfopenllm_v2/MUSR": 0.4284, - "hfopenllm_v2/MMLU-PRO": 0.3728 - } - }, - { - "id": "bunnycore/Qwen2.5-3B-RP-Thinker", - "name": "Qwen2.5-3B-RP-Thinker", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5894, - "hfopenllm_v2/BBH": 0.4164, - "hfopenllm_v2/MATH Level 5": 0.3353, - "hfopenllm_v2/GPQA": 0.2643, - "hfopenllm_v2/MUSR": 0.3287, - "hfopenllm_v2/MMLU-PRO": 0.315 - } - }, - { - "id": "bunnycore/Qwen2.5-3B-RP-Thinker-V2", - "name": "Qwen2.5-3B-RP-Thinker-V2", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.642, - "hfopenllm_v2/BBH": 0.4678, - "hfopenllm_v2/MATH Level 5": 0.3829, - "hfopenllm_v2/GPQA": 0.2852, - "hfopenllm_v2/MUSR": 0.3981, - "hfopenllm_v2/MMLU-PRO": 0.3271 - } - }, - { - "id": "bunnycore/Qwen2.5-7B-CyberRombos", - "name": "Qwen2.5-7B-CyberRombos", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7518, - "hfopenllm_v2/BBH": 0.5465, - "hfopenllm_v2/MATH Level 5": 0.4962, - "hfopenllm_v2/GPQA": 0.3045, - "hfopenllm_v2/MUSR": 0.4125, - "hfopenllm_v2/MMLU-PRO": 0.4391 - } - }, - { - "id": "bunnycore/Qwen2.5-7B-Fuse-Exp", - "name": "Qwen2.5-7B-Fuse-Exp", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5469, - "hfopenllm_v2/BBH": 0.5109, - "hfopenllm_v2/MATH Level 5": 0.3142, - "hfopenllm_v2/GPQA": 0.276, - "hfopenllm_v2/MUSR": 0.4573, - "hfopenllm_v2/MMLU-PRO": 0.3309 - } - }, - { - "id": "bunnycore/Qwen2.5-7B-Instruct-Fusion", - "name": "Qwen2.5-7B-Instruct-Fusion", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6962, - "hfopenllm_v2/BBH": 0.5492, - "hfopenllm_v2/MATH Level 5": 0.3406, - "hfopenllm_v2/GPQA": 0.3045, - "hfopenllm_v2/MUSR": 0.4297, - "hfopenllm_v2/MMLU-PRO": 0.4467 - } - }, - { - "id": "bunnycore/Qwen2.5-7B-Instruct-Merge-Stock-v0.1", - "name": "Qwen2.5-7B-Instruct-Merge-Stock-v0.1", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7509, - "hfopenllm_v2/BBH": 0.5529, - "hfopenllm_v2/MATH Level 5": 0.4894, - "hfopenllm_v2/GPQA": 0.3037, - "hfopenllm_v2/MUSR": 0.4231, - "hfopenllm_v2/MMLU-PRO": 0.4383 - } - }, - { - "id": "bunnycore/Qwen2.5-7B-MixStock-Sce-V0.3", - "name": "Qwen2.5-7B-MixStock-Sce-V0.3", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.212, - "hfopenllm_v2/BBH": 0.3479, - "hfopenllm_v2/MATH Level 5": 0.2576, - "hfopenllm_v2/GPQA": 0.2576, - "hfopenllm_v2/MUSR": 0.3714, - "hfopenllm_v2/MMLU-PRO": 0.1779 - } - }, - { - "id": "bunnycore/Qwen2.5-7B-MixStock-V0.1", - "name": "Qwen2.5-7B-MixStock-V0.1", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7673, - "hfopenllm_v2/BBH": 0.5479, - "hfopenllm_v2/MATH Level 5": 0.3172, - "hfopenllm_v2/GPQA": 0.3003, - "hfopenllm_v2/MUSR": 0.4416, - "hfopenllm_v2/MMLU-PRO": 0.4256 - } - }, - { - "id": "bunnycore/Qwen2.5-7B-R1-Bespoke-Stock", - "name": "Qwen2.5-7B-R1-Bespoke-Stock", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3726, - "hfopenllm_v2/BBH": 0.4822, - "hfopenllm_v2/MATH Level 5": 0.2047, - "hfopenllm_v2/GPQA": 0.2785, - "hfopenllm_v2/MUSR": 0.3926, - "hfopenllm_v2/MMLU-PRO": 0.3472 - } - }, - { - "id": "bunnycore/Qwen2.5-7B-R1-Bespoke-Task", - "name": "Qwen2.5-7B-R1-Bespoke-Task", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3787, - "hfopenllm_v2/BBH": 0.415, - "hfopenllm_v2/MATH Level 5": 0.1782, - "hfopenllm_v2/GPQA": 0.2534, - "hfopenllm_v2/MUSR": 0.3569, - "hfopenllm_v2/MMLU-PRO": 0.2688 - } - }, - { - "id": "bunnycore/Qwen2.5-7B-RRP-1M", - "name": "Qwen2.5-7B-RRP-1M", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7481, - "hfopenllm_v2/BBH": 0.5452, - "hfopenllm_v2/MATH Level 5": 0.3248, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.4483, - "hfopenllm_v2/MMLU-PRO": 0.4266 - } - }, - { - "id": "bunnycore/Qwen2.5-7B-RRP-1M-Thinker", - "name": "Qwen2.5-7B-RRP-1M-Thinker", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2308, - "hfopenllm_v2/BBH": 0.3482, - "hfopenllm_v2/MATH Level 5": 0.2719, - "hfopenllm_v2/GPQA": 0.2576, - "hfopenllm_v2/MUSR": 0.3767, - "hfopenllm_v2/MMLU-PRO": 0.1769 - } - }, - { - "id": "bunnycore/Qwen2.5-7B-RRP-ID", - "name": "Qwen2.5-7B-RRP-ID", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7473, - "hfopenllm_v2/BBH": 0.548, - "hfopenllm_v2/MATH Level 5": 0.4864, - "hfopenllm_v2/GPQA": 0.2827, - "hfopenllm_v2/MUSR": 0.418, - "hfopenllm_v2/MMLU-PRO": 0.4387 - } - }, - { - "id": "bunnycore/Qwen2.5-7B-Sky-R1-Mini", - "name": "Qwen2.5-7B-Sky-R1-Mini", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2305, - "hfopenllm_v2/BBH": 0.3503, - "hfopenllm_v2/MATH Level 5": 0.0295, - "hfopenllm_v2/GPQA": 0.2894, - "hfopenllm_v2/MUSR": 0.3448, - "hfopenllm_v2/MMLU-PRO": 0.1253 - } - }, - { - "id": "bunnycore/QwenMosaic-7B", - "name": "QwenMosaic-7B", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5819, - "hfopenllm_v2/BBH": 0.5564, - "hfopenllm_v2/MATH Level 5": 0.4441, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.4164, - "hfopenllm_v2/MMLU-PRO": 0.431 - } - }, - { - "id": "bunnycore/QwQen-3B-LCoT", - "name": "QwQen-3B-LCoT", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6025, - "hfopenllm_v2/BBH": 0.4899, - "hfopenllm_v2/MATH Level 5": 0.3618, - "hfopenllm_v2/GPQA": 0.2668, - "hfopenllm_v2/MUSR": 0.4178, - "hfopenllm_v2/MMLU-PRO": 0.3699 - } - }, - { - "id": "bunnycore/QwQen-3B-LCoT-R1", - "name": "QwQen-3B-LCoT-R1", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5342, - "hfopenllm_v2/BBH": 0.4799, - "hfopenllm_v2/MATH Level 5": 0.3353, - "hfopenllm_v2/GPQA": 0.2617, - "hfopenllm_v2/MUSR": 0.4138, - "hfopenllm_v2/MMLU-PRO": 0.3723 - } - }, - { - "id": "bunnycore/Smol-Llama-3.2-3B", - "name": "Smol-Llama-3.2-3B", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6679, - "hfopenllm_v2/BBH": 0.4539, - "hfopenllm_v2/MATH Level 5": 0.1382, - "hfopenllm_v2/GPQA": 0.2768, - "hfopenllm_v2/MUSR": 0.346, - "hfopenllm_v2/MMLU-PRO": 0.3228 - } - }, - { - "id": "bunnycore/SmolLM2-1.7-Persona", - "name": "SmolLM2-1.7-Persona", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5465, - "hfopenllm_v2/BBH": 0.3623, - "hfopenllm_v2/MATH Level 5": 0.0566, - "hfopenllm_v2/GPQA": 0.2634, - "hfopenllm_v2/MUSR": 0.3341, - "hfopenllm_v2/MMLU-PRO": 0.1974 - } - }, - { - "id": "bunnycore/SmolLM2-1.7B-roleplay-lora", - "name": "SmolLM2-1.7B-roleplay-lora", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5382, - "hfopenllm_v2/BBH": 0.361, - "hfopenllm_v2/MATH Level 5": 0.0529, - "hfopenllm_v2/GPQA": 0.2752, - "hfopenllm_v2/MUSR": 0.3395, - "hfopenllm_v2/MMLU-PRO": 0.1966 - } - }, - { - "id": "bunnycore/Tulu-3.1-8B-SuperNova", - "name": "Tulu-3.1-8B-SuperNova", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8194, - "hfopenllm_v2/BBH": 0.5254, - "hfopenllm_v2/MATH Level 5": 0.2462, - "hfopenllm_v2/GPQA": 0.302, - "hfopenllm_v2/MUSR": 0.3935, - "hfopenllm_v2/MMLU-PRO": 0.3814 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/byroneverson.json b/data/developers/byroneverson.json deleted file mode 100644 index 4b05f10e836206ccd27fd635e0336e8df718a806..0000000000000000000000000000000000000000 --- a/data/developers/byroneverson.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "developer": "byroneverson", - "models": [ - { - "id": "byroneverson/Mistral-Small-Instruct-2409-abliterated", - "name": "Mistral-Small-Instruct-2409-abliterated", - "developer": "byroneverson", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6971, - "hfopenllm_v2/BBH": 0.5238, - "hfopenllm_v2/MATH Level 5": 0.2477, - "hfopenllm_v2/GPQA": 0.3331, - "hfopenllm_v2/MUSR": 0.3697, - "hfopenllm_v2/MMLU-PRO": 0.3923 - } - }, - { - "id": "byroneverson/Yi-1.5-9B-Chat-16K-abliterated", - "name": "Yi-1.5-9B-Chat-16K-abliterated", - "developer": "byroneverson", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5528, - "hfopenllm_v2/BBH": 0.5282, - "hfopenllm_v2/MATH Level 5": 0.1412, - "hfopenllm_v2/GPQA": 0.3129, - "hfopenllm_v2/MUSR": 0.4734, - "hfopenllm_v2/MMLU-PRO": 0.3823 - } - }, - { - "id": "byroneverson/Yi-1.5-9B-Chat-abliterated", - "name": "Yi-1.5-9B-Chat-abliterated", - "developer": "byroneverson", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5723, - "hfopenllm_v2/BBH": 0.5401, - "hfopenllm_v2/MATH Level 5": 0.1662, - "hfopenllm_v2/GPQA": 0.2919, - "hfopenllm_v2/MUSR": 0.4389, - "hfopenllm_v2/MMLU-PRO": 0.3715 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/bytedance.json b/data/developers/bytedance.json deleted file mode 100644 index 9907018f50ee485ee6fbe6cf68ed3b72be8f6578..0000000000000000000000000000000000000000 --- a/data/developers/bytedance.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "developer": "ByteDance", - "models": [ - { - "id": "bytedance/doubao-seed-1-6-thinking-250615", - "name": "doubao-seed-1-6-thinking-250615", - "developer": "ByteDance", - "evaluator_relationship": null, - "benchmark_scores": { - "livecodebenchpro/Hard Problems": 0.0, - "livecodebenchpro/Medium Problems": 0.07042253521126761, - "livecodebenchpro/Easy Problems": 0.5774647887323944 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/c10x.json b/data/developers/c10x.json deleted file mode 100644 index 941dbe461d8a2d4202e51e65b8319d2f76b3fa0a..0000000000000000000000000000000000000000 --- a/data/developers/c10x.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "c10x", - "models": [ - { - "id": "c10x/longthinker", - "name": "longthinker", - "developer": "c10x", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3609, - "hfopenllm_v2/BBH": 0.4927, - "hfopenllm_v2/MATH Level 5": 0.2319, - "hfopenllm_v2/GPQA": 0.2643, - "hfopenllm_v2/MUSR": 0.391, - "hfopenllm_v2/MMLU-PRO": 0.3527 - } - }, - { - "id": "c10x/Q-Pluse", - "name": "Q-Pluse", - "developer": "c10x", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1123, - "hfopenllm_v2/BBH": 0.2875, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2466, - "hfopenllm_v2/MUSR": 0.3938, - "hfopenllm_v2/MMLU-PRO": 0.1135 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/carrotai.json b/data/developers/carrotai.json deleted file mode 100644 index 560ba820d0e2b653e59b98dd60b1707b9630a4fb..0000000000000000000000000000000000000000 --- a/data/developers/carrotai.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "CarrotAI", - "models": [ - { - "id": "CarrotAI/Llama-3.2-Rabbit-Ko-3B-Instruct", - "name": "Llama-3.2-Rabbit-Ko-3B-Instruct", - "developer": "CarrotAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7199, - "hfopenllm_v2/BBH": 0.4427, - "hfopenllm_v2/MATH Level 5": 0.2054, - "hfopenllm_v2/GPQA": 0.271, - "hfopenllm_v2/MUSR": 0.3649, - "hfopenllm_v2/MMLU-PRO": 0.2822 - } - }, - { - "id": "CarrotAI/Llama-3.2-Rabbit-Ko-3B-Instruct-2412", - "name": "Llama-3.2-Rabbit-Ko-3B-Instruct-2412", - "developer": "CarrotAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4782, - "hfopenllm_v2/BBH": 0.4358, - "hfopenllm_v2/MATH Level 5": 0.176, - "hfopenllm_v2/GPQA": 0.2928, - "hfopenllm_v2/MUSR": 0.3872, - "hfopenllm_v2/MMLU-PRO": 0.3134 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/carsenk.json b/data/developers/carsenk.json deleted file mode 100644 index e237086068941314940bb10d97f3842615a22d3d..0000000000000000000000000000000000000000 --- a/data/developers/carsenk.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "carsenk", - "models": [ - { - "id": "carsenk/flippa-v6", - "name": "flippa-v6", - "developer": "carsenk", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3439, - "hfopenllm_v2/BBH": 0.5047, - "hfopenllm_v2/MATH Level 5": 0.1405, - "hfopenllm_v2/GPQA": 0.2928, - "hfopenllm_v2/MUSR": 0.4089, - "hfopenllm_v2/MMLU-PRO": 0.3668 - } - }, - { - "id": "carsenk/phi3.5_mini_exp_825_uncensored", - "name": "phi3.5_mini_exp_825_uncensored", - "developer": "carsenk", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1364, - "hfopenllm_v2/BBH": 0.2965, - "hfopenllm_v2/MATH Level 5": 0.0106, - "hfopenllm_v2/GPQA": 0.2492, - "hfopenllm_v2/MUSR": 0.3644, - "hfopenllm_v2/MMLU-PRO": 0.1175 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/casual-autopsy.json b/data/developers/casual-autopsy.json deleted file mode 100644 index f44a0f3b5844f249f20428333dd1fa8ffda3387f..0000000000000000000000000000000000000000 --- a/data/developers/casual-autopsy.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "Casual-Autopsy", - "models": [ - { - "id": "Casual-Autopsy/L3-Umbral-Mind-RP-v2.0-8B", - "name": "L3-Umbral-Mind-RP-v2.0-8B", - "developer": "Casual-Autopsy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7123, - "hfopenllm_v2/BBH": 0.5262, - "hfopenllm_v2/MATH Level 5": 0.1095, - "hfopenllm_v2/GPQA": 0.2869, - "hfopenllm_v2/MUSR": 0.3687, - "hfopenllm_v2/MMLU-PRO": 0.3723 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/cat-searcher.json b/data/developers/cat-searcher.json deleted file mode 100644 index 1f3a47c36f373a2cecaea2380dd4d878e56fa04b..0000000000000000000000000000000000000000 --- a/data/developers/cat-searcher.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "cat-searcher", - "models": [ - { - "id": "cat-searcher/gemma-2-9b-it-sppo-iter-1", - "name": "gemma-2-9b-it-sppo-iter-1", - "developer": "cat-searcher", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3015, - "hfopenllm_v2/BBH": 0.5972, - "hfopenllm_v2/MATH Level 5": 0.0831, - "hfopenllm_v2/GPQA": 0.3448, - "hfopenllm_v2/MUSR": 0.3927, - "hfopenllm_v2/MMLU-PRO": 0.3854 - } - }, - { - "id": "cat-searcher/gemma-2-9b-it-sppo-iter-1-evol-1", - "name": "gemma-2-9b-it-sppo-iter-1-evol-1", - "developer": "cat-searcher", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2942, - "hfopenllm_v2/BBH": 0.5939, - "hfopenllm_v2/MATH Level 5": 0.0853, - "hfopenllm_v2/GPQA": 0.3406, - "hfopenllm_v2/MUSR": 0.3926, - "hfopenllm_v2/MMLU-PRO": 0.38 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/causallm.json b/data/developers/causallm.json deleted file mode 100644 index 102164bdeda9074354691687d74c13970d79628b..0000000000000000000000000000000000000000 --- a/data/developers/causallm.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "developer": "CausalLM", - "models": [ - { - "id": "CausalLM/14B", - "name": "14B", - "developer": "CausalLM", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2788, - "hfopenllm_v2/BBH": 0.47, - "hfopenllm_v2/MATH Level 5": 0.0755, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.4155, - "hfopenllm_v2/MMLU-PRO": 0.3221 - } - }, - { - "id": "CausalLM/34b-beta", - "name": "34b-beta", - "developer": "CausalLM", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3043, - "hfopenllm_v2/BBH": 0.5591, - "hfopenllm_v2/MATH Level 5": 0.0483, - "hfopenllm_v2/GPQA": 0.3465, - "hfopenllm_v2/MUSR": 0.3749, - "hfopenllm_v2/MMLU-PRO": 0.5325 - } - }, - { - "id": "CausalLM/preview-1-hf", - "name": "preview-1-hf", - "developer": "CausalLM", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5559, - "hfopenllm_v2/BBH": 0.3615, - "hfopenllm_v2/MATH Level 5": 0.0302, - "hfopenllm_v2/GPQA": 0.2617, - "hfopenllm_v2/MUSR": 0.3422, - "hfopenllm_v2/MMLU-PRO": 0.3597 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/cckm.json b/data/developers/cckm.json deleted file mode 100644 index 430066aff4bd7a50bac4899a6fc850677db7bcd4..0000000000000000000000000000000000000000 --- a/data/developers/cckm.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "cckm", - "models": [ - { - "id": "cckm/tinymistral_950m", - "name": "tinymistral_950m", - "developer": "cckm", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2395, - "hfopenllm_v2/BBH": 0.2969, - "hfopenllm_v2/MATH Level 5": 0.0053, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3554, - "hfopenllm_v2/MMLU-PRO": 0.1096 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/cgato.json b/data/developers/cgato.json deleted file mode 100644 index d6b97e4a6c18cc9b58fe1a44e92090c5d1ddaf6e..0000000000000000000000000000000000000000 --- a/data/developers/cgato.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "cgato", - "models": [ - { - "id": "cgato/TheSalt-L3-8b-v0.3.2", - "name": "TheSalt-L3-8b-v0.3.2", - "developer": "cgato", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2705, - "hfopenllm_v2/BBH": 0.2968, - "hfopenllm_v2/MATH Level 5": 0.0476, - "hfopenllm_v2/GPQA": 0.2659, - "hfopenllm_v2/MUSR": 0.3896, - "hfopenllm_v2/MMLU-PRO": 0.1139 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/changgil.json b/data/developers/changgil.json deleted file mode 100644 index c46c5aecbefbdde79b96e9e4f80e05ee09ea8fc9..0000000000000000000000000000000000000000 --- a/data/developers/changgil.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "Changgil", - "models": [ - { - "id": "Changgil/K2S3-14b-v0.2", - "name": "K2S3-14b-v0.2", - "developer": "Changgil", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3243, - "hfopenllm_v2/BBH": 0.4613, - "hfopenllm_v2/MATH Level 5": 0.0574, - "hfopenllm_v2/GPQA": 0.281, - "hfopenllm_v2/MUSR": 0.3923, - "hfopenllm_v2/MMLU-PRO": 0.2644 - } - }, - { - "id": "Changgil/K2S3-v0.1", - "name": "K2S3-v0.1", - "developer": "Changgil", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3277, - "hfopenllm_v2/BBH": 0.4655, - "hfopenllm_v2/MATH Level 5": 0.0461, - "hfopenllm_v2/GPQA": 0.2643, - "hfopenllm_v2/MUSR": 0.4014, - "hfopenllm_v2/MMLU-PRO": 0.2562 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/chargoddard.json b/data/developers/chargoddard.json deleted file mode 100644 index 9a102b01665825b5e9fb3ab649b4e74980fd10fe..0000000000000000000000000000000000000000 --- a/data/developers/chargoddard.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "chargoddard", - "models": [ - { - "id": "chargoddard/prometheus-2-llama-3-8b", - "name": "prometheus-2-llama-3-8b", - "developer": "chargoddard", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5289, - "hfopenllm_v2/BBH": 0.4931, - "hfopenllm_v2/MATH Level 5": 0.0823, - "hfopenllm_v2/GPQA": 0.2727, - "hfopenllm_v2/MUSR": 0.3396, - "hfopenllm_v2/MMLU-PRO": 0.3087 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/chujiezheng.json b/data/developers/chujiezheng.json deleted file mode 100644 index 2412580a2c434d737cc73ba56d1639f7493622c3..0000000000000000000000000000000000000000 --- a/data/developers/chujiezheng.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "chujiezheng", - "models": [ - { - "id": "chujiezheng/Llama-3-Instruct-8B-SimPO-ExPO", - "name": "Llama-3-Instruct-8B-SimPO-ExPO", - "developer": "chujiezheng", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6434, - "hfopenllm_v2/BBH": 0.4765, - "hfopenllm_v2/MATH Level 5": 0.0702, - "hfopenllm_v2/GPQA": 0.2869, - "hfopenllm_v2/MUSR": 0.392, - "hfopenllm_v2/MMLU-PRO": 0.3401 - } - }, - { - "id": "chujiezheng/Mistral7B-PairRM-SPPO-ExPO", - "name": "Mistral7B-PairRM-SPPO-ExPO", - "developer": "chujiezheng", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3673, - "hfopenllm_v2/BBH": 0.3882, - "hfopenllm_v2/MATH Level 5": 0.0181, - "hfopenllm_v2/GPQA": 0.2768, - "hfopenllm_v2/MUSR": 0.4055, - "hfopenllm_v2/MMLU-PRO": 0.2552 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/cir-ams.json b/data/developers/cir-ams.json deleted file mode 100644 index df9dcecb6f8fae5d901c2496b58813341797427e..0000000000000000000000000000000000000000 --- a/data/developers/cir-ams.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "developer": "CIR-AMS", - "models": [ - { - "id": "CIR-AMS/BTRM_Qwen2_7b_0613", - "name": "CIR-AMS/BTRM_Qwen2_7b_0613", - "developer": "CIR-AMS", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.8172, - "reward-bench/Factuality": 0.5347, - "reward-bench/Precise IF": 0.3563, - "reward-bench/Math": 0.6066, - "reward-bench/Safety": 0.9014, - "reward-bench/Focus": 0.5737, - "reward-bench/Ties": 0.6527, - "reward-bench/Chat": 0.9749, - "reward-bench/Chat Hard": 0.5724, - "reward-bench/Reasoning": 0.8775, - "reward-bench/Prior Sets (0.5 weight)": 0.7029 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/cjvt.json b/data/developers/cjvt.json deleted file mode 100644 index 5c34b1e5dcdc6d70822aaae14d19f1a36f506730..0000000000000000000000000000000000000000 --- a/data/developers/cjvt.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "cjvt", - "models": [ - { - "id": "cjvt/GaMS-1B", - "name": "GaMS-1B", - "developer": "cjvt", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1635, - "hfopenllm_v2/BBH": 0.3075, - "hfopenllm_v2/MATH Level 5": 0.0136, - "hfopenllm_v2/GPQA": 0.2584, - "hfopenllm_v2/MUSR": 0.3684, - "hfopenllm_v2/MMLU-PRO": 0.1149 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/claudioitaly.json b/data/developers/claudioitaly.json deleted file mode 100644 index 93d950b8e7f8145269ad3b3395032c17afda5c5d..0000000000000000000000000000000000000000 --- a/data/developers/claudioitaly.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "developer": "ClaudioItaly", - "models": [ - { - "id": "ClaudioItaly/Albacus", - "name": "Albacus", - "developer": "ClaudioItaly", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4667, - "hfopenllm_v2/BBH": 0.5113, - "hfopenllm_v2/MATH Level 5": 0.071, - "hfopenllm_v2/GPQA": 0.2718, - "hfopenllm_v2/MUSR": 0.4135, - "hfopenllm_v2/MMLU-PRO": 0.3165 - } - }, - { - "id": "ClaudioItaly/Book-Gut12B", - "name": "Book-Gut12B", - "developer": "ClaudioItaly", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3998, - "hfopenllm_v2/BBH": 0.5417, - "hfopenllm_v2/MATH Level 5": 0.102, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.4635, - "hfopenllm_v2/MMLU-PRO": 0.367 - } - }, - { - "id": "ClaudioItaly/Evolutionstory-7B-v2.2", - "name": "Evolutionstory-7B-v2.2", - "developer": "ClaudioItaly", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4814, - "hfopenllm_v2/BBH": 0.5108, - "hfopenllm_v2/MATH Level 5": 0.071, - "hfopenllm_v2/GPQA": 0.2752, - "hfopenllm_v2/MUSR": 0.4135, - "hfopenllm_v2/MMLU-PRO": 0.3159 - } - }, - { - "id": "ClaudioItaly/intelligence-cod-rag-7b-v3", - "name": "intelligence-cod-rag-7b-v3", - "developer": "ClaudioItaly", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6898, - "hfopenllm_v2/BBH": 0.5366, - "hfopenllm_v2/MATH Level 5": 0.3807, - "hfopenllm_v2/GPQA": 0.2727, - "hfopenllm_v2/MUSR": 0.4153, - "hfopenllm_v2/MMLU-PRO": 0.4195 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/cloudyu.json b/data/developers/cloudyu.json deleted file mode 100644 index 68e7805d1e3407eeb7307ca844c0c82c3c4c6d1b..0000000000000000000000000000000000000000 --- a/data/developers/cloudyu.json +++ /dev/null @@ -1,103 +0,0 @@ -{ - "developer": "cloudyu", - "models": [ - { - "id": "cloudyu/Llama-3-70Bx2-MOE", - "name": "Llama-3-70Bx2-MOE", - "developer": "cloudyu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5482, - "hfopenllm_v2/BBH": 0.6636, - "hfopenllm_v2/MATH Level 5": 0.2175, - "hfopenllm_v2/GPQA": 0.3935, - "hfopenllm_v2/MUSR": 0.4812, - "hfopenllm_v2/MMLU-PRO": 0.5142 - } - }, - { - "id": "cloudyu/Llama-3.2-3Bx4", - "name": "Llama-3.2-3Bx4", - "developer": "cloudyu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5069, - "hfopenllm_v2/BBH": 0.4332, - "hfopenllm_v2/MATH Level 5": 0.1073, - "hfopenllm_v2/GPQA": 0.2777, - "hfopenllm_v2/MUSR": 0.3496, - "hfopenllm_v2/MMLU-PRO": 0.2985 - } - }, - { - "id": "cloudyu/Mixtral_11Bx2_MoE_19B", - "name": "Mixtral_11Bx2_MoE_19B", - "developer": "cloudyu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3851, - "hfopenllm_v2/BBH": 0.5209, - "hfopenllm_v2/MATH Level 5": 0.0672, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.4297, - "hfopenllm_v2/MMLU-PRO": 0.3311 - } - }, - { - "id": "cloudyu/Mixtral_34Bx2_MoE_60B", - "name": "Mixtral_34Bx2_MoE_60B", - "developer": "cloudyu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4538, - "hfopenllm_v2/BBH": 0.587, - "hfopenllm_v2/MATH Level 5": 0.077, - "hfopenllm_v2/GPQA": 0.3381, - "hfopenllm_v2/MUSR": 0.4625, - "hfopenllm_v2/MMLU-PRO": 0.4766 - } - }, - { - "id": "cloudyu/Mixtral_7Bx2_MoE", - "name": "Mixtral_7Bx2_MoE", - "developer": "cloudyu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.448, - "hfopenllm_v2/BBH": 0.516, - "hfopenllm_v2/MATH Level 5": 0.0687, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.4473, - "hfopenllm_v2/MMLU-PRO": 0.3044 - } - }, - { - "id": "cloudyu/S1-Llama-3.2-3Bx4-MoE", - "name": "S1-Llama-3.2-3Bx4-MoE", - "developer": "cloudyu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5302, - "hfopenllm_v2/BBH": 0.4358, - "hfopenllm_v2/MATH Level 5": 0.1201, - "hfopenllm_v2/GPQA": 0.2936, - "hfopenllm_v2/MUSR": 0.3456, - "hfopenllm_v2/MMLU-PRO": 0.3044 - } - }, - { - "id": "cloudyu/Yi-34Bx2-MoE-60B-DPO", - "name": "Yi-34Bx2-MoE-60B-DPO", - "developer": "cloudyu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5319, - "hfopenllm_v2/BBH": 0.5168, - "hfopenllm_v2/MATH Level 5": 0.0702, - "hfopenllm_v2/GPQA": 0.3221, - "hfopenllm_v2/MUSR": 0.4375, - "hfopenllm_v2/MMLU-PRO": 0.4677 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/cluebbers.json b/data/developers/cluebbers.json deleted file mode 100644 index 6f44b35eb07aa37d3d9f261199745296092d2cf1..0000000000000000000000000000000000000000 --- a/data/developers/cluebbers.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "developer": "cluebbers", - "models": [ - { - "id": "cluebbers/Llama-3.1-8B-paraphrase-type-generation-apty-ipo", - "name": "Llama-3.1-8B-paraphrase-type-generation-apty-ipo", - "developer": "cluebbers", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1327, - "hfopenllm_v2/BBH": 0.38, - "hfopenllm_v2/MATH Level 5": 0.0249, - "hfopenllm_v2/GPQA": 0.2634, - "hfopenllm_v2/MUSR": 0.4332, - "hfopenllm_v2/MMLU-PRO": 0.2591 - } - }, - { - "id": "cluebbers/Llama-3.1-8B-paraphrase-type-generation-apty-sigmoid", - "name": "Llama-3.1-8B-paraphrase-type-generation-apty-sigmoid", - "developer": "cluebbers", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1318, - "hfopenllm_v2/BBH": 0.3789, - "hfopenllm_v2/MATH Level 5": 0.0264, - "hfopenllm_v2/GPQA": 0.2685, - "hfopenllm_v2/MUSR": 0.4306, - "hfopenllm_v2/MMLU-PRO": 0.2562 - } - }, - { - "id": "cluebbers/Llama-3.1-8B-paraphrase-type-generation-etpc", - "name": "Llama-3.1-8B-paraphrase-type-generation-etpc", - "developer": "cluebbers", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1209, - "hfopenllm_v2/BBH": 0.3781, - "hfopenllm_v2/MATH Level 5": 0.0196, - "hfopenllm_v2/GPQA": 0.2651, - "hfopenllm_v2/MUSR": 0.4319, - "hfopenllm_v2/MMLU-PRO": 0.2556 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/cognitivecomputations.json b/data/developers/cognitivecomputations.json deleted file mode 100644 index 27ef3ede420acfeab83ed5bb754062e32374c41a..0000000000000000000000000000000000000000 --- a/data/developers/cognitivecomputations.json +++ /dev/null @@ -1,243 +0,0 @@ -{ - "developer": "cognitivecomputations", - "models": [ - { - "id": "cognitivecomputations/dolphin-2.9-llama3-8b", - "name": "dolphin-2.9-llama3-8b", - "developer": "cognitivecomputations", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.385, - "hfopenllm_v2/BBH": 0.495, - "hfopenllm_v2/MATH Level 5": 0.0574, - "hfopenllm_v2/GPQA": 0.2869, - "hfopenllm_v2/MUSR": 0.4375, - "hfopenllm_v2/MMLU-PRO": 0.2771 - } - }, - { - "id": "cognitivecomputations/dolphin-2.9.1-llama-3-70b", - "name": "dolphin-2.9.1-llama-3-70b", - "developer": "cognitivecomputations", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.376, - "hfopenllm_v2/BBH": 0.5205, - "hfopenllm_v2/MATH Level 5": 0.182, - "hfopenllm_v2/GPQA": 0.3087, - "hfopenllm_v2/MUSR": 0.4976, - "hfopenllm_v2/MMLU-PRO": 0.413 - } - }, - { - "id": "cognitivecomputations/dolphin-2.9.1-yi-1.5-34b", - "name": "dolphin-2.9.1-yi-1.5-34b", - "developer": "cognitivecomputations", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3853, - "hfopenllm_v2/BBH": 0.6076, - "hfopenllm_v2/MATH Level 5": 0.1866, - "hfopenllm_v2/GPQA": 0.3431, - "hfopenllm_v2/MUSR": 0.4598, - "hfopenllm_v2/MMLU-PRO": 0.4519 - } - }, - { - "id": "cognitivecomputations/dolphin-2.9.1-yi-1.5-9b", - "name": "dolphin-2.9.1-yi-1.5-9b", - "developer": "cognitivecomputations", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4465, - "hfopenllm_v2/BBH": 0.5484, - "hfopenllm_v2/MATH Level 5": 0.1518, - "hfopenllm_v2/GPQA": 0.3381, - "hfopenllm_v2/MUSR": 0.4348, - "hfopenllm_v2/MMLU-PRO": 0.3967 - } - }, - { - "id": "cognitivecomputations/dolphin-2.9.2-Phi-3-Medium", - "name": "dolphin-2.9.2-Phi-3-Medium", - "developer": "cognitivecomputations", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4248, - "hfopenllm_v2/BBH": 0.6457, - "hfopenllm_v2/MATH Level 5": 0.1828, - "hfopenllm_v2/GPQA": 0.3272, - "hfopenllm_v2/MUSR": 0.4191, - "hfopenllm_v2/MMLU-PRO": 0.4555 - } - }, - { - "id": "cognitivecomputations/dolphin-2.9.2-Phi-3-Medium-abliterated", - "name": "dolphin-2.9.2-Phi-3-Medium-abliterated", - "developer": "cognitivecomputations", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4124, - "hfopenllm_v2/BBH": 0.6383, - "hfopenllm_v2/MATH Level 5": 0.182, - "hfopenllm_v2/GPQA": 0.3289, - "hfopenllm_v2/MUSR": 0.4349, - "hfopenllm_v2/MMLU-PRO": 0.4525 - } - }, - { - "id": "cognitivecomputations/dolphin-2.9.2-qwen2-72b", - "name": "dolphin-2.9.2-qwen2-72b", - "developer": "cognitivecomputations", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6344, - "hfopenllm_v2/BBH": 0.6296, - "hfopenllm_v2/MATH Level 5": 0.2802, - "hfopenllm_v2/GPQA": 0.37, - "hfopenllm_v2/MUSR": 0.4521, - "hfopenllm_v2/MMLU-PRO": 0.5471 - } - }, - { - "id": "cognitivecomputations/dolphin-2.9.2-qwen2-7b", - "name": "dolphin-2.9.2-qwen2-7b", - "developer": "cognitivecomputations", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3535, - "hfopenllm_v2/BBH": 0.4894, - "hfopenllm_v2/MATH Level 5": 0.1344, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.4191, - "hfopenllm_v2/MMLU-PRO": 0.4051 - } - }, - { - "id": "cognitivecomputations/dolphin-2.9.3-mistral-7B-32k", - "name": "dolphin-2.9.3-mistral-7B-32k", - "developer": "cognitivecomputations", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4126, - "hfopenllm_v2/BBH": 0.4813, - "hfopenllm_v2/MATH Level 5": 0.0506, - "hfopenllm_v2/GPQA": 0.2852, - "hfopenllm_v2/MUSR": 0.4643, - "hfopenllm_v2/MMLU-PRO": 0.2821 - } - }, - { - "id": "cognitivecomputations/dolphin-2.9.3-mistral-nemo-12b", - "name": "dolphin-2.9.3-mistral-nemo-12b", - "developer": "cognitivecomputations", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5601, - "hfopenllm_v2/BBH": 0.548, - "hfopenllm_v2/MATH Level 5": 0.074, - "hfopenllm_v2/GPQA": 0.3154, - "hfopenllm_v2/MUSR": 0.443, - "hfopenllm_v2/MMLU-PRO": 0.3377 - } - }, - { - "id": "cognitivecomputations/dolphin-2.9.3-Yi-1.5-34B-32k", - "name": "dolphin-2.9.3-Yi-1.5-34B-32k", - "developer": "cognitivecomputations", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3639, - "hfopenllm_v2/BBH": 0.6047, - "hfopenllm_v2/MATH Level 5": 0.1669, - "hfopenllm_v2/GPQA": 0.3431, - "hfopenllm_v2/MUSR": 0.4311, - "hfopenllm_v2/MMLU-PRO": 0.463 - } - }, - { - "id": "cognitivecomputations/dolphin-2.9.4-gemma2-2b", - "name": "dolphin-2.9.4-gemma2-2b", - "developer": "cognitivecomputations", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0896, - "hfopenllm_v2/BBH": 0.4081, - "hfopenllm_v2/MATH Level 5": 0.0491, - "hfopenllm_v2/GPQA": 0.2844, - "hfopenllm_v2/MUSR": 0.418, - "hfopenllm_v2/MMLU-PRO": 0.2105 - } - }, - { - "id": "cognitivecomputations/dolphin-2.9.4-llama3.1-8b", - "name": "dolphin-2.9.4-llama3.1-8b", - "developer": "cognitivecomputations", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2757, - "hfopenllm_v2/BBH": 0.3524, - "hfopenllm_v2/MATH Level 5": 0.0121, - "hfopenllm_v2/GPQA": 0.2634, - "hfopenllm_v2/MUSR": 0.3236, - "hfopenllm_v2/MMLU-PRO": 0.1237 - } - }, - { - "id": "cognitivecomputations/Dolphin3.0-Llama3.1-8B", - "name": "Dolphin3.0-Llama3.1-8B", - "developer": "cognitivecomputations", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7621, - "hfopenllm_v2/BBH": 0.4916, - "hfopenllm_v2/MATH Level 5": 0.1231, - "hfopenllm_v2/GPQA": 0.2827, - "hfopenllm_v2/MUSR": 0.3653, - "hfopenllm_v2/MMLU-PRO": 0.2992 - } - }, - { - "id": "cognitivecomputations/Dolphin3.0-Llama3.2-1B", - "name": "Dolphin3.0-Llama3.2-1B", - "developer": "cognitivecomputations", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5428, - "hfopenllm_v2/BBH": 0.3122, - "hfopenllm_v2/MATH Level 5": 0.0279, - "hfopenllm_v2/GPQA": 0.2299, - "hfopenllm_v2/MUSR": 0.3249, - "hfopenllm_v2/MMLU-PRO": 0.1375 - } - }, - { - "id": "cognitivecomputations/Dolphin3.0-Qwen2.5-0.5B", - "name": "Dolphin3.0-Qwen2.5-0.5B", - "developer": "cognitivecomputations", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4697, - "hfopenllm_v2/BBH": 0.3114, - "hfopenllm_v2/MATH Level 5": 0.0514, - "hfopenllm_v2/GPQA": 0.2349, - "hfopenllm_v2/MUSR": 0.3555, - "hfopenllm_v2/MMLU-PRO": 0.1413 - } - }, - { - "id": "cognitivecomputations/Dolphin3.0-R1-Mistral-24B", - "name": "Dolphin3.0-R1-Mistral-24B", - "developer": "cognitivecomputations", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4068, - "hfopenllm_v2/BBH": 0.536, - "hfopenllm_v2/MATH Level 5": 0.3119, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.3952, - "hfopenllm_v2/MMLU-PRO": 0.3005 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/cohere.json b/data/developers/cohere.json deleted file mode 100644 index e62de34f83cd73d87fb5e1be635eaf8f12b8c20b..0000000000000000000000000000000000000000 --- a/data/developers/cohere.json +++ /dev/null @@ -1,516 +0,0 @@ -{ - "developer": "cohere", - "models": [ - { - "id": "cohere/aya-expanse-32b", - "name": "aya-expanse-32b", - "developer": "cohere", - "evaluator_relationship": null, - "benchmark_scores": { - "global-mmlu-lite/Global MMLU Lite": 0.7353, - "global-mmlu-lite/Culturally Sensitive": 0.6891, - "global-mmlu-lite/Culturally Agnostic": 0.7815, - "global-mmlu-lite/Arabic": 0.7425, - "global-mmlu-lite/English": 0.7544, - "global-mmlu-lite/Bengali": 0.7343, - "global-mmlu-lite/German": 0.7425, - "global-mmlu-lite/French": 0.7325, - "global-mmlu-lite/Hindi": 0.7375, - "global-mmlu-lite/Indonesian": 0.7594, - "global-mmlu-lite/Italian": 0.7305, - "global-mmlu-lite/Japanese": 0.7419, - "global-mmlu-lite/Korean": 0.7525, - "global-mmlu-lite/Portuguese": 0.7544, - "global-mmlu-lite/Spanish": 0.7362, - "global-mmlu-lite/Swahili": 0.7071, - "global-mmlu-lite/Yoruba": 0.6942, - "global-mmlu-lite/Chinese": 0.743, - "global-mmlu-lite/Burmese": 0.7025 - } - }, - { - "id": "cohere/Cohere-Command-beta-52.4B", - "name": "Cohere Command beta 52.4B", - "developer": "cohere", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.874, - "helm_classic/MMLU": 0.452, - "helm_classic/BoolQ": 0.856, - "helm_classic/NarrativeQA": 0.752, - "helm_classic/NaturalQuestions (open-book)": 0.76, - "helm_classic/QuAC": 0.432, - "helm_classic/HellaSwag": 0.811, - "helm_classic/OpenbookQA": 0.582, - "helm_classic/TruthfulQA": 0.269, - "helm_classic/MS MARCO (TREC)": 0.762, - "helm_classic/CNN/DailyMail": 0.161, - "helm_classic/XSUM": 0.152, - "helm_classic/IMDB": 0.96, - "helm_classic/CivilComments": 0.601, - "helm_classic/RAFT": 0.667 - } - }, - { - "id": "cohere/Cohere-Command-beta-6.1B", - "name": "Cohere Command beta 6.1B", - "developer": "cohere", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.675, - "helm_classic/MMLU": 0.406, - "helm_classic/BoolQ": 0.798, - "helm_classic/NarrativeQA": 0.709, - "helm_classic/NaturalQuestions (open-book)": 0.717, - "helm_classic/QuAC": 0.375, - "helm_classic/HellaSwag": 0.752, - "helm_classic/OpenbookQA": 0.55, - "helm_classic/TruthfulQA": 0.203, - "helm_classic/MS MARCO (TREC)": 0.709, - "helm_classic/CNN/DailyMail": 0.153, - "helm_classic/XSUM": 0.122, - "helm_classic/IMDB": 0.961, - "helm_classic/CivilComments": 0.54, - "helm_classic/RAFT": 0.634 - } - }, - { - "id": "cohere/Cohere-large-v20220720-13.1B", - "name": "Cohere large v20220720 13.1B", - "developer": "cohere", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.372, - "helm_classic/MMLU": 0.324, - "helm_classic/BoolQ": 0.725, - "helm_classic/NarrativeQA": 0.625, - "helm_classic/NaturalQuestions (open-book)": 0.573, - "helm_classic/QuAC": 0.338, - "helm_classic/HellaSwag": 0.736, - "helm_classic/OpenbookQA": 0.542, - "helm_classic/TruthfulQA": 0.181, - "helm_classic/MS MARCO (TREC)": 0.33, - "helm_classic/CNN/DailyMail": 0.126, - "helm_classic/XSUM": 0.108, - "helm_classic/IMDB": 0.933, - "helm_classic/CivilComments": 0.507, - "helm_classic/RAFT": 0.596 - } - }, - { - "id": "cohere/Cohere-medium-v20220720-6.1B", - "name": "Cohere medium v20220720 6.1B", - "developer": "cohere", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.23, - "helm_classic/MMLU": 0.279, - "helm_classic/BoolQ": 0.659, - "helm_classic/NarrativeQA": 0.559, - "helm_classic/NaturalQuestions (open-book)": 0.504, - "helm_classic/QuAC": 0.279, - "helm_classic/HellaSwag": 0.706, - "helm_classic/OpenbookQA": 0.496, - "helm_classic/TruthfulQA": 0.19, - "helm_classic/MS MARCO (TREC)": 0.374, - "helm_classic/CNN/DailyMail": 0.077, - "helm_classic/XSUM": 0.087, - "helm_classic/IMDB": 0.935, - "helm_classic/CivilComments": 0.504, - "helm_classic/RAFT": 0.52 - } - }, - { - "id": "cohere/Cohere-medium-v20221108-6.1B", - "name": "Cohere medium v20221108 6.1B", - "developer": "cohere", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.312, - "helm_classic/MMLU": 0.254, - "helm_classic/BoolQ": 0.7, - "helm_classic/NarrativeQA": 0.61, - "helm_classic/NaturalQuestions (open-book)": 0.517, - "helm_classic/QuAC": 0.314, - "helm_classic/HellaSwag": 0.726, - "helm_classic/OpenbookQA": 0.538, - "helm_classic/TruthfulQA": 0.215, - "helm_classic/MS MARCO (TREC)": 0.373, - "helm_classic/CNN/DailyMail": 0.121, - "helm_classic/XSUM": 0.099, - "helm_classic/IMDB": 0.935, - "helm_classic/CivilComments": 0.5, - "helm_classic/RAFT": 0.591 - } - }, - { - "id": "cohere/Cohere-small-v20220720-410M", - "name": "Cohere small v20220720 410M", - "developer": "cohere", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.109, - "helm_classic/MMLU": 0.264, - "helm_classic/BoolQ": 0.457, - "helm_classic/NarrativeQA": 0.294, - "helm_classic/NaturalQuestions (open-book)": 0.309, - "helm_classic/QuAC": 0.219, - "helm_classic/HellaSwag": 0.483, - "helm_classic/OpenbookQA": 0.348, - "helm_classic/TruthfulQA": 0.217, - "helm_classic/MS MARCO (TREC)": 0.304, - "helm_classic/CNN/DailyMail": 0.063, - "helm_classic/XSUM": 0.033, - "helm_classic/IMDB": 0.578, - "helm_classic/CivilComments": 0.501, - "helm_classic/RAFT": 0.492 - } - }, - { - "id": "cohere/Cohere-xlarge-v20220609-52.4B", - "name": "Cohere xlarge v20220609 52.4B", - "developer": "cohere", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.56, - "helm_classic/MMLU": 0.353, - "helm_classic/BoolQ": 0.718, - "helm_classic/NarrativeQA": 0.65, - "helm_classic/NaturalQuestions (open-book)": 0.595, - "helm_classic/QuAC": 0.361, - "helm_classic/HellaSwag": 0.811, - "helm_classic/OpenbookQA": 0.55, - "helm_classic/TruthfulQA": 0.198, - "helm_classic/MS MARCO (TREC)": 0.459, - "helm_classic/CNN/DailyMail": 0.144, - "helm_classic/XSUM": 0.129, - "helm_classic/IMDB": 0.956, - "helm_classic/CivilComments": 0.532, - "helm_classic/RAFT": 0.633 - } - }, - { - "id": "cohere/Cohere-xlarge-v20221108-52.4B", - "name": "Cohere xlarge v20221108 52.4B", - "developer": "cohere", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.664, - "helm_classic/MMLU": 0.382, - "helm_classic/BoolQ": 0.762, - "helm_classic/NarrativeQA": 0.672, - "helm_classic/NaturalQuestions (open-book)": 0.628, - "helm_classic/QuAC": 0.374, - "helm_classic/HellaSwag": 0.81, - "helm_classic/OpenbookQA": 0.588, - "helm_classic/TruthfulQA": 0.169, - "helm_classic/MS MARCO (TREC)": 0.55, - "helm_classic/CNN/DailyMail": 0.153, - "helm_classic/XSUM": 0.153, - "helm_classic/IMDB": 0.956, - "helm_classic/CivilComments": 0.524, - "helm_classic/RAFT": 0.624 - } - }, - { - "id": "cohere/command", - "name": "Command", - "developer": "cohere", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.327, - "helm_lite/NarrativeQA": 0.749, - "helm_lite/NaturalQuestions (closed-book)": 0.391, - "helm_lite/OpenbookQA": 0.774, - "helm_lite/MMLU": 0.525, - "helm_lite/MATH": 0.236, - "helm_lite/GSM8K": 0.452, - "helm_lite/LegalBench": 0.578, - "helm_lite/MedQA": 0.445, - "helm_lite/WMT 2014": 0.088 - } - }, - { - "id": "cohere/command-a-03-2025", - "name": "command-a-03-2025", - "developer": "cohere", - "evaluator_relationship": null, - "benchmark_scores": { - "global-mmlu-lite/Global MMLU Lite": 0.8385, - "global-mmlu-lite/Culturally Sensitive": 0.7993, - "global-mmlu-lite/Culturally Agnostic": 0.8778, - "global-mmlu-lite/Arabic": 0.8425, - "global-mmlu-lite/English": 0.855, - "global-mmlu-lite/Bengali": 0.8225, - "global-mmlu-lite/German": 0.8425, - "global-mmlu-lite/French": 0.8375, - "global-mmlu-lite/Hindi": 0.8421, - "global-mmlu-lite/Indonesian": 0.8546, - "global-mmlu-lite/Italian": 0.8375, - "global-mmlu-lite/Japanese": 0.845, - "global-mmlu-lite/Korean": 0.85, - "global-mmlu-lite/Portuguese": 0.84, - "global-mmlu-lite/Spanish": 0.8525, - "global-mmlu-lite/Swahili": 0.8275, - "global-mmlu-lite/Yoruba": 0.815, - "global-mmlu-lite/Chinese": 0.835, - "global-mmlu-lite/Burmese": 0.8175 - } - }, - { - "id": "cohere/command-a-fc", - "name": "Command A (FC)", - "developer": "cohere", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 35.0, - "bfcl/bfcl.overall.overall_accuracy": 46.49, - "bfcl/bfcl.overall.total_cost_usd": 91.37, - "bfcl/bfcl.overall.latency_mean_s": 2.09, - "bfcl/bfcl.overall.latency_std_s": 7.36, - "bfcl/bfcl.overall.latency_p95_s": 4.94, - "bfcl/bfcl.non_live.ast_accuracy": 87.56, - "bfcl/bfcl.non_live.simple_ast_accuracy": 75.75, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 93.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 93.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 88.0, - "bfcl/bfcl.live.live_accuracy": 78.53, - "bfcl/bfcl.live.live_simple_ast_accuracy": 85.66, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 76.92, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 81.25, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 70.83, - "bfcl/bfcl.multi_turn.accuracy": 29.5, - "bfcl/bfcl.multi_turn.base_accuracy": 38.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 23.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 32.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 25.0, - "bfcl/bfcl.web_search.accuracy": 46.5, - "bfcl/bfcl.web_search.base_accuracy": 60.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 33.0, - "bfcl/bfcl.memory.accuracy": 16.56, - "bfcl/bfcl.memory.kv_accuracy": 4.52, - "bfcl/bfcl.memory.vector_accuracy": 5.16, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 40.0, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 81.25, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 84.19 - } - }, - { - "id": "cohere/command-a-reasoning-fc", - "name": "Command A Reasoning (FC)", - "developer": "cohere", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 13.0, - "bfcl/bfcl.overall.overall_accuracy": 57.06, - "bfcl/bfcl.overall.total_cost_usd": 3.04, - "bfcl/bfcl.overall.latency_mean_s": 3.44, - "bfcl/bfcl.overall.latency_std_s": 4.91, - "bfcl/bfcl.overall.latency_p95_s": 8.39, - "bfcl/bfcl.non_live.ast_accuracy": 86.27, - "bfcl/bfcl.non_live.simple_ast_accuracy": 73.58, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 93.5, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 89.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 88.5, - "bfcl/bfcl.live.live_accuracy": 78.61, - "bfcl/bfcl.live.live_simple_ast_accuracy": 80.23, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 78.35, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 75.0, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 75.0, - "bfcl/bfcl.multi_turn.accuracy": 50.12, - "bfcl/bfcl.multi_turn.base_accuracy": 61.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 41.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 49.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 48.5, - "bfcl/bfcl.web_search.accuracy": 55.5, - "bfcl/bfcl.web_search.base_accuracy": 65.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 46.0, - "bfcl/bfcl.memory.accuracy": 28.82, - "bfcl/bfcl.memory.kv_accuracy": 16.13, - "bfcl/bfcl.memory.vector_accuracy": 23.87, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 46.45, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 68.75, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 86.75 - } - }, - { - "id": "cohere/command-light", - "name": "Command Light", - "developer": "cohere", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.105, - "helm_lite/NarrativeQA": 0.629, - "helm_lite/NaturalQuestions (closed-book)": 0.195, - "helm_lite/OpenbookQA": 0.398, - "helm_lite/MMLU": 0.386, - "helm_lite/MATH": 0.098, - "helm_lite/GSM8K": 0.149, - "helm_lite/LegalBench": 0.397, - "helm_lite/MedQA": 0.312, - "helm_lite/WMT 2014": 0.023 - } - }, - { - "id": "cohere/command-r", - "name": "Command R", - "developer": "cohere", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.299, - "helm_lite/NarrativeQA": 0.742, - "helm_lite/NaturalQuestions (closed-book)": 0.352, - "helm_lite/OpenbookQA": 0.782, - "helm_lite/MMLU": 0.567, - "helm_lite/MATH": 0.266, - "helm_lite/GSM8K": 0.551, - "helm_lite/LegalBench": 0.507, - "helm_lite/MedQA": 0.555, - "helm_lite/WMT 2014": 0.149, - "helm_mmlu/MMLU All Subjects": 0.652, - "helm_mmlu/Abstract Algebra": 0.33, - "helm_mmlu/Anatomy": 0.615, - "helm_mmlu/College Physics": 0.382, - "helm_mmlu/Computer Security": 0.78, - "helm_mmlu/Econometrics": 0.456, - "helm_mmlu/Global Facts": 0.42, - "helm_mmlu/Jurisprudence": 0.796, - "helm_mmlu/Philosophy": 0.685, - "helm_mmlu/Professional Psychology": 0.681, - "helm_mmlu/Us Foreign Policy": 0.82, - "helm_mmlu/Astronomy": 0.743, - "helm_mmlu/Business Ethics": 0.63, - "helm_mmlu/Clinical Knowledge": 0.751, - "helm_mmlu/Conceptual Physics": 0.528, - "helm_mmlu/Electrical Engineering": 0.593, - "helm_mmlu/Elementary Mathematics": 0.437, - "helm_mmlu/Formal Logic": 0.405, - "helm_mmlu/High School World History": 0.84, - "helm_mmlu/Human Sexuality": 0.763, - "helm_mmlu/International Law": 0.802, - "helm_mmlu/Logical Fallacies": 0.798, - "helm_mmlu/Machine Learning": 0.446, - "helm_mmlu/Management": 0.796, - "helm_mmlu/Marketing": 0.872, - "helm_mmlu/Medical Genetics": 0.81, - "helm_mmlu/Miscellaneous": 0.848, - "helm_mmlu/Moral Scenarios": 0.451, - "helm_mmlu/Nutrition": 0.703, - "helm_mmlu/Prehistory": 0.728, - "helm_mmlu/Public Relations": 0.7, - "helm_mmlu/Security Studies": 0.714, - "helm_mmlu/Sociology": 0.866, - "helm_mmlu/Virology": 0.542, - "helm_mmlu/World Religions": 0.813, - "helm_mmlu/Mean win rate": 0.959 - } - }, - { - "id": "cohere/command-r-plus", - "name": "Command R Plus", - "developer": "cohere", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.441, - "helm_lite/NarrativeQA": 0.735, - "helm_lite/NaturalQuestions (closed-book)": 0.343, - "helm_lite/OpenbookQA": 0.828, - "helm_lite/MMLU": 0.59, - "helm_lite/MATH": 0.403, - "helm_lite/GSM8K": 0.738, - "helm_lite/LegalBench": 0.672, - "helm_lite/MedQA": 0.567, - "helm_lite/WMT 2014": 0.203, - "helm_mmlu/MMLU All Subjects": 0.694, - "helm_mmlu/Abstract Algebra": 0.21, - "helm_mmlu/Anatomy": 0.644, - "helm_mmlu/College Physics": 0.52, - "helm_mmlu/Computer Security": 0.74, - "helm_mmlu/Econometrics": 0.561, - "helm_mmlu/Global Facts": 0.5, - "helm_mmlu/Jurisprudence": 0.806, - "helm_mmlu/Philosophy": 0.695, - "helm_mmlu/Professional Psychology": 0.735, - "helm_mmlu/Us Foreign Policy": 0.89, - "helm_mmlu/Astronomy": 0.783, - "helm_mmlu/Business Ethics": 0.77, - "helm_mmlu/Clinical Knowledge": 0.743, - "helm_mmlu/Conceptual Physics": 0.591, - "helm_mmlu/Electrical Engineering": 0.71, - "helm_mmlu/Elementary Mathematics": 0.474, - "helm_mmlu/Formal Logic": 0.484, - "helm_mmlu/High School World History": 0.827, - "helm_mmlu/Human Sexuality": 0.786, - "helm_mmlu/International Law": 0.835, - "helm_mmlu/Logical Fallacies": 0.791, - "helm_mmlu/Machine Learning": 0.518, - "helm_mmlu/Management": 0.835, - "helm_mmlu/Marketing": 0.927, - "helm_mmlu/Medical Genetics": 0.77, - "helm_mmlu/Miscellaneous": 0.844, - "helm_mmlu/Moral Scenarios": 0.585, - "helm_mmlu/Nutrition": 0.742, - "helm_mmlu/Prehistory": 0.821, - "helm_mmlu/Public Relations": 0.709, - "helm_mmlu/Security Studies": 0.751, - "helm_mmlu/Sociology": 0.876, - "helm_mmlu/Virology": 0.56, - "helm_mmlu/World Religions": 0.842, - "helm_mmlu/Mean win rate": 0.825 - } - }, - { - "id": "cohere/command-r7b-fc", - "name": "Command R7B (FC)", - "developer": "cohere", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 61.0, - "bfcl/bfcl.overall.overall_accuracy": 32.07, - "bfcl/bfcl.overall.total_cost_usd": 1.5, - "bfcl/bfcl.overall.latency_mean_s": 1.38, - "bfcl/bfcl.overall.latency_std_s": 2.87, - "bfcl/bfcl.overall.latency_p95_s": 2.69, - "bfcl/bfcl.non_live.ast_accuracy": 80.96, - "bfcl/bfcl.non_live.simple_ast_accuracy": 67.33, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 89.5, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 85.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 81.5, - "bfcl/bfcl.live.live_accuracy": 69.06, - "bfcl/bfcl.live.live_simple_ast_accuracy": 62.79, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 70.94, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 43.75, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 70.83, - "bfcl/bfcl.multi_turn.accuracy": 8.25, - "bfcl/bfcl.multi_turn.base_accuracy": 12.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 0.5, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 10.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 10.0, - "bfcl/bfcl.web_search.accuracy": 27.0, - "bfcl/bfcl.web_search.base_accuracy": 43.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 11.0, - "bfcl/bfcl.memory.accuracy": 5.16, - "bfcl/bfcl.memory.kv_accuracy": 2.58, - "bfcl/bfcl.memory.vector_accuracy": 9.68, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 3.23, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 68.75, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 81.65 - } - }, - { - "id": "cohere/command-xlarge-beta", - "name": "Cohere Command beta 52.4B", - "developer": "cohere", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_instruct/Mean win rate": 0.089, - "helm_instruct/Anthropic RLHF dataset": 4.214, - "helm_instruct/Best ChatGPT Prompts": 4.988, - "helm_instruct/Koala test dataset": 4.969, - "helm_instruct/Open Assistant": 4.967, - "helm_instruct/Self Instruct": 4.971, - "helm_instruct/Vicuna": 4.995 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/cohereforai.json b/data/developers/cohereforai.json deleted file mode 100644 index 2b5ed1c9c0604bc8909ea77caf0c5a2e392f2636..0000000000000000000000000000000000000000 --- a/data/developers/cohereforai.json +++ /dev/null @@ -1,123 +0,0 @@ -{ - "developer": "CohereForAI", - "models": [ - { - "id": "CohereForAI/aya-23-35B", - "name": "aya-23-35B", - "developer": "CohereForAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6462, - "hfopenllm_v2/BBH": 0.54, - "hfopenllm_v2/MATH Level 5": 0.0347, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.431, - "hfopenllm_v2/MMLU-PRO": 0.3356 - } - }, - { - "id": "CohereForAI/aya-23-8B", - "name": "aya-23-8B", - "developer": "CohereForAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4699, - "hfopenllm_v2/BBH": 0.4296, - "hfopenllm_v2/MATH Level 5": 0.0166, - "hfopenllm_v2/GPQA": 0.2844, - "hfopenllm_v2/MUSR": 0.3941, - "hfopenllm_v2/MMLU-PRO": 0.2278 - } - }, - { - "id": "CohereForAI/aya-expanse-32b", - "name": "aya-expanse-32b", - "developer": "CohereForAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7302, - "hfopenllm_v2/BBH": 0.5649, - "hfopenllm_v2/MATH Level 5": 0.1533, - "hfopenllm_v2/GPQA": 0.3255, - "hfopenllm_v2/MUSR": 0.3873, - "hfopenllm_v2/MMLU-PRO": 0.413 - } - }, - { - "id": "CohereForAI/aya-expanse-8b", - "name": "aya-expanse-8b", - "developer": "CohereForAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6359, - "hfopenllm_v2/BBH": 0.4977, - "hfopenllm_v2/MATH Level 5": 0.0861, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.3729, - "hfopenllm_v2/MMLU-PRO": 0.3004 - } - }, - { - "id": "CohereForAI/c4ai-command-r-plus", - "name": "CohereForAI/c4ai-command-r-plus", - "developer": "CohereForAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7664, - "hfopenllm_v2/BBH": 0.5815, - "hfopenllm_v2/MATH Level 5": 0.0801, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.4807, - "hfopenllm_v2/MMLU-PRO": 0.3992, - "reward-bench/Score": 0.7057, - "reward-bench/Chat": 0.9511, - "reward-bench/Chat Hard": 0.5757, - "reward-bench/Safety": 0.5986, - "reward-bench/Reasoning": 0.704, - "reward-bench/Prior Sets (0.5 weight)": 0.6924 - } - }, - { - "id": "CohereForAI/c4ai-command-r-plus-08-2024", - "name": "c4ai-command-r-plus-08-2024", - "developer": "CohereForAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.754, - "hfopenllm_v2/BBH": 0.5996, - "hfopenllm_v2/MATH Level 5": 0.1239, - "hfopenllm_v2/GPQA": 0.3507, - "hfopenllm_v2/MUSR": 0.4829, - "hfopenllm_v2/MMLU-PRO": 0.4421 - } - }, - { - "id": "CohereForAI/c4ai-command-r-v01", - "name": "c4ai-command-r-v01", - "developer": "CohereForAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6748, - "hfopenllm_v2/BBH": 0.5406, - "hfopenllm_v2/MATH Level 5": 0.0347, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.4517, - "hfopenllm_v2/MMLU-PRO": 0.3369 - } - }, - { - "id": "CohereForAI/c4ai-command-r7b-12-2024", - "name": "c4ai-command-r7b-12-2024", - "developer": "CohereForAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7713, - "hfopenllm_v2/BBH": 0.5503, - "hfopenllm_v2/MATH Level 5": 0.2991, - "hfopenllm_v2/GPQA": 0.3087, - "hfopenllm_v2/MUSR": 0.4125, - "hfopenllm_v2/MMLU-PRO": 0.3572 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/collaiborateorg.json b/data/developers/collaiborateorg.json deleted file mode 100644 index 8d07059b317dd152a933e0ac5d8e5dfa6bb61c35..0000000000000000000000000000000000000000 --- a/data/developers/collaiborateorg.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "collaiborateorg", - "models": [ - { - "id": "collaiborateorg/Collaiborator-MEDLLM-Llama-3-8B-v2", - "name": "Collaiborator-MEDLLM-Llama-3-8B-v2", - "developer": "collaiborateorg", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3809, - "hfopenllm_v2/BBH": 0.4648, - "hfopenllm_v2/MATH Level 5": 0.0566, - "hfopenllm_v2/GPQA": 0.3331, - "hfopenllm_v2/MUSR": 0.3434, - "hfopenllm_v2/MMLU-PRO": 0.3481 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/columbia-nlp.json b/data/developers/columbia-nlp.json deleted file mode 100644 index b04d1f97bca6939c03a53d86cd396214edf82f72..0000000000000000000000000000000000000000 --- a/data/developers/columbia-nlp.json +++ /dev/null @@ -1,89 +0,0 @@ -{ - "developer": "Columbia-NLP", - "models": [ - { - "id": "Columbia-NLP/LION-Gemma-2b-dpo-v1.0", - "name": "LION-Gemma-2b-dpo-v1.0", - "developer": "Columbia-NLP", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3278, - "hfopenllm_v2/BBH": 0.392, - "hfopenllm_v2/MATH Level 5": 0.0431, - "hfopenllm_v2/GPQA": 0.2492, - "hfopenllm_v2/MUSR": 0.412, - "hfopenllm_v2/MMLU-PRO": 0.1666 - } - }, - { - "id": "Columbia-NLP/LION-Gemma-2b-odpo-v1.0", - "name": "LION-Gemma-2b-odpo-v1.0", - "developer": "Columbia-NLP", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3066, - "hfopenllm_v2/BBH": 0.3896, - "hfopenllm_v2/MATH Level 5": 0.0695, - "hfopenllm_v2/GPQA": 0.2424, - "hfopenllm_v2/MUSR": 0.4279, - "hfopenllm_v2/MMLU-PRO": 0.1692 - } - }, - { - "id": "Columbia-NLP/LION-Gemma-2b-sft-v1.0", - "name": "LION-Gemma-2b-sft-v1.0", - "developer": "Columbia-NLP", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3692, - "hfopenllm_v2/BBH": 0.3879, - "hfopenllm_v2/MATH Level 5": 0.068, - "hfopenllm_v2/GPQA": 0.2559, - "hfopenllm_v2/MUSR": 0.4027, - "hfopenllm_v2/MMLU-PRO": 0.1782 - } - }, - { - "id": "Columbia-NLP/LION-LLaMA-3-8b-dpo-v1.0", - "name": "LION-LLaMA-3-8b-dpo-v1.0", - "developer": "Columbia-NLP", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4957, - "hfopenllm_v2/BBH": 0.5028, - "hfopenllm_v2/MATH Level 5": 0.1171, - "hfopenllm_v2/GPQA": 0.281, - "hfopenllm_v2/MUSR": 0.4097, - "hfopenllm_v2/MMLU-PRO": 0.3219 - } - }, - { - "id": "Columbia-NLP/LION-LLaMA-3-8b-odpo-v1.0", - "name": "LION-LLaMA-3-8b-odpo-v1.0", - "developer": "Columbia-NLP", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3968, - "hfopenllm_v2/BBH": 0.5024, - "hfopenllm_v2/MATH Level 5": 0.1065, - "hfopenllm_v2/GPQA": 0.2852, - "hfopenllm_v2/MUSR": 0.4057, - "hfopenllm_v2/MMLU-PRO": 0.3152 - } - }, - { - "id": "Columbia-NLP/LION-LLaMA-3-8b-sft-v1.0", - "name": "LION-LLaMA-3-8b-sft-v1.0", - "developer": "Columbia-NLP", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3817, - "hfopenllm_v2/BBH": 0.5088, - "hfopenllm_v2/MATH Level 5": 0.114, - "hfopenllm_v2/GPQA": 0.2777, - "hfopenllm_v2/MUSR": 0.4503, - "hfopenllm_v2/MMLU-PRO": 0.3237 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/combinhorizon.json b/data/developers/combinhorizon.json deleted file mode 100644 index 5e09755a19cd8ebd0ac762e9a2f66d51023049ad..0000000000000000000000000000000000000000 --- a/data/developers/combinhorizon.json +++ /dev/null @@ -1,89 +0,0 @@ -{ - "developer": "CombinHorizon", - "models": [ - { - "id": "CombinHorizon/huihui-ai-abliterated-Qwen2.5-32B-Inst-BaseMerge-TIES", - "name": "huihui-ai-abliterated-Qwen2.5-32B-Inst-BaseMerge-TIES", - "developer": "CombinHorizon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8206, - "hfopenllm_v2/BBH": 0.6929, - "hfopenllm_v2/MATH Level 5": 0.5944, - "hfopenllm_v2/GPQA": 0.3389, - "hfopenllm_v2/MUSR": 0.4207, - "hfopenllm_v2/MMLU-PRO": 0.5721 - } - }, - { - "id": "CombinHorizon/huihui-ai-abliteratedV2-Qwen2.5-14B-Inst-BaseMerge-TIES", - "name": "huihui-ai-abliteratedV2-Qwen2.5-14B-Inst-BaseMerge-TIES", - "developer": "CombinHorizon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8176, - "hfopenllm_v2/BBH": 0.6336, - "hfopenllm_v2/MATH Level 5": 0.5476, - "hfopenllm_v2/GPQA": 0.3146, - "hfopenllm_v2/MUSR": 0.426, - "hfopenllm_v2/MMLU-PRO": 0.491 - } - }, - { - "id": "CombinHorizon/Josiefied-abliteratedV4-Qwen2.5-14B-Inst-BaseMerge-TIES", - "name": "Josiefied-abliteratedV4-Qwen2.5-14B-Inst-BaseMerge-TIES", - "developer": "CombinHorizon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.824, - "hfopenllm_v2/BBH": 0.637, - "hfopenllm_v2/MATH Level 5": 0.5317, - "hfopenllm_v2/GPQA": 0.3247, - "hfopenllm_v2/MUSR": 0.426, - "hfopenllm_v2/MMLU-PRO": 0.4979 - } - }, - { - "id": "CombinHorizon/Rombos-Qwen2.5-7B-Inst-BaseMerge-TIES", - "name": "Rombos-Qwen2.5-7B-Inst-BaseMerge-TIES", - "developer": "CombinHorizon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7564, - "hfopenllm_v2/BBH": 0.5402, - "hfopenllm_v2/MATH Level 5": 0.4932, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.4033, - "hfopenllm_v2/MMLU-PRO": 0.4342 - } - }, - { - "id": "CombinHorizon/YiSM-blossom5.1-34B-SLERP", - "name": "YiSM-blossom5.1-34B-SLERP", - "developer": "CombinHorizon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5033, - "hfopenllm_v2/BBH": 0.6208, - "hfopenllm_v2/MATH Level 5": 0.2153, - "hfopenllm_v2/GPQA": 0.3557, - "hfopenllm_v2/MUSR": 0.4413, - "hfopenllm_v2/MMLU-PRO": 0.4741 - } - }, - { - "id": "CombinHorizon/zetasepic-abliteratedV2-Qwen2.5-32B-Inst-BaseMerge-TIES", - "name": "zetasepic-abliteratedV2-Qwen2.5-32B-Inst-BaseMerge-TIES", - "developer": "CombinHorizon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8328, - "hfopenllm_v2/BBH": 0.6955, - "hfopenllm_v2/MATH Level 5": 0.5853, - "hfopenllm_v2/GPQA": 0.3674, - "hfopenllm_v2/MUSR": 0.4314, - "hfopenllm_v2/MMLU-PRO": 0.5685 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/contactdoctor.json b/data/developers/contactdoctor.json deleted file mode 100644 index dcb1d2e97c42931745bbb22aed62c3d49a409491..0000000000000000000000000000000000000000 --- a/data/developers/contactdoctor.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "ContactDoctor", - "models": [ - { - "id": "ContactDoctor/Bio-Medical-3B-CoT-012025", - "name": "Bio-Medical-3B-CoT-012025", - "developer": "ContactDoctor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3604, - "hfopenllm_v2/BBH": 0.4383, - "hfopenllm_v2/MATH Level 5": 0.2213, - "hfopenllm_v2/GPQA": 0.3045, - "hfopenllm_v2/MUSR": 0.3368, - "hfopenllm_v2/MMLU-PRO": 0.2934 - } - }, - { - "id": "ContactDoctor/Bio-Medical-Llama-3-8B", - "name": "Bio-Medical-Llama-3-8B", - "developer": "ContactDoctor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4422, - "hfopenllm_v2/BBH": 0.4863, - "hfopenllm_v2/MATH Level 5": 0.0672, - "hfopenllm_v2/GPQA": 0.3339, - "hfopenllm_v2/MUSR": 0.3514, - "hfopenllm_v2/MMLU-PRO": 0.3648 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/contextualai.json b/data/developers/contextualai.json deleted file mode 100644 index e77986d04b0893690694bd6f73b33ac9dd8bc506..0000000000000000000000000000000000000000 --- a/data/developers/contextualai.json +++ /dev/null @@ -1,231 +0,0 @@ -{ - "developer": "ContextualAI", - "models": [ - { - "id": "ContextualAI/archangel_sft-dpo_llama13b", - "name": "ContextualAI/archangel_sft-dpo_llama13b", - "developer": "ContextualAI", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.54, - "reward-bench/Chat": 0.7123, - "reward-bench/Chat Hard": 0.4298, - "reward-bench/Safety": 0.5649, - "reward-bench/Reasoning": 0.4401, - "reward-bench/Prior Sets (0.5 weight)": 0.5656 - } - }, - { - "id": "ContextualAI/archangel_sft-dpo_llama30b", - "name": "ContextualAI/archangel_sft-dpo_llama30b", - "developer": "ContextualAI", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5618, - "reward-bench/Chat": 0.6927, - "reward-bench/Chat Hard": 0.4474, - "reward-bench/Safety": 0.6284, - "reward-bench/Reasoning": 0.4745, - "reward-bench/Prior Sets (0.5 weight)": 0.5705 - } - }, - { - "id": "ContextualAI/archangel_sft-dpo_llama7b", - "name": "ContextualAI/archangel_sft-dpo_llama7b", - "developer": "ContextualAI", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5304, - "reward-bench/Chat": 0.5782, - "reward-bench/Chat Hard": 0.4452, - "reward-bench/Safety": 0.5203, - "reward-bench/Reasoning": 0.5658, - "reward-bench/Prior Sets (0.5 weight)": 0.5544 - } - }, - { - "id": "ContextualAI/archangel_sft-dpo_pythia1-4b", - "name": "ContextualAI/archangel_sft-dpo_pythia1-4b", - "developer": "ContextualAI", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5233, - "reward-bench/Chat": 0.6397, - "reward-bench/Chat Hard": 0.3728, - "reward-bench/Safety": 0.5041, - "reward-bench/Reasoning": 0.5672, - "reward-bench/Prior Sets (0.5 weight)": 0.5427 - } - }, - { - "id": "ContextualAI/archangel_sft-dpo_pythia12-0b", - "name": "ContextualAI/archangel_sft-dpo_pythia12-0b", - "developer": "ContextualAI", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5009, - "reward-bench/Chat": 0.6676, - "reward-bench/Chat Hard": 0.364, - "reward-bench/Safety": 0.5432, - "reward-bench/Reasoning": 0.4139, - "reward-bench/Prior Sets (0.5 weight)": 0.5303 - } - }, - { - "id": "ContextualAI/archangel_sft-dpo_pythia2-8b", - "name": "ContextualAI/archangel_sft-dpo_pythia2-8b", - "developer": "ContextualAI", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5286, - "reward-bench/Chat": 0.8073, - "reward-bench/Chat Hard": 0.3355, - "reward-bench/Safety": 0.4473, - "reward-bench/Reasoning": 0.5135, - "reward-bench/Prior Sets (0.5 weight)": 0.5501 - } - }, - { - "id": "ContextualAI/archangel_sft-dpo_pythia6-9b", - "name": "ContextualAI/archangel_sft-dpo_pythia6-9b", - "developer": "ContextualAI", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5263, - "reward-bench/Chat": 0.7486, - "reward-bench/Chat Hard": 0.3421, - "reward-bench/Safety": 0.5176, - "reward-bench/Reasoning": 0.4847, - "reward-bench/Prior Sets (0.5 weight)": 0.551 - } - }, - { - "id": "ContextualAI/archangel_sft-kto_llama13b", - "name": "ContextualAI/archangel_sft-kto_llama13b", - "developer": "ContextualAI", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5952, - "reward-bench/Chat": 0.8408, - "reward-bench/Chat Hard": 0.3772, - "reward-bench/Safety": 0.4649, - "reward-bench/Reasoning": 0.7077, - "reward-bench/Prior Sets (0.5 weight)": 0.576 - } - }, - { - "id": "ContextualAI/archangel_sft-kto_llama30b", - "name": "ContextualAI/archangel_sft-kto_llama30b", - "developer": "ContextualAI", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5901, - "reward-bench/Chat": 0.8436, - "reward-bench/Chat Hard": 0.4057, - "reward-bench/Safety": 0.6054, - "reward-bench/Reasoning": 0.5075, - "reward-bench/Prior Sets (0.5 weight)": 0.5862 - } - }, - { - "id": "ContextualAI/archangel_sft-kto_llama7b", - "name": "ContextualAI/archangel_sft-kto_llama7b", - "developer": "ContextualAI", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5388, - "reward-bench/Chat": 0.5587, - "reward-bench/Chat Hard": 0.4364, - "reward-bench/Safety": 0.4568, - "reward-bench/Reasoning": 0.6941, - "reward-bench/Prior Sets (0.5 weight)": 0.5575 - } - }, - { - "id": "ContextualAI/archangel_sft-kto_pythia1-4b", - "name": "ContextualAI/archangel_sft-kto_pythia1-4b", - "developer": "ContextualAI", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5581, - "reward-bench/Chat": 0.6844, - "reward-bench/Chat Hard": 0.3794, - "reward-bench/Safety": 0.5257, - "reward-bench/Reasoning": 0.6447, - "reward-bench/Prior Sets (0.5 weight)": 0.5546 - } - }, - { - "id": "ContextualAI/archangel_sft-kto_pythia12-0b", - "name": "ContextualAI/archangel_sft-kto_pythia12-0b", - "developer": "ContextualAI", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5053, - "reward-bench/Chat": 0.7486, - "reward-bench/Chat Hard": 0.3618, - "reward-bench/Safety": 0.4757, - "reward-bench/Reasoning": 0.4127, - "reward-bench/Prior Sets (0.5 weight)": 0.55 - } - }, - { - "id": "ContextualAI/archangel_sft-kto_pythia2-8b", - "name": "ContextualAI/archangel_sft-kto_pythia2-8b", - "developer": "ContextualAI", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5497, - "reward-bench/Chat": 0.757, - "reward-bench/Chat Hard": 0.3421, - "reward-bench/Safety": 0.4743, - "reward-bench/Reasoning": 0.6216, - "reward-bench/Prior Sets (0.5 weight)": 0.557 - } - }, - { - "id": "ContextualAI/archangel_sft-kto_pythia6-9b", - "name": "ContextualAI/archangel_sft-kto_pythia6-9b", - "developer": "ContextualAI", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5561, - "reward-bench/Chat": 0.7765, - "reward-bench/Chat Hard": 0.3618, - "reward-bench/Safety": 0.5365, - "reward-bench/Reasoning": 0.5415, - "reward-bench/Prior Sets (0.5 weight)": 0.5723 - } - }, - { - "id": "ContextualAI/LMUnit-llama3.1-70b", - "name": "ContextualAI/LMUnit-llama3.1-70b", - "developer": "ContextualAI", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.8054, - "reward-bench/Factuality": 0.8463, - "reward-bench/Precise IF": 0.4875, - "reward-bench/Math": 0.7158, - "reward-bench/Safety": 0.9067, - "reward-bench/Focus": 0.9697, - "reward-bench/Ties": 0.9063 - } - }, - { - "id": "ContextualAI/LMUnit-qwen2.5-72b", - "name": "ContextualAI/LMUnit-qwen2.5-72b", - "developer": "ContextualAI", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.8208, - "reward-bench/Factuality": 0.8716, - "reward-bench/Precise IF": 0.5437, - "reward-bench/Math": 0.7268, - "reward-bench/Safety": 0.9133, - "reward-bench/Focus": 0.9677, - "reward-bench/Ties": 0.9014 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/coolspring.json b/data/developers/coolspring.json deleted file mode 100644 index 54c42a45d8ef264f4abde70878b766561279410a..0000000000000000000000000000000000000000 --- a/data/developers/coolspring.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "developer": "CoolSpring", - "models": [ - { - "id": "CoolSpring/Qwen2-0.5B-Abyme", - "name": "Qwen2-0.5B-Abyme", - "developer": "CoolSpring", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1915, - "hfopenllm_v2/BBH": 0.2862, - "hfopenllm_v2/MATH Level 5": 0.0295, - "hfopenllm_v2/GPQA": 0.2534, - "hfopenllm_v2/MUSR": 0.3542, - "hfopenllm_v2/MMLU-PRO": 0.1333 - } - }, - { - "id": "CoolSpring/Qwen2-0.5B-Abyme-merge2", - "name": "Qwen2-0.5B-Abyme-merge2", - "developer": "CoolSpring", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2022, - "hfopenllm_v2/BBH": 0.2994, - "hfopenllm_v2/MATH Level 5": 0.0332, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3687, - "hfopenllm_v2/MMLU-PRO": 0.1489 - } - }, - { - "id": "CoolSpring/Qwen2-0.5B-Abyme-merge3", - "name": "Qwen2-0.5B-Abyme-merge3", - "developer": "CoolSpring", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2386, - "hfopenllm_v2/BBH": 0.3003, - "hfopenllm_v2/MATH Level 5": 0.0317, - "hfopenllm_v2/GPQA": 0.2643, - "hfopenllm_v2/MUSR": 0.3501, - "hfopenllm_v2/MMLU-PRO": 0.15 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/corianas.json b/data/developers/corianas.json deleted file mode 100644 index 8de58242a6c7c93098c3b2b3d233b7643ea69ee9..0000000000000000000000000000000000000000 --- a/data/developers/corianas.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "developer": "Corianas", - "models": [ - { - "id": "Corianas/llama-3-reactor", - "name": "llama-3-reactor", - "developer": "Corianas", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.23, - "hfopenllm_v2/BBH": 0.4457, - "hfopenllm_v2/MATH Level 5": 0.0468, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.3977, - "hfopenllm_v2/MMLU-PRO": 0.2801 - } - }, - { - "id": "Corianas/Neural-Mistral-7B", - "name": "Neural-Mistral-7B", - "developer": "Corianas", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5489, - "hfopenllm_v2/BBH": 0.4428, - "hfopenllm_v2/MATH Level 5": 0.0189, - "hfopenllm_v2/GPQA": 0.2836, - "hfopenllm_v2/MUSR": 0.3873, - "hfopenllm_v2/MMLU-PRO": 0.2738 - } - }, - { - "id": "Corianas/Quokka_2.7b", - "name": "Quokka_2.7b", - "developer": "Corianas", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1749, - "hfopenllm_v2/BBH": 0.3055, - "hfopenllm_v2/MATH Level 5": 0.0083, - "hfopenllm_v2/GPQA": 0.2559, - "hfopenllm_v2/MUSR": 0.3908, - "hfopenllm_v2/MMLU-PRO": 0.1145 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/cortexlm.json b/data/developers/cortexlm.json deleted file mode 100644 index 71f8ef9440c41c73700314b9148c2374292ded16..0000000000000000000000000000000000000000 --- a/data/developers/cortexlm.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "CortexLM", - "models": [ - { - "id": "CortexLM/btlm-7b-base-v0.2", - "name": "btlm-7b-base-v0.2", - "developer": "CortexLM", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1483, - "hfopenllm_v2/BBH": 0.4006, - "hfopenllm_v2/MATH Level 5": 0.0151, - "hfopenllm_v2/GPQA": 0.2534, - "hfopenllm_v2/MUSR": 0.3846, - "hfopenllm_v2/MMLU-PRO": 0.235 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/cpayne1303.json b/data/developers/cpayne1303.json deleted file mode 100644 index 6d735bd94a67b9fc86d407e5a74d4ec119a21a01..0000000000000000000000000000000000000000 --- a/data/developers/cpayne1303.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "developer": "cpayne1303", - "models": [ - { - "id": "cpayne1303/cp2024", - "name": "cp2024", - "developer": "cpayne1303", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1658, - "hfopenllm_v2/BBH": 0.2985, - "hfopenllm_v2/MATH Level 5": 0.0053, - "hfopenllm_v2/GPQA": 0.2559, - "hfopenllm_v2/MUSR": 0.3383, - "hfopenllm_v2/MMLU-PRO": 0.1101 - } - }, - { - "id": "cpayne1303/cp2024-instruct", - "name": "cp2024-instruct", - "developer": "cpayne1303", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1706, - "hfopenllm_v2/BBH": 0.2947, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3686, - "hfopenllm_v2/MMLU-PRO": 0.1167 - } - }, - { - "id": "cpayne1303/llama-43m-beta", - "name": "llama-43m-beta", - "developer": "cpayne1303", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1916, - "hfopenllm_v2/BBH": 0.2977, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2685, - "hfopenllm_v2/MUSR": 0.3872, - "hfopenllm_v2/MMLU-PRO": 0.1132 - } - }, - { - "id": "cpayne1303/smallcp2024", - "name": "smallcp2024", - "developer": "cpayne1303", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1582, - "hfopenllm_v2/BBH": 0.3027, - "hfopenllm_v2/MATH Level 5": 0.0053, - "hfopenllm_v2/GPQA": 0.2307, - "hfopenllm_v2/MUSR": 0.3425, - "hfopenllm_v2/MMLU-PRO": 0.1114 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/cran-may.json b/data/developers/cran-may.json deleted file mode 100644 index 30fa7f9c89a49484ecf95607c6aea9104e1fa041..0000000000000000000000000000000000000000 --- a/data/developers/cran-may.json +++ /dev/null @@ -1,103 +0,0 @@ -{ - "developer": "Cran-May", - "models": [ - { - "id": "Cran-May/merge_model_20250308_2", - "name": "merge_model_20250308_2", - "developer": "Cran-May", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5932, - "hfopenllm_v2/BBH": 0.6585, - "hfopenllm_v2/MATH Level 5": 0.4381, - "hfopenllm_v2/GPQA": 0.3909, - "hfopenllm_v2/MUSR": 0.4794, - "hfopenllm_v2/MMLU-PRO": 0.542 - } - }, - { - "id": "Cran-May/merge_model_20250308_3", - "name": "merge_model_20250308_3", - "developer": "Cran-May", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6018, - "hfopenllm_v2/BBH": 0.6271, - "hfopenllm_v2/MATH Level 5": 0.2545, - "hfopenllm_v2/GPQA": 0.3221, - "hfopenllm_v2/MUSR": 0.432, - "hfopenllm_v2/MMLU-PRO": 0.4962 - } - }, - { - "id": "Cran-May/merge_model_20250308_4", - "name": "merge_model_20250308_4", - "developer": "Cran-May", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.454, - "hfopenllm_v2/BBH": 0.6664, - "hfopenllm_v2/MATH Level 5": 0.4199, - "hfopenllm_v2/GPQA": 0.3977, - "hfopenllm_v2/MUSR": 0.4688, - "hfopenllm_v2/MMLU-PRO": 0.5367 - } - }, - { - "id": "Cran-May/SCE-2-24B", - "name": "SCE-2-24B", - "developer": "Cran-May", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5866, - "hfopenllm_v2/BBH": 0.6265, - "hfopenllm_v2/MATH Level 5": 0.1896, - "hfopenllm_v2/GPQA": 0.3372, - "hfopenllm_v2/MUSR": 0.4528, - "hfopenllm_v2/MMLU-PRO": 0.4612 - } - }, - { - "id": "Cran-May/SCE-3-24B", - "name": "SCE-3-24B", - "developer": "Cran-May", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5465, - "hfopenllm_v2/BBH": 0.5973, - "hfopenllm_v2/MATH Level 5": 0.1881, - "hfopenllm_v2/GPQA": 0.3465, - "hfopenllm_v2/MUSR": 0.4435, - "hfopenllm_v2/MMLU-PRO": 0.4647 - } - }, - { - "id": "Cran-May/T.E-8.1", - "name": "T.E-8.1", - "developer": "Cran-May", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7077, - "hfopenllm_v2/BBH": 0.5582, - "hfopenllm_v2/MATH Level 5": 0.4456, - "hfopenllm_v2/GPQA": 0.3129, - "hfopenllm_v2/MUSR": 0.4505, - "hfopenllm_v2/MMLU-PRO": 0.4432 - } - }, - { - "id": "Cran-May/tempmotacilla-cinerea-0308", - "name": "tempmotacilla-cinerea-0308", - "developer": "Cran-May", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8085, - "hfopenllm_v2/BBH": 0.6551, - "hfopenllm_v2/MATH Level 5": 0.5551, - "hfopenllm_v2/GPQA": 0.3624, - "hfopenllm_v2/MUSR": 0.4208, - "hfopenllm_v2/MMLU-PRO": 0.525 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/creitingameplays.json b/data/developers/creitingameplays.json deleted file mode 100644 index 4c545e3e9a2cc1a1e528e82b73d82085837f071f..0000000000000000000000000000000000000000 --- a/data/developers/creitingameplays.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "CreitinGameplays", - "models": [ - { - "id": "CreitinGameplays/Llama-3.1-8B-R1-v0.1", - "name": "Llama-3.1-8B-R1-v0.1", - "developer": "CreitinGameplays", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3235, - "hfopenllm_v2/BBH": 0.3057, - "hfopenllm_v2/MATH Level 5": 0.1813, - "hfopenllm_v2/GPQA": 0.2584, - "hfopenllm_v2/MUSR": 0.3622, - "hfopenllm_v2/MMLU-PRO": 0.1252 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/crestf411.json b/data/developers/crestf411.json deleted file mode 100644 index 79a1922450759ce563b5462cc77575548591305f..0000000000000000000000000000000000000000 --- a/data/developers/crestf411.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "crestf411", - "models": [ - { - "id": "crestf411/MN-Slush", - "name": "MN-Slush", - "developer": "crestf411", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4077, - "hfopenllm_v2/BBH": 0.534, - "hfopenllm_v2/MATH Level 5": 0.1269, - "hfopenllm_v2/GPQA": 0.3238, - "hfopenllm_v2/MUSR": 0.3933, - "hfopenllm_v2/MMLU-PRO": 0.3508 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/cstr.json b/data/developers/cstr.json deleted file mode 100644 index 0276c7cc7f9c8121a4528a183bb6747c330f08a7..0000000000000000000000000000000000000000 --- a/data/developers/cstr.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "cstr", - "models": [ - { - "id": "cstr/llama3.1-8b-spaetzle-v90", - "name": "llama3.1-8b-spaetzle-v90", - "developer": "cstr", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7356, - "hfopenllm_v2/BBH": 0.5303, - "hfopenllm_v2/MATH Level 5": 0.1495, - "hfopenllm_v2/GPQA": 0.2827, - "hfopenllm_v2/MUSR": 0.4134, - "hfopenllm_v2/MMLU-PRO": 0.3731 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/cultrix.json b/data/developers/cultrix.json deleted file mode 100644 index 0e775d2374e43f090e9c30eb6e47e1b19b229a3a..0000000000000000000000000000000000000000 --- a/data/developers/cultrix.json +++ /dev/null @@ -1,453 +0,0 @@ -{ - "developer": "CultriX", - "models": [ - { - "id": "CultriX/Qwen2.5-14B-Broca", - "name": "Qwen2.5-14B-Broca", - "developer": "CultriX", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5604, - "hfopenllm_v2/BBH": 0.6527, - "hfopenllm_v2/MATH Level 5": 0.358, - "hfopenllm_v2/GPQA": 0.3867, - "hfopenllm_v2/MUSR": 0.4767, - "hfopenllm_v2/MMLU-PRO": 0.5364 - } - }, - { - "id": "CultriX/Qwen2.5-14B-Brocav3", - "name": "Qwen2.5-14B-Brocav3", - "developer": "CultriX", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6952, - "hfopenllm_v2/BBH": 0.6452, - "hfopenllm_v2/MATH Level 5": 0.3875, - "hfopenllm_v2/GPQA": 0.3591, - "hfopenllm_v2/MUSR": 0.4756, - "hfopenllm_v2/MMLU-PRO": 0.5317 - } - }, - { - "id": "CultriX/Qwen2.5-14B-Brocav6", - "name": "Qwen2.5-14B-Brocav6", - "developer": "CultriX", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6995, - "hfopenllm_v2/BBH": 0.6389, - "hfopenllm_v2/MATH Level 5": 0.3875, - "hfopenllm_v2/GPQA": 0.3674, - "hfopenllm_v2/MUSR": 0.4742, - "hfopenllm_v2/MMLU-PRO": 0.5319 - } - }, - { - "id": "CultriX/Qwen2.5-14B-Brocav7", - "name": "Qwen2.5-14B-Brocav7", - "developer": "CultriX", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6724, - "hfopenllm_v2/BBH": 0.6444, - "hfopenllm_v2/MATH Level 5": 0.3844, - "hfopenllm_v2/GPQA": 0.3674, - "hfopenllm_v2/MUSR": 0.4796, - "hfopenllm_v2/MMLU-PRO": 0.5258 - } - }, - { - "id": "CultriX/Qwen2.5-14B-BrocaV9", - "name": "Qwen2.5-14B-BrocaV9", - "developer": "CultriX", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6763, - "hfopenllm_v2/BBH": 0.6391, - "hfopenllm_v2/MATH Level 5": 0.3814, - "hfopenllm_v2/GPQA": 0.3641, - "hfopenllm_v2/MUSR": 0.469, - "hfopenllm_v2/MMLU-PRO": 0.5331 - } - }, - { - "id": "CultriX/Qwen2.5-14B-Emerged", - "name": "Qwen2.5-14B-Emerged", - "developer": "CultriX", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7, - "hfopenllm_v2/BBH": 0.626, - "hfopenllm_v2/MATH Level 5": 0.3248, - "hfopenllm_v2/GPQA": 0.3574, - "hfopenllm_v2/MUSR": 0.4691, - "hfopenllm_v2/MMLU-PRO": 0.5186 - } - }, - { - "id": "CultriX/Qwen2.5-14B-Emergedv3", - "name": "Qwen2.5-14B-Emergedv3", - "developer": "CultriX", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6388, - "hfopenllm_v2/BBH": 0.6191, - "hfopenllm_v2/MATH Level 5": 0.4358, - "hfopenllm_v2/GPQA": 0.3607, - "hfopenllm_v2/MUSR": 0.4728, - "hfopenllm_v2/MMLU-PRO": 0.5174 - } - }, - { - "id": "CultriX/Qwen2.5-14B-FinalMerge", - "name": "Qwen2.5-14B-FinalMerge", - "developer": "CultriX", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4891, - "hfopenllm_v2/BBH": 0.5715, - "hfopenllm_v2/MATH Level 5": 0.3814, - "hfopenllm_v2/GPQA": 0.3549, - "hfopenllm_v2/MUSR": 0.4379, - "hfopenllm_v2/MMLU-PRO": 0.4574 - } - }, - { - "id": "CultriX/Qwen2.5-14B-Hyper", - "name": "Qwen2.5-14B-Hyper", - "developer": "CultriX", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5391, - "hfopenllm_v2/BBH": 0.6507, - "hfopenllm_v2/MATH Level 5": 0.3437, - "hfopenllm_v2/GPQA": 0.3918, - "hfopenllm_v2/MUSR": 0.4898, - "hfopenllm_v2/MMLU-PRO": 0.5374 - } - }, - { - "id": "CultriX/Qwen2.5-14B-Hyperionv3", - "name": "Qwen2.5-14B-Hyperionv3", - "developer": "CultriX", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6836, - "hfopenllm_v2/BBH": 0.6522, - "hfopenllm_v2/MATH Level 5": 0.3701, - "hfopenllm_v2/GPQA": 0.3708, - "hfopenllm_v2/MUSR": 0.473, - "hfopenllm_v2/MMLU-PRO": 0.534 - } - }, - { - "id": "CultriX/Qwen2.5-14B-Hyperionv4", - "name": "Qwen2.5-14B-Hyperionv4", - "developer": "CultriX", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5416, - "hfopenllm_v2/BBH": 0.6472, - "hfopenllm_v2/MATH Level 5": 0.3474, - "hfopenllm_v2/GPQA": 0.3977, - "hfopenllm_v2/MUSR": 0.4832, - "hfopenllm_v2/MMLU-PRO": 0.5364 - } - }, - { - "id": "CultriX/Qwen2.5-14B-Hyperionv5", - "name": "Qwen2.5-14B-Hyperionv5", - "developer": "CultriX", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6729, - "hfopenllm_v2/BBH": 0.6443, - "hfopenllm_v2/MATH Level 5": 0.3822, - "hfopenllm_v2/GPQA": 0.3716, - "hfopenllm_v2/MUSR": 0.4795, - "hfopenllm_v2/MMLU-PRO": 0.5302 - } - }, - { - "id": "CultriX/Qwen2.5-14B-HyperMarck-dl", - "name": "Qwen2.5-14B-HyperMarck-dl", - "developer": "CultriX", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.665, - "hfopenllm_v2/BBH": 0.6096, - "hfopenllm_v2/MATH Level 5": 0.5287, - "hfopenllm_v2/GPQA": 0.3674, - "hfopenllm_v2/MUSR": 0.4416, - "hfopenllm_v2/MMLU-PRO": 0.5091 - } - }, - { - "id": "CultriX/Qwen2.5-14B-MegaMerge-pt2", - "name": "Qwen2.5-14B-MegaMerge-pt2", - "developer": "CultriX", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5683, - "hfopenllm_v2/BBH": 0.6578, - "hfopenllm_v2/MATH Level 5": 0.3995, - "hfopenllm_v2/GPQA": 0.3792, - "hfopenllm_v2/MUSR": 0.4729, - "hfopenllm_v2/MMLU-PRO": 0.5421 - } - }, - { - "id": "CultriX/Qwen2.5-14B-MergeStock", - "name": "Qwen2.5-14B-MergeStock", - "developer": "CultriX", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5685, - "hfopenllm_v2/BBH": 0.6579, - "hfopenllm_v2/MATH Level 5": 0.4147, - "hfopenllm_v2/GPQA": 0.3733, - "hfopenllm_v2/MUSR": 0.4676, - "hfopenllm_v2/MMLU-PRO": 0.5396 - } - }, - { - "id": "CultriX/Qwen2.5-14B-partialmergept1", - "name": "Qwen2.5-14B-partialmergept1", - "developer": "CultriX", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6337, - "hfopenllm_v2/BBH": 0.6151, - "hfopenllm_v2/MATH Level 5": 0.4539, - "hfopenllm_v2/GPQA": 0.3616, - "hfopenllm_v2/MUSR": 0.4757, - "hfopenllm_v2/MMLU-PRO": 0.5208 - } - }, - { - "id": "CultriX/Qwen2.5-14B-ReasoningMerge", - "name": "Qwen2.5-14B-ReasoningMerge", - "developer": "CultriX", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4605, - "hfopenllm_v2/BBH": 0.6578, - "hfopenllm_v2/MATH Level 5": 0.5204, - "hfopenllm_v2/GPQA": 0.4077, - "hfopenllm_v2/MUSR": 0.5166, - "hfopenllm_v2/MMLU-PRO": 0.5345 - } - }, - { - "id": "CultriX/Qwen2.5-14B-Ultimav2", - "name": "Qwen2.5-14B-Ultimav2", - "developer": "CultriX", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.55, - "hfopenllm_v2/BBH": 0.6555, - "hfopenllm_v2/MATH Level 5": 0.3844, - "hfopenllm_v2/GPQA": 0.3851, - "hfopenllm_v2/MUSR": 0.4966, - "hfopenllm_v2/MMLU-PRO": 0.5417 - } - }, - { - "id": "CultriX/Qwen2.5-14B-Unity", - "name": "Qwen2.5-14B-Unity", - "developer": "CultriX", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6739, - "hfopenllm_v2/BBH": 0.602, - "hfopenllm_v2/MATH Level 5": 0.4313, - "hfopenllm_v2/GPQA": 0.3473, - "hfopenllm_v2/MUSR": 0.4679, - "hfopenllm_v2/MMLU-PRO": 0.5076 - } - }, - { - "id": "CultriX/Qwen2.5-14B-Wernicke", - "name": "Qwen2.5-14B-Wernicke", - "developer": "CultriX", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5235, - "hfopenllm_v2/BBH": 0.6568, - "hfopenllm_v2/MATH Level 5": 0.3814, - "hfopenllm_v2/GPQA": 0.3935, - "hfopenllm_v2/MUSR": 0.4689, - "hfopenllm_v2/MMLU-PRO": 0.5424 - } - }, - { - "id": "CultriX/Qwen2.5-14B-Wernicke-SFT", - "name": "Qwen2.5-14B-Wernicke-SFT", - "developer": "CultriX", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4937, - "hfopenllm_v2/BBH": 0.6461, - "hfopenllm_v2/MATH Level 5": 0.3595, - "hfopenllm_v2/GPQA": 0.354, - "hfopenllm_v2/MUSR": 0.39, - "hfopenllm_v2/MMLU-PRO": 0.507 - } - }, - { - "id": "CultriX/Qwen2.5-14B-Wernicke-SLERP", - "name": "Qwen2.5-14B-Wernicke-SLERP", - "developer": "CultriX", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5589, - "hfopenllm_v2/BBH": 0.6441, - "hfopenllm_v2/MATH Level 5": 0.4486, - "hfopenllm_v2/GPQA": 0.344, - "hfopenllm_v2/MUSR": 0.414, - "hfopenllm_v2/MMLU-PRO": 0.5094 - } - }, - { - "id": "CultriX/Qwen2.5-14B-Wernickev3", - "name": "Qwen2.5-14B-Wernickev3", - "developer": "CultriX", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7048, - "hfopenllm_v2/BBH": 0.6184, - "hfopenllm_v2/MATH Level 5": 0.3542, - "hfopenllm_v2/GPQA": 0.3624, - "hfopenllm_v2/MUSR": 0.4717, - "hfopenllm_v2/MMLU-PRO": 0.5151 - } - }, - { - "id": "CultriX/Qwenfinity-2.5-14B", - "name": "Qwenfinity-2.5-14B", - "developer": "CultriX", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4814, - "hfopenllm_v2/BBH": 0.5655, - "hfopenllm_v2/MATH Level 5": 0.4101, - "hfopenllm_v2/GPQA": 0.349, - "hfopenllm_v2/MUSR": 0.4506, - "hfopenllm_v2/MMLU-PRO": 0.4498 - } - }, - { - "id": "CultriX/Qwestion-14B", - "name": "Qwestion-14B", - "developer": "CultriX", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6318, - "hfopenllm_v2/BBH": 0.645, - "hfopenllm_v2/MATH Level 5": 0.3724, - "hfopenllm_v2/GPQA": 0.3683, - "hfopenllm_v2/MUSR": 0.4636, - "hfopenllm_v2/MMLU-PRO": 0.5422 - } - }, - { - "id": "CultriX/SeQwence-14B", - "name": "SeQwence-14B", - "developer": "CultriX", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5352, - "hfopenllm_v2/BBH": 0.6506, - "hfopenllm_v2/MATH Level 5": 0.3535, - "hfopenllm_v2/GPQA": 0.3607, - "hfopenllm_v2/MUSR": 0.4666, - "hfopenllm_v2/MMLU-PRO": 0.5419 - } - }, - { - "id": "CultriX/SeQwence-14B-EvolMerge", - "name": "SeQwence-14B-EvolMerge", - "developer": "CultriX", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5382, - "hfopenllm_v2/BBH": 0.6572, - "hfopenllm_v2/MATH Level 5": 0.3671, - "hfopenllm_v2/GPQA": 0.3809, - "hfopenllm_v2/MUSR": 0.4821, - "hfopenllm_v2/MMLU-PRO": 0.5419 - } - }, - { - "id": "CultriX/SeQwence-14B-EvolMergev1", - "name": "SeQwence-14B-EvolMergev1", - "developer": "CultriX", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5555, - "hfopenllm_v2/BBH": 0.6546, - "hfopenllm_v2/MATH Level 5": 0.4215, - "hfopenllm_v2/GPQA": 0.3767, - "hfopenllm_v2/MUSR": 0.4623, - "hfopenllm_v2/MMLU-PRO": 0.5393 - } - }, - { - "id": "CultriX/SeQwence-14B-v5", - "name": "SeQwence-14B-v5", - "developer": "CultriX", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.592, - "hfopenllm_v2/BBH": 0.6517, - "hfopenllm_v2/MATH Level 5": 0.3308, - "hfopenllm_v2/GPQA": 0.37, - "hfopenllm_v2/MUSR": 0.4714, - "hfopenllm_v2/MMLU-PRO": 0.5415 - } - }, - { - "id": "CultriX/SeQwence-14Bv1", - "name": "SeQwence-14Bv1", - "developer": "CultriX", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6678, - "hfopenllm_v2/BBH": 0.6345, - "hfopenllm_v2/MATH Level 5": 0.361, - "hfopenllm_v2/GPQA": 0.3616, - "hfopenllm_v2/MUSR": 0.4704, - "hfopenllm_v2/MMLU-PRO": 0.532 - } - }, - { - "id": "CultriX/SeQwence-14Bv2", - "name": "SeQwence-14Bv2", - "developer": "CultriX", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5786, - "hfopenllm_v2/BBH": 0.6305, - "hfopenllm_v2/MATH Level 5": 0.4758, - "hfopenllm_v2/GPQA": 0.3607, - "hfopenllm_v2/MUSR": 0.4601, - "hfopenllm_v2/MMLU-PRO": 0.5334 - } - }, - { - "id": "CultriX/SeQwence-14Bv3", - "name": "SeQwence-14Bv3", - "developer": "CultriX", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5719, - "hfopenllm_v2/BBH": 0.6302, - "hfopenllm_v2/MATH Level 5": 0.4766, - "hfopenllm_v2/GPQA": 0.3649, - "hfopenllm_v2/MUSR": 0.4624, - "hfopenllm_v2/MMLU-PRO": 0.5335 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/cyberagent.json b/data/developers/cyberagent.json deleted file mode 100644 index 671c959996d724a7093cd5abbf2aaff36891acab..0000000000000000000000000000000000000000 --- a/data/developers/cyberagent.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "cyberagent", - "models": [ - { - "id": "cyberagent/calm3-22b-chat", - "name": "calm3-22b-chat", - "developer": "cyberagent", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5091, - "hfopenllm_v2/BBH": 0.4992, - "hfopenllm_v2/MATH Level 5": 0.0695, - "hfopenllm_v2/GPQA": 0.2768, - "hfopenllm_v2/MUSR": 0.4553, - "hfopenllm_v2/MMLU-PRO": 0.295 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/cyfragovpl.json b/data/developers/cyfragovpl.json deleted file mode 100644 index 275204dc2ef859e3fbec74104c82df244ab3fe62..0000000000000000000000000000000000000000 --- a/data/developers/cyfragovpl.json +++ /dev/null @@ -1,89 +0,0 @@ -{ - "developer": "CYFRAGOVPL", - "models": [ - { - "id": "CYFRAGOVPL/Llama-PLLuM-8B-base", - "name": "Llama-PLLuM-8B-base", - "developer": "CYFRAGOVPL", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2899, - "hfopenllm_v2/BBH": 0.432, - "hfopenllm_v2/MATH Level 5": 0.0363, - "hfopenllm_v2/GPQA": 0.2852, - "hfopenllm_v2/MUSR": 0.397, - "hfopenllm_v2/MMLU-PRO": 0.2757 - } - }, - { - "id": "CYFRAGOVPL/Llama-PLLuM-8B-chat", - "name": "Llama-PLLuM-8B-chat", - "developer": "CYFRAGOVPL", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3515, - "hfopenllm_v2/BBH": 0.4077, - "hfopenllm_v2/MATH Level 5": 0.034, - "hfopenllm_v2/GPQA": 0.2643, - "hfopenllm_v2/MUSR": 0.4199, - "hfopenllm_v2/MMLU-PRO": 0.2719 - } - }, - { - "id": "CYFRAGOVPL/PLLuM-12B-base", - "name": "PLLuM-12B-base", - "developer": "CYFRAGOVPL", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2821, - "hfopenllm_v2/BBH": 0.4391, - "hfopenllm_v2/MATH Level 5": 0.0287, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.4142, - "hfopenllm_v2/MMLU-PRO": 0.274 - } - }, - { - "id": "CYFRAGOVPL/PLLuM-12B-chat", - "name": "PLLuM-12B-chat", - "developer": "CYFRAGOVPL", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3214, - "hfopenllm_v2/BBH": 0.4446, - "hfopenllm_v2/MATH Level 5": 0.0181, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.4115, - "hfopenllm_v2/MMLU-PRO": 0.2872 - } - }, - { - "id": "CYFRAGOVPL/PLLuM-12B-nc-base", - "name": "PLLuM-12B-nc-base", - "developer": "CYFRAGOVPL", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2405, - "hfopenllm_v2/BBH": 0.4277, - "hfopenllm_v2/MATH Level 5": 0.0219, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.3645, - "hfopenllm_v2/MMLU-PRO": 0.2559 - } - }, - { - "id": "CYFRAGOVPL/PLLuM-12B-nc-chat", - "name": "PLLuM-12B-nc-chat", - "developer": "CYFRAGOVPL", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2834, - "hfopenllm_v2/BBH": 0.4576, - "hfopenllm_v2/MATH Level 5": 0.0121, - "hfopenllm_v2/GPQA": 0.2827, - "hfopenllm_v2/MUSR": 0.4354, - "hfopenllm_v2/MMLU-PRO": 0.2597 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/daemontatox.json b/data/developers/daemontatox.json deleted file mode 100644 index 9b4de269d2d2e3992e83943985294da4a4347f97..0000000000000000000000000000000000000000 --- a/data/developers/daemontatox.json +++ /dev/null @@ -1,453 +0,0 @@ -{ - "developer": "Daemontatox", - "models": [ - { - "id": "Daemontatox/AetherDrake-SFT", - "name": "AetherDrake-SFT", - "developer": "Daemontatox", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4813, - "hfopenllm_v2/BBH": 0.4872, - "hfopenllm_v2/MATH Level 5": 0.1511, - "hfopenllm_v2/GPQA": 0.3205, - "hfopenllm_v2/MUSR": 0.4088, - "hfopenllm_v2/MMLU-PRO": 0.3499 - } - }, - { - "id": "Daemontatox/AetherSett", - "name": "AetherSett", - "developer": "Daemontatox", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.537, - "hfopenllm_v2/BBH": 0.5452, - "hfopenllm_v2/MATH Level 5": 0.3973, - "hfopenllm_v2/GPQA": 0.3079, - "hfopenllm_v2/MUSR": 0.4603, - "hfopenllm_v2/MMLU-PRO": 0.4279 - } - }, - { - "id": "Daemontatox/AetherTOT", - "name": "AetherTOT", - "developer": "Daemontatox", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4398, - "hfopenllm_v2/BBH": 0.5066, - "hfopenllm_v2/MATH Level 5": 0.1488, - "hfopenllm_v2/GPQA": 0.3238, - "hfopenllm_v2/MUSR": 0.4079, - "hfopenllm_v2/MMLU-PRO": 0.3804 - } - }, - { - "id": "Daemontatox/AetherUncensored", - "name": "AetherUncensored", - "developer": "Daemontatox", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4042, - "hfopenllm_v2/BBH": 0.4463, - "hfopenllm_v2/MATH Level 5": 0.145, - "hfopenllm_v2/GPQA": 0.2886, - "hfopenllm_v2/MUSR": 0.3747, - "hfopenllm_v2/MMLU-PRO": 0.271 - } - }, - { - "id": "Daemontatox/Cogito-MIS", - "name": "Cogito-MIS", - "developer": "Daemontatox", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1815, - "hfopenllm_v2/BBH": 0.506, - "hfopenllm_v2/MATH Level 5": 0.0861, - "hfopenllm_v2/GPQA": 0.2567, - "hfopenllm_v2/MUSR": 0.3768, - "hfopenllm_v2/MMLU-PRO": 0.1435 - } - }, - { - "id": "Daemontatox/CogitoDistil", - "name": "CogitoDistil", - "developer": "Daemontatox", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2776, - "hfopenllm_v2/BBH": 0.3677, - "hfopenllm_v2/MATH Level 5": 0.3927, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3755, - "hfopenllm_v2/MMLU-PRO": 0.2625 - } - }, - { - "id": "Daemontatox/CogitoZ", - "name": "CogitoZ", - "developer": "Daemontatox", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3967, - "hfopenllm_v2/BBH": 0.6734, - "hfopenllm_v2/MATH Level 5": 0.5242, - "hfopenllm_v2/GPQA": 0.3951, - "hfopenllm_v2/MUSR": 0.4793, - "hfopenllm_v2/MMLU-PRO": 0.5593 - } - }, - { - "id": "Daemontatox/CogitoZ14", - "name": "CogitoZ14", - "developer": "Daemontatox", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6637, - "hfopenllm_v2/BBH": 0.6298, - "hfopenllm_v2/MATH Level 5": 0.4222, - "hfopenllm_v2/GPQA": 0.3163, - "hfopenllm_v2/MUSR": 0.4059, - "hfopenllm_v2/MMLU-PRO": 0.3999 - } - }, - { - "id": "Daemontatox/DocumentCogito", - "name": "DocumentCogito", - "developer": "Daemontatox", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5064, - "hfopenllm_v2/BBH": 0.5112, - "hfopenllm_v2/MATH Level 5": 0.1631, - "hfopenllm_v2/GPQA": 0.3163, - "hfopenllm_v2/MUSR": 0.3973, - "hfopenllm_v2/MMLU-PRO": 0.3802 - } - }, - { - "id": "Daemontatox/Llama3.3-70B-CogniLink", - "name": "Llama3.3-70B-CogniLink", - "developer": "Daemontatox", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6931, - "hfopenllm_v2/BBH": 0.6668, - "hfopenllm_v2/MATH Level 5": 0.4139, - "hfopenllm_v2/GPQA": 0.4455, - "hfopenllm_v2/MUSR": 0.4877, - "hfopenllm_v2/MMLU-PRO": 0.5173 - } - }, - { - "id": "Daemontatox/Llama_cot", - "name": "Llama_cot", - "developer": "Daemontatox", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7549, - "hfopenllm_v2/BBH": 0.4838, - "hfopenllm_v2/MATH Level 5": 0.2024, - "hfopenllm_v2/GPQA": 0.2911, - "hfopenllm_v2/MUSR": 0.3872, - "hfopenllm_v2/MMLU-PRO": 0.3518 - } - }, - { - "id": "Daemontatox/MawaredT1", - "name": "MawaredT1", - "developer": "Daemontatox", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4199, - "hfopenllm_v2/BBH": 0.5215, - "hfopenllm_v2/MATH Level 5": 0.3021, - "hfopenllm_v2/GPQA": 0.3347, - "hfopenllm_v2/MUSR": 0.4702, - "hfopenllm_v2/MMLU-PRO": 0.4718 - } - }, - { - "id": "Daemontatox/mini-Cogito-R1", - "name": "mini-Cogito-R1", - "developer": "Daemontatox", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2298, - "hfopenllm_v2/BBH": 0.328, - "hfopenllm_v2/MATH Level 5": 0.2749, - "hfopenllm_v2/GPQA": 0.2869, - "hfopenllm_v2/MUSR": 0.3447, - "hfopenllm_v2/MMLU-PRO": 0.1482 - } - }, - { - "id": "Daemontatox/mini_Pathfinder", - "name": "mini_Pathfinder", - "developer": "Daemontatox", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2962, - "hfopenllm_v2/BBH": 0.3956, - "hfopenllm_v2/MATH Level 5": 0.4751, - "hfopenllm_v2/GPQA": 0.2584, - "hfopenllm_v2/MUSR": 0.3781, - "hfopenllm_v2/MMLU-PRO": 0.2809 - } - }, - { - "id": "Daemontatox/Mini_QwQ", - "name": "Mini_QwQ", - "developer": "Daemontatox", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4497, - "hfopenllm_v2/BBH": 0.5549, - "hfopenllm_v2/MATH Level 5": 0.4192, - "hfopenllm_v2/GPQA": 0.3037, - "hfopenllm_v2/MUSR": 0.4682, - "hfopenllm_v2/MMLU-PRO": 0.4373 - } - }, - { - "id": "Daemontatox/NemoR", - "name": "NemoR", - "developer": "Daemontatox", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2287, - "hfopenllm_v2/BBH": 0.5194, - "hfopenllm_v2/MATH Level 5": 0.0831, - "hfopenllm_v2/GPQA": 0.3272, - "hfopenllm_v2/MUSR": 0.3908, - "hfopenllm_v2/MMLU-PRO": 0.329 - } - }, - { - "id": "Daemontatox/PathfinderAI", - "name": "PathfinderAI", - "developer": "Daemontatox", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3745, - "hfopenllm_v2/BBH": 0.6668, - "hfopenllm_v2/MATH Level 5": 0.4758, - "hfopenllm_v2/GPQA": 0.3943, - "hfopenllm_v2/MUSR": 0.4858, - "hfopenllm_v2/MMLU-PRO": 0.5593 - } - }, - { - "id": "Daemontatox/PathFinderAI2.0", - "name": "PathFinderAI2.0", - "developer": "Daemontatox", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4541, - "hfopenllm_v2/BBH": 0.6658, - "hfopenllm_v2/MATH Level 5": 0.5076, - "hfopenllm_v2/GPQA": 0.302, - "hfopenllm_v2/MUSR": 0.4216, - "hfopenllm_v2/MMLU-PRO": 0.5547 - } - }, - { - "id": "Daemontatox/PathFinderAi3.0", - "name": "PathFinderAi3.0", - "developer": "Daemontatox", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4271, - "hfopenllm_v2/BBH": 0.6884, - "hfopenllm_v2/MATH Level 5": 0.5045, - "hfopenllm_v2/GPQA": 0.4086, - "hfopenllm_v2/MUSR": 0.4807, - "hfopenllm_v2/MMLU-PRO": 0.5757 - } - }, - { - "id": "Daemontatox/Phi-4-COT", - "name": "Phi-4-COT", - "developer": "Daemontatox", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1793, - "hfopenllm_v2/BBH": 0.6173, - "hfopenllm_v2/MATH Level 5": 0.2243, - "hfopenllm_v2/GPQA": 0.3356, - "hfopenllm_v2/MUSR": 0.453, - "hfopenllm_v2/MMLU-PRO": 0.5005 - } - }, - { - "id": "Daemontatox/PixelParse_AI", - "name": "PixelParse_AI", - "developer": "Daemontatox", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4383, - "hfopenllm_v2/BBH": 0.5034, - "hfopenllm_v2/MATH Level 5": 0.1473, - "hfopenllm_v2/GPQA": 0.3238, - "hfopenllm_v2/MUSR": 0.4052, - "hfopenllm_v2/MMLU-PRO": 0.3778 - } - }, - { - "id": "Daemontatox/RA2.0", - "name": "RA2.0", - "developer": "Daemontatox", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3784, - "hfopenllm_v2/BBH": 0.4889, - "hfopenllm_v2/MATH Level 5": 0.3837, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.4091, - "hfopenllm_v2/MMLU-PRO": 0.2616 - } - }, - { - "id": "Daemontatox/RA_Reasoner", - "name": "RA_Reasoner", - "developer": "Daemontatox", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5592, - "hfopenllm_v2/BBH": 0.6054, - "hfopenllm_v2/MATH Level 5": 0.2122, - "hfopenllm_v2/GPQA": 0.3314, - "hfopenllm_v2/MUSR": 0.3964, - "hfopenllm_v2/MMLU-PRO": 0.43 - } - }, - { - "id": "Daemontatox/RA_Reasoner2.0", - "name": "RA_Reasoner2.0", - "developer": "Daemontatox", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5366, - "hfopenllm_v2/BBH": 0.6062, - "hfopenllm_v2/MATH Level 5": 0.2311, - "hfopenllm_v2/GPQA": 0.3247, - "hfopenllm_v2/MUSR": 0.3884, - "hfopenllm_v2/MMLU-PRO": 0.4353 - } - }, - { - "id": "Daemontatox/ReasonTest", - "name": "ReasonTest", - "developer": "Daemontatox", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.408, - "hfopenllm_v2/BBH": 0.5435, - "hfopenllm_v2/MATH Level 5": 0.2137, - "hfopenllm_v2/GPQA": 0.3188, - "hfopenllm_v2/MUSR": 0.4315, - "hfopenllm_v2/MMLU-PRO": 0.4272 - } - }, - { - "id": "Daemontatox/Research_PathfinderAI", - "name": "Research_PathfinderAI", - "developer": "Daemontatox", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3457, - "hfopenllm_v2/BBH": 0.2872, - "hfopenllm_v2/MATH Level 5": 0.1699, - "hfopenllm_v2/GPQA": 0.2408, - "hfopenllm_v2/MUSR": 0.3394, - "hfopenllm_v2/MMLU-PRO": 0.113 - } - }, - { - "id": "Daemontatox/SphinX", - "name": "SphinX", - "developer": "Daemontatox", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5725, - "hfopenllm_v2/BBH": 0.5441, - "hfopenllm_v2/MATH Level 5": 0.3082, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.4405, - "hfopenllm_v2/MMLU-PRO": 0.4366 - } - }, - { - "id": "Daemontatox/Sphinx2.0", - "name": "Sphinx2.0", - "developer": "Daemontatox", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7123, - "hfopenllm_v2/BBH": 0.6473, - "hfopenllm_v2/MATH Level 5": 0.4018, - "hfopenllm_v2/GPQA": 0.2936, - "hfopenllm_v2/MUSR": 0.426, - "hfopenllm_v2/MMLU-PRO": 0.5184 - } - }, - { - "id": "Daemontatox/TinySphinx", - "name": "TinySphinx", - "developer": "Daemontatox", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2567, - "hfopenllm_v2/BBH": 0.331, - "hfopenllm_v2/MATH Level 5": 0.0431, - "hfopenllm_v2/GPQA": 0.2735, - "hfopenllm_v2/MUSR": 0.3328, - "hfopenllm_v2/MMLU-PRO": 0.1698 - } - }, - { - "id": "Daemontatox/TinySphinx2.0", - "name": "TinySphinx2.0", - "developer": "Daemontatox", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2535, - "hfopenllm_v2/BBH": 0.3168, - "hfopenllm_v2/MATH Level 5": 0.0325, - "hfopenllm_v2/GPQA": 0.2685, - "hfopenllm_v2/MUSR": 0.3382, - "hfopenllm_v2/MMLU-PRO": 0.1731 - } - }, - { - "id": "Daemontatox/Zirel-7B-Math", - "name": "Zirel-7B-Math", - "developer": "Daemontatox", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6639, - "hfopenllm_v2/BBH": 0.5448, - "hfopenllm_v2/MATH Level 5": 0.1979, - "hfopenllm_v2/GPQA": 0.3263, - "hfopenllm_v2/MUSR": 0.4789, - "hfopenllm_v2/MMLU-PRO": 0.4237 - } - }, - { - "id": "Daemontatox/Zirel_1.5", - "name": "Zirel_1.5", - "developer": "Daemontatox", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4168, - "hfopenllm_v2/BBH": 0.3985, - "hfopenllm_v2/MATH Level 5": 0.1133, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3658, - "hfopenllm_v2/MMLU-PRO": 0.2143 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/dampfinchen.json b/data/developers/dampfinchen.json deleted file mode 100644 index 2067a01e4f962631afa82d4370062d20be21ad66..0000000000000000000000000000000000000000 --- a/data/developers/dampfinchen.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "Dampfinchen", - "models": [ - { - "id": "Dampfinchen/Llama-3.1-8B-Ultra-Instruct", - "name": "Llama-3.1-8B-Ultra-Instruct", - "developer": "Dampfinchen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8081, - "hfopenllm_v2/BBH": 0.5258, - "hfopenllm_v2/MATH Level 5": 0.2205, - "hfopenllm_v2/GPQA": 0.2919, - "hfopenllm_v2/MUSR": 0.4003, - "hfopenllm_v2/MMLU-PRO": 0.3826 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/danielbrdz.json b/data/developers/danielbrdz.json deleted file mode 100644 index 4125accfc0491df06739e6b344e2093e59c20ead..0000000000000000000000000000000000000000 --- a/data/developers/danielbrdz.json +++ /dev/null @@ -1,103 +0,0 @@ -{ - "developer": "Danielbrdz", - "models": [ - { - "id": "Danielbrdz/Barcenas-10b", - "name": "Barcenas-10b", - "developer": "Danielbrdz", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6608, - "hfopenllm_v2/BBH": 0.6121, - "hfopenllm_v2/MATH Level 5": 0.2153, - "hfopenllm_v2/GPQA": 0.3414, - "hfopenllm_v2/MUSR": 0.4135, - "hfopenllm_v2/MMLU-PRO": 0.4361 - } - }, - { - "id": "Danielbrdz/Barcenas-14b-Phi-3-medium-ORPO", - "name": "Barcenas-14b-Phi-3-medium-ORPO", - "developer": "Danielbrdz", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4799, - "hfopenllm_v2/BBH": 0.6536, - "hfopenllm_v2/MATH Level 5": 0.2024, - "hfopenllm_v2/GPQA": 0.3263, - "hfopenllm_v2/MUSR": 0.4808, - "hfopenllm_v2/MMLU-PRO": 0.4723 - } - }, - { - "id": "Danielbrdz/Barcenas-14b-phi-4", - "name": "Barcenas-14b-phi-4", - "developer": "Danielbrdz", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0498, - "hfopenllm_v2/BBH": 0.6769, - "hfopenllm_v2/MATH Level 5": 0.2583, - "hfopenllm_v2/GPQA": 0.3834, - "hfopenllm_v2/MUSR": 0.5097, - "hfopenllm_v2/MMLU-PRO": 0.5175 - } - }, - { - "id": "Danielbrdz/Barcenas-14b-phi-4-v2", - "name": "Barcenas-14b-phi-4-v2", - "developer": "Danielbrdz", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2775, - "hfopenllm_v2/BBH": 0.6573, - "hfopenllm_v2/MATH Level 5": 0.3218, - "hfopenllm_v2/GPQA": 0.3784, - "hfopenllm_v2/MUSR": 0.4399, - "hfopenllm_v2/MMLU-PRO": 0.5244 - } - }, - { - "id": "Danielbrdz/Barcenas-3b-GRPO", - "name": "Barcenas-3b-GRPO", - "developer": "Danielbrdz", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5444, - "hfopenllm_v2/BBH": 0.4414, - "hfopenllm_v2/MATH Level 5": 0.1375, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.3576, - "hfopenllm_v2/MMLU-PRO": 0.3037 - } - }, - { - "id": "Danielbrdz/Barcenas-Llama3-8b-ORPO", - "name": "Barcenas-Llama3-8b-ORPO", - "developer": "Danielbrdz", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7372, - "hfopenllm_v2/BBH": 0.4987, - "hfopenllm_v2/MATH Level 5": 0.0657, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.419, - "hfopenllm_v2/MMLU-PRO": 0.383 - } - }, - { - "id": "Danielbrdz/Barcenas-R1-Qwen-1.5b", - "name": "Barcenas-R1-Qwen-1.5b", - "developer": "Danielbrdz", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2428, - "hfopenllm_v2/BBH": 0.3587, - "hfopenllm_v2/MATH Level 5": 0.3497, - "hfopenllm_v2/GPQA": 0.3037, - "hfopenllm_v2/MUSR": 0.3541, - "hfopenllm_v2/MMLU-PRO": 0.1909 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/dans-discountmodels.json b/data/developers/dans-discountmodels.json deleted file mode 100644 index 069699823793b23d817b4470d50a4e0f1321acd7..0000000000000000000000000000000000000000 --- a/data/developers/dans-discountmodels.json +++ /dev/null @@ -1,131 +0,0 @@ -{ - "developer": "Dans-DiscountModels", - "models": [ - { - "id": "Dans-DiscountModels/12b-mn-dans-reasoning-test-2", - "name": "12b-mn-dans-reasoning-test-2", - "developer": "Dans-DiscountModels", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3711, - "hfopenllm_v2/BBH": 0.4807, - "hfopenllm_v2/MATH Level 5": 0.0634, - "hfopenllm_v2/GPQA": 0.2735, - "hfopenllm_v2/MUSR": 0.3702, - "hfopenllm_v2/MMLU-PRO": 0.2507 - } - }, - { - "id": "Dans-DiscountModels/12b-mn-dans-reasoning-test-3", - "name": "12b-mn-dans-reasoning-test-3", - "developer": "Dans-DiscountModels", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5053, - "hfopenllm_v2/BBH": 0.4839, - "hfopenllm_v2/MATH Level 5": 0.0778, - "hfopenllm_v2/GPQA": 0.271, - "hfopenllm_v2/MUSR": 0.4168, - "hfopenllm_v2/MMLU-PRO": 0.2516 - } - }, - { - "id": "Dans-DiscountModels/Dans-Instruct-CoreCurriculum-12b-ChatML", - "name": "Dans-Instruct-CoreCurriculum-12b-ChatML", - "developer": "Dans-DiscountModels", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2111, - "hfopenllm_v2/BBH": 0.4792, - "hfopenllm_v2/MATH Level 5": 0.0431, - "hfopenllm_v2/GPQA": 0.2802, - "hfopenllm_v2/MUSR": 0.3606, - "hfopenllm_v2/MMLU-PRO": 0.2805 - } - }, - { - "id": "Dans-DiscountModels/Dans-Instruct-Mix-8b-ChatML", - "name": "Dans-Instruct-Mix-8b-ChatML", - "developer": "Dans-DiscountModels", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0825, - "hfopenllm_v2/BBH": 0.4738, - "hfopenllm_v2/MATH Level 5": 0.0551, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.3918, - "hfopenllm_v2/MMLU-PRO": 0.3288 - } - }, - { - "id": "Dans-DiscountModels/Dans-Instruct-Mix-8b-ChatML-V0.1.0", - "name": "Dans-Instruct-Mix-8b-ChatML-V0.1.0", - "developer": "Dans-DiscountModels", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0668, - "hfopenllm_v2/BBH": 0.4775, - "hfopenllm_v2/MATH Level 5": 0.0672, - "hfopenllm_v2/GPQA": 0.2861, - "hfopenllm_v2/MUSR": 0.3786, - "hfopenllm_v2/MMLU-PRO": 0.3284 - } - }, - { - "id": "Dans-DiscountModels/Dans-Instruct-Mix-8b-ChatML-V0.1.1", - "name": "Dans-Instruct-Mix-8b-ChatML-V0.1.1", - "developer": "Dans-DiscountModels", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0911, - "hfopenllm_v2/BBH": 0.4749, - "hfopenllm_v2/MATH Level 5": 0.0597, - "hfopenllm_v2/GPQA": 0.2911, - "hfopenllm_v2/MUSR": 0.3825, - "hfopenllm_v2/MMLU-PRO": 0.3279 - } - }, - { - "id": "Dans-DiscountModels/Dans-Instruct-Mix-8b-ChatML-V0.2.0", - "name": "Dans-Instruct-Mix-8b-ChatML-V0.2.0", - "developer": "Dans-DiscountModels", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5064, - "hfopenllm_v2/BBH": 0.4624, - "hfopenllm_v2/MATH Level 5": 0.0733, - "hfopenllm_v2/GPQA": 0.2936, - "hfopenllm_v2/MUSR": 0.3644, - "hfopenllm_v2/MMLU-PRO": 0.3 - } - }, - { - "id": "Dans-DiscountModels/mistral-7b-test-merged", - "name": "mistral-7b-test-merged", - "developer": "Dans-DiscountModels", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6678, - "hfopenllm_v2/BBH": 0.4898, - "hfopenllm_v2/MATH Level 5": 0.0446, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.3754, - "hfopenllm_v2/MMLU-PRO": 0.2978 - } - }, - { - "id": "Dans-DiscountModels/Mistral-7b-v0.3-Test-E0.7", - "name": "Mistral-7b-v0.3-Test-E0.7", - "developer": "Dans-DiscountModels", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5124, - "hfopenllm_v2/BBH": 0.475, - "hfopenllm_v2/MATH Level 5": 0.034, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.4005, - "hfopenllm_v2/MMLU-PRO": 0.2744 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/darkc0de.json b/data/developers/darkc0de.json deleted file mode 100644 index 9235fc136e323bd18cb94a2f6419cd9e31bbf791..0000000000000000000000000000000000000000 --- a/data/developers/darkc0de.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "developer": "darkc0de", - "models": [ - { - "id": "darkc0de/BuddyGlass_v0.3_Xortron7MethedUpSwitchedUp", - "name": "BuddyGlass_v0.3_Xortron7MethedUpSwitchedUp", - "developer": "darkc0de", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4358, - "hfopenllm_v2/BBH": 0.5243, - "hfopenllm_v2/MATH Level 5": 0.1284, - "hfopenllm_v2/GPQA": 0.2987, - "hfopenllm_v2/MUSR": 0.4143, - "hfopenllm_v2/MMLU-PRO": 0.3673 - } - }, - { - "id": "darkc0de/BuddyGlassNeverSleeps", - "name": "BuddyGlassNeverSleeps", - "developer": "darkc0de", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4239, - "hfopenllm_v2/BBH": 0.4977, - "hfopenllm_v2/MATH Level 5": 0.0627, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.3993, - "hfopenllm_v2/MMLU-PRO": 0.3452 - } - }, - { - "id": "darkc0de/BuddyGlassUncensored2025.2", - "name": "BuddyGlassUncensored2025.2", - "developer": "darkc0de", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7731, - "hfopenllm_v2/BBH": 0.6095, - "hfopenllm_v2/MATH Level 5": 0.2402, - "hfopenllm_v2/GPQA": 0.328, - "hfopenllm_v2/MUSR": 0.4071, - "hfopenllm_v2/MMLU-PRO": 0.4336 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/darkknight535.json b/data/developers/darkknight535.json deleted file mode 100644 index 27e2b821467b28cb196806825201f21e08fe0a34..0000000000000000000000000000000000000000 --- a/data/developers/darkknight535.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "Darkknight535", - "models": [ - { - "id": "Darkknight535/OpenCrystal-12B-L3", - "name": "OpenCrystal-12B-L3", - "developer": "Darkknight535", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4071, - "hfopenllm_v2/BBH": 0.5223, - "hfopenllm_v2/MATH Level 5": 0.0899, - "hfopenllm_v2/GPQA": 0.3062, - "hfopenllm_v2/MUSR": 0.3657, - "hfopenllm_v2/MMLU-PRO": 0.364 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/databricks-mosaic-research.json b/data/developers/databricks-mosaic-research.json deleted file mode 100644 index d54480ce8af664b382e1a483ecab0c45fb6bd496..0000000000000000000000000000000000000000 --- a/data/developers/databricks-mosaic-research.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "developer": "Databricks-Mosaic-Research", - "models": [ - { - "id": "Databricks-Mosaic-Research/PGRM", - "name": "Databricks-Mosaic-Research/PGRM", - "developer": "Databricks-Mosaic-Research", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.8002, - "reward-bench/Factuality": 0.7937, - "reward-bench/Precise IF": 0.5062, - "reward-bench/Math": 0.7404, - "reward-bench/Safety": 0.9289, - "reward-bench/Focus": 0.9424, - "reward-bench/Ties": 0.8893 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/databricks.json b/data/developers/databricks.json deleted file mode 100644 index baed75a2fae8ea193778c5b2817219afa46d4f73..0000000000000000000000000000000000000000 --- a/data/developers/databricks.json +++ /dev/null @@ -1,135 +0,0 @@ -{ - "developer": "databricks", - "models": [ - { - "id": "databricks/dbrx-base", - "name": "dbrx-base", - "developer": "databricks", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0821, - "hfopenllm_v2/BBH": 0.5196, - "hfopenllm_v2/MATH Level 5": 0.1, - "hfopenllm_v2/GPQA": 0.3267, - "hfopenllm_v2/MUSR": 0.4067, - "hfopenllm_v2/MMLU-PRO": 0.35 - } - }, - { - "id": "databricks/dbrx-instruct", - "name": "DBRX Instruct", - "developer": "databricks", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.289, - "helm_lite/NarrativeQA": 0.488, - "helm_lite/NaturalQuestions (closed-book)": 0.284, - "helm_lite/OpenbookQA": 0.91, - "helm_lite/MMLU": 0.643, - "helm_lite/MATH": 0.358, - "helm_lite/GSM8K": 0.671, - "helm_lite/LegalBench": 0.426, - "helm_lite/MedQA": 0.694, - "helm_lite/WMT 2014": 0.131, - "helm_mmlu/MMLU All Subjects": 0.741, - "helm_mmlu/Abstract Algebra": 0.34, - "helm_mmlu/Anatomy": 0.667, - "helm_mmlu/College Physics": 0.539, - "helm_mmlu/Computer Security": 0.83, - "helm_mmlu/Econometrics": 0.605, - "helm_mmlu/Global Facts": 0.46, - "helm_mmlu/Jurisprudence": 0.843, - "helm_mmlu/Philosophy": 0.804, - "helm_mmlu/Professional Psychology": 0.801, - "helm_mmlu/Us Foreign Policy": 0.93, - "helm_mmlu/Astronomy": 0.836, - "helm_mmlu/Business Ethics": 0.78, - "helm_mmlu/Clinical Knowledge": 0.789, - "helm_mmlu/Conceptual Physics": 0.74, - "helm_mmlu/Electrical Engineering": 0.71, - "helm_mmlu/Elementary Mathematics": 0.563, - "helm_mmlu/Formal Logic": 0.563, - "helm_mmlu/High School World History": 0.903, - "helm_mmlu/Human Sexuality": 0.878, - "helm_mmlu/International Law": 0.884, - "helm_mmlu/Logical Fallacies": 0.847, - "helm_mmlu/Machine Learning": 0.625, - "helm_mmlu/Management": 0.854, - "helm_mmlu/Marketing": 0.94, - "helm_mmlu/Medical Genetics": 0.85, - "helm_mmlu/Miscellaneous": 0.911, - "helm_mmlu/Moral Scenarios": 0.465, - "helm_mmlu/Nutrition": 0.814, - "helm_mmlu/Prehistory": 0.84, - "helm_mmlu/Public Relations": 0.691, - "helm_mmlu/Security Studies": 0.804, - "helm_mmlu/Sociology": 0.896, - "helm_mmlu/Virology": 0.566, - "helm_mmlu/World Religions": 0.871, - "helm_mmlu/Mean win rate": 0.537, - "hfopenllm_v2/IFEval": 0.5416, - "hfopenllm_v2/BBH": 0.5429, - "hfopenllm_v2/MATH Level 5": 0.0687, - "hfopenllm_v2/GPQA": 0.3414, - "hfopenllm_v2/MUSR": 0.4269, - "hfopenllm_v2/MMLU-PRO": 0.3683 - } - }, - { - "id": "databricks/dolly-v1-6b", - "name": "dolly-v1-6b", - "developer": "databricks", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2224, - "hfopenllm_v2/BBH": 0.3172, - "hfopenllm_v2/MATH Level 5": 0.0189, - "hfopenllm_v2/GPQA": 0.2643, - "hfopenllm_v2/MUSR": 0.4004, - "hfopenllm_v2/MMLU-PRO": 0.1266 - } - }, - { - "id": "databricks/dolly-v2-12b", - "name": "dolly-v2-12b", - "developer": "databricks", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2355, - "hfopenllm_v2/BBH": 0.332, - "hfopenllm_v2/MATH Level 5": 0.0136, - "hfopenllm_v2/GPQA": 0.2408, - "hfopenllm_v2/MUSR": 0.3739, - "hfopenllm_v2/MMLU-PRO": 0.1129 - } - }, - { - "id": "databricks/dolly-v2-3b", - "name": "dolly-v2-3b", - "developer": "databricks", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2247, - "hfopenllm_v2/BBH": 0.3079, - "hfopenllm_v2/MATH Level 5": 0.0151, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.3338, - "hfopenllm_v2/MMLU-PRO": 0.1145 - } - }, - { - "id": "databricks/dolly-v2-7b", - "name": "dolly-v2-7b", - "developer": "databricks", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.201, - "hfopenllm_v2/BBH": 0.3173, - "hfopenllm_v2/MATH Level 5": 0.0144, - "hfopenllm_v2/GPQA": 0.2685, - "hfopenllm_v2/MUSR": 0.3553, - "hfopenllm_v2/MMLU-PRO": 0.1149 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/davidau.json b/data/developers/davidau.json deleted file mode 100644 index eb025e176f06cf6e9f8e741f432299db8b01c688..0000000000000000000000000000000000000000 --- a/data/developers/davidau.json +++ /dev/null @@ -1,355 +0,0 @@ -{ - "developer": "DavidAU", - "models": [ - { - "id": "DavidAU/DeepHermes-3-Llama-3-8B-Preview-16.5B-Brainstorm", - "name": "DeepHermes-3-Llama-3-8B-Preview-16.5B-Brainstorm", - "developer": "DavidAU", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3136, - "hfopenllm_v2/BBH": 0.4762, - "hfopenllm_v2/MATH Level 5": 0.1057, - "hfopenllm_v2/GPQA": 0.3138, - "hfopenllm_v2/MUSR": 0.3928, - "hfopenllm_v2/MMLU-PRO": 0.3209 - } - }, - { - "id": "DavidAU/DeepSeek-BlackRoot-R1-Distill-Llama-3.1-8B", - "name": "DeepSeek-BlackRoot-R1-Distill-Llama-3.1-8B", - "developer": "DavidAU", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3685, - "hfopenllm_v2/BBH": 0.4887, - "hfopenllm_v2/MATH Level 5": 0.0657, - "hfopenllm_v2/GPQA": 0.318, - "hfopenllm_v2/MUSR": 0.432, - "hfopenllm_v2/MMLU-PRO": 0.2976 - } - }, - { - "id": "DavidAU/DeepSeek-Grand-Horror-SMB-R1-Distill-Llama-3.1-16B", - "name": "DeepSeek-Grand-Horror-SMB-R1-Distill-Llama-3.1-16B", - "developer": "DavidAU", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2507, - "hfopenllm_v2/BBH": 0.4488, - "hfopenllm_v2/MATH Level 5": 0.0295, - "hfopenllm_v2/GPQA": 0.3138, - "hfopenllm_v2/MUSR": 0.4164, - "hfopenllm_v2/MMLU-PRO": 0.2709 - } - }, - { - "id": "DavidAU/DeepSeek-MOE-4X8B-R1-Distill-Llama-3.1-Deep-Thinker-Uncensored-24B", - "name": "DeepSeek-MOE-4X8B-R1-Distill-Llama-3.1-Deep-Thinker-Uncensored-24B", - "developer": "DavidAU", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3883, - "hfopenllm_v2/BBH": 0.4886, - "hfopenllm_v2/MATH Level 5": 0.0816, - "hfopenllm_v2/GPQA": 0.323, - "hfopenllm_v2/MUSR": 0.4375, - "hfopenllm_v2/MMLU-PRO": 0.3024 - } - }, - { - "id": "DavidAU/DeepSeek-MOE-4X8B-R1-Distill-Llama-3.1-Mad-Scientist-24B", - "name": "DeepSeek-MOE-4X8B-R1-Distill-Llama-3.1-Mad-Scientist-24B", - "developer": "DavidAU", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3436, - "hfopenllm_v2/BBH": 0.4769, - "hfopenllm_v2/MATH Level 5": 0.0755, - "hfopenllm_v2/GPQA": 0.3372, - "hfopenllm_v2/MUSR": 0.4231, - "hfopenllm_v2/MMLU-PRO": 0.297 - } - }, - { - "id": "DavidAU/DeepSeek-R1-Distill-Qwen-25.5B-Brainstorm", - "name": "DeepSeek-R1-Distill-Qwen-25.5B-Brainstorm", - "developer": "DavidAU", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3416, - "hfopenllm_v2/BBH": 0.5807, - "hfopenllm_v2/MATH Level 5": 0.5536, - "hfopenllm_v2/GPQA": 0.3859, - "hfopenllm_v2/MUSR": 0.5155, - "hfopenllm_v2/MMLU-PRO": 0.4624 - } - }, - { - "id": "DavidAU/DeepSeek-V2-Grand-Horror-SMB-R1-Distill-Llama-3.1-Uncensored-16.5B", - "name": "DeepSeek-V2-Grand-Horror-SMB-R1-Distill-Llama-3.1-Uncensored-16.5B", - "developer": "DavidAU", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2853, - "hfopenllm_v2/BBH": 0.4462, - "hfopenllm_v2/MATH Level 5": 0.0174, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.4179, - "hfopenllm_v2/MMLU-PRO": 0.2778 - } - }, - { - "id": "DavidAU/DeepThought-MOE-8X3B-R1-Llama-3.2-Reasoning-18B", - "name": "DeepThought-MOE-8X3B-R1-Llama-3.2-Reasoning-18B", - "developer": "DavidAU", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3793, - "hfopenllm_v2/BBH": 0.4232, - "hfopenllm_v2/MATH Level 5": 0.108, - "hfopenllm_v2/GPQA": 0.2794, - "hfopenllm_v2/MUSR": 0.356, - "hfopenllm_v2/MMLU-PRO": 0.272 - } - }, - { - "id": "DavidAU/Gemma-The-Writer-9B", - "name": "Gemma-The-Writer-9B", - "developer": "DavidAU", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.174, - "hfopenllm_v2/BBH": 0.5905, - "hfopenllm_v2/MATH Level 5": 0.0876, - "hfopenllm_v2/GPQA": 0.3456, - "hfopenllm_v2/MUSR": 0.4099, - "hfopenllm_v2/MMLU-PRO": 0.3979 - } - }, - { - "id": "DavidAU/Gemma-The-Writer-DEADLINE-10B", - "name": "Gemma-The-Writer-DEADLINE-10B", - "developer": "DavidAU", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2332, - "hfopenllm_v2/BBH": 0.5896, - "hfopenllm_v2/MATH Level 5": 0.0989, - "hfopenllm_v2/GPQA": 0.3423, - "hfopenllm_v2/MUSR": 0.4189, - "hfopenllm_v2/MMLU-PRO": 0.3946 - } - }, - { - "id": "DavidAU/Gemma-The-Writer-J.GutenBerg-10B", - "name": "Gemma-The-Writer-J.GutenBerg-10B", - "developer": "DavidAU", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2858, - "hfopenllm_v2/BBH": 0.5909, - "hfopenllm_v2/MATH Level 5": 0.0921, - "hfopenllm_v2/GPQA": 0.3381, - "hfopenllm_v2/MUSR": 0.4176, - "hfopenllm_v2/MMLU-PRO": 0.3947 - } - }, - { - "id": "DavidAU/Gemma-The-Writer-Mighty-Sword-9B", - "name": "Gemma-The-Writer-Mighty-Sword-9B", - "developer": "DavidAU", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7528, - "hfopenllm_v2/BBH": 0.5912, - "hfopenllm_v2/MATH Level 5": 0.1911, - "hfopenllm_v2/GPQA": 0.3482, - "hfopenllm_v2/MUSR": 0.4112, - "hfopenllm_v2/MMLU-PRO": 0.3968 - } - }, - { - "id": "DavidAU/Gemma-The-Writer-N-Restless-Quill-10B-Uncensored", - "name": "Gemma-The-Writer-N-Restless-Quill-10B-Uncensored", - "developer": "DavidAU", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7071, - "hfopenllm_v2/BBH": 0.5922, - "hfopenllm_v2/MATH Level 5": 0.2296, - "hfopenllm_v2/GPQA": 0.3414, - "hfopenllm_v2/MUSR": 0.4163, - "hfopenllm_v2/MMLU-PRO": 0.3966 - } - }, - { - "id": "DavidAU/L3-Dark-Planet-8B", - "name": "L3-Dark-Planet-8B", - "developer": "DavidAU", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4134, - "hfopenllm_v2/BBH": 0.5084, - "hfopenllm_v2/MATH Level 5": 0.0823, - "hfopenllm_v2/GPQA": 0.3003, - "hfopenllm_v2/MUSR": 0.3616, - "hfopenllm_v2/MMLU-PRO": 0.3737 - } - }, - { - "id": "DavidAU/L3-DARKEST-PLANET-16.5B", - "name": "L3-DARKEST-PLANET-16.5B", - "developer": "DavidAU", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6231, - "hfopenllm_v2/BBH": 0.523, - "hfopenllm_v2/MATH Level 5": 0.0899, - "hfopenllm_v2/GPQA": 0.2953, - "hfopenllm_v2/MUSR": 0.3754, - "hfopenllm_v2/MMLU-PRO": 0.363 - } - }, - { - "id": "DavidAU/L3-Jamet-12.2B-MK.V-Blackroot-Instruct", - "name": "L3-Jamet-12.2B-MK.V-Blackroot-Instruct", - "developer": "DavidAU", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3962, - "hfopenllm_v2/BBH": 0.4766, - "hfopenllm_v2/MATH Level 5": 0.0408, - "hfopenllm_v2/GPQA": 0.2785, - "hfopenllm_v2/MUSR": 0.402, - "hfopenllm_v2/MMLU-PRO": 0.3291 - } - }, - { - "id": "DavidAU/L3-Lumimaid-12.2B-v0.1-OAS-Instruct", - "name": "L3-Lumimaid-12.2B-v0.1-OAS-Instruct", - "developer": "DavidAU", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3924, - "hfopenllm_v2/BBH": 0.4693, - "hfopenllm_v2/MATH Level 5": 0.0461, - "hfopenllm_v2/GPQA": 0.2768, - "hfopenllm_v2/MUSR": 0.4194, - "hfopenllm_v2/MMLU-PRO": 0.3142 - } - }, - { - "id": "DavidAU/L3-SMB-Instruct-12.2B-F32", - "name": "L3-SMB-Instruct-12.2B-F32", - "developer": "DavidAU", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4303, - "hfopenllm_v2/BBH": 0.4786, - "hfopenllm_v2/MATH Level 5": 0.0468, - "hfopenllm_v2/GPQA": 0.2819, - "hfopenllm_v2/MUSR": 0.4087, - "hfopenllm_v2/MMLU-PRO": 0.3312 - } - }, - { - "id": "DavidAU/L3-Stheno-Maid-Blackroot-Grand-HORROR-16B", - "name": "L3-Stheno-Maid-Blackroot-Grand-HORROR-16B", - "developer": "DavidAU", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3439, - "hfopenllm_v2/BBH": 0.4736, - "hfopenllm_v2/MATH Level 5": 0.0219, - "hfopenllm_v2/GPQA": 0.271, - "hfopenllm_v2/MUSR": 0.4031, - "hfopenllm_v2/MMLU-PRO": 0.357 - } - }, - { - "id": "DavidAU/L3-Stheno-v3.2-12.2B-Instruct", - "name": "L3-Stheno-v3.2-12.2B-Instruct", - "developer": "DavidAU", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4028, - "hfopenllm_v2/BBH": 0.4846, - "hfopenllm_v2/MATH Level 5": 0.0506, - "hfopenllm_v2/GPQA": 0.2752, - "hfopenllm_v2/MUSR": 0.4103, - "hfopenllm_v2/MMLU-PRO": 0.3345 - } - }, - { - "id": "DavidAU/L3.1-Dark-Planet-SpinFire-Uncensored-8B", - "name": "L3.1-Dark-Planet-SpinFire-Uncensored-8B", - "developer": "DavidAU", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7043, - "hfopenllm_v2/BBH": 0.5261, - "hfopenllm_v2/MATH Level 5": 0.0929, - "hfopenllm_v2/GPQA": 0.2794, - "hfopenllm_v2/MUSR": 0.3541, - "hfopenllm_v2/MMLU-PRO": 0.367 - } - }, - { - "id": "DavidAU/L3.1-MOE-2X8B-Deepseek-DeepHermes-e32-uncensored-abliterated-13.7B", - "name": "L3.1-MOE-2X8B-Deepseek-DeepHermes-e32-uncensored-abliterated-13.7B", - "developer": "DavidAU", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3345, - "hfopenllm_v2/BBH": 0.4421, - "hfopenllm_v2/MATH Level 5": 0.2606, - "hfopenllm_v2/GPQA": 0.3138, - "hfopenllm_v2/MUSR": 0.3749, - "hfopenllm_v2/MMLU-PRO": 0.2892 - } - }, - { - "id": "DavidAU/Qwen2.5-MOE-2X1.5B-DeepSeek-Uncensored-Censored-4B", - "name": "Qwen2.5-MOE-2X1.5B-DeepSeek-Uncensored-Censored-4B", - "developer": "DavidAU", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1783, - "hfopenllm_v2/BBH": 0.3033, - "hfopenllm_v2/MATH Level 5": 0.0249, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3715, - "hfopenllm_v2/MMLU-PRO": 0.1142 - } - }, - { - "id": "DavidAU/Qwen2.5-MOE-2X7B-DeepSeek-Abliterated-Censored-19B", - "name": "Qwen2.5-MOE-2X7B-DeepSeek-Abliterated-Censored-19B", - "developer": "DavidAU", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2835, - "hfopenllm_v2/BBH": 0.3592, - "hfopenllm_v2/MATH Level 5": 0.2417, - "hfopenllm_v2/GPQA": 0.2651, - "hfopenllm_v2/MUSR": 0.3847, - "hfopenllm_v2/MMLU-PRO": 0.1636 - } - }, - { - "id": "DavidAU/Qwen2.5-MOE-6x1.5B-DeepSeek-Reasoning-e32", - "name": "Qwen2.5-MOE-6x1.5B-DeepSeek-Reasoning-e32", - "developer": "DavidAU", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2107, - "hfopenllm_v2/BBH": 0.3286, - "hfopenllm_v2/MATH Level 5": 0.0665, - "hfopenllm_v2/GPQA": 0.2475, - "hfopenllm_v2/MUSR": 0.3404, - "hfopenllm_v2/MMLU-PRO": 0.1122 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/davidkim205.json b/data/developers/davidkim205.json deleted file mode 100644 index 09f9fb3f5ee2e593a053521a32e4ee348caa569e..0000000000000000000000000000000000000000 --- a/data/developers/davidkim205.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "davidkim205", - "models": [ - { - "id": "davidkim205/nox-solar-10.7b-v4", - "name": "nox-solar-10.7b-v4", - "developer": "davidkim205", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3753, - "hfopenllm_v2/BBH": 0.4814, - "hfopenllm_v2/MATH Level 5": 0.0083, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.4298, - "hfopenllm_v2/MMLU-PRO": 0.3333 - } - }, - { - "id": "davidkim205/Rhea-72b-v0.5", - "name": "Rhea-72b-v0.5", - "developer": "davidkim205", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0145, - "hfopenllm_v2/BBH": 0.3078, - "hfopenllm_v2/MATH Level 5": 0.1737, - "hfopenllm_v2/GPQA": 0.2525, - "hfopenllm_v2/MUSR": 0.4241, - "hfopenllm_v2/MMLU-PRO": 0.1166 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/davidsv.json b/data/developers/davidsv.json deleted file mode 100644 index 681040458292ee5182c8bc14f1b8cdb7d2059796..0000000000000000000000000000000000000000 --- a/data/developers/davidsv.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "Davidsv", - "models": [ - { - "id": "Davidsv/SUONG-1", - "name": "SUONG-1", - "developer": "Davidsv", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2497, - "hfopenllm_v2/BBH": 0.2817, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2441, - "hfopenllm_v2/MUSR": 0.3578, - "hfopenllm_v2/MMLU-PRO": 0.1085 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/davielion.json b/data/developers/davielion.json deleted file mode 100644 index ffdc7de10295de8981ccb0c2da137caa37979e5b..0000000000000000000000000000000000000000 --- a/data/developers/davielion.json +++ /dev/null @@ -1,75 +0,0 @@ -{ - "developer": "DavieLion", - "models": [ - { - "id": "DavieLion/Llama-3.2-1B-SPIN-iter0", - "name": "Llama-3.2-1B-SPIN-iter0", - "developer": "DavieLion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1507, - "hfopenllm_v2/BBH": 0.293, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2534, - "hfopenllm_v2/MUSR": 0.3565, - "hfopenllm_v2/MMLU-PRO": 0.1125 - } - }, - { - "id": "DavieLion/Llama-3.2-1B-SPIN-iter1", - "name": "Llama-3.2-1B-SPIN-iter1", - "developer": "DavieLion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1575, - "hfopenllm_v2/BBH": 0.294, - "hfopenllm_v2/MATH Level 5": 0.0023, - "hfopenllm_v2/GPQA": 0.2508, - "hfopenllm_v2/MUSR": 0.3646, - "hfopenllm_v2/MMLU-PRO": 0.1118 - } - }, - { - "id": "DavieLion/Llama-3.2-1B-SPIN-iter2", - "name": "Llama-3.2-1B-SPIN-iter2", - "developer": "DavieLion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1376, - "hfopenllm_v2/BBH": 0.298, - "hfopenllm_v2/MATH Level 5": 0.0053, - "hfopenllm_v2/GPQA": 0.2542, - "hfopenllm_v2/MUSR": 0.3553, - "hfopenllm_v2/MMLU-PRO": 0.1129 - } - }, - { - "id": "DavieLion/Llama-3.2-1B-SPIN-iter3", - "name": "Llama-3.2-1B-SPIN-iter3", - "developer": "DavieLion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1336, - "hfopenllm_v2/BBH": 0.2975, - "hfopenllm_v2/MATH Level 5": 0.0068, - "hfopenllm_v2/GPQA": 0.2534, - "hfopenllm_v2/MUSR": 0.35, - "hfopenllm_v2/MMLU-PRO": 0.1128 - } - }, - { - "id": "DavieLion/Lllma-3.2-1B", - "name": "Lllma-3.2-1B", - "developer": "DavieLion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1601, - "hfopenllm_v2/BBH": 0.2965, - "hfopenllm_v2/MATH Level 5": 0.0068, - "hfopenllm_v2/GPQA": 0.2441, - "hfopenllm_v2/MUSR": 0.3578, - "hfopenllm_v2/MMLU-PRO": 0.1126 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/debatelabkit.json b/data/developers/debatelabkit.json deleted file mode 100644 index 43228abf3989aca152b661c107d0571e3bdf50d9..0000000000000000000000000000000000000000 --- a/data/developers/debatelabkit.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "DebateLabKIT", - "models": [ - { - "id": "DebateLabKIT/Llama-3.1-Argunaut-1-8B-SFT", - "name": "Llama-3.1-Argunaut-1-8B-SFT", - "developer": "DebateLabKIT", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5519, - "hfopenllm_v2/BBH": 0.4824, - "hfopenllm_v2/MATH Level 5": 0.145, - "hfopenllm_v2/GPQA": 0.2836, - "hfopenllm_v2/MUSR": 0.4503, - "hfopenllm_v2/MMLU-PRO": 0.3472 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/deci.json b/data/developers/deci.json deleted file mode 100644 index 83da7bdeea3c354a4ff914e45297da9b4b117edb..0000000000000000000000000000000000000000 --- a/data/developers/deci.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "Deci", - "models": [ - { - "id": "Deci/DeciLM-7B", - "name": "DeciLM-7B", - "developer": "Deci", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2813, - "hfopenllm_v2/BBH": 0.4423, - "hfopenllm_v2/MATH Level 5": 0.0287, - "hfopenllm_v2/GPQA": 0.2953, - "hfopenllm_v2/MUSR": 0.4359, - "hfopenllm_v2/MMLU-PRO": 0.2692 - } - }, - { - "id": "Deci/DeciLM-7B-instruct", - "name": "DeciLM-7B-instruct", - "developer": "Deci", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.488, - "hfopenllm_v2/BBH": 0.459, - "hfopenllm_v2/MATH Level 5": 0.0302, - "hfopenllm_v2/GPQA": 0.2894, - "hfopenllm_v2/MUSR": 0.3884, - "hfopenllm_v2/MMLU-PRO": 0.2608 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/deepautoai.json b/data/developers/deepautoai.json deleted file mode 100644 index 57f195bee657d891faa1b91d64172e2bc51c5ecc..0000000000000000000000000000000000000000 --- a/data/developers/deepautoai.json +++ /dev/null @@ -1,173 +0,0 @@ -{ - "developer": "DeepAutoAI", - "models": [ - { - "id": "DeepAutoAI/causal_gpt2", - "name": "causal_gpt2", - "developer": "DeepAutoAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1813, - "hfopenllm_v2/BBH": 0.3026, - "hfopenllm_v2/MATH Level 5": 0.0053, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.427, - "hfopenllm_v2/MMLU-PRO": 0.1131 - } - }, - { - "id": "DeepAutoAI/d2nwg_causal_gpt2", - "name": "d2nwg_causal_gpt2", - "developer": "DeepAutoAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1916, - "hfopenllm_v2/BBH": 0.3027, - "hfopenllm_v2/MATH Level 5": 0.0045, - "hfopenllm_v2/GPQA": 0.2576, - "hfopenllm_v2/MUSR": 0.4297, - "hfopenllm_v2/MMLU-PRO": 0.1151 - } - }, - { - "id": "DeepAutoAI/d2nwg_causal_gpt2_v1", - "name": "d2nwg_causal_gpt2_v1", - "developer": "DeepAutoAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1989, - "hfopenllm_v2/BBH": 0.2992, - "hfopenllm_v2/MATH Level 5": 0.0038, - "hfopenllm_v2/GPQA": 0.2584, - "hfopenllm_v2/MUSR": 0.4337, - "hfopenllm_v2/MMLU-PRO": 0.1135 - } - }, - { - "id": "DeepAutoAI/d2nwg_Llama-3.1-8B-Instruct-v0.0", - "name": "d2nwg_Llama-3.1-8B-Instruct-v0.0", - "developer": "DeepAutoAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7893, - "hfopenllm_v2/BBH": 0.508, - "hfopenllm_v2/MATH Level 5": 0.1805, - "hfopenllm_v2/GPQA": 0.2919, - "hfopenllm_v2/MUSR": 0.4135, - "hfopenllm_v2/MMLU-PRO": 0.3877 - } - }, - { - "id": "DeepAutoAI/Explore_Llama-3.1-8B-Inst", - "name": "Explore_Llama-3.1-8B-Inst", - "developer": "DeepAutoAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7795, - "hfopenllm_v2/BBH": 0.5117, - "hfopenllm_v2/MATH Level 5": 0.2009, - "hfopenllm_v2/GPQA": 0.2836, - "hfopenllm_v2/MUSR": 0.391, - "hfopenllm_v2/MMLU-PRO": 0.3792 - } - }, - { - "id": "DeepAutoAI/Explore_Llama-3.2-1B-Inst", - "name": "Explore_Llama-3.2-1B-Inst", - "developer": "DeepAutoAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5649, - "hfopenllm_v2/BBH": 0.3505, - "hfopenllm_v2/MATH Level 5": 0.0748, - "hfopenllm_v2/GPQA": 0.2559, - "hfopenllm_v2/MUSR": 0.3183, - "hfopenllm_v2/MMLU-PRO": 0.1809 - } - }, - { - "id": "DeepAutoAI/Explore_Llama-3.2-1B-Inst_v0", - "name": "Explore_Llama-3.2-1B-Inst_v0", - "developer": "DeepAutoAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5597, - "hfopenllm_v2/BBH": 0.3365, - "hfopenllm_v2/MATH Level 5": 0.0597, - "hfopenllm_v2/GPQA": 0.2634, - "hfopenllm_v2/MUSR": 0.3103, - "hfopenllm_v2/MMLU-PRO": 0.1804 - } - }, - { - "id": "DeepAutoAI/Explore_Llama-3.2-1B-Inst_v1", - "name": "Explore_Llama-3.2-1B-Inst_v1", - "developer": "DeepAutoAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4999, - "hfopenllm_v2/BBH": 0.3141, - "hfopenllm_v2/MATH Level 5": 0.031, - "hfopenllm_v2/GPQA": 0.245, - "hfopenllm_v2/MUSR": 0.3781, - "hfopenllm_v2/MMLU-PRO": 0.1269 - } - }, - { - "id": "DeepAutoAI/Explore_Llama-3.2-1B-Inst_v1.1", - "name": "Explore_Llama-3.2-1B-Inst_v1.1", - "developer": "DeepAutoAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5844, - "hfopenllm_v2/BBH": 0.3513, - "hfopenllm_v2/MATH Level 5": 0.0718, - "hfopenllm_v2/GPQA": 0.2626, - "hfopenllm_v2/MUSR": 0.3117, - "hfopenllm_v2/MMLU-PRO": 0.1818 - } - }, - { - "id": "DeepAutoAI/ldm_soup_Llama-3.1-8B-Inst", - "name": "ldm_soup_Llama-3.1-8B-Inst", - "developer": "DeepAutoAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8033, - "hfopenllm_v2/BBH": 0.5121, - "hfopenllm_v2/MATH Level 5": 0.1888, - "hfopenllm_v2/GPQA": 0.2894, - "hfopenllm_v2/MUSR": 0.4161, - "hfopenllm_v2/MMLU-PRO": 0.3886 - } - }, - { - "id": "DeepAutoAI/ldm_soup_Llama-3.1-8B-Instruct-v0.0", - "name": "ldm_soup_Llama-3.1-8B-Instruct-v0.0", - "developer": "DeepAutoAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7889, - "hfopenllm_v2/BBH": 0.5125, - "hfopenllm_v2/MATH Level 5": 0.1918, - "hfopenllm_v2/GPQA": 0.2911, - "hfopenllm_v2/MUSR": 0.4121, - "hfopenllm_v2/MMLU-PRO": 0.3895 - } - }, - { - "id": "DeepAutoAI/ldm_soup_Llama-3.1-8B-Instruct-v0.1", - "name": "ldm_soup_Llama-3.1-8B-Instruct-v0.1", - "developer": "DeepAutoAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7889, - "hfopenllm_v2/BBH": 0.5125, - "hfopenllm_v2/MATH Level 5": 0.1918, - "hfopenllm_v2/GPQA": 0.2911, - "hfopenllm_v2/MUSR": 0.4121, - "hfopenllm_v2/MMLU-PRO": 0.3895 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/deepmount00.json b/data/developers/deepmount00.json deleted file mode 100644 index e898074e4a61782e05002f7b47eb2ee0411313aa..0000000000000000000000000000000000000000 --- a/data/developers/deepmount00.json +++ /dev/null @@ -1,187 +0,0 @@ -{ - "developer": "DeepMount00", - "models": [ - { - "id": "DeepMount00/Lexora-Lite-3B", - "name": "Lexora-Lite-3B", - "developer": "DeepMount00", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5776, - "hfopenllm_v2/BBH": 0.4873, - "hfopenllm_v2/MATH Level 5": 0.2304, - "hfopenllm_v2/GPQA": 0.2743, - "hfopenllm_v2/MUSR": 0.3966, - "hfopenllm_v2/MMLU-PRO": 0.3602 - } - }, - { - "id": "DeepMount00/Lexora-Lite-3B_v2", - "name": "Lexora-Lite-3B_v2", - "developer": "DeepMount00", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4943, - "hfopenllm_v2/BBH": 0.4812, - "hfopenllm_v2/MATH Level 5": 0.2281, - "hfopenllm_v2/GPQA": 0.271, - "hfopenllm_v2/MUSR": 0.3822, - "hfopenllm_v2/MMLU-PRO": 0.3544 - } - }, - { - "id": "DeepMount00/Lexora-Medium-7B", - "name": "Lexora-Medium-7B", - "developer": "DeepMount00", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4103, - "hfopenllm_v2/BBH": 0.5145, - "hfopenllm_v2/MATH Level 5": 0.2221, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.4439, - "hfopenllm_v2/MMLU-PRO": 0.4325 - } - }, - { - "id": "DeepMount00/Llama-3-8b-Ita", - "name": "Llama-3-8b-Ita", - "developer": "DeepMount00", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.753, - "hfopenllm_v2/BBH": 0.4936, - "hfopenllm_v2/MATH Level 5": 0.0665, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.4268, - "hfopenllm_v2/MMLU-PRO": 0.3852 - } - }, - { - "id": "DeepMount00/Llama-3.1-8b-ITA", - "name": "Llama-3.1-8b-ITA", - "developer": "DeepMount00", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5365, - "hfopenllm_v2/BBH": 0.517, - "hfopenllm_v2/MATH Level 5": 0.1707, - "hfopenllm_v2/GPQA": 0.3062, - "hfopenllm_v2/MUSR": 0.4487, - "hfopenllm_v2/MMLU-PRO": 0.396 - } - }, - { - "id": "DeepMount00/Llama-3.1-Distilled", - "name": "Llama-3.1-Distilled", - "developer": "DeepMount00", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7844, - "hfopenllm_v2/BBH": 0.5101, - "hfopenllm_v2/MATH Level 5": 0.2032, - "hfopenllm_v2/GPQA": 0.3037, - "hfopenllm_v2/MUSR": 0.4058, - "hfopenllm_v2/MMLU-PRO": 0.3782 - } - }, - { - "id": "DeepMount00/mergekit-ties-okvgjfz", - "name": "mergekit-ties-okvgjfz", - "developer": "DeepMount00", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.153, - "hfopenllm_v2/BBH": 0.2998, - "hfopenllm_v2/MATH Level 5": 0.0008, - "hfopenllm_v2/GPQA": 0.2626, - "hfopenllm_v2/MUSR": 0.3806, - "hfopenllm_v2/MMLU-PRO": 0.1118 - } - }, - { - "id": "DeepMount00/Qwen2-1.5B-Ita", - "name": "Qwen2-1.5B-Ita", - "developer": "DeepMount00", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5173, - "hfopenllm_v2/BBH": 0.3981, - "hfopenllm_v2/MATH Level 5": 0.114, - "hfopenllm_v2/GPQA": 0.2626, - "hfopenllm_v2/MUSR": 0.3504, - "hfopenllm_v2/MMLU-PRO": 0.2772 - } - }, - { - "id": "DeepMount00/Qwen2-1.5B-Ita_v2", - "name": "Qwen2-1.5B-Ita_v2", - "developer": "DeepMount00", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5, - "hfopenllm_v2/BBH": 0.3954, - "hfopenllm_v2/MATH Level 5": 0.0967, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3702, - "hfopenllm_v2/MMLU-PRO": 0.3032 - } - }, - { - "id": "DeepMount00/Qwen2-1.5B-Ita_v3", - "name": "Qwen2-1.5B-Ita_v3", - "developer": "DeepMount00", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.489, - "hfopenllm_v2/BBH": 0.3948, - "hfopenllm_v2/MATH Level 5": 0.1042, - "hfopenllm_v2/GPQA": 0.2534, - "hfopenllm_v2/MUSR": 0.3742, - "hfopenllm_v2/MMLU-PRO": 0.3018 - } - }, - { - "id": "DeepMount00/Qwen2-1.5B-Ita_v5", - "name": "Qwen2-1.5B-Ita_v5", - "developer": "DeepMount00", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4987, - "hfopenllm_v2/BBH": 0.4032, - "hfopenllm_v2/MATH Level 5": 0.1178, - "hfopenllm_v2/GPQA": 0.2542, - "hfopenllm_v2/MUSR": 0.3422, - "hfopenllm_v2/MMLU-PRO": 0.2943 - } - }, - { - "id": "DeepMount00/Qwen2-1.5B-Ita_v6", - "name": "Qwen2-1.5B-Ita_v6", - "developer": "DeepMount00", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2999, - "hfopenllm_v2/BBH": 0.4249, - "hfopenllm_v2/MATH Level 5": 0.0846, - "hfopenllm_v2/GPQA": 0.2827, - "hfopenllm_v2/MUSR": 0.3755, - "hfopenllm_v2/MMLU-PRO": 0.2872 - } - }, - { - "id": "DeepMount00/Qwen2.5-7B-Instruct-MathCoder", - "name": "Qwen2.5-7B-Instruct-MathCoder", - "developer": "DeepMount00", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.153, - "hfopenllm_v2/BBH": 0.2998, - "hfopenllm_v2/MATH Level 5": 0.0008, - "hfopenllm_v2/GPQA": 0.2626, - "hfopenllm_v2/MUSR": 0.3806, - "hfopenllm_v2/MMLU-PRO": 0.1118 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/deepseek-ai.json b/data/developers/deepseek-ai.json deleted file mode 100644 index 971bec3ded855a3b01d9662377922d63d5b600e8..0000000000000000000000000000000000000000 --- a/data/developers/deepseek-ai.json +++ /dev/null @@ -1,279 +0,0 @@ -{ - "developer": "deepseek-ai", - "models": [ - { - "id": "deepseek-ai/deepseek-llm-67b-chat", - "name": "DeepSeek LLM Chat 67B", - "developer": "deepseek-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.488, - "helm_lite/NarrativeQA": 0.581, - "helm_lite/NaturalQuestions (closed-book)": 0.412, - "helm_lite/OpenbookQA": 0.88, - "helm_lite/MMLU": 0.641, - "helm_lite/MATH": 0.615, - "helm_lite/GSM8K": 0.795, - "helm_lite/LegalBench": 0.637, - "helm_lite/MedQA": 0.628, - "helm_lite/WMT 2014": 0.186, - "helm_mmlu/MMLU All Subjects": 0.725, - "helm_mmlu/Abstract Algebra": 0.44, - "helm_mmlu/Anatomy": 0.667, - "helm_mmlu/College Physics": 0.363, - "helm_mmlu/Computer Security": 0.79, - "helm_mmlu/Econometrics": 0.553, - "helm_mmlu/Global Facts": 0.46, - "helm_mmlu/Jurisprudence": 0.852, - "helm_mmlu/Philosophy": 0.801, - "helm_mmlu/Professional Psychology": 0.809, - "helm_mmlu/Us Foreign Policy": 0.91, - "helm_mmlu/Astronomy": 0.822, - "helm_mmlu/Business Ethics": 0.86, - "helm_mmlu/Clinical Knowledge": 0.785, - "helm_mmlu/Conceptual Physics": 0.723, - "helm_mmlu/Electrical Engineering": 0.669, - "helm_mmlu/Elementary Mathematics": 0.548, - "helm_mmlu/Formal Logic": 0.548, - "helm_mmlu/High School World History": 0.911, - "helm_mmlu/Human Sexuality": 0.84, - "helm_mmlu/International Law": 0.851, - "helm_mmlu/Logical Fallacies": 0.847, - "helm_mmlu/Machine Learning": 0.562, - "helm_mmlu/Management": 0.903, - "helm_mmlu/Marketing": 0.923, - "helm_mmlu/Medical Genetics": 0.73, - "helm_mmlu/Miscellaneous": 0.904, - "helm_mmlu/Moral Scenarios": 0.544, - "helm_mmlu/Nutrition": 0.781, - "helm_mmlu/Prehistory": 0.858, - "helm_mmlu/Public Relations": 0.7, - "helm_mmlu/Security Studies": 0.796, - "helm_mmlu/Sociology": 0.876, - "helm_mmlu/Virology": 0.554, - "helm_mmlu/World Religions": 0.865, - "helm_mmlu/Mean win rate": 0.387, - "hfopenllm_v2/IFEval": 0.5587, - "hfopenllm_v2/BBH": 0.5243, - "hfopenllm_v2/MATH Level 5": 0.0929, - "hfopenllm_v2/GPQA": 0.3163, - "hfopenllm_v2/MUSR": 0.5059, - "hfopenllm_v2/MMLU-PRO": 0.3944 - } - }, - { - "id": "deepseek-ai/deepseek-llm-7b-base", - "name": "deepseek-llm-7b-base", - "developer": "deepseek-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2179, - "hfopenllm_v2/BBH": 0.3503, - "hfopenllm_v2/MATH Level 5": 0.0196, - "hfopenllm_v2/GPQA": 0.2735, - "hfopenllm_v2/MUSR": 0.3738, - "hfopenllm_v2/MMLU-PRO": 0.1806 - } - }, - { - "id": "deepseek-ai/deepseek-llm-7b-chat", - "name": "deepseek-llm-7b-chat", - "developer": "deepseek-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4171, - "hfopenllm_v2/BBH": 0.3632, - "hfopenllm_v2/MATH Level 5": 0.0204, - "hfopenllm_v2/GPQA": 0.2659, - "hfopenllm_v2/MUSR": 0.4668, - "hfopenllm_v2/MMLU-PRO": 0.2133 - } - }, - { - "id": "deepseek-ai/deepseek-moe-16b-base", - "name": "deepseek-moe-16b-base", - "developer": "deepseek-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.245, - "hfopenllm_v2/BBH": 0.3409, - "hfopenllm_v2/MATH Level 5": 0.0242, - "hfopenllm_v2/GPQA": 0.2542, - "hfopenllm_v2/MUSR": 0.3658, - "hfopenllm_v2/MMLU-PRO": 0.1505 - } - }, - { - "id": "deepseek-ai/deepseek-moe-16b-chat", - "name": "deepseek-moe-16b-chat", - "developer": "deepseek-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3663, - "hfopenllm_v2/BBH": 0.3275, - "hfopenllm_v2/MATH Level 5": 0.0257, - "hfopenllm_v2/GPQA": 0.2248, - "hfopenllm_v2/MUSR": 0.3808, - "hfopenllm_v2/MMLU-PRO": 0.1964 - } - }, - { - "id": "deepseek-ai/deepseek-r1-0528", - "name": "DeepSeek-R1-0528", - "developer": "deepseek-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.699, - "helm_capabilities/MMLU-Pro": 0.793, - "helm_capabilities/GPQA": 0.666, - "helm_capabilities/IFEval": 0.784, - "helm_capabilities/WildBench": 0.828, - "helm_capabilities/Omni-MATH": 0.424 - } - }, - { - "id": "deepseek-ai/DeepSeek-R1-Distill-Llama-70B", - "name": "DeepSeek-R1-Distill-Llama-70B", - "developer": "deepseek-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4336, - "hfopenllm_v2/BBH": 0.5635, - "hfopenllm_v2/MATH Level 5": 0.3074, - "hfopenllm_v2/GPQA": 0.2651, - "hfopenllm_v2/MUSR": 0.4342, - "hfopenllm_v2/MMLU-PRO": 0.4748 - } - }, - { - "id": "deepseek-ai/DeepSeek-R1-Distill-Llama-8B", - "name": "DeepSeek-R1-Distill-Llama-8B", - "developer": "deepseek-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3782, - "hfopenllm_v2/BBH": 0.3239, - "hfopenllm_v2/MATH Level 5": 0.2198, - "hfopenllm_v2/GPQA": 0.255, - "hfopenllm_v2/MUSR": 0.325, - "hfopenllm_v2/MMLU-PRO": 0.2089 - } - }, - { - "id": "deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B", - "name": "DeepSeek-R1-Distill-Qwen-1.5B", - "developer": "deepseek-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3463, - "hfopenllm_v2/BBH": 0.3241, - "hfopenllm_v2/MATH Level 5": 0.1692, - "hfopenllm_v2/GPQA": 0.2559, - "hfopenllm_v2/MUSR": 0.3635, - "hfopenllm_v2/MMLU-PRO": 0.1187 - } - }, - { - "id": "deepseek-ai/DeepSeek-R1-Distill-Qwen-14B", - "name": "DeepSeek-R1-Distill-Qwen-14B", - "developer": "deepseek-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4382, - "hfopenllm_v2/BBH": 0.5906, - "hfopenllm_v2/MATH Level 5": 0.5702, - "hfopenllm_v2/GPQA": 0.3876, - "hfopenllm_v2/MUSR": 0.5366, - "hfopenllm_v2/MMLU-PRO": 0.4667 - } - }, - { - "id": "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B", - "name": "DeepSeek-R1-Distill-Qwen-32B", - "developer": "deepseek-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4186, - "hfopenllm_v2/BBH": 0.4197, - "hfopenllm_v2/MATH Level 5": 0.1707, - "hfopenllm_v2/GPQA": 0.2844, - "hfopenllm_v2/MUSR": 0.4526, - "hfopenllm_v2/MMLU-PRO": 0.4687 - } - }, - { - "id": "deepseek-ai/DeepSeek-R1-Distill-Qwen-7B", - "name": "DeepSeek-R1-Distill-Qwen-7B", - "developer": "deepseek-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4038, - "hfopenllm_v2/BBH": 0.3443, - "hfopenllm_v2/MATH Level 5": 0.1956, - "hfopenllm_v2/GPQA": 0.2794, - "hfopenllm_v2/MUSR": 0.3663, - "hfopenllm_v2/MMLU-PRO": 0.2321 - } - }, - { - "id": "deepseek-ai/deepseek-v3", - "name": "DeepSeek v3", - "developer": "deepseek-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.665, - "helm_capabilities/MMLU-Pro": 0.723, - "helm_capabilities/GPQA": 0.538, - "helm_capabilities/IFEval": 0.832, - "helm_capabilities/WildBench": 0.831, - "helm_capabilities/Omni-MATH": 0.403, - "helm_lite/Mean win rate": 0.908, - "helm_lite/NarrativeQA": 0.796, - "helm_lite/NaturalQuestions (closed-book)": 0.467, - "helm_lite/OpenbookQA": 0.954, - "helm_lite/MMLU": 0.803, - "helm_lite/MATH": 0.912, - "helm_lite/GSM8K": 0.94, - "helm_lite/LegalBench": 0.718, - "helm_lite/MedQA": 0.809, - "helm_lite/WMT 2014": 0.209, - "helm_mmlu/MMLU All Subjects": 0.872, - "helm_mmlu/Abstract Algebra": 0.84, - "helm_mmlu/Anatomy": 0.867, - "helm_mmlu/College Physics": 0.814, - "helm_mmlu/Computer Security": 0.86, - "helm_mmlu/Econometrics": 0.746, - "helm_mmlu/Global Facts": 0.68, - "helm_mmlu/Jurisprudence": 0.898, - "helm_mmlu/Philosophy": 0.9, - "helm_mmlu/Professional Psychology": 0.887, - "helm_mmlu/Us Foreign Policy": 0.92, - "helm_mmlu/Astronomy": 0.921, - "helm_mmlu/Business Ethics": 0.89, - "helm_mmlu/Clinical Knowledge": 0.913, - "helm_mmlu/Conceptual Physics": 0.94, - "helm_mmlu/Electrical Engineering": 0.869, - "helm_mmlu/Elementary Mathematics": 0.942, - "helm_mmlu/Formal Logic": 0.77, - "helm_mmlu/High School World History": 0.928, - "helm_mmlu/Human Sexuality": 0.924, - "helm_mmlu/International Law": 0.95, - "helm_mmlu/Logical Fallacies": 0.914, - "helm_mmlu/Machine Learning": 0.786, - "helm_mmlu/Management": 0.903, - "helm_mmlu/Marketing": 0.949, - "helm_mmlu/Medical Genetics": 0.96, - "helm_mmlu/Miscellaneous": 0.949, - "helm_mmlu/Moral Scenarios": 0.808, - "helm_mmlu/Nutrition": 0.918, - "helm_mmlu/Prehistory": 0.923, - "helm_mmlu/Public Relations": 0.809, - "helm_mmlu/Security Studies": 0.837, - "helm_mmlu/Sociology": 0.955, - "helm_mmlu/Virology": 0.596, - "helm_mmlu/World Religions": 0.912, - "helm_mmlu/Mean win rate": 0.215 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/deepseek.json b/data/developers/deepseek.json deleted file mode 100644 index a295233018343352e4c903c1028ce412e1894956..0000000000000000000000000000000000000000 --- a/data/developers/deepseek.json +++ /dev/null @@ -1,190 +0,0 @@ -{ - "developer": "DeepSeek", - "models": [ - { - "id": "deepseek/chat-v3-0324", - "name": "deepseek/chat-v3-0324", - "developer": "DeepSeek", - "evaluator_relationship": null, - "benchmark_scores": { - "livecodebenchpro/Hard Problems": 0.0, - "livecodebenchpro/Medium Problems": 0.0, - "livecodebenchpro/Easy Problems": 0.19718309859154928 - } - }, - { - "id": "deepseek/deepseek-r1-0528", - "name": "deepseek-r1-0528", - "developer": "deepseek", - "evaluator_relationship": null, - "benchmark_scores": { - "global-mmlu-lite/Global MMLU Lite": 0.6744, - "global-mmlu-lite/Culturally Sensitive": 0.6672, - "global-mmlu-lite/Culturally Agnostic": 0.6816, - "global-mmlu-lite/Arabic": 0.6825, - "global-mmlu-lite/English": 0.715, - "global-mmlu-lite/Bengali": 0.655, - "global-mmlu-lite/German": 0.6375, - "global-mmlu-lite/French": 0.6925, - "global-mmlu-lite/Hindi": 0.6475, - "global-mmlu-lite/Indonesian": 0.655, - "global-mmlu-lite/Italian": 0.6775, - "global-mmlu-lite/Japanese": 0.7725, - "global-mmlu-lite/Korean": 0.6575, - "global-mmlu-lite/Portuguese": 0.635, - "global-mmlu-lite/Spanish": 0.7175, - "global-mmlu-lite/Swahili": 0.6775, - "global-mmlu-lite/Yoruba": 0.77, - "global-mmlu-lite/Chinese": 0.5075, - "global-mmlu-lite/Burmese": 0.69 - } - }, - { - "id": "deepseek/deepseek-v3-2-exp-fc", - "name": "DeepSeek-V3.2-Exp (FC)", - "developer": "deepseek", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 19.0, - "bfcl/bfcl.overall.overall_accuracy": 54.12, - "bfcl/bfcl.overall.total_cost_usd": 6.71, - "bfcl/bfcl.overall.latency_mean_s": 5.83, - "bfcl/bfcl.overall.latency_std_s": 11.71, - "bfcl/bfcl.overall.latency_p95_s": 10.59, - "bfcl/bfcl.non_live.ast_accuracy": 34.85, - "bfcl/bfcl.non_live.simple_ast_accuracy": 37.92, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 74.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 15.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 12.5, - "bfcl/bfcl.live.live_accuracy": 53.66, - "bfcl/bfcl.live.live_simple_ast_accuracy": 66.28, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 51.66, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 25.0, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 25.0, - "bfcl/bfcl.multi_turn.accuracy": 37.38, - "bfcl/bfcl.multi_turn.base_accuracy": 41.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 39.5, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 33.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 35.0, - "bfcl/bfcl.web_search.accuracy": 69.5, - "bfcl/bfcl.web_search.base_accuracy": 80.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 59.0, - "bfcl/bfcl.memory.accuracy": 54.19, - "bfcl/bfcl.memory.kv_accuracy": 41.94, - "bfcl/bfcl.memory.vector_accuracy": 61.29, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 59.35, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 37.5, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 93.18 - } - }, - { - "id": "deepseek/deepseek-v3-2-exp-prompt-thinking", - "name": "DeepSeek-V3.2-Exp (Prompt + Thinking)", - "developer": "deepseek", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 14.0, - "bfcl/bfcl.overall.overall_accuracy": 56.73, - "bfcl/bfcl.overall.total_cost_usd": 57.75, - "bfcl/bfcl.overall.latency_mean_s": 37.89, - "bfcl/bfcl.overall.latency_std_s": 49.56, - "bfcl/bfcl.overall.latency_p95_s": 102.09, - "bfcl/bfcl.non_live.ast_accuracy": 85.52, - "bfcl/bfcl.non_live.simple_ast_accuracy": 74.08, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 92.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 89.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 86.5, - "bfcl/bfcl.live.live_accuracy": 76.02, - "bfcl/bfcl.live.live_simple_ast_accuracy": 82.56, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 74.74, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 87.5, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 54.17, - "bfcl/bfcl.multi_turn.accuracy": 44.88, - "bfcl/bfcl.multi_turn.base_accuracy": 55.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 49.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 27.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 48.5, - "bfcl/bfcl.web_search.accuracy": 58.0, - "bfcl/bfcl.web_search.base_accuracy": 64.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 52.0, - "bfcl/bfcl.memory.accuracy": 44.09, - "bfcl/bfcl.memory.kv_accuracy": 46.45, - "bfcl/bfcl.memory.vector_accuracy": 46.45, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 39.35, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 93.75, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 67.0, - "bfcl/bfcl.format_sensitivity.max_delta": 10.0, - "bfcl/bfcl.format_sensitivity.stddev": 2.77 - } - }, - { - "id": "deepseek/deepseek-v3.1", - "name": "deepseek-v3.1", - "developer": "deepseek", - "evaluator_relationship": null, - "benchmark_scores": { - "global-mmlu-lite/Global MMLU Lite": 0.8044, - "global-mmlu-lite/Culturally Sensitive": 0.7793, - "global-mmlu-lite/Culturally Agnostic": 0.8295, - "global-mmlu-lite/Arabic": 0.805, - "global-mmlu-lite/English": 0.825, - "global-mmlu-lite/Bengali": 0.8157, - "global-mmlu-lite/German": 0.7925, - "global-mmlu-lite/French": 0.8175, - "global-mmlu-lite/Hindi": 0.7569, - "global-mmlu-lite/Indonesian": 0.7764, - "global-mmlu-lite/Italian": 0.8075, - "global-mmlu-lite/Japanese": 0.8312, - "global-mmlu-lite/Korean": 0.8125, - "global-mmlu-lite/Portuguese": 0.8246, - "global-mmlu-lite/Spanish": 0.8125, - "global-mmlu-lite/Swahili": 0.801, - "global-mmlu-lite/Yoruba": 0.7831, - "global-mmlu-lite/Chinese": 0.8161, - "global-mmlu-lite/Burmese": 0.7925 - } - }, - { - "id": "deepseek/deepseek-v3.2", - "name": "DeepSeek-V3.2", - "developer": "DeepSeek", - "evaluator_relationship": null, - "benchmark_scores": { - "terminal-bench-2.0/terminal-bench-2.0": 39.6 - } - }, - { - "id": "deepseek/ep-20250214004308-p7n89", - "name": "ep-20250214004308-p7n89", - "developer": "DeepSeek", - "evaluator_relationship": null, - "benchmark_scores": { - "livecodebenchpro/Hard Problems": 0.0, - "livecodebenchpro/Medium Problems": 0.014084507042253521, - "livecodebenchpro/Easy Problems": 0.4225352112676056 - } - }, - { - "id": "deepseek/ep-20250228232227-z44x5", - "name": "ep-20250228232227-z44x5", - "developer": "DeepSeek", - "evaluator_relationship": null, - "benchmark_scores": { - "livecodebenchpro/Hard Problems": 0.0, - "livecodebenchpro/Medium Problems": 0.0, - "livecodebenchpro/Easy Problems": 0.1267605633802817 - } - }, - { - "id": "deepseek/ep-20250603132404-cgpjm", - "name": "ep-20250603132404-cgpjm", - "developer": "DeepSeek", - "evaluator_relationship": null, - "benchmark_scores": { - "livecodebenchpro/Hard Problems": 0.0, - "livecodebenchpro/Medium Problems": 0.08450704225352113, - "livecodebenchpro/Easy Problems": 0.5774647887323944 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/delta-vector.json b/data/developers/delta-vector.json deleted file mode 100644 index f980692017f518fbbeec5b206efc8074ace9213c..0000000000000000000000000000000000000000 --- a/data/developers/delta-vector.json +++ /dev/null @@ -1,103 +0,0 @@ -{ - "developer": "Delta-Vector", - "models": [ - { - "id": "Delta-Vector/Baldur-8B", - "name": "Baldur-8B", - "developer": "Delta-Vector", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4782, - "hfopenllm_v2/BBH": 0.5306, - "hfopenllm_v2/MATH Level 5": 0.1435, - "hfopenllm_v2/GPQA": 0.302, - "hfopenllm_v2/MUSR": 0.4372, - "hfopenllm_v2/MMLU-PRO": 0.3654 - } - }, - { - "id": "Delta-Vector/Control-8B", - "name": "Control-8B", - "developer": "Delta-Vector", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.549, - "hfopenllm_v2/BBH": 0.5041, - "hfopenllm_v2/MATH Level 5": 0.139, - "hfopenllm_v2/GPQA": 0.3163, - "hfopenllm_v2/MUSR": 0.4355, - "hfopenllm_v2/MMLU-PRO": 0.3732 - } - }, - { - "id": "Delta-Vector/Control-8B-V1.1", - "name": "Control-8B-V1.1", - "developer": "Delta-Vector", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5697, - "hfopenllm_v2/BBH": 0.4993, - "hfopenllm_v2/MATH Level 5": 0.1276, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.4237, - "hfopenllm_v2/MMLU-PRO": 0.3745 - } - }, - { - "id": "Delta-Vector/Darkens-8B", - "name": "Darkens-8B", - "developer": "Delta-Vector", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2548, - "hfopenllm_v2/BBH": 0.5251, - "hfopenllm_v2/MATH Level 5": 0.0589, - "hfopenllm_v2/GPQA": 0.3247, - "hfopenllm_v2/MUSR": 0.4106, - "hfopenllm_v2/MMLU-PRO": 0.3736 - } - }, - { - "id": "Delta-Vector/Henbane-7b-attempt2", - "name": "Henbane-7b-attempt2", - "developer": "Delta-Vector", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4157, - "hfopenllm_v2/BBH": 0.5061, - "hfopenllm_v2/MATH Level 5": 0.2273, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.3973, - "hfopenllm_v2/MMLU-PRO": 0.4028 - } - }, - { - "id": "Delta-Vector/Odin-9B", - "name": "Odin-9B", - "developer": "Delta-Vector", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3692, - "hfopenllm_v2/BBH": 0.544, - "hfopenllm_v2/MATH Level 5": 0.145, - "hfopenllm_v2/GPQA": 0.3414, - "hfopenllm_v2/MUSR": 0.4648, - "hfopenllm_v2/MMLU-PRO": 0.4047 - } - }, - { - "id": "Delta-Vector/Tor-8B", - "name": "Tor-8B", - "developer": "Delta-Vector", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2382, - "hfopenllm_v2/BBH": 0.5209, - "hfopenllm_v2/MATH Level 5": 0.0589, - "hfopenllm_v2/GPQA": 0.3238, - "hfopenllm_v2/MUSR": 0.4092, - "hfopenllm_v2/MMLU-PRO": 0.373 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/devquasar.json b/data/developers/devquasar.json deleted file mode 100644 index d207cd7873b393f5c3f18840cd65f8a70406f7dc..0000000000000000000000000000000000000000 --- a/data/developers/devquasar.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "DevQuasar", - "models": [ - { - "id": "DevQuasar/DevQuasar-R1-Uncensored-Llama-8B", - "name": "DevQuasar-R1-Uncensored-Llama-8B", - "developer": "DevQuasar", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3849, - "hfopenllm_v2/BBH": 0.5118, - "hfopenllm_v2/MATH Level 5": 0.3308, - "hfopenllm_v2/GPQA": 0.3473, - "hfopenllm_v2/MUSR": 0.4436, - "hfopenllm_v2/MMLU-PRO": 0.3615 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/dfurman.json b/data/developers/dfurman.json deleted file mode 100644 index 2947dc3ef503295f24886c787e729305dcebb026..0000000000000000000000000000000000000000 --- a/data/developers/dfurman.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "developer": "dfurman", - "models": [ - { - "id": "dfurman/CalmeRys-78B-Orpo-v0.1", - "name": "CalmeRys-78B-Orpo-v0.1", - "developer": "dfurman", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8163, - "hfopenllm_v2/BBH": 0.7262, - "hfopenllm_v2/MATH Level 5": 0.4063, - "hfopenllm_v2/GPQA": 0.4002, - "hfopenllm_v2/MUSR": 0.5902, - "hfopenllm_v2/MMLU-PRO": 0.7012 - } - }, - { - "id": "dfurman/Llama-3-70B-Orpo-v0.1", - "name": "Llama-3-70B-Orpo-v0.1", - "developer": "dfurman", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2049, - "hfopenllm_v2/BBH": 0.4655, - "hfopenllm_v2/MATH Level 5": 0.1579, - "hfopenllm_v2/GPQA": 0.2576, - "hfopenllm_v2/MUSR": 0.4534, - "hfopenllm_v2/MMLU-PRO": 0.3893 - } - }, - { - "id": "dfurman/Llama-3-8B-Orpo-v0.1", - "name": "Llama-3-8B-Orpo-v0.1", - "developer": "dfurman", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2835, - "hfopenllm_v2/BBH": 0.3842, - "hfopenllm_v2/MATH Level 5": 0.0521, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.3566, - "hfopenllm_v2/MMLU-PRO": 0.2298 - } - }, - { - "id": "dfurman/Qwen2-72B-Orpo-v0.1", - "name": "Qwen2-72B-Orpo-v0.1", - "developer": "dfurman", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.788, - "hfopenllm_v2/BBH": 0.6969, - "hfopenllm_v2/MATH Level 5": 0.4056, - "hfopenllm_v2/GPQA": 0.3842, - "hfopenllm_v2/MUSR": 0.4784, - "hfopenllm_v2/MMLU-PRO": 0.5455 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/dicta-il.json b/data/developers/dicta-il.json deleted file mode 100644 index 96d95a28710fe141cefa709d98c71f0591b446ff..0000000000000000000000000000000000000000 --- a/data/developers/dicta-il.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "dicta-il", - "models": [ - { - "id": "dicta-il/dictalm2.0", - "name": "dictalm2.0", - "developer": "dicta-il", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2413, - "hfopenllm_v2/BBH": 0.4018, - "hfopenllm_v2/MATH Level 5": 0.0181, - "hfopenllm_v2/GPQA": 0.2919, - "hfopenllm_v2/MUSR": 0.382, - "hfopenllm_v2/MMLU-PRO": 0.2605 - } - }, - { - "id": "dicta-il/dictalm2.0-instruct", - "name": "dictalm2.0-instruct", - "developer": "dicta-il", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4412, - "hfopenllm_v2/BBH": 0.4256, - "hfopenllm_v2/MATH Level 5": 0.0227, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.3946, - "hfopenllm_v2/MMLU-PRO": 0.2605 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/distilbert.json b/data/developers/distilbert.json deleted file mode 100644 index e40493fad80544e55392936d1c3d06d2905b07f6..0000000000000000000000000000000000000000 --- a/data/developers/distilbert.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "distilbert", - "models": [ - { - "id": "distilbert/distilgpt2", - "name": "distilgpt2", - "developer": "distilbert", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0611, - "hfopenllm_v2/BBH": 0.3038, - "hfopenllm_v2/MATH Level 5": 0.006, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.4207, - "hfopenllm_v2/MMLU-PRO": 0.1187 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/divyanshukunwar.json b/data/developers/divyanshukunwar.json deleted file mode 100644 index 10b579d03cb68714ba306e97273cc6fd36fd44d7..0000000000000000000000000000000000000000 --- a/data/developers/divyanshukunwar.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "divyanshukunwar", - "models": [ - { - "id": "divyanshukunwar/SASTRI_1_9B", - "name": "SASTRI_1_9B", - "developer": "divyanshukunwar", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4207, - "hfopenllm_v2/BBH": 0.468, - "hfopenllm_v2/MATH Level 5": 0.1156, - "hfopenllm_v2/GPQA": 0.3213, - "hfopenllm_v2/MUSR": 0.3831, - "hfopenllm_v2/MMLU-PRO": 0.3187 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/djuna-test-lab.json b/data/developers/djuna-test-lab.json deleted file mode 100644 index a155d6ff32413d81bbc59a97ce20c2e851ad7bad..0000000000000000000000000000000000000000 --- a/data/developers/djuna-test-lab.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "djuna-test-lab", - "models": [ - { - "id": "djuna-test-lab/TEST-L3.2-ReWish-3B", - "name": "TEST-L3.2-ReWish-3B", - "developer": "djuna-test-lab", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6368, - "hfopenllm_v2/BBH": 0.4495, - "hfopenllm_v2/MATH Level 5": 0.1367, - "hfopenllm_v2/GPQA": 0.2836, - "hfopenllm_v2/MUSR": 0.3777, - "hfopenllm_v2/MMLU-PRO": 0.3126 - } - }, - { - "id": "djuna-test-lab/TEST-L3.2-ReWish-3B-ties-w-base", - "name": "TEST-L3.2-ReWish-3B-ties-w-base", - "developer": "djuna-test-lab", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6353, - "hfopenllm_v2/BBH": 0.4495, - "hfopenllm_v2/MATH Level 5": 0.1367, - "hfopenllm_v2/GPQA": 0.2836, - "hfopenllm_v2/MUSR": 0.3777, - "hfopenllm_v2/MMLU-PRO": 0.3126 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/djuna.json b/data/developers/djuna.json deleted file mode 100644 index 722d94d3c6d099c9e6a314b22e61bda73817fd42..0000000000000000000000000000000000000000 --- a/data/developers/djuna.json +++ /dev/null @@ -1,215 +0,0 @@ -{ - "developer": "djuna", - "models": [ - { - "id": "djuna/G2-BigGSHT-27B-2", - "name": "G2-BigGSHT-27B-2", - "developer": "djuna", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7974, - "hfopenllm_v2/BBH": 0.6415, - "hfopenllm_v2/MATH Level 5": 0.2349, - "hfopenllm_v2/GPQA": 0.3633, - "hfopenllm_v2/MUSR": 0.4072, - "hfopenllm_v2/MMLU-PRO": 0.4528 - } - }, - { - "id": "djuna/G2-GSHT", - "name": "G2-GSHT", - "developer": "djuna", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.563, - "hfopenllm_v2/BBH": 0.527, - "hfopenllm_v2/MATH Level 5": 0.1926, - "hfopenllm_v2/GPQA": 0.3255, - "hfopenllm_v2/MUSR": 0.4006, - "hfopenllm_v2/MMLU-PRO": 0.307 - } - }, - { - "id": "djuna/Gemma-2-gemmama-9b", - "name": "Gemma-2-gemmama-9b", - "developer": "djuna", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7703, - "hfopenllm_v2/BBH": 0.542, - "hfopenllm_v2/MATH Level 5": 0.1926, - "hfopenllm_v2/GPQA": 0.3356, - "hfopenllm_v2/MUSR": 0.4031, - "hfopenllm_v2/MMLU-PRO": 0.3109 - } - }, - { - "id": "djuna/L3.1-ForStHS", - "name": "L3.1-ForStHS", - "developer": "djuna", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7813, - "hfopenllm_v2/BBH": 0.5203, - "hfopenllm_v2/MATH Level 5": 0.1503, - "hfopenllm_v2/GPQA": 0.2911, - "hfopenllm_v2/MUSR": 0.4026, - "hfopenllm_v2/MMLU-PRO": 0.3735 - } - }, - { - "id": "djuna/L3.1-Promissum_Mane-8B-Della-1.5-calc", - "name": "L3.1-Promissum_Mane-8B-Della-1.5-calc", - "developer": "djuna", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7235, - "hfopenllm_v2/BBH": 0.5433, - "hfopenllm_v2/MATH Level 5": 0.1639, - "hfopenllm_v2/GPQA": 0.3146, - "hfopenllm_v2/MUSR": 0.4253, - "hfopenllm_v2/MMLU-PRO": 0.3904 - } - }, - { - "id": "djuna/L3.1-Promissum_Mane-8B-Della-calc", - "name": "L3.1-Promissum_Mane-8B-Della-calc", - "developer": "djuna", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5442, - "hfopenllm_v2/BBH": 0.5486, - "hfopenllm_v2/MATH Level 5": 0.1843, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.423, - "hfopenllm_v2/MMLU-PRO": 0.3802 - } - }, - { - "id": "djuna/L3.1-Purosani-2-8B", - "name": "L3.1-Purosani-2-8B", - "developer": "djuna", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4988, - "hfopenllm_v2/BBH": 0.5182, - "hfopenllm_v2/MATH Level 5": 0.1171, - "hfopenllm_v2/GPQA": 0.3012, - "hfopenllm_v2/MUSR": 0.3816, - "hfopenllm_v2/MMLU-PRO": 0.3752 - } - }, - { - "id": "djuna/L3.1-Suze-Vume-calc", - "name": "L3.1-Suze-Vume-calc", - "developer": "djuna", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7297, - "hfopenllm_v2/BBH": 0.5164, - "hfopenllm_v2/MATH Level 5": 0.114, - "hfopenllm_v2/GPQA": 0.2819, - "hfopenllm_v2/MUSR": 0.3843, - "hfopenllm_v2/MMLU-PRO": 0.3515 - } - }, - { - "id": "djuna/MN-Chinofun", - "name": "MN-Chinofun", - "developer": "djuna", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.611, - "hfopenllm_v2/BBH": 0.4953, - "hfopenllm_v2/MATH Level 5": 0.1307, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.4084, - "hfopenllm_v2/MMLU-PRO": 0.3603 - } - }, - { - "id": "djuna/MN-Chinofun-12B-2", - "name": "MN-Chinofun-12B-2", - "developer": "djuna", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6171, - "hfopenllm_v2/BBH": 0.5037, - "hfopenllm_v2/MATH Level 5": 0.1307, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.4268, - "hfopenllm_v2/MMLU-PRO": 0.3615 - } - }, - { - "id": "djuna/MN-Chinofun-12B-3", - "name": "MN-Chinofun-12B-3", - "developer": "djuna", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3053, - "hfopenllm_v2/BBH": 0.5348, - "hfopenllm_v2/MATH Level 5": 0.1005, - "hfopenllm_v2/GPQA": 0.2659, - "hfopenllm_v2/MUSR": 0.4198, - "hfopenllm_v2/MMLU-PRO": 0.3026 - } - }, - { - "id": "djuna/MN-Chinofun-12B-4", - "name": "MN-Chinofun-12B-4", - "developer": "djuna", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5404, - "hfopenllm_v2/BBH": 0.5348, - "hfopenllm_v2/MATH Level 5": 0.1118, - "hfopenllm_v2/GPQA": 0.2953, - "hfopenllm_v2/MUSR": 0.4307, - "hfopenllm_v2/MMLU-PRO": 0.3497 - } - }, - { - "id": "djuna/Q2.5-Partron-7B", - "name": "Q2.5-Partron-7B", - "developer": "djuna", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7321, - "hfopenllm_v2/BBH": 0.5418, - "hfopenllm_v2/MATH Level 5": 0.4826, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.4165, - "hfopenllm_v2/MMLU-PRO": 0.4283 - } - }, - { - "id": "djuna/Q2.5-Veltha-14B", - "name": "Q2.5-Veltha-14B", - "developer": "djuna", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8292, - "hfopenllm_v2/BBH": 0.6484, - "hfopenllm_v2/MATH Level 5": 0.4789, - "hfopenllm_v2/GPQA": 0.3591, - "hfopenllm_v2/MUSR": 0.4194, - "hfopenllm_v2/MMLU-PRO": 0.5298 - } - }, - { - "id": "djuna/Q2.5-Veltha-14B-0.5", - "name": "Q2.5-Veltha-14B-0.5", - "developer": "djuna", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7796, - "hfopenllm_v2/BBH": 0.6523, - "hfopenllm_v2/MATH Level 5": 0.4373, - "hfopenllm_v2/GPQA": 0.3683, - "hfopenllm_v2/MUSR": 0.4339, - "hfopenllm_v2/MMLU-PRO": 0.5295 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/dnhkng.json b/data/developers/dnhkng.json deleted file mode 100644 index 487ace7edbb1d45e7e3210bde7770cc18e628272..0000000000000000000000000000000000000000 --- a/data/developers/dnhkng.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "developer": "dnhkng", - "models": [ - { - "id": "dnhkng/RYS-Llama-3-8B-Instruct", - "name": "RYS-Llama-3-8B-Instruct", - "developer": "dnhkng", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6958, - "hfopenllm_v2/BBH": 0.4809, - "hfopenllm_v2/MATH Level 5": 0.0687, - "hfopenllm_v2/GPQA": 0.2576, - "hfopenllm_v2/MUSR": 0.3383, - "hfopenllm_v2/MMLU-PRO": 0.3557 - } - }, - { - "id": "dnhkng/RYS-Llama-3-Huge-Instruct", - "name": "RYS-Llama-3-Huge-Instruct", - "developer": "dnhkng", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7686, - "hfopenllm_v2/BBH": 0.6481, - "hfopenllm_v2/MATH Level 5": 0.2289, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.4208, - "hfopenllm_v2/MMLU-PRO": 0.511 - } - }, - { - "id": "dnhkng/RYS-Llama-3-Large-Instruct", - "name": "RYS-Llama-3-Large-Instruct", - "developer": "dnhkng", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8051, - "hfopenllm_v2/BBH": 0.6525, - "hfopenllm_v2/MATH Level 5": 0.2304, - "hfopenllm_v2/GPQA": 0.2894, - "hfopenllm_v2/MUSR": 0.418, - "hfopenllm_v2/MMLU-PRO": 0.5137 - } - }, - { - "id": "dnhkng/RYS-Llama-3.1-8B-Instruct", - "name": "RYS-Llama-3.1-8B-Instruct", - "developer": "dnhkng", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7685, - "hfopenllm_v2/BBH": 0.5164, - "hfopenllm_v2/MATH Level 5": 0.1329, - "hfopenllm_v2/GPQA": 0.2676, - "hfopenllm_v2/MUSR": 0.3681, - "hfopenllm_v2/MMLU-PRO": 0.3639 - } - }, - { - "id": "dnhkng/RYS-Llama3.1-Large", - "name": "RYS-Llama3.1-Large", - "developer": "dnhkng", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8492, - "hfopenllm_v2/BBH": 0.6899, - "hfopenllm_v2/MATH Level 5": 0.3505, - "hfopenllm_v2/GPQA": 0.3742, - "hfopenllm_v2/MUSR": 0.4554, - "hfopenllm_v2/MMLU-PRO": 0.5249 - } - }, - { - "id": "dnhkng/RYS-Medium", - "name": "RYS-Medium", - "developer": "dnhkng", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4406, - "hfopenllm_v2/BBH": 0.6285, - "hfopenllm_v2/MATH Level 5": 0.108, - "hfopenllm_v2/GPQA": 0.328, - "hfopenllm_v2/MUSR": 0.4069, - "hfopenllm_v2/MMLU-PRO": 0.4326 - } - }, - { - "id": "dnhkng/RYS-Phi-3-medium-4k-instruct", - "name": "RYS-Phi-3-medium-4k-instruct", - "developer": "dnhkng", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4391, - "hfopenllm_v2/BBH": 0.6226, - "hfopenllm_v2/MATH Level 5": 0.1609, - "hfopenllm_v2/GPQA": 0.3549, - "hfopenllm_v2/MUSR": 0.4253, - "hfopenllm_v2/MMLU-PRO": 0.4846 - } - }, - { - "id": "dnhkng/RYS-XLarge", - "name": "RYS-XLarge", - "developer": "dnhkng", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7996, - "hfopenllm_v2/BBH": 0.705, - "hfopenllm_v2/MATH Level 5": 0.4252, - "hfopenllm_v2/GPQA": 0.3842, - "hfopenllm_v2/MUSR": 0.497, - "hfopenllm_v2/MMLU-PRO": 0.5428 - } - }, - { - "id": "dnhkng/RYS-XLarge-base", - "name": "RYS-XLarge-base", - "developer": "dnhkng", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.791, - "hfopenllm_v2/BBH": 0.7047, - "hfopenllm_v2/MATH Level 5": 0.3792, - "hfopenllm_v2/GPQA": 0.3792, - "hfopenllm_v2/MUSR": 0.4903, - "hfopenllm_v2/MMLU-PRO": 0.5431 - } - }, - { - "id": "dnhkng/RYS-XLarge2", - "name": "RYS-XLarge2", - "developer": "dnhkng", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4902, - "hfopenllm_v2/BBH": 0.6574, - "hfopenllm_v2/MATH Level 5": 0.2749, - "hfopenllm_v2/GPQA": 0.3742, - "hfopenllm_v2/MUSR": 0.4508, - "hfopenllm_v2/MMLU-PRO": 0.5378 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/dongwei.json b/data/developers/dongwei.json deleted file mode 100644 index c29c9cae20cb8626ffd076e468a3c74680f49c2d..0000000000000000000000000000000000000000 --- a/data/developers/dongwei.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "Dongwei", - "models": [ - { - "id": "Dongwei/DeepSeek-R1-Distill-Qwen-7B-GRPO", - "name": "DeepSeek-R1-Distill-Qwen-7B-GRPO", - "developer": "Dongwei", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4038, - "hfopenllm_v2/BBH": 0.3443, - "hfopenllm_v2/MATH Level 5": 0.1956, - "hfopenllm_v2/GPQA": 0.2794, - "hfopenllm_v2/MUSR": 0.3663, - "hfopenllm_v2/MMLU-PRO": 0.2322 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/doppelreflex.json b/data/developers/doppelreflex.json deleted file mode 100644 index 0b78fc1cbec001d1df6b60b1fab265b2eab6799e..0000000000000000000000000000000000000000 --- a/data/developers/doppelreflex.json +++ /dev/null @@ -1,411 +0,0 @@ -{ - "developer": "DoppelReflEx", - "models": [ - { - "id": "DoppelReflEx/L3-8B-R1-WolfCore", - "name": "L3-8B-R1-WolfCore", - "developer": "DoppelReflEx", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3775, - "hfopenllm_v2/BBH": 0.5318, - "hfopenllm_v2/MATH Level 5": 0.1631, - "hfopenllm_v2/GPQA": 0.3289, - "hfopenllm_v2/MUSR": 0.4277, - "hfopenllm_v2/MMLU-PRO": 0.3717 - } - }, - { - "id": "DoppelReflEx/L3-8B-R1-WolfCore-V1.5-test", - "name": "L3-8B-R1-WolfCore-V1.5-test", - "developer": "DoppelReflEx", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3955, - "hfopenllm_v2/BBH": 0.5315, - "hfopenllm_v2/MATH Level 5": 0.1231, - "hfopenllm_v2/GPQA": 0.3263, - "hfopenllm_v2/MUSR": 0.3841, - "hfopenllm_v2/MMLU-PRO": 0.3728 - } - }, - { - "id": "DoppelReflEx/L3-8B-WolfCore", - "name": "L3-8B-WolfCore", - "developer": "DoppelReflEx", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4022, - "hfopenllm_v2/BBH": 0.5182, - "hfopenllm_v2/MATH Level 5": 0.0982, - "hfopenllm_v2/GPQA": 0.3096, - "hfopenllm_v2/MUSR": 0.3973, - "hfopenllm_v2/MMLU-PRO": 0.3705 - } - }, - { - "id": "DoppelReflEx/MiniusLight-24B", - "name": "MiniusLight-24B", - "developer": "DoppelReflEx", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2577, - "hfopenllm_v2/BBH": 0.6256, - "hfopenllm_v2/MATH Level 5": 0.1261, - "hfopenllm_v2/GPQA": 0.3582, - "hfopenllm_v2/MUSR": 0.4319, - "hfopenllm_v2/MMLU-PRO": 0.5091 - } - }, - { - "id": "DoppelReflEx/MiniusLight-24B-test", - "name": "MiniusLight-24B-test", - "developer": "DoppelReflEx", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0394, - "hfopenllm_v2/BBH": 0.6334, - "hfopenllm_v2/MATH Level 5": 0.0257, - "hfopenllm_v2/GPQA": 0.3683, - "hfopenllm_v2/MUSR": 0.4093, - "hfopenllm_v2/MMLU-PRO": 0.5182 - } - }, - { - "id": "DoppelReflEx/MiniusLight-24B-v1b-test", - "name": "MiniusLight-24B-v1b-test", - "developer": "DoppelReflEx", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3791, - "hfopenllm_v2/BBH": 0.6617, - "hfopenllm_v2/MATH Level 5": 0.2394, - "hfopenllm_v2/GPQA": 0.3792, - "hfopenllm_v2/MUSR": 0.4557, - "hfopenllm_v2/MMLU-PRO": 0.5365 - } - }, - { - "id": "DoppelReflEx/MiniusLight-24B-v1c-test", - "name": "MiniusLight-24B-v1c-test", - "developer": "DoppelReflEx", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3786, - "hfopenllm_v2/BBH": 0.6753, - "hfopenllm_v2/MATH Level 5": 0.2968, - "hfopenllm_v2/GPQA": 0.3951, - "hfopenllm_v2/MUSR": 0.4634, - "hfopenllm_v2/MMLU-PRO": 0.5487 - } - }, - { - "id": "DoppelReflEx/MiniusLight-24B-v1d-test", - "name": "MiniusLight-24B-v1d-test", - "developer": "DoppelReflEx", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4032, - "hfopenllm_v2/BBH": 0.6712, - "hfopenllm_v2/MATH Level 5": 0.2946, - "hfopenllm_v2/GPQA": 0.3951, - "hfopenllm_v2/MUSR": 0.4621, - "hfopenllm_v2/MMLU-PRO": 0.5489 - } - }, - { - "id": "DoppelReflEx/MN-12B-FoxFrame-test", - "name": "MN-12B-FoxFrame-test", - "developer": "DoppelReflEx", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4222, - "hfopenllm_v2/BBH": 0.5456, - "hfopenllm_v2/MATH Level 5": 0.1397, - "hfopenllm_v2/GPQA": 0.3079, - "hfopenllm_v2/MUSR": 0.4254, - "hfopenllm_v2/MMLU-PRO": 0.3503 - } - }, - { - "id": "DoppelReflEx/MN-12B-FoxFrame2-test", - "name": "MN-12B-FoxFrame2-test", - "developer": "DoppelReflEx", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4319, - "hfopenllm_v2/BBH": 0.5485, - "hfopenllm_v2/MATH Level 5": 0.1405, - "hfopenllm_v2/GPQA": 0.3146, - "hfopenllm_v2/MUSR": 0.4252, - "hfopenllm_v2/MMLU-PRO": 0.3569 - } - }, - { - "id": "DoppelReflEx/MN-12B-FoxFrame3-test", - "name": "MN-12B-FoxFrame3-test", - "developer": "DoppelReflEx", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4323, - "hfopenllm_v2/BBH": 0.5395, - "hfopenllm_v2/MATH Level 5": 0.1322, - "hfopenllm_v2/GPQA": 0.3012, - "hfopenllm_v2/MUSR": 0.4598, - "hfopenllm_v2/MMLU-PRO": 0.3529 - } - }, - { - "id": "DoppelReflEx/MN-12B-Kakigori", - "name": "MN-12B-Kakigori", - "developer": "DoppelReflEx", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3593, - "hfopenllm_v2/BBH": 0.5416, - "hfopenllm_v2/MATH Level 5": 0.1193, - "hfopenllm_v2/GPQA": 0.3247, - "hfopenllm_v2/MUSR": 0.4052, - "hfopenllm_v2/MMLU-PRO": 0.3581 - } - }, - { - "id": "DoppelReflEx/MN-12B-LilithFrame", - "name": "MN-12B-LilithFrame", - "developer": "DoppelReflEx", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.436, - "hfopenllm_v2/BBH": 0.4956, - "hfopenllm_v2/MATH Level 5": 0.0589, - "hfopenllm_v2/GPQA": 0.3205, - "hfopenllm_v2/MUSR": 0.3843, - "hfopenllm_v2/MMLU-PRO": 0.3237 - } - }, - { - "id": "DoppelReflEx/MN-12B-LilithFrame-Experiment-2", - "name": "MN-12B-LilithFrame-Experiment-2", - "developer": "DoppelReflEx", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4299, - "hfopenllm_v2/BBH": 0.4983, - "hfopenllm_v2/MATH Level 5": 0.1073, - "hfopenllm_v2/GPQA": 0.3255, - "hfopenllm_v2/MUSR": 0.3804, - "hfopenllm_v2/MMLU-PRO": 0.3276 - } - }, - { - "id": "DoppelReflEx/MN-12B-LilithFrame-Experiment-3", - "name": "MN-12B-LilithFrame-Experiment-3", - "developer": "DoppelReflEx", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4128, - "hfopenllm_v2/BBH": 0.5468, - "hfopenllm_v2/MATH Level 5": 0.1344, - "hfopenllm_v2/GPQA": 0.328, - "hfopenllm_v2/MUSR": 0.4039, - "hfopenllm_v2/MMLU-PRO": 0.3604 - } - }, - { - "id": "DoppelReflEx/MN-12B-LilithFrame-Experiment-4", - "name": "MN-12B-LilithFrame-Experiment-4", - "developer": "DoppelReflEx", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3981, - "hfopenllm_v2/BBH": 0.5534, - "hfopenllm_v2/MATH Level 5": 0.1224, - "hfopenllm_v2/GPQA": 0.3171, - "hfopenllm_v2/MUSR": 0.4371, - "hfopenllm_v2/MMLU-PRO": 0.3649 - } - }, - { - "id": "DoppelReflEx/MN-12B-Mimicore-GreenSnake", - "name": "MN-12B-Mimicore-GreenSnake", - "developer": "DoppelReflEx", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.478, - "hfopenllm_v2/BBH": 0.5481, - "hfopenllm_v2/MATH Level 5": 0.139, - "hfopenllm_v2/GPQA": 0.3247, - "hfopenllm_v2/MUSR": 0.4306, - "hfopenllm_v2/MMLU-PRO": 0.3651 - } - }, - { - "id": "DoppelReflEx/MN-12B-Mimicore-Nocturne", - "name": "MN-12B-Mimicore-Nocturne", - "developer": "DoppelReflEx", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3957, - "hfopenllm_v2/BBH": 0.5703, - "hfopenllm_v2/MATH Level 5": 0.1057, - "hfopenllm_v2/GPQA": 0.3196, - "hfopenllm_v2/MUSR": 0.4569, - "hfopenllm_v2/MMLU-PRO": 0.3634 - } - }, - { - "id": "DoppelReflEx/MN-12B-Mimicore-Orochi", - "name": "MN-12B-Mimicore-Orochi", - "developer": "DoppelReflEx", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.462, - "hfopenllm_v2/BBH": 0.5498, - "hfopenllm_v2/MATH Level 5": 0.136, - "hfopenllm_v2/GPQA": 0.3129, - "hfopenllm_v2/MUSR": 0.4546, - "hfopenllm_v2/MMLU-PRO": 0.3447 - } - }, - { - "id": "DoppelReflEx/MN-12B-Mimicore-Orochi-v2-Experiment", - "name": "MN-12B-Mimicore-Orochi-v2-Experiment", - "developer": "DoppelReflEx", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2842, - "hfopenllm_v2/BBH": 0.5323, - "hfopenllm_v2/MATH Level 5": 0.0612, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.4574, - "hfopenllm_v2/MMLU-PRO": 0.3423 - } - }, - { - "id": "DoppelReflEx/MN-12B-Mimicore-Orochi-v3-Experiment", - "name": "MN-12B-Mimicore-Orochi-v3-Experiment", - "developer": "DoppelReflEx", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4102, - "hfopenllm_v2/BBH": 0.5438, - "hfopenllm_v2/MATH Level 5": 0.1216, - "hfopenllm_v2/GPQA": 0.2928, - "hfopenllm_v2/MUSR": 0.4438, - "hfopenllm_v2/MMLU-PRO": 0.3396 - } - }, - { - "id": "DoppelReflEx/MN-12B-Mimicore-Orochi-v4-Experiment", - "name": "MN-12B-Mimicore-Orochi-v4-Experiment", - "developer": "DoppelReflEx", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4321, - "hfopenllm_v2/BBH": 0.5463, - "hfopenllm_v2/MATH Level 5": 0.1208, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.4449, - "hfopenllm_v2/MMLU-PRO": 0.352 - } - }, - { - "id": "DoppelReflEx/MN-12B-Mimicore-WhiteSnake", - "name": "MN-12B-Mimicore-WhiteSnake", - "developer": "DoppelReflEx", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4438, - "hfopenllm_v2/BBH": 0.5605, - "hfopenllm_v2/MATH Level 5": 0.1314, - "hfopenllm_v2/GPQA": 0.318, - "hfopenllm_v2/MUSR": 0.4569, - "hfopenllm_v2/MMLU-PRO": 0.3658 - } - }, - { - "id": "DoppelReflEx/MN-12B-Mimicore-WhiteSnake-v2-Experiment-1", - "name": "MN-12B-Mimicore-WhiteSnake-v2-Experiment-1", - "developer": "DoppelReflEx", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3909, - "hfopenllm_v2/BBH": 0.4866, - "hfopenllm_v2/MATH Level 5": 0.0785, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.379, - "hfopenllm_v2/MMLU-PRO": 0.3114 - } - }, - { - "id": "DoppelReflEx/MN-12B-Mimicore-WhiteSnake-v2-Experiment-2", - "name": "MN-12B-Mimicore-WhiteSnake-v2-Experiment-2", - "developer": "DoppelReflEx", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3124, - "hfopenllm_v2/BBH": 0.5126, - "hfopenllm_v2/MATH Level 5": 0.1125, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.3975, - "hfopenllm_v2/MMLU-PRO": 0.3314 - } - }, - { - "id": "DoppelReflEx/MN-12B-Mimicore-WhiteSnake-v2-Experiment-3", - "name": "MN-12B-Mimicore-WhiteSnake-v2-Experiment-3", - "developer": "DoppelReflEx", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4302, - "hfopenllm_v2/BBH": 0.4812, - "hfopenllm_v2/MATH Level 5": 0.0899, - "hfopenllm_v2/GPQA": 0.302, - "hfopenllm_v2/MUSR": 0.3684, - "hfopenllm_v2/MMLU-PRO": 0.3198 - } - }, - { - "id": "DoppelReflEx/MN-12B-Mimicore-WhiteSnake-v2-Experiment-4", - "name": "MN-12B-Mimicore-WhiteSnake-v2-Experiment-4", - "developer": "DoppelReflEx", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4241, - "hfopenllm_v2/BBH": 0.5185, - "hfopenllm_v2/MATH Level 5": 0.114, - "hfopenllm_v2/GPQA": 0.3104, - "hfopenllm_v2/MUSR": 0.4002, - "hfopenllm_v2/MMLU-PRO": 0.3342 - } - }, - { - "id": "DoppelReflEx/MN-12B-Unleashed-Twilight", - "name": "MN-12B-Unleashed-Twilight", - "developer": "DoppelReflEx", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3505, - "hfopenllm_v2/BBH": 0.5521, - "hfopenllm_v2/MATH Level 5": 0.0959, - "hfopenllm_v2/GPQA": 0.3289, - "hfopenllm_v2/MUSR": 0.4384, - "hfopenllm_v2/MMLU-PRO": 0.3678 - } - }, - { - "id": "DoppelReflEx/MN-12B-WolFrame", - "name": "MN-12B-WolFrame", - "developer": "DoppelReflEx", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4397, - "hfopenllm_v2/BBH": 0.5117, - "hfopenllm_v2/MATH Level 5": 0.1314, - "hfopenllm_v2/GPQA": 0.3104, - "hfopenllm_v2/MUSR": 0.4015, - "hfopenllm_v2/MMLU-PRO": 0.3393 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/dreadpoor.json b/data/developers/dreadpoor.json deleted file mode 100644 index 87d55478bbf844a03c23f356544a69e6cde1dd4e..0000000000000000000000000000000000000000 --- a/data/developers/dreadpoor.json +++ /dev/null @@ -1,1671 +0,0 @@ -{ - "developer": "DreadPoor", - "models": [ - { - "id": "DreadPoor/Again-8B-Model_Stock", - "name": "Again-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6724, - "hfopenllm_v2/BBH": 0.531, - "hfopenllm_v2/MATH Level 5": 0.1201, - "hfopenllm_v2/GPQA": 0.3012, - "hfopenllm_v2/MUSR": 0.3987, - "hfopenllm_v2/MMLU-PRO": 0.3518 - } - }, - { - "id": "DreadPoor/Alita99-8B-LINEAR", - "name": "Alita99-8B-LINEAR", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.719, - "hfopenllm_v2/BBH": 0.5442, - "hfopenllm_v2/MATH Level 5": 0.1647, - "hfopenllm_v2/GPQA": 0.3163, - "hfopenllm_v2/MUSR": 0.4266, - "hfopenllm_v2/MMLU-PRO": 0.3809 - } - }, - { - "id": "DreadPoor/AnotherTest", - "name": "AnotherTest", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4701, - "hfopenllm_v2/BBH": 0.4683, - "hfopenllm_v2/MATH Level 5": 0.0619, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.4213, - "hfopenllm_v2/MMLU-PRO": 0.2875 - } - }, - { - "id": "DreadPoor/Aspire-8B-model_stock", - "name": "Aspire-8B-model_stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7141, - "hfopenllm_v2/BBH": 0.5278, - "hfopenllm_v2/MATH Level 5": 0.1495, - "hfopenllm_v2/GPQA": 0.3146, - "hfopenllm_v2/MUSR": 0.4212, - "hfopenllm_v2/MMLU-PRO": 0.3763 - } - }, - { - "id": "DreadPoor/Aspire_1.3-8B_model-stock", - "name": "Aspire_1.3-8B_model-stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7062, - "hfopenllm_v2/BBH": 0.5302, - "hfopenllm_v2/MATH Level 5": 0.1692, - "hfopenllm_v2/GPQA": 0.3079, - "hfopenllm_v2/MUSR": 0.4105, - "hfopenllm_v2/MMLU-PRO": 0.3716 - } - }, - { - "id": "DreadPoor/Aspire_V2-8B-Model_Stock", - "name": "Aspire_V2-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7371, - "hfopenllm_v2/BBH": 0.533, - "hfopenllm_v2/MATH Level 5": 0.176, - "hfopenllm_v2/GPQA": 0.3205, - "hfopenllm_v2/MUSR": 0.3894, - "hfopenllm_v2/MMLU-PRO": 0.3697 - } - }, - { - "id": "DreadPoor/Aspire_V2.1-8B-Model_Stock", - "name": "Aspire_V2.1-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7238, - "hfopenllm_v2/BBH": 0.5236, - "hfopenllm_v2/MATH Level 5": 0.1767, - "hfopenllm_v2/GPQA": 0.3096, - "hfopenllm_v2/MUSR": 0.4136, - "hfopenllm_v2/MMLU-PRO": 0.3801 - } - }, - { - "id": "DreadPoor/Aspire_V2_ALT-8B-Model_Stock", - "name": "Aspire_V2_ALT-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7381, - "hfopenllm_v2/BBH": 0.5266, - "hfopenllm_v2/MATH Level 5": 0.173, - "hfopenllm_v2/GPQA": 0.3247, - "hfopenllm_v2/MUSR": 0.3975, - "hfopenllm_v2/MMLU-PRO": 0.3727 - } - }, - { - "id": "DreadPoor/Aspire_V2_ALT_ROW-8B-Model_Stock", - "name": "Aspire_V2_ALT_ROW-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7381, - "hfopenllm_v2/BBH": 0.5266, - "hfopenllm_v2/MATH Level 5": 0.173, - "hfopenllm_v2/GPQA": 0.3247, - "hfopenllm_v2/MUSR": 0.3975, - "hfopenllm_v2/MMLU-PRO": 0.3727 - } - }, - { - "id": "DreadPoor/Aspire_V3-8B-Model_Stock", - "name": "Aspire_V3-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5119, - "hfopenllm_v2/BBH": 0.5268, - "hfopenllm_v2/MATH Level 5": 0.1858, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.4015, - "hfopenllm_v2/MMLU-PRO": 0.3642 - } - }, - { - "id": "DreadPoor/Aspire_V4-8B-Model_Stock", - "name": "Aspire_V4-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7694, - "hfopenllm_v2/BBH": 0.5314, - "hfopenllm_v2/MATH Level 5": 0.1926, - "hfopenllm_v2/GPQA": 0.3045, - "hfopenllm_v2/MUSR": 0.3867, - "hfopenllm_v2/MMLU-PRO": 0.3708 - } - }, - { - "id": "DreadPoor/Aspire_V4_ALT-8B-Model_Stock", - "name": "Aspire_V4_ALT-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7366, - "hfopenllm_v2/BBH": 0.5268, - "hfopenllm_v2/MATH Level 5": 0.1813, - "hfopenllm_v2/GPQA": 0.3205, - "hfopenllm_v2/MUSR": 0.392, - "hfopenllm_v2/MMLU-PRO": 0.3682 - } - }, - { - "id": "DreadPoor/Asymmetric_Linearity-8B-Model_Stock", - "name": "Asymmetric_Linearity-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7174, - "hfopenllm_v2/BBH": 0.5465, - "hfopenllm_v2/MATH Level 5": 0.1647, - "hfopenllm_v2/GPQA": 0.3146, - "hfopenllm_v2/MUSR": 0.4199, - "hfopenllm_v2/MMLU-PRO": 0.3844 - } - }, - { - "id": "DreadPoor/Aurora_faustus-8B-LINEAR", - "name": "Aurora_faustus-8B-LINEAR", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7281, - "hfopenllm_v2/BBH": 0.5516, - "hfopenllm_v2/MATH Level 5": 0.1707, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.4146, - "hfopenllm_v2/MMLU-PRO": 0.3842 - } - }, - { - "id": "DreadPoor/Aurora_faustus-8B-LORABLATED", - "name": "Aurora_faustus-8B-LORABLATED", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7527, - "hfopenllm_v2/BBH": 0.5392, - "hfopenllm_v2/MATH Level 5": 0.1488, - "hfopenllm_v2/GPQA": 0.302, - "hfopenllm_v2/MUSR": 0.4239, - "hfopenllm_v2/MMLU-PRO": 0.3673 - } - }, - { - "id": "DreadPoor/Aurora_faustus-8B-LORABLATED_ALT", - "name": "Aurora_faustus-8B-LORABLATED_ALT", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7378, - "hfopenllm_v2/BBH": 0.5388, - "hfopenllm_v2/MATH Level 5": 0.1586, - "hfopenllm_v2/GPQA": 0.2987, - "hfopenllm_v2/MUSR": 0.4225, - "hfopenllm_v2/MMLU-PRO": 0.3694 - } - }, - { - "id": "DreadPoor/Autumn_Dawn-8B-LINEAR", - "name": "Autumn_Dawn-8B-LINEAR", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7293, - "hfopenllm_v2/BBH": 0.5459, - "hfopenllm_v2/MATH Level 5": 0.1858, - "hfopenllm_v2/GPQA": 0.2936, - "hfopenllm_v2/MUSR": 0.4186, - "hfopenllm_v2/MMLU-PRO": 0.3968 - } - }, - { - "id": "DreadPoor/BaeZel-8B-LINEAR", - "name": "BaeZel-8B-LINEAR", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7378, - "hfopenllm_v2/BBH": 0.5464, - "hfopenllm_v2/MATH Level 5": 0.1813, - "hfopenllm_v2/GPQA": 0.3213, - "hfopenllm_v2/MUSR": 0.4227, - "hfopenllm_v2/MMLU-PRO": 0.3861 - } - }, - { - "id": "DreadPoor/BaeZel-8B-Model_Stock", - "name": "BaeZel-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7713, - "hfopenllm_v2/BBH": 0.5408, - "hfopenllm_v2/MATH Level 5": 0.1639, - "hfopenllm_v2/GPQA": 0.3138, - "hfopenllm_v2/MUSR": 0.4199, - "hfopenllm_v2/MMLU-PRO": 0.388 - } - }, - { - "id": "DreadPoor/BaeZel_V2-8B-Model_Stock", - "name": "BaeZel_V2-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7677, - "hfopenllm_v2/BBH": 0.5374, - "hfopenllm_v2/MATH Level 5": 0.1798, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.4186, - "hfopenllm_v2/MMLU-PRO": 0.3947 - } - }, - { - "id": "DreadPoor/BaeZel_V2_ALT-8B-Model_Stock", - "name": "BaeZel_V2_ALT-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7677, - "hfopenllm_v2/BBH": 0.5374, - "hfopenllm_v2/MATH Level 5": 0.1798, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.4186, - "hfopenllm_v2/MMLU-PRO": 0.3947 - } - }, - { - "id": "DreadPoor/BaeZel_V3-8B-Model_Stock", - "name": "BaeZel_V3-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7832, - "hfopenllm_v2/BBH": 0.5392, - "hfopenllm_v2/MATH Level 5": 0.1896, - "hfopenllm_v2/GPQA": 0.3205, - "hfopenllm_v2/MUSR": 0.4174, - "hfopenllm_v2/MMLU-PRO": 0.3888 - } - }, - { - "id": "DreadPoor/Blunt_Edge-8B-SLERP", - "name": "Blunt_Edge-8B-SLERP", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7497, - "hfopenllm_v2/BBH": 0.5389, - "hfopenllm_v2/MATH Level 5": 0.1858, - "hfopenllm_v2/GPQA": 0.3112, - "hfopenllm_v2/MUSR": 0.4174, - "hfopenllm_v2/MMLU-PRO": 0.3767 - } - }, - { - "id": "DreadPoor/BulkUp", - "name": "BulkUp", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1778, - "hfopenllm_v2/BBH": 0.287, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2475, - "hfopenllm_v2/MUSR": 0.3447, - "hfopenllm_v2/MMLU-PRO": 0.111 - } - }, - { - "id": "DreadPoor/Cadence-8B-LINEAR", - "name": "Cadence-8B-LINEAR", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7682, - "hfopenllm_v2/BBH": 0.5433, - "hfopenllm_v2/MATH Level 5": 0.1677, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.4173, - "hfopenllm_v2/MMLU-PRO": 0.3803 - } - }, - { - "id": "DreadPoor/Caelid-8B-Model_Stock", - "name": "Caelid-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7247, - "hfopenllm_v2/BBH": 0.546, - "hfopenllm_v2/MATH Level 5": 0.1511, - "hfopenllm_v2/GPQA": 0.3104, - "hfopenllm_v2/MUSR": 0.4001, - "hfopenllm_v2/MMLU-PRO": 0.3816 - } - }, - { - "id": "DreadPoor/Casuar-9B-Model_Stock", - "name": "Casuar-9B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7765, - "hfopenllm_v2/BBH": 0.6107, - "hfopenllm_v2/MATH Level 5": 0.213, - "hfopenllm_v2/GPQA": 0.3448, - "hfopenllm_v2/MUSR": 0.4165, - "hfopenllm_v2/MMLU-PRO": 0.4156 - } - }, - { - "id": "DreadPoor/Condensed_Milk-8B-Model_Stock", - "name": "Condensed_Milk-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7536, - "hfopenllm_v2/BBH": 0.5435, - "hfopenllm_v2/MATH Level 5": 0.1745, - "hfopenllm_v2/GPQA": 0.3213, - "hfopenllm_v2/MUSR": 0.416, - "hfopenllm_v2/MMLU-PRO": 0.3876 - } - }, - { - "id": "DreadPoor/CoolerCoder-8B-LINEAR", - "name": "CoolerCoder-8B-LINEAR", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4519, - "hfopenllm_v2/BBH": 0.4762, - "hfopenllm_v2/MATH Level 5": 0.0793, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.3964, - "hfopenllm_v2/MMLU-PRO": 0.3159 - } - }, - { - "id": "DreadPoor/Damasteel-8B-LINEAR", - "name": "Damasteel-8B-LINEAR", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7384, - "hfopenllm_v2/BBH": 0.5388, - "hfopenllm_v2/MATH Level 5": 0.1669, - "hfopenllm_v2/GPQA": 0.2987, - "hfopenllm_v2/MUSR": 0.4212, - "hfopenllm_v2/MMLU-PRO": 0.3779 - } - }, - { - "id": "DreadPoor/Dearly_Beloved-8B-TIES", - "name": "Dearly_Beloved-8B-TIES", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8267, - "hfopenllm_v2/BBH": 0.405, - "hfopenllm_v2/MATH Level 5": 0.2115, - "hfopenllm_v2/GPQA": 0.2987, - "hfopenllm_v2/MUSR": 0.4175, - "hfopenllm_v2/MMLU-PRO": 0.2827 - } - }, - { - "id": "DreadPoor/Decayed-8B-LINEAR", - "name": "Decayed-8B-LINEAR", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7676, - "hfopenllm_v2/BBH": 0.5417, - "hfopenllm_v2/MATH Level 5": 0.1715, - "hfopenllm_v2/GPQA": 0.3096, - "hfopenllm_v2/MUSR": 0.4186, - "hfopenllm_v2/MMLU-PRO": 0.3763 - } - }, - { - "id": "DreadPoor/Derivative-8B-Model_Stock", - "name": "Derivative-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7667, - "hfopenllm_v2/BBH": 0.5395, - "hfopenllm_v2/MATH Level 5": 0.179, - "hfopenllm_v2/GPQA": 0.3171, - "hfopenllm_v2/MUSR": 0.42, - "hfopenllm_v2/MMLU-PRO": 0.3811 - } - }, - { - "id": "DreadPoor/Derivative_V2-8B-Model_Stock", - "name": "Derivative_V2-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7537, - "hfopenllm_v2/BBH": 0.5393, - "hfopenllm_v2/MATH Level 5": 0.1798, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.4123, - "hfopenllm_v2/MMLU-PRO": 0.3856 - } - }, - { - "id": "DreadPoor/Derivative_V2_ALT-8B-Model_Stock", - "name": "Derivative_V2_ALT-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.772, - "hfopenllm_v2/BBH": 0.5365, - "hfopenllm_v2/MATH Level 5": 0.1881, - "hfopenllm_v2/GPQA": 0.3112, - "hfopenllm_v2/MUSR": 0.4135, - "hfopenllm_v2/MMLU-PRO": 0.3882 - } - }, - { - "id": "DreadPoor/Derivative_V3-8B-Model_Stock", - "name": "Derivative_V3-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6964, - "hfopenllm_v2/BBH": 0.5243, - "hfopenllm_v2/MATH Level 5": 0.1465, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.415, - "hfopenllm_v2/MMLU-PRO": 0.3502 - } - }, - { - "id": "DreadPoor/Elusive_Dragon_Heart-8B-LINEAR", - "name": "Elusive_Dragon_Heart-8B-LINEAR", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7131, - "hfopenllm_v2/BBH": 0.5456, - "hfopenllm_v2/MATH Level 5": 0.148, - "hfopenllm_v2/GPQA": 0.3062, - "hfopenllm_v2/MUSR": 0.4146, - "hfopenllm_v2/MMLU-PRO": 0.3814 - } - }, - { - "id": "DreadPoor/Emu_Eggs-9B-Model_Stock", - "name": "Emu_Eggs-9B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7607, - "hfopenllm_v2/BBH": 0.6052, - "hfopenllm_v2/MATH Level 5": 0.21, - "hfopenllm_v2/GPQA": 0.3331, - "hfopenllm_v2/MUSR": 0.4071, - "hfopenllm_v2/MMLU-PRO": 0.4227 - } - }, - { - "id": "DreadPoor/Eunoia_Vespera-8B-LINEAR", - "name": "Eunoia_Vespera-8B-LINEAR", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7235, - "hfopenllm_v2/BBH": 0.5399, - "hfopenllm_v2/MATH Level 5": 0.1541, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.4185, - "hfopenllm_v2/MMLU-PRO": 0.3839 - } - }, - { - "id": "DreadPoor/felix_dies-mistral-7B-model_stock", - "name": "felix_dies-mistral-7B-model_stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3008, - "hfopenllm_v2/BBH": 0.4901, - "hfopenllm_v2/MATH Level 5": 0.0536, - "hfopenllm_v2/GPQA": 0.2919, - "hfopenllm_v2/MUSR": 0.4518, - "hfopenllm_v2/MMLU-PRO": 0.3109 - } - }, - { - "id": "DreadPoor/Fu_sion_HA-8B-SLERP", - "name": "Fu_sion_HA-8B-SLERP", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7609, - "hfopenllm_v2/BBH": 0.5373, - "hfopenllm_v2/MATH Level 5": 0.1752, - "hfopenllm_v2/GPQA": 0.323, - "hfopenllm_v2/MUSR": 0.416, - "hfopenllm_v2/MMLU-PRO": 0.3825 - } - }, - { - "id": "DreadPoor/H_the_eighth-8B-LINEAR", - "name": "H_the_eighth-8B-LINEAR", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7469, - "hfopenllm_v2/BBH": 0.5384, - "hfopenllm_v2/MATH Level 5": 0.1775, - "hfopenllm_v2/GPQA": 0.328, - "hfopenllm_v2/MUSR": 0.4173, - "hfopenllm_v2/MMLU-PRO": 0.3824 - } - }, - { - "id": "DreadPoor/hakuchido-8B-MODEL_STOCK", - "name": "hakuchido-8B-MODEL_STOCK", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7375, - "hfopenllm_v2/BBH": 0.5398, - "hfopenllm_v2/MATH Level 5": 0.1949, - "hfopenllm_v2/GPQA": 0.2953, - "hfopenllm_v2/MUSR": 0.4175, - "hfopenllm_v2/MMLU-PRO": 0.3782 - } - }, - { - "id": "DreadPoor/Happy_New_Year-8B-Model_Stock", - "name": "Happy_New_Year-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7616, - "hfopenllm_v2/BBH": 0.5368, - "hfopenllm_v2/MATH Level 5": 0.1594, - "hfopenllm_v2/GPQA": 0.3138, - "hfopenllm_v2/MUSR": 0.4186, - "hfopenllm_v2/MMLU-PRO": 0.3879 - } - }, - { - "id": "DreadPoor/Heart_Stolen-8B-Model_Stock", - "name": "Heart_Stolen-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7245, - "hfopenllm_v2/BBH": 0.5395, - "hfopenllm_v2/MATH Level 5": 0.1722, - "hfopenllm_v2/GPQA": 0.3171, - "hfopenllm_v2/MUSR": 0.4162, - "hfopenllm_v2/MMLU-PRO": 0.3794 - } - }, - { - "id": "DreadPoor/Heart_Stolen-ALT-8B-Model_Stock", - "name": "Heart_Stolen-ALT-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7184, - "hfopenllm_v2/BBH": 0.5263, - "hfopenllm_v2/MATH Level 5": 0.1563, - "hfopenllm_v2/GPQA": 0.3012, - "hfopenllm_v2/MUSR": 0.4055, - "hfopenllm_v2/MMLU-PRO": 0.3772 - } - }, - { - "id": "DreadPoor/Here_We_Go_Again-8B-SLERP", - "name": "Here_We_Go_Again-8B-SLERP", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7442, - "hfopenllm_v2/BBH": 0.546, - "hfopenllm_v2/MATH Level 5": 0.173, - "hfopenllm_v2/GPQA": 0.3188, - "hfopenllm_v2/MUSR": 0.4187, - "hfopenllm_v2/MMLU-PRO": 0.3873 - } - }, - { - "id": "DreadPoor/HOT_STINKING_GARBAGE", - "name": "HOT_STINKING_GARBAGE", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5754, - "hfopenllm_v2/BBH": 0.4884, - "hfopenllm_v2/MATH Level 5": 0.0672, - "hfopenllm_v2/GPQA": 0.2752, - "hfopenllm_v2/MUSR": 0.425, - "hfopenllm_v2/MMLU-PRO": 0.3017 - } - }, - { - "id": "DreadPoor/Howdy-8B-LINEAR", - "name": "Howdy-8B-LINEAR", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7378, - "hfopenllm_v2/BBH": 0.5384, - "hfopenllm_v2/MATH Level 5": 0.1775, - "hfopenllm_v2/GPQA": 0.3146, - "hfopenllm_v2/MUSR": 0.4121, - "hfopenllm_v2/MMLU-PRO": 0.3807 - } - }, - { - "id": "DreadPoor/ichor-8B-Model_Stock", - "name": "ichor-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5386, - "hfopenllm_v2/BBH": 0.5084, - "hfopenllm_v2/MATH Level 5": 0.1088, - "hfopenllm_v2/GPQA": 0.3238, - "hfopenllm_v2/MUSR": 0.4212, - "hfopenllm_v2/MMLU-PRO": 0.3151 - } - }, - { - "id": "DreadPoor/ichor_1.1-8B-Model_Stock", - "name": "ichor_1.1-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8096, - "hfopenllm_v2/BBH": 0.5281, - "hfopenllm_v2/MATH Level 5": 0.1775, - "hfopenllm_v2/GPQA": 0.3062, - "hfopenllm_v2/MUSR": 0.4068, - "hfopenllm_v2/MMLU-PRO": 0.3856 - } - }, - { - "id": "DreadPoor/Incidental-8B-Model_Stock", - "name": "Incidental-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7482, - "hfopenllm_v2/BBH": 0.5452, - "hfopenllm_v2/MATH Level 5": 0.1616, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.424, - "hfopenllm_v2/MMLU-PRO": 0.3873 - } - }, - { - "id": "DreadPoor/inexpertus-8B-Model_Stock", - "name": "inexpertus-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7795, - "hfopenllm_v2/BBH": 0.528, - "hfopenllm_v2/MATH Level 5": 0.1707, - "hfopenllm_v2/GPQA": 0.3096, - "hfopenllm_v2/MUSR": 0.4118, - "hfopenllm_v2/MMLU-PRO": 0.3791 - } - }, - { - "id": "DreadPoor/inexpertus_1.1-8B-LINEAR", - "name": "inexpertus_1.1-8B-LINEAR", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7527, - "hfopenllm_v2/BBH": 0.5525, - "hfopenllm_v2/MATH Level 5": 0.173, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.4173, - "hfopenllm_v2/MMLU-PRO": 0.3827 - } - }, - { - "id": "DreadPoor/inexpertus_1.2-8B-LINEAR", - "name": "inexpertus_1.2-8B-LINEAR", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7348, - "hfopenllm_v2/BBH": 0.5523, - "hfopenllm_v2/MATH Level 5": 0.1586, - "hfopenllm_v2/GPQA": 0.2953, - "hfopenllm_v2/MUSR": 0.4133, - "hfopenllm_v2/MMLU-PRO": 0.3788 - } - }, - { - "id": "DreadPoor/Irina-8B-model_stock", - "name": "Irina-8B-model_stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6799, - "hfopenllm_v2/BBH": 0.5237, - "hfopenllm_v2/MATH Level 5": 0.102, - "hfopenllm_v2/GPQA": 0.2844, - "hfopenllm_v2/MUSR": 0.4003, - "hfopenllm_v2/MMLU-PRO": 0.3574 - } - }, - { - "id": "DreadPoor/Kindling-8B-Model_Stock", - "name": "Kindling-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7308, - "hfopenllm_v2/BBH": 0.5492, - "hfopenllm_v2/MATH Level 5": 0.1752, - "hfopenllm_v2/GPQA": 0.318, - "hfopenllm_v2/MUSR": 0.4068, - "hfopenllm_v2/MMLU-PRO": 0.383 - } - }, - { - "id": "DreadPoor/L3.1-BaeZel-8B-Della", - "name": "L3.1-BaeZel-8B-Della", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.518, - "hfopenllm_v2/BBH": 0.5448, - "hfopenllm_v2/MATH Level 5": 0.1745, - "hfopenllm_v2/GPQA": 0.3196, - "hfopenllm_v2/MUSR": 0.42, - "hfopenllm_v2/MMLU-PRO": 0.3902 - } - }, - { - "id": "DreadPoor/Laughing_Stock-8B-Model_Stock", - "name": "Laughing_Stock-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.719, - "hfopenllm_v2/BBH": 0.5449, - "hfopenllm_v2/MATH Level 5": 0.1579, - "hfopenllm_v2/GPQA": 0.2894, - "hfopenllm_v2/MUSR": 0.4146, - "hfopenllm_v2/MMLU-PRO": 0.3764 - } - }, - { - "id": "DreadPoor/Lava_Lamp-8B-SLERP", - "name": "Lava_Lamp-8B-SLERP", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7381, - "hfopenllm_v2/BBH": 0.5368, - "hfopenllm_v2/MATH Level 5": 0.1737, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.4187, - "hfopenllm_v2/MMLU-PRO": 0.375 - } - }, - { - "id": "DreadPoor/LemonP-8B-Model_Stock", - "name": "LemonP-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7676, - "hfopenllm_v2/BBH": 0.5439, - "hfopenllm_v2/MATH Level 5": 0.1767, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.4081, - "hfopenllm_v2/MMLU-PRO": 0.4004 - } - }, - { - "id": "DreadPoor/Lydia_of_Whiterun-8B-LINEAR", - "name": "Lydia_of_Whiterun-8B-LINEAR", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7603, - "hfopenllm_v2/BBH": 0.538, - "hfopenllm_v2/MATH Level 5": 0.1767, - "hfopenllm_v2/GPQA": 0.3163, - "hfopenllm_v2/MUSR": 0.4251, - "hfopenllm_v2/MMLU-PRO": 0.3801 - } - }, - { - "id": "DreadPoor/Matryoshka-8B-LINEAR", - "name": "Matryoshka-8B-LINEAR", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7263, - "hfopenllm_v2/BBH": 0.5444, - "hfopenllm_v2/MATH Level 5": 0.1752, - "hfopenllm_v2/GPQA": 0.3205, - "hfopenllm_v2/MUSR": 0.4252, - "hfopenllm_v2/MMLU-PRO": 0.3866 - } - }, - { - "id": "DreadPoor/Mercury_In_Retrograde-8b-Model-Stock", - "name": "Mercury_In_Retrograde-8b-Model-Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7296, - "hfopenllm_v2/BBH": 0.5391, - "hfopenllm_v2/MATH Level 5": 0.1647, - "hfopenllm_v2/GPQA": 0.3163, - "hfopenllm_v2/MUSR": 0.4199, - "hfopenllm_v2/MMLU-PRO": 0.3829 - } - }, - { - "id": "DreadPoor/mergekit-nuslerp-nqzkedi", - "name": "mergekit-nuslerp-nqzkedi", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7765, - "hfopenllm_v2/BBH": 0.5362, - "hfopenllm_v2/MATH Level 5": 0.1881, - "hfopenllm_v2/GPQA": 0.3012, - "hfopenllm_v2/MUSR": 0.4225, - "hfopenllm_v2/MMLU-PRO": 0.3919 - } - }, - { - "id": "DreadPoor/Minthy-8B-Model_Stock", - "name": "Minthy-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7658, - "hfopenllm_v2/BBH": 0.5353, - "hfopenllm_v2/MATH Level 5": 0.1918, - "hfopenllm_v2/GPQA": 0.3037, - "hfopenllm_v2/MUSR": 0.4094, - "hfopenllm_v2/MMLU-PRO": 0.3993 - } - }, - { - "id": "DreadPoor/Minthy_ALT-8B-Model_Stock", - "name": "Minthy_ALT-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6992, - "hfopenllm_v2/BBH": 0.5375, - "hfopenllm_v2/MATH Level 5": 0.176, - "hfopenllm_v2/GPQA": 0.3062, - "hfopenllm_v2/MUSR": 0.4225, - "hfopenllm_v2/MMLU-PRO": 0.3674 - } - }, - { - "id": "DreadPoor/Minthy_V2-8B-Model_Stock", - "name": "Minthy_V2-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7126, - "hfopenllm_v2/BBH": 0.5491, - "hfopenllm_v2/MATH Level 5": 0.1594, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.4199, - "hfopenllm_v2/MMLU-PRO": 0.3737 - } - }, - { - "id": "DreadPoor/Minus_Penus-8B-Model_Stock", - "name": "Minus_Penus-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7311, - "hfopenllm_v2/BBH": 0.5344, - "hfopenllm_v2/MATH Level 5": 0.2002, - "hfopenllm_v2/GPQA": 0.3096, - "hfopenllm_v2/MUSR": 0.4019, - "hfopenllm_v2/MMLU-PRO": 0.3752 - } - }, - { - "id": "DreadPoor/Morphing-8B-Model_Stock", - "name": "Morphing-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7445, - "hfopenllm_v2/BBH": 0.5397, - "hfopenllm_v2/MATH Level 5": 0.1888, - "hfopenllm_v2/GPQA": 0.2936, - "hfopenllm_v2/MUSR": 0.4069, - "hfopenllm_v2/MMLU-PRO": 0.3852 - } - }, - { - "id": "DreadPoor/Not_Even_My_Final_Form-8B-Model_Stock", - "name": "Not_Even_My_Final_Form-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7722, - "hfopenllm_v2/BBH": 0.5351, - "hfopenllm_v2/MATH Level 5": 0.176, - "hfopenllm_v2/GPQA": 0.2953, - "hfopenllm_v2/MUSR": 0.4147, - "hfopenllm_v2/MMLU-PRO": 0.384 - } - }, - { - "id": "DreadPoor/Nother_One-8B-Model_Stock", - "name": "Nother_One-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6863, - "hfopenllm_v2/BBH": 0.5205, - "hfopenllm_v2/MATH Level 5": 0.1518, - "hfopenllm_v2/GPQA": 0.2894, - "hfopenllm_v2/MUSR": 0.387, - "hfopenllm_v2/MMLU-PRO": 0.3595 - } - }, - { - "id": "DreadPoor/Noxis-8B-LINEAR", - "name": "Noxis-8B-LINEAR", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6913, - "hfopenllm_v2/BBH": 0.5421, - "hfopenllm_v2/MATH Level 5": 0.1979, - "hfopenllm_v2/GPQA": 0.3188, - "hfopenllm_v2/MUSR": 0.4231, - "hfopenllm_v2/MMLU-PRO": 0.366 - } - }, - { - "id": "DreadPoor/Nullsworn-12B-LINEAR", - "name": "Nullsworn-12B-LINEAR", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4436, - "hfopenllm_v2/BBH": 0.5483, - "hfopenllm_v2/MATH Level 5": 0.1125, - "hfopenllm_v2/GPQA": 0.3079, - "hfopenllm_v2/MUSR": 0.435, - "hfopenllm_v2/MMLU-PRO": 0.3645 - } - }, - { - "id": "DreadPoor/Nwah-8B-Model_Stock", - "name": "Nwah-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7716, - "hfopenllm_v2/BBH": 0.5384, - "hfopenllm_v2/MATH Level 5": 0.1798, - "hfopenllm_v2/GPQA": 0.3104, - "hfopenllm_v2/MUSR": 0.4039, - "hfopenllm_v2/MMLU-PRO": 0.3807 - } - }, - { - "id": "DreadPoor/Oh_Boy-8B-LINEAR", - "name": "Oh_Boy-8B-LINEAR", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7503, - "hfopenllm_v2/BBH": 0.5375, - "hfopenllm_v2/MATH Level 5": 0.1782, - "hfopenllm_v2/GPQA": 0.3079, - "hfopenllm_v2/MUSR": 0.4108, - "hfopenllm_v2/MMLU-PRO": 0.3849 - } - }, - { - "id": "DreadPoor/ONeil-model_stock-8B", - "name": "ONeil-model_stock-8B", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6786, - "hfopenllm_v2/BBH": 0.5548, - "hfopenllm_v2/MATH Level 5": 0.1012, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.4173, - "hfopenllm_v2/MMLU-PRO": 0.3599 - } - }, - { - "id": "DreadPoor/OrangeJ-8B-Model_Stock", - "name": "OrangeJ-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7841, - "hfopenllm_v2/BBH": 0.5413, - "hfopenllm_v2/MATH Level 5": 0.176, - "hfopenllm_v2/GPQA": 0.3012, - "hfopenllm_v2/MUSR": 0.4028, - "hfopenllm_v2/MMLU-PRO": 0.3969 - } - }, - { - "id": "DreadPoor/Promissum_Mane-8B-LINEAR", - "name": "Promissum_Mane-8B-LINEAR", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.715, - "hfopenllm_v2/BBH": 0.5458, - "hfopenllm_v2/MATH Level 5": 0.1556, - "hfopenllm_v2/GPQA": 0.3045, - "hfopenllm_v2/MUSR": 0.42, - "hfopenllm_v2/MMLU-PRO": 0.3851 - } - }, - { - "id": "DreadPoor/Promissum_Mane-8B-LINEAR-lorablated", - "name": "Promissum_Mane-8B-LINEAR-lorablated", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7156, - "hfopenllm_v2/BBH": 0.5435, - "hfopenllm_v2/MATH Level 5": 0.1533, - "hfopenllm_v2/GPQA": 0.3037, - "hfopenllm_v2/MUSR": 0.4198, - "hfopenllm_v2/MMLU-PRO": 0.3739 - } - }, - { - "id": "DreadPoor/remember_to_breathe-8b-Model-Stock", - "name": "remember_to_breathe-8b-Model-Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7104, - "hfopenllm_v2/BBH": 0.5412, - "hfopenllm_v2/MATH Level 5": 0.1488, - "hfopenllm_v2/GPQA": 0.3012, - "hfopenllm_v2/MUSR": 0.4145, - "hfopenllm_v2/MMLU-PRO": 0.3761 - } - }, - { - "id": "DreadPoor/RPMash-8B-Model_Stock", - "name": "RPMash-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4564, - "hfopenllm_v2/BBH": 0.5169, - "hfopenllm_v2/MATH Level 5": 0.108, - "hfopenllm_v2/GPQA": 0.2869, - "hfopenllm_v2/MUSR": 0.4054, - "hfopenllm_v2/MMLU-PRO": 0.3604 - } - }, - { - "id": "DreadPoor/RPMash_V3-8B-Model_Stock", - "name": "RPMash_V3-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7049, - "hfopenllm_v2/BBH": 0.5217, - "hfopenllm_v2/MATH Level 5": 0.1042, - "hfopenllm_v2/GPQA": 0.3003, - "hfopenllm_v2/MUSR": 0.3778, - "hfopenllm_v2/MMLU-PRO": 0.3614 - } - }, - { - "id": "DreadPoor/Rusted_Gold-8B-LINEAR", - "name": "Rusted_Gold-8B-LINEAR", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7296, - "hfopenllm_v2/BBH": 0.5387, - "hfopenllm_v2/MATH Level 5": 0.1934, - "hfopenllm_v2/GPQA": 0.2987, - "hfopenllm_v2/MUSR": 0.4178, - "hfopenllm_v2/MMLU-PRO": 0.378 - } - }, - { - "id": "DreadPoor/Rusted_Platinum-8B-LINEAR", - "name": "Rusted_Platinum-8B-LINEAR", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.718, - "hfopenllm_v2/BBH": 0.5428, - "hfopenllm_v2/MATH Level 5": 0.1722, - "hfopenllm_v2/GPQA": 0.2802, - "hfopenllm_v2/MUSR": 0.3967, - "hfopenllm_v2/MMLU-PRO": 0.373 - } - }, - { - "id": "DreadPoor/Rusted_Platinum-8B-Model_Stock", - "name": "Rusted_Platinum-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4408, - "hfopenllm_v2/BBH": 0.5243, - "hfopenllm_v2/MATH Level 5": 0.102, - "hfopenllm_v2/GPQA": 0.2685, - "hfopenllm_v2/MUSR": 0.3741, - "hfopenllm_v2/MMLU-PRO": 0.3546 - } - }, - { - "id": "DreadPoor/Sellen-8B-model_stock", - "name": "Sellen-8B-model_stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7113, - "hfopenllm_v2/BBH": 0.5232, - "hfopenllm_v2/MATH Level 5": 0.1337, - "hfopenllm_v2/GPQA": 0.2743, - "hfopenllm_v2/MUSR": 0.396, - "hfopenllm_v2/MMLU-PRO": 0.357 - } - }, - { - "id": "DreadPoor/Something-8B-Model_Stock", - "name": "Something-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5043, - "hfopenllm_v2/BBH": 0.5395, - "hfopenllm_v2/MATH Level 5": 0.1798, - "hfopenllm_v2/GPQA": 0.3171, - "hfopenllm_v2/MUSR": 0.4187, - "hfopenllm_v2/MMLU-PRO": 0.3885 - } - }, - { - "id": "DreadPoor/Spring_Dusk-8B-SCE", - "name": "Spring_Dusk-8B-SCE", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6515, - "hfopenllm_v2/BBH": 0.5635, - "hfopenllm_v2/MATH Level 5": 0.0763, - "hfopenllm_v2/GPQA": 0.2878, - "hfopenllm_v2/MUSR": 0.46, - "hfopenllm_v2/MMLU-PRO": 0.3436 - } - }, - { - "id": "DreadPoor/Summer_Dawn-8B-SCE", - "name": "Summer_Dawn-8B-SCE", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6642, - "hfopenllm_v2/BBH": 0.5391, - "hfopenllm_v2/MATH Level 5": 0.1722, - "hfopenllm_v2/GPQA": 0.3003, - "hfopenllm_v2/MUSR": 0.412, - "hfopenllm_v2/MMLU-PRO": 0.3753 - } - }, - { - "id": "DreadPoor/Summer_Dusk-8B-TIES", - "name": "Summer_Dusk-8B-TIES", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4922, - "hfopenllm_v2/BBH": 0.536, - "hfopenllm_v2/MATH Level 5": 0.1805, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.4267, - "hfopenllm_v2/MMLU-PRO": 0.3856 - } - }, - { - "id": "DreadPoor/Summer_Rain-8B-SCE", - "name": "Summer_Rain-8B-SCE", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5459, - "hfopenllm_v2/BBH": 0.5846, - "hfopenllm_v2/MATH Level 5": 0.0702, - "hfopenllm_v2/GPQA": 0.3037, - "hfopenllm_v2/MUSR": 0.4477, - "hfopenllm_v2/MMLU-PRO": 0.3551 - } - }, - { - "id": "DreadPoor/Summer_Rain-8B-TIES", - "name": "Summer_Rain-8B-TIES", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5444, - "hfopenllm_v2/BBH": 0.5846, - "hfopenllm_v2/MATH Level 5": 0.0702, - "hfopenllm_v2/GPQA": 0.3037, - "hfopenllm_v2/MUSR": 0.4477, - "hfopenllm_v2/MMLU-PRO": 0.3551 - } - }, - { - "id": "DreadPoor/Sun-8B-Model_Stock", - "name": "Sun-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7758, - "hfopenllm_v2/BBH": 0.5264, - "hfopenllm_v2/MATH Level 5": 0.21, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.4098, - "hfopenllm_v2/MMLU-PRO": 0.3835 - } - }, - { - "id": "DreadPoor/Sweetened_Condensed_Milk-8B-Model_Stock", - "name": "Sweetened_Condensed_Milk-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7417, - "hfopenllm_v2/BBH": 0.5406, - "hfopenllm_v2/MATH Level 5": 0.1873, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.4107, - "hfopenllm_v2/MMLU-PRO": 0.3848 - } - }, - { - "id": "DreadPoor/test", - "name": "test", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4937, - "hfopenllm_v2/BBH": 0.5372, - "hfopenllm_v2/MATH Level 5": 0.1934, - "hfopenllm_v2/GPQA": 0.271, - "hfopenllm_v2/MUSR": 0.4351, - "hfopenllm_v2/MMLU-PRO": 0.3647 - } - }, - { - "id": "DreadPoor/TEST02-Ignore", - "name": "TEST02-Ignore", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6119, - "hfopenllm_v2/BBH": 0.5602, - "hfopenllm_v2/MATH Level 5": 0.0869, - "hfopenllm_v2/GPQA": 0.2844, - "hfopenllm_v2/MUSR": 0.4199, - "hfopenllm_v2/MMLU-PRO": 0.3468 - } - }, - { - "id": "DreadPoor/TEST03-ignore", - "name": "TEST03-ignore", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6967, - "hfopenllm_v2/BBH": 0.5383, - "hfopenllm_v2/MATH Level 5": 0.1654, - "hfopenllm_v2/GPQA": 0.3087, - "hfopenllm_v2/MUSR": 0.4186, - "hfopenllm_v2/MMLU-PRO": 0.3789 - } - }, - { - "id": "DreadPoor/TEST06-ignore", - "name": "TEST06-ignore", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7323, - "hfopenllm_v2/BBH": 0.5509, - "hfopenllm_v2/MATH Level 5": 0.1178, - "hfopenllm_v2/GPQA": 0.2869, - "hfopenllm_v2/MUSR": 0.4225, - "hfopenllm_v2/MMLU-PRO": 0.3615 - } - }, - { - "id": "DreadPoor/TEST07-ignore", - "name": "TEST07-ignore", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.74, - "hfopenllm_v2/BBH": 0.5561, - "hfopenllm_v2/MATH Level 5": 0.1662, - "hfopenllm_v2/GPQA": 0.3087, - "hfopenllm_v2/MUSR": 0.4094, - "hfopenllm_v2/MMLU-PRO": 0.388 - } - }, - { - "id": "DreadPoor/TEST08-ignore", - "name": "TEST08-ignore", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7467, - "hfopenllm_v2/BBH": 0.5454, - "hfopenllm_v2/MATH Level 5": 0.182, - "hfopenllm_v2/GPQA": 0.3129, - "hfopenllm_v2/MUSR": 0.4081, - "hfopenllm_v2/MMLU-PRO": 0.3853 - } - }, - { - "id": "DreadPoor/test_ALT", - "name": "test_ALT", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4997, - "hfopenllm_v2/BBH": 0.537, - "hfopenllm_v2/MATH Level 5": 0.1707, - "hfopenllm_v2/GPQA": 0.2693, - "hfopenllm_v2/MUSR": 0.4363, - "hfopenllm_v2/MMLU-PRO": 0.3492 - } - }, - { - "id": "DreadPoor/tests_pending-do_not_use_yet", - "name": "tests_pending-do_not_use_yet", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7691, - "hfopenllm_v2/BBH": 0.5408, - "hfopenllm_v2/MATH Level 5": 0.1979, - "hfopenllm_v2/GPQA": 0.297, - "hfopenllm_v2/MUSR": 0.4005, - "hfopenllm_v2/MMLU-PRO": 0.3827 - } - }, - { - "id": "DreadPoor/Trinas_Nectar-8B-model_stock", - "name": "Trinas_Nectar-8B-model_stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7259, - "hfopenllm_v2/BBH": 0.5256, - "hfopenllm_v2/MATH Level 5": 0.1526, - "hfopenllm_v2/GPQA": 0.2861, - "hfopenllm_v2/MUSR": 0.4068, - "hfopenllm_v2/MMLU-PRO": 0.3618 - } - }, - { - "id": "DreadPoor/UNTESTED-VENN_1.2-8B-Model_Stock", - "name": "UNTESTED-VENN_1.2-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4718, - "hfopenllm_v2/BBH": 0.5475, - "hfopenllm_v2/MATH Level 5": 0.1541, - "hfopenllm_v2/GPQA": 0.3154, - "hfopenllm_v2/MUSR": 0.4449, - "hfopenllm_v2/MMLU-PRO": 0.3787 - } - }, - { - "id": "DreadPoor/VENN_1.2-8B-Model_Stock", - "name": "VENN_1.2-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7226, - "hfopenllm_v2/BBH": 0.5459, - "hfopenllm_v2/MATH Level 5": 0.1707, - "hfopenllm_v2/GPQA": 0.297, - "hfopenllm_v2/MUSR": 0.42, - "hfopenllm_v2/MMLU-PRO": 0.3721 - } - }, - { - "id": "DreadPoor/Wannabe-8B-Model_Stock", - "name": "Wannabe-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7205, - "hfopenllm_v2/BBH": 0.539, - "hfopenllm_v2/MATH Level 5": 0.1775, - "hfopenllm_v2/GPQA": 0.3012, - "hfopenllm_v2/MUSR": 0.4135, - "hfopenllm_v2/MMLU-PRO": 0.3831 - } - }, - { - "id": "DreadPoor/What_A_Thrill-8B-Model_Stock", - "name": "What_A_Thrill-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7064, - "hfopenllm_v2/BBH": 0.5311, - "hfopenllm_v2/MATH Level 5": 0.182, - "hfopenllm_v2/GPQA": 0.297, - "hfopenllm_v2/MUSR": 0.408, - "hfopenllm_v2/MMLU-PRO": 0.3615 - } - }, - { - "id": "DreadPoor/Winter-8B-SCE", - "name": "Winter-8B-SCE", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7536, - "hfopenllm_v2/BBH": 0.5262, - "hfopenllm_v2/MATH Level 5": 0.1918, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.4071, - "hfopenllm_v2/MMLU-PRO": 0.3839 - } - }, - { - "id": "DreadPoor/Winter_Dawn-8B-TIES", - "name": "Winter_Dawn-8B-TIES", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5496, - "hfopenllm_v2/BBH": 0.5309, - "hfopenllm_v2/MATH Level 5": 0.1858, - "hfopenllm_v2/GPQA": 0.3096, - "hfopenllm_v2/MUSR": 0.4279, - "hfopenllm_v2/MMLU-PRO": 0.391 - } - }, - { - "id": "DreadPoor/Winter_Dusk-8B-TIES", - "name": "Winter_Dusk-8B-TIES", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7153, - "hfopenllm_v2/BBH": 0.4952, - "hfopenllm_v2/MATH Level 5": 0.0718, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.3688, - "hfopenllm_v2/MMLU-PRO": 0.3478 - } - }, - { - "id": "DreadPoor/Winter_Night-8B-Model_Stock", - "name": "Winter_Night-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.704, - "hfopenllm_v2/BBH": 0.5185, - "hfopenllm_v2/MATH Level 5": 0.1458, - "hfopenllm_v2/GPQA": 0.3062, - "hfopenllm_v2/MUSR": 0.3914, - "hfopenllm_v2/MMLU-PRO": 0.3666 - } - }, - { - "id": "DreadPoor/WIP-Acacia-8B-Model_Stock", - "name": "WIP-Acacia-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6246, - "hfopenllm_v2/BBH": 0.5195, - "hfopenllm_v2/MATH Level 5": 0.1669, - "hfopenllm_v2/GPQA": 0.3062, - "hfopenllm_v2/MUSR": 0.4226, - "hfopenllm_v2/MMLU-PRO": 0.3737 - } - }, - { - "id": "DreadPoor/WIP_Damascus-8B-TIES", - "name": "WIP_Damascus-8B-TIES", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4776, - "hfopenllm_v2/BBH": 0.5411, - "hfopenllm_v2/MATH Level 5": 0.1654, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.4119, - "hfopenllm_v2/MMLU-PRO": 0.3761 - } - }, - { - "id": "DreadPoor/Yafune-8B-Model_Stock", - "name": "Yafune-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7533, - "hfopenllm_v2/BBH": 0.5467, - "hfopenllm_v2/MATH Level 5": 0.1662, - "hfopenllm_v2/GPQA": 0.3272, - "hfopenllm_v2/MUSR": 0.4173, - "hfopenllm_v2/MMLU-PRO": 0.3851 - } - }, - { - "id": "DreadPoor/Yearn_V3-8B-Model_Stock", - "name": "Yearn_V3-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.729, - "hfopenllm_v2/BBH": 0.5322, - "hfopenllm_v2/MATH Level 5": 0.1896, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.3909, - "hfopenllm_v2/MMLU-PRO": 0.3802 - } - }, - { - "id": "DreadPoor/Zelus-8B-Model_Stock", - "name": "Zelus-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7788, - "hfopenllm_v2/BBH": 0.5307, - "hfopenllm_v2/MATH Level 5": 0.1647, - "hfopenllm_v2/GPQA": 0.3062, - "hfopenllm_v2/MUSR": 0.4214, - "hfopenllm_v2/MMLU-PRO": 0.3841 - } - }, - { - "id": "DreadPoor/Zelus_V2-8B-Model_Stock", - "name": "Zelus_V2-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7898, - "hfopenllm_v2/BBH": 0.5345, - "hfopenllm_v2/MATH Level 5": 0.2054, - "hfopenllm_v2/GPQA": 0.3096, - "hfopenllm_v2/MUSR": 0.3961, - "hfopenllm_v2/MMLU-PRO": 0.3833 - } - }, - { - "id": "DreadPoor/ZEUS-8B-V17-Abliterated_ALT", - "name": "ZEUS-8B-V17-Abliterated_ALT", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5511, - "hfopenllm_v2/BBH": 0.5231, - "hfopenllm_v2/MATH Level 5": 0.1903, - "hfopenllm_v2/GPQA": 0.3079, - "hfopenllm_v2/MUSR": 0.4149, - "hfopenllm_v2/MMLU-PRO": 0.389 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/dreamgen.json b/data/developers/dreamgen.json deleted file mode 100644 index 05126ced46ac0037000c9a12a2d2e993b783fadd..0000000000000000000000000000000000000000 --- a/data/developers/dreamgen.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "dreamgen", - "models": [ - { - "id": "dreamgen/WizardLM-2-7B", - "name": "WizardLM-2-7B", - "developer": "dreamgen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4583, - "hfopenllm_v2/BBH": 0.3487, - "hfopenllm_v2/MATH Level 5": 0.0332, - "hfopenllm_v2/GPQA": 0.2869, - "hfopenllm_v2/MUSR": 0.3941, - "hfopenllm_v2/MMLU-PRO": 0.266 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/drxd1000.json b/data/developers/drxd1000.json deleted file mode 100644 index 21836df503c7975786feb29506d0fd2ff71c7fa0..0000000000000000000000000000000000000000 --- a/data/developers/drxd1000.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "DRXD1000", - "models": [ - { - "id": "DRXD1000/Atlas-7B", - "name": "Atlas-7B", - "developer": "DRXD1000", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3704, - "hfopenllm_v2/BBH": 0.3302, - "hfopenllm_v2/MATH Level 5": 0.0189, - "hfopenllm_v2/GPQA": 0.2576, - "hfopenllm_v2/MUSR": 0.3342, - "hfopenllm_v2/MMLU-PRO": 0.1401 - } - }, - { - "id": "DRXD1000/Phoenix-7B", - "name": "Phoenix-7B", - "developer": "DRXD1000", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.321, - "hfopenllm_v2/BBH": 0.3932, - "hfopenllm_v2/MATH Level 5": 0.0166, - "hfopenllm_v2/GPQA": 0.2785, - "hfopenllm_v2/MUSR": 0.3849, - "hfopenllm_v2/MMLU-PRO": 0.2343 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/dual-gpo.json b/data/developers/dual-gpo.json deleted file mode 100644 index 8cb18864877c43a42607c65cff7a0e6df8aeac00..0000000000000000000000000000000000000000 --- a/data/developers/dual-gpo.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "DUAL-GPO", - "models": [ - { - "id": "DUAL-GPO/zephyr-7b-ipo-0k-15k-i1", - "name": "zephyr-7b-ipo-0k-15k-i1", - "developer": "DUAL-GPO", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2756, - "hfopenllm_v2/BBH": 0.4473, - "hfopenllm_v2/MATH Level 5": 0.0302, - "hfopenllm_v2/GPQA": 0.2911, - "hfopenllm_v2/MUSR": 0.4173, - "hfopenllm_v2/MMLU-PRO": 0.313 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/dustinwloring1988.json b/data/developers/dustinwloring1988.json deleted file mode 100644 index 95c00e3d31f713456520ba502375660b59ba6c39..0000000000000000000000000000000000000000 --- a/data/developers/dustinwloring1988.json +++ /dev/null @@ -1,103 +0,0 @@ -{ - "developer": "dustinwloring1988", - "models": [ - { - "id": "dustinwloring1988/Reflexis-8b-chat-v1", - "name": "Reflexis-8b-chat-v1", - "developer": "dustinwloring1988", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3658, - "hfopenllm_v2/BBH": 0.4664, - "hfopenllm_v2/MATH Level 5": 0.1156, - "hfopenllm_v2/GPQA": 0.2542, - "hfopenllm_v2/MUSR": 0.3754, - "hfopenllm_v2/MMLU-PRO": 0.3384 - } - }, - { - "id": "dustinwloring1988/Reflexis-8b-chat-v2", - "name": "Reflexis-8b-chat-v2", - "developer": "dustinwloring1988", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3912, - "hfopenllm_v2/BBH": 0.4724, - "hfopenllm_v2/MATH Level 5": 0.1163, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.3526, - "hfopenllm_v2/MMLU-PRO": 0.3378 - } - }, - { - "id": "dustinwloring1988/Reflexis-8b-chat-v3", - "name": "Reflexis-8b-chat-v3", - "developer": "dustinwloring1988", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5367, - "hfopenllm_v2/BBH": 0.4658, - "hfopenllm_v2/MATH Level 5": 0.1224, - "hfopenllm_v2/GPQA": 0.2424, - "hfopenllm_v2/MUSR": 0.3512, - "hfopenllm_v2/MMLU-PRO": 0.3548 - } - }, - { - "id": "dustinwloring1988/Reflexis-8b-chat-v4", - "name": "Reflexis-8b-chat-v4", - "developer": "dustinwloring1988", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4698, - "hfopenllm_v2/BBH": 0.4686, - "hfopenllm_v2/MATH Level 5": 0.1027, - "hfopenllm_v2/GPQA": 0.2341, - "hfopenllm_v2/MUSR": 0.3393, - "hfopenllm_v2/MMLU-PRO": 0.339 - } - }, - { - "id": "dustinwloring1988/Reflexis-8b-chat-v5", - "name": "Reflexis-8b-chat-v5", - "developer": "dustinwloring1988", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4238, - "hfopenllm_v2/BBH": 0.4782, - "hfopenllm_v2/MATH Level 5": 0.1216, - "hfopenllm_v2/GPQA": 0.271, - "hfopenllm_v2/MUSR": 0.3354, - "hfopenllm_v2/MMLU-PRO": 0.3217 - } - }, - { - "id": "dustinwloring1988/Reflexis-8b-chat-v6", - "name": "Reflexis-8b-chat-v6", - "developer": "dustinwloring1988", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4939, - "hfopenllm_v2/BBH": 0.481, - "hfopenllm_v2/MATH Level 5": 0.1299, - "hfopenllm_v2/GPQA": 0.2626, - "hfopenllm_v2/MUSR": 0.3753, - "hfopenllm_v2/MMLU-PRO": 0.3479 - } - }, - { - "id": "dustinwloring1988/Reflexis-8b-chat-v7", - "name": "Reflexis-8b-chat-v7", - "developer": "dustinwloring1988", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.398, - "hfopenllm_v2/BBH": 0.481, - "hfopenllm_v2/MATH Level 5": 0.1631, - "hfopenllm_v2/GPQA": 0.2617, - "hfopenllm_v2/MUSR": 0.3222, - "hfopenllm_v2/MMLU-PRO": 0.3643 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/duyhv1411.json b/data/developers/duyhv1411.json deleted file mode 100644 index 9a83aa2bbbe16700ebdbb1ef8cd4508cced5dc60..0000000000000000000000000000000000000000 --- a/data/developers/duyhv1411.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "duyhv1411", - "models": [ - { - "id": "duyhv1411/Llama-3.2-1B-en-vi", - "name": "Llama-3.2-1B-en-vi", - "developer": "duyhv1411", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4788, - "hfopenllm_v2/BBH": 0.3291, - "hfopenllm_v2/MATH Level 5": 0.0287, - "hfopenllm_v2/GPQA": 0.2768, - "hfopenllm_v2/MUSR": 0.3197, - "hfopenllm_v2/MMLU-PRO": 0.1341 - } - }, - { - "id": "duyhv1411/Llama-3.2-3B-en-vi", - "name": "Llama-3.2-3B-en-vi", - "developer": "duyhv1411", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4852, - "hfopenllm_v2/BBH": 0.3272, - "hfopenllm_v2/MATH Level 5": 0.0227, - "hfopenllm_v2/GPQA": 0.2752, - "hfopenllm_v2/MUSR": 0.321, - "hfopenllm_v2/MMLU-PRO": 0.1359 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/dwikitheduck.json b/data/developers/dwikitheduck.json deleted file mode 100644 index eb99106d7cc5453b2135a747e87d30df5a5952c6..0000000000000000000000000000000000000000 --- a/data/developers/dwikitheduck.json +++ /dev/null @@ -1,89 +0,0 @@ -{ - "developer": "dwikitheduck", - "models": [ - { - "id": "dwikitheduck/gemma-2-2b-id", - "name": "gemma-2-2b-id", - "developer": "dwikitheduck", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3879, - "hfopenllm_v2/BBH": 0.3962, - "hfopenllm_v2/MATH Level 5": 0.0453, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.4154, - "hfopenllm_v2/MMLU-PRO": 0.2173 - } - }, - { - "id": "dwikitheduck/gemma-2-2b-id-inst", - "name": "gemma-2-2b-id-inst", - "developer": "dwikitheduck", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3879, - "hfopenllm_v2/BBH": 0.3962, - "hfopenllm_v2/MATH Level 5": 0.0453, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.4154, - "hfopenllm_v2/MMLU-PRO": 0.2173 - } - }, - { - "id": "dwikitheduck/gemma-2-2b-id-instruct", - "name": "gemma-2-2b-id-instruct", - "developer": "dwikitheduck", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3879, - "hfopenllm_v2/BBH": 0.3962, - "hfopenllm_v2/MATH Level 5": 0.0453, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.4154, - "hfopenllm_v2/MMLU-PRO": 0.2173 - } - }, - { - "id": "dwikitheduck/gen-inst-1", - "name": "gen-inst-1", - "developer": "dwikitheduck", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.775, - "hfopenllm_v2/BBH": 0.642, - "hfopenllm_v2/MATH Level 5": 0.4554, - "hfopenllm_v2/GPQA": 0.3716, - "hfopenllm_v2/MUSR": 0.4205, - "hfopenllm_v2/MMLU-PRO": 0.5089 - } - }, - { - "id": "dwikitheduck/gen-try1", - "name": "gen-try1", - "developer": "dwikitheduck", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7522, - "hfopenllm_v2/BBH": 0.6359, - "hfopenllm_v2/MATH Level 5": 0.4101, - "hfopenllm_v2/GPQA": 0.3414, - "hfopenllm_v2/MUSR": 0.4416, - "hfopenllm_v2/MMLU-PRO": 0.5111 - } - }, - { - "id": "dwikitheduck/gen-try1-notemp", - "name": "gen-try1-notemp", - "developer": "dwikitheduck", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2627, - "hfopenllm_v2/BBH": 0.6263, - "hfopenllm_v2/MATH Level 5": 0.318, - "hfopenllm_v2/GPQA": 0.354, - "hfopenllm_v2/MUSR": 0.4714, - "hfopenllm_v2/MMLU-PRO": 0.521 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/dzakwan.json b/data/developers/dzakwan.json deleted file mode 100644 index 7df352cfc70a8a60b8af56b5f8ac2199d97a2a54..0000000000000000000000000000000000000000 --- a/data/developers/dzakwan.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "dzakwan", - "models": [ - { - "id": "dzakwan/dzakwan-MoE-4x7b-Beta", - "name": "dzakwan-MoE-4x7b-Beta", - "developer": "dzakwan", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4443, - "hfopenllm_v2/BBH": 0.514, - "hfopenllm_v2/MATH Level 5": 0.0778, - "hfopenllm_v2/GPQA": 0.2861, - "hfopenllm_v2/MUSR": 0.4267, - "hfopenllm_v2/MMLU-PRO": 0.3108 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/dzgas.json b/data/developers/dzgas.json deleted file mode 100644 index 533207e2f2eb79f315d16abfbdebb6768930c84e..0000000000000000000000000000000000000000 --- a/data/developers/dzgas.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "DZgas", - "models": [ - { - "id": "DZgas/GIGABATEMAN-7B", - "name": "GIGABATEMAN-7B", - "developer": "DZgas", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4607, - "hfopenllm_v2/BBH": 0.5032, - "hfopenllm_v2/MATH Level 5": 0.0551, - "hfopenllm_v2/GPQA": 0.2894, - "hfopenllm_v2/MUSR": 0.4328, - "hfopenllm_v2/MMLU-PRO": 0.3177 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/ece-ilab-prymmal.json b/data/developers/ece-ilab-prymmal.json deleted file mode 100644 index e19af4e3564183feba04bb48b33074e8d50df87d..0000000000000000000000000000000000000000 --- a/data/developers/ece-ilab-prymmal.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "ECE-ILAB-PRYMMAL", - "models": [ - { - "id": "ECE-ILAB-PRYMMAL/ILAB-Merging-3B-V2", - "name": "ILAB-Merging-3B-V2", - "developer": "ECE-ILAB-PRYMMAL", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4029, - "hfopenllm_v2/BBH": 0.5402, - "hfopenllm_v2/MATH Level 5": 0.1518, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.4332, - "hfopenllm_v2/MMLU-PRO": 0.3861 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/edgerunners.json b/data/developers/edgerunners.json deleted file mode 100644 index 4246bbe52b0a73a5b3b0b86aa437a799af1db85b..0000000000000000000000000000000000000000 --- a/data/developers/edgerunners.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "Edgerunners", - "models": [ - { - "id": "Edgerunners/meta-llama-3-8b-instruct-hf-ortho-baukit-34fail-3000total-bf16", - "name": "meta-llama-3-8b-instruct-hf-ortho-baukit-34fail-3000total-bf16", - "developer": "Edgerunners", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7147, - "hfopenllm_v2/BBH": 0.498, - "hfopenllm_v2/MATH Level 5": 0.0906, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3342, - "hfopenllm_v2/MMLU-PRO": 0.3636 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/ehristoforu.json b/data/developers/ehristoforu.json deleted file mode 100644 index b04f7a77f3c740f690b4cfab79b4df272b7d178e..0000000000000000000000000000000000000000 --- a/data/developers/ehristoforu.json +++ /dev/null @@ -1,509 +0,0 @@ -{ - "developer": "ehristoforu", - "models": [ - { - "id": "ehristoforu/coolqwen-3b-it", - "name": "coolqwen-3b-it", - "developer": "ehristoforu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6473, - "hfopenllm_v2/BBH": 0.4851, - "hfopenllm_v2/MATH Level 5": 0.3671, - "hfopenllm_v2/GPQA": 0.2827, - "hfopenllm_v2/MUSR": 0.4125, - "hfopenllm_v2/MMLU-PRO": 0.3601 - } - }, - { - "id": "ehristoforu/della-70b-test-v1", - "name": "della-70b-test-v1", - "developer": "ehristoforu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4979, - "hfopenllm_v2/BBH": 0.3029, - "hfopenllm_v2/MATH Level 5": 0.0098, - "hfopenllm_v2/GPQA": 0.2525, - "hfopenllm_v2/MUSR": 0.4555, - "hfopenllm_v2/MMLU-PRO": 0.1575 - } - }, - { - "id": "ehristoforu/Falcon3-8B-Franken-Basestruct", - "name": "Falcon3-8B-Franken-Basestruct", - "developer": "ehristoforu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1715, - "hfopenllm_v2/BBH": 0.5463, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.3406, - "hfopenllm_v2/MUSR": 0.3555, - "hfopenllm_v2/MMLU-PRO": 0.3947 - } - }, - { - "id": "ehristoforu/Falcon3-MoE-2x7B-Insruct", - "name": "Falcon3-MoE-2x7B-Insruct", - "developer": "ehristoforu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7643, - "hfopenllm_v2/BBH": 0.5648, - "hfopenllm_v2/MATH Level 5": 0.4124, - "hfopenllm_v2/GPQA": 0.3121, - "hfopenllm_v2/MUSR": 0.484, - "hfopenllm_v2/MMLU-PRO": 0.4095 - } - }, - { - "id": "ehristoforu/falcon3-ultraset", - "name": "falcon3-ultraset", - "developer": "ehristoforu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7135, - "hfopenllm_v2/BBH": 0.5584, - "hfopenllm_v2/MATH Level 5": 0.2122, - "hfopenllm_v2/GPQA": 0.3322, - "hfopenllm_v2/MUSR": 0.4853, - "hfopenllm_v2/MMLU-PRO": 0.3982 - } - }, - { - "id": "ehristoforu/fd-lora-merged-16x32", - "name": "fd-lora-merged-16x32", - "developer": "ehristoforu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3481, - "hfopenllm_v2/BBH": 0.3308, - "hfopenllm_v2/MATH Level 5": 0.1707, - "hfopenllm_v2/GPQA": 0.2534, - "hfopenllm_v2/MUSR": 0.3514, - "hfopenllm_v2/MMLU-PRO": 0.1205 - } - }, - { - "id": "ehristoforu/fd-lora-merged-64x128", - "name": "fd-lora-merged-64x128", - "developer": "ehristoforu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3281, - "hfopenllm_v2/BBH": 0.3345, - "hfopenllm_v2/MATH Level 5": 0.1873, - "hfopenllm_v2/GPQA": 0.255, - "hfopenllm_v2/MUSR": 0.3368, - "hfopenllm_v2/MMLU-PRO": 0.1537 - } - }, - { - "id": "ehristoforu/fp4-14b-it-v1", - "name": "fp4-14b-it-v1", - "developer": "ehristoforu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2535, - "hfopenllm_v2/BBH": 0.574, - "hfopenllm_v2/MATH Level 5": 0.0408, - "hfopenllm_v2/GPQA": 0.2953, - "hfopenllm_v2/MUSR": 0.3595, - "hfopenllm_v2/MMLU-PRO": 0.4205 - } - }, - { - "id": "ehristoforu/fp4-14b-v1-fix", - "name": "fp4-14b-v1-fix", - "developer": "ehristoforu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6742, - "hfopenllm_v2/BBH": 0.6817, - "hfopenllm_v2/MATH Level 5": 0.4207, - "hfopenllm_v2/GPQA": 0.354, - "hfopenllm_v2/MUSR": 0.4532, - "hfopenllm_v2/MMLU-PRO": 0.5353 - } - }, - { - "id": "ehristoforu/fq2.5-7b-it-normalize_false", - "name": "fq2.5-7b-it-normalize_false", - "developer": "ehristoforu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7399, - "hfopenllm_v2/BBH": 0.552, - "hfopenllm_v2/MATH Level 5": 0.4622, - "hfopenllm_v2/GPQA": 0.302, - "hfopenllm_v2/MUSR": 0.4612, - "hfopenllm_v2/MMLU-PRO": 0.4413 - } - }, - { - "id": "ehristoforu/fq2.5-7b-it-normalize_true", - "name": "fq2.5-7b-it-normalize_true", - "developer": "ehristoforu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7399, - "hfopenllm_v2/BBH": 0.552, - "hfopenllm_v2/MATH Level 5": 0.4622, - "hfopenllm_v2/GPQA": 0.302, - "hfopenllm_v2/MUSR": 0.4612, - "hfopenllm_v2/MMLU-PRO": 0.4413 - } - }, - { - "id": "ehristoforu/frqwen2.5-from7b-duable4layers-it", - "name": "frqwen2.5-from7b-duable4layers-it", - "developer": "ehristoforu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7729, - "hfopenllm_v2/BBH": 0.5264, - "hfopenllm_v2/MATH Level 5": 0.4509, - "hfopenllm_v2/GPQA": 0.2953, - "hfopenllm_v2/MUSR": 0.4166, - "hfopenllm_v2/MMLU-PRO": 0.4126 - } - }, - { - "id": "ehristoforu/frqwen2.5-from7b-it", - "name": "frqwen2.5-from7b-it", - "developer": "ehristoforu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6532, - "hfopenllm_v2/BBH": 0.5143, - "hfopenllm_v2/MATH Level 5": 0.2923, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.4086, - "hfopenllm_v2/MMLU-PRO": 0.3977 - } - }, - { - "id": "ehristoforu/Gemma2-9B-it-psy10k-mental_health", - "name": "Gemma2-9B-it-psy10k-mental_health", - "developer": "ehristoforu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5887, - "hfopenllm_v2/BBH": 0.5539, - "hfopenllm_v2/MATH Level 5": 0.1631, - "hfopenllm_v2/GPQA": 0.3372, - "hfopenllm_v2/MUSR": 0.4086, - "hfopenllm_v2/MMLU-PRO": 0.3829 - } - }, - { - "id": "ehristoforu/Gemma2-9b-it-train6", - "name": "Gemma2-9b-it-train6", - "developer": "ehristoforu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7025, - "hfopenllm_v2/BBH": 0.5898, - "hfopenllm_v2/MATH Level 5": 0.1911, - "hfopenllm_v2/GPQA": 0.3289, - "hfopenllm_v2/MUSR": 0.4084, - "hfopenllm_v2/MMLU-PRO": 0.3942 - } - }, - { - "id": "ehristoforu/HappyLlama1", - "name": "HappyLlama1", - "developer": "ehristoforu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7363, - "hfopenllm_v2/BBH": 0.4996, - "hfopenllm_v2/MATH Level 5": 0.1427, - "hfopenllm_v2/GPQA": 0.2836, - "hfopenllm_v2/MUSR": 0.4287, - "hfopenllm_v2/MMLU-PRO": 0.3546 - } - }, - { - "id": "ehristoforu/mllama-3.1-8b-instruct", - "name": "mllama-3.1-8b-instruct", - "developer": "ehristoforu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3458, - "hfopenllm_v2/BBH": 0.4718, - "hfopenllm_v2/MATH Level 5": 0.3776, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.338, - "hfopenllm_v2/MMLU-PRO": 0.2533 - } - }, - { - "id": "ehristoforu/mllama-3.1-8b-it", - "name": "mllama-3.1-8b-it", - "developer": "ehristoforu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3879, - "hfopenllm_v2/BBH": 0.4868, - "hfopenllm_v2/MATH Level 5": 0.3799, - "hfopenllm_v2/GPQA": 0.2768, - "hfopenllm_v2/MUSR": 0.3349, - "hfopenllm_v2/MMLU-PRO": 0.2622 - } - }, - { - "id": "ehristoforu/moremerge", - "name": "moremerge", - "developer": "ehristoforu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2019, - "hfopenllm_v2/BBH": 0.2868, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3566, - "hfopenllm_v2/MMLU-PRO": 0.1065 - } - }, - { - "id": "ehristoforu/moremerge-upscaled", - "name": "moremerge-upscaled", - "developer": "ehristoforu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1979, - "hfopenllm_v2/BBH": 0.2698, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2466, - "hfopenllm_v2/MUSR": 0.3593, - "hfopenllm_v2/MMLU-PRO": 0.1041 - } - }, - { - "id": "ehristoforu/phi-4-25b", - "name": "phi-4-25b", - "developer": "ehristoforu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6484, - "hfopenllm_v2/BBH": 0.6908, - "hfopenllm_v2/MATH Level 5": 0.4524, - "hfopenllm_v2/GPQA": 0.3188, - "hfopenllm_v2/MUSR": 0.4208, - "hfopenllm_v2/MMLU-PRO": 0.5351 - } - }, - { - "id": "ehristoforu/qwen2.5-test-32b-it", - "name": "qwen2.5-test-32b-it", - "developer": "ehristoforu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7889, - "hfopenllm_v2/BBH": 0.7081, - "hfopenllm_v2/MATH Level 5": 0.5974, - "hfopenllm_v2/GPQA": 0.3641, - "hfopenllm_v2/MUSR": 0.4578, - "hfopenllm_v2/MMLU-PRO": 0.5765 - } - }, - { - "id": "ehristoforu/qwen2.5-with-lora-think-3b-it", - "name": "qwen2.5-with-lora-think-3b-it", - "developer": "ehristoforu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5319, - "hfopenllm_v2/BBH": 0.4687, - "hfopenllm_v2/MATH Level 5": 0.2364, - "hfopenllm_v2/GPQA": 0.2802, - "hfopenllm_v2/MUSR": 0.431, - "hfopenllm_v2/MMLU-PRO": 0.3403 - } - }, - { - "id": "ehristoforu/QwenQwen2.5-7B-IT", - "name": "QwenQwen2.5-7B-IT", - "developer": "ehristoforu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7518, - "hfopenllm_v2/BBH": 0.5398, - "hfopenllm_v2/MATH Level 5": 0.5091, - "hfopenllm_v2/GPQA": 0.3037, - "hfopenllm_v2/MUSR": 0.4034, - "hfopenllm_v2/MMLU-PRO": 0.4289 - } - }, - { - "id": "ehristoforu/QwenQwen2.5-7B-IT-Dare", - "name": "QwenQwen2.5-7B-IT-Dare", - "developer": "ehristoforu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7509, - "hfopenllm_v2/BBH": 0.5398, - "hfopenllm_v2/MATH Level 5": 0.5091, - "hfopenllm_v2/GPQA": 0.3037, - "hfopenllm_v2/MUSR": 0.4034, - "hfopenllm_v2/MMLU-PRO": 0.4289 - } - }, - { - "id": "ehristoforu/rmoe-v1", - "name": "rmoe-v1", - "developer": "ehristoforu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.265, - "hfopenllm_v2/BBH": 0.2929, - "hfopenllm_v2/MATH Level 5": 0.0015, - "hfopenllm_v2/GPQA": 0.2584, - "hfopenllm_v2/MUSR": 0.3663, - "hfopenllm_v2/MMLU-PRO": 0.1125 - } - }, - { - "id": "ehristoforu/RQwen-v0.1", - "name": "RQwen-v0.1", - "developer": "ehristoforu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7625, - "hfopenllm_v2/BBH": 0.6446, - "hfopenllm_v2/MATH Level 5": 0.4645, - "hfopenllm_v2/GPQA": 0.3255, - "hfopenllm_v2/MUSR": 0.4139, - "hfopenllm_v2/MMLU-PRO": 0.5202 - } - }, - { - "id": "ehristoforu/RQwen-v0.2", - "name": "RQwen-v0.2", - "developer": "ehristoforu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7504, - "hfopenllm_v2/BBH": 0.6427, - "hfopenllm_v2/MATH Level 5": 0.327, - "hfopenllm_v2/GPQA": 0.3372, - "hfopenllm_v2/MUSR": 0.4207, - "hfopenllm_v2/MMLU-PRO": 0.5159 - } - }, - { - "id": "ehristoforu/rufalcon3-3b-it", - "name": "rufalcon3-3b-it", - "developer": "ehristoforu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5942, - "hfopenllm_v2/BBH": 0.4155, - "hfopenllm_v2/MATH Level 5": 0.1782, - "hfopenllm_v2/GPQA": 0.2727, - "hfopenllm_v2/MUSR": 0.3895, - "hfopenllm_v2/MMLU-PRO": 0.2348 - } - }, - { - "id": "ehristoforu/ruphi-4b", - "name": "ruphi-4b", - "developer": "ehristoforu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1752, - "hfopenllm_v2/BBH": 0.2906, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2399, - "hfopenllm_v2/MUSR": 0.3512, - "hfopenllm_v2/MMLU-PRO": 0.1126 - } - }, - { - "id": "ehristoforu/SoRu-0009", - "name": "SoRu-0009", - "developer": "ehristoforu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2582, - "hfopenllm_v2/BBH": 0.315, - "hfopenllm_v2/MATH Level 5": 0.0211, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.3369, - "hfopenllm_v2/MMLU-PRO": 0.1239 - } - }, - { - "id": "ehristoforu/testq-32b", - "name": "testq-32b", - "developer": "ehristoforu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1876, - "hfopenllm_v2/BBH": 0.2877, - "hfopenllm_v2/MATH Level 5": 0.003, - "hfopenllm_v2/GPQA": 0.2542, - "hfopenllm_v2/MUSR": 0.3715, - "hfopenllm_v2/MMLU-PRO": 0.1166 - } - }, - { - "id": "ehristoforu/tmoe", - "name": "tmoe", - "developer": "ehristoforu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1193, - "hfopenllm_v2/BBH": 0.3073, - "hfopenllm_v2/MATH Level 5": 0.0076, - "hfopenllm_v2/GPQA": 0.2232, - "hfopenllm_v2/MUSR": 0.3699, - "hfopenllm_v2/MMLU-PRO": 0.1191 - } - }, - { - "id": "ehristoforu/tmoe-v2", - "name": "tmoe-v2", - "developer": "ehristoforu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1903, - "hfopenllm_v2/BBH": 0.2897, - "hfopenllm_v2/MATH Level 5": 0.0023, - "hfopenllm_v2/GPQA": 0.2634, - "hfopenllm_v2/MUSR": 0.4151, - "hfopenllm_v2/MMLU-PRO": 0.11 - } - }, - { - "id": "ehristoforu/trd-7b-it", - "name": "trd-7b-it", - "developer": "ehristoforu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2185, - "hfopenllm_v2/BBH": 0.299, - "hfopenllm_v2/MATH Level 5": 0.0317, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.3794, - "hfopenllm_v2/MMLU-PRO": 0.1179 - } - }, - { - "id": "ehristoforu/ud-14b", - "name": "ud-14b", - "developer": "ehristoforu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4235, - "hfopenllm_v2/BBH": 0.3324, - "hfopenllm_v2/MATH Level 5": 0.1903, - "hfopenllm_v2/GPQA": 0.2374, - "hfopenllm_v2/MUSR": 0.4394, - "hfopenllm_v2/MMLU-PRO": 0.2415 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/eleutherai.json b/data/developers/eleutherai.json deleted file mode 100644 index f104cc6817fe7f506d98663412db91c3894ab990..0000000000000000000000000000000000000000 --- a/data/developers/eleutherai.json +++ /dev/null @@ -1,203 +0,0 @@ -{ - "developer": "EleutherAI", - "models": [ - { - "id": "EleutherAI/gpt-j-6b", - "name": "gpt-j-6b", - "developer": "EleutherAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2522, - "hfopenllm_v2/BBH": 0.3191, - "hfopenllm_v2/MATH Level 5": 0.0136, - "hfopenllm_v2/GPQA": 0.2458, - "hfopenllm_v2/MUSR": 0.3658, - "hfopenllm_v2/MMLU-PRO": 0.1241 - } - }, - { - "id": "EleutherAI/gpt-neo-1.3B", - "name": "gpt-neo-1.3B", - "developer": "EleutherAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2079, - "hfopenllm_v2/BBH": 0.3039, - "hfopenllm_v2/MATH Level 5": 0.0106, - "hfopenllm_v2/GPQA": 0.2559, - "hfopenllm_v2/MUSR": 0.3817, - "hfopenllm_v2/MMLU-PRO": 0.1164 - } - }, - { - "id": "EleutherAI/gpt-neo-125m", - "name": "gpt-neo-125m", - "developer": "EleutherAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1905, - "hfopenllm_v2/BBH": 0.3115, - "hfopenllm_v2/MATH Level 5": 0.006, - "hfopenllm_v2/GPQA": 0.2534, - "hfopenllm_v2/MUSR": 0.3593, - "hfopenllm_v2/MMLU-PRO": 0.1026 - } - }, - { - "id": "EleutherAI/gpt-neo-2.7B", - "name": "gpt-neo-2.7B", - "developer": "EleutherAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.259, - "hfopenllm_v2/BBH": 0.314, - "hfopenllm_v2/MATH Level 5": 0.0106, - "hfopenllm_v2/GPQA": 0.2659, - "hfopenllm_v2/MUSR": 0.3554, - "hfopenllm_v2/MMLU-PRO": 0.1163 - } - }, - { - "id": "EleutherAI/gpt-neox-20b", - "name": "gpt-neox-20b", - "developer": "EleutherAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2587, - "hfopenllm_v2/BBH": 0.3165, - "hfopenllm_v2/MATH Level 5": 0.0136, - "hfopenllm_v2/GPQA": 0.2433, - "hfopenllm_v2/MUSR": 0.3647, - "hfopenllm_v2/MMLU-PRO": 0.1155 - } - }, - { - "id": "EleutherAI/pythia-1.4b", - "name": "pythia-1.4b", - "developer": "EleutherAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2371, - "hfopenllm_v2/BBH": 0.315, - "hfopenllm_v2/MATH Level 5": 0.0151, - "hfopenllm_v2/GPQA": 0.2617, - "hfopenllm_v2/MUSR": 0.3538, - "hfopenllm_v2/MMLU-PRO": 0.1123 - } - }, - { - "id": "EleutherAI/pythia-12b", - "name": "Pythia 12B", - "developer": "EleutherAI", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.257, - "helm_classic/MMLU": 0.274, - "helm_classic/BoolQ": 0.662, - "helm_classic/NarrativeQA": 0.596, - "helm_classic/NaturalQuestions (open-book)": 0.581, - "helm_classic/QuAC": 0.313, - "helm_classic/HellaSwag": -1.0, - "helm_classic/OpenbookQA": -1.0, - "helm_classic/TruthfulQA": 0.177, - "helm_classic/MS MARCO (TREC)": -1.0, - "helm_classic/CNN/DailyMail": -1.0, - "helm_classic/XSUM": -1.0, - "helm_classic/IMDB": 0.931, - "helm_classic/CivilComments": 0.531, - "helm_classic/RAFT": 0.514, - "hfopenllm_v2/IFEval": 0.2471, - "hfopenllm_v2/BBH": 0.318, - "hfopenllm_v2/MATH Level 5": 0.0166, - "hfopenllm_v2/GPQA": 0.2466, - "hfopenllm_v2/MUSR": 0.3647, - "hfopenllm_v2/MMLU-PRO": 0.1109 - } - }, - { - "id": "EleutherAI/pythia-160m", - "name": "pythia-160m", - "developer": "EleutherAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1816, - "hfopenllm_v2/BBH": 0.297, - "hfopenllm_v2/MATH Level 5": 0.0091, - "hfopenllm_v2/GPQA": 0.2584, - "hfopenllm_v2/MUSR": 0.4179, - "hfopenllm_v2/MMLU-PRO": 0.112 - } - }, - { - "id": "EleutherAI/pythia-1b", - "name": "pythia-1b", - "developer": "EleutherAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2208, - "hfopenllm_v2/BBH": 0.3004, - "hfopenllm_v2/MATH Level 5": 0.0091, - "hfopenllm_v2/GPQA": 0.2567, - "hfopenllm_v2/MUSR": 0.3552, - "hfopenllm_v2/MMLU-PRO": 0.1136 - } - }, - { - "id": "EleutherAI/pythia-2.8b", - "name": "pythia-2.8b", - "developer": "EleutherAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2173, - "hfopenllm_v2/BBH": 0.3224, - "hfopenllm_v2/MATH Level 5": 0.0136, - "hfopenllm_v2/GPQA": 0.25, - "hfopenllm_v2/MUSR": 0.3486, - "hfopenllm_v2/MMLU-PRO": 0.1137 - } - }, - { - "id": "EleutherAI/pythia-410m", - "name": "pythia-410m", - "developer": "EleutherAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2195, - "hfopenllm_v2/BBH": 0.3028, - "hfopenllm_v2/MATH Level 5": 0.0098, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3578, - "hfopenllm_v2/MMLU-PRO": 0.1128 - } - }, - { - "id": "EleutherAI/pythia-6.9b", - "name": "Pythia 6.9B", - "developer": "EleutherAI", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.196, - "helm_classic/MMLU": 0.236, - "helm_classic/BoolQ": 0.631, - "helm_classic/NarrativeQA": 0.528, - "helm_classic/NaturalQuestions (open-book)": 0.539, - "helm_classic/QuAC": 0.296, - "helm_classic/HellaSwag": -1.0, - "helm_classic/OpenbookQA": -1.0, - "helm_classic/TruthfulQA": 0.213, - "helm_classic/MS MARCO (TREC)": -1.0, - "helm_classic/CNN/DailyMail": -1.0, - "helm_classic/XSUM": -1.0, - "helm_classic/IMDB": 0.928, - "helm_classic/CivilComments": 0.511, - "helm_classic/RAFT": 0.502, - "hfopenllm_v2/IFEval": 0.2281, - "hfopenllm_v2/BBH": 0.3232, - "hfopenllm_v2/MATH Level 5": 0.0144, - "hfopenllm_v2/GPQA": 0.2517, - "hfopenllm_v2/MUSR": 0.3591, - "hfopenllm_v2/MMLU-PRO": 0.1147 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/elinas.json b/data/developers/elinas.json deleted file mode 100644 index 653f6b1374f0460ec8082dd82f66e3a98606d82d..0000000000000000000000000000000000000000 --- a/data/developers/elinas.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "elinas", - "models": [ - { - "id": "elinas/Chronos-Gold-12B-1.0", - "name": "Chronos-Gold-12B-1.0", - "developer": "elinas", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3166, - "hfopenllm_v2/BBH": 0.5515, - "hfopenllm_v2/MATH Level 5": 0.0695, - "hfopenllm_v2/GPQA": 0.318, - "hfopenllm_v2/MUSR": 0.474, - "hfopenllm_v2/MMLU-PRO": 0.3518 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/ell44ot.json b/data/developers/ell44ot.json deleted file mode 100644 index 6cb8e6dc7481b51ca955f70a13a7130fea6e83d3..0000000000000000000000000000000000000000 --- a/data/developers/ell44ot.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "ell44ot", - "models": [ - { - "id": "ell44ot/gemma-2b-def", - "name": "gemma-2b-def", - "developer": "ell44ot", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2693, - "hfopenllm_v2/BBH": 0.3159, - "hfopenllm_v2/MATH Level 5": 0.0242, - "hfopenllm_v2/GPQA": 0.2735, - "hfopenllm_v2/MUSR": 0.367, - "hfopenllm_v2/MMLU-PRO": 0.1572 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/enno-ai.json b/data/developers/enno-ai.json deleted file mode 100644 index 81beef5d546a44b48c4a4a47dc0960691b5c38e9..0000000000000000000000000000000000000000 --- a/data/developers/enno-ai.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "developer": "Enno-Ai", - "models": [ - { - "id": "Enno-Ai/EnnoAi-Pro-French-Llama-3-8B-v0.4", - "name": "EnnoAi-Pro-French-Llama-3-8B-v0.4", - "developer": "Enno-Ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4189, - "hfopenllm_v2/BBH": 0.4075, - "hfopenllm_v2/MATH Level 5": 0.0363, - "hfopenllm_v2/GPQA": 0.271, - "hfopenllm_v2/MUSR": 0.417, - "hfopenllm_v2/MMLU-PRO": 0.2635 - } - }, - { - "id": "Enno-Ai/EnnoAi-Pro-Llama-3-8B", - "name": "EnnoAi-Pro-Llama-3-8B", - "developer": "Enno-Ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3195, - "hfopenllm_v2/BBH": 0.4152, - "hfopenllm_v2/MATH Level 5": 0.0219, - "hfopenllm_v2/GPQA": 0.2617, - "hfopenllm_v2/MUSR": 0.4071, - "hfopenllm_v2/MMLU-PRO": 0.2151 - } - }, - { - "id": "Enno-Ai/EnnoAi-Pro-Llama-3-8B-v0.3", - "name": "EnnoAi-Pro-Llama-3-8B-v0.3", - "developer": "Enno-Ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5083, - "hfopenllm_v2/BBH": 0.4101, - "hfopenllm_v2/MATH Level 5": 0.0483, - "hfopenllm_v2/GPQA": 0.2651, - "hfopenllm_v2/MUSR": 0.4236, - "hfopenllm_v2/MMLU-PRO": 0.299 - } - }, - { - "id": "Enno-Ai/EnnoAi-Pro-Llama-3.1-8B-v0.9", - "name": "EnnoAi-Pro-Llama-3.1-8B-v0.9", - "developer": "Enno-Ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4689, - "hfopenllm_v2/BBH": 0.416, - "hfopenllm_v2/MATH Level 5": 0.0378, - "hfopenllm_v2/GPQA": 0.2659, - "hfopenllm_v2/MUSR": 0.3832, - "hfopenllm_v2/MMLU-PRO": 0.2596 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/ennoai.json b/data/developers/ennoai.json deleted file mode 100644 index bb9a9b10584d6bf298bca59b742b8303b6f54bec..0000000000000000000000000000000000000000 --- a/data/developers/ennoai.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "EnnoAi", - "models": [ - { - "id": "EnnoAi/EnnoAi-7B-French-Instruct-202502", - "name": "EnnoAi-7B-French-Instruct-202502", - "developer": "EnnoAi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5564, - "hfopenllm_v2/BBH": 0.5575, - "hfopenllm_v2/MATH Level 5": 0.3724, - "hfopenllm_v2/GPQA": 0.2953, - "hfopenllm_v2/MUSR": 0.46, - "hfopenllm_v2/MMLU-PRO": 0.4013 - } - }, - { - "id": "EnnoAi/EnnoAi-Pro-Llama-3.1-8B-v1.0", - "name": "EnnoAi-Pro-Llama-3.1-8B-v1.0", - "developer": "EnnoAi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4704, - "hfopenllm_v2/BBH": 0.416, - "hfopenllm_v2/MATH Level 5": 0.0378, - "hfopenllm_v2/GPQA": 0.2659, - "hfopenllm_v2/MUSR": 0.3832, - "hfopenllm_v2/MMLU-PRO": 0.2596 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/epiculous.json b/data/developers/epiculous.json deleted file mode 100644 index c8128b33bb4fd177a979cf60b3ec6be0ca17039a..0000000000000000000000000000000000000000 --- a/data/developers/epiculous.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "developer": "Epiculous", - "models": [ - { - "id": "Epiculous/Azure_Dusk-v0.2", - "name": "Azure_Dusk-v0.2", - "developer": "Epiculous", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3467, - "hfopenllm_v2/BBH": 0.412, - "hfopenllm_v2/MATH Level 5": 0.0295, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.3835, - "hfopenllm_v2/MMLU-PRO": 0.3034 - } - }, - { - "id": "Epiculous/Crimson_Dawn-v0.2", - "name": "Crimson_Dawn-v0.2", - "developer": "Epiculous", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3103, - "hfopenllm_v2/BBH": 0.4482, - "hfopenllm_v2/MATH Level 5": 0.0431, - "hfopenllm_v2/GPQA": 0.276, - "hfopenllm_v2/MUSR": 0.4152, - "hfopenllm_v2/MMLU-PRO": 0.2721 - } - }, - { - "id": "Epiculous/NovaSpark", - "name": "NovaSpark", - "developer": "Epiculous", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6408, - "hfopenllm_v2/BBH": 0.5064, - "hfopenllm_v2/MATH Level 5": 0.1518, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.3882, - "hfopenllm_v2/MMLU-PRO": 0.3649 - } - }, - { - "id": "Epiculous/Violet_Twilight-v0.2", - "name": "Violet_Twilight-v0.2", - "developer": "Epiculous", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4532, - "hfopenllm_v2/BBH": 0.4615, - "hfopenllm_v2/MATH Level 5": 0.0287, - "hfopenllm_v2/GPQA": 0.2659, - "hfopenllm_v2/MUSR": 0.4299, - "hfopenllm_v2/MMLU-PRO": 0.3111 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/epistemeai.json b/data/developers/epistemeai.json deleted file mode 100644 index 11b25cd31eead1ba78e7fde536703493137e208b..0000000000000000000000000000000000000000 --- a/data/developers/epistemeai.json +++ /dev/null @@ -1,663 +0,0 @@ -{ - "developer": "EpistemeAI", - "models": [ - { - "id": "EpistemeAI/Alpaca-Llama3.1-8B", - "name": "Alpaca-Llama3.1-8B", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1599, - "hfopenllm_v2/BBH": 0.4755, - "hfopenllm_v2/MATH Level 5": 0.0506, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.3403, - "hfopenllm_v2/MMLU-PRO": 0.3246 - } - }, - { - "id": "EpistemeAI/Athena-gemma-2-2b-it", - "name": "Athena-gemma-2-2b-it", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3134, - "hfopenllm_v2/BBH": 0.4264, - "hfopenllm_v2/MATH Level 5": 0.0491, - "hfopenllm_v2/GPQA": 0.2685, - "hfopenllm_v2/MUSR": 0.4351, - "hfopenllm_v2/MMLU-PRO": 0.2422 - } - }, - { - "id": "EpistemeAI/Athena-gemma-2-2b-it-Philos", - "name": "Athena-gemma-2-2b-it-Philos", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4621, - "hfopenllm_v2/BBH": 0.3795, - "hfopenllm_v2/MATH Level 5": 0.037, - "hfopenllm_v2/GPQA": 0.281, - "hfopenllm_v2/MUSR": 0.4314, - "hfopenllm_v2/MMLU-PRO": 0.2248 - } - }, - { - "id": "EpistemeAI/Athene-codegemma-2-7b-it-alpaca-v1.3", - "name": "Athene-codegemma-2-7b-it-alpaca-v1.3", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.403, - "hfopenllm_v2/BBH": 0.4332, - "hfopenllm_v2/MATH Level 5": 0.0619, - "hfopenllm_v2/GPQA": 0.2802, - "hfopenllm_v2/MUSR": 0.4503, - "hfopenllm_v2/MMLU-PRO": 0.2587 - } - }, - { - "id": "EpistemeAI/DeepPhi-3.5-mini-instruct", - "name": "DeepPhi-3.5-mini-instruct", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1326, - "hfopenllm_v2/BBH": 0.2882, - "hfopenllm_v2/MATH Level 5": 0.0068, - "hfopenllm_v2/GPQA": 0.2332, - "hfopenllm_v2/MUSR": 0.3656, - "hfopenllm_v2/MMLU-PRO": 0.1103 - } - }, - { - "id": "EpistemeAI/DeepThinkers-Phi4", - "name": "DeepThinkers-Phi4", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.694, - "hfopenllm_v2/BBH": 0.679, - "hfopenllm_v2/MATH Level 5": 0.4585, - "hfopenllm_v2/GPQA": 0.3406, - "hfopenllm_v2/MUSR": 0.3981, - "hfopenllm_v2/MMLU-PRO": 0.5258 - } - }, - { - "id": "EpistemeAI/FineLlama3.1-8B-Instruct", - "name": "FineLlama3.1-8B-Instruct", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.08, - "hfopenllm_v2/BBH": 0.4557, - "hfopenllm_v2/MATH Level 5": 0.0347, - "hfopenllm_v2/GPQA": 0.2802, - "hfopenllm_v2/MUSR": 0.3482, - "hfopenllm_v2/MMLU-PRO": 0.3113 - } - }, - { - "id": "EpistemeAI/Fireball-12B", - "name": "Fireball-12B", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1834, - "hfopenllm_v2/BBH": 0.5111, - "hfopenllm_v2/MATH Level 5": 0.0408, - "hfopenllm_v2/GPQA": 0.2617, - "hfopenllm_v2/MUSR": 0.4236, - "hfopenllm_v2/MMLU-PRO": 0.3344 - } - }, - { - "id": "EpistemeAI/Fireball-12B-v1.13a-philosophers", - "name": "Fireball-12B-v1.13a-philosophers", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0876, - "hfopenllm_v2/BBH": 0.5103, - "hfopenllm_v2/MATH Level 5": 0.0461, - "hfopenllm_v2/GPQA": 0.3012, - "hfopenllm_v2/MUSR": 0.4081, - "hfopenllm_v2/MMLU-PRO": 0.3367 - } - }, - { - "id": "EpistemeAI/Fireball-Alpaca-Llama-3.1-8B-Philos-DPO-200", - "name": "Fireball-Alpaca-Llama-3.1-8B-Philos-DPO-200", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4577, - "hfopenllm_v2/BBH": 0.4838, - "hfopenllm_v2/MATH Level 5": 0.1231, - "hfopenllm_v2/GPQA": 0.3003, - "hfopenllm_v2/MUSR": 0.3945, - "hfopenllm_v2/MMLU-PRO": 0.3583 - } - }, - { - "id": "EpistemeAI/Fireball-Alpaca-Llama3.1.07-8B-Philos-Math-KTO-beta", - "name": "Fireball-Alpaca-Llama3.1.07-8B-Philos-Math-KTO-beta", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7274, - "hfopenllm_v2/BBH": 0.4865, - "hfopenllm_v2/MATH Level 5": 0.1526, - "hfopenllm_v2/GPQA": 0.2802, - "hfopenllm_v2/MUSR": 0.3619, - "hfopenllm_v2/MMLU-PRO": 0.3543 - } - }, - { - "id": "EpistemeAI/Fireball-Alpaca-Llama3.1.08-8B-Philos-C-R2", - "name": "Fireball-Alpaca-Llama3.1.08-8B-Philos-C-R2", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4673, - "hfopenllm_v2/BBH": 0.4932, - "hfopenllm_v2/MATH Level 5": 0.1239, - "hfopenllm_v2/GPQA": 0.2861, - "hfopenllm_v2/MUSR": 0.4624, - "hfopenllm_v2/MMLU-PRO": 0.3352 - } - }, - { - "id": "EpistemeAI/Fireball-Meta-Llama-3.1-8B-Instruct-0.001-128K-auto", - "name": "Fireball-Meta-Llama-3.1-8B-Instruct-0.001-128K-auto", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4432, - "hfopenllm_v2/BBH": 0.4824, - "hfopenllm_v2/MATH Level 5": 0.1329, - "hfopenllm_v2/GPQA": 0.3121, - "hfopenllm_v2/MUSR": 0.4066, - "hfopenllm_v2/MMLU-PRO": 0.3516 - } - }, - { - "id": "EpistemeAI/Fireball-Meta-Llama-3.1-8B-Instruct-Agent-0.003-128K", - "name": "Fireball-Meta-Llama-3.1-8B-Instruct-Agent-0.003-128K", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4457, - "hfopenllm_v2/BBH": 0.4897, - "hfopenllm_v2/MATH Level 5": 0.1208, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.3762, - "hfopenllm_v2/MMLU-PRO": 0.3543 - } - }, - { - "id": "EpistemeAI/Fireball-Meta-Llama-3.1-8B-Instruct-Agent-0.003-128K-code", - "name": "Fireball-Meta-Llama-3.1-8B-Instruct-Agent-0.003-128K-code", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5975, - "hfopenllm_v2/BBH": 0.4904, - "hfopenllm_v2/MATH Level 5": 0.1337, - "hfopenllm_v2/GPQA": 0.302, - "hfopenllm_v2/MUSR": 0.401, - "hfopenllm_v2/MMLU-PRO": 0.3423 - } - }, - { - "id": "EpistemeAI/Fireball-Meta-Llama-3.1-8B-Instruct-Agent-0.003-128K-code-ds", - "name": "Fireball-Meta-Llama-3.1-8B-Instruct-Agent-0.003-128K-code-ds", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6691, - "hfopenllm_v2/BBH": 0.4668, - "hfopenllm_v2/MATH Level 5": 0.1337, - "hfopenllm_v2/GPQA": 0.2727, - "hfopenllm_v2/MUSR": 0.3418, - "hfopenllm_v2/MMLU-PRO": 0.3389 - } - }, - { - "id": "EpistemeAI/Fireball-Meta-Llama-3.1-8B-Instruct-Agent-0.003-128K-code-ds-auto", - "name": "Fireball-Meta-Llama-3.1-8B-Instruct-Agent-0.003-128K-code-ds-auto", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7207, - "hfopenllm_v2/BBH": 0.461, - "hfopenllm_v2/MATH Level 5": 0.1314, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.3432, - "hfopenllm_v2/MMLU-PRO": 0.3354 - } - }, - { - "id": "EpistemeAI/Fireball-Meta-Llama-3.1-8B-Instruct-Agent-0.004-128K-code-COT", - "name": "Fireball-Meta-Llama-3.1-8B-Instruct-Agent-0.004-128K-code-COT", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4578, - "hfopenllm_v2/BBH": 0.4761, - "hfopenllm_v2/MATH Level 5": 0.1382, - "hfopenllm_v2/GPQA": 0.2936, - "hfopenllm_v2/MUSR": 0.3881, - "hfopenllm_v2/MMLU-PRO": 0.3471 - } - }, - { - "id": "EpistemeAI/Fireball-Meta-Llama-3.1-8B-Instruct-Agent-0.004-128K-code-ds-auto", - "name": "Fireball-Meta-Llama-3.1-8B-Instruct-Agent-0.004-128K-code-ds-auto", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7205, - "hfopenllm_v2/BBH": 0.4818, - "hfopenllm_v2/MATH Level 5": 0.1435, - "hfopenllm_v2/GPQA": 0.2483, - "hfopenllm_v2/MUSR": 0.33, - "hfopenllm_v2/MMLU-PRO": 0.3548 - } - }, - { - "id": "EpistemeAI/Fireball-Meta-Llama-3.1-8B-Instruct-Math", - "name": "Fireball-Meta-Llama-3.1-8B-Instruct-Math", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4623, - "hfopenllm_v2/BBH": 0.4983, - "hfopenllm_v2/MATH Level 5": 0.108, - "hfopenllm_v2/GPQA": 0.2911, - "hfopenllm_v2/MUSR": 0.3641, - "hfopenllm_v2/MMLU-PRO": 0.3331 - } - }, - { - "id": "EpistemeAI/Fireball-Meta-Llama-3.2-8B-Instruct-agent-003-128k-code-DPO", - "name": "Fireball-Meta-Llama-3.2-8B-Instruct-agent-003-128k-code-DPO", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4611, - "hfopenllm_v2/BBH": 0.4801, - "hfopenllm_v2/MATH Level 5": 0.1254, - "hfopenllm_v2/GPQA": 0.3003, - "hfopenllm_v2/MUSR": 0.3998, - "hfopenllm_v2/MMLU-PRO": 0.3521 - } - }, - { - "id": "EpistemeAI/Fireball-Mistral-Nemo-Base-2407-v1-DPO2", - "name": "Fireball-Mistral-Nemo-Base-2407-v1-DPO2", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1861, - "hfopenllm_v2/BBH": 0.4968, - "hfopenllm_v2/MATH Level 5": 0.0363, - "hfopenllm_v2/GPQA": 0.2919, - "hfopenllm_v2/MUSR": 0.404, - "hfopenllm_v2/MMLU-PRO": 0.3353 - } - }, - { - "id": "EpistemeAI/Fireball-R1-Llama-3.1-8B", - "name": "Fireball-R1-Llama-3.1-8B", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4427, - "hfopenllm_v2/BBH": 0.3643, - "hfopenllm_v2/MATH Level 5": 0.3112, - "hfopenllm_v2/GPQA": 0.2483, - "hfopenllm_v2/MUSR": 0.3288, - "hfopenllm_v2/MMLU-PRO": 0.1115 - } - }, - { - "id": "EpistemeAI/Fireball-R1-Llama-3.1-8B-Medical-COT", - "name": "Fireball-R1-Llama-3.1-8B-Medical-COT", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3216, - "hfopenllm_v2/BBH": 0.3716, - "hfopenllm_v2/MATH Level 5": 0.327, - "hfopenllm_v2/GPQA": 0.2743, - "hfopenllm_v2/MUSR": 0.3114, - "hfopenllm_v2/MMLU-PRO": 0.1402 - } - }, - { - "id": "EpistemeAI/Fireball-R1.1-Llama-3.1-8B", - "name": "Fireball-R1.1-Llama-3.1-8B", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3676, - "hfopenllm_v2/BBH": 0.3326, - "hfopenllm_v2/MATH Level 5": 0.1382, - "hfopenllm_v2/GPQA": 0.2517, - "hfopenllm_v2/MUSR": 0.3419, - "hfopenllm_v2/MMLU-PRO": 0.1115 - } - }, - { - "id": "EpistemeAI/Llama-3.2-3B-Agent007-Coder", - "name": "Llama-3.2-3B-Agent007-Coder", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.54, - "hfopenllm_v2/BBH": 0.4304, - "hfopenllm_v2/MATH Level 5": 0.111, - "hfopenllm_v2/GPQA": 0.2576, - "hfopenllm_v2/MUSR": 0.3668, - "hfopenllm_v2/MMLU-PRO": 0.2852 - } - }, - { - "id": "EpistemeAI/Mistral-Nemo-Instruct-12B-Philosophy-Math", - "name": "Mistral-Nemo-Instruct-12B-Philosophy-Math", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0695, - "hfopenllm_v2/BBH": 0.5365, - "hfopenllm_v2/MATH Level 5": 0.0959, - "hfopenllm_v2/GPQA": 0.3314, - "hfopenllm_v2/MUSR": 0.4292, - "hfopenllm_v2/MMLU-PRO": 0.3296 - } - }, - { - "id": "EpistemeAI/OpenReasoner-Llama-3.2-3B-rs1.0", - "name": "OpenReasoner-Llama-3.2-3B-rs1.0", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7274, - "hfopenllm_v2/BBH": 0.4519, - "hfopenllm_v2/MATH Level 5": 0.1344, - "hfopenllm_v2/GPQA": 0.2718, - "hfopenllm_v2/MUSR": 0.3461, - "hfopenllm_v2/MMLU-PRO": 0.3134 - } - }, - { - "id": "EpistemeAI/Polypsyche-Llama-3.1-8B-Instruct-Agent-0.003-128K-code-ds-auto-divergent", - "name": "Polypsyche-Llama-3.1-8B-Instruct-Agent-0.003-128K-code-ds-auto-divergent", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6915, - "hfopenllm_v2/BBH": 0.4525, - "hfopenllm_v2/MATH Level 5": 0.1292, - "hfopenllm_v2/GPQA": 0.2668, - "hfopenllm_v2/MUSR": 0.3578, - "hfopenllm_v2/MMLU-PRO": 0.329 - } - }, - { - "id": "EpistemeAI/Polypsyche-Llama-3.1-8B-Instruct-Agent-0.003-128K-code-ds-auto-Empathy", - "name": "Polypsyche-Llama-3.1-8B-Instruct-Agent-0.003-128K-code-ds-auto-Empathy", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7101, - "hfopenllm_v2/BBH": 0.4628, - "hfopenllm_v2/MATH Level 5": 0.1397, - "hfopenllm_v2/GPQA": 0.2768, - "hfopenllm_v2/MUSR": 0.3195, - "hfopenllm_v2/MMLU-PRO": 0.3311 - } - }, - { - "id": "EpistemeAI/Polypsyche-Llama-3.1-8B-Instruct-Agent-0.003-128K-code-ds-auto-Logic", - "name": "Polypsyche-Llama-3.1-8B-Instruct-Agent-0.003-128K-code-ds-auto-Logic", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7122, - "hfopenllm_v2/BBH": 0.4566, - "hfopenllm_v2/MATH Level 5": 0.1246, - "hfopenllm_v2/GPQA": 0.2844, - "hfopenllm_v2/MUSR": 0.3235, - "hfopenllm_v2/MMLU-PRO": 0.335 - } - }, - { - "id": "EpistemeAI/Reasoning-Llama-3.1-CoT-RE1-NMT", - "name": "Reasoning-Llama-3.1-CoT-RE1-NMT", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4829, - "hfopenllm_v2/BBH": 0.4736, - "hfopenllm_v2/MATH Level 5": 0.1299, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.3182, - "hfopenllm_v2/MMLU-PRO": 0.3343 - } - }, - { - "id": "EpistemeAI/Reasoning-Llama-3.1-CoT-RE1-NMT-V2-ORPO", - "name": "Reasoning-Llama-3.1-CoT-RE1-NMT-V2-ORPO", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4553, - "hfopenllm_v2/BBH": 0.4804, - "hfopenllm_v2/MATH Level 5": 0.1292, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.3931, - "hfopenllm_v2/MMLU-PRO": 0.3598 - } - }, - { - "id": "EpistemeAI/Reasoning-Llama-3.2-1B-Instruct-v1.2", - "name": "Reasoning-Llama-3.2-1B-Instruct-v1.2", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4087, - "hfopenllm_v2/BBH": 0.3324, - "hfopenllm_v2/MATH Level 5": 0.0506, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.3222, - "hfopenllm_v2/MMLU-PRO": 0.1179 - } - }, - { - "id": "EpistemeAI/Reasoning-Llama-3.2-1B-Instruct-v1.3", - "name": "Reasoning-Llama-3.2-1B-Instruct-v1.3", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3273, - "hfopenllm_v2/BBH": 0.3263, - "hfopenllm_v2/MATH Level 5": 0.0506, - "hfopenllm_v2/GPQA": 0.2584, - "hfopenllm_v2/MUSR": 0.326, - "hfopenllm_v2/MMLU-PRO": 0.1173 - } - }, - { - "id": "EpistemeAI/Reasoning-Llama-3.2-3B-Math-Instruct-RE1", - "name": "Reasoning-Llama-3.2-3B-Math-Instruct-RE1", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.512, - "hfopenllm_v2/BBH": 0.4381, - "hfopenllm_v2/MATH Level 5": 0.108, - "hfopenllm_v2/GPQA": 0.2643, - "hfopenllm_v2/MUSR": 0.3435, - "hfopenllm_v2/MMLU-PRO": 0.2789 - } - }, - { - "id": "EpistemeAI/Reasoning-Llama-3.2-3B-Math-Instruct-RE1-ORPO", - "name": "Reasoning-Llama-3.2-3B-Math-Instruct-RE1-ORPO", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.729, - "hfopenllm_v2/BBH": 0.4518, - "hfopenllm_v2/MATH Level 5": 0.1533, - "hfopenllm_v2/GPQA": 0.2735, - "hfopenllm_v2/MUSR": 0.3487, - "hfopenllm_v2/MMLU-PRO": 0.31 - } - }, - { - "id": "EpistemeAI/ReasoningCore-1.0-3B-Instruct-r01-Reflect-Math", - "name": "ReasoningCore-1.0-3B-Instruct-r01-Reflect-Math", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5903, - "hfopenllm_v2/BBH": 0.4364, - "hfopenllm_v2/MATH Level 5": 0.148, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3314, - "hfopenllm_v2/MMLU-PRO": 0.2823 - } - }, - { - "id": "EpistemeAI/ReasoningCore-3B-0", - "name": "ReasoningCore-3B-0", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7341, - "hfopenllm_v2/BBH": 0.4446, - "hfopenllm_v2/MATH Level 5": 0.1586, - "hfopenllm_v2/GPQA": 0.2727, - "hfopenllm_v2/MUSR": 0.3554, - "hfopenllm_v2/MMLU-PRO": 0.3172 - } - }, - { - "id": "EpistemeAI/ReasoningCore-3B-Instruct-r01-Reflect", - "name": "ReasoningCore-3B-Instruct-r01-Reflect", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7335, - "hfopenllm_v2/BBH": 0.445, - "hfopenllm_v2/MATH Level 5": 0.1541, - "hfopenllm_v2/GPQA": 0.2735, - "hfopenllm_v2/MUSR": 0.3527, - "hfopenllm_v2/MMLU-PRO": 0.3144 - } - }, - { - "id": "EpistemeAI/ReasoningCore-3B-R01", - "name": "ReasoningCore-3B-R01", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2976, - "hfopenllm_v2/BBH": 0.4373, - "hfopenllm_v2/MATH Level 5": 0.1299, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.3195, - "hfopenllm_v2/MMLU-PRO": 0.2591 - } - }, - { - "id": "EpistemeAI/ReasoningCore-3B-RE1-V2", - "name": "ReasoningCore-3B-RE1-V2", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7393, - "hfopenllm_v2/BBH": 0.4462, - "hfopenllm_v2/MATH Level 5": 0.1563, - "hfopenllm_v2/GPQA": 0.2735, - "hfopenllm_v2/MUSR": 0.3541, - "hfopenllm_v2/MMLU-PRO": 0.3181 - } - }, - { - "id": "EpistemeAI/ReasoningCore-3B-RE1-V2A", - "name": "ReasoningCore-3B-RE1-V2A", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5733, - "hfopenllm_v2/BBH": 0.419, - "hfopenllm_v2/MATH Level 5": 0.0929, - "hfopenllm_v2/GPQA": 0.2777, - "hfopenllm_v2/MUSR": 0.3352, - "hfopenllm_v2/MMLU-PRO": 0.2736 - } - }, - { - "id": "EpistemeAI/ReasoningCore-3B-RE1-V2B", - "name": "ReasoningCore-3B-RE1-V2B", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5051, - "hfopenllm_v2/BBH": 0.4168, - "hfopenllm_v2/MATH Level 5": 0.1073, - "hfopenllm_v2/GPQA": 0.2617, - "hfopenllm_v2/MUSR": 0.3448, - "hfopenllm_v2/MMLU-PRO": 0.2673 - } - }, - { - "id": "EpistemeAI/ReasoningCore-3B-RE1-V2C", - "name": "ReasoningCore-3B-RE1-V2C", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5057, - "hfopenllm_v2/BBH": 0.4177, - "hfopenllm_v2/MATH Level 5": 0.0974, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.3422, - "hfopenllm_v2/MMLU-PRO": 0.2691 - } - }, - { - "id": "EpistemeAI/ReasoningCore-3B-T1-V1", - "name": "ReasoningCore-3B-T1-V1", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7208, - "hfopenllm_v2/BBH": 0.4517, - "hfopenllm_v2/MATH Level 5": 0.1458, - "hfopenllm_v2/GPQA": 0.276, - "hfopenllm_v2/MUSR": 0.354, - "hfopenllm_v2/MMLU-PRO": 0.312 - } - }, - { - "id": "EpistemeAI/ReasoningCore-3B-T1_1", - "name": "ReasoningCore-3B-T1_1", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7275, - "hfopenllm_v2/BBH": 0.4524, - "hfopenllm_v2/MATH Level 5": 0.1541, - "hfopenllm_v2/GPQA": 0.276, - "hfopenllm_v2/MUSR": 0.3554, - "hfopenllm_v2/MMLU-PRO": 0.3117 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/epistemeai2.json b/data/developers/epistemeai2.json deleted file mode 100644 index da4444af114bf575cc00088ac628146148412404..0000000000000000000000000000000000000000 --- a/data/developers/epistemeai2.json +++ /dev/null @@ -1,215 +0,0 @@ -{ - "developer": "EpistemeAI2", - "models": [ - { - "id": "EpistemeAI2/Athene-codegemma-2-7b-it-alpaca-v1.2", - "name": "Athene-codegemma-2-7b-it-alpaca-v1.2", - "developer": "EpistemeAI2", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4351, - "hfopenllm_v2/BBH": 0.4175, - "hfopenllm_v2/MATH Level 5": 0.0423, - "hfopenllm_v2/GPQA": 0.271, - "hfopenllm_v2/MUSR": 0.417, - "hfopenllm_v2/MMLU-PRO": 0.2297 - } - }, - { - "id": "EpistemeAI2/Fireball-12B-v1.2", - "name": "Fireball-12B-v1.2", - "developer": "EpistemeAI2", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1355, - "hfopenllm_v2/BBH": 0.5019, - "hfopenllm_v2/MATH Level 5": 0.0415, - "hfopenllm_v2/GPQA": 0.2987, - "hfopenllm_v2/MUSR": 0.4173, - "hfopenllm_v2/MMLU-PRO": 0.3337 - } - }, - { - "id": "EpistemeAI2/Fireball-Alpaca-Llama3.1-8B-Philos", - "name": "Fireball-Alpaca-Llama3.1-8B-Philos", - "developer": "EpistemeAI2", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4986, - "hfopenllm_v2/BBH": 0.4978, - "hfopenllm_v2/MATH Level 5": 0.1186, - "hfopenllm_v2/GPQA": 0.2928, - "hfopenllm_v2/MUSR": 0.4277, - "hfopenllm_v2/MMLU-PRO": 0.3406 - } - }, - { - "id": "EpistemeAI2/Fireball-Alpaca-Llama3.1.01-8B-Philos", - "name": "Fireball-Alpaca-Llama3.1.01-8B-Philos", - "developer": "EpistemeAI2", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4212, - "hfopenllm_v2/BBH": 0.4956, - "hfopenllm_v2/MATH Level 5": 0.136, - "hfopenllm_v2/GPQA": 0.2886, - "hfopenllm_v2/MUSR": 0.4371, - "hfopenllm_v2/MMLU-PRO": 0.3383 - } - }, - { - "id": "EpistemeAI2/Fireball-Alpaca-Llama3.1.03-8B-Philos", - "name": "Fireball-Alpaca-Llama3.1.03-8B-Philos", - "developer": "EpistemeAI2", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3881, - "hfopenllm_v2/BBH": 0.4951, - "hfopenllm_v2/MATH Level 5": 0.1284, - "hfopenllm_v2/GPQA": 0.2785, - "hfopenllm_v2/MUSR": 0.428, - "hfopenllm_v2/MMLU-PRO": 0.3355 - } - }, - { - "id": "EpistemeAI2/Fireball-Alpaca-Llama3.1.04-8B-Philos", - "name": "Fireball-Alpaca-Llama3.1.04-8B-Philos", - "developer": "EpistemeAI2", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4084, - "hfopenllm_v2/BBH": 0.493, - "hfopenllm_v2/MATH Level 5": 0.1201, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.4372, - "hfopenllm_v2/MMLU-PRO": 0.3403 - } - }, - { - "id": "EpistemeAI2/Fireball-Alpaca-Llama3.1.06-8B-Philos-dpo", - "name": "Fireball-Alpaca-Llama3.1.06-8B-Philos-dpo", - "developer": "EpistemeAI2", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4866, - "hfopenllm_v2/BBH": 0.4881, - "hfopenllm_v2/MATH Level 5": 0.1307, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.3932, - "hfopenllm_v2/MMLU-PRO": 0.3615 - } - }, - { - "id": "EpistemeAI2/Fireball-Alpaca-Llama3.1.07-8B-Philos-Math", - "name": "Fireball-Alpaca-Llama3.1.07-8B-Philos-Math", - "developer": "EpistemeAI2", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5079, - "hfopenllm_v2/BBH": 0.4847, - "hfopenllm_v2/MATH Level 5": 0.1201, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.4063, - "hfopenllm_v2/MMLU-PRO": 0.3531 - } - }, - { - "id": "EpistemeAI2/Fireball-Alpaca-Llama3.1.08-8B-C-R1-KTO-Reflection", - "name": "Fireball-Alpaca-Llama3.1.08-8B-C-R1-KTO-Reflection", - "developer": "EpistemeAI2", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3952, - "hfopenllm_v2/BBH": 0.4955, - "hfopenllm_v2/MATH Level 5": 0.1246, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.4048, - "hfopenllm_v2/MMLU-PRO": 0.3593 - } - }, - { - "id": "EpistemeAI2/Fireball-Alpaca-Llama3.1.08-8B-Philos-C-R1", - "name": "Fireball-Alpaca-Llama3.1.08-8B-Philos-C-R1", - "developer": "EpistemeAI2", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5316, - "hfopenllm_v2/BBH": 0.4828, - "hfopenllm_v2/MATH Level 5": 0.1239, - "hfopenllm_v2/GPQA": 0.297, - "hfopenllm_v2/MUSR": 0.4103, - "hfopenllm_v2/MMLU-PRO": 0.3523 - } - }, - { - "id": "EpistemeAI2/Fireball-Llama-3.1-8B-Philos-Reflection", - "name": "Fireball-Llama-3.1-8B-Philos-Reflection", - "developer": "EpistemeAI2", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3596, - "hfopenllm_v2/BBH": 0.4898, - "hfopenllm_v2/MATH Level 5": 0.1284, - "hfopenllm_v2/GPQA": 0.3079, - "hfopenllm_v2/MUSR": 0.3957, - "hfopenllm_v2/MMLU-PRO": 0.3551 - } - }, - { - "id": "EpistemeAI2/Fireball-MathMistral-Nemo-Base-2407-v2dpo", - "name": "Fireball-MathMistral-Nemo-Base-2407-v2dpo", - "developer": "EpistemeAI2", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3097, - "hfopenllm_v2/BBH": 0.4328, - "hfopenllm_v2/MATH Level 5": 0.037, - "hfopenllm_v2/GPQA": 0.2634, - "hfopenllm_v2/MUSR": 0.403, - "hfopenllm_v2/MMLU-PRO": 0.1148 - } - }, - { - "id": "EpistemeAI2/Fireball-Meta-Llama-3.1-8B-Instruct-Agent-0.003-128K-code-math", - "name": "Fireball-Meta-Llama-3.1-8B-Instruct-Agent-0.003-128K-code-math", - "developer": "EpistemeAI2", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5515, - "hfopenllm_v2/BBH": 0.4808, - "hfopenllm_v2/MATH Level 5": 0.1352, - "hfopenllm_v2/GPQA": 0.3045, - "hfopenllm_v2/MUSR": 0.3693, - "hfopenllm_v2/MMLU-PRO": 0.342 - } - }, - { - "id": "EpistemeAI2/Fireball-Meta-Llama-3.1-8B-Instruct-Agent-0.005-128K-code-COT", - "name": "Fireball-Meta-Llama-3.1-8B-Instruct-Agent-0.005-128K-code-COT", - "developer": "EpistemeAI2", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4633, - "hfopenllm_v2/BBH": 0.4791, - "hfopenllm_v2/MATH Level 5": 0.1171, - "hfopenllm_v2/GPQA": 0.3121, - "hfopenllm_v2/MUSR": 0.3774, - "hfopenllm_v2/MMLU-PRO": 0.3565 - } - }, - { - "id": "EpistemeAI2/Fireball-Phi-3-medium-4k-inst-Philos", - "name": "Fireball-Phi-3-medium-4k-inst-Philos", - "developer": "EpistemeAI2", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5313, - "hfopenllm_v2/BBH": 0.6178, - "hfopenllm_v2/MATH Level 5": 0.1707, - "hfopenllm_v2/GPQA": 0.3322, - "hfopenllm_v2/MUSR": 0.4139, - "hfopenllm_v2/MMLU-PRO": 0.4599 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/eric111.json b/data/developers/eric111.json deleted file mode 100644 index 1258d247c8aad971653dcdf78b4107a103cd0ece..0000000000000000000000000000000000000000 --- a/data/developers/eric111.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "Eric111", - "models": [ - { - "id": "Eric111/CatunaMayo", - "name": "CatunaMayo", - "developer": "Eric111", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4074, - "hfopenllm_v2/BBH": 0.5244, - "hfopenllm_v2/MATH Level 5": 0.0846, - "hfopenllm_v2/GPQA": 0.2919, - "hfopenllm_v2/MUSR": 0.454, - "hfopenllm_v2/MMLU-PRO": 0.3178 - } - }, - { - "id": "Eric111/CatunaMayo-DPO", - "name": "CatunaMayo-DPO", - "developer": "Eric111", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4215, - "hfopenllm_v2/BBH": 0.5224, - "hfopenllm_v2/MATH Level 5": 0.0816, - "hfopenllm_v2/GPQA": 0.2919, - "hfopenllm_v2/MUSR": 0.445, - "hfopenllm_v2/MMLU-PRO": 0.317 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/etherll.json b/data/developers/etherll.json deleted file mode 100644 index 2be2455f8558b72fcbd342c72cc671c443f3155e..0000000000000000000000000000000000000000 --- a/data/developers/etherll.json +++ /dev/null @@ -1,117 +0,0 @@ -{ - "developer": "Etherll", - "models": [ - { - "id": "Etherll/Chocolatine-3B-Instruct-DPO-Revised-Ties", - "name": "Chocolatine-3B-Instruct-DPO-Revised-Ties", - "developer": "Etherll", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3725, - "hfopenllm_v2/BBH": 0.5411, - "hfopenllm_v2/MATH Level 5": 0.1631, - "hfopenllm_v2/GPQA": 0.3238, - "hfopenllm_v2/MUSR": 0.4649, - "hfopenllm_v2/MMLU-PRO": 0.3978 - } - }, - { - "id": "Etherll/Chocolatine-3B-Instruct-DPO-Revised-Ties-v2", - "name": "Chocolatine-3B-Instruct-DPO-Revised-Ties-v2", - "developer": "Etherll", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.374, - "hfopenllm_v2/BBH": 0.5411, - "hfopenllm_v2/MATH Level 5": 0.1631, - "hfopenllm_v2/GPQA": 0.3238, - "hfopenllm_v2/MUSR": 0.4649, - "hfopenllm_v2/MMLU-PRO": 0.3978 - } - }, - { - "id": "Etherll/Herplete-LLM-Llama-3.1-8b", - "name": "Herplete-LLM-Llama-3.1-8b", - "developer": "Etherll", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4672, - "hfopenllm_v2/BBH": 0.5013, - "hfopenllm_v2/MATH Level 5": 0.0279, - "hfopenllm_v2/GPQA": 0.2861, - "hfopenllm_v2/MUSR": 0.386, - "hfopenllm_v2/MMLU-PRO": 0.3482 - } - }, - { - "id": "Etherll/Herplete-LLM-Llama-3.1-8b-Ties", - "name": "Herplete-LLM-Llama-3.1-8b-Ties", - "developer": "Etherll", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6164, - "hfopenllm_v2/BBH": 0.5338, - "hfopenllm_v2/MATH Level 5": 0.1601, - "hfopenllm_v2/GPQA": 0.3171, - "hfopenllm_v2/MUSR": 0.4017, - "hfopenllm_v2/MMLU-PRO": 0.3752 - } - }, - { - "id": "Etherll/Qwen2.5-7B-della-test", - "name": "Qwen2.5-7B-della-test", - "developer": "Etherll", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7625, - "hfopenllm_v2/BBH": 0.5447, - "hfopenllm_v2/MATH Level 5": 0.4894, - "hfopenllm_v2/GPQA": 0.3087, - "hfopenllm_v2/MUSR": 0.4047, - "hfopenllm_v2/MMLU-PRO": 0.4361 - } - }, - { - "id": "Etherll/Qwen2.5-Coder-7B-Instruct-Ties", - "name": "Qwen2.5-Coder-7B-Instruct-Ties", - "developer": "Etherll", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5005, - "hfopenllm_v2/BBH": 0.4895, - "hfopenllm_v2/MATH Level 5": 0.2915, - "hfopenllm_v2/GPQA": 0.3297, - "hfopenllm_v2/MUSR": 0.4373, - "hfopenllm_v2/MMLU-PRO": 0.3503 - } - }, - { - "id": "Etherll/Replete-LLM-V3-Llama-3.1-8b", - "name": "Replete-LLM-V3-Llama-3.1-8b", - "developer": "Etherll", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5263, - "hfopenllm_v2/BBH": 0.4543, - "hfopenllm_v2/MATH Level 5": 0.2273, - "hfopenllm_v2/GPQA": 0.2685, - "hfopenllm_v2/MUSR": 0.3516, - "hfopenllm_v2/MMLU-PRO": 0.347 - } - }, - { - "id": "Etherll/SuperHermes", - "name": "SuperHermes", - "developer": "Etherll", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5459, - "hfopenllm_v2/BBH": 0.529, - "hfopenllm_v2/MATH Level 5": 0.1654, - "hfopenllm_v2/GPQA": 0.3238, - "hfopenllm_v2/MUSR": 0.44, - "hfopenllm_v2/MMLU-PRO": 0.3949 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/euclaise.json b/data/developers/euclaise.json deleted file mode 100644 index 93a139a4ab4cddc7cc9009dc8e0af3539662409f..0000000000000000000000000000000000000000 --- a/data/developers/euclaise.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "euclaise", - "models": [ - { - "id": "euclaise/ReMask-3B", - "name": "ReMask-3B", - "developer": "euclaise", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2419, - "hfopenllm_v2/BBH": 0.3517, - "hfopenllm_v2/MATH Level 5": 0.0196, - "hfopenllm_v2/GPQA": 0.2668, - "hfopenllm_v2/MUSR": 0.3341, - "hfopenllm_v2/MMLU-PRO": 0.1357 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/eurdem.json b/data/developers/eurdem.json deleted file mode 100644 index 09f01cf2fb5e4403a5b1d238151f0376eeb7f57f..0000000000000000000000000000000000000000 --- a/data/developers/eurdem.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "Eurdem", - "models": [ - { - "id": "Eurdem/Defne-llama3.1-8B", - "name": "Defne-llama3.1-8B", - "developer": "Eurdem", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5036, - "hfopenllm_v2/BBH": 0.5321, - "hfopenllm_v2/MATH Level 5": 0.1601, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.4331, - "hfopenllm_v2/MMLU-PRO": 0.3866 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/eva-unit-01.json b/data/developers/eva-unit-01.json deleted file mode 100644 index 95e29158f9545d9cb83c5139331a60df8a36ee69..0000000000000000000000000000000000000000 --- a/data/developers/eva-unit-01.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "EVA-UNIT-01", - "models": [ - { - "id": "EVA-UNIT-01/EVA-Qwen2.5-14B-v0.2", - "name": "EVA-Qwen2.5-14B-v0.2", - "developer": "EVA-UNIT-01", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4038, - "hfopenllm_v2/BBH": 0.609, - "hfopenllm_v2/MATH Level 5": 0.3406, - "hfopenllm_v2/GPQA": 0.3943, - "hfopenllm_v2/MUSR": 0.4794, - "hfopenllm_v2/MMLU-PRO": 0.5135 - } - }, - { - "id": "EVA-UNIT-01/EVA-Qwen2.5-72B-v0.2", - "name": "EVA-Qwen2.5-72B-v0.2", - "developer": "EVA-UNIT-01", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6879, - "hfopenllm_v2/BBH": 0.7088, - "hfopenllm_v2/MATH Level 5": 0.4313, - "hfopenllm_v2/GPQA": 0.4086, - "hfopenllm_v2/MUSR": 0.472, - "hfopenllm_v2/MMLU-PRO": 0.5813 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/eworojoshua.json b/data/developers/eworojoshua.json deleted file mode 100644 index 90484cd7279ed512276ecc4e78d37a5c24ae0289..0000000000000000000000000000000000000000 --- a/data/developers/eworojoshua.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "eworojoshua", - "models": [ - { - "id": "eworojoshua/vas-01", - "name": "vas-01", - "developer": "eworojoshua", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7612, - "hfopenllm_v2/BBH": 0.5418, - "hfopenllm_v2/MATH Level 5": 0.4736, - "hfopenllm_v2/GPQA": 0.3096, - "hfopenllm_v2/MUSR": 0.4432, - "hfopenllm_v2/MMLU-PRO": 0.4348 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/ewre324.json b/data/developers/ewre324.json deleted file mode 100644 index 0f308703f6255719e7d8fd86d3ca44cf1a7ac7e7..0000000000000000000000000000000000000000 --- a/data/developers/ewre324.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "developer": "ewre324", - "models": [ - { - "id": "ewre324/ewre324-R1-SmolLM2-135M-Distill", - "name": "ewre324-R1-SmolLM2-135M-Distill", - "developer": "ewre324", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1649, - "hfopenllm_v2/BBH": 0.3042, - "hfopenllm_v2/MATH Level 5": 0.0128, - "hfopenllm_v2/GPQA": 0.2617, - "hfopenllm_v2/MUSR": 0.3409, - "hfopenllm_v2/MMLU-PRO": 0.1134 - } - }, - { - "id": "ewre324/Thinker-Llama-3.2-3B-Instruct-Reasoning", - "name": "Thinker-Llama-3.2-3B-Instruct-Reasoning", - "developer": "ewre324", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4439, - "hfopenllm_v2/BBH": 0.4273, - "hfopenllm_v2/MATH Level 5": 0.0846, - "hfopenllm_v2/GPQA": 0.2768, - "hfopenllm_v2/MUSR": 0.3655, - "hfopenllm_v2/MMLU-PRO": 0.2886 - } - }, - { - "id": "ewre324/Thinker-Qwen2.5-0.5B-Instruct-Reasoning", - "name": "Thinker-Qwen2.5-0.5B-Instruct-Reasoning", - "developer": "ewre324", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2476, - "hfopenllm_v2/BBH": 0.3292, - "hfopenllm_v2/MATH Level 5": 0.0287, - "hfopenllm_v2/GPQA": 0.2852, - "hfopenllm_v2/MUSR": 0.3382, - "hfopenllm_v2/MMLU-PRO": 0.1647 - } - }, - { - "id": "ewre324/Thinker-SmolLM2-135M-Instruct-Reasoning", - "name": "Thinker-SmolLM2-135M-Instruct-Reasoning", - "developer": "ewre324", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2584, - "hfopenllm_v2/BBH": 0.3071, - "hfopenllm_v2/MATH Level 5": 0.0091, - "hfopenllm_v2/GPQA": 0.2525, - "hfopenllm_v2/MUSR": 0.3661, - "hfopenllm_v2/MMLU-PRO": 0.1094 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/experiment-llm.json b/data/developers/experiment-llm.json deleted file mode 100644 index fe679ed014b617a49cefb46b00c2228700dca44c..0000000000000000000000000000000000000000 --- a/data/developers/experiment-llm.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "experiment-llm", - "models": [ - { - "id": "experiment-llm/exp-3-q-r", - "name": "exp-3-q-r", - "developer": "experiment-llm", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6036, - "hfopenllm_v2/BBH": 0.5397, - "hfopenllm_v2/MATH Level 5": 0.2787, - "hfopenllm_v2/GPQA": 0.2936, - "hfopenllm_v2/MUSR": 0.4315, - "hfopenllm_v2/MMLU-PRO": 0.4316 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/facebook.json b/data/developers/facebook.json deleted file mode 100644 index 339d58ce2aefdf8a71a8fba045614f5fb23890d7..0000000000000000000000000000000000000000 --- a/data/developers/facebook.json +++ /dev/null @@ -1,59 +0,0 @@ -{ - "developer": "facebook", - "models": [ - { - "id": "facebook/opt-1.3b", - "name": "opt-1.3b", - "developer": "facebook", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2383, - "hfopenllm_v2/BBH": 0.3094, - "hfopenllm_v2/MATH Level 5": 0.0091, - "hfopenllm_v2/GPQA": 0.2424, - "hfopenllm_v2/MUSR": 0.342, - "hfopenllm_v2/MMLU-PRO": 0.1107 - } - }, - { - "id": "facebook/opt-30b", - "name": "opt-30b", - "developer": "facebook", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2453, - "hfopenllm_v2/BBH": 0.307, - "hfopenllm_v2/MATH Level 5": 0.0106, - "hfopenllm_v2/GPQA": 0.2693, - "hfopenllm_v2/MUSR": 0.3604, - "hfopenllm_v2/MMLU-PRO": 0.1164 - } - }, - { - "id": "facebook/Self-taught-evaluator-llama3.1-70B", - "name": "facebook/Self-taught-evaluator-llama3.1-70B", - "developer": "facebook", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.9001, - "reward-bench/Chat": 0.9693, - "reward-bench/Chat Hard": 0.8509, - "reward-bench/Safety": 0.8959, - "reward-bench/Reasoning": 0.8844 - } - }, - { - "id": "facebook/Self-taught-Llama-3-70B", - "name": "facebook/Self-taught-Llama-3-70B", - "developer": "facebook", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.8863, - "reward-bench/Chat": 0.9693, - "reward-bench/Chat Hard": 0.8399, - "reward-bench/Safety": 0.9108, - "reward-bench/Reasoning": 0.8251 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/failspy.json b/data/developers/failspy.json deleted file mode 100644 index 05dbbae9cacb27a89e0fb63e78a4d2526348acf5..0000000000000000000000000000000000000000 --- a/data/developers/failspy.json +++ /dev/null @@ -1,89 +0,0 @@ -{ - "developer": "failspy", - "models": [ - { - "id": "failspy/llama-3-70B-Instruct-abliterated", - "name": "llama-3-70B-Instruct-abliterated", - "developer": "failspy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8023, - "hfopenllm_v2/BBH": 0.6465, - "hfopenllm_v2/MATH Level 5": 0.2432, - "hfopenllm_v2/GPQA": 0.2894, - "hfopenllm_v2/MUSR": 0.4128, - "hfopenllm_v2/MMLU-PRO": 0.5145 - } - }, - { - "id": "failspy/Llama-3-8B-Instruct-abliterated", - "name": "Llama-3-8B-Instruct-abliterated", - "developer": "failspy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5909, - "hfopenllm_v2/BBH": 0.4354, - "hfopenllm_v2/MATH Level 5": 0.0385, - "hfopenllm_v2/GPQA": 0.276, - "hfopenllm_v2/MUSR": 0.4116, - "hfopenllm_v2/MMLU-PRO": 0.2742 - } - }, - { - "id": "failspy/Llama-3-8B-Instruct-MopeyMule", - "name": "Llama-3-8B-Instruct-MopeyMule", - "developer": "failspy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.675, - "hfopenllm_v2/BBH": 0.3839, - "hfopenllm_v2/MATH Level 5": 0.0196, - "hfopenllm_v2/GPQA": 0.2391, - "hfopenllm_v2/MUSR": 0.3513, - "hfopenllm_v2/MMLU-PRO": 0.1764 - } - }, - { - "id": "failspy/Meta-Llama-3-70B-Instruct-abliterated-v3.5", - "name": "Meta-Llama-3-70B-Instruct-abliterated-v3.5", - "developer": "failspy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7747, - "hfopenllm_v2/BBH": 0.5747, - "hfopenllm_v2/MATH Level 5": 0.1284, - "hfopenllm_v2/GPQA": 0.297, - "hfopenllm_v2/MUSR": 0.3982, - "hfopenllm_v2/MMLU-PRO": 0.4452 - } - }, - { - "id": "failspy/Meta-Llama-3-8B-Instruct-abliterated-v3", - "name": "Meta-Llama-3-8B-Instruct-abliterated-v3", - "developer": "failspy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7245, - "hfopenllm_v2/BBH": 0.4925, - "hfopenllm_v2/MATH Level 5": 0.0959, - "hfopenllm_v2/GPQA": 0.2643, - "hfopenllm_v2/MUSR": 0.3622, - "hfopenllm_v2/MMLU-PRO": 0.3654 - } - }, - { - "id": "failspy/Phi-3-medium-4k-instruct-abliterated-v3", - "name": "Phi-3-medium-4k-instruct-abliterated-v3", - "developer": "failspy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6319, - "hfopenllm_v2/BBH": 0.6305, - "hfopenllm_v2/MATH Level 5": 0.1594, - "hfopenllm_v2/GPQA": 0.3171, - "hfopenllm_v2/MUSR": 0.4604, - "hfopenllm_v2/MMLU-PRO": 0.44 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/fallenmerick.json b/data/developers/fallenmerick.json deleted file mode 100644 index 6b820742ad66046952037b8b731298080b8025de..0000000000000000000000000000000000000000 --- a/data/developers/fallenmerick.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "FallenMerick", - "models": [ - { - "id": "FallenMerick/Chewy-Lemon-Cookie-11B", - "name": "Chewy-Lemon-Cookie-11B", - "developer": "FallenMerick", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4875, - "hfopenllm_v2/BBH": 0.5251, - "hfopenllm_v2/MATH Level 5": 0.0544, - "hfopenllm_v2/GPQA": 0.2794, - "hfopenllm_v2/MUSR": 0.4546, - "hfopenllm_v2/MMLU-PRO": 0.3267 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/fblgit.json b/data/developers/fblgit.json deleted file mode 100644 index 6fc1e61b7aa2564e949656f946971c56834a0ca1..0000000000000000000000000000000000000000 --- a/data/developers/fblgit.json +++ /dev/null @@ -1,159 +0,0 @@ -{ - "developer": "fblgit", - "models": [ - { - "id": "fblgit/cybertron-v4-qw7B-MGS", - "name": "cybertron-v4-qw7B-MGS", - "developer": "fblgit", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6264, - "hfopenllm_v2/BBH": 0.5592, - "hfopenllm_v2/MATH Level 5": 0.3489, - "hfopenllm_v2/GPQA": 0.3104, - "hfopenllm_v2/MUSR": 0.4371, - "hfopenllm_v2/MMLU-PRO": 0.4473 - } - }, - { - "id": "fblgit/cybertron-v4-qw7B-UNAMGS", - "name": "cybertron-v4-qw7B-UNAMGS", - "developer": "fblgit", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.609, - "hfopenllm_v2/BBH": 0.5643, - "hfopenllm_v2/MATH Level 5": 0.3731, - "hfopenllm_v2/GPQA": 0.3314, - "hfopenllm_v2/MUSR": 0.4343, - "hfopenllm_v2/MMLU-PRO": 0.45 - } - }, - { - "id": "fblgit/juanako-7b-UNA", - "name": "juanako-7b-UNA", - "developer": "fblgit", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4837, - "hfopenllm_v2/BBH": 0.507, - "hfopenllm_v2/MATH Level 5": 0.034, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.4645, - "hfopenllm_v2/MMLU-PRO": 0.2771 - } - }, - { - "id": "fblgit/miniclaus-qw1.5B-UNAMGS", - "name": "miniclaus-qw1.5B-UNAMGS", - "developer": "fblgit", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3348, - "hfopenllm_v2/BBH": 0.4239, - "hfopenllm_v2/MATH Level 5": 0.1088, - "hfopenllm_v2/GPQA": 0.2919, - "hfopenllm_v2/MUSR": 0.4293, - "hfopenllm_v2/MMLU-PRO": 0.2937 - } - }, - { - "id": "fblgit/miniclaus-qw1.5B-UNAMGS-GRPO", - "name": "miniclaus-qw1.5B-UNAMGS-GRPO", - "developer": "fblgit", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3518, - "hfopenllm_v2/BBH": 0.4234, - "hfopenllm_v2/MATH Level 5": 0.1103, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.4254, - "hfopenllm_v2/MMLU-PRO": 0.2945 - } - }, - { - "id": "fblgit/pancho-v1-qw25-3B-UNAMGS", - "name": "pancho-v1-qw25-3B-UNAMGS", - "developer": "fblgit", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5361, - "hfopenllm_v2/BBH": 0.4926, - "hfopenllm_v2/MATH Level 5": 0.1571, - "hfopenllm_v2/GPQA": 0.297, - "hfopenllm_v2/MUSR": 0.4027, - "hfopenllm_v2/MMLU-PRO": 0.3766 - } - }, - { - "id": "fblgit/TheBeagle-v2beta-32B-MGS", - "name": "TheBeagle-v2beta-32B-MGS", - "developer": "fblgit", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5181, - "hfopenllm_v2/BBH": 0.7033, - "hfopenllm_v2/MATH Level 5": 0.4947, - "hfopenllm_v2/GPQA": 0.3826, - "hfopenllm_v2/MUSR": 0.5008, - "hfopenllm_v2/MMLU-PRO": 0.5915 - } - }, - { - "id": "fblgit/una-cybertron-7b-v2-bf16", - "name": "una-cybertron-7b-v2-bf16", - "developer": "fblgit", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4737, - "hfopenllm_v2/BBH": 0.3973, - "hfopenllm_v2/MATH Level 5": 0.0408, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.4473, - "hfopenllm_v2/MMLU-PRO": 0.2443 - } - }, - { - "id": "fblgit/UNA-SimpleSmaug-34b-v1beta", - "name": "UNA-SimpleSmaug-34b-v1beta", - "developer": "fblgit", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4556, - "hfopenllm_v2/BBH": 0.5287, - "hfopenllm_v2/MATH Level 5": 0.0718, - "hfopenllm_v2/GPQA": 0.3171, - "hfopenllm_v2/MUSR": 0.4256, - "hfopenllm_v2/MMLU-PRO": 0.454 - } - }, - { - "id": "fblgit/UNA-TheBeagle-7b-v1", - "name": "UNA-TheBeagle-7b-v1", - "developer": "fblgit", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3689, - "hfopenllm_v2/BBH": 0.5029, - "hfopenllm_v2/MATH Level 5": 0.077, - "hfopenllm_v2/GPQA": 0.2844, - "hfopenllm_v2/MUSR": 0.4564, - "hfopenllm_v2/MMLU-PRO": 0.3019 - } - }, - { - "id": "fblgit/UNA-ThePitbull-21.4B-v2", - "name": "UNA-ThePitbull-21.4B-v2", - "developer": "fblgit", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.379, - "hfopenllm_v2/BBH": 0.635, - "hfopenllm_v2/MATH Level 5": 0.1216, - "hfopenllm_v2/GPQA": 0.302, - "hfopenllm_v2/MUSR": 0.3922, - "hfopenllm_v2/MMLU-PRO": 0.3516 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/felladrin.json b/data/developers/felladrin.json deleted file mode 100644 index 0ed0c980df3982a097fdb38c763aa3c946af3e34..0000000000000000000000000000000000000000 --- a/data/developers/felladrin.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "Felladrin", - "models": [ - { - "id": "Felladrin/Llama-160M-Chat-v1", - "name": "Llama-160M-Chat-v1", - "developer": "Felladrin", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1575, - "hfopenllm_v2/BBH": 0.3036, - "hfopenllm_v2/MATH Level 5": 0.006, - "hfopenllm_v2/GPQA": 0.2576, - "hfopenllm_v2/MUSR": 0.3661, - "hfopenllm_v2/MMLU-PRO": 0.1136 - } - }, - { - "id": "Felladrin/Minueza-32M-UltraChat", - "name": "Minueza-32M-UltraChat", - "developer": "Felladrin", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1376, - "hfopenllm_v2/BBH": 0.2941, - "hfopenllm_v2/MATH Level 5": 0.0045, - "hfopenllm_v2/GPQA": 0.2559, - "hfopenllm_v2/MUSR": 0.3742, - "hfopenllm_v2/MMLU-PRO": 0.1133 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/fhai50032.json b/data/developers/fhai50032.json deleted file mode 100644 index 018ad4362c945f75a3b06b06adb8c08393c3d413..0000000000000000000000000000000000000000 --- a/data/developers/fhai50032.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "fhai50032", - "models": [ - { - "id": "fhai50032/RolePlayLake-7B", - "name": "RolePlayLake-7B", - "developer": "fhai50032", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5057, - "hfopenllm_v2/BBH": 0.5252, - "hfopenllm_v2/MATH Level 5": 0.0725, - "hfopenllm_v2/GPQA": 0.3037, - "hfopenllm_v2/MUSR": 0.4459, - "hfopenllm_v2/MMLU-PRO": 0.316 - } - }, - { - "id": "fhai50032/Unaligned-Thinker-PHI-4", - "name": "Unaligned-Thinker-PHI-4", - "developer": "fhai50032", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0563, - "hfopenllm_v2/BBH": 0.6643, - "hfopenllm_v2/MATH Level 5": 0.3353, - "hfopenllm_v2/GPQA": 0.3809, - "hfopenllm_v2/MUSR": 0.4679, - "hfopenllm_v2/MMLU-PRO": 0.5147 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/fingu-ai.json b/data/developers/fingu-ai.json deleted file mode 100644 index a9cb10b2300b37999737dcd9dee254ff2a6ae446..0000000000000000000000000000000000000000 --- a/data/developers/fingu-ai.json +++ /dev/null @@ -1,103 +0,0 @@ -{ - "developer": "FINGU-AI", - "models": [ - { - "id": "FINGU-AI/Chocolatine-Fusion-14B", - "name": "Chocolatine-Fusion-14B", - "developer": "FINGU-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6949, - "hfopenllm_v2/BBH": 0.6413, - "hfopenllm_v2/MATH Level 5": 0.3852, - "hfopenllm_v2/GPQA": 0.3716, - "hfopenllm_v2/MUSR": 0.494, - "hfopenllm_v2/MMLU-PRO": 0.5262 - } - }, - { - "id": "FINGU-AI/L3-8B", - "name": "L3-8B", - "developer": "FINGU-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7517, - "hfopenllm_v2/BBH": 0.4986, - "hfopenllm_v2/MATH Level 5": 0.2545, - "hfopenllm_v2/GPQA": 0.2953, - "hfopenllm_v2/MUSR": 0.3828, - "hfopenllm_v2/MMLU-PRO": 0.3639 - } - }, - { - "id": "FINGU-AI/Phi-4-RRStock", - "name": "Phi-4-RRStock", - "developer": "FINGU-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2855, - "hfopenllm_v2/BBH": 0.6443, - "hfopenllm_v2/MATH Level 5": 0.0582, - "hfopenllm_v2/GPQA": 0.38, - "hfopenllm_v2/MUSR": 0.4479, - "hfopenllm_v2/MMLU-PRO": 0.4883 - } - }, - { - "id": "FINGU-AI/Q-Small-3B", - "name": "Q-Small-3B", - "developer": "FINGU-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4145, - "hfopenllm_v2/BBH": 0.4319, - "hfopenllm_v2/MATH Level 5": 0.0831, - "hfopenllm_v2/GPQA": 0.2668, - "hfopenllm_v2/MUSR": 0.4005, - "hfopenllm_v2/MMLU-PRO": 0.279 - } - }, - { - "id": "FINGU-AI/QwQ-Buddy-32B-Alpha", - "name": "QwQ-Buddy-32B-Alpha", - "developer": "FINGU-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3446, - "hfopenllm_v2/BBH": 0.6424, - "hfopenllm_v2/MATH Level 5": 0.3852, - "hfopenllm_v2/GPQA": 0.3792, - "hfopenllm_v2/MUSR": 0.506, - "hfopenllm_v2/MMLU-PRO": 0.5294 - } - }, - { - "id": "FINGU-AI/RomboUltima-32B", - "name": "RomboUltima-32B", - "developer": "FINGU-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6672, - "hfopenllm_v2/BBH": 0.6938, - "hfopenllm_v2/MATH Level 5": 0.5385, - "hfopenllm_v2/GPQA": 0.3716, - "hfopenllm_v2/MUSR": 0.4836, - "hfopenllm_v2/MMLU-PRO": 0.5789 - } - }, - { - "id": "FINGU-AI/Ultimos-32B", - "name": "Ultimos-32B", - "developer": "FINGU-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1592, - "hfopenllm_v2/BBH": 0.2906, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2492, - "hfopenllm_v2/MUSR": 0.3286, - "hfopenllm_v2/MMLU-PRO": 0.1111 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/flammenai.json b/data/developers/flammenai.json deleted file mode 100644 index daaf52ac431a27940ea88621faadba9cc1cf9f16..0000000000000000000000000000000000000000 --- a/data/developers/flammenai.json +++ /dev/null @@ -1,89 +0,0 @@ -{ - "developer": "flammenai", - "models": [ - { - "id": "flammenai/flammen15-gutenberg-DPO-v1-7B", - "name": "flammen15-gutenberg-DPO-v1-7B", - "developer": "flammenai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4798, - "hfopenllm_v2/BBH": 0.5203, - "hfopenllm_v2/MATH Level 5": 0.0763, - "hfopenllm_v2/GPQA": 0.2844, - "hfopenllm_v2/MUSR": 0.4293, - "hfopenllm_v2/MMLU-PRO": 0.3186 - } - }, - { - "id": "flammenai/Llama3.1-Flammades-70B", - "name": "Llama3.1-Flammades-70B", - "developer": "flammenai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7058, - "hfopenllm_v2/BBH": 0.666, - "hfopenllm_v2/MATH Level 5": 0.2092, - "hfopenllm_v2/GPQA": 0.354, - "hfopenllm_v2/MUSR": 0.4871, - "hfopenllm_v2/MMLU-PRO": 0.4752 - } - }, - { - "id": "flammenai/Mahou-1.2a-llama3-8B", - "name": "Mahou-1.2a-llama3-8B", - "developer": "flammenai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5093, - "hfopenllm_v2/BBH": 0.5094, - "hfopenllm_v2/MATH Level 5": 0.0838, - "hfopenllm_v2/GPQA": 0.2886, - "hfopenllm_v2/MUSR": 0.3847, - "hfopenllm_v2/MMLU-PRO": 0.3817 - } - }, - { - "id": "flammenai/Mahou-1.2a-mistral-7B", - "name": "Mahou-1.2a-mistral-7B", - "developer": "flammenai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4552, - "hfopenllm_v2/BBH": 0.5118, - "hfopenllm_v2/MATH Level 5": 0.0687, - "hfopenllm_v2/GPQA": 0.2718, - "hfopenllm_v2/MUSR": 0.3896, - "hfopenllm_v2/MMLU-PRO": 0.3163 - } - }, - { - "id": "flammenai/Mahou-1.5-llama3.1-70B", - "name": "Mahou-1.5-llama3.1-70B", - "developer": "flammenai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7147, - "hfopenllm_v2/BBH": 0.6651, - "hfopenllm_v2/MATH Level 5": 0.21, - "hfopenllm_v2/GPQA": 0.354, - "hfopenllm_v2/MUSR": 0.495, - "hfopenllm_v2/MMLU-PRO": 0.4749 - } - }, - { - "id": "flammenai/Mahou-1.5-mistral-nemo-12B", - "name": "Mahou-1.5-mistral-nemo-12B", - "developer": "flammenai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6751, - "hfopenllm_v2/BBH": 0.5522, - "hfopenllm_v2/MATH Level 5": 0.0869, - "hfopenllm_v2/GPQA": 0.276, - "hfopenllm_v2/MUSR": 0.452, - "hfopenllm_v2/MMLU-PRO": 0.3602 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/floflob.json b/data/developers/floflob.json deleted file mode 100644 index ddf73ba12b92d64a299d70def26521c7e9deb31f..0000000000000000000000000000000000000000 --- a/data/developers/floflob.json +++ /dev/null @@ -1,383 +0,0 @@ -{ - "developer": "FlofloB", - "models": [ - { - "id": "FlofloB/100k_fineweb_continued_pretraining_Qwen2.5-0.5B-Instruct_Unsloth_merged_16bit", - "name": "100k_fineweb_continued_pretraining_Qwen2.5-0.5B-Instruct_Unsloth_merged_16bit", - "developer": "FlofloB", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3083, - "hfopenllm_v2/BBH": 0.3323, - "hfopenllm_v2/MATH Level 5": 0.0408, - "hfopenllm_v2/GPQA": 0.2693, - "hfopenllm_v2/MUSR": 0.3302, - "hfopenllm_v2/MMLU-PRO": 0.1498 - } - }, - { - "id": "FlofloB/10k_continued_pretraining_Phi-3-mini-4k-instruct_Unsloth_merged_16bit", - "name": "10k_continued_pretraining_Phi-3-mini-4k-instruct_Unsloth_merged_16bit", - "developer": "FlofloB", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5097, - "hfopenllm_v2/BBH": 0.5215, - "hfopenllm_v2/MATH Level 5": 0.0974, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.431, - "hfopenllm_v2/MMLU-PRO": 0.3769 - } - }, - { - "id": "FlofloB/10k_continued_pretraining_Qwen2.5-0.5B-Instruct_Unsloth_merged_16bit", - "name": "10k_continued_pretraining_Qwen2.5-0.5B-Instruct_Unsloth_merged_16bit", - "developer": "FlofloB", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2815, - "hfopenllm_v2/BBH": 0.3306, - "hfopenllm_v2/MATH Level 5": 0.031, - "hfopenllm_v2/GPQA": 0.2794, - "hfopenllm_v2/MUSR": 0.3302, - "hfopenllm_v2/MMLU-PRO": 0.1541 - } - }, - { - "id": "FlofloB/40k_continued_pretraining_Qwen2.5-0.5B-Instruct_Unsloth_merged_16bit", - "name": "40k_continued_pretraining_Qwen2.5-0.5B-Instruct_Unsloth_merged_16bit", - "developer": "FlofloB", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3016, - "hfopenllm_v2/BBH": 0.3325, - "hfopenllm_v2/MATH Level 5": 0.0332, - "hfopenllm_v2/GPQA": 0.2676, - "hfopenllm_v2/MUSR": 0.3408, - "hfopenllm_v2/MMLU-PRO": 0.1485 - } - }, - { - "id": "FlofloB/83k_continued_pretraining_Qwen2.5-0.5B-Instruct_Unsloth_merged_16bit", - "name": "83k_continued_pretraining_Qwen2.5-0.5B-Instruct_Unsloth_merged_16bit", - "developer": "FlofloB", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2869, - "hfopenllm_v2/BBH": 0.3347, - "hfopenllm_v2/MATH Level 5": 0.0302, - "hfopenllm_v2/GPQA": 0.2735, - "hfopenllm_v2/MUSR": 0.3289, - "hfopenllm_v2/MMLU-PRO": 0.1555 - } - }, - { - "id": "FlofloB/smollm2-135M_pretrained_1000k_fineweb", - "name": "smollm2-135M_pretrained_1000k_fineweb", - "developer": "FlofloB", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1485, - "hfopenllm_v2/BBH": 0.2918, - "hfopenllm_v2/MATH Level 5": 0.0091, - "hfopenllm_v2/GPQA": 0.2626, - "hfopenllm_v2/MUSR": 0.3581, - "hfopenllm_v2/MMLU-PRO": 0.1164 - } - }, - { - "id": "FlofloB/smollm2-135M_pretrained_1000k_fineweb_uncovai_human_removed", - "name": "smollm2-135M_pretrained_1000k_fineweb_uncovai_human_removed", - "developer": "FlofloB", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1554, - "hfopenllm_v2/BBH": 0.3066, - "hfopenllm_v2/MATH Level 5": 0.006, - "hfopenllm_v2/GPQA": 0.2508, - "hfopenllm_v2/MUSR": 0.358, - "hfopenllm_v2/MMLU-PRO": 0.1143 - } - }, - { - "id": "FlofloB/smollm2-135M_pretrained_1000k_fineweb_uncovai_selected", - "name": "smollm2-135M_pretrained_1000k_fineweb_uncovai_selected", - "developer": "FlofloB", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1468, - "hfopenllm_v2/BBH": 0.2932, - "hfopenllm_v2/MATH Level 5": 0.0068, - "hfopenllm_v2/GPQA": 0.2659, - "hfopenllm_v2/MUSR": 0.4048, - "hfopenllm_v2/MMLU-PRO": 0.1157 - } - }, - { - "id": "FlofloB/smollm2-135M_pretrained_1200k_fineweb", - "name": "smollm2-135M_pretrained_1200k_fineweb", - "developer": "FlofloB", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1581, - "hfopenllm_v2/BBH": 0.2941, - "hfopenllm_v2/MATH Level 5": 0.0068, - "hfopenllm_v2/GPQA": 0.2643, - "hfopenllm_v2/MUSR": 0.3714, - "hfopenllm_v2/MMLU-PRO": 0.1076 - } - }, - { - "id": "FlofloB/smollm2-135M_pretrained_1200k_fineweb_uncovai_human_removed", - "name": "smollm2-135M_pretrained_1200k_fineweb_uncovai_human_removed", - "developer": "FlofloB", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1578, - "hfopenllm_v2/BBH": 0.295, - "hfopenllm_v2/MATH Level 5": 0.0008, - "hfopenllm_v2/GPQA": 0.2651, - "hfopenllm_v2/MUSR": 0.37, - "hfopenllm_v2/MMLU-PRO": 0.1139 - } - }, - { - "id": "FlofloB/smollm2-135M_pretrained_1200k_fineweb_uncovai_selected", - "name": "smollm2-135M_pretrained_1200k_fineweb_uncovai_selected", - "developer": "FlofloB", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1585, - "hfopenllm_v2/BBH": 0.296, - "hfopenllm_v2/MATH Level 5": 0.0076, - "hfopenllm_v2/GPQA": 0.2634, - "hfopenllm_v2/MUSR": 0.3567, - "hfopenllm_v2/MMLU-PRO": 0.1164 - } - }, - { - "id": "FlofloB/smollm2-135M_pretrained_1400k_fineweb", - "name": "smollm2-135M_pretrained_1400k_fineweb", - "developer": "FlofloB", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1764, - "hfopenllm_v2/BBH": 0.2922, - "hfopenllm_v2/MATH Level 5": 0.0113, - "hfopenllm_v2/GPQA": 0.2659, - "hfopenllm_v2/MUSR": 0.3873, - "hfopenllm_v2/MMLU-PRO": 0.108 - } - }, - { - "id": "FlofloB/smollm2-135M_pretrained_1400k_fineweb_uncovai_human_removed", - "name": "smollm2-135M_pretrained_1400k_fineweb_uncovai_human_removed", - "developer": "FlofloB", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1707, - "hfopenllm_v2/BBH": 0.2992, - "hfopenllm_v2/MATH Level 5": 0.0106, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.3939, - "hfopenllm_v2/MMLU-PRO": 0.1105 - } - }, - { - "id": "FlofloB/smollm2-135M_pretrained_1400k_fineweb_uncovai_selected", - "name": "smollm2-135M_pretrained_1400k_fineweb_uncovai_selected", - "developer": "FlofloB", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1538, - "hfopenllm_v2/BBH": 0.2917, - "hfopenllm_v2/MATH Level 5": 0.0106, - "hfopenllm_v2/GPQA": 0.2685, - "hfopenllm_v2/MUSR": 0.3741, - "hfopenllm_v2/MMLU-PRO": 0.1137 - } - }, - { - "id": "FlofloB/smollm2-135M_pretrained_200k_fineweb_uncovai_human_removed", - "name": "smollm2-135M_pretrained_200k_fineweb_uncovai_human_removed", - "developer": "FlofloB", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1475, - "hfopenllm_v2/BBH": 0.3029, - "hfopenllm_v2/MATH Level 5": 0.0038, - "hfopenllm_v2/GPQA": 0.2584, - "hfopenllm_v2/MUSR": 0.3578, - "hfopenllm_v2/MMLU-PRO": 0.112 - } - }, - { - "id": "FlofloB/smollm2-135M_pretrained_200k_fineweb_uncovai_selected", - "name": "smollm2-135M_pretrained_200k_fineweb_uncovai_selected", - "developer": "FlofloB", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1345, - "hfopenllm_v2/BBH": 0.2927, - "hfopenllm_v2/MATH Level 5": 0.0076, - "hfopenllm_v2/GPQA": 0.2508, - "hfopenllm_v2/MUSR": 0.366, - "hfopenllm_v2/MMLU-PRO": 0.1131 - } - }, - { - "id": "FlofloB/smollm2-135M_pretrained_400k_fineweb", - "name": "smollm2-135M_pretrained_400k_fineweb", - "developer": "FlofloB", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1511, - "hfopenllm_v2/BBH": 0.2972, - "hfopenllm_v2/MATH Level 5": 0.0121, - "hfopenllm_v2/GPQA": 0.2525, - "hfopenllm_v2/MUSR": 0.3794, - "hfopenllm_v2/MMLU-PRO": 0.1163 - } - }, - { - "id": "FlofloB/smollm2-135M_pretrained_400k_fineweb_uncovai_human_removed", - "name": "smollm2-135M_pretrained_400k_fineweb_uncovai_human_removed", - "developer": "FlofloB", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1556, - "hfopenllm_v2/BBH": 0.3049, - "hfopenllm_v2/MATH Level 5": 0.0091, - "hfopenllm_v2/GPQA": 0.255, - "hfopenllm_v2/MUSR": 0.386, - "hfopenllm_v2/MMLU-PRO": 0.1138 - } - }, - { - "id": "FlofloB/smollm2-135M_pretrained_400k_fineweb_uncovai_selected", - "name": "smollm2-135M_pretrained_400k_fineweb_uncovai_selected", - "developer": "FlofloB", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1584, - "hfopenllm_v2/BBH": 0.2925, - "hfopenllm_v2/MATH Level 5": 0.0068, - "hfopenllm_v2/GPQA": 0.2542, - "hfopenllm_v2/MUSR": 0.382, - "hfopenllm_v2/MMLU-PRO": 0.1158 - } - }, - { - "id": "FlofloB/smollm2-135M_pretrained_600k_fineweb", - "name": "smollm2-135M_pretrained_600k_fineweb", - "developer": "FlofloB", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1639, - "hfopenllm_v2/BBH": 0.3014, - "hfopenllm_v2/MATH Level 5": 0.006, - "hfopenllm_v2/GPQA": 0.2659, - "hfopenllm_v2/MUSR": 0.3809, - "hfopenllm_v2/MMLU-PRO": 0.1126 - } - }, - { - "id": "FlofloB/smollm2-135M_pretrained_600k_fineweb_uncovai_human_removed", - "name": "smollm2-135M_pretrained_600k_fineweb_uncovai_human_removed", - "developer": "FlofloB", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1641, - "hfopenllm_v2/BBH": 0.3, - "hfopenllm_v2/MATH Level 5": 0.0091, - "hfopenllm_v2/GPQA": 0.2626, - "hfopenllm_v2/MUSR": 0.3793, - "hfopenllm_v2/MMLU-PRO": 0.1147 - } - }, - { - "id": "FlofloB/smollm2-135M_pretrained_600k_fineweb_uncovai_selected", - "name": "smollm2-135M_pretrained_600k_fineweb_uncovai_selected", - "developer": "FlofloB", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1606, - "hfopenllm_v2/BBH": 0.2983, - "hfopenllm_v2/MATH Level 5": 0.0076, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.3846, - "hfopenllm_v2/MMLU-PRO": 0.1162 - } - }, - { - "id": "FlofloB/smollm2-135M_pretrained_800k_fineweb", - "name": "smollm2-135M_pretrained_800k_fineweb", - "developer": "FlofloB", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1641, - "hfopenllm_v2/BBH": 0.2959, - "hfopenllm_v2/MATH Level 5": 0.0083, - "hfopenllm_v2/GPQA": 0.2492, - "hfopenllm_v2/MUSR": 0.3701, - "hfopenllm_v2/MMLU-PRO": 0.1152 - } - }, - { - "id": "FlofloB/smollm2-135M_pretrained_800k_fineweb_uncovai_human_removed", - "name": "smollm2-135M_pretrained_800k_fineweb_uncovai_human_removed", - "developer": "FlofloB", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1623, - "hfopenllm_v2/BBH": 0.3038, - "hfopenllm_v2/MATH Level 5": 0.0068, - "hfopenllm_v2/GPQA": 0.2525, - "hfopenllm_v2/MUSR": 0.3993, - "hfopenllm_v2/MMLU-PRO": 0.1138 - } - }, - { - "id": "FlofloB/smollm2-135M_pretrained_800k_fineweb_uncovai_selected", - "name": "smollm2-135M_pretrained_800k_fineweb_uncovai_selected", - "developer": "FlofloB", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1474, - "hfopenllm_v2/BBH": 0.2943, - "hfopenllm_v2/MATH Level 5": 0.0045, - "hfopenllm_v2/GPQA": 0.2617, - "hfopenllm_v2/MUSR": 0.3766, - "hfopenllm_v2/MMLU-PRO": 0.113 - } - }, - { - "id": "FlofloB/smollm2_pretrained_200k_fineweb", - "name": "smollm2_pretrained_200k_fineweb", - "developer": "FlofloB", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1527, - "hfopenllm_v2/BBH": 0.2995, - "hfopenllm_v2/MATH Level 5": 0.0038, - "hfopenllm_v2/GPQA": 0.2475, - "hfopenllm_v2/MUSR": 0.3699, - "hfopenllm_v2/MMLU-PRO": 0.1159 - } - }, - { - "id": "FlofloB/test_continued_pretraining_Phi-3-mini-4k-instruct_Unsloth_merged_16bit", - "name": "test_continued_pretraining_Phi-3-mini-4k-instruct_Unsloth_merged_16bit", - "developer": "FlofloB", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5215, - "hfopenllm_v2/BBH": 0.5241, - "hfopenllm_v2/MATH Level 5": 0.1103, - "hfopenllm_v2/GPQA": 0.3112, - "hfopenllm_v2/MUSR": 0.4244, - "hfopenllm_v2/MMLU-PRO": 0.3721 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/fluently-lm.json b/data/developers/fluently-lm.json deleted file mode 100644 index 0e6c9848b32304c6529c8bfaa085da0f02940129..0000000000000000000000000000000000000000 --- a/data/developers/fluently-lm.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "developer": "fluently-lm", - "models": [ - { - "id": "fluently-lm/FluentlyLM-Prinum", - "name": "FluentlyLM-Prinum", - "developer": "fluently-lm", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.809, - "hfopenllm_v2/BBH": 0.7144, - "hfopenllm_v2/MATH Level 5": 0.54, - "hfopenllm_v2/GPQA": 0.3867, - "hfopenllm_v2/MUSR": 0.4471, - "hfopenllm_v2/MMLU-PRO": 0.5808 - } - }, - { - "id": "fluently-lm/Llama-TI-8B", - "name": "Llama-TI-8B", - "developer": "fluently-lm", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.288, - "hfopenllm_v2/BBH": 0.5201, - "hfopenllm_v2/MATH Level 5": 0.1964, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.4103, - "hfopenllm_v2/MMLU-PRO": 0.344 - } - }, - { - "id": "fluently-lm/Llama-TI-8B-Instruct", - "name": "Llama-TI-8B-Instruct", - "developer": "fluently-lm", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7716, - "hfopenllm_v2/BBH": 0.5252, - "hfopenllm_v2/MATH Level 5": 0.2304, - "hfopenllm_v2/GPQA": 0.2953, - "hfopenllm_v2/MUSR": 0.3813, - "hfopenllm_v2/MMLU-PRO": 0.3726 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/fluently-sets.json b/data/developers/fluently-sets.json deleted file mode 100644 index 42343f86d625db8cce5578283dd28c1d1ca72350..0000000000000000000000000000000000000000 --- a/data/developers/fluently-sets.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "fluently-sets", - "models": [ - { - "id": "fluently-sets/FalconThink3-10B-IT", - "name": "FalconThink3-10B-IT", - "developer": "fluently-sets", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7326, - "hfopenllm_v2/BBH": 0.62, - "hfopenllm_v2/MATH Level 5": 0.2447, - "hfopenllm_v2/GPQA": 0.3347, - "hfopenllm_v2/MUSR": 0.4479, - "hfopenllm_v2/MMLU-PRO": 0.4435 - } - }, - { - "id": "fluently-sets/reasoning-1-1k-demo", - "name": "reasoning-1-1k-demo", - "developer": "fluently-sets", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7525, - "hfopenllm_v2/BBH": 0.6397, - "hfopenllm_v2/MATH Level 5": 0.4282, - "hfopenllm_v2/GPQA": 0.3356, - "hfopenllm_v2/MUSR": 0.4061, - "hfopenllm_v2/MMLU-PRO": 0.4774 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/formulae.json b/data/developers/formulae.json deleted file mode 100644 index a90660346e063775466ae46cd6fd73ac7b92344a..0000000000000000000000000000000000000000 --- a/data/developers/formulae.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "developer": "formulae", - "models": [ - { - "id": "formulae/mita-elite-sce-gen1.1-v1-7b-2-26-2025-exp", - "name": "mita-elite-sce-gen1.1-v1-7b-2-26-2025-exp", - "developer": "formulae", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1614, - "hfopenllm_v2/BBH": 0.2976, - "hfopenllm_v2/MATH Level 5": 0.0015, - "hfopenllm_v2/GPQA": 0.2534, - "hfopenllm_v2/MUSR": 0.4219, - "hfopenllm_v2/MMLU-PRO": 0.1174 - } - }, - { - "id": "formulae/mita-elite-v1.1-7b-2-25-2025", - "name": "mita-elite-v1.1-7b-2-25-2025", - "developer": "formulae", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.125, - "hfopenllm_v2/BBH": 0.2867, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2483, - "hfopenllm_v2/MUSR": 0.3487, - "hfopenllm_v2/MMLU-PRO": 0.1098 - } - }, - { - "id": "formulae/mita-elite-v1.1-gen2-7b-2-25-2025", - "name": "mita-elite-v1.1-gen2-7b-2-25-2025", - "developer": "formulae", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1411, - "hfopenllm_v2/BBH": 0.2924, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2525, - "hfopenllm_v2/MUSR": 0.3541, - "hfopenllm_v2/MMLU-PRO": 0.1101 - } - }, - { - "id": "formulae/mita-elite-v1.2-7b-2-26-2025", - "name": "mita-elite-v1.2-7b-2-26-2025", - "developer": "formulae", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.148, - "hfopenllm_v2/BBH": 0.293, - "hfopenllm_v2/MATH Level 5": 0.0023, - "hfopenllm_v2/GPQA": 0.2743, - "hfopenllm_v2/MUSR": 0.4287, - "hfopenllm_v2/MMLU-PRO": 0.1186 - } - }, - { - "id": "formulae/mita-gen3-7b-2-26-2025", - "name": "mita-gen3-7b-2-26-2025", - "developer": "formulae", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1964, - "hfopenllm_v2/BBH": 0.2916, - "hfopenllm_v2/MATH Level 5": 0.0023, - "hfopenllm_v2/GPQA": 0.2651, - "hfopenllm_v2/MUSR": 0.3912, - "hfopenllm_v2/MMLU-PRO": 0.1124 - } - }, - { - "id": "formulae/mita-gen3-v1.2-7b-2-26-2025", - "name": "mita-gen3-v1.2-7b-2-26-2025", - "developer": "formulae", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2044, - "hfopenllm_v2/BBH": 0.3058, - "hfopenllm_v2/MATH Level 5": 0.0023, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.39, - "hfopenllm_v2/MMLU-PRO": 0.1128 - } - }, - { - "id": "formulae/mita-math-v2.3-2-25-2025", - "name": "mita-math-v2.3-2-25-2025", - "developer": "formulae", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1373, - "hfopenllm_v2/BBH": 0.2949, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2508, - "hfopenllm_v2/MUSR": 0.3698, - "hfopenllm_v2/MMLU-PRO": 0.1118 - } - }, - { - "id": "formulae/mita-v1-7b", - "name": "mita-v1-7b", - "developer": "formulae", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1972, - "hfopenllm_v2/BBH": 0.3003, - "hfopenllm_v2/MATH Level 5": 0.0023, - "hfopenllm_v2/GPQA": 0.25, - "hfopenllm_v2/MUSR": 0.4152, - "hfopenllm_v2/MMLU-PRO": 0.1147 - } - }, - { - "id": "formulae/mita-v1.1-7b-2-24-2025", - "name": "mita-v1.1-7b-2-24-2025", - "developer": "formulae", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3412, - "hfopenllm_v2/BBH": 0.5442, - "hfopenllm_v2/MATH Level 5": 0.435, - "hfopenllm_v2/GPQA": 0.3146, - "hfopenllm_v2/MUSR": 0.4557, - "hfopenllm_v2/MMLU-PRO": 0.4524 - } - }, - { - "id": "formulae/mita-v1.2-7b-2-24-2025", - "name": "mita-v1.2-7b-2-24-2025", - "developer": "formulae", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2564, - "hfopenllm_v2/BBH": 0.4919, - "hfopenllm_v2/MATH Level 5": 0.4879, - "hfopenllm_v2/GPQA": 0.3062, - "hfopenllm_v2/MUSR": 0.4344, - "hfopenllm_v2/MMLU-PRO": 0.3359 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/frameai.json b/data/developers/frameai.json deleted file mode 100644 index f6cfe0eeaa937f1c2daddcc91f80312567ab07a3..0000000000000000000000000000000000000000 --- a/data/developers/frameai.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "frameai", - "models": [ - { - "id": "frameai/Loxa-4B", - "name": "Loxa-4B", - "developer": "frameai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4765, - "hfopenllm_v2/BBH": 0.4217, - "hfopenllm_v2/MATH Level 5": 0.1095, - "hfopenllm_v2/GPQA": 0.2836, - "hfopenllm_v2/MUSR": 0.3377, - "hfopenllm_v2/MMLU-PRO": 0.2802 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/freewheelin.json b/data/developers/freewheelin.json deleted file mode 100644 index 4cbcbd6d80c381c151916943a05fc6938679b59c..0000000000000000000000000000000000000000 --- a/data/developers/freewheelin.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "developer": "freewheelin", - "models": [ - { - "id": "freewheelin/free-evo-qwen72b-v0.8-re", - "name": "free-evo-qwen72b-v0.8-re", - "developer": "freewheelin", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5331, - "hfopenllm_v2/BBH": 0.6127, - "hfopenllm_v2/MATH Level 5": 0.1805, - "hfopenllm_v2/GPQA": 0.3565, - "hfopenllm_v2/MUSR": 0.4872, - "hfopenllm_v2/MMLU-PRO": 0.487 - } - }, - { - "id": "freewheelin/free-solar-evo-v0.1", - "name": "free-solar-evo-v0.1", - "developer": "freewheelin", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.205, - "hfopenllm_v2/BBH": 0.4502, - "hfopenllm_v2/MATH Level 5": 0.0083, - "hfopenllm_v2/GPQA": 0.2911, - "hfopenllm_v2/MUSR": 0.4946, - "hfopenllm_v2/MMLU-PRO": 0.3414 - } - }, - { - "id": "freewheelin/free-solar-evo-v0.11", - "name": "free-solar-evo-v0.11", - "developer": "freewheelin", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2027, - "hfopenllm_v2/BBH": 0.4545, - "hfopenllm_v2/MATH Level 5": 0.0083, - "hfopenllm_v2/GPQA": 0.2852, - "hfopenllm_v2/MUSR": 0.5052, - "hfopenllm_v2/MMLU-PRO": 0.3467 - } - }, - { - "id": "freewheelin/free-solar-evo-v0.13", - "name": "free-solar-evo-v0.13", - "developer": "freewheelin", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2321, - "hfopenllm_v2/BBH": 0.4555, - "hfopenllm_v2/MATH Level 5": 0.0121, - "hfopenllm_v2/GPQA": 0.2886, - "hfopenllm_v2/MUSR": 0.5052, - "hfopenllm_v2/MMLU-PRO": 0.347 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/fujhen.json b/data/developers/fujhen.json deleted file mode 100644 index 18df6b5a615decb777fa7c14bf593368991d43d7..0000000000000000000000000000000000000000 --- a/data/developers/fujhen.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "developer": "FuJhen", - "models": [ - { - "id": "FuJhen/ft-openhermes-25-mistral-7b-irca-dpo-pairs", - "name": "ft-openhermes-25-mistral-7b-irca-dpo-pairs", - "developer": "FuJhen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.542, - "hfopenllm_v2/BBH": 0.4773, - "hfopenllm_v2/MATH Level 5": 0.0483, - "hfopenllm_v2/GPQA": 0.2785, - "hfopenllm_v2/MUSR": 0.4174, - "hfopenllm_v2/MMLU-PRO": 0.2956 - } - }, - { - "id": "FuJhen/mistral-instruct-7B-DPO", - "name": "mistral-instruct-7B-DPO", - "developer": "FuJhen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4968, - "hfopenllm_v2/BBH": 0.4624, - "hfopenllm_v2/MATH Level 5": 0.0385, - "hfopenllm_v2/GPQA": 0.2777, - "hfopenllm_v2/MUSR": 0.4016, - "hfopenllm_v2/MMLU-PRO": 0.3034 - } - }, - { - "id": "FuJhen/mistral_7b_v0.1_structedData_e2e", - "name": "mistral_7b_v0.1_structedData_e2e", - "developer": "FuJhen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1727, - "hfopenllm_v2/BBH": 0.4114, - "hfopenllm_v2/MATH Level 5": 0.0045, - "hfopenllm_v2/GPQA": 0.2794, - "hfopenllm_v2/MUSR": 0.3723, - "hfopenllm_v2/MMLU-PRO": 0.2811 - } - }, - { - "id": "FuJhen/mistral_7b_v0.1_structedData_viggo", - "name": "mistral_7b_v0.1_structedData_viggo", - "developer": "FuJhen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1783, - "hfopenllm_v2/BBH": 0.4524, - "hfopenllm_v2/MATH Level 5": 0.0287, - "hfopenllm_v2/GPQA": 0.2836, - "hfopenllm_v2/MUSR": 0.3738, - "hfopenllm_v2/MMLU-PRO": 0.2942 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/fulim.json b/data/developers/fulim.json deleted file mode 100644 index 11f6fa5440952305b05a189555b1b36a11ed3c83..0000000000000000000000000000000000000000 --- a/data/developers/fulim.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "fulim", - "models": [ - { - "id": "fulim/FineLlama-3.1-8B", - "name": "FineLlama-3.1-8B", - "developer": "fulim", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1439, - "hfopenllm_v2/BBH": 0.4569, - "hfopenllm_v2/MATH Level 5": 0.0476, - "hfopenllm_v2/GPQA": 0.2928, - "hfopenllm_v2/MUSR": 0.3867, - "hfopenllm_v2/MMLU-PRO": 0.3167 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/fuseai.json b/data/developers/fuseai.json deleted file mode 100644 index 13a53260c5d7b9d584f47f65b16b17de97cfb59e..0000000000000000000000000000000000000000 --- a/data/developers/fuseai.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "developer": "FuseAI", - "models": [ - { - "id": "FuseAI/FuseChat-7B-v2.0", - "name": "FuseChat-7B-v2.0", - "developer": "FuseAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3423, - "hfopenllm_v2/BBH": 0.4954, - "hfopenllm_v2/MATH Level 5": 0.0612, - "hfopenllm_v2/GPQA": 0.302, - "hfopenllm_v2/MUSR": 0.4797, - "hfopenllm_v2/MMLU-PRO": 0.3162 - } - }, - { - "id": "FuseAI/FuseChat-Llama-3.1-8B-Instruct", - "name": "FuseChat-Llama-3.1-8B-Instruct", - "developer": "FuseAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7205, - "hfopenllm_v2/BBH": 0.512, - "hfopenllm_v2/MATH Level 5": 0.2477, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.382, - "hfopenllm_v2/MMLU-PRO": 0.3733 - } - }, - { - "id": "FuseAI/FuseChat-Llama-3.2-3B-Instruct", - "name": "FuseChat-Llama-3.2-3B-Instruct", - "developer": "FuseAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6849, - "hfopenllm_v2/BBH": 0.4658, - "hfopenllm_v2/MATH Level 5": 0.2424, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.3914, - "hfopenllm_v2/MMLU-PRO": 0.3132 - } - }, - { - "id": "FuseAI/FuseChat-Qwen-2.5-7B-Instruct", - "name": "FuseChat-Qwen-2.5-7B-Instruct", - "developer": "FuseAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5906, - "hfopenllm_v2/BBH": 0.5526, - "hfopenllm_v2/MATH Level 5": 0.4562, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.3874, - "hfopenllm_v2/MMLU-PRO": 0.4118 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/gabrielmbmb.json b/data/developers/gabrielmbmb.json deleted file mode 100644 index dacefab277230c321eb5a6384611464f503150d3..0000000000000000000000000000000000000000 --- a/data/developers/gabrielmbmb.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "gabrielmbmb", - "models": [ - { - "id": "gabrielmbmb/SmolLM-1.7B-Instruct-IFEval", - "name": "SmolLM-1.7B-Instruct-IFEval", - "developer": "gabrielmbmb", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2306, - "hfopenllm_v2/BBH": 0.3138, - "hfopenllm_v2/MATH Level 5": 0.0106, - "hfopenllm_v2/GPQA": 0.2534, - "hfopenllm_v2/MUSR": 0.3328, - "hfopenllm_v2/MMLU-PRO": 0.1156 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/galrionsoftworks.json b/data/developers/galrionsoftworks.json deleted file mode 100644 index 1472d8819ea1b0b1d0a1fc3c6be56da5fa9a8d81..0000000000000000000000000000000000000000 --- a/data/developers/galrionsoftworks.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "GalrionSoftworks", - "models": [ - { - "id": "GalrionSoftworks/MagnusIntellectus-12B-v1", - "name": "MagnusIntellectus-12B-v1", - "developer": "GalrionSoftworks", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4421, - "hfopenllm_v2/BBH": 0.5323, - "hfopenllm_v2/MATH Level 5": 0.065, - "hfopenllm_v2/GPQA": 0.2844, - "hfopenllm_v2/MUSR": 0.4428, - "hfopenllm_v2/MMLU-PRO": 0.3421 - } - }, - { - "id": "GalrionSoftworks/MN-LooseCannon-12B-v1", - "name": "MN-LooseCannon-12B-v1", - "developer": "GalrionSoftworks", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5418, - "hfopenllm_v2/BBH": 0.5128, - "hfopenllm_v2/MATH Level 5": 0.0853, - "hfopenllm_v2/GPQA": 0.2852, - "hfopenllm_v2/MUSR": 0.4138, - "hfopenllm_v2/MMLU-PRO": 0.3196 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/gaverfraxz.json b/data/developers/gaverfraxz.json deleted file mode 100644 index 0436f043f2d80f7a296880acd9db459d95075740..0000000000000000000000000000000000000000 --- a/data/developers/gaverfraxz.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "gaverfraxz", - "models": [ - { - "id": "gaverfraxz/Meta-Llama-3.1-8B-Instruct-HalfAbliterated-DELLA", - "name": "Meta-Llama-3.1-8B-Instruct-HalfAbliterated-DELLA", - "developer": "gaverfraxz", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4009, - "hfopenllm_v2/BBH": 0.3985, - "hfopenllm_v2/MATH Level 5": 0.0196, - "hfopenllm_v2/GPQA": 0.2844, - "hfopenllm_v2/MUSR": 0.365, - "hfopenllm_v2/MMLU-PRO": 0.1654 - } - }, - { - "id": "gaverfraxz/Meta-Llama-3.1-8B-Instruct-HalfAbliterated-TIES", - "name": "Meta-Llama-3.1-8B-Instruct-HalfAbliterated-TIES", - "developer": "gaverfraxz", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4551, - "hfopenllm_v2/BBH": 0.5044, - "hfopenllm_v2/MATH Level 5": 0.1299, - "hfopenllm_v2/GPQA": 0.2668, - "hfopenllm_v2/MUSR": 0.3738, - "hfopenllm_v2/MMLU-PRO": 0.3679 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/gbueno86.json b/data/developers/gbueno86.json deleted file mode 100644 index 5345de43e2ec85769193d3959b03a9fb7cc620e8..0000000000000000000000000000000000000000 --- a/data/developers/gbueno86.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "gbueno86", - "models": [ - { - "id": "gbueno86/Brinebreath-Llama-3.1-70B", - "name": "Brinebreath-Llama-3.1-70B", - "developer": "gbueno86", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5533, - "hfopenllm_v2/BBH": 0.6881, - "hfopenllm_v2/MATH Level 5": 0.2976, - "hfopenllm_v2/GPQA": 0.3465, - "hfopenllm_v2/MUSR": 0.4541, - "hfopenllm_v2/MMLU-PRO": 0.5196 - } - }, - { - "id": "gbueno86/Meta-LLama-3-Cat-Smaug-LLama-70b", - "name": "Meta-LLama-3-Cat-Smaug-LLama-70b", - "developer": "gbueno86", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8072, - "hfopenllm_v2/BBH": 0.6674, - "hfopenllm_v2/MATH Level 5": 0.2938, - "hfopenllm_v2/GPQA": 0.3272, - "hfopenllm_v2/MUSR": 0.4368, - "hfopenllm_v2/MMLU-PRO": 0.5075 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/general-preference.json b/data/developers/general-preference.json deleted file mode 100644 index e991453787cea41b4dba4804ca8afcc0731b4110..0000000000000000000000000000000000000000 --- a/data/developers/general-preference.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "developer": "general-preference", - "models": [ - { - "id": "general-preference/GPM-Gemma-2B", - "name": "general-preference/GPM-Gemma-2B", - "developer": "general-preference", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7449, - "reward-bench/Chat": 0.7151, - "reward-bench/Chat Hard": 0.6974, - "reward-bench/Safety": 0.8122, - "reward-bench/Reasoning": 0.755 - } - }, - { - "id": "general-preference/GPM-Llama-3.1-8B", - "name": "general-preference/GPM-Llama-3.1-8B", - "developer": "general-preference", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.9224, - "reward-bench/Chat": 0.933, - "reward-bench/Chat Hard": 0.886, - "reward-bench/Safety": 0.9108, - "reward-bench/Reasoning": 0.9597 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/genvradmin.json b/data/developers/genvradmin.json deleted file mode 100644 index ba52e8492d8f5c76d78f479e48e0f756596b4bac..0000000000000000000000000000000000000000 --- a/data/developers/genvradmin.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "developer": "GenVRadmin", - "models": [ - { - "id": "GenVRadmin/AryaBhatta-GemmaOrca-2-Merged", - "name": "AryaBhatta-GemmaOrca-2-Merged", - "developer": "GenVRadmin", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3064, - "hfopenllm_v2/BBH": 0.3887, - "hfopenllm_v2/MATH Level 5": 0.0498, - "hfopenllm_v2/GPQA": 0.2685, - "hfopenllm_v2/MUSR": 0.455, - "hfopenllm_v2/MMLU-PRO": 0.2384 - } - }, - { - "id": "GenVRadmin/AryaBhatta-GemmaOrca-Merged", - "name": "AryaBhatta-GemmaOrca-Merged", - "developer": "GenVRadmin", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3064, - "hfopenllm_v2/BBH": 0.4131, - "hfopenllm_v2/MATH Level 5": 0.0514, - "hfopenllm_v2/GPQA": 0.2559, - "hfopenllm_v2/MUSR": 0.3524, - "hfopenllm_v2/MMLU-PRO": 0.2228 - } - }, - { - "id": "GenVRadmin/AryaBhatta-GemmaUltra-Merged", - "name": "AryaBhatta-GemmaUltra-Merged", - "developer": "GenVRadmin", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3021, - "hfopenllm_v2/BBH": 0.4141, - "hfopenllm_v2/MATH Level 5": 0.0536, - "hfopenllm_v2/GPQA": 0.2534, - "hfopenllm_v2/MUSR": 0.4279, - "hfopenllm_v2/MMLU-PRO": 0.2266 - } - }, - { - "id": "GenVRadmin/llama38bGenZ_Vikas-Merged", - "name": "llama38bGenZ_Vikas-Merged", - "developer": "GenVRadmin", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3, - "hfopenllm_v2/BBH": 0.4536, - "hfopenllm_v2/MATH Level 5": 0.0574, - "hfopenllm_v2/GPQA": 0.2953, - "hfopenllm_v2/MUSR": 0.4402, - "hfopenllm_v2/MMLU-PRO": 0.2622 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/ghost-x.json b/data/developers/ghost-x.json deleted file mode 100644 index d4ed1cc45f217e142b7d757d09fda45f95c70952..0000000000000000000000000000000000000000 --- a/data/developers/ghost-x.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "ghost-x", - "models": [ - { - "id": "ghost-x/ghost-8b-beta-1608", - "name": "ghost-8b-beta-1608", - "developer": "ghost-x", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4273, - "hfopenllm_v2/BBH": 0.4517, - "hfopenllm_v2/MATH Level 5": 0.0695, - "hfopenllm_v2/GPQA": 0.2584, - "hfopenllm_v2/MUSR": 0.3516, - "hfopenllm_v2/MMLU-PRO": 0.284 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/glaiveai.json b/data/developers/glaiveai.json deleted file mode 100644 index fb54193fe0bd8cc8bd4fe5c9e80f2c0b40da388a..0000000000000000000000000000000000000000 --- a/data/developers/glaiveai.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "glaiveai", - "models": [ - { - "id": "glaiveai/Reflection-Llama-3.1-70B", - "name": "Reflection-Llama-3.1-70B", - "developer": "glaiveai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5991, - "hfopenllm_v2/BBH": 0.5681, - "hfopenllm_v2/MATH Level 5": 0.2757, - "hfopenllm_v2/GPQA": 0.3146, - "hfopenllm_v2/MUSR": 0.438, - "hfopenllm_v2/MMLU-PRO": 0.6341 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/gmonsoon.json b/data/developers/gmonsoon.json deleted file mode 100644 index cd5cde3e7e640505d0edfb3c519c0e5edfe9f9c3..0000000000000000000000000000000000000000 --- a/data/developers/gmonsoon.json +++ /dev/null @@ -1,75 +0,0 @@ -{ - "developer": "gmonsoon", - "models": [ - { - "id": "gmonsoon/gemma2-9b-sahabatai-v1-instruct-BaseTIES", - "name": "gemma2-9b-sahabatai-v1-instruct-BaseTIES", - "developer": "gmonsoon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7378, - "hfopenllm_v2/BBH": 0.6077, - "hfopenllm_v2/MATH Level 5": 0.1994, - "hfopenllm_v2/GPQA": 0.3205, - "hfopenllm_v2/MUSR": 0.4778, - "hfopenllm_v2/MMLU-PRO": 0.4347 - } - }, - { - "id": "gmonsoon/SahabatAI-Llama-11B-Test", - "name": "SahabatAI-Llama-11B-Test", - "developer": "gmonsoon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3376, - "hfopenllm_v2/BBH": 0.4728, - "hfopenllm_v2/MATH Level 5": 0.031, - "hfopenllm_v2/GPQA": 0.2819, - "hfopenllm_v2/MUSR": 0.4001, - "hfopenllm_v2/MMLU-PRO": 0.3182 - } - }, - { - "id": "gmonsoon/SahabatAI-MediChatIndo-8B-v1", - "name": "SahabatAI-MediChatIndo-8B-v1", - "developer": "gmonsoon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4163, - "hfopenllm_v2/BBH": 0.4509, - "hfopenllm_v2/MATH Level 5": 0.0619, - "hfopenllm_v2/GPQA": 0.2827, - "hfopenllm_v2/MUSR": 0.3754, - "hfopenllm_v2/MMLU-PRO": 0.3108 - } - }, - { - "id": "gmonsoon/SahabatAI-Rebase-8B-Test", - "name": "SahabatAI-Rebase-8B-Test", - "developer": "gmonsoon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5156, - "hfopenllm_v2/BBH": 0.523, - "hfopenllm_v2/MATH Level 5": 0.1148, - "hfopenllm_v2/GPQA": 0.2878, - "hfopenllm_v2/MUSR": 0.4133, - "hfopenllm_v2/MMLU-PRO": 0.3664 - } - }, - { - "id": "gmonsoon/StockSeaLLMs-7B-v1", - "name": "StockSeaLLMs-7B-v1", - "developer": "gmonsoon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4599, - "hfopenllm_v2/BBH": 0.5271, - "hfopenllm_v2/MATH Level 5": 0.1964, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.4214, - "hfopenllm_v2/MMLU-PRO": 0.3952 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/godlikehhd.json b/data/developers/godlikehhd.json deleted file mode 100644 index 3c08839703f026452326e3a7d0fcccd609482c94..0000000000000000000000000000000000000000 --- a/data/developers/godlikehhd.json +++ /dev/null @@ -1,369 +0,0 @@ -{ - "developer": "godlikehhd", - "models": [ - { - "id": "godlikehhd/alpaca_data_full_2", - "name": "alpaca_data_full_2", - "developer": "godlikehhd", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3178, - "hfopenllm_v2/BBH": 0.4217, - "hfopenllm_v2/MATH Level 5": 0.0929, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.4052, - "hfopenllm_v2/MMLU-PRO": 0.2854 - } - }, - { - "id": "godlikehhd/alpaca_data_full_3B", - "name": "alpaca_data_full_3B", - "developer": "godlikehhd", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3696, - "hfopenllm_v2/BBH": 0.4684, - "hfopenllm_v2/MATH Level 5": 0.1337, - "hfopenllm_v2/GPQA": 0.2777, - "hfopenllm_v2/MUSR": 0.4955, - "hfopenllm_v2/MMLU-PRO": 0.3357 - } - }, - { - "id": "godlikehhd/alpaca_data_ifd_max_2600", - "name": "alpaca_data_ifd_max_2600", - "developer": "godlikehhd", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3043, - "hfopenllm_v2/BBH": 0.4029, - "hfopenllm_v2/MATH Level 5": 0.0989, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.3509, - "hfopenllm_v2/MMLU-PRO": 0.2916 - } - }, - { - "id": "godlikehhd/alpaca_data_ifd_max_2600_3B", - "name": "alpaca_data_ifd_max_2600_3B", - "developer": "godlikehhd", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2982, - "hfopenllm_v2/BBH": 0.4626, - "hfopenllm_v2/MATH Level 5": 0.1594, - "hfopenllm_v2/GPQA": 0.2727, - "hfopenllm_v2/MUSR": 0.4346, - "hfopenllm_v2/MMLU-PRO": 0.3288 - } - }, - { - "id": "godlikehhd/alpaca_data_ifd_me_max_5200", - "name": "alpaca_data_ifd_me_max_5200", - "developer": "godlikehhd", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3683, - "hfopenllm_v2/BBH": 0.4153, - "hfopenllm_v2/MATH Level 5": 0.0974, - "hfopenllm_v2/GPQA": 0.2911, - "hfopenllm_v2/MUSR": 0.3483, - "hfopenllm_v2/MMLU-PRO": 0.2982 - } - }, - { - "id": "godlikehhd/alpaca_data_ifd_min_2600", - "name": "alpaca_data_ifd_min_2600", - "developer": "godlikehhd", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.375, - "hfopenllm_v2/BBH": 0.4219, - "hfopenllm_v2/MATH Level 5": 0.0967, - "hfopenllm_v2/GPQA": 0.2919, - "hfopenllm_v2/MUSR": 0.3656, - "hfopenllm_v2/MMLU-PRO": 0.2893 - } - }, - { - "id": "godlikehhd/alpaca_data_ins_ans_max_5200", - "name": "alpaca_data_ins_ans_max_5200", - "developer": "godlikehhd", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3479, - "hfopenllm_v2/BBH": 0.4098, - "hfopenllm_v2/MATH Level 5": 0.1027, - "hfopenllm_v2/GPQA": 0.2911, - "hfopenllm_v2/MUSR": 0.3602, - "hfopenllm_v2/MMLU-PRO": 0.2901 - } - }, - { - "id": "godlikehhd/alpaca_data_ins_max_5200", - "name": "alpaca_data_ins_max_5200", - "developer": "godlikehhd", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3275, - "hfopenllm_v2/BBH": 0.4155, - "hfopenllm_v2/MATH Level 5": 0.0997, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.3614, - "hfopenllm_v2/MMLU-PRO": 0.2916 - } - }, - { - "id": "godlikehhd/alpaca_data_ins_min_2600", - "name": "alpaca_data_ins_min_2600", - "developer": "godlikehhd", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.333, - "hfopenllm_v2/BBH": 0.4187, - "hfopenllm_v2/MATH Level 5": 0.111, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.3853, - "hfopenllm_v2/MMLU-PRO": 0.288 - } - }, - { - "id": "godlikehhd/alpaca_data_ins_min_5200", - "name": "alpaca_data_ins_min_5200", - "developer": "godlikehhd", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.336, - "hfopenllm_v2/BBH": 0.4289, - "hfopenllm_v2/MATH Level 5": 0.1035, - "hfopenllm_v2/GPQA": 0.2869, - "hfopenllm_v2/MUSR": 0.3906, - "hfopenllm_v2/MMLU-PRO": 0.2949 - } - }, - { - "id": "godlikehhd/alpaca_data_sampled_ifd_5200", - "name": "alpaca_data_sampled_ifd_5200", - "developer": "godlikehhd", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2924, - "hfopenllm_v2/BBH": 0.4033, - "hfopenllm_v2/MATH Level 5": 0.1254, - "hfopenllm_v2/GPQA": 0.3087, - "hfopenllm_v2/MUSR": 0.3521, - "hfopenllm_v2/MMLU-PRO": 0.2896 - } - }, - { - "id": "godlikehhd/alpaca_data_sampled_ifd_new_5200", - "name": "alpaca_data_sampled_ifd_new_5200", - "developer": "godlikehhd", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3663, - "hfopenllm_v2/BBH": 0.4178, - "hfopenllm_v2/MATH Level 5": 0.0944, - "hfopenllm_v2/GPQA": 0.2936, - "hfopenllm_v2/MUSR": 0.3613, - "hfopenllm_v2/MMLU-PRO": 0.2925 - } - }, - { - "id": "godlikehhd/alpaca_data_score_max_0.1_2600", - "name": "alpaca_data_score_max_0.1_2600", - "developer": "godlikehhd", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3288, - "hfopenllm_v2/BBH": 0.4252, - "hfopenllm_v2/MATH Level 5": 0.0989, - "hfopenllm_v2/GPQA": 0.2911, - "hfopenllm_v2/MUSR": 0.3706, - "hfopenllm_v2/MMLU-PRO": 0.2923 - } - }, - { - "id": "godlikehhd/alpaca_data_score_max_0.3_2600", - "name": "alpaca_data_score_max_0.3_2600", - "developer": "godlikehhd", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3375, - "hfopenllm_v2/BBH": 0.4151, - "hfopenllm_v2/MATH Level 5": 0.1035, - "hfopenllm_v2/GPQA": 0.2894, - "hfopenllm_v2/MUSR": 0.3759, - "hfopenllm_v2/MMLU-PRO": 0.2913 - } - }, - { - "id": "godlikehhd/alpaca_data_score_max_0.7_2600", - "name": "alpaca_data_score_max_0.7_2600", - "developer": "godlikehhd", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.364, - "hfopenllm_v2/BBH": 0.4185, - "hfopenllm_v2/MATH Level 5": 0.1073, - "hfopenllm_v2/GPQA": 0.3037, - "hfopenllm_v2/MUSR": 0.3469, - "hfopenllm_v2/MMLU-PRO": 0.2983 - } - }, - { - "id": "godlikehhd/alpaca_data_score_max_2500", - "name": "alpaca_data_score_max_2500", - "developer": "godlikehhd", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3564, - "hfopenllm_v2/BBH": 0.418, - "hfopenllm_v2/MATH Level 5": 0.0952, - "hfopenllm_v2/GPQA": 0.2953, - "hfopenllm_v2/MUSR": 0.3627, - "hfopenllm_v2/MMLU-PRO": 0.294 - } - }, - { - "id": "godlikehhd/alpaca_data_score_max_2600_3B", - "name": "alpaca_data_score_max_2600_3B", - "developer": "godlikehhd", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3358, - "hfopenllm_v2/BBH": 0.4716, - "hfopenllm_v2/MATH Level 5": 0.1548, - "hfopenllm_v2/GPQA": 0.2651, - "hfopenllm_v2/MUSR": 0.4474, - "hfopenllm_v2/MMLU-PRO": 0.3342 - } - }, - { - "id": "godlikehhd/alpaca_data_score_max_5200", - "name": "alpaca_data_score_max_5200", - "developer": "godlikehhd", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3445, - "hfopenllm_v2/BBH": 0.4242, - "hfopenllm_v2/MATH Level 5": 0.0974, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.3878, - "hfopenllm_v2/MMLU-PRO": 0.2945 - } - }, - { - "id": "godlikehhd/ifd_2500_qwen", - "name": "ifd_2500_qwen", - "developer": "godlikehhd", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3365, - "hfopenllm_v2/BBH": 0.4298, - "hfopenllm_v2/MATH Level 5": 0.0982, - "hfopenllm_v2/GPQA": 0.2953, - "hfopenllm_v2/MUSR": 0.3615, - "hfopenllm_v2/MMLU-PRO": 0.2921 - } - }, - { - "id": "godlikehhd/ifd_new_correct_all_sample_2500_qwen", - "name": "ifd_new_correct_all_sample_2500_qwen", - "developer": "godlikehhd", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3376, - "hfopenllm_v2/BBH": 0.402, - "hfopenllm_v2/MATH Level 5": 0.0959, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.3562, - "hfopenllm_v2/MMLU-PRO": 0.2889 - } - }, - { - "id": "godlikehhd/ifd_new_correct_sample_2500_qwen", - "name": "ifd_new_correct_sample_2500_qwen", - "developer": "godlikehhd", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3397, - "hfopenllm_v2/BBH": 0.411, - "hfopenllm_v2/MATH Level 5": 0.1042, - "hfopenllm_v2/GPQA": 0.3079, - "hfopenllm_v2/MUSR": 0.3627, - "hfopenllm_v2/MMLU-PRO": 0.2932 - } - }, - { - "id": "godlikehhd/ifd_new_qwen_2500", - "name": "ifd_new_qwen_2500", - "developer": "godlikehhd", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.324, - "hfopenllm_v2/BBH": 0.416, - "hfopenllm_v2/MATH Level 5": 0.1118, - "hfopenllm_v2/GPQA": 0.3003, - "hfopenllm_v2/MUSR": 0.359, - "hfopenllm_v2/MMLU-PRO": 0.2911 - } - }, - { - "id": "godlikehhd/qwen-2.5-1.5b-cherry", - "name": "qwen-2.5-1.5b-cherry", - "developer": "godlikehhd", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2893, - "hfopenllm_v2/BBH": 0.4036, - "hfopenllm_v2/MATH Level 5": 0.102, - "hfopenllm_v2/GPQA": 0.3003, - "hfopenllm_v2/MUSR": 0.3456, - "hfopenllm_v2/MMLU-PRO": 0.2923 - } - }, - { - "id": "godlikehhd/qwen_2.5-1.5b-cherry_new", - "name": "qwen_2.5-1.5b-cherry_new", - "developer": "godlikehhd", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.312, - "hfopenllm_v2/BBH": 0.415, - "hfopenllm_v2/MATH Level 5": 0.0967, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.3496, - "hfopenllm_v2/MMLU-PRO": 0.2894 - } - }, - { - "id": "godlikehhd/qwen_full_data_alpaca", - "name": "qwen_full_data_alpaca", - "developer": "godlikehhd", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3136, - "hfopenllm_v2/BBH": 0.4229, - "hfopenllm_v2/MATH Level 5": 0.0921, - "hfopenllm_v2/GPQA": 0.2928, - "hfopenllm_v2/MUSR": 0.4052, - "hfopenllm_v2/MMLU-PRO": 0.2851 - } - }, - { - "id": "godlikehhd/qwen_ins_ans_2500", - "name": "qwen_ins_ans_2500", - "developer": "godlikehhd", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2698, - "hfopenllm_v2/BBH": 0.4074, - "hfopenllm_v2/MATH Level 5": 0.114, - "hfopenllm_v2/GPQA": 0.2919, - "hfopenllm_v2/MUSR": 0.3589, - "hfopenllm_v2/MMLU-PRO": 0.2809 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/goekdeniz-guelmez.json b/data/developers/goekdeniz-guelmez.json deleted file mode 100644 index 2267332aba4462ef7fe64069f0fda516da5db997..0000000000000000000000000000000000000000 --- a/data/developers/goekdeniz-guelmez.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "developer": "Goekdeniz-Guelmez", - "models": [ - { - "id": "Goekdeniz-Guelmez/j.o.s.i.e.v4o-1.5b-dpo-stage1-v1", - "name": "j.o.s.i.e.v4o-1.5b-dpo-stage1-v1", - "developer": "Goekdeniz-Guelmez", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4188, - "hfopenllm_v2/BBH": 0.4124, - "hfopenllm_v2/MATH Level 5": 0.1201, - "hfopenllm_v2/GPQA": 0.2508, - "hfopenllm_v2/MUSR": 0.3529, - "hfopenllm_v2/MMLU-PRO": 0.2555 - } - }, - { - "id": "Goekdeniz-Guelmez/josie-3b-v6.0", - "name": "josie-3b-v6.0", - "developer": "Goekdeniz-Guelmez", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.601, - "hfopenllm_v2/BBH": 0.4496, - "hfopenllm_v2/MATH Level 5": 0.2938, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.3861, - "hfopenllm_v2/MMLU-PRO": 0.322 - } - }, - { - "id": "Goekdeniz-Guelmez/josie-7b-v6.0", - "name": "josie-7b-v6.0", - "developer": "Goekdeniz-Guelmez", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7412, - "hfopenllm_v2/BBH": 0.5105, - "hfopenllm_v2/MATH Level 5": 0.4358, - "hfopenllm_v2/GPQA": 0.2827, - "hfopenllm_v2/MUSR": 0.4154, - "hfopenllm_v2/MMLU-PRO": 0.3807 - } - }, - { - "id": "Goekdeniz-Guelmez/josie-7b-v6.0-step2000", - "name": "josie-7b-v6.0-step2000", - "developer": "Goekdeniz-Guelmez", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7628, - "hfopenllm_v2/BBH": 0.5098, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2802, - "hfopenllm_v2/MUSR": 0.4579, - "hfopenllm_v2/MMLU-PRO": 0.4033 - } - }, - { - "id": "Goekdeniz-Guelmez/Josiefied-Qwen2.5-0.5B-Instruct-abliterated-v1", - "name": "Josiefied-Qwen2.5-0.5B-Instruct-abliterated-v1", - "developer": "Goekdeniz-Guelmez", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3472, - "hfopenllm_v2/BBH": 0.3268, - "hfopenllm_v2/MATH Level 5": 0.0891, - "hfopenllm_v2/GPQA": 0.2517, - "hfopenllm_v2/MUSR": 0.3262, - "hfopenllm_v2/MMLU-PRO": 0.1641 - } - }, - { - "id": "Goekdeniz-Guelmez/Josiefied-Qwen2.5-1.5B-Instruct-abliterated-v1", - "name": "Josiefied-Qwen2.5-1.5B-Instruct-abliterated-v1", - "developer": "Goekdeniz-Guelmez", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4769, - "hfopenllm_v2/BBH": 0.4186, - "hfopenllm_v2/MATH Level 5": 0.2085, - "hfopenllm_v2/GPQA": 0.2433, - "hfopenllm_v2/MUSR": 0.3675, - "hfopenllm_v2/MMLU-PRO": 0.2783 - } - }, - { - "id": "Goekdeniz-Guelmez/Josiefied-Qwen2.5-1.5B-Instruct-abliterated-v2", - "name": "Josiefied-Qwen2.5-1.5B-Instruct-abliterated-v2", - "developer": "Goekdeniz-Guelmez", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4216, - "hfopenllm_v2/BBH": 0.4042, - "hfopenllm_v2/MATH Level 5": 0.1269, - "hfopenllm_v2/GPQA": 0.2399, - "hfopenllm_v2/MUSR": 0.3769, - "hfopenllm_v2/MMLU-PRO": 0.2562 - } - }, - { - "id": "Goekdeniz-Guelmez/Josiefied-Qwen2.5-1.5B-Instruct-abliterated-v3", - "name": "Josiefied-Qwen2.5-1.5B-Instruct-abliterated-v3", - "developer": "Goekdeniz-Guelmez", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4253, - "hfopenllm_v2/BBH": 0.4053, - "hfopenllm_v2/MATH Level 5": 0.1307, - "hfopenllm_v2/GPQA": 0.2433, - "hfopenllm_v2/MUSR": 0.3702, - "hfopenllm_v2/MMLU-PRO": 0.2556 - } - }, - { - "id": "Goekdeniz-Guelmez/Josiefied-Qwen2.5-14B-Instruct-abliterated-v4", - "name": "Josiefied-Qwen2.5-14B-Instruct-abliterated-v4", - "developer": "Goekdeniz-Guelmez", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8292, - "hfopenllm_v2/BBH": 0.6356, - "hfopenllm_v2/MATH Level 5": 0.5423, - "hfopenllm_v2/GPQA": 0.3423, - "hfopenllm_v2/MUSR": 0.4287, - "hfopenllm_v2/MMLU-PRO": 0.5018 - } - }, - { - "id": "Goekdeniz-Guelmez/Josiefied-Qwen2.5-7B-Instruct-abliterated-v2", - "name": "Josiefied-Qwen2.5-7B-Instruct-abliterated-v2", - "developer": "Goekdeniz-Guelmez", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7814, - "hfopenllm_v2/BBH": 0.531, - "hfopenllm_v2/MATH Level 5": 0.4532, - "hfopenllm_v2/GPQA": 0.2987, - "hfopenllm_v2/MUSR": 0.4354, - "hfopenllm_v2/MMLU-PRO": 0.412 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/google.json b/data/developers/google.json deleted file mode 100644 index 74805e9758d6a1e85b105b411bfda23c0bcf53ec..0000000000000000000000000000000000000000 --- a/data/developers/google.json +++ /dev/null @@ -1,2010 +0,0 @@ -{ - "developer": "Google", - "models": [ - { - "id": "google/codegemma-1.1-2b", - "name": "codegemma-1.1-2b", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2294, - "hfopenllm_v2/BBH": 0.3353, - "hfopenllm_v2/MATH Level 5": 0.0128, - "hfopenllm_v2/GPQA": 0.2651, - "hfopenllm_v2/MUSR": 0.3871, - "hfopenllm_v2/MMLU-PRO": 0.1278 - } - }, - { - "id": "google/flame-1.0-24B-july-2024", - "name": "google/flame-1.0-24B-july-2024", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.8781, - "reward-bench/Chat": 0.9218, - "reward-bench/Chat Hard": 0.7566, - "reward-bench/Safety": 0.8959, - "reward-bench/Reasoning": 0.938 - } - }, - { - "id": "google/flan-t5-base", - "name": "flan-t5-base", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1891, - "hfopenllm_v2/BBH": 0.3526, - "hfopenllm_v2/MATH Level 5": 0.0106, - "hfopenllm_v2/GPQA": 0.2383, - "hfopenllm_v2/MUSR": 0.3671, - "hfopenllm_v2/MMLU-PRO": 0.1357 - } - }, - { - "id": "google/flan-t5-large", - "name": "flan-t5-large", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2201, - "hfopenllm_v2/BBH": 0.4153, - "hfopenllm_v2/MATH Level 5": 0.0144, - "hfopenllm_v2/GPQA": 0.2508, - "hfopenllm_v2/MUSR": 0.4083, - "hfopenllm_v2/MMLU-PRO": 0.1709 - } - }, - { - "id": "google/flan-t5-small", - "name": "flan-t5-small", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1524, - "hfopenllm_v2/BBH": 0.3283, - "hfopenllm_v2/MATH Level 5": 0.0076, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.4123, - "hfopenllm_v2/MMLU-PRO": 0.1233 - } - }, - { - "id": "google/flan-t5-xl", - "name": "flan-t5-xl", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2207, - "hfopenllm_v2/BBH": 0.4537, - "hfopenllm_v2/MATH Level 5": 0.0008, - "hfopenllm_v2/GPQA": 0.2458, - "hfopenllm_v2/MUSR": 0.422, - "hfopenllm_v2/MMLU-PRO": 0.2142 - } - }, - { - "id": "google/flan-t5-xxl", - "name": "flan-t5-xxl", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.22, - "hfopenllm_v2/BBH": 0.5066, - "hfopenllm_v2/MATH Level 5": 0.0106, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.4218, - "hfopenllm_v2/MMLU-PRO": 0.2343 - } - }, - { - "id": "google/flan-ul2", - "name": "flan-ul2", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2393, - "hfopenllm_v2/BBH": 0.5054, - "hfopenllm_v2/MATH Level 5": 0.0091, - "hfopenllm_v2/GPQA": 0.2878, - "hfopenllm_v2/MUSR": 0.3844, - "hfopenllm_v2/MMLU-PRO": 0.2493 - } - }, - { - "id": "google/Gemini 2.5 Flash", - "name": "Gemini 2.5 Flash", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "ace/Overall Score": 0.38, - "ace/Gaming Score": 0.284, - "apex-v1/Overall Score": 0.604 - } - }, - { - "id": "google/Gemini 2.5 Pro", - "name": "Gemini 2.5 Pro", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "ace/Overall Score": 0.4, - "ace/Gaming Score": 0.285 - } - }, - { - "id": "google/Gemini 3 Flash", - "name": "Gemini 3 Flash", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "apex-agents/Overall Pass@1": 0.24, - "apex-agents/Overall Pass@8": 0.367, - "apex-agents/Overall Mean Score": 0.395, - "apex-agents/Investment Banking Pass@1": 0.267, - "apex-agents/Management Consulting Pass@1": 0.193, - "apex-agents/Corporate Law Pass@1": 0.259, - "apex-agents/Corporate Lawyer Mean Score": 0.524, - "ace/Gaming Score": 0.415, - "apex-v1/Overall Score": 0.64, - "apex-v1/Consulting Score": 0.64 - } - }, - { - "id": "google/Gemini 3 Pro", - "name": "Gemini 3 Pro", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "apex-agents/Overall Pass@1": 0.184, - "apex-agents/Overall Pass@8": 0.373, - "apex-agents/Overall Mean Score": 0.341, - "apex-agents/Investment Banking Pass@1": 0.188, - "apex-agents/Management Consulting Pass@1": 0.124, - "apex-agents/Corporate Law Pass@1": 0.239, - "apex-agents/Corporate Lawyer Mean Score": 0.487, - "ace/Overall Score": 0.47, - "ace/Gaming Score": 0.509, - "apex-v1/Overall Score": 0.643, - "apex-v1/Consulting Score": 0.64, - "apex-v1/Investment Banking Score": 0.63 - } - }, - { - "id": "google/Gemini 3.1 Pro", - "name": "Gemini 3.1 Pro", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "apex-agents/Overall Pass@1": 0.335, - "apex-agents/Corporate Lawyer Mean Score": 0.494 - } - }, - { - "id": "google/gemini-1.0-pro-001", - "name": "Gemini 1.0 Pro 001", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_mmlu/MMLU All Subjects": 0.7, - "helm_mmlu/Abstract Algebra": 0.34, - "helm_mmlu/Anatomy": 0.652, - "helm_mmlu/College Physics": 0.333, - "helm_mmlu/Computer Security": 0.84, - "helm_mmlu/Econometrics": 0.553, - "helm_mmlu/Global Facts": 0.49, - "helm_mmlu/Jurisprudence": 0.861, - "helm_mmlu/Philosophy": 0.762, - "helm_mmlu/Professional Psychology": 0.752, - "helm_mmlu/Us Foreign Policy": 0.89, - "helm_mmlu/Astronomy": 0.796, - "helm_mmlu/Business Ethics": 0.69, - "helm_mmlu/Clinical Knowledge": 0.758, - "helm_mmlu/Conceptual Physics": 0.706, - "helm_mmlu/Electrical Engineering": 0.69, - "helm_mmlu/Elementary Mathematics": 0.476, - "helm_mmlu/Formal Logic": 0.468, - "helm_mmlu/High School World History": 0.865, - "helm_mmlu/Human Sexuality": 0.618, - "helm_mmlu/International Law": 0.876, - "helm_mmlu/Logical Fallacies": 0.804, - "helm_mmlu/Machine Learning": 0.527, - "helm_mmlu/Management": 0.845, - "helm_mmlu/Marketing": 0.91, - "helm_mmlu/Medical Genetics": 0.8, - "helm_mmlu/Miscellaneous": 0.851, - "helm_mmlu/Moral Scenarios": 0.46, - "helm_mmlu/Nutrition": 0.788, - "helm_mmlu/Prehistory": 0.802, - "helm_mmlu/Public Relations": 0.691, - "helm_mmlu/Security Studies": 0.804, - "helm_mmlu/Sociology": 0.9, - "helm_mmlu/Virology": 0.536, - "helm_mmlu/World Religions": 0.86, - "helm_mmlu/Mean win rate": 0.677 - } - }, - { - "id": "google/gemini-1.0-pro-002", - "name": "Gemini 1.0 Pro 002", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.422, - "helm_lite/NarrativeQA": 0.751, - "helm_lite/NaturalQuestions (closed-book)": 0.391, - "helm_lite/OpenbookQA": 0.788, - "helm_lite/MMLU": 0.534, - "helm_lite/MATH": 0.665, - "helm_lite/GSM8K": 0.816, - "helm_lite/LegalBench": 0.475, - "helm_lite/MedQA": 0.483, - "helm_lite/WMT 2014": 0.194 - } - }, - { - "id": "google/gemini-1.5-flash-001", - "name": "Gemini 1.5 Flash 001", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.667, - "helm_lite/NarrativeQA": 0.783, - "helm_lite/NaturalQuestions (closed-book)": 0.332, - "helm_lite/OpenbookQA": 0.928, - "helm_lite/MMLU": 0.703, - "helm_lite/MATH": 0.753, - "helm_lite/GSM8K": 0.785, - "helm_lite/LegalBench": 0.661, - "helm_lite/MedQA": 0.68, - "helm_lite/WMT 2014": 0.225, - "helm_mmlu/MMLU All Subjects": 0.779, - "helm_mmlu/Abstract Algebra": 0.58, - "helm_mmlu/Anatomy": 0.8, - "helm_mmlu/College Physics": 0.696, - "helm_mmlu/Computer Security": 0.79, - "helm_mmlu/Econometrics": 0.614, - "helm_mmlu/Global Facts": 0.53, - "helm_mmlu/Jurisprudence": 0.889, - "helm_mmlu/Philosophy": 0.791, - "helm_mmlu/Professional Psychology": 0.828, - "helm_mmlu/Us Foreign Policy": 0.93, - "helm_mmlu/Astronomy": 0.882, - "helm_mmlu/Business Ethics": 0.81, - "helm_mmlu/Clinical Knowledge": 0.834, - "helm_mmlu/Conceptual Physics": 0.851, - "helm_mmlu/Electrical Engineering": 0.8, - "helm_mmlu/Elementary Mathematics": 0.754, - "helm_mmlu/Formal Logic": 0.627, - "helm_mmlu/High School World History": 0.907, - "helm_mmlu/Human Sexuality": 0.374, - "helm_mmlu/International Law": 0.901, - "helm_mmlu/Logical Fallacies": 0.853, - "helm_mmlu/Machine Learning": 0.571, - "helm_mmlu/Management": 0.864, - "helm_mmlu/Marketing": 0.94, - "helm_mmlu/Medical Genetics": 0.86, - "helm_mmlu/Miscellaneous": 0.886, - "helm_mmlu/Moral Scenarios": 0.637, - "helm_mmlu/Nutrition": 0.82, - "helm_mmlu/Prehistory": 0.867, - "helm_mmlu/Public Relations": 0.764, - "helm_mmlu/Security Studies": 0.808, - "helm_mmlu/Sociology": 0.915, - "helm_mmlu/Virology": 0.566, - "helm_mmlu/World Religions": 0.883, - "helm_mmlu/Mean win rate": 0.47, - "reward-bench/Score": 0.8054, - "reward-bench/Chat": 0.9218, - "reward-bench/Chat Hard": 0.6349, - "reward-bench/Safety": 0.8696, - "reward-bench/Reasoning": 0.8512, - "reward-bench/Prior Sets (0.5 weight)": 0.6937 - } - }, - { - "id": "google/gemini-1.5-flash-002", - "name": "Gemini 1.5 Flash 002", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.609, - "helm_capabilities/MMLU-Pro": 0.678, - "helm_capabilities/GPQA": 0.437, - "helm_capabilities/IFEval": 0.831, - "helm_capabilities/WildBench": 0.792, - "helm_capabilities/Omni-MATH": 0.305, - "helm_lite/Mean win rate": 0.573, - "helm_lite/NarrativeQA": 0.746, - "helm_lite/NaturalQuestions (closed-book)": 0.323, - "helm_lite/OpenbookQA": 0.914, - "helm_lite/MMLU": 0.679, - "helm_lite/MATH": 0.908, - "helm_lite/GSM8K": 0.328, - "helm_lite/LegalBench": 0.67, - "helm_lite/MedQA": 0.656, - "helm_lite/WMT 2014": 0.212, - "helm_mmlu/MMLU All Subjects": 0.739, - "helm_mmlu/Abstract Algebra": 0.63, - "helm_mmlu/Anatomy": 0.793, - "helm_mmlu/College Physics": 0.637, - "helm_mmlu/Computer Security": 0.72, - "helm_mmlu/Econometrics": 0.675, - "helm_mmlu/Global Facts": 0.47, - "helm_mmlu/Jurisprudence": 0.852, - "helm_mmlu/Philosophy": 0.797, - "helm_mmlu/Professional Psychology": 0.806, - "helm_mmlu/Us Foreign Policy": 0.81, - "helm_mmlu/Astronomy": 0.895, - "helm_mmlu/Business Ethics": 0.27, - "helm_mmlu/Clinical Knowledge": 0.792, - "helm_mmlu/Conceptual Physics": 0.851, - "helm_mmlu/Electrical Engineering": 0.772, - "helm_mmlu/Elementary Mathematics": 0.704, - "helm_mmlu/Formal Logic": 0.595, - "helm_mmlu/High School World History": 0.869, - "helm_mmlu/Human Sexuality": 0.847, - "helm_mmlu/International Law": 0.752, - "helm_mmlu/Logical Fallacies": 0.859, - "helm_mmlu/Machine Learning": 0.616, - "helm_mmlu/Management": 0.893, - "helm_mmlu/Marketing": 0.953, - "helm_mmlu/Medical Genetics": 0.89, - "helm_mmlu/Miscellaneous": 0.9, - "helm_mmlu/Moral Scenarios": 0.676, - "helm_mmlu/Nutrition": 0.588, - "helm_mmlu/Prehistory": 0.762, - "helm_mmlu/Public Relations": 0.7, - "helm_mmlu/Security Studies": 0.547, - "helm_mmlu/Sociology": 0.851, - "helm_mmlu/Virology": 0.524, - "helm_mmlu/World Religions": 0.865, - "helm_mmlu/Mean win rate": 0.817 - } - }, - { - "id": "google/gemini-1.5-flash-8b", - "name": "google/gemini-1.5-flash-8b", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.4851, - "reward-bench/Factuality": 0.4611, - "reward-bench/Precise IF": 0.3625, - "reward-bench/Math": 0.5082, - "reward-bench/Safety": 0.6622, - "reward-bench/Focus": 0.6747, - "reward-bench/Ties": 0.2421 - } - }, - { - "id": "google/gemini-1.5-flash-preview-0514", - "name": "Gemini 1.5 Flash 0514 preview", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_mmlu/MMLU All Subjects": 0.778, - "helm_mmlu/Abstract Algebra": 0.56, - "helm_mmlu/Anatomy": 0.807, - "helm_mmlu/College Physics": 0.667, - "helm_mmlu/Computer Security": 0.77, - "helm_mmlu/Econometrics": 0.64, - "helm_mmlu/Global Facts": 0.55, - "helm_mmlu/Jurisprudence": 0.889, - "helm_mmlu/Philosophy": 0.807, - "helm_mmlu/Professional Psychology": 0.825, - "helm_mmlu/Us Foreign Policy": 0.93, - "helm_mmlu/Astronomy": 0.868, - "helm_mmlu/Business Ethics": 0.82, - "helm_mmlu/Clinical Knowledge": 0.838, - "helm_mmlu/Conceptual Physics": 0.855, - "helm_mmlu/Electrical Engineering": 0.814, - "helm_mmlu/Elementary Mathematics": 0.778, - "helm_mmlu/Formal Logic": 0.611, - "helm_mmlu/High School World History": 0.907, - "helm_mmlu/Human Sexuality": 0.374, - "helm_mmlu/International Law": 0.876, - "helm_mmlu/Logical Fallacies": 0.853, - "helm_mmlu/Machine Learning": 0.562, - "helm_mmlu/Management": 0.854, - "helm_mmlu/Marketing": 0.936, - "helm_mmlu/Medical Genetics": 0.86, - "helm_mmlu/Miscellaneous": 0.884, - "helm_mmlu/Moral Scenarios": 0.631, - "helm_mmlu/Nutrition": 0.801, - "helm_mmlu/Prehistory": 0.867, - "helm_mmlu/Public Relations": 0.773, - "helm_mmlu/Security Studies": 0.812, - "helm_mmlu/Sociology": 0.9, - "helm_mmlu/Virology": 0.566, - "helm_mmlu/World Religions": 0.871, - "helm_mmlu/Mean win rate": 0.713 - } - }, - { - "id": "google/gemini-1.5-pro-001", - "name": "Gemini 1.5 Pro 001", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.739, - "helm_lite/NarrativeQA": 0.783, - "helm_lite/NaturalQuestions (closed-book)": 0.378, - "helm_lite/OpenbookQA": 0.902, - "helm_lite/MMLU": 0.772, - "helm_lite/MATH": 0.825, - "helm_lite/GSM8K": 0.836, - "helm_lite/LegalBench": 0.757, - "helm_lite/MedQA": 0.692, - "helm_lite/WMT 2014": 0.189, - "helm_mmlu/MMLU All Subjects": 0.827, - "helm_mmlu/Abstract Algebra": 0.75, - "helm_mmlu/Anatomy": 0.83, - "helm_mmlu/College Physics": 0.745, - "helm_mmlu/Computer Security": 0.83, - "helm_mmlu/Econometrics": 0.728, - "helm_mmlu/Global Facts": 0.66, - "helm_mmlu/Jurisprudence": 0.889, - "helm_mmlu/Philosophy": 0.871, - "helm_mmlu/Professional Psychology": 0.894, - "helm_mmlu/Us Foreign Policy": 0.93, - "helm_mmlu/Astronomy": 0.914, - "helm_mmlu/Business Ethics": 0.8, - "helm_mmlu/Clinical Knowledge": 0.853, - "helm_mmlu/Conceptual Physics": 0.949, - "helm_mmlu/Electrical Engineering": 0.745, - "helm_mmlu/Elementary Mathematics": 0.939, - "helm_mmlu/Formal Logic": 0.706, - "helm_mmlu/High School World History": 0.924, - "helm_mmlu/Human Sexuality": 0.374, - "helm_mmlu/International Law": 0.917, - "helm_mmlu/Logical Fallacies": 0.896, - "helm_mmlu/Machine Learning": 0.652, - "helm_mmlu/Management": 0.922, - "helm_mmlu/Marketing": 0.932, - "helm_mmlu/Medical Genetics": 0.91, - "helm_mmlu/Miscellaneous": 0.958, - "helm_mmlu/Moral Scenarios": 0.739, - "helm_mmlu/Nutrition": 0.879, - "helm_mmlu/Prehistory": 0.87, - "helm_mmlu/Public Relations": 0.818, - "helm_mmlu/Security Studies": 0.873, - "helm_mmlu/Sociology": 0.92, - "helm_mmlu/Virology": 0.554, - "helm_mmlu/World Religions": 0.854, - "helm_mmlu/Mean win rate": 0.349 - } - }, - { - "id": "google/gemini-1.5-pro-002", - "name": "Gemini 1.5 Pro 002", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.657, - "helm_capabilities/MMLU-Pro": 0.737, - "helm_capabilities/GPQA": 0.534, - "helm_capabilities/IFEval": 0.837, - "helm_capabilities/WildBench": 0.813, - "helm_capabilities/Omni-MATH": 0.364, - "helm_lite/Mean win rate": 0.842, - "helm_lite/NarrativeQA": 0.756, - "helm_lite/NaturalQuestions (closed-book)": 0.455, - "helm_lite/OpenbookQA": 0.952, - "helm_lite/MMLU": 0.795, - "helm_lite/MATH": 0.92, - "helm_lite/GSM8K": 0.817, - "helm_lite/LegalBench": 0.747, - "helm_lite/MedQA": 0.771, - "helm_lite/WMT 2014": 0.231, - "helm_mmlu/MMLU All Subjects": 0.869, - "helm_mmlu/Abstract Algebra": 0.82, - "helm_mmlu/Anatomy": 0.83, - "helm_mmlu/College Physics": 0.863, - "helm_mmlu/Computer Security": 0.85, - "helm_mmlu/Econometrics": 0.693, - "helm_mmlu/Global Facts": 0.77, - "helm_mmlu/Jurisprudence": 0.898, - "helm_mmlu/Philosophy": 0.887, - "helm_mmlu/Professional Psychology": 0.912, - "helm_mmlu/Us Foreign Policy": 0.94, - "helm_mmlu/Astronomy": 0.934, - "helm_mmlu/Business Ethics": 0.84, - "helm_mmlu/Clinical Knowledge": 0.906, - "helm_mmlu/Conceptual Physics": 0.945, - "helm_mmlu/Electrical Engineering": 0.855, - "helm_mmlu/Elementary Mathematics": 0.942, - "helm_mmlu/Formal Logic": 0.754, - "helm_mmlu/High School World History": 0.937, - "helm_mmlu/Human Sexuality": 0.878, - "helm_mmlu/International Law": 0.917, - "helm_mmlu/Logical Fallacies": 0.902, - "helm_mmlu/Machine Learning": 0.83, - "helm_mmlu/Management": 0.903, - "helm_mmlu/Marketing": 0.962, - "helm_mmlu/Medical Genetics": 0.92, - "helm_mmlu/Miscellaneous": 0.959, - "helm_mmlu/Moral Scenarios": 0.792, - "helm_mmlu/Nutrition": 0.886, - "helm_mmlu/Prehistory": 0.926, - "helm_mmlu/Public Relations": 0.809, - "helm_mmlu/Security Studies": 0.857, - "helm_mmlu/Sociology": 0.95, - "helm_mmlu/Virology": 0.566, - "helm_mmlu/World Religions": 0.889, - "helm_mmlu/Mean win rate": 0.334 - } - }, - { - "id": "google/gemini-1.5-pro-0514", - "name": "google/gemini-1.5-pro-0514", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.882, - "reward-bench/Chat": 0.9232, - "reward-bench/Chat Hard": 0.8059, - "reward-bench/Safety": 0.8791, - "reward-bench/Reasoning": 0.9199 - } - }, - { - "id": "google/gemini-1.5-pro-0924", - "name": "google/gemini-1.5-pro-0924", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.8678, - "reward-bench/Chat": 0.9413, - "reward-bench/Chat Hard": 0.7697, - "reward-bench/Safety": 0.8581, - "reward-bench/Reasoning": 0.9022 - } - }, - { - "id": "google/gemini-1.5-pro-preview-0409", - "name": "Gemini 1.5 Pro 0409 preview", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_mmlu/MMLU All Subjects": 0.81, - "helm_mmlu/Abstract Algebra": 0.6, - "helm_mmlu/Anatomy": 0.77, - "helm_mmlu/College Physics": 0.804, - "helm_mmlu/Computer Security": 0.81, - "helm_mmlu/Econometrics": 0.737, - "helm_mmlu/Global Facts": 0.66, - "helm_mmlu/Jurisprudence": 0.87, - "helm_mmlu/Philosophy": 0.846, - "helm_mmlu/Professional Psychology": 0.866, - "helm_mmlu/Us Foreign Policy": 0.94, - "helm_mmlu/Astronomy": 0.914, - "helm_mmlu/Business Ethics": 0.8, - "helm_mmlu/Clinical Knowledge": 0.868, - "helm_mmlu/Conceptual Physics": 0.915, - "helm_mmlu/Electrical Engineering": 0.772, - "helm_mmlu/Elementary Mathematics": 0.884, - "helm_mmlu/Formal Logic": 0.643, - "helm_mmlu/High School World History": 0.924, - "helm_mmlu/Human Sexuality": 0.397, - "helm_mmlu/International Law": 0.917, - "helm_mmlu/Logical Fallacies": 0.859, - "helm_mmlu/Machine Learning": 0.67, - "helm_mmlu/Management": 0.874, - "helm_mmlu/Marketing": 0.953, - "helm_mmlu/Medical Genetics": 0.91, - "helm_mmlu/Miscellaneous": 0.928, - "helm_mmlu/Moral Scenarios": 0.696, - "helm_mmlu/Nutrition": 0.846, - "helm_mmlu/Prehistory": 0.886, - "helm_mmlu/Public Relations": 0.755, - "helm_mmlu/Security Studies": 0.849, - "helm_mmlu/Sociology": 0.925, - "helm_mmlu/Virology": 0.584, - "helm_mmlu/World Religions": 0.877, - "helm_mmlu/Mean win rate": 0.118 - } - }, - { - "id": "google/gemini-2-5-flash-fc", - "name": "Gemini-2.5-Flash (FC)", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 15.0, - "bfcl/bfcl.overall.overall_accuracy": 56.24, - "bfcl/bfcl.overall.total_cost_usd": 26.36, - "bfcl/bfcl.overall.latency_mean_s": 2.99, - "bfcl/bfcl.overall.latency_std_s": 9.22, - "bfcl/bfcl.overall.latency_p95_s": 5.62, - "bfcl/bfcl.non_live.ast_accuracy": 84.96, - "bfcl/bfcl.non_live.simple_ast_accuracy": 74.33, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 92.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 94.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 79.5, - "bfcl/bfcl.live.live_accuracy": 74.39, - "bfcl/bfcl.live.live_simple_ast_accuracy": 85.27, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 71.7, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 81.25, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 70.83, - "bfcl/bfcl.multi_turn.accuracy": 36.25, - "bfcl/bfcl.multi_turn.base_accuracy": 41.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 36.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 32.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 35.5, - "bfcl/bfcl.web_search.accuracy": 59.0, - "bfcl/bfcl.web_search.base_accuracy": 59.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 59.0, - "bfcl/bfcl.memory.accuracy": 41.29, - "bfcl/bfcl.memory.kv_accuracy": 19.35, - "bfcl/bfcl.memory.vector_accuracy": 50.32, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 54.19, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 75.0, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 93.67 - } - }, - { - "id": "google/gemini-2-5-flash-lite-fc", - "name": "Gemini-2.5-Flash-Lite (FC)", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 52.0, - "bfcl/bfcl.overall.overall_accuracy": 36.87, - "bfcl/bfcl.overall.total_cost_usd": 7.55, - "bfcl/bfcl.overall.latency_mean_s": 1.18, - "bfcl/bfcl.overall.latency_std_s": 8.06, - "bfcl/bfcl.overall.latency_p95_s": 1.67, - "bfcl/bfcl.non_live.ast_accuracy": 86.6, - "bfcl/bfcl.non_live.simple_ast_accuracy": 70.92, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 90.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 93.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 92.0, - "bfcl/bfcl.live.live_accuracy": 65.8, - "bfcl/bfcl.live.live_simple_ast_accuracy": 73.26, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 63.82, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 75.0, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 66.67, - "bfcl/bfcl.multi_turn.accuracy": 13.5, - "bfcl/bfcl.multi_turn.base_accuracy": 20.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 1.5, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 15.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 17.5, - "bfcl/bfcl.web_search.accuracy": 21.0, - "bfcl/bfcl.web_search.base_accuracy": 26.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 16.0, - "bfcl/bfcl.memory.accuracy": 20.65, - "bfcl/bfcl.memory.kv_accuracy": 3.87, - "bfcl/bfcl.memory.vector_accuracy": 6.45, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 51.61, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 43.75, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 92.5 - } - }, - { - "id": "google/gemini-2-5-flash-lite-prompt", - "name": "Gemini-2.5-Flash-Lite (Prompt)", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 73.0, - "bfcl/bfcl.overall.overall_accuracy": 28.03, - "bfcl/bfcl.overall.total_cost_usd": 7.05, - "bfcl/bfcl.overall.latency_mean_s": 1.0, - "bfcl/bfcl.overall.latency_std_s": 4.75, - "bfcl/bfcl.overall.latency_p95_s": 1.4, - "bfcl/bfcl.non_live.ast_accuracy": 83.9, - "bfcl/bfcl.non_live.simple_ast_accuracy": 70.08, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 86.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 90.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 89.5, - "bfcl/bfcl.live.live_accuracy": 54.85, - "bfcl/bfcl.live.live_simple_ast_accuracy": 67.05, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 51.66, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 75.0, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 50.0, - "bfcl/bfcl.multi_turn.accuracy": 7.63, - "bfcl/bfcl.multi_turn.base_accuracy": 10.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 5.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 6.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 9.0, - "bfcl/bfcl.web_search.accuracy": 0.0, - "bfcl/bfcl.web_search.base_accuracy": 0.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 0.0, - "bfcl/bfcl.memory.accuracy": 12.69, - "bfcl/bfcl.memory.kv_accuracy": 1.94, - "bfcl/bfcl.memory.vector_accuracy": 6.45, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 29.68, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 50.0, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 93.33, - "bfcl/bfcl.format_sensitivity.max_delta": 25.5, - "bfcl/bfcl.format_sensitivity.stddev": 6.68 - } - }, - { - "id": "google/gemini-2-5-flash-prompt", - "name": "Gemini-2.5-Flash (Prompt)", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 26.0, - "bfcl/bfcl.overall.overall_accuracy": 50.9, - "bfcl/bfcl.overall.total_cost_usd": 33.45, - "bfcl/bfcl.overall.latency_mean_s": 3.18, - "bfcl/bfcl.overall.latency_std_s": 4.44, - "bfcl/bfcl.overall.latency_p95_s": 6.09, - "bfcl/bfcl.non_live.ast_accuracy": 88.08, - "bfcl/bfcl.non_live.simple_ast_accuracy": 77.33, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 91.5, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 96.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 87.5, - "bfcl/bfcl.live.live_accuracy": 78.16, - "bfcl/bfcl.live.live_simple_ast_accuracy": 87.21, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 75.97, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 81.25, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 75.0, - "bfcl/bfcl.multi_turn.accuracy": 16.75, - "bfcl/bfcl.multi_turn.base_accuracy": 14.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 16.5, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 17.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 18.5, - "bfcl/bfcl.web_search.accuracy": 62.0, - "bfcl/bfcl.web_search.base_accuracy": 60.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 64.0, - "bfcl/bfcl.memory.accuracy": 38.71, - "bfcl/bfcl.memory.kv_accuracy": 13.55, - "bfcl/bfcl.memory.vector_accuracy": 47.1, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 55.48, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 62.5, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 91.09, - "bfcl/bfcl.format_sensitivity.max_delta": 9.0, - "bfcl/bfcl.format_sensitivity.stddev": 2.45 - } - }, - { - "id": "google/gemini-2.0-flash-001", - "name": "Gemini 2.0 Flash", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.679, - "helm_capabilities/MMLU-Pro": 0.737, - "helm_capabilities/GPQA": 0.556, - "helm_capabilities/IFEval": 0.841, - "helm_capabilities/WildBench": 0.8, - "helm_capabilities/Omni-MATH": 0.459 - } - }, - { - "id": "google/gemini-2.0-flash-exp", - "name": "Gemini 2.0 Flash Experimental", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.813, - "helm_lite/NarrativeQA": 0.783, - "helm_lite/NaturalQuestions (closed-book)": 0.443, - "helm_lite/OpenbookQA": 0.946, - "helm_lite/MMLU": 0.717, - "helm_lite/MATH": 0.901, - "helm_lite/GSM8K": 0.946, - "helm_lite/LegalBench": 0.674, - "helm_lite/MedQA": 0.73, - "helm_lite/WMT 2014": 0.212, - "helm_mmlu/MMLU All Subjects": 0.797, - "helm_mmlu/Abstract Algebra": 0.72, - "helm_mmlu/Anatomy": 0.807, - "helm_mmlu/College Physics": 0.696, - "helm_mmlu/Computer Security": 0.83, - "helm_mmlu/Econometrics": 0.693, - "helm_mmlu/Global Facts": 0.66, - "helm_mmlu/Jurisprudence": 0.898, - "helm_mmlu/Philosophy": 0.887, - "helm_mmlu/Professional Psychology": 0.876, - "helm_mmlu/Us Foreign Policy": 0.78, - "helm_mmlu/Astronomy": 0.928, - "helm_mmlu/Business Ethics": 0.73, - "helm_mmlu/Clinical Knowledge": 0.879, - "helm_mmlu/Conceptual Physics": 0.813, - "helm_mmlu/Electrical Engineering": 0.834, - "helm_mmlu/Elementary Mathematics": 0.857, - "helm_mmlu/Formal Logic": 0.571, - "helm_mmlu/High School World History": 0.743, - "helm_mmlu/Human Sexuality": 0.901, - "helm_mmlu/International Law": 0.645, - "helm_mmlu/Logical Fallacies": 0.914, - "helm_mmlu/Machine Learning": 0.759, - "helm_mmlu/Management": 0.718, - "helm_mmlu/Marketing": 0.944, - "helm_mmlu/Medical Genetics": 0.89, - "helm_mmlu/Miscellaneous": 0.939, - "helm_mmlu/Moral Scenarios": 0.815, - "helm_mmlu/Nutrition": 0.856, - "helm_mmlu/Prehistory": 0.898, - "helm_mmlu/Public Relations": 0.791, - "helm_mmlu/Security Studies": 0.69, - "helm_mmlu/Sociology": 0.786, - "helm_mmlu/Virology": 0.554, - "helm_mmlu/World Religions": 0.731, - "helm_mmlu/Mean win rate": 0.567 - } - }, - { - "id": "google/gemini-2.0-flash-lite-preview-02-05", - "name": "Gemini 2.0 Flash Lite 02-05 preview", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.642, - "helm_capabilities/MMLU-Pro": 0.72, - "helm_capabilities/GPQA": 0.5, - "helm_capabilities/IFEval": 0.824, - "helm_capabilities/WildBench": 0.79, - "helm_capabilities/Omni-MATH": 0.374 - } - }, - { - "id": "google/gemini-2.5-flash", - "name": "Gemini 2.5 Flash", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "global-mmlu-lite/Global MMLU Lite": 0.9145, - "global-mmlu-lite/Culturally Sensitive": 0.9, - "global-mmlu-lite/Culturally Agnostic": 0.9291, - "global-mmlu-lite/Arabic": 0.9125, - "global-mmlu-lite/English": 0.9325, - "global-mmlu-lite/Bengali": 0.91, - "global-mmlu-lite/German": 0.9025, - "global-mmlu-lite/French": 0.91, - "global-mmlu-lite/Hindi": 0.925, - "global-mmlu-lite/Indonesian": 0.9075, - "global-mmlu-lite/Italian": 0.9225, - "global-mmlu-lite/Japanese": 0.9125, - "global-mmlu-lite/Korean": 0.915, - "global-mmlu-lite/Portuguese": 0.9125, - "global-mmlu-lite/Spanish": 0.9175, - "global-mmlu-lite/Swahili": 0.915, - "global-mmlu-lite/Yoruba": 0.9075, - "global-mmlu-lite/Chinese": 0.915, - "global-mmlu-lite/Burmese": 0.915, - "livecodebenchpro/Hard Problems": 0.0, - "livecodebenchpro/Medium Problems": 0.028169014084507043, - "livecodebenchpro/Easy Problems": 0.38028169014084506, - "reward-bench/Score": 0.7767, - "reward-bench/Factuality": 0.674, - "reward-bench/Precise IF": 0.575, - "reward-bench/Math": 0.852, - "reward-bench/Safety": 0.909, - "reward-bench/Focus": 0.841, - "reward-bench/Ties": 0.809, - "terminal-bench-2.0/terminal-bench-2.0": 16.9 - } - }, - { - "id": "google/gemini-2.5-flash-lite", - "name": "Gemini 2.5 Flash-Lite", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.591, - "helm_capabilities/MMLU-Pro": 0.537, - "helm_capabilities/GPQA": 0.309, - "helm_capabilities/IFEval": 0.81, - "helm_capabilities/WildBench": 0.818, - "helm_capabilities/Omni-MATH": 0.48 - } - }, - { - "id": "google/gemini-2.5-flash-preview-04-17", - "name": "Gemini 2.5 Flash 04-17 preview", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.626, - "helm_capabilities/MMLU-Pro": 0.639, - "helm_capabilities/GPQA": 0.39, - "helm_capabilities/IFEval": 0.898, - "helm_capabilities/WildBench": 0.817, - "helm_capabilities/Omni-MATH": 0.384, - "reward-bench/Score": 0.7721, - "reward-bench/Factuality": 0.6574, - "reward-bench/Precise IF": 0.5531, - "reward-bench/Math": 0.8115, - "reward-bench/Safety": 0.9094, - "reward-bench/Focus": 0.8672, - "reward-bench/Ties": 0.8341 - } - }, - { - "id": "google/gemini-2.5-flash-preview-05-20", - "name": "gemini-2.5-flash-preview-05-20", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "global-mmlu-lite/Global MMLU Lite": 0.9092, - "global-mmlu-lite/Culturally Sensitive": 0.8925, - "global-mmlu-lite/Culturally Agnostic": 0.9259, - "global-mmlu-lite/Arabic": 0.905, - "global-mmlu-lite/English": 0.9225, - "global-mmlu-lite/Bengali": 0.91, - "global-mmlu-lite/German": 0.905, - "global-mmlu-lite/French": 0.925, - "global-mmlu-lite/Hindi": 0.9125, - "global-mmlu-lite/Indonesian": 0.9075, - "global-mmlu-lite/Italian": 0.89, - "global-mmlu-lite/Japanese": 0.9125, - "global-mmlu-lite/Korean": 0.9075, - "global-mmlu-lite/Portuguese": 0.915, - "global-mmlu-lite/Spanish": 0.915, - "global-mmlu-lite/Swahili": 0.905, - "global-mmlu-lite/Yoruba": 0.8825, - "global-mmlu-lite/Chinese": 0.93, - "global-mmlu-lite/Burmese": 0.9025 - } - }, - { - "id": "google/gemini-2.5-pro", - "name": "Gemini 2.5 Pro", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "global-mmlu-lite/Global MMLU Lite": 0.9323, - "global-mmlu-lite/Culturally Sensitive": 0.9241, - "global-mmlu-lite/Culturally Agnostic": 0.9406, - "global-mmlu-lite/Arabic": 0.9475, - "global-mmlu-lite/English": 0.9275, - "global-mmlu-lite/Bengali": 0.9275, - "global-mmlu-lite/German": 0.93, - "global-mmlu-lite/French": 0.9425, - "global-mmlu-lite/Hindi": 0.9275, - "global-mmlu-lite/Indonesian": 0.925, - "global-mmlu-lite/Italian": 0.935, - "global-mmlu-lite/Japanese": 0.9375, - "global-mmlu-lite/Korean": 0.9275, - "global-mmlu-lite/Portuguese": 0.93, - "global-mmlu-lite/Spanish": 0.94, - "global-mmlu-lite/Swahili": 0.9375, - "global-mmlu-lite/Yoruba": 0.925, - "global-mmlu-lite/Chinese": 0.9275, - "global-mmlu-lite/Burmese": 0.93, - "livecodebenchpro/Hard Problems": 0.014084507042253521, - "livecodebenchpro/Medium Problems": 0.2112676056338028, - "livecodebenchpro/Easy Problems": 0.7183098591549296, - "reward-bench/Score": 0.7948, - "reward-bench/Factuality": 0.755, - "reward-bench/Precise IF": 0.619, - "reward-bench/Math": 0.898, - "reward-bench/Safety": 0.881, - "reward-bench/Focus": 0.805, - "reward-bench/Ties": 0.811, - "terminal-bench-2.0/terminal-bench-2.0": 19.6 - } - }, - { - "id": "google/gemini-2.5-pro-preview-03-25", - "name": "Gemini 2.5 Pro 03-25 preview", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.745, - "helm_capabilities/MMLU-Pro": 0.863, - "helm_capabilities/GPQA": 0.749, - "helm_capabilities/IFEval": 0.84, - "helm_capabilities/WildBench": 0.857, - "helm_capabilities/Omni-MATH": 0.416 - } - }, - { - "id": "google/gemini-2.5-pro-preview-05-06", - "name": "google/gemini-2.5-pro-preview-05-06", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6775, - "reward-bench/Factuality": 0.6532, - "reward-bench/Precise IF": 0.4688, - "reward-bench/Math": 0.5342, - "reward-bench/Safety": 0.8806, - "reward-bench/Focus": 0.8308, - "reward-bench/Ties": 0.6973 - } - }, - { - "id": "google/gemini-3-flash", - "name": "Gemini 3 Flash", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "terminal-bench-2.0/terminal-bench-2.0": 47.4 - } - }, - { - "id": "google/gemini-3-pro", - "name": "Gemini 3 Pro", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "terminal-bench-2.0/terminal-bench-2.0": 56.9 - } - }, - { - "id": "google/gemini-3-pro-preview", - "name": "gemini-3-pro-preview", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "appworld_test_normal/appworld/test_normal": 0.505, - "browsecompplus/browsecompplus": 0.48, - "global-mmlu-lite/Global MMLU Lite": 0.9453, - "global-mmlu-lite/Culturally Sensitive": 0.9397, - "global-mmlu-lite/Culturally Agnostic": 0.9509, - "global-mmlu-lite/Arabic": 0.9475, - "global-mmlu-lite/English": 0.9425, - "global-mmlu-lite/Bengali": 0.9425, - "global-mmlu-lite/German": 0.94, - "global-mmlu-lite/French": 0.9575, - "global-mmlu-lite/Hindi": 0.9425, - "global-mmlu-lite/Indonesian": 0.955, - "global-mmlu-lite/Italian": 0.955, - "global-mmlu-lite/Japanese": 0.94, - "global-mmlu-lite/Korean": 0.94, - "global-mmlu-lite/Portuguese": 0.9425, - "global-mmlu-lite/Spanish": 0.9475, - "global-mmlu-lite/Swahili": 0.94, - "global-mmlu-lite/Yoruba": 0.9425, - "global-mmlu-lite/Chinese": 0.9475, - "global-mmlu-lite/Burmese": 0.9425, - "swe-bench/swe-bench": 0.71, - "tau-bench-2_airline/tau-bench-2/airline": 0.62, - "tau-bench-2_retail/tau-bench-2/retail": 0.7576, - "tau-bench-2_telecom/tau-bench-2/telecom": 0.73 - } - }, - { - "id": "google/gemini-3-pro-preview-fc", - "name": "Gemini-3-Pro-Preview (FC)", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 7.0, - "bfcl/bfcl.overall.overall_accuracy": 68.14, - "bfcl/bfcl.overall.total_cost_usd": 224.69, - "bfcl/bfcl.overall.latency_mean_s": 15.87, - "bfcl/bfcl.overall.latency_std_s": 41.41, - "bfcl/bfcl.overall.latency_p95_s": 58.48, - "bfcl/bfcl.non_live.ast_accuracy": 85.75, - "bfcl/bfcl.non_live.simple_ast_accuracy": 75.5, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 94.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 91.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 82.5, - "bfcl/bfcl.live.live_accuracy": 81.72, - "bfcl/bfcl.live.live_simple_ast_accuracy": 87.6, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 80.44, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 75.0, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 79.17, - "bfcl/bfcl.multi_turn.accuracy": 63.12, - "bfcl/bfcl.multi_turn.base_accuracy": 69.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 63.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 56.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 64.0, - "bfcl/bfcl.web_search.accuracy": 68.5, - "bfcl/bfcl.web_search.base_accuracy": 63.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 74.0, - "bfcl/bfcl.memory.accuracy": 54.84, - "bfcl/bfcl.memory.kv_accuracy": 50.32, - "bfcl/bfcl.memory.vector_accuracy": 63.23, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 50.97, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 75.0, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 77.85 - } - }, - { - "id": "google/gemini-3-pro-preview-prompt", - "name": "Gemini-3-Pro-Preview (Prompt)", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 3.0, - "bfcl/bfcl.overall.overall_accuracy": 72.51, - "bfcl/bfcl.overall.total_cost_usd": 298.47, - "bfcl/bfcl.overall.latency_mean_s": 12.08, - "bfcl/bfcl.overall.latency_std_s": 21.3, - "bfcl/bfcl.overall.latency_p95_s": 32.73, - "bfcl/bfcl.non_live.ast_accuracy": 90.65, - "bfcl/bfcl.non_live.simple_ast_accuracy": 79.58, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 96.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 95.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 92.0, - "bfcl/bfcl.live.live_accuracy": 83.12, - "bfcl/bfcl.live.live_simple_ast_accuracy": 87.6, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 81.77, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 93.75, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 87.5, - "bfcl/bfcl.multi_turn.accuracy": 60.75, - "bfcl/bfcl.multi_turn.base_accuracy": 64.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 60.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 54.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 64.0, - "bfcl/bfcl.web_search.accuracy": 80.0, - "bfcl/bfcl.web_search.base_accuracy": 78.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 82.0, - "bfcl/bfcl.memory.accuracy": 61.72, - "bfcl/bfcl.memory.kv_accuracy": 59.35, - "bfcl/bfcl.memory.vector_accuracy": 62.58, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 63.23, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 68.75, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 85.59, - "bfcl/bfcl.format_sensitivity.max_delta": 8.5, - "bfcl/bfcl.format_sensitivity.stddev": 1.7 - } - }, - { - "id": "google/gemini-3.1-pro", - "name": "Gemini 3.1 Pro", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "terminal-bench-2.0/terminal-bench-2.0": 74.8 - } - }, - { - "id": "google/gemma-1.1-2b-it", - "name": "gemma-1.1-2b-it", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3067, - "hfopenllm_v2/BBH": 0.3185, - "hfopenllm_v2/MATH Level 5": 0.0181, - "hfopenllm_v2/GPQA": 0.2693, - "hfopenllm_v2/MUSR": 0.3394, - "hfopenllm_v2/MMLU-PRO": 0.1484 - } - }, - { - "id": "google/gemma-1.1-7b-it", - "name": "gemma-1.1-7b-it", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5039, - "hfopenllm_v2/BBH": 0.3935, - "hfopenllm_v2/MATH Level 5": 0.0491, - "hfopenllm_v2/GPQA": 0.2936, - "hfopenllm_v2/MUSR": 0.423, - "hfopenllm_v2/MMLU-PRO": 0.2584 - } - }, - { - "id": "google/gemma-2-27b", - "name": "Gemma 2 27B", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_mmlu/MMLU All Subjects": 0.757, - "helm_mmlu/Abstract Algebra": 0.4, - "helm_mmlu/Anatomy": 0.77, - "helm_mmlu/College Physics": 0.5, - "helm_mmlu/Computer Security": 0.84, - "helm_mmlu/Econometrics": 0.667, - "helm_mmlu/Global Facts": 0.43, - "helm_mmlu/Jurisprudence": 0.861, - "helm_mmlu/Philosophy": 0.849, - "helm_mmlu/Professional Psychology": 0.84, - "helm_mmlu/Us Foreign Policy": 0.95, - "helm_mmlu/Astronomy": 0.829, - "helm_mmlu/Business Ethics": 0.78, - "helm_mmlu/Clinical Knowledge": 0.808, - "helm_mmlu/Conceptual Physics": 0.834, - "helm_mmlu/Electrical Engineering": 0.738, - "helm_mmlu/Elementary Mathematics": 0.558, - "helm_mmlu/Formal Logic": 0.516, - "helm_mmlu/High School World History": 0.89, - "helm_mmlu/Human Sexuality": 0.84, - "helm_mmlu/International Law": 0.843, - "helm_mmlu/Logical Fallacies": 0.865, - "helm_mmlu/Machine Learning": 0.625, - "helm_mmlu/Management": 0.864, - "helm_mmlu/Marketing": 0.94, - "helm_mmlu/Medical Genetics": 0.87, - "helm_mmlu/Miscellaneous": 0.885, - "helm_mmlu/Moral Scenarios": 0.394, - "helm_mmlu/Nutrition": 0.824, - "helm_mmlu/Prehistory": 0.877, - "helm_mmlu/Public Relations": 0.745, - "helm_mmlu/Security Studies": 0.808, - "helm_mmlu/Sociology": 0.9, - "helm_mmlu/Virology": 0.56, - "helm_mmlu/World Religions": 0.924, - "helm_mmlu/Mean win rate": 0.05, - "hfopenllm_v2/IFEval": 0.2475, - "hfopenllm_v2/BBH": 0.5643, - "hfopenllm_v2/MATH Level 5": 0.1662, - "hfopenllm_v2/GPQA": 0.3507, - "hfopenllm_v2/MUSR": 0.4396, - "hfopenllm_v2/MMLU-PRO": 0.4371 - } - }, - { - "id": "google/gemma-2-27b-it", - "name": "Gemma 2 Instruct 27B", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.675, - "helm_lite/NarrativeQA": 0.79, - "helm_lite/NaturalQuestions (closed-book)": 0.353, - "helm_lite/OpenbookQA": 0.918, - "helm_lite/MMLU": 0.664, - "helm_lite/MATH": 0.746, - "helm_lite/GSM8K": 0.812, - "helm_lite/LegalBench": 0.7, - "helm_lite/MedQA": 0.684, - "helm_lite/WMT 2014": 0.214, - "hfopenllm_v2/IFEval": 0.7978, - "hfopenllm_v2/BBH": 0.6451, - "hfopenllm_v2/MATH Level 5": 0.2387, - "hfopenllm_v2/GPQA": 0.375, - "hfopenllm_v2/MUSR": 0.4033, - "hfopenllm_v2/MMLU-PRO": 0.4451, - "reward-bench/Score": 0.809, - "reward-bench/Chat": 0.9483, - "reward-bench/Chat Hard": 0.591, - "reward-bench/Safety": 0.8635, - "reward-bench/Reasoning": 0.833 - } - }, - { - "id": "google/gemma-2-2b", - "name": "gemma-2-2b", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1993, - "hfopenllm_v2/BBH": 0.3656, - "hfopenllm_v2/MATH Level 5": 0.0287, - "hfopenllm_v2/GPQA": 0.2626, - "hfopenllm_v2/MUSR": 0.4232, - "hfopenllm_v2/MMLU-PRO": 0.218 - } - }, - { - "id": "google/gemma-2-2b-it", - "name": "gemma-2-2b-it", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5668, - "hfopenllm_v2/BBH": 0.4199, - "hfopenllm_v2/MATH Level 5": 0.0008, - "hfopenllm_v2/GPQA": 0.2743, - "hfopenllm_v2/MUSR": 0.3929, - "hfopenllm_v2/MMLU-PRO": 0.255 - } - }, - { - "id": "google/gemma-2-2b-jpn-it", - "name": "gemma-2-2b-jpn-it", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5288, - "hfopenllm_v2/BBH": 0.4178, - "hfopenllm_v2/MATH Level 5": 0.0476, - "hfopenllm_v2/GPQA": 0.2752, - "hfopenllm_v2/MUSR": 0.3728, - "hfopenllm_v2/MMLU-PRO": 0.2467 - } - }, - { - "id": "google/gemma-2-9b", - "name": "Gemma 2 9B", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_mmlu/MMLU All Subjects": 0.721, - "helm_mmlu/Abstract Algebra": 0.4, - "helm_mmlu/Anatomy": 0.704, - "helm_mmlu/College Physics": 0.5, - "helm_mmlu/Computer Security": 0.81, - "helm_mmlu/Econometrics": 0.579, - "helm_mmlu/Global Facts": 0.53, - "helm_mmlu/Jurisprudence": 0.833, - "helm_mmlu/Philosophy": 0.772, - "helm_mmlu/Professional Psychology": 0.788, - "helm_mmlu/Us Foreign Policy": 0.9, - "helm_mmlu/Astronomy": 0.789, - "helm_mmlu/Business Ethics": 0.77, - "helm_mmlu/Clinical Knowledge": 0.777, - "helm_mmlu/Conceptual Physics": 0.732, - "helm_mmlu/Electrical Engineering": 0.724, - "helm_mmlu/Elementary Mathematics": 0.577, - "helm_mmlu/Formal Logic": 0.492, - "helm_mmlu/High School World History": 0.865, - "helm_mmlu/Human Sexuality": 0.809, - "helm_mmlu/International Law": 0.835, - "helm_mmlu/Logical Fallacies": 0.816, - "helm_mmlu/Machine Learning": 0.509, - "helm_mmlu/Management": 0.874, - "helm_mmlu/Marketing": 0.919, - "helm_mmlu/Medical Genetics": 0.84, - "helm_mmlu/Miscellaneous": 0.844, - "helm_mmlu/Moral Scenarios": 0.295, - "helm_mmlu/Nutrition": 0.775, - "helm_mmlu/Prehistory": 0.812, - "helm_mmlu/Public Relations": 0.736, - "helm_mmlu/Security Studies": 0.78, - "helm_mmlu/Sociology": 0.9, - "helm_mmlu/Virology": 0.53, - "helm_mmlu/World Religions": 0.86, - "helm_mmlu/Mean win rate": 0.265, - "hfopenllm_v2/IFEval": 0.204, - "hfopenllm_v2/BBH": 0.5377, - "hfopenllm_v2/MATH Level 5": 0.1344, - "hfopenllm_v2/GPQA": 0.3289, - "hfopenllm_v2/MUSR": 0.4461, - "hfopenllm_v2/MMLU-PRO": 0.4103 - } - }, - { - "id": "google/gemma-2-9b-it", - "name": "Gemma 2 Instruct 9B", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.562, - "helm_lite/NarrativeQA": 0.768, - "helm_lite/NaturalQuestions (closed-book)": 0.328, - "helm_lite/OpenbookQA": 0.91, - "helm_lite/MMLU": 0.645, - "helm_lite/MATH": 0.724, - "helm_lite/GSM8K": 0.762, - "helm_lite/LegalBench": 0.639, - "helm_lite/MedQA": 0.63, - "helm_lite/WMT 2014": 0.201, - "hfopenllm_v2/IFEval": 0.7436, - "hfopenllm_v2/BBH": 0.599, - "hfopenllm_v2/MATH Level 5": 0.1949, - "hfopenllm_v2/GPQA": 0.3607, - "hfopenllm_v2/MUSR": 0.4073, - "hfopenllm_v2/MMLU-PRO": 0.3875, - "la_leaderboard/la_leaderboard": 33.62 - } - }, - { - "id": "google/gemma-2b", - "name": "gemma-2b", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2038, - "hfopenllm_v2/BBH": 0.3366, - "hfopenllm_v2/MATH Level 5": 0.0302, - "hfopenllm_v2/GPQA": 0.255, - "hfopenllm_v2/MUSR": 0.3978, - "hfopenllm_v2/MMLU-PRO": 0.1366 - } - }, - { - "id": "google/gemma-2b-it", - "name": "gemma-2b-it", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.269, - "hfopenllm_v2/BBH": 0.3151, - "hfopenllm_v2/MATH Level 5": 0.0204, - "hfopenllm_v2/GPQA": 0.2785, - "hfopenllm_v2/MUSR": 0.3341, - "hfopenllm_v2/MMLU-PRO": 0.1353 - } - }, - { - "id": "google/gemma-3-12b-it-prompt", - "name": "Gemma-3-12b-it (Prompt)", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 66.0, - "bfcl/bfcl.overall.overall_accuracy": 30.43, - "bfcl/bfcl.overall.total_cost_usd": 10.77, - "bfcl/bfcl.overall.latency_mean_s": 11.1, - "bfcl/bfcl.overall.latency_std_s": 17.17, - "bfcl/bfcl.overall.latency_p95_s": 34.66, - "bfcl/bfcl.non_live.ast_accuracy": 79.44, - "bfcl/bfcl.non_live.simple_ast_accuracy": 76.25, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 94.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 91.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 56.5, - "bfcl/bfcl.live.live_accuracy": 74.24, - "bfcl/bfcl.live.live_simple_ast_accuracy": 85.66, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 71.89, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 87.5, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 45.83, - "bfcl/bfcl.multi_turn.accuracy": 5.75, - "bfcl/bfcl.multi_turn.base_accuracy": 6.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 7.5, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 5.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 4.0, - "bfcl/bfcl.web_search.accuracy": 4.0, - "bfcl/bfcl.web_search.base_accuracy": 4.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 4.0, - "bfcl/bfcl.memory.accuracy": 27.53, - "bfcl/bfcl.memory.kv_accuracy": 8.39, - "bfcl/bfcl.memory.vector_accuracy": 25.16, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 49.03, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 93.75, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 70.29, - "bfcl/bfcl.format_sensitivity.max_delta": 67.5, - "bfcl/bfcl.format_sensitivity.stddev": 22.41 - } - }, - { - "id": "google/gemma-3-1b-it-prompt", - "name": "Gemma-3-1b-it (Prompt)", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 109.0, - "bfcl/bfcl.overall.overall_accuracy": 7.17, - "bfcl/bfcl.overall.total_cost_usd": 3.4, - "bfcl/bfcl.overall.latency_mean_s": 3.98, - "bfcl/bfcl.overall.latency_std_s": 9.8, - "bfcl/bfcl.overall.latency_p95_s": 12.06, - "bfcl/bfcl.non_live.ast_accuracy": 20.21, - "bfcl/bfcl.non_live.simple_ast_accuracy": 43.33, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 36.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 0.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 1.5, - "bfcl/bfcl.live.live_accuracy": 11.84, - "bfcl/bfcl.live.live_simple_ast_accuracy": 36.43, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 6.27, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 0.0, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 0.0, - "bfcl/bfcl.multi_turn.accuracy": 0.0, - "bfcl/bfcl.multi_turn.base_accuracy": 0.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 0.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 0.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 0.0, - "bfcl/bfcl.web_search.accuracy": 0.0, - "bfcl/bfcl.web_search.base_accuracy": 0.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 0.0, - "bfcl/bfcl.memory.accuracy": 3.23, - "bfcl/bfcl.memory.kv_accuracy": 3.87, - "bfcl/bfcl.memory.vector_accuracy": 3.87, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 1.94, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 37.5, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 33.18, - "bfcl/bfcl.format_sensitivity.max_delta": 25.5, - "bfcl/bfcl.format_sensitivity.stddev": 9.76 - } - }, - { - "id": "google/gemma-3-27b-it", - "name": "gemma-3-27b-it", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "global-mmlu-lite/Global MMLU Lite": 0.763, - "global-mmlu-lite/Culturally Sensitive": 0.7528, - "global-mmlu-lite/Culturally Agnostic": 0.7733, - "global-mmlu-lite/Arabic": 0.78, - "global-mmlu-lite/English": 0.7337, - "global-mmlu-lite/Bengali": 0.75, - "global-mmlu-lite/German": 0.775, - "global-mmlu-lite/French": 0.7481, - "global-mmlu-lite/Hindi": 0.7335, - "global-mmlu-lite/Indonesian": 0.7563, - "global-mmlu-lite/Italian": 0.75, - "global-mmlu-lite/Japanese": 0.7925, - "global-mmlu-lite/Korean": 0.798, - "global-mmlu-lite/Portuguese": 0.7481, - "global-mmlu-lite/Spanish": 0.7494, - "global-mmlu-lite/Swahili": 0.785, - "global-mmlu-lite/Yoruba": 0.7444, - "global-mmlu-lite/Chinese": 0.7925, - "global-mmlu-lite/Burmese": 0.7719 - } - }, - { - "id": "google/gemma-3-27b-it-prompt", - "name": "Gemma-3-27b-it (Prompt)", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 69.0, - "bfcl/bfcl.overall.overall_accuracy": 29.47, - "bfcl/bfcl.overall.total_cost_usd": 11.82, - "bfcl/bfcl.overall.latency_mean_s": 10.88, - "bfcl/bfcl.overall.latency_std_s": 19.67, - "bfcl/bfcl.overall.latency_p95_s": 55.5, - "bfcl/bfcl.non_live.ast_accuracy": 87.17, - "bfcl/bfcl.non_live.simple_ast_accuracy": 77.67, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 92.5, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 89.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 89.5, - "bfcl/bfcl.live.live_accuracy": 74.54, - "bfcl/bfcl.live.live_simple_ast_accuracy": 84.5, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 72.46, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 93.75, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 45.83, - "bfcl/bfcl.multi_turn.accuracy": 10.75, - "bfcl/bfcl.multi_turn.base_accuracy": 16.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 4.5, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 8.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 14.0, - "bfcl/bfcl.web_search.accuracy": 0.0, - "bfcl/bfcl.web_search.base_accuracy": 0.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 0.0, - "bfcl/bfcl.memory.accuracy": 13.55, - "bfcl/bfcl.memory.kv_accuracy": 1.94, - "bfcl/bfcl.memory.vector_accuracy": 3.23, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 35.48, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 81.25, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 73.67, - "bfcl/bfcl.format_sensitivity.max_delta": 34.0, - "bfcl/bfcl.format_sensitivity.stddev": 8.06 - } - }, - { - "id": "google/gemma-3-4b-it", - "name": "gemma-3-4b-it", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "global-mmlu-lite/Global MMLU Lite": 0.6511, - "global-mmlu-lite/Culturally Sensitive": 0.6116, - "global-mmlu-lite/Culturally Agnostic": 0.6906, - "global-mmlu-lite/Arabic": 0.6525, - "global-mmlu-lite/English": 0.67, - "global-mmlu-lite/Bengali": 0.68, - "global-mmlu-lite/German": 0.6525, - "global-mmlu-lite/French": 0.6575, - "global-mmlu-lite/Hindi": 0.6475, - "global-mmlu-lite/Indonesian": 0.6775, - "global-mmlu-lite/Italian": 0.6675, - "global-mmlu-lite/Japanese": 0.6325, - "global-mmlu-lite/Korean": 0.66, - "global-mmlu-lite/Portuguese": 0.68, - "global-mmlu-lite/Spanish": 0.6725, - "global-mmlu-lite/Swahili": 0.6075, - "global-mmlu-lite/Yoruba": 0.5825, - "global-mmlu-lite/Chinese": 0.6475, - "global-mmlu-lite/Burmese": 0.63 - } - }, - { - "id": "google/gemma-3-4b-it-prompt", - "name": "Gemma-3-4b-it (Prompt)", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 101.0, - "bfcl/bfcl.overall.overall_accuracy": 19.62, - "bfcl/bfcl.overall.total_cost_usd": 4.14, - "bfcl/bfcl.overall.latency_mean_s": 4.69, - "bfcl/bfcl.overall.latency_std_s": 9.53, - "bfcl/bfcl.overall.latency_p95_s": 11.42, - "bfcl/bfcl.non_live.ast_accuracy": 61.12, - "bfcl/bfcl.non_live.simple_ast_accuracy": 64.5, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 88.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 56.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 36.0, - "bfcl/bfcl.live.live_accuracy": 60.84, - "bfcl/bfcl.live.live_simple_ast_accuracy": 70.93, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 59.35, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 25.0, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 41.67, - "bfcl/bfcl.multi_turn.accuracy": 0.38, - "bfcl/bfcl.multi_turn.base_accuracy": 0.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 0.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 0.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 0.5, - "bfcl/bfcl.web_search.accuracy": 1.0, - "bfcl/bfcl.web_search.base_accuracy": 1.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 1.0, - "bfcl/bfcl.memory.accuracy": 8.6, - "bfcl/bfcl.memory.kv_accuracy": 9.68, - "bfcl/bfcl.memory.vector_accuracy": 9.68, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 6.45, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 100.0, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 53.94, - "bfcl/bfcl.format_sensitivity.max_delta": 69.5, - "bfcl/bfcl.format_sensitivity.stddev": 23.67 - } - }, - { - "id": "google/gemma-7b", - "name": "Gemma 7B", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.336, - "helm_lite/NarrativeQA": 0.752, - "helm_lite/NaturalQuestions (closed-book)": 0.336, - "helm_lite/OpenbookQA": 0.808, - "helm_lite/MMLU": 0.571, - "helm_lite/MATH": 0.5, - "helm_lite/GSM8K": 0.559, - "helm_lite/LegalBench": 0.581, - "helm_lite/MedQA": 0.513, - "helm_lite/WMT 2014": 0.187, - "helm_mmlu/MMLU All Subjects": 0.661, - "helm_mmlu/Abstract Algebra": 0.28, - "helm_mmlu/Anatomy": 0.563, - "helm_mmlu/College Physics": 0.412, - "helm_mmlu/Computer Security": 0.75, - "helm_mmlu/Econometrics": 0.474, - "helm_mmlu/Global Facts": 0.42, - "helm_mmlu/Jurisprudence": 0.769, - "helm_mmlu/Philosophy": 0.727, - "helm_mmlu/Professional Psychology": 0.712, - "helm_mmlu/Us Foreign Policy": 0.87, - "helm_mmlu/Astronomy": 0.717, - "helm_mmlu/Business Ethics": 0.65, - "helm_mmlu/Clinical Knowledge": 0.698, - "helm_mmlu/Conceptual Physics": 0.621, - "helm_mmlu/Electrical Engineering": 0.628, - "helm_mmlu/Elementary Mathematics": 0.516, - "helm_mmlu/Formal Logic": 0.508, - "helm_mmlu/High School World History": 0.857, - "helm_mmlu/Human Sexuality": 0.733, - "helm_mmlu/International Law": 0.835, - "helm_mmlu/Logical Fallacies": 0.742, - "helm_mmlu/Machine Learning": 0.554, - "helm_mmlu/Management": 0.864, - "helm_mmlu/Marketing": 0.885, - "helm_mmlu/Medical Genetics": 0.7, - "helm_mmlu/Miscellaneous": 0.838, - "helm_mmlu/Moral Scenarios": 0.377, - "helm_mmlu/Nutrition": 0.778, - "helm_mmlu/Prehistory": 0.756, - "helm_mmlu/Public Relations": 0.682, - "helm_mmlu/Security Studies": 0.735, - "helm_mmlu/Sociology": 0.841, - "helm_mmlu/Virology": 0.548, - "helm_mmlu/World Religions": 0.842, - "helm_mmlu/Mean win rate": 0.824, - "hfopenllm_v2/IFEval": 0.2659, - "hfopenllm_v2/BBH": 0.4362, - "hfopenllm_v2/MATH Level 5": 0.074, - "hfopenllm_v2/GPQA": 0.2869, - "hfopenllm_v2/MUSR": 0.4062, - "hfopenllm_v2/MMLU-PRO": 0.2948 - } - }, - { - "id": "google/gemma-7b-it", - "name": "gemma-7b-it", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3868, - "hfopenllm_v2/BBH": 0.3646, - "hfopenllm_v2/MATH Level 5": 0.0295, - "hfopenllm_v2/GPQA": 0.2844, - "hfopenllm_v2/MUSR": 0.4274, - "hfopenllm_v2/MMLU-PRO": 0.1695 - } - }, - { - "id": "google/mt5-base", - "name": "mt5-base", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1645, - "hfopenllm_v2/BBH": 0.2883, - "hfopenllm_v2/MATH Level 5": 0.0091, - "hfopenllm_v2/GPQA": 0.2391, - "hfopenllm_v2/MUSR": 0.3672, - "hfopenllm_v2/MMLU-PRO": 0.107 - } - }, - { - "id": "google/mt5-small", - "name": "mt5-small", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1718, - "hfopenllm_v2/BBH": 0.2766, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2424, - "hfopenllm_v2/MUSR": 0.3857, - "hfopenllm_v2/MMLU-PRO": 0.1123 - } - }, - { - "id": "google/mt5-xl", - "name": "mt5-xl", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.196, - "hfopenllm_v2/BBH": 0.3047, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2643, - "hfopenllm_v2/MUSR": 0.3795, - "hfopenllm_v2/MMLU-PRO": 0.112 - } - }, - { - "id": "google/mt5-xxl", - "name": "mt5-xxl", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2358, - "hfopenllm_v2/BBH": 0.2959, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2416, - "hfopenllm_v2/MUSR": 0.3689, - "hfopenllm_v2/MMLU-PRO": 0.1089 - } - }, - { - "id": "google/Palmyra-X-43B", - "name": "Palmyra X 43B", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.732, - "helm_classic/MMLU": 0.609, - "helm_classic/BoolQ": 0.896, - "helm_classic/NarrativeQA": 0.742, - "helm_classic/NaturalQuestions (open-book)": -1.0, - "helm_classic/QuAC": 0.473, - "helm_classic/HellaSwag": -1.0, - "helm_classic/OpenbookQA": -1.0, - "helm_classic/TruthfulQA": 0.616, - "helm_classic/MS MARCO (TREC)": -1.0, - "helm_classic/CNN/DailyMail": 0.049, - "helm_classic/XSUM": 0.149, - "helm_classic/IMDB": 0.935, - "helm_classic/CivilComments": 0.008, - "helm_classic/RAFT": 0.701 - } - }, - { - "id": "google/recurrentgemma-2b", - "name": "recurrentgemma-2b", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3017, - "hfopenllm_v2/BBH": 0.3197, - "hfopenllm_v2/MATH Level 5": 0.0204, - "hfopenllm_v2/GPQA": 0.2458, - "hfopenllm_v2/MUSR": 0.3446, - "hfopenllm_v2/MMLU-PRO": 0.1176 - } - }, - { - "id": "google/recurrentgemma-2b-it", - "name": "recurrentgemma-2b-it", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2949, - "hfopenllm_v2/BBH": 0.333, - "hfopenllm_v2/MATH Level 5": 0.0196, - "hfopenllm_v2/GPQA": 0.2534, - "hfopenllm_v2/MUSR": 0.3341, - "hfopenllm_v2/MMLU-PRO": 0.1402 - } - }, - { - "id": "google/recurrentgemma-9b", - "name": "recurrentgemma-9b", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3116, - "hfopenllm_v2/BBH": 0.3956, - "hfopenllm_v2/MATH Level 5": 0.0665, - "hfopenllm_v2/GPQA": 0.2852, - "hfopenllm_v2/MUSR": 0.3803, - "hfopenllm_v2/MMLU-PRO": 0.2605 - } - }, - { - "id": "google/recurrentgemma-9b-it", - "name": "recurrentgemma-9b-it", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.501, - "hfopenllm_v2/BBH": 0.4367, - "hfopenllm_v2/MATH Level 5": 0.0665, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.4379, - "hfopenllm_v2/MMLU-PRO": 0.2843 - } - }, - { - "id": "google/switch-base-8", - "name": "switch-base-8", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1585, - "hfopenllm_v2/BBH": 0.2876, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.25, - "hfopenllm_v2/MUSR": 0.3517, - "hfopenllm_v2/MMLU-PRO": 0.1098 - } - }, - { - "id": "google/T5-11B", - "name": "T5 11B", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.131, - "helm_classic/MMLU": 0.29, - "helm_classic/BoolQ": 0.761, - "helm_classic/NarrativeQA": 0.086, - "helm_classic/NaturalQuestions (open-book)": 0.477, - "helm_classic/QuAC": 0.116, - "helm_classic/HellaSwag": -1.0, - "helm_classic/OpenbookQA": -1.0, - "helm_classic/TruthfulQA": 0.133, - "helm_classic/MS MARCO (TREC)": -1.0, - "helm_classic/CNN/DailyMail": 0.043, - "helm_classic/XSUM": 0.015, - "helm_classic/IMDB": 0.379, - "helm_classic/CivilComments": 0.509, - "helm_classic/RAFT": 0.37 - } - }, - { - "id": "google/text-bison@001", - "name": "PaLM-2 Bison", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.526, - "helm_lite/NarrativeQA": 0.718, - "helm_lite/NaturalQuestions (closed-book)": 0.39, - "helm_lite/OpenbookQA": 0.878, - "helm_lite/MMLU": 0.608, - "helm_lite/MATH": 0.421, - "helm_lite/GSM8K": 0.61, - "helm_lite/LegalBench": 0.645, - "helm_lite/MedQA": 0.547, - "helm_lite/WMT 2014": 0.241, - "helm_mmlu/MMLU All Subjects": 0.692, - "helm_mmlu/Abstract Algebra": 0.39, - "helm_mmlu/Anatomy": 0.644, - "helm_mmlu/College Physics": 0.51, - "helm_mmlu/Computer Security": 0.74, - "helm_mmlu/Econometrics": 0.518, - "helm_mmlu/Global Facts": 0.38, - "helm_mmlu/Jurisprudence": 0.769, - "helm_mmlu/Philosophy": 0.736, - "helm_mmlu/Professional Psychology": 0.761, - "helm_mmlu/Us Foreign Policy": 0.87, - "helm_mmlu/Astronomy": 0.803, - "helm_mmlu/Business Ethics": 0.76, - "helm_mmlu/Clinical Knowledge": 0.725, - "helm_mmlu/Conceptual Physics": 0.694, - "helm_mmlu/Electrical Engineering": 0.69, - "helm_mmlu/Elementary Mathematics": 0.487, - "helm_mmlu/Formal Logic": 0.5, - "helm_mmlu/High School World History": 0.869, - "helm_mmlu/Human Sexuality": 0.84, - "helm_mmlu/International Law": 0.835, - "helm_mmlu/Logical Fallacies": 0.853, - "helm_mmlu/Machine Learning": 0.562, - "helm_mmlu/Management": 0.893, - "helm_mmlu/Marketing": 0.893, - "helm_mmlu/Medical Genetics": 0.75, - "helm_mmlu/Miscellaneous": 0.866, - "helm_mmlu/Moral Scenarios": 0.369, - "helm_mmlu/Nutrition": 0.709, - "helm_mmlu/Prehistory": 0.812, - "helm_mmlu/Public Relations": 0.691, - "helm_mmlu/Security Studies": 0.812, - "helm_mmlu/Sociology": 0.92, - "helm_mmlu/Virology": 0.494, - "helm_mmlu/World Religions": 0.883, - "helm_mmlu/Mean win rate": 0.192 - } - }, - { - "id": "google/text-unicorn@001", - "name": "PaLM-2 Unicorn", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.644, - "helm_lite/NarrativeQA": 0.583, - "helm_lite/NaturalQuestions (closed-book)": 0.435, - "helm_lite/OpenbookQA": 0.938, - "helm_lite/MMLU": 0.702, - "helm_lite/MATH": 0.674, - "helm_lite/GSM8K": 0.831, - "helm_lite/LegalBench": 0.677, - "helm_lite/MedQA": 0.684, - "helm_lite/WMT 2014": 0.26, - "helm_mmlu/MMLU All Subjects": 0.786, - "helm_mmlu/Abstract Algebra": 0.51, - "helm_mmlu/Anatomy": 0.733, - "helm_mmlu/College Physics": 0.549, - "helm_mmlu/Computer Security": 0.77, - "helm_mmlu/Econometrics": 0.649, - "helm_mmlu/Global Facts": 0.53, - "helm_mmlu/Jurisprudence": 0.88, - "helm_mmlu/Philosophy": 0.836, - "helm_mmlu/Professional Psychology": 0.858, - "helm_mmlu/Us Foreign Policy": 0.96, - "helm_mmlu/Astronomy": 0.862, - "helm_mmlu/Business Ethics": 0.83, - "helm_mmlu/Clinical Knowledge": 0.804, - "helm_mmlu/Conceptual Physics": 0.809, - "helm_mmlu/Electrical Engineering": 0.772, - "helm_mmlu/Elementary Mathematics": 0.661, - "helm_mmlu/Formal Logic": 0.659, - "helm_mmlu/High School World History": 0.911, - "helm_mmlu/Human Sexuality": 0.924, - "helm_mmlu/International Law": 0.909, - "helm_mmlu/Logical Fallacies": 0.877, - "helm_mmlu/Machine Learning": 0.625, - "helm_mmlu/Management": 0.903, - "helm_mmlu/Marketing": 0.94, - "helm_mmlu/Medical Genetics": 0.83, - "helm_mmlu/Miscellaneous": 0.894, - "helm_mmlu/Moral Scenarios": 0.562, - "helm_mmlu/Nutrition": 0.856, - "helm_mmlu/Prehistory": 0.87, - "helm_mmlu/Public Relations": 0.773, - "helm_mmlu/Security Studies": 0.829, - "helm_mmlu/Sociology": 0.91, - "helm_mmlu/Virology": 0.572, - "helm_mmlu/World Religions": 0.877, - "helm_mmlu/Mean win rate": 0.142 - } - }, - { - "id": "google/UL2-20B", - "name": "UL2 20B", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.167, - "helm_classic/MMLU": 0.291, - "helm_classic/BoolQ": 0.746, - "helm_classic/NarrativeQA": 0.083, - "helm_classic/NaturalQuestions (open-book)": 0.349, - "helm_classic/QuAC": 0.144, - "helm_classic/HellaSwag": -1.0, - "helm_classic/OpenbookQA": -1.0, - "helm_classic/TruthfulQA": 0.193, - "helm_classic/MS MARCO (TREC)": -1.0, - "helm_classic/CNN/DailyMail": 0.03, - "helm_classic/XSUM": 0.058, - "helm_classic/IMDB": 0.337, - "helm_classic/CivilComments": 0.521, - "helm_classic/RAFT": 0.404 - } - }, - { - "id": "google/umt5-base", - "name": "umt5-base", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1746, - "hfopenllm_v2/BBH": 0.2788, - "hfopenllm_v2/MATH Level 5": 0.0045, - "hfopenllm_v2/GPQA": 0.2542, - "hfopenllm_v2/MUSR": 0.3382, - "hfopenllm_v2/MMLU-PRO": 0.1078 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/gotocompany.json b/data/developers/gotocompany.json deleted file mode 100644 index 4d0524802da3cdd1c3199a75ac08df458128a62c..0000000000000000000000000000000000000000 --- a/data/developers/gotocompany.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "GoToCompany", - "models": [ - { - "id": "GoToCompany/gemma2-9b-cpt-sahabatai-v1-instruct", - "name": "gemma2-9b-cpt-sahabatai-v1-instruct", - "developer": "GoToCompany", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6551, - "hfopenllm_v2/BBH": 0.5955, - "hfopenllm_v2/MATH Level 5": 0.2054, - "hfopenllm_v2/GPQA": 0.3347, - "hfopenllm_v2/MUSR": 0.4779, - "hfopenllm_v2/MMLU-PRO": 0.4264 - } - }, - { - "id": "GoToCompany/llama3-8b-cpt-sahabatai-v1-instruct", - "name": "llama3-8b-cpt-sahabatai-v1-instruct", - "developer": "GoToCompany", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5238, - "hfopenllm_v2/BBH": 0.4951, - "hfopenllm_v2/MATH Level 5": 0.1276, - "hfopenllm_v2/GPQA": 0.2668, - "hfopenllm_v2/MUSR": 0.4488, - "hfopenllm_v2/MMLU-PRO": 0.3453 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/goulue5.json b/data/developers/goulue5.json deleted file mode 100644 index 1784209cd1a91a98c08a6580371e4647e236069d..0000000000000000000000000000000000000000 --- a/data/developers/goulue5.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "goulue5", - "models": [ - { - "id": "goulue5/merging_LLM", - "name": "merging_LLM", - "developer": "goulue5", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3233, - "hfopenllm_v2/BBH": 0.4216, - "hfopenllm_v2/MATH Level 5": 0.0967, - "hfopenllm_v2/GPQA": 0.2911, - "hfopenllm_v2/MUSR": 0.4333, - "hfopenllm_v2/MMLU-PRO": 0.2958 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/gradientai.json b/data/developers/gradientai.json deleted file mode 100644 index 168231fa8cad153268e010d70aa3ff34d14b6188..0000000000000000000000000000000000000000 --- a/data/developers/gradientai.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "gradientai", - "models": [ - { - "id": "gradientai/Llama-3-8B-Instruct-Gradient-1048k", - "name": "Llama-3-8B-Instruct-Gradient-1048k", - "developer": "gradientai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4456, - "hfopenllm_v2/BBH": 0.4346, - "hfopenllm_v2/MATH Level 5": 0.0536, - "hfopenllm_v2/GPQA": 0.2777, - "hfopenllm_v2/MUSR": 0.4298, - "hfopenllm_v2/MMLU-PRO": 0.294 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/greennode.json b/data/developers/greennode.json deleted file mode 100644 index 384914502d43cf5d86dc21c52d2b35cf0ad5800b..0000000000000000000000000000000000000000 --- a/data/developers/greennode.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "GreenNode", - "models": [ - { - "id": "GreenNode/GreenNode-small-9B-it", - "name": "GreenNode-small-9B-it", - "developer": "GreenNode", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7436, - "hfopenllm_v2/BBH": 0.5994, - "hfopenllm_v2/MATH Level 5": 0.1745, - "hfopenllm_v2/GPQA": 0.3196, - "hfopenllm_v2/MUSR": 0.4204, - "hfopenllm_v2/MMLU-PRO": 0.3927 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/grimjim.json b/data/developers/grimjim.json deleted file mode 100644 index a403cecba778a226b14d7a850ad56a0126f49a93..0000000000000000000000000000000000000000 --- a/data/developers/grimjim.json +++ /dev/null @@ -1,355 +0,0 @@ -{ - "developer": "grimjim", - "models": [ - { - "id": "grimjim/DeepSauerHuatuoSkywork-R1-o1-Llama-3.1-8B", - "name": "DeepSauerHuatuoSkywork-R1-o1-Llama-3.1-8B", - "developer": "grimjim", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4797, - "hfopenllm_v2/BBH": 0.5269, - "hfopenllm_v2/MATH Level 5": 0.2221, - "hfopenllm_v2/GPQA": 0.3381, - "hfopenllm_v2/MUSR": 0.4408, - "hfopenllm_v2/MMLU-PRO": 0.3957 - } - }, - { - "id": "grimjim/Gigantes-v1-gemma2-9b-it", - "name": "Gigantes-v1-gemma2-9b-it", - "developer": "grimjim", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6925, - "hfopenllm_v2/BBH": 0.5978, - "hfopenllm_v2/MATH Level 5": 0.2145, - "hfopenllm_v2/GPQA": 0.3532, - "hfopenllm_v2/MUSR": 0.4555, - "hfopenllm_v2/MMLU-PRO": 0.4225 - } - }, - { - "id": "grimjim/Gigantes-v2-gemma2-9b-it", - "name": "Gigantes-v2-gemma2-9b-it", - "developer": "grimjim", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7351, - "hfopenllm_v2/BBH": 0.5987, - "hfopenllm_v2/MATH Level 5": 0.2017, - "hfopenllm_v2/GPQA": 0.3515, - "hfopenllm_v2/MUSR": 0.4595, - "hfopenllm_v2/MMLU-PRO": 0.4259 - } - }, - { - "id": "grimjim/Gigantes-v3-gemma2-9b-it", - "name": "Gigantes-v3-gemma2-9b-it", - "developer": "grimjim", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6976, - "hfopenllm_v2/BBH": 0.5984, - "hfopenllm_v2/MATH Level 5": 0.21, - "hfopenllm_v2/GPQA": 0.3565, - "hfopenllm_v2/MUSR": 0.4608, - "hfopenllm_v2/MMLU-PRO": 0.4226 - } - }, - { - "id": "grimjim/HuatuoSkywork-o1-Llama-3.1-8B", - "name": "HuatuoSkywork-o1-Llama-3.1-8B", - "developer": "grimjim", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3961, - "hfopenllm_v2/BBH": 0.4886, - "hfopenllm_v2/MATH Level 5": 0.3882, - "hfopenllm_v2/GPQA": 0.2928, - "hfopenllm_v2/MUSR": 0.3839, - "hfopenllm_v2/MMLU-PRO": 0.3095 - } - }, - { - "id": "grimjim/Llama-3-Instruct-8B-SimPO-SPPO-Iter3-merge", - "name": "Llama-3-Instruct-8B-SimPO-SPPO-Iter3-merge", - "developer": "grimjim", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6806, - "hfopenllm_v2/BBH": 0.5022, - "hfopenllm_v2/MATH Level 5": 0.0891, - "hfopenllm_v2/GPQA": 0.2626, - "hfopenllm_v2/MUSR": 0.3885, - "hfopenllm_v2/MMLU-PRO": 0.3684 - } - }, - { - "id": "grimjim/Llama-3-Instruct-8B-SPPO-Iter3-SimPO-merge", - "name": "Llama-3-Instruct-8B-SPPO-Iter3-SimPO-merge", - "developer": "grimjim", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4271, - "hfopenllm_v2/BBH": 0.4962, - "hfopenllm_v2/MATH Level 5": 0.0997, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.4043, - "hfopenllm_v2/MMLU-PRO": 0.3625 - } - }, - { - "id": "grimjim/llama-3-Nephilim-v1-8B", - "name": "llama-3-Nephilim-v1-8B", - "developer": "grimjim", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4277, - "hfopenllm_v2/BBH": 0.5132, - "hfopenllm_v2/MATH Level 5": 0.0906, - "hfopenllm_v2/GPQA": 0.302, - "hfopenllm_v2/MUSR": 0.4136, - "hfopenllm_v2/MMLU-PRO": 0.3796 - } - }, - { - "id": "grimjim/llama-3-Nephilim-v2-8B", - "name": "llama-3-Nephilim-v2-8B", - "developer": "grimjim", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3922, - "hfopenllm_v2/BBH": 0.5048, - "hfopenllm_v2/MATH Level 5": 0.1065, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.3895, - "hfopenllm_v2/MMLU-PRO": 0.3641 - } - }, - { - "id": "grimjim/llama-3-Nephilim-v2.1-8B", - "name": "llama-3-Nephilim-v2.1-8B", - "developer": "grimjim", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3895, - "hfopenllm_v2/BBH": 0.5095, - "hfopenllm_v2/MATH Level 5": 0.0997, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.3935, - "hfopenllm_v2/MMLU-PRO": 0.3644 - } - }, - { - "id": "grimjim/llama-3-Nephilim-v3-8B", - "name": "llama-3-Nephilim-v3-8B", - "developer": "grimjim", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4174, - "hfopenllm_v2/BBH": 0.5013, - "hfopenllm_v2/MATH Level 5": 0.0952, - "hfopenllm_v2/GPQA": 0.2953, - "hfopenllm_v2/MUSR": 0.3989, - "hfopenllm_v2/MMLU-PRO": 0.3612 - } - }, - { - "id": "grimjim/Llama-3.1-8B-Instruct-abliterated_via_adapter", - "name": "Llama-3.1-8B-Instruct-abliterated_via_adapter", - "developer": "grimjim", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.487, - "hfopenllm_v2/BBH": 0.5105, - "hfopenllm_v2/MATH Level 5": 0.1397, - "hfopenllm_v2/GPQA": 0.3138, - "hfopenllm_v2/MUSR": 0.401, - "hfopenllm_v2/MMLU-PRO": 0.3651 - } - }, - { - "id": "grimjim/Llama-3.1-Bonsaikraft-8B-Instruct", - "name": "Llama-3.1-Bonsaikraft-8B-Instruct", - "developer": "grimjim", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.425, - "hfopenllm_v2/BBH": 0.5287, - "hfopenllm_v2/MATH Level 5": 0.1314, - "hfopenllm_v2/GPQA": 0.3037, - "hfopenllm_v2/MUSR": 0.4235, - "hfopenllm_v2/MMLU-PRO": 0.3764 - } - }, - { - "id": "grimjim/Llama-Nephilim-Metamorphosis-v2-8B", - "name": "Llama-Nephilim-Metamorphosis-v2-8B", - "developer": "grimjim", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4545, - "hfopenllm_v2/BBH": 0.5013, - "hfopenllm_v2/MATH Level 5": 0.1397, - "hfopenllm_v2/GPQA": 0.323, - "hfopenllm_v2/MUSR": 0.4091, - "hfopenllm_v2/MMLU-PRO": 0.3809 - } - }, - { - "id": "grimjim/Llama3.1-SuperNovaLite-HuatuoSkywork-o1-8B", - "name": "Llama3.1-SuperNovaLite-HuatuoSkywork-o1-8B", - "developer": "grimjim", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4366, - "hfopenllm_v2/BBH": 0.5287, - "hfopenllm_v2/MATH Level 5": 0.3006, - "hfopenllm_v2/GPQA": 0.3112, - "hfopenllm_v2/MUSR": 0.3999, - "hfopenllm_v2/MMLU-PRO": 0.3684 - } - }, - { - "id": "grimjim/Magnolia-v1-Gemma2-8k-9B", - "name": "Magnolia-v1-Gemma2-8k-9B", - "developer": "grimjim", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3531, - "hfopenllm_v2/BBH": 0.5589, - "hfopenllm_v2/MATH Level 5": 0.1684, - "hfopenllm_v2/GPQA": 0.3364, - "hfopenllm_v2/MUSR": 0.4645, - "hfopenllm_v2/MMLU-PRO": 0.4242 - } - }, - { - "id": "grimjim/Magnolia-v2-12B", - "name": "Magnolia-v2-12B", - "developer": "grimjim", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3506, - "hfopenllm_v2/BBH": 0.529, - "hfopenllm_v2/MATH Level 5": 0.1292, - "hfopenllm_v2/GPQA": 0.3188, - "hfopenllm_v2/MUSR": 0.4171, - "hfopenllm_v2/MMLU-PRO": 0.3601 - } - }, - { - "id": "grimjim/Magnolia-v2-Gemma2-8k-9B", - "name": "Magnolia-v2-Gemma2-8k-9B", - "developer": "grimjim", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7384, - "hfopenllm_v2/BBH": 0.6016, - "hfopenllm_v2/MATH Level 5": 0.2281, - "hfopenllm_v2/GPQA": 0.3574, - "hfopenllm_v2/MUSR": 0.4488, - "hfopenllm_v2/MMLU-PRO": 0.4332 - } - }, - { - "id": "grimjim/Magnolia-v3-12B", - "name": "Magnolia-v3-12B", - "developer": "grimjim", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3965, - "hfopenllm_v2/BBH": 0.5327, - "hfopenllm_v2/MATH Level 5": 0.1352, - "hfopenllm_v2/GPQA": 0.3255, - "hfopenllm_v2/MUSR": 0.4184, - "hfopenllm_v2/MMLU-PRO": 0.3615 - } - }, - { - "id": "grimjim/Magnolia-v3-Gemma2-8k-9B", - "name": "Magnolia-v3-Gemma2-8k-9B", - "developer": "grimjim", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7378, - "hfopenllm_v2/BBH": 0.6015, - "hfopenllm_v2/MATH Level 5": 0.2319, - "hfopenllm_v2/GPQA": 0.3565, - "hfopenllm_v2/MUSR": 0.4488, - "hfopenllm_v2/MMLU-PRO": 0.4337 - } - }, - { - "id": "grimjim/Magnolia-v4-12B", - "name": "Magnolia-v4-12B", - "developer": "grimjim", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3418, - "hfopenllm_v2/BBH": 0.5431, - "hfopenllm_v2/MATH Level 5": 0.1314, - "hfopenllm_v2/GPQA": 0.328, - "hfopenllm_v2/MUSR": 0.4211, - "hfopenllm_v2/MMLU-PRO": 0.3672 - } - }, - { - "id": "grimjim/Magnolia-v5a-12B", - "name": "Magnolia-v5a-12B", - "developer": "grimjim", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4114, - "hfopenllm_v2/BBH": 0.5312, - "hfopenllm_v2/MATH Level 5": 0.1375, - "hfopenllm_v2/GPQA": 0.3221, - "hfopenllm_v2/MUSR": 0.4145, - "hfopenllm_v2/MMLU-PRO": 0.3601 - } - }, - { - "id": "grimjim/Magot-v1-Gemma2-8k-9B", - "name": "Magot-v1-Gemma2-8k-9B", - "developer": "grimjim", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2997, - "hfopenllm_v2/BBH": 0.6019, - "hfopenllm_v2/MATH Level 5": 0.0989, - "hfopenllm_v2/GPQA": 0.3465, - "hfopenllm_v2/MUSR": 0.4488, - "hfopenllm_v2/MMLU-PRO": 0.4337 - } - }, - { - "id": "grimjim/Magot-v2-Gemma2-8k-9B", - "name": "Magot-v2-Gemma2-8k-9B", - "developer": "grimjim", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7347, - "hfopenllm_v2/BBH": 0.5897, - "hfopenllm_v2/MATH Level 5": 0.2017, - "hfopenllm_v2/GPQA": 0.354, - "hfopenllm_v2/MUSR": 0.4344, - "hfopenllm_v2/MMLU-PRO": 0.4223 - } - }, - { - "id": "grimjim/SauerHuatuoSkywork-o1-Llama-3.1-8B", - "name": "SauerHuatuoSkywork-o1-Llama-3.1-8B", - "developer": "grimjim", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5219, - "hfopenllm_v2/BBH": 0.5222, - "hfopenllm_v2/MATH Level 5": 0.173, - "hfopenllm_v2/GPQA": 0.3213, - "hfopenllm_v2/MUSR": 0.4527, - "hfopenllm_v2/MMLU-PRO": 0.3991 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/gritlm.json b/data/developers/gritlm.json deleted file mode 100644 index 5600a5186bede48b0c614f8ee4392c434310cd12..0000000000000000000000000000000000000000 --- a/data/developers/gritlm.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "GritLM", - "models": [ - { - "id": "GritLM/GritLM-7B-KTO", - "name": "GritLM-7B-KTO", - "developer": "GritLM", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.531, - "hfopenllm_v2/BBH": 0.4853, - "hfopenllm_v2/MATH Level 5": 0.0272, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.371, - "hfopenllm_v2/MMLU-PRO": 0.268 - } - }, - { - "id": "GritLM/GritLM-8x7B-KTO", - "name": "GritLM-8x7B-KTO", - "developer": "GritLM", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5714, - "hfopenllm_v2/BBH": 0.582, - "hfopenllm_v2/MATH Level 5": 0.1224, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.4217, - "hfopenllm_v2/MMLU-PRO": 0.3648 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/groq.json b/data/developers/groq.json deleted file mode 100644 index 63e2893de00b9ebd1d7c878df5a729bcfa230e2a..0000000000000000000000000000000000000000 --- a/data/developers/groq.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "Groq", - "models": [ - { - "id": "Groq/Llama-3-Groq-8B-Tool-Use", - "name": "Llama-3-Groq-8B-Tool-Use", - "developer": "Groq", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6098, - "hfopenllm_v2/BBH": 0.4863, - "hfopenllm_v2/MATH Level 5": 0.0604, - "hfopenllm_v2/GPQA": 0.2676, - "hfopenllm_v2/MUSR": 0.366, - "hfopenllm_v2/MMLU-PRO": 0.3399 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/gryphe.json b/data/developers/gryphe.json deleted file mode 100644 index fb499a575b0a6874127b742b0e6e79f47ba72055..0000000000000000000000000000000000000000 --- a/data/developers/gryphe.json +++ /dev/null @@ -1,75 +0,0 @@ -{ - "developer": "Gryphe", - "models": [ - { - "id": "Gryphe/Pantheon-RP-1.0-8b-Llama-3", - "name": "Pantheon-RP-1.0-8b-Llama-3", - "developer": "Gryphe", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3933, - "hfopenllm_v2/BBH": 0.4539, - "hfopenllm_v2/MATH Level 5": 0.0634, - "hfopenllm_v2/GPQA": 0.276, - "hfopenllm_v2/MUSR": 0.3832, - "hfopenllm_v2/MMLU-PRO": 0.3067 - } - }, - { - "id": "Gryphe/Pantheon-RP-1.5-12b-Nemo", - "name": "Pantheon-RP-1.5-12b-Nemo", - "developer": "Gryphe", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4763, - "hfopenllm_v2/BBH": 0.5196, - "hfopenllm_v2/MATH Level 5": 0.0491, - "hfopenllm_v2/GPQA": 0.2727, - "hfopenllm_v2/MUSR": 0.442, - "hfopenllm_v2/MMLU-PRO": 0.3302 - } - }, - { - "id": "Gryphe/Pantheon-RP-1.6-12b-Nemo", - "name": "Pantheon-RP-1.6-12b-Nemo", - "developer": "Gryphe", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4481, - "hfopenllm_v2/BBH": 0.5204, - "hfopenllm_v2/MATH Level 5": 0.0461, - "hfopenllm_v2/GPQA": 0.2777, - "hfopenllm_v2/MUSR": 0.4288, - "hfopenllm_v2/MMLU-PRO": 0.3311 - } - }, - { - "id": "Gryphe/Pantheon-RP-1.6-12b-Nemo-KTO", - "name": "Pantheon-RP-1.6-12b-Nemo-KTO", - "developer": "Gryphe", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4636, - "hfopenllm_v2/BBH": 0.5277, - "hfopenllm_v2/MATH Level 5": 0.0529, - "hfopenllm_v2/GPQA": 0.2953, - "hfopenllm_v2/MUSR": 0.4248, - "hfopenllm_v2/MMLU-PRO": 0.3382 - } - }, - { - "id": "Gryphe/Pantheon-RP-Pure-1.6.2-22b-Small", - "name": "Pantheon-RP-Pure-1.6.2-22b-Small", - "developer": "Gryphe", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6931, - "hfopenllm_v2/BBH": 0.5305, - "hfopenllm_v2/MATH Level 5": 0.2024, - "hfopenllm_v2/GPQA": 0.3289, - "hfopenllm_v2/MUSR": 0.3765, - "hfopenllm_v2/MMLU-PRO": 0.3942 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/guilhermenaturaumana.json b/data/developers/guilhermenaturaumana.json deleted file mode 100644 index bc6bb765fac527e1a43413f17dc37f0bfd85ba71..0000000000000000000000000000000000000000 --- a/data/developers/guilhermenaturaumana.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "GuilhermeNaturaUmana", - "models": [ - { - "id": "GuilhermeNaturaUmana/Nature-Reason-1.2-reallysmall", - "name": "Nature-Reason-1.2-reallysmall", - "developer": "GuilhermeNaturaUmana", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4985, - "hfopenllm_v2/BBH": 0.5645, - "hfopenllm_v2/MATH Level 5": 0.2576, - "hfopenllm_v2/GPQA": 0.3003, - "hfopenllm_v2/MUSR": 0.4373, - "hfopenllm_v2/MMLU-PRO": 0.4429 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/gunulhona.json b/data/developers/gunulhona.json deleted file mode 100644 index 3d63c85d80df83c6632e31f815d8dc78510d19a0..0000000000000000000000000000000000000000 --- a/data/developers/gunulhona.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "Gunulhona", - "models": [ - { - "id": "Gunulhona/Gemma-Ko-Merge", - "name": "Gemma-Ko-Merge", - "developer": "Gunulhona", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6416, - "hfopenllm_v2/BBH": 0.5813, - "hfopenllm_v2/MATH Level 5": 0.1881, - "hfopenllm_v2/GPQA": 0.3356, - "hfopenllm_v2/MUSR": 0.4047, - "hfopenllm_v2/MMLU-PRO": 0.3879 - } - }, - { - "id": "Gunulhona/Gemma-Ko-Merge-PEFT", - "name": "Gemma-Ko-Merge-PEFT", - "developer": "Gunulhona", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.288, - "hfopenllm_v2/BBH": 0.5154, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.3247, - "hfopenllm_v2/MUSR": 0.408, - "hfopenllm_v2/MMLU-PRO": 0.3817 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/gupta-tanish.json b/data/developers/gupta-tanish.json deleted file mode 100644 index 3a454b1d3780807337f362fcd4281d424b869cb8..0000000000000000000000000000000000000000 --- a/data/developers/gupta-tanish.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "gupta-tanish", - "models": [ - { - "id": "gupta-tanish/llama-7b-dpo-baseline", - "name": "llama-7b-dpo-baseline", - "developer": "gupta-tanish", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2693, - "hfopenllm_v2/BBH": 0.3897, - "hfopenllm_v2/MATH Level 5": 0.0196, - "hfopenllm_v2/GPQA": 0.2626, - "hfopenllm_v2/MUSR": 0.4456, - "hfopenllm_v2/MMLU-PRO": 0.2028 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/gz987.json b/data/developers/gz987.json deleted file mode 100644 index 020082db9e4b469b79d1d3703cd5073eefcab4cd..0000000000000000000000000000000000000000 --- a/data/developers/gz987.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "developer": "gz987", - "models": [ - { - "id": "gz987/qwen2.5-7b-cabs-v0.1", - "name": "qwen2.5-7b-cabs-v0.1", - "developer": "gz987", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7506, - "hfopenllm_v2/BBH": 0.5482, - "hfopenllm_v2/MATH Level 5": 0.4796, - "hfopenllm_v2/GPQA": 0.3138, - "hfopenllm_v2/MUSR": 0.4376, - "hfopenllm_v2/MMLU-PRO": 0.4406 - } - }, - { - "id": "gz987/qwen2.5-7b-cabs-v0.2", - "name": "qwen2.5-7b-cabs-v0.2", - "developer": "gz987", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7418, - "hfopenllm_v2/BBH": 0.5516, - "hfopenllm_v2/MATH Level 5": 0.4902, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.4429, - "hfopenllm_v2/MMLU-PRO": 0.4397 - } - }, - { - "id": "gz987/qwen2.5-7b-cabs-v0.3", - "name": "qwen2.5-7b-cabs-v0.3", - "developer": "gz987", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.757, - "hfopenllm_v2/BBH": 0.5494, - "hfopenllm_v2/MATH Level 5": 0.4932, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.443, - "hfopenllm_v2/MMLU-PRO": 0.4402 - } - }, - { - "id": "gz987/qwen2.5-7b-cabs-v0.4", - "name": "qwen2.5-7b-cabs-v0.4", - "developer": "gz987", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7583, - "hfopenllm_v2/BBH": 0.5524, - "hfopenllm_v2/MATH Level 5": 0.4849, - "hfopenllm_v2/GPQA": 0.3079, - "hfopenllm_v2/MUSR": 0.443, - "hfopenllm_v2/MMLU-PRO": 0.4396 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/h2oai.json b/data/developers/h2oai.json deleted file mode 100644 index f4aa111f345aa1acf9f1cc6e0cca2ee77f66e54b..0000000000000000000000000000000000000000 --- a/data/developers/h2oai.json +++ /dev/null @@ -1,75 +0,0 @@ -{ - "developer": "h2oai", - "models": [ - { - "id": "h2oai/h2o-danube-1.8b-chat", - "name": "h2o-danube-1.8b-chat", - "developer": "h2oai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2199, - "hfopenllm_v2/BBH": 0.322, - "hfopenllm_v2/MATH Level 5": 0.0136, - "hfopenllm_v2/GPQA": 0.2542, - "hfopenllm_v2/MUSR": 0.3989, - "hfopenllm_v2/MMLU-PRO": 0.1314 - } - }, - { - "id": "h2oai/h2o-danube3-4b-base", - "name": "h2o-danube3-4b-base", - "developer": "h2oai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2338, - "hfopenllm_v2/BBH": 0.3599, - "hfopenllm_v2/MATH Level 5": 0.0227, - "hfopenllm_v2/GPQA": 0.2911, - "hfopenllm_v2/MUSR": 0.3778, - "hfopenllm_v2/MMLU-PRO": 0.2109 - } - }, - { - "id": "h2oai/h2o-danube3-4b-chat", - "name": "h2o-danube3-4b-chat", - "developer": "h2oai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3629, - "hfopenllm_v2/BBH": 0.3466, - "hfopenllm_v2/MATH Level 5": 0.0408, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3781, - "hfopenllm_v2/MMLU-PRO": 0.2228 - } - }, - { - "id": "h2oai/h2o-danube3-500m-chat", - "name": "h2o-danube3-500m-chat", - "developer": "h2oai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2208, - "hfopenllm_v2/BBH": 0.3035, - "hfopenllm_v2/MATH Level 5": 0.0166, - "hfopenllm_v2/GPQA": 0.2307, - "hfopenllm_v2/MUSR": 0.3434, - "hfopenllm_v2/MMLU-PRO": 0.1144 - } - }, - { - "id": "h2oai/h2o-danube3.1-4b-chat", - "name": "h2o-danube3.1-4b-chat", - "developer": "h2oai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5021, - "hfopenllm_v2/BBH": 0.3608, - "hfopenllm_v2/MATH Level 5": 0.0332, - "hfopenllm_v2/GPQA": 0.2852, - "hfopenllm_v2/MUSR": 0.4102, - "hfopenllm_v2/MMLU-PRO": 0.2719 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/haoranxu.json b/data/developers/haoranxu.json deleted file mode 100644 index 843256c0fe85c4f4e423c1c76f4a86daf1e80a57..0000000000000000000000000000000000000000 --- a/data/developers/haoranxu.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "developer": "haoranxu", - "models": [ - { - "id": "haoranxu/ALMA-13B-R", - "name": "ALMA-13B-R", - "developer": "haoranxu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0039, - "hfopenllm_v2/BBH": 0.3457, - "hfopenllm_v2/MATH Level 5": 0.0174, - "hfopenllm_v2/GPQA": 0.2576, - "hfopenllm_v2/MUSR": 0.3528, - "hfopenllm_v2/MMLU-PRO": 0.1817 - } - }, - { - "id": "haoranxu/Llama-3-Instruct-8B-CPO-SimPO", - "name": "Llama-3-Instruct-8B-CPO-SimPO", - "developer": "haoranxu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7046, - "hfopenllm_v2/BBH": 0.5048, - "hfopenllm_v2/MATH Level 5": 0.1027, - "hfopenllm_v2/GPQA": 0.2928, - "hfopenllm_v2/MUSR": 0.3567, - "hfopenllm_v2/MMLU-PRO": 0.3686 - } - }, - { - "id": "haoranxu/Llama-3-Instruct-8B-SimPO", - "name": "Llama-3-Instruct-8B-SimPO", - "developer": "haoranxu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7347, - "hfopenllm_v2/BBH": 0.4979, - "hfopenllm_v2/MATH Level 5": 0.0876, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.3566, - "hfopenllm_v2/MMLU-PRO": 0.3733 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/harbingerx.json b/data/developers/harbingerx.json deleted file mode 100644 index caedf22e726fadfc7fba92e0aa30e9f3d314bbdc..0000000000000000000000000000000000000000 --- a/data/developers/harbingerx.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "HarbingerX", - "models": [ - { - "id": "HarbingerX/Zeitgeist-3b-V1", - "name": "Zeitgeist-3b-V1", - "developer": "HarbingerX", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6712, - "hfopenllm_v2/BBH": 0.4441, - "hfopenllm_v2/MATH Level 5": 0.1035, - "hfopenllm_v2/GPQA": 0.2819, - "hfopenllm_v2/MUSR": 0.3579, - "hfopenllm_v2/MMLU-PRO": 0.3009 - } - }, - { - "id": "HarbingerX/Zeitgeist-3b-V1.2", - "name": "Zeitgeist-3b-V1.2", - "developer": "HarbingerX", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6754, - "hfopenllm_v2/BBH": 0.4441, - "hfopenllm_v2/MATH Level 5": 0.1012, - "hfopenllm_v2/GPQA": 0.2777, - "hfopenllm_v2/MUSR": 0.3579, - "hfopenllm_v2/MMLU-PRO": 0.3056 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/hastagaras.json b/data/developers/hastagaras.json deleted file mode 100644 index 4a3c55d43f86eaa49c1c3182528673849d0a1b88..0000000000000000000000000000000000000000 --- a/data/developers/hastagaras.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "developer": "Hastagaras", - "models": [ - { - "id": "Hastagaras/L3.2-JametMini-3B-MK.III", - "name": "L3.2-JametMini-3B-MK.III", - "developer": "Hastagaras", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6183, - "hfopenllm_v2/BBH": 0.4539, - "hfopenllm_v2/MATH Level 5": 0.1458, - "hfopenllm_v2/GPQA": 0.2827, - "hfopenllm_v2/MUSR": 0.3686, - "hfopenllm_v2/MMLU-PRO": 0.2983 - } - }, - { - "id": "Hastagaras/Llama-3.1-Jamet-8B-MK.I", - "name": "Llama-3.1-Jamet-8B-MK.I", - "developer": "Hastagaras", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7338, - "hfopenllm_v2/BBH": 0.5049, - "hfopenllm_v2/MATH Level 5": 0.1269, - "hfopenllm_v2/GPQA": 0.2743, - "hfopenllm_v2/MUSR": 0.3726, - "hfopenllm_v2/MMLU-PRO": 0.3482 - } - }, - { - "id": "Hastagaras/Zabuza-8B-Llama-3.1", - "name": "Zabuza-8B-Llama-3.1", - "developer": "Hastagaras", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6265, - "hfopenllm_v2/BBH": 0.4539, - "hfopenllm_v2/MATH Level 5": 0.0551, - "hfopenllm_v2/GPQA": 0.2643, - "hfopenllm_v2/MUSR": 0.3568, - "hfopenllm_v2/MMLU-PRO": 0.2923 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/hatemmahmoud.json b/data/developers/hatemmahmoud.json deleted file mode 100644 index ccc014c278b3b9da1f79c243abd7ecaacd707c1a..0000000000000000000000000000000000000000 --- a/data/developers/hatemmahmoud.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "hatemmahmoud", - "models": [ - { - "id": "hatemmahmoud/qwen2.5-1.5b-sft-raft-grpo-hra-doc", - "name": "qwen2.5-1.5b-sft-raft-grpo-hra-doc", - "developer": "hatemmahmoud", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4196, - "hfopenllm_v2/BBH": 0.427, - "hfopenllm_v2/MATH Level 5": 0.2175, - "hfopenllm_v2/GPQA": 0.2676, - "hfopenllm_v2/MUSR": 0.361, - "hfopenllm_v2/MMLU-PRO": 0.2776 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/helpingai.json b/data/developers/helpingai.json deleted file mode 100644 index 085694447262ec4c5f7e4be954741a8e8210e8e2..0000000000000000000000000000000000000000 --- a/data/developers/helpingai.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "developer": "HelpingAI", - "models": [ - { - "id": "HelpingAI/Cipher-20B", - "name": "Cipher-20B", - "developer": "HelpingAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5378, - "hfopenllm_v2/BBH": 0.6032, - "hfopenllm_v2/MATH Level 5": 0.1994, - "hfopenllm_v2/GPQA": 0.2953, - "hfopenllm_v2/MUSR": 0.4003, - "hfopenllm_v2/MMLU-PRO": 0.3744 - } - }, - { - "id": "HelpingAI/Dhanishtha-Large", - "name": "Dhanishtha-Large", - "developer": "HelpingAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2457, - "hfopenllm_v2/BBH": 0.4604, - "hfopenllm_v2/MATH Level 5": 0.3852, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.3845, - "hfopenllm_v2/MMLU-PRO": 0.2755 - } - }, - { - "id": "HelpingAI/Priya-10B", - "name": "Priya-10B", - "developer": "HelpingAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4043, - "hfopenllm_v2/BBH": 0.4441, - "hfopenllm_v2/MATH Level 5": 0.0189, - "hfopenllm_v2/GPQA": 0.2559, - "hfopenllm_v2/MUSR": 0.3793, - "hfopenllm_v2/MMLU-PRO": 0.2493 - } - }, - { - "id": "HelpingAI/Priya-3B", - "name": "Priya-3B", - "developer": "HelpingAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4526, - "hfopenllm_v2/BBH": 0.3961, - "hfopenllm_v2/MATH Level 5": 0.0144, - "hfopenllm_v2/GPQA": 0.2567, - "hfopenllm_v2/MUSR": 0.3713, - "hfopenllm_v2/MMLU-PRO": 0.2339 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/hendrydong.json b/data/developers/hendrydong.json deleted file mode 100644 index 56e05071d3b06a1289763aa84d052b49b755b3aa..0000000000000000000000000000000000000000 --- a/data/developers/hendrydong.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "developer": "hendrydong", - "models": [ - { - "id": "hendrydong/Mistral-RM-for-RAFT-GSHF-v0", - "name": "hendrydong/Mistral-RM-for-RAFT-GSHF-v0", - "developer": "hendrydong", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7847, - "reward-bench/Factuality": 0.5779, - "reward-bench/Precise IF": 0.3625, - "reward-bench/Math": 0.6011, - "reward-bench/Safety": 0.85, - "reward-bench/Focus": 0.6747, - "reward-bench/Ties": 0.5988, - "reward-bench/Chat": 0.9832, - "reward-bench/Chat Hard": 0.5789, - "reward-bench/Reasoning": 0.7434, - "reward-bench/Prior Sets (0.5 weight)": 0.7508 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/heraihench.json b/data/developers/heraihench.json deleted file mode 100644 index bc8b8bfdab897d4156c4ef68a19ecb0f65fe6582..0000000000000000000000000000000000000000 --- a/data/developers/heraihench.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "developer": "HeraiHench", - "models": [ - { - "id": "HeraiHench/DeepSeek-R1-Qwen-Coder-8B", - "name": "DeepSeek-R1-Qwen-Coder-8B", - "developer": "HeraiHench", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1869, - "hfopenllm_v2/BBH": 0.2913, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3738, - "hfopenllm_v2/MMLU-PRO": 0.1123 - } - }, - { - "id": "HeraiHench/Double-Down-Qwen-Math-7B", - "name": "Double-Down-Qwen-Math-7B", - "developer": "HeraiHench", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.167, - "hfopenllm_v2/BBH": 0.2845, - "hfopenllm_v2/MATH Level 5": 0.0008, - "hfopenllm_v2/GPQA": 0.2651, - "hfopenllm_v2/MUSR": 0.3737, - "hfopenllm_v2/MMLU-PRO": 0.1112 - } - }, - { - "id": "HeraiHench/Marge-Qwen-Math-7B", - "name": "Marge-Qwen-Math-7B", - "developer": "HeraiHench", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1262, - "hfopenllm_v2/BBH": 0.3069, - "hfopenllm_v2/MATH Level 5": 0.0053, - "hfopenllm_v2/GPQA": 0.2391, - "hfopenllm_v2/MUSR": 0.3939, - "hfopenllm_v2/MMLU-PRO": 0.1056 - } - }, - { - "id": "HeraiHench/Phi-4-slerp-ReasoningRP-14B", - "name": "Phi-4-slerp-ReasoningRP-14B", - "developer": "HeraiHench", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1575, - "hfopenllm_v2/BBH": 0.4196, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2936, - "hfopenllm_v2/MUSR": 0.3116, - "hfopenllm_v2/MMLU-PRO": 0.19 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/hfxm.json b/data/developers/hfxm.json deleted file mode 100644 index 0cc645611ef3598fb6468fc4c126c9a8eb5680a6..0000000000000000000000000000000000000000 --- a/data/developers/hfxm.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "developer": "HFXM", - "models": [ - { - "id": "HFXM/RAMO-Llama3.1-8B", - "name": "HFXM/RAMO-Llama3.1-8B", - "developer": "HFXM", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6917, - "reward-bench/Factuality": 0.6547, - "reward-bench/Precise IF": 0.375, - "reward-bench/Math": 0.5628, - "reward-bench/Safety": 0.9756, - "reward-bench/Focus": 0.9071, - "reward-bench/Ties": 0.6752 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/hirosekoichi.json b/data/developers/hirosekoichi.json deleted file mode 100644 index 3c3233e49f4e56f29c0e270ad345bced0ce862bb..0000000000000000000000000000000000000000 --- a/data/developers/hirosekoichi.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "HiroseKoichi", - "models": [ - { - "id": "HiroseKoichi/Llama-Salad-4x8B-V3", - "name": "Llama-Salad-4x8B-V3", - "developer": "HiroseKoichi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6654, - "hfopenllm_v2/BBH": 0.5245, - "hfopenllm_v2/MATH Level 5": 0.0959, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.374, - "hfopenllm_v2/MMLU-PRO": 0.3518 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/hoangha.json b/data/developers/hoangha.json deleted file mode 100644 index 8ad548c64bf7c7d209db7b3b149c712f0eb52220..0000000000000000000000000000000000000000 --- a/data/developers/hoangha.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "HoangHa", - "models": [ - { - "id": "HoangHa/Pensez-Llama3.1-8B", - "name": "Pensez-Llama3.1-8B", - "developer": "HoangHa", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3887, - "hfopenllm_v2/BBH": 0.4669, - "hfopenllm_v2/MATH Level 5": 0.1148, - "hfopenllm_v2/GPQA": 0.2886, - "hfopenllm_v2/MUSR": 0.3597, - "hfopenllm_v2/MMLU-PRO": 0.3126 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/hon9kon9ize.json b/data/developers/hon9kon9ize.json deleted file mode 100644 index f340e6f94fab3cb0f8a79f260c68bc5ec6b80d3d..0000000000000000000000000000000000000000 --- a/data/developers/hon9kon9ize.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "hon9kon9ize", - "models": [ - { - "id": "hon9kon9ize/CantoneseLLMChat-v0.5", - "name": "CantoneseLLMChat-v0.5", - "developer": "hon9kon9ize", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3231, - "hfopenllm_v2/BBH": 0.4345, - "hfopenllm_v2/MATH Level 5": 0.0415, - "hfopenllm_v2/GPQA": 0.2777, - "hfopenllm_v2/MUSR": 0.4706, - "hfopenllm_v2/MMLU-PRO": 0.2504 - } - }, - { - "id": "hon9kon9ize/CantoneseLLMChat-v1.0-7B", - "name": "CantoneseLLMChat-v1.0-7B", - "developer": "hon9kon9ize", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4455, - "hfopenllm_v2/BBH": 0.4866, - "hfopenllm_v2/MATH Level 5": 0.2107, - "hfopenllm_v2/GPQA": 0.3221, - "hfopenllm_v2/MUSR": 0.3883, - "hfopenllm_v2/MMLU-PRO": 0.3785 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/hongbai12.json b/data/developers/hongbai12.json deleted file mode 100644 index 8ff34dbce459ce15b852da0ccf849e37ce258554..0000000000000000000000000000000000000000 --- a/data/developers/hongbai12.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "hongbai12", - "models": [ - { - "id": "hongbai12/li-0.4-pre", - "name": "li-0.4-pre", - "developer": "hongbai12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.52, - "hfopenllm_v2/BBH": 0.6298, - "hfopenllm_v2/MATH Level 5": 0.4924, - "hfopenllm_v2/GPQA": 0.323, - "hfopenllm_v2/MUSR": 0.4513, - "hfopenllm_v2/MMLU-PRO": 0.5015 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/hotmailuser.json b/data/developers/hotmailuser.json deleted file mode 100644 index 0bf0b7e1420feb2969792760e6a930a76a40d2a7..0000000000000000000000000000000000000000 --- a/data/developers/hotmailuser.json +++ /dev/null @@ -1,481 +0,0 @@ -{ - "developer": "hotmailuser", - "models": [ - { - "id": "hotmailuser/Deepseek-qwen-modelstock-2B", - "name": "Deepseek-qwen-modelstock-2B", - "developer": "hotmailuser", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2149, - "hfopenllm_v2/BBH": 0.3549, - "hfopenllm_v2/MATH Level 5": 0.3399, - "hfopenllm_v2/GPQA": 0.2802, - "hfopenllm_v2/MUSR": 0.3475, - "hfopenllm_v2/MMLU-PRO": 0.1911 - } - }, - { - "id": "hotmailuser/Falcon3Slerp1-10B", - "name": "Falcon3Slerp1-10B", - "developer": "hotmailuser", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5694, - "hfopenllm_v2/BBH": 0.617, - "hfopenllm_v2/MATH Level 5": 0.2598, - "hfopenllm_v2/GPQA": 0.344, - "hfopenllm_v2/MUSR": 0.4318, - "hfopenllm_v2/MMLU-PRO": 0.4402 - } - }, - { - "id": "hotmailuser/Falcon3Slerp2-10B", - "name": "Falcon3Slerp2-10B", - "developer": "hotmailuser", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6118, - "hfopenllm_v2/BBH": 0.6164, - "hfopenllm_v2/MATH Level 5": 0.2319, - "hfopenllm_v2/GPQA": 0.3381, - "hfopenllm_v2/MUSR": 0.4096, - "hfopenllm_v2/MMLU-PRO": 0.4369 - } - }, - { - "id": "hotmailuser/Falcon3Slerp4-10B", - "name": "Falcon3Slerp4-10B", - "developer": "hotmailuser", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6072, - "hfopenllm_v2/BBH": 0.6114, - "hfopenllm_v2/MATH Level 5": 0.2289, - "hfopenllm_v2/GPQA": 0.3289, - "hfopenllm_v2/MUSR": 0.4017, - "hfopenllm_v2/MMLU-PRO": 0.4387 - } - }, - { - "id": "hotmailuser/FalconSlerp-3B", - "name": "FalconSlerp-3B", - "developer": "hotmailuser", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5695, - "hfopenllm_v2/BBH": 0.4624, - "hfopenllm_v2/MATH Level 5": 0.176, - "hfopenllm_v2/GPQA": 0.2878, - "hfopenllm_v2/MUSR": 0.3989, - "hfopenllm_v2/MMLU-PRO": 0.2968 - } - }, - { - "id": "hotmailuser/FalconSlerp1-7B", - "name": "FalconSlerp1-7B", - "developer": "hotmailuser", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5395, - "hfopenllm_v2/BBH": 0.5355, - "hfopenllm_v2/MATH Level 5": 0.2379, - "hfopenllm_v2/GPQA": 0.3196, - "hfopenllm_v2/MUSR": 0.4452, - "hfopenllm_v2/MMLU-PRO": 0.4129 - } - }, - { - "id": "hotmailuser/FalconSlerp2-7B", - "name": "FalconSlerp2-7B", - "developer": "hotmailuser", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.616, - "hfopenllm_v2/BBH": 0.5538, - "hfopenllm_v2/MATH Level 5": 0.2983, - "hfopenllm_v2/GPQA": 0.3196, - "hfopenllm_v2/MUSR": 0.4479, - "hfopenllm_v2/MMLU-PRO": 0.4141 - } - }, - { - "id": "hotmailuser/FalconSlerp3-10B", - "name": "FalconSlerp3-10B", - "developer": "hotmailuser", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6002, - "hfopenllm_v2/BBH": 0.606, - "hfopenllm_v2/MATH Level 5": 0.2273, - "hfopenllm_v2/GPQA": 0.3356, - "hfopenllm_v2/MUSR": 0.4031, - "hfopenllm_v2/MMLU-PRO": 0.4323 - } - }, - { - "id": "hotmailuser/FalconSlerp3-7B", - "name": "FalconSlerp3-7B", - "developer": "hotmailuser", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6096, - "hfopenllm_v2/BBH": 0.5533, - "hfopenllm_v2/MATH Level 5": 0.3157, - "hfopenllm_v2/GPQA": 0.3188, - "hfopenllm_v2/MUSR": 0.4507, - "hfopenllm_v2/MMLU-PRO": 0.4127 - } - }, - { - "id": "hotmailuser/FalconSlerp4-7B", - "name": "FalconSlerp4-7B", - "developer": "hotmailuser", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6285, - "hfopenllm_v2/BBH": 0.5524, - "hfopenllm_v2/MATH Level 5": 0.2213, - "hfopenllm_v2/GPQA": 0.3322, - "hfopenllm_v2/MUSR": 0.4585, - "hfopenllm_v2/MMLU-PRO": 0.4032 - } - }, - { - "id": "hotmailuser/FalconSlerp6-7B", - "name": "FalconSlerp6-7B", - "developer": "hotmailuser", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6027, - "hfopenllm_v2/BBH": 0.5384, - "hfopenllm_v2/MATH Level 5": 0.2047, - "hfopenllm_v2/GPQA": 0.318, - "hfopenllm_v2/MUSR": 0.4492, - "hfopenllm_v2/MMLU-PRO": 0.3995 - } - }, - { - "id": "hotmailuser/Gemma2atlas-27B", - "name": "Gemma2atlas-27B", - "developer": "hotmailuser", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7214, - "hfopenllm_v2/BBH": 0.6545, - "hfopenllm_v2/MATH Level 5": 0.2145, - "hfopenllm_v2/GPQA": 0.3557, - "hfopenllm_v2/MUSR": 0.4445, - "hfopenllm_v2/MMLU-PRO": 0.475 - } - }, - { - "id": "hotmailuser/Gemma2Crono-27B", - "name": "Gemma2Crono-27B", - "developer": "hotmailuser", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7086, - "hfopenllm_v2/BBH": 0.6505, - "hfopenllm_v2/MATH Level 5": 0.2424, - "hfopenllm_v2/GPQA": 0.3708, - "hfopenllm_v2/MUSR": 0.4567, - "hfopenllm_v2/MMLU-PRO": 0.4633 - } - }, - { - "id": "hotmailuser/Gemma2magnum-27b", - "name": "Gemma2magnum-27b", - "developer": "hotmailuser", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5051, - "hfopenllm_v2/BBH": 0.62, - "hfopenllm_v2/MATH Level 5": 0.2205, - "hfopenllm_v2/GPQA": 0.3851, - "hfopenllm_v2/MUSR": 0.4723, - "hfopenllm_v2/MMLU-PRO": 0.4596 - } - }, - { - "id": "hotmailuser/Gemma2SimPO-27B", - "name": "Gemma2SimPO-27B", - "developer": "hotmailuser", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7222, - "hfopenllm_v2/BBH": 0.6413, - "hfopenllm_v2/MATH Level 5": 0.2817, - "hfopenllm_v2/GPQA": 0.3582, - "hfopenllm_v2/MUSR": 0.4447, - "hfopenllm_v2/MMLU-PRO": 0.4642 - } - }, - { - "id": "hotmailuser/Llama-Hermes-slerp-8B", - "name": "Llama-Hermes-slerp-8B", - "developer": "hotmailuser", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.339, - "hfopenllm_v2/BBH": 0.531, - "hfopenllm_v2/MATH Level 5": 0.0801, - "hfopenllm_v2/GPQA": 0.2936, - "hfopenllm_v2/MUSR": 0.4078, - "hfopenllm_v2/MMLU-PRO": 0.3331 - } - }, - { - "id": "hotmailuser/Llama-Hermes-slerp2-8B", - "name": "Llama-Hermes-slerp2-8B", - "developer": "hotmailuser", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3728, - "hfopenllm_v2/BBH": 0.5265, - "hfopenllm_v2/MATH Level 5": 0.0974, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.4248, - "hfopenllm_v2/MMLU-PRO": 0.3379 - } - }, - { - "id": "hotmailuser/LlamaStock-8B", - "name": "LlamaStock-8B", - "developer": "hotmailuser", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.425, - "hfopenllm_v2/BBH": 0.5329, - "hfopenllm_v2/MATH Level 5": 0.1699, - "hfopenllm_v2/GPQA": 0.3272, - "hfopenllm_v2/MUSR": 0.4129, - "hfopenllm_v2/MMLU-PRO": 0.3807 - } - }, - { - "id": "hotmailuser/Mistral-modelstock-24B", - "name": "Mistral-modelstock-24B", - "developer": "hotmailuser", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3424, - "hfopenllm_v2/BBH": 0.6452, - "hfopenllm_v2/MATH Level 5": 0.1307, - "hfopenllm_v2/GPQA": 0.4102, - "hfopenllm_v2/MUSR": 0.459, - "hfopenllm_v2/MMLU-PRO": 0.507 - } - }, - { - "id": "hotmailuser/Mistral-modelstock2-24B", - "name": "Mistral-modelstock2-24B", - "developer": "hotmailuser", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4318, - "hfopenllm_v2/BBH": 0.6689, - "hfopenllm_v2/MATH Level 5": 0.2402, - "hfopenllm_v2/GPQA": 0.3926, - "hfopenllm_v2/MUSR": 0.4616, - "hfopenllm_v2/MMLU-PRO": 0.5318 - } - }, - { - "id": "hotmailuser/Phi4-Slerp4-14B", - "name": "Phi4-Slerp4-14B", - "developer": "hotmailuser", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0629, - "hfopenllm_v2/BBH": 0.6731, - "hfopenllm_v2/MATH Level 5": 0.3474, - "hfopenllm_v2/GPQA": 0.3968, - "hfopenllm_v2/MUSR": 0.5097, - "hfopenllm_v2/MMLU-PRO": 0.5278 - } - }, - { - "id": "hotmailuser/Qwen2.5-HomerSlerp-7B", - "name": "Qwen2.5-HomerSlerp-7B", - "developer": "hotmailuser", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4488, - "hfopenllm_v2/BBH": 0.5633, - "hfopenllm_v2/MATH Level 5": 0.3316, - "hfopenllm_v2/GPQA": 0.3138, - "hfopenllm_v2/MUSR": 0.4383, - "hfopenllm_v2/MMLU-PRO": 0.4549 - } - }, - { - "id": "hotmailuser/QwenModelStock-1.8B", - "name": "QwenModelStock-1.8B", - "developer": "hotmailuser", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3263, - "hfopenllm_v2/BBH": 0.4188, - "hfopenllm_v2/MATH Level 5": 0.0989, - "hfopenllm_v2/GPQA": 0.2869, - "hfopenllm_v2/MUSR": 0.4359, - "hfopenllm_v2/MMLU-PRO": 0.2959 - } - }, - { - "id": "hotmailuser/QwenSlerp-14B", - "name": "QwenSlerp-14B", - "developer": "hotmailuser", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7025, - "hfopenllm_v2/BBH": 0.6491, - "hfopenllm_v2/MATH Level 5": 0.3837, - "hfopenllm_v2/GPQA": 0.3876, - "hfopenllm_v2/MUSR": 0.4634, - "hfopenllm_v2/MMLU-PRO": 0.54 - } - }, - { - "id": "hotmailuser/QwenSlerp-3B", - "name": "QwenSlerp-3B", - "developer": "hotmailuser", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4334, - "hfopenllm_v2/BBH": 0.4892, - "hfopenllm_v2/MATH Level 5": 0.2749, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.4317, - "hfopenllm_v2/MMLU-PRO": 0.3693 - } - }, - { - "id": "hotmailuser/QwenSlerp-7B", - "name": "QwenSlerp-7B", - "developer": "hotmailuser", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4673, - "hfopenllm_v2/BBH": 0.5636, - "hfopenllm_v2/MATH Level 5": 0.3444, - "hfopenllm_v2/GPQA": 0.318, - "hfopenllm_v2/MUSR": 0.4409, - "hfopenllm_v2/MMLU-PRO": 0.4509 - } - }, - { - "id": "hotmailuser/QwenSlerp2-14B", - "name": "QwenSlerp2-14B", - "developer": "hotmailuser", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7037, - "hfopenllm_v2/BBH": 0.6493, - "hfopenllm_v2/MATH Level 5": 0.3965, - "hfopenllm_v2/GPQA": 0.3809, - "hfopenllm_v2/MUSR": 0.4807, - "hfopenllm_v2/MMLU-PRO": 0.5379 - } - }, - { - "id": "hotmailuser/QwenSlerp2-3B", - "name": "QwenSlerp2-3B", - "developer": "hotmailuser", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.428, - "hfopenllm_v2/BBH": 0.4802, - "hfopenllm_v2/MATH Level 5": 0.2606, - "hfopenllm_v2/GPQA": 0.297, - "hfopenllm_v2/MUSR": 0.4252, - "hfopenllm_v2/MMLU-PRO": 0.3742 - } - }, - { - "id": "hotmailuser/QwenSlerp3-14B", - "name": "QwenSlerp3-14B", - "developer": "hotmailuser", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6632, - "hfopenllm_v2/BBH": 0.6267, - "hfopenllm_v2/MATH Level 5": 0.4305, - "hfopenllm_v2/GPQA": 0.3666, - "hfopenllm_v2/MUSR": 0.4808, - "hfopenllm_v2/MMLU-PRO": 0.5263 - } - }, - { - "id": "hotmailuser/QwenSparse-7B", - "name": "QwenSparse-7B", - "developer": "hotmailuser", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1086, - "hfopenllm_v2/BBH": 0.2896, - "hfopenllm_v2/MATH Level 5": 0.0106, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3562, - "hfopenllm_v2/MMLU-PRO": 0.1122 - } - }, - { - "id": "hotmailuser/QwenStock-0.5B", - "name": "QwenStock-0.5B", - "developer": "hotmailuser", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2049, - "hfopenllm_v2/BBH": 0.2912, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3575, - "hfopenllm_v2/MMLU-PRO": 0.1167 - } - }, - { - "id": "hotmailuser/QwenStock-1.7B", - "name": "QwenStock-1.7B", - "developer": "hotmailuser", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3214, - "hfopenllm_v2/BBH": 0.4188, - "hfopenllm_v2/MATH Level 5": 0.0997, - "hfopenllm_v2/GPQA": 0.2878, - "hfopenllm_v2/MUSR": 0.4412, - "hfopenllm_v2/MMLU-PRO": 0.2955 - } - }, - { - "id": "hotmailuser/QwenStock1-14B", - "name": "QwenStock1-14B", - "developer": "hotmailuser", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6693, - "hfopenllm_v2/BBH": 0.6502, - "hfopenllm_v2/MATH Level 5": 0.3701, - "hfopenllm_v2/GPQA": 0.3859, - "hfopenllm_v2/MUSR": 0.4781, - "hfopenllm_v2/MMLU-PRO": 0.5416 - } - }, - { - "id": "hotmailuser/RombosBeagle-v2beta-MGS-32B", - "name": "RombosBeagle-v2beta-MGS-32B", - "developer": "hotmailuser", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5157, - "hfopenllm_v2/BBH": 0.7037, - "hfopenllm_v2/MATH Level 5": 0.4992, - "hfopenllm_v2/GPQA": 0.38, - "hfopenllm_v2/MUSR": 0.5021, - "hfopenllm_v2/MMLU-PRO": 0.5908 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/hpai-bsc.json b/data/developers/hpai-bsc.json deleted file mode 100644 index 8e5820265e260824ba737e99923df14285fb7e0e..0000000000000000000000000000000000000000 --- a/data/developers/hpai-bsc.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "developer": "HPAI-BSC", - "models": [ - { - "id": "HPAI-BSC/Llama3-Aloe-8B-Alpha", - "name": "Llama3-Aloe-8B-Alpha", - "developer": "HPAI-BSC", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5081, - "hfopenllm_v2/BBH": 0.4831, - "hfopenllm_v2/MATH Level 5": 0.0612, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.3673, - "hfopenllm_v2/MMLU-PRO": 0.3295 - } - }, - { - "id": "HPAI-BSC/Llama3.1-Aloe-Beta-8B", - "name": "Llama3.1-Aloe-Beta-8B", - "developer": "HPAI-BSC", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7253, - "hfopenllm_v2/BBH": 0.5093, - "hfopenllm_v2/MATH Level 5": 0.1828, - "hfopenllm_v2/GPQA": 0.2685, - "hfopenllm_v2/MUSR": 0.3835, - "hfopenllm_v2/MMLU-PRO": 0.358 - } - }, - { - "id": "HPAI-BSC/Qwen2.5-Aloe-Beta-7B", - "name": "Qwen2.5-Aloe-Beta-7B", - "developer": "HPAI-BSC", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4554, - "hfopenllm_v2/BBH": 0.5049, - "hfopenllm_v2/MATH Level 5": 0.3542, - "hfopenllm_v2/GPQA": 0.2911, - "hfopenllm_v2/MUSR": 0.426, - "hfopenllm_v2/MMLU-PRO": 0.4354 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/huawei-noah-ustc.json b/data/developers/huawei-noah-ustc.json deleted file mode 100644 index ac2fcf9fa26eb1cec6221a73bebcca38cce7bfbe..0000000000000000000000000000000000000000 --- a/data/developers/huawei-noah-ustc.json +++ /dev/null @@ -1,45 +0,0 @@ -{ - "developer": "huawei-noah-ustc", - "models": [ - { - "id": "huawei-noah-ustc/toolace-2-8b-fc", - "name": "ToolACE-2-8B (FC)", - "developer": "huawei-noah-ustc", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 40.0, - "bfcl/bfcl.overall.overall_accuracy": 42.44, - "bfcl/bfcl.overall.total_cost_usd": 24.43, - "bfcl/bfcl.overall.latency_mean_s": 15.95, - "bfcl/bfcl.overall.latency_std_s": 40.06, - "bfcl/bfcl.overall.latency_p95_s": 65.26, - "bfcl/bfcl.non_live.ast_accuracy": 87.1, - "bfcl/bfcl.non_live.simple_ast_accuracy": 73.42, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 91.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 93.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 91.0, - "bfcl/bfcl.live.live_accuracy": 77.42, - "bfcl/bfcl.live.live_simple_ast_accuracy": 71.32, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 79.39, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 68.75, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 62.5, - "bfcl/bfcl.multi_turn.accuracy": 38.38, - "bfcl/bfcl.multi_turn.base_accuracy": 49.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 28.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 30.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 46.0, - "bfcl/bfcl.web_search.accuracy": 8.5, - "bfcl/bfcl.web_search.base_accuracy": 13.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 4.0, - "bfcl/bfcl.memory.accuracy": 18.49, - "bfcl/bfcl.memory.kv_accuracy": 5.81, - "bfcl/bfcl.memory.vector_accuracy": 16.13, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 33.55, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 75.0, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 90.79, - "bfcl/bfcl.format_sensitivity.max_delta": 81.5, - "bfcl/bfcl.format_sensitivity.stddev": 27.92 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/huggingfaceh4.json b/data/developers/huggingfaceh4.json deleted file mode 100644 index d09a50838e73a777be131b3290796a075dbbf1cc..0000000000000000000000000000000000000000 --- a/data/developers/huggingfaceh4.json +++ /dev/null @@ -1,93 +0,0 @@ -{ - "developer": "HuggingFaceH4", - "models": [ - { - "id": "HuggingFaceH4/starchat2-15b-v0.1", - "name": "HuggingFaceH4/starchat2-15b-v0.1", - "developer": "HuggingFaceH4", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7322, - "reward-bench/Chat": 0.9385, - "reward-bench/Chat Hard": 0.5548, - "reward-bench/Safety": 0.7095, - "reward-bench/Reasoning": 0.8159, - "reward-bench/Prior Sets (0.5 weight)": 0.5525 - } - }, - { - "id": "HuggingFaceH4/zephyr-7b-alpha", - "name": "HuggingFaceH4/zephyr-7b-alpha", - "developer": "HuggingFaceH4", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5191, - "hfopenllm_v2/BBH": 0.4583, - "hfopenllm_v2/MATH Level 5": 0.0196, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.395, - "hfopenllm_v2/MMLU-PRO": 0.2795, - "reward-bench/Score": 0.7392, - "reward-bench/Chat": 0.9162, - "reward-bench/Chat Hard": 0.625, - "reward-bench/Safety": 0.7662, - "reward-bench/Reasoning": 0.7514, - "reward-bench/Prior Sets (0.5 weight)": 0.5353 - } - }, - { - "id": "HuggingFaceH4/zephyr-7b-beta", - "name": "HuggingFaceH4/zephyr-7b-beta", - "developer": "HuggingFaceH4", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.495, - "hfopenllm_v2/BBH": 0.4316, - "hfopenllm_v2/MATH Level 5": 0.0287, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.3925, - "hfopenllm_v2/MMLU-PRO": 0.2781, - "reward-bench/Score": 0.7281, - "reward-bench/Chat": 0.9525, - "reward-bench/Chat Hard": 0.6272, - "reward-bench/Safety": 0.6568, - "reward-bench/Reasoning": 0.7789, - "reward-bench/Prior Sets (0.5 weight)": 0.5216 - } - }, - { - "id": "HuggingFaceH4/zephyr-7b-gemma-v0.1", - "name": "HuggingFaceH4/zephyr-7b-gemma-v0.1", - "developer": "HuggingFaceH4", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3364, - "hfopenllm_v2/BBH": 0.4624, - "hfopenllm_v2/MATH Level 5": 0.0816, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.374, - "hfopenllm_v2/MMLU-PRO": 0.2847, - "reward-bench/Score": 0.6758, - "reward-bench/Chat": 0.9581, - "reward-bench/Chat Hard": 0.4956, - "reward-bench/Safety": 0.5824, - "reward-bench/Reasoning": 0.7463, - "reward-bench/Prior Sets (0.5 weight)": 0.5171 - } - }, - { - "id": "HuggingFaceH4/zephyr-orpo-141b-A35b-v0.1", - "name": "zephyr-orpo-141b-A35b-v0.1", - "developer": "HuggingFaceH4", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6511, - "hfopenllm_v2/BBH": 0.629, - "hfopenllm_v2/MATH Level 5": 0.2047, - "hfopenllm_v2/GPQA": 0.3784, - "hfopenllm_v2/MUSR": 0.4465, - "hfopenllm_v2/MMLU-PRO": 0.4586 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/huggingfacetb.json b/data/developers/huggingfacetb.json deleted file mode 100644 index bed31781473fb30427be579aab45ef01bf5054ce..0000000000000000000000000000000000000000 --- a/data/developers/huggingfacetb.json +++ /dev/null @@ -1,173 +0,0 @@ -{ - "developer": "HuggingFaceTB", - "models": [ - { - "id": "HuggingFaceTB/SmolLM-1.7B", - "name": "SmolLM-1.7B", - "developer": "HuggingFaceTB", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2362, - "hfopenllm_v2/BBH": 0.3181, - "hfopenllm_v2/MATH Level 5": 0.0166, - "hfopenllm_v2/GPQA": 0.2416, - "hfopenllm_v2/MUSR": 0.3421, - "hfopenllm_v2/MMLU-PRO": 0.1148 - } - }, - { - "id": "HuggingFaceTB/SmolLM-1.7B-Instruct", - "name": "SmolLM-1.7B-Instruct", - "developer": "HuggingFaceTB", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2348, - "hfopenllm_v2/BBH": 0.2885, - "hfopenllm_v2/MATH Level 5": 0.0211, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3487, - "hfopenllm_v2/MMLU-PRO": 0.1166 - } - }, - { - "id": "HuggingFaceTB/SmolLM-135M", - "name": "SmolLM-135M", - "developer": "HuggingFaceTB", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2125, - "hfopenllm_v2/BBH": 0.3046, - "hfopenllm_v2/MATH Level 5": 0.0136, - "hfopenllm_v2/GPQA": 0.2584, - "hfopenllm_v2/MUSR": 0.4366, - "hfopenllm_v2/MMLU-PRO": 0.1122 - } - }, - { - "id": "HuggingFaceTB/SmolLM-135M-Instruct", - "name": "SmolLM-135M-Instruct", - "developer": "HuggingFaceTB", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1214, - "hfopenllm_v2/BBH": 0.3015, - "hfopenllm_v2/MATH Level 5": 0.0053, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3635, - "hfopenllm_v2/MMLU-PRO": 0.1176 - } - }, - { - "id": "HuggingFaceTB/SmolLM-360M", - "name": "SmolLM-360M", - "developer": "HuggingFaceTB", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2134, - "hfopenllm_v2/BBH": 0.3065, - "hfopenllm_v2/MATH Level 5": 0.0113, - "hfopenllm_v2/GPQA": 0.2676, - "hfopenllm_v2/MUSR": 0.4018, - "hfopenllm_v2/MMLU-PRO": 0.1124 - } - }, - { - "id": "HuggingFaceTB/SmolLM-360M-Instruct", - "name": "SmolLM-360M-Instruct", - "developer": "HuggingFaceTB", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1952, - "hfopenllm_v2/BBH": 0.2885, - "hfopenllm_v2/MATH Level 5": 0.0181, - "hfopenllm_v2/GPQA": 0.2643, - "hfopenllm_v2/MUSR": 0.3472, - "hfopenllm_v2/MMLU-PRO": 0.1166 - } - }, - { - "id": "HuggingFaceTB/SmolLM2-1.7B", - "name": "SmolLM2-1.7B", - "developer": "HuggingFaceTB", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.244, - "hfopenllm_v2/BBH": 0.3453, - "hfopenllm_v2/MATH Level 5": 0.0264, - "hfopenllm_v2/GPQA": 0.2794, - "hfopenllm_v2/MUSR": 0.3485, - "hfopenllm_v2/MMLU-PRO": 0.2138 - } - }, - { - "id": "HuggingFaceTB/SmolLM2-1.7B-Instruct", - "name": "SmolLM2-1.7B-Instruct", - "developer": "HuggingFaceTB", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5368, - "hfopenllm_v2/BBH": 0.3599, - "hfopenllm_v2/MATH Level 5": 0.0582, - "hfopenllm_v2/GPQA": 0.2794, - "hfopenllm_v2/MUSR": 0.3421, - "hfopenllm_v2/MMLU-PRO": 0.2054 - } - }, - { - "id": "HuggingFaceTB/SmolLM2-135M", - "name": "SmolLM2-135M", - "developer": "HuggingFaceTB", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1818, - "hfopenllm_v2/BBH": 0.3044, - "hfopenllm_v2/MATH Level 5": 0.0121, - "hfopenllm_v2/GPQA": 0.2483, - "hfopenllm_v2/MUSR": 0.4112, - "hfopenllm_v2/MMLU-PRO": 0.1095 - } - }, - { - "id": "HuggingFaceTB/SmolLM2-135M-Instruct", - "name": "SmolLM2-135M-Instruct", - "developer": "HuggingFaceTB", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0593, - "hfopenllm_v2/BBH": 0.3135, - "hfopenllm_v2/MATH Level 5": 0.0144, - "hfopenllm_v2/GPQA": 0.2341, - "hfopenllm_v2/MUSR": 0.3871, - "hfopenllm_v2/MMLU-PRO": 0.1092 - } - }, - { - "id": "HuggingFaceTB/SmolLM2-360M", - "name": "SmolLM2-360M", - "developer": "HuggingFaceTB", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2115, - "hfopenllm_v2/BBH": 0.3233, - "hfopenllm_v2/MATH Level 5": 0.0121, - "hfopenllm_v2/GPQA": 0.2458, - "hfopenllm_v2/MUSR": 0.3954, - "hfopenllm_v2/MMLU-PRO": 0.1169 - } - }, - { - "id": "HuggingFaceTB/SmolLM2-360M-Instruct", - "name": "SmolLM2-360M-Instruct", - "developer": "HuggingFaceTB", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.083, - "hfopenllm_v2/BBH": 0.3053, - "hfopenllm_v2/MATH Level 5": 0.0083, - "hfopenllm_v2/GPQA": 0.2651, - "hfopenllm_v2/MUSR": 0.3423, - "hfopenllm_v2/MMLU-PRO": 0.1126 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/huggyllama.json b/data/developers/huggyllama.json deleted file mode 100644 index 385671a5d7ddc07d1e6a7c21e6d5bed23eeb69da..0000000000000000000000000000000000000000 --- a/data/developers/huggyllama.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "developer": "huggyllama", - "models": [ - { - "id": "huggyllama/llama-13b", - "name": "llama-13b", - "developer": "huggyllama", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2411, - "hfopenllm_v2/BBH": 0.3988, - "hfopenllm_v2/MATH Level 5": 0.0204, - "hfopenllm_v2/GPQA": 0.255, - "hfopenllm_v2/MUSR": 0.3462, - "hfopenllm_v2/MMLU-PRO": 0.1952 - } - }, - { - "id": "huggyllama/llama-65b", - "name": "llama-65b", - "developer": "huggyllama", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2526, - "hfopenllm_v2/BBH": 0.4703, - "hfopenllm_v2/MATH Level 5": 0.031, - "hfopenllm_v2/GPQA": 0.276, - "hfopenllm_v2/MUSR": 0.3595, - "hfopenllm_v2/MMLU-PRO": 0.3078 - } - }, - { - "id": "huggyllama/llama-7b", - "name": "llama-7b", - "developer": "huggyllama", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2501, - "hfopenllm_v2/BBH": 0.3277, - "hfopenllm_v2/MATH Level 5": 0.0083, - "hfopenllm_v2/GPQA": 0.2525, - "hfopenllm_v2/MUSR": 0.3354, - "hfopenllm_v2/MMLU-PRO": 0.1313 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/huihui-ai.json b/data/developers/huihui-ai.json deleted file mode 100644 index a8f76ff82aa1ce364cf4429ba16aacfbe50f29e0..0000000000000000000000000000000000000000 --- a/data/developers/huihui-ai.json +++ /dev/null @@ -1,117 +0,0 @@ -{ - "developer": "huihui-ai", - "models": [ - { - "id": "huihui-ai/DeepSeek-R1-Distill-Qwen-14B-abliterated-v2", - "name": "DeepSeek-R1-Distill-Qwen-14B-abliterated-v2", - "developer": "huihui-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4211, - "hfopenllm_v2/BBH": 0.3487, - "hfopenllm_v2/MATH Level 5": 0.2205, - "hfopenllm_v2/GPQA": 0.276, - "hfopenllm_v2/MUSR": 0.4701, - "hfopenllm_v2/MMLU-PRO": 0.1915 - } - }, - { - "id": "huihui-ai/Qwen2.5-14B-Instruct-abliterated-v2", - "name": "Qwen2.5-14B-Instruct-abliterated-v2", - "developer": "huihui-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8328, - "hfopenllm_v2/BBH": 0.6324, - "hfopenllm_v2/MATH Level 5": 0.5302, - "hfopenllm_v2/GPQA": 0.3339, - "hfopenllm_v2/MUSR": 0.422, - "hfopenllm_v2/MMLU-PRO": 0.4962 - } - }, - { - "id": "huihui-ai/Qwen2.5-72B-Instruct-abliterated", - "name": "Qwen2.5-72B-Instruct-abliterated", - "developer": "huihui-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8593, - "hfopenllm_v2/BBH": 0.719, - "hfopenllm_v2/MATH Level 5": 0.6012, - "hfopenllm_v2/GPQA": 0.3951, - "hfopenllm_v2/MUSR": 0.4233, - "hfopenllm_v2/MMLU-PRO": 0.5537 - } - }, - { - "id": "huihui-ai/Qwen2.5-7B-Instruct-abliterated", - "name": "Qwen2.5-7B-Instruct-abliterated", - "developer": "huihui-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7546, - "hfopenllm_v2/BBH": 0.5262, - "hfopenllm_v2/MATH Level 5": 0.4577, - "hfopenllm_v2/GPQA": 0.3154, - "hfopenllm_v2/MUSR": 0.3967, - "hfopenllm_v2/MMLU-PRO": 0.418 - } - }, - { - "id": "huihui-ai/Qwen2.5-7B-Instruct-abliterated-v2", - "name": "Qwen2.5-7B-Instruct-abliterated-v2", - "developer": "huihui-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7606, - "hfopenllm_v2/BBH": 0.5377, - "hfopenllm_v2/MATH Level 5": 0.4637, - "hfopenllm_v2/GPQA": 0.3087, - "hfopenllm_v2/MUSR": 0.3981, - "hfopenllm_v2/MMLU-PRO": 0.4208 - } - }, - { - "id": "huihui-ai/QwQ-32B-Coder-Fusion-7030", - "name": "QwQ-32B-Coder-Fusion-7030", - "developer": "huihui-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3865, - "hfopenllm_v2/BBH": 0.6178, - "hfopenllm_v2/MATH Level 5": 0.2795, - "hfopenllm_v2/GPQA": 0.2844, - "hfopenllm_v2/MUSR": 0.3922, - "hfopenllm_v2/MMLU-PRO": 0.4368 - } - }, - { - "id": "huihui-ai/QwQ-32B-Coder-Fusion-8020", - "name": "QwQ-32B-Coder-Fusion-8020", - "developer": "huihui-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6021, - "hfopenllm_v2/BBH": 0.6665, - "hfopenllm_v2/MATH Level 5": 0.4592, - "hfopenllm_v2/GPQA": 0.3549, - "hfopenllm_v2/MUSR": 0.4293, - "hfopenllm_v2/MMLU-PRO": 0.5367 - } - }, - { - "id": "huihui-ai/QwQ-32B-Coder-Fusion-9010", - "name": "QwQ-32B-Coder-Fusion-9010", - "developer": "huihui-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5778, - "hfopenllm_v2/BBH": 0.6727, - "hfopenllm_v2/MATH Level 5": 0.5317, - "hfopenllm_v2/GPQA": 0.3616, - "hfopenllm_v2/MUSR": 0.4682, - "hfopenllm_v2/MMLU-PRO": 0.56 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/humanllms.json b/data/developers/humanllms.json deleted file mode 100644 index d846fdb1c31c64fe2700aca95b192ad5efdb2343..0000000000000000000000000000000000000000 --- a/data/developers/humanllms.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "developer": "HumanLLMs", - "models": [ - { - "id": "HumanLLMs/Humanish-LLama3-8B-Instruct", - "name": "Humanish-LLama3-8B-Instruct", - "developer": "HumanLLMs", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6498, - "hfopenllm_v2/BBH": 0.4968, - "hfopenllm_v2/MATH Level 5": 0.1027, - "hfopenllm_v2/GPQA": 0.2559, - "hfopenllm_v2/MUSR": 0.3582, - "hfopenllm_v2/MMLU-PRO": 0.3702 - } - }, - { - "id": "HumanLLMs/Humanish-Mistral-Nemo-Instruct-2407", - "name": "Humanish-Mistral-Nemo-Instruct-2407", - "developer": "HumanLLMs", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5451, - "hfopenllm_v2/BBH": 0.5262, - "hfopenllm_v2/MATH Level 5": 0.1367, - "hfopenllm_v2/GPQA": 0.2878, - "hfopenllm_v2/MUSR": 0.3968, - "hfopenllm_v2/MMLU-PRO": 0.3521 - } - }, - { - "id": "HumanLLMs/Humanish-Qwen2.5-7B-Instruct", - "name": "Humanish-Qwen2.5-7B-Instruct", - "developer": "HumanLLMs", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7284, - "hfopenllm_v2/BBH": 0.5364, - "hfopenllm_v2/MATH Level 5": 0.5, - "hfopenllm_v2/GPQA": 0.2987, - "hfopenllm_v2/MUSR": 0.3981, - "hfopenllm_v2/MMLU-PRO": 0.4398 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/huu-ontocord.json b/data/developers/huu-ontocord.json deleted file mode 100644 index 55cf01e55b67d7b8aca6f1c18a51c194dc111f8a..0000000000000000000000000000000000000000 --- a/data/developers/huu-ontocord.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "huu-ontocord", - "models": [ - { - "id": "huu-ontocord/wide_3b_orpo_stage1.1-ss1-orpo3", - "name": "wide_3b_orpo_stage1.1-ss1-orpo3", - "developer": "huu-ontocord", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1505, - "hfopenllm_v2/BBH": 0.2937, - "hfopenllm_v2/MATH Level 5": 0.0098, - "hfopenllm_v2/GPQA": 0.2584, - "hfopenllm_v2/MUSR": 0.3618, - "hfopenllm_v2/MMLU-PRO": 0.1164 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/ibivibiv.json b/data/developers/ibivibiv.json deleted file mode 100644 index 8269b8ed59cb84b54b31344a04348454354da164..0000000000000000000000000000000000000000 --- a/data/developers/ibivibiv.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "ibivibiv", - "models": [ - { - "id": "ibivibiv/colossus_120b", - "name": "colossus_120b", - "developer": "ibivibiv", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4276, - "hfopenllm_v2/BBH": 0.6061, - "hfopenllm_v2/MATH Level 5": 0.0566, - "hfopenllm_v2/GPQA": 0.3087, - "hfopenllm_v2/MUSR": 0.4733, - "hfopenllm_v2/MMLU-PRO": 0.3961 - } - }, - { - "id": "ibivibiv/multimaster-7b-v6", - "name": "multimaster-7b-v6", - "developer": "ibivibiv", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4473, - "hfopenllm_v2/BBH": 0.5194, - "hfopenllm_v2/MATH Level 5": 0.0559, - "hfopenllm_v2/GPQA": 0.3037, - "hfopenllm_v2/MUSR": 0.4396, - "hfopenllm_v2/MMLU-PRO": 0.3095 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/ibm-granite.json b/data/developers/ibm-granite.json deleted file mode 100644 index cbdb43757d126217fef481bf71c9969a6e78a083..0000000000000000000000000000000000000000 --- a/data/developers/ibm-granite.json +++ /dev/null @@ -1,285 +0,0 @@ -{ - "developer": "ibm-granite", - "models": [ - { - "id": "ibm-granite/granite-3.0-1b-a400m-base", - "name": "granite-3.0-1b-a400m-base", - "developer": "ibm-granite", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2404, - "hfopenllm_v2/BBH": 0.3221, - "hfopenllm_v2/MATH Level 5": 0.0264, - "hfopenllm_v2/GPQA": 0.2475, - "hfopenllm_v2/MUSR": 0.3367, - "hfopenllm_v2/MMLU-PRO": 0.1152 - } - }, - { - "id": "ibm-granite/granite-3.0-1b-a400m-instruct", - "name": "granite-3.0-1b-a400m-instruct", - "developer": "ibm-granite", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3332, - "hfopenllm_v2/BBH": 0.3224, - "hfopenllm_v2/MATH Level 5": 0.0279, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.3623, - "hfopenllm_v2/MMLU-PRO": 0.1244 - } - }, - { - "id": "ibm-granite/granite-3.0-2b-base", - "name": "granite-3.0-2b-base", - "developer": "ibm-granite", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3874, - "hfopenllm_v2/BBH": 0.4047, - "hfopenllm_v2/MATH Level 5": 0.0544, - "hfopenllm_v2/GPQA": 0.2802, - "hfopenllm_v2/MUSR": 0.3434, - "hfopenllm_v2/MMLU-PRO": 0.2381 - } - }, - { - "id": "ibm-granite/granite-3.0-2b-instruct", - "name": "granite-3.0-2b-instruct", - "developer": "ibm-granite", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.514, - "hfopenllm_v2/BBH": 0.4412, - "hfopenllm_v2/MATH Level 5": 0.0921, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.3515, - "hfopenllm_v2/MMLU-PRO": 0.2814 - } - }, - { - "id": "ibm-granite/granite-3.0-3b-a800m-base", - "name": "granite-3.0-3b-a800m-base", - "developer": "ibm-granite", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2732, - "hfopenllm_v2/BBH": 0.3667, - "hfopenllm_v2/MATH Level 5": 0.0483, - "hfopenllm_v2/GPQA": 0.2517, - "hfopenllm_v2/MUSR": 0.342, - "hfopenllm_v2/MMLU-PRO": 0.1891 - } - }, - { - "id": "ibm-granite/granite-3.0-3b-a800m-instruct", - "name": "granite-3.0-3b-a800m-instruct", - "developer": "ibm-granite", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4298, - "hfopenllm_v2/BBH": 0.3753, - "hfopenllm_v2/MATH Level 5": 0.0702, - "hfopenllm_v2/GPQA": 0.281, - "hfopenllm_v2/MUSR": 0.3487, - "hfopenllm_v2/MMLU-PRO": 0.2152 - } - }, - { - "id": "ibm-granite/granite-3.0-8b-base", - "name": "granite-3.0-8b-base", - "developer": "ibm-granite", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4583, - "hfopenllm_v2/BBH": 0.4944, - "hfopenllm_v2/MATH Level 5": 0.1012, - "hfopenllm_v2/GPQA": 0.3255, - "hfopenllm_v2/MUSR": 0.4081, - "hfopenllm_v2/MMLU-PRO": 0.3313 - } - }, - { - "id": "ibm-granite/granite-3.0-8b-instruct", - "name": "granite-3.0-8b-instruct", - "developer": "ibm-granite", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.531, - "hfopenllm_v2/BBH": 0.5192, - "hfopenllm_v2/MATH Level 5": 0.142, - "hfopenllm_v2/GPQA": 0.3322, - "hfopenllm_v2/MUSR": 0.3901, - "hfopenllm_v2/MMLU-PRO": 0.3457 - } - }, - { - "id": "ibm-granite/granite-3.1-1b-a400m-base", - "name": "granite-3.1-1b-a400m-base", - "developer": "ibm-granite", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2519, - "hfopenllm_v2/BBH": 0.3299, - "hfopenllm_v2/MATH Level 5": 0.0272, - "hfopenllm_v2/GPQA": 0.2517, - "hfopenllm_v2/MUSR": 0.3501, - "hfopenllm_v2/MMLU-PRO": 0.1139 - } - }, - { - "id": "ibm-granite/granite-3.1-1b-a400m-instruct", - "name": "granite-3.1-1b-a400m-instruct", - "developer": "ibm-granite", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4686, - "hfopenllm_v2/BBH": 0.328, - "hfopenllm_v2/MATH Level 5": 0.0453, - "hfopenllm_v2/GPQA": 0.2399, - "hfopenllm_v2/MUSR": 0.3302, - "hfopenllm_v2/MMLU-PRO": 0.1217 - } - }, - { - "id": "ibm-granite/granite-3.1-2b-base", - "name": "granite-3.1-2b-base", - "developer": "ibm-granite", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3522, - "hfopenllm_v2/BBH": 0.4047, - "hfopenllm_v2/MATH Level 5": 0.0566, - "hfopenllm_v2/GPQA": 0.2777, - "hfopenllm_v2/MUSR": 0.3486, - "hfopenllm_v2/MMLU-PRO": 0.2251 - } - }, - { - "id": "ibm-granite/granite-3.1-2b-instruct", - "name": "granite-3.1-2b-instruct", - "developer": "ibm-granite", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6286, - "hfopenllm_v2/BBH": 0.4409, - "hfopenllm_v2/MATH Level 5": 0.1526, - "hfopenllm_v2/GPQA": 0.2894, - "hfopenllm_v2/MUSR": 0.3605, - "hfopenllm_v2/MMLU-PRO": 0.2819 - } - }, - { - "id": "ibm-granite/granite-3.1-3b-a800m-base", - "name": "granite-3.1-3b-a800m-base", - "developer": "ibm-granite", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2996, - "hfopenllm_v2/BBH": 0.3628, - "hfopenllm_v2/MATH Level 5": 0.0453, - "hfopenllm_v2/GPQA": 0.2777, - "hfopenllm_v2/MUSR": 0.3275, - "hfopenllm_v2/MMLU-PRO": 0.1793 - } - }, - { - "id": "ibm-granite/granite-3.1-3b-a800m-instruct", - "name": "granite-3.1-3b-a800m-instruct", - "developer": "ibm-granite", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5516, - "hfopenllm_v2/BBH": 0.4009, - "hfopenllm_v2/MATH Level 5": 0.114, - "hfopenllm_v2/GPQA": 0.2886, - "hfopenllm_v2/MUSR": 0.3486, - "hfopenllm_v2/MMLU-PRO": 0.2148 - } - }, - { - "id": "ibm-granite/granite-3.1-8b-base", - "name": "granite-3.1-8b-base", - "developer": "ibm-granite", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4221, - "hfopenllm_v2/BBH": 0.4777, - "hfopenllm_v2/MATH Level 5": 0.0944, - "hfopenllm_v2/GPQA": 0.3213, - "hfopenllm_v2/MUSR": 0.3922, - "hfopenllm_v2/MMLU-PRO": 0.3232 - } - }, - { - "id": "ibm-granite/granite-3.1-8b-instruct", - "name": "granite-3.1-8b-instruct", - "developer": "ibm-granite", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7208, - "hfopenllm_v2/BBH": 0.5364, - "hfopenllm_v2/MATH Level 5": 0.2198, - "hfopenllm_v2/GPQA": 0.3121, - "hfopenllm_v2/MUSR": 0.4707, - "hfopenllm_v2/MMLU-PRO": 0.3537 - } - }, - { - "id": "ibm-granite/granite-3.2-2b-instruct", - "name": "granite-3.2-2b-instruct", - "developer": "ibm-granite", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6152, - "hfopenllm_v2/BBH": 0.4387, - "hfopenllm_v2/MATH Level 5": 0.1443, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.3646, - "hfopenllm_v2/MMLU-PRO": 0.2783 - } - }, - { - "id": "ibm-granite/granite-3.2-8b-instruct", - "name": "granite-3.2-8b-instruct", - "developer": "ibm-granite", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7275, - "hfopenllm_v2/BBH": 0.5402, - "hfopenllm_v2/MATH Level 5": 0.2379, - "hfopenllm_v2/GPQA": 0.3154, - "hfopenllm_v2/MUSR": 0.4562, - "hfopenllm_v2/MMLU-PRO": 0.3512 - } - }, - { - "id": "ibm-granite/granite-7b-base", - "name": "granite-7b-base", - "developer": "ibm-granite", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2414, - "hfopenllm_v2/BBH": 0.348, - "hfopenllm_v2/MATH Level 5": 0.0159, - "hfopenllm_v2/GPQA": 0.2458, - "hfopenllm_v2/MUSR": 0.3555, - "hfopenllm_v2/MMLU-PRO": 0.1834 - } - }, - { - "id": "ibm-granite/granite-7b-instruct", - "name": "granite-7b-instruct", - "developer": "ibm-granite", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2972, - "hfopenllm_v2/BBH": 0.3723, - "hfopenllm_v2/MATH Level 5": 0.0204, - "hfopenllm_v2/GPQA": 0.2852, - "hfopenllm_v2/MUSR": 0.402, - "hfopenllm_v2/MMLU-PRO": 0.2286 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/ibm.json b/data/developers/ibm.json deleted file mode 100644 index 579fd6f0d99e43bb6118ee8c1737c0617d44c2a6..0000000000000000000000000000000000000000 --- a/data/developers/ibm.json +++ /dev/null @@ -1,226 +0,0 @@ -{ - "developer": "ibm", - "models": [ - { - "id": "ibm/granite-20b-functioncalling-fc", - "name": "Granite-20b-FunctionCalling (FC)", - "developer": "ibm", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 93.0, - "bfcl/bfcl.overall.overall_accuracy": 23.23, - "bfcl/bfcl.overall.total_cost_usd": 5.23, - "bfcl/bfcl.overall.latency_mean_s": 3.2, - "bfcl/bfcl.overall.latency_std_s": 3.43, - "bfcl/bfcl.overall.latency_p95_s": 9.97, - "bfcl/bfcl.non_live.ast_accuracy": 82.35, - "bfcl/bfcl.non_live.simple_ast_accuracy": 72.92, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 91.5, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 83.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 81.5, - "bfcl/bfcl.live.live_accuracy": 58.7, - "bfcl/bfcl.live.live_simple_ast_accuracy": 67.83, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 56.7, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 43.75, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 58.33, - "bfcl/bfcl.multi_turn.accuracy": 5.38, - "bfcl/bfcl.multi_turn.base_accuracy": 9.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 3.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 6.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 3.0, - "bfcl/bfcl.web_search.accuracy": 0.0, - "bfcl/bfcl.web_search.base_accuracy": 0.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 0.0, - "bfcl/bfcl.memory.accuracy": 0.0, - "bfcl/bfcl.memory.kv_accuracy": 0.0, - "bfcl/bfcl.memory.vector_accuracy": 0.0, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 0.0, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 87.5, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 75.13 - } - }, - { - "id": "ibm/granite-3-1-8b-instruct-fc", - "name": "Granite-3.1-8B-Instruct (FC)", - "developer": "ibm", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 81.0, - "bfcl/bfcl.overall.overall_accuracy": 27.1, - "bfcl/bfcl.overall.total_cost_usd": 9.32, - "bfcl/bfcl.overall.latency_mean_s": 13.23, - "bfcl/bfcl.overall.latency_std_s": 31.28, - "bfcl/bfcl.overall.latency_p95_s": 65.19, - "bfcl/bfcl.non_live.ast_accuracy": 78.33, - "bfcl/bfcl.non_live.simple_ast_accuracy": 67.33, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 92.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 84.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 70.0, - "bfcl/bfcl.live.live_accuracy": 60.33, - "bfcl/bfcl.live.live_simple_ast_accuracy": 58.53, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 61.82, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 18.75, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 41.67, - "bfcl/bfcl.multi_turn.accuracy": 7.5, - "bfcl/bfcl.multi_turn.base_accuracy": 11.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 2.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 7.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 9.0, - "bfcl/bfcl.web_search.accuracy": 0.5, - "bfcl/bfcl.web_search.base_accuracy": 1.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 0.0, - "bfcl/bfcl.memory.accuracy": 14.41, - "bfcl/bfcl.memory.kv_accuracy": 9.68, - "bfcl/bfcl.memory.vector_accuracy": 7.1, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 26.45, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 68.75, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 79.98 - } - }, - { - "id": "ibm/granite-3-2-8b-instruct-fc", - "name": "Granite-3.2-8B-Instruct (FC)", - "developer": "ibm", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 83.0, - "bfcl/bfcl.overall.overall_accuracy": 26.87, - "bfcl/bfcl.overall.total_cost_usd": 25.02, - "bfcl/bfcl.overall.latency_mean_s": 36.13, - "bfcl/bfcl.overall.latency_std_s": 81.76, - "bfcl/bfcl.overall.latency_p95_s": 216.28, - "bfcl/bfcl.non_live.ast_accuracy": 79.77, - "bfcl/bfcl.non_live.simple_ast_accuracy": 69.58, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 88.5, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 88.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 72.5, - "bfcl/bfcl.live.live_accuracy": 60.33, - "bfcl/bfcl.live.live_simple_ast_accuracy": 60.47, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 61.16, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 25.0, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 45.83, - "bfcl/bfcl.multi_turn.accuracy": 7.38, - "bfcl/bfcl.multi_turn.base_accuracy": 9.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 3.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 8.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 9.0, - "bfcl/bfcl.web_search.accuracy": 0.5, - "bfcl/bfcl.web_search.base_accuracy": 1.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 0.0, - "bfcl/bfcl.memory.accuracy": 12.47, - "bfcl/bfcl.memory.kv_accuracy": 6.45, - "bfcl/bfcl.memory.vector_accuracy": 9.68, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 21.29, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 75.0, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 80.53 - } - }, - { - "id": "ibm/granite-3.3-8b-instruct", - "name": "IBM Granite 3.3 8B Instruct", - "developer": "ibm", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.463, - "helm_capabilities/MMLU-Pro": 0.343, - "helm_capabilities/GPQA": 0.325, - "helm_capabilities/IFEval": 0.729, - "helm_capabilities/WildBench": 0.741, - "helm_capabilities/Omni-MATH": 0.176 - } - }, - { - "id": "ibm/granite-4-0-350m-fc", - "name": "Granite-4.0-350m (FC)", - "developer": "ibm", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 103.0, - "bfcl/bfcl.overall.overall_accuracy": 18.98, - "bfcl/bfcl.overall.total_cost_usd": 1.44, - "bfcl/bfcl.overall.latency_mean_s": 1.74, - "bfcl/bfcl.overall.latency_std_s": 4.85, - "bfcl/bfcl.overall.latency_p95_s": 3.44, - "bfcl/bfcl.non_live.ast_accuracy": 67.92, - "bfcl/bfcl.non_live.simple_ast_accuracy": 61.67, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 84.5, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 70.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 55.5, - "bfcl/bfcl.live.live_accuracy": 46.11, - "bfcl/bfcl.live.live_simple_ast_accuracy": 61.24, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 42.36, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 68.75, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 33.33, - "bfcl/bfcl.multi_turn.accuracy": 2.5, - "bfcl/bfcl.multi_turn.base_accuracy": 5.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 0.5, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 2.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 2.0, - "bfcl/bfcl.web_search.accuracy": 0.5, - "bfcl/bfcl.web_search.base_accuracy": 0.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 1.0, - "bfcl/bfcl.memory.accuracy": 3.23, - "bfcl/bfcl.memory.kv_accuracy": 1.94, - "bfcl/bfcl.memory.vector_accuracy": 1.29, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 6.45, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 81.25, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 60.84 - } - }, - { - "id": "ibm/granite-4.0-h-small", - "name": "granite-4.0-h-small", - "developer": "ibm", - "evaluator_relationship": null, - "benchmark_scores": { - "global-mmlu-lite/Global MMLU Lite": 0.7503, - "global-mmlu-lite/Culturally Sensitive": 0.7182, - "global-mmlu-lite/Culturally Agnostic": 0.7826, - "global-mmlu-lite/Arabic": 0.7613, - "global-mmlu-lite/English": 0.77, - "global-mmlu-lite/Bengali": 0.7613, - "global-mmlu-lite/German": 0.755, - "global-mmlu-lite/French": 0.7594, - "global-mmlu-lite/Hindi": 0.7575, - "global-mmlu-lite/Indonesian": 0.7614, - "global-mmlu-lite/Italian": 0.7525, - "global-mmlu-lite/Japanese": 0.7406, - "global-mmlu-lite/Korean": 0.7525, - "global-mmlu-lite/Portuguese": 0.757, - "global-mmlu-lite/Spanish": 0.7638, - "global-mmlu-lite/Swahili": 0.7318, - "global-mmlu-lite/Yoruba": 0.6921, - "global-mmlu-lite/Chinese": 0.7475, - "global-mmlu-lite/Burmese": 0.7419 - } - }, - { - "id": "ibm/merlinite-7b", - "name": "merlinite-7b", - "developer": "ibm", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2499, - "hfopenllm_v2/BBH": 0.5007, - "hfopenllm_v2/MATH Level 5": 0.0242, - "hfopenllm_v2/GPQA": 0.297, - "hfopenllm_v2/MUSR": 0.4412, - "hfopenllm_v2/MMLU-PRO": 0.3068 - } - }, - { - "id": "ibm/PowerLM-3b", - "name": "PowerLM-3b", - "developer": "ibm", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3321, - "hfopenllm_v2/BBH": 0.3679, - "hfopenllm_v2/MATH Level 5": 0.0363, - "hfopenllm_v2/GPQA": 0.2752, - "hfopenllm_v2/MUSR": 0.3563, - "hfopenllm_v2/MMLU-PRO": 0.2016 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/icefog72.json b/data/developers/icefog72.json deleted file mode 100644 index 0ea5ae912613ba6349dc7eed95da5f8d3afd5dbe..0000000000000000000000000000000000000000 --- a/data/developers/icefog72.json +++ /dev/null @@ -1,873 +0,0 @@ -{ - "developer": "icefog72", - "models": [ - { - "id": "icefog72/Ice0.15-02.10-RP", - "name": "Ice0.15-02.10-RP", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5343, - "hfopenllm_v2/BBH": 0.4976, - "hfopenllm_v2/MATH Level 5": 0.0574, - "hfopenllm_v2/GPQA": 0.2777, - "hfopenllm_v2/MUSR": 0.432, - "hfopenllm_v2/MMLU-PRO": 0.3066 - } - }, - { - "id": "icefog72/Ice0.16-02.10-RP", - "name": "Ice0.16-02.10-RP", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5069, - "hfopenllm_v2/BBH": 0.4946, - "hfopenllm_v2/MATH Level 5": 0.0589, - "hfopenllm_v2/GPQA": 0.2794, - "hfopenllm_v2/MUSR": 0.4334, - "hfopenllm_v2/MMLU-PRO": 0.3068 - } - }, - { - "id": "icefog72/Ice0.17-03.10-RP", - "name": "Ice0.17-03.10-RP", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5124, - "hfopenllm_v2/BBH": 0.5007, - "hfopenllm_v2/MATH Level 5": 0.0612, - "hfopenllm_v2/GPQA": 0.2819, - "hfopenllm_v2/MUSR": 0.4334, - "hfopenllm_v2/MMLU-PRO": 0.3085 - } - }, - { - "id": "icefog72/Ice0.27-06.11-RP", - "name": "Ice0.27-06.11-RP", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4918, - "hfopenllm_v2/BBH": 0.5112, - "hfopenllm_v2/MATH Level 5": 0.0566, - "hfopenllm_v2/GPQA": 0.3121, - "hfopenllm_v2/MUSR": 0.4328, - "hfopenllm_v2/MMLU-PRO": 0.3154 - } - }, - { - "id": "icefog72/Ice0.29-06.11-RP", - "name": "Ice0.29-06.11-RP", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4861, - "hfopenllm_v2/BBH": 0.5088, - "hfopenllm_v2/MATH Level 5": 0.0566, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.4459, - "hfopenllm_v2/MMLU-PRO": 0.3093 - } - }, - { - "id": "icefog72/Ice0.31-08.11-RP", - "name": "Ice0.31-08.11-RP", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5146, - "hfopenllm_v2/BBH": 0.5032, - "hfopenllm_v2/MATH Level 5": 0.0612, - "hfopenllm_v2/GPQA": 0.3079, - "hfopenllm_v2/MUSR": 0.4277, - "hfopenllm_v2/MMLU-PRO": 0.3131 - } - }, - { - "id": "icefog72/Ice0.32-10.11-RP", - "name": "Ice0.32-10.11-RP", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4915, - "hfopenllm_v2/BBH": 0.5048, - "hfopenllm_v2/MATH Level 5": 0.0514, - "hfopenllm_v2/GPQA": 0.3121, - "hfopenllm_v2/MUSR": 0.4382, - "hfopenllm_v2/MMLU-PRO": 0.31 - } - }, - { - "id": "icefog72/Ice0.34b-14.11-RP", - "name": "Ice0.34b-14.11-RP", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4762, - "hfopenllm_v2/BBH": 0.5067, - "hfopenllm_v2/MATH Level 5": 0.065, - "hfopenllm_v2/GPQA": 0.3096, - "hfopenllm_v2/MUSR": 0.442, - "hfopenllm_v2/MMLU-PRO": 0.3125 - } - }, - { - "id": "icefog72/Ice0.34n-14.11-RP", - "name": "Ice0.34n-14.11-RP", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4787, - "hfopenllm_v2/BBH": 0.5091, - "hfopenllm_v2/MATH Level 5": 0.0725, - "hfopenllm_v2/GPQA": 0.3138, - "hfopenllm_v2/MUSR": 0.438, - "hfopenllm_v2/MMLU-PRO": 0.3124 - } - }, - { - "id": "icefog72/Ice0.37-18.11-RP", - "name": "Ice0.37-18.11-RP", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4972, - "hfopenllm_v2/BBH": 0.5084, - "hfopenllm_v2/MATH Level 5": 0.0642, - "hfopenllm_v2/GPQA": 0.3121, - "hfopenllm_v2/MUSR": 0.4339, - "hfopenllm_v2/MMLU-PRO": 0.3143 - } - }, - { - "id": "icefog72/Ice0.38-19.11-RP", - "name": "Ice0.38-19.11-RP", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4403, - "hfopenllm_v2/BBH": 0.5101, - "hfopenllm_v2/MATH Level 5": 0.0551, - "hfopenllm_v2/GPQA": 0.3045, - "hfopenllm_v2/MUSR": 0.4367, - "hfopenllm_v2/MMLU-PRO": 0.314 - } - }, - { - "id": "icefog72/Ice0.39-19.11-RP", - "name": "Ice0.39-19.11-RP", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4757, - "hfopenllm_v2/BBH": 0.5093, - "hfopenllm_v2/MATH Level 5": 0.0498, - "hfopenllm_v2/GPQA": 0.3104, - "hfopenllm_v2/MUSR": 0.4341, - "hfopenllm_v2/MMLU-PRO": 0.3127 - } - }, - { - "id": "icefog72/Ice0.40-20.11-RP", - "name": "Ice0.40-20.11-RP", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4763, - "hfopenllm_v2/BBH": 0.5093, - "hfopenllm_v2/MATH Level 5": 0.0642, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.4446, - "hfopenllm_v2/MMLU-PRO": 0.3099 - } - }, - { - "id": "icefog72/Ice0.41-22.11-RP", - "name": "Ice0.41-22.11-RP", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.462, - "hfopenllm_v2/BBH": 0.4723, - "hfopenllm_v2/MATH Level 5": 0.031, - "hfopenllm_v2/GPQA": 0.2869, - "hfopenllm_v2/MUSR": 0.456, - "hfopenllm_v2/MMLU-PRO": 0.2618 - } - }, - { - "id": "icefog72/Ice0.50-16.01-RP", - "name": "Ice0.50-16.01-RP", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4385, - "hfopenllm_v2/BBH": 0.498, - "hfopenllm_v2/MATH Level 5": 0.0468, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.4381, - "hfopenllm_v2/MMLU-PRO": 0.3069 - } - }, - { - "id": "icefog72/Ice0.50.1-16.01-RP", - "name": "Ice0.50.1-16.01-RP", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4829, - "hfopenllm_v2/BBH": 0.5107, - "hfopenllm_v2/MATH Level 5": 0.0612, - "hfopenllm_v2/GPQA": 0.3096, - "hfopenllm_v2/MUSR": 0.4327, - "hfopenllm_v2/MMLU-PRO": 0.3132 - } - }, - { - "id": "icefog72/Ice0.51-16.01-RP", - "name": "Ice0.51-16.01-RP", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4431, - "hfopenllm_v2/BBH": 0.5044, - "hfopenllm_v2/MATH Level 5": 0.0514, - "hfopenllm_v2/GPQA": 0.3045, - "hfopenllm_v2/MUSR": 0.4437, - "hfopenllm_v2/MMLU-PRO": 0.306 - } - }, - { - "id": "icefog72/Ice0.51.1-16.01-RP", - "name": "Ice0.51.1-16.01-RP", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4573, - "hfopenllm_v2/BBH": 0.5121, - "hfopenllm_v2/MATH Level 5": 0.0642, - "hfopenllm_v2/GPQA": 0.3062, - "hfopenllm_v2/MUSR": 0.4394, - "hfopenllm_v2/MMLU-PRO": 0.3104 - } - }, - { - "id": "icefog72/Ice0.52-16.01-RP", - "name": "Ice0.52-16.01-RP", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4503, - "hfopenllm_v2/BBH": 0.5047, - "hfopenllm_v2/MATH Level 5": 0.0506, - "hfopenllm_v2/GPQA": 0.3087, - "hfopenllm_v2/MUSR": 0.4396, - "hfopenllm_v2/MMLU-PRO": 0.308 - } - }, - { - "id": "icefog72/Ice0.52.1-16.01-RP", - "name": "Ice0.52.1-16.01-RP", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4549, - "hfopenllm_v2/BBH": 0.5106, - "hfopenllm_v2/MATH Level 5": 0.0627, - "hfopenllm_v2/GPQA": 0.3037, - "hfopenllm_v2/MUSR": 0.4394, - "hfopenllm_v2/MMLU-PRO": 0.3105 - } - }, - { - "id": "icefog72/Ice0.53-16.01-RP", - "name": "Ice0.53-16.01-RP", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4741, - "hfopenllm_v2/BBH": 0.5102, - "hfopenllm_v2/MATH Level 5": 0.0634, - "hfopenllm_v2/GPQA": 0.3087, - "hfopenllm_v2/MUSR": 0.4327, - "hfopenllm_v2/MMLU-PRO": 0.313 - } - }, - { - "id": "icefog72/Ice0.54-17.01-RP", - "name": "Ice0.54-17.01-RP", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4379, - "hfopenllm_v2/BBH": 0.4853, - "hfopenllm_v2/MATH Level 5": 0.0408, - "hfopenllm_v2/GPQA": 0.3096, - "hfopenllm_v2/MUSR": 0.4874, - "hfopenllm_v2/MMLU-PRO": 0.2326 - } - }, - { - "id": "icefog72/Ice0.55-17.01-RP", - "name": "Ice0.55-17.01-RP", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4961, - "hfopenllm_v2/BBH": 0.5077, - "hfopenllm_v2/MATH Level 5": 0.0604, - "hfopenllm_v2/GPQA": 0.2869, - "hfopenllm_v2/MUSR": 0.4725, - "hfopenllm_v2/MMLU-PRO": 0.2658 - } - }, - { - "id": "icefog72/Ice0.57-17.01-RP", - "name": "Ice0.57-17.01-RP", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5152, - "hfopenllm_v2/BBH": 0.5064, - "hfopenllm_v2/MATH Level 5": 0.0514, - "hfopenllm_v2/GPQA": 0.297, - "hfopenllm_v2/MUSR": 0.4686, - "hfopenllm_v2/MMLU-PRO": 0.2651 - } - }, - { - "id": "icefog72/Ice0.60-18.01-RP", - "name": "Ice0.60-18.01-RP", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5374, - "hfopenllm_v2/BBH": 0.5094, - "hfopenllm_v2/MATH Level 5": 0.0536, - "hfopenllm_v2/GPQA": 0.3045, - "hfopenllm_v2/MUSR": 0.467, - "hfopenllm_v2/MMLU-PRO": 0.2837 - } - }, - { - "id": "icefog72/Ice0.60.1-18.01-RP", - "name": "Ice0.60.1-18.01-RP", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5188, - "hfopenllm_v2/BBH": 0.512, - "hfopenllm_v2/MATH Level 5": 0.0461, - "hfopenllm_v2/GPQA": 0.302, - "hfopenllm_v2/MUSR": 0.4498, - "hfopenllm_v2/MMLU-PRO": 0.2914 - } - }, - { - "id": "icefog72/Ice0.61-18.01-RP", - "name": "Ice0.61-18.01-RP", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5441, - "hfopenllm_v2/BBH": 0.5105, - "hfopenllm_v2/MATH Level 5": 0.0468, - "hfopenllm_v2/GPQA": 0.2987, - "hfopenllm_v2/MUSR": 0.4697, - "hfopenllm_v2/MMLU-PRO": 0.2709 - } - }, - { - "id": "icefog72/Ice0.62-18.01-RP", - "name": "Ice0.62-18.01-RP", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5367, - "hfopenllm_v2/BBH": 0.5103, - "hfopenllm_v2/MATH Level 5": 0.0574, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.4538, - "hfopenllm_v2/MMLU-PRO": 0.2877 - } - }, - { - "id": "icefog72/Ice0.62.1-24.01-RP", - "name": "Ice0.62.1-24.01-RP", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5182, - "hfopenllm_v2/BBH": 0.5109, - "hfopenllm_v2/MATH Level 5": 0.0559, - "hfopenllm_v2/GPQA": 0.3003, - "hfopenllm_v2/MUSR": 0.4551, - "hfopenllm_v2/MMLU-PRO": 0.2871 - } - }, - { - "id": "icefog72/Ice0.64-24.01-RP", - "name": "Ice0.64-24.01-RP", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5441, - "hfopenllm_v2/BBH": 0.506, - "hfopenllm_v2/MATH Level 5": 0.0627, - "hfopenllm_v2/GPQA": 0.3037, - "hfopenllm_v2/MUSR": 0.462, - "hfopenllm_v2/MMLU-PRO": 0.2933 - } - }, - { - "id": "icefog72/Ice0.64.1-24.01-RP", - "name": "Ice0.64.1-24.01-RP", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5447, - "hfopenllm_v2/BBH": 0.506, - "hfopenllm_v2/MATH Level 5": 0.0627, - "hfopenllm_v2/GPQA": 0.3037, - "hfopenllm_v2/MUSR": 0.462, - "hfopenllm_v2/MMLU-PRO": 0.2933 - } - }, - { - "id": "icefog72/Ice0.65-25.01-RP", - "name": "Ice0.65-25.01-RP", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5029, - "hfopenllm_v2/BBH": 0.5096, - "hfopenllm_v2/MATH Level 5": 0.065, - "hfopenllm_v2/GPQA": 0.3045, - "hfopenllm_v2/MUSR": 0.434, - "hfopenllm_v2/MMLU-PRO": 0.2997 - } - }, - { - "id": "icefog72/Ice0.66-25.01-RP", - "name": "Ice0.66-25.01-RP", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5325, - "hfopenllm_v2/BBH": 0.5129, - "hfopenllm_v2/MATH Level 5": 0.0604, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.4434, - "hfopenllm_v2/MMLU-PRO": 0.3039 - } - }, - { - "id": "icefog72/Ice0.67-25.01-RP", - "name": "Ice0.67-25.01-RP", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5361, - "hfopenllm_v2/BBH": 0.5113, - "hfopenllm_v2/MATH Level 5": 0.0748, - "hfopenllm_v2/GPQA": 0.2852, - "hfopenllm_v2/MUSR": 0.4279, - "hfopenllm_v2/MMLU-PRO": 0.3097 - } - }, - { - "id": "icefog72/Ice0.68-25.01-RP", - "name": "Ice0.68-25.01-RP", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5514, - "hfopenllm_v2/BBH": 0.513, - "hfopenllm_v2/MATH Level 5": 0.0725, - "hfopenllm_v2/GPQA": 0.3104, - "hfopenllm_v2/MUSR": 0.4446, - "hfopenllm_v2/MMLU-PRO": 0.3012 - } - }, - { - "id": "icefog72/Ice0.69-25.01-RP", - "name": "Ice0.69-25.01-RP", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5438, - "hfopenllm_v2/BBH": 0.5098, - "hfopenllm_v2/MATH Level 5": 0.0566, - "hfopenllm_v2/GPQA": 0.3129, - "hfopenllm_v2/MUSR": 0.4486, - "hfopenllm_v2/MMLU-PRO": 0.2965 - } - }, - { - "id": "icefog72/Ice0.7-29.09-RP", - "name": "Ice0.7-29.09-RP", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5176, - "hfopenllm_v2/BBH": 0.5048, - "hfopenllm_v2/MATH Level 5": 0.0665, - "hfopenllm_v2/GPQA": 0.2878, - "hfopenllm_v2/MUSR": 0.4238, - "hfopenllm_v2/MMLU-PRO": 0.3127 - } - }, - { - "id": "icefog72/Ice0.70-25.01-RP", - "name": "Ice0.70-25.01-RP", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5498, - "hfopenllm_v2/BBH": 0.5136, - "hfopenllm_v2/MATH Level 5": 0.0597, - "hfopenllm_v2/GPQA": 0.3079, - "hfopenllm_v2/MUSR": 0.4512, - "hfopenllm_v2/MMLU-PRO": 0.2996 - } - }, - { - "id": "icefog72/Ice0.70.1-01.02-RP", - "name": "Ice0.70.1-01.02-RP", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.507, - "hfopenllm_v2/BBH": 0.506, - "hfopenllm_v2/MATH Level 5": 0.034, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.4599, - "hfopenllm_v2/MMLU-PRO": 0.2749 - } - }, - { - "id": "icefog72/Ice0.73-01.02-RP", - "name": "Ice0.73-01.02-RP", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5292, - "hfopenllm_v2/BBH": 0.5103, - "hfopenllm_v2/MATH Level 5": 0.0385, - "hfopenllm_v2/GPQA": 0.2911, - "hfopenllm_v2/MUSR": 0.4664, - "hfopenllm_v2/MMLU-PRO": 0.2702 - } - }, - { - "id": "icefog72/Ice0.74-02.02-RP", - "name": "Ice0.74-02.02-RP", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2935, - "hfopenllm_v2/BBH": 0.4646, - "hfopenllm_v2/MATH Level 5": 0.0015, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.428, - "hfopenllm_v2/MMLU-PRO": 0.2143 - } - }, - { - "id": "icefog72/Ice0.76-02.02-RP", - "name": "Ice0.76-02.02-RP", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4529, - "hfopenllm_v2/BBH": 0.5086, - "hfopenllm_v2/MATH Level 5": 0.0144, - "hfopenllm_v2/GPQA": 0.2869, - "hfopenllm_v2/MUSR": 0.4362, - "hfopenllm_v2/MMLU-PRO": 0.2652 - } - }, - { - "id": "icefog72/Ice0.77-02.02-RP", - "name": "Ice0.77-02.02-RP", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.531, - "hfopenllm_v2/BBH": 0.5109, - "hfopenllm_v2/MATH Level 5": 0.0393, - "hfopenllm_v2/GPQA": 0.2852, - "hfopenllm_v2/MUSR": 0.4765, - "hfopenllm_v2/MMLU-PRO": 0.2999 - } - }, - { - "id": "icefog72/Ice0.78-02.02-RP", - "name": "Ice0.78-02.02-RP", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4053, - "hfopenllm_v2/BBH": 0.5002, - "hfopenllm_v2/MATH Level 5": 0.0438, - "hfopenllm_v2/GPQA": 0.2936, - "hfopenllm_v2/MUSR": 0.4686, - "hfopenllm_v2/MMLU-PRO": 0.2955 - } - }, - { - "id": "icefog72/Ice0.80-03.02-RP", - "name": "Ice0.80-03.02-RP", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5516, - "hfopenllm_v2/BBH": 0.5098, - "hfopenllm_v2/MATH Level 5": 0.0559, - "hfopenllm_v2/GPQA": 0.2785, - "hfopenllm_v2/MUSR": 0.4923, - "hfopenllm_v2/MMLU-PRO": 0.2912 - } - }, - { - "id": "icefog72/IceCocoaRP-7b", - "name": "IceCocoaRP-7b", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4962, - "hfopenllm_v2/BBH": 0.4938, - "hfopenllm_v2/MATH Level 5": 0.0574, - "hfopenllm_v2/GPQA": 0.2953, - "hfopenllm_v2/MUSR": 0.4198, - "hfopenllm_v2/MMLU-PRO": 0.3098 - } - }, - { - "id": "icefog72/IceCoffeeRP-7b", - "name": "IceCoffeeRP-7b", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4959, - "hfopenllm_v2/BBH": 0.4889, - "hfopenllm_v2/MATH Level 5": 0.0544, - "hfopenllm_v2/GPQA": 0.2852, - "hfopenllm_v2/MUSR": 0.416, - "hfopenllm_v2/MMLU-PRO": 0.2975 - } - }, - { - "id": "icefog72/IceDrinkByFrankensteinV3RP", - "name": "IceDrinkByFrankensteinV3RP", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4975, - "hfopenllm_v2/BBH": 0.4833, - "hfopenllm_v2/MATH Level 5": 0.0506, - "hfopenllm_v2/GPQA": 0.2617, - "hfopenllm_v2/MUSR": 0.4253, - "hfopenllm_v2/MMLU-PRO": 0.2927 - } - }, - { - "id": "icefog72/IceDrinkNameGoesHereRP-7b-Model_Stock", - "name": "IceDrinkNameGoesHereRP-7b-Model_Stock", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4968, - "hfopenllm_v2/BBH": 0.4658, - "hfopenllm_v2/MATH Level 5": 0.0408, - "hfopenllm_v2/GPQA": 0.2685, - "hfopenllm_v2/MUSR": 0.4067, - "hfopenllm_v2/MMLU-PRO": 0.2817 - } - }, - { - "id": "icefog72/IceDrinkNameNotFoundRP-7b-Model_Stock", - "name": "IceDrinkNameNotFoundRP-7b-Model_Stock", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.513, - "hfopenllm_v2/BBH": 0.5026, - "hfopenllm_v2/MATH Level 5": 0.0604, - "hfopenllm_v2/GPQA": 0.2777, - "hfopenllm_v2/MUSR": 0.4372, - "hfopenllm_v2/MMLU-PRO": 0.3064 - } - }, - { - "id": "icefog72/IceDrunkCherryRP-7b", - "name": "IceDrunkCherryRP-7b", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4898, - "hfopenllm_v2/BBH": 0.4847, - "hfopenllm_v2/MATH Level 5": 0.0612, - "hfopenllm_v2/GPQA": 0.2768, - "hfopenllm_v2/MUSR": 0.4292, - "hfopenllm_v2/MMLU-PRO": 0.3009 - } - }, - { - "id": "icefog72/IceDrunkenCherryRP-7b", - "name": "IceDrunkenCherryRP-7b", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4763, - "hfopenllm_v2/BBH": 0.5093, - "hfopenllm_v2/MATH Level 5": 0.0642, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.4446, - "hfopenllm_v2/MMLU-PRO": 0.3099 - } - }, - { - "id": "icefog72/IceEspressoRPv2-7b", - "name": "IceEspressoRPv2-7b", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4977, - "hfopenllm_v2/BBH": 0.5055, - "hfopenllm_v2/MATH Level 5": 0.0619, - "hfopenllm_v2/GPQA": 0.2894, - "hfopenllm_v2/MUSR": 0.4331, - "hfopenllm_v2/MMLU-PRO": 0.3061 - } - }, - { - "id": "icefog72/IceLemonTeaRP-32k-7b", - "name": "IceLemonTeaRP-32k-7b", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5212, - "hfopenllm_v2/BBH": 0.4997, - "hfopenllm_v2/MATH Level 5": 0.0544, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.429, - "hfopenllm_v2/MMLU-PRO": 0.3068 - } - }, - { - "id": "icefog72/IceMartiniRP-7b", - "name": "IceMartiniRP-7b", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5045, - "hfopenllm_v2/BBH": 0.4972, - "hfopenllm_v2/MATH Level 5": 0.0665, - "hfopenllm_v2/GPQA": 0.2794, - "hfopenllm_v2/MUSR": 0.4345, - "hfopenllm_v2/MMLU-PRO": 0.3073 - } - }, - { - "id": "icefog72/IceNalyvkaRP-7b", - "name": "IceNalyvkaRP-7b", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5498, - "hfopenllm_v2/BBH": 0.5136, - "hfopenllm_v2/MATH Level 5": 0.0597, - "hfopenllm_v2/GPQA": 0.3079, - "hfopenllm_v2/MUSR": 0.4512, - "hfopenllm_v2/MMLU-PRO": 0.2996 - } - }, - { - "id": "icefog72/IceSakeRP-7b", - "name": "IceSakeRP-7b", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5228, - "hfopenllm_v2/BBH": 0.5119, - "hfopenllm_v2/MATH Level 5": 0.0634, - "hfopenllm_v2/GPQA": 0.2852, - "hfopenllm_v2/MUSR": 0.413, - "hfopenllm_v2/MMLU-PRO": 0.3177 - } - }, - { - "id": "icefog72/IceSakeV4RP-7b", - "name": "IceSakeV4RP-7b", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4634, - "hfopenllm_v2/BBH": 0.493, - "hfopenllm_v2/MATH Level 5": 0.0559, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.4082, - "hfopenllm_v2/MMLU-PRO": 0.3103 - } - }, - { - "id": "icefog72/IceSakeV6RP-7b", - "name": "IceSakeV6RP-7b", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5033, - "hfopenllm_v2/BBH": 0.4976, - "hfopenllm_v2/MATH Level 5": 0.0619, - "hfopenllm_v2/GPQA": 0.2911, - "hfopenllm_v2/MUSR": 0.42, - "hfopenllm_v2/MMLU-PRO": 0.3093 - } - }, - { - "id": "icefog72/IceSakeV8RP-7b", - "name": "IceSakeV8RP-7b", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6086, - "hfopenllm_v2/BBH": 0.4885, - "hfopenllm_v2/MATH Level 5": 0.0597, - "hfopenllm_v2/GPQA": 0.276, - "hfopenllm_v2/MUSR": 0.3993, - "hfopenllm_v2/MMLU-PRO": 0.301 - } - }, - { - "id": "icefog72/IceTea21EnergyDrinkRPV13-DPOv3", - "name": "IceTea21EnergyDrinkRPV13-DPOv3", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5263, - "hfopenllm_v2/BBH": 0.502, - "hfopenllm_v2/MATH Level 5": 0.0582, - "hfopenllm_v2/GPQA": 0.2836, - "hfopenllm_v2/MUSR": 0.4372, - "hfopenllm_v2/MMLU-PRO": 0.3056 - } - }, - { - "id": "icefog72/IceTea21EnergyDrinkRPV13-DPOv3.5", - "name": "IceTea21EnergyDrinkRPV13-DPOv3.5", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4871, - "hfopenllm_v2/BBH": 0.44, - "hfopenllm_v2/MATH Level 5": 0.0363, - "hfopenllm_v2/GPQA": 0.2844, - "hfopenllm_v2/MUSR": 0.3964, - "hfopenllm_v2/MMLU-PRO": 0.2498 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/idea-ccnl.json b/data/developers/idea-ccnl.json deleted file mode 100644 index 6f1c086643897d19e96a9d5282c4cb8fb5a80256..0000000000000000000000000000000000000000 --- a/data/developers/idea-ccnl.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "IDEA-CCNL", - "models": [ - { - "id": "IDEA-CCNL/Ziya-LLaMA-13B-v1", - "name": "Ziya-LLaMA-13B-v1", - "developer": "IDEA-CCNL", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1697, - "hfopenllm_v2/BBH": 0.2877, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2492, - "hfopenllm_v2/MUSR": 0.3751, - "hfopenllm_v2/MMLU-PRO": 0.1101 - } - }, - { - "id": "IDEA-CCNL/Ziya-LLaMA-7B-Reward", - "name": "IDEA-CCNL/Ziya-LLaMA-7B-Reward", - "developer": "IDEA-CCNL", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6378, - "reward-bench/Chat": 0.8687, - "reward-bench/Chat Hard": 0.4605, - "reward-bench/Safety": 0.6405, - "reward-bench/Reasoning": 0.5775, - "reward-bench/Prior Sets (0.5 weight)": 0.6461 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/ifable.json b/data/developers/ifable.json deleted file mode 100644 index 601fc025187d79b202e9e36eb8bfbcdb29f13555..0000000000000000000000000000000000000000 --- a/data/developers/ifable.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "ifable", - "models": [ - { - "id": "ifable/gemma-2-Ifable-9B", - "name": "gemma-2-Ifable-9B", - "developer": "ifable", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2984, - "hfopenllm_v2/BBH": 0.5866, - "hfopenllm_v2/MATH Level 5": 0.1397, - "hfopenllm_v2/GPQA": 0.3414, - "hfopenllm_v2/MUSR": 0.4053, - "hfopenllm_v2/MMLU-PRO": 0.4226 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/ifaz.json b/data/developers/ifaz.json deleted file mode 100644 index 9664b16cfdd6fd46ee11e5e776b40f03fd77b55f..0000000000000000000000000000000000000000 --- a/data/developers/ifaz.json +++ /dev/null @@ -1,117 +0,0 @@ -{ - "developer": "iFaz", - "models": [ - { - "id": "iFaz/llama31_8B_en_emo_v4", - "name": "llama31_8B_en_emo_v4", - "developer": "iFaz", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3043, - "hfopenllm_v2/BBH": 0.4916, - "hfopenllm_v2/MATH Level 5": 0.0884, - "hfopenllm_v2/GPQA": 0.297, - "hfopenllm_v2/MUSR": 0.3643, - "hfopenllm_v2/MMLU-PRO": 0.3049 - } - }, - { - "id": "iFaz/llama32_1B_en_emo_v1", - "name": "llama32_1B_en_emo_v1", - "developer": "iFaz", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4408, - "hfopenllm_v2/BBH": 0.338, - "hfopenllm_v2/MATH Level 5": 0.0378, - "hfopenllm_v2/GPQA": 0.25, - "hfopenllm_v2/MUSR": 0.3489, - "hfopenllm_v2/MMLU-PRO": 0.1761 - } - }, - { - "id": "iFaz/llama32_3B_en_emo_1000_stp", - "name": "llama32_3B_en_emo_1000_stp", - "developer": "iFaz", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7295, - "hfopenllm_v2/BBH": 0.4522, - "hfopenllm_v2/MATH Level 5": 0.1465, - "hfopenllm_v2/GPQA": 0.2777, - "hfopenllm_v2/MUSR": 0.3621, - "hfopenllm_v2/MMLU-PRO": 0.3123 - } - }, - { - "id": "iFaz/llama32_3B_en_emo_2000_stp", - "name": "llama32_3B_en_emo_2000_stp", - "developer": "iFaz", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7369, - "hfopenllm_v2/BBH": 0.4535, - "hfopenllm_v2/MATH Level 5": 0.1533, - "hfopenllm_v2/GPQA": 0.2836, - "hfopenllm_v2/MUSR": 0.3527, - "hfopenllm_v2/MMLU-PRO": 0.3098 - } - }, - { - "id": "iFaz/llama32_3B_en_emo_300_stp", - "name": "llama32_3B_en_emo_300_stp", - "developer": "iFaz", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7256, - "hfopenllm_v2/BBH": 0.4505, - "hfopenllm_v2/MATH Level 5": 0.1601, - "hfopenllm_v2/GPQA": 0.2743, - "hfopenllm_v2/MUSR": 0.3621, - "hfopenllm_v2/MMLU-PRO": 0.3148 - } - }, - { - "id": "iFaz/llama32_3B_en_emo_5000_stp", - "name": "llama32_3B_en_emo_5000_stp", - "developer": "iFaz", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.71, - "hfopenllm_v2/BBH": 0.4568, - "hfopenllm_v2/MATH Level 5": 0.1292, - "hfopenllm_v2/GPQA": 0.2794, - "hfopenllm_v2/MUSR": 0.3446, - "hfopenllm_v2/MMLU-PRO": 0.3067 - } - }, - { - "id": "iFaz/llama32_3B_en_emo_v2", - "name": "llama32_3B_en_emo_v2", - "developer": "iFaz", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5454, - "hfopenllm_v2/BBH": 0.4284, - "hfopenllm_v2/MATH Level 5": 0.1088, - "hfopenllm_v2/GPQA": 0.2676, - "hfopenllm_v2/MUSR": 0.3482, - "hfopenllm_v2/MMLU-PRO": 0.3004 - } - }, - { - "id": "iFaz/llama32_3B_en_emo_v3", - "name": "llama32_3B_en_emo_v3", - "developer": "iFaz", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5759, - "hfopenllm_v2/BBH": 0.4301, - "hfopenllm_v2/MATH Level 5": 0.068, - "hfopenllm_v2/GPQA": 0.2676, - "hfopenllm_v2/MUSR": 0.3553, - "hfopenllm_v2/MMLU-PRO": 0.271 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/ilsp.json b/data/developers/ilsp.json deleted file mode 100644 index 7f7f270df6cb334f203b0a9cc17b7bbc4d4faef9..0000000000000000000000000000000000000000 --- a/data/developers/ilsp.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "ilsp", - "models": [ - { - "id": "ilsp/Llama-Krikri-8B-Instruct", - "name": "Llama-Krikri-8B-Instruct", - "developer": "ilsp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6079, - "hfopenllm_v2/BBH": 0.5047, - "hfopenllm_v2/MATH Level 5": 0.1178, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.408, - "hfopenllm_v2/MMLU-PRO": 0.3313 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/ilyagusev.json b/data/developers/ilyagusev.json deleted file mode 100644 index 5fa31a756a7150f9e665c6180fc5857924738f82..0000000000000000000000000000000000000000 --- a/data/developers/ilyagusev.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "IlyaGusev", - "models": [ - { - "id": "IlyaGusev/gemma-2-2b-it-abliterated", - "name": "gemma-2-2b-it-abliterated", - "developer": "IlyaGusev", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5331, - "hfopenllm_v2/BBH": 0.4119, - "hfopenllm_v2/MATH Level 5": 0.0612, - "hfopenllm_v2/GPQA": 0.2651, - "hfopenllm_v2/MUSR": 0.3782, - "hfopenllm_v2/MMLU-PRO": 0.2538 - } - }, - { - "id": "IlyaGusev/gemma-2-9b-it-abliterated", - "name": "gemma-2-9b-it-abliterated", - "developer": "IlyaGusev", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7473, - "hfopenllm_v2/BBH": 0.5906, - "hfopenllm_v2/MATH Level 5": 0.1775, - "hfopenllm_v2/GPQA": 0.3456, - "hfopenllm_v2/MUSR": 0.4034, - "hfopenllm_v2/MMLU-PRO": 0.3915 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/infinirc.json b/data/developers/infinirc.json deleted file mode 100644 index f08721bade7b8a610483249e7a508d48a51c77e6..0000000000000000000000000000000000000000 --- a/data/developers/infinirc.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "Infinirc", - "models": [ - { - "id": "Infinirc/Infinirc-Llama3-8B-2G-Release-v1.0", - "name": "Infinirc-Llama3-8B-2G-Release-v1.0", - "developer": "Infinirc", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2024, - "hfopenllm_v2/BBH": 0.4351, - "hfopenllm_v2/MATH Level 5": 0.0166, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.4609, - "hfopenllm_v2/MMLU-PRO": 0.216 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/inflatebot.json b/data/developers/inflatebot.json deleted file mode 100644 index eabe2eefaddd0757e69dc56209cfd6c076a5ca10..0000000000000000000000000000000000000000 --- a/data/developers/inflatebot.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "inflatebot", - "models": [ - { - "id": "inflatebot/MN-12B-Mag-Mell-R1", - "name": "MN-12B-Mag-Mell-R1", - "developer": "inflatebot", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4613, - "hfopenllm_v2/BBH": 0.5304, - "hfopenllm_v2/MATH Level 5": 0.1299, - "hfopenllm_v2/GPQA": 0.3163, - "hfopenllm_v2/MUSR": 0.4002, - "hfopenllm_v2/MMLU-PRO": 0.3438 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/infly.json b/data/developers/infly.json deleted file mode 100644 index fe3f0dc6f7a4b2c08dd2895544fd05de4f16df3c..0000000000000000000000000000000000000000 --- a/data/developers/infly.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "developer": "infly", - "models": [ - { - "id": "infly/INF-ORM-Llama3.1-70B", - "name": "infly/INF-ORM-Llama3.1-70B", - "developer": "infly", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7648, - "reward-bench/Chat": 0.9665, - "reward-bench/Chat Hard": 0.9101, - "reward-bench/Safety": 0.9644, - "reward-bench/Reasoning": 0.9912, - "reward-bench/Factuality": 0.7411, - "reward-bench/Precise IF": 0.4188, - "reward-bench/Math": 0.6995, - "reward-bench/Focus": 0.903, - "reward-bench/Ties": 0.8622 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/informatiker.json b/data/developers/informatiker.json deleted file mode 100644 index 431024c422c6128cfc084f87f0fbeb88ae3dc94b..0000000000000000000000000000000000000000 --- a/data/developers/informatiker.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "informatiker", - "models": [ - { - "id": "informatiker/Qwen2-7B-Instruct-abliterated", - "name": "Qwen2-7B-Instruct-abliterated", - "developer": "informatiker", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5822, - "hfopenllm_v2/BBH": 0.5534, - "hfopenllm_v2/MATH Level 5": 0.2636, - "hfopenllm_v2/GPQA": 0.3012, - "hfopenllm_v2/MUSR": 0.3888, - "hfopenllm_v2/MMLU-PRO": 0.3873 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/insait-institute.json b/data/developers/insait-institute.json deleted file mode 100644 index 3eb93e42a7e5a4cb23315af5f5a331cdf6b4feca..0000000000000000000000000000000000000000 --- a/data/developers/insait-institute.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "INSAIT-Institute", - "models": [ - { - "id": "INSAIT-Institute/BgGPT-Gemma-2-27B-IT-v1.0", - "name": "BgGPT-Gemma-2-27B-IT-v1.0", - "developer": "INSAIT-Institute", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0, - "hfopenllm_v2/BBH": 0.2912, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3575, - "hfopenllm_v2/MMLU-PRO": 0.1167 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/insightfactory.json b/data/developers/insightfactory.json deleted file mode 100644 index 19234d170aa685994456d0cd6208d08d15380df9..0000000000000000000000000000000000000000 --- a/data/developers/insightfactory.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "insightfactory", - "models": [ - { - "id": "insightfactory/Llama-3.2-3B-Instruct-unsloth-bnb-4bitlora_model", - "name": "Llama-3.2-3B-Instruct-unsloth-bnb-4bitlora_model", - "developer": "insightfactory", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4588, - "hfopenllm_v2/BBH": 0.4146, - "hfopenllm_v2/MATH Level 5": 0.105, - "hfopenllm_v2/GPQA": 0.2718, - "hfopenllm_v2/MUSR": 0.3499, - "hfopenllm_v2/MMLU-PRO": 0.296 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/instruction-pretrain.json b/data/developers/instruction-pretrain.json deleted file mode 100644 index 9af8328ba70698314b25fb0bbf55b956185c2412..0000000000000000000000000000000000000000 --- a/data/developers/instruction-pretrain.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "instruction-pretrain", - "models": [ - { - "id": "instruction-pretrain/InstructLM-500M", - "name": "InstructLM-500M", - "developer": "instruction-pretrain", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1028, - "hfopenllm_v2/BBH": 0.2941, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2567, - "hfopenllm_v2/MUSR": 0.3528, - "hfopenllm_v2/MMLU-PRO": 0.1141 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/intel.json b/data/developers/intel.json deleted file mode 100644 index 3c307773fc949afd4c3654b02313182481348954..0000000000000000000000000000000000000000 --- a/data/developers/intel.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "developer": "Intel", - "models": [ - { - "id": "Intel/neural-chat-7b-v3", - "name": "neural-chat-7b-v3", - "developer": "Intel", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2778, - "hfopenllm_v2/BBH": 0.5048, - "hfopenllm_v2/MATH Level 5": 0.0295, - "hfopenllm_v2/GPQA": 0.2919, - "hfopenllm_v2/MUSR": 0.5055, - "hfopenllm_v2/MMLU-PRO": 0.2699 - } - }, - { - "id": "Intel/neural-chat-7b-v3-1", - "name": "neural-chat-7b-v3-1", - "developer": "Intel", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4687, - "hfopenllm_v2/BBH": 0.5052, - "hfopenllm_v2/MATH Level 5": 0.0355, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.4979, - "hfopenllm_v2/MMLU-PRO": 0.2678 - } - }, - { - "id": "Intel/neural-chat-7b-v3-2", - "name": "neural-chat-7b-v3-2", - "developer": "Intel", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4988, - "hfopenllm_v2/BBH": 0.5032, - "hfopenllm_v2/MATH Level 5": 0.0476, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.4895, - "hfopenllm_v2/MMLU-PRO": 0.2667 - } - }, - { - "id": "Intel/neural-chat-7b-v3-3", - "name": "neural-chat-7b-v3-3", - "developer": "Intel", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4763, - "hfopenllm_v2/BBH": 0.4877, - "hfopenllm_v2/MATH Level 5": 0.0408, - "hfopenllm_v2/GPQA": 0.2894, - "hfopenllm_v2/MUSR": 0.486, - "hfopenllm_v2/MMLU-PRO": 0.2625 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/internlm.json b/data/developers/internlm.json deleted file mode 100644 index ba5efe87a4d9e153aadeeeaadd0b465b5955bdd6..0000000000000000000000000000000000000000 --- a/data/developers/internlm.json +++ /dev/null @@ -1,143 +0,0 @@ -{ - "developer": "internlm", - "models": [ - { - "id": "internlm/internlm2-1_8b", - "name": "internlm2-1_8b", - "developer": "internlm", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2198, - "hfopenllm_v2/BBH": 0.388, - "hfopenllm_v2/MATH Level 5": 0.0211, - "hfopenllm_v2/GPQA": 0.2483, - "hfopenllm_v2/MUSR": 0.3813, - "hfopenllm_v2/MMLU-PRO": 0.1588 - } - }, - { - "id": "internlm/internlm2-1_8b-reward", - "name": "internlm/internlm2-1_8b-reward", - "developer": "internlm", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.3902, - "reward-bench/Chat": 0.9358, - "reward-bench/Chat Hard": 0.6623, - "reward-bench/Safety": 0.4711, - "reward-bench/Reasoning": 0.8724, - "reward-bench/Factuality": 0.2758, - "reward-bench/Precise IF": 0.3625, - "reward-bench/Math": 0.4426, - "reward-bench/Focus": 0.596, - "reward-bench/Ties": 0.1934 - } - }, - { - "id": "internlm/internlm2-20b-reward", - "name": "internlm/internlm2-20b-reward", - "developer": "internlm", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.9016, - "reward-bench/Factuality": 0.5558, - "reward-bench/Precise IF": 0.3625, - "reward-bench/Math": 0.5738, - "reward-bench/Safety": 0.8946, - "reward-bench/Focus": 0.7253, - "reward-bench/Ties": 0.5483, - "reward-bench/Chat": 0.9888, - "reward-bench/Chat Hard": 0.7654, - "reward-bench/Reasoning": 0.9576 - } - }, - { - "id": "internlm/internlm2-7b", - "name": "internlm2-7b", - "developer": "internlm", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.228, - "hfopenllm_v2/BBH": 0.5825, - "hfopenllm_v2/MATH Level 5": 0.0857, - "hfopenllm_v2/GPQA": 0.3367, - "hfopenllm_v2/MUSR": 0.44, - "hfopenllm_v2/MMLU-PRO": 0.19 - } - }, - { - "id": "internlm/internlm2-7b-reward", - "name": "internlm/internlm2-7b-reward", - "developer": "internlm", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5335, - "reward-bench/Chat": 0.9916, - "reward-bench/Chat Hard": 0.6952, - "reward-bench/Safety": 0.5956, - "reward-bench/Reasoning": 0.9453, - "reward-bench/Factuality": 0.4211, - "reward-bench/Precise IF": 0.4, - "reward-bench/Math": 0.5628, - "reward-bench/Focus": 0.7051, - "reward-bench/Ties": 0.5164 - } - }, - { - "id": "internlm/internlm2-chat-1_8b", - "name": "internlm2-chat-1_8b", - "developer": "internlm", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2387, - "hfopenllm_v2/BBH": 0.4452, - "hfopenllm_v2/MATH Level 5": 0.0325, - "hfopenllm_v2/GPQA": 0.2659, - "hfopenllm_v2/MUSR": 0.3631, - "hfopenllm_v2/MMLU-PRO": 0.1839 - } - }, - { - "id": "internlm/internlm2_5-1_8b-chat", - "name": "internlm2_5-1_8b-chat", - "developer": "internlm", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3849, - "hfopenllm_v2/BBH": 0.4489, - "hfopenllm_v2/MATH Level 5": 0.1586, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.3594, - "hfopenllm_v2/MMLU-PRO": 0.1299 - } - }, - { - "id": "internlm/internlm2_5-20b-chat", - "name": "internlm2_5-20b-chat", - "developer": "internlm", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.701, - "hfopenllm_v2/BBH": 0.7474, - "hfopenllm_v2/MATH Level 5": 0.4079, - "hfopenllm_v2/GPQA": 0.3213, - "hfopenllm_v2/MUSR": 0.4558, - "hfopenllm_v2/MMLU-PRO": 0.3998 - } - }, - { - "id": "internlm/internlm2_5-7b-chat", - "name": "internlm2_5-7b-chat", - "developer": "internlm", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5539, - "hfopenllm_v2/BBH": 0.7073, - "hfopenllm_v2/MATH Level 5": 0.253, - "hfopenllm_v2/GPQA": 0.3473, - "hfopenllm_v2/MUSR": 0.4594, - "hfopenllm_v2/MMLU-PRO": 0.3777 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/intervitens.json b/data/developers/intervitens.json deleted file mode 100644 index 293151195dddaf964129ce365ecdb39e498abcb7..0000000000000000000000000000000000000000 --- a/data/developers/intervitens.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "intervitens", - "models": [ - { - "id": "intervitens/mini-magnum-12b-v1.1", - "name": "mini-magnum-12b-v1.1", - "developer": "intervitens", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5156, - "hfopenllm_v2/BBH": 0.5062, - "hfopenllm_v2/MATH Level 5": 0.0619, - "hfopenllm_v2/GPQA": 0.2886, - "hfopenllm_v2/MUSR": 0.4004, - "hfopenllm_v2/MMLU-PRO": 0.3291 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/intervitensinc.json b/data/developers/intervitensinc.json deleted file mode 100644 index 0ea9aa4a8d0ee5254edca8c7b51c3fd96c16439e..0000000000000000000000000000000000000000 --- a/data/developers/intervitensinc.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "IntervitensInc", - "models": [ - { - "id": "IntervitensInc/internlm2_5-20b-llamafied", - "name": "internlm2_5-20b-llamafied", - "developer": "IntervitensInc", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.341, - "hfopenllm_v2/BBH": 0.7478, - "hfopenllm_v2/MATH Level 5": 0.1715, - "hfopenllm_v2/GPQA": 0.3381, - "hfopenllm_v2/MUSR": 0.4475, - "hfopenllm_v2/MMLU-PRO": 0.4051 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/inumulaisk.json b/data/developers/inumulaisk.json deleted file mode 100644 index fe3306870802d32ab7048171bf1b41497caa7318..0000000000000000000000000000000000000000 --- a/data/developers/inumulaisk.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "inumulaisk", - "models": [ - { - "id": "inumulaisk/eval_model", - "name": "eval_model", - "developer": "inumulaisk", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1931, - "hfopenllm_v2/BBH": 0.3512, - "hfopenllm_v2/MATH Level 5": 0.2976, - "hfopenllm_v2/GPQA": 0.2794, - "hfopenllm_v2/MUSR": 0.358, - "hfopenllm_v2/MMLU-PRO": 0.1664 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/invalid-coder.json b/data/developers/invalid-coder.json deleted file mode 100644 index 1e4e77376b570ecc50521a6e198803bfb1f87e75..0000000000000000000000000000000000000000 --- a/data/developers/invalid-coder.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "invalid-coder", - "models": [ - { - "id": "invalid-coder/Sakura-SOLAR-Instruct-CarbonVillain-en-10.7B-v2-slerp", - "name": "Sakura-SOLAR-Instruct-CarbonVillain-en-10.7B-v2-slerp", - "developer": "invalid-coder", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4555, - "hfopenllm_v2/BBH": 0.5158, - "hfopenllm_v2/MATH Level 5": 0.0491, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.3992, - "hfopenllm_v2/MMLU-PRO": 0.3146 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/invalid-null.json b/data/developers/invalid-null.json deleted file mode 100644 index 092ae9a1b72cf4aa02d8fb66f5516fadfff0d956..0000000000000000000000000000000000000000 --- a/data/developers/invalid-null.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "Invalid-Null", - "models": [ - { - "id": "Invalid-Null/PeiYangMe-0.5", - "name": "PeiYangMe-0.5", - "developer": "Invalid-Null", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1409, - "hfopenllm_v2/BBH": 0.2791, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2441, - "hfopenllm_v2/MUSR": 0.3738, - "hfopenllm_v2/MMLU-PRO": 0.1109 - } - }, - { - "id": "Invalid-Null/PeiYangMe-0.7", - "name": "PeiYangMe-0.7", - "developer": "Invalid-Null", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1491, - "hfopenllm_v2/BBH": 0.3028, - "hfopenllm_v2/MATH Level 5": 0.0113, - "hfopenllm_v2/GPQA": 0.2332, - "hfopenllm_v2/MUSR": 0.3857, - "hfopenllm_v2/MMLU-PRO": 0.1101 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/invisietch.json b/data/developers/invisietch.json deleted file mode 100644 index 8583bb65940ddb12664addbe021e70440d09afd0..0000000000000000000000000000000000000000 --- a/data/developers/invisietch.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "developer": "invisietch", - "models": [ - { - "id": "invisietch/EtherealRainbow-v0.2-8B", - "name": "EtherealRainbow-v0.2-8B", - "developer": "invisietch", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3903, - "hfopenllm_v2/BBH": 0.5102, - "hfopenllm_v2/MATH Level 5": 0.0823, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.3827, - "hfopenllm_v2/MMLU-PRO": 0.3653 - } - }, - { - "id": "invisietch/EtherealRainbow-v0.3-8B", - "name": "EtherealRainbow-v0.3-8B", - "developer": "invisietch", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3682, - "hfopenllm_v2/BBH": 0.5097, - "hfopenllm_v2/MATH Level 5": 0.0763, - "hfopenllm_v2/GPQA": 0.3045, - "hfopenllm_v2/MUSR": 0.3904, - "hfopenllm_v2/MMLU-PRO": 0.3626 - } - }, - { - "id": "invisietch/MiS-Firefly-v0.2-22B", - "name": "MiS-Firefly-v0.2-22B", - "developer": "invisietch", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5371, - "hfopenllm_v2/BBH": 0.5514, - "hfopenllm_v2/MATH Level 5": 0.1654, - "hfopenllm_v2/GPQA": 0.3045, - "hfopenllm_v2/MUSR": 0.4694, - "hfopenllm_v2/MMLU-PRO": 0.362 - } - }, - { - "id": "invisietch/Nimbus-Miqu-v0.1-70B", - "name": "Nimbus-Miqu-v0.1-70B", - "developer": "invisietch", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4647, - "hfopenllm_v2/BBH": 0.601, - "hfopenllm_v2/MATH Level 5": 0.0604, - "hfopenllm_v2/GPQA": 0.3389, - "hfopenllm_v2/MUSR": 0.4133, - "hfopenllm_v2/MMLU-PRO": 0.3853 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/irahulpandey.json b/data/developers/irahulpandey.json deleted file mode 100644 index d636eca8af7304b695038d41c56949934cef8424..0000000000000000000000000000000000000000 --- a/data/developers/irahulpandey.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "irahulpandey", - "models": [ - { - "id": "irahulpandey/mistralai-7B-slerp-v0.1", - "name": "mistralai-7B-slerp-v0.1", - "developer": "irahulpandey", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4966, - "hfopenllm_v2/BBH": 0.5011, - "hfopenllm_v2/MATH Level 5": 0.0514, - "hfopenllm_v2/GPQA": 0.3037, - "hfopenllm_v2/MUSR": 0.455, - "hfopenllm_v2/MMLU-PRO": 0.2951 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/iryanbell.json b/data/developers/iryanbell.json deleted file mode 100644 index 2c50d54f05efbf1155feaa57a759bc15e4315d5a..0000000000000000000000000000000000000000 --- a/data/developers/iryanbell.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "iRyanBell", - "models": [ - { - "id": "iRyanBell/ARC1", - "name": "ARC1", - "developer": "iRyanBell", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4411, - "hfopenllm_v2/BBH": 0.4903, - "hfopenllm_v2/MATH Level 5": 0.0687, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.3991, - "hfopenllm_v2/MMLU-PRO": 0.3371 - } - }, - { - "id": "iRyanBell/ARC1-II", - "name": "ARC1-II", - "developer": "iRyanBell", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1708, - "hfopenllm_v2/BBH": 0.3382, - "hfopenllm_v2/MATH Level 5": 0.0219, - "hfopenllm_v2/GPQA": 0.2718, - "hfopenllm_v2/MUSR": 0.4913, - "hfopenllm_v2/MMLU-PRO": 0.1686 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/isaak-carter.json b/data/developers/isaak-carter.json deleted file mode 100644 index 690871bc9bb32834a871ee3cfe82d0488b7c6bad..0000000000000000000000000000000000000000 --- a/data/developers/isaak-carter.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "developer": "Isaak-Carter", - "models": [ - { - "id": "Isaak-Carter/Josiefied-Qwen2.5-7B-Instruct-abliterated", - "name": "Josiefied-Qwen2.5-7B-Instruct-abliterated", - "developer": "Isaak-Carter", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7317, - "hfopenllm_v2/BBH": 0.5396, - "hfopenllm_v2/MATH Level 5": 0.4924, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.4087, - "hfopenllm_v2/MMLU-PRO": 0.4276 - } - }, - { - "id": "Isaak-Carter/Josiefied-Qwen2.5-7B-Instruct-abliterated-v2", - "name": "Josiefied-Qwen2.5-7B-Instruct-abliterated-v2", - "developer": "Isaak-Carter", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7841, - "hfopenllm_v2/BBH": 0.5311, - "hfopenllm_v2/MATH Level 5": 0.4721, - "hfopenllm_v2/GPQA": 0.2987, - "hfopenllm_v2/MUSR": 0.4354, - "hfopenllm_v2/MMLU-PRO": 0.4128 - } - }, - { - "id": "Isaak-Carter/JOSIEv4o-8b-stage1-v4", - "name": "JOSIEv4o-8b-stage1-v4", - "developer": "Isaak-Carter", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2477, - "hfopenllm_v2/BBH": 0.4758, - "hfopenllm_v2/MATH Level 5": 0.0453, - "hfopenllm_v2/GPQA": 0.2911, - "hfopenllm_v2/MUSR": 0.3641, - "hfopenllm_v2/MMLU-PRO": 0.3292 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/j-lab.json b/data/developers/j-lab.json deleted file mode 100644 index 1d3396bc0913242e032ccd1c20ecfd0982c674e7..0000000000000000000000000000000000000000 --- a/data/developers/j-lab.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "J-LAB", - "models": [ - { - "id": "J-LAB/Thynk_orpo", - "name": "Thynk_orpo", - "developer": "J-LAB", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2102, - "hfopenllm_v2/BBH": 0.4463, - "hfopenllm_v2/MATH Level 5": 0.148, - "hfopenllm_v2/GPQA": 0.2928, - "hfopenllm_v2/MUSR": 0.4515, - "hfopenllm_v2/MMLU-PRO": 0.3231 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/jackfram.json b/data/developers/jackfram.json deleted file mode 100644 index 17a9ecc1505304f429e159e861c561da0bf6d73a..0000000000000000000000000000000000000000 --- a/data/developers/jackfram.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "JackFram", - "models": [ - { - "id": "JackFram/llama-160m", - "name": "llama-160m", - "developer": "JackFram", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1791, - "hfopenllm_v2/BBH": 0.2888, - "hfopenllm_v2/MATH Level 5": 0.0083, - "hfopenllm_v2/GPQA": 0.2617, - "hfopenllm_v2/MUSR": 0.3792, - "hfopenllm_v2/MMLU-PRO": 0.1128 - } - }, - { - "id": "JackFram/llama-68m", - "name": "llama-68m", - "developer": "JackFram", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1726, - "hfopenllm_v2/BBH": 0.2936, - "hfopenllm_v2/MATH Level 5": 0.006, - "hfopenllm_v2/GPQA": 0.2584, - "hfopenllm_v2/MUSR": 0.391, - "hfopenllm_v2/MMLU-PRO": 0.1144 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/jacoby746.json b/data/developers/jacoby746.json deleted file mode 100644 index 3b115885947526371d1548d3433037196ce9a203..0000000000000000000000000000000000000000 --- a/data/developers/jacoby746.json +++ /dev/null @@ -1,103 +0,0 @@ -{ - "developer": "Jacoby746", - "models": [ - { - "id": "Jacoby746/Casual-Magnum-34B", - "name": "Casual-Magnum-34B", - "developer": "Jacoby746", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.193, - "hfopenllm_v2/BBH": 0.6032, - "hfopenllm_v2/MATH Level 5": 0.0921, - "hfopenllm_v2/GPQA": 0.3725, - "hfopenllm_v2/MUSR": 0.4078, - "hfopenllm_v2/MMLU-PRO": 0.5184 - } - }, - { - "id": "Jacoby746/Inf-Silent-Kunoichi-v0.1-2x7B", - "name": "Inf-Silent-Kunoichi-v0.1-2x7B", - "developer": "Jacoby746", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.388, - "hfopenllm_v2/BBH": 0.5185, - "hfopenllm_v2/MATH Level 5": 0.071, - "hfopenllm_v2/GPQA": 0.2894, - "hfopenllm_v2/MUSR": 0.428, - "hfopenllm_v2/MMLU-PRO": 0.3271 - } - }, - { - "id": "Jacoby746/Inf-Silent-Kunoichi-v0.2-2x7B", - "name": "Inf-Silent-Kunoichi-v0.2-2x7B", - "developer": "Jacoby746", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3636, - "hfopenllm_v2/BBH": 0.5209, - "hfopenllm_v2/MATH Level 5": 0.0627, - "hfopenllm_v2/GPQA": 0.3003, - "hfopenllm_v2/MUSR": 0.432, - "hfopenllm_v2/MMLU-PRO": 0.3272 - } - }, - { - "id": "Jacoby746/Proto-Athena-4x7B", - "name": "Proto-Athena-4x7B", - "developer": "Jacoby746", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3703, - "hfopenllm_v2/BBH": 0.5107, - "hfopenllm_v2/MATH Level 5": 0.065, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.4348, - "hfopenllm_v2/MMLU-PRO": 0.3206 - } - }, - { - "id": "Jacoby746/Proto-Athena-v0.2-4x7B", - "name": "Proto-Athena-v0.2-4x7B", - "developer": "Jacoby746", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3752, - "hfopenllm_v2/BBH": 0.5068, - "hfopenllm_v2/MATH Level 5": 0.0634, - "hfopenllm_v2/GPQA": 0.2987, - "hfopenllm_v2/MUSR": 0.4213, - "hfopenllm_v2/MMLU-PRO": 0.3197 - } - }, - { - "id": "Jacoby746/Proto-Harpy-Blazing-Light-v0.1-2x7B", - "name": "Proto-Harpy-Blazing-Light-v0.1-2x7B", - "developer": "Jacoby746", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4905, - "hfopenllm_v2/BBH": 0.5187, - "hfopenllm_v2/MATH Level 5": 0.0748, - "hfopenllm_v2/GPQA": 0.2953, - "hfopenllm_v2/MUSR": 0.445, - "hfopenllm_v2/MMLU-PRO": 0.3301 - } - }, - { - "id": "Jacoby746/Proto-Harpy-Spark-v0.1-7B", - "name": "Proto-Harpy-Spark-v0.1-7B", - "developer": "Jacoby746", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4333, - "hfopenllm_v2/BBH": 0.4736, - "hfopenllm_v2/MATH Level 5": 0.0619, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.4317, - "hfopenllm_v2/MMLU-PRO": 0.3069 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/jaredjoss.json b/data/developers/jaredjoss.json deleted file mode 100644 index 26f23fdcced9da0fbd543c310df0ea11996f4638..0000000000000000000000000000000000000000 --- a/data/developers/jaredjoss.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "jaredjoss", - "models": [ - { - "id": "jaredjoss/pythia-410m-roberta-lr_8e7-kl_01-steps_12000-rlhf-model", - "name": "pythia-410m-roberta-lr_8e7-kl_01-steps_12000-rlhf-model", - "developer": "jaredjoss", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1572, - "hfopenllm_v2/BBH": 0.2863, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3607, - "hfopenllm_v2/MMLU-PRO": 0.1169 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/jaspionjader.json b/data/developers/jaspionjader.json deleted file mode 100644 index 9d9d1e268e56a9945ae657deca0493de6a22ce3d..0000000000000000000000000000000000000000 --- a/data/developers/jaspionjader.json +++ /dev/null @@ -1,2749 +0,0 @@ -{ - "developer": "jaspionjader", - "models": [ - { - "id": "jaspionjader/Auro-Kosmos-EVAA-v2-8B", - "name": "Auro-Kosmos-EVAA-v2-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4778, - "hfopenllm_v2/BBH": 0.5447, - "hfopenllm_v2/MATH Level 5": 0.1412, - "hfopenllm_v2/GPQA": 0.3154, - "hfopenllm_v2/MUSR": 0.425, - "hfopenllm_v2/MMLU-PRO": 0.3858 - } - }, - { - "id": "jaspionjader/Auro-Kosmos-EVAA-v2.1-8B", - "name": "Auro-Kosmos-EVAA-v2.1-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4666, - "hfopenllm_v2/BBH": 0.5444, - "hfopenllm_v2/MATH Level 5": 0.1458, - "hfopenllm_v2/GPQA": 0.3087, - "hfopenllm_v2/MUSR": 0.4317, - "hfopenllm_v2/MMLU-PRO": 0.3826 - } - }, - { - "id": "jaspionjader/Auro-Kosmos-EVAA-v2.2-8B", - "name": "Auro-Kosmos-EVAA-v2.2-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4268, - "hfopenllm_v2/BBH": 0.5431, - "hfopenllm_v2/MATH Level 5": 0.1412, - "hfopenllm_v2/GPQA": 0.3104, - "hfopenllm_v2/MUSR": 0.4251, - "hfopenllm_v2/MMLU-PRO": 0.3798 - } - }, - { - "id": "jaspionjader/Auro-Kosmos-EVAA-v2.3-8B", - "name": "Auro-Kosmos-EVAA-v2.3-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4271, - "hfopenllm_v2/BBH": 0.5441, - "hfopenllm_v2/MATH Level 5": 0.1344, - "hfopenllm_v2/GPQA": 0.3121, - "hfopenllm_v2/MUSR": 0.4278, - "hfopenllm_v2/MMLU-PRO": 0.3784 - } - }, - { - "id": "jaspionjader/bbb-1", - "name": "bbb-1", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4864, - "hfopenllm_v2/BBH": 0.5376, - "hfopenllm_v2/MATH Level 5": 0.1367, - "hfopenllm_v2/GPQA": 0.3138, - "hfopenllm_v2/MUSR": 0.4171, - "hfopenllm_v2/MMLU-PRO": 0.3897 - } - }, - { - "id": "jaspionjader/bbb-2", - "name": "bbb-2", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4077, - "hfopenllm_v2/BBH": 0.5067, - "hfopenllm_v2/MATH Level 5": 0.1125, - "hfopenllm_v2/GPQA": 0.302, - "hfopenllm_v2/MUSR": 0.4145, - "hfopenllm_v2/MMLU-PRO": 0.3635 - } - }, - { - "id": "jaspionjader/bbb-3", - "name": "bbb-3", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4168, - "hfopenllm_v2/BBH": 0.5158, - "hfopenllm_v2/MATH Level 5": 0.1405, - "hfopenllm_v2/GPQA": 0.3112, - "hfopenllm_v2/MUSR": 0.4265, - "hfopenllm_v2/MMLU-PRO": 0.3856 - } - }, - { - "id": "jaspionjader/bbb-4", - "name": "bbb-4", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4768, - "hfopenllm_v2/BBH": 0.5212, - "hfopenllm_v2/MATH Level 5": 0.1276, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.4092, - "hfopenllm_v2/MMLU-PRO": 0.3773 - } - }, - { - "id": "jaspionjader/bbb-5", - "name": "bbb-5", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4703, - "hfopenllm_v2/BBH": 0.5207, - "hfopenllm_v2/MATH Level 5": 0.1397, - "hfopenllm_v2/GPQA": 0.3045, - "hfopenllm_v2/MUSR": 0.3998, - "hfopenllm_v2/MMLU-PRO": 0.3834 - } - }, - { - "id": "jaspionjader/bbb-6", - "name": "bbb-6", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.488, - "hfopenllm_v2/BBH": 0.5211, - "hfopenllm_v2/MATH Level 5": 0.139, - "hfopenllm_v2/GPQA": 0.3104, - "hfopenllm_v2/MUSR": 0.4052, - "hfopenllm_v2/MMLU-PRO": 0.3871 - } - }, - { - "id": "jaspionjader/bbb-7", - "name": "bbb-7", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4828, - "hfopenllm_v2/BBH": 0.5211, - "hfopenllm_v2/MATH Level 5": 0.1367, - "hfopenllm_v2/GPQA": 0.3104, - "hfopenllm_v2/MUSR": 0.4038, - "hfopenllm_v2/MMLU-PRO": 0.386 - } - }, - { - "id": "jaspionjader/bh-1", - "name": "bh-1", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4284, - "hfopenllm_v2/BBH": 0.589, - "hfopenllm_v2/MATH Level 5": 0.0536, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.4441, - "hfopenllm_v2/MMLU-PRO": 0.3449 - } - }, - { - "id": "jaspionjader/bh-10", - "name": "bh-10", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4618, - "hfopenllm_v2/BBH": 0.5856, - "hfopenllm_v2/MATH Level 5": 0.1103, - "hfopenllm_v2/GPQA": 0.3003, - "hfopenllm_v2/MUSR": 0.4199, - "hfopenllm_v2/MMLU-PRO": 0.3708 - } - }, - { - "id": "jaspionjader/bh-11", - "name": "bh-11", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4575, - "hfopenllm_v2/BBH": 0.5851, - "hfopenllm_v2/MATH Level 5": 0.1178, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.4146, - "hfopenllm_v2/MMLU-PRO": 0.3738 - } - }, - { - "id": "jaspionjader/bh-12", - "name": "bh-12", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4734, - "hfopenllm_v2/BBH": 0.5802, - "hfopenllm_v2/MATH Level 5": 0.1186, - "hfopenllm_v2/GPQA": 0.3003, - "hfopenllm_v2/MUSR": 0.4145, - "hfopenllm_v2/MMLU-PRO": 0.3737 - } - }, - { - "id": "jaspionjader/bh-13", - "name": "bh-13", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4698, - "hfopenllm_v2/BBH": 0.5778, - "hfopenllm_v2/MATH Level 5": 0.1125, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.4159, - "hfopenllm_v2/MMLU-PRO": 0.373 - } - }, - { - "id": "jaspionjader/bh-15", - "name": "bh-15", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4745, - "hfopenllm_v2/BBH": 0.5819, - "hfopenllm_v2/MATH Level 5": 0.1246, - "hfopenllm_v2/GPQA": 0.2987, - "hfopenllm_v2/MUSR": 0.4105, - "hfopenllm_v2/MMLU-PRO": 0.3767 - } - }, - { - "id": "jaspionjader/bh-16", - "name": "bh-16", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4731, - "hfopenllm_v2/BBH": 0.5783, - "hfopenllm_v2/MATH Level 5": 0.1193, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.4159, - "hfopenllm_v2/MMLU-PRO": 0.3776 - } - }, - { - "id": "jaspionjader/bh-17", - "name": "bh-17", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4722, - "hfopenllm_v2/BBH": 0.5776, - "hfopenllm_v2/MATH Level 5": 0.1133, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.4158, - "hfopenllm_v2/MMLU-PRO": 0.3757 - } - }, - { - "id": "jaspionjader/bh-18", - "name": "bh-18", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4725, - "hfopenllm_v2/BBH": 0.5824, - "hfopenllm_v2/MATH Level 5": 0.1186, - "hfopenllm_v2/GPQA": 0.3003, - "hfopenllm_v2/MUSR": 0.4185, - "hfopenllm_v2/MMLU-PRO": 0.3757 - } - }, - { - "id": "jaspionjader/bh-19", - "name": "bh-19", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4584, - "hfopenllm_v2/BBH": 0.5766, - "hfopenllm_v2/MATH Level 5": 0.1193, - "hfopenllm_v2/GPQA": 0.297, - "hfopenllm_v2/MUSR": 0.4171, - "hfopenllm_v2/MMLU-PRO": 0.3775 - } - }, - { - "id": "jaspionjader/bh-2", - "name": "bh-2", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4579, - "hfopenllm_v2/BBH": 0.5937, - "hfopenllm_v2/MATH Level 5": 0.1027, - "hfopenllm_v2/GPQA": 0.3012, - "hfopenllm_v2/MUSR": 0.4186, - "hfopenllm_v2/MMLU-PRO": 0.3695 - } - }, - { - "id": "jaspionjader/bh-20", - "name": "bh-20", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4727, - "hfopenllm_v2/BBH": 0.575, - "hfopenllm_v2/MATH Level 5": 0.1201, - "hfopenllm_v2/GPQA": 0.2878, - "hfopenllm_v2/MUSR": 0.4105, - "hfopenllm_v2/MMLU-PRO": 0.3768 - } - }, - { - "id": "jaspionjader/bh-21", - "name": "bh-21", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.47, - "hfopenllm_v2/BBH": 0.5738, - "hfopenllm_v2/MATH Level 5": 0.1216, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.4158, - "hfopenllm_v2/MMLU-PRO": 0.3776 - } - }, - { - "id": "jaspionjader/bh-22", - "name": "bh-22", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.46, - "hfopenllm_v2/BBH": 0.5793, - "hfopenllm_v2/MATH Level 5": 0.1186, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.4172, - "hfopenllm_v2/MMLU-PRO": 0.3764 - } - }, - { - "id": "jaspionjader/bh-23", - "name": "bh-23", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4658, - "hfopenllm_v2/BBH": 0.57, - "hfopenllm_v2/MATH Level 5": 0.1201, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.4197, - "hfopenllm_v2/MMLU-PRO": 0.3796 - } - }, - { - "id": "jaspionjader/bh-24", - "name": "bh-24", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4715, - "hfopenllm_v2/BBH": 0.5717, - "hfopenllm_v2/MATH Level 5": 0.1269, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.4158, - "hfopenllm_v2/MMLU-PRO": 0.3809 - } - }, - { - "id": "jaspionjader/bh-25", - "name": "bh-25", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4752, - "hfopenllm_v2/BBH": 0.5706, - "hfopenllm_v2/MATH Level 5": 0.1133, - "hfopenllm_v2/GPQA": 0.2911, - "hfopenllm_v2/MUSR": 0.4118, - "hfopenllm_v2/MMLU-PRO": 0.3782 - } - }, - { - "id": "jaspionjader/bh-26", - "name": "bh-26", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4691, - "hfopenllm_v2/BBH": 0.5735, - "hfopenllm_v2/MATH Level 5": 0.1163, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.4277, - "hfopenllm_v2/MMLU-PRO": 0.3772 - } - }, - { - "id": "jaspionjader/bh-27", - "name": "bh-27", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4819, - "hfopenllm_v2/BBH": 0.5714, - "hfopenllm_v2/MATH Level 5": 0.1276, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.4091, - "hfopenllm_v2/MMLU-PRO": 0.3799 - } - }, - { - "id": "jaspionjader/bh-28", - "name": "bh-28", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4785, - "hfopenllm_v2/BBH": 0.5703, - "hfopenllm_v2/MATH Level 5": 0.1231, - "hfopenllm_v2/GPQA": 0.2987, - "hfopenllm_v2/MUSR": 0.4131, - "hfopenllm_v2/MMLU-PRO": 0.3812 - } - }, - { - "id": "jaspionjader/bh-29", - "name": "bh-29", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4688, - "hfopenllm_v2/BBH": 0.567, - "hfopenllm_v2/MATH Level 5": 0.1208, - "hfopenllm_v2/GPQA": 0.2953, - "hfopenllm_v2/MUSR": 0.4237, - "hfopenllm_v2/MMLU-PRO": 0.3819 - } - }, - { - "id": "jaspionjader/bh-3", - "name": "bh-3", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4664, - "hfopenllm_v2/BBH": 0.5891, - "hfopenllm_v2/MATH Level 5": 0.1148, - "hfopenllm_v2/GPQA": 0.302, - "hfopenllm_v2/MUSR": 0.4173, - "hfopenllm_v2/MMLU-PRO": 0.3702 - } - }, - { - "id": "jaspionjader/bh-30", - "name": "bh-30", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4666, - "hfopenllm_v2/BBH": 0.5706, - "hfopenllm_v2/MATH Level 5": 0.1231, - "hfopenllm_v2/GPQA": 0.2928, - "hfopenllm_v2/MUSR": 0.4144, - "hfopenllm_v2/MMLU-PRO": 0.3782 - } - }, - { - "id": "jaspionjader/bh-31", - "name": "bh-31", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4727, - "hfopenllm_v2/BBH": 0.5665, - "hfopenllm_v2/MATH Level 5": 0.1284, - "hfopenllm_v2/GPQA": 0.2936, - "hfopenllm_v2/MUSR": 0.4104, - "hfopenllm_v2/MMLU-PRO": 0.382 - } - }, - { - "id": "jaspionjader/bh-32", - "name": "bh-32", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4636, - "hfopenllm_v2/BBH": 0.5662, - "hfopenllm_v2/MATH Level 5": 0.1246, - "hfopenllm_v2/GPQA": 0.297, - "hfopenllm_v2/MUSR": 0.4157, - "hfopenllm_v2/MMLU-PRO": 0.3812 - } - }, - { - "id": "jaspionjader/bh-33", - "name": "bh-33", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4685, - "hfopenllm_v2/BBH": 0.5653, - "hfopenllm_v2/MATH Level 5": 0.1178, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.4157, - "hfopenllm_v2/MMLU-PRO": 0.3808 - } - }, - { - "id": "jaspionjader/bh-34", - "name": "bh-34", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4624, - "hfopenllm_v2/BBH": 0.5681, - "hfopenllm_v2/MATH Level 5": 0.1208, - "hfopenllm_v2/GPQA": 0.2919, - "hfopenllm_v2/MUSR": 0.4185, - "hfopenllm_v2/MMLU-PRO": 0.3804 - } - }, - { - "id": "jaspionjader/bh-35", - "name": "bh-35", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4721, - "hfopenllm_v2/BBH": 0.564, - "hfopenllm_v2/MATH Level 5": 0.1246, - "hfopenllm_v2/GPQA": 0.2953, - "hfopenllm_v2/MUSR": 0.4183, - "hfopenllm_v2/MMLU-PRO": 0.383 - } - }, - { - "id": "jaspionjader/bh-36", - "name": "bh-36", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4666, - "hfopenllm_v2/BBH": 0.5664, - "hfopenllm_v2/MATH Level 5": 0.1239, - "hfopenllm_v2/GPQA": 0.302, - "hfopenllm_v2/MUSR": 0.4196, - "hfopenllm_v2/MMLU-PRO": 0.3831 - } - }, - { - "id": "jaspionjader/bh-37", - "name": "bh-37", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.488, - "hfopenllm_v2/BBH": 0.5625, - "hfopenllm_v2/MATH Level 5": 0.1216, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.4156, - "hfopenllm_v2/MMLU-PRO": 0.3828 - } - }, - { - "id": "jaspionjader/bh-38", - "name": "bh-38", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4618, - "hfopenllm_v2/BBH": 0.5658, - "hfopenllm_v2/MATH Level 5": 0.1239, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.4117, - "hfopenllm_v2/MMLU-PRO": 0.3811 - } - }, - { - "id": "jaspionjader/bh-39", - "name": "bh-39", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4576, - "hfopenllm_v2/BBH": 0.5633, - "hfopenllm_v2/MATH Level 5": 0.1254, - "hfopenllm_v2/GPQA": 0.3003, - "hfopenllm_v2/MUSR": 0.4262, - "hfopenllm_v2/MMLU-PRO": 0.3831 - } - }, - { - "id": "jaspionjader/bh-4", - "name": "bh-4", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4673, - "hfopenllm_v2/BBH": 0.5892, - "hfopenllm_v2/MATH Level 5": 0.1095, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.4173, - "hfopenllm_v2/MMLU-PRO": 0.3705 - } - }, - { - "id": "jaspionjader/bh-40", - "name": "bh-40", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4536, - "hfopenllm_v2/BBH": 0.5634, - "hfopenllm_v2/MATH Level 5": 0.1246, - "hfopenllm_v2/GPQA": 0.2987, - "hfopenllm_v2/MUSR": 0.4236, - "hfopenllm_v2/MMLU-PRO": 0.3835 - } - }, - { - "id": "jaspionjader/bh-41", - "name": "bh-41", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.474, - "hfopenllm_v2/BBH": 0.5614, - "hfopenllm_v2/MATH Level 5": 0.1254, - "hfopenllm_v2/GPQA": 0.2928, - "hfopenllm_v2/MUSR": 0.4183, - "hfopenllm_v2/MMLU-PRO": 0.3825 - } - }, - { - "id": "jaspionjader/bh-42", - "name": "bh-42", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.466, - "hfopenllm_v2/BBH": 0.5646, - "hfopenllm_v2/MATH Level 5": 0.1269, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.421, - "hfopenllm_v2/MMLU-PRO": 0.3812 - } - }, - { - "id": "jaspionjader/bh-43", - "name": "bh-43", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.46, - "hfopenllm_v2/BBH": 0.5635, - "hfopenllm_v2/MATH Level 5": 0.1239, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.4156, - "hfopenllm_v2/MMLU-PRO": 0.382 - } - }, - { - "id": "jaspionjader/bh-44", - "name": "bh-44", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4706, - "hfopenllm_v2/BBH": 0.5643, - "hfopenllm_v2/MATH Level 5": 0.1216, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.4249, - "hfopenllm_v2/MMLU-PRO": 0.3834 - } - }, - { - "id": "jaspionjader/bh-46", - "name": "bh-46", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4727, - "hfopenllm_v2/BBH": 0.5632, - "hfopenllm_v2/MATH Level 5": 0.1276, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.4262, - "hfopenllm_v2/MMLU-PRO": 0.3822 - } - }, - { - "id": "jaspionjader/bh-47", - "name": "bh-47", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4652, - "hfopenllm_v2/BBH": 0.5546, - "hfopenllm_v2/MATH Level 5": 0.1276, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.4156, - "hfopenllm_v2/MMLU-PRO": 0.3855 - } - }, - { - "id": "jaspionjader/bh-48", - "name": "bh-48", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4688, - "hfopenllm_v2/BBH": 0.5541, - "hfopenllm_v2/MATH Level 5": 0.1254, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.4209, - "hfopenllm_v2/MMLU-PRO": 0.386 - } - }, - { - "id": "jaspionjader/bh-49", - "name": "bh-49", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4725, - "hfopenllm_v2/BBH": 0.554, - "hfopenllm_v2/MATH Level 5": 0.1201, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.4129, - "hfopenllm_v2/MMLU-PRO": 0.3808 - } - }, - { - "id": "jaspionjader/bh-5", - "name": "bh-5", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4652, - "hfopenllm_v2/BBH": 0.5882, - "hfopenllm_v2/MATH Level 5": 0.1057, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.4186, - "hfopenllm_v2/MMLU-PRO": 0.3702 - } - }, - { - "id": "jaspionjader/bh-50", - "name": "bh-50", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4725, - "hfopenllm_v2/BBH": 0.5553, - "hfopenllm_v2/MATH Level 5": 0.1208, - "hfopenllm_v2/GPQA": 0.2936, - "hfopenllm_v2/MUSR": 0.4169, - "hfopenllm_v2/MMLU-PRO": 0.3842 - } - }, - { - "id": "jaspionjader/bh-51", - "name": "bh-51", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.463, - "hfopenllm_v2/BBH": 0.5557, - "hfopenllm_v2/MATH Level 5": 0.1239, - "hfopenllm_v2/GPQA": 0.2928, - "hfopenllm_v2/MUSR": 0.4168, - "hfopenllm_v2/MMLU-PRO": 0.3831 - } - }, - { - "id": "jaspionjader/bh-52", - "name": "bh-52", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4536, - "hfopenllm_v2/BBH": 0.5444, - "hfopenllm_v2/MATH Level 5": 0.1201, - "hfopenllm_v2/GPQA": 0.2919, - "hfopenllm_v2/MUSR": 0.4169, - "hfopenllm_v2/MMLU-PRO": 0.3843 - } - }, - { - "id": "jaspionjader/bh-53", - "name": "bh-53", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.478, - "hfopenllm_v2/BBH": 0.5494, - "hfopenllm_v2/MATH Level 5": 0.1269, - "hfopenllm_v2/GPQA": 0.2987, - "hfopenllm_v2/MUSR": 0.4196, - "hfopenllm_v2/MMLU-PRO": 0.3858 - } - }, - { - "id": "jaspionjader/bh-54", - "name": "bh-54", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4841, - "hfopenllm_v2/BBH": 0.5548, - "hfopenllm_v2/MATH Level 5": 0.1292, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.4155, - "hfopenllm_v2/MMLU-PRO": 0.3825 - } - }, - { - "id": "jaspionjader/bh-55", - "name": "bh-55", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4709, - "hfopenllm_v2/BBH": 0.555, - "hfopenllm_v2/MATH Level 5": 0.1284, - "hfopenllm_v2/GPQA": 0.3062, - "hfopenllm_v2/MUSR": 0.4222, - "hfopenllm_v2/MMLU-PRO": 0.3846 - } - }, - { - "id": "jaspionjader/bh-56", - "name": "bh-56", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.46, - "hfopenllm_v2/BBH": 0.5447, - "hfopenllm_v2/MATH Level 5": 0.1231, - "hfopenllm_v2/GPQA": 0.3003, - "hfopenllm_v2/MUSR": 0.4116, - "hfopenllm_v2/MMLU-PRO": 0.3844 - } - }, - { - "id": "jaspionjader/bh-57", - "name": "bh-57", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4405, - "hfopenllm_v2/BBH": 0.5425, - "hfopenllm_v2/MATH Level 5": 0.1261, - "hfopenllm_v2/GPQA": 0.3037, - "hfopenllm_v2/MUSR": 0.421, - "hfopenllm_v2/MMLU-PRO": 0.3896 - } - }, - { - "id": "jaspionjader/bh-58", - "name": "bh-58", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.463, - "hfopenllm_v2/BBH": 0.5446, - "hfopenllm_v2/MATH Level 5": 0.1322, - "hfopenllm_v2/GPQA": 0.3062, - "hfopenllm_v2/MUSR": 0.4183, - "hfopenllm_v2/MMLU-PRO": 0.3896 - } - }, - { - "id": "jaspionjader/bh-59", - "name": "bh-59", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4341, - "hfopenllm_v2/BBH": 0.5512, - "hfopenllm_v2/MATH Level 5": 0.1541, - "hfopenllm_v2/GPQA": 0.3154, - "hfopenllm_v2/MUSR": 0.417, - "hfopenllm_v2/MMLU-PRO": 0.3838 - } - }, - { - "id": "jaspionjader/bh-6", - "name": "bh-6", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4621, - "hfopenllm_v2/BBH": 0.5891, - "hfopenllm_v2/MATH Level 5": 0.1088, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.4199, - "hfopenllm_v2/MMLU-PRO": 0.3698 - } - }, - { - "id": "jaspionjader/bh-60", - "name": "bh-60", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4207, - "hfopenllm_v2/BBH": 0.5369, - "hfopenllm_v2/MATH Level 5": 0.1579, - "hfopenllm_v2/GPQA": 0.3255, - "hfopenllm_v2/MUSR": 0.4289, - "hfopenllm_v2/MMLU-PRO": 0.3689 - } - }, - { - "id": "jaspionjader/bh-61", - "name": "bh-61", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4247, - "hfopenllm_v2/BBH": 0.5271, - "hfopenllm_v2/MATH Level 5": 0.1707, - "hfopenllm_v2/GPQA": 0.3188, - "hfopenllm_v2/MUSR": 0.4356, - "hfopenllm_v2/MMLU-PRO": 0.3679 - } - }, - { - "id": "jaspionjader/bh-62", - "name": "bh-62", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.415, - "hfopenllm_v2/BBH": 0.5379, - "hfopenllm_v2/MATH Level 5": 0.1624, - "hfopenllm_v2/GPQA": 0.3205, - "hfopenllm_v2/MUSR": 0.4289, - "hfopenllm_v2/MMLU-PRO": 0.3719 - } - }, - { - "id": "jaspionjader/bh-63", - "name": "bh-63", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4308, - "hfopenllm_v2/BBH": 0.4917, - "hfopenllm_v2/MATH Level 5": 0.111, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.4313, - "hfopenllm_v2/MMLU-PRO": 0.3248 - } - }, - { - "id": "jaspionjader/bh-64", - "name": "bh-64", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.414, - "hfopenllm_v2/BBH": 0.536, - "hfopenllm_v2/MATH Level 5": 0.1548, - "hfopenllm_v2/GPQA": 0.3213, - "hfopenllm_v2/MUSR": 0.4355, - "hfopenllm_v2/MMLU-PRO": 0.3693 - } - }, - { - "id": "jaspionjader/bh-7", - "name": "bh-7", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4624, - "hfopenllm_v2/BBH": 0.5861, - "hfopenllm_v2/MATH Level 5": 0.114, - "hfopenllm_v2/GPQA": 0.3037, - "hfopenllm_v2/MUSR": 0.4119, - "hfopenllm_v2/MMLU-PRO": 0.3715 - } - }, - { - "id": "jaspionjader/bh-8", - "name": "bh-8", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4597, - "hfopenllm_v2/BBH": 0.59, - "hfopenllm_v2/MATH Level 5": 0.1178, - "hfopenllm_v2/GPQA": 0.3012, - "hfopenllm_v2/MUSR": 0.4265, - "hfopenllm_v2/MMLU-PRO": 0.372 - } - }, - { - "id": "jaspionjader/bh-9", - "name": "bh-9", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4509, - "hfopenllm_v2/BBH": 0.585, - "hfopenllm_v2/MATH Level 5": 0.1156, - "hfopenllm_v2/GPQA": 0.302, - "hfopenllm_v2/MUSR": 0.4146, - "hfopenllm_v2/MMLU-PRO": 0.3703 - } - }, - { - "id": "jaspionjader/dp-6-8b", - "name": "dp-6-8b", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4806, - "hfopenllm_v2/BBH": 0.53, - "hfopenllm_v2/MATH Level 5": 0.1329, - "hfopenllm_v2/GPQA": 0.3079, - "hfopenllm_v2/MUSR": 0.4434, - "hfopenllm_v2/MMLU-PRO": 0.3897 - } - }, - { - "id": "jaspionjader/dp-7-8b", - "name": "dp-7-8b", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4498, - "hfopenllm_v2/BBH": 0.5291, - "hfopenllm_v2/MATH Level 5": 0.1261, - "hfopenllm_v2/GPQA": 0.3062, - "hfopenllm_v2/MUSR": 0.4407, - "hfopenllm_v2/MMLU-PRO": 0.3934 - } - }, - { - "id": "jaspionjader/ek-6", - "name": "ek-6", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4642, - "hfopenllm_v2/BBH": 0.5219, - "hfopenllm_v2/MATH Level 5": 0.1322, - "hfopenllm_v2/GPQA": 0.3087, - "hfopenllm_v2/MUSR": 0.4144, - "hfopenllm_v2/MMLU-PRO": 0.3861 - } - }, - { - "id": "jaspionjader/ek-7", - "name": "ek-7", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4767, - "hfopenllm_v2/BBH": 0.5194, - "hfopenllm_v2/MATH Level 5": 0.1329, - "hfopenllm_v2/GPQA": 0.3163, - "hfopenllm_v2/MUSR": 0.4171, - "hfopenllm_v2/MMLU-PRO": 0.3887 - } - }, - { - "id": "jaspionjader/f-1-8b", - "name": "f-1-8b", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4983, - "hfopenllm_v2/BBH": 0.5141, - "hfopenllm_v2/MATH Level 5": 0.1284, - "hfopenllm_v2/GPQA": 0.3087, - "hfopenllm_v2/MUSR": 0.4527, - "hfopenllm_v2/MMLU-PRO": 0.3907 - } - }, - { - "id": "jaspionjader/f-2-8b", - "name": "f-2-8b", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4824, - "hfopenllm_v2/BBH": 0.5294, - "hfopenllm_v2/MATH Level 5": 0.1171, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.4501, - "hfopenllm_v2/MMLU-PRO": 0.3962 - } - }, - { - "id": "jaspionjader/f-3-8b", - "name": "f-3-8b", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4803, - "hfopenllm_v2/BBH": 0.5275, - "hfopenllm_v2/MATH Level 5": 0.1216, - "hfopenllm_v2/GPQA": 0.3138, - "hfopenllm_v2/MUSR": 0.4421, - "hfopenllm_v2/MMLU-PRO": 0.3954 - } - }, - { - "id": "jaspionjader/f-4-8b", - "name": "f-4-8b", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4797, - "hfopenllm_v2/BBH": 0.5289, - "hfopenllm_v2/MATH Level 5": 0.1148, - "hfopenllm_v2/GPQA": 0.3087, - "hfopenllm_v2/MUSR": 0.4514, - "hfopenllm_v2/MMLU-PRO": 0.3956 - } - }, - { - "id": "jaspionjader/f-5-8b", - "name": "f-5-8b", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5044, - "hfopenllm_v2/BBH": 0.5313, - "hfopenllm_v2/MATH Level 5": 0.1239, - "hfopenllm_v2/GPQA": 0.3087, - "hfopenllm_v2/MUSR": 0.4461, - "hfopenllm_v2/MMLU-PRO": 0.3949 - } - }, - { - "id": "jaspionjader/f-6-8b", - "name": "f-6-8b", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4846, - "hfopenllm_v2/BBH": 0.5241, - "hfopenllm_v2/MATH Level 5": 0.1193, - "hfopenllm_v2/GPQA": 0.3079, - "hfopenllm_v2/MUSR": 0.4474, - "hfopenllm_v2/MMLU-PRO": 0.3939 - } - }, - { - "id": "jaspionjader/f-7-8b", - "name": "f-7-8b", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4462, - "hfopenllm_v2/BBH": 0.5277, - "hfopenllm_v2/MATH Level 5": 0.1239, - "hfopenllm_v2/GPQA": 0.3129, - "hfopenllm_v2/MUSR": 0.4315, - "hfopenllm_v2/MMLU-PRO": 0.3936 - } - }, - { - "id": "jaspionjader/f-8-8b", - "name": "f-8-8b", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4739, - "hfopenllm_v2/BBH": 0.5259, - "hfopenllm_v2/MATH Level 5": 0.1224, - "hfopenllm_v2/GPQA": 0.3096, - "hfopenllm_v2/MUSR": 0.4354, - "hfopenllm_v2/MMLU-PRO": 0.394 - } - }, - { - "id": "jaspionjader/f-9-8b", - "name": "f-9-8b", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4602, - "hfopenllm_v2/BBH": 0.5292, - "hfopenllm_v2/MATH Level 5": 0.1299, - "hfopenllm_v2/GPQA": 0.3062, - "hfopenllm_v2/MUSR": 0.4461, - "hfopenllm_v2/MMLU-PRO": 0.3944 - } - }, - { - "id": "jaspionjader/fct-14-8b", - "name": "fct-14-8b", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4129, - "hfopenllm_v2/BBH": 0.5206, - "hfopenllm_v2/MATH Level 5": 0.1201, - "hfopenllm_v2/GPQA": 0.3163, - "hfopenllm_v2/MUSR": 0.4186, - "hfopenllm_v2/MMLU-PRO": 0.3875 - } - }, - { - "id": "jaspionjader/fct-9-8b", - "name": "fct-9-8b", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4354, - "hfopenllm_v2/BBH": 0.5205, - "hfopenllm_v2/MATH Level 5": 0.1193, - "hfopenllm_v2/GPQA": 0.3079, - "hfopenllm_v2/MUSR": 0.4291, - "hfopenllm_v2/MMLU-PRO": 0.3932 - } - }, - { - "id": "jaspionjader/fr-1-8b", - "name": "fr-1-8b", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4211, - "hfopenllm_v2/BBH": 0.5142, - "hfopenllm_v2/MATH Level 5": 0.1118, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.4277, - "hfopenllm_v2/MMLU-PRO": 0.361 - } - }, - { - "id": "jaspionjader/fr-10-8b", - "name": "fr-10-8b", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4402, - "hfopenllm_v2/BBH": 0.5207, - "hfopenllm_v2/MATH Level 5": 0.1224, - "hfopenllm_v2/GPQA": 0.3171, - "hfopenllm_v2/MUSR": 0.4119, - "hfopenllm_v2/MMLU-PRO": 0.3863 - } - }, - { - "id": "jaspionjader/fr-3-8b", - "name": "fr-3-8b", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4326, - "hfopenllm_v2/BBH": 0.5255, - "hfopenllm_v2/MATH Level 5": 0.1133, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.4198, - "hfopenllm_v2/MMLU-PRO": 0.3863 - } - }, - { - "id": "jaspionjader/gamma-Kosmos-EVAA-8B", - "name": "gamma-Kosmos-EVAA-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.425, - "hfopenllm_v2/BBH": 0.5253, - "hfopenllm_v2/MATH Level 5": 0.0899, - "hfopenllm_v2/GPQA": 0.3138, - "hfopenllm_v2/MUSR": 0.4412, - "hfopenllm_v2/MMLU-PRO": 0.3776 - } - }, - { - "id": "jaspionjader/gamma-Kosmos-EVAA-v2-8B", - "name": "gamma-Kosmos-EVAA-v2-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4233, - "hfopenllm_v2/BBH": 0.5262, - "hfopenllm_v2/MATH Level 5": 0.1057, - "hfopenllm_v2/GPQA": 0.3205, - "hfopenllm_v2/MUSR": 0.4344, - "hfopenllm_v2/MMLU-PRO": 0.3756 - } - }, - { - "id": "jaspionjader/gamma-Kosmos-EVAA-v3-8B", - "name": "gamma-Kosmos-EVAA-v3-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4333, - "hfopenllm_v2/BBH": 0.5278, - "hfopenllm_v2/MATH Level 5": 0.111, - "hfopenllm_v2/GPQA": 0.3129, - "hfopenllm_v2/MUSR": 0.4263, - "hfopenllm_v2/MMLU-PRO": 0.3898 - } - }, - { - "id": "jaspionjader/knf-2-8b", - "name": "knf-2-8b", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.425, - "hfopenllm_v2/BBH": 0.5207, - "hfopenllm_v2/MATH Level 5": 0.1201, - "hfopenllm_v2/GPQA": 0.3104, - "hfopenllm_v2/MUSR": 0.4185, - "hfopenllm_v2/MMLU-PRO": 0.3875 - } - }, - { - "id": "jaspionjader/knfp-2-8b", - "name": "knfp-2-8b", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5327, - "hfopenllm_v2/BBH": 0.5305, - "hfopenllm_v2/MATH Level 5": 0.1427, - "hfopenllm_v2/GPQA": 0.2928, - "hfopenllm_v2/MUSR": 0.4185, - "hfopenllm_v2/MMLU-PRO": 0.3726 - } - }, - { - "id": "jaspionjader/knfp-3-8b", - "name": "knfp-3-8b", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4946, - "hfopenllm_v2/BBH": 0.52, - "hfopenllm_v2/MATH Level 5": 0.1224, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.4171, - "hfopenllm_v2/MMLU-PRO": 0.3881 - } - }, - { - "id": "jaspionjader/Kosmos-Aurora_faustus-8B", - "name": "Kosmos-Aurora_faustus-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4432, - "hfopenllm_v2/BBH": 0.526, - "hfopenllm_v2/MATH Level 5": 0.1125, - "hfopenllm_v2/GPQA": 0.2953, - "hfopenllm_v2/MUSR": 0.4117, - "hfopenllm_v2/MMLU-PRO": 0.3813 - } - }, - { - "id": "jaspionjader/Kosmos-Elusive-8b", - "name": "Kosmos-Elusive-8b", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4169, - "hfopenllm_v2/BBH": 0.5339, - "hfopenllm_v2/MATH Level 5": 0.1261, - "hfopenllm_v2/GPQA": 0.3079, - "hfopenllm_v2/MUSR": 0.4078, - "hfopenllm_v2/MMLU-PRO": 0.376 - } - }, - { - "id": "jaspionjader/Kosmos-Elusive-VENN-8B", - "name": "Kosmos-Elusive-VENN-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4233, - "hfopenllm_v2/BBH": 0.5356, - "hfopenllm_v2/MATH Level 5": 0.1246, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.4157, - "hfopenllm_v2/MMLU-PRO": 0.3797 - } - }, - { - "id": "jaspionjader/Kosmos-Elusive-VENN-Asymmetric-8B", - "name": "Kosmos-Elusive-VENN-Asymmetric-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4542, - "hfopenllm_v2/BBH": 0.5313, - "hfopenllm_v2/MATH Level 5": 0.1344, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.4251, - "hfopenllm_v2/MMLU-PRO": 0.3842 - } - }, - { - "id": "jaspionjader/Kosmos-Elusive-VENN-Aurora_faustus-8B", - "name": "Kosmos-Elusive-VENN-Aurora_faustus-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4335, - "hfopenllm_v2/BBH": 0.5304, - "hfopenllm_v2/MATH Level 5": 0.1125, - "hfopenllm_v2/GPQA": 0.2953, - "hfopenllm_v2/MUSR": 0.417, - "hfopenllm_v2/MMLU-PRO": 0.3795 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-8B", - "name": "Kosmos-EVAA-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4405, - "hfopenllm_v2/BBH": 0.5312, - "hfopenllm_v2/MATH Level 5": 0.1178, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.4237, - "hfopenllm_v2/MMLU-PRO": 0.3818 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-Franken-Immersive-v39-8B", - "name": "Kosmos-EVAA-Franken-Immersive-v39-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4378, - "hfopenllm_v2/BBH": 0.519, - "hfopenllm_v2/MATH Level 5": 0.1292, - "hfopenllm_v2/GPQA": 0.3154, - "hfopenllm_v2/MUSR": 0.4236, - "hfopenllm_v2/MMLU-PRO": 0.39 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-Franken-v38-8B", - "name": "Kosmos-EVAA-Franken-v38-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4356, - "hfopenllm_v2/BBH": 0.523, - "hfopenllm_v2/MATH Level 5": 0.1292, - "hfopenllm_v2/GPQA": 0.3087, - "hfopenllm_v2/MUSR": 0.4212, - "hfopenllm_v2/MMLU-PRO": 0.389 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-Fusion-8B", - "name": "Kosmos-EVAA-Fusion-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4418, - "hfopenllm_v2/BBH": 0.5406, - "hfopenllm_v2/MATH Level 5": 0.1352, - "hfopenllm_v2/GPQA": 0.3062, - "hfopenllm_v2/MUSR": 0.4277, - "hfopenllm_v2/MMLU-PRO": 0.386 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-gamma-8B", - "name": "Kosmos-EVAA-gamma-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4572, - "hfopenllm_v2/BBH": 0.5322, - "hfopenllm_v2/MATH Level 5": 0.105, - "hfopenllm_v2/GPQA": 0.3188, - "hfopenllm_v2/MUSR": 0.4306, - "hfopenllm_v2/MMLU-PRO": 0.3901 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-gamma-alt-8B", - "name": "Kosmos-EVAA-gamma-alt-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4542, - "hfopenllm_v2/BBH": 0.5298, - "hfopenllm_v2/MATH Level 5": 0.1095, - "hfopenllm_v2/GPQA": 0.3247, - "hfopenllm_v2/MUSR": 0.4292, - "hfopenllm_v2/MMLU-PRO": 0.3896 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-gamma-light-8B", - "name": "Kosmos-EVAA-gamma-light-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4581, - "hfopenllm_v2/BBH": 0.5376, - "hfopenllm_v2/MATH Level 5": 0.1103, - "hfopenllm_v2/GPQA": 0.3163, - "hfopenllm_v2/MUSR": 0.4291, - "hfopenllm_v2/MMLU-PRO": 0.3943 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-gamma-light-alt-8B", - "name": "Kosmos-EVAA-gamma-light-alt-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4454, - "hfopenllm_v2/BBH": 0.5327, - "hfopenllm_v2/MATH Level 5": 0.1133, - "hfopenllm_v2/GPQA": 0.3138, - "hfopenllm_v2/MUSR": 0.4305, - "hfopenllm_v2/MMLU-PRO": 0.3923 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-gamma-ultra-light-8B", - "name": "Kosmos-EVAA-gamma-ultra-light-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4563, - "hfopenllm_v2/BBH": 0.5316, - "hfopenllm_v2/MATH Level 5": 0.1178, - "hfopenllm_v2/GPQA": 0.3163, - "hfopenllm_v2/MUSR": 0.4197, - "hfopenllm_v2/MMLU-PRO": 0.3915 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-gamma-v13-8B", - "name": "Kosmos-EVAA-gamma-v13-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4429, - "hfopenllm_v2/BBH": 0.5359, - "hfopenllm_v2/MATH Level 5": 0.1118, - "hfopenllm_v2/GPQA": 0.3138, - "hfopenllm_v2/MUSR": 0.4278, - "hfopenllm_v2/MMLU-PRO": 0.393 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-gamma-v14-8B", - "name": "Kosmos-EVAA-gamma-v14-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.438, - "hfopenllm_v2/BBH": 0.5363, - "hfopenllm_v2/MATH Level 5": 0.1103, - "hfopenllm_v2/GPQA": 0.3129, - "hfopenllm_v2/MUSR": 0.4277, - "hfopenllm_v2/MMLU-PRO": 0.3931 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-gamma-v15-8B", - "name": "Kosmos-EVAA-gamma-v15-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4654, - "hfopenllm_v2/BBH": 0.5343, - "hfopenllm_v2/MATH Level 5": 0.111, - "hfopenllm_v2/GPQA": 0.3112, - "hfopenllm_v2/MUSR": 0.4277, - "hfopenllm_v2/MMLU-PRO": 0.3941 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-gamma-v16-8B", - "name": "Kosmos-EVAA-gamma-v16-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4557, - "hfopenllm_v2/BBH": 0.5344, - "hfopenllm_v2/MATH Level 5": 0.1171, - "hfopenllm_v2/GPQA": 0.3154, - "hfopenllm_v2/MUSR": 0.4264, - "hfopenllm_v2/MMLU-PRO": 0.3917 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-gamma-v17-8B", - "name": "Kosmos-EVAA-gamma-v17-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4462, - "hfopenllm_v2/BBH": 0.5347, - "hfopenllm_v2/MATH Level 5": 0.111, - "hfopenllm_v2/GPQA": 0.3112, - "hfopenllm_v2/MUSR": 0.4291, - "hfopenllm_v2/MMLU-PRO": 0.3923 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-gamma-v18-8B", - "name": "Kosmos-EVAA-gamma-v18-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4341, - "hfopenllm_v2/BBH": 0.5339, - "hfopenllm_v2/MATH Level 5": 0.111, - "hfopenllm_v2/GPQA": 0.3112, - "hfopenllm_v2/MUSR": 0.4317, - "hfopenllm_v2/MMLU-PRO": 0.3905 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-immersive-sof-v44-8B", - "name": "Kosmos-EVAA-immersive-sof-v44-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4408, - "hfopenllm_v2/BBH": 0.5215, - "hfopenllm_v2/MATH Level 5": 0.1186, - "hfopenllm_v2/GPQA": 0.3096, - "hfopenllm_v2/MUSR": 0.4144, - "hfopenllm_v2/MMLU-PRO": 0.3888 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-PRP-8B", - "name": "Kosmos-EVAA-PRP-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3405, - "hfopenllm_v2/BBH": 0.5196, - "hfopenllm_v2/MATH Level 5": 0.0884, - "hfopenllm_v2/GPQA": 0.3129, - "hfopenllm_v2/MUSR": 0.4301, - "hfopenllm_v2/MMLU-PRO": 0.3647 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-PRP-light-8B", - "name": "Kosmos-EVAA-PRP-light-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3824, - "hfopenllm_v2/BBH": 0.5271, - "hfopenllm_v2/MATH Level 5": 0.1103, - "hfopenllm_v2/GPQA": 0.3121, - "hfopenllm_v2/MUSR": 0.4249, - "hfopenllm_v2/MMLU-PRO": 0.3782 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-PRP-v23-8B", - "name": "Kosmos-EVAA-PRP-v23-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4041, - "hfopenllm_v2/BBH": 0.529, - "hfopenllm_v2/MATH Level 5": 0.1156, - "hfopenllm_v2/GPQA": 0.3087, - "hfopenllm_v2/MUSR": 0.4368, - "hfopenllm_v2/MMLU-PRO": 0.3706 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-PRP-v24-8B", - "name": "Kosmos-EVAA-PRP-v24-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4259, - "hfopenllm_v2/BBH": 0.5276, - "hfopenllm_v2/MATH Level 5": 0.1103, - "hfopenllm_v2/GPQA": 0.3104, - "hfopenllm_v2/MUSR": 0.429, - "hfopenllm_v2/MMLU-PRO": 0.3779 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-PRP-v25-8B", - "name": "Kosmos-EVAA-PRP-v25-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4421, - "hfopenllm_v2/BBH": 0.5291, - "hfopenllm_v2/MATH Level 5": 0.1186, - "hfopenllm_v2/GPQA": 0.318, - "hfopenllm_v2/MUSR": 0.4303, - "hfopenllm_v2/MMLU-PRO": 0.3716 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-PRP-v26-8B", - "name": "Kosmos-EVAA-PRP-v26-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4414, - "hfopenllm_v2/BBH": 0.5271, - "hfopenllm_v2/MATH Level 5": 0.1133, - "hfopenllm_v2/GPQA": 0.3045, - "hfopenllm_v2/MUSR": 0.4264, - "hfopenllm_v2/MMLU-PRO": 0.3793 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-PRP-v27-8B", - "name": "Kosmos-EVAA-PRP-v27-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4378, - "hfopenllm_v2/BBH": 0.529, - "hfopenllm_v2/MATH Level 5": 0.1193, - "hfopenllm_v2/GPQA": 0.3087, - "hfopenllm_v2/MUSR": 0.4343, - "hfopenllm_v2/MMLU-PRO": 0.3755 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-PRP-v28-8B", - "name": "Kosmos-EVAA-PRP-v28-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4366, - "hfopenllm_v2/BBH": 0.5295, - "hfopenllm_v2/MATH Level 5": 0.1171, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.433, - "hfopenllm_v2/MMLU-PRO": 0.375 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-PRP-v29-8B", - "name": "Kosmos-EVAA-PRP-v29-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4487, - "hfopenllm_v2/BBH": 0.5275, - "hfopenllm_v2/MATH Level 5": 0.1201, - "hfopenllm_v2/GPQA": 0.3104, - "hfopenllm_v2/MUSR": 0.4237, - "hfopenllm_v2/MMLU-PRO": 0.3765 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-PRP-v30-8B", - "name": "Kosmos-EVAA-PRP-v30-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4295, - "hfopenllm_v2/BBH": 0.5328, - "hfopenllm_v2/MATH Level 5": 0.1178, - "hfopenllm_v2/GPQA": 0.3045, - "hfopenllm_v2/MUSR": 0.4263, - "hfopenllm_v2/MMLU-PRO": 0.3938 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-PRP-v31-8B", - "name": "Kosmos-EVAA-PRP-v31-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4399, - "hfopenllm_v2/BBH": 0.5315, - "hfopenllm_v2/MATH Level 5": 0.1133, - "hfopenllm_v2/GPQA": 0.3138, - "hfopenllm_v2/MUSR": 0.4251, - "hfopenllm_v2/MMLU-PRO": 0.3935 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-PRP-v32-8B", - "name": "Kosmos-EVAA-PRP-v32-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4487, - "hfopenllm_v2/BBH": 0.5293, - "hfopenllm_v2/MATH Level 5": 0.1148, - "hfopenllm_v2/GPQA": 0.3163, - "hfopenllm_v2/MUSR": 0.4211, - "hfopenllm_v2/MMLU-PRO": 0.3777 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-PRP-v33-8B", - "name": "Kosmos-EVAA-PRP-v33-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4302, - "hfopenllm_v2/BBH": 0.5321, - "hfopenllm_v2/MATH Level 5": 0.1178, - "hfopenllm_v2/GPQA": 0.3129, - "hfopenllm_v2/MUSR": 0.4184, - "hfopenllm_v2/MMLU-PRO": 0.3909 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-PRP-v34-8B", - "name": "Kosmos-EVAA-PRP-v34-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4563, - "hfopenllm_v2/BBH": 0.5333, - "hfopenllm_v2/MATH Level 5": 0.1125, - "hfopenllm_v2/GPQA": 0.3112, - "hfopenllm_v2/MUSR": 0.4237, - "hfopenllm_v2/MMLU-PRO": 0.3927 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-TSN-8B", - "name": "Kosmos-EVAA-TSN-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4721, - "hfopenllm_v2/BBH": 0.5177, - "hfopenllm_v2/MATH Level 5": 0.1344, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.4329, - "hfopenllm_v2/MMLU-PRO": 0.3816 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-TSN-light-8B", - "name": "Kosmos-EVAA-TSN-light-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4685, - "hfopenllm_v2/BBH": 0.5235, - "hfopenllm_v2/MATH Level 5": 0.1216, - "hfopenllm_v2/GPQA": 0.3045, - "hfopenllm_v2/MUSR": 0.4289, - "hfopenllm_v2/MMLU-PRO": 0.3806 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-TSN-v19-8B", - "name": "Kosmos-EVAA-TSN-v19-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4564, - "hfopenllm_v2/BBH": 0.5316, - "hfopenllm_v2/MATH Level 5": 0.1156, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.4277, - "hfopenllm_v2/MMLU-PRO": 0.379 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-TSN-v20-8B", - "name": "Kosmos-EVAA-TSN-v20-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4423, - "hfopenllm_v2/BBH": 0.525, - "hfopenllm_v2/MATH Level 5": 0.1246, - "hfopenllm_v2/GPQA": 0.3138, - "hfopenllm_v2/MUSR": 0.421, - "hfopenllm_v2/MMLU-PRO": 0.3936 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-TSN-v21-8B", - "name": "Kosmos-EVAA-TSN-v21-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.467, - "hfopenllm_v2/BBH": 0.5248, - "hfopenllm_v2/MATH Level 5": 0.1193, - "hfopenllm_v2/GPQA": 0.3121, - "hfopenllm_v2/MUSR": 0.4343, - "hfopenllm_v2/MMLU-PRO": 0.3816 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-TSN-v22-8B", - "name": "Kosmos-EVAA-TSN-v22-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4673, - "hfopenllm_v2/BBH": 0.5246, - "hfopenllm_v2/MATH Level 5": 0.1133, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.4303, - "hfopenllm_v2/MMLU-PRO": 0.3812 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-v10-8B", - "name": "Kosmos-EVAA-v10-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4262, - "hfopenllm_v2/BBH": 0.5376, - "hfopenllm_v2/MATH Level 5": 0.1246, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.4224, - "hfopenllm_v2/MMLU-PRO": 0.3831 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-v11-8B", - "name": "Kosmos-EVAA-v11-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4426, - "hfopenllm_v2/BBH": 0.5359, - "hfopenllm_v2/MATH Level 5": 0.1322, - "hfopenllm_v2/GPQA": 0.3154, - "hfopenllm_v2/MUSR": 0.4184, - "hfopenllm_v2/MMLU-PRO": 0.3836 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-v12-8B", - "name": "Kosmos-EVAA-v12-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4378, - "hfopenllm_v2/BBH": 0.5349, - "hfopenllm_v2/MATH Level 5": 0.1367, - "hfopenllm_v2/GPQA": 0.3096, - "hfopenllm_v2/MUSR": 0.4211, - "hfopenllm_v2/MMLU-PRO": 0.3836 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-v2-8B", - "name": "Kosmos-EVAA-v2-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4396, - "hfopenllm_v2/BBH": 0.5341, - "hfopenllm_v2/MATH Level 5": 0.1322, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.4211, - "hfopenllm_v2/MMLU-PRO": 0.3826 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-v3-8B", - "name": "Kosmos-EVAA-v3-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4411, - "hfopenllm_v2/BBH": 0.5331, - "hfopenllm_v2/MATH Level 5": 0.1329, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.4224, - "hfopenllm_v2/MMLU-PRO": 0.3821 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-v4-8B", - "name": "Kosmos-EVAA-v4-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4289, - "hfopenllm_v2/BBH": 0.5337, - "hfopenllm_v2/MATH Level 5": 0.1254, - "hfopenllm_v2/GPQA": 0.3003, - "hfopenllm_v2/MUSR": 0.4197, - "hfopenllm_v2/MMLU-PRO": 0.3817 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-v5-8B", - "name": "Kosmos-EVAA-v5-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.446, - "hfopenllm_v2/BBH": 0.5345, - "hfopenllm_v2/MATH Level 5": 0.1261, - "hfopenllm_v2/GPQA": 0.3037, - "hfopenllm_v2/MUSR": 0.4224, - "hfopenllm_v2/MMLU-PRO": 0.3821 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-v6-8B", - "name": "Kosmos-EVAA-v6-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4396, - "hfopenllm_v2/BBH": 0.538, - "hfopenllm_v2/MATH Level 5": 0.1292, - "hfopenllm_v2/GPQA": 0.302, - "hfopenllm_v2/MUSR": 0.4184, - "hfopenllm_v2/MMLU-PRO": 0.3821 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-v7-8B", - "name": "Kosmos-EVAA-v7-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4277, - "hfopenllm_v2/BBH": 0.5335, - "hfopenllm_v2/MATH Level 5": 0.1337, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.4171, - "hfopenllm_v2/MMLU-PRO": 0.3836 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-v8-8B", - "name": "Kosmos-EVAA-v8-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4383, - "hfopenllm_v2/BBH": 0.5359, - "hfopenllm_v2/MATH Level 5": 0.1307, - "hfopenllm_v2/GPQA": 0.3037, - "hfopenllm_v2/MUSR": 0.421, - "hfopenllm_v2/MMLU-PRO": 0.3827 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-v9-8B", - "name": "Kosmos-EVAA-v9-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4369, - "hfopenllm_v2/BBH": 0.5361, - "hfopenllm_v2/MATH Level 5": 0.1276, - "hfopenllm_v2/GPQA": 0.3062, - "hfopenllm_v2/MUSR": 0.4184, - "hfopenllm_v2/MMLU-PRO": 0.382 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-v9-TitanFusion-Mix-8B", - "name": "Kosmos-EVAA-v9-TitanFusion-Mix-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4284, - "hfopenllm_v2/BBH": 0.554, - "hfopenllm_v2/MATH Level 5": 0.1148, - "hfopenllm_v2/GPQA": 0.2878, - "hfopenllm_v2/MUSR": 0.4354, - "hfopenllm_v2/MMLU-PRO": 0.3836 - } - }, - { - "id": "jaspionjader/Kosmos-VENN-8B", - "name": "Kosmos-VENN-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4332, - "hfopenllm_v2/BBH": 0.5318, - "hfopenllm_v2/MATH Level 5": 0.1412, - "hfopenllm_v2/GPQA": 0.2928, - "hfopenllm_v2/MUSR": 0.4211, - "hfopenllm_v2/MMLU-PRO": 0.3801 - } - }, - { - "id": "jaspionjader/kstc-1-8b", - "name": "kstc-1-8b", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4643, - "hfopenllm_v2/BBH": 0.5209, - "hfopenllm_v2/MATH Level 5": 0.1171, - "hfopenllm_v2/GPQA": 0.3171, - "hfopenllm_v2/MUSR": 0.4158, - "hfopenllm_v2/MMLU-PRO": 0.3892 - } - }, - { - "id": "jaspionjader/kstc-11-8b", - "name": "kstc-11-8b", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4757, - "hfopenllm_v2/BBH": 0.5189, - "hfopenllm_v2/MATH Level 5": 0.1201, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.4118, - "hfopenllm_v2/MMLU-PRO": 0.3879 - } - }, - { - "id": "jaspionjader/kstc-4-8b", - "name": "kstc-4-8b", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.477, - "hfopenllm_v2/BBH": 0.5216, - "hfopenllm_v2/MATH Level 5": 0.1239, - "hfopenllm_v2/GPQA": 0.3037, - "hfopenllm_v2/MUSR": 0.4118, - "hfopenllm_v2/MMLU-PRO": 0.3869 - } - }, - { - "id": "jaspionjader/kstc-5-8b", - "name": "kstc-5-8b", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4721, - "hfopenllm_v2/BBH": 0.5211, - "hfopenllm_v2/MATH Level 5": 0.1299, - "hfopenllm_v2/GPQA": 0.3154, - "hfopenllm_v2/MUSR": 0.4224, - "hfopenllm_v2/MMLU-PRO": 0.3892 - } - }, - { - "id": "jaspionjader/kstc-6-8b", - "name": "kstc-6-8b", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4944, - "hfopenllm_v2/BBH": 0.5231, - "hfopenllm_v2/MATH Level 5": 0.1246, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.4105, - "hfopenllm_v2/MMLU-PRO": 0.3857 - } - }, - { - "id": "jaspionjader/kstc-8-8b", - "name": "kstc-8-8b", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.491, - "hfopenllm_v2/BBH": 0.5239, - "hfopenllm_v2/MATH Level 5": 0.1307, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.4211, - "hfopenllm_v2/MMLU-PRO": 0.3889 - } - }, - { - "id": "jaspionjader/kstc-9-8b", - "name": "kstc-9-8b", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4861, - "hfopenllm_v2/BBH": 0.5238, - "hfopenllm_v2/MATH Level 5": 0.136, - "hfopenllm_v2/GPQA": 0.3012, - "hfopenllm_v2/MUSR": 0.4118, - "hfopenllm_v2/MMLU-PRO": 0.3872 - } - }, - { - "id": "jaspionjader/PRP-Kosmos-EVAA-8B", - "name": "PRP-Kosmos-EVAA-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3633, - "hfopenllm_v2/BBH": 0.5237, - "hfopenllm_v2/MATH Level 5": 0.0959, - "hfopenllm_v2/GPQA": 0.3096, - "hfopenllm_v2/MUSR": 0.425, - "hfopenllm_v2/MMLU-PRO": 0.3766 - } - }, - { - "id": "jaspionjader/PRP-Kosmos-EVAA-light-8B", - "name": "PRP-Kosmos-EVAA-light-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4321, - "hfopenllm_v2/BBH": 0.5275, - "hfopenllm_v2/MATH Level 5": 0.1103, - "hfopenllm_v2/GPQA": 0.3221, - "hfopenllm_v2/MUSR": 0.4235, - "hfopenllm_v2/MMLU-PRO": 0.3631 - } - }, - { - "id": "jaspionjader/slu-10", - "name": "slu-10", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.436, - "hfopenllm_v2/BBH": 0.5096, - "hfopenllm_v2/MATH Level 5": 0.0974, - "hfopenllm_v2/GPQA": 0.3138, - "hfopenllm_v2/MUSR": 0.392, - "hfopenllm_v2/MMLU-PRO": 0.3664 - } - }, - { - "id": "jaspionjader/slu-11", - "name": "slu-11", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3725, - "hfopenllm_v2/BBH": 0.489, - "hfopenllm_v2/MATH Level 5": 0.0559, - "hfopenllm_v2/GPQA": 0.3037, - "hfopenllm_v2/MUSR": 0.3919, - "hfopenllm_v2/MMLU-PRO": 0.3382 - } - }, - { - "id": "jaspionjader/slu-13", - "name": "slu-13", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4378, - "hfopenllm_v2/BBH": 0.5097, - "hfopenllm_v2/MATH Level 5": 0.0808, - "hfopenllm_v2/GPQA": 0.3079, - "hfopenllm_v2/MUSR": 0.3814, - "hfopenllm_v2/MMLU-PRO": 0.358 - } - }, - { - "id": "jaspionjader/slu-14", - "name": "slu-14", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4107, - "hfopenllm_v2/BBH": 0.5089, - "hfopenllm_v2/MATH Level 5": 0.0974, - "hfopenllm_v2/GPQA": 0.3079, - "hfopenllm_v2/MUSR": 0.396, - "hfopenllm_v2/MMLU-PRO": 0.3627 - } - }, - { - "id": "jaspionjader/slu-17", - "name": "slu-17", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4217, - "hfopenllm_v2/BBH": 0.5071, - "hfopenllm_v2/MATH Level 5": 0.0853, - "hfopenllm_v2/GPQA": 0.3087, - "hfopenllm_v2/MUSR": 0.3761, - "hfopenllm_v2/MMLU-PRO": 0.3619 - } - }, - { - "id": "jaspionjader/slu-2", - "name": "slu-2", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4016, - "hfopenllm_v2/BBH": 0.5008, - "hfopenllm_v2/MATH Level 5": 0.0634, - "hfopenllm_v2/GPQA": 0.2987, - "hfopenllm_v2/MUSR": 0.3959, - "hfopenllm_v2/MMLU-PRO": 0.3506 - } - }, - { - "id": "jaspionjader/slu-20", - "name": "slu-20", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4393, - "hfopenllm_v2/BBH": 0.5061, - "hfopenllm_v2/MATH Level 5": 0.0869, - "hfopenllm_v2/GPQA": 0.3087, - "hfopenllm_v2/MUSR": 0.3933, - "hfopenllm_v2/MMLU-PRO": 0.3665 - } - }, - { - "id": "jaspionjader/slu-22", - "name": "slu-22", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4321, - "hfopenllm_v2/BBH": 0.5082, - "hfopenllm_v2/MATH Level 5": 0.0793, - "hfopenllm_v2/GPQA": 0.3163, - "hfopenllm_v2/MUSR": 0.3893, - "hfopenllm_v2/MMLU-PRO": 0.365 - } - }, - { - "id": "jaspionjader/slu-23", - "name": "slu-23", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4478, - "hfopenllm_v2/BBH": 0.5132, - "hfopenllm_v2/MATH Level 5": 0.0944, - "hfopenllm_v2/GPQA": 0.3045, - "hfopenllm_v2/MUSR": 0.4092, - "hfopenllm_v2/MMLU-PRO": 0.3725 - } - }, - { - "id": "jaspionjader/slu-25", - "name": "slu-25", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.45, - "hfopenllm_v2/BBH": 0.5095, - "hfopenllm_v2/MATH Level 5": 0.0838, - "hfopenllm_v2/GPQA": 0.3087, - "hfopenllm_v2/MUSR": 0.3946, - "hfopenllm_v2/MMLU-PRO": 0.3684 - } - }, - { - "id": "jaspionjader/slu-29", - "name": "slu-29", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4431, - "hfopenllm_v2/BBH": 0.5096, - "hfopenllm_v2/MATH Level 5": 0.0869, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.3933, - "hfopenllm_v2/MMLU-PRO": 0.3669 - } - }, - { - "id": "jaspionjader/slu-32", - "name": "slu-32", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4516, - "hfopenllm_v2/BBH": 0.5167, - "hfopenllm_v2/MATH Level 5": 0.1073, - "hfopenllm_v2/GPQA": 0.302, - "hfopenllm_v2/MUSR": 0.4039, - "hfopenllm_v2/MMLU-PRO": 0.3766 - } - }, - { - "id": "jaspionjader/slu-33", - "name": "slu-33", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4457, - "hfopenllm_v2/BBH": 0.5081, - "hfopenllm_v2/MATH Level 5": 0.0997, - "hfopenllm_v2/GPQA": 0.3121, - "hfopenllm_v2/MUSR": 0.3867, - "hfopenllm_v2/MMLU-PRO": 0.3679 - } - }, - { - "id": "jaspionjader/slu-34", - "name": "slu-34", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4351, - "hfopenllm_v2/BBH": 0.5077, - "hfopenllm_v2/MATH Level 5": 0.0997, - "hfopenllm_v2/GPQA": 0.3079, - "hfopenllm_v2/MUSR": 0.388, - "hfopenllm_v2/MMLU-PRO": 0.372 - } - }, - { - "id": "jaspionjader/slu-35", - "name": "slu-35", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4242, - "hfopenllm_v2/BBH": 0.5103, - "hfopenllm_v2/MATH Level 5": 0.1012, - "hfopenllm_v2/GPQA": 0.3121, - "hfopenllm_v2/MUSR": 0.3946, - "hfopenllm_v2/MMLU-PRO": 0.3676 - } - }, - { - "id": "jaspionjader/slu-36", - "name": "slu-36", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4518, - "hfopenllm_v2/BBH": 0.5087, - "hfopenllm_v2/MATH Level 5": 0.0906, - "hfopenllm_v2/GPQA": 0.3121, - "hfopenllm_v2/MUSR": 0.3933, - "hfopenllm_v2/MMLU-PRO": 0.3711 - } - }, - { - "id": "jaspionjader/slu-37", - "name": "slu-37", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4534, - "hfopenllm_v2/BBH": 0.51, - "hfopenllm_v2/MATH Level 5": 0.0974, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.3946, - "hfopenllm_v2/MMLU-PRO": 0.3695 - } - }, - { - "id": "jaspionjader/slu-6", - "name": "slu-6", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4117, - "hfopenllm_v2/BBH": 0.5099, - "hfopenllm_v2/MATH Level 5": 0.0944, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.4066, - "hfopenllm_v2/MMLU-PRO": 0.3611 - } - }, - { - "id": "jaspionjader/slu-mix-1", - "name": "slu-mix-1", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4569, - "hfopenllm_v2/BBH": 0.524, - "hfopenllm_v2/MATH Level 5": 0.1118, - "hfopenllm_v2/GPQA": 0.3003, - "hfopenllm_v2/MUSR": 0.4277, - "hfopenllm_v2/MMLU-PRO": 0.393 - } - }, - { - "id": "jaspionjader/sof-1", - "name": "sof-1", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4314, - "hfopenllm_v2/BBH": 0.501, - "hfopenllm_v2/MATH Level 5": 0.114, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.4082, - "hfopenllm_v2/MMLU-PRO": 0.3674 - } - }, - { - "id": "jaspionjader/sof-10", - "name": "sof-10", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4648, - "hfopenllm_v2/BBH": 0.5197, - "hfopenllm_v2/MATH Level 5": 0.1239, - "hfopenllm_v2/GPQA": 0.3062, - "hfopenllm_v2/MUSR": 0.4091, - "hfopenllm_v2/MMLU-PRO": 0.3874 - } - }, - { - "id": "jaspionjader/sof-3", - "name": "sof-3", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4637, - "hfopenllm_v2/BBH": 0.5206, - "hfopenllm_v2/MATH Level 5": 0.1276, - "hfopenllm_v2/GPQA": 0.297, - "hfopenllm_v2/MUSR": 0.4131, - "hfopenllm_v2/MMLU-PRO": 0.3812 - } - }, - { - "id": "jaspionjader/sof-6", - "name": "sof-6", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4354, - "hfopenllm_v2/BBH": 0.5209, - "hfopenllm_v2/MATH Level 5": 0.1299, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.4171, - "hfopenllm_v2/MMLU-PRO": 0.3844 - } - }, - { - "id": "jaspionjader/test-10", - "name": "test-10", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4578, - "hfopenllm_v2/BBH": 0.5316, - "hfopenllm_v2/MATH Level 5": 0.114, - "hfopenllm_v2/GPQA": 0.3196, - "hfopenllm_v2/MUSR": 0.4251, - "hfopenllm_v2/MMLU-PRO": 0.3936 - } - }, - { - "id": "jaspionjader/test-11", - "name": "test-11", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4541, - "hfopenllm_v2/BBH": 0.535, - "hfopenllm_v2/MATH Level 5": 0.1201, - "hfopenllm_v2/GPQA": 0.3238, - "hfopenllm_v2/MUSR": 0.429, - "hfopenllm_v2/MMLU-PRO": 0.3939 - } - }, - { - "id": "jaspionjader/test-12", - "name": "test-12", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4368, - "hfopenllm_v2/BBH": 0.5347, - "hfopenllm_v2/MATH Level 5": 0.108, - "hfopenllm_v2/GPQA": 0.3188, - "hfopenllm_v2/MUSR": 0.425, - "hfopenllm_v2/MMLU-PRO": 0.3935 - } - }, - { - "id": "jaspionjader/test-13", - "name": "test-13", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4581, - "hfopenllm_v2/BBH": 0.5318, - "hfopenllm_v2/MATH Level 5": 0.1057, - "hfopenllm_v2/GPQA": 0.3163, - "hfopenllm_v2/MUSR": 0.4264, - "hfopenllm_v2/MMLU-PRO": 0.3935 - } - }, - { - "id": "jaspionjader/test-14", - "name": "test-14", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4444, - "hfopenllm_v2/BBH": 0.5323, - "hfopenllm_v2/MATH Level 5": 0.1103, - "hfopenllm_v2/GPQA": 0.3146, - "hfopenllm_v2/MUSR": 0.4317, - "hfopenllm_v2/MMLU-PRO": 0.393 - } - }, - { - "id": "jaspionjader/test-15", - "name": "test-15", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4365, - "hfopenllm_v2/BBH": 0.5328, - "hfopenllm_v2/MATH Level 5": 0.1118, - "hfopenllm_v2/GPQA": 0.3121, - "hfopenllm_v2/MUSR": 0.4264, - "hfopenllm_v2/MMLU-PRO": 0.393 - } - }, - { - "id": "jaspionjader/test-16", - "name": "test-16", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4599, - "hfopenllm_v2/BBH": 0.533, - "hfopenllm_v2/MATH Level 5": 0.1095, - "hfopenllm_v2/GPQA": 0.3138, - "hfopenllm_v2/MUSR": 0.4225, - "hfopenllm_v2/MMLU-PRO": 0.393 - } - }, - { - "id": "jaspionjader/test-17", - "name": "test-17", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4267, - "hfopenllm_v2/BBH": 0.5329, - "hfopenllm_v2/MATH Level 5": 0.1103, - "hfopenllm_v2/GPQA": 0.3129, - "hfopenllm_v2/MUSR": 0.429, - "hfopenllm_v2/MMLU-PRO": 0.3929 - } - }, - { - "id": "jaspionjader/test-18", - "name": "test-18", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4392, - "hfopenllm_v2/BBH": 0.5317, - "hfopenllm_v2/MATH Level 5": 0.1148, - "hfopenllm_v2/GPQA": 0.3121, - "hfopenllm_v2/MUSR": 0.4251, - "hfopenllm_v2/MMLU-PRO": 0.393 - } - }, - { - "id": "jaspionjader/test-19", - "name": "test-19", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4401, - "hfopenllm_v2/BBH": 0.5319, - "hfopenllm_v2/MATH Level 5": 0.1095, - "hfopenllm_v2/GPQA": 0.3096, - "hfopenllm_v2/MUSR": 0.4264, - "hfopenllm_v2/MMLU-PRO": 0.3929 - } - }, - { - "id": "jaspionjader/test-20", - "name": "test-20", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4529, - "hfopenllm_v2/BBH": 0.5327, - "hfopenllm_v2/MATH Level 5": 0.1118, - "hfopenllm_v2/GPQA": 0.3138, - "hfopenllm_v2/MUSR": 0.4251, - "hfopenllm_v2/MMLU-PRO": 0.392 - } - }, - { - "id": "jaspionjader/TSN-Kosmos-EVAA-8B", - "name": "TSN-Kosmos-EVAA-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4903, - "hfopenllm_v2/BBH": 0.5347, - "hfopenllm_v2/MATH Level 5": 0.145, - "hfopenllm_v2/GPQA": 0.3205, - "hfopenllm_v2/MUSR": 0.4173, - "hfopenllm_v2/MMLU-PRO": 0.3831 - } - }, - { - "id": "jaspionjader/TSN-Kosmos-EVAA-v2-8B", - "name": "TSN-Kosmos-EVAA-v2-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4667, - "hfopenllm_v2/BBH": 0.5343, - "hfopenllm_v2/MATH Level 5": 0.108, - "hfopenllm_v2/GPQA": 0.3221, - "hfopenllm_v2/MUSR": 0.4186, - "hfopenllm_v2/MMLU-PRO": 0.3762 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/jayasuryajsk.json b/data/developers/jayasuryajsk.json deleted file mode 100644 index bdf1e16539bcea7f4ba0e17c17b7cf1671762ab6..0000000000000000000000000000000000000000 --- a/data/developers/jayasuryajsk.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "jayasuryajsk", - "models": [ - { - "id": "jayasuryajsk/Qwen2.5-3B-reasoner", - "name": "Qwen2.5-3B-reasoner", - "developer": "jayasuryajsk", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.416, - "hfopenllm_v2/BBH": 0.4651, - "hfopenllm_v2/MATH Level 5": 0.2085, - "hfopenllm_v2/GPQA": 0.302, - "hfopenllm_v2/MUSR": 0.4123, - "hfopenllm_v2/MMLU-PRO": 0.3482 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/jayhyeon.json b/data/developers/jayhyeon.json deleted file mode 100644 index 4fff27cca4eef2d04b3884b1bd8e2522654fd9ee..0000000000000000000000000000000000000000 --- a/data/developers/jayhyeon.json +++ /dev/null @@ -1,2441 +0,0 @@ -{ - "developer": "JayHyeon", - "models": [ - { - "id": "JayHyeon/Qwen-0.5B-DPO-1epoch", - "name": "Qwen-0.5B-DPO-1epoch", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2647, - "hfopenllm_v2/BBH": 0.3191, - "hfopenllm_v2/MATH Level 5": 0.0287, - "hfopenllm_v2/GPQA": 0.2525, - "hfopenllm_v2/MUSR": 0.3352, - "hfopenllm_v2/MMLU-PRO": 0.1558 - } - }, - { - "id": "JayHyeon/Qwen-0.5B-DPO-5epoch", - "name": "Qwen-0.5B-DPO-5epoch", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.257, - "hfopenllm_v2/BBH": 0.3112, - "hfopenllm_v2/MATH Level 5": 0.04, - "hfopenllm_v2/GPQA": 0.2433, - "hfopenllm_v2/MUSR": 0.338, - "hfopenllm_v2/MMLU-PRO": 0.1533 - } - }, - { - "id": "JayHyeon/Qwen-0.5B-eDPO-1epoch", - "name": "Qwen-0.5B-eDPO-1epoch", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2623, - "hfopenllm_v2/BBH": 0.3181, - "hfopenllm_v2/MATH Level 5": 0.0347, - "hfopenllm_v2/GPQA": 0.2424, - "hfopenllm_v2/MUSR": 0.3327, - "hfopenllm_v2/MMLU-PRO": 0.1553 - } - }, - { - "id": "JayHyeon/Qwen-0.5B-eDPO-5epoch", - "name": "Qwen-0.5B-eDPO-5epoch", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2477, - "hfopenllm_v2/BBH": 0.3096, - "hfopenllm_v2/MATH Level 5": 0.0234, - "hfopenllm_v2/GPQA": 0.2492, - "hfopenllm_v2/MUSR": 0.3326, - "hfopenllm_v2/MMLU-PRO": 0.1523 - } - }, - { - "id": "JayHyeon/Qwen-0.5B-IRPO-1epoch", - "name": "Qwen-0.5B-IRPO-1epoch", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2589, - "hfopenllm_v2/BBH": 0.3164, - "hfopenllm_v2/MATH Level 5": 0.0317, - "hfopenllm_v2/GPQA": 0.2466, - "hfopenllm_v2/MUSR": 0.3286, - "hfopenllm_v2/MMLU-PRO": 0.15 - } - }, - { - "id": "JayHyeon/Qwen-0.5B-IRPO-5epoch", - "name": "Qwen-0.5B-IRPO-5epoch", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2487, - "hfopenllm_v2/BBH": 0.3189, - "hfopenllm_v2/MATH Level 5": 0.0325, - "hfopenllm_v2/GPQA": 0.2399, - "hfopenllm_v2/MUSR": 0.3287, - "hfopenllm_v2/MMLU-PRO": 0.1507 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-Instruct-SFT", - "name": "Qwen2.5-0.5B-Instruct-SFT", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2768, - "hfopenllm_v2/BBH": 0.3254, - "hfopenllm_v2/MATH Level 5": 0.0393, - "hfopenllm_v2/GPQA": 0.2827, - "hfopenllm_v2/MUSR": 0.3342, - "hfopenllm_v2/MMLU-PRO": 0.152 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-Instruct-SFT-DPO-1epoch_v1", - "name": "Qwen2.5-0.5B-Instruct-SFT-DPO-1epoch_v1", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2469, - "hfopenllm_v2/BBH": 0.326, - "hfopenllm_v2/MATH Level 5": 0.065, - "hfopenllm_v2/GPQA": 0.2727, - "hfopenllm_v2/MUSR": 0.3434, - "hfopenllm_v2/MMLU-PRO": 0.1575 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-Instruct-SFT-IRPO-1epoch_v1", - "name": "Qwen2.5-0.5B-Instruct-SFT-IRPO-1epoch_v1", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2606, - "hfopenllm_v2/BBH": 0.3308, - "hfopenllm_v2/MATH Level 5": 0.0498, - "hfopenllm_v2/GPQA": 0.2802, - "hfopenllm_v2/MUSR": 0.3288, - "hfopenllm_v2/MMLU-PRO": 0.1626 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-Instruct-SFT-MDPO-1epoch_v1", - "name": "Qwen2.5-0.5B-Instruct-SFT-MDPO-1epoch_v1", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2529, - "hfopenllm_v2/BBH": 0.3262, - "hfopenllm_v2/MATH Level 5": 0.0566, - "hfopenllm_v2/GPQA": 0.2685, - "hfopenllm_v2/MUSR": 0.3301, - "hfopenllm_v2/MMLU-PRO": 0.1576 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT", - "name": "Qwen2.5-0.5B-SFT", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1964, - "hfopenllm_v2/BBH": 0.3121, - "hfopenllm_v2/MATH Level 5": 0.0272, - "hfopenllm_v2/GPQA": 0.2785, - "hfopenllm_v2/MUSR": 0.3394, - "hfopenllm_v2/MMLU-PRO": 0.1673 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-1e-4", - "name": "Qwen2.5-0.5B-SFT-1e-4", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.202, - "hfopenllm_v2/BBH": 0.3017, - "hfopenllm_v2/MATH Level 5": 0.0189, - "hfopenllm_v2/GPQA": 0.2508, - "hfopenllm_v2/MUSR": 0.3446, - "hfopenllm_v2/MMLU-PRO": 0.1619 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-1e-4-2ep", - "name": "Qwen2.5-0.5B-SFT-1e-4-2ep", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.214, - "hfopenllm_v2/BBH": 0.3172, - "hfopenllm_v2/MATH Level 5": 0.0264, - "hfopenllm_v2/GPQA": 0.2466, - "hfopenllm_v2/MUSR": 0.3473, - "hfopenllm_v2/MMLU-PRO": 0.1537 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-1e-4-3ep", - "name": "Qwen2.5-0.5B-SFT-1e-4-3ep", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2257, - "hfopenllm_v2/BBH": 0.3064, - "hfopenllm_v2/MATH Level 5": 0.0264, - "hfopenllm_v2/GPQA": 0.2483, - "hfopenllm_v2/MUSR": 0.3661, - "hfopenllm_v2/MMLU-PRO": 0.1532 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-1e-4-5ep", - "name": "Qwen2.5-0.5B-SFT-1e-4-5ep", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1987, - "hfopenllm_v2/BBH": 0.3104, - "hfopenllm_v2/MATH Level 5": 0.0196, - "hfopenllm_v2/GPQA": 0.2534, - "hfopenllm_v2/MUSR": 0.3407, - "hfopenllm_v2/MMLU-PRO": 0.1558 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-1e-5", - "name": "Qwen2.5-0.5B-SFT-1e-5", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1986, - "hfopenllm_v2/BBH": 0.314, - "hfopenllm_v2/MATH Level 5": 0.0378, - "hfopenllm_v2/GPQA": 0.2685, - "hfopenllm_v2/MUSR": 0.346, - "hfopenllm_v2/MMLU-PRO": 0.1698 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-1e-5-2ep", - "name": "Qwen2.5-0.5B-SFT-1e-5-2ep", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1971, - "hfopenllm_v2/BBH": 0.3225, - "hfopenllm_v2/MATH Level 5": 0.0529, - "hfopenllm_v2/GPQA": 0.2693, - "hfopenllm_v2/MUSR": 0.3368, - "hfopenllm_v2/MMLU-PRO": 0.1651 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-1e-5-3ep", - "name": "Qwen2.5-0.5B-SFT-1e-5-3ep", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2241, - "hfopenllm_v2/BBH": 0.3247, - "hfopenllm_v2/MATH Level 5": 0.0536, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.3353, - "hfopenllm_v2/MMLU-PRO": 0.1689 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-1e-5-5ep", - "name": "Qwen2.5-0.5B-SFT-1e-5-5ep", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2292, - "hfopenllm_v2/BBH": 0.3259, - "hfopenllm_v2/MATH Level 5": 0.0521, - "hfopenllm_v2/GPQA": 0.2794, - "hfopenllm_v2/MUSR": 0.3235, - "hfopenllm_v2/MMLU-PRO": 0.1688 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-4", - "name": "Qwen2.5-0.5B-SFT-2e-4", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2034, - "hfopenllm_v2/BBH": 0.2936, - "hfopenllm_v2/MATH Level 5": 0.0242, - "hfopenllm_v2/GPQA": 0.2576, - "hfopenllm_v2/MUSR": 0.3434, - "hfopenllm_v2/MMLU-PRO": 0.1413 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-4-2ep", - "name": "Qwen2.5-0.5B-SFT-2e-4-2ep", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1831, - "hfopenllm_v2/BBH": 0.2984, - "hfopenllm_v2/MATH Level 5": 0.0249, - "hfopenllm_v2/GPQA": 0.2424, - "hfopenllm_v2/MUSR": 0.3568, - "hfopenllm_v2/MMLU-PRO": 0.1484 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-4-3ep", - "name": "Qwen2.5-0.5B-SFT-2e-4-3ep", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.199, - "hfopenllm_v2/BBH": 0.311, - "hfopenllm_v2/MATH Level 5": 0.0151, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.3449, - "hfopenllm_v2/MMLU-PRO": 0.1416 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-4-5ep", - "name": "Qwen2.5-0.5B-SFT-2e-4-5ep", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1897, - "hfopenllm_v2/BBH": 0.2936, - "hfopenllm_v2/MATH Level 5": 0.0181, - "hfopenllm_v2/GPQA": 0.2693, - "hfopenllm_v2/MUSR": 0.3874, - "hfopenllm_v2/MMLU-PRO": 0.1336 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5", - "name": "Qwen2.5-0.5B-SFT-2e-5", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2068, - "hfopenllm_v2/BBH": 0.3204, - "hfopenllm_v2/MATH Level 5": 0.037, - "hfopenllm_v2/GPQA": 0.2693, - "hfopenllm_v2/MUSR": 0.3487, - "hfopenllm_v2/MMLU-PRO": 0.1678 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2201, - "hfopenllm_v2/BBH": 0.3217, - "hfopenllm_v2/MATH Level 5": 0.0408, - "hfopenllm_v2/GPQA": 0.2777, - "hfopenllm_v2/MUSR": 0.3367, - "hfopenllm_v2/MMLU-PRO": 0.171 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_1e-6-1ep_0alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_1e-6-1ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2542, - "hfopenllm_v2/BBH": 0.3167, - "hfopenllm_v2/MATH Level 5": 0.0408, - "hfopenllm_v2/GPQA": 0.2718, - "hfopenllm_v2/MUSR": 0.3289, - "hfopenllm_v2/MMLU-PRO": 0.158 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_1e-6-2ep_0alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_1e-6-2ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2451, - "hfopenllm_v2/BBH": 0.316, - "hfopenllm_v2/MATH Level 5": 0.0408, - "hfopenllm_v2/GPQA": 0.2743, - "hfopenllm_v2/MUSR": 0.3302, - "hfopenllm_v2/MMLU-PRO": 0.1561 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_1e-6-3ep_0alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_1e-6-3ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2557, - "hfopenllm_v2/BBH": 0.3142, - "hfopenllm_v2/MATH Level 5": 0.04, - "hfopenllm_v2/GPQA": 0.2743, - "hfopenllm_v2/MUSR": 0.3315, - "hfopenllm_v2/MMLU-PRO": 0.1575 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_1e-7-2ep_0alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_1e-7-2ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2605, - "hfopenllm_v2/BBH": 0.3167, - "hfopenllm_v2/MATH Level 5": 0.0363, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.3341, - "hfopenllm_v2/MMLU-PRO": 0.1577 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_1e-7-3ep_0alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_1e-7-3ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2578, - "hfopenllm_v2/BBH": 0.3173, - "hfopenllm_v2/MATH Level 5": 0.0355, - "hfopenllm_v2/GPQA": 0.2634, - "hfopenllm_v2/MUSR": 0.3288, - "hfopenllm_v2/MMLU-PRO": 0.1583 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_2e-6-1ep_0alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_2e-6-1ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2335, - "hfopenllm_v2/BBH": 0.3198, - "hfopenllm_v2/MATH Level 5": 0.0385, - "hfopenllm_v2/GPQA": 0.2752, - "hfopenllm_v2/MUSR": 0.3276, - "hfopenllm_v2/MMLU-PRO": 0.1581 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_2e-6-2ep_0alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_2e-6-2ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2472, - "hfopenllm_v2/BBH": 0.3226, - "hfopenllm_v2/MATH Level 5": 0.0506, - "hfopenllm_v2/GPQA": 0.276, - "hfopenllm_v2/MUSR": 0.3262, - "hfopenllm_v2/MMLU-PRO": 0.1538 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_2e-6-3ep_0alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_2e-6-3ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2474, - "hfopenllm_v2/BBH": 0.3229, - "hfopenllm_v2/MATH Level 5": 0.0415, - "hfopenllm_v2/GPQA": 0.2727, - "hfopenllm_v2/MUSR": 0.3275, - "hfopenllm_v2/MMLU-PRO": 0.1539 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_3e-6-1ep_0alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_3e-6-1ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2403, - "hfopenllm_v2/BBH": 0.3245, - "hfopenllm_v2/MATH Level 5": 0.0431, - "hfopenllm_v2/GPQA": 0.2819, - "hfopenllm_v2/MUSR": 0.3262, - "hfopenllm_v2/MMLU-PRO": 0.1573 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_3e-6-2ep_0alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_3e-6-2ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2368, - "hfopenllm_v2/BBH": 0.3224, - "hfopenllm_v2/MATH Level 5": 0.0461, - "hfopenllm_v2/GPQA": 0.2743, - "hfopenllm_v2/MUSR": 0.3355, - "hfopenllm_v2/MMLU-PRO": 0.1516 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_3e-6-3ep_0alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_3e-6-3ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2372, - "hfopenllm_v2/BBH": 0.3248, - "hfopenllm_v2/MATH Level 5": 0.0476, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.3394, - "hfopenllm_v2/MMLU-PRO": 0.155 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_3e-7-3ep_0alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_3e-7-3ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2499, - "hfopenllm_v2/BBH": 0.3181, - "hfopenllm_v2/MATH Level 5": 0.0415, - "hfopenllm_v2/GPQA": 0.2651, - "hfopenllm_v2/MUSR": 0.3288, - "hfopenllm_v2/MMLU-PRO": 0.1574 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_5e-6-1ep_0alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_5e-6-1ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2381, - "hfopenllm_v2/BBH": 0.3242, - "hfopenllm_v2/MATH Level 5": 0.0498, - "hfopenllm_v2/GPQA": 0.2743, - "hfopenllm_v2/MUSR": 0.3328, - "hfopenllm_v2/MMLU-PRO": 0.1572 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_5e-6-2ep_0alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_5e-6-2ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2421, - "hfopenllm_v2/BBH": 0.3225, - "hfopenllm_v2/MATH Level 5": 0.04, - "hfopenllm_v2/GPQA": 0.2802, - "hfopenllm_v2/MUSR": 0.3408, - "hfopenllm_v2/MMLU-PRO": 0.1496 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_5e-6-3ep_0alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_5e-6-3ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2381, - "hfopenllm_v2/BBH": 0.3265, - "hfopenllm_v2/MATH Level 5": 0.0446, - "hfopenllm_v2/GPQA": 0.276, - "hfopenllm_v2/MUSR": 0.3408, - "hfopenllm_v2/MMLU-PRO": 0.1499 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_5e-7_1ep_0alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_5e-7_1ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2526, - "hfopenllm_v2/BBH": 0.3177, - "hfopenllm_v2/MATH Level 5": 0.0438, - "hfopenllm_v2/GPQA": 0.2735, - "hfopenllm_v2/MUSR": 0.3342, - "hfopenllm_v2/MMLU-PRO": 0.1572 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_5e-7_2ep_0alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_5e-7_2ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2457, - "hfopenllm_v2/BBH": 0.316, - "hfopenllm_v2/MATH Level 5": 0.0446, - "hfopenllm_v2/GPQA": 0.2727, - "hfopenllm_v2/MUSR": 0.3302, - "hfopenllm_v2/MMLU-PRO": 0.1572 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_5e-7_3ep_0alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_5e-7_3ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2442, - "hfopenllm_v2/BBH": 0.3194, - "hfopenllm_v2/MATH Level 5": 0.0483, - "hfopenllm_v2/GPQA": 0.2735, - "hfopenllm_v2/MUSR": 0.3315, - "hfopenllm_v2/MMLU-PRO": 0.1567 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_7e-7_1ep_0alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_7e-7_1ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2604, - "hfopenllm_v2/BBH": 0.3178, - "hfopenllm_v2/MATH Level 5": 0.0355, - "hfopenllm_v2/GPQA": 0.276, - "hfopenllm_v2/MUSR": 0.3288, - "hfopenllm_v2/MMLU-PRO": 0.1567 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_7e-7_2ep_0alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_7e-7_2ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.249, - "hfopenllm_v2/BBH": 0.3173, - "hfopenllm_v2/MATH Level 5": 0.0393, - "hfopenllm_v2/GPQA": 0.271, - "hfopenllm_v2/MUSR": 0.3302, - "hfopenllm_v2/MMLU-PRO": 0.1569 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_7e-7_3ep_0alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_7e-7_3ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2604, - "hfopenllm_v2/BBH": 0.315, - "hfopenllm_v2/MATH Level 5": 0.0378, - "hfopenllm_v2/GPQA": 0.2743, - "hfopenllm_v2/MUSR": 0.3342, - "hfopenllm_v2/MMLU-PRO": 0.1566 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-DPOP_3e-7-3ep_0alp_5lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-DPOP_3e-7-3ep_0alp_5lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2411, - "hfopenllm_v2/BBH": 0.3167, - "hfopenllm_v2/MATH Level 5": 0.0347, - "hfopenllm_v2/GPQA": 0.271, - "hfopenllm_v2/MUSR": 0.3301, - "hfopenllm_v2/MMLU-PRO": 0.1562 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-DPOP_5e-6-1ep_0alp_5lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-DPOP_5e-6-1ep_0alp_5lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2369, - "hfopenllm_v2/BBH": 0.326, - "hfopenllm_v2/MATH Level 5": 0.0453, - "hfopenllm_v2/GPQA": 0.276, - "hfopenllm_v2/MUSR": 0.3355, - "hfopenllm_v2/MMLU-PRO": 0.157 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-DPOP_5e-6-2ep_0alp_5lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-DPOP_5e-6-2ep_0alp_5lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2262, - "hfopenllm_v2/BBH": 0.3262, - "hfopenllm_v2/MATH Level 5": 0.0347, - "hfopenllm_v2/GPQA": 0.2794, - "hfopenllm_v2/MUSR": 0.3408, - "hfopenllm_v2/MMLU-PRO": 0.1541 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-DPOP_5e-6-3ep_0alp_5lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-DPOP_5e-6-3ep_0alp_5lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2508, - "hfopenllm_v2/BBH": 0.3199, - "hfopenllm_v2/MATH Level 5": 0.0408, - "hfopenllm_v2/GPQA": 0.276, - "hfopenllm_v2/MUSR": 0.3355, - "hfopenllm_v2/MMLU-PRO": 0.1555 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-DPOP_5e-7-1ep_0alp_5lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-DPOP_5e-7-1ep_0alp_5lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.239, - "hfopenllm_v2/BBH": 0.3182, - "hfopenllm_v2/MATH Level 5": 0.04, - "hfopenllm_v2/GPQA": 0.2676, - "hfopenllm_v2/MUSR": 0.3328, - "hfopenllm_v2/MMLU-PRO": 0.156 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-DPOP_5e-7-2ep_0alp_5lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-DPOP_5e-7-2ep_0alp_5lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2423, - "hfopenllm_v2/BBH": 0.3154, - "hfopenllm_v2/MATH Level 5": 0.0347, - "hfopenllm_v2/GPQA": 0.2676, - "hfopenllm_v2/MUSR": 0.3328, - "hfopenllm_v2/MMLU-PRO": 0.1548 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-DPOP_5e-7-3ep_0alp_5lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-DPOP_5e-7-3ep_0alp_5lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2493, - "hfopenllm_v2/BBH": 0.319, - "hfopenllm_v2/MATH Level 5": 0.0438, - "hfopenllm_v2/GPQA": 0.2651, - "hfopenllm_v2/MUSR": 0.3341, - "hfopenllm_v2/MMLU-PRO": 0.1561 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-IRPO_1e-7-1ep_1alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-IRPO_1e-7-1ep_1alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.255, - "hfopenllm_v2/BBH": 0.3211, - "hfopenllm_v2/MATH Level 5": 0.0491, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.3288, - "hfopenllm_v2/MMLU-PRO": 0.1571 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-IRPO_1e-7-2ep_1alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-IRPO_1e-7-2ep_1alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2478, - "hfopenllm_v2/BBH": 0.3198, - "hfopenllm_v2/MATH Level 5": 0.0423, - "hfopenllm_v2/GPQA": 0.2668, - "hfopenllm_v2/MUSR": 0.3315, - "hfopenllm_v2/MMLU-PRO": 0.1587 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-IRPO_1e-7-3ep_1alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-IRPO_1e-7-3ep_1alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2475, - "hfopenllm_v2/BBH": 0.3225, - "hfopenllm_v2/MATH Level 5": 0.04, - "hfopenllm_v2/GPQA": 0.271, - "hfopenllm_v2/MUSR": 0.3301, - "hfopenllm_v2/MMLU-PRO": 0.1556 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-IRPO_3e-7-3ep_1alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-IRPO_3e-7-3ep_1alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.259, - "hfopenllm_v2/BBH": 0.3185, - "hfopenllm_v2/MATH Level 5": 0.0363, - "hfopenllm_v2/GPQA": 0.2727, - "hfopenllm_v2/MUSR": 0.3275, - "hfopenllm_v2/MMLU-PRO": 0.1586 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-IRPO_5e-6-1ep_1alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-IRPO_5e-6-1ep_1alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2323, - "hfopenllm_v2/BBH": 0.3179, - "hfopenllm_v2/MATH Level 5": 0.0453, - "hfopenllm_v2/GPQA": 0.2827, - "hfopenllm_v2/MUSR": 0.3262, - "hfopenllm_v2/MMLU-PRO": 0.1548 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-IRPO_5e-6-2ep_1alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-IRPO_5e-6-2ep_1alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2315, - "hfopenllm_v2/BBH": 0.326, - "hfopenllm_v2/MATH Level 5": 0.0415, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.3383, - "hfopenllm_v2/MMLU-PRO": 0.1521 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-IRPO_5e-6-3ep_1alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-IRPO_5e-6-3ep_1alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2298, - "hfopenllm_v2/BBH": 0.332, - "hfopenllm_v2/MATH Level 5": 0.0431, - "hfopenllm_v2/GPQA": 0.2659, - "hfopenllm_v2/MUSR": 0.3329, - "hfopenllm_v2/MMLU-PRO": 0.1567 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-IRPO_5e-7-1ep_1alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-IRPO_5e-7-1ep_1alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2469, - "hfopenllm_v2/BBH": 0.3179, - "hfopenllm_v2/MATH Level 5": 0.0415, - "hfopenllm_v2/GPQA": 0.2794, - "hfopenllm_v2/MUSR": 0.3302, - "hfopenllm_v2/MMLU-PRO": 0.1575 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-IRPO_5e-7-2ep_1alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-IRPO_5e-7-2ep_1alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.252, - "hfopenllm_v2/BBH": 0.3168, - "hfopenllm_v2/MATH Level 5": 0.037, - "hfopenllm_v2/GPQA": 0.2752, - "hfopenllm_v2/MUSR": 0.3328, - "hfopenllm_v2/MMLU-PRO": 0.1576 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-IRPO_5e-7-3ep_1alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-IRPO_5e-7-3ep_1alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2666, - "hfopenllm_v2/BBH": 0.3191, - "hfopenllm_v2/MATH Level 5": 0.0347, - "hfopenllm_v2/GPQA": 0.2718, - "hfopenllm_v2/MUSR": 0.3289, - "hfopenllm_v2/MMLU-PRO": 0.1567 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_0.5_1e-7-1ep_0alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_0.5_1e-7-1ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2499, - "hfopenllm_v2/BBH": 0.3178, - "hfopenllm_v2/MATH Level 5": 0.037, - "hfopenllm_v2/GPQA": 0.2651, - "hfopenllm_v2/MUSR": 0.3341, - "hfopenllm_v2/MMLU-PRO": 0.1562 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_0.5_1e-7-2ep_0alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_0.5_1e-7-2ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2417, - "hfopenllm_v2/BBH": 0.3178, - "hfopenllm_v2/MATH Level 5": 0.04, - "hfopenllm_v2/GPQA": 0.2685, - "hfopenllm_v2/MUSR": 0.3328, - "hfopenllm_v2/MMLU-PRO": 0.1575 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_0.5_1e-7-3ep_0alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_0.5_1e-7-3ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2562, - "hfopenllm_v2/BBH": 0.319, - "hfopenllm_v2/MATH Level 5": 0.0423, - "hfopenllm_v2/GPQA": 0.2659, - "hfopenllm_v2/MUSR": 0.3341, - "hfopenllm_v2/MMLU-PRO": 0.1576 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_1e-6-3ep_0alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_1e-6-3ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2408, - "hfopenllm_v2/BBH": 0.3165, - "hfopenllm_v2/MATH Level 5": 0.0431, - "hfopenllm_v2/GPQA": 0.2735, - "hfopenllm_v2/MUSR": 0.3315, - "hfopenllm_v2/MMLU-PRO": 0.1557 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_1e-6_1ep_0alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_1e-6_1ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2481, - "hfopenllm_v2/BBH": 0.3204, - "hfopenllm_v2/MATH Level 5": 0.0476, - "hfopenllm_v2/GPQA": 0.276, - "hfopenllm_v2/MUSR": 0.3302, - "hfopenllm_v2/MMLU-PRO": 0.1592 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_1e-6_2ep_0alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_1e-6_2ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2545, - "hfopenllm_v2/BBH": 0.3186, - "hfopenllm_v2/MATH Level 5": 0.0498, - "hfopenllm_v2/GPQA": 0.2718, - "hfopenllm_v2/MUSR": 0.3289, - "hfopenllm_v2/MMLU-PRO": 0.1561 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_2e-6-3ep_0alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_2e-6-3ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.252, - "hfopenllm_v2/BBH": 0.3204, - "hfopenllm_v2/MATH Level 5": 0.0393, - "hfopenllm_v2/GPQA": 0.2727, - "hfopenllm_v2/MUSR": 0.3262, - "hfopenllm_v2/MMLU-PRO": 0.1538 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_2e-6_1ep_0alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_2e-6_1ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2315, - "hfopenllm_v2/BBH": 0.3213, - "hfopenllm_v2/MATH Level 5": 0.0453, - "hfopenllm_v2/GPQA": 0.2802, - "hfopenllm_v2/MUSR": 0.3222, - "hfopenllm_v2/MMLU-PRO": 0.1582 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_2e-6_2ep_0alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_2e-6_2ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2515, - "hfopenllm_v2/BBH": 0.3187, - "hfopenllm_v2/MATH Level 5": 0.0431, - "hfopenllm_v2/GPQA": 0.2718, - "hfopenllm_v2/MUSR": 0.3289, - "hfopenllm_v2/MMLU-PRO": 0.1539 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_3e-6-1ep_0alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_3e-6-1ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2472, - "hfopenllm_v2/BBH": 0.3213, - "hfopenllm_v2/MATH Level 5": 0.0347, - "hfopenllm_v2/GPQA": 0.2727, - "hfopenllm_v2/MUSR": 0.3262, - "hfopenllm_v2/MMLU-PRO": 0.1588 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_3e-6-2ep_0alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_3e-6-2ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.246, - "hfopenllm_v2/BBH": 0.3234, - "hfopenllm_v2/MATH Level 5": 0.0378, - "hfopenllm_v2/GPQA": 0.2794, - "hfopenllm_v2/MUSR": 0.3302, - "hfopenllm_v2/MMLU-PRO": 0.1533 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_3e-6-3ep_0alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_3e-6-3ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2524, - "hfopenllm_v2/BBH": 0.3256, - "hfopenllm_v2/MATH Level 5": 0.0536, - "hfopenllm_v2/GPQA": 0.2777, - "hfopenllm_v2/MUSR": 0.3368, - "hfopenllm_v2/MMLU-PRO": 0.1531 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_5e-6-1ep_0alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_5e-6-1ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2265, - "hfopenllm_v2/BBH": 0.3252, - "hfopenllm_v2/MATH Level 5": 0.0476, - "hfopenllm_v2/GPQA": 0.2735, - "hfopenllm_v2/MUSR": 0.3262, - "hfopenllm_v2/MMLU-PRO": 0.1568 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_5e-6-2ep_0alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_5e-6-2ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2302, - "hfopenllm_v2/BBH": 0.3224, - "hfopenllm_v2/MATH Level 5": 0.0438, - "hfopenllm_v2/GPQA": 0.2768, - "hfopenllm_v2/MUSR": 0.3408, - "hfopenllm_v2/MMLU-PRO": 0.15 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_5e-6-3ep_0alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_5e-6-3ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2524, - "hfopenllm_v2/BBH": 0.3278, - "hfopenllm_v2/MATH Level 5": 0.0408, - "hfopenllm_v2/GPQA": 0.2777, - "hfopenllm_v2/MUSR": 0.3395, - "hfopenllm_v2/MMLU-PRO": 0.1521 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_5e-7-3ep_0alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_5e-7-3ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2658, - "hfopenllm_v2/BBH": 0.3175, - "hfopenllm_v2/MATH Level 5": 0.0363, - "hfopenllm_v2/GPQA": 0.2617, - "hfopenllm_v2/MUSR": 0.3302, - "hfopenllm_v2/MMLU-PRO": 0.1575 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_5e-7_1ep_0alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_5e-7_1ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2487, - "hfopenllm_v2/BBH": 0.3189, - "hfopenllm_v2/MATH Level 5": 0.0378, - "hfopenllm_v2/GPQA": 0.2718, - "hfopenllm_v2/MUSR": 0.3275, - "hfopenllm_v2/MMLU-PRO": 0.1595 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_5e-7_2ep_0alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_5e-7_2ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.256, - "hfopenllm_v2/BBH": 0.3159, - "hfopenllm_v2/MATH Level 5": 0.0378, - "hfopenllm_v2/GPQA": 0.2768, - "hfopenllm_v2/MUSR": 0.3275, - "hfopenllm_v2/MMLU-PRO": 0.1562 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_7e-7-3ep_0alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_7e-7-3ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2499, - "hfopenllm_v2/BBH": 0.3156, - "hfopenllm_v2/MATH Level 5": 0.04, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.3302, - "hfopenllm_v2/MMLU-PRO": 0.1556 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_7e-7_1ep_0alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_7e-7_1ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2496, - "hfopenllm_v2/BBH": 0.3177, - "hfopenllm_v2/MATH Level 5": 0.0453, - "hfopenllm_v2/GPQA": 0.2626, - "hfopenllm_v2/MUSR": 0.3315, - "hfopenllm_v2/MMLU-PRO": 0.1567 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_7e-7_2ep_0alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_7e-7_2ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2515, - "hfopenllm_v2/BBH": 0.3172, - "hfopenllm_v2/MATH Level 5": 0.0438, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.3275, - "hfopenllm_v2/MMLU-PRO": 0.1553 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-3ep", - "name": "Qwen2.5-0.5B-SFT-2e-5-3ep", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2281, - "hfopenllm_v2/BBH": 0.324, - "hfopenllm_v2/MATH Level 5": 0.0453, - "hfopenllm_v2/GPQA": 0.2617, - "hfopenllm_v2/MUSR": 0.3301, - "hfopenllm_v2/MMLU-PRO": 0.1746 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-5ep", - "name": "Qwen2.5-0.5B-SFT-2e-5-5ep", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2348, - "hfopenllm_v2/BBH": 0.3308, - "hfopenllm_v2/MATH Level 5": 0.0506, - "hfopenllm_v2/GPQA": 0.2643, - "hfopenllm_v2/MUSR": 0.3409, - "hfopenllm_v2/MMLU-PRO": 0.1695 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-5ep-MDPO_5e-7_3ep_0alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-5ep-MDPO_5e-7_3ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2526, - "hfopenllm_v2/BBH": 0.3238, - "hfopenllm_v2/MATH Level 5": 0.0393, - "hfopenllm_v2/GPQA": 0.2676, - "hfopenllm_v2/MUSR": 0.3528, - "hfopenllm_v2/MMLU-PRO": 0.1574 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-5ep-MDPO_5e-7_3ep_0alp_0lam_1ep", - "name": "Qwen2.5-0.5B-SFT-2e-5-5ep-MDPO_5e-7_3ep_0alp_0lam_1ep", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2481, - "hfopenllm_v2/BBH": 0.3175, - "hfopenllm_v2/MATH Level 5": 0.0385, - "hfopenllm_v2/GPQA": 0.2626, - "hfopenllm_v2/MUSR": 0.3475, - "hfopenllm_v2/MMLU-PRO": 0.1597 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-5ep-MDPO_5e-7_3ep_0alp_0lam_2ep", - "name": "Qwen2.5-0.5B-SFT-2e-5-5ep-MDPO_5e-7_3ep_0alp_0lam_2ep", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2548, - "hfopenllm_v2/BBH": 0.3199, - "hfopenllm_v2/MATH Level 5": 0.0385, - "hfopenllm_v2/GPQA": 0.2651, - "hfopenllm_v2/MUSR": 0.3435, - "hfopenllm_v2/MMLU-PRO": 0.1562 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-5ep-MDPO_7e-7_3ep_0alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-5ep-MDPO_7e-7_3ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2423, - "hfopenllm_v2/BBH": 0.3219, - "hfopenllm_v2/MATH Level 5": 0.034, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.3515, - "hfopenllm_v2/MMLU-PRO": 0.1563 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-5ep-MDPO_7e-7_3ep_0alp_0lam_1ep", - "name": "Qwen2.5-0.5B-SFT-2e-5-5ep-MDPO_7e-7_3ep_0alp_0lam_1ep", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2493, - "hfopenllm_v2/BBH": 0.3191, - "hfopenllm_v2/MATH Level 5": 0.0393, - "hfopenllm_v2/GPQA": 0.2685, - "hfopenllm_v2/MUSR": 0.3475, - "hfopenllm_v2/MMLU-PRO": 0.1592 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-5ep-MDPO_7e-7_3ep_0alp_0lam_2ep", - "name": "Qwen2.5-0.5B-SFT-2e-5-5ep-MDPO_7e-7_3ep_0alp_0lam_2ep", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2478, - "hfopenllm_v2/BBH": 0.3218, - "hfopenllm_v2/MATH Level 5": 0.0415, - "hfopenllm_v2/GPQA": 0.2693, - "hfopenllm_v2/MUSR": 0.3515, - "hfopenllm_v2/MMLU-PRO": 0.1556 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-5e-5", - "name": "Qwen2.5-0.5B-SFT-5e-5", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.201, - "hfopenllm_v2/BBH": 0.3109, - "hfopenllm_v2/MATH Level 5": 0.034, - "hfopenllm_v2/GPQA": 0.2676, - "hfopenllm_v2/MUSR": 0.3381, - "hfopenllm_v2/MMLU-PRO": 0.1672 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-5e-5-2ep", - "name": "Qwen2.5-0.5B-SFT-5e-5-2ep", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2175, - "hfopenllm_v2/BBH": 0.318, - "hfopenllm_v2/MATH Level 5": 0.0378, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3368, - "hfopenllm_v2/MMLU-PRO": 0.1627 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-5e-5-3ep", - "name": "Qwen2.5-0.5B-SFT-5e-5-3ep", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2199, - "hfopenllm_v2/BBH": 0.3297, - "hfopenllm_v2/MATH Level 5": 0.0302, - "hfopenllm_v2/GPQA": 0.2534, - "hfopenllm_v2/MUSR": 0.3593, - "hfopenllm_v2/MMLU-PRO": 0.1651 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-5e-5-5ep", - "name": "Qwen2.5-0.5B-SFT-5e-5-5ep", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2077, - "hfopenllm_v2/BBH": 0.3276, - "hfopenllm_v2/MATH Level 5": 0.0272, - "hfopenllm_v2/GPQA": 0.2685, - "hfopenllm_v2/MUSR": 0.3766, - "hfopenllm_v2/MMLU-PRO": 0.1587 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-7e-5", - "name": "Qwen2.5-0.5B-SFT-7e-5", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2093, - "hfopenllm_v2/BBH": 0.3158, - "hfopenllm_v2/MATH Level 5": 0.0302, - "hfopenllm_v2/GPQA": 0.2567, - "hfopenllm_v2/MUSR": 0.3367, - "hfopenllm_v2/MMLU-PRO": 0.1622 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-7e-5-2ep", - "name": "Qwen2.5-0.5B-SFT-7e-5-2ep", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2156, - "hfopenllm_v2/BBH": 0.31, - "hfopenllm_v2/MATH Level 5": 0.0393, - "hfopenllm_v2/GPQA": 0.2424, - "hfopenllm_v2/MUSR": 0.3367, - "hfopenllm_v2/MMLU-PRO": 0.1567 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-7e-5-3ep", - "name": "Qwen2.5-0.5B-SFT-7e-5-3ep", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2381, - "hfopenllm_v2/BBH": 0.3199, - "hfopenllm_v2/MATH Level 5": 0.0332, - "hfopenllm_v2/GPQA": 0.2366, - "hfopenllm_v2/MUSR": 0.3554, - "hfopenllm_v2/MMLU-PRO": 0.1522 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-7e-5-5ep", - "name": "Qwen2.5-0.5B-SFT-7e-5-5ep", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.212, - "hfopenllm_v2/BBH": 0.32, - "hfopenllm_v2/MATH Level 5": 0.0219, - "hfopenllm_v2/GPQA": 0.2458, - "hfopenllm_v2/MUSR": 0.3713, - "hfopenllm_v2/MMLU-PRO": 0.1628 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-DPO-1epoch_v1", - "name": "Qwen2.5-0.5B-SFT-DPO-1epoch_v1", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2025, - "hfopenllm_v2/BBH": 0.3268, - "hfopenllm_v2/MATH Level 5": 0.0363, - "hfopenllm_v2/GPQA": 0.2727, - "hfopenllm_v2/MUSR": 0.3209, - "hfopenllm_v2/MMLU-PRO": 0.133 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-MDPO-1epoch_v1", - "name": "Qwen2.5-0.5B-SFT-MDPO-1epoch_v1", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1964, - "hfopenllm_v2/BBH": 0.3293, - "hfopenllm_v2/MATH Level 5": 0.0468, - "hfopenllm_v2/GPQA": 0.276, - "hfopenllm_v2/MUSR": 0.3262, - "hfopenllm_v2/MMLU-PRO": 0.1337 - } - }, - { - "id": "JayHyeon/Qwen_0.5-cDPO_5e-7-3ep_0vpo_const_0.1", - "name": "Qwen_0.5-cDPO_5e-7-3ep_0vpo_const_0.1", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2393, - "hfopenllm_v2/BBH": 0.3244, - "hfopenllm_v2/MATH Level 5": 0.0514, - "hfopenllm_v2/GPQA": 0.2777, - "hfopenllm_v2/MUSR": 0.3222, - "hfopenllm_v2/MMLU-PRO": 0.1573 - } - }, - { - "id": "JayHyeon/Qwen_0.5-cDPO_5e-7-3ep_0vpo_const_0.3", - "name": "Qwen_0.5-cDPO_5e-7-3ep_0vpo_const_0.3", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2475, - "hfopenllm_v2/BBH": 0.3209, - "hfopenllm_v2/MATH Level 5": 0.0461, - "hfopenllm_v2/GPQA": 0.281, - "hfopenllm_v2/MUSR": 0.3275, - "hfopenllm_v2/MMLU-PRO": 0.1567 - } - }, - { - "id": "JayHyeon/Qwen_0.5-DPO_1e-6-3ep_0alp_0lam", - "name": "Qwen_0.5-DPO_1e-6-3ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2316, - "hfopenllm_v2/BBH": 0.3258, - "hfopenllm_v2/MATH Level 5": 0.0529, - "hfopenllm_v2/GPQA": 0.2693, - "hfopenllm_v2/MUSR": 0.3221, - "hfopenllm_v2/MMLU-PRO": 0.158 - } - }, - { - "id": "JayHyeon/Qwen_0.5-DPO_1e-7-3ep_0alp_0lam", - "name": "Qwen_0.5-DPO_1e-7-3ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.236, - "hfopenllm_v2/BBH": 0.3225, - "hfopenllm_v2/MATH Level 5": 0.0438, - "hfopenllm_v2/GPQA": 0.271, - "hfopenllm_v2/MUSR": 0.3222, - "hfopenllm_v2/MMLU-PRO": 0.1596 - } - }, - { - "id": "JayHyeon/Qwen_0.5-DPO_3e-6-1ep_0alp_0lam", - "name": "Qwen_0.5-DPO_3e-6-1ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2337, - "hfopenllm_v2/BBH": 0.3132, - "hfopenllm_v2/MATH Level 5": 0.0347, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.3235, - "hfopenllm_v2/MMLU-PRO": 0.1533 - } - }, - { - "id": "JayHyeon/Qwen_0.5-DPO_3e-6-2ep_0alp_0lam", - "name": "Qwen_0.5-DPO_3e-6-2ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2569, - "hfopenllm_v2/BBH": 0.3276, - "hfopenllm_v2/MATH Level 5": 0.0544, - "hfopenllm_v2/GPQA": 0.2718, - "hfopenllm_v2/MUSR": 0.3156, - "hfopenllm_v2/MMLU-PRO": 0.1565 - } - }, - { - "id": "JayHyeon/Qwen_0.5-DPO_3e-6-3ep_0alp_0lam", - "name": "Qwen_0.5-DPO_3e-6-3ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.246, - "hfopenllm_v2/BBH": 0.3267, - "hfopenllm_v2/MATH Level 5": 0.0431, - "hfopenllm_v2/GPQA": 0.2685, - "hfopenllm_v2/MUSR": 0.3209, - "hfopenllm_v2/MMLU-PRO": 0.1543 - } - }, - { - "id": "JayHyeon/Qwen_0.5-DPO_3e-7-1ep_0alp_0lam", - "name": "Qwen_0.5-DPO_3e-7-1ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2529, - "hfopenllm_v2/BBH": 0.3229, - "hfopenllm_v2/MATH Level 5": 0.0551, - "hfopenllm_v2/GPQA": 0.2676, - "hfopenllm_v2/MUSR": 0.3195, - "hfopenllm_v2/MMLU-PRO": 0.1597 - } - }, - { - "id": "JayHyeon/Qwen_0.5-DPO_3e-7-2ep_0alp_0lam", - "name": "Qwen_0.5-DPO_3e-7-2ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2505, - "hfopenllm_v2/BBH": 0.3256, - "hfopenllm_v2/MATH Level 5": 0.0476, - "hfopenllm_v2/GPQA": 0.2718, - "hfopenllm_v2/MUSR": 0.3195, - "hfopenllm_v2/MMLU-PRO": 0.1599 - } - }, - { - "id": "JayHyeon/Qwen_0.5-DPO_3e-7-3ep_0alp_0lam", - "name": "Qwen_0.5-DPO_3e-7-3ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2387, - "hfopenllm_v2/BBH": 0.3258, - "hfopenllm_v2/MATH Level 5": 0.0446, - "hfopenllm_v2/GPQA": 0.2743, - "hfopenllm_v2/MUSR": 0.3169, - "hfopenllm_v2/MMLU-PRO": 0.1589 - } - }, - { - "id": "JayHyeon/Qwen_0.5-DPO_5e-7-1ep_0alp_0lam", - "name": "Qwen_0.5-DPO_5e-7-1ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2532, - "hfopenllm_v2/BBH": 0.3218, - "hfopenllm_v2/MATH Level 5": 0.0634, - "hfopenllm_v2/GPQA": 0.2685, - "hfopenllm_v2/MUSR": 0.3209, - "hfopenllm_v2/MMLU-PRO": 0.1593 - } - }, - { - "id": "JayHyeon/Qwen_0.5-DPO_5e-7-2ep_0alp_0lam", - "name": "Qwen_0.5-DPO_5e-7-2ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2456, - "hfopenllm_v2/BBH": 0.3299, - "hfopenllm_v2/MATH Level 5": 0.0536, - "hfopenllm_v2/GPQA": 0.271, - "hfopenllm_v2/MUSR": 0.3181, - "hfopenllm_v2/MMLU-PRO": 0.1602 - } - }, - { - "id": "JayHyeon/Qwen_0.5-DPO_5e-7-3ep_0alp_0lam", - "name": "Qwen_0.5-DPO_5e-7-3ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2423, - "hfopenllm_v2/BBH": 0.3271, - "hfopenllm_v2/MATH Level 5": 0.0514, - "hfopenllm_v2/GPQA": 0.2743, - "hfopenllm_v2/MUSR": 0.3181, - "hfopenllm_v2/MMLU-PRO": 0.1595 - } - }, - { - "id": "JayHyeon/Qwen_0.5-DPOP_1e-6-3ep_0alp_5lam", - "name": "Qwen_0.5-DPOP_1e-6-3ep_0alp_5lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2532, - "hfopenllm_v2/BBH": 0.314, - "hfopenllm_v2/MATH Level 5": 0.0491, - "hfopenllm_v2/GPQA": 0.2743, - "hfopenllm_v2/MUSR": 0.3315, - "hfopenllm_v2/MMLU-PRO": 0.1566 - } - }, - { - "id": "JayHyeon/Qwen_0.5-DPOP_1e-7-3ep_0alp_5lam", - "name": "Qwen_0.5-DPOP_1e-7-3ep_0alp_5lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.267, - "hfopenllm_v2/BBH": 0.3189, - "hfopenllm_v2/MATH Level 5": 0.0408, - "hfopenllm_v2/GPQA": 0.2668, - "hfopenllm_v2/MUSR": 0.3288, - "hfopenllm_v2/MMLU-PRO": 0.1562 - } - }, - { - "id": "JayHyeon/Qwen_0.5-DPOP_3e-6-1ep_0alp_5lam", - "name": "Qwen_0.5-DPOP_3e-6-1ep_0alp_5lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2481, - "hfopenllm_v2/BBH": 0.3261, - "hfopenllm_v2/MATH Level 5": 0.0438, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3368, - "hfopenllm_v2/MMLU-PRO": 0.1565 - } - }, - { - "id": "JayHyeon/Qwen_0.5-DPOP_3e-6-2ep_0alp_5lam", - "name": "Qwen_0.5-DPOP_3e-6-2ep_0alp_5lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2383, - "hfopenllm_v2/BBH": 0.3218, - "hfopenllm_v2/MATH Level 5": 0.0431, - "hfopenllm_v2/GPQA": 0.2794, - "hfopenllm_v2/MUSR": 0.3342, - "hfopenllm_v2/MMLU-PRO": 0.1503 - } - }, - { - "id": "JayHyeon/Qwen_0.5-DPOP_3e-6-3ep_0alp_5lam", - "name": "Qwen_0.5-DPOP_3e-6-3ep_0alp_5lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2471, - "hfopenllm_v2/BBH": 0.3224, - "hfopenllm_v2/MATH Level 5": 0.04, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.3328, - "hfopenllm_v2/MMLU-PRO": 0.1533 - } - }, - { - "id": "JayHyeon/Qwen_0.5-DPOP_3e-7-1ep_0alp_5lam", - "name": "Qwen_0.5-DPOP_3e-7-1ep_0alp_5lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2447, - "hfopenllm_v2/BBH": 0.3181, - "hfopenllm_v2/MATH Level 5": 0.0438, - "hfopenllm_v2/GPQA": 0.2617, - "hfopenllm_v2/MUSR": 0.3341, - "hfopenllm_v2/MMLU-PRO": 0.1565 - } - }, - { - "id": "JayHyeon/Qwen_0.5-DPOP_3e-7-2ep_0alp_5lam", - "name": "Qwen_0.5-DPOP_3e-7-2ep_0alp_5lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2551, - "hfopenllm_v2/BBH": 0.3194, - "hfopenllm_v2/MATH Level 5": 0.0446, - "hfopenllm_v2/GPQA": 0.2617, - "hfopenllm_v2/MUSR": 0.3262, - "hfopenllm_v2/MMLU-PRO": 0.1567 - } - }, - { - "id": "JayHyeon/Qwen_0.5-DPOP_3e-7-3ep_0alp_5lam", - "name": "Qwen_0.5-DPOP_3e-7-3ep_0alp_5lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2538, - "hfopenllm_v2/BBH": 0.3153, - "hfopenllm_v2/MATH Level 5": 0.0415, - "hfopenllm_v2/GPQA": 0.2676, - "hfopenllm_v2/MUSR": 0.3261, - "hfopenllm_v2/MMLU-PRO": 0.1583 - } - }, - { - "id": "JayHyeon/Qwen_0.5-DPOP_5e-7-1ep_0alp_5lam", - "name": "Qwen_0.5-DPOP_5e-7-1ep_0alp_5lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2402, - "hfopenllm_v2/BBH": 0.3168, - "hfopenllm_v2/MATH Level 5": 0.0378, - "hfopenllm_v2/GPQA": 0.2718, - "hfopenllm_v2/MUSR": 0.3328, - "hfopenllm_v2/MMLU-PRO": 0.1568 - } - }, - { - "id": "JayHyeon/Qwen_0.5-DPOP_5e-7-2ep_0alp_5lam", - "name": "Qwen_0.5-DPOP_5e-7-2ep_0alp_5lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2484, - "hfopenllm_v2/BBH": 0.3211, - "hfopenllm_v2/MATH Level 5": 0.0438, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.3288, - "hfopenllm_v2/MMLU-PRO": 0.1573 - } - }, - { - "id": "JayHyeon/Qwen_0.5-DPOP_5e-7-3ep_0alp_5lam", - "name": "Qwen_0.5-DPOP_5e-7-3ep_0alp_5lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2578, - "hfopenllm_v2/BBH": 0.3203, - "hfopenllm_v2/MATH Level 5": 0.0423, - "hfopenllm_v2/GPQA": 0.271, - "hfopenllm_v2/MUSR": 0.3289, - "hfopenllm_v2/MMLU-PRO": 0.1583 - } - }, - { - "id": "JayHyeon/Qwen_0.5-IPO_5e-7-1ep_0alp_0lam", - "name": "Qwen_0.5-IPO_5e-7-1ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2574, - "hfopenllm_v2/BBH": 0.3279, - "hfopenllm_v2/MATH Level 5": 0.0559, - "hfopenllm_v2/GPQA": 0.2693, - "hfopenllm_v2/MUSR": 0.3169, - "hfopenllm_v2/MMLU-PRO": 0.1651 - } - }, - { - "id": "JayHyeon/Qwen_0.5-IPO_5e-7-3ep_0alp_0lam", - "name": "Qwen_0.5-IPO_5e-7-3ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3072, - "hfopenllm_v2/BBH": 0.3264, - "hfopenllm_v2/MATH Level 5": 0.0582, - "hfopenllm_v2/GPQA": 0.2567, - "hfopenllm_v2/MUSR": 0.3156, - "hfopenllm_v2/MMLU-PRO": 0.1624 - } - }, - { - "id": "JayHyeon/Qwen_0.5-IRPO_1e-6-3ep_1alp_0lam", - "name": "Qwen_0.5-IRPO_1e-6-3ep_1alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2551, - "hfopenllm_v2/BBH": 0.3242, - "hfopenllm_v2/MATH Level 5": 0.0468, - "hfopenllm_v2/GPQA": 0.2668, - "hfopenllm_v2/MUSR": 0.3182, - "hfopenllm_v2/MMLU-PRO": 0.1574 - } - }, - { - "id": "JayHyeon/Qwen_0.5-IRPO_1e-7-3ep_1alp_0lam", - "name": "Qwen_0.5-IRPO_1e-7-3ep_1alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2636, - "hfopenllm_v2/BBH": 0.3198, - "hfopenllm_v2/MATH Level 5": 0.0514, - "hfopenllm_v2/GPQA": 0.276, - "hfopenllm_v2/MUSR": 0.3262, - "hfopenllm_v2/MMLU-PRO": 0.1586 - } - }, - { - "id": "JayHyeon/Qwen_0.5-IRPO_3e-6-1ep_1alp_0lam", - "name": "Qwen_0.5-IRPO_3e-6-1ep_1alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2323, - "hfopenllm_v2/BBH": 0.3255, - "hfopenllm_v2/MATH Level 5": 0.037, - "hfopenllm_v2/GPQA": 0.2508, - "hfopenllm_v2/MUSR": 0.3169, - "hfopenllm_v2/MMLU-PRO": 0.1612 - } - }, - { - "id": "JayHyeon/Qwen_0.5-IRPO_3e-6-2ep_1alp_0lam", - "name": "Qwen_0.5-IRPO_3e-6-2ep_1alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2414, - "hfopenllm_v2/BBH": 0.3314, - "hfopenllm_v2/MATH Level 5": 0.0347, - "hfopenllm_v2/GPQA": 0.2517, - "hfopenllm_v2/MUSR": 0.3342, - "hfopenllm_v2/MMLU-PRO": 0.1532 - } - }, - { - "id": "JayHyeon/Qwen_0.5-IRPO_3e-6-3ep_1alp_0lam", - "name": "Qwen_0.5-IRPO_3e-6-3ep_1alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2678, - "hfopenllm_v2/BBH": 0.3362, - "hfopenllm_v2/MATH Level 5": 0.0514, - "hfopenllm_v2/GPQA": 0.2542, - "hfopenllm_v2/MUSR": 0.3382, - "hfopenllm_v2/MMLU-PRO": 0.1561 - } - }, - { - "id": "JayHyeon/Qwen_0.5-IRPO_3e-7-1ep_1alp_0lam", - "name": "Qwen_0.5-IRPO_3e-7-1ep_1alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2561, - "hfopenllm_v2/BBH": 0.3231, - "hfopenllm_v2/MATH Level 5": 0.0536, - "hfopenllm_v2/GPQA": 0.2718, - "hfopenllm_v2/MUSR": 0.3196, - "hfopenllm_v2/MMLU-PRO": 0.1589 - } - }, - { - "id": "JayHyeon/Qwen_0.5-IRPO_3e-7-3ep_1alp_0lam", - "name": "Qwen_0.5-IRPO_3e-7-3ep_1alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2639, - "hfopenllm_v2/BBH": 0.3257, - "hfopenllm_v2/MATH Level 5": 0.0476, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.3209, - "hfopenllm_v2/MMLU-PRO": 0.1587 - } - }, - { - "id": "JayHyeon/Qwen_0.5-IRPO_5e-7-1ep_1alp_0lam", - "name": "Qwen_0.5-IRPO_5e-7-1ep_1alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2518, - "hfopenllm_v2/BBH": 0.3214, - "hfopenllm_v2/MATH Level 5": 0.0574, - "hfopenllm_v2/GPQA": 0.2735, - "hfopenllm_v2/MUSR": 0.3169, - "hfopenllm_v2/MMLU-PRO": 0.1585 - } - }, - { - "id": "JayHyeon/Qwen_0.5-IRPO_5e-7-2ep_1alp_0lam", - "name": "Qwen_0.5-IRPO_5e-7-2ep_1alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2438, - "hfopenllm_v2/BBH": 0.3266, - "hfopenllm_v2/MATH Level 5": 0.0619, - "hfopenllm_v2/GPQA": 0.2727, - "hfopenllm_v2/MUSR": 0.3196, - "hfopenllm_v2/MMLU-PRO": 0.1554 - } - }, - { - "id": "JayHyeon/Qwen_0.5-IRPO_5e-7-3ep_1alp_0lam", - "name": "Qwen_0.5-IRPO_5e-7-3ep_1alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2465, - "hfopenllm_v2/BBH": 0.3246, - "hfopenllm_v2/MATH Level 5": 0.0529, - "hfopenllm_v2/GPQA": 0.2718, - "hfopenllm_v2/MUSR": 0.3182, - "hfopenllm_v2/MMLU-PRO": 0.1563 - } - }, - { - "id": "JayHyeon/Qwen_0.5-MDPO_0.1_3e-6-3ep_0alp_0lam", - "name": "Qwen_0.5-MDPO_0.1_3e-6-3ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2506, - "hfopenllm_v2/BBH": 0.3261, - "hfopenllm_v2/MATH Level 5": 0.0498, - "hfopenllm_v2/GPQA": 0.2819, - "hfopenllm_v2/MUSR": 0.3382, - "hfopenllm_v2/MMLU-PRO": 0.1522 - } - }, - { - "id": "JayHyeon/Qwen_0.5-MDPO_0.1_5e-7-3ep_0alp_0lam", - "name": "Qwen_0.5-MDPO_0.1_5e-7-3ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2457, - "hfopenllm_v2/BBH": 0.318, - "hfopenllm_v2/MATH Level 5": 0.0347, - "hfopenllm_v2/GPQA": 0.2634, - "hfopenllm_v2/MUSR": 0.3315, - "hfopenllm_v2/MMLU-PRO": 0.1566 - } - }, - { - "id": "JayHyeon/Qwen_0.5-MDPO_0.3_3e-6-3ep_0alp_0lam", - "name": "Qwen_0.5-MDPO_0.3_3e-6-3ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2454, - "hfopenllm_v2/BBH": 0.3216, - "hfopenllm_v2/MATH Level 5": 0.0506, - "hfopenllm_v2/GPQA": 0.2802, - "hfopenllm_v2/MUSR": 0.3382, - "hfopenllm_v2/MMLU-PRO": 0.1544 - } - }, - { - "id": "JayHyeon/Qwen_0.5-MDPO_0.3_5e-7-3ep_0alp_0lam", - "name": "Qwen_0.5-MDPO_0.3_5e-7-3ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2342, - "hfopenllm_v2/BBH": 0.3189, - "hfopenllm_v2/MATH Level 5": 0.04, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.3302, - "hfopenllm_v2/MMLU-PRO": 0.158 - } - }, - { - "id": "JayHyeon/Qwen_0.5-MDPO_0.5_1e-5-3ep_0alp_0lam", - "name": "Qwen_0.5-MDPO_0.5_1e-5-3ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.232, - "hfopenllm_v2/BBH": 0.3234, - "hfopenllm_v2/MATH Level 5": 0.0393, - "hfopenllm_v2/GPQA": 0.2743, - "hfopenllm_v2/MUSR": 0.3369, - "hfopenllm_v2/MMLU-PRO": 0.1543 - } - }, - { - "id": "JayHyeon/Qwen_0.5-MDPO_0.5_3e-7-1ep_0alp_0lam", - "name": "Qwen_0.5-MDPO_0.5_3e-7-1ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2418, - "hfopenllm_v2/BBH": 0.3175, - "hfopenllm_v2/MATH Level 5": 0.0423, - "hfopenllm_v2/GPQA": 0.2626, - "hfopenllm_v2/MUSR": 0.3288, - "hfopenllm_v2/MMLU-PRO": 0.158 - } - }, - { - "id": "JayHyeon/Qwen_0.5-MDPO_0.5_3e-7-2ep_0alp_0lam", - "name": "Qwen_0.5-MDPO_0.5_3e-7-2ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2493, - "hfopenllm_v2/BBH": 0.3197, - "hfopenllm_v2/MATH Level 5": 0.0423, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.3315, - "hfopenllm_v2/MMLU-PRO": 0.1571 - } - }, - { - "id": "JayHyeon/Qwen_0.5-MDPO_0.5_3e-7-3ep_0alp_0lam", - "name": "Qwen_0.5-MDPO_0.5_3e-7-3ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.252, - "hfopenllm_v2/BBH": 0.3198, - "hfopenllm_v2/MATH Level 5": 0.0423, - "hfopenllm_v2/GPQA": 0.2634, - "hfopenllm_v2/MUSR": 0.3262, - "hfopenllm_v2/MMLU-PRO": 0.1551 - } - }, - { - "id": "JayHyeon/Qwen_0.5-MDPO_0.5_4e-6-3ep_0alp_0lam", - "name": "Qwen_0.5-MDPO_0.5_4e-6-3ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.258, - "hfopenllm_v2/BBH": 0.3248, - "hfopenllm_v2/MATH Level 5": 0.0476, - "hfopenllm_v2/GPQA": 0.2752, - "hfopenllm_v2/MUSR": 0.3422, - "hfopenllm_v2/MMLU-PRO": 0.1539 - } - }, - { - "id": "JayHyeon/Qwen_0.5-MDPO_0.5_6e-6-3ep_0alp_0lam", - "name": "Qwen_0.5-MDPO_0.5_6e-6-3ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.232, - "hfopenllm_v2/BBH": 0.3265, - "hfopenllm_v2/MATH Level 5": 0.0385, - "hfopenllm_v2/GPQA": 0.271, - "hfopenllm_v2/MUSR": 0.3395, - "hfopenllm_v2/MMLU-PRO": 0.1537 - } - }, - { - "id": "JayHyeon/Qwen_0.5-MDPO_0.5_7e-6-3ep_0alp_0lam", - "name": "Qwen_0.5-MDPO_0.5_7e-6-3ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2488, - "hfopenllm_v2/BBH": 0.3273, - "hfopenllm_v2/MATH Level 5": 0.0461, - "hfopenllm_v2/GPQA": 0.2718, - "hfopenllm_v2/MUSR": 0.3342, - "hfopenllm_v2/MMLU-PRO": 0.1531 - } - }, - { - "id": "JayHyeon/Qwen_0.5-MDPO_0.5_7e-7-3ep_0alp_0lam", - "name": "Qwen_0.5-MDPO_0.5_7e-7-3ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2524, - "hfopenllm_v2/BBH": 0.313, - "hfopenllm_v2/MATH Level 5": 0.0446, - "hfopenllm_v2/GPQA": 0.271, - "hfopenllm_v2/MUSR": 0.3289, - "hfopenllm_v2/MMLU-PRO": 0.1564 - } - }, - { - "id": "JayHyeon/Qwen_0.5-MDPO_0.7_3e-6-3ep_0alp_0lam", - "name": "Qwen_0.5-MDPO_0.7_3e-6-3ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2514, - "hfopenllm_v2/BBH": 0.3221, - "hfopenllm_v2/MATH Level 5": 0.0438, - "hfopenllm_v2/GPQA": 0.2752, - "hfopenllm_v2/MUSR": 0.3315, - "hfopenllm_v2/MMLU-PRO": 0.1538 - } - }, - { - "id": "JayHyeon/Qwen_0.5-MDPO_0.7_5e-7-3ep_0alp_0lam", - "name": "Qwen_0.5-MDPO_0.7_5e-7-3ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2457, - "hfopenllm_v2/BBH": 0.318, - "hfopenllm_v2/MATH Level 5": 0.0385, - "hfopenllm_v2/GPQA": 0.2668, - "hfopenllm_v2/MUSR": 0.3275, - "hfopenllm_v2/MMLU-PRO": 0.1572 - } - }, - { - "id": "JayHyeon/Qwen_0.5-MDPO_0.9_5e-7-3ep_0alp_0lam", - "name": "Qwen_0.5-MDPO_0.9_5e-7-3ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2636, - "hfopenllm_v2/BBH": 0.3181, - "hfopenllm_v2/MATH Level 5": 0.0476, - "hfopenllm_v2/GPQA": 0.2659, - "hfopenllm_v2/MUSR": 0.3235, - "hfopenllm_v2/MMLU-PRO": 0.1574 - } - }, - { - "id": "JayHyeon/Qwen_0.5-rDPO_3e-6-1ep_0vpo_const_0.1", - "name": "Qwen_0.5-rDPO_3e-6-1ep_0vpo_const_0.1", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2321, - "hfopenllm_v2/BBH": 0.3278, - "hfopenllm_v2/MATH Level 5": 0.0476, - "hfopenllm_v2/GPQA": 0.2576, - "hfopenllm_v2/MUSR": 0.3022, - "hfopenllm_v2/MMLU-PRO": 0.1496 - } - }, - { - "id": "JayHyeon/Qwen_0.5-rDPO_5e-7-3ep_0vpo_const_0.1", - "name": "Qwen_0.5-rDPO_5e-7-3ep_0vpo_const_0.1", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2542, - "hfopenllm_v2/BBH": 0.3253, - "hfopenllm_v2/MATH Level 5": 0.0529, - "hfopenllm_v2/GPQA": 0.271, - "hfopenllm_v2/MUSR": 0.3181, - "hfopenllm_v2/MMLU-PRO": 0.1609 - } - }, - { - "id": "JayHyeon/Qwen_0.5-rDPO_5e-7-3ep_0vpo_const_0.3", - "name": "Qwen_0.5-rDPO_5e-7-3ep_0vpo_const_0.3", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2739, - "hfopenllm_v2/BBH": 0.3245, - "hfopenllm_v2/MATH Level 5": 0.0461, - "hfopenllm_v2/GPQA": 0.2508, - "hfopenllm_v2/MUSR": 0.3089, - "hfopenllm_v2/MMLU-PRO": 0.1597 - } - }, - { - "id": "JayHyeon/Qwen_0.5-VDPO_3e-6-1ep_3vpo_const", - "name": "Qwen_0.5-VDPO_3e-6-1ep_3vpo_const", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2483, - "hfopenllm_v2/BBH": 0.3174, - "hfopenllm_v2/MATH Level 5": 0.0378, - "hfopenllm_v2/GPQA": 0.2542, - "hfopenllm_v2/MUSR": 0.3328, - "hfopenllm_v2/MMLU-PRO": 0.1558 - } - }, - { - "id": "JayHyeon/Qwen_0.5-VDPO_5e-7-1ep_0alp_0lam", - "name": "Qwen_0.5-VDPO_5e-7-1ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2518, - "hfopenllm_v2/BBH": 0.3218, - "hfopenllm_v2/MATH Level 5": 0.0529, - "hfopenllm_v2/GPQA": 0.2718, - "hfopenllm_v2/MUSR": 0.3235, - "hfopenllm_v2/MMLU-PRO": 0.1595 - } - }, - { - "id": "JayHyeon/Qwen_0.5-VDPO_5e-7-1ep_10vpo_const", - "name": "Qwen_0.5-VDPO_5e-7-1ep_10vpo_const", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2536, - "hfopenllm_v2/BBH": 0.3234, - "hfopenllm_v2/MATH Level 5": 0.0491, - "hfopenllm_v2/GPQA": 0.276, - "hfopenllm_v2/MUSR": 0.3236, - "hfopenllm_v2/MMLU-PRO": 0.1597 - } - }, - { - "id": "JayHyeon/Qwen_0.5-VDPO_5e-7-1ep_1vpo_const", - "name": "Qwen_0.5-VDPO_5e-7-1ep_1vpo_const", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2448, - "hfopenllm_v2/BBH": 0.324, - "hfopenllm_v2/MATH Level 5": 0.0604, - "hfopenllm_v2/GPQA": 0.2752, - "hfopenllm_v2/MUSR": 0.3249, - "hfopenllm_v2/MMLU-PRO": 0.1587 - } - }, - { - "id": "JayHyeon/Qwen_0.5-VDPO_5e-7-1ep_3vpo_const", - "name": "Qwen_0.5-VDPO_5e-7-1ep_3vpo_const", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2505, - "hfopenllm_v2/BBH": 0.3227, - "hfopenllm_v2/MATH Level 5": 0.0468, - "hfopenllm_v2/GPQA": 0.271, - "hfopenllm_v2/MUSR": 0.3209, - "hfopenllm_v2/MMLU-PRO": 0.1589 - } - }, - { - "id": "JayHyeon/Qwen_0.5-VDPO_5e-7-3ep_0alp_0lam", - "name": "Qwen_0.5-VDPO_5e-7-3ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2472, - "hfopenllm_v2/BBH": 0.3255, - "hfopenllm_v2/MATH Level 5": 0.0498, - "hfopenllm_v2/GPQA": 0.2752, - "hfopenllm_v2/MUSR": 0.3208, - "hfopenllm_v2/MMLU-PRO": 0.1587 - } - }, - { - "id": "JayHyeon/Qwen_0.5-VDPO_5e-7-3ep_1vpo_const", - "name": "Qwen_0.5-VDPO_5e-7-3ep_1vpo_const", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2417, - "hfopenllm_v2/BBH": 0.3256, - "hfopenllm_v2/MATH Level 5": 0.0582, - "hfopenllm_v2/GPQA": 0.2727, - "hfopenllm_v2/MUSR": 0.3275, - "hfopenllm_v2/MMLU-PRO": 0.1562 - } - }, - { - "id": "JayHyeon/Qwen_0.5-VDPO_5e-7-3ep_3vpo_const", - "name": "Qwen_0.5-VDPO_5e-7-3ep_3vpo_const", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2527, - "hfopenllm_v2/BBH": 0.3235, - "hfopenllm_v2/MATH Level 5": 0.0536, - "hfopenllm_v2/GPQA": 0.2785, - "hfopenllm_v2/MUSR": 0.3235, - "hfopenllm_v2/MMLU-PRO": 0.158 - } - }, - { - "id": "JayHyeon/Qwen_0.5-VIPO_5e-7-1ep_0alp_0lam", - "name": "Qwen_0.5-VIPO_5e-7-1ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2669, - "hfopenllm_v2/BBH": 0.3314, - "hfopenllm_v2/MATH Level 5": 0.071, - "hfopenllm_v2/GPQA": 0.2676, - "hfopenllm_v2/MUSR": 0.3168, - "hfopenllm_v2/MMLU-PRO": 0.1634 - } - }, - { - "id": "JayHyeon/Qwen_0.5-VIPO_5e-7-1ep_10vpo_const", - "name": "Qwen_0.5-VIPO_5e-7-1ep_10vpo_const", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2702, - "hfopenllm_v2/BBH": 0.33, - "hfopenllm_v2/MATH Level 5": 0.074, - "hfopenllm_v2/GPQA": 0.2752, - "hfopenllm_v2/MUSR": 0.3208, - "hfopenllm_v2/MMLU-PRO": 0.1635 - } - }, - { - "id": "JayHyeon/Qwen_0.5-VIPO_5e-7-1ep_1vpo_const", - "name": "Qwen_0.5-VIPO_5e-7-1ep_1vpo_const", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.248, - "hfopenllm_v2/BBH": 0.3309, - "hfopenllm_v2/MATH Level 5": 0.068, - "hfopenllm_v2/GPQA": 0.2643, - "hfopenllm_v2/MUSR": 0.3208, - "hfopenllm_v2/MMLU-PRO": 0.1649 - } - }, - { - "id": "JayHyeon/Qwen_0.5-VIPO_5e-7-1ep_30vpo_const", - "name": "Qwen_0.5-VIPO_5e-7-1ep_30vpo_const", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2622, - "hfopenllm_v2/BBH": 0.3282, - "hfopenllm_v2/MATH Level 5": 0.074, - "hfopenllm_v2/GPQA": 0.2693, - "hfopenllm_v2/MUSR": 0.3221, - "hfopenllm_v2/MMLU-PRO": 0.1634 - } - }, - { - "id": "JayHyeon/Qwen_0.5-VIPO_5e-7-1ep_3vpo_const", - "name": "Qwen_0.5-VIPO_5e-7-1ep_3vpo_const", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2609, - "hfopenllm_v2/BBH": 0.3298, - "hfopenllm_v2/MATH Level 5": 0.065, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.3168, - "hfopenllm_v2/MMLU-PRO": 0.1651 - } - }, - { - "id": "JayHyeon/Qwen_0.5-VIPO_5e-7-3ep_0alp_0lam", - "name": "Qwen_0.5-VIPO_5e-7-3ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.293, - "hfopenllm_v2/BBH": 0.322, - "hfopenllm_v2/MATH Level 5": 0.0627, - "hfopenllm_v2/GPQA": 0.2685, - "hfopenllm_v2/MUSR": 0.3116, - "hfopenllm_v2/MMLU-PRO": 0.1591 - } - }, - { - "id": "JayHyeon/Qwen_0.5-VIPO_5e-7-3ep_10vpo_const", - "name": "Qwen_0.5-VIPO_5e-7-3ep_10vpo_const", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2881, - "hfopenllm_v2/BBH": 0.3255, - "hfopenllm_v2/MATH Level 5": 0.0725, - "hfopenllm_v2/GPQA": 0.2752, - "hfopenllm_v2/MUSR": 0.3102, - "hfopenllm_v2/MMLU-PRO": 0.1582 - } - }, - { - "id": "JayHyeon/Qwen_0.5-VIPO_5e-7-3ep_1vpo_const", - "name": "Qwen_0.5-VIPO_5e-7-3ep_1vpo_const", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2887, - "hfopenllm_v2/BBH": 0.3237, - "hfopenllm_v2/MATH Level 5": 0.0748, - "hfopenllm_v2/GPQA": 0.2802, - "hfopenllm_v2/MUSR": 0.3142, - "hfopenllm_v2/MMLU-PRO": 0.1609 - } - }, - { - "id": "JayHyeon/Qwen_0.5-VIPO_5e-7-3ep_30vpo_const", - "name": "Qwen_0.5-VIPO_5e-7-3ep_30vpo_const", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2905, - "hfopenllm_v2/BBH": 0.3254, - "hfopenllm_v2/MATH Level 5": 0.077, - "hfopenllm_v2/GPQA": 0.2735, - "hfopenllm_v2/MUSR": 0.3129, - "hfopenllm_v2/MMLU-PRO": 0.1574 - } - }, - { - "id": "JayHyeon/Qwen_0.5-VIPO_5e-7-3ep_3vpo_const", - "name": "Qwen_0.5-VIPO_5e-7-3ep_3vpo_const", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2905, - "hfopenllm_v2/BBH": 0.3238, - "hfopenllm_v2/MATH Level 5": 0.0702, - "hfopenllm_v2/GPQA": 0.2735, - "hfopenllm_v2/MUSR": 0.3089, - "hfopenllm_v2/MMLU-PRO": 0.1592 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/jeanmichela.json b/data/developers/jeanmichela.json deleted file mode 100644 index fadc4dfc28a3579f5dccbf04caf169e5d950a8b7..0000000000000000000000000000000000000000 --- a/data/developers/jeanmichela.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "jeanmichela", - "models": [ - { - "id": "jeanmichela/o-distil-qwen", - "name": "o-distil-qwen", - "developer": "jeanmichela", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4482, - "hfopenllm_v2/BBH": 0.59, - "hfopenllm_v2/MATH Level 5": 0.565, - "hfopenllm_v2/GPQA": 0.3935, - "hfopenllm_v2/MUSR": 0.534, - "hfopenllm_v2/MMLU-PRO": 0.4658 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/jebcarter.json b/data/developers/jebcarter.json deleted file mode 100644 index d53b334a7bf817d5bf537f8b18a0e0ee8092de33..0000000000000000000000000000000000000000 --- a/data/developers/jebcarter.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "jebcarter", - "models": [ - { - "id": "jebcarter/psyonic-cetacean-20B", - "name": "psyonic-cetacean-20B", - "developer": "jebcarter", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2544, - "hfopenllm_v2/BBH": 0.4907, - "hfopenllm_v2/MATH Level 5": 0.0181, - "hfopenllm_v2/GPQA": 0.2735, - "hfopenllm_v2/MUSR": 0.4661, - "hfopenllm_v2/MMLU-PRO": 0.2886 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/jebish7.json b/data/developers/jebish7.json deleted file mode 100644 index cf4434cc3ffe6650e556ae52b51a4375e1296d6a..0000000000000000000000000000000000000000 --- a/data/developers/jebish7.json +++ /dev/null @@ -1,131 +0,0 @@ -{ - "developer": "jebish7", - "models": [ - { - "id": "jebish7/aya-expanse-8b", - "name": "aya-expanse-8b", - "developer": "jebish7", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3791, - "hfopenllm_v2/BBH": 0.4969, - "hfopenllm_v2/MATH Level 5": 0.0816, - "hfopenllm_v2/GPQA": 0.2836, - "hfopenllm_v2/MUSR": 0.3869, - "hfopenllm_v2/MMLU-PRO": 0.3103 - } - }, - { - "id": "jebish7/gemma-2-2b-it", - "name": "gemma-2-2b-it", - "developer": "jebish7", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1272, - "hfopenllm_v2/BBH": 0.4395, - "hfopenllm_v2/MATH Level 5": 0.034, - "hfopenllm_v2/GPQA": 0.297, - "hfopenllm_v2/MUSR": 0.4244, - "hfopenllm_v2/MMLU-PRO": 0.2715 - } - }, - { - "id": "jebish7/gemma-2-9b-it", - "name": "gemma-2-9b-it", - "developer": "jebish7", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1557, - "hfopenllm_v2/BBH": 0.5949, - "hfopenllm_v2/MATH Level 5": 0.0846, - "hfopenllm_v2/GPQA": 0.3473, - "hfopenllm_v2/MUSR": 0.4554, - "hfopenllm_v2/MMLU-PRO": 0.4143 - } - }, - { - "id": "jebish7/Llama-3-Nanda-10B-Chat", - "name": "Llama-3-Nanda-10B-Chat", - "developer": "jebish7", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2953, - "hfopenllm_v2/BBH": 0.4959, - "hfopenllm_v2/MATH Level 5": 0.0559, - "hfopenllm_v2/GPQA": 0.3012, - "hfopenllm_v2/MUSR": 0.4356, - "hfopenllm_v2/MMLU-PRO": 0.3157 - } - }, - { - "id": "jebish7/Llama-3.1-8B-Instruct", - "name": "Llama-3.1-8B-Instruct", - "developer": "jebish7", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5058, - "hfopenllm_v2/BBH": 0.5088, - "hfopenllm_v2/MATH Level 5": 0.1548, - "hfopenllm_v2/GPQA": 0.3213, - "hfopenllm_v2/MUSR": 0.3998, - "hfopenllm_v2/MMLU-PRO": 0.3777 - } - }, - { - "id": "jebish7/Nemotron-4-Mini-Hindi-4B-Base", - "name": "Nemotron-4-Mini-Hindi-4B-Base", - "developer": "jebish7", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2285, - "hfopenllm_v2/BBH": 0.3924, - "hfopenllm_v2/MATH Level 5": 0.0272, - "hfopenllm_v2/GPQA": 0.2836, - "hfopenllm_v2/MUSR": 0.4249, - "hfopenllm_v2/MMLU-PRO": 0.2503 - } - }, - { - "id": "jebish7/Nemotron-4-Mini-Hindi-4B-Instruct", - "name": "Nemotron-4-Mini-Hindi-4B-Instruct", - "developer": "jebish7", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3345, - "hfopenllm_v2/BBH": 0.4041, - "hfopenllm_v2/MATH Level 5": 0.0287, - "hfopenllm_v2/GPQA": 0.3087, - "hfopenllm_v2/MUSR": 0.4153, - "hfopenllm_v2/MMLU-PRO": 0.2595 - } - }, - { - "id": "jebish7/Nemotron-Mini-4B-Instruct", - "name": "Nemotron-Mini-4B-Instruct", - "developer": "jebish7", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3709, - "hfopenllm_v2/BBH": 0.4244, - "hfopenllm_v2/MATH Level 5": 0.0325, - "hfopenllm_v2/GPQA": 0.276, - "hfopenllm_v2/MUSR": 0.4727, - "hfopenllm_v2/MMLU-PRO": 0.2783 - } - }, - { - "id": "jebish7/qwen2.5-0.5B-IHA-Hin", - "name": "qwen2.5-0.5B-IHA-Hin", - "developer": "jebish7", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1416, - "hfopenllm_v2/BBH": 0.2989, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2525, - "hfopenllm_v2/MUSR": 0.3475, - "hfopenllm_v2/MMLU-PRO": 0.1094 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/jeffmeloy.json b/data/developers/jeffmeloy.json deleted file mode 100644 index 2f619d7825c8a5963de13a2bf82424cb60d1ae61..0000000000000000000000000000000000000000 --- a/data/developers/jeffmeloy.json +++ /dev/null @@ -1,257 +0,0 @@ -{ - "developer": "jeffmeloy", - "models": [ - { - "id": "jeffmeloy/jeffmeloy_Qwen2.5-7B-minperplexity-1", - "name": "jeffmeloy_Qwen2.5-7B-minperplexity-1", - "developer": "jeffmeloy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3757, - "hfopenllm_v2/BBH": 0.5582, - "hfopenllm_v2/MATH Level 5": 0.2915, - "hfopenllm_v2/GPQA": 0.3322, - "hfopenllm_v2/MUSR": 0.429, - "hfopenllm_v2/MMLU-PRO": 0.4368 - } - }, - { - "id": "jeffmeloy/Qwen-7B-nerd-uncensored-v1.0", - "name": "Qwen-7B-nerd-uncensored-v1.0", - "developer": "jeffmeloy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6136, - "hfopenllm_v2/BBH": 0.5421, - "hfopenllm_v2/MATH Level 5": 0.287, - "hfopenllm_v2/GPQA": 0.328, - "hfopenllm_v2/MUSR": 0.4793, - "hfopenllm_v2/MMLU-PRO": 0.4363 - } - }, - { - "id": "jeffmeloy/Qwen2.5-7B-minperplexity-2", - "name": "Qwen2.5-7B-minperplexity-2", - "developer": "jeffmeloy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5097, - "hfopenllm_v2/BBH": 0.5524, - "hfopenllm_v2/MATH Level 5": 0.3014, - "hfopenllm_v2/GPQA": 0.3112, - "hfopenllm_v2/MUSR": 0.4625, - "hfopenllm_v2/MMLU-PRO": 0.4346 - } - }, - { - "id": "jeffmeloy/Qwen2.5-7B-nerd-uncensored-v0.9", - "name": "Qwen2.5-7B-nerd-uncensored-v0.9", - "developer": "jeffmeloy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6048, - "hfopenllm_v2/BBH": 0.547, - "hfopenllm_v2/MATH Level 5": 0.2946, - "hfopenllm_v2/GPQA": 0.323, - "hfopenllm_v2/MUSR": 0.482, - "hfopenllm_v2/MMLU-PRO": 0.4363 - } - }, - { - "id": "jeffmeloy/Qwen2.5-7B-nerd-uncensored-v1.0", - "name": "Qwen2.5-7B-nerd-uncensored-v1.0", - "developer": "jeffmeloy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7695, - "hfopenllm_v2/BBH": 0.5418, - "hfopenllm_v2/MATH Level 5": 0.4713, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.4551, - "hfopenllm_v2/MMLU-PRO": 0.4254 - } - }, - { - "id": "jeffmeloy/Qwen2.5-7B-nerd-uncensored-v1.1", - "name": "Qwen2.5-7B-nerd-uncensored-v1.1", - "developer": "jeffmeloy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6626, - "hfopenllm_v2/BBH": 0.4864, - "hfopenllm_v2/MATH Level 5": 0.1329, - "hfopenllm_v2/GPQA": 0.2869, - "hfopenllm_v2/MUSR": 0.3843, - "hfopenllm_v2/MMLU-PRO": 0.385 - } - }, - { - "id": "jeffmeloy/Qwen2.5-7B-nerd-uncensored-v1.2", - "name": "Qwen2.5-7B-nerd-uncensored-v1.2", - "developer": "jeffmeloy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4965, - "hfopenllm_v2/BBH": 0.4946, - "hfopenllm_v2/MATH Level 5": 0.1208, - "hfopenllm_v2/GPQA": 0.3037, - "hfopenllm_v2/MUSR": 0.4172, - "hfopenllm_v2/MMLU-PRO": 0.3969 - } - }, - { - "id": "jeffmeloy/Qwen2.5-7B-nerd-uncensored-v1.3", - "name": "Qwen2.5-7B-nerd-uncensored-v1.3", - "developer": "jeffmeloy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4995, - "hfopenllm_v2/BBH": 0.5026, - "hfopenllm_v2/MATH Level 5": 0.1231, - "hfopenllm_v2/GPQA": 0.3129, - "hfopenllm_v2/MUSR": 0.4187, - "hfopenllm_v2/MMLU-PRO": 0.4016 - } - }, - { - "id": "jeffmeloy/Qwen2.5-7B-nerd-uncensored-v1.4", - "name": "Qwen2.5-7B-nerd-uncensored-v1.4", - "developer": "jeffmeloy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6079, - "hfopenllm_v2/BBH": 0.5467, - "hfopenllm_v2/MATH Level 5": 0.281, - "hfopenllm_v2/GPQA": 0.3238, - "hfopenllm_v2/MUSR": 0.4714, - "hfopenllm_v2/MMLU-PRO": 0.4419 - } - }, - { - "id": "jeffmeloy/Qwen2.5-7B-nerd-uncensored-v1.5", - "name": "Qwen2.5-7B-nerd-uncensored-v1.5", - "developer": "jeffmeloy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.565, - "hfopenllm_v2/BBH": 0.5523, - "hfopenllm_v2/MATH Level 5": 0.2757, - "hfopenllm_v2/GPQA": 0.3272, - "hfopenllm_v2/MUSR": 0.4982, - "hfopenllm_v2/MMLU-PRO": 0.4448 - } - }, - { - "id": "jeffmeloy/Qwen2.5-7B-nerd-uncensored-v1.7", - "name": "Qwen2.5-7B-nerd-uncensored-v1.7", - "developer": "jeffmeloy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4202, - "hfopenllm_v2/BBH": 0.5392, - "hfopenllm_v2/MATH Level 5": 0.2915, - "hfopenllm_v2/GPQA": 0.3238, - "hfopenllm_v2/MUSR": 0.4848, - "hfopenllm_v2/MMLU-PRO": 0.428 - } - }, - { - "id": "jeffmeloy/Qwen2.5-7B-nerd-uncensored-v1.8", - "name": "Qwen2.5-7B-nerd-uncensored-v1.8", - "developer": "jeffmeloy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6256, - "hfopenllm_v2/BBH": 0.5447, - "hfopenllm_v2/MATH Level 5": 0.2704, - "hfopenllm_v2/GPQA": 0.3238, - "hfopenllm_v2/MUSR": 0.4767, - "hfopenllm_v2/MMLU-PRO": 0.4343 - } - }, - { - "id": "jeffmeloy/Qwen2.5-7B-olm-v1.0", - "name": "Qwen2.5-7B-olm-v1.0", - "developer": "jeffmeloy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5331, - "hfopenllm_v2/BBH": 0.566, - "hfopenllm_v2/MATH Level 5": 0.2863, - "hfopenllm_v2/GPQA": 0.3205, - "hfopenllm_v2/MUSR": 0.4278, - "hfopenllm_v2/MMLU-PRO": 0.4566 - } - }, - { - "id": "jeffmeloy/Qwen2.5-7B-olm-v1.1", - "name": "Qwen2.5-7B-olm-v1.1", - "developer": "jeffmeloy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4329, - "hfopenllm_v2/BBH": 0.5478, - "hfopenllm_v2/MATH Level 5": 0.3829, - "hfopenllm_v2/GPQA": 0.3087, - "hfopenllm_v2/MUSR": 0.4808, - "hfopenllm_v2/MMLU-PRO": 0.4354 - } - }, - { - "id": "jeffmeloy/Qwen2.5-7B-olm-v1.2", - "name": "Qwen2.5-7B-olm-v1.2", - "developer": "jeffmeloy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4203, - "hfopenllm_v2/BBH": 0.5533, - "hfopenllm_v2/MATH Level 5": 0.2847, - "hfopenllm_v2/GPQA": 0.3171, - "hfopenllm_v2/MUSR": 0.4688, - "hfopenllm_v2/MMLU-PRO": 0.4387 - } - }, - { - "id": "jeffmeloy/Qwen2.5-7B-olm-v1.3", - "name": "Qwen2.5-7B-olm-v1.3", - "developer": "jeffmeloy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4219, - "hfopenllm_v2/BBH": 0.5532, - "hfopenllm_v2/MATH Level 5": 0.3104, - "hfopenllm_v2/GPQA": 0.3213, - "hfopenllm_v2/MUSR": 0.4701, - "hfopenllm_v2/MMLU-PRO": 0.447 - } - }, - { - "id": "jeffmeloy/Qwen2.5-7B-olm-v1.4", - "name": "Qwen2.5-7B-olm-v1.4", - "developer": "jeffmeloy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4545, - "hfopenllm_v2/BBH": 0.5582, - "hfopenllm_v2/MATH Level 5": 0.2923, - "hfopenllm_v2/GPQA": 0.3121, - "hfopenllm_v2/MUSR": 0.4622, - "hfopenllm_v2/MMLU-PRO": 0.4457 - } - }, - { - "id": "jeffmeloy/Qwen2.5-7B-olm-v1.5", - "name": "Qwen2.5-7B-olm-v1.5", - "developer": "jeffmeloy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4547, - "hfopenllm_v2/BBH": 0.5544, - "hfopenllm_v2/MATH Level 5": 0.2817, - "hfopenllm_v2/GPQA": 0.3398, - "hfopenllm_v2/MUSR": 0.4539, - "hfopenllm_v2/MMLU-PRO": 0.4399 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/jeonsworld.json b/data/developers/jeonsworld.json deleted file mode 100644 index 66a7da435776e15ee799e0eb55a917e7b5fc8d81..0000000000000000000000000000000000000000 --- a/data/developers/jeonsworld.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "jeonsworld", - "models": [ - { - "id": "jeonsworld/CarbonVillain-en-10.7B-v4", - "name": "CarbonVillain-en-10.7B-v4", - "developer": "jeonsworld", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4579, - "hfopenllm_v2/BBH": 0.5168, - "hfopenllm_v2/MATH Level 5": 0.0468, - "hfopenllm_v2/GPQA": 0.3062, - "hfopenllm_v2/MUSR": 0.3965, - "hfopenllm_v2/MMLU-PRO": 0.3142 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/jiangxinyang-shanda.json b/data/developers/jiangxinyang-shanda.json deleted file mode 100644 index a117dd12e1248c966f9863b47e12b1d37fb191c2..0000000000000000000000000000000000000000 --- a/data/developers/jiangxinyang-shanda.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "jiangxinyang-shanda", - "models": [ - { - "id": "jiangxinyang-shanda/Homer-LLama3-8B", - "name": "Homer-LLama3-8B", - "developer": "jiangxinyang-shanda", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3992, - "hfopenllm_v2/BBH": 0.5173, - "hfopenllm_v2/MATH Level 5": 0.0861, - "hfopenllm_v2/GPQA": 0.297, - "hfopenllm_v2/MUSR": 0.4056, - "hfopenllm_v2/MMLU-PRO": 0.3139 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/jieliu.json b/data/developers/jieliu.json deleted file mode 100644 index 3180e8761112c19266b8020f255f076dedb5ea93..0000000000000000000000000000000000000000 --- a/data/developers/jieliu.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "jieliu", - "models": [ - { - "id": "jieliu/Storm-7B", - "name": "Storm-7B", - "developer": "jieliu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3424, - "hfopenllm_v2/BBH": 0.5187, - "hfopenllm_v2/MATH Level 5": 0.0612, - "hfopenllm_v2/GPQA": 0.3079, - "hfopenllm_v2/MUSR": 0.4429, - "hfopenllm_v2/MMLU-PRO": 0.3119 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/jimmy19991222.json b/data/developers/jimmy19991222.json deleted file mode 100644 index 365c60d260b4e65c9f6769c397a302e1fb4ed6bc..0000000000000000000000000000000000000000 --- a/data/developers/jimmy19991222.json +++ /dev/null @@ -1,117 +0,0 @@ -{ - "developer": "Jimmy19991222", - "models": [ - { - "id": "Jimmy19991222/llama-3-8b-instruct-gapo-v2-bert-f1-beta10-gamma0.3-lr1.0e-6-1minus-rerun", - "name": "llama-3-8b-instruct-gapo-v2-bert-f1-beta10-gamma0.3-lr1.0e-6-1minus-rerun", - "developer": "Jimmy19991222", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6717, - "hfopenllm_v2/BBH": 0.488, - "hfopenllm_v2/MATH Level 5": 0.0604, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.4041, - "hfopenllm_v2/MMLU-PRO": 0.3634 - } - }, - { - "id": "Jimmy19991222/llama-3-8b-instruct-gapo-v2-bert_f1-beta10-gamma0.3-lr1.0e-6-scale-log", - "name": "llama-3-8b-instruct-gapo-v2-bert_f1-beta10-gamma0.3-lr1.0e-6-scale-log", - "developer": "Jimmy19991222", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6556, - "hfopenllm_v2/BBH": 0.4935, - "hfopenllm_v2/MATH Level 5": 0.0544, - "hfopenllm_v2/GPQA": 0.3045, - "hfopenllm_v2/MUSR": 0.4, - "hfopenllm_v2/MMLU-PRO": 0.3658 - } - }, - { - "id": "Jimmy19991222/llama-3-8b-instruct-gapo-v2-bert_p-beta10-gamma0.3-lr1.0e-6-scale-log", - "name": "llama-3-8b-instruct-gapo-v2-bert_p-beta10-gamma0.3-lr1.0e-6-scale-log", - "developer": "Jimmy19991222", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6315, - "hfopenllm_v2/BBH": 0.4916, - "hfopenllm_v2/MATH Level 5": 0.065, - "hfopenllm_v2/GPQA": 0.2861, - "hfopenllm_v2/MUSR": 0.3935, - "hfopenllm_v2/MMLU-PRO": 0.3611 - } - }, - { - "id": "Jimmy19991222/llama-3-8b-instruct-gapo-v2-bleu-beta0.1-no-length-scale-gamma0.4", - "name": "llama-3-8b-instruct-gapo-v2-bleu-beta0.1-no-length-scale-gamma0.4", - "developer": "Jimmy19991222", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6285, - "hfopenllm_v2/BBH": 0.4986, - "hfopenllm_v2/MATH Level 5": 0.0514, - "hfopenllm_v2/GPQA": 0.2928, - "hfopenllm_v2/MUSR": 0.4014, - "hfopenllm_v2/MMLU-PRO": 0.3545 - } - }, - { - "id": "Jimmy19991222/llama-3-8b-instruct-gapo-v2-rouge2-beta10-1minus-gamma0.3-rerun", - "name": "llama-3-8b-instruct-gapo-v2-rouge2-beta10-1minus-gamma0.3-rerun", - "developer": "Jimmy19991222", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6678, - "hfopenllm_v2/BBH": 0.494, - "hfopenllm_v2/MATH Level 5": 0.0612, - "hfopenllm_v2/GPQA": 0.3062, - "hfopenllm_v2/MUSR": 0.3987, - "hfopenllm_v2/MMLU-PRO": 0.3658 - } - }, - { - "id": "Jimmy19991222/llama-3-8b-instruct-gapo-v2-rouge2-beta10-gamma0.3-lr1.0e-6-scale-log", - "name": "llama-3-8b-instruct-gapo-v2-rouge2-beta10-gamma0.3-lr1.0e-6-scale-log", - "developer": "Jimmy19991222", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6605, - "hfopenllm_v2/BBH": 0.4916, - "hfopenllm_v2/MATH Level 5": 0.0657, - "hfopenllm_v2/GPQA": 0.3037, - "hfopenllm_v2/MUSR": 0.4, - "hfopenllm_v2/MMLU-PRO": 0.3664 - } - }, - { - "id": "Jimmy19991222/llama-3-8b-instruct-gapo-v2-rougeL-beta10-gamma0.3-lr1.0e-6-scale-log", - "name": "llama-3-8b-instruct-gapo-v2-rougeL-beta10-gamma0.3-lr1.0e-6-scale-log", - "developer": "Jimmy19991222", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6492, - "hfopenllm_v2/BBH": 0.4952, - "hfopenllm_v2/MATH Level 5": 0.0642, - "hfopenllm_v2/GPQA": 0.302, - "hfopenllm_v2/MUSR": 0.3961, - "hfopenllm_v2/MMLU-PRO": 0.3711 - } - }, - { - "id": "Jimmy19991222/Llama-3-Instruct-8B-SimPO-v0.2", - "name": "Llama-3-Instruct-8B-SimPO-v0.2", - "developer": "Jimmy19991222", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.654, - "hfopenllm_v2/BBH": 0.4984, - "hfopenllm_v2/MATH Level 5": 0.0619, - "hfopenllm_v2/GPQA": 0.3146, - "hfopenllm_v2/MUSR": 0.4013, - "hfopenllm_v2/MMLU-PRO": 0.3686 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/jiviai.json b/data/developers/jiviai.json deleted file mode 100644 index 0d7dce80a40ec9dabe53ceda7d61a4e9c8397c4e..0000000000000000000000000000000000000000 --- a/data/developers/jiviai.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "jiviai", - "models": [ - { - "id": "jiviai/medX_v2", - "name": "medX_v2", - "developer": "jiviai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3743, - "hfopenllm_v2/BBH": 0.4509, - "hfopenllm_v2/MATH Level 5": 0.0544, - "hfopenllm_v2/GPQA": 0.323, - "hfopenllm_v2/MUSR": 0.3498, - "hfopenllm_v2/MMLU-PRO": 0.3428 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/jlzhou.json b/data/developers/jlzhou.json deleted file mode 100644 index fbe69b6f4579a76ba77d51ba0528d76c911dd226..0000000000000000000000000000000000000000 --- a/data/developers/jlzhou.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "jlzhou", - "models": [ - { - "id": "jlzhou/Qwen2.5-3B-Infinity-Instruct-0625", - "name": "Qwen2.5-3B-Infinity-Instruct-0625", - "developer": "jlzhou", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3558, - "hfopenllm_v2/BBH": 0.4774, - "hfopenllm_v2/MATH Level 5": 0.1367, - "hfopenllm_v2/GPQA": 0.2693, - "hfopenllm_v2/MUSR": 0.3981, - "hfopenllm_v2/MMLU-PRO": 0.3199 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/johnsutor.json b/data/developers/johnsutor.json deleted file mode 100644 index f779ab2f470853f18bd8b0de0fa0b59de6f6df4f..0000000000000000000000000000000000000000 --- a/data/developers/johnsutor.json +++ /dev/null @@ -1,439 +0,0 @@ -{ - "developer": "johnsutor", - "models": [ - { - "id": "johnsutor/Llama-3-8B-Instruct_breadcrumbs-density-0.1-gamma-0.01", - "name": "Llama-3-8B-Instruct_breadcrumbs-density-0.1-gamma-0.01", - "developer": "johnsutor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4271, - "hfopenllm_v2/BBH": 0.5036, - "hfopenllm_v2/MATH Level 5": 0.0453, - "hfopenllm_v2/GPQA": 0.3221, - "hfopenllm_v2/MUSR": 0.4638, - "hfopenllm_v2/MMLU-PRO": 0.3739 - } - }, - { - "id": "johnsutor/Llama-3-8B-Instruct_breadcrumbs-density-0.1-gamma-0.1", - "name": "Llama-3-8B-Instruct_breadcrumbs-density-0.1-gamma-0.1", - "developer": "johnsutor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4253, - "hfopenllm_v2/BBH": 0.5019, - "hfopenllm_v2/MATH Level 5": 0.0967, - "hfopenllm_v2/GPQA": 0.3012, - "hfopenllm_v2/MUSR": 0.415, - "hfopenllm_v2/MMLU-PRO": 0.3724 - } - }, - { - "id": "johnsutor/Llama-3-8B-Instruct_breadcrumbs-density-0.3-gamma-0.01", - "name": "Llama-3-8B-Instruct_breadcrumbs-density-0.3-gamma-0.01", - "developer": "johnsutor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3377, - "hfopenllm_v2/BBH": 0.4917, - "hfopenllm_v2/MATH Level 5": 0.0106, - "hfopenllm_v2/GPQA": 0.3121, - "hfopenllm_v2/MUSR": 0.5018, - "hfopenllm_v2/MMLU-PRO": 0.3533 - } - }, - { - "id": "johnsutor/Llama-3-8B-Instruct_breadcrumbs-density-0.3-gamma-0.1", - "name": "Llama-3-8B-Instruct_breadcrumbs-density-0.3-gamma-0.1", - "developer": "johnsutor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4274, - "hfopenllm_v2/BBH": 0.5126, - "hfopenllm_v2/MATH Level 5": 0.0808, - "hfopenllm_v2/GPQA": 0.3087, - "hfopenllm_v2/MUSR": 0.4226, - "hfopenllm_v2/MMLU-PRO": 0.3739 - } - }, - { - "id": "johnsutor/Llama-3-8B-Instruct_breadcrumbs-density-0.5-gamma-0.01", - "name": "Llama-3-8B-Instruct_breadcrumbs-density-0.5-gamma-0.01", - "developer": "johnsutor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3204, - "hfopenllm_v2/BBH": 0.4884, - "hfopenllm_v2/MATH Level 5": 0.0038, - "hfopenllm_v2/GPQA": 0.302, - "hfopenllm_v2/MUSR": 0.5098, - "hfopenllm_v2/MMLU-PRO": 0.3344 - } - }, - { - "id": "johnsutor/Llama-3-8B-Instruct_breadcrumbs-density-0.5-gamma-0.1", - "name": "Llama-3-8B-Instruct_breadcrumbs-density-0.5-gamma-0.1", - "developer": "johnsutor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4396, - "hfopenllm_v2/BBH": 0.514, - "hfopenllm_v2/MATH Level 5": 0.0801, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.4398, - "hfopenllm_v2/MMLU-PRO": 0.3696 - } - }, - { - "id": "johnsutor/Llama-3-8B-Instruct_breadcrumbs-density-0.7-gamma-0.01", - "name": "Llama-3-8B-Instruct_breadcrumbs-density-0.7-gamma-0.01", - "developer": "johnsutor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2814, - "hfopenllm_v2/BBH": 0.4854, - "hfopenllm_v2/MATH Level 5": 0.0023, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.5163, - "hfopenllm_v2/MMLU-PRO": 0.3295 - } - }, - { - "id": "johnsutor/Llama-3-8B-Instruct_breadcrumbs-density-0.7-gamma-0.1", - "name": "Llama-3-8B-Instruct_breadcrumbs-density-0.7-gamma-0.1", - "developer": "johnsutor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4302, - "hfopenllm_v2/BBH": 0.5157, - "hfopenllm_v2/MATH Level 5": 0.0627, - "hfopenllm_v2/GPQA": 0.3079, - "hfopenllm_v2/MUSR": 0.4332, - "hfopenllm_v2/MMLU-PRO": 0.3663 - } - }, - { - "id": "johnsutor/Llama-3-8B-Instruct_breadcrumbs-density-0.9-gamma-0.01", - "name": "Llama-3-8B-Instruct_breadcrumbs-density-0.9-gamma-0.01", - "developer": "johnsutor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.279, - "hfopenllm_v2/BBH": 0.4861, - "hfopenllm_v2/MATH Level 5": 0.0015, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.515, - "hfopenllm_v2/MMLU-PRO": 0.3305 - } - }, - { - "id": "johnsutor/Llama-3-8B-Instruct_breadcrumbs-density-0.9-gamma-0.1", - "name": "Llama-3-8B-Instruct_breadcrumbs-density-0.9-gamma-0.1", - "developer": "johnsutor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4223, - "hfopenllm_v2/BBH": 0.5154, - "hfopenllm_v2/MATH Level 5": 0.074, - "hfopenllm_v2/GPQA": 0.3079, - "hfopenllm_v2/MUSR": 0.4384, - "hfopenllm_v2/MMLU-PRO": 0.365 - } - }, - { - "id": "johnsutor/Llama-3-8B-Instruct_breadcrumbs_ties-density-0.1-gamma-0.01", - "name": "Llama-3-8B-Instruct_breadcrumbs_ties-density-0.1-gamma-0.01", - "developer": "johnsutor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4359, - "hfopenllm_v2/BBH": 0.5041, - "hfopenllm_v2/MATH Level 5": 0.0483, - "hfopenllm_v2/GPQA": 0.3104, - "hfopenllm_v2/MUSR": 0.4532, - "hfopenllm_v2/MMLU-PRO": 0.3762 - } - }, - { - "id": "johnsutor/Llama-3-8B-Instruct_breadcrumbs_ties-density-0.1-gamma-0.1", - "name": "Llama-3-8B-Instruct_breadcrumbs_ties-density-0.1-gamma-0.1", - "developer": "johnsutor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4202, - "hfopenllm_v2/BBH": 0.5011, - "hfopenllm_v2/MATH Level 5": 0.0982, - "hfopenllm_v2/GPQA": 0.3003, - "hfopenllm_v2/MUSR": 0.415, - "hfopenllm_v2/MMLU-PRO": 0.3699 - } - }, - { - "id": "johnsutor/Llama-3-8B-Instruct_breadcrumbs_ties-density-0.3-gamma-0.01", - "name": "Llama-3-8B-Instruct_breadcrumbs_ties-density-0.3-gamma-0.01", - "developer": "johnsutor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3518, - "hfopenllm_v2/BBH": 0.4999, - "hfopenllm_v2/MATH Level 5": 0.0234, - "hfopenllm_v2/GPQA": 0.3062, - "hfopenllm_v2/MUSR": 0.4871, - "hfopenllm_v2/MMLU-PRO": 0.3611 - } - }, - { - "id": "johnsutor/Llama-3-8B-Instruct_breadcrumbs_ties-density-0.3-gamma-0.1", - "name": "Llama-3-8B-Instruct_breadcrumbs_ties-density-0.3-gamma-0.1", - "developer": "johnsutor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4204, - "hfopenllm_v2/BBH": 0.5107, - "hfopenllm_v2/MATH Level 5": 0.0876, - "hfopenllm_v2/GPQA": 0.3045, - "hfopenllm_v2/MUSR": 0.4279, - "hfopenllm_v2/MMLU-PRO": 0.371 - } - }, - { - "id": "johnsutor/Llama-3-8B-Instruct_breadcrumbs_ties-density-0.5-gamma-0.01", - "name": "Llama-3-8B-Instruct_breadcrumbs_ties-density-0.5-gamma-0.01", - "developer": "johnsutor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3454, - "hfopenllm_v2/BBH": 0.4984, - "hfopenllm_v2/MATH Level 5": 0.0219, - "hfopenllm_v2/GPQA": 0.297, - "hfopenllm_v2/MUSR": 0.4911, - "hfopenllm_v2/MMLU-PRO": 0.3531 - } - }, - { - "id": "johnsutor/Llama-3-8B-Instruct_breadcrumbs_ties-density-0.5-gamma-0.1", - "name": "Llama-3-8B-Instruct_breadcrumbs_ties-density-0.5-gamma-0.1", - "developer": "johnsutor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4092, - "hfopenllm_v2/BBH": 0.5137, - "hfopenllm_v2/MATH Level 5": 0.0808, - "hfopenllm_v2/GPQA": 0.2953, - "hfopenllm_v2/MUSR": 0.4357, - "hfopenllm_v2/MMLU-PRO": 0.3669 - } - }, - { - "id": "johnsutor/Llama-3-8B-Instruct_breadcrumbs_ties-density-0.7-gamma-0.01", - "name": "Llama-3-8B-Instruct_breadcrumbs_ties-density-0.7-gamma-0.01", - "developer": "johnsutor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2904, - "hfopenllm_v2/BBH": 0.4967, - "hfopenllm_v2/MATH Level 5": 0.0144, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.4991, - "hfopenllm_v2/MMLU-PRO": 0.349 - } - }, - { - "id": "johnsutor/Llama-3-8B-Instruct_breadcrumbs_ties-density-0.7-gamma-0.1", - "name": "Llama-3-8B-Instruct_breadcrumbs_ties-density-0.7-gamma-0.1", - "developer": "johnsutor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4199, - "hfopenllm_v2/BBH": 0.5147, - "hfopenllm_v2/MATH Level 5": 0.0808, - "hfopenllm_v2/GPQA": 0.2987, - "hfopenllm_v2/MUSR": 0.4358, - "hfopenllm_v2/MMLU-PRO": 0.3615 - } - }, - { - "id": "johnsutor/Llama-3-8B-Instruct_breadcrumbs_ties-density-0.9-gamma-0.01", - "name": "Llama-3-8B-Instruct_breadcrumbs_ties-density-0.9-gamma-0.01", - "developer": "johnsutor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2913, - "hfopenllm_v2/BBH": 0.4918, - "hfopenllm_v2/MATH Level 5": 0.0106, - "hfopenllm_v2/GPQA": 0.3003, - "hfopenllm_v2/MUSR": 0.4977, - "hfopenllm_v2/MMLU-PRO": 0.3454 - } - }, - { - "id": "johnsutor/Llama-3-8B-Instruct_breadcrumbs_ties-density-0.9-gamma-0.1", - "name": "Llama-3-8B-Instruct_breadcrumbs_ties-density-0.9-gamma-0.1", - "developer": "johnsutor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4162, - "hfopenllm_v2/BBH": 0.5139, - "hfopenllm_v2/MATH Level 5": 0.0778, - "hfopenllm_v2/GPQA": 0.297, - "hfopenllm_v2/MUSR": 0.4317, - "hfopenllm_v2/MMLU-PRO": 0.3625 - } - }, - { - "id": "johnsutor/Llama-3-8B-Instruct_dare_linear", - "name": "Llama-3-8B-Instruct_dare_linear", - "developer": "johnsutor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2145, - "hfopenllm_v2/BBH": 0.4283, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.4979, - "hfopenllm_v2/MMLU-PRO": 0.2414 - } - }, - { - "id": "johnsutor/Llama-3-8B-Instruct_dare_ties-density-0.1", - "name": "Llama-3-8B-Instruct_dare_ties-density-0.1", - "developer": "johnsutor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1891, - "hfopenllm_v2/BBH": 0.4119, - "hfopenllm_v2/MATH Level 5": 0.0008, - "hfopenllm_v2/GPQA": 0.2718, - "hfopenllm_v2/MUSR": 0.4658, - "hfopenllm_v2/MMLU-PRO": 0.2265 - } - }, - { - "id": "johnsutor/Llama-3-8B-Instruct_dare_ties-density-0.3", - "name": "Llama-3-8B-Instruct_dare_ties-density-0.3", - "developer": "johnsutor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2113, - "hfopenllm_v2/BBH": 0.4559, - "hfopenllm_v2/MATH Level 5": 0.0015, - "hfopenllm_v2/GPQA": 0.297, - "hfopenllm_v2/MUSR": 0.5069, - "hfopenllm_v2/MMLU-PRO": 0.304 - } - }, - { - "id": "johnsutor/Llama-3-8B-Instruct_dare_ties-density-0.7", - "name": "Llama-3-8B-Instruct_dare_ties-density-0.7", - "developer": "johnsutor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2034, - "hfopenllm_v2/BBH": 0.4723, - "hfopenllm_v2/MATH Level 5": 0.003, - "hfopenllm_v2/GPQA": 0.3037, - "hfopenllm_v2/MUSR": 0.511, - "hfopenllm_v2/MMLU-PRO": 0.3148 - } - }, - { - "id": "johnsutor/Llama-3-8B-Instruct_dare_ties-density-0.9", - "name": "Llama-3-8B-Instruct_dare_ties-density-0.9", - "developer": "johnsutor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2161, - "hfopenllm_v2/BBH": 0.4664, - "hfopenllm_v2/MATH Level 5": 0.0015, - "hfopenllm_v2/GPQA": 0.3079, - "hfopenllm_v2/MUSR": 0.523, - "hfopenllm_v2/MMLU-PRO": 0.3143 - } - }, - { - "id": "johnsutor/Llama-3-8B-Instruct_linear", - "name": "Llama-3-8B-Instruct_linear", - "developer": "johnsutor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4308, - "hfopenllm_v2/BBH": 0.5031, - "hfopenllm_v2/MATH Level 5": 0.1005, - "hfopenllm_v2/GPQA": 0.2953, - "hfopenllm_v2/MUSR": 0.4097, - "hfopenllm_v2/MMLU-PRO": 0.3712 - } - }, - { - "id": "johnsutor/Llama-3-8B-Instruct_ties-density-0.1", - "name": "Llama-3-8B-Instruct_ties-density-0.1", - "developer": "johnsutor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4116, - "hfopenllm_v2/BBH": 0.5021, - "hfopenllm_v2/MATH Level 5": 0.0793, - "hfopenllm_v2/GPQA": 0.2886, - "hfopenllm_v2/MUSR": 0.4174, - "hfopenllm_v2/MMLU-PRO": 0.36 - } - }, - { - "id": "johnsutor/Llama-3-8B-Instruct_ties-density-0.3", - "name": "Llama-3-8B-Instruct_ties-density-0.3", - "developer": "johnsutor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3626, - "hfopenllm_v2/BBH": 0.4906, - "hfopenllm_v2/MATH Level 5": 0.0672, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.4025, - "hfopenllm_v2/MMLU-PRO": 0.3321 - } - }, - { - "id": "johnsutor/Llama-3-8B-Instruct_ties-density-0.5", - "name": "Llama-3-8B-Instruct_ties-density-0.5", - "developer": "johnsutor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3797, - "hfopenllm_v2/BBH": 0.4793, - "hfopenllm_v2/MATH Level 5": 0.0612, - "hfopenllm_v2/GPQA": 0.3045, - "hfopenllm_v2/MUSR": 0.388, - "hfopenllm_v2/MMLU-PRO": 0.3175 - } - }, - { - "id": "johnsutor/Llama-3-8B-Instruct_ties-density-0.7", - "name": "Llama-3-8B-Instruct_ties-density-0.7", - "developer": "johnsutor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3681, - "hfopenllm_v2/BBH": 0.4738, - "hfopenllm_v2/MATH Level 5": 0.0672, - "hfopenllm_v2/GPQA": 0.3096, - "hfopenllm_v2/MUSR": 0.3881, - "hfopenllm_v2/MMLU-PRO": 0.3152 - } - }, - { - "id": "johnsutor/Llama-3-8B-Instruct_ties-density-0.9", - "name": "Llama-3-8B-Instruct_ties-density-0.9", - "developer": "johnsutor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3858, - "hfopenllm_v2/BBH": 0.4735, - "hfopenllm_v2/MATH Level 5": 0.0619, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.388, - "hfopenllm_v2/MMLU-PRO": 0.3182 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/jondurbin.json b/data/developers/jondurbin.json deleted file mode 100644 index 9184eff22aab70e07429839320522a10809d277e..0000000000000000000000000000000000000000 --- a/data/developers/jondurbin.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "jondurbin", - "models": [ - { - "id": "jondurbin/bagel-dpo-34b-v0.5", - "name": "jondurbin/bagel-dpo-34b-v0.5", - "developer": "jondurbin", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7215, - "reward-bench/Chat": 0.9385, - "reward-bench/Chat Hard": 0.5504, - "reward-bench/Safety": 0.6446, - "reward-bench/Reasoning": 0.8889, - "reward-bench/Prior Sets (0.5 weight)": 0.4487 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/joseph717171.json b/data/developers/joseph717171.json deleted file mode 100644 index 6363ccc8ae26d06a2ddb252774113b997a3919cc..0000000000000000000000000000000000000000 --- a/data/developers/joseph717171.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "Joseph717171", - "models": [ - { - "id": "Joseph717171/Hermes-3-Llama-3.1-8B_TIES_with_Base_Embeds_Initialized_to_Special_Instruct_Toks_dtypeF32", - "name": "Hermes-3-Llama-3.1-8B_TIES_with_Base_Embeds_Initialized_to_Special_Instruct_Toks_dtypeF32", - "developer": "Joseph717171", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6185, - "hfopenllm_v2/BBH": 0.5177, - "hfopenllm_v2/MATH Level 5": 0.0514, - "hfopenllm_v2/GPQA": 0.2827, - "hfopenllm_v2/MUSR": 0.4369, - "hfopenllm_v2/MMLU-PRO": 0.3144 - } - }, - { - "id": "Joseph717171/Llama-3.1-SuperNova-8B-Lite_TIES_with_Base", - "name": "Llama-3.1-SuperNova-8B-Lite_TIES_with_Base", - "developer": "Joseph717171", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8096, - "hfopenllm_v2/BBH": 0.5147, - "hfopenllm_v2/MATH Level 5": 0.1835, - "hfopenllm_v2/GPQA": 0.3096, - "hfopenllm_v2/MUSR": 0.411, - "hfopenllm_v2/MMLU-PRO": 0.388 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/josephgflowers.json b/data/developers/josephgflowers.json deleted file mode 100644 index 99e8d43d52dd93ef9d5e26004c70b5f5568cc7e7..0000000000000000000000000000000000000000 --- a/data/developers/josephgflowers.json +++ /dev/null @@ -1,103 +0,0 @@ -{ - "developer": "Josephgflowers", - "models": [ - { - "id": "Josephgflowers/Cinder-Phi-2-V1-F16-gguf", - "name": "Cinder-Phi-2-V1-F16-gguf", - "developer": "Josephgflowers", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2357, - "hfopenllm_v2/BBH": 0.4397, - "hfopenllm_v2/MATH Level 5": 0.0242, - "hfopenllm_v2/GPQA": 0.2819, - "hfopenllm_v2/MUSR": 0.3435, - "hfopenllm_v2/MMLU-PRO": 0.2161 - } - }, - { - "id": "Josephgflowers/Differential-Attention-Liquid-Metal-Tinyllama", - "name": "Differential-Attention-Liquid-Metal-Tinyllama", - "developer": "Josephgflowers", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2227, - "hfopenllm_v2/BBH": 0.2926, - "hfopenllm_v2/MATH Level 5": 0.0325, - "hfopenllm_v2/GPQA": 0.2508, - "hfopenllm_v2/MUSR": 0.3356, - "hfopenllm_v2/MMLU-PRO": 0.1214 - } - }, - { - "id": "Josephgflowers/TinyLlama-Cinder-Agent-v1", - "name": "TinyLlama-Cinder-Agent-v1", - "developer": "Josephgflowers", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.267, - "hfopenllm_v2/BBH": 0.3116, - "hfopenllm_v2/MATH Level 5": 0.0347, - "hfopenllm_v2/GPQA": 0.2441, - "hfopenllm_v2/MUSR": 0.3395, - "hfopenllm_v2/MMLU-PRO": 0.1161 - } - }, - { - "id": "Josephgflowers/Tinyllama-r1", - "name": "Tinyllama-r1", - "developer": "Josephgflowers", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2119, - "hfopenllm_v2/BBH": 0.3015, - "hfopenllm_v2/MATH Level 5": 0.0325, - "hfopenllm_v2/GPQA": 0.2567, - "hfopenllm_v2/MUSR": 0.3315, - "hfopenllm_v2/MMLU-PRO": 0.1134 - } - }, - { - "id": "Josephgflowers/Tinyllama-STEM-Cinder-Agent-v1", - "name": "Tinyllama-STEM-Cinder-Agent-v1", - "developer": "Josephgflowers", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2126, - "hfopenllm_v2/BBH": 0.3084, - "hfopenllm_v2/MATH Level 5": 0.0672, - "hfopenllm_v2/GPQA": 0.2349, - "hfopenllm_v2/MUSR": 0.3341, - "hfopenllm_v2/MMLU-PRO": 0.1086 - } - }, - { - "id": "Josephgflowers/TinyLlama-v1.1-Cinders-World", - "name": "TinyLlama-v1.1-Cinders-World", - "developer": "Josephgflowers", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2469, - "hfopenllm_v2/BBH": 0.2998, - "hfopenllm_v2/MATH Level 5": 0.0347, - "hfopenllm_v2/GPQA": 0.2441, - "hfopenllm_v2/MUSR": 0.3356, - "hfopenllm_v2/MMLU-PRO": 0.1198 - } - }, - { - "id": "Josephgflowers/TinyLlama_v1.1_math_code-world-test-1", - "name": "TinyLlama_v1.1_math_code-world-test-1", - "developer": "Josephgflowers", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0078, - "hfopenllm_v2/BBH": 0.3146, - "hfopenllm_v2/MATH Level 5": 0.0196, - "hfopenllm_v2/GPQA": 0.2341, - "hfopenllm_v2/MUSR": 0.3499, - "hfopenllm_v2/MMLU-PRO": 0.1132 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/jpacifico.json b/data/developers/jpacifico.json deleted file mode 100644 index 1969c4a11f52a585d9eecffb9498875e3a4668b6..0000000000000000000000000000000000000000 --- a/data/developers/jpacifico.json +++ /dev/null @@ -1,257 +0,0 @@ -{ - "developer": "jpacifico", - "models": [ - { - "id": "jpacifico/Chocolatine-14B-Instruct-4k-DPO", - "name": "Chocolatine-14B-Instruct-4k-DPO", - "developer": "jpacifico", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4689, - "hfopenllm_v2/BBH": 0.63, - "hfopenllm_v2/MATH Level 5": 0.1782, - "hfopenllm_v2/GPQA": 0.3414, - "hfopenllm_v2/MUSR": 0.4439, - "hfopenllm_v2/MMLU-PRO": 0.4764 - } - }, - { - "id": "jpacifico/Chocolatine-14B-Instruct-DPO-v1.2", - "name": "Chocolatine-14B-Instruct-DPO-v1.2", - "developer": "jpacifico", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6852, - "hfopenllm_v2/BBH": 0.6438, - "hfopenllm_v2/MATH Level 5": 0.2092, - "hfopenllm_v2/GPQA": 0.3255, - "hfopenllm_v2/MUSR": 0.4268, - "hfopenllm_v2/MMLU-PRO": 0.4697 - } - }, - { - "id": "jpacifico/Chocolatine-14B-Instruct-DPO-v1.3", - "name": "Chocolatine-14B-Instruct-DPO-v1.3", - "developer": "jpacifico", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.704, - "hfopenllm_v2/BBH": 0.6846, - "hfopenllm_v2/MATH Level 5": 0.5619, - "hfopenllm_v2/GPQA": 0.3414, - "hfopenllm_v2/MUSR": 0.4234, - "hfopenllm_v2/MMLU-PRO": 0.5374 - } - }, - { - "id": "jpacifico/Chocolatine-2-14B-Instruct-DPO-v2.0b1", - "name": "Chocolatine-2-14B-Instruct-DPO-v2.0b1", - "developer": "jpacifico", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1033, - "hfopenllm_v2/BBH": 0.6696, - "hfopenllm_v2/MATH Level 5": 0.2757, - "hfopenllm_v2/GPQA": 0.3758, - "hfopenllm_v2/MUSR": 0.4467, - "hfopenllm_v2/MMLU-PRO": 0.5124 - } - }, - { - "id": "jpacifico/Chocolatine-2-14B-Instruct-v2.0", - "name": "Chocolatine-2-14B-Instruct-v2.0", - "developer": "jpacifico", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0885, - "hfopenllm_v2/BBH": 0.677, - "hfopenllm_v2/MATH Level 5": 0.4804, - "hfopenllm_v2/GPQA": 0.3876, - "hfopenllm_v2/MUSR": 0.5021, - "hfopenllm_v2/MMLU-PRO": 0.5302 - } - }, - { - "id": "jpacifico/Chocolatine-2-14B-Instruct-v2.0.1", - "name": "Chocolatine-2-14B-Instruct-v2.0.1", - "developer": "jpacifico", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0742, - "hfopenllm_v2/BBH": 0.6736, - "hfopenllm_v2/MATH Level 5": 0.4796, - "hfopenllm_v2/GPQA": 0.3918, - "hfopenllm_v2/MUSR": 0.5008, - "hfopenllm_v2/MMLU-PRO": 0.5299 - } - }, - { - "id": "jpacifico/Chocolatine-2-14B-Instruct-v2.0.3", - "name": "Chocolatine-2-14B-Instruct-v2.0.3", - "developer": "jpacifico", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7037, - "hfopenllm_v2/BBH": 0.6548, - "hfopenllm_v2/MATH Level 5": 0.4207, - "hfopenllm_v2/GPQA": 0.3792, - "hfopenllm_v2/MUSR": 0.4768, - "hfopenllm_v2/MMLU-PRO": 0.5374 - } - }, - { - "id": "jpacifico/Chocolatine-2-14B-Instruct-v2.0b2", - "name": "Chocolatine-2-14B-Instruct-v2.0b2", - "developer": "jpacifico", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7241, - "hfopenllm_v2/BBH": 0.6476, - "hfopenllm_v2/MATH Level 5": 0.395, - "hfopenllm_v2/GPQA": 0.3834, - "hfopenllm_v2/MUSR": 0.4808, - "hfopenllm_v2/MMLU-PRO": 0.5369 - } - }, - { - "id": "jpacifico/Chocolatine-2-14B-Instruct-v2.0b3", - "name": "Chocolatine-2-14B-Instruct-v2.0b3", - "developer": "jpacifico", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7323, - "hfopenllm_v2/BBH": 0.6469, - "hfopenllm_v2/MATH Level 5": 0.4109, - "hfopenllm_v2/GPQA": 0.3792, - "hfopenllm_v2/MUSR": 0.4781, - "hfopenllm_v2/MMLU-PRO": 0.5337 - } - }, - { - "id": "jpacifico/Chocolatine-3B-Instruct-DPO-Revised", - "name": "Chocolatine-3B-Instruct-DPO-Revised", - "developer": "jpacifico", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5623, - "hfopenllm_v2/BBH": 0.554, - "hfopenllm_v2/MATH Level 5": 0.1805, - "hfopenllm_v2/GPQA": 0.3221, - "hfopenllm_v2/MUSR": 0.4453, - "hfopenllm_v2/MMLU-PRO": 0.3989 - } - }, - { - "id": "jpacifico/Chocolatine-3B-Instruct-DPO-v1.0", - "name": "Chocolatine-3B-Instruct-DPO-v1.0", - "developer": "jpacifico", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3737, - "hfopenllm_v2/BBH": 0.5471, - "hfopenllm_v2/MATH Level 5": 0.1782, - "hfopenllm_v2/GPQA": 0.3154, - "hfopenllm_v2/MUSR": 0.4755, - "hfopenllm_v2/MMLU-PRO": 0.3937 - } - }, - { - "id": "jpacifico/Chocolatine-3B-Instruct-DPO-v1.2", - "name": "Chocolatine-3B-Instruct-DPO-v1.2", - "developer": "jpacifico", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5455, - "hfopenllm_v2/BBH": 0.5487, - "hfopenllm_v2/MATH Level 5": 0.2047, - "hfopenllm_v2/GPQA": 0.3389, - "hfopenllm_v2/MUSR": 0.4154, - "hfopenllm_v2/MMLU-PRO": 0.3877 - } - }, - { - "id": "jpacifico/Distilucie-7B-Math-Instruct-DPO-v0.1", - "name": "Distilucie-7B-Math-Instruct-DPO-v0.1", - "developer": "jpacifico", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3048, - "hfopenllm_v2/BBH": 0.3835, - "hfopenllm_v2/MATH Level 5": 0.0257, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.3644, - "hfopenllm_v2/MMLU-PRO": 0.1809 - } - }, - { - "id": "jpacifico/Lucie-7B-Instruct-DPO-v1.1", - "name": "Lucie-7B-Instruct-DPO-v1.1", - "developer": "jpacifico", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3121, - "hfopenllm_v2/BBH": 0.3781, - "hfopenllm_v2/MATH Level 5": 0.0234, - "hfopenllm_v2/GPQA": 0.2878, - "hfopenllm_v2/MUSR": 0.4016, - "hfopenllm_v2/MMLU-PRO": 0.1838 - } - }, - { - "id": "jpacifico/Lucie-7B-Instruct-DPO-v1.1.3", - "name": "Lucie-7B-Instruct-DPO-v1.1.3", - "developer": "jpacifico", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3045, - "hfopenllm_v2/BBH": 0.3819, - "hfopenllm_v2/MATH Level 5": 0.0242, - "hfopenllm_v2/GPQA": 0.2861, - "hfopenllm_v2/MUSR": 0.3818, - "hfopenllm_v2/MMLU-PRO": 0.1764 - } - }, - { - "id": "jpacifico/Lucie-7B-Instruct-Merged-Model_Stock-v1.0", - "name": "Lucie-7B-Instruct-Merged-Model_Stock-v1.0", - "developer": "jpacifico", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3234, - "hfopenllm_v2/BBH": 0.3802, - "hfopenllm_v2/MATH Level 5": 0.0242, - "hfopenllm_v2/GPQA": 0.2886, - "hfopenllm_v2/MUSR": 0.3844, - "hfopenllm_v2/MMLU-PRO": 0.1871 - } - }, - { - "id": "jpacifico/Lucie-7B-Instruct-Merged-Model_Stock-v1.1", - "name": "Lucie-7B-Instruct-Merged-Model_Stock-v1.1", - "developer": "jpacifico", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3014, - "hfopenllm_v2/BBH": 0.3808, - "hfopenllm_v2/MATH Level 5": 0.0279, - "hfopenllm_v2/GPQA": 0.2827, - "hfopenllm_v2/MUSR": 0.375, - "hfopenllm_v2/MMLU-PRO": 0.1862 - } - }, - { - "id": "jpacifico/Lucie-Boosted-7B-Instruct", - "name": "Lucie-Boosted-7B-Instruct", - "developer": "jpacifico", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2566, - "hfopenllm_v2/BBH": 0.3465, - "hfopenllm_v2/MATH Level 5": 0.0128, - "hfopenllm_v2/GPQA": 0.2668, - "hfopenllm_v2/MUSR": 0.3699, - "hfopenllm_v2/MMLU-PRO": 0.163 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/jsfs11.json b/data/developers/jsfs11.json deleted file mode 100644 index d1bd91726362eb30d05bbdabbd0e3d2c21e9e432..0000000000000000000000000000000000000000 --- a/data/developers/jsfs11.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "developer": "jsfs11", - "models": [ - { - "id": "jsfs11/L3-8B-Stheno-slerp", - "name": "L3-8B-Stheno-slerp", - "developer": "jsfs11", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6752, - "hfopenllm_v2/BBH": 0.5326, - "hfopenllm_v2/MATH Level 5": 0.0989, - "hfopenllm_v2/GPQA": 0.2852, - "hfopenllm_v2/MUSR": 0.3725, - "hfopenllm_v2/MMLU-PRO": 0.3649 - } - }, - { - "id": "jsfs11/MixtureofMerges-MoE-4x7b-v4", - "name": "MixtureofMerges-MoE-4x7b-v4", - "developer": "jsfs11", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.403, - "hfopenllm_v2/BBH": 0.5169, - "hfopenllm_v2/MATH Level 5": 0.0634, - "hfopenllm_v2/GPQA": 0.2861, - "hfopenllm_v2/MUSR": 0.4386, - "hfopenllm_v2/MMLU-PRO": 0.3032 - } - }, - { - "id": "jsfs11/MixtureofMerges-MoE-4x7b-v5", - "name": "MixtureofMerges-MoE-4x7b-v5", - "developer": "jsfs11", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4199, - "hfopenllm_v2/BBH": 0.5198, - "hfopenllm_v2/MATH Level 5": 0.0755, - "hfopenllm_v2/GPQA": 0.2844, - "hfopenllm_v2/MUSR": 0.4305, - "hfopenllm_v2/MMLU-PRO": 0.3098 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/jungzoona.json b/data/developers/jungzoona.json deleted file mode 100644 index ac06e3f8c40f69e4c17995c20faab8440fb2647a..0000000000000000000000000000000000000000 --- a/data/developers/jungzoona.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "JungZoona", - "models": [ - { - "id": "JungZoona/T3Q-Qwen2.5-14B-Instruct-1M-e3", - "name": "T3Q-Qwen2.5-14B-Instruct-1M-e3", - "developer": "JungZoona", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7324, - "hfopenllm_v2/BBH": 0.7586, - "hfopenllm_v2/MATH Level 5": 0.2863, - "hfopenllm_v2/GPQA": 0.4169, - "hfopenllm_v2/MUSR": 0.5911, - "hfopenllm_v2/MMLU-PRO": 0.5884 - } - }, - { - "id": "JungZoona/T3Q-qwen2.5-14b-v1.0-e3", - "name": "T3Q-qwen2.5-14b-v1.0-e3", - "developer": "JungZoona", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7324, - "hfopenllm_v2/BBH": 0.7586, - "hfopenllm_v2/MATH Level 5": 0.2863, - "hfopenllm_v2/GPQA": 0.4169, - "hfopenllm_v2/MUSR": 0.5911, - "hfopenllm_v2/MMLU-PRO": 0.5884 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/junhoee.json b/data/developers/junhoee.json deleted file mode 100644 index 2b0c4f097bf8740d6939a8ab53509651fb64fb47..0000000000000000000000000000000000000000 --- a/data/developers/junhoee.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "Junhoee", - "models": [ - { - "id": "Junhoee/Qwen-Megumin", - "name": "Qwen-Megumin", - "developer": "Junhoee", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7141, - "hfopenllm_v2/BBH": 0.5285, - "hfopenllm_v2/MATH Level 5": 0.4902, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.398, - "hfopenllm_v2/MMLU-PRO": 0.4199 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/kaist-ai.json b/data/developers/kaist-ai.json deleted file mode 100644 index cafb4760301ac66a70e5dd87aada79bbc39137de..0000000000000000000000000000000000000000 --- a/data/developers/kaist-ai.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "developer": "kaist-ai", - "models": [ - { - "id": "kaist-ai/janus-7b", - "name": "janus-7b", - "developer": "kaist-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3775, - "hfopenllm_v2/BBH": 0.4694, - "hfopenllm_v2/MATH Level 5": 0.0408, - "hfopenllm_v2/GPQA": 0.2727, - "hfopenllm_v2/MUSR": 0.4401, - "hfopenllm_v2/MMLU-PRO": 0.2874 - } - }, - { - "id": "kaist-ai/janus-dpo-7b", - "name": "janus-dpo-7b", - "developer": "kaist-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4003, - "hfopenllm_v2/BBH": 0.4773, - "hfopenllm_v2/MATH Level 5": 0.0415, - "hfopenllm_v2/GPQA": 0.2819, - "hfopenllm_v2/MUSR": 0.4387, - "hfopenllm_v2/MMLU-PRO": 0.2976 - } - }, - { - "id": "kaist-ai/janus-rm-7b", - "name": "janus-rm-7b", - "developer": "kaist-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1778, - "hfopenllm_v2/BBH": 0.3056, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2517, - "hfopenllm_v2/MUSR": 0.3883, - "hfopenllm_v2/MMLU-PRO": 0.1126 - } - }, - { - "id": "kaist-ai/mistral-orpo-capybara-7k", - "name": "mistral-orpo-capybara-7k", - "developer": "kaist-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5367, - "hfopenllm_v2/BBH": 0.4489, - "hfopenllm_v2/MATH Level 5": 0.0393, - "hfopenllm_v2/GPQA": 0.2861, - "hfopenllm_v2/MUSR": 0.3964, - "hfopenllm_v2/MMLU-PRO": 0.2971 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/katanemo.json b/data/developers/katanemo.json deleted file mode 100644 index 3f211d7289ecaacb6ac08c98af815f255fb46d58..0000000000000000000000000000000000000000 --- a/data/developers/katanemo.json +++ /dev/null @@ -1,119 +0,0 @@ -{ - "developer": "katanemo", - "models": [ - { - "id": "katanemo/arch-agent-1-5b", - "name": "Arch-Agent-1.5B", - "developer": "katanemo", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 60.0, - "bfcl/bfcl.overall.overall_accuracy": 32.14, - "bfcl/bfcl.overall.total_cost_usd": 2.45, - "bfcl/bfcl.overall.latency_mean_s": 2.38, - "bfcl/bfcl.overall.latency_std_s": 4.01, - "bfcl/bfcl.overall.latency_p95_s": 5.3, - "bfcl/bfcl.non_live.ast_accuracy": 82.67, - "bfcl/bfcl.non_live.simple_ast_accuracy": 72.17, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 92.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 85.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 81.0, - "bfcl/bfcl.live.live_accuracy": 67.73, - "bfcl/bfcl.live.live_simple_ast_accuracy": 70.54, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 67.81, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 31.25, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 58.33, - "bfcl/bfcl.multi_turn.accuracy": 26.62, - "bfcl/bfcl.multi_turn.base_accuracy": 35.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 27.5, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 21.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 22.0, - "bfcl/bfcl.web_search.accuracy": 0.0, - "bfcl/bfcl.web_search.base_accuracy": 0.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 0.0, - "bfcl/bfcl.memory.accuracy": 8.17, - "bfcl/bfcl.memory.kv_accuracy": 5.81, - "bfcl/bfcl.memory.vector_accuracy": 5.81, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 12.9, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 75.0, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 74.83 - } - }, - { - "id": "katanemo/arch-agent-32b", - "name": "Arch-Agent-32B", - "developer": "katanemo", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 37.0, - "bfcl/bfcl.overall.overall_accuracy": 45.37, - "bfcl/bfcl.overall.total_cost_usd": 8.87, - "bfcl/bfcl.overall.latency_mean_s": 9.44, - "bfcl/bfcl.overall.latency_std_s": 21.44, - "bfcl/bfcl.overall.latency_p95_s": 24.87, - "bfcl/bfcl.non_live.ast_accuracy": 88.92, - "bfcl/bfcl.non_live.simple_ast_accuracy": 76.67, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 94.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 96.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 89.0, - "bfcl/bfcl.live.live_accuracy": 80.68, - "bfcl/bfcl.live.live_simple_ast_accuracy": 86.43, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 79.11, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 93.75, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 79.17, - "bfcl/bfcl.multi_turn.accuracy": 54.25, - "bfcl/bfcl.multi_turn.base_accuracy": 64.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 58.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 53.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 41.5, - "bfcl/bfcl.web_search.accuracy": 5.0, - "bfcl/bfcl.web_search.base_accuracy": 4.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 6.0, - "bfcl/bfcl.memory.accuracy": 14.62, - "bfcl/bfcl.memory.kv_accuracy": 5.81, - "bfcl/bfcl.memory.vector_accuracy": 9.03, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 29.03, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 81.25, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 82.15 - } - }, - { - "id": "katanemo/arch-agent-3b", - "name": "Arch-Agent-3B", - "developer": "katanemo", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 56.0, - "bfcl/bfcl.overall.overall_accuracy": 35.36, - "bfcl/bfcl.overall.total_cost_usd": 3.7, - "bfcl/bfcl.overall.latency_mean_s": 3.56, - "bfcl/bfcl.overall.latency_std_s": 6.65, - "bfcl/bfcl.overall.latency_p95_s": 8.19, - "bfcl/bfcl.non_live.ast_accuracy": 86.67, - "bfcl/bfcl.non_live.simple_ast_accuracy": 78.67, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 94.5, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 91.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 82.5, - "bfcl/bfcl.live.live_accuracy": 72.91, - "bfcl/bfcl.live.live_simple_ast_accuracy": 75.58, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 72.27, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 68.75, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 75.0, - "bfcl/bfcl.multi_turn.accuracy": 34.88, - "bfcl/bfcl.multi_turn.base_accuracy": 42.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 37.5, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 31.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 29.0, - "bfcl/bfcl.web_search.accuracy": 0.5, - "bfcl/bfcl.web_search.base_accuracy": 1.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 0.0, - "bfcl/bfcl.memory.accuracy": 6.88, - "bfcl/bfcl.memory.kv_accuracy": 5.16, - "bfcl/bfcl.memory.vector_accuracy": 5.81, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 9.68, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 68.75, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 74.67 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/kavonalds.json b/data/developers/kavonalds.json deleted file mode 100644 index c5a641658de284c1f8e0579f706e8c77674e2dfa..0000000000000000000000000000000000000000 --- a/data/developers/kavonalds.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "developer": "kavonalds", - "models": [ - { - "id": "kavonalds/BunderMaxx-0710", - "name": "BunderMaxx-0710", - "developer": "kavonalds", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2701, - "hfopenllm_v2/BBH": 0.5566, - "hfopenllm_v2/MATH Level 5": 0.068, - "hfopenllm_v2/GPQA": 0.2802, - "hfopenllm_v2/MUSR": 0.3682, - "hfopenllm_v2/MMLU-PRO": 0.1449 - } - }, - { - "id": "kavonalds/BunderMaxx-1010", - "name": "BunderMaxx-1010", - "developer": "kavonalds", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2981, - "hfopenllm_v2/BBH": 0.702, - "hfopenllm_v2/MATH Level 5": 0.105, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.3484, - "hfopenllm_v2/MMLU-PRO": 0.1224 - } - }, - { - "id": "kavonalds/Lancer-1-1b-Instruct", - "name": "Lancer-1-1b-Instruct", - "developer": "kavonalds", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5546, - "hfopenllm_v2/BBH": 0.3253, - "hfopenllm_v2/MATH Level 5": 0.0393, - "hfopenllm_v2/GPQA": 0.2617, - "hfopenllm_v2/MUSR": 0.3144, - "hfopenllm_v2/MMLU-PRO": 0.1568 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/kayfour.json b/data/developers/kayfour.json deleted file mode 100644 index 102ad7bc9a7c1e3e6a4966af227d9c5ec836ba95..0000000000000000000000000000000000000000 --- a/data/developers/kayfour.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "kayfour", - "models": [ - { - "id": "kayfour/T3Q-Qwen2.5-7B-it-KOR-Safe", - "name": "T3Q-Qwen2.5-7B-it-KOR-Safe", - "developer": "kayfour", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6081, - "hfopenllm_v2/BBH": 0.555, - "hfopenllm_v2/MATH Level 5": 0.3761, - "hfopenllm_v2/GPQA": 0.3213, - "hfopenllm_v2/MUSR": 0.4277, - "hfopenllm_v2/MMLU-PRO": 0.4464 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/keeeeenw.json b/data/developers/keeeeenw.json deleted file mode 100644 index c92573903f435f299efb6614a4dc6f1767e96059..0000000000000000000000000000000000000000 --- a/data/developers/keeeeenw.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "keeeeenw", - "models": [ - { - "id": "keeeeenw/MicroLlama", - "name": "MicroLlama", - "developer": "keeeeenw", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1985, - "hfopenllm_v2/BBH": 0.3007, - "hfopenllm_v2/MATH Level 5": 0.0113, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.3698, - "hfopenllm_v2/MMLU-PRO": 0.1138 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/kekmodel.json b/data/developers/kekmodel.json deleted file mode 100644 index 6b36fc62a538fa5126cc628205db84e8a3eccaff..0000000000000000000000000000000000000000 --- a/data/developers/kekmodel.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "kekmodel", - "models": [ - { - "id": "kekmodel/StopCarbon-10.7B-v5", - "name": "StopCarbon-10.7B-v5", - "developer": "kekmodel", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4728, - "hfopenllm_v2/BBH": 0.5178, - "hfopenllm_v2/MATH Level 5": 0.0559, - "hfopenllm_v2/GPQA": 0.3062, - "hfopenllm_v2/MUSR": 0.4019, - "hfopenllm_v2/MMLU-PRO": 0.3157 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/kevin009.json b/data/developers/kevin009.json deleted file mode 100644 index ff82ca19e303fe735ca9364c6b5a69e9d239a4df..0000000000000000000000000000000000000000 --- a/data/developers/kevin009.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "kevin009", - "models": [ - { - "id": "kevin009/llamaRAGdrama", - "name": "llamaRAGdrama", - "developer": "kevin009", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2598, - "hfopenllm_v2/BBH": 0.4007, - "hfopenllm_v2/MATH Level 5": 0.0431, - "hfopenllm_v2/GPQA": 0.2643, - "hfopenllm_v2/MUSR": 0.4316, - "hfopenllm_v2/MMLU-PRO": 0.2724 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/khetterman.json b/data/developers/khetterman.json deleted file mode 100644 index b0a72c659418ec6373be438380414d0ba3eea3ce..0000000000000000000000000000000000000000 --- a/data/developers/khetterman.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "Khetterman", - "models": [ - { - "id": "Khetterman/DarkAtom-12B-v3", - "name": "DarkAtom-12B-v3", - "developer": "Khetterman", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6173, - "hfopenllm_v2/BBH": 0.5154, - "hfopenllm_v2/MATH Level 5": 0.111, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.4468, - "hfopenllm_v2/MMLU-PRO": 0.3546 - } - }, - { - "id": "Khetterman/Kosmos-8B-v1", - "name": "Kosmos-8B-v1", - "developer": "Khetterman", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4129, - "hfopenllm_v2/BBH": 0.5234, - "hfopenllm_v2/MATH Level 5": 0.0989, - "hfopenllm_v2/GPQA": 0.2987, - "hfopenllm_v2/MUSR": 0.3919, - "hfopenllm_v2/MMLU-PRO": 0.3669 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/khoantap.json b/data/developers/khoantap.json deleted file mode 100644 index 39542944051aedda7a7c34cbbd69b94b255956a2..0000000000000000000000000000000000000000 --- a/data/developers/khoantap.json +++ /dev/null @@ -1,131 +0,0 @@ -{ - "developer": "khoantap", - "models": [ - { - "id": "khoantap/cheap-moe-merge", - "name": "cheap-moe-merge", - "developer": "khoantap", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4557, - "hfopenllm_v2/BBH": 0.5131, - "hfopenllm_v2/MATH Level 5": 0.0921, - "hfopenllm_v2/GPQA": 0.2953, - "hfopenllm_v2/MUSR": 0.4103, - "hfopenllm_v2/MMLU-PRO": 0.3339 - } - }, - { - "id": "khoantap/llama-3-8b-stock-merge", - "name": "llama-3-8b-stock-merge", - "developer": "khoantap", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4812, - "hfopenllm_v2/BBH": 0.5162, - "hfopenllm_v2/MATH Level 5": 0.1616, - "hfopenllm_v2/GPQA": 0.318, - "hfopenllm_v2/MUSR": 0.3946, - "hfopenllm_v2/MMLU-PRO": 0.38 - } - }, - { - "id": "khoantap/llama-breadcrumbs-ties-merge", - "name": "llama-breadcrumbs-ties-merge", - "developer": "khoantap", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2205, - "hfopenllm_v2/BBH": 0.5416, - "hfopenllm_v2/MATH Level 5": 0.1125, - "hfopenllm_v2/GPQA": 0.2659, - "hfopenllm_v2/MUSR": 0.4434, - "hfopenllm_v2/MMLU-PRO": 0.3172 - } - }, - { - "id": "khoantap/llama-evolve-ties-best-merge", - "name": "llama-evolve-ties-best-merge", - "developer": "khoantap", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6744, - "hfopenllm_v2/BBH": 0.5414, - "hfopenllm_v2/MATH Level 5": 0.1563, - "hfopenllm_v2/GPQA": 0.3171, - "hfopenllm_v2/MUSR": 0.3946, - "hfopenllm_v2/MMLU-PRO": 0.386 - } - }, - { - "id": "khoantap/llama-linear-0.5-0.5-1-merge", - "name": "llama-linear-0.5-0.5-1-merge", - "developer": "khoantap", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4812, - "hfopenllm_v2/BBH": 0.5643, - "hfopenllm_v2/MATH Level 5": 0.2054, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.4143, - "hfopenllm_v2/MMLU-PRO": 0.3833 - } - }, - { - "id": "khoantap/llama-linear-0.5-1-0.5-merge", - "name": "llama-linear-0.5-1-0.5-merge", - "developer": "khoantap", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5032, - "hfopenllm_v2/BBH": 0.5951, - "hfopenllm_v2/MATH Level 5": 0.148, - "hfopenllm_v2/GPQA": 0.2936, - "hfopenllm_v2/MUSR": 0.4172, - "hfopenllm_v2/MMLU-PRO": 0.369 - } - }, - { - "id": "khoantap/llama-linear-1-0.5-0.5-merge", - "name": "llama-linear-1-0.5-0.5-merge", - "developer": "khoantap", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4515, - "hfopenllm_v2/BBH": 0.5526, - "hfopenllm_v2/MATH Level 5": 0.2477, - "hfopenllm_v2/GPQA": 0.2928, - "hfopenllm_v2/MUSR": 0.4118, - "hfopenllm_v2/MMLU-PRO": 0.3635 - } - }, - { - "id": "khoantap/llama-slerp-merge", - "name": "llama-slerp-merge", - "developer": "khoantap", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.498, - "hfopenllm_v2/BBH": 0.5783, - "hfopenllm_v2/MATH Level 5": 0.0831, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.4053, - "hfopenllm_v2/MMLU-PRO": 0.3678 - } - }, - { - "id": "khoantap/moe-out-merge", - "name": "moe-out-merge", - "developer": "khoantap", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4505, - "hfopenllm_v2/BBH": 0.5151, - "hfopenllm_v2/MATH Level 5": 0.0929, - "hfopenllm_v2/GPQA": 0.2886, - "hfopenllm_v2/MUSR": 0.4063, - "hfopenllm_v2/MMLU-PRO": 0.3348 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/khulaifi95.json b/data/developers/khulaifi95.json deleted file mode 100644 index 61225eeefe15af2d1ba99c65423ebd1c5482c99b..0000000000000000000000000000000000000000 --- a/data/developers/khulaifi95.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "khulaifi95", - "models": [ - { - "id": "khulaifi95/Llama-3.1-8B-Reason-Blend-888k", - "name": "Llama-3.1-8B-Reason-Blend-888k", - "developer": "khulaifi95", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5832, - "hfopenllm_v2/BBH": 0.479, - "hfopenllm_v2/MATH Level 5": 0.1156, - "hfopenllm_v2/GPQA": 0.2794, - "hfopenllm_v2/MUSR": 0.3379, - "hfopenllm_v2/MMLU-PRO": 0.31 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/kimargin.json b/data/developers/kimargin.json deleted file mode 100644 index f2d25e0bd9e97f1a9b943f28a485582a78d710a4..0000000000000000000000000000000000000000 --- a/data/developers/kimargin.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "Kimargin", - "models": [ - { - "id": "Kimargin/GPT-NEO-1.3B-wiki", - "name": "GPT-NEO-1.3B-wiki", - "developer": "Kimargin", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1921, - "hfopenllm_v2/BBH": 0.3026, - "hfopenllm_v2/MATH Level 5": 0.0144, - "hfopenllm_v2/GPQA": 0.245, - "hfopenllm_v2/MUSR": 0.3883, - "hfopenllm_v2/MMLU-PRO": 0.1099 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/kimi.json b/data/developers/kimi.json deleted file mode 100644 index 1a69217fe4b32f418bb2d3f94f33e9c6502c5cfe..0000000000000000000000000000000000000000 --- a/data/developers/kimi.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "developer": "Kimi", - "models": [ - { - "id": "moonshot-ai/kimi-k2.5", - "name": "Kimi K2.5", - "developer": "Kimi", - "evaluator_relationship": null, - "benchmark_scores": { - "terminal-bench-2.0/terminal-bench-2.0": 43.2 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/kingnish.json b/data/developers/kingnish.json deleted file mode 100644 index c85edc134fbdbf494491a81a1f12f2b10c94b4c2..0000000000000000000000000000000000000000 --- a/data/developers/kingnish.json +++ /dev/null @@ -1,103 +0,0 @@ -{ - "developer": "KingNish", - "models": [ - { - "id": "KingNish/qwen-1b-continued", - "name": "qwen-1b-continued", - "developer": "KingNish", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1255, - "hfopenllm_v2/BBH": 0.2991, - "hfopenllm_v2/MATH Level 5": 0.0091, - "hfopenllm_v2/GPQA": 0.2676, - "hfopenllm_v2/MUSR": 0.3859, - "hfopenllm_v2/MMLU-PRO": 0.1261 - } - }, - { - "id": "KingNish/qwen-1b-continued-v2", - "name": "qwen-1b-continued-v2", - "developer": "KingNish", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1579, - "hfopenllm_v2/BBH": 0.3119, - "hfopenllm_v2/MATH Level 5": 0.0106, - "hfopenllm_v2/GPQA": 0.25, - "hfopenllm_v2/MUSR": 0.3393, - "hfopenllm_v2/MMLU-PRO": 0.1193 - } - }, - { - "id": "KingNish/qwen-1b-continued-v2.1", - "name": "qwen-1b-continued-v2.1", - "developer": "KingNish", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1127, - "hfopenllm_v2/BBH": 0.3042, - "hfopenllm_v2/MATH Level 5": 0.0091, - "hfopenllm_v2/GPQA": 0.2676, - "hfopenllm_v2/MUSR": 0.4154, - "hfopenllm_v2/MMLU-PRO": 0.1278 - } - }, - { - "id": "KingNish/qwen-1b-continued-v2.2", - "name": "qwen-1b-continued-v2.2", - "developer": "KingNish", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1413, - "hfopenllm_v2/BBH": 0.3059, - "hfopenllm_v2/MATH Level 5": 0.0151, - "hfopenllm_v2/GPQA": 0.2567, - "hfopenllm_v2/MUSR": 0.3513, - "hfopenllm_v2/MMLU-PRO": 0.1262 - } - }, - { - "id": "KingNish/Qwen2.5-0.5b-Test-ft", - "name": "Qwen2.5-0.5b-Test-ft", - "developer": "KingNish", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2671, - "hfopenllm_v2/BBH": 0.3232, - "hfopenllm_v2/MATH Level 5": 0.0355, - "hfopenllm_v2/GPQA": 0.2634, - "hfopenllm_v2/MUSR": 0.3421, - "hfopenllm_v2/MMLU-PRO": 0.1689 - } - }, - { - "id": "KingNish/Reasoning-0.5b", - "name": "Reasoning-0.5b", - "developer": "KingNish", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2174, - "hfopenllm_v2/BBH": 0.3354, - "hfopenllm_v2/MATH Level 5": 0.0219, - "hfopenllm_v2/GPQA": 0.2676, - "hfopenllm_v2/MUSR": 0.3513, - "hfopenllm_v2/MMLU-PRO": 0.1641 - } - }, - { - "id": "KingNish/Reasoning-Llama-3b-v0.1", - "name": "Reasoning-Llama-3b-v0.1", - "developer": "KingNish", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6225, - "hfopenllm_v2/BBH": 0.4343, - "hfopenllm_v2/MATH Level 5": 0.1299, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3168, - "hfopenllm_v2/MMLU-PRO": 0.3029 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/kms7530.json b/data/developers/kms7530.json deleted file mode 100644 index 3fbbe5339d95906fa350578a1f7278d91d76e541..0000000000000000000000000000000000000000 --- a/data/developers/kms7530.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "developer": "kms7530", - "models": [ - { - "id": "kms7530/chemeng_llama-3-8b-Instruct-bnb-4bit_24_1_100_1", - "name": "chemeng_llama-3-8b-Instruct-bnb-4bit_24_1_100_1", - "developer": "kms7530", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5455, - "hfopenllm_v2/BBH": 0.4289, - "hfopenllm_v2/MATH Level 5": 0.0619, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.3821, - "hfopenllm_v2/MMLU-PRO": 0.2798 - } - }, - { - "id": "kms7530/chemeng_phi-3-mini-4k-instruct-bnb-4bit_16_4_100_1_nonmath", - "name": "chemeng_phi-3-mini-4k-instruct-bnb-4bit_16_4_100_1_nonmath", - "developer": "kms7530", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4863, - "hfopenllm_v2/BBH": 0.4987, - "hfopenllm_v2/MATH Level 5": 0.108, - "hfopenllm_v2/GPQA": 0.3104, - "hfopenllm_v2/MUSR": 0.3983, - "hfopenllm_v2/MMLU-PRO": 0.3481 - } - }, - { - "id": "kms7530/chemeng_qwen-math-7b_24_1_100_1", - "name": "chemeng_qwen-math-7b_24_1_100_1", - "developer": "kms7530", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2111, - "hfopenllm_v2/BBH": 0.3578, - "hfopenllm_v2/MATH Level 5": 0.2243, - "hfopenllm_v2/GPQA": 0.2441, - "hfopenllm_v2/MUSR": 0.3687, - "hfopenllm_v2/MMLU-PRO": 0.2158 - } - }, - { - "id": "kms7530/chemeng_qwen-math-7b_24_1_100_1_nonmath", - "name": "chemeng_qwen-math-7b_24_1_100_1_nonmath", - "developer": "kms7530", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2584, - "hfopenllm_v2/BBH": 0.3893, - "hfopenllm_v2/MATH Level 5": 0.3097, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.4087, - "hfopenllm_v2/MMLU-PRO": 0.2452 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/kno10.json b/data/developers/kno10.json deleted file mode 100644 index 018c11f687911e11ce50496c2e0c018416cf3db1..0000000000000000000000000000000000000000 --- a/data/developers/kno10.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "kno10", - "models": [ - { - "id": "kno10/ende-chat-0.0.5", - "name": "ende-chat-0.0.5", - "developer": "kno10", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3404, - "hfopenllm_v2/BBH": 0.3604, - "hfopenllm_v2/MATH Level 5": 0.0204, - "hfopenllm_v2/GPQA": 0.2651, - "hfopenllm_v2/MUSR": 0.3938, - "hfopenllm_v2/MMLU-PRO": 0.179 - } - }, - { - "id": "kno10/ende-chat-0.0.7", - "name": "ende-chat-0.0.7", - "developer": "kno10", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4401, - "hfopenllm_v2/BBH": 0.3792, - "hfopenllm_v2/MATH Level 5": 0.0174, - "hfopenllm_v2/GPQA": 0.281, - "hfopenllm_v2/MUSR": 0.3861, - "hfopenllm_v2/MMLU-PRO": 0.1966 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/kquant03.json b/data/developers/kquant03.json deleted file mode 100644 index 580eddb7701d9eab6d1c5bcff0df7d5b7e676429..0000000000000000000000000000000000000000 --- a/data/developers/kquant03.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "Kquant03", - "models": [ - { - "id": "Kquant03/CognitiveFusion2-4x7B-BF16", - "name": "CognitiveFusion2-4x7B-BF16", - "developer": "Kquant03", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3567, - "hfopenllm_v2/BBH": 0.4108, - "hfopenllm_v2/MATH Level 5": 0.0574, - "hfopenllm_v2/GPQA": 0.2861, - "hfopenllm_v2/MUSR": 0.4146, - "hfopenllm_v2/MMLU-PRO": 0.2793 - } - }, - { - "id": "Kquant03/L3-Pneuma-8B", - "name": "L3-Pneuma-8B", - "developer": "Kquant03", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2374, - "hfopenllm_v2/BBH": 0.4955, - "hfopenllm_v2/MATH Level 5": 0.0506, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.4172, - "hfopenllm_v2/MMLU-PRO": 0.3184 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/krystalan.json b/data/developers/krystalan.json deleted file mode 100644 index 9353bf118687519f6b00451224b1ff4eba08729e..0000000000000000000000000000000000000000 --- a/data/developers/krystalan.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "Krystalan", - "models": [ - { - "id": "Krystalan/DRT-o1-14B", - "name": "DRT-o1-14B", - "developer": "Krystalan", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4068, - "hfopenllm_v2/BBH": 0.6379, - "hfopenllm_v2/MATH Level 5": 0.4826, - "hfopenllm_v2/GPQA": 0.3523, - "hfopenllm_v2/MUSR": 0.4795, - "hfopenllm_v2/MMLU-PRO": 0.5179 - } - }, - { - "id": "Krystalan/DRT-o1-7B", - "name": "DRT-o1-7B", - "developer": "Krystalan", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3928, - "hfopenllm_v2/BBH": 0.5468, - "hfopenllm_v2/MATH Level 5": 0.4479, - "hfopenllm_v2/GPQA": 0.3213, - "hfopenllm_v2/MUSR": 0.5087, - "hfopenllm_v2/MMLU-PRO": 0.4151 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/ksu-hw-sec.json b/data/developers/ksu-hw-sec.json deleted file mode 100644 index 7df9b6de5475b7da7c530ee8d3cdfc4b8b99d1f6..0000000000000000000000000000000000000000 --- a/data/developers/ksu-hw-sec.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "developer": "KSU-HW-SEC", - "models": [ - { - "id": "KSU-HW-SEC/Llama3-70b-SVA-FT-1415", - "name": "Llama3-70b-SVA-FT-1415", - "developer": "KSU-HW-SEC", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.618, - "hfopenllm_v2/BBH": 0.665, - "hfopenllm_v2/MATH Level 5": 0.2198, - "hfopenllm_v2/GPQA": 0.375, - "hfopenllm_v2/MUSR": 0.4565, - "hfopenllm_v2/MMLU-PRO": 0.5243 - } - }, - { - "id": "KSU-HW-SEC/Llama3-70b-SVA-FT-500", - "name": "Llama3-70b-SVA-FT-500", - "developer": "KSU-HW-SEC", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6105, - "hfopenllm_v2/BBH": 0.6692, - "hfopenllm_v2/MATH Level 5": 0.2137, - "hfopenllm_v2/GPQA": 0.3809, - "hfopenllm_v2/MUSR": 0.4511, - "hfopenllm_v2/MMLU-PRO": 0.5227 - } - }, - { - "id": "KSU-HW-SEC/Llama3-70b-SVA-FT-final", - "name": "Llama3-70b-SVA-FT-final", - "developer": "KSU-HW-SEC", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6165, - "hfopenllm_v2/BBH": 0.665, - "hfopenllm_v2/MATH Level 5": 0.2198, - "hfopenllm_v2/GPQA": 0.375, - "hfopenllm_v2/MUSR": 0.4565, - "hfopenllm_v2/MMLU-PRO": 0.5243 - } - }, - { - "id": "KSU-HW-SEC/Llama3.1-70b-SVA-FT-1000step", - "name": "Llama3.1-70b-SVA-FT-1000step", - "developer": "KSU-HW-SEC", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7238, - "hfopenllm_v2/BBH": 0.6903, - "hfopenllm_v2/MATH Level 5": 0.321, - "hfopenllm_v2/GPQA": 0.396, - "hfopenllm_v2/MUSR": 0.4592, - "hfopenllm_v2/MMLU-PRO": 0.5252 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/kuaishou.json b/data/developers/kuaishou.json deleted file mode 100644 index 511550dd6f527bdec6ac47d201b5a3d53f3a57a9..0000000000000000000000000000000000000000 --- a/data/developers/kuaishou.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "developer": "Kuaishou", - "models": [ - { - "id": "kuaishou/kwaipilot-40b-0604", - "name": "kwaipilot-40b-0604", - "developer": "Kuaishou", - "evaluator_relationship": null, - "benchmark_scores": { - "livecodebenchpro/Hard Problems": 0.0, - "livecodebenchpro/Medium Problems": 0.07042253521126761, - "livecodebenchpro/Easy Problems": 0.056338028169014086 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/kukedlc.json b/data/developers/kukedlc.json deleted file mode 100644 index 9c255dac8c9fb300c36fff0fb75c54544e7edc7c..0000000000000000000000000000000000000000 --- a/data/developers/kukedlc.json +++ /dev/null @@ -1,103 +0,0 @@ -{ - "developer": "Kukedlc", - "models": [ - { - "id": "Kukedlc/NeuralExperiment-7b-MagicCoder-v7.5", - "name": "NeuralExperiment-7b-MagicCoder-v7.5", - "developer": "Kukedlc", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4553, - "hfopenllm_v2/BBH": 0.3988, - "hfopenllm_v2/MATH Level 5": 0.0665, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.4282, - "hfopenllm_v2/MMLU-PRO": 0.2824 - } - }, - { - "id": "Kukedlc/NeuralLLaMa-3-8b-DT-v0.1", - "name": "NeuralLLaMa-3-8b-DT-v0.1", - "developer": "Kukedlc", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4371, - "hfopenllm_v2/BBH": 0.4987, - "hfopenllm_v2/MATH Level 5": 0.0808, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.4071, - "hfopenllm_v2/MMLU-PRO": 0.3792 - } - }, - { - "id": "Kukedlc/NeuralLLaMa-3-8b-ORPO-v0.3", - "name": "NeuralLLaMa-3-8b-ORPO-v0.3", - "developer": "Kukedlc", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5276, - "hfopenllm_v2/BBH": 0.4557, - "hfopenllm_v2/MATH Level 5": 0.0483, - "hfopenllm_v2/GPQA": 0.2391, - "hfopenllm_v2/MUSR": 0.37, - "hfopenllm_v2/MMLU-PRO": 0.3057 - } - }, - { - "id": "Kukedlc/NeuralSynthesis-7B-v0.1", - "name": "NeuralSynthesis-7B-v0.1", - "developer": "Kukedlc", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4185, - "hfopenllm_v2/BBH": 0.5145, - "hfopenllm_v2/MATH Level 5": 0.0634, - "hfopenllm_v2/GPQA": 0.281, - "hfopenllm_v2/MUSR": 0.4333, - "hfopenllm_v2/MMLU-PRO": 0.3049 - } - }, - { - "id": "Kukedlc/NeuralSynthesis-7B-v0.3", - "name": "NeuralSynthesis-7B-v0.3", - "developer": "Kukedlc", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4078, - "hfopenllm_v2/BBH": 0.5138, - "hfopenllm_v2/MATH Level 5": 0.0778, - "hfopenllm_v2/GPQA": 0.2802, - "hfopenllm_v2/MUSR": 0.4346, - "hfopenllm_v2/MMLU-PRO": 0.305 - } - }, - { - "id": "Kukedlc/NeuralSynthesis-7b-v0.4-slerp", - "name": "NeuralSynthesis-7b-v0.4-slerp", - "developer": "Kukedlc", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3947, - "hfopenllm_v2/BBH": 0.5143, - "hfopenllm_v2/MATH Level 5": 0.0627, - "hfopenllm_v2/GPQA": 0.2777, - "hfopenllm_v2/MUSR": 0.4332, - "hfopenllm_v2/MMLU-PRO": 0.3043 - } - }, - { - "id": "Kukedlc/Qwen-2.5-7b-Spanish-o1-CoT", - "name": "Qwen-2.5-7b-Spanish-o1-CoT", - "developer": "Kukedlc", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.421, - "hfopenllm_v2/BBH": 0.5602, - "hfopenllm_v2/MATH Level 5": 0.2727, - "hfopenllm_v2/GPQA": 0.3205, - "hfopenllm_v2/MUSR": 0.4777, - "hfopenllm_v2/MMLU-PRO": 0.4363 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/kumar955.json b/data/developers/kumar955.json deleted file mode 100644 index d7d571dddaed7e8c2351e382dd4b2e6d4c6dac3b..0000000000000000000000000000000000000000 --- a/data/developers/kumar955.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "Kumar955", - "models": [ - { - "id": "Kumar955/Hemanth-llm", - "name": "Hemanth-llm", - "developer": "Kumar955", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5045, - "hfopenllm_v2/BBH": 0.5225, - "hfopenllm_v2/MATH Level 5": 0.0702, - "hfopenllm_v2/GPQA": 0.2827, - "hfopenllm_v2/MUSR": 0.4486, - "hfopenllm_v2/MMLU-PRO": 0.3113 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/kyutai.json b/data/developers/kyutai.json deleted file mode 100644 index 7422ab7fa4a4ab4478ee22b9d7cd718cd595f093..0000000000000000000000000000000000000000 --- a/data/developers/kyutai.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "kyutai", - "models": [ - { - "id": "kyutai/helium-1-preview-2b", - "name": "helium-1-preview-2b", - "developer": "kyutai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2614, - "hfopenllm_v2/BBH": 0.3638, - "hfopenllm_v2/MATH Level 5": 0.0136, - "hfopenllm_v2/GPQA": 0.2785, - "hfopenllm_v2/MUSR": 0.355, - "hfopenllm_v2/MMLU-PRO": 0.1873 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/kz919.json b/data/developers/kz919.json deleted file mode 100644 index f07a8d4ed120988a54dd758837acf314e8b90f22..0000000000000000000000000000000000000000 --- a/data/developers/kz919.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "kz919", - "models": [ - { - "id": "kz919/QwQ-0.5B-Distilled-SFT", - "name": "QwQ-0.5B-Distilled-SFT", - "developer": "kz919", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3077, - "hfopenllm_v2/BBH": 0.3256, - "hfopenllm_v2/MATH Level 5": 0.074, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.3409, - "hfopenllm_v2/MMLU-PRO": 0.1587 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/l-rage.json b/data/developers/l-rage.json deleted file mode 100644 index 783915f8ad1bd3861f6eccd4ac963b0f1d17fab5..0000000000000000000000000000000000000000 --- a/data/developers/l-rage.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "L-RAGE", - "models": [ - { - "id": "L-RAGE/3_PRYMMAL-ECE-7B-SLERP-V1", - "name": "3_PRYMMAL-ECE-7B-SLERP-V1", - "developer": "L-RAGE", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2742, - "hfopenllm_v2/BBH": 0.4228, - "hfopenllm_v2/MATH Level 5": 0.108, - "hfopenllm_v2/GPQA": 0.2819, - "hfopenllm_v2/MUSR": 0.3841, - "hfopenllm_v2/MMLU-PRO": 0.2925 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/ladydaina.json b/data/developers/ladydaina.json deleted file mode 100644 index ab258388baf01a4d69d83b70adc3bc612368f60b..0000000000000000000000000000000000000000 --- a/data/developers/ladydaina.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "ladydaina", - "models": [ - { - "id": "ladydaina/ECE-FDF", - "name": "ECE-FDF", - "developer": "ladydaina", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3728, - "hfopenllm_v2/BBH": 0.515, - "hfopenllm_v2/MATH Level 5": 0.0816, - "hfopenllm_v2/GPQA": 0.2827, - "hfopenllm_v2/MUSR": 0.4504, - "hfopenllm_v2/MMLU-PRO": 0.3007 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/laislemke.json b/data/developers/laislemke.json deleted file mode 100644 index 4805ab06830ed4c83bd1b79bef90811b1eb79627..0000000000000000000000000000000000000000 --- a/data/developers/laislemke.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "laislemke", - "models": [ - { - "id": "laislemke/LLaMA-2-vicuna-7b-slerp", - "name": "LLaMA-2-vicuna-7b-slerp", - "developer": "laislemke", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2932, - "hfopenllm_v2/BBH": 0.2986, - "hfopenllm_v2/MATH Level 5": 0.0113, - "hfopenllm_v2/GPQA": 0.2735, - "hfopenllm_v2/MUSR": 0.3833, - "hfopenllm_v2/MMLU-PRO": 0.1342 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/lalainy.json b/data/developers/lalainy.json deleted file mode 100644 index 476a1593f27daec458637c1cc3ea80e442afd5f9..0000000000000000000000000000000000000000 --- a/data/developers/lalainy.json +++ /dev/null @@ -1,103 +0,0 @@ -{ - "developer": "lalainy", - "models": [ - { - "id": "lalainy/ECE-PRYMMAL-0.5B-FT-V5-MUSR", - "name": "ECE-PRYMMAL-0.5B-FT-V5-MUSR", - "developer": "lalainy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2138, - "hfopenllm_v2/BBH": 0.3269, - "hfopenllm_v2/MATH Level 5": 0.0453, - "hfopenllm_v2/GPQA": 0.2743, - "hfopenllm_v2/MUSR": 0.3262, - "hfopenllm_v2/MMLU-PRO": 0.1533 - } - }, - { - "id": "lalainy/ECE-PRYMMAL-0.5B-SLERP-V4", - "name": "ECE-PRYMMAL-0.5B-SLERP-V4", - "developer": "lalainy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1564, - "hfopenllm_v2/BBH": 0.2894, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2626, - "hfopenllm_v2/MUSR": 0.3789, - "hfopenllm_v2/MMLU-PRO": 0.1169 - } - }, - { - "id": "lalainy/ECE-PRYMMAL-YL-0.5B-SLERP-BIS-V1", - "name": "ECE-PRYMMAL-YL-0.5B-SLERP-BIS-V1", - "developer": "lalainy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1437, - "hfopenllm_v2/BBH": 0.3032, - "hfopenllm_v2/MATH Level 5": 0.0008, - "hfopenllm_v2/GPQA": 0.2349, - "hfopenllm_v2/MUSR": 0.3646, - "hfopenllm_v2/MMLU-PRO": 0.1121 - } - }, - { - "id": "lalainy/ECE-PRYMMAL-YL-1B-SLERP-V3", - "name": "ECE-PRYMMAL-YL-1B-SLERP-V3", - "developer": "lalainy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.325, - "hfopenllm_v2/BBH": 0.4225, - "hfopenllm_v2/MATH Level 5": 0.0974, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.4213, - "hfopenllm_v2/MMLU-PRO": 0.2931 - } - }, - { - "id": "lalainy/ECE-PRYMMAL-YL-1B-SLERP-V4", - "name": "ECE-PRYMMAL-YL-1B-SLERP-V4", - "developer": "lalainy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3324, - "hfopenllm_v2/BBH": 0.4171, - "hfopenllm_v2/MATH Level 5": 0.1005, - "hfopenllm_v2/GPQA": 0.2861, - "hfopenllm_v2/MUSR": 0.4306, - "hfopenllm_v2/MMLU-PRO": 0.2893 - } - }, - { - "id": "lalainy/ECE-PRYMMAL-YL-6B-SLERP-V1", - "name": "ECE-PRYMMAL-YL-6B-SLERP-V1", - "developer": "lalainy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3264, - "hfopenllm_v2/BBH": 0.4629, - "hfopenllm_v2/MATH Level 5": 0.1269, - "hfopenllm_v2/GPQA": 0.2886, - "hfopenllm_v2/MUSR": 0.4864, - "hfopenllm_v2/MMLU-PRO": 0.3214 - } - }, - { - "id": "lalainy/ECE-PRYMMAL-YL-6B-SLERP-V2", - "name": "ECE-PRYMMAL-YL-6B-SLERP-V2", - "developer": "lalainy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3249, - "hfopenllm_v2/BBH": 0.4629, - "hfopenllm_v2/MATH Level 5": 0.1269, - "hfopenllm_v2/GPQA": 0.2886, - "hfopenllm_v2/MUSR": 0.4864, - "hfopenllm_v2/MMLU-PRO": 0.3214 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/lambent.json b/data/developers/lambent.json deleted file mode 100644 index 01ac84d8a6579b3f404615e46c99ca996887260b..0000000000000000000000000000000000000000 --- a/data/developers/lambent.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "Lambent", - "models": [ - { - "id": "Lambent/qwen2.5-reinstruct-alternate-lumen-14B", - "name": "qwen2.5-reinstruct-alternate-lumen-14B", - "developer": "Lambent", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4794, - "hfopenllm_v2/BBH": 0.6459, - "hfopenllm_v2/MATH Level 5": 0.4622, - "hfopenllm_v2/GPQA": 0.3767, - "hfopenllm_v2/MUSR": 0.477, - "hfopenllm_v2/MMLU-PRO": 0.5388 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/langboat.json b/data/developers/langboat.json deleted file mode 100644 index 929f700b20c79cc3b08c22fb8dd90fc5d30fc929..0000000000000000000000000000000000000000 --- a/data/developers/langboat.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "Langboat", - "models": [ - { - "id": "Langboat/Mengzi3-8B-Chat", - "name": "Mengzi3-8B-Chat", - "developer": "Langboat", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.514, - "hfopenllm_v2/BBH": 0.4684, - "hfopenllm_v2/MATH Level 5": 0.0906, - "hfopenllm_v2/GPQA": 0.2743, - "hfopenllm_v2/MUSR": 0.4078, - "hfopenllm_v2/MMLU-PRO": 0.3142 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/langgptai.json b/data/developers/langgptai.json deleted file mode 100644 index 3f8c0e341080bbea317c93d76a30adf3cea0fbbf..0000000000000000000000000000000000000000 --- a/data/developers/langgptai.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "langgptai", - "models": [ - { - "id": "langgptai/Qwen-las-v0.1", - "name": "Qwen-las-v0.1", - "developer": "langgptai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3301, - "hfopenllm_v2/BBH": 0.3893, - "hfopenllm_v2/MATH Level 5": 0.037, - "hfopenllm_v2/GPQA": 0.2466, - "hfopenllm_v2/MUSR": 0.3701, - "hfopenllm_v2/MMLU-PRO": 0.2325 - } - }, - { - "id": "langgptai/qwen1.5-7b-chat-sa-v0.1", - "name": "qwen1.5-7b-chat-sa-v0.1", - "developer": "langgptai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4268, - "hfopenllm_v2/BBH": 0.4325, - "hfopenllm_v2/MATH Level 5": 0.0302, - "hfopenllm_v2/GPQA": 0.3121, - "hfopenllm_v2/MUSR": 0.3551, - "hfopenllm_v2/MMLU-PRO": 0.2993 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/lars1234.json b/data/developers/lars1234.json deleted file mode 100644 index e1d6249b8cfa9325134174278aafeb88d68057c8..0000000000000000000000000000000000000000 --- a/data/developers/lars1234.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "lars1234", - "models": [ - { - "id": "lars1234/Mistral-Small-24B-Instruct-2501-writer", - "name": "Mistral-Small-24B-Instruct-2501-writer", - "developer": "lars1234", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6565, - "hfopenllm_v2/BBH": 0.6733, - "hfopenllm_v2/MATH Level 5": 0.3557, - "hfopenllm_v2/GPQA": 0.3893, - "hfopenllm_v2/MUSR": 0.4645, - "hfopenllm_v2/MMLU-PRO": 0.5448 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/lawnakk.json b/data/developers/lawnakk.json deleted file mode 100644 index 846e685409048252cceb41219ec4c1e0b6c2fe7a..0000000000000000000000000000000000000000 --- a/data/developers/lawnakk.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "developer": "Lawnakk", - "models": [ - { - "id": "Lawnakk/BBA100", - "name": "BBA100", - "developer": "Lawnakk", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2076, - "hfopenllm_v2/BBH": 0.2826, - "hfopenllm_v2/MATH Level 5": 0.0098, - "hfopenllm_v2/GPQA": 0.2441, - "hfopenllm_v2/MUSR": 0.402, - "hfopenllm_v2/MMLU-PRO": 0.1122 - } - }, - { - "id": "Lawnakk/BBALAW1", - "name": "BBALAW1", - "developer": "Lawnakk", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1905, - "hfopenllm_v2/BBH": 0.2872, - "hfopenllm_v2/MATH Level 5": 0.0098, - "hfopenllm_v2/GPQA": 0.2433, - "hfopenllm_v2/MUSR": 0.4153, - "hfopenllm_v2/MMLU-PRO": 0.1121 - } - }, - { - "id": "Lawnakk/BBALAW1.0", - "name": "BBALAW1.0", - "developer": "Lawnakk", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1351, - "hfopenllm_v2/BBH": 0.2828, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2559, - "hfopenllm_v2/MUSR": 0.3526, - "hfopenllm_v2/MMLU-PRO": 0.1128 - } - }, - { - "id": "Lawnakk/BBALAW1.2", - "name": "BBALAW1.2", - "developer": "Lawnakk", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1354, - "hfopenllm_v2/BBH": 0.2811, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2643, - "hfopenllm_v2/MUSR": 0.3579, - "hfopenllm_v2/MMLU-PRO": 0.1105 - } - }, - { - "id": "Lawnakk/BBALAW1.3", - "name": "BBALAW1.3", - "developer": "Lawnakk", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1354, - "hfopenllm_v2/BBH": 0.2827, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.3619, - "hfopenllm_v2/MMLU-PRO": 0.1094 - } - }, - { - "id": "Lawnakk/BBALAW1.6", - "name": "BBALAW1.6", - "developer": "Lawnakk", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5245, - "hfopenllm_v2/BBH": 0.5554, - "hfopenllm_v2/MATH Level 5": 0.3603, - "hfopenllm_v2/GPQA": 0.3238, - "hfopenllm_v2/MUSR": 0.4368, - "hfopenllm_v2/MMLU-PRO": 0.4507 - } - }, - { - "id": "Lawnakk/BBALAW1.61", - "name": "BBALAW1.61", - "developer": "Lawnakk", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5771, - "hfopenllm_v2/BBH": 0.5549, - "hfopenllm_v2/MATH Level 5": 0.3663, - "hfopenllm_v2/GPQA": 0.3171, - "hfopenllm_v2/MUSR": 0.4355, - "hfopenllm_v2/MMLU-PRO": 0.4471 - } - }, - { - "id": "Lawnakk/BBALAW1.62", - "name": "BBALAW1.62", - "developer": "Lawnakk", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5046, - "hfopenllm_v2/BBH": 0.5581, - "hfopenllm_v2/MATH Level 5": 0.2825, - "hfopenllm_v2/GPQA": 0.3196, - "hfopenllm_v2/MUSR": 0.4343, - "hfopenllm_v2/MMLU-PRO": 0.4545 - } - }, - { - "id": "Lawnakk/BBALAW1.63", - "name": "BBALAW1.63", - "developer": "Lawnakk", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4407, - "hfopenllm_v2/BBH": 0.5541, - "hfopenllm_v2/MATH Level 5": 0.3701, - "hfopenllm_v2/GPQA": 0.3121, - "hfopenllm_v2/MUSR": 0.4303, - "hfopenllm_v2/MMLU-PRO": 0.4471 - } - }, - { - "id": "Lawnakk/BBALAW1.64", - "name": "BBALAW1.64", - "developer": "Lawnakk", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1395, - "hfopenllm_v2/BBH": 0.2779, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2483, - "hfopenllm_v2/MUSR": 0.3447, - "hfopenllm_v2/MMLU-PRO": 0.1115 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/leafspark.json b/data/developers/leafspark.json deleted file mode 100644 index 600febe95f303ade7fd5feeb906feac8e576ffc3..0000000000000000000000000000000000000000 --- a/data/developers/leafspark.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "leafspark", - "models": [ - { - "id": "leafspark/Llama-3.1-8B-MultiReflection-Instruct", - "name": "Llama-3.1-8B-MultiReflection-Instruct", - "developer": "leafspark", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7125, - "hfopenllm_v2/BBH": 0.5009, - "hfopenllm_v2/MATH Level 5": 0.1707, - "hfopenllm_v2/GPQA": 0.2928, - "hfopenllm_v2/MUSR": 0.3682, - "hfopenllm_v2/MMLU-PRO": 0.3724 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/leesm.json b/data/developers/leesm.json deleted file mode 100644 index cc5f95bada8da70e396ab3b966f6b04d7f03ead1..0000000000000000000000000000000000000000 --- a/data/developers/leesm.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "developer": "LEESM", - "models": [ - { - "id": "LEESM/llama-2-7b-hf-lora-oki100p", - "name": "llama-2-7b-hf-lora-oki100p", - "developer": "LEESM", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2513, - "hfopenllm_v2/BBH": 0.3492, - "hfopenllm_v2/MATH Level 5": 0.0166, - "hfopenllm_v2/GPQA": 0.2693, - "hfopenllm_v2/MUSR": 0.3687, - "hfopenllm_v2/MMLU-PRO": 0.1856 - } - }, - { - "id": "LEESM/llama-2-7b-hf-lora-oki10p", - "name": "llama-2-7b-hf-lora-oki10p", - "developer": "LEESM", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.227, - "hfopenllm_v2/BBH": 0.3531, - "hfopenllm_v2/MATH Level 5": 0.0166, - "hfopenllm_v2/GPQA": 0.2542, - "hfopenllm_v2/MUSR": 0.3475, - "hfopenllm_v2/MMLU-PRO": 0.1679 - } - }, - { - "id": "LEESM/llama-3-8b-bnb-4b-kowiki231101", - "name": "llama-3-8b-bnb-4b-kowiki231101", - "developer": "LEESM", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1685, - "hfopenllm_v2/BBH": 0.4131, - "hfopenllm_v2/MATH Level 5": 0.0136, - "hfopenllm_v2/GPQA": 0.271, - "hfopenllm_v2/MUSR": 0.3551, - "hfopenllm_v2/MMLU-PRO": 0.2425 - } - }, - { - "id": "LEESM/llama-3-Korean-Bllossom-8B-trexlab-oki10p", - "name": "llama-3-Korean-Bllossom-8B-trexlab-oki10p", - "developer": "LEESM", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2137, - "hfopenllm_v2/BBH": 0.4343, - "hfopenllm_v2/MATH Level 5": 0.0468, - "hfopenllm_v2/GPQA": 0.2752, - "hfopenllm_v2/MUSR": 0.3869, - "hfopenllm_v2/MMLU-PRO": 0.3177 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/lemon07r.json b/data/developers/lemon07r.json deleted file mode 100644 index 595d063195b99d8678cd42df0090ab56e38ba00f..0000000000000000000000000000000000000000 --- a/data/developers/lemon07r.json +++ /dev/null @@ -1,243 +0,0 @@ -{ - "developer": "lemon07r", - "models": [ - { - "id": "lemon07r/Gemma-2-Ataraxy-9B", - "name": "Gemma-2-Ataraxy-9B", - "developer": "lemon07r", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3009, - "hfopenllm_v2/BBH": 0.5931, - "hfopenllm_v2/MATH Level 5": 0.0853, - "hfopenllm_v2/GPQA": 0.3347, - "hfopenllm_v2/MUSR": 0.4424, - "hfopenllm_v2/MMLU-PRO": 0.4226 - } - }, - { - "id": "lemon07r/Gemma-2-Ataraxy-Advanced-9B", - "name": "Gemma-2-Ataraxy-Advanced-9B", - "developer": "lemon07r", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5516, - "hfopenllm_v2/BBH": 0.5889, - "hfopenllm_v2/MATH Level 5": 0.1979, - "hfopenllm_v2/GPQA": 0.3356, - "hfopenllm_v2/MUSR": 0.3761, - "hfopenllm_v2/MMLU-PRO": 0.4244 - } - }, - { - "id": "lemon07r/Gemma-2-Ataraxy-Remix-9B", - "name": "Gemma-2-Ataraxy-Remix-9B", - "developer": "lemon07r", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7083, - "hfopenllm_v2/BBH": 0.5892, - "hfopenllm_v2/MATH Level 5": 0.2017, - "hfopenllm_v2/GPQA": 0.3389, - "hfopenllm_v2/MUSR": 0.4372, - "hfopenllm_v2/MMLU-PRO": 0.4239 - } - }, - { - "id": "lemon07r/Gemma-2-Ataraxy-v2-9B", - "name": "Gemma-2-Ataraxy-v2-9B", - "developer": "lemon07r", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2136, - "hfopenllm_v2/BBH": 0.5766, - "hfopenllm_v2/MATH Level 5": 0.0846, - "hfopenllm_v2/GPQA": 0.3423, - "hfopenllm_v2/MUSR": 0.3484, - "hfopenllm_v2/MMLU-PRO": 0.4221 - } - }, - { - "id": "lemon07r/Gemma-2-Ataraxy-v2a-9B", - "name": "Gemma-2-Ataraxy-v2a-9B", - "developer": "lemon07r", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1595, - "hfopenllm_v2/BBH": 0.5182, - "hfopenllm_v2/MATH Level 5": 0.0612, - "hfopenllm_v2/GPQA": 0.3398, - "hfopenllm_v2/MUSR": 0.3165, - "hfopenllm_v2/MMLU-PRO": 0.3515 - } - }, - { - "id": "lemon07r/Gemma-2-Ataraxy-v2f-9B", - "name": "Gemma-2-Ataraxy-v2f-9B", - "developer": "lemon07r", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3791, - "hfopenllm_v2/BBH": 0.5193, - "hfopenllm_v2/MATH Level 5": 0.1163, - "hfopenllm_v2/GPQA": 0.3389, - "hfopenllm_v2/MUSR": 0.3231, - "hfopenllm_v2/MMLU-PRO": 0.3503 - } - }, - { - "id": "lemon07r/Gemma-2-Ataraxy-v3-Advanced-9B", - "name": "Gemma-2-Ataraxy-v3-Advanced-9B", - "developer": "lemon07r", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6602, - "hfopenllm_v2/BBH": 0.5935, - "hfopenllm_v2/MATH Level 5": 0.1873, - "hfopenllm_v2/GPQA": 0.3364, - "hfopenllm_v2/MUSR": 0.445, - "hfopenllm_v2/MMLU-PRO": 0.4196 - } - }, - { - "id": "lemon07r/Gemma-2-Ataraxy-v3b-9B", - "name": "Gemma-2-Ataraxy-v3b-9B", - "developer": "lemon07r", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6809, - "hfopenllm_v2/BBH": 0.5908, - "hfopenllm_v2/MATH Level 5": 0.2153, - "hfopenllm_v2/GPQA": 0.3331, - "hfopenllm_v2/MUSR": 0.4489, - "hfopenllm_v2/MMLU-PRO": 0.4205 - } - }, - { - "id": "lemon07r/Gemma-2-Ataraxy-v3i-9B", - "name": "Gemma-2-Ataraxy-v3i-9B", - "developer": "lemon07r", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4203, - "hfopenllm_v2/BBH": 0.5626, - "hfopenllm_v2/MATH Level 5": 0.1533, - "hfopenllm_v2/GPQA": 0.328, - "hfopenllm_v2/MUSR": 0.3181, - "hfopenllm_v2/MMLU-PRO": 0.4166 - } - }, - { - "id": "lemon07r/Gemma-2-Ataraxy-v3j-9B", - "name": "Gemma-2-Ataraxy-v3j-9B", - "developer": "lemon07r", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4169, - "hfopenllm_v2/BBH": 0.5632, - "hfopenllm_v2/MATH Level 5": 0.1692, - "hfopenllm_v2/GPQA": 0.328, - "hfopenllm_v2/MUSR": 0.318, - "hfopenllm_v2/MMLU-PRO": 0.4134 - } - }, - { - "id": "lemon07r/Gemma-2-Ataraxy-v4-Advanced-9B", - "name": "Gemma-2-Ataraxy-v4-Advanced-9B", - "developer": "lemon07r", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7015, - "hfopenllm_v2/BBH": 0.6024, - "hfopenllm_v2/MATH Level 5": 0.2153, - "hfopenllm_v2/GPQA": 0.3389, - "hfopenllm_v2/MUSR": 0.4581, - "hfopenllm_v2/MMLU-PRO": 0.4367 - } - }, - { - "id": "lemon07r/Gemma-2-Ataraxy-v4a-Advanced-9B", - "name": "Gemma-2-Ataraxy-v4a-Advanced-9B", - "developer": "lemon07r", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7135, - "hfopenllm_v2/BBH": 0.5988, - "hfopenllm_v2/MATH Level 5": 0.2115, - "hfopenllm_v2/GPQA": 0.344, - "hfopenllm_v2/MUSR": 0.4489, - "hfopenllm_v2/MMLU-PRO": 0.4309 - } - }, - { - "id": "lemon07r/Gemma-2-Ataraxy-v4b-9B", - "name": "Gemma-2-Ataraxy-v4b-9B", - "developer": "lemon07r", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6878, - "hfopenllm_v2/BBH": 0.6039, - "hfopenllm_v2/MATH Level 5": 0.2334, - "hfopenllm_v2/GPQA": 0.3406, - "hfopenllm_v2/MUSR": 0.4555, - "hfopenllm_v2/MMLU-PRO": 0.4357 - } - }, - { - "id": "lemon07r/Gemma-2-Ataraxy-v4c-9B", - "name": "Gemma-2-Ataraxy-v4c-9B", - "developer": "lemon07r", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6945, - "hfopenllm_v2/BBH": 0.6084, - "hfopenllm_v2/MATH Level 5": 0.2266, - "hfopenllm_v2/GPQA": 0.3339, - "hfopenllm_v2/MUSR": 0.4528, - "hfopenllm_v2/MMLU-PRO": 0.4395 - } - }, - { - "id": "lemon07r/Gemma-2-Ataraxy-v4d-9B", - "name": "Gemma-2-Ataraxy-v4d-9B", - "developer": "lemon07r", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.725, - "hfopenllm_v2/BBH": 0.6054, - "hfopenllm_v2/MATH Level 5": 0.2334, - "hfopenllm_v2/GPQA": 0.3473, - "hfopenllm_v2/MUSR": 0.4541, - "hfopenllm_v2/MMLU-PRO": 0.4346 - } - }, - { - "id": "lemon07r/llama-3-NeuralMahou-8b", - "name": "llama-3-NeuralMahou-8b", - "developer": "lemon07r", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4901, - "hfopenllm_v2/BBH": 0.4184, - "hfopenllm_v2/MATH Level 5": 0.102, - "hfopenllm_v2/GPQA": 0.2886, - "hfopenllm_v2/MUSR": 0.3873, - "hfopenllm_v2/MMLU-PRO": 0.369 - } - }, - { - "id": "lemon07r/Llama-3-RedMagic4-8B", - "name": "Llama-3-RedMagic4-8B", - "developer": "lemon07r", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4864, - "hfopenllm_v2/BBH": 0.4256, - "hfopenllm_v2/MATH Level 5": 0.0899, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.3766, - "hfopenllm_v2/MMLU-PRO": 0.3676 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/lenguajenaturalai.json b/data/developers/lenguajenaturalai.json deleted file mode 100644 index aeef2cf3b559975d29a9491a81b3abb61384a4d2..0000000000000000000000000000000000000000 --- a/data/developers/lenguajenaturalai.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "LenguajeNaturalAI", - "models": [ - { - "id": "LenguajeNaturalAI/leniachat-gemma-2b-v0", - "name": "leniachat-gemma-2b-v0", - "developer": "LenguajeNaturalAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.215, - "hfopenllm_v2/BBH": 0.3074, - "hfopenllm_v2/MATH Level 5": 0.0113, - "hfopenllm_v2/GPQA": 0.2659, - "hfopenllm_v2/MUSR": 0.3659, - "hfopenllm_v2/MMLU-PRO": 0.117 - } - }, - { - "id": "LenguajeNaturalAI/leniachat-qwen2-1.5B-v0", - "name": "leniachat-qwen2-1.5B-v0", - "developer": "LenguajeNaturalAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2221, - "hfopenllm_v2/BBH": 0.3684, - "hfopenllm_v2/MATH Level 5": 0.0128, - "hfopenllm_v2/GPQA": 0.2617, - "hfopenllm_v2/MUSR": 0.375, - "hfopenllm_v2/MMLU-PRO": 0.188 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/leroydyer.json b/data/developers/leroydyer.json deleted file mode 100644 index e1aa95462e1fed8d4e011735c7846112031a21a1..0000000000000000000000000000000000000000 --- a/data/developers/leroydyer.json +++ /dev/null @@ -1,817 +0,0 @@ -{ - "developer": "LeroyDyer", - "models": [ - { - "id": "LeroyDyer/_Spydaz_Web_AI_12", - "name": "_Spydaz_Web_AI_12", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2765, - "hfopenllm_v2/BBH": 0.3163, - "hfopenllm_v2/MATH Level 5": 0.0136, - "hfopenllm_v2/GPQA": 0.2685, - "hfopenllm_v2/MUSR": 0.3582, - "hfopenllm_v2/MMLU-PRO": 0.1137 - } - }, - { - "id": "LeroyDyer/_Spydaz_Web_AI_14", - "name": "_Spydaz_Web_AI_14", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1812, - "hfopenllm_v2/BBH": 0.2989, - "hfopenllm_v2/MATH Level 5": 0.0121, - "hfopenllm_v2/GPQA": 0.2659, - "hfopenllm_v2/MUSR": 0.3395, - "hfopenllm_v2/MMLU-PRO": 0.1139 - } - }, - { - "id": "LeroyDyer/_Spydaz_Web_AI_AGI_R1_001", - "name": "_Spydaz_Web_AI_AGI_R1_001", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4505, - "hfopenllm_v2/BBH": 0.4609, - "hfopenllm_v2/MATH Level 5": 0.0634, - "hfopenllm_v2/GPQA": 0.2676, - "hfopenllm_v2/MUSR": 0.4256, - "hfopenllm_v2/MMLU-PRO": 0.2734 - } - }, - { - "id": "LeroyDyer/_Spydaz_Web_AI_AGI_R1_002", - "name": "_Spydaz_Web_AI_AGI_R1_002", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5307, - "hfopenllm_v2/BBH": 0.4683, - "hfopenllm_v2/MATH Level 5": 0.0582, - "hfopenllm_v2/GPQA": 0.2685, - "hfopenllm_v2/MUSR": 0.4255, - "hfopenllm_v2/MMLU-PRO": 0.2894 - } - }, - { - "id": "LeroyDyer/_Spydaz_Web_AI_AGI_R1_MasterCoder", - "name": "_Spydaz_Web_AI_AGI_R1_MasterCoder", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4143, - "hfopenllm_v2/BBH": 0.4689, - "hfopenllm_v2/MATH Level 5": 0.0612, - "hfopenllm_v2/GPQA": 0.276, - "hfopenllm_v2/MUSR": 0.472, - "hfopenllm_v2/MMLU-PRO": 0.2719 - } - }, - { - "id": "LeroyDyer/_Spydaz_Web_AI_AGI_R1_Math_001", - "name": "_Spydaz_Web_AI_AGI_R1_Math_001", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4571, - "hfopenllm_v2/BBH": 0.4818, - "hfopenllm_v2/MATH Level 5": 0.0695, - "hfopenllm_v2/GPQA": 0.2768, - "hfopenllm_v2/MUSR": 0.4778, - "hfopenllm_v2/MMLU-PRO": 0.2681 - } - }, - { - "id": "LeroyDyer/_Spydaz_Web_AI_AGI_R1_Math_003", - "name": "_Spydaz_Web_AI_AGI_R1_Math_003", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.62, - "hfopenllm_v2/BBH": 0.4756, - "hfopenllm_v2/MATH Level 5": 0.0695, - "hfopenllm_v2/GPQA": 0.281, - "hfopenllm_v2/MUSR": 0.4202, - "hfopenllm_v2/MMLU-PRO": 0.2999 - } - }, - { - "id": "LeroyDyer/_Spydaz_Web_AI_AGI_R1_Math_AdvancedStudent", - "name": "_Spydaz_Web_AI_AGI_R1_Math_AdvancedStudent", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5951, - "hfopenllm_v2/BBH": 0.4927, - "hfopenllm_v2/MATH Level 5": 0.0544, - "hfopenllm_v2/GPQA": 0.2919, - "hfopenllm_v2/MUSR": 0.5198, - "hfopenllm_v2/MMLU-PRO": 0.3 - } - }, - { - "id": "LeroyDyer/_Spydaz_Web_AI_AGI_R1_Math_Student", - "name": "_Spydaz_Web_AI_AGI_R1_Math_Student", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5736, - "hfopenllm_v2/BBH": 0.4881, - "hfopenllm_v2/MATH Level 5": 0.0514, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.5098, - "hfopenllm_v2/MMLU-PRO": 0.2927 - } - }, - { - "id": "LeroyDyer/_Spydaz_Web_AI_AGI_R1_Math_Teacher", - "name": "_Spydaz_Web_AI_AGI_R1_Math_Teacher", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5772, - "hfopenllm_v2/BBH": 0.4805, - "hfopenllm_v2/MATH Level 5": 0.0544, - "hfopenllm_v2/GPQA": 0.2861, - "hfopenllm_v2/MUSR": 0.5222, - "hfopenllm_v2/MMLU-PRO": 0.2956 - } - }, - { - "id": "LeroyDyer/_Spydaz_Web_AI_AGI_R1_MUSR", - "name": "_Spydaz_Web_AI_AGI_R1_MUSR", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4786, - "hfopenllm_v2/BBH": 0.4672, - "hfopenllm_v2/MATH Level 5": 0.0604, - "hfopenllm_v2/GPQA": 0.2844, - "hfopenllm_v2/MUSR": 0.4869, - "hfopenllm_v2/MMLU-PRO": 0.2828 - } - }, - { - "id": "LeroyDyer/_Spydaz_Web_AI_AGI_R1_OmG_001", - "name": "_Spydaz_Web_AI_AGI_R1_OmG_001", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5818, - "hfopenllm_v2/BBH": 0.4908, - "hfopenllm_v2/MATH Level 5": 0.0506, - "hfopenllm_v2/GPQA": 0.3003, - "hfopenllm_v2/MUSR": 0.4486, - "hfopenllm_v2/MMLU-PRO": 0.2906 - } - }, - { - "id": "LeroyDyer/_Spydaz_Web_AI_AGI_R1_OmG_002", - "name": "_Spydaz_Web_AI_AGI_R1_OmG_002", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5462, - "hfopenllm_v2/BBH": 0.4655, - "hfopenllm_v2/MATH Level 5": 0.0498, - "hfopenllm_v2/GPQA": 0.2785, - "hfopenllm_v2/MUSR": 0.4511, - "hfopenllm_v2/MMLU-PRO": 0.2867 - } - }, - { - "id": "LeroyDyer/_Spydaz_Web_AI_AGI_R1_OmG_Coder", - "name": "_Spydaz_Web_AI_AGI_R1_OmG_Coder", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4924, - "hfopenllm_v2/BBH": 0.4638, - "hfopenllm_v2/MATH Level 5": 0.0544, - "hfopenllm_v2/GPQA": 0.2735, - "hfopenllm_v2/MUSR": 0.5625, - "hfopenllm_v2/MMLU-PRO": 0.289 - } - }, - { - "id": "LeroyDyer/_Spydaz_Web_AI_AGI_R1_OmG_Math", - "name": "_Spydaz_Web_AI_AGI_R1_OmG_Math", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5033, - "hfopenllm_v2/BBH": 0.4677, - "hfopenllm_v2/MATH Level 5": 0.0476, - "hfopenllm_v2/GPQA": 0.2827, - "hfopenllm_v2/MUSR": 0.4326, - "hfopenllm_v2/MMLU-PRO": 0.2913 - } - }, - { - "id": "LeroyDyer/_Spydaz_Web_AI_AGI_R1_OmG_MathMaster", - "name": "_Spydaz_Web_AI_AGI_R1_OmG_MathMaster", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5558, - "hfopenllm_v2/BBH": 0.4742, - "hfopenllm_v2/MATH Level 5": 0.0536, - "hfopenllm_v2/GPQA": 0.2878, - "hfopenllm_v2/MUSR": 0.451, - "hfopenllm_v2/MMLU-PRO": 0.2672 - } - }, - { - "id": "LeroyDyer/_Spydaz_Web_AI_AGI_R1_Student_Coder", - "name": "_Spydaz_Web_AI_AGI_R1_Student_Coder", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.545, - "hfopenllm_v2/BBH": 0.4651, - "hfopenllm_v2/MATH Level 5": 0.0657, - "hfopenllm_v2/GPQA": 0.2844, - "hfopenllm_v2/MUSR": 0.4388, - "hfopenllm_v2/MMLU-PRO": 0.2768 - } - }, - { - "id": "LeroyDyer/_Spydaz_Web_AI_AGI_R1_Teacher_Coder", - "name": "_Spydaz_Web_AI_AGI_R1_Teacher_Coder", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5082, - "hfopenllm_v2/BBH": 0.4797, - "hfopenllm_v2/MATH Level 5": 0.065, - "hfopenllm_v2/GPQA": 0.2911, - "hfopenllm_v2/MUSR": 0.4338, - "hfopenllm_v2/MMLU-PRO": 0.2845 - } - }, - { - "id": "LeroyDyer/_Spydaz_Web_AI_AGI_R1_Top_Student", - "name": "_Spydaz_Web_AI_AGI_R1_Top_Student", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.604, - "hfopenllm_v2/BBH": 0.4988, - "hfopenllm_v2/MATH Level 5": 0.0725, - "hfopenllm_v2/GPQA": 0.2727, - "hfopenllm_v2/MUSR": 0.5398, - "hfopenllm_v2/MMLU-PRO": 0.3024 - } - }, - { - "id": "LeroyDyer/_Spydaz_Web_AI_AGI_R1_X1", - "name": "_Spydaz_Web_AI_AGI_R1_X1", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4273, - "hfopenllm_v2/BBH": 0.4759, - "hfopenllm_v2/MATH Level 5": 0.0566, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.4232, - "hfopenllm_v2/MMLU-PRO": 0.2891 - } - }, - { - "id": "LeroyDyer/_Spydaz_Web_AI_AGI_R1_X2", - "name": "_Spydaz_Web_AI_AGI_R1_X2", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5434, - "hfopenllm_v2/BBH": 0.4786, - "hfopenllm_v2/MATH Level 5": 0.0612, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.4695, - "hfopenllm_v2/MMLU-PRO": 0.2921 - } - }, - { - "id": "LeroyDyer/_Spydaz_Web_AI_AGI_RP_R1", - "name": "_Spydaz_Web_AI_AGI_RP_R1", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5426, - "hfopenllm_v2/BBH": 0.4701, - "hfopenllm_v2/MATH Level 5": 0.0604, - "hfopenllm_v2/GPQA": 0.2693, - "hfopenllm_v2/MUSR": 0.4201, - "hfopenllm_v2/MMLU-PRO": 0.2894 - } - }, - { - "id": "LeroyDyer/_Spydaz_Web_AI_BIBLE_002", - "name": "_Spydaz_Web_AI_BIBLE_002", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2195, - "hfopenllm_v2/BBH": 0.3289, - "hfopenllm_v2/MATH Level 5": 0.0174, - "hfopenllm_v2/GPQA": 0.2844, - "hfopenllm_v2/MUSR": 0.3407, - "hfopenllm_v2/MMLU-PRO": 0.1368 - } - }, - { - "id": "LeroyDyer/_Spydaz_Web_AI_ChatML_002", - "name": "_Spydaz_Web_AI_ChatML_002", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2412, - "hfopenllm_v2/BBH": 0.3106, - "hfopenllm_v2/MATH Level 5": 0.0113, - "hfopenllm_v2/GPQA": 0.2576, - "hfopenllm_v2/MUSR": 0.3623, - "hfopenllm_v2/MMLU-PRO": 0.1095 - } - }, - { - "id": "LeroyDyer/_Spydaz_Web_AI_ChatQA", - "name": "_Spydaz_Web_AI_ChatQA", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1415, - "hfopenllm_v2/BBH": 0.3236, - "hfopenllm_v2/MATH Level 5": 0.0098, - "hfopenllm_v2/GPQA": 0.2659, - "hfopenllm_v2/MUSR": 0.3447, - "hfopenllm_v2/MMLU-PRO": 0.1475 - } - }, - { - "id": "LeroyDyer/_Spydaz_Web_AI_ChatQA_003", - "name": "_Spydaz_Web_AI_ChatQA_003", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2209, - "hfopenllm_v2/BBH": 0.3172, - "hfopenllm_v2/MATH Level 5": 0.0106, - "hfopenllm_v2/GPQA": 0.271, - "hfopenllm_v2/MUSR": 0.3818, - "hfopenllm_v2/MMLU-PRO": 0.1133 - } - }, - { - "id": "LeroyDyer/_Spydaz_Web_AI_TEMP_", - "name": "_Spydaz_Web_AI_TEMP_", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4795, - "hfopenllm_v2/BBH": 0.4957, - "hfopenllm_v2/MATH Level 5": 0.1239, - "hfopenllm_v2/GPQA": 0.2794, - "hfopenllm_v2/MUSR": 0.4218, - "hfopenllm_v2/MMLU-PRO": 0.3121 - } - }, - { - "id": "LeroyDyer/_Spydaz_Web_AI_Top_Teacher_", - "name": "_Spydaz_Web_AI_Top_Teacher_", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4404, - "hfopenllm_v2/BBH": 0.4891, - "hfopenllm_v2/MATH Level 5": 0.1156, - "hfopenllm_v2/GPQA": 0.2777, - "hfopenllm_v2/MUSR": 0.4366, - "hfopenllm_v2/MMLU-PRO": 0.315 - } - }, - { - "id": "LeroyDyer/CheckPoint_A", - "name": "CheckPoint_A", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4513, - "hfopenllm_v2/BBH": 0.4748, - "hfopenllm_v2/MATH Level 5": 0.0589, - "hfopenllm_v2/GPQA": 0.2836, - "hfopenllm_v2/MUSR": 0.4231, - "hfopenllm_v2/MMLU-PRO": 0.288 - } - }, - { - "id": "LeroyDyer/CheckPoint_B", - "name": "CheckPoint_B", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.444, - "hfopenllm_v2/BBH": 0.478, - "hfopenllm_v2/MATH Level 5": 0.0718, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.3898, - "hfopenllm_v2/MMLU-PRO": 0.2907 - } - }, - { - "id": "LeroyDyer/CheckPoint_C", - "name": "CheckPoint_C", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3477, - "hfopenllm_v2/BBH": 0.4586, - "hfopenllm_v2/MATH Level 5": 0.0551, - "hfopenllm_v2/GPQA": 0.271, - "hfopenllm_v2/MUSR": 0.4346, - "hfopenllm_v2/MMLU-PRO": 0.3021 - } - }, - { - "id": "LeroyDyer/CheckPoint_R1", - "name": "CheckPoint_R1", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1728, - "hfopenllm_v2/BBH": 0.4225, - "hfopenllm_v2/MATH Level 5": 0.0431, - "hfopenllm_v2/GPQA": 0.2743, - "hfopenllm_v2/MUSR": 0.4031, - "hfopenllm_v2/MMLU-PRO": 0.2205 - } - }, - { - "id": "LeroyDyer/LCARS_AI_001", - "name": "LCARS_AI_001", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3109, - "hfopenllm_v2/BBH": 0.4258, - "hfopenllm_v2/MATH Level 5": 0.0234, - "hfopenllm_v2/GPQA": 0.2634, - "hfopenllm_v2/MUSR": 0.4384, - "hfopenllm_v2/MMLU-PRO": 0.267 - } - }, - { - "id": "LeroyDyer/LCARS_AI_1x4_003_SuperAI", - "name": "LCARS_AI_1x4_003_SuperAI", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4111, - "hfopenllm_v2/BBH": 0.492, - "hfopenllm_v2/MATH Level 5": 0.0574, - "hfopenllm_v2/GPQA": 0.2827, - "hfopenllm_v2/MUSR": 0.4506, - "hfopenllm_v2/MMLU-PRO": 0.2972 - } - }, - { - "id": "LeroyDyer/LCARS_AI_StarTrek_Computer", - "name": "LCARS_AI_StarTrek_Computer", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3583, - "hfopenllm_v2/BBH": 0.4446, - "hfopenllm_v2/MATH Level 5": 0.0408, - "hfopenllm_v2/GPQA": 0.2676, - "hfopenllm_v2/MUSR": 0.395, - "hfopenllm_v2/MMLU-PRO": 0.2458 - } - }, - { - "id": "LeroyDyer/LCARS_TOP_SCORE", - "name": "LCARS_TOP_SCORE", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4371, - "hfopenllm_v2/BBH": 0.5127, - "hfopenllm_v2/MATH Level 5": 0.0672, - "hfopenllm_v2/GPQA": 0.2861, - "hfopenllm_v2/MUSR": 0.4293, - "hfopenllm_v2/MMLU-PRO": 0.3031 - } - }, - { - "id": "LeroyDyer/Mixtral_AI_SwahiliTron_7b", - "name": "Mixtral_AI_SwahiliTron_7b", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1534, - "hfopenllm_v2/BBH": 0.3055, - "hfopenllm_v2/MATH Level 5": 0.0136, - "hfopenllm_v2/GPQA": 0.2651, - "hfopenllm_v2/MUSR": 0.342, - "hfopenllm_v2/MMLU-PRO": 0.1208 - } - }, - { - "id": "LeroyDyer/SpydazWeb_AI_CyberTron_Ultra_7b", - "name": "SpydazWeb_AI_CyberTron_Ultra_7b", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1556, - "hfopenllm_v2/BBH": 0.4811, - "hfopenllm_v2/MATH Level 5": 0.0136, - "hfopenllm_v2/GPQA": 0.2928, - "hfopenllm_v2/MUSR": 0.4136, - "hfopenllm_v2/MMLU-PRO": 0.2866 - } - }, - { - "id": "LeroyDyer/SpydazWeb_AI_HumanAGI_001_M2", - "name": "SpydazWeb_AI_HumanAGI_001_M2", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.394, - "hfopenllm_v2/BBH": 0.4888, - "hfopenllm_v2/MATH Level 5": 0.0385, - "hfopenllm_v2/GPQA": 0.2894, - "hfopenllm_v2/MUSR": 0.4503, - "hfopenllm_v2/MMLU-PRO": 0.3005 - } - }, - { - "id": "LeroyDyer/SpydazWeb_AI_HumanAGI_002", - "name": "SpydazWeb_AI_HumanAGI_002", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4088, - "hfopenllm_v2/BBH": 0.5044, - "hfopenllm_v2/MATH Level 5": 0.0665, - "hfopenllm_v2/GPQA": 0.2869, - "hfopenllm_v2/MUSR": 0.4865, - "hfopenllm_v2/MMLU-PRO": 0.3059 - } - }, - { - "id": "LeroyDyer/SpydazWeb_AI_HumanAI_001", - "name": "SpydazWeb_AI_HumanAI_001", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2252, - "hfopenllm_v2/BBH": 0.3344, - "hfopenllm_v2/MATH Level 5": 0.0166, - "hfopenllm_v2/GPQA": 0.2886, - "hfopenllm_v2/MUSR": 0.386, - "hfopenllm_v2/MMLU-PRO": 0.1271 - } - }, - { - "id": "LeroyDyer/SpydazWeb_AI_HumanAI_006", - "name": "SpydazWeb_AI_HumanAI_006", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.143, - "hfopenllm_v2/BBH": 0.3302, - "hfopenllm_v2/MATH Level 5": 0.0106, - "hfopenllm_v2/GPQA": 0.2802, - "hfopenllm_v2/MUSR": 0.3568, - "hfopenllm_v2/MMLU-PRO": 0.1135 - } - }, - { - "id": "LeroyDyer/SpydazWeb_AI_HumanAI_007", - "name": "SpydazWeb_AI_HumanAI_007", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3352, - "hfopenllm_v2/BBH": 0.3416, - "hfopenllm_v2/MATH Level 5": 0.0227, - "hfopenllm_v2/GPQA": 0.2886, - "hfopenllm_v2/MUSR": 0.4096, - "hfopenllm_v2/MMLU-PRO": 0.1352 - } - }, - { - "id": "LeroyDyer/SpydazWeb_AI_HumanAI_009_CHAT", - "name": "SpydazWeb_AI_HumanAI_009_CHAT", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2973, - "hfopenllm_v2/BBH": 0.3307, - "hfopenllm_v2/MATH Level 5": 0.0166, - "hfopenllm_v2/GPQA": 0.281, - "hfopenllm_v2/MUSR": 0.4138, - "hfopenllm_v2/MMLU-PRO": 0.1433 - } - }, - { - "id": "LeroyDyer/SpydazWeb_AI_HumanAI_010_CHAT", - "name": "SpydazWeb_AI_HumanAI_010_CHAT", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2507, - "hfopenllm_v2/BBH": 0.3336, - "hfopenllm_v2/MATH Level 5": 0.0181, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.4137, - "hfopenllm_v2/MMLU-PRO": 0.143 - } - }, - { - "id": "LeroyDyer/SpydazWeb_AI_HumanAI_011_INSTRUCT", - "name": "SpydazWeb_AI_HumanAI_011_INSTRUCT", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3149, - "hfopenllm_v2/BBH": 0.3523, - "hfopenllm_v2/MATH Level 5": 0.0144, - "hfopenllm_v2/GPQA": 0.2794, - "hfopenllm_v2/MUSR": 0.3831, - "hfopenllm_v2/MMLU-PRO": 0.1595 - } - }, - { - "id": "LeroyDyer/SpydazWeb_AI_HumanAI_011_INSTRUCT_ML", - "name": "SpydazWeb_AI_HumanAI_011_INSTRUCT_ML", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3752, - "hfopenllm_v2/BBH": 0.3984, - "hfopenllm_v2/MATH Level 5": 0.0257, - "hfopenllm_v2/GPQA": 0.2928, - "hfopenllm_v2/MUSR": 0.4239, - "hfopenllm_v2/MMLU-PRO": 0.2019 - } - }, - { - "id": "LeroyDyer/SpydazWeb_AI_HumanAI_011_INSTRUCT_ML_r1", - "name": "SpydazWeb_AI_HumanAI_011_INSTRUCT_ML_r1", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.405, - "hfopenllm_v2/BBH": 0.4858, - "hfopenllm_v2/MATH Level 5": 0.0551, - "hfopenllm_v2/GPQA": 0.2928, - "hfopenllm_v2/MUSR": 0.3921, - "hfopenllm_v2/MMLU-PRO": 0.2956 - } - }, - { - "id": "LeroyDyer/SpydazWeb_AI_HumanAI_012_INSTRUCT_IA", - "name": "SpydazWeb_AI_HumanAI_012_INSTRUCT_IA", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3036, - "hfopenllm_v2/BBH": 0.4575, - "hfopenllm_v2/MATH Level 5": 0.0446, - "hfopenllm_v2/GPQA": 0.3012, - "hfopenllm_v2/MUSR": 0.4253, - "hfopenllm_v2/MMLU-PRO": 0.2329 - } - }, - { - "id": "LeroyDyer/SpydazWeb_AI_HumanAI_012_INSTRUCT_MX", - "name": "SpydazWeb_AI_HumanAI_012_INSTRUCT_MX", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3066, - "hfopenllm_v2/BBH": 0.3158, - "hfopenllm_v2/MATH Level 5": 0.0151, - "hfopenllm_v2/GPQA": 0.2911, - "hfopenllm_v2/MUSR": 0.3444, - "hfopenllm_v2/MMLU-PRO": 0.1107 - } - }, - { - "id": "LeroyDyer/SpydazWeb_AI_HumanAI_012_INSTRUCT_XA", - "name": "SpydazWeb_AI_HumanAI_012_INSTRUCT_XA", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3579, - "hfopenllm_v2/BBH": 0.4477, - "hfopenllm_v2/MATH Level 5": 0.0423, - "hfopenllm_v2/GPQA": 0.3096, - "hfopenllm_v2/MUSR": 0.4134, - "hfopenllm_v2/MMLU-PRO": 0.2376 - } - }, - { - "id": "LeroyDyer/SpydazWeb_AI_HumanAI_RP", - "name": "SpydazWeb_AI_HumanAI_RP", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2541, - "hfopenllm_v2/BBH": 0.3323, - "hfopenllm_v2/MATH Level 5": 0.0128, - "hfopenllm_v2/GPQA": 0.2752, - "hfopenllm_v2/MUSR": 0.3883, - "hfopenllm_v2/MMLU-PRO": 0.1324 - } - }, - { - "id": "LeroyDyer/SpydazWeb_AI_HumanAI_TextVision", - "name": "SpydazWeb_AI_HumanAI_TextVision", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3063, - "hfopenllm_v2/BBH": 0.3354, - "hfopenllm_v2/MATH Level 5": 0.0144, - "hfopenllm_v2/GPQA": 0.2919, - "hfopenllm_v2/MUSR": 0.3938, - "hfopenllm_v2/MMLU-PRO": 0.1387 - } - }, - { - "id": "LeroyDyer/SpydazWeb_HumanAI_M1", - "name": "SpydazWeb_HumanAI_M1", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3582, - "hfopenllm_v2/BBH": 0.3563, - "hfopenllm_v2/MATH Level 5": 0.0249, - "hfopenllm_v2/GPQA": 0.2676, - "hfopenllm_v2/MUSR": 0.3671, - "hfopenllm_v2/MMLU-PRO": 0.1663 - } - }, - { - "id": "LeroyDyer/SpydazWeb_HumanAI_M2", - "name": "SpydazWeb_HumanAI_M2", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.375, - "hfopenllm_v2/BBH": 0.3931, - "hfopenllm_v2/MATH Level 5": 0.0287, - "hfopenllm_v2/GPQA": 0.2794, - "hfopenllm_v2/MUSR": 0.3751, - "hfopenllm_v2/MMLU-PRO": 0.201 - } - }, - { - "id": "LeroyDyer/SpydazWeb_HumanAI_M3", - "name": "SpydazWeb_HumanAI_M3", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1579, - "hfopenllm_v2/BBH": 0.3127, - "hfopenllm_v2/MATH Level 5": 0.0091, - "hfopenllm_v2/GPQA": 0.271, - "hfopenllm_v2/MUSR": 0.3914, - "hfopenllm_v2/MMLU-PRO": 0.1149 - } - }, - { - "id": "LeroyDyer/SpydazWebAI_Human_AGI", - "name": "SpydazWebAI_Human_AGI", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3388, - "hfopenllm_v2/BBH": 0.3375, - "hfopenllm_v2/MATH Level 5": 0.0144, - "hfopenllm_v2/GPQA": 0.2827, - "hfopenllm_v2/MUSR": 0.3966, - "hfopenllm_v2/MMLU-PRO": 0.1479 - } - }, - { - "id": "LeroyDyer/SpydazWebAI_Human_AGI_001", - "name": "SpydazWebAI_Human_AGI_001", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3118, - "hfopenllm_v2/BBH": 0.3433, - "hfopenllm_v2/MATH Level 5": 0.0196, - "hfopenllm_v2/GPQA": 0.2987, - "hfopenllm_v2/MUSR": 0.3994, - "hfopenllm_v2/MMLU-PRO": 0.1426 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/lesubra.json b/data/developers/lesubra.json deleted file mode 100644 index 69fc58639a13523de36e07336784157369037d4d..0000000000000000000000000000000000000000 --- a/data/developers/lesubra.json +++ /dev/null @@ -1,117 +0,0 @@ -{ - "developer": "lesubra", - "models": [ - { - "id": "lesubra/ECE-EIFFEL-3B", - "name": "ECE-EIFFEL-3B", - "developer": "lesubra", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3469, - "hfopenllm_v2/BBH": 0.5102, - "hfopenllm_v2/MATH Level 5": 0.1216, - "hfopenllm_v2/GPQA": 0.3314, - "hfopenllm_v2/MUSR": 0.4362, - "hfopenllm_v2/MMLU-PRO": 0.3821 - } - }, - { - "id": "lesubra/ECE-EIFFEL-3Bv2", - "name": "ECE-EIFFEL-3Bv2", - "developer": "lesubra", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3013, - "hfopenllm_v2/BBH": 0.5424, - "hfopenllm_v2/MATH Level 5": 0.1186, - "hfopenllm_v2/GPQA": 0.3356, - "hfopenllm_v2/MUSR": 0.4443, - "hfopenllm_v2/MMLU-PRO": 0.3999 - } - }, - { - "id": "lesubra/ECE-EIFFEL-3Bv3", - "name": "ECE-EIFFEL-3Bv3", - "developer": "lesubra", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3786, - "hfopenllm_v2/BBH": 0.5469, - "hfopenllm_v2/MATH Level 5": 0.1669, - "hfopenllm_v2/GPQA": 0.3297, - "hfopenllm_v2/MUSR": 0.4675, - "hfopenllm_v2/MMLU-PRO": 0.3975 - } - }, - { - "id": "lesubra/ECE-PRYMMAL-3B-SLERP-V1", - "name": "ECE-PRYMMAL-3B-SLERP-V1", - "developer": "lesubra", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2933, - "hfopenllm_v2/BBH": 0.5341, - "hfopenllm_v2/MATH Level 5": 0.1662, - "hfopenllm_v2/GPQA": 0.3171, - "hfopenllm_v2/MUSR": 0.4595, - "hfopenllm_v2/MMLU-PRO": 0.39 - } - }, - { - "id": "lesubra/ECE-PRYMMAL-3B-SLERP-V2", - "name": "ECE-PRYMMAL-3B-SLERP-V2", - "developer": "lesubra", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2933, - "hfopenllm_v2/BBH": 0.5341, - "hfopenllm_v2/MATH Level 5": 0.1662, - "hfopenllm_v2/GPQA": 0.3171, - "hfopenllm_v2/MUSR": 0.4595, - "hfopenllm_v2/MMLU-PRO": 0.39 - } - }, - { - "id": "lesubra/ECE-PRYMMAL-3B-SLERP_2-V1", - "name": "ECE-PRYMMAL-3B-SLERP_2-V1", - "developer": "lesubra", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3649, - "hfopenllm_v2/BBH": 0.5411, - "hfopenllm_v2/MATH Level 5": 0.1677, - "hfopenllm_v2/GPQA": 0.3213, - "hfopenllm_v2/MUSR": 0.4661, - "hfopenllm_v2/MMLU-PRO": 0.399 - } - }, - { - "id": "lesubra/ECE-PRYMMAL-3B-SLERP_2-V2", - "name": "ECE-PRYMMAL-3B-SLERP_2-V2", - "developer": "lesubra", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3664, - "hfopenllm_v2/BBH": 0.5411, - "hfopenllm_v2/MATH Level 5": 0.1677, - "hfopenllm_v2/GPQA": 0.3213, - "hfopenllm_v2/MUSR": 0.4661, - "hfopenllm_v2/MMLU-PRO": 0.399 - } - }, - { - "id": "lesubra/merge-test", - "name": "merge-test", - "developer": "lesubra", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5383, - "hfopenllm_v2/BBH": 0.524, - "hfopenllm_v2/MATH Level 5": 0.1208, - "hfopenllm_v2/GPQA": 0.3221, - "hfopenllm_v2/MUSR": 0.4419, - "hfopenllm_v2/MMLU-PRO": 0.3874 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/lgai-exaone.json b/data/developers/lgai-exaone.json deleted file mode 100644 index 3ad004616133013115f31cdace4658aa65a9b3fb..0000000000000000000000000000000000000000 --- a/data/developers/lgai-exaone.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "developer": "LGAI-EXAONE", - "models": [ - { - "id": "LGAI-EXAONE/EXAONE-3.0-7.8B-Instruct", - "name": "EXAONE-3.0-7.8B-Instruct", - "developer": "LGAI-EXAONE", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7193, - "hfopenllm_v2/BBH": 0.4174, - "hfopenllm_v2/MATH Level 5": 0.3044, - "hfopenllm_v2/GPQA": 0.2659, - "hfopenllm_v2/MUSR": 0.3661, - "hfopenllm_v2/MMLU-PRO": 0.3577 - } - }, - { - "id": "LGAI-EXAONE/EXAONE-3.5-2.4B-Instruct", - "name": "EXAONE-3.5-2.4B-Instruct", - "developer": "LGAI-EXAONE", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.795, - "hfopenllm_v2/BBH": 0.4092, - "hfopenllm_v2/MATH Level 5": 0.3678, - "hfopenllm_v2/GPQA": 0.2659, - "hfopenllm_v2/MUSR": 0.3661, - "hfopenllm_v2/MMLU-PRO": 0.328 - } - }, - { - "id": "LGAI-EXAONE/EXAONE-3.5-32B-Instruct", - "name": "EXAONE-3.5-32B-Instruct", - "developer": "LGAI-EXAONE", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8392, - "hfopenllm_v2/BBH": 0.5761, - "hfopenllm_v2/MATH Level 5": 0.5128, - "hfopenllm_v2/GPQA": 0.2878, - "hfopenllm_v2/MUSR": 0.3807, - "hfopenllm_v2/MMLU-PRO": 0.4637 - } - }, - { - "id": "LGAI-EXAONE/EXAONE-3.5-7.8B-Instruct", - "name": "EXAONE-3.5-7.8B-Instruct", - "developer": "LGAI-EXAONE", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8136, - "hfopenllm_v2/BBH": 0.4728, - "hfopenllm_v2/MATH Level 5": 0.4751, - "hfopenllm_v2/GPQA": 0.2576, - "hfopenllm_v2/MUSR": 0.3779, - "hfopenllm_v2/MMLU-PRO": 0.4133 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/lightblue.json b/data/developers/lightblue.json deleted file mode 100644 index f33eda917b9f6e9ecf0fdaa52393b108e888d013..0000000000000000000000000000000000000000 --- a/data/developers/lightblue.json +++ /dev/null @@ -1,75 +0,0 @@ -{ - "developer": "lightblue", - "models": [ - { - "id": "lightblue/suzume-llama-3-8B-multilingual", - "name": "suzume-llama-3-8B-multilingual", - "developer": "lightblue", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6678, - "hfopenllm_v2/BBH": 0.495, - "hfopenllm_v2/MATH Level 5": 0.0944, - "hfopenllm_v2/GPQA": 0.2836, - "hfopenllm_v2/MUSR": 0.3977, - "hfopenllm_v2/MMLU-PRO": 0.3383 - } - }, - { - "id": "lightblue/suzume-llama-3-8B-multilingual-orpo-borda-full", - "name": "suzume-llama-3-8B-multilingual-orpo-borda-full", - "developer": "lightblue", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5817, - "hfopenllm_v2/BBH": 0.4714, - "hfopenllm_v2/MATH Level 5": 0.0763, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3222, - "hfopenllm_v2/MMLU-PRO": 0.331 - } - }, - { - "id": "lightblue/suzume-llama-3-8B-multilingual-orpo-borda-half", - "name": "suzume-llama-3-8B-multilingual-orpo-borda-half", - "developer": "lightblue", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6249, - "hfopenllm_v2/BBH": 0.4707, - "hfopenllm_v2/MATH Level 5": 0.0906, - "hfopenllm_v2/GPQA": 0.245, - "hfopenllm_v2/MUSR": 0.3516, - "hfopenllm_v2/MMLU-PRO": 0.3614 - } - }, - { - "id": "lightblue/suzume-llama-3-8B-multilingual-orpo-borda-top25", - "name": "suzume-llama-3-8B-multilingual-orpo-borda-top25", - "developer": "lightblue", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6637, - "hfopenllm_v2/BBH": 0.4865, - "hfopenllm_v2/MATH Level 5": 0.1042, - "hfopenllm_v2/GPQA": 0.2727, - "hfopenllm_v2/MUSR": 0.3566, - "hfopenllm_v2/MMLU-PRO": 0.3684 - } - }, - { - "id": "lightblue/suzume-llama-3-8B-multilingual-orpo-borda-top75", - "name": "suzume-llama-3-8B-multilingual-orpo-borda-top75", - "developer": "lightblue", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6687, - "hfopenllm_v2/BBH": 0.4833, - "hfopenllm_v2/MATH Level 5": 0.0785, - "hfopenllm_v2/GPQA": 0.2727, - "hfopenllm_v2/MUSR": 0.3817, - "hfopenllm_v2/MMLU-PRO": 0.3769 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/lightningrodlabs.json b/data/developers/lightningrodlabs.json deleted file mode 100644 index 2cac16c12675b0e38476f17047518aa0f6b980a5..0000000000000000000000000000000000000000 --- a/data/developers/lightningrodlabs.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "developer": "LightningRodLabs", - "models": [ - { - "id": "LightningRodLabs/Flashlight-v1.0", - "name": "Flashlight-v1.0", - "developer": "LightningRodLabs", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6745, - "hfopenllm_v2/BBH": 0.6877, - "hfopenllm_v2/MATH Level 5": 0.497, - "hfopenllm_v2/GPQA": 0.3423, - "hfopenllm_v2/MUSR": 0.4101, - "hfopenllm_v2/MMLU-PRO": 0.5402 - } - }, - { - "id": "LightningRodLabs/Flashlight-v1.1", - "name": "Flashlight-v1.1", - "developer": "LightningRodLabs", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6721, - "hfopenllm_v2/BBH": 0.6901, - "hfopenllm_v2/MATH Level 5": 0.5325, - "hfopenllm_v2/GPQA": 0.3398, - "hfopenllm_v2/MUSR": 0.4048, - "hfopenllm_v2/MMLU-PRO": 0.5416 - } - }, - { - "id": "LightningRodLabs/Flashlight-v1.2", - "name": "Flashlight-v1.2", - "developer": "LightningRodLabs", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.436, - "hfopenllm_v2/BBH": 0.3265, - "hfopenllm_v2/MATH Level 5": 0.1556, - "hfopenllm_v2/GPQA": 0.2357, - "hfopenllm_v2/MUSR": 0.4554, - "hfopenllm_v2/MMLU-PRO": 0.2485 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/lil-r.json b/data/developers/lil-r.json deleted file mode 100644 index 1a6a47cac2ee87327c9ea0e6eecfb82aaf1293ad..0000000000000000000000000000000000000000 --- a/data/developers/lil-r.json +++ /dev/null @@ -1,117 +0,0 @@ -{ - "developer": "Lil-R", - "models": [ - { - "id": "Lil-R/2_PRYMMAL-ECE-2B-SLERP-V1", - "name": "2_PRYMMAL-ECE-2B-SLERP-V1", - "developer": "Lil-R", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5823, - "hfopenllm_v2/BBH": 0.4287, - "hfopenllm_v2/MATH Level 5": 0.0914, - "hfopenllm_v2/GPQA": 0.3062, - "hfopenllm_v2/MUSR": 0.4375, - "hfopenllm_v2/MMLU-PRO": 0.2678 - } - }, - { - "id": "Lil-R/2_PRYMMAL-ECE-2B-SLERP-V2", - "name": "2_PRYMMAL-ECE-2B-SLERP-V2", - "developer": "Lil-R", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5543, - "hfopenllm_v2/BBH": 0.4376, - "hfopenllm_v2/MATH Level 5": 0.0944, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.4482, - "hfopenllm_v2/MMLU-PRO": 0.2744 - } - }, - { - "id": "Lil-R/2_PRYMMAL-ECE-7B-SLERP", - "name": "2_PRYMMAL-ECE-7B-SLERP", - "developer": "Lil-R", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5577, - "hfopenllm_v2/BBH": 0.5557, - "hfopenllm_v2/MATH Level 5": 0.3633, - "hfopenllm_v2/GPQA": 0.3104, - "hfopenllm_v2/MUSR": 0.4396, - "hfopenllm_v2/MMLU-PRO": 0.4507 - } - }, - { - "id": "Lil-R/2_PRYMMAL-ECE-7B-SLERP-V1", - "name": "2_PRYMMAL-ECE-7B-SLERP-V1", - "developer": "Lil-R", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1073, - "hfopenllm_v2/BBH": 0.3053, - "hfopenllm_v2/MATH Level 5": 0.0008, - "hfopenllm_v2/GPQA": 0.2508, - "hfopenllm_v2/MUSR": 0.3911, - "hfopenllm_v2/MMLU-PRO": 0.1124 - } - }, - { - "id": "Lil-R/2_PRYMMAL-ECE-7B-SLERP-V2", - "name": "2_PRYMMAL-ECE-7B-SLERP-V2", - "developer": "Lil-R", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1073, - "hfopenllm_v2/BBH": 0.3053, - "hfopenllm_v2/MATH Level 5": 0.0008, - "hfopenllm_v2/GPQA": 0.2508, - "hfopenllm_v2/MUSR": 0.3911, - "hfopenllm_v2/MMLU-PRO": 0.1124 - } - }, - { - "id": "Lil-R/2_PRYMMAL-ECE-7B-SLERP-V3", - "name": "2_PRYMMAL-ECE-7B-SLERP-V3", - "developer": "Lil-R", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2235, - "hfopenllm_v2/BBH": 0.3578, - "hfopenllm_v2/MATH Level 5": 0.006, - "hfopenllm_v2/GPQA": 0.2567, - "hfopenllm_v2/MUSR": 0.4107, - "hfopenllm_v2/MMLU-PRO": 0.1817 - } - }, - { - "id": "Lil-R/PRYMMAL-ECE-1B-SLERP-V1", - "name": "PRYMMAL-ECE-1B-SLERP-V1", - "developer": "Lil-R", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2874, - "hfopenllm_v2/BBH": 0.419, - "hfopenllm_v2/MATH Level 5": 0.1035, - "hfopenllm_v2/GPQA": 0.276, - "hfopenllm_v2/MUSR": 0.3974, - "hfopenllm_v2/MMLU-PRO": 0.2926 - } - }, - { - "id": "Lil-R/PRYMMAL-ECE-7B-SLERP-V8", - "name": "PRYMMAL-ECE-7B-SLERP-V8", - "developer": "Lil-R", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1258, - "hfopenllm_v2/BBH": 0.2955, - "hfopenllm_v2/MATH Level 5": 0.0098, - "hfopenllm_v2/GPQA": 0.25, - "hfopenllm_v2/MUSR": 0.3631, - "hfopenllm_v2/MMLU-PRO": 0.1128 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/lilrg.json b/data/developers/lilrg.json deleted file mode 100644 index 6ec2c46461d7079d0a9dd53ece713a8f1174cf8d..0000000000000000000000000000000000000000 --- a/data/developers/lilrg.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "developer": "LilRg", - "models": [ - { - "id": "LilRg/10PRYMMAL-3B-slerp", - "name": "10PRYMMAL-3B-slerp", - "developer": "LilRg", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1946, - "hfopenllm_v2/BBH": 0.532, - "hfopenllm_v2/MATH Level 5": 0.1495, - "hfopenllm_v2/GPQA": 0.3213, - "hfopenllm_v2/MUSR": 0.4529, - "hfopenllm_v2/MMLU-PRO": 0.3881 - } - }, - { - "id": "LilRg/ECE-1B-merge-PRYMMAL", - "name": "ECE-1B-merge-PRYMMAL", - "developer": "LilRg", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2712, - "hfopenllm_v2/BBH": 0.4235, - "hfopenllm_v2/MATH Level 5": 0.1012, - "hfopenllm_v2/GPQA": 0.281, - "hfopenllm_v2/MUSR": 0.3801, - "hfopenllm_v2/MMLU-PRO": 0.2906 - } - }, - { - "id": "LilRg/ECE_Finetunning", - "name": "ECE_Finetunning", - "developer": "LilRg", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0445, - "hfopenllm_v2/BBH": 0.4732, - "hfopenllm_v2/MATH Level 5": 0.0453, - "hfopenllm_v2/GPQA": 0.2827, - "hfopenllm_v2/MUSR": 0.3839, - "hfopenllm_v2/MMLU-PRO": 0.3191 - } - }, - { - "id": "LilRg/PRYMMAL-6B-slerp", - "name": "PRYMMAL-6B-slerp", - "developer": "LilRg", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1153, - "hfopenllm_v2/BBH": 0.2868, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2458, - "hfopenllm_v2/MUSR": 0.3698, - "hfopenllm_v2/MMLU-PRO": 0.1108 - } - }, - { - "id": "LilRg/PRYMMAL-ECE-7B-SLERP-V3", - "name": "PRYMMAL-ECE-7B-SLERP-V3", - "developer": "LilRg", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1243, - "hfopenllm_v2/BBH": 0.2957, - "hfopenllm_v2/MATH Level 5": 0.0098, - "hfopenllm_v2/GPQA": 0.2567, - "hfopenllm_v2/MUSR": 0.3671, - "hfopenllm_v2/MMLU-PRO": 0.1127 - } - }, - { - "id": "LilRg/PRYMMAL-ECE-7B-SLERP-V4", - "name": "PRYMMAL-ECE-7B-SLERP-V4", - "developer": "LilRg", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1249, - "hfopenllm_v2/BBH": 0.2957, - "hfopenllm_v2/MATH Level 5": 0.0098, - "hfopenllm_v2/GPQA": 0.2567, - "hfopenllm_v2/MUSR": 0.3671, - "hfopenllm_v2/MMLU-PRO": 0.1127 - } - }, - { - "id": "LilRg/PRYMMAL-ECE-7B-SLERP-V5", - "name": "PRYMMAL-ECE-7B-SLERP-V5", - "developer": "LilRg", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1249, - "hfopenllm_v2/BBH": 0.2957, - "hfopenllm_v2/MATH Level 5": 0.0098, - "hfopenllm_v2/GPQA": 0.2567, - "hfopenllm_v2/MUSR": 0.3671, - "hfopenllm_v2/MMLU-PRO": 0.1127 - } - }, - { - "id": "LilRg/PRYMMAL-ECE-7B-SLERP-V6", - "name": "PRYMMAL-ECE-7B-SLERP-V6", - "developer": "LilRg", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1243, - "hfopenllm_v2/BBH": 0.2957, - "hfopenllm_v2/MATH Level 5": 0.0098, - "hfopenllm_v2/GPQA": 0.2567, - "hfopenllm_v2/MUSR": 0.3671, - "hfopenllm_v2/MMLU-PRO": 0.1127 - } - }, - { - "id": "LilRg/PRYMMAL-ECE-7B-SLERP-V7", - "name": "PRYMMAL-ECE-7B-SLERP-V7", - "developer": "LilRg", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1249, - "hfopenllm_v2/BBH": 0.2957, - "hfopenllm_v2/MATH Level 5": 0.0098, - "hfopenllm_v2/GPQA": 0.2567, - "hfopenllm_v2/MUSR": 0.3671, - "hfopenllm_v2/MMLU-PRO": 0.1127 - } - }, - { - "id": "LilRg/PRYMMAL-slerp-Merge", - "name": "PRYMMAL-slerp-Merge", - "developer": "LilRg", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3044, - "hfopenllm_v2/BBH": 0.5364, - "hfopenllm_v2/MATH Level 5": 0.1616, - "hfopenllm_v2/GPQA": 0.3205, - "hfopenllm_v2/MUSR": 0.4635, - "hfopenllm_v2/MMLU-PRO": 0.3863 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/limyeri.json b/data/developers/limyeri.json deleted file mode 100644 index 48325ed0a50912b45c2e5ff82562c0d0c8ba5c21..0000000000000000000000000000000000000000 --- a/data/developers/limyeri.json +++ /dev/null @@ -1,75 +0,0 @@ -{ - "developer": "LimYeri", - "models": [ - { - "id": "LimYeri/CodeMind-Llama3-8B-unsloth_v2-merged", - "name": "CodeMind-Llama3-8B-unsloth_v2-merged", - "developer": "LimYeri", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6946, - "hfopenllm_v2/BBH": 0.486, - "hfopenllm_v2/MATH Level 5": 0.0665, - "hfopenllm_v2/GPQA": 0.2651, - "hfopenllm_v2/MUSR": 0.3316, - "hfopenllm_v2/MMLU-PRO": 0.3506 - } - }, - { - "id": "LimYeri/CodeMind-Llama3-8B-unsloth_v3-merged", - "name": "CodeMind-Llama3-8B-unsloth_v3-merged", - "developer": "LimYeri", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6763, - "hfopenllm_v2/BBH": 0.4908, - "hfopenllm_v2/MATH Level 5": 0.068, - "hfopenllm_v2/GPQA": 0.2584, - "hfopenllm_v2/MUSR": 0.3356, - "hfopenllm_v2/MMLU-PRO": 0.3496 - } - }, - { - "id": "LimYeri/CodeMind-Llama3-8B-unsloth_v4-one-DPO-merged", - "name": "CodeMind-Llama3-8B-unsloth_v4-one-DPO-merged", - "developer": "LimYeri", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6492, - "hfopenllm_v2/BBH": 0.4853, - "hfopenllm_v2/MATH Level 5": 0.068, - "hfopenllm_v2/GPQA": 0.2685, - "hfopenllm_v2/MUSR": 0.3608, - "hfopenllm_v2/MMLU-PRO": 0.3354 - } - }, - { - "id": "LimYeri/CodeMind-Llama3-8B-unsloth_v4-one-merged", - "name": "CodeMind-Llama3-8B-unsloth_v4-one-merged", - "developer": "LimYeri", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3211, - "hfopenllm_v2/BBH": 0.4739, - "hfopenllm_v2/MATH Level 5": 0.0551, - "hfopenllm_v2/GPQA": 0.3096, - "hfopenllm_v2/MUSR": 0.4069, - "hfopenllm_v2/MMLU-PRO": 0.3353 - } - }, - { - "id": "LimYeri/CodeMind-Llama3.1-8B-unsloth-merged", - "name": "CodeMind-Llama3.1-8B-unsloth-merged", - "developer": "LimYeri", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.649, - "hfopenllm_v2/BBH": 0.4695, - "hfopenllm_v2/MATH Level 5": 0.1088, - "hfopenllm_v2/GPQA": 0.2643, - "hfopenllm_v2/MUSR": 0.3752, - "hfopenllm_v2/MMLU-PRO": 0.334 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/lkoenig.json b/data/developers/lkoenig.json deleted file mode 100644 index 6a5d46d80d0d7a00b6fa27e52582996788c88c5e..0000000000000000000000000000000000000000 --- a/data/developers/lkoenig.json +++ /dev/null @@ -1,159 +0,0 @@ -{ - "developer": "lkoenig", - "models": [ - { - "id": "lkoenig/BBAI_145_", - "name": "BBAI_145_", - "developer": "lkoenig", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.445, - "hfopenllm_v2/BBH": 0.5567, - "hfopenllm_v2/MATH Level 5": 0.361, - "hfopenllm_v2/GPQA": 0.3163, - "hfopenllm_v2/MUSR": 0.4382, - "hfopenllm_v2/MMLU-PRO": 0.449 - } - }, - { - "id": "lkoenig/BBAI_200_Gemma", - "name": "BBAI_200_Gemma", - "developer": "lkoenig", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0705, - "hfopenllm_v2/BBH": 0.3449, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2668, - "hfopenllm_v2/MUSR": 0.3631, - "hfopenllm_v2/MMLU-PRO": 0.1679 - } - }, - { - "id": "lkoenig/BBAI_212_Qwencore", - "name": "BBAI_212_Qwencore", - "developer": "lkoenig", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4384, - "hfopenllm_v2/BBH": 0.5569, - "hfopenllm_v2/MATH Level 5": 0.3489, - "hfopenllm_v2/GPQA": 0.3163, - "hfopenllm_v2/MUSR": 0.4343, - "hfopenllm_v2/MMLU-PRO": 0.449 - } - }, - { - "id": "lkoenig/BBAI_212_QwenLawLo", - "name": "BBAI_212_QwenLawLo", - "developer": "lkoenig", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4566, - "hfopenllm_v2/BBH": 0.5574, - "hfopenllm_v2/MATH Level 5": 0.3603, - "hfopenllm_v2/GPQA": 0.3163, - "hfopenllm_v2/MUSR": 0.437, - "hfopenllm_v2/MMLU-PRO": 0.4489 - } - }, - { - "id": "lkoenig/BBAI_230_Xiaqwen", - "name": "BBAI_230_Xiaqwen", - "developer": "lkoenig", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4649, - "hfopenllm_v2/BBH": 0.5578, - "hfopenllm_v2/MATH Level 5": 0.3663, - "hfopenllm_v2/GPQA": 0.3138, - "hfopenllm_v2/MUSR": 0.4422, - "hfopenllm_v2/MMLU-PRO": 0.4481 - } - }, - { - "id": "lkoenig/BBAI_375_QwenDyancabs", - "name": "BBAI_375_QwenDyancabs", - "developer": "lkoenig", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4566, - "hfopenllm_v2/BBH": 0.5571, - "hfopenllm_v2/MATH Level 5": 0.3776, - "hfopenllm_v2/GPQA": 0.3129, - "hfopenllm_v2/MUSR": 0.4462, - "hfopenllm_v2/MMLU-PRO": 0.4476 - } - }, - { - "id": "lkoenig/BBAI_456_QwenKoen", - "name": "BBAI_456_QwenKoen", - "developer": "lkoenig", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4529, - "hfopenllm_v2/BBH": 0.5553, - "hfopenllm_v2/MATH Level 5": 0.3686, - "hfopenllm_v2/GPQA": 0.3129, - "hfopenllm_v2/MUSR": 0.4395, - "hfopenllm_v2/MMLU-PRO": 0.4469 - } - }, - { - "id": "lkoenig/BBAI_7B_KoenQwenDyan", - "name": "BBAI_7B_KoenQwenDyan", - "developer": "lkoenig", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5807, - "hfopenllm_v2/BBH": 0.5537, - "hfopenllm_v2/MATH Level 5": 0.3739, - "hfopenllm_v2/GPQA": 0.318, - "hfopenllm_v2/MUSR": 0.4369, - "hfopenllm_v2/MMLU-PRO": 0.446 - } - }, - { - "id": "lkoenig/BBAI_7B_Qwen2.5koen", - "name": "BBAI_7B_Qwen2.5koen", - "developer": "lkoenig", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.46, - "hfopenllm_v2/BBH": 0.5544, - "hfopenllm_v2/MATH Level 5": 0.3656, - "hfopenllm_v2/GPQA": 0.3129, - "hfopenllm_v2/MUSR": 0.4369, - "hfopenllm_v2/MMLU-PRO": 0.4485 - } - }, - { - "id": "lkoenig/BBAI_7B_QwenDyancabsLAW", - "name": "BBAI_7B_QwenDyancabsLAW", - "developer": "lkoenig", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.555, - "hfopenllm_v2/BBH": 0.5579, - "hfopenllm_v2/MATH Level 5": 0.3678, - "hfopenllm_v2/GPQA": 0.3188, - "hfopenllm_v2/MUSR": 0.4461, - "hfopenllm_v2/MMLU-PRO": 0.4471 - } - }, - { - "id": "lkoenig/BBAI_7B_QwenDyanKoenLo", - "name": "BBAI_7B_QwenDyanKoenLo", - "developer": "lkoenig", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4663, - "hfopenllm_v2/BBH": 0.5562, - "hfopenllm_v2/MATH Level 5": 0.364, - "hfopenllm_v2/GPQA": 0.3188, - "hfopenllm_v2/MUSR": 0.4343, - "hfopenllm_v2/MMLU-PRO": 0.4465 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/llm-blender.json b/data/developers/llm-blender.json deleted file mode 100644 index 20669383aaa6959bf51fde926169f3d0693eaf73..0000000000000000000000000000000000000000 --- a/data/developers/llm-blender.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "llm-blender", - "models": [ - { - "id": "llm-blender/PairRM-hf", - "name": "llm-blender/PairRM-hf", - "developer": "llm-blender", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6087, - "reward-bench/Chat": 0.9022, - "reward-bench/Chat Hard": 0.5219, - "reward-bench/Safety": 0.477, - "reward-bench/Reasoning": 0.4898, - "reward-bench/Prior Sets (0.5 weight)": 0.6961 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/llm360.json b/data/developers/llm360.json deleted file mode 100644 index 43e4b7d4754cc4528b7ea939ba53182d190b2ac5..0000000000000000000000000000000000000000 --- a/data/developers/llm360.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "LLM360", - "models": [ - { - "id": "LLM360/K2", - "name": "K2", - "developer": "LLM360", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2252, - "hfopenllm_v2/BBH": 0.4972, - "hfopenllm_v2/MATH Level 5": 0.0272, - "hfopenllm_v2/GPQA": 0.2768, - "hfopenllm_v2/MUSR": 0.398, - "hfopenllm_v2/MMLU-PRO": 0.3004 - } - }, - { - "id": "LLM360/K2-Chat", - "name": "K2-Chat", - "developer": "LLM360", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5152, - "hfopenllm_v2/BBH": 0.5358, - "hfopenllm_v2/MATH Level 5": 0.1035, - "hfopenllm_v2/GPQA": 0.3062, - "hfopenllm_v2/MUSR": 0.457, - "hfopenllm_v2/MMLU-PRO": 0.3371 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/llm4binary.json b/data/developers/llm4binary.json deleted file mode 100644 index e913f674439bcd63b70d593c0666708573616d83..0000000000000000000000000000000000000000 --- a/data/developers/llm4binary.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "LLM4Binary", - "models": [ - { - "id": "LLM4Binary/llm4decompile-1.3b-v2", - "name": "llm4decompile-1.3b-v2", - "developer": "LLM4Binary", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2268, - "hfopenllm_v2/BBH": 0.3272, - "hfopenllm_v2/MATH Level 5": 0.0128, - "hfopenllm_v2/GPQA": 0.2357, - "hfopenllm_v2/MUSR": 0.4072, - "hfopenllm_v2/MMLU-PRO": 0.1209 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/llmat.json b/data/developers/llmat.json deleted file mode 100644 index d073eb81547c22e04e5363702192b3a9654d7362..0000000000000000000000000000000000000000 --- a/data/developers/llmat.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "llmat", - "models": [ - { - "id": "llmat/Mistral-v0.3-7B-ORPO", - "name": "Mistral-v0.3-7B-ORPO", - "developer": "llmat", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.377, - "hfopenllm_v2/BBH": 0.3978, - "hfopenllm_v2/MATH Level 5": 0.0242, - "hfopenllm_v2/GPQA": 0.2668, - "hfopenllm_v2/MUSR": 0.3555, - "hfopenllm_v2/MMLU-PRO": 0.2278 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/llnyou.json b/data/developers/llnyou.json deleted file mode 100644 index 78c52c5cd89fc6383489e38a5b917a1b06465248..0000000000000000000000000000000000000000 --- a/data/developers/llnyou.json +++ /dev/null @@ -1,75 +0,0 @@ -{ - "developer": "llnYou", - "models": [ - { - "id": "llnYou/ECE-PRYMMAL-YL-1B-SLERP-V5", - "name": "ECE-PRYMMAL-YL-1B-SLERP-V5", - "developer": "llnYou", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3313, - "hfopenllm_v2/BBH": 0.4233, - "hfopenllm_v2/MATH Level 5": 0.111, - "hfopenllm_v2/GPQA": 0.2861, - "hfopenllm_v2/MUSR": 0.3868, - "hfopenllm_v2/MMLU-PRO": 0.2931 - } - }, - { - "id": "llnYou/ECE-PRYMMAL-YL-1B-SLERP-V6", - "name": "ECE-PRYMMAL-YL-1B-SLERP-V6", - "developer": "llnYou", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1388, - "hfopenllm_v2/BBH": 0.3944, - "hfopenllm_v2/MATH Level 5": 0.0023, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.3928, - "hfopenllm_v2/MMLU-PRO": 0.235 - } - }, - { - "id": "llnYou/ECE-PRYMMAL-YL-3B-SLERP-V1", - "name": "ECE-PRYMMAL-YL-3B-SLERP-V1", - "developer": "llnYou", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2346, - "hfopenllm_v2/BBH": 0.4018, - "hfopenllm_v2/MATH Level 5": 0.0091, - "hfopenllm_v2/GPQA": 0.2936, - "hfopenllm_v2/MUSR": 0.3364, - "hfopenllm_v2/MMLU-PRO": 0.285 - } - }, - { - "id": "llnYou/ECE-PRYMMAL-YL-3B-SLERP-V2", - "name": "ECE-PRYMMAL-YL-3B-SLERP-V2", - "developer": "llnYou", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2309, - "hfopenllm_v2/BBH": 0.399, - "hfopenllm_v2/MATH Level 5": 0.0128, - "hfopenllm_v2/GPQA": 0.2768, - "hfopenllm_v2/MUSR": 0.3588, - "hfopenllm_v2/MMLU-PRO": 0.29 - } - }, - { - "id": "llnYou/ECE-PRYMMAL-YL-3B-SLERP-V3", - "name": "ECE-PRYMMAL-YL-3B-SLERP-V3", - "developer": "llnYou", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3581, - "hfopenllm_v2/BBH": 0.5473, - "hfopenllm_v2/MATH Level 5": 0.1299, - "hfopenllm_v2/GPQA": 0.3045, - "hfopenllm_v2/MUSR": 0.4361, - "hfopenllm_v2/MMLU-PRO": 0.4043 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/lmsys.json b/data/developers/lmsys.json deleted file mode 100644 index 95664923653f5e2699a939e18e50607290f653d0..0000000000000000000000000000000000000000 --- a/data/developers/lmsys.json +++ /dev/null @@ -1,93 +0,0 @@ -{ - "developer": "lmsys", - "models": [ - { - "id": "lmsys/vicuna-13b-v1.3", - "name": "vicuna-13b-v1.3", - "developer": "lmsys", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3344, - "hfopenllm_v2/BBH": 0.3384, - "hfopenllm_v2/MATH Level 5": 0.0144, - "hfopenllm_v2/GPQA": 0.2676, - "hfopenllm_v2/MUSR": 0.3727, - "hfopenllm_v2/MMLU-PRO": 0.2243 - } - }, - { - "id": "lmsys/vicuna-7b-v1.3", - "name": "vicuna-7b-v1.3", - "developer": "lmsys", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2909, - "hfopenllm_v2/BBH": 0.3298, - "hfopenllm_v2/MATH Level 5": 0.0128, - "hfopenllm_v2/GPQA": 0.2424, - "hfopenllm_v2/MUSR": 0.3793, - "hfopenllm_v2/MMLU-PRO": 0.1838 - } - }, - { - "id": "lmsys/vicuna-7b-v1.5", - "name": "vicuna-7b-v1.5", - "developer": "lmsys", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2352, - "hfopenllm_v2/BBH": 0.3947, - "hfopenllm_v2/MATH Level 5": 0.0136, - "hfopenllm_v2/GPQA": 0.2584, - "hfopenllm_v2/MUSR": 0.4231, - "hfopenllm_v2/MMLU-PRO": 0.2147 - } - }, - { - "id": "lmsys/Vicuna-v1.3-13B", - "name": "Vicuna v1.3 13B", - "developer": "lmsys", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.706, - "helm_classic/MMLU": 0.462, - "helm_classic/BoolQ": 0.808, - "helm_classic/NarrativeQA": 0.691, - "helm_classic/NaturalQuestions (open-book)": 0.686, - "helm_classic/QuAC": 0.403, - "helm_classic/HellaSwag": -1.0, - "helm_classic/OpenbookQA": -1.0, - "helm_classic/TruthfulQA": 0.385, - "helm_classic/MS MARCO (TREC)": -1.0, - "helm_classic/CNN/DailyMail": -1.0, - "helm_classic/XSUM": -1.0, - "helm_classic/IMDB": 0.762, - "helm_classic/CivilComments": 0.645, - "helm_classic/RAFT": 0.657 - } - }, - { - "id": "lmsys/Vicuna-v1.3-7B", - "name": "Vicuna v1.3 7B", - "developer": "lmsys", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.625, - "helm_classic/MMLU": 0.434, - "helm_classic/BoolQ": 0.76, - "helm_classic/NarrativeQA": 0.643, - "helm_classic/NaturalQuestions (open-book)": 0.634, - "helm_classic/QuAC": 0.392, - "helm_classic/HellaSwag": -1.0, - "helm_classic/OpenbookQA": -1.0, - "helm_classic/TruthfulQA": 0.292, - "helm_classic/MS MARCO (TREC)": -1.0, - "helm_classic/CNN/DailyMail": -1.0, - "helm_classic/XSUM": -1.0, - "helm_classic/IMDB": 0.916, - "helm_classic/CivilComments": 0.62, - "helm_classic/RAFT": 0.693 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/locutusque.json b/data/developers/locutusque.json deleted file mode 100644 index 79fc85cde4cbae34291a4bb6ed193ab4ab0df365..0000000000000000000000000000000000000000 --- a/data/developers/locutusque.json +++ /dev/null @@ -1,89 +0,0 @@ -{ - "developer": "Locutusque", - "models": [ - { - "id": "Locutusque/CollectiveLM-Falcon-3-7B", - "name": "CollectiveLM-Falcon-3-7B", - "developer": "Locutusque", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3918, - "hfopenllm_v2/BBH": 0.5105, - "hfopenllm_v2/MATH Level 5": 0.2183, - "hfopenllm_v2/GPQA": 0.3255, - "hfopenllm_v2/MUSR": 0.3887, - "hfopenllm_v2/MMLU-PRO": 0.3599 - } - }, - { - "id": "Locutusque/Hercules-6.0-Llama-3.1-8B", - "name": "Hercules-6.0-Llama-3.1-8B", - "developer": "Locutusque", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.663, - "hfopenllm_v2/BBH": 0.4813, - "hfopenllm_v2/MATH Level 5": 0.1669, - "hfopenllm_v2/GPQA": 0.2643, - "hfopenllm_v2/MUSR": 0.3621, - "hfopenllm_v2/MMLU-PRO": 0.3615 - } - }, - { - "id": "Locutusque/Hercules-6.1-Llama-3.1-8B", - "name": "Hercules-6.1-Llama-3.1-8B", - "developer": "Locutusque", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6007, - "hfopenllm_v2/BBH": 0.4656, - "hfopenllm_v2/MATH Level 5": 0.176, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.3553, - "hfopenllm_v2/MMLU-PRO": 0.3669 - } - }, - { - "id": "Locutusque/Llama-3-NeuralHercules-5.0-8B", - "name": "Llama-3-NeuralHercules-5.0-8B", - "developer": "Locutusque", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4489, - "hfopenllm_v2/BBH": 0.394, - "hfopenllm_v2/MATH Level 5": 0.0431, - "hfopenllm_v2/GPQA": 0.2685, - "hfopenllm_v2/MUSR": 0.3881, - "hfopenllm_v2/MMLU-PRO": 0.2933 - } - }, - { - "id": "Locutusque/Llama-3-Yggdrasil-2.0-8B", - "name": "Llama-3-Yggdrasil-2.0-8B", - "developer": "Locutusque", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5371, - "hfopenllm_v2/BBH": 0.4772, - "hfopenllm_v2/MATH Level 5": 0.0831, - "hfopenllm_v2/GPQA": 0.2626, - "hfopenllm_v2/MUSR": 0.3977, - "hfopenllm_v2/MMLU-PRO": 0.3167 - } - }, - { - "id": "Locutusque/TinyMistral-248M-v2.5", - "name": "TinyMistral-248M-v2.5", - "developer": "Locutusque", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1336, - "hfopenllm_v2/BBH": 0.3039, - "hfopenllm_v2/MATH Level 5": 0.0098, - "hfopenllm_v2/GPQA": 0.2508, - "hfopenllm_v2/MUSR": 0.3782, - "hfopenllm_v2/MMLU-PRO": 0.1135 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/lodrick-the-lafted.json b/data/developers/lodrick-the-lafted.json deleted file mode 100644 index a81c387a5f606d1877cf135dda080cad901adb49..0000000000000000000000000000000000000000 --- a/data/developers/lodrick-the-lafted.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "lodrick-the-lafted", - "models": [ - { - "id": "lodrick-the-lafted/llama-3.1-8b-instruct-ortho-v7", - "name": "llama-3.1-8b-instruct-ortho-v7", - "developer": "lodrick-the-lafted", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3515, - "hfopenllm_v2/BBH": 0.3907, - "hfopenllm_v2/MATH Level 5": 0.0272, - "hfopenllm_v2/GPQA": 0.2727, - "hfopenllm_v2/MUSR": 0.3616, - "hfopenllm_v2/MMLU-PRO": 0.1974 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/lordjia.json b/data/developers/lordjia.json deleted file mode 100644 index f199a770c703a5000ae4040a1ad666f3c4cf3c52..0000000000000000000000000000000000000000 --- a/data/developers/lordjia.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "lordjia", - "models": [ - { - "id": "lordjia/Llama-3-Cantonese-8B-Instruct", - "name": "Llama-3-Cantonese-8B-Instruct", - "developer": "lordjia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6669, - "hfopenllm_v2/BBH": 0.4814, - "hfopenllm_v2/MATH Level 5": 0.0891, - "hfopenllm_v2/GPQA": 0.2936, - "hfopenllm_v2/MUSR": 0.4046, - "hfopenllm_v2/MMLU-PRO": 0.3515 - } - }, - { - "id": "lordjia/Qwen2-Cantonese-7B-Instruct", - "name": "Qwen2-Cantonese-7B-Instruct", - "developer": "lordjia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5435, - "hfopenllm_v2/BBH": 0.5215, - "hfopenllm_v2/MATH Level 5": 0.256, - "hfopenllm_v2/GPQA": 0.2953, - "hfopenllm_v2/MUSR": 0.4004, - "hfopenllm_v2/MMLU-PRO": 0.3843 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/lt-asset.json b/data/developers/lt-asset.json deleted file mode 100644 index 0bbb1e10736981f0b0824b3c6f95c492cc8700c4..0000000000000000000000000000000000000000 --- a/data/developers/lt-asset.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "lt-asset", - "models": [ - { - "id": "lt-asset/nova-1.3b", - "name": "nova-1.3b", - "developer": "lt-asset", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1214, - "hfopenllm_v2/BBH": 0.317, - "hfopenllm_v2/MATH Level 5": 0.0121, - "hfopenllm_v2/GPQA": 0.2492, - "hfopenllm_v2/MUSR": 0.3698, - "hfopenllm_v2/MMLU-PRO": 0.1142 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/lunahr.json b/data/developers/lunahr.json deleted file mode 100644 index 77a095e0a71c400736fd57c940aee25f3924e8c7..0000000000000000000000000000000000000000 --- a/data/developers/lunahr.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "lunahr", - "models": [ - { - "id": "lunahr/thea-3b-50r-u1", - "name": "thea-3b-50r-u1", - "developer": "lunahr", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.603, - "hfopenllm_v2/BBH": 0.4105, - "hfopenllm_v2/MATH Level 5": 0.1042, - "hfopenllm_v2/GPQA": 0.2836, - "hfopenllm_v2/MUSR": 0.3182, - "hfopenllm_v2/MMLU-PRO": 0.2808 - } - }, - { - "id": "lunahr/thea-v2-3b-50r", - "name": "thea-v2-3b-50r", - "developer": "lunahr", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3704, - "hfopenllm_v2/BBH": 0.4194, - "hfopenllm_v2/MATH Level 5": 0.0242, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.3222, - "hfopenllm_v2/MMLU-PRO": 0.2409 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/luni.json b/data/developers/luni.json deleted file mode 100644 index 5949c4e715c588b20c3086e11472f22b6fe16386..0000000000000000000000000000000000000000 --- a/data/developers/luni.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "Luni", - "models": [ - { - "id": "Luni/StarDust-12b-v1", - "name": "StarDust-12b-v1", - "developer": "Luni", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5459, - "hfopenllm_v2/BBH": 0.5366, - "hfopenllm_v2/MATH Level 5": 0.0763, - "hfopenllm_v2/GPQA": 0.276, - "hfopenllm_v2/MUSR": 0.4324, - "hfopenllm_v2/MMLU-PRO": 0.3412 - } - }, - { - "id": "Luni/StarDust-12b-v2", - "name": "StarDust-12b-v2", - "developer": "Luni", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5629, - "hfopenllm_v2/BBH": 0.5419, - "hfopenllm_v2/MATH Level 5": 0.0687, - "hfopenllm_v2/GPQA": 0.2936, - "hfopenllm_v2/MUSR": 0.4338, - "hfopenllm_v2/MMLU-PRO": 0.3439 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/lunzima.json b/data/developers/lunzima.json deleted file mode 100644 index a4fac6019f60ace84222176cca9db79aa521373e..0000000000000000000000000000000000000000 --- a/data/developers/lunzima.json +++ /dev/null @@ -1,257 +0,0 @@ -{ - "developer": "Lunzima", - "models": [ - { - "id": "Lunzima/NQLSG-Qwen2.5-14B-MegaFusion-v3", - "name": "NQLSG-Qwen2.5-14B-MegaFusion-v3", - "developer": "Lunzima", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7049, - "hfopenllm_v2/BBH": 0.6478, - "hfopenllm_v2/MATH Level 5": 0.4162, - "hfopenllm_v2/GPQA": 0.3817, - "hfopenllm_v2/MUSR": 0.4808, - "hfopenllm_v2/MMLU-PRO": 0.5394 - } - }, - { - "id": "Lunzima/NQLSG-Qwen2.5-14B-MegaFusion-v4", - "name": "NQLSG-Qwen2.5-14B-MegaFusion-v4", - "developer": "Lunzima", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6943, - "hfopenllm_v2/BBH": 0.642, - "hfopenllm_v2/MATH Level 5": 0.3467, - "hfopenllm_v2/GPQA": 0.3716, - "hfopenllm_v2/MUSR": 0.4769, - "hfopenllm_v2/MMLU-PRO": 0.5252 - } - }, - { - "id": "Lunzima/NQLSG-Qwen2.5-14B-MegaFusion-v5", - "name": "NQLSG-Qwen2.5-14B-MegaFusion-v5", - "developer": "Lunzima", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7485, - "hfopenllm_v2/BBH": 0.6467, - "hfopenllm_v2/MATH Level 5": 0.4358, - "hfopenllm_v2/GPQA": 0.3624, - "hfopenllm_v2/MUSR": 0.4473, - "hfopenllm_v2/MMLU-PRO": 0.514 - } - }, - { - "id": "Lunzima/NQLSG-Qwen2.5-14B-MegaFusion-v6", - "name": "NQLSG-Qwen2.5-14B-MegaFusion-v6", - "developer": "Lunzima", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7043, - "hfopenllm_v2/BBH": 0.6458, - "hfopenllm_v2/MATH Level 5": 0.3958, - "hfopenllm_v2/GPQA": 0.3775, - "hfopenllm_v2/MUSR": 0.4768, - "hfopenllm_v2/MMLU-PRO": 0.5392 - } - }, - { - "id": "Lunzima/NQLSG-Qwen2.5-14B-MegaFusion-v6-cpt", - "name": "NQLSG-Qwen2.5-14B-MegaFusion-v6-cpt", - "developer": "Lunzima", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4663, - "hfopenllm_v2/BBH": 0.6215, - "hfopenllm_v2/MATH Level 5": 0.3316, - "hfopenllm_v2/GPQA": 0.3758, - "hfopenllm_v2/MUSR": 0.4937, - "hfopenllm_v2/MMLU-PRO": 0.5204 - } - }, - { - "id": "Lunzima/NQLSG-Qwen2.5-14B-MegaFusion-v7", - "name": "NQLSG-Qwen2.5-14B-MegaFusion-v7", - "developer": "Lunzima", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6794, - "hfopenllm_v2/BBH": 0.6531, - "hfopenllm_v2/MATH Level 5": 0.4101, - "hfopenllm_v2/GPQA": 0.3792, - "hfopenllm_v2/MUSR": 0.4834, - "hfopenllm_v2/MMLU-PRO": 0.5376 - } - }, - { - "id": "Lunzima/NQLSG-Qwen2.5-14B-MegaFusion-v7-rebase", - "name": "NQLSG-Qwen2.5-14B-MegaFusion-v7-rebase", - "developer": "Lunzima", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6931, - "hfopenllm_v2/BBH": 0.6423, - "hfopenllm_v2/MATH Level 5": 0.3406, - "hfopenllm_v2/GPQA": 0.375, - "hfopenllm_v2/MUSR": 0.4888, - "hfopenllm_v2/MMLU-PRO": 0.5277 - } - }, - { - "id": "Lunzima/NQLSG-Qwen2.5-14B-MegaFusion-v8", - "name": "NQLSG-Qwen2.5-14B-MegaFusion-v8", - "developer": "Lunzima", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7875, - "hfopenllm_v2/BBH": 0.6419, - "hfopenllm_v2/MATH Level 5": 0.5559, - "hfopenllm_v2/GPQA": 0.3356, - "hfopenllm_v2/MUSR": 0.4394, - "hfopenllm_v2/MMLU-PRO": 0.5206 - } - }, - { - "id": "Lunzima/NQLSG-Qwen2.5-14B-MegaFusion-v8.5", - "name": "NQLSG-Qwen2.5-14B-MegaFusion-v8.5", - "developer": "Lunzima", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5929, - "hfopenllm_v2/BBH": 0.6451, - "hfopenllm_v2/MATH Level 5": 0.3656, - "hfopenllm_v2/GPQA": 0.38, - "hfopenllm_v2/MUSR": 0.477, - "hfopenllm_v2/MMLU-PRO": 0.529 - } - }, - { - "id": "Lunzima/NQLSG-Qwen2.5-14B-MegaFusion-v8.6", - "name": "NQLSG-Qwen2.5-14B-MegaFusion-v8.6", - "developer": "Lunzima", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5919, - "hfopenllm_v2/BBH": 0.6457, - "hfopenllm_v2/MATH Level 5": 0.4071, - "hfopenllm_v2/GPQA": 0.3842, - "hfopenllm_v2/MUSR": 0.4953, - "hfopenllm_v2/MMLU-PRO": 0.54 - } - }, - { - "id": "Lunzima/NQLSG-Qwen2.5-14B-MegaFusion-v8.7", - "name": "NQLSG-Qwen2.5-14B-MegaFusion-v8.7", - "developer": "Lunzima", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7875, - "hfopenllm_v2/BBH": 0.6483, - "hfopenllm_v2/MATH Level 5": 0.5408, - "hfopenllm_v2/GPQA": 0.3515, - "hfopenllm_v2/MUSR": 0.4381, - "hfopenllm_v2/MMLU-PRO": 0.5242 - } - }, - { - "id": "Lunzima/NQLSG-Qwen2.5-14B-MegaFusion-v8.8", - "name": "NQLSG-Qwen2.5-14B-MegaFusion-v8.8", - "developer": "Lunzima", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7028, - "hfopenllm_v2/BBH": 0.6566, - "hfopenllm_v2/MATH Level 5": 0.4237, - "hfopenllm_v2/GPQA": 0.3758, - "hfopenllm_v2/MUSR": 0.4912, - "hfopenllm_v2/MMLU-PRO": 0.5323 - } - }, - { - "id": "Lunzima/NQLSG-Qwen2.5-14B-MegaFusion-v8.9", - "name": "NQLSG-Qwen2.5-14B-MegaFusion-v8.9", - "developer": "Lunzima", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7993, - "hfopenllm_v2/BBH": 0.6483, - "hfopenllm_v2/MATH Level 5": 0.537, - "hfopenllm_v2/GPQA": 0.3297, - "hfopenllm_v2/MUSR": 0.4328, - "hfopenllm_v2/MMLU-PRO": 0.5199 - } - }, - { - "id": "Lunzima/NQLSG-Qwen2.5-14B-MegaFusion-v9", - "name": "NQLSG-Qwen2.5-14B-MegaFusion-v9", - "developer": "Lunzima", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5235, - "hfopenllm_v2/BBH": 0.6546, - "hfopenllm_v2/MATH Level 5": 0.4366, - "hfopenllm_v2/GPQA": 0.3884, - "hfopenllm_v2/MUSR": 0.4806, - "hfopenllm_v2/MMLU-PRO": 0.5422 - } - }, - { - "id": "Lunzima/NQLSG-Qwen2.5-14B-MegaFusion-v9-stock", - "name": "NQLSG-Qwen2.5-14B-MegaFusion-v9-stock", - "developer": "Lunzima", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6514, - "hfopenllm_v2/BBH": 0.6571, - "hfopenllm_v2/MATH Level 5": 0.4184, - "hfopenllm_v2/GPQA": 0.3842, - "hfopenllm_v2/MUSR": 0.482, - "hfopenllm_v2/MMLU-PRO": 0.5412 - } - }, - { - "id": "Lunzima/NQLSG-Qwen2.5-14B-MegaFusion-v9.1", - "name": "NQLSG-Qwen2.5-14B-MegaFusion-v9.1", - "developer": "Lunzima", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8003, - "hfopenllm_v2/BBH": 0.6555, - "hfopenllm_v2/MATH Level 5": 0.5468, - "hfopenllm_v2/GPQA": 0.3431, - "hfopenllm_v2/MUSR": 0.4354, - "hfopenllm_v2/MMLU-PRO": 0.5251 - } - }, - { - "id": "Lunzima/NQLSG-Qwen2.5-14B-MegaFusion-v9.2", - "name": "NQLSG-Qwen2.5-14B-MegaFusion-v9.2", - "developer": "Lunzima", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7862, - "hfopenllm_v2/BBH": 0.6538, - "hfopenllm_v2/MATH Level 5": 0.5332, - "hfopenllm_v2/GPQA": 0.3557, - "hfopenllm_v2/MUSR": 0.4381, - "hfopenllm_v2/MMLU-PRO": 0.5283 - } - }, - { - "id": "Lunzima/NQLSG-Qwen2.5-14B-OriginalFusion", - "name": "NQLSG-Qwen2.5-14B-OriginalFusion", - "developer": "Lunzima", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6142, - "hfopenllm_v2/BBH": 0.6592, - "hfopenllm_v2/MATH Level 5": 0.4275, - "hfopenllm_v2/GPQA": 0.3809, - "hfopenllm_v2/MUSR": 0.5122, - "hfopenllm_v2/MMLU-PRO": 0.5239 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/lxzgordon.json b/data/developers/lxzgordon.json deleted file mode 100644 index 7f802cf733857054e01537f3ecf745a3fdb38a05..0000000000000000000000000000000000000000 --- a/data/developers/lxzgordon.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - "developer": "LxzGordon", - "models": [ - { - "id": "LxzGordon/URM-LLaMa-3-8B", - "name": "LxzGordon/URM-LLaMa-3-8B", - "developer": "LxzGordon", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.8991, - "reward-bench/Chat": 0.9693, - "reward-bench/Chat Hard": 0.7873, - "reward-bench/Safety": 0.8824, - "reward-bench/Reasoning": 0.9574 - } - }, - { - "id": "LxzGordon/URM-LLaMa-3.1-8B", - "name": "LxzGordon/URM-LLaMa-3.1-8B", - "developer": "LxzGordon", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7394, - "reward-bench/Chat": 0.9553, - "reward-bench/Chat Hard": 0.8816, - "reward-bench/Safety": 0.9178, - "reward-bench/Reasoning": 0.9698, - "reward-bench/Factuality": 0.6884, - "reward-bench/Precise IF": 0.45, - "reward-bench/Math": 0.6393, - "reward-bench/Focus": 0.9758, - "reward-bench/Ties": 0.7653 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/lyte.json b/data/developers/lyte.json deleted file mode 100644 index e9dc729c1cc008adcb228cadfbe35ded3607f85f..0000000000000000000000000000000000000000 --- a/data/developers/lyte.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "developer": "Lyte", - "models": [ - { - "id": "Lyte/Llama-3.1-8B-Instruct-Reasoner-1o1_v0.3", - "name": "Llama-3.1-8B-Instruct-Reasoner-1o1_v0.3", - "developer": "Lyte", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7098, - "hfopenllm_v2/BBH": 0.495, - "hfopenllm_v2/MATH Level 5": 0.1903, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.3461, - "hfopenllm_v2/MMLU-PRO": 0.3618 - } - }, - { - "id": "Lyte/Llama-3.2-1B-Instruct-COT-RL-Expriement1-EP04", - "name": "Llama-3.2-1B-Instruct-COT-RL-Expriement1-EP04", - "developer": "Lyte", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5774, - "hfopenllm_v2/BBH": 0.3515, - "hfopenllm_v2/MATH Level 5": 0.0801, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3236, - "hfopenllm_v2/MMLU-PRO": 0.1843 - } - }, - { - "id": "Lyte/Llama-3.2-3B-Overthinker", - "name": "Llama-3.2-3B-Overthinker", - "developer": "Lyte", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6408, - "hfopenllm_v2/BBH": 0.432, - "hfopenllm_v2/MATH Level 5": 0.1563, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3419, - "hfopenllm_v2/MMLU-PRO": 0.2985 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/m4-ai.json b/data/developers/m4-ai.json deleted file mode 100644 index 09abbc2754581708d5cc628249887222f388d0f0..0000000000000000000000000000000000000000 --- a/data/developers/m4-ai.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "M4-ai", - "models": [ - { - "id": "M4-ai/TinyMistral-248M-v3", - "name": "TinyMistral-248M-v3", - "developer": "M4-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1639, - "hfopenllm_v2/BBH": 0.2885, - "hfopenllm_v2/MATH Level 5": 0.0045, - "hfopenllm_v2/GPQA": 0.2408, - "hfopenllm_v2/MUSR": 0.3793, - "hfopenllm_v2/MMLU-PRO": 0.1132 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/m42-health.json b/data/developers/m42-health.json deleted file mode 100644 index acda9f22f160f934ca72856d70298d1e988d23a6..0000000000000000000000000000000000000000 --- a/data/developers/m42-health.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "m42-health", - "models": [ - { - "id": "m42-health/Llama3-Med42-70B", - "name": "Llama3-Med42-70B", - "developer": "m42-health", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6291, - "hfopenllm_v2/BBH": 0.6688, - "hfopenllm_v2/MATH Level 5": 0.2258, - "hfopenllm_v2/GPQA": 0.3473, - "hfopenllm_v2/MUSR": 0.4629, - "hfopenllm_v2/MMLU-PRO": 0.4963 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/macadeliccc.json b/data/developers/macadeliccc.json deleted file mode 100644 index 8cfc7e6492542049620ef42f4fa4b058b4648da6..0000000000000000000000000000000000000000 --- a/data/developers/macadeliccc.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "developer": "macadeliccc", - "models": [ - { - "id": "macadeliccc/magistrate-3.2-3b-base", - "name": "magistrate-3.2-3b-base", - "developer": "macadeliccc", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1159, - "hfopenllm_v2/BBH": 0.3343, - "hfopenllm_v2/MATH Level 5": 0.0113, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.3976, - "hfopenllm_v2/MMLU-PRO": 0.1689 - } - }, - { - "id": "macadeliccc/magistrate-3.2-3b-it", - "name": "magistrate-3.2-3b-it", - "developer": "macadeliccc", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2292, - "hfopenllm_v2/BBH": 0.3257, - "hfopenllm_v2/MATH Level 5": 0.0196, - "hfopenllm_v2/GPQA": 0.2475, - "hfopenllm_v2/MUSR": 0.3763, - "hfopenllm_v2/MMLU-PRO": 0.1592 - } - }, - { - "id": "macadeliccc/Samantha-Qwen-2-7B", - "name": "Samantha-Qwen-2-7B", - "developer": "macadeliccc", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4377, - "hfopenllm_v2/BBH": 0.5082, - "hfopenllm_v2/MATH Level 5": 0.2115, - "hfopenllm_v2/GPQA": 0.2727, - "hfopenllm_v2/MUSR": 0.4799, - "hfopenllm_v2/MMLU-PRO": 0.3779 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/madeagents.json b/data/developers/madeagents.json deleted file mode 100644 index 137669493f3a9f385404927d39d76e96aa0783c2..0000000000000000000000000000000000000000 --- a/data/developers/madeagents.json +++ /dev/null @@ -1,157 +0,0 @@ -{ - "developer": "madeagents", - "models": [ - { - "id": "madeagents/hammer2-1-0-5b-fc", - "name": "Hammer2.1-0.5b (FC)", - "developer": "madeagents", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 100.0, - "bfcl/bfcl.overall.overall_accuracy": 21.22, - "bfcl/bfcl.overall.total_cost_usd": 2.82, - "bfcl/bfcl.overall.latency_mean_s": 2.79, - "bfcl/bfcl.overall.latency_std_s": 3.17, - "bfcl/bfcl.overall.latency_p95_s": 9.86, - "bfcl/bfcl.non_live.ast_accuracy": 65.98, - "bfcl/bfcl.non_live.simple_ast_accuracy": 62.42, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 81.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 69.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 51.5, - "bfcl/bfcl.live.live_accuracy": 54.63, - "bfcl/bfcl.live.live_simple_ast_accuracy": 56.59, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 54.42, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 62.5, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 37.5, - "bfcl/bfcl.multi_turn.accuracy": 2.88, - "bfcl/bfcl.multi_turn.base_accuracy": 4.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 0.5, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 4.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 2.5, - "bfcl/bfcl.web_search.accuracy": 0.0, - "bfcl/bfcl.web_search.base_accuracy": 0.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 0.0, - "bfcl/bfcl.memory.accuracy": 1.08, - "bfcl/bfcl.memory.kv_accuracy": 0.65, - "bfcl/bfcl.memory.vector_accuracy": 1.94, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 0.65, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 68.75, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 80.79 - } - }, - { - "id": "madeagents/hammer2-1-1-5b-fc", - "name": "Hammer2.1-1.5b (FC)", - "developer": "madeagents", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 75.0, - "bfcl/bfcl.overall.overall_accuracy": 27.88, - "bfcl/bfcl.overall.total_cost_usd": 6.83, - "bfcl/bfcl.overall.latency_mean_s": 6.28, - "bfcl/bfcl.overall.latency_std_s": 8.79, - "bfcl/bfcl.overall.latency_p95_s": 30.72, - "bfcl/bfcl.non_live.ast_accuracy": 82.98, - "bfcl/bfcl.non_live.simple_ast_accuracy": 73.92, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 92.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 85.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 80.5, - "bfcl/bfcl.live.live_accuracy": 69.5, - "bfcl/bfcl.live.live_simple_ast_accuracy": 72.09, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 69.33, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 50.0, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 62.5, - "bfcl/bfcl.multi_turn.accuracy": 15.62, - "bfcl/bfcl.multi_turn.base_accuracy": 20.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 16.5, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 9.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 16.5, - "bfcl/bfcl.web_search.accuracy": 0.0, - "bfcl/bfcl.web_search.base_accuracy": 0.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 0.0, - "bfcl/bfcl.memory.accuracy": 0.0, - "bfcl/bfcl.memory.kv_accuracy": 0.0, - "bfcl/bfcl.memory.vector_accuracy": 0.0, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 0.0, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 75.0, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 79.4 - } - }, - { - "id": "madeagents/hammer2-1-3b-fc", - "name": "Hammer2.1-3b (FC)", - "developer": "madeagents", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 68.0, - "bfcl/bfcl.overall.overall_accuracy": 29.71, - "bfcl/bfcl.overall.total_cost_usd": 10.89, - "bfcl/bfcl.overall.latency_mean_s": 11.24, - "bfcl/bfcl.overall.latency_std_s": 15.81, - "bfcl/bfcl.overall.latency_p95_s": 47.44, - "bfcl/bfcl.non_live.ast_accuracy": 84.96, - "bfcl/bfcl.non_live.simple_ast_accuracy": 79.33, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 93.5, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 86.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 80.5, - "bfcl/bfcl.live.live_accuracy": 70.54, - "bfcl/bfcl.live.live_simple_ast_accuracy": 68.22, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 71.32, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 62.5, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 66.67, - "bfcl/bfcl.multi_turn.accuracy": 16.5, - "bfcl/bfcl.multi_turn.base_accuracy": 22.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 12.5, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 16.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 15.5, - "bfcl/bfcl.web_search.accuracy": 0.0, - "bfcl/bfcl.web_search.base_accuracy": 0.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 0.0, - "bfcl/bfcl.memory.accuracy": 3.01, - "bfcl/bfcl.memory.kv_accuracy": 2.58, - "bfcl/bfcl.memory.vector_accuracy": 3.87, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 2.58, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 56.25, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 86.12 - } - }, - { - "id": "madeagents/hammer2-1-7b-fc", - "name": "Hammer2.1-7b (FC)", - "developer": "madeagents", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 64.0, - "bfcl/bfcl.overall.overall_accuracy": 31.67, - "bfcl/bfcl.overall.total_cost_usd": 4.99, - "bfcl/bfcl.overall.latency_mean_s": 5.77, - "bfcl/bfcl.overall.latency_std_s": 10.29, - "bfcl/bfcl.overall.latency_p95_s": 31.26, - "bfcl/bfcl.non_live.ast_accuracy": 85.5, - "bfcl/bfcl.non_live.simple_ast_accuracy": 72.5, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 92.5, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 91.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 86.0, - "bfcl/bfcl.live.live_accuracy": 69.5, - "bfcl/bfcl.live.live_simple_ast_accuracy": 66.67, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 69.99, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 75.0, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 75.0, - "bfcl/bfcl.multi_turn.accuracy": 23.87, - "bfcl/bfcl.multi_turn.base_accuracy": 24.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 28.5, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 21.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 21.5, - "bfcl/bfcl.web_search.accuracy": 0.0, - "bfcl/bfcl.web_search.base_accuracy": 0.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 0.0, - "bfcl/bfcl.memory.accuracy": 0.0, - "bfcl/bfcl.memory.kv_accuracy": 0.0, - "bfcl/bfcl.memory.vector_accuracy": 0.0, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 0.0, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 50.0, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 90.12 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/magnifi.json b/data/developers/magnifi.json deleted file mode 100644 index fefbb71d537ed40535f4c3c5326d83fa9d2fe79a..0000000000000000000000000000000000000000 --- a/data/developers/magnifi.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "magnifi", - "models": [ - { - "id": "magnifi/Phi3_intent_v56_3_w_unknown_5_lr_0.002", - "name": "Phi3_intent_v56_3_w_unknown_5_lr_0.002", - "developer": "magnifi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2018, - "hfopenllm_v2/BBH": 0.3282, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2643, - "hfopenllm_v2/MUSR": 0.4123, - "hfopenllm_v2/MMLU-PRO": 0.1472 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/magpie-align.json b/data/developers/magpie-align.json deleted file mode 100644 index 0dc0a43e89bb456a61006caa30051add55effb08..0000000000000000000000000000000000000000 --- a/data/developers/magpie-align.json +++ /dev/null @@ -1,117 +0,0 @@ -{ - "developer": "Magpie-Align", - "models": [ - { - "id": "Magpie-Align/Llama-3-8B-Magpie-Align-SFT-v0.1", - "name": "Llama-3-8B-Magpie-Align-SFT-v0.1", - "developer": "Magpie-Align", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4361, - "hfopenllm_v2/BBH": 0.4615, - "hfopenllm_v2/MATH Level 5": 0.0574, - "hfopenllm_v2/GPQA": 0.2626, - "hfopenllm_v2/MUSR": 0.3277, - "hfopenllm_v2/MMLU-PRO": 0.2863 - } - }, - { - "id": "Magpie-Align/Llama-3-8B-Magpie-Align-SFT-v0.3", - "name": "Llama-3-8B-Magpie-Align-SFT-v0.3", - "developer": "Magpie-Align", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5064, - "hfopenllm_v2/BBH": 0.4572, - "hfopenllm_v2/MATH Level 5": 0.0733, - "hfopenllm_v2/GPQA": 0.2659, - "hfopenllm_v2/MUSR": 0.3424, - "hfopenllm_v2/MMLU-PRO": 0.2902 - } - }, - { - "id": "Magpie-Align/Llama-3-8B-Magpie-Align-v0.1", - "name": "Llama-3-8B-Magpie-Align-v0.1", - "developer": "Magpie-Align", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4027, - "hfopenllm_v2/BBH": 0.4789, - "hfopenllm_v2/MATH Level 5": 0.0461, - "hfopenllm_v2/GPQA": 0.2768, - "hfopenllm_v2/MUSR": 0.3087, - "hfopenllm_v2/MMLU-PRO": 0.3001 - } - }, - { - "id": "Magpie-Align/Llama-3-8B-Magpie-Align-v0.3", - "name": "Llama-3-8B-Magpie-Align-v0.3", - "developer": "Magpie-Align", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4497, - "hfopenllm_v2/BBH": 0.457, - "hfopenllm_v2/MATH Level 5": 0.0566, - "hfopenllm_v2/GPQA": 0.2651, - "hfopenllm_v2/MUSR": 0.3406, - "hfopenllm_v2/MMLU-PRO": 0.3134 - } - }, - { - "id": "Magpie-Align/Llama-3.1-8B-Magpie-Align-SFT-v0.1", - "name": "Llama-3.1-8B-Magpie-Align-SFT-v0.1", - "developer": "Magpie-Align", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4782, - "hfopenllm_v2/BBH": 0.4764, - "hfopenllm_v2/MATH Level 5": 0.0899, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.3397, - "hfopenllm_v2/MMLU-PRO": 0.2943 - } - }, - { - "id": "Magpie-Align/Llama-3.1-8B-Magpie-Align-v0.1", - "name": "Llama-3.1-8B-Magpie-Align-v0.1", - "developer": "Magpie-Align", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4458, - "hfopenllm_v2/BBH": 0.4622, - "hfopenllm_v2/MATH Level 5": 0.0665, - "hfopenllm_v2/GPQA": 0.2634, - "hfopenllm_v2/MUSR": 0.3141, - "hfopenllm_v2/MMLU-PRO": 0.3262 - } - }, - { - "id": "Magpie-Align/MagpieLM-8B-Chat-v0.1", - "name": "MagpieLM-8B-Chat-v0.1", - "developer": "Magpie-Align", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3701, - "hfopenllm_v2/BBH": 0.4172, - "hfopenllm_v2/MATH Level 5": 0.0612, - "hfopenllm_v2/GPQA": 0.2617, - "hfopenllm_v2/MUSR": 0.3501, - "hfopenllm_v2/MMLU-PRO": 0.3195 - } - }, - { - "id": "Magpie-Align/MagpieLM-8B-SFT-v0.1", - "name": "MagpieLM-8B-SFT-v0.1", - "developer": "Magpie-Align", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4721, - "hfopenllm_v2/BBH": 0.4553, - "hfopenllm_v2/MATH Level 5": 0.0755, - "hfopenllm_v2/GPQA": 0.2676, - "hfopenllm_v2/MUSR": 0.3649, - "hfopenllm_v2/MMLU-PRO": 0.299 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/maguscorp.json b/data/developers/maguscorp.json deleted file mode 100644 index 03765be6117b10742a99027717bd71648696b76f..0000000000000000000000000000000000000000 --- a/data/developers/maguscorp.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "MagusCorp", - "models": [ - { - "id": "MagusCorp/grpo_lora_enem_llama3_7b", - "name": "grpo_lora_enem_llama3_7b", - "developer": "MagusCorp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4724, - "hfopenllm_v2/BBH": 0.4801, - "hfopenllm_v2/MATH Level 5": 0.1216, - "hfopenllm_v2/GPQA": 0.3096, - "hfopenllm_v2/MUSR": 0.3971, - "hfopenllm_v2/MMLU-PRO": 0.3574 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/maldv.json b/data/developers/maldv.json deleted file mode 100644 index c9fbeaf2766c06304ea467e69a2591eaf9f6a031..0000000000000000000000000000000000000000 --- a/data/developers/maldv.json +++ /dev/null @@ -1,103 +0,0 @@ -{ - "developer": "maldv", - "models": [ - { - "id": "maldv/Awqward2.5-32B-Instruct", - "name": "Awqward2.5-32B-Instruct", - "developer": "maldv", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8255, - "hfopenllm_v2/BBH": 0.6974, - "hfopenllm_v2/MATH Level 5": 0.6231, - "hfopenllm_v2/GPQA": 0.3406, - "hfopenllm_v2/MUSR": 0.4275, - "hfopenllm_v2/MMLU-PRO": 0.5723 - } - }, - { - "id": "maldv/badger-kappa-llama-3-8b", - "name": "badger-kappa-llama-3-8b", - "developer": "maldv", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4695, - "hfopenllm_v2/BBH": 0.5085, - "hfopenllm_v2/MATH Level 5": 0.0861, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.3765, - "hfopenllm_v2/MMLU-PRO": 0.3695 - } - }, - { - "id": "maldv/badger-lambda-llama-3-8b", - "name": "badger-lambda-llama-3-8b", - "developer": "maldv", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4861, - "hfopenllm_v2/BBH": 0.4963, - "hfopenllm_v2/MATH Level 5": 0.0944, - "hfopenllm_v2/GPQA": 0.2819, - "hfopenllm_v2/MUSR": 0.3754, - "hfopenllm_v2/MMLU-PRO": 0.3767 - } - }, - { - "id": "maldv/badger-mu-llama-3-8b", - "name": "badger-mu-llama-3-8b", - "developer": "maldv", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4919, - "hfopenllm_v2/BBH": 0.5143, - "hfopenllm_v2/MATH Level 5": 0.0559, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3555, - "hfopenllm_v2/MMLU-PRO": 0.3674 - } - }, - { - "id": "maldv/badger-writer-llama-3-8b", - "name": "badger-writer-llama-3-8b", - "developer": "maldv", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5303, - "hfopenllm_v2/BBH": 0.4864, - "hfopenllm_v2/MATH Level 5": 0.0755, - "hfopenllm_v2/GPQA": 0.2894, - "hfopenllm_v2/MUSR": 0.3581, - "hfopenllm_v2/MMLU-PRO": 0.376 - } - }, - { - "id": "maldv/Lytta2.5-32B-Instruct", - "name": "Lytta2.5-32B-Instruct", - "developer": "maldv", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2508, - "hfopenllm_v2/BBH": 0.56, - "hfopenllm_v2/MATH Level 5": 0.3444, - "hfopenllm_v2/GPQA": 0.2668, - "hfopenllm_v2/MUSR": 0.3769, - "hfopenllm_v2/MMLU-PRO": 0.5048 - } - }, - { - "id": "maldv/Qwentile2.5-32B-Instruct", - "name": "Qwentile2.5-32B-Instruct", - "developer": "maldv", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7393, - "hfopenllm_v2/BBH": 0.6963, - "hfopenllm_v2/MATH Level 5": 0.5219, - "hfopenllm_v2/GPQA": 0.3842, - "hfopenllm_v2/MUSR": 0.4682, - "hfopenllm_v2/MMLU-PRO": 0.5879 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/manolopueblo.json b/data/developers/manolopueblo.json deleted file mode 100644 index aa2d755da56eb00357d2cb595b16f96384ab5fbf..0000000000000000000000000000000000000000 --- a/data/developers/manolopueblo.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "developer": "ManoloPueblo", - "models": [ - { - "id": "ManoloPueblo/ContentCuisine_1-7B-slerp", - "name": "ContentCuisine_1-7B-slerp", - "developer": "ManoloPueblo", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3907, - "hfopenllm_v2/BBH": 0.5188, - "hfopenllm_v2/MATH Level 5": 0.0733, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.4672, - "hfopenllm_v2/MMLU-PRO": 0.3054 - } - }, - { - "id": "ManoloPueblo/LLM_MERGE_CC2", - "name": "LLM_MERGE_CC2", - "developer": "ManoloPueblo", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3853, - "hfopenllm_v2/BBH": 0.5209, - "hfopenllm_v2/MATH Level 5": 0.0642, - "hfopenllm_v2/GPQA": 0.3045, - "hfopenllm_v2/MUSR": 0.4593, - "hfopenllm_v2/MMLU-PRO": 0.3032 - } - }, - { - "id": "ManoloPueblo/LLM_MERGE_CC3", - "name": "LLM_MERGE_CC3", - "developer": "ManoloPueblo", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3959, - "hfopenllm_v2/BBH": 0.5246, - "hfopenllm_v2/MATH Level 5": 0.0793, - "hfopenllm_v2/GPQA": 0.3096, - "hfopenllm_v2/MUSR": 0.4672, - "hfopenllm_v2/MMLU-PRO": 0.3156 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/marcuscedricridia.json b/data/developers/marcuscedricridia.json deleted file mode 100644 index a18628920d587daa411d3e2a3545303a13e1c815..0000000000000000000000000000000000000000 --- a/data/developers/marcuscedricridia.json +++ /dev/null @@ -1,565 +0,0 @@ -{ - "developer": "marcuscedricridia", - "models": [ - { - "id": "marcuscedricridia/absolute-o1-7b", - "name": "absolute-o1-7b", - "developer": "marcuscedricridia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7516, - "hfopenllm_v2/BBH": 0.5469, - "hfopenllm_v2/MATH Level 5": 0.5083, - "hfopenllm_v2/GPQA": 0.3196, - "hfopenllm_v2/MUSR": 0.4114, - "hfopenllm_v2/MMLU-PRO": 0.4413 - } - }, - { - "id": "marcuscedricridia/Cheng-1", - "name": "Cheng-1", - "developer": "marcuscedricridia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7789, - "hfopenllm_v2/BBH": 0.5525, - "hfopenllm_v2/MATH Level 5": 0.4894, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.4073, - "hfopenllm_v2/MMLU-PRO": 0.4349 - } - }, - { - "id": "marcuscedricridia/Cheng-2", - "name": "Cheng-2", - "developer": "marcuscedricridia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8337, - "hfopenllm_v2/BBH": 0.6499, - "hfopenllm_v2/MATH Level 5": 0.5438, - "hfopenllm_v2/GPQA": 0.3456, - "hfopenllm_v2/MUSR": 0.4193, - "hfopenllm_v2/MMLU-PRO": 0.5013 - } - }, - { - "id": "marcuscedricridia/Cheng-2-v1.1", - "name": "Cheng-2-v1.1", - "developer": "marcuscedricridia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.827, - "hfopenllm_v2/BBH": 0.651, - "hfopenllm_v2/MATH Level 5": 0.5393, - "hfopenllm_v2/GPQA": 0.3431, - "hfopenllm_v2/MUSR": 0.4167, - "hfopenllm_v2/MMLU-PRO": 0.5076 - } - }, - { - "id": "marcuscedricridia/cursa-o1-7b", - "name": "cursa-o1-7b", - "developer": "marcuscedricridia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7628, - "hfopenllm_v2/BBH": 0.5466, - "hfopenllm_v2/MATH Level 5": 0.4955, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.4301, - "hfopenllm_v2/MMLU-PRO": 0.4392 - } - }, - { - "id": "marcuscedricridia/cursa-o1-7b-2-28-2025", - "name": "cursa-o1-7b-2-28-2025", - "developer": "marcuscedricridia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7467, - "hfopenllm_v2/BBH": 0.5384, - "hfopenllm_v2/MATH Level 5": 0.4811, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.4273, - "hfopenllm_v2/MMLU-PRO": 0.4365 - } - }, - { - "id": "marcuscedricridia/cursa-o1-7b-v1.1", - "name": "cursa-o1-7b-v1.1", - "developer": "marcuscedricridia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7528, - "hfopenllm_v2/BBH": 0.5493, - "hfopenllm_v2/MATH Level 5": 0.4985, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.4259, - "hfopenllm_v2/MMLU-PRO": 0.4392 - } - }, - { - "id": "marcuscedricridia/cursa-o1-7b-v1.2-normalize-false", - "name": "cursa-o1-7b-v1.2-normalize-false", - "developer": "marcuscedricridia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7616, - "hfopenllm_v2/BBH": 0.5492, - "hfopenllm_v2/MATH Level 5": 0.4992, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.4273, - "hfopenllm_v2/MMLU-PRO": 0.4436 - } - }, - { - "id": "marcuscedricridia/cursor-o1-7b", - "name": "cursor-o1-7b", - "developer": "marcuscedricridia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4107, - "hfopenllm_v2/BBH": 0.5007, - "hfopenllm_v2/MATH Level 5": 0.1412, - "hfopenllm_v2/GPQA": 0.281, - "hfopenllm_v2/MUSR": 0.4101, - "hfopenllm_v2/MMLU-PRO": 0.3251 - } - }, - { - "id": "marcuscedricridia/cursorr-o1.2-7b", - "name": "cursorr-o1.2-7b", - "developer": "marcuscedricridia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.166, - "hfopenllm_v2/BBH": 0.3068, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2542, - "hfopenllm_v2/MUSR": 0.3538, - "hfopenllm_v2/MMLU-PRO": 0.108 - } - }, - { - "id": "marcuscedricridia/etr1o-explicit-v1.1", - "name": "etr1o-explicit-v1.1", - "developer": "marcuscedricridia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.288, - "hfopenllm_v2/BBH": 0.3132, - "hfopenllm_v2/MATH Level 5": 0.0045, - "hfopenllm_v2/GPQA": 0.2777, - "hfopenllm_v2/MUSR": 0.4111, - "hfopenllm_v2/MMLU-PRO": 0.1195 - } - }, - { - "id": "marcuscedricridia/etr1o-explicit-v1.2", - "name": "etr1o-explicit-v1.2", - "developer": "marcuscedricridia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1504, - "hfopenllm_v2/BBH": 0.295, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.4031, - "hfopenllm_v2/MMLU-PRO": 0.1126 - } - }, - { - "id": "marcuscedricridia/etr1o-v1.1", - "name": "etr1o-v1.1", - "developer": "marcuscedricridia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1597, - "hfopenllm_v2/BBH": 0.31, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2567, - "hfopenllm_v2/MUSR": 0.4017, - "hfopenllm_v2/MMLU-PRO": 0.1157 - } - }, - { - "id": "marcuscedricridia/etr1o-v1.2", - "name": "etr1o-v1.2", - "developer": "marcuscedricridia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7287, - "hfopenllm_v2/BBH": 0.6349, - "hfopenllm_v2/MATH Level 5": 0.3588, - "hfopenllm_v2/GPQA": 0.3758, - "hfopenllm_v2/MUSR": 0.4714, - "hfopenllm_v2/MMLU-PRO": 0.5316 - } - }, - { - "id": "marcuscedricridia/fan-o1-7b", - "name": "fan-o1-7b", - "developer": "marcuscedricridia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4456, - "hfopenllm_v2/BBH": 0.4849, - "hfopenllm_v2/MATH Level 5": 0.1616, - "hfopenllm_v2/GPQA": 0.2844, - "hfopenllm_v2/MUSR": 0.3834, - "hfopenllm_v2/MMLU-PRO": 0.3274 - } - }, - { - "id": "marcuscedricridia/Hush-Qwen2.5-7B-MST", - "name": "Hush-Qwen2.5-7B-MST", - "developer": "marcuscedricridia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7488, - "hfopenllm_v2/BBH": 0.5458, - "hfopenllm_v2/MATH Level 5": 0.4245, - "hfopenllm_v2/GPQA": 0.3037, - "hfopenllm_v2/MUSR": 0.3914, - "hfopenllm_v2/MMLU-PRO": 0.4163 - } - }, - { - "id": "marcuscedricridia/Hush-Qwen2.5-7B-MST-v1.1", - "name": "Hush-Qwen2.5-7B-MST-v1.1", - "developer": "marcuscedricridia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7445, - "hfopenllm_v2/BBH": 0.5559, - "hfopenllm_v2/MATH Level 5": 0.4653, - "hfopenllm_v2/GPQA": 0.3062, - "hfopenllm_v2/MUSR": 0.4073, - "hfopenllm_v2/MMLU-PRO": 0.4299 - } - }, - { - "id": "marcuscedricridia/Hush-Qwen2.5-7B-MST-v1.3", - "name": "Hush-Qwen2.5-7B-MST-v1.3", - "developer": "marcuscedricridia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7043, - "hfopenllm_v2/BBH": 0.5516, - "hfopenllm_v2/MATH Level 5": 0.4758, - "hfopenllm_v2/GPQA": 0.3146, - "hfopenllm_v2/MUSR": 0.4311, - "hfopenllm_v2/MMLU-PRO": 0.444 - } - }, - { - "id": "marcuscedricridia/Hush-Qwen2.5-7B-Preview", - "name": "Hush-Qwen2.5-7B-Preview", - "developer": "marcuscedricridia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7962, - "hfopenllm_v2/BBH": 0.5431, - "hfopenllm_v2/MATH Level 5": 0.3754, - "hfopenllm_v2/GPQA": 0.3112, - "hfopenllm_v2/MUSR": 0.4298, - "hfopenllm_v2/MMLU-PRO": 0.4364 - } - }, - { - "id": "marcuscedricridia/Hush-Qwen2.5-7B-RP-v1.4-1M", - "name": "Hush-Qwen2.5-7B-RP-v1.4-1M", - "developer": "marcuscedricridia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7728, - "hfopenllm_v2/BBH": 0.5295, - "hfopenllm_v2/MATH Level 5": 0.3369, - "hfopenllm_v2/GPQA": 0.2987, - "hfopenllm_v2/MUSR": 0.4433, - "hfopenllm_v2/MMLU-PRO": 0.4135 - } - }, - { - "id": "marcuscedricridia/Hush-Qwen2.5-7B-v1.1", - "name": "Hush-Qwen2.5-7B-v1.1", - "developer": "marcuscedricridia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7889, - "hfopenllm_v2/BBH": 0.5384, - "hfopenllm_v2/MATH Level 5": 0.4381, - "hfopenllm_v2/GPQA": 0.3163, - "hfopenllm_v2/MUSR": 0.4179, - "hfopenllm_v2/MMLU-PRO": 0.4227 - } - }, - { - "id": "marcuscedricridia/Hush-Qwen2.5-7B-v1.2", - "name": "Hush-Qwen2.5-7B-v1.2", - "developer": "marcuscedricridia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7865, - "hfopenllm_v2/BBH": 0.5403, - "hfopenllm_v2/MATH Level 5": 0.4403, - "hfopenllm_v2/GPQA": 0.3146, - "hfopenllm_v2/MUSR": 0.4219, - "hfopenllm_v2/MMLU-PRO": 0.4197 - } - }, - { - "id": "marcuscedricridia/Hush-Qwen2.5-7B-v1.3", - "name": "Hush-Qwen2.5-7B-v1.3", - "developer": "marcuscedricridia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7856, - "hfopenllm_v2/BBH": 0.5327, - "hfopenllm_v2/MATH Level 5": 0.3323, - "hfopenllm_v2/GPQA": 0.3121, - "hfopenllm_v2/MUSR": 0.4246, - "hfopenllm_v2/MMLU-PRO": 0.4345 - } - }, - { - "id": "marcuscedricridia/Hush-Qwen2.5-7B-v1.4", - "name": "Hush-Qwen2.5-7B-v1.4", - "developer": "marcuscedricridia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7835, - "hfopenllm_v2/BBH": 0.5423, - "hfopenllm_v2/MATH Level 5": 0.426, - "hfopenllm_v2/GPQA": 0.3112, - "hfopenllm_v2/MUSR": 0.4232, - "hfopenllm_v2/MMLU-PRO": 0.4195 - } - }, - { - "id": "marcuscedricridia/olmner-7b", - "name": "olmner-7b", - "developer": "marcuscedricridia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7254, - "hfopenllm_v2/BBH": 0.5472, - "hfopenllm_v2/MATH Level 5": 0.463, - "hfopenllm_v2/GPQA": 0.3079, - "hfopenllm_v2/MUSR": 0.438, - "hfopenllm_v2/MMLU-PRO": 0.4309 - } - }, - { - "id": "marcuscedricridia/olmner-della-7b", - "name": "olmner-della-7b", - "developer": "marcuscedricridia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7637, - "hfopenllm_v2/BBH": 0.5491, - "hfopenllm_v2/MATH Level 5": 0.4962, - "hfopenllm_v2/GPQA": 0.3012, - "hfopenllm_v2/MUSR": 0.4208, - "hfopenllm_v2/MMLU-PRO": 0.4386 - } - }, - { - "id": "marcuscedricridia/olmner-o1-7b", - "name": "olmner-o1-7b", - "developer": "marcuscedricridia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7528, - "hfopenllm_v2/BBH": 0.5481, - "hfopenllm_v2/MATH Level 5": 0.4924, - "hfopenllm_v2/GPQA": 0.3012, - "hfopenllm_v2/MUSR": 0.4299, - "hfopenllm_v2/MMLU-PRO": 0.4386 - } - }, - { - "id": "marcuscedricridia/olmner-sbr-7b", - "name": "olmner-sbr-7b", - "developer": "marcuscedricridia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.76, - "hfopenllm_v2/BBH": 0.5462, - "hfopenllm_v2/MATH Level 5": 0.4947, - "hfopenllm_v2/GPQA": 0.3087, - "hfopenllm_v2/MUSR": 0.4154, - "hfopenllm_v2/MMLU-PRO": 0.4412 - } - }, - { - "id": "marcuscedricridia/post-cursa-o1", - "name": "post-cursa-o1", - "developer": "marcuscedricridia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7628, - "hfopenllm_v2/BBH": 0.548, - "hfopenllm_v2/MATH Level 5": 0.4872, - "hfopenllm_v2/GPQA": 0.3096, - "hfopenllm_v2/MUSR": 0.4351, - "hfopenllm_v2/MMLU-PRO": 0.4361 - } - }, - { - "id": "marcuscedricridia/pre-cursa-o1", - "name": "pre-cursa-o1", - "developer": "marcuscedricridia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7409, - "hfopenllm_v2/BBH": 0.5462, - "hfopenllm_v2/MATH Level 5": 0.5038, - "hfopenllm_v2/GPQA": 0.3096, - "hfopenllm_v2/MUSR": 0.426, - "hfopenllm_v2/MMLU-PRO": 0.4424 - } - }, - { - "id": "marcuscedricridia/pre-cursa-o1-v1.2", - "name": "pre-cursa-o1-v1.2", - "developer": "marcuscedricridia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7549, - "hfopenllm_v2/BBH": 0.5487, - "hfopenllm_v2/MATH Level 5": 0.5068, - "hfopenllm_v2/GPQA": 0.3129, - "hfopenllm_v2/MUSR": 0.4272, - "hfopenllm_v2/MMLU-PRO": 0.4402 - } - }, - { - "id": "marcuscedricridia/pre-cursa-o1-v1.3", - "name": "pre-cursa-o1-v1.3", - "developer": "marcuscedricridia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7507, - "hfopenllm_v2/BBH": 0.5455, - "hfopenllm_v2/MATH Level 5": 0.5076, - "hfopenllm_v2/GPQA": 0.3129, - "hfopenllm_v2/MUSR": 0.4271, - "hfopenllm_v2/MMLU-PRO": 0.442 - } - }, - { - "id": "marcuscedricridia/pre-cursa-o1-v1.4", - "name": "pre-cursa-o1-v1.4", - "developer": "marcuscedricridia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7488, - "hfopenllm_v2/BBH": 0.5493, - "hfopenllm_v2/MATH Level 5": 0.4834, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.4285, - "hfopenllm_v2/MMLU-PRO": 0.4436 - } - }, - { - "id": "marcuscedricridia/pre-cursa-o1-v1.6", - "name": "pre-cursa-o1-v1.6", - "developer": "marcuscedricridia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7528, - "hfopenllm_v2/BBH": 0.5473, - "hfopenllm_v2/MATH Level 5": 0.5, - "hfopenllm_v2/GPQA": 0.3205, - "hfopenllm_v2/MUSR": 0.4234, - "hfopenllm_v2/MMLU-PRO": 0.4413 - } - }, - { - "id": "marcuscedricridia/Qwen2.5-7B-Preview", - "name": "Qwen2.5-7B-Preview", - "developer": "marcuscedricridia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7679, - "hfopenllm_v2/BBH": 0.536, - "hfopenllm_v2/MATH Level 5": 0.3444, - "hfopenllm_v2/GPQA": 0.3238, - "hfopenllm_v2/MUSR": 0.414, - "hfopenllm_v2/MMLU-PRO": 0.4258 - } - }, - { - "id": "marcuscedricridia/r1o-et", - "name": "r1o-et", - "developer": "marcuscedricridia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3597, - "hfopenllm_v2/BBH": 0.4209, - "hfopenllm_v2/MATH Level 5": 0.0793, - "hfopenllm_v2/GPQA": 0.2727, - "hfopenllm_v2/MUSR": 0.3579, - "hfopenllm_v2/MMLU-PRO": 0.258 - } - }, - { - "id": "marcuscedricridia/sbr-o1-7b", - "name": "sbr-o1-7b", - "developer": "marcuscedricridia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7455, - "hfopenllm_v2/BBH": 0.5479, - "hfopenllm_v2/MATH Level 5": 0.4985, - "hfopenllm_v2/GPQA": 0.3104, - "hfopenllm_v2/MUSR": 0.4404, - "hfopenllm_v2/MMLU-PRO": 0.4355 - } - }, - { - "id": "marcuscedricridia/stray-r1o-et", - "name": "stray-r1o-et", - "developer": "marcuscedricridia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1562, - "hfopenllm_v2/BBH": 0.2967, - "hfopenllm_v2/MATH Level 5": 0.0045, - "hfopenllm_v2/GPQA": 0.2617, - "hfopenllm_v2/MUSR": 0.4086, - "hfopenllm_v2/MMLU-PRO": 0.1094 - } - }, - { - "id": "marcuscedricridia/Yell-Qwen2.5-7B-Preview", - "name": "Yell-Qwen2.5-7B-Preview", - "developer": "marcuscedricridia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5839, - "hfopenllm_v2/BBH": 0.5371, - "hfopenllm_v2/MATH Level 5": 0.1926, - "hfopenllm_v2/GPQA": 0.281, - "hfopenllm_v2/MUSR": 0.4046, - "hfopenllm_v2/MMLU-PRO": 0.3798 - } - }, - { - "id": "marcuscedricridia/Yell-Qwen2.5-7B-Preview-v1.1", - "name": "Yell-Qwen2.5-7B-Preview-v1.1", - "developer": "marcuscedricridia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5757, - "hfopenllm_v2/BBH": 0.5348, - "hfopenllm_v2/MATH Level 5": 0.1896, - "hfopenllm_v2/GPQA": 0.2861, - "hfopenllm_v2/MUSR": 0.4059, - "hfopenllm_v2/MMLU-PRO": 0.3831 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/marin-community.json b/data/developers/marin-community.json deleted file mode 100644 index ca6eb59d93bda99084a5861cffc99217581f03ee..0000000000000000000000000000000000000000 --- a/data/developers/marin-community.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "marin-community", - "models": [ - { - "id": "marin-community/marin-8b-instruct", - "name": "Marin 8B Instruct", - "developer": "marin-community", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.325, - "helm_capabilities/MMLU-Pro": 0.188, - "helm_capabilities/GPQA": 0.168, - "helm_capabilities/IFEval": 0.632, - "helm_capabilities/WildBench": 0.477, - "helm_capabilities/Omni-MATH": 0.16 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/marinaraspaghetti.json b/data/developers/marinaraspaghetti.json deleted file mode 100644 index 93215baca42819968f8721eac8bd11b3f2d54e9d..0000000000000000000000000000000000000000 --- a/data/developers/marinaraspaghetti.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "MarinaraSpaghetti", - "models": [ - { - "id": "MarinaraSpaghetti/Nemomix-v4.0-12B", - "name": "Nemomix-v4.0-12B", - "developer": "MarinaraSpaghetti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5575, - "hfopenllm_v2/BBH": 0.5275, - "hfopenllm_v2/MATH Level 5": 0.108, - "hfopenllm_v2/GPQA": 0.2919, - "hfopenllm_v2/MUSR": 0.4244, - "hfopenllm_v2/MMLU-PRO": 0.3613 - } - }, - { - "id": "MarinaraSpaghetti/NemoReRemix-12B", - "name": "NemoReRemix-12B", - "developer": "MarinaraSpaghetti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3343, - "hfopenllm_v2/BBH": 0.5537, - "hfopenllm_v2/MATH Level 5": 0.0906, - "hfopenllm_v2/GPQA": 0.318, - "hfopenllm_v2/MUSR": 0.4501, - "hfopenllm_v2/MMLU-PRO": 0.3598 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/marsouuu.json b/data/developers/marsouuu.json deleted file mode 100644 index 67e280f0212573b8f138f7caffea4c3a9d0bd87a..0000000000000000000000000000000000000000 --- a/data/developers/marsouuu.json +++ /dev/null @@ -1,117 +0,0 @@ -{ - "developer": "Marsouuu", - "models": [ - { - "id": "Marsouuu/general3B-ECE-PRYMMAL-Martial", - "name": "general3B-ECE-PRYMMAL-Martial", - "developer": "Marsouuu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2722, - "hfopenllm_v2/BBH": 0.5394, - "hfopenllm_v2/MATH Level 5": 0.1548, - "hfopenllm_v2/GPQA": 0.3196, - "hfopenllm_v2/MUSR": 0.4701, - "hfopenllm_v2/MMLU-PRO": 0.3876 - } - }, - { - "id": "Marsouuu/general3Bv2-ECE-PRYMMAL-Martial", - "name": "general3Bv2-ECE-PRYMMAL-Martial", - "developer": "Marsouuu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5693, - "hfopenllm_v2/BBH": 0.5637, - "hfopenllm_v2/MATH Level 5": 0.3671, - "hfopenllm_v2/GPQA": 0.3104, - "hfopenllm_v2/MUSR": 0.4396, - "hfopenllm_v2/MMLU-PRO": 0.4498 - } - }, - { - "id": "Marsouuu/lareneg1_78B-ECE-PRYMMAL-Martial", - "name": "lareneg1_78B-ECE-PRYMMAL-Martial", - "developer": "Marsouuu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2795, - "hfopenllm_v2/BBH": 0.423, - "hfopenllm_v2/MATH Level 5": 0.114, - "hfopenllm_v2/GPQA": 0.2819, - "hfopenllm_v2/MUSR": 0.3867, - "hfopenllm_v2/MMLU-PRO": 0.2922 - } - }, - { - "id": "Marsouuu/lareneg3B-ECE-PRYMMAL-Martial", - "name": "lareneg3B-ECE-PRYMMAL-Martial", - "developer": "Marsouuu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3303, - "hfopenllm_v2/BBH": 0.5453, - "hfopenllm_v2/MATH Level 5": 0.1518, - "hfopenllm_v2/GPQA": 0.3247, - "hfopenllm_v2/MUSR": 0.4725, - "hfopenllm_v2/MMLU-PRO": 0.3767 - } - }, - { - "id": "Marsouuu/lareneg3Bv2-ECE-PRYMMAL-Martial", - "name": "lareneg3Bv2-ECE-PRYMMAL-Martial", - "developer": "Marsouuu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5753, - "hfopenllm_v2/BBH": 0.5623, - "hfopenllm_v2/MATH Level 5": 0.3656, - "hfopenllm_v2/GPQA": 0.3196, - "hfopenllm_v2/MUSR": 0.4369, - "hfopenllm_v2/MMLU-PRO": 0.4511 - } - }, - { - "id": "Marsouuu/MiniMathExpert-2_61B-ECE-PRYMMAL-Martial", - "name": "MiniMathExpert-2_61B-ECE-PRYMMAL-Martial", - "developer": "Marsouuu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2548, - "hfopenllm_v2/BBH": 0.3953, - "hfopenllm_v2/MATH Level 5": 0.074, - "hfopenllm_v2/GPQA": 0.2752, - "hfopenllm_v2/MUSR": 0.4083, - "hfopenllm_v2/MMLU-PRO": 0.2274 - } - }, - { - "id": "Marsouuu/MiniQwenMathExpert-ECE-PRYMMAL-Martial", - "name": "MiniQwenMathExpert-ECE-PRYMMAL-Martial", - "developer": "Marsouuu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2795, - "hfopenllm_v2/BBH": 0.423, - "hfopenllm_v2/MATH Level 5": 0.114, - "hfopenllm_v2/GPQA": 0.2819, - "hfopenllm_v2/MUSR": 0.3867, - "hfopenllm_v2/MMLU-PRO": 0.2922 - } - }, - { - "id": "Marsouuu/MistralBase-4x7B-MoE-ECE-PRYMMAL-Martial", - "name": "MistralBase-4x7B-MoE-ECE-PRYMMAL-Martial", - "developer": "Marsouuu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1697, - "hfopenllm_v2/BBH": 0.3464, - "hfopenllm_v2/MATH Level 5": 0.0144, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3991, - "hfopenllm_v2/MMLU-PRO": 0.1379 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/matouleloup.json b/data/developers/matouleloup.json deleted file mode 100644 index 32ce94b71fbd9954a39bf98b1d00aabe591f2d98..0000000000000000000000000000000000000000 --- a/data/developers/matouleloup.json +++ /dev/null @@ -1,75 +0,0 @@ -{ - "developer": "matouLeLoup", - "models": [ - { - "id": "matouLeLoup/ECE-PRYMMAL-0.5B-FT-EnhancedMUSREnsembleV3", - "name": "ECE-PRYMMAL-0.5B-FT-EnhancedMUSREnsembleV3", - "developer": "matouLeLoup", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1873, - "hfopenllm_v2/BBH": 0.3239, - "hfopenllm_v2/MATH Level 5": 0.0264, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.3752, - "hfopenllm_v2/MMLU-PRO": 0.172 - } - }, - { - "id": "matouLeLoup/ECE-PRYMMAL-0.5B-FT-MUSR-ENSEMBLE-V2Mathis", - "name": "ECE-PRYMMAL-0.5B-FT-MUSR-ENSEMBLE-V2Mathis", - "developer": "matouLeLoup", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1873, - "hfopenllm_v2/BBH": 0.3239, - "hfopenllm_v2/MATH Level 5": 0.0264, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.3752, - "hfopenllm_v2/MMLU-PRO": 0.172 - } - }, - { - "id": "matouLeLoup/ECE-PRYMMAL-0.5B-FT-V4-MUSR-ENSEMBLE-Mathis", - "name": "ECE-PRYMMAL-0.5B-FT-V4-MUSR-ENSEMBLE-Mathis", - "developer": "matouLeLoup", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1873, - "hfopenllm_v2/BBH": 0.3239, - "hfopenllm_v2/MATH Level 5": 0.0264, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.3752, - "hfopenllm_v2/MMLU-PRO": 0.172 - } - }, - { - "id": "matouLeLoup/ECE-PRYMMAL-0.5B-FT-V4-MUSR-Mathis", - "name": "ECE-PRYMMAL-0.5B-FT-V4-MUSR-Mathis", - "developer": "matouLeLoup", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1882, - "hfopenllm_v2/BBH": 0.3233, - "hfopenllm_v2/MATH Level 5": 0.0272, - "hfopenllm_v2/GPQA": 0.2634, - "hfopenllm_v2/MUSR": 0.3685, - "hfopenllm_v2/MMLU-PRO": 0.172 - } - }, - { - "id": "matouLeLoup/ECE-PRYMMAL-0.5B-FT-V5-MUSR-Mathis", - "name": "ECE-PRYMMAL-0.5B-FT-V5-MUSR-Mathis", - "developer": "matouLeLoup", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1652, - "hfopenllm_v2/BBH": 0.3024, - "hfopenllm_v2/MATH Level 5": 0.0189, - "hfopenllm_v2/GPQA": 0.2567, - "hfopenllm_v2/MUSR": 0.4273, - "hfopenllm_v2/MMLU-PRO": 0.1116 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/mattshumer.json b/data/developers/mattshumer.json deleted file mode 100644 index 830afbe5e4c7ee8e3588967b168e27c5fd4434e1..0000000000000000000000000000000000000000 --- a/data/developers/mattshumer.json +++ /dev/null @@ -1,46 +0,0 @@ -{ - "developer": "mattshumer", - "models": [ - { - "id": "mattshumer/ref_70_e3", - "name": "ref_70_e3", - "developer": "mattshumer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6294, - "hfopenllm_v2/BBH": 0.6501, - "hfopenllm_v2/MATH Level 5": 0.2795, - "hfopenllm_v2/GPQA": 0.3356, - "hfopenllm_v2/MUSR": 0.4328, - "hfopenllm_v2/MMLU-PRO": 0.5303 - } - }, - { - "id": "mattshumer/Reflection-70B", - "name": "mattshumer/Reflection-70B", - "developer": "mattshumer", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.8422, - "reward-bench/Chat": 0.9749, - "reward-bench/Chat Hard": 0.7061, - "reward-bench/Safety": 0.8318, - "reward-bench/Reasoning": 0.8562 - } - }, - { - "id": "mattshumer/Reflection-Llama-3.1-70B", - "name": "Reflection-Llama-3.1-70B", - "developer": "mattshumer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0045, - "hfopenllm_v2/BBH": 0.645, - "hfopenllm_v2/MATH Level 5": 0.2145, - "hfopenllm_v2/GPQA": 0.3633, - "hfopenllm_v2/MUSR": 0.4577, - "hfopenllm_v2/MMLU-PRO": 0.4955 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/maywell.json b/data/developers/maywell.json deleted file mode 100644 index 0c4576902a7f911314c03825ca7c9d050bfeaf4f..0000000000000000000000000000000000000000 --- a/data/developers/maywell.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "maywell", - "models": [ - { - "id": "maywell/Qwen2-7B-Multilingual-RP", - "name": "Qwen2-7B-Multilingual-RP", - "developer": "maywell", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4347, - "hfopenllm_v2/BBH": 0.5062, - "hfopenllm_v2/MATH Level 5": 0.2243, - "hfopenllm_v2/GPQA": 0.297, - "hfopenllm_v2/MUSR": 0.3696, - "hfopenllm_v2/MMLU-PRO": 0.3859 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/maziyarpanahi.json b/data/developers/maziyarpanahi.json deleted file mode 100644 index 10d65c817209ad5800bbea0aa07d0943c266e0f5..0000000000000000000000000000000000000000 --- a/data/developers/maziyarpanahi.json +++ /dev/null @@ -1,621 +0,0 @@ -{ - "developer": "MaziyarPanahi", - "models": [ - { - "id": "MaziyarPanahi/calme-2.1-llama3.1-70b", - "name": "calme-2.1-llama3.1-70b", - "developer": "MaziyarPanahi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8434, - "hfopenllm_v2/BBH": 0.6448, - "hfopenllm_v2/MATH Level 5": 0.4101, - "hfopenllm_v2/GPQA": 0.328, - "hfopenllm_v2/MUSR": 0.438, - "hfopenllm_v2/MMLU-PRO": 0.5283 - } - }, - { - "id": "MaziyarPanahi/calme-2.1-phi3-4b", - "name": "calme-2.1-phi3-4b", - "developer": "MaziyarPanahi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5525, - "hfopenllm_v2/BBH": 0.5595, - "hfopenllm_v2/MATH Level 5": 0.1314, - "hfopenllm_v2/GPQA": 0.3297, - "hfopenllm_v2/MUSR": 0.4015, - "hfopenllm_v2/MMLU-PRO": 0.3746 - } - }, - { - "id": "MaziyarPanahi/calme-2.1-phi3.5-4b", - "name": "calme-2.1-phi3.5-4b", - "developer": "MaziyarPanahi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5659, - "hfopenllm_v2/BBH": 0.5484, - "hfopenllm_v2/MATH Level 5": 0.2039, - "hfopenllm_v2/GPQA": 0.344, - "hfopenllm_v2/MUSR": 0.3995, - "hfopenllm_v2/MMLU-PRO": 0.3935 - } - }, - { - "id": "MaziyarPanahi/calme-2.1-qwen2-72b", - "name": "calme-2.1-qwen2-72b", - "developer": "MaziyarPanahi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8163, - "hfopenllm_v2/BBH": 0.6966, - "hfopenllm_v2/MATH Level 5": 0.4079, - "hfopenllm_v2/GPQA": 0.3809, - "hfopenllm_v2/MUSR": 0.4732, - "hfopenllm_v2/MMLU-PRO": 0.5415 - } - }, - { - "id": "MaziyarPanahi/calme-2.1-qwen2-7b", - "name": "calme-2.1-qwen2-7b", - "developer": "MaziyarPanahi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3816, - "hfopenllm_v2/BBH": 0.5046, - "hfopenllm_v2/MATH Level 5": 0.2311, - "hfopenllm_v2/GPQA": 0.2894, - "hfopenllm_v2/MUSR": 0.4437, - "hfopenllm_v2/MMLU-PRO": 0.3693 - } - }, - { - "id": "MaziyarPanahi/calme-2.1-qwen2.5-72b", - "name": "calme-2.1-qwen2.5-72b", - "developer": "MaziyarPanahi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8662, - "hfopenllm_v2/BBH": 0.7262, - "hfopenllm_v2/MATH Level 5": 0.5914, - "hfopenllm_v2/GPQA": 0.3633, - "hfopenllm_v2/MUSR": 0.4298, - "hfopenllm_v2/MMLU-PRO": 0.5619 - } - }, - { - "id": "MaziyarPanahi/calme-2.1-rys-78b", - "name": "calme-2.1-rys-78b", - "developer": "MaziyarPanahi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8136, - "hfopenllm_v2/BBH": 0.7098, - "hfopenllm_v2/MATH Level 5": 0.3943, - "hfopenllm_v2/GPQA": 0.3943, - "hfopenllm_v2/MUSR": 0.4693, - "hfopenllm_v2/MMLU-PRO": 0.5444 - } - }, - { - "id": "MaziyarPanahi/calme-2.2-llama3-70b", - "name": "calme-2.2-llama3-70b", - "developer": "MaziyarPanahi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8208, - "hfopenllm_v2/BBH": 0.6435, - "hfopenllm_v2/MATH Level 5": 0.2394, - "hfopenllm_v2/GPQA": 0.3414, - "hfopenllm_v2/MUSR": 0.4446, - "hfopenllm_v2/MMLU-PRO": 0.5207 - } - }, - { - "id": "MaziyarPanahi/calme-2.2-llama3.1-70b", - "name": "calme-2.2-llama3.1-70b", - "developer": "MaziyarPanahi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8593, - "hfopenllm_v2/BBH": 0.6793, - "hfopenllm_v2/MATH Level 5": 0.4366, - "hfopenllm_v2/GPQA": 0.3247, - "hfopenllm_v2/MUSR": 0.4542, - "hfopenllm_v2/MMLU-PRO": 0.5415 - } - }, - { - "id": "MaziyarPanahi/calme-2.2-phi3-4b", - "name": "calme-2.2-phi3-4b", - "developer": "MaziyarPanahi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5069, - "hfopenllm_v2/BBH": 0.553, - "hfopenllm_v2/MATH Level 5": 0.145, - "hfopenllm_v2/GPQA": 0.3213, - "hfopenllm_v2/MUSR": 0.3976, - "hfopenllm_v2/MMLU-PRO": 0.3814 - } - }, - { - "id": "MaziyarPanahi/calme-2.2-qwen2-72b", - "name": "calme-2.2-qwen2-72b", - "developer": "MaziyarPanahi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8008, - "hfopenllm_v2/BBH": 0.694, - "hfopenllm_v2/MATH Level 5": 0.4532, - "hfopenllm_v2/GPQA": 0.3742, - "hfopenllm_v2/MUSR": 0.4508, - "hfopenllm_v2/MMLU-PRO": 0.5435 - } - }, - { - "id": "MaziyarPanahi/calme-2.2-qwen2-7b", - "name": "calme-2.2-qwen2-7b", - "developer": "MaziyarPanahi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3597, - "hfopenllm_v2/BBH": 0.5215, - "hfopenllm_v2/MATH Level 5": 0.2145, - "hfopenllm_v2/GPQA": 0.2911, - "hfopenllm_v2/MUSR": 0.4358, - "hfopenllm_v2/MMLU-PRO": 0.3899 - } - }, - { - "id": "MaziyarPanahi/calme-2.2-qwen2.5-72b", - "name": "calme-2.2-qwen2.5-72b", - "developer": "MaziyarPanahi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8477, - "hfopenllm_v2/BBH": 0.7276, - "hfopenllm_v2/MATH Level 5": 0.5891, - "hfopenllm_v2/GPQA": 0.3591, - "hfopenllm_v2/MUSR": 0.4207, - "hfopenllm_v2/MMLU-PRO": 0.5618 - } - }, - { - "id": "MaziyarPanahi/calme-2.2-rys-78b", - "name": "calme-2.2-rys-78b", - "developer": "MaziyarPanahi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7986, - "hfopenllm_v2/BBH": 0.7081, - "hfopenllm_v2/MATH Level 5": 0.4071, - "hfopenllm_v2/GPQA": 0.4069, - "hfopenllm_v2/MUSR": 0.4536, - "hfopenllm_v2/MMLU-PRO": 0.5386 - } - }, - { - "id": "MaziyarPanahi/calme-2.3-llama3-70b", - "name": "calme-2.3-llama3-70b", - "developer": "MaziyarPanahi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.801, - "hfopenllm_v2/BBH": 0.6399, - "hfopenllm_v2/MATH Level 5": 0.2326, - "hfopenllm_v2/GPQA": 0.3381, - "hfopenllm_v2/MUSR": 0.4261, - "hfopenllm_v2/MMLU-PRO": 0.5204 - } - }, - { - "id": "MaziyarPanahi/calme-2.3-llama3.1-70b", - "name": "calme-2.3-llama3.1-70b", - "developer": "MaziyarPanahi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8605, - "hfopenllm_v2/BBH": 0.6872, - "hfopenllm_v2/MATH Level 5": 0.3927, - "hfopenllm_v2/GPQA": 0.344, - "hfopenllm_v2/MUSR": 0.4568, - "hfopenllm_v2/MMLU-PRO": 0.5363 - } - }, - { - "id": "MaziyarPanahi/calme-2.3-phi3-4b", - "name": "calme-2.3-phi3-4b", - "developer": "MaziyarPanahi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4926, - "hfopenllm_v2/BBH": 0.5538, - "hfopenllm_v2/MATH Level 5": 0.1473, - "hfopenllm_v2/GPQA": 0.318, - "hfopenllm_v2/MUSR": 0.3988, - "hfopenllm_v2/MMLU-PRO": 0.3828 - } - }, - { - "id": "MaziyarPanahi/calme-2.3-qwen2-72b", - "name": "calme-2.3-qwen2-72b", - "developer": "MaziyarPanahi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.385, - "hfopenllm_v2/BBH": 0.6576, - "hfopenllm_v2/MATH Level 5": 0.3172, - "hfopenllm_v2/GPQA": 0.3716, - "hfopenllm_v2/MUSR": 0.4112, - "hfopenllm_v2/MMLU-PRO": 0.5419 - } - }, - { - "id": "MaziyarPanahi/calme-2.3-qwen2-7b", - "name": "calme-2.3-qwen2-7b", - "developer": "MaziyarPanahi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3825, - "hfopenllm_v2/BBH": 0.5064, - "hfopenllm_v2/MATH Level 5": 0.2069, - "hfopenllm_v2/GPQA": 0.297, - "hfopenllm_v2/MUSR": 0.4422, - "hfopenllm_v2/MMLU-PRO": 0.3611 - } - }, - { - "id": "MaziyarPanahi/calme-2.3-rys-78b", - "name": "calme-2.3-rys-78b", - "developer": "MaziyarPanahi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8066, - "hfopenllm_v2/BBH": 0.7108, - "hfopenllm_v2/MATH Level 5": 0.398, - "hfopenllm_v2/GPQA": 0.4044, - "hfopenllm_v2/MUSR": 0.4549, - "hfopenllm_v2/MMLU-PRO": 0.5475 - } - }, - { - "id": "MaziyarPanahi/calme-2.4-llama3-70b", - "name": "calme-2.4-llama3-70b", - "developer": "MaziyarPanahi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5027, - "hfopenllm_v2/BBH": 0.6418, - "hfopenllm_v2/MATH Level 5": 0.2447, - "hfopenllm_v2/GPQA": 0.3398, - "hfopenllm_v2/MUSR": 0.4288, - "hfopenllm_v2/MMLU-PRO": 0.5204 - } - }, - { - "id": "MaziyarPanahi/calme-2.4-qwen2-7b", - "name": "calme-2.4-qwen2-7b", - "developer": "MaziyarPanahi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.33, - "hfopenllm_v2/BBH": 0.5101, - "hfopenllm_v2/MATH Level 5": 0.2032, - "hfopenllm_v2/GPQA": 0.2836, - "hfopenllm_v2/MUSR": 0.4453, - "hfopenllm_v2/MMLU-PRO": 0.3977 - } - }, - { - "id": "MaziyarPanahi/calme-2.4-rys-78b", - "name": "calme-2.4-rys-78b", - "developer": "MaziyarPanahi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8011, - "hfopenllm_v2/BBH": 0.728, - "hfopenllm_v2/MATH Level 5": 0.4071, - "hfopenllm_v2/GPQA": 0.4027, - "hfopenllm_v2/MUSR": 0.5771, - "hfopenllm_v2/MMLU-PRO": 0.7002 - } - }, - { - "id": "MaziyarPanahi/calme-2.5-qwen2-7b", - "name": "calme-2.5-qwen2-7b", - "developer": "MaziyarPanahi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3145, - "hfopenllm_v2/BBH": 0.4887, - "hfopenllm_v2/MATH Level 5": 0.2258, - "hfopenllm_v2/GPQA": 0.3104, - "hfopenllm_v2/MUSR": 0.4565, - "hfopenllm_v2/MMLU-PRO": 0.3682 - } - }, - { - "id": "MaziyarPanahi/calme-2.6-qwen2-7b", - "name": "calme-2.6-qwen2-7b", - "developer": "MaziyarPanahi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3443, - "hfopenllm_v2/BBH": 0.493, - "hfopenllm_v2/MATH Level 5": 0.1216, - "hfopenllm_v2/GPQA": 0.2844, - "hfopenllm_v2/MUSR": 0.4586, - "hfopenllm_v2/MMLU-PRO": 0.3732 - } - }, - { - "id": "MaziyarPanahi/calme-2.7-qwen2-7b", - "name": "calme-2.7-qwen2-7b", - "developer": "MaziyarPanahi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3592, - "hfopenllm_v2/BBH": 0.4883, - "hfopenllm_v2/MATH Level 5": 0.1382, - "hfopenllm_v2/GPQA": 0.2911, - "hfopenllm_v2/MUSR": 0.4824, - "hfopenllm_v2/MMLU-PRO": 0.3705 - } - }, - { - "id": "MaziyarPanahi/calme-3.1-baguette-3b", - "name": "calme-3.1-baguette-3b", - "developer": "MaziyarPanahi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6234, - "hfopenllm_v2/BBH": 0.4683, - "hfopenllm_v2/MATH Level 5": 0.256, - "hfopenllm_v2/GPQA": 0.2861, - "hfopenllm_v2/MUSR": 0.4008, - "hfopenllm_v2/MMLU-PRO": 0.3399 - } - }, - { - "id": "MaziyarPanahi/calme-3.1-instruct-3b", - "name": "calme-3.1-instruct-3b", - "developer": "MaziyarPanahi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4336, - "hfopenllm_v2/BBH": 0.4813, - "hfopenllm_v2/MATH Level 5": 0.1775, - "hfopenllm_v2/GPQA": 0.2861, - "hfopenllm_v2/MUSR": 0.3952, - "hfopenllm_v2/MMLU-PRO": 0.3557 - } - }, - { - "id": "MaziyarPanahi/calme-3.1-instruct-78b", - "name": "calme-3.1-instruct-78b", - "developer": "MaziyarPanahi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8136, - "hfopenllm_v2/BBH": 0.7305, - "hfopenllm_v2/MATH Level 5": 0.3927, - "hfopenllm_v2/GPQA": 0.396, - "hfopenllm_v2/MUSR": 0.5891, - "hfopenllm_v2/MMLU-PRO": 0.7185 - } - }, - { - "id": "MaziyarPanahi/calme-3.1-llamaloi-3b", - "name": "calme-3.1-llamaloi-3b", - "developer": "MaziyarPanahi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7375, - "hfopenllm_v2/BBH": 0.4587, - "hfopenllm_v2/MATH Level 5": 0.173, - "hfopenllm_v2/GPQA": 0.281, - "hfopenllm_v2/MUSR": 0.3515, - "hfopenllm_v2/MMLU-PRO": 0.3205 - } - }, - { - "id": "MaziyarPanahi/calme-3.2-baguette-3b", - "name": "calme-3.2-baguette-3b", - "developer": "MaziyarPanahi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6338, - "hfopenllm_v2/BBH": 0.4709, - "hfopenllm_v2/MATH Level 5": 0.2825, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.4021, - "hfopenllm_v2/MMLU-PRO": 0.3338 - } - }, - { - "id": "MaziyarPanahi/calme-3.2-instruct-3b", - "name": "calme-3.2-instruct-3b", - "developer": "MaziyarPanahi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5533, - "hfopenllm_v2/BBH": 0.4866, - "hfopenllm_v2/MATH Level 5": 0.2168, - "hfopenllm_v2/GPQA": 0.2836, - "hfopenllm_v2/MUSR": 0.4047, - "hfopenllm_v2/MMLU-PRO": 0.3653 - } - }, - { - "id": "MaziyarPanahi/calme-3.2-instruct-78b", - "name": "calme-3.2-instruct-78b", - "developer": "MaziyarPanahi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8063, - "hfopenllm_v2/BBH": 0.7319, - "hfopenllm_v2/MATH Level 5": 0.4033, - "hfopenllm_v2/GPQA": 0.4027, - "hfopenllm_v2/MUSR": 0.6024, - "hfopenllm_v2/MMLU-PRO": 0.7303 - } - }, - { - "id": "MaziyarPanahi/calme-3.3-baguette-3b", - "name": "calme-3.3-baguette-3b", - "developer": "MaziyarPanahi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.636, - "hfopenllm_v2/BBH": 0.4678, - "hfopenllm_v2/MATH Level 5": 0.3807, - "hfopenllm_v2/GPQA": 0.2802, - "hfopenllm_v2/MUSR": 0.3928, - "hfopenllm_v2/MMLU-PRO": 0.3342 - } - }, - { - "id": "MaziyarPanahi/calme-3.3-instruct-3b", - "name": "calme-3.3-instruct-3b", - "developer": "MaziyarPanahi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6423, - "hfopenllm_v2/BBH": 0.4693, - "hfopenllm_v2/MATH Level 5": 0.3739, - "hfopenllm_v2/GPQA": 0.2827, - "hfopenllm_v2/MUSR": 0.4074, - "hfopenllm_v2/MMLU-PRO": 0.3305 - } - }, - { - "id": "MaziyarPanahi/Calme-4x7B-MoE-v0.1", - "name": "Calme-4x7B-MoE-v0.1", - "developer": "MaziyarPanahi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4315, - "hfopenllm_v2/BBH": 0.5103, - "hfopenllm_v2/MATH Level 5": 0.0801, - "hfopenllm_v2/GPQA": 0.2819, - "hfopenllm_v2/MUSR": 0.4199, - "hfopenllm_v2/MMLU-PRO": 0.3057 - } - }, - { - "id": "MaziyarPanahi/Calme-4x7B-MoE-v0.2", - "name": "Calme-4x7B-MoE-v0.2", - "developer": "MaziyarPanahi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4294, - "hfopenllm_v2/BBH": 0.5111, - "hfopenllm_v2/MATH Level 5": 0.074, - "hfopenllm_v2/GPQA": 0.2794, - "hfopenllm_v2/MUSR": 0.4318, - "hfopenllm_v2/MMLU-PRO": 0.3058 - } - }, - { - "id": "MaziyarPanahi/Llama-3-70B-Instruct-v0.1", - "name": "Llama-3-70B-Instruct-v0.1", - "developer": "MaziyarPanahi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4714, - "hfopenllm_v2/BBH": 0.5366, - "hfopenllm_v2/MATH Level 5": 0.1805, - "hfopenllm_v2/GPQA": 0.2844, - "hfopenllm_v2/MUSR": 0.4433, - "hfopenllm_v2/MMLU-PRO": 0.4618 - } - }, - { - "id": "MaziyarPanahi/Llama-3-8B-Instruct-v0.10", - "name": "Llama-3-8B-Instruct-v0.10", - "developer": "MaziyarPanahi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7667, - "hfopenllm_v2/BBH": 0.4924, - "hfopenllm_v2/MATH Level 5": 0.0574, - "hfopenllm_v2/GPQA": 0.3087, - "hfopenllm_v2/MUSR": 0.4214, - "hfopenllm_v2/MMLU-PRO": 0.3862 - } - }, - { - "id": "MaziyarPanahi/Llama-3-8B-Instruct-v0.8", - "name": "Llama-3-8B-Instruct-v0.8", - "developer": "MaziyarPanahi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7528, - "hfopenllm_v2/BBH": 0.4963, - "hfopenllm_v2/MATH Level 5": 0.0778, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.4202, - "hfopenllm_v2/MMLU-PRO": 0.3853 - } - }, - { - "id": "MaziyarPanahi/Llama-3-8B-Instruct-v0.9", - "name": "Llama-3-8B-Instruct-v0.9", - "developer": "MaziyarPanahi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.763, - "hfopenllm_v2/BBH": 0.4936, - "hfopenllm_v2/MATH Level 5": 0.0733, - "hfopenllm_v2/GPQA": 0.3079, - "hfopenllm_v2/MUSR": 0.4148, - "hfopenllm_v2/MMLU-PRO": 0.3846 - } - }, - { - "id": "MaziyarPanahi/Qwen1.5-MoE-A2.7B-Wikihow", - "name": "Qwen1.5-MoE-A2.7B-Wikihow", - "developer": "MaziyarPanahi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2954, - "hfopenllm_v2/BBH": 0.392, - "hfopenllm_v2/MATH Level 5": 0.0823, - "hfopenllm_v2/GPQA": 0.2752, - "hfopenllm_v2/MUSR": 0.3502, - "hfopenllm_v2/MMLU-PRO": 0.238 - } - }, - { - "id": "MaziyarPanahi/Qwen2-7B-Instruct-v0.1", - "name": "Qwen2-7B-Instruct-v0.1", - "developer": "MaziyarPanahi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3352, - "hfopenllm_v2/BBH": 0.5123, - "hfopenllm_v2/MATH Level 5": 0.2213, - "hfopenllm_v2/GPQA": 0.2852, - "hfopenllm_v2/MUSR": 0.4435, - "hfopenllm_v2/MMLU-PRO": 0.3857 - } - }, - { - "id": "MaziyarPanahi/Qwen2-7B-Instruct-v0.8", - "name": "Qwen2-7B-Instruct-v0.8", - "developer": "MaziyarPanahi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2775, - "hfopenllm_v2/BBH": 0.4637, - "hfopenllm_v2/MATH Level 5": 0.1767, - "hfopenllm_v2/GPQA": 0.2936, - "hfopenllm_v2/MUSR": 0.4293, - "hfopenllm_v2/MMLU-PRO": 0.3566 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/meditsolutions.json b/data/developers/meditsolutions.json deleted file mode 100644 index ce95036a7cd60f446f0c7ff48fb0d31de55604c7..0000000000000000000000000000000000000000 --- a/data/developers/meditsolutions.json +++ /dev/null @@ -1,173 +0,0 @@ -{ - "developer": "meditsolutions", - "models": [ - { - "id": "meditsolutions/Llama-3.1-MedIT-SUN-8B", - "name": "Llama-3.1-MedIT-SUN-8B", - "developer": "meditsolutions", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7837, - "hfopenllm_v2/BBH": 0.5187, - "hfopenllm_v2/MATH Level 5": 0.2092, - "hfopenllm_v2/GPQA": 0.3087, - "hfopenllm_v2/MUSR": 0.4056, - "hfopenllm_v2/MMLU-PRO": 0.3916 - } - }, - { - "id": "meditsolutions/Llama-3.2-SUN-1B-chat", - "name": "Llama-3.2-SUN-1B-chat", - "developer": "meditsolutions", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5482, - "hfopenllm_v2/BBH": 0.3514, - "hfopenllm_v2/MATH Level 5": 0.0642, - "hfopenllm_v2/GPQA": 0.2617, - "hfopenllm_v2/MUSR": 0.3249, - "hfopenllm_v2/MMLU-PRO": 0.1838 - } - }, - { - "id": "meditsolutions/Llama-3.2-SUN-1B-Instruct", - "name": "Llama-3.2-SUN-1B-Instruct", - "developer": "meditsolutions", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6413, - "hfopenllm_v2/BBH": 0.3474, - "hfopenllm_v2/MATH Level 5": 0.071, - "hfopenllm_v2/GPQA": 0.2424, - "hfopenllm_v2/MUSR": 0.3514, - "hfopenllm_v2/MMLU-PRO": 0.1781 - } - }, - { - "id": "meditsolutions/Llama-3.2-SUN-2.4B-checkpoint-26000", - "name": "Llama-3.2-SUN-2.4B-checkpoint-26000", - "developer": "meditsolutions", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2814, - "hfopenllm_v2/BBH": 0.3018, - "hfopenllm_v2/MATH Level 5": 0.0181, - "hfopenllm_v2/GPQA": 0.2777, - "hfopenllm_v2/MUSR": 0.4103, - "hfopenllm_v2/MMLU-PRO": 0.1345 - } - }, - { - "id": "meditsolutions/Llama-3.2-SUN-2.4B-checkpoint-34800", - "name": "Llama-3.2-SUN-2.4B-checkpoint-34800", - "developer": "meditsolutions", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2501, - "hfopenllm_v2/BBH": 0.3161, - "hfopenllm_v2/MATH Level 5": 0.0106, - "hfopenllm_v2/GPQA": 0.2861, - "hfopenllm_v2/MUSR": 0.4022, - "hfopenllm_v2/MMLU-PRO": 0.1357 - } - }, - { - "id": "meditsolutions/Llama-3.2-SUN-2.4B-v1.0.0", - "name": "Llama-3.2-SUN-2.4B-v1.0.0", - "developer": "meditsolutions", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5637, - "hfopenllm_v2/BBH": 0.3391, - "hfopenllm_v2/MATH Level 5": 0.0627, - "hfopenllm_v2/GPQA": 0.2576, - "hfopenllm_v2/MUSR": 0.3209, - "hfopenllm_v2/MMLU-PRO": 0.1543 - } - }, - { - "id": "meditsolutions/Llama-3.2-SUN-2.5B-chat", - "name": "Llama-3.2-SUN-2.5B-chat", - "developer": "meditsolutions", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5604, - "hfopenllm_v2/BBH": 0.3575, - "hfopenllm_v2/MATH Level 5": 0.071, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3155, - "hfopenllm_v2/MMLU-PRO": 0.1813 - } - }, - { - "id": "meditsolutions/Llama-3.2-SUN-HDIC-1B-Instruct", - "name": "Llama-3.2-SUN-HDIC-1B-Instruct", - "developer": "meditsolutions", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6827, - "hfopenllm_v2/BBH": 0.3508, - "hfopenllm_v2/MATH Level 5": 0.0619, - "hfopenllm_v2/GPQA": 0.2366, - "hfopenllm_v2/MUSR": 0.3594, - "hfopenllm_v2/MMLU-PRO": 0.1687 - } - }, - { - "id": "meditsolutions/MedIT-Mesh-3B-Instruct", - "name": "MedIT-Mesh-3B-Instruct", - "developer": "meditsolutions", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5814, - "hfopenllm_v2/BBH": 0.5576, - "hfopenllm_v2/MATH Level 5": 0.2032, - "hfopenllm_v2/GPQA": 0.3238, - "hfopenllm_v2/MUSR": 0.4048, - "hfopenllm_v2/MMLU-PRO": 0.4012 - } - }, - { - "id": "meditsolutions/MSH-Lite-7B-v1-Bielik-v2.3-Instruct-Llama-Prune", - "name": "MSH-Lite-7B-v1-Bielik-v2.3-Instruct-Llama-Prune", - "developer": "meditsolutions", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3655, - "hfopenllm_v2/BBH": 0.4035, - "hfopenllm_v2/MATH Level 5": 0.0264, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.4253, - "hfopenllm_v2/MMLU-PRO": 0.219 - } - }, - { - "id": "meditsolutions/MSH-v1-Bielik-v2.3-Instruct-MedIT-merge", - "name": "MSH-v1-Bielik-v2.3-Instruct-MedIT-merge", - "developer": "meditsolutions", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5814, - "hfopenllm_v2/BBH": 0.5672, - "hfopenllm_v2/MATH Level 5": 0.2077, - "hfopenllm_v2/GPQA": 0.3456, - "hfopenllm_v2/MUSR": 0.4385, - "hfopenllm_v2/MMLU-PRO": 0.35 - } - }, - { - "id": "meditsolutions/SmolLM2-MedIT-Upscale-2B", - "name": "SmolLM2-MedIT-Upscale-2B", - "developer": "meditsolutions", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6429, - "hfopenllm_v2/BBH": 0.3551, - "hfopenllm_v2/MATH Level 5": 0.0559, - "hfopenllm_v2/GPQA": 0.2643, - "hfopenllm_v2/MUSR": 0.3314, - "hfopenllm_v2/MMLU-PRO": 0.1971 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/meetkai.json b/data/developers/meetkai.json deleted file mode 100644 index d762477ec57ff7e5a541f046ef34cd69f3fb74b9..0000000000000000000000000000000000000000 --- a/data/developers/meetkai.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "meetkai", - "models": [ - { - "id": "meetkai/functionary-small-v3.1", - "name": "functionary-small-v3.1", - "developer": "meetkai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6275, - "hfopenllm_v2/BBH": 0.4982, - "hfopenllm_v2/MATH Level 5": 0.1571, - "hfopenllm_v2/GPQA": 0.2886, - "hfopenllm_v2/MUSR": 0.3834, - "hfopenllm_v2/MMLU-PRO": 0.3349 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/meragpt.json b/data/developers/meragpt.json deleted file mode 100644 index c87574afa87391bfd37d8e239fc87676a126cfb3..0000000000000000000000000000000000000000 --- a/data/developers/meragpt.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "meraGPT", - "models": [ - { - "id": "meraGPT/mera-mix-4x7B", - "name": "mera-mix-4x7B", - "developer": "meraGPT", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4832, - "hfopenllm_v2/BBH": 0.4019, - "hfopenllm_v2/MATH Level 5": 0.0536, - "hfopenllm_v2/GPQA": 0.3045, - "hfopenllm_v2/MUSR": 0.4057, - "hfopenllm_v2/MMLU-PRO": 0.2748 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/mergekit-community.json b/data/developers/mergekit-community.json deleted file mode 100644 index a580cf70c24f8d8399490700bd56e42422af9f07..0000000000000000000000000000000000000000 --- a/data/developers/mergekit-community.json +++ /dev/null @@ -1,159 +0,0 @@ -{ - "developer": "mergekit-community", - "models": [ - { - "id": "mergekit-community/diabolic6045_ELN-AOC-CAIN", - "name": "diabolic6045_ELN-AOC-CAIN", - "developer": "mergekit-community", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0862, - "hfopenllm_v2/BBH": 0.3126, - "hfopenllm_v2/MATH Level 5": 0.0121, - "hfopenllm_v2/GPQA": 0.2634, - "hfopenllm_v2/MUSR": 0.3658, - "hfopenllm_v2/MMLU-PRO": 0.1191 - } - }, - { - "id": "mergekit-community/JAJUKA-WEWILLNEVERFORGETYOU-3B", - "name": "JAJUKA-WEWILLNEVERFORGETYOU-3B", - "developer": "mergekit-community", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4941, - "hfopenllm_v2/BBH": 0.437, - "hfopenllm_v2/MATH Level 5": 0.1246, - "hfopenllm_v2/GPQA": 0.2928, - "hfopenllm_v2/MUSR": 0.3656, - "hfopenllm_v2/MMLU-PRO": 0.3033 - } - }, - { - "id": "mergekit-community/mergekit-dare_ties-ajgjgea", - "name": "mergekit-dare_ties-ajgjgea", - "developer": "mergekit-community", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5263, - "hfopenllm_v2/BBH": 0.3495, - "hfopenllm_v2/MATH Level 5": 0.0642, - "hfopenllm_v2/GPQA": 0.2643, - "hfopenllm_v2/MUSR": 0.3289, - "hfopenllm_v2/MMLU-PRO": 0.1744 - } - }, - { - "id": "mergekit-community/mergekit-della-zgowfmf", - "name": "mergekit-della-zgowfmf", - "developer": "mergekit-community", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4828, - "hfopenllm_v2/BBH": 0.6591, - "hfopenllm_v2/MATH Level 5": 0.3618, - "hfopenllm_v2/GPQA": 0.3901, - "hfopenllm_v2/MUSR": 0.4834, - "hfopenllm_v2/MMLU-PRO": 0.5415 - } - }, - { - "id": "mergekit-community/mergekit-model_stock-azgztvm", - "name": "mergekit-model_stock-azgztvm", - "developer": "mergekit-community", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5062, - "hfopenllm_v2/BBH": 0.6543, - "hfopenllm_v2/MATH Level 5": 0.4373, - "hfopenllm_v2/GPQA": 0.3817, - "hfopenllm_v2/MUSR": 0.473, - "hfopenllm_v2/MMLU-PRO": 0.5406 - } - }, - { - "id": "mergekit-community/mergekit-slerp-fmrazcr", - "name": "mergekit-slerp-fmrazcr", - "developer": "mergekit-community", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4174, - "hfopenllm_v2/BBH": 0.5342, - "hfopenllm_v2/MATH Level 5": 0.1193, - "hfopenllm_v2/GPQA": 0.3112, - "hfopenllm_v2/MUSR": 0.4105, - "hfopenllm_v2/MMLU-PRO": 0.3777 - } - }, - { - "id": "mergekit-community/mergekit-ties-rraxdhv", - "name": "mergekit-ties-rraxdhv", - "developer": "mergekit-community", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1123, - "hfopenllm_v2/BBH": 0.5184, - "hfopenllm_v2/MATH Level 5": 0.04, - "hfopenllm_v2/GPQA": 0.3079, - "hfopenllm_v2/MUSR": 0.4202, - "hfopenllm_v2/MMLU-PRO": 0.391 - } - }, - { - "id": "mergekit-community/mergekit-ties-ykqemwr", - "name": "mergekit-ties-ykqemwr", - "developer": "mergekit-community", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.36, - "hfopenllm_v2/BBH": 0.5455, - "hfopenllm_v2/MATH Level 5": 0.1224, - "hfopenllm_v2/GPQA": 0.3221, - "hfopenllm_v2/MUSR": 0.4198, - "hfopenllm_v2/MMLU-PRO": 0.3734 - } - }, - { - "id": "mergekit-community/sexeh_time_testing", - "name": "sexeh_time_testing", - "developer": "mergekit-community", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7329, - "hfopenllm_v2/BBH": 0.5241, - "hfopenllm_v2/MATH Level 5": 0.0899, - "hfopenllm_v2/GPQA": 0.2911, - "hfopenllm_v2/MUSR": 0.3619, - "hfopenllm_v2/MMLU-PRO": 0.3667 - } - }, - { - "id": "mergekit-community/SuperQwen-2.5-1.5B", - "name": "SuperQwen-2.5-1.5B", - "developer": "mergekit-community", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1336, - "hfopenllm_v2/BBH": 0.2907, - "hfopenllm_v2/MATH Level 5": 0.0196, - "hfopenllm_v2/GPQA": 0.2542, - "hfopenllm_v2/MUSR": 0.3355, - "hfopenllm_v2/MMLU-PRO": 0.1075 - } - }, - { - "id": "mergekit-community/VirtuosoSmall-InstructModelStock", - "name": "VirtuosoSmall-InstructModelStock", - "developer": "mergekit-community", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5238, - "hfopenllm_v2/BBH": 0.6518, - "hfopenllm_v2/MATH Level 5": 0.4094, - "hfopenllm_v2/GPQA": 0.3826, - "hfopenllm_v2/MUSR": 0.4756, - "hfopenllm_v2/MMLU-PRO": 0.5421 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/mescriva.json b/data/developers/mescriva.json deleted file mode 100644 index c6530a91eb22542864578782dab455ed8027a891..0000000000000000000000000000000000000000 --- a/data/developers/mescriva.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "MEscriva", - "models": [ - { - "id": "MEscriva/ECE-PRYMMAL-0.5B-FT-V5-MUSR-Mathis", - "name": "ECE-PRYMMAL-0.5B-FT-V5-MUSR-Mathis", - "developer": "MEscriva", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0866, - "hfopenllm_v2/BBH": 0.3057, - "hfopenllm_v2/MATH Level 5": 0.0106, - "hfopenllm_v2/GPQA": 0.2517, - "hfopenllm_v2/MUSR": 0.4017, - "hfopenllm_v2/MMLU-PRO": 0.1154 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/meta-llama.json b/data/developers/meta-llama.json deleted file mode 100644 index 76d923e16aab37465ed7dda94d7be71e39f4b29e..0000000000000000000000000000000000000000 --- a/data/developers/meta-llama.json +++ /dev/null @@ -1,335 +0,0 @@ -{ - "developer": "meta-llama", - "models": [ - { - "id": "meta-llama/Llama-2-13b-chat-hf", - "name": "Llama-2-13b-chat-hf", - "developer": "meta-llama", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3985, - "hfopenllm_v2/BBH": 0.3343, - "hfopenllm_v2/MATH Level 5": 0.0136, - "hfopenllm_v2/GPQA": 0.2315, - "hfopenllm_v2/MUSR": 0.4007, - "hfopenllm_v2/MMLU-PRO": 0.1923 - } - }, - { - "id": "meta-llama/Llama-2-13b-hf", - "name": "Llama-2-13b-hf", - "developer": "meta-llama", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2482, - "hfopenllm_v2/BBH": 0.4126, - "hfopenllm_v2/MATH Level 5": 0.0151, - "hfopenllm_v2/GPQA": 0.281, - "hfopenllm_v2/MUSR": 0.3538, - "hfopenllm_v2/MMLU-PRO": 0.2378 - } - }, - { - "id": "meta-llama/Llama-2-70b-chat-hf", - "name": "Llama-2-70b-chat-hf", - "developer": "meta-llama", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4958, - "hfopenllm_v2/BBH": 0.3042, - "hfopenllm_v2/MATH Level 5": 0.0295, - "hfopenllm_v2/GPQA": 0.2643, - "hfopenllm_v2/MUSR": 0.3687, - "hfopenllm_v2/MMLU-PRO": 0.2433 - } - }, - { - "id": "meta-llama/Llama-2-70b-hf", - "name": "Llama-2-70b-hf", - "developer": "meta-llama", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2407, - "hfopenllm_v2/BBH": 0.5473, - "hfopenllm_v2/MATH Level 5": 0.0325, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.4124, - "hfopenllm_v2/MMLU-PRO": 0.3718 - } - }, - { - "id": "meta-llama/Llama-2-7b-chat-hf", - "name": "Llama-2-7b-chat-hf", - "developer": "meta-llama", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3986, - "hfopenllm_v2/BBH": 0.3114, - "hfopenllm_v2/MATH Level 5": 0.0196, - "hfopenllm_v2/GPQA": 0.2534, - "hfopenllm_v2/MUSR": 0.3676, - "hfopenllm_v2/MMLU-PRO": 0.1688 - } - }, - { - "id": "meta-llama/Llama-2-7b-hf", - "name": "Llama-2-7b-hf", - "developer": "meta-llama", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2519, - "hfopenllm_v2/BBH": 0.3496, - "hfopenllm_v2/MATH Level 5": 0.0174, - "hfopenllm_v2/GPQA": 0.2668, - "hfopenllm_v2/MUSR": 0.3701, - "hfopenllm_v2/MMLU-PRO": 0.1861 - } - }, - { - "id": "meta-llama/Llama-3.1-70B", - "name": "Llama-3.1-70B", - "developer": "meta-llama", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1684, - "hfopenllm_v2/BBH": 0.626, - "hfopenllm_v2/MATH Level 5": 0.1843, - "hfopenllm_v2/GPQA": 0.3876, - "hfopenllm_v2/MUSR": 0.4572, - "hfopenllm_v2/MMLU-PRO": 0.4654 - } - }, - { - "id": "meta-llama/Llama-3.1-70B-Instruct", - "name": "Llama-3.1-70B-Instruct", - "developer": "meta-llama", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8669, - "hfopenllm_v2/BBH": 0.6917, - "hfopenllm_v2/MATH Level 5": 0.3807, - "hfopenllm_v2/GPQA": 0.3565, - "hfopenllm_v2/MUSR": 0.4581, - "hfopenllm_v2/MMLU-PRO": 0.5309 - } - }, - { - "id": "meta-llama/Llama-3.1-8B", - "name": "Llama-3.1-8B", - "developer": "meta-llama", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1246, - "hfopenllm_v2/BBH": 0.466, - "hfopenllm_v2/MATH Level 5": 0.0657, - "hfopenllm_v2/GPQA": 0.3104, - "hfopenllm_v2/MUSR": 0.3812, - "hfopenllm_v2/MMLU-PRO": 0.3288 - } - }, - { - "id": "meta-llama/Llama-3.1-8B-Instruct", - "name": "Llama-3.1-8B-Instruct", - "developer": "meta-llama", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4922, - "hfopenllm_v2/BBH": 0.5087, - "hfopenllm_v2/MATH Level 5": 0.1556, - "hfopenllm_v2/GPQA": 0.3154, - "hfopenllm_v2/MUSR": 0.3972, - "hfopenllm_v2/MMLU-PRO": 0.3798 - } - }, - { - "id": "meta-llama/Llama-3.2-1B", - "name": "Llama-3.2-1B", - "developer": "meta-llama", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1478, - "hfopenllm_v2/BBH": 0.3115, - "hfopenllm_v2/MATH Level 5": 0.0121, - "hfopenllm_v2/GPQA": 0.2282, - "hfopenllm_v2/MUSR": 0.3447, - "hfopenllm_v2/MMLU-PRO": 0.1203 - } - }, - { - "id": "meta-llama/Llama-3.2-1B-Instruct", - "name": "Llama-3.2-1B-Instruct", - "developer": "meta-llama", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5698, - "hfopenllm_v2/BBH": 0.3497, - "hfopenllm_v2/MATH Level 5": 0.0702, - "hfopenllm_v2/GPQA": 0.2752, - "hfopenllm_v2/MUSR": 0.3329, - "hfopenllm_v2/MMLU-PRO": 0.1682 - } - }, - { - "id": "meta-llama/Llama-3.2-3B", - "name": "Llama-3.2-3B", - "developer": "meta-llama", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1337, - "hfopenllm_v2/BBH": 0.3905, - "hfopenllm_v2/MATH Level 5": 0.0189, - "hfopenllm_v2/GPQA": 0.2676, - "hfopenllm_v2/MUSR": 0.3577, - "hfopenllm_v2/MMLU-PRO": 0.2488 - } - }, - { - "id": "meta-llama/Llama-3.2-3B-Instruct", - "name": "Llama-3.2-3B-Instruct", - "developer": "meta-llama", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7393, - "hfopenllm_v2/BBH": 0.461, - "hfopenllm_v2/MATH Level 5": 0.1767, - "hfopenllm_v2/GPQA": 0.2785, - "hfopenllm_v2/MUSR": 0.3529, - "hfopenllm_v2/MMLU-PRO": 0.3195 - } - }, - { - "id": "meta-llama/Llama-3.3-70B-Instruct", - "name": "Llama-3.3-70B-Instruct", - "developer": "meta-llama", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8998, - "hfopenllm_v2/BBH": 0.6919, - "hfopenllm_v2/MATH Level 5": 0.4834, - "hfopenllm_v2/GPQA": 0.3289, - "hfopenllm_v2/MUSR": 0.4461, - "hfopenllm_v2/MMLU-PRO": 0.5332 - } - }, - { - "id": "meta-llama/Meta-Llama-3-70B", - "name": "Meta-Llama-3-70B", - "developer": "meta-llama", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1603, - "hfopenllm_v2/BBH": 0.6461, - "hfopenllm_v2/MATH Level 5": 0.1858, - "hfopenllm_v2/GPQA": 0.3977, - "hfopenllm_v2/MUSR": 0.4518, - "hfopenllm_v2/MMLU-PRO": 0.4709 - } - }, - { - "id": "meta-llama/Meta-Llama-3-70B-Instruct", - "name": "Meta-Llama-3-70B-Instruct", - "developer": "meta-llama", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8099, - "hfopenllm_v2/BBH": 0.6547, - "hfopenllm_v2/MATH Level 5": 0.2447, - "hfopenllm_v2/GPQA": 0.2869, - "hfopenllm_v2/MUSR": 0.4154, - "hfopenllm_v2/MMLU-PRO": 0.5207, - "reward-bench/Score": 0.7627, - "reward-bench/Chat": 0.9763, - "reward-bench/Chat Hard": 0.5888, - "reward-bench/Safety": 0.7297, - "reward-bench/Reasoning": 0.7854, - "reward-bench/Prior Sets (0.5 weight)": 0.7035 - } - }, - { - "id": "meta-llama/Meta-Llama-3-8B", - "name": "Meta-Llama-3-8B", - "developer": "meta-llama", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1455, - "hfopenllm_v2/BBH": 0.4598, - "hfopenllm_v2/MATH Level 5": 0.0453, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.3614, - "hfopenllm_v2/MMLU-PRO": 0.321 - } - }, - { - "id": "meta-llama/Meta-Llama-3-8B-Instruct", - "name": "Meta-Llama-3-8B-Instruct", - "developer": "meta-llama", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7408, - "hfopenllm_v2/BBH": 0.4989, - "hfopenllm_v2/MATH Level 5": 0.0869, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3568, - "hfopenllm_v2/MMLU-PRO": 0.3664, - "reward-bench/Score": 0.645, - "reward-bench/Chat": 0.8547, - "reward-bench/Chat Hard": 0.4156, - "reward-bench/Safety": 0.6797, - "reward-bench/Reasoning": 0.6482, - "reward-bench/Prior Sets (0.5 weight)": 0.6082 - } - }, - { - "id": "meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo", - "name": "meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo", - "developer": "meta-llama", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.8412, - "reward-bench/Chat": 0.9721, - "reward-bench/Chat Hard": 0.7456, - "reward-bench/Safety": 0.7757, - "reward-bench/Reasoning": 0.8715 - } - }, - { - "id": "meta-llama/Meta-Llama-3.1-70B-Instruct", - "name": "meta-llama/Meta-Llama-3.1-70B-Instruct", - "developer": "meta-llama", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.8405, - "reward-bench/Chat": 0.9721, - "reward-bench/Chat Hard": 0.7018, - "reward-bench/Safety": 0.8284, - "reward-bench/Reasoning": 0.8599 - } - }, - { - "id": "meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo", - "name": "meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo", - "developer": "meta-llama", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7808, - "reward-bench/Chat": 0.8757, - "reward-bench/Chat Hard": 0.6689, - "reward-bench/Safety": 0.7507, - "reward-bench/Reasoning": 0.828 - } - }, - { - "id": "meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo", - "name": "meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo", - "developer": "meta-llama", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6565, - "reward-bench/Chat": 0.8073, - "reward-bench/Chat Hard": 0.4978, - "reward-bench/Safety": 0.6399, - "reward-bench/Reasoning": 0.6811 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/meta-metrics.json b/data/developers/meta-metrics.json deleted file mode 100644 index de6bbc909c82311f9b8856a2e528b221f7eb309b..0000000000000000000000000000000000000000 --- a/data/developers/meta-metrics.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "developer": "meta-metrics", - "models": [ - { - "id": "meta-metrics/MetaMetrics-RM-v1.0", - "name": "meta-metrics/MetaMetrics-RM-v1.0", - "developer": "meta-metrics", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.9342, - "reward-bench/Chat": 0.9832, - "reward-bench/Chat Hard": 0.864, - "reward-bench/Safety": 0.9081, - "reward-bench/Reasoning": 0.9816 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/meta.json b/data/developers/meta.json deleted file mode 100644 index 497b58d174e8d750c71dcd43b8d5e2daa877ceed..0000000000000000000000000000000000000000 --- a/data/developers/meta.json +++ /dev/null @@ -1,1079 +0,0 @@ -{ - "developer": "Meta", - "models": [ - { - "id": "meta/LLaMA-13B", - "name": "LLaMA 13B", - "developer": "Meta", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.595, - "helm_classic/MMLU": 0.422, - "helm_classic/BoolQ": 0.714, - "helm_classic/NarrativeQA": 0.711, - "helm_classic/NaturalQuestions (open-book)": 0.614, - "helm_classic/QuAC": 0.347, - "helm_classic/HellaSwag": -1.0, - "helm_classic/OpenbookQA": -1.0, - "helm_classic/TruthfulQA": 0.324, - "helm_classic/MS MARCO (TREC)": -1.0, - "helm_classic/CNN/DailyMail": -1.0, - "helm_classic/XSUM": -1.0, - "helm_classic/IMDB": 0.928, - "helm_classic/CivilComments": 0.6, - "helm_classic/RAFT": 0.643 - } - }, - { - "id": "meta/llama-2-13b", - "name": "Llama 2 13B", - "developer": "Meta", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.823, - "helm_classic/MMLU": 0.507, - "helm_classic/BoolQ": 0.811, - "helm_classic/NarrativeQA": 0.744, - "helm_classic/NaturalQuestions (open-book)": 0.637, - "helm_classic/QuAC": 0.424, - "helm_classic/HellaSwag": -1.0, - "helm_classic/OpenbookQA": -1.0, - "helm_classic/TruthfulQA": 0.33, - "helm_classic/MS MARCO (TREC)": -1.0, - "helm_classic/CNN/DailyMail": -1.0, - "helm_classic/XSUM": -1.0, - "helm_classic/IMDB": 0.962, - "helm_classic/CivilComments": 0.588, - "helm_classic/RAFT": 0.707, - "helm_lite/Mean win rate": 0.233, - "helm_lite/NarrativeQA": 0.741, - "helm_lite/NaturalQuestions (closed-book)": 0.371, - "helm_lite/OpenbookQA": 0.634, - "helm_lite/MMLU": 0.505, - "helm_lite/MATH": 0.102, - "helm_lite/GSM8K": 0.266, - "helm_lite/LegalBench": 0.591, - "helm_lite/MedQA": 0.392, - "helm_lite/WMT 2014": 0.167, - "helm_mmlu/MMLU All Subjects": 0.554, - "helm_mmlu/Abstract Algebra": 0.27, - "helm_mmlu/Anatomy": 0.496, - "helm_mmlu/College Physics": 0.235, - "helm_mmlu/Computer Security": 0.69, - "helm_mmlu/Econometrics": 0.307, - "helm_mmlu/Global Facts": 0.38, - "helm_mmlu/Jurisprudence": 0.704, - "helm_mmlu/Philosophy": 0.672, - "helm_mmlu/Professional Psychology": 0.567, - "helm_mmlu/Us Foreign Policy": 0.83, - "helm_mmlu/Astronomy": 0.546, - "helm_mmlu/Business Ethics": 0.55, - "helm_mmlu/Clinical Knowledge": 0.592, - "helm_mmlu/Conceptual Physics": 0.413, - "helm_mmlu/Electrical Engineering": 0.49, - "helm_mmlu/Elementary Mathematics": 0.307, - "helm_mmlu/Formal Logic": 0.381, - "helm_mmlu/High School World History": 0.705, - "helm_mmlu/Human Sexuality": 0.618, - "helm_mmlu/International Law": 0.752, - "helm_mmlu/Logical Fallacies": 0.687, - "helm_mmlu/Machine Learning": 0.286, - "helm_mmlu/Management": 0.738, - "helm_mmlu/Marketing": 0.786, - "helm_mmlu/Medical Genetics": 0.57, - "helm_mmlu/Miscellaneous": 0.748, - "helm_mmlu/Moral Scenarios": 0.407, - "helm_mmlu/Nutrition": 0.627, - "helm_mmlu/Prehistory": 0.654, - "helm_mmlu/Public Relations": 0.6, - "helm_mmlu/Security Studies": 0.608, - "helm_mmlu/Sociology": 0.761, - "helm_mmlu/Virology": 0.476, - "helm_mmlu/World Religions": 0.76, - "helm_mmlu/Mean win rate": 0.502 - } - }, - { - "id": "meta/llama-2-70b", - "name": "Llama 2 70B", - "developer": "Meta", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.944, - "helm_classic/MMLU": 0.582, - "helm_classic/BoolQ": 0.886, - "helm_classic/NarrativeQA": 0.77, - "helm_classic/NaturalQuestions (open-book)": 0.674, - "helm_classic/QuAC": 0.484, - "helm_classic/HellaSwag": -1.0, - "helm_classic/OpenbookQA": -1.0, - "helm_classic/TruthfulQA": 0.554, - "helm_classic/MS MARCO (TREC)": -1.0, - "helm_classic/CNN/DailyMail": -1.0, - "helm_classic/XSUM": -1.0, - "helm_classic/IMDB": 0.961, - "helm_classic/CivilComments": 0.652, - "helm_classic/RAFT": 0.727, - "helm_lite/Mean win rate": 0.482, - "helm_lite/NarrativeQA": 0.763, - "helm_lite/NaturalQuestions (closed-book)": 0.46, - "helm_lite/OpenbookQA": 0.838, - "helm_lite/MMLU": 0.58, - "helm_lite/MATH": 0.323, - "helm_lite/GSM8K": 0.567, - "helm_lite/LegalBench": 0.673, - "helm_lite/MedQA": 0.618, - "helm_lite/WMT 2014": 0.196, - "helm_mmlu/MMLU All Subjects": 0.695, - "helm_mmlu/Abstract Algebra": 0.31, - "helm_mmlu/Anatomy": 0.607, - "helm_mmlu/College Physics": 0.363, - "helm_mmlu/Computer Security": 0.77, - "helm_mmlu/Econometrics": 0.43, - "helm_mmlu/Global Facts": 0.47, - "helm_mmlu/Jurisprudence": 0.824, - "helm_mmlu/Philosophy": 0.791, - "helm_mmlu/Professional Psychology": 0.76, - "helm_mmlu/Us Foreign Policy": 0.92, - "helm_mmlu/Astronomy": 0.829, - "helm_mmlu/Business Ethics": 0.73, - "helm_mmlu/Clinical Knowledge": 0.717, - "helm_mmlu/Conceptual Physics": 0.668, - "helm_mmlu/Electrical Engineering": 0.634, - "helm_mmlu/Elementary Mathematics": 0.421, - "helm_mmlu/Formal Logic": 0.468, - "helm_mmlu/High School World History": 0.882, - "helm_mmlu/Human Sexuality": 0.84, - "helm_mmlu/International Law": 0.868, - "helm_mmlu/Logical Fallacies": 0.791, - "helm_mmlu/Machine Learning": 0.491, - "helm_mmlu/Management": 0.845, - "helm_mmlu/Marketing": 0.889, - "helm_mmlu/Medical Genetics": 0.72, - "helm_mmlu/Miscellaneous": 0.857, - "helm_mmlu/Moral Scenarios": 0.45, - "helm_mmlu/Nutrition": 0.758, - "helm_mmlu/Prehistory": 0.84, - "helm_mmlu/Public Relations": 0.745, - "helm_mmlu/Security Studies": 0.796, - "helm_mmlu/Sociology": 0.9, - "helm_mmlu/Virology": 0.53, - "helm_mmlu/World Religions": 0.854, - "helm_mmlu/Mean win rate": 0.508 - } - }, - { - "id": "meta/llama-2-7b", - "name": "Llama 2 7B", - "developer": "Meta", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.607, - "helm_classic/MMLU": 0.431, - "helm_classic/BoolQ": 0.762, - "helm_classic/NarrativeQA": 0.691, - "helm_classic/NaturalQuestions (open-book)": 0.611, - "helm_classic/QuAC": 0.406, - "helm_classic/HellaSwag": -1.0, - "helm_classic/OpenbookQA": -1.0, - "helm_classic/TruthfulQA": 0.272, - "helm_classic/MS MARCO (TREC)": -1.0, - "helm_classic/CNN/DailyMail": -1.0, - "helm_classic/XSUM": -1.0, - "helm_classic/IMDB": 0.907, - "helm_classic/CivilComments": 0.562, - "helm_classic/RAFT": 0.643, - "helm_lite/Mean win rate": 0.152, - "helm_lite/NarrativeQA": 0.686, - "helm_lite/NaturalQuestions (closed-book)": 0.333, - "helm_lite/OpenbookQA": 0.544, - "helm_lite/MMLU": 0.425, - "helm_lite/MATH": 0.097, - "helm_lite/GSM8K": 0.154, - "helm_lite/LegalBench": 0.502, - "helm_lite/MedQA": 0.392, - "helm_lite/WMT 2014": 0.144, - "helm_mmlu/MMLU All Subjects": 0.458, - "helm_mmlu/Abstract Algebra": 0.29, - "helm_mmlu/Anatomy": 0.452, - "helm_mmlu/College Physics": 0.196, - "helm_mmlu/Computer Security": 0.59, - "helm_mmlu/Econometrics": 0.316, - "helm_mmlu/Global Facts": 0.29, - "helm_mmlu/Jurisprudence": 0.519, - "helm_mmlu/Philosophy": 0.592, - "helm_mmlu/Professional Psychology": 0.459, - "helm_mmlu/Us Foreign Policy": 0.64, - "helm_mmlu/Astronomy": 0.408, - "helm_mmlu/Business Ethics": 0.48, - "helm_mmlu/Clinical Knowledge": 0.453, - "helm_mmlu/Conceptual Physics": 0.434, - "helm_mmlu/Electrical Engineering": 0.407, - "helm_mmlu/Elementary Mathematics": 0.254, - "helm_mmlu/Formal Logic": 0.27, - "helm_mmlu/High School World History": 0.662, - "helm_mmlu/Human Sexuality": 0.557, - "helm_mmlu/International Law": 0.628, - "helm_mmlu/Logical Fallacies": 0.466, - "helm_mmlu/Machine Learning": 0.402, - "helm_mmlu/Management": 0.563, - "helm_mmlu/Marketing": 0.697, - "helm_mmlu/Medical Genetics": 0.53, - "helm_mmlu/Miscellaneous": 0.632, - "helm_mmlu/Moral Scenarios": 0.238, - "helm_mmlu/Nutrition": 0.497, - "helm_mmlu/Prehistory": 0.503, - "helm_mmlu/Public Relations": 0.509, - "helm_mmlu/Security Studies": 0.433, - "helm_mmlu/Sociology": 0.617, - "helm_mmlu/Virology": 0.392, - "helm_mmlu/World Religions": 0.713, - "helm_mmlu/Mean win rate": 0.681 - } - }, - { - "id": "meta/llama-3-1-8b-instruct-prompt", - "name": "Llama-3.1-8B-Instruct (Prompt)", - "developer": "Meta", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 85.0, - "bfcl/bfcl.overall.overall_accuracy": 25.83, - "bfcl/bfcl.overall.total_cost_usd": 7.49, - "bfcl/bfcl.overall.latency_mean_s": 5.6, - "bfcl/bfcl.overall.latency_std_s": 19.37, - "bfcl/bfcl.overall.latency_p95_s": 22.6, - "bfcl/bfcl.non_live.ast_accuracy": 84.0, - "bfcl/bfcl.non_live.simple_ast_accuracy": 71.0, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 95.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 87.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 82.5, - "bfcl/bfcl.live.live_accuracy": 70.76, - "bfcl/bfcl.live.live_simple_ast_accuracy": 72.87, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 71.13, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 50.0, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 45.83, - "bfcl/bfcl.multi_turn.accuracy": 11.12, - "bfcl/bfcl.multi_turn.base_accuracy": 13.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 9.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 9.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 13.0, - "bfcl/bfcl.web_search.accuracy": 3.0, - "bfcl/bfcl.web_search.base_accuracy": 6.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 0.0, - "bfcl/bfcl.memory.accuracy": 10.75, - "bfcl/bfcl.memory.kv_accuracy": 7.74, - "bfcl/bfcl.memory.vector_accuracy": 5.81, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 18.71, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 93.75, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 42.7, - "bfcl/bfcl.format_sensitivity.max_delta": 74.5, - "bfcl/bfcl.format_sensitivity.stddev": 29.1 - } - }, - { - "id": "meta/llama-3-2-1b-instruct-fc", - "name": "Llama-3.2-1B-Instruct (FC)", - "developer": "Meta", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 107.0, - "bfcl/bfcl.overall.overall_accuracy": 10.82, - "bfcl/bfcl.overall.total_cost_usd": 1.64, - "bfcl/bfcl.overall.latency_mean_s": 3.21, - "bfcl/bfcl.overall.latency_std_s": 10.04, - "bfcl/bfcl.overall.latency_p95_s": 9.77, - "bfcl/bfcl.non_live.ast_accuracy": 38.38, - "bfcl/bfcl.non_live.simple_ast_accuracy": 44.0, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 50.5, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 44.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 15.0, - "bfcl/bfcl.live.live_accuracy": 11.77, - "bfcl/bfcl.live.live_simple_ast_accuracy": 31.78, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 7.31, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 0.0, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 0.0, - "bfcl/bfcl.multi_turn.accuracy": 0.0, - "bfcl/bfcl.multi_turn.base_accuracy": 0.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 0.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 0.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 0.0, - "bfcl/bfcl.web_search.accuracy": 0.0, - "bfcl/bfcl.web_search.base_accuracy": 0.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 0.0, - "bfcl/bfcl.memory.accuracy": 3.23, - "bfcl/bfcl.memory.kv_accuracy": 2.58, - "bfcl/bfcl.memory.vector_accuracy": 2.58, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 4.52, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 43.75, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 51.57 - } - }, - { - "id": "meta/llama-3-2-3b-instruct-fc", - "name": "Llama-3.2-3B-Instruct (FC)", - "developer": "Meta", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 98.0, - "bfcl/bfcl.overall.overall_accuracy": 21.95, - "bfcl/bfcl.overall.total_cost_usd": 6.2, - "bfcl/bfcl.overall.latency_mean_s": 6.1, - "bfcl/bfcl.overall.latency_std_s": 20.07, - "bfcl/bfcl.overall.latency_p95_s": 17.27, - "bfcl/bfcl.non_live.ast_accuracy": 82.67, - "bfcl/bfcl.non_live.simple_ast_accuracy": 70.67, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 92.5, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 88.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 79.0, - "bfcl/bfcl.live.live_accuracy": 58.33, - "bfcl/bfcl.live.live_simple_ast_accuracy": 65.12, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 57.64, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 25.0, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 37.5, - "bfcl/bfcl.multi_turn.accuracy": 4.0, - "bfcl/bfcl.multi_turn.base_accuracy": 5.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 3.5, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 4.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 3.5, - "bfcl/bfcl.web_search.accuracy": 1.0, - "bfcl/bfcl.web_search.base_accuracy": 2.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 0.0, - "bfcl/bfcl.memory.accuracy": 6.24, - "bfcl/bfcl.memory.kv_accuracy": 3.23, - "bfcl/bfcl.memory.vector_accuracy": 3.23, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 12.26, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 87.5, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 52.06 - } - }, - { - "id": "meta/llama-3-3-70b-instruct-fc", - "name": "Llama-3.3-70B-Instruct (FC)", - "developer": "Meta", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 62.0, - "bfcl/bfcl.overall.overall_accuracy": 31.9, - "bfcl/bfcl.overall.total_cost_usd": 29.54, - "bfcl/bfcl.overall.latency_mean_s": 26.11, - "bfcl/bfcl.overall.latency_std_s": 93.22, - "bfcl/bfcl.overall.latency_p95_s": 187.93, - "bfcl/bfcl.non_live.ast_accuracy": 88.02, - "bfcl/bfcl.non_live.simple_ast_accuracy": 76.08, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 95.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 90.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 91.0, - "bfcl/bfcl.live.live_accuracy": 76.61, - "bfcl/bfcl.live.live_simple_ast_accuracy": 81.4, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 75.5, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 81.25, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 70.83, - "bfcl/bfcl.multi_turn.accuracy": 21.5, - "bfcl/bfcl.multi_turn.base_accuracy": 26.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 19.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 14.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 26.5, - "bfcl/bfcl.web_search.accuracy": 10.0, - "bfcl/bfcl.web_search.base_accuracy": 14.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 6.0, - "bfcl/bfcl.memory.accuracy": 8.17, - "bfcl/bfcl.memory.kv_accuracy": 4.52, - "bfcl/bfcl.memory.vector_accuracy": 8.39, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 11.61, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 100.0, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 53.53 - } - }, - { - "id": "meta/llama-3-70b", - "name": "Llama 3 70B", - "developer": "Meta", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.793, - "helm_lite/NarrativeQA": 0.798, - "helm_lite/NaturalQuestions (closed-book)": 0.475, - "helm_lite/OpenbookQA": 0.934, - "helm_lite/MMLU": 0.695, - "helm_lite/MATH": 0.663, - "helm_lite/GSM8K": 0.805, - "helm_lite/LegalBench": 0.733, - "helm_lite/MedQA": 0.777, - "helm_lite/WMT 2014": 0.225, - "helm_mmlu/MMLU All Subjects": 0.793, - "helm_mmlu/Abstract Algebra": 0.43, - "helm_mmlu/Anatomy": 0.785, - "helm_mmlu/College Physics": 0.529, - "helm_mmlu/Computer Security": 0.85, - "helm_mmlu/Econometrics": 0.693, - "helm_mmlu/Global Facts": 0.49, - "helm_mmlu/Jurisprudence": 0.861, - "helm_mmlu/Philosophy": 0.865, - "helm_mmlu/Professional Psychology": 0.871, - "helm_mmlu/Us Foreign Policy": 0.94, - "helm_mmlu/Astronomy": 0.921, - "helm_mmlu/Business Ethics": 0.83, - "helm_mmlu/Clinical Knowledge": 0.845, - "helm_mmlu/Conceptual Physics": 0.838, - "helm_mmlu/Electrical Engineering": 0.766, - "helm_mmlu/Elementary Mathematics": 0.632, - "helm_mmlu/Formal Logic": 0.651, - "helm_mmlu/High School World History": 0.941, - "helm_mmlu/Human Sexuality": 0.878, - "helm_mmlu/International Law": 0.901, - "helm_mmlu/Logical Fallacies": 0.865, - "helm_mmlu/Machine Learning": 0.714, - "helm_mmlu/Management": 0.913, - "helm_mmlu/Marketing": 0.94, - "helm_mmlu/Medical Genetics": 0.89, - "helm_mmlu/Miscellaneous": 0.917, - "helm_mmlu/Moral Scenarios": 0.598, - "helm_mmlu/Nutrition": 0.876, - "helm_mmlu/Prehistory": 0.91, - "helm_mmlu/Public Relations": 0.727, - "helm_mmlu/Security Studies": 0.833, - "helm_mmlu/Sociology": 0.93, - "helm_mmlu/Virology": 0.59, - "helm_mmlu/World Religions": 0.906, - "helm_mmlu/Mean win rate": 0.524 - } - }, - { - "id": "meta/llama-3-8b", - "name": "Llama 3 8B", - "developer": "Meta", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.387, - "helm_lite/NarrativeQA": 0.754, - "helm_lite/NaturalQuestions (closed-book)": 0.378, - "helm_lite/OpenbookQA": 0.766, - "helm_lite/MMLU": 0.602, - "helm_lite/MATH": 0.391, - "helm_lite/GSM8K": 0.499, - "helm_lite/LegalBench": 0.637, - "helm_lite/MedQA": 0.581, - "helm_lite/WMT 2014": 0.183, - "helm_mmlu/MMLU All Subjects": 0.668, - "helm_mmlu/Abstract Algebra": 0.33, - "helm_mmlu/Anatomy": 0.696, - "helm_mmlu/College Physics": 0.451, - "helm_mmlu/Computer Security": 0.8, - "helm_mmlu/Econometrics": 0.518, - "helm_mmlu/Global Facts": 0.34, - "helm_mmlu/Jurisprudence": 0.741, - "helm_mmlu/Philosophy": 0.743, - "helm_mmlu/Professional Psychology": 0.711, - "helm_mmlu/Us Foreign Policy": 0.88, - "helm_mmlu/Astronomy": 0.711, - "helm_mmlu/Business Ethics": 0.65, - "helm_mmlu/Clinical Knowledge": 0.751, - "helm_mmlu/Conceptual Physics": 0.557, - "helm_mmlu/Electrical Engineering": 0.669, - "helm_mmlu/Elementary Mathematics": 0.426, - "helm_mmlu/Formal Logic": 0.468, - "helm_mmlu/High School World History": 0.823, - "helm_mmlu/Human Sexuality": 0.748, - "helm_mmlu/International Law": 0.843, - "helm_mmlu/Logical Fallacies": 0.755, - "helm_mmlu/Machine Learning": 0.545, - "helm_mmlu/Management": 0.874, - "helm_mmlu/Marketing": 0.885, - "helm_mmlu/Medical Genetics": 0.83, - "helm_mmlu/Miscellaneous": 0.831, - "helm_mmlu/Moral Scenarios": 0.416, - "helm_mmlu/Nutrition": 0.761, - "helm_mmlu/Prehistory": 0.738, - "helm_mmlu/Public Relations": 0.736, - "helm_mmlu/Security Studies": 0.771, - "helm_mmlu/Sociology": 0.866, - "helm_mmlu/Virology": 0.566, - "helm_mmlu/World Religions": 0.819, - "helm_mmlu/Mean win rate": 0.733 - } - }, - { - "id": "meta/llama-3.1-405b-instruct-turbo", - "name": "Llama 3.1 Instruct Turbo 405B", - "developer": "Meta", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.618, - "helm_capabilities/MMLU-Pro": 0.723, - "helm_capabilities/GPQA": 0.522, - "helm_capabilities/IFEval": 0.811, - "helm_capabilities/WildBench": 0.783, - "helm_capabilities/Omni-MATH": 0.249, - "helm_lite/Mean win rate": 0.854, - "helm_lite/NarrativeQA": 0.749, - "helm_lite/NaturalQuestions (closed-book)": 0.456, - "helm_lite/OpenbookQA": 0.94, - "helm_lite/MMLU": 0.759, - "helm_lite/MATH": 0.827, - "helm_lite/GSM8K": 0.949, - "helm_lite/LegalBench": 0.707, - "helm_lite/MedQA": 0.805, - "helm_lite/WMT 2014": 0.238, - "helm_mmlu/MMLU All Subjects": 0.845, - "helm_mmlu/Abstract Algebra": 0.7, - "helm_mmlu/Anatomy": 0.822, - "helm_mmlu/College Physics": 0.696, - "helm_mmlu/Computer Security": 0.81, - "helm_mmlu/Econometrics": 0.746, - "helm_mmlu/Global Facts": 0.71, - "helm_mmlu/Jurisprudence": 0.87, - "helm_mmlu/Philosophy": 0.878, - "helm_mmlu/Professional Psychology": 0.861, - "helm_mmlu/Us Foreign Policy": 0.94, - "helm_mmlu/Astronomy": 0.921, - "helm_mmlu/Business Ethics": 0.81, - "helm_mmlu/Clinical Knowledge": 0.879, - "helm_mmlu/Conceptual Physics": 0.877, - "helm_mmlu/Electrical Engineering": 0.821, - "helm_mmlu/Elementary Mathematics": 0.828, - "helm_mmlu/Formal Logic": 0.698, - "helm_mmlu/High School World History": 0.941, - "helm_mmlu/Human Sexuality": 0.855, - "helm_mmlu/International Law": 0.95, - "helm_mmlu/Logical Fallacies": 0.92, - "helm_mmlu/Machine Learning": 0.795, - "helm_mmlu/Management": 0.893, - "helm_mmlu/Marketing": 0.962, - "helm_mmlu/Medical Genetics": 0.93, - "helm_mmlu/Miscellaneous": 0.939, - "helm_mmlu/Moral Scenarios": 0.876, - "helm_mmlu/Nutrition": 0.928, - "helm_mmlu/Prehistory": 0.929, - "helm_mmlu/Public Relations": 0.818, - "helm_mmlu/Security Studies": 0.857, - "helm_mmlu/Sociology": 0.94, - "helm_mmlu/Virology": 0.572, - "helm_mmlu/World Religions": 0.906, - "helm_mmlu/Mean win rate": 0.33 - } - }, - { - "id": "meta/llama-3.1-70b-instruct-turbo", - "name": "Llama 3.1 Instruct Turbo 70B", - "developer": "Meta", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.574, - "helm_capabilities/MMLU-Pro": 0.653, - "helm_capabilities/GPQA": 0.426, - "helm_capabilities/IFEval": 0.821, - "helm_capabilities/WildBench": 0.758, - "helm_capabilities/Omni-MATH": 0.21, - "helm_lite/Mean win rate": 0.808, - "helm_lite/NarrativeQA": 0.772, - "helm_lite/NaturalQuestions (closed-book)": 0.452, - "helm_lite/OpenbookQA": 0.938, - "helm_lite/MMLU": 0.709, - "helm_lite/MATH": 0.783, - "helm_lite/GSM8K": 0.938, - "helm_lite/LegalBench": 0.687, - "helm_lite/MedQA": 0.769, - "helm_lite/WMT 2014": 0.223, - "helm_mmlu/MMLU All Subjects": 0.801, - "helm_mmlu/Abstract Algebra": 0.55, - "helm_mmlu/Anatomy": 0.8, - "helm_mmlu/College Physics": 0.559, - "helm_mmlu/Computer Security": 0.8, - "helm_mmlu/Econometrics": 0.675, - "helm_mmlu/Global Facts": 0.61, - "helm_mmlu/Jurisprudence": 0.889, - "helm_mmlu/Philosophy": 0.833, - "helm_mmlu/Professional Psychology": 0.846, - "helm_mmlu/Us Foreign Policy": 0.93, - "helm_mmlu/Astronomy": 0.908, - "helm_mmlu/Business Ethics": 0.72, - "helm_mmlu/Clinical Knowledge": 0.845, - "helm_mmlu/Conceptual Physics": 0.834, - "helm_mmlu/Electrical Engineering": 0.745, - "helm_mmlu/Elementary Mathematics": 0.701, - "helm_mmlu/Formal Logic": 0.675, - "helm_mmlu/High School World History": 0.937, - "helm_mmlu/Human Sexuality": 0.855, - "helm_mmlu/International Law": 0.926, - "helm_mmlu/Logical Fallacies": 0.84, - "helm_mmlu/Machine Learning": 0.696, - "helm_mmlu/Management": 0.913, - "helm_mmlu/Marketing": 0.936, - "helm_mmlu/Medical Genetics": 0.93, - "helm_mmlu/Miscellaneous": 0.913, - "helm_mmlu/Moral Scenarios": 0.834, - "helm_mmlu/Nutrition": 0.889, - "helm_mmlu/Prehistory": 0.88, - "helm_mmlu/Public Relations": 0.709, - "helm_mmlu/Security Studies": 0.849, - "helm_mmlu/Sociology": 0.92, - "helm_mmlu/Virology": 0.578, - "helm_mmlu/World Religions": 0.895, - "helm_mmlu/Mean win rate": 0.021 - } - }, - { - "id": "meta/llama-3.1-8b-instruct-turbo", - "name": "Llama 3.1 Instruct Turbo 8B", - "developer": "Meta", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.444, - "helm_capabilities/MMLU-Pro": 0.406, - "helm_capabilities/GPQA": 0.247, - "helm_capabilities/IFEval": 0.743, - "helm_capabilities/WildBench": 0.686, - "helm_capabilities/Omni-MATH": 0.137, - "helm_lite/Mean win rate": 0.303, - "helm_lite/NarrativeQA": 0.756, - "helm_lite/NaturalQuestions (closed-book)": 0.209, - "helm_lite/OpenbookQA": 0.74, - "helm_lite/MMLU": 0.5, - "helm_lite/MATH": 0.703, - "helm_lite/GSM8K": 0.798, - "helm_lite/LegalBench": 0.342, - "helm_lite/MedQA": 0.245, - "helm_lite/WMT 2014": 0.181, - "helm_mmlu/MMLU All Subjects": 0.561, - "helm_mmlu/Abstract Algebra": 0.26, - "helm_mmlu/Anatomy": 0.459, - "helm_mmlu/College Physics": 0.363, - "helm_mmlu/Computer Security": 0.71, - "helm_mmlu/Econometrics": 0.351, - "helm_mmlu/Global Facts": 0.26, - "helm_mmlu/Jurisprudence": 0.731, - "helm_mmlu/Philosophy": 0.64, - "helm_mmlu/Professional Psychology": 0.649, - "helm_mmlu/Us Foreign Policy": 0.79, - "helm_mmlu/Astronomy": 0.645, - "helm_mmlu/Business Ethics": 0.65, - "helm_mmlu/Clinical Knowledge": 0.615, - "helm_mmlu/Conceptual Physics": 0.528, - "helm_mmlu/Electrical Engineering": 0.441, - "helm_mmlu/Elementary Mathematics": 0.429, - "helm_mmlu/Formal Logic": 0.444, - "helm_mmlu/High School World History": 0.515, - "helm_mmlu/Human Sexuality": 0.733, - "helm_mmlu/International Law": 0.694, - "helm_mmlu/Logical Fallacies": 0.742, - "helm_mmlu/Machine Learning": 0.384, - "helm_mmlu/Management": 0.709, - "helm_mmlu/Marketing": 0.833, - "helm_mmlu/Medical Genetics": 0.66, - "helm_mmlu/Miscellaneous": 0.653, - "helm_mmlu/Moral Scenarios": 0.368, - "helm_mmlu/Nutrition": 0.712, - "helm_mmlu/Prehistory": 0.728, - "helm_mmlu/Public Relations": 0.664, - "helm_mmlu/Security Studies": 0.576, - "helm_mmlu/Sociology": 0.701, - "helm_mmlu/Virology": 0.446, - "helm_mmlu/World Religions": 0.789, - "helm_mmlu/Mean win rate": 0.475 - } - }, - { - "id": "meta/llama-3.2-11b-vision-instruct-turbo", - "name": "Llama 3.2 Vision Instruct Turbo 11B", - "developer": "Meta", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.325, - "helm_lite/NarrativeQA": 0.756, - "helm_lite/NaturalQuestions (closed-book)": 0.234, - "helm_lite/OpenbookQA": 0.724, - "helm_lite/MMLU": 0.511, - "helm_lite/MATH": 0.739, - "helm_lite/GSM8K": 0.823, - "helm_lite/LegalBench": 0.435, - "helm_lite/MedQA": 0.27, - "helm_lite/WMT 2014": 0.179, - "helm_mmlu/MMLU All Subjects": 0.565, - "helm_mmlu/Abstract Algebra": 0.28, - "helm_mmlu/Anatomy": 0.533, - "helm_mmlu/College Physics": 0.333, - "helm_mmlu/Computer Security": 0.71, - "helm_mmlu/Econometrics": 0.395, - "helm_mmlu/Global Facts": 0.25, - "helm_mmlu/Jurisprudence": 0.722, - "helm_mmlu/Philosophy": 0.646, - "helm_mmlu/Professional Psychology": 0.649, - "helm_mmlu/Us Foreign Policy": 0.78, - "helm_mmlu/Astronomy": 0.671, - "helm_mmlu/Business Ethics": 0.64, - "helm_mmlu/Clinical Knowledge": 0.638, - "helm_mmlu/Conceptual Physics": 0.536, - "helm_mmlu/Electrical Engineering": 0.51, - "helm_mmlu/Elementary Mathematics": 0.458, - "helm_mmlu/Formal Logic": 0.46, - "helm_mmlu/High School World History": 0.502, - "helm_mmlu/Human Sexuality": 0.763, - "helm_mmlu/International Law": 0.711, - "helm_mmlu/Logical Fallacies": 0.742, - "helm_mmlu/Machine Learning": 0.375, - "helm_mmlu/Management": 0.728, - "helm_mmlu/Marketing": 0.838, - "helm_mmlu/Medical Genetics": 0.7, - "helm_mmlu/Miscellaneous": 0.644, - "helm_mmlu/Moral Scenarios": 0.328, - "helm_mmlu/Nutrition": 0.752, - "helm_mmlu/Prehistory": 0.744, - "helm_mmlu/Public Relations": 0.645, - "helm_mmlu/Security Studies": 0.567, - "helm_mmlu/Sociology": 0.627, - "helm_mmlu/Virology": 0.446, - "helm_mmlu/World Religions": 0.696, - "helm_mmlu/Mean win rate": 0.897 - } - }, - { - "id": "meta/llama-3.2-90b-vision-instruct-turbo", - "name": "Llama 3.2 Vision Instruct Turbo 90B", - "developer": "Meta", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.819, - "helm_lite/NarrativeQA": 0.777, - "helm_lite/NaturalQuestions (closed-book)": 0.457, - "helm_lite/OpenbookQA": 0.942, - "helm_lite/MMLU": 0.703, - "helm_lite/MATH": 0.791, - "helm_lite/GSM8K": 0.936, - "helm_lite/LegalBench": 0.68, - "helm_lite/MedQA": 0.769, - "helm_lite/WMT 2014": 0.224, - "helm_mmlu/MMLU All Subjects": 0.803, - "helm_mmlu/Abstract Algebra": 0.52, - "helm_mmlu/Anatomy": 0.8, - "helm_mmlu/College Physics": 0.539, - "helm_mmlu/Computer Security": 0.81, - "helm_mmlu/Econometrics": 0.684, - "helm_mmlu/Global Facts": 0.6, - "helm_mmlu/Jurisprudence": 0.88, - "helm_mmlu/Philosophy": 0.839, - "helm_mmlu/Professional Psychology": 0.843, - "helm_mmlu/Us Foreign Policy": 0.93, - "helm_mmlu/Astronomy": 0.921, - "helm_mmlu/Business Ethics": 0.76, - "helm_mmlu/Clinical Knowledge": 0.845, - "helm_mmlu/Conceptual Physics": 0.826, - "helm_mmlu/Electrical Engineering": 0.759, - "helm_mmlu/Elementary Mathematics": 0.688, - "helm_mmlu/Formal Logic": 0.683, - "helm_mmlu/High School World History": 0.941, - "helm_mmlu/Human Sexuality": 0.87, - "helm_mmlu/International Law": 0.934, - "helm_mmlu/Logical Fallacies": 0.834, - "helm_mmlu/Machine Learning": 0.688, - "helm_mmlu/Management": 0.913, - "helm_mmlu/Marketing": 0.944, - "helm_mmlu/Medical Genetics": 0.92, - "helm_mmlu/Miscellaneous": 0.913, - "helm_mmlu/Moral Scenarios": 0.841, - "helm_mmlu/Nutrition": 0.889, - "helm_mmlu/Prehistory": 0.886, - "helm_mmlu/Public Relations": 0.718, - "helm_mmlu/Security Studies": 0.853, - "helm_mmlu/Sociology": 0.92, - "helm_mmlu/Virology": 0.584, - "helm_mmlu/World Religions": 0.901, - "helm_mmlu/Mean win rate": 0.773 - } - }, - { - "id": "meta/llama-3.3-70b-instruct-turbo", - "name": "Llama 3.3 Instruct Turbo 70B", - "developer": "Meta", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.812, - "helm_lite/NarrativeQA": 0.791, - "helm_lite/NaturalQuestions (closed-book)": 0.431, - "helm_lite/OpenbookQA": 0.928, - "helm_lite/MMLU": 0.7, - "helm_lite/MATH": 0.808, - "helm_lite/GSM8K": 0.942, - "helm_lite/LegalBench": 0.725, - "helm_lite/MedQA": 0.761, - "helm_lite/WMT 2014": 0.219, - "helm_mmlu/MMLU All Subjects": 0.791, - "helm_mmlu/Abstract Algebra": 0.5, - "helm_mmlu/Anatomy": 0.778, - "helm_mmlu/College Physics": 0.52, - "helm_mmlu/Computer Security": 0.8, - "helm_mmlu/Econometrics": 0.719, - "helm_mmlu/Global Facts": 0.58, - "helm_mmlu/Jurisprudence": 0.87, - "helm_mmlu/Philosophy": 0.83, - "helm_mmlu/Professional Psychology": 0.845, - "helm_mmlu/Us Foreign Policy": 0.93, - "helm_mmlu/Astronomy": 0.888, - "helm_mmlu/Business Ethics": 0.8, - "helm_mmlu/Clinical Knowledge": 0.83, - "helm_mmlu/Conceptual Physics": 0.821, - "helm_mmlu/Electrical Engineering": 0.745, - "helm_mmlu/Elementary Mathematics": 0.672, - "helm_mmlu/Formal Logic": 0.675, - "helm_mmlu/High School World History": 0.907, - "helm_mmlu/Human Sexuality": 0.855, - "helm_mmlu/International Law": 0.884, - "helm_mmlu/Logical Fallacies": 0.816, - "helm_mmlu/Machine Learning": 0.714, - "helm_mmlu/Management": 0.903, - "helm_mmlu/Marketing": 0.927, - "helm_mmlu/Medical Genetics": 0.9, - "helm_mmlu/Miscellaneous": 0.914, - "helm_mmlu/Moral Scenarios": 0.698, - "helm_mmlu/Nutrition": 0.882, - "helm_mmlu/Prehistory": 0.895, - "helm_mmlu/Public Relations": 0.727, - "helm_mmlu/Security Studies": 0.845, - "helm_mmlu/Sociology": 0.92, - "helm_mmlu/Virology": 0.566, - "helm_mmlu/World Religions": 0.883, - "helm_mmlu/Mean win rate": 0.722 - } - }, - { - "id": "meta/LLaMA-30B", - "name": "LLaMA 30B", - "developer": "Meta", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.781, - "helm_classic/MMLU": 0.531, - "helm_classic/BoolQ": 0.861, - "helm_classic/NarrativeQA": 0.752, - "helm_classic/NaturalQuestions (open-book)": 0.666, - "helm_classic/QuAC": 0.39, - "helm_classic/HellaSwag": -1.0, - "helm_classic/OpenbookQA": -1.0, - "helm_classic/TruthfulQA": 0.344, - "helm_classic/MS MARCO (TREC)": -1.0, - "helm_classic/CNN/DailyMail": -1.0, - "helm_classic/XSUM": -1.0, - "helm_classic/IMDB": 0.927, - "helm_classic/CivilComments": 0.549, - "helm_classic/RAFT": 0.752 - } - }, - { - "id": "meta/llama-4-maverick", - "name": "meta/llama-4-maverick", - "developer": "Meta", - "evaluator_relationship": null, - "benchmark_scores": { - "livecodebenchpro/Hard Problems": 0.0, - "livecodebenchpro/Medium Problems": 0.0, - "livecodebenchpro/Easy Problems": 0.09859154929577464 - } - }, - { - "id": "meta/llama-4-maverick-17b-128e-instruct-fp8", - "name": "Llama 4 Maverick 17Bx128E Instruct FP8", - "developer": "Meta", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.718, - "helm_capabilities/MMLU-Pro": 0.81, - "helm_capabilities/GPQA": 0.65, - "helm_capabilities/IFEval": 0.908, - "helm_capabilities/WildBench": 0.8, - "helm_capabilities/Omni-MATH": 0.422 - } - }, - { - "id": "meta/llama-4-maverick-17b-128e-instruct-fp8-fc", - "name": "Llama-4-Maverick-17B-128E-Instruct-FP8 (FC)", - "developer": "Meta", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 50.0, - "bfcl/bfcl.overall.overall_accuracy": 37.29, - "bfcl/bfcl.overall.total_cost_usd": 18.25, - "bfcl/bfcl.overall.latency_mean_s": 18.43, - "bfcl/bfcl.overall.latency_std_s": 34.11, - "bfcl/bfcl.overall.latency_p95_s": 102.75, - "bfcl/bfcl.non_live.ast_accuracy": 88.65, - "bfcl/bfcl.non_live.simple_ast_accuracy": 77.08, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 95.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 94.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 88.5, - "bfcl/bfcl.live.live_accuracy": 73.65, - "bfcl/bfcl.live.live_simple_ast_accuracy": 84.5, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 71.04, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 75.0, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 70.83, - "bfcl/bfcl.multi_turn.accuracy": 20.25, - "bfcl/bfcl.multi_turn.base_accuracy": 27.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 22.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 14.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 18.0, - "bfcl/bfcl.web_search.accuracy": 28.0, - "bfcl/bfcl.web_search.base_accuracy": 39.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 17.0, - "bfcl/bfcl.memory.accuracy": 18.92, - "bfcl/bfcl.memory.kv_accuracy": 8.39, - "bfcl/bfcl.memory.vector_accuracy": 32.9, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 15.48, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 100.0, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 55.97 - } - }, - { - "id": "meta/llama-4-scout-17b-16e-instruct", - "name": "Llama 4 Scout 17Bx16E Instruct", - "developer": "Meta", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.644, - "helm_capabilities/MMLU-Pro": 0.742, - "helm_capabilities/GPQA": 0.507, - "helm_capabilities/IFEval": 0.818, - "helm_capabilities/WildBench": 0.779, - "helm_capabilities/Omni-MATH": 0.373 - } - }, - { - "id": "meta/llama-4-scout-17b-16e-instruct-fc", - "name": "Llama-4-Scout-17B-16E-Instruct (FC)", - "developer": "Meta", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 72.0, - "bfcl/bfcl.overall.overall_accuracy": 28.13, - "bfcl/bfcl.overall.total_cost_usd": 24.68, - "bfcl/bfcl.overall.latency_mean_s": 17.86, - "bfcl/bfcl.overall.latency_std_s": 50.68, - "bfcl/bfcl.overall.latency_p95_s": 166.2, - "bfcl/bfcl.non_live.ast_accuracy": 89.38, - "bfcl/bfcl.non_live.simple_ast_accuracy": 79.0, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 94.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 94.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 90.5, - "bfcl/bfcl.live.live_accuracy": 74.69, - "bfcl/bfcl.live.live_simple_ast_accuracy": 81.78, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 72.74, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 81.25, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 79.17, - "bfcl/bfcl.multi_turn.accuracy": 9.0, - "bfcl/bfcl.multi_turn.base_accuracy": 12.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 7.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 7.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 9.5, - "bfcl/bfcl.web_search.accuracy": 14.5, - "bfcl/bfcl.web_search.base_accuracy": 18.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 11.0, - "bfcl/bfcl.memory.accuracy": 8.17, - "bfcl/bfcl.memory.kv_accuracy": 2.58, - "bfcl/bfcl.memory.vector_accuracy": 2.58, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 19.35, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 100.0, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 44.92 - } - }, - { - "id": "meta/LLaMA-65B", - "name": "LLaMA 65B", - "developer": "Meta", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.908, - "helm_classic/MMLU": 0.584, - "helm_classic/BoolQ": 0.871, - "helm_classic/NarrativeQA": 0.755, - "helm_classic/NaturalQuestions (open-book)": 0.672, - "helm_classic/QuAC": 0.401, - "helm_classic/HellaSwag": -1.0, - "helm_classic/OpenbookQA": -1.0, - "helm_classic/TruthfulQA": 0.508, - "helm_classic/MS MARCO (TREC)": -1.0, - "helm_classic/CNN/DailyMail": -1.0, - "helm_classic/XSUM": -1.0, - "helm_classic/IMDB": 0.962, - "helm_classic/CivilComments": 0.655, - "helm_classic/RAFT": 0.702, - "helm_lite/Mean win rate": 0.345, - "helm_lite/NarrativeQA": 0.755, - "helm_lite/NaturalQuestions (closed-book)": 0.433, - "helm_lite/OpenbookQA": 0.754, - "helm_lite/MMLU": 0.584, - "helm_lite/MATH": 0.257, - "helm_lite/GSM8K": 0.489, - "helm_lite/LegalBench": 0.48, - "helm_lite/MedQA": 0.507, - "helm_lite/WMT 2014": 0.189 - } - }, - { - "id": "meta/LLaMA-7B", - "name": "LLaMA 7B", - "developer": "Meta", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.533, - "helm_classic/MMLU": 0.321, - "helm_classic/BoolQ": 0.756, - "helm_classic/NarrativeQA": 0.669, - "helm_classic/NaturalQuestions (open-book)": 0.589, - "helm_classic/QuAC": 0.338, - "helm_classic/HellaSwag": -1.0, - "helm_classic/OpenbookQA": -1.0, - "helm_classic/TruthfulQA": 0.28, - "helm_classic/MS MARCO (TREC)": -1.0, - "helm_classic/CNN/DailyMail": -1.0, - "helm_classic/XSUM": -1.0, - "helm_classic/IMDB": 0.947, - "helm_classic/CivilComments": 0.563, - "helm_classic/RAFT": 0.573 - } - }, - { - "id": "meta/OPT-175B", - "name": "OPT 175B", - "developer": "Meta", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.609, - "helm_classic/MMLU": 0.318, - "helm_classic/BoolQ": 0.793, - "helm_classic/NarrativeQA": 0.671, - "helm_classic/NaturalQuestions (open-book)": 0.615, - "helm_classic/QuAC": 0.36, - "helm_classic/HellaSwag": 0.791, - "helm_classic/OpenbookQA": 0.586, - "helm_classic/TruthfulQA": 0.25, - "helm_classic/MS MARCO (TREC)": 0.448, - "helm_classic/CNN/DailyMail": 0.146, - "helm_classic/XSUM": 0.155, - "helm_classic/IMDB": 0.947, - "helm_classic/CivilComments": 0.505, - "helm_classic/RAFT": 0.606 - } - }, - { - "id": "meta/OPT-66B", - "name": "OPT 66B", - "developer": "Meta", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.448, - "helm_classic/MMLU": 0.276, - "helm_classic/BoolQ": 0.76, - "helm_classic/NarrativeQA": 0.638, - "helm_classic/NaturalQuestions (open-book)": 0.596, - "helm_classic/QuAC": 0.357, - "helm_classic/HellaSwag": 0.745, - "helm_classic/OpenbookQA": 0.534, - "helm_classic/TruthfulQA": 0.201, - "helm_classic/MS MARCO (TREC)": 0.482, - "helm_classic/CNN/DailyMail": 0.136, - "helm_classic/XSUM": 0.126, - "helm_classic/IMDB": 0.917, - "helm_classic/CivilComments": 0.506, - "helm_classic/RAFT": 0.557 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/mhl1.json b/data/developers/mhl1.json deleted file mode 100644 index 936e3681876c7812d0d3b012b89250105f55a50b..0000000000000000000000000000000000000000 --- a/data/developers/mhl1.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "mhl1", - "models": [ - { - "id": "mhl1/Qwen2.5-0.5B-cinstruct-stage1", - "name": "Qwen2.5-0.5B-cinstruct-stage1", - "developer": "mhl1", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1482, - "hfopenllm_v2/BBH": 0.3256, - "hfopenllm_v2/MATH Level 5": 0.0128, - "hfopenllm_v2/GPQA": 0.2651, - "hfopenllm_v2/MUSR": 0.35, - "hfopenllm_v2/MMLU-PRO": 0.1139 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/microsoft.json b/data/developers/microsoft.json deleted file mode 100644 index c64570948e3baf1a4260102e581572f859b9155b..0000000000000000000000000000000000000000 --- a/data/developers/microsoft.json +++ /dev/null @@ -1,453 +0,0 @@ -{ - "developer": "microsoft", - "models": [ - { - "id": "microsoft/DialoGPT-medium", - "name": "DialoGPT-medium", - "developer": "microsoft", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1479, - "hfopenllm_v2/BBH": 0.3014, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2542, - "hfopenllm_v2/MUSR": 0.4287, - "hfopenllm_v2/MMLU-PRO": 0.1119 - } - }, - { - "id": "microsoft/Orca-2-13b", - "name": "Orca-2-13b", - "developer": "microsoft", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3128, - "hfopenllm_v2/BBH": 0.4884, - "hfopenllm_v2/MATH Level 5": 0.0317, - "hfopenllm_v2/GPQA": 0.2802, - "hfopenllm_v2/MUSR": 0.513, - "hfopenllm_v2/MMLU-PRO": 0.2749 - } - }, - { - "id": "microsoft/Orca-2-7b", - "name": "Orca-2-7b", - "developer": "microsoft", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2183, - "hfopenllm_v2/BBH": 0.4452, - "hfopenllm_v2/MATH Level 5": 0.0196, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.5026, - "hfopenllm_v2/MMLU-PRO": 0.2319 - } - }, - { - "id": "microsoft/phi-1", - "name": "phi-1", - "developer": "microsoft", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2068, - "hfopenllm_v2/BBH": 0.3139, - "hfopenllm_v2/MATH Level 5": 0.0098, - "hfopenllm_v2/GPQA": 0.2651, - "hfopenllm_v2/MUSR": 0.3525, - "hfopenllm_v2/MMLU-PRO": 0.1162 - } - }, - { - "id": "microsoft/phi-1_5", - "name": "phi-1_5", - "developer": "microsoft", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2033, - "hfopenllm_v2/BBH": 0.336, - "hfopenllm_v2/MATH Level 5": 0.0181, - "hfopenllm_v2/GPQA": 0.2676, - "hfopenllm_v2/MUSR": 0.3404, - "hfopenllm_v2/MMLU-PRO": 0.1691 - } - }, - { - "id": "microsoft/phi-2", - "name": "Phi-2", - "developer": "microsoft", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.169, - "helm_lite/NarrativeQA": 0.703, - "helm_lite/NaturalQuestions (closed-book)": 0.155, - "helm_lite/OpenbookQA": 0.798, - "helm_lite/MMLU": 0.518, - "helm_lite/MATH": 0.255, - "helm_lite/GSM8K": 0.581, - "helm_lite/LegalBench": 0.334, - "helm_lite/MedQA": 0.41, - "helm_lite/WMT 2014": 0.038, - "helm_mmlu/MMLU All Subjects": 0.584, - "helm_mmlu/Abstract Algebra": 0.31, - "helm_mmlu/Anatomy": 0.437, - "helm_mmlu/College Physics": 0.382, - "helm_mmlu/Computer Security": 0.73, - "helm_mmlu/Econometrics": 0.342, - "helm_mmlu/Global Facts": 0.35, - "helm_mmlu/Jurisprudence": 0.694, - "helm_mmlu/Philosophy": 0.598, - "helm_mmlu/Professional Psychology": 0.572, - "helm_mmlu/Us Foreign Policy": 0.78, - "helm_mmlu/Astronomy": 0.605, - "helm_mmlu/Business Ethics": 0.59, - "helm_mmlu/Clinical Knowledge": 0.619, - "helm_mmlu/Conceptual Physics": 0.519, - "helm_mmlu/Electrical Engineering": 0.545, - "helm_mmlu/Elementary Mathematics": 0.463, - "helm_mmlu/Formal Logic": 0.389, - "helm_mmlu/High School World History": 0.73, - "helm_mmlu/Human Sexuality": 0.733, - "helm_mmlu/International Law": 0.752, - "helm_mmlu/Logical Fallacies": 0.767, - "helm_mmlu/Machine Learning": 0.5, - "helm_mmlu/Management": 0.748, - "helm_mmlu/Marketing": 0.833, - "helm_mmlu/Medical Genetics": 0.62, - "helm_mmlu/Miscellaneous": 0.688, - "helm_mmlu/Moral Scenarios": 0.231, - "helm_mmlu/Nutrition": 0.627, - "helm_mmlu/Prehistory": 0.605, - "helm_mmlu/Public Relations": 0.673, - "helm_mmlu/Security Studies": 0.702, - "helm_mmlu/Sociology": 0.816, - "helm_mmlu/Virology": 0.47, - "helm_mmlu/World Religions": 0.702, - "helm_mmlu/Mean win rate": 0.824, - "hfopenllm_v2/IFEval": 0.2739, - "hfopenllm_v2/BBH": 0.4881, - "hfopenllm_v2/MATH Level 5": 0.0295, - "hfopenllm_v2/GPQA": 0.2718, - "hfopenllm_v2/MUSR": 0.4099, - "hfopenllm_v2/MMLU-PRO": 0.2628 - } - }, - { - "id": "microsoft/Phi-3-medium-128k-instruct", - "name": "Phi-3-medium-128k-instruct", - "developer": "microsoft", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.604, - "hfopenllm_v2/BBH": 0.6382, - "hfopenllm_v2/MATH Level 5": 0.1918, - "hfopenllm_v2/GPQA": 0.3364, - "hfopenllm_v2/MUSR": 0.4129, - "hfopenllm_v2/MMLU-PRO": 0.4712 - } - }, - { - "id": "microsoft/phi-3-medium-4k-instruct", - "name": "Phi-3 14B", - "developer": "microsoft", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.509, - "helm_lite/NarrativeQA": 0.724, - "helm_lite/NaturalQuestions (closed-book)": 0.278, - "helm_lite/OpenbookQA": 0.916, - "helm_lite/MMLU": 0.675, - "helm_lite/MATH": 0.611, - "helm_lite/GSM8K": 0.878, - "helm_lite/LegalBench": 0.593, - "helm_lite/MedQA": 0.696, - "helm_lite/WMT 2014": 0.17, - "helm_mmlu/MMLU All Subjects": 0.775, - "helm_mmlu/Abstract Algebra": 0.5, - "helm_mmlu/Anatomy": 0.719, - "helm_mmlu/College Physics": 0.529, - "helm_mmlu/Computer Security": 0.79, - "helm_mmlu/Econometrics": 0.614, - "helm_mmlu/Global Facts": 0.5, - "helm_mmlu/Jurisprudence": 0.88, - "helm_mmlu/Philosophy": 0.804, - "helm_mmlu/Professional Psychology": 0.835, - "helm_mmlu/Us Foreign Policy": 0.95, - "helm_mmlu/Astronomy": 0.849, - "helm_mmlu/Business Ethics": 0.8, - "helm_mmlu/Clinical Knowledge": 0.826, - "helm_mmlu/Conceptual Physics": 0.809, - "helm_mmlu/Electrical Engineering": 0.683, - "helm_mmlu/Elementary Mathematics": 0.709, - "helm_mmlu/Formal Logic": 0.587, - "helm_mmlu/High School World History": 0.903, - "helm_mmlu/Human Sexuality": 0.863, - "helm_mmlu/International Law": 0.934, - "helm_mmlu/Logical Fallacies": 0.828, - "helm_mmlu/Machine Learning": 0.696, - "helm_mmlu/Management": 0.864, - "helm_mmlu/Marketing": 0.919, - "helm_mmlu/Medical Genetics": 0.91, - "helm_mmlu/Miscellaneous": 0.894, - "helm_mmlu/Moral Scenarios": 0.639, - "helm_mmlu/Nutrition": 0.837, - "helm_mmlu/Prehistory": 0.867, - "helm_mmlu/Public Relations": 0.755, - "helm_mmlu/Security Studies": 0.829, - "helm_mmlu/Sociology": 0.891, - "helm_mmlu/Virology": 0.554, - "helm_mmlu/World Religions": 0.865, - "helm_mmlu/Mean win rate": 0.015, - "hfopenllm_v2/IFEval": 0.6423, - "hfopenllm_v2/BBH": 0.6412, - "hfopenllm_v2/MATH Level 5": 0.1956, - "hfopenllm_v2/GPQA": 0.3364, - "hfopenllm_v2/MUSR": 0.4258, - "hfopenllm_v2/MMLU-PRO": 0.4676 - } - }, - { - "id": "microsoft/Phi-3-mini-128k-instruct", - "name": "Phi-3-mini-128k-instruct", - "developer": "microsoft", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5976, - "hfopenllm_v2/BBH": 0.5575, - "hfopenllm_v2/MATH Level 5": 0.1405, - "hfopenllm_v2/GPQA": 0.318, - "hfopenllm_v2/MUSR": 0.3937, - "hfopenllm_v2/MMLU-PRO": 0.3734 - } - }, - { - "id": "microsoft/Phi-3-mini-4k-instruct", - "name": "Phi-3-mini-4k-instruct", - "developer": "microsoft", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5613, - "hfopenllm_v2/BBH": 0.5676, - "hfopenllm_v2/MATH Level 5": 0.1163, - "hfopenllm_v2/GPQA": 0.3196, - "hfopenllm_v2/MUSR": 0.395, - "hfopenllm_v2/MMLU-PRO": 0.3866 - } - }, - { - "id": "microsoft/Phi-3-small-128k-instruct", - "name": "Phi-3-small-128k-instruct", - "developer": "microsoft", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6368, - "hfopenllm_v2/BBH": 0.6202, - "hfopenllm_v2/MATH Level 5": 0.2026, - "hfopenllm_v2/GPQA": 0.3171, - "hfopenllm_v2/MUSR": 0.4378, - "hfopenllm_v2/MMLU-PRO": 0.4491 - } - }, - { - "id": "microsoft/phi-3-small-8k-instruct", - "name": "Phi-3 7B", - "developer": "microsoft", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.473, - "helm_lite/NarrativeQA": 0.754, - "helm_lite/NaturalQuestions (closed-book)": 0.324, - "helm_lite/OpenbookQA": 0.912, - "helm_lite/MMLU": 0.659, - "helm_lite/MATH": 0.703, - "helm_lite/GSM8K": -1.0, - "helm_lite/LegalBench": 0.584, - "helm_lite/MedQA": 0.672, - "helm_lite/WMT 2014": 0.154, - "helm_mmlu/MMLU All Subjects": 0.757, - "helm_mmlu/Abstract Algebra": 0.44, - "helm_mmlu/Anatomy": 0.726, - "helm_mmlu/College Physics": 0.559, - "helm_mmlu/Computer Security": 0.77, - "helm_mmlu/Econometrics": 0.596, - "helm_mmlu/Global Facts": 0.52, - "helm_mmlu/Jurisprudence": 0.843, - "helm_mmlu/Philosophy": 0.82, - "helm_mmlu/Professional Psychology": 0.835, - "helm_mmlu/Us Foreign Policy": 0.95, - "helm_mmlu/Astronomy": 0.849, - "helm_mmlu/Business Ethics": 0.77, - "helm_mmlu/Clinical Knowledge": 0.83, - "helm_mmlu/Conceptual Physics": 0.779, - "helm_mmlu/Electrical Engineering": 0.69, - "helm_mmlu/Elementary Mathematics": 0.619, - "helm_mmlu/Formal Logic": 0.595, - "helm_mmlu/High School World History": 0.848, - "helm_mmlu/Human Sexuality": 0.817, - "helm_mmlu/International Law": 0.851, - "helm_mmlu/Logical Fallacies": 0.81, - "helm_mmlu/Machine Learning": 0.652, - "helm_mmlu/Management": 0.903, - "helm_mmlu/Marketing": 0.897, - "helm_mmlu/Medical Genetics": 0.84, - "helm_mmlu/Miscellaneous": 0.871, - "helm_mmlu/Moral Scenarios": 0.711, - "helm_mmlu/Nutrition": 0.833, - "helm_mmlu/Prehistory": 0.858, - "helm_mmlu/Public Relations": 0.727, - "helm_mmlu/Security Studies": 0.804, - "helm_mmlu/Sociology": 0.886, - "helm_mmlu/Virology": 0.548, - "helm_mmlu/World Religions": 0.825, - "helm_mmlu/Mean win rate": 0.708, - "hfopenllm_v2/IFEval": 0.6497, - "hfopenllm_v2/BBH": 0.6208, - "hfopenllm_v2/MATH Level 5": 0.1887, - "hfopenllm_v2/GPQA": 0.3121, - "hfopenllm_v2/MUSR": 0.4558, - "hfopenllm_v2/MMLU-PRO": 0.4506 - } - }, - { - "id": "microsoft/Phi-3.5-mini-instruct", - "name": "Phi-3.5-mini-instruct", - "developer": "microsoft", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5775, - "hfopenllm_v2/BBH": 0.5518, - "hfopenllm_v2/MATH Level 5": 0.1964, - "hfopenllm_v2/GPQA": 0.3398, - "hfopenllm_v2/MUSR": 0.4021, - "hfopenllm_v2/MMLU-PRO": 0.3962 - } - }, - { - "id": "microsoft/Phi-3.5-MoE-instruct", - "name": "Phi-3.5-MoE-instruct", - "developer": "microsoft", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6925, - "hfopenllm_v2/BBH": 0.6408, - "hfopenllm_v2/MATH Level 5": 0.3119, - "hfopenllm_v2/GPQA": 0.3557, - "hfopenllm_v2/MUSR": 0.4565, - "hfopenllm_v2/MMLU-PRO": 0.4658 - } - }, - { - "id": "microsoft/phi-4", - "name": "phi-4", - "developer": "microsoft", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0585, - "hfopenllm_v2/BBH": 0.6691, - "hfopenllm_v2/MATH Level 5": 0.3165, - "hfopenllm_v2/GPQA": 0.406, - "hfopenllm_v2/MUSR": 0.5034, - "hfopenllm_v2/MMLU-PRO": 0.5287 - } - }, - { - "id": "microsoft/Phi-4-mini-instruct", - "name": "Phi-4-mini-instruct", - "developer": "microsoft", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7378, - "hfopenllm_v2/BBH": 0.5689, - "hfopenllm_v2/MATH Level 5": 0.1699, - "hfopenllm_v2/GPQA": 0.3096, - "hfopenllm_v2/MUSR": 0.3873, - "hfopenllm_v2/MMLU-PRO": 0.3932 - } - }, - { - "id": "microsoft/phi-4-prompt", - "name": "Phi-4 (Prompt)", - "developer": "microsoft", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 70.0, - "bfcl/bfcl.overall.overall_accuracy": 28.79, - "bfcl/bfcl.overall.total_cost_usd": 8.72, - "bfcl/bfcl.overall.latency_mean_s": 9.49, - "bfcl/bfcl.overall.latency_std_s": 26.73, - "bfcl/bfcl.overall.latency_p95_s": 23.02, - "bfcl/bfcl.non_live.ast_accuracy": 69.56, - "bfcl/bfcl.non_live.simple_ast_accuracy": 74.25, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 89.5, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 65.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 49.5, - "bfcl/bfcl.live.live_accuracy": 60.7, - "bfcl/bfcl.live.live_simple_ast_accuracy": 65.5, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 59.64, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 81.25, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 41.67, - "bfcl/bfcl.multi_turn.accuracy": 3.88, - "bfcl/bfcl.multi_turn.base_accuracy": 9.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 0.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 3.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 3.0, - "bfcl/bfcl.web_search.accuracy": 4.5, - "bfcl/bfcl.web_search.base_accuracy": 4.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 5.0, - "bfcl/bfcl.memory.accuracy": 24.73, - "bfcl/bfcl.memory.kv_accuracy": 17.42, - "bfcl/bfcl.memory.vector_accuracy": 25.16, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 31.61, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 50.0, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 87.55, - "bfcl/bfcl.format_sensitivity.max_delta": 81.5, - "bfcl/bfcl.format_sensitivity.stddev": 23.34 - } - }, - { - "id": "microsoft/TNLG-v2-530B", - "name": "TNLG v2 530B", - "developer": "microsoft", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.787, - "helm_classic/MMLU": 0.469, - "helm_classic/BoolQ": 0.809, - "helm_classic/NarrativeQA": 0.722, - "helm_classic/NaturalQuestions (open-book)": 0.642, - "helm_classic/QuAC": 0.39, - "helm_classic/HellaSwag": 0.799, - "helm_classic/OpenbookQA": 0.562, - "helm_classic/TruthfulQA": 0.251, - "helm_classic/MS MARCO (TREC)": 0.643, - "helm_classic/CNN/DailyMail": 0.161, - "helm_classic/XSUM": 0.169, - "helm_classic/IMDB": 0.941, - "helm_classic/CivilComments": 0.601, - "helm_classic/RAFT": 0.679 - } - }, - { - "id": "microsoft/TNLG-v2-6.7B", - "name": "TNLG v2 6.7B", - "developer": "microsoft", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.309, - "helm_classic/MMLU": 0.242, - "helm_classic/BoolQ": 0.698, - "helm_classic/NarrativeQA": 0.631, - "helm_classic/NaturalQuestions (open-book)": 0.561, - "helm_classic/QuAC": 0.345, - "helm_classic/HellaSwag": 0.704, - "helm_classic/OpenbookQA": 0.478, - "helm_classic/TruthfulQA": 0.167, - "helm_classic/MS MARCO (TREC)": 0.332, - "helm_classic/CNN/DailyMail": 0.146, - "helm_classic/XSUM": 0.11, - "helm_classic/IMDB": 0.927, - "helm_classic/CivilComments": 0.532, - "helm_classic/RAFT": 0.525 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/mightbe.json b/data/developers/mightbe.json deleted file mode 100644 index ad73c924f7d1a8257a57195d8aa42a6544ae6187..0000000000000000000000000000000000000000 --- a/data/developers/mightbe.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "mightbe", - "models": [ - { - "id": "mightbe/Better-PairRM", - "name": "mightbe/Better-PairRM", - "developer": "mightbe", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.673, - "reward-bench/Chat": 0.9553, - "reward-bench/Chat Hard": 0.3925, - "reward-bench/Safety": 0.8203, - "reward-bench/Reasoning": 0.4983, - "reward-bench/Prior Sets (0.5 weight)": 0.724 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/migtissera.json b/data/developers/migtissera.json deleted file mode 100644 index 758dddd03daa89044ae5d24105c8c1c0de081425..0000000000000000000000000000000000000000 --- a/data/developers/migtissera.json +++ /dev/null @@ -1,117 +0,0 @@ -{ - "developer": "migtissera", - "models": [ - { - "id": "migtissera/Llama-3-70B-Synthia-v3.5", - "name": "Llama-3-70B-Synthia-v3.5", - "developer": "migtissera", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6076, - "hfopenllm_v2/BBH": 0.6489, - "hfopenllm_v2/MATH Level 5": 0.2115, - "hfopenllm_v2/GPQA": 0.3876, - "hfopenllm_v2/MUSR": 0.4922, - "hfopenllm_v2/MMLU-PRO": 0.4658 - } - }, - { - "id": "migtissera/Llama-3-8B-Synthia-v3.5", - "name": "Llama-3-8B-Synthia-v3.5", - "developer": "migtissera", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.507, - "hfopenllm_v2/BBH": 0.4888, - "hfopenllm_v2/MATH Level 5": 0.0657, - "hfopenllm_v2/GPQA": 0.2718, - "hfopenllm_v2/MUSR": 0.4044, - "hfopenllm_v2/MMLU-PRO": 0.303 - } - }, - { - "id": "migtissera/Tess-3-7B-SFT", - "name": "Tess-3-7B-SFT", - "developer": "migtissera", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3946, - "hfopenllm_v2/BBH": 0.4607, - "hfopenllm_v2/MATH Level 5": 0.04, - "hfopenllm_v2/GPQA": 0.271, - "hfopenllm_v2/MUSR": 0.4113, - "hfopenllm_v2/MMLU-PRO": 0.3034 - } - }, - { - "id": "migtissera/Tess-3-Mistral-Nemo-12B", - "name": "Tess-3-Mistral-Nemo-12B", - "developer": "migtissera", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3355, - "hfopenllm_v2/BBH": 0.4899, - "hfopenllm_v2/MATH Level 5": 0.0574, - "hfopenllm_v2/GPQA": 0.2508, - "hfopenllm_v2/MUSR": 0.4458, - "hfopenllm_v2/MMLU-PRO": 0.2565 - } - }, - { - "id": "migtissera/Tess-v2.5-Phi-3-medium-128k-14B", - "name": "Tess-v2.5-Phi-3-medium-128k-14B", - "developer": "migtissera", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4539, - "hfopenllm_v2/BBH": 0.6207, - "hfopenllm_v2/MATH Level 5": 0.0506, - "hfopenllm_v2/GPQA": 0.3079, - "hfopenllm_v2/MUSR": 0.4113, - "hfopenllm_v2/MMLU-PRO": 0.3732 - } - }, - { - "id": "migtissera/Tess-v2.5.2-Qwen2-72B", - "name": "Tess-v2.5.2-Qwen2-72B", - "developer": "migtissera", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4494, - "hfopenllm_v2/BBH": 0.6647, - "hfopenllm_v2/MATH Level 5": 0.2938, - "hfopenllm_v2/GPQA": 0.3507, - "hfopenllm_v2/MUSR": 0.4188, - "hfopenllm_v2/MMLU-PRO": 0.5561 - } - }, - { - "id": "migtissera/Trinity-2-Codestral-22B", - "name": "Trinity-2-Codestral-22B", - "developer": "migtissera", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4202, - "hfopenllm_v2/BBH": 0.5593, - "hfopenllm_v2/MATH Level 5": 0.0967, - "hfopenllm_v2/GPQA": 0.3146, - "hfopenllm_v2/MUSR": 0.4111, - "hfopenllm_v2/MMLU-PRO": 0.3308 - } - }, - { - "id": "migtissera/Trinity-2-Codestral-22B-v0.2", - "name": "Trinity-2-Codestral-22B-v0.2", - "developer": "migtissera", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.443, - "hfopenllm_v2/BBH": 0.5706, - "hfopenllm_v2/MATH Level 5": 0.0869, - "hfopenllm_v2/GPQA": 0.3079, - "hfopenllm_v2/MUSR": 0.4031, - "hfopenllm_v2/MMLU-PRO": 0.3354 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/minami-su.json b/data/developers/minami-su.json deleted file mode 100644 index ab1f6d2f21891005d8d34b394b96e4ef5daa1330..0000000000000000000000000000000000000000 --- a/data/developers/minami-su.json +++ /dev/null @@ -1,75 +0,0 @@ -{ - "developer": "Minami-su", - "models": [ - { - "id": "Minami-su/Amara-o1-7B-Qwen", - "name": "Amara-o1-7B-Qwen", - "developer": "Minami-su", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.739, - "hfopenllm_v2/BBH": 0.5199, - "hfopenllm_v2/MATH Level 5": 0.5181, - "hfopenllm_v2/GPQA": 0.2936, - "hfopenllm_v2/MUSR": 0.4007, - "hfopenllm_v2/MMLU-PRO": 0.4083 - } - }, - { - "id": "Minami-su/Amara-o2-7B-Qwen", - "name": "Amara-o2-7B-Qwen", - "developer": "Minami-su", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7147, - "hfopenllm_v2/BBH": 0.5173, - "hfopenllm_v2/MATH Level 5": 0.4086, - "hfopenllm_v2/GPQA": 0.2634, - "hfopenllm_v2/MUSR": 0.3781, - "hfopenllm_v2/MMLU-PRO": 0.4165 - } - }, - { - "id": "Minami-su/test-7B-00", - "name": "test-7B-00", - "developer": "Minami-su", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.669, - "hfopenllm_v2/BBH": 0.4466, - "hfopenllm_v2/MATH Level 5": 0.4517, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.4126, - "hfopenllm_v2/MMLU-PRO": 0.3588 - } - }, - { - "id": "Minami-su/test-7B-01", - "name": "test-7B-01", - "developer": "Minami-su", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6736, - "hfopenllm_v2/BBH": 0.4422, - "hfopenllm_v2/MATH Level 5": 0.4554, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.4153, - "hfopenllm_v2/MMLU-PRO": 0.3536 - } - }, - { - "id": "Minami-su/test-v2-7B-00", - "name": "test-v2-7B-00", - "developer": "Minami-su", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6747, - "hfopenllm_v2/BBH": 0.4416, - "hfopenllm_v2/MATH Level 5": 0.4418, - "hfopenllm_v2/GPQA": 0.2919, - "hfopenllm_v2/MUSR": 0.4154, - "hfopenllm_v2/MMLU-PRO": 0.3472 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/mindw96.json b/data/developers/mindw96.json deleted file mode 100644 index 849de50cbe76021240babe9df68c9bad5c76d358..0000000000000000000000000000000000000000 --- a/data/developers/mindw96.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "mindw96", - "models": [ - { - "id": "mindw96/DeepSeek-llama3.3-Bllossom-8B-DACON-LLM3", - "name": "DeepSeek-llama3.3-Bllossom-8B-DACON-LLM3", - "developer": "mindw96", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1388, - "hfopenllm_v2/BBH": 0.3068, - "hfopenllm_v2/MATH Level 5": 0.0083, - "hfopenllm_v2/GPQA": 0.2508, - "hfopenllm_v2/MUSR": 0.3792, - "hfopenllm_v2/MMLU-PRO": 0.1106 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/minghaowu.json b/data/developers/minghaowu.json deleted file mode 100644 index 5b01f9cd3a0fe33648498ba5463b6bc8b16a60b7..0000000000000000000000000000000000000000 --- a/data/developers/minghaowu.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "minghaowu", - "models": [ - { - "id": "minghaowu/Qwen1.5-1.8B-OpenHermes-2.5", - "name": "Qwen1.5-1.8B-OpenHermes-2.5", - "developer": "minghaowu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2778, - "hfopenllm_v2/BBH": 0.3375, - "hfopenllm_v2/MATH Level 5": 0.0242, - "hfopenllm_v2/GPQA": 0.2836, - "hfopenllm_v2/MUSR": 0.3529, - "hfopenllm_v2/MMLU-PRO": 0.1792 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/minimax.json b/data/developers/minimax.json deleted file mode 100644 index b575a16ccd7fb272ceb4b3067b0a9e48f65cff08..0000000000000000000000000000000000000000 --- a/data/developers/minimax.json +++ /dev/null @@ -1,41 +0,0 @@ -{ - "developer": "MiniMax", - "models": [ - { - "id": "minimax/Minimax-2.5", - "name": "Minimax-2.5", - "developer": "minimax", - "evaluator_relationship": null, - "benchmark_scores": { - "apex-agents/Corporate Lawyer Mean Score": 0.339 - } - }, - { - "id": "minimax/minimax-m2", - "name": "MiniMax M2", - "developer": "MiniMax", - "evaluator_relationship": null, - "benchmark_scores": { - "terminal-bench-2.0/terminal-bench-2.0": 30.0 - } - }, - { - "id": "minimax/minimax-m2.1", - "name": "MiniMax M2.1", - "developer": "MiniMax", - "evaluator_relationship": null, - "benchmark_scores": { - "terminal-bench-2.0/terminal-bench-2.0": 29.2 - } - }, - { - "id": "minimax/minimax-m2.5", - "name": "Minimax m2.5", - "developer": "Minimax", - "evaluator_relationship": null, - "benchmark_scores": { - "terminal-bench-2.0/terminal-bench-2.0": 42.2 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/ministral.json b/data/developers/ministral.json deleted file mode 100644 index 6955cf075a7bf1b3ace7fe1d40d931e0250d87d9..0000000000000000000000000000000000000000 --- a/data/developers/ministral.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "ministral", - "models": [ - { - "id": "ministral/Ministral-3b-instruct", - "name": "Ministral-3b-instruct", - "developer": "ministral", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1358, - "hfopenllm_v2/BBH": 0.3192, - "hfopenllm_v2/MATH Level 5": 0.0083, - "hfopenllm_v2/GPQA": 0.2517, - "hfopenllm_v2/MUSR": 0.3382, - "hfopenllm_v2/MMLU-PRO": 0.1093 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/mistral-community.json b/data/developers/mistral-community.json deleted file mode 100644 index bfaa5d6738731770955843d6671665c5e716c0d4..0000000000000000000000000000000000000000 --- a/data/developers/mistral-community.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "developer": "mistral-community", - "models": [ - { - "id": "mistral-community/Mistral-7B-v0.2", - "name": "Mistral-7B-v0.2", - "developer": "mistral-community", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2266, - "hfopenllm_v2/BBH": 0.451, - "hfopenllm_v2/MATH Level 5": 0.0302, - "hfopenllm_v2/GPQA": 0.2919, - "hfopenllm_v2/MUSR": 0.4032, - "hfopenllm_v2/MMLU-PRO": 0.2953 - } - }, - { - "id": "mistral-community/Mixtral-8x22B-v0.1", - "name": "Mixtral-8x22B-v0.1", - "developer": "mistral-community", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3167, - "hfopenllm_v2/BBH": 0.38, - "hfopenllm_v2/MATH Level 5": 0.1543, - "hfopenllm_v2/GPQA": 0.33, - "hfopenllm_v2/MUSR": 0.3533, - "hfopenllm_v2/MMLU-PRO": 0.36 - } - }, - { - "id": "mistral-community/mixtral-8x22B-v0.3", - "name": "mixtral-8x22B-v0.3", - "developer": "mistral-community", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2583, - "hfopenllm_v2/BBH": 0.625, - "hfopenllm_v2/MATH Level 5": 0.1835, - "hfopenllm_v2/GPQA": 0.3775, - "hfopenllm_v2/MUSR": 0.4037, - "hfopenllm_v2/MMLU-PRO": 0.4639 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/mistralai.json b/data/developers/mistralai.json deleted file mode 100644 index afb6ca7c6ffc67598d4e1e7d1d7ae896bc8d1ba5..0000000000000000000000000000000000000000 --- a/data/developers/mistralai.json +++ /dev/null @@ -1,1136 +0,0 @@ -{ - "developer": "mistralai", - "models": [ - { - "id": "mistralai/Codestral-22B-v0.1", - "name": "Codestral-22B-v0.1", - "developer": "mistralai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5772, - "hfopenllm_v2/BBH": 0.5139, - "hfopenllm_v2/MATH Level 5": 0.1005, - "hfopenllm_v2/GPQA": 0.2987, - "hfopenllm_v2/MUSR": 0.4187, - "hfopenllm_v2/MMLU-PRO": 0.3156 - } - }, - { - "id": "mistralai/Ministral-8B-Instruct-2410", - "name": "Ministral-8B-Instruct-2410", - "developer": "mistralai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5896, - "hfopenllm_v2/BBH": 0.4762, - "hfopenllm_v2/MATH Level 5": 0.1956, - "hfopenllm_v2/GPQA": 0.2844, - "hfopenllm_v2/MUSR": 0.4138, - "hfopenllm_v2/MMLU-PRO": 0.3291 - } - }, - { - "id": "mistralai/ministral-8b-instruct-2410-fc", - "name": "Ministral-8B-Instruct-2410 (FC)", - "developer": "mistralai", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 105.0, - "bfcl/bfcl.overall.overall_accuracy": 11.1, - "bfcl/bfcl.overall.total_cost_usd": 70.01, - "bfcl/bfcl.overall.latency_mean_s": 82.07, - "bfcl/bfcl.overall.latency_std_s": 212.99, - "bfcl/bfcl.overall.latency_p95_s": 568.59, - "bfcl/bfcl.non_live.ast_accuracy": 0.0, - "bfcl/bfcl.non_live.simple_ast_accuracy": 0.0, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 0.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 0.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 0.0, - "bfcl/bfcl.live.live_accuracy": 0.0, - "bfcl/bfcl.live.live_simple_ast_accuracy": 0.0, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 0.0, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 0.0, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 0.0, - "bfcl/bfcl.multi_turn.accuracy": 0.0, - "bfcl/bfcl.multi_turn.base_accuracy": 0.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 0.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 0.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 0.0, - "bfcl/bfcl.web_search.accuracy": 1.0, - "bfcl/bfcl.web_search.base_accuracy": 2.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 0.0, - "bfcl/bfcl.memory.accuracy": 4.52, - "bfcl/bfcl.memory.kv_accuracy": 3.87, - "bfcl/bfcl.memory.vector_accuracy": 7.1, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 2.58, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 0.0, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 100.0, - "bfcl/bfcl.format_sensitivity.max_delta": 0.0, - "bfcl/bfcl.format_sensitivity.stddev": 0.0 - } - }, - { - "id": "mistralai/Mistral-7B-Instruct-v0.1", - "name": "Mistral-7B-Instruct-v0.1", - "developer": "mistralai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4487, - "hfopenllm_v2/BBH": 0.3355, - "hfopenllm_v2/MATH Level 5": 0.0227, - "hfopenllm_v2/GPQA": 0.25, - "hfopenllm_v2/MUSR": 0.3848, - "hfopenllm_v2/MMLU-PRO": 0.2414 - } - }, - { - "id": "mistralai/Mistral-7B-Instruct-v0.2", - "name": "Mistral-7B-Instruct-v0.2", - "developer": "mistralai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5496, - "hfopenllm_v2/BBH": 0.446, - "hfopenllm_v2/MATH Level 5": 0.0302, - "hfopenllm_v2/GPQA": 0.276, - "hfopenllm_v2/MUSR": 0.3966, - "hfopenllm_v2/MMLU-PRO": 0.2717 - } - }, - { - "id": "mistralai/mistral-7b-instruct-v0.3", - "name": "Mistral Instruct v0.3 7B", - "developer": "mistralai", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.376, - "helm_capabilities/MMLU-Pro": 0.277, - "helm_capabilities/GPQA": 0.303, - "helm_capabilities/IFEval": 0.567, - "helm_capabilities/WildBench": 0.66, - "helm_capabilities/Omni-MATH": 0.072, - "helm_lite/Mean win rate": 0.196, - "helm_lite/NarrativeQA": 0.716, - "helm_lite/NaturalQuestions (closed-book)": 0.253, - "helm_lite/OpenbookQA": 0.79, - "helm_lite/MMLU": 0.51, - "helm_lite/MATH": 0.289, - "helm_lite/GSM8K": 0.538, - "helm_lite/LegalBench": 0.331, - "helm_lite/MedQA": 0.517, - "helm_lite/WMT 2014": 0.142, - "helm_mmlu/MMLU All Subjects": 0.599, - "helm_mmlu/Abstract Algebra": 0.27, - "helm_mmlu/Anatomy": 0.585, - "helm_mmlu/College Physics": 0.343, - "helm_mmlu/Computer Security": 0.7, - "helm_mmlu/Econometrics": 0.421, - "helm_mmlu/Global Facts": 0.33, - "helm_mmlu/Jurisprudence": 0.713, - "helm_mmlu/Philosophy": 0.659, - "helm_mmlu/Professional Psychology": 0.641, - "helm_mmlu/Us Foreign Policy": 0.79, - "helm_mmlu/Astronomy": 0.638, - "helm_mmlu/Business Ethics": 0.57, - "helm_mmlu/Clinical Knowledge": 0.687, - "helm_mmlu/Conceptual Physics": 0.549, - "helm_mmlu/Electrical Engineering": 0.572, - "helm_mmlu/Elementary Mathematics": 0.402, - "helm_mmlu/Formal Logic": 0.397, - "helm_mmlu/High School World History": 0.759, - "helm_mmlu/Human Sexuality": 0.702, - "helm_mmlu/International Law": 0.76, - "helm_mmlu/Logical Fallacies": 0.712, - "helm_mmlu/Machine Learning": 0.455, - "helm_mmlu/Management": 0.767, - "helm_mmlu/Marketing": 0.842, - "helm_mmlu/Medical Genetics": 0.75, - "helm_mmlu/Miscellaneous": 0.785, - "helm_mmlu/Moral Scenarios": 0.393, - "helm_mmlu/Nutrition": 0.676, - "helm_mmlu/Prehistory": 0.673, - "helm_mmlu/Public Relations": 0.636, - "helm_mmlu/Security Studies": 0.682, - "helm_mmlu/Sociology": 0.806, - "helm_mmlu/Virology": 0.47, - "helm_mmlu/World Religions": 0.825, - "helm_mmlu/Mean win rate": 0.509, - "hfopenllm_v2/IFEval": 0.5465, - "hfopenllm_v2/BBH": 0.4722, - "hfopenllm_v2/MATH Level 5": 0.0385, - "hfopenllm_v2/GPQA": 0.2794, - "hfopenllm_v2/MUSR": 0.3739, - "hfopenllm_v2/MMLU-PRO": 0.3075 - } - }, - { - "id": "mistralai/mistral-7b-v0.1", - "name": "Mistral v0.1 7B", - "developer": "mistralai", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.292, - "helm_lite/NarrativeQA": 0.716, - "helm_lite/NaturalQuestions (closed-book)": 0.367, - "helm_lite/OpenbookQA": 0.776, - "helm_lite/MMLU": 0.584, - "helm_lite/MATH": 0.297, - "helm_lite/GSM8K": 0.377, - "helm_lite/LegalBench": 0.58, - "helm_lite/MedQA": 0.525, - "helm_lite/WMT 2014": 0.16, - "helm_mmlu/MMLU All Subjects": 0.566, - "helm_mmlu/Abstract Algebra": 0.25, - "helm_mmlu/Anatomy": 0.467, - "helm_mmlu/College Physics": 0.314, - "helm_mmlu/Computer Security": 0.69, - "helm_mmlu/Econometrics": 0.351, - "helm_mmlu/Global Facts": 0.29, - "helm_mmlu/Jurisprudence": 0.667, - "helm_mmlu/Philosophy": 0.63, - "helm_mmlu/Professional Psychology": 0.578, - "helm_mmlu/Us Foreign Policy": 0.79, - "helm_mmlu/Astronomy": 0.599, - "helm_mmlu/Business Ethics": 0.56, - "helm_mmlu/Clinical Knowledge": 0.653, - "helm_mmlu/Conceptual Physics": 0.451, - "helm_mmlu/Electrical Engineering": 0.538, - "helm_mmlu/Elementary Mathematics": 0.32, - "helm_mmlu/Formal Logic": 0.365, - "helm_mmlu/High School World History": 0.726, - "helm_mmlu/Human Sexuality": 0.702, - "helm_mmlu/International Law": 0.76, - "helm_mmlu/Logical Fallacies": 0.693, - "helm_mmlu/Machine Learning": 0.438, - "helm_mmlu/Management": 0.709, - "helm_mmlu/Marketing": 0.833, - "helm_mmlu/Medical Genetics": 0.68, - "helm_mmlu/Miscellaneous": 0.72, - "helm_mmlu/Moral Scenarios": 0.33, - "helm_mmlu/Nutrition": 0.657, - "helm_mmlu/Prehistory": 0.642, - "helm_mmlu/Public Relations": 0.6, - "helm_mmlu/Security Studies": 0.731, - "helm_mmlu/Sociology": 0.831, - "helm_mmlu/Virology": 0.44, - "helm_mmlu/World Religions": 0.789, - "helm_mmlu/Mean win rate": 0.213, - "hfopenllm_v2/IFEval": 0.2386, - "hfopenllm_v2/BBH": 0.4419, - "hfopenllm_v2/MATH Level 5": 0.0295, - "hfopenllm_v2/GPQA": 0.2919, - "hfopenllm_v2/MUSR": 0.4139, - "hfopenllm_v2/MMLU-PRO": 0.3013 - } - }, - { - "id": "mistralai/Mistral-7B-v0.3", - "name": "Mistral-7B-v0.3", - "developer": "mistralai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2266, - "hfopenllm_v2/BBH": 0.4517, - "hfopenllm_v2/MATH Level 5": 0.0302, - "hfopenllm_v2/GPQA": 0.2919, - "hfopenllm_v2/MUSR": 0.4032, - "hfopenllm_v2/MMLU-PRO": 0.2953 - } - }, - { - "id": "mistralai/mistral-large-2402", - "name": "Mistral Large 2402", - "developer": "mistralai", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.328, - "helm_lite/NarrativeQA": 0.454, - "helm_lite/NaturalQuestions (closed-book)": 0.311, - "helm_lite/OpenbookQA": 0.894, - "helm_lite/MMLU": 0.638, - "helm_lite/MATH": 0.75, - "helm_lite/GSM8K": 0.694, - "helm_lite/LegalBench": 0.479, - "helm_lite/MedQA": 0.499, - "helm_lite/WMT 2014": 0.182, - "helm_mmlu/MMLU All Subjects": 0.688, - "helm_mmlu/Abstract Algebra": 0.45, - "helm_mmlu/Anatomy": 0.674, - "helm_mmlu/College Physics": 0.373, - "helm_mmlu/Computer Security": 0.8, - "helm_mmlu/Econometrics": 0.64, - "helm_mmlu/Global Facts": 0.34, - "helm_mmlu/Jurisprudence": 0.815, - "helm_mmlu/Philosophy": 0.794, - "helm_mmlu/Professional Psychology": 0.809, - "helm_mmlu/Us Foreign Policy": 0.92, - "helm_mmlu/Astronomy": 0.842, - "helm_mmlu/Business Ethics": 0.67, - "helm_mmlu/Clinical Knowledge": 0.751, - "helm_mmlu/Conceptual Physics": 0.574, - "helm_mmlu/Electrical Engineering": 0.545, - "helm_mmlu/Elementary Mathematics": 0.508, - "helm_mmlu/Formal Logic": 0.532, - "helm_mmlu/High School World History": 0.886, - "helm_mmlu/Human Sexuality": 0.847, - "helm_mmlu/International Law": 0.868, - "helm_mmlu/Logical Fallacies": 0.81, - "helm_mmlu/Machine Learning": 0.562, - "helm_mmlu/Management": 0.854, - "helm_mmlu/Marketing": 0.897, - "helm_mmlu/Medical Genetics": 0.74, - "helm_mmlu/Miscellaneous": 0.9, - "helm_mmlu/Moral Scenarios": 0.579, - "helm_mmlu/Nutrition": 0.791, - "helm_mmlu/Prehistory": 0.904, - "helm_mmlu/Public Relations": 0.709, - "helm_mmlu/Security Studies": 0.824, - "helm_mmlu/Sociology": 0.93, - "helm_mmlu/Virology": 0.554, - "helm_mmlu/World Religions": 0.883, - "helm_mmlu/Mean win rate": 0.464 - } - }, - { - "id": "mistralai/mistral-large-2407", - "name": "Mistral Large 2 2407", - "developer": "mistralai", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.744, - "helm_lite/NarrativeQA": 0.779, - "helm_lite/NaturalQuestions (closed-book)": 0.453, - "helm_lite/OpenbookQA": 0.932, - "helm_lite/MMLU": 0.725, - "helm_lite/MATH": 0.677, - "helm_lite/GSM8K": 0.912, - "helm_lite/LegalBench": 0.646, - "helm_lite/MedQA": 0.775, - "helm_lite/WMT 2014": 0.192, - "helm_mmlu/MMLU All Subjects": 0.8, - "helm_mmlu/Abstract Algebra": 0.7, - "helm_mmlu/Anatomy": 0.785, - "helm_mmlu/College Physics": 0.559, - "helm_mmlu/Computer Security": 0.81, - "helm_mmlu/Econometrics": 0.693, - "helm_mmlu/Global Facts": 0.56, - "helm_mmlu/Jurisprudence": 0.861, - "helm_mmlu/Philosophy": 0.826, - "helm_mmlu/Professional Psychology": 0.861, - "helm_mmlu/Us Foreign Policy": 0.9, - "helm_mmlu/Astronomy": 0.921, - "helm_mmlu/Business Ethics": 0.79, - "helm_mmlu/Clinical Knowledge": 0.864, - "helm_mmlu/Conceptual Physics": 0.864, - "helm_mmlu/Electrical Engineering": 0.793, - "helm_mmlu/Elementary Mathematics": 0.799, - "helm_mmlu/Formal Logic": 0.579, - "helm_mmlu/High School World History": 0.92, - "helm_mmlu/Human Sexuality": 0.924, - "helm_mmlu/International Law": 0.926, - "helm_mmlu/Logical Fallacies": 0.847, - "helm_mmlu/Machine Learning": 0.661, - "helm_mmlu/Management": 0.883, - "helm_mmlu/Marketing": 0.94, - "helm_mmlu/Medical Genetics": 0.9, - "helm_mmlu/Miscellaneous": 0.936, - "helm_mmlu/Moral Scenarios": 0.839, - "helm_mmlu/Nutrition": 0.827, - "helm_mmlu/Prehistory": 0.92, - "helm_mmlu/Public Relations": 0.764, - "helm_mmlu/Security Studies": 0.865, - "helm_mmlu/Sociology": 0.91, - "helm_mmlu/Virology": 0.59, - "helm_mmlu/World Religions": 0.865, - "helm_mmlu/Mean win rate": 0.24 - } - }, - { - "id": "mistralai/mistral-large-2411", - "name": "Mistral Large 2411", - "developer": "mistralai", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.598, - "helm_capabilities/MMLU-Pro": 0.599, - "helm_capabilities/GPQA": 0.435, - "helm_capabilities/IFEval": 0.876, - "helm_capabilities/WildBench": 0.801, - "helm_capabilities/Omni-MATH": 0.281 - } - }, - { - "id": "mistralai/mistral-large-2411-fc", - "name": "mistral-large-2411 (FC)", - "developer": "mistralai", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 46.0, - "bfcl/bfcl.overall.overall_accuracy": 38.37, - "bfcl/bfcl.overall.total_cost_usd": 115.98, - "bfcl/bfcl.overall.latency_mean_s": 2.04, - "bfcl/bfcl.overall.latency_std_s": 4.02, - "bfcl/bfcl.overall.latency_p95_s": 4.68, - "bfcl/bfcl.non_live.ast_accuracy": 84.65, - "bfcl/bfcl.non_live.simple_ast_accuracy": 72.08, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 93.5, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 89.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 83.5, - "bfcl/bfcl.live.live_accuracy": 81.87, - "bfcl/bfcl.live.live_simple_ast_accuracy": 87.21, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 80.72, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 81.25, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 75.0, - "bfcl/bfcl.multi_turn.accuracy": 14.12, - "bfcl/bfcl.multi_turn.base_accuracy": 18.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 11.5, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 13.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 13.5, - "bfcl/bfcl.web_search.accuracy": 28.0, - "bfcl/bfcl.web_search.base_accuracy": 41.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 15.0, - "bfcl/bfcl.memory.accuracy": 24.95, - "bfcl/bfcl.memory.kv_accuracy": 18.71, - "bfcl/bfcl.memory.vector_accuracy": 29.03, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 27.1, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 93.75, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 68.92 - } - }, - { - "id": "mistralai/mistral-large-2411-prompt", - "name": "mistral-large-2411 (Prompt)", - "developer": "mistralai", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 63.0, - "bfcl/bfcl.overall.overall_accuracy": 31.84, - "bfcl/bfcl.overall.total_cost_usd": 232.42, - "bfcl/bfcl.overall.latency_mean_s": 1.82, - "bfcl/bfcl.overall.latency_std_s": 7.15, - "bfcl/bfcl.overall.latency_p95_s": 4.08, - "bfcl/bfcl.non_live.ast_accuracy": 83.0, - "bfcl/bfcl.non_live.simple_ast_accuracy": 75.5, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 89.5, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 87.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 80.0, - "bfcl/bfcl.live.live_accuracy": 68.1, - "bfcl/bfcl.live.live_simple_ast_accuracy": 83.72, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 64.01, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 93.75, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 62.5, - "bfcl/bfcl.multi_turn.accuracy": 13.75, - "bfcl/bfcl.multi_turn.base_accuracy": 20.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 5.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 11.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 19.0, - "bfcl/bfcl.web_search.accuracy": 20.0, - "bfcl/bfcl.web_search.base_accuracy": 28.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 12.0, - "bfcl/bfcl.memory.accuracy": 23.66, - "bfcl/bfcl.memory.kv_accuracy": 16.77, - "bfcl/bfcl.memory.vector_accuracy": 30.97, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 23.23, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 93.75, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 38.77, - "bfcl/bfcl.format_sensitivity.max_delta": 13.5, - "bfcl/bfcl.format_sensitivity.stddev": 3.91 - } - }, - { - "id": "mistralai/Mistral-Large-Instruct-2411", - "name": "Mistral-Large-Instruct-2411", - "developer": "mistralai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8401, - "hfopenllm_v2/BBH": 0.6747, - "hfopenllm_v2/MATH Level 5": 0.4955, - "hfopenllm_v2/GPQA": 0.4371, - "hfopenllm_v2/MUSR": 0.454, - "hfopenllm_v2/MMLU-PRO": 0.5562 - } - }, - { - "id": "mistralai/mistral-medium-2312", - "name": "Mistral Medium 2312", - "developer": "mistralai", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.268, - "helm_lite/NarrativeQA": 0.449, - "helm_lite/NaturalQuestions (closed-book)": 0.29, - "helm_lite/OpenbookQA": 0.83, - "helm_lite/MMLU": 0.618, - "helm_lite/MATH": 0.565, - "helm_lite/GSM8K": 0.706, - "helm_lite/LegalBench": 0.452, - "helm_lite/MedQA": 0.61, - "helm_lite/WMT 2014": 0.169 - } - }, - { - "id": "mistralai/mistral-medium-2505", - "name": "Mistral-Medium-2505", - "developer": "mistralai", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 48.0, - "bfcl/bfcl.overall.overall_accuracy": 37.69, - "bfcl/bfcl.overall.total_cost_usd": 36.51, - "bfcl/bfcl.overall.latency_mean_s": 1.21, - "bfcl/bfcl.overall.latency_std_s": 3.5, - "bfcl/bfcl.overall.latency_p95_s": 2.86, - "bfcl/bfcl.non_live.ast_accuracy": 85.33, - "bfcl/bfcl.non_live.simple_ast_accuracy": 76.33, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 91.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 88.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 85.5, - "bfcl/bfcl.live.live_accuracy": 66.03, - "bfcl/bfcl.live.live_simple_ast_accuracy": 80.23, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 62.39, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 81.25, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 62.5, - "bfcl/bfcl.multi_turn.accuracy": 9.88, - "bfcl/bfcl.multi_turn.base_accuracy": 13.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 6.5, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 6.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 13.5, - "bfcl/bfcl.web_search.accuracy": 39.0, - "bfcl/bfcl.web_search.base_accuracy": 41.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 37.0, - "bfcl/bfcl.memory.accuracy": 21.72, - "bfcl/bfcl.memory.kv_accuracy": 16.13, - "bfcl/bfcl.memory.vector_accuracy": 14.84, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 34.19, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 75.0, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 74.49, - "bfcl/bfcl.format_sensitivity.max_delta": 21.5, - "bfcl/bfcl.format_sensitivity.stddev": 5.02 - } - }, - { - "id": "mistralai/mistral-medium-2505-fc", - "name": "Mistral-Medium-2505 (FC)", - "developer": "mistralai", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 49.0, - "bfcl/bfcl.overall.overall_accuracy": 37.56, - "bfcl/bfcl.overall.total_cost_usd": 18.8, - "bfcl/bfcl.overall.latency_mean_s": 1.6, - "bfcl/bfcl.overall.latency_std_s": 4.44, - "bfcl/bfcl.overall.latency_p95_s": 4.19, - "bfcl/bfcl.non_live.ast_accuracy": 67.44, - "bfcl/bfcl.non_live.simple_ast_accuracy": 39.75, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 78.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 83.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 69.0, - "bfcl/bfcl.live.live_accuracy": 67.95, - "bfcl/bfcl.live.live_simple_ast_accuracy": 67.05, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 68.09, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 81.25, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 62.5, - "bfcl/bfcl.multi_turn.accuracy": 10.75, - "bfcl/bfcl.multi_turn.base_accuracy": 15.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 7.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 7.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 13.0, - "bfcl/bfcl.web_search.accuracy": 35.0, - "bfcl/bfcl.web_search.base_accuracy": 36.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 34.0, - "bfcl/bfcl.memory.accuracy": 23.01, - "bfcl/bfcl.memory.kv_accuracy": 15.48, - "bfcl/bfcl.memory.vector_accuracy": 20.0, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 33.55, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 62.5, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 91.95 - } - }, - { - "id": "mistralai/mistral-medium-3", - "name": "mistral-medium-3", - "developer": "mistralai", - "evaluator_relationship": null, - "benchmark_scores": { - "global-mmlu-lite/Global MMLU Lite": 0.5511, - "global-mmlu-lite/Culturally Sensitive": 0.5391, - "global-mmlu-lite/Culturally Agnostic": 0.5631, - "global-mmlu-lite/Arabic": 0.455, - "global-mmlu-lite/English": 0.38, - "global-mmlu-lite/Bengali": 0.5175, - "global-mmlu-lite/German": 0.4775, - "global-mmlu-lite/French": 0.41, - "global-mmlu-lite/Hindi": 0.555, - "global-mmlu-lite/Indonesian": 0.515, - "global-mmlu-lite/Italian": 0.535, - "global-mmlu-lite/Japanese": 0.58, - "global-mmlu-lite/Korean": 0.595, - "global-mmlu-lite/Portuguese": 0.5175, - "global-mmlu-lite/Spanish": 0.5375, - "global-mmlu-lite/Swahili": 0.7075, - "global-mmlu-lite/Yoruba": 0.7675, - "global-mmlu-lite/Chinese": 0.535, - "global-mmlu-lite/Burmese": 0.7325 - } - }, - { - "id": "mistralai/Mistral-Nemo-Base-2407", - "name": "Mistral-Nemo-Base-2407", - "developer": "mistralai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.163, - "hfopenllm_v2/BBH": 0.5035, - "hfopenllm_v2/MATH Level 5": 0.0597, - "hfopenllm_v2/GPQA": 0.2936, - "hfopenllm_v2/MUSR": 0.3921, - "hfopenllm_v2/MMLU-PRO": 0.3472 - } - }, - { - "id": "mistralai/Mistral-Nemo-Instruct-2407", - "name": "Mistral-Nemo-Instruct-2407", - "developer": "mistralai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.638, - "hfopenllm_v2/BBH": 0.5037, - "hfopenllm_v2/MATH Level 5": 0.1269, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.39, - "hfopenllm_v2/MMLU-PRO": 0.3517 - } - }, - { - "id": "mistralai/mistral-small-2402", - "name": "Mistral Small 2402", - "developer": "mistralai", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.288, - "helm_lite/NarrativeQA": 0.519, - "helm_lite/NaturalQuestions (closed-book)": 0.304, - "helm_lite/OpenbookQA": 0.862, - "helm_lite/MMLU": 0.593, - "helm_lite/MATH": 0.621, - "helm_lite/GSM8K": 0.734, - "helm_lite/LegalBench": 0.389, - "helm_lite/MedQA": 0.616, - "helm_lite/WMT 2014": 0.169, - "helm_mmlu/MMLU All Subjects": 0.687, - "helm_mmlu/Abstract Algebra": 0.26, - "helm_mmlu/Anatomy": 0.674, - "helm_mmlu/College Physics": 0.402, - "helm_mmlu/Computer Security": 0.77, - "helm_mmlu/Econometrics": 0.614, - "helm_mmlu/Global Facts": 0.45, - "helm_mmlu/Jurisprudence": 0.833, - "helm_mmlu/Philosophy": 0.765, - "helm_mmlu/Professional Psychology": 0.768, - "helm_mmlu/Us Foreign Policy": 0.89, - "helm_mmlu/Astronomy": 0.77, - "helm_mmlu/Business Ethics": 0.71, - "helm_mmlu/Clinical Knowledge": 0.766, - "helm_mmlu/Conceptual Physics": 0.685, - "helm_mmlu/Electrical Engineering": 0.628, - "helm_mmlu/Elementary Mathematics": 0.415, - "helm_mmlu/Formal Logic": 0.516, - "helm_mmlu/High School World History": 0.857, - "helm_mmlu/Human Sexuality": 0.824, - "helm_mmlu/International Law": 0.826, - "helm_mmlu/Logical Fallacies": 0.804, - "helm_mmlu/Machine Learning": 0.562, - "helm_mmlu/Management": 0.786, - "helm_mmlu/Marketing": 0.906, - "helm_mmlu/Medical Genetics": 0.75, - "helm_mmlu/Miscellaneous": 0.844, - "helm_mmlu/Moral Scenarios": 0.575, - "helm_mmlu/Nutrition": 0.761, - "helm_mmlu/Prehistory": 0.802, - "helm_mmlu/Public Relations": 0.773, - "helm_mmlu/Security Studies": 0.788, - "helm_mmlu/Sociology": 0.871, - "helm_mmlu/Virology": 0.542, - "helm_mmlu/World Religions": 0.848, - "helm_mmlu/Mean win rate": 0.54 - } - }, - { - "id": "mistralai/Mistral-Small-24B-Base-2501", - "name": "Mistral-Small-24B-Base-2501", - "developer": "mistralai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1672, - "hfopenllm_v2/BBH": 0.6442, - "hfopenllm_v2/MATH Level 5": 0.1971, - "hfopenllm_v2/GPQA": 0.3876, - "hfopenllm_v2/MUSR": 0.4237, - "hfopenllm_v2/MMLU-PRO": 0.5406 - } - }, - { - "id": "mistralai/mistral-small-2503", - "name": "mistral-small-2503", - "developer": "mistralai", - "evaluator_relationship": null, - "benchmark_scores": { - "global-mmlu-lite/Global MMLU Lite": 0.7852, - "global-mmlu-lite/Culturally Sensitive": 0.7537, - "global-mmlu-lite/Culturally Agnostic": 0.8166, - "global-mmlu-lite/Arabic": 0.7875, - "global-mmlu-lite/English": 0.8, - "global-mmlu-lite/Bengali": 0.7725, - "global-mmlu-lite/German": 0.7975, - "global-mmlu-lite/French": 0.8, - "global-mmlu-lite/Hindi": 0.795, - "global-mmlu-lite/Indonesian": 0.785, - "global-mmlu-lite/Italian": 0.805, - "global-mmlu-lite/Japanese": 0.77, - "global-mmlu-lite/Korean": 0.79, - "global-mmlu-lite/Portuguese": 0.7925, - "global-mmlu-lite/Spanish": 0.7825, - "global-mmlu-lite/Swahili": 0.775, - "global-mmlu-lite/Yoruba": 0.735, - "global-mmlu-lite/Chinese": 0.7925, - "global-mmlu-lite/Burmese": 0.7825, - "helm_capabilities/Mean score": 0.558, - "helm_capabilities/MMLU-Pro": 0.61, - "helm_capabilities/GPQA": 0.392, - "helm_capabilities/IFEval": 0.75, - "helm_capabilities/WildBench": 0.788, - "helm_capabilities/Omni-MATH": 0.248 - } - }, - { - "id": "mistralai/mistral-small-2506-fc", - "name": "Mistral-small-2506 (FC)", - "developer": "mistralai", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 51.0, - "bfcl/bfcl.overall.overall_accuracy": 37.15, - "bfcl/bfcl.overall.total_cost_usd": 5.2, - "bfcl/bfcl.overall.latency_mean_s": 1.48, - "bfcl/bfcl.overall.latency_std_s": 18.25, - "bfcl/bfcl.overall.latency_p95_s": 2.5, - "bfcl/bfcl.non_live.ast_accuracy": 73.6, - "bfcl/bfcl.non_live.simple_ast_accuracy": 38.92, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 93.5, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 83.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 78.5, - "bfcl/bfcl.live.live_accuracy": 77.28, - "bfcl/bfcl.live.live_simple_ast_accuracy": 69.38, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 79.39, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 75.0, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 70.83, - "bfcl/bfcl.multi_turn.accuracy": 11.5, - "bfcl/bfcl.multi_turn.base_accuracy": 17.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 6.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 10.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 12.0, - "bfcl/bfcl.web_search.accuracy": 31.0, - "bfcl/bfcl.web_search.base_accuracy": 37.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 25.0, - "bfcl/bfcl.memory.accuracy": 18.06, - "bfcl/bfcl.memory.kv_accuracy": 8.39, - "bfcl/bfcl.memory.vector_accuracy": 14.19, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 31.61, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 87.5, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 87.94 - } - }, - { - "id": "mistralai/mistral-small-2506-prompt", - "name": "Mistral-Small-2506 (Prompt)", - "developer": "mistralai", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 59.0, - "bfcl/bfcl.overall.overall_accuracy": 32.38, - "bfcl/bfcl.overall.total_cost_usd": 6.91, - "bfcl/bfcl.overall.latency_mean_s": 0.92, - "bfcl/bfcl.overall.latency_std_s": 6.79, - "bfcl/bfcl.overall.latency_p95_s": 2.02, - "bfcl/bfcl.non_live.ast_accuracy": 89.69, - "bfcl/bfcl.non_live.simple_ast_accuracy": 78.75, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 96.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 92.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 91.5, - "bfcl/bfcl.live.live_accuracy": 79.05, - "bfcl/bfcl.live.live_simple_ast_accuracy": 81.4, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 78.54, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 93.75, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 66.67, - "bfcl/bfcl.multi_turn.accuracy": 14.75, - "bfcl/bfcl.multi_turn.base_accuracy": 20.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 17.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 9.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 12.0, - "bfcl/bfcl.web_search.accuracy": 7.5, - "bfcl/bfcl.web_search.base_accuracy": 9.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 6.0, - "bfcl/bfcl.memory.accuracy": 15.05, - "bfcl/bfcl.memory.kv_accuracy": 2.58, - "bfcl/bfcl.memory.vector_accuracy": 11.61, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 30.97, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 93.75, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 65.73, - "bfcl/bfcl.format_sensitivity.max_delta": 50.0, - "bfcl/bfcl.format_sensitivity.stddev": 13.57 - } - }, - { - "id": "mistralai/Mistral-Small-Instruct-2409", - "name": "Mistral-Small-Instruct-2409", - "developer": "mistralai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.667, - "hfopenllm_v2/BBH": 0.5213, - "hfopenllm_v2/MATH Level 5": 0.1435, - "hfopenllm_v2/GPQA": 0.3238, - "hfopenllm_v2/MUSR": 0.3632, - "hfopenllm_v2/MMLU-PRO": 0.396 - } - }, - { - "id": "mistralai/Mistral-v0.1-7B", - "name": "Mistral v0.1 7B", - "developer": "mistralai", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.884, - "helm_classic/MMLU": 0.572, - "helm_classic/BoolQ": 0.874, - "helm_classic/NarrativeQA": 0.716, - "helm_classic/NaturalQuestions (open-book)": 0.687, - "helm_classic/QuAC": 0.423, - "helm_classic/HellaSwag": -1.0, - "helm_classic/OpenbookQA": -1.0, - "helm_classic/TruthfulQA": 0.422, - "helm_classic/MS MARCO (TREC)": -1.0, - "helm_classic/CNN/DailyMail": -1.0, - "helm_classic/XSUM": -1.0, - "helm_classic/IMDB": 0.962, - "helm_classic/CivilComments": 0.624, - "helm_classic/RAFT": 0.707 - } - }, - { - "id": "mistralai/mixtral-8x22b", - "name": "Mixtral 8x22B", - "developer": "mistralai", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.705, - "helm_lite/NarrativeQA": 0.779, - "helm_lite/NaturalQuestions (closed-book)": 0.478, - "helm_lite/OpenbookQA": 0.882, - "helm_lite/MMLU": 0.701, - "helm_lite/MATH": 0.656, - "helm_lite/GSM8K": 0.8, - "helm_lite/LegalBench": 0.708, - "helm_lite/MedQA": 0.704, - "helm_lite/WMT 2014": 0.209, - "helm_mmlu/MMLU All Subjects": 0.778, - "helm_mmlu/Abstract Algebra": 0.48, - "helm_mmlu/Anatomy": 0.741, - "helm_mmlu/College Physics": 0.569, - "helm_mmlu/Computer Security": 0.84, - "helm_mmlu/Econometrics": 0.667, - "helm_mmlu/Global Facts": 0.56, - "helm_mmlu/Jurisprudence": 0.852, - "helm_mmlu/Philosophy": 0.842, - "helm_mmlu/Professional Psychology": 0.845, - "helm_mmlu/Us Foreign Policy": 0.95, - "helm_mmlu/Astronomy": 0.882, - "helm_mmlu/Business Ethics": 0.74, - "helm_mmlu/Clinical Knowledge": 0.819, - "helm_mmlu/Conceptual Physics": 0.796, - "helm_mmlu/Electrical Engineering": 0.766, - "helm_mmlu/Elementary Mathematics": 0.622, - "helm_mmlu/Formal Logic": 0.627, - "helm_mmlu/High School World History": 0.895, - "helm_mmlu/Human Sexuality": 0.885, - "helm_mmlu/International Law": 0.917, - "helm_mmlu/Logical Fallacies": 0.877, - "helm_mmlu/Machine Learning": 0.661, - "helm_mmlu/Management": 0.883, - "helm_mmlu/Marketing": 0.915, - "helm_mmlu/Medical Genetics": 0.85, - "helm_mmlu/Miscellaneous": 0.899, - "helm_mmlu/Moral Scenarios": 0.646, - "helm_mmlu/Nutrition": 0.866, - "helm_mmlu/Prehistory": 0.87, - "helm_mmlu/Public Relations": 0.755, - "helm_mmlu/Security Studies": 0.865, - "helm_mmlu/Sociology": 0.92, - "helm_mmlu/Virology": 0.596, - "helm_mmlu/World Religions": 0.901, - "helm_mmlu/Mean win rate": 0.598 - } - }, - { - "id": "mistralai/Mixtral-8x22B-Instruct-v0.1", - "name": "Mixtral-8x22B-Instruct-v0.1", - "developer": "mistralai", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.478, - "helm_capabilities/MMLU-Pro": 0.46, - "helm_capabilities/GPQA": 0.334, - "helm_capabilities/IFEval": 0.724, - "helm_capabilities/WildBench": 0.711, - "helm_capabilities/Omni-MATH": 0.163, - "hfopenllm_v2/IFEval": 0.7184, - "hfopenllm_v2/BBH": 0.6125, - "hfopenllm_v2/MATH Level 5": 0.1873, - "hfopenllm_v2/GPQA": 0.3733, - "hfopenllm_v2/MUSR": 0.4311, - "hfopenllm_v2/MMLU-PRO": 0.4483 - } - }, - { - "id": "mistralai/Mixtral-8x22B-v0.1", - "name": "Mixtral-8x22B-v0.1", - "developer": "mistralai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2583, - "hfopenllm_v2/BBH": 0.624, - "hfopenllm_v2/MATH Level 5": 0.1835, - "hfopenllm_v2/GPQA": 0.3758, - "hfopenllm_v2/MUSR": 0.4037, - "hfopenllm_v2/MMLU-PRO": 0.4639 - } - }, - { - "id": "mistralai/mixtral-8x7b-32kseqlen", - "name": "Mixtral 8x7B 32K seqlen", - "developer": "mistralai", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.51, - "helm_lite/NarrativeQA": 0.767, - "helm_lite/NaturalQuestions (closed-book)": 0.427, - "helm_lite/OpenbookQA": 0.868, - "helm_lite/MMLU": 0.649, - "helm_lite/MATH": 0.494, - "helm_lite/GSM8K": 0.622, - "helm_lite/LegalBench": 0.63, - "helm_lite/MedQA": 0.652, - "helm_lite/WMT 2014": 0.19, - "helm_mmlu/MMLU All Subjects": 0.717, - "helm_mmlu/Abstract Algebra": 0.38, - "helm_mmlu/Anatomy": 0.696, - "helm_mmlu/College Physics": 0.51, - "helm_mmlu/Computer Security": 0.81, - "helm_mmlu/Econometrics": 0.605, - "helm_mmlu/Global Facts": 0.46, - "helm_mmlu/Jurisprudence": 0.833, - "helm_mmlu/Philosophy": 0.797, - "helm_mmlu/Professional Psychology": 0.779, - "helm_mmlu/Us Foreign Policy": 0.93, - "helm_mmlu/Astronomy": 0.829, - "helm_mmlu/Business Ethics": 0.72, - "helm_mmlu/Clinical Knowledge": 0.785, - "helm_mmlu/Conceptual Physics": 0.681, - "helm_mmlu/Electrical Engineering": 0.676, - "helm_mmlu/Elementary Mathematics": 0.476, - "helm_mmlu/Formal Logic": 0.532, - "helm_mmlu/High School World History": 0.886, - "helm_mmlu/Human Sexuality": 0.87, - "helm_mmlu/International Law": 0.86, - "helm_mmlu/Logical Fallacies": 0.767, - "helm_mmlu/Machine Learning": 0.509, - "helm_mmlu/Management": 0.845, - "helm_mmlu/Marketing": 0.923, - "helm_mmlu/Medical Genetics": 0.76, - "helm_mmlu/Miscellaneous": 0.881, - "helm_mmlu/Moral Scenarios": 0.444, - "helm_mmlu/Nutrition": 0.83, - "helm_mmlu/Prehistory": 0.849, - "helm_mmlu/Public Relations": 0.682, - "helm_mmlu/Security Studies": 0.792, - "helm_mmlu/Sociology": 0.871, - "helm_mmlu/Virology": 0.506, - "helm_mmlu/World Religions": 0.871, - "helm_mmlu/Mean win rate": 0.689 - } - }, - { - "id": "mistralai/Mixtral-8x7B-Instruct-v0.1", - "name": "Mixtral-8x7B-Instruct-v0.1", - "developer": "mistralai", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.397, - "helm_capabilities/MMLU-Pro": 0.335, - "helm_capabilities/GPQA": 0.296, - "helm_capabilities/IFEval": 0.575, - "helm_capabilities/WildBench": 0.673, - "helm_capabilities/Omni-MATH": 0.105, - "hfopenllm_v2/IFEval": 0.5599, - "hfopenllm_v2/BBH": 0.4962, - "hfopenllm_v2/MATH Level 5": 0.0914, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.4203, - "hfopenllm_v2/MMLU-PRO": 0.3692, - "reward-bench/Score": 0.7455, - "reward-bench/Chat": 0.9497, - "reward-bench/Chat Hard": 0.6404, - "reward-bench/Safety": 0.7257, - "reward-bench/Reasoning": 0.7872, - "reward-bench/Prior Sets (0.5 weight)": 0.5033 - } - }, - { - "id": "mistralai/Mixtral-8x7B-v0.1", - "name": "Mixtral-8x7B-v0.1", - "developer": "mistralai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2326, - "hfopenllm_v2/BBH": 0.5098, - "hfopenllm_v2/MATH Level 5": 0.0937, - "hfopenllm_v2/GPQA": 0.3205, - "hfopenllm_v2/MUSR": 0.4413, - "hfopenllm_v2/MMLU-PRO": 0.3871 - } - }, - { - "id": "mistralai/open-mistral-nemo-2407", - "name": "Mistral NeMo 2402", - "developer": "mistralai", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.333, - "helm_lite/NarrativeQA": 0.731, - "helm_lite/NaturalQuestions (closed-book)": 0.265, - "helm_lite/OpenbookQA": 0.822, - "helm_lite/MMLU": 0.604, - "helm_lite/MATH": 0.668, - "helm_lite/GSM8K": 0.782, - "helm_lite/LegalBench": 0.415, - "helm_lite/MedQA": 0.59, - "helm_lite/WMT 2014": 0.177, - "helm_mmlu/MMLU All Subjects": 0.653, - "helm_mmlu/Abstract Algebra": 0.29, - "helm_mmlu/Anatomy": 0.607, - "helm_mmlu/College Physics": 0.373, - "helm_mmlu/Computer Security": 0.81, - "helm_mmlu/Econometrics": 0.561, - "helm_mmlu/Global Facts": 0.4, - "helm_mmlu/Jurisprudence": 0.796, - "helm_mmlu/Philosophy": 0.733, - "helm_mmlu/Professional Psychology": 0.588, - "helm_mmlu/Us Foreign Policy": 0.89, - "helm_mmlu/Astronomy": 0.691, - "helm_mmlu/Business Ethics": 0.49, - "helm_mmlu/Clinical Knowledge": 0.736, - "helm_mmlu/Conceptual Physics": 0.647, - "helm_mmlu/Electrical Engineering": 0.531, - "helm_mmlu/Elementary Mathematics": 0.439, - "helm_mmlu/Formal Logic": 0.405, - "helm_mmlu/High School World History": 0.848, - "helm_mmlu/Human Sexuality": 0.702, - "helm_mmlu/International Law": 0.769, - "helm_mmlu/Logical Fallacies": 0.791, - "helm_mmlu/Machine Learning": 0.402, - "helm_mmlu/Management": 0.796, - "helm_mmlu/Marketing": 0.889, - "helm_mmlu/Medical Genetics": 0.78, - "helm_mmlu/Miscellaneous": 0.861, - "helm_mmlu/Moral Scenarios": 0.381, - "helm_mmlu/Nutrition": 0.709, - "helm_mmlu/Prehistory": 0.765, - "helm_mmlu/Public Relations": 0.718, - "helm_mmlu/Security Studies": 0.771, - "helm_mmlu/Sociology": 0.726, - "helm_mmlu/Virology": 0.56, - "helm_mmlu/World Religions": 0.789, - "helm_mmlu/Mean win rate": 0.215 - } - }, - { - "id": "mistralai/open-mistral-nemo-2407-fc", - "name": "Open-Mistral-Nemo-2407 (FC)", - "developer": "mistralai", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 78.0, - "bfcl/bfcl.overall.overall_accuracy": 27.63, - "bfcl/bfcl.overall.total_cost_usd": 8.12, - "bfcl/bfcl.overall.latency_mean_s": 1.07, - "bfcl/bfcl.overall.latency_std_s": 11.93, - "bfcl/bfcl.overall.latency_p95_s": 1.39, - "bfcl/bfcl.non_live.ast_accuracy": 82.81, - "bfcl/bfcl.non_live.simple_ast_accuracy": 65.75, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 92.5, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 88.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 84.5, - "bfcl/bfcl.live.live_accuracy": 73.8, - "bfcl/bfcl.live.live_simple_ast_accuracy": 78.68, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 72.84, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 68.75, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 66.67, - "bfcl/bfcl.multi_turn.accuracy": 7.75, - "bfcl/bfcl.multi_turn.base_accuracy": 12.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 6.5, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 7.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 4.5, - "bfcl/bfcl.web_search.accuracy": 7.0, - "bfcl/bfcl.web_search.base_accuracy": 9.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 5.0, - "bfcl/bfcl.memory.accuracy": 10.32, - "bfcl/bfcl.memory.kv_accuracy": 8.39, - "bfcl/bfcl.memory.vector_accuracy": 9.68, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 12.9, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 81.25, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 61.77 - } - }, - { - "id": "mistralai/open-mistral-nemo-2407-prompt", - "name": "Open-Mistral-Nemo-2407 (Prompt)", - "developer": "mistralai", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 102.0, - "bfcl/bfcl.overall.overall_accuracy": 19.31, - "bfcl/bfcl.overall.total_cost_usd": 13.8, - "bfcl/bfcl.overall.latency_mean_s": 0.84, - "bfcl/bfcl.overall.latency_std_s": 7.05, - "bfcl/bfcl.overall.latency_p95_s": 1.32, - "bfcl/bfcl.non_live.ast_accuracy": 88.46, - "bfcl/bfcl.non_live.simple_ast_accuracy": 79.33, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 92.5, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 90.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 91.5, - "bfcl/bfcl.live.live_accuracy": 73.95, - "bfcl/bfcl.live.live_simple_ast_accuracy": 78.29, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 73.03, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 87.5, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 58.33, - "bfcl/bfcl.multi_turn.accuracy": 0.75, - "bfcl/bfcl.multi_turn.base_accuracy": 0.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 1.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 0.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 1.5, - "bfcl/bfcl.web_search.accuracy": 2.5, - "bfcl/bfcl.web_search.base_accuracy": 3.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 2.0, - "bfcl/bfcl.memory.accuracy": 8.6, - "bfcl/bfcl.memory.kv_accuracy": 9.68, - "bfcl/bfcl.memory.vector_accuracy": 9.68, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 6.45, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 93.75, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 6.28, - "bfcl/bfcl.format_sensitivity.max_delta": 14.5, - "bfcl/bfcl.format_sensitivity.stddev": 4.6 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/mixtao.json b/data/developers/mixtao.json deleted file mode 100644 index 0d228718cf1a13d82689d129c9ec3a15e10653bd..0000000000000000000000000000000000000000 --- a/data/developers/mixtao.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "mixtao", - "models": [ - { - "id": "mixtao/MixTAO-7Bx2-MoE-v8.1", - "name": "MixTAO-7Bx2-MoE-v8.1", - "developer": "mixtao", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4162, - "hfopenllm_v2/BBH": 0.5189, - "hfopenllm_v2/MATH Level 5": 0.0906, - "hfopenllm_v2/GPQA": 0.2844, - "hfopenllm_v2/MUSR": 0.4463, - "hfopenllm_v2/MMLU-PRO": 0.3123 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/mkurman.json b/data/developers/mkurman.json deleted file mode 100644 index 12ddb403e528047f75f58de4389102db97a272fe..0000000000000000000000000000000000000000 --- a/data/developers/mkurman.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "developer": "mkurman", - "models": [ - { - "id": "mkurman/llama-3.2-MEDIT-3B-o1", - "name": "llama-3.2-MEDIT-3B-o1", - "developer": "mkurman", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4382, - "hfopenllm_v2/BBH": 0.44, - "hfopenllm_v2/MATH Level 5": 0.1307, - "hfopenllm_v2/GPQA": 0.2659, - "hfopenllm_v2/MUSR": 0.3565, - "hfopenllm_v2/MMLU-PRO": 0.2741 - } - }, - { - "id": "mkurman/phi-4-MedIT-11B-exp-1", - "name": "phi-4-MedIT-11B-exp-1", - "developer": "mkurman", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5948, - "hfopenllm_v2/BBH": 0.5414, - "hfopenllm_v2/MATH Level 5": 0.0899, - "hfopenllm_v2/GPQA": 0.3012, - "hfopenllm_v2/MUSR": 0.3848, - "hfopenllm_v2/MMLU-PRO": 0.3825 - } - }, - { - "id": "mkurman/phi4-MedIT-10B-o1", - "name": "phi4-MedIT-10B-o1", - "developer": "mkurman", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3463, - "hfopenllm_v2/BBH": 0.5198, - "hfopenllm_v2/MATH Level 5": 0.1148, - "hfopenllm_v2/GPQA": 0.2458, - "hfopenllm_v2/MUSR": 0.3968, - "hfopenllm_v2/MMLU-PRO": 0.3507 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/mkxu.json b/data/developers/mkxu.json deleted file mode 100644 index 9d6fd0d4ad640a4d5ba72a7f3963c78e0818c27c..0000000000000000000000000000000000000000 --- a/data/developers/mkxu.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "mkxu", - "models": [ - { - "id": "mkxu/llama-3-8b-instruct-fpo", - "name": "llama-3-8b-instruct-fpo", - "developer": "mkxu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.679, - "hfopenllm_v2/BBH": 0.4959, - "hfopenllm_v2/MATH Level 5": 0.0733, - "hfopenllm_v2/GPQA": 0.2777, - "hfopenllm_v2/MUSR": 0.3658, - "hfopenllm_v2/MMLU-PRO": 0.3605 - } - }, - { - "id": "mkxu/llama-3-8b-po1", - "name": "llama-3-8b-po1", - "developer": "mkxu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4081, - "hfopenllm_v2/BBH": 0.4976, - "hfopenllm_v2/MATH Level 5": 0.0702, - "hfopenllm_v2/GPQA": 0.297, - "hfopenllm_v2/MUSR": 0.3804, - "hfopenllm_v2/MMLU-PRO": 0.3562 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/mlabonne.json b/data/developers/mlabonne.json deleted file mode 100644 index 2620a8c4e8931697abdcd44e4a4aae7c1e430da5..0000000000000000000000000000000000000000 --- a/data/developers/mlabonne.json +++ /dev/null @@ -1,201 +0,0 @@ -{ - "developer": "mlabonne", - "models": [ - { - "id": "mlabonne/AlphaMonarch-7B", - "name": "AlphaMonarch-7B", - "developer": "mlabonne", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4939, - "hfopenllm_v2/BBH": 0.4626, - "hfopenllm_v2/MATH Level 5": 0.0408, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.4121, - "hfopenllm_v2/MMLU-PRO": 0.2473 - } - }, - { - "id": "mlabonne/Beyonder-4x7B-v3", - "name": "Beyonder-4x7B-v3", - "developer": "mlabonne", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5608, - "hfopenllm_v2/BBH": 0.4671, - "hfopenllm_v2/MATH Level 5": 0.0536, - "hfopenllm_v2/GPQA": 0.2852, - "hfopenllm_v2/MUSR": 0.4045, - "hfopenllm_v2/MMLU-PRO": 0.2512 - } - }, - { - "id": "mlabonne/BigQwen2.5-52B-Instruct", - "name": "BigQwen2.5-52B-Instruct", - "developer": "mlabonne", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7913, - "hfopenllm_v2/BBH": 0.7121, - "hfopenllm_v2/MATH Level 5": 0.5476, - "hfopenllm_v2/GPQA": 0.302, - "hfopenllm_v2/MUSR": 0.4113, - "hfopenllm_v2/MMLU-PRO": 0.5519 - } - }, - { - "id": "mlabonne/BigQwen2.5-Echo-47B-Instruct", - "name": "BigQwen2.5-Echo-47B-Instruct", - "developer": "mlabonne", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7357, - "hfopenllm_v2/BBH": 0.6125, - "hfopenllm_v2/MATH Level 5": 0.4381, - "hfopenllm_v2/GPQA": 0.3146, - "hfopenllm_v2/MUSR": 0.4125, - "hfopenllm_v2/MMLU-PRO": 0.4734 - } - }, - { - "id": "mlabonne/ChimeraLlama-3-8B-v2", - "name": "ChimeraLlama-3-8B-v2", - "developer": "mlabonne", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4469, - "hfopenllm_v2/BBH": 0.5046, - "hfopenllm_v2/MATH Level 5": 0.0906, - "hfopenllm_v2/GPQA": 0.2852, - "hfopenllm_v2/MUSR": 0.3791, - "hfopenllm_v2/MMLU-PRO": 0.3569 - } - }, - { - "id": "mlabonne/ChimeraLlama-3-8B-v3", - "name": "ChimeraLlama-3-8B-v3", - "developer": "mlabonne", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4408, - "hfopenllm_v2/BBH": 0.4978, - "hfopenllm_v2/MATH Level 5": 0.0884, - "hfopenllm_v2/GPQA": 0.2919, - "hfopenllm_v2/MUSR": 0.4004, - "hfopenllm_v2/MMLU-PRO": 0.3669 - } - }, - { - "id": "mlabonne/Daredevil-8B", - "name": "Daredevil-8B", - "developer": "mlabonne", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4548, - "hfopenllm_v2/BBH": 0.5194, - "hfopenllm_v2/MATH Level 5": 0.1065, - "hfopenllm_v2/GPQA": 0.3079, - "hfopenllm_v2/MUSR": 0.3939, - "hfopenllm_v2/MMLU-PRO": 0.3831 - } - }, - { - "id": "mlabonne/Daredevil-8B-abliterated", - "name": "Daredevil-8B-abliterated", - "developer": "mlabonne", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4426, - "hfopenllm_v2/BBH": 0.4254, - "hfopenllm_v2/MATH Level 5": 0.0944, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.407, - "hfopenllm_v2/MMLU-PRO": 0.3701 - } - }, - { - "id": "mlabonne/Hermes-3-Llama-3.1-70B-lorablated", - "name": "Hermes-3-Llama-3.1-70B-lorablated", - "developer": "mlabonne", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3424, - "hfopenllm_v2/BBH": 0.6693, - "hfopenllm_v2/MATH Level 5": 0.2243, - "hfopenllm_v2/GPQA": 0.3658, - "hfopenllm_v2/MUSR": 0.5029, - "hfopenllm_v2/MMLU-PRO": 0.4679 - } - }, - { - "id": "mlabonne/Meta-Llama-3.1-8B-Instruct-abliterated", - "name": "Meta-Llama-3.1-8B-Instruct-abliterated", - "developer": "mlabonne", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7329, - "hfopenllm_v2/BBH": 0.4874, - "hfopenllm_v2/MATH Level 5": 0.0687, - "hfopenllm_v2/GPQA": 0.2567, - "hfopenllm_v2/MUSR": 0.3649, - "hfopenllm_v2/MMLU-PRO": 0.3503 - } - }, - { - "id": "mlabonne/NeuralBeagle14-7B", - "name": "NeuralBeagle14-7B", - "developer": "mlabonne", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4935, - "hfopenllm_v2/BBH": 0.4628, - "hfopenllm_v2/MATH Level 5": 0.0521, - "hfopenllm_v2/GPQA": 0.2819, - "hfopenllm_v2/MUSR": 0.4319, - "hfopenllm_v2/MMLU-PRO": 0.2601 - } - }, - { - "id": "mlabonne/NeuralDaredevil-8B-abliterated", - "name": "NeuralDaredevil-8B-abliterated", - "developer": "mlabonne", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7561, - "hfopenllm_v2/BBH": 0.5111, - "hfopenllm_v2/MATH Level 5": 0.0906, - "hfopenllm_v2/GPQA": 0.3062, - "hfopenllm_v2/MUSR": 0.4019, - "hfopenllm_v2/MMLU-PRO": 0.3841 - } - }, - { - "id": "mlabonne/OrpoLlama-3-8B", - "name": "OrpoLlama-3-8B", - "developer": "mlabonne", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3653, - "hfopenllm_v2/BBH": 0.4424, - "hfopenllm_v2/MATH Level 5": 0.0559, - "hfopenllm_v2/GPQA": 0.2794, - "hfopenllm_v2/MUSR": 0.3579, - "hfopenllm_v2/MMLU-PRO": 0.2705 - } - }, - { - "id": "mlabonne/phixtral-2x2_8", - "name": "phixtral-2x2_8", - "developer": "mlabonne", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3431, - "hfopenllm_v2/BBH": 0.4889, - "hfopenllm_v2/MATH Level 5": 0.0355, - "hfopenllm_v2/GPQA": 0.2651, - "hfopenllm_v2/MUSR": 0.3644, - "hfopenllm_v2/MMLU-PRO": 0.2551 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/mlp-ktlim.json b/data/developers/mlp-ktlim.json deleted file mode 100644 index e1744a40c6fb3456db7ed14ab1be534c66d21527..0000000000000000000000000000000000000000 --- a/data/developers/mlp-ktlim.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "MLP-KTLim", - "models": [ - { - "id": "MLP-KTLim/llama-3-Korean-Bllossom-8B", - "name": "llama-3-Korean-Bllossom-8B", - "developer": "MLP-KTLim", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5113, - "hfopenllm_v2/BBH": 0.49, - "hfopenllm_v2/MATH Level 5": 0.102, - "hfopenllm_v2/GPQA": 0.2626, - "hfopenllm_v2/MUSR": 0.3675, - "hfopenllm_v2/MMLU-PRO": 0.3594 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/mlx-community.json b/data/developers/mlx-community.json deleted file mode 100644 index cd8003f96a2e0f9e711b90ad5a115cb92e0bbb0d..0000000000000000000000000000000000000000 --- a/data/developers/mlx-community.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "mlx-community", - "models": [ - { - "id": "mlx-community/Josiefied-Qwen2.5-0.5B-Instruct-abliterated-v1-float32", - "name": "Josiefied-Qwen2.5-0.5B-Instruct-abliterated-v1-float32", - "developer": "mlx-community", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3369, - "hfopenllm_v2/BBH": 0.3292, - "hfopenllm_v2/MATH Level 5": 0.0846, - "hfopenllm_v2/GPQA": 0.2576, - "hfopenllm_v2/MUSR": 0.3249, - "hfopenllm_v2/MMLU-PRO": 0.1638 - } - }, - { - "id": "mlx-community/Mistral-Small-24B-Instruct-2501-bf16", - "name": "Mistral-Small-24B-Instruct-2501-bf16", - "developer": "mlx-community", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6283, - "hfopenllm_v2/BBH": 0.6713, - "hfopenllm_v2/MATH Level 5": 0.3225, - "hfopenllm_v2/GPQA": 0.3951, - "hfopenllm_v2/MUSR": 0.4618, - "hfopenllm_v2/MMLU-PRO": 0.5395 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/mmnga.json b/data/developers/mmnga.json deleted file mode 100644 index 094ed0afe1c425dd37754bed393f751628d653d7..0000000000000000000000000000000000000000 --- a/data/developers/mmnga.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "mmnga", - "models": [ - { - "id": "mmnga/Llama-3-70B-japanese-suzume-vector-v0.1", - "name": "Llama-3-70B-japanese-suzume-vector-v0.1", - "developer": "mmnga", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4649, - "hfopenllm_v2/BBH": 0.6542, - "hfopenllm_v2/MATH Level 5": 0.2326, - "hfopenllm_v2/GPQA": 0.2861, - "hfopenllm_v2/MUSR": 0.4141, - "hfopenllm_v2/MMLU-PRO": 0.5224 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/mobiuslabsgmbh.json b/data/developers/mobiuslabsgmbh.json deleted file mode 100644 index e3829cad0ef3f0085f581959646b05ecd692b6b0..0000000000000000000000000000000000000000 --- a/data/developers/mobiuslabsgmbh.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "mobiuslabsgmbh", - "models": [ - { - "id": "mobiuslabsgmbh/DeepSeek-R1-ReDistill-Llama3-8B-v1.1", - "name": "DeepSeek-R1-ReDistill-Llama3-8B-v1.1", - "developer": "mobiuslabsgmbh", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3704, - "hfopenllm_v2/BBH": 0.3473, - "hfopenllm_v2/MATH Level 5": 0.3285, - "hfopenllm_v2/GPQA": 0.271, - "hfopenllm_v2/MUSR": 0.3396, - "hfopenllm_v2/MMLU-PRO": 0.2198 - } - }, - { - "id": "mobiuslabsgmbh/DeepSeek-R1-ReDistill-Qwen-7B-v1.1", - "name": "DeepSeek-R1-ReDistill-Qwen-7B-v1.1", - "developer": "mobiuslabsgmbh", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3473, - "hfopenllm_v2/BBH": 0.3698, - "hfopenllm_v2/MATH Level 5": 0.3497, - "hfopenllm_v2/GPQA": 0.2651, - "hfopenllm_v2/MUSR": 0.4009, - "hfopenllm_v2/MMLU-PRO": 0.2326 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/modelcloud.json b/data/developers/modelcloud.json deleted file mode 100644 index 5f7a1ed1accb4f1de07451c248bdcb2c6e36288c..0000000000000000000000000000000000000000 --- a/data/developers/modelcloud.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "ModelCloud", - "models": [ - { - "id": "ModelCloud/Llama-3.2-1B-Instruct-gptqmodel-4bit-vortex-v1", - "name": "Llama-3.2-1B-Instruct-gptqmodel-4bit-vortex-v1", - "developer": "ModelCloud", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5269, - "hfopenllm_v2/BBH": 0.3253, - "hfopenllm_v2/MATH Level 5": 0.0604, - "hfopenllm_v2/GPQA": 0.2534, - "hfopenllm_v2/MUSR": 0.3249, - "hfopenllm_v2/MMLU-PRO": 0.1764 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/modelspace.json b/data/developers/modelspace.json deleted file mode 100644 index e2d1333815369d20f17555f9d4482294426e5fec..0000000000000000000000000000000000000000 --- a/data/developers/modelspace.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "ModelSpace", - "models": [ - { - "id": "ModelSpace/GemmaX2-28-9B-v0.1", - "name": "GemmaX2-28-9B-v0.1", - "developer": "ModelSpace", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0039, - "hfopenllm_v2/BBH": 0.3687, - "hfopenllm_v2/MATH Level 5": 0.0272, - "hfopenllm_v2/GPQA": 0.2768, - "hfopenllm_v2/MUSR": 0.3537, - "hfopenllm_v2/MMLU-PRO": 0.2231 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/moeru-ai.json b/data/developers/moeru-ai.json deleted file mode 100644 index 66cb52131be19f67ec9f4a03862bca0081d107c9..0000000000000000000000000000000000000000 --- a/data/developers/moeru-ai.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "developer": "moeru-ai", - "models": [ - { - "id": "moeru-ai/L3.1-Moe-2x8B-v0.2", - "name": "L3.1-Moe-2x8B-v0.2", - "developer": "moeru-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7348, - "hfopenllm_v2/BBH": 0.5256, - "hfopenllm_v2/MATH Level 5": 0.1699, - "hfopenllm_v2/GPQA": 0.3003, - "hfopenllm_v2/MUSR": 0.4199, - "hfopenllm_v2/MMLU-PRO": 0.3858 - } - }, - { - "id": "moeru-ai/L3.1-Moe-4x8B-v0.1", - "name": "L3.1-Moe-4x8B-v0.1", - "developer": "moeru-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4332, - "hfopenllm_v2/BBH": 0.4939, - "hfopenllm_v2/MATH Level 5": 0.1299, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3609, - "hfopenllm_v2/MMLU-PRO": 0.3454 - } - }, - { - "id": "moeru-ai/L3.1-Moe-4x8B-v0.2", - "name": "L3.1-Moe-4x8B-v0.2", - "developer": "moeru-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5407, - "hfopenllm_v2/BBH": 0.4466, - "hfopenllm_v2/MATH Level 5": 0.1035, - "hfopenllm_v2/GPQA": 0.2668, - "hfopenllm_v2/MUSR": 0.3234, - "hfopenllm_v2/MMLU-PRO": 0.2763 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/monsterapi.json b/data/developers/monsterapi.json deleted file mode 100644 index 28dd973594514c438ff479510a46de6b7e475972..0000000000000000000000000000000000000000 --- a/data/developers/monsterapi.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "monsterapi", - "models": [ - { - "id": "monsterapi/gemma-2-2b-LoRA-MonsterInstruct", - "name": "gemma-2-2b-LoRA-MonsterInstruct", - "developer": "monsterapi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3903, - "hfopenllm_v2/BBH": 0.365, - "hfopenllm_v2/MATH Level 5": 0.0506, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.3644, - "hfopenllm_v2/MMLU-PRO": 0.1987 - } - }, - { - "id": "monsterapi/Llama-3_1-8B-Instruct-orca-ORPO", - "name": "Llama-3_1-8B-Instruct-orca-ORPO", - "developer": "monsterapi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2273, - "hfopenllm_v2/BBH": 0.2865, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2492, - "hfopenllm_v2/MUSR": 0.3445, - "hfopenllm_v2/MMLU-PRO": 0.1168 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/moonride.json b/data/developers/moonride.json deleted file mode 100644 index 4fbd1f8f52394253d5382b982b906769dfc07443..0000000000000000000000000000000000000000 --- a/data/developers/moonride.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "MoonRide", - "models": [ - { - "id": "MoonRide/Llama-3.2-3B-Khelavaster", - "name": "Llama-3.2-3B-Khelavaster", - "developer": "MoonRide", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4925, - "hfopenllm_v2/BBH": 0.4516, - "hfopenllm_v2/MATH Level 5": 0.1616, - "hfopenllm_v2/GPQA": 0.2777, - "hfopenllm_v2/MUSR": 0.3699, - "hfopenllm_v2/MMLU-PRO": 0.3122 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/moonshot.json b/data/developers/moonshot.json deleted file mode 100644 index 69a15692e44ee14f98c4ae22bc5403fe4fd7daa6..0000000000000000000000000000000000000000 --- a/data/developers/moonshot.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "developer": "moonshot", - "models": [ - { - "id": "moonshot/Kimi K2 Thinking", - "name": "Kimi K2 Thinking", - "developer": "moonshot", - "evaluator_relationship": null, - "benchmark_scores": { - "apex-agents/Overall Pass@1": 0.04, - "apex-agents/Overall Pass@8": 0.144, - "apex-agents/Overall Mean Score": 0.115, - "apex-agents/Investment Banking Pass@1": 0.012, - "apex-agents/Management Consulting Pass@1": 0.029, - "apex-agents/Corporate Law Pass@1": 0.08, - "apex-agents/Corporate Lawyer Mean Score": 0.223 - } - }, - { - "id": "moonshot/Kimi K2.5", - "name": "Kimi K2.5", - "developer": "moonshot", - "evaluator_relationship": null, - "benchmark_scores": { - "apex-agents/Corporate Lawyer Mean Score": 0.402 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/moonshot_ai.json b/data/developers/moonshot_ai.json deleted file mode 100644 index 746185ce773a539bd025922ab856fdcf2f8a1d9f..0000000000000000000000000000000000000000 --- a/data/developers/moonshot_ai.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "developer": "Moonshot AI", - "models": [ - { - "id": "moonshot-ai/kimi-k2-instruct", - "name": "Kimi K2 Instruct", - "developer": "Moonshot AI", - "evaluator_relationship": null, - "benchmark_scores": { - "terminal-bench-2.0/terminal-bench-2.0": 27.8 - } - }, - { - "id": "moonshot-ai/kimi-k2-thinking", - "name": "Kimi K2 Thinking", - "developer": "Moonshot AI", - "evaluator_relationship": null, - "benchmark_scores": { - "terminal-bench-2.0/terminal-bench-2.0": 35.7 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/moonshotai.json b/data/developers/moonshotai.json deleted file mode 100644 index 5da9f6d3663ac38ffcb9fdcca497bcb16ba50ad6..0000000000000000000000000000000000000000 --- a/data/developers/moonshotai.json +++ /dev/null @@ -1,57 +0,0 @@ -{ - "developer": "moonshotai", - "models": [ - { - "id": "moonshotai/kimi-k2-instruct", - "name": "Kimi K2 Instruct", - "developer": "moonshotai", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.768, - "helm_capabilities/MMLU-Pro": 0.819, - "helm_capabilities/GPQA": 0.652, - "helm_capabilities/IFEval": 0.85, - "helm_capabilities/WildBench": 0.862, - "helm_capabilities/Omni-MATH": 0.654 - } - }, - { - "id": "moonshotai/moonshotai-kimi-k2-instruct-fc", - "name": "Moonshotai-Kimi-K2-Instruct (FC)", - "developer": "moonshotai", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 11.0, - "bfcl/bfcl.overall.overall_accuracy": 59.06, - "bfcl/bfcl.overall.total_cost_usd": 6.19, - "bfcl/bfcl.overall.latency_mean_s": 6.4, - "bfcl/bfcl.overall.latency_std_s": 9.38, - "bfcl/bfcl.overall.latency_p95_s": 13.78, - "bfcl/bfcl.non_live.ast_accuracy": 81.6, - "bfcl/bfcl.non_live.simple_ast_accuracy": 69.42, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 92.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 82.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 83.0, - "bfcl/bfcl.live.live_accuracy": 78.68, - "bfcl/bfcl.live.live_simple_ast_accuracy": 81.78, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 78.06, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 87.5, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 66.67, - "bfcl/bfcl.multi_turn.accuracy": 50.63, - "bfcl/bfcl.multi_turn.base_accuracy": 62.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 41.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 44.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 55.0, - "bfcl/bfcl.web_search.accuracy": 66.5, - "bfcl/bfcl.web_search.base_accuracy": 72.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 61.0, - "bfcl/bfcl.memory.accuracy": 29.03, - "bfcl/bfcl.memory.kv_accuracy": 21.94, - "bfcl/bfcl.memory.vector_accuracy": 20.0, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 45.16, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 75.0, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 87.34 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/mosaicml.json b/data/developers/mosaicml.json deleted file mode 100644 index 86170994191d12a0b90d2d4128cac71d701cfc01..0000000000000000000000000000000000000000 --- a/data/developers/mosaicml.json +++ /dev/null @@ -1,65 +0,0 @@ -{ - "developer": "mosaicml", - "models": [ - { - "id": "mosaicml/MPT-30B", - "name": "MPT 30B", - "developer": "mosaicml", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.714, - "helm_classic/MMLU": 0.437, - "helm_classic/BoolQ": 0.704, - "helm_classic/NarrativeQA": 0.732, - "helm_classic/NaturalQuestions (open-book)": 0.673, - "helm_classic/QuAC": 0.393, - "helm_classic/HellaSwag": -1.0, - "helm_classic/OpenbookQA": -1.0, - "helm_classic/TruthfulQA": 0.231, - "helm_classic/MS MARCO (TREC)": -1.0, - "helm_classic/CNN/DailyMail": -1.0, - "helm_classic/XSUM": -1.0, - "helm_classic/IMDB": 0.959, - "helm_classic/CivilComments": 0.599, - "helm_classic/RAFT": 0.723 - } - }, - { - "id": "mosaicml/mpt-7b", - "name": "mpt-7b", - "developer": "mosaicml", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2152, - "hfopenllm_v2/BBH": 0.33, - "hfopenllm_v2/MATH Level 5": 0.0159, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3672, - "hfopenllm_v2/MMLU-PRO": 0.1206 - } - }, - { - "id": "mosaicml/MPT-Instruct-30B", - "name": "MPT-Instruct 30B", - "developer": "mosaicml", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.716, - "helm_classic/MMLU": 0.444, - "helm_classic/BoolQ": 0.85, - "helm_classic/NarrativeQA": 0.733, - "helm_classic/NaturalQuestions (open-book)": 0.697, - "helm_classic/QuAC": 0.327, - "helm_classic/HellaSwag": -1.0, - "helm_classic/OpenbookQA": -1.0, - "helm_classic/TruthfulQA": 0.234, - "helm_classic/MS MARCO (TREC)": -1.0, - "helm_classic/CNN/DailyMail": -1.0, - "helm_classic/XSUM": -1.0, - "helm_classic/IMDB": 0.956, - "helm_classic/CivilComments": 0.573, - "helm_classic/RAFT": 0.68 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/mosama.json b/data/developers/mosama.json deleted file mode 100644 index 1256a4044b4b150c3257e24c8598003fb436e5bc..0000000000000000000000000000000000000000 --- a/data/developers/mosama.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "mosama", - "models": [ - { - "id": "mosama/Qwen2.5-1.5B-Instruct-CoT-Reflection", - "name": "Qwen2.5-1.5B-Instruct-CoT-Reflection", - "developer": "mosama", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.287, - "hfopenllm_v2/BBH": 0.4109, - "hfopenllm_v2/MATH Level 5": 0.0272, - "hfopenllm_v2/GPQA": 0.2617, - "hfopenllm_v2/MUSR": 0.3212, - "hfopenllm_v2/MMLU-PRO": 0.2651 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/mostafa8mehrabi.json b/data/developers/mostafa8mehrabi.json deleted file mode 100644 index 26762a1f3e8dd9a339a35475797600f1bbbf801d..0000000000000000000000000000000000000000 --- a/data/developers/mostafa8mehrabi.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "Mostafa8Mehrabi", - "models": [ - { - "id": "Mostafa8Mehrabi/llama-3.2-1b-Insomnia-ChatBot-merged", - "name": "llama-3.2-1b-Insomnia-ChatBot-merged", - "developer": "Mostafa8Mehrabi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1321, - "hfopenllm_v2/BBH": 0.3004, - "hfopenllm_v2/MATH Level 5": 0.0076, - "hfopenllm_v2/GPQA": 0.2366, - "hfopenllm_v2/MUSR": 0.3382, - "hfopenllm_v2/MMLU-PRO": 0.1131 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/mrdayl.json b/data/developers/mrdayl.json deleted file mode 100644 index 40195f7da75002a2f4c8fe34a6b972163d303c5a..0000000000000000000000000000000000000000 --- a/data/developers/mrdayl.json +++ /dev/null @@ -1,75 +0,0 @@ -{ - "developer": "mrdayl", - "models": [ - { - "id": "mrdayl/OpenCogito", - "name": "OpenCogito", - "developer": "mrdayl", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3934, - "hfopenllm_v2/BBH": 0.472, - "hfopenllm_v2/MATH Level 5": 0.2183, - "hfopenllm_v2/GPQA": 0.3003, - "hfopenllm_v2/MUSR": 0.424, - "hfopenllm_v2/MMLU-PRO": 0.3452 - } - }, - { - "id": "mrdayl/OpenCognito", - "name": "OpenCognito", - "developer": "mrdayl", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4062, - "hfopenllm_v2/BBH": 0.4706, - "hfopenllm_v2/MATH Level 5": 0.2115, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.4293, - "hfopenllm_v2/MMLU-PRO": 0.3443 - } - }, - { - "id": "mrdayl/OpenCognito-r1", - "name": "OpenCognito-r1", - "developer": "mrdayl", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4241, - "hfopenllm_v2/BBH": 0.4673, - "hfopenllm_v2/MATH Level 5": 0.1903, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.4241, - "hfopenllm_v2/MMLU-PRO": 0.3475 - } - }, - { - "id": "mrdayl/OpenCognito-r2", - "name": "OpenCognito-r2", - "developer": "mrdayl", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3959, - "hfopenllm_v2/BBH": 0.4688, - "hfopenllm_v2/MATH Level 5": 0.2024, - "hfopenllm_v2/GPQA": 0.3062, - "hfopenllm_v2/MUSR": 0.4202, - "hfopenllm_v2/MMLU-PRO": 0.3462 - } - }, - { - "id": "mrdayl/OpenThink", - "name": "OpenThink", - "developer": "mrdayl", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2054, - "hfopenllm_v2/BBH": 0.346, - "hfopenllm_v2/MATH Level 5": 0.2885, - "hfopenllm_v2/GPQA": 0.2827, - "hfopenllm_v2/MUSR": 0.3289, - "hfopenllm_v2/MMLU-PRO": 0.185 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/mrm8488.json b/data/developers/mrm8488.json deleted file mode 100644 index c81e9cb9e2a78f011fe0d3a763e255886a8a8105..0000000000000000000000000000000000000000 --- a/data/developers/mrm8488.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "mrm8488", - "models": [ - { - "id": "mrm8488/phi-4-14B-grpo-gsm8k-3e", - "name": "phi-4-14B-grpo-gsm8k-3e", - "developer": "mrm8488", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6885, - "hfopenllm_v2/BBH": 0.6805, - "hfopenllm_v2/MATH Level 5": 0.4524, - "hfopenllm_v2/GPQA": 0.3356, - "hfopenllm_v2/MUSR": 0.3994, - "hfopenllm_v2/MMLU-PRO": 0.5268 - } - }, - { - "id": "mrm8488/phi-4-14B-grpo-limo", - "name": "phi-4-14B-grpo-limo", - "developer": "mrm8488", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6812, - "hfopenllm_v2/BBH": 0.6785, - "hfopenllm_v2/MATH Level 5": 0.4569, - "hfopenllm_v2/GPQA": 0.3364, - "hfopenllm_v2/MUSR": 0.3981, - "hfopenllm_v2/MMLU-PRO": 0.5261 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/mrrobotoai.json b/data/developers/mrrobotoai.json deleted file mode 100644 index b9fdc0a7cabd982ae8ff8de14a4fc625e30515d7..0000000000000000000000000000000000000000 --- a/data/developers/mrrobotoai.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "MrRobotoAI", - "models": [ - { - "id": "MrRobotoAI/MrRoboto-ProLong-8b-v4i", - "name": "MrRoboto-ProLong-8b-v4i", - "developer": "MrRobotoAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3835, - "hfopenllm_v2/BBH": 0.4585, - "hfopenllm_v2/MATH Level 5": 0.0551, - "hfopenllm_v2/GPQA": 0.2894, - "hfopenllm_v2/MUSR": 0.4014, - "hfopenllm_v2/MMLU-PRO": 0.3068 - } - }, - { - "id": "MrRobotoAI/MrRoboto-ProLongBASE-pt8-unaligned-8b", - "name": "MrRoboto-ProLongBASE-pt8-unaligned-8b", - "developer": "MrRobotoAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3475, - "hfopenllm_v2/BBH": 0.4515, - "hfopenllm_v2/MATH Level 5": 0.0423, - "hfopenllm_v2/GPQA": 0.281, - "hfopenllm_v2/MUSR": 0.4279, - "hfopenllm_v2/MMLU-PRO": 0.2566 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/mtsair.json b/data/developers/mtsair.json deleted file mode 100644 index b31cfa942700007f43ba6fb3959ed442419ea4f1..0000000000000000000000000000000000000000 --- a/data/developers/mtsair.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "MTSAIR", - "models": [ - { - "id": "MTSAIR/Cotype-Nano", - "name": "Cotype-Nano", - "developer": "MTSAIR", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3748, - "hfopenllm_v2/BBH": 0.3865, - "hfopenllm_v2/MATH Level 5": 0.0974, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.3289, - "hfopenllm_v2/MMLU-PRO": 0.2477 - } - }, - { - "id": "MTSAIR/MultiVerse_70B", - "name": "MultiVerse_70B", - "developer": "MTSAIR", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5249, - "hfopenllm_v2/BBH": 0.6183, - "hfopenllm_v2/MATH Level 5": 0.1926, - "hfopenllm_v2/GPQA": 0.354, - "hfopenllm_v2/MUSR": 0.474, - "hfopenllm_v2/MMLU-PRO": 0.486 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/mukaj.json b/data/developers/mukaj.json deleted file mode 100644 index 386d271068ea1cac3eb6da1ae9eed760f0975f1f..0000000000000000000000000000000000000000 --- a/data/developers/mukaj.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "mukaj", - "models": [ - { - "id": "mukaj/Llama-3.1-Hawkish-8B", - "name": "Llama-3.1-Hawkish-8B", - "developer": "mukaj", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.672, - "hfopenllm_v2/BBH": 0.4884, - "hfopenllm_v2/MATH Level 5": 0.2432, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.3967, - "hfopenllm_v2/MMLU-PRO": 0.3331 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/multiple.json b/data/developers/multiple.json deleted file mode 100644 index 156ed1c99339b2b26434b892895ae159b12dbfec..0000000000000000000000000000000000000000 --- a/data/developers/multiple.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "developer": "Multiple", - "models": [ - { - "id": "multiple/multiple", - "name": "Multiple", - "developer": "Multiple", - "evaluator_relationship": null, - "benchmark_scores": { - "terminal-bench-2.0/terminal-bench-2.0": 59.1 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/multivexai.json b/data/developers/multivexai.json deleted file mode 100644 index 15d8d06c573f154d06df47c58778fb9bd81fb4fe..0000000000000000000000000000000000000000 --- a/data/developers/multivexai.json +++ /dev/null @@ -1,75 +0,0 @@ -{ - "developer": "MultivexAI", - "models": [ - { - "id": "MultivexAI/Gladiator-Mini-Exp-1211-3B", - "name": "Gladiator-Mini-Exp-1211-3B", - "developer": "MultivexAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6876, - "hfopenllm_v2/BBH": 0.4484, - "hfopenllm_v2/MATH Level 5": 0.1375, - "hfopenllm_v2/GPQA": 0.2727, - "hfopenllm_v2/MUSR": 0.326, - "hfopenllm_v2/MMLU-PRO": 0.3152 - } - }, - { - "id": "MultivexAI/Gladiator-Mini-Exp-1221-3B-Instruct", - "name": "Gladiator-Mini-Exp-1221-3B-Instruct", - "developer": "MultivexAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6079, - "hfopenllm_v2/BBH": 0.437, - "hfopenllm_v2/MATH Level 5": 0.1352, - "hfopenllm_v2/GPQA": 0.2634, - "hfopenllm_v2/MUSR": 0.3115, - "hfopenllm_v2/MMLU-PRO": 0.3049 - } - }, - { - "id": "MultivexAI/Gladiator-Mini-Exp-1221-3B-Instruct-V2", - "name": "Gladiator-Mini-Exp-1221-3B-Instruct-V2", - "developer": "MultivexAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6215, - "hfopenllm_v2/BBH": 0.4389, - "hfopenllm_v2/MATH Level 5": 0.1412, - "hfopenllm_v2/GPQA": 0.2634, - "hfopenllm_v2/MUSR": 0.3008, - "hfopenllm_v2/MMLU-PRO": 0.3025 - } - }, - { - "id": "MultivexAI/Gladiator-Mini-Exp-1222-3B-Instruct", - "name": "Gladiator-Mini-Exp-1222-3B-Instruct", - "developer": "MultivexAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6163, - "hfopenllm_v2/BBH": 0.4373, - "hfopenllm_v2/MATH Level 5": 0.1412, - "hfopenllm_v2/GPQA": 0.2634, - "hfopenllm_v2/MUSR": 0.3128, - "hfopenllm_v2/MMLU-PRO": 0.3017 - } - }, - { - "id": "MultivexAI/Phi-3.5-Mini-Instruct-MultiVex-v0.25-GGUF", - "name": "Phi-3.5-Mini-Instruct-MultiVex-v0.25-GGUF", - "developer": "MultivexAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.144, - "hfopenllm_v2/BBH": 0.2908, - "hfopenllm_v2/MATH Level 5": 0.006, - "hfopenllm_v2/GPQA": 0.255, - "hfopenllm_v2/MUSR": 0.3642, - "hfopenllm_v2/MMLU-PRO": 0.1109 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/mxode.json b/data/developers/mxode.json deleted file mode 100644 index 779e6ca24b641c50f0ea6e66402d308745ee5144..0000000000000000000000000000000000000000 --- a/data/developers/mxode.json +++ /dev/null @@ -1,75 +0,0 @@ -{ - "developer": "Mxode", - "models": [ - { - "id": "Mxode/NanoLM-0.3B-Instruct-v1", - "name": "NanoLM-0.3B-Instruct-v1", - "developer": "Mxode", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1537, - "hfopenllm_v2/BBH": 0.3028, - "hfopenllm_v2/MATH Level 5": 0.0144, - "hfopenllm_v2/GPQA": 0.2718, - "hfopenllm_v2/MUSR": 0.4155, - "hfopenllm_v2/MMLU-PRO": 0.1105 - } - }, - { - "id": "Mxode/NanoLM-0.3B-Instruct-v1.1", - "name": "NanoLM-0.3B-Instruct-v1.1", - "developer": "Mxode", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1783, - "hfopenllm_v2/BBH": 0.3014, - "hfopenllm_v2/MATH Level 5": 0.0136, - "hfopenllm_v2/GPQA": 0.25, - "hfopenllm_v2/MUSR": 0.4273, - "hfopenllm_v2/MMLU-PRO": 0.1121 - } - }, - { - "id": "Mxode/NanoLM-0.3B-Instruct-v2", - "name": "NanoLM-0.3B-Instruct-v2", - "developer": "Mxode", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1668, - "hfopenllm_v2/BBH": 0.2921, - "hfopenllm_v2/MATH Level 5": 0.0068, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.3955, - "hfopenllm_v2/MMLU-PRO": 0.1134 - } - }, - { - "id": "Mxode/NanoLM-1B-Instruct-v1.1", - "name": "NanoLM-1B-Instruct-v1.1", - "developer": "Mxode", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2395, - "hfopenllm_v2/BBH": 0.3184, - "hfopenllm_v2/MATH Level 5": 0.0363, - "hfopenllm_v2/GPQA": 0.2634, - "hfopenllm_v2/MUSR": 0.3433, - "hfopenllm_v2/MMLU-PRO": 0.1215 - } - }, - { - "id": "Mxode/NanoLM-1B-Instruct-v2", - "name": "NanoLM-1B-Instruct-v2", - "developer": "Mxode", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.263, - "hfopenllm_v2/BBH": 0.3123, - "hfopenllm_v2/MATH Level 5": 0.0415, - "hfopenllm_v2/GPQA": 0.2634, - "hfopenllm_v2/MUSR": 0.3552, - "hfopenllm_v2/MMLU-PRO": 0.1238 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/my_model.json b/data/developers/my_model.json deleted file mode 100644 index 5fd12b02258e615d31e894de4bd584d76311e1e7..0000000000000000000000000000000000000000 --- a/data/developers/my_model.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "developer": "my_model", - "models": [ - { - "id": "my_model/", - "name": "my_model/", - "developer": "my_model", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5267, - "reward-bench/Chat": 0.4553, - "reward-bench/Chat Hard": 0.5592, - "reward-bench/Safety": 0.4392, - "reward-bench/Reasoning": 0.6532 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/nanbeige.json b/data/developers/nanbeige.json deleted file mode 100644 index e64a37a7f887276abf3ae866b0ef75f3b18952f7..0000000000000000000000000000000000000000 --- a/data/developers/nanbeige.json +++ /dev/null @@ -1,81 +0,0 @@ -{ - "developer": "nanbeige", - "models": [ - { - "id": "nanbeige/nanbeige3-5-pro-thinking-fc", - "name": "Nanbeige3.5-Pro-Thinking (FC)", - "developer": "nanbeige", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 32.0, - "bfcl/bfcl.overall.overall_accuracy": 47.68, - "bfcl/bfcl.overall.total_cost_usd": 23.46, - "bfcl/bfcl.overall.latency_mean_s": 21.12, - "bfcl/bfcl.overall.latency_std_s": 28.61, - "bfcl/bfcl.overall.latency_p95_s": 63.29, - "bfcl/bfcl.non_live.ast_accuracy": 38.35, - "bfcl/bfcl.non_live.simple_ast_accuracy": 43.92, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 36.5, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 53.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 20.0, - "bfcl/bfcl.live.live_accuracy": 69.95, - "bfcl/bfcl.live.live_simple_ast_accuracy": 63.18, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 71.42, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 87.5, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 66.67, - "bfcl/bfcl.multi_turn.accuracy": 40.0, - "bfcl/bfcl.multi_turn.base_accuracy": 56.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 34.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 29.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 41.0, - "bfcl/bfcl.web_search.accuracy": 42.0, - "bfcl/bfcl.web_search.base_accuracy": 47.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 37.0, - "bfcl/bfcl.memory.accuracy": 45.16, - "bfcl/bfcl.memory.kv_accuracy": 38.06, - "bfcl/bfcl.memory.vector_accuracy": 58.06, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 39.35, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 100.0, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 74.2 - } - }, - { - "id": "nanbeige/nanbeige4-3b-thinking-2511-fc", - "name": "Nanbeige4-3B-Thinking-2511 (FC)", - "developer": "nanbeige", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 25.0, - "bfcl/bfcl.overall.overall_accuracy": 51.4, - "bfcl/bfcl.overall.total_cost_usd": 14.14, - "bfcl/bfcl.overall.latency_mean_s": 13.46, - "bfcl/bfcl.overall.latency_std_s": 26.41, - "bfcl/bfcl.overall.latency_p95_s": 37.45, - "bfcl/bfcl.non_live.ast_accuracy": 81.58, - "bfcl/bfcl.non_live.simple_ast_accuracy": 63.83, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 93.5, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 84.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 84.5, - "bfcl/bfcl.live.live_accuracy": 79.42, - "bfcl/bfcl.live.live_simple_ast_accuracy": 86.05, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 78.06, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 75.0, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 70.83, - "bfcl/bfcl.multi_turn.accuracy": 51.12, - "bfcl/bfcl.multi_turn.base_accuracy": 58.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 54.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 45.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 47.0, - "bfcl/bfcl.web_search.accuracy": 21.5, - "bfcl/bfcl.web_search.base_accuracy": 31.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 12.0, - "bfcl/bfcl.memory.accuracy": 36.77, - "bfcl/bfcl.memory.kv_accuracy": 31.61, - "bfcl/bfcl.memory.vector_accuracy": 34.19, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 44.52, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 75.0, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 83.09 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/naps-ai.json b/data/developers/naps-ai.json deleted file mode 100644 index 60aa8bbe4d0698b9b04228c9b0a0cc45135553e4..0000000000000000000000000000000000000000 --- a/data/developers/naps-ai.json +++ /dev/null @@ -1,103 +0,0 @@ -{ - "developer": "NAPS-ai", - "models": [ - { - "id": "NAPS-ai/naps-gemma-2-27b-v-0.1.0", - "name": "naps-gemma-2-27b-v-0.1.0", - "developer": "NAPS-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0, - "hfopenllm_v2/BBH": 0.2912, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3575, - "hfopenllm_v2/MMLU-PRO": 0.1168 - } - }, - { - "id": "NAPS-ai/naps-gemma-2-27b-v0.1.0", - "name": "naps-gemma-2-27b-v0.1.0", - "developer": "NAPS-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0, - "hfopenllm_v2/BBH": 0.2912, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3575, - "hfopenllm_v2/MMLU-PRO": 0.1168 - } - }, - { - "id": "NAPS-ai/naps-llama-3_1-8b-instruct-v0.3", - "name": "naps-llama-3_1-8b-instruct-v0.3", - "developer": "NAPS-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5391, - "hfopenllm_v2/BBH": 0.4901, - "hfopenllm_v2/MATH Level 5": 0.1903, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.3787, - "hfopenllm_v2/MMLU-PRO": 0.3398 - } - }, - { - "id": "NAPS-ai/naps-llama-3_1-8b-instruct-v0.4", - "name": "naps-llama-3_1-8b-instruct-v0.4", - "developer": "NAPS-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7344, - "hfopenllm_v2/BBH": 0.4862, - "hfopenllm_v2/MATH Level 5": 0.1964, - "hfopenllm_v2/GPQA": 0.2794, - "hfopenllm_v2/MUSR": 0.4421, - "hfopenllm_v2/MMLU-PRO": 0.3475 - } - }, - { - "id": "NAPS-ai/naps-llama-3_1-instruct-v0.5.0", - "name": "naps-llama-3_1-instruct-v0.5.0", - "developer": "NAPS-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.502, - "hfopenllm_v2/BBH": 0.4148, - "hfopenllm_v2/MATH Level 5": 0.0363, - "hfopenllm_v2/GPQA": 0.2685, - "hfopenllm_v2/MUSR": 0.3713, - "hfopenllm_v2/MMLU-PRO": 0.2614 - } - }, - { - "id": "NAPS-ai/naps-llama-3_1_instruct-v0.6.0", - "name": "naps-llama-3_1_instruct-v0.6.0", - "developer": "NAPS-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.328, - "hfopenllm_v2/BBH": 0.4528, - "hfopenllm_v2/MATH Level 5": 0.0642, - "hfopenllm_v2/GPQA": 0.2819, - "hfopenllm_v2/MUSR": 0.3739, - "hfopenllm_v2/MMLU-PRO": 0.3241 - } - }, - { - "id": "NAPS-ai/naps-llama3.1-70B-v0.2-fp16", - "name": "naps-llama3.1-70B-v0.2-fp16", - "developer": "NAPS-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1845, - "hfopenllm_v2/BBH": 0.3041, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2391, - "hfopenllm_v2/MUSR": 0.3486, - "hfopenllm_v2/MMLU-PRO": 0.1099 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/natong19.json b/data/developers/natong19.json deleted file mode 100644 index ea57a03181f4077ece6207733989872ede378a46..0000000000000000000000000000000000000000 --- a/data/developers/natong19.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "natong19", - "models": [ - { - "id": "natong19/Mistral-Nemo-Instruct-2407-abliterated", - "name": "Mistral-Nemo-Instruct-2407-abliterated", - "developer": "natong19", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6392, - "hfopenllm_v2/BBH": 0.5048, - "hfopenllm_v2/MATH Level 5": 0.1322, - "hfopenllm_v2/GPQA": 0.2869, - "hfopenllm_v2/MUSR": 0.4033, - "hfopenllm_v2/MMLU-PRO": 0.3518 - } - }, - { - "id": "natong19/Qwen2-7B-Instruct-abliterated", - "name": "Qwen2-7B-Instruct-abliterated", - "developer": "natong19", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5837, - "hfopenllm_v2/BBH": 0.5553, - "hfopenllm_v2/MATH Level 5": 0.2764, - "hfopenllm_v2/GPQA": 0.3012, - "hfopenllm_v2/MUSR": 0.4034, - "hfopenllm_v2/MMLU-PRO": 0.3842 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/naveenpoliasetty.json b/data/developers/naveenpoliasetty.json deleted file mode 100644 index 262b795646cd38997b32729c79e40e74b7a63971..0000000000000000000000000000000000000000 --- a/data/developers/naveenpoliasetty.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "Naveenpoliasetty", - "models": [ - { - "id": "Naveenpoliasetty/llama3-8B-V2", - "name": "llama3-8B-V2", - "developer": "Naveenpoliasetty", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4123, - "hfopenllm_v2/BBH": 0.5189, - "hfopenllm_v2/MATH Level 5": 0.0785, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.4081, - "hfopenllm_v2/MMLU-PRO": 0.3738 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/nazimali.json b/data/developers/nazimali.json deleted file mode 100644 index 34d47c9647d462b17a0fe6015c5c4f9fc00264e7..0000000000000000000000000000000000000000 --- a/data/developers/nazimali.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "nazimali", - "models": [ - { - "id": "nazimali/Mistral-Nemo-Kurdish", - "name": "Mistral-Nemo-Kurdish", - "developer": "nazimali", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3401, - "hfopenllm_v2/BBH": 0.5133, - "hfopenllm_v2/MATH Level 5": 0.0959, - "hfopenllm_v2/GPQA": 0.3012, - "hfopenllm_v2/MUSR": 0.4116, - "hfopenllm_v2/MMLU-PRO": 0.3235 - } - }, - { - "id": "nazimali/Mistral-Nemo-Kurdish-Instruct", - "name": "Mistral-Nemo-Kurdish-Instruct", - "developer": "nazimali", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.486, - "hfopenllm_v2/BBH": 0.4721, - "hfopenllm_v2/MATH Level 5": 0.0846, - "hfopenllm_v2/GPQA": 0.2844, - "hfopenllm_v2/MUSR": 0.4006, - "hfopenllm_v2/MMLU-PRO": 0.3087 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/nbailab.json b/data/developers/nbailab.json deleted file mode 100644 index 8989f7d6b04d7c1110a46b27b27e21fb7b728650..0000000000000000000000000000000000000000 --- a/data/developers/nbailab.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "NbAiLab", - "models": [ - { - "id": "NbAiLab/nb-llama-3.1-8B-Instruct", - "name": "nb-llama-3.1-8B-Instruct", - "developer": "NbAiLab", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3625, - "hfopenllm_v2/BBH": 0.3247, - "hfopenllm_v2/MATH Level 5": 0.0227, - "hfopenllm_v2/GPQA": 0.2735, - "hfopenllm_v2/MUSR": 0.3208, - "hfopenllm_v2/MMLU-PRO": 0.1197 - } - }, - { - "id": "NbAiLab/nb-llama-3.1-8B-sft", - "name": "nb-llama-3.1-8B-sft", - "developer": "NbAiLab", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3616, - "hfopenllm_v2/BBH": 0.3282, - "hfopenllm_v2/MATH Level 5": 0.0219, - "hfopenllm_v2/GPQA": 0.2542, - "hfopenllm_v2/MUSR": 0.3287, - "hfopenllm_v2/MMLU-PRO": 0.1222 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/nbeerbower.json b/data/developers/nbeerbower.json deleted file mode 100644 index e14166d6062dd90b9ceb3c06d84b83a82e99c141..0000000000000000000000000000000000000000 --- a/data/developers/nbeerbower.json +++ /dev/null @@ -1,719 +0,0 @@ -{ - "developer": "nbeerbower", - "models": [ - { - "id": "nbeerbower/BigKartoffel-mistral-nemo-20B", - "name": "BigKartoffel-mistral-nemo-20B", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5857, - "hfopenllm_v2/BBH": 0.5515, - "hfopenllm_v2/MATH Level 5": 0.0264, - "hfopenllm_v2/GPQA": 0.2869, - "hfopenllm_v2/MUSR": 0.428, - "hfopenllm_v2/MMLU-PRO": 0.353 - } - }, - { - "id": "nbeerbower/DoppelKartoffel-Mistral-Nemo-23B", - "name": "DoppelKartoffel-Mistral-Nemo-23B", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5191, - "hfopenllm_v2/BBH": 0.5218, - "hfopenllm_v2/MATH Level 5": 0.031, - "hfopenllm_v2/GPQA": 0.2752, - "hfopenllm_v2/MUSR": 0.3795, - "hfopenllm_v2/MMLU-PRO": 0.308 - } - }, - { - "id": "nbeerbower/DoublePotato-Mistral-Nemo-13B", - "name": "DoublePotato-Mistral-Nemo-13B", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6796, - "hfopenllm_v2/BBH": 0.5438, - "hfopenllm_v2/MATH Level 5": 0.04, - "hfopenllm_v2/GPQA": 0.3012, - "hfopenllm_v2/MUSR": 0.46, - "hfopenllm_v2/MMLU-PRO": 0.3596 - } - }, - { - "id": "nbeerbower/Dumpling-Qwen2.5-1.5B", - "name": "Dumpling-Qwen2.5-1.5B", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3699, - "hfopenllm_v2/BBH": 0.416, - "hfopenllm_v2/MATH Level 5": 0.1171, - "hfopenllm_v2/GPQA": 0.2685, - "hfopenllm_v2/MUSR": 0.3728, - "hfopenllm_v2/MMLU-PRO": 0.2772 - } - }, - { - "id": "nbeerbower/Dumpling-Qwen2.5-14B", - "name": "Dumpling-Qwen2.5-14B", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6064, - "hfopenllm_v2/BBH": 0.6451, - "hfopenllm_v2/MATH Level 5": 0.3097, - "hfopenllm_v2/GPQA": 0.3012, - "hfopenllm_v2/MUSR": 0.4354, - "hfopenllm_v2/MMLU-PRO": 0.517 - } - }, - { - "id": "nbeerbower/Dumpling-Qwen2.5-7B-1k-r16", - "name": "Dumpling-Qwen2.5-7B-1k-r16", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.486, - "hfopenllm_v2/BBH": 0.5214, - "hfopenllm_v2/MATH Level 5": 0.2364, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.423, - "hfopenllm_v2/MMLU-PRO": 0.3959 - } - }, - { - "id": "nbeerbower/Dumpling-Qwen2.5-7B-1k-r64-2e-5", - "name": "Dumpling-Qwen2.5-7B-1k-r64-2e-5", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4179, - "hfopenllm_v2/BBH": 0.5301, - "hfopenllm_v2/MATH Level 5": 0.2115, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.4486, - "hfopenllm_v2/MMLU-PRO": 0.4122 - } - }, - { - "id": "nbeerbower/EVA-abliterated-TIES-Qwen2.5-1.5B", - "name": "EVA-abliterated-TIES-Qwen2.5-1.5B", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4115, - "hfopenllm_v2/BBH": 0.3997, - "hfopenllm_v2/MATH Level 5": 0.1375, - "hfopenllm_v2/GPQA": 0.2651, - "hfopenllm_v2/MUSR": 0.3502, - "hfopenllm_v2/MMLU-PRO": 0.2712 - } - }, - { - "id": "nbeerbower/EVA-abliterated-TIES-Qwen2.5-14B", - "name": "EVA-abliterated-TIES-Qwen2.5-14B", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7836, - "hfopenllm_v2/BBH": 0.6372, - "hfopenllm_v2/MATH Level 5": 0.5045, - "hfopenllm_v2/GPQA": 0.3549, - "hfopenllm_v2/MUSR": 0.4407, - "hfopenllm_v2/MMLU-PRO": 0.5211 - } - }, - { - "id": "nbeerbower/Flammades-Mistral-Nemo-12B", - "name": "Flammades-Mistral-Nemo-12B", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3842, - "hfopenllm_v2/BBH": 0.53, - "hfopenllm_v2/MATH Level 5": 0.0755, - "hfopenllm_v2/GPQA": 0.3037, - "hfopenllm_v2/MUSR": 0.4806, - "hfopenllm_v2/MMLU-PRO": 0.3661 - } - }, - { - "id": "nbeerbower/gemma2-gutenberg-27B", - "name": "gemma2-gutenberg-27B", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2947, - "hfopenllm_v2/BBH": 0.3797, - "hfopenllm_v2/MATH Level 5": 0.0189, - "hfopenllm_v2/GPQA": 0.2727, - "hfopenllm_v2/MUSR": 0.3727, - "hfopenllm_v2/MMLU-PRO": 0.1982 - } - }, - { - "id": "nbeerbower/gemma2-gutenberg-9B", - "name": "gemma2-gutenberg-9B", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2796, - "hfopenllm_v2/BBH": 0.5951, - "hfopenllm_v2/MATH Level 5": 0.0808, - "hfopenllm_v2/GPQA": 0.3381, - "hfopenllm_v2/MUSR": 0.4595, - "hfopenllm_v2/MMLU-PRO": 0.4192 - } - }, - { - "id": "nbeerbower/Gemma2-Gutenberg-Doppel-9B", - "name": "Gemma2-Gutenberg-Doppel-9B", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7171, - "hfopenllm_v2/BBH": 0.587, - "hfopenllm_v2/MATH Level 5": 0.1979, - "hfopenllm_v2/GPQA": 0.3297, - "hfopenllm_v2/MUSR": 0.4608, - "hfopenllm_v2/MMLU-PRO": 0.4127 - } - }, - { - "id": "nbeerbower/Gutensuppe-mistral-nemo-12B", - "name": "Gutensuppe-mistral-nemo-12B", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2916, - "hfopenllm_v2/BBH": 0.5487, - "hfopenllm_v2/MATH Level 5": 0.1329, - "hfopenllm_v2/GPQA": 0.3372, - "hfopenllm_v2/MUSR": 0.429, - "hfopenllm_v2/MMLU-PRO": 0.368 - } - }, - { - "id": "nbeerbower/Hermes2-Gutenberg2-Mistral-7B", - "name": "Hermes2-Gutenberg2-Mistral-7B", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3721, - "hfopenllm_v2/BBH": 0.4981, - "hfopenllm_v2/MATH Level 5": 0.0574, - "hfopenllm_v2/GPQA": 0.2894, - "hfopenllm_v2/MUSR": 0.4623, - "hfopenllm_v2/MMLU-PRO": 0.2993 - } - }, - { - "id": "nbeerbower/Kartoffel-Deepfry-12B", - "name": "Kartoffel-Deepfry-12B", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5022, - "hfopenllm_v2/BBH": 0.5365, - "hfopenllm_v2/MATH Level 5": 0.0604, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.4792, - "hfopenllm_v2/MMLU-PRO": 0.3582 - } - }, - { - "id": "nbeerbower/llama-3-gutenberg-8B", - "name": "llama-3-gutenberg-8B", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4372, - "hfopenllm_v2/BBH": 0.4994, - "hfopenllm_v2/MATH Level 5": 0.0785, - "hfopenllm_v2/GPQA": 0.3012, - "hfopenllm_v2/MUSR": 0.4073, - "hfopenllm_v2/MMLU-PRO": 0.3831 - } - }, - { - "id": "nbeerbower/Llama-3.1-Nemotron-lorablated-70B", - "name": "Llama-3.1-Nemotron-lorablated-70B", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7229, - "hfopenllm_v2/BBH": 0.6825, - "hfopenllm_v2/MATH Level 5": 0.3338, - "hfopenllm_v2/GPQA": 0.3909, - "hfopenllm_v2/MUSR": 0.4682, - "hfopenllm_v2/MMLU-PRO": 0.5343 - } - }, - { - "id": "nbeerbower/llama3.1-cc-8B", - "name": "llama3.1-cc-8B", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5068, - "hfopenllm_v2/BBH": 0.4871, - "hfopenllm_v2/MATH Level 5": 0.071, - "hfopenllm_v2/GPQA": 0.2852, - "hfopenllm_v2/MUSR": 0.3885, - "hfopenllm_v2/MMLU-PRO": 0.3347 - } - }, - { - "id": "nbeerbower/Llama3.1-Gutenberg-Doppel-70B", - "name": "Llama3.1-Gutenberg-Doppel-70B", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7092, - "hfopenllm_v2/BBH": 0.6661, - "hfopenllm_v2/MATH Level 5": 0.2122, - "hfopenllm_v2/GPQA": 0.3448, - "hfopenllm_v2/MUSR": 0.4897, - "hfopenllm_v2/MMLU-PRO": 0.4737 - } - }, - { - "id": "nbeerbower/llama3.1-kartoffeldes-70B", - "name": "llama3.1-kartoffeldes-70B", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.823, - "hfopenllm_v2/BBH": 0.6894, - "hfopenllm_v2/MATH Level 5": 0.3218, - "hfopenllm_v2/GPQA": 0.3515, - "hfopenllm_v2/MUSR": 0.4646, - "hfopenllm_v2/MMLU-PRO": 0.4988 - } - }, - { - "id": "nbeerbower/Lyra-Gutenberg-mistral-nemo-12B", - "name": "Lyra-Gutenberg-mistral-nemo-12B", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3495, - "hfopenllm_v2/BBH": 0.5586, - "hfopenllm_v2/MATH Level 5": 0.1012, - "hfopenllm_v2/GPQA": 0.3339, - "hfopenllm_v2/MUSR": 0.4357, - "hfopenllm_v2/MMLU-PRO": 0.3628 - } - }, - { - "id": "nbeerbower/Lyra4-Gutenberg-12B", - "name": "Lyra4-Gutenberg-12B", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2212, - "hfopenllm_v2/BBH": 0.5387, - "hfopenllm_v2/MATH Level 5": 0.1299, - "hfopenllm_v2/GPQA": 0.3188, - "hfopenllm_v2/MUSR": 0.4038, - "hfopenllm_v2/MMLU-PRO": 0.3571 - } - }, - { - "id": "nbeerbower/Lyra4-Gutenberg2-12B", - "name": "Lyra4-Gutenberg2-12B", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2585, - "hfopenllm_v2/BBH": 0.5345, - "hfopenllm_v2/MATH Level 5": 0.1171, - "hfopenllm_v2/GPQA": 0.3129, - "hfopenllm_v2/MUSR": 0.3972, - "hfopenllm_v2/MMLU-PRO": 0.3565 - } - }, - { - "id": "nbeerbower/Mahou-1.5-mistral-nemo-12B-lorablated", - "name": "Mahou-1.5-mistral-nemo-12B-lorablated", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6825, - "hfopenllm_v2/BBH": 0.5496, - "hfopenllm_v2/MATH Level 5": 0.0891, - "hfopenllm_v2/GPQA": 0.2794, - "hfopenllm_v2/MUSR": 0.4522, - "hfopenllm_v2/MMLU-PRO": 0.3574 - } - }, - { - "id": "nbeerbower/Mistral-Gutenberg-Doppel-7B-FFT", - "name": "Mistral-Gutenberg-Doppel-7B-FFT", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5717, - "hfopenllm_v2/BBH": 0.4076, - "hfopenllm_v2/MATH Level 5": 0.0249, - "hfopenllm_v2/GPQA": 0.2836, - "hfopenllm_v2/MUSR": 0.4059, - "hfopenllm_v2/MMLU-PRO": 0.2729 - } - }, - { - "id": "nbeerbower/mistral-nemo-bophades-12B", - "name": "mistral-nemo-bophades-12B", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6794, - "hfopenllm_v2/BBH": 0.4988, - "hfopenllm_v2/MATH Level 5": 0.1231, - "hfopenllm_v2/GPQA": 0.2852, - "hfopenllm_v2/MUSR": 0.4178, - "hfopenllm_v2/MMLU-PRO": 0.3501 - } - }, - { - "id": "nbeerbower/mistral-nemo-bophades3-12B", - "name": "mistral-nemo-bophades3-12B", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6578, - "hfopenllm_v2/BBH": 0.5449, - "hfopenllm_v2/MATH Level 5": 0.0846, - "hfopenllm_v2/GPQA": 0.3121, - "hfopenllm_v2/MUSR": 0.4604, - "hfopenllm_v2/MMLU-PRO": 0.3371 - } - }, - { - "id": "nbeerbower/mistral-nemo-cc-12B", - "name": "mistral-nemo-cc-12B", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1435, - "hfopenllm_v2/BBH": 0.5399, - "hfopenllm_v2/MATH Level 5": 0.0257, - "hfopenllm_v2/GPQA": 0.3154, - "hfopenllm_v2/MUSR": 0.4424, - "hfopenllm_v2/MMLU-PRO": 0.3598 - } - }, - { - "id": "nbeerbower/mistral-nemo-gutades-12B", - "name": "mistral-nemo-gutades-12B", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3425, - "hfopenllm_v2/BBH": 0.5407, - "hfopenllm_v2/MATH Level 5": 0.1178, - "hfopenllm_v2/GPQA": 0.3154, - "hfopenllm_v2/MUSR": 0.404, - "hfopenllm_v2/MMLU-PRO": 0.3561 - } - }, - { - "id": "nbeerbower/mistral-nemo-gutenberg-12B", - "name": "mistral-nemo-gutenberg-12B", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3504, - "hfopenllm_v2/BBH": 0.5281, - "hfopenllm_v2/MATH Level 5": 0.1163, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.4171, - "hfopenllm_v2/MMLU-PRO": 0.3562 - } - }, - { - "id": "nbeerbower/mistral-nemo-gutenberg-12B-v2", - "name": "mistral-nemo-gutenberg-12B-v2", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6203, - "hfopenllm_v2/BBH": 0.5397, - "hfopenllm_v2/MATH Level 5": 0.1088, - "hfopenllm_v2/GPQA": 0.2777, - "hfopenllm_v2/MUSR": 0.4287, - "hfopenllm_v2/MMLU-PRO": 0.3499 - } - }, - { - "id": "nbeerbower/mistral-nemo-gutenberg-12B-v3", - "name": "mistral-nemo-gutenberg-12B-v3", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2183, - "hfopenllm_v2/BBH": 0.5441, - "hfopenllm_v2/MATH Level 5": 0.0597, - "hfopenllm_v2/GPQA": 0.3146, - "hfopenllm_v2/MUSR": 0.445, - "hfopenllm_v2/MMLU-PRO": 0.3644 - } - }, - { - "id": "nbeerbower/mistral-nemo-gutenberg-12B-v4", - "name": "mistral-nemo-gutenberg-12B-v4", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2379, - "hfopenllm_v2/BBH": 0.5269, - "hfopenllm_v2/MATH Level 5": 0.1261, - "hfopenllm_v2/GPQA": 0.3163, - "hfopenllm_v2/MUSR": 0.4104, - "hfopenllm_v2/MMLU-PRO": 0.3575 - } - }, - { - "id": "nbeerbower/Mistral-Nemo-Gutenberg-Doppel-12B", - "name": "Mistral-Nemo-Gutenberg-Doppel-12B", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3567, - "hfopenllm_v2/BBH": 0.5275, - "hfopenllm_v2/MATH Level 5": 0.1216, - "hfopenllm_v2/GPQA": 0.3163, - "hfopenllm_v2/MUSR": 0.4132, - "hfopenllm_v2/MMLU-PRO": 0.3579 - } - }, - { - "id": "nbeerbower/Mistral-Nemo-Gutenberg-Doppel-12B-v2", - "name": "Mistral-Nemo-Gutenberg-Doppel-12B-v2", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6536, - "hfopenllm_v2/BBH": 0.5374, - "hfopenllm_v2/MATH Level 5": 0.1156, - "hfopenllm_v2/GPQA": 0.271, - "hfopenllm_v2/MUSR": 0.4233, - "hfopenllm_v2/MMLU-PRO": 0.3546 - } - }, - { - "id": "nbeerbower/mistral-nemo-gutenberg2-12B-test", - "name": "mistral-nemo-gutenberg2-12B-test", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3385, - "hfopenllm_v2/BBH": 0.5255, - "hfopenllm_v2/MATH Level 5": 0.1163, - "hfopenllm_v2/GPQA": 0.3171, - "hfopenllm_v2/MUSR": 0.4157, - "hfopenllm_v2/MMLU-PRO": 0.3555 - } - }, - { - "id": "nbeerbower/mistral-nemo-kartoffel-12B", - "name": "mistral-nemo-kartoffel-12B", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7032, - "hfopenllm_v2/BBH": 0.5484, - "hfopenllm_v2/MATH Level 5": 0.0853, - "hfopenllm_v2/GPQA": 0.3045, - "hfopenllm_v2/MUSR": 0.4653, - "hfopenllm_v2/MMLU-PRO": 0.3585 - } - }, - { - "id": "nbeerbower/Mistral-Nemo-Moderne-12B-FFT-experimental", - "name": "Mistral-Nemo-Moderne-12B-FFT-experimental", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3352, - "hfopenllm_v2/BBH": 0.5234, - "hfopenllm_v2/MATH Level 5": 0.077, - "hfopenllm_v2/GPQA": 0.281, - "hfopenllm_v2/MUSR": 0.3715, - "hfopenllm_v2/MMLU-PRO": 0.3455 - } - }, - { - "id": "nbeerbower/mistral-nemo-narwhal-12B", - "name": "mistral-nemo-narwhal-12B", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5549, - "hfopenllm_v2/BBH": 0.5057, - "hfopenllm_v2/MATH Level 5": 0.0582, - "hfopenllm_v2/GPQA": 0.271, - "hfopenllm_v2/MUSR": 0.3847, - "hfopenllm_v2/MMLU-PRO": 0.3483 - } - }, - { - "id": "nbeerbower/Mistral-Nemo-Prism-12B", - "name": "Mistral-Nemo-Prism-12B", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6858, - "hfopenllm_v2/BBH": 0.5475, - "hfopenllm_v2/MATH Level 5": 0.0869, - "hfopenllm_v2/GPQA": 0.3079, - "hfopenllm_v2/MUSR": 0.4626, - "hfopenllm_v2/MMLU-PRO": 0.3581 - } - }, - { - "id": "nbeerbower/Mistral-Nemo-Prism-12B-v2", - "name": "Mistral-Nemo-Prism-12B-v2", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6974, - "hfopenllm_v2/BBH": 0.5492, - "hfopenllm_v2/MATH Level 5": 0.0891, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.46, - "hfopenllm_v2/MMLU-PRO": 0.3567 - } - }, - { - "id": "nbeerbower/Mistral-Nemo-Prism-12B-v7", - "name": "Mistral-Nemo-Prism-12B-v7", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6962, - "hfopenllm_v2/BBH": 0.5521, - "hfopenllm_v2/MATH Level 5": 0.0869, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.4639, - "hfopenllm_v2/MMLU-PRO": 0.359 - } - }, - { - "id": "nbeerbower/mistral-nemo-wissenschaft-12B", - "name": "mistral-nemo-wissenschaft-12B", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.652, - "hfopenllm_v2/BBH": 0.504, - "hfopenllm_v2/MATH Level 5": 0.1216, - "hfopenllm_v2/GPQA": 0.2928, - "hfopenllm_v2/MUSR": 0.4178, - "hfopenllm_v2/MMLU-PRO": 0.3532 - } - }, - { - "id": "nbeerbower/Mistral-Small-Drummer-22B", - "name": "Mistral-Small-Drummer-22B", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6331, - "hfopenllm_v2/BBH": 0.5793, - "hfopenllm_v2/MATH Level 5": 0.1888, - "hfopenllm_v2/GPQA": 0.3431, - "hfopenllm_v2/MUSR": 0.4064, - "hfopenllm_v2/MMLU-PRO": 0.4095 - } - }, - { - "id": "nbeerbower/Mistral-Small-Gutenberg-Doppel-22B", - "name": "Mistral-Small-Gutenberg-Doppel-22B", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4893, - "hfopenllm_v2/BBH": 0.5859, - "hfopenllm_v2/MATH Level 5": 0.2183, - "hfopenllm_v2/GPQA": 0.3465, - "hfopenllm_v2/MUSR": 0.3971, - "hfopenllm_v2/MMLU-PRO": 0.4124 - } - }, - { - "id": "nbeerbower/Nemo-Loony-12B-experimental", - "name": "Nemo-Loony-12B-experimental", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3734, - "hfopenllm_v2/BBH": 0.3822, - "hfopenllm_v2/MATH Level 5": 0.0151, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.3341, - "hfopenllm_v2/MMLU-PRO": 0.1589 - } - }, - { - "id": "nbeerbower/Nemoties-ChatML-12B", - "name": "Nemoties-ChatML-12B", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6382, - "hfopenllm_v2/BBH": 0.547, - "hfopenllm_v2/MATH Level 5": 0.0785, - "hfopenllm_v2/GPQA": 0.297, - "hfopenllm_v2/MUSR": 0.4509, - "hfopenllm_v2/MMLU-PRO": 0.3551 - } - }, - { - "id": "nbeerbower/Qwen2.5-Gutenberg-Doppel-14B", - "name": "Qwen2.5-Gutenberg-Doppel-14B", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8091, - "hfopenllm_v2/BBH": 0.6382, - "hfopenllm_v2/MATH Level 5": 0.5415, - "hfopenllm_v2/GPQA": 0.3331, - "hfopenllm_v2/MUSR": 0.4101, - "hfopenllm_v2/MMLU-PRO": 0.4921 - } - }, - { - "id": "nbeerbower/SmolNemo-12B-FFT-experimental", - "name": "SmolNemo-12B-FFT-experimental", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3348, - "hfopenllm_v2/BBH": 0.3336, - "hfopenllm_v2/MATH Level 5": 0.0128, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3847, - "hfopenllm_v2/MMLU-PRO": 0.1217 - } - }, - { - "id": "nbeerbower/Stella-mistral-nemo-12B-v2", - "name": "Stella-mistral-nemo-12B-v2", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3274, - "hfopenllm_v2/BBH": 0.5484, - "hfopenllm_v2/MATH Level 5": 0.1163, - "hfopenllm_v2/GPQA": 0.3322, - "hfopenllm_v2/MUSR": 0.4304, - "hfopenllm_v2/MMLU-PRO": 0.3684 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/nbrahme.json b/data/developers/nbrahme.json deleted file mode 100644 index 5b66226217bb36661e02e5e8f01691e3aae8a17f..0000000000000000000000000000000000000000 --- a/data/developers/nbrahme.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "nbrahme", - "models": [ - { - "id": "nbrahme/IndusQ", - "name": "IndusQ", - "developer": "nbrahme", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.244, - "hfopenllm_v2/BBH": 0.3062, - "hfopenllm_v2/MATH Level 5": 0.0008, - "hfopenllm_v2/GPQA": 0.2651, - "hfopenllm_v2/MUSR": 0.3366, - "hfopenllm_v2/MMLU-PRO": 0.112 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/ncsoft.json b/data/developers/ncsoft.json deleted file mode 100644 index 78886a7e33419ae72417f31ea13b0a41a2a3ced1..0000000000000000000000000000000000000000 --- a/data/developers/ncsoft.json +++ /dev/null @@ -1,50 +0,0 @@ -{ - "developer": "NCSOFT", - "models": [ - { - "id": "NCSOFT/Llama-3-OffsetBias-8B", - "name": "NCSOFT/Llama-3-OffsetBias-8B", - "developer": "NCSOFT", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.8397, - "reward-bench/Chat": 0.9246, - "reward-bench/Chat Hard": 0.8026, - "reward-bench/Safety": 0.8676, - "reward-bench/Reasoning": 0.7639 - } - }, - { - "id": "NCSOFT/Llama-3-OffsetBias-RM-8B", - "name": "NCSOFT/Llama-3-OffsetBias-RM-8B", - "developer": "NCSOFT", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.8942, - "reward-bench/Factuality": 0.6084, - "reward-bench/Precise IF": 0.4, - "reward-bench/Math": 0.5191, - "reward-bench/Safety": 0.8676, - "reward-bench/Focus": 0.9596, - "reward-bench/Ties": 0.6786, - "reward-bench/Chat": 0.9721, - "reward-bench/Chat Hard": 0.818, - "reward-bench/Reasoning": 0.9192 - } - }, - { - "id": "NCSOFT/Llama-VARCO-8B-Instruct", - "name": "Llama-VARCO-8B-Instruct", - "developer": "NCSOFT", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.447, - "hfopenllm_v2/BBH": 0.5023, - "hfopenllm_v2/MATH Level 5": 0.1065, - "hfopenllm_v2/GPQA": 0.297, - "hfopenllm_v2/MUSR": 0.3841, - "hfopenllm_v2/MMLU-PRO": 0.319 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/necva.json b/data/developers/necva.json deleted file mode 100644 index edc80018436dc75b2113f6a9422f4bd9c8eb566b..0000000000000000000000000000000000000000 --- a/data/developers/necva.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "necva", - "models": [ - { - "id": "necva/IE-cont-Llama3.1-8B", - "name": "IE-cont-Llama3.1-8B", - "developer": "necva", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2049, - "hfopenllm_v2/BBH": 0.2912, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3575, - "hfopenllm_v2/MMLU-PRO": 0.1167 - } - }, - { - "id": "necva/replica-IEPile", - "name": "replica-IEPile", - "developer": "necva", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4678, - "hfopenllm_v2/BBH": 0.4779, - "hfopenllm_v2/MATH Level 5": 0.1239, - "hfopenllm_v2/GPQA": 0.3062, - "hfopenllm_v2/MUSR": 0.3998, - "hfopenllm_v2/MMLU-PRO": 0.3561 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/nekochu.json b/data/developers/nekochu.json deleted file mode 100644 index 531b7a10eb1ae90d0651d8748cdb54c92c3cfabb..0000000000000000000000000000000000000000 --- a/data/developers/nekochu.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "developer": "Nekochu", - "models": [ - { - "id": "Nekochu/Llama-3.1-8B-french-DPO", - "name": "Llama-3.1-8B-french-DPO", - "developer": "Nekochu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4656, - "hfopenllm_v2/BBH": 0.5111, - "hfopenllm_v2/MATH Level 5": 0.0974, - "hfopenllm_v2/GPQA": 0.2911, - "hfopenllm_v2/MUSR": 0.4216, - "hfopenllm_v2/MMLU-PRO": 0.3414 - } - }, - { - "id": "Nekochu/Llama-3.1-8B-German-ORPO", - "name": "Llama-3.1-8B-German-ORPO", - "developer": "Nekochu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4611, - "hfopenllm_v2/BBH": 0.4983, - "hfopenllm_v2/MATH Level 5": 0.1171, - "hfopenllm_v2/GPQA": 0.3163, - "hfopenllm_v2/MUSR": 0.4647, - "hfopenllm_v2/MMLU-PRO": 0.3393 - } - }, - { - "id": "Nekochu/Luminia-13B-v3", - "name": "Luminia-13B-v3", - "developer": "Nekochu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2523, - "hfopenllm_v2/BBH": 0.4112, - "hfopenllm_v2/MATH Level 5": 0.0181, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.3983, - "hfopenllm_v2/MMLU-PRO": 0.2215 - } - }, - { - "id": "Nekochu/Luminia-8B-RP", - "name": "Luminia-8B-RP", - "developer": "Nekochu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5574, - "hfopenllm_v2/BBH": 0.5218, - "hfopenllm_v2/MATH Level 5": 0.136, - "hfopenllm_v2/GPQA": 0.297, - "hfopenllm_v2/MUSR": 0.3998, - "hfopenllm_v2/MMLU-PRO": 0.3631 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/neopolita.json b/data/developers/neopolita.json deleted file mode 100644 index f2d08b45e69605ff9b5860f25d070f5c01699763..0000000000000000000000000000000000000000 --- a/data/developers/neopolita.json +++ /dev/null @@ -1,159 +0,0 @@ -{ - "developer": "neopolita", - "models": [ - { - "id": "neopolita/jessi-v0.1-bf16-falcon3-7b-instruct", - "name": "jessi-v0.1-bf16-falcon3-7b-instruct", - "developer": "neopolita", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7527, - "hfopenllm_v2/BBH": 0.5516, - "hfopenllm_v2/MATH Level 5": 0.3807, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.4825, - "hfopenllm_v2/MMLU-PRO": 0.3924 - } - }, - { - "id": "neopolita/jessi-v0.1-falcon3-10b-instruct", - "name": "jessi-v0.1-falcon3-10b-instruct", - "developer": "neopolita", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7552, - "hfopenllm_v2/BBH": 0.5953, - "hfopenllm_v2/MATH Level 5": 0.2002, - "hfopenllm_v2/GPQA": 0.3188, - "hfopenllm_v2/MUSR": 0.4279, - "hfopenllm_v2/MMLU-PRO": 0.4188 - } - }, - { - "id": "neopolita/jessi-v0.1-qwen2.5-7b-instruct", - "name": "jessi-v0.1-qwen2.5-7b-instruct", - "developer": "neopolita", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7327, - "hfopenllm_v2/BBH": 0.5292, - "hfopenllm_v2/MATH Level 5": 0.4086, - "hfopenllm_v2/GPQA": 0.297, - "hfopenllm_v2/MUSR": 0.3914, - "hfopenllm_v2/MMLU-PRO": 0.4228 - } - }, - { - "id": "neopolita/jessi-v0.1-virtuoso-small", - "name": "jessi-v0.1-virtuoso-small", - "developer": "neopolita", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7959, - "hfopenllm_v2/BBH": 0.6443, - "hfopenllm_v2/MATH Level 5": 0.3399, - "hfopenllm_v2/GPQA": 0.3305, - "hfopenllm_v2/MUSR": 0.4362, - "hfopenllm_v2/MMLU-PRO": 0.513 - } - }, - { - "id": "neopolita/jessi-v0.2-falcon3-10b-instruct", - "name": "jessi-v0.2-falcon3-10b-instruct", - "developer": "neopolita", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7768, - "hfopenllm_v2/BBH": 0.6205, - "hfopenllm_v2/MATH Level 5": 0.2122, - "hfopenllm_v2/GPQA": 0.3289, - "hfopenllm_v2/MUSR": 0.4281, - "hfopenllm_v2/MMLU-PRO": 0.4354 - } - }, - { - "id": "neopolita/jessi-v0.2-falcon3-7b-instruct", - "name": "jessi-v0.2-falcon3-7b-instruct", - "developer": "neopolita", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5771, - "hfopenllm_v2/BBH": 0.5363, - "hfopenllm_v2/MATH Level 5": 0.2538, - "hfopenllm_v2/GPQA": 0.3171, - "hfopenllm_v2/MUSR": 0.4479, - "hfopenllm_v2/MMLU-PRO": 0.3905 - } - }, - { - "id": "neopolita/jessi-v0.3-falcon3-7b-instruct", - "name": "jessi-v0.3-falcon3-7b-instruct", - "developer": "neopolita", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7509, - "hfopenllm_v2/BBH": 0.5388, - "hfopenllm_v2/MATH Level 5": 0.1888, - "hfopenllm_v2/GPQA": 0.3196, - "hfopenllm_v2/MUSR": 0.4692, - "hfopenllm_v2/MMLU-PRO": 0.397 - } - }, - { - "id": "neopolita/jessi-v0.4-falcon3-7b-instruct", - "name": "jessi-v0.4-falcon3-7b-instruct", - "developer": "neopolita", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7604, - "hfopenllm_v2/BBH": 0.5522, - "hfopenllm_v2/MATH Level 5": 0.3769, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.4971, - "hfopenllm_v2/MMLU-PRO": 0.4004 - } - }, - { - "id": "neopolita/jessi-v0.5-falcon3-7b-instruct", - "name": "jessi-v0.5-falcon3-7b-instruct", - "developer": "neopolita", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7412, - "hfopenllm_v2/BBH": 0.559, - "hfopenllm_v2/MATH Level 5": 0.3739, - "hfopenllm_v2/GPQA": 0.3112, - "hfopenllm_v2/MUSR": 0.4865, - "hfopenllm_v2/MMLU-PRO": 0.3966 - } - }, - { - "id": "neopolita/jessi-v0.6-falcon3-7b-instruct", - "name": "jessi-v0.6-falcon3-7b-instruct", - "developer": "neopolita", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7402, - "hfopenllm_v2/BBH": 0.5509, - "hfopenllm_v2/MATH Level 5": 0.3565, - "hfopenllm_v2/GPQA": 0.3003, - "hfopenllm_v2/MUSR": 0.4904, - "hfopenllm_v2/MMLU-PRO": 0.3957 - } - }, - { - "id": "neopolita/loki-v0.1-virtuoso", - "name": "loki-v0.1-virtuoso", - "developer": "neopolita", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7819, - "hfopenllm_v2/BBH": 0.6467, - "hfopenllm_v2/MATH Level 5": 0.3391, - "hfopenllm_v2/GPQA": 0.3507, - "hfopenllm_v2/MUSR": 0.4375, - "hfopenllm_v2/MMLU-PRO": 0.5129 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/netcat420.json b/data/developers/netcat420.json deleted file mode 100644 index 8f0c15c90b3987f14176a8a09aeb791c356ac21b..0000000000000000000000000000000000000000 --- a/data/developers/netcat420.json +++ /dev/null @@ -1,677 +0,0 @@ -{ - "developer": "netcat420", - "models": [ - { - "id": "netcat420/DeepSeek-R1-Distill-Qwen-MFANN-Slerp-7b", - "name": "DeepSeek-R1-Distill-Qwen-MFANN-Slerp-7b", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.115, - "hfopenllm_v2/BBH": 0.2877, - "hfopenllm_v2/MATH Level 5": 0.0015, - "hfopenllm_v2/GPQA": 0.2643, - "hfopenllm_v2/MUSR": 0.3724, - "hfopenllm_v2/MMLU-PRO": 0.109 - } - }, - { - "id": "netcat420/DeepSeek-R1-MFANN-TIES-unretrained-7b", - "name": "DeepSeek-R1-MFANN-TIES-unretrained-7b", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2587, - "hfopenllm_v2/BBH": 0.3086, - "hfopenllm_v2/MATH Level 5": 0.0121, - "hfopenllm_v2/GPQA": 0.255, - "hfopenllm_v2/MUSR": 0.3527, - "hfopenllm_v2/MMLU-PRO": 0.1145 - } - }, - { - "id": "netcat420/Llama3.1-MFANN-8b", - "name": "Llama3.1-MFANN-8b", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.297, - "hfopenllm_v2/BBH": 0.4281, - "hfopenllm_v2/MATH Level 5": 0.0295, - "hfopenllm_v2/GPQA": 0.2878, - "hfopenllm_v2/MUSR": 0.3379, - "hfopenllm_v2/MMLU-PRO": 0.2725 - } - }, - { - "id": "netcat420/MFANN-abliterated-phi2-merge-unretrained", - "name": "MFANN-abliterated-phi2-merge-unretrained", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3005, - "hfopenllm_v2/BBH": 0.4104, - "hfopenllm_v2/MATH Level 5": 0.0287, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.3183, - "hfopenllm_v2/MMLU-PRO": 0.1478 - } - }, - { - "id": "netcat420/MFANN-llama3.1-Abliterated-SLERP", - "name": "MFANN-llama3.1-Abliterated-SLERP", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2591, - "hfopenllm_v2/BBH": 0.4574, - "hfopenllm_v2/MATH Level 5": 0.0483, - "hfopenllm_v2/GPQA": 0.2735, - "hfopenllm_v2/MUSR": 0.3809, - "hfopenllm_v2/MMLU-PRO": 0.2928 - } - }, - { - "id": "netcat420/MFANN-Llama3.1-Abliterated-Slerp-TIES", - "name": "MFANN-Llama3.1-Abliterated-Slerp-TIES", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4293, - "hfopenllm_v2/BBH": 0.4968, - "hfopenllm_v2/MATH Level 5": 0.0665, - "hfopenllm_v2/GPQA": 0.2919, - "hfopenllm_v2/MUSR": 0.3687, - "hfopenllm_v2/MMLU-PRO": 0.3531 - } - }, - { - "id": "netcat420/MFANN-Llama3.1-Abliterated-SLERP-TIES-V2", - "name": "MFANN-Llama3.1-Abliterated-SLERP-TIES-V2", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.421, - "hfopenllm_v2/BBH": 0.4924, - "hfopenllm_v2/MATH Level 5": 0.0763, - "hfopenllm_v2/GPQA": 0.297, - "hfopenllm_v2/MUSR": 0.3728, - "hfopenllm_v2/MMLU-PRO": 0.3522 - } - }, - { - "id": "netcat420/MFANN-Llama3.1-Abliterated-SLERP-TIES-V3", - "name": "MFANN-Llama3.1-Abliterated-SLERP-TIES-V3", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4238, - "hfopenllm_v2/BBH": 0.4914, - "hfopenllm_v2/MATH Level 5": 0.0755, - "hfopenllm_v2/GPQA": 0.297, - "hfopenllm_v2/MUSR": 0.3741, - "hfopenllm_v2/MMLU-PRO": 0.349 - } - }, - { - "id": "netcat420/MFANN-llama3.1-abliterated-SLERP-v3", - "name": "MFANN-llama3.1-abliterated-SLERP-v3", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3799, - "hfopenllm_v2/BBH": 0.4931, - "hfopenllm_v2/MATH Level 5": 0.0642, - "hfopenllm_v2/GPQA": 0.2911, - "hfopenllm_v2/MUSR": 0.366, - "hfopenllm_v2/MMLU-PRO": 0.3531 - } - }, - { - "id": "netcat420/MFANN-llama3.1-abliterated-SLERP-v3.1", - "name": "MFANN-llama3.1-abliterated-SLERP-v3.1", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4202, - "hfopenllm_v2/BBH": 0.4921, - "hfopenllm_v2/MATH Level 5": 0.0695, - "hfopenllm_v2/GPQA": 0.2928, - "hfopenllm_v2/MUSR": 0.3686, - "hfopenllm_v2/MMLU-PRO": 0.3543 - } - }, - { - "id": "netcat420/MFANN-Llama3.1-Abliterated-Slerp-V3.2", - "name": "MFANN-Llama3.1-Abliterated-Slerp-V3.2", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4128, - "hfopenllm_v2/BBH": 0.4978, - "hfopenllm_v2/MATH Level 5": 0.0702, - "hfopenllm_v2/GPQA": 0.2878, - "hfopenllm_v2/MUSR": 0.3754, - "hfopenllm_v2/MMLU-PRO": 0.3527 - } - }, - { - "id": "netcat420/MFANN-Llama3.1-Abliterated-SLERP-V4", - "name": "MFANN-Llama3.1-Abliterated-SLERP-V4", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4169, - "hfopenllm_v2/BBH": 0.4909, - "hfopenllm_v2/MATH Level 5": 0.068, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.3821, - "hfopenllm_v2/MMLU-PRO": 0.3516 - } - }, - { - "id": "netcat420/MFANN-Llama3.1-Abliterated-SLERP-V5", - "name": "MFANN-Llama3.1-Abliterated-SLERP-V5", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4329, - "hfopenllm_v2/BBH": 0.4952, - "hfopenllm_v2/MATH Level 5": 0.0816, - "hfopenllm_v2/GPQA": 0.2936, - "hfopenllm_v2/MUSR": 0.3781, - "hfopenllm_v2/MMLU-PRO": 0.3445 - } - }, - { - "id": "netcat420/MFANN-llama3.1-abliterated-v2", - "name": "MFANN-llama3.1-abliterated-v2", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4429, - "hfopenllm_v2/BBH": 0.4941, - "hfopenllm_v2/MATH Level 5": 0.074, - "hfopenllm_v2/GPQA": 0.2928, - "hfopenllm_v2/MUSR": 0.3845, - "hfopenllm_v2/MMLU-PRO": 0.3491 - } - }, - { - "id": "netcat420/MFANN-phigments-slerp-V2", - "name": "MFANN-phigments-slerp-V2", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3232, - "hfopenllm_v2/BBH": 0.4827, - "hfopenllm_v2/MATH Level 5": 0.0317, - "hfopenllm_v2/GPQA": 0.2727, - "hfopenllm_v2/MUSR": 0.4037, - "hfopenllm_v2/MMLU-PRO": 0.2717 - } - }, - { - "id": "netcat420/MFANN-phigments-slerp-V3.2", - "name": "MFANN-phigments-slerp-V3.2", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3524, - "hfopenllm_v2/BBH": 0.4809, - "hfopenllm_v2/MATH Level 5": 0.0332, - "hfopenllm_v2/GPQA": 0.2836, - "hfopenllm_v2/MUSR": 0.3708, - "hfopenllm_v2/MMLU-PRO": 0.2705 - } - }, - { - "id": "netcat420/MFANN-phigments-slerp-V3.3", - "name": "MFANN-phigments-slerp-V3.3", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3691, - "hfopenllm_v2/BBH": 0.4895, - "hfopenllm_v2/MATH Level 5": 0.0332, - "hfopenllm_v2/GPQA": 0.2752, - "hfopenllm_v2/MUSR": 0.3892, - "hfopenllm_v2/MMLU-PRO": 0.2803 - } - }, - { - "id": "netcat420/MFANN-SFT", - "name": "MFANN-SFT", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3682, - "hfopenllm_v2/BBH": 0.4852, - "hfopenllm_v2/MATH Level 5": 0.0597, - "hfopenllm_v2/GPQA": 0.3163, - "hfopenllm_v2/MUSR": 0.3725, - "hfopenllm_v2/MMLU-PRO": 0.3336 - } - }, - { - "id": "netcat420/MFANN3b", - "name": "MFANN3b", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2524, - "hfopenllm_v2/BBH": 0.4433, - "hfopenllm_v2/MATH Level 5": 0.0219, - "hfopenllm_v2/GPQA": 0.2919, - "hfopenllm_v2/MUSR": 0.3606, - "hfopenllm_v2/MMLU-PRO": 0.2306 - } - }, - { - "id": "netcat420/MFANN3bv0.15", - "name": "MFANN3bv0.15", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2012, - "hfopenllm_v2/BBH": 0.4539, - "hfopenllm_v2/MATH Level 5": 0.0264, - "hfopenllm_v2/GPQA": 0.2517, - "hfopenllm_v2/MUSR": 0.3958, - "hfopenllm_v2/MMLU-PRO": 0.2468 - } - }, - { - "id": "netcat420/MFANN3bv0.18", - "name": "MFANN3bv0.18", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2206, - "hfopenllm_v2/BBH": 0.4514, - "hfopenllm_v2/MATH Level 5": 0.0249, - "hfopenllm_v2/GPQA": 0.2576, - "hfopenllm_v2/MUSR": 0.4024, - "hfopenllm_v2/MMLU-PRO": 0.25 - } - }, - { - "id": "netcat420/MFANN3bv0.19", - "name": "MFANN3bv0.19", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2258, - "hfopenllm_v2/BBH": 0.4516, - "hfopenllm_v2/MATH Level 5": 0.0227, - "hfopenllm_v2/GPQA": 0.2576, - "hfopenllm_v2/MUSR": 0.4024, - "hfopenllm_v2/MMLU-PRO": 0.252 - } - }, - { - "id": "netcat420/MFANN3bv0.20", - "name": "MFANN3bv0.20", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2193, - "hfopenllm_v2/BBH": 0.4493, - "hfopenllm_v2/MATH Level 5": 0.0264, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.4077, - "hfopenllm_v2/MMLU-PRO": 0.25 - } - }, - { - "id": "netcat420/MFANN3bv0.21", - "name": "MFANN3bv0.21", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1909, - "hfopenllm_v2/BBH": 0.447, - "hfopenllm_v2/MATH Level 5": 0.0317, - "hfopenllm_v2/GPQA": 0.2643, - "hfopenllm_v2/MUSR": 0.3759, - "hfopenllm_v2/MMLU-PRO": 0.2393 - } - }, - { - "id": "netcat420/MFANN3bv0.22", - "name": "MFANN3bv0.22", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1979, - "hfopenllm_v2/BBH": 0.4485, - "hfopenllm_v2/MATH Level 5": 0.0264, - "hfopenllm_v2/GPQA": 0.2617, - "hfopenllm_v2/MUSR": 0.3521, - "hfopenllm_v2/MMLU-PRO": 0.2517 - } - }, - { - "id": "netcat420/MFANN3bv0.23", - "name": "MFANN3bv0.23", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2048, - "hfopenllm_v2/BBH": 0.4495, - "hfopenllm_v2/MATH Level 5": 0.0249, - "hfopenllm_v2/GPQA": 0.2517, - "hfopenllm_v2/MUSR": 0.3427, - "hfopenllm_v2/MMLU-PRO": 0.2418 - } - }, - { - "id": "netcat420/MFANN3bv0.24", - "name": "MFANN3bv0.24", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.22, - "hfopenllm_v2/BBH": 0.4407, - "hfopenllm_v2/MATH Level 5": 0.0279, - "hfopenllm_v2/GPQA": 0.2584, - "hfopenllm_v2/MUSR": 0.3521, - "hfopenllm_v2/MMLU-PRO": 0.2352 - } - }, - { - "id": "netcat420/MFANN3bv1.1", - "name": "MFANN3bv1.1", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2507, - "hfopenllm_v2/BBH": 0.3397, - "hfopenllm_v2/MATH Level 5": 0.0204, - "hfopenllm_v2/GPQA": 0.2668, - "hfopenllm_v2/MUSR": 0.3223, - "hfopenllm_v2/MMLU-PRO": 0.1159 - } - }, - { - "id": "netcat420/MFANN3bv1.2", - "name": "MFANN3bv1.2", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2686, - "hfopenllm_v2/BBH": 0.366, - "hfopenllm_v2/MATH Level 5": 0.0264, - "hfopenllm_v2/GPQA": 0.2634, - "hfopenllm_v2/MUSR": 0.3156, - "hfopenllm_v2/MMLU-PRO": 0.145 - } - }, - { - "id": "netcat420/MFANN3bv1.3", - "name": "MFANN3bv1.3", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2547, - "hfopenllm_v2/BBH": 0.4456, - "hfopenllm_v2/MATH Level 5": 0.0211, - "hfopenllm_v2/GPQA": 0.2576, - "hfopenllm_v2/MUSR": 0.3299, - "hfopenllm_v2/MMLU-PRO": 0.2276 - } - }, - { - "id": "netcat420/MFANN3bv1.4", - "name": "MFANN3bv1.4", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3524, - "hfopenllm_v2/BBH": 0.4809, - "hfopenllm_v2/MATH Level 5": 0.037, - "hfopenllm_v2/GPQA": 0.2827, - "hfopenllm_v2/MUSR": 0.3708, - "hfopenllm_v2/MMLU-PRO": 0.2705 - } - }, - { - "id": "netcat420/MFANNv0.19", - "name": "MFANNv0.19", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3057, - "hfopenllm_v2/BBH": 0.4731, - "hfopenllm_v2/MATH Level 5": 0.0415, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.3527, - "hfopenllm_v2/MMLU-PRO": 0.2473 - } - }, - { - "id": "netcat420/MFANNv0.20", - "name": "MFANNv0.20", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3479, - "hfopenllm_v2/BBH": 0.4574, - "hfopenllm_v2/MATH Level 5": 0.0498, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.3874, - "hfopenllm_v2/MMLU-PRO": 0.3202 - } - }, - { - "id": "netcat420/MFANNv0.21", - "name": "MFANNv0.21", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3233, - "hfopenllm_v2/BBH": 0.4576, - "hfopenllm_v2/MATH Level 5": 0.0574, - "hfopenllm_v2/GPQA": 0.2785, - "hfopenllm_v2/MUSR": 0.3993, - "hfopenllm_v2/MMLU-PRO": 0.3031 - } - }, - { - "id": "netcat420/MFANNv0.22.1", - "name": "MFANNv0.22.1", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3089, - "hfopenllm_v2/BBH": 0.4661, - "hfopenllm_v2/MATH Level 5": 0.0536, - "hfopenllm_v2/GPQA": 0.276, - "hfopenllm_v2/MUSR": 0.3753, - "hfopenllm_v2/MMLU-PRO": 0.3343 - } - }, - { - "id": "netcat420/MFANNv0.23", - "name": "MFANNv0.23", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3127, - "hfopenllm_v2/BBH": 0.4898, - "hfopenllm_v2/MATH Level 5": 0.0498, - "hfopenllm_v2/GPQA": 0.2844, - "hfopenllm_v2/MUSR": 0.3768, - "hfopenllm_v2/MMLU-PRO": 0.3388 - } - }, - { - "id": "netcat420/MFANNv0.24", - "name": "MFANNv0.24", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3162, - "hfopenllm_v2/BBH": 0.479, - "hfopenllm_v2/MATH Level 5": 0.0612, - "hfopenllm_v2/GPQA": 0.2844, - "hfopenllm_v2/MUSR": 0.3754, - "hfopenllm_v2/MMLU-PRO": 0.3348 - } - }, - { - "id": "netcat420/MFANNv0.25", - "name": "MFANNv0.25", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3467, - "hfopenllm_v2/BBH": 0.4794, - "hfopenllm_v2/MATH Level 5": 0.0582, - "hfopenllm_v2/GPQA": 0.2802, - "hfopenllm_v2/MUSR": 0.3688, - "hfopenllm_v2/MMLU-PRO": 0.3343 - } - }, - { - "id": "netcat420/Qwen2.5-7b-MFANN-slerp", - "name": "Qwen2.5-7b-MFANN-slerp", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6532, - "hfopenllm_v2/BBH": 0.5089, - "hfopenllm_v2/MATH Level 5": 0.287, - "hfopenllm_v2/GPQA": 0.2953, - "hfopenllm_v2/MUSR": 0.4073, - "hfopenllm_v2/MMLU-PRO": 0.3417 - } - }, - { - "id": "netcat420/Qwen2.5-7b-nerd-uncensored-MFANN-slerp", - "name": "Qwen2.5-7b-nerd-uncensored-MFANN-slerp", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1564, - "hfopenllm_v2/BBH": 0.292, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3792, - "hfopenllm_v2/MMLU-PRO": 0.11 - } - }, - { - "id": "netcat420/Qwen2.5-7B-nerd-uncensored-v0.9-MFANN", - "name": "Qwen2.5-7B-nerd-uncensored-v0.9-MFANN", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5878, - "hfopenllm_v2/BBH": 0.5237, - "hfopenllm_v2/MATH Level 5": 0.3376, - "hfopenllm_v2/GPQA": 0.281, - "hfopenllm_v2/MUSR": 0.3926, - "hfopenllm_v2/MMLU-PRO": 0.3904 - } - }, - { - "id": "netcat420/Qwen2.5-Coder-Scholar-7B-Abliterated-MFANN", - "name": "Qwen2.5-Coder-Scholar-7B-Abliterated-MFANN", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5742, - "hfopenllm_v2/BBH": 0.5071, - "hfopenllm_v2/MATH Level 5": 0.2568, - "hfopenllm_v2/GPQA": 0.2928, - "hfopenllm_v2/MUSR": 0.4058, - "hfopenllm_v2/MMLU-PRO": 0.3157 - } - }, - { - "id": "netcat420/Qwen2.5-Coder-Scholar-7B-Abliterated-MFANN-Slerp-Unretrained", - "name": "Qwen2.5-Coder-Scholar-7B-Abliterated-MFANN-Slerp-Unretrained", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6486, - "hfopenllm_v2/BBH": 0.5066, - "hfopenllm_v2/MATH Level 5": 0.2991, - "hfopenllm_v2/GPQA": 0.2987, - "hfopenllm_v2/MUSR": 0.4152, - "hfopenllm_v2/MMLU-PRO": 0.3432 - } - }, - { - "id": "netcat420/Qwen2.5-DeepSeek-R1-MFANN-Slerp-7b", - "name": "Qwen2.5-DeepSeek-R1-MFANN-Slerp-7b", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2676, - "hfopenllm_v2/BBH": 0.3789, - "hfopenllm_v2/MATH Level 5": 0.0181, - "hfopenllm_v2/GPQA": 0.2324, - "hfopenllm_v2/MUSR": 0.3528, - "hfopenllm_v2/MMLU-PRO": 0.1677 - } - }, - { - "id": "netcat420/Qwen2.5-MFANN-7b", - "name": "Qwen2.5-MFANN-7b", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6097, - "hfopenllm_v2/BBH": 0.5054, - "hfopenllm_v2/MATH Level 5": 0.2787, - "hfopenllm_v2/GPQA": 0.2861, - "hfopenllm_v2/MUSR": 0.4021, - "hfopenllm_v2/MMLU-PRO": 0.3233 - } - }, - { - "id": "netcat420/qwen2.5-MFANN-7b-SLERP-V1.2", - "name": "qwen2.5-MFANN-7b-SLERP-V1.2", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6606, - "hfopenllm_v2/BBH": 0.5111, - "hfopenllm_v2/MATH Level 5": 0.287, - "hfopenllm_v2/GPQA": 0.297, - "hfopenllm_v2/MUSR": 0.4259, - "hfopenllm_v2/MMLU-PRO": 0.3438 - } - }, - { - "id": "netcat420/qwen2.5-MFANN-7b-SLERPv1.1", - "name": "qwen2.5-MFANN-7b-SLERPv1.1", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6555, - "hfopenllm_v2/BBH": 0.5075, - "hfopenllm_v2/MATH Level 5": 0.2968, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.4126, - "hfopenllm_v2/MMLU-PRO": 0.3448 - } - }, - { - "id": "netcat420/qwen2.5-MFANN-7b-v1.1", - "name": "qwen2.5-MFANN-7b-v1.1", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6088, - "hfopenllm_v2/BBH": 0.4967, - "hfopenllm_v2/MATH Level 5": 0.2825, - "hfopenllm_v2/GPQA": 0.276, - "hfopenllm_v2/MUSR": 0.4114, - "hfopenllm_v2/MMLU-PRO": 0.3248 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/netease-youdao.json b/data/developers/netease-youdao.json deleted file mode 100644 index bdfbc64e891f992717eb6aa262681ba8949f553d..0000000000000000000000000000000000000000 --- a/data/developers/netease-youdao.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "netease-youdao", - "models": [ - { - "id": "netease-youdao/Confucius-o1-14B", - "name": "Confucius-o1-14B", - "developer": "netease-youdao", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6378, - "hfopenllm_v2/BBH": 0.63, - "hfopenllm_v2/MATH Level 5": 0.4313, - "hfopenllm_v2/GPQA": 0.3649, - "hfopenllm_v2/MUSR": 0.4338, - "hfopenllm_v2/MMLU-PRO": 0.5265 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/neversleep.json b/data/developers/neversleep.json deleted file mode 100644 index 42fe13428db7943afea84088f1b314bfebca83f1..0000000000000000000000000000000000000000 --- a/data/developers/neversleep.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "NeverSleep", - "models": [ - { - "id": "NeverSleep/Lumimaid-v0.2-12B", - "name": "Lumimaid-v0.2-12B", - "developer": "NeverSleep", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1099, - "hfopenllm_v2/BBH": 0.5396, - "hfopenllm_v2/MATH Level 5": 0.0566, - "hfopenllm_v2/GPQA": 0.3146, - "hfopenllm_v2/MUSR": 0.4821, - "hfopenllm_v2/MMLU-PRO": 0.3511 - } - }, - { - "id": "NeverSleep/Lumimaid-v0.2-8B", - "name": "Lumimaid-v0.2-8B", - "developer": "NeverSleep", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5038, - "hfopenllm_v2/BBH": 0.5238, - "hfopenllm_v2/MATH Level 5": 0.1435, - "hfopenllm_v2/GPQA": 0.3112, - "hfopenllm_v2/MUSR": 0.4303, - "hfopenllm_v2/MMLU-PRO": 0.3636 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/newsbang.json b/data/developers/newsbang.json deleted file mode 100644 index 98443dd516eb0045c4fd3eb3c8012c7292fa86dd..0000000000000000000000000000000000000000 --- a/data/developers/newsbang.json +++ /dev/null @@ -1,103 +0,0 @@ -{ - "developer": "newsbang", - "models": [ - { - "id": "newsbang/Homer-7B-v0.1", - "name": "Homer-7B-v0.1", - "developer": "newsbang", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6109, - "hfopenllm_v2/BBH": 0.5601, - "hfopenllm_v2/MATH Level 5": 0.386, - "hfopenllm_v2/GPQA": 0.3247, - "hfopenllm_v2/MUSR": 0.4357, - "hfopenllm_v2/MMLU-PRO": 0.4475 - } - }, - { - "id": "newsbang/Homer-7B-v0.2", - "name": "Homer-7B-v0.2", - "developer": "newsbang", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7494, - "hfopenllm_v2/BBH": 0.5517, - "hfopenllm_v2/MATH Level 5": 0.2477, - "hfopenllm_v2/GPQA": 0.3322, - "hfopenllm_v2/MUSR": 0.4298, - "hfopenllm_v2/MMLU-PRO": 0.441 - } - }, - { - "id": "newsbang/Homer-v0.3-Qwen2.5-7B", - "name": "Homer-v0.3-Qwen2.5-7B", - "developer": "newsbang", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5154, - "hfopenllm_v2/BBH": 0.5481, - "hfopenllm_v2/MATH Level 5": 0.3089, - "hfopenllm_v2/GPQA": 0.3339, - "hfopenllm_v2/MUSR": 0.4744, - "hfopenllm_v2/MMLU-PRO": 0.4456 - } - }, - { - "id": "newsbang/Homer-v0.4-Qwen2.5-7B", - "name": "Homer-v0.4-Qwen2.5-7B", - "developer": "newsbang", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7999, - "hfopenllm_v2/BBH": 0.5533, - "hfopenllm_v2/MATH Level 5": 0.2779, - "hfopenllm_v2/GPQA": 0.3154, - "hfopenllm_v2/MUSR": 0.4311, - "hfopenllm_v2/MMLU-PRO": 0.4363 - } - }, - { - "id": "newsbang/Homer-v0.5-Qwen2.5-7B", - "name": "Homer-v0.5-Qwen2.5-7B", - "developer": "newsbang", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7881, - "hfopenllm_v2/BBH": 0.554, - "hfopenllm_v2/MATH Level 5": 0.3724, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.4193, - "hfopenllm_v2/MMLU-PRO": 0.4369 - } - }, - { - "id": "newsbang/Homer-v1.0-Qwen2.5-72B", - "name": "Homer-v1.0-Qwen2.5-72B", - "developer": "newsbang", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7628, - "hfopenllm_v2/BBH": 0.731, - "hfopenllm_v2/MATH Level 5": 0.4902, - "hfopenllm_v2/GPQA": 0.4161, - "hfopenllm_v2/MUSR": 0.4677, - "hfopenllm_v2/MMLU-PRO": 0.6145 - } - }, - { - "id": "newsbang/Homer-v1.0-Qwen2.5-7B", - "name": "Homer-v1.0-Qwen2.5-7B", - "developer": "newsbang", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6393, - "hfopenllm_v2/BBH": 0.5655, - "hfopenllm_v2/MATH Level 5": 0.3323, - "hfopenllm_v2/GPQA": 0.3221, - "hfopenllm_v2/MUSR": 0.4278, - "hfopenllm_v2/MMLU-PRO": 0.4535 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/nexesenex.json b/data/developers/nexesenex.json deleted file mode 100644 index 6e28447b653002c1f000652d85b18668ef2ecdad..0000000000000000000000000000000000000000 --- a/data/developers/nexesenex.json +++ /dev/null @@ -1,621 +0,0 @@ -{ - "developer": "Nexesenex", - "models": [ - { - "id": "Nexesenex/Dolphin3.0-Llama3.1-1B-abliterated", - "name": "Dolphin3.0-Llama3.1-1B-abliterated", - "developer": "Nexesenex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5312, - "hfopenllm_v2/BBH": 0.3241, - "hfopenllm_v2/MATH Level 5": 0.0385, - "hfopenllm_v2/GPQA": 0.2408, - "hfopenllm_v2/MUSR": 0.3237, - "hfopenllm_v2/MMLU-PRO": 0.1373 - } - }, - { - "id": "Nexesenex/Llama_3.1_8b_DeepDive_3_Prev_v1.0", - "name": "Llama_3.1_8b_DeepDive_3_Prev_v1.0", - "developer": "Nexesenex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6809, - "hfopenllm_v2/BBH": 0.5155, - "hfopenllm_v2/MATH Level 5": 0.1866, - "hfopenllm_v2/GPQA": 0.2911, - "hfopenllm_v2/MUSR": 0.3666, - "hfopenllm_v2/MMLU-PRO": 0.3438 - } - }, - { - "id": "Nexesenex/Llama_3.1_8b_DeepDive_3_R1_Prev_v1.0", - "name": "Llama_3.1_8b_DeepDive_3_R1_Prev_v1.0", - "developer": "Nexesenex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7101, - "hfopenllm_v2/BBH": 0.512, - "hfopenllm_v2/MATH Level 5": 0.1926, - "hfopenllm_v2/GPQA": 0.3003, - "hfopenllm_v2/MUSR": 0.3758, - "hfopenllm_v2/MMLU-PRO": 0.3441 - } - }, - { - "id": "Nexesenex/Llama_3.1_8b_DoberWild_v2.01", - "name": "Llama_3.1_8b_DoberWild_v2.01", - "developer": "Nexesenex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7996, - "hfopenllm_v2/BBH": 0.5251, - "hfopenllm_v2/MATH Level 5": 0.2002, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.4012, - "hfopenllm_v2/MMLU-PRO": 0.3791 - } - }, - { - "id": "Nexesenex/Llama_3.1_8b_DoberWild_v2.03", - "name": "Llama_3.1_8b_DoberWild_v2.03", - "developer": "Nexesenex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7764, - "hfopenllm_v2/BBH": 0.5294, - "hfopenllm_v2/MATH Level 5": 0.2077, - "hfopenllm_v2/GPQA": 0.3045, - "hfopenllm_v2/MUSR": 0.3906, - "hfopenllm_v2/MMLU-PRO": 0.3722 - } - }, - { - "id": "Nexesenex/Llama_3.1_8b_DobHerWild_R1_v1.1R", - "name": "Llama_3.1_8b_DobHerWild_R1_v1.1R", - "developer": "Nexesenex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.76, - "hfopenllm_v2/BBH": 0.5257, - "hfopenllm_v2/MATH Level 5": 0.2319, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.3852, - "hfopenllm_v2/MMLU-PRO": 0.3688 - } - }, - { - "id": "Nexesenex/Llama_3.1_8b_DodoWild_v2.01", - "name": "Llama_3.1_8b_DodoWild_v2.01", - "developer": "Nexesenex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7978, - "hfopenllm_v2/BBH": 0.5253, - "hfopenllm_v2/MATH Level 5": 0.1986, - "hfopenllm_v2/GPQA": 0.3037, - "hfopenllm_v2/MUSR": 0.409, - "hfopenllm_v2/MMLU-PRO": 0.3738 - } - }, - { - "id": "Nexesenex/Llama_3.1_8b_DodoWild_v2.02", - "name": "Llama_3.1_8b_DodoWild_v2.02", - "developer": "Nexesenex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8017, - "hfopenllm_v2/BBH": 0.5262, - "hfopenllm_v2/MATH Level 5": 0.2273, - "hfopenllm_v2/GPQA": 0.3045, - "hfopenllm_v2/MUSR": 0.3971, - "hfopenllm_v2/MMLU-PRO": 0.3761 - } - }, - { - "id": "Nexesenex/Llama_3.1_8b_DodoWild_v2.03", - "name": "Llama_3.1_8b_DodoWild_v2.03", - "developer": "Nexesenex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7941, - "hfopenllm_v2/BBH": 0.5308, - "hfopenllm_v2/MATH Level 5": 0.2221, - "hfopenllm_v2/GPQA": 0.3079, - "hfopenllm_v2/MUSR": 0.3959, - "hfopenllm_v2/MMLU-PRO": 0.3786 - } - }, - { - "id": "Nexesenex/Llama_3.1_8b_DodoWild_v2.10", - "name": "Llama_3.1_8b_DodoWild_v2.10", - "developer": "Nexesenex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8054, - "hfopenllm_v2/BBH": 0.5278, - "hfopenllm_v2/MATH Level 5": 0.1971, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.4157, - "hfopenllm_v2/MMLU-PRO": 0.3855 - } - }, - { - "id": "Nexesenex/Llama_3.1_8b_Dolermed_R1_V1.01", - "name": "Llama_3.1_8b_Dolermed_R1_V1.01", - "developer": "Nexesenex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7534, - "hfopenllm_v2/BBH": 0.5312, - "hfopenllm_v2/MATH Level 5": 0.2017, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.3747, - "hfopenllm_v2/MMLU-PRO": 0.3733 - } - }, - { - "id": "Nexesenex/Llama_3.1_8b_Dolermed_R1_V1.03", - "name": "Llama_3.1_8b_Dolermed_R1_V1.03", - "developer": "Nexesenex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7564, - "hfopenllm_v2/BBH": 0.5316, - "hfopenllm_v2/MATH Level 5": 0.2092, - "hfopenllm_v2/GPQA": 0.318, - "hfopenllm_v2/MUSR": 0.38, - "hfopenllm_v2/MMLU-PRO": 0.372 - } - }, - { - "id": "Nexesenex/Llama_3.1_8b_Dolermed_V1.01", - "name": "Llama_3.1_8b_Dolermed_V1.01", - "developer": "Nexesenex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5087, - "hfopenllm_v2/BBH": 0.5194, - "hfopenllm_v2/MATH Level 5": 0.1344, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.3945, - "hfopenllm_v2/MMLU-PRO": 0.357 - } - }, - { - "id": "Nexesenex/Llama_3.1_8b_Dolerstormed_V1.04", - "name": "Llama_3.1_8b_Dolerstormed_V1.04", - "developer": "Nexesenex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7889, - "hfopenllm_v2/BBH": 0.5195, - "hfopenllm_v2/MATH Level 5": 0.1926, - "hfopenllm_v2/GPQA": 0.3221, - "hfopenllm_v2/MUSR": 0.403, - "hfopenllm_v2/MMLU-PRO": 0.3889 - } - }, - { - "id": "Nexesenex/Llama_3.1_8b_Hermedash_R1_V1.04", - "name": "Llama_3.1_8b_Hermedash_R1_V1.04", - "developer": "Nexesenex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7872, - "hfopenllm_v2/BBH": 0.5192, - "hfopenllm_v2/MATH Level 5": 0.1866, - "hfopenllm_v2/GPQA": 0.323, - "hfopenllm_v2/MUSR": 0.4111, - "hfopenllm_v2/MMLU-PRO": 0.3882 - } - }, - { - "id": "Nexesenex/Llama_3.1_8b_Hermedive_R1_V1.01", - "name": "Llama_3.1_8b_Hermedive_R1_V1.01", - "developer": "Nexesenex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5001, - "hfopenllm_v2/BBH": 0.5171, - "hfopenllm_v2/MATH Level 5": 0.1775, - "hfopenllm_v2/GPQA": 0.2827, - "hfopenllm_v2/MUSR": 0.4008, - "hfopenllm_v2/MMLU-PRO": 0.3427 - } - }, - { - "id": "Nexesenex/Llama_3.1_8b_Hermedive_R1_V1.03", - "name": "Llama_3.1_8b_Hermedive_R1_V1.03", - "developer": "Nexesenex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6648, - "hfopenllm_v2/BBH": 0.5141, - "hfopenllm_v2/MATH Level 5": 0.1858, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.3613, - "hfopenllm_v2/MMLU-PRO": 0.3488 - } - }, - { - "id": "Nexesenex/Llama_3.1_8b_Hermedive_V1.01", - "name": "Llama_3.1_8b_Hermedive_V1.01", - "developer": "Nexesenex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5062, - "hfopenllm_v2/BBH": 0.4918, - "hfopenllm_v2/MATH Level 5": 0.1647, - "hfopenllm_v2/GPQA": 0.2894, - "hfopenllm_v2/MUSR": 0.3697, - "hfopenllm_v2/MMLU-PRO": 0.3551 - } - }, - { - "id": "Nexesenex/Llama_3.1_8b_Mediver_V1.01", - "name": "Llama_3.1_8b_Mediver_V1.01", - "developer": "Nexesenex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1885, - "hfopenllm_v2/BBH": 0.4415, - "hfopenllm_v2/MATH Level 5": 0.0015, - "hfopenllm_v2/GPQA": 0.2777, - "hfopenllm_v2/MUSR": 0.3898, - "hfopenllm_v2/MMLU-PRO": 0.2994 - } - }, - { - "id": "Nexesenex/Llama_3.1_8b_Medusa_v1.01", - "name": "Llama_3.1_8b_Medusa_v1.01", - "developer": "Nexesenex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7685, - "hfopenllm_v2/BBH": 0.5018, - "hfopenllm_v2/MATH Level 5": 0.1465, - "hfopenllm_v2/GPQA": 0.2919, - "hfopenllm_v2/MUSR": 0.4067, - "hfopenllm_v2/MMLU-PRO": 0.3531 - } - }, - { - "id": "Nexesenex/Llama_3.1_8b_Smarteaz_0.2_R1", - "name": "Llama_3.1_8b_Smarteaz_0.2_R1", - "developer": "Nexesenex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6346, - "hfopenllm_v2/BBH": 0.5113, - "hfopenllm_v2/MATH Level 5": 0.2606, - "hfopenllm_v2/GPQA": 0.3003, - "hfopenllm_v2/MUSR": 0.4188, - "hfopenllm_v2/MMLU-PRO": 0.3645 - } - }, - { - "id": "Nexesenex/Llama_3.1_8b_Smarteaz_V1.01", - "name": "Llama_3.1_8b_Smarteaz_V1.01", - "developer": "Nexesenex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8151, - "hfopenllm_v2/BBH": 0.5241, - "hfopenllm_v2/MATH Level 5": 0.2341, - "hfopenllm_v2/GPQA": 0.3096, - "hfopenllm_v2/MUSR": 0.3789, - "hfopenllm_v2/MMLU-PRO": 0.3736 - } - }, - { - "id": "Nexesenex/Llama_3.1_8b_Stormeder_v1.04", - "name": "Llama_3.1_8b_Stormeder_v1.04", - "developer": "Nexesenex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7853, - "hfopenllm_v2/BBH": 0.5207, - "hfopenllm_v2/MATH Level 5": 0.185, - "hfopenllm_v2/GPQA": 0.3205, - "hfopenllm_v2/MUSR": 0.3949, - "hfopenllm_v2/MMLU-PRO": 0.3852 - } - }, - { - "id": "Nexesenex/Llama_3.1_8b_Typhoon_v1.03", - "name": "Llama_3.1_8b_Typhoon_v1.03", - "developer": "Nexesenex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8078, - "hfopenllm_v2/BBH": 0.5314, - "hfopenllm_v2/MATH Level 5": 0.2273, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.3815, - "hfopenllm_v2/MMLU-PRO": 0.3842 - } - }, - { - "id": "Nexesenex/Llama_3.2_1b_AquaSyn_0.1", - "name": "Llama_3.2_1b_AquaSyn_0.1", - "developer": "Nexesenex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2741, - "hfopenllm_v2/BBH": 0.3284, - "hfopenllm_v2/MATH Level 5": 0.0219, - "hfopenllm_v2/GPQA": 0.2483, - "hfopenllm_v2/MUSR": 0.346, - "hfopenllm_v2/MMLU-PRO": 0.1378 - } - }, - { - "id": "Nexesenex/Llama_3.2_1b_AquaSyn_0.11", - "name": "Llama_3.2_1b_AquaSyn_0.11", - "developer": "Nexesenex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2431, - "hfopenllm_v2/BBH": 0.3112, - "hfopenllm_v2/MATH Level 5": 0.0234, - "hfopenllm_v2/GPQA": 0.2651, - "hfopenllm_v2/MUSR": 0.3368, - "hfopenllm_v2/MMLU-PRO": 0.1116 - } - }, - { - "id": "Nexesenex/Llama_3.2_1b_Dolto_0.1", - "name": "Llama_3.2_1b_Dolto_0.1", - "developer": "Nexesenex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5434, - "hfopenllm_v2/BBH": 0.335, - "hfopenllm_v2/MATH Level 5": 0.037, - "hfopenllm_v2/GPQA": 0.2374, - "hfopenllm_v2/MUSR": 0.3421, - "hfopenllm_v2/MMLU-PRO": 0.1364 - } - }, - { - "id": "Nexesenex/Llama_3.2_1b_Odyssea_V1", - "name": "Llama_3.2_1b_Odyssea_V1", - "developer": "Nexesenex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2553, - "hfopenllm_v2/BBH": 0.301, - "hfopenllm_v2/MATH Level 5": 0.0144, - "hfopenllm_v2/GPQA": 0.2584, - "hfopenllm_v2/MUSR": 0.3394, - "hfopenllm_v2/MMLU-PRO": 0.1153 - } - }, - { - "id": "Nexesenex/Llama_3.2_1b_Odyssea_V1.01", - "name": "Llama_3.2_1b_Odyssea_V1.01", - "developer": "Nexesenex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2495, - "hfopenllm_v2/BBH": 0.3045, - "hfopenllm_v2/MATH Level 5": 0.0174, - "hfopenllm_v2/GPQA": 0.2559, - "hfopenllm_v2/MUSR": 0.342, - "hfopenllm_v2/MMLU-PRO": 0.1152 - } - }, - { - "id": "Nexesenex/Llama_3.2_1b_OpenTree_R1_0.1", - "name": "Llama_3.2_1b_OpenTree_R1_0.1", - "developer": "Nexesenex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5366, - "hfopenllm_v2/BBH": 0.328, - "hfopenllm_v2/MATH Level 5": 0.0476, - "hfopenllm_v2/GPQA": 0.2525, - "hfopenllm_v2/MUSR": 0.3131, - "hfopenllm_v2/MMLU-PRO": 0.1675 - } - }, - { - "id": "Nexesenex/Llama_3.2_1b_OrcaSun_V1", - "name": "Llama_3.2_1b_OrcaSun_V1", - "developer": "Nexesenex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5949, - "hfopenllm_v2/BBH": 0.355, - "hfopenllm_v2/MATH Level 5": 0.0597, - "hfopenllm_v2/GPQA": 0.2366, - "hfopenllm_v2/MUSR": 0.338, - "hfopenllm_v2/MMLU-PRO": 0.1904 - } - }, - { - "id": "Nexesenex/Llama_3.2_1b_RandomLego_RP_R1_0.1", - "name": "Llama_3.2_1b_RandomLego_RP_R1_0.1", - "developer": "Nexesenex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5543, - "hfopenllm_v2/BBH": 0.3428, - "hfopenllm_v2/MATH Level 5": 0.0566, - "hfopenllm_v2/GPQA": 0.25, - "hfopenllm_v2/MUSR": 0.3249, - "hfopenllm_v2/MMLU-PRO": 0.1563 - } - }, - { - "id": "Nexesenex/Llama_3.2_1b_SunOrca_V1", - "name": "Llama_3.2_1b_SunOrca_V1", - "developer": "Nexesenex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.543, - "hfopenllm_v2/BBH": 0.3431, - "hfopenllm_v2/MATH Level 5": 0.0672, - "hfopenllm_v2/GPQA": 0.2743, - "hfopenllm_v2/MUSR": 0.3262, - "hfopenllm_v2/MMLU-PRO": 0.1884 - } - }, - { - "id": "Nexesenex/Llama_3.2_1b_Sydonia_0.1", - "name": "Llama_3.2_1b_Sydonia_0.1", - "developer": "Nexesenex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2197, - "hfopenllm_v2/BBH": 0.3121, - "hfopenllm_v2/MATH Level 5": 0.0204, - "hfopenllm_v2/GPQA": 0.2282, - "hfopenllm_v2/MUSR": 0.3382, - "hfopenllm_v2/MMLU-PRO": 0.1224 - } - }, - { - "id": "Nexesenex/Llama_3.2_1b_Syneridol_0.2", - "name": "Llama_3.2_1b_Syneridol_0.2", - "developer": "Nexesenex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2157, - "hfopenllm_v2/BBH": 0.3139, - "hfopenllm_v2/MATH Level 5": 0.0219, - "hfopenllm_v2/GPQA": 0.2349, - "hfopenllm_v2/MUSR": 0.3343, - "hfopenllm_v2/MMLU-PRO": 0.1227 - } - }, - { - "id": "Nexesenex/Llama_3.2_1b_Synopsys_0.1", - "name": "Llama_3.2_1b_Synopsys_0.1", - "developer": "Nexesenex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1764, - "hfopenllm_v2/BBH": 0.3162, - "hfopenllm_v2/MATH Level 5": 0.0166, - "hfopenllm_v2/GPQA": 0.2391, - "hfopenllm_v2/MUSR": 0.3461, - "hfopenllm_v2/MMLU-PRO": 0.1231 - } - }, - { - "id": "Nexesenex/Llama_3.2_1b_Synopsys_0.11", - "name": "Llama_3.2_1b_Synopsys_0.11", - "developer": "Nexesenex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2842, - "hfopenllm_v2/BBH": 0.3102, - "hfopenllm_v2/MATH Level 5": 0.0128, - "hfopenllm_v2/GPQA": 0.2626, - "hfopenllm_v2/MUSR": 0.3513, - "hfopenllm_v2/MMLU-PRO": 0.1123 - } - }, - { - "id": "Nexesenex/Llama_3.2_3b_Kermes_v1", - "name": "Llama_3.2_3b_Kermes_v1", - "developer": "Nexesenex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4852, - "hfopenllm_v2/BBH": 0.441, - "hfopenllm_v2/MATH Level 5": 0.031, - "hfopenllm_v2/GPQA": 0.2735, - "hfopenllm_v2/MUSR": 0.407, - "hfopenllm_v2/MMLU-PRO": 0.2547 - } - }, - { - "id": "Nexesenex/Llama_3.2_3b_Kermes_v2", - "name": "Llama_3.2_3b_Kermes_v2", - "developer": "Nexesenex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5754, - "hfopenllm_v2/BBH": 0.4455, - "hfopenllm_v2/MATH Level 5": 0.0544, - "hfopenllm_v2/GPQA": 0.2651, - "hfopenllm_v2/MUSR": 0.3778, - "hfopenllm_v2/MMLU-PRO": 0.2734 - } - }, - { - "id": "Nexesenex/Llama_3.2_3b_Kermes_v2.1", - "name": "Llama_3.2_3b_Kermes_v2.1", - "developer": "Nexesenex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5584, - "hfopenllm_v2/BBH": 0.4464, - "hfopenllm_v2/MATH Level 5": 0.0521, - "hfopenllm_v2/GPQA": 0.2794, - "hfopenllm_v2/MUSR": 0.3964, - "hfopenllm_v2/MMLU-PRO": 0.2692 - } - }, - { - "id": "Nexesenex/Nemotron_W_4b_Halo_0.1", - "name": "Nemotron_W_4b_Halo_0.1", - "developer": "Nexesenex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3627, - "hfopenllm_v2/BBH": 0.4135, - "hfopenllm_v2/MATH Level 5": 0.0423, - "hfopenllm_v2/GPQA": 0.2802, - "hfopenllm_v2/MUSR": 0.4165, - "hfopenllm_v2/MMLU-PRO": 0.2505 - } - }, - { - "id": "Nexesenex/Nemotron_W_4b_MagLight_0.1", - "name": "Nemotron_W_4b_MagLight_0.1", - "developer": "Nexesenex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.423, - "hfopenllm_v2/BBH": 0.4231, - "hfopenllm_v2/MATH Level 5": 0.04, - "hfopenllm_v2/GPQA": 0.2836, - "hfopenllm_v2/MUSR": 0.4112, - "hfopenllm_v2/MMLU-PRO": 0.2545 - } - }, - { - "id": "Nexesenex/pankajmathur_orca_mini_v9_6_1B-instruct-Abliterated-LPL", - "name": "pankajmathur_orca_mini_v9_6_1B-instruct-Abliterated-LPL", - "developer": "Nexesenex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.589, - "hfopenllm_v2/BBH": 0.3562, - "hfopenllm_v2/MATH Level 5": 0.0748, - "hfopenllm_v2/GPQA": 0.2668, - "hfopenllm_v2/MUSR": 0.3396, - "hfopenllm_v2/MMLU-PRO": 0.1803 - } - }, - { - "id": "Nexesenex/Qwen_2.5_3b_Smarteaz_0.01a", - "name": "Qwen_2.5_3b_Smarteaz_0.01a", - "developer": "Nexesenex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4012, - "hfopenllm_v2/BBH": 0.4637, - "hfopenllm_v2/MATH Level 5": 0.1805, - "hfopenllm_v2/GPQA": 0.2777, - "hfopenllm_v2/MUSR": 0.432, - "hfopenllm_v2/MMLU-PRO": 0.286 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/nexusflow.json b/data/developers/nexusflow.json deleted file mode 100644 index 2f78cab8780e463261ce35ef251c295eb3f3fd0f..0000000000000000000000000000000000000000 --- a/data/developers/nexusflow.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "developer": "Nexusflow", - "models": [ - { - "id": "Nexusflow/NexusRaven-V2-13B", - "name": "NexusRaven-V2-13B", - "developer": "Nexusflow", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1791, - "hfopenllm_v2/BBH": 0.3949, - "hfopenllm_v2/MATH Level 5": 0.0295, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3737, - "hfopenllm_v2/MMLU-PRO": 0.1872 - } - }, - { - "id": "Nexusflow/Starling-RM-34B", - "name": "Nexusflow/Starling-RM-34B", - "developer": "Nexusflow", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.8133, - "reward-bench/Factuality": 0.4589, - "reward-bench/Precise IF": 0.3187, - "reward-bench/Math": 0.6175, - "reward-bench/Safety": 0.877, - "reward-bench/Focus": 0.4808, - "reward-bench/Ties": 0.1004, - "reward-bench/Chat": 0.9693, - "reward-bench/Chat Hard": 0.5724, - "reward-bench/Reasoning": 0.8845, - "reward-bench/Prior Sets (0.5 weight)": 0.7137 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/nguyentd.json b/data/developers/nguyentd.json deleted file mode 100644 index c3f968c4e67d3da8e4b0f3979310f7e550f561b6..0000000000000000000000000000000000000000 --- a/data/developers/nguyentd.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "nguyentd", - "models": [ - { - "id": "nguyentd/FinancialAdvice-Qwen2.5-7B", - "name": "FinancialAdvice-Qwen2.5-7B", - "developer": "nguyentd", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4496, - "hfopenllm_v2/BBH": 0.4731, - "hfopenllm_v2/MATH Level 5": 0.1148, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.4025, - "hfopenllm_v2/MMLU-PRO": 0.3752 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/ngxson.json b/data/developers/ngxson.json deleted file mode 100644 index 0eb48a0ae55dd8730e6e1ebbdc813d2849fd5a70..0000000000000000000000000000000000000000 --- a/data/developers/ngxson.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "ngxson", - "models": [ - { - "id": "ngxson/MiniThinky-1B-Llama-3.2", - "name": "MiniThinky-1B-Llama-3.2", - "developer": "ngxson", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2771, - "hfopenllm_v2/BBH": 0.3142, - "hfopenllm_v2/MATH Level 5": 0.0574, - "hfopenllm_v2/GPQA": 0.2391, - "hfopenllm_v2/MUSR": 0.3434, - "hfopenllm_v2/MMLU-PRO": 0.1147 - } - }, - { - "id": "ngxson/MiniThinky-v2-1B-Llama-3.2", - "name": "MiniThinky-v2-1B-Llama-3.2", - "developer": "ngxson", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2963, - "hfopenllm_v2/BBH": 0.3205, - "hfopenllm_v2/MATH Level 5": 0.0287, - "hfopenllm_v2/GPQA": 0.2399, - "hfopenllm_v2/MUSR": 0.3356, - "hfopenllm_v2/MMLU-PRO": 0.1116 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/nhyha.json b/data/developers/nhyha.json deleted file mode 100644 index 355703dc1e438ed8a6bdcbfc10704a3020a5772f..0000000000000000000000000000000000000000 --- a/data/developers/nhyha.json +++ /dev/null @@ -1,75 +0,0 @@ -{ - "developer": "nhyha", - "models": [ - { - "id": "nhyha/merge_Qwen2.5-7B-Instruct_20241023_0314", - "name": "merge_Qwen2.5-7B-Instruct_20241023_0314", - "developer": "nhyha", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5695, - "hfopenllm_v2/BBH": 0.5559, - "hfopenllm_v2/MATH Level 5": 0.3542, - "hfopenllm_v2/GPQA": 0.3213, - "hfopenllm_v2/MUSR": 0.4251, - "hfopenllm_v2/MMLU-PRO": 0.4542 - } - }, - { - "id": "nhyha/N3N_Delirium-v1_1030_0227", - "name": "N3N_Delirium-v1_1030_0227", - "developer": "nhyha", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8023, - "hfopenllm_v2/BBH": 0.5891, - "hfopenllm_v2/MATH Level 5": 0.2107, - "hfopenllm_v2/GPQA": 0.3372, - "hfopenllm_v2/MUSR": 0.4098, - "hfopenllm_v2/MMLU-PRO": 0.415 - } - }, - { - "id": "nhyha/N3N_gemma-2-9b-it_20241029_1532", - "name": "N3N_gemma-2-9b-it_20241029_1532", - "developer": "nhyha", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6752, - "hfopenllm_v2/BBH": 0.5863, - "hfopenllm_v2/MATH Level 5": 0.2122, - "hfopenllm_v2/GPQA": 0.3406, - "hfopenllm_v2/MUSR": 0.4594, - "hfopenllm_v2/MMLU-PRO": 0.4122 - } - }, - { - "id": "nhyha/N3N_gemma-2-9b-it_20241110_2026", - "name": "N3N_gemma-2-9b-it_20241110_2026", - "developer": "nhyha", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6283, - "hfopenllm_v2/BBH": 0.5867, - "hfopenllm_v2/MATH Level 5": 0.1609, - "hfopenllm_v2/GPQA": 0.3364, - "hfopenllm_v2/MUSR": 0.4073, - "hfopenllm_v2/MMLU-PRO": 0.402 - } - }, - { - "id": "nhyha/N3N_Llama-3.1-8B-Instruct_1028_0216", - "name": "N3N_Llama-3.1-8B-Instruct_1028_0216", - "developer": "nhyha", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4796, - "hfopenllm_v2/BBH": 0.5054, - "hfopenllm_v2/MATH Level 5": 0.1707, - "hfopenllm_v2/GPQA": 0.3062, - "hfopenllm_v2/MUSR": 0.405, - "hfopenllm_v2/MMLU-PRO": 0.3638 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/nicolinho.json b/data/developers/nicolinho.json deleted file mode 100644 index bf9706fe4ed328188b0945860efeebf7183abc65..0000000000000000000000000000000000000000 --- a/data/developers/nicolinho.json +++ /dev/null @@ -1,67 +0,0 @@ -{ - "developer": "nicolinho", - "models": [ - { - "id": "nicolinho/QRM-Gemma-2-27B", - "name": "nicolinho/QRM-Gemma-2-27B", - "developer": "nicolinho", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.9444, - "reward-bench/Factuality": 0.7853, - "reward-bench/Precise IF": 0.3719, - "reward-bench/Math": 0.6995, - "reward-bench/Safety": 0.927, - "reward-bench/Focus": 0.9535, - "reward-bench/Ties": 0.8321, - "reward-bench/Chat": 0.9665, - "reward-bench/Chat Hard": 0.9013, - "reward-bench/Reasoning": 0.9826 - } - }, - { - "id": "nicolinho/QRM-Llama3-8B", - "name": "nicolinho/QRM-Llama3-8B", - "developer": "nicolinho", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.911, - "reward-bench/Chat": 0.9581, - "reward-bench/Chat Hard": 0.8114, - "reward-bench/Safety": 0.8986, - "reward-bench/Reasoning": 0.9758 - } - }, - { - "id": "nicolinho/QRM-Llama3.1-8B", - "name": "nicolinho/QRM-Llama3.1-8B", - "developer": "nicolinho", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.9306, - "reward-bench/Chat": 0.9441, - "reward-bench/Chat Hard": 0.8969, - "reward-bench/Safety": 0.923, - "reward-bench/Reasoning": 0.9583 - } - }, - { - "id": "nicolinho/QRM-Llama3.1-8B-v2", - "name": "nicolinho/QRM-Llama3.1-8B-v2", - "developer": "nicolinho", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7074, - "reward-bench/Chat": 0.9637, - "reward-bench/Chat Hard": 0.8684, - "reward-bench/Safety": 0.9467, - "reward-bench/Reasoning": 0.9677, - "reward-bench/Factuality": 0.6653, - "reward-bench/Precise IF": 0.4062, - "reward-bench/Math": 0.612, - "reward-bench/Focus": 0.8909, - "reward-bench/Ties": 0.7234 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/nidum.json b/data/developers/nidum.json deleted file mode 100644 index af8e93bb5f8a01abfc8ee7d10496e70d2cff68b5..0000000000000000000000000000000000000000 --- a/data/developers/nidum.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "nidum", - "models": [ - { - "id": "nidum/Nidum-Limitless-Gemma-2B", - "name": "Nidum-Limitless-Gemma-2B", - "developer": "nidum", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2424, - "hfopenllm_v2/BBH": 0.3079, - "hfopenllm_v2/MATH Level 5": 0.0136, - "hfopenllm_v2/GPQA": 0.2643, - "hfopenllm_v2/MUSR": 0.374, - "hfopenllm_v2/MMLU-PRO": 0.1174 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/nikolasigmoid.json b/data/developers/nikolasigmoid.json deleted file mode 100644 index 03d6da9ff3ca1cbfea8bdda1e4d0e80ecd721a42..0000000000000000000000000000000000000000 --- a/data/developers/nikolasigmoid.json +++ /dev/null @@ -1,103 +0,0 @@ -{ - "developer": "NikolaSigmoid", - "models": [ - { - "id": "NikolaSigmoid/AceMath-1.5B-Instruct-1epoch", - "name": "AceMath-1.5B-Instruct-1epoch", - "developer": "NikolaSigmoid", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2849, - "hfopenllm_v2/BBH": 0.4263, - "hfopenllm_v2/MATH Level 5": 0.3051, - "hfopenllm_v2/GPQA": 0.2777, - "hfopenllm_v2/MUSR": 0.3925, - "hfopenllm_v2/MMLU-PRO": 0.2376 - } - }, - { - "id": "NikolaSigmoid/AceMath-1.5B-Instruct-dolphin-r1-200", - "name": "AceMath-1.5B-Instruct-dolphin-r1-200", - "developer": "NikolaSigmoid", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1808, - "hfopenllm_v2/BBH": 0.2815, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2559, - "hfopenllm_v2/MUSR": 0.375, - "hfopenllm_v2/MMLU-PRO": 0.1143 - } - }, - { - "id": "NikolaSigmoid/acemath-200", - "name": "acemath-200", - "developer": "NikolaSigmoid", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2849, - "hfopenllm_v2/BBH": 0.4263, - "hfopenllm_v2/MATH Level 5": 0.3051, - "hfopenllm_v2/GPQA": 0.2777, - "hfopenllm_v2/MUSR": 0.3925, - "hfopenllm_v2/MMLU-PRO": 0.2376 - } - }, - { - "id": "NikolaSigmoid/DeepSeek-R1-Distill-Qwen-1.5B-500", - "name": "DeepSeek-R1-Distill-Qwen-1.5B-500", - "developer": "NikolaSigmoid", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1749, - "hfopenllm_v2/BBH": 0.2602, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2458, - "hfopenllm_v2/MUSR": 0.338, - "hfopenllm_v2/MMLU-PRO": 0.1125 - } - }, - { - "id": "NikolaSigmoid/phi-4-14b", - "name": "phi-4-14b", - "developer": "NikolaSigmoid", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0561, - "hfopenllm_v2/BBH": 0.6695, - "hfopenllm_v2/MATH Level 5": 0.2938, - "hfopenllm_v2/GPQA": 0.4035, - "hfopenllm_v2/MUSR": 0.5047, - "hfopenllm_v2/MMLU-PRO": 0.5278 - } - }, - { - "id": "NikolaSigmoid/phi-4-1steps", - "name": "phi-4-1steps", - "developer": "NikolaSigmoid", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0528, - "hfopenllm_v2/BBH": 0.6707, - "hfopenllm_v2/MATH Level 5": 0.2983, - "hfopenllm_v2/GPQA": 0.4018, - "hfopenllm_v2/MUSR": 0.5021, - "hfopenllm_v2/MMLU-PRO": 0.5273 - } - }, - { - "id": "NikolaSigmoid/phi-4-300steps", - "name": "phi-4-300steps", - "developer": "NikolaSigmoid", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0561, - "hfopenllm_v2/BBH": 0.6701, - "hfopenllm_v2/MATH Level 5": 0.2946, - "hfopenllm_v2/GPQA": 0.4052, - "hfopenllm_v2/MUSR": 0.5034, - "hfopenllm_v2/MMLU-PRO": 0.5288 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/nisten.json b/data/developers/nisten.json deleted file mode 100644 index 7b275a3c64fa267662b1c7ec09c2c6db9c0fbfc6..0000000000000000000000000000000000000000 --- a/data/developers/nisten.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "nisten", - "models": [ - { - "id": "nisten/franqwenstein-35b", - "name": "franqwenstein-35b", - "developer": "nisten", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3914, - "hfopenllm_v2/BBH": 0.6591, - "hfopenllm_v2/MATH Level 5": 0.3044, - "hfopenllm_v2/GPQA": 0.3591, - "hfopenllm_v2/MUSR": 0.4681, - "hfopenllm_v2/MMLU-PRO": 0.5611 - } - }, - { - "id": "nisten/tqwendo-36b", - "name": "tqwendo-36b", - "developer": "nisten", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6778, - "hfopenllm_v2/BBH": 0.6432, - "hfopenllm_v2/MATH Level 5": 0.4154, - "hfopenllm_v2/GPQA": 0.3314, - "hfopenllm_v2/MUSR": 0.443, - "hfopenllm_v2/MMLU-PRO": 0.4381 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/nitral-ai.json b/data/developers/nitral-ai.json deleted file mode 100644 index c059df98d4988906f6fc0b24fb49a31144b6a067..0000000000000000000000000000000000000000 --- a/data/developers/nitral-ai.json +++ /dev/null @@ -1,117 +0,0 @@ -{ - "developer": "Nitral-AI", - "models": [ - { - "id": "Nitral-AI/Captain-Eris-BMO_Violent-GRPO-v0.420", - "name": "Captain-Eris-BMO_Violent-GRPO-v0.420", - "developer": "Nitral-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6313, - "hfopenllm_v2/BBH": 0.5079, - "hfopenllm_v2/MATH Level 5": 0.1314, - "hfopenllm_v2/GPQA": 0.3096, - "hfopenllm_v2/MUSR": 0.4228, - "hfopenllm_v2/MMLU-PRO": 0.3596 - } - }, - { - "id": "Nitral-AI/Captain-Eris_BMO-Violent-12B", - "name": "Captain-Eris_BMO-Violent-12B", - "developer": "Nitral-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6152, - "hfopenllm_v2/BBH": 0.5104, - "hfopenllm_v2/MATH Level 5": 0.1367, - "hfopenllm_v2/GPQA": 0.3096, - "hfopenllm_v2/MUSR": 0.4255, - "hfopenllm_v2/MMLU-PRO": 0.3571 - } - }, - { - "id": "Nitral-AI/Captain-Eris_Violet-GRPO-v0.420", - "name": "Captain-Eris_Violet-GRPO-v0.420", - "developer": "Nitral-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6262, - "hfopenllm_v2/BBH": 0.5159, - "hfopenllm_v2/MATH Level 5": 0.108, - "hfopenllm_v2/GPQA": 0.2987, - "hfopenllm_v2/MUSR": 0.4279, - "hfopenllm_v2/MMLU-PRO": 0.3535 - } - }, - { - "id": "Nitral-AI/Captain-Eris_Violet-V0.420-12B", - "name": "Captain-Eris_Violet-V0.420-12B", - "developer": "Nitral-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4339, - "hfopenllm_v2/BBH": 0.5478, - "hfopenllm_v2/MATH Level 5": 0.1073, - "hfopenllm_v2/GPQA": 0.3112, - "hfopenllm_v2/MUSR": 0.4331, - "hfopenllm_v2/MMLU-PRO": 0.3723 - } - }, - { - "id": "Nitral-AI/Captain_BMO-12B", - "name": "Captain_BMO-12B", - "developer": "Nitral-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4751, - "hfopenllm_v2/BBH": 0.5286, - "hfopenllm_v2/MATH Level 5": 0.1397, - "hfopenllm_v2/GPQA": 0.3196, - "hfopenllm_v2/MUSR": 0.3748, - "hfopenllm_v2/MMLU-PRO": 0.3569 - } - }, - { - "id": "Nitral-AI/Hathor_Stable-v0.2-L3-8B", - "name": "Hathor_Stable-v0.2-L3-8B", - "developer": "Nitral-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7175, - "hfopenllm_v2/BBH": 0.5286, - "hfopenllm_v2/MATH Level 5": 0.105, - "hfopenllm_v2/GPQA": 0.2869, - "hfopenllm_v2/MUSR": 0.3781, - "hfopenllm_v2/MMLU-PRO": 0.3696 - } - }, - { - "id": "Nitral-AI/Hathor_Tahsin-L3-8B-v0.85", - "name": "Hathor_Tahsin-L3-8B-v0.85", - "developer": "Nitral-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.711, - "hfopenllm_v2/BBH": 0.5279, - "hfopenllm_v2/MATH Level 5": 0.1005, - "hfopenllm_v2/GPQA": 0.2852, - "hfopenllm_v2/MUSR": 0.3647, - "hfopenllm_v2/MMLU-PRO": 0.372 - } - }, - { - "id": "Nitral-AI/Nera_Noctis-12B", - "name": "Nera_Noctis-12B", - "developer": "Nitral-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4562, - "hfopenllm_v2/BBH": 0.5194, - "hfopenllm_v2/MATH Level 5": 0.0876, - "hfopenllm_v2/GPQA": 0.2634, - "hfopenllm_v2/MUSR": 0.3979, - "hfopenllm_v2/MMLU-PRO": 0.3468 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/njs26.json b/data/developers/njs26.json deleted file mode 100644 index c8c02a590067afd0a7615bfcb7a0d012a7ca1ad9..0000000000000000000000000000000000000000 --- a/data/developers/njs26.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "NJS26", - "models": [ - { - "id": "NJS26/NJS_777", - "name": "NJS_777", - "developer": "NJS26", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1881, - "hfopenllm_v2/BBH": 0.2178, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2064, - "hfopenllm_v2/MUSR": 0.3538, - "hfopenllm_v2/MMLU-PRO": 0.1163 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/nlpark.json b/data/developers/nlpark.json deleted file mode 100644 index 75347595685858a623f9524c45e52afaf614b395..0000000000000000000000000000000000000000 --- a/data/developers/nlpark.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "developer": "NLPark", - "models": [ - { - "id": "NLPark/AnFeng_v3.1-Avocet", - "name": "AnFeng_v3.1-Avocet", - "developer": "NLPark", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5096, - "hfopenllm_v2/BBH": 0.5829, - "hfopenllm_v2/MATH Level 5": 0.1594, - "hfopenllm_v2/GPQA": 0.3247, - "hfopenllm_v2/MUSR": 0.4476, - "hfopenllm_v2/MMLU-PRO": 0.4438 - } - }, - { - "id": "NLPark/B-and-W_Flycatcher-3AD1E", - "name": "B-and-W_Flycatcher-3AD1E", - "developer": "NLPark", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4908, - "hfopenllm_v2/BBH": 0.6065, - "hfopenllm_v2/MATH Level 5": 0.2379, - "hfopenllm_v2/GPQA": 0.3305, - "hfopenllm_v2/MUSR": 0.4423, - "hfopenllm_v2/MMLU-PRO": 0.4741 - } - }, - { - "id": "NLPark/Shi-Ci-Robin-Test_3AD80", - "name": "Shi-Ci-Robin-Test_3AD80", - "developer": "NLPark", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7227, - "hfopenllm_v2/BBH": 0.6705, - "hfopenllm_v2/MATH Level 5": 0.3157, - "hfopenllm_v2/GPQA": 0.3599, - "hfopenllm_v2/MUSR": 0.4696, - "hfopenllm_v2/MMLU-PRO": 0.5121 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/nlpguy.json b/data/developers/nlpguy.json deleted file mode 100644 index 53da7ebbcfa5579e01d9df387a8a0c56ca3eec5d..0000000000000000000000000000000000000000 --- a/data/developers/nlpguy.json +++ /dev/null @@ -1,131 +0,0 @@ -{ - "developer": "nlpguy", - "models": [ - { - "id": "nlpguy/Lion-Lamarck-v.1.0.8", - "name": "Lion-Lamarck-v.1.0.8", - "developer": "nlpguy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4509, - "hfopenllm_v2/BBH": 0.5869, - "hfopenllm_v2/MATH Level 5": 0.5544, - "hfopenllm_v2/GPQA": 0.3582, - "hfopenllm_v2/MUSR": 0.4673, - "hfopenllm_v2/MMLU-PRO": 0.4643 - } - }, - { - "id": "nlpguy/Lion-Lamarck-v.1.0.9", - "name": "Lion-Lamarck-v.1.0.9", - "developer": "nlpguy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3409, - "hfopenllm_v2/BBH": 0.5918, - "hfopenllm_v2/MATH Level 5": 0.5642, - "hfopenllm_v2/GPQA": 0.3901, - "hfopenllm_v2/MUSR": 0.53, - "hfopenllm_v2/MMLU-PRO": 0.4704 - } - }, - { - "id": "nlpguy/Lion-Lamarck-v.1.1.0", - "name": "Lion-Lamarck-v.1.1.0", - "developer": "nlpguy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3658, - "hfopenllm_v2/BBH": 0.5962, - "hfopenllm_v2/MATH Level 5": 0.5755, - "hfopenllm_v2/GPQA": 0.3926, - "hfopenllm_v2/MUSR": 0.5325, - "hfopenllm_v2/MMLU-PRO": 0.4631 - } - }, - { - "id": "nlpguy/Miisce-one", - "name": "Miisce-one", - "developer": "nlpguy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6066, - "hfopenllm_v2/BBH": 0.6505, - "hfopenllm_v2/MATH Level 5": 0.4169, - "hfopenllm_v2/GPQA": 0.3859, - "hfopenllm_v2/MUSR": 0.482, - "hfopenllm_v2/MMLU-PRO": 0.5412 - } - }, - { - "id": "nlpguy/Mistral-NeMo-Minitron-Upscale-v1", - "name": "Mistral-NeMo-Minitron-Upscale-v1", - "developer": "nlpguy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1648, - "hfopenllm_v2/BBH": 0.4468, - "hfopenllm_v2/MATH Level 5": 0.0144, - "hfopenllm_v2/GPQA": 0.2802, - "hfopenllm_v2/MUSR": 0.3804, - "hfopenllm_v2/MMLU-PRO": 0.2537 - } - }, - { - "id": "nlpguy/Mistral-NeMo-Minitron-Upscale-v2", - "name": "Mistral-NeMo-Minitron-Upscale-v2", - "developer": "nlpguy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1573, - "hfopenllm_v2/BBH": 0.395, - "hfopenllm_v2/MATH Level 5": 0.0128, - "hfopenllm_v2/GPQA": 0.2735, - "hfopenllm_v2/MUSR": 0.3791, - "hfopenllm_v2/MMLU-PRO": 0.1927 - } - }, - { - "id": "nlpguy/Mistral-NeMo-Minitron-Upscale-v3", - "name": "Mistral-NeMo-Minitron-Upscale-v3", - "developer": "nlpguy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1412, - "hfopenllm_v2/BBH": 0.3052, - "hfopenllm_v2/MATH Level 5": 0.0113, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.4098, - "hfopenllm_v2/MMLU-PRO": 0.1171 - } - }, - { - "id": "nlpguy/StableProse", - "name": "StableProse", - "developer": "nlpguy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1972, - "hfopenllm_v2/BBH": 0.5117, - "hfopenllm_v2/MATH Level 5": 0.065, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.4067, - "hfopenllm_v2/MMLU-PRO": 0.3468 - } - }, - { - "id": "nlpguy/StarFusion-alpha1", - "name": "StarFusion-alpha1", - "developer": "nlpguy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.566, - "hfopenllm_v2/BBH": 0.4429, - "hfopenllm_v2/MATH Level 5": 0.0718, - "hfopenllm_v2/GPQA": 0.2953, - "hfopenllm_v2/MUSR": 0.4081, - "hfopenllm_v2/MMLU-PRO": 0.3191 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/nohobby.json b/data/developers/nohobby.json deleted file mode 100644 index 1f97b72706347a4d3531ecbf4b11cc2c3082e4d6..0000000000000000000000000000000000000000 --- a/data/developers/nohobby.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "Nohobby", - "models": [ - { - "id": "Nohobby/MS-Schisandra-22B-v0.1", - "name": "MS-Schisandra-22B-v0.1", - "developer": "Nohobby", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6331, - "hfopenllm_v2/BBH": 0.579, - "hfopenllm_v2/MATH Level 5": 0.2228, - "hfopenllm_v2/GPQA": 0.3322, - "hfopenllm_v2/MUSR": 0.3928, - "hfopenllm_v2/MMLU-PRO": 0.4096 - } - }, - { - "id": "Nohobby/MS-Schisandra-22B-v0.2", - "name": "MS-Schisandra-22B-v0.2", - "developer": "Nohobby", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6383, - "hfopenllm_v2/BBH": 0.5841, - "hfopenllm_v2/MATH Level 5": 0.2032, - "hfopenllm_v2/GPQA": 0.3356, - "hfopenllm_v2/MUSR": 0.4075, - "hfopenllm_v2/MMLU-PRO": 0.4136 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/noname0202.json b/data/developers/noname0202.json deleted file mode 100644 index 3f3a769adee89af5fcfd9a54a3f12fcba4bb240f..0000000000000000000000000000000000000000 --- a/data/developers/noname0202.json +++ /dev/null @@ -1,117 +0,0 @@ -{ - "developer": "noname0202", - "models": [ - { - "id": "noname0202/gemma-2-2b-it-ties", - "name": "gemma-2-2b-it-ties", - "developer": "noname0202", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1266, - "hfopenllm_v2/BBH": 0.4206, - "hfopenllm_v2/MATH Level 5": 0.0242, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.3929, - "hfopenllm_v2/MMLU-PRO": 0.2561 - } - }, - { - "id": "noname0202/gemma-2-9b-sft-jp-en-zh-v1", - "name": "gemma-2-9b-sft-jp-en-zh-v1", - "developer": "noname0202", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2988, - "hfopenllm_v2/BBH": 0.4519, - "hfopenllm_v2/MATH Level 5": 0.0891, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.408, - "hfopenllm_v2/MMLU-PRO": 0.3125 - } - }, - { - "id": "noname0202/gemma-2-9b-sft-jp-en-zh-v2", - "name": "gemma-2-9b-sft-jp-en-zh-v2", - "developer": "noname0202", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3993, - "hfopenllm_v2/BBH": 0.4515, - "hfopenllm_v2/MATH Level 5": 0.1042, - "hfopenllm_v2/GPQA": 0.2878, - "hfopenllm_v2/MUSR": 0.3612, - "hfopenllm_v2/MMLU-PRO": 0.3675 - } - }, - { - "id": "noname0202/Llama-3.2-4x3B-Instruct", - "name": "Llama-3.2-4x3B-Instruct", - "developer": "noname0202", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7067, - "hfopenllm_v2/BBH": 0.4647, - "hfopenllm_v2/MATH Level 5": 0.1586, - "hfopenllm_v2/GPQA": 0.2727, - "hfopenllm_v2/MUSR": 0.3674, - "hfopenllm_v2/MMLU-PRO": 0.3285 - } - }, - { - "id": "noname0202/llama-math-1b-r16-0to512tokens-test", - "name": "llama-math-1b-r16-0to512tokens-test", - "developer": "noname0202", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.547, - "hfopenllm_v2/BBH": 0.3488, - "hfopenllm_v2/MATH Level 5": 0.0816, - "hfopenllm_v2/GPQA": 0.2668, - "hfopenllm_v2/MUSR": 0.3143, - "hfopenllm_v2/MMLU-PRO": 0.1728 - } - }, - { - "id": "noname0202/llama-math-1b-r32-0to512tokens-test", - "name": "llama-math-1b-r32-0to512tokens-test", - "developer": "noname0202", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5683, - "hfopenllm_v2/BBH": 0.3495, - "hfopenllm_v2/MATH Level 5": 0.0906, - "hfopenllm_v2/GPQA": 0.2651, - "hfopenllm_v2/MUSR": 0.3209, - "hfopenllm_v2/MMLU-PRO": 0.176 - } - }, - { - "id": "noname0202/llama-math-1b-r32-test", - "name": "llama-math-1b-r32-test", - "developer": "noname0202", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5819, - "hfopenllm_v2/BBH": 0.3486, - "hfopenllm_v2/MATH Level 5": 0.0725, - "hfopenllm_v2/GPQA": 0.2617, - "hfopenllm_v2/MUSR": 0.3156, - "hfopenllm_v2/MMLU-PRO": 0.1781 - } - }, - { - "id": "noname0202/llama-math-1b-r8-512tokens-test", - "name": "llama-math-1b-r8-512tokens-test", - "developer": "noname0202", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5792, - "hfopenllm_v2/BBH": 0.3496, - "hfopenllm_v2/MATH Level 5": 0.0816, - "hfopenllm_v2/GPQA": 0.2685, - "hfopenllm_v2/MUSR": 0.3169, - "hfopenllm_v2/MMLU-PRO": 0.1753 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/norquinal.json b/data/developers/norquinal.json deleted file mode 100644 index cce1ebee228114767c2a6061ae2d77857f5702c6..0000000000000000000000000000000000000000 --- a/data/developers/norquinal.json +++ /dev/null @@ -1,117 +0,0 @@ -{ - "developer": "Norquinal", - "models": [ - { - "id": "Norquinal/Alpha", - "name": "Alpha", - "developer": "Norquinal", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2803, - "hfopenllm_v2/BBH": 0.3374, - "hfopenllm_v2/MATH Level 5": 0.0574, - "hfopenllm_v2/GPQA": 0.2651, - "hfopenllm_v2/MUSR": 0.3631, - "hfopenllm_v2/MMLU-PRO": 0.3003 - } - }, - { - "id": "Norquinal/Bravo", - "name": "Bravo", - "developer": "Norquinal", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3025, - "hfopenllm_v2/BBH": 0.3558, - "hfopenllm_v2/MATH Level 5": 0.0574, - "hfopenllm_v2/GPQA": 0.2819, - "hfopenllm_v2/MUSR": 0.3869, - "hfopenllm_v2/MMLU-PRO": 0.3127 - } - }, - { - "id": "Norquinal/Charlie", - "name": "Charlie", - "developer": "Norquinal", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3061, - "hfopenllm_v2/BBH": 0.3515, - "hfopenllm_v2/MATH Level 5": 0.0582, - "hfopenllm_v2/GPQA": 0.271, - "hfopenllm_v2/MUSR": 0.3737, - "hfopenllm_v2/MMLU-PRO": 0.3093 - } - }, - { - "id": "Norquinal/Delta", - "name": "Delta", - "developer": "Norquinal", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2538, - "hfopenllm_v2/BBH": 0.3435, - "hfopenllm_v2/MATH Level 5": 0.0612, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.3777, - "hfopenllm_v2/MMLU-PRO": 0.2959 - } - }, - { - "id": "Norquinal/Echo", - "name": "Echo", - "developer": "Norquinal", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3158, - "hfopenllm_v2/BBH": 0.353, - "hfopenllm_v2/MATH Level 5": 0.0574, - "hfopenllm_v2/GPQA": 0.2794, - "hfopenllm_v2/MUSR": 0.3804, - "hfopenllm_v2/MMLU-PRO": 0.3095 - } - }, - { - "id": "Norquinal/Foxtrot", - "name": "Foxtrot", - "developer": "Norquinal", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3012, - "hfopenllm_v2/BBH": 0.3558, - "hfopenllm_v2/MATH Level 5": 0.0582, - "hfopenllm_v2/GPQA": 0.2869, - "hfopenllm_v2/MUSR": 0.3804, - "hfopenllm_v2/MMLU-PRO": 0.305 - } - }, - { - "id": "Norquinal/Golf", - "name": "Golf", - "developer": "Norquinal", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3534, - "hfopenllm_v2/BBH": 0.3533, - "hfopenllm_v2/MATH Level 5": 0.0536, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.338, - "hfopenllm_v2/MMLU-PRO": 0.3056 - } - }, - { - "id": "Norquinal/Hotel", - "name": "Hotel", - "developer": "Norquinal", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3215, - "hfopenllm_v2/BBH": 0.3679, - "hfopenllm_v2/MATH Level 5": 0.0529, - "hfopenllm_v2/GPQA": 0.2794, - "hfopenllm_v2/MUSR": 0.3288, - "hfopenllm_v2/MMLU-PRO": 0.3157 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/notasi.json b/data/developers/notasi.json deleted file mode 100644 index 42c466260d274c745a84140a667b67e6cccdfc01..0000000000000000000000000000000000000000 --- a/data/developers/notasi.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "developer": "NotASI", - "models": [ - { - "id": "NotASI/FineTome-Llama3.2-1B-0929", - "name": "FineTome-Llama3.2-1B-0929", - "developer": "NotASI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3991, - "hfopenllm_v2/BBH": 0.3246, - "hfopenllm_v2/MATH Level 5": 0.0363, - "hfopenllm_v2/GPQA": 0.2727, - "hfopenllm_v2/MUSR": 0.3488, - "hfopenllm_v2/MMLU-PRO": 0.1429 - } - }, - { - "id": "NotASI/FineTome-Llama3.2-3B-1002", - "name": "FineTome-Llama3.2-3B-1002", - "developer": "NotASI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5474, - "hfopenllm_v2/BBH": 0.4319, - "hfopenllm_v2/MATH Level 5": 0.0627, - "hfopenllm_v2/GPQA": 0.2508, - "hfopenllm_v2/MUSR": 0.3685, - "hfopenllm_v2/MMLU-PRO": 0.2437 - } - }, - { - "id": "NotASI/FineTome-v1.5-Llama3.2-1B-1007", - "name": "FineTome-v1.5-Llama3.2-1B-1007", - "developer": "NotASI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3924, - "hfopenllm_v2/BBH": 0.3241, - "hfopenllm_v2/MATH Level 5": 0.0317, - "hfopenllm_v2/GPQA": 0.25, - "hfopenllm_v2/MUSR": 0.3475, - "hfopenllm_v2/MMLU-PRO": 0.1427 - } - }, - { - "id": "NotASI/FineTome-v1.5-Llama3.2-3B-1007", - "name": "FineTome-v1.5-Llama3.2-3B-1007", - "developer": "NotASI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5508, - "hfopenllm_v2/BBH": 0.4312, - "hfopenllm_v2/MATH Level 5": 0.0642, - "hfopenllm_v2/GPQA": 0.2617, - "hfopenllm_v2/MUSR": 0.3645, - "hfopenllm_v2/MMLU-PRO": 0.2448 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/notbdq.json b/data/developers/notbdq.json deleted file mode 100644 index de7f9a0a6578459c2cee836077e08bc53f540af8..0000000000000000000000000000000000000000 --- a/data/developers/notbdq.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "notbdq", - "models": [ - { - "id": "notbdq/Qwen2.5-14B-Instruct-1M-GRPO-Reasoning", - "name": "Qwen2.5-14B-Instruct-1M-GRPO-Reasoning", - "developer": "notbdq", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8414, - "hfopenllm_v2/BBH": 0.6198, - "hfopenllm_v2/MATH Level 5": 0.5302, - "hfopenllm_v2/GPQA": 0.3431, - "hfopenllm_v2/MUSR": 0.418, - "hfopenllm_v2/MMLU-PRO": 0.485 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/nothingiisreal.json b/data/developers/nothingiisreal.json deleted file mode 100644 index 92ab56bfd7ff6fc5c35e22c47a69c430dbab00d9..0000000000000000000000000000000000000000 --- a/data/developers/nothingiisreal.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "developer": "nothingiisreal", - "models": [ - { - "id": "nothingiisreal/L3.1-8B-Celeste-V1.5", - "name": "L3.1-8B-Celeste-V1.5", - "developer": "nothingiisreal", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7327, - "hfopenllm_v2/BBH": 0.5012, - "hfopenllm_v2/MATH Level 5": 0.1465, - "hfopenllm_v2/GPQA": 0.2844, - "hfopenllm_v2/MUSR": 0.3749, - "hfopenllm_v2/MMLU-PRO": 0.3704 - } - }, - { - "id": "nothingiisreal/MN-12B-Starcannon-v2", - "name": "MN-12B-Starcannon-v2", - "developer": "nothingiisreal", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3925, - "hfopenllm_v2/BBH": 0.5004, - "hfopenllm_v2/MATH Level 5": 0.0597, - "hfopenllm_v2/GPQA": 0.2785, - "hfopenllm_v2/MUSR": 0.3978, - "hfopenllm_v2/MMLU-PRO": 0.3128 - } - }, - { - "id": "nothingiisreal/MN-12B-Starcannon-v3", - "name": "MN-12B-Starcannon-v3", - "developer": "nothingiisreal", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3807, - "hfopenllm_v2/BBH": 0.5171, - "hfopenllm_v2/MATH Level 5": 0.0778, - "hfopenllm_v2/GPQA": 0.2735, - "hfopenllm_v2/MUSR": 0.4046, - "hfopenllm_v2/MMLU-PRO": 0.3265 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/nousresearch.json b/data/developers/nousresearch.json deleted file mode 100644 index 5eca3534d830aded2e15a419e7392ebd605b4769..0000000000000000000000000000000000000000 --- a/data/developers/nousresearch.json +++ /dev/null @@ -1,288 +0,0 @@ -{ - "developer": "NousResearch", - "models": [ - { - "id": "NousResearch/DeepHermes-3-Mistral-24B-Preview", - "name": "DeepHermes-3-Mistral-24B-Preview", - "developer": "NousResearch", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4536, - "hfopenllm_v2/BBH": 0.6488, - "hfopenllm_v2/MATH Level 5": 0.2576, - "hfopenllm_v2/GPQA": 0.37, - "hfopenllm_v2/MUSR": 0.4503, - "hfopenllm_v2/MMLU-PRO": 0.459 - } - }, - { - "id": "NousResearch/Hermes-2-Pro-Llama-3-8B", - "name": "Hermes-2-Pro-Llama-3-8B", - "developer": "NousResearch", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5362, - "hfopenllm_v2/BBH": 0.5071, - "hfopenllm_v2/MATH Level 5": 0.0838, - "hfopenllm_v2/GPQA": 0.2928, - "hfopenllm_v2/MUSR": 0.4262, - "hfopenllm_v2/MMLU-PRO": 0.3052 - } - }, - { - "id": "NousResearch/Hermes-2-Pro-Mistral-7B", - "name": "Hermes-2-Pro-Mistral-7B", - "developer": "NousResearch", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5668, - "hfopenllm_v2/BBH": 0.4995, - "hfopenllm_v2/MATH Level 5": 0.0604, - "hfopenllm_v2/GPQA": 0.2735, - "hfopenllm_v2/MUSR": 0.4376, - "hfopenllm_v2/MMLU-PRO": 0.2946 - } - }, - { - "id": "NousResearch/Hermes-2-Theta-Llama-3-8B", - "name": "Hermes-2-Theta-Llama-3-8B", - "developer": "NousResearch", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6518, - "hfopenllm_v2/BBH": 0.5207, - "hfopenllm_v2/MATH Level 5": 0.0967, - "hfopenllm_v2/GPQA": 0.3037, - "hfopenllm_v2/MUSR": 0.3949, - "hfopenllm_v2/MMLU-PRO": 0.3369 - } - }, - { - "id": "NousResearch/Hermes-3-Llama-3.1-70B", - "name": "NousResearch/Hermes-3-Llama-3.1-70B", - "developer": "NousResearch", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7661, - "hfopenllm_v2/BBH": 0.6756, - "hfopenllm_v2/MATH Level 5": 0.21, - "hfopenllm_v2/GPQA": 0.3616, - "hfopenllm_v2/MUSR": 0.4949, - "hfopenllm_v2/MMLU-PRO": 0.4727, - "reward-bench/Score": 0.7847, - "reward-bench/Chat": 0.9623, - "reward-bench/Chat Hard": 0.5669, - "reward-bench/Safety": 0.823, - "reward-bench/Reasoning": 0.7867 - } - }, - { - "id": "NousResearch/Hermes-3-Llama-3.1-8B", - "name": "Hermes-3-Llama-3.1-8B", - "developer": "NousResearch", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.617, - "hfopenllm_v2/BBH": 0.5177, - "hfopenllm_v2/MATH Level 5": 0.0476, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.4369, - "hfopenllm_v2/MMLU-PRO": 0.3139 - } - }, - { - "id": "NousResearch/Hermes-3-Llama-3.2-3B", - "name": "Hermes-3-Llama-3.2-3B", - "developer": "NousResearch", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3825, - "hfopenllm_v2/BBH": 0.4352, - "hfopenllm_v2/MATH Level 5": 0.0393, - "hfopenllm_v2/GPQA": 0.2752, - "hfopenllm_v2/MUSR": 0.403, - "hfopenllm_v2/MMLU-PRO": 0.2544 - } - }, - { - "id": "NousResearch/Nous-Hermes-2-Mistral-7B-DPO", - "name": "NousResearch/Nous-Hermes-2-Mistral-7B-DPO", - "developer": "NousResearch", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5763, - "hfopenllm_v2/BBH": 0.4853, - "hfopenllm_v2/MATH Level 5": 0.0476, - "hfopenllm_v2/GPQA": 0.2928, - "hfopenllm_v2/MUSR": 0.4, - "hfopenllm_v2/MMLU-PRO": 0.3015, - "reward-bench/Score": 0.7481, - "reward-bench/Chat": 0.9218, - "reward-bench/Chat Hard": 0.6053, - "reward-bench/Safety": 0.8243, - "reward-bench/Reasoning": 0.7375, - "reward-bench/Prior Sets (0.5 weight)": 0.555 - } - }, - { - "id": "NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO", - "name": "NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO", - "developer": "NousResearch", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5897, - "hfopenllm_v2/BBH": 0.5539, - "hfopenllm_v2/MATH Level 5": 0.1224, - "hfopenllm_v2/GPQA": 0.3213, - "hfopenllm_v2/MUSR": 0.4595, - "hfopenllm_v2/MMLU-PRO": 0.3666, - "reward-bench/Score": 0.7138, - "reward-bench/Chat": 0.9162, - "reward-bench/Chat Hard": 0.6053, - "reward-bench/Safety": 0.8149, - "reward-bench/Reasoning": 0.6126, - "reward-bench/Prior Sets (0.5 weight)": 0.5266 - } - }, - { - "id": "NousResearch/Nous-Hermes-2-Mixtral-8x7B-SFT", - "name": "Nous-Hermes-2-Mixtral-8x7B-SFT", - "developer": "NousResearch", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5731, - "hfopenllm_v2/BBH": 0.5058, - "hfopenllm_v2/MATH Level 5": 0.0211, - "hfopenllm_v2/GPQA": 0.302, - "hfopenllm_v2/MUSR": 0.4214, - "hfopenllm_v2/MMLU-PRO": 0.3066 - } - }, - { - "id": "NousResearch/Nous-Hermes-2-SOLAR-10.7B", - "name": "Nous-Hermes-2-SOLAR-10.7B", - "developer": "NousResearch", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5279, - "hfopenllm_v2/BBH": 0.5414, - "hfopenllm_v2/MATH Level 5": 0.0574, - "hfopenllm_v2/GPQA": 0.2936, - "hfopenllm_v2/MUSR": 0.4373, - "hfopenllm_v2/MMLU-PRO": 0.3458 - } - }, - { - "id": "NousResearch/Nous-Hermes-llama-2-7b", - "name": "Nous-Hermes-llama-2-7b", - "developer": "NousResearch", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1729, - "hfopenllm_v2/BBH": 0.3824, - "hfopenllm_v2/MATH Level 5": 0.0091, - "hfopenllm_v2/GPQA": 0.2634, - "hfopenllm_v2/MUSR": 0.4257, - "hfopenllm_v2/MMLU-PRO": 0.194 - } - }, - { - "id": "NousResearch/Yarn-Llama-2-13b-128k", - "name": "Yarn-Llama-2-13b-128k", - "developer": "NousResearch", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1655, - "hfopenllm_v2/BBH": 0.3827, - "hfopenllm_v2/MATH Level 5": 0.0174, - "hfopenllm_v2/GPQA": 0.2584, - "hfopenllm_v2/MUSR": 0.3458, - "hfopenllm_v2/MMLU-PRO": 0.232 - } - }, - { - "id": "NousResearch/Yarn-Llama-2-7b-128k", - "name": "Yarn-Llama-2-7b-128k", - "developer": "NousResearch", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1485, - "hfopenllm_v2/BBH": 0.3248, - "hfopenllm_v2/MATH Level 5": 0.0151, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3967, - "hfopenllm_v2/MMLU-PRO": 0.1791 - } - }, - { - "id": "NousResearch/Yarn-Llama-2-7b-64k", - "name": "Yarn-Llama-2-7b-64k", - "developer": "NousResearch", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.17, - "hfopenllm_v2/BBH": 0.3326, - "hfopenllm_v2/MATH Level 5": 0.0159, - "hfopenllm_v2/GPQA": 0.2643, - "hfopenllm_v2/MUSR": 0.3939, - "hfopenllm_v2/MMLU-PRO": 0.1799 - } - }, - { - "id": "NousResearch/Yarn-Mistral-7b-128k", - "name": "Yarn-Mistral-7b-128k", - "developer": "NousResearch", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1934, - "hfopenllm_v2/BBH": 0.4314, - "hfopenllm_v2/MATH Level 5": 0.0317, - "hfopenllm_v2/GPQA": 0.2987, - "hfopenllm_v2/MUSR": 0.4071, - "hfopenllm_v2/MMLU-PRO": 0.2893 - } - }, - { - "id": "NousResearch/Yarn-Mistral-7b-64k", - "name": "Yarn-Mistral-7b-64k", - "developer": "NousResearch", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.208, - "hfopenllm_v2/BBH": 0.4293, - "hfopenllm_v2/MATH Level 5": 0.037, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.4124, - "hfopenllm_v2/MMLU-PRO": 0.2914 - } - }, - { - "id": "NousResearch/Yarn-Solar-10b-32k", - "name": "Yarn-Solar-10b-32k", - "developer": "NousResearch", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1942, - "hfopenllm_v2/BBH": 0.4987, - "hfopenllm_v2/MATH Level 5": 0.0302, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.4146, - "hfopenllm_v2/MMLU-PRO": 0.3272 - } - }, - { - "id": "NousResearch/Yarn-Solar-10b-64k", - "name": "Yarn-Solar-10b-64k", - "developer": "NousResearch", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1989, - "hfopenllm_v2/BBH": 0.4922, - "hfopenllm_v2/MATH Level 5": 0.0287, - "hfopenllm_v2/GPQA": 0.302, - "hfopenllm_v2/MUSR": 0.4014, - "hfopenllm_v2/MMLU-PRO": 0.3148 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/novaciano.json b/data/developers/novaciano.json deleted file mode 100644 index 8f4bddfabc83d9f2d1641e77a712e29c99428196..0000000000000000000000000000000000000000 --- a/data/developers/novaciano.json +++ /dev/null @@ -1,159 +0,0 @@ -{ - "developer": "Novaciano", - "models": [ - { - "id": "Novaciano/ASTAROTH-3.2-1B", - "name": "ASTAROTH-3.2-1B", - "developer": "Novaciano", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5613, - "hfopenllm_v2/BBH": 0.3543, - "hfopenllm_v2/MATH Level 5": 0.0733, - "hfopenllm_v2/GPQA": 0.2559, - "hfopenllm_v2/MUSR": 0.3142, - "hfopenllm_v2/MMLU-PRO": 0.1909 - } - }, - { - "id": "Novaciano/BLAST_PROCESSING-3.2-1B", - "name": "BLAST_PROCESSING-3.2-1B", - "developer": "Novaciano", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3922, - "hfopenllm_v2/BBH": 0.346, - "hfopenllm_v2/MATH Level 5": 0.0748, - "hfopenllm_v2/GPQA": 0.2659, - "hfopenllm_v2/MUSR": 0.3351, - "hfopenllm_v2/MMLU-PRO": 0.1941 - } - }, - { - "id": "Novaciano/Cerberus-3.2-1B", - "name": "Cerberus-3.2-1B", - "developer": "Novaciano", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5017, - "hfopenllm_v2/BBH": 0.4165, - "hfopenllm_v2/MATH Level 5": 0.0582, - "hfopenllm_v2/GPQA": 0.2584, - "hfopenllm_v2/MUSR": 0.3289, - "hfopenllm_v2/MMLU-PRO": 0.1663 - } - }, - { - "id": "Novaciano/Cultist-3.2-1B", - "name": "Cultist-3.2-1B", - "developer": "Novaciano", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5295, - "hfopenllm_v2/BBH": 0.3399, - "hfopenllm_v2/MATH Level 5": 0.0589, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.333, - "hfopenllm_v2/MMLU-PRO": 0.1714 - } - }, - { - "id": "Novaciano/FuseChat-3.2-1B-GRPO_Creative_RP", - "name": "FuseChat-3.2-1B-GRPO_Creative_RP", - "developer": "Novaciano", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5598, - "hfopenllm_v2/BBH": 0.3488, - "hfopenllm_v2/MATH Level 5": 0.0801, - "hfopenllm_v2/GPQA": 0.2559, - "hfopenllm_v2/MUSR": 0.3329, - "hfopenllm_v2/MMLU-PRO": 0.1735 - } - }, - { - "id": "Novaciano/Fusetrix-3.2-1B-GRPO_RP_Creative", - "name": "Fusetrix-3.2-1B-GRPO_RP_Creative", - "developer": "Novaciano", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5366, - "hfopenllm_v2/BBH": 0.3435, - "hfopenllm_v2/MATH Level 5": 0.1148, - "hfopenllm_v2/GPQA": 0.25, - "hfopenllm_v2/MUSR": 0.3209, - "hfopenllm_v2/MMLU-PRO": 0.1758 - } - }, - { - "id": "Novaciano/Fusetrix-Dolphin-3.2-1B-GRPO_Creative_RP", - "name": "Fusetrix-Dolphin-3.2-1B-GRPO_Creative_RP", - "developer": "Novaciano", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5343, - "hfopenllm_v2/BBH": 0.3502, - "hfopenllm_v2/MATH Level 5": 0.105, - "hfopenllm_v2/GPQA": 0.2685, - "hfopenllm_v2/MUSR": 0.3183, - "hfopenllm_v2/MMLU-PRO": 0.1823 - } - }, - { - "id": "Novaciano/HarmfulProject-3.2-1B", - "name": "HarmfulProject-3.2-1B", - "developer": "Novaciano", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3874, - "hfopenllm_v2/BBH": 0.3274, - "hfopenllm_v2/MATH Level 5": 0.0476, - "hfopenllm_v2/GPQA": 0.2668, - "hfopenllm_v2/MUSR": 0.3419, - "hfopenllm_v2/MMLU-PRO": 0.1823 - } - }, - { - "id": "Novaciano/La_Mejor_Mezcla-3.2-1B", - "name": "La_Mejor_Mezcla-3.2-1B", - "developer": "Novaciano", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.551, - "hfopenllm_v2/BBH": 0.3488, - "hfopenllm_v2/MATH Level 5": 0.0899, - "hfopenllm_v2/GPQA": 0.2576, - "hfopenllm_v2/MUSR": 0.3196, - "hfopenllm_v2/MMLU-PRO": 0.1829 - } - }, - { - "id": "Novaciano/LEWD-Mental-Cultist-3.2-1B", - "name": "LEWD-Mental-Cultist-3.2-1B", - "developer": "Novaciano", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5309, - "hfopenllm_v2/BBH": 0.3513, - "hfopenllm_v2/MATH Level 5": 0.0529, - "hfopenllm_v2/GPQA": 0.2567, - "hfopenllm_v2/MUSR": 0.3223, - "hfopenllm_v2/MMLU-PRO": 0.1769 - } - }, - { - "id": "Novaciano/Sigil-Of-Satan-3.2-1B", - "name": "Sigil-Of-Satan-3.2-1B", - "developer": "Novaciano", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5494, - "hfopenllm_v2/BBH": 0.3546, - "hfopenllm_v2/MATH Level 5": 0.0544, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.3276, - "hfopenllm_v2/MMLU-PRO": 0.1855 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/ntqai.json b/data/developers/ntqai.json deleted file mode 100644 index e1a7d459a31e41649f5fd93f3471c3cbcba8e339..0000000000000000000000000000000000000000 --- a/data/developers/ntqai.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "NTQAI", - "models": [ - { - "id": "NTQAI/Nxcode-CQ-7B-orpo", - "name": "Nxcode-CQ-7B-orpo", - "developer": "NTQAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4007, - "hfopenllm_v2/BBH": 0.4143, - "hfopenllm_v2/MATH Level 5": 0.0219, - "hfopenllm_v2/GPQA": 0.2542, - "hfopenllm_v2/MUSR": 0.394, - "hfopenllm_v2/MMLU-PRO": 0.1612 - } - }, - { - "id": "NTQAI/NxMobileLM-1.5B-SFT", - "name": "NxMobileLM-1.5B-SFT", - "developer": "NTQAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6392, - "hfopenllm_v2/BBH": 0.3957, - "hfopenllm_v2/MATH Level 5": 0.0846, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3555, - "hfopenllm_v2/MMLU-PRO": 0.2817 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/nucleusai.json b/data/developers/nucleusai.json deleted file mode 100644 index 22a66fda36652e355f4ef17c5f1441f472e63eb9..0000000000000000000000000000000000000000 --- a/data/developers/nucleusai.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "NucleusAI", - "models": [ - { - "id": "NucleusAI/nucleus-22B-token-500B", - "name": "nucleus-22B-token-500B", - "developer": "NucleusAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0257, - "hfopenllm_v2/BBH": 0.292, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.25, - "hfopenllm_v2/MUSR": 0.3511, - "hfopenllm_v2/MMLU-PRO": 0.1162 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/nvidia.json b/data/developers/nvidia.json deleted file mode 100644 index 18070d4bff6dc36a41058554bf3e1da4fc153266..0000000000000000000000000000000000000000 --- a/data/developers/nvidia.json +++ /dev/null @@ -1,334 +0,0 @@ -{ - "developer": "nvidia", - "models": [ - { - "id": "nvidia/AceInstruct-1.5B", - "name": "AceInstruct-1.5B", - "developer": "nvidia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3948, - "hfopenllm_v2/BBH": 0.3932, - "hfopenllm_v2/MATH Level 5": 0.3127, - "hfopenllm_v2/GPQA": 0.2718, - "hfopenllm_v2/MUSR": 0.346, - "hfopenllm_v2/MMLU-PRO": 0.2574 - } - }, - { - "id": "nvidia/AceInstruct-72B", - "name": "AceInstruct-72B", - "developer": "nvidia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7119, - "hfopenllm_v2/BBH": 0.6139, - "hfopenllm_v2/MATH Level 5": 0.6261, - "hfopenllm_v2/GPQA": 0.3213, - "hfopenllm_v2/MUSR": 0.4206, - "hfopenllm_v2/MMLU-PRO": 0.4874 - } - }, - { - "id": "nvidia/AceInstruct-7B", - "name": "AceInstruct-7B", - "developer": "nvidia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5422, - "hfopenllm_v2/BBH": 0.5501, - "hfopenllm_v2/MATH Level 5": 0.5295, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.4255, - "hfopenllm_v2/MMLU-PRO": 0.4177 - } - }, - { - "id": "nvidia/AceMath-1.5B-Instruct", - "name": "AceMath-1.5B-Instruct", - "developer": "nvidia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3212, - "hfopenllm_v2/BBH": 0.4024, - "hfopenllm_v2/MATH Level 5": 0.5287, - "hfopenllm_v2/GPQA": 0.2743, - "hfopenllm_v2/MUSR": 0.3607, - "hfopenllm_v2/MMLU-PRO": 0.2064 - } - }, - { - "id": "nvidia/AceMath-72B-Instruct", - "name": "AceMath-72B-Instruct", - "developer": "nvidia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.495, - "hfopenllm_v2/BBH": 0.6402, - "hfopenllm_v2/MATH Level 5": 0.7145, - "hfopenllm_v2/GPQA": 0.271, - "hfopenllm_v2/MUSR": 0.4062, - "hfopenllm_v2/MMLU-PRO": 0.4411 - } - }, - { - "id": "nvidia/AceMath-72B-RM", - "name": "AceMath-72B-RM", - "developer": "nvidia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1413, - "hfopenllm_v2/BBH": 0.2717, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2341, - "hfopenllm_v2/MUSR": 0.3351, - "hfopenllm_v2/MMLU-PRO": 0.1179 - } - }, - { - "id": "nvidia/AceMath-7B-Instruct", - "name": "AceMath-7B-Instruct", - "developer": "nvidia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4532, - "hfopenllm_v2/BBH": 0.4994, - "hfopenllm_v2/MATH Level 5": 0.6337, - "hfopenllm_v2/GPQA": 0.2919, - "hfopenllm_v2/MUSR": 0.4193, - "hfopenllm_v2/MMLU-PRO": 0.3383 - } - }, - { - "id": "nvidia/AceMath-7B-RM", - "name": "AceMath-7B-RM", - "developer": "nvidia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1494, - "hfopenllm_v2/BBH": 0.2423, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2458, - "hfopenllm_v2/MUSR": 0.358, - "hfopenllm_v2/MMLU-PRO": 0.1139 - } - }, - { - "id": "nvidia/Hymba-1.5B-Base", - "name": "Hymba-1.5B-Base", - "developer": "nvidia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2295, - "hfopenllm_v2/BBH": 0.3256, - "hfopenllm_v2/MATH Level 5": 0.0136, - "hfopenllm_v2/GPQA": 0.2559, - "hfopenllm_v2/MUSR": 0.3566, - "hfopenllm_v2/MMLU-PRO": 0.1922 - } - }, - { - "id": "nvidia/Hymba-1.5B-Instruct", - "name": "Hymba-1.5B-Instruct", - "developer": "nvidia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6009, - "hfopenllm_v2/BBH": 0.3067, - "hfopenllm_v2/MATH Level 5": 0.0272, - "hfopenllm_v2/GPQA": 0.2886, - "hfopenllm_v2/MUSR": 0.3316, - "hfopenllm_v2/MMLU-PRO": 0.204 - } - }, - { - "id": "nvidia/llama-3-1-nemotron-ultra-253b-v1-fc", - "name": "Llama-3.1-Nemotron-Ultra-253B-v1 (FC)", - "developer": "nvidia", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 108.0, - "bfcl/bfcl.overall.overall_accuracy": 10.0, - "bfcl/bfcl.overall.total_cost_usd": 0.72, - "bfcl/bfcl.overall.latency_mean_s": 1.42, - "bfcl/bfcl.overall.latency_std_s": 1.84, - "bfcl/bfcl.overall.latency_p95_s": 2.4, - "bfcl/bfcl.non_live.ast_accuracy": 0.0, - "bfcl/bfcl.non_live.simple_ast_accuracy": 0.0, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 0.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 0.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 0.0, - "bfcl/bfcl.live.live_accuracy": 0.0, - "bfcl/bfcl.live.live_simple_ast_accuracy": 0.0, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 0.0, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 0.0, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 0.0, - "bfcl/bfcl.multi_turn.accuracy": 0.0, - "bfcl/bfcl.multi_turn.base_accuracy": 0.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 0.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 0.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 0.0, - "bfcl/bfcl.web_search.accuracy": 0.0, - "bfcl/bfcl.web_search.base_accuracy": 0.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 0.0, - "bfcl/bfcl.memory.accuracy": 0.0, - "bfcl/bfcl.memory.kv_accuracy": 0.0, - "bfcl/bfcl.memory.vector_accuracy": 0.0, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 0.0, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 0.0, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 100.0 - } - }, - { - "id": "nvidia/Llama-3.1-Minitron-4B-Depth-Base", - "name": "Llama-3.1-Minitron-4B-Depth-Base", - "developer": "nvidia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1607, - "hfopenllm_v2/BBH": 0.4171, - "hfopenllm_v2/MATH Level 5": 0.0196, - "hfopenllm_v2/GPQA": 0.2634, - "hfopenllm_v2/MUSR": 0.4011, - "hfopenllm_v2/MMLU-PRO": 0.2798 - } - }, - { - "id": "nvidia/Llama-3.1-Nemotron-70B-Instruct-HF", - "name": "Llama-3.1-Nemotron-70B-Instruct-HF", - "developer": "nvidia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7381, - "hfopenllm_v2/BBH": 0.6316, - "hfopenllm_v2/MATH Level 5": 0.4267, - "hfopenllm_v2/GPQA": 0.2584, - "hfopenllm_v2/MUSR": 0.4328, - "hfopenllm_v2/MMLU-PRO": 0.4919 - } - }, - { - "id": "nvidia/Llama-3.1-Nemotron-70B-Reward", - "name": "nvidia/Llama-3.1-Nemotron-70B-Reward", - "developer": "nvidia", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.9411, - "reward-bench/Chat": 0.9749, - "reward-bench/Chat Hard": 0.8575, - "reward-bench/Safety": 0.9514, - "reward-bench/Reasoning": 0.9807 - } - }, - { - "id": "nvidia/Llama3-70B-SteerLM-RM", - "name": "nvidia/Llama3-70B-SteerLM-RM", - "developer": "nvidia", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.8877, - "reward-bench/Chat": 0.9134, - "reward-bench/Chat Hard": 0.8026, - "reward-bench/Safety": 0.9284, - "reward-bench/Reasoning": 0.9064 - } - }, - { - "id": "nvidia/Minitron-4B-Base", - "name": "Minitron-4B-Base", - "developer": "nvidia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2218, - "hfopenllm_v2/BBH": 0.4084, - "hfopenllm_v2/MATH Level 5": 0.0196, - "hfopenllm_v2/GPQA": 0.2693, - "hfopenllm_v2/MUSR": 0.4134, - "hfopenllm_v2/MMLU-PRO": 0.262 - } - }, - { - "id": "nvidia/Minitron-8B-Base", - "name": "Minitron-8B-Base", - "developer": "nvidia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2424, - "hfopenllm_v2/BBH": 0.4395, - "hfopenllm_v2/MATH Level 5": 0.0257, - "hfopenllm_v2/GPQA": 0.2735, - "hfopenllm_v2/MUSR": 0.4026, - "hfopenllm_v2/MMLU-PRO": 0.3181 - } - }, - { - "id": "nvidia/Mistral-NeMo-Minitron-8B-Base", - "name": "Mistral-NeMo-Minitron-8B-Base", - "developer": "nvidia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1946, - "hfopenllm_v2/BBH": 0.5219, - "hfopenllm_v2/MATH Level 5": 0.0483, - "hfopenllm_v2/GPQA": 0.3255, - "hfopenllm_v2/MUSR": 0.4092, - "hfopenllm_v2/MMLU-PRO": 0.3796 - } - }, - { - "id": "nvidia/Mistral-NeMo-Minitron-8B-Instruct", - "name": "Mistral-NeMo-Minitron-8B-Instruct", - "developer": "nvidia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5004, - "hfopenllm_v2/BBH": 0.5321, - "hfopenllm_v2/MATH Level 5": 0.1163, - "hfopenllm_v2/GPQA": 0.2878, - "hfopenllm_v2/MUSR": 0.3886, - "hfopenllm_v2/MMLU-PRO": 0.3991 - } - }, - { - "id": "nvidia/Nemotron-4-340B-Reward", - "name": "nvidia/Nemotron-4-340B-Reward", - "developer": "nvidia", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.92, - "reward-bench/Chat": 0.9581, - "reward-bench/Chat Hard": 0.8706, - "reward-bench/Safety": 0.9149, - "reward-bench/Reasoning": 0.9363 - } - }, - { - "id": "nvidia/Nemotron-Mini-4B-Instruct", - "name": "Nemotron-Mini-4B-Instruct", - "developer": "nvidia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6669, - "hfopenllm_v2/BBH": 0.3865, - "hfopenllm_v2/MATH Level 5": 0.0257, - "hfopenllm_v2/GPQA": 0.2802, - "hfopenllm_v2/MUSR": 0.3767, - "hfopenllm_v2/MMLU-PRO": 0.2626 - } - }, - { - "id": "nvidia/OpenMath2-Llama3.1-8B", - "name": "OpenMath2-Llama3.1-8B", - "developer": "nvidia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2331, - "hfopenllm_v2/BBH": 0.4096, - "hfopenllm_v2/MATH Level 5": 0.2674, - "hfopenllm_v2/GPQA": 0.2651, - "hfopenllm_v2/MUSR": 0.3436, - "hfopenllm_v2/MMLU-PRO": 0.1553 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/nxmwxm.json b/data/developers/nxmwxm.json deleted file mode 100644 index 4bfd5d6679d4e288a25c6fc080fcde8e2ac4ee73..0000000000000000000000000000000000000000 --- a/data/developers/nxmwxm.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "nxmwxm", - "models": [ - { - "id": "nxmwxm/Beast-Soul-new", - "name": "Beast-Soul-new", - "developer": "nxmwxm", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4869, - "hfopenllm_v2/BBH": 0.5227, - "hfopenllm_v2/MATH Level 5": 0.074, - "hfopenllm_v2/GPQA": 0.2819, - "hfopenllm_v2/MUSR": 0.4459, - "hfopenllm_v2/MMLU-PRO": 0.3102 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/nytk.json b/data/developers/nytk.json deleted file mode 100644 index c8a4ec05353b2f4afd23852adae2b3d98bfecb55..0000000000000000000000000000000000000000 --- a/data/developers/nytk.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "NYTK", - "models": [ - { - "id": "NYTK/PULI-GPTrio", - "name": "PULI-GPTrio", - "developer": "NYTK", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.218, - "hfopenllm_v2/BBH": 0.306, - "hfopenllm_v2/MATH Level 5": 0.0121, - "hfopenllm_v2/GPQA": 0.2659, - "hfopenllm_v2/MUSR": 0.3819, - "hfopenllm_v2/MMLU-PRO": 0.1137 - } - }, - { - "id": "NYTK/PULI-LlumiX-32K", - "name": "PULI-LlumiX-32K", - "developer": "NYTK", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.17, - "hfopenllm_v2/BBH": 0.3189, - "hfopenllm_v2/MATH Level 5": 0.0128, - "hfopenllm_v2/GPQA": 0.2534, - "hfopenllm_v2/MUSR": 0.3964, - "hfopenllm_v2/MMLU-PRO": 0.1681 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/nyxkrage.json b/data/developers/nyxkrage.json deleted file mode 100644 index 465a044661f12dfb86b4e326cff543e179ac289b..0000000000000000000000000000000000000000 --- a/data/developers/nyxkrage.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "NyxKrage", - "models": [ - { - "id": "NyxKrage/Microsoft_Phi-4", - "name": "Microsoft_Phi-4", - "developer": "NyxKrage", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0585, - "hfopenllm_v2/BBH": 0.6691, - "hfopenllm_v2/MATH Level 5": 0.2991, - "hfopenllm_v2/GPQA": 0.406, - "hfopenllm_v2/MUSR": 0.5034, - "hfopenllm_v2/MMLU-PRO": 0.5287 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/occiglot.json b/data/developers/occiglot.json deleted file mode 100644 index 504d6f906b1432668570bd83990235a541e2e8b3..0000000000000000000000000000000000000000 --- a/data/developers/occiglot.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "occiglot", - "models": [ - { - "id": "occiglot/occiglot-7b-es-en-instruct", - "name": "occiglot-7b-es-en-instruct", - "developer": "occiglot", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3485, - "hfopenllm_v2/BBH": 0.4111, - "hfopenllm_v2/MATH Level 5": 0.0242, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3738, - "hfopenllm_v2/MMLU-PRO": 0.2311 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/odyssey-labs.json b/data/developers/odyssey-labs.json deleted file mode 100644 index 1e0ac40fc81dd082be186a296fb3d45f66972844..0000000000000000000000000000000000000000 --- a/data/developers/odyssey-labs.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "odyssey-labs", - "models": [ - { - "id": "odyssey-labs/Astral-1-10B", - "name": "Astral-1-10B", - "developer": "odyssey-labs", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3878, - "hfopenllm_v2/BBH": 0.4873, - "hfopenllm_v2/MATH Level 5": 0.0347, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.428, - "hfopenllm_v2/MMLU-PRO": 0.2985 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/oevortex.json b/data/developers/oevortex.json deleted file mode 100644 index 20e72c884066d591a9eeec3ef8d2e75f250ce485..0000000000000000000000000000000000000000 --- a/data/developers/oevortex.json +++ /dev/null @@ -1,75 +0,0 @@ -{ - "developer": "OEvortex", - "models": [ - { - "id": "OEvortex/Emotional-llama-8B", - "name": "Emotional-llama-8B", - "developer": "OEvortex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3516, - "hfopenllm_v2/BBH": 0.4839, - "hfopenllm_v2/MATH Level 5": 0.0816, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.3659, - "hfopenllm_v2/MMLU-PRO": 0.3535 - } - }, - { - "id": "OEvortex/HelpingAI-15B", - "name": "HelpingAI-15B", - "developer": "OEvortex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.203, - "hfopenllm_v2/BBH": 0.2936, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2576, - "hfopenllm_v2/MUSR": 0.3619, - "hfopenllm_v2/MMLU-PRO": 0.1111 - } - }, - { - "id": "OEvortex/HelpingAI-3B-reloaded", - "name": "HelpingAI-3B-reloaded", - "developer": "OEvortex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4647, - "hfopenllm_v2/BBH": 0.4129, - "hfopenllm_v2/MATH Level 5": 0.0136, - "hfopenllm_v2/GPQA": 0.2634, - "hfopenllm_v2/MUSR": 0.3524, - "hfopenllm_v2/MMLU-PRO": 0.2595 - } - }, - { - "id": "OEvortex/HelpingAI2-9B", - "name": "HelpingAI2-9B", - "developer": "OEvortex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4413, - "hfopenllm_v2/BBH": 0.4845, - "hfopenllm_v2/MATH Level 5": 0.0589, - "hfopenllm_v2/GPQA": 0.2584, - "hfopenllm_v2/MUSR": 0.3711, - "hfopenllm_v2/MMLU-PRO": 0.29 - } - }, - { - "id": "OEvortex/HelpingAI2.5-10B", - "name": "HelpingAI2.5-10B", - "developer": "OEvortex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3277, - "hfopenllm_v2/BBH": 0.4496, - "hfopenllm_v2/MATH Level 5": 0.0204, - "hfopenllm_v2/GPQA": 0.2693, - "hfopenllm_v2/MUSR": 0.3738, - "hfopenllm_v2/MMLU-PRO": 0.2575 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/olabs-ai.json b/data/developers/olabs-ai.json deleted file mode 100644 index 9c8f5e5288b93827b623178356c89affb926a845..0000000000000000000000000000000000000000 --- a/data/developers/olabs-ai.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "olabs-ai", - "models": [ - { - "id": "olabs-ai/reflection_model", - "name": "reflection_model", - "developer": "olabs-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1599, - "hfopenllm_v2/BBH": 0.4713, - "hfopenllm_v2/MATH Level 5": 0.0514, - "hfopenllm_v2/GPQA": 0.3003, - "hfopenllm_v2/MUSR": 0.3508, - "hfopenllm_v2/MMLU-PRO": 0.3311 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/oliveirajlt.json b/data/developers/oliveirajlt.json deleted file mode 100644 index 96425d9532f2e884859ea2f3d4edc2512f13356f..0000000000000000000000000000000000000000 --- a/data/developers/oliveirajlt.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "OliveiraJLT", - "models": [ - { - "id": "OliveiraJLT/Sagui-7B-Instruct-v0.1", - "name": "Sagui-7B-Instruct-v0.1", - "developer": "OliveiraJLT", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2892, - "hfopenllm_v2/BBH": 0.3111, - "hfopenllm_v2/MATH Level 5": 0.0151, - "hfopenllm_v2/GPQA": 0.2424, - "hfopenllm_v2/MUSR": 0.4191, - "hfopenllm_v2/MMLU-PRO": 0.1485 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/omkar1102.json b/data/developers/omkar1102.json deleted file mode 100644 index 1d044781189744d770af993cfdb651c1e02eee6f..0000000000000000000000000000000000000000 --- a/data/developers/omkar1102.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "Omkar1102", - "models": [ - { - "id": "Omkar1102/code-yi", - "name": "code-yi", - "developer": "Omkar1102", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2254, - "hfopenllm_v2/BBH": 0.275, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2576, - "hfopenllm_v2/MUSR": 0.3762, - "hfopenllm_v2/MMLU-PRO": 0.1123 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/omnicromsbrain.json b/data/developers/omnicromsbrain.json deleted file mode 100644 index 5e8901580b7e0479fc5b0e2b31b10bf904608a89..0000000000000000000000000000000000000000 --- a/data/developers/omnicromsbrain.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "OmnicromsBrain", - "models": [ - { - "id": "OmnicromsBrain/NeuralStar_FusionWriter_4x7b", - "name": "NeuralStar_FusionWriter_4x7b", - "developer": "OmnicromsBrain", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5964, - "hfopenllm_v2/BBH": 0.4776, - "hfopenllm_v2/MATH Level 5": 0.0491, - "hfopenllm_v2/GPQA": 0.2785, - "hfopenllm_v2/MUSR": 0.4019, - "hfopenllm_v2/MMLU-PRO": 0.2606 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/onlycheeini.json b/data/developers/onlycheeini.json deleted file mode 100644 index fb95652078e138cf9868b98c03d0f01121e777bc..0000000000000000000000000000000000000000 --- a/data/developers/onlycheeini.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "OnlyCheeini", - "models": [ - { - "id": "OnlyCheeini/greesychat-turbo", - "name": "greesychat-turbo", - "developer": "OnlyCheeini", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0233, - "hfopenllm_v2/BBH": 0.3092, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3314, - "hfopenllm_v2/MMLU-PRO": 0.1138 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/ontocord.json b/data/developers/ontocord.json deleted file mode 100644 index c16f4cbacdda3485b721f459b079923a6793a670..0000000000000000000000000000000000000000 --- a/data/developers/ontocord.json +++ /dev/null @@ -1,453 +0,0 @@ -{ - "developer": "ontocord", - "models": [ - { - "id": "ontocord/Llama_3.2_1b-autoredteam_helpfulness-train", - "name": "Llama_3.2_1b-autoredteam_helpfulness-train", - "developer": "ontocord", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2765, - "hfopenllm_v2/BBH": 0.3115, - "hfopenllm_v2/MATH Level 5": 0.0166, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3459, - "hfopenllm_v2/MMLU-PRO": 0.1132 - } - }, - { - "id": "ontocord/merged_0.2_expert_0.8", - "name": "merged_0.2_expert_0.8", - "developer": "ontocord", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1743, - "hfopenllm_v2/BBH": 0.3046, - "hfopenllm_v2/MATH Level 5": 0.0264, - "hfopenllm_v2/GPQA": 0.2617, - "hfopenllm_v2/MUSR": 0.3621, - "hfopenllm_v2/MMLU-PRO": 0.1111 - } - }, - { - "id": "ontocord/merged_0.2_expert_0.8-stack_2x", - "name": "merged_0.2_expert_0.8-stack_2x", - "developer": "ontocord", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1796, - "hfopenllm_v2/BBH": 0.3006, - "hfopenllm_v2/MATH Level 5": 0.0249, - "hfopenllm_v2/GPQA": 0.2626, - "hfopenllm_v2/MUSR": 0.3541, - "hfopenllm_v2/MMLU-PRO": 0.1103 - } - }, - { - "id": "ontocord/merged_0.5_expert_0.5", - "name": "merged_0.5_expert_0.5", - "developer": "ontocord", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1787, - "hfopenllm_v2/BBH": 0.3017, - "hfopenllm_v2/MATH Level 5": 0.0196, - "hfopenllm_v2/GPQA": 0.2643, - "hfopenllm_v2/MUSR": 0.3542, - "hfopenllm_v2/MMLU-PRO": 0.1108 - } - }, - { - "id": "ontocord/ontocord_wide_3b-stage1_shuf_sample1_jsonl-pretrained-autoredteam_helpful-0.25_helpful", - "name": "ontocord_wide_3b-stage1_shuf_sample1_jsonl-pretrained-autoredteam_helpful-0.25_helpful", - "developer": "ontocord", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1318, - "hfopenllm_v2/BBH": 0.3004, - "hfopenllm_v2/MATH Level 5": 0.0106, - "hfopenllm_v2/GPQA": 0.2676, - "hfopenllm_v2/MUSR": 0.3631, - "hfopenllm_v2/MMLU-PRO": 0.1142 - } - }, - { - "id": "ontocord/ontocord_wide_7b-stacked-stage1", - "name": "ontocord_wide_7b-stacked-stage1", - "developer": "ontocord", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1485, - "hfopenllm_v2/BBH": 0.2897, - "hfopenllm_v2/MATH Level 5": 0.0091, - "hfopenllm_v2/GPQA": 0.2534, - "hfopenllm_v2/MUSR": 0.3604, - "hfopenllm_v2/MMLU-PRO": 0.1105 - } - }, - { - "id": "ontocord/ontocord_wide_7b-stacked-stage1-instruct", - "name": "ontocord_wide_7b-stacked-stage1-instruct", - "developer": "ontocord", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.153, - "hfopenllm_v2/BBH": 0.2854, - "hfopenllm_v2/MATH Level 5": 0.0068, - "hfopenllm_v2/GPQA": 0.2466, - "hfopenllm_v2/MUSR": 0.3538, - "hfopenllm_v2/MMLU-PRO": 0.1117 - } - }, - { - "id": "ontocord/RedPajama-3B-v1-AutoRedteam", - "name": "RedPajama-3B-v1-AutoRedteam", - "developer": "ontocord", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1343, - "hfopenllm_v2/BBH": 0.3026, - "hfopenllm_v2/MATH Level 5": 0.0091, - "hfopenllm_v2/GPQA": 0.2424, - "hfopenllm_v2/MUSR": 0.3661, - "hfopenllm_v2/MMLU-PRO": 0.1108 - } - }, - { - "id": "ontocord/RedPajama-3B-v1-AutoRedteam-Harmless-only", - "name": "RedPajama-3B-v1-AutoRedteam-Harmless-only", - "developer": "ontocord", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1525, - "hfopenllm_v2/BBH": 0.3124, - "hfopenllm_v2/MATH Level 5": 0.006, - "hfopenllm_v2/GPQA": 0.2315, - "hfopenllm_v2/MUSR": 0.3661, - "hfopenllm_v2/MMLU-PRO": 0.11 - } - }, - { - "id": "ontocord/RedPajama3b_v1-autoredteam_helpfulness-train", - "name": "RedPajama3b_v1-autoredteam_helpfulness-train", - "developer": "ontocord", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2848, - "hfopenllm_v2/BBH": 0.3093, - "hfopenllm_v2/MATH Level 5": 0.0068, - "hfopenllm_v2/GPQA": 0.2458, - "hfopenllm_v2/MUSR": 0.358, - "hfopenllm_v2/MMLU-PRO": 0.1107 - } - }, - { - "id": "ontocord/starcoder2-29b-ls", - "name": "starcoder2-29b-ls", - "developer": "ontocord", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2149, - "hfopenllm_v2/BBH": 0.3735, - "hfopenllm_v2/MATH Level 5": 0.0189, - "hfopenllm_v2/GPQA": 0.2735, - "hfopenllm_v2/MUSR": 0.37, - "hfopenllm_v2/MMLU-PRO": 0.1869 - } - }, - { - "id": "ontocord/starcoder2_3b-AutoRedteam", - "name": "starcoder2_3b-AutoRedteam", - "developer": "ontocord", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1574, - "hfopenllm_v2/BBH": 0.3498, - "hfopenllm_v2/MATH Level 5": 0.0106, - "hfopenllm_v2/GPQA": 0.2517, - "hfopenllm_v2/MUSR": 0.3646, - "hfopenllm_v2/MMLU-PRO": 0.1336 - } - }, - { - "id": "ontocord/wide_3b-merge_test", - "name": "wide_3b-merge_test", - "developer": "ontocord", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1763, - "hfopenllm_v2/BBH": 0.3011, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2399, - "hfopenllm_v2/MUSR": 0.342, - "hfopenllm_v2/MMLU-PRO": 0.1066 - } - }, - { - "id": "ontocord/wide_3b-stage1_shuf_sample1_jsonl-pretrained", - "name": "wide_3b-stage1_shuf_sample1_jsonl-pretrained", - "developer": "ontocord", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1395, - "hfopenllm_v2/BBH": 0.3004, - "hfopenllm_v2/MATH Level 5": 0.0166, - "hfopenllm_v2/GPQA": 0.2659, - "hfopenllm_v2/MUSR": 0.3632, - "hfopenllm_v2/MMLU-PRO": 0.114 - } - }, - { - "id": "ontocord/wide_3b_sft_stag1.2-lyrical_law_news_software_howto_formattedtext_math_wiki-merge", - "name": "wide_3b_sft_stag1.2-lyrical_law_news_software_howto_formattedtext_math_wiki-merge", - "developer": "ontocord", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1664, - "hfopenllm_v2/BBH": 0.3031, - "hfopenllm_v2/MATH Level 5": 0.0113, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3845, - "hfopenllm_v2/MMLU-PRO": 0.1111 - } - }, - { - "id": "ontocord/wide_3b_sft_stag1.2-lyrical_news_software_howto_formattedtext-merge", - "name": "wide_3b_sft_stag1.2-lyrical_news_software_howto_formattedtext-merge", - "developer": "ontocord", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1697, - "hfopenllm_v2/BBH": 0.2975, - "hfopenllm_v2/MATH Level 5": 0.0136, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3778, - "hfopenllm_v2/MMLU-PRO": 0.1125 - } - }, - { - "id": "ontocord/wide_3b_sft_stage1.1-ss1-no_redteam_skg_poem.no_issue", - "name": "wide_3b_sft_stage1.1-ss1-no_redteam_skg_poem.no_issue", - "developer": "ontocord", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.148, - "hfopenllm_v2/BBH": 0.3095, - "hfopenllm_v2/MATH Level 5": 0.0204, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.3579, - "hfopenllm_v2/MMLU-PRO": 0.1108 - } - }, - { - "id": "ontocord/wide_3b_sft_stage1.1-ss1-with_generics_intr.no_issue", - "name": "wide_3b_sft_stage1.1-ss1-with_generics_intr.no_issue", - "developer": "ontocord", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1237, - "hfopenllm_v2/BBH": 0.306, - "hfopenllm_v2/MATH Level 5": 0.0106, - "hfopenllm_v2/GPQA": 0.2743, - "hfopenllm_v2/MUSR": 0.3673, - "hfopenllm_v2/MMLU-PRO": 0.1111 - } - }, - { - "id": "ontocord/wide_3b_sft_stage1.1-ss1-with_generics_intr_math.no_issue", - "name": "wide_3b_sft_stage1.1-ss1-with_generics_intr_math.no_issue", - "developer": "ontocord", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1192, - "hfopenllm_v2/BBH": 0.2956, - "hfopenllm_v2/MATH Level 5": 0.0068, - "hfopenllm_v2/GPQA": 0.2643, - "hfopenllm_v2/MUSR": 0.3553, - "hfopenllm_v2/MMLU-PRO": 0.1183 - } - }, - { - "id": "ontocord/wide_3b_sft_stage1.1-ss1-with_generics_intr_math_stories.no_issue", - "name": "wide_3b_sft_stage1.1-ss1-with_generics_intr_math_stories.no_issue", - "developer": "ontocord", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1128, - "hfopenllm_v2/BBH": 0.3171, - "hfopenllm_v2/MATH Level 5": 0.0113, - "hfopenllm_v2/GPQA": 0.2685, - "hfopenllm_v2/MUSR": 0.346, - "hfopenllm_v2/MMLU-PRO": 0.1129 - } - }, - { - "id": "ontocord/wide_3b_sft_stage1.1-ss1-with_generics_intr_math_stories_no_orig_instr.no_issue", - "name": "wide_3b_sft_stage1.1-ss1-with_generics_intr_math_stories_no_orig_instr.no_issue", - "developer": "ontocord", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1317, - "hfopenllm_v2/BBH": 0.3064, - "hfopenllm_v2/MATH Level 5": 0.0091, - "hfopenllm_v2/GPQA": 0.2651, - "hfopenllm_v2/MUSR": 0.3446, - "hfopenllm_v2/MMLU-PRO": 0.1144 - } - }, - { - "id": "ontocord/wide_3b_sft_stage1.1-ss1-with_generics_intr_stories.no_issue", - "name": "wide_3b_sft_stage1.1-ss1-with_generics_intr_stories.no_issue", - "developer": "ontocord", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1182, - "hfopenllm_v2/BBH": 0.3037, - "hfopenllm_v2/MATH Level 5": 0.0083, - "hfopenllm_v2/GPQA": 0.2659, - "hfopenllm_v2/MUSR": 0.3567, - "hfopenllm_v2/MMLU-PRO": 0.1162 - } - }, - { - "id": "ontocord/wide_3b_sft_stage1.1-ss1-with_generics_math.no_issue", - "name": "wide_3b_sft_stage1.1-ss1-with_generics_math.no_issue", - "developer": "ontocord", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.124, - "hfopenllm_v2/BBH": 0.3032, - "hfopenllm_v2/MATH Level 5": 0.0076, - "hfopenllm_v2/GPQA": 0.2584, - "hfopenllm_v2/MUSR": 0.3487, - "hfopenllm_v2/MMLU-PRO": 0.1128 - } - }, - { - "id": "ontocord/wide_3b_sft_stage1.1-ss1-with_math.no_issue", - "name": "wide_3b_sft_stage1.1-ss1-with_math.no_issue", - "developer": "ontocord", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1298, - "hfopenllm_v2/BBH": 0.3052, - "hfopenllm_v2/MATH Level 5": 0.0159, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3928, - "hfopenllm_v2/MMLU-PRO": 0.1147 - } - }, - { - "id": "ontocord/wide_3b_sft_stage1.1-ss1-with_r1_generics_intr_math_stories.no_issue", - "name": "wide_3b_sft_stage1.1-ss1-with_r1_generics_intr_math_stories.no_issue", - "developer": "ontocord", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2049, - "hfopenllm_v2/BBH": 0.2912, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3575, - "hfopenllm_v2/MMLU-PRO": 0.1167 - } - }, - { - "id": "ontocord/wide_3b_sft_stage1.2-ss1-expert_fictional_lyrical", - "name": "wide_3b_sft_stage1.2-ss1-expert_fictional_lyrical", - "developer": "ontocord", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1461, - "hfopenllm_v2/BBH": 0.2998, - "hfopenllm_v2/MATH Level 5": 0.0136, - "hfopenllm_v2/GPQA": 0.2643, - "hfopenllm_v2/MUSR": 0.3926, - "hfopenllm_v2/MMLU-PRO": 0.1141 - } - }, - { - "id": "ontocord/wide_3b_sft_stage1.2-ss1-expert_formatted_text", - "name": "wide_3b_sft_stage1.2-ss1-expert_formatted_text", - "developer": "ontocord", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1487, - "hfopenllm_v2/BBH": 0.3069, - "hfopenllm_v2/MATH Level 5": 0.0121, - "hfopenllm_v2/GPQA": 0.2617, - "hfopenllm_v2/MUSR": 0.3474, - "hfopenllm_v2/MMLU-PRO": 0.1146 - } - }, - { - "id": "ontocord/wide_3b_sft_stage1.2-ss1-expert_how-to", - "name": "wide_3b_sft_stage1.2-ss1-expert_how-to", - "developer": "ontocord", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1245, - "hfopenllm_v2/BBH": 0.3047, - "hfopenllm_v2/MATH Level 5": 0.0144, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3658, - "hfopenllm_v2/MMLU-PRO": 0.1153 - } - }, - { - "id": "ontocord/wide_3b_sft_stage1.2-ss1-expert_math", - "name": "wide_3b_sft_stage1.2-ss1-expert_math", - "developer": "ontocord", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1915, - "hfopenllm_v2/BBH": 0.306, - "hfopenllm_v2/MATH Level 5": 0.0279, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.37, - "hfopenllm_v2/MMLU-PRO": 0.1092 - } - }, - { - "id": "ontocord/wide_3b_sft_stage1.2-ss1-expert_news", - "name": "wide_3b_sft_stage1.2-ss1-expert_news", - "developer": "ontocord", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1658, - "hfopenllm_v2/BBH": 0.2926, - "hfopenllm_v2/MATH Level 5": 0.0166, - "hfopenllm_v2/GPQA": 0.2676, - "hfopenllm_v2/MUSR": 0.3621, - "hfopenllm_v2/MMLU-PRO": 0.1111 - } - }, - { - "id": "ontocord/wide_3b_sft_stage1.2-ss1-expert_software", - "name": "wide_3b_sft_stage1.2-ss1-expert_software", - "developer": "ontocord", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1734, - "hfopenllm_v2/BBH": 0.298, - "hfopenllm_v2/MATH Level 5": 0.0159, - "hfopenllm_v2/GPQA": 0.2584, - "hfopenllm_v2/MUSR": 0.3569, - "hfopenllm_v2/MMLU-PRO": 0.114 - } - }, - { - "id": "ontocord/wide_6.6b_sft_stag1.2-lyrical_law_news_software_howto_formattedtext_math_wiki-merge-stacked", - "name": "wide_6.6b_sft_stag1.2-lyrical_law_news_software_howto_formattedtext_math_wiki-merge-stacked", - "developer": "ontocord", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1244, - "hfopenllm_v2/BBH": 0.3026, - "hfopenllm_v2/MATH Level 5": 0.0144, - "hfopenllm_v2/GPQA": 0.2659, - "hfopenllm_v2/MUSR": 0.3686, - "hfopenllm_v2/MMLU-PRO": 0.1115 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/oobabooga.json b/data/developers/oobabooga.json deleted file mode 100644 index d6cce322b760eafcdd9d8d0a8f78e5eee9704c01..0000000000000000000000000000000000000000 --- a/data/developers/oobabooga.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "oobabooga", - "models": [ - { - "id": "oobabooga/CodeBooga-34B-v0.1", - "name": "CodeBooga-34B-v0.1", - "developer": "oobabooga", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.525, - "hfopenllm_v2/BBH": 0.3427, - "hfopenllm_v2/MATH Level 5": 0.0393, - "hfopenllm_v2/GPQA": 0.2567, - "hfopenllm_v2/MUSR": 0.431, - "hfopenllm_v2/MMLU-PRO": 0.236 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/oopere.json b/data/developers/oopere.json deleted file mode 100644 index 71d39c6c35063e6c97b2a688c804f115030e73dd..0000000000000000000000000000000000000000 --- a/data/developers/oopere.json +++ /dev/null @@ -1,131 +0,0 @@ -{ - "developer": "oopere", - "models": [ - { - "id": "oopere/Llama-FinSent-S", - "name": "Llama-FinSent-S", - "developer": "oopere", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2164, - "hfopenllm_v2/BBH": 0.3169, - "hfopenllm_v2/MATH Level 5": 0.0128, - "hfopenllm_v2/GPQA": 0.2584, - "hfopenllm_v2/MUSR": 0.3832, - "hfopenllm_v2/MMLU-PRO": 0.1134 - } - }, - { - "id": "oopere/pruned10-llama-3.2-3B", - "name": "pruned10-llama-3.2-3B", - "developer": "oopere", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1776, - "hfopenllm_v2/BBH": 0.334, - "hfopenllm_v2/MATH Level 5": 0.0196, - "hfopenllm_v2/GPQA": 0.2668, - "hfopenllm_v2/MUSR": 0.3722, - "hfopenllm_v2/MMLU-PRO": 0.164 - } - }, - { - "id": "oopere/pruned20-llama-1b", - "name": "pruned20-llama-1b", - "developer": "oopere", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1994, - "hfopenllm_v2/BBH": 0.3031, - "hfopenllm_v2/MATH Level 5": 0.0106, - "hfopenllm_v2/GPQA": 0.25, - "hfopenllm_v2/MUSR": 0.3631, - "hfopenllm_v2/MMLU-PRO": 0.1123 - } - }, - { - "id": "oopere/pruned20-llama-3.2-3b", - "name": "pruned20-llama-3.2-3b", - "developer": "oopere", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1789, - "hfopenllm_v2/BBH": 0.3248, - "hfopenllm_v2/MATH Level 5": 0.0159, - "hfopenllm_v2/GPQA": 0.2659, - "hfopenllm_v2/MUSR": 0.3418, - "hfopenllm_v2/MMLU-PRO": 0.128 - } - }, - { - "id": "oopere/pruned40-llama-1b", - "name": "pruned40-llama-1b", - "developer": "oopere", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2284, - "hfopenllm_v2/BBH": 0.2969, - "hfopenllm_v2/MATH Level 5": 0.0076, - "hfopenllm_v2/GPQA": 0.2433, - "hfopenllm_v2/MUSR": 0.4287, - "hfopenllm_v2/MMLU-PRO": 0.1082 - } - }, - { - "id": "oopere/pruned40-llama-3.2-1B", - "name": "pruned40-llama-3.2-1B", - "developer": "oopere", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2266, - "hfopenllm_v2/BBH": 0.2982, - "hfopenllm_v2/MATH Level 5": 0.0083, - "hfopenllm_v2/GPQA": 0.2542, - "hfopenllm_v2/MUSR": 0.4352, - "hfopenllm_v2/MMLU-PRO": 0.1115 - } - }, - { - "id": "oopere/pruned40-llama-3.2-3b", - "name": "pruned40-llama-3.2-3b", - "developer": "oopere", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2183, - "hfopenllm_v2/BBH": 0.3167, - "hfopenllm_v2/MATH Level 5": 0.0128, - "hfopenllm_v2/GPQA": 0.2299, - "hfopenllm_v2/MUSR": 0.3539, - "hfopenllm_v2/MMLU-PRO": 0.1177 - } - }, - { - "id": "oopere/pruned60-llama-1b", - "name": "pruned60-llama-1b", - "developer": "oopere", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1829, - "hfopenllm_v2/BBH": 0.3016, - "hfopenllm_v2/MATH Level 5": 0.0023, - "hfopenllm_v2/GPQA": 0.2492, - "hfopenllm_v2/MUSR": 0.4088, - "hfopenllm_v2/MMLU-PRO": 0.1173 - } - }, - { - "id": "oopere/pruned60-llama-3.2-3b", - "name": "pruned60-llama-3.2-3b", - "developer": "oopere", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1825, - "hfopenllm_v2/BBH": 0.3166, - "hfopenllm_v2/MATH Level 5": 0.0038, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.3633, - "hfopenllm_v2/MMLU-PRO": 0.1131 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/open-atlas.json b/data/developers/open-atlas.json deleted file mode 100644 index 2b4ab3cf42ca20f6291a93804bff62a5bc01d90b..0000000000000000000000000000000000000000 --- a/data/developers/open-atlas.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "open-atlas", - "models": [ - { - "id": "open-atlas/Atlas-Flash-1.5B-Preview", - "name": "Atlas-Flash-1.5B-Preview", - "developer": "open-atlas", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.327, - "hfopenllm_v2/BBH": 0.3215, - "hfopenllm_v2/MATH Level 5": 0.2213, - "hfopenllm_v2/GPQA": 0.2525, - "hfopenllm_v2/MUSR": 0.3488, - "hfopenllm_v2/MMLU-PRO": 0.1374 - } - }, - { - "id": "open-atlas/Atlas-Flash-7B-Preview", - "name": "Atlas-Flash-7B-Preview", - "developer": "open-atlas", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3908, - "hfopenllm_v2/BBH": 0.3542, - "hfopenllm_v2/MATH Level 5": 0.2576, - "hfopenllm_v2/GPQA": 0.2886, - "hfopenllm_v2/MUSR": 0.3836, - "hfopenllm_v2/MMLU-PRO": 0.2784 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/open-neo.json b/data/developers/open-neo.json deleted file mode 100644 index d3e72f4e3df9575c45fa609e3634c93986b615ee..0000000000000000000000000000000000000000 --- a/data/developers/open-neo.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "open-neo", - "models": [ - { - "id": "open-neo/Kyro-n1-3B", - "name": "Kyro-n1-3B", - "developer": "open-neo", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4595, - "hfopenllm_v2/BBH": 0.4685, - "hfopenllm_v2/MATH Level 5": 0.2855, - "hfopenllm_v2/GPQA": 0.2819, - "hfopenllm_v2/MUSR": 0.4088, - "hfopenllm_v2/MMLU-PRO": 0.3423 - } - }, - { - "id": "open-neo/Kyro-n1-7B", - "name": "Kyro-n1-7B", - "developer": "open-neo", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5573, - "hfopenllm_v2/BBH": 0.5387, - "hfopenllm_v2/MATH Level 5": 0.3897, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.3884, - "hfopenllm_v2/MMLU-PRO": 0.4333 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/open-orca.json b/data/developers/open-orca.json deleted file mode 100644 index 8f976ab95ea911176fa76bd1983b2b9e37e8dd39..0000000000000000000000000000000000000000 --- a/data/developers/open-orca.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "Open-Orca", - "models": [ - { - "id": "Open-Orca/Mistral-7B-OpenOrca", - "name": "Mistral-7B-OpenOrca", - "developer": "Open-Orca", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4978, - "hfopenllm_v2/BBH": 0.4768, - "hfopenllm_v2/MATH Level 5": 0.0355, - "hfopenllm_v2/GPQA": 0.2718, - "hfopenllm_v2/MUSR": 0.3858, - "hfopenllm_v2/MMLU-PRO": 0.2653 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/open-thoughts.json b/data/developers/open-thoughts.json deleted file mode 100644 index 1770bb79eb3160de246e6849b9e6cbf81aced3e6..0000000000000000000000000000000000000000 --- a/data/developers/open-thoughts.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "open-thoughts", - "models": [ - { - "id": "open-thoughts/OpenThinker-7B", - "name": "OpenThinker-7B", - "developer": "open-thoughts", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4089, - "hfopenllm_v2/BBH": 0.5343, - "hfopenllm_v2/MATH Level 5": 0.426, - "hfopenllm_v2/GPQA": 0.2567, - "hfopenllm_v2/MUSR": 0.382, - "hfopenllm_v2/MMLU-PRO": 0.4165 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/openai-community.json b/data/developers/openai-community.json deleted file mode 100644 index b65e0a6d2ad9fcf571ab1f95656f2b9e939df55e..0000000000000000000000000000000000000000 --- a/data/developers/openai-community.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "developer": "openai-community", - "models": [ - { - "id": "openai-community/gpt2", - "name": "gpt2", - "developer": "openai-community", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.178, - "hfopenllm_v2/BBH": 0.3017, - "hfopenllm_v2/MATH Level 5": 0.0053, - "hfopenllm_v2/GPQA": 0.2584, - "hfopenllm_v2/MUSR": 0.439, - "hfopenllm_v2/MMLU-PRO": 0.1165 - } - }, - { - "id": "openai-community/gpt2-large", - "name": "gpt2-large", - "developer": "openai-community", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2048, - "hfopenllm_v2/BBH": 0.3069, - "hfopenllm_v2/MATH Level 5": 0.0121, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3789, - "hfopenllm_v2/MMLU-PRO": 0.1142 - } - }, - { - "id": "openai-community/gpt2-medium", - "name": "gpt2-medium", - "developer": "openai-community", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2208, - "hfopenllm_v2/BBH": 0.305, - "hfopenllm_v2/MATH Level 5": 0.0076, - "hfopenllm_v2/GPQA": 0.2626, - "hfopenllm_v2/MUSR": 0.3884, - "hfopenllm_v2/MMLU-PRO": 0.1182 - } - }, - { - "id": "openai-community/gpt2-xl", - "name": "gpt2-xl", - "developer": "openai-community", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2039, - "hfopenllm_v2/BBH": 0.3009, - "hfopenllm_v2/MATH Level 5": 0.0098, - "hfopenllm_v2/GPQA": 0.2584, - "hfopenllm_v2/MUSR": 0.371, - "hfopenllm_v2/MMLU-PRO": 0.1131 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/openai.json b/data/developers/openai.json deleted file mode 100644 index 650104c582280807d0648ba825be14db81031213..0000000000000000000000000000000000000000 --- a/data/developers/openai.json +++ /dev/null @@ -1,2001 +0,0 @@ -{ - "developer": "OpenAI", - "models": [ - { - "id": "openai/ada-350M", - "name": "ada 350M", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.108, - "helm_classic/MMLU": 0.243, - "helm_classic/BoolQ": 0.581, - "helm_classic/NarrativeQA": 0.326, - "helm_classic/NaturalQuestions (open-book)": 0.365, - "helm_classic/QuAC": 0.242, - "helm_classic/HellaSwag": 0.435, - "helm_classic/OpenbookQA": 0.38, - "helm_classic/TruthfulQA": 0.215, - "helm_classic/MS MARCO (TREC)": 0.29, - "helm_classic/CNN/DailyMail": 0.09, - "helm_classic/XSUM": 0.022, - "helm_classic/IMDB": 0.849, - "helm_classic/CivilComments": 0.517, - "helm_classic/RAFT": 0.423 - } - }, - { - "id": "openai/babbage-1.3B", - "name": "babbage 1.3B", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.114, - "helm_classic/MMLU": 0.235, - "helm_classic/BoolQ": 0.574, - "helm_classic/NarrativeQA": 0.491, - "helm_classic/NaturalQuestions (open-book)": 0.451, - "helm_classic/QuAC": 0.273, - "helm_classic/HellaSwag": 0.555, - "helm_classic/OpenbookQA": 0.438, - "helm_classic/TruthfulQA": 0.188, - "helm_classic/MS MARCO (TREC)": 0.317, - "helm_classic/CNN/DailyMail": 0.079, - "helm_classic/XSUM": 0.045, - "helm_classic/IMDB": 0.597, - "helm_classic/CivilComments": 0.519, - "helm_classic/RAFT": 0.455 - } - }, - { - "id": "openai/curie-6.7B", - "name": "curie 6.7B", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.247, - "helm_classic/MMLU": 0.243, - "helm_classic/BoolQ": 0.656, - "helm_classic/NarrativeQA": 0.604, - "helm_classic/NaturalQuestions (open-book)": 0.552, - "helm_classic/QuAC": 0.321, - "helm_classic/HellaSwag": 0.682, - "helm_classic/OpenbookQA": 0.502, - "helm_classic/TruthfulQA": 0.232, - "helm_classic/MS MARCO (TREC)": 0.3, - "helm_classic/CNN/DailyMail": 0.113, - "helm_classic/XSUM": 0.091, - "helm_classic/IMDB": 0.889, - "helm_classic/CivilComments": 0.539, - "helm_classic/RAFT": 0.49 - } - }, - { - "id": "openai/davinci-175B", - "name": "davinci 175B", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.538, - "helm_classic/MMLU": 0.422, - "helm_classic/BoolQ": 0.722, - "helm_classic/NarrativeQA": 0.687, - "helm_classic/NaturalQuestions (open-book)": 0.625, - "helm_classic/QuAC": 0.36, - "helm_classic/HellaSwag": 0.775, - "helm_classic/OpenbookQA": 0.586, - "helm_classic/TruthfulQA": 0.194, - "helm_classic/MS MARCO (TREC)": 0.378, - "helm_classic/CNN/DailyMail": 0.127, - "helm_classic/XSUM": 0.126, - "helm_classic/IMDB": 0.933, - "helm_classic/CivilComments": 0.532, - "helm_classic/RAFT": 0.642 - } - }, - { - "id": "openai/GPT 4o", - "name": "GPT 4o", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "apex-v1/Overall Score": 0.359 - } - }, - { - "id": "openai/GPT 5", - "name": "GPT 5", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "apex-agents/Overall Pass@1": 0.183, - "apex-agents/Overall Pass@8": 0.31, - "apex-agents/Overall Mean Score": 0.329, - "apex-agents/Investment Banking Pass@1": 0.273, - "apex-agents/Management Consulting Pass@1": 0.123, - "apex-agents/Corporate Law Pass@1": 0.153, - "apex-agents/Corporate Lawyer Mean Score": 0.382, - "ace/Overall Score": 0.561, - "ace/DIY Score": 0.55, - "ace/Food Score": 0.7, - "ace/Gaming Score": 0.575, - "apex-v1/Overall Score": 0.67, - "apex-v1/Big Law Score": 0.78, - "apex-v1/Medicine (MD) Score": 0.66, - "apex-v1/Investment Banking Score": 0.61 - } - }, - { - "id": "openai/GPT 5 Codex", - "name": "GPT 5 Codex", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "apex-agents/Corporate Lawyer Mean Score": 0.362 - } - }, - { - "id": "openai/GPT 5.1", - "name": "GPT 5.1", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "apex-agents/Corporate Lawyer Mean Score": 0.376, - "ace/Overall Score": 0.551, - "ace/DIY Score": 0.56, - "ace/Gaming Score": 0.61, - "ace/Shopping Score": 0.45, - "apex-v1/Big Law Score": 0.77 - } - }, - { - "id": "openai/GPT 5.1 Codex", - "name": "GPT 5.1 Codex", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "apex-agents/Corporate Lawyer Mean Score": 0.366 - } - }, - { - "id": "openai/GPT 5.2", - "name": "GPT 5.2", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "ace/Overall Score": 0.515, - "ace/Food Score": 0.65, - "ace/Gaming Score": 0.578, - "apex-agents/Overall Pass@1": 0.23, - "apex-agents/Overall Pass@8": 0.4, - "apex-agents/Overall Mean Score": 0.387, - "apex-agents/Investment Banking Pass@1": 0.273, - "apex-agents/Management Consulting Pass@1": 0.227, - "apex-agents/Corporate Law Pass@1": 0.189, - "apex-agents/Corporate Lawyer Mean Score": 0.443 - } - }, - { - "id": "openai/GPT 5.2 Codex", - "name": "GPT 5.2 Codex", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "apex-agents/Overall Pass@1": 0.276, - "apex-agents/Corporate Lawyer Mean Score": 0.394 - } - }, - { - "id": "openai/GPT 5.2 Pro", - "name": "GPT 5.2 Pro", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "apex-v1/Overall Score": 0.668, - "apex-v1/Consulting Score": 0.64, - "apex-v1/Medicine (MD) Score": 0.65, - "apex-v1/Investment Banking Score": 0.64 - } - }, - { - "id": "openai/GPT 5.3 Codex", - "name": "GPT 5.3 Codex", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "apex-agents/Overall Pass@1": 0.317 - } - }, - { - "id": "openai/GPT OSS 120B", - "name": "GPT OSS 120B", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "apex-agents/Overall Pass@1": 0.047, - "apex-agents/Overall Pass@8": 0.115, - "apex-agents/Overall Mean Score": 0.145, - "apex-agents/Investment Banking Pass@1": 0.027, - "apex-agents/Management Consulting Pass@1": 0.035, - "apex-agents/Corporate Law Pass@1": 0.078, - "apex-agents/Corporate Lawyer Mean Score": 0.269 - } - }, - { - "id": "openai/gpt-3.5-turbo-0125", - "name": "GPT-3.5 Turbo 0125", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_mmlu/MMLU All Subjects": 0.673, - "helm_mmlu/Abstract Algebra": 0.31, - "helm_mmlu/Anatomy": 0.696, - "helm_mmlu/College Physics": 0.471, - "helm_mmlu/Computer Security": 0.78, - "helm_mmlu/Econometrics": 0.474, - "helm_mmlu/Global Facts": 0.39, - "helm_mmlu/Jurisprudence": 0.806, - "helm_mmlu/Philosophy": 0.746, - "helm_mmlu/Professional Psychology": 0.722, - "helm_mmlu/Us Foreign Policy": 0.89, - "helm_mmlu/Astronomy": 0.75, - "helm_mmlu/Business Ethics": 0.75, - "helm_mmlu/Clinical Knowledge": 0.755, - "helm_mmlu/Conceptual Physics": 0.634, - "helm_mmlu/Electrical Engineering": 0.669, - "helm_mmlu/Elementary Mathematics": 0.534, - "helm_mmlu/Formal Logic": 0.444, - "helm_mmlu/High School World History": 0.819, - "helm_mmlu/Human Sexuality": 0.779, - "helm_mmlu/International Law": 0.81, - "helm_mmlu/Logical Fallacies": 0.779, - "helm_mmlu/Machine Learning": 0.455, - "helm_mmlu/Management": 0.835, - "helm_mmlu/Marketing": 0.91, - "helm_mmlu/Medical Genetics": 0.73, - "helm_mmlu/Miscellaneous": 0.89, - "helm_mmlu/Moral Scenarios": 0.355, - "helm_mmlu/Nutrition": 0.748, - "helm_mmlu/Prehistory": 0.735, - "helm_mmlu/Public Relations": 0.727, - "helm_mmlu/Security Studies": 0.751, - "helm_mmlu/Sociology": 0.861, - "helm_mmlu/Virology": 0.536, - "helm_mmlu/World Religions": 0.842, - "helm_mmlu/Mean win rate": 0.493, - "reward-bench/Score": 0.6534, - "reward-bench/Chat": 0.9218, - "reward-bench/Chat Hard": 0.4452, - "reward-bench/Safety": 0.6547, - "reward-bench/Reasoning": 0.5912, - "reward-bench/Prior Sets (0.5 weight)": 0.6548 - } - }, - { - "id": "openai/gpt-3.5-turbo-0301", - "name": "gpt-3.5-turbo-0301", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.76, - "helm_classic/MMLU": 0.59, - "helm_classic/BoolQ": 0.74, - "helm_classic/NarrativeQA": 0.663, - "helm_classic/NaturalQuestions (open-book)": 0.624, - "helm_classic/QuAC": 0.512, - "helm_classic/HellaSwag": -1.0, - "helm_classic/OpenbookQA": -1.0, - "helm_classic/TruthfulQA": 0.609, - "helm_classic/MS MARCO (TREC)": -1.0, - "helm_classic/CNN/DailyMail": -1.0, - "helm_classic/XSUM": -1.0, - "helm_classic/IMDB": 0.899, - "helm_classic/CivilComments": 0.674, - "helm_classic/RAFT": 0.768 - } - }, - { - "id": "openai/gpt-3.5-turbo-0613", - "name": "GPT-3.5 Turbo 0613", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.783, - "helm_classic/MMLU": 0.391, - "helm_classic/BoolQ": 0.87, - "helm_classic/NarrativeQA": 0.625, - "helm_classic/NaturalQuestions (open-book)": 0.675, - "helm_classic/QuAC": 0.485, - "helm_classic/HellaSwag": -1.0, - "helm_classic/OpenbookQA": -1.0, - "helm_classic/TruthfulQA": 0.339, - "helm_classic/MS MARCO (TREC)": -1.0, - "helm_classic/CNN/DailyMail": -1.0, - "helm_classic/XSUM": -1.0, - "helm_classic/IMDB": 0.943, - "helm_classic/CivilComments": 0.696, - "helm_classic/RAFT": 0.748, - "helm_instruct/Mean win rate": 0.689, - "helm_instruct/Anthropic RLHF dataset": 4.964, - "helm_instruct/Best ChatGPT Prompts": 4.986, - "helm_instruct/Koala test dataset": 4.987, - "helm_instruct/Open Assistant": 4.987, - "helm_instruct/Self Instruct": 4.99, - "helm_instruct/Vicuna": 4.992, - "helm_lite/Mean win rate": 0.358, - "helm_lite/NarrativeQA": 0.655, - "helm_lite/NaturalQuestions (closed-book)": 0.335, - "helm_lite/OpenbookQA": 0.838, - "helm_lite/MMLU": 0.614, - "helm_lite/MATH": 0.667, - "helm_lite/GSM8K": 0.501, - "helm_lite/LegalBench": 0.528, - "helm_lite/MedQA": 0.622, - "helm_lite/WMT 2014": 0.187, - "helm_mmlu/MMLU All Subjects": 0.689, - "helm_mmlu/Abstract Algebra": 0.38, - "helm_mmlu/Anatomy": 0.659, - "helm_mmlu/College Physics": 0.461, - "helm_mmlu/Computer Security": 0.81, - "helm_mmlu/Econometrics": 0.5, - "helm_mmlu/Global Facts": 0.37, - "helm_mmlu/Jurisprudence": 0.806, - "helm_mmlu/Philosophy": 0.759, - "helm_mmlu/Professional Psychology": 0.732, - "helm_mmlu/Us Foreign Policy": 0.88, - "helm_mmlu/Astronomy": 0.763, - "helm_mmlu/Business Ethics": 0.75, - "helm_mmlu/Clinical Knowledge": 0.777, - "helm_mmlu/Conceptual Physics": 0.613, - "helm_mmlu/Electrical Engineering": 0.648, - "helm_mmlu/Elementary Mathematics": 0.5, - "helm_mmlu/Formal Logic": 0.397, - "helm_mmlu/High School World History": 0.857, - "helm_mmlu/Human Sexuality": 0.786, - "helm_mmlu/International Law": 0.843, - "helm_mmlu/Logical Fallacies": 0.791, - "helm_mmlu/Machine Learning": 0.455, - "helm_mmlu/Management": 0.845, - "helm_mmlu/Marketing": 0.91, - "helm_mmlu/Medical Genetics": 0.8, - "helm_mmlu/Miscellaneous": 0.893, - "helm_mmlu/Moral Scenarios": 0.404, - "helm_mmlu/Nutrition": 0.758, - "helm_mmlu/Prehistory": 0.787, - "helm_mmlu/Public Relations": 0.745, - "helm_mmlu/Security Studies": 0.8, - "helm_mmlu/Sociology": 0.871, - "helm_mmlu/Virology": 0.542, - "helm_mmlu/World Religions": 0.836, - "helm_mmlu/Mean win rate": 0.589 - } - }, - { - "id": "openai/gpt-4-0125-preview", - "name": "openai/gpt-4-0125-preview", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.8434, - "reward-bench/Chat": 0.9525, - "reward-bench/Chat Hard": 0.7434, - "reward-bench/Safety": 0.8757, - "reward-bench/Reasoning": 0.8692, - "reward-bench/Prior Sets (0.5 weight)": 0.7085 - } - }, - { - "id": "openai/gpt-4-0314", - "name": "GPT-4 0314", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_instruct/Mean win rate": 0.611, - "helm_instruct/Anthropic RLHF dataset": 4.934, - "helm_instruct/Best ChatGPT Prompts": 4.973, - "helm_instruct/Koala test dataset": 4.966, - "helm_instruct/Open Assistant": 4.986, - "helm_instruct/Self Instruct": 4.976, - "helm_instruct/Vicuna": 4.995 - } - }, - { - "id": "openai/gpt-4-0613", - "name": "GPT-4 0613", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.867, - "helm_lite/NarrativeQA": 0.768, - "helm_lite/NaturalQuestions (closed-book)": 0.457, - "helm_lite/OpenbookQA": 0.96, - "helm_lite/MMLU": 0.735, - "helm_lite/MATH": 0.802, - "helm_lite/GSM8K": 0.932, - "helm_lite/LegalBench": 0.713, - "helm_lite/MedQA": 0.815, - "helm_lite/WMT 2014": 0.211, - "helm_mmlu/MMLU All Subjects": 0.824, - "helm_mmlu/Abstract Algebra": 0.63, - "helm_mmlu/Anatomy": 0.8, - "helm_mmlu/College Physics": 0.627, - "helm_mmlu/Computer Security": 0.86, - "helm_mmlu/Econometrics": 0.684, - "helm_mmlu/Global Facts": 0.62, - "helm_mmlu/Jurisprudence": 0.889, - "helm_mmlu/Philosophy": 0.859, - "helm_mmlu/Professional Psychology": 0.891, - "helm_mmlu/Us Foreign Policy": 0.95, - "helm_mmlu/Astronomy": 0.934, - "helm_mmlu/Business Ethics": 0.79, - "helm_mmlu/Clinical Knowledge": 0.845, - "helm_mmlu/Conceptual Physics": 0.868, - "helm_mmlu/Electrical Engineering": 0.786, - "helm_mmlu/Elementary Mathematics": 0.807, - "helm_mmlu/Formal Logic": 0.643, - "helm_mmlu/High School World History": 0.945, - "helm_mmlu/Human Sexuality": 0.908, - "helm_mmlu/International Law": 0.917, - "helm_mmlu/Logical Fallacies": 0.871, - "helm_mmlu/Machine Learning": 0.759, - "helm_mmlu/Management": 0.932, - "helm_mmlu/Marketing": 0.962, - "helm_mmlu/Medical Genetics": 0.94, - "helm_mmlu/Miscellaneous": 0.949, - "helm_mmlu/Moral Scenarios": 0.902, - "helm_mmlu/Nutrition": 0.892, - "helm_mmlu/Prehistory": 0.926, - "helm_mmlu/Public Relations": 0.745, - "helm_mmlu/Security Studies": 0.861, - "helm_mmlu/Sociology": 0.93, - "helm_mmlu/Virology": 0.596, - "helm_mmlu/World Religions": 0.877, - "helm_mmlu/Mean win rate": 0.517 - } - }, - { - "id": "openai/gpt-4-1-2025-04-14-fc", - "name": "GPT-4.1-2025-04-14 (FC)", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 20.0, - "bfcl/bfcl.overall.overall_accuracy": 53.96, - "bfcl/bfcl.overall.total_cost_usd": 100.75, - "bfcl/bfcl.overall.latency_mean_s": 1.63, - "bfcl/bfcl.overall.latency_std_s": 3.05, - "bfcl/bfcl.overall.latency_p95_s": 4.01, - "bfcl/bfcl.non_live.ast_accuracy": 82.79, - "bfcl/bfcl.non_live.simple_ast_accuracy": 72.67, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 89.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 88.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 81.5, - "bfcl/bfcl.live.live_accuracy": 69.95, - "bfcl/bfcl.live.live_simple_ast_accuracy": 69.38, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 70.28, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 56.25, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 70.83, - "bfcl/bfcl.multi_turn.accuracy": 38.88, - "bfcl/bfcl.multi_turn.base_accuracy": 47.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 32.5, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 32.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 43.0, - "bfcl/bfcl.web_search.accuracy": 68.0, - "bfcl/bfcl.web_search.base_accuracy": 67.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 69.0, - "bfcl/bfcl.memory.accuracy": 23.87, - "bfcl/bfcl.memory.kv_accuracy": 16.13, - "bfcl/bfcl.memory.vector_accuracy": 18.06, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 37.42, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 87.5, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 86.52 - } - }, - { - "id": "openai/gpt-4-1-2025-04-14-prompt", - "name": "GPT-4.1-2025-04-14 (Prompt)", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 45.0, - "bfcl/bfcl.overall.overall_accuracy": 39.38, - "bfcl/bfcl.overall.total_cost_usd": 145.85, - "bfcl/bfcl.overall.latency_mean_s": 1.2, - "bfcl/bfcl.overall.latency_std_s": 3.23, - "bfcl/bfcl.overall.latency_p95_s": 2.53, - "bfcl/bfcl.non_live.ast_accuracy": 88.69, - "bfcl/bfcl.non_live.simple_ast_accuracy": 78.25, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 93.5, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 94.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 89.0, - "bfcl/bfcl.live.live_accuracy": 78.9, - "bfcl/bfcl.live.live_simple_ast_accuracy": 84.88, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 77.4, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 87.5, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 75.0, - "bfcl/bfcl.multi_turn.accuracy": 9.75, - "bfcl/bfcl.multi_turn.base_accuracy": 10.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 11.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 8.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 9.5, - "bfcl/bfcl.web_search.accuracy": 35.0, - "bfcl/bfcl.web_search.base_accuracy": 40.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 30.0, - "bfcl/bfcl.memory.accuracy": 21.51, - "bfcl/bfcl.memory.kv_accuracy": 9.68, - "bfcl/bfcl.memory.vector_accuracy": 19.35, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 35.48, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 100.0, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 83.99, - "bfcl/bfcl.format_sensitivity.max_delta": 23.5, - "bfcl/bfcl.format_sensitivity.stddev": 6.18 - } - }, - { - "id": "openai/gpt-4-1-mini-2025-04-14-fc", - "name": "GPT-4.1-mini-2025-04-14 (FC)", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 27.0, - "bfcl/bfcl.overall.overall_accuracy": 50.45, - "bfcl/bfcl.overall.total_cost_usd": 19.25, - "bfcl/bfcl.overall.latency_mean_s": 1.32, - "bfcl/bfcl.overall.latency_std_s": 3.65, - "bfcl/bfcl.overall.latency_p95_s": 2.4, - "bfcl/bfcl.non_live.ast_accuracy": 83.83, - "bfcl/bfcl.non_live.simple_ast_accuracy": 73.33, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 89.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 91.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 82.0, - "bfcl/bfcl.live.live_accuracy": 68.84, - "bfcl/bfcl.live.live_simple_ast_accuracy": 67.05, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 69.8, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 43.75, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 62.5, - "bfcl/bfcl.multi_turn.accuracy": 34.13, - "bfcl/bfcl.multi_turn.base_accuracy": 43.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 22.5, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 30.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 40.0, - "bfcl/bfcl.web_search.accuracy": 57.0, - "bfcl/bfcl.web_search.base_accuracy": 62.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 52.0, - "bfcl/bfcl.memory.accuracy": 26.88, - "bfcl/bfcl.memory.kv_accuracy": 22.58, - "bfcl/bfcl.memory.vector_accuracy": 16.13, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 41.94, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 81.25, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 81.69 - } - }, - { - "id": "openai/gpt-4-1-mini-2025-04-14-prompt", - "name": "GPT-4.1-mini-2025-04-14 (Prompt)", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 67.0, - "bfcl/bfcl.overall.overall_accuracy": 29.73, - "bfcl/bfcl.overall.total_cost_usd": 20.52, - "bfcl/bfcl.overall.latency_mean_s": 1.36, - "bfcl/bfcl.overall.latency_std_s": 4.5, - "bfcl/bfcl.overall.latency_p95_s": 3.38, - "bfcl/bfcl.non_live.ast_accuracy": 84.6, - "bfcl/bfcl.non_live.simple_ast_accuracy": 74.92, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 92.5, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 87.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 83.5, - "bfcl/bfcl.live.live_accuracy": 74.76, - "bfcl/bfcl.live.live_simple_ast_accuracy": 80.62, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 73.31, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 81.25, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 70.83, - "bfcl/bfcl.multi_turn.accuracy": 2.5, - "bfcl/bfcl.multi_turn.base_accuracy": 1.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 4.5, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 2.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 1.5, - "bfcl/bfcl.web_search.accuracy": 4.0, - "bfcl/bfcl.web_search.base_accuracy": 7.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 1.0, - "bfcl/bfcl.memory.accuracy": 24.3, - "bfcl/bfcl.memory.kv_accuracy": 20.65, - "bfcl/bfcl.memory.vector_accuracy": 13.55, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 38.71, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 87.5, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 73.88, - "bfcl/bfcl.format_sensitivity.max_delta": 45.0, - "bfcl/bfcl.format_sensitivity.stddev": 13.33 - } - }, - { - "id": "openai/gpt-4-1-nano-2025-04-14-fc", - "name": "GPT-4.1-nano-2025-04-14 (FC)", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 58.0, - "bfcl/bfcl.overall.overall_accuracy": 33.05, - "bfcl/bfcl.overall.total_cost_usd": 5.66, - "bfcl/bfcl.overall.latency_mean_s": 1.44, - "bfcl/bfcl.overall.latency_std_s": 10.84, - "bfcl/bfcl.overall.latency_p95_s": 2.26, - "bfcl/bfcl.non_live.ast_accuracy": 72.98, - "bfcl/bfcl.non_live.simple_ast_accuracy": 59.92, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 79.5, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 84.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 68.5, - "bfcl/bfcl.live.live_accuracy": 60.77, - "bfcl/bfcl.live.live_simple_ast_accuracy": 58.14, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 61.44, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 68.75, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 54.17, - "bfcl/bfcl.multi_turn.accuracy": 23.62, - "bfcl/bfcl.multi_turn.base_accuracy": 39.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 7.5, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 17.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 30.0, - "bfcl/bfcl.web_search.accuracy": 11.0, - "bfcl/bfcl.web_search.base_accuracy": 13.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 9.0, - "bfcl/bfcl.memory.accuracy": 18.92, - "bfcl/bfcl.memory.kv_accuracy": 10.32, - "bfcl/bfcl.memory.vector_accuracy": 19.35, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 27.1, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 93.75, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 66.0 - } - }, - { - "id": "openai/gpt-4-1-nano-2025-04-14-prompt", - "name": "GPT-4.1-nano-2025-04-14 (Prompt)", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 90.0, - "bfcl/bfcl.overall.overall_accuracy": 24.88, - "bfcl/bfcl.overall.total_cost_usd": 7.42, - "bfcl/bfcl.overall.latency_mean_s": 1.02, - "bfcl/bfcl.overall.latency_std_s": 7.3, - "bfcl/bfcl.overall.latency_p95_s": 1.88, - "bfcl/bfcl.non_live.ast_accuracy": 72.44, - "bfcl/bfcl.non_live.simple_ast_accuracy": 68.75, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 63.5, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 85.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 72.5, - "bfcl/bfcl.live.live_accuracy": 50.33, - "bfcl/bfcl.live.live_simple_ast_accuracy": 63.18, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 46.53, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 87.5, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 54.17, - "bfcl/bfcl.multi_turn.accuracy": 2.0, - "bfcl/bfcl.multi_turn.base_accuracy": 2.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 1.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 2.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 2.0, - "bfcl/bfcl.web_search.accuracy": 1.5, - "bfcl/bfcl.web_search.base_accuracy": 2.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 1.0, - "bfcl/bfcl.memory.accuracy": 16.77, - "bfcl/bfcl.memory.kv_accuracy": 9.03, - "bfcl/bfcl.memory.vector_accuracy": 14.19, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 27.1, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 68.75, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 83.44, - "bfcl/bfcl.format_sensitivity.max_delta": 73.0, - "bfcl/bfcl.format_sensitivity.stddev": 17.08 - } - }, - { - "id": "openai/gpt-4-1106-preview", - "name": "GPT-4 Turbo 1106 preview", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.698, - "helm_lite/NarrativeQA": 0.727, - "helm_lite/NaturalQuestions (closed-book)": 0.435, - "helm_lite/OpenbookQA": 0.95, - "helm_lite/MMLU": 0.699, - "helm_lite/MATH": 0.857, - "helm_lite/GSM8K": 0.668, - "helm_lite/LegalBench": 0.626, - "helm_lite/MedQA": 0.817, - "helm_lite/WMT 2014": 0.205, - "helm_mmlu/MMLU All Subjects": 0.796, - "helm_mmlu/Abstract Algebra": 0.53, - "helm_mmlu/Anatomy": 0.807, - "helm_mmlu/College Physics": 0.402, - "helm_mmlu/Computer Security": 0.86, - "helm_mmlu/Econometrics": 0.675, - "helm_mmlu/Global Facts": 0.58, - "helm_mmlu/Jurisprudence": 0.889, - "helm_mmlu/Philosophy": 0.852, - "helm_mmlu/Professional Psychology": 0.887, - "helm_mmlu/Us Foreign Policy": 0.96, - "helm_mmlu/Astronomy": 0.941, - "helm_mmlu/Business Ethics": 0.78, - "helm_mmlu/Clinical Knowledge": 0.864, - "helm_mmlu/Conceptual Physics": 0.894, - "helm_mmlu/Electrical Engineering": 0.772, - "helm_mmlu/Elementary Mathematics": 0.638, - "helm_mmlu/Formal Logic": 0.651, - "helm_mmlu/High School World History": 0.958, - "helm_mmlu/Human Sexuality": 0.908, - "helm_mmlu/International Law": 0.926, - "helm_mmlu/Logical Fallacies": 0.865, - "helm_mmlu/Machine Learning": 0.723, - "helm_mmlu/Management": 0.913, - "helm_mmlu/Marketing": 0.932, - "helm_mmlu/Medical Genetics": 0.93, - "helm_mmlu/Miscellaneous": 0.946, - "helm_mmlu/Moral Scenarios": 0.816, - "helm_mmlu/Nutrition": 0.879, - "helm_mmlu/Prehistory": 0.917, - "helm_mmlu/Public Relations": 0.782, - "helm_mmlu/Security Studies": 0.841, - "helm_mmlu/Sociology": 0.925, - "helm_mmlu/Virology": 0.59, - "helm_mmlu/World Religions": 0.854, - "helm_mmlu/Mean win rate": 0.416 - } - }, - { - "id": "openai/gpt-4-turbo-2024-04-09", - "name": "GPT-4 Turbo 2024-04-09", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.864, - "helm_lite/NarrativeQA": 0.761, - "helm_lite/NaturalQuestions (closed-book)": 0.482, - "helm_lite/OpenbookQA": 0.97, - "helm_lite/MMLU": 0.711, - "helm_lite/MATH": 0.833, - "helm_lite/GSM8K": 0.824, - "helm_lite/LegalBench": 0.727, - "helm_lite/MedQA": 0.783, - "helm_lite/WMT 2014": 0.218, - "helm_mmlu/MMLU All Subjects": 0.813, - "helm_mmlu/Abstract Algebra": 0.56, - "helm_mmlu/Anatomy": 0.822, - "helm_mmlu/College Physics": 0.539, - "helm_mmlu/Computer Security": 0.83, - "helm_mmlu/Econometrics": 0.675, - "helm_mmlu/Global Facts": 0.58, - "helm_mmlu/Jurisprudence": 0.88, - "helm_mmlu/Philosophy": 0.868, - "helm_mmlu/Professional Psychology": 0.873, - "helm_mmlu/Us Foreign Policy": 0.96, - "helm_mmlu/Astronomy": 0.941, - "helm_mmlu/Business Ethics": 0.82, - "helm_mmlu/Clinical Knowledge": 0.83, - "helm_mmlu/Conceptual Physics": 0.894, - "helm_mmlu/Electrical Engineering": 0.752, - "helm_mmlu/Elementary Mathematics": 0.72, - "helm_mmlu/Formal Logic": 0.706, - "helm_mmlu/High School World History": 0.941, - "helm_mmlu/Human Sexuality": 0.901, - "helm_mmlu/International Law": 0.942, - "helm_mmlu/Logical Fallacies": 0.871, - "helm_mmlu/Machine Learning": 0.741, - "helm_mmlu/Management": 0.883, - "helm_mmlu/Marketing": 0.949, - "helm_mmlu/Medical Genetics": 0.92, - "helm_mmlu/Miscellaneous": 0.945, - "helm_mmlu/Moral Scenarios": 0.803, - "helm_mmlu/Nutrition": 0.892, - "helm_mmlu/Prehistory": 0.92, - "helm_mmlu/Public Relations": 0.755, - "helm_mmlu/Security Studies": 0.8, - "helm_mmlu/Sociology": 0.915, - "helm_mmlu/Virology": 0.602, - "helm_mmlu/World Religions": 0.848, - "helm_mmlu/Mean win rate": 0.351, - "reward-bench/Score": 0.8395, - "reward-bench/Chat": 0.9525, - "reward-bench/Chat Hard": 0.7544, - "reward-bench/Safety": 0.8757, - "reward-bench/Reasoning": 0.827, - "reward-bench/Prior Sets (0.5 weight)": 0.7363 - } - }, - { - "id": "openai/gpt-4.1", - "name": "openai/gpt-4.1", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "livecodebenchpro/Hard Problems": 0.0, - "livecodebenchpro/Medium Problems": 0.0, - "livecodebenchpro/Easy Problems": 0.19718309859154928 - } - }, - { - "id": "openai/gpt-4.1-2025-04-14", - "name": "gpt-4.1-2025-04-14", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "global-mmlu-lite/Global MMLU Lite": 0.8755, - "global-mmlu-lite/Culturally Sensitive": 0.8541, - "global-mmlu-lite/Culturally Agnostic": 0.8969, - "global-mmlu-lite/Arabic": 0.88, - "global-mmlu-lite/English": 0.8825, - "global-mmlu-lite/Bengali": 0.8625, - "global-mmlu-lite/German": 0.875, - "global-mmlu-lite/French": 0.8875, - "global-mmlu-lite/Hindi": 0.8775, - "global-mmlu-lite/Indonesian": 0.885, - "global-mmlu-lite/Italian": 0.88, - "global-mmlu-lite/Japanese": 0.8725, - "global-mmlu-lite/Korean": 0.87, - "global-mmlu-lite/Portuguese": 0.875, - "global-mmlu-lite/Spanish": 0.885, - "global-mmlu-lite/Swahili": 0.8725, - "global-mmlu-lite/Yoruba": 0.875, - "global-mmlu-lite/Chinese": 0.87, - "global-mmlu-lite/Burmese": 0.8575, - "helm_capabilities/Mean score": 0.727, - "helm_capabilities/MMLU-Pro": 0.811, - "helm_capabilities/GPQA": 0.659, - "helm_capabilities/IFEval": 0.838, - "helm_capabilities/WildBench": 0.854, - "helm_capabilities/Omni-MATH": 0.471, - "reward-bench/Score": 0.7232, - "reward-bench/Factuality": 0.8289, - "reward-bench/Precise IF": 0.3974, - "reward-bench/Math": 0.6521, - "reward-bench/Safety": 0.8726, - "reward-bench/Focus": 0.7338, - "reward-bench/Ties": 0.8542 - } - }, - { - "id": "openai/gpt-4.1-mini-2025-04-14", - "name": "GPT-4.1 mini 2025-04-14", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.726, - "helm_capabilities/MMLU-Pro": 0.783, - "helm_capabilities/GPQA": 0.614, - "helm_capabilities/IFEval": 0.904, - "helm_capabilities/WildBench": 0.838, - "helm_capabilities/Omni-MATH": 0.491, - "reward-bench/Score": 0.6573, - "reward-bench/Factuality": 0.6084, - "reward-bench/Precise IF": 0.4125, - "reward-bench/Math": 0.7213, - "reward-bench/Safety": 0.7265, - "reward-bench/Focus": 0.7354, - "reward-bench/Ties": 0.74 - } - }, - { - "id": "openai/gpt-4.1-nano-2025-04-14", - "name": "GPT-4.1 nano 2025-04-14", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.616, - "helm_capabilities/MMLU-Pro": 0.55, - "helm_capabilities/GPQA": 0.507, - "helm_capabilities/IFEval": 0.843, - "helm_capabilities/WildBench": 0.811, - "helm_capabilities/Omni-MATH": 0.367, - "reward-bench/Score": 0.4849, - "reward-bench/Factuality": 0.4646, - "reward-bench/Precise IF": 0.2578, - "reward-bench/Math": 0.5041, - "reward-bench/Safety": 0.7156, - "reward-bench/Focus": 0.466, - "reward-bench/Ties": 0.5015 - } - }, - { - "id": "openai/gpt-4o-2024-05-13", - "name": "GPT-4o 2024-05-13", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.938, - "helm_lite/NarrativeQA": 0.804, - "helm_lite/NaturalQuestions (closed-book)": 0.501, - "helm_lite/OpenbookQA": 0.966, - "helm_lite/MMLU": 0.748, - "helm_lite/MATH": 0.829, - "helm_lite/GSM8K": 0.905, - "helm_lite/LegalBench": 0.733, - "helm_lite/MedQA": 0.857, - "helm_lite/WMT 2014": 0.231, - "helm_mmlu/MMLU All Subjects": 0.842, - "helm_mmlu/Abstract Algebra": 0.66, - "helm_mmlu/Anatomy": 0.911, - "helm_mmlu/College Physics": 0.686, - "helm_mmlu/Computer Security": 0.85, - "helm_mmlu/Econometrics": 0.693, - "helm_mmlu/Global Facts": 0.64, - "helm_mmlu/Jurisprudence": 0.898, - "helm_mmlu/Philosophy": 0.9, - "helm_mmlu/Professional Psychology": 0.905, - "helm_mmlu/Us Foreign Policy": 0.96, - "helm_mmlu/Astronomy": 0.941, - "helm_mmlu/Business Ethics": 0.85, - "helm_mmlu/Clinical Knowledge": 0.894, - "helm_mmlu/Conceptual Physics": 0.911, - "helm_mmlu/Electrical Engineering": 0.807, - "helm_mmlu/Elementary Mathematics": 0.741, - "helm_mmlu/Formal Logic": 0.683, - "helm_mmlu/High School World History": 0.945, - "helm_mmlu/Human Sexuality": 0.908, - "helm_mmlu/International Law": 0.934, - "helm_mmlu/Logical Fallacies": 0.883, - "helm_mmlu/Machine Learning": 0.768, - "helm_mmlu/Management": 0.942, - "helm_mmlu/Marketing": 0.936, - "helm_mmlu/Medical Genetics": 0.96, - "helm_mmlu/Miscellaneous": 0.954, - "helm_mmlu/Moral Scenarios": 0.841, - "helm_mmlu/Nutrition": 0.899, - "helm_mmlu/Prehistory": 0.938, - "helm_mmlu/Public Relations": 0.809, - "helm_mmlu/Security Studies": 0.837, - "helm_mmlu/Sociology": 0.94, - "helm_mmlu/Virology": 0.596, - "helm_mmlu/World Religions": 0.889, - "helm_mmlu/Mean win rate": 0.671, - "reward-bench/Score": 0.8327, - "reward-bench/Chat": 0.9665, - "reward-bench/Chat Hard": 0.7039, - "reward-bench/Safety": 0.8649, - "reward-bench/Reasoning": 0.8487, - "reward-bench/Prior Sets (0.5 weight)": 0.7262 - } - }, - { - "id": "openai/gpt-4o-2024-08-06", - "name": "GPT-4o 2024-08-06", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.928, - "helm_lite/NarrativeQA": 0.795, - "helm_lite/NaturalQuestions (closed-book)": 0.496, - "helm_lite/OpenbookQA": 0.968, - "helm_lite/MMLU": 0.738, - "helm_lite/MATH": 0.853, - "helm_lite/GSM8K": 0.909, - "helm_lite/LegalBench": 0.721, - "helm_lite/MedQA": 0.863, - "helm_lite/WMT 2014": 0.225, - "helm_mmlu/MMLU All Subjects": 0.843, - "helm_mmlu/Abstract Algebra": 0.58, - "helm_mmlu/Anatomy": 0.911, - "helm_mmlu/College Physics": 0.686, - "helm_mmlu/Computer Security": 0.85, - "helm_mmlu/Econometrics": 0.711, - "helm_mmlu/Global Facts": 0.69, - "helm_mmlu/Jurisprudence": 0.907, - "helm_mmlu/Philosophy": 0.894, - "helm_mmlu/Professional Psychology": 0.899, - "helm_mmlu/Us Foreign Policy": 0.95, - "helm_mmlu/Astronomy": 0.947, - "helm_mmlu/Business Ethics": 0.89, - "helm_mmlu/Clinical Knowledge": 0.894, - "helm_mmlu/Conceptual Physics": 0.923, - "helm_mmlu/Electrical Engineering": 0.793, - "helm_mmlu/Elementary Mathematics": 0.775, - "helm_mmlu/Formal Logic": 0.675, - "helm_mmlu/High School World History": 0.941, - "helm_mmlu/Human Sexuality": 0.901, - "helm_mmlu/International Law": 0.942, - "helm_mmlu/Logical Fallacies": 0.902, - "helm_mmlu/Machine Learning": 0.777, - "helm_mmlu/Management": 0.913, - "helm_mmlu/Marketing": 0.94, - "helm_mmlu/Medical Genetics": 0.98, - "helm_mmlu/Miscellaneous": 0.958, - "helm_mmlu/Moral Scenarios": 0.802, - "helm_mmlu/Nutrition": 0.905, - "helm_mmlu/Prehistory": 0.935, - "helm_mmlu/Public Relations": 0.782, - "helm_mmlu/Security Studies": 0.833, - "helm_mmlu/Sociology": 0.945, - "helm_mmlu/Virology": 0.578, - "helm_mmlu/World Religions": 0.883, - "helm_mmlu/Mean win rate": 0.52, - "reward-bench/Score": 0.6493, - "reward-bench/Chat": 0.9609, - "reward-bench/Chat Hard": 0.761, - "reward-bench/Safety": 0.8619, - "reward-bench/Reasoning": 0.8661, - "reward-bench/Factuality": 0.5684, - "reward-bench/Precise IF": 0.3312, - "reward-bench/Math": 0.623, - "reward-bench/Focus": 0.7293, - "reward-bench/Ties": 0.7819 - } - }, - { - "id": "openai/gpt-4o-2024-11-20", - "name": "GPT-4o 2024-11-20", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.634, - "helm_capabilities/MMLU-Pro": 0.713, - "helm_capabilities/GPQA": 0.52, - "helm_capabilities/IFEval": 0.817, - "helm_capabilities/WildBench": 0.828, - "helm_capabilities/Omni-MATH": 0.293, - "livecodebenchpro/Hard Problems": 0.0, - "livecodebenchpro/Medium Problems": 0.0, - "livecodebenchpro/Easy Problems": 0.07042253521126761 - } - }, - { - "id": "openai/gpt-4o-mini-2024-07-18", - "name": "GPT-4o mini 2024-07-18", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.565, - "helm_capabilities/MMLU-Pro": 0.603, - "helm_capabilities/GPQA": 0.368, - "helm_capabilities/IFEval": 0.782, - "helm_capabilities/WildBench": 0.791, - "helm_capabilities/Omni-MATH": 0.28, - "helm_lite/Mean win rate": 0.701, - "helm_lite/NarrativeQA": 0.768, - "helm_lite/NaturalQuestions (closed-book)": 0.386, - "helm_lite/OpenbookQA": 0.92, - "helm_lite/MMLU": 0.668, - "helm_lite/MATH": 0.802, - "helm_lite/GSM8K": 0.843, - "helm_lite/LegalBench": 0.653, - "helm_lite/MedQA": 0.748, - "helm_lite/WMT 2014": 0.206, - "helm_mmlu/MMLU All Subjects": 0.767, - "helm_mmlu/Abstract Algebra": 0.42, - "helm_mmlu/Anatomy": 0.77, - "helm_mmlu/College Physics": 0.559, - "helm_mmlu/Computer Security": 0.85, - "helm_mmlu/Econometrics": 0.649, - "helm_mmlu/Global Facts": 0.45, - "helm_mmlu/Jurisprudence": 0.87, - "helm_mmlu/Philosophy": 0.772, - "helm_mmlu/Professional Psychology": 0.833, - "helm_mmlu/Us Foreign Policy": 0.91, - "helm_mmlu/Astronomy": 0.849, - "helm_mmlu/Business Ethics": 0.79, - "helm_mmlu/Clinical Knowledge": 0.845, - "helm_mmlu/Conceptual Physics": 0.791, - "helm_mmlu/Electrical Engineering": 0.731, - "helm_mmlu/Elementary Mathematics": 0.651, - "helm_mmlu/Formal Logic": 0.556, - "helm_mmlu/High School World History": 0.903, - "helm_mmlu/Human Sexuality": 0.863, - "helm_mmlu/International Law": 0.926, - "helm_mmlu/Logical Fallacies": 0.871, - "helm_mmlu/Machine Learning": 0.616, - "helm_mmlu/Management": 0.845, - "helm_mmlu/Marketing": 0.927, - "helm_mmlu/Medical Genetics": 0.89, - "helm_mmlu/Miscellaneous": 0.913, - "helm_mmlu/Moral Scenarios": 0.485, - "helm_mmlu/Nutrition": 0.827, - "helm_mmlu/Prehistory": 0.833, - "helm_mmlu/Public Relations": 0.791, - "helm_mmlu/Security Studies": 0.788, - "helm_mmlu/Sociology": 0.9, - "helm_mmlu/Virology": 0.536, - "helm_mmlu/World Religions": 0.86, - "helm_mmlu/Mean win rate": 0.774, - "reward-bench/Score": 0.8007, - "reward-bench/Factuality": 0.4105, - "reward-bench/Precise IF": 0.3438, - "reward-bench/Math": 0.5191, - "reward-bench/Safety": 0.8081, - "reward-bench/Focus": 0.7414, - "reward-bench/Ties": 0.6962, - "reward-bench/Chat": 0.9497, - "reward-bench/Chat Hard": 0.6075, - "reward-bench/Reasoning": 0.8374 - } - }, - { - "id": "openai/gpt-5", - "name": "GPT-5", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "terminal-bench-2.0/terminal-bench-2.0": 49.6 - } - }, - { - "id": "openai/gpt-5-2-2025-12-11-fc", - "name": "GPT-5.2-2025-12-11 (FC)", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 16.0, - "bfcl/bfcl.overall.overall_accuracy": 55.87, - "bfcl/bfcl.overall.total_cost_usd": 85.65, - "bfcl/bfcl.overall.latency_mean_s": 2.23, - "bfcl/bfcl.overall.latency_std_s": 9.75, - "bfcl/bfcl.overall.latency_p95_s": 5.26, - "bfcl/bfcl.non_live.ast_accuracy": 81.85, - "bfcl/bfcl.non_live.simple_ast_accuracy": 72.92, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 88.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 89.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 77.5, - "bfcl/bfcl.live.live_accuracy": 70.39, - "bfcl/bfcl.live.live_simple_ast_accuracy": 71.71, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 70.37, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 68.75, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 58.33, - "bfcl/bfcl.multi_turn.accuracy": 28.12, - "bfcl/bfcl.multi_turn.base_accuracy": 36.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 18.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 27.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 30.5, - "bfcl/bfcl.web_search.accuracy": 75.5, - "bfcl/bfcl.web_search.base_accuracy": 78.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 73.0, - "bfcl/bfcl.memory.accuracy": 45.81, - "bfcl/bfcl.memory.kv_accuracy": 33.55, - "bfcl/bfcl.memory.vector_accuracy": 43.23, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 60.65, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 75.0, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 79.42 - } - }, - { - "id": "openai/gpt-5-2-2025-12-11-prompt", - "name": "GPT-5.2-2025-12-11 (Prompt)", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 38.0, - "bfcl/bfcl.overall.overall_accuracy": 45.27, - "bfcl/bfcl.overall.total_cost_usd": 164.58, - "bfcl/bfcl.overall.latency_mean_s": 4.21, - "bfcl/bfcl.overall.latency_std_s": 20.93, - "bfcl/bfcl.overall.latency_p95_s": 10.58, - "bfcl/bfcl.non_live.ast_accuracy": 78.29, - "bfcl/bfcl.non_live.simple_ast_accuracy": 71.17, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 83.5, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 84.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 74.5, - "bfcl/bfcl.live.live_accuracy": 67.14, - "bfcl/bfcl.live.live_simple_ast_accuracy": 77.91, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 64.58, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 75.0, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 58.33, - "bfcl/bfcl.multi_turn.accuracy": 43.75, - "bfcl/bfcl.multi_turn.base_accuracy": 54.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 40.5, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 33.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 46.5, - "bfcl/bfcl.web_search.accuracy": 40.5, - "bfcl/bfcl.web_search.base_accuracy": 45.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 36.0, - "bfcl/bfcl.memory.accuracy": 3.87, - "bfcl/bfcl.memory.kv_accuracy": 2.58, - "bfcl/bfcl.memory.vector_accuracy": 1.94, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 7.1, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 75.0, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 87.26, - "bfcl/bfcl.format_sensitivity.max_delta": 13.0, - "bfcl/bfcl.format_sensitivity.stddev": 3.25 - } - }, - { - "id": "openai/gpt-5-2025-08-07", - "name": "gpt-5-2025-08-07", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "global-mmlu-lite/Global MMLU Lite": 0.8895, - "global-mmlu-lite/Culturally Sensitive": 0.8913, - "global-mmlu-lite/Culturally Agnostic": 0.8878, - "global-mmlu-lite/Arabic": 0.8925, - "global-mmlu-lite/English": 0.8725, - "global-mmlu-lite/Bengali": 0.9, - "global-mmlu-lite/German": 0.91, - "global-mmlu-lite/French": 0.9075, - "global-mmlu-lite/Hindi": 0.865, - "global-mmlu-lite/Indonesian": 0.795, - "global-mmlu-lite/Italian": 0.9075, - "global-mmlu-lite/Japanese": 0.8875, - "global-mmlu-lite/Korean": 0.915, - "global-mmlu-lite/Portuguese": 0.8875, - "global-mmlu-lite/Spanish": 0.905, - "global-mmlu-lite/Swahili": 0.865, - "global-mmlu-lite/Yoruba": 0.9125, - "global-mmlu-lite/Chinese": 0.895, - "global-mmlu-lite/Burmese": 0.915, - "helm_capabilities/Mean score": 0.807, - "helm_capabilities/MMLU-Pro": 0.863, - "helm_capabilities/GPQA": 0.791, - "helm_capabilities/IFEval": 0.875, - "helm_capabilities/WildBench": 0.857, - "helm_capabilities/Omni-MATH": 0.647, - "livecodebenchpro/Hard Problems": 0.0423, - "livecodebenchpro/Medium Problems": 0.4085, - "livecodebenchpro/Easy Problems": 0.9014 - } - }, - { - "id": "openai/gpt-5-codex", - "name": "GPT-5-Codex", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "terminal-bench-2.0/terminal-bench-2.0": 44.3 - } - }, - { - "id": "openai/gpt-5-mini", - "name": "GPT-5-Mini", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "terminal-bench-2.0/terminal-bench-2.0": 31.9 - } - }, - { - "id": "openai/gpt-5-mini-2025-08-07", - "name": "GPT-5 mini 2025-08-07", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.819, - "helm_capabilities/MMLU-Pro": 0.835, - "helm_capabilities/GPQA": 0.756, - "helm_capabilities/IFEval": 0.927, - "helm_capabilities/WildBench": 0.855, - "helm_capabilities/Omni-MATH": 0.722 - } - }, - { - "id": "openai/gpt-5-mini-2025-08-07-fc", - "name": "GPT-5-mini-2025-08-07 (FC)", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 17.0, - "bfcl/bfcl.overall.overall_accuracy": 55.46, - "bfcl/bfcl.overall.total_cost_usd": 22.18, - "bfcl/bfcl.overall.latency_mean_s": 8.32, - "bfcl/bfcl.overall.latency_std_s": 17.35, - "bfcl/bfcl.overall.latency_p95_s": 19.8, - "bfcl/bfcl.non_live.ast_accuracy": 69.85, - "bfcl/bfcl.non_live.simple_ast_accuracy": 59.92, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 69.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 80.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 70.5, - "bfcl/bfcl.live.live_accuracy": 58.62, - "bfcl/bfcl.live.live_simple_ast_accuracy": 62.02, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 58.02, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 62.5, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 45.83, - "bfcl/bfcl.multi_turn.accuracy": 27.5, - "bfcl/bfcl.multi_turn.base_accuracy": 36.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 17.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 23.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 33.0, - "bfcl/bfcl.web_search.accuracy": 82.0, - "bfcl/bfcl.web_search.base_accuracy": 87.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 77.0, - "bfcl/bfcl.memory.accuracy": 44.3, - "bfcl/bfcl.memory.kv_accuracy": 36.77, - "bfcl/bfcl.memory.vector_accuracy": 43.87, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 52.26, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 62.5, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 91.01 - } - }, - { - "id": "openai/gpt-5-mini-2025-08-07-prompt", - "name": "GPT-5-mini-2025-08-07 (Prompt)", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 77.0, - "bfcl/bfcl.overall.overall_accuracy": 27.83, - "bfcl/bfcl.overall.total_cost_usd": 82.74, - "bfcl/bfcl.overall.latency_mean_s": 8.89, - "bfcl/bfcl.overall.latency_std_s": 11.08, - "bfcl/bfcl.overall.latency_p95_s": 19.72, - "bfcl/bfcl.non_live.ast_accuracy": 68.04, - "bfcl/bfcl.non_live.simple_ast_accuracy": 59.17, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 72.5, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 71.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 69.0, - "bfcl/bfcl.live.live_accuracy": 62.55, - "bfcl/bfcl.live.live_simple_ast_accuracy": 69.77, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 61.16, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 75.0, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 37.5, - "bfcl/bfcl.multi_turn.accuracy": 5.5, - "bfcl/bfcl.multi_turn.base_accuracy": 5.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 5.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 4.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 7.0, - "bfcl/bfcl.web_search.accuracy": 8.5, - "bfcl/bfcl.web_search.base_accuracy": 11.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 6.0, - "bfcl/bfcl.memory.accuracy": 29.25, - "bfcl/bfcl.memory.kv_accuracy": 19.35, - "bfcl/bfcl.memory.vector_accuracy": 29.68, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 38.71, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 93.75, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 55.71, - "bfcl/bfcl.format_sensitivity.max_delta": 16.0, - "bfcl/bfcl.format_sensitivity.stddev": 3.78 - } - }, - { - "id": "openai/gpt-5-nano", - "name": "GPT-5-Nano", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "terminal-bench-2.0/terminal-bench-2.0": 7.0 - } - }, - { - "id": "openai/gpt-5-nano-2025-08-07", - "name": "GPT-5 nano 2025-08-07", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.748, - "helm_capabilities/MMLU-Pro": 0.778, - "helm_capabilities/GPQA": 0.679, - "helm_capabilities/IFEval": 0.932, - "helm_capabilities/WildBench": 0.806, - "helm_capabilities/Omni-MATH": 0.547 - } - }, - { - "id": "openai/gpt-5-nano-2025-08-07-fc", - "name": "GPT-5-nano-2025-08-07 (FC)", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 24.0, - "bfcl/bfcl.overall.overall_accuracy": 51.45, - "bfcl/bfcl.overall.total_cost_usd": 8.79, - "bfcl/bfcl.overall.latency_mean_s": 10.36, - "bfcl/bfcl.overall.latency_std_s": 10.37, - "bfcl/bfcl.overall.latency_p95_s": 23.56, - "bfcl/bfcl.non_live.ast_accuracy": 68.0, - "bfcl/bfcl.non_live.simple_ast_accuracy": 57.0, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 64.5, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 79.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 71.5, - "bfcl/bfcl.live.live_accuracy": 59.44, - "bfcl/bfcl.live.live_simple_ast_accuracy": 58.91, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 59.83, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 50.0, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 54.17, - "bfcl/bfcl.multi_turn.accuracy": 34.5, - "bfcl/bfcl.multi_turn.base_accuracy": 44.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 23.5, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 32.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 38.0, - "bfcl/bfcl.web_search.accuracy": 72.5, - "bfcl/bfcl.web_search.base_accuracy": 74.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 71.0, - "bfcl/bfcl.memory.accuracy": 24.73, - "bfcl/bfcl.memory.kv_accuracy": 18.06, - "bfcl/bfcl.memory.vector_accuracy": 27.1, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 29.03, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 75.0, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 89.1 - } - }, - { - "id": "openai/gpt-5-nano-2025-08-07-prompt", - "name": "GPT-5-nano-2025-08-07 (Prompt)", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 79.0, - "bfcl/bfcl.overall.overall_accuracy": 27.55, - "bfcl/bfcl.overall.total_cost_usd": 21.47, - "bfcl/bfcl.overall.latency_mean_s": 10.67, - "bfcl/bfcl.overall.latency_std_s": 7.68, - "bfcl/bfcl.overall.latency_p95_s": 23.28, - "bfcl/bfcl.non_live.ast_accuracy": 80.81, - "bfcl/bfcl.non_live.simple_ast_accuracy": 69.25, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 86.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 87.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 80.5, - "bfcl/bfcl.live.live_accuracy": 70.69, - "bfcl/bfcl.live.live_simple_ast_accuracy": 76.36, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 69.71, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 68.75, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 54.17, - "bfcl/bfcl.multi_turn.accuracy": 0.75, - "bfcl/bfcl.multi_turn.base_accuracy": 1.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 1.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 0.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 1.0, - "bfcl/bfcl.web_search.accuracy": 13.5, - "bfcl/bfcl.web_search.base_accuracy": 10.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 17.0, - "bfcl/bfcl.memory.accuracy": 24.52, - "bfcl/bfcl.memory.kv_accuracy": 20.65, - "bfcl/bfcl.memory.vector_accuracy": 31.61, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 21.29, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 93.75, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 45.75, - "bfcl/bfcl.format_sensitivity.max_delta": 8.5, - "bfcl/bfcl.format_sensitivity.stddev": 2.57 - } - }, - { - "id": "openai/gpt-5.1", - "name": "GPT-5.1", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "terminal-bench-2.0/terminal-bench-2.0": 47.6 - } - }, - { - "id": "openai/gpt-5.1-codex", - "name": "GPT-5.1-Codex", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "terminal-bench-2.0/terminal-bench-2.0": 53.5 - } - }, - { - "id": "openai/gpt-5.1-codex-max", - "name": "GPT-5.1-Codex-Max", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "terminal-bench-2.0/terminal-bench-2.0": 60.4 - } - }, - { - "id": "openai/gpt-5.1-codex-mini", - "name": "GPT-5.1-Codex-Mini", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "terminal-bench-2.0/terminal-bench-2.0": 43.1 - } - }, - { - "id": "openai/gpt-5.2", - "name": "GPT-5.2", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "terminal-bench-2.0/terminal-bench-2.0": 64.9 - } - }, - { - "id": "openai/gpt-5.2-2025-12-11", - "name": "gpt-5.2-2025-12-11", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "appworld_test_normal/appworld/test_normal": 0.0, - "browsecompplus/browsecompplus": 0.26, - "livecodebenchpro/Hard Problems": 0.1594, - "livecodebenchpro/Medium Problems": 0.5211, - "livecodebenchpro/Easy Problems": 0.9014, - "swe-bench/swe-bench": 0.57, - "tau-bench-2_airline/tau-bench-2/airline": 0.54, - "tau-bench-2_retail/tau-bench-2/retail": 0.68, - "tau-bench-2_telecom/tau-bench-2/telecom": 0.5354 - } - }, - { - "id": "openai/gpt-5.2-codex", - "name": "GPT-5.2-Codex", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "terminal-bench-2.0/terminal-bench-2.0": 66.5 - } - }, - { - "id": "openai/gpt-5.3-codex", - "name": "GPT-5.3-Codex", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "terminal-bench-2.0/terminal-bench-2.0": 74.6 - } - }, - { - "id": "openai/GPT-J-6B", - "name": "GPT-J 6B", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.273, - "helm_classic/MMLU": 0.249, - "helm_classic/BoolQ": 0.649, - "helm_classic/NarrativeQA": 0.545, - "helm_classic/NaturalQuestions (open-book)": 0.559, - "helm_classic/QuAC": 0.33, - "helm_classic/HellaSwag": 0.663, - "helm_classic/OpenbookQA": 0.514, - "helm_classic/TruthfulQA": 0.199, - "helm_classic/MS MARCO (TREC)": 0.345, - "helm_classic/CNN/DailyMail": 0.131, - "helm_classic/XSUM": 0.096, - "helm_classic/IMDB": 0.939, - "helm_classic/CivilComments": 0.52, - "helm_classic/RAFT": 0.619 - } - }, - { - "id": "openai/GPT-NeoX-20B", - "name": "GPT-NeoX 20B", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.351, - "helm_classic/MMLU": 0.276, - "helm_classic/BoolQ": 0.683, - "helm_classic/NarrativeQA": 0.599, - "helm_classic/NaturalQuestions (open-book)": 0.596, - "helm_classic/QuAC": 0.326, - "helm_classic/HellaSwag": 0.718, - "helm_classic/OpenbookQA": 0.524, - "helm_classic/TruthfulQA": 0.216, - "helm_classic/MS MARCO (TREC)": 0.398, - "helm_classic/CNN/DailyMail": 0.123, - "helm_classic/XSUM": 0.102, - "helm_classic/IMDB": 0.948, - "helm_classic/CivilComments": 0.516, - "helm_classic/RAFT": 0.505 - } - }, - { - "id": "openai/gpt-oss-120b", - "name": "GPT-OSS-120B", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.77, - "helm_capabilities/MMLU-Pro": 0.795, - "helm_capabilities/GPQA": 0.684, - "helm_capabilities/IFEval": 0.836, - "helm_capabilities/WildBench": 0.845, - "helm_capabilities/Omni-MATH": 0.688, - "livecodebenchpro/Hard Problems": 0.0, - "livecodebenchpro/Medium Problems": 0.11267605633802817, - "livecodebenchpro/Easy Problems": 0.6619718309859155, - "terminal-bench-2.0/terminal-bench-2.0": 18.7 - } - }, - { - "id": "openai/gpt-oss-20b", - "name": "GPT-OSS-20B", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.674, - "helm_capabilities/MMLU-Pro": 0.74, - "helm_capabilities/GPQA": 0.594, - "helm_capabilities/IFEval": 0.732, - "helm_capabilities/WildBench": 0.737, - "helm_capabilities/Omni-MATH": 0.565, - "livecodebenchpro/Hard Problems": 0.0, - "livecodebenchpro/Medium Problems": 0.056338028169014086, - "livecodebenchpro/Easy Problems": 0.5070422535211268, - "terminal-bench-2.0/terminal-bench-2.0": 3.4 - } - }, - { - "id": "openai/o3", - "name": "o3", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "ace/Overall Score": 0.529, - "ace/Gaming Score": 0.585, - "ace/Shopping Score": 0.45, - "apex-v1/Big Law Score": 0.76 - } - }, - { - "id": "openai/o3 Pro", - "name": "o3 Pro", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "ace/Overall Score": 0.552, - "ace/DIY Score": 0.54, - "ace/Food Score": 0.6, - "ace/Gaming Score": 0.613, - "ace/Shopping Score": 0.45 - } - }, - { - "id": "openai/o3-2025-04-16", - "name": "o3-2025-04-16", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.811, - "helm_capabilities/MMLU-Pro": 0.859, - "helm_capabilities/GPQA": 0.753, - "helm_capabilities/IFEval": 0.869, - "helm_capabilities/WildBench": 0.861, - "helm_capabilities/Omni-MATH": 0.714, - "livecodebenchpro/Hard Problems": 0.0, - "livecodebenchpro/Medium Problems": 0.22535211267605634, - "livecodebenchpro/Easy Problems": 0.7183098591549296 - } - }, - { - "id": "openai/o3-2025-04-16-fc", - "name": "o3-2025-04-16 (FC)", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 30.0, - "bfcl/bfcl.overall.overall_accuracy": 48.56, - "bfcl/bfcl.overall.total_cost_usd": 133.45, - "bfcl/bfcl.overall.latency_mean_s": 3.5, - "bfcl/bfcl.overall.latency_std_s": 8.69, - "bfcl/bfcl.overall.latency_p95_s": 8.39, - "bfcl/bfcl.non_live.ast_accuracy": 40.38, - "bfcl/bfcl.non_live.simple_ast_accuracy": 74.5, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 87.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 0.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 0.0, - "bfcl/bfcl.live.live_accuracy": 66.17, - "bfcl/bfcl.live.live_simple_ast_accuracy": 70.54, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 67.62, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 0.0, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 0.0, - "bfcl/bfcl.multi_turn.accuracy": 14.75, - "bfcl/bfcl.multi_turn.base_accuracy": 16.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 11.5, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 14.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 16.5, - "bfcl/bfcl.web_search.accuracy": 77.0, - "bfcl/bfcl.web_search.base_accuracy": 79.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 75.0, - "bfcl/bfcl.memory.accuracy": 47.31, - "bfcl/bfcl.memory.kv_accuracy": 24.52, - "bfcl/bfcl.memory.vector_accuracy": 44.52, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 72.9, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 81.25, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 86.13 - } - }, - { - "id": "openai/o3-2025-04-16-prompt", - "name": "o3-2025-04-16 (Prompt)", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 8.0, - "bfcl/bfcl.overall.overall_accuracy": 63.05, - "bfcl/bfcl.overall.total_cost_usd": 234.64, - "bfcl/bfcl.overall.latency_mean_s": 4.83, - "bfcl/bfcl.overall.latency_std_s": 7.01, - "bfcl/bfcl.overall.latency_p95_s": 11.7, - "bfcl/bfcl.non_live.ast_accuracy": 81.94, - "bfcl/bfcl.non_live.simple_ast_accuracy": 74.25, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 89.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 86.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 78.0, - "bfcl/bfcl.live.live_accuracy": 73.21, - "bfcl/bfcl.live.live_simple_ast_accuracy": 83.33, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 70.75, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 75.0, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 70.83, - "bfcl/bfcl.multi_turn.accuracy": 62.25, - "bfcl/bfcl.multi_turn.base_accuracy": 68.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 63.5, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 54.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 63.0, - "bfcl/bfcl.web_search.accuracy": 50.5, - "bfcl/bfcl.web_search.base_accuracy": 51.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 50.0, - "bfcl/bfcl.memory.accuracy": 51.83, - "bfcl/bfcl.memory.kv_accuracy": 33.55, - "bfcl/bfcl.memory.vector_accuracy": 50.32, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 71.61, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 93.75, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 83.98, - "bfcl/bfcl.format_sensitivity.max_delta": 8.5, - "bfcl/bfcl.format_sensitivity.stddev": 2.75 - } - }, - { - "id": "openai/o3-mini-2025-01-31", - "name": "o3-mini-2025-01-31", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "global-mmlu-lite/Global MMLU Lite": 0.78, - "global-mmlu-lite/Culturally Sensitive": 0.765, - "global-mmlu-lite/Culturally Agnostic": 0.795, - "global-mmlu-lite/Arabic": 0.7725, - "global-mmlu-lite/English": 0.8025, - "global-mmlu-lite/Bengali": 0.77, - "global-mmlu-lite/German": 0.7525, - "global-mmlu-lite/French": 0.74, - "global-mmlu-lite/Hindi": 0.7525, - "global-mmlu-lite/Indonesian": 0.7425, - "global-mmlu-lite/Italian": 0.8, - "global-mmlu-lite/Japanese": 0.81, - "global-mmlu-lite/Korean": 0.8075, - "global-mmlu-lite/Portuguese": 0.7975, - "global-mmlu-lite/Spanish": 0.775, - "global-mmlu-lite/Swahili": 0.765, - "global-mmlu-lite/Yoruba": 0.7725, - "global-mmlu-lite/Chinese": 0.8125, - "global-mmlu-lite/Burmese": 0.8075 - } - }, - { - "id": "openai/o4-mini-2025-04-16", - "name": "o4-mini-2025-04-16", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "global-mmlu-lite/Global MMLU Lite": 0.8705, - "global-mmlu-lite/Culturally Sensitive": 0.8503, - "global-mmlu-lite/Culturally Agnostic": 0.8906, - "global-mmlu-lite/Arabic": 0.865, - "global-mmlu-lite/English": 0.8675, - "global-mmlu-lite/Bengali": 0.8875, - "global-mmlu-lite/German": 0.8775, - "global-mmlu-lite/French": 0.87, - "global-mmlu-lite/Hindi": 0.87, - "global-mmlu-lite/Indonesian": 0.8675, - "global-mmlu-lite/Italian": 0.855, - "global-mmlu-lite/Japanese": 0.885, - "global-mmlu-lite/Korean": 0.88, - "global-mmlu-lite/Portuguese": 0.88, - "global-mmlu-lite/Spanish": 0.855, - "global-mmlu-lite/Swahili": 0.8525, - "global-mmlu-lite/Yoruba": 0.8525, - "global-mmlu-lite/Chinese": 0.89, - "global-mmlu-lite/Burmese": 0.8725, - "helm_capabilities/Mean score": 0.812, - "helm_capabilities/MMLU-Pro": 0.82, - "helm_capabilities/GPQA": 0.735, - "helm_capabilities/IFEval": 0.929, - "helm_capabilities/WildBench": 0.854, - "helm_capabilities/Omni-MATH": 0.72, - "livecodebenchpro/Hard Problems": 0.0143, - "livecodebenchpro/Medium Problems": 0.2923, - "livecodebenchpro/Easy Problems": 0.8571 - } - }, - { - "id": "openai/o4-mini-2025-04-16-fc", - "name": "o4-mini-2025-04-16 (FC)", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 21.0, - "bfcl/bfcl.overall.overall_accuracy": 53.24, - "bfcl/bfcl.overall.total_cost_usd": 81.91, - "bfcl/bfcl.overall.latency_mean_s": 3.71, - "bfcl/bfcl.overall.latency_std_s": 7.18, - "bfcl/bfcl.overall.latency_p95_s": 9.33, - "bfcl/bfcl.non_live.ast_accuracy": 37.73, - "bfcl/bfcl.non_live.simple_ast_accuracy": 66.92, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 84.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 0.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 0.0, - "bfcl/bfcl.live.live_accuracy": 66.1, - "bfcl/bfcl.live.live_simple_ast_accuracy": 69.38, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 67.81, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 0.0, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 0.0, - "bfcl/bfcl.multi_turn.accuracy": 41.75, - "bfcl/bfcl.multi_turn.base_accuracy": 51.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 30.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 40.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 45.5, - "bfcl/bfcl.web_search.accuracy": 75.5, - "bfcl/bfcl.web_search.base_accuracy": 75.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 76.0, - "bfcl/bfcl.memory.accuracy": 34.19, - "bfcl/bfcl.memory.kv_accuracy": 19.35, - "bfcl/bfcl.memory.vector_accuracy": 24.52, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 58.71, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 81.25, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 83.91 - } - }, - { - "id": "openai/o4-mini-2025-04-16-prompt", - "name": "o4-mini-2025-04-16 (Prompt)", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 28.0, - "bfcl/bfcl.overall.overall_accuracy": 50.26, - "bfcl/bfcl.overall.total_cost_usd": 133.63, - "bfcl/bfcl.overall.latency_mean_s": 4.47, - "bfcl/bfcl.overall.latency_std_s": 5.19, - "bfcl/bfcl.overall.latency_p95_s": 10.19, - "bfcl/bfcl.non_live.ast_accuracy": 81.29, - "bfcl/bfcl.non_live.simple_ast_accuracy": 72.67, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 88.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 84.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 80.0, - "bfcl/bfcl.live.live_accuracy": 70.76, - "bfcl/bfcl.live.live_simple_ast_accuracy": 79.46, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 68.76, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 75.0, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 62.5, - "bfcl/bfcl.multi_turn.accuracy": 16.62, - "bfcl/bfcl.multi_turn.base_accuracy": 16.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 18.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 17.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 14.5, - "bfcl/bfcl.web_search.accuracy": 71.5, - "bfcl/bfcl.web_search.base_accuracy": 73.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 70.0, - "bfcl/bfcl.memory.accuracy": 35.27, - "bfcl/bfcl.memory.kv_accuracy": 22.58, - "bfcl/bfcl.memory.vector_accuracy": 25.16, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 58.06, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 81.25, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 87.16, - "bfcl/bfcl.format_sensitivity.max_delta": 9.5, - "bfcl/bfcl.format_sensitivity.stddev": 2.6 - } - }, - { - "id": "openai/text-ada-001", - "name": "text-ada-001", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.107, - "helm_classic/MMLU": 0.238, - "helm_classic/BoolQ": 0.464, - "helm_classic/NarrativeQA": 0.238, - "helm_classic/NaturalQuestions (open-book)": 0.149, - "helm_classic/QuAC": 0.176, - "helm_classic/HellaSwag": 0.429, - "helm_classic/OpenbookQA": 0.346, - "helm_classic/TruthfulQA": 0.232, - "helm_classic/MS MARCO (TREC)": 0.302, - "helm_classic/CNN/DailyMail": 0.136, - "helm_classic/XSUM": 0.034, - "helm_classic/IMDB": 0.822, - "helm_classic/CivilComments": 0.503, - "helm_classic/RAFT": 0.406 - } - }, - { - "id": "openai/text-babbage-001", - "name": "text-babbage-001", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.229, - "helm_classic/MMLU": 0.229, - "helm_classic/BoolQ": 0.451, - "helm_classic/NarrativeQA": 0.429, - "helm_classic/NaturalQuestions (open-book)": 0.33, - "helm_classic/QuAC": 0.284, - "helm_classic/HellaSwag": 0.561, - "helm_classic/OpenbookQA": 0.452, - "helm_classic/TruthfulQA": 0.233, - "helm_classic/MS MARCO (TREC)": 0.449, - "helm_classic/CNN/DailyMail": 0.151, - "helm_classic/XSUM": 0.046, - "helm_classic/IMDB": 0.913, - "helm_classic/CivilComments": 0.499, - "helm_classic/RAFT": 0.509 - } - }, - { - "id": "openai/text-curie-001", - "name": "text-curie-001", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.36, - "helm_classic/MMLU": 0.237, - "helm_classic/BoolQ": 0.62, - "helm_classic/NarrativeQA": 0.582, - "helm_classic/NaturalQuestions (open-book)": 0.571, - "helm_classic/QuAC": 0.358, - "helm_classic/HellaSwag": 0.676, - "helm_classic/OpenbookQA": 0.514, - "helm_classic/TruthfulQA": 0.257, - "helm_classic/MS MARCO (TREC)": 0.507, - "helm_classic/CNN/DailyMail": 0.152, - "helm_classic/XSUM": 0.076, - "helm_classic/IMDB": 0.923, - "helm_classic/CivilComments": 0.537, - "helm_classic/RAFT": 0.489 - } - }, - { - "id": "openai/text-davinci-002", - "name": "GPT-3.5 text-davinci-002", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.905, - "helm_classic/MMLU": 0.568, - "helm_classic/BoolQ": 0.877, - "helm_classic/NarrativeQA": 0.727, - "helm_classic/NaturalQuestions (open-book)": 0.713, - "helm_classic/QuAC": 0.445, - "helm_classic/HellaSwag": 0.815, - "helm_classic/OpenbookQA": 0.594, - "helm_classic/TruthfulQA": 0.61, - "helm_classic/MS MARCO (TREC)": 0.664, - "helm_classic/CNN/DailyMail": 0.153, - "helm_classic/XSUM": 0.144, - "helm_classic/IMDB": 0.948, - "helm_classic/CivilComments": 0.668, - "helm_classic/RAFT": 0.733, - "helm_lite/Mean win rate": 0.336, - "helm_lite/NarrativeQA": 0.719, - "helm_lite/NaturalQuestions (closed-book)": 0.394, - "helm_lite/OpenbookQA": 0.796, - "helm_lite/MMLU": 0.568, - "helm_lite/MATH": 0.428, - "helm_lite/GSM8K": 0.479, - "helm_lite/LegalBench": 0.58, - "helm_lite/MedQA": 0.525, - "helm_lite/WMT 2014": 0.174 - } - }, - { - "id": "openai/text-davinci-003", - "name": "GPT-3.5 text-davinci-003", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.872, - "helm_classic/MMLU": 0.569, - "helm_classic/BoolQ": 0.881, - "helm_classic/NarrativeQA": 0.727, - "helm_classic/NaturalQuestions (open-book)": 0.77, - "helm_classic/QuAC": 0.525, - "helm_classic/HellaSwag": 0.822, - "helm_classic/OpenbookQA": 0.646, - "helm_classic/TruthfulQA": 0.593, - "helm_classic/MS MARCO (TREC)": 0.644, - "helm_classic/CNN/DailyMail": 0.156, - "helm_classic/XSUM": 0.124, - "helm_classic/IMDB": 0.848, - "helm_classic/CivilComments": 0.684, - "helm_classic/RAFT": 0.759, - "helm_lite/Mean win rate": 0.439, - "helm_lite/NarrativeQA": 0.731, - "helm_lite/NaturalQuestions (closed-book)": 0.413, - "helm_lite/OpenbookQA": 0.828, - "helm_lite/MMLU": 0.555, - "helm_lite/MATH": 0.449, - "helm_lite/GSM8K": 0.615, - "helm_lite/LegalBench": 0.622, - "helm_lite/MedQA": 0.531, - "helm_lite/WMT 2014": 0.191 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/openassistant.json b/data/developers/openassistant.json deleted file mode 100644 index 94a84f4efb74c56aff95d97f1dc674cfa0445541..0000000000000000000000000000000000000000 --- a/data/developers/openassistant.json +++ /dev/null @@ -1,76 +0,0 @@ -{ - "developer": "OpenAssistant", - "models": [ - { - "id": "OpenAssistant/oasst-rm-2-pythia-6.9b-epoch-1", - "name": "OpenAssistant/oasst-rm-2-pythia-6.9b-epoch-1", - "developer": "OpenAssistant", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.2653, - "reward-bench/Chat": 0.9246, - "reward-bench/Chat Hard": 0.3728, - "reward-bench/Safety": 0.3289, - "reward-bench/Reasoning": 0.5855, - "reward-bench/Prior Sets (0.5 weight)": 0.6801, - "reward-bench/Factuality": 0.3979, - "reward-bench/Precise IF": 0.2875, - "reward-bench/Math": 0.377, - "reward-bench/Focus": 0.1535, - "reward-bench/Ties": 0.047 - } - }, - { - "id": "OpenAssistant/oasst-rm-2.1-pythia-1.4b-epoch-2.5", - "name": "OpenAssistant/oasst-rm-2.1-pythia-1.4b-epoch-2.5", - "developer": "OpenAssistant", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.2648, - "reward-bench/Chat": 0.8855, - "reward-bench/Chat Hard": 0.4868, - "reward-bench/Safety": 0.3244, - "reward-bench/Reasoning": 0.7752, - "reward-bench/Prior Sets (0.5 weight)": 0.6533, - "reward-bench/Factuality": 0.3179, - "reward-bench/Precise IF": 0.2625, - "reward-bench/Math": 0.3934, - "reward-bench/Focus": 0.2707, - "reward-bench/Ties": 0.0198 - } - }, - { - "id": "OpenAssistant/oasst-sft-1-pythia-12b", - "name": "oasst-sft-1-pythia-12b", - "developer": "OpenAssistant", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1055, - "hfopenllm_v2/BBH": 0.3147, - "hfopenllm_v2/MATH Level 5": 0.0151, - "hfopenllm_v2/GPQA": 0.2576, - "hfopenllm_v2/MUSR": 0.3327, - "hfopenllm_v2/MMLU-PRO": 0.1113 - } - }, - { - "id": "OpenAssistant/reward-model-deberta-v3-large-v2", - "name": "OpenAssistant/reward-model-deberta-v3-large-v2", - "developer": "OpenAssistant", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6126, - "reward-bench/Factuality": 0.3853, - "reward-bench/Precise IF": 0.2687, - "reward-bench/Math": 0.5027, - "reward-bench/Safety": 0.7338, - "reward-bench/Focus": 0.2768, - "reward-bench/Ties": 0.12, - "reward-bench/Chat": 0.8939, - "reward-bench/Chat Hard": 0.4518, - "reward-bench/Reasoning": 0.3855, - "reward-bench/Prior Sets (0.5 weight)": 0.5836 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/openbmb.json b/data/developers/openbmb.json deleted file mode 100644 index 331aeb03fb86add7a8702b0c38d6a8da0bbfb128..0000000000000000000000000000000000000000 --- a/data/developers/openbmb.json +++ /dev/null @@ -1,163 +0,0 @@ -{ - "developer": "openbmb", - "models": [ - { - "id": "openbmb/Eurus-7b-kto", - "name": "openbmb/Eurus-7b-kto", - "developer": "openbmb", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.69, - "reward-bench/Chat": 0.9525, - "reward-bench/Chat Hard": 0.5373, - "reward-bench/Safety": 0.6054, - "reward-bench/Reasoning": 0.7467, - "reward-bench/Prior Sets (0.5 weight)": 0.5261 - } - }, - { - "id": "openbmb/Eurus-RM-7b", - "name": "openbmb/Eurus-RM-7b", - "developer": "openbmb", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.8159, - "reward-bench/Factuality": 0.6, - "reward-bench/Precise IF": 0.3438, - "reward-bench/Math": 0.5683, - "reward-bench/Safety": 0.8135, - "reward-bench/Focus": 0.7475, - "reward-bench/Ties": 0.5972, - "reward-bench/Chat": 0.9804, - "reward-bench/Chat Hard": 0.6557, - "reward-bench/Reasoning": 0.8633, - "reward-bench/Prior Sets (0.5 weight)": 0.7172 - } - }, - { - "id": "openbmb/MiniCPM-2B-dpo-fp32", - "name": "openbmb/MiniCPM-2B-dpo-fp32", - "developer": "openbmb", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.673, - "reward-bench/Chat": 0.8911, - "reward-bench/Chat Hard": 0.4934, - "reward-bench/Safety": 0.573, - "reward-bench/Reasoning": 0.8233, - "reward-bench/Prior Sets (0.5 weight)": 0.4958 - } - }, - { - "id": "openbmb/MiniCPM-S-1B-sft-llama-format", - "name": "MiniCPM-S-1B-sft-llama-format", - "developer": "openbmb", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3329, - "hfopenllm_v2/BBH": 0.3049, - "hfopenllm_v2/MATH Level 5": 0.031, - "hfopenllm_v2/GPQA": 0.271, - "hfopenllm_v2/MUSR": 0.3317, - "hfopenllm_v2/MMLU-PRO": 0.1858 - } - }, - { - "id": "openbmb/minicpm3-4b-fc-fc", - "name": "MiniCPM3-4B-FC (FC)", - "developer": "openbmb", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 86.0, - "bfcl/bfcl.overall.overall_accuracy": 25.55, - "bfcl/bfcl.overall.total_cost_usd": 54.05, - "bfcl/bfcl.overall.latency_mean_s": 118.62, - "bfcl/bfcl.overall.latency_std_s": 143.98, - "bfcl/bfcl.overall.latency_p95_s": 388.67, - "bfcl/bfcl.non_live.ast_accuracy": 81.75, - "bfcl/bfcl.non_live.simple_ast_accuracy": 70.5, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 92.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 84.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 80.5, - "bfcl/bfcl.live.live_accuracy": 65.21, - "bfcl/bfcl.live.live_simple_ast_accuracy": 73.26, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 63.53, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 50.0, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 62.5, - "bfcl/bfcl.multi_turn.accuracy": 3.88, - "bfcl/bfcl.multi_turn.base_accuracy": 6.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 2.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 4.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 2.5, - "bfcl/bfcl.web_search.accuracy": 0.0, - "bfcl/bfcl.web_search.base_accuracy": 0.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 0.0, - "bfcl/bfcl.memory.accuracy": 12.04, - "bfcl/bfcl.memory.kv_accuracy": 9.68, - "bfcl/bfcl.memory.vector_accuracy": 15.48, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 10.97, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 68.75, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 72.84 - } - }, - { - "id": "openbmb/minicpm3-4b-prompt", - "name": "MiniCPM3-4B (Prompt)", - "developer": "openbmb", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 97.0, - "bfcl/bfcl.overall.overall_accuracy": 22.08, - "bfcl/bfcl.overall.total_cost_usd": 29.83, - "bfcl/bfcl.overall.latency_mean_s": 31.18, - "bfcl/bfcl.overall.latency_std_s": 35.61, - "bfcl/bfcl.overall.latency_p95_s": 102.02, - "bfcl/bfcl.non_live.ast_accuracy": 70.54, - "bfcl/bfcl.non_live.simple_ast_accuracy": 66.17, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 77.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 70.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 69.0, - "bfcl/bfcl.live.live_accuracy": 43.15, - "bfcl/bfcl.live.live_simple_ast_accuracy": 47.67, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 42.17, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 43.75, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 37.5, - "bfcl/bfcl.multi_turn.accuracy": 3.5, - "bfcl/bfcl.multi_turn.base_accuracy": 4.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 4.5, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 2.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 3.0, - "bfcl/bfcl.web_search.accuracy": 2.0, - "bfcl/bfcl.web_search.base_accuracy": 2.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 2.0, - "bfcl/bfcl.memory.accuracy": 9.46, - "bfcl/bfcl.memory.kv_accuracy": 8.39, - "bfcl/bfcl.memory.vector_accuracy": 10.32, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 9.68, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 56.25, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 73.71, - "bfcl/bfcl.format_sensitivity.max_delta": 68.0, - "bfcl/bfcl.format_sensitivity.stddev": 16.55 - } - }, - { - "id": "openbmb/UltraRM-13b", - "name": "openbmb/UltraRM-13b", - "developer": "openbmb", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.4683, - "reward-bench/Chat": 0.9637, - "reward-bench/Chat Hard": 0.5548, - "reward-bench/Safety": 0.5089, - "reward-bench/Reasoning": 0.6244, - "reward-bench/Prior Sets (0.5 weight)": 0.7294, - "reward-bench/Factuality": 0.5063, - "reward-bench/Precise IF": 0.3312, - "reward-bench/Math": 0.5519, - "reward-bench/Focus": 0.6081, - "reward-bench/Ties": 0.3036 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/openbuddy.json b/data/developers/openbuddy.json deleted file mode 100644 index a98e6fe11e595e2b8e16769938fa62415b797e60..0000000000000000000000000000000000000000 --- a/data/developers/openbuddy.json +++ /dev/null @@ -1,313 +0,0 @@ -{ - "developer": "OpenBuddy", - "models": [ - { - "id": "OpenBuddy/openbuddy-falcon3-10b-v24.2-131k", - "name": "openbuddy-falcon3-10b-v24.2-131k", - "developer": "OpenBuddy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5086, - "hfopenllm_v2/BBH": 0.6004, - "hfopenllm_v2/MATH Level 5": 0.213, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.4186, - "hfopenllm_v2/MMLU-PRO": 0.3834 - } - }, - { - "id": "OpenBuddy/openbuddy-llama3-70b-v21.2-32k", - "name": "openbuddy-llama3-70b-v21.2-32k", - "developer": "OpenBuddy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.701, - "hfopenllm_v2/BBH": 0.6507, - "hfopenllm_v2/MATH Level 5": 0.2032, - "hfopenllm_v2/GPQA": 0.3423, - "hfopenllm_v2/MUSR": 0.458, - "hfopenllm_v2/MMLU-PRO": 0.4832 - } - }, - { - "id": "OpenBuddy/openbuddy-llama3-8b-v21.1-8k", - "name": "openbuddy-llama3-8b-v21.1-8k", - "developer": "OpenBuddy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.557, - "hfopenllm_v2/BBH": 0.4788, - "hfopenllm_v2/MATH Level 5": 0.0431, - "hfopenllm_v2/GPQA": 0.271, - "hfopenllm_v2/MUSR": 0.3988, - "hfopenllm_v2/MMLU-PRO": 0.2955 - } - }, - { - "id": "OpenBuddy/openbuddy-llama3-8b-v21.2-32k", - "name": "openbuddy-llama3-8b-v21.2-32k", - "developer": "OpenBuddy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6192, - "hfopenllm_v2/BBH": 0.4856, - "hfopenllm_v2/MATH Level 5": 0.0785, - "hfopenllm_v2/GPQA": 0.2794, - "hfopenllm_v2/MUSR": 0.3779, - "hfopenllm_v2/MMLU-PRO": 0.3299 - } - }, - { - "id": "OpenBuddy/openbuddy-llama3.1-70b-v22.1-131k", - "name": "openbuddy-llama3.1-70b-v22.1-131k", - "developer": "OpenBuddy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7333, - "hfopenllm_v2/BBH": 0.6698, - "hfopenllm_v2/MATH Level 5": 0.395, - "hfopenllm_v2/GPQA": 0.375, - "hfopenllm_v2/MUSR": 0.463, - "hfopenllm_v2/MMLU-PRO": 0.5304 - } - }, - { - "id": "OpenBuddy/openbuddy-llama3.1-8b-v22.2-131k", - "name": "openbuddy-llama3.1-8b-v22.2-131k", - "developer": "OpenBuddy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6657, - "hfopenllm_v2/BBH": 0.5007, - "hfopenllm_v2/MATH Level 5": 0.1148, - "hfopenllm_v2/GPQA": 0.2794, - "hfopenllm_v2/MUSR": 0.4081, - "hfopenllm_v2/MMLU-PRO": 0.331 - } - }, - { - "id": "OpenBuddy/openbuddy-llama3.1-8b-v22.3-131k", - "name": "openbuddy-llama3.1-8b-v22.3-131k", - "developer": "OpenBuddy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5997, - "hfopenllm_v2/BBH": 0.5066, - "hfopenllm_v2/MATH Level 5": 0.1208, - "hfopenllm_v2/GPQA": 0.2794, - "hfopenllm_v2/MUSR": 0.4015, - "hfopenllm_v2/MMLU-PRO": 0.3277 - } - }, - { - "id": "OpenBuddy/openbuddy-llama3.2-1b-v23.1-131k", - "name": "openbuddy-llama3.2-1b-v23.1-131k", - "developer": "OpenBuddy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.359, - "hfopenllm_v2/BBH": 0.3267, - "hfopenllm_v2/MATH Level 5": 0.0249, - "hfopenllm_v2/GPQA": 0.2584, - "hfopenllm_v2/MUSR": 0.3342, - "hfopenllm_v2/MMLU-PRO": 0.184 - } - }, - { - "id": "OpenBuddy/openbuddy-llama3.2-3b-v23.2-131k", - "name": "openbuddy-llama3.2-3b-v23.2-131k", - "developer": "OpenBuddy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4319, - "hfopenllm_v2/BBH": 0.4073, - "hfopenllm_v2/MATH Level 5": 0.0264, - "hfopenllm_v2/GPQA": 0.276, - "hfopenllm_v2/MUSR": 0.3263, - "hfopenllm_v2/MMLU-PRO": 0.2479 - } - }, - { - "id": "OpenBuddy/openbuddy-llama3.3-70b-v24.1-131k", - "name": "openbuddy-llama3.3-70b-v24.1-131k", - "developer": "OpenBuddy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8121, - "hfopenllm_v2/BBH": 0.6858, - "hfopenllm_v2/MATH Level 5": 0.4411, - "hfopenllm_v2/GPQA": 0.4346, - "hfopenllm_v2/MUSR": 0.4869, - "hfopenllm_v2/MMLU-PRO": 0.5327 - } - }, - { - "id": "OpenBuddy/openbuddy-mixtral-7bx8-v18.1-32k", - "name": "openbuddy-mixtral-7bx8-v18.1-32k", - "developer": "OpenBuddy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5493, - "hfopenllm_v2/BBH": 0.4656, - "hfopenllm_v2/MATH Level 5": 0.108, - "hfopenllm_v2/GPQA": 0.3045, - "hfopenllm_v2/MUSR": 0.3831, - "hfopenllm_v2/MMLU-PRO": 0.3804 - } - }, - { - "id": "OpenBuddy/openbuddy-nemotron-70b-v23.1-131k", - "name": "openbuddy-nemotron-70b-v23.1-131k", - "developer": "OpenBuddy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7555, - "hfopenllm_v2/BBH": 0.6749, - "hfopenllm_v2/MATH Level 5": 0.321, - "hfopenllm_v2/GPQA": 0.3633, - "hfopenllm_v2/MUSR": 0.4538, - "hfopenllm_v2/MMLU-PRO": 0.5175 - } - }, - { - "id": "OpenBuddy/openbuddy-nemotron-70b-v23.2-131k", - "name": "openbuddy-nemotron-70b-v23.2-131k", - "developer": "OpenBuddy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7227, - "hfopenllm_v2/BBH": 0.6705, - "hfopenllm_v2/MATH Level 5": 0.3157, - "hfopenllm_v2/GPQA": 0.3599, - "hfopenllm_v2/MUSR": 0.4696, - "hfopenllm_v2/MMLU-PRO": 0.5121 - } - }, - { - "id": "OpenBuddy/openbuddy-qwen2.5llamaify-14b-v23.1-200k", - "name": "openbuddy-qwen2.5llamaify-14b-v23.1-200k", - "developer": "OpenBuddy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6309, - "hfopenllm_v2/BBH": 0.6013, - "hfopenllm_v2/MATH Level 5": 0.2538, - "hfopenllm_v2/GPQA": 0.3331, - "hfopenllm_v2/MUSR": 0.424, - "hfopenllm_v2/MMLU-PRO": 0.4673 - } - }, - { - "id": "OpenBuddy/openbuddy-qwen2.5llamaify-14b-v23.3-200k", - "name": "openbuddy-qwen2.5llamaify-14b-v23.3-200k", - "developer": "OpenBuddy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6131, - "hfopenllm_v2/BBH": 0.6081, - "hfopenllm_v2/MATH Level 5": 0.2311, - "hfopenllm_v2/GPQA": 0.3272, - "hfopenllm_v2/MUSR": 0.4346, - "hfopenllm_v2/MMLU-PRO": 0.4795 - } - }, - { - "id": "OpenBuddy/openbuddy-qwen2.5llamaify-7b-v23.1-200k", - "name": "openbuddy-qwen2.5llamaify-7b-v23.1-200k", - "developer": "OpenBuddy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5673, - "hfopenllm_v2/BBH": 0.5509, - "hfopenllm_v2/MATH Level 5": 0.1888, - "hfopenllm_v2/GPQA": 0.3146, - "hfopenllm_v2/MUSR": 0.4363, - "hfopenllm_v2/MMLU-PRO": 0.3948 - } - }, - { - "id": "OpenBuddy/openbuddy-qwq-32b-v24.1-200k", - "name": "openbuddy-qwq-32b-v24.1-200k", - "developer": "OpenBuddy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5937, - "hfopenllm_v2/BBH": 0.6798, - "hfopenllm_v2/MATH Level 5": 0.3739, - "hfopenllm_v2/GPQA": 0.3809, - "hfopenllm_v2/MUSR": 0.4849, - "hfopenllm_v2/MMLU-PRO": 0.549 - } - }, - { - "id": "OpenBuddy/openbuddy-qwq-32b-v24.2-200k", - "name": "openbuddy-qwq-32b-v24.2-200k", - "developer": "OpenBuddy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.597, - "hfopenllm_v2/BBH": 0.6772, - "hfopenllm_v2/MATH Level 5": 0.3776, - "hfopenllm_v2/GPQA": 0.3767, - "hfopenllm_v2/MUSR": 0.4718, - "hfopenllm_v2/MMLU-PRO": 0.5446 - } - }, - { - "id": "OpenBuddy/openbuddy-yi1.5-34b-v21.3-32k", - "name": "openbuddy-yi1.5-34b-v21.3-32k", - "developer": "OpenBuddy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.542, - "hfopenllm_v2/BBH": 0.6163, - "hfopenllm_v2/MATH Level 5": 0.1782, - "hfopenllm_v2/GPQA": 0.349, - "hfopenllm_v2/MUSR": 0.4439, - "hfopenllm_v2/MMLU-PRO": 0.4599 - } - }, - { - "id": "OpenBuddy/openbuddy-zero-14b-v22.3-32k", - "name": "openbuddy-zero-14b-v22.3-32k", - "developer": "OpenBuddy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3753, - "hfopenllm_v2/BBH": 0.486, - "hfopenllm_v2/MATH Level 5": 0.0937, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.4166, - "hfopenllm_v2/MMLU-PRO": 0.3187 - } - }, - { - "id": "OpenBuddy/openbuddy-zero-3b-v21.2-32k", - "name": "openbuddy-zero-3b-v21.2-32k", - "developer": "OpenBuddy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3802, - "hfopenllm_v2/BBH": 0.3935, - "hfopenllm_v2/MATH Level 5": 0.0189, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3566, - "hfopenllm_v2/MMLU-PRO": 0.2034 - } - }, - { - "id": "OpenBuddy/openbuddy-zero-56b-v21.2-32k", - "name": "openbuddy-zero-56b-v21.2-32k", - "developer": "OpenBuddy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5057, - "hfopenllm_v2/BBH": 0.6128, - "hfopenllm_v2/MATH Level 5": 0.1624, - "hfopenllm_v2/GPQA": 0.318, - "hfopenllm_v2/MUSR": 0.4305, - "hfopenllm_v2/MMLU-PRO": 0.4399 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/openchat.json b/data/developers/openchat.json deleted file mode 100644 index a7809e488044248dbd02599ca637f543387e3f9b..0000000000000000000000000000000000000000 --- a/data/developers/openchat.json +++ /dev/null @@ -1,89 +0,0 @@ -{ - "developer": "openchat", - "models": [ - { - "id": "openchat/openchat-3.5-0106", - "name": "openchat-3.5-0106", - "developer": "openchat", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5967, - "hfopenllm_v2/BBH": 0.4617, - "hfopenllm_v2/MATH Level 5": 0.0763, - "hfopenllm_v2/GPQA": 0.3079, - "hfopenllm_v2/MUSR": 0.4254, - "hfopenllm_v2/MMLU-PRO": 0.3291 - } - }, - { - "id": "openchat/openchat-3.5-1210", - "name": "openchat-3.5-1210", - "developer": "openchat", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6037, - "hfopenllm_v2/BBH": 0.4535, - "hfopenllm_v2/MATH Level 5": 0.0785, - "hfopenllm_v2/GPQA": 0.3012, - "hfopenllm_v2/MUSR": 0.4414, - "hfopenllm_v2/MMLU-PRO": 0.3142 - } - }, - { - "id": "openchat/openchat-3.6-8b-20240522", - "name": "openchat-3.6-8b-20240522", - "developer": "openchat", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5343, - "hfopenllm_v2/BBH": 0.5338, - "hfopenllm_v2/MATH Level 5": 0.0997, - "hfopenllm_v2/GPQA": 0.318, - "hfopenllm_v2/MUSR": 0.3999, - "hfopenllm_v2/MMLU-PRO": 0.3229 - } - }, - { - "id": "openchat/openchat_3.5", - "name": "openchat_3.5", - "developer": "openchat", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5931, - "hfopenllm_v2/BBH": 0.4426, - "hfopenllm_v2/MATH Level 5": 0.0725, - "hfopenllm_v2/GPQA": 0.2987, - "hfopenllm_v2/MUSR": 0.4229, - "hfopenllm_v2/MMLU-PRO": 0.3153 - } - }, - { - "id": "openchat/openchat_v3.2", - "name": "openchat_v3.2", - "developer": "openchat", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2981, - "hfopenllm_v2/BBH": 0.4331, - "hfopenllm_v2/MATH Level 5": 0.0128, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.4336, - "hfopenllm_v2/MMLU-PRO": 0.2422 - } - }, - { - "id": "openchat/openchat_v3.2_super", - "name": "openchat_v3.2_super", - "developer": "openchat", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2862, - "hfopenllm_v2/BBH": 0.4221, - "hfopenllm_v2/MATH Level 5": 0.0211, - "hfopenllm_v2/GPQA": 0.2643, - "hfopenllm_v2/MUSR": 0.4161, - "hfopenllm_v2/MMLU-PRO": 0.2425 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/opencompass.json b/data/developers/opencompass.json deleted file mode 100644 index 94e25e5bb03f1eaf9994b9c840b0dea07de68c21..0000000000000000000000000000000000000000 --- a/data/developers/opencompass.json +++ /dev/null @@ -1,57 +0,0 @@ -{ - "developer": "opencompass", - "models": [ - { - "id": "opencompass/CompassJudger-1-1.5B-Instruct", - "name": "opencompass/CompassJudger-1-1.5B-Instruct", - "developer": "opencompass", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7344, - "reward-bench/Chat": 0.9637, - "reward-bench/Chat Hard": 0.4923, - "reward-bench/Safety": 0.7818, - "reward-bench/Reasoning": 0.6999 - } - }, - { - "id": "opencompass/CompassJudger-1-14B-Instruct", - "name": "opencompass/CompassJudger-1-14B-Instruct", - "developer": "opencompass", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.8409, - "reward-bench/Chat": 0.9749, - "reward-bench/Chat Hard": 0.6228, - "reward-bench/Safety": 0.8392, - "reward-bench/Reasoning": 0.9268 - } - }, - { - "id": "opencompass/CompassJudger-1-32B-Instruct", - "name": "opencompass/CompassJudger-1-32B-Instruct", - "developer": "opencompass", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.8522, - "reward-bench/Chat": 0.9804, - "reward-bench/Chat Hard": 0.6513, - "reward-bench/Safety": 0.8527, - "reward-bench/Reasoning": 0.9244 - } - }, - { - "id": "opencompass/CompassJudger-1-7B-Instruct", - "name": "opencompass/CompassJudger-1-7B-Instruct", - "developer": "opencompass", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.8317, - "reward-bench/Chat": 0.9777, - "reward-bench/Chat Hard": 0.6096, - "reward-bench/Safety": 0.8446, - "reward-bench/Reasoning": 0.8948 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/opengenerativeai.json b/data/developers/opengenerativeai.json deleted file mode 100644 index e4348d42b5ebbf4a91a2f194731b6091cf2362c6..0000000000000000000000000000000000000000 --- a/data/developers/opengenerativeai.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "OpenGenerativeAI", - "models": [ - { - "id": "OpenGenerativeAI/Bifrost", - "name": "Bifrost", - "developer": "OpenGenerativeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6348, - "hfopenllm_v2/BBH": 0.6849, - "hfopenllm_v2/MATH Level 5": 0.2545, - "hfopenllm_v2/GPQA": 0.3683, - "hfopenllm_v2/MUSR": 0.4598, - "hfopenllm_v2/MMLU-PRO": 0.516 - } - }, - { - "id": "OpenGenerativeAI/Bifrost-14B", - "name": "Bifrost-14B", - "developer": "OpenGenerativeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6615, - "hfopenllm_v2/BBH": 0.6845, - "hfopenllm_v2/MATH Level 5": 0.2356, - "hfopenllm_v2/GPQA": 0.3792, - "hfopenllm_v2/MUSR": 0.4624, - "hfopenllm_v2/MMLU-PRO": 0.5074 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/openleecher.json b/data/developers/openleecher.json deleted file mode 100644 index 049d103cc2aafb7dad98cc0f49c1a8d98d36c1a7..0000000000000000000000000000000000000000 --- a/data/developers/openleecher.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "OpenLeecher", - "models": [ - { - "id": "OpenLeecher/llama3-8b-lima", - "name": "llama3-8b-lima", - "developer": "OpenLeecher", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4371, - "hfopenllm_v2/BBH": 0.4296, - "hfopenllm_v2/MATH Level 5": 0.0506, - "hfopenllm_v2/GPQA": 0.2383, - "hfopenllm_v2/MUSR": 0.3713, - "hfopenllm_v2/MMLU-PRO": 0.2626 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/openllm-france.json b/data/developers/openllm-france.json deleted file mode 100644 index 6ebba764bd5c8af0db5e7c6efeec343fba0ad06e..0000000000000000000000000000000000000000 --- a/data/developers/openllm-france.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "developer": "OpenLLM-France", - "models": [ - { - "id": "OpenLLM-France/Lucie-7B", - "name": "Lucie-7B", - "developer": "OpenLLM-France", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2496, - "hfopenllm_v2/BBH": 0.3492, - "hfopenllm_v2/MATH Level 5": 0.0144, - "hfopenllm_v2/GPQA": 0.2727, - "hfopenllm_v2/MUSR": 0.3923, - "hfopenllm_v2/MMLU-PRO": 0.1498 - } - }, - { - "id": "OpenLLM-France/Lucie-7B-Instruct", - "name": "Lucie-7B-Instruct", - "developer": "OpenLLM-France", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2796, - "hfopenllm_v2/BBH": 0.3254, - "hfopenllm_v2/MATH Level 5": 0.0166, - "hfopenllm_v2/GPQA": 0.2794, - "hfopenllm_v2/MUSR": 0.3662, - "hfopenllm_v2/MMLU-PRO": 0.1556 - } - }, - { - "id": "OpenLLM-France/Lucie-7B-Instruct-human-data", - "name": "Lucie-7B-Instruct-human-data", - "developer": "OpenLLM-France", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2946, - "hfopenllm_v2/BBH": 0.3284, - "hfopenllm_v2/MATH Level 5": 0.0219, - "hfopenllm_v2/GPQA": 0.2752, - "hfopenllm_v2/MUSR": 0.3729, - "hfopenllm_v2/MMLU-PRO": 0.143 - } - }, - { - "id": "OpenLLM-France/Lucie-7B-Instruct-v1.1", - "name": "Lucie-7B-Instruct-v1.1", - "developer": "OpenLLM-France", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3039, - "hfopenllm_v2/BBH": 0.3816, - "hfopenllm_v2/MATH Level 5": 0.0317, - "hfopenllm_v2/GPQA": 0.2819, - "hfopenllm_v2/MUSR": 0.375, - "hfopenllm_v2/MMLU-PRO": 0.1864 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/openscholar.json b/data/developers/openscholar.json deleted file mode 100644 index 72e39b0c055a9a51c7cb863cbd21bb77d6332d8f..0000000000000000000000000000000000000000 --- a/data/developers/openscholar.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "OpenScholar", - "models": [ - { - "id": "OpenScholar/Llama-3.1_OpenScholar-8B", - "name": "Llama-3.1_OpenScholar-8B", - "developer": "OpenScholar", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6064, - "hfopenllm_v2/BBH": 0.5208, - "hfopenllm_v2/MATH Level 5": 0.1654, - "hfopenllm_v2/GPQA": 0.2819, - "hfopenllm_v2/MUSR": 0.4275, - "hfopenllm_v2/MMLU-PRO": 0.3708 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/orai-nlp.json b/data/developers/orai-nlp.json deleted file mode 100644 index ae97aed54f747cee9f9d22196fe5b655770f58e7..0000000000000000000000000000000000000000 --- a/data/developers/orai-nlp.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "orai-nlp", - "models": [ - { - "id": "orai-nlp/Llama-eus-8B", - "name": "Llama-eus-8B", - "developer": "orai-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2161, - "hfopenllm_v2/BBH": 0.4418, - "hfopenllm_v2/MATH Level 5": 0.0468, - "hfopenllm_v2/GPQA": 0.2894, - "hfopenllm_v2/MUSR": 0.3919, - "hfopenllm_v2/MMLU-PRO": 0.3058 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/orenguteng.json b/data/developers/orenguteng.json deleted file mode 100644 index 9a10ade8d5678f6c0ab0a9dd6f8f24025ba5bec1..0000000000000000000000000000000000000000 --- a/data/developers/orenguteng.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "Orenguteng", - "models": [ - { - "id": "Orenguteng/Llama-3.1-8B-Lexi-Uncensored", - "name": "Llama-3.1-8B-Lexi-Uncensored", - "developer": "Orenguteng", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7777, - "hfopenllm_v2/BBH": 0.5057, - "hfopenllm_v2/MATH Level 5": 0.1571, - "hfopenllm_v2/GPQA": 0.2718, - "hfopenllm_v2/MUSR": 0.3871, - "hfopenllm_v2/MMLU-PRO": 0.379 - } - }, - { - "id": "Orenguteng/Llama-3.1-8B-Lexi-Uncensored-V2", - "name": "Llama-3.1-8B-Lexi-Uncensored-V2", - "developer": "Orenguteng", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7792, - "hfopenllm_v2/BBH": 0.5084, - "hfopenllm_v2/MATH Level 5": 0.1971, - "hfopenllm_v2/GPQA": 0.2827, - "hfopenllm_v2/MUSR": 0.3843, - "hfopenllm_v2/MMLU-PRO": 0.3781 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/orion-zhen.json b/data/developers/orion-zhen.json deleted file mode 100644 index 031e12dc68fa70cdcc03a1c0a95d59133c447de9..0000000000000000000000000000000000000000 --- a/data/developers/orion-zhen.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "Orion-zhen", - "models": [ - { - "id": "Orion-zhen/phi-4-abliterated", - "name": "phi-4-abliterated", - "developer": "Orion-zhen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0576, - "hfopenllm_v2/BBH": 0.6698, - "hfopenllm_v2/MATH Level 5": 0.3021, - "hfopenllm_v2/GPQA": 0.4044, - "hfopenllm_v2/MUSR": 0.5006, - "hfopenllm_v2/MMLU-PRO": 0.5292 - } - }, - { - "id": "Orion-zhen/Qwen2.5-7B-Instruct-Uncensored", - "name": "Qwen2.5-7B-Instruct-Uncensored", - "developer": "Orion-zhen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7204, - "hfopenllm_v2/BBH": 0.5474, - "hfopenllm_v2/MATH Level 5": 0.4773, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.4361, - "hfopenllm_v2/MMLU-PRO": 0.4427 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/oxyapi.json b/data/developers/oxyapi.json deleted file mode 100644 index db5cfa74854c055d13f627682108bebb7241409e..0000000000000000000000000000000000000000 --- a/data/developers/oxyapi.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "oxyapi", - "models": [ - { - "id": "oxyapi/oxy-1-small", - "name": "oxy-1-small", - "developer": "oxyapi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6245, - "hfopenllm_v2/BBH": 0.5885, - "hfopenllm_v2/MATH Level 5": 0.3603, - "hfopenllm_v2/GPQA": 0.3716, - "hfopenllm_v2/MUSR": 0.4487, - "hfopenllm_v2/MMLU-PRO": 0.5001 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/ozone-ai.json b/data/developers/ozone-ai.json deleted file mode 100644 index abe1b28ad015e6d4b54f97763783387da31eb70f..0000000000000000000000000000000000000000 --- a/data/developers/ozone-ai.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "ozone-ai", - "models": [ - { - "id": "ozone-ai/0x-lite", - "name": "0x-lite", - "developer": "ozone-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.774, - "hfopenllm_v2/BBH": 0.6341, - "hfopenllm_v2/MATH Level 5": 0.5045, - "hfopenllm_v2/GPQA": 0.3196, - "hfopenllm_v2/MUSR": 0.4221, - "hfopenllm_v2/MMLU-PRO": 0.5184 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/ozone-research.json b/data/developers/ozone-research.json deleted file mode 100644 index fd1aaf7ea4d20823384132dacf4797c8baeb36d3..0000000000000000000000000000000000000000 --- a/data/developers/ozone-research.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "ozone-research", - "models": [ - { - "id": "ozone-research/Chirp-01", - "name": "Chirp-01", - "developer": "ozone-research", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6348, - "hfopenllm_v2/BBH": 0.465, - "hfopenllm_v2/MATH Level 5": 0.3467, - "hfopenllm_v2/GPQA": 0.2718, - "hfopenllm_v2/MUSR": 0.4487, - "hfopenllm_v2/MMLU-PRO": 0.3508 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/p0x0.json b/data/developers/p0x0.json deleted file mode 100644 index 9eb75a9f1302a8c7d9b6f9eebe590f0b56dd2857..0000000000000000000000000000000000000000 --- a/data/developers/p0x0.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "P0x0", - "models": [ - { - "id": "P0x0/Astra-v1-12B", - "name": "Astra-v1-12B", - "developer": "P0x0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2806, - "hfopenllm_v2/BBH": 0.5215, - "hfopenllm_v2/MATH Level 5": 0.1133, - "hfopenllm_v2/GPQA": 0.3138, - "hfopenllm_v2/MUSR": 0.4052, - "hfopenllm_v2/MMLU-PRO": 0.3461 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/paloalma.json b/data/developers/paloalma.json deleted file mode 100644 index 6598701688bb611b27294d51194869f074d2a252..0000000000000000000000000000000000000000 --- a/data/developers/paloalma.json +++ /dev/null @@ -1,75 +0,0 @@ -{ - "developer": "paloalma", - "models": [ - { - "id": "paloalma/ECE-TW3-JRGL-V1", - "name": "ECE-TW3-JRGL-V1", - "developer": "paloalma", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5535, - "hfopenllm_v2/BBH": 0.6284, - "hfopenllm_v2/MATH Level 5": 0.1314, - "hfopenllm_v2/GPQA": 0.3473, - "hfopenllm_v2/MUSR": 0.4621, - "hfopenllm_v2/MMLU-PRO": 0.4221 - } - }, - { - "id": "paloalma/ECE-TW3-JRGL-V2", - "name": "ECE-TW3-JRGL-V2", - "developer": "paloalma", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2255, - "hfopenllm_v2/BBH": 0.6031, - "hfopenllm_v2/MATH Level 5": 0.185, - "hfopenllm_v2/GPQA": 0.3314, - "hfopenllm_v2/MUSR": 0.4793, - "hfopenllm_v2/MMLU-PRO": 0.4588 - } - }, - { - "id": "paloalma/ECE-TW3-JRGL-V5", - "name": "ECE-TW3-JRGL-V5", - "developer": "paloalma", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4553, - "hfopenllm_v2/BBH": 0.6025, - "hfopenllm_v2/MATH Level 5": 0.1835, - "hfopenllm_v2/GPQA": 0.3414, - "hfopenllm_v2/MUSR": 0.4621, - "hfopenllm_v2/MMLU-PRO": 0.4648 - } - }, - { - "id": "paloalma/Le_Triomphant-ECE-TW3", - "name": "Le_Triomphant-ECE-TW3", - "developer": "paloalma", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5402, - "hfopenllm_v2/BBH": 0.6112, - "hfopenllm_v2/MATH Level 5": 0.1949, - "hfopenllm_v2/GPQA": 0.349, - "hfopenllm_v2/MUSR": 0.4725, - "hfopenllm_v2/MMLU-PRO": 0.4763 - } - }, - { - "id": "paloalma/TW3-JRGL-v2", - "name": "TW3-JRGL-v2", - "developer": "paloalma", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5316, - "hfopenllm_v2/BBH": 0.6138, - "hfopenllm_v2/MATH Level 5": 0.179, - "hfopenllm_v2/GPQA": 0.3591, - "hfopenllm_v2/MUSR": 0.4858, - "hfopenllm_v2/MMLU-PRO": 0.4858 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/pankajmathur.json b/data/developers/pankajmathur.json deleted file mode 100644 index 817880875c15da047744ee954089b88c588562e4..0000000000000000000000000000000000000000 --- a/data/developers/pankajmathur.json +++ /dev/null @@ -1,411 +0,0 @@ -{ - "developer": "pankajmathur", - "models": [ - { - "id": "pankajmathur/Al_Dente_v1_8b", - "name": "Al_Dente_v1_8b", - "developer": "pankajmathur", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3694, - "hfopenllm_v2/BBH": 0.4835, - "hfopenllm_v2/MATH Level 5": 0.0408, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.3987, - "hfopenllm_v2/MMLU-PRO": 0.286 - } - }, - { - "id": "pankajmathur/model_007_13b_v2", - "name": "model_007_13b_v2", - "developer": "pankajmathur", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3056, - "hfopenllm_v2/BBH": 0.4702, - "hfopenllm_v2/MATH Level 5": 0.0211, - "hfopenllm_v2/GPQA": 0.2836, - "hfopenllm_v2/MUSR": 0.4611, - "hfopenllm_v2/MMLU-PRO": 0.2461 - } - }, - { - "id": "pankajmathur/orca_mini_3b", - "name": "orca_mini_3b", - "developer": "pankajmathur", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0742, - "hfopenllm_v2/BBH": 0.3196, - "hfopenllm_v2/MATH Level 5": 0.0083, - "hfopenllm_v2/GPQA": 0.2458, - "hfopenllm_v2/MUSR": 0.3349, - "hfopenllm_v2/MMLU-PRO": 0.1145 - } - }, - { - "id": "pankajmathur/orca_mini_7b", - "name": "orca_mini_7b", - "developer": "pankajmathur", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0412, - "hfopenllm_v2/BBH": 0.3332, - "hfopenllm_v2/MATH Level 5": 0.0128, - "hfopenllm_v2/GPQA": 0.2542, - "hfopenllm_v2/MUSR": 0.3698, - "hfopenllm_v2/MMLU-PRO": 0.1246 - } - }, - { - "id": "pankajmathur/orca_mini_phi-4", - "name": "orca_mini_phi-4", - "developer": "pankajmathur", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7781, - "hfopenllm_v2/BBH": 0.6856, - "hfopenllm_v2/MATH Level 5": 0.2953, - "hfopenllm_v2/GPQA": 0.3742, - "hfopenllm_v2/MUSR": 0.4703, - "hfopenllm_v2/MMLU-PRO": 0.5255 - } - }, - { - "id": "pankajmathur/orca_mini_v2_7b", - "name": "orca_mini_v2_7b", - "developer": "pankajmathur", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1358, - "hfopenllm_v2/BBH": 0.3536, - "hfopenllm_v2/MATH Level 5": 0.0113, - "hfopenllm_v2/GPQA": 0.2492, - "hfopenllm_v2/MUSR": 0.3593, - "hfopenllm_v2/MMLU-PRO": 0.1542 - } - }, - { - "id": "pankajmathur/orca_mini_v3_13b", - "name": "orca_mini_v3_13b", - "developer": "pankajmathur", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2897, - "hfopenllm_v2/BBH": 0.4711, - "hfopenllm_v2/MATH Level 5": 0.0211, - "hfopenllm_v2/GPQA": 0.2651, - "hfopenllm_v2/MUSR": 0.4598, - "hfopenllm_v2/MMLU-PRO": 0.2305 - } - }, - { - "id": "pankajmathur/orca_mini_v3_70b", - "name": "orca_mini_v3_70b", - "developer": "pankajmathur", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4015, - "hfopenllm_v2/BBH": 0.5949, - "hfopenllm_v2/MATH Level 5": 0.0385, - "hfopenllm_v2/GPQA": 0.318, - "hfopenllm_v2/MUSR": 0.5079, - "hfopenllm_v2/MMLU-PRO": 0.3757 - } - }, - { - "id": "pankajmathur/orca_mini_v3_7b", - "name": "orca_mini_v3_7b", - "developer": "pankajmathur", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2821, - "hfopenllm_v2/BBH": 0.4095, - "hfopenllm_v2/MATH Level 5": 0.0106, - "hfopenllm_v2/GPQA": 0.2466, - "hfopenllm_v2/MUSR": 0.4982, - "hfopenllm_v2/MMLU-PRO": 0.2084 - } - }, - { - "id": "pankajmathur/orca_mini_v5_8b", - "name": "orca_mini_v5_8b", - "developer": "pankajmathur", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4806, - "hfopenllm_v2/BBH": 0.5064, - "hfopenllm_v2/MATH Level 5": 0.0989, - "hfopenllm_v2/GPQA": 0.2869, - "hfopenllm_v2/MUSR": 0.4, - "hfopenllm_v2/MMLU-PRO": 0.3076 - } - }, - { - "id": "pankajmathur/orca_mini_v5_8b_dpo", - "name": "orca_mini_v5_8b_dpo", - "developer": "pankajmathur", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4896, - "hfopenllm_v2/BBH": 0.5075, - "hfopenllm_v2/MATH Level 5": 0.0974, - "hfopenllm_v2/GPQA": 0.2743, - "hfopenllm_v2/MUSR": 0.3894, - "hfopenllm_v2/MMLU-PRO": 0.3116 - } - }, - { - "id": "pankajmathur/orca_mini_v5_8b_orpo", - "name": "orca_mini_v5_8b_orpo", - "developer": "pankajmathur", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0824, - "hfopenllm_v2/BBH": 0.4964, - "hfopenllm_v2/MATH Level 5": 0.0665, - "hfopenllm_v2/GPQA": 0.2844, - "hfopenllm_v2/MUSR": 0.4131, - "hfopenllm_v2/MMLU-PRO": 0.2947 - } - }, - { - "id": "pankajmathur/orca_mini_v6_8b", - "name": "orca_mini_v6_8b", - "developer": "pankajmathur", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0111, - "hfopenllm_v2/BBH": 0.3029, - "hfopenllm_v2/MATH Level 5": 0.0038, - "hfopenllm_v2/GPQA": 0.2383, - "hfopenllm_v2/MUSR": 0.3555, - "hfopenllm_v2/MMLU-PRO": 0.1125 - } - }, - { - "id": "pankajmathur/orca_mini_v6_8b_dpo", - "name": "orca_mini_v6_8b_dpo", - "developer": "pankajmathur", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3883, - "hfopenllm_v2/BBH": 0.5203, - "hfopenllm_v2/MATH Level 5": 0.0612, - "hfopenllm_v2/GPQA": 0.3012, - "hfopenllm_v2/MUSR": 0.409, - "hfopenllm_v2/MMLU-PRO": 0.3596 - } - }, - { - "id": "pankajmathur/orca_mini_v7_72b", - "name": "orca_mini_v7_72b", - "developer": "pankajmathur", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.593, - "hfopenllm_v2/BBH": 0.6842, - "hfopenllm_v2/MATH Level 5": 0.0937, - "hfopenllm_v2/GPQA": 0.3851, - "hfopenllm_v2/MUSR": 0.507, - "hfopenllm_v2/MMLU-PRO": 0.5622 - } - }, - { - "id": "pankajmathur/orca_mini_v7_7b", - "name": "orca_mini_v7_7b", - "developer": "pankajmathur", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4388, - "hfopenllm_v2/BBH": 0.5275, - "hfopenllm_v2/MATH Level 5": 0.1208, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.436, - "hfopenllm_v2/MMLU-PRO": 0.4167 - } - }, - { - "id": "pankajmathur/orca_mini_v8_1_70b", - "name": "orca_mini_v8_1_70b", - "developer": "pankajmathur", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8571, - "hfopenllm_v2/BBH": 0.6781, - "hfopenllm_v2/MATH Level 5": 0.3527, - "hfopenllm_v2/GPQA": 0.4329, - "hfopenllm_v2/MUSR": 0.4437, - "hfopenllm_v2/MMLU-PRO": 0.4983 - } - }, - { - "id": "pankajmathur/orca_mini_v9_0_3B-Instruct", - "name": "orca_mini_v9_0_3B-Instruct", - "developer": "pankajmathur", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5754, - "hfopenllm_v2/BBH": 0.4413, - "hfopenllm_v2/MATH Level 5": 0.1465, - "hfopenllm_v2/GPQA": 0.3012, - "hfopenllm_v2/MUSR": 0.3659, - "hfopenllm_v2/MMLU-PRO": 0.2603 - } - }, - { - "id": "pankajmathur/orca_mini_v9_1_1B-Instruct", - "name": "orca_mini_v9_1_1B-Instruct", - "developer": "pankajmathur", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3629, - "hfopenllm_v2/BBH": 0.3205, - "hfopenllm_v2/MATH Level 5": 0.0461, - "hfopenllm_v2/GPQA": 0.2567, - "hfopenllm_v2/MUSR": 0.3381, - "hfopenllm_v2/MMLU-PRO": 0.1374 - } - }, - { - "id": "pankajmathur/orca_mini_v9_2_14B", - "name": "orca_mini_v9_2_14B", - "developer": "pankajmathur", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7781, - "hfopenllm_v2/BBH": 0.6856, - "hfopenllm_v2/MATH Level 5": 0.2953, - "hfopenllm_v2/GPQA": 0.3742, - "hfopenllm_v2/MUSR": 0.4703, - "hfopenllm_v2/MMLU-PRO": 0.5255 - } - }, - { - "id": "pankajmathur/orca_mini_v9_2_70b", - "name": "orca_mini_v9_2_70b", - "developer": "pankajmathur", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8383, - "hfopenllm_v2/BBH": 0.6745, - "hfopenllm_v2/MATH Level 5": 0.2938, - "hfopenllm_v2/GPQA": 0.3733, - "hfopenllm_v2/MUSR": 0.471, - "hfopenllm_v2/MMLU-PRO": 0.4821 - } - }, - { - "id": "pankajmathur/orca_mini_v9_4_70B", - "name": "orca_mini_v9_4_70B", - "developer": "pankajmathur", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8015, - "hfopenllm_v2/BBH": 0.6419, - "hfopenllm_v2/MATH Level 5": 0.3263, - "hfopenllm_v2/GPQA": 0.3658, - "hfopenllm_v2/MUSR": 0.4647, - "hfopenllm_v2/MMLU-PRO": 0.4536 - } - }, - { - "id": "pankajmathur/orca_mini_v9_5_1B-Instruct", - "name": "orca_mini_v9_5_1B-Instruct", - "developer": "pankajmathur", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4638, - "hfopenllm_v2/BBH": 0.3337, - "hfopenllm_v2/MATH Level 5": 0.0302, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.3182, - "hfopenllm_v2/MMLU-PRO": 0.137 - } - }, - { - "id": "pankajmathur/orca_mini_v9_5_1B-Instruct_preview", - "name": "orca_mini_v9_5_1B-Instruct_preview", - "developer": "pankajmathur", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3936, - "hfopenllm_v2/BBH": 0.3277, - "hfopenllm_v2/MATH Level 5": 0.0385, - "hfopenllm_v2/GPQA": 0.2634, - "hfopenllm_v2/MUSR": 0.3395, - "hfopenllm_v2/MMLU-PRO": 0.1327 - } - }, - { - "id": "pankajmathur/orca_mini_v9_5_3B-Instruct", - "name": "orca_mini_v9_5_3B-Instruct", - "developer": "pankajmathur", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7207, - "hfopenllm_v2/BBH": 0.4496, - "hfopenllm_v2/MATH Level 5": 0.1322, - "hfopenllm_v2/GPQA": 0.2869, - "hfopenllm_v2/MUSR": 0.427, - "hfopenllm_v2/MMLU-PRO": 0.2882 - } - }, - { - "id": "pankajmathur/orca_mini_v9_6_1B-Instruct", - "name": "orca_mini_v9_6_1B-Instruct", - "developer": "pankajmathur", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6086, - "hfopenllm_v2/BBH": 0.3561, - "hfopenllm_v2/MATH Level 5": 0.077, - "hfopenllm_v2/GPQA": 0.2685, - "hfopenllm_v2/MUSR": 0.3396, - "hfopenllm_v2/MMLU-PRO": 0.1809 - } - }, - { - "id": "pankajmathur/orca_mini_v9_6_3B-Instruct", - "name": "orca_mini_v9_6_3B-Instruct", - "developer": "pankajmathur", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7316, - "hfopenllm_v2/BBH": 0.4568, - "hfopenllm_v2/MATH Level 5": 0.1329, - "hfopenllm_v2/GPQA": 0.2936, - "hfopenllm_v2/MUSR": 0.4068, - "hfopenllm_v2/MMLU-PRO": 0.2851 - } - }, - { - "id": "pankajmathur/orca_mini_v9_7_1B-Instruct", - "name": "orca_mini_v9_7_1B-Instruct", - "developer": "pankajmathur", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.561, - "hfopenllm_v2/BBH": 0.3182, - "hfopenllm_v2/MATH Level 5": 0.0446, - "hfopenllm_v2/GPQA": 0.2727, - "hfopenllm_v2/MUSR": 0.3527, - "hfopenllm_v2/MMLU-PRO": 0.1345 - } - }, - { - "id": "pankajmathur/orca_mini_v9_7_3B-Instruct", - "name": "orca_mini_v9_7_3B-Instruct", - "developer": "pankajmathur", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5618, - "hfopenllm_v2/BBH": 0.3297, - "hfopenllm_v2/MATH Level 5": 0.0619, - "hfopenllm_v2/GPQA": 0.2617, - "hfopenllm_v2/MUSR": 0.3619, - "hfopenllm_v2/MMLU-PRO": 0.1375 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/parissa3.json b/data/developers/parissa3.json deleted file mode 100644 index 83eccf6cdb08cd71d0852659ed3f4e1c5dc29824..0000000000000000000000000000000000000000 --- a/data/developers/parissa3.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "Parissa3", - "models": [ - { - "id": "Parissa3/test-model", - "name": "test-model", - "developer": "Parissa3", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3883, - "hfopenllm_v2/BBH": 0.5194, - "hfopenllm_v2/MATH Level 5": 0.065, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.4685, - "hfopenllm_v2/MMLU-PRO": 0.3057 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/paulml.json b/data/developers/paulml.json deleted file mode 100644 index 79ccf1de04171581ead52be392d5d52d76f09fea..0000000000000000000000000000000000000000 --- a/data/developers/paulml.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "paulml", - "models": [ - { - "id": "paulml/ECE-ILAB-Q1", - "name": "ECE-ILAB-Q1", - "developer": "paulml", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7865, - "hfopenllm_v2/BBH": 0.6718, - "hfopenllm_v2/MATH Level 5": 0.3557, - "hfopenllm_v2/GPQA": 0.3867, - "hfopenllm_v2/MUSR": 0.4614, - "hfopenllm_v2/MMLU-PRO": 0.5505 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/phronetic-ai.json b/data/developers/phronetic-ai.json deleted file mode 100644 index 6361af98c7118026cfba773c4c13af0b8507c5a3..0000000000000000000000000000000000000000 --- a/data/developers/phronetic-ai.json +++ /dev/null @@ -1,45 +0,0 @@ -{ - "developer": "phronetic-ai", - "models": [ - { - "id": "phronetic-ai/rzn-t-prompt", - "name": "RZN-T (Prompt)", - "developer": "phronetic-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 96.0, - "bfcl/bfcl.overall.overall_accuracy": 22.25, - "bfcl/bfcl.overall.total_cost_usd": 12.31, - "bfcl/bfcl.overall.latency_mean_s": 12.32, - "bfcl/bfcl.overall.latency_std_s": 27.53, - "bfcl/bfcl.overall.latency_p95_s": 39.84, - "bfcl/bfcl.non_live.ast_accuracy": 67.94, - "bfcl/bfcl.non_live.simple_ast_accuracy": 63.75, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 75.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 69.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 63.5, - "bfcl/bfcl.live.live_accuracy": 49.74, - "bfcl/bfcl.live.live_simple_ast_accuracy": 61.24, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 47.2, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 43.75, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 41.67, - "bfcl/bfcl.multi_turn.accuracy": 2.88, - "bfcl/bfcl.multi_turn.base_accuracy": 4.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 2.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 2.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 2.5, - "bfcl/bfcl.web_search.accuracy": 0.0, - "bfcl/bfcl.web_search.base_accuracy": 0.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 0.0, - "bfcl/bfcl.memory.accuracy": 6.88, - "bfcl/bfcl.memory.kv_accuracy": 9.03, - "bfcl/bfcl.memory.vector_accuracy": 6.45, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 5.16, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 81.25, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 82.41, - "bfcl/bfcl.format_sensitivity.max_delta": 63.5, - "bfcl/bfcl.format_sensitivity.stddev": 25.53 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/pinkstack.json b/data/developers/pinkstack.json deleted file mode 100644 index a4a4d32c248f64869858ac2d7497f1b5f19ed120..0000000000000000000000000000000000000000 --- a/data/developers/pinkstack.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "developer": "Pinkstack", - "models": [ - { - "id": "Pinkstack/PARM-V1.5-base-QwQ-Qwen-2.5-o1-3B", - "name": "PARM-V1.5-base-QwQ-Qwen-2.5-o1-3B", - "developer": "Pinkstack", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5085, - "hfopenllm_v2/BBH": 0.4711, - "hfopenllm_v2/MATH Level 5": 0.1692, - "hfopenllm_v2/GPQA": 0.297, - "hfopenllm_v2/MUSR": 0.4479, - "hfopenllm_v2/MMLU-PRO": 0.3511 - } - }, - { - "id": "Pinkstack/SuperThoughts-CoT-14B-16k-o1-QwQ", - "name": "SuperThoughts-CoT-14B-16k-o1-QwQ", - "developer": "Pinkstack", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0515, - "hfopenllm_v2/BBH": 0.672, - "hfopenllm_v2/MATH Level 5": 0.4199, - "hfopenllm_v2/GPQA": 0.3926, - "hfopenllm_v2/MUSR": 0.4914, - "hfopenllm_v2/MMLU-PRO": 0.5268 - } - }, - { - "id": "Pinkstack/Superthoughts-lite-1.8B-experimental-o1", - "name": "Superthoughts-lite-1.8B-experimental-o1", - "developer": "Pinkstack", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0375, - "hfopenllm_v2/BBH": 0.3435, - "hfopenllm_v2/MATH Level 5": 0.0317, - "hfopenllm_v2/GPQA": 0.2752, - "hfopenllm_v2/MUSR": 0.3354, - "hfopenllm_v2/MMLU-PRO": 0.1851 - } - }, - { - "id": "Pinkstack/Superthoughts-lite-v1", - "name": "Superthoughts-lite-v1", - "developer": "Pinkstack", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1659, - "hfopenllm_v2/BBH": 0.3466, - "hfopenllm_v2/MATH Level 5": 0.0295, - "hfopenllm_v2/GPQA": 0.281, - "hfopenllm_v2/MUSR": 0.3672, - "hfopenllm_v2/MMLU-PRO": 0.1755 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/pints-ai.json b/data/developers/pints-ai.json deleted file mode 100644 index c84130dcb16187404bc96a785501ec5f09292a97..0000000000000000000000000000000000000000 --- a/data/developers/pints-ai.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "pints-ai", - "models": [ - { - "id": "pints-ai/1.5-Pints-16K-v0.1", - "name": "1.5-Pints-16K-v0.1", - "developer": "pints-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1636, - "hfopenllm_v2/BBH": 0.3133, - "hfopenllm_v2/MATH Level 5": 0.0144, - "hfopenllm_v2/GPQA": 0.2357, - "hfopenllm_v2/MUSR": 0.3579, - "hfopenllm_v2/MMLU-PRO": 0.1119 - } - }, - { - "id": "pints-ai/1.5-Pints-2K-v0.1", - "name": "1.5-Pints-2K-v0.1", - "developer": "pints-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1762, - "hfopenllm_v2/BBH": 0.298, - "hfopenllm_v2/MATH Level 5": 0.0128, - "hfopenllm_v2/GPQA": 0.2483, - "hfopenllm_v2/MUSR": 0.3502, - "hfopenllm_v2/MMLU-PRO": 0.1104 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/piotr25691.json b/data/developers/piotr25691.json deleted file mode 100644 index 1e4f9dc41e6d677d63260ee2a5456c180d5ccc40..0000000000000000000000000000000000000000 --- a/data/developers/piotr25691.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "developer": "piotr25691", - "models": [ - { - "id": "piotr25691/thea-3b-25r", - "name": "thea-3b-25r", - "developer": "piotr25691", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7344, - "hfopenllm_v2/BBH": 0.4484, - "hfopenllm_v2/MATH Level 5": 0.1782, - "hfopenllm_v2/GPQA": 0.2676, - "hfopenllm_v2/MUSR": 0.3315, - "hfopenllm_v2/MMLU-PRO": 0.3182 - } - }, - { - "id": "piotr25691/thea-c-3b-25r", - "name": "thea-c-3b-25r", - "developer": "piotr25691", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7402, - "hfopenllm_v2/BBH": 0.4532, - "hfopenllm_v2/MATH Level 5": 0.1526, - "hfopenllm_v2/GPQA": 0.2651, - "hfopenllm_v2/MUSR": 0.3315, - "hfopenllm_v2/MMLU-PRO": 0.3178 - } - }, - { - "id": "piotr25691/thea-rp-3b-25r", - "name": "thea-rp-3b-25r", - "developer": "piotr25691", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6578, - "hfopenllm_v2/BBH": 0.439, - "hfopenllm_v2/MATH Level 5": 0.1322, - "hfopenllm_v2/GPQA": 0.2743, - "hfopenllm_v2/MUSR": 0.3819, - "hfopenllm_v2/MMLU-PRO": 0.306 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/pjmixers-dev.json b/data/developers/pjmixers-dev.json deleted file mode 100644 index 0b7d3b30cc96b19fa424505465769a9547729ed5..0000000000000000000000000000000000000000 --- a/data/developers/pjmixers-dev.json +++ /dev/null @@ -1,131 +0,0 @@ -{ - "developer": "PJMixers-Dev", - "models": [ - { - "id": "PJMixers-Dev/L3.2-Instruct-Thinking-v0.1-1B", - "name": "L3.2-Instruct-Thinking-v0.1-1B", - "developer": "PJMixers-Dev", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4628, - "hfopenllm_v2/BBH": 0.3302, - "hfopenllm_v2/MATH Level 5": 0.0544, - "hfopenllm_v2/GPQA": 0.2576, - "hfopenllm_v2/MUSR": 0.3262, - "hfopenllm_v2/MMLU-PRO": 0.1483 - } - }, - { - "id": "PJMixers-Dev/LLaMa-3.1-Instruct-Interleaved-Zeroed-13B", - "name": "LLaMa-3.1-Instruct-Interleaved-Zeroed-13B", - "developer": "PJMixers-Dev", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7871, - "hfopenllm_v2/BBH": 0.5073, - "hfopenllm_v2/MATH Level 5": 0.2002, - "hfopenllm_v2/GPQA": 0.2919, - "hfopenllm_v2/MUSR": 0.387, - "hfopenllm_v2/MMLU-PRO": 0.3767 - } - }, - { - "id": "PJMixers-Dev/LLaMa-3.1-RomboTiesTest-8B", - "name": "LLaMa-3.1-RomboTiesTest-8B", - "developer": "PJMixers-Dev", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7825, - "hfopenllm_v2/BBH": 0.5073, - "hfopenllm_v2/MATH Level 5": 0.2002, - "hfopenllm_v2/GPQA": 0.2919, - "hfopenllm_v2/MUSR": 0.387, - "hfopenllm_v2/MMLU-PRO": 0.3767 - } - }, - { - "id": "PJMixers-Dev/LLaMa-3.1-RomboTiesTest2-8B", - "name": "LLaMa-3.1-RomboTiesTest2-8B", - "developer": "PJMixers-Dev", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7825, - "hfopenllm_v2/BBH": 0.5073, - "hfopenllm_v2/MATH Level 5": 0.2002, - "hfopenllm_v2/GPQA": 0.2919, - "hfopenllm_v2/MUSR": 0.387, - "hfopenllm_v2/MMLU-PRO": 0.3767 - } - }, - { - "id": "PJMixers-Dev/LLaMa-3.2-Instruct-JankMix-v0.1-SFT-3B", - "name": "LLaMa-3.2-Instruct-JankMix-v0.1-SFT-3B", - "developer": "PJMixers-Dev", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6931, - "hfopenllm_v2/BBH": 0.4556, - "hfopenllm_v2/MATH Level 5": 0.1216, - "hfopenllm_v2/GPQA": 0.2743, - "hfopenllm_v2/MUSR": 0.37, - "hfopenllm_v2/MMLU-PRO": 0.3127 - } - }, - { - "id": "PJMixers-Dev/LLaMa-3.2-Instruct-JankMix-v0.2-SFT-3B", - "name": "LLaMa-3.2-Instruct-JankMix-v0.2-SFT-3B", - "developer": "PJMixers-Dev", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6292, - "hfopenllm_v2/BBH": 0.4581, - "hfopenllm_v2/MATH Level 5": 0.1299, - "hfopenllm_v2/GPQA": 0.2727, - "hfopenllm_v2/MUSR": 0.3659, - "hfopenllm_v2/MMLU-PRO": 0.3115 - } - }, - { - "id": "PJMixers-Dev/LLaMa-3.2-Instruct-JankMix-v0.2-SFT-HailMary-v0.1-KTO-3B", - "name": "LLaMa-3.2-Instruct-JankMix-v0.2-SFT-HailMary-v0.1-KTO-3B", - "developer": "PJMixers-Dev", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6504, - "hfopenllm_v2/BBH": 0.4511, - "hfopenllm_v2/MATH Level 5": 0.1261, - "hfopenllm_v2/GPQA": 0.2718, - "hfopenllm_v2/MUSR": 0.3687, - "hfopenllm_v2/MMLU-PRO": 0.3108 - } - }, - { - "id": "PJMixers-Dev/LLaMa-3.2-Instruct-JankMixBread-v0.1-3B", - "name": "LLaMa-3.2-Instruct-JankMixBread-v0.1-3B", - "developer": "PJMixers-Dev", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5041, - "hfopenllm_v2/BBH": 0.4483, - "hfopenllm_v2/MATH Level 5": 0.1307, - "hfopenllm_v2/GPQA": 0.2827, - "hfopenllm_v2/MUSR": 0.3516, - "hfopenllm_v2/MMLU-PRO": 0.3083 - } - }, - { - "id": "PJMixers-Dev/Qwen2.5-RomboTiesTest-7B", - "name": "Qwen2.5-RomboTiesTest-7B", - "developer": "PJMixers-Dev", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7558, - "hfopenllm_v2/BBH": 0.5399, - "hfopenllm_v2/MATH Level 5": 0.4962, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.4034, - "hfopenllm_v2/MMLU-PRO": 0.4285 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/pjmixers.json b/data/developers/pjmixers.json deleted file mode 100644 index 114530932a47032429f611ae892cb01ef93e092f..0000000000000000000000000000000000000000 --- a/data/developers/pjmixers.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "PJMixers", - "models": [ - { - "id": "PJMixers/LLaMa-3-CursedStock-v2.0-8B", - "name": "LLaMa-3-CursedStock-v2.0-8B", - "developer": "PJMixers", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6331, - "hfopenllm_v2/BBH": 0.5271, - "hfopenllm_v2/MATH Level 5": 0.0944, - "hfopenllm_v2/GPQA": 0.2743, - "hfopenllm_v2/MUSR": 0.3856, - "hfopenllm_v2/MMLU-PRO": 0.3556 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/pku-alignment.json b/data/developers/pku-alignment.json deleted file mode 100644 index e10415940119893822695aee0e6a54e9dee27bc5..0000000000000000000000000000000000000000 --- a/data/developers/pku-alignment.json +++ /dev/null @@ -1,81 +0,0 @@ -{ - "developer": "PKU-Alignment", - "models": [ - { - "id": "PKU-Alignment/beaver-7b-v1.0-cost", - "name": "PKU-Alignment/beaver-7b-v1.0-cost", - "developer": "PKU-Alignment", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5798, - "reward-bench/Factuality": 0.3263, - "reward-bench/Precise IF": 0.2313, - "reward-bench/Math": 0.3989, - "reward-bench/Safety": 0.7351, - "reward-bench/Focus": 0.2939, - "reward-bench/Ties": -0.01, - "reward-bench/Chat": 0.6173, - "reward-bench/Chat Hard": 0.4232, - "reward-bench/Reasoning": 0.5482, - "reward-bench/Prior Sets (0.5 weight)": 0.57 - } - }, - { - "id": "PKU-Alignment/beaver-7b-v1.0-reward", - "name": "PKU-Alignment/beaver-7b-v1.0-reward", - "developer": "PKU-Alignment", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.4727, - "reward-bench/Factuality": 0.2105, - "reward-bench/Precise IF": 0.2938, - "reward-bench/Math": 0.2623, - "reward-bench/Safety": 0.3757, - "reward-bench/Focus": 0.0646, - "reward-bench/Ties": -0.01, - "reward-bench/Chat": 0.8184, - "reward-bench/Chat Hard": 0.2873, - "reward-bench/Reasoning": 0.346, - "reward-bench/Prior Sets (0.5 weight)": 0.5993 - } - }, - { - "id": "PKU-Alignment/beaver-7b-v2.0-cost", - "name": "PKU-Alignment/beaver-7b-v2.0-cost", - "developer": "PKU-Alignment", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5957, - "reward-bench/Factuality": 0.3789, - "reward-bench/Precise IF": 0.275, - "reward-bench/Math": 0.3333, - "reward-bench/Safety": 0.7608, - "reward-bench/Focus": 0.2828, - "reward-bench/Ties": -0.01, - "reward-bench/Chat": 0.5726, - "reward-bench/Chat Hard": 0.4561, - "reward-bench/Reasoning": 0.6211, - "reward-bench/Prior Sets (0.5 weight)": 0.5397 - } - }, - { - "id": "PKU-Alignment/beaver-7b-v2.0-reward", - "name": "PKU-Alignment/beaver-7b-v2.0-reward", - "developer": "PKU-Alignment", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6366, - "reward-bench/Factuality": 0.2168, - "reward-bench/Precise IF": 0.2562, - "reward-bench/Math": 0.3825, - "reward-bench/Safety": 0.6041, - "reward-bench/Focus": 0.2606, - "reward-bench/Ties": 0.0944, - "reward-bench/Chat": 0.8994, - "reward-bench/Chat Hard": 0.364, - "reward-bench/Reasoning": 0.6887, - "reward-bench/Prior Sets (0.5 weight)": 0.6171 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/pocketdoc.json b/data/developers/pocketdoc.json deleted file mode 100644 index 49f2a76d7d6900882895140c645441a81a158366..0000000000000000000000000000000000000000 --- a/data/developers/pocketdoc.json +++ /dev/null @@ -1,75 +0,0 @@ -{ - "developer": "PocketDoc", - "models": [ - { - "id": "PocketDoc/Dans-Instruct-CoreCurriculum-12b", - "name": "Dans-Instruct-CoreCurriculum-12b", - "developer": "PocketDoc", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2191, - "hfopenllm_v2/BBH": 0.3789, - "hfopenllm_v2/MATH Level 5": 0.0544, - "hfopenllm_v2/GPQA": 0.2827, - "hfopenllm_v2/MUSR": 0.4096, - "hfopenllm_v2/MMLU-PRO": 0.1219 - } - }, - { - "id": "PocketDoc/Dans-PersonalityEngine-v1.0.0-8b", - "name": "Dans-PersonalityEngine-v1.0.0-8b", - "developer": "PocketDoc", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4982, - "hfopenllm_v2/BBH": 0.4733, - "hfopenllm_v2/MATH Level 5": 0.0816, - "hfopenllm_v2/GPQA": 0.2852, - "hfopenllm_v2/MUSR": 0.3542, - "hfopenllm_v2/MMLU-PRO": 0.3065 - } - }, - { - "id": "PocketDoc/Dans-PersonalityEngine-V1.1.0-12b", - "name": "Dans-PersonalityEngine-V1.1.0-12b", - "developer": "PocketDoc", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7075, - "hfopenllm_v2/BBH": 0.5361, - "hfopenllm_v2/MATH Level 5": 0.105, - "hfopenllm_v2/GPQA": 0.2869, - "hfopenllm_v2/MUSR": 0.4587, - "hfopenllm_v2/MMLU-PRO": 0.3262 - } - }, - { - "id": "PocketDoc/Dans-PersonalityEngine-V1.2.0-24b", - "name": "Dans-PersonalityEngine-V1.2.0-24b", - "developer": "PocketDoc", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7886, - "hfopenllm_v2/BBH": 0.6421, - "hfopenllm_v2/MATH Level 5": 0.2455, - "hfopenllm_v2/GPQA": 0.3188, - "hfopenllm_v2/MUSR": 0.43, - "hfopenllm_v2/MMLU-PRO": 0.5026 - } - }, - { - "id": "PocketDoc/Dans-SakuraKaze-V1.0.0-12b", - "name": "Dans-SakuraKaze-V1.0.0-12b", - "developer": "PocketDoc", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.652, - "hfopenllm_v2/BBH": 0.5405, - "hfopenllm_v2/MATH Level 5": 0.0929, - "hfopenllm_v2/GPQA": 0.2936, - "hfopenllm_v2/MUSR": 0.4745, - "hfopenllm_v2/MMLU-PRO": 0.356 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/poll.json b/data/developers/poll.json deleted file mode 100644 index 654e212b1aaaf403fa50a809752fa41a54bc85fd..0000000000000000000000000000000000000000 --- a/data/developers/poll.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "developer": "PoLL", - "models": [ - { - "id": "PoLL/gpt-3.5-turbo-0125_claude-3-sonnet-2024022...", - "name": "PoLL/gpt-3.5-turbo-0125_claude-3-sonnet-2024022...", - "developer": "PoLL", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7578, - "reward-bench/Chat": 0.9525, - "reward-bench/Chat Hard": 0.5406, - "reward-bench/Safety": 0.8034, - "reward-bench/Reasoning": 0.7346 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/postbot.json b/data/developers/postbot.json deleted file mode 100644 index 663a02f652b9c08bc4ad5fb99a26081f82712198..0000000000000000000000000000000000000000 --- a/data/developers/postbot.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "postbot", - "models": [ - { - "id": "postbot/gpt2-medium-emailgen", - "name": "gpt2-medium-emailgen", - "developer": "postbot", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1492, - "hfopenllm_v2/BBH": 0.313, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3911, - "hfopenllm_v2/MMLU-PRO": 0.1147 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/powerinfer.json b/data/developers/powerinfer.json deleted file mode 100644 index 9191dec6772330f9ad0b1eeccf587f30deb4e088..0000000000000000000000000000000000000000 --- a/data/developers/powerinfer.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "PowerInfer", - "models": [ - { - "id": "PowerInfer/SmallThinker-3B-Preview", - "name": "SmallThinker-3B-Preview", - "developer": "PowerInfer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.62, - "hfopenllm_v2/BBH": 0.4495, - "hfopenllm_v2/MATH Level 5": 0.2779, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.3525, - "hfopenllm_v2/MMLU-PRO": 0.3018 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/pranavharshan.json b/data/developers/pranavharshan.json deleted file mode 100644 index 309a69bba59fe1c32060e0d57bf412cecc18d376..0000000000000000000000000000000000000000 --- a/data/developers/pranavharshan.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "PranavHarshan", - "models": [ - { - "id": "PranavHarshan/LaMistral-V4", - "name": "LaMistral-V4", - "developer": "PranavHarshan", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6239, - "hfopenllm_v2/BBH": 0.5184, - "hfopenllm_v2/MATH Level 5": 0.0687, - "hfopenllm_v2/GPQA": 0.328, - "hfopenllm_v2/MUSR": 0.3643, - "hfopenllm_v2/MMLU-PRO": 0.3599 - } - }, - { - "id": "PranavHarshan/MedNarra-X1", - "name": "MedNarra-X1", - "developer": "PranavHarshan", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4338, - "hfopenllm_v2/BBH": 0.4637, - "hfopenllm_v2/MATH Level 5": 0.0438, - "hfopenllm_v2/GPQA": 0.3079, - "hfopenllm_v2/MUSR": 0.354, - "hfopenllm_v2/MMLU-PRO": 0.3431 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/pretergeek.json b/data/developers/pretergeek.json deleted file mode 100644 index 78652e4ddb223f51b181d3459e21a2725b73cd6b..0000000000000000000000000000000000000000 --- a/data/developers/pretergeek.json +++ /dev/null @@ -1,131 +0,0 @@ -{ - "developer": "Pretergeek", - "models": [ - { - "id": "Pretergeek/OpenChat-3.5-0106_10.7B_48Layers-Appended", - "name": "OpenChat-3.5-0106_10.7B_48Layers-Appended", - "developer": "Pretergeek", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5961, - "hfopenllm_v2/BBH": 0.462, - "hfopenllm_v2/MATH Level 5": 0.0793, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.4254, - "hfopenllm_v2/MMLU-PRO": 0.329 - } - }, - { - "id": "Pretergeek/OpenChat-3.5-0106_10.7B_48Layers-Interleaved", - "name": "OpenChat-3.5-0106_10.7B_48Layers-Interleaved", - "developer": "Pretergeek", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5961, - "hfopenllm_v2/BBH": 0.462, - "hfopenllm_v2/MATH Level 5": 0.0778, - "hfopenllm_v2/GPQA": 0.3045, - "hfopenllm_v2/MUSR": 0.4254, - "hfopenllm_v2/MMLU-PRO": 0.3299 - } - }, - { - "id": "Pretergeek/OpenChat-3.5-0106_32K-PoSE", - "name": "OpenChat-3.5-0106_32K-PoSE", - "developer": "Pretergeek", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3969, - "hfopenllm_v2/BBH": 0.3471, - "hfopenllm_v2/MATH Level 5": 0.0264, - "hfopenllm_v2/GPQA": 0.276, - "hfopenllm_v2/MUSR": 0.4205, - "hfopenllm_v2/MMLU-PRO": 0.2031 - } - }, - { - "id": "Pretergeek/OpenChat-3.5-0106_8.11B_36Layers-Appended", - "name": "OpenChat-3.5-0106_8.11B_36Layers-Appended", - "developer": "Pretergeek", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5976, - "hfopenllm_v2/BBH": 0.462, - "hfopenllm_v2/MATH Level 5": 0.0793, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.4254, - "hfopenllm_v2/MMLU-PRO": 0.329 - } - }, - { - "id": "Pretergeek/OpenChat-3.5-0106_8.11B_36Layers-Interleaved", - "name": "OpenChat-3.5-0106_8.11B_36Layers-Interleaved", - "developer": "Pretergeek", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5961, - "hfopenllm_v2/BBH": 0.4621, - "hfopenllm_v2/MATH Level 5": 0.0778, - "hfopenllm_v2/GPQA": 0.3045, - "hfopenllm_v2/MUSR": 0.4241, - "hfopenllm_v2/MMLU-PRO": 0.3299 - } - }, - { - "id": "Pretergeek/OpenChat-3.5-0106_8.99B_40Layers-Appended", - "name": "OpenChat-3.5-0106_8.99B_40Layers-Appended", - "developer": "Pretergeek", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5961, - "hfopenllm_v2/BBH": 0.462, - "hfopenllm_v2/MATH Level 5": 0.0793, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.4254, - "hfopenllm_v2/MMLU-PRO": 0.329 - } - }, - { - "id": "Pretergeek/OpenChat-3.5-0106_8.99B_40Layers-Interleaved", - "name": "OpenChat-3.5-0106_8.99B_40Layers-Interleaved", - "developer": "Pretergeek", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5976, - "hfopenllm_v2/BBH": 0.4621, - "hfopenllm_v2/MATH Level 5": 0.0778, - "hfopenllm_v2/GPQA": 0.3045, - "hfopenllm_v2/MUSR": 0.4241, - "hfopenllm_v2/MMLU-PRO": 0.3299 - } - }, - { - "id": "Pretergeek/OpenChat-3.5-0106_9.86B_44Layers-Appended", - "name": "OpenChat-3.5-0106_9.86B_44Layers-Appended", - "developer": "Pretergeek", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5961, - "hfopenllm_v2/BBH": 0.462, - "hfopenllm_v2/MATH Level 5": 0.0793, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.4254, - "hfopenllm_v2/MMLU-PRO": 0.329 - } - }, - { - "id": "Pretergeek/openchat-3.5-0106_Rebased_Mistral-7B-v0.2", - "name": "openchat-3.5-0106_Rebased_Mistral-7B-v0.2", - "developer": "Pretergeek", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3706, - "hfopenllm_v2/BBH": 0.3627, - "hfopenllm_v2/MATH Level 5": 0.0453, - "hfopenllm_v2/GPQA": 0.2718, - "hfopenllm_v2/MUSR": 0.484, - "hfopenllm_v2/MMLU-PRO": 0.283 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/primeintellect.json b/data/developers/primeintellect.json deleted file mode 100644 index 160722785b06f80d9b220dee435fad1245d45495..0000000000000000000000000000000000000000 --- a/data/developers/primeintellect.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "PrimeIntellect", - "models": [ - { - "id": "PrimeIntellect/INTELLECT-1", - "name": "INTELLECT-1", - "developer": "PrimeIntellect", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1757, - "hfopenllm_v2/BBH": 0.274, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.25, - "hfopenllm_v2/MUSR": 0.3753, - "hfopenllm_v2/MMLU-PRO": 0.112 - } - }, - { - "id": "PrimeIntellect/INTELLECT-1-Instruct", - "name": "INTELLECT-1-Instruct", - "developer": "PrimeIntellect", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0, - "hfopenllm_v2/BBH": 0.287, - "hfopenllm_v2/MATH Level 5": 0.0227, - "hfopenllm_v2/GPQA": 0.2483, - "hfopenllm_v2/MUSR": 0.3577, - "hfopenllm_v2/MMLU-PRO": 0.1064 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/prince-canuma.json b/data/developers/prince-canuma.json deleted file mode 100644 index 1479ff9192e5e7f0ed71e7814a1c507d6d0537c1..0000000000000000000000000000000000000000 --- a/data/developers/prince-canuma.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "prince-canuma", - "models": [ - { - "id": "prince-canuma/Ministral-8B-Instruct-2410-HF", - "name": "Ministral-8B-Instruct-2410-HF", - "developer": "prince-canuma", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5912, - "hfopenllm_v2/BBH": 0.4586, - "hfopenllm_v2/MATH Level 5": 0.1918, - "hfopenllm_v2/GPQA": 0.281, - "hfopenllm_v2/MUSR": 0.4138, - "hfopenllm_v2/MMLU-PRO": 0.3298 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/princeton-nlp.json b/data/developers/princeton-nlp.json deleted file mode 100644 index c2ef64ee6de3b0368749e3cc226eadfed8e75e4e..0000000000000000000000000000000000000000 --- a/data/developers/princeton-nlp.json +++ /dev/null @@ -1,719 +0,0 @@ -{ - "developer": "princeton-nlp", - "models": [ - { - "id": "princeton-nlp/gemma-2-9b-it-DPO", - "name": "gemma-2-9b-it-DPO", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2769, - "hfopenllm_v2/BBH": 0.5941, - "hfopenllm_v2/MATH Level 5": 0.0831, - "hfopenllm_v2/GPQA": 0.3356, - "hfopenllm_v2/MUSR": 0.382, - "hfopenllm_v2/MMLU-PRO": 0.3723 - } - }, - { - "id": "princeton-nlp/gemma-2-9b-it-SimPO", - "name": "gemma-2-9b-it-SimPO", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3207, - "hfopenllm_v2/BBH": 0.5839, - "hfopenllm_v2/MATH Level 5": 0.071, - "hfopenllm_v2/GPQA": 0.3356, - "hfopenllm_v2/MUSR": 0.4123, - "hfopenllm_v2/MMLU-PRO": 0.3975 - } - }, - { - "id": "princeton-nlp/Llama-3-8B-ProLong-512k-Base", - "name": "Llama-3-8B-ProLong-512k-Base", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5322, - "hfopenllm_v2/BBH": 0.5033, - "hfopenllm_v2/MATH Level 5": 0.0687, - "hfopenllm_v2/GPQA": 0.2617, - "hfopenllm_v2/MUSR": 0.4223, - "hfopenllm_v2/MMLU-PRO": 0.3329 - } - }, - { - "id": "princeton-nlp/Llama-3-8B-ProLong-512k-Instruct", - "name": "Llama-3-8B-ProLong-512k-Instruct", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5508, - "hfopenllm_v2/BBH": 0.5028, - "hfopenllm_v2/MATH Level 5": 0.0529, - "hfopenllm_v2/GPQA": 0.2861, - "hfopenllm_v2/MUSR": 0.4266, - "hfopenllm_v2/MMLU-PRO": 0.3231 - } - }, - { - "id": "princeton-nlp/Llama-3-8B-ProLong-64k-Base", - "name": "Llama-3-8B-ProLong-64k-Base", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5201, - "hfopenllm_v2/BBH": 0.4927, - "hfopenllm_v2/MATH Level 5": 0.065, - "hfopenllm_v2/GPQA": 0.2651, - "hfopenllm_v2/MUSR": 0.4341, - "hfopenllm_v2/MMLU-PRO": 0.3348 - } - }, - { - "id": "princeton-nlp/Llama-3-8B-ProLong-64k-Instruct", - "name": "Llama-3-8B-ProLong-64k-Instruct", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5563, - "hfopenllm_v2/BBH": 0.5083, - "hfopenllm_v2/MATH Level 5": 0.065, - "hfopenllm_v2/GPQA": 0.2953, - "hfopenllm_v2/MUSR": 0.4397, - "hfopenllm_v2/MMLU-PRO": 0.3275 - } - }, - { - "id": "princeton-nlp/Llama-3-Base-8B-SFT", - "name": "Llama-3-Base-8B-SFT", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2796, - "hfopenllm_v2/BBH": 0.4643, - "hfopenllm_v2/MATH Level 5": 0.04, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.4118, - "hfopenllm_v2/MMLU-PRO": 0.3093 - } - }, - { - "id": "princeton-nlp/Llama-3-Base-8B-SFT-CPO", - "name": "Llama-3-Base-8B-SFT-CPO", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3703, - "hfopenllm_v2/BBH": 0.4595, - "hfopenllm_v2/MATH Level 5": 0.0544, - "hfopenllm_v2/GPQA": 0.2743, - "hfopenllm_v2/MUSR": 0.3609, - "hfopenllm_v2/MMLU-PRO": 0.2976 - } - }, - { - "id": "princeton-nlp/Llama-3-Base-8B-SFT-DPO", - "name": "Llama-3-Base-8B-SFT-DPO", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4111, - "hfopenllm_v2/BBH": 0.4666, - "hfopenllm_v2/MATH Level 5": 0.0415, - "hfopenllm_v2/GPQA": 0.3104, - "hfopenllm_v2/MUSR": 0.3867, - "hfopenllm_v2/MMLU-PRO": 0.3078 - } - }, - { - "id": "princeton-nlp/Llama-3-Base-8B-SFT-IPO", - "name": "Llama-3-Base-8B-SFT-IPO", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4487, - "hfopenllm_v2/BBH": 0.469, - "hfopenllm_v2/MATH Level 5": 0.0393, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.3919, - "hfopenllm_v2/MMLU-PRO": 0.3115 - } - }, - { - "id": "princeton-nlp/Llama-3-Base-8B-SFT-KTO", - "name": "Llama-3-Base-8B-SFT-KTO", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4523, - "hfopenllm_v2/BBH": 0.4693, - "hfopenllm_v2/MATH Level 5": 0.0529, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.3842, - "hfopenllm_v2/MMLU-PRO": 0.3054 - } - }, - { - "id": "princeton-nlp/Llama-3-Base-8B-SFT-ORPO", - "name": "Llama-3-Base-8B-SFT-ORPO", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4517, - "hfopenllm_v2/BBH": 0.4734, - "hfopenllm_v2/MATH Level 5": 0.0468, - "hfopenllm_v2/GPQA": 0.3138, - "hfopenllm_v2/MUSR": 0.3707, - "hfopenllm_v2/MMLU-PRO": 0.3083 - } - }, - { - "id": "princeton-nlp/Llama-3-Base-8B-SFT-RDPO", - "name": "Llama-3-Base-8B-SFT-RDPO", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.448, - "hfopenllm_v2/BBH": 0.4662, - "hfopenllm_v2/MATH Level 5": 0.0574, - "hfopenllm_v2/GPQA": 0.3062, - "hfopenllm_v2/MUSR": 0.4027, - "hfopenllm_v2/MMLU-PRO": 0.3014 - } - }, - { - "id": "princeton-nlp/Llama-3-Base-8B-SFT-RRHF", - "name": "Llama-3-Base-8B-SFT-RRHF", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3357, - "hfopenllm_v2/BBH": 0.452, - "hfopenllm_v2/MATH Level 5": 0.0453, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.3722, - "hfopenllm_v2/MMLU-PRO": 0.2889 - } - }, - { - "id": "princeton-nlp/Llama-3-Base-8B-SFT-SimPO", - "name": "Llama-3-Base-8B-SFT-SimPO", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4685, - "hfopenllm_v2/BBH": 0.4741, - "hfopenllm_v2/MATH Level 5": 0.0551, - "hfopenllm_v2/GPQA": 0.2886, - "hfopenllm_v2/MUSR": 0.4127, - "hfopenllm_v2/MMLU-PRO": 0.3105 - } - }, - { - "id": "princeton-nlp/Llama-3-Base-8B-SFT-SLiC-HF", - "name": "Llama-3-Base-8B-SFT-SLiC-HF", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.489, - "hfopenllm_v2/BBH": 0.4704, - "hfopenllm_v2/MATH Level 5": 0.0506, - "hfopenllm_v2/GPQA": 0.2869, - "hfopenllm_v2/MUSR": 0.4091, - "hfopenllm_v2/MMLU-PRO": 0.3063 - } - }, - { - "id": "princeton-nlp/Llama-3-Instruct-8B-CPO", - "name": "Llama-3-Instruct-8B-CPO", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7293, - "hfopenllm_v2/BBH": 0.4999, - "hfopenllm_v2/MATH Level 5": 0.0989, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3514, - "hfopenllm_v2/MMLU-PRO": 0.3652 - } - }, - { - "id": "princeton-nlp/Llama-3-Instruct-8B-CPO-v0.2", - "name": "Llama-3-Instruct-8B-CPO-v0.2", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7506, - "hfopenllm_v2/BBH": 0.5027, - "hfopenllm_v2/MATH Level 5": 0.108, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.3619, - "hfopenllm_v2/MMLU-PRO": 0.3706 - } - }, - { - "id": "princeton-nlp/Llama-3-Instruct-8B-DPO", - "name": "Llama-3-Instruct-8B-DPO", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6757, - "hfopenllm_v2/BBH": 0.4991, - "hfopenllm_v2/MATH Level 5": 0.0846, - "hfopenllm_v2/GPQA": 0.2718, - "hfopenllm_v2/MUSR": 0.3738, - "hfopenllm_v2/MMLU-PRO": 0.3665 - } - }, - { - "id": "princeton-nlp/Llama-3-Instruct-8B-DPO-v0.2", - "name": "Llama-3-Instruct-8B-DPO-v0.2", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7208, - "hfopenllm_v2/BBH": 0.5056, - "hfopenllm_v2/MATH Level 5": 0.0899, - "hfopenllm_v2/GPQA": 0.2869, - "hfopenllm_v2/MUSR": 0.3844, - "hfopenllm_v2/MMLU-PRO": 0.3769 - } - }, - { - "id": "princeton-nlp/Llama-3-Instruct-8B-KTO", - "name": "Llama-3-Instruct-8B-KTO", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6864, - "hfopenllm_v2/BBH": 0.4982, - "hfopenllm_v2/MATH Level 5": 0.0725, - "hfopenllm_v2/GPQA": 0.276, - "hfopenllm_v2/MUSR": 0.3698, - "hfopenllm_v2/MMLU-PRO": 0.3599 - } - }, - { - "id": "princeton-nlp/Llama-3-Instruct-8B-KTO-v0.2", - "name": "Llama-3-Instruct-8B-KTO-v0.2", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.729, - "hfopenllm_v2/BBH": 0.508, - "hfopenllm_v2/MATH Level 5": 0.0997, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3777, - "hfopenllm_v2/MMLU-PRO": 0.3668 - } - }, - { - "id": "princeton-nlp/Llama-3-Instruct-8B-ORPO", - "name": "Llama-3-Instruct-8B-ORPO", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7128, - "hfopenllm_v2/BBH": 0.5001, - "hfopenllm_v2/MATH Level 5": 0.0785, - "hfopenllm_v2/GPQA": 0.2584, - "hfopenllm_v2/MUSR": 0.3502, - "hfopenllm_v2/MMLU-PRO": 0.3646 - } - }, - { - "id": "princeton-nlp/Llama-3-Instruct-8B-ORPO-v0.2", - "name": "Llama-3-Instruct-8B-ORPO-v0.2", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7633, - "hfopenllm_v2/BBH": 0.5078, - "hfopenllm_v2/MATH Level 5": 0.102, - "hfopenllm_v2/GPQA": 0.2836, - "hfopenllm_v2/MUSR": 0.378, - "hfopenllm_v2/MMLU-PRO": 0.3731 - } - }, - { - "id": "princeton-nlp/Llama-3-Instruct-8B-RDPO", - "name": "Llama-3-Instruct-8B-RDPO", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.666, - "hfopenllm_v2/BBH": 0.5034, - "hfopenllm_v2/MATH Level 5": 0.0846, - "hfopenllm_v2/GPQA": 0.2827, - "hfopenllm_v2/MUSR": 0.3752, - "hfopenllm_v2/MMLU-PRO": 0.3607 - } - }, - { - "id": "princeton-nlp/Llama-3-Instruct-8B-RDPO-v0.2", - "name": "Llama-3-Instruct-8B-RDPO-v0.2", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7077, - "hfopenllm_v2/BBH": 0.5049, - "hfopenllm_v2/MATH Level 5": 0.0869, - "hfopenllm_v2/GPQA": 0.2928, - "hfopenllm_v2/MUSR": 0.3804, - "hfopenllm_v2/MMLU-PRO": 0.3774 - } - }, - { - "id": "princeton-nlp/Llama-3-Instruct-8B-RRHF", - "name": "Llama-3-Instruct-8B-RRHF", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7275, - "hfopenllm_v2/BBH": 0.4911, - "hfopenllm_v2/MATH Level 5": 0.0967, - "hfopenllm_v2/GPQA": 0.2802, - "hfopenllm_v2/MUSR": 0.3476, - "hfopenllm_v2/MMLU-PRO": 0.3644 - } - }, - { - "id": "princeton-nlp/Llama-3-Instruct-8B-RRHF-v0.2", - "name": "Llama-3-Instruct-8B-RRHF-v0.2", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7125, - "hfopenllm_v2/BBH": 0.4984, - "hfopenllm_v2/MATH Level 5": 0.0876, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3738, - "hfopenllm_v2/MMLU-PRO": 0.3482 - } - }, - { - "id": "princeton-nlp/Llama-3-Instruct-8B-SimPO", - "name": "Llama-3-Instruct-8B-SimPO", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6504, - "hfopenllm_v2/BBH": 0.4845, - "hfopenllm_v2/MATH Level 5": 0.0861, - "hfopenllm_v2/GPQA": 0.2936, - "hfopenllm_v2/MUSR": 0.3948, - "hfopenllm_v2/MMLU-PRO": 0.3489 - } - }, - { - "id": "princeton-nlp/Llama-3-Instruct-8B-SimPO-v0.2", - "name": "Llama-3-Instruct-8B-SimPO-v0.2", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6809, - "hfopenllm_v2/BBH": 0.5038, - "hfopenllm_v2/MATH Level 5": 0.074, - "hfopenllm_v2/GPQA": 0.3012, - "hfopenllm_v2/MUSR": 0.3988, - "hfopenllm_v2/MMLU-PRO": 0.3622 - } - }, - { - "id": "princeton-nlp/Llama-3-Instruct-8B-SLiC-HF", - "name": "Llama-3-Instruct-8B-SLiC-HF", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.74, - "hfopenllm_v2/BBH": 0.5029, - "hfopenllm_v2/MATH Level 5": 0.0974, - "hfopenllm_v2/GPQA": 0.2861, - "hfopenllm_v2/MUSR": 0.3723, - "hfopenllm_v2/MMLU-PRO": 0.3585 - } - }, - { - "id": "princeton-nlp/Llama-3-Instruct-8B-SLiC-HF-v0.2", - "name": "Llama-3-Instruct-8B-SLiC-HF-v0.2", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.711, - "hfopenllm_v2/BBH": 0.4984, - "hfopenllm_v2/MATH Level 5": 0.0876, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3738, - "hfopenllm_v2/MMLU-PRO": 0.3482 - } - }, - { - "id": "princeton-nlp/Mistral-7B-Base-SFT-CPO", - "name": "Mistral-7B-Base-SFT-CPO", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4655, - "hfopenllm_v2/BBH": 0.4382, - "hfopenllm_v2/MATH Level 5": 0.0279, - "hfopenllm_v2/GPQA": 0.2919, - "hfopenllm_v2/MUSR": 0.4071, - "hfopenllm_v2/MMLU-PRO": 0.2651 - } - }, - { - "id": "princeton-nlp/Mistral-7B-Base-SFT-DPO", - "name": "Mistral-7B-Base-SFT-DPO", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4403, - "hfopenllm_v2/BBH": 0.435, - "hfopenllm_v2/MATH Level 5": 0.0211, - "hfopenllm_v2/GPQA": 0.2727, - "hfopenllm_v2/MUSR": 0.4122, - "hfopenllm_v2/MMLU-PRO": 0.2645 - } - }, - { - "id": "princeton-nlp/Mistral-7B-Base-SFT-IPO", - "name": "Mistral-7B-Base-SFT-IPO", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.483, - "hfopenllm_v2/BBH": 0.4458, - "hfopenllm_v2/MATH Level 5": 0.0287, - "hfopenllm_v2/GPQA": 0.2802, - "hfopenllm_v2/MUSR": 0.3776, - "hfopenllm_v2/MMLU-PRO": 0.2792 - } - }, - { - "id": "princeton-nlp/Mistral-7B-Base-SFT-KTO", - "name": "Mistral-7B-Base-SFT-KTO", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4785, - "hfopenllm_v2/BBH": 0.4476, - "hfopenllm_v2/MATH Level 5": 0.0393, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.4368, - "hfopenllm_v2/MMLU-PRO": 0.2872 - } - }, - { - "id": "princeton-nlp/Mistral-7B-Base-SFT-RDPO", - "name": "Mistral-7B-Base-SFT-RDPO", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4606, - "hfopenllm_v2/BBH": 0.444, - "hfopenllm_v2/MATH Level 5": 0.0219, - "hfopenllm_v2/GPQA": 0.2777, - "hfopenllm_v2/MUSR": 0.3579, - "hfopenllm_v2/MMLU-PRO": 0.2777 - } - }, - { - "id": "princeton-nlp/Mistral-7B-Base-SFT-RRHF", - "name": "Mistral-7B-Base-SFT-RRHF", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4407, - "hfopenllm_v2/BBH": 0.4281, - "hfopenllm_v2/MATH Level 5": 0.0249, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.4187, - "hfopenllm_v2/MMLU-PRO": 0.2398 - } - }, - { - "id": "princeton-nlp/Mistral-7B-Base-SFT-SimPO", - "name": "Mistral-7B-Base-SFT-SimPO", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4701, - "hfopenllm_v2/BBH": 0.4398, - "hfopenllm_v2/MATH Level 5": 0.0144, - "hfopenllm_v2/GPQA": 0.2836, - "hfopenllm_v2/MUSR": 0.3971, - "hfopenllm_v2/MMLU-PRO": 0.2702 - } - }, - { - "id": "princeton-nlp/Mistral-7B-Base-SFT-SLiC-HF", - "name": "Mistral-7B-Base-SFT-SLiC-HF", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5127, - "hfopenllm_v2/BBH": 0.4422, - "hfopenllm_v2/MATH Level 5": 0.0355, - "hfopenllm_v2/GPQA": 0.2919, - "hfopenllm_v2/MUSR": 0.4261, - "hfopenllm_v2/MMLU-PRO": 0.2781 - } - }, - { - "id": "princeton-nlp/Mistral-7B-Instruct-CPO", - "name": "Mistral-7B-Instruct-CPO", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4203, - "hfopenllm_v2/BBH": 0.4069, - "hfopenllm_v2/MATH Level 5": 0.0204, - "hfopenllm_v2/GPQA": 0.2659, - "hfopenllm_v2/MUSR": 0.4178, - "hfopenllm_v2/MMLU-PRO": 0.2701 - } - }, - { - "id": "princeton-nlp/Mistral-7B-Instruct-DPO", - "name": "Mistral-7B-Instruct-DPO", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5176, - "hfopenllm_v2/BBH": 0.406, - "hfopenllm_v2/MATH Level 5": 0.031, - "hfopenllm_v2/GPQA": 0.2685, - "hfopenllm_v2/MUSR": 0.3833, - "hfopenllm_v2/MMLU-PRO": 0.2749 - } - }, - { - "id": "princeton-nlp/Mistral-7B-Instruct-IPO", - "name": "Mistral-7B-Instruct-IPO", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4929, - "hfopenllm_v2/BBH": 0.4322, - "hfopenllm_v2/MATH Level 5": 0.0204, - "hfopenllm_v2/GPQA": 0.2735, - "hfopenllm_v2/MUSR": 0.4324, - "hfopenllm_v2/MMLU-PRO": 0.2708 - } - }, - { - "id": "princeton-nlp/Mistral-7B-Instruct-KTO", - "name": "Mistral-7B-Instruct-KTO", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4908, - "hfopenllm_v2/BBH": 0.414, - "hfopenllm_v2/MATH Level 5": 0.0264, - "hfopenllm_v2/GPQA": 0.2735, - "hfopenllm_v2/MUSR": 0.3953, - "hfopenllm_v2/MMLU-PRO": 0.2812 - } - }, - { - "id": "princeton-nlp/Mistral-7B-Instruct-ORPO", - "name": "Mistral-7B-Instruct-ORPO", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.472, - "hfopenllm_v2/BBH": 0.4104, - "hfopenllm_v2/MATH Level 5": 0.0295, - "hfopenllm_v2/GPQA": 0.2743, - "hfopenllm_v2/MUSR": 0.3912, - "hfopenllm_v2/MMLU-PRO": 0.2662 - } - }, - { - "id": "princeton-nlp/Mistral-7B-Instruct-RDPO", - "name": "Mistral-7B-Instruct-RDPO", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4887, - "hfopenllm_v2/BBH": 0.405, - "hfopenllm_v2/MATH Level 5": 0.0249, - "hfopenllm_v2/GPQA": 0.2802, - "hfopenllm_v2/MUSR": 0.3873, - "hfopenllm_v2/MMLU-PRO": 0.2777 - } - }, - { - "id": "princeton-nlp/Mistral-7B-Instruct-RRHF", - "name": "Mistral-7B-Instruct-RRHF", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.496, - "hfopenllm_v2/BBH": 0.419, - "hfopenllm_v2/MATH Level 5": 0.0279, - "hfopenllm_v2/GPQA": 0.276, - "hfopenllm_v2/MUSR": 0.3979, - "hfopenllm_v2/MMLU-PRO": 0.2651 - } - }, - { - "id": "princeton-nlp/Mistral-7B-Instruct-SimPO", - "name": "Mistral-7B-Instruct-SimPO", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4687, - "hfopenllm_v2/BBH": 0.4507, - "hfopenllm_v2/MATH Level 5": 0.0287, - "hfopenllm_v2/GPQA": 0.2785, - "hfopenllm_v2/MUSR": 0.4098, - "hfopenllm_v2/MMLU-PRO": 0.2797 - } - }, - { - "id": "princeton-nlp/Mistral-7B-Instruct-SLiC-HF", - "name": "Mistral-7B-Instruct-SLiC-HF", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5115, - "hfopenllm_v2/BBH": 0.404, - "hfopenllm_v2/MATH Level 5": 0.0174, - "hfopenllm_v2/GPQA": 0.2727, - "hfopenllm_v2/MUSR": 0.3913, - "hfopenllm_v2/MMLU-PRO": 0.2715 - } - }, - { - "id": "princeton-nlp/Sheared-LLaMA-1.3B", - "name": "Sheared-LLaMA-1.3B", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2198, - "hfopenllm_v2/BBH": 0.3197, - "hfopenllm_v2/MATH Level 5": 0.0128, - "hfopenllm_v2/GPQA": 0.2399, - "hfopenllm_v2/MUSR": 0.3713, - "hfopenllm_v2/MMLU-PRO": 0.1171 - } - }, - { - "id": "princeton-nlp/Sheared-LLaMA-2.7B", - "name": "Sheared-LLaMA-2.7B", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2417, - "hfopenllm_v2/BBH": 0.3259, - "hfopenllm_v2/MATH Level 5": 0.0128, - "hfopenllm_v2/GPQA": 0.2752, - "hfopenllm_v2/MUSR": 0.3567, - "hfopenllm_v2/MMLU-PRO": 0.1187 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/prithivmlmods.json b/data/developers/prithivmlmods.json deleted file mode 100644 index 88e743a4279f310dec935aed3968189a78be084e..0000000000000000000000000000000000000000 --- a/data/developers/prithivmlmods.json +++ /dev/null @@ -1,1545 +0,0 @@ -{ - "developer": "prithivMLmods", - "models": [ - { - "id": "prithivMLmods/Bellatrix-1.5B-xElite", - "name": "Bellatrix-1.5B-xElite", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1964, - "hfopenllm_v2/BBH": 0.3501, - "hfopenllm_v2/MATH Level 5": 0.287, - "hfopenllm_v2/GPQA": 0.2785, - "hfopenllm_v2/MUSR": 0.3619, - "hfopenllm_v2/MMLU-PRO": 0.1657 - } - }, - { - "id": "prithivMLmods/Bellatrix-Tiny-1.5B-R1", - "name": "Bellatrix-Tiny-1.5B-R1", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3352, - "hfopenllm_v2/BBH": 0.4022, - "hfopenllm_v2/MATH Level 5": 0.0604, - "hfopenllm_v2/GPQA": 0.2987, - "hfopenllm_v2/MUSR": 0.3683, - "hfopenllm_v2/MMLU-PRO": 0.2751 - } - }, - { - "id": "prithivMLmods/Bellatrix-Tiny-1B-v2", - "name": "Bellatrix-Tiny-1B-v2", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.151, - "hfopenllm_v2/BBH": 0.3268, - "hfopenllm_v2/MATH Level 5": 0.0287, - "hfopenllm_v2/GPQA": 0.2727, - "hfopenllm_v2/MUSR": 0.343, - "hfopenllm_v2/MMLU-PRO": 0.1493 - } - }, - { - "id": "prithivMLmods/Blaze-14B-xElite", - "name": "Blaze-14B-xElite", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0363, - "hfopenllm_v2/BBH": 0.6628, - "hfopenllm_v2/MATH Level 5": 0.3693, - "hfopenllm_v2/GPQA": 0.3943, - "hfopenllm_v2/MUSR": 0.4625, - "hfopenllm_v2/MMLU-PRO": 0.5111 - } - }, - { - "id": "prithivMLmods/Calcium-Opus-14B-Elite", - "name": "Calcium-Opus-14B-Elite", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6052, - "hfopenllm_v2/BBH": 0.6317, - "hfopenllm_v2/MATH Level 5": 0.4789, - "hfopenllm_v2/GPQA": 0.3742, - "hfopenllm_v2/MUSR": 0.486, - "hfopenllm_v2/MMLU-PRO": 0.5302 - } - }, - { - "id": "prithivMLmods/Calcium-Opus-14B-Elite-1M", - "name": "Calcium-Opus-14B-Elite-1M", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5613, - "hfopenllm_v2/BBH": 0.6329, - "hfopenllm_v2/MATH Level 5": 0.4456, - "hfopenllm_v2/GPQA": 0.3523, - "hfopenllm_v2/MUSR": 0.4676, - "hfopenllm_v2/MMLU-PRO": 0.5152 - } - }, - { - "id": "prithivMLmods/Calcium-Opus-14B-Elite-Stock", - "name": "Calcium-Opus-14B-Elite-Stock", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6143, - "hfopenllm_v2/BBH": 0.6329, - "hfopenllm_v2/MATH Level 5": 0.4668, - "hfopenllm_v2/GPQA": 0.3683, - "hfopenllm_v2/MUSR": 0.4808, - "hfopenllm_v2/MMLU-PRO": 0.5284 - } - }, - { - "id": "prithivMLmods/Calcium-Opus-14B-Elite2", - "name": "Calcium-Opus-14B-Elite2", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6176, - "hfopenllm_v2/BBH": 0.6318, - "hfopenllm_v2/MATH Level 5": 0.469, - "hfopenllm_v2/GPQA": 0.37, - "hfopenllm_v2/MUSR": 0.494, - "hfopenllm_v2/MMLU-PRO": 0.5301 - } - }, - { - "id": "prithivMLmods/Calcium-Opus-14B-Elite2-R1", - "name": "Calcium-Opus-14B-Elite2-R1", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6326, - "hfopenllm_v2/BBH": 0.6362, - "hfopenllm_v2/MATH Level 5": 0.3338, - "hfopenllm_v2/GPQA": 0.3909, - "hfopenllm_v2/MUSR": 0.49, - "hfopenllm_v2/MMLU-PRO": 0.5248 - } - }, - { - "id": "prithivMLmods/Calcium-Opus-14B-Elite3", - "name": "Calcium-Opus-14B-Elite3", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5428, - "hfopenllm_v2/BBH": 0.635, - "hfopenllm_v2/MATH Level 5": 0.4705, - "hfopenllm_v2/GPQA": 0.3708, - "hfopenllm_v2/MUSR": 0.4795, - "hfopenllm_v2/MMLU-PRO": 0.5335 - } - }, - { - "id": "prithivMLmods/Calcium-Opus-14B-Elite4", - "name": "Calcium-Opus-14B-Elite4", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6112, - "hfopenllm_v2/BBH": 0.6195, - "hfopenllm_v2/MATH Level 5": 0.3625, - "hfopenllm_v2/GPQA": 0.3557, - "hfopenllm_v2/MUSR": 0.4687, - "hfopenllm_v2/MMLU-PRO": 0.5149 - } - }, - { - "id": "prithivMLmods/Calcium-Opus-14B-Merge", - "name": "Calcium-Opus-14B-Merge", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4949, - "hfopenllm_v2/BBH": 0.6319, - "hfopenllm_v2/MATH Level 5": 0.4637, - "hfopenllm_v2/GPQA": 0.3708, - "hfopenllm_v2/MUSR": 0.4861, - "hfopenllm_v2/MMLU-PRO": 0.5356 - } - }, - { - "id": "prithivMLmods/Calcium-Opus-20B-v1", - "name": "Calcium-Opus-20B-v1", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3093, - "hfopenllm_v2/BBH": 0.599, - "hfopenllm_v2/MATH Level 5": 0.3618, - "hfopenllm_v2/GPQA": 0.3532, - "hfopenllm_v2/MUSR": 0.4943, - "hfopenllm_v2/MMLU-PRO": 0.4734 - } - }, - { - "id": "prithivMLmods/COCO-7B-Instruct-1M", - "name": "COCO-7B-Instruct-1M", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4743, - "hfopenllm_v2/BBH": 0.541, - "hfopenllm_v2/MATH Level 5": 0.3497, - "hfopenllm_v2/GPQA": 0.3079, - "hfopenllm_v2/MUSR": 0.4382, - "hfopenllm_v2/MMLU-PRO": 0.4186 - } - }, - { - "id": "prithivMLmods/Codepy-Deepthink-3B", - "name": "Codepy-Deepthink-3B", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4327, - "hfopenllm_v2/BBH": 0.4259, - "hfopenllm_v2/MATH Level 5": 0.1156, - "hfopenllm_v2/GPQA": 0.2794, - "hfopenllm_v2/MUSR": 0.331, - "hfopenllm_v2/MMLU-PRO": 0.309 - } - }, - { - "id": "prithivMLmods/Coma-II-14B", - "name": "Coma-II-14B", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4168, - "hfopenllm_v2/BBH": 0.6321, - "hfopenllm_v2/MATH Level 5": 0.5514, - "hfopenllm_v2/GPQA": 0.4002, - "hfopenllm_v2/MUSR": 0.5351, - "hfopenllm_v2/MMLU-PRO": 0.504 - } - }, - { - "id": "prithivMLmods/Condor-Opus-14B-Exp", - "name": "Condor-Opus-14B-Exp", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4043, - "hfopenllm_v2/BBH": 0.6154, - "hfopenllm_v2/MATH Level 5": 0.5227, - "hfopenllm_v2/GPQA": 0.3918, - "hfopenllm_v2/MUSR": 0.5194, - "hfopenllm_v2/MMLU-PRO": 0.5014 - } - }, - { - "id": "prithivMLmods/Cygnus-II-14B", - "name": "Cygnus-II-14B", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6184, - "hfopenllm_v2/BBH": 0.6661, - "hfopenllm_v2/MATH Level 5": 0.4396, - "hfopenllm_v2/GPQA": 0.3876, - "hfopenllm_v2/MUSR": 0.4688, - "hfopenllm_v2/MMLU-PRO": 0.5391 - } - }, - { - "id": "prithivMLmods/Deepthink-Llama-3-8B-Preview", - "name": "Deepthink-Llama-3-8B-Preview", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2955, - "hfopenllm_v2/BBH": 0.4665, - "hfopenllm_v2/MATH Level 5": 0.355, - "hfopenllm_v2/GPQA": 0.3163, - "hfopenllm_v2/MUSR": 0.3707, - "hfopenllm_v2/MMLU-PRO": 0.2739 - } - }, - { - "id": "prithivMLmods/Deepthink-Reasoning-14B", - "name": "Deepthink-Reasoning-14B", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5424, - "hfopenllm_v2/BBH": 0.6334, - "hfopenllm_v2/MATH Level 5": 0.423, - "hfopenllm_v2/GPQA": 0.3666, - "hfopenllm_v2/MUSR": 0.4732, - "hfopenllm_v2/MMLU-PRO": 0.5296 - } - }, - { - "id": "prithivMLmods/Deepthink-Reasoning-7B", - "name": "Deepthink-Reasoning-7B", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.484, - "hfopenllm_v2/BBH": 0.5505, - "hfopenllm_v2/MATH Level 5": 0.3346, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.4432, - "hfopenllm_v2/MMLU-PRO": 0.4349 - } - }, - { - "id": "prithivMLmods/Dinobot-Opus-14B-Exp", - "name": "Dinobot-Opus-14B-Exp", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.824, - "hfopenllm_v2/BBH": 0.637, - "hfopenllm_v2/MATH Level 5": 0.5317, - "hfopenllm_v2/GPQA": 0.3247, - "hfopenllm_v2/MUSR": 0.426, - "hfopenllm_v2/MMLU-PRO": 0.4979 - } - }, - { - "id": "prithivMLmods/Elita-0.1-Distilled-R1-abliterated", - "name": "Elita-0.1-Distilled-R1-abliterated", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3542, - "hfopenllm_v2/BBH": 0.3828, - "hfopenllm_v2/MATH Level 5": 0.3066, - "hfopenllm_v2/GPQA": 0.2659, - "hfopenllm_v2/MUSR": 0.366, - "hfopenllm_v2/MMLU-PRO": 0.2758 - } - }, - { - "id": "prithivMLmods/Elita-1", - "name": "Elita-1", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4906, - "hfopenllm_v2/BBH": 0.652, - "hfopenllm_v2/MATH Level 5": 0.3429, - "hfopenllm_v2/GPQA": 0.3758, - "hfopenllm_v2/MUSR": 0.4834, - "hfopenllm_v2/MMLU-PRO": 0.5381 - } - }, - { - "id": "prithivMLmods/Epimetheus-14B-Axo", - "name": "Epimetheus-14B-Axo", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5546, - "hfopenllm_v2/BBH": 0.6613, - "hfopenllm_v2/MATH Level 5": 0.4101, - "hfopenllm_v2/GPQA": 0.3926, - "hfopenllm_v2/MUSR": 0.482, - "hfopenllm_v2/MMLU-PRO": 0.5304 - } - }, - { - "id": "prithivMLmods/Equuleus-Opus-14B-Exp", - "name": "Equuleus-Opus-14B-Exp", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7001, - "hfopenllm_v2/BBH": 0.6434, - "hfopenllm_v2/MATH Level 5": 0.4585, - "hfopenllm_v2/GPQA": 0.3867, - "hfopenllm_v2/MUSR": 0.4952, - "hfopenllm_v2/MMLU-PRO": 0.5374 - } - }, - { - "id": "prithivMLmods/Eridanus-Opus-14B-r999", - "name": "Eridanus-Opus-14B-r999", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6386, - "hfopenllm_v2/BBH": 0.6584, - "hfopenllm_v2/MATH Level 5": 0.386, - "hfopenllm_v2/GPQA": 0.3943, - "hfopenllm_v2/MUSR": 0.4769, - "hfopenllm_v2/MMLU-PRO": 0.5362 - } - }, - { - "id": "prithivMLmods/Evac-Opus-14B-Exp", - "name": "Evac-Opus-14B-Exp", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5916, - "hfopenllm_v2/BBH": 0.6475, - "hfopenllm_v2/MATH Level 5": 0.4215, - "hfopenllm_v2/GPQA": 0.3884, - "hfopenllm_v2/MUSR": 0.4728, - "hfopenllm_v2/MMLU-PRO": 0.5317 - } - }, - { - "id": "prithivMLmods/FastThink-0.5B-Tiny", - "name": "FastThink-0.5B-Tiny", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.258, - "hfopenllm_v2/BBH": 0.3206, - "hfopenllm_v2/MATH Level 5": 0.0204, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.3566, - "hfopenllm_v2/MMLU-PRO": 0.1649 - } - }, - { - "id": "prithivMLmods/Gaea-Opus-14B-Exp", - "name": "Gaea-Opus-14B-Exp", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5956, - "hfopenllm_v2/BBH": 0.656, - "hfopenllm_v2/MATH Level 5": 0.4275, - "hfopenllm_v2/GPQA": 0.3909, - "hfopenllm_v2/MUSR": 0.4859, - "hfopenllm_v2/MMLU-PRO": 0.5401 - } - }, - { - "id": "prithivMLmods/Galactic-Qwen-14B-Exp1", - "name": "Galactic-Qwen-14B-Exp1", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5832, - "hfopenllm_v2/BBH": 0.6582, - "hfopenllm_v2/MATH Level 5": 0.4018, - "hfopenllm_v2/GPQA": 0.3935, - "hfopenllm_v2/MUSR": 0.4781, - "hfopenllm_v2/MMLU-PRO": 0.5396 - } - }, - { - "id": "prithivMLmods/Galactic-Qwen-14B-Exp2", - "name": "Galactic-Qwen-14B-Exp2", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.662, - "hfopenllm_v2/BBH": 0.7203, - "hfopenllm_v2/MATH Level 5": 0.3474, - "hfopenllm_v2/GPQA": 0.3993, - "hfopenllm_v2/MUSR": 0.5354, - "hfopenllm_v2/MMLU-PRO": 0.5691 - } - }, - { - "id": "prithivMLmods/Gauss-Opus-14B-R999", - "name": "Gauss-Opus-14B-R999", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3907, - "hfopenllm_v2/BBH": 0.6228, - "hfopenllm_v2/MATH Level 5": 0.5755, - "hfopenllm_v2/GPQA": 0.3918, - "hfopenllm_v2/MUSR": 0.5338, - "hfopenllm_v2/MMLU-PRO": 0.5007 - } - }, - { - "id": "prithivMLmods/GWQ-9B-Preview", - "name": "GWQ-9B-Preview", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5066, - "hfopenllm_v2/BBH": 0.5806, - "hfopenllm_v2/MATH Level 5": 0.2266, - "hfopenllm_v2/GPQA": 0.3398, - "hfopenllm_v2/MUSR": 0.4951, - "hfopenllm_v2/MMLU-PRO": 0.3984 - } - }, - { - "id": "prithivMLmods/GWQ-9B-Preview2", - "name": "GWQ-9B-Preview2", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5209, - "hfopenllm_v2/BBH": 0.5797, - "hfopenllm_v2/MATH Level 5": 0.2372, - "hfopenllm_v2/GPQA": 0.3263, - "hfopenllm_v2/MUSR": 0.486, - "hfopenllm_v2/MMLU-PRO": 0.3997 - } - }, - { - "id": "prithivMLmods/GWQ2b", - "name": "GWQ2b", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4115, - "hfopenllm_v2/BBH": 0.4143, - "hfopenllm_v2/MATH Level 5": 0.0627, - "hfopenllm_v2/GPQA": 0.2827, - "hfopenllm_v2/MUSR": 0.4311, - "hfopenllm_v2/MMLU-PRO": 0.2473 - } - }, - { - "id": "prithivMLmods/Jolt-v0.1", - "name": "Jolt-v0.1", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5092, - "hfopenllm_v2/BBH": 0.6521, - "hfopenllm_v2/MATH Level 5": 0.3565, - "hfopenllm_v2/GPQA": 0.38, - "hfopenllm_v2/MUSR": 0.4847, - "hfopenllm_v2/MMLU-PRO": 0.5386 - } - }, - { - "id": "prithivMLmods/Lacerta-Opus-14B-Elite8", - "name": "Lacerta-Opus-14B-Elite8", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6141, - "hfopenllm_v2/BBH": 0.6401, - "hfopenllm_v2/MATH Level 5": 0.3648, - "hfopenllm_v2/GPQA": 0.3784, - "hfopenllm_v2/MUSR": 0.4635, - "hfopenllm_v2/MMLU-PRO": 0.5322 - } - }, - { - "id": "prithivMLmods/Llama-3.1-5B-Instruct", - "name": "Llama-3.1-5B-Instruct", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1407, - "hfopenllm_v2/BBH": 0.3051, - "hfopenllm_v2/MATH Level 5": 0.0151, - "hfopenllm_v2/GPQA": 0.2643, - "hfopenllm_v2/MUSR": 0.354, - "hfopenllm_v2/MMLU-PRO": 0.1184 - } - }, - { - "id": "prithivMLmods/Llama-3.1-8B-Open-SFT", - "name": "Llama-3.1-8B-Open-SFT", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4123, - "hfopenllm_v2/BBH": 0.4968, - "hfopenllm_v2/MATH Level 5": 0.1216, - "hfopenllm_v2/GPQA": 0.3096, - "hfopenllm_v2/MUSR": 0.3904, - "hfopenllm_v2/MMLU-PRO": 0.3522 - } - }, - { - "id": "prithivMLmods/Llama-3.2-3B-Math-Oct", - "name": "Llama-3.2-3B-Math-Oct", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4585, - "hfopenllm_v2/BBH": 0.4372, - "hfopenllm_v2/MATH Level 5": 0.1156, - "hfopenllm_v2/GPQA": 0.2584, - "hfopenllm_v2/MUSR": 0.347, - "hfopenllm_v2/MMLU-PRO": 0.2911 - } - }, - { - "id": "prithivMLmods/Llama-3.2-6B-AlgoCode", - "name": "Llama-3.2-6B-AlgoCode", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2136, - "hfopenllm_v2/BBH": 0.3748, - "hfopenllm_v2/MATH Level 5": 0.0136, - "hfopenllm_v2/GPQA": 0.2869, - "hfopenllm_v2/MUSR": 0.4013, - "hfopenllm_v2/MMLU-PRO": 0.1798 - } - }, - { - "id": "prithivMLmods/Llama-8B-Distill-CoT", - "name": "Llama-8B-Distill-CoT", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3342, - "hfopenllm_v2/BBH": 0.4298, - "hfopenllm_v2/MATH Level 5": 0.4003, - "hfopenllm_v2/GPQA": 0.2894, - "hfopenllm_v2/MUSR": 0.372, - "hfopenllm_v2/MMLU-PRO": 0.2732 - } - }, - { - "id": "prithivMLmods/Llama-Deepsync-1B", - "name": "Llama-Deepsync-1B", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.357, - "hfopenllm_v2/BBH": 0.3386, - "hfopenllm_v2/MATH Level 5": 0.0438, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3565, - "hfopenllm_v2/MMLU-PRO": 0.1738 - } - }, - { - "id": "prithivMLmods/Llama-Deepsync-3B", - "name": "Llama-Deepsync-3B", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4302, - "hfopenllm_v2/BBH": 0.4292, - "hfopenllm_v2/MATH Level 5": 0.1178, - "hfopenllm_v2/GPQA": 0.2718, - "hfopenllm_v2/MUSR": 0.3324, - "hfopenllm_v2/MMLU-PRO": 0.3031 - } - }, - { - "id": "prithivMLmods/Llama-Express.1-Math", - "name": "Llama-Express.1-Math", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5084, - "hfopenllm_v2/BBH": 0.3364, - "hfopenllm_v2/MATH Level 5": 0.0559, - "hfopenllm_v2/GPQA": 0.2634, - "hfopenllm_v2/MUSR": 0.3143, - "hfopenllm_v2/MMLU-PRO": 0.161 - } - }, - { - "id": "prithivMLmods/LwQ-10B-Instruct", - "name": "LwQ-10B-Instruct", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3935, - "hfopenllm_v2/BBH": 0.5122, - "hfopenllm_v2/MATH Level 5": 0.04, - "hfopenllm_v2/GPQA": 0.3121, - "hfopenllm_v2/MUSR": 0.4544, - "hfopenllm_v2/MMLU-PRO": 0.3318 - } - }, - { - "id": "prithivMLmods/LwQ-Reasoner-10B", - "name": "LwQ-Reasoner-10B", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2941, - "hfopenllm_v2/BBH": 0.5866, - "hfopenllm_v2/MATH Level 5": 0.358, - "hfopenllm_v2/GPQA": 0.3465, - "hfopenllm_v2/MUSR": 0.4079, - "hfopenllm_v2/MMLU-PRO": 0.4147 - } - }, - { - "id": "prithivMLmods/Magellanic-Opus-14B-Exp", - "name": "Magellanic-Opus-14B-Exp", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6866, - "hfopenllm_v2/BBH": 0.6383, - "hfopenllm_v2/MATH Level 5": 0.3799, - "hfopenllm_v2/GPQA": 0.3742, - "hfopenllm_v2/MUSR": 0.4926, - "hfopenllm_v2/MMLU-PRO": 0.5273 - } - }, - { - "id": "prithivMLmods/Magellanic-Qwen-25B-R999", - "name": "Magellanic-Qwen-25B-R999", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1873, - "hfopenllm_v2/BBH": 0.2608, - "hfopenllm_v2/MATH Level 5": 0.0053, - "hfopenllm_v2/GPQA": 0.2508, - "hfopenllm_v2/MUSR": 0.3831, - "hfopenllm_v2/MMLU-PRO": 0.13 - } - }, - { - "id": "prithivMLmods/Megatron-Corpus-14B-Exp", - "name": "Megatron-Corpus-14B-Exp", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4983, - "hfopenllm_v2/BBH": 0.6355, - "hfopenllm_v2/MATH Level 5": 0.3429, - "hfopenllm_v2/GPQA": 0.3633, - "hfopenllm_v2/MUSR": 0.4767, - "hfopenllm_v2/MMLU-PRO": 0.526 - } - }, - { - "id": "prithivMLmods/Megatron-Corpus-14B-Exp.v2", - "name": "Megatron-Corpus-14B-Exp.v2", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.487, - "hfopenllm_v2/BBH": 0.6321, - "hfopenllm_v2/MATH Level 5": 0.2591, - "hfopenllm_v2/GPQA": 0.3423, - "hfopenllm_v2/MUSR": 0.449, - "hfopenllm_v2/MMLU-PRO": 0.481 - } - }, - { - "id": "prithivMLmods/Megatron-Opus-14B-2.0", - "name": "Megatron-Opus-14B-2.0", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6694, - "hfopenllm_v2/BBH": 0.6871, - "hfopenllm_v2/MATH Level 5": 0.2779, - "hfopenllm_v2/GPQA": 0.3591, - "hfopenllm_v2/MUSR": 0.414, - "hfopenllm_v2/MMLU-PRO": 0.517 - } - }, - { - "id": "prithivMLmods/Megatron-Opus-14B-2.1", - "name": "Megatron-Opus-14B-2.1", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0246, - "hfopenllm_v2/BBH": 0.6727, - "hfopenllm_v2/MATH Level 5": 0.2998, - "hfopenllm_v2/GPQA": 0.3834, - "hfopenllm_v2/MUSR": 0.4928, - "hfopenllm_v2/MMLU-PRO": 0.5174 - } - }, - { - "id": "prithivMLmods/Megatron-Opus-14B-Exp", - "name": "Megatron-Opus-14B-Exp", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4979, - "hfopenllm_v2/BBH": 0.6516, - "hfopenllm_v2/MATH Level 5": 0.3535, - "hfopenllm_v2/GPQA": 0.375, - "hfopenllm_v2/MUSR": 0.4887, - "hfopenllm_v2/MMLU-PRO": 0.5401 - } - }, - { - "id": "prithivMLmods/Megatron-Opus-14B-Stock", - "name": "Megatron-Opus-14B-Stock", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5174, - "hfopenllm_v2/BBH": 0.6412, - "hfopenllm_v2/MATH Level 5": 0.3346, - "hfopenllm_v2/GPQA": 0.375, - "hfopenllm_v2/MUSR": 0.482, - "hfopenllm_v2/MMLU-PRO": 0.5293 - } - }, - { - "id": "prithivMLmods/Megatron-Opus-7B-Exp", - "name": "Megatron-Opus-7B-Exp", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6017, - "hfopenllm_v2/BBH": 0.5367, - "hfopenllm_v2/MATH Level 5": 0.1971, - "hfopenllm_v2/GPQA": 0.3112, - "hfopenllm_v2/MUSR": 0.4186, - "hfopenllm_v2/MMLU-PRO": 0.39 - } - }, - { - "id": "prithivMLmods/Messier-Opus-14B-Elite7", - "name": "Messier-Opus-14B-Elite7", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7113, - "hfopenllm_v2/BBH": 0.6499, - "hfopenllm_v2/MATH Level 5": 0.4071, - "hfopenllm_v2/GPQA": 0.3909, - "hfopenllm_v2/MUSR": 0.4886, - "hfopenllm_v2/MMLU-PRO": 0.5404 - } - }, - { - "id": "prithivMLmods/Omni-Reasoner-Merged", - "name": "Omni-Reasoner-Merged", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4599, - "hfopenllm_v2/BBH": 0.5508, - "hfopenllm_v2/MATH Level 5": 0.3331, - "hfopenllm_v2/GPQA": 0.3037, - "hfopenllm_v2/MUSR": 0.4616, - "hfopenllm_v2/MMLU-PRO": 0.4364 - } - }, - { - "id": "prithivMLmods/Omni-Reasoner3-Merged", - "name": "Omni-Reasoner3-Merged", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4935, - "hfopenllm_v2/BBH": 0.4388, - "hfopenllm_v2/MATH Level 5": 0.1088, - "hfopenllm_v2/GPQA": 0.2643, - "hfopenllm_v2/MUSR": 0.3522, - "hfopenllm_v2/MMLU-PRO": 0.295 - } - }, - { - "id": "prithivMLmods/Pegasus-Opus-14B-Exp", - "name": "Pegasus-Opus-14B-Exp", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6982, - "hfopenllm_v2/BBH": 0.6548, - "hfopenllm_v2/MATH Level 5": 0.4086, - "hfopenllm_v2/GPQA": 0.3951, - "hfopenllm_v2/MUSR": 0.486, - "hfopenllm_v2/MMLU-PRO": 0.5412 - } - }, - { - "id": "prithivMLmods/Phi-4-Empathetic", - "name": "Phi-4-Empathetic", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0497, - "hfopenllm_v2/BBH": 0.6727, - "hfopenllm_v2/MATH Level 5": 0.2621, - "hfopenllm_v2/GPQA": 0.38, - "hfopenllm_v2/MUSR": 0.4991, - "hfopenllm_v2/MMLU-PRO": 0.5066 - } - }, - { - "id": "prithivMLmods/Phi-4-Math-IO", - "name": "Phi-4-Math-IO", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.059, - "hfopenllm_v2/BBH": 0.6668, - "hfopenllm_v2/MATH Level 5": 0.4577, - "hfopenllm_v2/GPQA": 0.3985, - "hfopenllm_v2/MUSR": 0.4873, - "hfopenllm_v2/MMLU-PRO": 0.5205 - } - }, - { - "id": "prithivMLmods/Phi-4-o1", - "name": "Phi-4-o1", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.029, - "hfopenllm_v2/BBH": 0.6689, - "hfopenllm_v2/MATH Level 5": 0.3995, - "hfopenllm_v2/GPQA": 0.3826, - "hfopenllm_v2/MUSR": 0.4978, - "hfopenllm_v2/MMLU-PRO": 0.5174 - } - }, - { - "id": "prithivMLmods/Phi-4-QwQ", - "name": "Phi-4-QwQ", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0559, - "hfopenllm_v2/BBH": 0.6696, - "hfopenllm_v2/MATH Level 5": 0.4577, - "hfopenllm_v2/GPQA": 0.3909, - "hfopenllm_v2/MUSR": 0.4651, - "hfopenllm_v2/MMLU-PRO": 0.5275 - } - }, - { - "id": "prithivMLmods/Phi-4-Super", - "name": "Phi-4-Super", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0481, - "hfopenllm_v2/BBH": 0.672, - "hfopenllm_v2/MATH Level 5": 0.3489, - "hfopenllm_v2/GPQA": 0.3943, - "hfopenllm_v2/MUSR": 0.5044, - "hfopenllm_v2/MMLU-PRO": 0.5266 - } - }, - { - "id": "prithivMLmods/Phi-4-Super-1", - "name": "Phi-4-Super-1", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0418, - "hfopenllm_v2/BBH": 0.6729, - "hfopenllm_v2/MATH Level 5": 0.352, - "hfopenllm_v2/GPQA": 0.3935, - "hfopenllm_v2/MUSR": 0.5017, - "hfopenllm_v2/MMLU-PRO": 0.5235 - } - }, - { - "id": "prithivMLmods/Phi-4-Super-o1", - "name": "Phi-4-Super-o1", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0418, - "hfopenllm_v2/BBH": 0.6729, - "hfopenllm_v2/MATH Level 5": 0.352, - "hfopenllm_v2/GPQA": 0.3935, - "hfopenllm_v2/MUSR": 0.5017, - "hfopenllm_v2/MMLU-PRO": 0.5235 - } - }, - { - "id": "prithivMLmods/Phi4-Super", - "name": "Phi4-Super", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0481, - "hfopenllm_v2/BBH": 0.672, - "hfopenllm_v2/MATH Level 5": 0.3489, - "hfopenllm_v2/GPQA": 0.3943, - "hfopenllm_v2/MUSR": 0.5044, - "hfopenllm_v2/MMLU-PRO": 0.5266 - } - }, - { - "id": "prithivMLmods/Porpoise-Opus-14B-Exp", - "name": "Porpoise-Opus-14B-Exp", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7098, - "hfopenllm_v2/BBH": 0.6519, - "hfopenllm_v2/MATH Level 5": 0.4041, - "hfopenllm_v2/GPQA": 0.3935, - "hfopenllm_v2/MUSR": 0.4926, - "hfopenllm_v2/MMLU-PRO": 0.5396 - } - }, - { - "id": "prithivMLmods/Primal-Opus-14B-Optimus-v1", - "name": "Primal-Opus-14B-Optimus-v1", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5013, - "hfopenllm_v2/BBH": 0.6419, - "hfopenllm_v2/MATH Level 5": 0.3384, - "hfopenllm_v2/GPQA": 0.3725, - "hfopenllm_v2/MUSR": 0.4847, - "hfopenllm_v2/MMLU-PRO": 0.5259 - } - }, - { - "id": "prithivMLmods/Primal-Opus-14B-Optimus-v2", - "name": "Primal-Opus-14B-Optimus-v2", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6404, - "hfopenllm_v2/BBH": 0.6544, - "hfopenllm_v2/MATH Level 5": 0.4207, - "hfopenllm_v2/GPQA": 0.3918, - "hfopenllm_v2/MUSR": 0.49, - "hfopenllm_v2/MMLU-PRO": 0.5422 - } - }, - { - "id": "prithivMLmods/Qwen-7B-Distill-Reasoner", - "name": "Qwen-7B-Distill-Reasoner", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3396, - "hfopenllm_v2/BBH": 0.4409, - "hfopenllm_v2/MATH Level 5": 0.395, - "hfopenllm_v2/GPQA": 0.3272, - "hfopenllm_v2/MUSR": 0.366, - "hfopenllm_v2/MMLU-PRO": 0.2818 - } - }, - { - "id": "prithivMLmods/Qwen2.5-1.5B-DeepSeek-R1-Instruct", - "name": "Qwen2.5-1.5B-DeepSeek-R1-Instruct", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1397, - "hfopenllm_v2/BBH": 0.2824, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.276, - "hfopenllm_v2/MUSR": 0.3724, - "hfopenllm_v2/MMLU-PRO": 0.1123 - } - }, - { - "id": "prithivMLmods/Qwen2.5-14B-DeepSeek-R1-1M", - "name": "Qwen2.5-14B-DeepSeek-R1-1M", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4193, - "hfopenllm_v2/BBH": 0.5935, - "hfopenllm_v2/MATH Level 5": 0.5128, - "hfopenllm_v2/GPQA": 0.3322, - "hfopenllm_v2/MUSR": 0.4606, - "hfopenllm_v2/MMLU-PRO": 0.4899 - } - }, - { - "id": "prithivMLmods/Qwen2.5-7B-DeepSeek-R1-1M", - "name": "Qwen2.5-7B-DeepSeek-R1-1M", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1861, - "hfopenllm_v2/BBH": 0.3126, - "hfopenllm_v2/MATH Level 5": 0.0151, - "hfopenllm_v2/GPQA": 0.2617, - "hfopenllm_v2/MUSR": 0.3417, - "hfopenllm_v2/MMLU-PRO": 0.1201 - } - }, - { - "id": "prithivMLmods/QwQ-LCoT-14B-Conversational", - "name": "QwQ-LCoT-14B-Conversational", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4047, - "hfopenllm_v2/BBH": 0.624, - "hfopenllm_v2/MATH Level 5": 0.4653, - "hfopenllm_v2/GPQA": 0.3498, - "hfopenllm_v2/MUSR": 0.4847, - "hfopenllm_v2/MMLU-PRO": 0.5278 - } - }, - { - "id": "prithivMLmods/QwQ-LCoT-3B-Instruct", - "name": "QwQ-LCoT-3B-Instruct", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4354, - "hfopenllm_v2/BBH": 0.4763, - "hfopenllm_v2/MATH Level 5": 0.2825, - "hfopenllm_v2/GPQA": 0.2819, - "hfopenllm_v2/MUSR": 0.4358, - "hfopenllm_v2/MMLU-PRO": 0.3582 - } - }, - { - "id": "prithivMLmods/QwQ-LCoT-7B-Instruct", - "name": "QwQ-LCoT-7B-Instruct", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4987, - "hfopenllm_v2/BBH": 0.5466, - "hfopenllm_v2/MATH Level 5": 0.3716, - "hfopenllm_v2/GPQA": 0.302, - "hfopenllm_v2/MUSR": 0.4802, - "hfopenllm_v2/MMLU-PRO": 0.4334 - } - }, - { - "id": "prithivMLmods/QwQ-LCoT1-Merged", - "name": "QwQ-LCoT1-Merged", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4751, - "hfopenllm_v2/BBH": 0.5481, - "hfopenllm_v2/MATH Level 5": 0.3731, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.4696, - "hfopenllm_v2/MMLU-PRO": 0.4358 - } - }, - { - "id": "prithivMLmods/QwQ-LCoT2-7B-Instruct", - "name": "QwQ-LCoT2-7B-Instruct", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5561, - "hfopenllm_v2/BBH": 0.5425, - "hfopenllm_v2/MATH Level 5": 0.327, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.4564, - "hfopenllm_v2/MMLU-PRO": 0.4342 - } - }, - { - "id": "prithivMLmods/QwQ-MathOct-7B", - "name": "QwQ-MathOct-7B", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4684, - "hfopenllm_v2/BBH": 0.5486, - "hfopenllm_v2/MATH Level 5": 0.2953, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.4601, - "hfopenllm_v2/MMLU-PRO": 0.433 - } - }, - { - "id": "prithivMLmods/QwQ-R1-Distill-1.5B-CoT", - "name": "QwQ-R1-Distill-1.5B-CoT", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2194, - "hfopenllm_v2/BBH": 0.3666, - "hfopenllm_v2/MATH Level 5": 0.3346, - "hfopenllm_v2/GPQA": 0.2861, - "hfopenllm_v2/MUSR": 0.3434, - "hfopenllm_v2/MMLU-PRO": 0.1913 - } - }, - { - "id": "prithivMLmods/QwQ-R1-Distill-7B-CoT", - "name": "QwQ-R1-Distill-7B-CoT", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.35, - "hfopenllm_v2/BBH": 0.4388, - "hfopenllm_v2/MATH Level 5": 0.4683, - "hfopenllm_v2/GPQA": 0.2936, - "hfopenllm_v2/MUSR": 0.3779, - "hfopenllm_v2/MMLU-PRO": 0.2804 - } - }, - { - "id": "prithivMLmods/SmolLM2-CoT-360M", - "name": "SmolLM2-CoT-360M", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2216, - "hfopenllm_v2/BBH": 0.3135, - "hfopenllm_v2/MATH Level 5": 0.0204, - "hfopenllm_v2/GPQA": 0.2366, - "hfopenllm_v2/MUSR": 0.3794, - "hfopenllm_v2/MMLU-PRO": 0.1085 - } - }, - { - "id": "prithivMLmods/Sombrero-Opus-14B-Elite5", - "name": "Sombrero-Opus-14B-Elite5", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7881, - "hfopenllm_v2/BBH": 0.6502, - "hfopenllm_v2/MATH Level 5": 0.5355, - "hfopenllm_v2/GPQA": 0.3364, - "hfopenllm_v2/MUSR": 0.4287, - "hfopenllm_v2/MMLU-PRO": 0.52 - } - }, - { - "id": "prithivMLmods/Sombrero-Opus-14B-Elite6", - "name": "Sombrero-Opus-14B-Elite6", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7226, - "hfopenllm_v2/BBH": 0.6488, - "hfopenllm_v2/MATH Level 5": 0.4079, - "hfopenllm_v2/GPQA": 0.3935, - "hfopenllm_v2/MUSR": 0.4886, - "hfopenllm_v2/MMLU-PRO": 0.539 - } - }, - { - "id": "prithivMLmods/Sombrero-Opus-14B-Sm1", - "name": "Sombrero-Opus-14B-Sm1", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3813, - "hfopenllm_v2/BBH": 0.6355, - "hfopenllm_v2/MATH Level 5": 0.5665, - "hfopenllm_v2/GPQA": 0.4035, - "hfopenllm_v2/MUSR": 0.5299, - "hfopenllm_v2/MMLU-PRO": 0.5125 - } - }, - { - "id": "prithivMLmods/Sombrero-Opus-14B-Sm2", - "name": "Sombrero-Opus-14B-Sm2", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4272, - "hfopenllm_v2/BBH": 0.6609, - "hfopenllm_v2/MATH Level 5": 0.4864, - "hfopenllm_v2/GPQA": 0.3884, - "hfopenllm_v2/MUSR": 0.5088, - "hfopenllm_v2/MMLU-PRO": 0.5345 - } - }, - { - "id": "prithivMLmods/Sombrero-Opus-14B-Sm4", - "name": "Sombrero-Opus-14B-Sm4", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4347, - "hfopenllm_v2/BBH": 0.6613, - "hfopenllm_v2/MATH Level 5": 0.4879, - "hfopenllm_v2/GPQA": 0.3951, - "hfopenllm_v2/MUSR": 0.5192, - "hfopenllm_v2/MMLU-PRO": 0.53 - } - }, - { - "id": "prithivMLmods/Sombrero-Opus-14B-Sm5", - "name": "Sombrero-Opus-14B-Sm5", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6852, - "hfopenllm_v2/BBH": 0.6564, - "hfopenllm_v2/MATH Level 5": 0.4094, - "hfopenllm_v2/GPQA": 0.3867, - "hfopenllm_v2/MUSR": 0.4806, - "hfopenllm_v2/MMLU-PRO": 0.54 - } - }, - { - "id": "prithivMLmods/Sqweeks-7B-Instruct", - "name": "Sqweeks-7B-Instruct", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2158, - "hfopenllm_v2/BBH": 0.4667, - "hfopenllm_v2/MATH Level 5": 0.5144, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.4476, - "hfopenllm_v2/MMLU-PRO": 0.3133 - } - }, - { - "id": "prithivMLmods/Tadpole-Opus-14B-Exp", - "name": "Tadpole-Opus-14B-Exp", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.575, - "hfopenllm_v2/BBH": 0.6369, - "hfopenllm_v2/MATH Level 5": 0.3134, - "hfopenllm_v2/GPQA": 0.3859, - "hfopenllm_v2/MUSR": 0.4728, - "hfopenllm_v2/MMLU-PRO": 0.5322 - } - }, - { - "id": "prithivMLmods/Taurus-Opus-7B", - "name": "Taurus-Opus-7B", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4223, - "hfopenllm_v2/BBH": 0.5367, - "hfopenllm_v2/MATH Level 5": 0.2168, - "hfopenllm_v2/GPQA": 0.3263, - "hfopenllm_v2/MUSR": 0.4399, - "hfopenllm_v2/MMLU-PRO": 0.3951 - } - }, - { - "id": "prithivMLmods/Triangulum-10B", - "name": "Triangulum-10B", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3229, - "hfopenllm_v2/BBH": 0.5968, - "hfopenllm_v2/MATH Level 5": 0.355, - "hfopenllm_v2/GPQA": 0.354, - "hfopenllm_v2/MUSR": 0.4172, - "hfopenllm_v2/MMLU-PRO": 0.4178 - } - }, - { - "id": "prithivMLmods/Triangulum-5B", - "name": "Triangulum-5B", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1283, - "hfopenllm_v2/BBH": 0.3124, - "hfopenllm_v2/MATH Level 5": 0.0106, - "hfopenllm_v2/GPQA": 0.255, - "hfopenllm_v2/MUSR": 0.3445, - "hfopenllm_v2/MMLU-PRO": 0.1223 - } - }, - { - "id": "prithivMLmods/Triangulum-v2-10B", - "name": "Triangulum-v2-10B", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6705, - "hfopenllm_v2/BBH": 0.6065, - "hfopenllm_v2/MATH Level 5": 0.2447, - "hfopenllm_v2/GPQA": 0.3372, - "hfopenllm_v2/MUSR": 0.4281, - "hfopenllm_v2/MMLU-PRO": 0.4466 - } - }, - { - "id": "prithivMLmods/Tucana-Opus-14B-r999", - "name": "Tucana-Opus-14B-r999", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6067, - "hfopenllm_v2/BBH": 0.6557, - "hfopenllm_v2/MATH Level 5": 0.4063, - "hfopenllm_v2/GPQA": 0.3918, - "hfopenllm_v2/MUSR": 0.473, - "hfopenllm_v2/MMLU-PRO": 0.5384 - } - }, - { - "id": "prithivMLmods/Tulu-MathLingo-8B", - "name": "Tulu-MathLingo-8B", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5589, - "hfopenllm_v2/BBH": 0.4659, - "hfopenllm_v2/MATH Level 5": 0.145, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.3864, - "hfopenllm_v2/MMLU-PRO": 0.3044 - } - }, - { - "id": "prithivMLmods/Viper-Coder-7B-Elite14", - "name": "Viper-Coder-7B-Elite14", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1488, - "hfopenllm_v2/BBH": 0.2829, - "hfopenllm_v2/MATH Level 5": 0.0106, - "hfopenllm_v2/GPQA": 0.255, - "hfopenllm_v2/MUSR": 0.3422, - "hfopenllm_v2/MMLU-PRO": 0.1089 - } - }, - { - "id": "prithivMLmods/Viper-Coder-Hybrid-v1.2", - "name": "Viper-Coder-Hybrid-v1.2", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6736, - "hfopenllm_v2/BBH": 0.6391, - "hfopenllm_v2/MATH Level 5": 0.3331, - "hfopenllm_v2/GPQA": 0.3742, - "hfopenllm_v2/MUSR": 0.4822, - "hfopenllm_v2/MMLU-PRO": 0.5243 - } - }, - { - "id": "prithivMLmods/Viper-Coder-Hybrid-v1.3", - "name": "Viper-Coder-Hybrid-v1.3", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7555, - "hfopenllm_v2/BBH": 0.6471, - "hfopenllm_v2/MATH Level 5": 0.4517, - "hfopenllm_v2/GPQA": 0.3381, - "hfopenllm_v2/MUSR": 0.4403, - "hfopenllm_v2/MMLU-PRO": 0.5097 - } - }, - { - "id": "prithivMLmods/Viper-Coder-HybridMini-v1.3", - "name": "Viper-Coder-HybridMini-v1.3", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6104, - "hfopenllm_v2/BBH": 0.5365, - "hfopenllm_v2/MATH Level 5": 0.463, - "hfopenllm_v2/GPQA": 0.3171, - "hfopenllm_v2/MUSR": 0.4505, - "hfopenllm_v2/MMLU-PRO": 0.4352 - } - }, - { - "id": "prithivMLmods/Viper-Coder-v0.1", - "name": "Viper-Coder-v0.1", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5521, - "hfopenllm_v2/BBH": 0.6143, - "hfopenllm_v2/MATH Level 5": 0.327, - "hfopenllm_v2/GPQA": 0.354, - "hfopenllm_v2/MUSR": 0.4394, - "hfopenllm_v2/MMLU-PRO": 0.3928 - } - }, - { - "id": "prithivMLmods/Viper-Coder-v1.1", - "name": "Viper-Coder-v1.1", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4432, - "hfopenllm_v2/BBH": 0.6492, - "hfopenllm_v2/MATH Level 5": 0.5461, - "hfopenllm_v2/GPQA": 0.401, - "hfopenllm_v2/MUSR": 0.5219, - "hfopenllm_v2/MMLU-PRO": 0.5232 - } - }, - { - "id": "prithivMLmods/Viper-Coder-v1.6-r999", - "name": "Viper-Coder-v1.6-r999", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4433, - "hfopenllm_v2/BBH": 0.6492, - "hfopenllm_v2/MATH Level 5": 0.5657, - "hfopenllm_v2/GPQA": 0.401, - "hfopenllm_v2/MUSR": 0.5219, - "hfopenllm_v2/MMLU-PRO": 0.5232 - } - }, - { - "id": "prithivMLmods/Viper-Coder-v1.7-Vsm6", - "name": "Viper-Coder-v1.7-Vsm6", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5004, - "hfopenllm_v2/BBH": 0.6502, - "hfopenllm_v2/MATH Level 5": 0.4645, - "hfopenllm_v2/GPQA": 0.3968, - "hfopenllm_v2/MUSR": 0.4768, - "hfopenllm_v2/MMLU-PRO": 0.5288 - } - }, - { - "id": "prithivMLmods/Viper-OneCoder-UIGEN", - "name": "Viper-OneCoder-UIGEN", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4692, - "hfopenllm_v2/BBH": 0.6047, - "hfopenllm_v2/MATH Level 5": 0.3867, - "hfopenllm_v2/GPQA": 0.3423, - "hfopenllm_v2/MUSR": 0.4514, - "hfopenllm_v2/MMLU-PRO": 0.3904 - } - }, - { - "id": "prithivMLmods/Volans-Opus-14B-Exp", - "name": "Volans-Opus-14B-Exp", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5868, - "hfopenllm_v2/BBH": 0.6521, - "hfopenllm_v2/MATH Level 5": 0.4252, - "hfopenllm_v2/GPQA": 0.3851, - "hfopenllm_v2/MUSR": 0.4872, - "hfopenllm_v2/MMLU-PRO": 0.5385 - } - }, - { - "id": "prithivMLmods/WebMind-7B-v0.1", - "name": "WebMind-7B-v0.1", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5278, - "hfopenllm_v2/BBH": 0.5434, - "hfopenllm_v2/MATH Level 5": 0.3648, - "hfopenllm_v2/GPQA": 0.3171, - "hfopenllm_v2/MUSR": 0.4537, - "hfopenllm_v2/MMLU-PRO": 0.4279 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/prometheus-eval.json b/data/developers/prometheus-eval.json deleted file mode 100644 index 308d3c0f9b08b56dc84d18e21f526fb6b5763312..0000000000000000000000000000000000000000 --- a/data/developers/prometheus-eval.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "developer": "prometheus-eval", - "models": [ - { - "id": "prometheus-eval/prometheus-7b-v2.0", - "name": "prometheus-eval/prometheus-7b-v2.0", - "developer": "prometheus-eval", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7204, - "reward-bench/Chat": 0.8547, - "reward-bench/Chat Hard": 0.4912, - "reward-bench/Safety": 0.7709, - "reward-bench/Reasoning": 0.7648 - } - }, - { - "id": "prometheus-eval/prometheus-8x7b-v2.0", - "name": "prometheus-eval/prometheus-8x7b-v2.0", - "developer": "prometheus-eval", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7451, - "reward-bench/Chat": 0.9302, - "reward-bench/Chat Hard": 0.4715, - "reward-bench/Safety": 0.8047, - "reward-bench/Reasoning": 0.774 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/pszemraj.json b/data/developers/pszemraj.json deleted file mode 100644 index c0e0a36a5aea3f99dc9da3c6220482c82197c2cf..0000000000000000000000000000000000000000 --- a/data/developers/pszemraj.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "pszemraj", - "models": [ - { - "id": "pszemraj/Llama-3-6.3b-v0.1", - "name": "Llama-3-6.3b-v0.1", - "developer": "pszemraj", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1044, - "hfopenllm_v2/BBH": 0.4197, - "hfopenllm_v2/MATH Level 5": 0.0211, - "hfopenllm_v2/GPQA": 0.2836, - "hfopenllm_v2/MUSR": 0.3908, - "hfopenllm_v2/MMLU-PRO": 0.284 - } - }, - { - "id": "pszemraj/Mistral-v0.3-6B", - "name": "Mistral-v0.3-6B", - "developer": "pszemraj", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2454, - "hfopenllm_v2/BBH": 0.3774, - "hfopenllm_v2/MATH Level 5": 0.0136, - "hfopenllm_v2/GPQA": 0.2651, - "hfopenllm_v2/MUSR": 0.3908, - "hfopenllm_v2/MMLU-PRO": 0.2143 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/puxai.json b/data/developers/puxai.json deleted file mode 100644 index bfa39503dcfc20f324f1e8e3114d60da6d1bbd13..0000000000000000000000000000000000000000 --- a/data/developers/puxai.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "PuxAI", - "models": [ - { - "id": "PuxAI/LUA_model", - "name": "LUA_model", - "developer": "PuxAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2282, - "hfopenllm_v2/BBH": 0.2877, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3484, - "hfopenllm_v2/MMLU-PRO": 0.1123 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/pygmalionai.json b/data/developers/pygmalionai.json deleted file mode 100644 index 6a5917e144555e5d99f516ca381f3ee4f0a8f434..0000000000000000000000000000000000000000 --- a/data/developers/pygmalionai.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "PygmalionAI", - "models": [ - { - "id": "PygmalionAI/pygmalion-6b", - "name": "pygmalion-6b", - "developer": "PygmalionAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2091, - "hfopenllm_v2/BBH": 0.3199, - "hfopenllm_v2/MATH Level 5": 0.0083, - "hfopenllm_v2/GPQA": 0.2492, - "hfopenllm_v2/MUSR": 0.3684, - "hfopenllm_v2/MMLU-PRO": 0.1184 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/q-bert.json b/data/developers/q-bert.json deleted file mode 100644 index e42745fdd53475572843e194019b98f3f63a3074..0000000000000000000000000000000000000000 --- a/data/developers/q-bert.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "Q-bert", - "models": [ - { - "id": "Q-bert/MetaMath-1B", - "name": "MetaMath-1B", - "developer": "Q-bert", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.53, - "hfopenllm_v2/BBH": 0.3451, - "hfopenllm_v2/MATH Level 5": 0.0627, - "hfopenllm_v2/GPQA": 0.2517, - "hfopenllm_v2/MUSR": 0.3289, - "hfopenllm_v2/MMLU-PRO": 0.1495 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/qingy2019.json b/data/developers/qingy2019.json deleted file mode 100644 index 3885f54f7f780ebe87cee5b8aacc1c1136f4441f..0000000000000000000000000000000000000000 --- a/data/developers/qingy2019.json +++ /dev/null @@ -1,103 +0,0 @@ -{ - "developer": "qingy2019", - "models": [ - { - "id": "qingy2019/LLaMa_3.2_3B_Catalysts", - "name": "LLaMa_3.2_3B_Catalysts", - "developer": "qingy2019", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4992, - "hfopenllm_v2/BBH": 0.4468, - "hfopenllm_v2/MATH Level 5": 0.1292, - "hfopenllm_v2/GPQA": 0.2886, - "hfopenllm_v2/MUSR": 0.3788, - "hfopenllm_v2/MMLU-PRO": 0.3008 - } - }, - { - "id": "qingy2019/OpenMath2-Llama3.1-8B", - "name": "OpenMath2-Llama3.1-8B", - "developer": "qingy2019", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2331, - "hfopenllm_v2/BBH": 0.4096, - "hfopenllm_v2/MATH Level 5": 0.2674, - "hfopenllm_v2/GPQA": 0.2651, - "hfopenllm_v2/MUSR": 0.3436, - "hfopenllm_v2/MMLU-PRO": 0.1553 - } - }, - { - "id": "qingy2019/Oracle-14B", - "name": "Oracle-14B", - "developer": "qingy2019", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2358, - "hfopenllm_v2/BBH": 0.4612, - "hfopenllm_v2/MATH Level 5": 0.0642, - "hfopenllm_v2/GPQA": 0.2576, - "hfopenllm_v2/MUSR": 0.3717, - "hfopenllm_v2/MMLU-PRO": 0.2382 - } - }, - { - "id": "qingy2019/Qwen2.5-Math-14B-Instruct", - "name": "Qwen2.5-Math-14B-Instruct", - "developer": "qingy2019", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6066, - "hfopenllm_v2/BBH": 0.635, - "hfopenllm_v2/MATH Level 5": 0.3716, - "hfopenllm_v2/GPQA": 0.3725, - "hfopenllm_v2/MUSR": 0.4757, - "hfopenllm_v2/MMLU-PRO": 0.5331 - } - }, - { - "id": "qingy2019/Qwen2.5-Math-14B-Instruct-Alpha", - "name": "Qwen2.5-Math-14B-Instruct-Alpha", - "developer": "qingy2019", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5981, - "hfopenllm_v2/BBH": 0.6375, - "hfopenllm_v2/MATH Level 5": 0.3142, - "hfopenllm_v2/GPQA": 0.37, - "hfopenllm_v2/MUSR": 0.4649, - "hfopenllm_v2/MMLU-PRO": 0.5331 - } - }, - { - "id": "qingy2019/Qwen2.5-Math-14B-Instruct-Pro", - "name": "Qwen2.5-Math-14B-Instruct-Pro", - "developer": "qingy2019", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1922, - "hfopenllm_v2/BBH": 0.5319, - "hfopenllm_v2/MATH Level 5": 0.284, - "hfopenllm_v2/GPQA": 0.3112, - "hfopenllm_v2/MUSR": 0.374, - "hfopenllm_v2/MMLU-PRO": 0.3558 - } - }, - { - "id": "qingy2019/Qwen2.5-Ultimate-14B-Instruct", - "name": "Qwen2.5-Ultimate-14B-Instruct", - "developer": "qingy2019", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3938, - "hfopenllm_v2/BBH": 0.5842, - "hfopenllm_v2/MATH Level 5": 0.2893, - "hfopenllm_v2/GPQA": 0.3565, - "hfopenllm_v2/MUSR": 0.4135, - "hfopenllm_v2/MMLU-PRO": 0.4929 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/qingy2024.json b/data/developers/qingy2024.json deleted file mode 100644 index 0eae4fa9b79bfe46f0cd5f1efadfe14c23cfeb3b..0000000000000000000000000000000000000000 --- a/data/developers/qingy2024.json +++ /dev/null @@ -1,243 +0,0 @@ -{ - "developer": "qingy2024", - "models": [ - { - "id": "qingy2024/Benchmaxx-Llama-3.2-1B-Instruct", - "name": "Benchmaxx-Llama-3.2-1B-Instruct", - "developer": "qingy2024", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2014, - "hfopenllm_v2/BBH": 0.8269, - "hfopenllm_v2/MATH Level 5": 0.4804, - "hfopenllm_v2/GPQA": 0.2836, - "hfopenllm_v2/MUSR": 0.3446, - "hfopenllm_v2/MMLU-PRO": 0.1113 - } - }, - { - "id": "qingy2024/Eyas-17B-Instruct", - "name": "Eyas-17B-Instruct", - "developer": "qingy2024", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6575, - "hfopenllm_v2/BBH": 0.6085, - "hfopenllm_v2/MATH Level 5": 0.247, - "hfopenllm_v2/GPQA": 0.3146, - "hfopenllm_v2/MUSR": 0.4522, - "hfopenllm_v2/MMLU-PRO": 0.4343 - } - }, - { - "id": "qingy2024/Falcon3-2x10B-MoE-Instruct", - "name": "Falcon3-2x10B-MoE-Instruct", - "developer": "qingy2024", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.785, - "hfopenllm_v2/BBH": 0.6185, - "hfopenllm_v2/MATH Level 5": 0.2795, - "hfopenllm_v2/GPQA": 0.3305, - "hfopenllm_v2/MUSR": 0.4284, - "hfopenllm_v2/MMLU-PRO": 0.4423 - } - }, - { - "id": "qingy2024/Fusion-14B-Instruct", - "name": "Fusion-14B-Instruct", - "developer": "qingy2024", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.726, - "hfopenllm_v2/BBH": 0.6396, - "hfopenllm_v2/MATH Level 5": 0.3369, - "hfopenllm_v2/GPQA": 0.3549, - "hfopenllm_v2/MUSR": 0.44, - "hfopenllm_v2/MMLU-PRO": 0.5044 - } - }, - { - "id": "qingy2024/Fusion2-14B-Instruct", - "name": "Fusion2-14B-Instruct", - "developer": "qingy2024", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6064, - "hfopenllm_v2/BBH": 0.6119, - "hfopenllm_v2/MATH Level 5": 0.3127, - "hfopenllm_v2/GPQA": 0.3448, - "hfopenllm_v2/MUSR": 0.4634, - "hfopenllm_v2/MMLU-PRO": 0.5051 - } - }, - { - "id": "qingy2024/Fusion4-14B-Instruct", - "name": "Fusion4-14B-Instruct", - "developer": "qingy2024", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7649, - "hfopenllm_v2/BBH": 0.6543, - "hfopenllm_v2/MATH Level 5": 0.3882, - "hfopenllm_v2/GPQA": 0.3305, - "hfopenllm_v2/MUSR": 0.4326, - "hfopenllm_v2/MMLU-PRO": 0.5194 - } - }, - { - "id": "qingy2024/OwO-14B-Instruct", - "name": "OwO-14B-Instruct", - "developer": "qingy2024", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1383, - "hfopenllm_v2/BBH": 0.6165, - "hfopenllm_v2/MATH Level 5": 0.4162, - "hfopenllm_v2/GPQA": 0.3641, - "hfopenllm_v2/MUSR": 0.4407, - "hfopenllm_v2/MMLU-PRO": 0.5181 - } - }, - { - "id": "qingy2024/Qwarkstar-4B", - "name": "Qwarkstar-4B", - "developer": "qingy2024", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1994, - "hfopenllm_v2/BBH": 0.4015, - "hfopenllm_v2/MATH Level 5": 0.0861, - "hfopenllm_v2/GPQA": 0.3247, - "hfopenllm_v2/MUSR": 0.4428, - "hfopenllm_v2/MMLU-PRO": 0.2425 - } - }, - { - "id": "qingy2024/Qwarkstar-4B-Instruct-Preview", - "name": "Qwarkstar-4B-Instruct-Preview", - "developer": "qingy2024", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5324, - "hfopenllm_v2/BBH": 0.4358, - "hfopenllm_v2/MATH Level 5": 0.1284, - "hfopenllm_v2/GPQA": 0.2802, - "hfopenllm_v2/MUSR": 0.3896, - "hfopenllm_v2/MMLU-PRO": 0.2502 - } - }, - { - "id": "qingy2024/Qwen2.5-4B", - "name": "Qwen2.5-4B", - "developer": "qingy2024", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2158, - "hfopenllm_v2/BBH": 0.4269, - "hfopenllm_v2/MATH Level 5": 0.0514, - "hfopenllm_v2/GPQA": 0.2911, - "hfopenllm_v2/MUSR": 0.461, - "hfopenllm_v2/MMLU-PRO": 0.2525 - } - }, - { - "id": "qingy2024/Qwen2.5-Coder-Draft-1.5B-Instruct", - "name": "Qwen2.5-Coder-Draft-1.5B-Instruct", - "developer": "qingy2024", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4125, - "hfopenllm_v2/BBH": 0.3837, - "hfopenllm_v2/MATH Level 5": 0.1579, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.358, - "hfopenllm_v2/MMLU-PRO": 0.2244 - } - }, - { - "id": "qingy2024/Qwen2.5-Math-14B-Instruct-Alpha", - "name": "Qwen2.5-Math-14B-Instruct-Alpha", - "developer": "qingy2024", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7704, - "hfopenllm_v2/BBH": 0.6465, - "hfopenllm_v2/MATH Level 5": 0.429, - "hfopenllm_v2/GPQA": 0.349, - "hfopenllm_v2/MUSR": 0.4021, - "hfopenllm_v2/MMLU-PRO": 0.4966 - } - }, - { - "id": "qingy2024/Qwen2.5-Math-14B-Instruct-Preview", - "name": "Qwen2.5-Math-14B-Instruct-Preview", - "developer": "qingy2024", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7826, - "hfopenllm_v2/BBH": 0.6294, - "hfopenllm_v2/MATH Level 5": 0.4758, - "hfopenllm_v2/GPQA": 0.3406, - "hfopenllm_v2/MUSR": 0.4115, - "hfopenllm_v2/MMLU-PRO": 0.4993 - } - }, - { - "id": "qingy2024/Qwen2.6-14B-Instruct", - "name": "Qwen2.6-14B-Instruct", - "developer": "qingy2024", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5811, - "hfopenllm_v2/BBH": 0.6394, - "hfopenllm_v2/MATH Level 5": 0.3051, - "hfopenllm_v2/GPQA": 0.3792, - "hfopenllm_v2/MUSR": 0.4569, - "hfopenllm_v2/MMLU-PRO": 0.5285 - } - }, - { - "id": "qingy2024/Qwen2.6-Math-14B-Instruct", - "name": "Qwen2.6-Math-14B-Instruct", - "developer": "qingy2024", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3862, - "hfopenllm_v2/BBH": 0.6324, - "hfopenllm_v2/MATH Level 5": 0.429, - "hfopenllm_v2/GPQA": 0.37, - "hfopenllm_v2/MUSR": 0.4759, - "hfopenllm_v2/MMLU-PRO": 0.5241 - } - }, - { - "id": "qingy2024/QwEnlarge-16B-Instruct", - "name": "QwEnlarge-16B-Instruct", - "developer": "qingy2024", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7802, - "hfopenllm_v2/BBH": 0.5949, - "hfopenllm_v2/MATH Level 5": 0.46, - "hfopenllm_v2/GPQA": 0.3331, - "hfopenllm_v2/MUSR": 0.4101, - "hfopenllm_v2/MMLU-PRO": 0.4476 - } - }, - { - "id": "qingy2024/QwQ-14B-Math-v0.2", - "name": "QwQ-14B-Math-v0.2", - "developer": "qingy2024", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3391, - "hfopenllm_v2/BBH": 0.5731, - "hfopenllm_v2/MATH Level 5": 0.4811, - "hfopenllm_v2/GPQA": 0.2626, - "hfopenllm_v2/MUSR": 0.4021, - "hfopenllm_v2/MMLU-PRO": 0.48 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/qq8933.json b/data/developers/qq8933.json deleted file mode 100644 index 31cf81dc475087d5768bc0201101502018108d86..0000000000000000000000000000000000000000 --- a/data/developers/qq8933.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "qq8933", - "models": [ - { - "id": "qq8933/OpenLongCoT-Base-Gemma2-2B", - "name": "OpenLongCoT-Base-Gemma2-2B", - "developer": "qq8933", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1965, - "hfopenllm_v2/BBH": 0.3106, - "hfopenllm_v2/MATH Level 5": 0.0234, - "hfopenllm_v2/GPQA": 0.2626, - "hfopenllm_v2/MUSR": 0.3222, - "hfopenllm_v2/MMLU-PRO": 0.1316 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/quazim0t0.json b/data/developers/quazim0t0.json deleted file mode 100644 index 496082d204a36ed57a388cd540f10080c702a420..0000000000000000000000000000000000000000 --- a/data/developers/quazim0t0.json +++ /dev/null @@ -1,971 +0,0 @@ -{ - "developer": "Quazim0t0", - "models": [ - { - "id": "Quazim0t0/1up-14b", - "name": "1up-14b", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6888, - "hfopenllm_v2/BBH": 0.6921, - "hfopenllm_v2/MATH Level 5": 0.4162, - "hfopenllm_v2/GPQA": 0.3624, - "hfopenllm_v2/MUSR": 0.4583, - "hfopenllm_v2/MMLU-PRO": 0.5406 - } - }, - { - "id": "Quazim0t0/Adamant-14B-sce", - "name": "Adamant-14B-sce", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6858, - "hfopenllm_v2/BBH": 0.6859, - "hfopenllm_v2/MATH Level 5": 0.3988, - "hfopenllm_v2/GPQA": 0.3507, - "hfopenllm_v2/MUSR": 0.4558, - "hfopenllm_v2/MMLU-PRO": 0.5372 - } - }, - { - "id": "Quazim0t0/Alice-14B", - "name": "Alice-14B", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6836, - "hfopenllm_v2/BBH": 0.6938, - "hfopenllm_v2/MATH Level 5": 0.4569, - "hfopenllm_v2/GPQA": 0.3515, - "hfopenllm_v2/MUSR": 0.4479, - "hfopenllm_v2/MMLU-PRO": 0.5419 - } - }, - { - "id": "Quazim0t0/Alien-CoT-14B-sce", - "name": "Alien-CoT-14B-sce", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0749, - "hfopenllm_v2/BBH": 0.6395, - "hfopenllm_v2/MATH Level 5": 0.5204, - "hfopenllm_v2/GPQA": 0.3918, - "hfopenllm_v2/MUSR": 0.4785, - "hfopenllm_v2/MMLU-PRO": 0.517 - } - }, - { - "id": "Quazim0t0/Aura-8B-Linear", - "name": "Aura-8B-Linear", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7948, - "hfopenllm_v2/BBH": 0.5074, - "hfopenllm_v2/MATH Level 5": 0.1805, - "hfopenllm_v2/GPQA": 0.2693, - "hfopenllm_v2/MUSR": 0.3687, - "hfopenllm_v2/MMLU-PRO": 0.3801 - } - }, - { - "id": "Quazim0t0/bloom-14b-stock", - "name": "bloom-14b-stock", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6575, - "hfopenllm_v2/BBH": 0.6878, - "hfopenllm_v2/MATH Level 5": 0.4811, - "hfopenllm_v2/GPQA": 0.3314, - "hfopenllm_v2/MUSR": 0.431, - "hfopenllm_v2/MMLU-PRO": 0.5373 - } - }, - { - "id": "Quazim0t0/caramel-14B", - "name": "caramel-14B", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6745, - "hfopenllm_v2/BBH": 0.6919, - "hfopenllm_v2/MATH Level 5": 0.4713, - "hfopenllm_v2/GPQA": 0.3448, - "hfopenllm_v2/MUSR": 0.4454, - "hfopenllm_v2/MMLU-PRO": 0.5436 - } - }, - { - "id": "Quazim0t0/Casa-14b-sce", - "name": "Casa-14b-sce", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6654, - "hfopenllm_v2/BBH": 0.6901, - "hfopenllm_v2/MATH Level 5": 0.4698, - "hfopenllm_v2/GPQA": 0.3331, - "hfopenllm_v2/MUSR": 0.431, - "hfopenllm_v2/MMLU-PRO": 0.5426 - } - }, - { - "id": "Quazim0t0/Charlie-8B-Linear", - "name": "Charlie-8B-Linear", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7381, - "hfopenllm_v2/BBH": 0.5141, - "hfopenllm_v2/MATH Level 5": 0.2651, - "hfopenllm_v2/GPQA": 0.271, - "hfopenllm_v2/MUSR": 0.3485, - "hfopenllm_v2/MMLU-PRO": 0.3573 - } - }, - { - "id": "Quazim0t0/Chromatic-8b-sce", - "name": "Chromatic-8b-sce", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5085, - "hfopenllm_v2/BBH": 0.5063, - "hfopenllm_v2/MATH Level 5": 0.1556, - "hfopenllm_v2/GPQA": 0.3196, - "hfopenllm_v2/MUSR": 0.4051, - "hfopenllm_v2/MMLU-PRO": 0.3755 - } - }, - { - "id": "Quazim0t0/CoT_Phi", - "name": "CoT_Phi", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6159, - "hfopenllm_v2/BBH": 0.6751, - "hfopenllm_v2/MATH Level 5": 0.3308, - "hfopenllm_v2/GPQA": 0.3582, - "hfopenllm_v2/MUSR": 0.4244, - "hfopenllm_v2/MMLU-PRO": 0.4901 - } - }, - { - "id": "Quazim0t0/Dyson-14b", - "name": "Dyson-14b", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5857, - "hfopenllm_v2/BBH": 0.6863, - "hfopenllm_v2/MATH Level 5": 0.5393, - "hfopenllm_v2/GPQA": 0.3138, - "hfopenllm_v2/MUSR": 0.4259, - "hfopenllm_v2/MMLU-PRO": 0.5399 - } - }, - { - "id": "Quazim0t0/Edu-14B-Linear", - "name": "Edu-14B-Linear", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6158, - "hfopenllm_v2/BBH": 0.6758, - "hfopenllm_v2/MATH Level 5": 0.2447, - "hfopenllm_v2/GPQA": 0.3171, - "hfopenllm_v2/MUSR": 0.4378, - "hfopenllm_v2/MMLU-PRO": 0.5086 - } - }, - { - "id": "Quazim0t0/Fugazi14b", - "name": "Fugazi14b", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6998, - "hfopenllm_v2/BBH": 0.6941, - "hfopenllm_v2/MATH Level 5": 0.4653, - "hfopenllm_v2/GPQA": 0.3515, - "hfopenllm_v2/MUSR": 0.4546, - "hfopenllm_v2/MMLU-PRO": 0.5417 - } - }, - { - "id": "Quazim0t0/Geedorah-14B", - "name": "Geedorah-14B", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6873, - "hfopenllm_v2/BBH": 0.6964, - "hfopenllm_v2/MATH Level 5": 0.4449, - "hfopenllm_v2/GPQA": 0.3473, - "hfopenllm_v2/MUSR": 0.4547, - "hfopenllm_v2/MMLU-PRO": 0.5421 - } - }, - { - "id": "Quazim0t0/GivingTree-8b-sce", - "name": "GivingTree-8b-sce", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5006, - "hfopenllm_v2/BBH": 0.504, - "hfopenllm_v2/MATH Level 5": 0.1526, - "hfopenllm_v2/GPQA": 0.3221, - "hfopenllm_v2/MUSR": 0.4051, - "hfopenllm_v2/MMLU-PRO": 0.3761 - } - }, - { - "id": "Quazim0t0/graphite-14b-sce", - "name": "graphite-14b-sce", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3217, - "hfopenllm_v2/BBH": 0.6631, - "hfopenllm_v2/MATH Level 5": 0.3006, - "hfopenllm_v2/GPQA": 0.2894, - "hfopenllm_v2/MUSR": 0.3981, - "hfopenllm_v2/MMLU-PRO": 0.528 - } - }, - { - "id": "Quazim0t0/GuiltySpark-14B-ties", - "name": "GuiltySpark-14B-ties", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6854, - "hfopenllm_v2/BBH": 0.6914, - "hfopenllm_v2/MATH Level 5": 0.3837, - "hfopenllm_v2/GPQA": 0.3649, - "hfopenllm_v2/MUSR": 0.4557, - "hfopenllm_v2/MMLU-PRO": 0.54 - } - }, - { - "id": "Quazim0t0/GZA-14B-sce", - "name": "GZA-14B-sce", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6274, - "hfopenllm_v2/BBH": 0.6687, - "hfopenllm_v2/MATH Level 5": 0.4721, - "hfopenllm_v2/GPQA": 0.302, - "hfopenllm_v2/MUSR": 0.4285, - "hfopenllm_v2/MMLU-PRO": 0.5232 - } - }, - { - "id": "Quazim0t0/Halo-14B-sce", - "name": "Halo-14B-sce", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6754, - "hfopenllm_v2/BBH": 0.6876, - "hfopenllm_v2/MATH Level 5": 0.429, - "hfopenllm_v2/GPQA": 0.3473, - "hfopenllm_v2/MUSR": 0.4401, - "hfopenllm_v2/MMLU-PRO": 0.5376 - } - }, - { - "id": "Quazim0t0/Heretic1.5b", - "name": "Heretic1.5b", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2062, - "hfopenllm_v2/BBH": 0.3529, - "hfopenllm_v2/MATH Level 5": 0.244, - "hfopenllm_v2/GPQA": 0.2685, - "hfopenllm_v2/MUSR": 0.3511, - "hfopenllm_v2/MMLU-PRO": 0.1728 - } - }, - { - "id": "Quazim0t0/Hyde-14b-sce", - "name": "Hyde-14b-sce", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6715, - "hfopenllm_v2/BBH": 0.6885, - "hfopenllm_v2/MATH Level 5": 0.2734, - "hfopenllm_v2/GPQA": 0.3414, - "hfopenllm_v2/MUSR": 0.4141, - "hfopenllm_v2/MMLU-PRO": 0.53 - } - }, - { - "id": "Quazim0t0/Imagine-v0.5-16bit", - "name": "Imagine-v0.5-16bit", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2759, - "hfopenllm_v2/BBH": 0.6769, - "hfopenllm_v2/MATH Level 5": 0.1397, - "hfopenllm_v2/GPQA": 0.3649, - "hfopenllm_v2/MUSR": 0.4349, - "hfopenllm_v2/MMLU-PRO": 0.5354 - } - }, - { - "id": "Quazim0t0/Imbue-14b", - "name": "Imbue-14b", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.52, - "hfopenllm_v2/BBH": 0.6845, - "hfopenllm_v2/MATH Level 5": 0.5317, - "hfopenllm_v2/GPQA": 0.3129, - "hfopenllm_v2/MUSR": 0.4167, - "hfopenllm_v2/MMLU-PRO": 0.5402 - } - }, - { - "id": "Quazim0t0/Insom", - "name": "Insom", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6818, - "hfopenllm_v2/BBH": 0.6881, - "hfopenllm_v2/MATH Level 5": 0.3852, - "hfopenllm_v2/GPQA": 0.3498, - "hfopenllm_v2/MUSR": 0.4311, - "hfopenllm_v2/MMLU-PRO": 0.5352 - } - }, - { - "id": "Quazim0t0/InspectorDeck-14B-sce", - "name": "InspectorDeck-14B-sce", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3241, - "hfopenllm_v2/BBH": 0.6668, - "hfopenllm_v2/MATH Level 5": 0.3165, - "hfopenllm_v2/GPQA": 0.297, - "hfopenllm_v2/MUSR": 0.3982, - "hfopenllm_v2/MMLU-PRO": 0.5261 - } - }, - { - "id": "Quazim0t0/Jekyl-8b-sce", - "name": "Jekyl-8b-sce", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4697, - "hfopenllm_v2/BBH": 0.4994, - "hfopenllm_v2/MATH Level 5": 0.1616, - "hfopenllm_v2/GPQA": 0.3381, - "hfopenllm_v2/MUSR": 0.4197, - "hfopenllm_v2/MMLU-PRO": 0.3686 - } - }, - { - "id": "Quazim0t0/Jigsaw-14B-Linear", - "name": "Jigsaw-14B-Linear", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.648, - "hfopenllm_v2/BBH": 0.6865, - "hfopenllm_v2/MATH Level 5": 0.2651, - "hfopenllm_v2/GPQA": 0.3406, - "hfopenllm_v2/MUSR": 0.4483, - "hfopenllm_v2/MMLU-PRO": 0.5234 - } - }, - { - "id": "Quazim0t0/Katana-8b-sce", - "name": "Katana-8b-sce", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5107, - "hfopenllm_v2/BBH": 0.5075, - "hfopenllm_v2/MATH Level 5": 0.1511, - "hfopenllm_v2/GPQA": 0.3247, - "hfopenllm_v2/MUSR": 0.4038, - "hfopenllm_v2/MMLU-PRO": 0.3771 - } - }, - { - "id": "Quazim0t0/Knot-CoT-14B-sce", - "name": "Knot-CoT-14B-sce", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4832, - "hfopenllm_v2/BBH": 0.6616, - "hfopenllm_v2/MATH Level 5": 0.3995, - "hfopenllm_v2/GPQA": 0.2936, - "hfopenllm_v2/MUSR": 0.414, - "hfopenllm_v2/MMLU-PRO": 0.5154 - } - }, - { - "id": "Quazim0t0/Lineage-14B", - "name": "Lineage-14B", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.707, - "hfopenllm_v2/BBH": 0.6934, - "hfopenllm_v2/MATH Level 5": 0.4245, - "hfopenllm_v2/GPQA": 0.3599, - "hfopenllm_v2/MUSR": 0.4597, - "hfopenllm_v2/MMLU-PRO": 0.5411 - } - }, - { - "id": "Quazim0t0/Lo-Phi-14b", - "name": "Lo-Phi-14b", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4941, - "hfopenllm_v2/BBH": 0.6852, - "hfopenllm_v2/MATH Level 5": 0.5196, - "hfopenllm_v2/GPQA": 0.328, - "hfopenllm_v2/MUSR": 0.4232, - "hfopenllm_v2/MMLU-PRO": 0.5369 - } - }, - { - "id": "Quazim0t0/Loke-14B-sce", - "name": "Loke-14B-sce", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6848, - "hfopenllm_v2/BBH": 0.6924, - "hfopenllm_v2/MATH Level 5": 0.3905, - "hfopenllm_v2/GPQA": 0.3649, - "hfopenllm_v2/MUSR": 0.4637, - "hfopenllm_v2/MMLU-PRO": 0.5401 - } - }, - { - "id": "Quazim0t0/Math_Phi4_Reason", - "name": "Math_Phi4_Reason", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.322, - "hfopenllm_v2/BBH": 0.624, - "hfopenllm_v2/MATH Level 5": 0.3278, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.4034, - "hfopenllm_v2/MMLU-PRO": 0.503 - } - }, - { - "id": "Quazim0t0/MFDOOM-14B", - "name": "MFDOOM-14B", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6736, - "hfopenllm_v2/BBH": 0.6916, - "hfopenllm_v2/MATH Level 5": 0.5264, - "hfopenllm_v2/GPQA": 0.323, - "hfopenllm_v2/MUSR": 0.4377, - "hfopenllm_v2/MMLU-PRO": 0.5426 - } - }, - { - "id": "Quazim0t0/MFGRIMM-14B", - "name": "MFGRIMM-14B", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6894, - "hfopenllm_v2/BBH": 0.6909, - "hfopenllm_v2/MATH Level 5": 0.506, - "hfopenllm_v2/GPQA": 0.3339, - "hfopenllm_v2/MUSR": 0.4361, - "hfopenllm_v2/MMLU-PRO": 0.5416 - } - }, - { - "id": "Quazim0t0/Mithril-14B-sce", - "name": "Mithril-14B-sce", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6958, - "hfopenllm_v2/BBH": 0.6926, - "hfopenllm_v2/MATH Level 5": 0.3822, - "hfopenllm_v2/GPQA": 0.3691, - "hfopenllm_v2/MUSR": 0.4611, - "hfopenllm_v2/MMLU-PRO": 0.5403 - } - }, - { - "id": "Quazim0t0/mocha-14B", - "name": "mocha-14B", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5893, - "hfopenllm_v2/BBH": 0.6895, - "hfopenllm_v2/MATH Level 5": 0.5264, - "hfopenllm_v2/GPQA": 0.3305, - "hfopenllm_v2/MUSR": 0.4272, - "hfopenllm_v2/MMLU-PRO": 0.5384 - } - }, - { - "id": "Quazim0t0/Mononoke-14B-sce", - "name": "Mononoke-14B-sce", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3502, - "hfopenllm_v2/BBH": 0.6744, - "hfopenllm_v2/MATH Level 5": 0.4698, - "hfopenllm_v2/GPQA": 0.323, - "hfopenllm_v2/MUSR": 0.4155, - "hfopenllm_v2/MMLU-PRO": 0.5298 - } - }, - { - "id": "Quazim0t0/mosaic-14b-sce", - "name": "mosaic-14b-sce", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6876, - "hfopenllm_v2/BBH": 0.6907, - "hfopenllm_v2/MATH Level 5": 0.4026, - "hfopenllm_v2/GPQA": 0.3624, - "hfopenllm_v2/MUSR": 0.4558, - "hfopenllm_v2/MMLU-PRO": 0.5396 - } - }, - { - "id": "Quazim0t0/Motion-8B-Linear", - "name": "Motion-8B-Linear", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7686, - "hfopenllm_v2/BBH": 0.5084, - "hfopenllm_v2/MATH Level 5": 0.1888, - "hfopenllm_v2/GPQA": 0.271, - "hfopenllm_v2/MUSR": 0.3606, - "hfopenllm_v2/MMLU-PRO": 0.3785 - } - }, - { - "id": "Quazim0t0/Mouse-9B", - "name": "Mouse-9B", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1325, - "hfopenllm_v2/BBH": 0.2979, - "hfopenllm_v2/MATH Level 5": 0.0053, - "hfopenllm_v2/GPQA": 0.2542, - "hfopenllm_v2/MUSR": 0.347, - "hfopenllm_v2/MMLU-PRO": 0.1139 - } - }, - { - "id": "Quazim0t0/Nova-14b-sce", - "name": "Nova-14b-sce", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7022, - "hfopenllm_v2/BBH": 0.6935, - "hfopenllm_v2/MATH Level 5": 0.4162, - "hfopenllm_v2/GPQA": 0.3633, - "hfopenllm_v2/MUSR": 0.4571, - "hfopenllm_v2/MMLU-PRO": 0.5413 - } - }, - { - "id": "Quazim0t0/NovaScotia-14b-stock", - "name": "NovaScotia-14b-stock", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6787, - "hfopenllm_v2/BBH": 0.6935, - "hfopenllm_v2/MATH Level 5": 0.463, - "hfopenllm_v2/GPQA": 0.349, - "hfopenllm_v2/MUSR": 0.4493, - "hfopenllm_v2/MMLU-PRO": 0.5409 - } - }, - { - "id": "Quazim0t0/Oasis-14B-ties", - "name": "Oasis-14B-ties", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6937, - "hfopenllm_v2/BBH": 0.6915, - "hfopenllm_v2/MATH Level 5": 0.3754, - "hfopenllm_v2/GPQA": 0.3649, - "hfopenllm_v2/MUSR": 0.4571, - "hfopenllm_v2/MMLU-PRO": 0.5405 - } - }, - { - "id": "Quazim0t0/ODB-14B-sce", - "name": "ODB-14B-sce", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2922, - "hfopenllm_v2/BBH": 0.6559, - "hfopenllm_v2/MATH Level 5": 0.2545, - "hfopenllm_v2/GPQA": 0.2659, - "hfopenllm_v2/MUSR": 0.3929, - "hfopenllm_v2/MMLU-PRO": 0.5207 - } - }, - { - "id": "Quazim0t0/Origami-14B-sce", - "name": "Origami-14B-sce", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3259, - "hfopenllm_v2/BBH": 0.662, - "hfopenllm_v2/MATH Level 5": 0.2915, - "hfopenllm_v2/GPQA": 0.2836, - "hfopenllm_v2/MUSR": 0.4035, - "hfopenllm_v2/MMLU-PRO": 0.5244 - } - }, - { - "id": "Quazim0t0/Phi4.Turn.R1Distill.16bit", - "name": "Phi4.Turn.R1Distill.16bit", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3126, - "hfopenllm_v2/BBH": 0.6563, - "hfopenllm_v2/MATH Level 5": 0.2311, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.3902, - "hfopenllm_v2/MMLU-PRO": 0.5257 - } - }, - { - "id": "Quazim0t0/Phi4.Turn.R1Distill_v1.5.1-Tensors", - "name": "Phi4.Turn.R1Distill_v1.5.1-Tensors", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2995, - "hfopenllm_v2/BBH": 0.6456, - "hfopenllm_v2/MATH Level 5": 0.219, - "hfopenllm_v2/GPQA": 0.2685, - "hfopenllm_v2/MUSR": 0.3929, - "hfopenllm_v2/MMLU-PRO": 0.5117 - } - }, - { - "id": "Quazim0t0/Phi4Basis-14B-sce", - "name": "Phi4Basis-14B-sce", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6502, - "hfopenllm_v2/BBH": 0.6909, - "hfopenllm_v2/MATH Level 5": 0.4789, - "hfopenllm_v2/GPQA": 0.3289, - "hfopenllm_v2/MUSR": 0.4338, - "hfopenllm_v2/MMLU-PRO": 0.539 - } - }, - { - "id": "Quazim0t0/Ponder-14B-linear", - "name": "Ponder-14B-linear", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6906, - "hfopenllm_v2/BBH": 0.6943, - "hfopenllm_v2/MATH Level 5": 0.4282, - "hfopenllm_v2/GPQA": 0.3582, - "hfopenllm_v2/MUSR": 0.4558, - "hfopenllm_v2/MMLU-PRO": 0.5408 - } - }, - { - "id": "Quazim0t0/Rosemary-14b", - "name": "Rosemary-14b", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6915, - "hfopenllm_v2/BBH": 0.6955, - "hfopenllm_v2/MATH Level 5": 0.4388, - "hfopenllm_v2/GPQA": 0.3565, - "hfopenllm_v2/MUSR": 0.4492, - "hfopenllm_v2/MMLU-PRO": 0.5396 - } - }, - { - "id": "Quazim0t0/Rune-14b", - "name": "Rune-14b", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7016, - "hfopenllm_v2/BBH": 0.6937, - "hfopenllm_v2/MATH Level 5": 0.4585, - "hfopenllm_v2/GPQA": 0.3515, - "hfopenllm_v2/MUSR": 0.4533, - "hfopenllm_v2/MMLU-PRO": 0.5411 - } - }, - { - "id": "Quazim0t0/RZA-14B-sce", - "name": "RZA-14B-sce", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4774, - "hfopenllm_v2/BBH": 0.6686, - "hfopenllm_v2/MATH Level 5": 0.5189, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.4113, - "hfopenllm_v2/MMLU-PRO": 0.5383 - } - }, - { - "id": "Quazim0t0/Sake-20b", - "name": "Sake-20b", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6693, - "hfopenllm_v2/BBH": 0.677, - "hfopenllm_v2/MATH Level 5": 0.4653, - "hfopenllm_v2/GPQA": 0.3188, - "hfopenllm_v2/MUSR": 0.4494, - "hfopenllm_v2/MMLU-PRO": 0.5391 - } - }, - { - "id": "Quazim0t0/Spok-14b-sce", - "name": "Spok-14b-sce", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6682, - "hfopenllm_v2/BBH": 0.6899, - "hfopenllm_v2/MATH Level 5": 0.2719, - "hfopenllm_v2/GPQA": 0.3456, - "hfopenllm_v2/MUSR": 0.4141, - "hfopenllm_v2/MMLU-PRO": 0.5298 - } - }, - { - "id": "Quazim0t0/Sumatra-20b", - "name": "Sumatra-20b", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6738, - "hfopenllm_v2/BBH": 0.6855, - "hfopenllm_v2/MATH Level 5": 0.3671, - "hfopenllm_v2/GPQA": 0.3263, - "hfopenllm_v2/MUSR": 0.456, - "hfopenllm_v2/MMLU-PRO": 0.5415 - } - }, - { - "id": "Quazim0t0/SuperNova14b", - "name": "SuperNova14b", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7076, - "hfopenllm_v2/BBH": 0.6937, - "hfopenllm_v2/MATH Level 5": 0.4396, - "hfopenllm_v2/GPQA": 0.3523, - "hfopenllm_v2/MUSR": 0.4545, - "hfopenllm_v2/MMLU-PRO": 0.5435 - } - }, - { - "id": "Quazim0t0/SZA-14B-sce", - "name": "SZA-14B-sce", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5659, - "hfopenllm_v2/BBH": 0.6889, - "hfopenllm_v2/MATH Level 5": 0.5242, - "hfopenllm_v2/GPQA": 0.3305, - "hfopenllm_v2/MUSR": 0.4339, - "hfopenllm_v2/MMLU-PRO": 0.5353 - } - }, - { - "id": "Quazim0t0/TB0-8B-sce", - "name": "TB0-8B-sce", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5107, - "hfopenllm_v2/BBH": 0.5075, - "hfopenllm_v2/MATH Level 5": 0.1511, - "hfopenllm_v2/GPQA": 0.3247, - "hfopenllm_v2/MUSR": 0.4038, - "hfopenllm_v2/MMLU-PRO": 0.3771 - } - }, - { - "id": "Quazim0t0/TBL-8B-sce", - "name": "TBL-8B-sce", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4581, - "hfopenllm_v2/BBH": 0.5008, - "hfopenllm_v2/MATH Level 5": 0.1533, - "hfopenllm_v2/GPQA": 0.3339, - "hfopenllm_v2/MUSR": 0.4236, - "hfopenllm_v2/MMLU-PRO": 0.3689 - } - }, - { - "id": "Quazim0t0/tesseract-14b-stock", - "name": "tesseract-14b-stock", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5848, - "hfopenllm_v2/BBH": 0.688, - "hfopenllm_v2/MATH Level 5": 0.5144, - "hfopenllm_v2/GPQA": 0.3272, - "hfopenllm_v2/MUSR": 0.4232, - "hfopenllm_v2/MMLU-PRO": 0.5389 - } - }, - { - "id": "Quazim0t0/ThinkPhi1.1-Tensors", - "name": "ThinkPhi1.1-Tensors", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3908, - "hfopenllm_v2/BBH": 0.6449, - "hfopenllm_v2/MATH Level 5": 0.182, - "hfopenllm_v2/GPQA": 0.2987, - "hfopenllm_v2/MUSR": 0.418, - "hfopenllm_v2/MMLU-PRO": 0.4908 - } - }, - { - "id": "Quazim0t0/time-14b-stock", - "name": "time-14b-stock", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6699, - "hfopenllm_v2/BBH": 0.6897, - "hfopenllm_v2/MATH Level 5": 0.5083, - "hfopenllm_v2/GPQA": 0.3347, - "hfopenllm_v2/MUSR": 0.4323, - "hfopenllm_v2/MMLU-PRO": 0.5419 - } - }, - { - "id": "Quazim0t0/Venti-20b", - "name": "Venti-20b", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6641, - "hfopenllm_v2/BBH": 0.6901, - "hfopenllm_v2/MATH Level 5": 0.3391, - "hfopenllm_v2/GPQA": 0.3322, - "hfopenllm_v2/MUSR": 0.448, - "hfopenllm_v2/MMLU-PRO": 0.5386 - } - }, - { - "id": "Quazim0t0/Venti-Blend-sce", - "name": "Venti-Blend-sce", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6879, - "hfopenllm_v2/BBH": 0.6843, - "hfopenllm_v2/MATH Level 5": 0.4056, - "hfopenllm_v2/GPQA": 0.3163, - "hfopenllm_v2/MUSR": 0.4389, - "hfopenllm_v2/MMLU-PRO": 0.5414 - } - }, - { - "id": "Quazim0t0/Vine-14b-sce", - "name": "Vine-14b-sce", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6733, - "hfopenllm_v2/BBH": 0.6891, - "hfopenllm_v2/MATH Level 5": 0.5008, - "hfopenllm_v2/GPQA": 0.3339, - "hfopenllm_v2/MUSR": 0.4323, - "hfopenllm_v2/MMLU-PRO": 0.5408 - } - }, - { - "id": "Quazim0t0/Wendy-14B", - "name": "Wendy-14B", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6772, - "hfopenllm_v2/BBH": 0.6958, - "hfopenllm_v2/MATH Level 5": 0.4834, - "hfopenllm_v2/GPQA": 0.3322, - "hfopenllm_v2/MUSR": 0.4428, - "hfopenllm_v2/MMLU-PRO": 0.5435 - } - }, - { - "id": "Quazim0t0/Wu-14b-sce", - "name": "Wu-14b-sce", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6718, - "hfopenllm_v2/BBH": 0.6885, - "hfopenllm_v2/MATH Level 5": 0.2613, - "hfopenllm_v2/GPQA": 0.3465, - "hfopenllm_v2/MUSR": 0.4114, - "hfopenllm_v2/MMLU-PRO": 0.5293 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/qwen.json b/data/developers/qwen.json deleted file mode 100644 index 13e8e69646f6fc076f9d2221ca49b53a5a260c91..0000000000000000000000000000000000000000 --- a/data/developers/qwen.json +++ /dev/null @@ -1,1900 +0,0 @@ -{ - "developer": "Qwen", - "models": [ - { - "id": "Qwen/Qwen1.5-0.5B", - "name": "Qwen1.5-0.5B", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1706, - "hfopenllm_v2/BBH": 0.3154, - "hfopenllm_v2/MATH Level 5": 0.0174, - "hfopenllm_v2/GPQA": 0.2542, - "hfopenllm_v2/MUSR": 0.3616, - "hfopenllm_v2/MMLU-PRO": 0.1307 - } - }, - { - "id": "Qwen/Qwen1.5-0.5B-Chat", - "name": "Qwen/Qwen1.5-0.5B-Chat", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1807, - "hfopenllm_v2/BBH": 0.3167, - "hfopenllm_v2/MATH Level 5": 0.0068, - "hfopenllm_v2/GPQA": 0.2693, - "hfopenllm_v2/MUSR": 0.3837, - "hfopenllm_v2/MMLU-PRO": 0.1213, - "reward-bench/Score": 0.5298, - "reward-bench/Chat": 0.3547, - "reward-bench/Chat Hard": 0.6294, - "reward-bench/Safety": 0.5703, - "reward-bench/Reasoning": 0.5984, - "reward-bench/Prior Sets (0.5 weight)": 0.4629 - } - }, - { - "id": "Qwen/Qwen1.5-1.8B", - "name": "Qwen1.5-1.8B", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2154, - "hfopenllm_v2/BBH": 0.3476, - "hfopenllm_v2/MATH Level 5": 0.0317, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.3605, - "hfopenllm_v2/MMLU-PRO": 0.1882 - } - }, - { - "id": "Qwen/Qwen1.5-1.8B-Chat", - "name": "Qwen/Qwen1.5-1.8B-Chat", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2019, - "hfopenllm_v2/BBH": 0.3256, - "hfopenllm_v2/MATH Level 5": 0.0196, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.426, - "hfopenllm_v2/MMLU-PRO": 0.1804, - "reward-bench/Score": 0.589, - "reward-bench/Chat": 0.5615, - "reward-bench/Chat Hard": 0.6031, - "reward-bench/Safety": 0.4838, - "reward-bench/Reasoning": 0.7793, - "reward-bench/Prior Sets (0.5 weight)": 0.4453 - } - }, - { - "id": "Qwen/Qwen1.5-110B", - "name": "Qwen1.5-110B", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3422, - "hfopenllm_v2/BBH": 0.61, - "hfopenllm_v2/MATH Level 5": 0.247, - "hfopenllm_v2/GPQA": 0.3523, - "hfopenllm_v2/MUSR": 0.4408, - "hfopenllm_v2/MMLU-PRO": 0.5361 - } - }, - { - "id": "qwen/qwen1.5-110b-chat", - "name": "Qwen1.5 Chat 110B", - "developer": "qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.55, - "helm_lite/NarrativeQA": 0.721, - "helm_lite/NaturalQuestions (closed-book)": 0.35, - "helm_lite/OpenbookQA": 0.922, - "helm_lite/MMLU": 0.704, - "helm_lite/MATH": 0.568, - "helm_lite/GSM8K": 0.815, - "helm_lite/LegalBench": 0.624, - "helm_lite/MedQA": 0.64, - "helm_lite/WMT 2014": 0.192, - "helm_mmlu/MMLU All Subjects": 0.768, - "helm_mmlu/Abstract Algebra": 0.57, - "helm_mmlu/Anatomy": 0.696, - "helm_mmlu/College Physics": 0.51, - "helm_mmlu/Computer Security": 0.82, - "helm_mmlu/Econometrics": 0.64, - "helm_mmlu/Global Facts": 0.51, - "helm_mmlu/Jurisprudence": 0.833, - "helm_mmlu/Philosophy": 0.823, - "helm_mmlu/Professional Psychology": 0.82, - "helm_mmlu/Us Foreign Policy": 0.87, - "helm_mmlu/Astronomy": 0.901, - "helm_mmlu/Business Ethics": 0.8, - "helm_mmlu/Clinical Knowledge": 0.766, - "helm_mmlu/Conceptual Physics": 0.838, - "helm_mmlu/Electrical Engineering": 0.752, - "helm_mmlu/Elementary Mathematics": 0.669, - "helm_mmlu/Formal Logic": 0.643, - "helm_mmlu/High School World History": 0.903, - "helm_mmlu/Human Sexuality": 0.855, - "helm_mmlu/International Law": 0.876, - "helm_mmlu/Logical Fallacies": 0.828, - "helm_mmlu/Machine Learning": 0.634, - "helm_mmlu/Management": 0.835, - "helm_mmlu/Marketing": 0.919, - "helm_mmlu/Medical Genetics": 0.85, - "helm_mmlu/Miscellaneous": 0.934, - "helm_mmlu/Moral Scenarios": 0.783, - "helm_mmlu/Nutrition": 0.804, - "helm_mmlu/Prehistory": 0.867, - "helm_mmlu/Public Relations": 0.773, - "helm_mmlu/Security Studies": 0.735, - "helm_mmlu/Sociology": 0.866, - "helm_mmlu/Virology": 0.542, - "helm_mmlu/World Religions": 0.871, - "helm_mmlu/Mean win rate": 0.875, - "hfopenllm_v2/IFEval": 0.5939, - "hfopenllm_v2/BBH": 0.6184, - "hfopenllm_v2/MATH Level 5": 0.2341, - "hfopenllm_v2/GPQA": 0.3414, - "hfopenllm_v2/MUSR": 0.4522, - "hfopenllm_v2/MMLU-PRO": 0.4825 - } - }, - { - "id": "qwen/qwen1.5-14b", - "name": "Qwen1.5 14B", - "developer": "qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.425, - "helm_lite/NarrativeQA": 0.711, - "helm_lite/NaturalQuestions (closed-book)": 0.3, - "helm_lite/OpenbookQA": 0.862, - "helm_lite/MMLU": 0.626, - "helm_lite/MATH": 0.686, - "helm_lite/GSM8K": 0.693, - "helm_lite/LegalBench": 0.593, - "helm_lite/MedQA": 0.515, - "helm_lite/WMT 2014": 0.178, - "helm_mmlu/MMLU All Subjects": 0.686, - "helm_mmlu/Abstract Algebra": 0.4, - "helm_mmlu/Anatomy": 0.637, - "helm_mmlu/College Physics": 0.48, - "helm_mmlu/Computer Security": 0.84, - "helm_mmlu/Econometrics": 0.561, - "helm_mmlu/Global Facts": 0.49, - "helm_mmlu/Jurisprudence": 0.769, - "helm_mmlu/Philosophy": 0.717, - "helm_mmlu/Professional Psychology": 0.699, - "helm_mmlu/Us Foreign Policy": 0.87, - "helm_mmlu/Astronomy": 0.724, - "helm_mmlu/Business Ethics": 0.75, - "helm_mmlu/Clinical Knowledge": 0.736, - "helm_mmlu/Conceptual Physics": 0.694, - "helm_mmlu/Electrical Engineering": 0.683, - "helm_mmlu/Elementary Mathematics": 0.603, - "helm_mmlu/Formal Logic": 0.492, - "helm_mmlu/High School World History": 0.84, - "helm_mmlu/Human Sexuality": 0.756, - "helm_mmlu/International Law": 0.826, - "helm_mmlu/Logical Fallacies": 0.736, - "helm_mmlu/Machine Learning": 0.509, - "helm_mmlu/Management": 0.816, - "helm_mmlu/Marketing": 0.893, - "helm_mmlu/Medical Genetics": 0.76, - "helm_mmlu/Miscellaneous": 0.835, - "helm_mmlu/Moral Scenarios": 0.368, - "helm_mmlu/Nutrition": 0.742, - "helm_mmlu/Prehistory": 0.71, - "helm_mmlu/Public Relations": 0.655, - "helm_mmlu/Security Studies": 0.8, - "helm_mmlu/Sociology": 0.841, - "helm_mmlu/Virology": 0.458, - "helm_mmlu/World Religions": 0.842, - "helm_mmlu/Mean win rate": 0.796, - "hfopenllm_v2/IFEval": 0.2905, - "hfopenllm_v2/BBH": 0.508, - "hfopenllm_v2/MATH Level 5": 0.2024, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.4186, - "hfopenllm_v2/MMLU-PRO": 0.3644 - } - }, - { - "id": "Qwen/Qwen1.5-14B-Chat", - "name": "Qwen/Qwen1.5-14B-Chat", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4768, - "hfopenllm_v2/BBH": 0.5229, - "hfopenllm_v2/MATH Level 5": 0.1526, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.44, - "hfopenllm_v2/MMLU-PRO": 0.3618, - "reward-bench/Score": 0.6864, - "reward-bench/Chat": 0.5726, - "reward-bench/Chat Hard": 0.7018, - "reward-bench/Safety": 0.7122, - "reward-bench/Reasoning": 0.8961, - "reward-bench/Prior Sets (0.5 weight)": 0.4123 - } - }, - { - "id": "qwen/qwen1.5-32b", - "name": "Qwen1.5 32B", - "developer": "qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.546, - "helm_lite/NarrativeQA": 0.589, - "helm_lite/NaturalQuestions (closed-book)": 0.353, - "helm_lite/OpenbookQA": 0.932, - "helm_lite/MMLU": 0.628, - "helm_lite/MATH": 0.733, - "helm_lite/GSM8K": 0.773, - "helm_lite/LegalBench": 0.636, - "helm_lite/MedQA": 0.656, - "helm_lite/WMT 2014": 0.193, - "helm_mmlu/MMLU All Subjects": 0.744, - "helm_mmlu/Abstract Algebra": 0.4, - "helm_mmlu/Anatomy": 0.644, - "helm_mmlu/College Physics": 0.51, - "helm_mmlu/Computer Security": 0.77, - "helm_mmlu/Econometrics": 0.561, - "helm_mmlu/Global Facts": 0.47, - "helm_mmlu/Jurisprudence": 0.843, - "helm_mmlu/Philosophy": 0.826, - "helm_mmlu/Professional Psychology": 0.75, - "helm_mmlu/Us Foreign Policy": 0.91, - "helm_mmlu/Astronomy": 0.855, - "helm_mmlu/Business Ethics": 0.77, - "helm_mmlu/Clinical Knowledge": 0.781, - "helm_mmlu/Conceptual Physics": 0.766, - "helm_mmlu/Electrical Engineering": 0.731, - "helm_mmlu/Elementary Mathematics": 0.685, - "helm_mmlu/Formal Logic": 0.524, - "helm_mmlu/High School World History": 0.869, - "helm_mmlu/Human Sexuality": 0.847, - "helm_mmlu/International Law": 0.884, - "helm_mmlu/Logical Fallacies": 0.822, - "helm_mmlu/Machine Learning": 0.616, - "helm_mmlu/Management": 0.874, - "helm_mmlu/Marketing": 0.936, - "helm_mmlu/Medical Genetics": 0.85, - "helm_mmlu/Miscellaneous": 0.884, - "helm_mmlu/Moral Scenarios": 0.545, - "helm_mmlu/Nutrition": 0.81, - "helm_mmlu/Prehistory": 0.83, - "helm_mmlu/Public Relations": 0.664, - "helm_mmlu/Security Studies": 0.829, - "helm_mmlu/Sociology": 0.881, - "helm_mmlu/Virology": 0.578, - "helm_mmlu/World Religions": 0.854, - "helm_mmlu/Mean win rate": 0.624, - "hfopenllm_v2/IFEval": 0.3297, - "hfopenllm_v2/BBH": 0.5715, - "hfopenllm_v2/MATH Level 5": 0.3029, - "hfopenllm_v2/GPQA": 0.3297, - "hfopenllm_v2/MUSR": 0.4278, - "hfopenllm_v2/MMLU-PRO": 0.45 - } - }, - { - "id": "Qwen/Qwen1.5-32B-Chat", - "name": "Qwen1.5-32B-Chat", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5532, - "hfopenllm_v2/BBH": 0.6067, - "hfopenllm_v2/MATH Level 5": 0.1956, - "hfopenllm_v2/GPQA": 0.3062, - "hfopenllm_v2/MUSR": 0.416, - "hfopenllm_v2/MMLU-PRO": 0.4457 - } - }, - { - "id": "Qwen/Qwen1.5-4B", - "name": "Qwen1.5-4B", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2445, - "hfopenllm_v2/BBH": 0.4054, - "hfopenllm_v2/MATH Level 5": 0.0529, - "hfopenllm_v2/GPQA": 0.2768, - "hfopenllm_v2/MUSR": 0.3604, - "hfopenllm_v2/MMLU-PRO": 0.246 - } - }, - { - "id": "Qwen/Qwen1.5-4B-Chat", - "name": "Qwen/Qwen1.5-4B-Chat", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3157, - "hfopenllm_v2/BBH": 0.4006, - "hfopenllm_v2/MATH Level 5": 0.0279, - "hfopenllm_v2/GPQA": 0.2668, - "hfopenllm_v2/MUSR": 0.3978, - "hfopenllm_v2/MMLU-PRO": 0.2396, - "reward-bench/Score": 0.5477, - "reward-bench/Chat": 0.3883, - "reward-bench/Chat Hard": 0.6272, - "reward-bench/Safety": 0.5568, - "reward-bench/Reasoning": 0.6689, - "reward-bench/Prior Sets (0.5 weight)": 0.447 - } - }, - { - "id": "qwen/qwen1.5-72b", - "name": "Qwen1.5 72B", - "developer": "qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.608, - "helm_lite/NarrativeQA": 0.601, - "helm_lite/NaturalQuestions (closed-book)": 0.417, - "helm_lite/OpenbookQA": 0.93, - "helm_lite/MMLU": 0.647, - "helm_lite/MATH": 0.683, - "helm_lite/GSM8K": 0.799, - "helm_lite/LegalBench": 0.694, - "helm_lite/MedQA": 0.67, - "helm_lite/WMT 2014": 0.201, - "helm_mmlu/MMLU All Subjects": 0.774, - "helm_mmlu/Abstract Algebra": 0.44, - "helm_mmlu/Anatomy": 0.733, - "helm_mmlu/College Physics": 0.559, - "helm_mmlu/Computer Security": 0.81, - "helm_mmlu/Econometrics": 0.544, - "helm_mmlu/Global Facts": 0.56, - "helm_mmlu/Jurisprudence": 0.824, - "helm_mmlu/Philosophy": 0.83, - "helm_mmlu/Professional Psychology": 0.809, - "helm_mmlu/Us Foreign Policy": 0.94, - "helm_mmlu/Astronomy": 0.868, - "helm_mmlu/Business Ethics": 0.79, - "helm_mmlu/Clinical Knowledge": 0.834, - "helm_mmlu/Conceptual Physics": 0.821, - "helm_mmlu/Electrical Engineering": 0.779, - "helm_mmlu/Elementary Mathematics": 0.696, - "helm_mmlu/Formal Logic": 0.556, - "helm_mmlu/High School World History": 0.899, - "helm_mmlu/Human Sexuality": 0.878, - "helm_mmlu/International Law": 0.909, - "helm_mmlu/Logical Fallacies": 0.853, - "helm_mmlu/Machine Learning": 0.67, - "helm_mmlu/Management": 0.854, - "helm_mmlu/Marketing": 0.949, - "helm_mmlu/Medical Genetics": 0.87, - "helm_mmlu/Miscellaneous": 0.921, - "helm_mmlu/Moral Scenarios": 0.669, - "helm_mmlu/Nutrition": 0.859, - "helm_mmlu/Prehistory": 0.88, - "helm_mmlu/Public Relations": 0.755, - "helm_mmlu/Security Studies": 0.824, - "helm_mmlu/Sociology": 0.9, - "helm_mmlu/Virology": 0.584, - "helm_mmlu/World Religions": 0.883, - "helm_mmlu/Mean win rate": 0.65 - } - }, - { - "id": "Qwen/Qwen1.5-72B-Chat", - "name": "Qwen/Qwen1.5-72B-Chat", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6723, - "reward-bench/Chat": 0.6229, - "reward-bench/Chat Hard": 0.6601, - "reward-bench/Safety": 0.6757, - "reward-bench/Reasoning": 0.8554, - "reward-bench/Prior Sets (0.5 weight)": 0.4226 - } - }, - { - "id": "qwen/qwen1.5-7b", - "name": "Qwen1.5 7B", - "developer": "qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.275, - "helm_lite/NarrativeQA": 0.448, - "helm_lite/NaturalQuestions (closed-book)": 0.27, - "helm_lite/OpenbookQA": 0.806, - "helm_lite/MMLU": 0.569, - "helm_lite/MATH": 0.561, - "helm_lite/GSM8K": 0.6, - "helm_lite/LegalBench": 0.523, - "helm_lite/MedQA": 0.479, - "helm_lite/WMT 2014": 0.153, - "helm_mmlu/MMLU All Subjects": 0.626, - "helm_mmlu/Abstract Algebra": 0.39, - "helm_mmlu/Anatomy": 0.526, - "helm_mmlu/College Physics": 0.471, - "helm_mmlu/Computer Security": 0.76, - "helm_mmlu/Econometrics": 0.447, - "helm_mmlu/Global Facts": 0.4, - "helm_mmlu/Jurisprudence": 0.778, - "helm_mmlu/Philosophy": 0.691, - "helm_mmlu/Professional Psychology": 0.603, - "helm_mmlu/Us Foreign Policy": 0.84, - "helm_mmlu/Astronomy": 0.671, - "helm_mmlu/Business Ethics": 0.69, - "helm_mmlu/Clinical Knowledge": 0.691, - "helm_mmlu/Conceptual Physics": 0.579, - "helm_mmlu/Electrical Engineering": 0.572, - "helm_mmlu/Elementary Mathematics": 0.5, - "helm_mmlu/Formal Logic": 0.397, - "helm_mmlu/High School World History": 0.789, - "helm_mmlu/Human Sexuality": 0.695, - "helm_mmlu/International Law": 0.76, - "helm_mmlu/Logical Fallacies": 0.706, - "helm_mmlu/Machine Learning": 0.411, - "helm_mmlu/Management": 0.816, - "helm_mmlu/Marketing": 0.863, - "helm_mmlu/Medical Genetics": 0.69, - "helm_mmlu/Miscellaneous": 0.765, - "helm_mmlu/Moral Scenarios": 0.372, - "helm_mmlu/Nutrition": 0.696, - "helm_mmlu/Prehistory": 0.688, - "helm_mmlu/Public Relations": 0.627, - "helm_mmlu/Security Studies": 0.727, - "helm_mmlu/Sociology": 0.836, - "helm_mmlu/Virology": 0.488, - "helm_mmlu/World Religions": 0.778, - "helm_mmlu/Mean win rate": 0.843, - "hfopenllm_v2/IFEval": 0.2684, - "hfopenllm_v2/BBH": 0.456, - "hfopenllm_v2/MATH Level 5": 0.0929, - "hfopenllm_v2/GPQA": 0.2987, - "hfopenllm_v2/MUSR": 0.4103, - "hfopenllm_v2/MMLU-PRO": 0.2916 - } - }, - { - "id": "Qwen/Qwen1.5-7B-Chat", - "name": "Qwen/Qwen1.5-7B-Chat", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4371, - "hfopenllm_v2/BBH": 0.451, - "hfopenllm_v2/MATH Level 5": 0.0627, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.3779, - "hfopenllm_v2/MMLU-PRO": 0.2951, - "reward-bench/Score": 0.675, - "reward-bench/Chat": 0.5363, - "reward-bench/Chat Hard": 0.6908, - "reward-bench/Safety": 0.6919, - "reward-bench/Reasoning": 0.9041, - "reward-bench/Prior Sets (0.5 weight)": 0.4288 - } - }, - { - "id": "Qwen/Qwen1.5-MoE-A2.7B", - "name": "Qwen1.5-MoE-A2.7B", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.266, - "hfopenllm_v2/BBH": 0.4114, - "hfopenllm_v2/MATH Level 5": 0.0929, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.4013, - "hfopenllm_v2/MMLU-PRO": 0.2778 - } - }, - { - "id": "Qwen/Qwen1.5-MoE-A2.7B-Chat", - "name": "Qwen/Qwen1.5-MoE-A2.7B-Chat", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3795, - "hfopenllm_v2/BBH": 0.4272, - "hfopenllm_v2/MATH Level 5": 0.0634, - "hfopenllm_v2/GPQA": 0.2743, - "hfopenllm_v2/MUSR": 0.3899, - "hfopenllm_v2/MMLU-PRO": 0.2923, - "reward-bench/Score": 0.6644, - "reward-bench/Chat": 0.7291, - "reward-bench/Chat Hard": 0.6316, - "reward-bench/Safety": 0.6284, - "reward-bench/Reasoning": 0.774, - "reward-bench/Prior Sets (0.5 weight)": 0.4536 - } - }, - { - "id": "Qwen/Qwen2-0.5B", - "name": "Qwen2-0.5B", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1873, - "hfopenllm_v2/BBH": 0.3239, - "hfopenllm_v2/MATH Level 5": 0.0264, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.3752, - "hfopenllm_v2/MMLU-PRO": 0.172 - } - }, - { - "id": "Qwen/Qwen2-0.5B-Instruct", - "name": "Qwen2-0.5B-Instruct", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2247, - "hfopenllm_v2/BBH": 0.3173, - "hfopenllm_v2/MATH Level 5": 0.0287, - "hfopenllm_v2/GPQA": 0.2466, - "hfopenllm_v2/MUSR": 0.3353, - "hfopenllm_v2/MMLU-PRO": 0.1531 - } - }, - { - "id": "Qwen/Qwen2-1.5B", - "name": "Qwen2-1.5B", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2113, - "hfopenllm_v2/BBH": 0.3575, - "hfopenllm_v2/MATH Level 5": 0.0702, - "hfopenllm_v2/GPQA": 0.2643, - "hfopenllm_v2/MUSR": 0.3658, - "hfopenllm_v2/MMLU-PRO": 0.2552 - } - }, - { - "id": "Qwen/Qwen2-1.5B-Instruct", - "name": "Qwen2-1.5B-Instruct", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3371, - "hfopenllm_v2/BBH": 0.3852, - "hfopenllm_v2/MATH Level 5": 0.0718, - "hfopenllm_v2/GPQA": 0.2617, - "hfopenllm_v2/MUSR": 0.4293, - "hfopenllm_v2/MMLU-PRO": 0.2501 - } - }, - { - "id": "Qwen/Qwen2-57B-A14B", - "name": "Qwen2-57B-A14B", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3113, - "hfopenllm_v2/BBH": 0.5618, - "hfopenllm_v2/MATH Level 5": 0.1866, - "hfopenllm_v2/GPQA": 0.3062, - "hfopenllm_v2/MUSR": 0.4174, - "hfopenllm_v2/MMLU-PRO": 0.4916 - } - }, - { - "id": "Qwen/Qwen2-57B-A14B-Instruct", - "name": "Qwen2-57B-A14B-Instruct", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6338, - "hfopenllm_v2/BBH": 0.5888, - "hfopenllm_v2/MATH Level 5": 0.2817, - "hfopenllm_v2/GPQA": 0.3314, - "hfopenllm_v2/MUSR": 0.4361, - "hfopenllm_v2/MMLU-PRO": 0.4575 - } - }, - { - "id": "Qwen/Qwen2-72B", - "name": "Qwen2-72B", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3824, - "hfopenllm_v2/BBH": 0.6617, - "hfopenllm_v2/MATH Level 5": 0.3112, - "hfopenllm_v2/GPQA": 0.3943, - "hfopenllm_v2/MUSR": 0.4704, - "hfopenllm_v2/MMLU-PRO": 0.5731 - } - }, - { - "id": "qwen/qwen2-72b-instruct", - "name": "Qwen2 Instruct 72B", - "developer": "qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.77, - "helm_lite/NarrativeQA": 0.727, - "helm_lite/NaturalQuestions (closed-book)": 0.39, - "helm_lite/OpenbookQA": 0.954, - "helm_lite/MMLU": 0.769, - "helm_lite/MATH": 0.79, - "helm_lite/GSM8K": 0.92, - "helm_lite/LegalBench": 0.712, - "helm_lite/MedQA": 0.746, - "helm_lite/WMT 2014": 0.207, - "helm_mmlu/MMLU All Subjects": 0.824, - "helm_mmlu/Abstract Algebra": 0.67, - "helm_mmlu/Anatomy": 0.793, - "helm_mmlu/College Physics": 0.598, - "helm_mmlu/Computer Security": 0.85, - "helm_mmlu/Econometrics": 0.737, - "helm_mmlu/Global Facts": 0.58, - "helm_mmlu/Jurisprudence": 0.87, - "helm_mmlu/Philosophy": 0.859, - "helm_mmlu/Professional Psychology": 0.886, - "helm_mmlu/Us Foreign Policy": 0.94, - "helm_mmlu/Astronomy": 0.934, - "helm_mmlu/Business Ethics": 0.82, - "helm_mmlu/Clinical Knowledge": 0.868, - "helm_mmlu/Conceptual Physics": 0.872, - "helm_mmlu/Electrical Engineering": 0.793, - "helm_mmlu/Elementary Mathematics": 0.825, - "helm_mmlu/Formal Logic": 0.667, - "helm_mmlu/High School World History": 0.932, - "helm_mmlu/Human Sexuality": 0.893, - "helm_mmlu/International Law": 0.893, - "helm_mmlu/Logical Fallacies": 0.914, - "helm_mmlu/Machine Learning": 0.768, - "helm_mmlu/Management": 0.903, - "helm_mmlu/Marketing": 0.953, - "helm_mmlu/Medical Genetics": 0.9, - "helm_mmlu/Miscellaneous": 0.943, - "helm_mmlu/Moral Scenarios": 0.815, - "helm_mmlu/Nutrition": 0.902, - "helm_mmlu/Prehistory": 0.914, - "helm_mmlu/Public Relations": 0.745, - "helm_mmlu/Security Studies": 0.837, - "helm_mmlu/Sociology": 0.935, - "helm_mmlu/Virology": 0.56, - "helm_mmlu/World Religions": 0.848, - "helm_mmlu/Mean win rate": 0.826, - "hfopenllm_v2/IFEval": 0.7989, - "hfopenllm_v2/BBH": 0.6977, - "hfopenllm_v2/MATH Level 5": 0.4177, - "hfopenllm_v2/GPQA": 0.3725, - "hfopenllm_v2/MUSR": 0.456, - "hfopenllm_v2/MMLU-PRO": 0.5403 - } - }, - { - "id": "Qwen/Qwen2-7B", - "name": "Qwen2-7B", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3149, - "hfopenllm_v2/BBH": 0.5315, - "hfopenllm_v2/MATH Level 5": 0.2039, - "hfopenllm_v2/GPQA": 0.3045, - "hfopenllm_v2/MUSR": 0.4439, - "hfopenllm_v2/MMLU-PRO": 0.4183 - } - }, - { - "id": "Qwen/Qwen2-7B-Instruct", - "name": "Qwen2-7B-Instruct", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5679, - "hfopenllm_v2/BBH": 0.5545, - "hfopenllm_v2/MATH Level 5": 0.2764, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.3928, - "hfopenllm_v2/MMLU-PRO": 0.3847 - } - }, - { - "id": "Qwen/Qwen2-Math-72B-Instruct", - "name": "Qwen2-Math-72B-Instruct", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5694, - "hfopenllm_v2/BBH": 0.6343, - "hfopenllm_v2/MATH Level 5": 0.5536, - "hfopenllm_v2/GPQA": 0.3683, - "hfopenllm_v2/MUSR": 0.4517, - "hfopenllm_v2/MMLU-PRO": 0.4273 - } - }, - { - "id": "Qwen/Qwen2-Math-7B", - "name": "Qwen2-Math-7B", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2687, - "hfopenllm_v2/BBH": 0.387, - "hfopenllm_v2/MATH Level 5": 0.2477, - "hfopenllm_v2/GPQA": 0.2634, - "hfopenllm_v2/MUSR": 0.3593, - "hfopenllm_v2/MMLU-PRO": 0.1197 - } - }, - { - "id": "Qwen/Qwen2-VL-72B-Instruct", - "name": "Qwen2-VL-72B-Instruct", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5982, - "hfopenllm_v2/BBH": 0.6946, - "hfopenllm_v2/MATH Level 5": 0.3444, - "hfopenllm_v2/GPQA": 0.3876, - "hfopenllm_v2/MUSR": 0.4492, - "hfopenllm_v2/MMLU-PRO": 0.5717 - } - }, - { - "id": "Qwen/Qwen2-VL-7B-Instruct", - "name": "Qwen2-VL-7B-Instruct", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4599, - "hfopenllm_v2/BBH": 0.5465, - "hfopenllm_v2/MATH Level 5": 0.1986, - "hfopenllm_v2/GPQA": 0.3196, - "hfopenllm_v2/MUSR": 0.4375, - "hfopenllm_v2/MMLU-PRO": 0.4095 - } - }, - { - "id": "Qwen/Qwen2.5-0.5B", - "name": "Qwen2.5-0.5B", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1627, - "hfopenllm_v2/BBH": 0.3275, - "hfopenllm_v2/MATH Level 5": 0.0393, - "hfopenllm_v2/GPQA": 0.2466, - "hfopenllm_v2/MUSR": 0.3433, - "hfopenllm_v2/MMLU-PRO": 0.1906 - } - }, - { - "id": "Qwen/Qwen2.5-0.5B-Instruct", - "name": "Qwen2.5-0.5B-Instruct", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3071, - "hfopenllm_v2/BBH": 0.3341, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2576, - "hfopenllm_v2/MUSR": 0.3329, - "hfopenllm_v2/MMLU-PRO": 0.1697 - } - }, - { - "id": "Qwen/Qwen2.5-1.5B", - "name": "Qwen2.5-1.5B", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2674, - "hfopenllm_v2/BBH": 0.4078, - "hfopenllm_v2/MATH Level 5": 0.0914, - "hfopenllm_v2/GPQA": 0.2852, - "hfopenllm_v2/MUSR": 0.3576, - "hfopenllm_v2/MMLU-PRO": 0.2855 - } - }, - { - "id": "Qwen/Qwen2.5-1.5B-Instruct", - "name": "Qwen2.5-1.5B-Instruct", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4476, - "hfopenllm_v2/BBH": 0.4289, - "hfopenllm_v2/MATH Level 5": 0.2205, - "hfopenllm_v2/GPQA": 0.2559, - "hfopenllm_v2/MUSR": 0.3663, - "hfopenllm_v2/MMLU-PRO": 0.2799 - } - }, - { - "id": "Qwen/Qwen2.5-14B", - "name": "Qwen2.5-14B", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3694, - "hfopenllm_v2/BBH": 0.6161, - "hfopenllm_v2/MATH Level 5": 0.29, - "hfopenllm_v2/GPQA": 0.3817, - "hfopenllm_v2/MUSR": 0.4502, - "hfopenllm_v2/MMLU-PRO": 0.5249 - } - }, - { - "id": "Qwen/Qwen2.5-14B-Instruct", - "name": "Qwen2.5-14B-Instruct", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8158, - "hfopenllm_v2/BBH": 0.639, - "hfopenllm_v2/MATH Level 5": 0.5476, - "hfopenllm_v2/GPQA": 0.3221, - "hfopenllm_v2/MUSR": 0.4101, - "hfopenllm_v2/MMLU-PRO": 0.4904 - } - }, - { - "id": "Qwen/Qwen2.5-14B-Instruct-1M", - "name": "Qwen2.5-14B-Instruct-1M", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8414, - "hfopenllm_v2/BBH": 0.6198, - "hfopenllm_v2/MATH Level 5": 0.5302, - "hfopenllm_v2/GPQA": 0.3431, - "hfopenllm_v2/MUSR": 0.418, - "hfopenllm_v2/MMLU-PRO": 0.485 - } - }, - { - "id": "Qwen/Qwen2.5-32B", - "name": "Qwen2.5-32B", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4077, - "hfopenllm_v2/BBH": 0.6771, - "hfopenllm_v2/MATH Level 5": 0.3565, - "hfopenllm_v2/GPQA": 0.4119, - "hfopenllm_v2/MUSR": 0.4978, - "hfopenllm_v2/MMLU-PRO": 0.5805 - } - }, - { - "id": "Qwen/Qwen2.5-32B-Instruct", - "name": "Qwen2.5-32B-Instruct", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8346, - "hfopenllm_v2/BBH": 0.6913, - "hfopenllm_v2/MATH Level 5": 0.6254, - "hfopenllm_v2/GPQA": 0.3381, - "hfopenllm_v2/MUSR": 0.4261, - "hfopenllm_v2/MMLU-PRO": 0.5667 - } - }, - { - "id": "Qwen/Qwen2.5-3B", - "name": "Qwen2.5-3B", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.269, - "hfopenllm_v2/BBH": 0.4612, - "hfopenllm_v2/MATH Level 5": 0.148, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.4303, - "hfopenllm_v2/MMLU-PRO": 0.3203 - } - }, - { - "id": "Qwen/Qwen2.5-3B-Instruct", - "name": "Qwen2.5-3B-Instruct", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6475, - "hfopenllm_v2/BBH": 0.4693, - "hfopenllm_v2/MATH Level 5": 0.3678, - "hfopenllm_v2/GPQA": 0.2727, - "hfopenllm_v2/MUSR": 0.3968, - "hfopenllm_v2/MMLU-PRO": 0.3255, - "theory_of_mind/accuracy on theory_of_mind for scorer model_graded_fact": 0.78 - } - }, - { - "id": "Qwen/Qwen2.5-72B", - "name": "Qwen2.5-72B", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4137, - "hfopenllm_v2/BBH": 0.6797, - "hfopenllm_v2/MATH Level 5": 0.3912, - "hfopenllm_v2/GPQA": 0.4052, - "hfopenllm_v2/MUSR": 0.4771, - "hfopenllm_v2/MMLU-PRO": 0.5968 - } - }, - { - "id": "Qwen/Qwen2.5-72B-Instruct", - "name": "Qwen2.5-72B-Instruct", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8638, - "hfopenllm_v2/BBH": 0.7273, - "hfopenllm_v2/MATH Level 5": 0.5982, - "hfopenllm_v2/GPQA": 0.375, - "hfopenllm_v2/MUSR": 0.4206, - "hfopenllm_v2/MMLU-PRO": 0.5626 - } - }, - { - "id": "qwen/qwen2.5-72b-instruct-turbo", - "name": "Qwen2.5 Instruct Turbo 72B", - "developer": "qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.599, - "helm_capabilities/MMLU-Pro": 0.631, - "helm_capabilities/GPQA": 0.426, - "helm_capabilities/IFEval": 0.806, - "helm_capabilities/WildBench": 0.802, - "helm_capabilities/Omni-MATH": 0.33, - "helm_lite/Mean win rate": 0.745, - "helm_lite/NarrativeQA": 0.745, - "helm_lite/NaturalQuestions (closed-book)": 0.359, - "helm_lite/OpenbookQA": 0.962, - "helm_lite/MMLU": 0.77, - "helm_lite/MATH": 0.884, - "helm_lite/GSM8K": 0.9, - "helm_lite/LegalBench": 0.74, - "helm_lite/MedQA": 0.753, - "helm_lite/WMT 2014": 0.207, - "helm_mmlu/MMLU All Subjects": 0.834, - "helm_mmlu/Abstract Algebra": 0.68, - "helm_mmlu/Anatomy": 0.822, - "helm_mmlu/College Physics": 0.588, - "helm_mmlu/Computer Security": 0.86, - "helm_mmlu/Econometrics": 0.728, - "helm_mmlu/Global Facts": 0.61, - "helm_mmlu/Jurisprudence": 0.87, - "helm_mmlu/Philosophy": 0.839, - "helm_mmlu/Professional Psychology": 0.864, - "helm_mmlu/Us Foreign Policy": 0.96, - "helm_mmlu/Astronomy": 0.934, - "helm_mmlu/Business Ethics": 0.85, - "helm_mmlu/Clinical Knowledge": 0.872, - "helm_mmlu/Conceptual Physics": 0.885, - "helm_mmlu/Electrical Engineering": 0.8, - "helm_mmlu/Elementary Mathematics": 0.87, - "helm_mmlu/Formal Logic": 0.73, - "helm_mmlu/High School World History": 0.92, - "helm_mmlu/Human Sexuality": 0.878, - "helm_mmlu/International Law": 0.893, - "helm_mmlu/Logical Fallacies": 0.89, - "helm_mmlu/Machine Learning": 0.777, - "helm_mmlu/Management": 0.913, - "helm_mmlu/Marketing": 0.953, - "helm_mmlu/Medical Genetics": 0.92, - "helm_mmlu/Miscellaneous": 0.932, - "helm_mmlu/Moral Scenarios": 0.787, - "helm_mmlu/Nutrition": 0.886, - "helm_mmlu/Prehistory": 0.91, - "helm_mmlu/Public Relations": 0.782, - "helm_mmlu/Security Studies": 0.849, - "helm_mmlu/Sociology": 0.925, - "helm_mmlu/Virology": 0.584, - "helm_mmlu/World Religions": 0.901, - "helm_mmlu/Mean win rate": 0.548 - } - }, - { - "id": "Qwen/Qwen2.5-7B", - "name": "Qwen2.5-7B", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3374, - "hfopenllm_v2/BBH": 0.5416, - "hfopenllm_v2/MATH Level 5": 0.2508, - "hfopenllm_v2/GPQA": 0.3247, - "hfopenllm_v2/MUSR": 0.4424, - "hfopenllm_v2/MMLU-PRO": 0.4365, - "la_leaderboard/la_leaderboard": 27.61 - } - }, - { - "id": "Qwen/Qwen2.5-7B-Instruct", - "name": "Qwen2.5-7B-Instruct", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7585, - "hfopenllm_v2/BBH": 0.5394, - "hfopenllm_v2/MATH Level 5": 0.5, - "hfopenllm_v2/GPQA": 0.2911, - "hfopenllm_v2/MUSR": 0.402, - "hfopenllm_v2/MMLU-PRO": 0.4287 - } - }, - { - "id": "Qwen/Qwen2.5-7B-Instruct-1M", - "name": "Qwen2.5-7B-Instruct-1M", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7448, - "hfopenllm_v2/BBH": 0.5404, - "hfopenllm_v2/MATH Level 5": 0.4335, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.4087, - "hfopenllm_v2/MMLU-PRO": 0.3505 - } - }, - { - "id": "qwen/qwen2.5-7b-instruct-turbo", - "name": "Qwen2.5 Instruct Turbo 7B", - "developer": "qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.529, - "helm_capabilities/MMLU-Pro": 0.539, - "helm_capabilities/GPQA": 0.341, - "helm_capabilities/IFEval": 0.741, - "helm_capabilities/WildBench": 0.731, - "helm_capabilities/Omni-MATH": 0.294, - "helm_lite/Mean win rate": 0.488, - "helm_lite/NarrativeQA": 0.742, - "helm_lite/NaturalQuestions (closed-book)": 0.205, - "helm_lite/OpenbookQA": 0.862, - "helm_lite/MMLU": 0.658, - "helm_lite/MATH": 0.835, - "helm_lite/GSM8K": 0.83, - "helm_lite/LegalBench": 0.632, - "helm_lite/MedQA": 0.6, - "helm_lite/WMT 2014": 0.155, - "helm_mmlu/MMLU All Subjects": 0.729, - "helm_mmlu/Abstract Algebra": 0.49, - "helm_mmlu/Anatomy": 0.689, - "helm_mmlu/College Physics": 0.51, - "helm_mmlu/Computer Security": 0.79, - "helm_mmlu/Econometrics": 0.64, - "helm_mmlu/Global Facts": 0.42, - "helm_mmlu/Jurisprudence": 0.796, - "helm_mmlu/Philosophy": 0.746, - "helm_mmlu/Professional Psychology": 0.757, - "helm_mmlu/Us Foreign Policy": 0.86, - "helm_mmlu/Astronomy": 0.836, - "helm_mmlu/Business Ethics": 0.82, - "helm_mmlu/Clinical Knowledge": 0.785, - "helm_mmlu/Conceptual Physics": 0.736, - "helm_mmlu/Electrical Engineering": 0.717, - "helm_mmlu/Elementary Mathematics": 0.643, - "helm_mmlu/Formal Logic": 0.587, - "helm_mmlu/High School World History": 0.878, - "helm_mmlu/Human Sexuality": 0.794, - "helm_mmlu/International Law": 0.86, - "helm_mmlu/Logical Fallacies": 0.773, - "helm_mmlu/Machine Learning": 0.554, - "helm_mmlu/Management": 0.845, - "helm_mmlu/Marketing": 0.919, - "helm_mmlu/Medical Genetics": 0.85, - "helm_mmlu/Miscellaneous": 0.852, - "helm_mmlu/Moral Scenarios": 0.511, - "helm_mmlu/Nutrition": 0.778, - "helm_mmlu/Prehistory": 0.836, - "helm_mmlu/Public Relations": 0.709, - "helm_mmlu/Security Studies": 0.682, - "helm_mmlu/Sociology": 0.861, - "helm_mmlu/Virology": 0.578, - "helm_mmlu/World Religions": 0.83, - "helm_mmlu/Mean win rate": 0.887 - } - }, - { - "id": "Qwen/Qwen2.5-Coder-14B", - "name": "Qwen2.5-Coder-14B", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3473, - "hfopenllm_v2/BBH": 0.5865, - "hfopenllm_v2/MATH Level 5": 0.2251, - "hfopenllm_v2/GPQA": 0.2928, - "hfopenllm_v2/MUSR": 0.3874, - "hfopenllm_v2/MMLU-PRO": 0.4521 - } - }, - { - "id": "Qwen/Qwen2.5-Coder-14B-Instruct", - "name": "Qwen2.5-Coder-14B-Instruct", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6908, - "hfopenllm_v2/BBH": 0.614, - "hfopenllm_v2/MATH Level 5": 0.3248, - "hfopenllm_v2/GPQA": 0.3045, - "hfopenllm_v2/MUSR": 0.3915, - "hfopenllm_v2/MMLU-PRO": 0.3939 - } - }, - { - "id": "Qwen/Qwen2.5-Coder-32B", - "name": "Qwen2.5-Coder-32B", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4363, - "hfopenllm_v2/BBH": 0.6404, - "hfopenllm_v2/MATH Level 5": 0.3089, - "hfopenllm_v2/GPQA": 0.3465, - "hfopenllm_v2/MUSR": 0.4528, - "hfopenllm_v2/MMLU-PRO": 0.5303 - } - }, - { - "id": "Qwen/Qwen2.5-Coder-32B-Instruct", - "name": "Qwen2.5-Coder-32B-Instruct", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7265, - "hfopenllm_v2/BBH": 0.6625, - "hfopenllm_v2/MATH Level 5": 0.4955, - "hfopenllm_v2/GPQA": 0.349, - "hfopenllm_v2/MUSR": 0.4386, - "hfopenllm_v2/MMLU-PRO": 0.4413 - } - }, - { - "id": "Qwen/Qwen2.5-Coder-7B", - "name": "Qwen2.5-Coder-7B", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3446, - "hfopenllm_v2/BBH": 0.4856, - "hfopenllm_v2/MATH Level 5": 0.1918, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3449, - "hfopenllm_v2/MMLU-PRO": 0.3679 - } - }, - { - "id": "Qwen/Qwen2.5-Coder-7B-Instruct", - "name": "Qwen2.5-Coder-7B-Instruct", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6147, - "hfopenllm_v2/BBH": 0.4999, - "hfopenllm_v2/MATH Level 5": 0.031, - "hfopenllm_v2/GPQA": 0.2936, - "hfopenllm_v2/MUSR": 0.4099, - "hfopenllm_v2/MMLU-PRO": 0.3354 - } - }, - { - "id": "Qwen/Qwen2.5-Math-1.5B-Instruct", - "name": "Qwen2.5-Math-1.5B-Instruct", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1856, - "hfopenllm_v2/BBH": 0.3752, - "hfopenllm_v2/MATH Level 5": 0.2628, - "hfopenllm_v2/GPQA": 0.2651, - "hfopenllm_v2/MUSR": 0.3685, - "hfopenllm_v2/MMLU-PRO": 0.1801 - } - }, - { - "id": "Qwen/Qwen2.5-Math-72B-Instruct", - "name": "Qwen2.5-Math-72B-Instruct", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4003, - "hfopenllm_v2/BBH": 0.6452, - "hfopenllm_v2/MATH Level 5": 0.6239, - "hfopenllm_v2/GPQA": 0.3314, - "hfopenllm_v2/MUSR": 0.4473, - "hfopenllm_v2/MMLU-PRO": 0.4812 - } - }, - { - "id": "Qwen/Qwen2.5-Math-7B", - "name": "Qwen2.5-Math-7B", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.246, - "hfopenllm_v2/BBH": 0.4455, - "hfopenllm_v2/MATH Level 5": 0.3051, - "hfopenllm_v2/GPQA": 0.2936, - "hfopenllm_v2/MUSR": 0.3781, - "hfopenllm_v2/MMLU-PRO": 0.2718 - } - }, - { - "id": "Qwen/Qwen2.5-Math-7B-Instruct", - "name": "Qwen2.5-Math-7B-Instruct", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2636, - "hfopenllm_v2/BBH": 0.4388, - "hfopenllm_v2/MATH Level 5": 0.5808, - "hfopenllm_v2/GPQA": 0.2617, - "hfopenllm_v2/MUSR": 0.3647, - "hfopenllm_v2/MMLU-PRO": 0.282 - } - }, - { - "id": "qwen/qwen3-0-6b-fc", - "name": "Qwen3-0.6B (FC)", - "developer": "qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 92.0, - "bfcl/bfcl.overall.overall_accuracy": 23.93, - "bfcl/bfcl.overall.total_cost_usd": 0.46, - "bfcl/bfcl.overall.latency_mean_s": 0.68, - "bfcl/bfcl.overall.latency_std_s": 8.45, - "bfcl/bfcl.overall.latency_p95_s": 0.96, - "bfcl/bfcl.non_live.ast_accuracy": 71.79, - "bfcl/bfcl.non_live.simple_ast_accuracy": 64.17, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 86.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 67.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 69.5, - "bfcl/bfcl.live.live_accuracy": 56.62, - "bfcl/bfcl.live.live_simple_ast_accuracy": 61.24, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 56.13, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 43.75, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 37.5, - "bfcl/bfcl.multi_turn.accuracy": 3.62, - "bfcl/bfcl.multi_turn.base_accuracy": 5.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 2.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 3.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 4.0, - "bfcl/bfcl.web_search.accuracy": 1.0, - "bfcl/bfcl.web_search.base_accuracy": 1.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 1.0, - "bfcl/bfcl.memory.accuracy": 8.6, - "bfcl/bfcl.memory.kv_accuracy": 2.58, - "bfcl/bfcl.memory.vector_accuracy": 1.94, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 21.29, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 75.0, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 80.84 - } - }, - { - "id": "qwen/qwen3-0-6b-prompt", - "name": "Qwen3-0.6B (Prompt)", - "developer": "qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 94.0, - "bfcl/bfcl.overall.overall_accuracy": 22.38, - "bfcl/bfcl.overall.total_cost_usd": 3.65, - "bfcl/bfcl.overall.latency_mean_s": 3.1, - "bfcl/bfcl.overall.latency_std_s": 4.32, - "bfcl/bfcl.overall.latency_p95_s": 10.31, - "bfcl/bfcl.non_live.ast_accuracy": 70.0, - "bfcl/bfcl.non_live.simple_ast_accuracy": 64.0, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 78.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 75.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 63.0, - "bfcl/bfcl.live.live_accuracy": 49.37, - "bfcl/bfcl.live.live_simple_ast_accuracy": 57.75, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 47.77, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 37.5, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 37.5, - "bfcl/bfcl.multi_turn.accuracy": 1.38, - "bfcl/bfcl.multi_turn.base_accuracy": 1.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 1.5, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 1.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 1.0, - "bfcl/bfcl.web_search.accuracy": 0.5, - "bfcl/bfcl.web_search.base_accuracy": 1.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 0.0, - "bfcl/bfcl.memory.accuracy": 8.39, - "bfcl/bfcl.memory.kv_accuracy": 1.29, - "bfcl/bfcl.memory.vector_accuracy": 2.58, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 21.29, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 75.0, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 82.5, - "bfcl/bfcl.format_sensitivity.max_delta": 60.5, - "bfcl/bfcl.format_sensitivity.stddev": 24.35 - } - }, - { - "id": "qwen/qwen3-1-7b-fc", - "name": "Qwen3-1.7B (FC)", - "developer": "qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 71.0, - "bfcl/bfcl.overall.overall_accuracy": 28.41, - "bfcl/bfcl.overall.total_cost_usd": 4.33, - "bfcl/bfcl.overall.latency_mean_s": 5.12, - "bfcl/bfcl.overall.latency_std_s": 7.37, - "bfcl/bfcl.overall.latency_p95_s": 13.35, - "bfcl/bfcl.non_live.ast_accuracy": 82.92, - "bfcl/bfcl.non_live.simple_ast_accuracy": 70.67, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 92.5, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 88.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 80.0, - "bfcl/bfcl.live.live_accuracy": 74.61, - "bfcl/bfcl.live.live_simple_ast_accuracy": 76.74, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 74.26, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 62.5, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 75.0, - "bfcl/bfcl.multi_turn.accuracy": 11.0, - "bfcl/bfcl.multi_turn.base_accuracy": 15.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 6.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 12.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 11.0, - "bfcl/bfcl.web_search.accuracy": 2.5, - "bfcl/bfcl.web_search.base_accuracy": 3.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 2.0, - "bfcl/bfcl.memory.accuracy": 6.02, - "bfcl/bfcl.memory.kv_accuracy": 4.52, - "bfcl/bfcl.memory.vector_accuracy": 7.74, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 5.81, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 81.25, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 76.54 - } - }, - { - "id": "qwen/qwen3-14b-fc", - "name": "Qwen3-14B (FC)", - "developer": "qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 43.0, - "bfcl/bfcl.overall.overall_accuracy": 41.03, - "bfcl/bfcl.overall.total_cost_usd": 3.38, - "bfcl/bfcl.overall.latency_mean_s": 4.5, - "bfcl/bfcl.overall.latency_std_s": 18.84, - "bfcl/bfcl.overall.latency_p95_s": 13.34, - "bfcl/bfcl.non_live.ast_accuracy": 84.94, - "bfcl/bfcl.non_live.simple_ast_accuracy": 74.75, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 93.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 80.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 92.0, - "bfcl/bfcl.live.live_accuracy": 80.01, - "bfcl/bfcl.live.live_simple_ast_accuracy": 85.66, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 79.01, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 68.75, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 70.83, - "bfcl/bfcl.multi_turn.accuracy": 34.75, - "bfcl/bfcl.multi_turn.base_accuracy": 39.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 34.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 33.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 32.5, - "bfcl/bfcl.web_search.accuracy": 10.0, - "bfcl/bfcl.web_search.base_accuracy": 8.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 12.0, - "bfcl/bfcl.memory.accuracy": 19.57, - "bfcl/bfcl.memory.kv_accuracy": 7.1, - "bfcl/bfcl.memory.vector_accuracy": 16.77, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 34.84, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 87.5, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 81.94 - } - }, - { - "id": "qwen/qwen3-14b-prompt", - "name": "Qwen3-14B (Prompt)", - "developer": "qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 47.0, - "bfcl/bfcl.overall.overall_accuracy": 37.77, - "bfcl/bfcl.overall.total_cost_usd": 1.35, - "bfcl/bfcl.overall.latency_mean_s": 1.2, - "bfcl/bfcl.overall.latency_std_s": 8.5, - "bfcl/bfcl.overall.latency_p95_s": 2.3, - "bfcl/bfcl.non_live.ast_accuracy": 89.46, - "bfcl/bfcl.non_live.simple_ast_accuracy": 76.83, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 93.5, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 95.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 92.0, - "bfcl/bfcl.live.live_accuracy": 79.35, - "bfcl/bfcl.live.live_simple_ast_accuracy": 84.5, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 78.06, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 87.5, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 75.0, - "bfcl/bfcl.multi_turn.accuracy": 26.13, - "bfcl/bfcl.multi_turn.base_accuracy": 16.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 37.5, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 31.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 19.5, - "bfcl/bfcl.web_search.accuracy": 10.5, - "bfcl/bfcl.web_search.base_accuracy": 6.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 15.0, - "bfcl/bfcl.memory.accuracy": 11.18, - "bfcl/bfcl.memory.kv_accuracy": 4.52, - "bfcl/bfcl.memory.vector_accuracy": 6.45, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 22.58, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 81.25, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 87.18, - "bfcl/bfcl.format_sensitivity.max_delta": 14.0, - "bfcl/bfcl.format_sensitivity.stddev": 3.97 - } - }, - { - "id": "qwen/qwen3-235b-a22b-fp8-tput", - "name": "Qwen3 235B A22B FP8 Throughput", - "developer": "qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.726, - "helm_capabilities/MMLU-Pro": 0.817, - "helm_capabilities/GPQA": 0.623, - "helm_capabilities/IFEval": 0.816, - "helm_capabilities/WildBench": 0.828, - "helm_capabilities/Omni-MATH": 0.548 - } - }, - { - "id": "qwen/qwen3-235b-a22b-instruct-2507-fc", - "name": "Qwen3-235B-A22B-Instruct-2507 (FC)", - "developer": "qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 31.0, - "bfcl/bfcl.overall.overall_accuracy": 47.99, - "bfcl/bfcl.overall.total_cost_usd": 2.5, - "bfcl/bfcl.overall.latency_mean_s": 2.57, - "bfcl/bfcl.overall.latency_std_s": 2.44, - "bfcl/bfcl.overall.latency_p95_s": 6.27, - "bfcl/bfcl.non_live.ast_accuracy": 37.4, - "bfcl/bfcl.non_live.simple_ast_accuracy": 40.58, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 36.5, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 53.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 19.5, - "bfcl/bfcl.live.live_accuracy": 68.91, - "bfcl/bfcl.live.live_simple_ast_accuracy": 58.53, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 71.6, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 68.75, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 62.5, - "bfcl/bfcl.multi_turn.accuracy": 45.38, - "bfcl/bfcl.multi_turn.base_accuracy": 57.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 35.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 33.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 55.5, - "bfcl/bfcl.web_search.accuracy": 54.0, - "bfcl/bfcl.web_search.base_accuracy": 57.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 51.0, - "bfcl/bfcl.memory.accuracy": 23.87, - "bfcl/bfcl.memory.kv_accuracy": 7.1, - "bfcl/bfcl.memory.vector_accuracy": 18.71, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 45.81, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 87.5, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 81.73 - } - }, - { - "id": "qwen/qwen3-235b-a22b-instruct-2507-fp8", - "name": "Qwen3 235B A22B Instruct 2507 FP8", - "developer": "qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.798, - "helm_capabilities/MMLU-Pro": 0.844, - "helm_capabilities/GPQA": 0.726, - "helm_capabilities/IFEval": 0.835, - "helm_capabilities/WildBench": 0.866, - "helm_capabilities/Omni-MATH": 0.718 - } - }, - { - "id": "qwen/qwen3-235b-a22b-instruct-2507-prompt", - "name": "Qwen3-235B-A22B-Instruct-2507 (Prompt)", - "developer": "qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 23.0, - "bfcl/bfcl.overall.overall_accuracy": 52.15, - "bfcl/bfcl.overall.total_cost_usd": 3.12, - "bfcl/bfcl.overall.latency_mean_s": 2.56, - "bfcl/bfcl.overall.latency_std_s": 2.75, - "bfcl/bfcl.overall.latency_p95_s": 7.61, - "bfcl/bfcl.non_live.ast_accuracy": 90.33, - "bfcl/bfcl.non_live.simple_ast_accuracy": 79.83, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 95.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 95.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 91.0, - "bfcl/bfcl.live.live_accuracy": 78.68, - "bfcl/bfcl.live.live_simple_ast_accuracy": 82.95, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 77.78, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 81.25, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 70.83, - "bfcl/bfcl.multi_turn.accuracy": 44.62, - "bfcl/bfcl.multi_turn.base_accuracy": 54.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 42.5, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 31.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 50.5, - "bfcl/bfcl.web_search.accuracy": 50.5, - "bfcl/bfcl.web_search.base_accuracy": 56.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 45.0, - "bfcl/bfcl.memory.accuracy": 19.35, - "bfcl/bfcl.memory.kv_accuracy": 12.9, - "bfcl/bfcl.memory.vector_accuracy": 11.61, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 33.55, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 93.75, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 78.89, - "bfcl/bfcl.format_sensitivity.max_delta": 8.0, - "bfcl/bfcl.format_sensitivity.stddev": 1.95 - } - }, - { - "id": "qwen/qwen3-30b-a3b-instruct-2507-fc", - "name": "Qwen3-30B-A3B-Instruct-2507 (FC)", - "developer": "qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 41.0, - "bfcl/bfcl.overall.overall_accuracy": 41.39, - "bfcl/bfcl.overall.total_cost_usd": 5.62, - "bfcl/bfcl.overall.latency_mean_s": 5.95, - "bfcl/bfcl.overall.latency_std_s": 25.48, - "bfcl/bfcl.overall.latency_p95_s": 12.7, - "bfcl/bfcl.non_live.ast_accuracy": 85.77, - "bfcl/bfcl.non_live.simple_ast_accuracy": 68.58, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 94.5, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 91.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 88.5, - "bfcl/bfcl.live.live_accuracy": 77.94, - "bfcl/bfcl.live.live_simple_ast_accuracy": 83.33, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 76.83, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 68.75, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 75.0, - "bfcl/bfcl.multi_turn.accuracy": 30.0, - "bfcl/bfcl.multi_turn.base_accuracy": 43.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 10.5, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 25.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 41.0, - "bfcl/bfcl.web_search.accuracy": 22.5, - "bfcl/bfcl.web_search.base_accuracy": 21.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 24.0, - "bfcl/bfcl.memory.accuracy": 17.63, - "bfcl/bfcl.memory.kv_accuracy": 9.03, - "bfcl/bfcl.memory.vector_accuracy": 9.03, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 34.84, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 81.25, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 79.9 - } - }, - { - "id": "qwen/qwen3-30b-a3b-instruct-2507-prompt", - "name": "Qwen3-30B-A3B-Instruct-2507 (Prompt)", - "developer": "qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 53.0, - "bfcl/bfcl.overall.overall_accuracy": 36.7, - "bfcl/bfcl.overall.total_cost_usd": 1.56, - "bfcl/bfcl.overall.latency_mean_s": 1.24, - "bfcl/bfcl.overall.latency_std_s": 7.9, - "bfcl/bfcl.overall.latency_p95_s": 2.84, - "bfcl/bfcl.non_live.ast_accuracy": 88.92, - "bfcl/bfcl.non_live.simple_ast_accuracy": 80.67, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 93.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 94.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 88.0, - "bfcl/bfcl.live.live_accuracy": 78.39, - "bfcl/bfcl.live.live_simple_ast_accuracy": 82.56, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 77.49, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 87.5, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 66.67, - "bfcl/bfcl.multi_turn.accuracy": 23.5, - "bfcl/bfcl.multi_turn.base_accuracy": 33.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 16.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 16.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 29.0, - "bfcl/bfcl.web_search.accuracy": 17.5, - "bfcl/bfcl.web_search.base_accuracy": 15.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 20.0, - "bfcl/bfcl.memory.accuracy": 9.68, - "bfcl/bfcl.memory.kv_accuracy": 5.81, - "bfcl/bfcl.memory.vector_accuracy": 6.45, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 16.77, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 93.75, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 74.85, - "bfcl/bfcl.format_sensitivity.max_delta": 16.0, - "bfcl/bfcl.format_sensitivity.stddev": 4.13 - } - }, - { - "id": "qwen/qwen3-32b-fc", - "name": "Qwen3-32B (FC)", - "developer": "qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 29.0, - "bfcl/bfcl.overall.overall_accuracy": 48.71, - "bfcl/bfcl.overall.total_cost_usd": 153.08, - "bfcl/bfcl.overall.latency_mean_s": 169.87, - "bfcl/bfcl.overall.latency_std_s": 164.27, - "bfcl/bfcl.overall.latency_p95_s": 473.49, - "bfcl/bfcl.non_live.ast_accuracy": 88.77, - "bfcl/bfcl.non_live.simple_ast_accuracy": 75.58, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 94.5, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 93.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 91.5, - "bfcl/bfcl.live.live_accuracy": 82.01, - "bfcl/bfcl.live.live_simple_ast_accuracy": 89.53, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 80.91, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 81.25, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 50.0, - "bfcl/bfcl.multi_turn.accuracy": 47.87, - "bfcl/bfcl.multi_turn.base_accuracy": 56.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 52.5, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 40.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 43.0, - "bfcl/bfcl.web_search.accuracy": 21.5, - "bfcl/bfcl.web_search.base_accuracy": 25.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 18.0, - "bfcl/bfcl.memory.accuracy": 26.67, - "bfcl/bfcl.memory.kv_accuracy": 12.26, - "bfcl/bfcl.memory.vector_accuracy": 25.81, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 41.94, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 93.75, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 76.37 - } - }, - { - "id": "qwen/qwen3-32b-prompt", - "name": "Qwen3-32B (Prompt)", - "developer": "qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 33.0, - "bfcl/bfcl.overall.overall_accuracy": 46.78, - "bfcl/bfcl.overall.total_cost_usd": 199.47, - "bfcl/bfcl.overall.latency_mean_s": 167.54, - "bfcl/bfcl.overall.latency_std_s": 160.5, - "bfcl/bfcl.overall.latency_p95_s": 457.87, - "bfcl/bfcl.non_live.ast_accuracy": 90.27, - "bfcl/bfcl.non_live.simple_ast_accuracy": 79.08, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 97.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 93.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 91.5, - "bfcl/bfcl.live.live_accuracy": 82.01, - "bfcl/bfcl.live.live_simple_ast_accuracy": 87.21, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 81.2, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 81.25, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 62.5, - "bfcl/bfcl.multi_turn.accuracy": 43.25, - "bfcl/bfcl.multi_turn.base_accuracy": 54.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 46.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 36.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 36.5, - "bfcl/bfcl.web_search.accuracy": 26.0, - "bfcl/bfcl.web_search.base_accuracy": 34.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 18.0, - "bfcl/bfcl.memory.accuracy": 15.7, - "bfcl/bfcl.memory.kv_accuracy": 13.55, - "bfcl/bfcl.memory.vector_accuracy": 14.19, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 19.35, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 81.25, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 82.39, - "bfcl/bfcl.format_sensitivity.max_delta": 15.5, - "bfcl/bfcl.format_sensitivity.stddev": 3.75 - } - }, - { - "id": "qwen/qwen3-4b-instruct-2507-fc", - "name": "Qwen3-4B-Instruct-2507 (FC)", - "developer": "qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 54.0, - "bfcl/bfcl.overall.overall_accuracy": 35.68, - "bfcl/bfcl.overall.total_cost_usd": 6.37, - "bfcl/bfcl.overall.latency_mean_s": 7.61, - "bfcl/bfcl.overall.latency_std_s": 20.36, - "bfcl/bfcl.overall.latency_p95_s": 49.18, - "bfcl/bfcl.non_live.ast_accuracy": 87.88, - "bfcl/bfcl.non_live.simple_ast_accuracy": 75.5, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 93.5, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 92.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 90.0, - "bfcl/bfcl.live.live_accuracy": 76.39, - "bfcl/bfcl.live.live_simple_ast_accuracy": 79.07, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 76.16, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 62.5, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 66.67, - "bfcl/bfcl.multi_turn.accuracy": 22.12, - "bfcl/bfcl.multi_turn.base_accuracy": 26.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 21.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 15.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 25.5, - "bfcl/bfcl.web_search.accuracy": 3.0, - "bfcl/bfcl.web_search.base_accuracy": 4.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 2.0, - "bfcl/bfcl.memory.accuracy": 17.63, - "bfcl/bfcl.memory.kv_accuracy": 16.13, - "bfcl/bfcl.memory.vector_accuracy": 12.26, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 24.52, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 87.5, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 84.93 - } - }, - { - "id": "qwen/qwen3-4b-instruct-2507-prompt", - "name": "Qwen3-4B-Instruct-2507 (Prompt)", - "developer": "qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 55.0, - "bfcl/bfcl.overall.overall_accuracy": 35.52, - "bfcl/bfcl.overall.total_cost_usd": 53.66, - "bfcl/bfcl.overall.latency_mean_s": 44.7, - "bfcl/bfcl.overall.latency_std_s": 163.79, - "bfcl/bfcl.overall.latency_p95_s": 208.06, - "bfcl/bfcl.non_live.ast_accuracy": 86.44, - "bfcl/bfcl.non_live.simple_ast_accuracy": 77.25, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 91.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 88.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 89.5, - "bfcl/bfcl.live.live_accuracy": 74.69, - "bfcl/bfcl.live.live_simple_ast_accuracy": 77.91, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 74.17, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 81.25, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 58.33, - "bfcl/bfcl.multi_turn.accuracy": 20.5, - "bfcl/bfcl.multi_turn.base_accuracy": 24.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 21.5, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 16.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 20.0, - "bfcl/bfcl.web_search.accuracy": 4.5, - "bfcl/bfcl.web_search.base_accuracy": 4.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 5.0, - "bfcl/bfcl.memory.accuracy": 23.87, - "bfcl/bfcl.memory.kv_accuracy": 12.9, - "bfcl/bfcl.memory.vector_accuracy": 14.19, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 44.52, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 87.5, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 75.87, - "bfcl/bfcl.format_sensitivity.max_delta": 18.0, - "bfcl/bfcl.format_sensitivity.stddev": 5.22 - } - }, - { - "id": "qwen/qwen3-8b-fc", - "name": "Qwen3-8B (FC)", - "developer": "qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 39.0, - "bfcl/bfcl.overall.overall_accuracy": 42.57, - "bfcl/bfcl.overall.total_cost_usd": 43.32, - "bfcl/bfcl.overall.latency_mean_s": 51.36, - "bfcl/bfcl.overall.latency_std_s": 76.14, - "bfcl/bfcl.overall.latency_p95_s": 188.98, - "bfcl/bfcl.non_live.ast_accuracy": 87.58, - "bfcl/bfcl.non_live.simple_ast_accuracy": 72.83, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 96.5, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 92.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 89.0, - "bfcl/bfcl.live.live_accuracy": 80.53, - "bfcl/bfcl.live.live_simple_ast_accuracy": 84.5, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 79.68, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 75.0, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 79.17, - "bfcl/bfcl.multi_turn.accuracy": 41.75, - "bfcl/bfcl.multi_turn.base_accuracy": 50.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 42.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 40.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 34.5, - "bfcl/bfcl.web_search.accuracy": 12.0, - "bfcl/bfcl.web_search.base_accuracy": 15.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 9.0, - "bfcl/bfcl.memory.accuracy": 14.62, - "bfcl/bfcl.memory.kv_accuracy": 5.16, - "bfcl/bfcl.memory.vector_accuracy": 7.1, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 31.61, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 93.75, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 79.07 - } - }, - { - "id": "qwen/qwen3-8b-prompt", - "name": "Qwen3-8B (Prompt)", - "developer": "qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 44.0, - "bfcl/bfcl.overall.overall_accuracy": 40.43, - "bfcl/bfcl.overall.total_cost_usd": 63.95, - "bfcl/bfcl.overall.latency_mean_s": 54.17, - "bfcl/bfcl.overall.latency_std_s": 79.9, - "bfcl/bfcl.overall.latency_p95_s": 194.15, - "bfcl/bfcl.non_live.ast_accuracy": 88.56, - "bfcl/bfcl.non_live.simple_ast_accuracy": 75.25, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 95.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 94.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 89.5, - "bfcl/bfcl.live.live_accuracy": 80.09, - "bfcl/bfcl.live.live_simple_ast_accuracy": 84.5, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 78.92, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 93.75, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 75.0, - "bfcl/bfcl.multi_turn.accuracy": 33.38, - "bfcl/bfcl.multi_turn.base_accuracy": 41.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 38.5, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 27.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 26.5, - "bfcl/bfcl.web_search.accuracy": 13.5, - "bfcl/bfcl.web_search.base_accuracy": 19.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 8.0, - "bfcl/bfcl.memory.accuracy": 13.12, - "bfcl/bfcl.memory.kv_accuracy": 3.87, - "bfcl/bfcl.memory.vector_accuracy": 10.32, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 25.16, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 75.0, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 82.27, - "bfcl/bfcl.format_sensitivity.max_delta": 16.5, - "bfcl/bfcl.format_sensitivity.stddev": 5.09 - } - }, - { - "id": "Qwen/QwQ-32B", - "name": "QwQ-32B", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3977, - "hfopenllm_v2/BBH": 0.2983, - "hfopenllm_v2/MATH Level 5": 0.1609, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.4206, - "hfopenllm_v2/MMLU-PRO": 0.1196 - } - }, - { - "id": "Qwen/QwQ-32B-Preview", - "name": "QwQ-32B-Preview", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4035, - "hfopenllm_v2/BBH": 0.6691, - "hfopenllm_v2/MATH Level 5": 0.4494, - "hfopenllm_v2/GPQA": 0.2819, - "hfopenllm_v2/MUSR": 0.411, - "hfopenllm_v2/MMLU-PRO": 0.5678 - } - }, - { - "id": "Qwen/WorldPM-72B", - "name": "Qwen/WorldPM-72B", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6333, - "reward-bench/Factuality": 0.7074, - "reward-bench/Precise IF": 0.3125, - "reward-bench/Math": 0.6557, - "reward-bench/Safety": 0.8533, - "reward-bench/Focus": 0.9172, - "reward-bench/Ties": 0.3535 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/r-i-s-e.json b/data/developers/r-i-s-e.json deleted file mode 100644 index 05d6bd52d17e381ac81a1c18a0dba0bf121dcda2..0000000000000000000000000000000000000000 --- a/data/developers/r-i-s-e.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "developer": "R-I-S-E", - "models": [ - { - "id": "R-I-S-E/RISE-Judge-Qwen2.5-32B", - "name": "R-I-S-E/RISE-Judge-Qwen2.5-32B", - "developer": "R-I-S-E", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.9266, - "reward-bench/Chat": 0.9665, - "reward-bench/Chat Hard": 0.8333, - "reward-bench/Safety": 0.9189, - "reward-bench/Reasoning": 0.9877 - } - }, - { - "id": "R-I-S-E/RISE-Judge-Qwen2.5-7B", - "name": "R-I-S-E/RISE-Judge-Qwen2.5-7B", - "developer": "R-I-S-E", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.8819, - "reward-bench/Chat": 0.9218, - "reward-bench/Chat Hard": 0.7654, - "reward-bench/Safety": 0.8797, - "reward-bench/Reasoning": 0.9608 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/rakuten.json b/data/developers/rakuten.json deleted file mode 100644 index 877ffb5c05690ede4d79770a9f14e03e050382e5..0000000000000000000000000000000000000000 --- a/data/developers/rakuten.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "developer": "Rakuten", - "models": [ - { - "id": "Rakuten/RakutenAI-2.0-mini-instruct", - "name": "RakutenAI-2.0-mini-instruct", - "developer": "Rakuten", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6794, - "hfopenllm_v2/BBH": 0.2867, - "hfopenllm_v2/MATH Level 5": 0.0521, - "hfopenllm_v2/GPQA": 0.2668, - "hfopenllm_v2/MUSR": 0.3249, - "hfopenllm_v2/MMLU-PRO": 0.1118 - } - }, - { - "id": "Rakuten/RakutenAI-7B", - "name": "RakutenAI-7B", - "developer": "Rakuten", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1556, - "hfopenllm_v2/BBH": 0.4315, - "hfopenllm_v2/MATH Level 5": 0.0196, - "hfopenllm_v2/GPQA": 0.2894, - "hfopenllm_v2/MUSR": 0.3738, - "hfopenllm_v2/MMLU-PRO": 0.2877 - } - }, - { - "id": "Rakuten/RakutenAI-7B-chat", - "name": "RakutenAI-7B-chat", - "developer": "Rakuten", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2686, - "hfopenllm_v2/BBH": 0.4316, - "hfopenllm_v2/MATH Level 5": 0.0295, - "hfopenllm_v2/GPQA": 0.2567, - "hfopenllm_v2/MUSR": 0.379, - "hfopenllm_v2/MMLU-PRO": 0.2798 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/raphgg.json b/data/developers/raphgg.json deleted file mode 100644 index e0cca6a377c18f8c30320729487a6c71178a4345..0000000000000000000000000000000000000000 --- a/data/developers/raphgg.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "raphgg", - "models": [ - { - "id": "raphgg/test-2.5-72B", - "name": "test-2.5-72B", - "developer": "raphgg", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8437, - "hfopenllm_v2/BBH": 0.7266, - "hfopenllm_v2/MATH Level 5": 0.4109, - "hfopenllm_v2/GPQA": 0.3893, - "hfopenllm_v2/MUSR": 0.4812, - "hfopenllm_v2/MMLU-PRO": 0.5837 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/rasyosef.json b/data/developers/rasyosef.json deleted file mode 100644 index a04dad9cab18f27426a8f9116460793f6d035061..0000000000000000000000000000000000000000 --- a/data/developers/rasyosef.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "developer": "rasyosef", - "models": [ - { - "id": "rasyosef/Mistral-NeMo-Minitron-8B-Chat", - "name": "Mistral-NeMo-Minitron-8B-Chat", - "developer": "rasyosef", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4452, - "hfopenllm_v2/BBH": 0.4759, - "hfopenllm_v2/MATH Level 5": 0.0272, - "hfopenllm_v2/GPQA": 0.276, - "hfopenllm_v2/MUSR": 0.4304, - "hfopenllm_v2/MMLU-PRO": 0.2404 - } - }, - { - "id": "rasyosef/Phi-1_5-Instruct-v0.1", - "name": "Phi-1_5-Instruct-v0.1", - "developer": "rasyosef", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2402, - "hfopenllm_v2/BBH": 0.3118, - "hfopenllm_v2/MATH Level 5": 0.0136, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3422, - "hfopenllm_v2/MMLU-PRO": 0.1562 - } - }, - { - "id": "rasyosef/phi-2-instruct-apo", - "name": "phi-2-instruct-apo", - "developer": "rasyosef", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3146, - "hfopenllm_v2/BBH": 0.4445, - "hfopenllm_v2/MATH Level 5": 0.0302, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.3342, - "hfopenllm_v2/MMLU-PRO": 0.2155 - } - }, - { - "id": "rasyosef/phi-2-instruct-v0.1", - "name": "phi-2-instruct-v0.1", - "developer": "rasyosef", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3681, - "hfopenllm_v2/BBH": 0.4726, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2743, - "hfopenllm_v2/MUSR": 0.3524, - "hfopenllm_v2/MMLU-PRO": 0.2247 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/ray2333.json b/data/developers/ray2333.json deleted file mode 100644 index 4d9b39e75f5711f2e8d73a3686c6b1195057b998..0000000000000000000000000000000000000000 --- a/data/developers/ray2333.json +++ /dev/null @@ -1,160 +0,0 @@ -{ - "developer": "Ray2333", - "models": [ - { - "id": "Ray2333/Gemma-2B-rewardmodel-baseline", - "name": "Ray2333/Gemma-2B-rewardmodel-baseline", - "developer": "Ray2333", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.729, - "reward-bench/Chat": 0.9413, - "reward-bench/Chat Hard": 0.4693, - "reward-bench/Safety": 0.7865, - "reward-bench/Reasoning": 0.7384, - "reward-bench/Prior Sets (0.5 weight)": 0.6897 - } - }, - { - "id": "Ray2333/Gemma-2B-rewardmodel-ft", - "name": "Ray2333/Gemma-2B-rewardmodel-ft", - "developer": "Ray2333", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.8048, - "reward-bench/Chat": 0.7793, - "reward-bench/Chat Hard": 0.7478, - "reward-bench/Safety": 0.8527, - "reward-bench/Reasoning": 0.8393 - } - }, - { - "id": "Ray2333/GRM-Gemma-2B-rewardmodel-ft", - "name": "Ray2333/GRM-Gemma-2B-rewardmodel-ft", - "developer": "Ray2333", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.8447, - "reward-bench/Chat": 0.8939, - "reward-bench/Chat Hard": 0.7522, - "reward-bench/Safety": 0.8446, - "reward-bench/Reasoning": 0.8881 - } - }, - { - "id": "Ray2333/GRM-Gemma-2B-sftreg", - "name": "Ray2333/GRM-Gemma-2B-sftreg", - "developer": "Ray2333", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7451, - "reward-bench/Chat": 0.9553, - "reward-bench/Chat Hard": 0.4868, - "reward-bench/Safety": 0.7932, - "reward-bench/Reasoning": 0.7684, - "reward-bench/Prior Sets (0.5 weight)": 0.6983 - } - }, - { - "id": "Ray2333/GRM-gemma2-2B-rewardmodel-ft", - "name": "Ray2333/GRM-gemma2-2B-rewardmodel-ft", - "developer": "Ray2333", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5966, - "reward-bench/Chat": 0.9302, - "reward-bench/Chat Hard": 0.7719, - "reward-bench/Safety": 0.9222, - "reward-bench/Reasoning": 0.912, - "reward-bench/Factuality": 0.5305, - "reward-bench/Precise IF": 0.3125, - "reward-bench/Math": 0.5902, - "reward-bench/Focus": 0.7455, - "reward-bench/Ties": 0.4788 - } - }, - { - "id": "Ray2333/GRM-llama3-8B-distill", - "name": "Ray2333/GRM-llama3-8B-distill", - "developer": "Ray2333", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.589, - "reward-bench/Chat": 0.9832, - "reward-bench/Chat Hard": 0.6842, - "reward-bench/Safety": 0.7222, - "reward-bench/Reasoning": 0.9133, - "reward-bench/Prior Sets (0.5 weight)": 0.7209, - "reward-bench/Factuality": 0.5874, - "reward-bench/Precise IF": 0.3875, - "reward-bench/Math": 0.5902, - "reward-bench/Focus": 0.6727, - "reward-bench/Ties": 0.5743 - } - }, - { - "id": "Ray2333/GRM-Llama3-8B-rewardmodel-ft", - "name": "Ray2333/GRM-Llama3-8B-rewardmodel-ft", - "developer": "Ray2333", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6766, - "reward-bench/Chat": 0.9553, - "reward-bench/Chat Hard": 0.8618, - "reward-bench/Safety": 0.9222, - "reward-bench/Reasoning": 0.9362, - "reward-bench/Factuality": 0.6274, - "reward-bench/Precise IF": 0.35, - "reward-bench/Math": 0.5847, - "reward-bench/Focus": 0.8929, - "reward-bench/Ties": 0.6824 - } - }, - { - "id": "Ray2333/GRM-llama3-8B-sftreg", - "name": "Ray2333/GRM-llama3-8B-sftreg", - "developer": "Ray2333", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6089, - "reward-bench/Chat": 0.986, - "reward-bench/Chat Hard": 0.6776, - "reward-bench/Safety": 0.7867, - "reward-bench/Reasoning": 0.9229, - "reward-bench/Prior Sets (0.5 weight)": 0.7309, - "reward-bench/Factuality": 0.6189, - "reward-bench/Precise IF": 0.3875, - "reward-bench/Math": 0.5792, - "reward-bench/Focus": 0.6828, - "reward-bench/Ties": 0.5981 - } - }, - { - "id": "Ray2333/GRM-llama3.2-3B-rewardmodel-ft", - "name": "Ray2333/GRM-llama3.2-3B-rewardmodel-ft", - "developer": "Ray2333", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.9092, - "reward-bench/Chat": 0.9162, - "reward-bench/Chat Hard": 0.8487, - "reward-bench/Safety": 0.927, - "reward-bench/Reasoning": 0.945 - } - }, - { - "id": "Ray2333/reward-model-Mistral-7B-instruct-Unifie...", - "name": "Ray2333/reward-model-Mistral-7B-instruct-Unifie...", - "developer": "Ray2333", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7661, - "reward-bench/Chat": 0.9777, - "reward-bench/Chat Hard": 0.5066, - "reward-bench/Safety": 0.8527, - "reward-bench/Reasoning": 0.7389, - "reward-bench/Prior Sets (0.5 weight)": 0.7434 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/rdson.json b/data/developers/rdson.json deleted file mode 100644 index 36f1051289198450d90b1967f86a100056cc2aa6..0000000000000000000000000000000000000000 --- a/data/developers/rdson.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "RDson", - "models": [ - { - "id": "RDson/WomboCombo-R1-Coder-14B-Preview", - "name": "WomboCombo-R1-Coder-14B-Preview", - "developer": "RDson", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6286, - "hfopenllm_v2/BBH": 0.6392, - "hfopenllm_v2/MATH Level 5": 0.5989, - "hfopenllm_v2/GPQA": 0.3213, - "hfopenllm_v2/MUSR": 0.4844, - "hfopenllm_v2/MMLU-PRO": 0.5168 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/realtreetune.json b/data/developers/realtreetune.json deleted file mode 100644 index 9db7762c41305893f8f3af8c0a1731c6478daa5c..0000000000000000000000000000000000000000 --- a/data/developers/realtreetune.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "realtreetune", - "models": [ - { - "id": "realtreetune/rho-1b-sft-MATH", - "name": "rho-1b-sft-MATH", - "developer": "realtreetune", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2121, - "hfopenllm_v2/BBH": 0.3144, - "hfopenllm_v2/MATH Level 5": 0.0347, - "hfopenllm_v2/GPQA": 0.2525, - "hfopenllm_v2/MUSR": 0.3458, - "hfopenllm_v2/MMLU-PRO": 0.1117 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/recoilme.json b/data/developers/recoilme.json deleted file mode 100644 index a964ddb489dc6886290c7f2b96295a41f9f7bf6b..0000000000000000000000000000000000000000 --- a/data/developers/recoilme.json +++ /dev/null @@ -1,89 +0,0 @@ -{ - "developer": "recoilme", - "models": [ - { - "id": "recoilme/Gemma-2-Ataraxy-Gemmasutra-9B-slerp", - "name": "Gemma-2-Ataraxy-Gemmasutra-9B-slerp", - "developer": "recoilme", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7649, - "hfopenllm_v2/BBH": 0.5974, - "hfopenllm_v2/MATH Level 5": 0.0174, - "hfopenllm_v2/GPQA": 0.3305, - "hfopenllm_v2/MUSR": 0.4245, - "hfopenllm_v2/MMLU-PRO": 0.4207 - } - }, - { - "id": "recoilme/recoilme-gemma-2-9B-v0.1", - "name": "recoilme-gemma-2-9B-v0.1", - "developer": "recoilme", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7515, - "hfopenllm_v2/BBH": 0.5995, - "hfopenllm_v2/MATH Level 5": 0.2039, - "hfopenllm_v2/GPQA": 0.3389, - "hfopenllm_v2/MUSR": 0.4191, - "hfopenllm_v2/MMLU-PRO": 0.4159 - } - }, - { - "id": "recoilme/recoilme-gemma-2-9B-v0.2", - "name": "recoilme-gemma-2-9B-v0.2", - "developer": "recoilme", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2747, - "hfopenllm_v2/BBH": 0.6031, - "hfopenllm_v2/MATH Level 5": 0.0831, - "hfopenllm_v2/GPQA": 0.3305, - "hfopenllm_v2/MUSR": 0.4686, - "hfopenllm_v2/MMLU-PRO": 0.4122 - } - }, - { - "id": "recoilme/recoilme-gemma-2-9B-v0.3", - "name": "recoilme-gemma-2-9B-v0.3", - "developer": "recoilme", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5761, - "hfopenllm_v2/BBH": 0.602, - "hfopenllm_v2/MATH Level 5": 0.1888, - "hfopenllm_v2/GPQA": 0.3372, - "hfopenllm_v2/MUSR": 0.4632, - "hfopenllm_v2/MMLU-PRO": 0.4039 - } - }, - { - "id": "recoilme/recoilme-gemma-2-9B-v0.4", - "name": "recoilme-gemma-2-9B-v0.4", - "developer": "recoilme", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2562, - "hfopenllm_v2/BBH": 0.5967, - "hfopenllm_v2/MATH Level 5": 0.0846, - "hfopenllm_v2/GPQA": 0.3406, - "hfopenllm_v2/MUSR": 0.4727, - "hfopenllm_v2/MMLU-PRO": 0.4406 - } - }, - { - "id": "recoilme/recoilme-gemma-2-9B-v0.5", - "name": "recoilme-gemma-2-9B-v0.5", - "developer": "recoilme", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7664, - "hfopenllm_v2/BBH": 0.5981, - "hfopenllm_v2/MATH Level 5": 0.2115, - "hfopenllm_v2/GPQA": 0.3364, - "hfopenllm_v2/MUSR": 0.4232, - "hfopenllm_v2/MMLU-PRO": 0.42 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/redrix.json b/data/developers/redrix.json deleted file mode 100644 index 9dc4b1ebd40cd7d1a6293364f9deaf001b148cd4..0000000000000000000000000000000000000000 --- a/data/developers/redrix.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "redrix", - "models": [ - { - "id": "redrix/AngelSlayer-12B-Unslop-Mell-RPMax-DARKNESS", - "name": "AngelSlayer-12B-Unslop-Mell-RPMax-DARKNESS", - "developer": "redrix", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.536, - "hfopenllm_v2/BBH": 0.5129, - "hfopenllm_v2/MATH Level 5": 0.1133, - "hfopenllm_v2/GPQA": 0.3154, - "hfopenllm_v2/MUSR": 0.3818, - "hfopenllm_v2/MMLU-PRO": 0.318 - } - }, - { - "id": "redrix/patricide-12B-Unslop-Mell", - "name": "patricide-12B-Unslop-Mell", - "developer": "redrix", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4074, - "hfopenllm_v2/BBH": 0.5399, - "hfopenllm_v2/MATH Level 5": 0.1314, - "hfopenllm_v2/GPQA": 0.3238, - "hfopenllm_v2/MUSR": 0.4026, - "hfopenllm_v2/MMLU-PRO": 0.357 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/refuelai.json b/data/developers/refuelai.json deleted file mode 100644 index ebeaea95c700a74e10fbc9648a1d22c0b467313c..0000000000000000000000000000000000000000 --- a/data/developers/refuelai.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "refuelai", - "models": [ - { - "id": "refuelai/Llama-3-Refueled", - "name": "Llama-3-Refueled", - "developer": "refuelai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.462, - "hfopenllm_v2/BBH": 0.5871, - "hfopenllm_v2/MATH Level 5": 0.0665, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.4454, - "hfopenllm_v2/MMLU-PRO": 0.3095 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/replete-ai.json b/data/developers/replete-ai.json deleted file mode 100644 index dbb06f00736a7fcddf76b24a5c7673e098ecab57..0000000000000000000000000000000000000000 --- a/data/developers/replete-ai.json +++ /dev/null @@ -1,131 +0,0 @@ -{ - "developer": "Replete-AI", - "models": [ - { - "id": "Replete-AI/L3-Pneuma-8B", - "name": "L3-Pneuma-8B", - "developer": "Replete-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2413, - "hfopenllm_v2/BBH": 0.4909, - "hfopenllm_v2/MATH Level 5": 0.0544, - "hfopenllm_v2/GPQA": 0.318, - "hfopenllm_v2/MUSR": 0.4105, - "hfopenllm_v2/MMLU-PRO": 0.3176 - } - }, - { - "id": "Replete-AI/L3.1-Pneuma-8B", - "name": "L3.1-Pneuma-8B", - "developer": "Replete-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7076, - "hfopenllm_v2/BBH": 0.505, - "hfopenllm_v2/MATH Level 5": 0.2198, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.3871, - "hfopenllm_v2/MMLU-PRO": 0.3691 - } - }, - { - "id": "Replete-AI/Llama3-8B-Instruct-Replete-Adapted", - "name": "Llama3-8B-Instruct-Replete-Adapted", - "developer": "Replete-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6915, - "hfopenllm_v2/BBH": 0.487, - "hfopenllm_v2/MATH Level 5": 0.071, - "hfopenllm_v2/GPQA": 0.281, - "hfopenllm_v2/MUSR": 0.3634, - "hfopenllm_v2/MMLU-PRO": 0.3391 - } - }, - { - "id": "Replete-AI/Replete-Coder-Instruct-8b-Merged", - "name": "Replete-Coder-Instruct-8b-Merged", - "developer": "Replete-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5388, - "hfopenllm_v2/BBH": 0.4462, - "hfopenllm_v2/MATH Level 5": 0.0778, - "hfopenllm_v2/GPQA": 0.2693, - "hfopenllm_v2/MUSR": 0.366, - "hfopenllm_v2/MMLU-PRO": 0.1805 - } - }, - { - "id": "Replete-AI/Replete-Coder-Llama3-8B", - "name": "Replete-Coder-Llama3-8B", - "developer": "Replete-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4729, - "hfopenllm_v2/BBH": 0.3271, - "hfopenllm_v2/MATH Level 5": 0.0476, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.3953, - "hfopenllm_v2/MMLU-PRO": 0.1331 - } - }, - { - "id": "Replete-AI/Replete-Coder-Qwen2-1.5b", - "name": "Replete-Coder-Qwen2-1.5b", - "developer": "Replete-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3014, - "hfopenllm_v2/BBH": 0.3475, - "hfopenllm_v2/MATH Level 5": 0.0385, - "hfopenllm_v2/GPQA": 0.2685, - "hfopenllm_v2/MUSR": 0.4073, - "hfopenllm_v2/MMLU-PRO": 0.2147 - } - }, - { - "id": "Replete-AI/Replete-LLM-Qwen2-7b", - "name": "Replete-LLM-Qwen2-7b", - "developer": "Replete-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0932, - "hfopenllm_v2/BBH": 0.2977, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2475, - "hfopenllm_v2/MUSR": 0.3941, - "hfopenllm_v2/MMLU-PRO": 0.1157 - } - }, - { - "id": "Replete-AI/Replete-LLM-Qwen2-7b_Beta-Preview", - "name": "Replete-LLM-Qwen2-7b_Beta-Preview", - "developer": "Replete-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0858, - "hfopenllm_v2/BBH": 0.2929, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2483, - "hfopenllm_v2/MUSR": 0.3981, - "hfopenllm_v2/MMLU-PRO": 0.1285 - } - }, - { - "id": "Replete-AI/Replete-LLM-V2-Llama-3.1-8b", - "name": "Replete-LLM-V2-Llama-3.1-8b", - "developer": "Replete-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5515, - "hfopenllm_v2/BBH": 0.5339, - "hfopenllm_v2/MATH Level 5": 0.1405, - "hfopenllm_v2/GPQA": 0.3138, - "hfopenllm_v2/MUSR": 0.4001, - "hfopenllm_v2/MMLU-PRO": 0.3753 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/resmpdev.json b/data/developers/resmpdev.json deleted file mode 100644 index 272e604c50494381f3fdb725c611a8e0bb67c4fb..0000000000000000000000000000000000000000 --- a/data/developers/resmpdev.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "RESMPDEV", - "models": [ - { - "id": "RESMPDEV/EVA-Qwen2.5-1.5B-FRFR", - "name": "EVA-Qwen2.5-1.5B-FRFR", - "developer": "RESMPDEV", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3082, - "hfopenllm_v2/BBH": 0.3932, - "hfopenllm_v2/MATH Level 5": 0.1027, - "hfopenllm_v2/GPQA": 0.2794, - "hfopenllm_v2/MUSR": 0.3539, - "hfopenllm_v2/MMLU-PRO": 0.277 - } - }, - { - "id": "RESMPDEV/Qwen2-Wukong-0.5B", - "name": "Qwen2-Wukong-0.5B", - "developer": "RESMPDEV", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1854, - "hfopenllm_v2/BBH": 0.3085, - "hfopenllm_v2/MATH Level 5": 0.0015, - "hfopenllm_v2/GPQA": 0.2366, - "hfopenllm_v2/MUSR": 0.3525, - "hfopenllm_v2/MMLU-PRO": 0.1327 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/rezvortex.json b/data/developers/rezvortex.json deleted file mode 100644 index 2bba05e881305c661e7b0e1bebd4cc8ad3daf762..0000000000000000000000000000000000000000 --- a/data/developers/rezvortex.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "RezVortex", - "models": [ - { - "id": "RezVortex/Jajuka-3b", - "name": "Jajuka-3b", - "developer": "RezVortex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6925, - "hfopenllm_v2/BBH": 0.4594, - "hfopenllm_v2/MATH Level 5": 0.1594, - "hfopenllm_v2/GPQA": 0.2659, - "hfopenllm_v2/MUSR": 0.3671, - "hfopenllm_v2/MMLU-PRO": 0.3137 - } - }, - { - "id": "RezVortex/JAJUKA-WEWILLNEVERFORGETYOU-3B", - "name": "JAJUKA-WEWILLNEVERFORGETYOU-3B", - "developer": "RezVortex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6858, - "hfopenllm_v2/BBH": 0.4619, - "hfopenllm_v2/MATH Level 5": 0.1548, - "hfopenllm_v2/GPQA": 0.2576, - "hfopenllm_v2/MUSR": 0.363, - "hfopenllm_v2/MMLU-PRO": 0.3143 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/rhplus0831.json b/data/developers/rhplus0831.json deleted file mode 100644 index 3ec2e804f5d7f0cbcdbc1f85b0968eeaaa6c7709..0000000000000000000000000000000000000000 --- a/data/developers/rhplus0831.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "rhplus0831", - "models": [ - { - "id": "rhplus0831/maid-yuzu-v7", - "name": "maid-yuzu-v7", - "developer": "rhplus0831", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6462, - "hfopenllm_v2/BBH": 0.4805, - "hfopenllm_v2/MATH Level 5": 0.102, - "hfopenllm_v2/GPQA": 0.3096, - "hfopenllm_v2/MUSR": 0.4136, - "hfopenllm_v2/MMLU-PRO": 0.354 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/rhymes-ai.json b/data/developers/rhymes-ai.json deleted file mode 100644 index eaa17557e6f865ccf6660d0f4b2846647fdc7ddc..0000000000000000000000000000000000000000 --- a/data/developers/rhymes-ai.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "rhymes-ai", - "models": [ - { - "id": "rhymes-ai/Aria", - "name": "Aria", - "developer": "rhymes-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4773, - "hfopenllm_v2/BBH": 0.5695, - "hfopenllm_v2/MATH Level 5": 0.1934, - "hfopenllm_v2/GPQA": 0.3624, - "hfopenllm_v2/MUSR": 0.4338, - "hfopenllm_v2/MMLU-PRO": 0.4405 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/rhysjones.json b/data/developers/rhysjones.json deleted file mode 100644 index fae66ec44b4b65528290d286c26a405e0c49ba6e..0000000000000000000000000000000000000000 --- a/data/developers/rhysjones.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "rhysjones", - "models": [ - { - "id": "rhysjones/phi-2-orange-v2", - "name": "phi-2-orange-v2", - "developer": "rhysjones", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.367, - "hfopenllm_v2/BBH": 0.477, - "hfopenllm_v2/MATH Level 5": 0.0408, - "hfopenllm_v2/GPQA": 0.2617, - "hfopenllm_v2/MUSR": 0.363, - "hfopenllm_v2/MMLU-PRO": 0.2532 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/riaz.json b/data/developers/riaz.json deleted file mode 100644 index 342d5654379425c9866753c909eb0413c18e5c77..0000000000000000000000000000000000000000 --- a/data/developers/riaz.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "riaz", - "models": [ - { - "id": "riaz/FineLlama-3.1-8B", - "name": "FineLlama-3.1-8B", - "developer": "riaz", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4137, - "hfopenllm_v2/BBH": 0.4565, - "hfopenllm_v2/MATH Level 5": 0.0453, - "hfopenllm_v2/GPQA": 0.276, - "hfopenllm_v2/MUSR": 0.3776, - "hfopenllm_v2/MMLU-PRO": 0.2978 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/rlhflow.json b/data/developers/rlhflow.json deleted file mode 100644 index ded4687b9b57367ba736fc197812722b45b282e2..0000000000000000000000000000000000000000 --- a/data/developers/rlhflow.json +++ /dev/null @@ -1,78 +0,0 @@ -{ - "developer": "RLHFlow", - "models": [ - { - "id": "RLHFlow/ArmoRM-Llama3-8B-v0.1", - "name": "RLHFlow/ArmoRM-Llama3-8B-v0.1", - "developer": "RLHFlow", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1897, - "hfopenllm_v2/BBH": 0.2876, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2492, - "hfopenllm_v2/MUSR": 0.3948, - "hfopenllm_v2/MMLU-PRO": 0.1078, - "reward-bench/Score": 0.886, - "reward-bench/Factuality": 0.6568, - "reward-bench/Precise IF": 0.4188, - "reward-bench/Math": 0.6612, - "reward-bench/Safety": 0.9054, - "reward-bench/Focus": 0.7657, - "reward-bench/Ties": 0.6629, - "reward-bench/Chat": 0.9693, - "reward-bench/Chat Hard": 0.7675, - "reward-bench/Reasoning": 0.9735, - "reward-bench/Prior Sets (0.5 weight)": 0.7429 - } - }, - { - "id": "RLHFlow/LLaMA3-iterative-DPO-final", - "name": "RLHFlow/LLaMA3-iterative-DPO-final", - "developer": "RLHFlow", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.534, - "hfopenllm_v2/BBH": 0.5058, - "hfopenllm_v2/MATH Level 5": 0.0884, - "hfopenllm_v2/GPQA": 0.2836, - "hfopenllm_v2/MUSR": 0.3673, - "hfopenllm_v2/MMLU-PRO": 0.3257, - "reward-bench/Score": 0.6783, - "reward-bench/Chat": 0.838, - "reward-bench/Chat Hard": 0.5921, - "reward-bench/Safety": 0.7865, - "reward-bench/Reasoning": 0.6161, - "reward-bench/Prior Sets (0.5 weight)": 0.4392 - } - }, - { - "id": "RLHFlow/pair-preference-model-LLaMA3-8B", - "name": "RLHFlow/pair-preference-model-LLaMA3-8B", - "developer": "RLHFlow", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.8575, - "reward-bench/Chat": 0.9832, - "reward-bench/Chat Hard": 0.6579, - "reward-bench/Safety": 0.8973, - "reward-bench/Reasoning": 0.9473, - "reward-bench/Prior Sets (0.5 weight)": 0.7458 - } - }, - { - "id": "RLHFlow/RewardModel-Mistral-7B-for-DPA-v1", - "name": "RLHFlow/RewardModel-Mistral-7B-for-DPA-v1", - "developer": "RLHFlow", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6633, - "reward-bench/Chat": 0.8799, - "reward-bench/Chat Hard": 0.4978, - "reward-bench/Safety": 0.7068, - "reward-bench/Reasoning": 0.5971, - "reward-bench/Prior Sets (0.5 weight)": 0.6068 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/rmdhirr.json b/data/developers/rmdhirr.json deleted file mode 100644 index 509e60dd2a8e7570dc1b1d5079abbc23cd489a5a..0000000000000000000000000000000000000000 --- a/data/developers/rmdhirr.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "rmdhirr", - "models": [ - { - "id": "rmdhirr/Gluon-8B", - "name": "Gluon-8B", - "developer": "rmdhirr", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5053, - "hfopenllm_v2/BBH": 0.5153, - "hfopenllm_v2/MATH Level 5": 0.1443, - "hfopenllm_v2/GPQA": 0.3121, - "hfopenllm_v2/MUSR": 0.4039, - "hfopenllm_v2/MMLU-PRO": 0.3808 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/ro-xe.json b/data/developers/ro-xe.json deleted file mode 100644 index 1888889d0865258e380c0aea95a46b43e7022139..0000000000000000000000000000000000000000 --- a/data/developers/ro-xe.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "developer": "Ro-xe", - "models": [ - { - "id": "Ro-xe/FMixIA-7B-DARE-0", - "name": "FMixIA-7B-DARE-0", - "developer": "Ro-xe", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3341, - "hfopenllm_v2/BBH": 0.5035, - "hfopenllm_v2/MATH Level 5": 0.0529, - "hfopenllm_v2/GPQA": 0.2894, - "hfopenllm_v2/MUSR": 0.4545, - "hfopenllm_v2/MMLU-PRO": 0.3016 - } - }, - { - "id": "Ro-xe/FMixIA-7B-SLERP-27", - "name": "FMixIA-7B-SLERP-27", - "developer": "Ro-xe", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3765, - "hfopenllm_v2/BBH": 0.5151, - "hfopenllm_v2/MATH Level 5": 0.0634, - "hfopenllm_v2/GPQA": 0.2953, - "hfopenllm_v2/MUSR": 0.4412, - "hfopenllm_v2/MMLU-PRO": 0.3008 - } - }, - { - "id": "Ro-xe/FMixIA-7B-TIES-1", - "name": "FMixIA-7B-TIES-1", - "developer": "Ro-xe", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3453, - "hfopenllm_v2/BBH": 0.5092, - "hfopenllm_v2/MATH Level 5": 0.0566, - "hfopenllm_v2/GPQA": 0.2886, - "hfopenllm_v2/MUSR": 0.4689, - "hfopenllm_v2/MMLU-PRO": 0.2992 - } - }, - { - "id": "Ro-xe/FMixIA-FrankenMerge-9.5B-PT-9", - "name": "FMixIA-FrankenMerge-9.5B-PT-9", - "developer": "Ro-xe", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.194, - "hfopenllm_v2/BBH": 0.5088, - "hfopenllm_v2/MATH Level 5": 0.003, - "hfopenllm_v2/GPQA": 0.3079, - "hfopenllm_v2/MUSR": 0.417, - "hfopenllm_v2/MMLU-PRO": 0.3657 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/rombo-org.json b/data/developers/rombo-org.json deleted file mode 100644 index b79d7a7f39a0dea35feab8529f6276806b3465d6..0000000000000000000000000000000000000000 --- a/data/developers/rombo-org.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "Rombo-Org", - "models": [ - { - "id": "Rombo-Org/Rombo-LLM-V2.5-Qwen-7b", - "name": "Rombo-LLM-V2.5-Qwen-7b", - "developer": "Rombo-Org", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7482, - "hfopenllm_v2/BBH": 0.54, - "hfopenllm_v2/MATH Level 5": 0.5068, - "hfopenllm_v2/GPQA": 0.3012, - "hfopenllm_v2/MUSR": 0.398, - "hfopenllm_v2/MMLU-PRO": 0.4283 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/rombodawg.json b/data/developers/rombodawg.json deleted file mode 100644 index e7b347a91d5be7f4c52661e255a50c99caa3a3fd..0000000000000000000000000000000000000000 --- a/data/developers/rombodawg.json +++ /dev/null @@ -1,201 +0,0 @@ -{ - "developer": "rombodawg", - "models": [ - { - "id": "rombodawg/Rombos-Coder-V2.5-Qwen-14b", - "name": "Rombos-Coder-V2.5-Qwen-14b", - "developer": "rombodawg", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7047, - "hfopenllm_v2/BBH": 0.6165, - "hfopenllm_v2/MATH Level 5": 0.3301, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.3915, - "hfopenllm_v2/MMLU-PRO": 0.3939 - } - }, - { - "id": "rombodawg/Rombos-Coder-V2.5-Qwen-7b", - "name": "Rombos-Coder-V2.5-Qwen-7b", - "developer": "rombodawg", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.621, - "hfopenllm_v2/BBH": 0.5077, - "hfopenllm_v2/MATH Level 5": 0.3338, - "hfopenllm_v2/GPQA": 0.2836, - "hfopenllm_v2/MUSR": 0.3979, - "hfopenllm_v2/MMLU-PRO": 0.3398 - } - }, - { - "id": "rombodawg/Rombos-LLM-V2.5-Qwen-0.5b", - "name": "Rombos-LLM-V2.5-Qwen-0.5b", - "developer": "rombodawg", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2847, - "hfopenllm_v2/BBH": 0.3294, - "hfopenllm_v2/MATH Level 5": 0.068, - "hfopenllm_v2/GPQA": 0.2668, - "hfopenllm_v2/MUSR": 0.3236, - "hfopenllm_v2/MMLU-PRO": 0.1866 - } - }, - { - "id": "rombodawg/Rombos-LLM-V2.5-Qwen-1.5b", - "name": "Rombos-LLM-V2.5-Qwen-1.5b", - "developer": "rombodawg", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3402, - "hfopenllm_v2/BBH": 0.4257, - "hfopenllm_v2/MATH Level 5": 0.0853, - "hfopenllm_v2/GPQA": 0.2886, - "hfopenllm_v2/MUSR": 0.4186, - "hfopenllm_v2/MMLU-PRO": 0.2922 - } - }, - { - "id": "rombodawg/Rombos-LLM-V2.5-Qwen-14b", - "name": "Rombos-LLM-V2.5-Qwen-14b", - "developer": "rombodawg", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.584, - "hfopenllm_v2/BBH": 0.6481, - "hfopenllm_v2/MATH Level 5": 0.4554, - "hfopenllm_v2/GPQA": 0.3716, - "hfopenllm_v2/MUSR": 0.4717, - "hfopenllm_v2/MMLU-PRO": 0.5376 - } - }, - { - "id": "rombodawg/Rombos-LLM-V2.5-Qwen-32b", - "name": "Rombos-LLM-V2.5-Qwen-32b", - "developer": "rombodawg", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6827, - "hfopenllm_v2/BBH": 0.7046, - "hfopenllm_v2/MATH Level 5": 0.4955, - "hfopenllm_v2/GPQA": 0.3968, - "hfopenllm_v2/MUSR": 0.5034, - "hfopenllm_v2/MMLU-PRO": 0.5916 - } - }, - { - "id": "rombodawg/Rombos-LLM-V2.5-Qwen-3b", - "name": "Rombos-LLM-V2.5-Qwen-3b", - "developer": "rombodawg", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5342, - "hfopenllm_v2/BBH": 0.4809, - "hfopenllm_v2/MATH Level 5": 0.2795, - "hfopenllm_v2/GPQA": 0.3079, - "hfopenllm_v2/MUSR": 0.4042, - "hfopenllm_v2/MMLU-PRO": 0.3761 - } - }, - { - "id": "rombodawg/Rombos-LLM-V2.5-Qwen-72b", - "name": "Rombos-LLM-V2.5-Qwen-72b", - "developer": "rombodawg", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7155, - "hfopenllm_v2/BBH": 0.723, - "hfopenllm_v2/MATH Level 5": 0.5423, - "hfopenllm_v2/GPQA": 0.3985, - "hfopenllm_v2/MUSR": 0.4599, - "hfopenllm_v2/MMLU-PRO": 0.5935 - } - }, - { - "id": "rombodawg/Rombos-LLM-V2.5-Qwen-7b", - "name": "Rombos-LLM-V2.5-Qwen-7b", - "developer": "rombodawg", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6237, - "hfopenllm_v2/BBH": 0.5544, - "hfopenllm_v2/MATH Level 5": 0.3814, - "hfopenllm_v2/GPQA": 0.318, - "hfopenllm_v2/MUSR": 0.4291, - "hfopenllm_v2/MMLU-PRO": 0.4469 - } - }, - { - "id": "rombodawg/Rombos-LLM-V2.5.1-Qwen-3b", - "name": "Rombos-LLM-V2.5.1-Qwen-3b", - "developer": "rombodawg", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2566, - "hfopenllm_v2/BBH": 0.39, - "hfopenllm_v2/MATH Level 5": 0.1208, - "hfopenllm_v2/GPQA": 0.2626, - "hfopenllm_v2/MUSR": 0.3991, - "hfopenllm_v2/MMLU-PRO": 0.2741 - } - }, - { - "id": "rombodawg/Rombos-LLM-V2.6-Nemotron-70b", - "name": "Rombos-LLM-V2.6-Nemotron-70b", - "developer": "rombodawg", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7527, - "hfopenllm_v2/BBH": 0.6938, - "hfopenllm_v2/MATH Level 5": 0.3331, - "hfopenllm_v2/GPQA": 0.406, - "hfopenllm_v2/MUSR": 0.4669, - "hfopenllm_v2/MMLU-PRO": 0.5329 - } - }, - { - "id": "rombodawg/Rombos-LLM-V2.6-Qwen-14b", - "name": "Rombos-LLM-V2.6-Qwen-14b", - "developer": "rombodawg", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8432, - "hfopenllm_v2/BBH": 0.6442, - "hfopenllm_v2/MATH Level 5": 0.5211, - "hfopenllm_v2/GPQA": 0.3339, - "hfopenllm_v2/MUSR": 0.4221, - "hfopenllm_v2/MMLU-PRO": 0.4961 - } - }, - { - "id": "rombodawg/rombos_Replete-Coder-Instruct-8b-Merged", - "name": "rombos_Replete-Coder-Instruct-8b-Merged", - "developer": "rombodawg", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5388, - "hfopenllm_v2/BBH": 0.4462, - "hfopenllm_v2/MATH Level 5": 0.0778, - "hfopenllm_v2/GPQA": 0.2693, - "hfopenllm_v2/MUSR": 0.366, - "hfopenllm_v2/MMLU-PRO": 0.1809 - } - }, - { - "id": "rombodawg/rombos_Replete-Coder-Llama3-8B", - "name": "rombos_Replete-Coder-Llama3-8B", - "developer": "rombodawg", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4714, - "hfopenllm_v2/BBH": 0.3276, - "hfopenllm_v2/MATH Level 5": 0.0393, - "hfopenllm_v2/GPQA": 0.2668, - "hfopenllm_v2/MUSR": 0.3966, - "hfopenllm_v2/MMLU-PRO": 0.1335 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/rootxhacker.json b/data/developers/rootxhacker.json deleted file mode 100644 index 1710809ec6de34c4865e5abcc14149c25536165b..0000000000000000000000000000000000000000 --- a/data/developers/rootxhacker.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "developer": "rootxhacker", - "models": [ - { - "id": "rootxhacker/Apollo-70B", - "name": "Apollo-70B", - "developer": "rootxhacker", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5099, - "hfopenllm_v2/BBH": 0.6804, - "hfopenllm_v2/MATH Level 5": 0.5612, - "hfopenllm_v2/GPQA": 0.4572, - "hfopenllm_v2/MUSR": 0.4948, - "hfopenllm_v2/MMLU-PRO": 0.5279 - } - }, - { - "id": "rootxhacker/apollo-7B", - "name": "apollo-7B", - "developer": "rootxhacker", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2953, - "hfopenllm_v2/BBH": 0.3636, - "hfopenllm_v2/MATH Level 5": 0.0257, - "hfopenllm_v2/GPQA": 0.2785, - "hfopenllm_v2/MUSR": 0.4131, - "hfopenllm_v2/MMLU-PRO": 0.1748 - } - }, - { - "id": "rootxhacker/Apollo_v2-32B", - "name": "Apollo_v2-32B", - "developer": "rootxhacker", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.428, - "hfopenllm_v2/BBH": 0.7072, - "hfopenllm_v2/MATH Level 5": 0.4275, - "hfopenllm_v2/GPQA": 0.3784, - "hfopenllm_v2/MUSR": 0.4994, - "hfopenllm_v2/MMLU-PRO": 0.5869 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/rsh345.json b/data/developers/rsh345.json deleted file mode 100644 index 4882bfdab00aa1707b38ac6f21aaebfc93e138b7..0000000000000000000000000000000000000000 --- a/data/developers/rsh345.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "rsh345", - "models": [ - { - "id": "rsh345/mistral-ft-optimized-1218-NeuralHermes-2.5-Mistral-7B", - "name": "mistral-ft-optimized-1218-NeuralHermes-2.5-Mistral-7B", - "developer": "rsh345", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3892, - "hfopenllm_v2/BBH": 0.5188, - "hfopenllm_v2/MATH Level 5": 0.0733, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.4672, - "hfopenllm_v2/MMLU-PRO": 0.3054 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/rubenroy.json b/data/developers/rubenroy.json deleted file mode 100644 index ad954ec8ab7d387b8b13601760cb5a1d671314ad..0000000000000000000000000000000000000000 --- a/data/developers/rubenroy.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "developer": "rubenroy", - "models": [ - { - "id": "rubenroy/Geneva-12B-GCv2-5m", - "name": "Geneva-12B-GCv2-5m", - "developer": "rubenroy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2586, - "hfopenllm_v2/BBH": 0.5278, - "hfopenllm_v2/MATH Level 5": 0.0801, - "hfopenllm_v2/GPQA": 0.2878, - "hfopenllm_v2/MUSR": 0.3525, - "hfopenllm_v2/MMLU-PRO": 0.325 - } - }, - { - "id": "rubenroy/Gilgamesh-72B", - "name": "Gilgamesh-72B", - "developer": "rubenroy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8486, - "hfopenllm_v2/BBH": 0.7253, - "hfopenllm_v2/MATH Level 5": 0.4381, - "hfopenllm_v2/GPQA": 0.3943, - "hfopenllm_v2/MUSR": 0.4626, - "hfopenllm_v2/MMLU-PRO": 0.5802 - } - }, - { - "id": "rubenroy/Zurich-14B-GCv2-5m", - "name": "Zurich-14B-GCv2-5m", - "developer": "rubenroy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6164, - "hfopenllm_v2/BBH": 0.6308, - "hfopenllm_v2/MATH Level 5": 0.3074, - "hfopenllm_v2/GPQA": 0.3616, - "hfopenllm_v2/MUSR": 0.4874, - "hfopenllm_v2/MMLU-PRO": 0.5233 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/rubiellabarta.json b/data/developers/rubiellabarta.json deleted file mode 100644 index b07aef707c469238132c77be4ee5e8a60fe2458c..0000000000000000000000000000000000000000 --- a/data/developers/rubiellabarta.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "RubielLabarta", - "models": [ - { - "id": "RubielLabarta/LogoS-7Bx2-MoE-13B-v0.2", - "name": "LogoS-7Bx2-MoE-13B-v0.2", - "developer": "RubielLabarta", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4379, - "hfopenllm_v2/BBH": 0.5207, - "hfopenllm_v2/MATH Level 5": 0.0574, - "hfopenllm_v2/GPQA": 0.2777, - "hfopenllm_v2/MUSR": 0.4226, - "hfopenllm_v2/MMLU-PRO": 0.3088 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/ruizhe1217.json b/data/developers/ruizhe1217.json deleted file mode 100644 index e9a768054ffd872c8248ceed6cd26b3e2250c26f..0000000000000000000000000000000000000000 --- a/data/developers/ruizhe1217.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "ruizhe1217", - "models": [ - { - "id": "ruizhe1217/sft-s1-qwen-0.5b", - "name": "sft-s1-qwen-0.5b", - "developer": "ruizhe1217", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2749, - "hfopenllm_v2/BBH": 0.3301, - "hfopenllm_v2/MATH Level 5": 0.0619, - "hfopenllm_v2/GPQA": 0.271, - "hfopenllm_v2/MUSR": 0.3196, - "hfopenllm_v2/MMLU-PRO": 0.1892 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/rwitz.json b/data/developers/rwitz.json deleted file mode 100644 index 70890afac405774e5d2d9ef4c3d2094f2a227a5c..0000000000000000000000000000000000000000 --- a/data/developers/rwitz.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "rwitz", - "models": [ - { - "id": "rwitz/go-bruins-v2", - "name": "go-bruins-v2", - "developer": "rwitz", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4096, - "hfopenllm_v2/BBH": 0.3799, - "hfopenllm_v2/MATH Level 5": 0.0672, - "hfopenllm_v2/GPQA": 0.2626, - "hfopenllm_v2/MUSR": 0.4138, - "hfopenllm_v2/MMLU-PRO": 0.2761 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/rwkv.json b/data/developers/rwkv.json deleted file mode 100644 index 11a2d290f58b9571e1d2dfdc73fb201adaee5f40..0000000000000000000000000000000000000000 --- a/data/developers/rwkv.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "RWKV", - "models": [ - { - "id": "RWKV/rwkv-raven-14b", - "name": "rwkv-raven-14b", - "developer": "RWKV", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0768, - "hfopenllm_v2/BBH": 0.3307, - "hfopenllm_v2/MATH Level 5": 0.0045, - "hfopenllm_v2/GPQA": 0.229, - "hfopenllm_v2/MUSR": 0.3951, - "hfopenllm_v2/MMLU-PRO": 0.115 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/sabersaleh.json b/data/developers/sabersaleh.json deleted file mode 100644 index 1fd7aa43f325aaa60f4a7b6947e6929b39c987d9..0000000000000000000000000000000000000000 --- a/data/developers/sabersaleh.json +++ /dev/null @@ -1,103 +0,0 @@ -{ - "developer": "sabersaleh", - "models": [ - { - "id": "sabersaleh/Llama2-7B-CPO", - "name": "Llama2-7B-CPO", - "developer": "sabersaleh", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1545, - "hfopenllm_v2/BBH": 0.3458, - "hfopenllm_v2/MATH Level 5": 0.0136, - "hfopenllm_v2/GPQA": 0.2676, - "hfopenllm_v2/MUSR": 0.4048, - "hfopenllm_v2/MMLU-PRO": 0.1606 - } - }, - { - "id": "sabersaleh/Llama2-7B-DPO", - "name": "Llama2-7B-DPO", - "developer": "sabersaleh", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1453, - "hfopenllm_v2/BBH": 0.3512, - "hfopenllm_v2/MATH Level 5": 0.0159, - "hfopenllm_v2/GPQA": 0.2685, - "hfopenllm_v2/MUSR": 0.4114, - "hfopenllm_v2/MMLU-PRO": 0.1626 - } - }, - { - "id": "sabersaleh/Llama2-7B-IPO", - "name": "Llama2-7B-IPO", - "developer": "sabersaleh", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1769, - "hfopenllm_v2/BBH": 0.3475, - "hfopenllm_v2/MATH Level 5": 0.0159, - "hfopenllm_v2/GPQA": 0.2676, - "hfopenllm_v2/MUSR": 0.4048, - "hfopenllm_v2/MMLU-PRO": 0.1617 - } - }, - { - "id": "sabersaleh/Llama2-7B-KTO", - "name": "Llama2-7B-KTO", - "developer": "sabersaleh", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1528, - "hfopenllm_v2/BBH": 0.3501, - "hfopenllm_v2/MATH Level 5": 0.0189, - "hfopenllm_v2/GPQA": 0.2676, - "hfopenllm_v2/MUSR": 0.4167, - "hfopenllm_v2/MMLU-PRO": 0.1636 - } - }, - { - "id": "sabersaleh/Llama2-7B-SimPO", - "name": "Llama2-7B-SimPO", - "developer": "sabersaleh", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1659, - "hfopenllm_v2/BBH": 0.3489, - "hfopenllm_v2/MATH Level 5": 0.0159, - "hfopenllm_v2/GPQA": 0.271, - "hfopenllm_v2/MUSR": 0.4007, - "hfopenllm_v2/MMLU-PRO": 0.1641 - } - }, - { - "id": "sabersaleh/Llama2-7B-SPO", - "name": "Llama2-7B-SPO", - "developer": "sabersaleh", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1567, - "hfopenllm_v2/BBH": 0.3383, - "hfopenllm_v2/MATH Level 5": 0.0196, - "hfopenllm_v2/GPQA": 0.2768, - "hfopenllm_v2/MUSR": 0.3874, - "hfopenllm_v2/MMLU-PRO": 0.1757 - } - }, - { - "id": "sabersaleh/Llama3", - "name": "Llama3", - "developer": "sabersaleh", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3321, - "hfopenllm_v2/BBH": 0.4782, - "hfopenllm_v2/MATH Level 5": 0.0566, - "hfopenllm_v2/GPQA": 0.3104, - "hfopenllm_v2/MUSR": 0.3933, - "hfopenllm_v2/MMLU-PRO": 0.3162 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/sabersalehk.json b/data/developers/sabersalehk.json deleted file mode 100644 index 41e1940ce854636569840b6b1c4642be344c3f39..0000000000000000000000000000000000000000 --- a/data/developers/sabersalehk.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "developer": "sabersalehk", - "models": [ - { - "id": "sabersalehk/Llama3-001-300", - "name": "Llama3-001-300", - "developer": "sabersalehk", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3179, - "hfopenllm_v2/BBH": 0.4745, - "hfopenllm_v2/MATH Level 5": 0.0529, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.4064, - "hfopenllm_v2/MMLU-PRO": 0.3158 - } - }, - { - "id": "sabersalehk/Llama3-SimPO", - "name": "Llama3-SimPO", - "developer": "sabersalehk", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3642, - "hfopenllm_v2/BBH": 0.4874, - "hfopenllm_v2/MATH Level 5": 0.0574, - "hfopenllm_v2/GPQA": 0.3079, - "hfopenllm_v2/MUSR": 0.4046, - "hfopenllm_v2/MMLU-PRO": 0.3157 - } - }, - { - "id": "sabersalehk/Llama3_001_200", - "name": "Llama3_001_200", - "developer": "sabersalehk", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3218, - "hfopenllm_v2/BBH": 0.4728, - "hfopenllm_v2/MATH Level 5": 0.0514, - "hfopenllm_v2/GPQA": 0.3037, - "hfopenllm_v2/MUSR": 0.4037, - "hfopenllm_v2/MMLU-PRO": 0.3183 - } - }, - { - "id": "sabersalehk/Llama3_01_300", - "name": "Llama3_01_300", - "developer": "sabersalehk", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2959, - "hfopenllm_v2/BBH": 0.4691, - "hfopenllm_v2/MATH Level 5": 0.0498, - "hfopenllm_v2/GPQA": 0.3079, - "hfopenllm_v2/MUSR": 0.4065, - "hfopenllm_v2/MMLU-PRO": 0.3124 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/saisexperiments.json b/data/developers/saisexperiments.json deleted file mode 100644 index 802705b381af64db71ac86a6e6fccc0d765cc9d5..0000000000000000000000000000000000000000 --- a/data/developers/saisexperiments.json +++ /dev/null @@ -1,89 +0,0 @@ -{ - "developer": "SaisExperiments", - "models": [ - { - "id": "SaisExperiments/Evil-Alpaca-3B-L3.2", - "name": "Evil-Alpaca-3B-L3.2", - "developer": "SaisExperiments", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3251, - "hfopenllm_v2/BBH": 0.4341, - "hfopenllm_v2/MATH Level 5": 0.0702, - "hfopenllm_v2/GPQA": 0.2634, - "hfopenllm_v2/MUSR": 0.4198, - "hfopenllm_v2/MMLU-PRO": 0.2621 - } - }, - { - "id": "SaisExperiments/Gemma-2-2B-Opus-Instruct", - "name": "Gemma-2-2B-Opus-Instruct", - "developer": "SaisExperiments", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.475, - "hfopenllm_v2/BBH": 0.4293, - "hfopenllm_v2/MATH Level 5": 0.0506, - "hfopenllm_v2/GPQA": 0.2836, - "hfopenllm_v2/MUSR": 0.4057, - "hfopenllm_v2/MMLU-PRO": 0.265 - } - }, - { - "id": "SaisExperiments/Gemma-2-2B-Stheno-Filtered", - "name": "Gemma-2-2B-Stheno-Filtered", - "developer": "SaisExperiments", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4197, - "hfopenllm_v2/BBH": 0.4149, - "hfopenllm_v2/MATH Level 5": 0.0461, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.4003, - "hfopenllm_v2/MMLU-PRO": 0.263 - } - }, - { - "id": "SaisExperiments/Not-So-Small-Alpaca-24B", - "name": "Not-So-Small-Alpaca-24B", - "developer": "SaisExperiments", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6244, - "hfopenllm_v2/BBH": 0.5339, - "hfopenllm_v2/MATH Level 5": 0.1828, - "hfopenllm_v2/GPQA": 0.3591, - "hfopenllm_v2/MUSR": 0.4282, - "hfopenllm_v2/MMLU-PRO": 0.3694 - } - }, - { - "id": "SaisExperiments/QwOwO-7B-V1", - "name": "QwOwO-7B-V1", - "developer": "SaisExperiments", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4556, - "hfopenllm_v2/BBH": 0.5431, - "hfopenllm_v2/MATH Level 5": 0.386, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3835, - "hfopenllm_v2/MMLU-PRO": 0.4224 - } - }, - { - "id": "SaisExperiments/RightSheep-Llama3.2-3B", - "name": "RightSheep-Llama3.2-3B", - "developer": "SaisExperiments", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4156, - "hfopenllm_v2/BBH": 0.4241, - "hfopenllm_v2/MATH Level 5": 0.0808, - "hfopenllm_v2/GPQA": 0.2869, - "hfopenllm_v2/MUSR": 0.3767, - "hfopenllm_v2/MMLU-PRO": 0.254 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/saishf.json b/data/developers/saishf.json deleted file mode 100644 index 213fdc9814d2a3044881c868cc722095c8b5929b..0000000000000000000000000000000000000000 --- a/data/developers/saishf.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "saishf", - "models": [ - { - "id": "saishf/Fimbulvetr-Kuro-Lotus-10.7B", - "name": "Fimbulvetr-Kuro-Lotus-10.7B", - "developer": "saishf", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4939, - "hfopenllm_v2/BBH": 0.4342, - "hfopenllm_v2/MATH Level 5": 0.0536, - "hfopenllm_v2/GPQA": 0.3012, - "hfopenllm_v2/MUSR": 0.4445, - "hfopenllm_v2/MMLU-PRO": 0.3389 - } - }, - { - "id": "saishf/Neural-SOVLish-Devil-8B-L3", - "name": "Neural-SOVLish-Devil-8B-L3", - "developer": "saishf", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4199, - "hfopenllm_v2/BBH": 0.5142, - "hfopenllm_v2/MATH Level 5": 0.0891, - "hfopenllm_v2/GPQA": 0.3079, - "hfopenllm_v2/MUSR": 0.411, - "hfopenllm_v2/MMLU-PRO": 0.3807 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/saishshinde15.json b/data/developers/saishshinde15.json deleted file mode 100644 index 9447e0f64fedf17428662c5533aa3c413d08be90..0000000000000000000000000000000000000000 --- a/data/developers/saishshinde15.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "developer": "saishshinde15", - "models": [ - { - "id": "saishshinde15/TethysAI_Base_Reasoning", - "name": "TethysAI_Base_Reasoning", - "developer": "saishshinde15", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6369, - "hfopenllm_v2/BBH": 0.4519, - "hfopenllm_v2/MATH Level 5": 0.3142, - "hfopenllm_v2/GPQA": 0.2861, - "hfopenllm_v2/MUSR": 0.4075, - "hfopenllm_v2/MMLU-PRO": 0.3236 - } - }, - { - "id": "saishshinde15/TethysAI_Vortex", - "name": "TethysAI_Vortex", - "developer": "saishshinde15", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4298, - "hfopenllm_v2/BBH": 0.4749, - "hfopenllm_v2/MATH Level 5": 0.315, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.4458, - "hfopenllm_v2/MMLU-PRO": 0.3241 - } - }, - { - "id": "saishshinde15/TethysAI_Vortex_Reasoning", - "name": "TethysAI_Vortex_Reasoning", - "developer": "saishshinde15", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4021, - "hfopenllm_v2/BBH": 0.4694, - "hfopenllm_v2/MATH Level 5": 0.2145, - "hfopenllm_v2/GPQA": 0.3045, - "hfopenllm_v2/MUSR": 0.4084, - "hfopenllm_v2/MMLU-PRO": 0.3381 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/sakaltcommunity.json b/data/developers/sakaltcommunity.json deleted file mode 100644 index 2286491f7e40f5bfd7b8b0c7e94e931b1b8ac15f..0000000000000000000000000000000000000000 --- a/data/developers/sakaltcommunity.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "sakaltcommunity", - "models": [ - { - "id": "sakaltcommunity/novablast-preview", - "name": "novablast-preview", - "developer": "sakaltcommunity", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.453, - "hfopenllm_v2/BBH": 0.7043, - "hfopenllm_v2/MATH Level 5": 0.4894, - "hfopenllm_v2/GPQA": 0.3817, - "hfopenllm_v2/MUSR": 0.5021, - "hfopenllm_v2/MMLU-PRO": 0.5915 - } - }, - { - "id": "sakaltcommunity/sakaltum-7b", - "name": "sakaltum-7b", - "developer": "sakaltcommunity", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2604, - "hfopenllm_v2/BBH": 0.4575, - "hfopenllm_v2/MATH Level 5": 0.0295, - "hfopenllm_v2/GPQA": 0.2727, - "hfopenllm_v2/MUSR": 0.3775, - "hfopenllm_v2/MMLU-PRO": 0.2769 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/sakalti.json b/data/developers/sakalti.json deleted file mode 100644 index 0685628dda18a928aedc49913dc15451f6dfdfd0..0000000000000000000000000000000000000000 --- a/data/developers/sakalti.json +++ /dev/null @@ -1,929 +0,0 @@ -{ - "developer": "Sakalti", - "models": [ - { - "id": "Sakalti/Anemoi-3B", - "name": "Anemoi-3B", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3804, - "hfopenllm_v2/BBH": 0.4922, - "hfopenllm_v2/MATH Level 5": 0.1775, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.4371, - "hfopenllm_v2/MMLU-PRO": 0.3766 - } - }, - { - "id": "Sakalti/Euphrates-14B", - "name": "Euphrates-14B", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2647, - "hfopenllm_v2/BBH": 0.6138, - "hfopenllm_v2/MATH Level 5": 0.3051, - "hfopenllm_v2/GPQA": 0.3935, - "hfopenllm_v2/MUSR": 0.4516, - "hfopenllm_v2/MMLU-PRO": 0.5255 - } - }, - { - "id": "Sakalti/light-1.1-3B", - "name": "light-1.1-3B", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2735, - "hfopenllm_v2/BBH": 0.2803, - "hfopenllm_v2/MATH Level 5": 0.0113, - "hfopenllm_v2/GPQA": 0.2617, - "hfopenllm_v2/MUSR": 0.3901, - "hfopenllm_v2/MMLU-PRO": 0.1209 - } - }, - { - "id": "Sakalti/light-3B", - "name": "light-3B", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5337, - "hfopenllm_v2/BBH": 0.4831, - "hfopenllm_v2/MATH Level 5": 0.2591, - "hfopenllm_v2/GPQA": 0.2953, - "hfopenllm_v2/MUSR": 0.4015, - "hfopenllm_v2/MMLU-PRO": 0.3775 - } - }, - { - "id": "Sakalti/light-3b-beta", - "name": "light-3b-beta", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5485, - "hfopenllm_v2/BBH": 0.4815, - "hfopenllm_v2/MATH Level 5": 0.2772, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.4015, - "hfopenllm_v2/MMLU-PRO": 0.3758 - } - }, - { - "id": "Sakalti/light-7b-beta", - "name": "light-7b-beta", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6234, - "hfopenllm_v2/BBH": 0.5548, - "hfopenllm_v2/MATH Level 5": 0.3769, - "hfopenllm_v2/GPQA": 0.3213, - "hfopenllm_v2/MUSR": 0.4291, - "hfopenllm_v2/MMLU-PRO": 0.4456 - } - }, - { - "id": "Sakalti/llama-3-yanyuedao-8b-instruct", - "name": "llama-3-yanyuedao-8b-instruct", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2186, - "hfopenllm_v2/BBH": 0.435, - "hfopenllm_v2/MATH Level 5": 0.0385, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.4199, - "hfopenllm_v2/MMLU-PRO": 0.2911 - } - }, - { - "id": "Sakalti/Llama3.2-3B-Uranus-1", - "name": "Llama3.2-3B-Uranus-1", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5335, - "hfopenllm_v2/BBH": 0.4437, - "hfopenllm_v2/MATH Level 5": 0.1495, - "hfopenllm_v2/GPQA": 0.297, - "hfopenllm_v2/MUSR": 0.3669, - "hfopenllm_v2/MMLU-PRO": 0.3094 - } - }, - { - "id": "Sakalti/magro-7B", - "name": "magro-7B", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1344, - "hfopenllm_v2/BBH": 0.4186, - "hfopenllm_v2/MATH Level 5": 0.0204, - "hfopenllm_v2/GPQA": 0.2953, - "hfopenllm_v2/MUSR": 0.446, - "hfopenllm_v2/MMLU-PRO": 0.2765 - } - }, - { - "id": "Sakalti/Magro-7B-v1.1", - "name": "Magro-7B-v1.1", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1204, - "hfopenllm_v2/BBH": 0.4179, - "hfopenllm_v2/MATH Level 5": 0.0249, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.4433, - "hfopenllm_v2/MMLU-PRO": 0.2764 - } - }, - { - "id": "Sakalti/mergekit-01", - "name": "mergekit-01", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6234, - "hfopenllm_v2/BBH": 0.5548, - "hfopenllm_v2/MATH Level 5": 0.3769, - "hfopenllm_v2/GPQA": 0.3213, - "hfopenllm_v2/MUSR": 0.4291, - "hfopenllm_v2/MMLU-PRO": 0.4456 - } - }, - { - "id": "Sakalti/mergekit-della_linear-vmeykci", - "name": "mergekit-della_linear-vmeykci", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1126, - "hfopenllm_v2/BBH": 0.2816, - "hfopenllm_v2/MATH Level 5": 0.0106, - "hfopenllm_v2/GPQA": 0.2634, - "hfopenllm_v2/MUSR": 0.3897, - "hfopenllm_v2/MMLU-PRO": 0.1089 - } - }, - { - "id": "Sakalti/model-3", - "name": "model-3", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6264, - "hfopenllm_v2/BBH": 0.5542, - "hfopenllm_v2/MATH Level 5": 0.3708, - "hfopenllm_v2/GPQA": 0.3213, - "hfopenllm_v2/MUSR": 0.4264, - "hfopenllm_v2/MMLU-PRO": 0.4455 - } - }, - { - "id": "Sakalti/Neptuno-3B", - "name": "Neptuno-3B", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4296, - "hfopenllm_v2/BBH": 0.4834, - "hfopenllm_v2/MATH Level 5": 0.2553, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.4002, - "hfopenllm_v2/MMLU-PRO": 0.3773 - } - }, - { - "id": "Sakalti/Neptuno-Alpha", - "name": "Neptuno-Alpha", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.378, - "hfopenllm_v2/BBH": 0.4925, - "hfopenllm_v2/MATH Level 5": 0.1835, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.4371, - "hfopenllm_v2/MMLU-PRO": 0.3767 - } - }, - { - "id": "Sakalti/Oxyge1-33B", - "name": "Oxyge1-33B", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4548, - "hfopenllm_v2/BBH": 0.7033, - "hfopenllm_v2/MATH Level 5": 0.4962, - "hfopenllm_v2/GPQA": 0.3826, - "hfopenllm_v2/MUSR": 0.5008, - "hfopenllm_v2/MMLU-PRO": 0.5909 - } - }, - { - "id": "Sakalti/Phi3.5-Comets-3.8B", - "name": "Phi3.5-Comets-3.8B", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2094, - "hfopenllm_v2/BBH": 0.3335, - "hfopenllm_v2/MATH Level 5": 0.0008, - "hfopenllm_v2/GPQA": 0.2492, - "hfopenllm_v2/MUSR": 0.3764, - "hfopenllm_v2/MMLU-PRO": 0.1153 - } - }, - { - "id": "Sakalti/Qwen2.5-1B-Instruct", - "name": "Qwen2.5-1B-Instruct", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1751, - "hfopenllm_v2/BBH": 0.3027, - "hfopenllm_v2/MATH Level 5": 0.006, - "hfopenllm_v2/GPQA": 0.2559, - "hfopenllm_v2/MUSR": 0.3369, - "hfopenllm_v2/MMLU-PRO": 0.1213 - } - }, - { - "id": "Sakalti/qwen2.5-2.3B", - "name": "qwen2.5-2.3B", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1288, - "hfopenllm_v2/BBH": 0.2849, - "hfopenllm_v2/MATH Level 5": 0.0053, - "hfopenllm_v2/GPQA": 0.2517, - "hfopenllm_v2/MUSR": 0.3857, - "hfopenllm_v2/MMLU-PRO": 0.1173 - } - }, - { - "id": "Sakalti/QwenTest-7", - "name": "QwenTest-7", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1672, - "hfopenllm_v2/BBH": 0.3063, - "hfopenllm_v2/MATH Level 5": 0.0038, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3422, - "hfopenllm_v2/MMLU-PRO": 0.1212 - } - }, - { - "id": "Sakalti/Saba-Passthrough-2", - "name": "Saba-Passthrough-2", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1691, - "hfopenllm_v2/BBH": 0.3672, - "hfopenllm_v2/MATH Level 5": 0.0008, - "hfopenllm_v2/GPQA": 0.2634, - "hfopenllm_v2/MUSR": 0.3844, - "hfopenllm_v2/MMLU-PRO": 0.2077 - } - }, - { - "id": "Sakalti/Saba1-1.8B", - "name": "Saba1-1.8B", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3333, - "hfopenllm_v2/BBH": 0.4147, - "hfopenllm_v2/MATH Level 5": 0.1541, - "hfopenllm_v2/GPQA": 0.2827, - "hfopenllm_v2/MUSR": 0.4239, - "hfopenllm_v2/MMLU-PRO": 0.2926 - } - }, - { - "id": "Sakalti/Saba1-7B", - "name": "Saba1-7B", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4585, - "hfopenllm_v2/BBH": 0.5489, - "hfopenllm_v2/MATH Level 5": 0.3663, - "hfopenllm_v2/GPQA": 0.3163, - "hfopenllm_v2/MUSR": 0.4793, - "hfopenllm_v2/MMLU-PRO": 0.4376 - } - }, - { - "id": "Sakalti/Saba1.5-1.5B", - "name": "Saba1.5-1.5B", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3333, - "hfopenllm_v2/BBH": 0.4147, - "hfopenllm_v2/MATH Level 5": 0.1541, - "hfopenllm_v2/GPQA": 0.2827, - "hfopenllm_v2/MUSR": 0.4239, - "hfopenllm_v2/MMLU-PRO": 0.2926 - } - }, - { - "id": "Sakalti/Saba1.5-Pro-3B", - "name": "Saba1.5-Pro-3B", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2386, - "hfopenllm_v2/BBH": 0.3623, - "hfopenllm_v2/MATH Level 5": 0.0272, - "hfopenllm_v2/GPQA": 0.2685, - "hfopenllm_v2/MUSR": 0.4405, - "hfopenllm_v2/MMLU-PRO": 0.1958 - } - }, - { - "id": "Sakalti/Saba2-14B-Preview", - "name": "Saba2-14B-Preview", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4722, - "hfopenllm_v2/BBH": 0.6496, - "hfopenllm_v2/MATH Level 5": 0.3127, - "hfopenllm_v2/GPQA": 0.3826, - "hfopenllm_v2/MUSR": 0.4781, - "hfopenllm_v2/MMLU-PRO": 0.5384 - } - }, - { - "id": "Sakalti/Saba2-3B", - "name": "Saba2-3B", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2865, - "hfopenllm_v2/BBH": 0.2801, - "hfopenllm_v2/MATH Level 5": 0.006, - "hfopenllm_v2/GPQA": 0.2617, - "hfopenllm_v2/MUSR": 0.3927, - "hfopenllm_v2/MMLU-PRO": 0.121 - } - }, - { - "id": "Sakalti/Sailor-japanese", - "name": "Sailor-japanese", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1605, - "hfopenllm_v2/BBH": 0.2913, - "hfopenllm_v2/MATH Level 5": 0.003, - "hfopenllm_v2/GPQA": 0.2534, - "hfopenllm_v2/MUSR": 0.3912, - "hfopenllm_v2/MMLU-PRO": 0.1164 - } - }, - { - "id": "Sakalti/Saka-1.5B", - "name": "Saka-1.5B", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2726, - "hfopenllm_v2/BBH": 0.3988, - "hfopenllm_v2/MATH Level 5": 0.0801, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.3739, - "hfopenllm_v2/MMLU-PRO": 0.2415 - } - }, - { - "id": "Sakalti/Saka-14B", - "name": "Saka-14B", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7174, - "hfopenllm_v2/BBH": 0.6497, - "hfopenllm_v2/MATH Level 5": 0.4094, - "hfopenllm_v2/GPQA": 0.396, - "hfopenllm_v2/MUSR": 0.4886, - "hfopenllm_v2/MMLU-PRO": 0.5396 - } - }, - { - "id": "Sakalti/Saka-24B", - "name": "Saka-24B", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3819, - "hfopenllm_v2/BBH": 0.6072, - "hfopenllm_v2/MATH Level 5": 0.1805, - "hfopenllm_v2/GPQA": 0.3423, - "hfopenllm_v2/MUSR": 0.4541, - "hfopenllm_v2/MMLU-PRO": 0.4766 - } - }, - { - "id": "Sakalti/Saka-7.2B", - "name": "Saka-7.2B", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1545, - "hfopenllm_v2/BBH": 0.2945, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2391, - "hfopenllm_v2/MUSR": 0.3711, - "hfopenllm_v2/MMLU-PRO": 0.116 - } - }, - { - "id": "Sakalti/Saka-7.6B", - "name": "Saka-7.6B", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4524, - "hfopenllm_v2/BBH": 0.5655, - "hfopenllm_v2/MATH Level 5": 0.3255, - "hfopenllm_v2/GPQA": 0.3163, - "hfopenllm_v2/MUSR": 0.4489, - "hfopenllm_v2/MMLU-PRO": 0.454 - } - }, - { - "id": "Sakalti/SakalFusion-7B-Alpha", - "name": "SakalFusion-7B-Alpha", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.529, - "hfopenllm_v2/BBH": 0.5591, - "hfopenllm_v2/MATH Level 5": 0.3844, - "hfopenllm_v2/GPQA": 0.3255, - "hfopenllm_v2/MUSR": 0.4581, - "hfopenllm_v2/MMLU-PRO": 0.4474 - } - }, - { - "id": "Sakalti/SakalFusion-7B-Beta", - "name": "SakalFusion-7B-Beta", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1809, - "hfopenllm_v2/BBH": 0.2881, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2433, - "hfopenllm_v2/MUSR": 0.3872, - "hfopenllm_v2/MMLU-PRO": 0.109 - } - }, - { - "id": "Sakalti/SakaMoe-3x1.6B-Instruct", - "name": "SakaMoe-3x1.6B-Instruct", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2371, - "hfopenllm_v2/BBH": 0.3282, - "hfopenllm_v2/MATH Level 5": 0.0544, - "hfopenllm_v2/GPQA": 0.2668, - "hfopenllm_v2/MUSR": 0.3342, - "hfopenllm_v2/MMLU-PRO": 0.1882 - } - }, - { - "id": "Sakalti/SJT-0.5B", - "name": "SJT-0.5B", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2425, - "hfopenllm_v2/BBH": 0.3306, - "hfopenllm_v2/MATH Level 5": 0.0521, - "hfopenllm_v2/GPQA": 0.2718, - "hfopenllm_v2/MUSR": 0.3196, - "hfopenllm_v2/MMLU-PRO": 0.1891 - } - }, - { - "id": "Sakalti/SJT-1.5B-Alpha", - "name": "SJT-1.5B-Alpha", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3449, - "hfopenllm_v2/BBH": 0.4241, - "hfopenllm_v2/MATH Level 5": 0.0997, - "hfopenllm_v2/GPQA": 0.2919, - "hfopenllm_v2/MUSR": 0.4226, - "hfopenllm_v2/MMLU-PRO": 0.2961 - } - }, - { - "id": "Sakalti/SJT-1.5B-Alpha-1.1", - "name": "SJT-1.5B-Alpha-1.1", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3439, - "hfopenllm_v2/BBH": 0.4243, - "hfopenllm_v2/MATH Level 5": 0.0959, - "hfopenllm_v2/GPQA": 0.2894, - "hfopenllm_v2/MUSR": 0.4239, - "hfopenllm_v2/MMLU-PRO": 0.2966 - } - }, - { - "id": "Sakalti/SJT-1.7B", - "name": "SJT-1.7B", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1776, - "hfopenllm_v2/BBH": 0.2934, - "hfopenllm_v2/MATH Level 5": 0.0015, - "hfopenllm_v2/GPQA": 0.2416, - "hfopenllm_v2/MUSR": 0.3964, - "hfopenllm_v2/MMLU-PRO": 0.1133 - } - }, - { - "id": "Sakalti/SJT-14B", - "name": "SJT-14B", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5494, - "hfopenllm_v2/BBH": 0.6536, - "hfopenllm_v2/MATH Level 5": 0.3844, - "hfopenllm_v2/GPQA": 0.3867, - "hfopenllm_v2/MUSR": 0.4766, - "hfopenllm_v2/MMLU-PRO": 0.5381 - } - }, - { - "id": "Sakalti/SJT-2.4B", - "name": "SJT-2.4B", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2804, - "hfopenllm_v2/BBH": 0.349, - "hfopenllm_v2/MATH Level 5": 0.0219, - "hfopenllm_v2/GPQA": 0.2559, - "hfopenllm_v2/MUSR": 0.3699, - "hfopenllm_v2/MMLU-PRO": 0.1858 - } - }, - { - "id": "Sakalti/SJT-24B-Alpha", - "name": "SJT-24B-Alpha", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3206, - "hfopenllm_v2/BBH": 0.6081, - "hfopenllm_v2/MATH Level 5": 0.253, - "hfopenllm_v2/GPQA": 0.3809, - "hfopenllm_v2/MUSR": 0.4595, - "hfopenllm_v2/MMLU-PRO": 0.4857 - } - }, - { - "id": "Sakalti/SJT-2B", - "name": "SJT-2B", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2151, - "hfopenllm_v2/BBH": 0.2936, - "hfopenllm_v2/MATH Level 5": 0.0008, - "hfopenllm_v2/GPQA": 0.2416, - "hfopenllm_v2/MUSR": 0.3564, - "hfopenllm_v2/MMLU-PRO": 0.1187 - } - }, - { - "id": "Sakalti/SJT-2B-V1.1", - "name": "SJT-2B-V1.1", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3977, - "hfopenllm_v2/BBH": 0.3984, - "hfopenllm_v2/MATH Level 5": 0.0483, - "hfopenllm_v2/GPQA": 0.2676, - "hfopenllm_v2/MUSR": 0.4299, - "hfopenllm_v2/MMLU-PRO": 0.2124 - } - }, - { - "id": "Sakalti/SJT-3.7B", - "name": "SJT-3.7B", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1078, - "hfopenllm_v2/BBH": 0.3393, - "hfopenllm_v2/MATH Level 5": 0.0121, - "hfopenllm_v2/GPQA": 0.2559, - "hfopenllm_v2/MUSR": 0.3617, - "hfopenllm_v2/MMLU-PRO": 0.1505 - } - }, - { - "id": "Sakalti/SJT-4B", - "name": "SJT-4B", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4077, - "hfopenllm_v2/BBH": 0.4886, - "hfopenllm_v2/MATH Level 5": 0.1156, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.478, - "hfopenllm_v2/MMLU-PRO": 0.3281 - } - }, - { - "id": "Sakalti/SJT-7.5B", - "name": "SJT-7.5B", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4223, - "hfopenllm_v2/BBH": 0.5367, - "hfopenllm_v2/MATH Level 5": 0.2168, - "hfopenllm_v2/GPQA": 0.3263, - "hfopenllm_v2/MUSR": 0.4399, - "hfopenllm_v2/MMLU-PRO": 0.3951 - } - }, - { - "id": "Sakalti/SJT-7B-V1.1", - "name": "SJT-7B-V1.1", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4703, - "hfopenllm_v2/BBH": 0.5419, - "hfopenllm_v2/MATH Level 5": 0.2432, - "hfopenllm_v2/GPQA": 0.3339, - "hfopenllm_v2/MUSR": 0.4411, - "hfopenllm_v2/MMLU-PRO": 0.4412 - } - }, - { - "id": "Sakalti/SJT-7B-V1.1-Multilingal", - "name": "SJT-7B-V1.1-Multilingal", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1949, - "hfopenllm_v2/BBH": 0.292, - "hfopenllm_v2/MATH Level 5": 0.0045, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3621, - "hfopenllm_v2/MMLU-PRO": 0.1137 - } - }, - { - "id": "Sakalti/SJT-8B", - "name": "SJT-8B", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6535, - "hfopenllm_v2/BBH": 0.5282, - "hfopenllm_v2/MATH Level 5": 0.2538, - "hfopenllm_v2/GPQA": 0.3297, - "hfopenllm_v2/MUSR": 0.408, - "hfopenllm_v2/MMLU-PRO": 0.4266 - } - }, - { - "id": "Sakalti/SJT-8B-V1.1", - "name": "SJT-8B-V1.1", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4621, - "hfopenllm_v2/BBH": 0.5121, - "hfopenllm_v2/MATH Level 5": 0.2069, - "hfopenllm_v2/GPQA": 0.3364, - "hfopenllm_v2/MUSR": 0.4266, - "hfopenllm_v2/MMLU-PRO": 0.4231 - } - }, - { - "id": "Sakalti/SJT-900M", - "name": "SJT-900M", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.241, - "hfopenllm_v2/BBH": 0.3169, - "hfopenllm_v2/MATH Level 5": 0.0136, - "hfopenllm_v2/GPQA": 0.2534, - "hfopenllm_v2/MUSR": 0.3595, - "hfopenllm_v2/MMLU-PRO": 0.1142 - } - }, - { - "id": "Sakalti/SJT-Moe2x7.5B", - "name": "SJT-Moe2x7.5B", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4117, - "hfopenllm_v2/BBH": 0.5371, - "hfopenllm_v2/MATH Level 5": 0.2145, - "hfopenllm_v2/GPQA": 0.3263, - "hfopenllm_v2/MUSR": 0.4399, - "hfopenllm_v2/MMLU-PRO": 0.3954 - } - }, - { - "id": "Sakalti/SJTPass-2", - "name": "SJTPass-2", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.24, - "hfopenllm_v2/BBH": 0.3302, - "hfopenllm_v2/MATH Level 5": 0.0529, - "hfopenllm_v2/GPQA": 0.2727, - "hfopenllm_v2/MUSR": 0.3222, - "hfopenllm_v2/MMLU-PRO": 0.1902 - } - }, - { - "id": "Sakalti/SJTPass-4", - "name": "SJTPass-4", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1913, - "hfopenllm_v2/BBH": 0.2964, - "hfopenllm_v2/MATH Level 5": 0.0023, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3898, - "hfopenllm_v2/MMLU-PRO": 0.1083 - } - }, - { - "id": "Sakalti/SJTPass-5", - "name": "SJTPass-5", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2425, - "hfopenllm_v2/BBH": 0.3103, - "hfopenllm_v2/MATH Level 5": 0.0159, - "hfopenllm_v2/GPQA": 0.2668, - "hfopenllm_v2/MUSR": 0.3794, - "hfopenllm_v2/MMLU-PRO": 0.1327 - } - }, - { - "id": "Sakalti/tara-3.8B", - "name": "tara-3.8B", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4077, - "hfopenllm_v2/BBH": 0.4886, - "hfopenllm_v2/MATH Level 5": 0.1156, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.478, - "hfopenllm_v2/MMLU-PRO": 0.3281 - } - }, - { - "id": "Sakalti/Tara-3.8B-v1.1", - "name": "Tara-3.8B-v1.1", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4062, - "hfopenllm_v2/BBH": 0.4886, - "hfopenllm_v2/MATH Level 5": 0.1156, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.478, - "hfopenllm_v2/MMLU-PRO": 0.3281 - } - }, - { - "id": "Sakalti/ultiima-14B", - "name": "ultiima-14B", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5701, - "hfopenllm_v2/BBH": 0.6491, - "hfopenllm_v2/MATH Level 5": 0.4698, - "hfopenllm_v2/GPQA": 0.3742, - "hfopenllm_v2/MUSR": 0.4718, - "hfopenllm_v2/MMLU-PRO": 0.5381 - } - }, - { - "id": "Sakalti/ultiima-14B-v0.2", - "name": "ultiima-14B-v0.2", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.707, - "hfopenllm_v2/BBH": 0.6472, - "hfopenllm_v2/MATH Level 5": 0.3995, - "hfopenllm_v2/GPQA": 0.3826, - "hfopenllm_v2/MUSR": 0.4794, - "hfopenllm_v2/MMLU-PRO": 0.5387 - } - }, - { - "id": "Sakalti/ultiima-14B-v0.3", - "name": "ultiima-14B-v0.3", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.704, - "hfopenllm_v2/BBH": 0.6398, - "hfopenllm_v2/MATH Level 5": 0.3965, - "hfopenllm_v2/GPQA": 0.3767, - "hfopenllm_v2/MUSR": 0.4754, - "hfopenllm_v2/MMLU-PRO": 0.5337 - } - }, - { - "id": "Sakalti/ultiima-14B-v0.4", - "name": "ultiima-14B-v0.4", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3008, - "hfopenllm_v2/BBH": 0.642, - "hfopenllm_v2/MATH Level 5": 0.3535, - "hfopenllm_v2/GPQA": 0.396, - "hfopenllm_v2/MUSR": 0.4886, - "hfopenllm_v2/MMLU-PRO": 0.5278 - } - }, - { - "id": "Sakalti/ultiima-32B", - "name": "ultiima-32B", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6854, - "hfopenllm_v2/BBH": 0.7037, - "hfopenllm_v2/MATH Level 5": 0.4962, - "hfopenllm_v2/GPQA": 0.3809, - "hfopenllm_v2/MUSR": 0.4995, - "hfopenllm_v2/MMLU-PRO": 0.591 - } - }, - { - "id": "Sakalti/ultiima-72B", - "name": "ultiima-72B", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.714, - "hfopenllm_v2/BBH": 0.7218, - "hfopenllm_v2/MATH Level 5": 0.5355, - "hfopenllm_v2/GPQA": 0.4144, - "hfopenllm_v2/MUSR": 0.4652, - "hfopenllm_v2/MMLU-PRO": 0.5906 - } - }, - { - "id": "Sakalti/ultiima-72B-v1.5", - "name": "ultiima-72B-v1.5", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.655, - "hfopenllm_v2/BBH": 0.7392, - "hfopenllm_v2/MATH Level 5": 0.4396, - "hfopenllm_v2/GPQA": 0.4136, - "hfopenllm_v2/MUSR": 0.4691, - "hfopenllm_v2/MMLU-PRO": 0.6054 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/sakhan10.json b/data/developers/sakhan10.json deleted file mode 100644 index b12902c83010dbcb46cf75c5d23653f21f4afc03..0000000000000000000000000000000000000000 --- a/data/developers/sakhan10.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "sakhan10", - "models": [ - { - "id": "sakhan10/quantized_open_llama_3b_v2", - "name": "quantized_open_llama_3b_v2", - "developer": "sakhan10", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1872, - "hfopenllm_v2/BBH": 0.302, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2768, - "hfopenllm_v2/MUSR": 0.3682, - "hfopenllm_v2/MMLU-PRO": 0.1095 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/salesforce.json b/data/developers/salesforce.json deleted file mode 100644 index e75d089c85c971d7b40af38b2ddc9bda8edebe01..0000000000000000000000000000000000000000 --- a/data/developers/salesforce.json +++ /dev/null @@ -1,248 +0,0 @@ -{ - "developer": "salesforce", - "models": [ - { - "id": "Salesforce/LLaMA-3-8B-SFR-Iterative-DPO-R", - "name": "LLaMA-3-8B-SFR-Iterative-DPO-R", - "developer": "Salesforce", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3816, - "hfopenllm_v2/BBH": 0.5012, - "hfopenllm_v2/MATH Level 5": 0.0914, - "hfopenllm_v2/GPQA": 0.2878, - "hfopenllm_v2/MUSR": 0.3633, - "hfopenllm_v2/MMLU-PRO": 0.3172 - } - }, - { - "id": "Salesforce/SFR-LLaMa-3.1-70B-Judge-r", - "name": "Salesforce/SFR-LLaMa-3.1-70B-Judge-r", - "developer": "Salesforce", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.9272, - "reward-bench/Chat": 0.9693, - "reward-bench/Chat Hard": 0.8476, - "reward-bench/Safety": 0.9162, - "reward-bench/Reasoning": 0.9757 - } - }, - { - "id": "Salesforce/SFR-LLaMa-3.1-8B-Judge-r", - "name": "Salesforce/SFR-LLaMa-3.1-8B-Judge-r", - "developer": "Salesforce", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.8865, - "reward-bench/Chat": 0.9553, - "reward-bench/Chat Hard": 0.7774, - "reward-bench/Safety": 0.8622, - "reward-bench/Reasoning": 0.9513 - } - }, - { - "id": "Salesforce/SFR-nemo-12B-Judge-r", - "name": "Salesforce/SFR-nemo-12B-Judge-r", - "developer": "Salesforce", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.9027, - "reward-bench/Chat": 0.9721, - "reward-bench/Chat Hard": 0.8224, - "reward-bench/Safety": 0.8649, - "reward-bench/Reasoning": 0.9513 - } - }, - { - "id": "salesforce/xlam-2-1b-fc-r-fc", - "name": "xLAM-2-1b-fc-r (FC)", - "developer": "salesforce", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 65.0, - "bfcl/bfcl.overall.overall_accuracy": 30.44, - "bfcl/bfcl.overall.total_cost_usd": 2.79, - "bfcl/bfcl.overall.latency_mean_s": 2.84, - "bfcl/bfcl.overall.latency_std_s": 2.35, - "bfcl/bfcl.overall.latency_p95_s": 6.52, - "bfcl/bfcl.non_live.ast_accuracy": 69.04, - "bfcl/bfcl.non_live.simple_ast_accuracy": 64.17, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 82.5, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 73.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 56.0, - "bfcl/bfcl.live.live_accuracy": 55.14, - "bfcl/bfcl.live.live_simple_ast_accuracy": 68.22, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 52.8, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 43.75, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 25.0, - "bfcl/bfcl.multi_turn.accuracy": 36.0, - "bfcl/bfcl.multi_turn.base_accuracy": 45.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 36.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 37.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 25.5, - "bfcl/bfcl.web_search.accuracy": 0.0, - "bfcl/bfcl.web_search.base_accuracy": 0.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 0.0, - "bfcl/bfcl.memory.accuracy": 3.87, - "bfcl/bfcl.memory.kv_accuracy": 3.87, - "bfcl/bfcl.memory.vector_accuracy": 3.87, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 3.87, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 87.5, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 64.47 - } - }, - { - "id": "salesforce/xlam-2-32b-fc-r-fc", - "name": "xLAM-2-32b-fc-r (FC)", - "developer": "salesforce", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 18.0, - "bfcl/bfcl.overall.overall_accuracy": 54.66, - "bfcl/bfcl.overall.total_cost_usd": 6.0, - "bfcl/bfcl.overall.latency_mean_s": 6.94, - "bfcl/bfcl.overall.latency_std_s": 8.21, - "bfcl/bfcl.overall.latency_p95_s": 17.66, - "bfcl/bfcl.non_live.ast_accuracy": 89.6, - "bfcl/bfcl.non_live.simple_ast_accuracy": 80.42, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 94.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 93.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 91.0, - "bfcl/bfcl.live.live_accuracy": 75.5, - "bfcl/bfcl.live.live_simple_ast_accuracy": 82.17, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 74.64, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 50.0, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 58.33, - "bfcl/bfcl.multi_turn.accuracy": 69.5, - "bfcl/bfcl.multi_turn.base_accuracy": 81.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 72.5, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 67.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 56.5, - "bfcl/bfcl.web_search.accuracy": 25.5, - "bfcl/bfcl.web_search.base_accuracy": 37.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 14.0, - "bfcl/bfcl.memory.accuracy": 20.86, - "bfcl/bfcl.memory.kv_accuracy": 6.45, - "bfcl/bfcl.memory.vector_accuracy": 10.32, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 45.81, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 81.25, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 80.23 - } - }, - { - "id": "salesforce/xlam-2-3b-fc-r-fc", - "name": "xLAM-2-3b-fc-r (FC)", - "developer": "salesforce", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 42.0, - "bfcl/bfcl.overall.overall_accuracy": 41.22, - "bfcl/bfcl.overall.total_cost_usd": 3.36, - "bfcl/bfcl.overall.latency_mean_s": 3.8, - "bfcl/bfcl.overall.latency_std_s": 3.59, - "bfcl/bfcl.overall.latency_p95_s": 8.79, - "bfcl/bfcl.non_live.ast_accuracy": 82.96, - "bfcl/bfcl.non_live.simple_ast_accuracy": 75.33, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 91.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 86.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 79.0, - "bfcl/bfcl.live.live_accuracy": 62.92, - "bfcl/bfcl.live.live_simple_ast_accuracy": 73.26, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 60.68, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 62.5, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 50.0, - "bfcl/bfcl.multi_turn.accuracy": 58.38, - "bfcl/bfcl.multi_turn.base_accuracy": 71.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 59.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 57.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 45.5, - "bfcl/bfcl.web_search.accuracy": 2.5, - "bfcl/bfcl.web_search.base_accuracy": 3.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 2.0, - "bfcl/bfcl.memory.accuracy": 11.4, - "bfcl/bfcl.memory.kv_accuracy": 5.81, - "bfcl/bfcl.memory.vector_accuracy": 5.81, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 22.58, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 87.5, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 63.45 - } - }, - { - "id": "salesforce/xlam-2-70b-fc-r-fc", - "name": "xLAM-2-70b-fc-r (FC)", - "developer": "salesforce", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 22.0, - "bfcl/bfcl.overall.overall_accuracy": 53.07, - "bfcl/bfcl.overall.total_cost_usd": 25.1, - "bfcl/bfcl.overall.latency_mean_s": 28.06, - "bfcl/bfcl.overall.latency_std_s": 68.77, - "bfcl/bfcl.overall.latency_p95_s": 91.21, - "bfcl/bfcl.non_live.ast_accuracy": 88.44, - "bfcl/bfcl.non_live.simple_ast_accuracy": 78.25, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 94.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 92.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 89.5, - "bfcl/bfcl.live.live_accuracy": 72.17, - "bfcl/bfcl.live.live_simple_ast_accuracy": 77.91, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 71.13, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 68.75, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 58.33, - "bfcl/bfcl.multi_turn.accuracy": 77.38, - "bfcl/bfcl.multi_turn.base_accuracy": 82.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 77.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 74.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 76.0, - "bfcl/bfcl.web_search.accuracy": 15.0, - "bfcl/bfcl.web_search.base_accuracy": 17.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 13.0, - "bfcl/bfcl.memory.accuracy": 14.41, - "bfcl/bfcl.memory.kv_accuracy": 2.58, - "bfcl/bfcl.memory.vector_accuracy": 10.97, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 29.68, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 75.0, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 79.11 - } - }, - { - "id": "salesforce/xlam-2-8b-fc-r-fc", - "name": "xLAM-2-8b-fc-r (FC)", - "developer": "salesforce", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 34.0, - "bfcl/bfcl.overall.overall_accuracy": 46.68, - "bfcl/bfcl.overall.total_cost_usd": 20.92, - "bfcl/bfcl.overall.latency_mean_s": 22.65, - "bfcl/bfcl.overall.latency_std_s": 46.92, - "bfcl/bfcl.overall.latency_p95_s": 108.81, - "bfcl/bfcl.non_live.ast_accuracy": 84.58, - "bfcl/bfcl.non_live.simple_ast_accuracy": 73.83, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 93.5, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 87.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 83.5, - "bfcl/bfcl.live.live_accuracy": 67.95, - "bfcl/bfcl.live.live_simple_ast_accuracy": 75.58, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 66.57, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 56.25, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 54.17, - "bfcl/bfcl.multi_turn.accuracy": 70.0, - "bfcl/bfcl.multi_turn.base_accuracy": 76.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 72.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 65.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 67.0, - "bfcl/bfcl.web_search.accuracy": 6.5, - "bfcl/bfcl.web_search.base_accuracy": 11.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 2.0, - "bfcl/bfcl.memory.accuracy": 13.98, - "bfcl/bfcl.memory.kv_accuracy": 5.81, - "bfcl/bfcl.memory.vector_accuracy": 15.48, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 20.65, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 87.5, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 63.28 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/saltlux.json b/data/developers/saltlux.json deleted file mode 100644 index 24babe4e487ab3e00fc37985120480dceba70b97..0000000000000000000000000000000000000000 --- a/data/developers/saltlux.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "saltlux", - "models": [ - { - "id": "saltlux/luxia-21.4b-alignment-v1.0", - "name": "luxia-21.4b-alignment-v1.0", - "developer": "saltlux", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3693, - "hfopenllm_v2/BBH": 0.6373, - "hfopenllm_v2/MATH Level 5": 0.0974, - "hfopenllm_v2/GPQA": 0.3012, - "hfopenllm_v2/MUSR": 0.4328, - "hfopenllm_v2/MMLU-PRO": 0.3403 - } - }, - { - "id": "saltlux/luxia-21.4b-alignment-v1.2", - "name": "luxia-21.4b-alignment-v1.2", - "developer": "saltlux", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4115, - "hfopenllm_v2/BBH": 0.6371, - "hfopenllm_v2/MATH Level 5": 0.0846, - "hfopenllm_v2/GPQA": 0.3079, - "hfopenllm_v2/MUSR": 0.4459, - "hfopenllm_v2/MMLU-PRO": 0.3473 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/sam-paech.json b/data/developers/sam-paech.json deleted file mode 100644 index 709c4d51b66e2e5651f027f2da2034940aba8b82..0000000000000000000000000000000000000000 --- a/data/developers/sam-paech.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "developer": "sam-paech", - "models": [ - { - "id": "sam-paech/Darkest-muse-v1", - "name": "Darkest-muse-v1", - "developer": "sam-paech", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7344, - "hfopenllm_v2/BBH": 0.5968, - "hfopenllm_v2/MATH Level 5": 0.2145, - "hfopenllm_v2/GPQA": 0.344, - "hfopenllm_v2/MUSR": 0.4502, - "hfopenllm_v2/MMLU-PRO": 0.4184 - } - }, - { - "id": "sam-paech/Delirium-v1", - "name": "Delirium-v1", - "developer": "sam-paech", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7208, - "hfopenllm_v2/BBH": 0.5962, - "hfopenllm_v2/MATH Level 5": 0.2107, - "hfopenllm_v2/GPQA": 0.3431, - "hfopenllm_v2/MUSR": 0.4514, - "hfopenllm_v2/MMLU-PRO": 0.419 - } - }, - { - "id": "sam-paech/Quill-v1", - "name": "Quill-v1", - "developer": "sam-paech", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7122, - "hfopenllm_v2/BBH": 0.5969, - "hfopenllm_v2/MATH Level 5": 0.2122, - "hfopenllm_v2/GPQA": 0.3398, - "hfopenllm_v2/MUSR": 0.4555, - "hfopenllm_v2/MMLU-PRO": 0.4171 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/sanjiwatsuki.json b/data/developers/sanjiwatsuki.json deleted file mode 100644 index 9886f491af2c048e9849271814dfcde82a97dc56..0000000000000000000000000000000000000000 --- a/data/developers/sanjiwatsuki.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "SanjiWatsuki", - "models": [ - { - "id": "SanjiWatsuki/Kunoichi-DPO-v2-7B", - "name": "Kunoichi-DPO-v2-7B", - "developer": "SanjiWatsuki", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5431, - "hfopenllm_v2/BBH": 0.4416, - "hfopenllm_v2/MATH Level 5": 0.0763, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.4188, - "hfopenllm_v2/MMLU-PRO": 0.3107 - } - }, - { - "id": "SanjiWatsuki/Silicon-Maid-7B", - "name": "Silicon-Maid-7B", - "developer": "SanjiWatsuki", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5368, - "hfopenllm_v2/BBH": 0.4128, - "hfopenllm_v2/MATH Level 5": 0.065, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.4188, - "hfopenllm_v2/MMLU-PRO": 0.3083 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/sao10k.json b/data/developers/sao10k.json deleted file mode 100644 index 6569058a39b468de833ea3ab9419e073af211e5a..0000000000000000000000000000000000000000 --- a/data/developers/sao10k.json +++ /dev/null @@ -1,117 +0,0 @@ -{ - "developer": "Sao10K", - "models": [ - { - "id": "Sao10K/70B-L3.3-Cirrus-x1", - "name": "70B-L3.3-Cirrus-x1", - "developer": "Sao10K", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6681, - "hfopenllm_v2/BBH": 0.7029, - "hfopenllm_v2/MATH Level 5": 0.3739, - "hfopenllm_v2/GPQA": 0.4497, - "hfopenllm_v2/MUSR": 0.4842, - "hfopenllm_v2/MMLU-PRO": 0.5378 - } - }, - { - "id": "Sao10K/Fimbulvetr-11B-v2", - "name": "Fimbulvetr-11B-v2", - "developer": "Sao10K", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.51, - "hfopenllm_v2/BBH": 0.4544, - "hfopenllm_v2/MATH Level 5": 0.068, - "hfopenllm_v2/GPQA": 0.2919, - "hfopenllm_v2/MUSR": 0.4354, - "hfopenllm_v2/MMLU-PRO": 0.3301 - } - }, - { - "id": "Sao10K/L3-70B-Euryale-v2.1", - "name": "L3-70B-Euryale-v2.1", - "developer": "Sao10K", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7281, - "hfopenllm_v2/BBH": 0.6503, - "hfopenllm_v2/MATH Level 5": 0.2243, - "hfopenllm_v2/GPQA": 0.3314, - "hfopenllm_v2/MUSR": 0.4196, - "hfopenllm_v2/MMLU-PRO": 0.5096 - } - }, - { - "id": "Sao10K/L3-8B-Lunaris-v1", - "name": "L3-8B-Lunaris-v1", - "developer": "Sao10K", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6895, - "hfopenllm_v2/BBH": 0.5235, - "hfopenllm_v2/MATH Level 5": 0.0906, - "hfopenllm_v2/GPQA": 0.3012, - "hfopenllm_v2/MUSR": 0.3727, - "hfopenllm_v2/MMLU-PRO": 0.3787 - } - }, - { - "id": "Sao10K/L3-8B-Niitama-v1", - "name": "L3-8B-Niitama-v1", - "developer": "Sao10K", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6791, - "hfopenllm_v2/BBH": 0.5303, - "hfopenllm_v2/MATH Level 5": 0.0982, - "hfopenllm_v2/GPQA": 0.3079, - "hfopenllm_v2/MUSR": 0.3807, - "hfopenllm_v2/MMLU-PRO": 0.3701 - } - }, - { - "id": "Sao10K/L3-8B-Stheno-v3.2", - "name": "L3-8B-Stheno-v3.2", - "developer": "Sao10K", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6873, - "hfopenllm_v2/BBH": 0.5228, - "hfopenllm_v2/MATH Level 5": 0.0929, - "hfopenllm_v2/GPQA": 0.3104, - "hfopenllm_v2/MUSR": 0.3794, - "hfopenllm_v2/MMLU-PRO": 0.3768 - } - }, - { - "id": "Sao10K/L3-8B-Stheno-v3.3-32K", - "name": "L3-8B-Stheno-v3.3-32K", - "developer": "Sao10K", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4604, - "hfopenllm_v2/BBH": 0.3844, - "hfopenllm_v2/MATH Level 5": 0.0144, - "hfopenllm_v2/GPQA": 0.2567, - "hfopenllm_v2/MUSR": 0.3725, - "hfopenllm_v2/MMLU-PRO": 0.1896 - } - }, - { - "id": "Sao10K/MN-12B-Lyra-v3", - "name": "MN-12B-Lyra-v3", - "developer": "Sao10K", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4486, - "hfopenllm_v2/BBH": 0.4804, - "hfopenllm_v2/MATH Level 5": 0.0937, - "hfopenllm_v2/GPQA": 0.2777, - "hfopenllm_v2/MUSR": 0.4019, - "hfopenllm_v2/MMLU-PRO": 0.3249 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/sarvamai.json b/data/developers/sarvamai.json deleted file mode 100644 index 8984c3337cb73c343c38193daf6552f8d18e1b48..0000000000000000000000000000000000000000 --- a/data/developers/sarvamai.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "sarvamai", - "models": [ - { - "id": "sarvamai/OpenHathi-7B-Hi-v0.1-Base", - "name": "OpenHathi-7B-Hi-v0.1-Base", - "developer": "sarvamai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1804, - "hfopenllm_v2/BBH": 0.3354, - "hfopenllm_v2/MATH Level 5": 0.0083, - "hfopenllm_v2/GPQA": 0.2534, - "hfopenllm_v2/MUSR": 0.3658, - "hfopenllm_v2/MMLU-PRO": 0.1543 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/saxo.json b/data/developers/saxo.json deleted file mode 100644 index ebdad02ee1b1d5395feaa13ee41b292cd1c12286..0000000000000000000000000000000000000000 --- a/data/developers/saxo.json +++ /dev/null @@ -1,159 +0,0 @@ -{ - "developer": "Saxo", - "models": [ - { - "id": "Saxo/Linkbricks-Horizon-AI-Avengers-V1-32B", - "name": "Linkbricks-Horizon-AI-Avengers-V1-32B", - "developer": "Saxo", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7972, - "hfopenllm_v2/BBH": 0.7001, - "hfopenllm_v2/MATH Level 5": 0.6027, - "hfopenllm_v2/GPQA": 0.3624, - "hfopenllm_v2/MUSR": 0.4538, - "hfopenllm_v2/MMLU-PRO": 0.5793 - } - }, - { - "id": "Saxo/Linkbricks-Horizon-AI-Avengers-V2-32B", - "name": "Linkbricks-Horizon-AI-Avengers-V2-32B", - "developer": "Saxo", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7956, - "hfopenllm_v2/BBH": 0.7023, - "hfopenllm_v2/MATH Level 5": 0.5665, - "hfopenllm_v2/GPQA": 0.2659, - "hfopenllm_v2/MUSR": 0.4166, - "hfopenllm_v2/MMLU-PRO": 0.572 - } - }, - { - "id": "Saxo/Linkbricks-Horizon-AI-Avengers-V3-32B", - "name": "Linkbricks-Horizon-AI-Avengers-V3-32B", - "developer": "Saxo", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8249, - "hfopenllm_v2/BBH": 0.6913, - "hfopenllm_v2/MATH Level 5": 0.6178, - "hfopenllm_v2/GPQA": 0.3381, - "hfopenllm_v2/MUSR": 0.4275, - "hfopenllm_v2/MMLU-PRO": 0.5664 - } - }, - { - "id": "Saxo/Linkbricks-Horizon-AI-Avengers-V4-32B", - "name": "Linkbricks-Horizon-AI-Avengers-V4-32B", - "developer": "Saxo", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7631, - "hfopenllm_v2/BBH": 0.692, - "hfopenllm_v2/MATH Level 5": 0.5363, - "hfopenllm_v2/GPQA": 0.3616, - "hfopenllm_v2/MUSR": 0.4643, - "hfopenllm_v2/MMLU-PRO": 0.5752 - } - }, - { - "id": "Saxo/Linkbricks-Horizon-AI-Avengers-V5-32B", - "name": "Linkbricks-Horizon-AI-Avengers-V5-32B", - "developer": "Saxo", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7516, - "hfopenllm_v2/BBH": 0.6929, - "hfopenllm_v2/MATH Level 5": 0.5461, - "hfopenllm_v2/GPQA": 0.3557, - "hfopenllm_v2/MUSR": 0.4709, - "hfopenllm_v2/MMLU-PRO": 0.5762 - } - }, - { - "id": "Saxo/Linkbricks-Horizon-AI-Avengers-V6-32B", - "name": "Linkbricks-Horizon-AI-Avengers-V6-32B", - "developer": "Saxo", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8209, - "hfopenllm_v2/BBH": 0.689, - "hfopenllm_v2/MATH Level 5": 0.6224, - "hfopenllm_v2/GPQA": 0.3347, - "hfopenllm_v2/MUSR": 0.4274, - "hfopenllm_v2/MMLU-PRO": 0.5672 - } - }, - { - "id": "Saxo/Linkbricks-Horizon-AI-Korean-Avengers-V2-27B", - "name": "Linkbricks-Horizon-AI-Korean-Avengers-V2-27B", - "developer": "Saxo", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8146, - "hfopenllm_v2/BBH": 0.6463, - "hfopenllm_v2/MATH Level 5": 0.2802, - "hfopenllm_v2/GPQA": 0.3473, - "hfopenllm_v2/MUSR": 0.4139, - "hfopenllm_v2/MMLU-PRO": 0.4599 - } - }, - { - "id": "Saxo/Linkbricks-Horizon-AI-Korean-Avengers-V3-27B", - "name": "Linkbricks-Horizon-AI-Korean-Avengers-V3-27B", - "developer": "Saxo", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8142, - "hfopenllm_v2/BBH": 0.6404, - "hfopenllm_v2/MATH Level 5": 0.2492, - "hfopenllm_v2/GPQA": 0.3591, - "hfopenllm_v2/MUSR": 0.4467, - "hfopenllm_v2/MMLU-PRO": 0.4524 - } - }, - { - "id": "Saxo/Linkbricks-Horizon-AI-Korean-Superb-22B", - "name": "Linkbricks-Horizon-AI-Korean-Superb-22B", - "developer": "Saxo", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6767, - "hfopenllm_v2/BBH": 0.5626, - "hfopenllm_v2/MATH Level 5": 0.2372, - "hfopenllm_v2/GPQA": 0.3263, - "hfopenllm_v2/MUSR": 0.3908, - "hfopenllm_v2/MMLU-PRO": 0.3871 - } - }, - { - "id": "Saxo/Linkbricks-Horizon-AI-Korean-Superb-27B", - "name": "Linkbricks-Horizon-AI-Korean-Superb-27B", - "developer": "Saxo", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7768, - "hfopenllm_v2/BBH": 0.6518, - "hfopenllm_v2/MATH Level 5": 0.2719, - "hfopenllm_v2/GPQA": 0.3599, - "hfopenllm_v2/MUSR": 0.4791, - "hfopenllm_v2/MMLU-PRO": 0.4647 - } - }, - { - "id": "Saxo/Linkbricks-Horizon-AI-Superb-27B", - "name": "Linkbricks-Horizon-AI-Superb-27B", - "developer": "Saxo", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7302, - "hfopenllm_v2/BBH": 0.6186, - "hfopenllm_v2/MATH Level 5": 0.2221, - "hfopenllm_v2/GPQA": 0.3574, - "hfopenllm_v2/MUSR": 0.465, - "hfopenllm_v2/MMLU-PRO": 0.406 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/schnapss.json b/data/developers/schnapss.json deleted file mode 100644 index bcd5766199ddd0e9e653dd3e7d48a1f21af08243..0000000000000000000000000000000000000000 --- a/data/developers/schnapss.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "schnapss", - "models": [ - { - "id": "schnapss/testmerge-7b", - "name": "testmerge-7b", - "developer": "schnapss", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3922, - "hfopenllm_v2/BBH": 0.5187, - "hfopenllm_v2/MATH Level 5": 0.0687, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.4686, - "hfopenllm_v2/MMLU-PRO": 0.306 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/schrieffer.json b/data/developers/schrieffer.json deleted file mode 100644 index 3126aaf3fafa22fcfcbb0e55bb08dbfc8e04baeb..0000000000000000000000000000000000000000 --- a/data/developers/schrieffer.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "developer": "Schrieffer", - "models": [ - { - "id": "Schrieffer/Llama-SARM-4B", - "name": "Schrieffer/Llama-SARM-4B", - "developer": "Schrieffer", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7379, - "reward-bench/Factuality": 0.6874, - "reward-bench/Precise IF": 0.4281, - "reward-bench/Math": 0.6448, - "reward-bench/Safety": 0.9178, - "reward-bench/Focus": 0.9556, - "reward-bench/Ties": 0.7939 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/sci-m-wang.json b/data/developers/sci-m-wang.json deleted file mode 100644 index cdf4776cff7c1f8bb7944912c72f972ea977f301..0000000000000000000000000000000000000000 --- a/data/developers/sci-m-wang.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "developer": "sci-m-wang", - "models": [ - { - "id": "sci-m-wang/deepseek-llm-7b-chat-sa-v0.1", - "name": "deepseek-llm-7b-chat-sa-v0.1", - "developer": "sci-m-wang", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4036, - "hfopenllm_v2/BBH": 0.3718, - "hfopenllm_v2/MATH Level 5": 0.0264, - "hfopenllm_v2/GPQA": 0.2567, - "hfopenllm_v2/MUSR": 0.4173, - "hfopenllm_v2/MMLU-PRO": 0.2209 - } - }, - { - "id": "sci-m-wang/Mistral-7B-Instruct-sa-v0.1", - "name": "Mistral-7B-Instruct-sa-v0.1", - "developer": "sci-m-wang", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4335, - "hfopenllm_v2/BBH": 0.3273, - "hfopenllm_v2/MATH Level 5": 0.0144, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.39, - "hfopenllm_v2/MMLU-PRO": 0.2362 - } - }, - { - "id": "sci-m-wang/Phi-3-mini-4k-instruct-sa-v0.1", - "name": "Phi-3-mini-4k-instruct-sa-v0.1", - "developer": "sci-m-wang", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5021, - "hfopenllm_v2/BBH": 0.5502, - "hfopenllm_v2/MATH Level 5": 0.148, - "hfopenllm_v2/GPQA": 0.3289, - "hfopenllm_v2/MUSR": 0.4073, - "hfopenllm_v2/MMLU-PRO": 0.3985 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/seallms.json b/data/developers/seallms.json deleted file mode 100644 index 30ec679931075fdd65d50e21ff501eb142faf18c..0000000000000000000000000000000000000000 --- a/data/developers/seallms.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "developer": "SeaLLMs", - "models": [ - { - "id": "SeaLLMs/SeaLLM-7B-v2", - "name": "SeaLLM-7B-v2", - "developer": "SeaLLMs", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3671, - "hfopenllm_v2/BBH": 0.4902, - "hfopenllm_v2/MATH Level 5": 0.0853, - "hfopenllm_v2/GPQA": 0.2785, - "hfopenllm_v2/MUSR": 0.407, - "hfopenllm_v2/MMLU-PRO": 0.3083 - } - }, - { - "id": "SeaLLMs/SeaLLM-7B-v2.5", - "name": "SeaLLM-7B-v2.5", - "developer": "SeaLLMs", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4522, - "hfopenllm_v2/BBH": 0.498, - "hfopenllm_v2/MATH Level 5": 0.1088, - "hfopenllm_v2/GPQA": 0.276, - "hfopenllm_v2/MUSR": 0.4203, - "hfopenllm_v2/MMLU-PRO": 0.3203 - } - }, - { - "id": "SeaLLMs/SeaLLMs-v3-7B-Chat", - "name": "SeaLLMs-v3-7B-Chat", - "developer": "SeaLLMs", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4377, - "hfopenllm_v2/BBH": 0.5266, - "hfopenllm_v2/MATH Level 5": 0.1858, - "hfopenllm_v2/GPQA": 0.2987, - "hfopenllm_v2/MUSR": 0.4174, - "hfopenllm_v2/MMLU-PRO": 0.3895 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/securin.json b/data/developers/securin.json deleted file mode 100644 index 5ff352e96b7128943192bcdd7bbd31ecf10c814b..0000000000000000000000000000000000000000 --- a/data/developers/securin.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "securin", - "models": [ - { - "id": "securin/Securin-LLM-V2.5-Qwen-1.5B", - "name": "Securin-LLM-V2.5-Qwen-1.5B", - "developer": "securin", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1492, - "hfopenllm_v2/BBH": 0.3158, - "hfopenllm_v2/MATH Level 5": 0.0249, - "hfopenllm_v2/GPQA": 0.25, - "hfopenllm_v2/MUSR": 0.3606, - "hfopenllm_v2/MMLU-PRO": 0.1615 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/senseable.json b/data/developers/senseable.json deleted file mode 100644 index b8ae08e14ae9a18180ae7bfeff29701a4a77156b..0000000000000000000000000000000000000000 --- a/data/developers/senseable.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "senseable", - "models": [ - { - "id": "senseable/WestLake-7B-v2", - "name": "WestLake-7B-v2", - "developer": "senseable", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4419, - "hfopenllm_v2/BBH": 0.4073, - "hfopenllm_v2/MATH Level 5": 0.0483, - "hfopenllm_v2/GPQA": 0.2768, - "hfopenllm_v2/MUSR": 0.3937, - "hfopenllm_v2/MMLU-PRO": 0.2764 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/sensellm.json b/data/developers/sensellm.json deleted file mode 100644 index c213f7326b642f4cff4035bb78e11bc136f1b002..0000000000000000000000000000000000000000 --- a/data/developers/sensellm.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "SenseLLM", - "models": [ - { - "id": "SenseLLM/ReflectionCoder-CL-34B", - "name": "ReflectionCoder-CL-34B", - "developer": "SenseLLM", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4008, - "hfopenllm_v2/BBH": 0.3953, - "hfopenllm_v2/MATH Level 5": 0.0332, - "hfopenllm_v2/GPQA": 0.2508, - "hfopenllm_v2/MUSR": 0.4155, - "hfopenllm_v2/MMLU-PRO": 0.1424 - } - }, - { - "id": "SenseLLM/ReflectionCoder-DS-33B", - "name": "ReflectionCoder-DS-33B", - "developer": "SenseLLM", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3787, - "hfopenllm_v2/BBH": 0.3449, - "hfopenllm_v2/MATH Level 5": 0.0302, - "hfopenllm_v2/GPQA": 0.2743, - "hfopenllm_v2/MUSR": 0.3343, - "hfopenllm_v2/MMLU-PRO": 0.1202 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/sentientagi.json b/data/developers/sentientagi.json deleted file mode 100644 index 9e3052f693fdbc8b0cc8ba88c6f61ca9b99f0ccd..0000000000000000000000000000000000000000 --- a/data/developers/sentientagi.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "SentientAGI", - "models": [ - { - "id": "SentientAGI/Dobby-Mini-Leashed-Llama-3.1-8B", - "name": "Dobby-Mini-Leashed-Llama-3.1-8B", - "developer": "SentientAGI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7847, - "hfopenllm_v2/BBH": 0.5138, - "hfopenllm_v2/MATH Level 5": 0.1858, - "hfopenllm_v2/GPQA": 0.302, - "hfopenllm_v2/MUSR": 0.4254, - "hfopenllm_v2/MMLU-PRO": 0.3694 - } - }, - { - "id": "SentientAGI/Dobby-Mini-Unhinged-Llama-3.1-8B", - "name": "Dobby-Mini-Unhinged-Llama-3.1-8B", - "developer": "SentientAGI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7457, - "hfopenllm_v2/BBH": 0.5142, - "hfopenllm_v2/MATH Level 5": 0.1563, - "hfopenllm_v2/GPQA": 0.3062, - "hfopenllm_v2/MUSR": 0.4013, - "hfopenllm_v2/MMLU-PRO": 0.3585 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/seppev.json b/data/developers/seppev.json deleted file mode 100644 index a36959d2d4ad27dc7b193839473927f74de9f14b..0000000000000000000000000000000000000000 --- a/data/developers/seppev.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "SeppeV", - "models": [ - { - "id": "SeppeV/SmolLM_pretrained_with_sft_trained_with_1pc_data_on_a_preference_dpo", - "name": "SmolLM_pretrained_with_sft_trained_with_1pc_data_on_a_preference_dpo", - "developer": "SeppeV", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0955, - "hfopenllm_v2/BBH": 0.3073, - "hfopenllm_v2/MATH Level 5": 0.0121, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.4032, - "hfopenllm_v2/MMLU-PRO": 0.1161 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/sequelbox.json b/data/developers/sequelbox.json deleted file mode 100644 index e82470efcced51f7e1ac3e43c8735eb3cbc14f30..0000000000000000000000000000000000000000 --- a/data/developers/sequelbox.json +++ /dev/null @@ -1,89 +0,0 @@ -{ - "developer": "sequelbox", - "models": [ - { - "id": "sequelbox/gemma-2-9B-MOTH", - "name": "gemma-2-9B-MOTH", - "developer": "sequelbox", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2059, - "hfopenllm_v2/BBH": 0.308, - "hfopenllm_v2/MATH Level 5": 0.0106, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3409, - "hfopenllm_v2/MMLU-PRO": 0.114 - } - }, - { - "id": "sequelbox/Llama3.1-70B-PlumChat", - "name": "Llama3.1-70B-PlumChat", - "developer": "sequelbox", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5616, - "hfopenllm_v2/BBH": 0.6753, - "hfopenllm_v2/MATH Level 5": 0.3029, - "hfopenllm_v2/GPQA": 0.3909, - "hfopenllm_v2/MUSR": 0.4774, - "hfopenllm_v2/MMLU-PRO": 0.5164 - } - }, - { - "id": "sequelbox/Llama3.1-8B-MOTH", - "name": "Llama3.1-8B-MOTH", - "developer": "sequelbox", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5245, - "hfopenllm_v2/BBH": 0.4902, - "hfopenllm_v2/MATH Level 5": 0.1216, - "hfopenllm_v2/GPQA": 0.2685, - "hfopenllm_v2/MUSR": 0.3689, - "hfopenllm_v2/MMLU-PRO": 0.3339 - } - }, - { - "id": "sequelbox/Llama3.1-8B-PlumChat", - "name": "Llama3.1-8B-PlumChat", - "developer": "sequelbox", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4243, - "hfopenllm_v2/BBH": 0.3873, - "hfopenllm_v2/MATH Level 5": 0.0363, - "hfopenllm_v2/GPQA": 0.2651, - "hfopenllm_v2/MUSR": 0.3755, - "hfopenllm_v2/MMLU-PRO": 0.2127 - } - }, - { - "id": "sequelbox/Llama3.1-8B-PlumCode", - "name": "Llama3.1-8B-PlumCode", - "developer": "sequelbox", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2045, - "hfopenllm_v2/BBH": 0.3368, - "hfopenllm_v2/MATH Level 5": 0.0272, - "hfopenllm_v2/GPQA": 0.276, - "hfopenllm_v2/MUSR": 0.3773, - "hfopenllm_v2/MMLU-PRO": 0.2335 - } - }, - { - "id": "sequelbox/Llama3.1-8B-PlumMath", - "name": "Llama3.1-8B-PlumMath", - "developer": "sequelbox", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2242, - "hfopenllm_v2/BBH": 0.4032, - "hfopenllm_v2/MATH Level 5": 0.0476, - "hfopenllm_v2/GPQA": 0.318, - "hfopenllm_v2/MUSR": 0.3919, - "hfopenllm_v2/MMLU-PRO": 0.2975 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/sethuiyer.json b/data/developers/sethuiyer.json deleted file mode 100644 index 8c2a5d6156f4238828149cbb80ac84aad02697e0..0000000000000000000000000000000000000000 --- a/data/developers/sethuiyer.json +++ /dev/null @@ -1,89 +0,0 @@ -{ - "developer": "sethuiyer", - "models": [ - { - "id": "sethuiyer/Llama-3.1-8B-Experimental-1206-Instruct", - "name": "Llama-3.1-8B-Experimental-1206-Instruct", - "developer": "sethuiyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6967, - "hfopenllm_v2/BBH": 0.5104, - "hfopenllm_v2/MATH Level 5": 0.1118, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.3966, - "hfopenllm_v2/MMLU-PRO": 0.3529 - } - }, - { - "id": "sethuiyer/Llama-3.1-8B-Experimental-1208-Instruct", - "name": "Llama-3.1-8B-Experimental-1208-Instruct", - "developer": "sethuiyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.61, - "hfopenllm_v2/BBH": 0.4964, - "hfopenllm_v2/MATH Level 5": 0.0891, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.379, - "hfopenllm_v2/MMLU-PRO": 0.3511 - } - }, - { - "id": "sethuiyer/Llamaverse-3.1-8B-Instruct", - "name": "Llamaverse-3.1-8B-Instruct", - "developer": "sethuiyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6185, - "hfopenllm_v2/BBH": 0.5414, - "hfopenllm_v2/MATH Level 5": 0.1858, - "hfopenllm_v2/GPQA": 0.2911, - "hfopenllm_v2/MUSR": 0.3762, - "hfopenllm_v2/MMLU-PRO": 0.3523 - } - }, - { - "id": "sethuiyer/LlamaZero-3.1-8B-Experimental-1208", - "name": "LlamaZero-3.1-8B-Experimental-1208", - "developer": "sethuiyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6051, - "hfopenllm_v2/BBH": 0.4981, - "hfopenllm_v2/MATH Level 5": 0.108, - "hfopenllm_v2/GPQA": 0.2685, - "hfopenllm_v2/MUSR": 0.382, - "hfopenllm_v2/MMLU-PRO": 0.3 - } - }, - { - "id": "sethuiyer/Llamazing-3.1-8B-Instruct", - "name": "Llamazing-3.1-8B-Instruct", - "developer": "sethuiyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5711, - "hfopenllm_v2/BBH": 0.5291, - "hfopenllm_v2/MATH Level 5": 0.0544, - "hfopenllm_v2/GPQA": 0.3121, - "hfopenllm_v2/MUSR": 0.3976, - "hfopenllm_v2/MMLU-PRO": 0.3606 - } - }, - { - "id": "sethuiyer/Qwen2.5-7B-Anvita", - "name": "Qwen2.5-7B-Anvita", - "developer": "sethuiyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.648, - "hfopenllm_v2/BBH": 0.5466, - "hfopenllm_v2/MATH Level 5": 0.2017, - "hfopenllm_v2/GPQA": 0.3272, - "hfopenllm_v2/MUSR": 0.4337, - "hfopenllm_v2/MMLU-PRO": 0.4166 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/sf-foundation.json b/data/developers/sf-foundation.json deleted file mode 100644 index aad345870c7d1ee261bdf988b97dd013cae7289e..0000000000000000000000000000000000000000 --- a/data/developers/sf-foundation.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "developer": "SF-Foundation", - "models": [ - { - "id": "SF-Foundation/TextEval-Llama3.1-70B", - "name": "SF-Foundation/TextEval-Llama3.1-70B", - "developer": "SF-Foundation", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.9348, - "reward-bench/Chat": 0.9413, - "reward-bench/Chat Hard": 0.9013, - "reward-bench/Safety": 0.9324, - "reward-bench/Reasoning": 0.9641 - } - }, - { - "id": "SF-Foundation/TextEval-OffsetBias-12B", - "name": "SF-Foundation/TextEval-OffsetBias-12B", - "developer": "SF-Foundation", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.9105, - "reward-bench/Chat": 0.919, - "reward-bench/Chat Hard": 0.8662, - "reward-bench/Safety": 0.9203, - "reward-bench/Reasoning": 0.9365 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/sfairxc.json b/data/developers/sfairxc.json deleted file mode 100644 index f511504fa472f754b653d8fff6432038f2fa5642..0000000000000000000000000000000000000000 --- a/data/developers/sfairxc.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "developer": "sfairXC", - "models": [ - { - "id": "sfairXC/FsfairX-LLaMA3-RM-v0.1", - "name": "sfairXC/FsfairX-LLaMA3-RM-v0.1", - "developer": "sfairXC", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6292, - "reward-bench/Chat": 0.9944, - "reward-bench/Chat Hard": 0.6513, - "reward-bench/Safety": 0.7667, - "reward-bench/Reasoning": 0.8644, - "reward-bench/Prior Sets (0.5 weight)": 0.7492, - "reward-bench/Factuality": 0.5916, - "reward-bench/Precise IF": 0.4188, - "reward-bench/Math": 0.6284, - "reward-bench/Focus": 0.7051, - "reward-bench/Ties": 0.6647 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/shadowml.json b/data/developers/shadowml.json deleted file mode 100644 index 1bc11cfd3b8cf26a1a26b53cc430206434b59b47..0000000000000000000000000000000000000000 --- a/data/developers/shadowml.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "shadowml", - "models": [ - { - "id": "shadowml/BeagSake-7B", - "name": "BeagSake-7B", - "developer": "shadowml", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5216, - "hfopenllm_v2/BBH": 0.4711, - "hfopenllm_v2/MATH Level 5": 0.0506, - "hfopenllm_v2/GPQA": 0.281, - "hfopenllm_v2/MUSR": 0.4124, - "hfopenllm_v2/MMLU-PRO": 0.2585 - } - }, - { - "id": "shadowml/Mixolar-4x7b", - "name": "Mixolar-4x7b", - "developer": "shadowml", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3893, - "hfopenllm_v2/BBH": 0.5216, - "hfopenllm_v2/MATH Level 5": 0.0582, - "hfopenllm_v2/GPQA": 0.2928, - "hfopenllm_v2/MUSR": 0.4258, - "hfopenllm_v2/MMLU-PRO": 0.3305 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/sharathhebbar24.json b/data/developers/sharathhebbar24.json deleted file mode 100644 index efc3c20484e1afb781c77bd2f8d823bfcb4a31fc..0000000000000000000000000000000000000000 --- a/data/developers/sharathhebbar24.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "Sharathhebbar24", - "models": [ - { - "id": "Sharathhebbar24/chat_gpt2_dpo", - "name": "chat_gpt2_dpo", - "developer": "Sharathhebbar24", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0986, - "hfopenllm_v2/BBH": 0.2902, - "hfopenllm_v2/MATH Level 5": 0.0053, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3818, - "hfopenllm_v2/MMLU-PRO": 0.1142 - } - }, - { - "id": "Sharathhebbar24/SSH_355M", - "name": "SSH_355M", - "developer": "Sharathhebbar24", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1424, - "hfopenllm_v2/BBH": 0.3099, - "hfopenllm_v2/MATH Level 5": 0.0091, - "hfopenllm_v2/GPQA": 0.2584, - "hfopenllm_v2/MUSR": 0.4178, - "hfopenllm_v2/MMLU-PRO": 0.1176 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/shastraai.json b/data/developers/shastraai.json deleted file mode 100644 index 62f077e19bd887791b211081c450b14e52aa6e09..0000000000000000000000000000000000000000 --- a/data/developers/shastraai.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "shastraai", - "models": [ - { - "id": "shastraai/Shastra-LLAMA2-Math-Commonsense-SFT", - "name": "Shastra-LLAMA2-Math-Commonsense-SFT", - "developer": "shastraai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3042, - "hfopenllm_v2/BBH": 0.3843, - "hfopenllm_v2/MATH Level 5": 0.0174, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3604, - "hfopenllm_v2/MMLU-PRO": 0.1997 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/shikaichen.json b/data/developers/shikaichen.json deleted file mode 100644 index 6162cba6dec7eaed27af88272ffb98342af2522b..0000000000000000000000000000000000000000 --- a/data/developers/shikaichen.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "developer": "ShikaiChen", - "models": [ - { - "id": "ShikaiChen/LDL-Reward-Gemma-2-27B-v0.1", - "name": "ShikaiChen/LDL-Reward-Gemma-2-27B-v0.1", - "developer": "ShikaiChen", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.9499, - "reward-bench/Factuality": 0.7558, - "reward-bench/Precise IF": 0.35, - "reward-bench/Math": 0.6448, - "reward-bench/Safety": 0.9378, - "reward-bench/Focus": 0.9131, - "reward-bench/Ties": 0.7633, - "reward-bench/Chat": 0.9637, - "reward-bench/Chat Hard": 0.9079, - "reward-bench/Reasoning": 0.9903 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/shivam9980.json b/data/developers/shivam9980.json deleted file mode 100644 index 4b480049d9dffdf995929a88536fbe6580e19b67..0000000000000000000000000000000000000000 --- a/data/developers/shivam9980.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "shivam9980", - "models": [ - { - "id": "shivam9980/mistral-7b-news-cnn-merged", - "name": "mistral-7b-news-cnn-merged", - "developer": "shivam9980", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4634, - "hfopenllm_v2/BBH": 0.3635, - "hfopenllm_v2/MATH Level 5": 0.0189, - "hfopenllm_v2/GPQA": 0.3087, - "hfopenllm_v2/MUSR": 0.4523, - "hfopenllm_v2/MMLU-PRO": 0.2827 - } - }, - { - "id": "shivam9980/NEPALI-LLM", - "name": "NEPALI-LLM", - "developer": "shivam9980", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0417, - "hfopenllm_v2/BBH": 0.3828, - "hfopenllm_v2/MATH Level 5": 0.0091, - "hfopenllm_v2/GPQA": 0.2617, - "hfopenllm_v2/MUSR": 0.4122, - "hfopenllm_v2/MMLU-PRO": 0.2064 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/shivank21.json b/data/developers/shivank21.json deleted file mode 100644 index 41171d70fe0b0a7b8593970b825e48eb5476a53e..0000000000000000000000000000000000000000 --- a/data/developers/shivank21.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "shivank21", - "models": [ - { - "id": "shivank21/mistral_dpo_self", - "name": "mistral_dpo_self", - "developer": "shivank21", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3403, - "hfopenllm_v2/BBH": 0.3216, - "hfopenllm_v2/MATH Level 5": 0.0219, - "hfopenllm_v2/GPQA": 0.2408, - "hfopenllm_v2/MUSR": 0.3247, - "hfopenllm_v2/MMLU-PRO": 0.2214 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/shreyash2010.json b/data/developers/shreyash2010.json deleted file mode 100644 index 0b37aeb427e3399bb567b08cc303f7d6e7c4f332..0000000000000000000000000000000000000000 --- a/data/developers/shreyash2010.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "Shreyash2010", - "models": [ - { - "id": "Shreyash2010/Uma-4x4B-Instruct-v0.1", - "name": "Uma-4x4B-Instruct-v0.1", - "developer": "Shreyash2010", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5517, - "hfopenllm_v2/BBH": 0.5512, - "hfopenllm_v2/MATH Level 5": 0.1775, - "hfopenllm_v2/GPQA": 0.3347, - "hfopenllm_v2/MUSR": 0.4441, - "hfopenllm_v2/MMLU-PRO": 0.387 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/shuttleai.json b/data/developers/shuttleai.json deleted file mode 100644 index bf42fb42f571fa56667c3be3048193995b18d540..0000000000000000000000000000000000000000 --- a/data/developers/shuttleai.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "shuttleai", - "models": [ - { - "id": "shuttleai/shuttle-3", - "name": "shuttle-3", - "developer": "shuttleai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8154, - "hfopenllm_v2/BBH": 0.742, - "hfopenllm_v2/MATH Level 5": 0.46, - "hfopenllm_v2/GPQA": 0.4119, - "hfopenllm_v2/MUSR": 0.4377, - "hfopenllm_v2/MMLU-PRO": 0.5716 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/shyamieee.json b/data/developers/shyamieee.json deleted file mode 100644 index 0cdd6010e858d13672f44add131ae67503b8821d..0000000000000000000000000000000000000000 --- a/data/developers/shyamieee.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "shyamieee", - "models": [ - { - "id": "shyamieee/Padma-v7.0", - "name": "Padma-v7.0", - "developer": "shyamieee", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3841, - "hfopenllm_v2/BBH": 0.5119, - "hfopenllm_v2/MATH Level 5": 0.0702, - "hfopenllm_v2/GPQA": 0.2861, - "hfopenllm_v2/MUSR": 0.4386, - "hfopenllm_v2/MMLU-PRO": 0.3029 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/sicarius-prototyping.json b/data/developers/sicarius-prototyping.json deleted file mode 100644 index 63756f4c626668b117b7d90628ce388cf2afeaa0..0000000000000000000000000000000000000000 --- a/data/developers/sicarius-prototyping.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "developer": "Sicarius-Prototyping", - "models": [ - { - "id": "Sicarius-Prototyping/bacon_and_food", - "name": "bacon_and_food", - "developer": "Sicarius-Prototyping", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.586, - "hfopenllm_v2/BBH": 0.4725, - "hfopenllm_v2/MATH Level 5": 0.0982, - "hfopenllm_v2/GPQA": 0.3096, - "hfopenllm_v2/MUSR": 0.3884, - "hfopenllm_v2/MMLU-PRO": 0.3263 - } - }, - { - "id": "Sicarius-Prototyping/Brainy_LLAMA", - "name": "Brainy_LLAMA", - "developer": "Sicarius-Prototyping", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5204, - "hfopenllm_v2/BBH": 0.5117, - "hfopenllm_v2/MATH Level 5": 0.1337, - "hfopenllm_v2/GPQA": 0.3138, - "hfopenllm_v2/MUSR": 0.4143, - "hfopenllm_v2/MMLU-PRO": 0.3849 - } - }, - { - "id": "Sicarius-Prototyping/Micropenis_1B", - "name": "Micropenis_1B", - "developer": "Sicarius-Prototyping", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3461, - "hfopenllm_v2/BBH": 0.3372, - "hfopenllm_v2/MATH Level 5": 0.0461, - "hfopenllm_v2/GPQA": 0.2626, - "hfopenllm_v2/MUSR": 0.3325, - "hfopenllm_v2/MMLU-PRO": 0.186 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/sicariussicariistuff.json b/data/developers/sicariussicariistuff.json deleted file mode 100644 index 0d2028ef01da9c1d75147248c52590b10b58262a..0000000000000000000000000000000000000000 --- a/data/developers/sicariussicariistuff.json +++ /dev/null @@ -1,271 +0,0 @@ -{ - "developer": "SicariusSicariiStuff", - "models": [ - { - "id": "SicariusSicariiStuff/2B-ad", - "name": "2B-ad", - "developer": "SicariusSicariiStuff", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4379, - "hfopenllm_v2/BBH": 0.4092, - "hfopenllm_v2/MATH Level 5": 0.0506, - "hfopenllm_v2/GPQA": 0.281, - "hfopenllm_v2/MUSR": 0.4015, - "hfopenllm_v2/MMLU-PRO": 0.2662 - } - }, - { - "id": "SicariusSicariiStuff/2B_or_not_2B", - "name": "2B_or_not_2B", - "developer": "SicariusSicariiStuff", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2062, - "hfopenllm_v2/BBH": 0.3416, - "hfopenllm_v2/MATH Level 5": 0.0196, - "hfopenllm_v2/GPQA": 0.2475, - "hfopenllm_v2/MUSR": 0.3791, - "hfopenllm_v2/MMLU-PRO": 0.1399 - } - }, - { - "id": "SicariusSicariiStuff/dn_ep02", - "name": "dn_ep02", - "developer": "SicariusSicariiStuff", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5064, - "hfopenllm_v2/BBH": 0.5266, - "hfopenllm_v2/MATH Level 5": 0.142, - "hfopenllm_v2/GPQA": 0.3154, - "hfopenllm_v2/MUSR": 0.4316, - "hfopenllm_v2/MMLU-PRO": 0.3998 - } - }, - { - "id": "SicariusSicariiStuff/Dusk_Rainbow", - "name": "Dusk_Rainbow", - "developer": "SicariusSicariiStuff", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3588, - "hfopenllm_v2/BBH": 0.4772, - "hfopenllm_v2/MATH Level 5": 0.0748, - "hfopenllm_v2/GPQA": 0.3087, - "hfopenllm_v2/MUSR": 0.4025, - "hfopenllm_v2/MMLU-PRO": 0.3443 - } - }, - { - "id": "SicariusSicariiStuff/Eximius_Persona_5B", - "name": "Eximius_Persona_5B", - "developer": "SicariusSicariiStuff", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.656, - "hfopenllm_v2/BBH": 0.4512, - "hfopenllm_v2/MATH Level 5": 0.102, - "hfopenllm_v2/GPQA": 0.2643, - "hfopenllm_v2/MUSR": 0.3818, - "hfopenllm_v2/MMLU-PRO": 0.314 - } - }, - { - "id": "SicariusSicariiStuff/Impish_LLAMA_3B", - "name": "Impish_LLAMA_3B", - "developer": "SicariusSicariiStuff", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.463, - "hfopenllm_v2/BBH": 0.4091, - "hfopenllm_v2/MATH Level 5": 0.1125, - "hfopenllm_v2/GPQA": 0.2878, - "hfopenllm_v2/MUSR": 0.3673, - "hfopenllm_v2/MMLU-PRO": 0.2941 - } - }, - { - "id": "SicariusSicariiStuff/Impish_Mind_8B", - "name": "Impish_Mind_8B", - "developer": "SicariusSicariiStuff", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3179, - "hfopenllm_v2/BBH": 0.4674, - "hfopenllm_v2/MATH Level 5": 0.105, - "hfopenllm_v2/GPQA": 0.3045, - "hfopenllm_v2/MUSR": 0.407, - "hfopenllm_v2/MMLU-PRO": 0.3309 - } - }, - { - "id": "SicariusSicariiStuff/Impish_QWEN_14B-1M", - "name": "Impish_QWEN_14B-1M", - "developer": "SicariusSicariiStuff", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7868, - "hfopenllm_v2/BBH": 0.6283, - "hfopenllm_v2/MATH Level 5": 0.3965, - "hfopenllm_v2/GPQA": 0.3507, - "hfopenllm_v2/MUSR": 0.4615, - "hfopenllm_v2/MMLU-PRO": 0.5044 - } - }, - { - "id": "SicariusSicariiStuff/Impish_QWEN_7B-1M", - "name": "Impish_QWEN_7B-1M", - "developer": "SicariusSicariiStuff", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6382, - "hfopenllm_v2/BBH": 0.5372, - "hfopenllm_v2/MATH Level 5": 0.3089, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.4074, - "hfopenllm_v2/MMLU-PRO": 0.4265 - } - }, - { - "id": "SicariusSicariiStuff/LLAMA-3_8B_Unaligned_BETA", - "name": "LLAMA-3_8B_Unaligned_BETA", - "developer": "SicariusSicariiStuff", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3713, - "hfopenllm_v2/BBH": 0.4717, - "hfopenllm_v2/MATH Level 5": 0.0838, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.4119, - "hfopenllm_v2/MMLU-PRO": 0.3465 - } - }, - { - "id": "SicariusSicariiStuff/Phi-Line_14B", - "name": "Phi-Line_14B", - "developer": "SicariusSicariiStuff", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6496, - "hfopenllm_v2/BBH": 0.6154, - "hfopenllm_v2/MATH Level 5": 0.386, - "hfopenllm_v2/GPQA": 0.3532, - "hfopenllm_v2/MUSR": 0.4479, - "hfopenllm_v2/MMLU-PRO": 0.5454 - } - }, - { - "id": "SicariusSicariiStuff/Phi-lthy4", - "name": "Phi-lthy4", - "developer": "SicariusSicariiStuff", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7679, - "hfopenllm_v2/BBH": 0.5879, - "hfopenllm_v2/MATH Level 5": 0.1367, - "hfopenllm_v2/GPQA": 0.2869, - "hfopenllm_v2/MUSR": 0.4083, - "hfopenllm_v2/MMLU-PRO": 0.4333 - } - }, - { - "id": "SicariusSicariiStuff/Qwen2.5-14B_Uncencored", - "name": "Qwen2.5-14B_Uncencored", - "developer": "SicariusSicariiStuff", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3158, - "hfopenllm_v2/BBH": 0.6309, - "hfopenllm_v2/MATH Level 5": 0.318, - "hfopenllm_v2/GPQA": 0.3817, - "hfopenllm_v2/MUSR": 0.4517, - "hfopenllm_v2/MMLU-PRO": 0.5266 - } - }, - { - "id": "SicariusSicariiStuff/Qwen2.5-14B_Uncensored", - "name": "Qwen2.5-14B_Uncensored", - "developer": "SicariusSicariiStuff", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3173, - "hfopenllm_v2/BBH": 0.6309, - "hfopenllm_v2/MATH Level 5": 0.318, - "hfopenllm_v2/GPQA": 0.3817, - "hfopenllm_v2/MUSR": 0.4517, - "hfopenllm_v2/MMLU-PRO": 0.5266 - } - }, - { - "id": "SicariusSicariiStuff/Qwen2.5-14B_Uncensored_Instruct", - "name": "Qwen2.5-14B_Uncensored_Instruct", - "developer": "SicariusSicariiStuff", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3789, - "hfopenllm_v2/BBH": 0.5937, - "hfopenllm_v2/MATH Level 5": 0.3285, - "hfopenllm_v2/GPQA": 0.3297, - "hfopenllm_v2/MUSR": 0.3697, - "hfopenllm_v2/MMLU-PRO": 0.5127 - } - }, - { - "id": "SicariusSicariiStuff/Redemption_Wind_24B", - "name": "Redemption_Wind_24B", - "developer": "SicariusSicariiStuff", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2501, - "hfopenllm_v2/BBH": 0.6428, - "hfopenllm_v2/MATH Level 5": 0.1858, - "hfopenllm_v2/GPQA": 0.3834, - "hfopenllm_v2/MUSR": 0.4262, - "hfopenllm_v2/MMLU-PRO": 0.5432 - } - }, - { - "id": "SicariusSicariiStuff/Winged_Imp_8B", - "name": "Winged_Imp_8B", - "developer": "SicariusSicariiStuff", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.743, - "hfopenllm_v2/BBH": 0.512, - "hfopenllm_v2/MATH Level 5": 0.1201, - "hfopenllm_v2/GPQA": 0.2827, - "hfopenllm_v2/MUSR": 0.4148, - "hfopenllm_v2/MMLU-PRO": 0.3639 - } - }, - { - "id": "SicariusSicariiStuff/Wingless_Imp_8B", - "name": "Wingless_Imp_8B", - "developer": "SicariusSicariiStuff", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.743, - "hfopenllm_v2/BBH": 0.512, - "hfopenllm_v2/MATH Level 5": 0.1201, - "hfopenllm_v2/GPQA": 0.2827, - "hfopenllm_v2/MUSR": 0.4148, - "hfopenllm_v2/MMLU-PRO": 0.3639 - } - }, - { - "id": "SicariusSicariiStuff/Zion_Alpha", - "name": "Zion_Alpha", - "developer": "SicariusSicariiStuff", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3324, - "hfopenllm_v2/BBH": 0.4932, - "hfopenllm_v2/MATH Level 5": 0.0521, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.4727, - "hfopenllm_v2/MMLU-PRO": 0.3132 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/silma-ai.json b/data/developers/silma-ai.json deleted file mode 100644 index c223d015a0a261f76d652d17c9574ec7fa11716a..0000000000000000000000000000000000000000 --- a/data/developers/silma-ai.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "silma-ai", - "models": [ - { - "id": "silma-ai/SILMA-9B-Instruct-v1.0", - "name": "SILMA-9B-Instruct-v1.0", - "developer": "silma-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5842, - "hfopenllm_v2/BBH": 0.5219, - "hfopenllm_v2/MATH Level 5": 0.1163, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.4637, - "hfopenllm_v2/MMLU-PRO": 0.392 - } - }, - { - "id": "silma-ai/SILMA-Kashif-2B-Instruct-v1.0", - "name": "SILMA-Kashif-2B-Instruct-v1.0", - "developer": "silma-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1181, - "hfopenllm_v2/BBH": 0.3793, - "hfopenllm_v2/MATH Level 5": 0.0113, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.4043, - "hfopenllm_v2/MMLU-PRO": 0.2258 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/siqi00.json b/data/developers/siqi00.json deleted file mode 100644 index 6129b58a40776dc03eb6e76ca56d3ab5f56ae29d..0000000000000000000000000000000000000000 --- a/data/developers/siqi00.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "siqi00", - "models": [ - { - "id": "siqi00/Mistral-7B-DFT", - "name": "Mistral-7B-DFT", - "developer": "siqi00", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5569, - "hfopenllm_v2/BBH": 0.4665, - "hfopenllm_v2/MATH Level 5": 0.0378, - "hfopenllm_v2/GPQA": 0.3045, - "hfopenllm_v2/MUSR": 0.4191, - "hfopenllm_v2/MMLU-PRO": 0.2963 - } - }, - { - "id": "siqi00/Mistral-7B-DFT2", - "name": "Mistral-7B-DFT2", - "developer": "siqi00", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5804, - "hfopenllm_v2/BBH": 0.3968, - "hfopenllm_v2/MATH Level 5": 0.0453, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.4401, - "hfopenllm_v2/MMLU-PRO": 0.2852 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/skumar9.json b/data/developers/skumar9.json deleted file mode 100644 index 16cb68038507902f71851df1a6b09705dad757dd..0000000000000000000000000000000000000000 --- a/data/developers/skumar9.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "skumar9", - "models": [ - { - "id": "skumar9/Llama-medx_v2", - "name": "Llama-medx_v2", - "developer": "skumar9", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4462, - "hfopenllm_v2/BBH": 0.4909, - "hfopenllm_v2/MATH Level 5": 0.0914, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.3661, - "hfopenllm_v2/MMLU-PRO": 0.3463 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/skymizer.json b/data/developers/skymizer.json deleted file mode 100644 index 2410cff2b8d9ce2e8486e2a7c862f0517c474dbf..0000000000000000000000000000000000000000 --- a/data/developers/skymizer.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "skymizer", - "models": [ - { - "id": "skymizer/Llama2-7b-sft-chat-custom-template-dpo", - "name": "Llama2-7b-sft-chat-custom-template-dpo", - "developer": "skymizer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2353, - "hfopenllm_v2/BBH": 0.3688, - "hfopenllm_v2/MATH Level 5": 0.0144, - "hfopenllm_v2/GPQA": 0.2391, - "hfopenllm_v2/MUSR": 0.4429, - "hfopenllm_v2/MMLU-PRO": 0.1946 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/skyorbis.json b/data/developers/skyorbis.json deleted file mode 100644 index 8bfc544d3f0143c039b4e56871c1f9a9d900a33e..0000000000000000000000000000000000000000 --- a/data/developers/skyorbis.json +++ /dev/null @@ -1,173 +0,0 @@ -{ - "developer": "SkyOrbis", - "models": [ - { - "id": "SkyOrbis/SKY-Ko-Llama3.1-8B-lora", - "name": "SKY-Ko-Llama3.1-8B-lora", - "developer": "SkyOrbis", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5058, - "hfopenllm_v2/BBH": 0.5088, - "hfopenllm_v2/MATH Level 5": 0.1548, - "hfopenllm_v2/GPQA": 0.3213, - "hfopenllm_v2/MUSR": 0.3998, - "hfopenllm_v2/MMLU-PRO": 0.3777 - } - }, - { - "id": "SkyOrbis/SKY-Ko-Llama3.1-8B-lora-epoch1", - "name": "SKY-Ko-Llama3.1-8B-lora-epoch1", - "developer": "SkyOrbis", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5058, - "hfopenllm_v2/BBH": 0.5088, - "hfopenllm_v2/MATH Level 5": 0.1548, - "hfopenllm_v2/GPQA": 0.3213, - "hfopenllm_v2/MUSR": 0.3998, - "hfopenllm_v2/MMLU-PRO": 0.3777 - } - }, - { - "id": "SkyOrbis/SKY-Ko-Llama3.2-1B-lora-epoch3", - "name": "SKY-Ko-Llama3.2-1B-lora-epoch3", - "developer": "SkyOrbis", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3247, - "hfopenllm_v2/BBH": 0.3167, - "hfopenllm_v2/MATH Level 5": 0.0272, - "hfopenllm_v2/GPQA": 0.2517, - "hfopenllm_v2/MUSR": 0.3382, - "hfopenllm_v2/MMLU-PRO": 0.1279 - } - }, - { - "id": "SkyOrbis/SKY-Ko-Llama3.2-1B-lora-epoch5", - "name": "SKY-Ko-Llama3.2-1B-lora-epoch5", - "developer": "SkyOrbis", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.436, - "hfopenllm_v2/BBH": 0.3406, - "hfopenllm_v2/MATH Level 5": 0.0521, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3471, - "hfopenllm_v2/MMLU-PRO": 0.1946 - } - }, - { - "id": "SkyOrbis/SKY-Ko-Llama3.2-1B-lora-v2-epoch3", - "name": "SKY-Ko-Llama3.2-1B-lora-v2-epoch3", - "developer": "SkyOrbis", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.436, - "hfopenllm_v2/BBH": 0.3406, - "hfopenllm_v2/MATH Level 5": 0.0521, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3471, - "hfopenllm_v2/MMLU-PRO": 0.1946 - } - }, - { - "id": "SkyOrbis/SKY-Ko-Llama3.2-1B-lora-v2-epoch5", - "name": "SKY-Ko-Llama3.2-1B-lora-v2-epoch5", - "developer": "SkyOrbis", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4247, - "hfopenllm_v2/BBH": 0.3397, - "hfopenllm_v2/MATH Level 5": 0.0506, - "hfopenllm_v2/GPQA": 0.2542, - "hfopenllm_v2/MUSR": 0.3458, - "hfopenllm_v2/MMLU-PRO": 0.1946 - } - }, - { - "id": "SkyOrbis/SKY-Ko-Llama3.2-3B-lora-epoch1", - "name": "SKY-Ko-Llama3.2-3B-lora-epoch1", - "developer": "SkyOrbis", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5331, - "hfopenllm_v2/BBH": 0.44, - "hfopenllm_v2/MATH Level 5": 0.1458, - "hfopenllm_v2/GPQA": 0.2919, - "hfopenllm_v2/MUSR": 0.3522, - "hfopenllm_v2/MMLU-PRO": 0.3004 - } - }, - { - "id": "SkyOrbis/SKY-Ko-Llama3.2-3B-lora-epoch2", - "name": "SKY-Ko-Llama3.2-3B-lora-epoch2", - "developer": "SkyOrbis", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5331, - "hfopenllm_v2/BBH": 0.44, - "hfopenllm_v2/MATH Level 5": 0.1458, - "hfopenllm_v2/GPQA": 0.2919, - "hfopenllm_v2/MUSR": 0.3522, - "hfopenllm_v2/MMLU-PRO": 0.3004 - } - }, - { - "id": "SkyOrbis/SKY-Ko-Llama3.2-3B-lora-epoch3", - "name": "SKY-Ko-Llama3.2-3B-lora-epoch3", - "developer": "SkyOrbis", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5331, - "hfopenllm_v2/BBH": 0.44, - "hfopenllm_v2/MATH Level 5": 0.1458, - "hfopenllm_v2/GPQA": 0.2919, - "hfopenllm_v2/MUSR": 0.3522, - "hfopenllm_v2/MMLU-PRO": 0.3004 - } - }, - { - "id": "SkyOrbis/SKY-Ko-Qwen2.5-3B-Instruct", - "name": "SKY-Ko-Qwen2.5-3B-Instruct", - "developer": "SkyOrbis", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3534, - "hfopenllm_v2/BBH": 0.4265, - "hfopenllm_v2/MATH Level 5": 0.0695, - "hfopenllm_v2/GPQA": 0.2794, - "hfopenllm_v2/MUSR": 0.4024, - "hfopenllm_v2/MMLU-PRO": 0.2812 - } - }, - { - "id": "SkyOrbis/SKY-Ko-Qwen2.5-7B-Instruct-SFT-step-15000", - "name": "SKY-Ko-Qwen2.5-7B-Instruct-SFT-step-15000", - "developer": "SkyOrbis", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3819, - "hfopenllm_v2/BBH": 0.5078, - "hfopenllm_v2/MATH Level 5": 0.1866, - "hfopenllm_v2/GPQA": 0.3272, - "hfopenllm_v2/MUSR": 0.4436, - "hfopenllm_v2/MMLU-PRO": 0.3914 - } - }, - { - "id": "SkyOrbis/SKY-Ko-Qwen2.5-7B-Instruct-SFT-step-5000", - "name": "SKY-Ko-Qwen2.5-7B-Instruct-SFT-step-5000", - "developer": "SkyOrbis", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3812, - "hfopenllm_v2/BBH": 0.539, - "hfopenllm_v2/MATH Level 5": 0.21, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.4238, - "hfopenllm_v2/MMLU-PRO": 0.4238 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/skywork.json b/data/developers/skywork.json deleted file mode 100644 index b75ac4fa3e86dd1c341887c14306e39e694944d2..0000000000000000000000000000000000000000 --- a/data/developers/skywork.json +++ /dev/null @@ -1,246 +0,0 @@ -{ - "developer": "Skywork", - "models": [ - { - "id": "Skywork/Skywork-Critic-Llama-3.1-70B", - "name": "Skywork/Skywork-Critic-Llama-3.1-70B", - "developer": "Skywork", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.9331, - "reward-bench/Chat": 0.9665, - "reward-bench/Chat Hard": 0.8794, - "reward-bench/Safety": 0.9311, - "reward-bench/Reasoning": 0.9554 - } - }, - { - "id": "Skywork/Skywork-Critic-Llama-3.1-8B", - "name": "Skywork/Skywork-Critic-Llama-3.1-8B", - "developer": "Skywork", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.8896, - "reward-bench/Chat": 0.9358, - "reward-bench/Chat Hard": 0.8136, - "reward-bench/Safety": 0.9108, - "reward-bench/Reasoning": 0.898 - } - }, - { - "id": "Skywork/Skywork-o1-Open-Llama-3.1-8B", - "name": "Skywork-o1-Open-Llama-3.1-8B", - "developer": "Skywork", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3518, - "hfopenllm_v2/BBH": 0.4516, - "hfopenllm_v2/MATH Level 5": 0.5211, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3156, - "hfopenllm_v2/MMLU-PRO": 0.203 - } - }, - { - "id": "Skywork/Skywork-Reward-Gemma-2-27B", - "name": "Skywork/Skywork-Reward-Gemma-2-27B", - "developer": "Skywork", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.938, - "reward-bench/Factuality": 0.7368, - "reward-bench/Precise IF": 0.4031, - "reward-bench/Math": 0.7049, - "reward-bench/Safety": 0.9189, - "reward-bench/Focus": 0.9323, - "reward-bench/Ties": 0.8261, - "reward-bench/Chat": 0.9581, - "reward-bench/Chat Hard": 0.9145, - "reward-bench/Reasoning": 0.9606 - } - }, - { - "id": "Skywork/Skywork-Reward-Gemma-2-27B-v0.2", - "name": "Skywork/Skywork-Reward-Gemma-2-27B-v0.2", - "developer": "Skywork", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7807, - "hfopenllm_v2/BBH": 0.636, - "hfopenllm_v2/MATH Level 5": 0.2273, - "hfopenllm_v2/GPQA": 0.344, - "hfopenllm_v2/MUSR": 0.4231, - "hfopenllm_v2/MMLU-PRO": 0.4103, - "reward-bench/Score": 0.9426, - "reward-bench/Factuality": 0.7674, - "reward-bench/Precise IF": 0.375, - "reward-bench/Math": 0.6721, - "reward-bench/Safety": 0.9297, - "reward-bench/Focus": 0.9172, - "reward-bench/Ties": 0.8182, - "reward-bench/Chat": 0.9609, - "reward-bench/Chat Hard": 0.8991, - "reward-bench/Reasoning": 0.9807 - } - }, - { - "id": "Skywork/Skywork-Reward-Llama-3.1-8B", - "name": "Skywork/Skywork-Reward-Llama-3.1-8B", - "developer": "Skywork", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7314, - "reward-bench/Chat": 0.9581, - "reward-bench/Chat Hard": 0.8728, - "reward-bench/Safety": 0.9333, - "reward-bench/Reasoning": 0.962, - "reward-bench/Factuality": 0.6989, - "reward-bench/Precise IF": 0.425, - "reward-bench/Math": 0.6284, - "reward-bench/Focus": 0.9616, - "reward-bench/Ties": 0.741 - } - }, - { - "id": "Skywork/Skywork-Reward-Llama-3.1-8B-v0.2", - "name": "Skywork/Skywork-Reward-Llama-3.1-8B-v0.2", - "developer": "Skywork", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7175, - "reward-bench/Chat": 0.9469, - "reward-bench/Chat Hard": 0.8838, - "reward-bench/Safety": 0.9422, - "reward-bench/Reasoning": 0.9675, - "reward-bench/Factuality": 0.6968, - "reward-bench/Precise IF": 0.4062, - "reward-bench/Math": 0.6011, - "reward-bench/Focus": 0.9414, - "reward-bench/Ties": 0.7169 - } - }, - { - "id": "Skywork/Skywork-Reward-V2-Llama-3.1-8B", - "name": "Skywork/Skywork-Reward-V2-Llama-3.1-8B", - "developer": "Skywork", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.8413, - "reward-bench/Factuality": 0.8463, - "reward-bench/Precise IF": 0.6625, - "reward-bench/Math": 0.776, - "reward-bench/Safety": 0.9667, - "reward-bench/Focus": 0.9838, - "reward-bench/Ties": 0.8124 - } - }, - { - "id": "Skywork/Skywork-Reward-V2-Llama-3.2-1B", - "name": "Skywork/Skywork-Reward-V2-Llama-3.2-1B", - "developer": "Skywork", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6438, - "reward-bench/Factuality": 0.6084, - "reward-bench/Precise IF": 0.4562, - "reward-bench/Math": 0.6011, - "reward-bench/Safety": 0.8733, - "reward-bench/Focus": 0.8929, - "reward-bench/Ties": 0.4306 - } - }, - { - "id": "Skywork/Skywork-Reward-V2-Llama-3.2-3B", - "name": "Skywork/Skywork-Reward-V2-Llama-3.2-3B", - "developer": "Skywork", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7466, - "reward-bench/Factuality": 0.7621, - "reward-bench/Precise IF": 0.4562, - "reward-bench/Math": 0.694, - "reward-bench/Safety": 0.9311, - "reward-bench/Focus": 0.9596, - "reward-bench/Ties": 0.6768 - } - }, - { - "id": "Skywork/Skywork-Reward-V2-Qwen3-0.6B", - "name": "Skywork/Skywork-Reward-V2-Qwen3-0.6B", - "developer": "Skywork", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6125, - "reward-bench/Factuality": 0.58, - "reward-bench/Precise IF": 0.4, - "reward-bench/Math": 0.7158, - "reward-bench/Safety": 0.8444, - "reward-bench/Focus": 0.7949, - "reward-bench/Ties": 0.3397 - } - }, - { - "id": "Skywork/Skywork-Reward-V2-Qwen3-1.7B", - "name": "Skywork/Skywork-Reward-V2-Qwen3-1.7B", - "developer": "Skywork", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6818, - "reward-bench/Factuality": 0.6568, - "reward-bench/Precise IF": 0.4437, - "reward-bench/Math": 0.7268, - "reward-bench/Safety": 0.8911, - "reward-bench/Focus": 0.8848, - "reward-bench/Ties": 0.4872 - } - }, - { - "id": "Skywork/Skywork-Reward-V2-Qwen3-4B", - "name": "Skywork/Skywork-Reward-V2-Qwen3-4B", - "developer": "Skywork", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7551, - "reward-bench/Factuality": 0.7737, - "reward-bench/Precise IF": 0.4625, - "reward-bench/Math": 0.7322, - "reward-bench/Safety": 0.9222, - "reward-bench/Focus": 0.9657, - "reward-bench/Ties": 0.6743 - } - }, - { - "id": "Skywork/Skywork-Reward-V2-Qwen3-8B", - "name": "Skywork/Skywork-Reward-V2-Qwen3-8B", - "developer": "Skywork", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7837, - "reward-bench/Factuality": 0.7989, - "reward-bench/Precise IF": 0.5, - "reward-bench/Math": 0.7705, - "reward-bench/Safety": 0.94, - "reward-bench/Focus": 0.9636, - "reward-bench/Ties": 0.7294 - } - }, - { - "id": "Skywork/Skywork-VL-Reward-7B", - "name": "Skywork/Skywork-VL-Reward-7B", - "developer": "Skywork", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.9007, - "reward-bench/Factuality": 0.6063, - "reward-bench/Precise IF": 0.35, - "reward-bench/Math": 0.6339, - "reward-bench/Safety": 0.9108, - "reward-bench/Focus": 0.8909, - "reward-bench/Ties": 0.7586, - "reward-bench/Chat": 0.8994, - "reward-bench/Chat Hard": 0.875, - "reward-bench/Reasoning": 0.9176 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/snowflake.json b/data/developers/snowflake.json deleted file mode 100644 index d36a86d899a25b9271cd05a105d08de1ef5c396f..0000000000000000000000000000000000000000 --- a/data/developers/snowflake.json +++ /dev/null @@ -1,59 +0,0 @@ -{ - "developer": "snowflake", - "models": [ - { - "id": "snowflake/snowflake-arctic-instruct", - "name": "Arctic Instruct", - "developer": "snowflake", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.338, - "helm_lite/NarrativeQA": 0.654, - "helm_lite/NaturalQuestions (closed-book)": 0.39, - "helm_lite/OpenbookQA": 0.828, - "helm_lite/MMLU": 0.575, - "helm_lite/MATH": 0.519, - "helm_lite/GSM8K": 0.768, - "helm_lite/LegalBench": 0.588, - "helm_lite/MedQA": 0.581, - "helm_lite/WMT 2014": 0.172, - "helm_mmlu/MMLU All Subjects": 0.677, - "helm_mmlu/Abstract Algebra": 0.35, - "helm_mmlu/Anatomy": 0.652, - "helm_mmlu/College Physics": 0.461, - "helm_mmlu/Computer Security": 0.84, - "helm_mmlu/Econometrics": 0.5, - "helm_mmlu/Global Facts": 0.39, - "helm_mmlu/Jurisprudence": 0.741, - "helm_mmlu/Philosophy": 0.752, - "helm_mmlu/Professional Psychology": 0.724, - "helm_mmlu/Us Foreign Policy": 0.88, - "helm_mmlu/Astronomy": 0.763, - "helm_mmlu/Business Ethics": 0.69, - "helm_mmlu/Clinical Knowledge": 0.781, - "helm_mmlu/Conceptual Physics": 0.634, - "helm_mmlu/Electrical Engineering": 0.662, - "helm_mmlu/Elementary Mathematics": 0.481, - "helm_mmlu/Formal Logic": 0.444, - "helm_mmlu/High School World History": 0.827, - "helm_mmlu/Human Sexuality": 0.847, - "helm_mmlu/International Law": 0.826, - "helm_mmlu/Logical Fallacies": 0.779, - "helm_mmlu/Machine Learning": 0.473, - "helm_mmlu/Management": 0.796, - "helm_mmlu/Marketing": 0.902, - "helm_mmlu/Medical Genetics": 0.76, - "helm_mmlu/Miscellaneous": 0.875, - "helm_mmlu/Moral Scenarios": 0.28, - "helm_mmlu/Nutrition": 0.725, - "helm_mmlu/Prehistory": 0.79, - "helm_mmlu/Public Relations": 0.664, - "helm_mmlu/Security Studies": 0.78, - "helm_mmlu/Sociology": 0.891, - "helm_mmlu/Virology": 0.536, - "helm_mmlu/World Religions": 0.854, - "helm_mmlu/Mean win rate": 0.565 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/solshine.json b/data/developers/solshine.json deleted file mode 100644 index e830a92839463f45e681eaf15d27461bf7e55475..0000000000000000000000000000000000000000 --- a/data/developers/solshine.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "Solshine", - "models": [ - { - "id": "Solshine/Brimful-merged-replete", - "name": "Brimful-merged-replete", - "developer": "Solshine", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1761, - "hfopenllm_v2/BBH": 0.2883, - "hfopenllm_v2/MATH Level 5": 0.003, - "hfopenllm_v2/GPQA": 0.2576, - "hfopenllm_v2/MUSR": 0.3421, - "hfopenllm_v2/MMLU-PRO": 0.1085 - } - }, - { - "id": "Solshine/Llama-3-1-big-thoughtful-passthrough-merge-2", - "name": "Llama-3-1-big-thoughtful-passthrough-merge-2", - "developer": "Solshine", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2547, - "hfopenllm_v2/BBH": 0.3209, - "hfopenllm_v2/MATH Level 5": 0.0106, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3889, - "hfopenllm_v2/MMLU-PRO": 0.1185 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/someon98.json b/data/developers/someon98.json deleted file mode 100644 index ae9aa961e9a6fc0b2ae82aec18c44097dd97e780..0000000000000000000000000000000000000000 --- a/data/developers/someon98.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "someon98", - "models": [ - { - "id": "someon98/qwen-CoMa-0.5b", - "name": "qwen-CoMa-0.5b", - "developer": "someon98", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2277, - "hfopenllm_v2/BBH": 0.2953, - "hfopenllm_v2/MATH Level 5": 0.0045, - "hfopenllm_v2/GPQA": 0.2399, - "hfopenllm_v2/MUSR": 0.4046, - "hfopenllm_v2/MMLU-PRO": 0.1099 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/sometimesanotion.json b/data/developers/sometimesanotion.json deleted file mode 100644 index c020e85fc5c690d215cbd09f94a114e50c6a6936..0000000000000000000000000000000000000000 --- a/data/developers/sometimesanotion.json +++ /dev/null @@ -1,817 +0,0 @@ -{ - "developer": "sometimesanotion", - "models": [ - { - "id": "sometimesanotion/ChocoTrio-14B-v1", - "name": "ChocoTrio-14B-v1", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7089, - "hfopenllm_v2/BBH": 0.6506, - "hfopenllm_v2/MATH Level 5": 0.3973, - "hfopenllm_v2/GPQA": 0.3851, - "hfopenllm_v2/MUSR": 0.4821, - "hfopenllm_v2/MMLU-PRO": 0.537 - } - }, - { - "id": "sometimesanotion/IF-reasoning-experiment-40", - "name": "IF-reasoning-experiment-40", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.633, - "hfopenllm_v2/BBH": 0.6112, - "hfopenllm_v2/MATH Level 5": 0.3716, - "hfopenllm_v2/GPQA": 0.38, - "hfopenllm_v2/MUSR": 0.5194, - "hfopenllm_v2/MMLU-PRO": 0.5025 - } - }, - { - "id": "sometimesanotion/IF-reasoning-experiment-80", - "name": "IF-reasoning-experiment-80", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5463, - "hfopenllm_v2/BBH": 0.421, - "hfopenllm_v2/MATH Level 5": 0.0989, - "hfopenllm_v2/GPQA": 0.2844, - "hfopenllm_v2/MUSR": 0.5025, - "hfopenllm_v2/MMLU-PRO": 0.3368 - } - }, - { - "id": "sometimesanotion/KytheraMix-7B-v0.2", - "name": "KytheraMix-7B-v0.2", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6129, - "hfopenllm_v2/BBH": 0.5635, - "hfopenllm_v2/MATH Level 5": 0.2923, - "hfopenllm_v2/GPQA": 0.3356, - "hfopenllm_v2/MUSR": 0.4594, - "hfopenllm_v2/MMLU-PRO": 0.4505 - } - }, - { - "id": "sometimesanotion/lamarck-14b-prose-model_stock", - "name": "lamarck-14b-prose-model_stock", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4276, - "hfopenllm_v2/BBH": 0.6488, - "hfopenllm_v2/MATH Level 5": 0.3414, - "hfopenllm_v2/GPQA": 0.3935, - "hfopenllm_v2/MUSR": 0.4846, - "hfopenllm_v2/MMLU-PRO": 0.5354 - } - }, - { - "id": "sometimesanotion/lamarck-14b-reason-model_stock", - "name": "lamarck-14b-reason-model_stock", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4965, - "hfopenllm_v2/BBH": 0.6569, - "hfopenllm_v2/MATH Level 5": 0.358, - "hfopenllm_v2/GPQA": 0.3842, - "hfopenllm_v2/MUSR": 0.4741, - "hfopenllm_v2/MMLU-PRO": 0.5402 - } - }, - { - "id": "sometimesanotion/Lamarck-14B-v0.1-experimental", - "name": "Lamarck-14B-v0.1-experimental", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5354, - "hfopenllm_v2/BBH": 0.6583, - "hfopenllm_v2/MATH Level 5": 0.358, - "hfopenllm_v2/GPQA": 0.3817, - "hfopenllm_v2/MUSR": 0.4728, - "hfopenllm_v2/MMLU-PRO": 0.5408 - } - }, - { - "id": "sometimesanotion/Lamarck-14B-v0.3", - "name": "Lamarck-14B-v0.3", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5032, - "hfopenllm_v2/BBH": 0.6611, - "hfopenllm_v2/MATH Level 5": 0.3406, - "hfopenllm_v2/GPQA": 0.3884, - "hfopenllm_v2/MUSR": 0.4688, - "hfopenllm_v2/MMLU-PRO": 0.5411 - } - }, - { - "id": "sometimesanotion/Lamarck-14B-v0.4-Qwenvergence", - "name": "Lamarck-14B-v0.4-Qwenvergence", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4906, - "hfopenllm_v2/BBH": 0.6535, - "hfopenllm_v2/MATH Level 5": 0.3399, - "hfopenllm_v2/GPQA": 0.3784, - "hfopenllm_v2/MUSR": 0.4847, - "hfopenllm_v2/MMLU-PRO": 0.5406 - } - }, - { - "id": "sometimesanotion/Lamarck-14B-v0.6", - "name": "Lamarck-14B-v0.6", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6973, - "hfopenllm_v2/BBH": 0.646, - "hfopenllm_v2/MATH Level 5": 0.4041, - "hfopenllm_v2/GPQA": 0.3893, - "hfopenllm_v2/MUSR": 0.4847, - "hfopenllm_v2/MMLU-PRO": 0.54 - } - }, - { - "id": "sometimesanotion/Lamarck-14B-v0.6-002-model_stock", - "name": "Lamarck-14B-v0.6-002-model_stock", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6692, - "hfopenllm_v2/BBH": 0.6143, - "hfopenllm_v2/MATH Level 5": 0.3776, - "hfopenllm_v2/GPQA": 0.3742, - "hfopenllm_v2/MUSR": 0.518, - "hfopenllm_v2/MMLU-PRO": 0.5054 - } - }, - { - "id": "sometimesanotion/Lamarck-14B-v0.6-model_stock", - "name": "Lamarck-14B-v0.6-model_stock", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.679, - "hfopenllm_v2/BBH": 0.6269, - "hfopenllm_v2/MATH Level 5": 0.4245, - "hfopenllm_v2/GPQA": 0.3842, - "hfopenllm_v2/MUSR": 0.5007, - "hfopenllm_v2/MMLU-PRO": 0.5198 - } - }, - { - "id": "sometimesanotion/Lamarck-14B-v0.7-Fusion", - "name": "Lamarck-14B-v0.7-Fusion", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6821, - "hfopenllm_v2/BBH": 0.6544, - "hfopenllm_v2/MATH Level 5": 0.4041, - "hfopenllm_v2/GPQA": 0.401, - "hfopenllm_v2/MUSR": 0.4991, - "hfopenllm_v2/MMLU-PRO": 0.5391 - } - }, - { - "id": "sometimesanotion/Lamarck-14B-v0.7-rc1", - "name": "Lamarck-14B-v0.7-rc1", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7305, - "hfopenllm_v2/BBH": 0.6486, - "hfopenllm_v2/MATH Level 5": 0.3852, - "hfopenllm_v2/GPQA": 0.3893, - "hfopenllm_v2/MUSR": 0.4715, - "hfopenllm_v2/MMLU-PRO": 0.5416 - } - }, - { - "id": "sometimesanotion/Lamarck-14B-v0.7-rc4", - "name": "Lamarck-14B-v0.7-rc4", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7211, - "hfopenllm_v2/BBH": 0.651, - "hfopenllm_v2/MATH Level 5": 0.4026, - "hfopenllm_v2/GPQA": 0.3893, - "hfopenllm_v2/MUSR": 0.4912, - "hfopenllm_v2/MMLU-PRO": 0.54 - } - }, - { - "id": "sometimesanotion/LamarckInfusion-14B-v1", - "name": "LamarckInfusion-14B-v1", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7198, - "hfopenllm_v2/BBH": 0.6539, - "hfopenllm_v2/MATH Level 5": 0.4169, - "hfopenllm_v2/GPQA": 0.3909, - "hfopenllm_v2/MUSR": 0.4899, - "hfopenllm_v2/MMLU-PRO": 0.5376 - } - }, - { - "id": "sometimesanotion/LamarckInfusion-14B-v2", - "name": "LamarckInfusion-14B-v2", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6812, - "hfopenllm_v2/BBH": 0.6564, - "hfopenllm_v2/MATH Level 5": 0.4388, - "hfopenllm_v2/GPQA": 0.3876, - "hfopenllm_v2/MUSR": 0.4993, - "hfopenllm_v2/MMLU-PRO": 0.5416 - } - }, - { - "id": "sometimesanotion/LamarckInfusion-14B-v2-hi", - "name": "LamarckInfusion-14B-v2-hi", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6855, - "hfopenllm_v2/BBH": 0.6555, - "hfopenllm_v2/MATH Level 5": 0.423, - "hfopenllm_v2/GPQA": 0.3884, - "hfopenllm_v2/MUSR": 0.4847, - "hfopenllm_v2/MMLU-PRO": 0.5405 - } - }, - { - "id": "sometimesanotion/LamarckInfusion-14B-v2-lo", - "name": "LamarckInfusion-14B-v2-lo", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6788, - "hfopenllm_v2/BBH": 0.6528, - "hfopenllm_v2/MATH Level 5": 0.4237, - "hfopenllm_v2/GPQA": 0.3859, - "hfopenllm_v2/MUSR": 0.4991, - "hfopenllm_v2/MMLU-PRO": 0.5397 - } - }, - { - "id": "sometimesanotion/LamarckInfusion-14B-v3", - "name": "LamarckInfusion-14B-v3", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7131, - "hfopenllm_v2/BBH": 0.6518, - "hfopenllm_v2/MATH Level 5": 0.4124, - "hfopenllm_v2/GPQA": 0.3867, - "hfopenllm_v2/MUSR": 0.482, - "hfopenllm_v2/MMLU-PRO": 0.5407 - } - }, - { - "id": "sometimesanotion/Qwen-14B-ProseStock-v4", - "name": "Qwen-14B-ProseStock-v4", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4942, - "hfopenllm_v2/BBH": 0.6498, - "hfopenllm_v2/MATH Level 5": 0.364, - "hfopenllm_v2/GPQA": 0.3884, - "hfopenllm_v2/MUSR": 0.4938, - "hfopenllm_v2/MMLU-PRO": 0.5386 - } - }, - { - "id": "sometimesanotion/Qwen-2.5-14B-Virmarckeoso", - "name": "Qwen-2.5-14B-Virmarckeoso", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4813, - "hfopenllm_v2/BBH": 0.657, - "hfopenllm_v2/MATH Level 5": 0.3565, - "hfopenllm_v2/GPQA": 0.3792, - "hfopenllm_v2/MUSR": 0.4794, - "hfopenllm_v2/MMLU-PRO": 0.5377 - } - }, - { - "id": "sometimesanotion/Qwen2.5-14B-Vimarckoso", - "name": "Qwen2.5-14B-Vimarckoso", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4574, - "hfopenllm_v2/BBH": 0.6446, - "hfopenllm_v2/MATH Level 5": 0.3384, - "hfopenllm_v2/GPQA": 0.3926, - "hfopenllm_v2/MUSR": 0.4859, - "hfopenllm_v2/MMLU-PRO": 0.5329 - } - }, - { - "id": "sometimesanotion/Qwen2.5-14B-Vimarckoso-v2", - "name": "Qwen2.5-14B-Vimarckoso-v2", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4505, - "hfopenllm_v2/BBH": 0.655, - "hfopenllm_v2/MATH Level 5": 0.358, - "hfopenllm_v2/GPQA": 0.3826, - "hfopenllm_v2/MUSR": 0.4819, - "hfopenllm_v2/MMLU-PRO": 0.538 - } - }, - { - "id": "sometimesanotion/Qwen2.5-14B-Vimarckoso-v3", - "name": "Qwen2.5-14B-Vimarckoso-v3", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7257, - "hfopenllm_v2/BBH": 0.6415, - "hfopenllm_v2/MATH Level 5": 0.4003, - "hfopenllm_v2/GPQA": 0.38, - "hfopenllm_v2/MUSR": 0.4807, - "hfopenllm_v2/MMLU-PRO": 0.5343 - } - }, - { - "id": "sometimesanotion/Qwen2.5-14B-Vimarckoso-v3-IF-Variant", - "name": "Qwen2.5-14B-Vimarckoso-v3-IF-Variant", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6413, - "hfopenllm_v2/BBH": 0.5521, - "hfopenllm_v2/MATH Level 5": 0.2545, - "hfopenllm_v2/GPQA": 0.3473, - "hfopenllm_v2/MUSR": 0.5319, - "hfopenllm_v2/MMLU-PRO": 0.4589 - } - }, - { - "id": "sometimesanotion/Qwen2.5-14B-Vimarckoso-v3-model_stock", - "name": "Qwen2.5-14B-Vimarckoso-v3-model_stock", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7162, - "hfopenllm_v2/BBH": 0.6421, - "hfopenllm_v2/MATH Level 5": 0.4245, - "hfopenllm_v2/GPQA": 0.38, - "hfopenllm_v2/MUSR": 0.4781, - "hfopenllm_v2/MMLU-PRO": 0.5316 - } - }, - { - "id": "sometimesanotion/Qwen2.5-14B-Vimarckoso-v3-Prose01", - "name": "Qwen2.5-14B-Vimarckoso-v3-Prose01", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6872, - "hfopenllm_v2/BBH": 0.6359, - "hfopenllm_v2/MATH Level 5": 0.3995, - "hfopenllm_v2/GPQA": 0.3867, - "hfopenllm_v2/MUSR": 0.4807, - "hfopenllm_v2/MMLU-PRO": 0.5275 - } - }, - { - "id": "sometimesanotion/Qwen2.5-7B-Gordion-v0.1", - "name": "Qwen2.5-7B-Gordion-v0.1", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7482, - "hfopenllm_v2/BBH": 0.5524, - "hfopenllm_v2/MATH Level 5": 0.2915, - "hfopenllm_v2/GPQA": 0.3079, - "hfopenllm_v2/MUSR": 0.4016, - "hfopenllm_v2/MMLU-PRO": 0.43 - } - }, - { - "id": "sometimesanotion/Qwen2.5-7B-Gordion-v0.1-Prose", - "name": "Qwen2.5-7B-Gordion-v0.1-Prose", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5347, - "hfopenllm_v2/BBH": 0.5599, - "hfopenllm_v2/MATH Level 5": 0.2893, - "hfopenllm_v2/GPQA": 0.3205, - "hfopenllm_v2/MUSR": 0.4502, - "hfopenllm_v2/MMLU-PRO": 0.4525 - } - }, - { - "id": "sometimesanotion/Qwen2.5-7B-Gordion-v0.1-Reason", - "name": "Qwen2.5-7B-Gordion-v0.1-Reason", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4917, - "hfopenllm_v2/BBH": 0.5498, - "hfopenllm_v2/MATH Level 5": 0.2621, - "hfopenllm_v2/GPQA": 0.3406, - "hfopenllm_v2/MUSR": 0.4434, - "hfopenllm_v2/MMLU-PRO": 0.4307 - } - }, - { - "id": "sometimesanotion/Qwentessential-14B-v1", - "name": "Qwentessential-14B-v1", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6279, - "hfopenllm_v2/BBH": 0.6545, - "hfopenllm_v2/MATH Level 5": 0.4071, - "hfopenllm_v2/GPQA": 0.3876, - "hfopenllm_v2/MUSR": 0.4873, - "hfopenllm_v2/MMLU-PRO": 0.5381 - } - }, - { - "id": "sometimesanotion/Qwentinuum-14B-v013", - "name": "Qwentinuum-14B-v013", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6711, - "hfopenllm_v2/BBH": 0.6087, - "hfopenllm_v2/MATH Level 5": 0.3708, - "hfopenllm_v2/GPQA": 0.3574, - "hfopenllm_v2/MUSR": 0.5154, - "hfopenllm_v2/MMLU-PRO": 0.4991 - } - }, - { - "id": "sometimesanotion/Qwentinuum-14B-v1", - "name": "Qwentinuum-14B-v1", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5032, - "hfopenllm_v2/BBH": 0.6573, - "hfopenllm_v2/MATH Level 5": 0.3603, - "hfopenllm_v2/GPQA": 0.3826, - "hfopenllm_v2/MUSR": 0.4781, - "hfopenllm_v2/MMLU-PRO": 0.541 - } - }, - { - "id": "sometimesanotion/Qwentinuum-14B-v2", - "name": "Qwentinuum-14B-v2", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5378, - "hfopenllm_v2/BBH": 0.6555, - "hfopenllm_v2/MATH Level 5": 0.3754, - "hfopenllm_v2/GPQA": 0.3884, - "hfopenllm_v2/MUSR": 0.4714, - "hfopenllm_v2/MMLU-PRO": 0.5409 - } - }, - { - "id": "sometimesanotion/Qwentinuum-14B-v3", - "name": "Qwentinuum-14B-v3", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6158, - "hfopenllm_v2/BBH": 0.6539, - "hfopenllm_v2/MATH Level 5": 0.3535, - "hfopenllm_v2/GPQA": 0.3876, - "hfopenllm_v2/MUSR": 0.486, - "hfopenllm_v2/MMLU-PRO": 0.5413 - } - }, - { - "id": "sometimesanotion/Qwentinuum-14B-v5", - "name": "Qwentinuum-14B-v5", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6286, - "hfopenllm_v2/BBH": 0.655, - "hfopenllm_v2/MATH Level 5": 0.3444, - "hfopenllm_v2/GPQA": 0.3876, - "hfopenllm_v2/MUSR": 0.4874, - "hfopenllm_v2/MMLU-PRO": 0.5418 - } - }, - { - "id": "sometimesanotion/Qwentinuum-14B-v6", - "name": "Qwentinuum-14B-v6", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6304, - "hfopenllm_v2/BBH": 0.6545, - "hfopenllm_v2/MATH Level 5": 0.3603, - "hfopenllm_v2/GPQA": 0.3867, - "hfopenllm_v2/MUSR": 0.49, - "hfopenllm_v2/MMLU-PRO": 0.54 - } - }, - { - "id": "sometimesanotion/Qwentinuum-14B-v6-Prose", - "name": "Qwentinuum-14B-v6-Prose", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5643, - "hfopenllm_v2/BBH": 0.6545, - "hfopenllm_v2/MATH Level 5": 0.3701, - "hfopenllm_v2/GPQA": 0.3884, - "hfopenllm_v2/MUSR": 0.4913, - "hfopenllm_v2/MMLU-PRO": 0.5392 - } - }, - { - "id": "sometimesanotion/Qwentinuum-14B-v7", - "name": "Qwentinuum-14B-v7", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6109, - "hfopenllm_v2/BBH": 0.6551, - "hfopenllm_v2/MATH Level 5": 0.3573, - "hfopenllm_v2/GPQA": 0.3909, - "hfopenllm_v2/MUSR": 0.482, - "hfopenllm_v2/MMLU-PRO": 0.541 - } - }, - { - "id": "sometimesanotion/Qwentinuum-14B-v8", - "name": "Qwentinuum-14B-v8", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5412, - "hfopenllm_v2/BBH": 0.6534, - "hfopenllm_v2/MATH Level 5": 0.3912, - "hfopenllm_v2/GPQA": 0.3834, - "hfopenllm_v2/MUSR": 0.4873, - "hfopenllm_v2/MMLU-PRO": 0.5412 - } - }, - { - "id": "sometimesanotion/Qwentinuum-14B-v9", - "name": "Qwentinuum-14B-v9", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5107, - "hfopenllm_v2/BBH": 0.658, - "hfopenllm_v2/MATH Level 5": 0.3482, - "hfopenllm_v2/GPQA": 0.3859, - "hfopenllm_v2/MUSR": 0.4781, - "hfopenllm_v2/MMLU-PRO": 0.5421 - } - }, - { - "id": "sometimesanotion/Qwenvergence-14B-qv256", - "name": "Qwenvergence-14B-qv256", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7006, - "hfopenllm_v2/BBH": 0.6312, - "hfopenllm_v2/MATH Level 5": 0.3897, - "hfopenllm_v2/GPQA": 0.3784, - "hfopenllm_v2/MUSR": 0.4926, - "hfopenllm_v2/MMLU-PRO": 0.5178 - } - }, - { - "id": "sometimesanotion/Qwenvergence-14B-v0.6-004-model_stock", - "name": "Qwenvergence-14B-v0.6-004-model_stock", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.686, - "hfopenllm_v2/BBH": 0.6249, - "hfopenllm_v2/MATH Level 5": 0.4094, - "hfopenllm_v2/GPQA": 0.3834, - "hfopenllm_v2/MUSR": 0.5033, - "hfopenllm_v2/MMLU-PRO": 0.5193 - } - }, - { - "id": "sometimesanotion/Qwenvergence-14B-v10", - "name": "Qwenvergence-14B-v10", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6757, - "hfopenllm_v2/BBH": 0.6316, - "hfopenllm_v2/MATH Level 5": 0.4789, - "hfopenllm_v2/GPQA": 0.3792, - "hfopenllm_v2/MUSR": 0.4991, - "hfopenllm_v2/MMLU-PRO": 0.5239 - } - }, - { - "id": "sometimesanotion/Qwenvergence-14B-v11", - "name": "Qwenvergence-14B-v11", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7192, - "hfopenllm_v2/BBH": 0.6368, - "hfopenllm_v2/MATH Level 5": 0.4645, - "hfopenllm_v2/GPQA": 0.3725, - "hfopenllm_v2/MUSR": 0.4754, - "hfopenllm_v2/MMLU-PRO": 0.5327 - } - }, - { - "id": "sometimesanotion/Qwenvergence-14B-v12-Prose", - "name": "Qwenvergence-14B-v12-Prose", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5412, - "hfopenllm_v2/BBH": 0.6504, - "hfopenllm_v2/MATH Level 5": 0.3535, - "hfopenllm_v2/GPQA": 0.3867, - "hfopenllm_v2/MUSR": 0.4991, - "hfopenllm_v2/MMLU-PRO": 0.5381 - } - }, - { - "id": "sometimesanotion/Qwenvergence-14B-v12-Prose-DS", - "name": "Qwenvergence-14B-v12-Prose-DS", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6173, - "hfopenllm_v2/BBH": 0.6507, - "hfopenllm_v2/MATH Level 5": 0.4305, - "hfopenllm_v2/GPQA": 0.3943, - "hfopenllm_v2/MUSR": 0.5151, - "hfopenllm_v2/MMLU-PRO": 0.5369 - } - }, - { - "id": "sometimesanotion/Qwenvergence-14B-v13-Prose-DS", - "name": "Qwenvergence-14B-v13-Prose-DS", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7178, - "hfopenllm_v2/BBH": 0.6405, - "hfopenllm_v2/MATH Level 5": 0.386, - "hfopenllm_v2/GPQA": 0.3834, - "hfopenllm_v2/MUSR": 0.4927, - "hfopenllm_v2/MMLU-PRO": 0.5349 - } - }, - { - "id": "sometimesanotion/Qwenvergence-14B-v15-Prose-MS", - "name": "Qwenvergence-14B-v15-Prose-MS", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5032, - "hfopenllm_v2/BBH": 0.655, - "hfopenllm_v2/MATH Level 5": 0.3633, - "hfopenllm_v2/GPQA": 0.3951, - "hfopenllm_v2/MUSR": 0.4913, - "hfopenllm_v2/MMLU-PRO": 0.5393 - } - }, - { - "id": "sometimesanotion/Qwenvergence-14B-v2-Prose", - "name": "Qwenvergence-14B-v2-Prose", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4705, - "hfopenllm_v2/BBH": 0.6519, - "hfopenllm_v2/MATH Level 5": 0.3557, - "hfopenllm_v2/GPQA": 0.3935, - "hfopenllm_v2/MUSR": 0.4926, - "hfopenllm_v2/MMLU-PRO": 0.5372 - } - }, - { - "id": "sometimesanotion/Qwenvergence-14B-v3", - "name": "Qwenvergence-14B-v3", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5044, - "hfopenllm_v2/BBH": 0.6548, - "hfopenllm_v2/MATH Level 5": 0.3693, - "hfopenllm_v2/GPQA": 0.3842, - "hfopenllm_v2/MUSR": 0.4886, - "hfopenllm_v2/MMLU-PRO": 0.5386 - } - }, - { - "id": "sometimesanotion/Qwenvergence-14B-v3-Prose", - "name": "Qwenvergence-14B-v3-Prose", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4918, - "hfopenllm_v2/BBH": 0.6513, - "hfopenllm_v2/MATH Level 5": 0.3648, - "hfopenllm_v2/GPQA": 0.3951, - "hfopenllm_v2/MUSR": 0.4939, - "hfopenllm_v2/MMLU-PRO": 0.537 - } - }, - { - "id": "sometimesanotion/Qwenvergence-14B-v3-Reason", - "name": "Qwenvergence-14B-v3-Reason", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5278, - "hfopenllm_v2/BBH": 0.6557, - "hfopenllm_v2/MATH Level 5": 0.3119, - "hfopenllm_v2/GPQA": 0.3842, - "hfopenllm_v2/MUSR": 0.4754, - "hfopenllm_v2/MMLU-PRO": 0.5396 - } - }, - { - "id": "sometimesanotion/Qwenvergence-14B-v6-Prose", - "name": "Qwenvergence-14B-v6-Prose", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.599, - "hfopenllm_v2/BBH": 0.6544, - "hfopenllm_v2/MATH Level 5": 0.3565, - "hfopenllm_v2/GPQA": 0.3884, - "hfopenllm_v2/MUSR": 0.4887, - "hfopenllm_v2/MMLU-PRO": 0.5371 - } - }, - { - "id": "sometimesanotion/Qwenvergence-14B-v6-Prose-model_stock", - "name": "Qwenvergence-14B-v6-Prose-model_stock", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4811, - "hfopenllm_v2/BBH": 0.653, - "hfopenllm_v2/MATH Level 5": 0.3603, - "hfopenllm_v2/GPQA": 0.3935, - "hfopenllm_v2/MUSR": 0.4899, - "hfopenllm_v2/MMLU-PRO": 0.5387 - } - }, - { - "id": "sometimesanotion/Qwenvergence-14B-v8", - "name": "Qwenvergence-14B-v8", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5913, - "hfopenllm_v2/BBH": 0.6522, - "hfopenllm_v2/MATH Level 5": 0.4048, - "hfopenllm_v2/GPQA": 0.3809, - "hfopenllm_v2/MUSR": 0.4768, - "hfopenllm_v2/MMLU-PRO": 0.5435 - } - }, - { - "id": "sometimesanotion/Qwenvergence-14B-v9", - "name": "Qwenvergence-14B-v9", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6598, - "hfopenllm_v2/BBH": 0.6166, - "hfopenllm_v2/MATH Level 5": 0.4139, - "hfopenllm_v2/GPQA": 0.3683, - "hfopenllm_v2/MUSR": 0.5141, - "hfopenllm_v2/MMLU-PRO": 0.5111 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/sonthenguyen.json b/data/developers/sonthenguyen.json deleted file mode 100644 index ea8ed7f630d2c7af0c6488b18d61f14b7cb8f4c2..0000000000000000000000000000000000000000 --- a/data/developers/sonthenguyen.json +++ /dev/null @@ -1,89 +0,0 @@ -{ - "developer": "sonthenguyen", - "models": [ - { - "id": "sonthenguyen/ft-unsloth-zephyr-sft-bnb-4bit-20241014-161415", - "name": "ft-unsloth-zephyr-sft-bnb-4bit-20241014-161415", - "developer": "sonthenguyen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2893, - "hfopenllm_v2/BBH": 0.3804, - "hfopenllm_v2/MATH Level 5": 0.0113, - "hfopenllm_v2/GPQA": 0.2466, - "hfopenllm_v2/MUSR": 0.3861, - "hfopenllm_v2/MMLU-PRO": 0.1401 - } - }, - { - "id": "sonthenguyen/ft-unsloth-zephyr-sft-bnb-4bit-20241014-164205", - "name": "ft-unsloth-zephyr-sft-bnb-4bit-20241014-164205", - "developer": "sonthenguyen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3199, - "hfopenllm_v2/BBH": 0.3959, - "hfopenllm_v2/MATH Level 5": 0.0083, - "hfopenllm_v2/GPQA": 0.276, - "hfopenllm_v2/MUSR": 0.4272, - "hfopenllm_v2/MMLU-PRO": 0.2124 - } - }, - { - "id": "sonthenguyen/ft-unsloth-zephyr-sft-bnb-4bit-20241014-170522", - "name": "ft-unsloth-zephyr-sft-bnb-4bit-20241014-170522", - "developer": "sonthenguyen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3764, - "hfopenllm_v2/BBH": 0.3828, - "hfopenllm_v2/MATH Level 5": 0.0091, - "hfopenllm_v2/GPQA": 0.2651, - "hfopenllm_v2/MUSR": 0.4404, - "hfopenllm_v2/MMLU-PRO": 0.2055 - } - }, - { - "id": "sonthenguyen/zephyr-sft-bnb-4bit-DPO-mtbc-213steps", - "name": "zephyr-sft-bnb-4bit-DPO-mtbc-213steps", - "developer": "sonthenguyen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4275, - "hfopenllm_v2/BBH": 0.4197, - "hfopenllm_v2/MATH Level 5": 0.0257, - "hfopenllm_v2/GPQA": 0.2617, - "hfopenllm_v2/MUSR": 0.4086, - "hfopenllm_v2/MMLU-PRO": 0.2709 - } - }, - { - "id": "sonthenguyen/zephyr-sft-bnb-4bit-DPO-mtbo-180steps", - "name": "zephyr-sft-bnb-4bit-DPO-mtbo-180steps", - "developer": "sonthenguyen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4087, - "hfopenllm_v2/BBH": 0.4323, - "hfopenllm_v2/MATH Level 5": 0.0234, - "hfopenllm_v2/GPQA": 0.276, - "hfopenllm_v2/MUSR": 0.3885, - "hfopenllm_v2/MMLU-PRO": 0.2748 - } - }, - { - "id": "sonthenguyen/zephyr-sft-bnb-4bit-DPO-mtbr-180steps", - "name": "zephyr-sft-bnb-4bit-DPO-mtbr-180steps", - "developer": "sonthenguyen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4032, - "hfopenllm_v2/BBH": 0.4305, - "hfopenllm_v2/MATH Level 5": 0.0249, - "hfopenllm_v2/GPQA": 0.2802, - "hfopenllm_v2/MUSR": 0.4258, - "hfopenllm_v2/MMLU-PRO": 0.2711 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/sophosympatheia.json b/data/developers/sophosympatheia.json deleted file mode 100644 index daf2aa2b7fe4ae3ff5f5e14351377c30775172a8..0000000000000000000000000000000000000000 --- a/data/developers/sophosympatheia.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "sophosympatheia", - "models": [ - { - "id": "sophosympatheia/Midnight-Miqu-70B-v1.5", - "name": "Midnight-Miqu-70B-v1.5", - "developer": "sophosympatheia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6118, - "hfopenllm_v2/BBH": 0.5606, - "hfopenllm_v2/MATH Level 5": 0.0702, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.4244, - "hfopenllm_v2/MMLU-PRO": 0.3825 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/sorawiz.json b/data/developers/sorawiz.json deleted file mode 100644 index 2650c3b4fd620a7263bae026629f54caad0a4d31..0000000000000000000000000000000000000000 --- a/data/developers/sorawiz.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "Sorawiz", - "models": [ - { - "id": "Sorawiz/Gemma-9B-Base", - "name": "Gemma-9B-Base", - "developer": "Sorawiz", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1667, - "hfopenllm_v2/BBH": 0.593, - "hfopenllm_v2/MATH Level 5": 0.0982, - "hfopenllm_v2/GPQA": 0.3398, - "hfopenllm_v2/MUSR": 0.4045, - "hfopenllm_v2/MMLU-PRO": 0.4235 - } - }, - { - "id": "Sorawiz/Gemma-Creative-9B-Base", - "name": "Gemma-Creative-9B-Base", - "developer": "Sorawiz", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1515, - "hfopenllm_v2/BBH": 0.5459, - "hfopenllm_v2/MATH Level 5": 0.0778, - "hfopenllm_v2/GPQA": 0.3297, - "hfopenllm_v2/MUSR": 0.4019, - "hfopenllm_v2/MMLU-PRO": 0.4008 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/sourjayon.json b/data/developers/sourjayon.json deleted file mode 100644 index 01fa6b43cb5bdaf67d532ba142857849618af89f..0000000000000000000000000000000000000000 --- a/data/developers/sourjayon.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "Sourjayon", - "models": [ - { - "id": "Sourjayon/DeepSeek-R1-8b-Sify", - "name": "DeepSeek-R1-8b-Sify", - "developer": "Sourjayon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3679, - "hfopenllm_v2/BBH": 0.3379, - "hfopenllm_v2/MATH Level 5": 0.2447, - "hfopenllm_v2/GPQA": 0.2525, - "hfopenllm_v2/MUSR": 0.3303, - "hfopenllm_v2/MMLU-PRO": 0.1981 - } - }, - { - "id": "Sourjayon/DeepSeek-R1-ForumNXT", - "name": "DeepSeek-R1-ForumNXT", - "developer": "Sourjayon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2603, - "hfopenllm_v2/BBH": 0.331, - "hfopenllm_v2/MATH Level 5": 0.2576, - "hfopenllm_v2/GPQA": 0.2743, - "hfopenllm_v2/MUSR": 0.3392, - "hfopenllm_v2/MMLU-PRO": 0.1648 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/spaceyl.json b/data/developers/spaceyl.json deleted file mode 100644 index c9fc888c11d4b241d443803a3cf12907f61998f9..0000000000000000000000000000000000000000 --- a/data/developers/spaceyl.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "SpaceYL", - "models": [ - { - "id": "SpaceYL/ECE_Poirot", - "name": "ECE_Poirot", - "developer": "SpaceYL", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3107, - "hfopenllm_v2/BBH": 0.4262, - "hfopenllm_v2/MATH Level 5": 0.0914, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.4026, - "hfopenllm_v2/MMLU-PRO": 0.2883 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/speakleash-ack-cyfronet-agh.json b/data/developers/speakleash-ack-cyfronet-agh.json deleted file mode 100644 index 4636dc9c3b7038944204927d0a99c08a2b2229b8..0000000000000000000000000000000000000000 --- a/data/developers/speakleash-ack-cyfronet-agh.json +++ /dev/null @@ -1,45 +0,0 @@ -{ - "developer": "speakleash-ack-cyfronet-agh", - "models": [ - { - "id": "speakleash-ack-cyfronet-agh/bielik-11b-v2-3-instruct-prompt", - "name": "Bielik-11B-v2.3-Instruct (Prompt)", - "developer": "speakleash-ack-cyfronet-agh", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 99.0, - "bfcl/bfcl.overall.overall_accuracy": 21.9, - "bfcl/bfcl.overall.total_cost_usd": 22.44, - "bfcl/bfcl.overall.latency_mean_s": 23.75, - "bfcl/bfcl.overall.latency_std_s": 61.76, - "bfcl/bfcl.overall.latency_p95_s": 72.8, - "bfcl/bfcl.non_live.ast_accuracy": 81.5, - "bfcl/bfcl.non_live.simple_ast_accuracy": 73.0, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 92.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 85.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 75.5, - "bfcl/bfcl.live.live_accuracy": 67.8, - "bfcl/bfcl.live.live_simple_ast_accuracy": 75.58, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 66.19, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 62.5, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 58.33, - "bfcl/bfcl.multi_turn.accuracy": 2.62, - "bfcl/bfcl.multi_turn.base_accuracy": 4.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 0.5, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 3.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 2.5, - "bfcl/bfcl.web_search.accuracy": 1.5, - "bfcl/bfcl.web_search.base_accuracy": 1.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 2.0, - "bfcl/bfcl.memory.accuracy": 11.4, - "bfcl/bfcl.memory.kv_accuracy": 7.1, - "bfcl/bfcl.memory.vector_accuracy": 4.52, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 22.58, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 93.75, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 36.01, - "bfcl/bfcl.format_sensitivity.max_delta": 35.0, - "bfcl/bfcl.format_sensitivity.stddev": 9.74 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/speakleash.json b/data/developers/speakleash.json deleted file mode 100644 index 09c644aaa535e62ebd4dc0b391f0749b08b56ffe..0000000000000000000000000000000000000000 --- a/data/developers/speakleash.json +++ /dev/null @@ -1,75 +0,0 @@ -{ - "developer": "speakleash", - "models": [ - { - "id": "speakleash/Bielik-11B-v2", - "name": "Bielik-11B-v2", - "developer": "speakleash", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2381, - "hfopenllm_v2/BBH": 0.4931, - "hfopenllm_v2/MATH Level 5": 0.0785, - "hfopenllm_v2/GPQA": 0.2886, - "hfopenllm_v2/MUSR": 0.3924, - "hfopenllm_v2/MMLU-PRO": 0.3137 - } - }, - { - "id": "speakleash/Bielik-11B-v2.0-Instruct", - "name": "Bielik-11B-v2.0-Instruct", - "developer": "speakleash", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5252, - "hfopenllm_v2/BBH": 0.5362, - "hfopenllm_v2/MATH Level 5": 0.1186, - "hfopenllm_v2/GPQA": 0.3171, - "hfopenllm_v2/MUSR": 0.4467, - "hfopenllm_v2/MMLU-PRO": 0.3351 - } - }, - { - "id": "speakleash/Bielik-11B-v2.1-Instruct", - "name": "Bielik-11B-v2.1-Instruct", - "developer": "speakleash", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.509, - "hfopenllm_v2/BBH": 0.553, - "hfopenllm_v2/MATH Level 5": 0.2666, - "hfopenllm_v2/GPQA": 0.3372, - "hfopenllm_v2/MUSR": 0.4185, - "hfopenllm_v2/MMLU-PRO": 0.3447 - } - }, - { - "id": "speakleash/Bielik-11B-v2.2-Instruct", - "name": "Bielik-11B-v2.2-Instruct", - "developer": "speakleash", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5552, - "hfopenllm_v2/BBH": 0.5597, - "hfopenllm_v2/MATH Level 5": 0.2681, - "hfopenllm_v2/GPQA": 0.3314, - "hfopenllm_v2/MUSR": 0.4171, - "hfopenllm_v2/MMLU-PRO": 0.3487 - } - }, - { - "id": "speakleash/Bielik-11B-v2.3-Instruct", - "name": "Bielik-11B-v2.3-Instruct", - "developer": "speakleash", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5583, - "hfopenllm_v2/BBH": 0.5663, - "hfopenllm_v2/MATH Level 5": 0.2085, - "hfopenllm_v2/GPQA": 0.3406, - "hfopenllm_v2/MUSR": 0.4518, - "hfopenllm_v2/MMLU-PRO": 0.3444 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/spestly.json b/data/developers/spestly.json deleted file mode 100644 index 0b4037b864eb053cd4c0642e5b3b289ce874ab09..0000000000000000000000000000000000000000 --- a/data/developers/spestly.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "developer": "Spestly", - "models": [ - { - "id": "Spestly/Athena-1-3B", - "name": "Athena-1-3B", - "developer": "Spestly", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5569, - "hfopenllm_v2/BBH": 0.4702, - "hfopenllm_v2/MATH Level 5": 0.2379, - "hfopenllm_v2/GPQA": 0.2936, - "hfopenllm_v2/MUSR": 0.4362, - "hfopenllm_v2/MMLU-PRO": 0.3519 - } - }, - { - "id": "Spestly/Atlas-Pro-1.5B-Preview", - "name": "Atlas-Pro-1.5B-Preview", - "developer": "Spestly", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.243, - "hfopenllm_v2/BBH": 0.3499, - "hfopenllm_v2/MATH Level 5": 0.3195, - "hfopenllm_v2/GPQA": 0.297, - "hfopenllm_v2/MUSR": 0.3354, - "hfopenllm_v2/MMLU-PRO": 0.1925 - } - }, - { - "id": "Spestly/Atlas-Pro-7B-Preview", - "name": "Atlas-Pro-7B-Preview", - "developer": "Spestly", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3154, - "hfopenllm_v2/BBH": 0.4668, - "hfopenllm_v2/MATH Level 5": 0.5083, - "hfopenllm_v2/GPQA": 0.3372, - "hfopenllm_v2/MUSR": 0.3911, - "hfopenllm_v2/MMLU-PRO": 0.297 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/spmurrayzzz.json b/data/developers/spmurrayzzz.json deleted file mode 100644 index 7697a5ccd81534909759899096ef6805baf01a65..0000000000000000000000000000000000000000 --- a/data/developers/spmurrayzzz.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "spmurrayzzz", - "models": [ - { - "id": "spmurrayzzz/Mistral-Syndicate-7B", - "name": "Mistral-Syndicate-7B", - "developer": "spmurrayzzz", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2496, - "hfopenllm_v2/BBH": 0.4245, - "hfopenllm_v2/MATH Level 5": 0.034, - "hfopenllm_v2/GPQA": 0.276, - "hfopenllm_v2/MUSR": 0.4386, - "hfopenllm_v2/MMLU-PRO": 0.2631 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/spow12.json b/data/developers/spow12.json deleted file mode 100644 index 39b5b7b70e78310868166d1a8d2f2ae80dc77b8f..0000000000000000000000000000000000000000 --- a/data/developers/spow12.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "developer": "spow12", - "models": [ - { - "id": "spow12/ChatWaifu_12B_v2.0", - "name": "ChatWaifu_12B_v2.0", - "developer": "spow12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4768, - "hfopenllm_v2/BBH": 0.5208, - "hfopenllm_v2/MATH Level 5": 0.071, - "hfopenllm_v2/GPQA": 0.2768, - "hfopenllm_v2/MUSR": 0.4432, - "hfopenllm_v2/MMLU-PRO": 0.3388 - } - }, - { - "id": "spow12/ChatWaifu_22B_v2.0_preview", - "name": "ChatWaifu_22B_v2.0_preview", - "developer": "spow12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6745, - "hfopenllm_v2/BBH": 0.617, - "hfopenllm_v2/MATH Level 5": 0.1888, - "hfopenllm_v2/GPQA": 0.3154, - "hfopenllm_v2/MUSR": 0.3685, - "hfopenllm_v2/MMLU-PRO": 0.3988 - } - }, - { - "id": "spow12/ChatWaifu_v1.4", - "name": "ChatWaifu_v1.4", - "developer": "spow12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5691, - "hfopenllm_v2/BBH": 0.5176, - "hfopenllm_v2/MATH Level 5": 0.1057, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.4743, - "hfopenllm_v2/MMLU-PRO": 0.3475 - } - }, - { - "id": "spow12/ChatWaifu_v2.0_22B", - "name": "ChatWaifu_v2.0_22B", - "developer": "spow12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6511, - "hfopenllm_v2/BBH": 0.5926, - "hfopenllm_v2/MATH Level 5": 0.1858, - "hfopenllm_v2/GPQA": 0.3247, - "hfopenllm_v2/MUSR": 0.3842, - "hfopenllm_v2/MMLU-PRO": 0.3836 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/ssmits.json b/data/developers/ssmits.json deleted file mode 100644 index b18b4bd4706de08526b971bc0323d4163a095a37..0000000000000000000000000000000000000000 --- a/data/developers/ssmits.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "ssmits", - "models": [ - { - "id": "ssmits/Qwen2.5-95B-Instruct", - "name": "Qwen2.5-95B-Instruct", - "developer": "ssmits", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8431, - "hfopenllm_v2/BBH": 0.7038, - "hfopenllm_v2/MATH Level 5": 0.5302, - "hfopenllm_v2/GPQA": 0.3641, - "hfopenllm_v2/MUSR": 0.4284, - "hfopenllm_v2/MMLU-PRO": 0.5217 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/stabilityai.json b/data/developers/stabilityai.json deleted file mode 100644 index fa056e36c8ea5518e291812fa80267171985efd8..0000000000000000000000000000000000000000 --- a/data/developers/stabilityai.json +++ /dev/null @@ -1,149 +0,0 @@ -{ - "developer": "stabilityai", - "models": [ - { - "id": "stabilityai/stable-code-instruct-3b", - "name": "stabilityai/stable-code-instruct-3b", - "developer": "stabilityai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6216, - "reward-bench/Chat": 0.5782, - "reward-bench/Chat Hard": 0.5855, - "reward-bench/Safety": 0.6554, - "reward-bench/Reasoning": 0.7528, - "reward-bench/Prior Sets (0.5 weight)": 0.4506 - } - }, - { - "id": "stabilityai/StableBeluga2", - "name": "StableBeluga2", - "developer": "stabilityai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3787, - "hfopenllm_v2/BBH": 0.5824, - "hfopenllm_v2/MATH Level 5": 0.0438, - "hfopenllm_v2/GPQA": 0.3163, - "hfopenllm_v2/MUSR": 0.473, - "hfopenllm_v2/MMLU-PRO": 0.3326 - } - }, - { - "id": "stabilityai/stablelm-2-12b", - "name": "stablelm-2-12b", - "developer": "stabilityai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1569, - "hfopenllm_v2/BBH": 0.4509, - "hfopenllm_v2/MATH Level 5": 0.0431, - "hfopenllm_v2/GPQA": 0.2785, - "hfopenllm_v2/MUSR": 0.4479, - "hfopenllm_v2/MMLU-PRO": 0.3072 - } - }, - { - "id": "stabilityai/stablelm-2-12b-chat", - "name": "stabilityai/stablelm-2-12b-chat", - "developer": "stabilityai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4082, - "hfopenllm_v2/BBH": 0.4672, - "hfopenllm_v2/MATH Level 5": 0.0536, - "hfopenllm_v2/GPQA": 0.2668, - "hfopenllm_v2/MUSR": 0.3914, - "hfopenllm_v2/MMLU-PRO": 0.2734, - "reward-bench/Score": 0.7642, - "reward-bench/Chat": 0.9665, - "reward-bench/Chat Hard": 0.5548, - "reward-bench/Safety": 0.7811, - "reward-bench/Reasoning": 0.8945, - "reward-bench/Prior Sets (0.5 weight)": 0.4839 - } - }, - { - "id": "stabilityai/stablelm-2-1_6b", - "name": "stablelm-2-1_6b", - "developer": "stabilityai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1157, - "hfopenllm_v2/BBH": 0.3385, - "hfopenllm_v2/MATH Level 5": 0.0076, - "hfopenllm_v2/GPQA": 0.2483, - "hfopenllm_v2/MUSR": 0.3882, - "hfopenllm_v2/MMLU-PRO": 0.1464 - } - }, - { - "id": "stabilityai/stablelm-2-1_6b-chat", - "name": "stablelm-2-1_6b-chat", - "developer": "stabilityai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.306, - "hfopenllm_v2/BBH": 0.339, - "hfopenllm_v2/MATH Level 5": 0.0249, - "hfopenllm_v2/GPQA": 0.2475, - "hfopenllm_v2/MUSR": 0.358, - "hfopenllm_v2/MMLU-PRO": 0.1622 - } - }, - { - "id": "stabilityai/stablelm-2-zephyr-1_6b", - "name": "stabilityai/stablelm-2-zephyr-1_6b", - "developer": "stabilityai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3279, - "hfopenllm_v2/BBH": 0.3352, - "hfopenllm_v2/MATH Level 5": 0.0332, - "hfopenllm_v2/GPQA": 0.2433, - "hfopenllm_v2/MUSR": 0.3511, - "hfopenllm_v2/MMLU-PRO": 0.1714, - "reward-bench/Score": 0.6574, - "reward-bench/Chat": 0.9665, - "reward-bench/Chat Hard": 0.4671, - "reward-bench/Safety": 0.6027, - "reward-bench/Reasoning": 0.6784, - "reward-bench/Prior Sets (0.5 weight)": 0.4868 - } - }, - { - "id": "stabilityai/stablelm-3b-4e1t", - "name": "stablelm-3b-4e1t", - "developer": "stabilityai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2203, - "hfopenllm_v2/BBH": 0.3504, - "hfopenllm_v2/MATH Level 5": 0.0106, - "hfopenllm_v2/GPQA": 0.2374, - "hfopenllm_v2/MUSR": 0.3778, - "hfopenllm_v2/MMLU-PRO": 0.1669 - } - }, - { - "id": "stabilityai/stablelm-zephyr-3b", - "name": "stabilityai/stablelm-zephyr-3b", - "developer": "stabilityai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3683, - "hfopenllm_v2/BBH": 0.3866, - "hfopenllm_v2/MATH Level 5": 0.0431, - "hfopenllm_v2/GPQA": 0.2391, - "hfopenllm_v2/MUSR": 0.4183, - "hfopenllm_v2/MMLU-PRO": 0.1768, - "reward-bench/Score": 0.7146, - "reward-bench/Chat": 0.8631, - "reward-bench/Chat Hard": 0.6009, - "reward-bench/Safety": 0.7405, - "reward-bench/Reasoning": 0.7573, - "reward-bench/Prior Sets (0.5 weight)": 0.5075 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/stanford.json b/data/developers/stanford.json deleted file mode 100644 index 95c10daf7e4483a9621f542461b1d7e157b2d5e2..0000000000000000000000000000000000000000 --- a/data/developers/stanford.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "developer": "stanford", - "models": [ - { - "id": "stanford/Alpaca-7B", - "name": "Alpaca 7B", - "developer": "stanford", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.381, - "helm_classic/MMLU": 0.385, - "helm_classic/BoolQ": 0.778, - "helm_classic/NarrativeQA": 0.396, - "helm_classic/NaturalQuestions (open-book)": 0.592, - "helm_classic/QuAC": 0.27, - "helm_classic/HellaSwag": -1.0, - "helm_classic/OpenbookQA": -1.0, - "helm_classic/TruthfulQA": 0.243, - "helm_classic/MS MARCO (TREC)": -1.0, - "helm_classic/CNN/DailyMail": -1.0, - "helm_classic/XSUM": -1.0, - "helm_classic/IMDB": 0.738, - "helm_classic/CivilComments": 0.566, - "helm_classic/RAFT": 0.486 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/stanfordnlp.json b/data/developers/stanfordnlp.json deleted file mode 100644 index 2aaadfecfb9ec3215a55d094f249a23b6717aec2..0000000000000000000000000000000000000000 --- a/data/developers/stanfordnlp.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "stanfordnlp", - "models": [ - { - "id": "stanfordnlp/SteamSHP-flan-t5-large", - "name": "stanfordnlp/SteamSHP-flan-t5-large", - "developer": "stanfordnlp", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.4962, - "reward-bench/Chat": 0.8575, - "reward-bench/Chat Hard": 0.3311, - "reward-bench/Safety": 0.3743, - "reward-bench/Reasoning": 0.3563, - "reward-bench/Prior Sets (0.5 weight)": 0.6273 - } - }, - { - "id": "stanfordnlp/SteamSHP-flan-t5-xl", - "name": "stanfordnlp/SteamSHP-flan-t5-xl", - "developer": "stanfordnlp", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5135, - "reward-bench/Chat": 0.8547, - "reward-bench/Chat Hard": 0.3684, - "reward-bench/Safety": 0.3784, - "reward-bench/Reasoning": 0.3841, - "reward-bench/Prior Sets (0.5 weight)": 0.6498 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/stark2008.json b/data/developers/stark2008.json deleted file mode 100644 index 50633df72c9efad74f05c46342aae6f44173516f..0000000000000000000000000000000000000000 --- a/data/developers/stark2008.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "developer": "Stark2008", - "models": [ - { - "id": "Stark2008/GutenLaserPi", - "name": "GutenLaserPi", - "developer": "Stark2008", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4227, - "hfopenllm_v2/BBH": 0.5212, - "hfopenllm_v2/MATH Level 5": 0.0785, - "hfopenllm_v2/GPQA": 0.2869, - "hfopenllm_v2/MUSR": 0.462, - "hfopenllm_v2/MMLU-PRO": 0.3106 - } - }, - { - "id": "Stark2008/LayleleFlamPi", - "name": "LayleleFlamPi", - "developer": "Stark2008", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4284, - "hfopenllm_v2/BBH": 0.5116, - "hfopenllm_v2/MATH Level 5": 0.0665, - "hfopenllm_v2/GPQA": 0.2852, - "hfopenllm_v2/MUSR": 0.4608, - "hfopenllm_v2/MMLU-PRO": 0.3093 - } - }, - { - "id": "Stark2008/VisFlamCat", - "name": "VisFlamCat", - "developer": "Stark2008", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4366, - "hfopenllm_v2/BBH": 0.5217, - "hfopenllm_v2/MATH Level 5": 0.0763, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.4463, - "hfopenllm_v2/MMLU-PRO": 0.3144 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/steelskull.json b/data/developers/steelskull.json deleted file mode 100644 index c52479b54b19affd00493eea0310042cc8cea472..0000000000000000000000000000000000000000 --- a/data/developers/steelskull.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "Steelskull", - "models": [ - { - "id": "Steelskull/L3.3-MS-Nevoria-70b", - "name": "L3.3-MS-Nevoria-70b", - "developer": "Steelskull", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6963, - "hfopenllm_v2/BBH": 0.6998, - "hfopenllm_v2/MATH Level 5": 0.3958, - "hfopenllm_v2/GPQA": 0.4706, - "hfopenllm_v2/MUSR": 0.4682, - "hfopenllm_v2/MMLU-PRO": 0.5535 - } - }, - { - "id": "Steelskull/L3.3-Nevoria-R1-70b", - "name": "L3.3-Nevoria-R1-70b", - "developer": "Steelskull", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6024, - "hfopenllm_v2/BBH": 0.6972, - "hfopenllm_v2/MATH Level 5": 0.463, - "hfopenllm_v2/GPQA": 0.469, - "hfopenllm_v2/MUSR": 0.4775, - "hfopenllm_v2/MMLU-PRO": 0.5463 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/stellex.json b/data/developers/stellex.json deleted file mode 100644 index 76954e170183fd4864448255c6d8328f7b2922c3..0000000000000000000000000000000000000000 --- a/data/developers/stellex.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "StelleX", - "models": [ - { - "id": "StelleX/Qwen2.5_Math_7B_Cot", - "name": "Qwen2.5_Math_7B_Cot", - "developer": "StelleX", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2143, - "hfopenllm_v2/BBH": 0.4313, - "hfopenllm_v2/MATH Level 5": 0.3263, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.3924, - "hfopenllm_v2/MMLU-PRO": 0.281 - } - }, - { - "id": "StelleX/Vorisatex-7B-preview", - "name": "Vorisatex-7B-preview", - "developer": "StelleX", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1515, - "hfopenllm_v2/BBH": 0.3112, - "hfopenllm_v2/MATH Level 5": 0.0287, - "hfopenllm_v2/GPQA": 0.2517, - "hfopenllm_v2/MUSR": 0.4192, - "hfopenllm_v2/MMLU-PRO": 0.1166 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/sthenno-com.json b/data/developers/sthenno-com.json deleted file mode 100644 index 6079039a25dc05d46a87af22b79b137766bfb81a..0000000000000000000000000000000000000000 --- a/data/developers/sthenno-com.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "developer": "sthenno-com", - "models": [ - { - "id": "sthenno-com/miscii-14b-0130", - "name": "miscii-14b-0130", - "developer": "sthenno-com", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6647, - "hfopenllm_v2/BBH": 0.6505, - "hfopenllm_v2/MATH Level 5": 0.432, - "hfopenllm_v2/GPQA": 0.3817, - "hfopenllm_v2/MUSR": 0.4912, - "hfopenllm_v2/MMLU-PRO": 0.5363 - } - }, - { - "id": "sthenno-com/miscii-14b-0218", - "name": "miscii-14b-0218", - "developer": "sthenno-com", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7656, - "hfopenllm_v2/BBH": 0.6559, - "hfopenllm_v2/MATH Level 5": 0.5144, - "hfopenllm_v2/GPQA": 0.3834, - "hfopenllm_v2/MUSR": 0.4273, - "hfopenllm_v2/MMLU-PRO": 0.5298 - } - }, - { - "id": "sthenno-com/miscii-14b-1028", - "name": "miscii-14b-1028", - "developer": "sthenno-com", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8237, - "hfopenllm_v2/BBH": 0.6448, - "hfopenllm_v2/MATH Level 5": 0.503, - "hfopenllm_v2/GPQA": 0.3565, - "hfopenllm_v2/MUSR": 0.4182, - "hfopenllm_v2/MMLU-PRO": 0.5153 - } - }, - { - "id": "sthenno-com/miscii-14b-1225", - "name": "miscii-14b-1225", - "developer": "sthenno-com", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7878, - "hfopenllm_v2/BBH": 0.6572, - "hfopenllm_v2/MATH Level 5": 0.4517, - "hfopenllm_v2/GPQA": 0.3775, - "hfopenllm_v2/MUSR": 0.4366, - "hfopenllm_v2/MMLU-PRO": 0.5272 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/sthenno.json b/data/developers/sthenno.json deleted file mode 100644 index 144f6005135a51b3707516c6521804423fd6f58e..0000000000000000000000000000000000000000 --- a/data/developers/sthenno.json +++ /dev/null @@ -1,131 +0,0 @@ -{ - "developer": "sthenno", - "models": [ - { - "id": "sthenno/tempesthenno-0120", - "name": "tempesthenno-0120", - "developer": "sthenno", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.539, - "hfopenllm_v2/BBH": 0.6373, - "hfopenllm_v2/MATH Level 5": 0.3353, - "hfopenllm_v2/GPQA": 0.3943, - "hfopenllm_v2/MUSR": 0.4633, - "hfopenllm_v2/MMLU-PRO": 0.529 - } - }, - { - "id": "sthenno/tempesthenno-fusion-0309", - "name": "tempesthenno-fusion-0309", - "developer": "sthenno", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7692, - "hfopenllm_v2/BBH": 0.6581, - "hfopenllm_v2/MATH Level 5": 0.4766, - "hfopenllm_v2/GPQA": 0.37, - "hfopenllm_v2/MUSR": 0.4325, - "hfopenllm_v2/MMLU-PRO": 0.5258 - } - }, - { - "id": "sthenno/tempesthenno-kto-0205-ckpt80", - "name": "tempesthenno-kto-0205-ckpt80", - "developer": "sthenno", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8054, - "hfopenllm_v2/BBH": 0.6543, - "hfopenllm_v2/MATH Level 5": 0.4592, - "hfopenllm_v2/GPQA": 0.3482, - "hfopenllm_v2/MUSR": 0.4248, - "hfopenllm_v2/MMLU-PRO": 0.5286 - } - }, - { - "id": "sthenno/tempesthenno-nuslerp-001", - "name": "tempesthenno-nuslerp-001", - "developer": "sthenno", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7926, - "hfopenllm_v2/BBH": 0.6578, - "hfopenllm_v2/MATH Level 5": 0.4758, - "hfopenllm_v2/GPQA": 0.3733, - "hfopenllm_v2/MUSR": 0.43, - "hfopenllm_v2/MMLU-PRO": 0.5257 - } - }, - { - "id": "sthenno/tempesthenno-nuslerp-0124", - "name": "tempesthenno-nuslerp-0124", - "developer": "sthenno", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7004, - "hfopenllm_v2/BBH": 0.6469, - "hfopenllm_v2/MATH Level 5": 0.4116, - "hfopenllm_v2/GPQA": 0.3901, - "hfopenllm_v2/MUSR": 0.4859, - "hfopenllm_v2/MMLU-PRO": 0.5352 - } - }, - { - "id": "sthenno/tempesthenno-ppo-ckpt40", - "name": "tempesthenno-ppo-ckpt40", - "developer": "sthenno", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7923, - "hfopenllm_v2/BBH": 0.655, - "hfopenllm_v2/MATH Level 5": 0.4736, - "hfopenllm_v2/GPQA": 0.3775, - "hfopenllm_v2/MUSR": 0.4352, - "hfopenllm_v2/MMLU-PRO": 0.5292 - } - }, - { - "id": "sthenno/tempesthenno-sft-0309-ckpt10", - "name": "tempesthenno-sft-0309-ckpt10", - "developer": "sthenno", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7744, - "hfopenllm_v2/BBH": 0.6552, - "hfopenllm_v2/MATH Level 5": 0.4721, - "hfopenllm_v2/GPQA": 0.3716, - "hfopenllm_v2/MUSR": 0.4364, - "hfopenllm_v2/MMLU-PRO": 0.5258 - } - }, - { - "id": "sthenno/tempesthenno-sft-0314-stage1-ckpt50", - "name": "tempesthenno-sft-0314-stage1-ckpt50", - "developer": "sthenno", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7394, - "hfopenllm_v2/BBH": 0.6601, - "hfopenllm_v2/MATH Level 5": 0.4683, - "hfopenllm_v2/GPQA": 0.3733, - "hfopenllm_v2/MUSR": 0.4429, - "hfopenllm_v2/MMLU-PRO": 0.5302 - } - }, - { - "id": "sthenno/tempestissimo-14b-0309", - "name": "tempestissimo-14b-0309", - "developer": "sthenno", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7549, - "hfopenllm_v2/BBH": 0.6587, - "hfopenllm_v2/MATH Level 5": 0.4796, - "hfopenllm_v2/GPQA": 0.3666, - "hfopenllm_v2/MUSR": 0.4312, - "hfopenllm_v2/MMLU-PRO": 0.5281 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/streamerbtw1002.json b/data/developers/streamerbtw1002.json deleted file mode 100644 index f847a3c8df86df18738c76fc4ac41247a4f99db8..0000000000000000000000000000000000000000 --- a/data/developers/streamerbtw1002.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "streamerbtw1002", - "models": [ - { - "id": "streamerbtw1002/Nexuim-R1-7B-Instruct", - "name": "Nexuim-R1-7B-Instruct", - "developer": "streamerbtw1002", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6934, - "hfopenllm_v2/BBH": 0.5175, - "hfopenllm_v2/MATH Level 5": 0.4456, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3356, - "hfopenllm_v2/MMLU-PRO": 0.4138 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/stupidity-ai.json b/data/developers/stupidity-ai.json deleted file mode 100644 index 841dccd4e39b0e7b87c676655545ed12f5073d77..0000000000000000000000000000000000000000 --- a/data/developers/stupidity-ai.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "stupidity-ai", - "models": [ - { - "id": "stupidity-ai/Llama-3-8B-Instruct-MultiMoose", - "name": "Llama-3-8B-Instruct-MultiMoose", - "developer": "stupidity-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2318, - "hfopenllm_v2/BBH": 0.2823, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2534, - "hfopenllm_v2/MUSR": 0.3485, - "hfopenllm_v2/MMLU-PRO": 0.1094 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/suayptalha.json b/data/developers/suayptalha.json deleted file mode 100644 index 31fa829fc447e018d00a0eccdff5106bcd916280..0000000000000000000000000000000000000000 --- a/data/developers/suayptalha.json +++ /dev/null @@ -1,173 +0,0 @@ -{ - "developer": "suayptalha", - "models": [ - { - "id": "suayptalha/Clarus-7B-v0.1", - "name": "Clarus-7B-v0.1", - "developer": "suayptalha", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7454, - "hfopenllm_v2/BBH": 0.5497, - "hfopenllm_v2/MATH Level 5": 0.4924, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.443, - "hfopenllm_v2/MMLU-PRO": 0.4387 - } - }, - { - "id": "suayptalha/Clarus-7B-v0.2", - "name": "Clarus-7B-v0.2", - "developer": "suayptalha", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7679, - "hfopenllm_v2/BBH": 0.549, - "hfopenllm_v2/MATH Level 5": 0.4856, - "hfopenllm_v2/GPQA": 0.302, - "hfopenllm_v2/MUSR": 0.4417, - "hfopenllm_v2/MMLU-PRO": 0.44 - } - }, - { - "id": "suayptalha/Clarus-7B-v0.3", - "name": "Clarus-7B-v0.3", - "developer": "suayptalha", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7509, - "hfopenllm_v2/BBH": 0.5526, - "hfopenllm_v2/MATH Level 5": 0.4879, - "hfopenllm_v2/GPQA": 0.3121, - "hfopenllm_v2/MUSR": 0.4402, - "hfopenllm_v2/MMLU-PRO": 0.4385 - } - }, - { - "id": "suayptalha/DeepSeek-R1-Distill-Llama-3B", - "name": "DeepSeek-R1-Distill-Llama-3B", - "developer": "suayptalha", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7093, - "hfopenllm_v2/BBH": 0.4452, - "hfopenllm_v2/MATH Level 5": 0.2092, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.3396, - "hfopenllm_v2/MMLU-PRO": 0.2978 - } - }, - { - "id": "suayptalha/Falcon3-Jessi-v0.4-7B-Slerp", - "name": "Falcon3-Jessi-v0.4-7B-Slerp", - "developer": "suayptalha", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7676, - "hfopenllm_v2/BBH": 0.5591, - "hfopenllm_v2/MATH Level 5": 0.3965, - "hfopenllm_v2/GPQA": 0.3121, - "hfopenllm_v2/MUSR": 0.4812, - "hfopenllm_v2/MMLU-PRO": 0.406 - } - }, - { - "id": "suayptalha/HomerCreativeAnvita-Mix-Qw7B", - "name": "HomerCreativeAnvita-Mix-Qw7B", - "developer": "suayptalha", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7808, - "hfopenllm_v2/BBH": 0.5565, - "hfopenllm_v2/MATH Level 5": 0.361, - "hfopenllm_v2/GPQA": 0.3146, - "hfopenllm_v2/MUSR": 0.4416, - "hfopenllm_v2/MMLU-PRO": 0.4445 - } - }, - { - "id": "suayptalha/Komodo-Llama-3.2-3B-v2-fp16", - "name": "Komodo-Llama-3.2-3B-v2-fp16", - "developer": "suayptalha", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6341, - "hfopenllm_v2/BBH": 0.4355, - "hfopenllm_v2/MATH Level 5": 0.1065, - "hfopenllm_v2/GPQA": 0.2777, - "hfopenllm_v2/MUSR": 0.3406, - "hfopenllm_v2/MMLU-PRO": 0.2852 - } - }, - { - "id": "suayptalha/Lamarckvergence-14B", - "name": "Lamarckvergence-14B", - "developer": "suayptalha", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7656, - "hfopenllm_v2/BBH": 0.6517, - "hfopenllm_v2/MATH Level 5": 0.54, - "hfopenllm_v2/GPQA": 0.3633, - "hfopenllm_v2/MUSR": 0.4422, - "hfopenllm_v2/MMLU-PRO": 0.5283 - } - }, - { - "id": "suayptalha/Lix-14B-v0.1", - "name": "Lix-14B-v0.1", - "developer": "suayptalha", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7813, - "hfopenllm_v2/BBH": 0.6608, - "hfopenllm_v2/MATH Level 5": 0.5295, - "hfopenllm_v2/GPQA": 0.37, - "hfopenllm_v2/MUSR": 0.4338, - "hfopenllm_v2/MMLU-PRO": 0.5314 - } - }, - { - "id": "suayptalha/Luminis-phi-4", - "name": "Luminis-phi-4", - "developer": "suayptalha", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.69, - "hfopenllm_v2/BBH": 0.692, - "hfopenllm_v2/MATH Level 5": 0.4637, - "hfopenllm_v2/GPQA": 0.3515, - "hfopenllm_v2/MUSR": 0.4572, - "hfopenllm_v2/MMLU-PRO": 0.5424 - } - }, - { - "id": "suayptalha/Maestro-10B", - "name": "Maestro-10B", - "developer": "suayptalha", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7768, - "hfopenllm_v2/BBH": 0.5746, - "hfopenllm_v2/MATH Level 5": 0.1911, - "hfopenllm_v2/GPQA": 0.3331, - "hfopenllm_v2/MUSR": 0.4397, - "hfopenllm_v2/MMLU-PRO": 0.4218 - } - }, - { - "id": "suayptalha/Rombos-2.5-T.E-8.1", - "name": "Rombos-2.5-T.E-8.1", - "developer": "suayptalha", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6925, - "hfopenllm_v2/BBH": 0.5515, - "hfopenllm_v2/MATH Level 5": 0.4924, - "hfopenllm_v2/GPQA": 0.3112, - "hfopenllm_v2/MUSR": 0.4166, - "hfopenllm_v2/MMLU-PRO": 0.4446 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/sultanr.json b/data/developers/sultanr.json deleted file mode 100644 index c320049febba8aa937915baa7877fb901641f218..0000000000000000000000000000000000000000 --- a/data/developers/sultanr.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "developer": "SultanR", - "models": [ - { - "id": "SultanR/SmolTulu-1.7b-Instruct", - "name": "SmolTulu-1.7b-Instruct", - "developer": "SultanR", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6541, - "hfopenllm_v2/BBH": 0.3713, - "hfopenllm_v2/MATH Level 5": 0.0793, - "hfopenllm_v2/GPQA": 0.2693, - "hfopenllm_v2/MUSR": 0.354, - "hfopenllm_v2/MMLU-PRO": 0.171 - } - }, - { - "id": "SultanR/SmolTulu-1.7b-it-v0", - "name": "SmolTulu-1.7b-it-v0", - "developer": "SultanR", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6541, - "hfopenllm_v2/BBH": 0.3713, - "hfopenllm_v2/MATH Level 5": 0.0793, - "hfopenllm_v2/GPQA": 0.2693, - "hfopenllm_v2/MUSR": 0.354, - "hfopenllm_v2/MMLU-PRO": 0.171 - } - }, - { - "id": "SultanR/SmolTulu-1.7b-Reinforced", - "name": "SmolTulu-1.7b-Reinforced", - "developer": "SultanR", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6791, - "hfopenllm_v2/BBH": 0.3552, - "hfopenllm_v2/MATH Level 5": 0.0718, - "hfopenllm_v2/GPQA": 0.276, - "hfopenllm_v2/MUSR": 0.3406, - "hfopenllm_v2/MMLU-PRO": 0.1763 - } - }, - { - "id": "SultanR/SmolTulu-1.7b-RM", - "name": "SultanR/SmolTulu-1.7b-RM", - "developer": "SultanR", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5094, - "reward-bench/Chat": 0.743, - "reward-bench/Chat Hard": 0.4408, - "reward-bench/Safety": 0.5716, - "reward-bench/Reasoning": 0.2821 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/sumink.json b/data/developers/sumink.json deleted file mode 100644 index 6e7f0f5128686ba1a8c73f71e203d7195edc1a05..0000000000000000000000000000000000000000 --- a/data/developers/sumink.json +++ /dev/null @@ -1,313 +0,0 @@ -{ - "developer": "sumink", - "models": [ - { - "id": "sumink/bbhqwen", - "name": "bbhqwen", - "developer": "sumink", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1809, - "hfopenllm_v2/BBH": 0.3388, - "hfopenllm_v2/MATH Level 5": 0.0106, - "hfopenllm_v2/GPQA": 0.2576, - "hfopenllm_v2/MUSR": 0.4352, - "hfopenllm_v2/MMLU-PRO": 0.1617 - } - }, - { - "id": "sumink/bbhqwen2", - "name": "bbhqwen2", - "developer": "sumink", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1533, - "hfopenllm_v2/BBH": 0.3066, - "hfopenllm_v2/MATH Level 5": 0.006, - "hfopenllm_v2/GPQA": 0.2626, - "hfopenllm_v2/MUSR": 0.4431, - "hfopenllm_v2/MMLU-PRO": 0.1149 - } - }, - { - "id": "sumink/bbhqwen3", - "name": "bbhqwen3", - "developer": "sumink", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1943, - "hfopenllm_v2/BBH": 0.2951, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2576, - "hfopenllm_v2/MUSR": 0.3796, - "hfopenllm_v2/MMLU-PRO": 0.1166 - } - }, - { - "id": "sumink/bbhqwen4", - "name": "bbhqwen4", - "developer": "sumink", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1449, - "hfopenllm_v2/BBH": 0.3199, - "hfopenllm_v2/MATH Level 5": 0.006, - "hfopenllm_v2/GPQA": 0.2441, - "hfopenllm_v2/MUSR": 0.4029, - "hfopenllm_v2/MMLU-PRO": 0.1509 - } - }, - { - "id": "sumink/bbhqwen5", - "name": "bbhqwen5", - "developer": "sumink", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1522, - "hfopenllm_v2/BBH": 0.2913, - "hfopenllm_v2/MATH Level 5": 0.0023, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.4019, - "hfopenllm_v2/MMLU-PRO": 0.1131 - } - }, - { - "id": "sumink/bbhqwen6", - "name": "bbhqwen6", - "developer": "sumink", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1893, - "hfopenllm_v2/BBH": 0.2782, - "hfopenllm_v2/MATH Level 5": 0.0008, - "hfopenllm_v2/GPQA": 0.2584, - "hfopenllm_v2/MUSR": 0.358, - "hfopenllm_v2/MMLU-PRO": 0.1153 - } - }, - { - "id": "sumink/flflmillama", - "name": "flflmillama", - "developer": "sumink", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1676, - "hfopenllm_v2/BBH": 0.3851, - "hfopenllm_v2/MATH Level 5": 0.0196, - "hfopenllm_v2/GPQA": 0.2919, - "hfopenllm_v2/MUSR": 0.3591, - "hfopenllm_v2/MMLU-PRO": 0.2096 - } - }, - { - "id": "sumink/ftgpt", - "name": "ftgpt", - "developer": "sumink", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0787, - "hfopenllm_v2/BBH": 0.2919, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2643, - "hfopenllm_v2/MUSR": 0.4138, - "hfopenllm_v2/MMLU-PRO": 0.1172 - } - }, - { - "id": "sumink/llamaft", - "name": "llamaft", - "developer": "sumink", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1609, - "hfopenllm_v2/BBH": 0.3763, - "hfopenllm_v2/MATH Level 5": 0.0166, - "hfopenllm_v2/GPQA": 0.271, - "hfopenllm_v2/MUSR": 0.3498, - "hfopenllm_v2/MMLU-PRO": 0.2114 - } - }, - { - "id": "sumink/llamamerge", - "name": "llamamerge", - "developer": "sumink", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2672, - "hfopenllm_v2/BBH": 0.4632, - "hfopenllm_v2/MATH Level 5": 0.0151, - "hfopenllm_v2/GPQA": 0.2987, - "hfopenllm_v2/MUSR": 0.424, - "hfopenllm_v2/MMLU-PRO": 0.259 - } - }, - { - "id": "sumink/llftfl7", - "name": "llftfl7", - "developer": "sumink", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1714, - "hfopenllm_v2/BBH": 0.3786, - "hfopenllm_v2/MATH Level 5": 0.0106, - "hfopenllm_v2/GPQA": 0.281, - "hfopenllm_v2/MUSR": 0.3632, - "hfopenllm_v2/MMLU-PRO": 0.1743 - } - }, - { - "id": "sumink/llmer", - "name": "llmer", - "developer": "sumink", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3191, - "hfopenllm_v2/BBH": 0.4885, - "hfopenllm_v2/MATH Level 5": 0.065, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.4039, - "hfopenllm_v2/MMLU-PRO": 0.3529 - } - }, - { - "id": "sumink/Qmerft", - "name": "Qmerft", - "developer": "sumink", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1564, - "hfopenllm_v2/BBH": 0.2939, - "hfopenllm_v2/MATH Level 5": 0.0023, - "hfopenllm_v2/GPQA": 0.2525, - "hfopenllm_v2/MUSR": 0.3688, - "hfopenllm_v2/MMLU-PRO": 0.1157 - } - }, - { - "id": "sumink/Qwenftmodel", - "name": "Qwenftmodel", - "developer": "sumink", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1729, - "hfopenllm_v2/BBH": 0.3823, - "hfopenllm_v2/MATH Level 5": 0.0891, - "hfopenllm_v2/GPQA": 0.2567, - "hfopenllm_v2/MUSR": 0.3617, - "hfopenllm_v2/MMLU-PRO": 0.2339 - } - }, - { - "id": "sumink/Qwenmplus", - "name": "Qwenmplus", - "developer": "sumink", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.204, - "hfopenllm_v2/BBH": 0.3676, - "hfopenllm_v2/MATH Level 5": 0.0249, - "hfopenllm_v2/GPQA": 0.2852, - "hfopenllm_v2/MUSR": 0.3828, - "hfopenllm_v2/MMLU-PRO": 0.1992 - } - }, - { - "id": "sumink/Qwensci", - "name": "Qwensci", - "developer": "sumink", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.174, - "hfopenllm_v2/BBH": 0.3282, - "hfopenllm_v2/MATH Level 5": 0.0204, - "hfopenllm_v2/GPQA": 0.2584, - "hfopenllm_v2/MUSR": 0.3609, - "hfopenllm_v2/MMLU-PRO": 0.126 - } - }, - { - "id": "sumink/qwft", - "name": "qwft", - "developer": "sumink", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1197, - "hfopenllm_v2/BBH": 0.3002, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2525, - "hfopenllm_v2/MUSR": 0.3581, - "hfopenllm_v2/MMLU-PRO": 0.1129 - } - }, - { - "id": "sumink/qwmer", - "name": "qwmer", - "developer": "sumink", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2212, - "hfopenllm_v2/BBH": 0.4299, - "hfopenllm_v2/MATH Level 5": 0.0008, - "hfopenllm_v2/GPQA": 0.2869, - "hfopenllm_v2/MUSR": 0.4032, - "hfopenllm_v2/MMLU-PRO": 0.2215 - } - }, - { - "id": "sumink/solarmer3", - "name": "solarmer3", - "developer": "sumink", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3741, - "hfopenllm_v2/BBH": 0.5266, - "hfopenllm_v2/MATH Level 5": 0.0582, - "hfopenllm_v2/GPQA": 0.2911, - "hfopenllm_v2/MUSR": 0.4401, - "hfopenllm_v2/MMLU-PRO": 0.3323 - } - }, - { - "id": "sumink/somer", - "name": "somer", - "developer": "sumink", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.299, - "hfopenllm_v2/BBH": 0.5194, - "hfopenllm_v2/MATH Level 5": 0.0415, - "hfopenllm_v2/GPQA": 0.2987, - "hfopenllm_v2/MUSR": 0.465, - "hfopenllm_v2/MMLU-PRO": 0.3447 - } - }, - { - "id": "sumink/somer2", - "name": "somer2", - "developer": "sumink", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3132, - "hfopenllm_v2/BBH": 0.5167, - "hfopenllm_v2/MATH Level 5": 0.0468, - "hfopenllm_v2/GPQA": 0.3037, - "hfopenllm_v2/MUSR": 0.4663, - "hfopenllm_v2/MMLU-PRO": 0.3433 - } - }, - { - "id": "sumink/somerft", - "name": "somerft", - "developer": "sumink", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1431, - "hfopenllm_v2/BBH": 0.3093, - "hfopenllm_v2/MATH Level 5": 0.0144, - "hfopenllm_v2/GPQA": 0.2483, - "hfopenllm_v2/MUSR": 0.4045, - "hfopenllm_v2/MMLU-PRO": 0.1117 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/sunbaby.json b/data/developers/sunbaby.json deleted file mode 100644 index 281b12d8e58e49d898fe276b6cc13a9606253f6c..0000000000000000000000000000000000000000 --- a/data/developers/sunbaby.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "sunbaby", - "models": [ - { - "id": "sunbaby/BrainCog-8B-0.1-Instruct", - "name": "BrainCog-8B-0.1-Instruct", - "developer": "sunbaby", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4253, - "hfopenllm_v2/BBH": 0.4618, - "hfopenllm_v2/MATH Level 5": 0.0967, - "hfopenllm_v2/GPQA": 0.3012, - "hfopenllm_v2/MUSR": 0.3656, - "hfopenllm_v2/MMLU-PRO": 0.2858 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/supichi.json b/data/developers/supichi.json deleted file mode 100644 index be41814ef66baa8ee077cb346f080e74aa5594e5..0000000000000000000000000000000000000000 --- a/data/developers/supichi.json +++ /dev/null @@ -1,159 +0,0 @@ -{ - "developer": "Supichi", - "models": [ - { - "id": "Supichi/BBA-123", - "name": "BBA-123", - "developer": "Supichi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.208, - "hfopenllm_v2/BBH": 0.292, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3499, - "hfopenllm_v2/MMLU-PRO": 0.1167 - } - }, - { - "id": "Supichi/BBA99", - "name": "BBA99", - "developer": "Supichi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1407, - "hfopenllm_v2/BBH": 0.2769, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2634, - "hfopenllm_v2/MUSR": 0.3218, - "hfopenllm_v2/MMLU-PRO": 0.1112 - } - }, - { - "id": "Supichi/BBAI_135_Gemma", - "name": "BBAI_135_Gemma", - "developer": "Supichi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0656, - "hfopenllm_v2/BBH": 0.3568, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2676, - "hfopenllm_v2/MUSR": 0.3805, - "hfopenllm_v2/MMLU-PRO": 0.1672 - } - }, - { - "id": "Supichi/BBAI_250_Xia0_gZ", - "name": "BBAI_250_Xia0_gZ", - "developer": "Supichi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4685, - "hfopenllm_v2/BBH": 0.5568, - "hfopenllm_v2/MATH Level 5": 0.364, - "hfopenllm_v2/GPQA": 0.3213, - "hfopenllm_v2/MUSR": 0.4579, - "hfopenllm_v2/MMLU-PRO": 0.4465 - } - }, - { - "id": "Supichi/BBAI_275_Tsunami_gZ", - "name": "BBAI_275_Tsunami_gZ", - "developer": "Supichi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.537, - "hfopenllm_v2/BBH": 0.5531, - "hfopenllm_v2/MATH Level 5": 0.3285, - "hfopenllm_v2/GPQA": 0.3213, - "hfopenllm_v2/MUSR": 0.4448, - "hfopenllm_v2/MMLU-PRO": 0.4492 - } - }, - { - "id": "Supichi/BBAI_525_Tsu_gZ_Xia0", - "name": "BBAI_525_Tsu_gZ_Xia0", - "developer": "Supichi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5339, - "hfopenllm_v2/BBH": 0.5562, - "hfopenllm_v2/MATH Level 5": 0.3429, - "hfopenllm_v2/GPQA": 0.3121, - "hfopenllm_v2/MUSR": 0.4474, - "hfopenllm_v2/MMLU-PRO": 0.4477 - } - }, - { - "id": "Supichi/BBAI_78B_Calme_3_1_Ties", - "name": "BBAI_78B_Calme_3_1_Ties", - "developer": "Supichi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1828, - "hfopenllm_v2/BBH": 0.2828, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.229, - "hfopenllm_v2/MUSR": 0.31, - "hfopenllm_v2/MMLU-PRO": 0.1144 - } - }, - { - "id": "Supichi/BBAI_QWEEN_V000000_LUMEN_14B", - "name": "BBAI_QWEEN_V000000_LUMEN_14B", - "developer": "Supichi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1815, - "hfopenllm_v2/BBH": 0.2297, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2315, - "hfopenllm_v2/MUSR": 0.3445, - "hfopenllm_v2/MMLU-PRO": 0.116 - } - }, - { - "id": "Supichi/BBAIK29", - "name": "BBAIK29", - "developer": "Supichi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4588, - "hfopenllm_v2/BBH": 0.559, - "hfopenllm_v2/MATH Level 5": 0.3678, - "hfopenllm_v2/GPQA": 0.3121, - "hfopenllm_v2/MUSR": 0.4501, - "hfopenllm_v2/MMLU-PRO": 0.4469 - } - }, - { - "id": "Supichi/HF_TOKEN", - "name": "HF_TOKEN", - "developer": "Supichi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.138, - "hfopenllm_v2/BBH": 0.2764, - "hfopenllm_v2/MATH Level 5": 0.0008, - "hfopenllm_v2/GPQA": 0.2634, - "hfopenllm_v2/MUSR": 0.3272, - "hfopenllm_v2/MMLU-PRO": 0.111 - } - }, - { - "id": "Supichi/NJS26", - "name": "NJS26", - "developer": "Supichi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0448, - "hfopenllm_v2/BBH": 0.478, - "hfopenllm_v2/MATH Level 5": 0.0325, - "hfopenllm_v2/GPQA": 0.318, - "hfopenllm_v2/MUSR": 0.3854, - "hfopenllm_v2/MMLU-PRO": 0.3037 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/svak.json b/data/developers/svak.json deleted file mode 100644 index b354380187fdce2c4780478c1151e96721aa43f5..0000000000000000000000000000000000000000 --- a/data/developers/svak.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "Svak", - "models": [ - { - "id": "Svak/MN-12B-Inferor-v0.0", - "name": "MN-12B-Inferor-v0.0", - "developer": "Svak", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5708, - "hfopenllm_v2/BBH": 0.5195, - "hfopenllm_v2/MATH Level 5": 0.102, - "hfopenllm_v2/GPQA": 0.3087, - "hfopenllm_v2/MUSR": 0.4639, - "hfopenllm_v2/MMLU-PRO": 0.3559 - } - }, - { - "id": "Svak/MN-12B-Inferor-v0.1", - "name": "MN-12B-Inferor-v0.1", - "developer": "Svak", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6347, - "hfopenllm_v2/BBH": 0.5147, - "hfopenllm_v2/MATH Level 5": 0.1261, - "hfopenllm_v2/GPQA": 0.3255, - "hfopenllm_v2/MUSR": 0.4351, - "hfopenllm_v2/MMLU-PRO": 0.3662 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/swap-uniba.json b/data/developers/swap-uniba.json deleted file mode 100644 index 57d840de260b85652c7d17020ce275b4a2e9782a..0000000000000000000000000000000000000000 --- a/data/developers/swap-uniba.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "swap-uniba", - "models": [ - { - "id": "swap-uniba/LLaMAntino-3-ANITA-8B-Inst-DPO-ITA", - "name": "LLaMAntino-3-ANITA-8B-Inst-DPO-ITA", - "developer": "swap-uniba", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4815, - "hfopenllm_v2/BBH": 0.4936, - "hfopenllm_v2/MATH Level 5": 0.0483, - "hfopenllm_v2/GPQA": 0.2987, - "hfopenllm_v2/MUSR": 0.4387, - "hfopenllm_v2/MMLU-PRO": 0.3723 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/syed-hasan-8503.json b/data/developers/syed-hasan-8503.json deleted file mode 100644 index 54097e5df4fffe1a64a108ab1414f546df9428a9..0000000000000000000000000000000000000000 --- a/data/developers/syed-hasan-8503.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "Syed-Hasan-8503", - "models": [ - { - "id": "Syed-Hasan-8503/Phi-3-mini-4K-instruct-cpo-simpo", - "name": "Phi-3-mini-4K-instruct-cpo-simpo", - "developer": "Syed-Hasan-8503", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5714, - "hfopenllm_v2/BBH": 0.5682, - "hfopenllm_v2/MATH Level 5": 0.1571, - "hfopenllm_v2/GPQA": 0.3305, - "hfopenllm_v2/MUSR": 0.3964, - "hfopenllm_v2/MMLU-PRO": 0.3861 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/synergetic.json b/data/developers/synergetic.json deleted file mode 100644 index 2b5cd94290293780c1c36a296640d863780a989a..0000000000000000000000000000000000000000 --- a/data/developers/synergetic.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "synergetic", - "models": [ - { - "id": "synergetic/FrankenQwen2.5-14B", - "name": "FrankenQwen2.5-14B", - "developer": "synergetic", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1869, - "hfopenllm_v2/BBH": 0.6048, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.3843, - "hfopenllm_v2/MMLU-PRO": 0.4382 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/t145.json b/data/developers/t145.json deleted file mode 100644 index ad61070ed0a4e5da5fa3512b14d771ccdc6f5dd1..0000000000000000000000000000000000000000 --- a/data/developers/t145.json +++ /dev/null @@ -1,719 +0,0 @@ -{ - "developer": "T145", - "models": [ - { - "id": "T145/KRONOS-8B-V1-P1", - "name": "KRONOS-8B-V1-P1", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.785, - "hfopenllm_v2/BBH": 0.5085, - "hfopenllm_v2/MATH Level 5": 0.1979, - "hfopenllm_v2/GPQA": 0.2953, - "hfopenllm_v2/MUSR": 0.3881, - "hfopenllm_v2/MMLU-PRO": 0.376 - } - }, - { - "id": "T145/KRONOS-8B-V1-P2", - "name": "KRONOS-8B-V1-P2", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6724, - "hfopenllm_v2/BBH": 0.4772, - "hfopenllm_v2/MATH Level 5": 0.1601, - "hfopenllm_v2/GPQA": 0.2919, - "hfopenllm_v2/MUSR": 0.3568, - "hfopenllm_v2/MMLU-PRO": 0.3453 - } - }, - { - "id": "T145/KRONOS-8B-V1-P3", - "name": "KRONOS-8B-V1-P3", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7137, - "hfopenllm_v2/BBH": 0.5128, - "hfopenllm_v2/MATH Level 5": 0.1926, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3616, - "hfopenllm_v2/MMLU-PRO": 0.3405 - } - }, - { - "id": "T145/KRONOS-8B-V2", - "name": "KRONOS-8B-V2", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.518, - "hfopenllm_v2/BBH": 0.5133, - "hfopenllm_v2/MATH Level 5": 0.2266, - "hfopenllm_v2/GPQA": 0.2987, - "hfopenllm_v2/MUSR": 0.3829, - "hfopenllm_v2/MMLU-PRO": 0.3738 - } - }, - { - "id": "T145/KRONOS-8B-V3", - "name": "KRONOS-8B-V3", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5475, - "hfopenllm_v2/BBH": 0.5119, - "hfopenllm_v2/MATH Level 5": 0.2598, - "hfopenllm_v2/GPQA": 0.2886, - "hfopenllm_v2/MUSR": 0.3922, - "hfopenllm_v2/MMLU-PRO": 0.3738 - } - }, - { - "id": "T145/KRONOS-8B-V4", - "name": "KRONOS-8B-V4", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7889, - "hfopenllm_v2/BBH": 0.5092, - "hfopenllm_v2/MATH Level 5": 0.1949, - "hfopenllm_v2/GPQA": 0.2894, - "hfopenllm_v2/MUSR": 0.383, - "hfopenllm_v2/MMLU-PRO": 0.3786 - } - }, - { - "id": "T145/KRONOS-8B-V5", - "name": "KRONOS-8B-V5", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5405, - "hfopenllm_v2/BBH": 0.5089, - "hfopenllm_v2/MATH Level 5": 0.2689, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.4055, - "hfopenllm_v2/MMLU-PRO": 0.3759 - } - }, - { - "id": "T145/KRONOS-8B-V6", - "name": "KRONOS-8B-V6", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7022, - "hfopenllm_v2/BBH": 0.5034, - "hfopenllm_v2/MATH Level 5": 0.2598, - "hfopenllm_v2/GPQA": 0.2794, - "hfopenllm_v2/MUSR": 0.4121, - "hfopenllm_v2/MMLU-PRO": 0.3501 - } - }, - { - "id": "T145/KRONOS-8B-V7", - "name": "KRONOS-8B-V7", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3529, - "hfopenllm_v2/BBH": 0.4526, - "hfopenllm_v2/MATH Level 5": 0.111, - "hfopenllm_v2/GPQA": 0.2668, - "hfopenllm_v2/MUSR": 0.3671, - "hfopenllm_v2/MMLU-PRO": 0.2697 - } - }, - { - "id": "T145/KRONOS-8B-V8", - "name": "KRONOS-8B-V8", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.777, - "hfopenllm_v2/BBH": 0.5094, - "hfopenllm_v2/MATH Level 5": 0.2047, - "hfopenllm_v2/GPQA": 0.2894, - "hfopenllm_v2/MUSR": 0.3869, - "hfopenllm_v2/MMLU-PRO": 0.3782 - } - }, - { - "id": "T145/KRONOS-8B-V9", - "name": "KRONOS-8B-V9", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7856, - "hfopenllm_v2/BBH": 0.5099, - "hfopenllm_v2/MATH Level 5": 0.1986, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.3868, - "hfopenllm_v2/MMLU-PRO": 0.3752 - } - }, - { - "id": "T145/Llama-3.1-8B-Instruct-Zeus", - "name": "Llama-3.1-8B-Instruct-Zeus", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7941, - "hfopenllm_v2/BBH": 0.5174, - "hfopenllm_v2/MATH Level 5": 0.1956, - "hfopenllm_v2/GPQA": 0.3012, - "hfopenllm_v2/MUSR": 0.3976, - "hfopenllm_v2/MMLU-PRO": 0.3893 - } - }, - { - "id": "T145/Llama-3.1-8B-Zeus", - "name": "Llama-3.1-8B-Zeus", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3518, - "hfopenllm_v2/BBH": 0.3671, - "hfopenllm_v2/MATH Level 5": 0.0144, - "hfopenllm_v2/GPQA": 0.2651, - "hfopenllm_v2/MUSR": 0.3316, - "hfopenllm_v2/MMLU-PRO": 0.1332 - } - }, - { - "id": "T145/Meta-Llama-3.1-8B-Instruct-TIES", - "name": "Meta-Llama-3.1-8B-Instruct-TIES", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5424, - "hfopenllm_v2/BBH": 0.507, - "hfopenllm_v2/MATH Level 5": 0.21, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.3843, - "hfopenllm_v2/MMLU-PRO": 0.378 - } - }, - { - "id": "T145/qwen-2.5-3B-merge-test", - "name": "qwen-2.5-3B-merge-test", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5751, - "hfopenllm_v2/BBH": 0.4842, - "hfopenllm_v2/MATH Level 5": 0.3202, - "hfopenllm_v2/GPQA": 0.2852, - "hfopenllm_v2/MUSR": 0.4007, - "hfopenllm_v2/MMLU-PRO": 0.329 - } - }, - { - "id": "T145/ZEUS-8B-V10", - "name": "ZEUS-8B-V10", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7707, - "hfopenllm_v2/BBH": 0.527, - "hfopenllm_v2/MATH Level 5": 0.2115, - "hfopenllm_v2/GPQA": 0.3247, - "hfopenllm_v2/MUSR": 0.3898, - "hfopenllm_v2/MMLU-PRO": 0.3904 - } - }, - { - "id": "T145/ZEUS-8B-V11", - "name": "ZEUS-8B-V11", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.81, - "hfopenllm_v2/BBH": 0.5162, - "hfopenllm_v2/MATH Level 5": 0.1964, - "hfopenllm_v2/GPQA": 0.3146, - "hfopenllm_v2/MUSR": 0.3807, - "hfopenllm_v2/MMLU-PRO": 0.3884 - } - }, - { - "id": "T145/ZEUS-8B-V12", - "name": "ZEUS-8B-V12", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7816, - "hfopenllm_v2/BBH": 0.5254, - "hfopenllm_v2/MATH Level 5": 0.2115, - "hfopenllm_v2/GPQA": 0.3205, - "hfopenllm_v2/MUSR": 0.3858, - "hfopenllm_v2/MMLU-PRO": 0.3912 - } - }, - { - "id": "T145/ZEUS-8B-V13", - "name": "ZEUS-8B-V13", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7904, - "hfopenllm_v2/BBH": 0.5277, - "hfopenllm_v2/MATH Level 5": 0.2137, - "hfopenllm_v2/GPQA": 0.3238, - "hfopenllm_v2/MUSR": 0.3845, - "hfopenllm_v2/MMLU-PRO": 0.3911 - } - }, - { - "id": "T145/ZEUS-8B-V13-abliterated", - "name": "ZEUS-8B-V13-abliterated", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7878, - "hfopenllm_v2/BBH": 0.5198, - "hfopenllm_v2/MATH Level 5": 0.179, - "hfopenllm_v2/GPQA": 0.3112, - "hfopenllm_v2/MUSR": 0.3871, - "hfopenllm_v2/MMLU-PRO": 0.3872 - } - }, - { - "id": "T145/ZEUS-8B-V14", - "name": "ZEUS-8B-V14", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7709, - "hfopenllm_v2/BBH": 0.5275, - "hfopenllm_v2/MATH Level 5": 0.213, - "hfopenllm_v2/GPQA": 0.3205, - "hfopenllm_v2/MUSR": 0.3844, - "hfopenllm_v2/MMLU-PRO": 0.3914 - } - }, - { - "id": "T145/ZEUS-8B-V15", - "name": "ZEUS-8B-V15", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7013, - "hfopenllm_v2/BBH": 0.5538, - "hfopenllm_v2/MATH Level 5": 0.2304, - "hfopenllm_v2/GPQA": 0.276, - "hfopenllm_v2/MUSR": 0.402, - "hfopenllm_v2/MMLU-PRO": 0.4059 - } - }, - { - "id": "T145/ZEUS-8B-V16", - "name": "ZEUS-8B-V16", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7925, - "hfopenllm_v2/BBH": 0.5266, - "hfopenllm_v2/MATH Level 5": 0.2205, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.3951, - "hfopenllm_v2/MMLU-PRO": 0.3926 - } - }, - { - "id": "T145/ZEUS-8B-V17", - "name": "ZEUS-8B-V17", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7941, - "hfopenllm_v2/BBH": 0.5251, - "hfopenllm_v2/MATH Level 5": 0.2243, - "hfopenllm_v2/GPQA": 0.3221, - "hfopenllm_v2/MUSR": 0.4016, - "hfopenllm_v2/MMLU-PRO": 0.3935 - } - }, - { - "id": "T145/ZEUS-8B-V17-abliterated", - "name": "ZEUS-8B-V17-abliterated", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7576, - "hfopenllm_v2/BBH": 0.52, - "hfopenllm_v2/MATH Level 5": 0.0438, - "hfopenllm_v2/GPQA": 0.3037, - "hfopenllm_v2/MUSR": 0.4269, - "hfopenllm_v2/MMLU-PRO": 0.3622 - } - }, - { - "id": "T145/ZEUS-8B-V17-abliterated-V2", - "name": "ZEUS-8B-V17-abliterated-V2", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6532, - "hfopenllm_v2/BBH": 0.4928, - "hfopenllm_v2/MATH Level 5": 0.1118, - "hfopenllm_v2/GPQA": 0.2735, - "hfopenllm_v2/MUSR": 0.3407, - "hfopenllm_v2/MMLU-PRO": 0.3402 - } - }, - { - "id": "T145/ZEUS-8B-V17-abliterated-V4", - "name": "ZEUS-8B-V17-abliterated-V4", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7228, - "hfopenllm_v2/BBH": 0.5169, - "hfopenllm_v2/MATH Level 5": 0.0937, - "hfopenllm_v2/GPQA": 0.2836, - "hfopenllm_v2/MUSR": 0.4187, - "hfopenllm_v2/MMLU-PRO": 0.3774 - } - }, - { - "id": "T145/ZEUS-8B-V18", - "name": "ZEUS-8B-V18", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7834, - "hfopenllm_v2/BBH": 0.527, - "hfopenllm_v2/MATH Level 5": 0.2183, - "hfopenllm_v2/GPQA": 0.3213, - "hfopenllm_v2/MUSR": 0.4043, - "hfopenllm_v2/MMLU-PRO": 0.3942 - } - }, - { - "id": "T145/ZEUS-8B-V19", - "name": "ZEUS-8B-V19", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7883, - "hfopenllm_v2/BBH": 0.5276, - "hfopenllm_v2/MATH Level 5": 0.2205, - "hfopenllm_v2/GPQA": 0.3221, - "hfopenllm_v2/MUSR": 0.4043, - "hfopenllm_v2/MMLU-PRO": 0.3934 - } - }, - { - "id": "T145/ZEUS-8B-V2", - "name": "ZEUS-8B-V2", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8029, - "hfopenllm_v2/BBH": 0.5194, - "hfopenllm_v2/MATH Level 5": 0.216, - "hfopenllm_v2/GPQA": 0.302, - "hfopenllm_v2/MUSR": 0.391, - "hfopenllm_v2/MMLU-PRO": 0.3896 - } - }, - { - "id": "T145/ZEUS-8B-V2-abliterated", - "name": "ZEUS-8B-V2-abliterated", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7895, - "hfopenllm_v2/BBH": 0.5129, - "hfopenllm_v2/MATH Level 5": 0.2115, - "hfopenllm_v2/GPQA": 0.3129, - "hfopenllm_v2/MUSR": 0.3911, - "hfopenllm_v2/MMLU-PRO": 0.3825 - } - }, - { - "id": "T145/ZEUS-8B-V2-ORPO", - "name": "ZEUS-8B-V2-ORPO", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7187, - "hfopenllm_v2/BBH": 0.5075, - "hfopenllm_v2/MATH Level 5": 0.1828, - "hfopenllm_v2/GPQA": 0.3104, - "hfopenllm_v2/MUSR": 0.3935, - "hfopenllm_v2/MMLU-PRO": 0.3678 - } - }, - { - "id": "T145/ZEUS-8B-V20", - "name": "ZEUS-8B-V20", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7956, - "hfopenllm_v2/BBH": 0.5244, - "hfopenllm_v2/MATH Level 5": 0.219, - "hfopenllm_v2/GPQA": 0.323, - "hfopenllm_v2/MUSR": 0.4043, - "hfopenllm_v2/MMLU-PRO": 0.393 - } - }, - { - "id": "T145/ZEUS-8B-V21", - "name": "ZEUS-8B-V21", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3785, - "hfopenllm_v2/BBH": 0.3398, - "hfopenllm_v2/MATH Level 5": 0.1594, - "hfopenllm_v2/GPQA": 0.2643, - "hfopenllm_v2/MUSR": 0.3262, - "hfopenllm_v2/MMLU-PRO": 0.1714 - } - }, - { - "id": "T145/ZEUS-8B-V22", - "name": "ZEUS-8B-V22", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7995, - "hfopenllm_v2/BBH": 0.5245, - "hfopenllm_v2/MATH Level 5": 0.2228, - "hfopenllm_v2/GPQA": 0.328, - "hfopenllm_v2/MUSR": 0.399, - "hfopenllm_v2/MMLU-PRO": 0.3938 - } - }, - { - "id": "T145/ZEUS-8B-V23", - "name": "ZEUS-8B-V23", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7621, - "hfopenllm_v2/BBH": 0.5195, - "hfopenllm_v2/MATH Level 5": 0.182, - "hfopenllm_v2/GPQA": 0.3096, - "hfopenllm_v2/MUSR": 0.3922, - "hfopenllm_v2/MMLU-PRO": 0.3666 - } - }, - { - "id": "T145/ZEUS-8B-V24", - "name": "ZEUS-8B-V24", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6, - "hfopenllm_v2/BBH": 0.4778, - "hfopenllm_v2/MATH Level 5": 0.1458, - "hfopenllm_v2/GPQA": 0.2617, - "hfopenllm_v2/MUSR": 0.3729, - "hfopenllm_v2/MMLU-PRO": 0.3285 - } - }, - { - "id": "T145/ZEUS-8B-V25", - "name": "ZEUS-8B-V25", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.332, - "hfopenllm_v2/BBH": 0.4547, - "hfopenllm_v2/MATH Level 5": 0.2039, - "hfopenllm_v2/GPQA": 0.2643, - "hfopenllm_v2/MUSR": 0.3488, - "hfopenllm_v2/MMLU-PRO": 0.2885 - } - }, - { - "id": "T145/ZEUS-8B-V26", - "name": "ZEUS-8B-V26", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6708, - "hfopenllm_v2/BBH": 0.5232, - "hfopenllm_v2/MATH Level 5": 0.1246, - "hfopenllm_v2/GPQA": 0.2953, - "hfopenllm_v2/MUSR": 0.4016, - "hfopenllm_v2/MMLU-PRO": 0.3907 - } - }, - { - "id": "T145/ZEUS-8B-V27", - "name": "ZEUS-8B-V27", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6544, - "hfopenllm_v2/BBH": 0.523, - "hfopenllm_v2/MATH Level 5": 0.1344, - "hfopenllm_v2/GPQA": 0.3079, - "hfopenllm_v2/MUSR": 0.3977, - "hfopenllm_v2/MMLU-PRO": 0.3902 - } - }, - { - "id": "T145/ZEUS-8B-V28", - "name": "ZEUS-8B-V28", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6353, - "hfopenllm_v2/BBH": 0.5254, - "hfopenllm_v2/MATH Level 5": 0.1269, - "hfopenllm_v2/GPQA": 0.3037, - "hfopenllm_v2/MUSR": 0.3896, - "hfopenllm_v2/MMLU-PRO": 0.3902 - } - }, - { - "id": "T145/ZEUS-8B-V29", - "name": "ZEUS-8B-V29", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7418, - "hfopenllm_v2/BBH": 0.5253, - "hfopenllm_v2/MATH Level 5": 0.1601, - "hfopenllm_v2/GPQA": 0.3263, - "hfopenllm_v2/MUSR": 0.4003, - "hfopenllm_v2/MMLU-PRO": 0.392 - } - }, - { - "id": "T145/ZEUS-8B-V2L1", - "name": "ZEUS-8B-V2L1", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3192, - "hfopenllm_v2/BBH": 0.5013, - "hfopenllm_v2/MATH Level 5": 0.1239, - "hfopenllm_v2/GPQA": 0.3129, - "hfopenllm_v2/MUSR": 0.3882, - "hfopenllm_v2/MMLU-PRO": 0.3638 - } - }, - { - "id": "T145/ZEUS-8B-V2L2", - "name": "ZEUS-8B-V2L2", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8021, - "hfopenllm_v2/BBH": 0.5203, - "hfopenllm_v2/MATH Level 5": 0.2017, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.3975, - "hfopenllm_v2/MMLU-PRO": 0.3884 - } - }, - { - "id": "T145/ZEUS-8B-V3", - "name": "ZEUS-8B-V3", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7887, - "hfopenllm_v2/BBH": 0.5265, - "hfopenllm_v2/MATH Level 5": 0.1677, - "hfopenllm_v2/GPQA": 0.3221, - "hfopenllm_v2/MUSR": 0.4017, - "hfopenllm_v2/MMLU-PRO": 0.3804 - } - }, - { - "id": "T145/ZEUS-8B-V30", - "name": "ZEUS-8B-V30", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7436, - "hfopenllm_v2/BBH": 0.5243, - "hfopenllm_v2/MATH Level 5": 0.1586, - "hfopenllm_v2/GPQA": 0.3205, - "hfopenllm_v2/MUSR": 0.4029, - "hfopenllm_v2/MMLU-PRO": 0.3944 - } - }, - { - "id": "T145/ZEUS-8B-V4", - "name": "ZEUS-8B-V4", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7807, - "hfopenllm_v2/BBH": 0.5246, - "hfopenllm_v2/MATH Level 5": 0.1926, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.4029, - "hfopenllm_v2/MMLU-PRO": 0.3788 - } - }, - { - "id": "T145/ZEUS-8B-V6", - "name": "ZEUS-8B-V6", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7838, - "hfopenllm_v2/BBH": 0.524, - "hfopenllm_v2/MATH Level 5": 0.2024, - "hfopenllm_v2/GPQA": 0.3045, - "hfopenllm_v2/MUSR": 0.4068, - "hfopenllm_v2/MMLU-PRO": 0.3759 - } - }, - { - "id": "T145/ZEUS-8B-V7", - "name": "ZEUS-8B-V7", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7786, - "hfopenllm_v2/BBH": 0.507, - "hfopenllm_v2/MATH Level 5": 0.148, - "hfopenllm_v2/GPQA": 0.297, - "hfopenllm_v2/MUSR": 0.4162, - "hfopenllm_v2/MMLU-PRO": 0.3812 - } - }, - { - "id": "T145/ZEUS-8B-V8", - "name": "ZEUS-8B-V8", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7914, - "hfopenllm_v2/BBH": 0.5065, - "hfopenllm_v2/MATH Level 5": 0.1329, - "hfopenllm_v2/GPQA": 0.2878, - "hfopenllm_v2/MUSR": 0.4214, - "hfopenllm_v2/MMLU-PRO": 0.3761 - } - }, - { - "id": "T145/ZEUS-8B-V9", - "name": "ZEUS-8B-V9", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5551, - "hfopenllm_v2/BBH": 0.5207, - "hfopenllm_v2/MATH Level 5": 0.2137, - "hfopenllm_v2/GPQA": 0.2911, - "hfopenllm_v2/MUSR": 0.3949, - "hfopenllm_v2/MMLU-PRO": 0.3901 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/talha2001.json b/data/developers/talha2001.json deleted file mode 100644 index 52c08146595f4c5ea023cd8df02edf3acd2ef3dd..0000000000000000000000000000000000000000 --- a/data/developers/talha2001.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "talha2001", - "models": [ - { - "id": "talha2001/Beast-Soul-new", - "name": "Beast-Soul-new", - "developer": "talha2001", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4854, - "hfopenllm_v2/BBH": 0.5227, - "hfopenllm_v2/MATH Level 5": 0.074, - "hfopenllm_v2/GPQA": 0.2819, - "hfopenllm_v2/MUSR": 0.4459, - "hfopenllm_v2/MMLU-PRO": 0.3102 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/tangledgroup.json b/data/developers/tangledgroup.json deleted file mode 100644 index 12057061e89bb04a722bd13b96087a62d695a722..0000000000000000000000000000000000000000 --- a/data/developers/tangledgroup.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "tangledgroup", - "models": [ - { - "id": "tangledgroup/tangled-llama-pints-1.5b-v0.1-instruct", - "name": "tangled-llama-pints-1.5b-v0.1-instruct", - "developer": "tangledgroup", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1509, - "hfopenllm_v2/BBH": 0.3143, - "hfopenllm_v2/MATH Level 5": 0.0121, - "hfopenllm_v2/GPQA": 0.2399, - "hfopenllm_v2/MUSR": 0.3761, - "hfopenllm_v2/MMLU-PRO": 0.1109 - } - }, - { - "id": "tangledgroup/tangled-llama-pints-1.5b-v0.2-instruct", - "name": "tangled-llama-pints-1.5b-v0.2-instruct", - "developer": "tangledgroup", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1724, - "hfopenllm_v2/BBH": 0.3158, - "hfopenllm_v2/MATH Level 5": 0.0128, - "hfopenllm_v2/GPQA": 0.2416, - "hfopenllm_v2/MUSR": 0.3643, - "hfopenllm_v2/MMLU-PRO": 0.1117 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/tanliboy.json b/data/developers/tanliboy.json deleted file mode 100644 index daa17945601d8528cbfc7c883e4dc4315ea363fc..0000000000000000000000000000000000000000 --- a/data/developers/tanliboy.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "developer": "tanliboy", - "models": [ - { - "id": "tanliboy/lambda-gemma-2-9b-dpo", - "name": "lambda-gemma-2-9b-dpo", - "developer": "tanliboy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4501, - "hfopenllm_v2/BBH": 0.5472, - "hfopenllm_v2/MATH Level 5": 0.0944, - "hfopenllm_v2/GPQA": 0.3138, - "hfopenllm_v2/MUSR": 0.4017, - "hfopenllm_v2/MMLU-PRO": 0.3792 - } - }, - { - "id": "tanliboy/lambda-qwen2.5-14b-dpo-test", - "name": "lambda-qwen2.5-14b-dpo-test", - "developer": "tanliboy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8231, - "hfopenllm_v2/BBH": 0.6394, - "hfopenllm_v2/MATH Level 5": 0.5461, - "hfopenllm_v2/GPQA": 0.3624, - "hfopenllm_v2/MUSR": 0.426, - "hfopenllm_v2/MMLU-PRO": 0.4848 - } - }, - { - "id": "tanliboy/lambda-qwen2.5-32b-dpo-test", - "name": "lambda-qwen2.5-32b-dpo-test", - "developer": "tanliboy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8084, - "hfopenllm_v2/BBH": 0.6764, - "hfopenllm_v2/MATH Level 5": 0.6103, - "hfopenllm_v2/GPQA": 0.3565, - "hfopenllm_v2/MUSR": 0.4274, - "hfopenllm_v2/MMLU-PRO": 0.5657 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/tannedbum.json b/data/developers/tannedbum.json deleted file mode 100644 index 19b7cd9a7d2aac309c9ef8c8091ea782d7354a70..0000000000000000000000000000000000000000 --- a/data/developers/tannedbum.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "developer": "tannedbum", - "models": [ - { - "id": "tannedbum/Ellaria-9B", - "name": "Ellaria-9B", - "developer": "tannedbum", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7826, - "hfopenllm_v2/BBH": 0.5942, - "hfopenllm_v2/MATH Level 5": 0.2077, - "hfopenllm_v2/GPQA": 0.3331, - "hfopenllm_v2/MUSR": 0.4151, - "hfopenllm_v2/MMLU-PRO": 0.4205 - } - }, - { - "id": "tannedbum/L3-Nymeria-Maid-8B", - "name": "L3-Nymeria-Maid-8B", - "developer": "tannedbum", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.725, - "hfopenllm_v2/BBH": 0.5146, - "hfopenllm_v2/MATH Level 5": 0.0937, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.3751, - "hfopenllm_v2/MMLU-PRO": 0.3747 - } - }, - { - "id": "tannedbum/L3-Nymeria-v2-8B", - "name": "L3-Nymeria-v2-8B", - "developer": "tannedbum", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7168, - "hfopenllm_v2/BBH": 0.5224, - "hfopenllm_v2/MATH Level 5": 0.0921, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.3699, - "hfopenllm_v2/MMLU-PRO": 0.3753 - } - }, - { - "id": "tannedbum/L3-Rhaenys-8B", - "name": "L3-Rhaenys-8B", - "developer": "tannedbum", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7363, - "hfopenllm_v2/BBH": 0.5299, - "hfopenllm_v2/MATH Level 5": 0.0876, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.3725, - "hfopenllm_v2/MMLU-PRO": 0.3799 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/tarek07.json b/data/developers/tarek07.json deleted file mode 100644 index 8b3233c296ab77e65724be7e95482f31b6b0200b..0000000000000000000000000000000000000000 --- a/data/developers/tarek07.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "Tarek07", - "models": [ - { - "id": "Tarek07/Progenitor-V1.1-LLaMa-70B", - "name": "Progenitor-V1.1-LLaMa-70B", - "developer": "Tarek07", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6906, - "hfopenllm_v2/BBH": 0.6971, - "hfopenllm_v2/MATH Level 5": 0.3573, - "hfopenllm_v2/GPQA": 0.4581, - "hfopenllm_v2/MUSR": 0.4736, - "hfopenllm_v2/MMLU-PRO": 0.5465 - } - }, - { - "id": "Tarek07/Thalassic-Alpha-LLaMa-70B", - "name": "Thalassic-Alpha-LLaMa-70B", - "developer": "Tarek07", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7003, - "hfopenllm_v2/BBH": 0.694, - "hfopenllm_v2/MATH Level 5": 0.315, - "hfopenllm_v2/GPQA": 0.4438, - "hfopenllm_v2/MUSR": 0.4802, - "hfopenllm_v2/MMLU-PRO": 0.5435 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/teezee.json b/data/developers/teezee.json deleted file mode 100644 index 144d54ca187a75a6a7a41a675e96d0e2498668c9..0000000000000000000000000000000000000000 --- a/data/developers/teezee.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "TeeZee", - "models": [ - { - "id": "TeeZee/DoubleBagel-57B-v1.0", - "name": "DoubleBagel-57B-v1.0", - "developer": "TeeZee", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2336, - "hfopenllm_v2/BBH": 0.3251, - "hfopenllm_v2/MATH Level 5": 0.0098, - "hfopenllm_v2/GPQA": 0.276, - "hfopenllm_v2/MUSR": 0.4315, - "hfopenllm_v2/MMLU-PRO": 0.1478 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/teknium.json b/data/developers/teknium.json deleted file mode 100644 index e391d123290feb9376748c2a1f02aa900cfda727..0000000000000000000000000000000000000000 --- a/data/developers/teknium.json +++ /dev/null @@ -1,75 +0,0 @@ -{ - "developer": "teknium", - "models": [ - { - "id": "teknium/CollectiveCognition-v1.1-Mistral-7B", - "name": "CollectiveCognition-v1.1-Mistral-7B", - "developer": "teknium", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.279, - "hfopenllm_v2/BBH": 0.4493, - "hfopenllm_v2/MATH Level 5": 0.031, - "hfopenllm_v2/GPQA": 0.2869, - "hfopenllm_v2/MUSR": 0.3869, - "hfopenllm_v2/MMLU-PRO": 0.2837 - } - }, - { - "id": "teknium/OpenHermes-13B", - "name": "OpenHermes-13B", - "developer": "teknium", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2668, - "hfopenllm_v2/BBH": 0.4206, - "hfopenllm_v2/MATH Level 5": 0.0121, - "hfopenllm_v2/GPQA": 0.2727, - "hfopenllm_v2/MUSR": 0.4043, - "hfopenllm_v2/MMLU-PRO": 0.2389 - } - }, - { - "id": "teknium/OpenHermes-2-Mistral-7B", - "name": "OpenHermes-2-Mistral-7B", - "developer": "teknium", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5286, - "hfopenllm_v2/BBH": 0.4948, - "hfopenllm_v2/MATH Level 5": 0.0453, - "hfopenllm_v2/GPQA": 0.2836, - "hfopenllm_v2/MUSR": 0.452, - "hfopenllm_v2/MMLU-PRO": 0.2931 - } - }, - { - "id": "teknium/OpenHermes-2.5-Mistral-7B", - "name": "OpenHermes-2.5-Mistral-7B", - "developer": "teknium", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5571, - "hfopenllm_v2/BBH": 0.487, - "hfopenllm_v2/MATH Level 5": 0.0506, - "hfopenllm_v2/GPQA": 0.2836, - "hfopenllm_v2/MUSR": 0.4242, - "hfopenllm_v2/MMLU-PRO": 0.3054 - } - }, - { - "id": "teknium/OpenHermes-7B", - "name": "OpenHermes-7B", - "developer": "teknium", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1813, - "hfopenllm_v2/BBH": 0.362, - "hfopenllm_v2/MATH Level 5": 0.0159, - "hfopenllm_v2/GPQA": 0.2693, - "hfopenllm_v2/MUSR": 0.4324, - "hfopenllm_v2/MMLU-PRO": 0.1933 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/telugu-llm-labs.json b/data/developers/telugu-llm-labs.json deleted file mode 100644 index 1b4f9bae71f6b7e1507f62ac83550ac483c30b65..0000000000000000000000000000000000000000 --- a/data/developers/telugu-llm-labs.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "Telugu-LLM-Labs", - "models": [ - { - "id": "Telugu-LLM-Labs/Indic-gemma-2b-finetuned-sft-Navarasa-2.0", - "name": "Indic-gemma-2b-finetuned-sft-Navarasa-2.0", - "developer": "Telugu-LLM-Labs", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2103, - "hfopenllm_v2/BBH": 0.3241, - "hfopenllm_v2/MATH Level 5": 0.0272, - "hfopenllm_v2/GPQA": 0.2433, - "hfopenllm_v2/MUSR": 0.3899, - "hfopenllm_v2/MMLU-PRO": 0.1279 - } - }, - { - "id": "Telugu-LLM-Labs/Indic-gemma-7b-finetuned-sft-Navarasa-2.0", - "name": "Indic-gemma-7b-finetuned-sft-Navarasa-2.0", - "developer": "Telugu-LLM-Labs", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3237, - "hfopenllm_v2/BBH": 0.4023, - "hfopenllm_v2/MATH Level 5": 0.0257, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.4083, - "hfopenllm_v2/MMLU-PRO": 0.235 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/tencentarc.json b/data/developers/tencentarc.json deleted file mode 100644 index 6bdcb11bbff177b71f03310639417506e24254cb..0000000000000000000000000000000000000000 --- a/data/developers/tencentarc.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "developer": "TencentARC", - "models": [ - { - "id": "TencentARC/LLaMA-Pro-8B", - "name": "LLaMA-Pro-8B", - "developer": "TencentARC", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2277, - "hfopenllm_v2/BBH": 0.3484, - "hfopenllm_v2/MATH Level 5": 0.0189, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.4018, - "hfopenllm_v2/MMLU-PRO": 0.1811 - } - }, - { - "id": "TencentARC/LLaMA-Pro-8B-Instruct", - "name": "LLaMA-Pro-8B-Instruct", - "developer": "TencentARC", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4486, - "hfopenllm_v2/BBH": 0.4224, - "hfopenllm_v2/MATH Level 5": 0.0249, - "hfopenllm_v2/GPQA": 0.2743, - "hfopenllm_v2/MUSR": 0.419, - "hfopenllm_v2/MMLU-PRO": 0.1946 - } - }, - { - "id": "TencentARC/MetaMath-Mistral-Pro", - "name": "MetaMath-Mistral-Pro", - "developer": "TencentARC", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2119, - "hfopenllm_v2/BBH": 0.4413, - "hfopenllm_v2/MATH Level 5": 0.0763, - "hfopenllm_v2/GPQA": 0.2693, - "hfopenllm_v2/MUSR": 0.3524, - "hfopenllm_v2/MMLU-PRO": 0.2472 - } - }, - { - "id": "TencentARC/Mistral_Pro_8B_v0.1", - "name": "Mistral_Pro_8B_v0.1", - "developer": "TencentARC", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2115, - "hfopenllm_v2/BBH": 0.4526, - "hfopenllm_v2/MATH Level 5": 0.0566, - "hfopenllm_v2/GPQA": 0.2802, - "hfopenllm_v2/MUSR": 0.4242, - "hfopenllm_v2/MMLU-PRO": 0.2765 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/tensopolis.json b/data/developers/tensopolis.json deleted file mode 100644 index ef6a38c9fc2daee3b5bb5e68fa5563d4aee82191..0000000000000000000000000000000000000000 --- a/data/developers/tensopolis.json +++ /dev/null @@ -1,215 +0,0 @@ -{ - "developer": "tensopolis", - "models": [ - { - "id": "tensopolis/falcon3-10b-tensopolis-v1", - "name": "falcon3-10b-tensopolis-v1", - "developer": "tensopolis", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7817, - "hfopenllm_v2/BBH": 0.6182, - "hfopenllm_v2/MATH Level 5": 0.2749, - "hfopenllm_v2/GPQA": 0.3297, - "hfopenllm_v2/MUSR": 0.4375, - "hfopenllm_v2/MMLU-PRO": 0.442 - } - }, - { - "id": "tensopolis/falcon3-10b-tensopolis-v2", - "name": "falcon3-10b-tensopolis-v2", - "developer": "tensopolis", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7792, - "hfopenllm_v2/BBH": 0.6182, - "hfopenllm_v2/MATH Level 5": 0.2666, - "hfopenllm_v2/GPQA": 0.3272, - "hfopenllm_v2/MUSR": 0.4297, - "hfopenllm_v2/MMLU-PRO": 0.4424 - } - }, - { - "id": "tensopolis/lamarckvergence-14b-tensopolis-v1", - "name": "lamarckvergence-14b-tensopolis-v1", - "developer": "tensopolis", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7604, - "hfopenllm_v2/BBH": 0.6561, - "hfopenllm_v2/MATH Level 5": 0.5166, - "hfopenllm_v2/GPQA": 0.3607, - "hfopenllm_v2/MUSR": 0.4475, - "hfopenllm_v2/MMLU-PRO": 0.525 - } - }, - { - "id": "tensopolis/mistral-small-2501-tensopolis-v1", - "name": "mistral-small-2501-tensopolis-v1", - "developer": "tensopolis", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7762, - "hfopenllm_v2/BBH": 0.6475, - "hfopenllm_v2/MATH Level 5": 0.4441, - "hfopenllm_v2/GPQA": 0.3574, - "hfopenllm_v2/MUSR": 0.428, - "hfopenllm_v2/MMLU-PRO": 0.4465 - } - }, - { - "id": "tensopolis/mistral-small-r1-tensopolis", - "name": "mistral-small-r1-tensopolis", - "developer": "tensopolis", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4622, - "hfopenllm_v2/BBH": 0.5436, - "hfopenllm_v2/MATH Level 5": 0.2908, - "hfopenllm_v2/GPQA": 0.2819, - "hfopenllm_v2/MUSR": 0.3738, - "hfopenllm_v2/MMLU-PRO": 0.4035 - } - }, - { - "id": "tensopolis/phi-4-tensopolis-v1", - "name": "phi-4-tensopolis-v1", - "developer": "tensopolis", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6767, - "hfopenllm_v2/BBH": 0.6872, - "hfopenllm_v2/MATH Level 5": 0.494, - "hfopenllm_v2/GPQA": 0.3347, - "hfopenllm_v2/MUSR": 0.4141, - "hfopenllm_v2/MMLU-PRO": 0.5384 - } - }, - { - "id": "tensopolis/qwen2.5-14b-tensopolis-v1", - "name": "qwen2.5-14b-tensopolis-v1", - "developer": "tensopolis", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.799, - "hfopenllm_v2/BBH": 0.6364, - "hfopenllm_v2/MATH Level 5": 0.5295, - "hfopenllm_v2/GPQA": 0.3347, - "hfopenllm_v2/MUSR": 0.4193, - "hfopenllm_v2/MMLU-PRO": 0.4911 - } - }, - { - "id": "tensopolis/qwen2.5-3b-or1-tensopolis", - "name": "qwen2.5-3b-or1-tensopolis", - "developer": "tensopolis", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.354, - "hfopenllm_v2/BBH": 0.4421, - "hfopenllm_v2/MATH Level 5": 0.173, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.3749, - "hfopenllm_v2/MMLU-PRO": 0.3197 - } - }, - { - "id": "tensopolis/qwen2.5-7b-tensopolis-v1", - "name": "qwen2.5-7b-tensopolis-v1", - "developer": "tensopolis", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7661, - "hfopenllm_v2/BBH": 0.5379, - "hfopenllm_v2/MATH Level 5": 0.4562, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.4339, - "hfopenllm_v2/MMLU-PRO": 0.4269 - } - }, - { - "id": "tensopolis/qwen2.5-7b-tensopolis-v2", - "name": "qwen2.5-7b-tensopolis-v2", - "developer": "tensopolis", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7521, - "hfopenllm_v2/BBH": 0.5415, - "hfopenllm_v2/MATH Level 5": 0.4819, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.4246, - "hfopenllm_v2/MMLU-PRO": 0.4243 - } - }, - { - "id": "tensopolis/virtuoso-lite-tensopolis-v1", - "name": "virtuoso-lite-tensopolis-v1", - "developer": "tensopolis", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8069, - "hfopenllm_v2/BBH": 0.6102, - "hfopenllm_v2/MATH Level 5": 0.2545, - "hfopenllm_v2/GPQA": 0.3448, - "hfopenllm_v2/MUSR": 0.4582, - "hfopenllm_v2/MMLU-PRO": 0.4435 - } - }, - { - "id": "tensopolis/virtuoso-lite-tensopolis-v2", - "name": "virtuoso-lite-tensopolis-v2", - "developer": "tensopolis", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8029, - "hfopenllm_v2/BBH": 0.61, - "hfopenllm_v2/MATH Level 5": 0.25, - "hfopenllm_v2/GPQA": 0.3431, - "hfopenllm_v2/MUSR": 0.4595, - "hfopenllm_v2/MMLU-PRO": 0.444 - } - }, - { - "id": "tensopolis/virtuoso-small-tensopolis-v1", - "name": "virtuoso-small-tensopolis-v1", - "developer": "tensopolis", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7856, - "hfopenllm_v2/BBH": 0.6415, - "hfopenllm_v2/MATH Level 5": 0.3527, - "hfopenllm_v2/GPQA": 0.328, - "hfopenllm_v2/MUSR": 0.4326, - "hfopenllm_v2/MMLU-PRO": 0.4968 - } - }, - { - "id": "tensopolis/virtuoso-small-tensopolis-v2", - "name": "virtuoso-small-tensopolis-v2", - "developer": "tensopolis", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.802, - "hfopenllm_v2/BBH": 0.6516, - "hfopenllm_v2/MATH Level 5": 0.3875, - "hfopenllm_v2/GPQA": 0.3289, - "hfopenllm_v2/MUSR": 0.4352, - "hfopenllm_v2/MMLU-PRO": 0.5154 - } - }, - { - "id": "tensopolis/virtuoso-small-v2-tensopolis-v1", - "name": "virtuoso-small-v2-tensopolis-v1", - "developer": "tensopolis", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8419, - "hfopenllm_v2/BBH": 0.6545, - "hfopenllm_v2/MATH Level 5": 0.4524, - "hfopenllm_v2/GPQA": 0.3465, - "hfopenllm_v2/MUSR": 0.4509, - "hfopenllm_v2/MMLU-PRO": 0.5175 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/tensoropera.json b/data/developers/tensoropera.json deleted file mode 100644 index 714352a4146412b4ece92148d7c9d727448baf8b..0000000000000000000000000000000000000000 --- a/data/developers/tensoropera.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "tensoropera", - "models": [ - { - "id": "tensoropera/Fox-1-1.6B", - "name": "Fox-1-1.6B", - "developer": "tensoropera", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2766, - "hfopenllm_v2/BBH": 0.3307, - "hfopenllm_v2/MATH Level 5": 0.0174, - "hfopenllm_v2/GPQA": 0.2634, - "hfopenllm_v2/MUSR": 0.355, - "hfopenllm_v2/MMLU-PRO": 0.1371 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/tenyx.json b/data/developers/tenyx.json deleted file mode 100644 index 2710fe40e7067b955166223da475f149bf037f62..0000000000000000000000000000000000000000 --- a/data/developers/tenyx.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "tenyx", - "models": [ - { - "id": "tenyx/Llama3-TenyxChat-70B", - "name": "Llama3-TenyxChat-70B", - "developer": "tenyx", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8087, - "hfopenllm_v2/BBH": 0.6511, - "hfopenllm_v2/MATH Level 5": 0.2356, - "hfopenllm_v2/GPQA": 0.3012, - "hfopenllm_v2/MUSR": 0.426, - "hfopenllm_v2/MMLU-PRO": 0.521 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/thedrummer.json b/data/developers/thedrummer.json deleted file mode 100644 index 113c04c24075576f2c3900dfda1978f8623653fc..0000000000000000000000000000000000000000 --- a/data/developers/thedrummer.json +++ /dev/null @@ -1,131 +0,0 @@ -{ - "developer": "TheDrummer", - "models": [ - { - "id": "TheDrummer/Cydonia-22B-v1.2", - "name": "Cydonia-22B-v1.2", - "developer": "TheDrummer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5635, - "hfopenllm_v2/BBH": 0.5809, - "hfopenllm_v2/MATH Level 5": 0.2032, - "hfopenllm_v2/GPQA": 0.3305, - "hfopenllm_v2/MUSR": 0.4022, - "hfopenllm_v2/MMLU-PRO": 0.4141 - } - }, - { - "id": "TheDrummer/Gemmasutra-9B-v1", - "name": "Gemmasutra-9B-v1", - "developer": "TheDrummer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2416, - "hfopenllm_v2/BBH": 0.5887, - "hfopenllm_v2/MATH Level 5": 0.0831, - "hfopenllm_v2/GPQA": 0.3104, - "hfopenllm_v2/MUSR": 0.4846, - "hfopenllm_v2/MMLU-PRO": 0.4045 - } - }, - { - "id": "TheDrummer/Gemmasutra-Mini-2B-v1", - "name": "Gemmasutra-Mini-2B-v1", - "developer": "TheDrummer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2549, - "hfopenllm_v2/BBH": 0.3575, - "hfopenllm_v2/MATH Level 5": 0.0378, - "hfopenllm_v2/GPQA": 0.271, - "hfopenllm_v2/MUSR": 0.349, - "hfopenllm_v2/MMLU-PRO": 0.2055 - } - }, - { - "id": "TheDrummer/Llama-3SOME-8B-v2", - "name": "Llama-3SOME-8B-v2", - "developer": "TheDrummer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4508, - "hfopenllm_v2/BBH": 0.5203, - "hfopenllm_v2/MATH Level 5": 0.0937, - "hfopenllm_v2/GPQA": 0.302, - "hfopenllm_v2/MUSR": 0.3833, - "hfopenllm_v2/MMLU-PRO": 0.3753 - } - }, - { - "id": "TheDrummer/Ministrations-8B-v1", - "name": "Ministrations-8B-v1", - "developer": "TheDrummer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2822, - "hfopenllm_v2/BBH": 0.4877, - "hfopenllm_v2/MATH Level 5": 0.1843, - "hfopenllm_v2/GPQA": 0.3247, - "hfopenllm_v2/MUSR": 0.4449, - "hfopenllm_v2/MMLU-PRO": 0.3644 - } - }, - { - "id": "TheDrummer/Rocinante-12B-v1", - "name": "Rocinante-12B-v1", - "developer": "TheDrummer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6076, - "hfopenllm_v2/BBH": 0.5065, - "hfopenllm_v2/MATH Level 5": 0.1269, - "hfopenllm_v2/GPQA": 0.2911, - "hfopenllm_v2/MUSR": 0.4017, - "hfopenllm_v2/MMLU-PRO": 0.3477 - } - }, - { - "id": "TheDrummer/Tiger-Gemma-9B-v1", - "name": "Tiger-Gemma-9B-v1", - "developer": "TheDrummer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7282, - "hfopenllm_v2/BBH": 0.5704, - "hfopenllm_v2/MATH Level 5": 0.1835, - "hfopenllm_v2/GPQA": 0.3389, - "hfopenllm_v2/MUSR": 0.4162, - "hfopenllm_v2/MMLU-PRO": 0.4118 - } - }, - { - "id": "TheDrummer/Tiger-Gemma-9B-v2", - "name": "Tiger-Gemma-9B-v2", - "developer": "TheDrummer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6986, - "hfopenllm_v2/BBH": 0.5617, - "hfopenllm_v2/MATH Level 5": 0.182, - "hfopenllm_v2/GPQA": 0.3398, - "hfopenllm_v2/MUSR": 0.4084, - "hfopenllm_v2/MMLU-PRO": 0.4112 - } - }, - { - "id": "TheDrummer/Tiger-Gemma-9B-v3", - "name": "Tiger-Gemma-9B-v3", - "developer": "TheDrummer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6821, - "hfopenllm_v2/BBH": 0.5812, - "hfopenllm_v2/MATH Level 5": 0.1624, - "hfopenllm_v2/GPQA": 0.3389, - "hfopenllm_v2/MUSR": 0.4004, - "hfopenllm_v2/MMLU-PRO": 0.4059 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/thedrunkensnail.json b/data/developers/thedrunkensnail.json deleted file mode 100644 index 30efc6e1511d9b33e12c49f5c4903c88e8b660fe..0000000000000000000000000000000000000000 --- a/data/developers/thedrunkensnail.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "developer": "TheDrunkenSnail", - "models": [ - { - "id": "TheDrunkenSnail/Daughter-of-Rhodia-12B", - "name": "Daughter-of-Rhodia-12B", - "developer": "TheDrunkenSnail", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6904, - "hfopenllm_v2/BBH": 0.5179, - "hfopenllm_v2/MATH Level 5": 0.1224, - "hfopenllm_v2/GPQA": 0.3171, - "hfopenllm_v2/MUSR": 0.4348, - "hfopenllm_v2/MMLU-PRO": 0.3641 - } - }, - { - "id": "TheDrunkenSnail/Mother-of-Rhodia-12B", - "name": "Mother-of-Rhodia-12B", - "developer": "TheDrunkenSnail", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6505, - "hfopenllm_v2/BBH": 0.4948, - "hfopenllm_v2/MATH Level 5": 0.1224, - "hfopenllm_v2/GPQA": 0.2987, - "hfopenllm_v2/MUSR": 0.4124, - "hfopenllm_v2/MMLU-PRO": 0.3551 - } - }, - { - "id": "TheDrunkenSnail/Son-of-Rhodia", - "name": "Son-of-Rhodia", - "developer": "TheDrunkenSnail", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7046, - "hfopenllm_v2/BBH": 0.5097, - "hfopenllm_v2/MATH Level 5": 0.1314, - "hfopenllm_v2/GPQA": 0.3129, - "hfopenllm_v2/MUSR": 0.4203, - "hfopenllm_v2/MMLU-PRO": 0.3608 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/thehierophant.json b/data/developers/thehierophant.json deleted file mode 100644 index c5942c1f64f18b40b781b7a930f223e096946f47..0000000000000000000000000000000000000000 --- a/data/developers/thehierophant.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "TheHierophant", - "models": [ - { - "id": "TheHierophant/Underground-Cognitive-V0.3-test", - "name": "Underground-Cognitive-V0.3-test", - "developer": "TheHierophant", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4808, - "hfopenllm_v2/BBH": 0.529, - "hfopenllm_v2/MATH Level 5": 0.0589, - "hfopenllm_v2/GPQA": 0.2987, - "hfopenllm_v2/MUSR": 0.4351, - "hfopenllm_v2/MMLU-PRO": 0.3318 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/theo77186.json b/data/developers/theo77186.json deleted file mode 100644 index 1808ec42373f0178babb160f553f004c1f2357cd..0000000000000000000000000000000000000000 --- a/data/developers/theo77186.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "theo77186", - "models": [ - { - "id": "theo77186/Qwen2.5-Coder-7B-Instruct-20241106", - "name": "Qwen2.5-Coder-7B-Instruct-20241106", - "developer": "theo77186", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6101, - "hfopenllm_v2/BBH": 0.5008, - "hfopenllm_v2/MATH Level 5": 0.3882, - "hfopenllm_v2/GPQA": 0.2919, - "hfopenllm_v2/MUSR": 0.4073, - "hfopenllm_v2/MMLU-PRO": 0.3353 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/theprint.json b/data/developers/theprint.json deleted file mode 100644 index 317c1e9ace20eaa4ae354ffa2c6a7785705bc3ff..0000000000000000000000000000000000000000 --- a/data/developers/theprint.json +++ /dev/null @@ -1,257 +0,0 @@ -{ - "developer": "theprint", - "models": [ - { - "id": "theprint/Boptruth-Agatha-7B", - "name": "Boptruth-Agatha-7B", - "developer": "theprint", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3124, - "hfopenllm_v2/BBH": 0.4984, - "hfopenllm_v2/MATH Level 5": 0.0551, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.4277, - "hfopenllm_v2/MMLU-PRO": 0.2861 - } - }, - { - "id": "theprint/CleverBoi-7B-v2", - "name": "CleverBoi-7B-v2", - "developer": "theprint", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.217, - "hfopenllm_v2/BBH": 0.4532, - "hfopenllm_v2/MATH Level 5": 0.0264, - "hfopenllm_v2/GPQA": 0.2886, - "hfopenllm_v2/MUSR": 0.4695, - "hfopenllm_v2/MMLU-PRO": 0.2709 - } - }, - { - "id": "theprint/CleverBoi-7B-v3", - "name": "CleverBoi-7B-v3", - "developer": "theprint", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2382, - "hfopenllm_v2/BBH": 0.4414, - "hfopenllm_v2/MATH Level 5": 0.04, - "hfopenllm_v2/GPQA": 0.2659, - "hfopenllm_v2/MUSR": 0.4072, - "hfopenllm_v2/MMLU-PRO": 0.2868 - } - }, - { - "id": "theprint/CleverBoi-Llama-3.1-8B-Instruct", - "name": "CleverBoi-Llama-3.1-8B-Instruct", - "developer": "theprint", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1682, - "hfopenllm_v2/BBH": 0.456, - "hfopenllm_v2/MATH Level 5": 0.0491, - "hfopenllm_v2/GPQA": 0.3003, - "hfopenllm_v2/MUSR": 0.4014, - "hfopenllm_v2/MMLU-PRO": 0.3075 - } - }, - { - "id": "theprint/CleverBoi-Llama-3.1-8B-v2", - "name": "CleverBoi-Llama-3.1-8B-v2", - "developer": "theprint", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1961, - "hfopenllm_v2/BBH": 0.4668, - "hfopenllm_v2/MATH Level 5": 0.0529, - "hfopenllm_v2/GPQA": 0.2861, - "hfopenllm_v2/MUSR": 0.3735, - "hfopenllm_v2/MMLU-PRO": 0.3188 - } - }, - { - "id": "theprint/CleverBoi-Nemo-12B-v2", - "name": "CleverBoi-Nemo-12B-v2", - "developer": "theprint", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2046, - "hfopenllm_v2/BBH": 0.5241, - "hfopenllm_v2/MATH Level 5": 0.1035, - "hfopenllm_v2/GPQA": 0.3138, - "hfopenllm_v2/MUSR": 0.4187, - "hfopenllm_v2/MMLU-PRO": 0.3228 - } - }, - { - "id": "theprint/Code-Llama-Bagel-8B", - "name": "Code-Llama-Bagel-8B", - "developer": "theprint", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.253, - "hfopenllm_v2/BBH": 0.4697, - "hfopenllm_v2/MATH Level 5": 0.0612, - "hfopenllm_v2/GPQA": 0.276, - "hfopenllm_v2/MUSR": 0.368, - "hfopenllm_v2/MMLU-PRO": 0.2822 - } - }, - { - "id": "theprint/Conversely-Mistral-7B", - "name": "Conversely-Mistral-7B", - "developer": "theprint", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2608, - "hfopenllm_v2/BBH": 0.4672, - "hfopenllm_v2/MATH Level 5": 0.0279, - "hfopenllm_v2/GPQA": 0.2852, - "hfopenllm_v2/MUSR": 0.4189, - "hfopenllm_v2/MMLU-PRO": 0.2826 - } - }, - { - "id": "theprint/Llama-3.2-3B-VanRossum", - "name": "Llama-3.2-3B-VanRossum", - "developer": "theprint", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4783, - "hfopenllm_v2/BBH": 0.4279, - "hfopenllm_v2/MATH Level 5": 0.0974, - "hfopenllm_v2/GPQA": 0.2676, - "hfopenllm_v2/MUSR": 0.3442, - "hfopenllm_v2/MMLU-PRO": 0.277 - } - }, - { - "id": "theprint/phi-3-mini-4k-python", - "name": "phi-3-mini-4k-python", - "developer": "theprint", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2409, - "hfopenllm_v2/BBH": 0.4938, - "hfopenllm_v2/MATH Level 5": 0.105, - "hfopenllm_v2/GPQA": 0.2911, - "hfopenllm_v2/MUSR": 0.3922, - "hfopenllm_v2/MMLU-PRO": 0.3577 - } - }, - { - "id": "theprint/ReWiz-7B", - "name": "ReWiz-7B", - "developer": "theprint", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4048, - "hfopenllm_v2/BBH": 0.4564, - "hfopenllm_v2/MATH Level 5": 0.0408, - "hfopenllm_v2/GPQA": 0.2752, - "hfopenllm_v2/MUSR": 0.4612, - "hfopenllm_v2/MMLU-PRO": 0.267 - } - }, - { - "id": "theprint/ReWiz-Llama-3.1-8B-v2", - "name": "ReWiz-Llama-3.1-8B-v2", - "developer": "theprint", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2379, - "hfopenllm_v2/BBH": 0.4632, - "hfopenllm_v2/MATH Level 5": 0.0574, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.3814, - "hfopenllm_v2/MMLU-PRO": 0.331 - } - }, - { - "id": "theprint/ReWiz-Llama-3.2-3B", - "name": "ReWiz-Llama-3.2-3B", - "developer": "theprint", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4649, - "hfopenllm_v2/BBH": 0.4343, - "hfopenllm_v2/MATH Level 5": 0.1095, - "hfopenllm_v2/GPQA": 0.2836, - "hfopenllm_v2/MUSR": 0.3614, - "hfopenllm_v2/MMLU-PRO": 0.2887 - } - }, - { - "id": "theprint/ReWiz-Nemo-12B-Instruct", - "name": "ReWiz-Nemo-12B-Instruct", - "developer": "theprint", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1062, - "hfopenllm_v2/BBH": 0.5092, - "hfopenllm_v2/MATH Level 5": 0.1042, - "hfopenllm_v2/GPQA": 0.3238, - "hfopenllm_v2/MUSR": 0.4096, - "hfopenllm_v2/MMLU-PRO": 0.3339 - } - }, - { - "id": "theprint/ReWiz-Qwen-2.5-14B", - "name": "ReWiz-Qwen-2.5-14B", - "developer": "theprint", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2785, - "hfopenllm_v2/BBH": 0.6179, - "hfopenllm_v2/MATH Level 5": 0.2923, - "hfopenllm_v2/GPQA": 0.38, - "hfopenllm_v2/MUSR": 0.4539, - "hfopenllm_v2/MMLU-PRO": 0.5092 - } - }, - { - "id": "theprint/ReWiz-Worldbuilder-7B", - "name": "ReWiz-Worldbuilder-7B", - "developer": "theprint", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.251, - "hfopenllm_v2/BBH": 0.4636, - "hfopenllm_v2/MATH Level 5": 0.037, - "hfopenllm_v2/GPQA": 0.2693, - "hfopenllm_v2/MUSR": 0.4572, - "hfopenllm_v2/MMLU-PRO": 0.2971 - } - }, - { - "id": "theprint/RuDolph-Hermes-7B", - "name": "RuDolph-Hermes-7B", - "developer": "theprint", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3604, - "hfopenllm_v2/BBH": 0.5053, - "hfopenllm_v2/MATH Level 5": 0.0514, - "hfopenllm_v2/GPQA": 0.3121, - "hfopenllm_v2/MUSR": 0.4226, - "hfopenllm_v2/MMLU-PRO": 0.3073 - } - }, - { - "id": "theprint/WorldBuilder-12B", - "name": "WorldBuilder-12B", - "developer": "theprint", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1374, - "hfopenllm_v2/BBH": 0.501, - "hfopenllm_v2/MATH Level 5": 0.0446, - "hfopenllm_v2/GPQA": 0.297, - "hfopenllm_v2/MUSR": 0.4066, - "hfopenllm_v2/MMLU-PRO": 0.3192 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/thetsar1209.json b/data/developers/thetsar1209.json deleted file mode 100644 index 3287d82fbdad8add23c36e578b883cdb742fbacb..0000000000000000000000000000000000000000 --- a/data/developers/thetsar1209.json +++ /dev/null @@ -1,103 +0,0 @@ -{ - "developer": "TheTsar1209", - "models": [ - { - "id": "TheTsar1209/nemo-carpmuscle-v0.1", - "name": "nemo-carpmuscle-v0.1", - "developer": "TheTsar1209", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2276, - "hfopenllm_v2/BBH": 0.5084, - "hfopenllm_v2/MATH Level 5": 0.0476, - "hfopenllm_v2/GPQA": 0.297, - "hfopenllm_v2/MUSR": 0.4135, - "hfopenllm_v2/MMLU-PRO": 0.3406 - } - }, - { - "id": "TheTsar1209/qwen-carpmuscle-r-v0.3", - "name": "qwen-carpmuscle-r-v0.3", - "developer": "TheTsar1209", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4455, - "hfopenllm_v2/BBH": 0.6227, - "hfopenllm_v2/MATH Level 5": 0.3006, - "hfopenllm_v2/GPQA": 0.3507, - "hfopenllm_v2/MUSR": 0.4278, - "hfopenllm_v2/MMLU-PRO": 0.5103 - } - }, - { - "id": "TheTsar1209/qwen-carpmuscle-v0.1", - "name": "qwen-carpmuscle-v0.1", - "developer": "TheTsar1209", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5622, - "hfopenllm_v2/BBH": 0.6434, - "hfopenllm_v2/MATH Level 5": 0.2628, - "hfopenllm_v2/GPQA": 0.344, - "hfopenllm_v2/MUSR": 0.4161, - "hfopenllm_v2/MMLU-PRO": 0.52 - } - }, - { - "id": "TheTsar1209/qwen-carpmuscle-v0.2", - "name": "qwen-carpmuscle-v0.2", - "developer": "TheTsar1209", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5257, - "hfopenllm_v2/BBH": 0.6387, - "hfopenllm_v2/MATH Level 5": 0.2832, - "hfopenllm_v2/GPQA": 0.3557, - "hfopenllm_v2/MUSR": 0.4346, - "hfopenllm_v2/MMLU-PRO": 0.5147 - } - }, - { - "id": "TheTsar1209/qwen-carpmuscle-v0.3", - "name": "qwen-carpmuscle-v0.3", - "developer": "TheTsar1209", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4476, - "hfopenllm_v2/BBH": 0.6152, - "hfopenllm_v2/MATH Level 5": 0.3134, - "hfopenllm_v2/GPQA": 0.3565, - "hfopenllm_v2/MUSR": 0.4132, - "hfopenllm_v2/MMLU-PRO": 0.5062 - } - }, - { - "id": "TheTsar1209/qwen-carpmuscle-v0.4", - "name": "qwen-carpmuscle-v0.4", - "developer": "TheTsar1209", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7202, - "hfopenllm_v2/BBH": 0.6454, - "hfopenllm_v2/MATH Level 5": 0.2772, - "hfopenllm_v2/GPQA": 0.3523, - "hfopenllm_v2/MUSR": 0.4516, - "hfopenllm_v2/MMLU-PRO": 0.5144 - } - }, - { - "id": "TheTsar1209/qwen-carpmuscle-v0.4.1", - "name": "qwen-carpmuscle-v0.4.1", - "developer": "TheTsar1209", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.736, - "hfopenllm_v2/BBH": 0.6507, - "hfopenllm_v2/MATH Level 5": 0.2779, - "hfopenllm_v2/GPQA": 0.3456, - "hfopenllm_v2/MUSR": 0.4489, - "hfopenllm_v2/MMLU-PRO": 0.5191 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/thinkcoder.json b/data/developers/thinkcoder.json deleted file mode 100644 index 758defeed9e7e5eef830591f8aa3bbc16e3d953e..0000000000000000000000000000000000000000 --- a/data/developers/thinkcoder.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "thinkcoder", - "models": [ - { - "id": "thinkcoder/llama3-8b-instruct-lora-8-sft", - "name": "llama3-8b-instruct-lora-8-sft", - "developer": "thinkcoder", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.648, - "hfopenllm_v2/BBH": 0.4865, - "hfopenllm_v2/MATH Level 5": 0.102, - "hfopenllm_v2/GPQA": 0.2668, - "hfopenllm_v2/MUSR": 0.3235, - "hfopenllm_v2/MMLU-PRO": 0.3476 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/thirdeyeai.json b/data/developers/thirdeyeai.json deleted file mode 100644 index eadc873895e11ba0dccbb2fae8600f3324eb346e..0000000000000000000000000000000000000000 --- a/data/developers/thirdeyeai.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "thirdeyeai", - "models": [ - { - "id": "thirdeyeai/elevate360m", - "name": "elevate360m", - "developer": "thirdeyeai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0445, - "hfopenllm_v2/BBH": 0.2963, - "hfopenllm_v2/MATH Level 5": 0.0159, - "hfopenllm_v2/GPQA": 0.2408, - "hfopenllm_v2/MUSR": 0.3462, - "hfopenllm_v2/MMLU-PRO": 0.1077 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/thomas-yanxin.json b/data/developers/thomas-yanxin.json deleted file mode 100644 index 73a7fd9f779612c0dc692aaccb0edd0e53f86ba9..0000000000000000000000000000000000000000 --- a/data/developers/thomas-yanxin.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "developer": "thomas-yanxin", - "models": [ - { - "id": "thomas-yanxin/XinYuan-Qwen2-1_5B", - "name": "XinYuan-Qwen2-1_5B", - "developer": "thomas-yanxin", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2986, - "hfopenllm_v2/BBH": 0.3635, - "hfopenllm_v2/MATH Level 5": 0.0672, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.3634, - "hfopenllm_v2/MMLU-PRO": 0.2357 - } - }, - { - "id": "thomas-yanxin/XinYuan-Qwen2-7B", - "name": "XinYuan-Qwen2-7B", - "developer": "thomas-yanxin", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4438, - "hfopenllm_v2/BBH": 0.4937, - "hfopenllm_v2/MATH Level 5": 0.1458, - "hfopenllm_v2/GPQA": 0.2911, - "hfopenllm_v2/MUSR": 0.4058, - "hfopenllm_v2/MMLU-PRO": 0.3925 - } - }, - { - "id": "thomas-yanxin/XinYuan-Qwen2-7B-0917", - "name": "XinYuan-Qwen2-7B-0917", - "developer": "thomas-yanxin", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3719, - "hfopenllm_v2/BBH": 0.5169, - "hfopenllm_v2/MATH Level 5": 0.1979, - "hfopenllm_v2/GPQA": 0.3096, - "hfopenllm_v2/MUSR": 0.4401, - "hfopenllm_v2/MMLU-PRO": 0.4245 - } - }, - { - "id": "thomas-yanxin/XinYuan-Qwen2.5-7B-0917", - "name": "XinYuan-Qwen2.5-7B-0917", - "developer": "thomas-yanxin", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3577, - "hfopenllm_v2/BBH": 0.5184, - "hfopenllm_v2/MATH Level 5": 0.1934, - "hfopenllm_v2/GPQA": 0.281, - "hfopenllm_v2/MUSR": 0.3676, - "hfopenllm_v2/MMLU-PRO": 0.3882 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/thudm.json b/data/developers/thudm.json deleted file mode 100644 index 03f33b182a312a7c7bdb9d8d0ede927124c9fcd0..0000000000000000000000000000000000000000 --- a/data/developers/thudm.json +++ /dev/null @@ -1,75 +0,0 @@ -{ - "developer": "THUDM", - "models": [ - { - "id": "THUDM/glm-4-9b", - "name": "glm-4-9b", - "developer": "THUDM", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1426, - "hfopenllm_v2/BBH": 0.5528, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.3163, - "hfopenllm_v2/MUSR": 0.4386, - "hfopenllm_v2/MMLU-PRO": 0.4145 - } - }, - { - "id": "THUDM/glm-4-9b-chat", - "name": "glm-4-9b-chat", - "developer": "THUDM", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0, - "hfopenllm_v2/BBH": 0.4736, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.3138, - "hfopenllm_v2/MUSR": 0.3994, - "hfopenllm_v2/MMLU-PRO": 0.3167 - } - }, - { - "id": "THUDM/glm-4-9b-chat-1m", - "name": "glm-4-9b-chat-1m", - "developer": "THUDM", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0, - "hfopenllm_v2/BBH": 0.418, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.3037, - "hfopenllm_v2/MUSR": 0.3795, - "hfopenllm_v2/MMLU-PRO": 0.3163 - } - }, - { - "id": "THUDM/glm-4-9b-chat-1m-hf", - "name": "glm-4-9b-chat-1m-hf", - "developer": "THUDM", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5341, - "hfopenllm_v2/BBH": 0.3901, - "hfopenllm_v2/MATH Level 5": 0.0483, - "hfopenllm_v2/GPQA": 0.2919, - "hfopenllm_v2/MUSR": 0.3689, - "hfopenllm_v2/MMLU-PRO": 0.1814 - } - }, - { - "id": "THUDM/glm-4-9b-chat-hf", - "name": "glm-4-9b-chat-hf", - "developer": "THUDM", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6513, - "hfopenllm_v2/BBH": 0.4432, - "hfopenllm_v2/MATH Level 5": 0.0846, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.3593, - "hfopenllm_v2/MMLU-PRO": 0.2774 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/tianyil1.json b/data/developers/tianyil1.json deleted file mode 100644 index e4d404fedb4aebf09d86018218a9fc7fb5280009..0000000000000000000000000000000000000000 --- a/data/developers/tianyil1.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "tianyil1", - "models": [ - { - "id": "tianyil1/MistralForCausalLM_Cal_DPO", - "name": "MistralForCausalLM_Cal_DPO", - "developer": "tianyil1", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5328, - "hfopenllm_v2/BBH": 0.4381, - "hfopenllm_v2/MATH Level 5": 0.0287, - "hfopenllm_v2/GPQA": 0.276, - "hfopenllm_v2/MUSR": 0.3977, - "hfopenllm_v2/MMLU-PRO": 0.2763 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/tiger-lab.json b/data/developers/tiger-lab.json deleted file mode 100644 index d7fb41d995ca9e3ab4e0908fbc299ebfa8f667e0..0000000000000000000000000000000000000000 --- a/data/developers/tiger-lab.json +++ /dev/null @@ -1,89 +0,0 @@ -{ - "developer": "TIGER-Lab", - "models": [ - { - "id": "TIGER-Lab/AceCoder-Qwen2.5-7B-Ins-Rule", - "name": "AceCoder-Qwen2.5-7B-Ins-Rule", - "developer": "TIGER-Lab", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7424, - "hfopenllm_v2/BBH": 0.5404, - "hfopenllm_v2/MATH Level 5": 0.4992, - "hfopenllm_v2/GPQA": 0.3012, - "hfopenllm_v2/MUSR": 0.398, - "hfopenllm_v2/MMLU-PRO": 0.4322 - } - }, - { - "id": "TIGER-Lab/AceCoder-Qwen2.5-Coder-7B-Base-Rule", - "name": "AceCoder-Qwen2.5-Coder-7B-Base-Rule", - "developer": "TIGER-Lab", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4408, - "hfopenllm_v2/BBH": 0.4902, - "hfopenllm_v2/MATH Level 5": 0.2017, - "hfopenllm_v2/GPQA": 0.2718, - "hfopenllm_v2/MUSR": 0.3449, - "hfopenllm_v2/MMLU-PRO": 0.3745 - } - }, - { - "id": "TIGER-Lab/AceCoder-Qwen2.5-Coder-7B-Ins-Rule", - "name": "AceCoder-Qwen2.5-Coder-7B-Ins-Rule", - "developer": "TIGER-Lab", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6222, - "hfopenllm_v2/BBH": 0.5089, - "hfopenllm_v2/MATH Level 5": 0.3603, - "hfopenllm_v2/GPQA": 0.2777, - "hfopenllm_v2/MUSR": 0.4046, - "hfopenllm_v2/MMLU-PRO": 0.3428 - } - }, - { - "id": "TIGER-Lab/AceCodeRM-7B", - "name": "AceCodeRM-7B", - "developer": "TIGER-Lab", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5855, - "hfopenllm_v2/BBH": 0.4773, - "hfopenllm_v2/MATH Level 5": 0.3467, - "hfopenllm_v2/GPQA": 0.3045, - "hfopenllm_v2/MUSR": 0.4192, - "hfopenllm_v2/MMLU-PRO": 0.3361 - } - }, - { - "id": "TIGER-Lab/MAmmoTH2-7B-Plus", - "name": "MAmmoTH2-7B-Plus", - "developer": "TIGER-Lab", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5575, - "hfopenllm_v2/BBH": 0.4235, - "hfopenllm_v2/MATH Level 5": 0.1858, - "hfopenllm_v2/GPQA": 0.2802, - "hfopenllm_v2/MUSR": 0.4124, - "hfopenllm_v2/MMLU-PRO": 0.3017 - } - }, - { - "id": "TIGER-Lab/Qwen2.5-Math-7B-CFT", - "name": "Qwen2.5-Math-7B-CFT", - "developer": "TIGER-Lab", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2777, - "hfopenllm_v2/BBH": 0.4637, - "hfopenllm_v2/MATH Level 5": 0.5574, - "hfopenllm_v2/GPQA": 0.2861, - "hfopenllm_v2/MUSR": 0.3887, - "hfopenllm_v2/MMLU-PRO": 0.2945 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/tii-uae.json b/data/developers/tii-uae.json deleted file mode 100644 index 59fa6fec7eedf159bb54179677bddb06087f5693..0000000000000000000000000000000000000000 --- a/data/developers/tii-uae.json +++ /dev/null @@ -1,157 +0,0 @@ -{ - "developer": "tii-uae", - "models": [ - { - "id": "tii-uae/falcon3-10b-instruct-fc", - "name": "Falcon3-10B-Instruct (FC)", - "developer": "tii-uae", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 82.0, - "bfcl/bfcl.overall.overall_accuracy": 27.01, - "bfcl/bfcl.overall.total_cost_usd": 52.59, - "bfcl/bfcl.overall.latency_mean_s": 69.27, - "bfcl/bfcl.overall.latency_std_s": 92.22, - "bfcl/bfcl.overall.latency_p95_s": 190.96, - "bfcl/bfcl.non_live.ast_accuracy": 85.0, - "bfcl/bfcl.non_live.simple_ast_accuracy": 70.5, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 93.5, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 88.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 87.5, - "bfcl/bfcl.live.live_accuracy": 75.43, - "bfcl/bfcl.live.live_simple_ast_accuracy": 77.13, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 76.16, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 50.0, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 41.67, - "bfcl/bfcl.multi_turn.accuracy": 6.5, - "bfcl/bfcl.multi_turn.base_accuracy": 6.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 9.5, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 5.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 5.0, - "bfcl/bfcl.web_search.accuracy": 1.5, - "bfcl/bfcl.web_search.base_accuracy": 2.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 1.0, - "bfcl/bfcl.memory.accuracy": 27.53, - "bfcl/bfcl.memory.kv_accuracy": 12.26, - "bfcl/bfcl.memory.vector_accuracy": 19.35, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 50.97, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 93.75, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 32.09 - } - }, - { - "id": "tii-uae/falcon3-1b-instruct-fc", - "name": "Falcon3-1B-Instruct (FC)", - "developer": "tii-uae", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 106.0, - "bfcl/bfcl.overall.overall_accuracy": 11.08, - "bfcl/bfcl.overall.total_cost_usd": 1.72, - "bfcl/bfcl.overall.latency_mean_s": 5.23, - "bfcl/bfcl.overall.latency_std_s": 14.34, - "bfcl/bfcl.overall.latency_p95_s": 11.48, - "bfcl/bfcl.non_live.ast_accuracy": 9.02, - "bfcl/bfcl.non_live.simple_ast_accuracy": 2.58, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 6.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 18.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 9.5, - "bfcl/bfcl.live.live_accuracy": 2.89, - "bfcl/bfcl.live.live_simple_ast_accuracy": 4.26, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 2.37, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 0.0, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 12.5, - "bfcl/bfcl.multi_turn.accuracy": 0.0, - "bfcl/bfcl.multi_turn.base_accuracy": 0.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 0.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 0.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 0.0, - "bfcl/bfcl.web_search.accuracy": 0.0, - "bfcl/bfcl.web_search.base_accuracy": 0.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 0.0, - "bfcl/bfcl.memory.accuracy": 5.81, - "bfcl/bfcl.memory.kv_accuracy": 5.16, - "bfcl/bfcl.memory.vector_accuracy": 7.74, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 4.52, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 0.0, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 87.3 - } - }, - { - "id": "tii-uae/falcon3-3b-instruct-fc", - "name": "Falcon3-3B-Instruct (FC)", - "developer": "tii-uae", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 104.0, - "bfcl/bfcl.overall.overall_accuracy": 16.25, - "bfcl/bfcl.overall.total_cost_usd": 36.7, - "bfcl/bfcl.overall.latency_mean_s": 38.52, - "bfcl/bfcl.overall.latency_std_s": 107.47, - "bfcl/bfcl.overall.latency_p95_s": 103.62, - "bfcl/bfcl.non_live.ast_accuracy": 54.62, - "bfcl/bfcl.non_live.simple_ast_accuracy": 56.5, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 69.5, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 67.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 25.5, - "bfcl/bfcl.live.live_accuracy": 54.48, - "bfcl/bfcl.live.live_simple_ast_accuracy": 57.36, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 54.7, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 25.0, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 33.33, - "bfcl/bfcl.multi_turn.accuracy": 1.0, - "bfcl/bfcl.multi_turn.base_accuracy": 1.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 0.5, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 0.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 1.5, - "bfcl/bfcl.web_search.accuracy": 1.0, - "bfcl/bfcl.web_search.base_accuracy": 1.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 1.0, - "bfcl/bfcl.memory.accuracy": 7.74, - "bfcl/bfcl.memory.kv_accuracy": 6.45, - "bfcl/bfcl.memory.vector_accuracy": 8.39, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 8.39, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 81.25, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 32.92 - } - }, - { - "id": "tii-uae/falcon3-7b-instruct-fc", - "name": "Falcon3-7B-Instruct (FC)", - "developer": "tii-uae", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 91.0, - "bfcl/bfcl.overall.overall_accuracy": 24.03, - "bfcl/bfcl.overall.total_cost_usd": 73.61, - "bfcl/bfcl.overall.latency_mean_s": 93.11, - "bfcl/bfcl.overall.latency_std_s": 117.8, - "bfcl/bfcl.overall.latency_p95_s": 315.7, - "bfcl/bfcl.non_live.ast_accuracy": 82.69, - "bfcl/bfcl.non_live.simple_ast_accuracy": 65.75, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 89.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 87.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 89.0, - "bfcl/bfcl.live.live_accuracy": 68.32, - "bfcl/bfcl.live.live_simple_ast_accuracy": 74.81, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 66.76, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 75.0, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 62.5, - "bfcl/bfcl.multi_turn.accuracy": 5.0, - "bfcl/bfcl.multi_turn.base_accuracy": 7.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 4.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 5.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 4.0, - "bfcl/bfcl.web_search.accuracy": 0.5, - "bfcl/bfcl.web_search.base_accuracy": 1.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 0.0, - "bfcl/bfcl.memory.accuracy": 20.65, - "bfcl/bfcl.memory.kv_accuracy": 10.32, - "bfcl/bfcl.memory.vector_accuracy": 12.9, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 38.71, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 100.0, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 31.99 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/tiiuae.json b/data/developers/tiiuae.json deleted file mode 100644 index 0134425f8dd645edf05a0f5359175c618b059029..0000000000000000000000000000000000000000 --- a/data/developers/tiiuae.json +++ /dev/null @@ -1,325 +0,0 @@ -{ - "developer": "tiiuae", - "models": [ - { - "id": "tiiuae/falcon-11B", - "name": "falcon-11B", - "developer": "tiiuae", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3261, - "hfopenllm_v2/BBH": 0.4392, - "hfopenllm_v2/MATH Level 5": 0.0279, - "hfopenllm_v2/GPQA": 0.271, - "hfopenllm_v2/MUSR": 0.3986, - "hfopenllm_v2/MMLU-PRO": 0.2389 - } - }, - { - "id": "tiiuae/falcon-40b", - "name": "Falcon 40B", - "developer": "tiiuae", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.729, - "helm_classic/MMLU": 0.509, - "helm_classic/BoolQ": 0.819, - "helm_classic/NarrativeQA": 0.673, - "helm_classic/NaturalQuestions (open-book)": 0.675, - "helm_classic/QuAC": 0.307, - "helm_classic/HellaSwag": -1.0, - "helm_classic/OpenbookQA": -1.0, - "helm_classic/TruthfulQA": 0.353, - "helm_classic/MS MARCO (TREC)": -1.0, - "helm_classic/CNN/DailyMail": -1.0, - "helm_classic/XSUM": -1.0, - "helm_classic/IMDB": 0.959, - "helm_classic/CivilComments": 0.552, - "helm_classic/RAFT": 0.661, - "helm_lite/Mean win rate": 0.217, - "helm_lite/NarrativeQA": 0.671, - "helm_lite/NaturalQuestions (closed-book)": 0.392, - "helm_lite/OpenbookQA": 0.662, - "helm_lite/MMLU": 0.507, - "helm_lite/MATH": 0.128, - "helm_lite/GSM8K": 0.267, - "helm_lite/LegalBench": 0.442, - "helm_lite/MedQA": 0.419, - "helm_lite/WMT 2014": 0.162, - "hfopenllm_v2/IFEval": 0.2496, - "hfopenllm_v2/BBH": 0.4019, - "hfopenllm_v2/MATH Level 5": 0.0181, - "hfopenllm_v2/GPQA": 0.2735, - "hfopenllm_v2/MUSR": 0.3631, - "hfopenllm_v2/MMLU-PRO": 0.2505 - } - }, - { - "id": "tiiuae/falcon-40b-instruct", - "name": "falcon-40b-instruct", - "developer": "tiiuae", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2454, - "hfopenllm_v2/BBH": 0.4054, - "hfopenllm_v2/MATH Level 5": 0.0196, - "hfopenllm_v2/GPQA": 0.25, - "hfopenllm_v2/MUSR": 0.3762, - "hfopenllm_v2/MMLU-PRO": 0.2261 - } - }, - { - "id": "tiiuae/falcon-7b", - "name": "Falcon 7B", - "developer": "tiiuae", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.378, - "helm_classic/MMLU": 0.286, - "helm_classic/BoolQ": 0.753, - "helm_classic/NarrativeQA": 0.621, - "helm_classic/NaturalQuestions (open-book)": 0.579, - "helm_classic/QuAC": 0.332, - "helm_classic/HellaSwag": -1.0, - "helm_classic/OpenbookQA": -1.0, - "helm_classic/TruthfulQA": 0.234, - "helm_classic/MS MARCO (TREC)": -1.0, - "helm_classic/CNN/DailyMail": -1.0, - "helm_classic/XSUM": -1.0, - "helm_classic/IMDB": 0.836, - "helm_classic/CivilComments": 0.514, - "helm_classic/RAFT": 0.602, - "helm_lite/Mean win rate": 0.064, - "helm_lite/NarrativeQA": 0.621, - "helm_lite/NaturalQuestions (closed-book)": 0.285, - "helm_lite/OpenbookQA": 0.26, - "helm_lite/MMLU": 0.288, - "helm_lite/MATH": 0.044, - "helm_lite/GSM8K": 0.055, - "helm_lite/LegalBench": 0.346, - "helm_lite/MedQA": 0.254, - "helm_lite/WMT 2014": 0.094, - "hfopenllm_v2/IFEval": 0.1821, - "hfopenllm_v2/BBH": 0.3285, - "hfopenllm_v2/MATH Level 5": 0.0098, - "hfopenllm_v2/GPQA": 0.245, - "hfopenllm_v2/MUSR": 0.3778, - "hfopenllm_v2/MMLU-PRO": 0.1125 - } - }, - { - "id": "tiiuae/falcon-7b-instruct", - "name": "falcon-7b-instruct", - "developer": "tiiuae", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1969, - "hfopenllm_v2/BBH": 0.3203, - "hfopenllm_v2/MATH Level 5": 0.0121, - "hfopenllm_v2/GPQA": 0.2475, - "hfopenllm_v2/MUSR": 0.3634, - "hfopenllm_v2/MMLU-PRO": 0.1155 - } - }, - { - "id": "tiiuae/Falcon-Instruct-40B", - "name": "Falcon-Instruct 40B", - "developer": "tiiuae", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.727, - "helm_classic/MMLU": 0.497, - "helm_classic/BoolQ": 0.829, - "helm_classic/NarrativeQA": 0.625, - "helm_classic/NaturalQuestions (open-book)": 0.666, - "helm_classic/QuAC": 0.371, - "helm_classic/HellaSwag": -1.0, - "helm_classic/OpenbookQA": -1.0, - "helm_classic/TruthfulQA": 0.384, - "helm_classic/MS MARCO (TREC)": -1.0, - "helm_classic/CNN/DailyMail": -1.0, - "helm_classic/XSUM": -1.0, - "helm_classic/IMDB": 0.959, - "helm_classic/CivilComments": 0.603, - "helm_classic/RAFT": 0.586 - } - }, - { - "id": "tiiuae/Falcon-Instruct-7B", - "name": "Falcon-Instruct 7B", - "developer": "tiiuae", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.244, - "helm_classic/MMLU": 0.275, - "helm_classic/BoolQ": 0.72, - "helm_classic/NarrativeQA": 0.476, - "helm_classic/NaturalQuestions (open-book)": 0.449, - "helm_classic/QuAC": 0.311, - "helm_classic/HellaSwag": -1.0, - "helm_classic/OpenbookQA": -1.0, - "helm_classic/TruthfulQA": 0.213, - "helm_classic/MS MARCO (TREC)": -1.0, - "helm_classic/CNN/DailyMail": -1.0, - "helm_classic/XSUM": -1.0, - "helm_classic/IMDB": 0.852, - "helm_classic/CivilComments": 0.511, - "helm_classic/RAFT": 0.523 - } - }, - { - "id": "tiiuae/falcon-mamba-7b", - "name": "falcon-mamba-7b", - "developer": "tiiuae", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3336, - "hfopenllm_v2/BBH": 0.4285, - "hfopenllm_v2/MATH Level 5": 0.0446, - "hfopenllm_v2/GPQA": 0.3104, - "hfopenllm_v2/MUSR": 0.421, - "hfopenllm_v2/MMLU-PRO": 0.2302 - } - }, - { - "id": "tiiuae/Falcon3-10B-Base", - "name": "Falcon3-10B-Base", - "developer": "tiiuae", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3648, - "hfopenllm_v2/BBH": 0.595, - "hfopenllm_v2/MATH Level 5": 0.2492, - "hfopenllm_v2/GPQA": 0.3456, - "hfopenllm_v2/MUSR": 0.4398, - "hfopenllm_v2/MMLU-PRO": 0.424 - } - }, - { - "id": "tiiuae/Falcon3-10B-Instruct", - "name": "Falcon3-10B-Instruct", - "developer": "tiiuae", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7817, - "hfopenllm_v2/BBH": 0.617, - "hfopenllm_v2/MATH Level 5": 0.2764, - "hfopenllm_v2/GPQA": 0.3289, - "hfopenllm_v2/MUSR": 0.4323, - "hfopenllm_v2/MMLU-PRO": 0.4429 - } - }, - { - "id": "tiiuae/Falcon3-1B-Base", - "name": "Falcon3-1B-Base", - "developer": "tiiuae", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2428, - "hfopenllm_v2/BBH": 0.3571, - "hfopenllm_v2/MATH Level 5": 0.0332, - "hfopenllm_v2/GPQA": 0.2794, - "hfopenllm_v2/MUSR": 0.4147, - "hfopenllm_v2/MMLU-PRO": 0.1608 - } - }, - { - "id": "tiiuae/Falcon3-1B-Instruct", - "name": "Falcon3-1B-Instruct", - "developer": "tiiuae", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5557, - "hfopenllm_v2/BBH": 0.3745, - "hfopenllm_v2/MATH Level 5": 0.0634, - "hfopenllm_v2/GPQA": 0.2668, - "hfopenllm_v2/MUSR": 0.4189, - "hfopenllm_v2/MMLU-PRO": 0.1838 - } - }, - { - "id": "tiiuae/Falcon3-3B-Base", - "name": "Falcon3-3B-Base", - "developer": "tiiuae", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2765, - "hfopenllm_v2/BBH": 0.4421, - "hfopenllm_v2/MATH Level 5": 0.1178, - "hfopenllm_v2/GPQA": 0.297, - "hfopenllm_v2/MUSR": 0.375, - "hfopenllm_v2/MMLU-PRO": 0.2879 - } - }, - { - "id": "tiiuae/Falcon3-3B-Instruct", - "name": "Falcon3-3B-Instruct", - "developer": "tiiuae", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6977, - "hfopenllm_v2/BBH": 0.4754, - "hfopenllm_v2/MATH Level 5": 0.25, - "hfopenllm_v2/GPQA": 0.2886, - "hfopenllm_v2/MUSR": 0.4136, - "hfopenllm_v2/MMLU-PRO": 0.3005 - } - }, - { - "id": "tiiuae/Falcon3-7B-Base", - "name": "Falcon3-7B-Base", - "developer": "tiiuae", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3416, - "hfopenllm_v2/BBH": 0.5099, - "hfopenllm_v2/MATH Level 5": 0.1941, - "hfopenllm_v2/GPQA": 0.3465, - "hfopenllm_v2/MUSR": 0.4702, - "hfopenllm_v2/MMLU-PRO": 0.391 - } - }, - { - "id": "tiiuae/Falcon3-7B-Instruct", - "name": "Falcon3-7B-Instruct", - "developer": "tiiuae", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7612, - "hfopenllm_v2/BBH": 0.5632, - "hfopenllm_v2/MATH Level 5": 0.4086, - "hfopenllm_v2/GPQA": 0.3104, - "hfopenllm_v2/MUSR": 0.4827, - "hfopenllm_v2/MMLU-PRO": 0.4087 - } - }, - { - "id": "tiiuae/Falcon3-Mamba-7B-Base", - "name": "Falcon3-Mamba-7B-Base", - "developer": "tiiuae", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2891, - "hfopenllm_v2/BBH": 0.4699, - "hfopenllm_v2/MATH Level 5": 0.1941, - "hfopenllm_v2/GPQA": 0.3096, - "hfopenllm_v2/MUSR": 0.3431, - "hfopenllm_v2/MMLU-PRO": 0.3038 - } - }, - { - "id": "tiiuae/Falcon3-Mamba-7B-Instruct", - "name": "Falcon3-Mamba-7B-Instruct", - "developer": "tiiuae", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7165, - "hfopenllm_v2/BBH": 0.4679, - "hfopenllm_v2/MATH Level 5": 0.3006, - "hfopenllm_v2/GPQA": 0.3037, - "hfopenllm_v2/MUSR": 0.3869, - "hfopenllm_v2/MMLU-PRO": 0.3369 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/tijmen2.json b/data/developers/tijmen2.json deleted file mode 100644 index 704d058bd3030289404726f9aae333117b3bed0b..0000000000000000000000000000000000000000 --- a/data/developers/tijmen2.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "Tijmen2", - "models": [ - { - "id": "Tijmen2/cosmosage-v3", - "name": "cosmosage-v3", - "developer": "Tijmen2", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4482, - "hfopenllm_v2/BBH": 0.4551, - "hfopenllm_v2/MATH Level 5": 0.0506, - "hfopenllm_v2/GPQA": 0.2827, - "hfopenllm_v2/MUSR": 0.4199, - "hfopenllm_v2/MMLU-PRO": 0.2486 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/tinycompany.json b/data/developers/tinycompany.json deleted file mode 100644 index aefb31ec443bdaa534331eee717201b46eaa5bf5..0000000000000000000000000000000000000000 --- a/data/developers/tinycompany.json +++ /dev/null @@ -1,215 +0,0 @@ -{ - "developer": "tinycompany", - "models": [ - { - "id": "tinycompany/BiBo-v0.3", - "name": "BiBo-v0.3", - "developer": "tinycompany", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5184, - "hfopenllm_v2/BBH": 0.4642, - "hfopenllm_v2/MATH Level 5": 0.0876, - "hfopenllm_v2/GPQA": 0.2676, - "hfopenllm_v2/MUSR": 0.395, - "hfopenllm_v2/MMLU-PRO": 0.2995 - } - }, - { - "id": "tinycompany/BiBo-v0.7", - "name": "BiBo-v0.7", - "developer": "tinycompany", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3738, - "hfopenllm_v2/BBH": 0.4311, - "hfopenllm_v2/MATH Level 5": 0.0823, - "hfopenllm_v2/GPQA": 0.2768, - "hfopenllm_v2/MUSR": 0.4044, - "hfopenllm_v2/MMLU-PRO": 0.265 - } - }, - { - "id": "tinycompany/ShawtyIsBad-bgem3", - "name": "ShawtyIsBad-bgem3", - "developer": "tinycompany", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2608, - "hfopenllm_v2/BBH": 0.3853, - "hfopenllm_v2/MATH Level 5": 0.0483, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.3695, - "hfopenllm_v2/MMLU-PRO": 0.2583 - } - }, - { - "id": "tinycompany/ShawtyIsBad-e5-large", - "name": "ShawtyIsBad-e5-large", - "developer": "tinycompany", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2468, - "hfopenllm_v2/BBH": 0.3873, - "hfopenllm_v2/MATH Level 5": 0.0453, - "hfopenllm_v2/GPQA": 0.302, - "hfopenllm_v2/MUSR": 0.372, - "hfopenllm_v2/MMLU-PRO": 0.2569 - } - }, - { - "id": "tinycompany/ShawtyIsBad-ib", - "name": "ShawtyIsBad-ib", - "developer": "tinycompany", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2565, - "hfopenllm_v2/BBH": 0.388, - "hfopenllm_v2/MATH Level 5": 0.0491, - "hfopenllm_v2/GPQA": 0.2987, - "hfopenllm_v2/MUSR": 0.3641, - "hfopenllm_v2/MMLU-PRO": 0.2581 - } - }, - { - "id": "tinycompany/ShawtyIsBad-nomic-moe", - "name": "ShawtyIsBad-nomic-moe", - "developer": "tinycompany", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2608, - "hfopenllm_v2/BBH": 0.3878, - "hfopenllm_v2/MATH Level 5": 0.0431, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.3747, - "hfopenllm_v2/MMLU-PRO": 0.2572 - } - }, - { - "id": "tinycompany/ShawtyIsBad-nomic1.5", - "name": "ShawtyIsBad-nomic1.5", - "developer": "tinycompany", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2544, - "hfopenllm_v2/BBH": 0.3874, - "hfopenllm_v2/MATH Level 5": 0.0431, - "hfopenllm_v2/GPQA": 0.3112, - "hfopenllm_v2/MUSR": 0.3628, - "hfopenllm_v2/MMLU-PRO": 0.2567 - } - }, - { - "id": "tinycompany/SigmaBoi-base", - "name": "SigmaBoi-base", - "developer": "tinycompany", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2447, - "hfopenllm_v2/BBH": 0.4314, - "hfopenllm_v2/MATH Level 5": 0.0778, - "hfopenllm_v2/GPQA": 0.2936, - "hfopenllm_v2/MUSR": 0.4343, - "hfopenllm_v2/MMLU-PRO": 0.2817 - } - }, - { - "id": "tinycompany/SigmaBoi-bge-m3", - "name": "SigmaBoi-bge-m3", - "developer": "tinycompany", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.245, - "hfopenllm_v2/BBH": 0.4351, - "hfopenllm_v2/MATH Level 5": 0.0763, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.4383, - "hfopenllm_v2/MMLU-PRO": 0.2819 - } - }, - { - "id": "tinycompany/SigmaBoi-bgem3", - "name": "SigmaBoi-bgem3", - "developer": "tinycompany", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.245, - "hfopenllm_v2/BBH": 0.4351, - "hfopenllm_v2/MATH Level 5": 0.0763, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.4383, - "hfopenllm_v2/MMLU-PRO": 0.2819 - } - }, - { - "id": "tinycompany/SigmaBoi-ib", - "name": "SigmaBoi-ib", - "developer": "tinycompany", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2477, - "hfopenllm_v2/BBH": 0.4344, - "hfopenllm_v2/MATH Level 5": 0.074, - "hfopenllm_v2/GPQA": 0.2878, - "hfopenllm_v2/MUSR": 0.429, - "hfopenllm_v2/MMLU-PRO": 0.2824 - } - }, - { - "id": "tinycompany/SigmaBoi-nomic-moe", - "name": "SigmaBoi-nomic-moe", - "developer": "tinycompany", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2474, - "hfopenllm_v2/BBH": 0.4334, - "hfopenllm_v2/MATH Level 5": 0.0718, - "hfopenllm_v2/GPQA": 0.2928, - "hfopenllm_v2/MUSR": 0.4316, - "hfopenllm_v2/MMLU-PRO": 0.2837 - } - }, - { - "id": "tinycompany/SigmaBoi-nomic1.5", - "name": "SigmaBoi-nomic1.5", - "developer": "tinycompany", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2447, - "hfopenllm_v2/BBH": 0.4371, - "hfopenllm_v2/MATH Level 5": 0.0831, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.4316, - "hfopenllm_v2/MMLU-PRO": 0.2841 - } - }, - { - "id": "tinycompany/SigmaBoi-nomic1.5-fp32", - "name": "SigmaBoi-nomic1.5-fp32", - "developer": "tinycompany", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2462, - "hfopenllm_v2/BBH": 0.4371, - "hfopenllm_v2/MATH Level 5": 0.0831, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.4316, - "hfopenllm_v2/MMLU-PRO": 0.2841 - } - }, - { - "id": "tinycompany/Tamed-Shawty", - "name": "Tamed-Shawty", - "developer": "tinycompany", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3831, - "hfopenllm_v2/BBH": 0.3837, - "hfopenllm_v2/MATH Level 5": 0.0718, - "hfopenllm_v2/GPQA": 0.2626, - "hfopenllm_v2/MUSR": 0.3501, - "hfopenllm_v2/MMLU-PRO": 0.2601 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/tinyllama.json b/data/developers/tinyllama.json deleted file mode 100644 index 497ffeb47f52828836f7758f7a9a1082c24b52ea..0000000000000000000000000000000000000000 --- a/data/developers/tinyllama.json +++ /dev/null @@ -1,89 +0,0 @@ -{ - "developer": "TinyLlama", - "models": [ - { - "id": "TinyLlama/TinyLlama-1.1B-Chat-v0.1", - "name": "TinyLlama-1.1B-Chat-v0.1", - "developer": "TinyLlama", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1479, - "hfopenllm_v2/BBH": 0.3084, - "hfopenllm_v2/MATH Level 5": 0.006, - "hfopenllm_v2/GPQA": 0.229, - "hfopenllm_v2/MUSR": 0.3592, - "hfopenllm_v2/MMLU-PRO": 0.1098 - } - }, - { - "id": "TinyLlama/TinyLlama-1.1B-Chat-v0.5", - "name": "TinyLlama-1.1B-Chat-v0.5", - "developer": "TinyLlama", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1634, - "hfopenllm_v2/BBH": 0.3105, - "hfopenllm_v2/MATH Level 5": 0.0038, - "hfopenllm_v2/GPQA": 0.2483, - "hfopenllm_v2/MUSR": 0.3661, - "hfopenllm_v2/MMLU-PRO": 0.1096 - } - }, - { - "id": "TinyLlama/TinyLlama-1.1B-Chat-v0.6", - "name": "TinyLlama-1.1B-Chat-v0.6", - "developer": "TinyLlama", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1574, - "hfopenllm_v2/BBH": 0.3067, - "hfopenllm_v2/MATH Level 5": 0.0159, - "hfopenllm_v2/GPQA": 0.2584, - "hfopenllm_v2/MUSR": 0.3422, - "hfopenllm_v2/MMLU-PRO": 0.1149 - } - }, - { - "id": "TinyLlama/TinyLlama-1.1B-Chat-v1.0", - "name": "TinyLlama-1.1B-Chat-v1.0", - "developer": "TinyLlama", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0596, - "hfopenllm_v2/BBH": 0.3104, - "hfopenllm_v2/MATH Level 5": 0.0151, - "hfopenllm_v2/GPQA": 0.25, - "hfopenllm_v2/MUSR": 0.3515, - "hfopenllm_v2/MMLU-PRO": 0.1101 - } - }, - { - "id": "TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T", - "name": "TinyLlama-1.1B-intermediate-step-1431k-3T", - "developer": "TinyLlama", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2277, - "hfopenllm_v2/BBH": 0.3071, - "hfopenllm_v2/MATH Level 5": 0.0121, - "hfopenllm_v2/GPQA": 0.2525, - "hfopenllm_v2/MUSR": 0.338, - "hfopenllm_v2/MMLU-PRO": 0.112 - } - }, - { - "id": "TinyLlama/TinyLlama_v1.1", - "name": "TinyLlama_v1.1", - "developer": "TinyLlama", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2001, - "hfopenllm_v2/BBH": 0.3024, - "hfopenllm_v2/MATH Level 5": 0.0121, - "hfopenllm_v2/GPQA": 0.2458, - "hfopenllm_v2/MUSR": 0.37, - "hfopenllm_v2/MMLU-PRO": 0.1049 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/tklohj.json b/data/developers/tklohj.json deleted file mode 100644 index b7fe7986665fbf31cb59d414701c967231e18665..0000000000000000000000000000000000000000 --- a/data/developers/tklohj.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "tklohj", - "models": [ - { - "id": "tklohj/WindyFloLLM", - "name": "WindyFloLLM", - "developer": "tklohj", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2669, - "hfopenllm_v2/BBH": 0.4637, - "hfopenllm_v2/MATH Level 5": 0.0159, - "hfopenllm_v2/GPQA": 0.2752, - "hfopenllm_v2/MUSR": 0.4253, - "hfopenllm_v2/MMLU-PRO": 0.2581 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/toastypigeon.json b/data/developers/toastypigeon.json deleted file mode 100644 index 78d9a15ae7c4e769870922b86d113659a479616b..0000000000000000000000000000000000000000 --- a/data/developers/toastypigeon.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "ToastyPigeon", - "models": [ - { - "id": "ToastyPigeon/Sto-vo-kor-12B", - "name": "Sto-vo-kor-12B", - "developer": "ToastyPigeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5501, - "hfopenllm_v2/BBH": 0.5065, - "hfopenllm_v2/MATH Level 5": 0.1088, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.3938, - "hfopenllm_v2/MMLU-PRO": 0.3398 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/together.json b/data/developers/together.json deleted file mode 100644 index 406f7bcf87e4781c2c0c59b6c4757d81c413c501..0000000000000000000000000000000000000000 --- a/data/developers/together.json +++ /dev/null @@ -1,97 +0,0 @@ -{ - "developer": "together", - "models": [ - { - "id": "together/RedPajama-INCITE-Base-7B", - "name": "RedPajama-INCITE-Base 7B", - "developer": "together", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.378, - "helm_classic/MMLU": 0.302, - "helm_classic/BoolQ": 0.713, - "helm_classic/NarrativeQA": 0.617, - "helm_classic/NaturalQuestions (open-book)": 0.586, - "helm_classic/QuAC": 0.336, - "helm_classic/HellaSwag": -1.0, - "helm_classic/OpenbookQA": -1.0, - "helm_classic/TruthfulQA": 0.205, - "helm_classic/MS MARCO (TREC)": -1.0, - "helm_classic/CNN/DailyMail": -1.0, - "helm_classic/XSUM": -1.0, - "helm_classic/IMDB": 0.752, - "helm_classic/CivilComments": 0.547, - "helm_classic/RAFT": 0.648 - } - }, - { - "id": "together/RedPajama-INCITE-Base-v1-3B", - "name": "RedPajama-INCITE-Base-v1 3B", - "developer": "together", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.311, - "helm_classic/MMLU": 0.263, - "helm_classic/BoolQ": 0.685, - "helm_classic/NarrativeQA": 0.555, - "helm_classic/NaturalQuestions (open-book)": 0.52, - "helm_classic/QuAC": 0.309, - "helm_classic/HellaSwag": -1.0, - "helm_classic/OpenbookQA": -1.0, - "helm_classic/TruthfulQA": 0.277, - "helm_classic/MS MARCO (TREC)": -1.0, - "helm_classic/CNN/DailyMail": -1.0, - "helm_classic/XSUM": -1.0, - "helm_classic/IMDB": 0.907, - "helm_classic/CivilComments": 0.549, - "helm_classic/RAFT": 0.502 - } - }, - { - "id": "together/RedPajama-INCITE-Instruct-7B", - "name": "RedPajama-INCITE-Instruct 7B", - "developer": "together", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.524, - "helm_classic/MMLU": 0.363, - "helm_classic/BoolQ": 0.705, - "helm_classic/NarrativeQA": 0.638, - "helm_classic/NaturalQuestions (open-book)": 0.659, - "helm_classic/QuAC": 0.26, - "helm_classic/HellaSwag": -1.0, - "helm_classic/OpenbookQA": -1.0, - "helm_classic/TruthfulQA": 0.243, - "helm_classic/MS MARCO (TREC)": -1.0, - "helm_classic/CNN/DailyMail": -1.0, - "helm_classic/XSUM": -1.0, - "helm_classic/IMDB": 0.927, - "helm_classic/CivilComments": 0.664, - "helm_classic/RAFT": 0.695 - } - }, - { - "id": "together/RedPajama-INCITE-Instruct-v1-3B", - "name": "RedPajama-INCITE-Instruct-v1 3B", - "developer": "together", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.366, - "helm_classic/MMLU": 0.257, - "helm_classic/BoolQ": 0.677, - "helm_classic/NarrativeQA": 0.638, - "helm_classic/NaturalQuestions (open-book)": 0.637, - "helm_classic/QuAC": 0.259, - "helm_classic/HellaSwag": -1.0, - "helm_classic/OpenbookQA": -1.0, - "helm_classic/TruthfulQA": 0.208, - "helm_classic/MS MARCO (TREC)": -1.0, - "helm_classic/CNN/DailyMail": -1.0, - "helm_classic/XSUM": -1.0, - "helm_classic/IMDB": 0.894, - "helm_classic/CivilComments": 0.549, - "helm_classic/RAFT": 0.661 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/togethercomputer.json b/data/developers/togethercomputer.json deleted file mode 100644 index 152a501a0475b14c1abac56c7280e5ff47fb39ab..0000000000000000000000000000000000000000 --- a/data/developers/togethercomputer.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "developer": "togethercomputer", - "models": [ - { - "id": "togethercomputer/GPT-JT-6B-v1", - "name": "GPT-JT-6B-v1", - "developer": "togethercomputer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2061, - "hfopenllm_v2/BBH": 0.3303, - "hfopenllm_v2/MATH Level 5": 0.0106, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.3737, - "hfopenllm_v2/MMLU-PRO": 0.1626 - } - }, - { - "id": "togethercomputer/GPT-NeoXT-Chat-Base-20B", - "name": "GPT-NeoXT-Chat-Base-20B", - "developer": "togethercomputer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.183, - "hfopenllm_v2/BBH": 0.3321, - "hfopenllm_v2/MATH Level 5": 0.0234, - "hfopenllm_v2/GPQA": 0.25, - "hfopenllm_v2/MUSR": 0.3461, - "hfopenllm_v2/MMLU-PRO": 0.1145 - } - }, - { - "id": "togethercomputer/LLaMA-2-7B-32K", - "name": "LLaMA-2-7B-32K", - "developer": "togethercomputer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1865, - "hfopenllm_v2/BBH": 0.34, - "hfopenllm_v2/MATH Level 5": 0.0144, - "hfopenllm_v2/GPQA": 0.25, - "hfopenllm_v2/MUSR": 0.3754, - "hfopenllm_v2/MMLU-PRO": 0.1768 - } - }, - { - "id": "togethercomputer/Llama-2-7B-32K-Instruct", - "name": "Llama-2-7B-32K-Instruct", - "developer": "togethercomputer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.213, - "hfopenllm_v2/BBH": 0.3443, - "hfopenllm_v2/MATH Level 5": 0.0159, - "hfopenllm_v2/GPQA": 0.2517, - "hfopenllm_v2/MUSR": 0.4056, - "hfopenllm_v2/MMLU-PRO": 0.1781 - } - }, - { - "id": "togethercomputer/RedPajama-INCITE-7B-Base", - "name": "RedPajama-INCITE-7B-Base", - "developer": "togethercomputer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2082, - "hfopenllm_v2/BBH": 0.3195, - "hfopenllm_v2/MATH Level 5": 0.0159, - "hfopenllm_v2/GPQA": 0.255, - "hfopenllm_v2/MUSR": 0.362, - "hfopenllm_v2/MMLU-PRO": 0.1197 - } - }, - { - "id": "togethercomputer/RedPajama-INCITE-7B-Chat", - "name": "RedPajama-INCITE-7B-Chat", - "developer": "togethercomputer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1558, - "hfopenllm_v2/BBH": 0.3175, - "hfopenllm_v2/MATH Level 5": 0.0068, - "hfopenllm_v2/GPQA": 0.2525, - "hfopenllm_v2/MUSR": 0.3448, - "hfopenllm_v2/MMLU-PRO": 0.1121 - } - }, - { - "id": "togethercomputer/RedPajama-INCITE-7B-Instruct", - "name": "RedPajama-INCITE-7B-Instruct", - "developer": "togethercomputer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2055, - "hfopenllm_v2/BBH": 0.3377, - "hfopenllm_v2/MATH Level 5": 0.0211, - "hfopenllm_v2/GPQA": 0.2508, - "hfopenllm_v2/MUSR": 0.3685, - "hfopenllm_v2/MMLU-PRO": 0.1272 - } - }, - { - "id": "togethercomputer/RedPajama-INCITE-Base-3B-v1", - "name": "RedPajama-INCITE-Base-3B-v1", - "developer": "togethercomputer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2294, - "hfopenllm_v2/BBH": 0.306, - "hfopenllm_v2/MATH Level 5": 0.0144, - "hfopenllm_v2/GPQA": 0.2433, - "hfopenllm_v2/MUSR": 0.3739, - "hfopenllm_v2/MMLU-PRO": 0.1111 - } - }, - { - "id": "togethercomputer/RedPajama-INCITE-Chat-3B-v1", - "name": "RedPajama-INCITE-Chat-3B-v1", - "developer": "togethercomputer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1652, - "hfopenllm_v2/BBH": 0.3217, - "hfopenllm_v2/MATH Level 5": 0.0091, - "hfopenllm_v2/GPQA": 0.2441, - "hfopenllm_v2/MUSR": 0.3684, - "hfopenllm_v2/MMLU-PRO": 0.1127 - } - }, - { - "id": "togethercomputer/RedPajama-INCITE-Instruct-3B-v1", - "name": "RedPajama-INCITE-Instruct-3B-v1", - "developer": "togethercomputer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2124, - "hfopenllm_v2/BBH": 0.3146, - "hfopenllm_v2/MATH Level 5": 0.0128, - "hfopenllm_v2/GPQA": 0.2475, - "hfopenllm_v2/MUSR": 0.3886, - "hfopenllm_v2/MMLU-PRO": 0.111 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/tokyotech-llm.json b/data/developers/tokyotech-llm.json deleted file mode 100644 index d89a0c9d89d224fd8856c2b701dbf3579e2488c2..0000000000000000000000000000000000000000 --- a/data/developers/tokyotech-llm.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "tokyotech-llm", - "models": [ - { - "id": "tokyotech-llm/Llama-3-Swallow-8B-Instruct-v0.1", - "name": "Llama-3-Swallow-8B-Instruct-v0.1", - "developer": "tokyotech-llm", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5508, - "hfopenllm_v2/BBH": 0.5009, - "hfopenllm_v2/MATH Level 5": 0.0748, - "hfopenllm_v2/GPQA": 0.2894, - "hfopenllm_v2/MUSR": 0.4357, - "hfopenllm_v2/MMLU-PRO": 0.3088 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/tomasmcm.json b/data/developers/tomasmcm.json deleted file mode 100644 index a65cb2a8174542e5cfaecbe8a958bccab476acc2..0000000000000000000000000000000000000000 --- a/data/developers/tomasmcm.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "tomasmcm", - "models": [ - { - "id": "tomasmcm/sky-t1-coder-32b-flash", - "name": "sky-t1-coder-32b-flash", - "developer": "tomasmcm", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.778, - "hfopenllm_v2/BBH": 0.6822, - "hfopenllm_v2/MATH Level 5": 0.5423, - "hfopenllm_v2/GPQA": 0.3683, - "hfopenllm_v2/MUSR": 0.4233, - "hfopenllm_v2/MMLU-PRO": 0.5782 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/trappu.json b/data/developers/trappu.json deleted file mode 100644 index 68f5894c486680acbd0d60271a2cad4b3980837d..0000000000000000000000000000000000000000 --- a/data/developers/trappu.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "Trappu", - "models": [ - { - "id": "Trappu/Magnum-Picaro-0.7-v2-12b", - "name": "Magnum-Picaro-0.7-v2-12b", - "developer": "Trappu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3003, - "hfopenllm_v2/BBH": 0.5507, - "hfopenllm_v2/MATH Level 5": 0.0665, - "hfopenllm_v2/GPQA": 0.323, - "hfopenllm_v2/MUSR": 0.4727, - "hfopenllm_v2/MMLU-PRO": 0.358 - } - }, - { - "id": "Trappu/Nemo-Picaro-12B", - "name": "Nemo-Picaro-12B", - "developer": "Trappu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2577, - "hfopenllm_v2/BBH": 0.549, - "hfopenllm_v2/MATH Level 5": 0.0846, - "hfopenllm_v2/GPQA": 0.3272, - "hfopenllm_v2/MUSR": 0.4726, - "hfopenllm_v2/MMLU-PRO": 0.3605 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/tremontaine.json b/data/developers/tremontaine.json deleted file mode 100644 index 20b1ca478985df9227ce5715f269cd38654609db..0000000000000000000000000000000000000000 --- a/data/developers/tremontaine.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "Tremontaine", - "models": [ - { - "id": "Tremontaine/L3-12B-Lunaris-v1", - "name": "L3-12B-Lunaris-v1", - "developer": "Tremontaine", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6909, - "hfopenllm_v2/BBH": 0.523, - "hfopenllm_v2/MATH Level 5": 0.0876, - "hfopenllm_v2/GPQA": 0.3096, - "hfopenllm_v2/MUSR": 0.3674, - "hfopenllm_v2/MMLU-PRO": 0.3775 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/triangle104.json b/data/developers/triangle104.json deleted file mode 100644 index 86646ba029b55228f0615ec25b16293d85ab7a94..0000000000000000000000000000000000000000 --- a/data/developers/triangle104.json +++ /dev/null @@ -1,859 +0,0 @@ -{ - "developer": "Triangle104", - "models": [ - { - "id": "Triangle104/Annunaki-12b", - "name": "Annunaki-12b", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3872, - "hfopenllm_v2/BBH": 0.5499, - "hfopenllm_v2/MATH Level 5": 0.1216, - "hfopenllm_v2/GPQA": 0.3213, - "hfopenllm_v2/MUSR": 0.4409, - "hfopenllm_v2/MMLU-PRO": 0.3721 - } - }, - { - "id": "Triangle104/BigTalker-Lite-8B", - "name": "BigTalker-Lite-8B", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3689, - "hfopenllm_v2/BBH": 0.5308, - "hfopenllm_v2/MATH Level 5": 0.102, - "hfopenllm_v2/GPQA": 0.3104, - "hfopenllm_v2/MUSR": 0.4208, - "hfopenllm_v2/MMLU-PRO": 0.3431 - } - }, - { - "id": "Triangle104/Chatty-Harry_V2.0", - "name": "Chatty-Harry_V2.0", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3326, - "hfopenllm_v2/BBH": 0.5319, - "hfopenllm_v2/MATH Level 5": 0.139, - "hfopenllm_v2/GPQA": 0.323, - "hfopenllm_v2/MUSR": 0.4078, - "hfopenllm_v2/MMLU-PRO": 0.3683 - } - }, - { - "id": "Triangle104/Chatty-Harry_V3.0", - "name": "Chatty-Harry_V3.0", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3675, - "hfopenllm_v2/BBH": 0.5526, - "hfopenllm_v2/MATH Level 5": 0.1125, - "hfopenllm_v2/GPQA": 0.323, - "hfopenllm_v2/MUSR": 0.4408, - "hfopenllm_v2/MMLU-PRO": 0.3702 - } - }, - { - "id": "Triangle104/Chronos-Prism_V1.0", - "name": "Chronos-Prism_V1.0", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3259, - "hfopenllm_v2/BBH": 0.5554, - "hfopenllm_v2/MATH Level 5": 0.1201, - "hfopenllm_v2/GPQA": 0.3096, - "hfopenllm_v2/MUSR": 0.4263, - "hfopenllm_v2/MMLU-PRO": 0.3673 - } - }, - { - "id": "Triangle104/Dark-Chivalry_V1.0", - "name": "Dark-Chivalry_V1.0", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4326, - "hfopenllm_v2/BBH": 0.4974, - "hfopenllm_v2/MATH Level 5": 0.1314, - "hfopenllm_v2/GPQA": 0.2936, - "hfopenllm_v2/MUSR": 0.4182, - "hfopenllm_v2/MMLU-PRO": 0.3444 - } - }, - { - "id": "Triangle104/Distilled-DarkPlanet-Allades-8B", - "name": "Distilled-DarkPlanet-Allades-8B", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.346, - "hfopenllm_v2/BBH": 0.4634, - "hfopenllm_v2/MATH Level 5": 0.4003, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.3538, - "hfopenllm_v2/MMLU-PRO": 0.2901 - } - }, - { - "id": "Triangle104/Distilled-DarkPlanet-Allades-8B_TIES", - "name": "Distilled-DarkPlanet-Allades-8B_TIES", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3892, - "hfopenllm_v2/BBH": 0.5042, - "hfopenllm_v2/MATH Level 5": 0.0906, - "hfopenllm_v2/GPQA": 0.3146, - "hfopenllm_v2/MUSR": 0.3868, - "hfopenllm_v2/MMLU-PRO": 0.3401 - } - }, - { - "id": "Triangle104/Distilled-Whiskey-8b", - "name": "Distilled-Whiskey-8b", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3448, - "hfopenllm_v2/BBH": 0.5028, - "hfopenllm_v2/MATH Level 5": 0.2545, - "hfopenllm_v2/GPQA": 0.3314, - "hfopenllm_v2/MUSR": 0.4172, - "hfopenllm_v2/MMLU-PRO": 0.3367 - } - }, - { - "id": "Triangle104/Dolphin3-Llama3.2-Smart", - "name": "Dolphin3-Llama3.2-Smart", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4137, - "hfopenllm_v2/BBH": 0.3975, - "hfopenllm_v2/MATH Level 5": 0.0438, - "hfopenllm_v2/GPQA": 0.2693, - "hfopenllm_v2/MUSR": 0.3922, - "hfopenllm_v2/MMLU-PRO": 0.2195 - } - }, - { - "id": "Triangle104/DS-Distilled-Hermes-Llama-3.1", - "name": "DS-Distilled-Hermes-Llama-3.1", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3229, - "hfopenllm_v2/BBH": 0.5117, - "hfopenllm_v2/MATH Level 5": 0.2931, - "hfopenllm_v2/GPQA": 0.3188, - "hfopenllm_v2/MUSR": 0.4039, - "hfopenllm_v2/MMLU-PRO": 0.311 - } - }, - { - "id": "Triangle104/DS-Distilled-Hermes-Llama-3.1_TIES", - "name": "DS-Distilled-Hermes-Llama-3.1_TIES", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1364, - "hfopenllm_v2/BBH": 0.2928, - "hfopenllm_v2/MATH Level 5": 0.0091, - "hfopenllm_v2/GPQA": 0.245, - "hfopenllm_v2/MUSR": 0.3621, - "hfopenllm_v2/MMLU-PRO": 0.1104 - } - }, - { - "id": "Triangle104/DS-R1-Distill-Q2.5-10B-Harmony", - "name": "DS-R1-Distill-Q2.5-10B-Harmony", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1751, - "hfopenllm_v2/BBH": 0.2643, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2106, - "hfopenllm_v2/MUSR": 0.3128, - "hfopenllm_v2/MMLU-PRO": 0.1173 - } - }, - { - "id": "Triangle104/DS-R1-Distill-Q2.5-14B-Harmony_V0.1", - "name": "DS-R1-Distill-Q2.5-14B-Harmony_V0.1", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4515, - "hfopenllm_v2/BBH": 0.5783, - "hfopenllm_v2/MATH Level 5": 0.5551, - "hfopenllm_v2/GPQA": 0.3935, - "hfopenllm_v2/MUSR": 0.5567, - "hfopenllm_v2/MMLU-PRO": 0.4601 - } - }, - { - "id": "Triangle104/DS-R1-Distill-Q2.5-7B-RP", - "name": "DS-R1-Distill-Q2.5-7B-RP", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3445, - "hfopenllm_v2/BBH": 0.4383, - "hfopenllm_v2/MATH Level 5": 0.4683, - "hfopenllm_v2/GPQA": 0.3138, - "hfopenllm_v2/MUSR": 0.403, - "hfopenllm_v2/MMLU-PRO": 0.2891 - } - }, - { - "id": "Triangle104/DS-R1-Llama-8B-Harmony", - "name": "DS-R1-Llama-8B-Harmony", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3566, - "hfopenllm_v2/BBH": 0.4154, - "hfopenllm_v2/MATH Level 5": 0.4282, - "hfopenllm_v2/GPQA": 0.2919, - "hfopenllm_v2/MUSR": 0.3762, - "hfopenllm_v2/MMLU-PRO": 0.2744 - } - }, - { - "id": "Triangle104/DSR1-Distill-Llama-Lit-8B", - "name": "DSR1-Distill-Llama-Lit-8B", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1885, - "hfopenllm_v2/BBH": 0.4284, - "hfopenllm_v2/MATH Level 5": 0.352, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.3535, - "hfopenllm_v2/MMLU-PRO": 0.2798 - } - }, - { - "id": "Triangle104/DSR1-Distill-Qwen-7B-RP", - "name": "DSR1-Distill-Qwen-7B-RP", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3609, - "hfopenllm_v2/BBH": 0.4326, - "hfopenllm_v2/MATH Level 5": 0.4804, - "hfopenllm_v2/GPQA": 0.3196, - "hfopenllm_v2/MUSR": 0.4045, - "hfopenllm_v2/MMLU-PRO": 0.3028 - } - }, - { - "id": "Triangle104/Gemmadevi-Stock-10B", - "name": "Gemmadevi-Stock-10B", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1582, - "hfopenllm_v2/BBH": 0.6066, - "hfopenllm_v2/MATH Level 5": 0.0967, - "hfopenllm_v2/GPQA": 0.3532, - "hfopenllm_v2/MUSR": 0.4621, - "hfopenllm_v2/MMLU-PRO": 0.4262 - } - }, - { - "id": "Triangle104/Hermes-Llama-3.2-CoT", - "name": "Hermes-Llama-3.2-CoT", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4178, - "hfopenllm_v2/BBH": 0.4616, - "hfopenllm_v2/MATH Level 5": 0.0952, - "hfopenllm_v2/GPQA": 0.2794, - "hfopenllm_v2/MUSR": 0.3698, - "hfopenllm_v2/MMLU-PRO": 0.2947 - } - }, - { - "id": "Triangle104/Hermes-Llama-3.2-CoT-Summary", - "name": "Hermes-Llama-3.2-CoT-Summary", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.483, - "hfopenllm_v2/BBH": 0.42, - "hfopenllm_v2/MATH Level 5": 0.0831, - "hfopenllm_v2/GPQA": 0.2559, - "hfopenllm_v2/MUSR": 0.3575, - "hfopenllm_v2/MMLU-PRO": 0.2901 - } - }, - { - "id": "Triangle104/Hermes3-L3.1-DirtyHarry-8B", - "name": "Hermes3-L3.1-DirtyHarry-8B", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3242, - "hfopenllm_v2/BBH": 0.5066, - "hfopenllm_v2/MATH Level 5": 0.0718, - "hfopenllm_v2/GPQA": 0.302, - "hfopenllm_v2/MUSR": 0.4069, - "hfopenllm_v2/MMLU-PRO": 0.3339 - } - }, - { - "id": "Triangle104/Herodotos-14B", - "name": "Herodotos-14B", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4667, - "hfopenllm_v2/BBH": 0.6435, - "hfopenllm_v2/MATH Level 5": 0.5045, - "hfopenllm_v2/GPQA": 0.3733, - "hfopenllm_v2/MUSR": 0.4795, - "hfopenllm_v2/MMLU-PRO": 0.529 - } - }, - { - "id": "Triangle104/Herodotos-14B_V0.1", - "name": "Herodotos-14B_V0.1", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1879, - "hfopenllm_v2/BBH": 0.3017, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.224, - "hfopenllm_v2/MUSR": 0.3684, - "hfopenllm_v2/MMLU-PRO": 0.1164 - } - }, - { - "id": "Triangle104/L3.1-8B-Dusky-Ink", - "name": "L3.1-8B-Dusky-Ink", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.453, - "hfopenllm_v2/BBH": 0.5098, - "hfopenllm_v2/MATH Level 5": 0.1231, - "hfopenllm_v2/GPQA": 0.2894, - "hfopenllm_v2/MUSR": 0.4224, - "hfopenllm_v2/MMLU-PRO": 0.3683 - } - }, - { - "id": "Triangle104/L3.1-8B-Dusky-Ink_v0.r1", - "name": "L3.1-8B-Dusky-Ink_v0.r1", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1985, - "hfopenllm_v2/BBH": 0.4337, - "hfopenllm_v2/MATH Level 5": 0.0431, - "hfopenllm_v2/GPQA": 0.3037, - "hfopenllm_v2/MUSR": 0.3988, - "hfopenllm_v2/MMLU-PRO": 0.3206 - } - }, - { - "id": "Triangle104/Llama3.1-Allades-Lit-8b", - "name": "Llama3.1-Allades-Lit-8b", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2461, - "hfopenllm_v2/BBH": 0.4183, - "hfopenllm_v2/MATH Level 5": 0.0023, - "hfopenllm_v2/GPQA": 0.2844, - "hfopenllm_v2/MUSR": 0.3708, - "hfopenllm_v2/MMLU-PRO": 0.2724 - } - }, - { - "id": "Triangle104/Llama3.1-cc-Lit-8b", - "name": "Llama3.1-cc-Lit-8b", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2993, - "hfopenllm_v2/BBH": 0.3848, - "hfopenllm_v2/MATH Level 5": 0.003, - "hfopenllm_v2/GPQA": 0.2777, - "hfopenllm_v2/MUSR": 0.3854, - "hfopenllm_v2/MMLU-PRO": 0.3004 - } - }, - { - "id": "Triangle104/LThreePointOne-8B-HermesBlackroot", - "name": "LThreePointOne-8B-HermesBlackroot", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1792, - "hfopenllm_v2/BBH": 0.4998, - "hfopenllm_v2/MATH Level 5": 0.0196, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.3586, - "hfopenllm_v2/MMLU-PRO": 0.3285 - } - }, - { - "id": "Triangle104/LThreePointOne-8B-HermesInk", - "name": "LThreePointOne-8B-HermesInk", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4031, - "hfopenllm_v2/BBH": 0.5223, - "hfopenllm_v2/MATH Level 5": 0.1722, - "hfopenllm_v2/GPQA": 0.323, - "hfopenllm_v2/MUSR": 0.4129, - "hfopenllm_v2/MMLU-PRO": 0.3467 - } - }, - { - "id": "Triangle104/Minerva-1.5b", - "name": "Minerva-1.5b", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2694, - "hfopenllm_v2/BBH": 0.4026, - "hfopenllm_v2/MATH Level 5": 0.1027, - "hfopenllm_v2/GPQA": 0.3104, - "hfopenllm_v2/MUSR": 0.3655, - "hfopenllm_v2/MMLU-PRO": 0.2698 - } - }, - { - "id": "Triangle104/Minerva-1.5b_V0.2", - "name": "Minerva-1.5b_V0.2", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3083, - "hfopenllm_v2/BBH": 0.3989, - "hfopenllm_v2/MATH Level 5": 0.114, - "hfopenllm_v2/GPQA": 0.2852, - "hfopenllm_v2/MUSR": 0.396, - "hfopenllm_v2/MMLU-PRO": 0.2911 - } - }, - { - "id": "Triangle104/Minerva-10b", - "name": "Minerva-10b", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1879, - "hfopenllm_v2/BBH": 0.4462, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.281, - "hfopenllm_v2/MUSR": 0.3627, - "hfopenllm_v2/MMLU-PRO": 0.2318 - } - }, - { - "id": "Triangle104/Minerva-14b", - "name": "Minerva-14b", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3468, - "hfopenllm_v2/BBH": 0.6301, - "hfopenllm_v2/MATH Level 5": 0.3051, - "hfopenllm_v2/GPQA": 0.3742, - "hfopenllm_v2/MUSR": 0.4766, - "hfopenllm_v2/MMLU-PRO": 0.5194 - } - }, - { - "id": "Triangle104/Minerva-14b-V0.1", - "name": "Minerva-14b-V0.1", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0861, - "hfopenllm_v2/BBH": 0.609, - "hfopenllm_v2/MATH Level 5": 0.3051, - "hfopenllm_v2/GPQA": 0.3658, - "hfopenllm_v2/MUSR": 0.47, - "hfopenllm_v2/MMLU-PRO": 0.5118 - } - }, - { - "id": "Triangle104/Minerva-7b", - "name": "Minerva-7b", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3724, - "hfopenllm_v2/BBH": 0.5498, - "hfopenllm_v2/MATH Level 5": 0.284, - "hfopenllm_v2/GPQA": 0.323, - "hfopenllm_v2/MUSR": 0.4143, - "hfopenllm_v2/MMLU-PRO": 0.4444 - } - }, - { - "id": "Triangle104/Minerva-8b", - "name": "Minerva-8b", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1721, - "hfopenllm_v2/BBH": 0.4669, - "hfopenllm_v2/MATH Level 5": 0.0045, - "hfopenllm_v2/GPQA": 0.3121, - "hfopenllm_v2/MUSR": 0.4273, - "hfopenllm_v2/MMLU-PRO": 0.3089 - } - }, - { - "id": "Triangle104/Mistral-Redemption-Arc", - "name": "Mistral-Redemption-Arc", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4029, - "hfopenllm_v2/BBH": 0.6255, - "hfopenllm_v2/MATH Level 5": 0.4101, - "hfopenllm_v2/GPQA": 0.3473, - "hfopenllm_v2/MUSR": 0.4595, - "hfopenllm_v2/MMLU-PRO": 0.451 - } - }, - { - "id": "Triangle104/Mistral-Small-24b-Harmony", - "name": "Mistral-Small-24b-Harmony", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1687, - "hfopenllm_v2/BBH": 0.6434, - "hfopenllm_v2/MATH Level 5": 0.1911, - "hfopenllm_v2/GPQA": 0.3842, - "hfopenllm_v2/MUSR": 0.4276, - "hfopenllm_v2/MMLU-PRO": 0.5431 - } - }, - { - "id": "Triangle104/Pans_Gutenbergum_V0.1", - "name": "Pans_Gutenbergum_V0.1", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3097, - "hfopenllm_v2/BBH": 0.5541, - "hfopenllm_v2/MATH Level 5": 0.1057, - "hfopenllm_v2/GPQA": 0.323, - "hfopenllm_v2/MUSR": 0.4528, - "hfopenllm_v2/MMLU-PRO": 0.3697 - } - }, - { - "id": "Triangle104/Pans_Gutenbergum_V0.2", - "name": "Pans_Gutenbergum_V0.2", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3215, - "hfopenllm_v2/BBH": 0.5526, - "hfopenllm_v2/MATH Level 5": 0.0687, - "hfopenllm_v2/GPQA": 0.3121, - "hfopenllm_v2/MUSR": 0.4673, - "hfopenllm_v2/MMLU-PRO": 0.3585 - } - }, - { - "id": "Triangle104/Pantheon_ChatWaifu_V0.2", - "name": "Pantheon_ChatWaifu_V0.2", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2683, - "hfopenllm_v2/BBH": 0.5532, - "hfopenllm_v2/MATH Level 5": 0.0566, - "hfopenllm_v2/GPQA": 0.318, - "hfopenllm_v2/MUSR": 0.4755, - "hfopenllm_v2/MMLU-PRO": 0.3442 - } - }, - { - "id": "Triangle104/Phi-4-AbliteratedRP", - "name": "Phi-4-AbliteratedRP", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4923, - "hfopenllm_v2/BBH": 0.6709, - "hfopenllm_v2/MATH Level 5": 0.3074, - "hfopenllm_v2/GPQA": 0.3951, - "hfopenllm_v2/MUSR": 0.5098, - "hfopenllm_v2/MMLU-PRO": 0.5308 - } - }, - { - "id": "Triangle104/Phi4-RP-o1", - "name": "Phi4-RP-o1", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.022, - "hfopenllm_v2/BBH": 0.6653, - "hfopenllm_v2/MATH Level 5": 0.3776, - "hfopenllm_v2/GPQA": 0.3733, - "hfopenllm_v2/MUSR": 0.4756, - "hfopenllm_v2/MMLU-PRO": 0.5111 - } - }, - { - "id": "Triangle104/Phi4-RP-o1-Ablit", - "name": "Phi4-RP-o1-Ablit", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0239, - "hfopenllm_v2/BBH": 0.663, - "hfopenllm_v2/MATH Level 5": 0.3882, - "hfopenllm_v2/GPQA": 0.3633, - "hfopenllm_v2/MUSR": 0.4754, - "hfopenllm_v2/MMLU-PRO": 0.5105 - } - }, - { - "id": "Triangle104/Porpoise-R1-Llama3.2-3b", - "name": "Porpoise-R1-Llama3.2-3b", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4352, - "hfopenllm_v2/BBH": 0.3824, - "hfopenllm_v2/MATH Level 5": 0.0423, - "hfopenllm_v2/GPQA": 0.2668, - "hfopenllm_v2/MUSR": 0.3576, - "hfopenllm_v2/MMLU-PRO": 0.2117 - } - }, - { - "id": "Triangle104/Q2.5-14B-Instruct-1M-Harmony", - "name": "Q2.5-14B-Instruct-1M-Harmony", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5986, - "hfopenllm_v2/BBH": 0.6339, - "hfopenllm_v2/MATH Level 5": 0.3769, - "hfopenllm_v2/GPQA": 0.375, - "hfopenllm_v2/MUSR": 0.4795, - "hfopenllm_v2/MMLU-PRO": 0.5075 - } - }, - { - "id": "Triangle104/Q2.5-AthensCOT", - "name": "Q2.5-AthensCOT", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4573, - "hfopenllm_v2/BBH": 0.5542, - "hfopenllm_v2/MATH Level 5": 0.2915, - "hfopenllm_v2/GPQA": 0.3003, - "hfopenllm_v2/MUSR": 0.4578, - "hfopenllm_v2/MMLU-PRO": 0.4379 - } - }, - { - "id": "Triangle104/Q2.5-CodeR1-3B", - "name": "Q2.5-CodeR1-3B", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3588, - "hfopenllm_v2/BBH": 0.4661, - "hfopenllm_v2/MATH Level 5": 0.1639, - "hfopenllm_v2/GPQA": 0.3037, - "hfopenllm_v2/MUSR": 0.4315, - "hfopenllm_v2/MMLU-PRO": 0.2979 - } - }, - { - "id": "Triangle104/Q2.5-EVACOT-7b", - "name": "Q2.5-EVACOT-7b", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5784, - "hfopenllm_v2/BBH": 0.5506, - "hfopenllm_v2/MATH Level 5": 0.2825, - "hfopenllm_v2/GPQA": 0.318, - "hfopenllm_v2/MUSR": 0.4499, - "hfopenllm_v2/MMLU-PRO": 0.4331 - } - }, - { - "id": "Triangle104/Q2.5-EvaHumane-RP", - "name": "Q2.5-EvaHumane-RP", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3676, - "hfopenllm_v2/BBH": 0.5328, - "hfopenllm_v2/MATH Level 5": 0.2923, - "hfopenllm_v2/GPQA": 0.3188, - "hfopenllm_v2/MUSR": 0.4276, - "hfopenllm_v2/MMLU-PRO": 0.4412 - } - }, - { - "id": "Triangle104/Q2.5-Humane-RP", - "name": "Q2.5-Humane-RP", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4412, - "hfopenllm_v2/BBH": 0.5649, - "hfopenllm_v2/MATH Level 5": 0.3391, - "hfopenllm_v2/GPQA": 0.3188, - "hfopenllm_v2/MUSR": 0.4528, - "hfopenllm_v2/MMLU-PRO": 0.4492 - } - }, - { - "id": "Triangle104/Q2.5-Instruct-1M_Harmony", - "name": "Q2.5-Instruct-1M_Harmony", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6038, - "hfopenllm_v2/BBH": 0.5373, - "hfopenllm_v2/MATH Level 5": 0.3323, - "hfopenllm_v2/GPQA": 0.323, - "hfopenllm_v2/MUSR": 0.4688, - "hfopenllm_v2/MMLU-PRO": 0.4366 - } - }, - { - "id": "Triangle104/Q2.5-R1-3B", - "name": "Q2.5-R1-3B", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4214, - "hfopenllm_v2/BBH": 0.4812, - "hfopenllm_v2/MATH Level 5": 0.2674, - "hfopenllm_v2/GPQA": 0.3096, - "hfopenllm_v2/MUSR": 0.432, - "hfopenllm_v2/MMLU-PRO": 0.3813 - } - }, - { - "id": "Triangle104/Q2.5-R1-7B", - "name": "Q2.5-R1-7B", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1346, - "hfopenllm_v2/BBH": 0.3007, - "hfopenllm_v2/MATH Level 5": 0.0166, - "hfopenllm_v2/GPQA": 0.2525, - "hfopenllm_v2/MUSR": 0.3607, - "hfopenllm_v2/MMLU-PRO": 0.118 - } - }, - { - "id": "Triangle104/Robo-Gutenberg_V1.0", - "name": "Robo-Gutenberg_V1.0", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6008, - "hfopenllm_v2/BBH": 0.6537, - "hfopenllm_v2/MATH Level 5": 0.4562, - "hfopenllm_v2/GPQA": 0.3859, - "hfopenllm_v2/MUSR": 0.4744, - "hfopenllm_v2/MMLU-PRO": 0.5391 - } - }, - { - "id": "Triangle104/Rocinante-Prism_V2.0", - "name": "Rocinante-Prism_V2.0", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2616, - "hfopenllm_v2/BBH": 0.5361, - "hfopenllm_v2/MATH Level 5": 0.111, - "hfopenllm_v2/GPQA": 0.3205, - "hfopenllm_v2/MUSR": 0.445, - "hfopenllm_v2/MMLU-PRO": 0.364 - } - }, - { - "id": "Triangle104/Rocinante-Prism_V2.1", - "name": "Rocinante-Prism_V2.1", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2558, - "hfopenllm_v2/BBH": 0.5333, - "hfopenllm_v2/MATH Level 5": 0.1125, - "hfopenllm_v2/GPQA": 0.3196, - "hfopenllm_v2/MUSR": 0.449, - "hfopenllm_v2/MMLU-PRO": 0.3651 - } - }, - { - "id": "Triangle104/RomboHermes3-R1-Llama3.2-3b", - "name": "RomboHermes3-R1-Llama3.2-3b", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3007, - "hfopenllm_v2/BBH": 0.4264, - "hfopenllm_v2/MATH Level 5": 0.0816, - "hfopenllm_v2/GPQA": 0.2836, - "hfopenllm_v2/MUSR": 0.3657, - "hfopenllm_v2/MMLU-PRO": 0.2957 - } - }, - { - "id": "Triangle104/Rombos-Novasky-7B_V1c", - "name": "Rombos-Novasky-7B_V1c", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.408, - "hfopenllm_v2/BBH": 0.4349, - "hfopenllm_v2/MATH Level 5": 0.0853, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.4465, - "hfopenllm_v2/MMLU-PRO": 0.2738 - } - }, - { - "id": "Triangle104/Set-70b", - "name": "Set-70b", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7643, - "hfopenllm_v2/BBH": 0.7014, - "hfopenllm_v2/MATH Level 5": 0.364, - "hfopenllm_v2/GPQA": 0.4463, - "hfopenllm_v2/MUSR": 0.4696, - "hfopenllm_v2/MMLU-PRO": 0.5442 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/trthminh1112.json b/data/developers/trthminh1112.json deleted file mode 100644 index 8775717a53c5fdbd88894639f6139f7ec78bafc4..0000000000000000000000000000000000000000 --- a/data/developers/trthminh1112.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "trthminh1112", - "models": [ - { - "id": "trthminh1112/autotrain-llama32-1b-finetune", - "name": "autotrain-llama32-1b-finetune", - "developer": "trthminh1112", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1769, - "hfopenllm_v2/BBH": 0.2996, - "hfopenllm_v2/MATH Level 5": 0.0151, - "hfopenllm_v2/GPQA": 0.2567, - "hfopenllm_v2/MUSR": 0.3513, - "hfopenllm_v2/MMLU-PRO": 0.1099 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/tsunami-th.json b/data/developers/tsunami-th.json deleted file mode 100644 index 64c5b78b88f5bfca2f52ae7bd367015c771d5caf..0000000000000000000000000000000000000000 --- a/data/developers/tsunami-th.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "developer": "Tsunami-th", - "models": [ - { - "id": "Tsunami-th/Tsunami-0.5-7B-Instruct", - "name": "Tsunami-0.5-7B-Instruct", - "developer": "Tsunami-th", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.74, - "hfopenllm_v2/BBH": 0.5524, - "hfopenllm_v2/MATH Level 5": 0.5045, - "hfopenllm_v2/GPQA": 0.3087, - "hfopenllm_v2/MUSR": 0.4257, - "hfopenllm_v2/MMLU-PRO": 0.4413 - } - }, - { - "id": "Tsunami-th/Tsunami-0.5x-7B-Instruct", - "name": "Tsunami-0.5x-7B-Instruct", - "developer": "Tsunami-th", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7099, - "hfopenllm_v2/BBH": 0.5593, - "hfopenllm_v2/MATH Level 5": 0.4207, - "hfopenllm_v2/GPQA": 0.3146, - "hfopenllm_v2/MUSR": 0.4667, - "hfopenllm_v2/MMLU-PRO": 0.4458 - } - }, - { - "id": "Tsunami-th/Tsunami-1.0-14B-Instruct", - "name": "Tsunami-1.0-14B-Instruct", - "developer": "Tsunami-th", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7829, - "hfopenllm_v2/BBH": 0.6439, - "hfopenllm_v2/MATH Level 5": 0.4585, - "hfopenllm_v2/GPQA": 0.3565, - "hfopenllm_v2/MUSR": 0.4459, - "hfopenllm_v2/MMLU-PRO": 0.5249 - } - }, - { - "id": "Tsunami-th/Tsunami-1.0-7B-Instruct", - "name": "Tsunami-1.0-7B-Instruct", - "developer": "Tsunami-th", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7309, - "hfopenllm_v2/BBH": 0.5491, - "hfopenllm_v2/MATH Level 5": 0.4335, - "hfopenllm_v2/GPQA": 0.3129, - "hfopenllm_v2/MUSR": 0.4493, - "hfopenllm_v2/MMLU-PRO": 0.4424 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/tttxxx01.json b/data/developers/tttxxx01.json deleted file mode 100644 index bb862daad6569fae6d4762f1274132ef4b4a48d3..0000000000000000000000000000000000000000 --- a/data/developers/tttxxx01.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "TTTXXX01", - "models": [ - { - "id": "TTTXXX01/Mistral-7B-Base-SimPO2-5e-7", - "name": "Mistral-7B-Base-SimPO2-5e-7", - "developer": "TTTXXX01", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4392, - "hfopenllm_v2/BBH": 0.432, - "hfopenllm_v2/MATH Level 5": 0.0264, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.3604, - "hfopenllm_v2/MMLU-PRO": 0.2766 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/tugstugi.json b/data/developers/tugstugi.json deleted file mode 100644 index 7d7f6df5a055f9cd8d1ca8265a8df27a0c2b2174..0000000000000000000000000000000000000000 --- a/data/developers/tugstugi.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "tugstugi", - "models": [ - { - "id": "tugstugi/Qwen2.5-7B-Instruct-QwQ-v0.1", - "name": "Qwen2.5-7B-Instruct-QwQ-v0.1", - "developer": "tugstugi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6017, - "hfopenllm_v2/BBH": 0.5101, - "hfopenllm_v2/MATH Level 5": 0.3814, - "hfopenllm_v2/GPQA": 0.2685, - "hfopenllm_v2/MUSR": 0.3794, - "hfopenllm_v2/MMLU-PRO": 0.4081 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/ucla-agi.json b/data/developers/ucla-agi.json deleted file mode 100644 index 93fb82e5b4d9ef216b964e7c543f4f7a063aa502..0000000000000000000000000000000000000000 --- a/data/developers/ucla-agi.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "developer": "UCLA-AGI", - "models": [ - { - "id": "UCLA-AGI/Gemma-2-9B-It-SPPO-Iter1", - "name": "Gemma-2-9B-It-SPPO-Iter1", - "developer": "UCLA-AGI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3082, - "hfopenllm_v2/BBH": 0.5969, - "hfopenllm_v2/MATH Level 5": 0.0899, - "hfopenllm_v2/GPQA": 0.3364, - "hfopenllm_v2/MUSR": 0.4099, - "hfopenllm_v2/MMLU-PRO": 0.3907 - } - }, - { - "id": "UCLA-AGI/Gemma-2-9B-It-SPPO-Iter2", - "name": "Gemma-2-9B-It-SPPO-Iter2", - "developer": "UCLA-AGI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.31, - "hfopenllm_v2/BBH": 0.599, - "hfopenllm_v2/MATH Level 5": 0.0808, - "hfopenllm_v2/GPQA": 0.3347, - "hfopenllm_v2/MUSR": 0.4139, - "hfopenllm_v2/MMLU-PRO": 0.387 - } - }, - { - "id": "UCLA-AGI/Gemma-2-9B-It-SPPO-Iter3", - "name": "Gemma-2-9B-It-SPPO-Iter3", - "developer": "UCLA-AGI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3167, - "hfopenllm_v2/BBH": 0.6007, - "hfopenllm_v2/MATH Level 5": 0.071, - "hfopenllm_v2/GPQA": 0.3389, - "hfopenllm_v2/MUSR": 0.4166, - "hfopenllm_v2/MMLU-PRO": 0.3826 - } - }, - { - "id": "UCLA-AGI/Llama-3-Instruct-8B-SPPO-Iter1", - "name": "Llama-3-Instruct-8B-SPPO-Iter1", - "developer": "UCLA-AGI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7299, - "hfopenllm_v2/BBH": 0.5058, - "hfopenllm_v2/MATH Level 5": 0.1148, - "hfopenllm_v2/GPQA": 0.2676, - "hfopenllm_v2/MUSR": 0.3568, - "hfopenllm_v2/MMLU-PRO": 0.3711 - } - }, - { - "id": "UCLA-AGI/Llama-3-Instruct-8B-SPPO-Iter2", - "name": "Llama-3-Instruct-8B-SPPO-Iter2", - "developer": "UCLA-AGI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6989, - "hfopenllm_v2/BBH": 0.5089, - "hfopenllm_v2/MATH Level 5": 0.1035, - "hfopenllm_v2/GPQA": 0.2668, - "hfopenllm_v2/MUSR": 0.3594, - "hfopenllm_v2/MMLU-PRO": 0.3692 - } - }, - { - "id": "UCLA-AGI/Llama-3-Instruct-8B-SPPO-Iter3", - "name": "Llama-3-Instruct-8B-SPPO-Iter3", - "developer": "UCLA-AGI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6703, - "hfopenllm_v2/BBH": 0.5076, - "hfopenllm_v2/MATH Level 5": 0.0718, - "hfopenllm_v2/GPQA": 0.2651, - "hfopenllm_v2/MUSR": 0.3647, - "hfopenllm_v2/MMLU-PRO": 0.3658 - } - }, - { - "id": "UCLA-AGI/Mistral7B-PairRM-SPPO", - "name": "Mistral7B-PairRM-SPPO", - "developer": "UCLA-AGI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4355, - "hfopenllm_v2/BBH": 0.4439, - "hfopenllm_v2/MATH Level 5": 0.031, - "hfopenllm_v2/GPQA": 0.281, - "hfopenllm_v2/MUSR": 0.3965, - "hfopenllm_v2/MMLU-PRO": 0.2621 - } - }, - { - "id": "UCLA-AGI/Mistral7B-PairRM-SPPO-Iter1", - "name": "Mistral7B-PairRM-SPPO-Iter1", - "developer": "UCLA-AGI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5047, - "hfopenllm_v2/BBH": 0.4468, - "hfopenllm_v2/MATH Level 5": 0.0249, - "hfopenllm_v2/GPQA": 0.2836, - "hfopenllm_v2/MUSR": 0.3992, - "hfopenllm_v2/MMLU-PRO": 0.2695 - } - }, - { - "id": "UCLA-AGI/Mistral7B-PairRM-SPPO-Iter2", - "name": "Mistral7B-PairRM-SPPO-Iter2", - "developer": "UCLA-AGI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4446, - "hfopenllm_v2/BBH": 0.4466, - "hfopenllm_v2/MATH Level 5": 0.0219, - "hfopenllm_v2/GPQA": 0.2886, - "hfopenllm_v2/MUSR": 0.4085, - "hfopenllm_v2/MMLU-PRO": 0.2677 - } - }, - { - "id": "UCLA-AGI/Mistral7B-PairRM-SPPO-Iter3", - "name": "Mistral7B-PairRM-SPPO-Iter3", - "developer": "UCLA-AGI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4351, - "hfopenllm_v2/BBH": 0.4397, - "hfopenllm_v2/MATH Level 5": 0.0234, - "hfopenllm_v2/GPQA": 0.2752, - "hfopenllm_v2/MUSR": 0.4071, - "hfopenllm_v2/MMLU-PRO": 0.2658 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/uiuc-oumi.json b/data/developers/uiuc-oumi.json deleted file mode 100644 index b7d6f00e6d487724a78b74c77f9119817eef1255..0000000000000000000000000000000000000000 --- a/data/developers/uiuc-oumi.json +++ /dev/null @@ -1,85 +0,0 @@ -{ - "developer": "uiuc-oumi", - "models": [ - { - "id": "uiuc-oumi/coalm-70b", - "name": "CoALM-70B", - "developer": "uiuc-oumi", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 74.0, - "bfcl/bfcl.overall.overall_accuracy": 27.99, - "bfcl/bfcl.overall.total_cost_usd": 19.89, - "bfcl/bfcl.overall.latency_mean_s": 16.22, - "bfcl/bfcl.overall.latency_std_s": 59.91, - "bfcl/bfcl.overall.latency_p95_s": 36.0, - "bfcl/bfcl.non_live.ast_accuracy": 83.44, - "bfcl/bfcl.non_live.simple_ast_accuracy": 70.25, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 92.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 88.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 83.0, - "bfcl/bfcl.live.live_accuracy": 67.28, - "bfcl/bfcl.live.live_simple_ast_accuracy": 70.54, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 66.57, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 68.75, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 62.5, - "bfcl/bfcl.multi_turn.accuracy": 10.62, - "bfcl/bfcl.multi_turn.base_accuracy": 11.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 14.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 9.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 8.5, - "bfcl/bfcl.web_search.accuracy": 0.0, - "bfcl/bfcl.web_search.base_accuracy": 0.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 0.0, - "bfcl/bfcl.memory.accuracy": 5.81, - "bfcl/bfcl.memory.kv_accuracy": 9.03, - "bfcl/bfcl.memory.vector_accuracy": 5.16, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 3.23, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 93.75, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 85.65, - "bfcl/bfcl.format_sensitivity.max_delta": 72.0, - "bfcl/bfcl.format_sensitivity.stddev": 27.76 - } - }, - { - "id": "uiuc-oumi/coalm-8b", - "name": "CoALM-8B", - "developer": "uiuc-oumi", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 84.0, - "bfcl/bfcl.overall.overall_accuracy": 26.81, - "bfcl/bfcl.overall.total_cost_usd": 25.33, - "bfcl/bfcl.overall.latency_mean_s": 20.36, - "bfcl/bfcl.overall.latency_std_s": 73.74, - "bfcl/bfcl.overall.latency_p95_s": 138.04, - "bfcl/bfcl.non_live.ast_accuracy": 84.87, - "bfcl/bfcl.non_live.simple_ast_accuracy": 69.5, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 93.5, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 88.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 88.5, - "bfcl/bfcl.live.live_accuracy": 66.77, - "bfcl/bfcl.live.live_simple_ast_accuracy": 70.54, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 66.19, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 62.5, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 54.17, - "bfcl/bfcl.multi_turn.accuracy": 8.0, - "bfcl/bfcl.multi_turn.base_accuracy": 10.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 7.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 8.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 7.0, - "bfcl/bfcl.web_search.accuracy": 0.0, - "bfcl/bfcl.web_search.base_accuracy": 0.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 0.0, - "bfcl/bfcl.memory.accuracy": 2.8, - "bfcl/bfcl.memory.kv_accuracy": 3.23, - "bfcl/bfcl.memory.vector_accuracy": 3.87, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 1.29, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 87.5, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 86.9, - "bfcl/bfcl.format_sensitivity.max_delta": 79.0, - "bfcl/bfcl.format_sensitivity.stddev": 34.18 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/ukzexecution.json b/data/developers/ukzexecution.json deleted file mode 100644 index 4bcbd0d70ea224adc75713c79f642ffd360ec781..0000000000000000000000000000000000000000 --- a/data/developers/ukzexecution.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "UKzExecution", - "models": [ - { - "id": "UKzExecution/LlamaExecutor-8B-3.0.5", - "name": "LlamaExecutor-8B-3.0.5", - "developer": "UKzExecution", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7403, - "hfopenllm_v2/BBH": 0.5006, - "hfopenllm_v2/MATH Level 5": 0.102, - "hfopenllm_v2/GPQA": 0.2559, - "hfopenllm_v2/MUSR": 0.3754, - "hfopenllm_v2/MMLU-PRO": 0.3625 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/unbabel.json b/data/developers/unbabel.json deleted file mode 100644 index 5a60588fe344e5c7316edcb47d8d3aa47ea2f5c1..0000000000000000000000000000000000000000 --- a/data/developers/unbabel.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "Unbabel", - "models": [ - { - "id": "Unbabel/TowerInstruct-Mistral-7B-v0.2", - "name": "TowerInstruct-Mistral-7B-v0.2", - "developer": "Unbabel", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2843, - "hfopenllm_v2/BBH": 0.3882, - "hfopenllm_v2/MATH Level 5": 0.0204, - "hfopenllm_v2/GPQA": 0.2475, - "hfopenllm_v2/MUSR": 0.4522, - "hfopenllm_v2/MMLU-PRO": 0.1968 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/undi95.json b/data/developers/undi95.json deleted file mode 100644 index f448e36f91def4ad862286f0aed2b6c5faea3274..0000000000000000000000000000000000000000 --- a/data/developers/undi95.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "Undi95", - "models": [ - { - "id": "Undi95/MG-FinalMix-72B", - "name": "MG-FinalMix-72B", - "developer": "Undi95", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8014, - "hfopenllm_v2/BBH": 0.6973, - "hfopenllm_v2/MATH Level 5": 0.3973, - "hfopenllm_v2/GPQA": 0.3851, - "hfopenllm_v2/MUSR": 0.4823, - "hfopenllm_v2/MMLU-PRO": 0.5427 - } - }, - { - "id": "Undi95/Phi4-abliterated", - "name": "Phi4-abliterated", - "developer": "Undi95", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6618, - "hfopenllm_v2/BBH": 0.6809, - "hfopenllm_v2/MATH Level 5": 0.3701, - "hfopenllm_v2/GPQA": 0.3305, - "hfopenllm_v2/MUSR": 0.4034, - "hfopenllm_v2/MMLU-PRO": 0.5281 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/universalml.json b/data/developers/universalml.json deleted file mode 100644 index 6a875805611a502f473751f7dcb735d0ed736e8b..0000000000000000000000000000000000000000 --- a/data/developers/universalml.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "universalml", - "models": [ - { - "id": "universalml/NepaliGPT-2.0", - "name": "NepaliGPT-2.0", - "developer": "universalml", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0365, - "hfopenllm_v2/BBH": 0.466, - "hfopenllm_v2/MATH Level 5": 0.0045, - "hfopenllm_v2/GPQA": 0.281, - "hfopenllm_v2/MUSR": 0.4657, - "hfopenllm_v2/MMLU-PRO": 0.33 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/unknown.json b/data/developers/unknown.json deleted file mode 100644 index caadd81b81e09e8905405fc2663af88694315b8a..0000000000000000000000000000000000000000 --- a/data/developers/unknown.json +++ /dev/null @@ -1,177 +0,0 @@ -{ - "developer": "unknown", - "models": [ - { - "id": "Anthropic-LM-v4-s3-52B", - "name": "Anthropic-LM v4-s3 52B", - "developer": "unknown", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.78, - "helm_classic/MMLU": 0.481, - "helm_classic/BoolQ": 0.815, - "helm_classic/NarrativeQA": 0.728, - "helm_classic/NaturalQuestions (open-book)": 0.686, - "helm_classic/QuAC": 0.431, - "helm_classic/HellaSwag": 0.807, - "helm_classic/OpenbookQA": 0.558, - "helm_classic/TruthfulQA": 0.368, - "helm_classic/MS MARCO (TREC)": -1.0, - "helm_classic/CNN/DailyMail": 0.154, - "helm_classic/XSUM": 0.134, - "helm_classic/IMDB": 0.934, - "helm_classic/CivilComments": 0.61, - "helm_classic/RAFT": 0.699 - } - }, - { - "id": "Cohere March 2024", - "name": "Cohere March 2024", - "developer": "unknown", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.8511, - "reward-bench/Chat": 0.9469, - "reward-bench/Chat Hard": 0.6513, - "reward-bench/Safety": 0.877, - "reward-bench/Reasoning": 0.9817, - "reward-bench/Prior Sets (0.5 weight)": 0.7458 - } - }, - { - "id": "Cohere May 2024", - "name": "Cohere May 2024", - "developer": "unknown", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.8816, - "reward-bench/Chat": 0.9637, - "reward-bench/Chat Hard": 0.7127, - "reward-bench/Safety": 0.923, - "reward-bench/Reasoning": 0.9768, - "reward-bench/Prior Sets (0.5 weight)": 0.782 - } - }, - { - "id": "gemini-1.5-flash-8b", - "name": "gemini-1.5-flash-8b", - "developer": "unknown", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7601, - "reward-bench/Chat": 0.9441, - "reward-bench/Chat Hard": 0.5987, - "reward-bench/Safety": 0.7399, - "reward-bench/Reasoning": 0.7575 - } - }, - { - "id": "meta-llama/Meta-Llama-3.1-8B", - "name": "Meta Llama 3.1 8B", - "developer": "unknown", - "evaluator_relationship": null, - "benchmark_scores": { - "la_leaderboard/la_leaderboard": 27.04 - } - }, - { - "id": "meta-llama/Meta-Llama-3.1-8B-Instruct", - "name": "Meta Llama 3.1 8B Instruct", - "developer": "unknown", - "evaluator_relationship": null, - "benchmark_scores": { - "la_leaderboard/la_leaderboard": 30.23 - } - }, - { - "id": "unknown/aya-expanse-32b", - "name": "aya-expanse-32b", - "developer": "unknown", - "evaluator_relationship": null, - "benchmark_scores": { - "global-mmlu-lite/Global MMLU Lite": 0.7353, - "global-mmlu-lite/Culturally Sensitive": 0.6891, - "global-mmlu-lite/Culturally Agnostic": 0.7815, - "global-mmlu-lite/Arabic": 0.7425, - "global-mmlu-lite/English": 0.7544, - "global-mmlu-lite/Bengali": 0.7343, - "global-mmlu-lite/German": 0.7425, - "global-mmlu-lite/French": 0.7325, - "global-mmlu-lite/Hindi": 0.7375, - "global-mmlu-lite/Indonesian": 0.7594, - "global-mmlu-lite/Italian": 0.7305, - "global-mmlu-lite/Japanese": 0.7419, - "global-mmlu-lite/Korean": 0.7525, - "global-mmlu-lite/Portuguese": 0.7544, - "global-mmlu-lite/Spanish": 0.7362, - "global-mmlu-lite/Swahili": 0.7071, - "global-mmlu-lite/Yoruba": 0.6942, - "global-mmlu-lite/Chinese": 0.743, - "global-mmlu-lite/Burmese": 0.7025 - } - }, - { - "id": "unknown/granite-4.0-h-small", - "name": "granite-4.0-h-small", - "developer": "unknown", - "evaluator_relationship": null, - "benchmark_scores": { - "global-mmlu-lite/Global MMLU Lite": 0.7503, - "global-mmlu-lite/Culturally Sensitive": 0.7182, - "global-mmlu-lite/Culturally Agnostic": 0.7826, - "global-mmlu-lite/Arabic": 0.7613, - "global-mmlu-lite/English": 0.77, - "global-mmlu-lite/Bengali": 0.7613, - "global-mmlu-lite/German": 0.755, - "global-mmlu-lite/French": 0.7594, - "global-mmlu-lite/Hindi": 0.7575, - "global-mmlu-lite/Indonesian": 0.7614, - "global-mmlu-lite/Italian": 0.7525, - "global-mmlu-lite/Japanese": 0.7406, - "global-mmlu-lite/Korean": 0.7525, - "global-mmlu-lite/Portuguese": 0.757, - "global-mmlu-lite/Spanish": 0.7638, - "global-mmlu-lite/Swahili": 0.7318, - "global-mmlu-lite/Yoruba": 0.6921, - "global-mmlu-lite/Chinese": 0.7475, - "global-mmlu-lite/Burmese": 0.7419 - } - }, - { - "id": "unknown/o4-mini-2025-04-16", - "name": "o4-mini-2025-04-16", - "developer": "unknown", - "evaluator_relationship": null, - "benchmark_scores": { - "global-mmlu-lite/Global MMLU Lite": 0.8705, - "global-mmlu-lite/Culturally Sensitive": 0.8503, - "global-mmlu-lite/Culturally Agnostic": 0.8906, - "global-mmlu-lite/Arabic": 0.865, - "global-mmlu-lite/English": 0.8675, - "global-mmlu-lite/Bengali": 0.8875, - "global-mmlu-lite/German": 0.8775, - "global-mmlu-lite/French": 0.87, - "global-mmlu-lite/Hindi": 0.87, - "global-mmlu-lite/Indonesian": 0.8675, - "global-mmlu-lite/Italian": 0.855, - "global-mmlu-lite/Japanese": 0.885, - "global-mmlu-lite/Korean": 0.88, - "global-mmlu-lite/Portuguese": 0.88, - "global-mmlu-lite/Spanish": 0.855, - "global-mmlu-lite/Swahili": 0.8525, - "global-mmlu-lite/Yoruba": 0.8525, - "global-mmlu-lite/Chinese": 0.89, - "global-mmlu-lite/Burmese": 0.8725 - } - }, - { - "id": "utter-project/EuroLLM-9B", - "name": "EuroLLM 9B", - "developer": "unknown", - "evaluator_relationship": null, - "benchmark_scores": { - "la_leaderboard/la_leaderboard": 25.87 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/unsloth.json b/data/developers/unsloth.json deleted file mode 100644 index 9fce6b255e1d7c1ace5fec08071a66eeeff9c3b7..0000000000000000000000000000000000000000 --- a/data/developers/unsloth.json +++ /dev/null @@ -1,89 +0,0 @@ -{ - "developer": "unsloth", - "models": [ - { - "id": "unsloth/Llama-3.2-1B-Instruct", - "name": "Llama-3.2-1B-Instruct", - "developer": "unsloth", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.581, - "hfopenllm_v2/BBH": 0.3485, - "hfopenllm_v2/MATH Level 5": 0.0823, - "hfopenllm_v2/GPQA": 0.2676, - "hfopenllm_v2/MUSR": 0.3196, - "hfopenllm_v2/MMLU-PRO": 0.1742 - } - }, - { - "id": "unsloth/Llama-3.2-1B-Instruct-no-system-message", - "name": "Llama-3.2-1B-Instruct-no-system-message", - "developer": "unsloth", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.565, - "hfopenllm_v2/BBH": 0.3544, - "hfopenllm_v2/MATH Level 5": 0.0755, - "hfopenllm_v2/GPQA": 0.2727, - "hfopenllm_v2/MUSR": 0.3341, - "hfopenllm_v2/MMLU-PRO": 0.1669 - } - }, - { - "id": "unsloth/Phi-3-mini-4k-instruct", - "name": "Phi-3-mini-4k-instruct", - "developer": "unsloth", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.544, - "hfopenllm_v2/BBH": 0.55, - "hfopenllm_v2/MATH Level 5": 0.1639, - "hfopenllm_v2/GPQA": 0.323, - "hfopenllm_v2/MUSR": 0.4284, - "hfopenllm_v2/MMLU-PRO": 0.4031 - } - }, - { - "id": "unsloth/phi-4", - "name": "phi-4", - "developer": "unsloth", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6882, - "hfopenllm_v2/BBH": 0.6886, - "hfopenllm_v2/MATH Level 5": 0.5, - "hfopenllm_v2/GPQA": 0.3364, - "hfopenllm_v2/MUSR": 0.4114, - "hfopenllm_v2/MMLU-PRO": 0.5378 - } - }, - { - "id": "unsloth/phi-4-bnb-4bit", - "name": "phi-4-bnb-4bit", - "developer": "unsloth", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.673, - "hfopenllm_v2/BBH": 0.677, - "hfopenllm_v2/MATH Level 5": 0.4607, - "hfopenllm_v2/GPQA": 0.3381, - "hfopenllm_v2/MUSR": 0.4007, - "hfopenllm_v2/MMLU-PRO": 0.5256 - } - }, - { - "id": "unsloth/phi-4-unsloth-bnb-4bit", - "name": "phi-4-unsloth-bnb-4bit", - "developer": "unsloth", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6794, - "hfopenllm_v2/BBH": 0.6791, - "hfopenllm_v2/MATH Level 5": 0.4562, - "hfopenllm_v2/GPQA": 0.3364, - "hfopenllm_v2/MUSR": 0.4034, - "hfopenllm_v2/MMLU-PRO": 0.5286 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/upstage.json b/data/developers/upstage.json deleted file mode 100644 index ca8a8f601c091f416d00421e8a147f7e016ea392..0000000000000000000000000000000000000000 --- a/data/developers/upstage.json +++ /dev/null @@ -1,107 +0,0 @@ -{ - "developer": "upstage", - "models": [ - { - "id": "upstage/SOLAR-10.7B-Instruct-v1.0", - "name": "SOLAR-10.7B-Instruct-v1.0", - "developer": "upstage", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4737, - "hfopenllm_v2/BBH": 0.5162, - "hfopenllm_v2/MATH Level 5": 0.0566, - "hfopenllm_v2/GPQA": 0.3087, - "hfopenllm_v2/MUSR": 0.3899, - "hfopenllm_v2/MMLU-PRO": 0.3138, - "reward-bench/Score": 0.7391, - "reward-bench/Chat": 0.8156, - "reward-bench/Chat Hard": 0.6864, - "reward-bench/Safety": 0.8514, - "reward-bench/Reasoning": 0.7252, - "reward-bench/Prior Sets (0.5 weight)": 0.4949 - } - }, - { - "id": "upstage/SOLAR-10.7B-v1.0", - "name": "SOLAR-10.7B-v1.0", - "developer": "upstage", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2421, - "hfopenllm_v2/BBH": 0.5094, - "hfopenllm_v2/MATH Level 5": 0.0264, - "hfopenllm_v2/GPQA": 0.281, - "hfopenllm_v2/MUSR": 0.4372, - "hfopenllm_v2/MMLU-PRO": 0.34 - } - }, - { - "id": "upstage/solar-pro-241126", - "name": "Solar Pro", - "developer": "upstage", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.602, - "helm_lite/NarrativeQA": 0.753, - "helm_lite/NaturalQuestions (closed-book)": 0.297, - "helm_lite/OpenbookQA": 0.922, - "helm_lite/MMLU": 0.679, - "helm_lite/MATH": 0.567, - "helm_lite/GSM8K": 0.871, - "helm_lite/LegalBench": 0.67, - "helm_lite/MedQA": 0.698, - "helm_lite/WMT 2014": 0.169, - "helm_mmlu/MMLU All Subjects": 0.776, - "helm_mmlu/Abstract Algebra": 0.46, - "helm_mmlu/Anatomy": 0.719, - "helm_mmlu/College Physics": 0.559, - "helm_mmlu/Computer Security": 0.82, - "helm_mmlu/Econometrics": 0.605, - "helm_mmlu/Global Facts": 0.5, - "helm_mmlu/Jurisprudence": 0.898, - "helm_mmlu/Philosophy": 0.817, - "helm_mmlu/Professional Psychology": 0.85, - "helm_mmlu/Us Foreign Policy": 0.97, - "helm_mmlu/Astronomy": 0.868, - "helm_mmlu/Business Ethics": 0.8, - "helm_mmlu/Clinical Knowledge": 0.808, - "helm_mmlu/Conceptual Physics": 0.826, - "helm_mmlu/Electrical Engineering": 0.697, - "helm_mmlu/Elementary Mathematics": 0.611, - "helm_mmlu/Formal Logic": 0.579, - "helm_mmlu/High School World History": 0.907, - "helm_mmlu/Human Sexuality": 0.847, - "helm_mmlu/International Law": 0.901, - "helm_mmlu/Logical Fallacies": 0.865, - "helm_mmlu/Machine Learning": 0.616, - "helm_mmlu/Management": 0.864, - "helm_mmlu/Marketing": 0.953, - "helm_mmlu/Medical Genetics": 0.91, - "helm_mmlu/Miscellaneous": 0.888, - "helm_mmlu/Moral Scenarios": 0.811, - "helm_mmlu/Nutrition": 0.859, - "helm_mmlu/Prehistory": 0.867, - "helm_mmlu/Public Relations": 0.764, - "helm_mmlu/Security Studies": 0.82, - "helm_mmlu/Sociology": 0.886, - "helm_mmlu/Virology": 0.572, - "helm_mmlu/World Religions": 0.883, - "helm_mmlu/Mean win rate": 0.462 - } - }, - { - "id": "upstage/solar-pro-preview-instruct", - "name": "solar-pro-preview-instruct", - "developer": "upstage", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8416, - "hfopenllm_v2/BBH": 0.6817, - "hfopenllm_v2/MATH Level 5": 0.2205, - "hfopenllm_v2/GPQA": 0.3708, - "hfopenllm_v2/MUSR": 0.4417, - "hfopenllm_v2/MMLU-PRO": 0.5273 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/utkmst.json b/data/developers/utkmst.json deleted file mode 100644 index 40be9162c0df392098568cdd7f1a63f35dd4f11b..0000000000000000000000000000000000000000 --- a/data/developers/utkmst.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "utkmst", - "models": [ - { - "id": "utkmst/chimera-beta-test2-lora-merged", - "name": "chimera-beta-test2-lora-merged", - "developer": "utkmst", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6054, - "hfopenllm_v2/BBH": 0.4796, - "hfopenllm_v2/MATH Level 5": 0.0952, - "hfopenllm_v2/GPQA": 0.3037, - "hfopenllm_v2/MUSR": 0.4118, - "hfopenllm_v2/MMLU-PRO": 0.2992 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/uukuguy.json b/data/developers/uukuguy.json deleted file mode 100644 index 7d022d1529e04b5afb867ef1b09439a985a98928..0000000000000000000000000000000000000000 --- a/data/developers/uukuguy.json +++ /dev/null @@ -1,103 +0,0 @@ -{ - "developer": "uukuguy", - "models": [ - { - "id": "uukuguy/speechless-code-mistral-7b-v1.0", - "name": "speechless-code-mistral-7b-v1.0", - "developer": "uukuguy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3665, - "hfopenllm_v2/BBH": 0.4572, - "hfopenllm_v2/MATH Level 5": 0.0521, - "hfopenllm_v2/GPQA": 0.2844, - "hfopenllm_v2/MUSR": 0.4502, - "hfopenllm_v2/MMLU-PRO": 0.3146 - } - }, - { - "id": "uukuguy/speechless-codellama-34b-v2.0", - "name": "speechless-codellama-34b-v2.0", - "developer": "uukuguy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4604, - "hfopenllm_v2/BBH": 0.4813, - "hfopenllm_v2/MATH Level 5": 0.0431, - "hfopenllm_v2/GPQA": 0.2693, - "hfopenllm_v2/MUSR": 0.3787, - "hfopenllm_v2/MMLU-PRO": 0.2542 - } - }, - { - "id": "uukuguy/speechless-coder-ds-6.7b", - "name": "speechless-coder-ds-6.7b", - "developer": "uukuguy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2505, - "hfopenllm_v2/BBH": 0.4036, - "hfopenllm_v2/MATH Level 5": 0.0211, - "hfopenllm_v2/GPQA": 0.2643, - "hfopenllm_v2/MUSR": 0.3819, - "hfopenllm_v2/MMLU-PRO": 0.1719 - } - }, - { - "id": "uukuguy/speechless-instruct-mistral-7b-v0.2", - "name": "speechless-instruct-mistral-7b-v0.2", - "developer": "uukuguy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3261, - "hfopenllm_v2/BBH": 0.4607, - "hfopenllm_v2/MATH Level 5": 0.0491, - "hfopenllm_v2/GPQA": 0.2819, - "hfopenllm_v2/MUSR": 0.4902, - "hfopenllm_v2/MMLU-PRO": 0.2902 - } - }, - { - "id": "uukuguy/speechless-llama2-hermes-orca-platypus-wizardlm-13b", - "name": "speechless-llama2-hermes-orca-platypus-wizardlm-13b", - "developer": "uukuguy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4562, - "hfopenllm_v2/BBH": 0.4846, - "hfopenllm_v2/MATH Level 5": 0.0204, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.4655, - "hfopenllm_v2/MMLU-PRO": 0.2559 - } - }, - { - "id": "uukuguy/speechless-mistral-dolphin-orca-platypus-samantha-7b", - "name": "speechless-mistral-dolphin-orca-platypus-samantha-7b", - "developer": "uukuguy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.37, - "hfopenllm_v2/BBH": 0.4983, - "hfopenllm_v2/MATH Level 5": 0.0295, - "hfopenllm_v2/GPQA": 0.2836, - "hfopenllm_v2/MUSR": 0.4361, - "hfopenllm_v2/MMLU-PRO": 0.299 - } - }, - { - "id": "uukuguy/speechless-zephyr-code-functionary-7b", - "name": "speechless-zephyr-code-functionary-7b", - "developer": "uukuguy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2696, - "hfopenllm_v2/BBH": 0.4664, - "hfopenllm_v2/MATH Level 5": 0.0423, - "hfopenllm_v2/GPQA": 0.3003, - "hfopenllm_v2/MUSR": 0.4268, - "hfopenllm_v2/MMLU-PRO": 0.3094 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/v000000.json b/data/developers/v000000.json deleted file mode 100644 index 5e3125570bb166596fe3c396ac640a981eb6699e..0000000000000000000000000000000000000000 --- a/data/developers/v000000.json +++ /dev/null @@ -1,89 +0,0 @@ -{ - "developer": "v000000", - "models": [ - { - "id": "v000000/L3-8B-Stheno-v3.2-abliterated", - "name": "L3-8B-Stheno-v3.2-abliterated", - "developer": "v000000", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6718, - "hfopenllm_v2/BBH": 0.5141, - "hfopenllm_v2/MATH Level 5": 0.0695, - "hfopenllm_v2/GPQA": 0.3096, - "hfopenllm_v2/MUSR": 0.362, - "hfopenllm_v2/MMLU-PRO": 0.3604 - } - }, - { - "id": "v000000/L3.1-Niitorm-8B-DPO-t0.0001", - "name": "L3.1-Niitorm-8B-DPO-t0.0001", - "developer": "v000000", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7689, - "hfopenllm_v2/BBH": 0.5134, - "hfopenllm_v2/MATH Level 5": 0.1624, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.388, - "hfopenllm_v2/MMLU-PRO": 0.3866 - } - }, - { - "id": "v000000/L3.1-Storniitova-8B", - "name": "L3.1-Storniitova-8B", - "developer": "v000000", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7817, - "hfopenllm_v2/BBH": 0.5151, - "hfopenllm_v2/MATH Level 5": 0.1465, - "hfopenllm_v2/GPQA": 0.2894, - "hfopenllm_v2/MUSR": 0.4029, - "hfopenllm_v2/MMLU-PRO": 0.3776 - } - }, - { - "id": "v000000/Qwen2.5-14B-Gutenberg-1e-Delta", - "name": "Qwen2.5-14B-Gutenberg-1e-Delta", - "developer": "v000000", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8045, - "hfopenllm_v2/BBH": 0.6398, - "hfopenllm_v2/MATH Level 5": 0.5264, - "hfopenllm_v2/GPQA": 0.3289, - "hfopenllm_v2/MUSR": 0.4073, - "hfopenllm_v2/MMLU-PRO": 0.493 - } - }, - { - "id": "v000000/Qwen2.5-14B-Gutenberg-Instruct-Slerpeno", - "name": "Qwen2.5-14B-Gutenberg-Instruct-Slerpeno", - "developer": "v000000", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8197, - "hfopenllm_v2/BBH": 0.639, - "hfopenllm_v2/MATH Level 5": 0.5325, - "hfopenllm_v2/GPQA": 0.3314, - "hfopenllm_v2/MUSR": 0.4114, - "hfopenllm_v2/MMLU-PRO": 0.4924 - } - }, - { - "id": "v000000/Qwen2.5-Lumen-14B", - "name": "Qwen2.5-Lumen-14B", - "developer": "v000000", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8064, - "hfopenllm_v2/BBH": 0.6391, - "hfopenllm_v2/MATH Level 5": 0.5363, - "hfopenllm_v2/GPQA": 0.328, - "hfopenllm_v2/MUSR": 0.4114, - "hfopenllm_v2/MMLU-PRO": 0.4903 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/v3n0m.json b/data/developers/v3n0m.json deleted file mode 100644 index 046d2dddb46255aa6e08144f75c10d7203668a12..0000000000000000000000000000000000000000 --- a/data/developers/v3n0m.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "V3N0M", - "models": [ - { - "id": "V3N0M/Jenna-Tiny-2.0", - "name": "Jenna-Tiny-2.0", - "developer": "V3N0M", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2309, - "hfopenllm_v2/BBH": 0.3148, - "hfopenllm_v2/MATH Level 5": 0.0121, - "hfopenllm_v2/GPQA": 0.25, - "hfopenllm_v2/MUSR": 0.3367, - "hfopenllm_v2/MMLU-PRO": 0.1147 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/vagosolutions.json b/data/developers/vagosolutions.json deleted file mode 100644 index 8d73ab72e4adfede7d8d2179c11946e8bf8eb580..0000000000000000000000000000000000000000 --- a/data/developers/vagosolutions.json +++ /dev/null @@ -1,243 +0,0 @@ -{ - "developer": "VAGOsolutions", - "models": [ - { - "id": "VAGOsolutions/Llama-3-SauerkrautLM-70b-Instruct", - "name": "Llama-3-SauerkrautLM-70b-Instruct", - "developer": "VAGOsolutions", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8045, - "hfopenllm_v2/BBH": 0.6663, - "hfopenllm_v2/MATH Level 5": 0.2281, - "hfopenllm_v2/GPQA": 0.328, - "hfopenllm_v2/MUSR": 0.4339, - "hfopenllm_v2/MMLU-PRO": 0.5392 - } - }, - { - "id": "VAGOsolutions/Llama-3-SauerkrautLM-8b-Instruct", - "name": "Llama-3-SauerkrautLM-8b-Instruct", - "developer": "VAGOsolutions", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7445, - "hfopenllm_v2/BBH": 0.4943, - "hfopenllm_v2/MATH Level 5": 0.0665, - "hfopenllm_v2/GPQA": 0.3087, - "hfopenllm_v2/MUSR": 0.4241, - "hfopenllm_v2/MMLU-PRO": 0.3857 - } - }, - { - "id": "VAGOsolutions/Llama-3.1-SauerkrautLM-70b-Instruct", - "name": "Llama-3.1-SauerkrautLM-70b-Instruct", - "developer": "VAGOsolutions", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8656, - "hfopenllm_v2/BBH": 0.7006, - "hfopenllm_v2/MATH Level 5": 0.3693, - "hfopenllm_v2/GPQA": 0.3414, - "hfopenllm_v2/MUSR": 0.4711, - "hfopenllm_v2/MMLU-PRO": 0.5335 - } - }, - { - "id": "VAGOsolutions/Llama-3.1-SauerkrautLM-8b-Instruct", - "name": "Llama-3.1-SauerkrautLM-8b-Instruct", - "developer": "VAGOsolutions", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8017, - "hfopenllm_v2/BBH": 0.5115, - "hfopenllm_v2/MATH Level 5": 0.1941, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.4148, - "hfopenllm_v2/MMLU-PRO": 0.389 - } - }, - { - "id": "VAGOsolutions/SauerkrautLM-1.5b", - "name": "SauerkrautLM-1.5b", - "developer": "VAGOsolutions", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2404, - "hfopenllm_v2/BBH": 0.3704, - "hfopenllm_v2/MATH Level 5": 0.0363, - "hfopenllm_v2/GPQA": 0.271, - "hfopenllm_v2/MUSR": 0.3739, - "hfopenllm_v2/MMLU-PRO": 0.2151 - } - }, - { - "id": "VAGOsolutions/SauerkrautLM-7b-HerO", - "name": "SauerkrautLM-7b-HerO", - "developer": "VAGOsolutions", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5346, - "hfopenllm_v2/BBH": 0.4904, - "hfopenllm_v2/MATH Level 5": 0.0393, - "hfopenllm_v2/GPQA": 0.2727, - "hfopenllm_v2/MUSR": 0.3924, - "hfopenllm_v2/MMLU-PRO": 0.3046 - } - }, - { - "id": "VAGOsolutions/SauerkrautLM-7b-LaserChat", - "name": "SauerkrautLM-7b-LaserChat", - "developer": "VAGOsolutions", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5988, - "hfopenllm_v2/BBH": 0.4543, - "hfopenllm_v2/MATH Level 5": 0.0778, - "hfopenllm_v2/GPQA": 0.3003, - "hfopenllm_v2/MUSR": 0.4148, - "hfopenllm_v2/MMLU-PRO": 0.3305 - } - }, - { - "id": "VAGOsolutions/SauerkrautLM-gemma-2-2b-it", - "name": "SauerkrautLM-gemma-2-2b-it", - "developer": "VAGOsolutions", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1321, - "hfopenllm_v2/BBH": 0.4241, - "hfopenllm_v2/MATH Level 5": 0.0219, - "hfopenllm_v2/GPQA": 0.2727, - "hfopenllm_v2/MUSR": 0.3995, - "hfopenllm_v2/MMLU-PRO": 0.2693 - } - }, - { - "id": "VAGOsolutions/SauerkrautLM-gemma-2-9b-it", - "name": "SauerkrautLM-gemma-2-9b-it", - "developer": "VAGOsolutions", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3024, - "hfopenllm_v2/BBH": 0.6073, - "hfopenllm_v2/MATH Level 5": 0.0838, - "hfopenllm_v2/GPQA": 0.3272, - "hfopenllm_v2/MUSR": 0.4318, - "hfopenllm_v2/MMLU-PRO": 0.4091 - } - }, - { - "id": "VAGOsolutions/SauerkrautLM-Gemma-2b", - "name": "SauerkrautLM-Gemma-2b", - "developer": "VAGOsolutions", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2475, - "hfopenllm_v2/BBH": 0.3416, - "hfopenllm_v2/MATH Level 5": 0.0279, - "hfopenllm_v2/GPQA": 0.2567, - "hfopenllm_v2/MUSR": 0.3676, - "hfopenllm_v2/MMLU-PRO": 0.1469 - } - }, - { - "id": "VAGOsolutions/SauerkrautLM-Gemma-7b", - "name": "SauerkrautLM-Gemma-7b", - "developer": "VAGOsolutions", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3407, - "hfopenllm_v2/BBH": 0.4188, - "hfopenllm_v2/MATH Level 5": 0.0672, - "hfopenllm_v2/GPQA": 0.2861, - "hfopenllm_v2/MUSR": 0.3594, - "hfopenllm_v2/MMLU-PRO": 0.2961 - } - }, - { - "id": "VAGOsolutions/SauerkrautLM-Mixtral-8x7B-Instruct", - "name": "SauerkrautLM-Mixtral-8x7B-Instruct", - "developer": "VAGOsolutions", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5602, - "hfopenllm_v2/BBH": 0.5277, - "hfopenllm_v2/MATH Level 5": 0.0982, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.4204, - "hfopenllm_v2/MMLU-PRO": 0.365 - } - }, - { - "id": "VAGOsolutions/SauerkrautLM-Nemo-12b-Instruct", - "name": "SauerkrautLM-Nemo-12b-Instruct", - "developer": "VAGOsolutions", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6113, - "hfopenllm_v2/BBH": 0.5214, - "hfopenllm_v2/MATH Level 5": 0.1224, - "hfopenllm_v2/GPQA": 0.3096, - "hfopenllm_v2/MUSR": 0.4469, - "hfopenllm_v2/MMLU-PRO": 0.3385 - } - }, - { - "id": "VAGOsolutions/SauerkrautLM-Phi-3-medium", - "name": "SauerkrautLM-Phi-3-medium", - "developer": "VAGOsolutions", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4409, - "hfopenllm_v2/BBH": 0.6433, - "hfopenllm_v2/MATH Level 5": 0.1601, - "hfopenllm_v2/GPQA": 0.3347, - "hfopenllm_v2/MUSR": 0.4845, - "hfopenllm_v2/MMLU-PRO": 0.4665 - } - }, - { - "id": "VAGOsolutions/SauerkrautLM-SOLAR-Instruct", - "name": "SauerkrautLM-SOLAR-Instruct", - "developer": "VAGOsolutions", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4917, - "hfopenllm_v2/BBH": 0.5169, - "hfopenllm_v2/MATH Level 5": 0.0634, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.3965, - "hfopenllm_v2/MMLU-PRO": 0.3183 - } - }, - { - "id": "VAGOsolutions/SauerkrautLM-v2-14b-DPO", - "name": "SauerkrautLM-v2-14b-DPO", - "developer": "VAGOsolutions", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7412, - "hfopenllm_v2/BBH": 0.656, - "hfopenllm_v2/MATH Level 5": 0.3165, - "hfopenllm_v2/GPQA": 0.3196, - "hfopenllm_v2/MUSR": 0.4375, - "hfopenllm_v2/MMLU-PRO": 0.5117 - } - }, - { - "id": "VAGOsolutions/SauerkrautLM-v2-14b-SFT", - "name": "SauerkrautLM-v2-14b-SFT", - "developer": "VAGOsolutions", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6949, - "hfopenllm_v2/BBH": 0.621, - "hfopenllm_v2/MATH Level 5": 0.3285, - "hfopenllm_v2/GPQA": 0.3356, - "hfopenllm_v2/MUSR": 0.4179, - "hfopenllm_v2/MMLU-PRO": 0.5205 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/valiantlabs.json b/data/developers/valiantlabs.json deleted file mode 100644 index bf9fb4afa6f318f8db15eb852084df0656489446..0000000000000000000000000000000000000000 --- a/data/developers/valiantlabs.json +++ /dev/null @@ -1,159 +0,0 @@ -{ - "developer": "ValiantLabs", - "models": [ - { - "id": "ValiantLabs/Llama3-70B-Fireplace", - "name": "Llama3-70B-Fireplace", - "developer": "ValiantLabs", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7774, - "hfopenllm_v2/BBH": 0.6489, - "hfopenllm_v2/MATH Level 5": 0.2145, - "hfopenllm_v2/GPQA": 0.3549, - "hfopenllm_v2/MUSR": 0.4449, - "hfopenllm_v2/MMLU-PRO": 0.4893 - } - }, - { - "id": "ValiantLabs/Llama3-70B-ShiningValiant2", - "name": "Llama3-70B-ShiningValiant2", - "developer": "ValiantLabs", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6122, - "hfopenllm_v2/BBH": 0.6338, - "hfopenllm_v2/MATH Level 5": 0.2077, - "hfopenllm_v2/GPQA": 0.3305, - "hfopenllm_v2/MUSR": 0.4326, - "hfopenllm_v2/MMLU-PRO": 0.4898 - } - }, - { - "id": "ValiantLabs/Llama3.1-70B-ShiningValiant2", - "name": "Llama3.1-70B-ShiningValiant2", - "developer": "ValiantLabs", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5355, - "hfopenllm_v2/BBH": 0.6738, - "hfopenllm_v2/MATH Level 5": 0.2915, - "hfopenllm_v2/GPQA": 0.3926, - "hfopenllm_v2/MUSR": 0.4681, - "hfopenllm_v2/MMLU-PRO": 0.5173 - } - }, - { - "id": "ValiantLabs/Llama3.1-8B-Cobalt", - "name": "Llama3.1-8B-Cobalt", - "developer": "ValiantLabs", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3496, - "hfopenllm_v2/BBH": 0.4947, - "hfopenllm_v2/MATH Level 5": 0.1269, - "hfopenllm_v2/GPQA": 0.3037, - "hfopenllm_v2/MUSR": 0.3959, - "hfopenllm_v2/MMLU-PRO": 0.3644 - } - }, - { - "id": "ValiantLabs/Llama3.1-8B-Enigma", - "name": "Llama3.1-8B-Enigma", - "developer": "ValiantLabs", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2681, - "hfopenllm_v2/BBH": 0.4478, - "hfopenllm_v2/MATH Level 5": 0.0891, - "hfopenllm_v2/GPQA": 0.2878, - "hfopenllm_v2/MUSR": 0.4196, - "hfopenllm_v2/MMLU-PRO": 0.3409 - } - }, - { - "id": "ValiantLabs/Llama3.1-8B-Esper2", - "name": "Llama3.1-8B-Esper2", - "developer": "ValiantLabs", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2567, - "hfopenllm_v2/BBH": 0.447, - "hfopenllm_v2/MATH Level 5": 0.0589, - "hfopenllm_v2/GPQA": 0.2727, - "hfopenllm_v2/MUSR": 0.3561, - "hfopenllm_v2/MMLU-PRO": 0.2904 - } - }, - { - "id": "ValiantLabs/Llama3.1-8B-Fireplace2", - "name": "Llama3.1-8B-Fireplace2", - "developer": "ValiantLabs", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5328, - "hfopenllm_v2/BBH": 0.4613, - "hfopenllm_v2/MATH Level 5": 0.0876, - "hfopenllm_v2/GPQA": 0.2894, - "hfopenllm_v2/MUSR": 0.3367, - "hfopenllm_v2/MMLU-PRO": 0.2424 - } - }, - { - "id": "ValiantLabs/Llama3.1-8B-ShiningValiant2", - "name": "Llama3.1-8B-ShiningValiant2", - "developer": "ValiantLabs", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2678, - "hfopenllm_v2/BBH": 0.4429, - "hfopenllm_v2/MATH Level 5": 0.0521, - "hfopenllm_v2/GPQA": 0.302, - "hfopenllm_v2/MUSR": 0.3959, - "hfopenllm_v2/MMLU-PRO": 0.2927 - } - }, - { - "id": "ValiantLabs/Llama3.2-3B-Enigma", - "name": "Llama3.2-3B-Enigma", - "developer": "ValiantLabs", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2786, - "hfopenllm_v2/BBH": 0.3723, - "hfopenllm_v2/MATH Level 5": 0.0438, - "hfopenllm_v2/GPQA": 0.2617, - "hfopenllm_v2/MUSR": 0.3921, - "hfopenllm_v2/MMLU-PRO": 0.2428 - } - }, - { - "id": "ValiantLabs/Llama3.2-3B-Esper2", - "name": "Llama3.2-3B-Esper2", - "developer": "ValiantLabs", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.275, - "hfopenllm_v2/BBH": 0.3808, - "hfopenllm_v2/MATH Level 5": 0.0363, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.355, - "hfopenllm_v2/MMLU-PRO": 0.2257 - } - }, - { - "id": "ValiantLabs/Llama3.2-3B-ShiningValiant2", - "name": "Llama3.2-3B-ShiningValiant2", - "developer": "ValiantLabs", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2625, - "hfopenllm_v2/BBH": 0.4226, - "hfopenllm_v2/MATH Level 5": 0.0823, - "hfopenllm_v2/GPQA": 0.2802, - "hfopenllm_v2/MUSR": 0.3866, - "hfopenllm_v2/MMLU-PRO": 0.2829 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/vhab10.json b/data/developers/vhab10.json deleted file mode 100644 index 40e6271ce21ffab2cf76fc396d6d50660b8fb800..0000000000000000000000000000000000000000 --- a/data/developers/vhab10.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "developer": "vhab10", - "models": [ - { - "id": "vhab10/llama-3-8b-merged-linear", - "name": "llama-3-8b-merged-linear", - "developer": "vhab10", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5917, - "hfopenllm_v2/BBH": 0.4937, - "hfopenllm_v2/MATH Level 5": 0.0816, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.4191, - "hfopenllm_v2/MMLU-PRO": 0.3704 - } - }, - { - "id": "vhab10/Llama-3.1-8B-Base-Instruct-SLERP", - "name": "Llama-3.1-8B-Base-Instruct-SLERP", - "developer": "vhab10", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2907, - "hfopenllm_v2/BBH": 0.5057, - "hfopenllm_v2/MATH Level 5": 0.1201, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.4011, - "hfopenllm_v2/MMLU-PRO": 0.3621 - } - }, - { - "id": "vhab10/Llama-3.2-Instruct-3B-TIES", - "name": "Llama-3.2-Instruct-3B-TIES", - "developer": "vhab10", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4727, - "hfopenllm_v2/BBH": 0.4332, - "hfopenllm_v2/MATH Level 5": 0.0982, - "hfopenllm_v2/GPQA": 0.2693, - "hfopenllm_v2/MUSR": 0.3497, - "hfopenllm_v2/MMLU-PRO": 0.2916 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/vicgalle.json b/data/developers/vicgalle.json deleted file mode 100644 index 0c6f2105e3f597f030b6d21192c9b14c4770a980..0000000000000000000000000000000000000000 --- a/data/developers/vicgalle.json +++ /dev/null @@ -1,173 +0,0 @@ -{ - "developer": "vicgalle", - "models": [ - { - "id": "vicgalle/CarbonBeagle-11B", - "name": "CarbonBeagle-11B", - "developer": "vicgalle", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5415, - "hfopenllm_v2/BBH": 0.5294, - "hfopenllm_v2/MATH Level 5": 0.0619, - "hfopenllm_v2/GPQA": 0.302, - "hfopenllm_v2/MUSR": 0.402, - "hfopenllm_v2/MMLU-PRO": 0.3276 - } - }, - { - "id": "vicgalle/CarbonBeagle-11B-truthy", - "name": "CarbonBeagle-11B-truthy", - "developer": "vicgalle", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5212, - "hfopenllm_v2/BBH": 0.5348, - "hfopenllm_v2/MATH Level 5": 0.0491, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.374, - "hfopenllm_v2/MMLU-PRO": 0.3357 - } - }, - { - "id": "vicgalle/Configurable-Hermes-2-Pro-Llama-3-8B", - "name": "Configurable-Hermes-2-Pro-Llama-3-8B", - "developer": "vicgalle", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5763, - "hfopenllm_v2/BBH": 0.5055, - "hfopenllm_v2/MATH Level 5": 0.0763, - "hfopenllm_v2/GPQA": 0.297, - "hfopenllm_v2/MUSR": 0.4184, - "hfopenllm_v2/MMLU-PRO": 0.3098 - } - }, - { - "id": "vicgalle/Configurable-Llama-3.1-8B-Instruct", - "name": "Configurable-Llama-3.1-8B-Instruct", - "developer": "vicgalle", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8312, - "hfopenllm_v2/BBH": 0.5045, - "hfopenllm_v2/MATH Level 5": 0.173, - "hfopenllm_v2/GPQA": 0.2743, - "hfopenllm_v2/MUSR": 0.3845, - "hfopenllm_v2/MMLU-PRO": 0.3592 - } - }, - { - "id": "vicgalle/Configurable-Yi-1.5-9B-Chat", - "name": "Configurable-Yi-1.5-9B-Chat", - "developer": "vicgalle", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4323, - "hfopenllm_v2/BBH": 0.5452, - "hfopenllm_v2/MATH Level 5": 0.2047, - "hfopenllm_v2/GPQA": 0.3431, - "hfopenllm_v2/MUSR": 0.4271, - "hfopenllm_v2/MMLU-PRO": 0.4015 - } - }, - { - "id": "vicgalle/ConfigurableBeagle-11B", - "name": "ConfigurableBeagle-11B", - "developer": "vicgalle", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5834, - "hfopenllm_v2/BBH": 0.5287, - "hfopenllm_v2/MATH Level 5": 0.0431, - "hfopenllm_v2/GPQA": 0.302, - "hfopenllm_v2/MUSR": 0.3953, - "hfopenllm_v2/MMLU-PRO": 0.3374 - } - }, - { - "id": "vicgalle/ConfigurableHermes-7B", - "name": "ConfigurableHermes-7B", - "developer": "vicgalle", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5411, - "hfopenllm_v2/BBH": 0.4573, - "hfopenllm_v2/MATH Level 5": 0.0476, - "hfopenllm_v2/GPQA": 0.2768, - "hfopenllm_v2/MUSR": 0.4057, - "hfopenllm_v2/MMLU-PRO": 0.3025 - } - }, - { - "id": "vicgalle/ConfigurableSOLAR-10.7B", - "name": "ConfigurableSOLAR-10.7B", - "developer": "vicgalle", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.51, - "hfopenllm_v2/BBH": 0.4867, - "hfopenllm_v2/MATH Level 5": 0.0665, - "hfopenllm_v2/GPQA": 0.2987, - "hfopenllm_v2/MUSR": 0.3805, - "hfopenllm_v2/MMLU-PRO": 0.3173 - } - }, - { - "id": "vicgalle/Humanish-RP-Llama-3.1-8B", - "name": "Humanish-RP-Llama-3.1-8B", - "developer": "vicgalle", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6669, - "hfopenllm_v2/BBH": 0.51, - "hfopenllm_v2/MATH Level 5": 0.1518, - "hfopenllm_v2/GPQA": 0.2869, - "hfopenllm_v2/MUSR": 0.3952, - "hfopenllm_v2/MMLU-PRO": 0.3477 - } - }, - { - "id": "vicgalle/Merge-Mistral-Prometheus-7B", - "name": "Merge-Mistral-Prometheus-7B", - "developer": "vicgalle", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4848, - "hfopenllm_v2/BBH": 0.4201, - "hfopenllm_v2/MATH Level 5": 0.0181, - "hfopenllm_v2/GPQA": 0.2634, - "hfopenllm_v2/MUSR": 0.41, - "hfopenllm_v2/MMLU-PRO": 0.2717 - } - }, - { - "id": "vicgalle/Merge-Mixtral-Prometheus-8x7B", - "name": "Merge-Mixtral-Prometheus-8x7B", - "developer": "vicgalle", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5744, - "hfopenllm_v2/BBH": 0.5351, - "hfopenllm_v2/MATH Level 5": 0.0929, - "hfopenllm_v2/GPQA": 0.3087, - "hfopenllm_v2/MUSR": 0.4098, - "hfopenllm_v2/MMLU-PRO": 0.3684 - } - }, - { - "id": "vicgalle/Roleplay-Llama-3-8B", - "name": "Roleplay-Llama-3-8B", - "developer": "vicgalle", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.732, - "hfopenllm_v2/BBH": 0.5012, - "hfopenllm_v2/MATH Level 5": 0.0914, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.3529, - "hfopenllm_v2/MMLU-PRO": 0.3708 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/viettelsecurity-ai.json b/data/developers/viettelsecurity-ai.json deleted file mode 100644 index c998c562c115539194ae54bb33f2621840449fa1..0000000000000000000000000000000000000000 --- a/data/developers/viettelsecurity-ai.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "viettelsecurity-ai", - "models": [ - { - "id": "viettelsecurity-ai/security-llama3.2-3b", - "name": "security-llama3.2-3b", - "developer": "viettelsecurity-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5909, - "hfopenllm_v2/BBH": 0.4401, - "hfopenllm_v2/MATH Level 5": 0.1261, - "hfopenllm_v2/GPQA": 0.2743, - "hfopenllm_v2/MUSR": 0.3379, - "hfopenllm_v2/MMLU-PRO": 0.2837 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/vihangd.json b/data/developers/vihangd.json deleted file mode 100644 index 436df44fa5738837b462253bc3008a4ec60e02bb..0000000000000000000000000000000000000000 --- a/data/developers/vihangd.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "vihangd", - "models": [ - { - "id": "vihangd/smart-dan-sft-v0.1", - "name": "smart-dan-sft-v0.1", - "developer": "vihangd", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1576, - "hfopenllm_v2/BBH": 0.3062, - "hfopenllm_v2/MATH Level 5": 0.0098, - "hfopenllm_v2/GPQA": 0.255, - "hfopenllm_v2/MUSR": 0.3502, - "hfopenllm_v2/MMLU-PRO": 0.1142 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/vikhrmodels.json b/data/developers/vikhrmodels.json deleted file mode 100644 index 5cc67a891b263df5a424866528225e042c9fa782..0000000000000000000000000000000000000000 --- a/data/developers/vikhrmodels.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "Vikhrmodels", - "models": [ - { - "id": "Vikhrmodels/Vikhr-Llama3.1-8B-Instruct-R-21-09-24", - "name": "Vikhr-Llama3.1-8B-Instruct-R-21-09-24", - "developer": "Vikhrmodels", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6431, - "hfopenllm_v2/BBH": 0.5272, - "hfopenllm_v2/MATH Level 5": 0.2175, - "hfopenllm_v2/GPQA": 0.245, - "hfopenllm_v2/MUSR": 0.3754, - "hfopenllm_v2/MMLU-PRO": 0.3547 - } - }, - { - "id": "Vikhrmodels/Vikhr-Nemo-12B-Instruct-R-21-09-24", - "name": "Vikhr-Nemo-12B-Instruct-R-21-09-24", - "developer": "Vikhrmodels", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5999, - "hfopenllm_v2/BBH": 0.5212, - "hfopenllm_v2/MATH Level 5": 0.1715, - "hfopenllm_v2/GPQA": 0.2911, - "hfopenllm_v2/MUSR": 0.4073, - "hfopenllm_v2/MMLU-PRO": 0.3398 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/virnect.json b/data/developers/virnect.json deleted file mode 100644 index c4d108ef590950a6696d9775575feeb937623918..0000000000000000000000000000000000000000 --- a/data/developers/virnect.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "VIRNECT", - "models": [ - { - "id": "VIRNECT/llama-3-Korean-8B", - "name": "llama-3-Korean-8B", - "developer": "VIRNECT", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5058, - "hfopenllm_v2/BBH": 0.4908, - "hfopenllm_v2/MATH Level 5": 0.0929, - "hfopenllm_v2/GPQA": 0.271, - "hfopenllm_v2/MUSR": 0.3662, - "hfopenllm_v2/MMLU-PRO": 0.3539 - } - }, - { - "id": "VIRNECT/llama-3-Korean-8B-r-v-0.1", - "name": "llama-3-Korean-8B-r-v-0.1", - "developer": "VIRNECT", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4916, - "hfopenllm_v2/BBH": 0.4806, - "hfopenllm_v2/MATH Level 5": 0.0861, - "hfopenllm_v2/GPQA": 0.2424, - "hfopenllm_v2/MUSR": 0.3675, - "hfopenllm_v2/MMLU-PRO": 0.326 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/voidful.json b/data/developers/voidful.json deleted file mode 100644 index a6de5e1eae715c6d4a98bf6e858459c94a0e37a1..0000000000000000000000000000000000000000 --- a/data/developers/voidful.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "voidful", - "models": [ - { - "id": "voidful/smol-360m-ft", - "name": "smol-360m-ft", - "developer": "voidful", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2013, - "hfopenllm_v2/BBH": 0.3012, - "hfopenllm_v2/MATH Level 5": 0.0083, - "hfopenllm_v2/GPQA": 0.2458, - "hfopenllm_v2/MUSR": 0.3714, - "hfopenllm_v2/MMLU-PRO": 0.1087 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/vonjack.json b/data/developers/vonjack.json deleted file mode 100644 index 7ae9750c00c58a489b48463780a5edf88ec14ed0..0000000000000000000000000000000000000000 --- a/data/developers/vonjack.json +++ /dev/null @@ -1,103 +0,0 @@ -{ - "developer": "vonjack", - "models": [ - { - "id": "vonjack/MobileLLM-125M-HF", - "name": "MobileLLM-125M-HF", - "developer": "vonjack", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2107, - "hfopenllm_v2/BBH": 0.3027, - "hfopenllm_v2/MATH Level 5": 0.0091, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3782, - "hfopenllm_v2/MMLU-PRO": 0.1164 - } - }, - { - "id": "vonjack/Phi-3-mini-4k-instruct-LLaMAfied", - "name": "Phi-3-mini-4k-instruct-LLaMAfied", - "developer": "vonjack", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5787, - "hfopenllm_v2/BBH": 0.5741, - "hfopenllm_v2/MATH Level 5": 0.1382, - "hfopenllm_v2/GPQA": 0.3305, - "hfopenllm_v2/MUSR": 0.3924, - "hfopenllm_v2/MMLU-PRO": 0.3885 - } - }, - { - "id": "vonjack/Phi-3.5-mini-instruct-hermes-fc-json", - "name": "Phi-3.5-mini-instruct-hermes-fc-json", - "developer": "vonjack", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1416, - "hfopenllm_v2/BBH": 0.2975, - "hfopenllm_v2/MATH Level 5": 0.0076, - "hfopenllm_v2/GPQA": 0.2542, - "hfopenllm_v2/MUSR": 0.4041, - "hfopenllm_v2/MMLU-PRO": 0.1139 - } - }, - { - "id": "vonjack/Qwen2.5-Coder-0.5B-Merged", - "name": "Qwen2.5-Coder-0.5B-Merged", - "developer": "vonjack", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.31, - "hfopenllm_v2/BBH": 0.3076, - "hfopenllm_v2/MATH Level 5": 0.0378, - "hfopenllm_v2/GPQA": 0.2534, - "hfopenllm_v2/MUSR": 0.3303, - "hfopenllm_v2/MMLU-PRO": 0.1202 - } - }, - { - "id": "vonjack/SmolLM2-1.7B-Merged", - "name": "SmolLM2-1.7B-Merged", - "developer": "vonjack", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3698, - "hfopenllm_v2/BBH": 0.3587, - "hfopenllm_v2/MATH Level 5": 0.0627, - "hfopenllm_v2/GPQA": 0.2794, - "hfopenllm_v2/MUSR": 0.3408, - "hfopenllm_v2/MMLU-PRO": 0.2048 - } - }, - { - "id": "vonjack/SmolLM2-135M-Merged", - "name": "SmolLM2-135M-Merged", - "developer": "vonjack", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2483, - "hfopenllm_v2/BBH": 0.31, - "hfopenllm_v2/MATH Level 5": 0.0113, - "hfopenllm_v2/GPQA": 0.2383, - "hfopenllm_v2/MUSR": 0.3662, - "hfopenllm_v2/MMLU-PRO": 0.1112 - } - }, - { - "id": "vonjack/SmolLM2-360M-Merged", - "name": "SmolLM2-360M-Merged", - "developer": "vonjack", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3206, - "hfopenllm_v2/BBH": 0.3155, - "hfopenllm_v2/MATH Level 5": 0.0174, - "hfopenllm_v2/GPQA": 0.2559, - "hfopenllm_v2/MUSR": 0.3527, - "hfopenllm_v2/MMLU-PRO": 0.1098 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/w4r10ck.json b/data/developers/w4r10ck.json deleted file mode 100644 index af6efbb76c0ab7ca146aee9c95291c7f48412e41..0000000000000000000000000000000000000000 --- a/data/developers/w4r10ck.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "w4r10ck", - "models": [ - { - "id": "w4r10ck/SOLAR-10.7B-Instruct-v1.0-uncensored", - "name": "SOLAR-10.7B-Instruct-v1.0-uncensored", - "developer": "w4r10ck", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3884, - "hfopenllm_v2/BBH": 0.5302, - "hfopenllm_v2/MATH Level 5": 0.0657, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.4639, - "hfopenllm_v2/MMLU-PRO": 0.3344 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/wanlige.json b/data/developers/wanlige.json deleted file mode 100644 index 463f87ec60baeec83beadbb3c8f277c9fcc94551..0000000000000000000000000000000000000000 --- a/data/developers/wanlige.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "developer": "wanlige", - "models": [ - { - "id": "wanlige/li-14b-v0.4", - "name": "li-14b-v0.4", - "developer": "wanlige", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8133, - "hfopenllm_v2/BBH": 0.6544, - "hfopenllm_v2/MATH Level 5": 0.5574, - "hfopenllm_v2/GPQA": 0.3389, - "hfopenllm_v2/MUSR": 0.446, - "hfopenllm_v2/MMLU-PRO": 0.5167 - } - }, - { - "id": "wanlige/li-14b-v0.4-slerp", - "name": "li-14b-v0.4-slerp", - "developer": "wanlige", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4606, - "hfopenllm_v2/BBH": 0.6587, - "hfopenllm_v2/MATH Level 5": 0.4192, - "hfopenllm_v2/GPQA": 0.4002, - "hfopenllm_v2/MUSR": 0.4768, - "hfopenllm_v2/MMLU-PRO": 0.5372 - } - }, - { - "id": "wanlige/li-14b-v0.4-slerp0.1", - "name": "li-14b-v0.4-slerp0.1", - "developer": "wanlige", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7923, - "hfopenllm_v2/BBH": 0.6572, - "hfopenllm_v2/MATH Level 5": 0.5332, - "hfopenllm_v2/GPQA": 0.3591, - "hfopenllm_v2/MUSR": 0.4207, - "hfopenllm_v2/MMLU-PRO": 0.5294 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/wannaphong.json b/data/developers/wannaphong.json deleted file mode 100644 index 50f8660f474e3b402487a715f4b5797e1c58accf..0000000000000000000000000000000000000000 --- a/data/developers/wannaphong.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "wannaphong", - "models": [ - { - "id": "wannaphong/KhanomTanLLM-Instruct", - "name": "KhanomTanLLM-Instruct", - "developer": "wannaphong", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1621, - "hfopenllm_v2/BBH": 0.3093, - "hfopenllm_v2/MATH Level 5": 0.0136, - "hfopenllm_v2/GPQA": 0.2634, - "hfopenllm_v2/MUSR": 0.3701, - "hfopenllm_v2/MMLU-PRO": 0.1119 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/waqasali1707.json b/data/developers/waqasali1707.json deleted file mode 100644 index 57f8257bc9490ec8162ed63344b5bf450368a234..0000000000000000000000000000000000000000 --- a/data/developers/waqasali1707.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "waqasali1707", - "models": [ - { - "id": "waqasali1707/Beast-Soul-new", - "name": "Beast-Soul-new", - "developer": "waqasali1707", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.503, - "hfopenllm_v2/BBH": 0.5225, - "hfopenllm_v2/MATH Level 5": 0.0702, - "hfopenllm_v2/GPQA": 0.2827, - "hfopenllm_v2/MUSR": 0.4486, - "hfopenllm_v2/MMLU-PRO": 0.3108 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/wave-on-discord.json b/data/developers/wave-on-discord.json deleted file mode 100644 index 40e374e65eb42588d697b5d8299aa8e4008bd0f0..0000000000000000000000000000000000000000 --- a/data/developers/wave-on-discord.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "wave-on-discord", - "models": [ - { - "id": "wave-on-discord/qwent-7b", - "name": "qwent-7b", - "developer": "wave-on-discord", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2015, - "hfopenllm_v2/BBH": 0.4228, - "hfopenllm_v2/MATH Level 5": 0.0038, - "hfopenllm_v2/GPQA": 0.2651, - "hfopenllm_v2/MUSR": 0.3817, - "hfopenllm_v2/MMLU-PRO": 0.1603 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/weathermanj.json b/data/developers/weathermanj.json deleted file mode 100644 index f7cec2ba4e2118205feba545342ba05a9e285a1b..0000000000000000000000000000000000000000 --- a/data/developers/weathermanj.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "developer": "weathermanj", - "models": [ - { - "id": "weathermanj/Menda-3B-500", - "name": "Menda-3B-500", - "developer": "weathermanj", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6353, - "hfopenllm_v2/BBH": 0.4766, - "hfopenllm_v2/MATH Level 5": 0.3724, - "hfopenllm_v2/GPQA": 0.2878, - "hfopenllm_v2/MUSR": 0.3968, - "hfopenllm_v2/MMLU-PRO": 0.3475 - } - }, - { - "id": "weathermanj/Menda-3b-750", - "name": "Menda-3b-750", - "developer": "weathermanj", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6335, - "hfopenllm_v2/BBH": 0.4737, - "hfopenllm_v2/MATH Level 5": 0.3716, - "hfopenllm_v2/GPQA": 0.2878, - "hfopenllm_v2/MUSR": 0.3942, - "hfopenllm_v2/MMLU-PRO": 0.3506 - } - }, - { - "id": "weathermanj/Menda-3b-Optim-100", - "name": "Menda-3b-Optim-100", - "developer": "weathermanj", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6398, - "hfopenllm_v2/BBH": 0.4735, - "hfopenllm_v2/MATH Level 5": 0.3716, - "hfopenllm_v2/GPQA": 0.2894, - "hfopenllm_v2/MUSR": 0.3993, - "hfopenllm_v2/MMLU-PRO": 0.3461 - } - }, - { - "id": "weathermanj/Menda-3b-Optim-200", - "name": "Menda-3b-Optim-200", - "developer": "weathermanj", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6375, - "hfopenllm_v2/BBH": 0.4746, - "hfopenllm_v2/MATH Level 5": 0.3731, - "hfopenllm_v2/GPQA": 0.2827, - "hfopenllm_v2/MUSR": 0.4033, - "hfopenllm_v2/MMLU-PRO": 0.3484 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/wenbopan.json b/data/developers/wenbopan.json deleted file mode 100644 index baa05d5ec0959b065992ea52981ce1b0efee773b..0000000000000000000000000000000000000000 --- a/data/developers/wenbopan.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "wenbopan", - "models": [ - { - "id": "wenbopan/Faro-Yi-9B-DPO", - "name": "wenbopan/Faro-Yi-9B-DPO", - "developer": "wenbopan", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6461, - "reward-bench/Chat": 0.9218, - "reward-bench/Chat Hard": 0.5307, - "reward-bench/Safety": 0.5514, - "reward-bench/Reasoning": 0.5839, - "reward-bench/Prior Sets (0.5 weight)": 0.6395 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/weqweasdas.json b/data/developers/weqweasdas.json deleted file mode 100644 index 53701e6ba9018d055fac2463f132ffbb9e4ba26c..0000000000000000000000000000000000000000 --- a/data/developers/weqweasdas.json +++ /dev/null @@ -1,95 +0,0 @@ -{ - "developer": "weqweasdas", - "models": [ - { - "id": "weqweasdas/hh_rlhf_rm_open_llama_3b", - "name": "weqweasdas/hh_rlhf_rm_open_llama_3b", - "developer": "weqweasdas", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.2498, - "reward-bench/Chat": 0.8184, - "reward-bench/Chat Hard": 0.3728, - "reward-bench/Safety": 0.24, - "reward-bench/Reasoning": 0.3281, - "reward-bench/Prior Sets (0.5 weight)": 0.6564, - "reward-bench/Factuality": 0.3642, - "reward-bench/Precise IF": 0.275, - "reward-bench/Math": 0.3497, - "reward-bench/Focus": 0.2384, - "reward-bench/Ties": 0.0315 - } - }, - { - "id": "weqweasdas/RM-Gemma-2B", - "name": "weqweasdas/RM-Gemma-2B", - "developer": "weqweasdas", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6549, - "reward-bench/Factuality": 0.3705, - "reward-bench/Precise IF": 0.2812, - "reward-bench/Math": 0.4317, - "reward-bench/Safety": 0.4986, - "reward-bench/Focus": 0.2343, - "reward-bench/Ties": 0.1851, - "reward-bench/Chat": 0.9441, - "reward-bench/Chat Hard": 0.4079, - "reward-bench/Reasoning": 0.7637, - "reward-bench/Prior Sets (0.5 weight)": 0.6652 - } - }, - { - "id": "weqweasdas/RM-Gemma-7B", - "name": "weqweasdas/RM-Gemma-7B", - "developer": "weqweasdas", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6967, - "reward-bench/Factuality": 0.4926, - "reward-bench/Precise IF": 0.3937, - "reward-bench/Math": 0.6066, - "reward-bench/Safety": 0.5784, - "reward-bench/Focus": 0.497, - "reward-bench/Ties": 0.4232, - "reward-bench/Chat": 0.9693, - "reward-bench/Chat Hard": 0.4978, - "reward-bench/Reasoning": 0.7362, - "reward-bench/Prior Sets (0.5 weight)": 0.7069 - } - }, - { - "id": "weqweasdas/RM-Gemma-7B-4096", - "name": "weqweasdas/RM-Gemma-7B-4096", - "developer": "weqweasdas", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6922, - "reward-bench/Chat": 0.9497, - "reward-bench/Chat Hard": 0.5022, - "reward-bench/Safety": 0.5608, - "reward-bench/Reasoning": 0.7511, - "reward-bench/Prior Sets (0.5 weight)": 0.7024 - } - }, - { - "id": "weqweasdas/RM-Mistral-7B", - "name": "weqweasdas/RM-Mistral-7B", - "developer": "weqweasdas", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7982, - "reward-bench/Factuality": 0.5937, - "reward-bench/Precise IF": 0.3438, - "reward-bench/Math": 0.5956, - "reward-bench/Safety": 0.8703, - "reward-bench/Focus": 0.7293, - "reward-bench/Ties": 0.6226, - "reward-bench/Chat": 0.9665, - "reward-bench/Chat Hard": 0.6053, - "reward-bench/Reasoning": 0.7736, - "reward-bench/Prior Sets (0.5 weight)": 0.753 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/weyaxi.json b/data/developers/weyaxi.json deleted file mode 100644 index ea911216e69bc46da866d69f16443b70106ab7ab..0000000000000000000000000000000000000000 --- a/data/developers/weyaxi.json +++ /dev/null @@ -1,117 +0,0 @@ -{ - "developer": "Weyaxi", - "models": [ - { - "id": "Weyaxi/Bagel-Hermes-2x34B", - "name": "Bagel-Hermes-2x34B", - "developer": "Weyaxi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5432, - "hfopenllm_v2/BBH": 0.4917, - "hfopenllm_v2/MATH Level 5": 0.0604, - "hfopenllm_v2/GPQA": 0.328, - "hfopenllm_v2/MUSR": 0.4517, - "hfopenllm_v2/MMLU-PRO": 0.4589 - } - }, - { - "id": "Weyaxi/Bagel-Hermes-34B-Slerp", - "name": "Bagel-Hermes-34B-Slerp", - "developer": "Weyaxi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4603, - "hfopenllm_v2/BBH": 0.5922, - "hfopenllm_v2/MATH Level 5": 0.0604, - "hfopenllm_v2/GPQA": 0.3347, - "hfopenllm_v2/MUSR": 0.4622, - "hfopenllm_v2/MMLU-PRO": 0.4703 - } - }, - { - "id": "Weyaxi/Einstein-v4-7B", - "name": "Einstein-v4-7B", - "developer": "Weyaxi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4708, - "hfopenllm_v2/BBH": 0.3849, - "hfopenllm_v2/MATH Level 5": 0.0189, - "hfopenllm_v2/GPQA": 0.2819, - "hfopenllm_v2/MUSR": 0.4682, - "hfopenllm_v2/MMLU-PRO": 0.2259 - } - }, - { - "id": "Weyaxi/Einstein-v6.1-developed-by-Weyaxi-Llama3-8B", - "name": "Einstein-v6.1-developed-by-Weyaxi-Llama3-8B", - "developer": "Weyaxi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3927, - "hfopenllm_v2/BBH": 0.5044, - "hfopenllm_v2/MATH Level 5": 0.0718, - "hfopenllm_v2/GPQA": 0.2735, - "hfopenllm_v2/MUSR": 0.4332, - "hfopenllm_v2/MMLU-PRO": 0.3093 - } - }, - { - "id": "Weyaxi/Einstein-v6.1-Llama3-8B", - "name": "Einstein-v6.1-Llama3-8B", - "developer": "Weyaxi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4568, - "hfopenllm_v2/BBH": 0.5008, - "hfopenllm_v2/MATH Level 5": 0.068, - "hfopenllm_v2/GPQA": 0.2819, - "hfopenllm_v2/MUSR": 0.4213, - "hfopenllm_v2/MMLU-PRO": 0.3131 - } - }, - { - "id": "Weyaxi/Einstein-v7-Qwen2-7B", - "name": "Einstein-v7-Qwen2-7B", - "developer": "Weyaxi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.41, - "hfopenllm_v2/BBH": 0.5161, - "hfopenllm_v2/MATH Level 5": 0.1994, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.44, - "hfopenllm_v2/MMLU-PRO": 0.4096 - } - }, - { - "id": "Weyaxi/Einstein-v8-Llama3.2-1B", - "name": "Einstein-v8-Llama3.2-1B", - "developer": "Weyaxi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1862, - "hfopenllm_v2/BBH": 0.3018, - "hfopenllm_v2/MATH Level 5": 0.0008, - "hfopenllm_v2/GPQA": 0.2584, - "hfopenllm_v2/MUSR": 0.3618, - "hfopenllm_v2/MMLU-PRO": 0.1161 - } - }, - { - "id": "Weyaxi/SauerkrautLM-UNA-SOLAR-Instruct", - "name": "SauerkrautLM-UNA-SOLAR-Instruct", - "developer": "Weyaxi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4573, - "hfopenllm_v2/BBH": 0.5166, - "hfopenllm_v2/MATH Level 5": 0.0461, - "hfopenllm_v2/GPQA": 0.3112, - "hfopenllm_v2/MUSR": 0.3979, - "hfopenllm_v2/MMLU-PRO": 0.3153 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/win10.json b/data/developers/win10.json deleted file mode 100644 index ab266d88ecc7d9801029dcffcb50e3677f52036b..0000000000000000000000000000000000000000 --- a/data/developers/win10.json +++ /dev/null @@ -1,131 +0,0 @@ -{ - "developer": "win10", - "models": [ - { - "id": "win10/ArliAI-RPMax-v1.3-merge-13.3B", - "name": "ArliAI-RPMax-v1.3-merge-13.3B", - "developer": "win10", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3038, - "hfopenllm_v2/BBH": 0.4581, - "hfopenllm_v2/MATH Level 5": 0.0393, - "hfopenllm_v2/GPQA": 0.2743, - "hfopenllm_v2/MUSR": 0.4325, - "hfopenllm_v2/MMLU-PRO": 0.32 - } - }, - { - "id": "win10/Breeze-13B-32k-Instruct-v1_0", - "name": "Breeze-13B-32k-Instruct-v1_0", - "developer": "win10", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3584, - "hfopenllm_v2/BBH": 0.4611, - "hfopenllm_v2/MATH Level 5": 0.0128, - "hfopenllm_v2/GPQA": 0.2643, - "hfopenllm_v2/MUSR": 0.4202, - "hfopenllm_v2/MMLU-PRO": 0.2568 - } - }, - { - "id": "win10/EVA-Norns-Qwen2.5-v0.1", - "name": "EVA-Norns-Qwen2.5-v0.1", - "developer": "win10", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.622, - "hfopenllm_v2/BBH": 0.5072, - "hfopenllm_v2/MATH Level 5": 0.2613, - "hfopenllm_v2/GPQA": 0.2852, - "hfopenllm_v2/MUSR": 0.4045, - "hfopenllm_v2/MMLU-PRO": 0.3425 - } - }, - { - "id": "win10/Llama-3.2-3B-Instruct-24-9-29", - "name": "Llama-3.2-3B-Instruct-24-9-29", - "developer": "win10", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7332, - "hfopenllm_v2/BBH": 0.4614, - "hfopenllm_v2/MATH Level 5": 0.1707, - "hfopenllm_v2/GPQA": 0.2743, - "hfopenllm_v2/MUSR": 0.3555, - "hfopenllm_v2/MMLU-PRO": 0.3228 - } - }, - { - "id": "win10/llama3-13.45b-Instruct", - "name": "llama3-13.45b-Instruct", - "developer": "win10", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4144, - "hfopenllm_v2/BBH": 0.4865, - "hfopenllm_v2/MATH Level 5": 0.0242, - "hfopenllm_v2/GPQA": 0.2584, - "hfopenllm_v2/MUSR": 0.3848, - "hfopenllm_v2/MMLU-PRO": 0.3345 - } - }, - { - "id": "win10/miscii-14b-1M-0128", - "name": "miscii-14b-1M-0128", - "developer": "win10", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4181, - "hfopenllm_v2/BBH": 0.5742, - "hfopenllm_v2/MATH Level 5": 0.4773, - "hfopenllm_v2/GPQA": 0.3826, - "hfopenllm_v2/MUSR": 0.5431, - "hfopenllm_v2/MMLU-PRO": 0.4491 - } - }, - { - "id": "win10/Norns-Qwen2.5-12B", - "name": "Norns-Qwen2.5-12B", - "developer": "win10", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4897, - "hfopenllm_v2/BBH": 0.4619, - "hfopenllm_v2/MATH Level 5": 0.0838, - "hfopenllm_v2/GPQA": 0.2836, - "hfopenllm_v2/MUSR": 0.3555, - "hfopenllm_v2/MMLU-PRO": 0.266 - } - }, - { - "id": "win10/Norns-Qwen2.5-7B", - "name": "Norns-Qwen2.5-7B", - "developer": "win10", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6122, - "hfopenllm_v2/BBH": 0.5073, - "hfopenllm_v2/MATH Level 5": 0.2628, - "hfopenllm_v2/GPQA": 0.2844, - "hfopenllm_v2/MUSR": 0.4085, - "hfopenllm_v2/MMLU-PRO": 0.3413 - } - }, - { - "id": "win10/Qwen2.5-2B-Instruct", - "name": "Qwen2.5-2B-Instruct", - "developer": "win10", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2273, - "hfopenllm_v2/BBH": 0.3706, - "hfopenllm_v2/MATH Level 5": 0.0227, - "hfopenllm_v2/GPQA": 0.2676, - "hfopenllm_v2/MUSR": 0.4378, - "hfopenllm_v2/MMLU-PRO": 0.1934 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/winglian.json b/data/developers/winglian.json deleted file mode 100644 index 6f9a26d92467199e44eea3d2dda2ac615a540348..0000000000000000000000000000000000000000 --- a/data/developers/winglian.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "winglian", - "models": [ - { - "id": "winglian/llama-3-8b-256k-PoSE", - "name": "llama-3-8b-256k-PoSE", - "developer": "winglian", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2909, - "hfopenllm_v2/BBH": 0.3157, - "hfopenllm_v2/MATH Level 5": 0.0196, - "hfopenllm_v2/GPQA": 0.2576, - "hfopenllm_v2/MUSR": 0.3316, - "hfopenllm_v2/MMLU-PRO": 0.1116 - } - }, - { - "id": "winglian/Llama-3-8b-64k-PoSE", - "name": "Llama-3-8b-64k-PoSE", - "developer": "winglian", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2857, - "hfopenllm_v2/BBH": 0.3702, - "hfopenllm_v2/MATH Level 5": 0.0415, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.3396, - "hfopenllm_v2/MMLU-PRO": 0.2467 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/wizardlmteam.json b/data/developers/wizardlmteam.json deleted file mode 100644 index 4c9fb2823b79284575bcc095b3d2c6b47611bd1c..0000000000000000000000000000000000000000 --- a/data/developers/wizardlmteam.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "developer": "WizardLMTeam", - "models": [ - { - "id": "WizardLMTeam/WizardLM-13B-V1.0", - "name": "WizardLM-13B-V1.0", - "developer": "WizardLMTeam", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.185, - "hfopenllm_v2/BBH": 0.2913, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3497, - "hfopenllm_v2/MMLU-PRO": 0.1166 - } - }, - { - "id": "WizardLMTeam/WizardLM-13B-V1.2", - "name": "WizardLM-13B-V1.2", - "developer": "WizardLMTeam", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3392, - "hfopenllm_v2/BBH": 0.4462, - "hfopenllm_v2/MATH Level 5": 0.0189, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.4378, - "hfopenllm_v2/MMLU-PRO": 0.2519 - } - }, - { - "id": "WizardLMTeam/WizardLM-70B-V1.0", - "name": "WizardLM-70B-V1.0", - "developer": "WizardLMTeam", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4951, - "hfopenllm_v2/BBH": 0.559, - "hfopenllm_v2/MATH Level 5": 0.0393, - "hfopenllm_v2/GPQA": 0.2659, - "hfopenllm_v2/MUSR": 0.4391, - "hfopenllm_v2/MMLU-PRO": 0.3447 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/wladastic.json b/data/developers/wladastic.json deleted file mode 100644 index f0bb4a704305145cf952e72fb6f156140ab45ac9..0000000000000000000000000000000000000000 --- a/data/developers/wladastic.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "Wladastic", - "models": [ - { - "id": "Wladastic/Mini-Think-Base-1B", - "name": "Mini-Think-Base-1B", - "developer": "Wladastic", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5588, - "hfopenllm_v2/BBH": 0.3574, - "hfopenllm_v2/MATH Level 5": 0.0733, - "hfopenllm_v2/GPQA": 0.2634, - "hfopenllm_v2/MUSR": 0.3275, - "hfopenllm_v2/MMLU-PRO": 0.1772 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/writer.json b/data/developers/writer.json deleted file mode 100644 index f363f703ef09f6c69b87c319dd6c8d2c2b25d7e0..0000000000000000000000000000000000000000 --- a/data/developers/writer.json +++ /dev/null @@ -1,240 +0,0 @@ -{ - "developer": "writer", - "models": [ - { - "id": "writer/InstructPalmyra-30B", - "name": "InstructPalmyra 30B", - "developer": "writer", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.568, - "helm_classic/MMLU": 0.403, - "helm_classic/BoolQ": 0.751, - "helm_classic/NarrativeQA": 0.496, - "helm_classic/NaturalQuestions (open-book)": 0.682, - "helm_classic/QuAC": 0.433, - "helm_classic/HellaSwag": -1.0, - "helm_classic/OpenbookQA": -1.0, - "helm_classic/TruthfulQA": 0.185, - "helm_classic/MS MARCO (TREC)": -1.0, - "helm_classic/CNN/DailyMail": 0.152, - "helm_classic/XSUM": 0.104, - "helm_classic/IMDB": 0.94, - "helm_classic/CivilComments": 0.555, - "helm_classic/RAFT": 0.652 - } - }, - { - "id": "writer/palmyra-fin", - "name": "Palmyra Fin", - "developer": "writer", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.577, - "helm_capabilities/MMLU-Pro": 0.591, - "helm_capabilities/GPQA": 0.422, - "helm_capabilities/IFEval": 0.793, - "helm_capabilities/WildBench": 0.783, - "helm_capabilities/Omni-MATH": 0.295 - } - }, - { - "id": "writer/palmyra-med", - "name": "Palmyra Med", - "developer": "writer", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.476, - "helm_capabilities/MMLU-Pro": 0.411, - "helm_capabilities/GPQA": 0.368, - "helm_capabilities/IFEval": 0.767, - "helm_capabilities/WildBench": 0.676, - "helm_capabilities/Omni-MATH": 0.156 - } - }, - { - "id": "writer/palmyra-x-004", - "name": "Palmyra-X-004", - "developer": "writer", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.609, - "helm_capabilities/MMLU-Pro": 0.657, - "helm_capabilities/GPQA": 0.395, - "helm_capabilities/IFEval": 0.872, - "helm_capabilities/WildBench": 0.802, - "helm_capabilities/Omni-MATH": 0.32, - "helm_lite/Mean win rate": 0.808, - "helm_lite/NarrativeQA": 0.773, - "helm_lite/NaturalQuestions (closed-book)": 0.457, - "helm_lite/OpenbookQA": 0.926, - "helm_lite/MMLU": 0.739, - "helm_lite/MATH": 0.767, - "helm_lite/GSM8K": 0.905, - "helm_lite/LegalBench": 0.73, - "helm_lite/MedQA": 0.775, - "helm_lite/WMT 2014": 0.203, - "helm_mmlu/MMLU All Subjects": 0.813, - "helm_mmlu/Abstract Algebra": 0.75, - "helm_mmlu/Anatomy": 0.822, - "helm_mmlu/College Physics": 0.647, - "helm_mmlu/Computer Security": 0.82, - "helm_mmlu/Econometrics": 0.684, - "helm_mmlu/Global Facts": 0.62, - "helm_mmlu/Jurisprudence": 0.843, - "helm_mmlu/Philosophy": 0.83, - "helm_mmlu/Professional Psychology": 0.845, - "helm_mmlu/Us Foreign Policy": 0.92, - "helm_mmlu/Astronomy": 0.928, - "helm_mmlu/Business Ethics": 0.76, - "helm_mmlu/Clinical Knowledge": 0.879, - "helm_mmlu/Conceptual Physics": 0.885, - "helm_mmlu/Electrical Engineering": 0.793, - "helm_mmlu/Elementary Mathematics": 0.841, - "helm_mmlu/Formal Logic": 0.579, - "helm_mmlu/High School World History": 0.911, - "helm_mmlu/Human Sexuality": 0.924, - "helm_mmlu/International Law": 0.901, - "helm_mmlu/Logical Fallacies": 0.877, - "helm_mmlu/Machine Learning": 0.679, - "helm_mmlu/Management": 0.903, - "helm_mmlu/Marketing": 0.932, - "helm_mmlu/Medical Genetics": 0.87, - "helm_mmlu/Miscellaneous": 0.934, - "helm_mmlu/Moral Scenarios": 0.825, - "helm_mmlu/Nutrition": 0.869, - "helm_mmlu/Prehistory": 0.917, - "helm_mmlu/Public Relations": 0.791, - "helm_mmlu/Security Studies": 0.849, - "helm_mmlu/Sociology": 0.915, - "helm_mmlu/Virology": 0.584, - "helm_mmlu/World Religions": 0.842, - "helm_mmlu/Mean win rate": 0.629 - } - }, - { - "id": "writer/palmyra-x-004-fc", - "name": "palmyra-x-004 (FC)", - "developer": "writer", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 76.0, - "bfcl/bfcl.overall.overall_accuracy": 27.87, - "bfcl/bfcl.overall.total_cost_usd": 178.15, - "bfcl/bfcl.overall.latency_mean_s": 3.71, - "bfcl/bfcl.overall.latency_std_s": 7.62, - "bfcl/bfcl.overall.latency_p95_s": 8.04, - "bfcl/bfcl.non_live.ast_accuracy": 87.46, - "bfcl/bfcl.non_live.simple_ast_accuracy": 71.33, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 96.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 93.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 89.5, - "bfcl/bfcl.live.live_accuracy": 77.87, - "bfcl/bfcl.live.live_simple_ast_accuracy": 79.46, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 77.97, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 56.25, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 70.83, - "bfcl/bfcl.multi_turn.accuracy": 0.38, - "bfcl/bfcl.multi_turn.base_accuracy": 0.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 0.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 0.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 0.5, - "bfcl/bfcl.web_search.accuracy": 2.5, - "bfcl/bfcl.web_search.base_accuracy": 4.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 1.0, - "bfcl/bfcl.memory.accuracy": 13.12, - "bfcl/bfcl.memory.kv_accuracy": 6.45, - "bfcl/bfcl.memory.vector_accuracy": 14.19, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 18.71, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 81.25, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 80.99 - } - }, - { - "id": "writer/palmyra-x-v2", - "name": "Palmyra X V2 33B", - "developer": "writer", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.589, - "helm_lite/NarrativeQA": 0.752, - "helm_lite/NaturalQuestions (closed-book)": 0.428, - "helm_lite/OpenbookQA": 0.878, - "helm_lite/MMLU": 0.621, - "helm_lite/MATH": 0.58, - "helm_lite/GSM8K": 0.735, - "helm_lite/LegalBench": 0.644, - "helm_lite/MedQA": 0.598, - "helm_lite/WMT 2014": 0.239 - } - }, - { - "id": "writer/palmyra-x-v3", - "name": "Palmyra X V3 72B", - "developer": "writer", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.679, - "helm_lite/NarrativeQA": 0.706, - "helm_lite/NaturalQuestions (closed-book)": 0.407, - "helm_lite/OpenbookQA": 0.938, - "helm_lite/MMLU": 0.702, - "helm_lite/MATH": 0.723, - "helm_lite/GSM8K": 0.831, - "helm_lite/LegalBench": 0.709, - "helm_lite/MedQA": 0.684, - "helm_lite/WMT 2014": 0.262, - "helm_mmlu/MMLU All Subjects": 0.786, - "helm_mmlu/Abstract Algebra": 0.53, - "helm_mmlu/Anatomy": 0.733, - "helm_mmlu/College Physics": 0.549, - "helm_mmlu/Computer Security": 0.78, - "helm_mmlu/Econometrics": 0.649, - "helm_mmlu/Global Facts": 0.53, - "helm_mmlu/Jurisprudence": 0.88, - "helm_mmlu/Philosophy": 0.836, - "helm_mmlu/Professional Psychology": 0.858, - "helm_mmlu/Us Foreign Policy": 0.96, - "helm_mmlu/Astronomy": 0.862, - "helm_mmlu/Business Ethics": 0.83, - "helm_mmlu/Clinical Knowledge": 0.804, - "helm_mmlu/Conceptual Physics": 0.809, - "helm_mmlu/Electrical Engineering": 0.772, - "helm_mmlu/Elementary Mathematics": 0.661, - "helm_mmlu/Formal Logic": 0.659, - "helm_mmlu/High School World History": 0.911, - "helm_mmlu/Human Sexuality": 0.924, - "helm_mmlu/International Law": 0.909, - "helm_mmlu/Logical Fallacies": 0.877, - "helm_mmlu/Machine Learning": 0.625, - "helm_mmlu/Management": 0.903, - "helm_mmlu/Marketing": 0.94, - "helm_mmlu/Medical Genetics": 0.83, - "helm_mmlu/Miscellaneous": 0.894, - "helm_mmlu/Moral Scenarios": 0.562, - "helm_mmlu/Nutrition": 0.856, - "helm_mmlu/Prehistory": 0.87, - "helm_mmlu/Public Relations": 0.773, - "helm_mmlu/Security Studies": 0.833, - "helm_mmlu/Sociology": 0.91, - "helm_mmlu/Virology": 0.572, - "helm_mmlu/World Religions": 0.877, - "helm_mmlu/Mean win rate": 0.325 - } - }, - { - "id": "writer/palmyra-x5", - "name": "Palmyra X5", - "developer": "writer", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.696, - "helm_capabilities/MMLU-Pro": 0.804, - "helm_capabilities/GPQA": 0.661, - "helm_capabilities/IFEval": 0.823, - "helm_capabilities/WildBench": 0.78, - "helm_capabilities/Omni-MATH": 0.414 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/wzhouad.json b/data/developers/wzhouad.json deleted file mode 100644 index 453797409e63aafdb8b07eeb1ddaefaa100cd036..0000000000000000000000000000000000000000 --- a/data/developers/wzhouad.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "wzhouad", - "models": [ - { - "id": "wzhouad/gemma-2-9b-it-WPO-HB", - "name": "gemma-2-9b-it-WPO-HB", - "developer": "wzhouad", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5437, - "hfopenllm_v2/BBH": 0.5629, - "hfopenllm_v2/MATH Level 5": 0.1533, - "hfopenllm_v2/GPQA": 0.3498, - "hfopenllm_v2/MUSR": 0.3675, - "hfopenllm_v2/MMLU-PRO": 0.336 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/x0000001.json b/data/developers/x0000001.json deleted file mode 100644 index 042a41cd2a9dd6984ca2dddab28f1b0c182af361..0000000000000000000000000000000000000000 --- a/data/developers/x0000001.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "x0000001", - "models": [ - { - "id": "x0000001/Deepseek-Lumen-R1-Qwen2.5-14B", - "name": "Deepseek-Lumen-R1-Qwen2.5-14B", - "developer": "x0000001", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4436, - "hfopenllm_v2/BBH": 0.4569, - "hfopenllm_v2/MATH Level 5": 0.2779, - "hfopenllm_v2/GPQA": 0.2852, - "hfopenllm_v2/MUSR": 0.474, - "hfopenllm_v2/MMLU-PRO": 0.4379 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/xai.json b/data/developers/xai.json deleted file mode 100644 index fd8750f39099c713363adac45ce08c294f19db40..0000000000000000000000000000000000000000 --- a/data/developers/xai.json +++ /dev/null @@ -1,281 +0,0 @@ -{ - "developer": "xAI", - "models": [ - { - "id": "xai/Grok 4", - "name": "Grok 4", - "developer": "xAI", - "evaluator_relationship": null, - "benchmark_scores": { - "apex-agents/Overall Pass@1": 0.152, - "apex-agents/Overall Pass@8": 0.329, - "apex-agents/Overall Mean Score": 0.303, - "apex-agents/Investment Banking Pass@1": 0.17, - "apex-agents/Management Consulting Pass@1": 0.12, - "apex-agents/Corporate Law Pass@1": 0.165, - "apex-agents/Corporate Lawyer Mean Score": 0.41, - "apex-v1/Overall Score": 0.635 - } - }, - { - "id": "xai/grok-3-beta", - "name": "Grok 3 Beta", - "developer": "xAI", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.727, - "helm_capabilities/MMLU-Pro": 0.788, - "helm_capabilities/GPQA": 0.65, - "helm_capabilities/IFEval": 0.884, - "helm_capabilities/WildBench": 0.849, - "helm_capabilities/Omni-MATH": 0.464 - } - }, - { - "id": "xai/grok-3-mini", - "name": "grok-3-mini", - "developer": "xAI", - "evaluator_relationship": null, - "benchmark_scores": { - "global-mmlu-lite/Global MMLU Lite": 0.673, - "global-mmlu-lite/Culturally Sensitive": 0.6717, - "global-mmlu-lite/Culturally Agnostic": 0.6743, - "global-mmlu-lite/Arabic": 0.755, - "global-mmlu-lite/English": 0.5075, - "global-mmlu-lite/Bengali": 0.7355, - "global-mmlu-lite/German": 0.6591, - "global-mmlu-lite/French": 0.485, - "global-mmlu-lite/Hindi": 0.56, - "global-mmlu-lite/Indonesian": 0.725, - "global-mmlu-lite/Italian": 0.696, - "global-mmlu-lite/Japanese": 0.6575, - "global-mmlu-lite/Korean": 0.7325, - "global-mmlu-lite/Portuguese": 0.6275, - "global-mmlu-lite/Spanish": 0.61, - "global-mmlu-lite/Swahili": 0.7625, - "global-mmlu-lite/Yoruba": 0.8296, - "global-mmlu-lite/Chinese": 0.5564, - "global-mmlu-lite/Burmese": 0.8693 - } - }, - { - "id": "xai/grok-3-mini-beta", - "name": "Grok 3 mini Beta", - "developer": "xAI", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.679, - "helm_capabilities/MMLU-Pro": 0.799, - "helm_capabilities/GPQA": 0.675, - "helm_capabilities/IFEval": 0.951, - "helm_capabilities/WildBench": 0.651, - "helm_capabilities/Omni-MATH": 0.318 - } - }, - { - "id": "xai/grok-4", - "name": "Grok 4", - "developer": "xAI", - "evaluator_relationship": null, - "benchmark_scores": { - "terminal-bench-2.0/terminal-bench-2.0": 23.1 - } - }, - { - "id": "xai/grok-4-0709", - "name": "grok-4-0709", - "developer": "xAI", - "evaluator_relationship": null, - "benchmark_scores": { - "global-mmlu-lite/Global MMLU Lite": 0.8881, - "global-mmlu-lite/Culturally Sensitive": 0.8862, - "global-mmlu-lite/Culturally Agnostic": 0.89, - "global-mmlu-lite/Arabic": 0.885, - "global-mmlu-lite/English": 0.905, - "global-mmlu-lite/Bengali": 0.8925, - "global-mmlu-lite/German": 0.8725, - "global-mmlu-lite/French": 0.875, - "global-mmlu-lite/Hindi": 0.8675, - "global-mmlu-lite/Indonesian": 0.89, - "global-mmlu-lite/Italian": 0.9025, - "global-mmlu-lite/Japanese": 0.87, - "global-mmlu-lite/Korean": 0.895, - "global-mmlu-lite/Portuguese": 0.8725, - "global-mmlu-lite/Spanish": 0.9075, - "global-mmlu-lite/Swahili": 0.91, - "global-mmlu-lite/Yoruba": 0.905, - "global-mmlu-lite/Chinese": 0.8525, - "global-mmlu-lite/Burmese": 0.9075, - "helm_capabilities/Mean score": 0.785, - "helm_capabilities/MMLU-Pro": 0.851, - "helm_capabilities/GPQA": 0.726, - "helm_capabilities/IFEval": 0.949, - "helm_capabilities/WildBench": 0.797, - "helm_capabilities/Omni-MATH": 0.603 - } - }, - { - "id": "xai/grok-4-0709-fc", - "name": "Grok-4-0709 (FC)", - "developer": "xAI", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 10.0, - "bfcl/bfcl.overall.overall_accuracy": 61.38, - "bfcl/bfcl.overall.total_cost_usd": 355.17, - "bfcl/bfcl.overall.latency_mean_s": 15.49, - "bfcl/bfcl.overall.latency_std_s": 26.22, - "bfcl/bfcl.overall.latency_p95_s": 44.28, - "bfcl/bfcl.non_live.ast_accuracy": 85.38, - "bfcl/bfcl.non_live.simple_ast_accuracy": 73.5, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 92.5, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 88.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 87.0, - "bfcl/bfcl.live.live_accuracy": 75.57, - "bfcl/bfcl.live.live_simple_ast_accuracy": 82.17, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 73.88, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 75.0, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 79.17, - "bfcl/bfcl.multi_turn.accuracy": 33.88, - "bfcl/bfcl.multi_turn.base_accuracy": 44.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 19.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 28.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 44.0, - "bfcl/bfcl.web_search.accuracy": 82.0, - "bfcl/bfcl.web_search.base_accuracy": 80.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 84.0, - "bfcl/bfcl.memory.accuracy": 55.91, - "bfcl/bfcl.memory.kv_accuracy": 57.42, - "bfcl/bfcl.memory.vector_accuracy": 58.71, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 51.61, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 87.5, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 75.4 - } - }, - { - "id": "xai/grok-4-0709-prompt", - "name": "Grok-4-0709 (Prompt)", - "developer": "xAI", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 9.0, - "bfcl/bfcl.overall.overall_accuracy": 62.97, - "bfcl/bfcl.overall.total_cost_usd": 348.19, - "bfcl/bfcl.overall.latency_mean_s": 30.38, - "bfcl/bfcl.overall.latency_std_s": 36.19, - "bfcl/bfcl.overall.latency_p95_s": 101.54, - "bfcl/bfcl.non_live.ast_accuracy": 82.75, - "bfcl/bfcl.non_live.simple_ast_accuracy": 67.0, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 93.5, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 89.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 81.5, - "bfcl/bfcl.live.live_accuracy": 72.54, - "bfcl/bfcl.live.live_simple_ast_accuracy": 81.78, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 70.18, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 81.25, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 70.83, - "bfcl/bfcl.multi_turn.accuracy": 47.0, - "bfcl/bfcl.multi_turn.base_accuracy": 55.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 46.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 36.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 50.5, - "bfcl/bfcl.web_search.accuracy": 74.0, - "bfcl/bfcl.web_search.base_accuracy": 74.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 74.0, - "bfcl/bfcl.memory.accuracy": 50.54, - "bfcl/bfcl.memory.kv_accuracy": 43.87, - "bfcl/bfcl.memory.vector_accuracy": 59.35, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 48.39, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 81.25, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 84.3, - "bfcl/bfcl.format_sensitivity.max_delta": 13.0, - "bfcl/bfcl.format_sensitivity.stddev": 2.88 - } - }, - { - "id": "xai/grok-4-1-fast-non-reasoning-fc", - "name": "Grok-4-1-fast-non-reasoning (FC)", - "developer": "xAI", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 12.0, - "bfcl/bfcl.overall.overall_accuracy": 58.29, - "bfcl/bfcl.overall.total_cost_usd": 16.27, - "bfcl/bfcl.overall.latency_mean_s": 2.29, - "bfcl/bfcl.overall.latency_std_s": 7.31, - "bfcl/bfcl.overall.latency_p95_s": 5.34, - "bfcl/bfcl.non_live.ast_accuracy": 88.13, - "bfcl/bfcl.non_live.simple_ast_accuracy": 76.0, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 93.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 93.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 90.5, - "bfcl/bfcl.live.live_accuracy": 77.94, - "bfcl/bfcl.live.live_simple_ast_accuracy": 82.95, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 76.92, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 75.0, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 70.83, - "bfcl/bfcl.multi_turn.accuracy": 46.75, - "bfcl/bfcl.multi_turn.base_accuracy": 58.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 39.5, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 37.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 52.0, - "bfcl/bfcl.web_search.accuracy": 75.0, - "bfcl/bfcl.web_search.base_accuracy": 74.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 76.0, - "bfcl/bfcl.memory.accuracy": 26.24, - "bfcl/bfcl.memory.kv_accuracy": 20.65, - "bfcl/bfcl.memory.vector_accuracy": 20.0, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 38.06, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 81.25, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 74.09 - } - }, - { - "id": "xai/grok-4-1-fast-reasoning-fc", - "name": "Grok-4-1-fast-reasoning (FC)", - "developer": "xAI", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 5.0, - "bfcl/bfcl.overall.overall_accuracy": 69.57, - "bfcl/bfcl.overall.total_cost_usd": 17.26, - "bfcl/bfcl.overall.latency_mean_s": 6.74, - "bfcl/bfcl.overall.latency_std_s": 12.78, - "bfcl/bfcl.overall.latency_p95_s": 17.57, - "bfcl/bfcl.non_live.ast_accuracy": 88.27, - "bfcl/bfcl.non_live.simple_ast_accuracy": 77.58, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 93.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 92.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 90.0, - "bfcl/bfcl.live.live_accuracy": 78.46, - "bfcl/bfcl.live.live_simple_ast_accuracy": 84.11, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 77.3, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 75.0, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 70.83, - "bfcl/bfcl.multi_turn.accuracy": 58.87, - "bfcl/bfcl.multi_turn.base_accuracy": 70.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 59.5, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 43.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 62.5, - "bfcl/bfcl.web_search.accuracy": 82.5, - "bfcl/bfcl.web_search.base_accuracy": 82.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 83.0, - "bfcl/bfcl.memory.accuracy": 53.98, - "bfcl/bfcl.memory.kv_accuracy": 41.29, - "bfcl/bfcl.memory.vector_accuracy": 57.42, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 63.23, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 81.25, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 79.43 - } - }, - { - "id": "xai/grok-code-fast-1", - "name": "Grok Code Fast 1", - "developer": "xAI", - "evaluator_relationship": null, - "benchmark_scores": { - "terminal-bench-2.0/terminal-bench-2.0": 14.2 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/xclbr7.json b/data/developers/xclbr7.json deleted file mode 100644 index 2e2c000d3a74eadfd8d24930f7ef63d3f4b7d7d9..0000000000000000000000000000000000000000 --- a/data/developers/xclbr7.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "developer": "Xclbr7", - "models": [ - { - "id": "Xclbr7/Arcanum-12b", - "name": "Arcanum-12b", - "developer": "Xclbr7", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2907, - "hfopenllm_v2/BBH": 0.5265, - "hfopenllm_v2/MATH Level 5": 0.1193, - "hfopenllm_v2/GPQA": 0.3205, - "hfopenllm_v2/MUSR": 0.417, - "hfopenllm_v2/MMLU-PRO": 0.3586 - } - }, - { - "id": "Xclbr7/caliburn-12b", - "name": "caliburn-12b", - "developer": "Xclbr7", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3576, - "hfopenllm_v2/BBH": 0.5519, - "hfopenllm_v2/MATH Level 5": 0.1125, - "hfopenllm_v2/GPQA": 0.3364, - "hfopenllm_v2/MUSR": 0.4292, - "hfopenllm_v2/MMLU-PRO": 0.3675 - } - }, - { - "id": "Xclbr7/caliburn-v2-12b", - "name": "caliburn-v2-12b", - "developer": "Xclbr7", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2967, - "hfopenllm_v2/BBH": 0.5141, - "hfopenllm_v2/MATH Level 5": 0.105, - "hfopenllm_v2/GPQA": 0.3263, - "hfopenllm_v2/MUSR": 0.437, - "hfopenllm_v2/MMLU-PRO": 0.3784 - } - }, - { - "id": "Xclbr7/Hyena-12b", - "name": "Hyena-12b", - "developer": "Xclbr7", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3404, - "hfopenllm_v2/BBH": 0.5457, - "hfopenllm_v2/MATH Level 5": 0.1133, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.3984, - "hfopenllm_v2/MMLU-PRO": 0.3439 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/xiaojian9992024.json b/data/developers/xiaojian9992024.json deleted file mode 100644 index 143c5d039208c97a6954b4619330d62b00e55a1a..0000000000000000000000000000000000000000 --- a/data/developers/xiaojian9992024.json +++ /dev/null @@ -1,173 +0,0 @@ -{ - "developer": "Xiaojian9992024", - "models": [ - { - "id": "Xiaojian9992024/Llama3.2-1B-THREADRIPPER", - "name": "Llama3.2-1B-THREADRIPPER", - "developer": "Xiaojian9992024", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5576, - "hfopenllm_v2/BBH": 0.3544, - "hfopenllm_v2/MATH Level 5": 0.074, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.313, - "hfopenllm_v2/MMLU-PRO": 0.1763 - } - }, - { - "id": "Xiaojian9992024/Llama3.2-1B-THREADRIPPER-v0.2", - "name": "Llama3.2-1B-THREADRIPPER-v0.2", - "developer": "Xiaojian9992024", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5318, - "hfopenllm_v2/BBH": 0.3528, - "hfopenllm_v2/MATH Level 5": 0.0657, - "hfopenllm_v2/GPQA": 0.2659, - "hfopenllm_v2/MUSR": 0.3316, - "hfopenllm_v2/MMLU-PRO": 0.1745 - } - }, - { - "id": "Xiaojian9992024/Phi-4-Megatron-Empathetic", - "name": "Phi-4-Megatron-Empathetic", - "developer": "Xiaojian9992024", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0173, - "hfopenllm_v2/BBH": 0.6673, - "hfopenllm_v2/MATH Level 5": 0.2696, - "hfopenllm_v2/GPQA": 0.3859, - "hfopenllm_v2/MUSR": 0.5071, - "hfopenllm_v2/MMLU-PRO": 0.5082 - } - }, - { - "id": "Xiaojian9992024/Phi-4-mini-UNOFFICAL", - "name": "Phi-4-mini-UNOFFICAL", - "developer": "Xiaojian9992024", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1273, - "hfopenllm_v2/BBH": 0.2944, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2408, - "hfopenllm_v2/MUSR": 0.3368, - "hfopenllm_v2/MMLU-PRO": 0.1144 - } - }, - { - "id": "Xiaojian9992024/Qwen2.5-7B-MS-Destroyer", - "name": "Qwen2.5-7B-MS-Destroyer", - "developer": "Xiaojian9992024", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7296, - "hfopenllm_v2/BBH": 0.547, - "hfopenllm_v2/MATH Level 5": 0.4592, - "hfopenllm_v2/GPQA": 0.3045, - "hfopenllm_v2/MUSR": 0.427, - "hfopenllm_v2/MMLU-PRO": 0.4412 - } - }, - { - "id": "Xiaojian9992024/Qwen2.5-Dyanka-7B-Preview", - "name": "Qwen2.5-Dyanka-7B-Preview", - "developer": "Xiaojian9992024", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.764, - "hfopenllm_v2/BBH": 0.5543, - "hfopenllm_v2/MATH Level 5": 0.4879, - "hfopenllm_v2/GPQA": 0.3171, - "hfopenllm_v2/MUSR": 0.4481, - "hfopenllm_v2/MMLU-PRO": 0.4376 - } - }, - { - "id": "Xiaojian9992024/Qwen2.5-Dyanka-7B-Preview-v0.2", - "name": "Qwen2.5-Dyanka-7B-Preview-v0.2", - "developer": "Xiaojian9992024", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6702, - "hfopenllm_v2/BBH": 0.5374, - "hfopenllm_v2/MATH Level 5": 0.4721, - "hfopenllm_v2/GPQA": 0.2936, - "hfopenllm_v2/MUSR": 0.4467, - "hfopenllm_v2/MMLU-PRO": 0.4371 - } - }, - { - "id": "Xiaojian9992024/Qwen2.5-THREADRIPPER-Medium-Censored", - "name": "Qwen2.5-THREADRIPPER-Medium-Censored", - "developer": "Xiaojian9992024", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8112, - "hfopenllm_v2/BBH": 0.6431, - "hfopenllm_v2/MATH Level 5": 0.534, - "hfopenllm_v2/GPQA": 0.3347, - "hfopenllm_v2/MUSR": 0.414, - "hfopenllm_v2/MMLU-PRO": 0.4929 - } - }, - { - "id": "Xiaojian9992024/Qwen2.5-THREADRIPPER-Small", - "name": "Qwen2.5-THREADRIPPER-Small", - "developer": "Xiaojian9992024", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7689, - "hfopenllm_v2/BBH": 0.549, - "hfopenllm_v2/MATH Level 5": 0.4736, - "hfopenllm_v2/GPQA": 0.3104, - "hfopenllm_v2/MUSR": 0.4349, - "hfopenllm_v2/MMLU-PRO": 0.4357 - } - }, - { - "id": "Xiaojian9992024/Qwen2.5-THREADRIPPER-Small-AnniversaryEdition", - "name": "Qwen2.5-THREADRIPPER-Small-AnniversaryEdition", - "developer": "Xiaojian9992024", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7404, - "hfopenllm_v2/BBH": 0.5465, - "hfopenllm_v2/MATH Level 5": 0.5076, - "hfopenllm_v2/GPQA": 0.2685, - "hfopenllm_v2/MUSR": 0.3807, - "hfopenllm_v2/MMLU-PRO": 0.4393 - } - }, - { - "id": "Xiaojian9992024/Qwen2.5-Ultra-1.5B-25.02-Exp", - "name": "Qwen2.5-Ultra-1.5B-25.02-Exp", - "developer": "Xiaojian9992024", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4073, - "hfopenllm_v2/BBH": 0.4066, - "hfopenllm_v2/MATH Level 5": 0.0831, - "hfopenllm_v2/GPQA": 0.2584, - "hfopenllm_v2/MUSR": 0.3383, - "hfopenllm_v2/MMLU-PRO": 0.2641 - } - }, - { - "id": "Xiaojian9992024/Reflection-L3.2-JametMiniMix-3B", - "name": "Reflection-L3.2-JametMiniMix-3B", - "developer": "Xiaojian9992024", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4619, - "hfopenllm_v2/BBH": 0.439, - "hfopenllm_v2/MATH Level 5": 0.1193, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.3667, - "hfopenllm_v2/MMLU-PRO": 0.2988 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/xinchen9.json b/data/developers/xinchen9.json deleted file mode 100644 index 3caed50dd5bad9825ff7414a57353a764f10c902..0000000000000000000000000000000000000000 --- a/data/developers/xinchen9.json +++ /dev/null @@ -1,75 +0,0 @@ -{ - "developer": "xinchen9", - "models": [ - { - "id": "xinchen9/llama3-b8-ft-dis", - "name": "llama3-b8-ft-dis", - "developer": "xinchen9", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1546, - "hfopenllm_v2/BBH": 0.4626, - "hfopenllm_v2/MATH Level 5": 0.0393, - "hfopenllm_v2/GPQA": 0.3129, - "hfopenllm_v2/MUSR": 0.3654, - "hfopenllm_v2/MMLU-PRO": 0.3244 - } - }, - { - "id": "xinchen9/Llama3.1_8B_Instruct_CoT", - "name": "Llama3.1_8B_Instruct_CoT", - "developer": "xinchen9", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2974, - "hfopenllm_v2/BBH": 0.4398, - "hfopenllm_v2/MATH Level 5": 0.0604, - "hfopenllm_v2/GPQA": 0.302, - "hfopenllm_v2/MUSR": 0.4371, - "hfopenllm_v2/MMLU-PRO": 0.2879 - } - }, - { - "id": "xinchen9/Llama3.1_CoT", - "name": "Llama3.1_CoT", - "developer": "xinchen9", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2246, - "hfopenllm_v2/BBH": 0.4341, - "hfopenllm_v2/MATH Level 5": 0.0385, - "hfopenllm_v2/GPQA": 0.2886, - "hfopenllm_v2/MUSR": 0.4305, - "hfopenllm_v2/MMLU-PRO": 0.2739 - } - }, - { - "id": "xinchen9/Llama3.1_CoT_V1", - "name": "Llama3.1_CoT_V1", - "developer": "xinchen9", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2453, - "hfopenllm_v2/BBH": 0.4376, - "hfopenllm_v2/MATH Level 5": 0.0332, - "hfopenllm_v2/GPQA": 0.2794, - "hfopenllm_v2/MUSR": 0.4572, - "hfopenllm_v2/MMLU-PRO": 0.2805 - } - }, - { - "id": "xinchen9/Mistral-7B-CoT", - "name": "Mistral-7B-CoT", - "developer": "xinchen9", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2783, - "hfopenllm_v2/BBH": 0.3873, - "hfopenllm_v2/MATH Level 5": 0.0249, - "hfopenllm_v2/GPQA": 0.2492, - "hfopenllm_v2/MUSR": 0.3994, - "hfopenllm_v2/MMLU-PRO": 0.2284 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/xkev.json b/data/developers/xkev.json deleted file mode 100644 index 5612b6593ae88e8cf3c29929039db59c9e95e540..0000000000000000000000000000000000000000 --- a/data/developers/xkev.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "Xkev", - "models": [ - { - "id": "Xkev/Llama-3.2V-11B-cot", - "name": "Llama-3.2V-11B-cot", - "developer": "Xkev", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4158, - "hfopenllm_v2/BBH": 0.4959, - "hfopenllm_v2/MATH Level 5": 0.1556, - "hfopenllm_v2/GPQA": 0.2953, - "hfopenllm_v2/MUSR": 0.4159, - "hfopenllm_v2/MMLU-PRO": 0.3587 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/xkp24.json b/data/developers/xkp24.json deleted file mode 100644 index 4083497ab0a287c873007f5bd866a54a7499662d..0000000000000000000000000000000000000000 --- a/data/developers/xkp24.json +++ /dev/null @@ -1,117 +0,0 @@ -{ - "developer": "xkp24", - "models": [ - { - "id": "xkp24/Llama-3-8B-Instruct-SPPO-Iter2_bt_2b-table", - "name": "Llama-3-8B-Instruct-SPPO-Iter2_bt_2b-table", - "developer": "xkp24", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6375, - "hfopenllm_v2/BBH": 0.4912, - "hfopenllm_v2/MATH Level 5": 0.0921, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.382, - "hfopenllm_v2/MMLU-PRO": 0.3686 - } - }, - { - "id": "xkp24/Llama-3-8B-Instruct-SPPO-Iter2_bt_8b-table", - "name": "Llama-3-8B-Instruct-SPPO-Iter2_bt_8b-table", - "developer": "xkp24", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7275, - "hfopenllm_v2/BBH": 0.5057, - "hfopenllm_v2/MATH Level 5": 0.0846, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3819, - "hfopenllm_v2/MMLU-PRO": 0.3697 - } - }, - { - "id": "xkp24/Llama-3-8B-Instruct-SPPO-Iter2_gp_2b-table", - "name": "Llama-3-8B-Instruct-SPPO-Iter2_gp_2b-table", - "developer": "xkp24", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6569, - "hfopenllm_v2/BBH": 0.4952, - "hfopenllm_v2/MATH Level 5": 0.0891, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3594, - "hfopenllm_v2/MMLU-PRO": 0.3702 - } - }, - { - "id": "xkp24/Llama-3-8B-Instruct-SPPO-Iter2_gp_8b-table", - "name": "Llama-3-8B-Instruct-SPPO-Iter2_gp_8b-table", - "developer": "xkp24", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6621, - "hfopenllm_v2/BBH": 0.5004, - "hfopenllm_v2/MATH Level 5": 0.0861, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3805, - "hfopenllm_v2/MMLU-PRO": 0.36 - } - }, - { - "id": "xkp24/Llama-3-8B-Instruct-SPPO-score-Iter2_bt_2b-table-0.001", - "name": "Llama-3-8B-Instruct-SPPO-score-Iter2_bt_2b-table-0.001", - "developer": "xkp24", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6042, - "hfopenllm_v2/BBH": 0.4936, - "hfopenllm_v2/MATH Level 5": 0.0997, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3793, - "hfopenllm_v2/MMLU-PRO": 0.3708 - } - }, - { - "id": "xkp24/Llama-3-8B-Instruct-SPPO-score-Iter2_bt_8b-table-0.002", - "name": "Llama-3-8B-Instruct-SPPO-score-Iter2_bt_8b-table-0.002", - "developer": "xkp24", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7132, - "hfopenllm_v2/BBH": 0.4996, - "hfopenllm_v2/MATH Level 5": 0.0853, - "hfopenllm_v2/GPQA": 0.2584, - "hfopenllm_v2/MUSR": 0.3872, - "hfopenllm_v2/MMLU-PRO": 0.3664 - } - }, - { - "id": "xkp24/Llama-3-8B-Instruct-SPPO-score-Iter2_gp_2b-table-0.001", - "name": "Llama-3-8B-Instruct-SPPO-score-Iter2_gp_2b-table-0.001", - "developer": "xkp24", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5947, - "hfopenllm_v2/BBH": 0.4899, - "hfopenllm_v2/MATH Level 5": 0.1073, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3581, - "hfopenllm_v2/MMLU-PRO": 0.3704 - } - }, - { - "id": "xkp24/Llama-3-8B-Instruct-SPPO-score-Iter2_gp_8b-table-0.002", - "name": "Llama-3-8B-Instruct-SPPO-score-Iter2_gp_8b-table-0.002", - "developer": "xkp24", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6453, - "hfopenllm_v2/BBH": 0.4951, - "hfopenllm_v2/MATH Level 5": 0.0937, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3939, - "hfopenllm_v2/MMLU-PRO": 0.353 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/xmaulana.json b/data/developers/xmaulana.json deleted file mode 100644 index b4d6b577aa49240ab6bcf77c3897b8d702f000e1..0000000000000000000000000000000000000000 --- a/data/developers/xmaulana.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "xMaulana", - "models": [ - { - "id": "xMaulana/FinMatcha-3B-Instruct", - "name": "FinMatcha-3B-Instruct", - "developer": "xMaulana", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7548, - "hfopenllm_v2/BBH": 0.4536, - "hfopenllm_v2/MATH Level 5": 0.1435, - "hfopenllm_v2/GPQA": 0.2693, - "hfopenllm_v2/MUSR": 0.3633, - "hfopenllm_v2/MMLU-PRO": 0.3182 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/xukp20.json b/data/developers/xukp20.json deleted file mode 100644 index ab51e2b1b49eb65e77d508fc3bff37a0f1179db5..0000000000000000000000000000000000000000 --- a/data/developers/xukp20.json +++ /dev/null @@ -1,117 +0,0 @@ -{ - "developer": "xukp20", - "models": [ - { - "id": "xukp20/llama-3-8b-instruct-sppo-iter1-gp-2b-tau01-table", - "name": "llama-3-8b-instruct-sppo-iter1-gp-2b-tau01-table", - "developer": "xukp20", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.69, - "hfopenllm_v2/BBH": 0.4978, - "hfopenllm_v2/MATH Level 5": 0.105, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3673, - "hfopenllm_v2/MMLU-PRO": 0.3716 - } - }, - { - "id": "xukp20/Llama-3-8B-Instruct-SPPO-Iter3_bt_2b-table", - "name": "Llama-3-8B-Instruct-SPPO-Iter3_bt_2b-table", - "developer": "xukp20", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5756, - "hfopenllm_v2/BBH": 0.4901, - "hfopenllm_v2/MATH Level 5": 0.0997, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.366, - "hfopenllm_v2/MMLU-PRO": 0.3659 - } - }, - { - "id": "xukp20/Llama-3-8B-Instruct-SPPO-Iter3_bt_8b-table", - "name": "Llama-3-8B-Instruct-SPPO-Iter3_bt_8b-table", - "developer": "xukp20", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7034, - "hfopenllm_v2/BBH": 0.5092, - "hfopenllm_v2/MATH Level 5": 0.0967, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3739, - "hfopenllm_v2/MMLU-PRO": 0.3693 - } - }, - { - "id": "xukp20/Llama-3-8B-Instruct-SPPO-Iter3_gp_2b-table", - "name": "Llama-3-8B-Instruct-SPPO-Iter3_gp_2b-table", - "developer": "xukp20", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6024, - "hfopenllm_v2/BBH": 0.497, - "hfopenllm_v2/MATH Level 5": 0.1042, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3674, - "hfopenllm_v2/MMLU-PRO": 0.3658 - } - }, - { - "id": "xukp20/Llama-3-8B-Instruct-SPPO-Iter3_gp_8b-table", - "name": "Llama-3-8B-Instruct-SPPO-Iter3_gp_8b-table", - "developer": "xukp20", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.662, - "hfopenllm_v2/BBH": 0.5, - "hfopenllm_v2/MATH Level 5": 0.0937, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3818, - "hfopenllm_v2/MMLU-PRO": 0.3615 - } - }, - { - "id": "xukp20/Llama-3-8B-Instruct-SPPO-score-Iter3_bt_2b-table-0.001", - "name": "Llama-3-8B-Instruct-SPPO-score-Iter3_bt_2b-table-0.001", - "developer": "xukp20", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5336, - "hfopenllm_v2/BBH": 0.4915, - "hfopenllm_v2/MATH Level 5": 0.0982, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.378, - "hfopenllm_v2/MMLU-PRO": 0.3625 - } - }, - { - "id": "xukp20/Llama-3-8B-Instruct-SPPO-score-Iter3_bt_8b-table-0.002", - "name": "Llama-3-8B-Instruct-SPPO-score-Iter3_bt_8b-table-0.002", - "developer": "xukp20", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6852, - "hfopenllm_v2/BBH": 0.5075, - "hfopenllm_v2/MATH Level 5": 0.0718, - "hfopenllm_v2/GPQA": 0.2584, - "hfopenllm_v2/MUSR": 0.3832, - "hfopenllm_v2/MMLU-PRO": 0.3621 - } - }, - { - "id": "xukp20/Llama-3-8B-Instruct-SPPO-score-Iter3_gp_2b-table-0.001", - "name": "Llama-3-8B-Instruct-SPPO-score-Iter3_gp_2b-table-0.001", - "developer": "xukp20", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5482, - "hfopenllm_v2/BBH": 0.4887, - "hfopenllm_v2/MATH Level 5": 0.0891, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.3633, - "hfopenllm_v2/MMLU-PRO": 0.3671 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/xwen-team.json b/data/developers/xwen-team.json deleted file mode 100644 index 60d7a6f2f80b797def4b2a5bf64155dcbc5adbff..0000000000000000000000000000000000000000 --- a/data/developers/xwen-team.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "xwen-team", - "models": [ - { - "id": "xwen-team/Xwen-7B-Chat", - "name": "Xwen-7B-Chat", - "developer": "xwen-team", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6864, - "hfopenllm_v2/BBH": 0.5068, - "hfopenllm_v2/MATH Level 5": 0.4509, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.3914, - "hfopenllm_v2/MMLU-PRO": 0.429 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/xxx777xxxasd.json b/data/developers/xxx777xxxasd.json deleted file mode 100644 index 9624ed023877d4f46e564f2b1046890ec687436e..0000000000000000000000000000000000000000 --- a/data/developers/xxx777xxxasd.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "xxx777xxxASD", - "models": [ - { - "id": "xxx777xxxASD/L3.1-ClaudeMaid-4x8B", - "name": "L3.1-ClaudeMaid-4x8B", - "developer": "xxx777xxxASD", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6696, - "hfopenllm_v2/BBH": 0.5071, - "hfopenllm_v2/MATH Level 5": 0.1412, - "hfopenllm_v2/GPQA": 0.2911, - "hfopenllm_v2/MUSR": 0.4289, - "hfopenllm_v2/MMLU-PRO": 0.358 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/yam-peleg.json b/data/developers/yam-peleg.json deleted file mode 100644 index f415128b8e93253c12ae15436b68cf36d5f248bf..0000000000000000000000000000000000000000 --- a/data/developers/yam-peleg.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "developer": "yam-peleg", - "models": [ - { - "id": "yam-peleg/Hebrew-Gemma-11B-Instruct", - "name": "Hebrew-Gemma-11B-Instruct", - "developer": "yam-peleg", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3021, - "hfopenllm_v2/BBH": 0.4036, - "hfopenllm_v2/MATH Level 5": 0.0657, - "hfopenllm_v2/GPQA": 0.276, - "hfopenllm_v2/MUSR": 0.4089, - "hfopenllm_v2/MMLU-PRO": 0.2554 - } - }, - { - "id": "yam-peleg/Hebrew-Mistral-7B", - "name": "Hebrew-Mistral-7B", - "developer": "yam-peleg", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2328, - "hfopenllm_v2/BBH": 0.4334, - "hfopenllm_v2/MATH Level 5": 0.0498, - "hfopenllm_v2/GPQA": 0.2794, - "hfopenllm_v2/MUSR": 0.3977, - "hfopenllm_v2/MMLU-PRO": 0.278 - } - }, - { - "id": "yam-peleg/Hebrew-Mistral-7B-200K", - "name": "Hebrew-Mistral-7B-200K", - "developer": "yam-peleg", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.177, - "hfopenllm_v2/BBH": 0.3411, - "hfopenllm_v2/MATH Level 5": 0.031, - "hfopenllm_v2/GPQA": 0.2534, - "hfopenllm_v2/MUSR": 0.374, - "hfopenllm_v2/MMLU-PRO": 0.2529 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/yandex.json b/data/developers/yandex.json deleted file mode 100644 index b112206ad32f8f95bdaef8cfd7791c6d8a424936..0000000000000000000000000000000000000000 --- a/data/developers/yandex.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "developer": "yandex", - "models": [ - { - "id": "yandex/YaLM-100B", - "name": "YaLM 100B", - "developer": "yandex", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.075, - "helm_classic/MMLU": 0.243, - "helm_classic/BoolQ": 0.634, - "helm_classic/NarrativeQA": 0.252, - "helm_classic/NaturalQuestions (open-book)": 0.227, - "helm_classic/QuAC": 0.162, - "helm_classic/HellaSwag": -1.0, - "helm_classic/OpenbookQA": -1.0, - "helm_classic/TruthfulQA": 0.202, - "helm_classic/MS MARCO (TREC)": -1.0, - "helm_classic/CNN/DailyMail": 0.017, - "helm_classic/XSUM": 0.021, - "helm_classic/IMDB": 0.836, - "helm_classic/CivilComments": 0.49, - "helm_classic/RAFT": 0.395 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/yanng1242.json b/data/developers/yanng1242.json deleted file mode 100644 index ef7b82c81b1cc22d6beb15bce60394b80153904c..0000000000000000000000000000000000000000 --- a/data/developers/yanng1242.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "yanng1242", - "models": [ - { - "id": "yanng1242/Marcoro14-7B-slerp", - "name": "Marcoro14-7B-slerp", - "developer": "yanng1242", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.406, - "hfopenllm_v2/BBH": 0.5252, - "hfopenllm_v2/MATH Level 5": 0.0748, - "hfopenllm_v2/GPQA": 0.3146, - "hfopenllm_v2/MUSR": 0.4686, - "hfopenllm_v2/MMLU-PRO": 0.3168 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/yash21.json b/data/developers/yash21.json deleted file mode 100644 index ab19941e98732d96ef334b7798f9aaa9e1ede929..0000000000000000000000000000000000000000 --- a/data/developers/yash21.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "Yash21", - "models": [ - { - "id": "Yash21/TinyYi-7B-Test", - "name": "TinyYi-7B-Test", - "developer": "Yash21", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1856, - "hfopenllm_v2/BBH": 0.291, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2643, - "hfopenllm_v2/MUSR": 0.3364, - "hfopenllm_v2/MMLU-PRO": 0.1091 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/yasserrmd.json b/data/developers/yasserrmd.json deleted file mode 100644 index 5564a88140dae2972387cd1d19a8316870614c39..0000000000000000000000000000000000000000 --- a/data/developers/yasserrmd.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "yasserrmd", - "models": [ - { - "id": "yasserrmd/Coder-GRPO-3B", - "name": "Coder-GRPO-3B", - "developer": "yasserrmd", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6208, - "hfopenllm_v2/BBH": 0.4469, - "hfopenllm_v2/MATH Level 5": 0.3202, - "hfopenllm_v2/GPQA": 0.2777, - "hfopenllm_v2/MUSR": 0.4115, - "hfopenllm_v2/MMLU-PRO": 0.3197 - } - }, - { - "id": "yasserrmd/Text2SQL-1.5B", - "name": "Text2SQL-1.5B", - "developer": "yasserrmd", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2857, - "hfopenllm_v2/BBH": 0.3858, - "hfopenllm_v2/MATH Level 5": 0.068, - "hfopenllm_v2/GPQA": 0.2878, - "hfopenllm_v2/MUSR": 0.3942, - "hfopenllm_v2/MMLU-PRO": 0.2363 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/ycros.json b/data/developers/ycros.json deleted file mode 100644 index 9b83f7f872f197f98fd71538cda202af618f5659..0000000000000000000000000000000000000000 --- a/data/developers/ycros.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "ycros", - "models": [ - { - "id": "ycros/BagelMIsteryTour-v2-8x7B", - "name": "BagelMIsteryTour-v2-8x7B", - "developer": "ycros", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5994, - "hfopenllm_v2/BBH": 0.5159, - "hfopenllm_v2/MATH Level 5": 0.0785, - "hfopenllm_v2/GPQA": 0.3045, - "hfopenllm_v2/MUSR": 0.4203, - "hfopenllm_v2/MMLU-PRO": 0.3473 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/yfzp.json b/data/developers/yfzp.json deleted file mode 100644 index af69616605e9d8864220b6bd0050288576f02596..0000000000000000000000000000000000000000 --- a/data/developers/yfzp.json +++ /dev/null @@ -1,117 +0,0 @@ -{ - "developer": "yfzp", - "models": [ - { - "id": "yfzp/Llama-3-8B-Instruct-SPPO-Iter1_bt_2b-table", - "name": "Llama-3-8B-Instruct-SPPO-Iter1_bt_2b-table", - "developer": "yfzp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6709, - "hfopenllm_v2/BBH": 0.4987, - "hfopenllm_v2/MATH Level 5": 0.1118, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3727, - "hfopenllm_v2/MMLU-PRO": 0.3716 - } - }, - { - "id": "yfzp/Llama-3-8B-Instruct-SPPO-Iter1_bt_8b-table", - "name": "Llama-3-8B-Instruct-SPPO-Iter1_bt_8b-table", - "developer": "yfzp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7333, - "hfopenllm_v2/BBH": 0.508, - "hfopenllm_v2/MATH Level 5": 0.1035, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3806, - "hfopenllm_v2/MMLU-PRO": 0.3748 - } - }, - { - "id": "yfzp/Llama-3-8B-Instruct-SPPO-Iter1_gp_2b-table", - "name": "Llama-3-8B-Instruct-SPPO-Iter1_gp_2b-table", - "developer": "yfzp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6785, - "hfopenllm_v2/BBH": 0.4941, - "hfopenllm_v2/MATH Level 5": 0.1125, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3647, - "hfopenllm_v2/MMLU-PRO": 0.3718 - } - }, - { - "id": "yfzp/Llama-3-8B-Instruct-SPPO-Iter1_gp_8b-table", - "name": "Llama-3-8B-Instruct-SPPO-Iter1_gp_8b-table", - "developer": "yfzp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7132, - "hfopenllm_v2/BBH": 0.5025, - "hfopenllm_v2/MATH Level 5": 0.0989, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3713, - "hfopenllm_v2/MMLU-PRO": 0.3683 - } - }, - { - "id": "yfzp/Llama-3-8B-Instruct-SPPO-score-Iter1_bt_2b-table-0.001", - "name": "Llama-3-8B-Instruct-SPPO-score-Iter1_bt_2b-table-0.001", - "developer": "yfzp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6496, - "hfopenllm_v2/BBH": 0.4979, - "hfopenllm_v2/MATH Level 5": 0.1012, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.378, - "hfopenllm_v2/MMLU-PRO": 0.372 - } - }, - { - "id": "yfzp/Llama-3-8B-Instruct-SPPO-score-Iter1_bt_8b-table-0.002", - "name": "Llama-3-8B-Instruct-SPPO-score-Iter1_bt_8b-table-0.002", - "developer": "yfzp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7196, - "hfopenllm_v2/BBH": 0.5045, - "hfopenllm_v2/MATH Level 5": 0.0876, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3831, - "hfopenllm_v2/MMLU-PRO": 0.3734 - } - }, - { - "id": "yfzp/Llama-3-8B-Instruct-SPPO-score-Iter1_gp_2b-table-0.001", - "name": "Llama-3-8B-Instruct-SPPO-score-Iter1_gp_2b-table-0.001", - "developer": "yfzp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6504, - "hfopenllm_v2/BBH": 0.4958, - "hfopenllm_v2/MATH Level 5": 0.0937, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.366, - "hfopenllm_v2/MMLU-PRO": 0.3703 - } - }, - { - "id": "yfzp/Llama-3-8B-Instruct-SPPO-score-Iter1_gp_8b-table-0.002", - "name": "Llama-3-8B-Instruct-SPPO-score-Iter1_gp_8b-table-0.002", - "developer": "yfzp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7016, - "hfopenllm_v2/BBH": 0.4992, - "hfopenllm_v2/MATH Level 5": 0.0869, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3779, - "hfopenllm_v2/MMLU-PRO": 0.3669 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/yifai.json b/data/developers/yifai.json deleted file mode 100644 index 0ffcb54953079f72e995c1ad81eb2c4aeb958202..0000000000000000000000000000000000000000 --- a/data/developers/yifai.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "yifAI", - "models": [ - { - "id": "yifAI/Llama-3-8B-Instruct-SPPO-score-Iter3_gp_8b-table-0.002", - "name": "Llama-3-8B-Instruct-SPPO-score-Iter3_gp_8b-table-0.002", - "developer": "yifAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.649, - "hfopenllm_v2/BBH": 0.4915, - "hfopenllm_v2/MATH Level 5": 0.0755, - "hfopenllm_v2/GPQA": 0.2617, - "hfopenllm_v2/MUSR": 0.3899, - "hfopenllm_v2/MMLU-PRO": 0.352 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/ylalain.json b/data/developers/ylalain.json deleted file mode 100644 index 4f5ac89537e2a3fe1a315987b6c78585ba8a18c6..0000000000000000000000000000000000000000 --- a/data/developers/ylalain.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "ylalain", - "models": [ - { - "id": "ylalain/ECE-PRYMMAL-YL-1B-SLERP-V8", - "name": "ECE-PRYMMAL-YL-1B-SLERP-V8", - "developer": "ylalain", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1505, - "hfopenllm_v2/BBH": 0.3976, - "hfopenllm_v2/MATH Level 5": 0.0045, - "hfopenllm_v2/GPQA": 0.2894, - "hfopenllm_v2/MUSR": 0.3875, - "hfopenllm_v2/MMLU-PRO": 0.2384 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/ymcki.json b/data/developers/ymcki.json deleted file mode 100644 index 522152b195db93b535cb213cee3863371f59f211..0000000000000000000000000000000000000000 --- a/data/developers/ymcki.json +++ /dev/null @@ -1,159 +0,0 @@ -{ - "developer": "ymcki", - "models": [ - { - "id": "ymcki/gemma-2-2b-jpn-it-abliterated-17", - "name": "gemma-2-2b-jpn-it-abliterated-17", - "developer": "ymcki", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5082, - "hfopenllm_v2/BBH": 0.4076, - "hfopenllm_v2/MATH Level 5": 0.0385, - "hfopenllm_v2/GPQA": 0.2718, - "hfopenllm_v2/MUSR": 0.3701, - "hfopenllm_v2/MMLU-PRO": 0.2455 - } - }, - { - "id": "ymcki/gemma-2-2b-jpn-it-abliterated-17-18-24", - "name": "gemma-2-2b-jpn-it-abliterated-17-18-24", - "developer": "ymcki", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5055, - "hfopenllm_v2/BBH": 0.3812, - "hfopenllm_v2/MATH Level 5": 0.0257, - "hfopenllm_v2/GPQA": 0.281, - "hfopenllm_v2/MUSR": 0.3502, - "hfopenllm_v2/MMLU-PRO": 0.2282 - } - }, - { - "id": "ymcki/gemma-2-2b-jpn-it-abliterated-17-ORPO", - "name": "gemma-2-2b-jpn-it-abliterated-17-ORPO", - "developer": "ymcki", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4748, - "hfopenllm_v2/BBH": 0.3898, - "hfopenllm_v2/MATH Level 5": 0.0619, - "hfopenllm_v2/GPQA": 0.2743, - "hfopenllm_v2/MUSR": 0.3768, - "hfopenllm_v2/MMLU-PRO": 0.2191 - } - }, - { - "id": "ymcki/gemma-2-2b-jpn-it-abliterated-17-ORPO-alpaca", - "name": "gemma-2-2b-jpn-it-abliterated-17-ORPO-alpaca", - "developer": "ymcki", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3065, - "hfopenllm_v2/BBH": 0.4072, - "hfopenllm_v2/MATH Level 5": 0.0325, - "hfopenllm_v2/GPQA": 0.2693, - "hfopenllm_v2/MUSR": 0.3969, - "hfopenllm_v2/MMLU-PRO": 0.2249 - } - }, - { - "id": "ymcki/gemma-2-2b-jpn-it-abliterated-18", - "name": "gemma-2-2b-jpn-it-abliterated-18", - "developer": "ymcki", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5175, - "hfopenllm_v2/BBH": 0.4132, - "hfopenllm_v2/MATH Level 5": 0.0446, - "hfopenllm_v2/GPQA": 0.2735, - "hfopenllm_v2/MUSR": 0.3742, - "hfopenllm_v2/MMLU-PRO": 0.2505 - } - }, - { - "id": "ymcki/gemma-2-2b-jpn-it-abliterated-18-ORPO", - "name": "gemma-2-2b-jpn-it-abliterated-18-ORPO", - "developer": "ymcki", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4742, - "hfopenllm_v2/BBH": 0.4039, - "hfopenllm_v2/MATH Level 5": 0.0468, - "hfopenllm_v2/GPQA": 0.2617, - "hfopenllm_v2/MUSR": 0.3953, - "hfopenllm_v2/MMLU-PRO": 0.2185 - } - }, - { - "id": "ymcki/gemma-2-2b-jpn-it-abliterated-24", - "name": "gemma-2-2b-jpn-it-abliterated-24", - "developer": "ymcki", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4979, - "hfopenllm_v2/BBH": 0.411, - "hfopenllm_v2/MATH Level 5": 0.0438, - "hfopenllm_v2/GPQA": 0.2777, - "hfopenllm_v2/MUSR": 0.3915, - "hfopenllm_v2/MMLU-PRO": 0.2473 - } - }, - { - "id": "ymcki/gemma-2-2b-ORPO-jpn-it-abliterated-18", - "name": "gemma-2-2b-ORPO-jpn-it-abliterated-18", - "developer": "ymcki", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4631, - "hfopenllm_v2/BBH": 0.4053, - "hfopenllm_v2/MATH Level 5": 0.0431, - "hfopenllm_v2/GPQA": 0.2886, - "hfopenllm_v2/MUSR": 0.3754, - "hfopenllm_v2/MMLU-PRO": 0.2345 - } - }, - { - "id": "ymcki/gemma-2-2b-ORPO-jpn-it-abliterated-18-merge", - "name": "gemma-2-2b-ORPO-jpn-it-abliterated-18-merge", - "developer": "ymcki", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5218, - "hfopenllm_v2/BBH": 0.4147, - "hfopenllm_v2/MATH Level 5": 0.0544, - "hfopenllm_v2/GPQA": 0.2836, - "hfopenllm_v2/MUSR": 0.3514, - "hfopenllm_v2/MMLU-PRO": 0.2461 - } - }, - { - "id": "ymcki/Llama-3.1-8B-GRPO-Instruct", - "name": "Llama-3.1-8B-GRPO-Instruct", - "developer": "ymcki", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7445, - "hfopenllm_v2/BBH": 0.5132, - "hfopenllm_v2/MATH Level 5": 0.2024, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.3817, - "hfopenllm_v2/MMLU-PRO": 0.3738 - } - }, - { - "id": "ymcki/Llama-3.1-8B-SFT-GRPO-Instruct", - "name": "Llama-3.1-8B-SFT-GRPO-Instruct", - "developer": "ymcki", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3354, - "hfopenllm_v2/BBH": 0.3126, - "hfopenllm_v2/MATH Level 5": 0.04, - "hfopenllm_v2/GPQA": 0.2534, - "hfopenllm_v2/MUSR": 0.3526, - "hfopenllm_v2/MMLU-PRO": 0.1098 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/youlln.json b/data/developers/youlln.json deleted file mode 100644 index b5a516e1321358bc2316ecb79a0a15334b9c7292..0000000000000000000000000000000000000000 --- a/data/developers/youlln.json +++ /dev/null @@ -1,271 +0,0 @@ -{ - "developer": "Youlln", - "models": [ - { - "id": "Youlln/1PARAMMYL-8B-ModelStock", - "name": "1PARAMMYL-8B-ModelStock", - "developer": "Youlln", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5371, - "hfopenllm_v2/BBH": 0.5216, - "hfopenllm_v2/MATH Level 5": 0.1488, - "hfopenllm_v2/GPQA": 0.3238, - "hfopenllm_v2/MUSR": 0.4409, - "hfopenllm_v2/MMLU-PRO": 0.4 - } - }, - { - "id": "Youlln/2PRYMMAL-Yi1.5-6B-SLERP", - "name": "2PRYMMAL-Yi1.5-6B-SLERP", - "developer": "Youlln", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2826, - "hfopenllm_v2/BBH": 0.4665, - "hfopenllm_v2/MATH Level 5": 0.1133, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.4756, - "hfopenllm_v2/MMLU-PRO": 0.317 - } - }, - { - "id": "Youlln/3PRYMMAL-PHI3-3B-SLERP", - "name": "3PRYMMAL-PHI3-3B-SLERP", - "developer": "Youlln", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3656, - "hfopenllm_v2/BBH": 0.5422, - "hfopenllm_v2/MATH Level 5": 0.1715, - "hfopenllm_v2/GPQA": 0.3263, - "hfopenllm_v2/MUSR": 0.4648, - "hfopenllm_v2/MMLU-PRO": 0.4002 - } - }, - { - "id": "Youlln/4PRYMMAL-GEMMA2-9B-SLERP", - "name": "4PRYMMAL-GEMMA2-9B-SLERP", - "developer": "Youlln", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2714, - "hfopenllm_v2/BBH": 0.5923, - "hfopenllm_v2/MATH Level 5": 0.0906, - "hfopenllm_v2/GPQA": 0.3305, - "hfopenllm_v2/MUSR": 0.4672, - "hfopenllm_v2/MMLU-PRO": 0.421 - } - }, - { - "id": "Youlln/ECE-MIRAGE-1-12B", - "name": "ECE-MIRAGE-1-12B", - "developer": "Youlln", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.207, - "hfopenllm_v2/BBH": 0.3011, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2634, - "hfopenllm_v2/MUSR": 0.3219, - "hfopenllm_v2/MMLU-PRO": 0.111 - } - }, - { - "id": "Youlln/ECE-MIRAGE-1-15B", - "name": "ECE-MIRAGE-1-15B", - "developer": "Youlln", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.207, - "hfopenllm_v2/BBH": 0.3011, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2634, - "hfopenllm_v2/MUSR": 0.3219, - "hfopenllm_v2/MMLU-PRO": 0.111 - } - }, - { - "id": "Youlln/ECE-PRYMMAL-0.5B-FT-V3", - "name": "ECE-PRYMMAL-0.5B-FT-V3", - "developer": "Youlln", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1642, - "hfopenllm_v2/BBH": 0.3093, - "hfopenllm_v2/MATH Level 5": 0.003, - "hfopenllm_v2/GPQA": 0.2576, - "hfopenllm_v2/MUSR": 0.3644, - "hfopenllm_v2/MMLU-PRO": 0.1161 - } - }, - { - "id": "Youlln/ECE-PRYMMAL-0.5B-FT-V3-MUSR", - "name": "ECE-PRYMMAL-0.5B-FT-V3-MUSR", - "developer": "Youlln", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1533, - "hfopenllm_v2/BBH": 0.3041, - "hfopenllm_v2/MATH Level 5": 0.0242, - "hfopenllm_v2/GPQA": 0.2492, - "hfopenllm_v2/MUSR": 0.366, - "hfopenllm_v2/MMLU-PRO": 0.1645 - } - }, - { - "id": "Youlln/ECE-PRYMMAL-0.5B-FT-V4-MUSR", - "name": "ECE-PRYMMAL-0.5B-FT-V4-MUSR", - "developer": "Youlln", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1138, - "hfopenllm_v2/BBH": 0.3038, - "hfopenllm_v2/MATH Level 5": 0.0121, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.3529, - "hfopenllm_v2/MMLU-PRO": 0.1321 - } - }, - { - "id": "Youlln/ECE-PRYMMAL-0.5B-SLERP-V2", - "name": "ECE-PRYMMAL-0.5B-SLERP-V2", - "developer": "Youlln", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1612, - "hfopenllm_v2/BBH": 0.2935, - "hfopenllm_v2/MATH Level 5": 0.0008, - "hfopenllm_v2/GPQA": 0.2743, - "hfopenllm_v2/MUSR": 0.3831, - "hfopenllm_v2/MMLU-PRO": 0.1095 - } - }, - { - "id": "Youlln/ECE-PRYMMAL-0.5B-SLERP-V3", - "name": "ECE-PRYMMAL-0.5B-SLERP-V3", - "developer": "Youlln", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.167, - "hfopenllm_v2/BBH": 0.2938, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2517, - "hfopenllm_v2/MUSR": 0.3541, - "hfopenllm_v2/MMLU-PRO": 0.1087 - } - }, - { - "id": "Youlln/ECE-PRYMMAL-YL-1B-SLERP-V1", - "name": "ECE-PRYMMAL-YL-1B-SLERP-V1", - "developer": "Youlln", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3251, - "hfopenllm_v2/BBH": 0.4209, - "hfopenllm_v2/MATH Level 5": 0.1073, - "hfopenllm_v2/GPQA": 0.2911, - "hfopenllm_v2/MUSR": 0.4266, - "hfopenllm_v2/MMLU-PRO": 0.2936 - } - }, - { - "id": "Youlln/ECE-PRYMMAL-YL-1B-SLERP-V2", - "name": "ECE-PRYMMAL-YL-1B-SLERP-V2", - "developer": "Youlln", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3251, - "hfopenllm_v2/BBH": 0.4209, - "hfopenllm_v2/MATH Level 5": 0.1073, - "hfopenllm_v2/GPQA": 0.2911, - "hfopenllm_v2/MUSR": 0.4266, - "hfopenllm_v2/MMLU-PRO": 0.2936 - } - }, - { - "id": "Youlln/ECE-PRYMMAL-YL-7B-SLERP-V4", - "name": "ECE-PRYMMAL-YL-7B-SLERP-V4", - "developer": "Youlln", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.251, - "hfopenllm_v2/BBH": 0.377, - "hfopenllm_v2/MATH Level 5": 0.0536, - "hfopenllm_v2/GPQA": 0.2651, - "hfopenllm_v2/MUSR": 0.3745, - "hfopenllm_v2/MMLU-PRO": 0.2132 - } - }, - { - "id": "Youlln/ECE-PRYMMAL0.5-FT", - "name": "ECE-PRYMMAL0.5-FT", - "developer": "Youlln", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1851, - "hfopenllm_v2/BBH": 0.3132, - "hfopenllm_v2/MATH Level 5": 0.0234, - "hfopenllm_v2/GPQA": 0.2559, - "hfopenllm_v2/MUSR": 0.3301, - "hfopenllm_v2/MMLU-PRO": 0.1477 - } - }, - { - "id": "Youlln/ECE-PRYMMAL0.5B-Youri", - "name": "ECE-PRYMMAL0.5B-Youri", - "developer": "Youlln", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1446, - "hfopenllm_v2/BBH": 0.2817, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2433, - "hfopenllm_v2/MUSR": 0.3697, - "hfopenllm_v2/MMLU-PRO": 0.1095 - } - }, - { - "id": "Youlln/ECE-PRYMMAL1B-FT-V1", - "name": "ECE-PRYMMAL1B-FT-V1", - "developer": "Youlln", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2144, - "hfopenllm_v2/BBH": 0.4033, - "hfopenllm_v2/MATH Level 5": 0.0642, - "hfopenllm_v2/GPQA": 0.2785, - "hfopenllm_v2/MUSR": 0.3417, - "hfopenllm_v2/MMLU-PRO": 0.2743 - } - }, - { - "id": "Youlln/ECE-Qwen0.5B-FT-V2", - "name": "ECE-Qwen0.5B-FT-V2", - "developer": "Youlln", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2526, - "hfopenllm_v2/BBH": 0.329, - "hfopenllm_v2/MATH Level 5": 0.0204, - "hfopenllm_v2/GPQA": 0.2668, - "hfopenllm_v2/MUSR": 0.3063, - "hfopenllm_v2/MMLU-PRO": 0.1666 - } - }, - { - "id": "Youlln/ECE.EIFFEIL.ia-0.5B-SLERP", - "name": "ECE.EIFFEIL.ia-0.5B-SLERP", - "developer": "Youlln", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2561, - "hfopenllm_v2/BBH": 0.3306, - "hfopenllm_v2/MATH Level 5": 0.0597, - "hfopenllm_v2/GPQA": 0.2651, - "hfopenllm_v2/MUSR": 0.3102, - "hfopenllm_v2/MMLU-PRO": 0.1903 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/youngpanda.json b/data/developers/youngpanda.json deleted file mode 100644 index a2cbd319801c160fef029841a7e5ed06a606a524..0000000000000000000000000000000000000000 --- a/data/developers/youngpanda.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "YoungPanda", - "models": [ - { - "id": "YoungPanda/qwenqwen", - "name": "qwenqwen", - "developer": "YoungPanda", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1264, - "hfopenllm_v2/BBH": 0.3379, - "hfopenllm_v2/MATH Level 5": 0.0355, - "hfopenllm_v2/GPQA": 0.25, - "hfopenllm_v2/MUSR": 0.3434, - "hfopenllm_v2/MMLU-PRO": 0.1168 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/yoyo-ai.json b/data/developers/yoyo-ai.json deleted file mode 100644 index 9dcbc81e86094f67bbf3f24e1917205fb4cc44f8..0000000000000000000000000000000000000000 --- a/data/developers/yoyo-ai.json +++ /dev/null @@ -1,299 +0,0 @@ -{ - "developer": "YOYO-AI", - "models": [ - { - "id": "YOYO-AI/Qwen2.5-14B-1M-YOYO-V3", - "name": "Qwen2.5-14B-1M-YOYO-V3", - "developer": "YOYO-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8398, - "hfopenllm_v2/BBH": 0.6448, - "hfopenllm_v2/MATH Level 5": 0.5355, - "hfopenllm_v2/GPQA": 0.3289, - "hfopenllm_v2/MUSR": 0.4141, - "hfopenllm_v2/MMLU-PRO": 0.5207 - } - }, - { - "id": "YOYO-AI/Qwen2.5-14B-it-restore", - "name": "Qwen2.5-14B-it-restore", - "developer": "YOYO-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8209, - "hfopenllm_v2/BBH": 0.6388, - "hfopenllm_v2/MATH Level 5": 0.537, - "hfopenllm_v2/GPQA": 0.3372, - "hfopenllm_v2/MUSR": 0.4087, - "hfopenllm_v2/MMLU-PRO": 0.49 - } - }, - { - "id": "YOYO-AI/Qwen2.5-14B-YOYO-0505", - "name": "Qwen2.5-14B-YOYO-0505", - "developer": "YOYO-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5883, - "hfopenllm_v2/BBH": 0.6539, - "hfopenllm_v2/MATH Level 5": 0.4434, - "hfopenllm_v2/GPQA": 0.3733, - "hfopenllm_v2/MUSR": 0.4757, - "hfopenllm_v2/MMLU-PRO": 0.5371 - } - }, - { - "id": "YOYO-AI/Qwen2.5-14B-YOYO-0510-v2", - "name": "Qwen2.5-14B-YOYO-0510-v2", - "developer": "YOYO-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5947, - "hfopenllm_v2/BBH": 0.6553, - "hfopenllm_v2/MATH Level 5": 0.4441, - "hfopenllm_v2/GPQA": 0.3817, - "hfopenllm_v2/MUSR": 0.4744, - "hfopenllm_v2/MMLU-PRO": 0.5381 - } - }, - { - "id": "YOYO-AI/Qwen2.5-14B-YOYO-0805", - "name": "Qwen2.5-14B-YOYO-0805", - "developer": "YOYO-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5883, - "hfopenllm_v2/BBH": 0.6539, - "hfopenllm_v2/MATH Level 5": 0.4434, - "hfopenllm_v2/GPQA": 0.3733, - "hfopenllm_v2/MUSR": 0.4757, - "hfopenllm_v2/MMLU-PRO": 0.5371 - } - }, - { - "id": "YOYO-AI/Qwen2.5-14B-YOYO-1005", - "name": "Qwen2.5-14B-YOYO-1005", - "developer": "YOYO-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5972, - "hfopenllm_v2/BBH": 0.6542, - "hfopenllm_v2/MATH Level 5": 0.4524, - "hfopenllm_v2/GPQA": 0.3809, - "hfopenllm_v2/MUSR": 0.473, - "hfopenllm_v2/MMLU-PRO": 0.5382 - } - }, - { - "id": "YOYO-AI/Qwen2.5-14B-YOYO-1005-v2", - "name": "Qwen2.5-14B-YOYO-1005-v2", - "developer": "YOYO-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5953, - "hfopenllm_v2/BBH": 0.6551, - "hfopenllm_v2/MATH Level 5": 0.4434, - "hfopenllm_v2/GPQA": 0.3842, - "hfopenllm_v2/MUSR": 0.4731, - "hfopenllm_v2/MMLU-PRO": 0.5372 - } - }, - { - "id": "YOYO-AI/Qwen2.5-14B-YOYO-1010", - "name": "Qwen2.5-14B-YOYO-1010", - "developer": "YOYO-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5899, - "hfopenllm_v2/BBH": 0.654, - "hfopenllm_v2/MATH Level 5": 0.4509, - "hfopenllm_v2/GPQA": 0.3834, - "hfopenllm_v2/MUSR": 0.4744, - "hfopenllm_v2/MMLU-PRO": 0.5376 - } - }, - { - "id": "YOYO-AI/Qwen2.5-14B-YOYO-1010-v2", - "name": "Qwen2.5-14B-YOYO-1010-v2", - "developer": "YOYO-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5947, - "hfopenllm_v2/BBH": 0.6553, - "hfopenllm_v2/MATH Level 5": 0.4441, - "hfopenllm_v2/GPQA": 0.3817, - "hfopenllm_v2/MUSR": 0.4744, - "hfopenllm_v2/MMLU-PRO": 0.5381 - } - }, - { - "id": "YOYO-AI/Qwen2.5-14B-YOYO-latest", - "name": "Qwen2.5-14B-YOYO-latest", - "developer": "YOYO-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5911, - "hfopenllm_v2/BBH": 0.6656, - "hfopenllm_v2/MATH Level 5": 0.4418, - "hfopenllm_v2/GPQA": 0.3826, - "hfopenllm_v2/MUSR": 0.4691, - "hfopenllm_v2/MMLU-PRO": 0.5371 - } - }, - { - "id": "YOYO-AI/Qwen2.5-14B-YOYO-latest-V2", - "name": "Qwen2.5-14B-YOYO-latest-V2", - "developer": "YOYO-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7771, - "hfopenllm_v2/BBH": 0.6299, - "hfopenllm_v2/MATH Level 5": 0.5159, - "hfopenllm_v2/GPQA": 0.354, - "hfopenllm_v2/MUSR": 0.4299, - "hfopenllm_v2/MMLU-PRO": 0.5224 - } - }, - { - "id": "YOYO-AI/Qwen2.5-14B-YOYO-SCE", - "name": "Qwen2.5-14B-YOYO-SCE", - "developer": "YOYO-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5844, - "hfopenllm_v2/BBH": 0.6489, - "hfopenllm_v2/MATH Level 5": 0.4615, - "hfopenllm_v2/GPQA": 0.3742, - "hfopenllm_v2/MUSR": 0.4704, - "hfopenllm_v2/MMLU-PRO": 0.5381 - } - }, - { - "id": "YOYO-AI/Qwen2.5-14B-YOYO-V4", - "name": "Qwen2.5-14B-YOYO-V4", - "developer": "YOYO-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8398, - "hfopenllm_v2/BBH": 0.649, - "hfopenllm_v2/MATH Level 5": 0.5347, - "hfopenllm_v2/GPQA": 0.3221, - "hfopenllm_v2/MUSR": 0.4115, - "hfopenllm_v2/MMLU-PRO": 0.517 - } - }, - { - "id": "YOYO-AI/Qwen2.5-14B-YOYO-V4-p1", - "name": "Qwen2.5-14B-YOYO-V4-p1", - "developer": "YOYO-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8203, - "hfopenllm_v2/BBH": 0.6516, - "hfopenllm_v2/MATH Level 5": 0.5332, - "hfopenllm_v2/GPQA": 0.3456, - "hfopenllm_v2/MUSR": 0.4194, - "hfopenllm_v2/MMLU-PRO": 0.502 - } - }, - { - "id": "YOYO-AI/Qwen2.5-14B-YOYO-V4-p2", - "name": "Qwen2.5-14B-YOYO-V4-p2", - "developer": "YOYO-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8048, - "hfopenllm_v2/BBH": 0.6339, - "hfopenllm_v2/MATH Level 5": 0.5166, - "hfopenllm_v2/GPQA": 0.3272, - "hfopenllm_v2/MUSR": 0.4435, - "hfopenllm_v2/MMLU-PRO": 0.4968 - } - }, - { - "id": "YOYO-AI/Qwen2.5-7B-it-restore", - "name": "Qwen2.5-7B-it-restore", - "developer": "YOYO-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7531, - "hfopenllm_v2/BBH": 0.5407, - "hfopenllm_v2/MATH Level 5": 0.5, - "hfopenllm_v2/GPQA": 0.3012, - "hfopenllm_v2/MUSR": 0.4007, - "hfopenllm_v2/MMLU-PRO": 0.4288 - } - }, - { - "id": "YOYO-AI/Qwen2.5-Coder-14B-YOYO-1010", - "name": "Qwen2.5-Coder-14B-YOYO-1010", - "developer": "YOYO-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5336, - "hfopenllm_v2/BBH": 0.6187, - "hfopenllm_v2/MATH Level 5": 0.3218, - "hfopenllm_v2/GPQA": 0.3523, - "hfopenllm_v2/MUSR": 0.4422, - "hfopenllm_v2/MMLU-PRO": 0.4075 - } - }, - { - "id": "YOYO-AI/ZYH-LLM-Qwen2.5-14B", - "name": "ZYH-LLM-Qwen2.5-14B", - "developer": "YOYO-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5941, - "hfopenllm_v2/BBH": 0.6644, - "hfopenllm_v2/MATH Level 5": 0.4116, - "hfopenllm_v2/GPQA": 0.3859, - "hfopenllm_v2/MUSR": 0.4757, - "hfopenllm_v2/MMLU-PRO": 0.5351 - } - }, - { - "id": "YOYO-AI/ZYH-LLM-Qwen2.5-14B-V2", - "name": "ZYH-LLM-Qwen2.5-14B-V2", - "developer": "YOYO-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5071, - "hfopenllm_v2/BBH": 0.6452, - "hfopenllm_v2/MATH Level 5": 0.3542, - "hfopenllm_v2/GPQA": 0.3792, - "hfopenllm_v2/MUSR": 0.4689, - "hfopenllm_v2/MMLU-PRO": 0.5372 - } - }, - { - "id": "YOYO-AI/ZYH-LLM-Qwen2.5-14B-V3", - "name": "ZYH-LLM-Qwen2.5-14B-V3", - "developer": "YOYO-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8578, - "hfopenllm_v2/BBH": 0.6359, - "hfopenllm_v2/MATH Level 5": 0.5272, - "hfopenllm_v2/GPQA": 0.3322, - "hfopenllm_v2/MUSR": 0.4022, - "hfopenllm_v2/MMLU-PRO": 0.4881 - } - }, - { - "id": "YOYO-AI/ZYH-LLM-Qwen2.5-14B-V4", - "name": "ZYH-LLM-Qwen2.5-14B-V4", - "developer": "YOYO-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8365, - "hfopenllm_v2/BBH": 0.6515, - "hfopenllm_v2/MATH Level 5": 0.5393, - "hfopenllm_v2/GPQA": 0.3146, - "hfopenllm_v2/MUSR": 0.4434, - "hfopenllm_v2/MMLU-PRO": 0.5204 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/yuchenxie.json b/data/developers/yuchenxie.json deleted file mode 100644 index 34dd07aac11cacaf468d8ba95f3cd57442859f6c..0000000000000000000000000000000000000000 --- a/data/developers/yuchenxie.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "yuchenxie", - "models": [ - { - "id": "yuchenxie/ArlowGPT-3B-Multilingual", - "name": "ArlowGPT-3B-Multilingual", - "developer": "yuchenxie", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6395, - "hfopenllm_v2/BBH": 0.4301, - "hfopenllm_v2/MATH Level 5": 0.1125, - "hfopenllm_v2/GPQA": 0.2802, - "hfopenllm_v2/MUSR": 0.3727, - "hfopenllm_v2/MMLU-PRO": 0.2817 - } - }, - { - "id": "yuchenxie/ArlowGPT-8B", - "name": "ArlowGPT-8B", - "developer": "yuchenxie", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7847, - "hfopenllm_v2/BBH": 0.508, - "hfopenllm_v2/MATH Level 5": 0.2039, - "hfopenllm_v2/GPQA": 0.2936, - "hfopenllm_v2/MUSR": 0.3882, - "hfopenllm_v2/MMLU-PRO": 0.3787 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/yuma42.json b/data/developers/yuma42.json deleted file mode 100644 index 86fa973f880ed9c54d4e33501ca47dd3bc9324bc..0000000000000000000000000000000000000000 --- a/data/developers/yuma42.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "developer": "Yuma42", - "models": [ - { - "id": "Yuma42/KangalKhan-RawRuby-7B", - "name": "KangalKhan-RawRuby-7B", - "developer": "Yuma42", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5477, - "hfopenllm_v2/BBH": 0.4755, - "hfopenllm_v2/MATH Level 5": 0.0665, - "hfopenllm_v2/GPQA": 0.2878, - "hfopenllm_v2/MUSR": 0.395, - "hfopenllm_v2/MMLU-PRO": 0.3023 - } - }, - { - "id": "Yuma42/Llama3.1-IgneousIguana-8B", - "name": "Llama3.1-IgneousIguana-8B", - "developer": "Yuma42", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8133, - "hfopenllm_v2/BBH": 0.5191, - "hfopenllm_v2/MATH Level 5": 0.2198, - "hfopenllm_v2/GPQA": 0.3104, - "hfopenllm_v2/MUSR": 0.4203, - "hfopenllm_v2/MMLU-PRO": 0.3974 - } - }, - { - "id": "Yuma42/Llama3.1-SuperHawk-8B", - "name": "Llama3.1-SuperHawk-8B", - "developer": "Yuma42", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7986, - "hfopenllm_v2/BBH": 0.52, - "hfopenllm_v2/MATH Level 5": 0.2349, - "hfopenllm_v2/GPQA": 0.3129, - "hfopenllm_v2/MUSR": 0.4084, - "hfopenllm_v2/MMLU-PRO": 0.3945 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/yuvraj17.json b/data/developers/yuvraj17.json deleted file mode 100644 index a8f978afc031af7ed9f5351099fc921cabd75e4d..0000000000000000000000000000000000000000 --- a/data/developers/yuvraj17.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "developer": "yuvraj17", - "models": [ - { - "id": "yuvraj17/Llama3-8B-abliterated-Spectrum-slerp", - "name": "Llama3-8B-abliterated-Spectrum-slerp", - "developer": "yuvraj17", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2885, - "hfopenllm_v2/BBH": 0.4978, - "hfopenllm_v2/MATH Level 5": 0.0604, - "hfopenllm_v2/GPQA": 0.3012, - "hfopenllm_v2/MUSR": 0.3998, - "hfopenllm_v2/MMLU-PRO": 0.3257 - } - }, - { - "id": "yuvraj17/Llama3-8B-SuperNova-Spectrum-dare_ties", - "name": "Llama3-8B-SuperNova-Spectrum-dare_ties", - "developer": "yuvraj17", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4013, - "hfopenllm_v2/BBH": 0.4616, - "hfopenllm_v2/MATH Level 5": 0.0846, - "hfopenllm_v2/GPQA": 0.2752, - "hfopenllm_v2/MUSR": 0.4211, - "hfopenllm_v2/MMLU-PRO": 0.3574 - } - }, - { - "id": "yuvraj17/Llama3-8B-SuperNova-Spectrum-Hermes-DPO", - "name": "Llama3-8B-SuperNova-Spectrum-Hermes-DPO", - "developer": "yuvraj17", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4691, - "hfopenllm_v2/BBH": 0.44, - "hfopenllm_v2/MATH Level 5": 0.0566, - "hfopenllm_v2/GPQA": 0.302, - "hfopenllm_v2/MUSR": 0.4012, - "hfopenllm_v2/MMLU-PRO": 0.2635 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/z-ai.json b/data/developers/z-ai.json deleted file mode 100644 index d4ada9599b117d1b4893fef821933bb44c4fb1d9..0000000000000000000000000000000000000000 --- a/data/developers/z-ai.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "developer": "Z-AI", - "models": [ - { - "id": "zhipu-ai/glm-4.7", - "name": "GLM 4.7", - "developer": "Z-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "terminal-bench-2.0/terminal-bench-2.0": 33.3 - } - }, - { - "id": "zhipu-ai/glm-5", - "name": "GLM 5", - "developer": "Z-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "terminal-bench-2.0/terminal-bench-2.0": 52.4 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/z.ai.json b/data/developers/z.ai.json deleted file mode 100644 index 8921fb0501217b4c718467219685dda1990f317c..0000000000000000000000000000000000000000 --- a/data/developers/z.ai.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "developer": "Z.ai", - "models": [ - { - "id": "z-ai/glm-4.5", - "name": "z-ai/glm-4.5", - "developer": "Z.ai", - "evaluator_relationship": null, - "benchmark_scores": { - "livecodebenchpro/Hard Problems": 0.0, - "livecodebenchpro/Medium Problems": 0.028169014084507043, - "livecodebenchpro/Easy Problems": 0.1267605633802817 - } - }, - { - "id": "zhipu-ai/glm-4.6", - "name": "GLM 4.6", - "developer": "Z.ai", - "evaluator_relationship": null, - "benchmark_scores": { - "terminal-bench-2.0/terminal-bench-2.0": 24.5 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/z1-coder.json b/data/developers/z1-coder.json deleted file mode 100644 index 0ddc003961050c2c441db68bf3a0a10aa617937d..0000000000000000000000000000000000000000 --- a/data/developers/z1-coder.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "Z1-Coder", - "models": [ - { - "id": "Z1-Coder/Z1-Coder-7B", - "name": "Z1-Coder-7B", - "developer": "Z1-Coder", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3215, - "hfopenllm_v2/BBH": 0.4842, - "hfopenllm_v2/MATH Level 5": 0.3248, - "hfopenllm_v2/GPQA": 0.2727, - "hfopenllm_v2/MUSR": 0.3622, - "hfopenllm_v2/MMLU-PRO": 0.3759 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/zai-org.json b/data/developers/zai-org.json deleted file mode 100644 index 97d318c43138cf1428c836472cd08b522e456bd3..0000000000000000000000000000000000000000 --- a/data/developers/zai-org.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "zai-org", - "models": [ - { - "id": "zai-org/glm-4.5-air-fp8", - "name": "GLM-4.5-Air-FP8", - "developer": "zai-org", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.67, - "helm_capabilities/MMLU-Pro": 0.762, - "helm_capabilities/GPQA": 0.594, - "helm_capabilities/IFEval": 0.812, - "helm_capabilities/WildBench": 0.789, - "helm_capabilities/Omni-MATH": 0.391 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/zake7749.json b/data/developers/zake7749.json deleted file mode 100644 index a63543b72cbd283b991e4cdc1c436c2e985c9c93..0000000000000000000000000000000000000000 --- a/data/developers/zake7749.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "zake7749", - "models": [ - { - "id": "zake7749/gemma-2-2b-it-chinese-kyara-dpo", - "name": "gemma-2-2b-it-chinese-kyara-dpo", - "developer": "zake7749", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5382, - "hfopenllm_v2/BBH": 0.4257, - "hfopenllm_v2/MATH Level 5": 0.0838, - "hfopenllm_v2/GPQA": 0.2668, - "hfopenllm_v2/MUSR": 0.4576, - "hfopenllm_v2/MMLU-PRO": 0.2573 - } - }, - { - "id": "zake7749/gemma-2-9b-it-chinese-kyara", - "name": "gemma-2-9b-it-chinese-kyara", - "developer": "zake7749", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1764, - "hfopenllm_v2/BBH": 0.5954, - "hfopenllm_v2/MATH Level 5": 0.105, - "hfopenllm_v2/GPQA": 0.3381, - "hfopenllm_v2/MUSR": 0.4242, - "hfopenllm_v2/MMLU-PRO": 0.4179 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/zelk12.json b/data/developers/zelk12.json deleted file mode 100644 index 7354c9ad3962812f9191091f1806b21899dae4e6..0000000000000000000000000000000000000000 --- a/data/developers/zelk12.json +++ /dev/null @@ -1,1097 +0,0 @@ -{ - "developer": "zelk12", - "models": [ - { - "id": "zelk12/gemma-2-S2MTM-9B", - "name": "gemma-2-S2MTM-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7823, - "hfopenllm_v2/BBH": 0.6061, - "hfopenllm_v2/MATH Level 5": 0.2047, - "hfopenllm_v2/GPQA": 0.3456, - "hfopenllm_v2/MUSR": 0.4218, - "hfopenllm_v2/MMLU-PRO": 0.4297 - } - }, - { - "id": "zelk12/Gemma-2-TM-9B", - "name": "Gemma-2-TM-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8045, - "hfopenllm_v2/BBH": 0.5987, - "hfopenllm_v2/MATH Level 5": 0.2024, - "hfopenllm_v2/GPQA": 0.3465, - "hfopenllm_v2/MUSR": 0.4152, - "hfopenllm_v2/MMLU-PRO": 0.4088 - } - }, - { - "id": "zelk12/MT-gemma-2-9B", - "name": "MT-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7968, - "hfopenllm_v2/BBH": 0.6064, - "hfopenllm_v2/MATH Level 5": 0.2054, - "hfopenllm_v2/GPQA": 0.3456, - "hfopenllm_v2/MUSR": 0.4071, - "hfopenllm_v2/MMLU-PRO": 0.4224 - } - }, - { - "id": "zelk12/MT-Gen1-gemma-2-9B", - "name": "MT-Gen1-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7886, - "hfopenllm_v2/BBH": 0.61, - "hfopenllm_v2/MATH Level 5": 0.2221, - "hfopenllm_v2/GPQA": 0.3465, - "hfopenllm_v2/MUSR": 0.4217, - "hfopenllm_v2/MMLU-PRO": 0.4381 - } - }, - { - "id": "zelk12/MT-Gen2-gemma-2-9B", - "name": "MT-Gen2-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7907, - "hfopenllm_v2/BBH": 0.61, - "hfopenllm_v2/MATH Level 5": 0.219, - "hfopenllm_v2/GPQA": 0.3465, - "hfopenllm_v2/MUSR": 0.4323, - "hfopenllm_v2/MMLU-PRO": 0.4387 - } - }, - { - "id": "zelk12/MT-Gen2-GI-gemma-2-9B", - "name": "MT-Gen2-GI-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7914, - "hfopenllm_v2/BBH": 0.6096, - "hfopenllm_v2/MATH Level 5": 0.2205, - "hfopenllm_v2/GPQA": 0.3507, - "hfopenllm_v2/MUSR": 0.4283, - "hfopenllm_v2/MMLU-PRO": 0.4356 - } - }, - { - "id": "zelk12/MT-Gen3-gemma-2-9B", - "name": "MT-Gen3-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.802, - "hfopenllm_v2/BBH": 0.6097, - "hfopenllm_v2/MATH Level 5": 0.2296, - "hfopenllm_v2/GPQA": 0.349, - "hfopenllm_v2/MUSR": 0.4217, - "hfopenllm_v2/MMLU-PRO": 0.4356 - } - }, - { - "id": "zelk12/MT-Gen4-gemma-2-9B", - "name": "MT-Gen4-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7883, - "hfopenllm_v2/BBH": 0.611, - "hfopenllm_v2/MATH Level 5": 0.2236, - "hfopenllm_v2/GPQA": 0.3549, - "hfopenllm_v2/MUSR": 0.4228, - "hfopenllm_v2/MMLU-PRO": 0.4387 - } - }, - { - "id": "zelk12/MT-Gen5-gemma-2-9B", - "name": "MT-Gen5-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7923, - "hfopenllm_v2/BBH": 0.6133, - "hfopenllm_v2/MATH Level 5": 0.2153, - "hfopenllm_v2/GPQA": 0.3515, - "hfopenllm_v2/MUSR": 0.4202, - "hfopenllm_v2/MMLU-PRO": 0.4402 - } - }, - { - "id": "zelk12/MT-Gen6-gemma-2-9B", - "name": "MT-Gen6-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1616, - "hfopenllm_v2/BBH": 0.5845, - "hfopenllm_v2/MATH Level 5": 0.0823, - "hfopenllm_v2/GPQA": 0.3331, - "hfopenllm_v2/MUSR": 0.4069, - "hfopenllm_v2/MMLU-PRO": 0.4166 - } - }, - { - "id": "zelk12/MT-Gen6fix-gemma-2-9B", - "name": "MT-Gen6fix-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1576, - "hfopenllm_v2/BBH": 0.5917, - "hfopenllm_v2/MATH Level 5": 0.0816, - "hfopenllm_v2/GPQA": 0.3372, - "hfopenllm_v2/MUSR": 0.4084, - "hfopenllm_v2/MMLU-PRO": 0.412 - } - }, - { - "id": "zelk12/MT-Gen7-gemma-2-9B", - "name": "MT-Gen7-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1664, - "hfopenllm_v2/BBH": 0.5935, - "hfopenllm_v2/MATH Level 5": 0.0891, - "hfopenllm_v2/GPQA": 0.3356, - "hfopenllm_v2/MUSR": 0.4098, - "hfopenllm_v2/MMLU-PRO": 0.4122 - } - }, - { - "id": "zelk12/MT-Max-Merge_02012025163610-gemma-2-9B", - "name": "MT-Max-Merge_02012025163610-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7907, - "hfopenllm_v2/BBH": 0.6142, - "hfopenllm_v2/MATH Level 5": 0.2213, - "hfopenllm_v2/GPQA": 0.3515, - "hfopenllm_v2/MUSR": 0.4228, - "hfopenllm_v2/MMLU-PRO": 0.4396 - } - }, - { - "id": "zelk12/MT-Merge-gemma-2-9B", - "name": "MT-Merge-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8035, - "hfopenllm_v2/BBH": 0.6118, - "hfopenllm_v2/MATH Level 5": 0.2205, - "hfopenllm_v2/GPQA": 0.3482, - "hfopenllm_v2/MUSR": 0.4256, - "hfopenllm_v2/MMLU-PRO": 0.4362 - } - }, - { - "id": "zelk12/MT-Merge1-gemma-2-9B", - "name": "MT-Merge1-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7901, - "hfopenllm_v2/BBH": 0.61, - "hfopenllm_v2/MATH Level 5": 0.2289, - "hfopenllm_v2/GPQA": 0.3515, - "hfopenllm_v2/MUSR": 0.4244, - "hfopenllm_v2/MMLU-PRO": 0.4374 - } - }, - { - "id": "zelk12/MT-Merge2-gemma-2-9B", - "name": "MT-Merge2-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7877, - "hfopenllm_v2/BBH": 0.6107, - "hfopenllm_v2/MATH Level 5": 0.2349, - "hfopenllm_v2/GPQA": 0.3507, - "hfopenllm_v2/MUSR": 0.4217, - "hfopenllm_v2/MMLU-PRO": 0.4382 - } - }, - { - "id": "zelk12/MT-Merge2-MU-gemma-2-MTg2MT1g2-9B", - "name": "MT-Merge2-MU-gemma-2-MTg2MT1g2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7956, - "hfopenllm_v2/BBH": 0.6084, - "hfopenllm_v2/MATH Level 5": 0.2183, - "hfopenllm_v2/GPQA": 0.3507, - "hfopenllm_v2/MUSR": 0.4322, - "hfopenllm_v2/MMLU-PRO": 0.4373 - } - }, - { - "id": "zelk12/MT-Merge3-gemma-2-9B", - "name": "MT-Merge3-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7859, - "hfopenllm_v2/BBH": 0.6102, - "hfopenllm_v2/MATH Level 5": 0.2205, - "hfopenllm_v2/GPQA": 0.349, - "hfopenllm_v2/MUSR": 0.4258, - "hfopenllm_v2/MMLU-PRO": 0.4373 - } - }, - { - "id": "zelk12/MT-Merge4-gemma-2-9B", - "name": "MT-Merge4-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7807, - "hfopenllm_v2/BBH": 0.6118, - "hfopenllm_v2/MATH Level 5": 0.2168, - "hfopenllm_v2/GPQA": 0.3523, - "hfopenllm_v2/MUSR": 0.4294, - "hfopenllm_v2/MMLU-PRO": 0.439 - } - }, - { - "id": "zelk12/MT-Merge5-gemma-2-9B", - "name": "MT-Merge5-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7844, - "hfopenllm_v2/BBH": 0.6123, - "hfopenllm_v2/MATH Level 5": 0.2183, - "hfopenllm_v2/GPQA": 0.3532, - "hfopenllm_v2/MUSR": 0.4281, - "hfopenllm_v2/MMLU-PRO": 0.4387 - } - }, - { - "id": "zelk12/MT-Merge6-gemma-2-9B", - "name": "MT-Merge6-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1695, - "hfopenllm_v2/BBH": 0.5949, - "hfopenllm_v2/MATH Level 5": 0.0801, - "hfopenllm_v2/GPQA": 0.3289, - "hfopenllm_v2/MUSR": 0.4098, - "hfopenllm_v2/MMLU-PRO": 0.4115 - } - }, - { - "id": "zelk12/MT1-gemma-2-9B", - "name": "MT1-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7947, - "hfopenllm_v2/BBH": 0.6109, - "hfopenllm_v2/MATH Level 5": 0.2236, - "hfopenllm_v2/GPQA": 0.3456, - "hfopenllm_v2/MUSR": 0.4322, - "hfopenllm_v2/MMLU-PRO": 0.4358 - } - }, - { - "id": "zelk12/MT1-Gen1-gemma-2-9B", - "name": "MT1-Gen1-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7974, - "hfopenllm_v2/BBH": 0.6118, - "hfopenllm_v2/MATH Level 5": 0.2243, - "hfopenllm_v2/GPQA": 0.344, - "hfopenllm_v2/MUSR": 0.431, - "hfopenllm_v2/MMLU-PRO": 0.4376 - } - }, - { - "id": "zelk12/MT1-Gen2-gemma-2-9B", - "name": "MT1-Gen2-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7984, - "hfopenllm_v2/BBH": 0.6096, - "hfopenllm_v2/MATH Level 5": 0.2251, - "hfopenllm_v2/GPQA": 0.3523, - "hfopenllm_v2/MUSR": 0.4284, - "hfopenllm_v2/MMLU-PRO": 0.4355 - } - }, - { - "id": "zelk12/MT1-Gen3-gemma-2-9B", - "name": "MT1-Gen3-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.796, - "hfopenllm_v2/BBH": 0.6102, - "hfopenllm_v2/MATH Level 5": 0.2243, - "hfopenllm_v2/GPQA": 0.349, - "hfopenllm_v2/MUSR": 0.4243, - "hfopenllm_v2/MMLU-PRO": 0.4349 - } - }, - { - "id": "zelk12/MT1-Gen4-gemma-2-9B", - "name": "MT1-Gen4-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7941, - "hfopenllm_v2/BBH": 0.6058, - "hfopenllm_v2/MATH Level 5": 0.216, - "hfopenllm_v2/GPQA": 0.3473, - "hfopenllm_v2/MUSR": 0.4231, - "hfopenllm_v2/MMLU-PRO": 0.4286 - } - }, - { - "id": "zelk12/MT1-Gen5-gemma-2-9B", - "name": "MT1-Gen5-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7795, - "hfopenllm_v2/BBH": 0.6017, - "hfopenllm_v2/MATH Level 5": 0.2077, - "hfopenllm_v2/GPQA": 0.3465, - "hfopenllm_v2/MUSR": 0.4191, - "hfopenllm_v2/MMLU-PRO": 0.4222 - } - }, - { - "id": "zelk12/MT1-Gen5-IF-gemma-2-S2DMv1-9B", - "name": "MT1-Gen5-IF-gemma-2-S2DMv1-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7929, - "hfopenllm_v2/BBH": 0.6, - "hfopenllm_v2/MATH Level 5": 0.2032, - "hfopenllm_v2/GPQA": 0.344, - "hfopenllm_v2/MUSR": 0.4245, - "hfopenllm_v2/MMLU-PRO": 0.4218 - } - }, - { - "id": "zelk12/MT1-Gen6-gemma-2-9B", - "name": "MT1-Gen6-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1634, - "hfopenllm_v2/BBH": 0.5944, - "hfopenllm_v2/MATH Level 5": 0.0808, - "hfopenllm_v2/GPQA": 0.328, - "hfopenllm_v2/MUSR": 0.4044, - "hfopenllm_v2/MMLU-PRO": 0.4133 - } - }, - { - "id": "zelk12/MT1-Gen7-gemma-2-9B", - "name": "MT1-Gen7-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1634, - "hfopenllm_v2/BBH": 0.5938, - "hfopenllm_v2/MATH Level 5": 0.0831, - "hfopenllm_v2/GPQA": 0.328, - "hfopenllm_v2/MUSR": 0.4111, - "hfopenllm_v2/MMLU-PRO": 0.4145 - } - }, - { - "id": "zelk12/MT1-Max-Merge_02012025163610-gemma-2-9B", - "name": "MT1-Max-Merge_02012025163610-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7929, - "hfopenllm_v2/BBH": 0.6123, - "hfopenllm_v2/MATH Level 5": 0.2228, - "hfopenllm_v2/GPQA": 0.3549, - "hfopenllm_v2/MUSR": 0.4255, - "hfopenllm_v2/MMLU-PRO": 0.4382 - } - }, - { - "id": "zelk12/MT2-gemma-2-9B", - "name": "MT2-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7886, - "hfopenllm_v2/BBH": 0.6115, - "hfopenllm_v2/MATH Level 5": 0.2213, - "hfopenllm_v2/GPQA": 0.3473, - "hfopenllm_v2/MUSR": 0.4217, - "hfopenllm_v2/MMLU-PRO": 0.4368 - } - }, - { - "id": "zelk12/MT2-Gen1-gemma-2-9B", - "name": "MT2-Gen1-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7856, - "hfopenllm_v2/BBH": 0.6101, - "hfopenllm_v2/MATH Level 5": 0.2213, - "hfopenllm_v2/GPQA": 0.3431, - "hfopenllm_v2/MUSR": 0.4243, - "hfopenllm_v2/MMLU-PRO": 0.4377 - } - }, - { - "id": "zelk12/MT2-Gen2-gemma-2-9B", - "name": "MT2-Gen2-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7889, - "hfopenllm_v2/BBH": 0.6093, - "hfopenllm_v2/MATH Level 5": 0.2183, - "hfopenllm_v2/GPQA": 0.3465, - "hfopenllm_v2/MUSR": 0.427, - "hfopenllm_v2/MMLU-PRO": 0.4388 - } - }, - { - "id": "zelk12/MT2-Gen3-gemma-2-9B", - "name": "MT2-Gen3-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.781, - "hfopenllm_v2/BBH": 0.6105, - "hfopenllm_v2/MATH Level 5": 0.2107, - "hfopenllm_v2/GPQA": 0.3465, - "hfopenllm_v2/MUSR": 0.4231, - "hfopenllm_v2/MMLU-PRO": 0.4374 - } - }, - { - "id": "zelk12/MT2-Gen4-gemma-2-9B", - "name": "MT2-Gen4-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7896, - "hfopenllm_v2/BBH": 0.6097, - "hfopenllm_v2/MATH Level 5": 0.2236, - "hfopenllm_v2/GPQA": 0.3456, - "hfopenllm_v2/MUSR": 0.4125, - "hfopenllm_v2/MMLU-PRO": 0.4321 - } - }, - { - "id": "zelk12/MT2-Gen5-gemma-2-9B", - "name": "MT2-Gen5-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7749, - "hfopenllm_v2/BBH": 0.6064, - "hfopenllm_v2/MATH Level 5": 0.2107, - "hfopenllm_v2/GPQA": 0.3515, - "hfopenllm_v2/MUSR": 0.4244, - "hfopenllm_v2/MMLU-PRO": 0.4302 - } - }, - { - "id": "zelk12/MT2-Gen6-gemma-2-9B", - "name": "MT2-Gen6-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1664, - "hfopenllm_v2/BBH": 0.596, - "hfopenllm_v2/MATH Level 5": 0.0846, - "hfopenllm_v2/GPQA": 0.3381, - "hfopenllm_v2/MUSR": 0.4137, - "hfopenllm_v2/MMLU-PRO": 0.421 - } - }, - { - "id": "zelk12/MT2-Gen7-gemma-2-9B", - "name": "MT2-Gen7-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1762, - "hfopenllm_v2/BBH": 0.6079, - "hfopenllm_v2/MATH Level 5": 0.102, - "hfopenllm_v2/GPQA": 0.3549, - "hfopenllm_v2/MUSR": 0.4203, - "hfopenllm_v2/MMLU-PRO": 0.4311 - } - }, - { - "id": "zelk12/MT2-Max-Merge_02012025163610-gemma-2-9B", - "name": "MT2-Max-Merge_02012025163610-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7901, - "hfopenllm_v2/BBH": 0.6108, - "hfopenllm_v2/MATH Level 5": 0.2243, - "hfopenllm_v2/GPQA": 0.3515, - "hfopenllm_v2/MUSR": 0.4228, - "hfopenllm_v2/MMLU-PRO": 0.4391 - } - }, - { - "id": "zelk12/MT3-gemma-2-9B", - "name": "MT3-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7786, - "hfopenllm_v2/BBH": 0.6131, - "hfopenllm_v2/MATH Level 5": 0.2168, - "hfopenllm_v2/GPQA": 0.3448, - "hfopenllm_v2/MUSR": 0.4243, - "hfopenllm_v2/MMLU-PRO": 0.4327 - } - }, - { - "id": "zelk12/MT3-Gen1-gemma-2-9B", - "name": "MT3-Gen1-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7838, - "hfopenllm_v2/BBH": 0.6107, - "hfopenllm_v2/MATH Level 5": 0.2145, - "hfopenllm_v2/GPQA": 0.3465, - "hfopenllm_v2/MUSR": 0.4151, - "hfopenllm_v2/MMLU-PRO": 0.4327 - } - }, - { - "id": "zelk12/MT3-Gen2-gemma-2-9B", - "name": "MT3-Gen2-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7843, - "hfopenllm_v2/BBH": 0.6091, - "hfopenllm_v2/MATH Level 5": 0.2236, - "hfopenllm_v2/GPQA": 0.3574, - "hfopenllm_v2/MUSR": 0.4111, - "hfopenllm_v2/MMLU-PRO": 0.4333 - } - }, - { - "id": "zelk12/MT3-Gen3-gemma-2-9B", - "name": "MT3-Gen3-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7856, - "hfopenllm_v2/BBH": 0.6089, - "hfopenllm_v2/MATH Level 5": 0.2153, - "hfopenllm_v2/GPQA": 0.3515, - "hfopenllm_v2/MUSR": 0.4258, - "hfopenllm_v2/MMLU-PRO": 0.4303 - } - }, - { - "id": "zelk12/MT3-Gen4-gemma-2-9B", - "name": "MT3-Gen4-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7737, - "hfopenllm_v2/BBH": 0.6101, - "hfopenllm_v2/MATH Level 5": 0.2062, - "hfopenllm_v2/GPQA": 0.3473, - "hfopenllm_v2/MUSR": 0.4476, - "hfopenllm_v2/MMLU-PRO": 0.4387 - } - }, - { - "id": "zelk12/MT3-Gen5-gemma-2-9B", - "name": "MT3-Gen5-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.799, - "hfopenllm_v2/BBH": 0.6099, - "hfopenllm_v2/MATH Level 5": 0.2266, - "hfopenllm_v2/GPQA": 0.3532, - "hfopenllm_v2/MUSR": 0.4191, - "hfopenllm_v2/MMLU-PRO": 0.4317 - } - }, - { - "id": "zelk12/MT3-Gen5-gemma-2-9B_v1", - "name": "MT3-Gen5-gemma-2-9B_v1", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7996, - "hfopenllm_v2/BBH": 0.6113, - "hfopenllm_v2/MATH Level 5": 0.2228, - "hfopenllm_v2/GPQA": 0.349, - "hfopenllm_v2/MUSR": 0.4204, - "hfopenllm_v2/MMLU-PRO": 0.4359 - } - }, - { - "id": "zelk12/MT3-Gen6-gemma-2-9B", - "name": "MT3-Gen6-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1762, - "hfopenllm_v2/BBH": 0.602, - "hfopenllm_v2/MATH Level 5": 0.0884, - "hfopenllm_v2/GPQA": 0.3431, - "hfopenllm_v2/MUSR": 0.4126, - "hfopenllm_v2/MMLU-PRO": 0.4102 - } - }, - { - "id": "zelk12/MT3-Max-Merge_02012025163610-gemma-2-9B", - "name": "MT3-Max-Merge_02012025163610-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1762, - "hfopenllm_v2/BBH": 0.6123, - "hfopenllm_v2/MATH Level 5": 0.1012, - "hfopenllm_v2/GPQA": 0.3507, - "hfopenllm_v2/MUSR": 0.4255, - "hfopenllm_v2/MMLU-PRO": 0.4389 - } - }, - { - "id": "zelk12/MT4-gemma-2-9B", - "name": "MT4-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7762, - "hfopenllm_v2/BBH": 0.6073, - "hfopenllm_v2/MATH Level 5": 0.2085, - "hfopenllm_v2/GPQA": 0.3381, - "hfopenllm_v2/MUSR": 0.4309, - "hfopenllm_v2/MMLU-PRO": 0.4366 - } - }, - { - "id": "zelk12/MT4-Gen1-gemma-2-9B", - "name": "MT4-Gen1-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7895, - "hfopenllm_v2/BBH": 0.6094, - "hfopenllm_v2/MATH Level 5": 0.2198, - "hfopenllm_v2/GPQA": 0.344, - "hfopenllm_v2/MUSR": 0.4322, - "hfopenllm_v2/MMLU-PRO": 0.4389 - } - }, - { - "id": "zelk12/MT4-Gen2-gemma-2-9B", - "name": "MT4-Gen2-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8051, - "hfopenllm_v2/BBH": 0.6108, - "hfopenllm_v2/MATH Level 5": 0.2326, - "hfopenllm_v2/GPQA": 0.3456, - "hfopenllm_v2/MUSR": 0.4257, - "hfopenllm_v2/MMLU-PRO": 0.4368 - } - }, - { - "id": "zelk12/MT4-Gen3-gemma-2-9B", - "name": "MT4-Gen3-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7841, - "hfopenllm_v2/BBH": 0.6087, - "hfopenllm_v2/MATH Level 5": 0.219, - "hfopenllm_v2/GPQA": 0.344, - "hfopenllm_v2/MUSR": 0.4243, - "hfopenllm_v2/MMLU-PRO": 0.4381 - } - }, - { - "id": "zelk12/MT4-Gen4-gemma-2-9B", - "name": "MT4-Gen4-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7874, - "hfopenllm_v2/BBH": 0.6076, - "hfopenllm_v2/MATH Level 5": 0.2145, - "hfopenllm_v2/GPQA": 0.3523, - "hfopenllm_v2/MUSR": 0.4244, - "hfopenllm_v2/MMLU-PRO": 0.4323 - } - }, - { - "id": "zelk12/MT4-Gen5-gemma-2-9B", - "name": "MT4-Gen5-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7789, - "hfopenllm_v2/BBH": 0.6107, - "hfopenllm_v2/MATH Level 5": 0.2266, - "hfopenllm_v2/GPQA": 0.3565, - "hfopenllm_v2/MUSR": 0.4268, - "hfopenllm_v2/MMLU-PRO": 0.4384 - } - }, - { - "id": "zelk12/MT4-Max-Merge_02012025163610-gemma-2-9B", - "name": "MT4-Max-Merge_02012025163610-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1771, - "hfopenllm_v2/BBH": 0.612, - "hfopenllm_v2/MATH Level 5": 0.0952, - "hfopenllm_v2/GPQA": 0.3515, - "hfopenllm_v2/MUSR": 0.4228, - "hfopenllm_v2/MMLU-PRO": 0.4391 - } - }, - { - "id": "zelk12/MT5-gemma-2-9B", - "name": "MT5-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8048, - "hfopenllm_v2/BBH": 0.6112, - "hfopenllm_v2/MATH Level 5": 0.2258, - "hfopenllm_v2/GPQA": 0.3431, - "hfopenllm_v2/MUSR": 0.4204, - "hfopenllm_v2/MMLU-PRO": 0.4367 - } - }, - { - "id": "zelk12/MT5-Gen1-gemma-2-9B", - "name": "MT5-Gen1-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7831, - "hfopenllm_v2/BBH": 0.611, - "hfopenllm_v2/MATH Level 5": 0.2213, - "hfopenllm_v2/GPQA": 0.3473, - "hfopenllm_v2/MUSR": 0.4204, - "hfopenllm_v2/MMLU-PRO": 0.4368 - } - }, - { - "id": "zelk12/MT5-Gen2-gemma-2-9B", - "name": "MT5-Gen2-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7962, - "hfopenllm_v2/BBH": 0.6105, - "hfopenllm_v2/MATH Level 5": 0.2205, - "hfopenllm_v2/GPQA": 0.3515, - "hfopenllm_v2/MUSR": 0.4163, - "hfopenllm_v2/MMLU-PRO": 0.4379 - } - }, - { - "id": "zelk12/MT5-Gen3-gemma-2-9B", - "name": "MT5-Gen3-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7825, - "hfopenllm_v2/BBH": 0.609, - "hfopenllm_v2/MATH Level 5": 0.2168, - "hfopenllm_v2/GPQA": 0.3515, - "hfopenllm_v2/MUSR": 0.4231, - "hfopenllm_v2/MMLU-PRO": 0.4375 - } - }, - { - "id": "zelk12/MT5-Gen4-gemma-2-9B", - "name": "MT5-Gen4-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7835, - "hfopenllm_v2/BBH": 0.6131, - "hfopenllm_v2/MATH Level 5": 0.2243, - "hfopenllm_v2/GPQA": 0.3532, - "hfopenllm_v2/MUSR": 0.4228, - "hfopenllm_v2/MMLU-PRO": 0.4397 - } - }, - { - "id": "zelk12/MT5-Gen5-gemma-2-9B", - "name": "MT5-Gen5-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7947, - "hfopenllm_v2/BBH": 0.6112, - "hfopenllm_v2/MATH Level 5": 0.2258, - "hfopenllm_v2/GPQA": 0.3482, - "hfopenllm_v2/MUSR": 0.4191, - "hfopenllm_v2/MMLU-PRO": 0.4329 - } - }, - { - "id": "zelk12/MT5-Max-Merge_02012025163610-gemma-2-9B", - "name": "MT5-Max-Merge_02012025163610-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1762, - "hfopenllm_v2/BBH": 0.6127, - "hfopenllm_v2/MATH Level 5": 0.0982, - "hfopenllm_v2/GPQA": 0.3515, - "hfopenllm_v2/MUSR": 0.4228, - "hfopenllm_v2/MMLU-PRO": 0.439 - } - }, - { - "id": "zelk12/MTM-Merge-gemma-2-9B", - "name": "MTM-Merge-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7798, - "hfopenllm_v2/BBH": 0.6133, - "hfopenllm_v2/MATH Level 5": 0.2175, - "hfopenllm_v2/GPQA": 0.3549, - "hfopenllm_v2/MUSR": 0.4268, - "hfopenllm_v2/MMLU-PRO": 0.4388 - } - }, - { - "id": "zelk12/MTMaMe-Merge_02012025163610-gemma-2-9B", - "name": "MTMaMe-Merge_02012025163610-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1786, - "hfopenllm_v2/BBH": 0.6117, - "hfopenllm_v2/MATH Level 5": 0.0959, - "hfopenllm_v2/GPQA": 0.3523, - "hfopenllm_v2/MUSR": 0.4241, - "hfopenllm_v2/MMLU-PRO": 0.4382 - } - }, - { - "id": "zelk12/recoilme-gemma-2-Ataraxy-9B-v0.1", - "name": "recoilme-gemma-2-Ataraxy-9B-v0.1", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7649, - "hfopenllm_v2/BBH": 0.6075, - "hfopenllm_v2/MATH Level 5": 0.2281, - "hfopenllm_v2/GPQA": 0.3498, - "hfopenllm_v2/MUSR": 0.4136, - "hfopenllm_v2/MMLU-PRO": 0.4321 - } - }, - { - "id": "zelk12/recoilme-gemma-2-Ataraxy-9B-v0.1-t0.25", - "name": "recoilme-gemma-2-Ataraxy-9B-v0.1-t0.25", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7707, - "hfopenllm_v2/BBH": 0.6075, - "hfopenllm_v2/MATH Level 5": 0.2145, - "hfopenllm_v2/GPQA": 0.3431, - "hfopenllm_v2/MUSR": 0.4323, - "hfopenllm_v2/MMLU-PRO": 0.44 - } - }, - { - "id": "zelk12/recoilme-gemma-2-Ataraxy-9B-v0.1-t0.75", - "name": "recoilme-gemma-2-Ataraxy-9B-v0.1-t0.75", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7208, - "hfopenllm_v2/BBH": 0.5995, - "hfopenllm_v2/MATH Level 5": 0.2017, - "hfopenllm_v2/GPQA": 0.3498, - "hfopenllm_v2/MUSR": 0.3951, - "hfopenllm_v2/MMLU-PRO": 0.4141 - } - }, - { - "id": "zelk12/recoilme-gemma-2-Ataraxy-9B-v0.2", - "name": "recoilme-gemma-2-Ataraxy-9B-v0.2", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.76, - "hfopenllm_v2/BBH": 0.6066, - "hfopenllm_v2/MATH Level 5": 0.2228, - "hfopenllm_v2/GPQA": 0.3482, - "hfopenllm_v2/MUSR": 0.411, - "hfopenllm_v2/MMLU-PRO": 0.4323 - } - }, - { - "id": "zelk12/recoilme-gemma-2-Gutenberg-Doppel-9B-v0.1", - "name": "recoilme-gemma-2-Gutenberg-Doppel-9B-v0.1", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7615, - "hfopenllm_v2/BBH": 0.6099, - "hfopenllm_v2/MATH Level 5": 0.21, - "hfopenllm_v2/GPQA": 0.3414, - "hfopenllm_v2/MUSR": 0.431, - "hfopenllm_v2/MMLU-PRO": 0.4315 - } - }, - { - "id": "zelk12/recoilme-gemma-2-Ifable-9B-v0.1", - "name": "recoilme-gemma-2-Ifable-9B-v0.1", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7944, - "hfopenllm_v2/BBH": 0.6064, - "hfopenllm_v2/MATH Level 5": 0.2205, - "hfopenllm_v2/GPQA": 0.3515, - "hfopenllm_v2/MUSR": 0.4202, - "hfopenllm_v2/MMLU-PRO": 0.4323 - } - }, - { - "id": "zelk12/recoilme-gemma-2-psy10k-mental_healt-9B-v0.1", - "name": "recoilme-gemma-2-psy10k-mental_healt-9B-v0.1", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7445, - "hfopenllm_v2/BBH": 0.5978, - "hfopenllm_v2/MATH Level 5": 0.1888, - "hfopenllm_v2/GPQA": 0.344, - "hfopenllm_v2/MUSR": 0.4295, - "hfopenllm_v2/MMLU-PRO": 0.4181 - } - }, - { - "id": "zelk12/Rv0.4DMv1t0.25-gemma-2-9B", - "name": "Rv0.4DMv1t0.25-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7497, - "hfopenllm_v2/BBH": 0.607, - "hfopenllm_v2/MATH Level 5": 0.2258, - "hfopenllm_v2/GPQA": 0.3456, - "hfopenllm_v2/MUSR": 0.4309, - "hfopenllm_v2/MMLU-PRO": 0.4401 - } - }, - { - "id": "zelk12/Rv0.4DMv1t0.25Tt0.25-gemma-2-9B", - "name": "Rv0.4DMv1t0.25Tt0.25-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7646, - "hfopenllm_v2/BBH": 0.6098, - "hfopenllm_v2/MATH Level 5": 0.2069, - "hfopenllm_v2/GPQA": 0.3423, - "hfopenllm_v2/MUSR": 0.4283, - "hfopenllm_v2/MMLU-PRO": 0.4347 - } - }, - { - "id": "zelk12/Rv0.4MT4g2-gemma-2-9B", - "name": "Rv0.4MT4g2-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.732, - "hfopenllm_v2/BBH": 0.6041, - "hfopenllm_v2/MATH Level 5": 0.1949, - "hfopenllm_v2/GPQA": 0.3532, - "hfopenllm_v2/MUSR": 0.4231, - "hfopenllm_v2/MMLU-PRO": 0.4417 - } - }, - { - "id": "zelk12/T31122024203920-gemma-2-9B", - "name": "T31122024203920-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7676, - "hfopenllm_v2/BBH": 0.6096, - "hfopenllm_v2/MATH Level 5": 0.2054, - "hfopenllm_v2/GPQA": 0.3507, - "hfopenllm_v2/MUSR": 0.4322, - "hfopenllm_v2/MMLU-PRO": 0.4373 - } - }, - { - "id": "zelk12/Test01012025155054", - "name": "Test01012025155054", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1555, - "hfopenllm_v2/BBH": 0.283, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2416, - "hfopenllm_v2/MUSR": 0.367, - "hfopenllm_v2/MMLU-PRO": 0.109 - } - }, - { - "id": "zelk12/Test01012025155054t0.5_gemma-2", - "name": "Test01012025155054t0.5_gemma-2", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1555, - "hfopenllm_v2/BBH": 0.283, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2416, - "hfopenllm_v2/MUSR": 0.367, - "hfopenllm_v2/MMLU-PRO": 0.109 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/zeroxclem.json b/data/developers/zeroxclem.json deleted file mode 100644 index 3da31c4394d2fcde462d5ea16ed3a93f4dfc601d..0000000000000000000000000000000000000000 --- a/data/developers/zeroxclem.json +++ /dev/null @@ -1,159 +0,0 @@ -{ - "developer": "ZeroXClem", - "models": [ - { - "id": "ZeroXClem/L3-Aspire-Heart-Matrix-8B", - "name": "L3-Aspire-Heart-Matrix-8B", - "developer": "ZeroXClem", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4834, - "hfopenllm_v2/BBH": 0.5384, - "hfopenllm_v2/MATH Level 5": 0.1828, - "hfopenllm_v2/GPQA": 0.3247, - "hfopenllm_v2/MUSR": 0.4187, - "hfopenllm_v2/MMLU-PRO": 0.3785 - } - }, - { - "id": "ZeroXClem/Llama-3.1-8B-AthenaSky-MegaMix", - "name": "Llama-3.1-8B-AthenaSky-MegaMix", - "developer": "ZeroXClem", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6301, - "hfopenllm_v2/BBH": 0.5163, - "hfopenllm_v2/MATH Level 5": 0.2795, - "hfopenllm_v2/GPQA": 0.2777, - "hfopenllm_v2/MUSR": 0.3538, - "hfopenllm_v2/MMLU-PRO": 0.3504 - } - }, - { - "id": "ZeroXClem/Llama-3.1-8B-RainbowLight-EtherealMix", - "name": "Llama-3.1-8B-RainbowLight-EtherealMix", - "developer": "ZeroXClem", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4973, - "hfopenllm_v2/BBH": 0.5155, - "hfopenllm_v2/MATH Level 5": 0.1216, - "hfopenllm_v2/GPQA": 0.2869, - "hfopenllm_v2/MUSR": 0.3947, - "hfopenllm_v2/MMLU-PRO": 0.363 - } - }, - { - "id": "ZeroXClem/Llama-3.1-8B-SpecialTitanFusion", - "name": "Llama-3.1-8B-SpecialTitanFusion", - "developer": "ZeroXClem", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7402, - "hfopenllm_v2/BBH": 0.5439, - "hfopenllm_v2/MATH Level 5": 0.2334, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.3874, - "hfopenllm_v2/MMLU-PRO": 0.3621 - } - }, - { - "id": "ZeroXClem/Llama-3.1-8B-SuperNova-EtherealHermes", - "name": "Llama-3.1-8B-SuperNova-EtherealHermes", - "developer": "ZeroXClem", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7339, - "hfopenllm_v2/BBH": 0.5244, - "hfopenllm_v2/MATH Level 5": 0.1745, - "hfopenllm_v2/GPQA": 0.2928, - "hfopenllm_v2/MUSR": 0.4066, - "hfopenllm_v2/MMLU-PRO": 0.3745 - } - }, - { - "id": "ZeroXClem/Llama-3.1-8B-SuperTulu-LexiNova", - "name": "Llama-3.1-8B-SuperTulu-LexiNova", - "developer": "ZeroXClem", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4165, - "hfopenllm_v2/BBH": 0.5079, - "hfopenllm_v2/MATH Level 5": 0.253, - "hfopenllm_v2/GPQA": 0.2861, - "hfopenllm_v2/MUSR": 0.3971, - "hfopenllm_v2/MMLU-PRO": 0.3368 - } - }, - { - "id": "ZeroXClem/Qwen-2.5-Aether-SlerpFusion-7B", - "name": "Qwen-2.5-Aether-SlerpFusion-7B", - "developer": "ZeroXClem", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6262, - "hfopenllm_v2/BBH": 0.5462, - "hfopenllm_v2/MATH Level 5": 0.2734, - "hfopenllm_v2/GPQA": 0.2987, - "hfopenllm_v2/MUSR": 0.4178, - "hfopenllm_v2/MMLU-PRO": 0.4327 - } - }, - { - "id": "ZeroXClem/Qwen2.5-7B-CelestialHarmony-1M", - "name": "Qwen2.5-7B-CelestialHarmony-1M", - "developer": "ZeroXClem", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5944, - "hfopenllm_v2/BBH": 0.5431, - "hfopenllm_v2/MATH Level 5": 0.3474, - "hfopenllm_v2/GPQA": 0.3188, - "hfopenllm_v2/MUSR": 0.4595, - "hfopenllm_v2/MMLU-PRO": 0.4387 - } - }, - { - "id": "ZeroXClem/Qwen2.5-7B-HomerAnvita-NerdMix", - "name": "Qwen2.5-7B-HomerAnvita-NerdMix", - "developer": "ZeroXClem", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7708, - "hfopenllm_v2/BBH": 0.5541, - "hfopenllm_v2/MATH Level 5": 0.3837, - "hfopenllm_v2/GPQA": 0.3196, - "hfopenllm_v2/MUSR": 0.4391, - "hfopenllm_v2/MMLU-PRO": 0.4432 - } - }, - { - "id": "ZeroXClem/Qwen2.5-7B-HomerCreative-Mix", - "name": "Qwen2.5-7B-HomerCreative-Mix", - "developer": "ZeroXClem", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7835, - "hfopenllm_v2/BBH": 0.5548, - "hfopenllm_v2/MATH Level 5": 0.3565, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.435, - "hfopenllm_v2/MMLU-PRO": 0.4447 - } - }, - { - "id": "ZeroXClem/Qwen2.5-7B-Qandora-CySec", - "name": "Qwen2.5-7B-Qandora-CySec", - "developer": "ZeroXClem", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6773, - "hfopenllm_v2/BBH": 0.549, - "hfopenllm_v2/MATH Level 5": 0.2931, - "hfopenllm_v2/GPQA": 0.3003, - "hfopenllm_v2/MUSR": 0.4286, - "hfopenllm_v2/MMLU-PRO": 0.4485 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/zetasepic.json b/data/developers/zetasepic.json deleted file mode 100644 index 1086f4e5f9c03e4c2dcb6387043dd4fb20abddf5..0000000000000000000000000000000000000000 --- a/data/developers/zetasepic.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "zetasepic", - "models": [ - { - "id": "zetasepic/Qwen2.5-32B-Instruct-abliterated-v2", - "name": "Qwen2.5-32B-Instruct-abliterated-v2", - "developer": "zetasepic", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8334, - "hfopenllm_v2/BBH": 0.6934, - "hfopenllm_v2/MATH Level 5": 0.5952, - "hfopenllm_v2/GPQA": 0.3674, - "hfopenllm_v2/MUSR": 0.4354, - "hfopenllm_v2/MMLU-PRO": 0.5622 - } - }, - { - "id": "zetasepic/Qwen2.5-72B-Instruct-abliterated", - "name": "Qwen2.5-72B-Instruct-abliterated", - "developer": "zetasepic", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7153, - "hfopenllm_v2/BBH": 0.7152, - "hfopenllm_v2/MATH Level 5": 0.5242, - "hfopenllm_v2/GPQA": 0.4069, - "hfopenllm_v2/MUSR": 0.4719, - "hfopenllm_v2/MMLU-PRO": 0.5872 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/zeuslabs.json b/data/developers/zeuslabs.json deleted file mode 100644 index 83dccb02e142f371a1ba7a31d170ad01689e5601..0000000000000000000000000000000000000000 --- a/data/developers/zeuslabs.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "ZeusLabs", - "models": [ - { - "id": "ZeusLabs/L3-Aethora-15B-V2", - "name": "L3-Aethora-15B-V2", - "developer": "ZeusLabs", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7208, - "hfopenllm_v2/BBH": 0.5011, - "hfopenllm_v2/MATH Level 5": 0.0808, - "hfopenllm_v2/GPQA": 0.2878, - "hfopenllm_v2/MUSR": 0.3871, - "hfopenllm_v2/MMLU-PRO": 0.35 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/zhangshenao.json b/data/developers/zhangshenao.json deleted file mode 100644 index 182ed81b639b78ebf31540c427577135f11c8ab9..0000000000000000000000000000000000000000 --- a/data/developers/zhangshenao.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "ZhangShenao", - "models": [ - { - "id": "ZhangShenao/SELM-Llama-3-8B-Instruct-iter-3", - "name": "SELM-Llama-3-8B-Instruct-iter-3", - "developer": "ZhangShenao", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6903, - "hfopenllm_v2/BBH": 0.5046, - "hfopenllm_v2/MATH Level 5": 0.0861, - "hfopenllm_v2/GPQA": 0.2584, - "hfopenllm_v2/MUSR": 0.3845, - "hfopenllm_v2/MMLU-PRO": 0.3783 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/zhengr.json b/data/developers/zhengr.json deleted file mode 100644 index fd148552e2b6e6a9168c94acdb94db08b9ff9f46..0000000000000000000000000000000000000000 --- a/data/developers/zhengr.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "developer": "zhengr", - "models": [ - { - "id": "zhengr/MixTAO-7Bx2-MoE-v8.1", - "name": "MixTAO-7Bx2-MoE-v8.1", - "developer": "zhengr", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4188, - "hfopenllm_v2/BBH": 0.4202, - "hfopenllm_v2/MATH Level 5": 0.0604, - "hfopenllm_v2/GPQA": 0.2987, - "hfopenllm_v2/MUSR": 0.3976, - "hfopenllm_v2/MMLU-PRO": 0.2847 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/zhipu-ai.json b/data/developers/zhipu-ai.json deleted file mode 100644 index b914350b9f9666b3ff0559ce5693c595b07fee2f..0000000000000000000000000000000000000000 --- a/data/developers/zhipu-ai.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "developer": "zhipu-ai", - "models": [ - { - "id": "zhipu-ai/GLM-130B", - "name": "GLM 130B", - "developer": "zhipu-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.512, - "helm_classic/MMLU": 0.344, - "helm_classic/BoolQ": 0.784, - "helm_classic/NarrativeQA": 0.706, - "helm_classic/NaturalQuestions (open-book)": 0.642, - "helm_classic/QuAC": 0.272, - "helm_classic/HellaSwag": -1.0, - "helm_classic/OpenbookQA": -1.0, - "helm_classic/TruthfulQA": 0.218, - "helm_classic/MS MARCO (TREC)": -1.0, - "helm_classic/CNN/DailyMail": 0.154, - "helm_classic/XSUM": 0.132, - "helm_classic/IMDB": 0.955, - "helm_classic/CivilComments": 0.5, - "helm_classic/RAFT": 0.598 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/zhipu.json b/data/developers/zhipu.json deleted file mode 100644 index 735a5a0a6108be8ee32a8b9881335c349cc75969..0000000000000000000000000000000000000000 --- a/data/developers/zhipu.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "developer": "zhipu", - "models": [ - { - "id": "zhipu/GLM 4.6", - "name": "GLM 4.6", - "developer": "zhipu", - "evaluator_relationship": null, - "benchmark_scores": { - "apex-agents/Corporate Lawyer Mean Score": 0.196 - } - }, - { - "id": "zhipu/GLM 4.7", - "name": "GLM 4.7", - "developer": "zhipu", - "evaluator_relationship": null, - "benchmark_scores": { - "apex-agents/Corporate Lawyer Mean Score": 0.147 - } - }, - { - "id": "zhipu/glm-4-6-fc-thinking", - "name": "GLM-4.6 (FC thinking)", - "developer": "zhipu", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 4.0, - "bfcl/bfcl.overall.overall_accuracy": 72.38, - "bfcl/bfcl.overall.total_cost_usd": 4.64, - "bfcl/bfcl.overall.latency_mean_s": 4.34, - "bfcl/bfcl.overall.latency_std_s": 7.22, - "bfcl/bfcl.overall.latency_p95_s": 13.5, - "bfcl/bfcl.non_live.ast_accuracy": 87.56, - "bfcl/bfcl.non_live.simple_ast_accuracy": 74.25, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 95.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 91.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 89.5, - "bfcl/bfcl.live.live_accuracy": 80.9, - "bfcl/bfcl.live.live_simple_ast_accuracy": 89.53, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 78.92, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 81.25, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 75.0, - "bfcl/bfcl.multi_turn.accuracy": 68.0, - "bfcl/bfcl.multi_turn.base_accuracy": 74.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 68.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 63.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 66.5, - "bfcl/bfcl.web_search.accuracy": 77.5, - "bfcl/bfcl.web_search.base_accuracy": 79.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 76.0, - "bfcl/bfcl.memory.accuracy": 55.7, - "bfcl/bfcl.memory.kv_accuracy": 43.87, - "bfcl/bfcl.memory.vector_accuracy": 56.13, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 67.1, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 75.0, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 84.96 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/zhliu627.json b/data/developers/zhliu627.json deleted file mode 100644 index bfd999c370e89411f80f7ac2b53269ebd8da056f..0000000000000000000000000000000000000000 --- a/data/developers/zhliu627.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "developer": "ZHLiu627", - "models": [ - { - "id": "ZHLiu627/zephyr-7b-gemma-dpo-avg", - "name": "zephyr-7b-gemma-dpo-avg", - "developer": "ZHLiu627", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.309, - "hfopenllm_v2/BBH": 0.4149, - "hfopenllm_v2/MATH Level 5": 0.0453, - "hfopenllm_v2/GPQA": 0.2785, - "hfopenllm_v2/MUSR": 0.4107, - "hfopenllm_v2/MMLU-PRO": 0.2851 - } - }, - { - "id": "ZHLiu627/zephyr-7b-gemma-rpo-avg", - "name": "zephyr-7b-gemma-rpo-avg", - "developer": "ZHLiu627", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3006, - "hfopenllm_v2/BBH": 0.4183, - "hfopenllm_v2/MATH Level 5": 0.0498, - "hfopenllm_v2/GPQA": 0.2768, - "hfopenllm_v2/MUSR": 0.4081, - "hfopenllm_v2/MMLU-PRO": 0.2831 - } - } - ] -} \ No newline at end of file diff --git a/data/developers/ziyiye.json b/data/developers/ziyiye.json deleted file mode 100644 index 1b84dc0cda7c9fc81602b625fc2bd93f040b0dcd..0000000000000000000000000000000000000000 --- a/data/developers/ziyiye.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "developer": "ZiyiYe", - "models": [ - { - "id": "ZiyiYe/Con-J-Qwen2-7B", - "name": "ZiyiYe/Con-J-Qwen2-7B", - "developer": "ZiyiYe", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.8712, - "reward-bench/Chat": 0.919, - "reward-bench/Chat Hard": 0.8026, - "reward-bench/Safety": 0.8824, - "reward-bench/Reasoning": 0.8808 - } - } - ] -} \ No newline at end of file diff --git a/data/models.json b/data/models.json deleted file mode 100644 index fb0887357f489599a32846dc1003be0e21361be1..0000000000000000000000000000000000000000 --- a/data/models.json +++ /dev/null @@ -1,79831 +0,0 @@ -[ - { - "id": "0-hero/Matter-0.1-7B-boost-DPO-preview", - "name": "0-hero/Matter-0.1-7B-boost-DPO-preview", - "developer": "0-hero", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7448, - "reward-bench/Chat": 0.9106, - "reward-bench/Chat Hard": 0.6096, - "reward-bench/Safety": 0.7135, - "reward-bench/Reasoning": 0.8395, - "reward-bench/Prior Sets (0.5 weight)": 0.5566 - } - }, - { - "id": "0-hero/Matter-0.1-7B-DPO-preview", - "name": "0-hero/Matter-0.1-7B-DPO-preview", - "developer": "0-hero", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7247, - "reward-bench/Chat": 0.8939, - "reward-bench/Chat Hard": 0.5768, - "reward-bench/Safety": 0.6378, - "reward-bench/Reasoning": 0.8854, - "reward-bench/Prior Sets (0.5 weight)": 0.5348 - } - }, - { - "id": "0-hero/Matter-0.2-7B-DPO", - "name": "Matter-0.2-7B-DPO", - "developer": "0-hero", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3303, - "hfopenllm_v2/BBH": 0.3596, - "hfopenllm_v2/MATH Level 5": 0.0144, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3814, - "hfopenllm_v2/MMLU-PRO": 0.1164 - } - }, - { - "id": "01-ai/Yi-1.5-34B", - "name": "Yi-1.5-34B", - "developer": "01-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2841, - "hfopenllm_v2/BBH": 0.5976, - "hfopenllm_v2/MATH Level 5": 0.1533, - "hfopenllm_v2/GPQA": 0.3658, - "hfopenllm_v2/MUSR": 0.4236, - "hfopenllm_v2/MMLU-PRO": 0.4666 - } - }, - { - "id": "01-ai/Yi-1.5-34B-32K", - "name": "Yi-1.5-34B-32K", - "developer": "01-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3119, - "hfopenllm_v2/BBH": 0.6016, - "hfopenllm_v2/MATH Level 5": 0.1541, - "hfopenllm_v2/GPQA": 0.3633, - "hfopenllm_v2/MUSR": 0.4398, - "hfopenllm_v2/MMLU-PRO": 0.4709 - } - }, - { - "id": "01-ai/Yi-1.5-34B-Chat", - "name": "Yi-1.5-34B-Chat", - "developer": "01-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6067, - "hfopenllm_v2/BBH": 0.6084, - "hfopenllm_v2/MATH Level 5": 0.2772, - "hfopenllm_v2/GPQA": 0.3649, - "hfopenllm_v2/MUSR": 0.4282, - "hfopenllm_v2/MMLU-PRO": 0.452 - } - }, - { - "id": "01-ai/Yi-1.5-34B-Chat-16K", - "name": "Yi-1.5-34B-Chat-16K", - "developer": "01-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4564, - "hfopenllm_v2/BBH": 0.61, - "hfopenllm_v2/MATH Level 5": 0.2137, - "hfopenllm_v2/GPQA": 0.3381, - "hfopenllm_v2/MUSR": 0.4398, - "hfopenllm_v2/MMLU-PRO": 0.4545 - } - }, - { - "id": "01-ai/Yi-1.5-6B", - "name": "Yi-1.5-6B", - "developer": "01-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2617, - "hfopenllm_v2/BBH": 0.4493, - "hfopenllm_v2/MATH Level 5": 0.0665, - "hfopenllm_v2/GPQA": 0.3138, - "hfopenllm_v2/MUSR": 0.4374, - "hfopenllm_v2/MMLU-PRO": 0.3144 - } - }, - { - "id": "01-ai/Yi-1.5-6B-Chat", - "name": "Yi-1.5-6B-Chat", - "developer": "01-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5145, - "hfopenllm_v2/BBH": 0.4571, - "hfopenllm_v2/MATH Level 5": 0.1624, - "hfopenllm_v2/GPQA": 0.302, - "hfopenllm_v2/MUSR": 0.4392, - "hfopenllm_v2/MMLU-PRO": 0.3193 - } - }, - { - "id": "01-ai/Yi-1.5-9B", - "name": "Yi-1.5-9B", - "developer": "01-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2936, - "hfopenllm_v2/BBH": 0.5143, - "hfopenllm_v2/MATH Level 5": 0.114, - "hfopenllm_v2/GPQA": 0.3792, - "hfopenllm_v2/MUSR": 0.4328, - "hfopenllm_v2/MMLU-PRO": 0.3916 - } - }, - { - "id": "01-ai/Yi-1.5-9B-32K", - "name": "Yi-1.5-9B-32K", - "developer": "01-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2303, - "hfopenllm_v2/BBH": 0.4963, - "hfopenllm_v2/MATH Level 5": 0.108, - "hfopenllm_v2/GPQA": 0.3591, - "hfopenllm_v2/MUSR": 0.4186, - "hfopenllm_v2/MMLU-PRO": 0.3765 - } - }, - { - "id": "01-ai/Yi-1.5-9B-Chat", - "name": "Yi-1.5-9B-Chat", - "developer": "01-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6046, - "hfopenllm_v2/BBH": 0.5559, - "hfopenllm_v2/MATH Level 5": 0.2258, - "hfopenllm_v2/GPQA": 0.3347, - "hfopenllm_v2/MUSR": 0.4259, - "hfopenllm_v2/MMLU-PRO": 0.3975 - } - }, - { - "id": "01-ai/Yi-1.5-9B-Chat-16K", - "name": "Yi-1.5-9B-Chat-16K", - "developer": "01-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4214, - "hfopenllm_v2/BBH": 0.5153, - "hfopenllm_v2/MATH Level 5": 0.1782, - "hfopenllm_v2/GPQA": 0.3087, - "hfopenllm_v2/MUSR": 0.4099, - "hfopenllm_v2/MMLU-PRO": 0.3994 - } - }, - { - "id": "01-ai/yi-34b", - "name": "Yi 34B", - "developer": "01-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.57, - "helm_lite/NarrativeQA": 0.782, - "helm_lite/NaturalQuestions (closed-book)": 0.443, - "helm_lite/OpenbookQA": 0.92, - "helm_lite/MMLU": 0.65, - "helm_lite/MATH": 0.375, - "helm_lite/GSM8K": 0.648, - "helm_lite/LegalBench": 0.618, - "helm_lite/MedQA": 0.656, - "helm_lite/WMT 2014": 0.172, - "helm_mmlu/MMLU All Subjects": 0.762, - "helm_mmlu/Abstract Algebra": 0.4, - "helm_mmlu/Anatomy": 0.748, - "helm_mmlu/College Physics": 0.5, - "helm_mmlu/Computer Security": 0.83, - "helm_mmlu/Econometrics": 0.588, - "helm_mmlu/Global Facts": 0.53, - "helm_mmlu/Jurisprudence": 0.898, - "helm_mmlu/Philosophy": 0.82, - "helm_mmlu/Professional Psychology": 0.835, - "helm_mmlu/Us Foreign Policy": 0.91, - "helm_mmlu/Astronomy": 0.901, - "helm_mmlu/Business Ethics": 0.75, - "helm_mmlu/Clinical Knowledge": 0.8, - "helm_mmlu/Conceptual Physics": 0.77, - "helm_mmlu/Electrical Engineering": 0.779, - "helm_mmlu/Elementary Mathematics": 0.656, - "helm_mmlu/Formal Logic": 0.548, - "helm_mmlu/High School World History": 0.907, - "helm_mmlu/Human Sexuality": 0.87, - "helm_mmlu/International Law": 0.909, - "helm_mmlu/Logical Fallacies": 0.883, - "helm_mmlu/Machine Learning": 0.58, - "helm_mmlu/Management": 0.893, - "helm_mmlu/Marketing": 0.936, - "helm_mmlu/Medical Genetics": 0.87, - "helm_mmlu/Miscellaneous": 0.902, - "helm_mmlu/Moral Scenarios": 0.606, - "helm_mmlu/Nutrition": 0.869, - "helm_mmlu/Prehistory": 0.877, - "helm_mmlu/Public Relations": 0.745, - "helm_mmlu/Security Studies": 0.833, - "helm_mmlu/Sociology": 0.9, - "helm_mmlu/Virology": 0.572, - "helm_mmlu/World Religions": 0.877, - "helm_mmlu/Mean win rate": 0.315, - "hfopenllm_v2/IFEval": 0.3046, - "hfopenllm_v2/BBH": 0.5457, - "hfopenllm_v2/MATH Level 5": 0.0514, - "hfopenllm_v2/GPQA": 0.3666, - "hfopenllm_v2/MUSR": 0.4119, - "hfopenllm_v2/MMLU-PRO": 0.4412 - } - }, - { - "id": "01-ai/Yi-34B-200K", - "name": "Yi-34B-200K", - "developer": "01-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1542, - "hfopenllm_v2/BBH": 0.5442, - "hfopenllm_v2/MATH Level 5": 0.0574, - "hfopenllm_v2/GPQA": 0.3565, - "hfopenllm_v2/MUSR": 0.3817, - "hfopenllm_v2/MMLU-PRO": 0.4535 - } - }, - { - "id": "01-ai/Yi-34B-Chat", - "name": "Yi-34B-Chat", - "developer": "01-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4699, - "hfopenllm_v2/BBH": 0.5561, - "hfopenllm_v2/MATH Level 5": 0.0627, - "hfopenllm_v2/GPQA": 0.3381, - "hfopenllm_v2/MUSR": 0.3978, - "hfopenllm_v2/MMLU-PRO": 0.4093 - } - }, - { - "id": "01-ai/yi-6b", - "name": "Yi 6B", - "developer": "01-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.253, - "helm_lite/NarrativeQA": 0.702, - "helm_lite/NaturalQuestions (closed-book)": 0.31, - "helm_lite/OpenbookQA": 0.8, - "helm_lite/MMLU": 0.53, - "helm_lite/MATH": 0.126, - "helm_lite/GSM8K": 0.375, - "helm_lite/LegalBench": 0.519, - "helm_lite/MedQA": 0.497, - "helm_lite/WMT 2014": 0.117, - "helm_mmlu/MMLU All Subjects": 0.64, - "helm_mmlu/Abstract Algebra": 0.3, - "helm_mmlu/Anatomy": 0.6, - "helm_mmlu/College Physics": 0.422, - "helm_mmlu/Computer Security": 0.73, - "helm_mmlu/Econometrics": 0.351, - "helm_mmlu/Global Facts": 0.43, - "helm_mmlu/Jurisprudence": 0.796, - "helm_mmlu/Philosophy": 0.678, - "helm_mmlu/Professional Psychology": 0.668, - "helm_mmlu/Us Foreign Policy": 0.87, - "helm_mmlu/Astronomy": 0.684, - "helm_mmlu/Business Ethics": 0.67, - "helm_mmlu/Clinical Knowledge": 0.66, - "helm_mmlu/Conceptual Physics": 0.621, - "helm_mmlu/Electrical Engineering": 0.662, - "helm_mmlu/Elementary Mathematics": 0.452, - "helm_mmlu/Formal Logic": 0.452, - "helm_mmlu/High School World History": 0.785, - "helm_mmlu/Human Sexuality": 0.763, - "helm_mmlu/International Law": 0.769, - "helm_mmlu/Logical Fallacies": 0.779, - "helm_mmlu/Machine Learning": 0.411, - "helm_mmlu/Management": 0.806, - "helm_mmlu/Marketing": 0.893, - "helm_mmlu/Medical Genetics": 0.77, - "helm_mmlu/Miscellaneous": 0.796, - "helm_mmlu/Moral Scenarios": 0.335, - "helm_mmlu/Nutrition": 0.739, - "helm_mmlu/Prehistory": 0.713, - "helm_mmlu/Public Relations": 0.718, - "helm_mmlu/Security Studies": 0.735, - "helm_mmlu/Sociology": 0.831, - "helm_mmlu/Virology": 0.452, - "helm_mmlu/World Religions": 0.836, - "helm_mmlu/Mean win rate": 0.651, - "hfopenllm_v2/IFEval": 0.2893, - "hfopenllm_v2/BBH": 0.4309, - "hfopenllm_v2/MATH Level 5": 0.0159, - "hfopenllm_v2/GPQA": 0.2693, - "hfopenllm_v2/MUSR": 0.3937, - "hfopenllm_v2/MMLU-PRO": 0.2991 - } - }, - { - "id": "01-ai/Yi-6B-200K", - "name": "Yi-6B-200K", - "developer": "01-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0843, - "hfopenllm_v2/BBH": 0.4289, - "hfopenllm_v2/MATH Level 5": 0.0181, - "hfopenllm_v2/GPQA": 0.2819, - "hfopenllm_v2/MUSR": 0.4587, - "hfopenllm_v2/MMLU-PRO": 0.2844 - } - }, - { - "id": "01-ai/Yi-6B-Chat", - "name": "Yi-6B-Chat", - "developer": "01-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3395, - "hfopenllm_v2/BBH": 0.4133, - "hfopenllm_v2/MATH Level 5": 0.0136, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.3688, - "hfopenllm_v2/MMLU-PRO": 0.3061 - } - }, - { - "id": "01-ai/Yi-9B", - "name": "Yi-9B", - "developer": "01-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2709, - "hfopenllm_v2/BBH": 0.494, - "hfopenllm_v2/MATH Level 5": 0.0559, - "hfopenllm_v2/GPQA": 0.318, - "hfopenllm_v2/MUSR": 0.4054, - "hfopenllm_v2/MMLU-PRO": 0.3574 - } - }, - { - "id": "01-ai/Yi-9B-200K", - "name": "Yi-9B-200K", - "developer": "01-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2327, - "hfopenllm_v2/BBH": 0.4793, - "hfopenllm_v2/MATH Level 5": 0.0665, - "hfopenllm_v2/GPQA": 0.3154, - "hfopenllm_v2/MUSR": 0.4294, - "hfopenllm_v2/MMLU-PRO": 0.3622 - } - }, - { - "id": "01-ai/Yi-Coder-9B-Chat", - "name": "Yi-Coder-9B-Chat", - "developer": "01-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4817, - "hfopenllm_v2/BBH": 0.4814, - "hfopenllm_v2/MATH Level 5": 0.04, - "hfopenllm_v2/GPQA": 0.2475, - "hfopenllm_v2/MUSR": 0.3992, - "hfopenllm_v2/MMLU-PRO": 0.2425 - } - }, - { - "id": "01-ai/yi-large-preview", - "name": "Yi Large Preview", - "developer": "01-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.471, - "helm_lite/NarrativeQA": 0.373, - "helm_lite/NaturalQuestions (closed-book)": 0.428, - "helm_lite/OpenbookQA": 0.946, - "helm_lite/MMLU": 0.712, - "helm_lite/MATH": 0.712, - "helm_lite/GSM8K": 0.69, - "helm_lite/LegalBench": 0.519, - "helm_lite/MedQA": 0.66, - "helm_lite/WMT 2014": 0.176, - "helm_mmlu/MMLU All Subjects": 0.793, - "helm_mmlu/Abstract Algebra": 0.6, - "helm_mmlu/Anatomy": 0.83, - "helm_mmlu/College Physics": 0.569, - "helm_mmlu/Computer Security": 0.86, - "helm_mmlu/Econometrics": 0.728, - "helm_mmlu/Global Facts": 0.52, - "helm_mmlu/Jurisprudence": 0.852, - "helm_mmlu/Philosophy": 0.842, - "helm_mmlu/Professional Psychology": 0.853, - "helm_mmlu/Us Foreign Policy": 0.85, - "helm_mmlu/Astronomy": 0.914, - "helm_mmlu/Business Ethics": 0.8, - "helm_mmlu/Clinical Knowledge": 0.857, - "helm_mmlu/Conceptual Physics": 0.864, - "helm_mmlu/Electrical Engineering": 0.779, - "helm_mmlu/Elementary Mathematics": 0.685, - "helm_mmlu/Formal Logic": 0.603, - "helm_mmlu/High School World History": 0.928, - "helm_mmlu/Human Sexuality": 0.901, - "helm_mmlu/International Law": 0.917, - "helm_mmlu/Logical Fallacies": 0.865, - "helm_mmlu/Machine Learning": 0.616, - "helm_mmlu/Management": 0.903, - "helm_mmlu/Marketing": 0.927, - "helm_mmlu/Medical Genetics": 0.83, - "helm_mmlu/Miscellaneous": 0.916, - "helm_mmlu/Moral Scenarios": 0.831, - "helm_mmlu/Nutrition": 0.846, - "helm_mmlu/Prehistory": 0.892, - "helm_mmlu/Public Relations": 0.827, - "helm_mmlu/Security Studies": 0.82, - "helm_mmlu/Sociology": 0.881, - "helm_mmlu/Virology": 0.59, - "helm_mmlu/World Religions": 0.871, - "helm_mmlu/Mean win rate": 0.258 - } - }, - { - "id": "1-800-LLMs/Qwen-2.5-14B-Hindi", - "name": "Qwen-2.5-14B-Hindi", - "developer": "1-800-LLMs", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5826, - "hfopenllm_v2/BBH": 0.6524, - "hfopenllm_v2/MATH Level 5": 0.3331, - "hfopenllm_v2/GPQA": 0.3624, - "hfopenllm_v2/MUSR": 0.4489, - "hfopenllm_v2/MMLU-PRO": 0.5263 - } - }, - { - "id": "1-800-LLMs/Qwen-2.5-14B-Hindi-Custom-Instruct", - "name": "Qwen-2.5-14B-Hindi-Custom-Instruct", - "developer": "1-800-LLMs", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3077, - "hfopenllm_v2/BBH": 0.6284, - "hfopenllm_v2/MATH Level 5": 0.3112, - "hfopenllm_v2/GPQA": 0.37, - "hfopenllm_v2/MUSR": 0.4491, - "hfopenllm_v2/MMLU-PRO": 0.5164 - } - }, - { - "id": "1024m/PHI-4-Hindi", - "name": "PHI-4-Hindi", - "developer": "1024m", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0082, - "hfopenllm_v2/BBH": 0.671, - "hfopenllm_v2/MATH Level 5": 0.2334, - "hfopenllm_v2/GPQA": 0.3977, - "hfopenllm_v2/MUSR": 0.4914, - "hfopenllm_v2/MMLU-PRO": 0.5239 - } - }, - { - "id": "1024m/QWEN-14B-B100", - "name": "QWEN-14B-B100", - "developer": "1024m", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7762, - "hfopenllm_v2/BBH": 0.6533, - "hfopenllm_v2/MATH Level 5": 0.5438, - "hfopenllm_v2/GPQA": 0.3507, - "hfopenllm_v2/MUSR": 0.41, - "hfopenllm_v2/MMLU-PRO": 0.5179 - } - }, - { - "id": "152334H/miqu-1-70b-sf", - "name": "miqu-1-70b-sf", - "developer": "152334H", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5182, - "hfopenllm_v2/BBH": 0.6102, - "hfopenllm_v2/MATH Level 5": 0.1246, - "hfopenllm_v2/GPQA": 0.3507, - "hfopenllm_v2/MUSR": 0.4582, - "hfopenllm_v2/MMLU-PRO": 0.4228 - } - }, - { - "id": "1TuanPham/T-VisStar-7B-v0.1", - "name": "T-VisStar-7B-v0.1", - "developer": "1TuanPham", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3607, - "hfopenllm_v2/BBH": 0.5052, - "hfopenllm_v2/MATH Level 5": 0.0574, - "hfopenllm_v2/GPQA": 0.2852, - "hfopenllm_v2/MUSR": 0.4375, - "hfopenllm_v2/MMLU-PRO": 0.3211 - } - }, - { - "id": "1TuanPham/T-VisStar-v0.1", - "name": "T-VisStar-v0.1", - "developer": "1TuanPham", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3607, - "hfopenllm_v2/BBH": 0.5052, - "hfopenllm_v2/MATH Level 5": 0.0574, - "hfopenllm_v2/GPQA": 0.2852, - "hfopenllm_v2/MUSR": 0.4375, - "hfopenllm_v2/MMLU-PRO": 0.3211 - } - }, - { - "id": "3rd-Degree-Burn/L-3.1-Science-Writer-8B", - "name": "L-3.1-Science-Writer-8B", - "developer": "3rd-Degree-Burn", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4263, - "hfopenllm_v2/BBH": 0.5041, - "hfopenllm_v2/MATH Level 5": 0.1035, - "hfopenllm_v2/GPQA": 0.2743, - "hfopenllm_v2/MUSR": 0.3959, - "hfopenllm_v2/MMLU-PRO": 0.3649 - } - }, - { - "id": "3rd-Degree-Burn/Llama-3.1-8B-Squareroot", - "name": "Llama-3.1-8B-Squareroot", - "developer": "3rd-Degree-Burn", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2213, - "hfopenllm_v2/BBH": 0.3461, - "hfopenllm_v2/MATH Level 5": 0.2659, - "hfopenllm_v2/GPQA": 0.2567, - "hfopenllm_v2/MUSR": 0.3089, - "hfopenllm_v2/MMLU-PRO": 0.175 - } - }, - { - "id": "3rd-Degree-Burn/Llama-3.1-8B-Squareroot-v1", - "name": "Llama-3.1-8B-Squareroot-v1", - "developer": "3rd-Degree-Burn", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2892, - "hfopenllm_v2/BBH": 0.3343, - "hfopenllm_v2/MATH Level 5": 0.0884, - "hfopenllm_v2/GPQA": 0.2559, - "hfopenllm_v2/MUSR": 0.3341, - "hfopenllm_v2/MMLU-PRO": 0.1127 - } - }, - { - "id": "3rd-Degree-Burn/Llama-Squared-8B", - "name": "Llama-Squared-8B", - "developer": "3rd-Degree-Burn", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2755, - "hfopenllm_v2/BBH": 0.4431, - "hfopenllm_v2/MATH Level 5": 0.0574, - "hfopenllm_v2/GPQA": 0.2718, - "hfopenllm_v2/MUSR": 0.3089, - "hfopenllm_v2/MMLU-PRO": 0.2366 - } - }, - { - "id": "4season/final_model_test_v2", - "name": "final_model_test_v2", - "developer": "4season", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3191, - "hfopenllm_v2/BBH": 0.6342, - "hfopenllm_v2/MATH Level 5": 0.0838, - "hfopenllm_v2/GPQA": 0.3272, - "hfopenllm_v2/MUSR": 0.4314, - "hfopenllm_v2/MMLU-PRO": 0.3528 - } - }, - { - "id": "aaditya/Llama3-OpenBioLLM-70B", - "name": "Llama3-OpenBioLLM-70B", - "developer": "aaditya", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7597, - "hfopenllm_v2/BBH": 0.6399, - "hfopenllm_v2/MATH Level 5": 0.1971, - "hfopenllm_v2/GPQA": 0.323, - "hfopenllm_v2/MUSR": 0.4417, - "hfopenllm_v2/MMLU-PRO": 0.4867 - } - }, - { - "id": "AALF/FuseChat-Llama-3.1-8B-Instruct-preview", - "name": "FuseChat-Llama-3.1-8B-Instruct-preview", - "developer": "AALF", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.719, - "hfopenllm_v2/BBH": 0.512, - "hfopenllm_v2/MATH Level 5": 0.2477, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.382, - "hfopenllm_v2/MMLU-PRO": 0.3733 - } - }, - { - "id": "AALF/FuseChat-Llama-3.1-8B-SFT-preview", - "name": "FuseChat-Llama-3.1-8B-SFT-preview", - "developer": "AALF", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7281, - "hfopenllm_v2/BBH": 0.524, - "hfopenllm_v2/MATH Level 5": 0.2251, - "hfopenllm_v2/GPQA": 0.3045, - "hfopenllm_v2/MUSR": 0.402, - "hfopenllm_v2/MMLU-PRO": 0.3743 - } - }, - { - "id": "AALF/gemma-2-27b-it-SimPO-37K", - "name": "gemma-2-27b-it-SimPO-37K", - "developer": "AALF", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2407, - "hfopenllm_v2/BBH": 0.3911, - "hfopenllm_v2/MATH Level 5": 0.0128, - "hfopenllm_v2/GPQA": 0.2802, - "hfopenllm_v2/MUSR": 0.3488, - "hfopenllm_v2/MMLU-PRO": 0.1971 - } - }, - { - "id": "AALF/gemma-2-27b-it-SimPO-37K-100steps", - "name": "gemma-2-27b-it-SimPO-37K-100steps", - "developer": "AALF", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2568, - "hfopenllm_v2/BBH": 0.3931, - "hfopenllm_v2/MATH Level 5": 0.0211, - "hfopenllm_v2/GPQA": 0.2886, - "hfopenllm_v2/MUSR": 0.3329, - "hfopenllm_v2/MMLU-PRO": 0.2125 - } - }, - { - "id": "Aashraf995/Creative-7B-nerd", - "name": "Creative-7B-nerd", - "developer": "Aashraf995", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4722, - "hfopenllm_v2/BBH": 0.5607, - "hfopenllm_v2/MATH Level 5": 0.3165, - "hfopenllm_v2/GPQA": 0.3263, - "hfopenllm_v2/MUSR": 0.4515, - "hfopenllm_v2/MMLU-PRO": 0.4492 - } - }, - { - "id": "Aashraf995/Gemma-Evo-10B", - "name": "Gemma-Evo-10B", - "developer": "Aashraf995", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7332, - "hfopenllm_v2/BBH": 0.6044, - "hfopenllm_v2/MATH Level 5": 0.2228, - "hfopenllm_v2/GPQA": 0.354, - "hfopenllm_v2/MUSR": 0.4595, - "hfopenllm_v2/MMLU-PRO": 0.4275 - } - }, - { - "id": "Aashraf995/Qwen-Evo-7B", - "name": "Qwen-Evo-7B", - "developer": "Aashraf995", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4757, - "hfopenllm_v2/BBH": 0.5709, - "hfopenllm_v2/MATH Level 5": 0.3142, - "hfopenllm_v2/GPQA": 0.3255, - "hfopenllm_v2/MUSR": 0.4541, - "hfopenllm_v2/MMLU-PRO": 0.4462 - } - }, - { - "id": "Aashraf995/QwenStock-14B", - "name": "QwenStock-14B", - "developer": "Aashraf995", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5009, - "hfopenllm_v2/BBH": 0.655, - "hfopenllm_v2/MATH Level 5": 0.3573, - "hfopenllm_v2/GPQA": 0.3893, - "hfopenllm_v2/MUSR": 0.4793, - "hfopenllm_v2/MMLU-PRO": 0.5382 - } - }, - { - "id": "abacusai/bigstral-12b-32k", - "name": "bigstral-12b-32k", - "developer": "abacusai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4194, - "hfopenllm_v2/BBH": 0.47, - "hfopenllm_v2/MATH Level 5": 0.0151, - "hfopenllm_v2/GPQA": 0.2928, - "hfopenllm_v2/MUSR": 0.456, - "hfopenllm_v2/MMLU-PRO": 0.2641 - } - }, - { - "id": "abacusai/bigyi-15b", - "name": "bigyi-15b", - "developer": "abacusai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2094, - "hfopenllm_v2/BBH": 0.4345, - "hfopenllm_v2/MATH Level 5": 0.0295, - "hfopenllm_v2/GPQA": 0.3096, - "hfopenllm_v2/MUSR": 0.3538, - "hfopenllm_v2/MMLU-PRO": 0.3003 - } - }, - { - "id": "abacusai/Dracarys-72B-Instruct", - "name": "Dracarys-72B-Instruct", - "developer": "abacusai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7856, - "hfopenllm_v2/BBH": 0.6944, - "hfopenllm_v2/MATH Level 5": 0.3965, - "hfopenllm_v2/GPQA": 0.3909, - "hfopenllm_v2/MUSR": 0.4558, - "hfopenllm_v2/MMLU-PRO": 0.5456 - } - }, - { - "id": "abacusai/Liberated-Qwen1.5-14B", - "name": "Liberated-Qwen1.5-14B", - "developer": "abacusai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3631, - "hfopenllm_v2/BBH": 0.4948, - "hfopenllm_v2/MATH Level 5": 0.1601, - "hfopenllm_v2/GPQA": 0.2836, - "hfopenllm_v2/MUSR": 0.4175, - "hfopenllm_v2/MMLU-PRO": 0.3512 - } - }, - { - "id": "abacusai/Llama-3-Smaug-8B", - "name": "Llama-3-Smaug-8B", - "developer": "abacusai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4867, - "hfopenllm_v2/BBH": 0.4931, - "hfopenllm_v2/MATH Level 5": 0.0853, - "hfopenllm_v2/GPQA": 0.2483, - "hfopenllm_v2/MUSR": 0.3622, - "hfopenllm_v2/MMLU-PRO": 0.3185 - } - }, - { - "id": "abacusai/Smaug-34B-v0.1", - "name": "Smaug-34B-v0.1", - "developer": "abacusai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5016, - "hfopenllm_v2/BBH": 0.5358, - "hfopenllm_v2/MATH Level 5": 0.0718, - "hfopenllm_v2/GPQA": 0.3297, - "hfopenllm_v2/MUSR": 0.3979, - "hfopenllm_v2/MMLU-PRO": 0.4543 - } - }, - { - "id": "abacusai/Smaug-72B-v0.1", - "name": "Smaug-72B-v0.1", - "developer": "abacusai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5167, - "hfopenllm_v2/BBH": 0.5996, - "hfopenllm_v2/MATH Level 5": 0.1911, - "hfopenllm_v2/GPQA": 0.3238, - "hfopenllm_v2/MUSR": 0.4473, - "hfopenllm_v2/MMLU-PRO": 0.4624 - } - }, - { - "id": "abacusai/Smaug-Llama-3-70B-Instruct-32K", - "name": "Smaug-Llama-3-70B-Instruct-32K", - "developer": "abacusai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7761, - "hfopenllm_v2/BBH": 0.6493, - "hfopenllm_v2/MATH Level 5": 0.2749, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.4208, - "hfopenllm_v2/MMLU-PRO": 0.4765 - } - }, - { - "id": "abacusai/Smaug-Mixtral-v0.1", - "name": "Smaug-Mixtral-v0.1", - "developer": "abacusai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5554, - "hfopenllm_v2/BBH": 0.5162, - "hfopenllm_v2/MATH Level 5": 0.0952, - "hfopenllm_v2/GPQA": 0.3012, - "hfopenllm_v2/MUSR": 0.4298, - "hfopenllm_v2/MMLU-PRO": 0.3352 - } - }, - { - "id": "abacusai/Smaug-Qwen2-72B-Instruct", - "name": "Smaug-Qwen2-72B-Instruct", - "developer": "abacusai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7825, - "hfopenllm_v2/BBH": 0.691, - "hfopenllm_v2/MATH Level 5": 0.4131, - "hfopenllm_v2/GPQA": 0.3616, - "hfopenllm_v2/MUSR": 0.4401, - "hfopenllm_v2/MMLU-PRO": 0.519 - } - }, - { - "id": "AbacusResearch/Jallabi-34B", - "name": "Jallabi-34B", - "developer": "AbacusResearch", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3529, - "hfopenllm_v2/BBH": 0.6023, - "hfopenllm_v2/MATH Level 5": 0.0521, - "hfopenllm_v2/GPQA": 0.3389, - "hfopenllm_v2/MUSR": 0.4822, - "hfopenllm_v2/MMLU-PRO": 0.4682 - } - }, - { - "id": "abhishek/autotrain-0tmgq-5tpbg", - "name": "autotrain-0tmgq-5tpbg", - "developer": "abhishek", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1952, - "hfopenllm_v2/BBH": 0.3127, - "hfopenllm_v2/MATH Level 5": 0.0128, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3584, - "hfopenllm_v2/MMLU-PRO": 0.1144 - } - }, - { - "id": "abhishek/autotrain-llama3-70b-orpo-v1", - "name": "autotrain-llama3-70b-orpo-v1", - "developer": "abhishek", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4233, - "hfopenllm_v2/BBH": 0.5998, - "hfopenllm_v2/MATH Level 5": 0.0106, - "hfopenllm_v2/GPQA": 0.2441, - "hfopenllm_v2/MUSR": 0.3579, - "hfopenllm_v2/MMLU-PRO": 0.1122 - } - }, - { - "id": "abhishek/autotrain-llama3-70b-orpo-v2", - "name": "autotrain-llama3-70b-orpo-v2", - "developer": "abhishek", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5406, - "hfopenllm_v2/BBH": 0.5899, - "hfopenllm_v2/MATH Level 5": 0.2107, - "hfopenllm_v2/GPQA": 0.2936, - "hfopenllm_v2/MUSR": 0.4113, - "hfopenllm_v2/MMLU-PRO": 0.4818 - } - }, - { - "id": "abhishek/autotrain-llama3-orpo-v2", - "name": "autotrain-llama3-orpo-v2", - "developer": "abhishek", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4372, - "hfopenllm_v2/BBH": 0.3159, - "hfopenllm_v2/MATH Level 5": 0.0468, - "hfopenllm_v2/GPQA": 0.2668, - "hfopenllm_v2/MUSR": 0.3792, - "hfopenllm_v2/MMLU-PRO": 0.2218 - } - }, - { - "id": "abhishek/autotrain-vr4a1-e5mms", - "name": "autotrain-vr4a1-e5mms", - "developer": "abhishek", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2142, - "hfopenllm_v2/BBH": 0.5001, - "hfopenllm_v2/MATH Level 5": 0.1412, - "hfopenllm_v2/GPQA": 0.3196, - "hfopenllm_v2/MUSR": 0.3891, - "hfopenllm_v2/MMLU-PRO": 0.3667 - } - }, - { - "id": "abideen/MedPhi-4-14B-v1", - "name": "MedPhi-4-14B-v1", - "developer": "abideen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6277, - "hfopenllm_v2/BBH": 0.6897, - "hfopenllm_v2/MATH Level 5": 0.2931, - "hfopenllm_v2/GPQA": 0.344, - "hfopenllm_v2/MUSR": 0.4155, - "hfopenllm_v2/MMLU-PRO": 0.5338 - } - }, - { - "id": "adamo1139/Yi-34B-200K-AEZAKMI-v2", - "name": "Yi-34B-200K-AEZAKMI-v2", - "developer": "adamo1139", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4555, - "hfopenllm_v2/BBH": 0.5384, - "hfopenllm_v2/MATH Level 5": 0.0566, - "hfopenllm_v2/GPQA": 0.3322, - "hfopenllm_v2/MUSR": 0.3886, - "hfopenllm_v2/MMLU-PRO": 0.4513 - } - }, - { - "id": "adriszmar/QAIMath-Qwen2.5-7B-TIES", - "name": "QAIMath-Qwen2.5-7B-TIES", - "developer": "adriszmar", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1746, - "hfopenllm_v2/BBH": 0.3126, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.245, - "hfopenllm_v2/MUSR": 0.4096, - "hfopenllm_v2/MMLU-PRO": 0.1087 - } - }, - { - "id": "AELLM/gemma-2-aeria-infinity-9b", - "name": "gemma-2-aeria-infinity-9b", - "developer": "AELLM", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7594, - "hfopenllm_v2/BBH": 0.5983, - "hfopenllm_v2/MATH Level 5": 0.2145, - "hfopenllm_v2/GPQA": 0.3339, - "hfopenllm_v2/MUSR": 0.402, - "hfopenllm_v2/MMLU-PRO": 0.3862 - } - }, - { - "id": "AELLM/gemma-2-lyco-infinity-9b", - "name": "gemma-2-lyco-infinity-9b", - "developer": "AELLM", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7316, - "hfopenllm_v2/BBH": 0.584, - "hfopenllm_v2/MATH Level 5": 0.1707, - "hfopenllm_v2/GPQA": 0.328, - "hfopenllm_v2/MUSR": 0.4006, - "hfopenllm_v2/MMLU-PRO": 0.3787 - } - }, - { - "id": "aevalone/distill_qw_test", - "name": "distill_qw_test", - "developer": "aevalone", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7409, - "hfopenllm_v2/BBH": 0.5246, - "hfopenllm_v2/MATH Level 5": 0.4781, - "hfopenllm_v2/GPQA": 0.3003, - "hfopenllm_v2/MUSR": 0.386, - "hfopenllm_v2/MMLU-PRO": 0.4092 - } - }, - { - "id": "agentlans/Gemma2-9B-AdvancedFuse", - "name": "Gemma2-9B-AdvancedFuse", - "developer": "agentlans", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1543, - "hfopenllm_v2/BBH": 0.5859, - "hfopenllm_v2/MATH Level 5": 0.1005, - "hfopenllm_v2/GPQA": 0.3347, - "hfopenllm_v2/MUSR": 0.4231, - "hfopenllm_v2/MMLU-PRO": 0.4 - } - }, - { - "id": "agentlans/Llama-3.2-1B-Instruct-CrashCourse12K", - "name": "Llama-3.2-1B-Instruct-CrashCourse12K", - "developer": "agentlans", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5395, - "hfopenllm_v2/BBH": 0.3548, - "hfopenllm_v2/MATH Level 5": 0.071, - "hfopenllm_v2/GPQA": 0.2408, - "hfopenllm_v2/MUSR": 0.321, - "hfopenllm_v2/MMLU-PRO": 0.1809 - } - }, - { - "id": "agentlans/Llama3.1-8B-drill", - "name": "Llama3.1-8B-drill", - "developer": "agentlans", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7652, - "hfopenllm_v2/BBH": 0.5016, - "hfopenllm_v2/MATH Level 5": 0.1715, - "hfopenllm_v2/GPQA": 0.2676, - "hfopenllm_v2/MUSR": 0.3672, - "hfopenllm_v2/MMLU-PRO": 0.3776 - } - }, - { - "id": "agentlans/Llama3.1-Daredevilish", - "name": "Llama3.1-Daredevilish", - "developer": "agentlans", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6292, - "hfopenllm_v2/BBH": 0.5013, - "hfopenllm_v2/MATH Level 5": 0.1292, - "hfopenllm_v2/GPQA": 0.3012, - "hfopenllm_v2/MUSR": 0.4091, - "hfopenllm_v2/MMLU-PRO": 0.3697 - } - }, - { - "id": "agentlans/Llama3.1-Daredevilish-Instruct", - "name": "Llama3.1-Daredevilish-Instruct", - "developer": "agentlans", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7926, - "hfopenllm_v2/BBH": 0.5235, - "hfopenllm_v2/MATH Level 5": 0.1722, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.3911, - "hfopenllm_v2/MMLU-PRO": 0.3877 - } - }, - { - "id": "agentlans/Llama3.1-LexiHermes-SuperStorm", - "name": "Llama3.1-LexiHermes-SuperStorm", - "developer": "agentlans", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7835, - "hfopenllm_v2/BBH": 0.5266, - "hfopenllm_v2/MATH Level 5": 0.1616, - "hfopenllm_v2/GPQA": 0.323, - "hfopenllm_v2/MUSR": 0.3963, - "hfopenllm_v2/MMLU-PRO": 0.3844 - } - }, - { - "id": "agentlans/Llama3.1-SuperDeepFuse", - "name": "Llama3.1-SuperDeepFuse", - "developer": "agentlans", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7762, - "hfopenllm_v2/BBH": 0.5049, - "hfopenllm_v2/MATH Level 5": 0.1828, - "hfopenllm_v2/GPQA": 0.2743, - "hfopenllm_v2/MUSR": 0.3699, - "hfopenllm_v2/MMLU-PRO": 0.3775 - } - }, - { - "id": "agentlans/Llama3.1-SuperDeepFuse-CrashCourse12K", - "name": "Llama3.1-SuperDeepFuse-CrashCourse12K", - "developer": "agentlans", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7187, - "hfopenllm_v2/BBH": 0.5216, - "hfopenllm_v2/MATH Level 5": 0.1805, - "hfopenllm_v2/GPQA": 0.3129, - "hfopenllm_v2/MUSR": 0.4026, - "hfopenllm_v2/MMLU-PRO": 0.3631 - } - }, - { - "id": "agentlans/Qwen2.5-0.5B-Instruct-CrashCourse-dropout", - "name": "Qwen2.5-0.5B-Instruct-CrashCourse-dropout", - "developer": "agentlans", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2949, - "hfopenllm_v2/BBH": 0.3312, - "hfopenllm_v2/MATH Level 5": 0.0423, - "hfopenllm_v2/GPQA": 0.2634, - "hfopenllm_v2/MUSR": 0.3342, - "hfopenllm_v2/MMLU-PRO": 0.1608 - } - }, - { - "id": "AGI-0/Art-v0-3B", - "name": "Art-v0-3B", - "developer": "AGI-0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3192, - "hfopenllm_v2/BBH": 0.3401, - "hfopenllm_v2/MATH Level 5": 0.2462, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3768, - "hfopenllm_v2/MMLU-PRO": 0.1179 - } - }, - { - "id": "AGI-0/Artificium-llama3.1-8B-001", - "name": "Artificium-llama3.1-8B-001", - "developer": "AGI-0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5248, - "hfopenllm_v2/BBH": 0.4256, - "hfopenllm_v2/MATH Level 5": 0.136, - "hfopenllm_v2/GPQA": 0.2659, - "hfopenllm_v2/MUSR": 0.3795, - "hfopenllm_v2/MMLU-PRO": 0.3182 - } - }, - { - "id": "AGI-0/smartllama3.1-8B-001", - "name": "smartllama3.1-8B-001", - "developer": "AGI-0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3518, - "hfopenllm_v2/BBH": 0.467, - "hfopenllm_v2/MATH Level 5": 0.1299, - "hfopenllm_v2/GPQA": 0.3062, - "hfopenllm_v2/MUSR": 0.4386, - "hfopenllm_v2/MMLU-PRO": 0.3487 - } - }, - { - "id": "Ahdoot/StructuredThinker-v0.3-MoreStructure", - "name": "StructuredThinker-v0.3-MoreStructure", - "developer": "Ahdoot", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4193, - "hfopenllm_v2/BBH": 0.4838, - "hfopenllm_v2/MATH Level 5": 0.2908, - "hfopenllm_v2/GPQA": 0.297, - "hfopenllm_v2/MUSR": 0.4158, - "hfopenllm_v2/MMLU-PRO": 0.361 - } - }, - { - "id": "Ahdoot/Test_StealthThinker", - "name": "Test_StealthThinker", - "developer": "Ahdoot", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.422, - "hfopenllm_v2/BBH": 0.4647, - "hfopenllm_v2/MATH Level 5": 0.179, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.428, - "hfopenllm_v2/MMLU-PRO": 0.3597 - } - }, - { - "id": "Ahjeong/MMPO_Gemma_7b", - "name": "Ahjeong/MMPO_Gemma_7b", - "developer": "Ahjeong", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7587, - "reward-bench/Chat": 0.9693, - "reward-bench/Chat Hard": 0.614, - "reward-bench/Safety": 0.7135, - "reward-bench/Reasoning": 0.7756, - "reward-bench/Prior Sets (0.5 weight)": 0.6831 - } - }, - { - "id": "Ahjeong/MMPO_Gemma_7b_gamma1.1_epoch3", - "name": "Ahjeong/MMPO_Gemma_7b_gamma1.1_epoch3", - "developer": "Ahjeong", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7652, - "reward-bench/Chat": 0.9721, - "reward-bench/Chat Hard": 0.6338, - "reward-bench/Safety": 0.7635, - "reward-bench/Reasoning": 0.7284, - "reward-bench/Prior Sets (0.5 weight)": 0.6913 - } - }, - { - "id": "ahmeda335/13_outOf_32_pruned_layers_llama3.1-8b", - "name": "13_outOf_32_pruned_layers_llama3.1-8b", - "developer": "ahmeda335", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1748, - "hfopenllm_v2/BBH": 0.2883, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3803, - "hfopenllm_v2/MMLU-PRO": 0.1129 - } - }, - { - "id": "AI-MO/NuminaMath-7B-CoT", - "name": "NuminaMath-7B-CoT", - "developer": "AI-MO", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2689, - "hfopenllm_v2/BBH": 0.4314, - "hfopenllm_v2/MATH Level 5": 0.2696, - "hfopenllm_v2/GPQA": 0.2659, - "hfopenllm_v2/MUSR": 0.3303, - "hfopenllm_v2/MMLU-PRO": 0.2868 - } - }, - { - "id": "AI-MO/NuminaMath-7B-TIR", - "name": "NuminaMath-7B-TIR", - "developer": "AI-MO", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2756, - "hfopenllm_v2/BBH": 0.4144, - "hfopenllm_v2/MATH Level 5": 0.1609, - "hfopenllm_v2/GPQA": 0.2584, - "hfopenllm_v2/MUSR": 0.3509, - "hfopenllm_v2/MMLU-PRO": 0.2733 - } - }, - { - "id": "AI-Sweden-Models/gpt-sw3-40b", - "name": "gpt-sw3-40b", - "developer": "AI-Sweden-Models", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.147, - "hfopenllm_v2/BBH": 0.3268, - "hfopenllm_v2/MATH Level 5": 0.0174, - "hfopenllm_v2/GPQA": 0.2349, - "hfopenllm_v2/MUSR": 0.3632, - "hfopenllm_v2/MMLU-PRO": 0.1276 - } - }, - { - "id": "AI-Sweden-Models/Llama-3-8B-instruct", - "name": "Llama-3-8B-instruct", - "developer": "AI-Sweden-Models", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2401, - "hfopenllm_v2/BBH": 0.4173, - "hfopenllm_v2/MATH Level 5": 0.0385, - "hfopenllm_v2/GPQA": 0.2659, - "hfopenllm_v2/MUSR": 0.4771, - "hfopenllm_v2/MMLU-PRO": 0.2597 - } - }, - { - "id": "ai2/llama-2-chat-7b-nectar-3.8m.json", - "name": "ai2/llama-2-chat-7b-nectar-3.8m.json", - "developer": "AI2", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5843, - "reward-bench/Chat": 0.8631, - "reward-bench/Chat Hard": 0.2654, - "reward-bench/Safety": 0.6243 - } - }, - { - "id": "ai2/llama-2-chat-nectar-180k.json", - "name": "ai2/llama-2-chat-nectar-180k.json", - "developer": "AI2", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5235, - "reward-bench/Chat": 0.8827, - "reward-bench/Chat Hard": 0.2851, - "reward-bench/Safety": 0.4027 - } - }, - { - "id": "ai2/llama-2-chat-ultrafeedback-60k.jsonl", - "name": "ai2/llama-2-chat-ultrafeedback-60k.jsonl", - "developer": "AI2", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.644, - "reward-bench/Chat": 0.9441, - "reward-bench/Chat Hard": 0.4539, - "reward-bench/Safety": 0.5338 - } - }, - { - "id": "ai2/tulu-2-7b-rm-v0-nectar-binarized-3.8m-check...", - "name": "ai2/tulu-2-7b-rm-v0-nectar-binarized-3.8m-check...", - "developer": "AI2", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7008, - "reward-bench/Chat": 0.9385, - "reward-bench/Chat Hard": 0.3882, - "reward-bench/Safety": 0.7757 - } - }, - { - "id": "ai2/tulu-2-7b-rm-v0-nectar-binarized-700k.json", - "name": "ai2/tulu-2-7b-rm-v0-nectar-binarized-700k.json", - "developer": "AI2", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7127, - "reward-bench/Chat": 0.9358, - "reward-bench/Chat Hard": 0.4079, - "reward-bench/Safety": 0.7946 - } - }, - { - "id": "ai2/tulu-2-7b-rm-v0-nectar-binarized.json", - "name": "ai2/tulu-2-7b-rm-v0-nectar-binarized.json", - "developer": "AI2", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6756, - "reward-bench/Chat": 0.9134, - "reward-bench/Chat Hard": 0.3904, - "reward-bench/Safety": 0.723 - } - }, - { - "id": "ai2/tulu-2-7b-rm-v0.json", - "name": "ai2/tulu-2-7b-rm-v0.json", - "developer": "AI2", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6655, - "reward-bench/Chat": 0.933, - "reward-bench/Chat Hard": 0.4539, - "reward-bench/Safety": 0.6095 - } - }, - { - "id": "ai21/J1-Grande-v1-17B", - "name": "J1-Grande v1 17B", - "developer": "ai21", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.433, - "helm_classic/MMLU": 0.27, - "helm_classic/BoolQ": 0.722, - "helm_classic/NarrativeQA": 0.672, - "helm_classic/NaturalQuestions (open-book)": 0.578, - "helm_classic/QuAC": 0.362, - "helm_classic/HellaSwag": 0.739, - "helm_classic/OpenbookQA": 0.52, - "helm_classic/TruthfulQA": 0.193, - "helm_classic/MS MARCO (TREC)": 0.341, - "helm_classic/CNN/DailyMail": 0.143, - "helm_classic/XSUM": 0.122, - "helm_classic/IMDB": 0.953, - "helm_classic/CivilComments": 0.529, - "helm_classic/RAFT": 0.658 - } - }, - { - "id": "ai21/J1-Grande-v2-beta-17B", - "name": "J1-Grande v2 beta 17B", - "developer": "ai21", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.706, - "helm_classic/MMLU": 0.445, - "helm_classic/BoolQ": 0.812, - "helm_classic/NarrativeQA": 0.725, - "helm_classic/NaturalQuestions (open-book)": 0.625, - "helm_classic/QuAC": 0.392, - "helm_classic/HellaSwag": 0.764, - "helm_classic/OpenbookQA": 0.56, - "helm_classic/TruthfulQA": 0.306, - "helm_classic/MS MARCO (TREC)": 0.46, - "helm_classic/CNN/DailyMail": 0.146, - "helm_classic/XSUM": 0.152, - "helm_classic/IMDB": 0.957, - "helm_classic/CivilComments": 0.546, - "helm_classic/RAFT": 0.679 - } - }, - { - "id": "ai21/J1-Jumbo-v1-178B", - "name": "J1-Jumbo v1 178B", - "developer": "ai21", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.517, - "helm_classic/MMLU": 0.259, - "helm_classic/BoolQ": 0.776, - "helm_classic/NarrativeQA": 0.695, - "helm_classic/NaturalQuestions (open-book)": 0.595, - "helm_classic/QuAC": 0.358, - "helm_classic/HellaSwag": 0.765, - "helm_classic/OpenbookQA": 0.534, - "helm_classic/TruthfulQA": 0.175, - "helm_classic/MS MARCO (TREC)": 0.363, - "helm_classic/CNN/DailyMail": 0.144, - "helm_classic/XSUM": 0.129, - "helm_classic/IMDB": 0.943, - "helm_classic/CivilComments": 0.553, - "helm_classic/RAFT": 0.681 - } - }, - { - "id": "ai21/J1-Large-v1-7.5B", - "name": "J1-Large v1 7.5B", - "developer": "ai21", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.285, - "helm_classic/MMLU": 0.241, - "helm_classic/BoolQ": 0.683, - "helm_classic/NarrativeQA": 0.623, - "helm_classic/NaturalQuestions (open-book)": 0.532, - "helm_classic/QuAC": 0.328, - "helm_classic/HellaSwag": 0.7, - "helm_classic/OpenbookQA": 0.514, - "helm_classic/TruthfulQA": 0.197, - "helm_classic/MS MARCO (TREC)": 0.292, - "helm_classic/CNN/DailyMail": 0.134, - "helm_classic/XSUM": 0.102, - "helm_classic/IMDB": 0.956, - "helm_classic/CivilComments": 0.532, - "helm_classic/RAFT": 0.545 - } - }, - { - "id": "ai21/j2-grande", - "name": "Jurassic-2 Grande 17B", - "developer": "ai21", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.172, - "helm_lite/NarrativeQA": 0.744, - "helm_lite/NaturalQuestions (closed-book)": 0.35, - "helm_lite/OpenbookQA": 0.614, - "helm_lite/MMLU": 0.471, - "helm_lite/MATH": 0.064, - "helm_lite/GSM8K": 0.159, - "helm_lite/LegalBench": 0.468, - "helm_lite/MedQA": 0.39, - "helm_lite/WMT 2014": 0.102 - } - }, - { - "id": "ai21/j2-jumbo", - "name": "Jurassic-2 Jumbo 178B", - "developer": "ai21", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.215, - "helm_lite/NarrativeQA": 0.728, - "helm_lite/NaturalQuestions (closed-book)": 0.385, - "helm_lite/OpenbookQA": 0.688, - "helm_lite/MMLU": 0.483, - "helm_lite/MATH": 0.103, - "helm_lite/GSM8K": 0.239, - "helm_lite/LegalBench": 0.533, - "helm_lite/MedQA": 0.431, - "helm_lite/WMT 2014": 0.114 - } - }, - { - "id": "ai21/jamba-1.5-large", - "name": "Jamba 1.5 Large", - "developer": "ai21", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.637, - "helm_lite/NarrativeQA": 0.664, - "helm_lite/NaturalQuestions (closed-book)": 0.394, - "helm_lite/OpenbookQA": 0.948, - "helm_lite/MMLU": 0.683, - "helm_lite/MATH": 0.692, - "helm_lite/GSM8K": 0.846, - "helm_lite/LegalBench": 0.675, - "helm_lite/MedQA": 0.698, - "helm_lite/WMT 2014": 0.203, - "helm_mmlu/MMLU All Subjects": 0.782, - "helm_mmlu/Abstract Algebra": 0.53, - "helm_mmlu/Anatomy": 0.793, - "helm_mmlu/College Physics": 0.51, - "helm_mmlu/Computer Security": 0.8, - "helm_mmlu/Econometrics": 0.614, - "helm_mmlu/Global Facts": 0.54, - "helm_mmlu/Jurisprudence": 0.87, - "helm_mmlu/Philosophy": 0.849, - "helm_mmlu/Professional Psychology": 0.842, - "helm_mmlu/Us Foreign Policy": 0.92, - "helm_mmlu/Astronomy": 0.882, - "helm_mmlu/Business Ethics": 0.77, - "helm_mmlu/Clinical Knowledge": 0.849, - "helm_mmlu/Conceptual Physics": 0.779, - "helm_mmlu/Electrical Engineering": 0.793, - "helm_mmlu/Elementary Mathematics": 0.656, - "helm_mmlu/Formal Logic": 0.619, - "helm_mmlu/High School World History": 0.911, - "helm_mmlu/Human Sexuality": 0.832, - "helm_mmlu/International Law": 0.884, - "helm_mmlu/Logical Fallacies": 0.859, - "helm_mmlu/Machine Learning": 0.688, - "helm_mmlu/Management": 0.864, - "helm_mmlu/Marketing": 0.94, - "helm_mmlu/Medical Genetics": 0.89, - "helm_mmlu/Miscellaneous": 0.931, - "helm_mmlu/Moral Scenarios": 0.686, - "helm_mmlu/Nutrition": 0.869, - "helm_mmlu/Prehistory": 0.892, - "helm_mmlu/Public Relations": 0.755, - "helm_mmlu/Security Studies": 0.771, - "helm_mmlu/Sociology": 0.93, - "helm_mmlu/Virology": 0.554, - "helm_mmlu/World Religions": 0.865, - "helm_mmlu/Mean win rate": 0.147 - } - }, - { - "id": "ai21/jamba-1.5-mini", - "name": "Jamba 1.5 Mini", - "developer": "ai21", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.414, - "helm_lite/NarrativeQA": 0.746, - "helm_lite/NaturalQuestions (closed-book)": 0.388, - "helm_lite/OpenbookQA": 0.89, - "helm_lite/MMLU": 0.582, - "helm_lite/MATH": 0.318, - "helm_lite/GSM8K": 0.691, - "helm_lite/LegalBench": 0.503, - "helm_lite/MedQA": 0.632, - "helm_lite/WMT 2014": 0.179, - "helm_mmlu/MMLU All Subjects": 0.699, - "helm_mmlu/Abstract Algebra": 0.33, - "helm_mmlu/Anatomy": 0.711, - "helm_mmlu/College Physics": 0.48, - "helm_mmlu/Computer Security": 0.73, - "helm_mmlu/Econometrics": 0.491, - "helm_mmlu/Global Facts": 0.43, - "helm_mmlu/Jurisprudence": 0.88, - "helm_mmlu/Philosophy": 0.752, - "helm_mmlu/Professional Psychology": 0.76, - "helm_mmlu/Us Foreign Policy": 0.9, - "helm_mmlu/Astronomy": 0.822, - "helm_mmlu/Business Ethics": 0.76, - "helm_mmlu/Clinical Knowledge": 0.74, - "helm_mmlu/Conceptual Physics": 0.677, - "helm_mmlu/Electrical Engineering": 0.683, - "helm_mmlu/Elementary Mathematics": 0.553, - "helm_mmlu/Formal Logic": 0.452, - "helm_mmlu/High School World History": 0.84, - "helm_mmlu/Human Sexuality": 0.809, - "helm_mmlu/International Law": 0.893, - "helm_mmlu/Logical Fallacies": 0.81, - "helm_mmlu/Machine Learning": 0.509, - "helm_mmlu/Management": 0.825, - "helm_mmlu/Marketing": 0.915, - "helm_mmlu/Medical Genetics": 0.69, - "helm_mmlu/Miscellaneous": 0.902, - "helm_mmlu/Moral Scenarios": 0.269, - "helm_mmlu/Nutrition": 0.801, - "helm_mmlu/Prehistory": 0.824, - "helm_mmlu/Public Relations": 0.727, - "helm_mmlu/Security Studies": 0.755, - "helm_mmlu/Sociology": 0.876, - "helm_mmlu/Virology": 0.578, - "helm_mmlu/World Religions": 0.842, - "helm_mmlu/Mean win rate": 0.206 - } - }, - { - "id": "ai21/jamba-instruct", - "name": "Jamba Instruct", - "developer": "ai21", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.287, - "helm_lite/NarrativeQA": 0.658, - "helm_lite/NaturalQuestions (closed-book)": 0.384, - "helm_lite/OpenbookQA": 0.796, - "helm_lite/MMLU": 0.582, - "helm_lite/MATH": 0.38, - "helm_lite/GSM8K": 0.67, - "helm_lite/LegalBench": 0.54, - "helm_lite/MedQA": 0.519, - "helm_lite/WMT 2014": 0.164, - "helm_mmlu/MMLU All Subjects": 0.659, - "helm_mmlu/Abstract Algebra": 0.36, - "helm_mmlu/Anatomy": 0.615, - "helm_mmlu/College Physics": 0.422, - "helm_mmlu/Computer Security": 0.76, - "helm_mmlu/Econometrics": 0.439, - "helm_mmlu/Global Facts": 0.4, - "helm_mmlu/Jurisprudence": 0.796, - "helm_mmlu/Philosophy": 0.749, - "helm_mmlu/Professional Psychology": 0.716, - "helm_mmlu/Us Foreign Policy": 0.91, - "helm_mmlu/Astronomy": 0.73, - "helm_mmlu/Business Ethics": 0.6, - "helm_mmlu/Clinical Knowledge": 0.702, - "helm_mmlu/Conceptual Physics": 0.677, - "helm_mmlu/Electrical Engineering": 0.621, - "helm_mmlu/Elementary Mathematics": 0.497, - "helm_mmlu/Formal Logic": 0.444, - "helm_mmlu/High School World History": 0.797, - "helm_mmlu/Human Sexuality": 0.794, - "helm_mmlu/International Law": 0.835, - "helm_mmlu/Logical Fallacies": 0.706, - "helm_mmlu/Machine Learning": 0.536, - "helm_mmlu/Management": 0.786, - "helm_mmlu/Marketing": 0.885, - "helm_mmlu/Medical Genetics": 0.67, - "helm_mmlu/Miscellaneous": 0.865, - "helm_mmlu/Moral Scenarios": 0.465, - "helm_mmlu/Nutrition": 0.745, - "helm_mmlu/Prehistory": 0.796, - "helm_mmlu/Public Relations": 0.682, - "helm_mmlu/Security Studies": 0.743, - "helm_mmlu/Sociology": 0.891, - "helm_mmlu/Virology": 0.53, - "helm_mmlu/World Religions": 0.813, - "helm_mmlu/Mean win rate": 0.887 - } - }, - { - "id": "ai21/Jurassic-2-Grande-17B", - "name": "Jurassic-2 Grande 17B", - "developer": "ai21", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.743, - "helm_classic/MMLU": 0.475, - "helm_classic/BoolQ": 0.826, - "helm_classic/NarrativeQA": 0.737, - "helm_classic/NaturalQuestions (open-book)": 0.639, - "helm_classic/QuAC": 0.418, - "helm_classic/HellaSwag": 0.781, - "helm_classic/OpenbookQA": 0.542, - "helm_classic/TruthfulQA": 0.348, - "helm_classic/MS MARCO (TREC)": 0.514, - "helm_classic/CNN/DailyMail": 0.144, - "helm_classic/XSUM": 0.167, - "helm_classic/IMDB": 0.938, - "helm_classic/CivilComments": 0.547, - "helm_classic/RAFT": 0.712 - } - }, - { - "id": "ai21/Jurassic-2-Jumbo-178B", - "name": "Jurassic-2 Jumbo 178B", - "developer": "ai21", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.824, - "helm_classic/MMLU": 0.48, - "helm_classic/BoolQ": 0.829, - "helm_classic/NarrativeQA": 0.733, - "helm_classic/NaturalQuestions (open-book)": 0.669, - "helm_classic/QuAC": 0.435, - "helm_classic/HellaSwag": 0.788, - "helm_classic/OpenbookQA": 0.558, - "helm_classic/TruthfulQA": 0.437, - "helm_classic/MS MARCO (TREC)": 0.661, - "helm_classic/CNN/DailyMail": 0.149, - "helm_classic/XSUM": 0.182, - "helm_classic/IMDB": 0.938, - "helm_classic/CivilComments": 0.57, - "helm_classic/RAFT": 0.746 - } - }, - { - "id": "ai21/Jurassic-2-Large-7.5B", - "name": "Jurassic-2 Large 7.5B", - "developer": "ai21", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.553, - "helm_classic/MMLU": 0.339, - "helm_classic/BoolQ": 0.742, - "helm_classic/NarrativeQA": -1.0, - "helm_classic/NaturalQuestions (open-book)": 0.589, - "helm_classic/QuAC": -1.0, - "helm_classic/HellaSwag": 0.729, - "helm_classic/OpenbookQA": 0.53, - "helm_classic/TruthfulQA": 0.245, - "helm_classic/MS MARCO (TREC)": 0.464, - "helm_classic/CNN/DailyMail": 0.136, - "helm_classic/XSUM": 0.142, - "helm_classic/IMDB": 0.956, - "helm_classic/CivilComments": 0.57, - "helm_classic/RAFT": 0.622 - } - }, - { - "id": "ai21labs/Jamba-v0.1", - "name": "Jamba-v0.1", - "developer": "ai21labs", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2026, - "hfopenllm_v2/BBH": 0.3602, - "hfopenllm_v2/MATH Level 5": 0.0159, - "hfopenllm_v2/GPQA": 0.2685, - "hfopenllm_v2/MUSR": 0.359, - "hfopenllm_v2/MMLU-PRO": 0.2492 - } - }, - { - "id": "ai4bharat/Airavata", - "name": "Airavata", - "developer": "ai4bharat", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0559, - "hfopenllm_v2/BBH": 0.3628, - "hfopenllm_v2/MATH Level 5": 0.0181, - "hfopenllm_v2/GPQA": 0.2743, - "hfopenllm_v2/MUSR": 0.3763, - "hfopenllm_v2/MMLU-PRO": 0.1635 - } - }, - { - "id": "AI4free/Dhanishtha", - "name": "Dhanishtha", - "developer": "AI4free", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2451, - "hfopenllm_v2/BBH": 0.3404, - "hfopenllm_v2/MATH Level 5": 0.256, - "hfopenllm_v2/GPQA": 0.2525, - "hfopenllm_v2/MUSR": 0.3569, - "hfopenllm_v2/MMLU-PRO": 0.1643 - } - }, - { - "id": "AI4free/t2", - "name": "t2", - "developer": "AI4free", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3867, - "hfopenllm_v2/BBH": 0.291, - "hfopenllm_v2/MATH Level 5": 0.1896, - "hfopenllm_v2/GPQA": 0.2576, - "hfopenllm_v2/MUSR": 0.3846, - "hfopenllm_v2/MMLU-PRO": 0.1144 - } - }, - { - "id": "AicoresSecurity/Cybernet-Sec-3B-R1-V0", - "name": "Cybernet-Sec-3B-R1-V0", - "developer": "AicoresSecurity", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6358, - "hfopenllm_v2/BBH": 0.4497, - "hfopenllm_v2/MATH Level 5": 0.1156, - "hfopenllm_v2/GPQA": 0.2634, - "hfopenllm_v2/MUSR": 0.3314, - "hfopenllm_v2/MMLU-PRO": 0.301 - } - }, - { - "id": "AicoresSecurity/Cybernet-Sec-3B-R1-V0-Coder", - "name": "Cybernet-Sec-3B-R1-V0-Coder", - "developer": "AicoresSecurity", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7098, - "hfopenllm_v2/BBH": 0.4478, - "hfopenllm_v2/MATH Level 5": 0.1488, - "hfopenllm_v2/GPQA": 0.2718, - "hfopenllm_v2/MUSR": 0.3408, - "hfopenllm_v2/MMLU-PRO": 0.3178 - } - }, - { - "id": "AicoresSecurity/Cybernet-Sec-3B-R1-V1", - "name": "Cybernet-Sec-3B-R1-V1", - "developer": "AicoresSecurity", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6146, - "hfopenllm_v2/BBH": 0.4282, - "hfopenllm_v2/MATH Level 5": 0.1518, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.3287, - "hfopenllm_v2/MMLU-PRO": 0.2876 - } - }, - { - "id": "AicoresSecurity/Cybernet-Sec-3B-R1-V1.1", - "name": "Cybernet-Sec-3B-R1-V1.1", - "developer": "AicoresSecurity", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.673, - "hfopenllm_v2/BBH": 0.4392, - "hfopenllm_v2/MATH Level 5": 0.176, - "hfopenllm_v2/GPQA": 0.271, - "hfopenllm_v2/MUSR": 0.3541, - "hfopenllm_v2/MMLU-PRO": 0.3088 - } - }, - { - "id": "AIDC-AI/Marco-o1", - "name": "Marco-o1", - "developer": "AIDC-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4771, - "hfopenllm_v2/BBH": 0.5364, - "hfopenllm_v2/MATH Level 5": 0.3746, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.4138, - "hfopenllm_v2/MMLU-PRO": 0.4117 - } - }, - { - "id": "aixonlab/Aether-12b", - "name": "Aether-12b", - "developer": "aixonlab", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2347, - "hfopenllm_v2/BBH": 0.5179, - "hfopenllm_v2/MATH Level 5": 0.1065, - "hfopenllm_v2/GPQA": 0.3163, - "hfopenllm_v2/MUSR": 0.3829, - "hfopenllm_v2/MMLU-PRO": 0.341 - } - }, - { - "id": "aixonlab/Grey-12b", - "name": "Grey-12b", - "developer": "aixonlab", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3968, - "hfopenllm_v2/BBH": 0.5699, - "hfopenllm_v2/MATH Level 5": 0.0982, - "hfopenllm_v2/GPQA": 0.3003, - "hfopenllm_v2/MUSR": 0.4516, - "hfopenllm_v2/MMLU-PRO": 0.3779 - } - }, - { - "id": "aixonlab/Zara-14b-v1.2", - "name": "Zara-14b-v1.2", - "developer": "aixonlab", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6197, - "hfopenllm_v2/BBH": 0.6405, - "hfopenllm_v2/MATH Level 5": 0.3535, - "hfopenllm_v2/GPQA": 0.3817, - "hfopenllm_v2/MUSR": 0.4675, - "hfopenllm_v2/MMLU-PRO": 0.5263 - } - }, - { - "id": "akhadangi/Llama3.2.1B.0.01-First", - "name": "Llama3.2.1B.0.01-First", - "developer": "akhadangi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0814, - "hfopenllm_v2/BBH": 0.3189, - "hfopenllm_v2/MATH Level 5": 0.0181, - "hfopenllm_v2/GPQA": 0.2483, - "hfopenllm_v2/MUSR": 0.3194, - "hfopenllm_v2/MMLU-PRO": 0.1197 - } - }, - { - "id": "akhadangi/Llama3.2.1B.0.01-Last", - "name": "Llama3.2.1B.0.01-Last", - "developer": "akhadangi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0917, - "hfopenllm_v2/BBH": 0.3159, - "hfopenllm_v2/MATH Level 5": 0.0136, - "hfopenllm_v2/GPQA": 0.2433, - "hfopenllm_v2/MUSR": 0.3206, - "hfopenllm_v2/MMLU-PRO": 0.1227 - } - }, - { - "id": "akhadangi/Llama3.2.1B.0.1-First", - "name": "Llama3.2.1B.0.1-First", - "developer": "akhadangi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1001, - "hfopenllm_v2/BBH": 0.312, - "hfopenllm_v2/MATH Level 5": 0.0211, - "hfopenllm_v2/GPQA": 0.245, - "hfopenllm_v2/MUSR": 0.3301, - "hfopenllm_v2/MMLU-PRO": 0.1169 - } - }, - { - "id": "akhadangi/Llama3.2.1B.0.1-Last", - "name": "Llama3.2.1B.0.1-Last", - "developer": "akhadangi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.095, - "hfopenllm_v2/BBH": 0.3164, - "hfopenllm_v2/MATH Level 5": 0.0211, - "hfopenllm_v2/GPQA": 0.2383, - "hfopenllm_v2/MUSR": 0.3341, - "hfopenllm_v2/MMLU-PRO": 0.1178 - } - }, - { - "id": "akhadangi/Llama3.2.1B.BaseFiT", - "name": "Llama3.2.1B.BaseFiT", - "developer": "akhadangi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0883, - "hfopenllm_v2/BBH": 0.3175, - "hfopenllm_v2/MATH Level 5": 0.0242, - "hfopenllm_v2/GPQA": 0.2534, - "hfopenllm_v2/MUSR": 0.3221, - "hfopenllm_v2/MMLU-PRO": 0.1172 - } - }, - { - "id": "akjindal53244/Llama-3.1-Storm-8B", - "name": "Llama-3.1-Storm-8B", - "developer": "akjindal53244", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8051, - "hfopenllm_v2/BBH": 0.5189, - "hfopenllm_v2/MATH Level 5": 0.1722, - "hfopenllm_v2/GPQA": 0.3263, - "hfopenllm_v2/MUSR": 0.4028, - "hfopenllm_v2/MMLU-PRO": 0.3803 - } - }, - { - "id": "alcholjung/llama3_medical_tuned", - "name": "llama3_medical_tuned", - "developer": "alcholjung", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0106, - "hfopenllm_v2/BBH": 0.4513, - "hfopenllm_v2/MATH Level 5": 0.0468, - "hfopenllm_v2/GPQA": 0.2861, - "hfopenllm_v2/MUSR": 0.466, - "hfopenllm_v2/MMLU-PRO": 0.2946 - } - }, - { - "id": "Alepach/notHumpback-M0", - "name": "notHumpback-M0", - "developer": "Alepach", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.235, - "hfopenllm_v2/BBH": 0.2785, - "hfopenllm_v2/MATH Level 5": 0.0189, - "hfopenllm_v2/GPQA": 0.2492, - "hfopenllm_v2/MUSR": 0.3552, - "hfopenllm_v2/MMLU-PRO": 0.1119 - } - }, - { - "id": "Alepach/notHumpback-M1", - "name": "notHumpback-M1", - "developer": "Alepach", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2207, - "hfopenllm_v2/BBH": 0.2882, - "hfopenllm_v2/MATH Level 5": 0.0159, - "hfopenllm_v2/GPQA": 0.2374, - "hfopenllm_v2/MUSR": 0.342, - "hfopenllm_v2/MMLU-PRO": 0.1091 - } - }, - { - "id": "Alepach/notHumpback-M1-v2", - "name": "notHumpback-M1-v2", - "developer": "Alepach", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2277, - "hfopenllm_v2/BBH": 0.2776, - "hfopenllm_v2/MATH Level 5": 0.0219, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3473, - "hfopenllm_v2/MMLU-PRO": 0.1119 - } - }, - { - "id": "aleph-alpha/Luminous-Base-13B", - "name": "Luminous Base 13B", - "developer": "aleph-alpha", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.315, - "helm_classic/MMLU": 0.27, - "helm_classic/BoolQ": 0.719, - "helm_classic/NarrativeQA": 0.605, - "helm_classic/NaturalQuestions (open-book)": 0.568, - "helm_classic/QuAC": 0.334, - "helm_classic/HellaSwag": -1.0, - "helm_classic/OpenbookQA": -1.0, - "helm_classic/TruthfulQA": 0.182, - "helm_classic/MS MARCO (TREC)": -1.0, - "helm_classic/CNN/DailyMail": 0.11, - "helm_classic/XSUM": 0.105, - "helm_classic/IMDB": 0.939, - "helm_classic/CivilComments": 0.544, - "helm_classic/RAFT": 0.473 - } - }, - { - "id": "aleph-alpha/Luminous-Extended-30B", - "name": "Luminous Extended 30B", - "developer": "aleph-alpha", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.485, - "helm_classic/MMLU": 0.321, - "helm_classic/BoolQ": 0.767, - "helm_classic/NarrativeQA": 0.665, - "helm_classic/NaturalQuestions (open-book)": 0.609, - "helm_classic/QuAC": 0.349, - "helm_classic/HellaSwag": -1.0, - "helm_classic/OpenbookQA": -1.0, - "helm_classic/TruthfulQA": 0.221, - "helm_classic/MS MARCO (TREC)": -1.0, - "helm_classic/CNN/DailyMail": 0.139, - "helm_classic/XSUM": 0.124, - "helm_classic/IMDB": 0.947, - "helm_classic/CivilComments": 0.524, - "helm_classic/RAFT": 0.523 - } - }, - { - "id": "aleph-alpha/Luminous-Supreme-70B", - "name": "Luminous Supreme 70B", - "developer": "aleph-alpha", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.662, - "helm_classic/MMLU": 0.38, - "helm_classic/BoolQ": 0.775, - "helm_classic/NarrativeQA": 0.711, - "helm_classic/NaturalQuestions (open-book)": 0.649, - "helm_classic/QuAC": 0.37, - "helm_classic/HellaSwag": -1.0, - "helm_classic/OpenbookQA": -1.0, - "helm_classic/TruthfulQA": 0.222, - "helm_classic/MS MARCO (TREC)": -1.0, - "helm_classic/CNN/DailyMail": 0.15, - "helm_classic/XSUM": 0.136, - "helm_classic/IMDB": 0.959, - "helm_classic/CivilComments": 0.562, - "helm_classic/RAFT": 0.653 - } - }, - { - "id": "AlephAlpha/luminous-base", - "name": "Luminous Base 13B", - "developer": "AlephAlpha", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.041, - "helm_lite/NarrativeQA": 0.633, - "helm_lite/NaturalQuestions (closed-book)": 0.197, - "helm_lite/OpenbookQA": 0.286, - "helm_lite/MMLU": 0.243, - "helm_lite/MATH": 0.026, - "helm_lite/GSM8K": 0.028, - "helm_lite/LegalBench": 0.332, - "helm_lite/MedQA": 0.26, - "helm_lite/WMT 2014": 0.066 - } - }, - { - "id": "AlephAlpha/luminous-extended", - "name": "Luminous Extended 30B", - "developer": "AlephAlpha", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.078, - "helm_lite/NarrativeQA": 0.684, - "helm_lite/NaturalQuestions (closed-book)": 0.253, - "helm_lite/OpenbookQA": 0.272, - "helm_lite/MMLU": 0.248, - "helm_lite/MATH": 0.04, - "helm_lite/GSM8K": 0.075, - "helm_lite/LegalBench": 0.421, - "helm_lite/MedQA": 0.276, - "helm_lite/WMT 2014": 0.083 - } - }, - { - "id": "AlephAlpha/luminous-supreme", - "name": "Luminous Supreme 70B", - "developer": "AlephAlpha", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.145, - "helm_lite/NarrativeQA": 0.743, - "helm_lite/NaturalQuestions (closed-book)": 0.299, - "helm_lite/OpenbookQA": 0.284, - "helm_lite/MMLU": 0.316, - "helm_lite/MATH": 0.078, - "helm_lite/GSM8K": 0.137, - "helm_lite/LegalBench": 0.452, - "helm_lite/MedQA": 0.276, - "helm_lite/WMT 2014": 0.102 - } - }, - { - "id": "Alibaba-NLP/gte-Qwen2-7B-instruct", - "name": "gte-Qwen2-7B-instruct", - "developer": "Alibaba-NLP", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2255, - "hfopenllm_v2/BBH": 0.4495, - "hfopenllm_v2/MATH Level 5": 0.0642, - "hfopenllm_v2/GPQA": 0.245, - "hfopenllm_v2/MUSR": 0.3559, - "hfopenllm_v2/MMLU-PRO": 0.3321 - } - }, - { - "id": "alibaba/qwen-3-coder-480b", - "name": "Qwen 3 Coder 480B", - "developer": "Alibaba", - "evaluator_relationship": null, - "benchmark_scores": { - "terminal-bench-2.0/terminal-bench-2.0": 23.9 - } - }, - { - "id": "alibaba/qwen3-235b-a22b-instruct-2507", - "name": "qwen3-235b-a22b-instruct-2507", - "developer": "alibaba", - "evaluator_relationship": null, - "benchmark_scores": { - "global-mmlu-lite/Global MMLU Lite": 0.8798, - "global-mmlu-lite/Culturally Sensitive": 0.8522, - "global-mmlu-lite/Culturally Agnostic": 0.9075, - "global-mmlu-lite/Arabic": 0.88, - "global-mmlu-lite/English": 0.89, - "global-mmlu-lite/Bengali": 0.8875, - "global-mmlu-lite/German": 0.885, - "global-mmlu-lite/French": 0.88, - "global-mmlu-lite/Hindi": 0.8775, - "global-mmlu-lite/Indonesian": 0.88, - "global-mmlu-lite/Italian": 0.88, - "global-mmlu-lite/Japanese": 0.88, - "global-mmlu-lite/Korean": 0.875, - "global-mmlu-lite/Portuguese": 0.8875, - "global-mmlu-lite/Spanish": 0.875, - "global-mmlu-lite/Swahili": 0.87, - "global-mmlu-lite/Yoruba": 0.8725, - "global-mmlu-lite/Chinese": 0.8775, - "global-mmlu-lite/Burmese": 0.88 - } - }, - { - "id": "alibaba/qwen3-235b-a22b-thinking-2507", - "name": "qwen3-235b-a22b-thinking-2507", - "developer": "Alibaba", - "evaluator_relationship": null, - "benchmark_scores": { - "livecodebenchpro/Hard Problems": 0.0, - "livecodebenchpro/Medium Problems": 0.1267605633802817, - "livecodebenchpro/Easy Problems": 0.7605633802816901 - } - }, - { - "id": "alibaba/qwen3-30b-a3b", - "name": "qwen3-30b-a3b", - "developer": "Alibaba", - "evaluator_relationship": null, - "benchmark_scores": { - "livecodebenchpro/Hard Problems": 0.0, - "livecodebenchpro/Medium Problems": 0.028169014084507043, - "livecodebenchpro/Easy Problems": 0.5774647887323944 - } - }, - { - "id": "alibaba/qwen3-max", - "name": "alibaba/qwen3-max", - "developer": "Alibaba", - "evaluator_relationship": null, - "benchmark_scores": { - "livecodebenchpro/Hard Problems": 0.0, - "livecodebenchpro/Medium Problems": 0.04225352112676056, - "livecodebenchpro/Easy Problems": 0.36619718309859156 - } - }, - { - "id": "alibaba/qwen3-next-80b-a3b-thinking", - "name": "qwen3-next-80b-a3b-thinking", - "developer": "Alibaba", - "evaluator_relationship": null, - "benchmark_scores": { - "livecodebenchpro/Hard Problems": 0.0, - "livecodebenchpro/Medium Problems": 0.14084507042253522, - "livecodebenchpro/Easy Problems": 0.7464788732394366 - } - }, - { - "id": "aliyun/qwen3-next-80b-a3b-thinking", - "name": "qwen3-next-80b-a3b-thinking", - "developer": "aliyun", - "evaluator_relationship": null, - "benchmark_scores": { - "livecodebenchpro/Hard Problems": 0.0, - "livecodebenchpro/Medium Problems": 0.0704, - "livecodebenchpro/Easy Problems": 0.6901 - } - }, - { - "id": "allenai/llama-3-tulu-2-70b-uf-mean-rm", - "name": "allenai/llama-3-tulu-2-70b-uf-mean-rm", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7019, - "reward-bench/Chat": 0.8631, - "reward-bench/Chat Hard": 0.5614, - "reward-bench/Safety": 0.6095, - "reward-bench/Reasoning": 0.8268, - "reward-bench/Prior Sets (0.5 weight)": 0.5957 - } - }, - { - "id": "allenai/llama-3-tulu-2-8b-uf-mean-rm", - "name": "allenai/llama-3-tulu-2-8b-uf-mean-rm", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7342, - "reward-bench/Chat": 0.9525, - "reward-bench/Chat Hard": 0.5921, - "reward-bench/Safety": 0.6162, - "reward-bench/Reasoning": 0.8212, - "reward-bench/Prior Sets (0.5 weight)": 0.6434 - } - }, - { - "id": "allenai/llama-3-tulu-2-dpo-70b", - "name": "allenai/llama-3-tulu-2-dpo-70b", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7496, - "reward-bench/Chat": 0.9637, - "reward-bench/Chat Hard": 0.5746, - "reward-bench/Safety": 0.7486, - "reward-bench/Reasoning": 0.802, - "reward-bench/Prior Sets (0.5 weight)": 0.5687 - } - }, - { - "id": "allenai/llama-3-tulu-2-dpo-8b", - "name": "allenai/llama-3-tulu-2-dpo-8b", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7275, - "reward-bench/Chat": 0.9525, - "reward-bench/Chat Hard": 0.5351, - "reward-bench/Safety": 0.6649, - "reward-bench/Reasoning": 0.8663, - "reward-bench/Prior Sets (0.5 weight)": 0.5097 - } - }, - { - "id": "allenai/Llama-3.1-70B-Instruct-RM-RB2", - "name": "allenai/Llama-3.1-70B-Instruct-RM-RB2", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.9021, - "reward-bench/Factuality": 0.8126, - "reward-bench/Precise IF": 0.4188, - "reward-bench/Math": 0.6995, - "reward-bench/Safety": 0.9095, - "reward-bench/Focus": 0.8646, - "reward-bench/Ties": 0.8835, - "reward-bench/Chat": 0.9665, - "reward-bench/Chat Hard": 0.8355, - "reward-bench/Reasoning": 0.8969, - "reward-bench/Prior Sets (0.5 weight)": 0.0 - } - }, - { - "id": "allenai/Llama-3.1-8B-Base-RM-RB2", - "name": "allenai/Llama-3.1-8B-Base-RM-RB2", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.649, - "reward-bench/Chat": 0.933, - "reward-bench/Chat Hard": 0.7785, - "reward-bench/Safety": 0.8267, - "reward-bench/Reasoning": 0.7886, - "reward-bench/Prior Sets (0.5 weight)": 0.0, - "reward-bench/Factuality": 0.72, - "reward-bench/Precise IF": 0.3625, - "reward-bench/Math": 0.612, - "reward-bench/Focus": 0.8323, - "reward-bench/Ties": 0.5406 - } - }, - { - "id": "allenai/Llama-3.1-8B-Instruct-RM-RB2", - "name": "allenai/Llama-3.1-8B-Instruct-RM-RB2", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.8885, - "reward-bench/Factuality": 0.7432, - "reward-bench/Precise IF": 0.4437, - "reward-bench/Math": 0.6175, - "reward-bench/Safety": 0.8932, - "reward-bench/Focus": 0.9071, - "reward-bench/Ties": 0.7638, - "reward-bench/Chat": 0.9581, - "reward-bench/Chat Hard": 0.8158, - "reward-bench/Reasoning": 0.887, - "reward-bench/Prior Sets (0.5 weight)": 0.0 - } - }, - { - "id": "allenai/Llama-3.1-Tulu-3-70B", - "name": "Llama-3.1-Tulu-3-70B", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8379, - "hfopenllm_v2/BBH": 0.6157, - "hfopenllm_v2/MATH Level 5": 0.3829, - "hfopenllm_v2/GPQA": 0.3733, - "hfopenllm_v2/MUSR": 0.4988, - "hfopenllm_v2/MMLU-PRO": 0.4656 - } - }, - { - "id": "allenai/Llama-3.1-Tulu-3-70B-DPO", - "name": "Llama-3.1-Tulu-3-70B-DPO", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8282, - "hfopenllm_v2/BBH": 0.6146, - "hfopenllm_v2/MATH Level 5": 0.4494, - "hfopenllm_v2/GPQA": 0.3758, - "hfopenllm_v2/MUSR": 0.4923, - "hfopenllm_v2/MMLU-PRO": 0.4633 - } - }, - { - "id": "allenai/Llama-3.1-Tulu-3-70B-SFT", - "name": "Llama-3.1-Tulu-3-70B-SFT", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8051, - "hfopenllm_v2/BBH": 0.5951, - "hfopenllm_v2/MATH Level 5": 0.3316, - "hfopenllm_v2/GPQA": 0.3448, - "hfopenllm_v2/MUSR": 0.5026, - "hfopenllm_v2/MMLU-PRO": 0.4624 - } - }, - { - "id": "allenai/Llama-3.1-Tulu-3-70B-SFT-RM-RB2", - "name": "allenai/Llama-3.1-Tulu-3-70B-SFT-RM-RB2", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.722, - "reward-bench/Chat": 0.9693, - "reward-bench/Chat Hard": 0.8268, - "reward-bench/Safety": 0.8689, - "reward-bench/Reasoning": 0.8583, - "reward-bench/Prior Sets (0.5 weight)": 0.0, - "reward-bench/Factuality": 0.8084, - "reward-bench/Precise IF": 0.3688, - "reward-bench/Math": 0.6776, - "reward-bench/Focus": 0.7778, - "reward-bench/Ties": 0.8308 - } - }, - { - "id": "allenai/Llama-3.1-Tulu-3-8B", - "name": "Llama-3.1-Tulu-3-8B", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8267, - "hfopenllm_v2/BBH": 0.405, - "hfopenllm_v2/MATH Level 5": 0.1964, - "hfopenllm_v2/GPQA": 0.2987, - "hfopenllm_v2/MUSR": 0.4175, - "hfopenllm_v2/MMLU-PRO": 0.2827 - } - }, - { - "id": "allenai/Llama-3.1-Tulu-3-8B-DPO", - "name": "Llama-3.1-Tulu-3-8B-DPO", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8029, - "hfopenllm_v2/BBH": 0.4079, - "hfopenllm_v2/MATH Level 5": 0.2364, - "hfopenllm_v2/GPQA": 0.2936, - "hfopenllm_v2/MUSR": 0.4161, - "hfopenllm_v2/MMLU-PRO": 0.2898 - } - }, - { - "id": "allenai/Llama-3.1-Tulu-3-8B-DPO-RM-RB2", - "name": "allenai/Llama-3.1-Tulu-3-8B-DPO-RM-RB2", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.8431, - "reward-bench/Factuality": 0.7516, - "reward-bench/Precise IF": 0.3875, - "reward-bench/Math": 0.6284, - "reward-bench/Safety": 0.8662, - "reward-bench/Focus": 0.8545, - "reward-bench/Ties": 0.6397, - "reward-bench/Chat": 0.9553, - "reward-bench/Chat Hard": 0.761, - "reward-bench/Reasoning": 0.7898, - "reward-bench/Prior Sets (0.5 weight)": 0.0 - } - }, - { - "id": "allenai/Llama-3.1-Tulu-3-8B-RL-RM-RB2", - "name": "allenai/Llama-3.1-Tulu-3-8B-RL-RM-RB2", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6871, - "reward-bench/Chat": 0.9469, - "reward-bench/Chat Hard": 0.7588, - "reward-bench/Safety": 0.8644, - "reward-bench/Reasoning": 0.7715, - "reward-bench/Prior Sets (0.5 weight)": 0.0, - "reward-bench/Factuality": 0.7642, - "reward-bench/Precise IF": 0.4, - "reward-bench/Math": 0.6175, - "reward-bench/Focus": 0.8485, - "reward-bench/Ties": 0.6281 - } - }, - { - "id": "allenai/Llama-3.1-Tulu-3-8B-RM", - "name": "Llama-3.1-Tulu-3-8B-RM", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.167, - "hfopenllm_v2/BBH": 0.295, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2567, - "hfopenllm_v2/MUSR": 0.3764, - "hfopenllm_v2/MMLU-PRO": 0.1082, - "reward-bench/Score": 0.59, - "reward-bench/Factuality": 0.7453, - "reward-bench/Precise IF": 0.3469, - "reward-bench/Math": 0.6448, - "reward-bench/Safety": 0.7422, - "reward-bench/Focus": 0.5364, - "reward-bench/Ties": 0.5243 - } - }, - { - "id": "allenai/Llama-3.1-Tulu-3-8B-SFT", - "name": "Llama-3.1-Tulu-3-8B-SFT", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7403, - "hfopenllm_v2/BBH": 0.3872, - "hfopenllm_v2/MATH Level 5": 0.1178, - "hfopenllm_v2/GPQA": 0.2777, - "hfopenllm_v2/MUSR": 0.4268, - "hfopenllm_v2/MMLU-PRO": 0.2812 - } - }, - { - "id": "allenai/Llama-3.1-Tulu-3-8B-SFT-RM-RB2", - "name": "allenai/Llama-3.1-Tulu-3-8B-SFT-RM-RB2", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6821, - "reward-bench/Chat": 0.9497, - "reward-bench/Chat Hard": 0.7917, - "reward-bench/Safety": 0.8978, - "reward-bench/Reasoning": 0.8005, - "reward-bench/Prior Sets (0.5 weight)": 0.0, - "reward-bench/Factuality": 0.7326, - "reward-bench/Precise IF": 0.3875, - "reward-bench/Math": 0.5792, - "reward-bench/Focus": 0.8889, - "reward-bench/Ties": 0.6063 - } - }, - { - "id": "allenai/olmo-1.7-7b", - "name": "OLMo 1.7 7B", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_mmlu/MMLU All Subjects": 0.538, - "helm_mmlu/Abstract Algebra": 0.33, - "helm_mmlu/Anatomy": 0.496, - "helm_mmlu/College Physics": 0.333, - "helm_mmlu/Computer Security": 0.65, - "helm_mmlu/Econometrics": 0.404, - "helm_mmlu/Global Facts": 0.34, - "helm_mmlu/Jurisprudence": 0.565, - "helm_mmlu/Philosophy": 0.592, - "helm_mmlu/Professional Psychology": 0.526, - "helm_mmlu/Us Foreign Policy": 0.76, - "helm_mmlu/Astronomy": 0.526, - "helm_mmlu/Business Ethics": 0.59, - "helm_mmlu/Clinical Knowledge": 0.57, - "helm_mmlu/Conceptual Physics": 0.434, - "helm_mmlu/Electrical Engineering": 0.517, - "helm_mmlu/Elementary Mathematics": 0.307, - "helm_mmlu/Formal Logic": 0.325, - "helm_mmlu/High School World History": 0.713, - "helm_mmlu/Human Sexuality": 0.595, - "helm_mmlu/International Law": 0.612, - "helm_mmlu/Logical Fallacies": 0.607, - "helm_mmlu/Machine Learning": 0.375, - "helm_mmlu/Management": 0.689, - "helm_mmlu/Marketing": 0.769, - "helm_mmlu/Medical Genetics": 0.56, - "helm_mmlu/Miscellaneous": 0.734, - "helm_mmlu/Moral Scenarios": 0.335, - "helm_mmlu/Nutrition": 0.608, - "helm_mmlu/Prehistory": 0.593, - "helm_mmlu/Public Relations": 0.6, - "helm_mmlu/Security Studies": 0.522, - "helm_mmlu/Sociology": 0.751, - "helm_mmlu/Virology": 0.452, - "helm_mmlu/World Religions": 0.731, - "helm_mmlu/Mean win rate": 0.196 - } - }, - { - "id": "allenai/OLMo-1.7-7B-hf", - "name": "OLMo-1.7-7B-hf", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1569, - "hfopenllm_v2/BBH": 0.3014, - "hfopenllm_v2/MATH Level 5": 0.0023, - "hfopenllm_v2/GPQA": 0.255, - "hfopenllm_v2/MUSR": 0.3475, - "hfopenllm_v2/MMLU-PRO": 0.1124 - } - }, - { - "id": "allenai/OLMo-1B-hf", - "name": "OLMo-1B-hf", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2182, - "hfopenllm_v2/BBH": 0.3052, - "hfopenllm_v2/MATH Level 5": 0.0174, - "hfopenllm_v2/GPQA": 0.2617, - "hfopenllm_v2/MUSR": 0.4098, - "hfopenllm_v2/MMLU-PRO": 0.1174 - } - }, - { - "id": "allenai/olmo-2-0325-32b-instruct", - "name": "OLMo 2 32B Instruct March 2025", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.475, - "helm_capabilities/MMLU-Pro": 0.414, - "helm_capabilities/GPQA": 0.287, - "helm_capabilities/IFEval": 0.78, - "helm_capabilities/WildBench": 0.734, - "helm_capabilities/Omni-MATH": 0.161 - } - }, - { - "id": "allenai/olmo-2-1124-13b-instruct", - "name": "OLMo 2 13B Instruct November 2024", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.44, - "helm_capabilities/MMLU-Pro": 0.31, - "helm_capabilities/GPQA": 0.316, - "helm_capabilities/IFEval": 0.73, - "helm_capabilities/WildBench": 0.689, - "helm_capabilities/Omni-MATH": 0.156 - } - }, - { - "id": "allenai/OLMo-2-1124-7B-Instruct", - "name": "OLMo 2 7B Instruct November 2024", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.405, - "helm_capabilities/MMLU-Pro": 0.292, - "helm_capabilities/GPQA": 0.296, - "helm_capabilities/IFEval": 0.693, - "helm_capabilities/WildBench": 0.628, - "helm_capabilities/Omni-MATH": 0.116, - "hfopenllm_v2/IFEval": 0.7244, - "hfopenllm_v2/BBH": 0.4022, - "hfopenllm_v2/MATH Level 5": 0.1488, - "hfopenllm_v2/GPQA": 0.2785, - "hfopenllm_v2/MUSR": 0.3508, - "hfopenllm_v2/MMLU-PRO": 0.2672 - } - }, - { - "id": "allenai/olmo-7b", - "name": "OLMo 7B", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.052, - "helm_lite/NarrativeQA": 0.597, - "helm_lite/NaturalQuestions (closed-book)": 0.259, - "helm_lite/OpenbookQA": 0.222, - "helm_lite/MMLU": 0.305, - "helm_lite/MATH": 0.029, - "helm_lite/GSM8K": 0.044, - "helm_lite/LegalBench": 0.341, - "helm_lite/MedQA": 0.229, - "helm_lite/WMT 2014": 0.097, - "helm_mmlu/MMLU All Subjects": 0.295, - "helm_mmlu/Abstract Algebra": 0.26, - "helm_mmlu/Anatomy": 0.222, - "helm_mmlu/College Physics": 0.294, - "helm_mmlu/Computer Security": 0.3, - "helm_mmlu/Econometrics": 0.325, - "helm_mmlu/Global Facts": 0.32, - "helm_mmlu/Jurisprudence": 0.25, - "helm_mmlu/Philosophy": 0.325, - "helm_mmlu/Professional Psychology": 0.232, - "helm_mmlu/Us Foreign Policy": 0.26, - "helm_mmlu/Astronomy": 0.342, - "helm_mmlu/Business Ethics": 0.24, - "helm_mmlu/Clinical Knowledge": 0.26, - "helm_mmlu/Conceptual Physics": 0.319, - "helm_mmlu/Electrical Engineering": 0.29, - "helm_mmlu/Elementary Mathematics": 0.254, - "helm_mmlu/Formal Logic": 0.278, - "helm_mmlu/High School World History": 0.253, - "helm_mmlu/Human Sexuality": 0.267, - "helm_mmlu/International Law": 0.306, - "helm_mmlu/Logical Fallacies": 0.264, - "helm_mmlu/Machine Learning": 0.286, - "helm_mmlu/Management": 0.272, - "helm_mmlu/Marketing": 0.269, - "helm_mmlu/Medical Genetics": 0.28, - "helm_mmlu/Miscellaneous": 0.292, - "helm_mmlu/Moral Scenarios": 0.265, - "helm_mmlu/Nutrition": 0.34, - "helm_mmlu/Prehistory": 0.318, - "helm_mmlu/Public Relations": 0.345, - "helm_mmlu/Security Studies": 0.408, - "helm_mmlu/Sociology": 0.383, - "helm_mmlu/Virology": 0.416, - "helm_mmlu/World Religions": 0.234, - "helm_mmlu/Mean win rate": 0.68 - } - }, - { - "id": "allenai/OLMo-7B-hf", - "name": "OLMo-7B-hf", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2719, - "hfopenllm_v2/BBH": 0.3279, - "hfopenllm_v2/MATH Level 5": 0.0121, - "hfopenllm_v2/GPQA": 0.2727, - "hfopenllm_v2/MUSR": 0.3487, - "hfopenllm_v2/MMLU-PRO": 0.1173 - } - }, - { - "id": "allenai/OLMo-7B-Instruct", - "name": "allenai/OLMo-7B-Instruct", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6727, - "reward-bench/Chat": 0.8966, - "reward-bench/Chat Hard": 0.5066, - "reward-bench/Safety": 0.6486, - "reward-bench/Reasoning": 0.7168, - "reward-bench/Prior Sets (0.5 weight)": 0.5173 - } - }, - { - "id": "allenai/OLMo-7B-Instruct-hf", - "name": "OLMo-7B-Instruct-hf", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3473, - "hfopenllm_v2/BBH": 0.3706, - "hfopenllm_v2/MATH Level 5": 0.0136, - "hfopenllm_v2/GPQA": 0.271, - "hfopenllm_v2/MUSR": 0.3765, - "hfopenllm_v2/MMLU-PRO": 0.1785 - } - }, - { - "id": "allenai/OLMoE-1B-7B-0125-Instruct", - "name": "OLMoE 1B-7B Instruct January 2025", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.332, - "helm_capabilities/MMLU-Pro": 0.169, - "helm_capabilities/GPQA": 0.22, - "helm_capabilities/IFEval": 0.628, - "helm_capabilities/WildBench": 0.551, - "helm_capabilities/Omni-MATH": 0.093, - "hfopenllm_v2/IFEval": 0.6757, - "hfopenllm_v2/BBH": 0.3825, - "hfopenllm_v2/MATH Level 5": 0.0899, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3636, - "hfopenllm_v2/MMLU-PRO": 0.1915 - } - }, - { - "id": "allenai/OLMoE-1B-7B-0924", - "name": "OLMoE-1B-7B-0924", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2185, - "hfopenllm_v2/BBH": 0.3393, - "hfopenllm_v2/MATH Level 5": 0.0166, - "hfopenllm_v2/GPQA": 0.2475, - "hfopenllm_v2/MUSR": 0.3488, - "hfopenllm_v2/MMLU-PRO": 0.174 - } - }, - { - "id": "allenai/OLMoE-1B-7B-0924-Instruct", - "name": "OLMoE-1B-7B-0924-Instruct", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4667, - "hfopenllm_v2/BBH": 0.3902, - "hfopenllm_v2/MATH Level 5": 0.0279, - "hfopenllm_v2/GPQA": 0.2676, - "hfopenllm_v2/MUSR": 0.3848, - "hfopenllm_v2/MMLU-PRO": 0.1876 - } - }, - { - "id": "allenai/open_instruct_dev-reward_modeling__1__1739590997", - "name": "allenai/open_instruct_dev-reward_modeling__1__1739590997", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6004, - "reward-bench/Factuality": 0.7032, - "reward-bench/Precise IF": 0.375, - "reward-bench/Math": 0.623, - "reward-bench/Safety": 0.7867, - "reward-bench/Focus": 0.598, - "reward-bench/Ties": 0.5165 - } - }, - { - "id": "allenai/open_instruct_dev-reward_modeling__1__1739871066", - "name": "allenai/open_instruct_dev-reward_modeling__1__1739871066", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6012, - "reward-bench/Factuality": 0.6989, - "reward-bench/Precise IF": 0.425, - "reward-bench/Math": 0.6284, - "reward-bench/Safety": 0.7978, - "reward-bench/Focus": 0.604, - "reward-bench/Ties": 0.4527 - } - }, - { - "id": "allenai/open_instruct_dev-reward_modeling__1__1739925892", - "name": "allenai/open_instruct_dev-reward_modeling__1__1739925892", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6345, - "reward-bench/Factuality": 0.7432, - "reward-bench/Precise IF": 0.3563, - "reward-bench/Math": 0.623, - "reward-bench/Safety": 0.8111, - "reward-bench/Focus": 0.7131, - "reward-bench/Ties": 0.5606 - } - }, - { - "id": "allenai/open_instruct_dev-reward_modeling__1__1739943850", - "name": "allenai/open_instruct_dev-reward_modeling__1__1739943850", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.4978, - "reward-bench/Factuality": 0.5726, - "reward-bench/Precise IF": 0.3125, - "reward-bench/Math": 0.5191, - "reward-bench/Safety": 0.6489, - "reward-bench/Focus": 0.6222, - "reward-bench/Ties": 0.3114 - } - }, - { - "id": "allenai/open_instruct_dev-reward_modeling__1__1739943881", - "name": "allenai/open_instruct_dev-reward_modeling__1__1739943881", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5998, - "reward-bench/Factuality": 0.7032, - "reward-bench/Precise IF": 0.3187, - "reward-bench/Math": 0.5792, - "reward-bench/Safety": 0.8222, - "reward-bench/Focus": 0.6727, - "reward-bench/Ties": 0.5025 - } - }, - { - "id": "allenai/open_instruct_dev-reward_modeling__1__1739943972", - "name": "allenai/open_instruct_dev-reward_modeling__1__1739943972", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5289, - "reward-bench/Factuality": 0.6168, - "reward-bench/Precise IF": 0.375, - "reward-bench/Math": 0.5738, - "reward-bench/Safety": 0.6844, - "reward-bench/Focus": 0.5657, - "reward-bench/Ties": 0.3577 - } - }, - { - "id": "allenai/open_instruct_dev-reward_modeling__1__1739957701", - "name": "allenai/open_instruct_dev-reward_modeling__1__1739957701", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6194, - "reward-bench/Factuality": 0.6779, - "reward-bench/Precise IF": 0.3563, - "reward-bench/Math": 0.6011, - "reward-bench/Safety": 0.8022, - "reward-bench/Focus": 0.697, - "reward-bench/Ties": 0.5822 - } - }, - { - "id": "allenai/open_instruct_dev-reward_modeling__1__1739971507", - "name": "allenai/open_instruct_dev-reward_modeling__1__1739971507", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5717, - "reward-bench/Factuality": 0.68, - "reward-bench/Precise IF": 0.375, - "reward-bench/Math": 0.6066, - "reward-bench/Safety": 0.7667, - "reward-bench/Focus": 0.5475, - "reward-bench/Ties": 0.4545 - } - }, - { - "id": "allenai/open_instruct_dev-reward_modeling__1__1739971529", - "name": "allenai/open_instruct_dev-reward_modeling__1__1739971529", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5564, - "reward-bench/Factuality": 0.6568, - "reward-bench/Precise IF": 0.3563, - "reward-bench/Math": 0.5956, - "reward-bench/Safety": 0.7533, - "reward-bench/Focus": 0.5737, - "reward-bench/Ties": 0.4027 - } - }, - { - "id": "allenai/open_instruct_dev-reward_modeling__1__1739998765", - "name": "allenai/open_instruct_dev-reward_modeling__1__1739998765", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6008, - "reward-bench/Factuality": 0.7095, - "reward-bench/Precise IF": 0.4125, - "reward-bench/Math": 0.6066, - "reward-bench/Safety": 0.8022, - "reward-bench/Focus": 0.5859, - "reward-bench/Ties": 0.4883 - } - }, - { - "id": "allenai/open_instruct_dev-reward_modeling__1__1740005072", - "name": "allenai/open_instruct_dev-reward_modeling__1__1740005072", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6097, - "reward-bench/Factuality": 0.7137, - "reward-bench/Precise IF": 0.3937, - "reward-bench/Math": 0.6339, - "reward-bench/Safety": 0.7778, - "reward-bench/Focus": 0.6343, - "reward-bench/Ties": 0.5047 - } - }, - { - "id": "allenai/open_instruct_dev-reward_modeling__1__1740129284", - "name": "allenai/open_instruct_dev-reward_modeling__1__1740129284", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6129, - "reward-bench/Factuality": 0.7116, - "reward-bench/Precise IF": 0.4437, - "reward-bench/Math": 0.6448, - "reward-bench/Safety": 0.8022, - "reward-bench/Focus": 0.6101, - "reward-bench/Ties": 0.4652 - } - }, - { - "id": "allenai/open_instruct_dev-reward_modeling__1__1741286813", - "name": "allenai/open_instruct_dev-reward_modeling__1__1741286813", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6557, - "reward-bench/Factuality": 0.6295, - "reward-bench/Precise IF": 0.4188, - "reward-bench/Math": 0.612, - "reward-bench/Safety": 0.9111, - "reward-bench/Focus": 0.8263, - "reward-bench/Ties": 0.5365 - } - }, - { - "id": "allenai/open_instruct_dev-reward_modeling__1__1741287363", - "name": "allenai/open_instruct_dev-reward_modeling__1__1741287363", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6672, - "reward-bench/Factuality": 0.6295, - "reward-bench/Precise IF": 0.375, - "reward-bench/Math": 0.6066, - "reward-bench/Safety": 0.88, - "reward-bench/Focus": 0.9374, - "reward-bench/Ties": 0.5748 - } - }, - { - "id": "allenai/open_instruct_dev-reward_modeling__1__1741292911", - "name": "allenai/open_instruct_dev-reward_modeling__1__1741292911", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6607, - "reward-bench/Factuality": 0.6589, - "reward-bench/Precise IF": 0.4, - "reward-bench/Math": 0.6066, - "reward-bench/Safety": 0.9089, - "reward-bench/Focus": 0.8869, - "reward-bench/Ties": 0.5028 - } - }, - { - "id": "allenai/open_instruct_dev-reward_modeling__1__1742338142", - "name": "allenai/open_instruct_dev-reward_modeling__1__1742338142", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6344, - "reward-bench/Factuality": 0.7326, - "reward-bench/Precise IF": 0.3812, - "reward-bench/Math": 0.7049, - "reward-bench/Safety": 0.88, - "reward-bench/Focus": 0.6323, - "reward-bench/Ties": 0.475 - } - }, - { - "id": "allenai/open_instruct_dev-reward_modeling__1__1742519610", - "name": "allenai/open_instruct_dev-reward_modeling__1__1742519610", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6361, - "reward-bench/Factuality": 0.7074, - "reward-bench/Precise IF": 0.3812, - "reward-bench/Math": 0.6721, - "reward-bench/Safety": 0.82, - "reward-bench/Focus": 0.6444, - "reward-bench/Ties": 0.5915 - } - }, - { - "id": "allenai/open_instruct_dev-reward_modeling__1__1742519628", - "name": "allenai/open_instruct_dev-reward_modeling__1__1742519628", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5609, - "reward-bench/Factuality": 0.5179, - "reward-bench/Precise IF": 0.3563, - "reward-bench/Math": 0.623, - "reward-bench/Safety": 0.8356, - "reward-bench/Focus": 0.5071, - "reward-bench/Ties": 0.5254 - } - }, - { - "id": "allenai/open_instruct_dev-rm_1e-6_1_100pctflipped__1__1744241455", - "name": "allenai/open_instruct_dev-rm_1e-6_1_100pctflipped__1__1744241455", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.0576, - "reward-bench/Factuality": 0.04, - "reward-bench/Precise IF": 0.1313, - "reward-bench/Math": 0.0546, - "reward-bench/Safety": 0.0489, - "reward-bench/Focus": 0.0808, - "reward-bench/Ties": -0.01 - } - }, - { - "id": "allenai/open_instruct_dev-rm_1e-6_1_10pctflipped__1__1743295511", - "name": "allenai/open_instruct_dev-rm_1e-6_1_10pctflipped__1__1743295511", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5499, - "reward-bench/Factuality": 0.6821, - "reward-bench/Precise IF": 0.3937, - "reward-bench/Math": 0.5956, - "reward-bench/Safety": 0.7356, - "reward-bench/Focus": 0.5212, - "reward-bench/Ties": 0.3711 - } - }, - { - "id": "allenai/open_instruct_dev-rm_1e-6_1_20pctflipped__1__1743295406", - "name": "allenai/open_instruct_dev-rm_1e-6_1_20pctflipped__1__1743295406", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5054, - "reward-bench/Factuality": 0.6358, - "reward-bench/Precise IF": 0.3688, - "reward-bench/Math": 0.6066, - "reward-bench/Safety": 0.6867, - "reward-bench/Focus": 0.4424, - "reward-bench/Ties": 0.2922 - } - }, - { - "id": "allenai/open_instruct_dev-rm_1e-6_1_30pctflipped__1__1743325136", - "name": "allenai/open_instruct_dev-rm_1e-6_1_30pctflipped__1__1743325136", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.478, - "reward-bench/Factuality": 0.6442, - "reward-bench/Precise IF": 0.3563, - "reward-bench/Math": 0.612, - "reward-bench/Safety": 0.6356, - "reward-bench/Focus": 0.2707, - "reward-bench/Ties": 0.3496 - } - }, - { - "id": "allenai/open_instruct_dev-rm_1e-6_1_50pctflipped__1__1744241398", - "name": "allenai/open_instruct_dev-rm_1e-6_1_50pctflipped__1__1744241398", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.219, - "reward-bench/Factuality": 0.2484, - "reward-bench/Precise IF": 0.2812, - "reward-bench/Math": 0.2623, - "reward-bench/Safety": 0.3422, - "reward-bench/Focus": 0.1717, - "reward-bench/Ties": 0.008 - } - }, - { - "id": "allenai/open_instruct_dev-rm_1e-6_1_5pctflipped__1__1743444535", - "name": "allenai/open_instruct_dev-rm_1e-6_1_5pctflipped__1__1743444535", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5625, - "reward-bench/Factuality": 0.6821, - "reward-bench/Precise IF": 0.4062, - "reward-bench/Math": 0.6011, - "reward-bench/Safety": 0.7511, - "reward-bench/Focus": 0.5313, - "reward-bench/Ties": 0.403 - } - }, - { - "id": "allenai/open_instruct_dev-rm_1e-6_1_dpo__1__1743550054", - "name": "allenai/open_instruct_dev-rm_1e-6_1_dpo__1__1743550054", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5759, - "reward-bench/Factuality": 0.7074, - "reward-bench/Precise IF": 0.375, - "reward-bench/Math": 0.623, - "reward-bench/Safety": 0.7578, - "reward-bench/Focus": 0.5333, - "reward-bench/Ties": 0.459 - } - }, - { - "id": "allenai/open_instruct_dev-rm_1e-6_1_dpo_skyworks__1__1744530271", - "name": "allenai/open_instruct_dev-rm_1e-6_1_dpo_skyworks__1__1744530271", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6057, - "reward-bench/Factuality": 0.5053, - "reward-bench/Precise IF": 0.375, - "reward-bench/Math": 0.5902, - "reward-bench/Safety": 0.8422, - "reward-bench/Focus": 0.7798, - "reward-bench/Ties": 0.5419 - } - }, - { - "id": "allenai/open_instruct_dev-rm_1e-6_1_dpo_skyworkstulufull__1__1743550181", - "name": "allenai/open_instruct_dev-rm_1e-6_1_dpo_skyworkstulufull__1__1743550181", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6535, - "reward-bench/Factuality": 0.7137, - "reward-bench/Precise IF": 0.3812, - "reward-bench/Math": 0.6175, - "reward-bench/Safety": 0.8244, - "reward-bench/Focus": 0.7737, - "reward-bench/Ties": 0.6101 - } - }, - { - "id": "allenai/open_instruct_dev-rm_1e-6_1_rl__1__1743551221", - "name": "allenai/open_instruct_dev-rm_1e-6_1_rl__1__1743551221", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5799, - "reward-bench/Factuality": 0.7116, - "reward-bench/Precise IF": 0.3812, - "reward-bench/Math": 0.6284, - "reward-bench/Safety": 0.76, - "reward-bench/Focus": 0.5374, - "reward-bench/Ties": 0.461 - } - }, - { - "id": "allenai/open_instruct_dev-rm_1e-6_1_rl_skyworks__1__1744530262", - "name": "allenai/open_instruct_dev-rm_1e-6_1_rl_skyworks__1__1744530262", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5903, - "reward-bench/Factuality": 0.4863, - "reward-bench/Precise IF": 0.3625, - "reward-bench/Math": 0.5738, - "reward-bench/Safety": 0.8489, - "reward-bench/Focus": 0.7778, - "reward-bench/Ties": 0.4926 - } - }, - { - "id": "allenai/open_instruct_dev-rm_1e-6_1_rl_skyworkstulufull__1__1743551523", - "name": "allenai/open_instruct_dev-rm_1e-6_1_rl_skyworkstulufull__1__1743551523", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6483, - "reward-bench/Factuality": 0.7074, - "reward-bench/Precise IF": 0.3625, - "reward-bench/Math": 0.6175, - "reward-bench/Safety": 0.8222, - "reward-bench/Focus": 0.7758, - "reward-bench/Ties": 0.6044 - } - }, - { - "id": "allenai/open_instruct_dev-rm_1e-6_1_skyworkstulumix__1__1743205750", - "name": "allenai/open_instruct_dev-rm_1e-6_1_skyworkstulumix__1__1743205750", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5157, - "reward-bench/Factuality": 0.6084, - "reward-bench/Precise IF": 0.3688, - "reward-bench/Math": 0.6066, - "reward-bench/Safety": 0.7089, - "reward-bench/Focus": 0.4222, - "reward-bench/Ties": 0.3791 - } - }, - { - "id": "allenai/open_instruct_dev-rm_1e-6_2_10pctflipped__1__1743295427", - "name": "allenai/open_instruct_dev-rm_1e-6_2_10pctflipped__1__1743295427", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6009, - "reward-bench/Factuality": 0.7263, - "reward-bench/Precise IF": 0.375, - "reward-bench/Math": 0.5902, - "reward-bench/Safety": 0.7933, - "reward-bench/Focus": 0.7273, - "reward-bench/Ties": 0.3931 - } - }, - { - "id": "allenai/open_instruct_dev-rm_1e-6_2_20pctflipped__1__1743295446", - "name": "allenai/open_instruct_dev-rm_1e-6_2_20pctflipped__1__1743295446", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5716, - "reward-bench/Factuality": 0.6779, - "reward-bench/Precise IF": 0.3937, - "reward-bench/Math": 0.5464, - "reward-bench/Safety": 0.7533, - "reward-bench/Focus": 0.7051, - "reward-bench/Ties": 0.3534 - } - }, - { - "id": "allenai/open_instruct_dev-rm_1e-6_2_30pctflipped__1__1743325094", - "name": "allenai/open_instruct_dev-rm_1e-6_2_30pctflipped__1__1743325094", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5151, - "reward-bench/Factuality": 0.6484, - "reward-bench/Precise IF": 0.3312, - "reward-bench/Math": 0.5574, - "reward-bench/Safety": 0.7289, - "reward-bench/Focus": 0.4889, - "reward-bench/Ties": 0.3357 - } - }, - { - "id": "allenai/open_instruct_dev-rm_1e-6_2_5pctflipped__1__1743444636", - "name": "allenai/open_instruct_dev-rm_1e-6_2_5pctflipped__1__1743444636", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6119, - "reward-bench/Factuality": 0.72, - "reward-bench/Precise IF": 0.4062, - "reward-bench/Math": 0.6284, - "reward-bench/Safety": 0.8067, - "reward-bench/Focus": 0.6889, - "reward-bench/Ties": 0.421 - } - }, - { - "id": "allenai/open_instruct_dev-rm_1e-6_2_dpo__1__1743549325", - "name": "allenai/open_instruct_dev-rm_1e-6_2_dpo__1__1743549325", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6008, - "reward-bench/Factuality": 0.7179, - "reward-bench/Precise IF": 0.35, - "reward-bench/Math": 0.5956, - "reward-bench/Safety": 0.8, - "reward-bench/Focus": 0.6707, - "reward-bench/Ties": 0.4707 - } - }, - { - "id": "allenai/open_instruct_dev-rm_1e-6_2_rl__1__1743551238", - "name": "allenai/open_instruct_dev-rm_1e-6_2_rl__1__1743551238", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5965, - "reward-bench/Factuality": 0.7095, - "reward-bench/Precise IF": 0.3438, - "reward-bench/Math": 0.612, - "reward-bench/Safety": 0.8044, - "reward-bench/Focus": 0.6566, - "reward-bench/Ties": 0.453 - } - }, - { - "id": "allenai/open_instruct_dev-rm_1e-6_2_skyworkstulumix__1__1743205906", - "name": "allenai/open_instruct_dev-rm_1e-6_2_skyworkstulumix__1__1743205906", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5574, - "reward-bench/Factuality": 0.6526, - "reward-bench/Precise IF": 0.3937, - "reward-bench/Math": 0.6011, - "reward-bench/Safety": 0.7711, - "reward-bench/Focus": 0.5051, - "reward-bench/Ties": 0.4208 - } - }, - { - "id": "allenai/open_instruct_dev-rm_2e-5_1_100pctflipped__1__1744241529", - "name": "allenai/open_instruct_dev-rm_2e-5_1_100pctflipped__1__1744241529", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.0719, - "reward-bench/Factuality": 0.0421, - "reward-bench/Precise IF": 0.2062, - "reward-bench/Math": 0.0601, - "reward-bench/Safety": 0.0378, - "reward-bench/Focus": 0.0949, - "reward-bench/Ties": -0.01 - } - }, - { - "id": "allenai/open_instruct_dev-rm_2e-5_1_10pctflipped__1__1743295305", - "name": "allenai/open_instruct_dev-rm_2e-5_1_10pctflipped__1__1743295305", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.553, - "reward-bench/Factuality": 0.6674, - "reward-bench/Precise IF": 0.3563, - "reward-bench/Math": 0.6284, - "reward-bench/Safety": 0.6733, - "reward-bench/Focus": 0.5697, - "reward-bench/Ties": 0.4227 - } - }, - { - "id": "allenai/open_instruct_dev-rm_2e-5_1_20pctflipped__1__1743324778", - "name": "allenai/open_instruct_dev-rm_2e-5_1_20pctflipped__1__1743324778", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.4955, - "reward-bench/Factuality": 0.6189, - "reward-bench/Precise IF": 0.325, - "reward-bench/Math": 0.5792, - "reward-bench/Safety": 0.6378, - "reward-bench/Focus": 0.5657, - "reward-bench/Ties": 0.2466 - } - }, - { - "id": "allenai/open_instruct_dev-rm_2e-5_1_30pctflipped__1__1743326459", - "name": "allenai/open_instruct_dev-rm_2e-5_1_30pctflipped__1__1743326459", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.4198, - "reward-bench/Factuality": 0.5747, - "reward-bench/Precise IF": 0.3375, - "reward-bench/Math": 0.5464, - "reward-bench/Safety": 0.4933, - "reward-bench/Focus": 0.3596, - "reward-bench/Ties": 0.2073 - } - }, - { - "id": "allenai/open_instruct_dev-rm_2e-5_1_5pctflipped__1__1743443747", - "name": "allenai/open_instruct_dev-rm_2e-5_1_5pctflipped__1__1743443747", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5465, - "reward-bench/Factuality": 0.6821, - "reward-bench/Precise IF": 0.375, - "reward-bench/Math": 0.612, - "reward-bench/Safety": 0.7333, - "reward-bench/Focus": 0.5051, - "reward-bench/Ties": 0.3713 - } - }, - { - "id": "allenai/open_instruct_dev-rm_2e-5_1_skyworkstulumix__1__1743205935", - "name": "allenai/open_instruct_dev-rm_2e-5_1_skyworkstulumix__1__1743205935", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5197, - "reward-bench/Factuality": 0.6126, - "reward-bench/Precise IF": 0.3375, - "reward-bench/Math": 0.5847, - "reward-bench/Safety": 0.7333, - "reward-bench/Focus": 0.4646, - "reward-bench/Ties": 0.3855 - } - }, - { - "id": "allenai/open_instruct_dev-rm_2e-5_2_10pctflipped__1__1743295360", - "name": "allenai/open_instruct_dev-rm_2e-5_2_10pctflipped__1__1743295360", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.4555, - "reward-bench/Factuality": 0.5495, - "reward-bench/Precise IF": 0.3063, - "reward-bench/Math": 0.4262, - "reward-bench/Safety": 0.5711, - "reward-bench/Focus": 0.6101, - "reward-bench/Ties": 0.2696 - } - }, - { - "id": "allenai/open_instruct_dev-rm_2e-5_2_20pctflipped__1__1743295366", - "name": "allenai/open_instruct_dev-rm_2e-5_2_20pctflipped__1__1743295366", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.4422, - "reward-bench/Factuality": 0.5053, - "reward-bench/Precise IF": 0.3375, - "reward-bench/Math": 0.4044, - "reward-bench/Safety": 0.5422, - "reward-bench/Focus": 0.6646, - "reward-bench/Ties": 0.1991 - } - }, - { - "id": "allenai/open_instruct_dev-rm_2e-5_2_30pctflipped__1__1743326352", - "name": "allenai/open_instruct_dev-rm_2e-5_2_30pctflipped__1__1743326352", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.341, - "reward-bench/Factuality": 0.4674, - "reward-bench/Precise IF": 0.2875, - "reward-bench/Math": 0.3333, - "reward-bench/Safety": 0.3711, - "reward-bench/Focus": 0.3919, - "reward-bench/Ties": 0.195 - } - }, - { - "id": "allenai/open_instruct_dev-rm_2e-5_2_5pctflipped__1__1743444634", - "name": "allenai/open_instruct_dev-rm_2e-5_2_5pctflipped__1__1743444634", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.4698, - "reward-bench/Factuality": 0.5853, - "reward-bench/Precise IF": 0.2562, - "reward-bench/Math": 0.5027, - "reward-bench/Safety": 0.6489, - "reward-bench/Focus": 0.5697, - "reward-bench/Ties": 0.2562 - } - }, - { - "id": "allenai/open_instruct_dev-rm_2e-5_2_skyworkstulumix__1__1743205988", - "name": "allenai/open_instruct_dev-rm_2e-5_2_skyworkstulumix__1__1743205988", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.4791, - "reward-bench/Factuality": 0.6421, - "reward-bench/Precise IF": 0.3125, - "reward-bench/Math": 0.541, - "reward-bench/Safety": 0.6911, - "reward-bench/Focus": 0.4182, - "reward-bench/Ties": 0.27 - } - }, - { - "id": "allenai/open_instruct_dev-rm_3e-6_1_100pctflipped__1__1744242103", - "name": "allenai/open_instruct_dev-rm_3e-6_1_100pctflipped__1__1744242103", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.0607, - "reward-bench/Factuality": 0.0274, - "reward-bench/Precise IF": 0.1625, - "reward-bench/Math": 0.0656, - "reward-bench/Safety": 0.04, - "reward-bench/Focus": 0.0788, - "reward-bench/Ties": -0.01 - } - }, - { - "id": "allenai/open_instruct_dev-rm_3e-6_1_10pctflipped__1__1743324835", - "name": "allenai/open_instruct_dev-rm_3e-6_1_10pctflipped__1__1743324835", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6089, - "reward-bench/Factuality": 0.7284, - "reward-bench/Precise IF": 0.4375, - "reward-bench/Math": 0.612, - "reward-bench/Safety": 0.7622, - "reward-bench/Focus": 0.6444, - "reward-bench/Ties": 0.4686 - } - }, - { - "id": "allenai/open_instruct_dev-rm_3e-6_1_1pctflipped__1__1743445221", - "name": "allenai/open_instruct_dev-rm_3e-6_1_1pctflipped__1__1743445221", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6032, - "reward-bench/Factuality": 0.7158, - "reward-bench/Precise IF": 0.4062, - "reward-bench/Math": 0.6284, - "reward-bench/Safety": 0.7778, - "reward-bench/Focus": 0.5859, - "reward-bench/Ties": 0.5051 - } - }, - { - "id": "allenai/open_instruct_dev-rm_3e-6_1_20pctflipped__1__1743324826", - "name": "allenai/open_instruct_dev-rm_3e-6_1_20pctflipped__1__1743324826", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5831, - "reward-bench/Factuality": 0.6947, - "reward-bench/Precise IF": 0.4188, - "reward-bench/Math": 0.623, - "reward-bench/Safety": 0.74, - "reward-bench/Focus": 0.5758, - "reward-bench/Ties": 0.4465 - } - }, - { - "id": "allenai/open_instruct_dev-rm_3e-6_1_30pctflipped__1__1743326363", - "name": "allenai/open_instruct_dev-rm_3e-6_1_30pctflipped__1__1743326363", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5268, - "reward-bench/Factuality": 0.68, - "reward-bench/Precise IF": 0.3688, - "reward-bench/Math": 0.5792, - "reward-bench/Safety": 0.7178, - "reward-bench/Focus": 0.4343, - "reward-bench/Ties": 0.3809 - } - }, - { - "id": "allenai/open_instruct_dev-rm_3e-6_1_5pctflipped__1__1743444498", - "name": "allenai/open_instruct_dev-rm_3e-6_1_5pctflipped__1__1743444498", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6093, - "reward-bench/Factuality": 0.7326, - "reward-bench/Precise IF": 0.4313, - "reward-bench/Math": 0.6339, - "reward-bench/Safety": 0.7578, - "reward-bench/Focus": 0.5859, - "reward-bench/Ties": 0.5143 - } - }, - { - "id": "allenai/open_instruct_dev-rm_3e-6_1__2__1743897475", - "name": "allenai/open_instruct_dev-rm_3e-6_1__2__1743897475", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6122, - "reward-bench/Factuality": 0.7368, - "reward-bench/Precise IF": 0.4, - "reward-bench/Math": 0.623, - "reward-bench/Safety": 0.8044, - "reward-bench/Focus": 0.602, - "reward-bench/Ties": 0.5071 - } - }, - { - "id": "allenai/open_instruct_dev-rm_3e-6_1__3__1744311421", - "name": "allenai/open_instruct_dev-rm_3e-6_1__3__1744311421", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5995, - "reward-bench/Factuality": 0.7179, - "reward-bench/Precise IF": 0.3375, - "reward-bench/Math": 0.6066, - "reward-bench/Safety": 0.8, - "reward-bench/Focus": 0.6323, - "reward-bench/Ties": 0.503 - } - }, - { - "id": "allenai/open_instruct_dev-rm_3e-6_1_dpo__1__1743549903", - "name": "allenai/open_instruct_dev-rm_3e-6_1_dpo__1__1743549903", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6154, - "reward-bench/Factuality": 0.7326, - "reward-bench/Precise IF": 0.4375, - "reward-bench/Math": 0.6339, - "reward-bench/Safety": 0.7778, - "reward-bench/Focus": 0.6061, - "reward-bench/Ties": 0.5043 - } - }, - { - "id": "allenai/open_instruct_dev-rm_3e-6_1_dpo_skyworks__1__1744530368", - "name": "allenai/open_instruct_dev-rm_3e-6_1_dpo_skyworks__1__1744530368", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6604, - "reward-bench/Factuality": 0.6316, - "reward-bench/Precise IF": 0.3937, - "reward-bench/Math": 0.5792, - "reward-bench/Safety": 0.9044, - "reward-bench/Focus": 0.8929, - "reward-bench/Ties": 0.5604 - } - }, - { - "id": "allenai/open_instruct_dev-rm_3e-6_1_dpo_skyworkstulufull__1__1743550182", - "name": "allenai/open_instruct_dev-rm_3e-6_1_dpo_skyworkstulufull__1__1743550182", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6783, - "reward-bench/Factuality": 0.7705, - "reward-bench/Precise IF": 0.4, - "reward-bench/Math": 0.6066, - "reward-bench/Safety": 0.84, - "reward-bench/Focus": 0.8101, - "reward-bench/Ties": 0.6427 - } - }, - { - "id": "allenai/open_instruct_dev-rm_3e-6_1_no_if__2__1744316012", - "name": "allenai/open_instruct_dev-rm_3e-6_1_no_if__2__1744316012", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5911, - "reward-bench/Factuality": 0.7347, - "reward-bench/Precise IF": 0.4, - "reward-bench/Math": 0.6284, - "reward-bench/Safety": 0.74, - "reward-bench/Focus": 0.604, - "reward-bench/Ties": 0.4392 - } - }, - { - "id": "allenai/open_instruct_dev-rm_3e-6_1_no_if__3__1744315765", - "name": "allenai/open_instruct_dev-rm_3e-6_1_no_if__3__1744315765", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5926, - "reward-bench/Factuality": 0.7263, - "reward-bench/Precise IF": 0.3563, - "reward-bench/Math": 0.623, - "reward-bench/Safety": 0.7889, - "reward-bench/Focus": 0.5879, - "reward-bench/Ties": 0.4733 - } - }, - { - "id": "allenai/open_instruct_dev-rm_3e-6_1_rl__1__1743551527", - "name": "allenai/open_instruct_dev-rm_3e-6_1_rl__1__1743551527", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6126, - "reward-bench/Factuality": 0.7411, - "reward-bench/Precise IF": 0.425, - "reward-bench/Math": 0.623, - "reward-bench/Safety": 0.7822, - "reward-bench/Focus": 0.5939, - "reward-bench/Ties": 0.5104 - } - }, - { - "id": "allenai/open_instruct_dev-rm_3e-6_1_rl_skyworks__1__1744530236", - "name": "allenai/open_instruct_dev-rm_3e-6_1_rl_skyworks__1__1744530236", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6525, - "reward-bench/Factuality": 0.6021, - "reward-bench/Precise IF": 0.3875, - "reward-bench/Math": 0.5792, - "reward-bench/Safety": 0.8933, - "reward-bench/Focus": 0.8626, - "reward-bench/Ties": 0.59 - } - }, - { - "id": "allenai/open_instruct_dev-rm_3e-6_1_rl_skyworkstulufull__1__1743551530", - "name": "allenai/open_instruct_dev-rm_3e-6_1_rl_skyworkstulufull__1__1743551530", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6849, - "reward-bench/Factuality": 0.7453, - "reward-bench/Precise IF": 0.3812, - "reward-bench/Math": 0.612, - "reward-bench/Safety": 0.8422, - "reward-bench/Focus": 0.8404, - "reward-bench/Ties": 0.6885 - } - }, - { - "id": "allenai/open_instruct_dev-rm_3e-6_1_skyworkstulu75__1__1743534417", - "name": "allenai/open_instruct_dev-rm_3e-6_1_skyworkstulu75__1__1743534417", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.586, - "reward-bench/Factuality": 0.6632, - "reward-bench/Precise IF": 0.425, - "reward-bench/Math": 0.6557, - "reward-bench/Safety": 0.7778, - "reward-bench/Focus": 0.5172, - "reward-bench/Ties": 0.477 - } - }, - { - "id": "allenai/open_instruct_dev-rm_3e-6_1_skyworkstulufull__1__1743446486", - "name": "allenai/open_instruct_dev-rm_3e-6_1_skyworkstulufull__1__1743446486", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6773, - "reward-bench/Factuality": 0.7432, - "reward-bench/Precise IF": 0.4, - "reward-bench/Math": 0.612, - "reward-bench/Safety": 0.8422, - "reward-bench/Focus": 0.804, - "reward-bench/Ties": 0.6626 - } - }, - { - "id": "allenai/open_instruct_dev-rm_3e-6_1_skyworkstulufull__2__1744314745", - "name": "allenai/open_instruct_dev-rm_3e-6_1_skyworkstulufull__2__1744314745", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6793, - "reward-bench/Factuality": 0.7558, - "reward-bench/Precise IF": 0.4062, - "reward-bench/Math": 0.6284, - "reward-bench/Safety": 0.8311, - "reward-bench/Focus": 0.8061, - "reward-bench/Ties": 0.6485 - } - }, - { - "id": "allenai/open_instruct_dev-rm_3e-6_1_skyworkstulufull__3__1744311661", - "name": "allenai/open_instruct_dev-rm_3e-6_1_skyworkstulufull__3__1744311661", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6611, - "reward-bench/Factuality": 0.72, - "reward-bench/Precise IF": 0.3563, - "reward-bench/Math": 0.6393, - "reward-bench/Safety": 0.8444, - "reward-bench/Focus": 0.7636, - "reward-bench/Ties": 0.6428 - } - }, - { - "id": "allenai/open_instruct_dev-rm_3e-6_1_skyworkstulumix__1__1743204472", - "name": "allenai/open_instruct_dev-rm_3e-6_1_skyworkstulumix__1__1743204472", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5778, - "reward-bench/Factuality": 0.6674, - "reward-bench/Precise IF": 0.3875, - "reward-bench/Math": 0.6011, - "reward-bench/Safety": 0.7933, - "reward-bench/Focus": 0.5172, - "reward-bench/Ties": 0.5003 - } - }, - { - "id": "allenai/open_instruct_dev-rm_3e-6_2_10pctflipped__1__1743295267", - "name": "allenai/open_instruct_dev-rm_3e-6_2_10pctflipped__1__1743295267", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5746, - "reward-bench/Factuality": 0.6505, - "reward-bench/Precise IF": 0.35, - "reward-bench/Math": 0.5082, - "reward-bench/Safety": 0.7844, - "reward-bench/Focus": 0.7414, - "reward-bench/Ties": 0.4128 - } - }, - { - "id": "allenai/open_instruct_dev-rm_3e-6_2_1pctflipped__1__1743445759", - "name": "allenai/open_instruct_dev-rm_3e-6_2_1pctflipped__1__1743445759", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6065, - "reward-bench/Factuality": 0.7116, - "reward-bench/Precise IF": 0.35, - "reward-bench/Math": 0.5792, - "reward-bench/Safety": 0.8178, - "reward-bench/Focus": 0.7152, - "reward-bench/Ties": 0.465 - } - }, - { - "id": "allenai/open_instruct_dev-rm_3e-6_2_20pctflipped__1__1743324905", - "name": "allenai/open_instruct_dev-rm_3e-6_2_20pctflipped__1__1743324905", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5305, - "reward-bench/Factuality": 0.5832, - "reward-bench/Precise IF": 0.3312, - "reward-bench/Math": 0.459, - "reward-bench/Safety": 0.7178, - "reward-bench/Focus": 0.7071, - "reward-bench/Ties": 0.3849 - } - }, - { - "id": "allenai/open_instruct_dev-rm_3e-6_2_30pctflipped__1__1743326363", - "name": "allenai/open_instruct_dev-rm_3e-6_2_30pctflipped__1__1743326363", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.4436, - "reward-bench/Factuality": 0.5411, - "reward-bench/Precise IF": 0.3312, - "reward-bench/Math": 0.3115, - "reward-bench/Safety": 0.6267, - "reward-bench/Focus": 0.5414, - "reward-bench/Ties": 0.31 - } - }, - { - "id": "allenai/open_instruct_dev-rm_3e-6_2_5pctflipped__1__1743444505", - "name": "allenai/open_instruct_dev-rm_3e-6_2_5pctflipped__1__1743444505", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5925, - "reward-bench/Factuality": 0.68, - "reward-bench/Precise IF": 0.3688, - "reward-bench/Math": 0.5519, - "reward-bench/Safety": 0.78, - "reward-bench/Focus": 0.7434, - "reward-bench/Ties": 0.431 - } - }, - { - "id": "allenai/open_instruct_dev-rm_3e-6_2_dpo__1__1743550180", - "name": "allenai/open_instruct_dev-rm_3e-6_2_dpo__1__1743550180", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6198, - "reward-bench/Factuality": 0.7263, - "reward-bench/Precise IF": 0.3312, - "reward-bench/Math": 0.6339, - "reward-bench/Safety": 0.8133, - "reward-bench/Focus": 0.7232, - "reward-bench/Ties": 0.4908 - } - }, - { - "id": "allenai/open_instruct_dev-rm_3e-6_2_dpo_skyworkstulufull__1__1743550187", - "name": "allenai/open_instruct_dev-rm_3e-6_2_dpo_skyworkstulufull__1__1743550187", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6763, - "reward-bench/Factuality": 0.7411, - "reward-bench/Precise IF": 0.375, - "reward-bench/Math": 0.612, - "reward-bench/Safety": 0.8844, - "reward-bench/Focus": 0.8545, - "reward-bench/Ties": 0.5908 - } - }, - { - "id": "allenai/open_instruct_dev-rm_3e-6_2_rl__1__1743551509", - "name": "allenai/open_instruct_dev-rm_3e-6_2_rl__1__1743551509", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6245, - "reward-bench/Factuality": 0.7242, - "reward-bench/Precise IF": 0.35, - "reward-bench/Math": 0.6175, - "reward-bench/Safety": 0.8178, - "reward-bench/Focus": 0.7253, - "reward-bench/Ties": 0.5124 - } - }, - { - "id": "allenai/open_instruct_dev-rm_3e-6_2_rl_skyworkstulufull__1__1743551498", - "name": "allenai/open_instruct_dev-rm_3e-6_2_rl_skyworkstulufull__1__1743551498", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6673, - "reward-bench/Factuality": 0.7326, - "reward-bench/Precise IF": 0.3438, - "reward-bench/Math": 0.6175, - "reward-bench/Safety": 0.8622, - "reward-bench/Focus": 0.8566, - "reward-bench/Ties": 0.5911 - } - }, - { - "id": "allenai/open_instruct_dev-rm_3e-6_2_skyworkstulu75__1__1743548926", - "name": "allenai/open_instruct_dev-rm_3e-6_2_skyworkstulu75__1__1743548926", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5863, - "reward-bench/Factuality": 0.6674, - "reward-bench/Precise IF": 0.3937, - "reward-bench/Math": 0.6284, - "reward-bench/Safety": 0.8, - "reward-bench/Focus": 0.5515, - "reward-bench/Ties": 0.4768 - } - }, - { - "id": "allenai/open_instruct_dev-rm_3e-6_2_skyworkstulumix__1__1743205661", - "name": "allenai/open_instruct_dev-rm_3e-6_2_skyworkstulumix__1__1743205661", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.589, - "reward-bench/Factuality": 0.6842, - "reward-bench/Precise IF": 0.3688, - "reward-bench/Math": 0.6393, - "reward-bench/Safety": 0.7867, - "reward-bench/Focus": 0.6081, - "reward-bench/Ties": 0.447 - } - }, - { - "id": "allenai/open_instruct_dev-rm_llama70b_skyworkstulufull__4__1747266598", - "name": "allenai/open_instruct_dev-rm_llama70b_skyworkstulufull__4__1747266598", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7306, - "reward-bench/Factuality": 0.7474, - "reward-bench/Precise IF": 0.375, - "reward-bench/Math": 0.694, - "reward-bench/Safety": 0.8622, - "reward-bench/Focus": 0.8061, - "reward-bench/Ties": 0.8992 - } - }, - { - "id": "allenai/open_instruct_dev-rm_llama70b_skyworkstulufull__8__1745387923", - "name": "allenai/open_instruct_dev-rm_llama70b_skyworkstulufull__8__1745387923", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7573, - "reward-bench/Factuality": 0.8168, - "reward-bench/Precise IF": 0.4125, - "reward-bench/Math": 0.7049, - "reward-bench/Safety": 0.8733, - "reward-bench/Focus": 0.8545, - "reward-bench/Ties": 0.8814 - } - }, - { - "id": "allenai/open_instruct_dev-rm_llama_1e-6_1__1__1743896628", - "name": "allenai/open_instruct_dev-rm_llama_1e-6_1__1__1743896628", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6637, - "reward-bench/Factuality": 0.6947, - "reward-bench/Precise IF": 0.4062, - "reward-bench/Math": 0.6284, - "reward-bench/Safety": 0.8422, - "reward-bench/Focus": 0.7273, - "reward-bench/Ties": 0.6834 - } - }, - { - "id": "allenai/open_instruct_dev-rm_llama_1e-6_1_skyworks__1__1744062999", - "name": "allenai/open_instruct_dev-rm_llama_1e-6_1_skyworks__1__1744062999", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6665, - "reward-bench/Factuality": 0.5979, - "reward-bench/Precise IF": 0.3688, - "reward-bench/Math": 0.6339, - "reward-bench/Safety": 0.8956, - "reward-bench/Focus": 0.8606, - "reward-bench/Ties": 0.6422 - } - }, - { - "id": "allenai/open_instruct_dev-rm_llama_1e-6_1_skyworkstulufull__1__1743712777", - "name": "allenai/open_instruct_dev-rm_llama_1e-6_1_skyworkstulufull__1__1743712777", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7038, - "reward-bench/Factuality": 0.6947, - "reward-bench/Precise IF": 0.3937, - "reward-bench/Math": 0.6557, - "reward-bench/Safety": 0.8867, - "reward-bench/Focus": 0.8586, - "reward-bench/Ties": 0.7331 - } - }, - { - "id": "allenai/open_instruct_dev-rm_llama_1e-6_2__1__1743896638", - "name": "allenai/open_instruct_dev-rm_llama_1e-6_2__1__1743896638", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6754, - "reward-bench/Factuality": 0.6716, - "reward-bench/Precise IF": 0.4, - "reward-bench/Math": 0.6339, - "reward-bench/Safety": 0.8756, - "reward-bench/Focus": 0.7737, - "reward-bench/Ties": 0.6976 - } - }, - { - "id": "allenai/open_instruct_dev-rm_llama_1e-6_2_skyworkstulufull__1__1743800938", - "name": "allenai/open_instruct_dev-rm_llama_1e-6_2_skyworkstulufull__1__1743800938", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7241, - "reward-bench/Factuality": 0.7305, - "reward-bench/Precise IF": 0.4, - "reward-bench/Math": 0.6667, - "reward-bench/Safety": 0.9422, - "reward-bench/Focus": 0.9414, - "reward-bench/Ties": 0.6635 - } - }, - { - "id": "allenai/open_instruct_dev-rm_llama_2e-5_1_skyworkstulufull__1__1743712885", - "name": "allenai/open_instruct_dev-rm_llama_2e-5_1_skyworkstulufull__1__1743712885", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6716, - "reward-bench/Factuality": 0.6632, - "reward-bench/Precise IF": 0.3688, - "reward-bench/Math": 0.6284, - "reward-bench/Safety": 0.82, - "reward-bench/Focus": 0.8303, - "reward-bench/Ties": 0.719 - } - }, - { - "id": "allenai/open_instruct_dev-rm_llama_2e-5_2_skyworkstulufull__1__1743800773", - "name": "allenai/open_instruct_dev-rm_llama_2e-5_2_skyworkstulufull__1__1743800773", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6207, - "reward-bench/Factuality": 0.6358, - "reward-bench/Precise IF": 0.375, - "reward-bench/Math": 0.5902, - "reward-bench/Safety": 0.8267, - "reward-bench/Focus": 0.802, - "reward-bench/Ties": 0.4948 - } - }, - { - "id": "allenai/open_instruct_dev-rm_llama_2e-6_1_skyworkstulufull__1__1743893867", - "name": "allenai/open_instruct_dev-rm_llama_2e-6_1_skyworkstulufull__1__1743893867", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.719, - "reward-bench/Factuality": 0.7263, - "reward-bench/Precise IF": 0.3875, - "reward-bench/Math": 0.6393, - "reward-bench/Safety": 0.8956, - "reward-bench/Focus": 0.9273, - "reward-bench/Ties": 0.738 - } - }, - { - "id": "allenai/open_instruct_dev-rm_llama_3e-6_1__1__1743929424", - "name": "allenai/open_instruct_dev-rm_llama_3e-6_1__1__1743929424", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6572, - "reward-bench/Factuality": 0.7305, - "reward-bench/Precise IF": 0.3688, - "reward-bench/Math": 0.6284, - "reward-bench/Safety": 0.8289, - "reward-bench/Focus": 0.703, - "reward-bench/Ties": 0.6837 - } - }, - { - "id": "allenai/open_instruct_dev-rm_llama_3e-6_1__2__1744311395", - "name": "allenai/open_instruct_dev-rm_llama_3e-6_1__2__1744311395", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6938, - "reward-bench/Factuality": 0.7537, - "reward-bench/Precise IF": 0.45, - "reward-bench/Math": 0.6393, - "reward-bench/Safety": 0.8667, - "reward-bench/Focus": 0.7616, - "reward-bench/Ties": 0.6913 - } - }, - { - "id": "allenai/open_instruct_dev-rm_llama_3e-6_1__3__1744311491", - "name": "allenai/open_instruct_dev-rm_llama_3e-6_1__3__1744311491", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6754, - "reward-bench/Factuality": 0.7242, - "reward-bench/Precise IF": 0.4062, - "reward-bench/Math": 0.6284, - "reward-bench/Safety": 0.8422, - "reward-bench/Focus": 0.7535, - "reward-bench/Ties": 0.6976 - } - }, - { - "id": "allenai/open_instruct_dev-rm_llama_3e-6_1_skyworks__1__1744062787", - "name": "allenai/open_instruct_dev-rm_llama_3e-6_1_skyworks__1__1744062787", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7045, - "reward-bench/Factuality": 0.6253, - "reward-bench/Precise IF": 0.3812, - "reward-bench/Math": 0.6667, - "reward-bench/Safety": 0.92, - "reward-bench/Focus": 0.9232, - "reward-bench/Ties": 0.7109 - } - }, - { - "id": "allenai/open_instruct_dev-rm_llama_3e-6_1_skyworkstulufull__2__1744311461", - "name": "allenai/open_instruct_dev-rm_llama_3e-6_1_skyworkstulufull__2__1744311461", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7189, - "reward-bench/Factuality": 0.7305, - "reward-bench/Precise IF": 0.3937, - "reward-bench/Math": 0.6066, - "reward-bench/Safety": 0.8978, - "reward-bench/Focus": 0.9374, - "reward-bench/Ties": 0.7475 - } - }, - { - "id": "allenai/open_instruct_dev-rm_llama_3e-6_1_skyworkstulufull__3__1744311780", - "name": "allenai/open_instruct_dev-rm_llama_3e-6_1_skyworkstulufull__3__1744311780", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7172, - "reward-bench/Factuality": 0.7242, - "reward-bench/Precise IF": 0.4313, - "reward-bench/Math": 0.6175, - "reward-bench/Safety": 0.8778, - "reward-bench/Focus": 0.897, - "reward-bench/Ties": 0.7555 - } - }, - { - "id": "allenai/open_instruct_dev-rm_llama_3e-6_2__1__1743896489", - "name": "allenai/open_instruct_dev-rm_llama_3e-6_2__1__1743896489", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6813, - "reward-bench/Factuality": 0.7137, - "reward-bench/Precise IF": 0.4437, - "reward-bench/Math": 0.6284, - "reward-bench/Safety": 0.8644, - "reward-bench/Focus": 0.7596, - "reward-bench/Ties": 0.6781 - } - }, - { - "id": "allenai/open_instruct_dev-rm_llama_3e-6_2_skyworkstulufull__1__1743800713", - "name": "allenai/open_instruct_dev-rm_llama_3e-6_2_skyworkstulufull__1__1743800713", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7209, - "reward-bench/Factuality": 0.7116, - "reward-bench/Precise IF": 0.3875, - "reward-bench/Math": 0.6612, - "reward-bench/Safety": 0.9067, - "reward-bench/Focus": 0.9172, - "reward-bench/Ties": 0.7414 - } - }, - { - "id": "allenai/open_instruct_dev-rm_llama_4e-6_1_skyworkstulufull__1__1743893911", - "name": "allenai/open_instruct_dev-rm_llama_4e-6_1_skyworkstulufull__1__1743893911", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7266, - "reward-bench/Factuality": 0.7347, - "reward-bench/Precise IF": 0.4313, - "reward-bench/Math": 0.6339, - "reward-bench/Safety": 0.8933, - "reward-bench/Focus": 0.897, - "reward-bench/Ties": 0.7697 - } - }, - { - "id": "allenai/open_instruct_dev-rm_llamabase_1e-6_1_skyworkstulufull__1__1745386412", - "name": "allenai/open_instruct_dev-rm_llamabase_1e-6_1_skyworkstulufull__1__1745386412", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5342, - "reward-bench/Factuality": 0.6042, - "reward-bench/Precise IF": 0.275, - "reward-bench/Math": 0.6284, - "reward-bench/Safety": 0.7222, - "reward-bench/Focus": 0.5818, - "reward-bench/Ties": 0.3935 - } - }, - { - "id": "allenai/open_instruct_dev-rm_llamabase_1e-6_2_skyworkstulufull__1__1745441922", - "name": "allenai/open_instruct_dev-rm_llamabase_1e-6_2_skyworkstulufull__1__1745441922", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6111, - "reward-bench/Factuality": 0.6884, - "reward-bench/Precise IF": 0.3063, - "reward-bench/Math": 0.623, - "reward-bench/Safety": 0.8289, - "reward-bench/Focus": 0.7576, - "reward-bench/Ties": 0.4628 - } - }, - { - "id": "allenai/open_instruct_dev-rm_llamabase_2e-5_1_skyworkstulufull__1__1745386495", - "name": "allenai/open_instruct_dev-rm_llamabase_2e-5_1_skyworkstulufull__1__1745386495", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5825, - "reward-bench/Factuality": 0.6379, - "reward-bench/Precise IF": 0.325, - "reward-bench/Math": 0.5355, - "reward-bench/Safety": 0.8222, - "reward-bench/Focus": 0.7051, - "reward-bench/Ties": 0.4691 - } - }, - { - "id": "allenai/open_instruct_dev-rm_llamabase_2e-5_2_skyworkstulufull__1__1745386507", - "name": "allenai/open_instruct_dev-rm_llamabase_2e-5_2_skyworkstulufull__1__1745386507", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5598, - "reward-bench/Factuality": 0.5495, - "reward-bench/Precise IF": 0.3563, - "reward-bench/Math": 0.5902, - "reward-bench/Safety": 0.76, - "reward-bench/Focus": 0.7273, - "reward-bench/Ties": 0.3754 - } - }, - { - "id": "allenai/open_instruct_dev-rm_llamabase_3e-6_1_skyworkstulufull__1__1745386507", - "name": "allenai/open_instruct_dev-rm_llamabase_3e-6_1_skyworkstulufull__1__1745386507", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6101, - "reward-bench/Factuality": 0.6632, - "reward-bench/Precise IF": 0.35, - "reward-bench/Math": 0.6175, - "reward-bench/Safety": 0.7778, - "reward-bench/Focus": 0.7111, - "reward-bench/Ties": 0.5408 - } - }, - { - "id": "allenai/open_instruct_dev-rm_qwen32b_1e-6_skyworkstulufull__8__1748235917", - "name": "allenai/open_instruct_dev-rm_qwen32b_1e-6_skyworkstulufull__8__1748235917", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7185, - "reward-bench/Factuality": 0.7305, - "reward-bench/Precise IF": 0.4125, - "reward-bench/Math": 0.7158, - "reward-bench/Safety": 0.7933, - "reward-bench/Focus": 0.8545, - "reward-bench/Ties": 0.804 - } - }, - { - "id": "allenai/open_instruct_dev-rm_qwen32b_3e-6_skyworkstulufull__8__1748288961", - "name": "allenai/open_instruct_dev-rm_qwen32b_3e-6_skyworkstulufull__8__1748288961", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7325, - "reward-bench/Factuality": 0.7474, - "reward-bench/Precise IF": 0.4437, - "reward-bench/Math": 0.7158, - "reward-bench/Safety": 0.7978, - "reward-bench/Focus": 0.8141, - "reward-bench/Ties": 0.8763 - } - }, - { - "id": "allenai/open_instruct_dev-rm_qwen_1e-6_1_skyworks__1__1744062830", - "name": "allenai/open_instruct_dev-rm_qwen_1e-6_1_skyworks__1__1744062830", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6022, - "reward-bench/Factuality": 0.5284, - "reward-bench/Precise IF": 0.325, - "reward-bench/Math": 0.694, - "reward-bench/Safety": 0.7556, - "reward-bench/Focus": 0.7616, - "reward-bench/Ties": 0.5486 - } - }, - { - "id": "allenai/open_instruct_dev-rm_qwen_1e-6_1_skyworks__2__1744576024", - "name": "allenai/open_instruct_dev-rm_qwen_1e-6_1_skyworks__2__1744576024", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5948, - "reward-bench/Factuality": 0.5579, - "reward-bench/Precise IF": 0.2875, - "reward-bench/Math": 0.6776, - "reward-bench/Safety": 0.72, - "reward-bench/Focus": 0.7394, - "reward-bench/Ties": 0.5863 - } - }, - { - "id": "allenai/open_instruct_dev-rm_qwen_1e-6_1_skyworkstulufull__1__1743712914", - "name": "allenai/open_instruct_dev-rm_qwen_1e-6_1_skyworkstulufull__1__1743712914", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6492, - "reward-bench/Factuality": 0.6084, - "reward-bench/Precise IF": 0.35, - "reward-bench/Math": 0.6776, - "reward-bench/Safety": 0.76, - "reward-bench/Focus": 0.8, - "reward-bench/Ties": 0.699 - } - }, - { - "id": "allenai/open_instruct_dev-rm_qwen_2e-5_1_skyworkstulufull__1__1743713091", - "name": "allenai/open_instruct_dev-rm_qwen_2e-5_1_skyworkstulufull__1__1743713091", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6764, - "reward-bench/Factuality": 0.7074, - "reward-bench/Precise IF": 0.3, - "reward-bench/Math": 0.6885, - "reward-bench/Safety": 0.8622, - "reward-bench/Focus": 0.802, - "reward-bench/Ties": 0.6984 - } - }, - { - "id": "allenai/open_instruct_dev-rm_qwen_3e-6_1_skyworks__1__1744062829", - "name": "allenai/open_instruct_dev-rm_qwen_3e-6_1_skyworks__1__1744062829", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6408, - "reward-bench/Factuality": 0.6337, - "reward-bench/Precise IF": 0.3063, - "reward-bench/Math": 0.6831, - "reward-bench/Safety": 0.8467, - "reward-bench/Focus": 0.8222, - "reward-bench/Ties": 0.5529 - } - }, - { - "id": "allenai/open_instruct_dev-rm_qwen_3e-6_1_skyworks__2__1744576050", - "name": "allenai/open_instruct_dev-rm_qwen_3e-6_1_skyworks__2__1744576050", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6452, - "reward-bench/Factuality": 0.6063, - "reward-bench/Precise IF": 0.3187, - "reward-bench/Math": 0.7158, - "reward-bench/Safety": 0.8356, - "reward-bench/Focus": 0.8343, - "reward-bench/Ties": 0.5603 - } - }, - { - "id": "allenai/open_instruct_dev-rm_qwen_3e-6_1_skyworkstulufull__1__1743712916", - "name": "allenai/open_instruct_dev-rm_qwen_3e-6_1_skyworkstulufull__1__1743712916", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7013, - "reward-bench/Factuality": 0.7263, - "reward-bench/Precise IF": 0.3438, - "reward-bench/Math": 0.6995, - "reward-bench/Safety": 0.8222, - "reward-bench/Focus": 0.8444, - "reward-bench/Ties": 0.7714 - } - }, - { - "id": "allenai/open_instruct_dev-rm_qwen_3e-6_2__1__1743023576", - "name": "allenai/open_instruct_dev-rm_qwen_3e-6_2__1__1743023576", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6369, - "reward-bench/Factuality": 0.6905, - "reward-bench/Precise IF": 0.3187, - "reward-bench/Math": 0.6448, - "reward-bench/Safety": 0.7844, - "reward-bench/Focus": 0.7596, - "reward-bench/Ties": 0.6236 - } - }, - { - "id": "allenai/open_instruct_dev-rm_qwen_3e-6_3__1__1743023619", - "name": "allenai/open_instruct_dev-rm_qwen_3e-6_3__1__1743023619", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6221, - "reward-bench/Factuality": 0.6674, - "reward-bench/Precise IF": 0.325, - "reward-bench/Math": 0.612, - "reward-bench/Safety": 0.7978, - "reward-bench/Focus": 0.7455, - "reward-bench/Ties": 0.5852 - } - }, - { - "id": "allenai/open_instruct_dev-rm_qwenbase_1e-6_1_skyworkstulufull__1__1745388583", - "name": "allenai/open_instruct_dev-rm_qwenbase_1e-6_1_skyworkstulufull__1__1745388583", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5735, - "reward-bench/Factuality": 0.5895, - "reward-bench/Precise IF": 0.2625, - "reward-bench/Math": 0.6448, - "reward-bench/Safety": 0.6889, - "reward-bench/Focus": 0.6727, - "reward-bench/Ties": 0.5823 - } - }, - { - "id": "allenai/open_instruct_dev-rm_qwenbase_1e-6_2_skyworkstulufull__1__1745388604", - "name": "allenai/open_instruct_dev-rm_qwenbase_1e-6_2_skyworkstulufull__1__1745388604", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6336, - "reward-bench/Factuality": 0.6337, - "reward-bench/Precise IF": 0.3063, - "reward-bench/Math": 0.6885, - "reward-bench/Safety": 0.7244, - "reward-bench/Focus": 0.802, - "reward-bench/Ties": 0.6465 - } - }, - { - "id": "allenai/open_instruct_dev-rm_qwenbase_2e-5_1_skyworkstulufull__1__1745388738", - "name": "allenai/open_instruct_dev-rm_qwenbase_2e-5_1_skyworkstulufull__1__1745388738", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6824, - "reward-bench/Factuality": 0.6989, - "reward-bench/Precise IF": 0.3625, - "reward-bench/Math": 0.6831, - "reward-bench/Safety": 0.8311, - "reward-bench/Focus": 0.8081, - "reward-bench/Ties": 0.7107 - } - }, - { - "id": "allenai/open_instruct_dev-rm_qwenbase_2e-5_2_skyworkstulufull__1__1745388191", - "name": "allenai/open_instruct_dev-rm_qwenbase_2e-5_2_skyworkstulufull__1__1745388191", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6392, - "reward-bench/Factuality": 0.6589, - "reward-bench/Precise IF": 0.3312, - "reward-bench/Math": 0.6995, - "reward-bench/Safety": 0.7933, - "reward-bench/Focus": 0.7717, - "reward-bench/Ties": 0.5804 - } - }, - { - "id": "allenai/open_instruct_dev-rm_qwenbase_3e-6_1_skyworkstulufull__1__1745388737", - "name": "allenai/open_instruct_dev-rm_qwenbase_3e-6_1_skyworkstulufull__1__1745388737", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.664, - "reward-bench/Factuality": 0.6821, - "reward-bench/Precise IF": 0.3312, - "reward-bench/Math": 0.6448, - "reward-bench/Safety": 0.8133, - "reward-bench/Focus": 0.8061, - "reward-bench/Ties": 0.7066 - } - }, - { - "id": "allenai/open_instruct_dev-rm_qwenbase_3e-6_2_skyworkstulufull__1__1745388138", - "name": "allenai/open_instruct_dev-rm_qwenbase_3e-6_2_skyworkstulufull__1__1745388138", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6678, - "reward-bench/Factuality": 0.6505, - "reward-bench/Precise IF": 0.3312, - "reward-bench/Math": 0.6831, - "reward-bench/Safety": 0.7978, - "reward-bench/Focus": 0.8808, - "reward-bench/Ties": 0.6632 - } - }, - { - "id": "allenai/open_instruct_dev-rm_tulu3_70b_1__8__1742924455", - "name": "allenai/open_instruct_dev-rm_tulu3_70b_1__8__1742924455", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6618, - "reward-bench/Factuality": 0.7958, - "reward-bench/Precise IF": 0.325, - "reward-bench/Math": 0.6557, - "reward-bench/Safety": 0.8311, - "reward-bench/Focus": 0.6323, - "reward-bench/Ties": 0.7311 - } - }, - { - "id": "allenai/open_instruct_dev-rm_tulu3_70b_2__8__1742982964", - "name": "allenai/open_instruct_dev-rm_tulu3_70b_2__8__1742982964", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6605, - "reward-bench/Factuality": 0.7789, - "reward-bench/Precise IF": 0.3688, - "reward-bench/Math": 0.6448, - "reward-bench/Safety": 0.8844, - "reward-bench/Focus": 0.6667, - "reward-bench/Ties": 0.6195 - } - }, - { - "id": "allenai/tulu-2-dpo-13b", - "name": "allenai/tulu-2-dpo-13b", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7368, - "reward-bench/Chat": 0.9581, - "reward-bench/Chat Hard": 0.5833, - "reward-bench/Safety": 0.7946, - "reward-bench/Reasoning": 0.7323, - "reward-bench/Prior Sets (0.5 weight)": 0.4947 - } - }, - { - "id": "allenai/tulu-2-dpo-70b", - "name": "allenai/tulu-2-dpo-70b", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7621, - "reward-bench/Chat": 0.9749, - "reward-bench/Chat Hard": 0.6053, - "reward-bench/Safety": 0.8446, - "reward-bench/Reasoning": 0.7407, - "reward-bench/Prior Sets (0.5 weight)": 0.5278 - } - }, - { - "id": "allenai/tulu-2-dpo-7b", - "name": "allenai/tulu-2-dpo-7b", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7212, - "reward-bench/Chat": 0.9749, - "reward-bench/Chat Hard": 0.5614, - "reward-bench/Safety": 0.7527, - "reward-bench/Reasoning": 0.7176, - "reward-bench/Prior Sets (0.5 weight)": 0.4774 - } - }, - { - "id": "allenai/tulu-v2.5-13b-preference-mix-rm", - "name": "allenai/tulu-v2.5-13b-preference-mix-rm", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.8027, - "reward-bench/Chat": 0.9358, - "reward-bench/Chat Hard": 0.682, - "reward-bench/Safety": 0.773, - "reward-bench/Reasoning": 0.885, - "reward-bench/Prior Sets (0.5 weight)": 0.6724 - } - }, - { - "id": "allenai/tulu-v2.5-13b-uf-rm", - "name": "allenai/tulu-v2.5-13b-uf-rm", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.4806, - "reward-bench/Chat": 0.3939, - "reward-bench/Chat Hard": 0.4232, - "reward-bench/Safety": 0.5554, - "reward-bench/Reasoning": 0.4737, - "reward-bench/Prior Sets (0.5 weight)": 0.6326 - } - }, - { - "id": "allenai/tulu-v2.5-70b-preference-mix-rm", - "name": "allenai/tulu-v2.5-70b-preference-mix-rm", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6516, - "reward-bench/Chat": 0.7737, - "reward-bench/Chat Hard": 0.5921, - "reward-bench/Safety": 0.8486, - "reward-bench/Reasoning": 0.4138, - "reward-bench/Prior Sets (0.5 weight)": 0.6079 - } - }, - { - "id": "allenai/tulu-v2.5-70b-uf-rm", - "name": "allenai/tulu-v2.5-70b-uf-rm", - "developer": "allenai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7398, - "reward-bench/Chat": 0.8659, - "reward-bench/Chat Hard": 0.7171, - "reward-bench/Safety": 0.7014, - "reward-bench/Reasoning": 0.757, - "reward-bench/Prior Sets (0.5 weight)": 0.5757 - } - }, - { - "id": "allknowingroger/Chocolatine-24B", - "name": "Chocolatine-24B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1958, - "hfopenllm_v2/BBH": 0.6191, - "hfopenllm_v2/MATH Level 5": 0.0008, - "hfopenllm_v2/GPQA": 0.3255, - "hfopenllm_v2/MUSR": 0.4323, - "hfopenllm_v2/MMLU-PRO": 0.4566 - } - }, - { - "id": "allknowingroger/Gemma2Slerp1-2.6B", - "name": "Gemma2Slerp1-2.6B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5354, - "hfopenllm_v2/BBH": 0.4343, - "hfopenllm_v2/MATH Level 5": 0.1065, - "hfopenllm_v2/GPQA": 0.2836, - "hfopenllm_v2/MUSR": 0.4562, - "hfopenllm_v2/MMLU-PRO": 0.2689 - } - }, - { - "id": "allknowingroger/Gemma2Slerp1-27B", - "name": "Gemma2Slerp1-27B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7186, - "hfopenllm_v2/BBH": 0.6399, - "hfopenllm_v2/MATH Level 5": 0.2583, - "hfopenllm_v2/GPQA": 0.3641, - "hfopenllm_v2/MUSR": 0.4767, - "hfopenllm_v2/MMLU-PRO": 0.4456 - } - }, - { - "id": "allknowingroger/Gemma2Slerp2-2.6B", - "name": "Gemma2Slerp2-2.6B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5747, - "hfopenllm_v2/BBH": 0.4308, - "hfopenllm_v2/MATH Level 5": 0.0906, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.4468, - "hfopenllm_v2/MMLU-PRO": 0.2696 - } - }, - { - "id": "allknowingroger/Gemma2Slerp2-27B", - "name": "Gemma2Slerp2-27B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7546, - "hfopenllm_v2/BBH": 0.6557, - "hfopenllm_v2/MATH Level 5": 0.2787, - "hfopenllm_v2/GPQA": 0.37, - "hfopenllm_v2/MUSR": 0.4621, - "hfopenllm_v2/MMLU-PRO": 0.4623 - } - }, - { - "id": "allknowingroger/Gemma2Slerp3-27B", - "name": "Gemma2Slerp3-27B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7426, - "hfopenllm_v2/BBH": 0.65, - "hfopenllm_v2/MATH Level 5": 0.2742, - "hfopenllm_v2/GPQA": 0.3549, - "hfopenllm_v2/MUSR": 0.474, - "hfopenllm_v2/MMLU-PRO": 0.4641 - } - }, - { - "id": "allknowingroger/Gemma2Slerp4-27B", - "name": "Gemma2Slerp4-27B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7497, - "hfopenllm_v2/BBH": 0.653, - "hfopenllm_v2/MATH Level 5": 0.2719, - "hfopenllm_v2/GPQA": 0.3666, - "hfopenllm_v2/MUSR": 0.4502, - "hfopenllm_v2/MMLU-PRO": 0.4649 - } - }, - { - "id": "allknowingroger/GemmaSlerp-9B", - "name": "GemmaSlerp-9B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7043, - "hfopenllm_v2/BBH": 0.5921, - "hfopenllm_v2/MATH Level 5": 0.216, - "hfopenllm_v2/GPQA": 0.344, - "hfopenllm_v2/MUSR": 0.4673, - "hfopenllm_v2/MMLU-PRO": 0.4161 - } - }, - { - "id": "allknowingroger/GemmaSlerp2-9B", - "name": "GemmaSlerp2-9B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7281, - "hfopenllm_v2/BBH": 0.5983, - "hfopenllm_v2/MATH Level 5": 0.2107, - "hfopenllm_v2/GPQA": 0.3523, - "hfopenllm_v2/MUSR": 0.4767, - "hfopenllm_v2/MMLU-PRO": 0.4239 - } - }, - { - "id": "allknowingroger/GemmaSlerp4-10B", - "name": "GemmaSlerp4-10B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7326, - "hfopenllm_v2/BBH": 0.6028, - "hfopenllm_v2/MATH Level 5": 0.2243, - "hfopenllm_v2/GPQA": 0.3532, - "hfopenllm_v2/MUSR": 0.454, - "hfopenllm_v2/MMLU-PRO": 0.425 - } - }, - { - "id": "allknowingroger/GemmaSlerp5-10B", - "name": "GemmaSlerp5-10B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7353, - "hfopenllm_v2/BBH": 0.6054, - "hfopenllm_v2/MATH Level 5": 0.2183, - "hfopenllm_v2/GPQA": 0.3523, - "hfopenllm_v2/MUSR": 0.4608, - "hfopenllm_v2/MMLU-PRO": 0.4328 - } - }, - { - "id": "allknowingroger/GemmaStock1-27B", - "name": "GemmaStock1-27B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7509, - "hfopenllm_v2/BBH": 0.6566, - "hfopenllm_v2/MATH Level 5": 0.2636, - "hfopenllm_v2/GPQA": 0.3641, - "hfopenllm_v2/MUSR": 0.4527, - "hfopenllm_v2/MMLU-PRO": 0.473 - } - }, - { - "id": "allknowingroger/HomerSlerp1-7B", - "name": "HomerSlerp1-7B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4621, - "hfopenllm_v2/BBH": 0.5518, - "hfopenllm_v2/MATH Level 5": 0.2719, - "hfopenllm_v2/GPQA": 0.318, - "hfopenllm_v2/MUSR": 0.4359, - "hfopenllm_v2/MMLU-PRO": 0.4504 - } - }, - { - "id": "allknowingroger/HomerSlerp2-7B", - "name": "HomerSlerp2-7B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4487, - "hfopenllm_v2/BBH": 0.5649, - "hfopenllm_v2/MATH Level 5": 0.2968, - "hfopenllm_v2/GPQA": 0.3196, - "hfopenllm_v2/MUSR": 0.4356, - "hfopenllm_v2/MMLU-PRO": 0.4515 - } - }, - { - "id": "allknowingroger/HomerSlerp3-7B", - "name": "HomerSlerp3-7B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4363, - "hfopenllm_v2/BBH": 0.5598, - "hfopenllm_v2/MATH Level 5": 0.3021, - "hfopenllm_v2/GPQA": 0.3171, - "hfopenllm_v2/MUSR": 0.4462, - "hfopenllm_v2/MMLU-PRO": 0.4535 - } - }, - { - "id": "allknowingroger/HomerSlerp4-7B", - "name": "HomerSlerp4-7B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4374, - "hfopenllm_v2/BBH": 0.5571, - "hfopenllm_v2/MATH Level 5": 0.327, - "hfopenllm_v2/GPQA": 0.3196, - "hfopenllm_v2/MUSR": 0.4408, - "hfopenllm_v2/MMLU-PRO": 0.4472 - } - }, - { - "id": "allknowingroger/limyClown-7B-slerp", - "name": "limyClown-7B-slerp", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4017, - "hfopenllm_v2/BBH": 0.5148, - "hfopenllm_v2/MATH Level 5": 0.0687, - "hfopenllm_v2/GPQA": 0.281, - "hfopenllm_v2/MUSR": 0.4293, - "hfopenllm_v2/MMLU-PRO": 0.3038 - } - }, - { - "id": "allknowingroger/LimyQstar-7B-slerp", - "name": "LimyQstar-7B-slerp", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3491, - "hfopenllm_v2/BBH": 0.5024, - "hfopenllm_v2/MATH Level 5": 0.0687, - "hfopenllm_v2/GPQA": 0.2987, - "hfopenllm_v2/MUSR": 0.4146, - "hfopenllm_v2/MMLU-PRO": 0.3103 - } - }, - { - "id": "allknowingroger/llama3-Jallabi-40B-s", - "name": "llama3-Jallabi-40B-s", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1921, - "hfopenllm_v2/BBH": 0.3252, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2374, - "hfopenllm_v2/MUSR": 0.375, - "hfopenllm_v2/MMLU-PRO": 0.1088 - } - }, - { - "id": "allknowingroger/Llama3.1-60B", - "name": "Llama3.1-60B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1815, - "hfopenllm_v2/BBH": 0.3242, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.3596, - "hfopenllm_v2/MMLU-PRO": 0.331 - } - }, - { - "id": "allknowingroger/llama3AnFeng-40B", - "name": "llama3AnFeng-40B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1742, - "hfopenllm_v2/BBH": 0.3794, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.3062, - "hfopenllm_v2/MUSR": 0.394, - "hfopenllm_v2/MMLU-PRO": 0.198 - } - }, - { - "id": "allknowingroger/Marco-01-slerp1-7B", - "name": "Marco-01-slerp1-7B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4681, - "hfopenllm_v2/BBH": 0.5541, - "hfopenllm_v2/MATH Level 5": 0.3157, - "hfopenllm_v2/GPQA": 0.3171, - "hfopenllm_v2/MUSR": 0.4452, - "hfopenllm_v2/MMLU-PRO": 0.4483 - } - }, - { - "id": "allknowingroger/Meme-7B-slerp", - "name": "Meme-7B-slerp", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5164, - "hfopenllm_v2/BBH": 0.4661, - "hfopenllm_v2/MATH Level 5": 0.0438, - "hfopenllm_v2/GPQA": 0.2861, - "hfopenllm_v2/MUSR": 0.4223, - "hfopenllm_v2/MMLU-PRO": 0.281 - } - }, - { - "id": "allknowingroger/Ministral-8B-slerp", - "name": "Ministral-8B-slerp", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1961, - "hfopenllm_v2/BBH": 0.4686, - "hfopenllm_v2/MATH Level 5": 0.0038, - "hfopenllm_v2/GPQA": 0.3121, - "hfopenllm_v2/MUSR": 0.4285, - "hfopenllm_v2/MMLU-PRO": 0.3119 - } - }, - { - "id": "allknowingroger/Mistralmash1-7B-s", - "name": "Mistralmash1-7B-s", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3961, - "hfopenllm_v2/BBH": 0.5277, - "hfopenllm_v2/MATH Level 5": 0.0921, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.4267, - "hfopenllm_v2/MMLU-PRO": 0.3293 - } - }, - { - "id": "allknowingroger/Mistralmash2-7B-s", - "name": "Mistralmash2-7B-s", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4102, - "hfopenllm_v2/BBH": 0.5305, - "hfopenllm_v2/MATH Level 5": 0.0793, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.4372, - "hfopenllm_v2/MMLU-PRO": 0.3345 - } - }, - { - "id": "allknowingroger/MistralPhi3-11B", - "name": "MistralPhi3-11B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1943, - "hfopenllm_v2/BBH": 0.6234, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.3322, - "hfopenllm_v2/MUSR": 0.4267, - "hfopenllm_v2/MMLU-PRO": 0.4688 - } - }, - { - "id": "allknowingroger/MixTAO-19B-pass", - "name": "MixTAO-19B-pass", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3814, - "hfopenllm_v2/BBH": 0.5128, - "hfopenllm_v2/MATH Level 5": 0.0612, - "hfopenllm_v2/GPQA": 0.2844, - "hfopenllm_v2/MUSR": 0.4783, - "hfopenllm_v2/MMLU-PRO": 0.3105 - } - }, - { - "id": "allknowingroger/MixTaoTruthful-13B-slerp", - "name": "MixTaoTruthful-13B-slerp", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4139, - "hfopenllm_v2/BBH": 0.5207, - "hfopenllm_v2/MATH Level 5": 0.0665, - "hfopenllm_v2/GPQA": 0.2844, - "hfopenllm_v2/MUSR": 0.4292, - "hfopenllm_v2/MMLU-PRO": 0.31 - } - }, - { - "id": "allknowingroger/MultiCalm-7B-slerp", - "name": "MultiCalm-7B-slerp", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3927, - "hfopenllm_v2/BBH": 0.5122, - "hfopenllm_v2/MATH Level 5": 0.0619, - "hfopenllm_v2/GPQA": 0.2827, - "hfopenllm_v2/MUSR": 0.4319, - "hfopenllm_v2/MMLU-PRO": 0.3033 - } - }, - { - "id": "allknowingroger/MultiMash-12B-slerp", - "name": "MultiMash-12B-slerp", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3974, - "hfopenllm_v2/BBH": 0.5142, - "hfopenllm_v2/MATH Level 5": 0.0808, - "hfopenllm_v2/GPQA": 0.2768, - "hfopenllm_v2/MUSR": 0.4438, - "hfopenllm_v2/MMLU-PRO": 0.3068 - } - }, - { - "id": "allknowingroger/MultiMash10-13B-slerp", - "name": "MultiMash10-13B-slerp", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4163, - "hfopenllm_v2/BBH": 0.5186, - "hfopenllm_v2/MATH Level 5": 0.0718, - "hfopenllm_v2/GPQA": 0.2861, - "hfopenllm_v2/MUSR": 0.4318, - "hfopenllm_v2/MMLU-PRO": 0.3117 - } - }, - { - "id": "allknowingroger/MultiMash11-13B-slerp", - "name": "MultiMash11-13B-slerp", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4251, - "hfopenllm_v2/BBH": 0.5194, - "hfopenllm_v2/MATH Level 5": 0.0702, - "hfopenllm_v2/GPQA": 0.2827, - "hfopenllm_v2/MUSR": 0.4373, - "hfopenllm_v2/MMLU-PRO": 0.3085 - } - }, - { - "id": "allknowingroger/MultiMash2-12B-slerp", - "name": "MultiMash2-12B-slerp", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4261, - "hfopenllm_v2/BBH": 0.5134, - "hfopenllm_v2/MATH Level 5": 0.0642, - "hfopenllm_v2/GPQA": 0.2794, - "hfopenllm_v2/MUSR": 0.4228, - "hfopenllm_v2/MMLU-PRO": 0.3043 - } - }, - { - "id": "allknowingroger/Multimash3-12B-slerp", - "name": "Multimash3-12B-slerp", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4437, - "hfopenllm_v2/BBH": 0.5177, - "hfopenllm_v2/MATH Level 5": 0.0627, - "hfopenllm_v2/GPQA": 0.2802, - "hfopenllm_v2/MUSR": 0.4344, - "hfopenllm_v2/MMLU-PRO": 0.3068 - } - }, - { - "id": "allknowingroger/MultiMash5-12B-slerp", - "name": "MultiMash5-12B-slerp", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4142, - "hfopenllm_v2/BBH": 0.5145, - "hfopenllm_v2/MATH Level 5": 0.0634, - "hfopenllm_v2/GPQA": 0.2777, - "hfopenllm_v2/MUSR": 0.4203, - "hfopenllm_v2/MMLU-PRO": 0.3028 - } - }, - { - "id": "allknowingroger/MultiMash6-12B-slerp", - "name": "MultiMash6-12B-slerp", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.43, - "hfopenllm_v2/BBH": 0.5196, - "hfopenllm_v2/MATH Level 5": 0.0725, - "hfopenllm_v2/GPQA": 0.2743, - "hfopenllm_v2/MUSR": 0.4306, - "hfopenllm_v2/MMLU-PRO": 0.3091 - } - }, - { - "id": "allknowingroger/MultiMash7-12B-slerp", - "name": "MultiMash7-12B-slerp", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4213, - "hfopenllm_v2/BBH": 0.5111, - "hfopenllm_v2/MATH Level 5": 0.0695, - "hfopenllm_v2/GPQA": 0.2785, - "hfopenllm_v2/MUSR": 0.4279, - "hfopenllm_v2/MMLU-PRO": 0.3029 - } - }, - { - "id": "allknowingroger/MultiMash8-13B-slerp", - "name": "MultiMash8-13B-slerp", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4321, - "hfopenllm_v2/BBH": 0.5178, - "hfopenllm_v2/MATH Level 5": 0.077, - "hfopenllm_v2/GPQA": 0.2886, - "hfopenllm_v2/MUSR": 0.4424, - "hfopenllm_v2/MMLU-PRO": 0.3126 - } - }, - { - "id": "allknowingroger/MultiMash9-13B-slerp", - "name": "MultiMash9-13B-slerp", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4188, - "hfopenllm_v2/BBH": 0.5194, - "hfopenllm_v2/MATH Level 5": 0.0785, - "hfopenllm_v2/GPQA": 0.2802, - "hfopenllm_v2/MUSR": 0.4398, - "hfopenllm_v2/MMLU-PRO": 0.31 - } - }, - { - "id": "allknowingroger/Multimerge-19B-pass", - "name": "Multimerge-19B-pass", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1773, - "hfopenllm_v2/BBH": 0.2892, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.343, - "hfopenllm_v2/MMLU-PRO": 0.1169 - } - }, - { - "id": "allknowingroger/MultiMerge-7B-slerp", - "name": "MultiMerge-7B-slerp", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3948, - "hfopenllm_v2/BBH": 0.514, - "hfopenllm_v2/MATH Level 5": 0.0665, - "hfopenllm_v2/GPQA": 0.2827, - "hfopenllm_v2/MUSR": 0.428, - "hfopenllm_v2/MMLU-PRO": 0.3037 - } - }, - { - "id": "allknowingroger/MultiverseEx26-7B-slerp", - "name": "MultiverseEx26-7B-slerp", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3939, - "hfopenllm_v2/BBH": 0.5134, - "hfopenllm_v2/MATH Level 5": 0.0755, - "hfopenllm_v2/GPQA": 0.2827, - "hfopenllm_v2/MUSR": 0.4293, - "hfopenllm_v2/MMLU-PRO": 0.3035 - } - }, - { - "id": "allknowingroger/Neuralcoven-7B-slerp", - "name": "Neuralcoven-7B-slerp", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3859, - "hfopenllm_v2/BBH": 0.5303, - "hfopenllm_v2/MATH Level 5": 0.0785, - "hfopenllm_v2/GPQA": 0.2852, - "hfopenllm_v2/MUSR": 0.429, - "hfopenllm_v2/MMLU-PRO": 0.3294 - } - }, - { - "id": "allknowingroger/Neuralmultiverse-7B-slerp", - "name": "Neuralmultiverse-7B-slerp", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3769, - "hfopenllm_v2/BBH": 0.5166, - "hfopenllm_v2/MATH Level 5": 0.065, - "hfopenllm_v2/GPQA": 0.2844, - "hfopenllm_v2/MUSR": 0.428, - "hfopenllm_v2/MMLU-PRO": 0.3042 - } - }, - { - "id": "allknowingroger/NeuralWestSeverus-7B-slerp", - "name": "NeuralWestSeverus-7B-slerp", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4136, - "hfopenllm_v2/BBH": 0.5244, - "hfopenllm_v2/MATH Level 5": 0.0733, - "hfopenllm_v2/GPQA": 0.271, - "hfopenllm_v2/MUSR": 0.4529, - "hfopenllm_v2/MMLU-PRO": 0.3137 - } - }, - { - "id": "allknowingroger/Ph3della5-14B", - "name": "Ph3della5-14B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4799, - "hfopenllm_v2/BBH": 0.6332, - "hfopenllm_v2/MATH Level 5": 0.1767, - "hfopenllm_v2/GPQA": 0.3423, - "hfopenllm_v2/MUSR": 0.4386, - "hfopenllm_v2/MMLU-PRO": 0.4787 - } - }, - { - "id": "allknowingroger/Ph3merge-14B", - "name": "Ph3merge-14B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2701, - "hfopenllm_v2/BBH": 0.6381, - "hfopenllm_v2/MATH Level 5": 0.0106, - "hfopenllm_v2/GPQA": 0.3381, - "hfopenllm_v2/MUSR": 0.4334, - "hfopenllm_v2/MMLU-PRO": 0.4611 - } - }, - { - "id": "allknowingroger/Ph3merge2-14B", - "name": "Ph3merge2-14B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1706, - "hfopenllm_v2/BBH": 0.3607, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2911, - "hfopenllm_v2/MUSR": 0.3911, - "hfopenllm_v2/MMLU-PRO": 0.1723 - } - }, - { - "id": "allknowingroger/Ph3merge3-14B", - "name": "Ph3merge3-14B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1645, - "hfopenllm_v2/BBH": 0.3597, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2852, - "hfopenllm_v2/MUSR": 0.4082, - "hfopenllm_v2/MMLU-PRO": 0.1647 - } - }, - { - "id": "allknowingroger/Ph3task1-14B", - "name": "Ph3task1-14B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4695, - "hfopenllm_v2/BBH": 0.6318, - "hfopenllm_v2/MATH Level 5": 0.1669, - "hfopenllm_v2/GPQA": 0.3507, - "hfopenllm_v2/MUSR": 0.4508, - "hfopenllm_v2/MMLU-PRO": 0.4734 - } - }, - { - "id": "allknowingroger/Ph3task2-14B", - "name": "Ph3task2-14B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4713, - "hfopenllm_v2/BBH": 0.6098, - "hfopenllm_v2/MATH Level 5": 0.1465, - "hfopenllm_v2/GPQA": 0.3305, - "hfopenllm_v2/MUSR": 0.4535, - "hfopenllm_v2/MMLU-PRO": 0.446 - } - }, - { - "id": "allknowingroger/Ph3task3-14B", - "name": "Ph3task3-14B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4962, - "hfopenllm_v2/BBH": 0.6298, - "hfopenllm_v2/MATH Level 5": 0.176, - "hfopenllm_v2/GPQA": 0.3414, - "hfopenllm_v2/MUSR": 0.4426, - "hfopenllm_v2/MMLU-PRO": 0.4771 - } - }, - { - "id": "allknowingroger/Ph3unsloth-3B-slerp", - "name": "Ph3unsloth-3B-slerp", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1894, - "hfopenllm_v2/BBH": 0.5468, - "hfopenllm_v2/MATH Level 5": 0.1012, - "hfopenllm_v2/GPQA": 0.3247, - "hfopenllm_v2/MUSR": 0.4528, - "hfopenllm_v2/MMLU-PRO": 0.3701 - } - }, - { - "id": "allknowingroger/Phi3mash1-17B-pass", - "name": "Phi3mash1-17B-pass", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1884, - "hfopenllm_v2/BBH": 0.6129, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.3196, - "hfopenllm_v2/MUSR": 0.4451, - "hfopenllm_v2/MMLU-PRO": 0.4589 - } - }, - { - "id": "allknowingroger/Quen2-65B", - "name": "Quen2-65B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1758, - "hfopenllm_v2/BBH": 0.2757, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2357, - "hfopenllm_v2/MUSR": 0.3209, - "hfopenllm_v2/MMLU-PRO": 0.1114 - } - }, - { - "id": "allknowingroger/Qwen2.5-42B-AGI", - "name": "Qwen2.5-42B-AGI", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1913, - "hfopenllm_v2/BBH": 0.2942, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.362, - "hfopenllm_v2/MMLU-PRO": 0.1168 - } - }, - { - "id": "allknowingroger/Qwen2.5-7B-task2", - "name": "Qwen2.5-7B-task2", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4527, - "hfopenllm_v2/BBH": 0.5626, - "hfopenllm_v2/MATH Level 5": 0.355, - "hfopenllm_v2/GPQA": 0.3163, - "hfopenllm_v2/MUSR": 0.437, - "hfopenllm_v2/MMLU-PRO": 0.4517 - } - }, - { - "id": "allknowingroger/Qwen2.5-7B-task3", - "name": "Qwen2.5-7B-task3", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5129, - "hfopenllm_v2/BBH": 0.5398, - "hfopenllm_v2/MATH Level 5": 0.2606, - "hfopenllm_v2/GPQA": 0.3171, - "hfopenllm_v2/MUSR": 0.4356, - "hfopenllm_v2/MMLU-PRO": 0.4501 - } - }, - { - "id": "allknowingroger/Qwen2.5-7B-task4", - "name": "Qwen2.5-7B-task4", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5005, - "hfopenllm_v2/BBH": 0.5583, - "hfopenllm_v2/MATH Level 5": 0.3112, - "hfopenllm_v2/GPQA": 0.3205, - "hfopenllm_v2/MUSR": 0.4395, - "hfopenllm_v2/MMLU-PRO": 0.4561 - } - }, - { - "id": "allknowingroger/Qwen2.5-7B-task7", - "name": "Qwen2.5-7B-task7", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4284, - "hfopenllm_v2/BBH": 0.5552, - "hfopenllm_v2/MATH Level 5": 0.065, - "hfopenllm_v2/GPQA": 0.3205, - "hfopenllm_v2/MUSR": 0.4326, - "hfopenllm_v2/MMLU-PRO": 0.4133 - } - }, - { - "id": "allknowingroger/Qwen2.5-7B-task8", - "name": "Qwen2.5-7B-task8", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4645, - "hfopenllm_v2/BBH": 0.5525, - "hfopenllm_v2/MATH Level 5": 0.3527, - "hfopenllm_v2/GPQA": 0.3205, - "hfopenllm_v2/MUSR": 0.4514, - "hfopenllm_v2/MMLU-PRO": 0.4433 - } - }, - { - "id": "allknowingroger/Qwen2.5-slerp-14B", - "name": "Qwen2.5-slerp-14B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4928, - "hfopenllm_v2/BBH": 0.6512, - "hfopenllm_v2/MATH Level 5": 0.4622, - "hfopenllm_v2/GPQA": 0.3674, - "hfopenllm_v2/MUSR": 0.4744, - "hfopenllm_v2/MMLU-PRO": 0.5379 - } - }, - { - "id": "allknowingroger/QwenSlerp12-7B", - "name": "QwenSlerp12-7B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5076, - "hfopenllm_v2/BBH": 0.5556, - "hfopenllm_v2/MATH Level 5": 0.2946, - "hfopenllm_v2/GPQA": 0.3154, - "hfopenllm_v2/MUSR": 0.4595, - "hfopenllm_v2/MMLU-PRO": 0.4461 - } - }, - { - "id": "allknowingroger/Qwenslerp2-14B", - "name": "Qwenslerp2-14B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5007, - "hfopenllm_v2/BBH": 0.6555, - "hfopenllm_v2/MATH Level 5": 0.4456, - "hfopenllm_v2/GPQA": 0.3683, - "hfopenllm_v2/MUSR": 0.4729, - "hfopenllm_v2/MMLU-PRO": 0.5403 - } - }, - { - "id": "allknowingroger/Qwenslerp2-7B", - "name": "Qwenslerp2-7B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5294, - "hfopenllm_v2/BBH": 0.5609, - "hfopenllm_v2/MATH Level 5": 0.3421, - "hfopenllm_v2/GPQA": 0.3129, - "hfopenllm_v2/MUSR": 0.4356, - "hfopenllm_v2/MMLU-PRO": 0.4515 - } - }, - { - "id": "allknowingroger/Qwenslerp3-14B", - "name": "Qwenslerp3-14B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5052, - "hfopenllm_v2/BBH": 0.6521, - "hfopenllm_v2/MATH Level 5": 0.4464, - "hfopenllm_v2/GPQA": 0.375, - "hfopenllm_v2/MUSR": 0.4676, - "hfopenllm_v2/MMLU-PRO": 0.5395 - } - }, - { - "id": "allknowingroger/Qwenslerp3-7B", - "name": "Qwenslerp3-7B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5018, - "hfopenllm_v2/BBH": 0.558, - "hfopenllm_v2/MATH Level 5": 0.3218, - "hfopenllm_v2/GPQA": 0.3247, - "hfopenllm_v2/MUSR": 0.4515, - "hfopenllm_v2/MMLU-PRO": 0.4542 - } - }, - { - "id": "allknowingroger/QwenSlerp4-14B", - "name": "QwenSlerp4-14B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6328, - "hfopenllm_v2/BBH": 0.6483, - "hfopenllm_v2/MATH Level 5": 0.3693, - "hfopenllm_v2/GPQA": 0.3725, - "hfopenllm_v2/MUSR": 0.465, - "hfopenllm_v2/MMLU-PRO": 0.5436 - } - }, - { - "id": "allknowingroger/QwenSlerp5-14B", - "name": "QwenSlerp5-14B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7119, - "hfopenllm_v2/BBH": 0.6357, - "hfopenllm_v2/MATH Level 5": 0.3565, - "hfopenllm_v2/GPQA": 0.3649, - "hfopenllm_v2/MUSR": 0.4675, - "hfopenllm_v2/MMLU-PRO": 0.5391 - } - }, - { - "id": "allknowingroger/QwenSlerp6-14B", - "name": "QwenSlerp6-14B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6867, - "hfopenllm_v2/BBH": 0.6384, - "hfopenllm_v2/MATH Level 5": 0.3724, - "hfopenllm_v2/GPQA": 0.3733, - "hfopenllm_v2/MUSR": 0.469, - "hfopenllm_v2/MMLU-PRO": 0.5406 - } - }, - { - "id": "allknowingroger/QwenStock1-14B", - "name": "QwenStock1-14B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5634, - "hfopenllm_v2/BBH": 0.6528, - "hfopenllm_v2/MATH Level 5": 0.3769, - "hfopenllm_v2/GPQA": 0.3767, - "hfopenllm_v2/MUSR": 0.473, - "hfopenllm_v2/MMLU-PRO": 0.5418 - } - }, - { - "id": "allknowingroger/QwenStock2-14B", - "name": "QwenStock2-14B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5563, - "hfopenllm_v2/BBH": 0.6569, - "hfopenllm_v2/MATH Level 5": 0.3882, - "hfopenllm_v2/GPQA": 0.3792, - "hfopenllm_v2/MUSR": 0.4756, - "hfopenllm_v2/MMLU-PRO": 0.5406 - } - }, - { - "id": "allknowingroger/QwenStock3-14B", - "name": "QwenStock3-14B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5615, - "hfopenllm_v2/BBH": 0.6565, - "hfopenllm_v2/MATH Level 5": 0.3776, - "hfopenllm_v2/GPQA": 0.3784, - "hfopenllm_v2/MUSR": 0.4756, - "hfopenllm_v2/MMLU-PRO": 0.5428 - } - }, - { - "id": "allknowingroger/RogerMerge-7B-slerp", - "name": "RogerMerge-7B-slerp", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3933, - "hfopenllm_v2/BBH": 0.516, - "hfopenllm_v2/MATH Level 5": 0.0687, - "hfopenllm_v2/GPQA": 0.2802, - "hfopenllm_v2/MUSR": 0.432, - "hfopenllm_v2/MMLU-PRO": 0.303 - } - }, - { - "id": "allknowingroger/ROGERphi-7B-slerp", - "name": "ROGERphi-7B-slerp", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3861, - "hfopenllm_v2/BBH": 0.5196, - "hfopenllm_v2/MATH Level 5": 0.0733, - "hfopenllm_v2/GPQA": 0.2886, - "hfopenllm_v2/MUSR": 0.4685, - "hfopenllm_v2/MMLU-PRO": 0.3053 - } - }, - { - "id": "allknowingroger/Rombos-LLM-V2.5-Qwen-42b", - "name": "Rombos-LLM-V2.5-Qwen-42b", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1879, - "hfopenllm_v2/BBH": 0.2969, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2626, - "hfopenllm_v2/MUSR": 0.3633, - "hfopenllm_v2/MMLU-PRO": 0.1168 - } - }, - { - "id": "allknowingroger/Strangecoven-7B-slerp", - "name": "Strangecoven-7B-slerp", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3746, - "hfopenllm_v2/BBH": 0.5368, - "hfopenllm_v2/MATH Level 5": 0.0763, - "hfopenllm_v2/GPQA": 0.2894, - "hfopenllm_v2/MUSR": 0.4199, - "hfopenllm_v2/MMLU-PRO": 0.3364 - } - }, - { - "id": "allknowingroger/Weirdslerp2-25B", - "name": "Weirdslerp2-25B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1754, - "hfopenllm_v2/BBH": 0.2874, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2492, - "hfopenllm_v2/MUSR": 0.3524, - "hfopenllm_v2/MMLU-PRO": 0.1128 - } - }, - { - "id": "allknowingroger/WestlakeMaziyar-7B-slerp", - "name": "WestlakeMaziyar-7B-slerp", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4838, - "hfopenllm_v2/BBH": 0.5245, - "hfopenllm_v2/MATH Level 5": 0.0665, - "hfopenllm_v2/GPQA": 0.3037, - "hfopenllm_v2/MUSR": 0.4474, - "hfopenllm_v2/MMLU-PRO": 0.3078 - } - }, - { - "id": "allknowingroger/YamMaths-7B-slerp", - "name": "YamMaths-7B-slerp", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4148, - "hfopenllm_v2/BBH": 0.5156, - "hfopenllm_v2/MATH Level 5": 0.0853, - "hfopenllm_v2/GPQA": 0.2802, - "hfopenllm_v2/MUSR": 0.4384, - "hfopenllm_v2/MMLU-PRO": 0.3131 - } - }, - { - "id": "allknowingroger/Yi-1.5-34B", - "name": "Yi-1.5-34B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1639, - "hfopenllm_v2/BBH": 0.2827, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2584, - "hfopenllm_v2/MUSR": 0.3857, - "hfopenllm_v2/MMLU-PRO": 0.1095 - } - }, - { - "id": "allknowingroger/Yi-blossom-40B", - "name": "Yi-blossom-40B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2009, - "hfopenllm_v2/BBH": 0.3215, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2743, - "hfopenllm_v2/MUSR": 0.3843, - "hfopenllm_v2/MMLU-PRO": 0.108 - } - }, - { - "id": "allknowingroger/Yibuddy-35B", - "name": "Yibuddy-35B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4235, - "hfopenllm_v2/BBH": 0.5916, - "hfopenllm_v2/MATH Level 5": 0.1571, - "hfopenllm_v2/GPQA": 0.3557, - "hfopenllm_v2/MUSR": 0.4505, - "hfopenllm_v2/MMLU-PRO": 0.4489 - } - }, - { - "id": "allknowingroger/Yillama-40B", - "name": "Yillama-40B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1697, - "hfopenllm_v2/BBH": 0.4063, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2827, - "hfopenllm_v2/MUSR": 0.3501, - "hfopenllm_v2/MMLU-PRO": 0.1981 - } - }, - { - "id": "allknowingroger/Yislerp-34B", - "name": "Yislerp-34B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3692, - "hfopenllm_v2/BBH": 0.6159, - "hfopenllm_v2/MATH Level 5": 0.216, - "hfopenllm_v2/GPQA": 0.3582, - "hfopenllm_v2/MUSR": 0.4566, - "hfopenllm_v2/MMLU-PRO": 0.4751 - } - }, - { - "id": "allknowingroger/Yislerp2-34B", - "name": "Yislerp2-34B", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3999, - "hfopenllm_v2/BBH": 0.6246, - "hfopenllm_v2/MATH Level 5": 0.2296, - "hfopenllm_v2/GPQA": 0.3641, - "hfopenllm_v2/MUSR": 0.453, - "hfopenllm_v2/MMLU-PRO": 0.4724 - } - }, - { - "id": "allknowingroger/Yunconglong-13B-slerp", - "name": "Yunconglong-13B-slerp", - "developer": "allknowingroger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4242, - "hfopenllm_v2/BBH": 0.5166, - "hfopenllm_v2/MATH Level 5": 0.0544, - "hfopenllm_v2/GPQA": 0.281, - "hfopenllm_v2/MUSR": 0.4161, - "hfopenllm_v2/MMLU-PRO": 0.3036 - } - }, - { - "id": "allura-org/L3.1-8b-RP-Ink", - "name": "L3.1-8b-RP-Ink", - "developer": "allura-org", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7811, - "hfopenllm_v2/BBH": 0.4828, - "hfopenllm_v2/MATH Level 5": 0.148, - "hfopenllm_v2/GPQA": 0.2643, - "hfopenllm_v2/MUSR": 0.3608, - "hfopenllm_v2/MMLU-PRO": 0.3428 - } - }, - { - "id": "allura-org/Mistral-Small-24b-Sertraline-0304", - "name": "Mistral-Small-24b-Sertraline-0304", - "developer": "allura-org", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.68, - "hfopenllm_v2/BBH": 0.6525, - "hfopenllm_v2/MATH Level 5": 0.2228, - "hfopenllm_v2/GPQA": 0.3515, - "hfopenllm_v2/MUSR": 0.4395, - "hfopenllm_v2/MMLU-PRO": 0.5106 - } - }, - { - "id": "allura-org/Mistral-Small-Sisyphus-24b-2503", - "name": "Mistral-Small-Sisyphus-24b-2503", - "developer": "allura-org", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6848, - "hfopenllm_v2/BBH": 0.627, - "hfopenllm_v2/MATH Level 5": 0.25, - "hfopenllm_v2/GPQA": 0.2626, - "hfopenllm_v2/MUSR": 0.3977, - "hfopenllm_v2/MMLU-PRO": 0.5127 - } - }, - { - "id": "allura-org/MN-12b-RP-Ink", - "name": "MN-12b-RP-Ink", - "developer": "allura-org", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7186, - "hfopenllm_v2/BBH": 0.4834, - "hfopenllm_v2/MATH Level 5": 0.1186, - "hfopenllm_v2/GPQA": 0.2852, - "hfopenllm_v2/MUSR": 0.3818, - "hfopenllm_v2/MMLU-PRO": 0.3514 - } - }, - { - "id": "allura-org/MoE-Girl-1BA-7BT", - "name": "MoE-Girl-1BA-7BT", - "developer": "allura-org", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2705, - "hfopenllm_v2/BBH": 0.3139, - "hfopenllm_v2/MATH Level 5": 0.0151, - "hfopenllm_v2/GPQA": 0.2584, - "hfopenllm_v2/MUSR": 0.3436, - "hfopenllm_v2/MMLU-PRO": 0.1218 - } - }, - { - "id": "allura-org/MS-Meadowlark-22B", - "name": "MS-Meadowlark-22B", - "developer": "allura-org", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6697, - "hfopenllm_v2/BBH": 0.5163, - "hfopenllm_v2/MATH Level 5": 0.1835, - "hfopenllm_v2/GPQA": 0.3255, - "hfopenllm_v2/MUSR": 0.3843, - "hfopenllm_v2/MMLU-PRO": 0.3823 - } - }, - { - "id": "allura-org/Teleut-7b", - "name": "Teleut-7b", - "developer": "allura-org", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6379, - "hfopenllm_v2/BBH": 0.5141, - "hfopenllm_v2/MATH Level 5": 0.2409, - "hfopenllm_v2/GPQA": 0.3263, - "hfopenllm_v2/MUSR": 0.464, - "hfopenllm_v2/MMLU-PRO": 0.4131 - } - }, - { - "id": "allura-org/TQ2.5-14B-Aletheia-v1", - "name": "TQ2.5-14B-Aletheia-v1", - "developer": "allura-org", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.753, - "hfopenllm_v2/BBH": 0.6585, - "hfopenllm_v2/MATH Level 5": 0.3399, - "hfopenllm_v2/GPQA": 0.3624, - "hfopenllm_v2/MUSR": 0.4452, - "hfopenllm_v2/MMLU-PRO": 0.5241 - } - }, - { - "id": "allura-org/TQ2.5-14B-Neon-v1", - "name": "TQ2.5-14B-Neon-v1", - "developer": "allura-org", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6754, - "hfopenllm_v2/BBH": 0.6553, - "hfopenllm_v2/MATH Level 5": 0.3603, - "hfopenllm_v2/GPQA": 0.3716, - "hfopenllm_v2/MUSR": 0.461, - "hfopenllm_v2/MMLU-PRO": 0.5253 - } - }, - { - "id": "aloobun/d-SmolLM2-360M", - "name": "d-SmolLM2-360M", - "developer": "aloobun", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2097, - "hfopenllm_v2/BBH": 0.3196, - "hfopenllm_v2/MATH Level 5": 0.0128, - "hfopenllm_v2/GPQA": 0.2534, - "hfopenllm_v2/MUSR": 0.3981, - "hfopenllm_v2/MMLU-PRO": 0.1169 - } - }, - { - "id": "aloobun/Meta-Llama-3-7B-28Layers", - "name": "Meta-Llama-3-7B-28Layers", - "developer": "aloobun", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1964, - "hfopenllm_v2/BBH": 0.4437, - "hfopenllm_v2/MATH Level 5": 0.0279, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.3589, - "hfopenllm_v2/MMLU-PRO": 0.316 - } - }, - { - "id": "alpindale/magnum-72b-v1", - "name": "magnum-72b-v1", - "developer": "alpindale", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7606, - "hfopenllm_v2/BBH": 0.6982, - "hfopenllm_v2/MATH Level 5": 0.398, - "hfopenllm_v2/GPQA": 0.3909, - "hfopenllm_v2/MUSR": 0.4489, - "hfopenllm_v2/MMLU-PRO": 0.5468 - } - }, - { - "id": "alpindale/WizardLM-2-8x22B", - "name": "WizardLM-2-8x22B", - "developer": "alpindale", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5272, - "hfopenllm_v2/BBH": 0.6377, - "hfopenllm_v2/MATH Level 5": 0.25, - "hfopenllm_v2/GPQA": 0.3817, - "hfopenllm_v2/MUSR": 0.4387, - "hfopenllm_v2/MMLU-PRO": 0.4596 - } - }, - { - "id": "Alsebay/Qwen2.5-7B-test-novelist", - "name": "Qwen2.5-7B-test-novelist", - "developer": "Alsebay", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5352, - "hfopenllm_v2/BBH": 0.5151, - "hfopenllm_v2/MATH Level 5": 0.2349, - "hfopenllm_v2/GPQA": 0.2911, - "hfopenllm_v2/MUSR": 0.4749, - "hfopenllm_v2/MMLU-PRO": 0.3866 - } - }, - { - "id": "altomek/YiSM-34B-0rn", - "name": "YiSM-34B-0rn", - "developer": "altomek", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4284, - "hfopenllm_v2/BBH": 0.614, - "hfopenllm_v2/MATH Level 5": 0.2281, - "hfopenllm_v2/GPQA": 0.3716, - "hfopenllm_v2/MUSR": 0.445, - "hfopenllm_v2/MMLU-PRO": 0.4696 - } - }, - { - "id": "Amaorynho/BBAI2006", - "name": "BBAI2006", - "developer": "Amaorynho", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1467, - "hfopenllm_v2/BBH": 0.2704, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2525, - "hfopenllm_v2/MUSR": 0.3605, - "hfopenllm_v2/MMLU-PRO": 0.1123 - } - }, - { - "id": "Amaorynho/BBAI270V4", - "name": "BBAI270V4", - "developer": "Amaorynho", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.199, - "hfopenllm_v2/BBH": 0.3071, - "hfopenllm_v2/MATH Level 5": 0.0083, - "hfopenllm_v2/GPQA": 0.2458, - "hfopenllm_v2/MUSR": 0.3314, - "hfopenllm_v2/MMLU-PRO": 0.1114 - } - }, - { - "id": "Amaorynho/BBAI_375", - "name": "BBAI_375", - "developer": "Amaorynho", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1467, - "hfopenllm_v2/BBH": 0.2704, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2525, - "hfopenllm_v2/MUSR": 0.3605, - "hfopenllm_v2/MMLU-PRO": 0.1123 - } - }, - { - "id": "Amaorynho/BBAIIFEV1", - "name": "BBAIIFEV1", - "developer": "Amaorynho", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8047, - "hfopenllm_v2/BBH": 0.5292, - "hfopenllm_v2/MATH Level 5": 0.1934, - "hfopenllm_v2/GPQA": 0.3104, - "hfopenllm_v2/MUSR": 0.4185, - "hfopenllm_v2/MMLU-PRO": 0.3857 - } - }, - { - "id": "amazon/amazon-nova-2-lite-v1-0-fc", - "name": "Amazon-Nova-2-Lite-v1:0 (FC)", - "developer": "amazon", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 80.0, - "bfcl/bfcl.overall.overall_accuracy": 27.1, - "bfcl/bfcl.overall.total_cost_usd": 78.19, - "bfcl/bfcl.overall.latency_mean_s": 8.55, - "bfcl/bfcl.overall.latency_std_s": 9.85, - "bfcl/bfcl.overall.latency_p95_s": 27.62, - "bfcl/bfcl.non_live.ast_accuracy": 86.96, - "bfcl/bfcl.non_live.simple_ast_accuracy": 76.33, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 94.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 91.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 86.0, - "bfcl/bfcl.live.live_accuracy": 80.83, - "bfcl/bfcl.live.live_simple_ast_accuracy": 83.33, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 80.15, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 87.5, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 79.17, - "bfcl/bfcl.multi_turn.accuracy": 2.12, - "bfcl/bfcl.multi_turn.base_accuracy": 2.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 1.5, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 2.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 2.5, - "bfcl/bfcl.web_search.accuracy": 5.0, - "bfcl/bfcl.web_search.base_accuracy": 4.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 6.0, - "bfcl/bfcl.memory.accuracy": 2.37, - "bfcl/bfcl.memory.kv_accuracy": 1.94, - "bfcl/bfcl.memory.vector_accuracy": 3.23, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 1.94, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 75.0, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 82.11 - } - }, - { - "id": "amazon/amazon-nova-micro-v1-0-fc", - "name": "Amazon-Nova-Micro-v1:0 (FC)", - "developer": "amazon", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 95.0, - "bfcl/bfcl.overall.overall_accuracy": 22.29, - "bfcl/bfcl.overall.total_cost_usd": 1.81, - "bfcl/bfcl.overall.latency_mean_s": 1.12, - "bfcl/bfcl.overall.latency_std_s": 0.45, - "bfcl/bfcl.overall.latency_p95_s": 1.79, - "bfcl/bfcl.non_live.ast_accuracy": 74.1, - "bfcl/bfcl.non_live.simple_ast_accuracy": 70.92, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 87.5, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 75.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 62.5, - "bfcl/bfcl.live.live_accuracy": 66.32, - "bfcl/bfcl.live.live_simple_ast_accuracy": 72.09, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 64.96, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 81.25, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 54.17, - "bfcl/bfcl.multi_turn.accuracy": 1.38, - "bfcl/bfcl.multi_turn.base_accuracy": 1.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 1.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 2.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 1.0, - "bfcl/bfcl.web_search.accuracy": 1.5, - "bfcl/bfcl.web_search.base_accuracy": 3.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 0.0, - "bfcl/bfcl.memory.accuracy": 2.37, - "bfcl/bfcl.memory.kv_accuracy": 1.94, - "bfcl/bfcl.memory.vector_accuracy": 2.58, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 2.58, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 81.25, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 70.65 - } - }, - { - "id": "amazon/amazon-nova-pro-v1-0-fc", - "name": "Amazon-Nova-Pro-v1:0 (FC)", - "developer": "amazon", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 88.0, - "bfcl/bfcl.overall.overall_accuracy": 24.97, - "bfcl/bfcl.overall.total_cost_usd": 48.44, - "bfcl/bfcl.overall.latency_mean_s": 2.25, - "bfcl/bfcl.overall.latency_std_s": 1.91, - "bfcl/bfcl.overall.latency_p95_s": 3.29, - "bfcl/bfcl.non_live.ast_accuracy": 86.58, - "bfcl/bfcl.non_live.simple_ast_accuracy": 75.83, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 93.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 93.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 84.0, - "bfcl/bfcl.live.live_accuracy": 78.53, - "bfcl/bfcl.live.live_simple_ast_accuracy": 81.4, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 77.97, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 81.25, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 70.83, - "bfcl/bfcl.multi_turn.accuracy": 1.88, - "bfcl/bfcl.multi_turn.base_accuracy": 1.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 0.5, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 2.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 3.0, - "bfcl/bfcl.web_search.accuracy": 2.5, - "bfcl/bfcl.web_search.base_accuracy": 4.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 1.0, - "bfcl/bfcl.memory.accuracy": 1.94, - "bfcl/bfcl.memory.kv_accuracy": 2.58, - "bfcl/bfcl.memory.vector_accuracy": 1.29, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 1.94, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 93.75, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 70.06 - } - }, - { - "id": "amazon/MegaBeam-Mistral-7B-300k", - "name": "MegaBeam-Mistral-7B-300k", - "developer": "amazon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5203, - "hfopenllm_v2/BBH": 0.4228, - "hfopenllm_v2/MATH Level 5": 0.0211, - "hfopenllm_v2/GPQA": 0.2735, - "hfopenllm_v2/MUSR": 0.398, - "hfopenllm_v2/MMLU-PRO": 0.2549 - } - }, - { - "id": "amazon/nova-lite-v1:0", - "name": "Amazon Nova Lite", - "developer": "amazon", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.551, - "helm_capabilities/MMLU-Pro": 0.6, - "helm_capabilities/GPQA": 0.397, - "helm_capabilities/IFEval": 0.776, - "helm_capabilities/WildBench": 0.75, - "helm_capabilities/Omni-MATH": 0.233, - "helm_lite/Mean win rate": 0.708, - "helm_lite/NarrativeQA": 0.768, - "helm_lite/NaturalQuestions (closed-book)": 0.352, - "helm_lite/OpenbookQA": 0.928, - "helm_lite/MMLU": 0.693, - "helm_lite/MATH": 0.779, - "helm_lite/GSM8K": 0.829, - "helm_lite/LegalBench": 0.659, - "helm_lite/MedQA": 0.696, - "helm_lite/WMT 2014": 0.204, - "helm_mmlu/MMLU All Subjects": 0.77, - "helm_mmlu/Abstract Algebra": 0.52, - "helm_mmlu/Anatomy": 0.719, - "helm_mmlu/College Physics": 0.608, - "helm_mmlu/Computer Security": 0.79, - "helm_mmlu/Econometrics": 0.675, - "helm_mmlu/Global Facts": 0.55, - "helm_mmlu/Jurisprudence": 0.852, - "helm_mmlu/Philosophy": 0.817, - "helm_mmlu/Professional Psychology": 0.812, - "helm_mmlu/Us Foreign Policy": 0.92, - "helm_mmlu/Astronomy": 0.862, - "helm_mmlu/Business Ethics": 0.73, - "helm_mmlu/Clinical Knowledge": 0.8, - "helm_mmlu/Conceptual Physics": 0.796, - "helm_mmlu/Electrical Engineering": 0.779, - "helm_mmlu/Elementary Mathematics": 0.757, - "helm_mmlu/Formal Logic": 0.643, - "helm_mmlu/High School World History": 0.886, - "helm_mmlu/Human Sexuality": 0.84, - "helm_mmlu/International Law": 0.843, - "helm_mmlu/Logical Fallacies": 0.81, - "helm_mmlu/Machine Learning": 0.509, - "helm_mmlu/Management": 0.864, - "helm_mmlu/Marketing": 0.889, - "helm_mmlu/Medical Genetics": 0.9, - "helm_mmlu/Miscellaneous": 0.872, - "helm_mmlu/Moral Scenarios": 0.694, - "helm_mmlu/Nutrition": 0.788, - "helm_mmlu/Prehistory": 0.849, - "helm_mmlu/Public Relations": 0.682, - "helm_mmlu/Security Studies": 0.788, - "helm_mmlu/Sociology": 0.896, - "helm_mmlu/Virology": 0.542, - "helm_mmlu/World Religions": 0.871, - "helm_mmlu/Mean win rate": 0.987 - } - }, - { - "id": "amazon/nova-micro-v1:0", - "name": "Amazon Nova Micro", - "developer": "amazon", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.522, - "helm_capabilities/MMLU-Pro": 0.511, - "helm_capabilities/GPQA": 0.383, - "helm_capabilities/IFEval": 0.76, - "helm_capabilities/WildBench": 0.743, - "helm_capabilities/Omni-MATH": 0.214, - "helm_lite/Mean win rate": 0.524, - "helm_lite/NarrativeQA": 0.744, - "helm_lite/NaturalQuestions (closed-book)": 0.285, - "helm_lite/OpenbookQA": 0.888, - "helm_lite/MMLU": 0.64, - "helm_lite/MATH": 0.76, - "helm_lite/GSM8K": 0.794, - "helm_lite/LegalBench": 0.615, - "helm_lite/MedQA": 0.608, - "helm_lite/WMT 2014": 0.192, - "helm_mmlu/MMLU All Subjects": 0.708, - "helm_mmlu/Abstract Algebra": 0.42, - "helm_mmlu/Anatomy": 0.726, - "helm_mmlu/College Physics": 0.5, - "helm_mmlu/Computer Security": 0.77, - "helm_mmlu/Econometrics": 0.57, - "helm_mmlu/Global Facts": 0.44, - "helm_mmlu/Jurisprudence": 0.815, - "helm_mmlu/Philosophy": 0.733, - "helm_mmlu/Professional Psychology": 0.739, - "helm_mmlu/Us Foreign Policy": 0.9, - "helm_mmlu/Astronomy": 0.822, - "helm_mmlu/Business Ethics": 0.71, - "helm_mmlu/Clinical Knowledge": 0.751, - "helm_mmlu/Conceptual Physics": 0.706, - "helm_mmlu/Electrical Engineering": 0.683, - "helm_mmlu/Elementary Mathematics": 0.55, - "helm_mmlu/Formal Logic": 0.508, - "helm_mmlu/High School World History": 0.84, - "helm_mmlu/Human Sexuality": 0.824, - "helm_mmlu/International Law": 0.843, - "helm_mmlu/Logical Fallacies": 0.798, - "helm_mmlu/Machine Learning": 0.562, - "helm_mmlu/Management": 0.816, - "helm_mmlu/Marketing": 0.91, - "helm_mmlu/Medical Genetics": 0.82, - "helm_mmlu/Miscellaneous": 0.83, - "helm_mmlu/Moral Scenarios": 0.464, - "helm_mmlu/Nutrition": 0.778, - "helm_mmlu/Prehistory": 0.787, - "helm_mmlu/Public Relations": 0.673, - "helm_mmlu/Security Studies": 0.718, - "helm_mmlu/Sociology": 0.846, - "helm_mmlu/Virology": 0.524, - "helm_mmlu/World Religions": 0.825, - "helm_mmlu/Mean win rate": 1.0 - } - }, - { - "id": "amazon/nova-premier-v1:0", - "name": "Amazon Nova Premier", - "developer": "amazon", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.637, - "helm_capabilities/MMLU-Pro": 0.726, - "helm_capabilities/GPQA": 0.518, - "helm_capabilities/IFEval": 0.803, - "helm_capabilities/WildBench": 0.788, - "helm_capabilities/Omni-MATH": 0.35 - } - }, - { - "id": "amazon/nova-pro-v1:0", - "name": "Amazon Nova Pro", - "developer": "amazon", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.591, - "helm_capabilities/MMLU-Pro": 0.673, - "helm_capabilities/GPQA": 0.446, - "helm_capabilities/IFEval": 0.815, - "helm_capabilities/WildBench": 0.777, - "helm_capabilities/Omni-MATH": 0.242, - "helm_lite/Mean win rate": 0.885, - "helm_lite/NarrativeQA": 0.791, - "helm_lite/NaturalQuestions (closed-book)": 0.405, - "helm_lite/OpenbookQA": 0.96, - "helm_lite/MMLU": 0.758, - "helm_lite/MATH": 0.821, - "helm_lite/GSM8K": 0.87, - "helm_lite/LegalBench": 0.736, - "helm_lite/MedQA": 0.811, - "helm_lite/WMT 2014": 0.229, - "helm_mmlu/MMLU All Subjects": 0.82, - "helm_mmlu/Abstract Algebra": 0.69, - "helm_mmlu/Anatomy": 0.807, - "helm_mmlu/College Physics": 0.647, - "helm_mmlu/Computer Security": 0.84, - "helm_mmlu/Econometrics": 0.702, - "helm_mmlu/Global Facts": 0.54, - "helm_mmlu/Jurisprudence": 0.861, - "helm_mmlu/Philosophy": 0.826, - "helm_mmlu/Professional Psychology": 0.864, - "helm_mmlu/Us Foreign Policy": 0.93, - "helm_mmlu/Astronomy": 0.895, - "helm_mmlu/Business Ethics": 0.81, - "helm_mmlu/Clinical Knowledge": 0.875, - "helm_mmlu/Conceptual Physics": 0.851, - "helm_mmlu/Electrical Engineering": 0.8, - "helm_mmlu/Elementary Mathematics": 0.831, - "helm_mmlu/Formal Logic": 0.714, - "helm_mmlu/High School World History": 0.928, - "helm_mmlu/Human Sexuality": 0.885, - "helm_mmlu/International Law": 0.901, - "helm_mmlu/Logical Fallacies": 0.871, - "helm_mmlu/Machine Learning": 0.625, - "helm_mmlu/Management": 0.922, - "helm_mmlu/Marketing": 0.923, - "helm_mmlu/Medical Genetics": 0.87, - "helm_mmlu/Miscellaneous": 0.912, - "helm_mmlu/Moral Scenarios": 0.76, - "helm_mmlu/Nutrition": 0.866, - "helm_mmlu/Prehistory": 0.926, - "helm_mmlu/Public Relations": 0.8, - "helm_mmlu/Security Studies": 0.849, - "helm_mmlu/Sociology": 0.905, - "helm_mmlu/Virology": 0.59, - "helm_mmlu/World Religions": 0.877, - "helm_mmlu/Mean win rate": 0.975 - } - }, - { - "id": "amd/AMD-Llama-135m", - "name": "AMD-Llama-135m", - "developer": "amd", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1842, - "hfopenllm_v2/BBH": 0.2974, - "hfopenllm_v2/MATH Level 5": 0.0053, - "hfopenllm_v2/GPQA": 0.2525, - "hfopenllm_v2/MUSR": 0.378, - "hfopenllm_v2/MMLU-PRO": 0.1169 - } - }, - { - "id": "Amu/t1-1.5B", - "name": "t1-1.5B", - "developer": "Amu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3394, - "hfopenllm_v2/BBH": 0.4008, - "hfopenllm_v2/MATH Level 5": 0.0514, - "hfopenllm_v2/GPQA": 0.2433, - "hfopenllm_v2/MUSR": 0.3517, - "hfopenllm_v2/MMLU-PRO": 0.2566 - } - }, - { - "id": "Amu/t1-3B", - "name": "t1-3B", - "developer": "Amu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3328, - "hfopenllm_v2/BBH": 0.3999, - "hfopenllm_v2/MATH Level 5": 0.1375, - "hfopenllm_v2/GPQA": 0.2408, - "hfopenllm_v2/MUSR": 0.3435, - "hfopenllm_v2/MMLU-PRO": 0.1284 - } - }, - { - "id": "anakin87/gemma-2b-orpo", - "name": "gemma-2b-orpo", - "developer": "anakin87", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2478, - "hfopenllm_v2/BBH": 0.3426, - "hfopenllm_v2/MATH Level 5": 0.0189, - "hfopenllm_v2/GPQA": 0.2617, - "hfopenllm_v2/MUSR": 0.3728, - "hfopenllm_v2/MMLU-PRO": 0.1306 - } - }, - { - "id": "anthracite-org/magnum-v1-72b", - "name": "magnum-v1-72b", - "developer": "anthracite-org", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7606, - "hfopenllm_v2/BBH": 0.6982, - "hfopenllm_v2/MATH Level 5": 0.398, - "hfopenllm_v2/GPQA": 0.3909, - "hfopenllm_v2/MUSR": 0.4489, - "hfopenllm_v2/MMLU-PRO": 0.5486 - } - }, - { - "id": "anthracite-org/magnum-v2-12b", - "name": "magnum-v2-12b", - "developer": "anthracite-org", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3762, - "hfopenllm_v2/BBH": 0.5021, - "hfopenllm_v2/MATH Level 5": 0.0544, - "hfopenllm_v2/GPQA": 0.2911, - "hfopenllm_v2/MUSR": 0.4179, - "hfopenllm_v2/MMLU-PRO": 0.3167 - } - }, - { - "id": "anthracite-org/magnum-v2-72b", - "name": "magnum-v2-72b", - "developer": "anthracite-org", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.756, - "hfopenllm_v2/BBH": 0.7005, - "hfopenllm_v2/MATH Level 5": 0.3542, - "hfopenllm_v2/GPQA": 0.3859, - "hfopenllm_v2/MUSR": 0.4372, - "hfopenllm_v2/MMLU-PRO": 0.5456 - } - }, - { - "id": "anthracite-org/magnum-v2.5-12b-kto", - "name": "magnum-v2.5-12b-kto", - "developer": "anthracite-org", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3866, - "hfopenllm_v2/BBH": 0.5077, - "hfopenllm_v2/MATH Level 5": 0.0521, - "hfopenllm_v2/GPQA": 0.2936, - "hfopenllm_v2/MUSR": 0.4086, - "hfopenllm_v2/MMLU-PRO": 0.3215 - } - }, - { - "id": "anthracite-org/magnum-v3-27b-kto", - "name": "magnum-v3-27b-kto", - "developer": "anthracite-org", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5675, - "hfopenllm_v2/BBH": 0.586, - "hfopenllm_v2/MATH Level 5": 0.1813, - "hfopenllm_v2/GPQA": 0.3557, - "hfopenllm_v2/MUSR": 0.3855, - "hfopenllm_v2/MMLU-PRO": 0.4238 - } - }, - { - "id": "anthracite-org/magnum-v3-34b", - "name": "magnum-v3-34b", - "developer": "anthracite-org", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5115, - "hfopenllm_v2/BBH": 0.6088, - "hfopenllm_v2/MATH Level 5": 0.1949, - "hfopenllm_v2/GPQA": 0.3607, - "hfopenllm_v2/MUSR": 0.3872, - "hfopenllm_v2/MMLU-PRO": 0.4752 - } - }, - { - "id": "anthracite-org/magnum-v3-9b-chatml", - "name": "magnum-v3-9b-chatml", - "developer": "anthracite-org", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1275, - "hfopenllm_v2/BBH": 0.5428, - "hfopenllm_v2/MATH Level 5": 0.0695, - "hfopenllm_v2/GPQA": 0.3456, - "hfopenllm_v2/MUSR": 0.4432, - "hfopenllm_v2/MMLU-PRO": 0.4242 - } - }, - { - "id": "anthracite-org/magnum-v3-9b-customgemma2", - "name": "magnum-v3-9b-customgemma2", - "developer": "anthracite-org", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1273, - "hfopenllm_v2/BBH": 0.534, - "hfopenllm_v2/MATH Level 5": 0.0718, - "hfopenllm_v2/GPQA": 0.3289, - "hfopenllm_v2/MUSR": 0.4565, - "hfopenllm_v2/MMLU-PRO": 0.4205 - } - }, - { - "id": "anthracite-org/magnum-v4-12b", - "name": "magnum-v4-12b", - "developer": "anthracite-org", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3393, - "hfopenllm_v2/BBH": 0.5177, - "hfopenllm_v2/MATH Level 5": 0.1178, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.4093, - "hfopenllm_v2/MMLU-PRO": 0.3604 - } - }, - { - "id": "anthracite-org/magnum-v4-22b", - "name": "magnum-v4-22b", - "developer": "anthracite-org", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5629, - "hfopenllm_v2/BBH": 0.5486, - "hfopenllm_v2/MATH Level 5": 0.2002, - "hfopenllm_v2/GPQA": 0.328, - "hfopenllm_v2/MUSR": 0.4408, - "hfopenllm_v2/MMLU-PRO": 0.383 - } - }, - { - "id": "anthracite-org/magnum-v4-27b", - "name": "magnum-v4-27b", - "developer": "anthracite-org", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3454, - "hfopenllm_v2/BBH": 0.5867, - "hfopenllm_v2/MATH Level 5": 0.1798, - "hfopenllm_v2/GPQA": 0.37, - "hfopenllm_v2/MUSR": 0.438, - "hfopenllm_v2/MMLU-PRO": 0.4376 - } - }, - { - "id": "anthracite-org/magnum-v4-9b", - "name": "magnum-v4-9b", - "developer": "anthracite-org", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3503, - "hfopenllm_v2/BBH": 0.5336, - "hfopenllm_v2/MATH Level 5": 0.1307, - "hfopenllm_v2/GPQA": 0.3473, - "hfopenllm_v2/MUSR": 0.4516, - "hfopenllm_v2/MMLU-PRO": 0.3953 - } - }, - { - "id": "Anthropic-LM-v4-s3-52B", - "name": "Anthropic-LM v4-s3 52B", - "developer": "unknown", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.78, - "helm_classic/MMLU": 0.481, - "helm_classic/BoolQ": 0.815, - "helm_classic/NarrativeQA": 0.728, - "helm_classic/NaturalQuestions (open-book)": 0.686, - "helm_classic/QuAC": 0.431, - "helm_classic/HellaSwag": 0.807, - "helm_classic/OpenbookQA": 0.558, - "helm_classic/TruthfulQA": 0.368, - "helm_classic/MS MARCO (TREC)": -1.0, - "helm_classic/CNN/DailyMail": 0.154, - "helm_classic/XSUM": 0.134, - "helm_classic/IMDB": 0.934, - "helm_classic/CivilComments": 0.61, - "helm_classic/RAFT": 0.699 - } - }, - { - "id": "anthropic/claude-2.0", - "name": "Claude 2.0", - "developer": "Anthropic", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.489, - "helm_lite/NarrativeQA": 0.718, - "helm_lite/NaturalQuestions (closed-book)": 0.428, - "helm_lite/OpenbookQA": 0.862, - "helm_lite/MMLU": 0.639, - "helm_lite/MATH": 0.603, - "helm_lite/GSM8K": 0.583, - "helm_lite/LegalBench": 0.643, - "helm_lite/MedQA": 0.652, - "helm_lite/WMT 2014": 0.219 - } - }, - { - "id": "anthropic/claude-2.1", - "name": "Claude 2.1", - "developer": "Anthropic", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.437, - "helm_lite/NarrativeQA": 0.677, - "helm_lite/NaturalQuestions (closed-book)": 0.375, - "helm_lite/OpenbookQA": 0.872, - "helm_lite/MMLU": 0.643, - "helm_lite/MATH": 0.632, - "helm_lite/GSM8K": 0.604, - "helm_lite/LegalBench": 0.643, - "helm_lite/MedQA": 0.644, - "helm_lite/WMT 2014": 0.204, - "helm_mmlu/MMLU All Subjects": 0.735, - "helm_mmlu/Abstract Algebra": 0.4, - "helm_mmlu/Anatomy": 0.726, - "helm_mmlu/College Physics": 0.5, - "helm_mmlu/Computer Security": 0.81, - "helm_mmlu/Econometrics": 0.596, - "helm_mmlu/Global Facts": 0.55, - "helm_mmlu/Jurisprudence": 0.87, - "helm_mmlu/Philosophy": 0.794, - "helm_mmlu/Professional Psychology": 0.797, - "helm_mmlu/Us Foreign Policy": 0.92, - "helm_mmlu/Astronomy": 0.855, - "helm_mmlu/Business Ethics": 0.73, - "helm_mmlu/Clinical Knowledge": 0.785, - "helm_mmlu/Conceptual Physics": 0.766, - "helm_mmlu/Electrical Engineering": 0.724, - "helm_mmlu/Elementary Mathematics": 0.521, - "helm_mmlu/Formal Logic": 0.5, - "helm_mmlu/High School World History": 0.903, - "helm_mmlu/Human Sexuality": 0.847, - "helm_mmlu/International Law": 0.901, - "helm_mmlu/Logical Fallacies": 0.834, - "helm_mmlu/Machine Learning": 0.482, - "helm_mmlu/Management": 0.825, - "helm_mmlu/Marketing": 0.923, - "helm_mmlu/Medical Genetics": 0.81, - "helm_mmlu/Miscellaneous": 0.88, - "helm_mmlu/Moral Scenarios": 0.52, - "helm_mmlu/Nutrition": 0.781, - "helm_mmlu/Prehistory": 0.821, - "helm_mmlu/Public Relations": 0.773, - "helm_mmlu/Security Studies": 0.812, - "helm_mmlu/Sociology": 0.886, - "helm_mmlu/Virology": 0.554, - "helm_mmlu/World Religions": 0.854, - "helm_mmlu/Mean win rate": 0.048 - } - }, - { - "id": "anthropic/claude-3-5-haiku-20241022", - "name": "Claude 3.5 Haiku 20241022", - "developer": "Anthropic", - "evaluator_relationship": null, - "benchmark_scores": { - "global-mmlu-lite/Global MMLU Lite": 0.6114, - "global-mmlu-lite/Culturally Sensitive": 0.5834, - "global-mmlu-lite/Culturally Agnostic": 0.6394, - "global-mmlu-lite/Arabic": 0.695, - "global-mmlu-lite/English": 0.485, - "global-mmlu-lite/Bengali": 0.675, - "global-mmlu-lite/German": 0.565, - "global-mmlu-lite/French": 0.61, - "global-mmlu-lite/Hindi": 0.6575, - "global-mmlu-lite/Indonesian": 0.5475, - "global-mmlu-lite/Italian": 0.48, - "global-mmlu-lite/Japanese": 0.655, - "global-mmlu-lite/Korean": 0.6575, - "global-mmlu-lite/Portuguese": 0.5225, - "global-mmlu-lite/Spanish": 0.485, - "global-mmlu-lite/Swahili": 0.69, - "global-mmlu-lite/Yoruba": 0.6675, - "global-mmlu-lite/Chinese": 0.69, - "global-mmlu-lite/Burmese": 0.7, - "helm_capabilities/Mean score": 0.549, - "helm_capabilities/MMLU-Pro": 0.605, - "helm_capabilities/GPQA": 0.363, - "helm_capabilities/IFEval": 0.792, - "helm_capabilities/WildBench": 0.76, - "helm_capabilities/Omni-MATH": 0.224, - "helm_lite/Mean win rate": 0.531, - "helm_lite/NarrativeQA": 0.763, - "helm_lite/NaturalQuestions (closed-book)": 0.344, - "helm_lite/OpenbookQA": 0.854, - "helm_lite/MMLU": 0.671, - "helm_lite/MATH": 0.872, - "helm_lite/GSM8K": 0.815, - "helm_lite/LegalBench": 0.631, - "helm_lite/MedQA": 0.722, - "helm_lite/WMT 2014": 0.135, - "helm_mmlu/MMLU All Subjects": 0.743, - "helm_mmlu/Abstract Algebra": 0.47, - "helm_mmlu/Anatomy": 0.793, - "helm_mmlu/College Physics": 0.52, - "helm_mmlu/Computer Security": 0.84, - "helm_mmlu/Econometrics": 0.596, - "helm_mmlu/Global Facts": 0.5, - "helm_mmlu/Jurisprudence": 0.861, - "helm_mmlu/Philosophy": 0.823, - "helm_mmlu/Professional Psychology": 0.825, - "helm_mmlu/Us Foreign Policy": 0.94, - "helm_mmlu/Astronomy": 0.829, - "helm_mmlu/Business Ethics": 0.8, - "helm_mmlu/Clinical Knowledge": 0.823, - "helm_mmlu/Conceptual Physics": 0.723, - "helm_mmlu/Electrical Engineering": 0.717, - "helm_mmlu/Elementary Mathematics": 0.561, - "helm_mmlu/Formal Logic": 0.619, - "helm_mmlu/High School World History": 0.882, - "helm_mmlu/Human Sexuality": 0.885, - "helm_mmlu/International Law": 0.884, - "helm_mmlu/Logical Fallacies": 0.822, - "helm_mmlu/Machine Learning": 0.518, - "helm_mmlu/Management": 0.845, - "helm_mmlu/Marketing": 0.897, - "helm_mmlu/Medical Genetics": 0.83, - "helm_mmlu/Miscellaneous": 0.905, - "helm_mmlu/Moral Scenarios": 0.476, - "helm_mmlu/Nutrition": 0.846, - "helm_mmlu/Prehistory": 0.877, - "helm_mmlu/Public Relations": 0.727, - "helm_mmlu/Security Studies": 0.792, - "helm_mmlu/Sociology": 0.905, - "helm_mmlu/Virology": 0.566, - "helm_mmlu/World Religions": 0.865, - "helm_mmlu/Mean win rate": 0.128 - } - }, - { - "id": "anthropic/claude-3-5-sonnet-20240620", - "name": "Claude 3.5 Sonnet 20240620", - "developer": "Anthropic", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.885, - "helm_lite/NarrativeQA": 0.746, - "helm_lite/NaturalQuestions (closed-book)": 0.502, - "helm_lite/OpenbookQA": 0.972, - "helm_lite/MMLU": 0.799, - "helm_lite/MATH": 0.813, - "helm_lite/GSM8K": 0.949, - "helm_lite/LegalBench": 0.707, - "helm_lite/MedQA": 0.825, - "helm_lite/WMT 2014": 0.229, - "helm_mmlu/MMLU All Subjects": 0.865, - "helm_mmlu/Abstract Algebra": 0.75, - "helm_mmlu/Anatomy": 0.844, - "helm_mmlu/College Physics": 0.696, - "helm_mmlu/Computer Security": 0.89, - "helm_mmlu/Econometrics": 0.807, - "helm_mmlu/Global Facts": 0.72, - "helm_mmlu/Jurisprudence": 0.889, - "helm_mmlu/Philosophy": 0.891, - "helm_mmlu/Professional Psychology": 0.922, - "helm_mmlu/Us Foreign Policy": 0.96, - "helm_mmlu/Astronomy": 0.961, - "helm_mmlu/Business Ethics": 0.85, - "helm_mmlu/Clinical Knowledge": 0.913, - "helm_mmlu/Conceptual Physics": 0.885, - "helm_mmlu/Electrical Engineering": 0.828, - "helm_mmlu/Elementary Mathematics": 0.892, - "helm_mmlu/Formal Logic": 0.698, - "helm_mmlu/High School World History": 0.954, - "helm_mmlu/Human Sexuality": 0.939, - "helm_mmlu/International Law": 0.959, - "helm_mmlu/Logical Fallacies": 0.926, - "helm_mmlu/Machine Learning": 0.786, - "helm_mmlu/Management": 0.942, - "helm_mmlu/Marketing": 0.949, - "helm_mmlu/Medical Genetics": 0.98, - "helm_mmlu/Miscellaneous": 0.962, - "helm_mmlu/Moral Scenarios": 0.882, - "helm_mmlu/Nutrition": 0.912, - "helm_mmlu/Prehistory": 0.951, - "helm_mmlu/Public Relations": 0.855, - "helm_mmlu/Security Studies": 0.878, - "helm_mmlu/Sociology": 0.96, - "helm_mmlu/Virology": 0.602, - "helm_mmlu/World Religions": 0.924, - "helm_mmlu/Mean win rate": 0.17, - "reward-bench/Score": 0.6466, - "reward-bench/Chat": 0.9637, - "reward-bench/Chat Hard": 0.7401, - "reward-bench/Safety": 0.8519, - "reward-bench/Reasoning": 0.8469, - "reward-bench/Factuality": 0.5284, - "reward-bench/Precise IF": 0.3875, - "reward-bench/Math": 0.5683, - "reward-bench/Focus": 0.8697, - "reward-bench/Ties": 0.674 - } - }, - { - "id": "anthropic/claude-3-5-sonnet-20241022", - "name": "Claude 3.5 Sonnet 20241022", - "developer": "Anthropic", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.653, - "helm_capabilities/MMLU-Pro": 0.777, - "helm_capabilities/GPQA": 0.565, - "helm_capabilities/IFEval": 0.856, - "helm_capabilities/WildBench": 0.792, - "helm_capabilities/Omni-MATH": 0.276, - "helm_lite/Mean win rate": 0.846, - "helm_lite/NarrativeQA": 0.77, - "helm_lite/NaturalQuestions (closed-book)": 0.467, - "helm_lite/OpenbookQA": 0.966, - "helm_lite/MMLU": 0.809, - "helm_lite/MATH": 0.904, - "helm_lite/GSM8K": 0.956, - "helm_lite/LegalBench": 0.647, - "helm_lite/MedQA": 0.859, - "helm_lite/WMT 2014": 0.226, - "helm_mmlu/MMLU All Subjects": 0.873, - "helm_mmlu/Abstract Algebra": 0.78, - "helm_mmlu/Anatomy": 0.859, - "helm_mmlu/College Physics": 0.775, - "helm_mmlu/Computer Security": 0.87, - "helm_mmlu/Econometrics": 0.807, - "helm_mmlu/Global Facts": 0.8, - "helm_mmlu/Jurisprudence": 0.898, - "helm_mmlu/Philosophy": 0.891, - "helm_mmlu/Professional Psychology": 0.922, - "helm_mmlu/Us Foreign Policy": 0.96, - "helm_mmlu/Astronomy": 0.974, - "helm_mmlu/Business Ethics": 0.83, - "helm_mmlu/Clinical Knowledge": 0.928, - "helm_mmlu/Conceptual Physics": 0.906, - "helm_mmlu/Electrical Engineering": 0.848, - "helm_mmlu/Elementary Mathematics": 0.918, - "helm_mmlu/Formal Logic": 0.786, - "helm_mmlu/High School World History": 0.958, - "helm_mmlu/Human Sexuality": 0.939, - "helm_mmlu/International Law": 0.959, - "helm_mmlu/Logical Fallacies": 0.914, - "helm_mmlu/Machine Learning": 0.839, - "helm_mmlu/Management": 0.932, - "helm_mmlu/Marketing": 0.953, - "helm_mmlu/Medical Genetics": 0.96, - "helm_mmlu/Miscellaneous": 0.964, - "helm_mmlu/Moral Scenarios": 0.888, - "helm_mmlu/Nutrition": 0.922, - "helm_mmlu/Prehistory": 0.941, - "helm_mmlu/Public Relations": 0.8, - "helm_mmlu/Security Studies": 0.882, - "helm_mmlu/Sociology": 0.955, - "helm_mmlu/Virology": 0.584, - "helm_mmlu/World Religions": 0.901, - "helm_mmlu/Mean win rate": 0.311 - } - }, - { - "id": "anthropic/claude-3-7-sonnet-20250219", - "name": "claude-3-7-sonnet-20250219", - "developer": "Anthropic", - "evaluator_relationship": null, - "benchmark_scores": { - "global-mmlu-lite/Global MMLU Lite": 0.8078, - "global-mmlu-lite/Culturally Sensitive": 0.7794, - "global-mmlu-lite/Culturally Agnostic": 0.8362, - "global-mmlu-lite/Arabic": 0.7925, - "global-mmlu-lite/English": 0.7625, - "global-mmlu-lite/Bengali": 0.825, - "global-mmlu-lite/German": 0.8125, - "global-mmlu-lite/French": 0.7675, - "global-mmlu-lite/Hindi": 0.805, - "global-mmlu-lite/Indonesian": 0.8175, - "global-mmlu-lite/Italian": 0.8225, - "global-mmlu-lite/Japanese": 0.8425, - "global-mmlu-lite/Korean": 0.83, - "global-mmlu-lite/Portuguese": 0.77, - "global-mmlu-lite/Spanish": 0.8075, - "global-mmlu-lite/Swahili": 0.8125, - "global-mmlu-lite/Yoruba": 0.81, - "global-mmlu-lite/Chinese": 0.835, - "global-mmlu-lite/Burmese": 0.8125, - "helm_capabilities/Mean score": 0.674, - "helm_capabilities/MMLU-Pro": 0.784, - "helm_capabilities/GPQA": 0.608, - "helm_capabilities/IFEval": 0.834, - "helm_capabilities/WildBench": 0.814, - "helm_capabilities/Omni-MATH": 0.33, - "livecodebenchpro/Hard Problems": 0.0, - "livecodebenchpro/Medium Problems": 0.0, - "livecodebenchpro/Easy Problems": 0.28169014084507044, - "reward-bench/Score": 0.7539, - "reward-bench/Factuality": 0.7326, - "reward-bench/Precise IF": 0.5437, - "reward-bench/Math": 0.75, - "reward-bench/Safety": 0.9033, - "reward-bench/Focus": 0.9212, - "reward-bench/Ties": 0.6723 - } - }, - { - "id": "anthropic/claude-3-haiku-20240307", - "name": "Claude 3 Haiku 20240307", - "developer": "Anthropic", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.263, - "helm_lite/NarrativeQA": 0.244, - "helm_lite/NaturalQuestions (closed-book)": 0.144, - "helm_lite/OpenbookQA": 0.838, - "helm_lite/MMLU": 0.662, - "helm_lite/MATH": 0.131, - "helm_lite/GSM8K": 0.699, - "helm_lite/LegalBench": 0.46, - "helm_lite/MedQA": 0.702, - "helm_lite/WMT 2014": 0.148, - "helm_mmlu/MMLU All Subjects": 0.738, - "helm_mmlu/Abstract Algebra": 0.42, - "helm_mmlu/Anatomy": 0.711, - "helm_mmlu/College Physics": 0.48, - "helm_mmlu/Computer Security": 0.79, - "helm_mmlu/Econometrics": 0.632, - "helm_mmlu/Global Facts": 0.47, - "helm_mmlu/Jurisprudence": 0.861, - "helm_mmlu/Philosophy": 0.814, - "helm_mmlu/Professional Psychology": 0.802, - "helm_mmlu/Us Foreign Policy": 0.95, - "helm_mmlu/Astronomy": 0.901, - "helm_mmlu/Business Ethics": 0.78, - "helm_mmlu/Clinical Knowledge": 0.789, - "helm_mmlu/Conceptual Physics": 0.715, - "helm_mmlu/Electrical Engineering": 0.69, - "helm_mmlu/Elementary Mathematics": 0.558, - "helm_mmlu/Formal Logic": 0.579, - "helm_mmlu/High School World History": 0.878, - "helm_mmlu/Human Sexuality": 0.824, - "helm_mmlu/International Law": 0.901, - "helm_mmlu/Logical Fallacies": 0.791, - "helm_mmlu/Machine Learning": 0.589, - "helm_mmlu/Management": 0.874, - "helm_mmlu/Marketing": 0.91, - "helm_mmlu/Medical Genetics": 0.8, - "helm_mmlu/Miscellaneous": 0.893, - "helm_mmlu/Moral Scenarios": 0.502, - "helm_mmlu/Nutrition": 0.83, - "helm_mmlu/Prehistory": 0.824, - "helm_mmlu/Public Relations": 0.755, - "helm_mmlu/Security Studies": 0.808, - "helm_mmlu/Sociology": 0.9, - "helm_mmlu/Virology": 0.542, - "helm_mmlu/World Religions": 0.871, - "helm_mmlu/Mean win rate": 0.28, - "reward-bench/Score": 0.3711, - "reward-bench/Chat": 0.9274, - "reward-bench/Chat Hard": 0.5197, - "reward-bench/Safety": 0.595, - "reward-bench/Reasoning": 0.706, - "reward-bench/Prior Sets (0.5 weight)": 0.6635, - "reward-bench/Factuality": 0.4042, - "reward-bench/Precise IF": 0.2812, - "reward-bench/Math": 0.3552, - "reward-bench/Focus": 0.501, - "reward-bench/Ties": 0.0899 - } - }, - { - "id": "anthropic/claude-3-opus-20240229", - "name": "Claude 3 Opus 20240229", - "developer": "Anthropic", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.683, - "helm_lite/NarrativeQA": 0.351, - "helm_lite/NaturalQuestions (closed-book)": 0.441, - "helm_lite/OpenbookQA": 0.956, - "helm_lite/MMLU": 0.768, - "helm_lite/MATH": 0.76, - "helm_lite/GSM8K": 0.924, - "helm_lite/LegalBench": 0.662, - "helm_lite/MedQA": 0.775, - "helm_lite/WMT 2014": 0.24, - "helm_mmlu/MMLU All Subjects": 0.846, - "helm_mmlu/Abstract Algebra": 0.64, - "helm_mmlu/Anatomy": 0.8, - "helm_mmlu/College Physics": 0.716, - "helm_mmlu/Computer Security": 0.85, - "helm_mmlu/Econometrics": 0.789, - "helm_mmlu/Global Facts": 0.66, - "helm_mmlu/Jurisprudence": 0.88, - "helm_mmlu/Philosophy": 0.9, - "helm_mmlu/Professional Psychology": 0.904, - "helm_mmlu/Us Foreign Policy": 0.96, - "helm_mmlu/Astronomy": 0.967, - "helm_mmlu/Business Ethics": 0.86, - "helm_mmlu/Clinical Knowledge": 0.879, - "helm_mmlu/Conceptual Physics": 0.881, - "helm_mmlu/Electrical Engineering": 0.814, - "helm_mmlu/Elementary Mathematics": 0.862, - "helm_mmlu/Formal Logic": 0.698, - "helm_mmlu/High School World History": 0.941, - "helm_mmlu/Human Sexuality": 0.908, - "helm_mmlu/International Law": 0.901, - "helm_mmlu/Logical Fallacies": 0.896, - "helm_mmlu/Machine Learning": 0.741, - "helm_mmlu/Management": 0.942, - "helm_mmlu/Marketing": 0.944, - "helm_mmlu/Medical Genetics": 0.93, - "helm_mmlu/Miscellaneous": 0.951, - "helm_mmlu/Moral Scenarios": 0.826, - "helm_mmlu/Nutrition": 0.925, - "helm_mmlu/Prehistory": 0.941, - "helm_mmlu/Public Relations": 0.827, - "helm_mmlu/Security Studies": 0.886, - "helm_mmlu/Sociology": 0.94, - "helm_mmlu/Virology": 0.578, - "helm_mmlu/World Religions": 0.901, - "helm_mmlu/Mean win rate": 0.014, - "reward-bench/Score": 0.5744, - "reward-bench/Chat": 0.9469, - "reward-bench/Chat Hard": 0.6031, - "reward-bench/Safety": 0.8378, - "reward-bench/Reasoning": 0.7868, - "reward-bench/Factuality": 0.5389, - "reward-bench/Precise IF": 0.3312, - "reward-bench/Math": 0.5137, - "reward-bench/Focus": 0.6646, - "reward-bench/Ties": 0.5601 - } - }, - { - "id": "anthropic/claude-3-sonnet-20240229", - "name": "Claude 3 Sonnet 20240229", - "developer": "Anthropic", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.377, - "helm_lite/NarrativeQA": 0.111, - "helm_lite/NaturalQuestions (closed-book)": 0.028, - "helm_lite/OpenbookQA": 0.918, - "helm_lite/MMLU": 0.652, - "helm_lite/MATH": 0.084, - "helm_lite/GSM8K": 0.907, - "helm_lite/LegalBench": 0.49, - "helm_lite/MedQA": 0.684, - "helm_lite/WMT 2014": 0.218, - "helm_mmlu/MMLU All Subjects": 0.759, - "helm_mmlu/Abstract Algebra": 0.39, - "helm_mmlu/Anatomy": 0.711, - "helm_mmlu/College Physics": 0.559, - "helm_mmlu/Computer Security": 0.79, - "helm_mmlu/Econometrics": 0.64, - "helm_mmlu/Global Facts": 0.53, - "helm_mmlu/Jurisprudence": 0.861, - "helm_mmlu/Philosophy": 0.852, - "helm_mmlu/Professional Psychology": 0.814, - "helm_mmlu/Us Foreign Policy": 0.94, - "helm_mmlu/Astronomy": 0.855, - "helm_mmlu/Business Ethics": 0.82, - "helm_mmlu/Clinical Knowledge": 0.804, - "helm_mmlu/Conceptual Physics": 0.774, - "helm_mmlu/Electrical Engineering": 0.703, - "helm_mmlu/Elementary Mathematics": 0.635, - "helm_mmlu/Formal Logic": 0.579, - "helm_mmlu/High School World History": 0.895, - "helm_mmlu/Human Sexuality": 0.809, - "helm_mmlu/International Law": 0.909, - "helm_mmlu/Logical Fallacies": 0.853, - "helm_mmlu/Machine Learning": 0.643, - "helm_mmlu/Management": 0.922, - "helm_mmlu/Marketing": 0.85, - "helm_mmlu/Medical Genetics": 0.79, - "helm_mmlu/Miscellaneous": 0.872, - "helm_mmlu/Moral Scenarios": 0.626, - "helm_mmlu/Nutrition": 0.82, - "helm_mmlu/Prehistory": 0.864, - "helm_mmlu/Public Relations": 0.782, - "helm_mmlu/Security Studies": 0.865, - "helm_mmlu/Sociology": 0.905, - "helm_mmlu/Virology": 0.578, - "helm_mmlu/World Religions": 0.871, - "helm_mmlu/Mean win rate": 0.082, - "reward-bench/Score": 0.7458, - "reward-bench/Chat": 0.9344, - "reward-bench/Chat Hard": 0.5658, - "reward-bench/Safety": 0.8169, - "reward-bench/Reasoning": 0.6907, - "reward-bench/Prior Sets (0.5 weight)": 0.6963 - } - }, - { - "id": "anthropic/claude-3.7-sonnet", - "name": "anthropic/claude-3.7-sonnet", - "developer": "Anthropic", - "evaluator_relationship": null, - "benchmark_scores": { - "livecodebenchpro/Hard Problems": 0.0, - "livecodebenchpro/Medium Problems": 0.014084507042253521, - "livecodebenchpro/Easy Problems": 0.15492957746478872 - } - }, - { - "id": "anthropic/claude-haiku-4-5-20251001-fc", - "name": "Claude-Haiku-4-5-20251001 (FC)", - "developer": "Anthropic", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 6.0, - "bfcl/bfcl.overall.overall_accuracy": 68.7, - "bfcl/bfcl.overall.total_cost_usd": 14.23, - "bfcl/bfcl.overall.latency_mean_s": 1.68, - "bfcl/bfcl.overall.latency_std_s": 3.92, - "bfcl/bfcl.overall.latency_p95_s": 3.15, - "bfcl/bfcl.non_live.ast_accuracy": 86.5, - "bfcl/bfcl.non_live.simple_ast_accuracy": 71.0, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 94.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 92.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 88.5, - "bfcl/bfcl.live.live_accuracy": 78.68, - "bfcl/bfcl.live.live_simple_ast_accuracy": 83.72, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 77.59, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 75.0, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 75.0, - "bfcl/bfcl.multi_turn.accuracy": 53.62, - "bfcl/bfcl.multi_turn.base_accuracy": 63.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 42.5, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 52.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 56.0, - "bfcl/bfcl.web_search.accuracy": 83.5, - "bfcl/bfcl.web_search.base_accuracy": 86.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 81.0, - "bfcl/bfcl.memory.accuracy": 54.41, - "bfcl/bfcl.memory.kv_accuracy": 51.61, - "bfcl/bfcl.memory.vector_accuracy": 55.48, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 56.13, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 62.5, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 85.11 - } - }, - { - "id": "anthropic/claude-haiku-4-5-20251001-prompt", - "name": "Claude-Haiku-4-5-20251001 (Prompt)", - "developer": "Anthropic", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 87.0, - "bfcl/bfcl.overall.overall_accuracy": 25.26, - "bfcl/bfcl.overall.total_cost_usd": 45.13, - "bfcl/bfcl.overall.latency_mean_s": 3.75, - "bfcl/bfcl.overall.latency_std_s": 19.96, - "bfcl/bfcl.overall.latency_p95_s": 3.77, - "bfcl/bfcl.non_live.ast_accuracy": 55.42, - "bfcl/bfcl.non_live.simple_ast_accuracy": 55.67, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 84.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 38.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 44.0, - "bfcl/bfcl.live.live_accuracy": 52.48, - "bfcl/bfcl.live.live_simple_ast_accuracy": 66.67, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 49.76, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 56.25, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 16.67, - "bfcl/bfcl.multi_turn.accuracy": 1.75, - "bfcl/bfcl.multi_turn.base_accuracy": 1.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 0.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 4.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 1.5, - "bfcl/bfcl.web_search.accuracy": 19.5, - "bfcl/bfcl.web_search.base_accuracy": 20.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 19.0, - "bfcl/bfcl.memory.accuracy": 2.58, - "bfcl/bfcl.memory.kv_accuracy": 2.58, - "bfcl/bfcl.memory.vector_accuracy": 1.94, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 3.23, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 31.25, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 95.29, - "bfcl/bfcl.format_sensitivity.max_delta": 67.5, - "bfcl/bfcl.format_sensitivity.stddev": 20.07 - } - }, - { - "id": "anthropic/claude-haiku-4.5", - "name": "Claude Haiku 4.5", - "developer": "Anthropic", - "evaluator_relationship": null, - "benchmark_scores": { - "terminal-bench-2.0/terminal-bench-2.0": 35.5 - } - }, - { - "id": "anthropic/claude-instant-1.2", - "name": "Claude Instant 1.2", - "developer": "Anthropic", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.399, - "helm_lite/NarrativeQA": 0.616, - "helm_lite/NaturalQuestions (closed-book)": 0.343, - "helm_lite/OpenbookQA": 0.844, - "helm_lite/MMLU": 0.631, - "helm_lite/MATH": 0.499, - "helm_lite/GSM8K": 0.721, - "helm_lite/LegalBench": 0.586, - "helm_lite/MedQA": 0.559, - "helm_lite/WMT 2014": 0.194, - "helm_mmlu/MMLU All Subjects": 0.688, - "helm_mmlu/Abstract Algebra": 0.37, - "helm_mmlu/Anatomy": 0.637, - "helm_mmlu/College Physics": 0.49, - "helm_mmlu/Computer Security": 0.76, - "helm_mmlu/Econometrics": 0.614, - "helm_mmlu/Global Facts": 0.38, - "helm_mmlu/Jurisprudence": 0.833, - "helm_mmlu/Philosophy": 0.756, - "helm_mmlu/Professional Psychology": 0.724, - "helm_mmlu/Us Foreign Policy": 0.9, - "helm_mmlu/Astronomy": 0.743, - "helm_mmlu/Business Ethics": 0.7, - "helm_mmlu/Clinical Knowledge": 0.709, - "helm_mmlu/Conceptual Physics": 0.613, - "helm_mmlu/Electrical Engineering": 0.641, - "helm_mmlu/Elementary Mathematics": 0.45, - "helm_mmlu/Formal Logic": 0.444, - "helm_mmlu/High School World History": 0.878, - "helm_mmlu/Human Sexuality": 0.794, - "helm_mmlu/International Law": 0.851, - "helm_mmlu/Logical Fallacies": 0.81, - "helm_mmlu/Machine Learning": 0.67, - "helm_mmlu/Management": 0.835, - "helm_mmlu/Marketing": 0.885, - "helm_mmlu/Medical Genetics": 0.71, - "helm_mmlu/Miscellaneous": 0.828, - "helm_mmlu/Moral Scenarios": 0.488, - "helm_mmlu/Nutrition": 0.735, - "helm_mmlu/Prehistory": 0.762, - "helm_mmlu/Public Relations": 0.627, - "helm_mmlu/Security Studies": 0.784, - "helm_mmlu/Sociology": 0.841, - "helm_mmlu/Virology": 0.548, - "helm_mmlu/World Religions": 0.784, - "helm_mmlu/Mean win rate": 0.186 - } - }, - { - "id": "anthropic/claude-opus-4-1-20250805", - "name": "claude-opus-4-1-20250805", - "developer": "Anthropic", - "evaluator_relationship": null, - "benchmark_scores": { - "global-mmlu-lite/Global MMLU Lite": 0.943, - "global-mmlu-lite/Culturally Sensitive": 0.9331, - "global-mmlu-lite/Culturally Agnostic": 0.9528, - "global-mmlu-lite/Arabic": 0.945, - "global-mmlu-lite/English": 0.9475, - "global-mmlu-lite/Bengali": 0.9425, - "global-mmlu-lite/German": 0.94, - "global-mmlu-lite/French": 0.945, - "global-mmlu-lite/Hindi": 0.9475, - "global-mmlu-lite/Indonesian": 0.9425, - "global-mmlu-lite/Italian": 0.94, - "global-mmlu-lite/Japanese": 0.94, - "global-mmlu-lite/Korean": 0.95, - "global-mmlu-lite/Portuguese": 0.945, - "global-mmlu-lite/Spanish": 0.945, - "global-mmlu-lite/Swahili": 0.93, - "global-mmlu-lite/Yoruba": 0.9375, - "global-mmlu-lite/Chinese": 0.945, - "global-mmlu-lite/Burmese": 0.945 - } - }, - { - "id": "anthropic/claude-opus-4-20250514", - "name": "Claude 4 Opus 20250514", - "developer": "Anthropic", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.757, - "helm_capabilities/MMLU-Pro": 0.859, - "helm_capabilities/GPQA": 0.666, - "helm_capabilities/IFEval": 0.918, - "helm_capabilities/WildBench": 0.833, - "helm_capabilities/Omni-MATH": 0.511, - "reward-bench/Score": 0.7648, - "reward-bench/Factuality": 0.8267, - "reward-bench/Precise IF": 0.4188, - "reward-bench/Math": 0.7491, - "reward-bench/Safety": 0.8954, - "reward-bench/Focus": 0.8616, - "reward-bench/Ties": 0.8375 - } - }, - { - "id": "anthropic/claude-opus-4-20250514-thinking-10k", - "name": "Claude 4 Opus 20250514, extended thinking", - "developer": "Anthropic", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.78, - "helm_capabilities/MMLU-Pro": 0.875, - "helm_capabilities/GPQA": 0.709, - "helm_capabilities/IFEval": 0.849, - "helm_capabilities/WildBench": 0.852, - "helm_capabilities/Omni-MATH": 0.616 - } - }, - { - "id": "anthropic/claude-opus-4-5", - "name": "claude-opus-4-5", - "developer": "Anthropic", - "evaluator_relationship": null, - "benchmark_scores": { - "appworld_test_normal/appworld/test_normal": 0.68, - "browsecompplus/browsecompplus": 0.61, - "swe-bench/swe-bench": 0.65, - "tau-bench-2_airline/tau-bench-2/airline": 0.66, - "tau-bench-2_retail/tau-bench-2/retail": 0.78, - "tau-bench-2_telecom/tau-bench-2/telecom": 0.84 - } - }, - { - "id": "anthropic/claude-opus-4-5-20251101-fc", - "name": "Claude-Opus-4-5-20251101 (FC)", - "developer": "Anthropic", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 1.0, - "bfcl/bfcl.overall.overall_accuracy": 77.47, - "bfcl/bfcl.overall.total_cost_usd": 86.55, - "bfcl/bfcl.overall.latency_mean_s": 4.38, - "bfcl/bfcl.overall.latency_std_s": 3.13, - "bfcl/bfcl.overall.latency_p95_s": 7.56, - "bfcl/bfcl.non_live.ast_accuracy": 88.58, - "bfcl/bfcl.non_live.simple_ast_accuracy": 76.83, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 95.5, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 93.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 88.5, - "bfcl/bfcl.live.live_accuracy": 79.79, - "bfcl/bfcl.live.live_simple_ast_accuracy": 86.43, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 78.16, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 87.5, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 75.0, - "bfcl/bfcl.multi_turn.accuracy": 68.38, - "bfcl/bfcl.multi_turn.base_accuracy": 81.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 64.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 58.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 70.5, - "bfcl/bfcl.web_search.accuracy": 84.5, - "bfcl/bfcl.web_search.base_accuracy": 84.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 85.0, - "bfcl/bfcl.memory.accuracy": 73.76, - "bfcl/bfcl.memory.kv_accuracy": 70.97, - "bfcl/bfcl.memory.vector_accuracy": 72.9, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 77.42, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 62.5, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 84.72 - } - }, - { - "id": "anthropic/claude-opus-4-5-20251101-prompt", - "name": "Claude-Opus-4-5-20251101 (Prompt)", - "developer": "Anthropic", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 57.0, - "bfcl/bfcl.overall.overall_accuracy": 33.47, - "bfcl/bfcl.overall.total_cost_usd": 88.33, - "bfcl/bfcl.overall.latency_mean_s": 3.76, - "bfcl/bfcl.overall.latency_std_s": 13.19, - "bfcl/bfcl.overall.latency_p95_s": 5.52, - "bfcl/bfcl.non_live.ast_accuracy": 89.65, - "bfcl/bfcl.non_live.simple_ast_accuracy": 79.58, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 93.5, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 93.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 92.5, - "bfcl/bfcl.live.live_accuracy": 76.02, - "bfcl/bfcl.live.live_simple_ast_accuracy": 84.5, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 74.17, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 81.25, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 62.5, - "bfcl/bfcl.multi_turn.accuracy": 16.12, - "bfcl/bfcl.multi_turn.base_accuracy": 20.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 9.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 21.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 13.5, - "bfcl/bfcl.web_search.accuracy": 13.0, - "bfcl/bfcl.web_search.base_accuracy": 13.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 13.0, - "bfcl/bfcl.memory.accuracy": 1.94, - "bfcl/bfcl.memory.kv_accuracy": 1.29, - "bfcl/bfcl.memory.vector_accuracy": 1.94, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 2.58, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 68.75, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 90.75, - "bfcl/bfcl.format_sensitivity.max_delta": 13.0, - "bfcl/bfcl.format_sensitivity.stddev": 3.65 - } - }, - { - "id": "anthropic/claude-opus-4.1", - "name": "Claude Opus 4.1", - "developer": "Anthropic", - "evaluator_relationship": null, - "benchmark_scores": { - "terminal-bench-2.0/terminal-bench-2.0": 35.1 - } - }, - { - "id": "anthropic/claude-opus-4.5", - "name": "Claude Opus 4.5", - "developer": "Anthropic", - "evaluator_relationship": null, - "benchmark_scores": { - "terminal-bench-2.0/terminal-bench-2.0": 63.1 - } - }, - { - "id": "anthropic/claude-opus-4.6", - "name": "Claude Opus 4.6", - "developer": "Anthropic", - "evaluator_relationship": null, - "benchmark_scores": { - "terminal-bench-2.0/terminal-bench-2.0": 74.7 - } - }, - { - "id": "anthropic/claude-sonnet-4-20250514", - "name": "claude-sonnet-4-20250514", - "developer": "Anthropic", - "evaluator_relationship": null, - "benchmark_scores": { - "global-mmlu-lite/Global MMLU Lite": 0.9058, - "global-mmlu-lite/Culturally Sensitive": 0.8913, - "global-mmlu-lite/Culturally Agnostic": 0.9203, - "global-mmlu-lite/Arabic": 0.9125, - "global-mmlu-lite/English": 0.905, - "global-mmlu-lite/Bengali": 0.9075, - "global-mmlu-lite/German": 0.9125, - "global-mmlu-lite/French": 0.91, - "global-mmlu-lite/Hindi": 0.9, - "global-mmlu-lite/Indonesian": 0.9025, - "global-mmlu-lite/Italian": 0.9075, - "global-mmlu-lite/Japanese": 0.9, - "global-mmlu-lite/Korean": 0.9125, - "global-mmlu-lite/Portuguese": 0.91, - "global-mmlu-lite/Spanish": 0.9075, - "global-mmlu-lite/Swahili": 0.8975, - "global-mmlu-lite/Yoruba": 0.8975, - "global-mmlu-lite/Chinese": 0.9175, - "global-mmlu-lite/Burmese": 0.8925, - "helm_capabilities/Mean score": 0.733, - "helm_capabilities/MMLU-Pro": 0.843, - "helm_capabilities/GPQA": 0.643, - "helm_capabilities/IFEval": 0.839, - "helm_capabilities/WildBench": 0.825, - "helm_capabilities/Omni-MATH": 0.512, - "reward-bench/Score": 0.7117, - "reward-bench/Factuality": 0.7612, - "reward-bench/Precise IF": 0.3594, - "reward-bench/Math": 0.7049, - "reward-bench/Safety": 0.8909, - "reward-bench/Focus": 0.7596, - "reward-bench/Ties": 0.7939 - } - }, - { - "id": "anthropic/claude-sonnet-4-20250514-thinking-10k", - "name": "Claude 4 Sonnet 20250514, extended thinking", - "developer": "Anthropic", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.766, - "helm_capabilities/MMLU-Pro": 0.843, - "helm_capabilities/GPQA": 0.706, - "helm_capabilities/IFEval": 0.84, - "helm_capabilities/WildBench": 0.838, - "helm_capabilities/Omni-MATH": 0.602 - } - }, - { - "id": "anthropic/claude-sonnet-4-5-20250929", - "name": "claude-sonnet-4-5-20250929", - "developer": "Anthropic", - "evaluator_relationship": null, - "benchmark_scores": { - "livecodebenchpro/Hard Problems": 0.0, - "livecodebenchpro/Medium Problems": 0.0, - "livecodebenchpro/Easy Problems": 0.5352 - } - }, - { - "id": "anthropic/claude-sonnet-4-5-20250929-fc", - "name": "Claude-Sonnet-4-5-20250929 (FC)", - "developer": "Anthropic", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 2.0, - "bfcl/bfcl.overall.overall_accuracy": 73.24, - "bfcl/bfcl.overall.total_cost_usd": 43.73, - "bfcl/bfcl.overall.latency_mean_s": 4.31, - "bfcl/bfcl.overall.latency_std_s": 4.43, - "bfcl/bfcl.overall.latency_p95_s": 7.27, - "bfcl/bfcl.non_live.ast_accuracy": 88.65, - "bfcl/bfcl.non_live.simple_ast_accuracy": 72.58, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 95.5, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 94.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 92.0, - "bfcl/bfcl.live.live_accuracy": 81.13, - "bfcl/bfcl.live.live_simple_ast_accuracy": 89.53, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 78.92, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 87.5, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 83.33, - "bfcl/bfcl.multi_turn.accuracy": 61.37, - "bfcl/bfcl.multi_turn.base_accuracy": 69.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 65.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 52.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 59.0, - "bfcl/bfcl.web_search.accuracy": 81.0, - "bfcl/bfcl.web_search.base_accuracy": 82.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 80.0, - "bfcl/bfcl.memory.accuracy": 64.95, - "bfcl/bfcl.memory.kv_accuracy": 54.19, - "bfcl/bfcl.memory.vector_accuracy": 57.42, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 83.23, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 68.75, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 86.61 - } - }, - { - "id": "anthropic/claude-sonnet-4-5-20250929-prompt", - "name": "Claude-Sonnet-4-5-20250929 (Prompt)", - "developer": "Anthropic", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 89.0, - "bfcl/bfcl.overall.overall_accuracy": 24.9, - "bfcl/bfcl.overall.total_cost_usd": 47.82, - "bfcl/bfcl.overall.latency_mean_s": 3.84, - "bfcl/bfcl.overall.latency_std_s": 1.53, - "bfcl/bfcl.overall.latency_p95_s": 6.66, - "bfcl/bfcl.non_live.ast_accuracy": 59.81, - "bfcl/bfcl.non_live.simple_ast_accuracy": 47.25, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 79.5, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 53.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 59.0, - "bfcl/bfcl.live.live_accuracy": 46.56, - "bfcl/bfcl.live.live_simple_ast_accuracy": 73.26, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 40.17, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 56.25, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 33.33, - "bfcl/bfcl.multi_turn.accuracy": 1.62, - "bfcl/bfcl.multi_turn.base_accuracy": 2.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 0.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 3.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 1.5, - "bfcl/bfcl.web_search.accuracy": 16.0, - "bfcl/bfcl.web_search.base_accuracy": 16.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 16.0, - "bfcl/bfcl.memory.accuracy": 5.38, - "bfcl/bfcl.memory.kv_accuracy": 4.52, - "bfcl/bfcl.memory.vector_accuracy": 9.68, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 1.94, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 37.5, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 95.03, - "bfcl/bfcl.format_sensitivity.max_delta": 37.5, - "bfcl/bfcl.format_sensitivity.stddev": 10.07 - } - }, - { - "id": "anthropic/claude-sonnet-4.5", - "name": "Claude Sonnet 4.5", - "developer": "Anthropic", - "evaluator_relationship": null, - "benchmark_scores": { - "terminal-bench-2.0/terminal-bench-2.0": 42.5 - } - }, - { - "id": "anthropic/claude-v1.3", - "name": "Anthropic Claude v1.3", - "developer": "Anthropic", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_instruct/Mean win rate": 0.611, - "helm_instruct/Anthropic RLHF dataset": 4.965, - "helm_instruct/Best ChatGPT Prompts": 4.995, - "helm_instruct/Koala test dataset": 4.981, - "helm_instruct/Open Assistant": 4.975, - "helm_instruct/Self Instruct": 4.992, - "helm_instruct/Vicuna": 4.989, - "helm_lite/Mean win rate": 0.518, - "helm_lite/NarrativeQA": 0.723, - "helm_lite/NaturalQuestions (closed-book)": 0.409, - "helm_lite/OpenbookQA": 0.908, - "helm_lite/MMLU": 0.631, - "helm_lite/MATH": 0.54, - "helm_lite/GSM8K": 0.784, - "helm_lite/LegalBench": 0.629, - "helm_lite/MedQA": 0.618, - "helm_lite/WMT 2014": 0.219 - } - }, - { - "id": "anthropic/Opus 4.1", - "name": "Opus 4.1", - "developer": "Anthropic", - "evaluator_relationship": null, - "benchmark_scores": { - "ace/Overall Score": 0.4, - "ace/Gaming Score": 0.318 - } - }, - { - "id": "anthropic/Opus 4.5", - "name": "Opus 4.5", - "developer": "Anthropic", - "evaluator_relationship": null, - "benchmark_scores": { - "ace/Overall Score": 0.478, - "ace/Gaming Score": 0.391, - "apex-agents/Overall Pass@1": 0.184, - "apex-agents/Overall Pass@8": 0.34, - "apex-agents/Overall Mean Score": 0.348, - "apex-agents/Investment Banking Pass@1": 0.216, - "apex-agents/Management Consulting Pass@1": 0.132, - "apex-agents/Corporate Law Pass@1": 0.202, - "apex-agents/Corporate Lawyer Mean Score": 0.471, - "apex-v1/Medicine (MD) Score": 0.65 - } - }, - { - "id": "anthropic/Opus 4.6", - "name": "Opus 4.6", - "developer": "Anthropic", - "evaluator_relationship": null, - "benchmark_scores": { - "apex-agents/Overall Pass@1": 0.298, - "apex-agents/Corporate Lawyer Mean Score": 0.502 - } - }, - { - "id": "anthropic/Sonnet 4.5", - "name": "Sonnet 4.5", - "developer": "Anthropic", - "evaluator_relationship": null, - "benchmark_scores": { - "ace/Overall Score": 0.44, - "ace/Gaming Score": 0.373 - } - }, - { - "id": "apple/DCLM-7B", - "name": "DCLM-7B", - "developer": "apple", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2173, - "hfopenllm_v2/BBH": 0.4232, - "hfopenllm_v2/MATH Level 5": 0.037, - "hfopenllm_v2/GPQA": 0.3154, - "hfopenllm_v2/MUSR": 0.3921, - "hfopenllm_v2/MMLU-PRO": 0.3111 - } - }, - { - "id": "applied-compute/Applied Compute: Small", - "name": "Applied Compute: Small", - "developer": "applied-compute", - "evaluator_relationship": null, - "benchmark_scores": { - "apex-agents/Overall Pass@1": 0.23, - "apex-agents/Overall Mean Score": 0.401, - "apex-agents/Corporate Law Pass@1": 0.266, - "apex-agents/Corporate Lawyer Mean Score": 0.548 - } - }, - { - "id": "appvoid/arco-2", - "name": "arco-2", - "developer": "appvoid", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1991, - "hfopenllm_v2/BBH": 0.3146, - "hfopenllm_v2/MATH Level 5": 0.0136, - "hfopenllm_v2/GPQA": 0.2391, - "hfopenllm_v2/MUSR": 0.3536, - "hfopenllm_v2/MMLU-PRO": 0.1116 - } - }, - { - "id": "appvoid/arco-2-instruct", - "name": "arco-2-instruct", - "developer": "appvoid", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2164, - "hfopenllm_v2/BBH": 0.3133, - "hfopenllm_v2/MATH Level 5": 0.0128, - "hfopenllm_v2/GPQA": 0.2383, - "hfopenllm_v2/MUSR": 0.3496, - "hfopenllm_v2/MMLU-PRO": 0.1113 - } - }, - { - "id": "arcee-ai/Arcee-Blitz", - "name": "Arcee-Blitz", - "developer": "arcee-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5543, - "hfopenllm_v2/BBH": 0.6607, - "hfopenllm_v2/MATH Level 5": 0.3482, - "hfopenllm_v2/GPQA": 0.3851, - "hfopenllm_v2/MUSR": 0.5047, - "hfopenllm_v2/MMLU-PRO": 0.6154 - } - }, - { - "id": "arcee-ai/Arcee-Maestro-7B-Preview", - "name": "Arcee-Maestro-7B-Preview", - "developer": "arcee-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.275, - "hfopenllm_v2/BBH": 0.4648, - "hfopenllm_v2/MATH Level 5": 0.4992, - "hfopenllm_v2/GPQA": 0.3322, - "hfopenllm_v2/MUSR": 0.3885, - "hfopenllm_v2/MMLU-PRO": 0.3039 - } - }, - { - "id": "arcee-ai/Arcee-Nova", - "name": "Arcee-Nova", - "developer": "arcee-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7907, - "hfopenllm_v2/BBH": 0.6942, - "hfopenllm_v2/MATH Level 5": 0.4381, - "hfopenllm_v2/GPQA": 0.3851, - "hfopenllm_v2/MUSR": 0.4562, - "hfopenllm_v2/MMLU-PRO": 0.5452 - } - }, - { - "id": "arcee-ai/Arcee-Spark", - "name": "Arcee-Spark", - "developer": "arcee-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5718, - "hfopenllm_v2/BBH": 0.5481, - "hfopenllm_v2/MATH Level 5": 0.114, - "hfopenllm_v2/GPQA": 0.3062, - "hfopenllm_v2/MUSR": 0.4008, - "hfopenllm_v2/MMLU-PRO": 0.3813 - } - }, - { - "id": "arcee-ai/Llama-3.1-SuperNova-Lite", - "name": "Llama-3.1-SuperNova-Lite", - "developer": "arcee-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8017, - "hfopenllm_v2/BBH": 0.5152, - "hfopenllm_v2/MATH Level 5": 0.1828, - "hfopenllm_v2/GPQA": 0.3062, - "hfopenllm_v2/MUSR": 0.4163, - "hfopenllm_v2/MMLU-PRO": 0.3877 - } - }, - { - "id": "arcee-ai/Llama-Spark", - "name": "Llama-Spark", - "developer": "arcee-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7911, - "hfopenllm_v2/BBH": 0.5054, - "hfopenllm_v2/MATH Level 5": 0.139, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.3593, - "hfopenllm_v2/MMLU-PRO": 0.3721 - } - }, - { - "id": "arcee-ai/raspberry-3B", - "name": "raspberry-3B", - "developer": "arcee-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3154, - "hfopenllm_v2/BBH": 0.4269, - "hfopenllm_v2/MATH Level 5": 0.1035, - "hfopenllm_v2/GPQA": 0.2777, - "hfopenllm_v2/MUSR": 0.4123, - "hfopenllm_v2/MMLU-PRO": 0.2854 - } - }, - { - "id": "arcee-ai/SuperNova-Medius", - "name": "SuperNova-Medius", - "developer": "arcee-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7184, - "hfopenllm_v2/BBH": 0.6377, - "hfopenllm_v2/MATH Level 5": 0.469, - "hfopenllm_v2/GPQA": 0.3331, - "hfopenllm_v2/MUSR": 0.4233, - "hfopenllm_v2/MMLU-PRO": 0.5035 - } - }, - { - "id": "arcee-ai/Virtuoso-Lite", - "name": "Virtuoso-Lite", - "developer": "arcee-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.81, - "hfopenllm_v2/BBH": 0.6099, - "hfopenllm_v2/MATH Level 5": 0.253, - "hfopenllm_v2/GPQA": 0.344, - "hfopenllm_v2/MUSR": 0.4595, - "hfopenllm_v2/MMLU-PRO": 0.4441 - } - }, - { - "id": "arcee-ai/Virtuoso-Small", - "name": "Virtuoso-Small", - "developer": "arcee-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7935, - "hfopenllm_v2/BBH": 0.6518, - "hfopenllm_v2/MATH Level 5": 0.4094, - "hfopenllm_v2/GPQA": 0.3364, - "hfopenllm_v2/MUSR": 0.4339, - "hfopenllm_v2/MMLU-PRO": 0.5191 - } - }, - { - "id": "arcee-ai/Virtuoso-Small-v2", - "name": "Virtuoso-Small-v2", - "developer": "arcee-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8273, - "hfopenllm_v2/BBH": 0.6554, - "hfopenllm_v2/MATH Level 5": 0.466, - "hfopenllm_v2/GPQA": 0.3532, - "hfopenllm_v2/MUSR": 0.4313, - "hfopenllm_v2/MMLU-PRO": 0.5188 - } - }, - { - "id": "argilla-warehouse/Llama-3.1-8B-MagPie-Ultra", - "name": "Llama-3.1-8B-MagPie-Ultra", - "developer": "argilla-warehouse", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5757, - "hfopenllm_v2/BBH": 0.462, - "hfopenllm_v2/MATH Level 5": 0.077, - "hfopenllm_v2/GPQA": 0.2668, - "hfopenllm_v2/MUSR": 0.3543, - "hfopenllm_v2/MMLU-PRO": 0.3144 - } - }, - { - "id": "argilla/notus-7b-v1", - "name": "notus-7b-v1", - "developer": "argilla", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5082, - "hfopenllm_v2/BBH": 0.4512, - "hfopenllm_v2/MATH Level 5": 0.0317, - "hfopenllm_v2/GPQA": 0.2894, - "hfopenllm_v2/MUSR": 0.3364, - "hfopenllm_v2/MMLU-PRO": 0.3004 - } - }, - { - "id": "argilla/notux-8x7b-v1", - "name": "notux-8x7b-v1", - "developer": "argilla", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5422, - "hfopenllm_v2/BBH": 0.5363, - "hfopenllm_v2/MATH Level 5": 0.0997, - "hfopenllm_v2/GPQA": 0.3087, - "hfopenllm_v2/MUSR": 0.4176, - "hfopenllm_v2/MMLU-PRO": 0.366 - } - }, - { - "id": "arisin/orca-platypus-13B-slerp", - "name": "orca-platypus-13B-slerp", - "developer": "arisin", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2672, - "hfopenllm_v2/BBH": 0.4631, - "hfopenllm_v2/MATH Level 5": 0.0159, - "hfopenllm_v2/GPQA": 0.2987, - "hfopenllm_v2/MUSR": 0.4253, - "hfopenllm_v2/MMLU-PRO": 0.2592 - } - }, - { - "id": "ark/ep-20250603132404-cgpjm", - "name": "ep-20250603132404-cgpjm", - "developer": "ark", - "evaluator_relationship": null, - "benchmark_scores": { - "livecodebenchpro/Hard Problems": 0.0, - "livecodebenchpro/Medium Problems": 0.0141, - "livecodebenchpro/Easy Problems": 0.507 - } - }, - { - "id": "ArliAI/ArliAI-RPMax-12B-v1.1", - "name": "ArliAI-RPMax-12B-v1.1", - "developer": "ArliAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5349, - "hfopenllm_v2/BBH": 0.4752, - "hfopenllm_v2/MATH Level 5": 0.1125, - "hfopenllm_v2/GPQA": 0.2819, - "hfopenllm_v2/MUSR": 0.3618, - "hfopenllm_v2/MMLU-PRO": 0.3384 - } - }, - { - "id": "ArliAI/Llama-3.1-8B-ArliAI-RPMax-v1.1", - "name": "Llama-3.1-8B-ArliAI-RPMax-v1.1", - "developer": "ArliAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6359, - "hfopenllm_v2/BBH": 0.5016, - "hfopenllm_v2/MATH Level 5": 0.1314, - "hfopenllm_v2/GPQA": 0.2836, - "hfopenllm_v2/MUSR": 0.3577, - "hfopenllm_v2/MMLU-PRO": 0.3551 - } - }, - { - "id": "arshiaafshani/Arsh-V1", - "name": "Arsh-V1", - "developer": "arshiaafshani", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6043, - "hfopenllm_v2/BBH": 0.674, - "hfopenllm_v2/MATH Level 5": 0.2621, - "hfopenllm_v2/GPQA": 0.3733, - "hfopenllm_v2/MUSR": 0.4899, - "hfopenllm_v2/MMLU-PRO": 0.5257 - } - }, - { - "id": "Arthur-LAGACHERIE/Precis-1B-Instruct", - "name": "Precis-1B-Instruct", - "developer": "Arthur-LAGACHERIE", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3671, - "hfopenllm_v2/BBH": 0.3224, - "hfopenllm_v2/MATH Level 5": 0.0038, - "hfopenllm_v2/GPQA": 0.2659, - "hfopenllm_v2/MUSR": 0.3436, - "hfopenllm_v2/MMLU-PRO": 0.1426 - } - }, - { - "id": "Artples/L-MChat-7b", - "name": "L-MChat-7b", - "developer": "Artples", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5297, - "hfopenllm_v2/BBH": 0.46, - "hfopenllm_v2/MATH Level 5": 0.0921, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.4029, - "hfopenllm_v2/MMLU-PRO": 0.3299 - } - }, - { - "id": "Artples/L-MChat-Small", - "name": "L-MChat-Small", - "developer": "Artples", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3287, - "hfopenllm_v2/BBH": 0.4823, - "hfopenllm_v2/MATH Level 5": 0.0378, - "hfopenllm_v2/GPQA": 0.2676, - "hfopenllm_v2/MUSR": 0.3696, - "hfopenllm_v2/MMLU-PRO": 0.2464 - } - }, - { - "id": "Aryanne/QwentileSwap", - "name": "QwentileSwap", - "developer": "Aryanne", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7378, - "hfopenllm_v2/BBH": 0.7008, - "hfopenllm_v2/MATH Level 5": 0.4222, - "hfopenllm_v2/GPQA": 0.3674, - "hfopenllm_v2/MUSR": 0.464, - "hfopenllm_v2/MMLU-PRO": 0.5946 - } - }, - { - "id": "Aryanne/SHBA", - "name": "SHBA", - "developer": "Aryanne", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7817, - "hfopenllm_v2/BBH": 0.5233, - "hfopenllm_v2/MATH Level 5": 0.1798, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.4161, - "hfopenllm_v2/MMLU-PRO": 0.3892 - } - }, - { - "id": "Aryanne/SuperHeart", - "name": "SuperHeart", - "developer": "Aryanne", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5192, - "hfopenllm_v2/BBH": 0.5215, - "hfopenllm_v2/MATH Level 5": 0.1563, - "hfopenllm_v2/GPQA": 0.3012, - "hfopenllm_v2/MUSR": 0.4436, - "hfopenllm_v2/MMLU-PRO": 0.3912 - } - }, - { - "id": "asharsha30/LLAMA_Harsha_8_B_ORDP_10k", - "name": "LLAMA_Harsha_8_B_ORDP_10k", - "developer": "asharsha30", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3464, - "hfopenllm_v2/BBH": 0.4669, - "hfopenllm_v2/MATH Level 5": 0.0665, - "hfopenllm_v2/GPQA": 0.2735, - "hfopenllm_v2/MUSR": 0.3697, - "hfopenllm_v2/MMLU-PRO": 0.281 - } - }, - { - "id": "ashercn97/a1-v0.0.1", - "name": "a1-v0.0.1", - "developer": "ashercn97", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2198, - "hfopenllm_v2/BBH": 0.5188, - "hfopenllm_v2/MATH Level 5": 0.2145, - "hfopenllm_v2/GPQA": 0.3112, - "hfopenllm_v2/MUSR": 0.412, - "hfopenllm_v2/MMLU-PRO": 0.4165 - } - }, - { - "id": "ashercn97/a1-v002", - "name": "a1-v002", - "developer": "ashercn97", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2585, - "hfopenllm_v2/BBH": 0.5261, - "hfopenllm_v2/MATH Level 5": 0.2341, - "hfopenllm_v2/GPQA": 0.3188, - "hfopenllm_v2/MUSR": 0.4159, - "hfopenllm_v2/MMLU-PRO": 0.4175 - } - }, - { - "id": "assskelad/smollm2-360M-sft_SmallThoughts", - "name": "smollm2-360M-sft_SmallThoughts", - "developer": "assskelad", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2007, - "hfopenllm_v2/BBH": 0.315, - "hfopenllm_v2/MATH Level 5": 0.0166, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3395, - "hfopenllm_v2/MMLU-PRO": 0.1182 - } - }, - { - "id": "AtAndDev/Qwen2.5-1.5B-continuous-learnt", - "name": "Qwen2.5-1.5B-continuous-learnt", - "developer": "AtAndDev", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4511, - "hfopenllm_v2/BBH": 0.4275, - "hfopenllm_v2/MATH Level 5": 0.1473, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.3623, - "hfopenllm_v2/MMLU-PRO": 0.2806 - } - }, - { - "id": "Ateron/Glowing-Forest-12B", - "name": "Glowing-Forest-12B", - "developer": "Ateron", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3592, - "hfopenllm_v2/BBH": 0.5492, - "hfopenllm_v2/MATH Level 5": 0.0778, - "hfopenllm_v2/GPQA": 0.3331, - "hfopenllm_v2/MUSR": 0.4449, - "hfopenllm_v2/MMLU-PRO": 0.3718 - } - }, - { - "id": "Ateron/Lotus-Magpic", - "name": "Lotus-Magpic", - "developer": "Ateron", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6286, - "hfopenllm_v2/BBH": 0.5254, - "hfopenllm_v2/MATH Level 5": 0.0997, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.4332, - "hfopenllm_v2/MMLU-PRO": 0.3491 - } - }, - { - "id": "Ateron/Way_of_MagPicaro", - "name": "Way_of_MagPicaro", - "developer": "Ateron", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2637, - "hfopenllm_v2/BBH": 0.5427, - "hfopenllm_v2/MATH Level 5": 0.0589, - "hfopenllm_v2/GPQA": 0.3339, - "hfopenllm_v2/MUSR": 0.4649, - "hfopenllm_v2/MMLU-PRO": 0.3536 - } - }, - { - "id": "athirdpath/Llama-3.1-Instruct_NSFW-pretrained_e1-plus_reddit", - "name": "Llama-3.1-Instruct_NSFW-pretrained_e1-plus_reddit", - "developer": "athirdpath", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4521, - "hfopenllm_v2/BBH": 0.4939, - "hfopenllm_v2/MATH Level 5": 0.102, - "hfopenllm_v2/GPQA": 0.2919, - "hfopenllm_v2/MUSR": 0.3864, - "hfopenllm_v2/MMLU-PRO": 0.3565 - } - }, - { - "id": "AtlaAI/Selene-1", - "name": "AtlaAI/Selene-1", - "developer": "AtlaAI", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.9241, - "reward-bench/Chat": 0.9777, - "reward-bench/Chat Hard": 0.8399, - "reward-bench/Safety": 0.9216, - "reward-bench/Reasoning": 0.9572 - } - }, - { - "id": "AtlaAI/Selene-1-Mini-Llama-3.1-8B", - "name": "AtlaAI/Selene-1-Mini-Llama-3.1-8B", - "developer": "AtlaAI", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.8913, - "reward-bench/Chat": 0.9358, - "reward-bench/Chat Hard": 0.7939, - "reward-bench/Safety": 0.8926, - "reward-bench/Reasoning": 0.9429 - } - }, - { - "id": "AuraIndustries/Aura-4B", - "name": "Aura-4B", - "developer": "AuraIndustries", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3816, - "hfopenllm_v2/BBH": 0.449, - "hfopenllm_v2/MATH Level 5": 0.0423, - "hfopenllm_v2/GPQA": 0.2878, - "hfopenllm_v2/MUSR": 0.3938, - "hfopenllm_v2/MMLU-PRO": 0.2706 - } - }, - { - "id": "AuraIndustries/Aura-8B", - "name": "Aura-8B", - "developer": "AuraIndustries", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7205, - "hfopenllm_v2/BBH": 0.5131, - "hfopenllm_v2/MATH Level 5": 0.1518, - "hfopenllm_v2/GPQA": 0.2861, - "hfopenllm_v2/MUSR": 0.4004, - "hfopenllm_v2/MMLU-PRO": 0.3874 - } - }, - { - "id": "AuraIndustries/Aura-MoE-2x4B", - "name": "Aura-MoE-2x4B", - "developer": "AuraIndustries", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4601, - "hfopenllm_v2/BBH": 0.4339, - "hfopenllm_v2/MATH Level 5": 0.031, - "hfopenllm_v2/GPQA": 0.2718, - "hfopenllm_v2/MUSR": 0.4085, - "hfopenllm_v2/MMLU-PRO": 0.265 - } - }, - { - "id": "AuraIndustries/Aura-MoE-2x4B-v2", - "name": "Aura-MoE-2x4B-v2", - "developer": "AuraIndustries", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4778, - "hfopenllm_v2/BBH": 0.4315, - "hfopenllm_v2/MATH Level 5": 0.0317, - "hfopenllm_v2/GPQA": 0.2878, - "hfopenllm_v2/MUSR": 0.4101, - "hfopenllm_v2/MMLU-PRO": 0.261 - } - }, - { - "id": "Aurel9/testmerge-7b", - "name": "testmerge-7b", - "developer": "Aurel9", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.398, - "hfopenllm_v2/BBH": 0.519, - "hfopenllm_v2/MATH Level 5": 0.0657, - "hfopenllm_v2/GPQA": 0.3003, - "hfopenllm_v2/MUSR": 0.4659, - "hfopenllm_v2/MMLU-PRO": 0.3053 - } - }, - { - "id": "automerger/YamshadowExperiment28-7B", - "name": "YamshadowExperiment28-7B", - "developer": "automerger", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.407, - "hfopenllm_v2/BBH": 0.515, - "hfopenllm_v2/MATH Level 5": 0.0612, - "hfopenllm_v2/GPQA": 0.2869, - "hfopenllm_v2/MUSR": 0.4306, - "hfopenllm_v2/MMLU-PRO": 0.306 - } - }, - { - "id": "avemio/GRAG-NEMO-12B-ORPO-HESSIAN-AI", - "name": "GRAG-NEMO-12B-ORPO-HESSIAN-AI", - "developer": "avemio", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0, - "hfopenllm_v2/BBH": 0.2607, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3447, - "hfopenllm_v2/MMLU-PRO": 0.1061 - } - }, - { - "id": "awnr/Mistral-7B-v0.1-signtensors-1-over-2", - "name": "Mistral-7B-v0.1-signtensors-1-over-2", - "developer": "awnr", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2179, - "hfopenllm_v2/BBH": 0.4423, - "hfopenllm_v2/MATH Level 5": 0.034, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.4006, - "hfopenllm_v2/MMLU-PRO": 0.3 - } - }, - { - "id": "awnr/Mistral-7B-v0.1-signtensors-1-over-4", - "name": "Mistral-7B-v0.1-signtensors-1-over-4", - "developer": "awnr", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2133, - "hfopenllm_v2/BBH": 0.3507, - "hfopenllm_v2/MATH Level 5": 0.0249, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.346, - "hfopenllm_v2/MMLU-PRO": 0.2311 - } - }, - { - "id": "awnr/Mistral-7B-v0.1-signtensors-3-over-8", - "name": "Mistral-7B-v0.1-signtensors-3-over-8", - "developer": "awnr", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2394, - "hfopenllm_v2/BBH": 0.43, - "hfopenllm_v2/MATH Level 5": 0.0332, - "hfopenllm_v2/GPQA": 0.3037, - "hfopenllm_v2/MUSR": 0.3818, - "hfopenllm_v2/MMLU-PRO": 0.3001 - } - }, - { - "id": "awnr/Mistral-7B-v0.1-signtensors-5-over-16", - "name": "Mistral-7B-v0.1-signtensors-5-over-16", - "developer": "awnr", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2118, - "hfopenllm_v2/BBH": 0.4124, - "hfopenllm_v2/MATH Level 5": 0.0295, - "hfopenllm_v2/GPQA": 0.281, - "hfopenllm_v2/MUSR": 0.3686, - "hfopenllm_v2/MMLU-PRO": 0.2958 - } - }, - { - "id": "awnr/Mistral-7B-v0.1-signtensors-7-over-16", - "name": "Mistral-7B-v0.1-signtensors-7-over-16", - "developer": "awnr", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2294, - "hfopenllm_v2/BBH": 0.4316, - "hfopenllm_v2/MATH Level 5": 0.0385, - "hfopenllm_v2/GPQA": 0.3037, - "hfopenllm_v2/MUSR": 0.3952, - "hfopenllm_v2/MMLU-PRO": 0.303 - } - }, - { - "id": "aws-prototyping/MegaBeam-Mistral-7B-512k", - "name": "MegaBeam-Mistral-7B-512k", - "developer": "aws-prototyping", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5973, - "hfopenllm_v2/BBH": 0.3662, - "hfopenllm_v2/MATH Level 5": 0.0287, - "hfopenllm_v2/GPQA": 0.2827, - "hfopenllm_v2/MUSR": 0.3994, - "hfopenllm_v2/MMLU-PRO": 0.2589 - } - }, - { - "id": "axolotl-ai-co/romulus-mistral-nemo-12b-simpo", - "name": "romulus-mistral-nemo-12b-simpo", - "developer": "axolotl-ai-co", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6079, - "hfopenllm_v2/BBH": 0.5395, - "hfopenllm_v2/MATH Level 5": 0.114, - "hfopenllm_v2/GPQA": 0.2785, - "hfopenllm_v2/MUSR": 0.4233, - "hfopenllm_v2/MMLU-PRO": 0.3469 - } - }, - { - "id": "Ayush-Singh/Llama1B-sft-2", - "name": "Llama1B-sft-2", - "developer": "Ayush-Singh", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1374, - "hfopenllm_v2/BBH": 0.2834, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2458, - "hfopenllm_v2/MUSR": 0.3552, - "hfopenllm_v2/MMLU-PRO": 0.1117 - } - }, - { - "id": "Azure99/blossom-v5-32b", - "name": "blossom-v5-32b", - "developer": "Azure99", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5235, - "hfopenllm_v2/BBH": 0.5955, - "hfopenllm_v2/MATH Level 5": 0.1866, - "hfopenllm_v2/GPQA": 0.3112, - "hfopenllm_v2/MUSR": 0.402, - "hfopenllm_v2/MMLU-PRO": 0.4235 - } - }, - { - "id": "Azure99/blossom-v5-llama3-8b", - "name": "blossom-v5-llama3-8b", - "developer": "Azure99", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4343, - "hfopenllm_v2/BBH": 0.4185, - "hfopenllm_v2/MATH Level 5": 0.0514, - "hfopenllm_v2/GPQA": 0.2651, - "hfopenllm_v2/MUSR": 0.367, - "hfopenllm_v2/MMLU-PRO": 0.2206 - } - }, - { - "id": "Azure99/blossom-v5.1-34b", - "name": "blossom-v5.1-34b", - "developer": "Azure99", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5697, - "hfopenllm_v2/BBH": 0.6109, - "hfopenllm_v2/MATH Level 5": 0.2591, - "hfopenllm_v2/GPQA": 0.3096, - "hfopenllm_v2/MUSR": 0.3928, - "hfopenllm_v2/MMLU-PRO": 0.4558 - } - }, - { - "id": "Azure99/blossom-v5.1-9b", - "name": "blossom-v5.1-9b", - "developer": "Azure99", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5086, - "hfopenllm_v2/BBH": 0.5343, - "hfopenllm_v2/MATH Level 5": 0.2122, - "hfopenllm_v2/GPQA": 0.3356, - "hfopenllm_v2/MUSR": 0.3994, - "hfopenllm_v2/MMLU-PRO": 0.3979 - } - }, - { - "id": "Azure99/Blossom-V6-14B", - "name": "Blossom-V6-14B", - "developer": "Azure99", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6395, - "hfopenllm_v2/BBH": 0.5069, - "hfopenllm_v2/MATH Level 5": 0.5257, - "hfopenllm_v2/GPQA": 0.2626, - "hfopenllm_v2/MUSR": 0.4035, - "hfopenllm_v2/MMLU-PRO": 0.4544 - } - }, - { - "id": "Azure99/Blossom-V6-7B", - "name": "Blossom-V6-7B", - "developer": "Azure99", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5538, - "hfopenllm_v2/BBH": 0.4974, - "hfopenllm_v2/MATH Level 5": 0.4585, - "hfopenllm_v2/GPQA": 0.3045, - "hfopenllm_v2/MUSR": 0.4301, - "hfopenllm_v2/MMLU-PRO": 0.4144 - } - }, - { - "id": "Ba2han/Llama-Phi-3_DoRA", - "name": "Llama-Phi-3_DoRA", - "developer": "Ba2han", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5131, - "hfopenllm_v2/BBH": 0.5515, - "hfopenllm_v2/MATH Level 5": 0.1216, - "hfopenllm_v2/GPQA": 0.3263, - "hfopenllm_v2/MUSR": 0.4069, - "hfopenllm_v2/MMLU-PRO": 0.3915 - } - }, - { - "id": "BAAI/Gemma2-9B-IT-Simpo-Infinity-Preference", - "name": "Gemma2-9B-IT-Simpo-Infinity-Preference", - "developer": "BAAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3176, - "hfopenllm_v2/BBH": 0.5979, - "hfopenllm_v2/MATH Level 5": 0.0974, - "hfopenllm_v2/GPQA": 0.3398, - "hfopenllm_v2/MUSR": 0.3966, - "hfopenllm_v2/MMLU-PRO": 0.3869 - } - }, - { - "id": "BAAI/Infinity-Instruct-3M-0613-Llama3-70B", - "name": "Infinity-Instruct-3M-0613-Llama3-70B", - "developer": "BAAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6821, - "hfopenllm_v2/BBH": 0.6642, - "hfopenllm_v2/MATH Level 5": 0.2153, - "hfopenllm_v2/GPQA": 0.3582, - "hfopenllm_v2/MUSR": 0.4523, - "hfopenllm_v2/MMLU-PRO": 0.473 - } - }, - { - "id": "BAAI/Infinity-Instruct-3M-0613-Mistral-7B", - "name": "Infinity-Instruct-3M-0613-Mistral-7B", - "developer": "BAAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.532, - "hfopenllm_v2/BBH": 0.4958, - "hfopenllm_v2/MATH Level 5": 0.0816, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.4351, - "hfopenllm_v2/MMLU-PRO": 0.3161 - } - }, - { - "id": "BAAI/Infinity-Instruct-3M-0625-Llama3-70B", - "name": "Infinity-Instruct-3M-0625-Llama3-70B", - "developer": "BAAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7442, - "hfopenllm_v2/BBH": 0.667, - "hfopenllm_v2/MATH Level 5": 0.2251, - "hfopenllm_v2/GPQA": 0.3574, - "hfopenllm_v2/MUSR": 0.4617, - "hfopenllm_v2/MMLU-PRO": 0.4586 - } - }, - { - "id": "BAAI/Infinity-Instruct-3M-0625-Llama3-8B", - "name": "Infinity-Instruct-3M-0625-Llama3-8B", - "developer": "BAAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.605, - "hfopenllm_v2/BBH": 0.4955, - "hfopenllm_v2/MATH Level 5": 0.0884, - "hfopenllm_v2/GPQA": 0.2752, - "hfopenllm_v2/MUSR": 0.3712, - "hfopenllm_v2/MMLU-PRO": 0.3252 - } - }, - { - "id": "BAAI/Infinity-Instruct-3M-0625-Mistral-7B", - "name": "Infinity-Instruct-3M-0625-Mistral-7B", - "developer": "BAAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5867, - "hfopenllm_v2/BBH": 0.494, - "hfopenllm_v2/MATH Level 5": 0.0763, - "hfopenllm_v2/GPQA": 0.2869, - "hfopenllm_v2/MUSR": 0.4272, - "hfopenllm_v2/MMLU-PRO": 0.323 - } - }, - { - "id": "BAAI/Infinity-Instruct-3M-0625-Qwen2-7B", - "name": "Infinity-Instruct-3M-0625-Qwen2-7B", - "developer": "BAAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5554, - "hfopenllm_v2/BBH": 0.5346, - "hfopenllm_v2/MATH Level 5": 0.1926, - "hfopenllm_v2/GPQA": 0.3129, - "hfopenllm_v2/MUSR": 0.3888, - "hfopenllm_v2/MMLU-PRO": 0.396 - } - }, - { - "id": "BAAI/Infinity-Instruct-3M-0625-Yi-1.5-9B", - "name": "Infinity-Instruct-3M-0625-Yi-1.5-9B", - "developer": "BAAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5186, - "hfopenllm_v2/BBH": 0.5509, - "hfopenllm_v2/MATH Level 5": 0.1639, - "hfopenllm_v2/GPQA": 0.354, - "hfopenllm_v2/MUSR": 0.4575, - "hfopenllm_v2/MMLU-PRO": 0.4118 - } - }, - { - "id": "BAAI/Infinity-Instruct-7M-0729-Llama3_1-8B", - "name": "Infinity-Instruct-7M-0729-Llama3_1-8B", - "developer": "BAAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6132, - "hfopenllm_v2/BBH": 0.5077, - "hfopenllm_v2/MATH Level 5": 0.1276, - "hfopenllm_v2/GPQA": 0.2928, - "hfopenllm_v2/MUSR": 0.3578, - "hfopenllm_v2/MMLU-PRO": 0.3224 - } - }, - { - "id": "BAAI/Infinity-Instruct-7M-0729-mistral-7B", - "name": "Infinity-Instruct-7M-0729-mistral-7B", - "developer": "BAAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6162, - "hfopenllm_v2/BBH": 0.4964, - "hfopenllm_v2/MATH Level 5": 0.0831, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.4062, - "hfopenllm_v2/MMLU-PRO": 0.3274 - } - }, - { - "id": "BAAI/Infinity-Instruct-7M-Gen-Llama3_1-70B", - "name": "Infinity-Instruct-7M-Gen-Llama3_1-70B", - "developer": "BAAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7335, - "hfopenllm_v2/BBH": 0.6695, - "hfopenllm_v2/MATH Level 5": 0.2523, - "hfopenllm_v2/GPQA": 0.3758, - "hfopenllm_v2/MUSR": 0.4539, - "hfopenllm_v2/MMLU-PRO": 0.4607 - } - }, - { - "id": "BAAI/Infinity-Instruct-7M-Gen-Llama3_1-8B", - "name": "Infinity-Instruct-7M-Gen-Llama3_1-8B", - "developer": "BAAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6132, - "hfopenllm_v2/BBH": 0.5077, - "hfopenllm_v2/MATH Level 5": 0.1276, - "hfopenllm_v2/GPQA": 0.2928, - "hfopenllm_v2/MUSR": 0.3578, - "hfopenllm_v2/MMLU-PRO": 0.3224 - } - }, - { - "id": "BAAI/Infinity-Instruct-7M-Gen-mistral-7B", - "name": "Infinity-Instruct-7M-Gen-mistral-7B", - "developer": "BAAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6147, - "hfopenllm_v2/BBH": 0.4964, - "hfopenllm_v2/MATH Level 5": 0.0831, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.4062, - "hfopenllm_v2/MMLU-PRO": 0.3274 - } - }, - { - "id": "BAAI/OPI-Llama-3.1-8B-Instruct", - "name": "OPI-Llama-3.1-8B-Instruct", - "developer": "BAAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2075, - "hfopenllm_v2/BBH": 0.3551, - "hfopenllm_v2/MATH Level 5": 0.0136, - "hfopenllm_v2/GPQA": 0.2743, - "hfopenllm_v2/MUSR": 0.3233, - "hfopenllm_v2/MMLU-PRO": 0.2124 - } - }, - { - "id": "baconnier/Napoleon_24B_V0.0", - "name": "Napoleon_24B_V0.0", - "developer": "baconnier", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1801, - "hfopenllm_v2/BBH": 0.6367, - "hfopenllm_v2/MATH Level 5": 0.2273, - "hfopenllm_v2/GPQA": 0.3792, - "hfopenllm_v2/MUSR": 0.442, - "hfopenllm_v2/MMLU-PRO": 0.504 - } - }, - { - "id": "baconnier/Napoleon_24B_V0.2", - "name": "Napoleon_24B_V0.2", - "developer": "baconnier", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2527, - "hfopenllm_v2/BBH": 0.5911, - "hfopenllm_v2/MATH Level 5": 0.1435, - "hfopenllm_v2/GPQA": 0.3381, - "hfopenllm_v2/MUSR": 0.446, - "hfopenllm_v2/MMLU-PRO": 0.4357 - } - }, - { - "id": "baebee/7B-Cetacea", - "name": "7B-Cetacea", - "developer": "baebee", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5279, - "hfopenllm_v2/BBH": 0.4757, - "hfopenllm_v2/MATH Level 5": 0.0468, - "hfopenllm_v2/GPQA": 0.2861, - "hfopenllm_v2/MUSR": 0.4136, - "hfopenllm_v2/MMLU-PRO": 0.2955 - } - }, - { - "id": "baebee/mergekit-model_stock-nzjnheg", - "name": "mergekit-model_stock-nzjnheg", - "developer": "baebee", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4844, - "hfopenllm_v2/BBH": 0.5287, - "hfopenllm_v2/MATH Level 5": 0.1677, - "hfopenllm_v2/GPQA": 0.2802, - "hfopenllm_v2/MUSR": 0.3847, - "hfopenllm_v2/MMLU-PRO": 0.3699 - } - }, - { - "id": "baebee/mergekit-ties-fnjenli", - "name": "mergekit-ties-fnjenli", - "developer": "baebee", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1988, - "hfopenllm_v2/BBH": 0.3024, - "hfopenllm_v2/MATH Level 5": 0.0023, - "hfopenllm_v2/GPQA": 0.245, - "hfopenllm_v2/MUSR": 0.4019, - "hfopenllm_v2/MMLU-PRO": 0.1129 - } - }, - { - "id": "bamec66557/mergekit-model_stock-zdaysvi", - "name": "mergekit-model_stock-zdaysvi", - "developer": "bamec66557", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6426, - "hfopenllm_v2/BBH": 0.5063, - "hfopenllm_v2/MATH Level 5": 0.1352, - "hfopenllm_v2/GPQA": 0.3138, - "hfopenllm_v2/MUSR": 0.4124, - "hfopenllm_v2/MMLU-PRO": 0.3688 - } - }, - { - "id": "bamec66557/mergekit-ties-sinbkow", - "name": "mergekit-ties-sinbkow", - "developer": "bamec66557", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6432, - "hfopenllm_v2/BBH": 0.5092, - "hfopenllm_v2/MATH Level 5": 0.145, - "hfopenllm_v2/GPQA": 0.3196, - "hfopenllm_v2/MUSR": 0.4045, - "hfopenllm_v2/MMLU-PRO": 0.3603 - } - }, - { - "id": "bamec66557/MISCHIEVOUS-12B", - "name": "MISCHIEVOUS-12B", - "developer": "bamec66557", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3852, - "hfopenllm_v2/BBH": 0.5405, - "hfopenllm_v2/MATH Level 5": 0.1276, - "hfopenllm_v2/GPQA": 0.3205, - "hfopenllm_v2/MUSR": 0.4145, - "hfopenllm_v2/MMLU-PRO": 0.3672 - } - }, - { - "id": "bamec66557/MISCHIEVOUS-12B-Mix_0.1v", - "name": "MISCHIEVOUS-12B-Mix_0.1v", - "developer": "bamec66557", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3636, - "hfopenllm_v2/BBH": 0.5436, - "hfopenllm_v2/MATH Level 5": 0.1329, - "hfopenllm_v2/GPQA": 0.328, - "hfopenllm_v2/MUSR": 0.4132, - "hfopenllm_v2/MMLU-PRO": 0.3674 - } - }, - { - "id": "bamec66557/MISCHIEVOUS-12B-Mix_0.2v", - "name": "MISCHIEVOUS-12B-Mix_0.2v", - "developer": "bamec66557", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3624, - "hfopenllm_v2/BBH": 0.5434, - "hfopenllm_v2/MATH Level 5": 0.1261, - "hfopenllm_v2/GPQA": 0.3255, - "hfopenllm_v2/MUSR": 0.4158, - "hfopenllm_v2/MMLU-PRO": 0.3663 - } - }, - { - "id": "bamec66557/MISCHIEVOUS-12B-Mix_0.3v", - "name": "MISCHIEVOUS-12B-Mix_0.3v", - "developer": "bamec66557", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.387, - "hfopenllm_v2/BBH": 0.5431, - "hfopenllm_v2/MATH Level 5": 0.1337, - "hfopenllm_v2/GPQA": 0.3196, - "hfopenllm_v2/MUSR": 0.4131, - "hfopenllm_v2/MMLU-PRO": 0.3664 - } - }, - { - "id": "bamec66557/MISCHIEVOUS-12B-Mix_0.4v", - "name": "MISCHIEVOUS-12B-Mix_0.4v", - "developer": "bamec66557", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6508, - "hfopenllm_v2/BBH": 0.5094, - "hfopenllm_v2/MATH Level 5": 0.1352, - "hfopenllm_v2/GPQA": 0.3171, - "hfopenllm_v2/MUSR": 0.4176, - "hfopenllm_v2/MMLU-PRO": 0.3683 - } - }, - { - "id": "bamec66557/MISCHIEVOUS-12B-Mix_0.5v", - "name": "MISCHIEVOUS-12B-Mix_0.5v", - "developer": "bamec66557", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3746, - "hfopenllm_v2/BBH": 0.5422, - "hfopenllm_v2/MATH Level 5": 0.1367, - "hfopenllm_v2/GPQA": 0.3205, - "hfopenllm_v2/MUSR": 0.4132, - "hfopenllm_v2/MMLU-PRO": 0.3661 - } - }, - { - "id": "bamec66557/MISCHIEVOUS-12B-Mix_0.6v", - "name": "MISCHIEVOUS-12B-Mix_0.6v", - "developer": "bamec66557", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4366, - "hfopenllm_v2/BBH": 0.5449, - "hfopenllm_v2/MATH Level 5": 0.1254, - "hfopenllm_v2/GPQA": 0.328, - "hfopenllm_v2/MUSR": 0.4185, - "hfopenllm_v2/MMLU-PRO": 0.3662 - } - }, - { - "id": "bamec66557/MISCHIEVOUS-12B-Mix_III_ex_V", - "name": "MISCHIEVOUS-12B-Mix_III_ex_V", - "developer": "bamec66557", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4316, - "hfopenllm_v2/BBH": 0.5449, - "hfopenllm_v2/MATH Level 5": 0.1322, - "hfopenllm_v2/GPQA": 0.3205, - "hfopenllm_v2/MUSR": 0.4198, - "hfopenllm_v2/MMLU-PRO": 0.3649 - } - }, - { - "id": "bamec66557/MISCHIEVOUS-12B-Mix_III_IV_V", - "name": "MISCHIEVOUS-12B-Mix_III_IV_V", - "developer": "bamec66557", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4031, - "hfopenllm_v2/BBH": 0.5465, - "hfopenllm_v2/MATH Level 5": 0.1292, - "hfopenllm_v2/GPQA": 0.3205, - "hfopenllm_v2/MUSR": 0.4198, - "hfopenllm_v2/MMLU-PRO": 0.3664 - } - }, - { - "id": "bamec66557/MISCHIEVOUS-12B-Mix_Neo", - "name": "MISCHIEVOUS-12B-Mix_Neo", - "developer": "bamec66557", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.625, - "hfopenllm_v2/BBH": 0.5078, - "hfopenllm_v2/MATH Level 5": 0.1329, - "hfopenllm_v2/GPQA": 0.3163, - "hfopenllm_v2/MUSR": 0.415, - "hfopenllm_v2/MMLU-PRO": 0.3685 - } - }, - { - "id": "bamec66557/Mistral-Nemo-VICIOUS_MESH-12B-2407", - "name": "Mistral-Nemo-VICIOUS_MESH-12B-2407", - "developer": "bamec66557", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6706, - "hfopenllm_v2/BBH": 0.5156, - "hfopenllm_v2/MATH Level 5": 0.1367, - "hfopenllm_v2/GPQA": 0.3154, - "hfopenllm_v2/MUSR": 0.431, - "hfopenllm_v2/MMLU-PRO": 0.3677 - } - }, - { - "id": "bamec66557/NameLess-12B-prob", - "name": "NameLess-12B-prob", - "developer": "bamec66557", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6602, - "hfopenllm_v2/BBH": 0.5158, - "hfopenllm_v2/MATH Level 5": 0.1261, - "hfopenllm_v2/GPQA": 0.3146, - "hfopenllm_v2/MUSR": 0.4336, - "hfopenllm_v2/MMLU-PRO": 0.3684 - } - }, - { - "id": "bamec66557/VICIOUS_MESH-12B", - "name": "VICIOUS_MESH-12B", - "developer": "bamec66557", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3716, - "hfopenllm_v2/BBH": 0.5436, - "hfopenllm_v2/MATH Level 5": 0.1344, - "hfopenllm_v2/GPQA": 0.328, - "hfopenllm_v2/MUSR": 0.4105, - "hfopenllm_v2/MMLU-PRO": 0.3679 - } - }, - { - "id": "bamec66557/VICIOUS_MESH-12B-0.1v", - "name": "VICIOUS_MESH-12B-0.1v", - "developer": "bamec66557", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3657, - "hfopenllm_v2/BBH": 0.5412, - "hfopenllm_v2/MATH Level 5": 0.1322, - "hfopenllm_v2/GPQA": 0.3247, - "hfopenllm_v2/MUSR": 0.4158, - "hfopenllm_v2/MMLU-PRO": 0.3683 - } - }, - { - "id": "bamec66557/VICIOUS_MESH-12B-0.X.ver", - "name": "VICIOUS_MESH-12B-0.X.ver", - "developer": "bamec66557", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3776, - "hfopenllm_v2/BBH": 0.5416, - "hfopenllm_v2/MATH Level 5": 0.1201, - "hfopenllm_v2/GPQA": 0.3213, - "hfopenllm_v2/MUSR": 0.4198, - "hfopenllm_v2/MMLU-PRO": 0.3671 - } - }, - { - "id": "bamec66557/VICIOUS_MESH-12B-ALPHA", - "name": "VICIOUS_MESH-12B-ALPHA", - "developer": "bamec66557", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6365, - "hfopenllm_v2/BBH": 0.5094, - "hfopenllm_v2/MATH Level 5": 0.1367, - "hfopenllm_v2/GPQA": 0.3138, - "hfopenllm_v2/MUSR": 0.4203, - "hfopenllm_v2/MMLU-PRO": 0.3697 - } - }, - { - "id": "bamec66557/VICIOUS_MESH-12B-BETA", - "name": "VICIOUS_MESH-12B-BETA", - "developer": "bamec66557", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6721, - "hfopenllm_v2/BBH": 0.5156, - "hfopenllm_v2/MATH Level 5": 0.1329, - "hfopenllm_v2/GPQA": 0.3163, - "hfopenllm_v2/MUSR": 0.431, - "hfopenllm_v2/MMLU-PRO": 0.3679 - } - }, - { - "id": "bamec66557/VICIOUS_MESH-12B-DELTA", - "name": "VICIOUS_MESH-12B-DELTA", - "developer": "bamec66557", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6469, - "hfopenllm_v2/BBH": 0.5055, - "hfopenllm_v2/MATH Level 5": 0.1375, - "hfopenllm_v2/GPQA": 0.3121, - "hfopenllm_v2/MUSR": 0.4057, - "hfopenllm_v2/MMLU-PRO": 0.3651 - } - }, - { - "id": "bamec66557/VICIOUS_MESH-12B-DIGAMMA", - "name": "VICIOUS_MESH-12B-DIGAMMA", - "developer": "bamec66557", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6429, - "hfopenllm_v2/BBH": 0.5061, - "hfopenllm_v2/MATH Level 5": 0.1337, - "hfopenllm_v2/GPQA": 0.3129, - "hfopenllm_v2/MUSR": 0.4097, - "hfopenllm_v2/MMLU-PRO": 0.3659 - } - }, - { - "id": "bamec66557/VICIOUS_MESH-12B-EPSILON", - "name": "VICIOUS_MESH-12B-EPSILON", - "developer": "bamec66557", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6305, - "hfopenllm_v2/BBH": 0.5038, - "hfopenllm_v2/MATH Level 5": 0.1261, - "hfopenllm_v2/GPQA": 0.3146, - "hfopenllm_v2/MUSR": 0.407, - "hfopenllm_v2/MMLU-PRO": 0.3648 - } - }, - { - "id": "bamec66557/VICIOUS_MESH-12B-GAMMA", - "name": "VICIOUS_MESH-12B-GAMMA", - "developer": "bamec66557", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6362, - "hfopenllm_v2/BBH": 0.5182, - "hfopenllm_v2/MATH Level 5": 0.1307, - "hfopenllm_v2/GPQA": 0.3138, - "hfopenllm_v2/MUSR": 0.4363, - "hfopenllm_v2/MMLU-PRO": 0.3666 - } - }, - { - "id": "bamec66557/VICIOUS_MESH-12B-NEMO", - "name": "VICIOUS_MESH-12B-NEMO", - "developer": "bamec66557", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4022, - "hfopenllm_v2/BBH": 0.5442, - "hfopenllm_v2/MATH Level 5": 0.1269, - "hfopenllm_v2/GPQA": 0.3238, - "hfopenllm_v2/MUSR": 0.4251, - "hfopenllm_v2/MMLU-PRO": 0.3716 - } - }, - { - "id": "bamec66557/VICIOUS_MESH-12B-OMEGA", - "name": "VICIOUS_MESH-12B-OMEGA", - "developer": "bamec66557", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.67, - "hfopenllm_v2/BBH": 0.5166, - "hfopenllm_v2/MATH Level 5": 0.1344, - "hfopenllm_v2/GPQA": 0.3154, - "hfopenllm_v2/MUSR": 0.4323, - "hfopenllm_v2/MMLU-PRO": 0.3677 - } - }, - { - "id": "bamec66557/VICIOUS_MESH-12B-UNION", - "name": "VICIOUS_MESH-12B-UNION", - "developer": "bamec66557", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6429, - "hfopenllm_v2/BBH": 0.5107, - "hfopenllm_v2/MATH Level 5": 0.139, - "hfopenllm_v2/GPQA": 0.3121, - "hfopenllm_v2/MUSR": 0.4257, - "hfopenllm_v2/MMLU-PRO": 0.3672 - } - }, - { - "id": "bamec66557/VICIOUS_MESH-12B_Razor", - "name": "VICIOUS_MESH-12B_Razor", - "developer": "bamec66557", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3736, - "hfopenllm_v2/BBH": 0.5447, - "hfopenllm_v2/MATH Level 5": 0.1299, - "hfopenllm_v2/GPQA": 0.323, - "hfopenllm_v2/MUSR": 0.4092, - "hfopenllm_v2/MMLU-PRO": 0.3669 - } - }, - { - "id": "Baptiste-HUVELLE-10/LeTriomphant2.2_ECE_iLAB", - "name": "LeTriomphant2.2_ECE_iLAB", - "developer": "Baptiste-HUVELLE-10", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5076, - "hfopenllm_v2/BBH": 0.7256, - "hfopenllm_v2/MATH Level 5": 0.4449, - "hfopenllm_v2/GPQA": 0.3993, - "hfopenllm_v2/MUSR": 0.4626, - "hfopenllm_v2/MMLU-PRO": 0.5851 - } - }, - { - "id": "BEE-spoke-data/Meta-Llama-3-8Bee", - "name": "Meta-Llama-3-8Bee", - "developer": "BEE-spoke-data", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1951, - "hfopenllm_v2/BBH": 0.4626, - "hfopenllm_v2/MATH Level 5": 0.0483, - "hfopenllm_v2/GPQA": 0.3138, - "hfopenllm_v2/MUSR": 0.3654, - "hfopenllm_v2/MMLU-PRO": 0.322 - } - }, - { - "id": "BEE-spoke-data/smol_llama-101M-GQA", - "name": "smol_llama-101M-GQA", - "developer": "BEE-spoke-data", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1384, - "hfopenllm_v2/BBH": 0.3018, - "hfopenllm_v2/MATH Level 5": 0.006, - "hfopenllm_v2/GPQA": 0.2576, - "hfopenllm_v2/MUSR": 0.3713, - "hfopenllm_v2/MMLU-PRO": 0.1107 - } - }, - { - "id": "BEE-spoke-data/smol_llama-220M-GQA", - "name": "smol_llama-220M-GQA", - "developer": "BEE-spoke-data", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2386, - "hfopenllm_v2/BBH": 0.3032, - "hfopenllm_v2/MATH Level 5": 0.0106, - "hfopenllm_v2/GPQA": 0.2559, - "hfopenllm_v2/MUSR": 0.4059, - "hfopenllm_v2/MMLU-PRO": 0.1149 - } - }, - { - "id": "BEE-spoke-data/smol_llama-220M-GQA-fineweb_edu", - "name": "smol_llama-220M-GQA-fineweb_edu", - "developer": "BEE-spoke-data", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1988, - "hfopenllm_v2/BBH": 0.2929, - "hfopenllm_v2/MATH Level 5": 0.0068, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.4368, - "hfopenllm_v2/MMLU-PRO": 0.1127 - } - }, - { - "id": "BEE-spoke-data/smol_llama-220M-openhermes", - "name": "smol_llama-220M-openhermes", - "developer": "BEE-spoke-data", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1555, - "hfopenllm_v2/BBH": 0.3028, - "hfopenllm_v2/MATH Level 5": 0.0106, - "hfopenllm_v2/GPQA": 0.2676, - "hfopenllm_v2/MUSR": 0.3847, - "hfopenllm_v2/MMLU-PRO": 0.112 - } - }, - { - "id": "BEE-spoke-data/tFINE-900m-e16-d32-flan", - "name": "tFINE-900m-e16-d32-flan", - "developer": "BEE-spoke-data", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1506, - "hfopenllm_v2/BBH": 0.3028, - "hfopenllm_v2/MATH Level 5": 0.0098, - "hfopenllm_v2/GPQA": 0.2332, - "hfopenllm_v2/MUSR": 0.3724, - "hfopenllm_v2/MMLU-PRO": 0.1307 - } - }, - { - "id": "BEE-spoke-data/tFINE-900m-e16-d32-flan-infinity-instruct-7m-T2T_en-1024", - "name": "tFINE-900m-e16-d32-flan-infinity-instruct-7m-T2T_en-1024", - "developer": "BEE-spoke-data", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1321, - "hfopenllm_v2/BBH": 0.3138, - "hfopenllm_v2/MATH Level 5": 0.0106, - "hfopenllm_v2/GPQA": 0.2542, - "hfopenllm_v2/MUSR": 0.4393, - "hfopenllm_v2/MMLU-PRO": 0.1237 - } - }, - { - "id": "BEE-spoke-data/tFINE-900m-e16-d32-instruct_2e", - "name": "tFINE-900m-e16-d32-instruct_2e", - "developer": "BEE-spoke-data", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1403, - "hfopenllm_v2/BBH": 0.3135, - "hfopenllm_v2/MATH Level 5": 0.0136, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.4207, - "hfopenllm_v2/MMLU-PRO": 0.1237 - } - }, - { - "id": "BEE-spoke-data/tFINE-900m-instruct-orpo", - "name": "tFINE-900m-instruct-orpo", - "developer": "BEE-spoke-data", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.133, - "hfopenllm_v2/BBH": 0.3022, - "hfopenllm_v2/MATH Level 5": 0.0159, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3409, - "hfopenllm_v2/MMLU-PRO": 0.1152 - } - }, - { - "id": "belztjti/dffghgjh", - "name": "dffghgjh", - "developer": "belztjti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5784, - "hfopenllm_v2/BBH": 0.3582, - "hfopenllm_v2/MATH Level 5": 0.0234, - "hfopenllm_v2/GPQA": 0.2634, - "hfopenllm_v2/MUSR": 0.3475, - "hfopenllm_v2/MMLU-PRO": 0.3422 - } - }, - { - "id": "belztjti/dtfgv", - "name": "dtfgv", - "developer": "belztjti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3345, - "hfopenllm_v2/BBH": 0.3282, - "hfopenllm_v2/MATH Level 5": 0.0181, - "hfopenllm_v2/GPQA": 0.2693, - "hfopenllm_v2/MUSR": 0.3794, - "hfopenllm_v2/MMLU-PRO": 0.1504 - } - }, - { - "id": "BenevolenceMessiah/Qwen2.5-72B-2x-Instruct-TIES-v1.0", - "name": "Qwen2.5-72B-2x-Instruct-TIES-v1.0", - "developer": "BenevolenceMessiah", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5473, - "hfopenllm_v2/BBH": 0.7273, - "hfopenllm_v2/MATH Level 5": 0.5785, - "hfopenllm_v2/GPQA": 0.3674, - "hfopenllm_v2/MUSR": 0.4207, - "hfopenllm_v2/MMLU-PRO": 0.5628 - } - }, - { - "id": "BenevolenceMessiah/Yi-Coder-9B-Chat-Instruct-TIES-MoE-v1.0", - "name": "Yi-Coder-9B-Chat-Instruct-TIES-MoE-v1.0", - "developer": "BenevolenceMessiah", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3012, - "hfopenllm_v2/BBH": 0.4909, - "hfopenllm_v2/MATH Level 5": 0.0415, - "hfopenllm_v2/GPQA": 0.2626, - "hfopenllm_v2/MUSR": 0.408, - "hfopenllm_v2/MMLU-PRO": 0.268 - } - }, - { - "id": "benhaotang/phi4-qwq-sky-t1", - "name": "phi4-qwq-sky-t1", - "developer": "benhaotang", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.046, - "hfopenllm_v2/BBH": 0.6711, - "hfopenllm_v2/MATH Level 5": 0.4101, - "hfopenllm_v2/GPQA": 0.3951, - "hfopenllm_v2/MUSR": 0.49, - "hfopenllm_v2/MMLU-PRO": 0.5244 - } - }, - { - "id": "beomi/gemma-mling-7b", - "name": "gemma-mling-7b", - "developer": "beomi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2029, - "hfopenllm_v2/BBH": 0.4068, - "hfopenllm_v2/MATH Level 5": 0.0544, - "hfopenllm_v2/GPQA": 0.25, - "hfopenllm_v2/MUSR": 0.3759, - "hfopenllm_v2/MMLU-PRO": 0.2633 - } - }, - { - "id": "beowolx/CodeNinja-1.0-OpenChat-7B", - "name": "CodeNinja-1.0-OpenChat-7B", - "developer": "beowolx", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5447, - "hfopenllm_v2/BBH": 0.4441, - "hfopenllm_v2/MATH Level 5": 0.0672, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.4243, - "hfopenllm_v2/MMLU-PRO": 0.3015 - } - }, - { - "id": "berkeley-nest/Starling-LM-7B-alpha", - "name": "Starling-LM-7B-alpha", - "developer": "berkeley-nest", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.548, - "hfopenllm_v2/BBH": 0.444, - "hfopenllm_v2/MATH Level 5": 0.0838, - "hfopenllm_v2/GPQA": 0.297, - "hfopenllm_v2/MUSR": 0.412, - "hfopenllm_v2/MMLU-PRO": 0.3172 - } - }, - { - "id": "berkeley-nest/Starling-RM-7B-alpha", - "name": "berkeley-nest/Starling-RM-7B-alpha", - "developer": "berkeley-nest", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7113, - "reward-bench/Chat": 0.9804, - "reward-bench/Chat Hard": 0.4561, - "reward-bench/Safety": 0.8446, - "reward-bench/Reasoning": 0.58, - "reward-bench/Prior Sets (0.5 weight)": 0.6794 - } - }, - { - "id": "bfuzzy1/acheron", - "name": "acheron", - "developer": "bfuzzy1", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1983, - "hfopenllm_v2/BBH": 0.3108, - "hfopenllm_v2/MATH Level 5": 0.0166, - "hfopenllm_v2/GPQA": 0.2391, - "hfopenllm_v2/MUSR": 0.3511, - "hfopenllm_v2/MMLU-PRO": 0.1096 - } - }, - { - "id": "bfuzzy1/acheron-c", - "name": "acheron-c", - "developer": "bfuzzy1", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1929, - "hfopenllm_v2/BBH": 0.3026, - "hfopenllm_v2/MATH Level 5": 0.003, - "hfopenllm_v2/GPQA": 0.2475, - "hfopenllm_v2/MUSR": 0.3382, - "hfopenllm_v2/MMLU-PRO": 0.1172 - } - }, - { - "id": "bfuzzy1/acheron-d", - "name": "acheron-d", - "developer": "bfuzzy1", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1925, - "hfopenllm_v2/BBH": 0.314, - "hfopenllm_v2/MATH Level 5": 0.0151, - "hfopenllm_v2/GPQA": 0.2366, - "hfopenllm_v2/MUSR": 0.3497, - "hfopenllm_v2/MMLU-PRO": 0.1134 - } - }, - { - "id": "bfuzzy1/acheron-m", - "name": "acheron-m", - "developer": "bfuzzy1", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1758, - "hfopenllm_v2/BBH": 0.2928, - "hfopenllm_v2/MATH Level 5": 0.0091, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3487, - "hfopenllm_v2/MMLU-PRO": 0.1113 - } - }, - { - "id": "bfuzzy1/acheron-m1a-llama", - "name": "acheron-m1a-llama", - "developer": "bfuzzy1", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1125, - "hfopenllm_v2/BBH": 0.2956, - "hfopenllm_v2/MATH Level 5": 0.0076, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3633, - "hfopenllm_v2/MMLU-PRO": 0.1146 - } - }, - { - "id": "bfuzzy1/Gunny", - "name": "Gunny", - "developer": "bfuzzy1", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7129, - "hfopenllm_v2/BBH": 0.4546, - "hfopenllm_v2/MATH Level 5": 0.173, - "hfopenllm_v2/GPQA": 0.2785, - "hfopenllm_v2/MUSR": 0.3583, - "hfopenllm_v2/MMLU-PRO": 0.3039 - } - }, - { - "id": "bfuzzy1/llambses-1", - "name": "llambses-1", - "developer": "bfuzzy1", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3554, - "hfopenllm_v2/BBH": 0.5047, - "hfopenllm_v2/MATH Level 5": 0.0687, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.4529, - "hfopenllm_v2/MMLU-PRO": 0.314 - } - }, - { - "id": "bhuvneshsaini/merged_model", - "name": "merged_model", - "developer": "bhuvneshsaini", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1813, - "hfopenllm_v2/BBH": 0.336, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.25, - "hfopenllm_v2/MUSR": 0.3497, - "hfopenllm_v2/MMLU-PRO": 0.1445 - } - }, - { - "id": "bigcode/starcoder2-15b", - "name": "starcoder2-15b", - "developer": "bigcode", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.278, - "hfopenllm_v2/BBH": 0.4448, - "hfopenllm_v2/MATH Level 5": 0.0597, - "hfopenllm_v2/GPQA": 0.2735, - "hfopenllm_v2/MUSR": 0.3501, - "hfopenllm_v2/MMLU-PRO": 0.2353 - } - }, - { - "id": "bigcode/starcoder2-3b", - "name": "starcoder2-3b", - "developer": "bigcode", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2037, - "hfopenllm_v2/BBH": 0.3509, - "hfopenllm_v2/MATH Level 5": 0.0151, - "hfopenllm_v2/GPQA": 0.2441, - "hfopenllm_v2/MUSR": 0.3435, - "hfopenllm_v2/MMLU-PRO": 0.1636 - } - }, - { - "id": "bigcode/starcoder2-7b", - "name": "starcoder2-7b", - "developer": "bigcode", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2209, - "hfopenllm_v2/BBH": 0.3661, - "hfopenllm_v2/MATH Level 5": 0.031, - "hfopenllm_v2/GPQA": 0.2517, - "hfopenllm_v2/MUSR": 0.3793, - "hfopenllm_v2/MMLU-PRO": 0.1642 - } - }, - { - "id": "bigscience/BLOOM-176B", - "name": "BLOOM 176B", - "developer": "bigscience", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.446, - "helm_classic/MMLU": 0.299, - "helm_classic/BoolQ": 0.704, - "helm_classic/NarrativeQA": 0.662, - "helm_classic/NaturalQuestions (open-book)": 0.621, - "helm_classic/QuAC": 0.361, - "helm_classic/HellaSwag": 0.744, - "helm_classic/OpenbookQA": 0.534, - "helm_classic/TruthfulQA": 0.205, - "helm_classic/MS MARCO (TREC)": 0.386, - "helm_classic/CNN/DailyMail": 0.08, - "helm_classic/XSUM": 0.03, - "helm_classic/IMDB": 0.945, - "helm_classic/CivilComments": 0.62, - "helm_classic/RAFT": 0.592 - } - }, - { - "id": "bigscience/bloom-1b1", - "name": "bloom-1b1", - "developer": "bigscience", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1373, - "hfopenllm_v2/BBH": 0.3107, - "hfopenllm_v2/MATH Level 5": 0.0053, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.37, - "hfopenllm_v2/MMLU-PRO": 0.1108 - } - }, - { - "id": "bigscience/bloom-1b7", - "name": "bloom-1b7", - "developer": "bigscience", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1044, - "hfopenllm_v2/BBH": 0.3141, - "hfopenllm_v2/MATH Level 5": 0.0053, - "hfopenllm_v2/GPQA": 0.2584, - "hfopenllm_v2/MUSR": 0.3886, - "hfopenllm_v2/MMLU-PRO": 0.1086 - } - }, - { - "id": "bigscience/bloom-3b", - "name": "bloom-3b", - "developer": "bigscience", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1271, - "hfopenllm_v2/BBH": 0.3063, - "hfopenllm_v2/MATH Level 5": 0.0083, - "hfopenllm_v2/GPQA": 0.2399, - "hfopenllm_v2/MUSR": 0.3981, - "hfopenllm_v2/MMLU-PRO": 0.1133 - } - }, - { - "id": "bigscience/bloom-560m", - "name": "bloom-560m", - "developer": "bigscience", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.062, - "hfopenllm_v2/BBH": 0.3026, - "hfopenllm_v2/MATH Level 5": 0.0038, - "hfopenllm_v2/GPQA": 0.2617, - "hfopenllm_v2/MUSR": 0.4031, - "hfopenllm_v2/MMLU-PRO": 0.1164 - } - }, - { - "id": "bigscience/bloom-7b1", - "name": "bloom-7b1", - "developer": "bigscience", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1322, - "hfopenllm_v2/BBH": 0.3114, - "hfopenllm_v2/MATH Level 5": 0.0053, - "hfopenllm_v2/GPQA": 0.2643, - "hfopenllm_v2/MUSR": 0.3487, - "hfopenllm_v2/MMLU-PRO": 0.1105 - } - }, - { - "id": "bigscience/T0pp-11B", - "name": "T0pp 11B", - "developer": "bigscience", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.197, - "helm_classic/MMLU": 0.407, - "helm_classic/BoolQ": 0.0, - "helm_classic/NarrativeQA": 0.151, - "helm_classic/NaturalQuestions (open-book)": 0.19, - "helm_classic/QuAC": 0.121, - "helm_classic/HellaSwag": -1.0, - "helm_classic/OpenbookQA": -1.0, - "helm_classic/TruthfulQA": 0.377, - "helm_classic/MS MARCO (TREC)": -1.0, - "helm_classic/CNN/DailyMail": 0.122, - "helm_classic/XSUM": 0.09, - "helm_classic/IMDB": 0.207, - "helm_classic/CivilComments": 0.234, - "helm_classic/RAFT": 0.118 - } - }, - { - "id": "bittensor/bitagent-bounty-8b", - "name": "BitAgent-Bounty-8B", - "developer": "bittensor", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 36.0, - "bfcl/bfcl.overall.overall_accuracy": 46.23, - "bfcl/bfcl.overall.total_cost_usd": 18.02, - "bfcl/bfcl.overall.latency_mean_s": 16.52, - "bfcl/bfcl.overall.latency_std_s": 30.73, - "bfcl/bfcl.overall.latency_p95_s": 77.12, - "bfcl/bfcl.non_live.ast_accuracy": 81.6, - "bfcl/bfcl.non_live.simple_ast_accuracy": 72.42, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 93.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 83.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 78.0, - "bfcl/bfcl.live.live_accuracy": 93.12, - "bfcl/bfcl.live.live_simple_ast_accuracy": 90.31, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 94.02, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 75.0, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 95.83, - "bfcl/bfcl.multi_turn.accuracy": 62.38, - "bfcl/bfcl.multi_turn.base_accuracy": 75.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 49.5, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 68.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 57.0, - "bfcl/bfcl.web_search.accuracy": 0.0, - "bfcl/bfcl.web_search.base_accuracy": 0.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 0.0, - "bfcl/bfcl.memory.accuracy": 1.51, - "bfcl/bfcl.memory.kv_accuracy": 1.29, - "bfcl/bfcl.memory.vector_accuracy": 1.29, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 1.94, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 68.75, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 97.48 - } - }, - { - "id": "BlackBeenie/Bloslain-8B-v0.2", - "name": "Bloslain-8B-v0.2", - "developer": "BlackBeenie", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5023, - "hfopenllm_v2/BBH": 0.5111, - "hfopenllm_v2/MATH Level 5": 0.145, - "hfopenllm_v2/GPQA": 0.3062, - "hfopenllm_v2/MUSR": 0.4076, - "hfopenllm_v2/MMLU-PRO": 0.3654 - } - }, - { - "id": "BlackBeenie/llama-3-luminous-merged", - "name": "llama-3-luminous-merged", - "developer": "BlackBeenie", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4323, - "hfopenllm_v2/BBH": 0.5154, - "hfopenllm_v2/MATH Level 5": 0.0869, - "hfopenllm_v2/GPQA": 0.2928, - "hfopenllm_v2/MUSR": 0.4149, - "hfopenllm_v2/MMLU-PRO": 0.3773 - } - }, - { - "id": "BlackBeenie/llama-3.1-8B-Galore-openassistant-guanaco", - "name": "llama-3.1-8B-Galore-openassistant-guanaco", - "developer": "BlackBeenie", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2635, - "hfopenllm_v2/BBH": 0.5213, - "hfopenllm_v2/MATH Level 5": 0.0665, - "hfopenllm_v2/GPQA": 0.3003, - "hfopenllm_v2/MUSR": 0.4406, - "hfopenllm_v2/MMLU-PRO": 0.3206 - } - }, - { - "id": "BlackBeenie/Llama-3.1-8B-OpenO1-SFT-v0.1", - "name": "Llama-3.1-8B-OpenO1-SFT-v0.1", - "developer": "BlackBeenie", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5124, - "hfopenllm_v2/BBH": 0.4787, - "hfopenllm_v2/MATH Level 5": 0.1526, - "hfopenllm_v2/GPQA": 0.2685, - "hfopenllm_v2/MUSR": 0.3618, - "hfopenllm_v2/MMLU-PRO": 0.3492 - } - }, - { - "id": "BlackBeenie/Llama-3.1-8B-pythonic-passthrough-merge", - "name": "Llama-3.1-8B-pythonic-passthrough-merge", - "developer": "BlackBeenie", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2316, - "hfopenllm_v2/BBH": 0.3454, - "hfopenllm_v2/MATH Level 5": 0.0113, - "hfopenllm_v2/GPQA": 0.2685, - "hfopenllm_v2/MUSR": 0.3778, - "hfopenllm_v2/MMLU-PRO": 0.1332 - } - }, - { - "id": "BlackBeenie/Neos-Gemma-2-9b", - "name": "Neos-Gemma-2-9b", - "developer": "BlackBeenie", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5876, - "hfopenllm_v2/BBH": 0.5503, - "hfopenllm_v2/MATH Level 5": 0.0982, - "hfopenllm_v2/GPQA": 0.323, - "hfopenllm_v2/MUSR": 0.3618, - "hfopenllm_v2/MMLU-PRO": 0.3981 - } - }, - { - "id": "BlackBeenie/Neos-Llama-3.1-8B", - "name": "Neos-Llama-3.1-8B", - "developer": "BlackBeenie", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4944, - "hfopenllm_v2/BBH": 0.4425, - "hfopenllm_v2/MATH Level 5": 0.1322, - "hfopenllm_v2/GPQA": 0.2685, - "hfopenllm_v2/MUSR": 0.375, - "hfopenllm_v2/MMLU-PRO": 0.3262 - } - }, - { - "id": "BlackBeenie/Neos-Llama-3.1-base", - "name": "Neos-Llama-3.1-base", - "developer": "BlackBeenie", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1751, - "hfopenllm_v2/BBH": 0.293, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2374, - "hfopenllm_v2/MUSR": 0.3499, - "hfopenllm_v2/MMLU-PRO": 0.1112 - } - }, - { - "id": "BlackBeenie/Neos-Phi-3-14B-v0.1", - "name": "Neos-Phi-3-14B-v0.1", - "developer": "BlackBeenie", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4022, - "hfopenllm_v2/BBH": 0.6212, - "hfopenllm_v2/MATH Level 5": 0.1782, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.4125, - "hfopenllm_v2/MMLU-PRO": 0.4564 - } - }, - { - "id": "Bllossom/llama-3.2-Korean-Bllossom-AICA-5B", - "name": "llama-3.2-Korean-Bllossom-AICA-5B", - "developer": "Bllossom", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5172, - "hfopenllm_v2/BBH": 0.4293, - "hfopenllm_v2/MATH Level 5": 0.1239, - "hfopenllm_v2/GPQA": 0.2987, - "hfopenllm_v2/MUSR": 0.3834, - "hfopenllm_v2/MMLU-PRO": 0.271 - } - }, - { - "id": "bluuwhale/L3-SthenoMaid-8B-V1", - "name": "L3-SthenoMaid-8B-V1", - "developer": "bluuwhale", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7345, - "hfopenllm_v2/BBH": 0.5219, - "hfopenllm_v2/MATH Level 5": 0.108, - "hfopenllm_v2/GPQA": 0.2802, - "hfopenllm_v2/MUSR": 0.3687, - "hfopenllm_v2/MMLU-PRO": 0.3656 - } - }, - { - "id": "BoltMonkey/DreadMix", - "name": "DreadMix", - "developer": "BoltMonkey", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7095, - "hfopenllm_v2/BBH": 0.5435, - "hfopenllm_v2/MATH Level 5": 0.1556, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.4212, - "hfopenllm_v2/MMLU-PRO": 0.379 - } - }, - { - "id": "BoltMonkey/NeuralDaredevil-SuperNova-Lite-7B-DARETIES-abliterated", - "name": "NeuralDaredevil-SuperNova-Lite-7B-DARETIES-abliterated", - "developer": "BoltMonkey", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7999, - "hfopenllm_v2/BBH": 0.5152, - "hfopenllm_v2/MATH Level 5": 0.1193, - "hfopenllm_v2/GPQA": 0.281, - "hfopenllm_v2/MUSR": 0.4019, - "hfopenllm_v2/MMLU-PRO": 0.3733 - } - }, - { - "id": "BoltMonkey/SuperNeuralDreadDevil-8b", - "name": "SuperNeuralDreadDevil-8b", - "developer": "BoltMonkey", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.771, - "hfopenllm_v2/BBH": 0.5286, - "hfopenllm_v2/MATH Level 5": 0.0929, - "hfopenllm_v2/GPQA": 0.2919, - "hfopenllm_v2/MUSR": 0.3977, - "hfopenllm_v2/MMLU-PRO": 0.3679 - } - }, - { - "id": "bond005/meno-tiny-0.1", - "name": "meno-tiny-0.1", - "developer": "bond005", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.455, - "hfopenllm_v2/BBH": 0.4263, - "hfopenllm_v2/MATH Level 5": 0.139, - "hfopenllm_v2/GPQA": 0.2819, - "hfopenllm_v2/MUSR": 0.4185, - "hfopenllm_v2/MMLU-PRO": 0.2786 - } - }, - { - "id": "bosonai/Higgs-Llama-3-70B", - "name": "Higgs-Llama-3-70B", - "developer": "bosonai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5561, - "hfopenllm_v2/BBH": 0.6258, - "hfopenllm_v2/MATH Level 5": 0.2523, - "hfopenllm_v2/GPQA": 0.3666, - "hfopenllm_v2/MUSR": 0.4471, - "hfopenllm_v2/MMLU-PRO": 0.4902 - } - }, - { - "id": "braindao/DeepSeek-R1-Distill-Qwen-1.5B-Blunt", - "name": "DeepSeek-R1-Distill-Qwen-1.5B-Blunt", - "developer": "braindao", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2611, - "hfopenllm_v2/BBH": 0.2774, - "hfopenllm_v2/MATH Level 5": 0.1382, - "hfopenllm_v2/GPQA": 0.2475, - "hfopenllm_v2/MUSR": 0.3595, - "hfopenllm_v2/MMLU-PRO": 0.1184 - } - }, - { - "id": "braindao/DeepSeek-R1-Distill-Qwen-1.5B-Reflective", - "name": "DeepSeek-R1-Distill-Qwen-1.5B-Reflective", - "developer": "braindao", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3033, - "hfopenllm_v2/BBH": 0.2908, - "hfopenllm_v2/MATH Level 5": 0.1631, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.3356, - "hfopenllm_v2/MMLU-PRO": 0.113 - } - }, - { - "id": "braindao/DeepSeek-R1-Distill-Qwen-14B", - "name": "DeepSeek-R1-Distill-Qwen-14B", - "developer": "braindao", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4172, - "hfopenllm_v2/BBH": 0.3033, - "hfopenllm_v2/MATH Level 5": 0.176, - "hfopenllm_v2/GPQA": 0.2802, - "hfopenllm_v2/MUSR": 0.4488, - "hfopenllm_v2/MMLU-PRO": 0.1127 - } - }, - { - "id": "braindao/DeepSeek-R1-Distill-Qwen-14B-ABUB-ST", - "name": "DeepSeek-R1-Distill-Qwen-14B-ABUB-ST", - "developer": "braindao", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3752, - "hfopenllm_v2/BBH": 0.4927, - "hfopenllm_v2/MATH Level 5": 0.5015, - "hfopenllm_v2/GPQA": 0.3448, - "hfopenllm_v2/MUSR": 0.4221, - "hfopenllm_v2/MMLU-PRO": 0.4243 - } - }, - { - "id": "braindao/DeepSeek-R1-Distill-Qwen-14B-Blunt", - "name": "DeepSeek-R1-Distill-Qwen-14B-Blunt", - "developer": "braindao", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5612, - "hfopenllm_v2/BBH": 0.3283, - "hfopenllm_v2/MATH Level 5": 0.1639, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.4554, - "hfopenllm_v2/MMLU-PRO": 0.1447 - } - }, - { - "id": "braindao/DeepSeek-R1-Distill-Qwen-14B-Blunt-Uncensored", - "name": "DeepSeek-R1-Distill-Qwen-14B-Blunt-Uncensored", - "developer": "braindao", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5422, - "hfopenllm_v2/BBH": 0.317, - "hfopenllm_v2/MATH Level 5": 0.1631, - "hfopenllm_v2/GPQA": 0.2827, - "hfopenllm_v2/MUSR": 0.4487, - "hfopenllm_v2/MMLU-PRO": 0.1431 - } - }, - { - "id": "braindao/DeepSeek-R1-Distill-Qwen-14B-Blunt-Uncensored-Blunt", - "name": "DeepSeek-R1-Distill-Qwen-14B-Blunt-Uncensored-Blunt", - "developer": "braindao", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5221, - "hfopenllm_v2/BBH": 0.3199, - "hfopenllm_v2/MATH Level 5": 0.2508, - "hfopenllm_v2/GPQA": 0.2785, - "hfopenllm_v2/MUSR": 0.4527, - "hfopenllm_v2/MMLU-PRO": 0.1484 - } - }, - { - "id": "braindao/DeepSeek-R1-Distill-Qwen-14B-Blunt-Uncensored-Blunt-Reflective", - "name": "DeepSeek-R1-Distill-Qwen-14B-Blunt-Uncensored-Blunt-Reflective", - "developer": "braindao", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.554, - "hfopenllm_v2/BBH": 0.3371, - "hfopenllm_v2/MATH Level 5": 0.2372, - "hfopenllm_v2/GPQA": 0.2777, - "hfopenllm_v2/MUSR": 0.4248, - "hfopenllm_v2/MMLU-PRO": 0.1504 - } - }, - { - "id": "braindao/DeepSeek-R1-Distill-Qwen-14B-Blunt-Uncensored-Reflective", - "name": "DeepSeek-R1-Distill-Qwen-14B-Blunt-Uncensored-Reflective", - "developer": "braindao", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5139, - "hfopenllm_v2/BBH": 0.3013, - "hfopenllm_v2/MATH Level 5": 0.1473, - "hfopenllm_v2/GPQA": 0.2878, - "hfopenllm_v2/MUSR": 0.4433, - "hfopenllm_v2/MMLU-PRO": 0.1289 - } - }, - { - "id": "braindao/DeepSeek-R1-Distill-Qwen-14B-Reflective", - "name": "DeepSeek-R1-Distill-Qwen-14B-Reflective", - "developer": "braindao", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.429, - "hfopenllm_v2/BBH": 0.3012, - "hfopenllm_v2/MATH Level 5": 0.1918, - "hfopenllm_v2/GPQA": 0.2727, - "hfopenllm_v2/MUSR": 0.4554, - "hfopenllm_v2/MMLU-PRO": 0.1129 - } - }, - { - "id": "braindao/DeepSeek-R1-Distill-Qwen-7B", - "name": "DeepSeek-R1-Distill-Qwen-7B", - "developer": "braindao", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3968, - "hfopenllm_v2/BBH": 0.2887, - "hfopenllm_v2/MATH Level 5": 0.1918, - "hfopenllm_v2/GPQA": 0.2617, - "hfopenllm_v2/MUSR": 0.3767, - "hfopenllm_v2/MMLU-PRO": 0.1141 - } - }, - { - "id": "braindao/DeepSeek-R1-Distill-Qwen-7B-Blunt", - "name": "DeepSeek-R1-Distill-Qwen-7B-Blunt", - "developer": "braindao", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4266, - "hfopenllm_v2/BBH": 0.2902, - "hfopenllm_v2/MATH Level 5": 0.2145, - "hfopenllm_v2/GPQA": 0.271, - "hfopenllm_v2/MUSR": 0.3885, - "hfopenllm_v2/MMLU-PRO": 0.1169 - } - }, - { - "id": "braindao/DeepSeek-R1-Distill-Qwen-7B-ORPO-Uncensored", - "name": "DeepSeek-R1-Distill-Qwen-7B-ORPO-Uncensored", - "developer": "braindao", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3655, - "hfopenllm_v2/BBH": 0.2958, - "hfopenllm_v2/MATH Level 5": 0.1737, - "hfopenllm_v2/GPQA": 0.2534, - "hfopenllm_v2/MUSR": 0.3846, - "hfopenllm_v2/MMLU-PRO": 0.1133 - } - }, - { - "id": "braindao/DeepSeek-R1-Distill-Qwen-7B-Reflective", - "name": "DeepSeek-R1-Distill-Qwen-7B-Reflective", - "developer": "braindao", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3922, - "hfopenllm_v2/BBH": 0.2907, - "hfopenllm_v2/MATH Level 5": 0.2024, - "hfopenllm_v2/GPQA": 0.2542, - "hfopenllm_v2/MUSR": 0.39, - "hfopenllm_v2/MMLU-PRO": 0.1155 - } - }, - { - "id": "braindao/iq-code-evmind-0.5b", - "name": "iq-code-evmind-0.5b", - "developer": "braindao", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3216, - "hfopenllm_v2/BBH": 0.3164, - "hfopenllm_v2/MATH Level 5": 0.0242, - "hfopenllm_v2/GPQA": 0.2416, - "hfopenllm_v2/MUSR": 0.3304, - "hfopenllm_v2/MMLU-PRO": 0.1189 - } - }, - { - "id": "braindao/Qwen2.5-14B", - "name": "Qwen2.5-14B", - "developer": "braindao", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5409, - "hfopenllm_v2/BBH": 0.5853, - "hfopenllm_v2/MATH Level 5": 0.2923, - "hfopenllm_v2/GPQA": 0.3733, - "hfopenllm_v2/MUSR": 0.4124, - "hfopenllm_v2/MMLU-PRO": 0.4884 - } - }, - { - "id": "braindao/Qwen2.5-14B-Instruct", - "name": "Qwen2.5-14B-Instruct", - "developer": "braindao", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8143, - "hfopenllm_v2/BBH": 0.6404, - "hfopenllm_v2/MATH Level 5": 0.5529, - "hfopenllm_v2/GPQA": 0.3289, - "hfopenllm_v2/MUSR": 0.414, - "hfopenllm_v2/MMLU-PRO": 0.4889 - } - }, - { - "id": "BrainWave-ML/llama3.2-3B-maths-orpo", - "name": "llama3.2-3B-maths-orpo", - "developer": "BrainWave-ML", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2049, - "hfopenllm_v2/BBH": 0.2912, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3575, - "hfopenllm_v2/MMLU-PRO": 0.1168 - } - }, - { - "id": "BramVanroy/fietje-2", - "name": "fietje-2", - "developer": "BramVanroy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2098, - "hfopenllm_v2/BBH": 0.4036, - "hfopenllm_v2/MATH Level 5": 0.0159, - "hfopenllm_v2/GPQA": 0.2542, - "hfopenllm_v2/MUSR": 0.3696, - "hfopenllm_v2/MMLU-PRO": 0.1986 - } - }, - { - "id": "BramVanroy/fietje-2-chat", - "name": "fietje-2-chat", - "developer": "BramVanroy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2917, - "hfopenllm_v2/BBH": 0.415, - "hfopenllm_v2/MATH Level 5": 0.0189, - "hfopenllm_v2/GPQA": 0.2399, - "hfopenllm_v2/MUSR": 0.3528, - "hfopenllm_v2/MMLU-PRO": 0.2055 - } - }, - { - "id": "BramVanroy/fietje-2-instruct", - "name": "fietje-2-instruct", - "developer": "BramVanroy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.279, - "hfopenllm_v2/BBH": 0.4136, - "hfopenllm_v2/MATH Level 5": 0.0227, - "hfopenllm_v2/GPQA": 0.2332, - "hfopenllm_v2/MUSR": 0.3369, - "hfopenllm_v2/MMLU-PRO": 0.2104 - } - }, - { - "id": "BramVanroy/GEITje-7B-ultra", - "name": "GEITje-7B-ultra", - "developer": "BramVanroy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3723, - "hfopenllm_v2/BBH": 0.3776, - "hfopenllm_v2/MATH Level 5": 0.0159, - "hfopenllm_v2/GPQA": 0.2626, - "hfopenllm_v2/MUSR": 0.329, - "hfopenllm_v2/MMLU-PRO": 0.2011 - } - }, - { - "id": "brgx53/3Bgeneral-ECE-PRYMMAL-Martial", - "name": "3Bgeneral-ECE-PRYMMAL-Martial", - "developer": "brgx53", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3289, - "hfopenllm_v2/BBH": 0.5458, - "hfopenllm_v2/MATH Level 5": 0.1314, - "hfopenllm_v2/GPQA": 0.3247, - "hfopenllm_v2/MUSR": 0.4373, - "hfopenllm_v2/MMLU-PRO": 0.3934 - } - }, - { - "id": "brgx53/3Bgeneralv2-ECE-PRYMMAL-Martial", - "name": "3Bgeneralv2-ECE-PRYMMAL-Martial", - "developer": "brgx53", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5677, - "hfopenllm_v2/BBH": 0.5607, - "hfopenllm_v2/MATH Level 5": 0.3497, - "hfopenllm_v2/GPQA": 0.3112, - "hfopenllm_v2/MUSR": 0.4356, - "hfopenllm_v2/MMLU-PRO": 0.4505 - } - }, - { - "id": "brgx53/3Blareneg-ECE-PRYMMAL-Martial", - "name": "3Blareneg-ECE-PRYMMAL-Martial", - "developer": "brgx53", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2876, - "hfopenllm_v2/BBH": 0.5358, - "hfopenllm_v2/MATH Level 5": 0.1208, - "hfopenllm_v2/GPQA": 0.3347, - "hfopenllm_v2/MUSR": 0.4429, - "hfopenllm_v2/MMLU-PRO": 0.4016 - } - }, - { - "id": "brgx53/3Blarenegv2-ECE-PRYMMAL-Martial", - "name": "3Blarenegv2-ECE-PRYMMAL-Martial", - "developer": "brgx53", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5662, - "hfopenllm_v2/BBH": 0.5607, - "hfopenllm_v2/MATH Level 5": 0.3497, - "hfopenllm_v2/GPQA": 0.3112, - "hfopenllm_v2/MUSR": 0.4356, - "hfopenllm_v2/MMLU-PRO": 0.4505 - } - }, - { - "id": "brgx53/Barracuda-PRYMMAL-ECE-TW3", - "name": "Barracuda-PRYMMAL-ECE-TW3", - "developer": "brgx53", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.164, - "hfopenllm_v2/BBH": 0.3002, - "hfopenllm_v2/MATH Level 5": 0.0023, - "hfopenllm_v2/GPQA": 0.2534, - "hfopenllm_v2/MUSR": 0.3609, - "hfopenllm_v2/MMLU-PRO": 0.1093 - } - }, - { - "id": "brgx53/LaConfiance-PRYMMAL-ECE-TW3", - "name": "LaConfiance-PRYMMAL-ECE-TW3", - "developer": "brgx53", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1579, - "hfopenllm_v2/BBH": 0.2962, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2517, - "hfopenllm_v2/MUSR": 0.3846, - "hfopenllm_v2/MMLU-PRO": 0.1146 - } - }, - { - "id": "BSC-LT/salamandra-7b", - "name": "salamandra-7b", - "developer": "BSC-LT", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1367, - "hfopenllm_v2/BBH": 0.3517, - "hfopenllm_v2/MATH Level 5": 0.0038, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.3501, - "hfopenllm_v2/MMLU-PRO": 0.1493 - } - }, - { - "id": "BSC-LT/salamandra-7b-instruct", - "name": "salamandra-7b-instruct", - "developer": "BSC-LT", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2451, - "hfopenllm_v2/BBH": 0.3851, - "hfopenllm_v2/MATH Level 5": 0.0083, - "hfopenllm_v2/GPQA": 0.2643, - "hfopenllm_v2/MUSR": 0.4134, - "hfopenllm_v2/MMLU-PRO": 0.1805 - } - }, - { - "id": "bunnycore/Best-Mix-Llama-3.1-8B", - "name": "Best-Mix-Llama-3.1-8B", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2067, - "hfopenllm_v2/BBH": 0.3432, - "hfopenllm_v2/MATH Level 5": 0.2054, - "hfopenllm_v2/GPQA": 0.2651, - "hfopenllm_v2/MUSR": 0.2929, - "hfopenllm_v2/MMLU-PRO": 0.1565 - } - }, - { - "id": "bunnycore/Blabbertron-1.0", - "name": "Blabbertron-1.0", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7433, - "hfopenllm_v2/BBH": 0.5497, - "hfopenllm_v2/MATH Level 5": 0.4924, - "hfopenllm_v2/GPQA": 0.302, - "hfopenllm_v2/MUSR": 0.4337, - "hfopenllm_v2/MMLU-PRO": 0.4354 - } - }, - { - "id": "bunnycore/Blabbertron-1.1", - "name": "Blabbertron-1.1", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7265, - "hfopenllm_v2/BBH": 0.5534, - "hfopenllm_v2/MATH Level 5": 0.4804, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.4416, - "hfopenllm_v2/MMLU-PRO": 0.4431 - } - }, - { - "id": "bunnycore/CyberCore-Qwen-2.1-7B", - "name": "CyberCore-Qwen-2.1-7B", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5766, - "hfopenllm_v2/BBH": 0.5572, - "hfopenllm_v2/MATH Level 5": 0.3588, - "hfopenllm_v2/GPQA": 0.3079, - "hfopenllm_v2/MUSR": 0.4145, - "hfopenllm_v2/MMLU-PRO": 0.4445 - } - }, - { - "id": "bunnycore/DeepQwen-3B-LCoT-SCE", - "name": "DeepQwen-3B-LCoT-SCE", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.449, - "hfopenllm_v2/BBH": 0.4512, - "hfopenllm_v2/MATH Level 5": 0.247, - "hfopenllm_v2/GPQA": 0.2626, - "hfopenllm_v2/MUSR": 0.3514, - "hfopenllm_v2/MMLU-PRO": 0.329 - } - }, - { - "id": "bunnycore/DeepSeek-R1-Distill-Qwen-7B-RRP-Ex", - "name": "DeepSeek-R1-Distill-Qwen-7B-RRP-Ex", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3901, - "hfopenllm_v2/BBH": 0.3494, - "hfopenllm_v2/MATH Level 5": 0.1654, - "hfopenllm_v2/GPQA": 0.2785, - "hfopenllm_v2/MUSR": 0.3663, - "hfopenllm_v2/MMLU-PRO": 0.2508 - } - }, - { - "id": "bunnycore/DeepThinker-7B-Sce-v1", - "name": "DeepThinker-7B-Sce-v1", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1218, - "hfopenllm_v2/BBH": 0.3018, - "hfopenllm_v2/MATH Level 5": 0.0098, - "hfopenllm_v2/GPQA": 0.2517, - "hfopenllm_v2/MUSR": 0.4194, - "hfopenllm_v2/MMLU-PRO": 0.1123 - } - }, - { - "id": "bunnycore/DeepThinker-7B-Sce-v2", - "name": "DeepThinker-7B-Sce-v2", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1631, - "hfopenllm_v2/BBH": 0.3057, - "hfopenllm_v2/MATH Level 5": 0.0113, - "hfopenllm_v2/GPQA": 0.2584, - "hfopenllm_v2/MUSR": 0.4101, - "hfopenllm_v2/MMLU-PRO": 0.1146 - } - }, - { - "id": "bunnycore/FuseCyberMix-Qwen-2.5-7B-Instruct", - "name": "FuseCyberMix-Qwen-2.5-7B-Instruct", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7019, - "hfopenllm_v2/BBH": 0.5518, - "hfopenllm_v2/MATH Level 5": 0.4841, - "hfopenllm_v2/GPQA": 0.297, - "hfopenllm_v2/MUSR": 0.402, - "hfopenllm_v2/MMLU-PRO": 0.4337 - } - }, - { - "id": "bunnycore/FuseQwQen-7B", - "name": "FuseQwQen-7B", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7275, - "hfopenllm_v2/BBH": 0.5504, - "hfopenllm_v2/MATH Level 5": 0.4366, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.4217, - "hfopenllm_v2/MMLU-PRO": 0.4407 - } - }, - { - "id": "bunnycore/FwF-Qwen-7B-0.1", - "name": "FwF-Qwen-7B-0.1", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3005, - "hfopenllm_v2/BBH": 0.5019, - "hfopenllm_v2/MATH Level 5": 0.2764, - "hfopenllm_v2/GPQA": 0.271, - "hfopenllm_v2/MUSR": 0.3952, - "hfopenllm_v2/MMLU-PRO": 0.4061 - } - }, - { - "id": "bunnycore/FwF-Qwen-7B-0.2", - "name": "FwF-Qwen-7B-0.2", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4479, - "hfopenllm_v2/BBH": 0.5596, - "hfopenllm_v2/MATH Level 5": 0.426, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.4218, - "hfopenllm_v2/MMLU-PRO": 0.4382 - } - }, - { - "id": "bunnycore/Gemma-2-2B-Smart", - "name": "Gemma-2-2B-Smart", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1321, - "hfopenllm_v2/BBH": 0.3974, - "hfopenllm_v2/MATH Level 5": 0.0332, - "hfopenllm_v2/GPQA": 0.2827, - "hfopenllm_v2/MUSR": 0.4249, - "hfopenllm_v2/MMLU-PRO": 0.2426 - } - }, - { - "id": "bunnycore/Gemma2-9B-TitanFusion", - "name": "Gemma2-9B-TitanFusion", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1618, - "hfopenllm_v2/BBH": 0.5712, - "hfopenllm_v2/MATH Level 5": 0.077, - "hfopenllm_v2/GPQA": 0.3322, - "hfopenllm_v2/MUSR": 0.4136, - "hfopenllm_v2/MMLU-PRO": 0.396 - } - }, - { - "id": "bunnycore/HyperLlama-3.1-8B", - "name": "HyperLlama-3.1-8B", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7883, - "hfopenllm_v2/BBH": 0.5103, - "hfopenllm_v2/MATH Level 5": 0.1828, - "hfopenllm_v2/GPQA": 0.2869, - "hfopenllm_v2/MUSR": 0.3829, - "hfopenllm_v2/MMLU-PRO": 0.3783 - } - }, - { - "id": "bunnycore/Llama-3.1-8B-TitanFusion-Mix", - "name": "Llama-3.1-8B-TitanFusion-Mix", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4925, - "hfopenllm_v2/BBH": 0.5756, - "hfopenllm_v2/MATH Level 5": 0.1284, - "hfopenllm_v2/GPQA": 0.2953, - "hfopenllm_v2/MUSR": 0.4317, - "hfopenllm_v2/MMLU-PRO": 0.3695 - } - }, - { - "id": "bunnycore/Llama-3.1-8B-TitanFusion-v3", - "name": "Llama-3.1-8B-TitanFusion-v3", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.481, - "hfopenllm_v2/BBH": 0.5262, - "hfopenllm_v2/MATH Level 5": 0.142, - "hfopenllm_v2/GPQA": 0.3087, - "hfopenllm_v2/MUSR": 0.4302, - "hfopenllm_v2/MMLU-PRO": 0.3806 - } - }, - { - "id": "bunnycore/Llama-3.2-3B-All-Mix", - "name": "Llama-3.2-3B-All-Mix", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7226, - "hfopenllm_v2/BBH": 0.4508, - "hfopenllm_v2/MATH Level 5": 0.1503, - "hfopenllm_v2/GPQA": 0.2626, - "hfopenllm_v2/MUSR": 0.3287, - "hfopenllm_v2/MMLU-PRO": 0.316 - } - }, - { - "id": "bunnycore/Llama-3.2-3B-Bespoke-Thought", - "name": "Llama-3.2-3B-Bespoke-Thought", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4113, - "hfopenllm_v2/BBH": 0.4522, - "hfopenllm_v2/MATH Level 5": 0.1647, - "hfopenllm_v2/GPQA": 0.2659, - "hfopenllm_v2/MUSR": 0.3302, - "hfopenllm_v2/MMLU-PRO": 0.311 - } - }, - { - "id": "bunnycore/Llama-3.2-3B-Booval", - "name": "Llama-3.2-3B-Booval", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6669, - "hfopenllm_v2/BBH": 0.4514, - "hfopenllm_v2/MATH Level 5": 0.1269, - "hfopenllm_v2/GPQA": 0.2668, - "hfopenllm_v2/MUSR": 0.3394, - "hfopenllm_v2/MMLU-PRO": 0.3058 - } - }, - { - "id": "bunnycore/Llama-3.2-3B-Deep-Test", - "name": "Llama-3.2-3B-Deep-Test", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4652, - "hfopenllm_v2/BBH": 0.4531, - "hfopenllm_v2/MATH Level 5": 0.1284, - "hfopenllm_v2/GPQA": 0.2643, - "hfopenllm_v2/MUSR": 0.3394, - "hfopenllm_v2/MMLU-PRO": 0.3152 - } - }, - { - "id": "bunnycore/Llama-3.2-3B-Della", - "name": "Llama-3.2-3B-Della", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3561, - "hfopenllm_v2/BBH": 0.3683, - "hfopenllm_v2/MATH Level 5": 0.0302, - "hfopenllm_v2/GPQA": 0.276, - "hfopenllm_v2/MUSR": 0.3902, - "hfopenllm_v2/MMLU-PRO": 0.2128 - } - }, - { - "id": "bunnycore/Llama-3.2-3B-Long-Think", - "name": "Llama-3.2-3B-Long-Think", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5473, - "hfopenllm_v2/BBH": 0.461, - "hfopenllm_v2/MATH Level 5": 0.1458, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.3396, - "hfopenllm_v2/MMLU-PRO": 0.3048 - } - }, - { - "id": "bunnycore/Llama-3.2-3B-Mix-Skill", - "name": "Llama-3.2-3B-Mix-Skill", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6404, - "hfopenllm_v2/BBH": 0.4582, - "hfopenllm_v2/MATH Level 5": 0.1473, - "hfopenllm_v2/GPQA": 0.2617, - "hfopenllm_v2/MUSR": 0.3396, - "hfopenllm_v2/MMLU-PRO": 0.3121 - } - }, - { - "id": "bunnycore/Llama-3.2-3B-ProdigyPlus", - "name": "Llama-3.2-3B-ProdigyPlus", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4015, - "hfopenllm_v2/BBH": 0.4392, - "hfopenllm_v2/MATH Level 5": 0.1156, - "hfopenllm_v2/GPQA": 0.2685, - "hfopenllm_v2/MUSR": 0.358, - "hfopenllm_v2/MMLU-PRO": 0.2817 - } - }, - { - "id": "bunnycore/Llama-3.2-3B-ProdigyPlusPlus", - "name": "Llama-3.2-3B-ProdigyPlusPlus", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1645, - "hfopenllm_v2/BBH": 0.369, - "hfopenllm_v2/MATH Level 5": 0.0453, - "hfopenllm_v2/GPQA": 0.2534, - "hfopenllm_v2/MUSR": 0.3541, - "hfopenllm_v2/MMLU-PRO": 0.15 - } - }, - { - "id": "bunnycore/Llama-3.2-3B-RP-DeepThink", - "name": "Llama-3.2-3B-RP-DeepThink", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7144, - "hfopenllm_v2/BBH": 0.4563, - "hfopenllm_v2/MATH Level 5": 0.1609, - "hfopenllm_v2/GPQA": 0.2659, - "hfopenllm_v2/MUSR": 0.3302, - "hfopenllm_v2/MMLU-PRO": 0.3242 - } - }, - { - "id": "bunnycore/Llama-3.2-3b-RP-Toxic-Fuse", - "name": "Llama-3.2-3b-RP-Toxic-Fuse", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6834, - "hfopenllm_v2/BBH": 0.465, - "hfopenllm_v2/MATH Level 5": 0.2402, - "hfopenllm_v2/GPQA": 0.2777, - "hfopenllm_v2/MUSR": 0.3954, - "hfopenllm_v2/MMLU-PRO": 0.3106 - } - }, - { - "id": "bunnycore/Llama-3.2-3B-RRStock", - "name": "Llama-3.2-3B-RRStock", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6657, - "hfopenllm_v2/BBH": 0.4568, - "hfopenllm_v2/MATH Level 5": 0.1699, - "hfopenllm_v2/GPQA": 0.2659, - "hfopenllm_v2/MUSR": 0.3314, - "hfopenllm_v2/MMLU-PRO": 0.3236 - } - }, - { - "id": "bunnycore/Llama-3.2-3B-ToxicKod", - "name": "Llama-3.2-3B-ToxicKod", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6319, - "hfopenllm_v2/BBH": 0.4525, - "hfopenllm_v2/MATH Level 5": 0.1699, - "hfopenllm_v2/GPQA": 0.2659, - "hfopenllm_v2/MUSR": 0.3475, - "hfopenllm_v2/MMLU-PRO": 0.288 - } - }, - { - "id": "bunnycore/Maestro-S1k-7B-Sce", - "name": "Maestro-S1k-7B-Sce", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2523, - "hfopenllm_v2/BBH": 0.3104, - "hfopenllm_v2/MATH Level 5": 0.0279, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.3768, - "hfopenllm_v2/MMLU-PRO": 0.117 - } - }, - { - "id": "bunnycore/Phi-3.5-mini-TitanFusion-0.1", - "name": "Phi-3.5-mini-TitanFusion-0.1", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5228, - "hfopenllm_v2/BBH": 0.5374, - "hfopenllm_v2/MATH Level 5": 0.1186, - "hfopenllm_v2/GPQA": 0.3314, - "hfopenllm_v2/MUSR": 0.4453, - "hfopenllm_v2/MMLU-PRO": 0.3807 - } - }, - { - "id": "bunnycore/Phi-4-Model-Stock", - "name": "Phi-4-Model-Stock", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6879, - "hfopenllm_v2/BBH": 0.689, - "hfopenllm_v2/MATH Level 5": 0.4298, - "hfopenllm_v2/GPQA": 0.3549, - "hfopenllm_v2/MUSR": 0.4441, - "hfopenllm_v2/MMLU-PRO": 0.5368 - } - }, - { - "id": "bunnycore/Phi-4-Model-Stock-v2", - "name": "Phi-4-Model-Stock-v2", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6375, - "hfopenllm_v2/BBH": 0.6825, - "hfopenllm_v2/MATH Level 5": 0.3754, - "hfopenllm_v2/GPQA": 0.349, - "hfopenllm_v2/MUSR": 0.4662, - "hfopenllm_v2/MMLU-PRO": 0.5331 - } - }, - { - "id": "bunnycore/Phi-4-Model-Stock-v3", - "name": "Phi-4-Model-Stock-v3", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5912, - "hfopenllm_v2/BBH": 0.6726, - "hfopenllm_v2/MATH Level 5": 0.4902, - "hfopenllm_v2/GPQA": 0.2894, - "hfopenllm_v2/MUSR": 0.4166, - "hfopenllm_v2/MMLU-PRO": 0.5381 - } - }, - { - "id": "bunnycore/Phi-4-Model-Stock-v4", - "name": "Phi-4-Model-Stock-v4", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.711, - "hfopenllm_v2/BBH": 0.6924, - "hfopenllm_v2/MATH Level 5": 0.3829, - "hfopenllm_v2/GPQA": 0.3691, - "hfopenllm_v2/MUSR": 0.4611, - "hfopenllm_v2/MMLU-PRO": 0.5394 - } - }, - { - "id": "bunnycore/Phi-4-ReasoningRP", - "name": "Phi-4-ReasoningRP", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6736, - "hfopenllm_v2/BBH": 0.6922, - "hfopenllm_v2/MATH Level 5": 0.4569, - "hfopenllm_v2/GPQA": 0.344, - "hfopenllm_v2/MUSR": 0.4491, - "hfopenllm_v2/MMLU-PRO": 0.5421 - } - }, - { - "id": "bunnycore/Phi-4-RP-v0", - "name": "Phi-4-RP-v0", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6827, - "hfopenllm_v2/BBH": 0.6856, - "hfopenllm_v2/MATH Level 5": 0.3316, - "hfopenllm_v2/GPQA": 0.3523, - "hfopenllm_v2/MUSR": 0.4141, - "hfopenllm_v2/MMLU-PRO": 0.5364 - } - }, - { - "id": "bunnycore/Phi-4-RR-Shoup", - "name": "Phi-4-RR-Shoup", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6587, - "hfopenllm_v2/BBH": 0.6947, - "hfopenllm_v2/MATH Level 5": 0.4992, - "hfopenllm_v2/GPQA": 0.3372, - "hfopenllm_v2/MUSR": 0.444, - "hfopenllm_v2/MMLU-PRO": 0.5429 - } - }, - { - "id": "bunnycore/Phi-4-RStock-v0.1", - "name": "Phi-4-RStock-v0.1", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7019, - "hfopenllm_v2/BBH": 0.6928, - "hfopenllm_v2/MATH Level 5": 0.395, - "hfopenllm_v2/GPQA": 0.3649, - "hfopenllm_v2/MUSR": 0.4584, - "hfopenllm_v2/MMLU-PRO": 0.5401 - } - }, - { - "id": "bunnycore/Phi-4-Sce-exp-v0.1", - "name": "Phi-4-Sce-exp-v0.1", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6595, - "hfopenllm_v2/BBH": 0.6943, - "hfopenllm_v2/MATH Level 5": 0.503, - "hfopenllm_v2/GPQA": 0.3356, - "hfopenllm_v2/MUSR": 0.4441, - "hfopenllm_v2/MMLU-PRO": 0.5423 - } - }, - { - "id": "bunnycore/Phi-4-Stock-Ex", - "name": "Phi-4-Stock-Ex", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6575, - "hfopenllm_v2/BBH": 0.6864, - "hfopenllm_v2/MATH Level 5": 0.4086, - "hfopenllm_v2/GPQA": 0.3507, - "hfopenllm_v2/MUSR": 0.4624, - "hfopenllm_v2/MMLU-PRO": 0.5375 - } - }, - { - "id": "bunnycore/Phi-4-Stock-RP", - "name": "Phi-4-Stock-RP", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6399, - "hfopenllm_v2/BBH": 0.686, - "hfopenllm_v2/MATH Level 5": 0.3414, - "hfopenllm_v2/GPQA": 0.3582, - "hfopenllm_v2/MUSR": 0.4715, - "hfopenllm_v2/MMLU-PRO": 0.5317 - } - }, - { - "id": "bunnycore/Phi-4-Trim-Exp1", - "name": "Phi-4-Trim-Exp1", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1219, - "hfopenllm_v2/BBH": 0.2852, - "hfopenllm_v2/MATH Level 5": 0.0053, - "hfopenllm_v2/GPQA": 0.255, - "hfopenllm_v2/MUSR": 0.4177, - "hfopenllm_v2/MMLU-PRO": 0.1147 - } - }, - { - "id": "bunnycore/Phi-Seek-4-Sce-V1", - "name": "Phi-Seek-4-Sce-V1", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2935, - "hfopenllm_v2/BBH": 0.6459, - "hfopenllm_v2/MATH Level 5": 0.2145, - "hfopenllm_v2/GPQA": 0.276, - "hfopenllm_v2/MUSR": 0.3982, - "hfopenllm_v2/MMLU-PRO": 0.5123 - } - }, - { - "id": "bunnycore/Qandora-2.5-7B-Creative", - "name": "Qandora-2.5-7B-Creative", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6803, - "hfopenllm_v2/BBH": 0.5542, - "hfopenllm_v2/MATH Level 5": 0.3059, - "hfopenllm_v2/GPQA": 0.3104, - "hfopenllm_v2/MUSR": 0.4212, - "hfopenllm_v2/MMLU-PRO": 0.448 - } - }, - { - "id": "bunnycore/QandoraExp-7B", - "name": "QandoraExp-7B", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7509, - "hfopenllm_v2/BBH": 0.5478, - "hfopenllm_v2/MATH Level 5": 0.4743, - "hfopenllm_v2/GPQA": 0.3104, - "hfopenllm_v2/MUSR": 0.4312, - "hfopenllm_v2/MMLU-PRO": 0.441 - } - }, - { - "id": "bunnycore/QandoraExp-7B-Persona", - "name": "QandoraExp-7B-Persona", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6247, - "hfopenllm_v2/BBH": 0.5558, - "hfopenllm_v2/MATH Level 5": 0.3104, - "hfopenllm_v2/GPQA": 0.3146, - "hfopenllm_v2/MUSR": 0.4372, - "hfopenllm_v2/MMLU-PRO": 0.4407 - } - }, - { - "id": "bunnycore/QandoraExp-7B-v2", - "name": "QandoraExp-7B-v2", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5607, - "hfopenllm_v2/BBH": 0.5445, - "hfopenllm_v2/MATH Level 5": 0.4713, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.4045, - "hfopenllm_v2/MMLU-PRO": 0.3909 - } - }, - { - "id": "bunnycore/Qwen-2.5-7B-Deep-Sky-T1", - "name": "Qwen-2.5-7B-Deep-Sky-T1", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4208, - "hfopenllm_v2/BBH": 0.414, - "hfopenllm_v2/MATH Level 5": 0.0551, - "hfopenllm_v2/GPQA": 0.281, - "hfopenllm_v2/MUSR": 0.4018, - "hfopenllm_v2/MMLU-PRO": 0.2104 - } - }, - { - "id": "bunnycore/Qwen-2.5-7B-Deep-Stock-v1", - "name": "Qwen-2.5-7B-Deep-Stock-v1", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5695, - "hfopenllm_v2/BBH": 0.5361, - "hfopenllm_v2/MATH Level 5": 0.2644, - "hfopenllm_v2/GPQA": 0.2777, - "hfopenllm_v2/MUSR": 0.4109, - "hfopenllm_v2/MMLU-PRO": 0.4066 - } - }, - { - "id": "bunnycore/Qwen-2.5-7B-Deep-Stock-v4", - "name": "Qwen-2.5-7B-Deep-Stock-v4", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7753, - "hfopenllm_v2/BBH": 0.5453, - "hfopenllm_v2/MATH Level 5": 0.4894, - "hfopenllm_v2/GPQA": 0.3003, - "hfopenllm_v2/MUSR": 0.4127, - "hfopenllm_v2/MMLU-PRO": 0.4342 - } - }, - { - "id": "bunnycore/Qwen-2.5-7B-Deep-Stock-v5", - "name": "Qwen-2.5-7B-Deep-Stock-v5", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4509, - "hfopenllm_v2/BBH": 0.4672, - "hfopenllm_v2/MATH Level 5": 0.1473, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.3648, - "hfopenllm_v2/MMLU-PRO": 0.2832 - } - }, - { - "id": "bunnycore/Qwen-2.5-7B-Exp-Sce", - "name": "Qwen-2.5-7B-Exp-Sce", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7652, - "hfopenllm_v2/BBH": 0.5506, - "hfopenllm_v2/MATH Level 5": 0.3255, - "hfopenllm_v2/GPQA": 0.2987, - "hfopenllm_v2/MUSR": 0.443, - "hfopenllm_v2/MMLU-PRO": 0.4259 - } - }, - { - "id": "bunnycore/Qwen-2.5-7B-R1-Stock", - "name": "Qwen-2.5-7B-R1-Stock", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7573, - "hfopenllm_v2/BBH": 0.5393, - "hfopenllm_v2/MATH Level 5": 0.5008, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.3994, - "hfopenllm_v2/MMLU-PRO": 0.4294 - } - }, - { - "id": "bunnycore/Qwen-2.5-7b-S1k", - "name": "Qwen-2.5-7b-S1k", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7162, - "hfopenllm_v2/BBH": 0.5563, - "hfopenllm_v2/MATH Level 5": 0.4781, - "hfopenllm_v2/GPQA": 0.2844, - "hfopenllm_v2/MUSR": 0.4071, - "hfopenllm_v2/MMLU-PRO": 0.4382 - } - }, - { - "id": "bunnycore/Qwen-2.5-7B-Stock-Deep-Bespoke", - "name": "Qwen-2.5-7B-Stock-Deep-Bespoke", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5206, - "hfopenllm_v2/BBH": 0.492, - "hfopenllm_v2/MATH Level 5": 0.1888, - "hfopenllm_v2/GPQA": 0.281, - "hfopenllm_v2/MUSR": 0.4068, - "hfopenllm_v2/MMLU-PRO": 0.358 - } - }, - { - "id": "bunnycore/Qwen2.5-1.5B-Model-Stock", - "name": "Qwen2.5-1.5B-Model-Stock", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1829, - "hfopenllm_v2/BBH": 0.2874, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3674, - "hfopenllm_v2/MMLU-PRO": 0.11 - } - }, - { - "id": "bunnycore/Qwen2.5-3B-Model-Stock", - "name": "Qwen2.5-3B-Model-Stock", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6381, - "hfopenllm_v2/BBH": 0.4712, - "hfopenllm_v2/MATH Level 5": 0.3799, - "hfopenllm_v2/GPQA": 0.2886, - "hfopenllm_v2/MUSR": 0.3942, - "hfopenllm_v2/MMLU-PRO": 0.325 - } - }, - { - "id": "bunnycore/Qwen2.5-3B-Model-Stock-v2", - "name": "Qwen2.5-3B-Model-Stock-v2", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.649, - "hfopenllm_v2/BBH": 0.4677, - "hfopenllm_v2/MATH Level 5": 0.3867, - "hfopenllm_v2/GPQA": 0.2869, - "hfopenllm_v2/MUSR": 0.3915, - "hfopenllm_v2/MMLU-PRO": 0.327 - } - }, - { - "id": "bunnycore/Qwen2.5-3B-Model-Stock-v3.1", - "name": "Qwen2.5-3B-Model-Stock-v3.1", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6481, - "hfopenllm_v2/BBH": 0.4737, - "hfopenllm_v2/MATH Level 5": 0.3897, - "hfopenllm_v2/GPQA": 0.2844, - "hfopenllm_v2/MUSR": 0.3968, - "hfopenllm_v2/MMLU-PRO": 0.329 - } - }, - { - "id": "bunnycore/Qwen2.5-3B-Model-Stock-v3.2", - "name": "Qwen2.5-3B-Model-Stock-v3.2", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6353, - "hfopenllm_v2/BBH": 0.4727, - "hfopenllm_v2/MATH Level 5": 0.3754, - "hfopenllm_v2/GPQA": 0.2836, - "hfopenllm_v2/MUSR": 0.3928, - "hfopenllm_v2/MMLU-PRO": 0.3294 - } - }, - { - "id": "bunnycore/Qwen2.5-3B-Model-Stock-v4.1", - "name": "Qwen2.5-3B-Model-Stock-v4.1", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6381, - "hfopenllm_v2/BBH": 0.482, - "hfopenllm_v2/MATH Level 5": 0.3769, - "hfopenllm_v2/GPQA": 0.2794, - "hfopenllm_v2/MUSR": 0.3941, - "hfopenllm_v2/MMLU-PRO": 0.3387 - } - }, - { - "id": "bunnycore/Qwen2.5-3B-RP-Mix", - "name": "Qwen2.5-3B-RP-Mix", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5721, - "hfopenllm_v2/BBH": 0.4894, - "hfopenllm_v2/MATH Level 5": 0.2153, - "hfopenllm_v2/GPQA": 0.2735, - "hfopenllm_v2/MUSR": 0.4284, - "hfopenllm_v2/MMLU-PRO": 0.3728 - } - }, - { - "id": "bunnycore/Qwen2.5-3B-RP-Thinker", - "name": "Qwen2.5-3B-RP-Thinker", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5894, - "hfopenllm_v2/BBH": 0.4164, - "hfopenllm_v2/MATH Level 5": 0.3353, - "hfopenllm_v2/GPQA": 0.2643, - "hfopenllm_v2/MUSR": 0.3287, - "hfopenllm_v2/MMLU-PRO": 0.315 - } - }, - { - "id": "bunnycore/Qwen2.5-3B-RP-Thinker-V2", - "name": "Qwen2.5-3B-RP-Thinker-V2", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.642, - "hfopenllm_v2/BBH": 0.4678, - "hfopenllm_v2/MATH Level 5": 0.3829, - "hfopenllm_v2/GPQA": 0.2852, - "hfopenllm_v2/MUSR": 0.3981, - "hfopenllm_v2/MMLU-PRO": 0.3271 - } - }, - { - "id": "bunnycore/Qwen2.5-7B-CyberRombos", - "name": "Qwen2.5-7B-CyberRombos", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7518, - "hfopenllm_v2/BBH": 0.5465, - "hfopenllm_v2/MATH Level 5": 0.4962, - "hfopenllm_v2/GPQA": 0.3045, - "hfopenllm_v2/MUSR": 0.4125, - "hfopenllm_v2/MMLU-PRO": 0.4391 - } - }, - { - "id": "bunnycore/Qwen2.5-7B-Fuse-Exp", - "name": "Qwen2.5-7B-Fuse-Exp", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5469, - "hfopenllm_v2/BBH": 0.5109, - "hfopenllm_v2/MATH Level 5": 0.3142, - "hfopenllm_v2/GPQA": 0.276, - "hfopenllm_v2/MUSR": 0.4573, - "hfopenllm_v2/MMLU-PRO": 0.3309 - } - }, - { - "id": "bunnycore/Qwen2.5-7B-Instruct-Fusion", - "name": "Qwen2.5-7B-Instruct-Fusion", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6962, - "hfopenllm_v2/BBH": 0.5492, - "hfopenllm_v2/MATH Level 5": 0.3406, - "hfopenllm_v2/GPQA": 0.3045, - "hfopenllm_v2/MUSR": 0.4297, - "hfopenllm_v2/MMLU-PRO": 0.4467 - } - }, - { - "id": "bunnycore/Qwen2.5-7B-Instruct-Merge-Stock-v0.1", - "name": "Qwen2.5-7B-Instruct-Merge-Stock-v0.1", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7509, - "hfopenllm_v2/BBH": 0.5529, - "hfopenllm_v2/MATH Level 5": 0.4894, - "hfopenllm_v2/GPQA": 0.3037, - "hfopenllm_v2/MUSR": 0.4231, - "hfopenllm_v2/MMLU-PRO": 0.4383 - } - }, - { - "id": "bunnycore/Qwen2.5-7B-MixStock-Sce-V0.3", - "name": "Qwen2.5-7B-MixStock-Sce-V0.3", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.212, - "hfopenllm_v2/BBH": 0.3479, - "hfopenllm_v2/MATH Level 5": 0.2576, - "hfopenllm_v2/GPQA": 0.2576, - "hfopenllm_v2/MUSR": 0.3714, - "hfopenllm_v2/MMLU-PRO": 0.1779 - } - }, - { - "id": "bunnycore/Qwen2.5-7B-MixStock-V0.1", - "name": "Qwen2.5-7B-MixStock-V0.1", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7673, - "hfopenllm_v2/BBH": 0.5479, - "hfopenllm_v2/MATH Level 5": 0.3172, - "hfopenllm_v2/GPQA": 0.3003, - "hfopenllm_v2/MUSR": 0.4416, - "hfopenllm_v2/MMLU-PRO": 0.4256 - } - }, - { - "id": "bunnycore/Qwen2.5-7B-R1-Bespoke-Stock", - "name": "Qwen2.5-7B-R1-Bespoke-Stock", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3726, - "hfopenllm_v2/BBH": 0.4822, - "hfopenllm_v2/MATH Level 5": 0.2047, - "hfopenllm_v2/GPQA": 0.2785, - "hfopenllm_v2/MUSR": 0.3926, - "hfopenllm_v2/MMLU-PRO": 0.3472 - } - }, - { - "id": "bunnycore/Qwen2.5-7B-R1-Bespoke-Task", - "name": "Qwen2.5-7B-R1-Bespoke-Task", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3787, - "hfopenllm_v2/BBH": 0.415, - "hfopenllm_v2/MATH Level 5": 0.1782, - "hfopenllm_v2/GPQA": 0.2534, - "hfopenllm_v2/MUSR": 0.3569, - "hfopenllm_v2/MMLU-PRO": 0.2688 - } - }, - { - "id": "bunnycore/Qwen2.5-7B-RRP-1M", - "name": "Qwen2.5-7B-RRP-1M", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7481, - "hfopenllm_v2/BBH": 0.5452, - "hfopenllm_v2/MATH Level 5": 0.3248, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.4483, - "hfopenllm_v2/MMLU-PRO": 0.4266 - } - }, - { - "id": "bunnycore/Qwen2.5-7B-RRP-1M-Thinker", - "name": "Qwen2.5-7B-RRP-1M-Thinker", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2308, - "hfopenllm_v2/BBH": 0.3482, - "hfopenllm_v2/MATH Level 5": 0.2719, - "hfopenllm_v2/GPQA": 0.2576, - "hfopenllm_v2/MUSR": 0.3767, - "hfopenllm_v2/MMLU-PRO": 0.1769 - } - }, - { - "id": "bunnycore/Qwen2.5-7B-RRP-ID", - "name": "Qwen2.5-7B-RRP-ID", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7473, - "hfopenllm_v2/BBH": 0.548, - "hfopenllm_v2/MATH Level 5": 0.4864, - "hfopenllm_v2/GPQA": 0.2827, - "hfopenllm_v2/MUSR": 0.418, - "hfopenllm_v2/MMLU-PRO": 0.4387 - } - }, - { - "id": "bunnycore/Qwen2.5-7B-Sky-R1-Mini", - "name": "Qwen2.5-7B-Sky-R1-Mini", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2305, - "hfopenllm_v2/BBH": 0.3503, - "hfopenllm_v2/MATH Level 5": 0.0295, - "hfopenllm_v2/GPQA": 0.2894, - "hfopenllm_v2/MUSR": 0.3448, - "hfopenllm_v2/MMLU-PRO": 0.1253 - } - }, - { - "id": "bunnycore/QwenMosaic-7B", - "name": "QwenMosaic-7B", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5819, - "hfopenllm_v2/BBH": 0.5564, - "hfopenllm_v2/MATH Level 5": 0.4441, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.4164, - "hfopenllm_v2/MMLU-PRO": 0.431 - } - }, - { - "id": "bunnycore/QwQen-3B-LCoT", - "name": "QwQen-3B-LCoT", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6025, - "hfopenllm_v2/BBH": 0.4899, - "hfopenllm_v2/MATH Level 5": 0.3618, - "hfopenllm_v2/GPQA": 0.2668, - "hfopenllm_v2/MUSR": 0.4178, - "hfopenllm_v2/MMLU-PRO": 0.3699 - } - }, - { - "id": "bunnycore/QwQen-3B-LCoT-R1", - "name": "QwQen-3B-LCoT-R1", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5342, - "hfopenllm_v2/BBH": 0.4799, - "hfopenllm_v2/MATH Level 5": 0.3353, - "hfopenllm_v2/GPQA": 0.2617, - "hfopenllm_v2/MUSR": 0.4138, - "hfopenllm_v2/MMLU-PRO": 0.3723 - } - }, - { - "id": "bunnycore/Smol-Llama-3.2-3B", - "name": "Smol-Llama-3.2-3B", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6679, - "hfopenllm_v2/BBH": 0.4539, - "hfopenllm_v2/MATH Level 5": 0.1382, - "hfopenllm_v2/GPQA": 0.2768, - "hfopenllm_v2/MUSR": 0.346, - "hfopenllm_v2/MMLU-PRO": 0.3228 - } - }, - { - "id": "bunnycore/SmolLM2-1.7-Persona", - "name": "SmolLM2-1.7-Persona", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5465, - "hfopenllm_v2/BBH": 0.3623, - "hfopenllm_v2/MATH Level 5": 0.0566, - "hfopenllm_v2/GPQA": 0.2634, - "hfopenllm_v2/MUSR": 0.3341, - "hfopenllm_v2/MMLU-PRO": 0.1974 - } - }, - { - "id": "bunnycore/SmolLM2-1.7B-roleplay-lora", - "name": "SmolLM2-1.7B-roleplay-lora", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5382, - "hfopenllm_v2/BBH": 0.361, - "hfopenllm_v2/MATH Level 5": 0.0529, - "hfopenllm_v2/GPQA": 0.2752, - "hfopenllm_v2/MUSR": 0.3395, - "hfopenllm_v2/MMLU-PRO": 0.1966 - } - }, - { - "id": "bunnycore/Tulu-3.1-8B-SuperNova", - "name": "Tulu-3.1-8B-SuperNova", - "developer": "bunnycore", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8194, - "hfopenllm_v2/BBH": 0.5254, - "hfopenllm_v2/MATH Level 5": 0.2462, - "hfopenllm_v2/GPQA": 0.302, - "hfopenllm_v2/MUSR": 0.3935, - "hfopenllm_v2/MMLU-PRO": 0.3814 - } - }, - { - "id": "byroneverson/Mistral-Small-Instruct-2409-abliterated", - "name": "Mistral-Small-Instruct-2409-abliterated", - "developer": "byroneverson", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6971, - "hfopenllm_v2/BBH": 0.5238, - "hfopenllm_v2/MATH Level 5": 0.2477, - "hfopenllm_v2/GPQA": 0.3331, - "hfopenllm_v2/MUSR": 0.3697, - "hfopenllm_v2/MMLU-PRO": 0.3923 - } - }, - { - "id": "byroneverson/Yi-1.5-9B-Chat-16K-abliterated", - "name": "Yi-1.5-9B-Chat-16K-abliterated", - "developer": "byroneverson", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5528, - "hfopenllm_v2/BBH": 0.5282, - "hfopenllm_v2/MATH Level 5": 0.1412, - "hfopenllm_v2/GPQA": 0.3129, - "hfopenllm_v2/MUSR": 0.4734, - "hfopenllm_v2/MMLU-PRO": 0.3823 - } - }, - { - "id": "byroneverson/Yi-1.5-9B-Chat-abliterated", - "name": "Yi-1.5-9B-Chat-abliterated", - "developer": "byroneverson", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5723, - "hfopenllm_v2/BBH": 0.5401, - "hfopenllm_v2/MATH Level 5": 0.1662, - "hfopenllm_v2/GPQA": 0.2919, - "hfopenllm_v2/MUSR": 0.4389, - "hfopenllm_v2/MMLU-PRO": 0.3715 - } - }, - { - "id": "bytedance/doubao-seed-1-6-thinking-250615", - "name": "doubao-seed-1-6-thinking-250615", - "developer": "ByteDance", - "evaluator_relationship": null, - "benchmark_scores": { - "livecodebenchpro/Hard Problems": 0.0, - "livecodebenchpro/Medium Problems": 0.07042253521126761, - "livecodebenchpro/Easy Problems": 0.5774647887323944 - } - }, - { - "id": "c10x/longthinker", - "name": "longthinker", - "developer": "c10x", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3609, - "hfopenllm_v2/BBH": 0.4927, - "hfopenllm_v2/MATH Level 5": 0.2319, - "hfopenllm_v2/GPQA": 0.2643, - "hfopenllm_v2/MUSR": 0.391, - "hfopenllm_v2/MMLU-PRO": 0.3527 - } - }, - { - "id": "c10x/Q-Pluse", - "name": "Q-Pluse", - "developer": "c10x", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1123, - "hfopenllm_v2/BBH": 0.2875, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2466, - "hfopenllm_v2/MUSR": 0.3938, - "hfopenllm_v2/MMLU-PRO": 0.1135 - } - }, - { - "id": "CarrotAI/Llama-3.2-Rabbit-Ko-3B-Instruct", - "name": "Llama-3.2-Rabbit-Ko-3B-Instruct", - "developer": "CarrotAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7199, - "hfopenllm_v2/BBH": 0.4427, - "hfopenllm_v2/MATH Level 5": 0.2054, - "hfopenllm_v2/GPQA": 0.271, - "hfopenllm_v2/MUSR": 0.3649, - "hfopenllm_v2/MMLU-PRO": 0.2822 - } - }, - { - "id": "CarrotAI/Llama-3.2-Rabbit-Ko-3B-Instruct-2412", - "name": "Llama-3.2-Rabbit-Ko-3B-Instruct-2412", - "developer": "CarrotAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4782, - "hfopenllm_v2/BBH": 0.4358, - "hfopenllm_v2/MATH Level 5": 0.176, - "hfopenllm_v2/GPQA": 0.2928, - "hfopenllm_v2/MUSR": 0.3872, - "hfopenllm_v2/MMLU-PRO": 0.3134 - } - }, - { - "id": "carsenk/flippa-v6", - "name": "flippa-v6", - "developer": "carsenk", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3439, - "hfopenllm_v2/BBH": 0.5047, - "hfopenllm_v2/MATH Level 5": 0.1405, - "hfopenllm_v2/GPQA": 0.2928, - "hfopenllm_v2/MUSR": 0.4089, - "hfopenllm_v2/MMLU-PRO": 0.3668 - } - }, - { - "id": "carsenk/phi3.5_mini_exp_825_uncensored", - "name": "phi3.5_mini_exp_825_uncensored", - "developer": "carsenk", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1364, - "hfopenllm_v2/BBH": 0.2965, - "hfopenllm_v2/MATH Level 5": 0.0106, - "hfopenllm_v2/GPQA": 0.2492, - "hfopenllm_v2/MUSR": 0.3644, - "hfopenllm_v2/MMLU-PRO": 0.1175 - } - }, - { - "id": "Casual-Autopsy/L3-Umbral-Mind-RP-v2.0-8B", - "name": "L3-Umbral-Mind-RP-v2.0-8B", - "developer": "Casual-Autopsy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7123, - "hfopenllm_v2/BBH": 0.5262, - "hfopenllm_v2/MATH Level 5": 0.1095, - "hfopenllm_v2/GPQA": 0.2869, - "hfopenllm_v2/MUSR": 0.3687, - "hfopenllm_v2/MMLU-PRO": 0.3723 - } - }, - { - "id": "cat-searcher/gemma-2-9b-it-sppo-iter-1", - "name": "gemma-2-9b-it-sppo-iter-1", - "developer": "cat-searcher", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3015, - "hfopenllm_v2/BBH": 0.5972, - "hfopenllm_v2/MATH Level 5": 0.0831, - "hfopenllm_v2/GPQA": 0.3448, - "hfopenllm_v2/MUSR": 0.3927, - "hfopenllm_v2/MMLU-PRO": 0.3854 - } - }, - { - "id": "cat-searcher/gemma-2-9b-it-sppo-iter-1-evol-1", - "name": "gemma-2-9b-it-sppo-iter-1-evol-1", - "developer": "cat-searcher", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2942, - "hfopenllm_v2/BBH": 0.5939, - "hfopenllm_v2/MATH Level 5": 0.0853, - "hfopenllm_v2/GPQA": 0.3406, - "hfopenllm_v2/MUSR": 0.3926, - "hfopenllm_v2/MMLU-PRO": 0.38 - } - }, - { - "id": "CausalLM/14B", - "name": "14B", - "developer": "CausalLM", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2788, - "hfopenllm_v2/BBH": 0.47, - "hfopenllm_v2/MATH Level 5": 0.0755, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.4155, - "hfopenllm_v2/MMLU-PRO": 0.3221 - } - }, - { - "id": "CausalLM/34b-beta", - "name": "34b-beta", - "developer": "CausalLM", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3043, - "hfopenllm_v2/BBH": 0.5591, - "hfopenllm_v2/MATH Level 5": 0.0483, - "hfopenllm_v2/GPQA": 0.3465, - "hfopenllm_v2/MUSR": 0.3749, - "hfopenllm_v2/MMLU-PRO": 0.5325 - } - }, - { - "id": "CausalLM/preview-1-hf", - "name": "preview-1-hf", - "developer": "CausalLM", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5559, - "hfopenllm_v2/BBH": 0.3615, - "hfopenllm_v2/MATH Level 5": 0.0302, - "hfopenllm_v2/GPQA": 0.2617, - "hfopenllm_v2/MUSR": 0.3422, - "hfopenllm_v2/MMLU-PRO": 0.3597 - } - }, - { - "id": "cckm/tinymistral_950m", - "name": "tinymistral_950m", - "developer": "cckm", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2395, - "hfopenllm_v2/BBH": 0.2969, - "hfopenllm_v2/MATH Level 5": 0.0053, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3554, - "hfopenllm_v2/MMLU-PRO": 0.1096 - } - }, - { - "id": "cgato/TheSalt-L3-8b-v0.3.2", - "name": "TheSalt-L3-8b-v0.3.2", - "developer": "cgato", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2705, - "hfopenllm_v2/BBH": 0.2968, - "hfopenllm_v2/MATH Level 5": 0.0476, - "hfopenllm_v2/GPQA": 0.2659, - "hfopenllm_v2/MUSR": 0.3896, - "hfopenllm_v2/MMLU-PRO": 0.1139 - } - }, - { - "id": "Changgil/K2S3-14b-v0.2", - "name": "K2S3-14b-v0.2", - "developer": "Changgil", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3243, - "hfopenllm_v2/BBH": 0.4613, - "hfopenllm_v2/MATH Level 5": 0.0574, - "hfopenllm_v2/GPQA": 0.281, - "hfopenllm_v2/MUSR": 0.3923, - "hfopenllm_v2/MMLU-PRO": 0.2644 - } - }, - { - "id": "Changgil/K2S3-v0.1", - "name": "K2S3-v0.1", - "developer": "Changgil", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3277, - "hfopenllm_v2/BBH": 0.4655, - "hfopenllm_v2/MATH Level 5": 0.0461, - "hfopenllm_v2/GPQA": 0.2643, - "hfopenllm_v2/MUSR": 0.4014, - "hfopenllm_v2/MMLU-PRO": 0.2562 - } - }, - { - "id": "chargoddard/prometheus-2-llama-3-8b", - "name": "prometheus-2-llama-3-8b", - "developer": "chargoddard", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5289, - "hfopenllm_v2/BBH": 0.4931, - "hfopenllm_v2/MATH Level 5": 0.0823, - "hfopenllm_v2/GPQA": 0.2727, - "hfopenllm_v2/MUSR": 0.3396, - "hfopenllm_v2/MMLU-PRO": 0.3087 - } - }, - { - "id": "chujiezheng/Llama-3-Instruct-8B-SimPO-ExPO", - "name": "Llama-3-Instruct-8B-SimPO-ExPO", - "developer": "chujiezheng", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6434, - "hfopenllm_v2/BBH": 0.4765, - "hfopenllm_v2/MATH Level 5": 0.0702, - "hfopenllm_v2/GPQA": 0.2869, - "hfopenllm_v2/MUSR": 0.392, - "hfopenllm_v2/MMLU-PRO": 0.3401 - } - }, - { - "id": "chujiezheng/Mistral7B-PairRM-SPPO-ExPO", - "name": "Mistral7B-PairRM-SPPO-ExPO", - "developer": "chujiezheng", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3673, - "hfopenllm_v2/BBH": 0.3882, - "hfopenllm_v2/MATH Level 5": 0.0181, - "hfopenllm_v2/GPQA": 0.2768, - "hfopenllm_v2/MUSR": 0.4055, - "hfopenllm_v2/MMLU-PRO": 0.2552 - } - }, - { - "id": "CIR-AMS/BTRM_Qwen2_7b_0613", - "name": "CIR-AMS/BTRM_Qwen2_7b_0613", - "developer": "CIR-AMS", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.8172, - "reward-bench/Factuality": 0.5347, - "reward-bench/Precise IF": 0.3563, - "reward-bench/Math": 0.6066, - "reward-bench/Safety": 0.9014, - "reward-bench/Focus": 0.5737, - "reward-bench/Ties": 0.6527, - "reward-bench/Chat": 0.9749, - "reward-bench/Chat Hard": 0.5724, - "reward-bench/Reasoning": 0.8775, - "reward-bench/Prior Sets (0.5 weight)": 0.7029 - } - }, - { - "id": "cjvt/GaMS-1B", - "name": "GaMS-1B", - "developer": "cjvt", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1635, - "hfopenllm_v2/BBH": 0.3075, - "hfopenllm_v2/MATH Level 5": 0.0136, - "hfopenllm_v2/GPQA": 0.2584, - "hfopenllm_v2/MUSR": 0.3684, - "hfopenllm_v2/MMLU-PRO": 0.1149 - } - }, - { - "id": "ClaudioItaly/Albacus", - "name": "Albacus", - "developer": "ClaudioItaly", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4667, - "hfopenllm_v2/BBH": 0.5113, - "hfopenllm_v2/MATH Level 5": 0.071, - "hfopenllm_v2/GPQA": 0.2718, - "hfopenllm_v2/MUSR": 0.4135, - "hfopenllm_v2/MMLU-PRO": 0.3165 - } - }, - { - "id": "ClaudioItaly/Book-Gut12B", - "name": "Book-Gut12B", - "developer": "ClaudioItaly", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3998, - "hfopenllm_v2/BBH": 0.5417, - "hfopenllm_v2/MATH Level 5": 0.102, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.4635, - "hfopenllm_v2/MMLU-PRO": 0.367 - } - }, - { - "id": "ClaudioItaly/Evolutionstory-7B-v2.2", - "name": "Evolutionstory-7B-v2.2", - "developer": "ClaudioItaly", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4814, - "hfopenllm_v2/BBH": 0.5108, - "hfopenllm_v2/MATH Level 5": 0.071, - "hfopenllm_v2/GPQA": 0.2752, - "hfopenllm_v2/MUSR": 0.4135, - "hfopenllm_v2/MMLU-PRO": 0.3159 - } - }, - { - "id": "ClaudioItaly/intelligence-cod-rag-7b-v3", - "name": "intelligence-cod-rag-7b-v3", - "developer": "ClaudioItaly", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6898, - "hfopenllm_v2/BBH": 0.5366, - "hfopenllm_v2/MATH Level 5": 0.3807, - "hfopenllm_v2/GPQA": 0.2727, - "hfopenllm_v2/MUSR": 0.4153, - "hfopenllm_v2/MMLU-PRO": 0.4195 - } - }, - { - "id": "cloudyu/Llama-3-70Bx2-MOE", - "name": "Llama-3-70Bx2-MOE", - "developer": "cloudyu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5482, - "hfopenllm_v2/BBH": 0.6636, - "hfopenllm_v2/MATH Level 5": 0.2175, - "hfopenllm_v2/GPQA": 0.3935, - "hfopenllm_v2/MUSR": 0.4812, - "hfopenllm_v2/MMLU-PRO": 0.5142 - } - }, - { - "id": "cloudyu/Llama-3.2-3Bx4", - "name": "Llama-3.2-3Bx4", - "developer": "cloudyu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5069, - "hfopenllm_v2/BBH": 0.4332, - "hfopenllm_v2/MATH Level 5": 0.1073, - "hfopenllm_v2/GPQA": 0.2777, - "hfopenllm_v2/MUSR": 0.3496, - "hfopenllm_v2/MMLU-PRO": 0.2985 - } - }, - { - "id": "cloudyu/Mixtral_11Bx2_MoE_19B", - "name": "Mixtral_11Bx2_MoE_19B", - "developer": "cloudyu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3851, - "hfopenllm_v2/BBH": 0.5209, - "hfopenllm_v2/MATH Level 5": 0.0672, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.4297, - "hfopenllm_v2/MMLU-PRO": 0.3311 - } - }, - { - "id": "cloudyu/Mixtral_34Bx2_MoE_60B", - "name": "Mixtral_34Bx2_MoE_60B", - "developer": "cloudyu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4538, - "hfopenllm_v2/BBH": 0.587, - "hfopenllm_v2/MATH Level 5": 0.077, - "hfopenllm_v2/GPQA": 0.3381, - "hfopenllm_v2/MUSR": 0.4625, - "hfopenllm_v2/MMLU-PRO": 0.4766 - } - }, - { - "id": "cloudyu/Mixtral_7Bx2_MoE", - "name": "Mixtral_7Bx2_MoE", - "developer": "cloudyu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.448, - "hfopenllm_v2/BBH": 0.516, - "hfopenllm_v2/MATH Level 5": 0.0687, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.4473, - "hfopenllm_v2/MMLU-PRO": 0.3044 - } - }, - { - "id": "cloudyu/S1-Llama-3.2-3Bx4-MoE", - "name": "S1-Llama-3.2-3Bx4-MoE", - "developer": "cloudyu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5302, - "hfopenllm_v2/BBH": 0.4358, - "hfopenllm_v2/MATH Level 5": 0.1201, - "hfopenllm_v2/GPQA": 0.2936, - "hfopenllm_v2/MUSR": 0.3456, - "hfopenllm_v2/MMLU-PRO": 0.3044 - } - }, - { - "id": "cloudyu/Yi-34Bx2-MoE-60B-DPO", - "name": "Yi-34Bx2-MoE-60B-DPO", - "developer": "cloudyu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5319, - "hfopenllm_v2/BBH": 0.5168, - "hfopenllm_v2/MATH Level 5": 0.0702, - "hfopenllm_v2/GPQA": 0.3221, - "hfopenllm_v2/MUSR": 0.4375, - "hfopenllm_v2/MMLU-PRO": 0.4677 - } - }, - { - "id": "cluebbers/Llama-3.1-8B-paraphrase-type-generation-apty-ipo", - "name": "Llama-3.1-8B-paraphrase-type-generation-apty-ipo", - "developer": "cluebbers", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1327, - "hfopenllm_v2/BBH": 0.38, - "hfopenllm_v2/MATH Level 5": 0.0249, - "hfopenllm_v2/GPQA": 0.2634, - "hfopenllm_v2/MUSR": 0.4332, - "hfopenllm_v2/MMLU-PRO": 0.2591 - } - }, - { - "id": "cluebbers/Llama-3.1-8B-paraphrase-type-generation-apty-sigmoid", - "name": "Llama-3.1-8B-paraphrase-type-generation-apty-sigmoid", - "developer": "cluebbers", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1318, - "hfopenllm_v2/BBH": 0.3789, - "hfopenllm_v2/MATH Level 5": 0.0264, - "hfopenllm_v2/GPQA": 0.2685, - "hfopenllm_v2/MUSR": 0.4306, - "hfopenllm_v2/MMLU-PRO": 0.2562 - } - }, - { - "id": "cluebbers/Llama-3.1-8B-paraphrase-type-generation-etpc", - "name": "Llama-3.1-8B-paraphrase-type-generation-etpc", - "developer": "cluebbers", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1209, - "hfopenllm_v2/BBH": 0.3781, - "hfopenllm_v2/MATH Level 5": 0.0196, - "hfopenllm_v2/GPQA": 0.2651, - "hfopenllm_v2/MUSR": 0.4319, - "hfopenllm_v2/MMLU-PRO": 0.2556 - } - }, - { - "id": "cognitivecomputations/dolphin-2.9-llama3-8b", - "name": "dolphin-2.9-llama3-8b", - "developer": "cognitivecomputations", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.385, - "hfopenllm_v2/BBH": 0.495, - "hfopenllm_v2/MATH Level 5": 0.0574, - "hfopenllm_v2/GPQA": 0.2869, - "hfopenllm_v2/MUSR": 0.4375, - "hfopenllm_v2/MMLU-PRO": 0.2771 - } - }, - { - "id": "cognitivecomputations/dolphin-2.9.1-llama-3-70b", - "name": "dolphin-2.9.1-llama-3-70b", - "developer": "cognitivecomputations", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.376, - "hfopenllm_v2/BBH": 0.5205, - "hfopenllm_v2/MATH Level 5": 0.182, - "hfopenllm_v2/GPQA": 0.3087, - "hfopenllm_v2/MUSR": 0.4976, - "hfopenllm_v2/MMLU-PRO": 0.413 - } - }, - { - "id": "cognitivecomputations/dolphin-2.9.1-yi-1.5-34b", - "name": "dolphin-2.9.1-yi-1.5-34b", - "developer": "cognitivecomputations", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3853, - "hfopenllm_v2/BBH": 0.6076, - "hfopenllm_v2/MATH Level 5": 0.1866, - "hfopenllm_v2/GPQA": 0.3431, - "hfopenllm_v2/MUSR": 0.4598, - "hfopenllm_v2/MMLU-PRO": 0.4519 - } - }, - { - "id": "cognitivecomputations/dolphin-2.9.1-yi-1.5-9b", - "name": "dolphin-2.9.1-yi-1.5-9b", - "developer": "cognitivecomputations", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4465, - "hfopenllm_v2/BBH": 0.5484, - "hfopenllm_v2/MATH Level 5": 0.1518, - "hfopenllm_v2/GPQA": 0.3381, - "hfopenllm_v2/MUSR": 0.4348, - "hfopenllm_v2/MMLU-PRO": 0.3967 - } - }, - { - "id": "cognitivecomputations/dolphin-2.9.2-Phi-3-Medium", - "name": "dolphin-2.9.2-Phi-3-Medium", - "developer": "cognitivecomputations", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4248, - "hfopenllm_v2/BBH": 0.6457, - "hfopenllm_v2/MATH Level 5": 0.1828, - "hfopenllm_v2/GPQA": 0.3272, - "hfopenllm_v2/MUSR": 0.4191, - "hfopenllm_v2/MMLU-PRO": 0.4555 - } - }, - { - "id": "cognitivecomputations/dolphin-2.9.2-Phi-3-Medium-abliterated", - "name": "dolphin-2.9.2-Phi-3-Medium-abliterated", - "developer": "cognitivecomputations", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4124, - "hfopenllm_v2/BBH": 0.6383, - "hfopenllm_v2/MATH Level 5": 0.182, - "hfopenllm_v2/GPQA": 0.3289, - "hfopenllm_v2/MUSR": 0.4349, - "hfopenllm_v2/MMLU-PRO": 0.4525 - } - }, - { - "id": "cognitivecomputations/dolphin-2.9.2-qwen2-72b", - "name": "dolphin-2.9.2-qwen2-72b", - "developer": "cognitivecomputations", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6344, - "hfopenllm_v2/BBH": 0.6296, - "hfopenllm_v2/MATH Level 5": 0.2802, - "hfopenllm_v2/GPQA": 0.37, - "hfopenllm_v2/MUSR": 0.4521, - "hfopenllm_v2/MMLU-PRO": 0.5471 - } - }, - { - "id": "cognitivecomputations/dolphin-2.9.2-qwen2-7b", - "name": "dolphin-2.9.2-qwen2-7b", - "developer": "cognitivecomputations", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3535, - "hfopenllm_v2/BBH": 0.4894, - "hfopenllm_v2/MATH Level 5": 0.1344, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.4191, - "hfopenllm_v2/MMLU-PRO": 0.4051 - } - }, - { - "id": "cognitivecomputations/dolphin-2.9.3-mistral-7B-32k", - "name": "dolphin-2.9.3-mistral-7B-32k", - "developer": "cognitivecomputations", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4126, - "hfopenllm_v2/BBH": 0.4813, - "hfopenllm_v2/MATH Level 5": 0.0506, - "hfopenllm_v2/GPQA": 0.2852, - "hfopenllm_v2/MUSR": 0.4643, - "hfopenllm_v2/MMLU-PRO": 0.2821 - } - }, - { - "id": "cognitivecomputations/dolphin-2.9.3-mistral-nemo-12b", - "name": "dolphin-2.9.3-mistral-nemo-12b", - "developer": "cognitivecomputations", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5601, - "hfopenllm_v2/BBH": 0.548, - "hfopenllm_v2/MATH Level 5": 0.074, - "hfopenllm_v2/GPQA": 0.3154, - "hfopenllm_v2/MUSR": 0.443, - "hfopenllm_v2/MMLU-PRO": 0.3377 - } - }, - { - "id": "cognitivecomputations/dolphin-2.9.3-Yi-1.5-34B-32k", - "name": "dolphin-2.9.3-Yi-1.5-34B-32k", - "developer": "cognitivecomputations", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3639, - "hfopenllm_v2/BBH": 0.6047, - "hfopenllm_v2/MATH Level 5": 0.1669, - "hfopenllm_v2/GPQA": 0.3431, - "hfopenllm_v2/MUSR": 0.4311, - "hfopenllm_v2/MMLU-PRO": 0.463 - } - }, - { - "id": "cognitivecomputations/dolphin-2.9.4-gemma2-2b", - "name": "dolphin-2.9.4-gemma2-2b", - "developer": "cognitivecomputations", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0896, - "hfopenllm_v2/BBH": 0.4081, - "hfopenllm_v2/MATH Level 5": 0.0491, - "hfopenllm_v2/GPQA": 0.2844, - "hfopenllm_v2/MUSR": 0.418, - "hfopenllm_v2/MMLU-PRO": 0.2105 - } - }, - { - "id": "cognitivecomputations/dolphin-2.9.4-llama3.1-8b", - "name": "dolphin-2.9.4-llama3.1-8b", - "developer": "cognitivecomputations", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2757, - "hfopenllm_v2/BBH": 0.3524, - "hfopenllm_v2/MATH Level 5": 0.0121, - "hfopenllm_v2/GPQA": 0.2634, - "hfopenllm_v2/MUSR": 0.3236, - "hfopenllm_v2/MMLU-PRO": 0.1237 - } - }, - { - "id": "cognitivecomputations/Dolphin3.0-Llama3.1-8B", - "name": "Dolphin3.0-Llama3.1-8B", - "developer": "cognitivecomputations", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7621, - "hfopenllm_v2/BBH": 0.4916, - "hfopenllm_v2/MATH Level 5": 0.1231, - "hfopenllm_v2/GPQA": 0.2827, - "hfopenllm_v2/MUSR": 0.3653, - "hfopenllm_v2/MMLU-PRO": 0.2992 - } - }, - { - "id": "cognitivecomputations/Dolphin3.0-Llama3.2-1B", - "name": "Dolphin3.0-Llama3.2-1B", - "developer": "cognitivecomputations", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5428, - "hfopenllm_v2/BBH": 0.3122, - "hfopenllm_v2/MATH Level 5": 0.0279, - "hfopenllm_v2/GPQA": 0.2299, - "hfopenllm_v2/MUSR": 0.3249, - "hfopenllm_v2/MMLU-PRO": 0.1375 - } - }, - { - "id": "cognitivecomputations/Dolphin3.0-Qwen2.5-0.5B", - "name": "Dolphin3.0-Qwen2.5-0.5B", - "developer": "cognitivecomputations", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4697, - "hfopenllm_v2/BBH": 0.3114, - "hfopenllm_v2/MATH Level 5": 0.0514, - "hfopenllm_v2/GPQA": 0.2349, - "hfopenllm_v2/MUSR": 0.3555, - "hfopenllm_v2/MMLU-PRO": 0.1413 - } - }, - { - "id": "cognitivecomputations/Dolphin3.0-R1-Mistral-24B", - "name": "Dolphin3.0-R1-Mistral-24B", - "developer": "cognitivecomputations", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4068, - "hfopenllm_v2/BBH": 0.536, - "hfopenllm_v2/MATH Level 5": 0.3119, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.3952, - "hfopenllm_v2/MMLU-PRO": 0.3005 - } - }, - { - "id": "Cohere March 2024", - "name": "Cohere March 2024", - "developer": "unknown", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.8511, - "reward-bench/Chat": 0.9469, - "reward-bench/Chat Hard": 0.6513, - "reward-bench/Safety": 0.877, - "reward-bench/Reasoning": 0.9817, - "reward-bench/Prior Sets (0.5 weight)": 0.7458 - } - }, - { - "id": "Cohere May 2024", - "name": "Cohere May 2024", - "developer": "unknown", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.8816, - "reward-bench/Chat": 0.9637, - "reward-bench/Chat Hard": 0.7127, - "reward-bench/Safety": 0.923, - "reward-bench/Reasoning": 0.9768, - "reward-bench/Prior Sets (0.5 weight)": 0.782 - } - }, - { - "id": "cohere/aya-expanse-32b", - "name": "aya-expanse-32b", - "developer": "cohere", - "evaluator_relationship": null, - "benchmark_scores": { - "global-mmlu-lite/Global MMLU Lite": 0.7353, - "global-mmlu-lite/Culturally Sensitive": 0.6891, - "global-mmlu-lite/Culturally Agnostic": 0.7815, - "global-mmlu-lite/Arabic": 0.7425, - "global-mmlu-lite/English": 0.7544, - "global-mmlu-lite/Bengali": 0.7343, - "global-mmlu-lite/German": 0.7425, - "global-mmlu-lite/French": 0.7325, - "global-mmlu-lite/Hindi": 0.7375, - "global-mmlu-lite/Indonesian": 0.7594, - "global-mmlu-lite/Italian": 0.7305, - "global-mmlu-lite/Japanese": 0.7419, - "global-mmlu-lite/Korean": 0.7525, - "global-mmlu-lite/Portuguese": 0.7544, - "global-mmlu-lite/Spanish": 0.7362, - "global-mmlu-lite/Swahili": 0.7071, - "global-mmlu-lite/Yoruba": 0.6942, - "global-mmlu-lite/Chinese": 0.743, - "global-mmlu-lite/Burmese": 0.7025 - } - }, - { - "id": "cohere/Cohere-Command-beta-52.4B", - "name": "Cohere Command beta 52.4B", - "developer": "cohere", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.874, - "helm_classic/MMLU": 0.452, - "helm_classic/BoolQ": 0.856, - "helm_classic/NarrativeQA": 0.752, - "helm_classic/NaturalQuestions (open-book)": 0.76, - "helm_classic/QuAC": 0.432, - "helm_classic/HellaSwag": 0.811, - "helm_classic/OpenbookQA": 0.582, - "helm_classic/TruthfulQA": 0.269, - "helm_classic/MS MARCO (TREC)": 0.762, - "helm_classic/CNN/DailyMail": 0.161, - "helm_classic/XSUM": 0.152, - "helm_classic/IMDB": 0.96, - "helm_classic/CivilComments": 0.601, - "helm_classic/RAFT": 0.667 - } - }, - { - "id": "cohere/Cohere-Command-beta-6.1B", - "name": "Cohere Command beta 6.1B", - "developer": "cohere", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.675, - "helm_classic/MMLU": 0.406, - "helm_classic/BoolQ": 0.798, - "helm_classic/NarrativeQA": 0.709, - "helm_classic/NaturalQuestions (open-book)": 0.717, - "helm_classic/QuAC": 0.375, - "helm_classic/HellaSwag": 0.752, - "helm_classic/OpenbookQA": 0.55, - "helm_classic/TruthfulQA": 0.203, - "helm_classic/MS MARCO (TREC)": 0.709, - "helm_classic/CNN/DailyMail": 0.153, - "helm_classic/XSUM": 0.122, - "helm_classic/IMDB": 0.961, - "helm_classic/CivilComments": 0.54, - "helm_classic/RAFT": 0.634 - } - }, - { - "id": "cohere/Cohere-large-v20220720-13.1B", - "name": "Cohere large v20220720 13.1B", - "developer": "cohere", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.372, - "helm_classic/MMLU": 0.324, - "helm_classic/BoolQ": 0.725, - "helm_classic/NarrativeQA": 0.625, - "helm_classic/NaturalQuestions (open-book)": 0.573, - "helm_classic/QuAC": 0.338, - "helm_classic/HellaSwag": 0.736, - "helm_classic/OpenbookQA": 0.542, - "helm_classic/TruthfulQA": 0.181, - "helm_classic/MS MARCO (TREC)": 0.33, - "helm_classic/CNN/DailyMail": 0.126, - "helm_classic/XSUM": 0.108, - "helm_classic/IMDB": 0.933, - "helm_classic/CivilComments": 0.507, - "helm_classic/RAFT": 0.596 - } - }, - { - "id": "cohere/Cohere-medium-v20220720-6.1B", - "name": "Cohere medium v20220720 6.1B", - "developer": "cohere", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.23, - "helm_classic/MMLU": 0.279, - "helm_classic/BoolQ": 0.659, - "helm_classic/NarrativeQA": 0.559, - "helm_classic/NaturalQuestions (open-book)": 0.504, - "helm_classic/QuAC": 0.279, - "helm_classic/HellaSwag": 0.706, - "helm_classic/OpenbookQA": 0.496, - "helm_classic/TruthfulQA": 0.19, - "helm_classic/MS MARCO (TREC)": 0.374, - "helm_classic/CNN/DailyMail": 0.077, - "helm_classic/XSUM": 0.087, - "helm_classic/IMDB": 0.935, - "helm_classic/CivilComments": 0.504, - "helm_classic/RAFT": 0.52 - } - }, - { - "id": "cohere/Cohere-medium-v20221108-6.1B", - "name": "Cohere medium v20221108 6.1B", - "developer": "cohere", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.312, - "helm_classic/MMLU": 0.254, - "helm_classic/BoolQ": 0.7, - "helm_classic/NarrativeQA": 0.61, - "helm_classic/NaturalQuestions (open-book)": 0.517, - "helm_classic/QuAC": 0.314, - "helm_classic/HellaSwag": 0.726, - "helm_classic/OpenbookQA": 0.538, - "helm_classic/TruthfulQA": 0.215, - "helm_classic/MS MARCO (TREC)": 0.373, - "helm_classic/CNN/DailyMail": 0.121, - "helm_classic/XSUM": 0.099, - "helm_classic/IMDB": 0.935, - "helm_classic/CivilComments": 0.5, - "helm_classic/RAFT": 0.591 - } - }, - { - "id": "cohere/Cohere-small-v20220720-410M", - "name": "Cohere small v20220720 410M", - "developer": "cohere", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.109, - "helm_classic/MMLU": 0.264, - "helm_classic/BoolQ": 0.457, - "helm_classic/NarrativeQA": 0.294, - "helm_classic/NaturalQuestions (open-book)": 0.309, - "helm_classic/QuAC": 0.219, - "helm_classic/HellaSwag": 0.483, - "helm_classic/OpenbookQA": 0.348, - "helm_classic/TruthfulQA": 0.217, - "helm_classic/MS MARCO (TREC)": 0.304, - "helm_classic/CNN/DailyMail": 0.063, - "helm_classic/XSUM": 0.033, - "helm_classic/IMDB": 0.578, - "helm_classic/CivilComments": 0.501, - "helm_classic/RAFT": 0.492 - } - }, - { - "id": "cohere/Cohere-xlarge-v20220609-52.4B", - "name": "Cohere xlarge v20220609 52.4B", - "developer": "cohere", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.56, - "helm_classic/MMLU": 0.353, - "helm_classic/BoolQ": 0.718, - "helm_classic/NarrativeQA": 0.65, - "helm_classic/NaturalQuestions (open-book)": 0.595, - "helm_classic/QuAC": 0.361, - "helm_classic/HellaSwag": 0.811, - "helm_classic/OpenbookQA": 0.55, - "helm_classic/TruthfulQA": 0.198, - "helm_classic/MS MARCO (TREC)": 0.459, - "helm_classic/CNN/DailyMail": 0.144, - "helm_classic/XSUM": 0.129, - "helm_classic/IMDB": 0.956, - "helm_classic/CivilComments": 0.532, - "helm_classic/RAFT": 0.633 - } - }, - { - "id": "cohere/Cohere-xlarge-v20221108-52.4B", - "name": "Cohere xlarge v20221108 52.4B", - "developer": "cohere", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.664, - "helm_classic/MMLU": 0.382, - "helm_classic/BoolQ": 0.762, - "helm_classic/NarrativeQA": 0.672, - "helm_classic/NaturalQuestions (open-book)": 0.628, - "helm_classic/QuAC": 0.374, - "helm_classic/HellaSwag": 0.81, - "helm_classic/OpenbookQA": 0.588, - "helm_classic/TruthfulQA": 0.169, - "helm_classic/MS MARCO (TREC)": 0.55, - "helm_classic/CNN/DailyMail": 0.153, - "helm_classic/XSUM": 0.153, - "helm_classic/IMDB": 0.956, - "helm_classic/CivilComments": 0.524, - "helm_classic/RAFT": 0.624 - } - }, - { - "id": "cohere/command", - "name": "Command", - "developer": "cohere", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.327, - "helm_lite/NarrativeQA": 0.749, - "helm_lite/NaturalQuestions (closed-book)": 0.391, - "helm_lite/OpenbookQA": 0.774, - "helm_lite/MMLU": 0.525, - "helm_lite/MATH": 0.236, - "helm_lite/GSM8K": 0.452, - "helm_lite/LegalBench": 0.578, - "helm_lite/MedQA": 0.445, - "helm_lite/WMT 2014": 0.088 - } - }, - { - "id": "cohere/command-a-03-2025", - "name": "command-a-03-2025", - "developer": "cohere", - "evaluator_relationship": null, - "benchmark_scores": { - "global-mmlu-lite/Global MMLU Lite": 0.8385, - "global-mmlu-lite/Culturally Sensitive": 0.7993, - "global-mmlu-lite/Culturally Agnostic": 0.8778, - "global-mmlu-lite/Arabic": 0.8425, - "global-mmlu-lite/English": 0.855, - "global-mmlu-lite/Bengali": 0.8225, - "global-mmlu-lite/German": 0.8425, - "global-mmlu-lite/French": 0.8375, - "global-mmlu-lite/Hindi": 0.8421, - "global-mmlu-lite/Indonesian": 0.8546, - "global-mmlu-lite/Italian": 0.8375, - "global-mmlu-lite/Japanese": 0.845, - "global-mmlu-lite/Korean": 0.85, - "global-mmlu-lite/Portuguese": 0.84, - "global-mmlu-lite/Spanish": 0.8525, - "global-mmlu-lite/Swahili": 0.8275, - "global-mmlu-lite/Yoruba": 0.815, - "global-mmlu-lite/Chinese": 0.835, - "global-mmlu-lite/Burmese": 0.8175 - } - }, - { - "id": "cohere/command-a-fc", - "name": "Command A (FC)", - "developer": "cohere", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 35.0, - "bfcl/bfcl.overall.overall_accuracy": 46.49, - "bfcl/bfcl.overall.total_cost_usd": 91.37, - "bfcl/bfcl.overall.latency_mean_s": 2.09, - "bfcl/bfcl.overall.latency_std_s": 7.36, - "bfcl/bfcl.overall.latency_p95_s": 4.94, - "bfcl/bfcl.non_live.ast_accuracy": 87.56, - "bfcl/bfcl.non_live.simple_ast_accuracy": 75.75, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 93.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 93.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 88.0, - "bfcl/bfcl.live.live_accuracy": 78.53, - "bfcl/bfcl.live.live_simple_ast_accuracy": 85.66, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 76.92, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 81.25, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 70.83, - "bfcl/bfcl.multi_turn.accuracy": 29.5, - "bfcl/bfcl.multi_turn.base_accuracy": 38.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 23.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 32.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 25.0, - "bfcl/bfcl.web_search.accuracy": 46.5, - "bfcl/bfcl.web_search.base_accuracy": 60.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 33.0, - "bfcl/bfcl.memory.accuracy": 16.56, - "bfcl/bfcl.memory.kv_accuracy": 4.52, - "bfcl/bfcl.memory.vector_accuracy": 5.16, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 40.0, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 81.25, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 84.19 - } - }, - { - "id": "cohere/command-a-reasoning-fc", - "name": "Command A Reasoning (FC)", - "developer": "cohere", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 13.0, - "bfcl/bfcl.overall.overall_accuracy": 57.06, - "bfcl/bfcl.overall.total_cost_usd": 3.04, - "bfcl/bfcl.overall.latency_mean_s": 3.44, - "bfcl/bfcl.overall.latency_std_s": 4.91, - "bfcl/bfcl.overall.latency_p95_s": 8.39, - "bfcl/bfcl.non_live.ast_accuracy": 86.27, - "bfcl/bfcl.non_live.simple_ast_accuracy": 73.58, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 93.5, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 89.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 88.5, - "bfcl/bfcl.live.live_accuracy": 78.61, - "bfcl/bfcl.live.live_simple_ast_accuracy": 80.23, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 78.35, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 75.0, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 75.0, - "bfcl/bfcl.multi_turn.accuracy": 50.12, - "bfcl/bfcl.multi_turn.base_accuracy": 61.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 41.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 49.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 48.5, - "bfcl/bfcl.web_search.accuracy": 55.5, - "bfcl/bfcl.web_search.base_accuracy": 65.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 46.0, - "bfcl/bfcl.memory.accuracy": 28.82, - "bfcl/bfcl.memory.kv_accuracy": 16.13, - "bfcl/bfcl.memory.vector_accuracy": 23.87, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 46.45, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 68.75, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 86.75 - } - }, - { - "id": "cohere/command-light", - "name": "Command Light", - "developer": "cohere", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.105, - "helm_lite/NarrativeQA": 0.629, - "helm_lite/NaturalQuestions (closed-book)": 0.195, - "helm_lite/OpenbookQA": 0.398, - "helm_lite/MMLU": 0.386, - "helm_lite/MATH": 0.098, - "helm_lite/GSM8K": 0.149, - "helm_lite/LegalBench": 0.397, - "helm_lite/MedQA": 0.312, - "helm_lite/WMT 2014": 0.023 - } - }, - { - "id": "cohere/command-r", - "name": "Command R", - "developer": "cohere", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.299, - "helm_lite/NarrativeQA": 0.742, - "helm_lite/NaturalQuestions (closed-book)": 0.352, - "helm_lite/OpenbookQA": 0.782, - "helm_lite/MMLU": 0.567, - "helm_lite/MATH": 0.266, - "helm_lite/GSM8K": 0.551, - "helm_lite/LegalBench": 0.507, - "helm_lite/MedQA": 0.555, - "helm_lite/WMT 2014": 0.149, - "helm_mmlu/MMLU All Subjects": 0.652, - "helm_mmlu/Abstract Algebra": 0.33, - "helm_mmlu/Anatomy": 0.615, - "helm_mmlu/College Physics": 0.382, - "helm_mmlu/Computer Security": 0.78, - "helm_mmlu/Econometrics": 0.456, - "helm_mmlu/Global Facts": 0.42, - "helm_mmlu/Jurisprudence": 0.796, - "helm_mmlu/Philosophy": 0.685, - "helm_mmlu/Professional Psychology": 0.681, - "helm_mmlu/Us Foreign Policy": 0.82, - "helm_mmlu/Astronomy": 0.743, - "helm_mmlu/Business Ethics": 0.63, - "helm_mmlu/Clinical Knowledge": 0.751, - "helm_mmlu/Conceptual Physics": 0.528, - "helm_mmlu/Electrical Engineering": 0.593, - "helm_mmlu/Elementary Mathematics": 0.437, - "helm_mmlu/Formal Logic": 0.405, - "helm_mmlu/High School World History": 0.84, - "helm_mmlu/Human Sexuality": 0.763, - "helm_mmlu/International Law": 0.802, - "helm_mmlu/Logical Fallacies": 0.798, - "helm_mmlu/Machine Learning": 0.446, - "helm_mmlu/Management": 0.796, - "helm_mmlu/Marketing": 0.872, - "helm_mmlu/Medical Genetics": 0.81, - "helm_mmlu/Miscellaneous": 0.848, - "helm_mmlu/Moral Scenarios": 0.451, - "helm_mmlu/Nutrition": 0.703, - "helm_mmlu/Prehistory": 0.728, - "helm_mmlu/Public Relations": 0.7, - "helm_mmlu/Security Studies": 0.714, - "helm_mmlu/Sociology": 0.866, - "helm_mmlu/Virology": 0.542, - "helm_mmlu/World Religions": 0.813, - "helm_mmlu/Mean win rate": 0.959 - } - }, - { - "id": "cohere/command-r-plus", - "name": "Command R Plus", - "developer": "cohere", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.441, - "helm_lite/NarrativeQA": 0.735, - "helm_lite/NaturalQuestions (closed-book)": 0.343, - "helm_lite/OpenbookQA": 0.828, - "helm_lite/MMLU": 0.59, - "helm_lite/MATH": 0.403, - "helm_lite/GSM8K": 0.738, - "helm_lite/LegalBench": 0.672, - "helm_lite/MedQA": 0.567, - "helm_lite/WMT 2014": 0.203, - "helm_mmlu/MMLU All Subjects": 0.694, - "helm_mmlu/Abstract Algebra": 0.21, - "helm_mmlu/Anatomy": 0.644, - "helm_mmlu/College Physics": 0.52, - "helm_mmlu/Computer Security": 0.74, - "helm_mmlu/Econometrics": 0.561, - "helm_mmlu/Global Facts": 0.5, - "helm_mmlu/Jurisprudence": 0.806, - "helm_mmlu/Philosophy": 0.695, - "helm_mmlu/Professional Psychology": 0.735, - "helm_mmlu/Us Foreign Policy": 0.89, - "helm_mmlu/Astronomy": 0.783, - "helm_mmlu/Business Ethics": 0.77, - "helm_mmlu/Clinical Knowledge": 0.743, - "helm_mmlu/Conceptual Physics": 0.591, - "helm_mmlu/Electrical Engineering": 0.71, - "helm_mmlu/Elementary Mathematics": 0.474, - "helm_mmlu/Formal Logic": 0.484, - "helm_mmlu/High School World History": 0.827, - "helm_mmlu/Human Sexuality": 0.786, - "helm_mmlu/International Law": 0.835, - "helm_mmlu/Logical Fallacies": 0.791, - "helm_mmlu/Machine Learning": 0.518, - "helm_mmlu/Management": 0.835, - "helm_mmlu/Marketing": 0.927, - "helm_mmlu/Medical Genetics": 0.77, - "helm_mmlu/Miscellaneous": 0.844, - "helm_mmlu/Moral Scenarios": 0.585, - "helm_mmlu/Nutrition": 0.742, - "helm_mmlu/Prehistory": 0.821, - "helm_mmlu/Public Relations": 0.709, - "helm_mmlu/Security Studies": 0.751, - "helm_mmlu/Sociology": 0.876, - "helm_mmlu/Virology": 0.56, - "helm_mmlu/World Religions": 0.842, - "helm_mmlu/Mean win rate": 0.825 - } - }, - { - "id": "cohere/command-r7b-fc", - "name": "Command R7B (FC)", - "developer": "cohere", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 61.0, - "bfcl/bfcl.overall.overall_accuracy": 32.07, - "bfcl/bfcl.overall.total_cost_usd": 1.5, - "bfcl/bfcl.overall.latency_mean_s": 1.38, - "bfcl/bfcl.overall.latency_std_s": 2.87, - "bfcl/bfcl.overall.latency_p95_s": 2.69, - "bfcl/bfcl.non_live.ast_accuracy": 80.96, - "bfcl/bfcl.non_live.simple_ast_accuracy": 67.33, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 89.5, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 85.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 81.5, - "bfcl/bfcl.live.live_accuracy": 69.06, - "bfcl/bfcl.live.live_simple_ast_accuracy": 62.79, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 70.94, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 43.75, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 70.83, - "bfcl/bfcl.multi_turn.accuracy": 8.25, - "bfcl/bfcl.multi_turn.base_accuracy": 12.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 0.5, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 10.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 10.0, - "bfcl/bfcl.web_search.accuracy": 27.0, - "bfcl/bfcl.web_search.base_accuracy": 43.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 11.0, - "bfcl/bfcl.memory.accuracy": 5.16, - "bfcl/bfcl.memory.kv_accuracy": 2.58, - "bfcl/bfcl.memory.vector_accuracy": 9.68, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 3.23, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 68.75, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 81.65 - } - }, - { - "id": "cohere/command-xlarge-beta", - "name": "Cohere Command beta 52.4B", - "developer": "cohere", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_instruct/Mean win rate": 0.089, - "helm_instruct/Anthropic RLHF dataset": 4.214, - "helm_instruct/Best ChatGPT Prompts": 4.988, - "helm_instruct/Koala test dataset": 4.969, - "helm_instruct/Open Assistant": 4.967, - "helm_instruct/Self Instruct": 4.971, - "helm_instruct/Vicuna": 4.995 - } - }, - { - "id": "CohereForAI/aya-23-35B", - "name": "aya-23-35B", - "developer": "CohereForAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6462, - "hfopenllm_v2/BBH": 0.54, - "hfopenllm_v2/MATH Level 5": 0.0347, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.431, - "hfopenllm_v2/MMLU-PRO": 0.3356 - } - }, - { - "id": "CohereForAI/aya-23-8B", - "name": "aya-23-8B", - "developer": "CohereForAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4699, - "hfopenllm_v2/BBH": 0.4296, - "hfopenllm_v2/MATH Level 5": 0.0166, - "hfopenllm_v2/GPQA": 0.2844, - "hfopenllm_v2/MUSR": 0.3941, - "hfopenllm_v2/MMLU-PRO": 0.2278 - } - }, - { - "id": "CohereForAI/aya-expanse-32b", - "name": "aya-expanse-32b", - "developer": "CohereForAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7302, - "hfopenllm_v2/BBH": 0.5649, - "hfopenllm_v2/MATH Level 5": 0.1533, - "hfopenllm_v2/GPQA": 0.3255, - "hfopenllm_v2/MUSR": 0.3873, - "hfopenllm_v2/MMLU-PRO": 0.413 - } - }, - { - "id": "CohereForAI/aya-expanse-8b", - "name": "aya-expanse-8b", - "developer": "CohereForAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6359, - "hfopenllm_v2/BBH": 0.4977, - "hfopenllm_v2/MATH Level 5": 0.0861, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.3729, - "hfopenllm_v2/MMLU-PRO": 0.3004 - } - }, - { - "id": "CohereForAI/c4ai-command-r-plus", - "name": "CohereForAI/c4ai-command-r-plus", - "developer": "CohereForAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7664, - "hfopenllm_v2/BBH": 0.5815, - "hfopenllm_v2/MATH Level 5": 0.0801, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.4807, - "hfopenllm_v2/MMLU-PRO": 0.3992, - "reward-bench/Score": 0.7057, - "reward-bench/Chat": 0.9511, - "reward-bench/Chat Hard": 0.5757, - "reward-bench/Safety": 0.5986, - "reward-bench/Reasoning": 0.704, - "reward-bench/Prior Sets (0.5 weight)": 0.6924 - } - }, - { - "id": "CohereForAI/c4ai-command-r-plus-08-2024", - "name": "c4ai-command-r-plus-08-2024", - "developer": "CohereForAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.754, - "hfopenllm_v2/BBH": 0.5996, - "hfopenllm_v2/MATH Level 5": 0.1239, - "hfopenllm_v2/GPQA": 0.3507, - "hfopenllm_v2/MUSR": 0.4829, - "hfopenllm_v2/MMLU-PRO": 0.4421 - } - }, - { - "id": "CohereForAI/c4ai-command-r-v01", - "name": "c4ai-command-r-v01", - "developer": "CohereForAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6748, - "hfopenllm_v2/BBH": 0.5406, - "hfopenllm_v2/MATH Level 5": 0.0347, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.4517, - "hfopenllm_v2/MMLU-PRO": 0.3369 - } - }, - { - "id": "CohereForAI/c4ai-command-r7b-12-2024", - "name": "c4ai-command-r7b-12-2024", - "developer": "CohereForAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7713, - "hfopenllm_v2/BBH": 0.5503, - "hfopenllm_v2/MATH Level 5": 0.2991, - "hfopenllm_v2/GPQA": 0.3087, - "hfopenllm_v2/MUSR": 0.4125, - "hfopenllm_v2/MMLU-PRO": 0.3572 - } - }, - { - "id": "collaiborateorg/Collaiborator-MEDLLM-Llama-3-8B-v2", - "name": "Collaiborator-MEDLLM-Llama-3-8B-v2", - "developer": "collaiborateorg", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3809, - "hfopenllm_v2/BBH": 0.4648, - "hfopenllm_v2/MATH Level 5": 0.0566, - "hfopenllm_v2/GPQA": 0.3331, - "hfopenllm_v2/MUSR": 0.3434, - "hfopenllm_v2/MMLU-PRO": 0.3481 - } - }, - { - "id": "Columbia-NLP/LION-Gemma-2b-dpo-v1.0", - "name": "LION-Gemma-2b-dpo-v1.0", - "developer": "Columbia-NLP", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3278, - "hfopenllm_v2/BBH": 0.392, - "hfopenllm_v2/MATH Level 5": 0.0431, - "hfopenllm_v2/GPQA": 0.2492, - "hfopenllm_v2/MUSR": 0.412, - "hfopenllm_v2/MMLU-PRO": 0.1666 - } - }, - { - "id": "Columbia-NLP/LION-Gemma-2b-odpo-v1.0", - "name": "LION-Gemma-2b-odpo-v1.0", - "developer": "Columbia-NLP", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3066, - "hfopenllm_v2/BBH": 0.3896, - "hfopenllm_v2/MATH Level 5": 0.0695, - "hfopenllm_v2/GPQA": 0.2424, - "hfopenllm_v2/MUSR": 0.4279, - "hfopenllm_v2/MMLU-PRO": 0.1692 - } - }, - { - "id": "Columbia-NLP/LION-Gemma-2b-sft-v1.0", - "name": "LION-Gemma-2b-sft-v1.0", - "developer": "Columbia-NLP", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3692, - "hfopenllm_v2/BBH": 0.3879, - "hfopenllm_v2/MATH Level 5": 0.068, - "hfopenllm_v2/GPQA": 0.2559, - "hfopenllm_v2/MUSR": 0.4027, - "hfopenllm_v2/MMLU-PRO": 0.1782 - } - }, - { - "id": "Columbia-NLP/LION-LLaMA-3-8b-dpo-v1.0", - "name": "LION-LLaMA-3-8b-dpo-v1.0", - "developer": "Columbia-NLP", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4957, - "hfopenllm_v2/BBH": 0.5028, - "hfopenllm_v2/MATH Level 5": 0.1171, - "hfopenllm_v2/GPQA": 0.281, - "hfopenllm_v2/MUSR": 0.4097, - "hfopenllm_v2/MMLU-PRO": 0.3219 - } - }, - { - "id": "Columbia-NLP/LION-LLaMA-3-8b-odpo-v1.0", - "name": "LION-LLaMA-3-8b-odpo-v1.0", - "developer": "Columbia-NLP", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3968, - "hfopenllm_v2/BBH": 0.5024, - "hfopenllm_v2/MATH Level 5": 0.1065, - "hfopenllm_v2/GPQA": 0.2852, - "hfopenllm_v2/MUSR": 0.4057, - "hfopenllm_v2/MMLU-PRO": 0.3152 - } - }, - { - "id": "Columbia-NLP/LION-LLaMA-3-8b-sft-v1.0", - "name": "LION-LLaMA-3-8b-sft-v1.0", - "developer": "Columbia-NLP", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3817, - "hfopenllm_v2/BBH": 0.5088, - "hfopenllm_v2/MATH Level 5": 0.114, - "hfopenllm_v2/GPQA": 0.2777, - "hfopenllm_v2/MUSR": 0.4503, - "hfopenllm_v2/MMLU-PRO": 0.3237 - } - }, - { - "id": "CombinHorizon/huihui-ai-abliterated-Qwen2.5-32B-Inst-BaseMerge-TIES", - "name": "huihui-ai-abliterated-Qwen2.5-32B-Inst-BaseMerge-TIES", - "developer": "CombinHorizon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8206, - "hfopenllm_v2/BBH": 0.6929, - "hfopenllm_v2/MATH Level 5": 0.5944, - "hfopenllm_v2/GPQA": 0.3389, - "hfopenllm_v2/MUSR": 0.4207, - "hfopenllm_v2/MMLU-PRO": 0.5721 - } - }, - { - "id": "CombinHorizon/huihui-ai-abliteratedV2-Qwen2.5-14B-Inst-BaseMerge-TIES", - "name": "huihui-ai-abliteratedV2-Qwen2.5-14B-Inst-BaseMerge-TIES", - "developer": "CombinHorizon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8176, - "hfopenllm_v2/BBH": 0.6336, - "hfopenllm_v2/MATH Level 5": 0.5476, - "hfopenllm_v2/GPQA": 0.3146, - "hfopenllm_v2/MUSR": 0.426, - "hfopenllm_v2/MMLU-PRO": 0.491 - } - }, - { - "id": "CombinHorizon/Josiefied-abliteratedV4-Qwen2.5-14B-Inst-BaseMerge-TIES", - "name": "Josiefied-abliteratedV4-Qwen2.5-14B-Inst-BaseMerge-TIES", - "developer": "CombinHorizon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.824, - "hfopenllm_v2/BBH": 0.637, - "hfopenllm_v2/MATH Level 5": 0.5317, - "hfopenllm_v2/GPQA": 0.3247, - "hfopenllm_v2/MUSR": 0.426, - "hfopenllm_v2/MMLU-PRO": 0.4979 - } - }, - { - "id": "CombinHorizon/Rombos-Qwen2.5-7B-Inst-BaseMerge-TIES", - "name": "Rombos-Qwen2.5-7B-Inst-BaseMerge-TIES", - "developer": "CombinHorizon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7564, - "hfopenllm_v2/BBH": 0.5402, - "hfopenllm_v2/MATH Level 5": 0.4932, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.4033, - "hfopenllm_v2/MMLU-PRO": 0.4342 - } - }, - { - "id": "CombinHorizon/YiSM-blossom5.1-34B-SLERP", - "name": "YiSM-blossom5.1-34B-SLERP", - "developer": "CombinHorizon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5033, - "hfopenllm_v2/BBH": 0.6208, - "hfopenllm_v2/MATH Level 5": 0.2153, - "hfopenllm_v2/GPQA": 0.3557, - "hfopenllm_v2/MUSR": 0.4413, - "hfopenllm_v2/MMLU-PRO": 0.4741 - } - }, - { - "id": "CombinHorizon/zetasepic-abliteratedV2-Qwen2.5-32B-Inst-BaseMerge-TIES", - "name": "zetasepic-abliteratedV2-Qwen2.5-32B-Inst-BaseMerge-TIES", - "developer": "CombinHorizon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8328, - "hfopenllm_v2/BBH": 0.6955, - "hfopenllm_v2/MATH Level 5": 0.5853, - "hfopenllm_v2/GPQA": 0.3674, - "hfopenllm_v2/MUSR": 0.4314, - "hfopenllm_v2/MMLU-PRO": 0.5685 - } - }, - { - "id": "ContactDoctor/Bio-Medical-3B-CoT-012025", - "name": "Bio-Medical-3B-CoT-012025", - "developer": "ContactDoctor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3604, - "hfopenllm_v2/BBH": 0.4383, - "hfopenllm_v2/MATH Level 5": 0.2213, - "hfopenllm_v2/GPQA": 0.3045, - "hfopenllm_v2/MUSR": 0.3368, - "hfopenllm_v2/MMLU-PRO": 0.2934 - } - }, - { - "id": "ContactDoctor/Bio-Medical-Llama-3-8B", - "name": "Bio-Medical-Llama-3-8B", - "developer": "ContactDoctor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4422, - "hfopenllm_v2/BBH": 0.4863, - "hfopenllm_v2/MATH Level 5": 0.0672, - "hfopenllm_v2/GPQA": 0.3339, - "hfopenllm_v2/MUSR": 0.3514, - "hfopenllm_v2/MMLU-PRO": 0.3648 - } - }, - { - "id": "ContextualAI/archangel_sft-dpo_llama13b", - "name": "ContextualAI/archangel_sft-dpo_llama13b", - "developer": "ContextualAI", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.54, - "reward-bench/Chat": 0.7123, - "reward-bench/Chat Hard": 0.4298, - "reward-bench/Safety": 0.5649, - "reward-bench/Reasoning": 0.4401, - "reward-bench/Prior Sets (0.5 weight)": 0.5656 - } - }, - { - "id": "ContextualAI/archangel_sft-dpo_llama30b", - "name": "ContextualAI/archangel_sft-dpo_llama30b", - "developer": "ContextualAI", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5618, - "reward-bench/Chat": 0.6927, - "reward-bench/Chat Hard": 0.4474, - "reward-bench/Safety": 0.6284, - "reward-bench/Reasoning": 0.4745, - "reward-bench/Prior Sets (0.5 weight)": 0.5705 - } - }, - { - "id": "ContextualAI/archangel_sft-dpo_llama7b", - "name": "ContextualAI/archangel_sft-dpo_llama7b", - "developer": "ContextualAI", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5304, - "reward-bench/Chat": 0.5782, - "reward-bench/Chat Hard": 0.4452, - "reward-bench/Safety": 0.5203, - "reward-bench/Reasoning": 0.5658, - "reward-bench/Prior Sets (0.5 weight)": 0.5544 - } - }, - { - "id": "ContextualAI/archangel_sft-dpo_pythia1-4b", - "name": "ContextualAI/archangel_sft-dpo_pythia1-4b", - "developer": "ContextualAI", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5233, - "reward-bench/Chat": 0.6397, - "reward-bench/Chat Hard": 0.3728, - "reward-bench/Safety": 0.5041, - "reward-bench/Reasoning": 0.5672, - "reward-bench/Prior Sets (0.5 weight)": 0.5427 - } - }, - { - "id": "ContextualAI/archangel_sft-dpo_pythia12-0b", - "name": "ContextualAI/archangel_sft-dpo_pythia12-0b", - "developer": "ContextualAI", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5009, - "reward-bench/Chat": 0.6676, - "reward-bench/Chat Hard": 0.364, - "reward-bench/Safety": 0.5432, - "reward-bench/Reasoning": 0.4139, - "reward-bench/Prior Sets (0.5 weight)": 0.5303 - } - }, - { - "id": "ContextualAI/archangel_sft-dpo_pythia2-8b", - "name": "ContextualAI/archangel_sft-dpo_pythia2-8b", - "developer": "ContextualAI", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5286, - "reward-bench/Chat": 0.8073, - "reward-bench/Chat Hard": 0.3355, - "reward-bench/Safety": 0.4473, - "reward-bench/Reasoning": 0.5135, - "reward-bench/Prior Sets (0.5 weight)": 0.5501 - } - }, - { - "id": "ContextualAI/archangel_sft-dpo_pythia6-9b", - "name": "ContextualAI/archangel_sft-dpo_pythia6-9b", - "developer": "ContextualAI", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5263, - "reward-bench/Chat": 0.7486, - "reward-bench/Chat Hard": 0.3421, - "reward-bench/Safety": 0.5176, - "reward-bench/Reasoning": 0.4847, - "reward-bench/Prior Sets (0.5 weight)": 0.551 - } - }, - { - "id": "ContextualAI/archangel_sft-kto_llama13b", - "name": "ContextualAI/archangel_sft-kto_llama13b", - "developer": "ContextualAI", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5952, - "reward-bench/Chat": 0.8408, - "reward-bench/Chat Hard": 0.3772, - "reward-bench/Safety": 0.4649, - "reward-bench/Reasoning": 0.7077, - "reward-bench/Prior Sets (0.5 weight)": 0.576 - } - }, - { - "id": "ContextualAI/archangel_sft-kto_llama30b", - "name": "ContextualAI/archangel_sft-kto_llama30b", - "developer": "ContextualAI", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5901, - "reward-bench/Chat": 0.8436, - "reward-bench/Chat Hard": 0.4057, - "reward-bench/Safety": 0.6054, - "reward-bench/Reasoning": 0.5075, - "reward-bench/Prior Sets (0.5 weight)": 0.5862 - } - }, - { - "id": "ContextualAI/archangel_sft-kto_llama7b", - "name": "ContextualAI/archangel_sft-kto_llama7b", - "developer": "ContextualAI", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5388, - "reward-bench/Chat": 0.5587, - "reward-bench/Chat Hard": 0.4364, - "reward-bench/Safety": 0.4568, - "reward-bench/Reasoning": 0.6941, - "reward-bench/Prior Sets (0.5 weight)": 0.5575 - } - }, - { - "id": "ContextualAI/archangel_sft-kto_pythia1-4b", - "name": "ContextualAI/archangel_sft-kto_pythia1-4b", - "developer": "ContextualAI", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5581, - "reward-bench/Chat": 0.6844, - "reward-bench/Chat Hard": 0.3794, - "reward-bench/Safety": 0.5257, - "reward-bench/Reasoning": 0.6447, - "reward-bench/Prior Sets (0.5 weight)": 0.5546 - } - }, - { - "id": "ContextualAI/archangel_sft-kto_pythia12-0b", - "name": "ContextualAI/archangel_sft-kto_pythia12-0b", - "developer": "ContextualAI", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5053, - "reward-bench/Chat": 0.7486, - "reward-bench/Chat Hard": 0.3618, - "reward-bench/Safety": 0.4757, - "reward-bench/Reasoning": 0.4127, - "reward-bench/Prior Sets (0.5 weight)": 0.55 - } - }, - { - "id": "ContextualAI/archangel_sft-kto_pythia2-8b", - "name": "ContextualAI/archangel_sft-kto_pythia2-8b", - "developer": "ContextualAI", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5497, - "reward-bench/Chat": 0.757, - "reward-bench/Chat Hard": 0.3421, - "reward-bench/Safety": 0.4743, - "reward-bench/Reasoning": 0.6216, - "reward-bench/Prior Sets (0.5 weight)": 0.557 - } - }, - { - "id": "ContextualAI/archangel_sft-kto_pythia6-9b", - "name": "ContextualAI/archangel_sft-kto_pythia6-9b", - "developer": "ContextualAI", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5561, - "reward-bench/Chat": 0.7765, - "reward-bench/Chat Hard": 0.3618, - "reward-bench/Safety": 0.5365, - "reward-bench/Reasoning": 0.5415, - "reward-bench/Prior Sets (0.5 weight)": 0.5723 - } - }, - { - "id": "ContextualAI/LMUnit-llama3.1-70b", - "name": "ContextualAI/LMUnit-llama3.1-70b", - "developer": "ContextualAI", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.8054, - "reward-bench/Factuality": 0.8463, - "reward-bench/Precise IF": 0.4875, - "reward-bench/Math": 0.7158, - "reward-bench/Safety": 0.9067, - "reward-bench/Focus": 0.9697, - "reward-bench/Ties": 0.9063 - } - }, - { - "id": "ContextualAI/LMUnit-qwen2.5-72b", - "name": "ContextualAI/LMUnit-qwen2.5-72b", - "developer": "ContextualAI", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.8208, - "reward-bench/Factuality": 0.8716, - "reward-bench/Precise IF": 0.5437, - "reward-bench/Math": 0.7268, - "reward-bench/Safety": 0.9133, - "reward-bench/Focus": 0.9677, - "reward-bench/Ties": 0.9014 - } - }, - { - "id": "CoolSpring/Qwen2-0.5B-Abyme", - "name": "Qwen2-0.5B-Abyme", - "developer": "CoolSpring", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1915, - "hfopenllm_v2/BBH": 0.2862, - "hfopenllm_v2/MATH Level 5": 0.0295, - "hfopenllm_v2/GPQA": 0.2534, - "hfopenllm_v2/MUSR": 0.3542, - "hfopenllm_v2/MMLU-PRO": 0.1333 - } - }, - { - "id": "CoolSpring/Qwen2-0.5B-Abyme-merge2", - "name": "Qwen2-0.5B-Abyme-merge2", - "developer": "CoolSpring", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2022, - "hfopenllm_v2/BBH": 0.2994, - "hfopenllm_v2/MATH Level 5": 0.0332, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3687, - "hfopenllm_v2/MMLU-PRO": 0.1489 - } - }, - { - "id": "CoolSpring/Qwen2-0.5B-Abyme-merge3", - "name": "Qwen2-0.5B-Abyme-merge3", - "developer": "CoolSpring", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2386, - "hfopenllm_v2/BBH": 0.3003, - "hfopenllm_v2/MATH Level 5": 0.0317, - "hfopenllm_v2/GPQA": 0.2643, - "hfopenllm_v2/MUSR": 0.3501, - "hfopenllm_v2/MMLU-PRO": 0.15 - } - }, - { - "id": "Corianas/llama-3-reactor", - "name": "llama-3-reactor", - "developer": "Corianas", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.23, - "hfopenllm_v2/BBH": 0.4457, - "hfopenllm_v2/MATH Level 5": 0.0468, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.3977, - "hfopenllm_v2/MMLU-PRO": 0.2801 - } - }, - { - "id": "Corianas/Neural-Mistral-7B", - "name": "Neural-Mistral-7B", - "developer": "Corianas", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5489, - "hfopenllm_v2/BBH": 0.4428, - "hfopenllm_v2/MATH Level 5": 0.0189, - "hfopenllm_v2/GPQA": 0.2836, - "hfopenllm_v2/MUSR": 0.3873, - "hfopenllm_v2/MMLU-PRO": 0.2738 - } - }, - { - "id": "Corianas/Quokka_2.7b", - "name": "Quokka_2.7b", - "developer": "Corianas", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1749, - "hfopenllm_v2/BBH": 0.3055, - "hfopenllm_v2/MATH Level 5": 0.0083, - "hfopenllm_v2/GPQA": 0.2559, - "hfopenllm_v2/MUSR": 0.3908, - "hfopenllm_v2/MMLU-PRO": 0.1145 - } - }, - { - "id": "CortexLM/btlm-7b-base-v0.2", - "name": "btlm-7b-base-v0.2", - "developer": "CortexLM", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1483, - "hfopenllm_v2/BBH": 0.4006, - "hfopenllm_v2/MATH Level 5": 0.0151, - "hfopenllm_v2/GPQA": 0.2534, - "hfopenllm_v2/MUSR": 0.3846, - "hfopenllm_v2/MMLU-PRO": 0.235 - } - }, - { - "id": "cpayne1303/cp2024", - "name": "cp2024", - "developer": "cpayne1303", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1658, - "hfopenllm_v2/BBH": 0.2985, - "hfopenllm_v2/MATH Level 5": 0.0053, - "hfopenllm_v2/GPQA": 0.2559, - "hfopenllm_v2/MUSR": 0.3383, - "hfopenllm_v2/MMLU-PRO": 0.1101 - } - }, - { - "id": "cpayne1303/cp2024-instruct", - "name": "cp2024-instruct", - "developer": "cpayne1303", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1706, - "hfopenllm_v2/BBH": 0.2947, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3686, - "hfopenllm_v2/MMLU-PRO": 0.1167 - } - }, - { - "id": "cpayne1303/llama-43m-beta", - "name": "llama-43m-beta", - "developer": "cpayne1303", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1916, - "hfopenllm_v2/BBH": 0.2977, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2685, - "hfopenllm_v2/MUSR": 0.3872, - "hfopenllm_v2/MMLU-PRO": 0.1132 - } - }, - { - "id": "cpayne1303/smallcp2024", - "name": "smallcp2024", - "developer": "cpayne1303", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1582, - "hfopenllm_v2/BBH": 0.3027, - "hfopenllm_v2/MATH Level 5": 0.0053, - "hfopenllm_v2/GPQA": 0.2307, - "hfopenllm_v2/MUSR": 0.3425, - "hfopenllm_v2/MMLU-PRO": 0.1114 - } - }, - { - "id": "Cran-May/merge_model_20250308_2", - "name": "merge_model_20250308_2", - "developer": "Cran-May", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5932, - "hfopenllm_v2/BBH": 0.6585, - "hfopenllm_v2/MATH Level 5": 0.4381, - "hfopenllm_v2/GPQA": 0.3909, - "hfopenllm_v2/MUSR": 0.4794, - "hfopenllm_v2/MMLU-PRO": 0.542 - } - }, - { - "id": "Cran-May/merge_model_20250308_3", - "name": "merge_model_20250308_3", - "developer": "Cran-May", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6018, - "hfopenllm_v2/BBH": 0.6271, - "hfopenllm_v2/MATH Level 5": 0.2545, - "hfopenllm_v2/GPQA": 0.3221, - "hfopenllm_v2/MUSR": 0.432, - "hfopenllm_v2/MMLU-PRO": 0.4962 - } - }, - { - "id": "Cran-May/merge_model_20250308_4", - "name": "merge_model_20250308_4", - "developer": "Cran-May", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.454, - "hfopenllm_v2/BBH": 0.6664, - "hfopenllm_v2/MATH Level 5": 0.4199, - "hfopenllm_v2/GPQA": 0.3977, - "hfopenllm_v2/MUSR": 0.4688, - "hfopenllm_v2/MMLU-PRO": 0.5367 - } - }, - { - "id": "Cran-May/SCE-2-24B", - "name": "SCE-2-24B", - "developer": "Cran-May", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5866, - "hfopenllm_v2/BBH": 0.6265, - "hfopenllm_v2/MATH Level 5": 0.1896, - "hfopenllm_v2/GPQA": 0.3372, - "hfopenllm_v2/MUSR": 0.4528, - "hfopenllm_v2/MMLU-PRO": 0.4612 - } - }, - { - "id": "Cran-May/SCE-3-24B", - "name": "SCE-3-24B", - "developer": "Cran-May", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5465, - "hfopenllm_v2/BBH": 0.5973, - "hfopenllm_v2/MATH Level 5": 0.1881, - "hfopenllm_v2/GPQA": 0.3465, - "hfopenllm_v2/MUSR": 0.4435, - "hfopenllm_v2/MMLU-PRO": 0.4647 - } - }, - { - "id": "Cran-May/T.E-8.1", - "name": "T.E-8.1", - "developer": "Cran-May", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7077, - "hfopenllm_v2/BBH": 0.5582, - "hfopenllm_v2/MATH Level 5": 0.4456, - "hfopenllm_v2/GPQA": 0.3129, - "hfopenllm_v2/MUSR": 0.4505, - "hfopenllm_v2/MMLU-PRO": 0.4432 - } - }, - { - "id": "Cran-May/tempmotacilla-cinerea-0308", - "name": "tempmotacilla-cinerea-0308", - "developer": "Cran-May", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8085, - "hfopenllm_v2/BBH": 0.6551, - "hfopenllm_v2/MATH Level 5": 0.5551, - "hfopenllm_v2/GPQA": 0.3624, - "hfopenllm_v2/MUSR": 0.4208, - "hfopenllm_v2/MMLU-PRO": 0.525 - } - }, - { - "id": "CreitinGameplays/Llama-3.1-8B-R1-v0.1", - "name": "Llama-3.1-8B-R1-v0.1", - "developer": "CreitinGameplays", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3235, - "hfopenllm_v2/BBH": 0.3057, - "hfopenllm_v2/MATH Level 5": 0.1813, - "hfopenllm_v2/GPQA": 0.2584, - "hfopenllm_v2/MUSR": 0.3622, - "hfopenllm_v2/MMLU-PRO": 0.1252 - } - }, - { - "id": "crestf411/MN-Slush", - "name": "MN-Slush", - "developer": "crestf411", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4077, - "hfopenllm_v2/BBH": 0.534, - "hfopenllm_v2/MATH Level 5": 0.1269, - "hfopenllm_v2/GPQA": 0.3238, - "hfopenllm_v2/MUSR": 0.3933, - "hfopenllm_v2/MMLU-PRO": 0.3508 - } - }, - { - "id": "cstr/llama3.1-8b-spaetzle-v90", - "name": "llama3.1-8b-spaetzle-v90", - "developer": "cstr", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7356, - "hfopenllm_v2/BBH": 0.5303, - "hfopenllm_v2/MATH Level 5": 0.1495, - "hfopenllm_v2/GPQA": 0.2827, - "hfopenllm_v2/MUSR": 0.4134, - "hfopenllm_v2/MMLU-PRO": 0.3731 - } - }, - { - "id": "CultriX/Qwen2.5-14B-Broca", - "name": "Qwen2.5-14B-Broca", - "developer": "CultriX", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5604, - "hfopenllm_v2/BBH": 0.6527, - "hfopenllm_v2/MATH Level 5": 0.358, - "hfopenllm_v2/GPQA": 0.3867, - "hfopenllm_v2/MUSR": 0.4767, - "hfopenllm_v2/MMLU-PRO": 0.5364 - } - }, - { - "id": "CultriX/Qwen2.5-14B-Brocav3", - "name": "Qwen2.5-14B-Brocav3", - "developer": "CultriX", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6952, - "hfopenllm_v2/BBH": 0.6452, - "hfopenllm_v2/MATH Level 5": 0.3875, - "hfopenllm_v2/GPQA": 0.3591, - "hfopenllm_v2/MUSR": 0.4756, - "hfopenllm_v2/MMLU-PRO": 0.5317 - } - }, - { - "id": "CultriX/Qwen2.5-14B-Brocav6", - "name": "Qwen2.5-14B-Brocav6", - "developer": "CultriX", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6995, - "hfopenllm_v2/BBH": 0.6389, - "hfopenllm_v2/MATH Level 5": 0.3875, - "hfopenllm_v2/GPQA": 0.3674, - "hfopenllm_v2/MUSR": 0.4742, - "hfopenllm_v2/MMLU-PRO": 0.5319 - } - }, - { - "id": "CultriX/Qwen2.5-14B-Brocav7", - "name": "Qwen2.5-14B-Brocav7", - "developer": "CultriX", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6724, - "hfopenllm_v2/BBH": 0.6444, - "hfopenllm_v2/MATH Level 5": 0.3844, - "hfopenllm_v2/GPQA": 0.3674, - "hfopenllm_v2/MUSR": 0.4796, - "hfopenllm_v2/MMLU-PRO": 0.5258 - } - }, - { - "id": "CultriX/Qwen2.5-14B-BrocaV9", - "name": "Qwen2.5-14B-BrocaV9", - "developer": "CultriX", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6763, - "hfopenllm_v2/BBH": 0.6391, - "hfopenllm_v2/MATH Level 5": 0.3814, - "hfopenllm_v2/GPQA": 0.3641, - "hfopenllm_v2/MUSR": 0.469, - "hfopenllm_v2/MMLU-PRO": 0.5331 - } - }, - { - "id": "CultriX/Qwen2.5-14B-Emerged", - "name": "Qwen2.5-14B-Emerged", - "developer": "CultriX", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7, - "hfopenllm_v2/BBH": 0.626, - "hfopenllm_v2/MATH Level 5": 0.3248, - "hfopenllm_v2/GPQA": 0.3574, - "hfopenllm_v2/MUSR": 0.4691, - "hfopenllm_v2/MMLU-PRO": 0.5186 - } - }, - { - "id": "CultriX/Qwen2.5-14B-Emergedv3", - "name": "Qwen2.5-14B-Emergedv3", - "developer": "CultriX", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6388, - "hfopenllm_v2/BBH": 0.6191, - "hfopenllm_v2/MATH Level 5": 0.4358, - "hfopenllm_v2/GPQA": 0.3607, - "hfopenllm_v2/MUSR": 0.4728, - "hfopenllm_v2/MMLU-PRO": 0.5174 - } - }, - { - "id": "CultriX/Qwen2.5-14B-FinalMerge", - "name": "Qwen2.5-14B-FinalMerge", - "developer": "CultriX", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4891, - "hfopenllm_v2/BBH": 0.5715, - "hfopenllm_v2/MATH Level 5": 0.3814, - "hfopenllm_v2/GPQA": 0.3549, - "hfopenllm_v2/MUSR": 0.4379, - "hfopenllm_v2/MMLU-PRO": 0.4574 - } - }, - { - "id": "CultriX/Qwen2.5-14B-Hyper", - "name": "Qwen2.5-14B-Hyper", - "developer": "CultriX", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5391, - "hfopenllm_v2/BBH": 0.6507, - "hfopenllm_v2/MATH Level 5": 0.3437, - "hfopenllm_v2/GPQA": 0.3918, - "hfopenllm_v2/MUSR": 0.4898, - "hfopenllm_v2/MMLU-PRO": 0.5374 - } - }, - { - "id": "CultriX/Qwen2.5-14B-Hyperionv3", - "name": "Qwen2.5-14B-Hyperionv3", - "developer": "CultriX", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6836, - "hfopenllm_v2/BBH": 0.6522, - "hfopenllm_v2/MATH Level 5": 0.3701, - "hfopenllm_v2/GPQA": 0.3708, - "hfopenllm_v2/MUSR": 0.473, - "hfopenllm_v2/MMLU-PRO": 0.534 - } - }, - { - "id": "CultriX/Qwen2.5-14B-Hyperionv4", - "name": "Qwen2.5-14B-Hyperionv4", - "developer": "CultriX", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5416, - "hfopenllm_v2/BBH": 0.6472, - "hfopenllm_v2/MATH Level 5": 0.3474, - "hfopenllm_v2/GPQA": 0.3977, - "hfopenllm_v2/MUSR": 0.4832, - "hfopenllm_v2/MMLU-PRO": 0.5364 - } - }, - { - "id": "CultriX/Qwen2.5-14B-Hyperionv5", - "name": "Qwen2.5-14B-Hyperionv5", - "developer": "CultriX", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6729, - "hfopenllm_v2/BBH": 0.6443, - "hfopenllm_v2/MATH Level 5": 0.3822, - "hfopenllm_v2/GPQA": 0.3716, - "hfopenllm_v2/MUSR": 0.4795, - "hfopenllm_v2/MMLU-PRO": 0.5302 - } - }, - { - "id": "CultriX/Qwen2.5-14B-HyperMarck-dl", - "name": "Qwen2.5-14B-HyperMarck-dl", - "developer": "CultriX", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.665, - "hfopenllm_v2/BBH": 0.6096, - "hfopenllm_v2/MATH Level 5": 0.5287, - "hfopenllm_v2/GPQA": 0.3674, - "hfopenllm_v2/MUSR": 0.4416, - "hfopenllm_v2/MMLU-PRO": 0.5091 - } - }, - { - "id": "CultriX/Qwen2.5-14B-MegaMerge-pt2", - "name": "Qwen2.5-14B-MegaMerge-pt2", - "developer": "CultriX", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5683, - "hfopenllm_v2/BBH": 0.6578, - "hfopenllm_v2/MATH Level 5": 0.3995, - "hfopenllm_v2/GPQA": 0.3792, - "hfopenllm_v2/MUSR": 0.4729, - "hfopenllm_v2/MMLU-PRO": 0.5421 - } - }, - { - "id": "CultriX/Qwen2.5-14B-MergeStock", - "name": "Qwen2.5-14B-MergeStock", - "developer": "CultriX", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5685, - "hfopenllm_v2/BBH": 0.6579, - "hfopenllm_v2/MATH Level 5": 0.4147, - "hfopenllm_v2/GPQA": 0.3733, - "hfopenllm_v2/MUSR": 0.4676, - "hfopenllm_v2/MMLU-PRO": 0.5396 - } - }, - { - "id": "CultriX/Qwen2.5-14B-partialmergept1", - "name": "Qwen2.5-14B-partialmergept1", - "developer": "CultriX", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6337, - "hfopenllm_v2/BBH": 0.6151, - "hfopenllm_v2/MATH Level 5": 0.4539, - "hfopenllm_v2/GPQA": 0.3616, - "hfopenllm_v2/MUSR": 0.4757, - "hfopenllm_v2/MMLU-PRO": 0.5208 - } - }, - { - "id": "CultriX/Qwen2.5-14B-ReasoningMerge", - "name": "Qwen2.5-14B-ReasoningMerge", - "developer": "CultriX", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4605, - "hfopenllm_v2/BBH": 0.6578, - "hfopenllm_v2/MATH Level 5": 0.5204, - "hfopenllm_v2/GPQA": 0.4077, - "hfopenllm_v2/MUSR": 0.5166, - "hfopenllm_v2/MMLU-PRO": 0.5345 - } - }, - { - "id": "CultriX/Qwen2.5-14B-Ultimav2", - "name": "Qwen2.5-14B-Ultimav2", - "developer": "CultriX", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.55, - "hfopenllm_v2/BBH": 0.6555, - "hfopenllm_v2/MATH Level 5": 0.3844, - "hfopenllm_v2/GPQA": 0.3851, - "hfopenllm_v2/MUSR": 0.4966, - "hfopenllm_v2/MMLU-PRO": 0.5417 - } - }, - { - "id": "CultriX/Qwen2.5-14B-Unity", - "name": "Qwen2.5-14B-Unity", - "developer": "CultriX", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6739, - "hfopenllm_v2/BBH": 0.602, - "hfopenllm_v2/MATH Level 5": 0.4313, - "hfopenllm_v2/GPQA": 0.3473, - "hfopenllm_v2/MUSR": 0.4679, - "hfopenllm_v2/MMLU-PRO": 0.5076 - } - }, - { - "id": "CultriX/Qwen2.5-14B-Wernicke", - "name": "Qwen2.5-14B-Wernicke", - "developer": "CultriX", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5235, - "hfopenllm_v2/BBH": 0.6568, - "hfopenllm_v2/MATH Level 5": 0.3814, - "hfopenllm_v2/GPQA": 0.3935, - "hfopenllm_v2/MUSR": 0.4689, - "hfopenllm_v2/MMLU-PRO": 0.5424 - } - }, - { - "id": "CultriX/Qwen2.5-14B-Wernicke-SFT", - "name": "Qwen2.5-14B-Wernicke-SFT", - "developer": "CultriX", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4937, - "hfopenllm_v2/BBH": 0.6461, - "hfopenllm_v2/MATH Level 5": 0.3595, - "hfopenllm_v2/GPQA": 0.354, - "hfopenllm_v2/MUSR": 0.39, - "hfopenllm_v2/MMLU-PRO": 0.507 - } - }, - { - "id": "CultriX/Qwen2.5-14B-Wernicke-SLERP", - "name": "Qwen2.5-14B-Wernicke-SLERP", - "developer": "CultriX", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5589, - "hfopenllm_v2/BBH": 0.6441, - "hfopenllm_v2/MATH Level 5": 0.4486, - "hfopenllm_v2/GPQA": 0.344, - "hfopenllm_v2/MUSR": 0.414, - "hfopenllm_v2/MMLU-PRO": 0.5094 - } - }, - { - "id": "CultriX/Qwen2.5-14B-Wernickev3", - "name": "Qwen2.5-14B-Wernickev3", - "developer": "CultriX", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7048, - "hfopenllm_v2/BBH": 0.6184, - "hfopenllm_v2/MATH Level 5": 0.3542, - "hfopenllm_v2/GPQA": 0.3624, - "hfopenllm_v2/MUSR": 0.4717, - "hfopenllm_v2/MMLU-PRO": 0.5151 - } - }, - { - "id": "CultriX/Qwenfinity-2.5-14B", - "name": "Qwenfinity-2.5-14B", - "developer": "CultriX", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4814, - "hfopenllm_v2/BBH": 0.5655, - "hfopenllm_v2/MATH Level 5": 0.4101, - "hfopenllm_v2/GPQA": 0.349, - "hfopenllm_v2/MUSR": 0.4506, - "hfopenllm_v2/MMLU-PRO": 0.4498 - } - }, - { - "id": "CultriX/Qwestion-14B", - "name": "Qwestion-14B", - "developer": "CultriX", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6318, - "hfopenllm_v2/BBH": 0.645, - "hfopenllm_v2/MATH Level 5": 0.3724, - "hfopenllm_v2/GPQA": 0.3683, - "hfopenllm_v2/MUSR": 0.4636, - "hfopenllm_v2/MMLU-PRO": 0.5422 - } - }, - { - "id": "CultriX/SeQwence-14B", - "name": "SeQwence-14B", - "developer": "CultriX", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5352, - "hfopenllm_v2/BBH": 0.6506, - "hfopenllm_v2/MATH Level 5": 0.3535, - "hfopenllm_v2/GPQA": 0.3607, - "hfopenllm_v2/MUSR": 0.4666, - "hfopenllm_v2/MMLU-PRO": 0.5419 - } - }, - { - "id": "CultriX/SeQwence-14B-EvolMerge", - "name": "SeQwence-14B-EvolMerge", - "developer": "CultriX", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5382, - "hfopenllm_v2/BBH": 0.6572, - "hfopenllm_v2/MATH Level 5": 0.3671, - "hfopenllm_v2/GPQA": 0.3809, - "hfopenllm_v2/MUSR": 0.4821, - "hfopenllm_v2/MMLU-PRO": 0.5419 - } - }, - { - "id": "CultriX/SeQwence-14B-EvolMergev1", - "name": "SeQwence-14B-EvolMergev1", - "developer": "CultriX", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5555, - "hfopenllm_v2/BBH": 0.6546, - "hfopenllm_v2/MATH Level 5": 0.4215, - "hfopenllm_v2/GPQA": 0.3767, - "hfopenllm_v2/MUSR": 0.4623, - "hfopenllm_v2/MMLU-PRO": 0.5393 - } - }, - { - "id": "CultriX/SeQwence-14B-v5", - "name": "SeQwence-14B-v5", - "developer": "CultriX", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.592, - "hfopenllm_v2/BBH": 0.6517, - "hfopenllm_v2/MATH Level 5": 0.3308, - "hfopenllm_v2/GPQA": 0.37, - "hfopenllm_v2/MUSR": 0.4714, - "hfopenllm_v2/MMLU-PRO": 0.5415 - } - }, - { - "id": "CultriX/SeQwence-14Bv1", - "name": "SeQwence-14Bv1", - "developer": "CultriX", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6678, - "hfopenllm_v2/BBH": 0.6345, - "hfopenllm_v2/MATH Level 5": 0.361, - "hfopenllm_v2/GPQA": 0.3616, - "hfopenllm_v2/MUSR": 0.4704, - "hfopenllm_v2/MMLU-PRO": 0.532 - } - }, - { - "id": "CultriX/SeQwence-14Bv2", - "name": "SeQwence-14Bv2", - "developer": "CultriX", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5786, - "hfopenllm_v2/BBH": 0.6305, - "hfopenllm_v2/MATH Level 5": 0.4758, - "hfopenllm_v2/GPQA": 0.3607, - "hfopenllm_v2/MUSR": 0.4601, - "hfopenllm_v2/MMLU-PRO": 0.5334 - } - }, - { - "id": "CultriX/SeQwence-14Bv3", - "name": "SeQwence-14Bv3", - "developer": "CultriX", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5719, - "hfopenllm_v2/BBH": 0.6302, - "hfopenllm_v2/MATH Level 5": 0.4766, - "hfopenllm_v2/GPQA": 0.3649, - "hfopenllm_v2/MUSR": 0.4624, - "hfopenllm_v2/MMLU-PRO": 0.5335 - } - }, - { - "id": "cyberagent/calm3-22b-chat", - "name": "calm3-22b-chat", - "developer": "cyberagent", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5091, - "hfopenllm_v2/BBH": 0.4992, - "hfopenllm_v2/MATH Level 5": 0.0695, - "hfopenllm_v2/GPQA": 0.2768, - "hfopenllm_v2/MUSR": 0.4553, - "hfopenllm_v2/MMLU-PRO": 0.295 - } - }, - { - "id": "CYFRAGOVPL/Llama-PLLuM-8B-base", - "name": "Llama-PLLuM-8B-base", - "developer": "CYFRAGOVPL", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2899, - "hfopenllm_v2/BBH": 0.432, - "hfopenllm_v2/MATH Level 5": 0.0363, - "hfopenllm_v2/GPQA": 0.2852, - "hfopenllm_v2/MUSR": 0.397, - "hfopenllm_v2/MMLU-PRO": 0.2757 - } - }, - { - "id": "CYFRAGOVPL/Llama-PLLuM-8B-chat", - "name": "Llama-PLLuM-8B-chat", - "developer": "CYFRAGOVPL", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3515, - "hfopenllm_v2/BBH": 0.4077, - "hfopenllm_v2/MATH Level 5": 0.034, - "hfopenllm_v2/GPQA": 0.2643, - "hfopenllm_v2/MUSR": 0.4199, - "hfopenllm_v2/MMLU-PRO": 0.2719 - } - }, - { - "id": "CYFRAGOVPL/PLLuM-12B-base", - "name": "PLLuM-12B-base", - "developer": "CYFRAGOVPL", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2821, - "hfopenllm_v2/BBH": 0.4391, - "hfopenllm_v2/MATH Level 5": 0.0287, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.4142, - "hfopenllm_v2/MMLU-PRO": 0.274 - } - }, - { - "id": "CYFRAGOVPL/PLLuM-12B-chat", - "name": "PLLuM-12B-chat", - "developer": "CYFRAGOVPL", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3214, - "hfopenllm_v2/BBH": 0.4446, - "hfopenllm_v2/MATH Level 5": 0.0181, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.4115, - "hfopenllm_v2/MMLU-PRO": 0.2872 - } - }, - { - "id": "CYFRAGOVPL/PLLuM-12B-nc-base", - "name": "PLLuM-12B-nc-base", - "developer": "CYFRAGOVPL", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2405, - "hfopenllm_v2/BBH": 0.4277, - "hfopenllm_v2/MATH Level 5": 0.0219, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.3645, - "hfopenllm_v2/MMLU-PRO": 0.2559 - } - }, - { - "id": "CYFRAGOVPL/PLLuM-12B-nc-chat", - "name": "PLLuM-12B-nc-chat", - "developer": "CYFRAGOVPL", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2834, - "hfopenllm_v2/BBH": 0.4576, - "hfopenllm_v2/MATH Level 5": 0.0121, - "hfopenllm_v2/GPQA": 0.2827, - "hfopenllm_v2/MUSR": 0.4354, - "hfopenllm_v2/MMLU-PRO": 0.2597 - } - }, - { - "id": "Daemontatox/AetherDrake-SFT", - "name": "AetherDrake-SFT", - "developer": "Daemontatox", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4813, - "hfopenllm_v2/BBH": 0.4872, - "hfopenllm_v2/MATH Level 5": 0.1511, - "hfopenllm_v2/GPQA": 0.3205, - "hfopenllm_v2/MUSR": 0.4088, - "hfopenllm_v2/MMLU-PRO": 0.3499 - } - }, - { - "id": "Daemontatox/AetherSett", - "name": "AetherSett", - "developer": "Daemontatox", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.537, - "hfopenllm_v2/BBH": 0.5452, - "hfopenllm_v2/MATH Level 5": 0.3973, - "hfopenllm_v2/GPQA": 0.3079, - "hfopenllm_v2/MUSR": 0.4603, - "hfopenllm_v2/MMLU-PRO": 0.4279 - } - }, - { - "id": "Daemontatox/AetherTOT", - "name": "AetherTOT", - "developer": "Daemontatox", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4398, - "hfopenllm_v2/BBH": 0.5066, - "hfopenllm_v2/MATH Level 5": 0.1488, - "hfopenllm_v2/GPQA": 0.3238, - "hfopenllm_v2/MUSR": 0.4079, - "hfopenllm_v2/MMLU-PRO": 0.3804 - } - }, - { - "id": "Daemontatox/AetherUncensored", - "name": "AetherUncensored", - "developer": "Daemontatox", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4042, - "hfopenllm_v2/BBH": 0.4463, - "hfopenllm_v2/MATH Level 5": 0.145, - "hfopenllm_v2/GPQA": 0.2886, - "hfopenllm_v2/MUSR": 0.3747, - "hfopenllm_v2/MMLU-PRO": 0.271 - } - }, - { - "id": "Daemontatox/Cogito-MIS", - "name": "Cogito-MIS", - "developer": "Daemontatox", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1815, - "hfopenllm_v2/BBH": 0.506, - "hfopenllm_v2/MATH Level 5": 0.0861, - "hfopenllm_v2/GPQA": 0.2567, - "hfopenllm_v2/MUSR": 0.3768, - "hfopenllm_v2/MMLU-PRO": 0.1435 - } - }, - { - "id": "Daemontatox/CogitoDistil", - "name": "CogitoDistil", - "developer": "Daemontatox", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2776, - "hfopenllm_v2/BBH": 0.3677, - "hfopenllm_v2/MATH Level 5": 0.3927, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3755, - "hfopenllm_v2/MMLU-PRO": 0.2625 - } - }, - { - "id": "Daemontatox/CogitoZ", - "name": "CogitoZ", - "developer": "Daemontatox", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3967, - "hfopenllm_v2/BBH": 0.6734, - "hfopenllm_v2/MATH Level 5": 0.5242, - "hfopenllm_v2/GPQA": 0.3951, - "hfopenllm_v2/MUSR": 0.4793, - "hfopenllm_v2/MMLU-PRO": 0.5593 - } - }, - { - "id": "Daemontatox/CogitoZ14", - "name": "CogitoZ14", - "developer": "Daemontatox", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6637, - "hfopenllm_v2/BBH": 0.6298, - "hfopenllm_v2/MATH Level 5": 0.4222, - "hfopenllm_v2/GPQA": 0.3163, - "hfopenllm_v2/MUSR": 0.4059, - "hfopenllm_v2/MMLU-PRO": 0.3999 - } - }, - { - "id": "Daemontatox/DocumentCogito", - "name": "DocumentCogito", - "developer": "Daemontatox", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5064, - "hfopenllm_v2/BBH": 0.5112, - "hfopenllm_v2/MATH Level 5": 0.1631, - "hfopenllm_v2/GPQA": 0.3163, - "hfopenllm_v2/MUSR": 0.3973, - "hfopenllm_v2/MMLU-PRO": 0.3802 - } - }, - { - "id": "Daemontatox/Llama3.3-70B-CogniLink", - "name": "Llama3.3-70B-CogniLink", - "developer": "Daemontatox", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6931, - "hfopenllm_v2/BBH": 0.6668, - "hfopenllm_v2/MATH Level 5": 0.4139, - "hfopenllm_v2/GPQA": 0.4455, - "hfopenllm_v2/MUSR": 0.4877, - "hfopenllm_v2/MMLU-PRO": 0.5173 - } - }, - { - "id": "Daemontatox/Llama_cot", - "name": "Llama_cot", - "developer": "Daemontatox", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7549, - "hfopenllm_v2/BBH": 0.4838, - "hfopenllm_v2/MATH Level 5": 0.2024, - "hfopenllm_v2/GPQA": 0.2911, - "hfopenllm_v2/MUSR": 0.3872, - "hfopenllm_v2/MMLU-PRO": 0.3518 - } - }, - { - "id": "Daemontatox/MawaredT1", - "name": "MawaredT1", - "developer": "Daemontatox", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4199, - "hfopenllm_v2/BBH": 0.5215, - "hfopenllm_v2/MATH Level 5": 0.3021, - "hfopenllm_v2/GPQA": 0.3347, - "hfopenllm_v2/MUSR": 0.4702, - "hfopenllm_v2/MMLU-PRO": 0.4718 - } - }, - { - "id": "Daemontatox/mini-Cogito-R1", - "name": "mini-Cogito-R1", - "developer": "Daemontatox", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2298, - "hfopenllm_v2/BBH": 0.328, - "hfopenllm_v2/MATH Level 5": 0.2749, - "hfopenllm_v2/GPQA": 0.2869, - "hfopenllm_v2/MUSR": 0.3447, - "hfopenllm_v2/MMLU-PRO": 0.1482 - } - }, - { - "id": "Daemontatox/mini_Pathfinder", - "name": "mini_Pathfinder", - "developer": "Daemontatox", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2962, - "hfopenllm_v2/BBH": 0.3956, - "hfopenllm_v2/MATH Level 5": 0.4751, - "hfopenllm_v2/GPQA": 0.2584, - "hfopenllm_v2/MUSR": 0.3781, - "hfopenllm_v2/MMLU-PRO": 0.2809 - } - }, - { - "id": "Daemontatox/Mini_QwQ", - "name": "Mini_QwQ", - "developer": "Daemontatox", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4497, - "hfopenllm_v2/BBH": 0.5549, - "hfopenllm_v2/MATH Level 5": 0.4192, - "hfopenllm_v2/GPQA": 0.3037, - "hfopenllm_v2/MUSR": 0.4682, - "hfopenllm_v2/MMLU-PRO": 0.4373 - } - }, - { - "id": "Daemontatox/NemoR", - "name": "NemoR", - "developer": "Daemontatox", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2287, - "hfopenllm_v2/BBH": 0.5194, - "hfopenllm_v2/MATH Level 5": 0.0831, - "hfopenllm_v2/GPQA": 0.3272, - "hfopenllm_v2/MUSR": 0.3908, - "hfopenllm_v2/MMLU-PRO": 0.329 - } - }, - { - "id": "Daemontatox/PathfinderAI", - "name": "PathfinderAI", - "developer": "Daemontatox", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3745, - "hfopenllm_v2/BBH": 0.6668, - "hfopenllm_v2/MATH Level 5": 0.4758, - "hfopenllm_v2/GPQA": 0.3943, - "hfopenllm_v2/MUSR": 0.4858, - "hfopenllm_v2/MMLU-PRO": 0.5593 - } - }, - { - "id": "Daemontatox/PathFinderAI2.0", - "name": "PathFinderAI2.0", - "developer": "Daemontatox", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4541, - "hfopenllm_v2/BBH": 0.6658, - "hfopenllm_v2/MATH Level 5": 0.5076, - "hfopenllm_v2/GPQA": 0.302, - "hfopenllm_v2/MUSR": 0.4216, - "hfopenllm_v2/MMLU-PRO": 0.5547 - } - }, - { - "id": "Daemontatox/PathFinderAi3.0", - "name": "PathFinderAi3.0", - "developer": "Daemontatox", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4271, - "hfopenllm_v2/BBH": 0.6884, - "hfopenllm_v2/MATH Level 5": 0.5045, - "hfopenllm_v2/GPQA": 0.4086, - "hfopenllm_v2/MUSR": 0.4807, - "hfopenllm_v2/MMLU-PRO": 0.5757 - } - }, - { - "id": "Daemontatox/Phi-4-COT", - "name": "Phi-4-COT", - "developer": "Daemontatox", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1793, - "hfopenllm_v2/BBH": 0.6173, - "hfopenllm_v2/MATH Level 5": 0.2243, - "hfopenllm_v2/GPQA": 0.3356, - "hfopenllm_v2/MUSR": 0.453, - "hfopenllm_v2/MMLU-PRO": 0.5005 - } - }, - { - "id": "Daemontatox/PixelParse_AI", - "name": "PixelParse_AI", - "developer": "Daemontatox", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4383, - "hfopenllm_v2/BBH": 0.5034, - "hfopenllm_v2/MATH Level 5": 0.1473, - "hfopenllm_v2/GPQA": 0.3238, - "hfopenllm_v2/MUSR": 0.4052, - "hfopenllm_v2/MMLU-PRO": 0.3778 - } - }, - { - "id": "Daemontatox/RA2.0", - "name": "RA2.0", - "developer": "Daemontatox", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3784, - "hfopenllm_v2/BBH": 0.4889, - "hfopenllm_v2/MATH Level 5": 0.3837, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.4091, - "hfopenllm_v2/MMLU-PRO": 0.2616 - } - }, - { - "id": "Daemontatox/RA_Reasoner", - "name": "RA_Reasoner", - "developer": "Daemontatox", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5592, - "hfopenllm_v2/BBH": 0.6054, - "hfopenllm_v2/MATH Level 5": 0.2122, - "hfopenllm_v2/GPQA": 0.3314, - "hfopenllm_v2/MUSR": 0.3964, - "hfopenllm_v2/MMLU-PRO": 0.43 - } - }, - { - "id": "Daemontatox/RA_Reasoner2.0", - "name": "RA_Reasoner2.0", - "developer": "Daemontatox", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5366, - "hfopenllm_v2/BBH": 0.6062, - "hfopenllm_v2/MATH Level 5": 0.2311, - "hfopenllm_v2/GPQA": 0.3247, - "hfopenllm_v2/MUSR": 0.3884, - "hfopenllm_v2/MMLU-PRO": 0.4353 - } - }, - { - "id": "Daemontatox/ReasonTest", - "name": "ReasonTest", - "developer": "Daemontatox", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.408, - "hfopenllm_v2/BBH": 0.5435, - "hfopenllm_v2/MATH Level 5": 0.2137, - "hfopenllm_v2/GPQA": 0.3188, - "hfopenllm_v2/MUSR": 0.4315, - "hfopenllm_v2/MMLU-PRO": 0.4272 - } - }, - { - "id": "Daemontatox/Research_PathfinderAI", - "name": "Research_PathfinderAI", - "developer": "Daemontatox", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3457, - "hfopenllm_v2/BBH": 0.2872, - "hfopenllm_v2/MATH Level 5": 0.1699, - "hfopenllm_v2/GPQA": 0.2408, - "hfopenllm_v2/MUSR": 0.3394, - "hfopenllm_v2/MMLU-PRO": 0.113 - } - }, - { - "id": "Daemontatox/SphinX", - "name": "SphinX", - "developer": "Daemontatox", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5725, - "hfopenllm_v2/BBH": 0.5441, - "hfopenllm_v2/MATH Level 5": 0.3082, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.4405, - "hfopenllm_v2/MMLU-PRO": 0.4366 - } - }, - { - "id": "Daemontatox/Sphinx2.0", - "name": "Sphinx2.0", - "developer": "Daemontatox", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7123, - "hfopenllm_v2/BBH": 0.6473, - "hfopenllm_v2/MATH Level 5": 0.4018, - "hfopenllm_v2/GPQA": 0.2936, - "hfopenllm_v2/MUSR": 0.426, - "hfopenllm_v2/MMLU-PRO": 0.5184 - } - }, - { - "id": "Daemontatox/TinySphinx", - "name": "TinySphinx", - "developer": "Daemontatox", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2567, - "hfopenllm_v2/BBH": 0.331, - "hfopenllm_v2/MATH Level 5": 0.0431, - "hfopenllm_v2/GPQA": 0.2735, - "hfopenllm_v2/MUSR": 0.3328, - "hfopenllm_v2/MMLU-PRO": 0.1698 - } - }, - { - "id": "Daemontatox/TinySphinx2.0", - "name": "TinySphinx2.0", - "developer": "Daemontatox", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2535, - "hfopenllm_v2/BBH": 0.3168, - "hfopenllm_v2/MATH Level 5": 0.0325, - "hfopenllm_v2/GPQA": 0.2685, - "hfopenllm_v2/MUSR": 0.3382, - "hfopenllm_v2/MMLU-PRO": 0.1731 - } - }, - { - "id": "Daemontatox/Zirel-7B-Math", - "name": "Zirel-7B-Math", - "developer": "Daemontatox", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6639, - "hfopenllm_v2/BBH": 0.5448, - "hfopenllm_v2/MATH Level 5": 0.1979, - "hfopenllm_v2/GPQA": 0.3263, - "hfopenllm_v2/MUSR": 0.4789, - "hfopenllm_v2/MMLU-PRO": 0.4237 - } - }, - { - "id": "Daemontatox/Zirel_1.5", - "name": "Zirel_1.5", - "developer": "Daemontatox", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4168, - "hfopenllm_v2/BBH": 0.3985, - "hfopenllm_v2/MATH Level 5": 0.1133, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3658, - "hfopenllm_v2/MMLU-PRO": 0.2143 - } - }, - { - "id": "Dampfinchen/Llama-3.1-8B-Ultra-Instruct", - "name": "Llama-3.1-8B-Ultra-Instruct", - "developer": "Dampfinchen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8081, - "hfopenllm_v2/BBH": 0.5258, - "hfopenllm_v2/MATH Level 5": 0.2205, - "hfopenllm_v2/GPQA": 0.2919, - "hfopenllm_v2/MUSR": 0.4003, - "hfopenllm_v2/MMLU-PRO": 0.3826 - } - }, - { - "id": "Danielbrdz/Barcenas-10b", - "name": "Barcenas-10b", - "developer": "Danielbrdz", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6608, - "hfopenllm_v2/BBH": 0.6121, - "hfopenllm_v2/MATH Level 5": 0.2153, - "hfopenllm_v2/GPQA": 0.3414, - "hfopenllm_v2/MUSR": 0.4135, - "hfopenllm_v2/MMLU-PRO": 0.4361 - } - }, - { - "id": "Danielbrdz/Barcenas-14b-Phi-3-medium-ORPO", - "name": "Barcenas-14b-Phi-3-medium-ORPO", - "developer": "Danielbrdz", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4799, - "hfopenllm_v2/BBH": 0.6536, - "hfopenllm_v2/MATH Level 5": 0.2024, - "hfopenllm_v2/GPQA": 0.3263, - "hfopenllm_v2/MUSR": 0.4808, - "hfopenllm_v2/MMLU-PRO": 0.4723 - } - }, - { - "id": "Danielbrdz/Barcenas-14b-phi-4", - "name": "Barcenas-14b-phi-4", - "developer": "Danielbrdz", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0498, - "hfopenllm_v2/BBH": 0.6769, - "hfopenllm_v2/MATH Level 5": 0.2583, - "hfopenllm_v2/GPQA": 0.3834, - "hfopenllm_v2/MUSR": 0.5097, - "hfopenllm_v2/MMLU-PRO": 0.5175 - } - }, - { - "id": "Danielbrdz/Barcenas-14b-phi-4-v2", - "name": "Barcenas-14b-phi-4-v2", - "developer": "Danielbrdz", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2775, - "hfopenllm_v2/BBH": 0.6573, - "hfopenllm_v2/MATH Level 5": 0.3218, - "hfopenllm_v2/GPQA": 0.3784, - "hfopenllm_v2/MUSR": 0.4399, - "hfopenllm_v2/MMLU-PRO": 0.5244 - } - }, - { - "id": "Danielbrdz/Barcenas-3b-GRPO", - "name": "Barcenas-3b-GRPO", - "developer": "Danielbrdz", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5444, - "hfopenllm_v2/BBH": 0.4414, - "hfopenllm_v2/MATH Level 5": 0.1375, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.3576, - "hfopenllm_v2/MMLU-PRO": 0.3037 - } - }, - { - "id": "Danielbrdz/Barcenas-Llama3-8b-ORPO", - "name": "Barcenas-Llama3-8b-ORPO", - "developer": "Danielbrdz", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7372, - "hfopenllm_v2/BBH": 0.4987, - "hfopenllm_v2/MATH Level 5": 0.0657, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.419, - "hfopenllm_v2/MMLU-PRO": 0.383 - } - }, - { - "id": "Danielbrdz/Barcenas-R1-Qwen-1.5b", - "name": "Barcenas-R1-Qwen-1.5b", - "developer": "Danielbrdz", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2428, - "hfopenllm_v2/BBH": 0.3587, - "hfopenllm_v2/MATH Level 5": 0.3497, - "hfopenllm_v2/GPQA": 0.3037, - "hfopenllm_v2/MUSR": 0.3541, - "hfopenllm_v2/MMLU-PRO": 0.1909 - } - }, - { - "id": "Dans-DiscountModels/12b-mn-dans-reasoning-test-2", - "name": "12b-mn-dans-reasoning-test-2", - "developer": "Dans-DiscountModels", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3711, - "hfopenllm_v2/BBH": 0.4807, - "hfopenllm_v2/MATH Level 5": 0.0634, - "hfopenllm_v2/GPQA": 0.2735, - "hfopenllm_v2/MUSR": 0.3702, - "hfopenllm_v2/MMLU-PRO": 0.2507 - } - }, - { - "id": "Dans-DiscountModels/12b-mn-dans-reasoning-test-3", - "name": "12b-mn-dans-reasoning-test-3", - "developer": "Dans-DiscountModels", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5053, - "hfopenllm_v2/BBH": 0.4839, - "hfopenllm_v2/MATH Level 5": 0.0778, - "hfopenllm_v2/GPQA": 0.271, - "hfopenllm_v2/MUSR": 0.4168, - "hfopenllm_v2/MMLU-PRO": 0.2516 - } - }, - { - "id": "Dans-DiscountModels/Dans-Instruct-CoreCurriculum-12b-ChatML", - "name": "Dans-Instruct-CoreCurriculum-12b-ChatML", - "developer": "Dans-DiscountModels", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2111, - "hfopenllm_v2/BBH": 0.4792, - "hfopenllm_v2/MATH Level 5": 0.0431, - "hfopenllm_v2/GPQA": 0.2802, - "hfopenllm_v2/MUSR": 0.3606, - "hfopenllm_v2/MMLU-PRO": 0.2805 - } - }, - { - "id": "Dans-DiscountModels/Dans-Instruct-Mix-8b-ChatML", - "name": "Dans-Instruct-Mix-8b-ChatML", - "developer": "Dans-DiscountModels", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0825, - "hfopenllm_v2/BBH": 0.4738, - "hfopenllm_v2/MATH Level 5": 0.0551, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.3918, - "hfopenllm_v2/MMLU-PRO": 0.3288 - } - }, - { - "id": "Dans-DiscountModels/Dans-Instruct-Mix-8b-ChatML-V0.1.0", - "name": "Dans-Instruct-Mix-8b-ChatML-V0.1.0", - "developer": "Dans-DiscountModels", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0668, - "hfopenllm_v2/BBH": 0.4775, - "hfopenllm_v2/MATH Level 5": 0.0672, - "hfopenllm_v2/GPQA": 0.2861, - "hfopenllm_v2/MUSR": 0.3786, - "hfopenllm_v2/MMLU-PRO": 0.3284 - } - }, - { - "id": "Dans-DiscountModels/Dans-Instruct-Mix-8b-ChatML-V0.1.1", - "name": "Dans-Instruct-Mix-8b-ChatML-V0.1.1", - "developer": "Dans-DiscountModels", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0911, - "hfopenllm_v2/BBH": 0.4749, - "hfopenllm_v2/MATH Level 5": 0.0597, - "hfopenllm_v2/GPQA": 0.2911, - "hfopenllm_v2/MUSR": 0.3825, - "hfopenllm_v2/MMLU-PRO": 0.3279 - } - }, - { - "id": "Dans-DiscountModels/Dans-Instruct-Mix-8b-ChatML-V0.2.0", - "name": "Dans-Instruct-Mix-8b-ChatML-V0.2.0", - "developer": "Dans-DiscountModels", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5064, - "hfopenllm_v2/BBH": 0.4624, - "hfopenllm_v2/MATH Level 5": 0.0733, - "hfopenllm_v2/GPQA": 0.2936, - "hfopenllm_v2/MUSR": 0.3644, - "hfopenllm_v2/MMLU-PRO": 0.3 - } - }, - { - "id": "Dans-DiscountModels/mistral-7b-test-merged", - "name": "mistral-7b-test-merged", - "developer": "Dans-DiscountModels", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6678, - "hfopenllm_v2/BBH": 0.4898, - "hfopenllm_v2/MATH Level 5": 0.0446, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.3754, - "hfopenllm_v2/MMLU-PRO": 0.2978 - } - }, - { - "id": "Dans-DiscountModels/Mistral-7b-v0.3-Test-E0.7", - "name": "Mistral-7b-v0.3-Test-E0.7", - "developer": "Dans-DiscountModels", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5124, - "hfopenllm_v2/BBH": 0.475, - "hfopenllm_v2/MATH Level 5": 0.034, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.4005, - "hfopenllm_v2/MMLU-PRO": 0.2744 - } - }, - { - "id": "darkc0de/BuddyGlass_v0.3_Xortron7MethedUpSwitchedUp", - "name": "BuddyGlass_v0.3_Xortron7MethedUpSwitchedUp", - "developer": "darkc0de", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4358, - "hfopenllm_v2/BBH": 0.5243, - "hfopenllm_v2/MATH Level 5": 0.1284, - "hfopenllm_v2/GPQA": 0.2987, - "hfopenllm_v2/MUSR": 0.4143, - "hfopenllm_v2/MMLU-PRO": 0.3673 - } - }, - { - "id": "darkc0de/BuddyGlassNeverSleeps", - "name": "BuddyGlassNeverSleeps", - "developer": "darkc0de", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4239, - "hfopenllm_v2/BBH": 0.4977, - "hfopenllm_v2/MATH Level 5": 0.0627, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.3993, - "hfopenllm_v2/MMLU-PRO": 0.3452 - } - }, - { - "id": "darkc0de/BuddyGlassUncensored2025.2", - "name": "BuddyGlassUncensored2025.2", - "developer": "darkc0de", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7731, - "hfopenllm_v2/BBH": 0.6095, - "hfopenllm_v2/MATH Level 5": 0.2402, - "hfopenllm_v2/GPQA": 0.328, - "hfopenllm_v2/MUSR": 0.4071, - "hfopenllm_v2/MMLU-PRO": 0.4336 - } - }, - { - "id": "Darkknight535/OpenCrystal-12B-L3", - "name": "OpenCrystal-12B-L3", - "developer": "Darkknight535", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4071, - "hfopenllm_v2/BBH": 0.5223, - "hfopenllm_v2/MATH Level 5": 0.0899, - "hfopenllm_v2/GPQA": 0.3062, - "hfopenllm_v2/MUSR": 0.3657, - "hfopenllm_v2/MMLU-PRO": 0.364 - } - }, - { - "id": "Databricks-Mosaic-Research/PGRM", - "name": "Databricks-Mosaic-Research/PGRM", - "developer": "Databricks-Mosaic-Research", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.8002, - "reward-bench/Factuality": 0.7937, - "reward-bench/Precise IF": 0.5062, - "reward-bench/Math": 0.7404, - "reward-bench/Safety": 0.9289, - "reward-bench/Focus": 0.9424, - "reward-bench/Ties": 0.8893 - } - }, - { - "id": "databricks/dbrx-base", - "name": "dbrx-base", - "developer": "databricks", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0821, - "hfopenllm_v2/BBH": 0.5196, - "hfopenllm_v2/MATH Level 5": 0.1, - "hfopenllm_v2/GPQA": 0.3267, - "hfopenllm_v2/MUSR": 0.4067, - "hfopenllm_v2/MMLU-PRO": 0.35 - } - }, - { - "id": "databricks/dbrx-instruct", - "name": "DBRX Instruct", - "developer": "databricks", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.289, - "helm_lite/NarrativeQA": 0.488, - "helm_lite/NaturalQuestions (closed-book)": 0.284, - "helm_lite/OpenbookQA": 0.91, - "helm_lite/MMLU": 0.643, - "helm_lite/MATH": 0.358, - "helm_lite/GSM8K": 0.671, - "helm_lite/LegalBench": 0.426, - "helm_lite/MedQA": 0.694, - "helm_lite/WMT 2014": 0.131, - "helm_mmlu/MMLU All Subjects": 0.741, - "helm_mmlu/Abstract Algebra": 0.34, - "helm_mmlu/Anatomy": 0.667, - "helm_mmlu/College Physics": 0.539, - "helm_mmlu/Computer Security": 0.83, - "helm_mmlu/Econometrics": 0.605, - "helm_mmlu/Global Facts": 0.46, - "helm_mmlu/Jurisprudence": 0.843, - "helm_mmlu/Philosophy": 0.804, - "helm_mmlu/Professional Psychology": 0.801, - "helm_mmlu/Us Foreign Policy": 0.93, - "helm_mmlu/Astronomy": 0.836, - "helm_mmlu/Business Ethics": 0.78, - "helm_mmlu/Clinical Knowledge": 0.789, - "helm_mmlu/Conceptual Physics": 0.74, - "helm_mmlu/Electrical Engineering": 0.71, - "helm_mmlu/Elementary Mathematics": 0.563, - "helm_mmlu/Formal Logic": 0.563, - "helm_mmlu/High School World History": 0.903, - "helm_mmlu/Human Sexuality": 0.878, - "helm_mmlu/International Law": 0.884, - "helm_mmlu/Logical Fallacies": 0.847, - "helm_mmlu/Machine Learning": 0.625, - "helm_mmlu/Management": 0.854, - "helm_mmlu/Marketing": 0.94, - "helm_mmlu/Medical Genetics": 0.85, - "helm_mmlu/Miscellaneous": 0.911, - "helm_mmlu/Moral Scenarios": 0.465, - "helm_mmlu/Nutrition": 0.814, - "helm_mmlu/Prehistory": 0.84, - "helm_mmlu/Public Relations": 0.691, - "helm_mmlu/Security Studies": 0.804, - "helm_mmlu/Sociology": 0.896, - "helm_mmlu/Virology": 0.566, - "helm_mmlu/World Religions": 0.871, - "helm_mmlu/Mean win rate": 0.537, - "hfopenllm_v2/IFEval": 0.5416, - "hfopenllm_v2/BBH": 0.5429, - "hfopenllm_v2/MATH Level 5": 0.0687, - "hfopenllm_v2/GPQA": 0.3414, - "hfopenllm_v2/MUSR": 0.4269, - "hfopenllm_v2/MMLU-PRO": 0.3683 - } - }, - { - "id": "databricks/dolly-v1-6b", - "name": "dolly-v1-6b", - "developer": "databricks", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2224, - "hfopenllm_v2/BBH": 0.3172, - "hfopenllm_v2/MATH Level 5": 0.0189, - "hfopenllm_v2/GPQA": 0.2643, - "hfopenllm_v2/MUSR": 0.4004, - "hfopenllm_v2/MMLU-PRO": 0.1266 - } - }, - { - "id": "databricks/dolly-v2-12b", - "name": "dolly-v2-12b", - "developer": "databricks", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2355, - "hfopenllm_v2/BBH": 0.332, - "hfopenllm_v2/MATH Level 5": 0.0136, - "hfopenllm_v2/GPQA": 0.2408, - "hfopenllm_v2/MUSR": 0.3739, - "hfopenllm_v2/MMLU-PRO": 0.1129 - } - }, - { - "id": "databricks/dolly-v2-3b", - "name": "dolly-v2-3b", - "developer": "databricks", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2247, - "hfopenllm_v2/BBH": 0.3079, - "hfopenllm_v2/MATH Level 5": 0.0151, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.3338, - "hfopenllm_v2/MMLU-PRO": 0.1145 - } - }, - { - "id": "databricks/dolly-v2-7b", - "name": "dolly-v2-7b", - "developer": "databricks", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.201, - "hfopenllm_v2/BBH": 0.3173, - "hfopenllm_v2/MATH Level 5": 0.0144, - "hfopenllm_v2/GPQA": 0.2685, - "hfopenllm_v2/MUSR": 0.3553, - "hfopenllm_v2/MMLU-PRO": 0.1149 - } - }, - { - "id": "DavidAU/DeepHermes-3-Llama-3-8B-Preview-16.5B-Brainstorm", - "name": "DeepHermes-3-Llama-3-8B-Preview-16.5B-Brainstorm", - "developer": "DavidAU", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3136, - "hfopenllm_v2/BBH": 0.4762, - "hfopenllm_v2/MATH Level 5": 0.1057, - "hfopenllm_v2/GPQA": 0.3138, - "hfopenllm_v2/MUSR": 0.3928, - "hfopenllm_v2/MMLU-PRO": 0.3209 - } - }, - { - "id": "DavidAU/DeepSeek-BlackRoot-R1-Distill-Llama-3.1-8B", - "name": "DeepSeek-BlackRoot-R1-Distill-Llama-3.1-8B", - "developer": "DavidAU", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3685, - "hfopenllm_v2/BBH": 0.4887, - "hfopenllm_v2/MATH Level 5": 0.0657, - "hfopenllm_v2/GPQA": 0.318, - "hfopenllm_v2/MUSR": 0.432, - "hfopenllm_v2/MMLU-PRO": 0.2976 - } - }, - { - "id": "DavidAU/DeepSeek-Grand-Horror-SMB-R1-Distill-Llama-3.1-16B", - "name": "DeepSeek-Grand-Horror-SMB-R1-Distill-Llama-3.1-16B", - "developer": "DavidAU", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2507, - "hfopenllm_v2/BBH": 0.4488, - "hfopenllm_v2/MATH Level 5": 0.0295, - "hfopenllm_v2/GPQA": 0.3138, - "hfopenllm_v2/MUSR": 0.4164, - "hfopenllm_v2/MMLU-PRO": 0.2709 - } - }, - { - "id": "DavidAU/DeepSeek-MOE-4X8B-R1-Distill-Llama-3.1-Deep-Thinker-Uncensored-24B", - "name": "DeepSeek-MOE-4X8B-R1-Distill-Llama-3.1-Deep-Thinker-Uncensored-24B", - "developer": "DavidAU", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3883, - "hfopenllm_v2/BBH": 0.4886, - "hfopenllm_v2/MATH Level 5": 0.0816, - "hfopenllm_v2/GPQA": 0.323, - "hfopenllm_v2/MUSR": 0.4375, - "hfopenllm_v2/MMLU-PRO": 0.3024 - } - }, - { - "id": "DavidAU/DeepSeek-MOE-4X8B-R1-Distill-Llama-3.1-Mad-Scientist-24B", - "name": "DeepSeek-MOE-4X8B-R1-Distill-Llama-3.1-Mad-Scientist-24B", - "developer": "DavidAU", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3436, - "hfopenllm_v2/BBH": 0.4769, - "hfopenllm_v2/MATH Level 5": 0.0755, - "hfopenllm_v2/GPQA": 0.3372, - "hfopenllm_v2/MUSR": 0.4231, - "hfopenllm_v2/MMLU-PRO": 0.297 - } - }, - { - "id": "DavidAU/DeepSeek-R1-Distill-Qwen-25.5B-Brainstorm", - "name": "DeepSeek-R1-Distill-Qwen-25.5B-Brainstorm", - "developer": "DavidAU", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3416, - "hfopenllm_v2/BBH": 0.5807, - "hfopenllm_v2/MATH Level 5": 0.5536, - "hfopenllm_v2/GPQA": 0.3859, - "hfopenllm_v2/MUSR": 0.5155, - "hfopenllm_v2/MMLU-PRO": 0.4624 - } - }, - { - "id": "DavidAU/DeepSeek-V2-Grand-Horror-SMB-R1-Distill-Llama-3.1-Uncensored-16.5B", - "name": "DeepSeek-V2-Grand-Horror-SMB-R1-Distill-Llama-3.1-Uncensored-16.5B", - "developer": "DavidAU", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2853, - "hfopenllm_v2/BBH": 0.4462, - "hfopenllm_v2/MATH Level 5": 0.0174, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.4179, - "hfopenllm_v2/MMLU-PRO": 0.2778 - } - }, - { - "id": "DavidAU/DeepThought-MOE-8X3B-R1-Llama-3.2-Reasoning-18B", - "name": "DeepThought-MOE-8X3B-R1-Llama-3.2-Reasoning-18B", - "developer": "DavidAU", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3793, - "hfopenllm_v2/BBH": 0.4232, - "hfopenllm_v2/MATH Level 5": 0.108, - "hfopenllm_v2/GPQA": 0.2794, - "hfopenllm_v2/MUSR": 0.356, - "hfopenllm_v2/MMLU-PRO": 0.272 - } - }, - { - "id": "DavidAU/Gemma-The-Writer-9B", - "name": "Gemma-The-Writer-9B", - "developer": "DavidAU", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.174, - "hfopenllm_v2/BBH": 0.5905, - "hfopenllm_v2/MATH Level 5": 0.0876, - "hfopenllm_v2/GPQA": 0.3456, - "hfopenllm_v2/MUSR": 0.4099, - "hfopenllm_v2/MMLU-PRO": 0.3979 - } - }, - { - "id": "DavidAU/Gemma-The-Writer-DEADLINE-10B", - "name": "Gemma-The-Writer-DEADLINE-10B", - "developer": "DavidAU", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2332, - "hfopenllm_v2/BBH": 0.5896, - "hfopenllm_v2/MATH Level 5": 0.0989, - "hfopenllm_v2/GPQA": 0.3423, - "hfopenllm_v2/MUSR": 0.4189, - "hfopenllm_v2/MMLU-PRO": 0.3946 - } - }, - { - "id": "DavidAU/Gemma-The-Writer-J.GutenBerg-10B", - "name": "Gemma-The-Writer-J.GutenBerg-10B", - "developer": "DavidAU", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2858, - "hfopenllm_v2/BBH": 0.5909, - "hfopenllm_v2/MATH Level 5": 0.0921, - "hfopenllm_v2/GPQA": 0.3381, - "hfopenllm_v2/MUSR": 0.4176, - "hfopenllm_v2/MMLU-PRO": 0.3947 - } - }, - { - "id": "DavidAU/Gemma-The-Writer-Mighty-Sword-9B", - "name": "Gemma-The-Writer-Mighty-Sword-9B", - "developer": "DavidAU", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7528, - "hfopenllm_v2/BBH": 0.5912, - "hfopenllm_v2/MATH Level 5": 0.1911, - "hfopenllm_v2/GPQA": 0.3482, - "hfopenllm_v2/MUSR": 0.4112, - "hfopenllm_v2/MMLU-PRO": 0.3968 - } - }, - { - "id": "DavidAU/Gemma-The-Writer-N-Restless-Quill-10B-Uncensored", - "name": "Gemma-The-Writer-N-Restless-Quill-10B-Uncensored", - "developer": "DavidAU", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7071, - "hfopenllm_v2/BBH": 0.5922, - "hfopenllm_v2/MATH Level 5": 0.2296, - "hfopenllm_v2/GPQA": 0.3414, - "hfopenllm_v2/MUSR": 0.4163, - "hfopenllm_v2/MMLU-PRO": 0.3966 - } - }, - { - "id": "DavidAU/L3-Dark-Planet-8B", - "name": "L3-Dark-Planet-8B", - "developer": "DavidAU", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4134, - "hfopenllm_v2/BBH": 0.5084, - "hfopenllm_v2/MATH Level 5": 0.0823, - "hfopenllm_v2/GPQA": 0.3003, - "hfopenllm_v2/MUSR": 0.3616, - "hfopenllm_v2/MMLU-PRO": 0.3737 - } - }, - { - "id": "DavidAU/L3-DARKEST-PLANET-16.5B", - "name": "L3-DARKEST-PLANET-16.5B", - "developer": "DavidAU", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6231, - "hfopenllm_v2/BBH": 0.523, - "hfopenllm_v2/MATH Level 5": 0.0899, - "hfopenllm_v2/GPQA": 0.2953, - "hfopenllm_v2/MUSR": 0.3754, - "hfopenllm_v2/MMLU-PRO": 0.363 - } - }, - { - "id": "DavidAU/L3-Jamet-12.2B-MK.V-Blackroot-Instruct", - "name": "L3-Jamet-12.2B-MK.V-Blackroot-Instruct", - "developer": "DavidAU", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3962, - "hfopenllm_v2/BBH": 0.4766, - "hfopenllm_v2/MATH Level 5": 0.0408, - "hfopenllm_v2/GPQA": 0.2785, - "hfopenllm_v2/MUSR": 0.402, - "hfopenllm_v2/MMLU-PRO": 0.3291 - } - }, - { - "id": "DavidAU/L3-Lumimaid-12.2B-v0.1-OAS-Instruct", - "name": "L3-Lumimaid-12.2B-v0.1-OAS-Instruct", - "developer": "DavidAU", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3924, - "hfopenllm_v2/BBH": 0.4693, - "hfopenllm_v2/MATH Level 5": 0.0461, - "hfopenllm_v2/GPQA": 0.2768, - "hfopenllm_v2/MUSR": 0.4194, - "hfopenllm_v2/MMLU-PRO": 0.3142 - } - }, - { - "id": "DavidAU/L3-SMB-Instruct-12.2B-F32", - "name": "L3-SMB-Instruct-12.2B-F32", - "developer": "DavidAU", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4303, - "hfopenllm_v2/BBH": 0.4786, - "hfopenllm_v2/MATH Level 5": 0.0468, - "hfopenllm_v2/GPQA": 0.2819, - "hfopenllm_v2/MUSR": 0.4087, - "hfopenllm_v2/MMLU-PRO": 0.3312 - } - }, - { - "id": "DavidAU/L3-Stheno-Maid-Blackroot-Grand-HORROR-16B", - "name": "L3-Stheno-Maid-Blackroot-Grand-HORROR-16B", - "developer": "DavidAU", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3439, - "hfopenllm_v2/BBH": 0.4736, - "hfopenllm_v2/MATH Level 5": 0.0219, - "hfopenllm_v2/GPQA": 0.271, - "hfopenllm_v2/MUSR": 0.4031, - "hfopenllm_v2/MMLU-PRO": 0.357 - } - }, - { - "id": "DavidAU/L3-Stheno-v3.2-12.2B-Instruct", - "name": "L3-Stheno-v3.2-12.2B-Instruct", - "developer": "DavidAU", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4028, - "hfopenllm_v2/BBH": 0.4846, - "hfopenllm_v2/MATH Level 5": 0.0506, - "hfopenllm_v2/GPQA": 0.2752, - "hfopenllm_v2/MUSR": 0.4103, - "hfopenllm_v2/MMLU-PRO": 0.3345 - } - }, - { - "id": "DavidAU/L3.1-Dark-Planet-SpinFire-Uncensored-8B", - "name": "L3.1-Dark-Planet-SpinFire-Uncensored-8B", - "developer": "DavidAU", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7043, - "hfopenllm_v2/BBH": 0.5261, - "hfopenllm_v2/MATH Level 5": 0.0929, - "hfopenllm_v2/GPQA": 0.2794, - "hfopenllm_v2/MUSR": 0.3541, - "hfopenllm_v2/MMLU-PRO": 0.367 - } - }, - { - "id": "DavidAU/L3.1-MOE-2X8B-Deepseek-DeepHermes-e32-uncensored-abliterated-13.7B", - "name": "L3.1-MOE-2X8B-Deepseek-DeepHermes-e32-uncensored-abliterated-13.7B", - "developer": "DavidAU", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3345, - "hfopenllm_v2/BBH": 0.4421, - "hfopenllm_v2/MATH Level 5": 0.2606, - "hfopenllm_v2/GPQA": 0.3138, - "hfopenllm_v2/MUSR": 0.3749, - "hfopenllm_v2/MMLU-PRO": 0.2892 - } - }, - { - "id": "DavidAU/Qwen2.5-MOE-2X1.5B-DeepSeek-Uncensored-Censored-4B", - "name": "Qwen2.5-MOE-2X1.5B-DeepSeek-Uncensored-Censored-4B", - "developer": "DavidAU", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1783, - "hfopenllm_v2/BBH": 0.3033, - "hfopenllm_v2/MATH Level 5": 0.0249, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3715, - "hfopenllm_v2/MMLU-PRO": 0.1142 - } - }, - { - "id": "DavidAU/Qwen2.5-MOE-2X7B-DeepSeek-Abliterated-Censored-19B", - "name": "Qwen2.5-MOE-2X7B-DeepSeek-Abliterated-Censored-19B", - "developer": "DavidAU", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2835, - "hfopenllm_v2/BBH": 0.3592, - "hfopenllm_v2/MATH Level 5": 0.2417, - "hfopenllm_v2/GPQA": 0.2651, - "hfopenllm_v2/MUSR": 0.3847, - "hfopenllm_v2/MMLU-PRO": 0.1636 - } - }, - { - "id": "DavidAU/Qwen2.5-MOE-6x1.5B-DeepSeek-Reasoning-e32", - "name": "Qwen2.5-MOE-6x1.5B-DeepSeek-Reasoning-e32", - "developer": "DavidAU", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2107, - "hfopenllm_v2/BBH": 0.3286, - "hfopenllm_v2/MATH Level 5": 0.0665, - "hfopenllm_v2/GPQA": 0.2475, - "hfopenllm_v2/MUSR": 0.3404, - "hfopenllm_v2/MMLU-PRO": 0.1122 - } - }, - { - "id": "davidkim205/nox-solar-10.7b-v4", - "name": "nox-solar-10.7b-v4", - "developer": "davidkim205", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3753, - "hfopenllm_v2/BBH": 0.4814, - "hfopenllm_v2/MATH Level 5": 0.0083, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.4298, - "hfopenllm_v2/MMLU-PRO": 0.3333 - } - }, - { - "id": "davidkim205/Rhea-72b-v0.5", - "name": "Rhea-72b-v0.5", - "developer": "davidkim205", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0145, - "hfopenllm_v2/BBH": 0.3078, - "hfopenllm_v2/MATH Level 5": 0.1737, - "hfopenllm_v2/GPQA": 0.2525, - "hfopenllm_v2/MUSR": 0.4241, - "hfopenllm_v2/MMLU-PRO": 0.1166 - } - }, - { - "id": "Davidsv/SUONG-1", - "name": "SUONG-1", - "developer": "Davidsv", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2497, - "hfopenllm_v2/BBH": 0.2817, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2441, - "hfopenllm_v2/MUSR": 0.3578, - "hfopenllm_v2/MMLU-PRO": 0.1085 - } - }, - { - "id": "DavieLion/Llama-3.2-1B-SPIN-iter0", - "name": "Llama-3.2-1B-SPIN-iter0", - "developer": "DavieLion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1507, - "hfopenllm_v2/BBH": 0.293, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2534, - "hfopenllm_v2/MUSR": 0.3565, - "hfopenllm_v2/MMLU-PRO": 0.1125 - } - }, - { - "id": "DavieLion/Llama-3.2-1B-SPIN-iter1", - "name": "Llama-3.2-1B-SPIN-iter1", - "developer": "DavieLion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1575, - "hfopenllm_v2/BBH": 0.294, - "hfopenllm_v2/MATH Level 5": 0.0023, - "hfopenllm_v2/GPQA": 0.2508, - "hfopenllm_v2/MUSR": 0.3646, - "hfopenllm_v2/MMLU-PRO": 0.1118 - } - }, - { - "id": "DavieLion/Llama-3.2-1B-SPIN-iter2", - "name": "Llama-3.2-1B-SPIN-iter2", - "developer": "DavieLion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1376, - "hfopenllm_v2/BBH": 0.298, - "hfopenllm_v2/MATH Level 5": 0.0053, - "hfopenllm_v2/GPQA": 0.2542, - "hfopenllm_v2/MUSR": 0.3553, - "hfopenllm_v2/MMLU-PRO": 0.1129 - } - }, - { - "id": "DavieLion/Llama-3.2-1B-SPIN-iter3", - "name": "Llama-3.2-1B-SPIN-iter3", - "developer": "DavieLion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1336, - "hfopenllm_v2/BBH": 0.2975, - "hfopenllm_v2/MATH Level 5": 0.0068, - "hfopenllm_v2/GPQA": 0.2534, - "hfopenllm_v2/MUSR": 0.35, - "hfopenllm_v2/MMLU-PRO": 0.1128 - } - }, - { - "id": "DavieLion/Lllma-3.2-1B", - "name": "Lllma-3.2-1B", - "developer": "DavieLion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1601, - "hfopenllm_v2/BBH": 0.2965, - "hfopenllm_v2/MATH Level 5": 0.0068, - "hfopenllm_v2/GPQA": 0.2441, - "hfopenllm_v2/MUSR": 0.3578, - "hfopenllm_v2/MMLU-PRO": 0.1126 - } - }, - { - "id": "DebateLabKIT/Llama-3.1-Argunaut-1-8B-SFT", - "name": "Llama-3.1-Argunaut-1-8B-SFT", - "developer": "DebateLabKIT", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5519, - "hfopenllm_v2/BBH": 0.4824, - "hfopenllm_v2/MATH Level 5": 0.145, - "hfopenllm_v2/GPQA": 0.2836, - "hfopenllm_v2/MUSR": 0.4503, - "hfopenllm_v2/MMLU-PRO": 0.3472 - } - }, - { - "id": "Deci/DeciLM-7B", - "name": "DeciLM-7B", - "developer": "Deci", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2813, - "hfopenllm_v2/BBH": 0.4423, - "hfopenllm_v2/MATH Level 5": 0.0287, - "hfopenllm_v2/GPQA": 0.2953, - "hfopenllm_v2/MUSR": 0.4359, - "hfopenllm_v2/MMLU-PRO": 0.2692 - } - }, - { - "id": "Deci/DeciLM-7B-instruct", - "name": "DeciLM-7B-instruct", - "developer": "Deci", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.488, - "hfopenllm_v2/BBH": 0.459, - "hfopenllm_v2/MATH Level 5": 0.0302, - "hfopenllm_v2/GPQA": 0.2894, - "hfopenllm_v2/MUSR": 0.3884, - "hfopenllm_v2/MMLU-PRO": 0.2608 - } - }, - { - "id": "DeepAutoAI/causal_gpt2", - "name": "causal_gpt2", - "developer": "DeepAutoAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1813, - "hfopenllm_v2/BBH": 0.3026, - "hfopenllm_v2/MATH Level 5": 0.0053, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.427, - "hfopenllm_v2/MMLU-PRO": 0.1131 - } - }, - { - "id": "DeepAutoAI/d2nwg_causal_gpt2", - "name": "d2nwg_causal_gpt2", - "developer": "DeepAutoAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1916, - "hfopenllm_v2/BBH": 0.3027, - "hfopenllm_v2/MATH Level 5": 0.0045, - "hfopenllm_v2/GPQA": 0.2576, - "hfopenllm_v2/MUSR": 0.4297, - "hfopenllm_v2/MMLU-PRO": 0.1151 - } - }, - { - "id": "DeepAutoAI/d2nwg_causal_gpt2_v1", - "name": "d2nwg_causal_gpt2_v1", - "developer": "DeepAutoAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1989, - "hfopenllm_v2/BBH": 0.2992, - "hfopenllm_v2/MATH Level 5": 0.0038, - "hfopenllm_v2/GPQA": 0.2584, - "hfopenllm_v2/MUSR": 0.4337, - "hfopenllm_v2/MMLU-PRO": 0.1135 - } - }, - { - "id": "DeepAutoAI/d2nwg_Llama-3.1-8B-Instruct-v0.0", - "name": "d2nwg_Llama-3.1-8B-Instruct-v0.0", - "developer": "DeepAutoAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7893, - "hfopenllm_v2/BBH": 0.508, - "hfopenllm_v2/MATH Level 5": 0.1805, - "hfopenllm_v2/GPQA": 0.2919, - "hfopenllm_v2/MUSR": 0.4135, - "hfopenllm_v2/MMLU-PRO": 0.3877 - } - }, - { - "id": "DeepAutoAI/Explore_Llama-3.1-8B-Inst", - "name": "Explore_Llama-3.1-8B-Inst", - "developer": "DeepAutoAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7795, - "hfopenllm_v2/BBH": 0.5117, - "hfopenllm_v2/MATH Level 5": 0.2009, - "hfopenllm_v2/GPQA": 0.2836, - "hfopenllm_v2/MUSR": 0.391, - "hfopenllm_v2/MMLU-PRO": 0.3792 - } - }, - { - "id": "DeepAutoAI/Explore_Llama-3.2-1B-Inst", - "name": "Explore_Llama-3.2-1B-Inst", - "developer": "DeepAutoAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5649, - "hfopenllm_v2/BBH": 0.3505, - "hfopenllm_v2/MATH Level 5": 0.0748, - "hfopenllm_v2/GPQA": 0.2559, - "hfopenllm_v2/MUSR": 0.3183, - "hfopenllm_v2/MMLU-PRO": 0.1809 - } - }, - { - "id": "DeepAutoAI/Explore_Llama-3.2-1B-Inst_v0", - "name": "Explore_Llama-3.2-1B-Inst_v0", - "developer": "DeepAutoAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5597, - "hfopenllm_v2/BBH": 0.3365, - "hfopenllm_v2/MATH Level 5": 0.0597, - "hfopenllm_v2/GPQA": 0.2634, - "hfopenllm_v2/MUSR": 0.3103, - "hfopenllm_v2/MMLU-PRO": 0.1804 - } - }, - { - "id": "DeepAutoAI/Explore_Llama-3.2-1B-Inst_v1", - "name": "Explore_Llama-3.2-1B-Inst_v1", - "developer": "DeepAutoAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4999, - "hfopenllm_v2/BBH": 0.3141, - "hfopenllm_v2/MATH Level 5": 0.031, - "hfopenllm_v2/GPQA": 0.245, - "hfopenllm_v2/MUSR": 0.3781, - "hfopenllm_v2/MMLU-PRO": 0.1269 - } - }, - { - "id": "DeepAutoAI/Explore_Llama-3.2-1B-Inst_v1.1", - "name": "Explore_Llama-3.2-1B-Inst_v1.1", - "developer": "DeepAutoAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5844, - "hfopenllm_v2/BBH": 0.3513, - "hfopenllm_v2/MATH Level 5": 0.0718, - "hfopenllm_v2/GPQA": 0.2626, - "hfopenllm_v2/MUSR": 0.3117, - "hfopenllm_v2/MMLU-PRO": 0.1818 - } - }, - { - "id": "DeepAutoAI/ldm_soup_Llama-3.1-8B-Inst", - "name": "ldm_soup_Llama-3.1-8B-Inst", - "developer": "DeepAutoAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8033, - "hfopenllm_v2/BBH": 0.5121, - "hfopenllm_v2/MATH Level 5": 0.1888, - "hfopenllm_v2/GPQA": 0.2894, - "hfopenllm_v2/MUSR": 0.4161, - "hfopenllm_v2/MMLU-PRO": 0.3886 - } - }, - { - "id": "DeepAutoAI/ldm_soup_Llama-3.1-8B-Instruct-v0.0", - "name": "ldm_soup_Llama-3.1-8B-Instruct-v0.0", - "developer": "DeepAutoAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7889, - "hfopenllm_v2/BBH": 0.5125, - "hfopenllm_v2/MATH Level 5": 0.1918, - "hfopenllm_v2/GPQA": 0.2911, - "hfopenllm_v2/MUSR": 0.4121, - "hfopenllm_v2/MMLU-PRO": 0.3895 - } - }, - { - "id": "DeepAutoAI/ldm_soup_Llama-3.1-8B-Instruct-v0.1", - "name": "ldm_soup_Llama-3.1-8B-Instruct-v0.1", - "developer": "DeepAutoAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7889, - "hfopenllm_v2/BBH": 0.5125, - "hfopenllm_v2/MATH Level 5": 0.1918, - "hfopenllm_v2/GPQA": 0.2911, - "hfopenllm_v2/MUSR": 0.4121, - "hfopenllm_v2/MMLU-PRO": 0.3895 - } - }, - { - "id": "DeepMount00/Lexora-Lite-3B", - "name": "Lexora-Lite-3B", - "developer": "DeepMount00", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5776, - "hfopenllm_v2/BBH": 0.4873, - "hfopenllm_v2/MATH Level 5": 0.2304, - "hfopenllm_v2/GPQA": 0.2743, - "hfopenllm_v2/MUSR": 0.3966, - "hfopenllm_v2/MMLU-PRO": 0.3602 - } - }, - { - "id": "DeepMount00/Lexora-Lite-3B_v2", - "name": "Lexora-Lite-3B_v2", - "developer": "DeepMount00", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4943, - "hfopenllm_v2/BBH": 0.4812, - "hfopenllm_v2/MATH Level 5": 0.2281, - "hfopenllm_v2/GPQA": 0.271, - "hfopenllm_v2/MUSR": 0.3822, - "hfopenllm_v2/MMLU-PRO": 0.3544 - } - }, - { - "id": "DeepMount00/Lexora-Medium-7B", - "name": "Lexora-Medium-7B", - "developer": "DeepMount00", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4103, - "hfopenllm_v2/BBH": 0.5145, - "hfopenllm_v2/MATH Level 5": 0.2221, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.4439, - "hfopenllm_v2/MMLU-PRO": 0.4325 - } - }, - { - "id": "DeepMount00/Llama-3-8b-Ita", - "name": "Llama-3-8b-Ita", - "developer": "DeepMount00", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.753, - "hfopenllm_v2/BBH": 0.4936, - "hfopenllm_v2/MATH Level 5": 0.0665, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.4268, - "hfopenllm_v2/MMLU-PRO": 0.3852 - } - }, - { - "id": "DeepMount00/Llama-3.1-8b-ITA", - "name": "Llama-3.1-8b-ITA", - "developer": "DeepMount00", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5365, - "hfopenllm_v2/BBH": 0.517, - "hfopenllm_v2/MATH Level 5": 0.1707, - "hfopenllm_v2/GPQA": 0.3062, - "hfopenllm_v2/MUSR": 0.4487, - "hfopenllm_v2/MMLU-PRO": 0.396 - } - }, - { - "id": "DeepMount00/Llama-3.1-Distilled", - "name": "Llama-3.1-Distilled", - "developer": "DeepMount00", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7844, - "hfopenllm_v2/BBH": 0.5101, - "hfopenllm_v2/MATH Level 5": 0.2032, - "hfopenllm_v2/GPQA": 0.3037, - "hfopenllm_v2/MUSR": 0.4058, - "hfopenllm_v2/MMLU-PRO": 0.3782 - } - }, - { - "id": "DeepMount00/mergekit-ties-okvgjfz", - "name": "mergekit-ties-okvgjfz", - "developer": "DeepMount00", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.153, - "hfopenllm_v2/BBH": 0.2998, - "hfopenllm_v2/MATH Level 5": 0.0008, - "hfopenllm_v2/GPQA": 0.2626, - "hfopenllm_v2/MUSR": 0.3806, - "hfopenllm_v2/MMLU-PRO": 0.1118 - } - }, - { - "id": "DeepMount00/Qwen2-1.5B-Ita", - "name": "Qwen2-1.5B-Ita", - "developer": "DeepMount00", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5173, - "hfopenllm_v2/BBH": 0.3981, - "hfopenllm_v2/MATH Level 5": 0.114, - "hfopenllm_v2/GPQA": 0.2626, - "hfopenllm_v2/MUSR": 0.3504, - "hfopenllm_v2/MMLU-PRO": 0.2772 - } - }, - { - "id": "DeepMount00/Qwen2-1.5B-Ita_v2", - "name": "Qwen2-1.5B-Ita_v2", - "developer": "DeepMount00", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5, - "hfopenllm_v2/BBH": 0.3954, - "hfopenllm_v2/MATH Level 5": 0.0967, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3702, - "hfopenllm_v2/MMLU-PRO": 0.3032 - } - }, - { - "id": "DeepMount00/Qwen2-1.5B-Ita_v3", - "name": "Qwen2-1.5B-Ita_v3", - "developer": "DeepMount00", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.489, - "hfopenllm_v2/BBH": 0.3948, - "hfopenllm_v2/MATH Level 5": 0.1042, - "hfopenllm_v2/GPQA": 0.2534, - "hfopenllm_v2/MUSR": 0.3742, - "hfopenllm_v2/MMLU-PRO": 0.3018 - } - }, - { - "id": "DeepMount00/Qwen2-1.5B-Ita_v5", - "name": "Qwen2-1.5B-Ita_v5", - "developer": "DeepMount00", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4987, - "hfopenllm_v2/BBH": 0.4032, - "hfopenllm_v2/MATH Level 5": 0.1178, - "hfopenllm_v2/GPQA": 0.2542, - "hfopenllm_v2/MUSR": 0.3422, - "hfopenllm_v2/MMLU-PRO": 0.2943 - } - }, - { - "id": "DeepMount00/Qwen2-1.5B-Ita_v6", - "name": "Qwen2-1.5B-Ita_v6", - "developer": "DeepMount00", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2999, - "hfopenllm_v2/BBH": 0.4249, - "hfopenllm_v2/MATH Level 5": 0.0846, - "hfopenllm_v2/GPQA": 0.2827, - "hfopenllm_v2/MUSR": 0.3755, - "hfopenllm_v2/MMLU-PRO": 0.2872 - } - }, - { - "id": "DeepMount00/Qwen2.5-7B-Instruct-MathCoder", - "name": "Qwen2.5-7B-Instruct-MathCoder", - "developer": "DeepMount00", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.153, - "hfopenllm_v2/BBH": 0.2998, - "hfopenllm_v2/MATH Level 5": 0.0008, - "hfopenllm_v2/GPQA": 0.2626, - "hfopenllm_v2/MUSR": 0.3806, - "hfopenllm_v2/MMLU-PRO": 0.1118 - } - }, - { - "id": "deepseek-ai/deepseek-llm-67b-chat", - "name": "DeepSeek LLM Chat 67B", - "developer": "deepseek-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.488, - "helm_lite/NarrativeQA": 0.581, - "helm_lite/NaturalQuestions (closed-book)": 0.412, - "helm_lite/OpenbookQA": 0.88, - "helm_lite/MMLU": 0.641, - "helm_lite/MATH": 0.615, - "helm_lite/GSM8K": 0.795, - "helm_lite/LegalBench": 0.637, - "helm_lite/MedQA": 0.628, - "helm_lite/WMT 2014": 0.186, - "helm_mmlu/MMLU All Subjects": 0.725, - "helm_mmlu/Abstract Algebra": 0.44, - "helm_mmlu/Anatomy": 0.667, - "helm_mmlu/College Physics": 0.363, - "helm_mmlu/Computer Security": 0.79, - "helm_mmlu/Econometrics": 0.553, - "helm_mmlu/Global Facts": 0.46, - "helm_mmlu/Jurisprudence": 0.852, - "helm_mmlu/Philosophy": 0.801, - "helm_mmlu/Professional Psychology": 0.809, - "helm_mmlu/Us Foreign Policy": 0.91, - "helm_mmlu/Astronomy": 0.822, - "helm_mmlu/Business Ethics": 0.86, - "helm_mmlu/Clinical Knowledge": 0.785, - "helm_mmlu/Conceptual Physics": 0.723, - "helm_mmlu/Electrical Engineering": 0.669, - "helm_mmlu/Elementary Mathematics": 0.548, - "helm_mmlu/Formal Logic": 0.548, - "helm_mmlu/High School World History": 0.911, - "helm_mmlu/Human Sexuality": 0.84, - "helm_mmlu/International Law": 0.851, - "helm_mmlu/Logical Fallacies": 0.847, - "helm_mmlu/Machine Learning": 0.562, - "helm_mmlu/Management": 0.903, - "helm_mmlu/Marketing": 0.923, - "helm_mmlu/Medical Genetics": 0.73, - "helm_mmlu/Miscellaneous": 0.904, - "helm_mmlu/Moral Scenarios": 0.544, - "helm_mmlu/Nutrition": 0.781, - "helm_mmlu/Prehistory": 0.858, - "helm_mmlu/Public Relations": 0.7, - "helm_mmlu/Security Studies": 0.796, - "helm_mmlu/Sociology": 0.876, - "helm_mmlu/Virology": 0.554, - "helm_mmlu/World Religions": 0.865, - "helm_mmlu/Mean win rate": 0.387, - "hfopenllm_v2/IFEval": 0.5587, - "hfopenllm_v2/BBH": 0.5243, - "hfopenllm_v2/MATH Level 5": 0.0929, - "hfopenllm_v2/GPQA": 0.3163, - "hfopenllm_v2/MUSR": 0.5059, - "hfopenllm_v2/MMLU-PRO": 0.3944 - } - }, - { - "id": "deepseek-ai/deepseek-llm-7b-base", - "name": "deepseek-llm-7b-base", - "developer": "deepseek-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2179, - "hfopenllm_v2/BBH": 0.3503, - "hfopenllm_v2/MATH Level 5": 0.0196, - "hfopenllm_v2/GPQA": 0.2735, - "hfopenllm_v2/MUSR": 0.3738, - "hfopenllm_v2/MMLU-PRO": 0.1806 - } - }, - { - "id": "deepseek-ai/deepseek-llm-7b-chat", - "name": "deepseek-llm-7b-chat", - "developer": "deepseek-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4171, - "hfopenllm_v2/BBH": 0.3632, - "hfopenllm_v2/MATH Level 5": 0.0204, - "hfopenllm_v2/GPQA": 0.2659, - "hfopenllm_v2/MUSR": 0.4668, - "hfopenllm_v2/MMLU-PRO": 0.2133 - } - }, - { - "id": "deepseek-ai/deepseek-moe-16b-base", - "name": "deepseek-moe-16b-base", - "developer": "deepseek-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.245, - "hfopenllm_v2/BBH": 0.3409, - "hfopenllm_v2/MATH Level 5": 0.0242, - "hfopenllm_v2/GPQA": 0.2542, - "hfopenllm_v2/MUSR": 0.3658, - "hfopenllm_v2/MMLU-PRO": 0.1505 - } - }, - { - "id": "deepseek-ai/deepseek-moe-16b-chat", - "name": "deepseek-moe-16b-chat", - "developer": "deepseek-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3663, - "hfopenllm_v2/BBH": 0.3275, - "hfopenllm_v2/MATH Level 5": 0.0257, - "hfopenllm_v2/GPQA": 0.2248, - "hfopenllm_v2/MUSR": 0.3808, - "hfopenllm_v2/MMLU-PRO": 0.1964 - } - }, - { - "id": "deepseek-ai/deepseek-r1-0528", - "name": "DeepSeek-R1-0528", - "developer": "deepseek-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.699, - "helm_capabilities/MMLU-Pro": 0.793, - "helm_capabilities/GPQA": 0.666, - "helm_capabilities/IFEval": 0.784, - "helm_capabilities/WildBench": 0.828, - "helm_capabilities/Omni-MATH": 0.424 - } - }, - { - "id": "deepseek-ai/DeepSeek-R1-Distill-Llama-70B", - "name": "DeepSeek-R1-Distill-Llama-70B", - "developer": "deepseek-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4336, - "hfopenllm_v2/BBH": 0.5635, - "hfopenllm_v2/MATH Level 5": 0.3074, - "hfopenllm_v2/GPQA": 0.2651, - "hfopenllm_v2/MUSR": 0.4342, - "hfopenllm_v2/MMLU-PRO": 0.4748 - } - }, - { - "id": "deepseek-ai/DeepSeek-R1-Distill-Llama-8B", - "name": "DeepSeek-R1-Distill-Llama-8B", - "developer": "deepseek-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3782, - "hfopenllm_v2/BBH": 0.3239, - "hfopenllm_v2/MATH Level 5": 0.2198, - "hfopenllm_v2/GPQA": 0.255, - "hfopenllm_v2/MUSR": 0.325, - "hfopenllm_v2/MMLU-PRO": 0.2089 - } - }, - { - "id": "deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B", - "name": "DeepSeek-R1-Distill-Qwen-1.5B", - "developer": "deepseek-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3463, - "hfopenllm_v2/BBH": 0.3241, - "hfopenllm_v2/MATH Level 5": 0.1692, - "hfopenllm_v2/GPQA": 0.2559, - "hfopenllm_v2/MUSR": 0.3635, - "hfopenllm_v2/MMLU-PRO": 0.1187 - } - }, - { - "id": "deepseek-ai/DeepSeek-R1-Distill-Qwen-14B", - "name": "DeepSeek-R1-Distill-Qwen-14B", - "developer": "deepseek-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4382, - "hfopenllm_v2/BBH": 0.5906, - "hfopenllm_v2/MATH Level 5": 0.5702, - "hfopenllm_v2/GPQA": 0.3876, - "hfopenllm_v2/MUSR": 0.5366, - "hfopenllm_v2/MMLU-PRO": 0.4667 - } - }, - { - "id": "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B", - "name": "DeepSeek-R1-Distill-Qwen-32B", - "developer": "deepseek-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4186, - "hfopenllm_v2/BBH": 0.4197, - "hfopenllm_v2/MATH Level 5": 0.1707, - "hfopenllm_v2/GPQA": 0.2844, - "hfopenllm_v2/MUSR": 0.4526, - "hfopenllm_v2/MMLU-PRO": 0.4687 - } - }, - { - "id": "deepseek-ai/DeepSeek-R1-Distill-Qwen-7B", - "name": "DeepSeek-R1-Distill-Qwen-7B", - "developer": "deepseek-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4038, - "hfopenllm_v2/BBH": 0.3443, - "hfopenllm_v2/MATH Level 5": 0.1956, - "hfopenllm_v2/GPQA": 0.2794, - "hfopenllm_v2/MUSR": 0.3663, - "hfopenllm_v2/MMLU-PRO": 0.2321 - } - }, - { - "id": "deepseek-ai/deepseek-v3", - "name": "DeepSeek v3", - "developer": "deepseek-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.665, - "helm_capabilities/MMLU-Pro": 0.723, - "helm_capabilities/GPQA": 0.538, - "helm_capabilities/IFEval": 0.832, - "helm_capabilities/WildBench": 0.831, - "helm_capabilities/Omni-MATH": 0.403, - "helm_lite/Mean win rate": 0.908, - "helm_lite/NarrativeQA": 0.796, - "helm_lite/NaturalQuestions (closed-book)": 0.467, - "helm_lite/OpenbookQA": 0.954, - "helm_lite/MMLU": 0.803, - "helm_lite/MATH": 0.912, - "helm_lite/GSM8K": 0.94, - "helm_lite/LegalBench": 0.718, - "helm_lite/MedQA": 0.809, - "helm_lite/WMT 2014": 0.209, - "helm_mmlu/MMLU All Subjects": 0.872, - "helm_mmlu/Abstract Algebra": 0.84, - "helm_mmlu/Anatomy": 0.867, - "helm_mmlu/College Physics": 0.814, - "helm_mmlu/Computer Security": 0.86, - "helm_mmlu/Econometrics": 0.746, - "helm_mmlu/Global Facts": 0.68, - "helm_mmlu/Jurisprudence": 0.898, - "helm_mmlu/Philosophy": 0.9, - "helm_mmlu/Professional Psychology": 0.887, - "helm_mmlu/Us Foreign Policy": 0.92, - "helm_mmlu/Astronomy": 0.921, - "helm_mmlu/Business Ethics": 0.89, - "helm_mmlu/Clinical Knowledge": 0.913, - "helm_mmlu/Conceptual Physics": 0.94, - "helm_mmlu/Electrical Engineering": 0.869, - "helm_mmlu/Elementary Mathematics": 0.942, - "helm_mmlu/Formal Logic": 0.77, - "helm_mmlu/High School World History": 0.928, - "helm_mmlu/Human Sexuality": 0.924, - "helm_mmlu/International Law": 0.95, - "helm_mmlu/Logical Fallacies": 0.914, - "helm_mmlu/Machine Learning": 0.786, - "helm_mmlu/Management": 0.903, - "helm_mmlu/Marketing": 0.949, - "helm_mmlu/Medical Genetics": 0.96, - "helm_mmlu/Miscellaneous": 0.949, - "helm_mmlu/Moral Scenarios": 0.808, - "helm_mmlu/Nutrition": 0.918, - "helm_mmlu/Prehistory": 0.923, - "helm_mmlu/Public Relations": 0.809, - "helm_mmlu/Security Studies": 0.837, - "helm_mmlu/Sociology": 0.955, - "helm_mmlu/Virology": 0.596, - "helm_mmlu/World Religions": 0.912, - "helm_mmlu/Mean win rate": 0.215 - } - }, - { - "id": "deepseek/chat-v3-0324", - "name": "deepseek/chat-v3-0324", - "developer": "DeepSeek", - "evaluator_relationship": null, - "benchmark_scores": { - "livecodebenchpro/Hard Problems": 0.0, - "livecodebenchpro/Medium Problems": 0.0, - "livecodebenchpro/Easy Problems": 0.19718309859154928 - } - }, - { - "id": "deepseek/deepseek-r1-0528", - "name": "deepseek-r1-0528", - "developer": "deepseek", - "evaluator_relationship": null, - "benchmark_scores": { - "global-mmlu-lite/Global MMLU Lite": 0.6744, - "global-mmlu-lite/Culturally Sensitive": 0.6672, - "global-mmlu-lite/Culturally Agnostic": 0.6816, - "global-mmlu-lite/Arabic": 0.6825, - "global-mmlu-lite/English": 0.715, - "global-mmlu-lite/Bengali": 0.655, - "global-mmlu-lite/German": 0.6375, - "global-mmlu-lite/French": 0.6925, - "global-mmlu-lite/Hindi": 0.6475, - "global-mmlu-lite/Indonesian": 0.655, - "global-mmlu-lite/Italian": 0.6775, - "global-mmlu-lite/Japanese": 0.7725, - "global-mmlu-lite/Korean": 0.6575, - "global-mmlu-lite/Portuguese": 0.635, - "global-mmlu-lite/Spanish": 0.7175, - "global-mmlu-lite/Swahili": 0.6775, - "global-mmlu-lite/Yoruba": 0.77, - "global-mmlu-lite/Chinese": 0.5075, - "global-mmlu-lite/Burmese": 0.69 - } - }, - { - "id": "deepseek/deepseek-v3-2-exp-fc", - "name": "DeepSeek-V3.2-Exp (FC)", - "developer": "deepseek", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 19.0, - "bfcl/bfcl.overall.overall_accuracy": 54.12, - "bfcl/bfcl.overall.total_cost_usd": 6.71, - "bfcl/bfcl.overall.latency_mean_s": 5.83, - "bfcl/bfcl.overall.latency_std_s": 11.71, - "bfcl/bfcl.overall.latency_p95_s": 10.59, - "bfcl/bfcl.non_live.ast_accuracy": 34.85, - "bfcl/bfcl.non_live.simple_ast_accuracy": 37.92, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 74.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 15.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 12.5, - "bfcl/bfcl.live.live_accuracy": 53.66, - "bfcl/bfcl.live.live_simple_ast_accuracy": 66.28, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 51.66, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 25.0, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 25.0, - "bfcl/bfcl.multi_turn.accuracy": 37.38, - "bfcl/bfcl.multi_turn.base_accuracy": 41.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 39.5, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 33.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 35.0, - "bfcl/bfcl.web_search.accuracy": 69.5, - "bfcl/bfcl.web_search.base_accuracy": 80.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 59.0, - "bfcl/bfcl.memory.accuracy": 54.19, - "bfcl/bfcl.memory.kv_accuracy": 41.94, - "bfcl/bfcl.memory.vector_accuracy": 61.29, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 59.35, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 37.5, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 93.18 - } - }, - { - "id": "deepseek/deepseek-v3-2-exp-prompt-thinking", - "name": "DeepSeek-V3.2-Exp (Prompt + Thinking)", - "developer": "deepseek", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 14.0, - "bfcl/bfcl.overall.overall_accuracy": 56.73, - "bfcl/bfcl.overall.total_cost_usd": 57.75, - "bfcl/bfcl.overall.latency_mean_s": 37.89, - "bfcl/bfcl.overall.latency_std_s": 49.56, - "bfcl/bfcl.overall.latency_p95_s": 102.09, - "bfcl/bfcl.non_live.ast_accuracy": 85.52, - "bfcl/bfcl.non_live.simple_ast_accuracy": 74.08, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 92.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 89.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 86.5, - "bfcl/bfcl.live.live_accuracy": 76.02, - "bfcl/bfcl.live.live_simple_ast_accuracy": 82.56, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 74.74, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 87.5, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 54.17, - "bfcl/bfcl.multi_turn.accuracy": 44.88, - "bfcl/bfcl.multi_turn.base_accuracy": 55.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 49.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 27.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 48.5, - "bfcl/bfcl.web_search.accuracy": 58.0, - "bfcl/bfcl.web_search.base_accuracy": 64.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 52.0, - "bfcl/bfcl.memory.accuracy": 44.09, - "bfcl/bfcl.memory.kv_accuracy": 46.45, - "bfcl/bfcl.memory.vector_accuracy": 46.45, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 39.35, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 93.75, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 67.0, - "bfcl/bfcl.format_sensitivity.max_delta": 10.0, - "bfcl/bfcl.format_sensitivity.stddev": 2.77 - } - }, - { - "id": "deepseek/deepseek-v3.1", - "name": "deepseek-v3.1", - "developer": "deepseek", - "evaluator_relationship": null, - "benchmark_scores": { - "global-mmlu-lite/Global MMLU Lite": 0.8044, - "global-mmlu-lite/Culturally Sensitive": 0.7793, - "global-mmlu-lite/Culturally Agnostic": 0.8295, - "global-mmlu-lite/Arabic": 0.805, - "global-mmlu-lite/English": 0.825, - "global-mmlu-lite/Bengali": 0.8157, - "global-mmlu-lite/German": 0.7925, - "global-mmlu-lite/French": 0.8175, - "global-mmlu-lite/Hindi": 0.7569, - "global-mmlu-lite/Indonesian": 0.7764, - "global-mmlu-lite/Italian": 0.8075, - "global-mmlu-lite/Japanese": 0.8312, - "global-mmlu-lite/Korean": 0.8125, - "global-mmlu-lite/Portuguese": 0.8246, - "global-mmlu-lite/Spanish": 0.8125, - "global-mmlu-lite/Swahili": 0.801, - "global-mmlu-lite/Yoruba": 0.7831, - "global-mmlu-lite/Chinese": 0.8161, - "global-mmlu-lite/Burmese": 0.7925 - } - }, - { - "id": "deepseek/deepseek-v3.2", - "name": "DeepSeek-V3.2", - "developer": "DeepSeek", - "evaluator_relationship": null, - "benchmark_scores": { - "terminal-bench-2.0/terminal-bench-2.0": 39.6 - } - }, - { - "id": "deepseek/ep-20250214004308-p7n89", - "name": "ep-20250214004308-p7n89", - "developer": "DeepSeek", - "evaluator_relationship": null, - "benchmark_scores": { - "livecodebenchpro/Hard Problems": 0.0, - "livecodebenchpro/Medium Problems": 0.014084507042253521, - "livecodebenchpro/Easy Problems": 0.4225352112676056 - } - }, - { - "id": "deepseek/ep-20250228232227-z44x5", - "name": "ep-20250228232227-z44x5", - "developer": "DeepSeek", - "evaluator_relationship": null, - "benchmark_scores": { - "livecodebenchpro/Hard Problems": 0.0, - "livecodebenchpro/Medium Problems": 0.0, - "livecodebenchpro/Easy Problems": 0.1267605633802817 - } - }, - { - "id": "deepseek/ep-20250603132404-cgpjm", - "name": "ep-20250603132404-cgpjm", - "developer": "DeepSeek", - "evaluator_relationship": null, - "benchmark_scores": { - "livecodebenchpro/Hard Problems": 0.0, - "livecodebenchpro/Medium Problems": 0.08450704225352113, - "livecodebenchpro/Easy Problems": 0.5774647887323944 - } - }, - { - "id": "Delta-Vector/Baldur-8B", - "name": "Baldur-8B", - "developer": "Delta-Vector", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4782, - "hfopenllm_v2/BBH": 0.5306, - "hfopenllm_v2/MATH Level 5": 0.1435, - "hfopenllm_v2/GPQA": 0.302, - "hfopenllm_v2/MUSR": 0.4372, - "hfopenllm_v2/MMLU-PRO": 0.3654 - } - }, - { - "id": "Delta-Vector/Control-8B", - "name": "Control-8B", - "developer": "Delta-Vector", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.549, - "hfopenllm_v2/BBH": 0.5041, - "hfopenllm_v2/MATH Level 5": 0.139, - "hfopenllm_v2/GPQA": 0.3163, - "hfopenllm_v2/MUSR": 0.4355, - "hfopenllm_v2/MMLU-PRO": 0.3732 - } - }, - { - "id": "Delta-Vector/Control-8B-V1.1", - "name": "Control-8B-V1.1", - "developer": "Delta-Vector", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5697, - "hfopenllm_v2/BBH": 0.4993, - "hfopenllm_v2/MATH Level 5": 0.1276, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.4237, - "hfopenllm_v2/MMLU-PRO": 0.3745 - } - }, - { - "id": "Delta-Vector/Darkens-8B", - "name": "Darkens-8B", - "developer": "Delta-Vector", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2548, - "hfopenllm_v2/BBH": 0.5251, - "hfopenllm_v2/MATH Level 5": 0.0589, - "hfopenllm_v2/GPQA": 0.3247, - "hfopenllm_v2/MUSR": 0.4106, - "hfopenllm_v2/MMLU-PRO": 0.3736 - } - }, - { - "id": "Delta-Vector/Henbane-7b-attempt2", - "name": "Henbane-7b-attempt2", - "developer": "Delta-Vector", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4157, - "hfopenllm_v2/BBH": 0.5061, - "hfopenllm_v2/MATH Level 5": 0.2273, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.3973, - "hfopenllm_v2/MMLU-PRO": 0.4028 - } - }, - { - "id": "Delta-Vector/Odin-9B", - "name": "Odin-9B", - "developer": "Delta-Vector", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3692, - "hfopenllm_v2/BBH": 0.544, - "hfopenllm_v2/MATH Level 5": 0.145, - "hfopenllm_v2/GPQA": 0.3414, - "hfopenllm_v2/MUSR": 0.4648, - "hfopenllm_v2/MMLU-PRO": 0.4047 - } - }, - { - "id": "Delta-Vector/Tor-8B", - "name": "Tor-8B", - "developer": "Delta-Vector", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2382, - "hfopenllm_v2/BBH": 0.5209, - "hfopenllm_v2/MATH Level 5": 0.0589, - "hfopenllm_v2/GPQA": 0.3238, - "hfopenllm_v2/MUSR": 0.4092, - "hfopenllm_v2/MMLU-PRO": 0.373 - } - }, - { - "id": "DevQuasar/DevQuasar-R1-Uncensored-Llama-8B", - "name": "DevQuasar-R1-Uncensored-Llama-8B", - "developer": "DevQuasar", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3849, - "hfopenllm_v2/BBH": 0.5118, - "hfopenllm_v2/MATH Level 5": 0.3308, - "hfopenllm_v2/GPQA": 0.3473, - "hfopenllm_v2/MUSR": 0.4436, - "hfopenllm_v2/MMLU-PRO": 0.3615 - } - }, - { - "id": "dfurman/CalmeRys-78B-Orpo-v0.1", - "name": "CalmeRys-78B-Orpo-v0.1", - "developer": "dfurman", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8163, - "hfopenllm_v2/BBH": 0.7262, - "hfopenllm_v2/MATH Level 5": 0.4063, - "hfopenllm_v2/GPQA": 0.4002, - "hfopenllm_v2/MUSR": 0.5902, - "hfopenllm_v2/MMLU-PRO": 0.7012 - } - }, - { - "id": "dfurman/Llama-3-70B-Orpo-v0.1", - "name": "Llama-3-70B-Orpo-v0.1", - "developer": "dfurman", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2049, - "hfopenllm_v2/BBH": 0.4655, - "hfopenllm_v2/MATH Level 5": 0.1579, - "hfopenllm_v2/GPQA": 0.2576, - "hfopenllm_v2/MUSR": 0.4534, - "hfopenllm_v2/MMLU-PRO": 0.3893 - } - }, - { - "id": "dfurman/Llama-3-8B-Orpo-v0.1", - "name": "Llama-3-8B-Orpo-v0.1", - "developer": "dfurman", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2835, - "hfopenllm_v2/BBH": 0.3842, - "hfopenllm_v2/MATH Level 5": 0.0521, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.3566, - "hfopenllm_v2/MMLU-PRO": 0.2298 - } - }, - { - "id": "dfurman/Qwen2-72B-Orpo-v0.1", - "name": "Qwen2-72B-Orpo-v0.1", - "developer": "dfurman", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.788, - "hfopenllm_v2/BBH": 0.6969, - "hfopenllm_v2/MATH Level 5": 0.4056, - "hfopenllm_v2/GPQA": 0.3842, - "hfopenllm_v2/MUSR": 0.4784, - "hfopenllm_v2/MMLU-PRO": 0.5455 - } - }, - { - "id": "dicta-il/dictalm2.0", - "name": "dictalm2.0", - "developer": "dicta-il", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2413, - "hfopenllm_v2/BBH": 0.4018, - "hfopenllm_v2/MATH Level 5": 0.0181, - "hfopenllm_v2/GPQA": 0.2919, - "hfopenllm_v2/MUSR": 0.382, - "hfopenllm_v2/MMLU-PRO": 0.2605 - } - }, - { - "id": "dicta-il/dictalm2.0-instruct", - "name": "dictalm2.0-instruct", - "developer": "dicta-il", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4412, - "hfopenllm_v2/BBH": 0.4256, - "hfopenllm_v2/MATH Level 5": 0.0227, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.3946, - "hfopenllm_v2/MMLU-PRO": 0.2605 - } - }, - { - "id": "distilbert/distilgpt2", - "name": "distilgpt2", - "developer": "distilbert", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0611, - "hfopenllm_v2/BBH": 0.3038, - "hfopenllm_v2/MATH Level 5": 0.006, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.4207, - "hfopenllm_v2/MMLU-PRO": 0.1187 - } - }, - { - "id": "divyanshukunwar/SASTRI_1_9B", - "name": "SASTRI_1_9B", - "developer": "divyanshukunwar", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4207, - "hfopenllm_v2/BBH": 0.468, - "hfopenllm_v2/MATH Level 5": 0.1156, - "hfopenllm_v2/GPQA": 0.3213, - "hfopenllm_v2/MUSR": 0.3831, - "hfopenllm_v2/MMLU-PRO": 0.3187 - } - }, - { - "id": "djuna-test-lab/TEST-L3.2-ReWish-3B", - "name": "TEST-L3.2-ReWish-3B", - "developer": "djuna-test-lab", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6368, - "hfopenllm_v2/BBH": 0.4495, - "hfopenllm_v2/MATH Level 5": 0.1367, - "hfopenllm_v2/GPQA": 0.2836, - "hfopenllm_v2/MUSR": 0.3777, - "hfopenllm_v2/MMLU-PRO": 0.3126 - } - }, - { - "id": "djuna-test-lab/TEST-L3.2-ReWish-3B-ties-w-base", - "name": "TEST-L3.2-ReWish-3B-ties-w-base", - "developer": "djuna-test-lab", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6353, - "hfopenllm_v2/BBH": 0.4495, - "hfopenllm_v2/MATH Level 5": 0.1367, - "hfopenllm_v2/GPQA": 0.2836, - "hfopenllm_v2/MUSR": 0.3777, - "hfopenllm_v2/MMLU-PRO": 0.3126 - } - }, - { - "id": "djuna/G2-BigGSHT-27B-2", - "name": "G2-BigGSHT-27B-2", - "developer": "djuna", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7974, - "hfopenllm_v2/BBH": 0.6415, - "hfopenllm_v2/MATH Level 5": 0.2349, - "hfopenllm_v2/GPQA": 0.3633, - "hfopenllm_v2/MUSR": 0.4072, - "hfopenllm_v2/MMLU-PRO": 0.4528 - } - }, - { - "id": "djuna/G2-GSHT", - "name": "G2-GSHT", - "developer": "djuna", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.563, - "hfopenllm_v2/BBH": 0.527, - "hfopenllm_v2/MATH Level 5": 0.1926, - "hfopenllm_v2/GPQA": 0.3255, - "hfopenllm_v2/MUSR": 0.4006, - "hfopenllm_v2/MMLU-PRO": 0.307 - } - }, - { - "id": "djuna/Gemma-2-gemmama-9b", - "name": "Gemma-2-gemmama-9b", - "developer": "djuna", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7703, - "hfopenllm_v2/BBH": 0.542, - "hfopenllm_v2/MATH Level 5": 0.1926, - "hfopenllm_v2/GPQA": 0.3356, - "hfopenllm_v2/MUSR": 0.4031, - "hfopenllm_v2/MMLU-PRO": 0.3109 - } - }, - { - "id": "djuna/L3.1-ForStHS", - "name": "L3.1-ForStHS", - "developer": "djuna", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7813, - "hfopenllm_v2/BBH": 0.5203, - "hfopenllm_v2/MATH Level 5": 0.1503, - "hfopenllm_v2/GPQA": 0.2911, - "hfopenllm_v2/MUSR": 0.4026, - "hfopenllm_v2/MMLU-PRO": 0.3735 - } - }, - { - "id": "djuna/L3.1-Promissum_Mane-8B-Della-1.5-calc", - "name": "L3.1-Promissum_Mane-8B-Della-1.5-calc", - "developer": "djuna", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7235, - "hfopenllm_v2/BBH": 0.5433, - "hfopenllm_v2/MATH Level 5": 0.1639, - "hfopenllm_v2/GPQA": 0.3146, - "hfopenllm_v2/MUSR": 0.4253, - "hfopenllm_v2/MMLU-PRO": 0.3904 - } - }, - { - "id": "djuna/L3.1-Promissum_Mane-8B-Della-calc", - "name": "L3.1-Promissum_Mane-8B-Della-calc", - "developer": "djuna", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5442, - "hfopenllm_v2/BBH": 0.5486, - "hfopenllm_v2/MATH Level 5": 0.1843, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.423, - "hfopenllm_v2/MMLU-PRO": 0.3802 - } - }, - { - "id": "djuna/L3.1-Purosani-2-8B", - "name": "L3.1-Purosani-2-8B", - "developer": "djuna", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4988, - "hfopenllm_v2/BBH": 0.5182, - "hfopenllm_v2/MATH Level 5": 0.1171, - "hfopenllm_v2/GPQA": 0.3012, - "hfopenllm_v2/MUSR": 0.3816, - "hfopenllm_v2/MMLU-PRO": 0.3752 - } - }, - { - "id": "djuna/L3.1-Suze-Vume-calc", - "name": "L3.1-Suze-Vume-calc", - "developer": "djuna", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7297, - "hfopenllm_v2/BBH": 0.5164, - "hfopenllm_v2/MATH Level 5": 0.114, - "hfopenllm_v2/GPQA": 0.2819, - "hfopenllm_v2/MUSR": 0.3843, - "hfopenllm_v2/MMLU-PRO": 0.3515 - } - }, - { - "id": "djuna/MN-Chinofun", - "name": "MN-Chinofun", - "developer": "djuna", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.611, - "hfopenllm_v2/BBH": 0.4953, - "hfopenllm_v2/MATH Level 5": 0.1307, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.4084, - "hfopenllm_v2/MMLU-PRO": 0.3603 - } - }, - { - "id": "djuna/MN-Chinofun-12B-2", - "name": "MN-Chinofun-12B-2", - "developer": "djuna", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6171, - "hfopenllm_v2/BBH": 0.5037, - "hfopenllm_v2/MATH Level 5": 0.1307, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.4268, - "hfopenllm_v2/MMLU-PRO": 0.3615 - } - }, - { - "id": "djuna/MN-Chinofun-12B-3", - "name": "MN-Chinofun-12B-3", - "developer": "djuna", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3053, - "hfopenllm_v2/BBH": 0.5348, - "hfopenllm_v2/MATH Level 5": 0.1005, - "hfopenllm_v2/GPQA": 0.2659, - "hfopenllm_v2/MUSR": 0.4198, - "hfopenllm_v2/MMLU-PRO": 0.3026 - } - }, - { - "id": "djuna/MN-Chinofun-12B-4", - "name": "MN-Chinofun-12B-4", - "developer": "djuna", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5404, - "hfopenllm_v2/BBH": 0.5348, - "hfopenllm_v2/MATH Level 5": 0.1118, - "hfopenllm_v2/GPQA": 0.2953, - "hfopenllm_v2/MUSR": 0.4307, - "hfopenllm_v2/MMLU-PRO": 0.3497 - } - }, - { - "id": "djuna/Q2.5-Partron-7B", - "name": "Q2.5-Partron-7B", - "developer": "djuna", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7321, - "hfopenllm_v2/BBH": 0.5418, - "hfopenllm_v2/MATH Level 5": 0.4826, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.4165, - "hfopenllm_v2/MMLU-PRO": 0.4283 - } - }, - { - "id": "djuna/Q2.5-Veltha-14B", - "name": "Q2.5-Veltha-14B", - "developer": "djuna", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8292, - "hfopenllm_v2/BBH": 0.6484, - "hfopenllm_v2/MATH Level 5": 0.4789, - "hfopenllm_v2/GPQA": 0.3591, - "hfopenllm_v2/MUSR": 0.4194, - "hfopenllm_v2/MMLU-PRO": 0.5298 - } - }, - { - "id": "djuna/Q2.5-Veltha-14B-0.5", - "name": "Q2.5-Veltha-14B-0.5", - "developer": "djuna", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7796, - "hfopenllm_v2/BBH": 0.6523, - "hfopenllm_v2/MATH Level 5": 0.4373, - "hfopenllm_v2/GPQA": 0.3683, - "hfopenllm_v2/MUSR": 0.4339, - "hfopenllm_v2/MMLU-PRO": 0.5295 - } - }, - { - "id": "dnhkng/RYS-Llama-3-8B-Instruct", - "name": "RYS-Llama-3-8B-Instruct", - "developer": "dnhkng", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6958, - "hfopenllm_v2/BBH": 0.4809, - "hfopenllm_v2/MATH Level 5": 0.0687, - "hfopenllm_v2/GPQA": 0.2576, - "hfopenllm_v2/MUSR": 0.3383, - "hfopenllm_v2/MMLU-PRO": 0.3557 - } - }, - { - "id": "dnhkng/RYS-Llama-3-Huge-Instruct", - "name": "RYS-Llama-3-Huge-Instruct", - "developer": "dnhkng", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7686, - "hfopenllm_v2/BBH": 0.6481, - "hfopenllm_v2/MATH Level 5": 0.2289, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.4208, - "hfopenllm_v2/MMLU-PRO": 0.511 - } - }, - { - "id": "dnhkng/RYS-Llama-3-Large-Instruct", - "name": "RYS-Llama-3-Large-Instruct", - "developer": "dnhkng", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8051, - "hfopenllm_v2/BBH": 0.6525, - "hfopenllm_v2/MATH Level 5": 0.2304, - "hfopenllm_v2/GPQA": 0.2894, - "hfopenllm_v2/MUSR": 0.418, - "hfopenllm_v2/MMLU-PRO": 0.5137 - } - }, - { - "id": "dnhkng/RYS-Llama-3.1-8B-Instruct", - "name": "RYS-Llama-3.1-8B-Instruct", - "developer": "dnhkng", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7685, - "hfopenllm_v2/BBH": 0.5164, - "hfopenllm_v2/MATH Level 5": 0.1329, - "hfopenllm_v2/GPQA": 0.2676, - "hfopenllm_v2/MUSR": 0.3681, - "hfopenllm_v2/MMLU-PRO": 0.3639 - } - }, - { - "id": "dnhkng/RYS-Llama3.1-Large", - "name": "RYS-Llama3.1-Large", - "developer": "dnhkng", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8492, - "hfopenllm_v2/BBH": 0.6899, - "hfopenllm_v2/MATH Level 5": 0.3505, - "hfopenllm_v2/GPQA": 0.3742, - "hfopenllm_v2/MUSR": 0.4554, - "hfopenllm_v2/MMLU-PRO": 0.5249 - } - }, - { - "id": "dnhkng/RYS-Medium", - "name": "RYS-Medium", - "developer": "dnhkng", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4406, - "hfopenllm_v2/BBH": 0.6285, - "hfopenllm_v2/MATH Level 5": 0.108, - "hfopenllm_v2/GPQA": 0.328, - "hfopenllm_v2/MUSR": 0.4069, - "hfopenllm_v2/MMLU-PRO": 0.4326 - } - }, - { - "id": "dnhkng/RYS-Phi-3-medium-4k-instruct", - "name": "RYS-Phi-3-medium-4k-instruct", - "developer": "dnhkng", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4391, - "hfopenllm_v2/BBH": 0.6226, - "hfopenllm_v2/MATH Level 5": 0.1609, - "hfopenllm_v2/GPQA": 0.3549, - "hfopenllm_v2/MUSR": 0.4253, - "hfopenllm_v2/MMLU-PRO": 0.4846 - } - }, - { - "id": "dnhkng/RYS-XLarge", - "name": "RYS-XLarge", - "developer": "dnhkng", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7996, - "hfopenllm_v2/BBH": 0.705, - "hfopenllm_v2/MATH Level 5": 0.4252, - "hfopenllm_v2/GPQA": 0.3842, - "hfopenllm_v2/MUSR": 0.497, - "hfopenllm_v2/MMLU-PRO": 0.5428 - } - }, - { - "id": "dnhkng/RYS-XLarge-base", - "name": "RYS-XLarge-base", - "developer": "dnhkng", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.791, - "hfopenllm_v2/BBH": 0.7047, - "hfopenllm_v2/MATH Level 5": 0.3792, - "hfopenllm_v2/GPQA": 0.3792, - "hfopenllm_v2/MUSR": 0.4903, - "hfopenllm_v2/MMLU-PRO": 0.5431 - } - }, - { - "id": "dnhkng/RYS-XLarge2", - "name": "RYS-XLarge2", - "developer": "dnhkng", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4902, - "hfopenllm_v2/BBH": 0.6574, - "hfopenllm_v2/MATH Level 5": 0.2749, - "hfopenllm_v2/GPQA": 0.3742, - "hfopenllm_v2/MUSR": 0.4508, - "hfopenllm_v2/MMLU-PRO": 0.5378 - } - }, - { - "id": "Dongwei/DeepSeek-R1-Distill-Qwen-7B-GRPO", - "name": "DeepSeek-R1-Distill-Qwen-7B-GRPO", - "developer": "Dongwei", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4038, - "hfopenllm_v2/BBH": 0.3443, - "hfopenllm_v2/MATH Level 5": 0.1956, - "hfopenllm_v2/GPQA": 0.2794, - "hfopenllm_v2/MUSR": 0.3663, - "hfopenllm_v2/MMLU-PRO": 0.2322 - } - }, - { - "id": "DoppelReflEx/L3-8B-R1-WolfCore", - "name": "L3-8B-R1-WolfCore", - "developer": "DoppelReflEx", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3775, - "hfopenllm_v2/BBH": 0.5318, - "hfopenllm_v2/MATH Level 5": 0.1631, - "hfopenllm_v2/GPQA": 0.3289, - "hfopenllm_v2/MUSR": 0.4277, - "hfopenllm_v2/MMLU-PRO": 0.3717 - } - }, - { - "id": "DoppelReflEx/L3-8B-R1-WolfCore-V1.5-test", - "name": "L3-8B-R1-WolfCore-V1.5-test", - "developer": "DoppelReflEx", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3955, - "hfopenllm_v2/BBH": 0.5315, - "hfopenllm_v2/MATH Level 5": 0.1231, - "hfopenllm_v2/GPQA": 0.3263, - "hfopenllm_v2/MUSR": 0.3841, - "hfopenllm_v2/MMLU-PRO": 0.3728 - } - }, - { - "id": "DoppelReflEx/L3-8B-WolfCore", - "name": "L3-8B-WolfCore", - "developer": "DoppelReflEx", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4022, - "hfopenllm_v2/BBH": 0.5182, - "hfopenllm_v2/MATH Level 5": 0.0982, - "hfopenllm_v2/GPQA": 0.3096, - "hfopenllm_v2/MUSR": 0.3973, - "hfopenllm_v2/MMLU-PRO": 0.3705 - } - }, - { - "id": "DoppelReflEx/MiniusLight-24B", - "name": "MiniusLight-24B", - "developer": "DoppelReflEx", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2577, - "hfopenllm_v2/BBH": 0.6256, - "hfopenllm_v2/MATH Level 5": 0.1261, - "hfopenllm_v2/GPQA": 0.3582, - "hfopenllm_v2/MUSR": 0.4319, - "hfopenllm_v2/MMLU-PRO": 0.5091 - } - }, - { - "id": "DoppelReflEx/MiniusLight-24B-test", - "name": "MiniusLight-24B-test", - "developer": "DoppelReflEx", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0394, - "hfopenllm_v2/BBH": 0.6334, - "hfopenllm_v2/MATH Level 5": 0.0257, - "hfopenllm_v2/GPQA": 0.3683, - "hfopenllm_v2/MUSR": 0.4093, - "hfopenllm_v2/MMLU-PRO": 0.5182 - } - }, - { - "id": "DoppelReflEx/MiniusLight-24B-v1b-test", - "name": "MiniusLight-24B-v1b-test", - "developer": "DoppelReflEx", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3791, - "hfopenllm_v2/BBH": 0.6617, - "hfopenllm_v2/MATH Level 5": 0.2394, - "hfopenllm_v2/GPQA": 0.3792, - "hfopenllm_v2/MUSR": 0.4557, - "hfopenllm_v2/MMLU-PRO": 0.5365 - } - }, - { - "id": "DoppelReflEx/MiniusLight-24B-v1c-test", - "name": "MiniusLight-24B-v1c-test", - "developer": "DoppelReflEx", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3786, - "hfopenllm_v2/BBH": 0.6753, - "hfopenllm_v2/MATH Level 5": 0.2968, - "hfopenllm_v2/GPQA": 0.3951, - "hfopenllm_v2/MUSR": 0.4634, - "hfopenllm_v2/MMLU-PRO": 0.5487 - } - }, - { - "id": "DoppelReflEx/MiniusLight-24B-v1d-test", - "name": "MiniusLight-24B-v1d-test", - "developer": "DoppelReflEx", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4032, - "hfopenllm_v2/BBH": 0.6712, - "hfopenllm_v2/MATH Level 5": 0.2946, - "hfopenllm_v2/GPQA": 0.3951, - "hfopenllm_v2/MUSR": 0.4621, - "hfopenllm_v2/MMLU-PRO": 0.5489 - } - }, - { - "id": "DoppelReflEx/MN-12B-FoxFrame-test", - "name": "MN-12B-FoxFrame-test", - "developer": "DoppelReflEx", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4222, - "hfopenllm_v2/BBH": 0.5456, - "hfopenllm_v2/MATH Level 5": 0.1397, - "hfopenllm_v2/GPQA": 0.3079, - "hfopenllm_v2/MUSR": 0.4254, - "hfopenllm_v2/MMLU-PRO": 0.3503 - } - }, - { - "id": "DoppelReflEx/MN-12B-FoxFrame2-test", - "name": "MN-12B-FoxFrame2-test", - "developer": "DoppelReflEx", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4319, - "hfopenllm_v2/BBH": 0.5485, - "hfopenllm_v2/MATH Level 5": 0.1405, - "hfopenllm_v2/GPQA": 0.3146, - "hfopenllm_v2/MUSR": 0.4252, - "hfopenllm_v2/MMLU-PRO": 0.3569 - } - }, - { - "id": "DoppelReflEx/MN-12B-FoxFrame3-test", - "name": "MN-12B-FoxFrame3-test", - "developer": "DoppelReflEx", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4323, - "hfopenllm_v2/BBH": 0.5395, - "hfopenllm_v2/MATH Level 5": 0.1322, - "hfopenllm_v2/GPQA": 0.3012, - "hfopenllm_v2/MUSR": 0.4598, - "hfopenllm_v2/MMLU-PRO": 0.3529 - } - }, - { - "id": "DoppelReflEx/MN-12B-Kakigori", - "name": "MN-12B-Kakigori", - "developer": "DoppelReflEx", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3593, - "hfopenllm_v2/BBH": 0.5416, - "hfopenllm_v2/MATH Level 5": 0.1193, - "hfopenllm_v2/GPQA": 0.3247, - "hfopenllm_v2/MUSR": 0.4052, - "hfopenllm_v2/MMLU-PRO": 0.3581 - } - }, - { - "id": "DoppelReflEx/MN-12B-LilithFrame", - "name": "MN-12B-LilithFrame", - "developer": "DoppelReflEx", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.436, - "hfopenllm_v2/BBH": 0.4956, - "hfopenllm_v2/MATH Level 5": 0.0589, - "hfopenllm_v2/GPQA": 0.3205, - "hfopenllm_v2/MUSR": 0.3843, - "hfopenllm_v2/MMLU-PRO": 0.3237 - } - }, - { - "id": "DoppelReflEx/MN-12B-LilithFrame-Experiment-2", - "name": "MN-12B-LilithFrame-Experiment-2", - "developer": "DoppelReflEx", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4299, - "hfopenllm_v2/BBH": 0.4983, - "hfopenllm_v2/MATH Level 5": 0.1073, - "hfopenllm_v2/GPQA": 0.3255, - "hfopenllm_v2/MUSR": 0.3804, - "hfopenllm_v2/MMLU-PRO": 0.3276 - } - }, - { - "id": "DoppelReflEx/MN-12B-LilithFrame-Experiment-3", - "name": "MN-12B-LilithFrame-Experiment-3", - "developer": "DoppelReflEx", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4128, - "hfopenllm_v2/BBH": 0.5468, - "hfopenllm_v2/MATH Level 5": 0.1344, - "hfopenllm_v2/GPQA": 0.328, - "hfopenllm_v2/MUSR": 0.4039, - "hfopenllm_v2/MMLU-PRO": 0.3604 - } - }, - { - "id": "DoppelReflEx/MN-12B-LilithFrame-Experiment-4", - "name": "MN-12B-LilithFrame-Experiment-4", - "developer": "DoppelReflEx", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3981, - "hfopenllm_v2/BBH": 0.5534, - "hfopenllm_v2/MATH Level 5": 0.1224, - "hfopenllm_v2/GPQA": 0.3171, - "hfopenllm_v2/MUSR": 0.4371, - "hfopenllm_v2/MMLU-PRO": 0.3649 - } - }, - { - "id": "DoppelReflEx/MN-12B-Mimicore-GreenSnake", - "name": "MN-12B-Mimicore-GreenSnake", - "developer": "DoppelReflEx", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.478, - "hfopenllm_v2/BBH": 0.5481, - "hfopenllm_v2/MATH Level 5": 0.139, - "hfopenllm_v2/GPQA": 0.3247, - "hfopenllm_v2/MUSR": 0.4306, - "hfopenllm_v2/MMLU-PRO": 0.3651 - } - }, - { - "id": "DoppelReflEx/MN-12B-Mimicore-Nocturne", - "name": "MN-12B-Mimicore-Nocturne", - "developer": "DoppelReflEx", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3957, - "hfopenllm_v2/BBH": 0.5703, - "hfopenllm_v2/MATH Level 5": 0.1057, - "hfopenllm_v2/GPQA": 0.3196, - "hfopenllm_v2/MUSR": 0.4569, - "hfopenllm_v2/MMLU-PRO": 0.3634 - } - }, - { - "id": "DoppelReflEx/MN-12B-Mimicore-Orochi", - "name": "MN-12B-Mimicore-Orochi", - "developer": "DoppelReflEx", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.462, - "hfopenllm_v2/BBH": 0.5498, - "hfopenllm_v2/MATH Level 5": 0.136, - "hfopenllm_v2/GPQA": 0.3129, - "hfopenllm_v2/MUSR": 0.4546, - "hfopenllm_v2/MMLU-PRO": 0.3447 - } - }, - { - "id": "DoppelReflEx/MN-12B-Mimicore-Orochi-v2-Experiment", - "name": "MN-12B-Mimicore-Orochi-v2-Experiment", - "developer": "DoppelReflEx", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2842, - "hfopenllm_v2/BBH": 0.5323, - "hfopenllm_v2/MATH Level 5": 0.0612, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.4574, - "hfopenllm_v2/MMLU-PRO": 0.3423 - } - }, - { - "id": "DoppelReflEx/MN-12B-Mimicore-Orochi-v3-Experiment", - "name": "MN-12B-Mimicore-Orochi-v3-Experiment", - "developer": "DoppelReflEx", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4102, - "hfopenllm_v2/BBH": 0.5438, - "hfopenllm_v2/MATH Level 5": 0.1216, - "hfopenllm_v2/GPQA": 0.2928, - "hfopenllm_v2/MUSR": 0.4438, - "hfopenllm_v2/MMLU-PRO": 0.3396 - } - }, - { - "id": "DoppelReflEx/MN-12B-Mimicore-Orochi-v4-Experiment", - "name": "MN-12B-Mimicore-Orochi-v4-Experiment", - "developer": "DoppelReflEx", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4321, - "hfopenllm_v2/BBH": 0.5463, - "hfopenllm_v2/MATH Level 5": 0.1208, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.4449, - "hfopenllm_v2/MMLU-PRO": 0.352 - } - }, - { - "id": "DoppelReflEx/MN-12B-Mimicore-WhiteSnake", - "name": "MN-12B-Mimicore-WhiteSnake", - "developer": "DoppelReflEx", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4438, - "hfopenllm_v2/BBH": 0.5605, - "hfopenllm_v2/MATH Level 5": 0.1314, - "hfopenllm_v2/GPQA": 0.318, - "hfopenllm_v2/MUSR": 0.4569, - "hfopenllm_v2/MMLU-PRO": 0.3658 - } - }, - { - "id": "DoppelReflEx/MN-12B-Mimicore-WhiteSnake-v2-Experiment-1", - "name": "MN-12B-Mimicore-WhiteSnake-v2-Experiment-1", - "developer": "DoppelReflEx", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3909, - "hfopenllm_v2/BBH": 0.4866, - "hfopenllm_v2/MATH Level 5": 0.0785, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.379, - "hfopenllm_v2/MMLU-PRO": 0.3114 - } - }, - { - "id": "DoppelReflEx/MN-12B-Mimicore-WhiteSnake-v2-Experiment-2", - "name": "MN-12B-Mimicore-WhiteSnake-v2-Experiment-2", - "developer": "DoppelReflEx", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3124, - "hfopenllm_v2/BBH": 0.5126, - "hfopenllm_v2/MATH Level 5": 0.1125, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.3975, - "hfopenllm_v2/MMLU-PRO": 0.3314 - } - }, - { - "id": "DoppelReflEx/MN-12B-Mimicore-WhiteSnake-v2-Experiment-3", - "name": "MN-12B-Mimicore-WhiteSnake-v2-Experiment-3", - "developer": "DoppelReflEx", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4302, - "hfopenllm_v2/BBH": 0.4812, - "hfopenllm_v2/MATH Level 5": 0.0899, - "hfopenllm_v2/GPQA": 0.302, - "hfopenllm_v2/MUSR": 0.3684, - "hfopenllm_v2/MMLU-PRO": 0.3198 - } - }, - { - "id": "DoppelReflEx/MN-12B-Mimicore-WhiteSnake-v2-Experiment-4", - "name": "MN-12B-Mimicore-WhiteSnake-v2-Experiment-4", - "developer": "DoppelReflEx", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4241, - "hfopenllm_v2/BBH": 0.5185, - "hfopenllm_v2/MATH Level 5": 0.114, - "hfopenllm_v2/GPQA": 0.3104, - "hfopenllm_v2/MUSR": 0.4002, - "hfopenllm_v2/MMLU-PRO": 0.3342 - } - }, - { - "id": "DoppelReflEx/MN-12B-Unleashed-Twilight", - "name": "MN-12B-Unleashed-Twilight", - "developer": "DoppelReflEx", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3505, - "hfopenllm_v2/BBH": 0.5521, - "hfopenllm_v2/MATH Level 5": 0.0959, - "hfopenllm_v2/GPQA": 0.3289, - "hfopenllm_v2/MUSR": 0.4384, - "hfopenllm_v2/MMLU-PRO": 0.3678 - } - }, - { - "id": "DoppelReflEx/MN-12B-WolFrame", - "name": "MN-12B-WolFrame", - "developer": "DoppelReflEx", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4397, - "hfopenllm_v2/BBH": 0.5117, - "hfopenllm_v2/MATH Level 5": 0.1314, - "hfopenllm_v2/GPQA": 0.3104, - "hfopenllm_v2/MUSR": 0.4015, - "hfopenllm_v2/MMLU-PRO": 0.3393 - } - }, - { - "id": "DreadPoor/Again-8B-Model_Stock", - "name": "Again-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6724, - "hfopenllm_v2/BBH": 0.531, - "hfopenllm_v2/MATH Level 5": 0.1201, - "hfopenllm_v2/GPQA": 0.3012, - "hfopenllm_v2/MUSR": 0.3987, - "hfopenllm_v2/MMLU-PRO": 0.3518 - } - }, - { - "id": "DreadPoor/Alita99-8B-LINEAR", - "name": "Alita99-8B-LINEAR", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.719, - "hfopenllm_v2/BBH": 0.5442, - "hfopenllm_v2/MATH Level 5": 0.1647, - "hfopenllm_v2/GPQA": 0.3163, - "hfopenllm_v2/MUSR": 0.4266, - "hfopenllm_v2/MMLU-PRO": 0.3809 - } - }, - { - "id": "DreadPoor/AnotherTest", - "name": "AnotherTest", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4701, - "hfopenllm_v2/BBH": 0.4683, - "hfopenllm_v2/MATH Level 5": 0.0619, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.4213, - "hfopenllm_v2/MMLU-PRO": 0.2875 - } - }, - { - "id": "DreadPoor/Aspire-8B-model_stock", - "name": "Aspire-8B-model_stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7141, - "hfopenllm_v2/BBH": 0.5278, - "hfopenllm_v2/MATH Level 5": 0.1495, - "hfopenllm_v2/GPQA": 0.3146, - "hfopenllm_v2/MUSR": 0.4212, - "hfopenllm_v2/MMLU-PRO": 0.3763 - } - }, - { - "id": "DreadPoor/Aspire_1.3-8B_model-stock", - "name": "Aspire_1.3-8B_model-stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7062, - "hfopenllm_v2/BBH": 0.5302, - "hfopenllm_v2/MATH Level 5": 0.1692, - "hfopenllm_v2/GPQA": 0.3079, - "hfopenllm_v2/MUSR": 0.4105, - "hfopenllm_v2/MMLU-PRO": 0.3716 - } - }, - { - "id": "DreadPoor/Aspire_V2-8B-Model_Stock", - "name": "Aspire_V2-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7371, - "hfopenllm_v2/BBH": 0.533, - "hfopenllm_v2/MATH Level 5": 0.176, - "hfopenllm_v2/GPQA": 0.3205, - "hfopenllm_v2/MUSR": 0.3894, - "hfopenllm_v2/MMLU-PRO": 0.3697 - } - }, - { - "id": "DreadPoor/Aspire_V2.1-8B-Model_Stock", - "name": "Aspire_V2.1-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7238, - "hfopenllm_v2/BBH": 0.5236, - "hfopenllm_v2/MATH Level 5": 0.1767, - "hfopenllm_v2/GPQA": 0.3096, - "hfopenllm_v2/MUSR": 0.4136, - "hfopenllm_v2/MMLU-PRO": 0.3801 - } - }, - { - "id": "DreadPoor/Aspire_V2_ALT-8B-Model_Stock", - "name": "Aspire_V2_ALT-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7381, - "hfopenllm_v2/BBH": 0.5266, - "hfopenllm_v2/MATH Level 5": 0.173, - "hfopenllm_v2/GPQA": 0.3247, - "hfopenllm_v2/MUSR": 0.3975, - "hfopenllm_v2/MMLU-PRO": 0.3727 - } - }, - { - "id": "DreadPoor/Aspire_V2_ALT_ROW-8B-Model_Stock", - "name": "Aspire_V2_ALT_ROW-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7381, - "hfopenllm_v2/BBH": 0.5266, - "hfopenllm_v2/MATH Level 5": 0.173, - "hfopenllm_v2/GPQA": 0.3247, - "hfopenllm_v2/MUSR": 0.3975, - "hfopenllm_v2/MMLU-PRO": 0.3727 - } - }, - { - "id": "DreadPoor/Aspire_V3-8B-Model_Stock", - "name": "Aspire_V3-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5119, - "hfopenllm_v2/BBH": 0.5268, - "hfopenllm_v2/MATH Level 5": 0.1858, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.4015, - "hfopenllm_v2/MMLU-PRO": 0.3642 - } - }, - { - "id": "DreadPoor/Aspire_V4-8B-Model_Stock", - "name": "Aspire_V4-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7694, - "hfopenllm_v2/BBH": 0.5314, - "hfopenllm_v2/MATH Level 5": 0.1926, - "hfopenllm_v2/GPQA": 0.3045, - "hfopenllm_v2/MUSR": 0.3867, - "hfopenllm_v2/MMLU-PRO": 0.3708 - } - }, - { - "id": "DreadPoor/Aspire_V4_ALT-8B-Model_Stock", - "name": "Aspire_V4_ALT-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7366, - "hfopenllm_v2/BBH": 0.5268, - "hfopenllm_v2/MATH Level 5": 0.1813, - "hfopenllm_v2/GPQA": 0.3205, - "hfopenllm_v2/MUSR": 0.392, - "hfopenllm_v2/MMLU-PRO": 0.3682 - } - }, - { - "id": "DreadPoor/Asymmetric_Linearity-8B-Model_Stock", - "name": "Asymmetric_Linearity-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7174, - "hfopenllm_v2/BBH": 0.5465, - "hfopenllm_v2/MATH Level 5": 0.1647, - "hfopenllm_v2/GPQA": 0.3146, - "hfopenllm_v2/MUSR": 0.4199, - "hfopenllm_v2/MMLU-PRO": 0.3844 - } - }, - { - "id": "DreadPoor/Aurora_faustus-8B-LINEAR", - "name": "Aurora_faustus-8B-LINEAR", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7281, - "hfopenllm_v2/BBH": 0.5516, - "hfopenllm_v2/MATH Level 5": 0.1707, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.4146, - "hfopenllm_v2/MMLU-PRO": 0.3842 - } - }, - { - "id": "DreadPoor/Aurora_faustus-8B-LORABLATED", - "name": "Aurora_faustus-8B-LORABLATED", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7527, - "hfopenllm_v2/BBH": 0.5392, - "hfopenllm_v2/MATH Level 5": 0.1488, - "hfopenllm_v2/GPQA": 0.302, - "hfopenllm_v2/MUSR": 0.4239, - "hfopenllm_v2/MMLU-PRO": 0.3673 - } - }, - { - "id": "DreadPoor/Aurora_faustus-8B-LORABLATED_ALT", - "name": "Aurora_faustus-8B-LORABLATED_ALT", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7378, - "hfopenllm_v2/BBH": 0.5388, - "hfopenllm_v2/MATH Level 5": 0.1586, - "hfopenllm_v2/GPQA": 0.2987, - "hfopenllm_v2/MUSR": 0.4225, - "hfopenllm_v2/MMLU-PRO": 0.3694 - } - }, - { - "id": "DreadPoor/Autumn_Dawn-8B-LINEAR", - "name": "Autumn_Dawn-8B-LINEAR", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7293, - "hfopenllm_v2/BBH": 0.5459, - "hfopenllm_v2/MATH Level 5": 0.1858, - "hfopenllm_v2/GPQA": 0.2936, - "hfopenllm_v2/MUSR": 0.4186, - "hfopenllm_v2/MMLU-PRO": 0.3968 - } - }, - { - "id": "DreadPoor/BaeZel-8B-LINEAR", - "name": "BaeZel-8B-LINEAR", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7378, - "hfopenllm_v2/BBH": 0.5464, - "hfopenllm_v2/MATH Level 5": 0.1813, - "hfopenllm_v2/GPQA": 0.3213, - "hfopenllm_v2/MUSR": 0.4227, - "hfopenllm_v2/MMLU-PRO": 0.3861 - } - }, - { - "id": "DreadPoor/BaeZel-8B-Model_Stock", - "name": "BaeZel-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7713, - "hfopenllm_v2/BBH": 0.5408, - "hfopenllm_v2/MATH Level 5": 0.1639, - "hfopenllm_v2/GPQA": 0.3138, - "hfopenllm_v2/MUSR": 0.4199, - "hfopenllm_v2/MMLU-PRO": 0.388 - } - }, - { - "id": "DreadPoor/BaeZel_V2-8B-Model_Stock", - "name": "BaeZel_V2-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7677, - "hfopenllm_v2/BBH": 0.5374, - "hfopenllm_v2/MATH Level 5": 0.1798, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.4186, - "hfopenllm_v2/MMLU-PRO": 0.3947 - } - }, - { - "id": "DreadPoor/BaeZel_V2_ALT-8B-Model_Stock", - "name": "BaeZel_V2_ALT-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7677, - "hfopenllm_v2/BBH": 0.5374, - "hfopenllm_v2/MATH Level 5": 0.1798, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.4186, - "hfopenllm_v2/MMLU-PRO": 0.3947 - } - }, - { - "id": "DreadPoor/BaeZel_V3-8B-Model_Stock", - "name": "BaeZel_V3-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7832, - "hfopenllm_v2/BBH": 0.5392, - "hfopenllm_v2/MATH Level 5": 0.1896, - "hfopenllm_v2/GPQA": 0.3205, - "hfopenllm_v2/MUSR": 0.4174, - "hfopenllm_v2/MMLU-PRO": 0.3888 - } - }, - { - "id": "DreadPoor/Blunt_Edge-8B-SLERP", - "name": "Blunt_Edge-8B-SLERP", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7497, - "hfopenllm_v2/BBH": 0.5389, - "hfopenllm_v2/MATH Level 5": 0.1858, - "hfopenllm_v2/GPQA": 0.3112, - "hfopenllm_v2/MUSR": 0.4174, - "hfopenllm_v2/MMLU-PRO": 0.3767 - } - }, - { - "id": "DreadPoor/BulkUp", - "name": "BulkUp", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1778, - "hfopenllm_v2/BBH": 0.287, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2475, - "hfopenllm_v2/MUSR": 0.3447, - "hfopenllm_v2/MMLU-PRO": 0.111 - } - }, - { - "id": "DreadPoor/Cadence-8B-LINEAR", - "name": "Cadence-8B-LINEAR", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7682, - "hfopenllm_v2/BBH": 0.5433, - "hfopenllm_v2/MATH Level 5": 0.1677, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.4173, - "hfopenllm_v2/MMLU-PRO": 0.3803 - } - }, - { - "id": "DreadPoor/Caelid-8B-Model_Stock", - "name": "Caelid-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7247, - "hfopenllm_v2/BBH": 0.546, - "hfopenllm_v2/MATH Level 5": 0.1511, - "hfopenllm_v2/GPQA": 0.3104, - "hfopenllm_v2/MUSR": 0.4001, - "hfopenllm_v2/MMLU-PRO": 0.3816 - } - }, - { - "id": "DreadPoor/Casuar-9B-Model_Stock", - "name": "Casuar-9B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7765, - "hfopenllm_v2/BBH": 0.6107, - "hfopenllm_v2/MATH Level 5": 0.213, - "hfopenllm_v2/GPQA": 0.3448, - "hfopenllm_v2/MUSR": 0.4165, - "hfopenllm_v2/MMLU-PRO": 0.4156 - } - }, - { - "id": "DreadPoor/Condensed_Milk-8B-Model_Stock", - "name": "Condensed_Milk-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7536, - "hfopenllm_v2/BBH": 0.5435, - "hfopenllm_v2/MATH Level 5": 0.1745, - "hfopenllm_v2/GPQA": 0.3213, - "hfopenllm_v2/MUSR": 0.416, - "hfopenllm_v2/MMLU-PRO": 0.3876 - } - }, - { - "id": "DreadPoor/CoolerCoder-8B-LINEAR", - "name": "CoolerCoder-8B-LINEAR", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4519, - "hfopenllm_v2/BBH": 0.4762, - "hfopenllm_v2/MATH Level 5": 0.0793, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.3964, - "hfopenllm_v2/MMLU-PRO": 0.3159 - } - }, - { - "id": "DreadPoor/Damasteel-8B-LINEAR", - "name": "Damasteel-8B-LINEAR", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7384, - "hfopenllm_v2/BBH": 0.5388, - "hfopenllm_v2/MATH Level 5": 0.1669, - "hfopenllm_v2/GPQA": 0.2987, - "hfopenllm_v2/MUSR": 0.4212, - "hfopenllm_v2/MMLU-PRO": 0.3779 - } - }, - { - "id": "DreadPoor/Dearly_Beloved-8B-TIES", - "name": "Dearly_Beloved-8B-TIES", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8267, - "hfopenllm_v2/BBH": 0.405, - "hfopenllm_v2/MATH Level 5": 0.2115, - "hfopenllm_v2/GPQA": 0.2987, - "hfopenllm_v2/MUSR": 0.4175, - "hfopenllm_v2/MMLU-PRO": 0.2827 - } - }, - { - "id": "DreadPoor/Decayed-8B-LINEAR", - "name": "Decayed-8B-LINEAR", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7676, - "hfopenllm_v2/BBH": 0.5417, - "hfopenllm_v2/MATH Level 5": 0.1715, - "hfopenllm_v2/GPQA": 0.3096, - "hfopenllm_v2/MUSR": 0.4186, - "hfopenllm_v2/MMLU-PRO": 0.3763 - } - }, - { - "id": "DreadPoor/Derivative-8B-Model_Stock", - "name": "Derivative-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7667, - "hfopenllm_v2/BBH": 0.5395, - "hfopenllm_v2/MATH Level 5": 0.179, - "hfopenllm_v2/GPQA": 0.3171, - "hfopenllm_v2/MUSR": 0.42, - "hfopenllm_v2/MMLU-PRO": 0.3811 - } - }, - { - "id": "DreadPoor/Derivative_V2-8B-Model_Stock", - "name": "Derivative_V2-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7537, - "hfopenllm_v2/BBH": 0.5393, - "hfopenllm_v2/MATH Level 5": 0.1798, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.4123, - "hfopenllm_v2/MMLU-PRO": 0.3856 - } - }, - { - "id": "DreadPoor/Derivative_V2_ALT-8B-Model_Stock", - "name": "Derivative_V2_ALT-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.772, - "hfopenllm_v2/BBH": 0.5365, - "hfopenllm_v2/MATH Level 5": 0.1881, - "hfopenllm_v2/GPQA": 0.3112, - "hfopenllm_v2/MUSR": 0.4135, - "hfopenllm_v2/MMLU-PRO": 0.3882 - } - }, - { - "id": "DreadPoor/Derivative_V3-8B-Model_Stock", - "name": "Derivative_V3-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6964, - "hfopenllm_v2/BBH": 0.5243, - "hfopenllm_v2/MATH Level 5": 0.1465, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.415, - "hfopenllm_v2/MMLU-PRO": 0.3502 - } - }, - { - "id": "DreadPoor/Elusive_Dragon_Heart-8B-LINEAR", - "name": "Elusive_Dragon_Heart-8B-LINEAR", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7131, - "hfopenllm_v2/BBH": 0.5456, - "hfopenllm_v2/MATH Level 5": 0.148, - "hfopenllm_v2/GPQA": 0.3062, - "hfopenllm_v2/MUSR": 0.4146, - "hfopenllm_v2/MMLU-PRO": 0.3814 - } - }, - { - "id": "DreadPoor/Emu_Eggs-9B-Model_Stock", - "name": "Emu_Eggs-9B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7607, - "hfopenllm_v2/BBH": 0.6052, - "hfopenllm_v2/MATH Level 5": 0.21, - "hfopenllm_v2/GPQA": 0.3331, - "hfopenllm_v2/MUSR": 0.4071, - "hfopenllm_v2/MMLU-PRO": 0.4227 - } - }, - { - "id": "DreadPoor/Eunoia_Vespera-8B-LINEAR", - "name": "Eunoia_Vespera-8B-LINEAR", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7235, - "hfopenllm_v2/BBH": 0.5399, - "hfopenllm_v2/MATH Level 5": 0.1541, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.4185, - "hfopenllm_v2/MMLU-PRO": 0.3839 - } - }, - { - "id": "DreadPoor/felix_dies-mistral-7B-model_stock", - "name": "felix_dies-mistral-7B-model_stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3008, - "hfopenllm_v2/BBH": 0.4901, - "hfopenllm_v2/MATH Level 5": 0.0536, - "hfopenllm_v2/GPQA": 0.2919, - "hfopenllm_v2/MUSR": 0.4518, - "hfopenllm_v2/MMLU-PRO": 0.3109 - } - }, - { - "id": "DreadPoor/Fu_sion_HA-8B-SLERP", - "name": "Fu_sion_HA-8B-SLERP", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7609, - "hfopenllm_v2/BBH": 0.5373, - "hfopenllm_v2/MATH Level 5": 0.1752, - "hfopenllm_v2/GPQA": 0.323, - "hfopenllm_v2/MUSR": 0.416, - "hfopenllm_v2/MMLU-PRO": 0.3825 - } - }, - { - "id": "DreadPoor/H_the_eighth-8B-LINEAR", - "name": "H_the_eighth-8B-LINEAR", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7469, - "hfopenllm_v2/BBH": 0.5384, - "hfopenllm_v2/MATH Level 5": 0.1775, - "hfopenllm_v2/GPQA": 0.328, - "hfopenllm_v2/MUSR": 0.4173, - "hfopenllm_v2/MMLU-PRO": 0.3824 - } - }, - { - "id": "DreadPoor/hakuchido-8B-MODEL_STOCK", - "name": "hakuchido-8B-MODEL_STOCK", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7375, - "hfopenllm_v2/BBH": 0.5398, - "hfopenllm_v2/MATH Level 5": 0.1949, - "hfopenllm_v2/GPQA": 0.2953, - "hfopenllm_v2/MUSR": 0.4175, - "hfopenllm_v2/MMLU-PRO": 0.3782 - } - }, - { - "id": "DreadPoor/Happy_New_Year-8B-Model_Stock", - "name": "Happy_New_Year-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7616, - "hfopenllm_v2/BBH": 0.5368, - "hfopenllm_v2/MATH Level 5": 0.1594, - "hfopenllm_v2/GPQA": 0.3138, - "hfopenllm_v2/MUSR": 0.4186, - "hfopenllm_v2/MMLU-PRO": 0.3879 - } - }, - { - "id": "DreadPoor/Heart_Stolen-8B-Model_Stock", - "name": "Heart_Stolen-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7245, - "hfopenllm_v2/BBH": 0.5395, - "hfopenllm_v2/MATH Level 5": 0.1722, - "hfopenllm_v2/GPQA": 0.3171, - "hfopenllm_v2/MUSR": 0.4162, - "hfopenllm_v2/MMLU-PRO": 0.3794 - } - }, - { - "id": "DreadPoor/Heart_Stolen-ALT-8B-Model_Stock", - "name": "Heart_Stolen-ALT-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7184, - "hfopenllm_v2/BBH": 0.5263, - "hfopenllm_v2/MATH Level 5": 0.1563, - "hfopenllm_v2/GPQA": 0.3012, - "hfopenllm_v2/MUSR": 0.4055, - "hfopenllm_v2/MMLU-PRO": 0.3772 - } - }, - { - "id": "DreadPoor/Here_We_Go_Again-8B-SLERP", - "name": "Here_We_Go_Again-8B-SLERP", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7442, - "hfopenllm_v2/BBH": 0.546, - "hfopenllm_v2/MATH Level 5": 0.173, - "hfopenllm_v2/GPQA": 0.3188, - "hfopenllm_v2/MUSR": 0.4187, - "hfopenllm_v2/MMLU-PRO": 0.3873 - } - }, - { - "id": "DreadPoor/HOT_STINKING_GARBAGE", - "name": "HOT_STINKING_GARBAGE", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5754, - "hfopenllm_v2/BBH": 0.4884, - "hfopenllm_v2/MATH Level 5": 0.0672, - "hfopenllm_v2/GPQA": 0.2752, - "hfopenllm_v2/MUSR": 0.425, - "hfopenllm_v2/MMLU-PRO": 0.3017 - } - }, - { - "id": "DreadPoor/Howdy-8B-LINEAR", - "name": "Howdy-8B-LINEAR", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7378, - "hfopenllm_v2/BBH": 0.5384, - "hfopenllm_v2/MATH Level 5": 0.1775, - "hfopenllm_v2/GPQA": 0.3146, - "hfopenllm_v2/MUSR": 0.4121, - "hfopenllm_v2/MMLU-PRO": 0.3807 - } - }, - { - "id": "DreadPoor/ichor-8B-Model_Stock", - "name": "ichor-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5386, - "hfopenllm_v2/BBH": 0.5084, - "hfopenllm_v2/MATH Level 5": 0.1088, - "hfopenllm_v2/GPQA": 0.3238, - "hfopenllm_v2/MUSR": 0.4212, - "hfopenllm_v2/MMLU-PRO": 0.3151 - } - }, - { - "id": "DreadPoor/ichor_1.1-8B-Model_Stock", - "name": "ichor_1.1-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8096, - "hfopenllm_v2/BBH": 0.5281, - "hfopenllm_v2/MATH Level 5": 0.1775, - "hfopenllm_v2/GPQA": 0.3062, - "hfopenllm_v2/MUSR": 0.4068, - "hfopenllm_v2/MMLU-PRO": 0.3856 - } - }, - { - "id": "DreadPoor/Incidental-8B-Model_Stock", - "name": "Incidental-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7482, - "hfopenllm_v2/BBH": 0.5452, - "hfopenllm_v2/MATH Level 5": 0.1616, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.424, - "hfopenllm_v2/MMLU-PRO": 0.3873 - } - }, - { - "id": "DreadPoor/inexpertus-8B-Model_Stock", - "name": "inexpertus-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7795, - "hfopenllm_v2/BBH": 0.528, - "hfopenllm_v2/MATH Level 5": 0.1707, - "hfopenllm_v2/GPQA": 0.3096, - "hfopenllm_v2/MUSR": 0.4118, - "hfopenllm_v2/MMLU-PRO": 0.3791 - } - }, - { - "id": "DreadPoor/inexpertus_1.1-8B-LINEAR", - "name": "inexpertus_1.1-8B-LINEAR", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7527, - "hfopenllm_v2/BBH": 0.5525, - "hfopenllm_v2/MATH Level 5": 0.173, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.4173, - "hfopenllm_v2/MMLU-PRO": 0.3827 - } - }, - { - "id": "DreadPoor/inexpertus_1.2-8B-LINEAR", - "name": "inexpertus_1.2-8B-LINEAR", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7348, - "hfopenllm_v2/BBH": 0.5523, - "hfopenllm_v2/MATH Level 5": 0.1586, - "hfopenllm_v2/GPQA": 0.2953, - "hfopenllm_v2/MUSR": 0.4133, - "hfopenllm_v2/MMLU-PRO": 0.3788 - } - }, - { - "id": "DreadPoor/Irina-8B-model_stock", - "name": "Irina-8B-model_stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6799, - "hfopenllm_v2/BBH": 0.5237, - "hfopenllm_v2/MATH Level 5": 0.102, - "hfopenllm_v2/GPQA": 0.2844, - "hfopenllm_v2/MUSR": 0.4003, - "hfopenllm_v2/MMLU-PRO": 0.3574 - } - }, - { - "id": "DreadPoor/Kindling-8B-Model_Stock", - "name": "Kindling-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7308, - "hfopenllm_v2/BBH": 0.5492, - "hfopenllm_v2/MATH Level 5": 0.1752, - "hfopenllm_v2/GPQA": 0.318, - "hfopenllm_v2/MUSR": 0.4068, - "hfopenllm_v2/MMLU-PRO": 0.383 - } - }, - { - "id": "DreadPoor/L3.1-BaeZel-8B-Della", - "name": "L3.1-BaeZel-8B-Della", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.518, - "hfopenllm_v2/BBH": 0.5448, - "hfopenllm_v2/MATH Level 5": 0.1745, - "hfopenllm_v2/GPQA": 0.3196, - "hfopenllm_v2/MUSR": 0.42, - "hfopenllm_v2/MMLU-PRO": 0.3902 - } - }, - { - "id": "DreadPoor/Laughing_Stock-8B-Model_Stock", - "name": "Laughing_Stock-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.719, - "hfopenllm_v2/BBH": 0.5449, - "hfopenllm_v2/MATH Level 5": 0.1579, - "hfopenllm_v2/GPQA": 0.2894, - "hfopenllm_v2/MUSR": 0.4146, - "hfopenllm_v2/MMLU-PRO": 0.3764 - } - }, - { - "id": "DreadPoor/Lava_Lamp-8B-SLERP", - "name": "Lava_Lamp-8B-SLERP", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7381, - "hfopenllm_v2/BBH": 0.5368, - "hfopenllm_v2/MATH Level 5": 0.1737, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.4187, - "hfopenllm_v2/MMLU-PRO": 0.375 - } - }, - { - "id": "DreadPoor/LemonP-8B-Model_Stock", - "name": "LemonP-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7676, - "hfopenllm_v2/BBH": 0.5439, - "hfopenllm_v2/MATH Level 5": 0.1767, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.4081, - "hfopenllm_v2/MMLU-PRO": 0.4004 - } - }, - { - "id": "DreadPoor/Lydia_of_Whiterun-8B-LINEAR", - "name": "Lydia_of_Whiterun-8B-LINEAR", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7603, - "hfopenllm_v2/BBH": 0.538, - "hfopenllm_v2/MATH Level 5": 0.1767, - "hfopenllm_v2/GPQA": 0.3163, - "hfopenllm_v2/MUSR": 0.4251, - "hfopenllm_v2/MMLU-PRO": 0.3801 - } - }, - { - "id": "DreadPoor/Matryoshka-8B-LINEAR", - "name": "Matryoshka-8B-LINEAR", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7263, - "hfopenllm_v2/BBH": 0.5444, - "hfopenllm_v2/MATH Level 5": 0.1752, - "hfopenllm_v2/GPQA": 0.3205, - "hfopenllm_v2/MUSR": 0.4252, - "hfopenllm_v2/MMLU-PRO": 0.3866 - } - }, - { - "id": "DreadPoor/Mercury_In_Retrograde-8b-Model-Stock", - "name": "Mercury_In_Retrograde-8b-Model-Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7296, - "hfopenllm_v2/BBH": 0.5391, - "hfopenllm_v2/MATH Level 5": 0.1647, - "hfopenllm_v2/GPQA": 0.3163, - "hfopenllm_v2/MUSR": 0.4199, - "hfopenllm_v2/MMLU-PRO": 0.3829 - } - }, - { - "id": "DreadPoor/mergekit-nuslerp-nqzkedi", - "name": "mergekit-nuslerp-nqzkedi", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7765, - "hfopenllm_v2/BBH": 0.5362, - "hfopenllm_v2/MATH Level 5": 0.1881, - "hfopenllm_v2/GPQA": 0.3012, - "hfopenllm_v2/MUSR": 0.4225, - "hfopenllm_v2/MMLU-PRO": 0.3919 - } - }, - { - "id": "DreadPoor/Minthy-8B-Model_Stock", - "name": "Minthy-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7658, - "hfopenllm_v2/BBH": 0.5353, - "hfopenllm_v2/MATH Level 5": 0.1918, - "hfopenllm_v2/GPQA": 0.3037, - "hfopenllm_v2/MUSR": 0.4094, - "hfopenllm_v2/MMLU-PRO": 0.3993 - } - }, - { - "id": "DreadPoor/Minthy_ALT-8B-Model_Stock", - "name": "Minthy_ALT-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6992, - "hfopenllm_v2/BBH": 0.5375, - "hfopenllm_v2/MATH Level 5": 0.176, - "hfopenllm_v2/GPQA": 0.3062, - "hfopenllm_v2/MUSR": 0.4225, - "hfopenllm_v2/MMLU-PRO": 0.3674 - } - }, - { - "id": "DreadPoor/Minthy_V2-8B-Model_Stock", - "name": "Minthy_V2-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7126, - "hfopenllm_v2/BBH": 0.5491, - "hfopenllm_v2/MATH Level 5": 0.1594, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.4199, - "hfopenllm_v2/MMLU-PRO": 0.3737 - } - }, - { - "id": "DreadPoor/Minus_Penus-8B-Model_Stock", - "name": "Minus_Penus-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7311, - "hfopenllm_v2/BBH": 0.5344, - "hfopenllm_v2/MATH Level 5": 0.2002, - "hfopenllm_v2/GPQA": 0.3096, - "hfopenllm_v2/MUSR": 0.4019, - "hfopenllm_v2/MMLU-PRO": 0.3752 - } - }, - { - "id": "DreadPoor/Morphing-8B-Model_Stock", - "name": "Morphing-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7445, - "hfopenllm_v2/BBH": 0.5397, - "hfopenllm_v2/MATH Level 5": 0.1888, - "hfopenllm_v2/GPQA": 0.2936, - "hfopenllm_v2/MUSR": 0.4069, - "hfopenllm_v2/MMLU-PRO": 0.3852 - } - }, - { - "id": "DreadPoor/Not_Even_My_Final_Form-8B-Model_Stock", - "name": "Not_Even_My_Final_Form-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7722, - "hfopenllm_v2/BBH": 0.5351, - "hfopenllm_v2/MATH Level 5": 0.176, - "hfopenllm_v2/GPQA": 0.2953, - "hfopenllm_v2/MUSR": 0.4147, - "hfopenllm_v2/MMLU-PRO": 0.384 - } - }, - { - "id": "DreadPoor/Nother_One-8B-Model_Stock", - "name": "Nother_One-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6863, - "hfopenllm_v2/BBH": 0.5205, - "hfopenllm_v2/MATH Level 5": 0.1518, - "hfopenllm_v2/GPQA": 0.2894, - "hfopenllm_v2/MUSR": 0.387, - "hfopenllm_v2/MMLU-PRO": 0.3595 - } - }, - { - "id": "DreadPoor/Noxis-8B-LINEAR", - "name": "Noxis-8B-LINEAR", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6913, - "hfopenllm_v2/BBH": 0.5421, - "hfopenllm_v2/MATH Level 5": 0.1979, - "hfopenllm_v2/GPQA": 0.3188, - "hfopenllm_v2/MUSR": 0.4231, - "hfopenllm_v2/MMLU-PRO": 0.366 - } - }, - { - "id": "DreadPoor/Nullsworn-12B-LINEAR", - "name": "Nullsworn-12B-LINEAR", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4436, - "hfopenllm_v2/BBH": 0.5483, - "hfopenllm_v2/MATH Level 5": 0.1125, - "hfopenllm_v2/GPQA": 0.3079, - "hfopenllm_v2/MUSR": 0.435, - "hfopenllm_v2/MMLU-PRO": 0.3645 - } - }, - { - "id": "DreadPoor/Nwah-8B-Model_Stock", - "name": "Nwah-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7716, - "hfopenllm_v2/BBH": 0.5384, - "hfopenllm_v2/MATH Level 5": 0.1798, - "hfopenllm_v2/GPQA": 0.3104, - "hfopenllm_v2/MUSR": 0.4039, - "hfopenllm_v2/MMLU-PRO": 0.3807 - } - }, - { - "id": "DreadPoor/Oh_Boy-8B-LINEAR", - "name": "Oh_Boy-8B-LINEAR", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7503, - "hfopenllm_v2/BBH": 0.5375, - "hfopenllm_v2/MATH Level 5": 0.1782, - "hfopenllm_v2/GPQA": 0.3079, - "hfopenllm_v2/MUSR": 0.4108, - "hfopenllm_v2/MMLU-PRO": 0.3849 - } - }, - { - "id": "DreadPoor/ONeil-model_stock-8B", - "name": "ONeil-model_stock-8B", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6786, - "hfopenllm_v2/BBH": 0.5548, - "hfopenllm_v2/MATH Level 5": 0.1012, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.4173, - "hfopenllm_v2/MMLU-PRO": 0.3599 - } - }, - { - "id": "DreadPoor/OrangeJ-8B-Model_Stock", - "name": "OrangeJ-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7841, - "hfopenllm_v2/BBH": 0.5413, - "hfopenllm_v2/MATH Level 5": 0.176, - "hfopenllm_v2/GPQA": 0.3012, - "hfopenllm_v2/MUSR": 0.4028, - "hfopenllm_v2/MMLU-PRO": 0.3969 - } - }, - { - "id": "DreadPoor/Promissum_Mane-8B-LINEAR", - "name": "Promissum_Mane-8B-LINEAR", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.715, - "hfopenllm_v2/BBH": 0.5458, - "hfopenllm_v2/MATH Level 5": 0.1556, - "hfopenllm_v2/GPQA": 0.3045, - "hfopenllm_v2/MUSR": 0.42, - "hfopenllm_v2/MMLU-PRO": 0.3851 - } - }, - { - "id": "DreadPoor/Promissum_Mane-8B-LINEAR-lorablated", - "name": "Promissum_Mane-8B-LINEAR-lorablated", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7156, - "hfopenllm_v2/BBH": 0.5435, - "hfopenllm_v2/MATH Level 5": 0.1533, - "hfopenllm_v2/GPQA": 0.3037, - "hfopenllm_v2/MUSR": 0.4198, - "hfopenllm_v2/MMLU-PRO": 0.3739 - } - }, - { - "id": "DreadPoor/remember_to_breathe-8b-Model-Stock", - "name": "remember_to_breathe-8b-Model-Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7104, - "hfopenllm_v2/BBH": 0.5412, - "hfopenllm_v2/MATH Level 5": 0.1488, - "hfopenllm_v2/GPQA": 0.3012, - "hfopenllm_v2/MUSR": 0.4145, - "hfopenllm_v2/MMLU-PRO": 0.3761 - } - }, - { - "id": "DreadPoor/RPMash-8B-Model_Stock", - "name": "RPMash-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4564, - "hfopenllm_v2/BBH": 0.5169, - "hfopenllm_v2/MATH Level 5": 0.108, - "hfopenllm_v2/GPQA": 0.2869, - "hfopenllm_v2/MUSR": 0.4054, - "hfopenllm_v2/MMLU-PRO": 0.3604 - } - }, - { - "id": "DreadPoor/RPMash_V3-8B-Model_Stock", - "name": "RPMash_V3-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7049, - "hfopenllm_v2/BBH": 0.5217, - "hfopenllm_v2/MATH Level 5": 0.1042, - "hfopenllm_v2/GPQA": 0.3003, - "hfopenllm_v2/MUSR": 0.3778, - "hfopenllm_v2/MMLU-PRO": 0.3614 - } - }, - { - "id": "DreadPoor/Rusted_Gold-8B-LINEAR", - "name": "Rusted_Gold-8B-LINEAR", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7296, - "hfopenllm_v2/BBH": 0.5387, - "hfopenllm_v2/MATH Level 5": 0.1934, - "hfopenllm_v2/GPQA": 0.2987, - "hfopenllm_v2/MUSR": 0.4178, - "hfopenllm_v2/MMLU-PRO": 0.378 - } - }, - { - "id": "DreadPoor/Rusted_Platinum-8B-LINEAR", - "name": "Rusted_Platinum-8B-LINEAR", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.718, - "hfopenllm_v2/BBH": 0.5428, - "hfopenllm_v2/MATH Level 5": 0.1722, - "hfopenllm_v2/GPQA": 0.2802, - "hfopenllm_v2/MUSR": 0.3967, - "hfopenllm_v2/MMLU-PRO": 0.373 - } - }, - { - "id": "DreadPoor/Rusted_Platinum-8B-Model_Stock", - "name": "Rusted_Platinum-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4408, - "hfopenllm_v2/BBH": 0.5243, - "hfopenllm_v2/MATH Level 5": 0.102, - "hfopenllm_v2/GPQA": 0.2685, - "hfopenllm_v2/MUSR": 0.3741, - "hfopenllm_v2/MMLU-PRO": 0.3546 - } - }, - { - "id": "DreadPoor/Sellen-8B-model_stock", - "name": "Sellen-8B-model_stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7113, - "hfopenllm_v2/BBH": 0.5232, - "hfopenllm_v2/MATH Level 5": 0.1337, - "hfopenllm_v2/GPQA": 0.2743, - "hfopenllm_v2/MUSR": 0.396, - "hfopenllm_v2/MMLU-PRO": 0.357 - } - }, - { - "id": "DreadPoor/Something-8B-Model_Stock", - "name": "Something-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5043, - "hfopenllm_v2/BBH": 0.5395, - "hfopenllm_v2/MATH Level 5": 0.1798, - "hfopenllm_v2/GPQA": 0.3171, - "hfopenllm_v2/MUSR": 0.4187, - "hfopenllm_v2/MMLU-PRO": 0.3885 - } - }, - { - "id": "DreadPoor/Spring_Dusk-8B-SCE", - "name": "Spring_Dusk-8B-SCE", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6515, - "hfopenllm_v2/BBH": 0.5635, - "hfopenllm_v2/MATH Level 5": 0.0763, - "hfopenllm_v2/GPQA": 0.2878, - "hfopenllm_v2/MUSR": 0.46, - "hfopenllm_v2/MMLU-PRO": 0.3436 - } - }, - { - "id": "DreadPoor/Summer_Dawn-8B-SCE", - "name": "Summer_Dawn-8B-SCE", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6642, - "hfopenllm_v2/BBH": 0.5391, - "hfopenllm_v2/MATH Level 5": 0.1722, - "hfopenllm_v2/GPQA": 0.3003, - "hfopenllm_v2/MUSR": 0.412, - "hfopenllm_v2/MMLU-PRO": 0.3753 - } - }, - { - "id": "DreadPoor/Summer_Dusk-8B-TIES", - "name": "Summer_Dusk-8B-TIES", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4922, - "hfopenllm_v2/BBH": 0.536, - "hfopenllm_v2/MATH Level 5": 0.1805, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.4267, - "hfopenllm_v2/MMLU-PRO": 0.3856 - } - }, - { - "id": "DreadPoor/Summer_Rain-8B-SCE", - "name": "Summer_Rain-8B-SCE", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5459, - "hfopenllm_v2/BBH": 0.5846, - "hfopenllm_v2/MATH Level 5": 0.0702, - "hfopenllm_v2/GPQA": 0.3037, - "hfopenllm_v2/MUSR": 0.4477, - "hfopenllm_v2/MMLU-PRO": 0.3551 - } - }, - { - "id": "DreadPoor/Summer_Rain-8B-TIES", - "name": "Summer_Rain-8B-TIES", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5444, - "hfopenllm_v2/BBH": 0.5846, - "hfopenllm_v2/MATH Level 5": 0.0702, - "hfopenllm_v2/GPQA": 0.3037, - "hfopenllm_v2/MUSR": 0.4477, - "hfopenllm_v2/MMLU-PRO": 0.3551 - } - }, - { - "id": "DreadPoor/Sun-8B-Model_Stock", - "name": "Sun-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7758, - "hfopenllm_v2/BBH": 0.5264, - "hfopenllm_v2/MATH Level 5": 0.21, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.4098, - "hfopenllm_v2/MMLU-PRO": 0.3835 - } - }, - { - "id": "DreadPoor/Sweetened_Condensed_Milk-8B-Model_Stock", - "name": "Sweetened_Condensed_Milk-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7417, - "hfopenllm_v2/BBH": 0.5406, - "hfopenllm_v2/MATH Level 5": 0.1873, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.4107, - "hfopenllm_v2/MMLU-PRO": 0.3848 - } - }, - { - "id": "DreadPoor/test", - "name": "test", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4937, - "hfopenllm_v2/BBH": 0.5372, - "hfopenllm_v2/MATH Level 5": 0.1934, - "hfopenllm_v2/GPQA": 0.271, - "hfopenllm_v2/MUSR": 0.4351, - "hfopenllm_v2/MMLU-PRO": 0.3647 - } - }, - { - "id": "DreadPoor/TEST02-Ignore", - "name": "TEST02-Ignore", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6119, - "hfopenllm_v2/BBH": 0.5602, - "hfopenllm_v2/MATH Level 5": 0.0869, - "hfopenllm_v2/GPQA": 0.2844, - "hfopenllm_v2/MUSR": 0.4199, - "hfopenllm_v2/MMLU-PRO": 0.3468 - } - }, - { - "id": "DreadPoor/TEST03-ignore", - "name": "TEST03-ignore", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6967, - "hfopenllm_v2/BBH": 0.5383, - "hfopenllm_v2/MATH Level 5": 0.1654, - "hfopenllm_v2/GPQA": 0.3087, - "hfopenllm_v2/MUSR": 0.4186, - "hfopenllm_v2/MMLU-PRO": 0.3789 - } - }, - { - "id": "DreadPoor/TEST06-ignore", - "name": "TEST06-ignore", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7323, - "hfopenllm_v2/BBH": 0.5509, - "hfopenllm_v2/MATH Level 5": 0.1178, - "hfopenllm_v2/GPQA": 0.2869, - "hfopenllm_v2/MUSR": 0.4225, - "hfopenllm_v2/MMLU-PRO": 0.3615 - } - }, - { - "id": "DreadPoor/TEST07-ignore", - "name": "TEST07-ignore", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.74, - "hfopenllm_v2/BBH": 0.5561, - "hfopenllm_v2/MATH Level 5": 0.1662, - "hfopenllm_v2/GPQA": 0.3087, - "hfopenllm_v2/MUSR": 0.4094, - "hfopenllm_v2/MMLU-PRO": 0.388 - } - }, - { - "id": "DreadPoor/TEST08-ignore", - "name": "TEST08-ignore", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7467, - "hfopenllm_v2/BBH": 0.5454, - "hfopenllm_v2/MATH Level 5": 0.182, - "hfopenllm_v2/GPQA": 0.3129, - "hfopenllm_v2/MUSR": 0.4081, - "hfopenllm_v2/MMLU-PRO": 0.3853 - } - }, - { - "id": "DreadPoor/test_ALT", - "name": "test_ALT", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4997, - "hfopenllm_v2/BBH": 0.537, - "hfopenllm_v2/MATH Level 5": 0.1707, - "hfopenllm_v2/GPQA": 0.2693, - "hfopenllm_v2/MUSR": 0.4363, - "hfopenllm_v2/MMLU-PRO": 0.3492 - } - }, - { - "id": "DreadPoor/tests_pending-do_not_use_yet", - "name": "tests_pending-do_not_use_yet", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7691, - "hfopenllm_v2/BBH": 0.5408, - "hfopenllm_v2/MATH Level 5": 0.1979, - "hfopenllm_v2/GPQA": 0.297, - "hfopenllm_v2/MUSR": 0.4005, - "hfopenllm_v2/MMLU-PRO": 0.3827 - } - }, - { - "id": "DreadPoor/Trinas_Nectar-8B-model_stock", - "name": "Trinas_Nectar-8B-model_stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7259, - "hfopenllm_v2/BBH": 0.5256, - "hfopenllm_v2/MATH Level 5": 0.1526, - "hfopenllm_v2/GPQA": 0.2861, - "hfopenllm_v2/MUSR": 0.4068, - "hfopenllm_v2/MMLU-PRO": 0.3618 - } - }, - { - "id": "DreadPoor/UNTESTED-VENN_1.2-8B-Model_Stock", - "name": "UNTESTED-VENN_1.2-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4718, - "hfopenllm_v2/BBH": 0.5475, - "hfopenllm_v2/MATH Level 5": 0.1541, - "hfopenllm_v2/GPQA": 0.3154, - "hfopenllm_v2/MUSR": 0.4449, - "hfopenllm_v2/MMLU-PRO": 0.3787 - } - }, - { - "id": "DreadPoor/VENN_1.2-8B-Model_Stock", - "name": "VENN_1.2-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7226, - "hfopenllm_v2/BBH": 0.5459, - "hfopenllm_v2/MATH Level 5": 0.1707, - "hfopenllm_v2/GPQA": 0.297, - "hfopenllm_v2/MUSR": 0.42, - "hfopenllm_v2/MMLU-PRO": 0.3721 - } - }, - { - "id": "DreadPoor/Wannabe-8B-Model_Stock", - "name": "Wannabe-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7205, - "hfopenllm_v2/BBH": 0.539, - "hfopenllm_v2/MATH Level 5": 0.1775, - "hfopenllm_v2/GPQA": 0.3012, - "hfopenllm_v2/MUSR": 0.4135, - "hfopenllm_v2/MMLU-PRO": 0.3831 - } - }, - { - "id": "DreadPoor/What_A_Thrill-8B-Model_Stock", - "name": "What_A_Thrill-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7064, - "hfopenllm_v2/BBH": 0.5311, - "hfopenllm_v2/MATH Level 5": 0.182, - "hfopenllm_v2/GPQA": 0.297, - "hfopenllm_v2/MUSR": 0.408, - "hfopenllm_v2/MMLU-PRO": 0.3615 - } - }, - { - "id": "DreadPoor/Winter-8B-SCE", - "name": "Winter-8B-SCE", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7536, - "hfopenllm_v2/BBH": 0.5262, - "hfopenllm_v2/MATH Level 5": 0.1918, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.4071, - "hfopenllm_v2/MMLU-PRO": 0.3839 - } - }, - { - "id": "DreadPoor/Winter_Dawn-8B-TIES", - "name": "Winter_Dawn-8B-TIES", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5496, - "hfopenllm_v2/BBH": 0.5309, - "hfopenllm_v2/MATH Level 5": 0.1858, - "hfopenllm_v2/GPQA": 0.3096, - "hfopenllm_v2/MUSR": 0.4279, - "hfopenllm_v2/MMLU-PRO": 0.391 - } - }, - { - "id": "DreadPoor/Winter_Dusk-8B-TIES", - "name": "Winter_Dusk-8B-TIES", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7153, - "hfopenllm_v2/BBH": 0.4952, - "hfopenllm_v2/MATH Level 5": 0.0718, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.3688, - "hfopenllm_v2/MMLU-PRO": 0.3478 - } - }, - { - "id": "DreadPoor/Winter_Night-8B-Model_Stock", - "name": "Winter_Night-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.704, - "hfopenllm_v2/BBH": 0.5185, - "hfopenllm_v2/MATH Level 5": 0.1458, - "hfopenllm_v2/GPQA": 0.3062, - "hfopenllm_v2/MUSR": 0.3914, - "hfopenllm_v2/MMLU-PRO": 0.3666 - } - }, - { - "id": "DreadPoor/WIP-Acacia-8B-Model_Stock", - "name": "WIP-Acacia-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6246, - "hfopenllm_v2/BBH": 0.5195, - "hfopenllm_v2/MATH Level 5": 0.1669, - "hfopenllm_v2/GPQA": 0.3062, - "hfopenllm_v2/MUSR": 0.4226, - "hfopenllm_v2/MMLU-PRO": 0.3737 - } - }, - { - "id": "DreadPoor/WIP_Damascus-8B-TIES", - "name": "WIP_Damascus-8B-TIES", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4776, - "hfopenllm_v2/BBH": 0.5411, - "hfopenllm_v2/MATH Level 5": 0.1654, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.4119, - "hfopenllm_v2/MMLU-PRO": 0.3761 - } - }, - { - "id": "DreadPoor/Yafune-8B-Model_Stock", - "name": "Yafune-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7533, - "hfopenllm_v2/BBH": 0.5467, - "hfopenllm_v2/MATH Level 5": 0.1662, - "hfopenllm_v2/GPQA": 0.3272, - "hfopenllm_v2/MUSR": 0.4173, - "hfopenllm_v2/MMLU-PRO": 0.3851 - } - }, - { - "id": "DreadPoor/Yearn_V3-8B-Model_Stock", - "name": "Yearn_V3-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.729, - "hfopenllm_v2/BBH": 0.5322, - "hfopenllm_v2/MATH Level 5": 0.1896, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.3909, - "hfopenllm_v2/MMLU-PRO": 0.3802 - } - }, - { - "id": "DreadPoor/Zelus-8B-Model_Stock", - "name": "Zelus-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7788, - "hfopenllm_v2/BBH": 0.5307, - "hfopenllm_v2/MATH Level 5": 0.1647, - "hfopenllm_v2/GPQA": 0.3062, - "hfopenllm_v2/MUSR": 0.4214, - "hfopenllm_v2/MMLU-PRO": 0.3841 - } - }, - { - "id": "DreadPoor/Zelus_V2-8B-Model_Stock", - "name": "Zelus_V2-8B-Model_Stock", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7898, - "hfopenllm_v2/BBH": 0.5345, - "hfopenllm_v2/MATH Level 5": 0.2054, - "hfopenllm_v2/GPQA": 0.3096, - "hfopenllm_v2/MUSR": 0.3961, - "hfopenllm_v2/MMLU-PRO": 0.3833 - } - }, - { - "id": "DreadPoor/ZEUS-8B-V17-Abliterated_ALT", - "name": "ZEUS-8B-V17-Abliterated_ALT", - "developer": "DreadPoor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5511, - "hfopenllm_v2/BBH": 0.5231, - "hfopenllm_v2/MATH Level 5": 0.1903, - "hfopenllm_v2/GPQA": 0.3079, - "hfopenllm_v2/MUSR": 0.4149, - "hfopenllm_v2/MMLU-PRO": 0.389 - } - }, - { - "id": "dreamgen/WizardLM-2-7B", - "name": "WizardLM-2-7B", - "developer": "dreamgen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4583, - "hfopenllm_v2/BBH": 0.3487, - "hfopenllm_v2/MATH Level 5": 0.0332, - "hfopenllm_v2/GPQA": 0.2869, - "hfopenllm_v2/MUSR": 0.3941, - "hfopenllm_v2/MMLU-PRO": 0.266 - } - }, - { - "id": "DRXD1000/Atlas-7B", - "name": "Atlas-7B", - "developer": "DRXD1000", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3704, - "hfopenllm_v2/BBH": 0.3302, - "hfopenllm_v2/MATH Level 5": 0.0189, - "hfopenllm_v2/GPQA": 0.2576, - "hfopenllm_v2/MUSR": 0.3342, - "hfopenllm_v2/MMLU-PRO": 0.1401 - } - }, - { - "id": "DRXD1000/Phoenix-7B", - "name": "Phoenix-7B", - "developer": "DRXD1000", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.321, - "hfopenllm_v2/BBH": 0.3932, - "hfopenllm_v2/MATH Level 5": 0.0166, - "hfopenllm_v2/GPQA": 0.2785, - "hfopenllm_v2/MUSR": 0.3849, - "hfopenllm_v2/MMLU-PRO": 0.2343 - } - }, - { - "id": "DUAL-GPO/zephyr-7b-ipo-0k-15k-i1", - "name": "zephyr-7b-ipo-0k-15k-i1", - "developer": "DUAL-GPO", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2756, - "hfopenllm_v2/BBH": 0.4473, - "hfopenllm_v2/MATH Level 5": 0.0302, - "hfopenllm_v2/GPQA": 0.2911, - "hfopenllm_v2/MUSR": 0.4173, - "hfopenllm_v2/MMLU-PRO": 0.313 - } - }, - { - "id": "dustinwloring1988/Reflexis-8b-chat-v1", - "name": "Reflexis-8b-chat-v1", - "developer": "dustinwloring1988", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3658, - "hfopenllm_v2/BBH": 0.4664, - "hfopenllm_v2/MATH Level 5": 0.1156, - "hfopenllm_v2/GPQA": 0.2542, - "hfopenllm_v2/MUSR": 0.3754, - "hfopenllm_v2/MMLU-PRO": 0.3384 - } - }, - { - "id": "dustinwloring1988/Reflexis-8b-chat-v2", - "name": "Reflexis-8b-chat-v2", - "developer": "dustinwloring1988", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3912, - "hfopenllm_v2/BBH": 0.4724, - "hfopenllm_v2/MATH Level 5": 0.1163, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.3526, - "hfopenllm_v2/MMLU-PRO": 0.3378 - } - }, - { - "id": "dustinwloring1988/Reflexis-8b-chat-v3", - "name": "Reflexis-8b-chat-v3", - "developer": "dustinwloring1988", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5367, - "hfopenllm_v2/BBH": 0.4658, - "hfopenllm_v2/MATH Level 5": 0.1224, - "hfopenllm_v2/GPQA": 0.2424, - "hfopenllm_v2/MUSR": 0.3512, - "hfopenllm_v2/MMLU-PRO": 0.3548 - } - }, - { - "id": "dustinwloring1988/Reflexis-8b-chat-v4", - "name": "Reflexis-8b-chat-v4", - "developer": "dustinwloring1988", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4698, - "hfopenllm_v2/BBH": 0.4686, - "hfopenllm_v2/MATH Level 5": 0.1027, - "hfopenllm_v2/GPQA": 0.2341, - "hfopenllm_v2/MUSR": 0.3393, - "hfopenllm_v2/MMLU-PRO": 0.339 - } - }, - { - "id": "dustinwloring1988/Reflexis-8b-chat-v5", - "name": "Reflexis-8b-chat-v5", - "developer": "dustinwloring1988", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4238, - "hfopenllm_v2/BBH": 0.4782, - "hfopenllm_v2/MATH Level 5": 0.1216, - "hfopenllm_v2/GPQA": 0.271, - "hfopenllm_v2/MUSR": 0.3354, - "hfopenllm_v2/MMLU-PRO": 0.3217 - } - }, - { - "id": "dustinwloring1988/Reflexis-8b-chat-v6", - "name": "Reflexis-8b-chat-v6", - "developer": "dustinwloring1988", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4939, - "hfopenllm_v2/BBH": 0.481, - "hfopenllm_v2/MATH Level 5": 0.1299, - "hfopenllm_v2/GPQA": 0.2626, - "hfopenllm_v2/MUSR": 0.3753, - "hfopenllm_v2/MMLU-PRO": 0.3479 - } - }, - { - "id": "dustinwloring1988/Reflexis-8b-chat-v7", - "name": "Reflexis-8b-chat-v7", - "developer": "dustinwloring1988", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.398, - "hfopenllm_v2/BBH": 0.481, - "hfopenllm_v2/MATH Level 5": 0.1631, - "hfopenllm_v2/GPQA": 0.2617, - "hfopenllm_v2/MUSR": 0.3222, - "hfopenllm_v2/MMLU-PRO": 0.3643 - } - }, - { - "id": "duyhv1411/Llama-3.2-1B-en-vi", - "name": "Llama-3.2-1B-en-vi", - "developer": "duyhv1411", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4788, - "hfopenllm_v2/BBH": 0.3291, - "hfopenllm_v2/MATH Level 5": 0.0287, - "hfopenllm_v2/GPQA": 0.2768, - "hfopenllm_v2/MUSR": 0.3197, - "hfopenllm_v2/MMLU-PRO": 0.1341 - } - }, - { - "id": "duyhv1411/Llama-3.2-3B-en-vi", - "name": "Llama-3.2-3B-en-vi", - "developer": "duyhv1411", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4852, - "hfopenllm_v2/BBH": 0.3272, - "hfopenllm_v2/MATH Level 5": 0.0227, - "hfopenllm_v2/GPQA": 0.2752, - "hfopenllm_v2/MUSR": 0.321, - "hfopenllm_v2/MMLU-PRO": 0.1359 - } - }, - { - "id": "dwikitheduck/gemma-2-2b-id", - "name": "gemma-2-2b-id", - "developer": "dwikitheduck", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3879, - "hfopenllm_v2/BBH": 0.3962, - "hfopenllm_v2/MATH Level 5": 0.0453, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.4154, - "hfopenllm_v2/MMLU-PRO": 0.2173 - } - }, - { - "id": "dwikitheduck/gemma-2-2b-id-inst", - "name": "gemma-2-2b-id-inst", - "developer": "dwikitheduck", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3879, - "hfopenllm_v2/BBH": 0.3962, - "hfopenllm_v2/MATH Level 5": 0.0453, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.4154, - "hfopenllm_v2/MMLU-PRO": 0.2173 - } - }, - { - "id": "dwikitheduck/gemma-2-2b-id-instruct", - "name": "gemma-2-2b-id-instruct", - "developer": "dwikitheduck", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3879, - "hfopenllm_v2/BBH": 0.3962, - "hfopenllm_v2/MATH Level 5": 0.0453, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.4154, - "hfopenllm_v2/MMLU-PRO": 0.2173 - } - }, - { - "id": "dwikitheduck/gen-inst-1", - "name": "gen-inst-1", - "developer": "dwikitheduck", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.775, - "hfopenllm_v2/BBH": 0.642, - "hfopenllm_v2/MATH Level 5": 0.4554, - "hfopenllm_v2/GPQA": 0.3716, - "hfopenllm_v2/MUSR": 0.4205, - "hfopenllm_v2/MMLU-PRO": 0.5089 - } - }, - { - "id": "dwikitheduck/gen-try1", - "name": "gen-try1", - "developer": "dwikitheduck", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7522, - "hfopenllm_v2/BBH": 0.6359, - "hfopenllm_v2/MATH Level 5": 0.4101, - "hfopenllm_v2/GPQA": 0.3414, - "hfopenllm_v2/MUSR": 0.4416, - "hfopenllm_v2/MMLU-PRO": 0.5111 - } - }, - { - "id": "dwikitheduck/gen-try1-notemp", - "name": "gen-try1-notemp", - "developer": "dwikitheduck", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2627, - "hfopenllm_v2/BBH": 0.6263, - "hfopenllm_v2/MATH Level 5": 0.318, - "hfopenllm_v2/GPQA": 0.354, - "hfopenllm_v2/MUSR": 0.4714, - "hfopenllm_v2/MMLU-PRO": 0.521 - } - }, - { - "id": "dzakwan/dzakwan-MoE-4x7b-Beta", - "name": "dzakwan-MoE-4x7b-Beta", - "developer": "dzakwan", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4443, - "hfopenllm_v2/BBH": 0.514, - "hfopenllm_v2/MATH Level 5": 0.0778, - "hfopenllm_v2/GPQA": 0.2861, - "hfopenllm_v2/MUSR": 0.4267, - "hfopenllm_v2/MMLU-PRO": 0.3108 - } - }, - { - "id": "DZgas/GIGABATEMAN-7B", - "name": "GIGABATEMAN-7B", - "developer": "DZgas", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4607, - "hfopenllm_v2/BBH": 0.5032, - "hfopenllm_v2/MATH Level 5": 0.0551, - "hfopenllm_v2/GPQA": 0.2894, - "hfopenllm_v2/MUSR": 0.4328, - "hfopenllm_v2/MMLU-PRO": 0.3177 - } - }, - { - "id": "ECE-ILAB-PRYMMAL/ILAB-Merging-3B-V2", - "name": "ILAB-Merging-3B-V2", - "developer": "ECE-ILAB-PRYMMAL", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4029, - "hfopenllm_v2/BBH": 0.5402, - "hfopenllm_v2/MATH Level 5": 0.1518, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.4332, - "hfopenllm_v2/MMLU-PRO": 0.3861 - } - }, - { - "id": "Edgerunners/meta-llama-3-8b-instruct-hf-ortho-baukit-34fail-3000total-bf16", - "name": "meta-llama-3-8b-instruct-hf-ortho-baukit-34fail-3000total-bf16", - "developer": "Edgerunners", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7147, - "hfopenllm_v2/BBH": 0.498, - "hfopenllm_v2/MATH Level 5": 0.0906, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3342, - "hfopenllm_v2/MMLU-PRO": 0.3636 - } - }, - { - "id": "ehristoforu/coolqwen-3b-it", - "name": "coolqwen-3b-it", - "developer": "ehristoforu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6473, - "hfopenllm_v2/BBH": 0.4851, - "hfopenllm_v2/MATH Level 5": 0.3671, - "hfopenllm_v2/GPQA": 0.2827, - "hfopenllm_v2/MUSR": 0.4125, - "hfopenllm_v2/MMLU-PRO": 0.3601 - } - }, - { - "id": "ehristoforu/della-70b-test-v1", - "name": "della-70b-test-v1", - "developer": "ehristoforu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4979, - "hfopenllm_v2/BBH": 0.3029, - "hfopenllm_v2/MATH Level 5": 0.0098, - "hfopenllm_v2/GPQA": 0.2525, - "hfopenllm_v2/MUSR": 0.4555, - "hfopenllm_v2/MMLU-PRO": 0.1575 - } - }, - { - "id": "ehristoforu/Falcon3-8B-Franken-Basestruct", - "name": "Falcon3-8B-Franken-Basestruct", - "developer": "ehristoforu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1715, - "hfopenllm_v2/BBH": 0.5463, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.3406, - "hfopenllm_v2/MUSR": 0.3555, - "hfopenllm_v2/MMLU-PRO": 0.3947 - } - }, - { - "id": "ehristoforu/Falcon3-MoE-2x7B-Insruct", - "name": "Falcon3-MoE-2x7B-Insruct", - "developer": "ehristoforu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7643, - "hfopenllm_v2/BBH": 0.5648, - "hfopenllm_v2/MATH Level 5": 0.4124, - "hfopenllm_v2/GPQA": 0.3121, - "hfopenllm_v2/MUSR": 0.484, - "hfopenllm_v2/MMLU-PRO": 0.4095 - } - }, - { - "id": "ehristoforu/falcon3-ultraset", - "name": "falcon3-ultraset", - "developer": "ehristoforu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7135, - "hfopenllm_v2/BBH": 0.5584, - "hfopenllm_v2/MATH Level 5": 0.2122, - "hfopenllm_v2/GPQA": 0.3322, - "hfopenllm_v2/MUSR": 0.4853, - "hfopenllm_v2/MMLU-PRO": 0.3982 - } - }, - { - "id": "ehristoforu/fd-lora-merged-16x32", - "name": "fd-lora-merged-16x32", - "developer": "ehristoforu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3481, - "hfopenllm_v2/BBH": 0.3308, - "hfopenllm_v2/MATH Level 5": 0.1707, - "hfopenllm_v2/GPQA": 0.2534, - "hfopenllm_v2/MUSR": 0.3514, - "hfopenllm_v2/MMLU-PRO": 0.1205 - } - }, - { - "id": "ehristoforu/fd-lora-merged-64x128", - "name": "fd-lora-merged-64x128", - "developer": "ehristoforu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3281, - "hfopenllm_v2/BBH": 0.3345, - "hfopenllm_v2/MATH Level 5": 0.1873, - "hfopenllm_v2/GPQA": 0.255, - "hfopenllm_v2/MUSR": 0.3368, - "hfopenllm_v2/MMLU-PRO": 0.1537 - } - }, - { - "id": "ehristoforu/fp4-14b-it-v1", - "name": "fp4-14b-it-v1", - "developer": "ehristoforu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2535, - "hfopenllm_v2/BBH": 0.574, - "hfopenllm_v2/MATH Level 5": 0.0408, - "hfopenllm_v2/GPQA": 0.2953, - "hfopenllm_v2/MUSR": 0.3595, - "hfopenllm_v2/MMLU-PRO": 0.4205 - } - }, - { - "id": "ehristoforu/fp4-14b-v1-fix", - "name": "fp4-14b-v1-fix", - "developer": "ehristoforu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6742, - "hfopenllm_v2/BBH": 0.6817, - "hfopenllm_v2/MATH Level 5": 0.4207, - "hfopenllm_v2/GPQA": 0.354, - "hfopenllm_v2/MUSR": 0.4532, - "hfopenllm_v2/MMLU-PRO": 0.5353 - } - }, - { - "id": "ehristoforu/fq2.5-7b-it-normalize_false", - "name": "fq2.5-7b-it-normalize_false", - "developer": "ehristoforu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7399, - "hfopenllm_v2/BBH": 0.552, - "hfopenllm_v2/MATH Level 5": 0.4622, - "hfopenllm_v2/GPQA": 0.302, - "hfopenllm_v2/MUSR": 0.4612, - "hfopenllm_v2/MMLU-PRO": 0.4413 - } - }, - { - "id": "ehristoforu/fq2.5-7b-it-normalize_true", - "name": "fq2.5-7b-it-normalize_true", - "developer": "ehristoforu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7399, - "hfopenllm_v2/BBH": 0.552, - "hfopenllm_v2/MATH Level 5": 0.4622, - "hfopenllm_v2/GPQA": 0.302, - "hfopenllm_v2/MUSR": 0.4612, - "hfopenllm_v2/MMLU-PRO": 0.4413 - } - }, - { - "id": "ehristoforu/frqwen2.5-from7b-duable4layers-it", - "name": "frqwen2.5-from7b-duable4layers-it", - "developer": "ehristoforu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7729, - "hfopenllm_v2/BBH": 0.5264, - "hfopenllm_v2/MATH Level 5": 0.4509, - "hfopenllm_v2/GPQA": 0.2953, - "hfopenllm_v2/MUSR": 0.4166, - "hfopenllm_v2/MMLU-PRO": 0.4126 - } - }, - { - "id": "ehristoforu/frqwen2.5-from7b-it", - "name": "frqwen2.5-from7b-it", - "developer": "ehristoforu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6532, - "hfopenllm_v2/BBH": 0.5143, - "hfopenllm_v2/MATH Level 5": 0.2923, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.4086, - "hfopenllm_v2/MMLU-PRO": 0.3977 - } - }, - { - "id": "ehristoforu/Gemma2-9B-it-psy10k-mental_health", - "name": "Gemma2-9B-it-psy10k-mental_health", - "developer": "ehristoforu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5887, - "hfopenllm_v2/BBH": 0.5539, - "hfopenllm_v2/MATH Level 5": 0.1631, - "hfopenllm_v2/GPQA": 0.3372, - "hfopenllm_v2/MUSR": 0.4086, - "hfopenllm_v2/MMLU-PRO": 0.3829 - } - }, - { - "id": "ehristoforu/Gemma2-9b-it-train6", - "name": "Gemma2-9b-it-train6", - "developer": "ehristoforu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7025, - "hfopenllm_v2/BBH": 0.5898, - "hfopenllm_v2/MATH Level 5": 0.1911, - "hfopenllm_v2/GPQA": 0.3289, - "hfopenllm_v2/MUSR": 0.4084, - "hfopenllm_v2/MMLU-PRO": 0.3942 - } - }, - { - "id": "ehristoforu/HappyLlama1", - "name": "HappyLlama1", - "developer": "ehristoforu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7363, - "hfopenllm_v2/BBH": 0.4996, - "hfopenllm_v2/MATH Level 5": 0.1427, - "hfopenllm_v2/GPQA": 0.2836, - "hfopenllm_v2/MUSR": 0.4287, - "hfopenllm_v2/MMLU-PRO": 0.3546 - } - }, - { - "id": "ehristoforu/mllama-3.1-8b-instruct", - "name": "mllama-3.1-8b-instruct", - "developer": "ehristoforu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3458, - "hfopenllm_v2/BBH": 0.4718, - "hfopenllm_v2/MATH Level 5": 0.3776, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.338, - "hfopenllm_v2/MMLU-PRO": 0.2533 - } - }, - { - "id": "ehristoforu/mllama-3.1-8b-it", - "name": "mllama-3.1-8b-it", - "developer": "ehristoforu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3879, - "hfopenllm_v2/BBH": 0.4868, - "hfopenllm_v2/MATH Level 5": 0.3799, - "hfopenllm_v2/GPQA": 0.2768, - "hfopenllm_v2/MUSR": 0.3349, - "hfopenllm_v2/MMLU-PRO": 0.2622 - } - }, - { - "id": "ehristoforu/moremerge", - "name": "moremerge", - "developer": "ehristoforu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2019, - "hfopenllm_v2/BBH": 0.2868, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3566, - "hfopenllm_v2/MMLU-PRO": 0.1065 - } - }, - { - "id": "ehristoforu/moremerge-upscaled", - "name": "moremerge-upscaled", - "developer": "ehristoforu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1979, - "hfopenllm_v2/BBH": 0.2698, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2466, - "hfopenllm_v2/MUSR": 0.3593, - "hfopenllm_v2/MMLU-PRO": 0.1041 - } - }, - { - "id": "ehristoforu/phi-4-25b", - "name": "phi-4-25b", - "developer": "ehristoforu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6484, - "hfopenllm_v2/BBH": 0.6908, - "hfopenllm_v2/MATH Level 5": 0.4524, - "hfopenllm_v2/GPQA": 0.3188, - "hfopenllm_v2/MUSR": 0.4208, - "hfopenllm_v2/MMLU-PRO": 0.5351 - } - }, - { - "id": "ehristoforu/qwen2.5-test-32b-it", - "name": "qwen2.5-test-32b-it", - "developer": "ehristoforu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7889, - "hfopenllm_v2/BBH": 0.7081, - "hfopenllm_v2/MATH Level 5": 0.5974, - "hfopenllm_v2/GPQA": 0.3641, - "hfopenllm_v2/MUSR": 0.4578, - "hfopenllm_v2/MMLU-PRO": 0.5765 - } - }, - { - "id": "ehristoforu/qwen2.5-with-lora-think-3b-it", - "name": "qwen2.5-with-lora-think-3b-it", - "developer": "ehristoforu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5319, - "hfopenllm_v2/BBH": 0.4687, - "hfopenllm_v2/MATH Level 5": 0.2364, - "hfopenllm_v2/GPQA": 0.2802, - "hfopenllm_v2/MUSR": 0.431, - "hfopenllm_v2/MMLU-PRO": 0.3403 - } - }, - { - "id": "ehristoforu/QwenQwen2.5-7B-IT", - "name": "QwenQwen2.5-7B-IT", - "developer": "ehristoforu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7518, - "hfopenllm_v2/BBH": 0.5398, - "hfopenllm_v2/MATH Level 5": 0.5091, - "hfopenllm_v2/GPQA": 0.3037, - "hfopenllm_v2/MUSR": 0.4034, - "hfopenllm_v2/MMLU-PRO": 0.4289 - } - }, - { - "id": "ehristoforu/QwenQwen2.5-7B-IT-Dare", - "name": "QwenQwen2.5-7B-IT-Dare", - "developer": "ehristoforu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7509, - "hfopenllm_v2/BBH": 0.5398, - "hfopenllm_v2/MATH Level 5": 0.5091, - "hfopenllm_v2/GPQA": 0.3037, - "hfopenllm_v2/MUSR": 0.4034, - "hfopenllm_v2/MMLU-PRO": 0.4289 - } - }, - { - "id": "ehristoforu/rmoe-v1", - "name": "rmoe-v1", - "developer": "ehristoforu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.265, - "hfopenllm_v2/BBH": 0.2929, - "hfopenllm_v2/MATH Level 5": 0.0015, - "hfopenllm_v2/GPQA": 0.2584, - "hfopenllm_v2/MUSR": 0.3663, - "hfopenllm_v2/MMLU-PRO": 0.1125 - } - }, - { - "id": "ehristoforu/RQwen-v0.1", - "name": "RQwen-v0.1", - "developer": "ehristoforu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7625, - "hfopenllm_v2/BBH": 0.6446, - "hfopenllm_v2/MATH Level 5": 0.4645, - "hfopenllm_v2/GPQA": 0.3255, - "hfopenllm_v2/MUSR": 0.4139, - "hfopenllm_v2/MMLU-PRO": 0.5202 - } - }, - { - "id": "ehristoforu/RQwen-v0.2", - "name": "RQwen-v0.2", - "developer": "ehristoforu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7504, - "hfopenllm_v2/BBH": 0.6427, - "hfopenllm_v2/MATH Level 5": 0.327, - "hfopenllm_v2/GPQA": 0.3372, - "hfopenllm_v2/MUSR": 0.4207, - "hfopenllm_v2/MMLU-PRO": 0.5159 - } - }, - { - "id": "ehristoforu/rufalcon3-3b-it", - "name": "rufalcon3-3b-it", - "developer": "ehristoforu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5942, - "hfopenllm_v2/BBH": 0.4155, - "hfopenllm_v2/MATH Level 5": 0.1782, - "hfopenllm_v2/GPQA": 0.2727, - "hfopenllm_v2/MUSR": 0.3895, - "hfopenllm_v2/MMLU-PRO": 0.2348 - } - }, - { - "id": "ehristoforu/ruphi-4b", - "name": "ruphi-4b", - "developer": "ehristoforu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1752, - "hfopenllm_v2/BBH": 0.2906, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2399, - "hfopenllm_v2/MUSR": 0.3512, - "hfopenllm_v2/MMLU-PRO": 0.1126 - } - }, - { - "id": "ehristoforu/SoRu-0009", - "name": "SoRu-0009", - "developer": "ehristoforu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2582, - "hfopenllm_v2/BBH": 0.315, - "hfopenllm_v2/MATH Level 5": 0.0211, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.3369, - "hfopenllm_v2/MMLU-PRO": 0.1239 - } - }, - { - "id": "ehristoforu/testq-32b", - "name": "testq-32b", - "developer": "ehristoforu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1876, - "hfopenllm_v2/BBH": 0.2877, - "hfopenllm_v2/MATH Level 5": 0.003, - "hfopenllm_v2/GPQA": 0.2542, - "hfopenllm_v2/MUSR": 0.3715, - "hfopenllm_v2/MMLU-PRO": 0.1166 - } - }, - { - "id": "ehristoforu/tmoe", - "name": "tmoe", - "developer": "ehristoforu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1193, - "hfopenllm_v2/BBH": 0.3073, - "hfopenllm_v2/MATH Level 5": 0.0076, - "hfopenllm_v2/GPQA": 0.2232, - "hfopenllm_v2/MUSR": 0.3699, - "hfopenllm_v2/MMLU-PRO": 0.1191 - } - }, - { - "id": "ehristoforu/tmoe-v2", - "name": "tmoe-v2", - "developer": "ehristoforu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1903, - "hfopenllm_v2/BBH": 0.2897, - "hfopenllm_v2/MATH Level 5": 0.0023, - "hfopenllm_v2/GPQA": 0.2634, - "hfopenllm_v2/MUSR": 0.4151, - "hfopenllm_v2/MMLU-PRO": 0.11 - } - }, - { - "id": "ehristoforu/trd-7b-it", - "name": "trd-7b-it", - "developer": "ehristoforu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2185, - "hfopenllm_v2/BBH": 0.299, - "hfopenllm_v2/MATH Level 5": 0.0317, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.3794, - "hfopenllm_v2/MMLU-PRO": 0.1179 - } - }, - { - "id": "ehristoforu/ud-14b", - "name": "ud-14b", - "developer": "ehristoforu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4235, - "hfopenllm_v2/BBH": 0.3324, - "hfopenllm_v2/MATH Level 5": 0.1903, - "hfopenllm_v2/GPQA": 0.2374, - "hfopenllm_v2/MUSR": 0.4394, - "hfopenllm_v2/MMLU-PRO": 0.2415 - } - }, - { - "id": "EleutherAI/gpt-j-6b", - "name": "gpt-j-6b", - "developer": "EleutherAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2522, - "hfopenllm_v2/BBH": 0.3191, - "hfopenllm_v2/MATH Level 5": 0.0136, - "hfopenllm_v2/GPQA": 0.2458, - "hfopenllm_v2/MUSR": 0.3658, - "hfopenllm_v2/MMLU-PRO": 0.1241 - } - }, - { - "id": "EleutherAI/gpt-neo-1.3B", - "name": "gpt-neo-1.3B", - "developer": "EleutherAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2079, - "hfopenllm_v2/BBH": 0.3039, - "hfopenllm_v2/MATH Level 5": 0.0106, - "hfopenllm_v2/GPQA": 0.2559, - "hfopenllm_v2/MUSR": 0.3817, - "hfopenllm_v2/MMLU-PRO": 0.1164 - } - }, - { - "id": "EleutherAI/gpt-neo-125m", - "name": "gpt-neo-125m", - "developer": "EleutherAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1905, - "hfopenllm_v2/BBH": 0.3115, - "hfopenllm_v2/MATH Level 5": 0.006, - "hfopenllm_v2/GPQA": 0.2534, - "hfopenllm_v2/MUSR": 0.3593, - "hfopenllm_v2/MMLU-PRO": 0.1026 - } - }, - { - "id": "EleutherAI/gpt-neo-2.7B", - "name": "gpt-neo-2.7B", - "developer": "EleutherAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.259, - "hfopenllm_v2/BBH": 0.314, - "hfopenllm_v2/MATH Level 5": 0.0106, - "hfopenllm_v2/GPQA": 0.2659, - "hfopenllm_v2/MUSR": 0.3554, - "hfopenllm_v2/MMLU-PRO": 0.1163 - } - }, - { - "id": "EleutherAI/gpt-neox-20b", - "name": "gpt-neox-20b", - "developer": "EleutherAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2587, - "hfopenllm_v2/BBH": 0.3165, - "hfopenllm_v2/MATH Level 5": 0.0136, - "hfopenllm_v2/GPQA": 0.2433, - "hfopenllm_v2/MUSR": 0.3647, - "hfopenllm_v2/MMLU-PRO": 0.1155 - } - }, - { - "id": "EleutherAI/pythia-1.4b", - "name": "pythia-1.4b", - "developer": "EleutherAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2371, - "hfopenllm_v2/BBH": 0.315, - "hfopenllm_v2/MATH Level 5": 0.0151, - "hfopenllm_v2/GPQA": 0.2617, - "hfopenllm_v2/MUSR": 0.3538, - "hfopenllm_v2/MMLU-PRO": 0.1123 - } - }, - { - "id": "EleutherAI/pythia-12b", - "name": "Pythia 12B", - "developer": "EleutherAI", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.257, - "helm_classic/MMLU": 0.274, - "helm_classic/BoolQ": 0.662, - "helm_classic/NarrativeQA": 0.596, - "helm_classic/NaturalQuestions (open-book)": 0.581, - "helm_classic/QuAC": 0.313, - "helm_classic/HellaSwag": -1.0, - "helm_classic/OpenbookQA": -1.0, - "helm_classic/TruthfulQA": 0.177, - "helm_classic/MS MARCO (TREC)": -1.0, - "helm_classic/CNN/DailyMail": -1.0, - "helm_classic/XSUM": -1.0, - "helm_classic/IMDB": 0.931, - "helm_classic/CivilComments": 0.531, - "helm_classic/RAFT": 0.514, - "hfopenllm_v2/IFEval": 0.2471, - "hfopenllm_v2/BBH": 0.318, - "hfopenllm_v2/MATH Level 5": 0.0166, - "hfopenllm_v2/GPQA": 0.2466, - "hfopenllm_v2/MUSR": 0.3647, - "hfopenllm_v2/MMLU-PRO": 0.1109 - } - }, - { - "id": "EleutherAI/pythia-160m", - "name": "pythia-160m", - "developer": "EleutherAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1816, - "hfopenllm_v2/BBH": 0.297, - "hfopenllm_v2/MATH Level 5": 0.0091, - "hfopenllm_v2/GPQA": 0.2584, - "hfopenllm_v2/MUSR": 0.4179, - "hfopenllm_v2/MMLU-PRO": 0.112 - } - }, - { - "id": "EleutherAI/pythia-1b", - "name": "pythia-1b", - "developer": "EleutherAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2208, - "hfopenllm_v2/BBH": 0.3004, - "hfopenllm_v2/MATH Level 5": 0.0091, - "hfopenllm_v2/GPQA": 0.2567, - "hfopenllm_v2/MUSR": 0.3552, - "hfopenllm_v2/MMLU-PRO": 0.1136 - } - }, - { - "id": "EleutherAI/pythia-2.8b", - "name": "pythia-2.8b", - "developer": "EleutherAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2173, - "hfopenllm_v2/BBH": 0.3224, - "hfopenllm_v2/MATH Level 5": 0.0136, - "hfopenllm_v2/GPQA": 0.25, - "hfopenllm_v2/MUSR": 0.3486, - "hfopenllm_v2/MMLU-PRO": 0.1137 - } - }, - { - "id": "EleutherAI/pythia-410m", - "name": "pythia-410m", - "developer": "EleutherAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2195, - "hfopenllm_v2/BBH": 0.3028, - "hfopenllm_v2/MATH Level 5": 0.0098, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3578, - "hfopenllm_v2/MMLU-PRO": 0.1128 - } - }, - { - "id": "EleutherAI/pythia-6.9b", - "name": "Pythia 6.9B", - "developer": "EleutherAI", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.196, - "helm_classic/MMLU": 0.236, - "helm_classic/BoolQ": 0.631, - "helm_classic/NarrativeQA": 0.528, - "helm_classic/NaturalQuestions (open-book)": 0.539, - "helm_classic/QuAC": 0.296, - "helm_classic/HellaSwag": -1.0, - "helm_classic/OpenbookQA": -1.0, - "helm_classic/TruthfulQA": 0.213, - "helm_classic/MS MARCO (TREC)": -1.0, - "helm_classic/CNN/DailyMail": -1.0, - "helm_classic/XSUM": -1.0, - "helm_classic/IMDB": 0.928, - "helm_classic/CivilComments": 0.511, - "helm_classic/RAFT": 0.502, - "hfopenllm_v2/IFEval": 0.2281, - "hfopenllm_v2/BBH": 0.3232, - "hfopenllm_v2/MATH Level 5": 0.0144, - "hfopenllm_v2/GPQA": 0.2517, - "hfopenllm_v2/MUSR": 0.3591, - "hfopenllm_v2/MMLU-PRO": 0.1147 - } - }, - { - "id": "elinas/Chronos-Gold-12B-1.0", - "name": "Chronos-Gold-12B-1.0", - "developer": "elinas", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3166, - "hfopenllm_v2/BBH": 0.5515, - "hfopenllm_v2/MATH Level 5": 0.0695, - "hfopenllm_v2/GPQA": 0.318, - "hfopenllm_v2/MUSR": 0.474, - "hfopenllm_v2/MMLU-PRO": 0.3518 - } - }, - { - "id": "ell44ot/gemma-2b-def", - "name": "gemma-2b-def", - "developer": "ell44ot", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2693, - "hfopenllm_v2/BBH": 0.3159, - "hfopenllm_v2/MATH Level 5": 0.0242, - "hfopenllm_v2/GPQA": 0.2735, - "hfopenllm_v2/MUSR": 0.367, - "hfopenllm_v2/MMLU-PRO": 0.1572 - } - }, - { - "id": "Enno-Ai/EnnoAi-Pro-French-Llama-3-8B-v0.4", - "name": "EnnoAi-Pro-French-Llama-3-8B-v0.4", - "developer": "Enno-Ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4189, - "hfopenllm_v2/BBH": 0.4075, - "hfopenllm_v2/MATH Level 5": 0.0363, - "hfopenllm_v2/GPQA": 0.271, - "hfopenllm_v2/MUSR": 0.417, - "hfopenllm_v2/MMLU-PRO": 0.2635 - } - }, - { - "id": "Enno-Ai/EnnoAi-Pro-Llama-3-8B", - "name": "EnnoAi-Pro-Llama-3-8B", - "developer": "Enno-Ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3195, - "hfopenllm_v2/BBH": 0.4152, - "hfopenllm_v2/MATH Level 5": 0.0219, - "hfopenllm_v2/GPQA": 0.2617, - "hfopenllm_v2/MUSR": 0.4071, - "hfopenllm_v2/MMLU-PRO": 0.2151 - } - }, - { - "id": "Enno-Ai/EnnoAi-Pro-Llama-3-8B-v0.3", - "name": "EnnoAi-Pro-Llama-3-8B-v0.3", - "developer": "Enno-Ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5083, - "hfopenllm_v2/BBH": 0.4101, - "hfopenllm_v2/MATH Level 5": 0.0483, - "hfopenllm_v2/GPQA": 0.2651, - "hfopenllm_v2/MUSR": 0.4236, - "hfopenllm_v2/MMLU-PRO": 0.299 - } - }, - { - "id": "Enno-Ai/EnnoAi-Pro-Llama-3.1-8B-v0.9", - "name": "EnnoAi-Pro-Llama-3.1-8B-v0.9", - "developer": "Enno-Ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4689, - "hfopenllm_v2/BBH": 0.416, - "hfopenllm_v2/MATH Level 5": 0.0378, - "hfopenllm_v2/GPQA": 0.2659, - "hfopenllm_v2/MUSR": 0.3832, - "hfopenllm_v2/MMLU-PRO": 0.2596 - } - }, - { - "id": "EnnoAi/EnnoAi-7B-French-Instruct-202502", - "name": "EnnoAi-7B-French-Instruct-202502", - "developer": "EnnoAi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5564, - "hfopenllm_v2/BBH": 0.5575, - "hfopenllm_v2/MATH Level 5": 0.3724, - "hfopenllm_v2/GPQA": 0.2953, - "hfopenllm_v2/MUSR": 0.46, - "hfopenllm_v2/MMLU-PRO": 0.4013 - } - }, - { - "id": "EnnoAi/EnnoAi-Pro-Llama-3.1-8B-v1.0", - "name": "EnnoAi-Pro-Llama-3.1-8B-v1.0", - "developer": "EnnoAi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4704, - "hfopenllm_v2/BBH": 0.416, - "hfopenllm_v2/MATH Level 5": 0.0378, - "hfopenllm_v2/GPQA": 0.2659, - "hfopenllm_v2/MUSR": 0.3832, - "hfopenllm_v2/MMLU-PRO": 0.2596 - } - }, - { - "id": "Epiculous/Azure_Dusk-v0.2", - "name": "Azure_Dusk-v0.2", - "developer": "Epiculous", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3467, - "hfopenllm_v2/BBH": 0.412, - "hfopenllm_v2/MATH Level 5": 0.0295, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.3835, - "hfopenllm_v2/MMLU-PRO": 0.3034 - } - }, - { - "id": "Epiculous/Crimson_Dawn-v0.2", - "name": "Crimson_Dawn-v0.2", - "developer": "Epiculous", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3103, - "hfopenllm_v2/BBH": 0.4482, - "hfopenllm_v2/MATH Level 5": 0.0431, - "hfopenllm_v2/GPQA": 0.276, - "hfopenllm_v2/MUSR": 0.4152, - "hfopenllm_v2/MMLU-PRO": 0.2721 - } - }, - { - "id": "Epiculous/NovaSpark", - "name": "NovaSpark", - "developer": "Epiculous", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6408, - "hfopenllm_v2/BBH": 0.5064, - "hfopenllm_v2/MATH Level 5": 0.1518, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.3882, - "hfopenllm_v2/MMLU-PRO": 0.3649 - } - }, - { - "id": "Epiculous/Violet_Twilight-v0.2", - "name": "Violet_Twilight-v0.2", - "developer": "Epiculous", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4532, - "hfopenllm_v2/BBH": 0.4615, - "hfopenllm_v2/MATH Level 5": 0.0287, - "hfopenllm_v2/GPQA": 0.2659, - "hfopenllm_v2/MUSR": 0.4299, - "hfopenllm_v2/MMLU-PRO": 0.3111 - } - }, - { - "id": "EpistemeAI/Alpaca-Llama3.1-8B", - "name": "Alpaca-Llama3.1-8B", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1599, - "hfopenllm_v2/BBH": 0.4755, - "hfopenllm_v2/MATH Level 5": 0.0506, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.3403, - "hfopenllm_v2/MMLU-PRO": 0.3246 - } - }, - { - "id": "EpistemeAI/Athena-gemma-2-2b-it", - "name": "Athena-gemma-2-2b-it", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3134, - "hfopenllm_v2/BBH": 0.4264, - "hfopenllm_v2/MATH Level 5": 0.0491, - "hfopenllm_v2/GPQA": 0.2685, - "hfopenllm_v2/MUSR": 0.4351, - "hfopenllm_v2/MMLU-PRO": 0.2422 - } - }, - { - "id": "EpistemeAI/Athena-gemma-2-2b-it-Philos", - "name": "Athena-gemma-2-2b-it-Philos", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4621, - "hfopenllm_v2/BBH": 0.3795, - "hfopenllm_v2/MATH Level 5": 0.037, - "hfopenllm_v2/GPQA": 0.281, - "hfopenllm_v2/MUSR": 0.4314, - "hfopenllm_v2/MMLU-PRO": 0.2248 - } - }, - { - "id": "EpistemeAI/Athene-codegemma-2-7b-it-alpaca-v1.3", - "name": "Athene-codegemma-2-7b-it-alpaca-v1.3", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.403, - "hfopenllm_v2/BBH": 0.4332, - "hfopenllm_v2/MATH Level 5": 0.0619, - "hfopenllm_v2/GPQA": 0.2802, - "hfopenllm_v2/MUSR": 0.4503, - "hfopenllm_v2/MMLU-PRO": 0.2587 - } - }, - { - "id": "EpistemeAI/DeepPhi-3.5-mini-instruct", - "name": "DeepPhi-3.5-mini-instruct", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1326, - "hfopenllm_v2/BBH": 0.2882, - "hfopenllm_v2/MATH Level 5": 0.0068, - "hfopenllm_v2/GPQA": 0.2332, - "hfopenllm_v2/MUSR": 0.3656, - "hfopenllm_v2/MMLU-PRO": 0.1103 - } - }, - { - "id": "EpistemeAI/DeepThinkers-Phi4", - "name": "DeepThinkers-Phi4", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.694, - "hfopenllm_v2/BBH": 0.679, - "hfopenllm_v2/MATH Level 5": 0.4585, - "hfopenllm_v2/GPQA": 0.3406, - "hfopenllm_v2/MUSR": 0.3981, - "hfopenllm_v2/MMLU-PRO": 0.5258 - } - }, - { - "id": "EpistemeAI/FineLlama3.1-8B-Instruct", - "name": "FineLlama3.1-8B-Instruct", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.08, - "hfopenllm_v2/BBH": 0.4557, - "hfopenllm_v2/MATH Level 5": 0.0347, - "hfopenllm_v2/GPQA": 0.2802, - "hfopenllm_v2/MUSR": 0.3482, - "hfopenllm_v2/MMLU-PRO": 0.3113 - } - }, - { - "id": "EpistemeAI/Fireball-12B", - "name": "Fireball-12B", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1834, - "hfopenllm_v2/BBH": 0.5111, - "hfopenllm_v2/MATH Level 5": 0.0408, - "hfopenllm_v2/GPQA": 0.2617, - "hfopenllm_v2/MUSR": 0.4236, - "hfopenllm_v2/MMLU-PRO": 0.3344 - } - }, - { - "id": "EpistemeAI/Fireball-12B-v1.13a-philosophers", - "name": "Fireball-12B-v1.13a-philosophers", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0876, - "hfopenllm_v2/BBH": 0.5103, - "hfopenllm_v2/MATH Level 5": 0.0461, - "hfopenllm_v2/GPQA": 0.3012, - "hfopenllm_v2/MUSR": 0.4081, - "hfopenllm_v2/MMLU-PRO": 0.3367 - } - }, - { - "id": "EpistemeAI/Fireball-Alpaca-Llama-3.1-8B-Philos-DPO-200", - "name": "Fireball-Alpaca-Llama-3.1-8B-Philos-DPO-200", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4577, - "hfopenllm_v2/BBH": 0.4838, - "hfopenllm_v2/MATH Level 5": 0.1231, - "hfopenllm_v2/GPQA": 0.3003, - "hfopenllm_v2/MUSR": 0.3945, - "hfopenllm_v2/MMLU-PRO": 0.3583 - } - }, - { - "id": "EpistemeAI/Fireball-Alpaca-Llama3.1.07-8B-Philos-Math-KTO-beta", - "name": "Fireball-Alpaca-Llama3.1.07-8B-Philos-Math-KTO-beta", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7274, - "hfopenllm_v2/BBH": 0.4865, - "hfopenllm_v2/MATH Level 5": 0.1526, - "hfopenllm_v2/GPQA": 0.2802, - "hfopenllm_v2/MUSR": 0.3619, - "hfopenllm_v2/MMLU-PRO": 0.3543 - } - }, - { - "id": "EpistemeAI/Fireball-Alpaca-Llama3.1.08-8B-Philos-C-R2", - "name": "Fireball-Alpaca-Llama3.1.08-8B-Philos-C-R2", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4673, - "hfopenllm_v2/BBH": 0.4932, - "hfopenllm_v2/MATH Level 5": 0.1239, - "hfopenllm_v2/GPQA": 0.2861, - "hfopenllm_v2/MUSR": 0.4624, - "hfopenllm_v2/MMLU-PRO": 0.3352 - } - }, - { - "id": "EpistemeAI/Fireball-Meta-Llama-3.1-8B-Instruct-0.001-128K-auto", - "name": "Fireball-Meta-Llama-3.1-8B-Instruct-0.001-128K-auto", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4432, - "hfopenllm_v2/BBH": 0.4824, - "hfopenllm_v2/MATH Level 5": 0.1329, - "hfopenllm_v2/GPQA": 0.3121, - "hfopenllm_v2/MUSR": 0.4066, - "hfopenllm_v2/MMLU-PRO": 0.3516 - } - }, - { - "id": "EpistemeAI/Fireball-Meta-Llama-3.1-8B-Instruct-Agent-0.003-128K", - "name": "Fireball-Meta-Llama-3.1-8B-Instruct-Agent-0.003-128K", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4457, - "hfopenllm_v2/BBH": 0.4897, - "hfopenllm_v2/MATH Level 5": 0.1208, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.3762, - "hfopenllm_v2/MMLU-PRO": 0.3543 - } - }, - { - "id": "EpistemeAI/Fireball-Meta-Llama-3.1-8B-Instruct-Agent-0.003-128K-code", - "name": "Fireball-Meta-Llama-3.1-8B-Instruct-Agent-0.003-128K-code", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5975, - "hfopenllm_v2/BBH": 0.4904, - "hfopenllm_v2/MATH Level 5": 0.1337, - "hfopenllm_v2/GPQA": 0.302, - "hfopenllm_v2/MUSR": 0.401, - "hfopenllm_v2/MMLU-PRO": 0.3423 - } - }, - { - "id": "EpistemeAI/Fireball-Meta-Llama-3.1-8B-Instruct-Agent-0.003-128K-code-ds", - "name": "Fireball-Meta-Llama-3.1-8B-Instruct-Agent-0.003-128K-code-ds", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6691, - "hfopenllm_v2/BBH": 0.4668, - "hfopenllm_v2/MATH Level 5": 0.1337, - "hfopenllm_v2/GPQA": 0.2727, - "hfopenllm_v2/MUSR": 0.3418, - "hfopenllm_v2/MMLU-PRO": 0.3389 - } - }, - { - "id": "EpistemeAI/Fireball-Meta-Llama-3.1-8B-Instruct-Agent-0.003-128K-code-ds-auto", - "name": "Fireball-Meta-Llama-3.1-8B-Instruct-Agent-0.003-128K-code-ds-auto", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7207, - "hfopenllm_v2/BBH": 0.461, - "hfopenllm_v2/MATH Level 5": 0.1314, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.3432, - "hfopenllm_v2/MMLU-PRO": 0.3354 - } - }, - { - "id": "EpistemeAI/Fireball-Meta-Llama-3.1-8B-Instruct-Agent-0.004-128K-code-COT", - "name": "Fireball-Meta-Llama-3.1-8B-Instruct-Agent-0.004-128K-code-COT", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4578, - "hfopenllm_v2/BBH": 0.4761, - "hfopenllm_v2/MATH Level 5": 0.1382, - "hfopenllm_v2/GPQA": 0.2936, - "hfopenllm_v2/MUSR": 0.3881, - "hfopenllm_v2/MMLU-PRO": 0.3471 - } - }, - { - "id": "EpistemeAI/Fireball-Meta-Llama-3.1-8B-Instruct-Agent-0.004-128K-code-ds-auto", - "name": "Fireball-Meta-Llama-3.1-8B-Instruct-Agent-0.004-128K-code-ds-auto", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7205, - "hfopenllm_v2/BBH": 0.4818, - "hfopenllm_v2/MATH Level 5": 0.1435, - "hfopenllm_v2/GPQA": 0.2483, - "hfopenllm_v2/MUSR": 0.33, - "hfopenllm_v2/MMLU-PRO": 0.3548 - } - }, - { - "id": "EpistemeAI/Fireball-Meta-Llama-3.1-8B-Instruct-Math", - "name": "Fireball-Meta-Llama-3.1-8B-Instruct-Math", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4623, - "hfopenllm_v2/BBH": 0.4983, - "hfopenllm_v2/MATH Level 5": 0.108, - "hfopenllm_v2/GPQA": 0.2911, - "hfopenllm_v2/MUSR": 0.3641, - "hfopenllm_v2/MMLU-PRO": 0.3331 - } - }, - { - "id": "EpistemeAI/Fireball-Meta-Llama-3.2-8B-Instruct-agent-003-128k-code-DPO", - "name": "Fireball-Meta-Llama-3.2-8B-Instruct-agent-003-128k-code-DPO", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4611, - "hfopenllm_v2/BBH": 0.4801, - "hfopenllm_v2/MATH Level 5": 0.1254, - "hfopenllm_v2/GPQA": 0.3003, - "hfopenllm_v2/MUSR": 0.3998, - "hfopenllm_v2/MMLU-PRO": 0.3521 - } - }, - { - "id": "EpistemeAI/Fireball-Mistral-Nemo-Base-2407-v1-DPO2", - "name": "Fireball-Mistral-Nemo-Base-2407-v1-DPO2", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1861, - "hfopenllm_v2/BBH": 0.4968, - "hfopenllm_v2/MATH Level 5": 0.0363, - "hfopenllm_v2/GPQA": 0.2919, - "hfopenllm_v2/MUSR": 0.404, - "hfopenllm_v2/MMLU-PRO": 0.3353 - } - }, - { - "id": "EpistemeAI/Fireball-R1-Llama-3.1-8B", - "name": "Fireball-R1-Llama-3.1-8B", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4427, - "hfopenllm_v2/BBH": 0.3643, - "hfopenllm_v2/MATH Level 5": 0.3112, - "hfopenllm_v2/GPQA": 0.2483, - "hfopenllm_v2/MUSR": 0.3288, - "hfopenllm_v2/MMLU-PRO": 0.1115 - } - }, - { - "id": "EpistemeAI/Fireball-R1-Llama-3.1-8B-Medical-COT", - "name": "Fireball-R1-Llama-3.1-8B-Medical-COT", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3216, - "hfopenllm_v2/BBH": 0.3716, - "hfopenllm_v2/MATH Level 5": 0.327, - "hfopenllm_v2/GPQA": 0.2743, - "hfopenllm_v2/MUSR": 0.3114, - "hfopenllm_v2/MMLU-PRO": 0.1402 - } - }, - { - "id": "EpistemeAI/Fireball-R1.1-Llama-3.1-8B", - "name": "Fireball-R1.1-Llama-3.1-8B", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3676, - "hfopenllm_v2/BBH": 0.3326, - "hfopenllm_v2/MATH Level 5": 0.1382, - "hfopenllm_v2/GPQA": 0.2517, - "hfopenllm_v2/MUSR": 0.3419, - "hfopenllm_v2/MMLU-PRO": 0.1115 - } - }, - { - "id": "EpistemeAI/Llama-3.2-3B-Agent007-Coder", - "name": "Llama-3.2-3B-Agent007-Coder", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.54, - "hfopenllm_v2/BBH": 0.4304, - "hfopenllm_v2/MATH Level 5": 0.111, - "hfopenllm_v2/GPQA": 0.2576, - "hfopenllm_v2/MUSR": 0.3668, - "hfopenllm_v2/MMLU-PRO": 0.2852 - } - }, - { - "id": "EpistemeAI/Mistral-Nemo-Instruct-12B-Philosophy-Math", - "name": "Mistral-Nemo-Instruct-12B-Philosophy-Math", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0695, - "hfopenllm_v2/BBH": 0.5365, - "hfopenllm_v2/MATH Level 5": 0.0959, - "hfopenllm_v2/GPQA": 0.3314, - "hfopenllm_v2/MUSR": 0.4292, - "hfopenllm_v2/MMLU-PRO": 0.3296 - } - }, - { - "id": "EpistemeAI/OpenReasoner-Llama-3.2-3B-rs1.0", - "name": "OpenReasoner-Llama-3.2-3B-rs1.0", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7274, - "hfopenllm_v2/BBH": 0.4519, - "hfopenllm_v2/MATH Level 5": 0.1344, - "hfopenllm_v2/GPQA": 0.2718, - "hfopenllm_v2/MUSR": 0.3461, - "hfopenllm_v2/MMLU-PRO": 0.3134 - } - }, - { - "id": "EpistemeAI/Polypsyche-Llama-3.1-8B-Instruct-Agent-0.003-128K-code-ds-auto-divergent", - "name": "Polypsyche-Llama-3.1-8B-Instruct-Agent-0.003-128K-code-ds-auto-divergent", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6915, - "hfopenllm_v2/BBH": 0.4525, - "hfopenllm_v2/MATH Level 5": 0.1292, - "hfopenllm_v2/GPQA": 0.2668, - "hfopenllm_v2/MUSR": 0.3578, - "hfopenllm_v2/MMLU-PRO": 0.329 - } - }, - { - "id": "EpistemeAI/Polypsyche-Llama-3.1-8B-Instruct-Agent-0.003-128K-code-ds-auto-Empathy", - "name": "Polypsyche-Llama-3.1-8B-Instruct-Agent-0.003-128K-code-ds-auto-Empathy", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7101, - "hfopenllm_v2/BBH": 0.4628, - "hfopenllm_v2/MATH Level 5": 0.1397, - "hfopenllm_v2/GPQA": 0.2768, - "hfopenllm_v2/MUSR": 0.3195, - "hfopenllm_v2/MMLU-PRO": 0.3311 - } - }, - { - "id": "EpistemeAI/Polypsyche-Llama-3.1-8B-Instruct-Agent-0.003-128K-code-ds-auto-Logic", - "name": "Polypsyche-Llama-3.1-8B-Instruct-Agent-0.003-128K-code-ds-auto-Logic", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7122, - "hfopenllm_v2/BBH": 0.4566, - "hfopenllm_v2/MATH Level 5": 0.1246, - "hfopenllm_v2/GPQA": 0.2844, - "hfopenllm_v2/MUSR": 0.3235, - "hfopenllm_v2/MMLU-PRO": 0.335 - } - }, - { - "id": "EpistemeAI/Reasoning-Llama-3.1-CoT-RE1-NMT", - "name": "Reasoning-Llama-3.1-CoT-RE1-NMT", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4829, - "hfopenllm_v2/BBH": 0.4736, - "hfopenllm_v2/MATH Level 5": 0.1299, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.3182, - "hfopenllm_v2/MMLU-PRO": 0.3343 - } - }, - { - "id": "EpistemeAI/Reasoning-Llama-3.1-CoT-RE1-NMT-V2-ORPO", - "name": "Reasoning-Llama-3.1-CoT-RE1-NMT-V2-ORPO", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4553, - "hfopenllm_v2/BBH": 0.4804, - "hfopenllm_v2/MATH Level 5": 0.1292, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.3931, - "hfopenllm_v2/MMLU-PRO": 0.3598 - } - }, - { - "id": "EpistemeAI/Reasoning-Llama-3.2-1B-Instruct-v1.2", - "name": "Reasoning-Llama-3.2-1B-Instruct-v1.2", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4087, - "hfopenllm_v2/BBH": 0.3324, - "hfopenllm_v2/MATH Level 5": 0.0506, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.3222, - "hfopenllm_v2/MMLU-PRO": 0.1179 - } - }, - { - "id": "EpistemeAI/Reasoning-Llama-3.2-1B-Instruct-v1.3", - "name": "Reasoning-Llama-3.2-1B-Instruct-v1.3", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3273, - "hfopenllm_v2/BBH": 0.3263, - "hfopenllm_v2/MATH Level 5": 0.0506, - "hfopenllm_v2/GPQA": 0.2584, - "hfopenllm_v2/MUSR": 0.326, - "hfopenllm_v2/MMLU-PRO": 0.1173 - } - }, - { - "id": "EpistemeAI/Reasoning-Llama-3.2-3B-Math-Instruct-RE1", - "name": "Reasoning-Llama-3.2-3B-Math-Instruct-RE1", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.512, - "hfopenllm_v2/BBH": 0.4381, - "hfopenllm_v2/MATH Level 5": 0.108, - "hfopenllm_v2/GPQA": 0.2643, - "hfopenllm_v2/MUSR": 0.3435, - "hfopenllm_v2/MMLU-PRO": 0.2789 - } - }, - { - "id": "EpistemeAI/Reasoning-Llama-3.2-3B-Math-Instruct-RE1-ORPO", - "name": "Reasoning-Llama-3.2-3B-Math-Instruct-RE1-ORPO", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.729, - "hfopenllm_v2/BBH": 0.4518, - "hfopenllm_v2/MATH Level 5": 0.1533, - "hfopenllm_v2/GPQA": 0.2735, - "hfopenllm_v2/MUSR": 0.3487, - "hfopenllm_v2/MMLU-PRO": 0.31 - } - }, - { - "id": "EpistemeAI/ReasoningCore-1.0-3B-Instruct-r01-Reflect-Math", - "name": "ReasoningCore-1.0-3B-Instruct-r01-Reflect-Math", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5903, - "hfopenllm_v2/BBH": 0.4364, - "hfopenllm_v2/MATH Level 5": 0.148, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3314, - "hfopenllm_v2/MMLU-PRO": 0.2823 - } - }, - { - "id": "EpistemeAI/ReasoningCore-3B-0", - "name": "ReasoningCore-3B-0", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7341, - "hfopenllm_v2/BBH": 0.4446, - "hfopenllm_v2/MATH Level 5": 0.1586, - "hfopenllm_v2/GPQA": 0.2727, - "hfopenllm_v2/MUSR": 0.3554, - "hfopenllm_v2/MMLU-PRO": 0.3172 - } - }, - { - "id": "EpistemeAI/ReasoningCore-3B-Instruct-r01-Reflect", - "name": "ReasoningCore-3B-Instruct-r01-Reflect", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7335, - "hfopenllm_v2/BBH": 0.445, - "hfopenllm_v2/MATH Level 5": 0.1541, - "hfopenllm_v2/GPQA": 0.2735, - "hfopenllm_v2/MUSR": 0.3527, - "hfopenllm_v2/MMLU-PRO": 0.3144 - } - }, - { - "id": "EpistemeAI/ReasoningCore-3B-R01", - "name": "ReasoningCore-3B-R01", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2976, - "hfopenllm_v2/BBH": 0.4373, - "hfopenllm_v2/MATH Level 5": 0.1299, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.3195, - "hfopenllm_v2/MMLU-PRO": 0.2591 - } - }, - { - "id": "EpistemeAI/ReasoningCore-3B-RE1-V2", - "name": "ReasoningCore-3B-RE1-V2", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7393, - "hfopenllm_v2/BBH": 0.4462, - "hfopenllm_v2/MATH Level 5": 0.1563, - "hfopenllm_v2/GPQA": 0.2735, - "hfopenllm_v2/MUSR": 0.3541, - "hfopenllm_v2/MMLU-PRO": 0.3181 - } - }, - { - "id": "EpistemeAI/ReasoningCore-3B-RE1-V2A", - "name": "ReasoningCore-3B-RE1-V2A", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5733, - "hfopenllm_v2/BBH": 0.419, - "hfopenllm_v2/MATH Level 5": 0.0929, - "hfopenllm_v2/GPQA": 0.2777, - "hfopenllm_v2/MUSR": 0.3352, - "hfopenllm_v2/MMLU-PRO": 0.2736 - } - }, - { - "id": "EpistemeAI/ReasoningCore-3B-RE1-V2B", - "name": "ReasoningCore-3B-RE1-V2B", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5051, - "hfopenllm_v2/BBH": 0.4168, - "hfopenllm_v2/MATH Level 5": 0.1073, - "hfopenllm_v2/GPQA": 0.2617, - "hfopenllm_v2/MUSR": 0.3448, - "hfopenllm_v2/MMLU-PRO": 0.2673 - } - }, - { - "id": "EpistemeAI/ReasoningCore-3B-RE1-V2C", - "name": "ReasoningCore-3B-RE1-V2C", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5057, - "hfopenllm_v2/BBH": 0.4177, - "hfopenllm_v2/MATH Level 5": 0.0974, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.3422, - "hfopenllm_v2/MMLU-PRO": 0.2691 - } - }, - { - "id": "EpistemeAI/ReasoningCore-3B-T1-V1", - "name": "ReasoningCore-3B-T1-V1", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7208, - "hfopenllm_v2/BBH": 0.4517, - "hfopenllm_v2/MATH Level 5": 0.1458, - "hfopenllm_v2/GPQA": 0.276, - "hfopenllm_v2/MUSR": 0.354, - "hfopenllm_v2/MMLU-PRO": 0.312 - } - }, - { - "id": "EpistemeAI/ReasoningCore-3B-T1_1", - "name": "ReasoningCore-3B-T1_1", - "developer": "EpistemeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7275, - "hfopenllm_v2/BBH": 0.4524, - "hfopenllm_v2/MATH Level 5": 0.1541, - "hfopenllm_v2/GPQA": 0.276, - "hfopenllm_v2/MUSR": 0.3554, - "hfopenllm_v2/MMLU-PRO": 0.3117 - } - }, - { - "id": "EpistemeAI2/Athene-codegemma-2-7b-it-alpaca-v1.2", - "name": "Athene-codegemma-2-7b-it-alpaca-v1.2", - "developer": "EpistemeAI2", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4351, - "hfopenllm_v2/BBH": 0.4175, - "hfopenllm_v2/MATH Level 5": 0.0423, - "hfopenllm_v2/GPQA": 0.271, - "hfopenllm_v2/MUSR": 0.417, - "hfopenllm_v2/MMLU-PRO": 0.2297 - } - }, - { - "id": "EpistemeAI2/Fireball-12B-v1.2", - "name": "Fireball-12B-v1.2", - "developer": "EpistemeAI2", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1355, - "hfopenllm_v2/BBH": 0.5019, - "hfopenllm_v2/MATH Level 5": 0.0415, - "hfopenllm_v2/GPQA": 0.2987, - "hfopenllm_v2/MUSR": 0.4173, - "hfopenllm_v2/MMLU-PRO": 0.3337 - } - }, - { - "id": "EpistemeAI2/Fireball-Alpaca-Llama3.1-8B-Philos", - "name": "Fireball-Alpaca-Llama3.1-8B-Philos", - "developer": "EpistemeAI2", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4986, - "hfopenllm_v2/BBH": 0.4978, - "hfopenllm_v2/MATH Level 5": 0.1186, - "hfopenllm_v2/GPQA": 0.2928, - "hfopenllm_v2/MUSR": 0.4277, - "hfopenllm_v2/MMLU-PRO": 0.3406 - } - }, - { - "id": "EpistemeAI2/Fireball-Alpaca-Llama3.1.01-8B-Philos", - "name": "Fireball-Alpaca-Llama3.1.01-8B-Philos", - "developer": "EpistemeAI2", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4212, - "hfopenllm_v2/BBH": 0.4956, - "hfopenllm_v2/MATH Level 5": 0.136, - "hfopenllm_v2/GPQA": 0.2886, - "hfopenllm_v2/MUSR": 0.4371, - "hfopenllm_v2/MMLU-PRO": 0.3383 - } - }, - { - "id": "EpistemeAI2/Fireball-Alpaca-Llama3.1.03-8B-Philos", - "name": "Fireball-Alpaca-Llama3.1.03-8B-Philos", - "developer": "EpistemeAI2", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3881, - "hfopenllm_v2/BBH": 0.4951, - "hfopenllm_v2/MATH Level 5": 0.1284, - "hfopenllm_v2/GPQA": 0.2785, - "hfopenllm_v2/MUSR": 0.428, - "hfopenllm_v2/MMLU-PRO": 0.3355 - } - }, - { - "id": "EpistemeAI2/Fireball-Alpaca-Llama3.1.04-8B-Philos", - "name": "Fireball-Alpaca-Llama3.1.04-8B-Philos", - "developer": "EpistemeAI2", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4084, - "hfopenllm_v2/BBH": 0.493, - "hfopenllm_v2/MATH Level 5": 0.1201, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.4372, - "hfopenllm_v2/MMLU-PRO": 0.3403 - } - }, - { - "id": "EpistemeAI2/Fireball-Alpaca-Llama3.1.06-8B-Philos-dpo", - "name": "Fireball-Alpaca-Llama3.1.06-8B-Philos-dpo", - "developer": "EpistemeAI2", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4866, - "hfopenllm_v2/BBH": 0.4881, - "hfopenllm_v2/MATH Level 5": 0.1307, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.3932, - "hfopenllm_v2/MMLU-PRO": 0.3615 - } - }, - { - "id": "EpistemeAI2/Fireball-Alpaca-Llama3.1.07-8B-Philos-Math", - "name": "Fireball-Alpaca-Llama3.1.07-8B-Philos-Math", - "developer": "EpistemeAI2", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5079, - "hfopenllm_v2/BBH": 0.4847, - "hfopenllm_v2/MATH Level 5": 0.1201, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.4063, - "hfopenllm_v2/MMLU-PRO": 0.3531 - } - }, - { - "id": "EpistemeAI2/Fireball-Alpaca-Llama3.1.08-8B-C-R1-KTO-Reflection", - "name": "Fireball-Alpaca-Llama3.1.08-8B-C-R1-KTO-Reflection", - "developer": "EpistemeAI2", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3952, - "hfopenllm_v2/BBH": 0.4955, - "hfopenllm_v2/MATH Level 5": 0.1246, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.4048, - "hfopenllm_v2/MMLU-PRO": 0.3593 - } - }, - { - "id": "EpistemeAI2/Fireball-Alpaca-Llama3.1.08-8B-Philos-C-R1", - "name": "Fireball-Alpaca-Llama3.1.08-8B-Philos-C-R1", - "developer": "EpistemeAI2", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5316, - "hfopenllm_v2/BBH": 0.4828, - "hfopenllm_v2/MATH Level 5": 0.1239, - "hfopenllm_v2/GPQA": 0.297, - "hfopenllm_v2/MUSR": 0.4103, - "hfopenllm_v2/MMLU-PRO": 0.3523 - } - }, - { - "id": "EpistemeAI2/Fireball-Llama-3.1-8B-Philos-Reflection", - "name": "Fireball-Llama-3.1-8B-Philos-Reflection", - "developer": "EpistemeAI2", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3596, - "hfopenllm_v2/BBH": 0.4898, - "hfopenllm_v2/MATH Level 5": 0.1284, - "hfopenllm_v2/GPQA": 0.3079, - "hfopenllm_v2/MUSR": 0.3957, - "hfopenllm_v2/MMLU-PRO": 0.3551 - } - }, - { - "id": "EpistemeAI2/Fireball-MathMistral-Nemo-Base-2407-v2dpo", - "name": "Fireball-MathMistral-Nemo-Base-2407-v2dpo", - "developer": "EpistemeAI2", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3097, - "hfopenllm_v2/BBH": 0.4328, - "hfopenllm_v2/MATH Level 5": 0.037, - "hfopenllm_v2/GPQA": 0.2634, - "hfopenllm_v2/MUSR": 0.403, - "hfopenllm_v2/MMLU-PRO": 0.1148 - } - }, - { - "id": "EpistemeAI2/Fireball-Meta-Llama-3.1-8B-Instruct-Agent-0.003-128K-code-math", - "name": "Fireball-Meta-Llama-3.1-8B-Instruct-Agent-0.003-128K-code-math", - "developer": "EpistemeAI2", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5515, - "hfopenllm_v2/BBH": 0.4808, - "hfopenllm_v2/MATH Level 5": 0.1352, - "hfopenllm_v2/GPQA": 0.3045, - "hfopenllm_v2/MUSR": 0.3693, - "hfopenllm_v2/MMLU-PRO": 0.342 - } - }, - { - "id": "EpistemeAI2/Fireball-Meta-Llama-3.1-8B-Instruct-Agent-0.005-128K-code-COT", - "name": "Fireball-Meta-Llama-3.1-8B-Instruct-Agent-0.005-128K-code-COT", - "developer": "EpistemeAI2", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4633, - "hfopenllm_v2/BBH": 0.4791, - "hfopenllm_v2/MATH Level 5": 0.1171, - "hfopenllm_v2/GPQA": 0.3121, - "hfopenllm_v2/MUSR": 0.3774, - "hfopenllm_v2/MMLU-PRO": 0.3565 - } - }, - { - "id": "EpistemeAI2/Fireball-Phi-3-medium-4k-inst-Philos", - "name": "Fireball-Phi-3-medium-4k-inst-Philos", - "developer": "EpistemeAI2", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5313, - "hfopenllm_v2/BBH": 0.6178, - "hfopenllm_v2/MATH Level 5": 0.1707, - "hfopenllm_v2/GPQA": 0.3322, - "hfopenllm_v2/MUSR": 0.4139, - "hfopenllm_v2/MMLU-PRO": 0.4599 - } - }, - { - "id": "Eric111/CatunaMayo", - "name": "CatunaMayo", - "developer": "Eric111", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4074, - "hfopenllm_v2/BBH": 0.5244, - "hfopenllm_v2/MATH Level 5": 0.0846, - "hfopenllm_v2/GPQA": 0.2919, - "hfopenllm_v2/MUSR": 0.454, - "hfopenllm_v2/MMLU-PRO": 0.3178 - } - }, - { - "id": "Eric111/CatunaMayo-DPO", - "name": "CatunaMayo-DPO", - "developer": "Eric111", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4215, - "hfopenllm_v2/BBH": 0.5224, - "hfopenllm_v2/MATH Level 5": 0.0816, - "hfopenllm_v2/GPQA": 0.2919, - "hfopenllm_v2/MUSR": 0.445, - "hfopenllm_v2/MMLU-PRO": 0.317 - } - }, - { - "id": "Etherll/Chocolatine-3B-Instruct-DPO-Revised-Ties", - "name": "Chocolatine-3B-Instruct-DPO-Revised-Ties", - "developer": "Etherll", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3725, - "hfopenllm_v2/BBH": 0.5411, - "hfopenllm_v2/MATH Level 5": 0.1631, - "hfopenllm_v2/GPQA": 0.3238, - "hfopenllm_v2/MUSR": 0.4649, - "hfopenllm_v2/MMLU-PRO": 0.3978 - } - }, - { - "id": "Etherll/Chocolatine-3B-Instruct-DPO-Revised-Ties-v2", - "name": "Chocolatine-3B-Instruct-DPO-Revised-Ties-v2", - "developer": "Etherll", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.374, - "hfopenllm_v2/BBH": 0.5411, - "hfopenllm_v2/MATH Level 5": 0.1631, - "hfopenllm_v2/GPQA": 0.3238, - "hfopenllm_v2/MUSR": 0.4649, - "hfopenllm_v2/MMLU-PRO": 0.3978 - } - }, - { - "id": "Etherll/Herplete-LLM-Llama-3.1-8b", - "name": "Herplete-LLM-Llama-3.1-8b", - "developer": "Etherll", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4672, - "hfopenllm_v2/BBH": 0.5013, - "hfopenllm_v2/MATH Level 5": 0.0279, - "hfopenllm_v2/GPQA": 0.2861, - "hfopenllm_v2/MUSR": 0.386, - "hfopenllm_v2/MMLU-PRO": 0.3482 - } - }, - { - "id": "Etherll/Herplete-LLM-Llama-3.1-8b-Ties", - "name": "Herplete-LLM-Llama-3.1-8b-Ties", - "developer": "Etherll", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6164, - "hfopenllm_v2/BBH": 0.5338, - "hfopenllm_v2/MATH Level 5": 0.1601, - "hfopenllm_v2/GPQA": 0.3171, - "hfopenllm_v2/MUSR": 0.4017, - "hfopenllm_v2/MMLU-PRO": 0.3752 - } - }, - { - "id": "Etherll/Qwen2.5-7B-della-test", - "name": "Qwen2.5-7B-della-test", - "developer": "Etherll", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7625, - "hfopenllm_v2/BBH": 0.5447, - "hfopenllm_v2/MATH Level 5": 0.4894, - "hfopenllm_v2/GPQA": 0.3087, - "hfopenllm_v2/MUSR": 0.4047, - "hfopenllm_v2/MMLU-PRO": 0.4361 - } - }, - { - "id": "Etherll/Qwen2.5-Coder-7B-Instruct-Ties", - "name": "Qwen2.5-Coder-7B-Instruct-Ties", - "developer": "Etherll", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5005, - "hfopenllm_v2/BBH": 0.4895, - "hfopenllm_v2/MATH Level 5": 0.2915, - "hfopenllm_v2/GPQA": 0.3297, - "hfopenllm_v2/MUSR": 0.4373, - "hfopenllm_v2/MMLU-PRO": 0.3503 - } - }, - { - "id": "Etherll/Replete-LLM-V3-Llama-3.1-8b", - "name": "Replete-LLM-V3-Llama-3.1-8b", - "developer": "Etherll", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5263, - "hfopenllm_v2/BBH": 0.4543, - "hfopenllm_v2/MATH Level 5": 0.2273, - "hfopenllm_v2/GPQA": 0.2685, - "hfopenllm_v2/MUSR": 0.3516, - "hfopenllm_v2/MMLU-PRO": 0.347 - } - }, - { - "id": "Etherll/SuperHermes", - "name": "SuperHermes", - "developer": "Etherll", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5459, - "hfopenllm_v2/BBH": 0.529, - "hfopenllm_v2/MATH Level 5": 0.1654, - "hfopenllm_v2/GPQA": 0.3238, - "hfopenllm_v2/MUSR": 0.44, - "hfopenllm_v2/MMLU-PRO": 0.3949 - } - }, - { - "id": "euclaise/ReMask-3B", - "name": "ReMask-3B", - "developer": "euclaise", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2419, - "hfopenllm_v2/BBH": 0.3517, - "hfopenllm_v2/MATH Level 5": 0.0196, - "hfopenllm_v2/GPQA": 0.2668, - "hfopenllm_v2/MUSR": 0.3341, - "hfopenllm_v2/MMLU-PRO": 0.1357 - } - }, - { - "id": "Eurdem/Defne-llama3.1-8B", - "name": "Defne-llama3.1-8B", - "developer": "Eurdem", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5036, - "hfopenllm_v2/BBH": 0.5321, - "hfopenllm_v2/MATH Level 5": 0.1601, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.4331, - "hfopenllm_v2/MMLU-PRO": 0.3866 - } - }, - { - "id": "EVA-UNIT-01/EVA-Qwen2.5-14B-v0.2", - "name": "EVA-Qwen2.5-14B-v0.2", - "developer": "EVA-UNIT-01", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4038, - "hfopenllm_v2/BBH": 0.609, - "hfopenllm_v2/MATH Level 5": 0.3406, - "hfopenllm_v2/GPQA": 0.3943, - "hfopenllm_v2/MUSR": 0.4794, - "hfopenllm_v2/MMLU-PRO": 0.5135 - } - }, - { - "id": "EVA-UNIT-01/EVA-Qwen2.5-72B-v0.2", - "name": "EVA-Qwen2.5-72B-v0.2", - "developer": "EVA-UNIT-01", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6879, - "hfopenllm_v2/BBH": 0.7088, - "hfopenllm_v2/MATH Level 5": 0.4313, - "hfopenllm_v2/GPQA": 0.4086, - "hfopenllm_v2/MUSR": 0.472, - "hfopenllm_v2/MMLU-PRO": 0.5813 - } - }, - { - "id": "eworojoshua/vas-01", - "name": "vas-01", - "developer": "eworojoshua", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7612, - "hfopenllm_v2/BBH": 0.5418, - "hfopenllm_v2/MATH Level 5": 0.4736, - "hfopenllm_v2/GPQA": 0.3096, - "hfopenllm_v2/MUSR": 0.4432, - "hfopenllm_v2/MMLU-PRO": 0.4348 - } - }, - { - "id": "ewre324/ewre324-R1-SmolLM2-135M-Distill", - "name": "ewre324-R1-SmolLM2-135M-Distill", - "developer": "ewre324", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1649, - "hfopenllm_v2/BBH": 0.3042, - "hfopenllm_v2/MATH Level 5": 0.0128, - "hfopenllm_v2/GPQA": 0.2617, - "hfopenllm_v2/MUSR": 0.3409, - "hfopenllm_v2/MMLU-PRO": 0.1134 - } - }, - { - "id": "ewre324/Thinker-Llama-3.2-3B-Instruct-Reasoning", - "name": "Thinker-Llama-3.2-3B-Instruct-Reasoning", - "developer": "ewre324", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4439, - "hfopenllm_v2/BBH": 0.4273, - "hfopenllm_v2/MATH Level 5": 0.0846, - "hfopenllm_v2/GPQA": 0.2768, - "hfopenllm_v2/MUSR": 0.3655, - "hfopenllm_v2/MMLU-PRO": 0.2886 - } - }, - { - "id": "ewre324/Thinker-Qwen2.5-0.5B-Instruct-Reasoning", - "name": "Thinker-Qwen2.5-0.5B-Instruct-Reasoning", - "developer": "ewre324", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2476, - "hfopenllm_v2/BBH": 0.3292, - "hfopenllm_v2/MATH Level 5": 0.0287, - "hfopenllm_v2/GPQA": 0.2852, - "hfopenllm_v2/MUSR": 0.3382, - "hfopenllm_v2/MMLU-PRO": 0.1647 - } - }, - { - "id": "ewre324/Thinker-SmolLM2-135M-Instruct-Reasoning", - "name": "Thinker-SmolLM2-135M-Instruct-Reasoning", - "developer": "ewre324", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2584, - "hfopenllm_v2/BBH": 0.3071, - "hfopenllm_v2/MATH Level 5": 0.0091, - "hfopenllm_v2/GPQA": 0.2525, - "hfopenllm_v2/MUSR": 0.3661, - "hfopenllm_v2/MMLU-PRO": 0.1094 - } - }, - { - "id": "experiment-llm/exp-3-q-r", - "name": "exp-3-q-r", - "developer": "experiment-llm", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6036, - "hfopenllm_v2/BBH": 0.5397, - "hfopenllm_v2/MATH Level 5": 0.2787, - "hfopenllm_v2/GPQA": 0.2936, - "hfopenllm_v2/MUSR": 0.4315, - "hfopenllm_v2/MMLU-PRO": 0.4316 - } - }, - { - "id": "facebook/opt-1.3b", - "name": "opt-1.3b", - "developer": "facebook", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2383, - "hfopenllm_v2/BBH": 0.3094, - "hfopenllm_v2/MATH Level 5": 0.0091, - "hfopenllm_v2/GPQA": 0.2424, - "hfopenllm_v2/MUSR": 0.342, - "hfopenllm_v2/MMLU-PRO": 0.1107 - } - }, - { - "id": "facebook/opt-30b", - "name": "opt-30b", - "developer": "facebook", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2453, - "hfopenllm_v2/BBH": 0.307, - "hfopenllm_v2/MATH Level 5": 0.0106, - "hfopenllm_v2/GPQA": 0.2693, - "hfopenllm_v2/MUSR": 0.3604, - "hfopenllm_v2/MMLU-PRO": 0.1164 - } - }, - { - "id": "facebook/Self-taught-evaluator-llama3.1-70B", - "name": "facebook/Self-taught-evaluator-llama3.1-70B", - "developer": "facebook", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.9001, - "reward-bench/Chat": 0.9693, - "reward-bench/Chat Hard": 0.8509, - "reward-bench/Safety": 0.8959, - "reward-bench/Reasoning": 0.8844 - } - }, - { - "id": "facebook/Self-taught-Llama-3-70B", - "name": "facebook/Self-taught-Llama-3-70B", - "developer": "facebook", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.8863, - "reward-bench/Chat": 0.9693, - "reward-bench/Chat Hard": 0.8399, - "reward-bench/Safety": 0.9108, - "reward-bench/Reasoning": 0.8251 - } - }, - { - "id": "failspy/llama-3-70B-Instruct-abliterated", - "name": "llama-3-70B-Instruct-abliterated", - "developer": "failspy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8023, - "hfopenllm_v2/BBH": 0.6465, - "hfopenllm_v2/MATH Level 5": 0.2432, - "hfopenllm_v2/GPQA": 0.2894, - "hfopenllm_v2/MUSR": 0.4128, - "hfopenllm_v2/MMLU-PRO": 0.5145 - } - }, - { - "id": "failspy/Llama-3-8B-Instruct-abliterated", - "name": "Llama-3-8B-Instruct-abliterated", - "developer": "failspy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5909, - "hfopenllm_v2/BBH": 0.4354, - "hfopenllm_v2/MATH Level 5": 0.0385, - "hfopenllm_v2/GPQA": 0.276, - "hfopenllm_v2/MUSR": 0.4116, - "hfopenllm_v2/MMLU-PRO": 0.2742 - } - }, - { - "id": "failspy/Llama-3-8B-Instruct-MopeyMule", - "name": "Llama-3-8B-Instruct-MopeyMule", - "developer": "failspy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.675, - "hfopenllm_v2/BBH": 0.3839, - "hfopenllm_v2/MATH Level 5": 0.0196, - "hfopenllm_v2/GPQA": 0.2391, - "hfopenllm_v2/MUSR": 0.3513, - "hfopenllm_v2/MMLU-PRO": 0.1764 - } - }, - { - "id": "failspy/Meta-Llama-3-70B-Instruct-abliterated-v3.5", - "name": "Meta-Llama-3-70B-Instruct-abliterated-v3.5", - "developer": "failspy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7747, - "hfopenllm_v2/BBH": 0.5747, - "hfopenllm_v2/MATH Level 5": 0.1284, - "hfopenllm_v2/GPQA": 0.297, - "hfopenllm_v2/MUSR": 0.3982, - "hfopenllm_v2/MMLU-PRO": 0.4452 - } - }, - { - "id": "failspy/Meta-Llama-3-8B-Instruct-abliterated-v3", - "name": "Meta-Llama-3-8B-Instruct-abliterated-v3", - "developer": "failspy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7245, - "hfopenllm_v2/BBH": 0.4925, - "hfopenllm_v2/MATH Level 5": 0.0959, - "hfopenllm_v2/GPQA": 0.2643, - "hfopenllm_v2/MUSR": 0.3622, - "hfopenllm_v2/MMLU-PRO": 0.3654 - } - }, - { - "id": "failspy/Phi-3-medium-4k-instruct-abliterated-v3", - "name": "Phi-3-medium-4k-instruct-abliterated-v3", - "developer": "failspy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6319, - "hfopenllm_v2/BBH": 0.6305, - "hfopenllm_v2/MATH Level 5": 0.1594, - "hfopenllm_v2/GPQA": 0.3171, - "hfopenllm_v2/MUSR": 0.4604, - "hfopenllm_v2/MMLU-PRO": 0.44 - } - }, - { - "id": "FallenMerick/Chewy-Lemon-Cookie-11B", - "name": "Chewy-Lemon-Cookie-11B", - "developer": "FallenMerick", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4875, - "hfopenllm_v2/BBH": 0.5251, - "hfopenllm_v2/MATH Level 5": 0.0544, - "hfopenllm_v2/GPQA": 0.2794, - "hfopenllm_v2/MUSR": 0.4546, - "hfopenllm_v2/MMLU-PRO": 0.3267 - } - }, - { - "id": "fblgit/cybertron-v4-qw7B-MGS", - "name": "cybertron-v4-qw7B-MGS", - "developer": "fblgit", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6264, - "hfopenllm_v2/BBH": 0.5592, - "hfopenllm_v2/MATH Level 5": 0.3489, - "hfopenllm_v2/GPQA": 0.3104, - "hfopenllm_v2/MUSR": 0.4371, - "hfopenllm_v2/MMLU-PRO": 0.4473 - } - }, - { - "id": "fblgit/cybertron-v4-qw7B-UNAMGS", - "name": "cybertron-v4-qw7B-UNAMGS", - "developer": "fblgit", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.609, - "hfopenllm_v2/BBH": 0.5643, - "hfopenllm_v2/MATH Level 5": 0.3731, - "hfopenllm_v2/GPQA": 0.3314, - "hfopenllm_v2/MUSR": 0.4343, - "hfopenllm_v2/MMLU-PRO": 0.45 - } - }, - { - "id": "fblgit/juanako-7b-UNA", - "name": "juanako-7b-UNA", - "developer": "fblgit", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4837, - "hfopenllm_v2/BBH": 0.507, - "hfopenllm_v2/MATH Level 5": 0.034, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.4645, - "hfopenllm_v2/MMLU-PRO": 0.2771 - } - }, - { - "id": "fblgit/miniclaus-qw1.5B-UNAMGS", - "name": "miniclaus-qw1.5B-UNAMGS", - "developer": "fblgit", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3348, - "hfopenllm_v2/BBH": 0.4239, - "hfopenllm_v2/MATH Level 5": 0.1088, - "hfopenllm_v2/GPQA": 0.2919, - "hfopenllm_v2/MUSR": 0.4293, - "hfopenllm_v2/MMLU-PRO": 0.2937 - } - }, - { - "id": "fblgit/miniclaus-qw1.5B-UNAMGS-GRPO", - "name": "miniclaus-qw1.5B-UNAMGS-GRPO", - "developer": "fblgit", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3518, - "hfopenllm_v2/BBH": 0.4234, - "hfopenllm_v2/MATH Level 5": 0.1103, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.4254, - "hfopenllm_v2/MMLU-PRO": 0.2945 - } - }, - { - "id": "fblgit/pancho-v1-qw25-3B-UNAMGS", - "name": "pancho-v1-qw25-3B-UNAMGS", - "developer": "fblgit", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5361, - "hfopenllm_v2/BBH": 0.4926, - "hfopenllm_v2/MATH Level 5": 0.1571, - "hfopenllm_v2/GPQA": 0.297, - "hfopenllm_v2/MUSR": 0.4027, - "hfopenllm_v2/MMLU-PRO": 0.3766 - } - }, - { - "id": "fblgit/TheBeagle-v2beta-32B-MGS", - "name": "TheBeagle-v2beta-32B-MGS", - "developer": "fblgit", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5181, - "hfopenllm_v2/BBH": 0.7033, - "hfopenllm_v2/MATH Level 5": 0.4947, - "hfopenllm_v2/GPQA": 0.3826, - "hfopenllm_v2/MUSR": 0.5008, - "hfopenllm_v2/MMLU-PRO": 0.5915 - } - }, - { - "id": "fblgit/una-cybertron-7b-v2-bf16", - "name": "una-cybertron-7b-v2-bf16", - "developer": "fblgit", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4737, - "hfopenllm_v2/BBH": 0.3973, - "hfopenllm_v2/MATH Level 5": 0.0408, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.4473, - "hfopenllm_v2/MMLU-PRO": 0.2443 - } - }, - { - "id": "fblgit/UNA-SimpleSmaug-34b-v1beta", - "name": "UNA-SimpleSmaug-34b-v1beta", - "developer": "fblgit", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4556, - "hfopenllm_v2/BBH": 0.5287, - "hfopenllm_v2/MATH Level 5": 0.0718, - "hfopenllm_v2/GPQA": 0.3171, - "hfopenllm_v2/MUSR": 0.4256, - "hfopenllm_v2/MMLU-PRO": 0.454 - } - }, - { - "id": "fblgit/UNA-TheBeagle-7b-v1", - "name": "UNA-TheBeagle-7b-v1", - "developer": "fblgit", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3689, - "hfopenllm_v2/BBH": 0.5029, - "hfopenllm_v2/MATH Level 5": 0.077, - "hfopenllm_v2/GPQA": 0.2844, - "hfopenllm_v2/MUSR": 0.4564, - "hfopenllm_v2/MMLU-PRO": 0.3019 - } - }, - { - "id": "fblgit/UNA-ThePitbull-21.4B-v2", - "name": "UNA-ThePitbull-21.4B-v2", - "developer": "fblgit", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.379, - "hfopenllm_v2/BBH": 0.635, - "hfopenllm_v2/MATH Level 5": 0.1216, - "hfopenllm_v2/GPQA": 0.302, - "hfopenllm_v2/MUSR": 0.3922, - "hfopenllm_v2/MMLU-PRO": 0.3516 - } - }, - { - "id": "Felladrin/Llama-160M-Chat-v1", - "name": "Llama-160M-Chat-v1", - "developer": "Felladrin", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1575, - "hfopenllm_v2/BBH": 0.3036, - "hfopenllm_v2/MATH Level 5": 0.006, - "hfopenllm_v2/GPQA": 0.2576, - "hfopenllm_v2/MUSR": 0.3661, - "hfopenllm_v2/MMLU-PRO": 0.1136 - } - }, - { - "id": "Felladrin/Minueza-32M-UltraChat", - "name": "Minueza-32M-UltraChat", - "developer": "Felladrin", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1376, - "hfopenllm_v2/BBH": 0.2941, - "hfopenllm_v2/MATH Level 5": 0.0045, - "hfopenllm_v2/GPQA": 0.2559, - "hfopenllm_v2/MUSR": 0.3742, - "hfopenllm_v2/MMLU-PRO": 0.1133 - } - }, - { - "id": "fhai50032/RolePlayLake-7B", - "name": "RolePlayLake-7B", - "developer": "fhai50032", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5057, - "hfopenllm_v2/BBH": 0.5252, - "hfopenllm_v2/MATH Level 5": 0.0725, - "hfopenllm_v2/GPQA": 0.3037, - "hfopenllm_v2/MUSR": 0.4459, - "hfopenllm_v2/MMLU-PRO": 0.316 - } - }, - { - "id": "fhai50032/Unaligned-Thinker-PHI-4", - "name": "Unaligned-Thinker-PHI-4", - "developer": "fhai50032", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0563, - "hfopenllm_v2/BBH": 0.6643, - "hfopenllm_v2/MATH Level 5": 0.3353, - "hfopenllm_v2/GPQA": 0.3809, - "hfopenllm_v2/MUSR": 0.4679, - "hfopenllm_v2/MMLU-PRO": 0.5147 - } - }, - { - "id": "FINGU-AI/Chocolatine-Fusion-14B", - "name": "Chocolatine-Fusion-14B", - "developer": "FINGU-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6949, - "hfopenllm_v2/BBH": 0.6413, - "hfopenllm_v2/MATH Level 5": 0.3852, - "hfopenllm_v2/GPQA": 0.3716, - "hfopenllm_v2/MUSR": 0.494, - "hfopenllm_v2/MMLU-PRO": 0.5262 - } - }, - { - "id": "FINGU-AI/L3-8B", - "name": "L3-8B", - "developer": "FINGU-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7517, - "hfopenllm_v2/BBH": 0.4986, - "hfopenllm_v2/MATH Level 5": 0.2545, - "hfopenllm_v2/GPQA": 0.2953, - "hfopenllm_v2/MUSR": 0.3828, - "hfopenllm_v2/MMLU-PRO": 0.3639 - } - }, - { - "id": "FINGU-AI/Phi-4-RRStock", - "name": "Phi-4-RRStock", - "developer": "FINGU-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2855, - "hfopenllm_v2/BBH": 0.6443, - "hfopenllm_v2/MATH Level 5": 0.0582, - "hfopenllm_v2/GPQA": 0.38, - "hfopenllm_v2/MUSR": 0.4479, - "hfopenllm_v2/MMLU-PRO": 0.4883 - } - }, - { - "id": "FINGU-AI/Q-Small-3B", - "name": "Q-Small-3B", - "developer": "FINGU-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4145, - "hfopenllm_v2/BBH": 0.4319, - "hfopenllm_v2/MATH Level 5": 0.0831, - "hfopenllm_v2/GPQA": 0.2668, - "hfopenllm_v2/MUSR": 0.4005, - "hfopenllm_v2/MMLU-PRO": 0.279 - } - }, - { - "id": "FINGU-AI/QwQ-Buddy-32B-Alpha", - "name": "QwQ-Buddy-32B-Alpha", - "developer": "FINGU-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3446, - "hfopenllm_v2/BBH": 0.6424, - "hfopenllm_v2/MATH Level 5": 0.3852, - "hfopenllm_v2/GPQA": 0.3792, - "hfopenllm_v2/MUSR": 0.506, - "hfopenllm_v2/MMLU-PRO": 0.5294 - } - }, - { - "id": "FINGU-AI/RomboUltima-32B", - "name": "RomboUltima-32B", - "developer": "FINGU-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6672, - "hfopenllm_v2/BBH": 0.6938, - "hfopenllm_v2/MATH Level 5": 0.5385, - "hfopenllm_v2/GPQA": 0.3716, - "hfopenllm_v2/MUSR": 0.4836, - "hfopenllm_v2/MMLU-PRO": 0.5789 - } - }, - { - "id": "FINGU-AI/Ultimos-32B", - "name": "Ultimos-32B", - "developer": "FINGU-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1592, - "hfopenllm_v2/BBH": 0.2906, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2492, - "hfopenllm_v2/MUSR": 0.3286, - "hfopenllm_v2/MMLU-PRO": 0.1111 - } - }, - { - "id": "flammenai/flammen15-gutenberg-DPO-v1-7B", - "name": "flammen15-gutenberg-DPO-v1-7B", - "developer": "flammenai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4798, - "hfopenllm_v2/BBH": 0.5203, - "hfopenllm_v2/MATH Level 5": 0.0763, - "hfopenllm_v2/GPQA": 0.2844, - "hfopenllm_v2/MUSR": 0.4293, - "hfopenllm_v2/MMLU-PRO": 0.3186 - } - }, - { - "id": "flammenai/Llama3.1-Flammades-70B", - "name": "Llama3.1-Flammades-70B", - "developer": "flammenai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7058, - "hfopenllm_v2/BBH": 0.666, - "hfopenllm_v2/MATH Level 5": 0.2092, - "hfopenllm_v2/GPQA": 0.354, - "hfopenllm_v2/MUSR": 0.4871, - "hfopenllm_v2/MMLU-PRO": 0.4752 - } - }, - { - "id": "flammenai/Mahou-1.2a-llama3-8B", - "name": "Mahou-1.2a-llama3-8B", - "developer": "flammenai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5093, - "hfopenllm_v2/BBH": 0.5094, - "hfopenllm_v2/MATH Level 5": 0.0838, - "hfopenllm_v2/GPQA": 0.2886, - "hfopenllm_v2/MUSR": 0.3847, - "hfopenllm_v2/MMLU-PRO": 0.3817 - } - }, - { - "id": "flammenai/Mahou-1.2a-mistral-7B", - "name": "Mahou-1.2a-mistral-7B", - "developer": "flammenai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4552, - "hfopenllm_v2/BBH": 0.5118, - "hfopenllm_v2/MATH Level 5": 0.0687, - "hfopenllm_v2/GPQA": 0.2718, - "hfopenllm_v2/MUSR": 0.3896, - "hfopenllm_v2/MMLU-PRO": 0.3163 - } - }, - { - "id": "flammenai/Mahou-1.5-llama3.1-70B", - "name": "Mahou-1.5-llama3.1-70B", - "developer": "flammenai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7147, - "hfopenllm_v2/BBH": 0.6651, - "hfopenllm_v2/MATH Level 5": 0.21, - "hfopenllm_v2/GPQA": 0.354, - "hfopenllm_v2/MUSR": 0.495, - "hfopenllm_v2/MMLU-PRO": 0.4749 - } - }, - { - "id": "flammenai/Mahou-1.5-mistral-nemo-12B", - "name": "Mahou-1.5-mistral-nemo-12B", - "developer": "flammenai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6751, - "hfopenllm_v2/BBH": 0.5522, - "hfopenllm_v2/MATH Level 5": 0.0869, - "hfopenllm_v2/GPQA": 0.276, - "hfopenllm_v2/MUSR": 0.452, - "hfopenllm_v2/MMLU-PRO": 0.3602 - } - }, - { - "id": "FlofloB/100k_fineweb_continued_pretraining_Qwen2.5-0.5B-Instruct_Unsloth_merged_16bit", - "name": "100k_fineweb_continued_pretraining_Qwen2.5-0.5B-Instruct_Unsloth_merged_16bit", - "developer": "FlofloB", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3083, - "hfopenllm_v2/BBH": 0.3323, - "hfopenllm_v2/MATH Level 5": 0.0408, - "hfopenllm_v2/GPQA": 0.2693, - "hfopenllm_v2/MUSR": 0.3302, - "hfopenllm_v2/MMLU-PRO": 0.1498 - } - }, - { - "id": "FlofloB/10k_continued_pretraining_Phi-3-mini-4k-instruct_Unsloth_merged_16bit", - "name": "10k_continued_pretraining_Phi-3-mini-4k-instruct_Unsloth_merged_16bit", - "developer": "FlofloB", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5097, - "hfopenllm_v2/BBH": 0.5215, - "hfopenllm_v2/MATH Level 5": 0.0974, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.431, - "hfopenllm_v2/MMLU-PRO": 0.3769 - } - }, - { - "id": "FlofloB/10k_continued_pretraining_Qwen2.5-0.5B-Instruct_Unsloth_merged_16bit", - "name": "10k_continued_pretraining_Qwen2.5-0.5B-Instruct_Unsloth_merged_16bit", - "developer": "FlofloB", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2815, - "hfopenllm_v2/BBH": 0.3306, - "hfopenllm_v2/MATH Level 5": 0.031, - "hfopenllm_v2/GPQA": 0.2794, - "hfopenllm_v2/MUSR": 0.3302, - "hfopenllm_v2/MMLU-PRO": 0.1541 - } - }, - { - "id": "FlofloB/40k_continued_pretraining_Qwen2.5-0.5B-Instruct_Unsloth_merged_16bit", - "name": "40k_continued_pretraining_Qwen2.5-0.5B-Instruct_Unsloth_merged_16bit", - "developer": "FlofloB", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3016, - "hfopenllm_v2/BBH": 0.3325, - "hfopenllm_v2/MATH Level 5": 0.0332, - "hfopenllm_v2/GPQA": 0.2676, - "hfopenllm_v2/MUSR": 0.3408, - "hfopenllm_v2/MMLU-PRO": 0.1485 - } - }, - { - "id": "FlofloB/83k_continued_pretraining_Qwen2.5-0.5B-Instruct_Unsloth_merged_16bit", - "name": "83k_continued_pretraining_Qwen2.5-0.5B-Instruct_Unsloth_merged_16bit", - "developer": "FlofloB", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2869, - "hfopenllm_v2/BBH": 0.3347, - "hfopenllm_v2/MATH Level 5": 0.0302, - "hfopenllm_v2/GPQA": 0.2735, - "hfopenllm_v2/MUSR": 0.3289, - "hfopenllm_v2/MMLU-PRO": 0.1555 - } - }, - { - "id": "FlofloB/smollm2-135M_pretrained_1000k_fineweb", - "name": "smollm2-135M_pretrained_1000k_fineweb", - "developer": "FlofloB", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1485, - "hfopenllm_v2/BBH": 0.2918, - "hfopenllm_v2/MATH Level 5": 0.0091, - "hfopenllm_v2/GPQA": 0.2626, - "hfopenllm_v2/MUSR": 0.3581, - "hfopenllm_v2/MMLU-PRO": 0.1164 - } - }, - { - "id": "FlofloB/smollm2-135M_pretrained_1000k_fineweb_uncovai_human_removed", - "name": "smollm2-135M_pretrained_1000k_fineweb_uncovai_human_removed", - "developer": "FlofloB", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1554, - "hfopenllm_v2/BBH": 0.3066, - "hfopenllm_v2/MATH Level 5": 0.006, - "hfopenllm_v2/GPQA": 0.2508, - "hfopenllm_v2/MUSR": 0.358, - "hfopenllm_v2/MMLU-PRO": 0.1143 - } - }, - { - "id": "FlofloB/smollm2-135M_pretrained_1000k_fineweb_uncovai_selected", - "name": "smollm2-135M_pretrained_1000k_fineweb_uncovai_selected", - "developer": "FlofloB", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1468, - "hfopenllm_v2/BBH": 0.2932, - "hfopenllm_v2/MATH Level 5": 0.0068, - "hfopenllm_v2/GPQA": 0.2659, - "hfopenllm_v2/MUSR": 0.4048, - "hfopenllm_v2/MMLU-PRO": 0.1157 - } - }, - { - "id": "FlofloB/smollm2-135M_pretrained_1200k_fineweb", - "name": "smollm2-135M_pretrained_1200k_fineweb", - "developer": "FlofloB", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1581, - "hfopenllm_v2/BBH": 0.2941, - "hfopenllm_v2/MATH Level 5": 0.0068, - "hfopenllm_v2/GPQA": 0.2643, - "hfopenllm_v2/MUSR": 0.3714, - "hfopenllm_v2/MMLU-PRO": 0.1076 - } - }, - { - "id": "FlofloB/smollm2-135M_pretrained_1200k_fineweb_uncovai_human_removed", - "name": "smollm2-135M_pretrained_1200k_fineweb_uncovai_human_removed", - "developer": "FlofloB", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1578, - "hfopenllm_v2/BBH": 0.295, - "hfopenllm_v2/MATH Level 5": 0.0008, - "hfopenllm_v2/GPQA": 0.2651, - "hfopenllm_v2/MUSR": 0.37, - "hfopenllm_v2/MMLU-PRO": 0.1139 - } - }, - { - "id": "FlofloB/smollm2-135M_pretrained_1200k_fineweb_uncovai_selected", - "name": "smollm2-135M_pretrained_1200k_fineweb_uncovai_selected", - "developer": "FlofloB", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1585, - "hfopenllm_v2/BBH": 0.296, - "hfopenllm_v2/MATH Level 5": 0.0076, - "hfopenllm_v2/GPQA": 0.2634, - "hfopenllm_v2/MUSR": 0.3567, - "hfopenllm_v2/MMLU-PRO": 0.1164 - } - }, - { - "id": "FlofloB/smollm2-135M_pretrained_1400k_fineweb", - "name": "smollm2-135M_pretrained_1400k_fineweb", - "developer": "FlofloB", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1764, - "hfopenllm_v2/BBH": 0.2922, - "hfopenllm_v2/MATH Level 5": 0.0113, - "hfopenllm_v2/GPQA": 0.2659, - "hfopenllm_v2/MUSR": 0.3873, - "hfopenllm_v2/MMLU-PRO": 0.108 - } - }, - { - "id": "FlofloB/smollm2-135M_pretrained_1400k_fineweb_uncovai_human_removed", - "name": "smollm2-135M_pretrained_1400k_fineweb_uncovai_human_removed", - "developer": "FlofloB", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1707, - "hfopenllm_v2/BBH": 0.2992, - "hfopenllm_v2/MATH Level 5": 0.0106, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.3939, - "hfopenllm_v2/MMLU-PRO": 0.1105 - } - }, - { - "id": "FlofloB/smollm2-135M_pretrained_1400k_fineweb_uncovai_selected", - "name": "smollm2-135M_pretrained_1400k_fineweb_uncovai_selected", - "developer": "FlofloB", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1538, - "hfopenllm_v2/BBH": 0.2917, - "hfopenllm_v2/MATH Level 5": 0.0106, - "hfopenllm_v2/GPQA": 0.2685, - "hfopenllm_v2/MUSR": 0.3741, - "hfopenllm_v2/MMLU-PRO": 0.1137 - } - }, - { - "id": "FlofloB/smollm2-135M_pretrained_200k_fineweb_uncovai_human_removed", - "name": "smollm2-135M_pretrained_200k_fineweb_uncovai_human_removed", - "developer": "FlofloB", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1475, - "hfopenllm_v2/BBH": 0.3029, - "hfopenllm_v2/MATH Level 5": 0.0038, - "hfopenllm_v2/GPQA": 0.2584, - "hfopenllm_v2/MUSR": 0.3578, - "hfopenllm_v2/MMLU-PRO": 0.112 - } - }, - { - "id": "FlofloB/smollm2-135M_pretrained_200k_fineweb_uncovai_selected", - "name": "smollm2-135M_pretrained_200k_fineweb_uncovai_selected", - "developer": "FlofloB", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1345, - "hfopenllm_v2/BBH": 0.2927, - "hfopenllm_v2/MATH Level 5": 0.0076, - "hfopenllm_v2/GPQA": 0.2508, - "hfopenllm_v2/MUSR": 0.366, - "hfopenllm_v2/MMLU-PRO": 0.1131 - } - }, - { - "id": "FlofloB/smollm2-135M_pretrained_400k_fineweb", - "name": "smollm2-135M_pretrained_400k_fineweb", - "developer": "FlofloB", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1511, - "hfopenllm_v2/BBH": 0.2972, - "hfopenllm_v2/MATH Level 5": 0.0121, - "hfopenllm_v2/GPQA": 0.2525, - "hfopenllm_v2/MUSR": 0.3794, - "hfopenllm_v2/MMLU-PRO": 0.1163 - } - }, - { - "id": "FlofloB/smollm2-135M_pretrained_400k_fineweb_uncovai_human_removed", - "name": "smollm2-135M_pretrained_400k_fineweb_uncovai_human_removed", - "developer": "FlofloB", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1556, - "hfopenllm_v2/BBH": 0.3049, - "hfopenllm_v2/MATH Level 5": 0.0091, - "hfopenllm_v2/GPQA": 0.255, - "hfopenllm_v2/MUSR": 0.386, - "hfopenllm_v2/MMLU-PRO": 0.1138 - } - }, - { - "id": "FlofloB/smollm2-135M_pretrained_400k_fineweb_uncovai_selected", - "name": "smollm2-135M_pretrained_400k_fineweb_uncovai_selected", - "developer": "FlofloB", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1584, - "hfopenllm_v2/BBH": 0.2925, - "hfopenllm_v2/MATH Level 5": 0.0068, - "hfopenllm_v2/GPQA": 0.2542, - "hfopenllm_v2/MUSR": 0.382, - "hfopenllm_v2/MMLU-PRO": 0.1158 - } - }, - { - "id": "FlofloB/smollm2-135M_pretrained_600k_fineweb", - "name": "smollm2-135M_pretrained_600k_fineweb", - "developer": "FlofloB", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1639, - "hfopenllm_v2/BBH": 0.3014, - "hfopenllm_v2/MATH Level 5": 0.006, - "hfopenllm_v2/GPQA": 0.2659, - "hfopenllm_v2/MUSR": 0.3809, - "hfopenllm_v2/MMLU-PRO": 0.1126 - } - }, - { - "id": "FlofloB/smollm2-135M_pretrained_600k_fineweb_uncovai_human_removed", - "name": "smollm2-135M_pretrained_600k_fineweb_uncovai_human_removed", - "developer": "FlofloB", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1641, - "hfopenllm_v2/BBH": 0.3, - "hfopenllm_v2/MATH Level 5": 0.0091, - "hfopenllm_v2/GPQA": 0.2626, - "hfopenllm_v2/MUSR": 0.3793, - "hfopenllm_v2/MMLU-PRO": 0.1147 - } - }, - { - "id": "FlofloB/smollm2-135M_pretrained_600k_fineweb_uncovai_selected", - "name": "smollm2-135M_pretrained_600k_fineweb_uncovai_selected", - "developer": "FlofloB", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1606, - "hfopenllm_v2/BBH": 0.2983, - "hfopenllm_v2/MATH Level 5": 0.0076, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.3846, - "hfopenllm_v2/MMLU-PRO": 0.1162 - } - }, - { - "id": "FlofloB/smollm2-135M_pretrained_800k_fineweb", - "name": "smollm2-135M_pretrained_800k_fineweb", - "developer": "FlofloB", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1641, - "hfopenllm_v2/BBH": 0.2959, - "hfopenllm_v2/MATH Level 5": 0.0083, - "hfopenllm_v2/GPQA": 0.2492, - "hfopenllm_v2/MUSR": 0.3701, - "hfopenllm_v2/MMLU-PRO": 0.1152 - } - }, - { - "id": "FlofloB/smollm2-135M_pretrained_800k_fineweb_uncovai_human_removed", - "name": "smollm2-135M_pretrained_800k_fineweb_uncovai_human_removed", - "developer": "FlofloB", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1623, - "hfopenllm_v2/BBH": 0.3038, - "hfopenllm_v2/MATH Level 5": 0.0068, - "hfopenllm_v2/GPQA": 0.2525, - "hfopenllm_v2/MUSR": 0.3993, - "hfopenllm_v2/MMLU-PRO": 0.1138 - } - }, - { - "id": "FlofloB/smollm2-135M_pretrained_800k_fineweb_uncovai_selected", - "name": "smollm2-135M_pretrained_800k_fineweb_uncovai_selected", - "developer": "FlofloB", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1474, - "hfopenllm_v2/BBH": 0.2943, - "hfopenllm_v2/MATH Level 5": 0.0045, - "hfopenllm_v2/GPQA": 0.2617, - "hfopenllm_v2/MUSR": 0.3766, - "hfopenllm_v2/MMLU-PRO": 0.113 - } - }, - { - "id": "FlofloB/smollm2_pretrained_200k_fineweb", - "name": "smollm2_pretrained_200k_fineweb", - "developer": "FlofloB", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1527, - "hfopenllm_v2/BBH": 0.2995, - "hfopenllm_v2/MATH Level 5": 0.0038, - "hfopenllm_v2/GPQA": 0.2475, - "hfopenllm_v2/MUSR": 0.3699, - "hfopenllm_v2/MMLU-PRO": 0.1159 - } - }, - { - "id": "FlofloB/test_continued_pretraining_Phi-3-mini-4k-instruct_Unsloth_merged_16bit", - "name": "test_continued_pretraining_Phi-3-mini-4k-instruct_Unsloth_merged_16bit", - "developer": "FlofloB", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5215, - "hfopenllm_v2/BBH": 0.5241, - "hfopenllm_v2/MATH Level 5": 0.1103, - "hfopenllm_v2/GPQA": 0.3112, - "hfopenllm_v2/MUSR": 0.4244, - "hfopenllm_v2/MMLU-PRO": 0.3721 - } - }, - { - "id": "fluently-lm/FluentlyLM-Prinum", - "name": "FluentlyLM-Prinum", - "developer": "fluently-lm", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.809, - "hfopenllm_v2/BBH": 0.7144, - "hfopenllm_v2/MATH Level 5": 0.54, - "hfopenllm_v2/GPQA": 0.3867, - "hfopenllm_v2/MUSR": 0.4471, - "hfopenllm_v2/MMLU-PRO": 0.5808 - } - }, - { - "id": "fluently-lm/Llama-TI-8B", - "name": "Llama-TI-8B", - "developer": "fluently-lm", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.288, - "hfopenllm_v2/BBH": 0.5201, - "hfopenllm_v2/MATH Level 5": 0.1964, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.4103, - "hfopenllm_v2/MMLU-PRO": 0.344 - } - }, - { - "id": "fluently-lm/Llama-TI-8B-Instruct", - "name": "Llama-TI-8B-Instruct", - "developer": "fluently-lm", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7716, - "hfopenllm_v2/BBH": 0.5252, - "hfopenllm_v2/MATH Level 5": 0.2304, - "hfopenllm_v2/GPQA": 0.2953, - "hfopenllm_v2/MUSR": 0.3813, - "hfopenllm_v2/MMLU-PRO": 0.3726 - } - }, - { - "id": "fluently-sets/FalconThink3-10B-IT", - "name": "FalconThink3-10B-IT", - "developer": "fluently-sets", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7326, - "hfopenllm_v2/BBH": 0.62, - "hfopenllm_v2/MATH Level 5": 0.2447, - "hfopenllm_v2/GPQA": 0.3347, - "hfopenllm_v2/MUSR": 0.4479, - "hfopenllm_v2/MMLU-PRO": 0.4435 - } - }, - { - "id": "fluently-sets/reasoning-1-1k-demo", - "name": "reasoning-1-1k-demo", - "developer": "fluently-sets", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7525, - "hfopenllm_v2/BBH": 0.6397, - "hfopenllm_v2/MATH Level 5": 0.4282, - "hfopenllm_v2/GPQA": 0.3356, - "hfopenllm_v2/MUSR": 0.4061, - "hfopenllm_v2/MMLU-PRO": 0.4774 - } - }, - { - "id": "formulae/mita-elite-sce-gen1.1-v1-7b-2-26-2025-exp", - "name": "mita-elite-sce-gen1.1-v1-7b-2-26-2025-exp", - "developer": "formulae", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1614, - "hfopenllm_v2/BBH": 0.2976, - "hfopenllm_v2/MATH Level 5": 0.0015, - "hfopenllm_v2/GPQA": 0.2534, - "hfopenllm_v2/MUSR": 0.4219, - "hfopenllm_v2/MMLU-PRO": 0.1174 - } - }, - { - "id": "formulae/mita-elite-v1.1-7b-2-25-2025", - "name": "mita-elite-v1.1-7b-2-25-2025", - "developer": "formulae", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.125, - "hfopenllm_v2/BBH": 0.2867, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2483, - "hfopenllm_v2/MUSR": 0.3487, - "hfopenllm_v2/MMLU-PRO": 0.1098 - } - }, - { - "id": "formulae/mita-elite-v1.1-gen2-7b-2-25-2025", - "name": "mita-elite-v1.1-gen2-7b-2-25-2025", - "developer": "formulae", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1411, - "hfopenllm_v2/BBH": 0.2924, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2525, - "hfopenllm_v2/MUSR": 0.3541, - "hfopenllm_v2/MMLU-PRO": 0.1101 - } - }, - { - "id": "formulae/mita-elite-v1.2-7b-2-26-2025", - "name": "mita-elite-v1.2-7b-2-26-2025", - "developer": "formulae", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.148, - "hfopenllm_v2/BBH": 0.293, - "hfopenllm_v2/MATH Level 5": 0.0023, - "hfopenllm_v2/GPQA": 0.2743, - "hfopenllm_v2/MUSR": 0.4287, - "hfopenllm_v2/MMLU-PRO": 0.1186 - } - }, - { - "id": "formulae/mita-gen3-7b-2-26-2025", - "name": "mita-gen3-7b-2-26-2025", - "developer": "formulae", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1964, - "hfopenllm_v2/BBH": 0.2916, - "hfopenllm_v2/MATH Level 5": 0.0023, - "hfopenllm_v2/GPQA": 0.2651, - "hfopenllm_v2/MUSR": 0.3912, - "hfopenllm_v2/MMLU-PRO": 0.1124 - } - }, - { - "id": "formulae/mita-gen3-v1.2-7b-2-26-2025", - "name": "mita-gen3-v1.2-7b-2-26-2025", - "developer": "formulae", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2044, - "hfopenllm_v2/BBH": 0.3058, - "hfopenllm_v2/MATH Level 5": 0.0023, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.39, - "hfopenllm_v2/MMLU-PRO": 0.1128 - } - }, - { - "id": "formulae/mita-math-v2.3-2-25-2025", - "name": "mita-math-v2.3-2-25-2025", - "developer": "formulae", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1373, - "hfopenllm_v2/BBH": 0.2949, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2508, - "hfopenllm_v2/MUSR": 0.3698, - "hfopenllm_v2/MMLU-PRO": 0.1118 - } - }, - { - "id": "formulae/mita-v1-7b", - "name": "mita-v1-7b", - "developer": "formulae", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1972, - "hfopenllm_v2/BBH": 0.3003, - "hfopenllm_v2/MATH Level 5": 0.0023, - "hfopenllm_v2/GPQA": 0.25, - "hfopenllm_v2/MUSR": 0.4152, - "hfopenllm_v2/MMLU-PRO": 0.1147 - } - }, - { - "id": "formulae/mita-v1.1-7b-2-24-2025", - "name": "mita-v1.1-7b-2-24-2025", - "developer": "formulae", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3412, - "hfopenllm_v2/BBH": 0.5442, - "hfopenllm_v2/MATH Level 5": 0.435, - "hfopenllm_v2/GPQA": 0.3146, - "hfopenllm_v2/MUSR": 0.4557, - "hfopenllm_v2/MMLU-PRO": 0.4524 - } - }, - { - "id": "formulae/mita-v1.2-7b-2-24-2025", - "name": "mita-v1.2-7b-2-24-2025", - "developer": "formulae", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2564, - "hfopenllm_v2/BBH": 0.4919, - "hfopenllm_v2/MATH Level 5": 0.4879, - "hfopenllm_v2/GPQA": 0.3062, - "hfopenllm_v2/MUSR": 0.4344, - "hfopenllm_v2/MMLU-PRO": 0.3359 - } - }, - { - "id": "frameai/Loxa-4B", - "name": "Loxa-4B", - "developer": "frameai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4765, - "hfopenllm_v2/BBH": 0.4217, - "hfopenllm_v2/MATH Level 5": 0.1095, - "hfopenllm_v2/GPQA": 0.2836, - "hfopenllm_v2/MUSR": 0.3377, - "hfopenllm_v2/MMLU-PRO": 0.2802 - } - }, - { - "id": "freewheelin/free-evo-qwen72b-v0.8-re", - "name": "free-evo-qwen72b-v0.8-re", - "developer": "freewheelin", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5331, - "hfopenllm_v2/BBH": 0.6127, - "hfopenllm_v2/MATH Level 5": 0.1805, - "hfopenllm_v2/GPQA": 0.3565, - "hfopenllm_v2/MUSR": 0.4872, - "hfopenllm_v2/MMLU-PRO": 0.487 - } - }, - { - "id": "freewheelin/free-solar-evo-v0.1", - "name": "free-solar-evo-v0.1", - "developer": "freewheelin", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.205, - "hfopenllm_v2/BBH": 0.4502, - "hfopenllm_v2/MATH Level 5": 0.0083, - "hfopenllm_v2/GPQA": 0.2911, - "hfopenllm_v2/MUSR": 0.4946, - "hfopenllm_v2/MMLU-PRO": 0.3414 - } - }, - { - "id": "freewheelin/free-solar-evo-v0.11", - "name": "free-solar-evo-v0.11", - "developer": "freewheelin", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2027, - "hfopenllm_v2/BBH": 0.4545, - "hfopenllm_v2/MATH Level 5": 0.0083, - "hfopenllm_v2/GPQA": 0.2852, - "hfopenllm_v2/MUSR": 0.5052, - "hfopenllm_v2/MMLU-PRO": 0.3467 - } - }, - { - "id": "freewheelin/free-solar-evo-v0.13", - "name": "free-solar-evo-v0.13", - "developer": "freewheelin", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2321, - "hfopenllm_v2/BBH": 0.4555, - "hfopenllm_v2/MATH Level 5": 0.0121, - "hfopenllm_v2/GPQA": 0.2886, - "hfopenllm_v2/MUSR": 0.5052, - "hfopenllm_v2/MMLU-PRO": 0.347 - } - }, - { - "id": "FuJhen/ft-openhermes-25-mistral-7b-irca-dpo-pairs", - "name": "ft-openhermes-25-mistral-7b-irca-dpo-pairs", - "developer": "FuJhen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.542, - "hfopenllm_v2/BBH": 0.4773, - "hfopenllm_v2/MATH Level 5": 0.0483, - "hfopenllm_v2/GPQA": 0.2785, - "hfopenllm_v2/MUSR": 0.4174, - "hfopenllm_v2/MMLU-PRO": 0.2956 - } - }, - { - "id": "FuJhen/mistral-instruct-7B-DPO", - "name": "mistral-instruct-7B-DPO", - "developer": "FuJhen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4968, - "hfopenllm_v2/BBH": 0.4624, - "hfopenllm_v2/MATH Level 5": 0.0385, - "hfopenllm_v2/GPQA": 0.2777, - "hfopenllm_v2/MUSR": 0.4016, - "hfopenllm_v2/MMLU-PRO": 0.3034 - } - }, - { - "id": "FuJhen/mistral_7b_v0.1_structedData_e2e", - "name": "mistral_7b_v0.1_structedData_e2e", - "developer": "FuJhen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1727, - "hfopenllm_v2/BBH": 0.4114, - "hfopenllm_v2/MATH Level 5": 0.0045, - "hfopenllm_v2/GPQA": 0.2794, - "hfopenllm_v2/MUSR": 0.3723, - "hfopenllm_v2/MMLU-PRO": 0.2811 - } - }, - { - "id": "FuJhen/mistral_7b_v0.1_structedData_viggo", - "name": "mistral_7b_v0.1_structedData_viggo", - "developer": "FuJhen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1783, - "hfopenllm_v2/BBH": 0.4524, - "hfopenllm_v2/MATH Level 5": 0.0287, - "hfopenllm_v2/GPQA": 0.2836, - "hfopenllm_v2/MUSR": 0.3738, - "hfopenllm_v2/MMLU-PRO": 0.2942 - } - }, - { - "id": "fulim/FineLlama-3.1-8B", - "name": "FineLlama-3.1-8B", - "developer": "fulim", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1439, - "hfopenllm_v2/BBH": 0.4569, - "hfopenllm_v2/MATH Level 5": 0.0476, - "hfopenllm_v2/GPQA": 0.2928, - "hfopenllm_v2/MUSR": 0.3867, - "hfopenllm_v2/MMLU-PRO": 0.3167 - } - }, - { - "id": "FuseAI/FuseChat-7B-v2.0", - "name": "FuseChat-7B-v2.0", - "developer": "FuseAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3423, - "hfopenllm_v2/BBH": 0.4954, - "hfopenllm_v2/MATH Level 5": 0.0612, - "hfopenllm_v2/GPQA": 0.302, - "hfopenllm_v2/MUSR": 0.4797, - "hfopenllm_v2/MMLU-PRO": 0.3162 - } - }, - { - "id": "FuseAI/FuseChat-Llama-3.1-8B-Instruct", - "name": "FuseChat-Llama-3.1-8B-Instruct", - "developer": "FuseAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7205, - "hfopenllm_v2/BBH": 0.512, - "hfopenllm_v2/MATH Level 5": 0.2477, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.382, - "hfopenllm_v2/MMLU-PRO": 0.3733 - } - }, - { - "id": "FuseAI/FuseChat-Llama-3.2-3B-Instruct", - "name": "FuseChat-Llama-3.2-3B-Instruct", - "developer": "FuseAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6849, - "hfopenllm_v2/BBH": 0.4658, - "hfopenllm_v2/MATH Level 5": 0.2424, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.3914, - "hfopenllm_v2/MMLU-PRO": 0.3132 - } - }, - { - "id": "FuseAI/FuseChat-Qwen-2.5-7B-Instruct", - "name": "FuseChat-Qwen-2.5-7B-Instruct", - "developer": "FuseAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5906, - "hfopenllm_v2/BBH": 0.5526, - "hfopenllm_v2/MATH Level 5": 0.4562, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.3874, - "hfopenllm_v2/MMLU-PRO": 0.4118 - } - }, - { - "id": "gabrielmbmb/SmolLM-1.7B-Instruct-IFEval", - "name": "SmolLM-1.7B-Instruct-IFEval", - "developer": "gabrielmbmb", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2306, - "hfopenllm_v2/BBH": 0.3138, - "hfopenllm_v2/MATH Level 5": 0.0106, - "hfopenllm_v2/GPQA": 0.2534, - "hfopenllm_v2/MUSR": 0.3328, - "hfopenllm_v2/MMLU-PRO": 0.1156 - } - }, - { - "id": "GalrionSoftworks/MagnusIntellectus-12B-v1", - "name": "MagnusIntellectus-12B-v1", - "developer": "GalrionSoftworks", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4421, - "hfopenllm_v2/BBH": 0.5323, - "hfopenllm_v2/MATH Level 5": 0.065, - "hfopenllm_v2/GPQA": 0.2844, - "hfopenllm_v2/MUSR": 0.4428, - "hfopenllm_v2/MMLU-PRO": 0.3421 - } - }, - { - "id": "GalrionSoftworks/MN-LooseCannon-12B-v1", - "name": "MN-LooseCannon-12B-v1", - "developer": "GalrionSoftworks", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5418, - "hfopenllm_v2/BBH": 0.5128, - "hfopenllm_v2/MATH Level 5": 0.0853, - "hfopenllm_v2/GPQA": 0.2852, - "hfopenllm_v2/MUSR": 0.4138, - "hfopenllm_v2/MMLU-PRO": 0.3196 - } - }, - { - "id": "gaverfraxz/Meta-Llama-3.1-8B-Instruct-HalfAbliterated-DELLA", - "name": "Meta-Llama-3.1-8B-Instruct-HalfAbliterated-DELLA", - "developer": "gaverfraxz", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4009, - "hfopenllm_v2/BBH": 0.3985, - "hfopenllm_v2/MATH Level 5": 0.0196, - "hfopenllm_v2/GPQA": 0.2844, - "hfopenllm_v2/MUSR": 0.365, - "hfopenllm_v2/MMLU-PRO": 0.1654 - } - }, - { - "id": "gaverfraxz/Meta-Llama-3.1-8B-Instruct-HalfAbliterated-TIES", - "name": "Meta-Llama-3.1-8B-Instruct-HalfAbliterated-TIES", - "developer": "gaverfraxz", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4551, - "hfopenllm_v2/BBH": 0.5044, - "hfopenllm_v2/MATH Level 5": 0.1299, - "hfopenllm_v2/GPQA": 0.2668, - "hfopenllm_v2/MUSR": 0.3738, - "hfopenllm_v2/MMLU-PRO": 0.3679 - } - }, - { - "id": "gbueno86/Brinebreath-Llama-3.1-70B", - "name": "Brinebreath-Llama-3.1-70B", - "developer": "gbueno86", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5533, - "hfopenllm_v2/BBH": 0.6881, - "hfopenllm_v2/MATH Level 5": 0.2976, - "hfopenllm_v2/GPQA": 0.3465, - "hfopenllm_v2/MUSR": 0.4541, - "hfopenllm_v2/MMLU-PRO": 0.5196 - } - }, - { - "id": "gbueno86/Meta-LLama-3-Cat-Smaug-LLama-70b", - "name": "Meta-LLama-3-Cat-Smaug-LLama-70b", - "developer": "gbueno86", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8072, - "hfopenllm_v2/BBH": 0.6674, - "hfopenllm_v2/MATH Level 5": 0.2938, - "hfopenllm_v2/GPQA": 0.3272, - "hfopenllm_v2/MUSR": 0.4368, - "hfopenllm_v2/MMLU-PRO": 0.5075 - } - }, - { - "id": "gemini-1.5-flash-8b", - "name": "gemini-1.5-flash-8b", - "developer": "unknown", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7601, - "reward-bench/Chat": 0.9441, - "reward-bench/Chat Hard": 0.5987, - "reward-bench/Safety": 0.7399, - "reward-bench/Reasoning": 0.7575 - } - }, - { - "id": "general-preference/GPM-Gemma-2B", - "name": "general-preference/GPM-Gemma-2B", - "developer": "general-preference", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7449, - "reward-bench/Chat": 0.7151, - "reward-bench/Chat Hard": 0.6974, - "reward-bench/Safety": 0.8122, - "reward-bench/Reasoning": 0.755 - } - }, - { - "id": "general-preference/GPM-Llama-3.1-8B", - "name": "general-preference/GPM-Llama-3.1-8B", - "developer": "general-preference", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.9224, - "reward-bench/Chat": 0.933, - "reward-bench/Chat Hard": 0.886, - "reward-bench/Safety": 0.9108, - "reward-bench/Reasoning": 0.9597 - } - }, - { - "id": "GenVRadmin/AryaBhatta-GemmaOrca-2-Merged", - "name": "AryaBhatta-GemmaOrca-2-Merged", - "developer": "GenVRadmin", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3064, - "hfopenllm_v2/BBH": 0.3887, - "hfopenllm_v2/MATH Level 5": 0.0498, - "hfopenllm_v2/GPQA": 0.2685, - "hfopenllm_v2/MUSR": 0.455, - "hfopenllm_v2/MMLU-PRO": 0.2384 - } - }, - { - "id": "GenVRadmin/AryaBhatta-GemmaOrca-Merged", - "name": "AryaBhatta-GemmaOrca-Merged", - "developer": "GenVRadmin", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3064, - "hfopenllm_v2/BBH": 0.4131, - "hfopenllm_v2/MATH Level 5": 0.0514, - "hfopenllm_v2/GPQA": 0.2559, - "hfopenllm_v2/MUSR": 0.3524, - "hfopenllm_v2/MMLU-PRO": 0.2228 - } - }, - { - "id": "GenVRadmin/AryaBhatta-GemmaUltra-Merged", - "name": "AryaBhatta-GemmaUltra-Merged", - "developer": "GenVRadmin", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3021, - "hfopenllm_v2/BBH": 0.4141, - "hfopenllm_v2/MATH Level 5": 0.0536, - "hfopenllm_v2/GPQA": 0.2534, - "hfopenllm_v2/MUSR": 0.4279, - "hfopenllm_v2/MMLU-PRO": 0.2266 - } - }, - { - "id": "GenVRadmin/llama38bGenZ_Vikas-Merged", - "name": "llama38bGenZ_Vikas-Merged", - "developer": "GenVRadmin", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3, - "hfopenllm_v2/BBH": 0.4536, - "hfopenllm_v2/MATH Level 5": 0.0574, - "hfopenllm_v2/GPQA": 0.2953, - "hfopenllm_v2/MUSR": 0.4402, - "hfopenllm_v2/MMLU-PRO": 0.2622 - } - }, - { - "id": "ghost-x/ghost-8b-beta-1608", - "name": "ghost-8b-beta-1608", - "developer": "ghost-x", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4273, - "hfopenllm_v2/BBH": 0.4517, - "hfopenllm_v2/MATH Level 5": 0.0695, - "hfopenllm_v2/GPQA": 0.2584, - "hfopenllm_v2/MUSR": 0.3516, - "hfopenllm_v2/MMLU-PRO": 0.284 - } - }, - { - "id": "glaiveai/Reflection-Llama-3.1-70B", - "name": "Reflection-Llama-3.1-70B", - "developer": "glaiveai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5991, - "hfopenllm_v2/BBH": 0.5681, - "hfopenllm_v2/MATH Level 5": 0.2757, - "hfopenllm_v2/GPQA": 0.3146, - "hfopenllm_v2/MUSR": 0.438, - "hfopenllm_v2/MMLU-PRO": 0.6341 - } - }, - { - "id": "gmonsoon/gemma2-9b-sahabatai-v1-instruct-BaseTIES", - "name": "gemma2-9b-sahabatai-v1-instruct-BaseTIES", - "developer": "gmonsoon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7378, - "hfopenllm_v2/BBH": 0.6077, - "hfopenllm_v2/MATH Level 5": 0.1994, - "hfopenllm_v2/GPQA": 0.3205, - "hfopenllm_v2/MUSR": 0.4778, - "hfopenllm_v2/MMLU-PRO": 0.4347 - } - }, - { - "id": "gmonsoon/SahabatAI-Llama-11B-Test", - "name": "SahabatAI-Llama-11B-Test", - "developer": "gmonsoon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3376, - "hfopenllm_v2/BBH": 0.4728, - "hfopenllm_v2/MATH Level 5": 0.031, - "hfopenllm_v2/GPQA": 0.2819, - "hfopenllm_v2/MUSR": 0.4001, - "hfopenllm_v2/MMLU-PRO": 0.3182 - } - }, - { - "id": "gmonsoon/SahabatAI-MediChatIndo-8B-v1", - "name": "SahabatAI-MediChatIndo-8B-v1", - "developer": "gmonsoon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4163, - "hfopenllm_v2/BBH": 0.4509, - "hfopenllm_v2/MATH Level 5": 0.0619, - "hfopenllm_v2/GPQA": 0.2827, - "hfopenllm_v2/MUSR": 0.3754, - "hfopenllm_v2/MMLU-PRO": 0.3108 - } - }, - { - "id": "gmonsoon/SahabatAI-Rebase-8B-Test", - "name": "SahabatAI-Rebase-8B-Test", - "developer": "gmonsoon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5156, - "hfopenllm_v2/BBH": 0.523, - "hfopenllm_v2/MATH Level 5": 0.1148, - "hfopenllm_v2/GPQA": 0.2878, - "hfopenllm_v2/MUSR": 0.4133, - "hfopenllm_v2/MMLU-PRO": 0.3664 - } - }, - { - "id": "gmonsoon/StockSeaLLMs-7B-v1", - "name": "StockSeaLLMs-7B-v1", - "developer": "gmonsoon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4599, - "hfopenllm_v2/BBH": 0.5271, - "hfopenllm_v2/MATH Level 5": 0.1964, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.4214, - "hfopenllm_v2/MMLU-PRO": 0.3952 - } - }, - { - "id": "godlikehhd/alpaca_data_full_2", - "name": "alpaca_data_full_2", - "developer": "godlikehhd", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3178, - "hfopenllm_v2/BBH": 0.4217, - "hfopenllm_v2/MATH Level 5": 0.0929, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.4052, - "hfopenllm_v2/MMLU-PRO": 0.2854 - } - }, - { - "id": "godlikehhd/alpaca_data_full_3B", - "name": "alpaca_data_full_3B", - "developer": "godlikehhd", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3696, - "hfopenllm_v2/BBH": 0.4684, - "hfopenllm_v2/MATH Level 5": 0.1337, - "hfopenllm_v2/GPQA": 0.2777, - "hfopenllm_v2/MUSR": 0.4955, - "hfopenllm_v2/MMLU-PRO": 0.3357 - } - }, - { - "id": "godlikehhd/alpaca_data_ifd_max_2600", - "name": "alpaca_data_ifd_max_2600", - "developer": "godlikehhd", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3043, - "hfopenllm_v2/BBH": 0.4029, - "hfopenllm_v2/MATH Level 5": 0.0989, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.3509, - "hfopenllm_v2/MMLU-PRO": 0.2916 - } - }, - { - "id": "godlikehhd/alpaca_data_ifd_max_2600_3B", - "name": "alpaca_data_ifd_max_2600_3B", - "developer": "godlikehhd", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2982, - "hfopenllm_v2/BBH": 0.4626, - "hfopenllm_v2/MATH Level 5": 0.1594, - "hfopenllm_v2/GPQA": 0.2727, - "hfopenllm_v2/MUSR": 0.4346, - "hfopenllm_v2/MMLU-PRO": 0.3288 - } - }, - { - "id": "godlikehhd/alpaca_data_ifd_me_max_5200", - "name": "alpaca_data_ifd_me_max_5200", - "developer": "godlikehhd", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3683, - "hfopenllm_v2/BBH": 0.4153, - "hfopenllm_v2/MATH Level 5": 0.0974, - "hfopenllm_v2/GPQA": 0.2911, - "hfopenllm_v2/MUSR": 0.3483, - "hfopenllm_v2/MMLU-PRO": 0.2982 - } - }, - { - "id": "godlikehhd/alpaca_data_ifd_min_2600", - "name": "alpaca_data_ifd_min_2600", - "developer": "godlikehhd", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.375, - "hfopenllm_v2/BBH": 0.4219, - "hfopenllm_v2/MATH Level 5": 0.0967, - "hfopenllm_v2/GPQA": 0.2919, - "hfopenllm_v2/MUSR": 0.3656, - "hfopenllm_v2/MMLU-PRO": 0.2893 - } - }, - { - "id": "godlikehhd/alpaca_data_ins_ans_max_5200", - "name": "alpaca_data_ins_ans_max_5200", - "developer": "godlikehhd", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3479, - "hfopenllm_v2/BBH": 0.4098, - "hfopenllm_v2/MATH Level 5": 0.1027, - "hfopenllm_v2/GPQA": 0.2911, - "hfopenllm_v2/MUSR": 0.3602, - "hfopenllm_v2/MMLU-PRO": 0.2901 - } - }, - { - "id": "godlikehhd/alpaca_data_ins_max_5200", - "name": "alpaca_data_ins_max_5200", - "developer": "godlikehhd", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3275, - "hfopenllm_v2/BBH": 0.4155, - "hfopenllm_v2/MATH Level 5": 0.0997, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.3614, - "hfopenllm_v2/MMLU-PRO": 0.2916 - } - }, - { - "id": "godlikehhd/alpaca_data_ins_min_2600", - "name": "alpaca_data_ins_min_2600", - "developer": "godlikehhd", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.333, - "hfopenllm_v2/BBH": 0.4187, - "hfopenllm_v2/MATH Level 5": 0.111, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.3853, - "hfopenllm_v2/MMLU-PRO": 0.288 - } - }, - { - "id": "godlikehhd/alpaca_data_ins_min_5200", - "name": "alpaca_data_ins_min_5200", - "developer": "godlikehhd", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.336, - "hfopenllm_v2/BBH": 0.4289, - "hfopenllm_v2/MATH Level 5": 0.1035, - "hfopenllm_v2/GPQA": 0.2869, - "hfopenllm_v2/MUSR": 0.3906, - "hfopenllm_v2/MMLU-PRO": 0.2949 - } - }, - { - "id": "godlikehhd/alpaca_data_sampled_ifd_5200", - "name": "alpaca_data_sampled_ifd_5200", - "developer": "godlikehhd", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2924, - "hfopenllm_v2/BBH": 0.4033, - "hfopenllm_v2/MATH Level 5": 0.1254, - "hfopenllm_v2/GPQA": 0.3087, - "hfopenllm_v2/MUSR": 0.3521, - "hfopenllm_v2/MMLU-PRO": 0.2896 - } - }, - { - "id": "godlikehhd/alpaca_data_sampled_ifd_new_5200", - "name": "alpaca_data_sampled_ifd_new_5200", - "developer": "godlikehhd", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3663, - "hfopenllm_v2/BBH": 0.4178, - "hfopenllm_v2/MATH Level 5": 0.0944, - "hfopenllm_v2/GPQA": 0.2936, - "hfopenllm_v2/MUSR": 0.3613, - "hfopenllm_v2/MMLU-PRO": 0.2925 - } - }, - { - "id": "godlikehhd/alpaca_data_score_max_0.1_2600", - "name": "alpaca_data_score_max_0.1_2600", - "developer": "godlikehhd", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3288, - "hfopenllm_v2/BBH": 0.4252, - "hfopenllm_v2/MATH Level 5": 0.0989, - "hfopenllm_v2/GPQA": 0.2911, - "hfopenllm_v2/MUSR": 0.3706, - "hfopenllm_v2/MMLU-PRO": 0.2923 - } - }, - { - "id": "godlikehhd/alpaca_data_score_max_0.3_2600", - "name": "alpaca_data_score_max_0.3_2600", - "developer": "godlikehhd", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3375, - "hfopenllm_v2/BBH": 0.4151, - "hfopenllm_v2/MATH Level 5": 0.1035, - "hfopenllm_v2/GPQA": 0.2894, - "hfopenllm_v2/MUSR": 0.3759, - "hfopenllm_v2/MMLU-PRO": 0.2913 - } - }, - { - "id": "godlikehhd/alpaca_data_score_max_0.7_2600", - "name": "alpaca_data_score_max_0.7_2600", - "developer": "godlikehhd", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.364, - "hfopenllm_v2/BBH": 0.4185, - "hfopenllm_v2/MATH Level 5": 0.1073, - "hfopenllm_v2/GPQA": 0.3037, - "hfopenllm_v2/MUSR": 0.3469, - "hfopenllm_v2/MMLU-PRO": 0.2983 - } - }, - { - "id": "godlikehhd/alpaca_data_score_max_2500", - "name": "alpaca_data_score_max_2500", - "developer": "godlikehhd", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3564, - "hfopenllm_v2/BBH": 0.418, - "hfopenllm_v2/MATH Level 5": 0.0952, - "hfopenllm_v2/GPQA": 0.2953, - "hfopenllm_v2/MUSR": 0.3627, - "hfopenllm_v2/MMLU-PRO": 0.294 - } - }, - { - "id": "godlikehhd/alpaca_data_score_max_2600_3B", - "name": "alpaca_data_score_max_2600_3B", - "developer": "godlikehhd", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3358, - "hfopenllm_v2/BBH": 0.4716, - "hfopenllm_v2/MATH Level 5": 0.1548, - "hfopenllm_v2/GPQA": 0.2651, - "hfopenllm_v2/MUSR": 0.4474, - "hfopenllm_v2/MMLU-PRO": 0.3342 - } - }, - { - "id": "godlikehhd/alpaca_data_score_max_5200", - "name": "alpaca_data_score_max_5200", - "developer": "godlikehhd", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3445, - "hfopenllm_v2/BBH": 0.4242, - "hfopenllm_v2/MATH Level 5": 0.0974, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.3878, - "hfopenllm_v2/MMLU-PRO": 0.2945 - } - }, - { - "id": "godlikehhd/ifd_2500_qwen", - "name": "ifd_2500_qwen", - "developer": "godlikehhd", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3365, - "hfopenllm_v2/BBH": 0.4298, - "hfopenllm_v2/MATH Level 5": 0.0982, - "hfopenllm_v2/GPQA": 0.2953, - "hfopenllm_v2/MUSR": 0.3615, - "hfopenllm_v2/MMLU-PRO": 0.2921 - } - }, - { - "id": "godlikehhd/ifd_new_correct_all_sample_2500_qwen", - "name": "ifd_new_correct_all_sample_2500_qwen", - "developer": "godlikehhd", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3376, - "hfopenllm_v2/BBH": 0.402, - "hfopenllm_v2/MATH Level 5": 0.0959, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.3562, - "hfopenllm_v2/MMLU-PRO": 0.2889 - } - }, - { - "id": "godlikehhd/ifd_new_correct_sample_2500_qwen", - "name": "ifd_new_correct_sample_2500_qwen", - "developer": "godlikehhd", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3397, - "hfopenllm_v2/BBH": 0.411, - "hfopenllm_v2/MATH Level 5": 0.1042, - "hfopenllm_v2/GPQA": 0.3079, - "hfopenllm_v2/MUSR": 0.3627, - "hfopenllm_v2/MMLU-PRO": 0.2932 - } - }, - { - "id": "godlikehhd/ifd_new_qwen_2500", - "name": "ifd_new_qwen_2500", - "developer": "godlikehhd", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.324, - "hfopenllm_v2/BBH": 0.416, - "hfopenllm_v2/MATH Level 5": 0.1118, - "hfopenllm_v2/GPQA": 0.3003, - "hfopenllm_v2/MUSR": 0.359, - "hfopenllm_v2/MMLU-PRO": 0.2911 - } - }, - { - "id": "godlikehhd/qwen-2.5-1.5b-cherry", - "name": "qwen-2.5-1.5b-cherry", - "developer": "godlikehhd", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2893, - "hfopenllm_v2/BBH": 0.4036, - "hfopenllm_v2/MATH Level 5": 0.102, - "hfopenllm_v2/GPQA": 0.3003, - "hfopenllm_v2/MUSR": 0.3456, - "hfopenllm_v2/MMLU-PRO": 0.2923 - } - }, - { - "id": "godlikehhd/qwen_2.5-1.5b-cherry_new", - "name": "qwen_2.5-1.5b-cherry_new", - "developer": "godlikehhd", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.312, - "hfopenllm_v2/BBH": 0.415, - "hfopenllm_v2/MATH Level 5": 0.0967, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.3496, - "hfopenllm_v2/MMLU-PRO": 0.2894 - } - }, - { - "id": "godlikehhd/qwen_full_data_alpaca", - "name": "qwen_full_data_alpaca", - "developer": "godlikehhd", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3136, - "hfopenllm_v2/BBH": 0.4229, - "hfopenllm_v2/MATH Level 5": 0.0921, - "hfopenllm_v2/GPQA": 0.2928, - "hfopenllm_v2/MUSR": 0.4052, - "hfopenllm_v2/MMLU-PRO": 0.2851 - } - }, - { - "id": "godlikehhd/qwen_ins_ans_2500", - "name": "qwen_ins_ans_2500", - "developer": "godlikehhd", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2698, - "hfopenllm_v2/BBH": 0.4074, - "hfopenllm_v2/MATH Level 5": 0.114, - "hfopenllm_v2/GPQA": 0.2919, - "hfopenllm_v2/MUSR": 0.3589, - "hfopenllm_v2/MMLU-PRO": 0.2809 - } - }, - { - "id": "Goekdeniz-Guelmez/j.o.s.i.e.v4o-1.5b-dpo-stage1-v1", - "name": "j.o.s.i.e.v4o-1.5b-dpo-stage1-v1", - "developer": "Goekdeniz-Guelmez", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4188, - "hfopenllm_v2/BBH": 0.4124, - "hfopenllm_v2/MATH Level 5": 0.1201, - "hfopenllm_v2/GPQA": 0.2508, - "hfopenllm_v2/MUSR": 0.3529, - "hfopenllm_v2/MMLU-PRO": 0.2555 - } - }, - { - "id": "Goekdeniz-Guelmez/josie-3b-v6.0", - "name": "josie-3b-v6.0", - "developer": "Goekdeniz-Guelmez", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.601, - "hfopenllm_v2/BBH": 0.4496, - "hfopenllm_v2/MATH Level 5": 0.2938, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.3861, - "hfopenllm_v2/MMLU-PRO": 0.322 - } - }, - { - "id": "Goekdeniz-Guelmez/josie-7b-v6.0", - "name": "josie-7b-v6.0", - "developer": "Goekdeniz-Guelmez", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7412, - "hfopenllm_v2/BBH": 0.5105, - "hfopenllm_v2/MATH Level 5": 0.4358, - "hfopenllm_v2/GPQA": 0.2827, - "hfopenllm_v2/MUSR": 0.4154, - "hfopenllm_v2/MMLU-PRO": 0.3807 - } - }, - { - "id": "Goekdeniz-Guelmez/josie-7b-v6.0-step2000", - "name": "josie-7b-v6.0-step2000", - "developer": "Goekdeniz-Guelmez", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7628, - "hfopenllm_v2/BBH": 0.5098, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2802, - "hfopenllm_v2/MUSR": 0.4579, - "hfopenllm_v2/MMLU-PRO": 0.4033 - } - }, - { - "id": "Goekdeniz-Guelmez/Josiefied-Qwen2.5-0.5B-Instruct-abliterated-v1", - "name": "Josiefied-Qwen2.5-0.5B-Instruct-abliterated-v1", - "developer": "Goekdeniz-Guelmez", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3472, - "hfopenllm_v2/BBH": 0.3268, - "hfopenllm_v2/MATH Level 5": 0.0891, - "hfopenllm_v2/GPQA": 0.2517, - "hfopenllm_v2/MUSR": 0.3262, - "hfopenllm_v2/MMLU-PRO": 0.1641 - } - }, - { - "id": "Goekdeniz-Guelmez/Josiefied-Qwen2.5-1.5B-Instruct-abliterated-v1", - "name": "Josiefied-Qwen2.5-1.5B-Instruct-abliterated-v1", - "developer": "Goekdeniz-Guelmez", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4769, - "hfopenllm_v2/BBH": 0.4186, - "hfopenllm_v2/MATH Level 5": 0.2085, - "hfopenllm_v2/GPQA": 0.2433, - "hfopenllm_v2/MUSR": 0.3675, - "hfopenllm_v2/MMLU-PRO": 0.2783 - } - }, - { - "id": "Goekdeniz-Guelmez/Josiefied-Qwen2.5-1.5B-Instruct-abliterated-v2", - "name": "Josiefied-Qwen2.5-1.5B-Instruct-abliterated-v2", - "developer": "Goekdeniz-Guelmez", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4216, - "hfopenllm_v2/BBH": 0.4042, - "hfopenllm_v2/MATH Level 5": 0.1269, - "hfopenllm_v2/GPQA": 0.2399, - "hfopenllm_v2/MUSR": 0.3769, - "hfopenllm_v2/MMLU-PRO": 0.2562 - } - }, - { - "id": "Goekdeniz-Guelmez/Josiefied-Qwen2.5-1.5B-Instruct-abliterated-v3", - "name": "Josiefied-Qwen2.5-1.5B-Instruct-abliterated-v3", - "developer": "Goekdeniz-Guelmez", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4253, - "hfopenllm_v2/BBH": 0.4053, - "hfopenllm_v2/MATH Level 5": 0.1307, - "hfopenllm_v2/GPQA": 0.2433, - "hfopenllm_v2/MUSR": 0.3702, - "hfopenllm_v2/MMLU-PRO": 0.2556 - } - }, - { - "id": "Goekdeniz-Guelmez/Josiefied-Qwen2.5-14B-Instruct-abliterated-v4", - "name": "Josiefied-Qwen2.5-14B-Instruct-abliterated-v4", - "developer": "Goekdeniz-Guelmez", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8292, - "hfopenllm_v2/BBH": 0.6356, - "hfopenllm_v2/MATH Level 5": 0.5423, - "hfopenllm_v2/GPQA": 0.3423, - "hfopenllm_v2/MUSR": 0.4287, - "hfopenllm_v2/MMLU-PRO": 0.5018 - } - }, - { - "id": "Goekdeniz-Guelmez/Josiefied-Qwen2.5-7B-Instruct-abliterated-v2", - "name": "Josiefied-Qwen2.5-7B-Instruct-abliterated-v2", - "developer": "Goekdeniz-Guelmez", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7814, - "hfopenllm_v2/BBH": 0.531, - "hfopenllm_v2/MATH Level 5": 0.4532, - "hfopenllm_v2/GPQA": 0.2987, - "hfopenllm_v2/MUSR": 0.4354, - "hfopenllm_v2/MMLU-PRO": 0.412 - } - }, - { - "id": "google/codegemma-1.1-2b", - "name": "codegemma-1.1-2b", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2294, - "hfopenllm_v2/BBH": 0.3353, - "hfopenllm_v2/MATH Level 5": 0.0128, - "hfopenllm_v2/GPQA": 0.2651, - "hfopenllm_v2/MUSR": 0.3871, - "hfopenllm_v2/MMLU-PRO": 0.1278 - } - }, - { - "id": "google/flame-1.0-24B-july-2024", - "name": "google/flame-1.0-24B-july-2024", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.8781, - "reward-bench/Chat": 0.9218, - "reward-bench/Chat Hard": 0.7566, - "reward-bench/Safety": 0.8959, - "reward-bench/Reasoning": 0.938 - } - }, - { - "id": "google/flan-t5-base", - "name": "flan-t5-base", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1891, - "hfopenllm_v2/BBH": 0.3526, - "hfopenllm_v2/MATH Level 5": 0.0106, - "hfopenllm_v2/GPQA": 0.2383, - "hfopenllm_v2/MUSR": 0.3671, - "hfopenllm_v2/MMLU-PRO": 0.1357 - } - }, - { - "id": "google/flan-t5-large", - "name": "flan-t5-large", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2201, - "hfopenllm_v2/BBH": 0.4153, - "hfopenllm_v2/MATH Level 5": 0.0144, - "hfopenllm_v2/GPQA": 0.2508, - "hfopenllm_v2/MUSR": 0.4083, - "hfopenllm_v2/MMLU-PRO": 0.1709 - } - }, - { - "id": "google/flan-t5-small", - "name": "flan-t5-small", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1524, - "hfopenllm_v2/BBH": 0.3283, - "hfopenllm_v2/MATH Level 5": 0.0076, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.4123, - "hfopenllm_v2/MMLU-PRO": 0.1233 - } - }, - { - "id": "google/flan-t5-xl", - "name": "flan-t5-xl", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2207, - "hfopenllm_v2/BBH": 0.4537, - "hfopenllm_v2/MATH Level 5": 0.0008, - "hfopenllm_v2/GPQA": 0.2458, - "hfopenllm_v2/MUSR": 0.422, - "hfopenllm_v2/MMLU-PRO": 0.2142 - } - }, - { - "id": "google/flan-t5-xxl", - "name": "flan-t5-xxl", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.22, - "hfopenllm_v2/BBH": 0.5066, - "hfopenllm_v2/MATH Level 5": 0.0106, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.4218, - "hfopenllm_v2/MMLU-PRO": 0.2343 - } - }, - { - "id": "google/flan-ul2", - "name": "flan-ul2", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2393, - "hfopenllm_v2/BBH": 0.5054, - "hfopenllm_v2/MATH Level 5": 0.0091, - "hfopenllm_v2/GPQA": 0.2878, - "hfopenllm_v2/MUSR": 0.3844, - "hfopenllm_v2/MMLU-PRO": 0.2493 - } - }, - { - "id": "google/Gemini 2.5 Flash", - "name": "Gemini 2.5 Flash", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "ace/Overall Score": 0.38, - "ace/Gaming Score": 0.284, - "apex-v1/Overall Score": 0.604 - } - }, - { - "id": "google/Gemini 2.5 Pro", - "name": "Gemini 2.5 Pro", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "ace/Overall Score": 0.4, - "ace/Gaming Score": 0.285 - } - }, - { - "id": "google/Gemini 3 Flash", - "name": "Gemini 3 Flash", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "apex-agents/Overall Pass@1": 0.24, - "apex-agents/Overall Pass@8": 0.367, - "apex-agents/Overall Mean Score": 0.395, - "apex-agents/Investment Banking Pass@1": 0.267, - "apex-agents/Management Consulting Pass@1": 0.193, - "apex-agents/Corporate Law Pass@1": 0.259, - "apex-agents/Corporate Lawyer Mean Score": 0.524, - "ace/Gaming Score": 0.415, - "apex-v1/Overall Score": 0.64, - "apex-v1/Consulting Score": 0.64 - } - }, - { - "id": "google/Gemini 3 Pro", - "name": "Gemini 3 Pro", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "apex-agents/Overall Pass@1": 0.184, - "apex-agents/Overall Pass@8": 0.373, - "apex-agents/Overall Mean Score": 0.341, - "apex-agents/Investment Banking Pass@1": 0.188, - "apex-agents/Management Consulting Pass@1": 0.124, - "apex-agents/Corporate Law Pass@1": 0.239, - "apex-agents/Corporate Lawyer Mean Score": 0.487, - "ace/Overall Score": 0.47, - "ace/Gaming Score": 0.509, - "apex-v1/Overall Score": 0.643, - "apex-v1/Consulting Score": 0.64, - "apex-v1/Investment Banking Score": 0.63 - } - }, - { - "id": "google/Gemini 3.1 Pro", - "name": "Gemini 3.1 Pro", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "apex-agents/Overall Pass@1": 0.335, - "apex-agents/Corporate Lawyer Mean Score": 0.494 - } - }, - { - "id": "google/gemini-1.0-pro-001", - "name": "Gemini 1.0 Pro 001", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_mmlu/MMLU All Subjects": 0.7, - "helm_mmlu/Abstract Algebra": 0.34, - "helm_mmlu/Anatomy": 0.652, - "helm_mmlu/College Physics": 0.333, - "helm_mmlu/Computer Security": 0.84, - "helm_mmlu/Econometrics": 0.553, - "helm_mmlu/Global Facts": 0.49, - "helm_mmlu/Jurisprudence": 0.861, - "helm_mmlu/Philosophy": 0.762, - "helm_mmlu/Professional Psychology": 0.752, - "helm_mmlu/Us Foreign Policy": 0.89, - "helm_mmlu/Astronomy": 0.796, - "helm_mmlu/Business Ethics": 0.69, - "helm_mmlu/Clinical Knowledge": 0.758, - "helm_mmlu/Conceptual Physics": 0.706, - "helm_mmlu/Electrical Engineering": 0.69, - "helm_mmlu/Elementary Mathematics": 0.476, - "helm_mmlu/Formal Logic": 0.468, - "helm_mmlu/High School World History": 0.865, - "helm_mmlu/Human Sexuality": 0.618, - "helm_mmlu/International Law": 0.876, - "helm_mmlu/Logical Fallacies": 0.804, - "helm_mmlu/Machine Learning": 0.527, - "helm_mmlu/Management": 0.845, - "helm_mmlu/Marketing": 0.91, - "helm_mmlu/Medical Genetics": 0.8, - "helm_mmlu/Miscellaneous": 0.851, - "helm_mmlu/Moral Scenarios": 0.46, - "helm_mmlu/Nutrition": 0.788, - "helm_mmlu/Prehistory": 0.802, - "helm_mmlu/Public Relations": 0.691, - "helm_mmlu/Security Studies": 0.804, - "helm_mmlu/Sociology": 0.9, - "helm_mmlu/Virology": 0.536, - "helm_mmlu/World Religions": 0.86, - "helm_mmlu/Mean win rate": 0.677 - } - }, - { - "id": "google/gemini-1.0-pro-002", - "name": "Gemini 1.0 Pro 002", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.422, - "helm_lite/NarrativeQA": 0.751, - "helm_lite/NaturalQuestions (closed-book)": 0.391, - "helm_lite/OpenbookQA": 0.788, - "helm_lite/MMLU": 0.534, - "helm_lite/MATH": 0.665, - "helm_lite/GSM8K": 0.816, - "helm_lite/LegalBench": 0.475, - "helm_lite/MedQA": 0.483, - "helm_lite/WMT 2014": 0.194 - } - }, - { - "id": "google/gemini-1.5-flash-001", - "name": "Gemini 1.5 Flash 001", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.667, - "helm_lite/NarrativeQA": 0.783, - "helm_lite/NaturalQuestions (closed-book)": 0.332, - "helm_lite/OpenbookQA": 0.928, - "helm_lite/MMLU": 0.703, - "helm_lite/MATH": 0.753, - "helm_lite/GSM8K": 0.785, - "helm_lite/LegalBench": 0.661, - "helm_lite/MedQA": 0.68, - "helm_lite/WMT 2014": 0.225, - "helm_mmlu/MMLU All Subjects": 0.779, - "helm_mmlu/Abstract Algebra": 0.58, - "helm_mmlu/Anatomy": 0.8, - "helm_mmlu/College Physics": 0.696, - "helm_mmlu/Computer Security": 0.79, - "helm_mmlu/Econometrics": 0.614, - "helm_mmlu/Global Facts": 0.53, - "helm_mmlu/Jurisprudence": 0.889, - "helm_mmlu/Philosophy": 0.791, - "helm_mmlu/Professional Psychology": 0.828, - "helm_mmlu/Us Foreign Policy": 0.93, - "helm_mmlu/Astronomy": 0.882, - "helm_mmlu/Business Ethics": 0.81, - "helm_mmlu/Clinical Knowledge": 0.834, - "helm_mmlu/Conceptual Physics": 0.851, - "helm_mmlu/Electrical Engineering": 0.8, - "helm_mmlu/Elementary Mathematics": 0.754, - "helm_mmlu/Formal Logic": 0.627, - "helm_mmlu/High School World History": 0.907, - "helm_mmlu/Human Sexuality": 0.374, - "helm_mmlu/International Law": 0.901, - "helm_mmlu/Logical Fallacies": 0.853, - "helm_mmlu/Machine Learning": 0.571, - "helm_mmlu/Management": 0.864, - "helm_mmlu/Marketing": 0.94, - "helm_mmlu/Medical Genetics": 0.86, - "helm_mmlu/Miscellaneous": 0.886, - "helm_mmlu/Moral Scenarios": 0.637, - "helm_mmlu/Nutrition": 0.82, - "helm_mmlu/Prehistory": 0.867, - "helm_mmlu/Public Relations": 0.764, - "helm_mmlu/Security Studies": 0.808, - "helm_mmlu/Sociology": 0.915, - "helm_mmlu/Virology": 0.566, - "helm_mmlu/World Religions": 0.883, - "helm_mmlu/Mean win rate": 0.47, - "reward-bench/Score": 0.8054, - "reward-bench/Chat": 0.9218, - "reward-bench/Chat Hard": 0.6349, - "reward-bench/Safety": 0.8696, - "reward-bench/Reasoning": 0.8512, - "reward-bench/Prior Sets (0.5 weight)": 0.6937 - } - }, - { - "id": "google/gemini-1.5-flash-002", - "name": "Gemini 1.5 Flash 002", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.609, - "helm_capabilities/MMLU-Pro": 0.678, - "helm_capabilities/GPQA": 0.437, - "helm_capabilities/IFEval": 0.831, - "helm_capabilities/WildBench": 0.792, - "helm_capabilities/Omni-MATH": 0.305, - "helm_lite/Mean win rate": 0.573, - "helm_lite/NarrativeQA": 0.746, - "helm_lite/NaturalQuestions (closed-book)": 0.323, - "helm_lite/OpenbookQA": 0.914, - "helm_lite/MMLU": 0.679, - "helm_lite/MATH": 0.908, - "helm_lite/GSM8K": 0.328, - "helm_lite/LegalBench": 0.67, - "helm_lite/MedQA": 0.656, - "helm_lite/WMT 2014": 0.212, - "helm_mmlu/MMLU All Subjects": 0.739, - "helm_mmlu/Abstract Algebra": 0.63, - "helm_mmlu/Anatomy": 0.793, - "helm_mmlu/College Physics": 0.637, - "helm_mmlu/Computer Security": 0.72, - "helm_mmlu/Econometrics": 0.675, - "helm_mmlu/Global Facts": 0.47, - "helm_mmlu/Jurisprudence": 0.852, - "helm_mmlu/Philosophy": 0.797, - "helm_mmlu/Professional Psychology": 0.806, - "helm_mmlu/Us Foreign Policy": 0.81, - "helm_mmlu/Astronomy": 0.895, - "helm_mmlu/Business Ethics": 0.27, - "helm_mmlu/Clinical Knowledge": 0.792, - "helm_mmlu/Conceptual Physics": 0.851, - "helm_mmlu/Electrical Engineering": 0.772, - "helm_mmlu/Elementary Mathematics": 0.704, - "helm_mmlu/Formal Logic": 0.595, - "helm_mmlu/High School World History": 0.869, - "helm_mmlu/Human Sexuality": 0.847, - "helm_mmlu/International Law": 0.752, - "helm_mmlu/Logical Fallacies": 0.859, - "helm_mmlu/Machine Learning": 0.616, - "helm_mmlu/Management": 0.893, - "helm_mmlu/Marketing": 0.953, - "helm_mmlu/Medical Genetics": 0.89, - "helm_mmlu/Miscellaneous": 0.9, - "helm_mmlu/Moral Scenarios": 0.676, - "helm_mmlu/Nutrition": 0.588, - "helm_mmlu/Prehistory": 0.762, - "helm_mmlu/Public Relations": 0.7, - "helm_mmlu/Security Studies": 0.547, - "helm_mmlu/Sociology": 0.851, - "helm_mmlu/Virology": 0.524, - "helm_mmlu/World Religions": 0.865, - "helm_mmlu/Mean win rate": 0.817 - } - }, - { - "id": "google/gemini-1.5-flash-8b", - "name": "google/gemini-1.5-flash-8b", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.4851, - "reward-bench/Factuality": 0.4611, - "reward-bench/Precise IF": 0.3625, - "reward-bench/Math": 0.5082, - "reward-bench/Safety": 0.6622, - "reward-bench/Focus": 0.6747, - "reward-bench/Ties": 0.2421 - } - }, - { - "id": "google/gemini-1.5-flash-preview-0514", - "name": "Gemini 1.5 Flash 0514 preview", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_mmlu/MMLU All Subjects": 0.778, - "helm_mmlu/Abstract Algebra": 0.56, - "helm_mmlu/Anatomy": 0.807, - "helm_mmlu/College Physics": 0.667, - "helm_mmlu/Computer Security": 0.77, - "helm_mmlu/Econometrics": 0.64, - "helm_mmlu/Global Facts": 0.55, - "helm_mmlu/Jurisprudence": 0.889, - "helm_mmlu/Philosophy": 0.807, - "helm_mmlu/Professional Psychology": 0.825, - "helm_mmlu/Us Foreign Policy": 0.93, - "helm_mmlu/Astronomy": 0.868, - "helm_mmlu/Business Ethics": 0.82, - "helm_mmlu/Clinical Knowledge": 0.838, - "helm_mmlu/Conceptual Physics": 0.855, - "helm_mmlu/Electrical Engineering": 0.814, - "helm_mmlu/Elementary Mathematics": 0.778, - "helm_mmlu/Formal Logic": 0.611, - "helm_mmlu/High School World History": 0.907, - "helm_mmlu/Human Sexuality": 0.374, - "helm_mmlu/International Law": 0.876, - "helm_mmlu/Logical Fallacies": 0.853, - "helm_mmlu/Machine Learning": 0.562, - "helm_mmlu/Management": 0.854, - "helm_mmlu/Marketing": 0.936, - "helm_mmlu/Medical Genetics": 0.86, - "helm_mmlu/Miscellaneous": 0.884, - "helm_mmlu/Moral Scenarios": 0.631, - "helm_mmlu/Nutrition": 0.801, - "helm_mmlu/Prehistory": 0.867, - "helm_mmlu/Public Relations": 0.773, - "helm_mmlu/Security Studies": 0.812, - "helm_mmlu/Sociology": 0.9, - "helm_mmlu/Virology": 0.566, - "helm_mmlu/World Religions": 0.871, - "helm_mmlu/Mean win rate": 0.713 - } - }, - { - "id": "google/gemini-1.5-pro-001", - "name": "Gemini 1.5 Pro 001", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.739, - "helm_lite/NarrativeQA": 0.783, - "helm_lite/NaturalQuestions (closed-book)": 0.378, - "helm_lite/OpenbookQA": 0.902, - "helm_lite/MMLU": 0.772, - "helm_lite/MATH": 0.825, - "helm_lite/GSM8K": 0.836, - "helm_lite/LegalBench": 0.757, - "helm_lite/MedQA": 0.692, - "helm_lite/WMT 2014": 0.189, - "helm_mmlu/MMLU All Subjects": 0.827, - "helm_mmlu/Abstract Algebra": 0.75, - "helm_mmlu/Anatomy": 0.83, - "helm_mmlu/College Physics": 0.745, - "helm_mmlu/Computer Security": 0.83, - "helm_mmlu/Econometrics": 0.728, - "helm_mmlu/Global Facts": 0.66, - "helm_mmlu/Jurisprudence": 0.889, - "helm_mmlu/Philosophy": 0.871, - "helm_mmlu/Professional Psychology": 0.894, - "helm_mmlu/Us Foreign Policy": 0.93, - "helm_mmlu/Astronomy": 0.914, - "helm_mmlu/Business Ethics": 0.8, - "helm_mmlu/Clinical Knowledge": 0.853, - "helm_mmlu/Conceptual Physics": 0.949, - "helm_mmlu/Electrical Engineering": 0.745, - "helm_mmlu/Elementary Mathematics": 0.939, - "helm_mmlu/Formal Logic": 0.706, - "helm_mmlu/High School World History": 0.924, - "helm_mmlu/Human Sexuality": 0.374, - "helm_mmlu/International Law": 0.917, - "helm_mmlu/Logical Fallacies": 0.896, - "helm_mmlu/Machine Learning": 0.652, - "helm_mmlu/Management": 0.922, - "helm_mmlu/Marketing": 0.932, - "helm_mmlu/Medical Genetics": 0.91, - "helm_mmlu/Miscellaneous": 0.958, - "helm_mmlu/Moral Scenarios": 0.739, - "helm_mmlu/Nutrition": 0.879, - "helm_mmlu/Prehistory": 0.87, - "helm_mmlu/Public Relations": 0.818, - "helm_mmlu/Security Studies": 0.873, - "helm_mmlu/Sociology": 0.92, - "helm_mmlu/Virology": 0.554, - "helm_mmlu/World Religions": 0.854, - "helm_mmlu/Mean win rate": 0.349 - } - }, - { - "id": "google/gemini-1.5-pro-002", - "name": "Gemini 1.5 Pro 002", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.657, - "helm_capabilities/MMLU-Pro": 0.737, - "helm_capabilities/GPQA": 0.534, - "helm_capabilities/IFEval": 0.837, - "helm_capabilities/WildBench": 0.813, - "helm_capabilities/Omni-MATH": 0.364, - "helm_lite/Mean win rate": 0.842, - "helm_lite/NarrativeQA": 0.756, - "helm_lite/NaturalQuestions (closed-book)": 0.455, - "helm_lite/OpenbookQA": 0.952, - "helm_lite/MMLU": 0.795, - "helm_lite/MATH": 0.92, - "helm_lite/GSM8K": 0.817, - "helm_lite/LegalBench": 0.747, - "helm_lite/MedQA": 0.771, - "helm_lite/WMT 2014": 0.231, - "helm_mmlu/MMLU All Subjects": 0.869, - "helm_mmlu/Abstract Algebra": 0.82, - "helm_mmlu/Anatomy": 0.83, - "helm_mmlu/College Physics": 0.863, - "helm_mmlu/Computer Security": 0.85, - "helm_mmlu/Econometrics": 0.693, - "helm_mmlu/Global Facts": 0.77, - "helm_mmlu/Jurisprudence": 0.898, - "helm_mmlu/Philosophy": 0.887, - "helm_mmlu/Professional Psychology": 0.912, - "helm_mmlu/Us Foreign Policy": 0.94, - "helm_mmlu/Astronomy": 0.934, - "helm_mmlu/Business Ethics": 0.84, - "helm_mmlu/Clinical Knowledge": 0.906, - "helm_mmlu/Conceptual Physics": 0.945, - "helm_mmlu/Electrical Engineering": 0.855, - "helm_mmlu/Elementary Mathematics": 0.942, - "helm_mmlu/Formal Logic": 0.754, - "helm_mmlu/High School World History": 0.937, - "helm_mmlu/Human Sexuality": 0.878, - "helm_mmlu/International Law": 0.917, - "helm_mmlu/Logical Fallacies": 0.902, - "helm_mmlu/Machine Learning": 0.83, - "helm_mmlu/Management": 0.903, - "helm_mmlu/Marketing": 0.962, - "helm_mmlu/Medical Genetics": 0.92, - "helm_mmlu/Miscellaneous": 0.959, - "helm_mmlu/Moral Scenarios": 0.792, - "helm_mmlu/Nutrition": 0.886, - "helm_mmlu/Prehistory": 0.926, - "helm_mmlu/Public Relations": 0.809, - "helm_mmlu/Security Studies": 0.857, - "helm_mmlu/Sociology": 0.95, - "helm_mmlu/Virology": 0.566, - "helm_mmlu/World Religions": 0.889, - "helm_mmlu/Mean win rate": 0.334 - } - }, - { - "id": "google/gemini-1.5-pro-0514", - "name": "google/gemini-1.5-pro-0514", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.882, - "reward-bench/Chat": 0.9232, - "reward-bench/Chat Hard": 0.8059, - "reward-bench/Safety": 0.8791, - "reward-bench/Reasoning": 0.9199 - } - }, - { - "id": "google/gemini-1.5-pro-0924", - "name": "google/gemini-1.5-pro-0924", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.8678, - "reward-bench/Chat": 0.9413, - "reward-bench/Chat Hard": 0.7697, - "reward-bench/Safety": 0.8581, - "reward-bench/Reasoning": 0.9022 - } - }, - { - "id": "google/gemini-1.5-pro-preview-0409", - "name": "Gemini 1.5 Pro 0409 preview", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_mmlu/MMLU All Subjects": 0.81, - "helm_mmlu/Abstract Algebra": 0.6, - "helm_mmlu/Anatomy": 0.77, - "helm_mmlu/College Physics": 0.804, - "helm_mmlu/Computer Security": 0.81, - "helm_mmlu/Econometrics": 0.737, - "helm_mmlu/Global Facts": 0.66, - "helm_mmlu/Jurisprudence": 0.87, - "helm_mmlu/Philosophy": 0.846, - "helm_mmlu/Professional Psychology": 0.866, - "helm_mmlu/Us Foreign Policy": 0.94, - "helm_mmlu/Astronomy": 0.914, - "helm_mmlu/Business Ethics": 0.8, - "helm_mmlu/Clinical Knowledge": 0.868, - "helm_mmlu/Conceptual Physics": 0.915, - "helm_mmlu/Electrical Engineering": 0.772, - "helm_mmlu/Elementary Mathematics": 0.884, - "helm_mmlu/Formal Logic": 0.643, - "helm_mmlu/High School World History": 0.924, - "helm_mmlu/Human Sexuality": 0.397, - "helm_mmlu/International Law": 0.917, - "helm_mmlu/Logical Fallacies": 0.859, - "helm_mmlu/Machine Learning": 0.67, - "helm_mmlu/Management": 0.874, - "helm_mmlu/Marketing": 0.953, - "helm_mmlu/Medical Genetics": 0.91, - "helm_mmlu/Miscellaneous": 0.928, - "helm_mmlu/Moral Scenarios": 0.696, - "helm_mmlu/Nutrition": 0.846, - "helm_mmlu/Prehistory": 0.886, - "helm_mmlu/Public Relations": 0.755, - "helm_mmlu/Security Studies": 0.849, - "helm_mmlu/Sociology": 0.925, - "helm_mmlu/Virology": 0.584, - "helm_mmlu/World Religions": 0.877, - "helm_mmlu/Mean win rate": 0.118 - } - }, - { - "id": "google/gemini-2-5-flash-fc", - "name": "Gemini-2.5-Flash (FC)", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 15.0, - "bfcl/bfcl.overall.overall_accuracy": 56.24, - "bfcl/bfcl.overall.total_cost_usd": 26.36, - "bfcl/bfcl.overall.latency_mean_s": 2.99, - "bfcl/bfcl.overall.latency_std_s": 9.22, - "bfcl/bfcl.overall.latency_p95_s": 5.62, - "bfcl/bfcl.non_live.ast_accuracy": 84.96, - "bfcl/bfcl.non_live.simple_ast_accuracy": 74.33, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 92.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 94.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 79.5, - "bfcl/bfcl.live.live_accuracy": 74.39, - "bfcl/bfcl.live.live_simple_ast_accuracy": 85.27, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 71.7, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 81.25, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 70.83, - "bfcl/bfcl.multi_turn.accuracy": 36.25, - "bfcl/bfcl.multi_turn.base_accuracy": 41.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 36.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 32.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 35.5, - "bfcl/bfcl.web_search.accuracy": 59.0, - "bfcl/bfcl.web_search.base_accuracy": 59.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 59.0, - "bfcl/bfcl.memory.accuracy": 41.29, - "bfcl/bfcl.memory.kv_accuracy": 19.35, - "bfcl/bfcl.memory.vector_accuracy": 50.32, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 54.19, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 75.0, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 93.67 - } - }, - { - "id": "google/gemini-2-5-flash-lite-fc", - "name": "Gemini-2.5-Flash-Lite (FC)", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 52.0, - "bfcl/bfcl.overall.overall_accuracy": 36.87, - "bfcl/bfcl.overall.total_cost_usd": 7.55, - "bfcl/bfcl.overall.latency_mean_s": 1.18, - "bfcl/bfcl.overall.latency_std_s": 8.06, - "bfcl/bfcl.overall.latency_p95_s": 1.67, - "bfcl/bfcl.non_live.ast_accuracy": 86.6, - "bfcl/bfcl.non_live.simple_ast_accuracy": 70.92, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 90.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 93.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 92.0, - "bfcl/bfcl.live.live_accuracy": 65.8, - "bfcl/bfcl.live.live_simple_ast_accuracy": 73.26, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 63.82, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 75.0, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 66.67, - "bfcl/bfcl.multi_turn.accuracy": 13.5, - "bfcl/bfcl.multi_turn.base_accuracy": 20.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 1.5, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 15.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 17.5, - "bfcl/bfcl.web_search.accuracy": 21.0, - "bfcl/bfcl.web_search.base_accuracy": 26.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 16.0, - "bfcl/bfcl.memory.accuracy": 20.65, - "bfcl/bfcl.memory.kv_accuracy": 3.87, - "bfcl/bfcl.memory.vector_accuracy": 6.45, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 51.61, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 43.75, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 92.5 - } - }, - { - "id": "google/gemini-2-5-flash-lite-prompt", - "name": "Gemini-2.5-Flash-Lite (Prompt)", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 73.0, - "bfcl/bfcl.overall.overall_accuracy": 28.03, - "bfcl/bfcl.overall.total_cost_usd": 7.05, - "bfcl/bfcl.overall.latency_mean_s": 1.0, - "bfcl/bfcl.overall.latency_std_s": 4.75, - "bfcl/bfcl.overall.latency_p95_s": 1.4, - "bfcl/bfcl.non_live.ast_accuracy": 83.9, - "bfcl/bfcl.non_live.simple_ast_accuracy": 70.08, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 86.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 90.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 89.5, - "bfcl/bfcl.live.live_accuracy": 54.85, - "bfcl/bfcl.live.live_simple_ast_accuracy": 67.05, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 51.66, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 75.0, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 50.0, - "bfcl/bfcl.multi_turn.accuracy": 7.63, - "bfcl/bfcl.multi_turn.base_accuracy": 10.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 5.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 6.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 9.0, - "bfcl/bfcl.web_search.accuracy": 0.0, - "bfcl/bfcl.web_search.base_accuracy": 0.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 0.0, - "bfcl/bfcl.memory.accuracy": 12.69, - "bfcl/bfcl.memory.kv_accuracy": 1.94, - "bfcl/bfcl.memory.vector_accuracy": 6.45, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 29.68, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 50.0, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 93.33, - "bfcl/bfcl.format_sensitivity.max_delta": 25.5, - "bfcl/bfcl.format_sensitivity.stddev": 6.68 - } - }, - { - "id": "google/gemini-2-5-flash-prompt", - "name": "Gemini-2.5-Flash (Prompt)", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 26.0, - "bfcl/bfcl.overall.overall_accuracy": 50.9, - "bfcl/bfcl.overall.total_cost_usd": 33.45, - "bfcl/bfcl.overall.latency_mean_s": 3.18, - "bfcl/bfcl.overall.latency_std_s": 4.44, - "bfcl/bfcl.overall.latency_p95_s": 6.09, - "bfcl/bfcl.non_live.ast_accuracy": 88.08, - "bfcl/bfcl.non_live.simple_ast_accuracy": 77.33, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 91.5, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 96.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 87.5, - "bfcl/bfcl.live.live_accuracy": 78.16, - "bfcl/bfcl.live.live_simple_ast_accuracy": 87.21, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 75.97, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 81.25, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 75.0, - "bfcl/bfcl.multi_turn.accuracy": 16.75, - "bfcl/bfcl.multi_turn.base_accuracy": 14.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 16.5, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 17.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 18.5, - "bfcl/bfcl.web_search.accuracy": 62.0, - "bfcl/bfcl.web_search.base_accuracy": 60.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 64.0, - "bfcl/bfcl.memory.accuracy": 38.71, - "bfcl/bfcl.memory.kv_accuracy": 13.55, - "bfcl/bfcl.memory.vector_accuracy": 47.1, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 55.48, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 62.5, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 91.09, - "bfcl/bfcl.format_sensitivity.max_delta": 9.0, - "bfcl/bfcl.format_sensitivity.stddev": 2.45 - } - }, - { - "id": "google/gemini-2.0-flash-001", - "name": "Gemini 2.0 Flash", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.679, - "helm_capabilities/MMLU-Pro": 0.737, - "helm_capabilities/GPQA": 0.556, - "helm_capabilities/IFEval": 0.841, - "helm_capabilities/WildBench": 0.8, - "helm_capabilities/Omni-MATH": 0.459 - } - }, - { - "id": "google/gemini-2.0-flash-exp", - "name": "Gemini 2.0 Flash Experimental", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.813, - "helm_lite/NarrativeQA": 0.783, - "helm_lite/NaturalQuestions (closed-book)": 0.443, - "helm_lite/OpenbookQA": 0.946, - "helm_lite/MMLU": 0.717, - "helm_lite/MATH": 0.901, - "helm_lite/GSM8K": 0.946, - "helm_lite/LegalBench": 0.674, - "helm_lite/MedQA": 0.73, - "helm_lite/WMT 2014": 0.212, - "helm_mmlu/MMLU All Subjects": 0.797, - "helm_mmlu/Abstract Algebra": 0.72, - "helm_mmlu/Anatomy": 0.807, - "helm_mmlu/College Physics": 0.696, - "helm_mmlu/Computer Security": 0.83, - "helm_mmlu/Econometrics": 0.693, - "helm_mmlu/Global Facts": 0.66, - "helm_mmlu/Jurisprudence": 0.898, - "helm_mmlu/Philosophy": 0.887, - "helm_mmlu/Professional Psychology": 0.876, - "helm_mmlu/Us Foreign Policy": 0.78, - "helm_mmlu/Astronomy": 0.928, - "helm_mmlu/Business Ethics": 0.73, - "helm_mmlu/Clinical Knowledge": 0.879, - "helm_mmlu/Conceptual Physics": 0.813, - "helm_mmlu/Electrical Engineering": 0.834, - "helm_mmlu/Elementary Mathematics": 0.857, - "helm_mmlu/Formal Logic": 0.571, - "helm_mmlu/High School World History": 0.743, - "helm_mmlu/Human Sexuality": 0.901, - "helm_mmlu/International Law": 0.645, - "helm_mmlu/Logical Fallacies": 0.914, - "helm_mmlu/Machine Learning": 0.759, - "helm_mmlu/Management": 0.718, - "helm_mmlu/Marketing": 0.944, - "helm_mmlu/Medical Genetics": 0.89, - "helm_mmlu/Miscellaneous": 0.939, - "helm_mmlu/Moral Scenarios": 0.815, - "helm_mmlu/Nutrition": 0.856, - "helm_mmlu/Prehistory": 0.898, - "helm_mmlu/Public Relations": 0.791, - "helm_mmlu/Security Studies": 0.69, - "helm_mmlu/Sociology": 0.786, - "helm_mmlu/Virology": 0.554, - "helm_mmlu/World Religions": 0.731, - "helm_mmlu/Mean win rate": 0.567 - } - }, - { - "id": "google/gemini-2.0-flash-lite-preview-02-05", - "name": "Gemini 2.0 Flash Lite 02-05 preview", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.642, - "helm_capabilities/MMLU-Pro": 0.72, - "helm_capabilities/GPQA": 0.5, - "helm_capabilities/IFEval": 0.824, - "helm_capabilities/WildBench": 0.79, - "helm_capabilities/Omni-MATH": 0.374 - } - }, - { - "id": "google/gemini-2.5-flash", - "name": "Gemini 2.5 Flash", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "global-mmlu-lite/Global MMLU Lite": 0.9145, - "global-mmlu-lite/Culturally Sensitive": 0.9, - "global-mmlu-lite/Culturally Agnostic": 0.9291, - "global-mmlu-lite/Arabic": 0.9125, - "global-mmlu-lite/English": 0.9325, - "global-mmlu-lite/Bengali": 0.91, - "global-mmlu-lite/German": 0.9025, - "global-mmlu-lite/French": 0.91, - "global-mmlu-lite/Hindi": 0.925, - "global-mmlu-lite/Indonesian": 0.9075, - "global-mmlu-lite/Italian": 0.9225, - "global-mmlu-lite/Japanese": 0.9125, - "global-mmlu-lite/Korean": 0.915, - "global-mmlu-lite/Portuguese": 0.9125, - "global-mmlu-lite/Spanish": 0.9175, - "global-mmlu-lite/Swahili": 0.915, - "global-mmlu-lite/Yoruba": 0.9075, - "global-mmlu-lite/Chinese": 0.915, - "global-mmlu-lite/Burmese": 0.915, - "livecodebenchpro/Hard Problems": 0.0, - "livecodebenchpro/Medium Problems": 0.028169014084507043, - "livecodebenchpro/Easy Problems": 0.38028169014084506, - "reward-bench/Score": 0.7767, - "reward-bench/Factuality": 0.674, - "reward-bench/Precise IF": 0.575, - "reward-bench/Math": 0.852, - "reward-bench/Safety": 0.909, - "reward-bench/Focus": 0.841, - "reward-bench/Ties": 0.809, - "terminal-bench-2.0/terminal-bench-2.0": 16.9 - } - }, - { - "id": "google/gemini-2.5-flash-lite", - "name": "Gemini 2.5 Flash-Lite", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.591, - "helm_capabilities/MMLU-Pro": 0.537, - "helm_capabilities/GPQA": 0.309, - "helm_capabilities/IFEval": 0.81, - "helm_capabilities/WildBench": 0.818, - "helm_capabilities/Omni-MATH": 0.48 - } - }, - { - "id": "google/gemini-2.5-flash-preview-04-17", - "name": "Gemini 2.5 Flash 04-17 preview", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.626, - "helm_capabilities/MMLU-Pro": 0.639, - "helm_capabilities/GPQA": 0.39, - "helm_capabilities/IFEval": 0.898, - "helm_capabilities/WildBench": 0.817, - "helm_capabilities/Omni-MATH": 0.384, - "reward-bench/Score": 0.7721, - "reward-bench/Factuality": 0.6574, - "reward-bench/Precise IF": 0.5531, - "reward-bench/Math": 0.8115, - "reward-bench/Safety": 0.9094, - "reward-bench/Focus": 0.8672, - "reward-bench/Ties": 0.8341 - } - }, - { - "id": "google/gemini-2.5-flash-preview-05-20", - "name": "gemini-2.5-flash-preview-05-20", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "global-mmlu-lite/Global MMLU Lite": 0.9092, - "global-mmlu-lite/Culturally Sensitive": 0.8925, - "global-mmlu-lite/Culturally Agnostic": 0.9259, - "global-mmlu-lite/Arabic": 0.905, - "global-mmlu-lite/English": 0.9225, - "global-mmlu-lite/Bengali": 0.91, - "global-mmlu-lite/German": 0.905, - "global-mmlu-lite/French": 0.925, - "global-mmlu-lite/Hindi": 0.9125, - "global-mmlu-lite/Indonesian": 0.9075, - "global-mmlu-lite/Italian": 0.89, - "global-mmlu-lite/Japanese": 0.9125, - "global-mmlu-lite/Korean": 0.9075, - "global-mmlu-lite/Portuguese": 0.915, - "global-mmlu-lite/Spanish": 0.915, - "global-mmlu-lite/Swahili": 0.905, - "global-mmlu-lite/Yoruba": 0.8825, - "global-mmlu-lite/Chinese": 0.93, - "global-mmlu-lite/Burmese": 0.9025 - } - }, - { - "id": "google/gemini-2.5-pro", - "name": "Gemini 2.5 Pro", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "global-mmlu-lite/Global MMLU Lite": 0.9323, - "global-mmlu-lite/Culturally Sensitive": 0.9241, - "global-mmlu-lite/Culturally Agnostic": 0.9406, - "global-mmlu-lite/Arabic": 0.9475, - "global-mmlu-lite/English": 0.9275, - "global-mmlu-lite/Bengali": 0.9275, - "global-mmlu-lite/German": 0.93, - "global-mmlu-lite/French": 0.9425, - "global-mmlu-lite/Hindi": 0.9275, - "global-mmlu-lite/Indonesian": 0.925, - "global-mmlu-lite/Italian": 0.935, - "global-mmlu-lite/Japanese": 0.9375, - "global-mmlu-lite/Korean": 0.9275, - "global-mmlu-lite/Portuguese": 0.93, - "global-mmlu-lite/Spanish": 0.94, - "global-mmlu-lite/Swahili": 0.9375, - "global-mmlu-lite/Yoruba": 0.925, - "global-mmlu-lite/Chinese": 0.9275, - "global-mmlu-lite/Burmese": 0.93, - "livecodebenchpro/Hard Problems": 0.014084507042253521, - "livecodebenchpro/Medium Problems": 0.2112676056338028, - "livecodebenchpro/Easy Problems": 0.7183098591549296, - "reward-bench/Score": 0.7948, - "reward-bench/Factuality": 0.755, - "reward-bench/Precise IF": 0.619, - "reward-bench/Math": 0.898, - "reward-bench/Safety": 0.881, - "reward-bench/Focus": 0.805, - "reward-bench/Ties": 0.811, - "terminal-bench-2.0/terminal-bench-2.0": 19.6 - } - }, - { - "id": "google/gemini-2.5-pro-preview-03-25", - "name": "Gemini 2.5 Pro 03-25 preview", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.745, - "helm_capabilities/MMLU-Pro": 0.863, - "helm_capabilities/GPQA": 0.749, - "helm_capabilities/IFEval": 0.84, - "helm_capabilities/WildBench": 0.857, - "helm_capabilities/Omni-MATH": 0.416 - } - }, - { - "id": "google/gemini-2.5-pro-preview-05-06", - "name": "google/gemini-2.5-pro-preview-05-06", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6775, - "reward-bench/Factuality": 0.6532, - "reward-bench/Precise IF": 0.4688, - "reward-bench/Math": 0.5342, - "reward-bench/Safety": 0.8806, - "reward-bench/Focus": 0.8308, - "reward-bench/Ties": 0.6973 - } - }, - { - "id": "google/gemini-3-flash", - "name": "Gemini 3 Flash", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "terminal-bench-2.0/terminal-bench-2.0": 47.4 - } - }, - { - "id": "google/gemini-3-pro", - "name": "Gemini 3 Pro", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "terminal-bench-2.0/terminal-bench-2.0": 56.9 - } - }, - { - "id": "google/gemini-3-pro-preview", - "name": "gemini-3-pro-preview", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "appworld_test_normal/appworld/test_normal": 0.505, - "browsecompplus/browsecompplus": 0.48, - "global-mmlu-lite/Global MMLU Lite": 0.9453, - "global-mmlu-lite/Culturally Sensitive": 0.9397, - "global-mmlu-lite/Culturally Agnostic": 0.9509, - "global-mmlu-lite/Arabic": 0.9475, - "global-mmlu-lite/English": 0.9425, - "global-mmlu-lite/Bengali": 0.9425, - "global-mmlu-lite/German": 0.94, - "global-mmlu-lite/French": 0.9575, - "global-mmlu-lite/Hindi": 0.9425, - "global-mmlu-lite/Indonesian": 0.955, - "global-mmlu-lite/Italian": 0.955, - "global-mmlu-lite/Japanese": 0.94, - "global-mmlu-lite/Korean": 0.94, - "global-mmlu-lite/Portuguese": 0.9425, - "global-mmlu-lite/Spanish": 0.9475, - "global-mmlu-lite/Swahili": 0.94, - "global-mmlu-lite/Yoruba": 0.9425, - "global-mmlu-lite/Chinese": 0.9475, - "global-mmlu-lite/Burmese": 0.9425, - "swe-bench/swe-bench": 0.71, - "tau-bench-2_airline/tau-bench-2/airline": 0.62, - "tau-bench-2_retail/tau-bench-2/retail": 0.7576, - "tau-bench-2_telecom/tau-bench-2/telecom": 0.73 - } - }, - { - "id": "google/gemini-3-pro-preview-fc", - "name": "Gemini-3-Pro-Preview (FC)", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 7.0, - "bfcl/bfcl.overall.overall_accuracy": 68.14, - "bfcl/bfcl.overall.total_cost_usd": 224.69, - "bfcl/bfcl.overall.latency_mean_s": 15.87, - "bfcl/bfcl.overall.latency_std_s": 41.41, - "bfcl/bfcl.overall.latency_p95_s": 58.48, - "bfcl/bfcl.non_live.ast_accuracy": 85.75, - "bfcl/bfcl.non_live.simple_ast_accuracy": 75.5, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 94.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 91.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 82.5, - "bfcl/bfcl.live.live_accuracy": 81.72, - "bfcl/bfcl.live.live_simple_ast_accuracy": 87.6, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 80.44, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 75.0, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 79.17, - "bfcl/bfcl.multi_turn.accuracy": 63.12, - "bfcl/bfcl.multi_turn.base_accuracy": 69.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 63.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 56.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 64.0, - "bfcl/bfcl.web_search.accuracy": 68.5, - "bfcl/bfcl.web_search.base_accuracy": 63.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 74.0, - "bfcl/bfcl.memory.accuracy": 54.84, - "bfcl/bfcl.memory.kv_accuracy": 50.32, - "bfcl/bfcl.memory.vector_accuracy": 63.23, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 50.97, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 75.0, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 77.85 - } - }, - { - "id": "google/gemini-3-pro-preview-prompt", - "name": "Gemini-3-Pro-Preview (Prompt)", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 3.0, - "bfcl/bfcl.overall.overall_accuracy": 72.51, - "bfcl/bfcl.overall.total_cost_usd": 298.47, - "bfcl/bfcl.overall.latency_mean_s": 12.08, - "bfcl/bfcl.overall.latency_std_s": 21.3, - "bfcl/bfcl.overall.latency_p95_s": 32.73, - "bfcl/bfcl.non_live.ast_accuracy": 90.65, - "bfcl/bfcl.non_live.simple_ast_accuracy": 79.58, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 96.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 95.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 92.0, - "bfcl/bfcl.live.live_accuracy": 83.12, - "bfcl/bfcl.live.live_simple_ast_accuracy": 87.6, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 81.77, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 93.75, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 87.5, - "bfcl/bfcl.multi_turn.accuracy": 60.75, - "bfcl/bfcl.multi_turn.base_accuracy": 64.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 60.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 54.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 64.0, - "bfcl/bfcl.web_search.accuracy": 80.0, - "bfcl/bfcl.web_search.base_accuracy": 78.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 82.0, - "bfcl/bfcl.memory.accuracy": 61.72, - "bfcl/bfcl.memory.kv_accuracy": 59.35, - "bfcl/bfcl.memory.vector_accuracy": 62.58, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 63.23, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 68.75, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 85.59, - "bfcl/bfcl.format_sensitivity.max_delta": 8.5, - "bfcl/bfcl.format_sensitivity.stddev": 1.7 - } - }, - { - "id": "google/gemini-3.1-pro", - "name": "Gemini 3.1 Pro", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "terminal-bench-2.0/terminal-bench-2.0": 74.8 - } - }, - { - "id": "google/gemma-1.1-2b-it", - "name": "gemma-1.1-2b-it", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3067, - "hfopenllm_v2/BBH": 0.3185, - "hfopenllm_v2/MATH Level 5": 0.0181, - "hfopenllm_v2/GPQA": 0.2693, - "hfopenllm_v2/MUSR": 0.3394, - "hfopenllm_v2/MMLU-PRO": 0.1484 - } - }, - { - "id": "google/gemma-1.1-7b-it", - "name": "gemma-1.1-7b-it", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5039, - "hfopenllm_v2/BBH": 0.3935, - "hfopenllm_v2/MATH Level 5": 0.0491, - "hfopenllm_v2/GPQA": 0.2936, - "hfopenllm_v2/MUSR": 0.423, - "hfopenllm_v2/MMLU-PRO": 0.2584 - } - }, - { - "id": "google/gemma-2-27b", - "name": "Gemma 2 27B", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_mmlu/MMLU All Subjects": 0.757, - "helm_mmlu/Abstract Algebra": 0.4, - "helm_mmlu/Anatomy": 0.77, - "helm_mmlu/College Physics": 0.5, - "helm_mmlu/Computer Security": 0.84, - "helm_mmlu/Econometrics": 0.667, - "helm_mmlu/Global Facts": 0.43, - "helm_mmlu/Jurisprudence": 0.861, - "helm_mmlu/Philosophy": 0.849, - "helm_mmlu/Professional Psychology": 0.84, - "helm_mmlu/Us Foreign Policy": 0.95, - "helm_mmlu/Astronomy": 0.829, - "helm_mmlu/Business Ethics": 0.78, - "helm_mmlu/Clinical Knowledge": 0.808, - "helm_mmlu/Conceptual Physics": 0.834, - "helm_mmlu/Electrical Engineering": 0.738, - "helm_mmlu/Elementary Mathematics": 0.558, - "helm_mmlu/Formal Logic": 0.516, - "helm_mmlu/High School World History": 0.89, - "helm_mmlu/Human Sexuality": 0.84, - "helm_mmlu/International Law": 0.843, - "helm_mmlu/Logical Fallacies": 0.865, - "helm_mmlu/Machine Learning": 0.625, - "helm_mmlu/Management": 0.864, - "helm_mmlu/Marketing": 0.94, - "helm_mmlu/Medical Genetics": 0.87, - "helm_mmlu/Miscellaneous": 0.885, - "helm_mmlu/Moral Scenarios": 0.394, - "helm_mmlu/Nutrition": 0.824, - "helm_mmlu/Prehistory": 0.877, - "helm_mmlu/Public Relations": 0.745, - "helm_mmlu/Security Studies": 0.808, - "helm_mmlu/Sociology": 0.9, - "helm_mmlu/Virology": 0.56, - "helm_mmlu/World Religions": 0.924, - "helm_mmlu/Mean win rate": 0.05, - "hfopenllm_v2/IFEval": 0.2475, - "hfopenllm_v2/BBH": 0.5643, - "hfopenllm_v2/MATH Level 5": 0.1662, - "hfopenllm_v2/GPQA": 0.3507, - "hfopenllm_v2/MUSR": 0.4396, - "hfopenllm_v2/MMLU-PRO": 0.4371 - } - }, - { - "id": "google/gemma-2-27b-it", - "name": "Gemma 2 Instruct 27B", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.675, - "helm_lite/NarrativeQA": 0.79, - "helm_lite/NaturalQuestions (closed-book)": 0.353, - "helm_lite/OpenbookQA": 0.918, - "helm_lite/MMLU": 0.664, - "helm_lite/MATH": 0.746, - "helm_lite/GSM8K": 0.812, - "helm_lite/LegalBench": 0.7, - "helm_lite/MedQA": 0.684, - "helm_lite/WMT 2014": 0.214, - "hfopenllm_v2/IFEval": 0.7978, - "hfopenllm_v2/BBH": 0.6451, - "hfopenllm_v2/MATH Level 5": 0.2387, - "hfopenllm_v2/GPQA": 0.375, - "hfopenllm_v2/MUSR": 0.4033, - "hfopenllm_v2/MMLU-PRO": 0.4451, - "reward-bench/Score": 0.809, - "reward-bench/Chat": 0.9483, - "reward-bench/Chat Hard": 0.591, - "reward-bench/Safety": 0.8635, - "reward-bench/Reasoning": 0.833 - } - }, - { - "id": "google/gemma-2-2b", - "name": "gemma-2-2b", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1993, - "hfopenllm_v2/BBH": 0.3656, - "hfopenllm_v2/MATH Level 5": 0.0287, - "hfopenllm_v2/GPQA": 0.2626, - "hfopenllm_v2/MUSR": 0.4232, - "hfopenllm_v2/MMLU-PRO": 0.218 - } - }, - { - "id": "google/gemma-2-2b-it", - "name": "gemma-2-2b-it", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5668, - "hfopenllm_v2/BBH": 0.4199, - "hfopenllm_v2/MATH Level 5": 0.0008, - "hfopenllm_v2/GPQA": 0.2743, - "hfopenllm_v2/MUSR": 0.3929, - "hfopenllm_v2/MMLU-PRO": 0.255 - } - }, - { - "id": "google/gemma-2-2b-jpn-it", - "name": "gemma-2-2b-jpn-it", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5288, - "hfopenllm_v2/BBH": 0.4178, - "hfopenllm_v2/MATH Level 5": 0.0476, - "hfopenllm_v2/GPQA": 0.2752, - "hfopenllm_v2/MUSR": 0.3728, - "hfopenllm_v2/MMLU-PRO": 0.2467 - } - }, - { - "id": "google/gemma-2-9b", - "name": "Gemma 2 9B", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_mmlu/MMLU All Subjects": 0.721, - "helm_mmlu/Abstract Algebra": 0.4, - "helm_mmlu/Anatomy": 0.704, - "helm_mmlu/College Physics": 0.5, - "helm_mmlu/Computer Security": 0.81, - "helm_mmlu/Econometrics": 0.579, - "helm_mmlu/Global Facts": 0.53, - "helm_mmlu/Jurisprudence": 0.833, - "helm_mmlu/Philosophy": 0.772, - "helm_mmlu/Professional Psychology": 0.788, - "helm_mmlu/Us Foreign Policy": 0.9, - "helm_mmlu/Astronomy": 0.789, - "helm_mmlu/Business Ethics": 0.77, - "helm_mmlu/Clinical Knowledge": 0.777, - "helm_mmlu/Conceptual Physics": 0.732, - "helm_mmlu/Electrical Engineering": 0.724, - "helm_mmlu/Elementary Mathematics": 0.577, - "helm_mmlu/Formal Logic": 0.492, - "helm_mmlu/High School World History": 0.865, - "helm_mmlu/Human Sexuality": 0.809, - "helm_mmlu/International Law": 0.835, - "helm_mmlu/Logical Fallacies": 0.816, - "helm_mmlu/Machine Learning": 0.509, - "helm_mmlu/Management": 0.874, - "helm_mmlu/Marketing": 0.919, - "helm_mmlu/Medical Genetics": 0.84, - "helm_mmlu/Miscellaneous": 0.844, - "helm_mmlu/Moral Scenarios": 0.295, - "helm_mmlu/Nutrition": 0.775, - "helm_mmlu/Prehistory": 0.812, - "helm_mmlu/Public Relations": 0.736, - "helm_mmlu/Security Studies": 0.78, - "helm_mmlu/Sociology": 0.9, - "helm_mmlu/Virology": 0.53, - "helm_mmlu/World Religions": 0.86, - "helm_mmlu/Mean win rate": 0.265, - "hfopenllm_v2/IFEval": 0.204, - "hfopenllm_v2/BBH": 0.5377, - "hfopenllm_v2/MATH Level 5": 0.1344, - "hfopenllm_v2/GPQA": 0.3289, - "hfopenllm_v2/MUSR": 0.4461, - "hfopenllm_v2/MMLU-PRO": 0.4103 - } - }, - { - "id": "google/gemma-2-9b-it", - "name": "Gemma 2 Instruct 9B", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.562, - "helm_lite/NarrativeQA": 0.768, - "helm_lite/NaturalQuestions (closed-book)": 0.328, - "helm_lite/OpenbookQA": 0.91, - "helm_lite/MMLU": 0.645, - "helm_lite/MATH": 0.724, - "helm_lite/GSM8K": 0.762, - "helm_lite/LegalBench": 0.639, - "helm_lite/MedQA": 0.63, - "helm_lite/WMT 2014": 0.201, - "hfopenllm_v2/IFEval": 0.7436, - "hfopenllm_v2/BBH": 0.599, - "hfopenllm_v2/MATH Level 5": 0.1949, - "hfopenllm_v2/GPQA": 0.3607, - "hfopenllm_v2/MUSR": 0.4073, - "hfopenllm_v2/MMLU-PRO": 0.3875, - "la_leaderboard/la_leaderboard": 33.62 - } - }, - { - "id": "google/gemma-2b", - "name": "gemma-2b", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2038, - "hfopenllm_v2/BBH": 0.3366, - "hfopenllm_v2/MATH Level 5": 0.0302, - "hfopenllm_v2/GPQA": 0.255, - "hfopenllm_v2/MUSR": 0.3978, - "hfopenllm_v2/MMLU-PRO": 0.1366 - } - }, - { - "id": "google/gemma-2b-it", - "name": "gemma-2b-it", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.269, - "hfopenllm_v2/BBH": 0.3151, - "hfopenllm_v2/MATH Level 5": 0.0204, - "hfopenllm_v2/GPQA": 0.2785, - "hfopenllm_v2/MUSR": 0.3341, - "hfopenllm_v2/MMLU-PRO": 0.1353 - } - }, - { - "id": "google/gemma-3-12b-it-prompt", - "name": "Gemma-3-12b-it (Prompt)", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 66.0, - "bfcl/bfcl.overall.overall_accuracy": 30.43, - "bfcl/bfcl.overall.total_cost_usd": 10.77, - "bfcl/bfcl.overall.latency_mean_s": 11.1, - "bfcl/bfcl.overall.latency_std_s": 17.17, - "bfcl/bfcl.overall.latency_p95_s": 34.66, - "bfcl/bfcl.non_live.ast_accuracy": 79.44, - "bfcl/bfcl.non_live.simple_ast_accuracy": 76.25, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 94.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 91.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 56.5, - "bfcl/bfcl.live.live_accuracy": 74.24, - "bfcl/bfcl.live.live_simple_ast_accuracy": 85.66, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 71.89, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 87.5, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 45.83, - "bfcl/bfcl.multi_turn.accuracy": 5.75, - "bfcl/bfcl.multi_turn.base_accuracy": 6.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 7.5, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 5.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 4.0, - "bfcl/bfcl.web_search.accuracy": 4.0, - "bfcl/bfcl.web_search.base_accuracy": 4.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 4.0, - "bfcl/bfcl.memory.accuracy": 27.53, - "bfcl/bfcl.memory.kv_accuracy": 8.39, - "bfcl/bfcl.memory.vector_accuracy": 25.16, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 49.03, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 93.75, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 70.29, - "bfcl/bfcl.format_sensitivity.max_delta": 67.5, - "bfcl/bfcl.format_sensitivity.stddev": 22.41 - } - }, - { - "id": "google/gemma-3-1b-it-prompt", - "name": "Gemma-3-1b-it (Prompt)", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 109.0, - "bfcl/bfcl.overall.overall_accuracy": 7.17, - "bfcl/bfcl.overall.total_cost_usd": 3.4, - "bfcl/bfcl.overall.latency_mean_s": 3.98, - "bfcl/bfcl.overall.latency_std_s": 9.8, - "bfcl/bfcl.overall.latency_p95_s": 12.06, - "bfcl/bfcl.non_live.ast_accuracy": 20.21, - "bfcl/bfcl.non_live.simple_ast_accuracy": 43.33, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 36.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 0.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 1.5, - "bfcl/bfcl.live.live_accuracy": 11.84, - "bfcl/bfcl.live.live_simple_ast_accuracy": 36.43, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 6.27, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 0.0, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 0.0, - "bfcl/bfcl.multi_turn.accuracy": 0.0, - "bfcl/bfcl.multi_turn.base_accuracy": 0.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 0.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 0.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 0.0, - "bfcl/bfcl.web_search.accuracy": 0.0, - "bfcl/bfcl.web_search.base_accuracy": 0.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 0.0, - "bfcl/bfcl.memory.accuracy": 3.23, - "bfcl/bfcl.memory.kv_accuracy": 3.87, - "bfcl/bfcl.memory.vector_accuracy": 3.87, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 1.94, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 37.5, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 33.18, - "bfcl/bfcl.format_sensitivity.max_delta": 25.5, - "bfcl/bfcl.format_sensitivity.stddev": 9.76 - } - }, - { - "id": "google/gemma-3-27b-it", - "name": "gemma-3-27b-it", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "global-mmlu-lite/Global MMLU Lite": 0.763, - "global-mmlu-lite/Culturally Sensitive": 0.7528, - "global-mmlu-lite/Culturally Agnostic": 0.7733, - "global-mmlu-lite/Arabic": 0.78, - "global-mmlu-lite/English": 0.7337, - "global-mmlu-lite/Bengali": 0.75, - "global-mmlu-lite/German": 0.775, - "global-mmlu-lite/French": 0.7481, - "global-mmlu-lite/Hindi": 0.7335, - "global-mmlu-lite/Indonesian": 0.7563, - "global-mmlu-lite/Italian": 0.75, - "global-mmlu-lite/Japanese": 0.7925, - "global-mmlu-lite/Korean": 0.798, - "global-mmlu-lite/Portuguese": 0.7481, - "global-mmlu-lite/Spanish": 0.7494, - "global-mmlu-lite/Swahili": 0.785, - "global-mmlu-lite/Yoruba": 0.7444, - "global-mmlu-lite/Chinese": 0.7925, - "global-mmlu-lite/Burmese": 0.7719 - } - }, - { - "id": "google/gemma-3-27b-it-prompt", - "name": "Gemma-3-27b-it (Prompt)", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 69.0, - "bfcl/bfcl.overall.overall_accuracy": 29.47, - "bfcl/bfcl.overall.total_cost_usd": 11.82, - "bfcl/bfcl.overall.latency_mean_s": 10.88, - "bfcl/bfcl.overall.latency_std_s": 19.67, - "bfcl/bfcl.overall.latency_p95_s": 55.5, - "bfcl/bfcl.non_live.ast_accuracy": 87.17, - "bfcl/bfcl.non_live.simple_ast_accuracy": 77.67, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 92.5, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 89.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 89.5, - "bfcl/bfcl.live.live_accuracy": 74.54, - "bfcl/bfcl.live.live_simple_ast_accuracy": 84.5, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 72.46, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 93.75, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 45.83, - "bfcl/bfcl.multi_turn.accuracy": 10.75, - "bfcl/bfcl.multi_turn.base_accuracy": 16.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 4.5, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 8.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 14.0, - "bfcl/bfcl.web_search.accuracy": 0.0, - "bfcl/bfcl.web_search.base_accuracy": 0.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 0.0, - "bfcl/bfcl.memory.accuracy": 13.55, - "bfcl/bfcl.memory.kv_accuracy": 1.94, - "bfcl/bfcl.memory.vector_accuracy": 3.23, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 35.48, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 81.25, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 73.67, - "bfcl/bfcl.format_sensitivity.max_delta": 34.0, - "bfcl/bfcl.format_sensitivity.stddev": 8.06 - } - }, - { - "id": "google/gemma-3-4b-it", - "name": "gemma-3-4b-it", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "global-mmlu-lite/Global MMLU Lite": 0.6511, - "global-mmlu-lite/Culturally Sensitive": 0.6116, - "global-mmlu-lite/Culturally Agnostic": 0.6906, - "global-mmlu-lite/Arabic": 0.6525, - "global-mmlu-lite/English": 0.67, - "global-mmlu-lite/Bengali": 0.68, - "global-mmlu-lite/German": 0.6525, - "global-mmlu-lite/French": 0.6575, - "global-mmlu-lite/Hindi": 0.6475, - "global-mmlu-lite/Indonesian": 0.6775, - "global-mmlu-lite/Italian": 0.6675, - "global-mmlu-lite/Japanese": 0.6325, - "global-mmlu-lite/Korean": 0.66, - "global-mmlu-lite/Portuguese": 0.68, - "global-mmlu-lite/Spanish": 0.6725, - "global-mmlu-lite/Swahili": 0.6075, - "global-mmlu-lite/Yoruba": 0.5825, - "global-mmlu-lite/Chinese": 0.6475, - "global-mmlu-lite/Burmese": 0.63 - } - }, - { - "id": "google/gemma-3-4b-it-prompt", - "name": "Gemma-3-4b-it (Prompt)", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 101.0, - "bfcl/bfcl.overall.overall_accuracy": 19.62, - "bfcl/bfcl.overall.total_cost_usd": 4.14, - "bfcl/bfcl.overall.latency_mean_s": 4.69, - "bfcl/bfcl.overall.latency_std_s": 9.53, - "bfcl/bfcl.overall.latency_p95_s": 11.42, - "bfcl/bfcl.non_live.ast_accuracy": 61.12, - "bfcl/bfcl.non_live.simple_ast_accuracy": 64.5, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 88.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 56.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 36.0, - "bfcl/bfcl.live.live_accuracy": 60.84, - "bfcl/bfcl.live.live_simple_ast_accuracy": 70.93, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 59.35, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 25.0, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 41.67, - "bfcl/bfcl.multi_turn.accuracy": 0.38, - "bfcl/bfcl.multi_turn.base_accuracy": 0.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 0.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 0.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 0.5, - "bfcl/bfcl.web_search.accuracy": 1.0, - "bfcl/bfcl.web_search.base_accuracy": 1.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 1.0, - "bfcl/bfcl.memory.accuracy": 8.6, - "bfcl/bfcl.memory.kv_accuracy": 9.68, - "bfcl/bfcl.memory.vector_accuracy": 9.68, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 6.45, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 100.0, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 53.94, - "bfcl/bfcl.format_sensitivity.max_delta": 69.5, - "bfcl/bfcl.format_sensitivity.stddev": 23.67 - } - }, - { - "id": "google/gemma-7b", - "name": "Gemma 7B", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.336, - "helm_lite/NarrativeQA": 0.752, - "helm_lite/NaturalQuestions (closed-book)": 0.336, - "helm_lite/OpenbookQA": 0.808, - "helm_lite/MMLU": 0.571, - "helm_lite/MATH": 0.5, - "helm_lite/GSM8K": 0.559, - "helm_lite/LegalBench": 0.581, - "helm_lite/MedQA": 0.513, - "helm_lite/WMT 2014": 0.187, - "helm_mmlu/MMLU All Subjects": 0.661, - "helm_mmlu/Abstract Algebra": 0.28, - "helm_mmlu/Anatomy": 0.563, - "helm_mmlu/College Physics": 0.412, - "helm_mmlu/Computer Security": 0.75, - "helm_mmlu/Econometrics": 0.474, - "helm_mmlu/Global Facts": 0.42, - "helm_mmlu/Jurisprudence": 0.769, - "helm_mmlu/Philosophy": 0.727, - "helm_mmlu/Professional Psychology": 0.712, - "helm_mmlu/Us Foreign Policy": 0.87, - "helm_mmlu/Astronomy": 0.717, - "helm_mmlu/Business Ethics": 0.65, - "helm_mmlu/Clinical Knowledge": 0.698, - "helm_mmlu/Conceptual Physics": 0.621, - "helm_mmlu/Electrical Engineering": 0.628, - "helm_mmlu/Elementary Mathematics": 0.516, - "helm_mmlu/Formal Logic": 0.508, - "helm_mmlu/High School World History": 0.857, - "helm_mmlu/Human Sexuality": 0.733, - "helm_mmlu/International Law": 0.835, - "helm_mmlu/Logical Fallacies": 0.742, - "helm_mmlu/Machine Learning": 0.554, - "helm_mmlu/Management": 0.864, - "helm_mmlu/Marketing": 0.885, - "helm_mmlu/Medical Genetics": 0.7, - "helm_mmlu/Miscellaneous": 0.838, - "helm_mmlu/Moral Scenarios": 0.377, - "helm_mmlu/Nutrition": 0.778, - "helm_mmlu/Prehistory": 0.756, - "helm_mmlu/Public Relations": 0.682, - "helm_mmlu/Security Studies": 0.735, - "helm_mmlu/Sociology": 0.841, - "helm_mmlu/Virology": 0.548, - "helm_mmlu/World Religions": 0.842, - "helm_mmlu/Mean win rate": 0.824, - "hfopenllm_v2/IFEval": 0.2659, - "hfopenllm_v2/BBH": 0.4362, - "hfopenllm_v2/MATH Level 5": 0.074, - "hfopenllm_v2/GPQA": 0.2869, - "hfopenllm_v2/MUSR": 0.4062, - "hfopenllm_v2/MMLU-PRO": 0.2948 - } - }, - { - "id": "google/gemma-7b-it", - "name": "gemma-7b-it", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3868, - "hfopenllm_v2/BBH": 0.3646, - "hfopenllm_v2/MATH Level 5": 0.0295, - "hfopenllm_v2/GPQA": 0.2844, - "hfopenllm_v2/MUSR": 0.4274, - "hfopenllm_v2/MMLU-PRO": 0.1695 - } - }, - { - "id": "google/mt5-base", - "name": "mt5-base", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1645, - "hfopenllm_v2/BBH": 0.2883, - "hfopenllm_v2/MATH Level 5": 0.0091, - "hfopenllm_v2/GPQA": 0.2391, - "hfopenllm_v2/MUSR": 0.3672, - "hfopenllm_v2/MMLU-PRO": 0.107 - } - }, - { - "id": "google/mt5-small", - "name": "mt5-small", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1718, - "hfopenllm_v2/BBH": 0.2766, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2424, - "hfopenllm_v2/MUSR": 0.3857, - "hfopenllm_v2/MMLU-PRO": 0.1123 - } - }, - { - "id": "google/mt5-xl", - "name": "mt5-xl", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.196, - "hfopenllm_v2/BBH": 0.3047, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2643, - "hfopenllm_v2/MUSR": 0.3795, - "hfopenllm_v2/MMLU-PRO": 0.112 - } - }, - { - "id": "google/mt5-xxl", - "name": "mt5-xxl", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2358, - "hfopenllm_v2/BBH": 0.2959, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2416, - "hfopenllm_v2/MUSR": 0.3689, - "hfopenllm_v2/MMLU-PRO": 0.1089 - } - }, - { - "id": "google/Palmyra-X-43B", - "name": "Palmyra X 43B", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.732, - "helm_classic/MMLU": 0.609, - "helm_classic/BoolQ": 0.896, - "helm_classic/NarrativeQA": 0.742, - "helm_classic/NaturalQuestions (open-book)": -1.0, - "helm_classic/QuAC": 0.473, - "helm_classic/HellaSwag": -1.0, - "helm_classic/OpenbookQA": -1.0, - "helm_classic/TruthfulQA": 0.616, - "helm_classic/MS MARCO (TREC)": -1.0, - "helm_classic/CNN/DailyMail": 0.049, - "helm_classic/XSUM": 0.149, - "helm_classic/IMDB": 0.935, - "helm_classic/CivilComments": 0.008, - "helm_classic/RAFT": 0.701 - } - }, - { - "id": "google/recurrentgemma-2b", - "name": "recurrentgemma-2b", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3017, - "hfopenllm_v2/BBH": 0.3197, - "hfopenllm_v2/MATH Level 5": 0.0204, - "hfopenllm_v2/GPQA": 0.2458, - "hfopenllm_v2/MUSR": 0.3446, - "hfopenllm_v2/MMLU-PRO": 0.1176 - } - }, - { - "id": "google/recurrentgemma-2b-it", - "name": "recurrentgemma-2b-it", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2949, - "hfopenllm_v2/BBH": 0.333, - "hfopenllm_v2/MATH Level 5": 0.0196, - "hfopenllm_v2/GPQA": 0.2534, - "hfopenllm_v2/MUSR": 0.3341, - "hfopenllm_v2/MMLU-PRO": 0.1402 - } - }, - { - "id": "google/recurrentgemma-9b", - "name": "recurrentgemma-9b", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3116, - "hfopenllm_v2/BBH": 0.3956, - "hfopenllm_v2/MATH Level 5": 0.0665, - "hfopenllm_v2/GPQA": 0.2852, - "hfopenllm_v2/MUSR": 0.3803, - "hfopenllm_v2/MMLU-PRO": 0.2605 - } - }, - { - "id": "google/recurrentgemma-9b-it", - "name": "recurrentgemma-9b-it", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.501, - "hfopenllm_v2/BBH": 0.4367, - "hfopenllm_v2/MATH Level 5": 0.0665, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.4379, - "hfopenllm_v2/MMLU-PRO": 0.2843 - } - }, - { - "id": "google/switch-base-8", - "name": "switch-base-8", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1585, - "hfopenllm_v2/BBH": 0.2876, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.25, - "hfopenllm_v2/MUSR": 0.3517, - "hfopenllm_v2/MMLU-PRO": 0.1098 - } - }, - { - "id": "google/T5-11B", - "name": "T5 11B", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.131, - "helm_classic/MMLU": 0.29, - "helm_classic/BoolQ": 0.761, - "helm_classic/NarrativeQA": 0.086, - "helm_classic/NaturalQuestions (open-book)": 0.477, - "helm_classic/QuAC": 0.116, - "helm_classic/HellaSwag": -1.0, - "helm_classic/OpenbookQA": -1.0, - "helm_classic/TruthfulQA": 0.133, - "helm_classic/MS MARCO (TREC)": -1.0, - "helm_classic/CNN/DailyMail": 0.043, - "helm_classic/XSUM": 0.015, - "helm_classic/IMDB": 0.379, - "helm_classic/CivilComments": 0.509, - "helm_classic/RAFT": 0.37 - } - }, - { - "id": "google/text-bison@001", - "name": "PaLM-2 Bison", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.526, - "helm_lite/NarrativeQA": 0.718, - "helm_lite/NaturalQuestions (closed-book)": 0.39, - "helm_lite/OpenbookQA": 0.878, - "helm_lite/MMLU": 0.608, - "helm_lite/MATH": 0.421, - "helm_lite/GSM8K": 0.61, - "helm_lite/LegalBench": 0.645, - "helm_lite/MedQA": 0.547, - "helm_lite/WMT 2014": 0.241, - "helm_mmlu/MMLU All Subjects": 0.692, - "helm_mmlu/Abstract Algebra": 0.39, - "helm_mmlu/Anatomy": 0.644, - "helm_mmlu/College Physics": 0.51, - "helm_mmlu/Computer Security": 0.74, - "helm_mmlu/Econometrics": 0.518, - "helm_mmlu/Global Facts": 0.38, - "helm_mmlu/Jurisprudence": 0.769, - "helm_mmlu/Philosophy": 0.736, - "helm_mmlu/Professional Psychology": 0.761, - "helm_mmlu/Us Foreign Policy": 0.87, - "helm_mmlu/Astronomy": 0.803, - "helm_mmlu/Business Ethics": 0.76, - "helm_mmlu/Clinical Knowledge": 0.725, - "helm_mmlu/Conceptual Physics": 0.694, - "helm_mmlu/Electrical Engineering": 0.69, - "helm_mmlu/Elementary Mathematics": 0.487, - "helm_mmlu/Formal Logic": 0.5, - "helm_mmlu/High School World History": 0.869, - "helm_mmlu/Human Sexuality": 0.84, - "helm_mmlu/International Law": 0.835, - "helm_mmlu/Logical Fallacies": 0.853, - "helm_mmlu/Machine Learning": 0.562, - "helm_mmlu/Management": 0.893, - "helm_mmlu/Marketing": 0.893, - "helm_mmlu/Medical Genetics": 0.75, - "helm_mmlu/Miscellaneous": 0.866, - "helm_mmlu/Moral Scenarios": 0.369, - "helm_mmlu/Nutrition": 0.709, - "helm_mmlu/Prehistory": 0.812, - "helm_mmlu/Public Relations": 0.691, - "helm_mmlu/Security Studies": 0.812, - "helm_mmlu/Sociology": 0.92, - "helm_mmlu/Virology": 0.494, - "helm_mmlu/World Religions": 0.883, - "helm_mmlu/Mean win rate": 0.192 - } - }, - { - "id": "google/text-unicorn@001", - "name": "PaLM-2 Unicorn", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.644, - "helm_lite/NarrativeQA": 0.583, - "helm_lite/NaturalQuestions (closed-book)": 0.435, - "helm_lite/OpenbookQA": 0.938, - "helm_lite/MMLU": 0.702, - "helm_lite/MATH": 0.674, - "helm_lite/GSM8K": 0.831, - "helm_lite/LegalBench": 0.677, - "helm_lite/MedQA": 0.684, - "helm_lite/WMT 2014": 0.26, - "helm_mmlu/MMLU All Subjects": 0.786, - "helm_mmlu/Abstract Algebra": 0.51, - "helm_mmlu/Anatomy": 0.733, - "helm_mmlu/College Physics": 0.549, - "helm_mmlu/Computer Security": 0.77, - "helm_mmlu/Econometrics": 0.649, - "helm_mmlu/Global Facts": 0.53, - "helm_mmlu/Jurisprudence": 0.88, - "helm_mmlu/Philosophy": 0.836, - "helm_mmlu/Professional Psychology": 0.858, - "helm_mmlu/Us Foreign Policy": 0.96, - "helm_mmlu/Astronomy": 0.862, - "helm_mmlu/Business Ethics": 0.83, - "helm_mmlu/Clinical Knowledge": 0.804, - "helm_mmlu/Conceptual Physics": 0.809, - "helm_mmlu/Electrical Engineering": 0.772, - "helm_mmlu/Elementary Mathematics": 0.661, - "helm_mmlu/Formal Logic": 0.659, - "helm_mmlu/High School World History": 0.911, - "helm_mmlu/Human Sexuality": 0.924, - "helm_mmlu/International Law": 0.909, - "helm_mmlu/Logical Fallacies": 0.877, - "helm_mmlu/Machine Learning": 0.625, - "helm_mmlu/Management": 0.903, - "helm_mmlu/Marketing": 0.94, - "helm_mmlu/Medical Genetics": 0.83, - "helm_mmlu/Miscellaneous": 0.894, - "helm_mmlu/Moral Scenarios": 0.562, - "helm_mmlu/Nutrition": 0.856, - "helm_mmlu/Prehistory": 0.87, - "helm_mmlu/Public Relations": 0.773, - "helm_mmlu/Security Studies": 0.829, - "helm_mmlu/Sociology": 0.91, - "helm_mmlu/Virology": 0.572, - "helm_mmlu/World Religions": 0.877, - "helm_mmlu/Mean win rate": 0.142 - } - }, - { - "id": "google/UL2-20B", - "name": "UL2 20B", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.167, - "helm_classic/MMLU": 0.291, - "helm_classic/BoolQ": 0.746, - "helm_classic/NarrativeQA": 0.083, - "helm_classic/NaturalQuestions (open-book)": 0.349, - "helm_classic/QuAC": 0.144, - "helm_classic/HellaSwag": -1.0, - "helm_classic/OpenbookQA": -1.0, - "helm_classic/TruthfulQA": 0.193, - "helm_classic/MS MARCO (TREC)": -1.0, - "helm_classic/CNN/DailyMail": 0.03, - "helm_classic/XSUM": 0.058, - "helm_classic/IMDB": 0.337, - "helm_classic/CivilComments": 0.521, - "helm_classic/RAFT": 0.404 - } - }, - { - "id": "google/umt5-base", - "name": "umt5-base", - "developer": "Google", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1746, - "hfopenllm_v2/BBH": 0.2788, - "hfopenllm_v2/MATH Level 5": 0.0045, - "hfopenllm_v2/GPQA": 0.2542, - "hfopenllm_v2/MUSR": 0.3382, - "hfopenllm_v2/MMLU-PRO": 0.1078 - } - }, - { - "id": "GoToCompany/gemma2-9b-cpt-sahabatai-v1-instruct", - "name": "gemma2-9b-cpt-sahabatai-v1-instruct", - "developer": "GoToCompany", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6551, - "hfopenllm_v2/BBH": 0.5955, - "hfopenllm_v2/MATH Level 5": 0.2054, - "hfopenllm_v2/GPQA": 0.3347, - "hfopenllm_v2/MUSR": 0.4779, - "hfopenllm_v2/MMLU-PRO": 0.4264 - } - }, - { - "id": "GoToCompany/llama3-8b-cpt-sahabatai-v1-instruct", - "name": "llama3-8b-cpt-sahabatai-v1-instruct", - "developer": "GoToCompany", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5238, - "hfopenllm_v2/BBH": 0.4951, - "hfopenllm_v2/MATH Level 5": 0.1276, - "hfopenllm_v2/GPQA": 0.2668, - "hfopenllm_v2/MUSR": 0.4488, - "hfopenllm_v2/MMLU-PRO": 0.3453 - } - }, - { - "id": "goulue5/merging_LLM", - "name": "merging_LLM", - "developer": "goulue5", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3233, - "hfopenllm_v2/BBH": 0.4216, - "hfopenllm_v2/MATH Level 5": 0.0967, - "hfopenllm_v2/GPQA": 0.2911, - "hfopenllm_v2/MUSR": 0.4333, - "hfopenllm_v2/MMLU-PRO": 0.2958 - } - }, - { - "id": "gradientai/Llama-3-8B-Instruct-Gradient-1048k", - "name": "Llama-3-8B-Instruct-Gradient-1048k", - "developer": "gradientai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4456, - "hfopenllm_v2/BBH": 0.4346, - "hfopenllm_v2/MATH Level 5": 0.0536, - "hfopenllm_v2/GPQA": 0.2777, - "hfopenllm_v2/MUSR": 0.4298, - "hfopenllm_v2/MMLU-PRO": 0.294 - } - }, - { - "id": "GreenNode/GreenNode-small-9B-it", - "name": "GreenNode-small-9B-it", - "developer": "GreenNode", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7436, - "hfopenllm_v2/BBH": 0.5994, - "hfopenllm_v2/MATH Level 5": 0.1745, - "hfopenllm_v2/GPQA": 0.3196, - "hfopenllm_v2/MUSR": 0.4204, - "hfopenllm_v2/MMLU-PRO": 0.3927 - } - }, - { - "id": "grimjim/DeepSauerHuatuoSkywork-R1-o1-Llama-3.1-8B", - "name": "DeepSauerHuatuoSkywork-R1-o1-Llama-3.1-8B", - "developer": "grimjim", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4797, - "hfopenllm_v2/BBH": 0.5269, - "hfopenllm_v2/MATH Level 5": 0.2221, - "hfopenllm_v2/GPQA": 0.3381, - "hfopenllm_v2/MUSR": 0.4408, - "hfopenllm_v2/MMLU-PRO": 0.3957 - } - }, - { - "id": "grimjim/Gigantes-v1-gemma2-9b-it", - "name": "Gigantes-v1-gemma2-9b-it", - "developer": "grimjim", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6925, - "hfopenllm_v2/BBH": 0.5978, - "hfopenllm_v2/MATH Level 5": 0.2145, - "hfopenllm_v2/GPQA": 0.3532, - "hfopenllm_v2/MUSR": 0.4555, - "hfopenllm_v2/MMLU-PRO": 0.4225 - } - }, - { - "id": "grimjim/Gigantes-v2-gemma2-9b-it", - "name": "Gigantes-v2-gemma2-9b-it", - "developer": "grimjim", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7351, - "hfopenllm_v2/BBH": 0.5987, - "hfopenllm_v2/MATH Level 5": 0.2017, - "hfopenllm_v2/GPQA": 0.3515, - "hfopenllm_v2/MUSR": 0.4595, - "hfopenllm_v2/MMLU-PRO": 0.4259 - } - }, - { - "id": "grimjim/Gigantes-v3-gemma2-9b-it", - "name": "Gigantes-v3-gemma2-9b-it", - "developer": "grimjim", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6976, - "hfopenllm_v2/BBH": 0.5984, - "hfopenllm_v2/MATH Level 5": 0.21, - "hfopenllm_v2/GPQA": 0.3565, - "hfopenllm_v2/MUSR": 0.4608, - "hfopenllm_v2/MMLU-PRO": 0.4226 - } - }, - { - "id": "grimjim/HuatuoSkywork-o1-Llama-3.1-8B", - "name": "HuatuoSkywork-o1-Llama-3.1-8B", - "developer": "grimjim", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3961, - "hfopenllm_v2/BBH": 0.4886, - "hfopenllm_v2/MATH Level 5": 0.3882, - "hfopenllm_v2/GPQA": 0.2928, - "hfopenllm_v2/MUSR": 0.3839, - "hfopenllm_v2/MMLU-PRO": 0.3095 - } - }, - { - "id": "grimjim/Llama-3-Instruct-8B-SimPO-SPPO-Iter3-merge", - "name": "Llama-3-Instruct-8B-SimPO-SPPO-Iter3-merge", - "developer": "grimjim", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6806, - "hfopenllm_v2/BBH": 0.5022, - "hfopenllm_v2/MATH Level 5": 0.0891, - "hfopenllm_v2/GPQA": 0.2626, - "hfopenllm_v2/MUSR": 0.3885, - "hfopenllm_v2/MMLU-PRO": 0.3684 - } - }, - { - "id": "grimjim/Llama-3-Instruct-8B-SPPO-Iter3-SimPO-merge", - "name": "Llama-3-Instruct-8B-SPPO-Iter3-SimPO-merge", - "developer": "grimjim", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4271, - "hfopenllm_v2/BBH": 0.4962, - "hfopenllm_v2/MATH Level 5": 0.0997, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.4043, - "hfopenllm_v2/MMLU-PRO": 0.3625 - } - }, - { - "id": "grimjim/llama-3-Nephilim-v1-8B", - "name": "llama-3-Nephilim-v1-8B", - "developer": "grimjim", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4277, - "hfopenllm_v2/BBH": 0.5132, - "hfopenllm_v2/MATH Level 5": 0.0906, - "hfopenllm_v2/GPQA": 0.302, - "hfopenllm_v2/MUSR": 0.4136, - "hfopenllm_v2/MMLU-PRO": 0.3796 - } - }, - { - "id": "grimjim/llama-3-Nephilim-v2-8B", - "name": "llama-3-Nephilim-v2-8B", - "developer": "grimjim", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3922, - "hfopenllm_v2/BBH": 0.5048, - "hfopenllm_v2/MATH Level 5": 0.1065, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.3895, - "hfopenllm_v2/MMLU-PRO": 0.3641 - } - }, - { - "id": "grimjim/llama-3-Nephilim-v2.1-8B", - "name": "llama-3-Nephilim-v2.1-8B", - "developer": "grimjim", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3895, - "hfopenllm_v2/BBH": 0.5095, - "hfopenllm_v2/MATH Level 5": 0.0997, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.3935, - "hfopenllm_v2/MMLU-PRO": 0.3644 - } - }, - { - "id": "grimjim/llama-3-Nephilim-v3-8B", - "name": "llama-3-Nephilim-v3-8B", - "developer": "grimjim", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4174, - "hfopenllm_v2/BBH": 0.5013, - "hfopenllm_v2/MATH Level 5": 0.0952, - "hfopenllm_v2/GPQA": 0.2953, - "hfopenllm_v2/MUSR": 0.3989, - "hfopenllm_v2/MMLU-PRO": 0.3612 - } - }, - { - "id": "grimjim/Llama-3.1-8B-Instruct-abliterated_via_adapter", - "name": "Llama-3.1-8B-Instruct-abliterated_via_adapter", - "developer": "grimjim", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.487, - "hfopenllm_v2/BBH": 0.5105, - "hfopenllm_v2/MATH Level 5": 0.1397, - "hfopenllm_v2/GPQA": 0.3138, - "hfopenllm_v2/MUSR": 0.401, - "hfopenllm_v2/MMLU-PRO": 0.3651 - } - }, - { - "id": "grimjim/Llama-3.1-Bonsaikraft-8B-Instruct", - "name": "Llama-3.1-Bonsaikraft-8B-Instruct", - "developer": "grimjim", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.425, - "hfopenllm_v2/BBH": 0.5287, - "hfopenllm_v2/MATH Level 5": 0.1314, - "hfopenllm_v2/GPQA": 0.3037, - "hfopenllm_v2/MUSR": 0.4235, - "hfopenllm_v2/MMLU-PRO": 0.3764 - } - }, - { - "id": "grimjim/Llama-Nephilim-Metamorphosis-v2-8B", - "name": "Llama-Nephilim-Metamorphosis-v2-8B", - "developer": "grimjim", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4545, - "hfopenllm_v2/BBH": 0.5013, - "hfopenllm_v2/MATH Level 5": 0.1397, - "hfopenllm_v2/GPQA": 0.323, - "hfopenllm_v2/MUSR": 0.4091, - "hfopenllm_v2/MMLU-PRO": 0.3809 - } - }, - { - "id": "grimjim/Llama3.1-SuperNovaLite-HuatuoSkywork-o1-8B", - "name": "Llama3.1-SuperNovaLite-HuatuoSkywork-o1-8B", - "developer": "grimjim", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4366, - "hfopenllm_v2/BBH": 0.5287, - "hfopenllm_v2/MATH Level 5": 0.3006, - "hfopenllm_v2/GPQA": 0.3112, - "hfopenllm_v2/MUSR": 0.3999, - "hfopenllm_v2/MMLU-PRO": 0.3684 - } - }, - { - "id": "grimjim/Magnolia-v1-Gemma2-8k-9B", - "name": "Magnolia-v1-Gemma2-8k-9B", - "developer": "grimjim", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3531, - "hfopenllm_v2/BBH": 0.5589, - "hfopenllm_v2/MATH Level 5": 0.1684, - "hfopenllm_v2/GPQA": 0.3364, - "hfopenllm_v2/MUSR": 0.4645, - "hfopenllm_v2/MMLU-PRO": 0.4242 - } - }, - { - "id": "grimjim/Magnolia-v2-12B", - "name": "Magnolia-v2-12B", - "developer": "grimjim", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3506, - "hfopenllm_v2/BBH": 0.529, - "hfopenllm_v2/MATH Level 5": 0.1292, - "hfopenllm_v2/GPQA": 0.3188, - "hfopenllm_v2/MUSR": 0.4171, - "hfopenllm_v2/MMLU-PRO": 0.3601 - } - }, - { - "id": "grimjim/Magnolia-v2-Gemma2-8k-9B", - "name": "Magnolia-v2-Gemma2-8k-9B", - "developer": "grimjim", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7384, - "hfopenllm_v2/BBH": 0.6016, - "hfopenllm_v2/MATH Level 5": 0.2281, - "hfopenllm_v2/GPQA": 0.3574, - "hfopenllm_v2/MUSR": 0.4488, - "hfopenllm_v2/MMLU-PRO": 0.4332 - } - }, - { - "id": "grimjim/Magnolia-v3-12B", - "name": "Magnolia-v3-12B", - "developer": "grimjim", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3965, - "hfopenllm_v2/BBH": 0.5327, - "hfopenllm_v2/MATH Level 5": 0.1352, - "hfopenllm_v2/GPQA": 0.3255, - "hfopenllm_v2/MUSR": 0.4184, - "hfopenllm_v2/MMLU-PRO": 0.3615 - } - }, - { - "id": "grimjim/Magnolia-v3-Gemma2-8k-9B", - "name": "Magnolia-v3-Gemma2-8k-9B", - "developer": "grimjim", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7378, - "hfopenllm_v2/BBH": 0.6015, - "hfopenllm_v2/MATH Level 5": 0.2319, - "hfopenllm_v2/GPQA": 0.3565, - "hfopenllm_v2/MUSR": 0.4488, - "hfopenllm_v2/MMLU-PRO": 0.4337 - } - }, - { - "id": "grimjim/Magnolia-v4-12B", - "name": "Magnolia-v4-12B", - "developer": "grimjim", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3418, - "hfopenllm_v2/BBH": 0.5431, - "hfopenllm_v2/MATH Level 5": 0.1314, - "hfopenllm_v2/GPQA": 0.328, - "hfopenllm_v2/MUSR": 0.4211, - "hfopenllm_v2/MMLU-PRO": 0.3672 - } - }, - { - "id": "grimjim/Magnolia-v5a-12B", - "name": "Magnolia-v5a-12B", - "developer": "grimjim", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4114, - "hfopenllm_v2/BBH": 0.5312, - "hfopenllm_v2/MATH Level 5": 0.1375, - "hfopenllm_v2/GPQA": 0.3221, - "hfopenllm_v2/MUSR": 0.4145, - "hfopenllm_v2/MMLU-PRO": 0.3601 - } - }, - { - "id": "grimjim/Magot-v1-Gemma2-8k-9B", - "name": "Magot-v1-Gemma2-8k-9B", - "developer": "grimjim", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2997, - "hfopenllm_v2/BBH": 0.6019, - "hfopenllm_v2/MATH Level 5": 0.0989, - "hfopenllm_v2/GPQA": 0.3465, - "hfopenllm_v2/MUSR": 0.4488, - "hfopenllm_v2/MMLU-PRO": 0.4337 - } - }, - { - "id": "grimjim/Magot-v2-Gemma2-8k-9B", - "name": "Magot-v2-Gemma2-8k-9B", - "developer": "grimjim", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7347, - "hfopenllm_v2/BBH": 0.5897, - "hfopenllm_v2/MATH Level 5": 0.2017, - "hfopenllm_v2/GPQA": 0.354, - "hfopenllm_v2/MUSR": 0.4344, - "hfopenllm_v2/MMLU-PRO": 0.4223 - } - }, - { - "id": "grimjim/SauerHuatuoSkywork-o1-Llama-3.1-8B", - "name": "SauerHuatuoSkywork-o1-Llama-3.1-8B", - "developer": "grimjim", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5219, - "hfopenllm_v2/BBH": 0.5222, - "hfopenllm_v2/MATH Level 5": 0.173, - "hfopenllm_v2/GPQA": 0.3213, - "hfopenllm_v2/MUSR": 0.4527, - "hfopenllm_v2/MMLU-PRO": 0.3991 - } - }, - { - "id": "GritLM/GritLM-7B-KTO", - "name": "GritLM-7B-KTO", - "developer": "GritLM", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.531, - "hfopenllm_v2/BBH": 0.4853, - "hfopenllm_v2/MATH Level 5": 0.0272, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.371, - "hfopenllm_v2/MMLU-PRO": 0.268 - } - }, - { - "id": "GritLM/GritLM-8x7B-KTO", - "name": "GritLM-8x7B-KTO", - "developer": "GritLM", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5714, - "hfopenllm_v2/BBH": 0.582, - "hfopenllm_v2/MATH Level 5": 0.1224, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.4217, - "hfopenllm_v2/MMLU-PRO": 0.3648 - } - }, - { - "id": "Groq/Llama-3-Groq-8B-Tool-Use", - "name": "Llama-3-Groq-8B-Tool-Use", - "developer": "Groq", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6098, - "hfopenllm_v2/BBH": 0.4863, - "hfopenllm_v2/MATH Level 5": 0.0604, - "hfopenllm_v2/GPQA": 0.2676, - "hfopenllm_v2/MUSR": 0.366, - "hfopenllm_v2/MMLU-PRO": 0.3399 - } - }, - { - "id": "Gryphe/Pantheon-RP-1.0-8b-Llama-3", - "name": "Pantheon-RP-1.0-8b-Llama-3", - "developer": "Gryphe", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3933, - "hfopenllm_v2/BBH": 0.4539, - "hfopenllm_v2/MATH Level 5": 0.0634, - "hfopenllm_v2/GPQA": 0.276, - "hfopenllm_v2/MUSR": 0.3832, - "hfopenllm_v2/MMLU-PRO": 0.3067 - } - }, - { - "id": "Gryphe/Pantheon-RP-1.5-12b-Nemo", - "name": "Pantheon-RP-1.5-12b-Nemo", - "developer": "Gryphe", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4763, - "hfopenllm_v2/BBH": 0.5196, - "hfopenllm_v2/MATH Level 5": 0.0491, - "hfopenllm_v2/GPQA": 0.2727, - "hfopenllm_v2/MUSR": 0.442, - "hfopenllm_v2/MMLU-PRO": 0.3302 - } - }, - { - "id": "Gryphe/Pantheon-RP-1.6-12b-Nemo", - "name": "Pantheon-RP-1.6-12b-Nemo", - "developer": "Gryphe", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4481, - "hfopenllm_v2/BBH": 0.5204, - "hfopenllm_v2/MATH Level 5": 0.0461, - "hfopenllm_v2/GPQA": 0.2777, - "hfopenllm_v2/MUSR": 0.4288, - "hfopenllm_v2/MMLU-PRO": 0.3311 - } - }, - { - "id": "Gryphe/Pantheon-RP-1.6-12b-Nemo-KTO", - "name": "Pantheon-RP-1.6-12b-Nemo-KTO", - "developer": "Gryphe", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4636, - "hfopenllm_v2/BBH": 0.5277, - "hfopenllm_v2/MATH Level 5": 0.0529, - "hfopenllm_v2/GPQA": 0.2953, - "hfopenllm_v2/MUSR": 0.4248, - "hfopenllm_v2/MMLU-PRO": 0.3382 - } - }, - { - "id": "Gryphe/Pantheon-RP-Pure-1.6.2-22b-Small", - "name": "Pantheon-RP-Pure-1.6.2-22b-Small", - "developer": "Gryphe", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6931, - "hfopenllm_v2/BBH": 0.5305, - "hfopenllm_v2/MATH Level 5": 0.2024, - "hfopenllm_v2/GPQA": 0.3289, - "hfopenllm_v2/MUSR": 0.3765, - "hfopenllm_v2/MMLU-PRO": 0.3942 - } - }, - { - "id": "GuilhermeNaturaUmana/Nature-Reason-1.2-reallysmall", - "name": "Nature-Reason-1.2-reallysmall", - "developer": "GuilhermeNaturaUmana", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4985, - "hfopenllm_v2/BBH": 0.5645, - "hfopenllm_v2/MATH Level 5": 0.2576, - "hfopenllm_v2/GPQA": 0.3003, - "hfopenllm_v2/MUSR": 0.4373, - "hfopenllm_v2/MMLU-PRO": 0.4429 - } - }, - { - "id": "Gunulhona/Gemma-Ko-Merge", - "name": "Gemma-Ko-Merge", - "developer": "Gunulhona", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6416, - "hfopenllm_v2/BBH": 0.5813, - "hfopenllm_v2/MATH Level 5": 0.1881, - "hfopenllm_v2/GPQA": 0.3356, - "hfopenllm_v2/MUSR": 0.4047, - "hfopenllm_v2/MMLU-PRO": 0.3879 - } - }, - { - "id": "Gunulhona/Gemma-Ko-Merge-PEFT", - "name": "Gemma-Ko-Merge-PEFT", - "developer": "Gunulhona", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.288, - "hfopenllm_v2/BBH": 0.5154, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.3247, - "hfopenllm_v2/MUSR": 0.408, - "hfopenllm_v2/MMLU-PRO": 0.3817 - } - }, - { - "id": "gupta-tanish/llama-7b-dpo-baseline", - "name": "llama-7b-dpo-baseline", - "developer": "gupta-tanish", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2693, - "hfopenllm_v2/BBH": 0.3897, - "hfopenllm_v2/MATH Level 5": 0.0196, - "hfopenllm_v2/GPQA": 0.2626, - "hfopenllm_v2/MUSR": 0.4456, - "hfopenllm_v2/MMLU-PRO": 0.2028 - } - }, - { - "id": "gz987/qwen2.5-7b-cabs-v0.1", - "name": "qwen2.5-7b-cabs-v0.1", - "developer": "gz987", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7506, - "hfopenllm_v2/BBH": 0.5482, - "hfopenllm_v2/MATH Level 5": 0.4796, - "hfopenllm_v2/GPQA": 0.3138, - "hfopenllm_v2/MUSR": 0.4376, - "hfopenllm_v2/MMLU-PRO": 0.4406 - } - }, - { - "id": "gz987/qwen2.5-7b-cabs-v0.2", - "name": "qwen2.5-7b-cabs-v0.2", - "developer": "gz987", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7418, - "hfopenllm_v2/BBH": 0.5516, - "hfopenllm_v2/MATH Level 5": 0.4902, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.4429, - "hfopenllm_v2/MMLU-PRO": 0.4397 - } - }, - { - "id": "gz987/qwen2.5-7b-cabs-v0.3", - "name": "qwen2.5-7b-cabs-v0.3", - "developer": "gz987", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.757, - "hfopenllm_v2/BBH": 0.5494, - "hfopenllm_v2/MATH Level 5": 0.4932, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.443, - "hfopenllm_v2/MMLU-PRO": 0.4402 - } - }, - { - "id": "gz987/qwen2.5-7b-cabs-v0.4", - "name": "qwen2.5-7b-cabs-v0.4", - "developer": "gz987", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7583, - "hfopenllm_v2/BBH": 0.5524, - "hfopenllm_v2/MATH Level 5": 0.4849, - "hfopenllm_v2/GPQA": 0.3079, - "hfopenllm_v2/MUSR": 0.443, - "hfopenllm_v2/MMLU-PRO": 0.4396 - } - }, - { - "id": "h2oai/h2o-danube-1.8b-chat", - "name": "h2o-danube-1.8b-chat", - "developer": "h2oai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2199, - "hfopenllm_v2/BBH": 0.322, - "hfopenllm_v2/MATH Level 5": 0.0136, - "hfopenllm_v2/GPQA": 0.2542, - "hfopenllm_v2/MUSR": 0.3989, - "hfopenllm_v2/MMLU-PRO": 0.1314 - } - }, - { - "id": "h2oai/h2o-danube3-4b-base", - "name": "h2o-danube3-4b-base", - "developer": "h2oai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2338, - "hfopenllm_v2/BBH": 0.3599, - "hfopenllm_v2/MATH Level 5": 0.0227, - "hfopenllm_v2/GPQA": 0.2911, - "hfopenllm_v2/MUSR": 0.3778, - "hfopenllm_v2/MMLU-PRO": 0.2109 - } - }, - { - "id": "h2oai/h2o-danube3-4b-chat", - "name": "h2o-danube3-4b-chat", - "developer": "h2oai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3629, - "hfopenllm_v2/BBH": 0.3466, - "hfopenllm_v2/MATH Level 5": 0.0408, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3781, - "hfopenllm_v2/MMLU-PRO": 0.2228 - } - }, - { - "id": "h2oai/h2o-danube3-500m-chat", - "name": "h2o-danube3-500m-chat", - "developer": "h2oai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2208, - "hfopenllm_v2/BBH": 0.3035, - "hfopenllm_v2/MATH Level 5": 0.0166, - "hfopenllm_v2/GPQA": 0.2307, - "hfopenllm_v2/MUSR": 0.3434, - "hfopenllm_v2/MMLU-PRO": 0.1144 - } - }, - { - "id": "h2oai/h2o-danube3.1-4b-chat", - "name": "h2o-danube3.1-4b-chat", - "developer": "h2oai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5021, - "hfopenllm_v2/BBH": 0.3608, - "hfopenllm_v2/MATH Level 5": 0.0332, - "hfopenllm_v2/GPQA": 0.2852, - "hfopenllm_v2/MUSR": 0.4102, - "hfopenllm_v2/MMLU-PRO": 0.2719 - } - }, - { - "id": "haoranxu/ALMA-13B-R", - "name": "ALMA-13B-R", - "developer": "haoranxu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0039, - "hfopenllm_v2/BBH": 0.3457, - "hfopenllm_v2/MATH Level 5": 0.0174, - "hfopenllm_v2/GPQA": 0.2576, - "hfopenllm_v2/MUSR": 0.3528, - "hfopenllm_v2/MMLU-PRO": 0.1817 - } - }, - { - "id": "haoranxu/Llama-3-Instruct-8B-CPO-SimPO", - "name": "Llama-3-Instruct-8B-CPO-SimPO", - "developer": "haoranxu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7046, - "hfopenllm_v2/BBH": 0.5048, - "hfopenllm_v2/MATH Level 5": 0.1027, - "hfopenllm_v2/GPQA": 0.2928, - "hfopenllm_v2/MUSR": 0.3567, - "hfopenllm_v2/MMLU-PRO": 0.3686 - } - }, - { - "id": "haoranxu/Llama-3-Instruct-8B-SimPO", - "name": "Llama-3-Instruct-8B-SimPO", - "developer": "haoranxu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7347, - "hfopenllm_v2/BBH": 0.4979, - "hfopenllm_v2/MATH Level 5": 0.0876, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.3566, - "hfopenllm_v2/MMLU-PRO": 0.3733 - } - }, - { - "id": "HarbingerX/Zeitgeist-3b-V1", - "name": "Zeitgeist-3b-V1", - "developer": "HarbingerX", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6712, - "hfopenllm_v2/BBH": 0.4441, - "hfopenllm_v2/MATH Level 5": 0.1035, - "hfopenllm_v2/GPQA": 0.2819, - "hfopenllm_v2/MUSR": 0.3579, - "hfopenllm_v2/MMLU-PRO": 0.3009 - } - }, - { - "id": "HarbingerX/Zeitgeist-3b-V1.2", - "name": "Zeitgeist-3b-V1.2", - "developer": "HarbingerX", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6754, - "hfopenllm_v2/BBH": 0.4441, - "hfopenllm_v2/MATH Level 5": 0.1012, - "hfopenllm_v2/GPQA": 0.2777, - "hfopenllm_v2/MUSR": 0.3579, - "hfopenllm_v2/MMLU-PRO": 0.3056 - } - }, - { - "id": "Hastagaras/L3.2-JametMini-3B-MK.III", - "name": "L3.2-JametMini-3B-MK.III", - "developer": "Hastagaras", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6183, - "hfopenllm_v2/BBH": 0.4539, - "hfopenllm_v2/MATH Level 5": 0.1458, - "hfopenllm_v2/GPQA": 0.2827, - "hfopenllm_v2/MUSR": 0.3686, - "hfopenllm_v2/MMLU-PRO": 0.2983 - } - }, - { - "id": "Hastagaras/Llama-3.1-Jamet-8B-MK.I", - "name": "Llama-3.1-Jamet-8B-MK.I", - "developer": "Hastagaras", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7338, - "hfopenllm_v2/BBH": 0.5049, - "hfopenllm_v2/MATH Level 5": 0.1269, - "hfopenllm_v2/GPQA": 0.2743, - "hfopenllm_v2/MUSR": 0.3726, - "hfopenllm_v2/MMLU-PRO": 0.3482 - } - }, - { - "id": "Hastagaras/Zabuza-8B-Llama-3.1", - "name": "Zabuza-8B-Llama-3.1", - "developer": "Hastagaras", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6265, - "hfopenllm_v2/BBH": 0.4539, - "hfopenllm_v2/MATH Level 5": 0.0551, - "hfopenllm_v2/GPQA": 0.2643, - "hfopenllm_v2/MUSR": 0.3568, - "hfopenllm_v2/MMLU-PRO": 0.2923 - } - }, - { - "id": "hatemmahmoud/qwen2.5-1.5b-sft-raft-grpo-hra-doc", - "name": "qwen2.5-1.5b-sft-raft-grpo-hra-doc", - "developer": "hatemmahmoud", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4196, - "hfopenllm_v2/BBH": 0.427, - "hfopenllm_v2/MATH Level 5": 0.2175, - "hfopenllm_v2/GPQA": 0.2676, - "hfopenllm_v2/MUSR": 0.361, - "hfopenllm_v2/MMLU-PRO": 0.2776 - } - }, - { - "id": "HelpingAI/Cipher-20B", - "name": "Cipher-20B", - "developer": "HelpingAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5378, - "hfopenllm_v2/BBH": 0.6032, - "hfopenllm_v2/MATH Level 5": 0.1994, - "hfopenllm_v2/GPQA": 0.2953, - "hfopenllm_v2/MUSR": 0.4003, - "hfopenllm_v2/MMLU-PRO": 0.3744 - } - }, - { - "id": "HelpingAI/Dhanishtha-Large", - "name": "Dhanishtha-Large", - "developer": "HelpingAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2457, - "hfopenllm_v2/BBH": 0.4604, - "hfopenllm_v2/MATH Level 5": 0.3852, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.3845, - "hfopenllm_v2/MMLU-PRO": 0.2755 - } - }, - { - "id": "HelpingAI/Priya-10B", - "name": "Priya-10B", - "developer": "HelpingAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4043, - "hfopenllm_v2/BBH": 0.4441, - "hfopenllm_v2/MATH Level 5": 0.0189, - "hfopenllm_v2/GPQA": 0.2559, - "hfopenllm_v2/MUSR": 0.3793, - "hfopenllm_v2/MMLU-PRO": 0.2493 - } - }, - { - "id": "HelpingAI/Priya-3B", - "name": "Priya-3B", - "developer": "HelpingAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4526, - "hfopenllm_v2/BBH": 0.3961, - "hfopenllm_v2/MATH Level 5": 0.0144, - "hfopenllm_v2/GPQA": 0.2567, - "hfopenllm_v2/MUSR": 0.3713, - "hfopenllm_v2/MMLU-PRO": 0.2339 - } - }, - { - "id": "hendrydong/Mistral-RM-for-RAFT-GSHF-v0", - "name": "hendrydong/Mistral-RM-for-RAFT-GSHF-v0", - "developer": "hendrydong", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7847, - "reward-bench/Factuality": 0.5779, - "reward-bench/Precise IF": 0.3625, - "reward-bench/Math": 0.6011, - "reward-bench/Safety": 0.85, - "reward-bench/Focus": 0.6747, - "reward-bench/Ties": 0.5988, - "reward-bench/Chat": 0.9832, - "reward-bench/Chat Hard": 0.5789, - "reward-bench/Reasoning": 0.7434, - "reward-bench/Prior Sets (0.5 weight)": 0.7508 - } - }, - { - "id": "HeraiHench/DeepSeek-R1-Qwen-Coder-8B", - "name": "DeepSeek-R1-Qwen-Coder-8B", - "developer": "HeraiHench", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1869, - "hfopenllm_v2/BBH": 0.2913, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3738, - "hfopenllm_v2/MMLU-PRO": 0.1123 - } - }, - { - "id": "HeraiHench/Double-Down-Qwen-Math-7B", - "name": "Double-Down-Qwen-Math-7B", - "developer": "HeraiHench", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.167, - "hfopenllm_v2/BBH": 0.2845, - "hfopenllm_v2/MATH Level 5": 0.0008, - "hfopenllm_v2/GPQA": 0.2651, - "hfopenllm_v2/MUSR": 0.3737, - "hfopenllm_v2/MMLU-PRO": 0.1112 - } - }, - { - "id": "HeraiHench/Marge-Qwen-Math-7B", - "name": "Marge-Qwen-Math-7B", - "developer": "HeraiHench", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1262, - "hfopenllm_v2/BBH": 0.3069, - "hfopenllm_v2/MATH Level 5": 0.0053, - "hfopenllm_v2/GPQA": 0.2391, - "hfopenllm_v2/MUSR": 0.3939, - "hfopenllm_v2/MMLU-PRO": 0.1056 - } - }, - { - "id": "HeraiHench/Phi-4-slerp-ReasoningRP-14B", - "name": "Phi-4-slerp-ReasoningRP-14B", - "developer": "HeraiHench", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1575, - "hfopenllm_v2/BBH": 0.4196, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2936, - "hfopenllm_v2/MUSR": 0.3116, - "hfopenllm_v2/MMLU-PRO": 0.19 - } - }, - { - "id": "HFXM/RAMO-Llama3.1-8B", - "name": "HFXM/RAMO-Llama3.1-8B", - "developer": "HFXM", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6917, - "reward-bench/Factuality": 0.6547, - "reward-bench/Precise IF": 0.375, - "reward-bench/Math": 0.5628, - "reward-bench/Safety": 0.9756, - "reward-bench/Focus": 0.9071, - "reward-bench/Ties": 0.6752 - } - }, - { - "id": "HiroseKoichi/Llama-Salad-4x8B-V3", - "name": "Llama-Salad-4x8B-V3", - "developer": "HiroseKoichi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6654, - "hfopenllm_v2/BBH": 0.5245, - "hfopenllm_v2/MATH Level 5": 0.0959, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.374, - "hfopenllm_v2/MMLU-PRO": 0.3518 - } - }, - { - "id": "HoangHa/Pensez-Llama3.1-8B", - "name": "Pensez-Llama3.1-8B", - "developer": "HoangHa", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3887, - "hfopenllm_v2/BBH": 0.4669, - "hfopenllm_v2/MATH Level 5": 0.1148, - "hfopenllm_v2/GPQA": 0.2886, - "hfopenllm_v2/MUSR": 0.3597, - "hfopenllm_v2/MMLU-PRO": 0.3126 - } - }, - { - "id": "hon9kon9ize/CantoneseLLMChat-v0.5", - "name": "CantoneseLLMChat-v0.5", - "developer": "hon9kon9ize", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3231, - "hfopenllm_v2/BBH": 0.4345, - "hfopenllm_v2/MATH Level 5": 0.0415, - "hfopenllm_v2/GPQA": 0.2777, - "hfopenllm_v2/MUSR": 0.4706, - "hfopenllm_v2/MMLU-PRO": 0.2504 - } - }, - { - "id": "hon9kon9ize/CantoneseLLMChat-v1.0-7B", - "name": "CantoneseLLMChat-v1.0-7B", - "developer": "hon9kon9ize", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4455, - "hfopenllm_v2/BBH": 0.4866, - "hfopenllm_v2/MATH Level 5": 0.2107, - "hfopenllm_v2/GPQA": 0.3221, - "hfopenllm_v2/MUSR": 0.3883, - "hfopenllm_v2/MMLU-PRO": 0.3785 - } - }, - { - "id": "hongbai12/li-0.4-pre", - "name": "li-0.4-pre", - "developer": "hongbai12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.52, - "hfopenllm_v2/BBH": 0.6298, - "hfopenllm_v2/MATH Level 5": 0.4924, - "hfopenllm_v2/GPQA": 0.323, - "hfopenllm_v2/MUSR": 0.4513, - "hfopenllm_v2/MMLU-PRO": 0.5015 - } - }, - { - "id": "hotmailuser/Deepseek-qwen-modelstock-2B", - "name": "Deepseek-qwen-modelstock-2B", - "developer": "hotmailuser", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2149, - "hfopenllm_v2/BBH": 0.3549, - "hfopenllm_v2/MATH Level 5": 0.3399, - "hfopenllm_v2/GPQA": 0.2802, - "hfopenllm_v2/MUSR": 0.3475, - "hfopenllm_v2/MMLU-PRO": 0.1911 - } - }, - { - "id": "hotmailuser/Falcon3Slerp1-10B", - "name": "Falcon3Slerp1-10B", - "developer": "hotmailuser", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5694, - "hfopenllm_v2/BBH": 0.617, - "hfopenllm_v2/MATH Level 5": 0.2598, - "hfopenllm_v2/GPQA": 0.344, - "hfopenllm_v2/MUSR": 0.4318, - "hfopenllm_v2/MMLU-PRO": 0.4402 - } - }, - { - "id": "hotmailuser/Falcon3Slerp2-10B", - "name": "Falcon3Slerp2-10B", - "developer": "hotmailuser", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6118, - "hfopenllm_v2/BBH": 0.6164, - "hfopenllm_v2/MATH Level 5": 0.2319, - "hfopenllm_v2/GPQA": 0.3381, - "hfopenllm_v2/MUSR": 0.4096, - "hfopenllm_v2/MMLU-PRO": 0.4369 - } - }, - { - "id": "hotmailuser/Falcon3Slerp4-10B", - "name": "Falcon3Slerp4-10B", - "developer": "hotmailuser", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6072, - "hfopenllm_v2/BBH": 0.6114, - "hfopenllm_v2/MATH Level 5": 0.2289, - "hfopenllm_v2/GPQA": 0.3289, - "hfopenllm_v2/MUSR": 0.4017, - "hfopenllm_v2/MMLU-PRO": 0.4387 - } - }, - { - "id": "hotmailuser/FalconSlerp-3B", - "name": "FalconSlerp-3B", - "developer": "hotmailuser", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5695, - "hfopenllm_v2/BBH": 0.4624, - "hfopenllm_v2/MATH Level 5": 0.176, - "hfopenllm_v2/GPQA": 0.2878, - "hfopenllm_v2/MUSR": 0.3989, - "hfopenllm_v2/MMLU-PRO": 0.2968 - } - }, - { - "id": "hotmailuser/FalconSlerp1-7B", - "name": "FalconSlerp1-7B", - "developer": "hotmailuser", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5395, - "hfopenllm_v2/BBH": 0.5355, - "hfopenllm_v2/MATH Level 5": 0.2379, - "hfopenllm_v2/GPQA": 0.3196, - "hfopenllm_v2/MUSR": 0.4452, - "hfopenllm_v2/MMLU-PRO": 0.4129 - } - }, - { - "id": "hotmailuser/FalconSlerp2-7B", - "name": "FalconSlerp2-7B", - "developer": "hotmailuser", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.616, - "hfopenllm_v2/BBH": 0.5538, - "hfopenllm_v2/MATH Level 5": 0.2983, - "hfopenllm_v2/GPQA": 0.3196, - "hfopenllm_v2/MUSR": 0.4479, - "hfopenllm_v2/MMLU-PRO": 0.4141 - } - }, - { - "id": "hotmailuser/FalconSlerp3-10B", - "name": "FalconSlerp3-10B", - "developer": "hotmailuser", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6002, - "hfopenllm_v2/BBH": 0.606, - "hfopenllm_v2/MATH Level 5": 0.2273, - "hfopenllm_v2/GPQA": 0.3356, - "hfopenllm_v2/MUSR": 0.4031, - "hfopenllm_v2/MMLU-PRO": 0.4323 - } - }, - { - "id": "hotmailuser/FalconSlerp3-7B", - "name": "FalconSlerp3-7B", - "developer": "hotmailuser", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6096, - "hfopenllm_v2/BBH": 0.5533, - "hfopenllm_v2/MATH Level 5": 0.3157, - "hfopenllm_v2/GPQA": 0.3188, - "hfopenllm_v2/MUSR": 0.4507, - "hfopenllm_v2/MMLU-PRO": 0.4127 - } - }, - { - "id": "hotmailuser/FalconSlerp4-7B", - "name": "FalconSlerp4-7B", - "developer": "hotmailuser", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6285, - "hfopenllm_v2/BBH": 0.5524, - "hfopenllm_v2/MATH Level 5": 0.2213, - "hfopenllm_v2/GPQA": 0.3322, - "hfopenllm_v2/MUSR": 0.4585, - "hfopenllm_v2/MMLU-PRO": 0.4032 - } - }, - { - "id": "hotmailuser/FalconSlerp6-7B", - "name": "FalconSlerp6-7B", - "developer": "hotmailuser", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6027, - "hfopenllm_v2/BBH": 0.5384, - "hfopenllm_v2/MATH Level 5": 0.2047, - "hfopenllm_v2/GPQA": 0.318, - "hfopenllm_v2/MUSR": 0.4492, - "hfopenllm_v2/MMLU-PRO": 0.3995 - } - }, - { - "id": "hotmailuser/Gemma2atlas-27B", - "name": "Gemma2atlas-27B", - "developer": "hotmailuser", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7214, - "hfopenllm_v2/BBH": 0.6545, - "hfopenllm_v2/MATH Level 5": 0.2145, - "hfopenllm_v2/GPQA": 0.3557, - "hfopenllm_v2/MUSR": 0.4445, - "hfopenllm_v2/MMLU-PRO": 0.475 - } - }, - { - "id": "hotmailuser/Gemma2Crono-27B", - "name": "Gemma2Crono-27B", - "developer": "hotmailuser", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7086, - "hfopenllm_v2/BBH": 0.6505, - "hfopenllm_v2/MATH Level 5": 0.2424, - "hfopenllm_v2/GPQA": 0.3708, - "hfopenllm_v2/MUSR": 0.4567, - "hfopenllm_v2/MMLU-PRO": 0.4633 - } - }, - { - "id": "hotmailuser/Gemma2magnum-27b", - "name": "Gemma2magnum-27b", - "developer": "hotmailuser", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5051, - "hfopenllm_v2/BBH": 0.62, - "hfopenllm_v2/MATH Level 5": 0.2205, - "hfopenllm_v2/GPQA": 0.3851, - "hfopenllm_v2/MUSR": 0.4723, - "hfopenllm_v2/MMLU-PRO": 0.4596 - } - }, - { - "id": "hotmailuser/Gemma2SimPO-27B", - "name": "Gemma2SimPO-27B", - "developer": "hotmailuser", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7222, - "hfopenllm_v2/BBH": 0.6413, - "hfopenllm_v2/MATH Level 5": 0.2817, - "hfopenllm_v2/GPQA": 0.3582, - "hfopenllm_v2/MUSR": 0.4447, - "hfopenllm_v2/MMLU-PRO": 0.4642 - } - }, - { - "id": "hotmailuser/Llama-Hermes-slerp-8B", - "name": "Llama-Hermes-slerp-8B", - "developer": "hotmailuser", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.339, - "hfopenllm_v2/BBH": 0.531, - "hfopenllm_v2/MATH Level 5": 0.0801, - "hfopenllm_v2/GPQA": 0.2936, - "hfopenllm_v2/MUSR": 0.4078, - "hfopenllm_v2/MMLU-PRO": 0.3331 - } - }, - { - "id": "hotmailuser/Llama-Hermes-slerp2-8B", - "name": "Llama-Hermes-slerp2-8B", - "developer": "hotmailuser", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3728, - "hfopenllm_v2/BBH": 0.5265, - "hfopenllm_v2/MATH Level 5": 0.0974, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.4248, - "hfopenllm_v2/MMLU-PRO": 0.3379 - } - }, - { - "id": "hotmailuser/LlamaStock-8B", - "name": "LlamaStock-8B", - "developer": "hotmailuser", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.425, - "hfopenllm_v2/BBH": 0.5329, - "hfopenllm_v2/MATH Level 5": 0.1699, - "hfopenllm_v2/GPQA": 0.3272, - "hfopenllm_v2/MUSR": 0.4129, - "hfopenllm_v2/MMLU-PRO": 0.3807 - } - }, - { - "id": "hotmailuser/Mistral-modelstock-24B", - "name": "Mistral-modelstock-24B", - "developer": "hotmailuser", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3424, - "hfopenllm_v2/BBH": 0.6452, - "hfopenllm_v2/MATH Level 5": 0.1307, - "hfopenllm_v2/GPQA": 0.4102, - "hfopenllm_v2/MUSR": 0.459, - "hfopenllm_v2/MMLU-PRO": 0.507 - } - }, - { - "id": "hotmailuser/Mistral-modelstock2-24B", - "name": "Mistral-modelstock2-24B", - "developer": "hotmailuser", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4318, - "hfopenllm_v2/BBH": 0.6689, - "hfopenllm_v2/MATH Level 5": 0.2402, - "hfopenllm_v2/GPQA": 0.3926, - "hfopenllm_v2/MUSR": 0.4616, - "hfopenllm_v2/MMLU-PRO": 0.5318 - } - }, - { - "id": "hotmailuser/Phi4-Slerp4-14B", - "name": "Phi4-Slerp4-14B", - "developer": "hotmailuser", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0629, - "hfopenllm_v2/BBH": 0.6731, - "hfopenllm_v2/MATH Level 5": 0.3474, - "hfopenllm_v2/GPQA": 0.3968, - "hfopenllm_v2/MUSR": 0.5097, - "hfopenllm_v2/MMLU-PRO": 0.5278 - } - }, - { - "id": "hotmailuser/Qwen2.5-HomerSlerp-7B", - "name": "Qwen2.5-HomerSlerp-7B", - "developer": "hotmailuser", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4488, - "hfopenllm_v2/BBH": 0.5633, - "hfopenllm_v2/MATH Level 5": 0.3316, - "hfopenllm_v2/GPQA": 0.3138, - "hfopenllm_v2/MUSR": 0.4383, - "hfopenllm_v2/MMLU-PRO": 0.4549 - } - }, - { - "id": "hotmailuser/QwenModelStock-1.8B", - "name": "QwenModelStock-1.8B", - "developer": "hotmailuser", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3263, - "hfopenllm_v2/BBH": 0.4188, - "hfopenllm_v2/MATH Level 5": 0.0989, - "hfopenllm_v2/GPQA": 0.2869, - "hfopenllm_v2/MUSR": 0.4359, - "hfopenllm_v2/MMLU-PRO": 0.2959 - } - }, - { - "id": "hotmailuser/QwenSlerp-14B", - "name": "QwenSlerp-14B", - "developer": "hotmailuser", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7025, - "hfopenllm_v2/BBH": 0.6491, - "hfopenllm_v2/MATH Level 5": 0.3837, - "hfopenllm_v2/GPQA": 0.3876, - "hfopenllm_v2/MUSR": 0.4634, - "hfopenllm_v2/MMLU-PRO": 0.54 - } - }, - { - "id": "hotmailuser/QwenSlerp-3B", - "name": "QwenSlerp-3B", - "developer": "hotmailuser", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4334, - "hfopenllm_v2/BBH": 0.4892, - "hfopenllm_v2/MATH Level 5": 0.2749, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.4317, - "hfopenllm_v2/MMLU-PRO": 0.3693 - } - }, - { - "id": "hotmailuser/QwenSlerp-7B", - "name": "QwenSlerp-7B", - "developer": "hotmailuser", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4673, - "hfopenllm_v2/BBH": 0.5636, - "hfopenllm_v2/MATH Level 5": 0.3444, - "hfopenllm_v2/GPQA": 0.318, - "hfopenllm_v2/MUSR": 0.4409, - "hfopenllm_v2/MMLU-PRO": 0.4509 - } - }, - { - "id": "hotmailuser/QwenSlerp2-14B", - "name": "QwenSlerp2-14B", - "developer": "hotmailuser", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7037, - "hfopenllm_v2/BBH": 0.6493, - "hfopenllm_v2/MATH Level 5": 0.3965, - "hfopenllm_v2/GPQA": 0.3809, - "hfopenllm_v2/MUSR": 0.4807, - "hfopenllm_v2/MMLU-PRO": 0.5379 - } - }, - { - "id": "hotmailuser/QwenSlerp2-3B", - "name": "QwenSlerp2-3B", - "developer": "hotmailuser", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.428, - "hfopenllm_v2/BBH": 0.4802, - "hfopenllm_v2/MATH Level 5": 0.2606, - "hfopenllm_v2/GPQA": 0.297, - "hfopenllm_v2/MUSR": 0.4252, - "hfopenllm_v2/MMLU-PRO": 0.3742 - } - }, - { - "id": "hotmailuser/QwenSlerp3-14B", - "name": "QwenSlerp3-14B", - "developer": "hotmailuser", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6632, - "hfopenllm_v2/BBH": 0.6267, - "hfopenllm_v2/MATH Level 5": 0.4305, - "hfopenllm_v2/GPQA": 0.3666, - "hfopenllm_v2/MUSR": 0.4808, - "hfopenllm_v2/MMLU-PRO": 0.5263 - } - }, - { - "id": "hotmailuser/QwenSparse-7B", - "name": "QwenSparse-7B", - "developer": "hotmailuser", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1086, - "hfopenllm_v2/BBH": 0.2896, - "hfopenllm_v2/MATH Level 5": 0.0106, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3562, - "hfopenllm_v2/MMLU-PRO": 0.1122 - } - }, - { - "id": "hotmailuser/QwenStock-0.5B", - "name": "QwenStock-0.5B", - "developer": "hotmailuser", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2049, - "hfopenllm_v2/BBH": 0.2912, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3575, - "hfopenllm_v2/MMLU-PRO": 0.1167 - } - }, - { - "id": "hotmailuser/QwenStock-1.7B", - "name": "QwenStock-1.7B", - "developer": "hotmailuser", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3214, - "hfopenllm_v2/BBH": 0.4188, - "hfopenllm_v2/MATH Level 5": 0.0997, - "hfopenllm_v2/GPQA": 0.2878, - "hfopenllm_v2/MUSR": 0.4412, - "hfopenllm_v2/MMLU-PRO": 0.2955 - } - }, - { - "id": "hotmailuser/QwenStock1-14B", - "name": "QwenStock1-14B", - "developer": "hotmailuser", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6693, - "hfopenllm_v2/BBH": 0.6502, - "hfopenllm_v2/MATH Level 5": 0.3701, - "hfopenllm_v2/GPQA": 0.3859, - "hfopenllm_v2/MUSR": 0.4781, - "hfopenllm_v2/MMLU-PRO": 0.5416 - } - }, - { - "id": "hotmailuser/RombosBeagle-v2beta-MGS-32B", - "name": "RombosBeagle-v2beta-MGS-32B", - "developer": "hotmailuser", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5157, - "hfopenllm_v2/BBH": 0.7037, - "hfopenllm_v2/MATH Level 5": 0.4992, - "hfopenllm_v2/GPQA": 0.38, - "hfopenllm_v2/MUSR": 0.5021, - "hfopenllm_v2/MMLU-PRO": 0.5908 - } - }, - { - "id": "HPAI-BSC/Llama3-Aloe-8B-Alpha", - "name": "Llama3-Aloe-8B-Alpha", - "developer": "HPAI-BSC", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5081, - "hfopenllm_v2/BBH": 0.4831, - "hfopenllm_v2/MATH Level 5": 0.0612, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.3673, - "hfopenllm_v2/MMLU-PRO": 0.3295 - } - }, - { - "id": "HPAI-BSC/Llama3.1-Aloe-Beta-8B", - "name": "Llama3.1-Aloe-Beta-8B", - "developer": "HPAI-BSC", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7253, - "hfopenllm_v2/BBH": 0.5093, - "hfopenllm_v2/MATH Level 5": 0.1828, - "hfopenllm_v2/GPQA": 0.2685, - "hfopenllm_v2/MUSR": 0.3835, - "hfopenllm_v2/MMLU-PRO": 0.358 - } - }, - { - "id": "HPAI-BSC/Qwen2.5-Aloe-Beta-7B", - "name": "Qwen2.5-Aloe-Beta-7B", - "developer": "HPAI-BSC", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4554, - "hfopenllm_v2/BBH": 0.5049, - "hfopenllm_v2/MATH Level 5": 0.3542, - "hfopenllm_v2/GPQA": 0.2911, - "hfopenllm_v2/MUSR": 0.426, - "hfopenllm_v2/MMLU-PRO": 0.4354 - } - }, - { - "id": "huawei-noah-ustc/toolace-2-8b-fc", - "name": "ToolACE-2-8B (FC)", - "developer": "huawei-noah-ustc", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 40.0, - "bfcl/bfcl.overall.overall_accuracy": 42.44, - "bfcl/bfcl.overall.total_cost_usd": 24.43, - "bfcl/bfcl.overall.latency_mean_s": 15.95, - "bfcl/bfcl.overall.latency_std_s": 40.06, - "bfcl/bfcl.overall.latency_p95_s": 65.26, - "bfcl/bfcl.non_live.ast_accuracy": 87.1, - "bfcl/bfcl.non_live.simple_ast_accuracy": 73.42, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 91.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 93.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 91.0, - "bfcl/bfcl.live.live_accuracy": 77.42, - "bfcl/bfcl.live.live_simple_ast_accuracy": 71.32, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 79.39, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 68.75, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 62.5, - "bfcl/bfcl.multi_turn.accuracy": 38.38, - "bfcl/bfcl.multi_turn.base_accuracy": 49.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 28.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 30.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 46.0, - "bfcl/bfcl.web_search.accuracy": 8.5, - "bfcl/bfcl.web_search.base_accuracy": 13.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 4.0, - "bfcl/bfcl.memory.accuracy": 18.49, - "bfcl/bfcl.memory.kv_accuracy": 5.81, - "bfcl/bfcl.memory.vector_accuracy": 16.13, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 33.55, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 75.0, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 90.79, - "bfcl/bfcl.format_sensitivity.max_delta": 81.5, - "bfcl/bfcl.format_sensitivity.stddev": 27.92 - } - }, - { - "id": "HuggingFaceH4/starchat2-15b-v0.1", - "name": "HuggingFaceH4/starchat2-15b-v0.1", - "developer": "HuggingFaceH4", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7322, - "reward-bench/Chat": 0.9385, - "reward-bench/Chat Hard": 0.5548, - "reward-bench/Safety": 0.7095, - "reward-bench/Reasoning": 0.8159, - "reward-bench/Prior Sets (0.5 weight)": 0.5525 - } - }, - { - "id": "HuggingFaceH4/zephyr-7b-alpha", - "name": "HuggingFaceH4/zephyr-7b-alpha", - "developer": "HuggingFaceH4", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5191, - "hfopenllm_v2/BBH": 0.4583, - "hfopenllm_v2/MATH Level 5": 0.0196, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.395, - "hfopenllm_v2/MMLU-PRO": 0.2795, - "reward-bench/Score": 0.7392, - "reward-bench/Chat": 0.9162, - "reward-bench/Chat Hard": 0.625, - "reward-bench/Safety": 0.7662, - "reward-bench/Reasoning": 0.7514, - "reward-bench/Prior Sets (0.5 weight)": 0.5353 - } - }, - { - "id": "HuggingFaceH4/zephyr-7b-beta", - "name": "HuggingFaceH4/zephyr-7b-beta", - "developer": "HuggingFaceH4", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.495, - "hfopenllm_v2/BBH": 0.4316, - "hfopenllm_v2/MATH Level 5": 0.0287, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.3925, - "hfopenllm_v2/MMLU-PRO": 0.2781, - "reward-bench/Score": 0.7281, - "reward-bench/Chat": 0.9525, - "reward-bench/Chat Hard": 0.6272, - "reward-bench/Safety": 0.6568, - "reward-bench/Reasoning": 0.7789, - "reward-bench/Prior Sets (0.5 weight)": 0.5216 - } - }, - { - "id": "HuggingFaceH4/zephyr-7b-gemma-v0.1", - "name": "HuggingFaceH4/zephyr-7b-gemma-v0.1", - "developer": "HuggingFaceH4", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3364, - "hfopenllm_v2/BBH": 0.4624, - "hfopenllm_v2/MATH Level 5": 0.0816, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.374, - "hfopenllm_v2/MMLU-PRO": 0.2847, - "reward-bench/Score": 0.6758, - "reward-bench/Chat": 0.9581, - "reward-bench/Chat Hard": 0.4956, - "reward-bench/Safety": 0.5824, - "reward-bench/Reasoning": 0.7463, - "reward-bench/Prior Sets (0.5 weight)": 0.5171 - } - }, - { - "id": "HuggingFaceH4/zephyr-orpo-141b-A35b-v0.1", - "name": "zephyr-orpo-141b-A35b-v0.1", - "developer": "HuggingFaceH4", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6511, - "hfopenllm_v2/BBH": 0.629, - "hfopenllm_v2/MATH Level 5": 0.2047, - "hfopenllm_v2/GPQA": 0.3784, - "hfopenllm_v2/MUSR": 0.4465, - "hfopenllm_v2/MMLU-PRO": 0.4586 - } - }, - { - "id": "HuggingFaceTB/SmolLM-1.7B", - "name": "SmolLM-1.7B", - "developer": "HuggingFaceTB", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2362, - "hfopenllm_v2/BBH": 0.3181, - "hfopenllm_v2/MATH Level 5": 0.0166, - "hfopenllm_v2/GPQA": 0.2416, - "hfopenllm_v2/MUSR": 0.3421, - "hfopenllm_v2/MMLU-PRO": 0.1148 - } - }, - { - "id": "HuggingFaceTB/SmolLM-1.7B-Instruct", - "name": "SmolLM-1.7B-Instruct", - "developer": "HuggingFaceTB", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2348, - "hfopenllm_v2/BBH": 0.2885, - "hfopenllm_v2/MATH Level 5": 0.0211, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3487, - "hfopenllm_v2/MMLU-PRO": 0.1166 - } - }, - { - "id": "HuggingFaceTB/SmolLM-135M", - "name": "SmolLM-135M", - "developer": "HuggingFaceTB", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2125, - "hfopenllm_v2/BBH": 0.3046, - "hfopenllm_v2/MATH Level 5": 0.0136, - "hfopenllm_v2/GPQA": 0.2584, - "hfopenllm_v2/MUSR": 0.4366, - "hfopenllm_v2/MMLU-PRO": 0.1122 - } - }, - { - "id": "HuggingFaceTB/SmolLM-135M-Instruct", - "name": "SmolLM-135M-Instruct", - "developer": "HuggingFaceTB", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1214, - "hfopenllm_v2/BBH": 0.3015, - "hfopenllm_v2/MATH Level 5": 0.0053, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3635, - "hfopenllm_v2/MMLU-PRO": 0.1176 - } - }, - { - "id": "HuggingFaceTB/SmolLM-360M", - "name": "SmolLM-360M", - "developer": "HuggingFaceTB", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2134, - "hfopenllm_v2/BBH": 0.3065, - "hfopenllm_v2/MATH Level 5": 0.0113, - "hfopenllm_v2/GPQA": 0.2676, - "hfopenllm_v2/MUSR": 0.4018, - "hfopenllm_v2/MMLU-PRO": 0.1124 - } - }, - { - "id": "HuggingFaceTB/SmolLM-360M-Instruct", - "name": "SmolLM-360M-Instruct", - "developer": "HuggingFaceTB", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1952, - "hfopenllm_v2/BBH": 0.2885, - "hfopenllm_v2/MATH Level 5": 0.0181, - "hfopenllm_v2/GPQA": 0.2643, - "hfopenllm_v2/MUSR": 0.3472, - "hfopenllm_v2/MMLU-PRO": 0.1166 - } - }, - { - "id": "HuggingFaceTB/SmolLM2-1.7B", - "name": "SmolLM2-1.7B", - "developer": "HuggingFaceTB", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.244, - "hfopenllm_v2/BBH": 0.3453, - "hfopenllm_v2/MATH Level 5": 0.0264, - "hfopenllm_v2/GPQA": 0.2794, - "hfopenllm_v2/MUSR": 0.3485, - "hfopenllm_v2/MMLU-PRO": 0.2138 - } - }, - { - "id": "HuggingFaceTB/SmolLM2-1.7B-Instruct", - "name": "SmolLM2-1.7B-Instruct", - "developer": "HuggingFaceTB", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5368, - "hfopenllm_v2/BBH": 0.3599, - "hfopenllm_v2/MATH Level 5": 0.0582, - "hfopenllm_v2/GPQA": 0.2794, - "hfopenllm_v2/MUSR": 0.3421, - "hfopenllm_v2/MMLU-PRO": 0.2054 - } - }, - { - "id": "HuggingFaceTB/SmolLM2-135M", - "name": "SmolLM2-135M", - "developer": "HuggingFaceTB", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1818, - "hfopenllm_v2/BBH": 0.3044, - "hfopenllm_v2/MATH Level 5": 0.0121, - "hfopenllm_v2/GPQA": 0.2483, - "hfopenllm_v2/MUSR": 0.4112, - "hfopenllm_v2/MMLU-PRO": 0.1095 - } - }, - { - "id": "HuggingFaceTB/SmolLM2-135M-Instruct", - "name": "SmolLM2-135M-Instruct", - "developer": "HuggingFaceTB", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0593, - "hfopenllm_v2/BBH": 0.3135, - "hfopenllm_v2/MATH Level 5": 0.0144, - "hfopenllm_v2/GPQA": 0.2341, - "hfopenllm_v2/MUSR": 0.3871, - "hfopenllm_v2/MMLU-PRO": 0.1092 - } - }, - { - "id": "HuggingFaceTB/SmolLM2-360M", - "name": "SmolLM2-360M", - "developer": "HuggingFaceTB", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2115, - "hfopenllm_v2/BBH": 0.3233, - "hfopenllm_v2/MATH Level 5": 0.0121, - "hfopenllm_v2/GPQA": 0.2458, - "hfopenllm_v2/MUSR": 0.3954, - "hfopenllm_v2/MMLU-PRO": 0.1169 - } - }, - { - "id": "HuggingFaceTB/SmolLM2-360M-Instruct", - "name": "SmolLM2-360M-Instruct", - "developer": "HuggingFaceTB", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.083, - "hfopenllm_v2/BBH": 0.3053, - "hfopenllm_v2/MATH Level 5": 0.0083, - "hfopenllm_v2/GPQA": 0.2651, - "hfopenllm_v2/MUSR": 0.3423, - "hfopenllm_v2/MMLU-PRO": 0.1126 - } - }, - { - "id": "huggyllama/llama-13b", - "name": "llama-13b", - "developer": "huggyllama", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2411, - "hfopenllm_v2/BBH": 0.3988, - "hfopenllm_v2/MATH Level 5": 0.0204, - "hfopenllm_v2/GPQA": 0.255, - "hfopenllm_v2/MUSR": 0.3462, - "hfopenllm_v2/MMLU-PRO": 0.1952 - } - }, - { - "id": "huggyllama/llama-65b", - "name": "llama-65b", - "developer": "huggyllama", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2526, - "hfopenllm_v2/BBH": 0.4703, - "hfopenllm_v2/MATH Level 5": 0.031, - "hfopenllm_v2/GPQA": 0.276, - "hfopenllm_v2/MUSR": 0.3595, - "hfopenllm_v2/MMLU-PRO": 0.3078 - } - }, - { - "id": "huggyllama/llama-7b", - "name": "llama-7b", - "developer": "huggyllama", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2501, - "hfopenllm_v2/BBH": 0.3277, - "hfopenllm_v2/MATH Level 5": 0.0083, - "hfopenllm_v2/GPQA": 0.2525, - "hfopenllm_v2/MUSR": 0.3354, - "hfopenllm_v2/MMLU-PRO": 0.1313 - } - }, - { - "id": "huihui-ai/DeepSeek-R1-Distill-Qwen-14B-abliterated-v2", - "name": "DeepSeek-R1-Distill-Qwen-14B-abliterated-v2", - "developer": "huihui-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4211, - "hfopenllm_v2/BBH": 0.3487, - "hfopenllm_v2/MATH Level 5": 0.2205, - "hfopenllm_v2/GPQA": 0.276, - "hfopenllm_v2/MUSR": 0.4701, - "hfopenllm_v2/MMLU-PRO": 0.1915 - } - }, - { - "id": "huihui-ai/Qwen2.5-14B-Instruct-abliterated-v2", - "name": "Qwen2.5-14B-Instruct-abliterated-v2", - "developer": "huihui-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8328, - "hfopenllm_v2/BBH": 0.6324, - "hfopenllm_v2/MATH Level 5": 0.5302, - "hfopenllm_v2/GPQA": 0.3339, - "hfopenllm_v2/MUSR": 0.422, - "hfopenllm_v2/MMLU-PRO": 0.4962 - } - }, - { - "id": "huihui-ai/Qwen2.5-72B-Instruct-abliterated", - "name": "Qwen2.5-72B-Instruct-abliterated", - "developer": "huihui-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8593, - "hfopenllm_v2/BBH": 0.719, - "hfopenllm_v2/MATH Level 5": 0.6012, - "hfopenllm_v2/GPQA": 0.3951, - "hfopenllm_v2/MUSR": 0.4233, - "hfopenllm_v2/MMLU-PRO": 0.5537 - } - }, - { - "id": "huihui-ai/Qwen2.5-7B-Instruct-abliterated", - "name": "Qwen2.5-7B-Instruct-abliterated", - "developer": "huihui-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7546, - "hfopenllm_v2/BBH": 0.5262, - "hfopenllm_v2/MATH Level 5": 0.4577, - "hfopenllm_v2/GPQA": 0.3154, - "hfopenllm_v2/MUSR": 0.3967, - "hfopenllm_v2/MMLU-PRO": 0.418 - } - }, - { - "id": "huihui-ai/Qwen2.5-7B-Instruct-abliterated-v2", - "name": "Qwen2.5-7B-Instruct-abliterated-v2", - "developer": "huihui-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7606, - "hfopenllm_v2/BBH": 0.5377, - "hfopenllm_v2/MATH Level 5": 0.4637, - "hfopenllm_v2/GPQA": 0.3087, - "hfopenllm_v2/MUSR": 0.3981, - "hfopenllm_v2/MMLU-PRO": 0.4208 - } - }, - { - "id": "huihui-ai/QwQ-32B-Coder-Fusion-7030", - "name": "QwQ-32B-Coder-Fusion-7030", - "developer": "huihui-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3865, - "hfopenllm_v2/BBH": 0.6178, - "hfopenllm_v2/MATH Level 5": 0.2795, - "hfopenllm_v2/GPQA": 0.2844, - "hfopenllm_v2/MUSR": 0.3922, - "hfopenllm_v2/MMLU-PRO": 0.4368 - } - }, - { - "id": "huihui-ai/QwQ-32B-Coder-Fusion-8020", - "name": "QwQ-32B-Coder-Fusion-8020", - "developer": "huihui-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6021, - "hfopenllm_v2/BBH": 0.6665, - "hfopenllm_v2/MATH Level 5": 0.4592, - "hfopenllm_v2/GPQA": 0.3549, - "hfopenllm_v2/MUSR": 0.4293, - "hfopenllm_v2/MMLU-PRO": 0.5367 - } - }, - { - "id": "huihui-ai/QwQ-32B-Coder-Fusion-9010", - "name": "QwQ-32B-Coder-Fusion-9010", - "developer": "huihui-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5778, - "hfopenllm_v2/BBH": 0.6727, - "hfopenllm_v2/MATH Level 5": 0.5317, - "hfopenllm_v2/GPQA": 0.3616, - "hfopenllm_v2/MUSR": 0.4682, - "hfopenllm_v2/MMLU-PRO": 0.56 - } - }, - { - "id": "HumanLLMs/Humanish-LLama3-8B-Instruct", - "name": "Humanish-LLama3-8B-Instruct", - "developer": "HumanLLMs", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6498, - "hfopenllm_v2/BBH": 0.4968, - "hfopenllm_v2/MATH Level 5": 0.1027, - "hfopenllm_v2/GPQA": 0.2559, - "hfopenllm_v2/MUSR": 0.3582, - "hfopenllm_v2/MMLU-PRO": 0.3702 - } - }, - { - "id": "HumanLLMs/Humanish-Mistral-Nemo-Instruct-2407", - "name": "Humanish-Mistral-Nemo-Instruct-2407", - "developer": "HumanLLMs", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5451, - "hfopenllm_v2/BBH": 0.5262, - "hfopenllm_v2/MATH Level 5": 0.1367, - "hfopenllm_v2/GPQA": 0.2878, - "hfopenllm_v2/MUSR": 0.3968, - "hfopenllm_v2/MMLU-PRO": 0.3521 - } - }, - { - "id": "HumanLLMs/Humanish-Qwen2.5-7B-Instruct", - "name": "Humanish-Qwen2.5-7B-Instruct", - "developer": "HumanLLMs", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7284, - "hfopenllm_v2/BBH": 0.5364, - "hfopenllm_v2/MATH Level 5": 0.5, - "hfopenllm_v2/GPQA": 0.2987, - "hfopenllm_v2/MUSR": 0.3981, - "hfopenllm_v2/MMLU-PRO": 0.4398 - } - }, - { - "id": "huu-ontocord/wide_3b_orpo_stage1.1-ss1-orpo3", - "name": "wide_3b_orpo_stage1.1-ss1-orpo3", - "developer": "huu-ontocord", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1505, - "hfopenllm_v2/BBH": 0.2937, - "hfopenllm_v2/MATH Level 5": 0.0098, - "hfopenllm_v2/GPQA": 0.2584, - "hfopenllm_v2/MUSR": 0.3618, - "hfopenllm_v2/MMLU-PRO": 0.1164 - } - }, - { - "id": "ibivibiv/colossus_120b", - "name": "colossus_120b", - "developer": "ibivibiv", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4276, - "hfopenllm_v2/BBH": 0.6061, - "hfopenllm_v2/MATH Level 5": 0.0566, - "hfopenllm_v2/GPQA": 0.3087, - "hfopenllm_v2/MUSR": 0.4733, - "hfopenllm_v2/MMLU-PRO": 0.3961 - } - }, - { - "id": "ibivibiv/multimaster-7b-v6", - "name": "multimaster-7b-v6", - "developer": "ibivibiv", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4473, - "hfopenllm_v2/BBH": 0.5194, - "hfopenllm_v2/MATH Level 5": 0.0559, - "hfopenllm_v2/GPQA": 0.3037, - "hfopenllm_v2/MUSR": 0.4396, - "hfopenllm_v2/MMLU-PRO": 0.3095 - } - }, - { - "id": "ibm-granite/granite-3.0-1b-a400m-base", - "name": "granite-3.0-1b-a400m-base", - "developer": "ibm-granite", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2404, - "hfopenllm_v2/BBH": 0.3221, - "hfopenllm_v2/MATH Level 5": 0.0264, - "hfopenllm_v2/GPQA": 0.2475, - "hfopenllm_v2/MUSR": 0.3367, - "hfopenllm_v2/MMLU-PRO": 0.1152 - } - }, - { - "id": "ibm-granite/granite-3.0-1b-a400m-instruct", - "name": "granite-3.0-1b-a400m-instruct", - "developer": "ibm-granite", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3332, - "hfopenllm_v2/BBH": 0.3224, - "hfopenllm_v2/MATH Level 5": 0.0279, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.3623, - "hfopenllm_v2/MMLU-PRO": 0.1244 - } - }, - { - "id": "ibm-granite/granite-3.0-2b-base", - "name": "granite-3.0-2b-base", - "developer": "ibm-granite", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3874, - "hfopenllm_v2/BBH": 0.4047, - "hfopenllm_v2/MATH Level 5": 0.0544, - "hfopenllm_v2/GPQA": 0.2802, - "hfopenllm_v2/MUSR": 0.3434, - "hfopenllm_v2/MMLU-PRO": 0.2381 - } - }, - { - "id": "ibm-granite/granite-3.0-2b-instruct", - "name": "granite-3.0-2b-instruct", - "developer": "ibm-granite", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.514, - "hfopenllm_v2/BBH": 0.4412, - "hfopenllm_v2/MATH Level 5": 0.0921, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.3515, - "hfopenllm_v2/MMLU-PRO": 0.2814 - } - }, - { - "id": "ibm-granite/granite-3.0-3b-a800m-base", - "name": "granite-3.0-3b-a800m-base", - "developer": "ibm-granite", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2732, - "hfopenllm_v2/BBH": 0.3667, - "hfopenllm_v2/MATH Level 5": 0.0483, - "hfopenllm_v2/GPQA": 0.2517, - "hfopenllm_v2/MUSR": 0.342, - "hfopenllm_v2/MMLU-PRO": 0.1891 - } - }, - { - "id": "ibm-granite/granite-3.0-3b-a800m-instruct", - "name": "granite-3.0-3b-a800m-instruct", - "developer": "ibm-granite", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4298, - "hfopenllm_v2/BBH": 0.3753, - "hfopenllm_v2/MATH Level 5": 0.0702, - "hfopenllm_v2/GPQA": 0.281, - "hfopenllm_v2/MUSR": 0.3487, - "hfopenllm_v2/MMLU-PRO": 0.2152 - } - }, - { - "id": "ibm-granite/granite-3.0-8b-base", - "name": "granite-3.0-8b-base", - "developer": "ibm-granite", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4583, - "hfopenllm_v2/BBH": 0.4944, - "hfopenllm_v2/MATH Level 5": 0.1012, - "hfopenllm_v2/GPQA": 0.3255, - "hfopenllm_v2/MUSR": 0.4081, - "hfopenllm_v2/MMLU-PRO": 0.3313 - } - }, - { - "id": "ibm-granite/granite-3.0-8b-instruct", - "name": "granite-3.0-8b-instruct", - "developer": "ibm-granite", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.531, - "hfopenllm_v2/BBH": 0.5192, - "hfopenllm_v2/MATH Level 5": 0.142, - "hfopenllm_v2/GPQA": 0.3322, - "hfopenllm_v2/MUSR": 0.3901, - "hfopenllm_v2/MMLU-PRO": 0.3457 - } - }, - { - "id": "ibm-granite/granite-3.1-1b-a400m-base", - "name": "granite-3.1-1b-a400m-base", - "developer": "ibm-granite", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2519, - "hfopenllm_v2/BBH": 0.3299, - "hfopenllm_v2/MATH Level 5": 0.0272, - "hfopenllm_v2/GPQA": 0.2517, - "hfopenllm_v2/MUSR": 0.3501, - "hfopenllm_v2/MMLU-PRO": 0.1139 - } - }, - { - "id": "ibm-granite/granite-3.1-1b-a400m-instruct", - "name": "granite-3.1-1b-a400m-instruct", - "developer": "ibm-granite", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4686, - "hfopenllm_v2/BBH": 0.328, - "hfopenllm_v2/MATH Level 5": 0.0453, - "hfopenllm_v2/GPQA": 0.2399, - "hfopenllm_v2/MUSR": 0.3302, - "hfopenllm_v2/MMLU-PRO": 0.1217 - } - }, - { - "id": "ibm-granite/granite-3.1-2b-base", - "name": "granite-3.1-2b-base", - "developer": "ibm-granite", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3522, - "hfopenllm_v2/BBH": 0.4047, - "hfopenllm_v2/MATH Level 5": 0.0566, - "hfopenllm_v2/GPQA": 0.2777, - "hfopenllm_v2/MUSR": 0.3486, - "hfopenllm_v2/MMLU-PRO": 0.2251 - } - }, - { - "id": "ibm-granite/granite-3.1-2b-instruct", - "name": "granite-3.1-2b-instruct", - "developer": "ibm-granite", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6286, - "hfopenllm_v2/BBH": 0.4409, - "hfopenllm_v2/MATH Level 5": 0.1526, - "hfopenllm_v2/GPQA": 0.2894, - "hfopenllm_v2/MUSR": 0.3605, - "hfopenllm_v2/MMLU-PRO": 0.2819 - } - }, - { - "id": "ibm-granite/granite-3.1-3b-a800m-base", - "name": "granite-3.1-3b-a800m-base", - "developer": "ibm-granite", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2996, - "hfopenllm_v2/BBH": 0.3628, - "hfopenllm_v2/MATH Level 5": 0.0453, - "hfopenllm_v2/GPQA": 0.2777, - "hfopenllm_v2/MUSR": 0.3275, - "hfopenllm_v2/MMLU-PRO": 0.1793 - } - }, - { - "id": "ibm-granite/granite-3.1-3b-a800m-instruct", - "name": "granite-3.1-3b-a800m-instruct", - "developer": "ibm-granite", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5516, - "hfopenllm_v2/BBH": 0.4009, - "hfopenllm_v2/MATH Level 5": 0.114, - "hfopenllm_v2/GPQA": 0.2886, - "hfopenllm_v2/MUSR": 0.3486, - "hfopenllm_v2/MMLU-PRO": 0.2148 - } - }, - { - "id": "ibm-granite/granite-3.1-8b-base", - "name": "granite-3.1-8b-base", - "developer": "ibm-granite", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4221, - "hfopenllm_v2/BBH": 0.4777, - "hfopenllm_v2/MATH Level 5": 0.0944, - "hfopenllm_v2/GPQA": 0.3213, - "hfopenllm_v2/MUSR": 0.3922, - "hfopenllm_v2/MMLU-PRO": 0.3232 - } - }, - { - "id": "ibm-granite/granite-3.1-8b-instruct", - "name": "granite-3.1-8b-instruct", - "developer": "ibm-granite", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7208, - "hfopenllm_v2/BBH": 0.5364, - "hfopenllm_v2/MATH Level 5": 0.2198, - "hfopenllm_v2/GPQA": 0.3121, - "hfopenllm_v2/MUSR": 0.4707, - "hfopenllm_v2/MMLU-PRO": 0.3537 - } - }, - { - "id": "ibm-granite/granite-3.2-2b-instruct", - "name": "granite-3.2-2b-instruct", - "developer": "ibm-granite", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6152, - "hfopenllm_v2/BBH": 0.4387, - "hfopenllm_v2/MATH Level 5": 0.1443, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.3646, - "hfopenllm_v2/MMLU-PRO": 0.2783 - } - }, - { - "id": "ibm-granite/granite-3.2-8b-instruct", - "name": "granite-3.2-8b-instruct", - "developer": "ibm-granite", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7275, - "hfopenllm_v2/BBH": 0.5402, - "hfopenllm_v2/MATH Level 5": 0.2379, - "hfopenllm_v2/GPQA": 0.3154, - "hfopenllm_v2/MUSR": 0.4562, - "hfopenllm_v2/MMLU-PRO": 0.3512 - } - }, - { - "id": "ibm-granite/granite-7b-base", - "name": "granite-7b-base", - "developer": "ibm-granite", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2414, - "hfopenllm_v2/BBH": 0.348, - "hfopenllm_v2/MATH Level 5": 0.0159, - "hfopenllm_v2/GPQA": 0.2458, - "hfopenllm_v2/MUSR": 0.3555, - "hfopenllm_v2/MMLU-PRO": 0.1834 - } - }, - { - "id": "ibm-granite/granite-7b-instruct", - "name": "granite-7b-instruct", - "developer": "ibm-granite", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2972, - "hfopenllm_v2/BBH": 0.3723, - "hfopenllm_v2/MATH Level 5": 0.0204, - "hfopenllm_v2/GPQA": 0.2852, - "hfopenllm_v2/MUSR": 0.402, - "hfopenllm_v2/MMLU-PRO": 0.2286 - } - }, - { - "id": "ibm/granite-20b-functioncalling-fc", - "name": "Granite-20b-FunctionCalling (FC)", - "developer": "ibm", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 93.0, - "bfcl/bfcl.overall.overall_accuracy": 23.23, - "bfcl/bfcl.overall.total_cost_usd": 5.23, - "bfcl/bfcl.overall.latency_mean_s": 3.2, - "bfcl/bfcl.overall.latency_std_s": 3.43, - "bfcl/bfcl.overall.latency_p95_s": 9.97, - "bfcl/bfcl.non_live.ast_accuracy": 82.35, - "bfcl/bfcl.non_live.simple_ast_accuracy": 72.92, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 91.5, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 83.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 81.5, - "bfcl/bfcl.live.live_accuracy": 58.7, - "bfcl/bfcl.live.live_simple_ast_accuracy": 67.83, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 56.7, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 43.75, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 58.33, - "bfcl/bfcl.multi_turn.accuracy": 5.38, - "bfcl/bfcl.multi_turn.base_accuracy": 9.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 3.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 6.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 3.0, - "bfcl/bfcl.web_search.accuracy": 0.0, - "bfcl/bfcl.web_search.base_accuracy": 0.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 0.0, - "bfcl/bfcl.memory.accuracy": 0.0, - "bfcl/bfcl.memory.kv_accuracy": 0.0, - "bfcl/bfcl.memory.vector_accuracy": 0.0, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 0.0, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 87.5, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 75.13 - } - }, - { - "id": "ibm/granite-3-1-8b-instruct-fc", - "name": "Granite-3.1-8B-Instruct (FC)", - "developer": "ibm", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 81.0, - "bfcl/bfcl.overall.overall_accuracy": 27.1, - "bfcl/bfcl.overall.total_cost_usd": 9.32, - "bfcl/bfcl.overall.latency_mean_s": 13.23, - "bfcl/bfcl.overall.latency_std_s": 31.28, - "bfcl/bfcl.overall.latency_p95_s": 65.19, - "bfcl/bfcl.non_live.ast_accuracy": 78.33, - "bfcl/bfcl.non_live.simple_ast_accuracy": 67.33, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 92.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 84.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 70.0, - "bfcl/bfcl.live.live_accuracy": 60.33, - "bfcl/bfcl.live.live_simple_ast_accuracy": 58.53, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 61.82, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 18.75, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 41.67, - "bfcl/bfcl.multi_turn.accuracy": 7.5, - "bfcl/bfcl.multi_turn.base_accuracy": 11.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 2.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 7.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 9.0, - "bfcl/bfcl.web_search.accuracy": 0.5, - "bfcl/bfcl.web_search.base_accuracy": 1.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 0.0, - "bfcl/bfcl.memory.accuracy": 14.41, - "bfcl/bfcl.memory.kv_accuracy": 9.68, - "bfcl/bfcl.memory.vector_accuracy": 7.1, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 26.45, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 68.75, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 79.98 - } - }, - { - "id": "ibm/granite-3-2-8b-instruct-fc", - "name": "Granite-3.2-8B-Instruct (FC)", - "developer": "ibm", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 83.0, - "bfcl/bfcl.overall.overall_accuracy": 26.87, - "bfcl/bfcl.overall.total_cost_usd": 25.02, - "bfcl/bfcl.overall.latency_mean_s": 36.13, - "bfcl/bfcl.overall.latency_std_s": 81.76, - "bfcl/bfcl.overall.latency_p95_s": 216.28, - "bfcl/bfcl.non_live.ast_accuracy": 79.77, - "bfcl/bfcl.non_live.simple_ast_accuracy": 69.58, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 88.5, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 88.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 72.5, - "bfcl/bfcl.live.live_accuracy": 60.33, - "bfcl/bfcl.live.live_simple_ast_accuracy": 60.47, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 61.16, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 25.0, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 45.83, - "bfcl/bfcl.multi_turn.accuracy": 7.38, - "bfcl/bfcl.multi_turn.base_accuracy": 9.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 3.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 8.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 9.0, - "bfcl/bfcl.web_search.accuracy": 0.5, - "bfcl/bfcl.web_search.base_accuracy": 1.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 0.0, - "bfcl/bfcl.memory.accuracy": 12.47, - "bfcl/bfcl.memory.kv_accuracy": 6.45, - "bfcl/bfcl.memory.vector_accuracy": 9.68, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 21.29, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 75.0, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 80.53 - } - }, - { - "id": "ibm/granite-3.3-8b-instruct", - "name": "IBM Granite 3.3 8B Instruct", - "developer": "ibm", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.463, - "helm_capabilities/MMLU-Pro": 0.343, - "helm_capabilities/GPQA": 0.325, - "helm_capabilities/IFEval": 0.729, - "helm_capabilities/WildBench": 0.741, - "helm_capabilities/Omni-MATH": 0.176 - } - }, - { - "id": "ibm/granite-4-0-350m-fc", - "name": "Granite-4.0-350m (FC)", - "developer": "ibm", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 103.0, - "bfcl/bfcl.overall.overall_accuracy": 18.98, - "bfcl/bfcl.overall.total_cost_usd": 1.44, - "bfcl/bfcl.overall.latency_mean_s": 1.74, - "bfcl/bfcl.overall.latency_std_s": 4.85, - "bfcl/bfcl.overall.latency_p95_s": 3.44, - "bfcl/bfcl.non_live.ast_accuracy": 67.92, - "bfcl/bfcl.non_live.simple_ast_accuracy": 61.67, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 84.5, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 70.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 55.5, - "bfcl/bfcl.live.live_accuracy": 46.11, - "bfcl/bfcl.live.live_simple_ast_accuracy": 61.24, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 42.36, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 68.75, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 33.33, - "bfcl/bfcl.multi_turn.accuracy": 2.5, - "bfcl/bfcl.multi_turn.base_accuracy": 5.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 0.5, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 2.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 2.0, - "bfcl/bfcl.web_search.accuracy": 0.5, - "bfcl/bfcl.web_search.base_accuracy": 0.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 1.0, - "bfcl/bfcl.memory.accuracy": 3.23, - "bfcl/bfcl.memory.kv_accuracy": 1.94, - "bfcl/bfcl.memory.vector_accuracy": 1.29, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 6.45, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 81.25, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 60.84 - } - }, - { - "id": "ibm/granite-4.0-h-small", - "name": "granite-4.0-h-small", - "developer": "ibm", - "evaluator_relationship": null, - "benchmark_scores": { - "global-mmlu-lite/Global MMLU Lite": 0.7503, - "global-mmlu-lite/Culturally Sensitive": 0.7182, - "global-mmlu-lite/Culturally Agnostic": 0.7826, - "global-mmlu-lite/Arabic": 0.7613, - "global-mmlu-lite/English": 0.77, - "global-mmlu-lite/Bengali": 0.7613, - "global-mmlu-lite/German": 0.755, - "global-mmlu-lite/French": 0.7594, - "global-mmlu-lite/Hindi": 0.7575, - "global-mmlu-lite/Indonesian": 0.7614, - "global-mmlu-lite/Italian": 0.7525, - "global-mmlu-lite/Japanese": 0.7406, - "global-mmlu-lite/Korean": 0.7525, - "global-mmlu-lite/Portuguese": 0.757, - "global-mmlu-lite/Spanish": 0.7638, - "global-mmlu-lite/Swahili": 0.7318, - "global-mmlu-lite/Yoruba": 0.6921, - "global-mmlu-lite/Chinese": 0.7475, - "global-mmlu-lite/Burmese": 0.7419 - } - }, - { - "id": "ibm/merlinite-7b", - "name": "merlinite-7b", - "developer": "ibm", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2499, - "hfopenllm_v2/BBH": 0.5007, - "hfopenllm_v2/MATH Level 5": 0.0242, - "hfopenllm_v2/GPQA": 0.297, - "hfopenllm_v2/MUSR": 0.4412, - "hfopenllm_v2/MMLU-PRO": 0.3068 - } - }, - { - "id": "ibm/PowerLM-3b", - "name": "PowerLM-3b", - "developer": "ibm", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3321, - "hfopenllm_v2/BBH": 0.3679, - "hfopenllm_v2/MATH Level 5": 0.0363, - "hfopenllm_v2/GPQA": 0.2752, - "hfopenllm_v2/MUSR": 0.3563, - "hfopenllm_v2/MMLU-PRO": 0.2016 - } - }, - { - "id": "icefog72/Ice0.15-02.10-RP", - "name": "Ice0.15-02.10-RP", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5343, - "hfopenllm_v2/BBH": 0.4976, - "hfopenllm_v2/MATH Level 5": 0.0574, - "hfopenllm_v2/GPQA": 0.2777, - "hfopenllm_v2/MUSR": 0.432, - "hfopenllm_v2/MMLU-PRO": 0.3066 - } - }, - { - "id": "icefog72/Ice0.16-02.10-RP", - "name": "Ice0.16-02.10-RP", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5069, - "hfopenllm_v2/BBH": 0.4946, - "hfopenllm_v2/MATH Level 5": 0.0589, - "hfopenllm_v2/GPQA": 0.2794, - "hfopenllm_v2/MUSR": 0.4334, - "hfopenllm_v2/MMLU-PRO": 0.3068 - } - }, - { - "id": "icefog72/Ice0.17-03.10-RP", - "name": "Ice0.17-03.10-RP", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5124, - "hfopenllm_v2/BBH": 0.5007, - "hfopenllm_v2/MATH Level 5": 0.0612, - "hfopenllm_v2/GPQA": 0.2819, - "hfopenllm_v2/MUSR": 0.4334, - "hfopenllm_v2/MMLU-PRO": 0.3085 - } - }, - { - "id": "icefog72/Ice0.27-06.11-RP", - "name": "Ice0.27-06.11-RP", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4918, - "hfopenllm_v2/BBH": 0.5112, - "hfopenllm_v2/MATH Level 5": 0.0566, - "hfopenllm_v2/GPQA": 0.3121, - "hfopenllm_v2/MUSR": 0.4328, - "hfopenllm_v2/MMLU-PRO": 0.3154 - } - }, - { - "id": "icefog72/Ice0.29-06.11-RP", - "name": "Ice0.29-06.11-RP", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4861, - "hfopenllm_v2/BBH": 0.5088, - "hfopenllm_v2/MATH Level 5": 0.0566, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.4459, - "hfopenllm_v2/MMLU-PRO": 0.3093 - } - }, - { - "id": "icefog72/Ice0.31-08.11-RP", - "name": "Ice0.31-08.11-RP", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5146, - "hfopenllm_v2/BBH": 0.5032, - "hfopenllm_v2/MATH Level 5": 0.0612, - "hfopenllm_v2/GPQA": 0.3079, - "hfopenllm_v2/MUSR": 0.4277, - "hfopenllm_v2/MMLU-PRO": 0.3131 - } - }, - { - "id": "icefog72/Ice0.32-10.11-RP", - "name": "Ice0.32-10.11-RP", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4915, - "hfopenllm_v2/BBH": 0.5048, - "hfopenllm_v2/MATH Level 5": 0.0514, - "hfopenllm_v2/GPQA": 0.3121, - "hfopenllm_v2/MUSR": 0.4382, - "hfopenllm_v2/MMLU-PRO": 0.31 - } - }, - { - "id": "icefog72/Ice0.34b-14.11-RP", - "name": "Ice0.34b-14.11-RP", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4762, - "hfopenllm_v2/BBH": 0.5067, - "hfopenllm_v2/MATH Level 5": 0.065, - "hfopenllm_v2/GPQA": 0.3096, - "hfopenllm_v2/MUSR": 0.442, - "hfopenllm_v2/MMLU-PRO": 0.3125 - } - }, - { - "id": "icefog72/Ice0.34n-14.11-RP", - "name": "Ice0.34n-14.11-RP", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4787, - "hfopenllm_v2/BBH": 0.5091, - "hfopenllm_v2/MATH Level 5": 0.0725, - "hfopenllm_v2/GPQA": 0.3138, - "hfopenllm_v2/MUSR": 0.438, - "hfopenllm_v2/MMLU-PRO": 0.3124 - } - }, - { - "id": "icefog72/Ice0.37-18.11-RP", - "name": "Ice0.37-18.11-RP", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4972, - "hfopenllm_v2/BBH": 0.5084, - "hfopenllm_v2/MATH Level 5": 0.0642, - "hfopenllm_v2/GPQA": 0.3121, - "hfopenllm_v2/MUSR": 0.4339, - "hfopenllm_v2/MMLU-PRO": 0.3143 - } - }, - { - "id": "icefog72/Ice0.38-19.11-RP", - "name": "Ice0.38-19.11-RP", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4403, - "hfopenllm_v2/BBH": 0.5101, - "hfopenllm_v2/MATH Level 5": 0.0551, - "hfopenllm_v2/GPQA": 0.3045, - "hfopenllm_v2/MUSR": 0.4367, - "hfopenllm_v2/MMLU-PRO": 0.314 - } - }, - { - "id": "icefog72/Ice0.39-19.11-RP", - "name": "Ice0.39-19.11-RP", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4757, - "hfopenllm_v2/BBH": 0.5093, - "hfopenllm_v2/MATH Level 5": 0.0498, - "hfopenllm_v2/GPQA": 0.3104, - "hfopenllm_v2/MUSR": 0.4341, - "hfopenllm_v2/MMLU-PRO": 0.3127 - } - }, - { - "id": "icefog72/Ice0.40-20.11-RP", - "name": "Ice0.40-20.11-RP", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4763, - "hfopenllm_v2/BBH": 0.5093, - "hfopenllm_v2/MATH Level 5": 0.0642, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.4446, - "hfopenllm_v2/MMLU-PRO": 0.3099 - } - }, - { - "id": "icefog72/Ice0.41-22.11-RP", - "name": "Ice0.41-22.11-RP", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.462, - "hfopenllm_v2/BBH": 0.4723, - "hfopenllm_v2/MATH Level 5": 0.031, - "hfopenllm_v2/GPQA": 0.2869, - "hfopenllm_v2/MUSR": 0.456, - "hfopenllm_v2/MMLU-PRO": 0.2618 - } - }, - { - "id": "icefog72/Ice0.50-16.01-RP", - "name": "Ice0.50-16.01-RP", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4385, - "hfopenllm_v2/BBH": 0.498, - "hfopenllm_v2/MATH Level 5": 0.0468, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.4381, - "hfopenllm_v2/MMLU-PRO": 0.3069 - } - }, - { - "id": "icefog72/Ice0.50.1-16.01-RP", - "name": "Ice0.50.1-16.01-RP", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4829, - "hfopenllm_v2/BBH": 0.5107, - "hfopenllm_v2/MATH Level 5": 0.0612, - "hfopenllm_v2/GPQA": 0.3096, - "hfopenllm_v2/MUSR": 0.4327, - "hfopenllm_v2/MMLU-PRO": 0.3132 - } - }, - { - "id": "icefog72/Ice0.51-16.01-RP", - "name": "Ice0.51-16.01-RP", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4431, - "hfopenllm_v2/BBH": 0.5044, - "hfopenllm_v2/MATH Level 5": 0.0514, - "hfopenllm_v2/GPQA": 0.3045, - "hfopenllm_v2/MUSR": 0.4437, - "hfopenllm_v2/MMLU-PRO": 0.306 - } - }, - { - "id": "icefog72/Ice0.51.1-16.01-RP", - "name": "Ice0.51.1-16.01-RP", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4573, - "hfopenllm_v2/BBH": 0.5121, - "hfopenllm_v2/MATH Level 5": 0.0642, - "hfopenllm_v2/GPQA": 0.3062, - "hfopenllm_v2/MUSR": 0.4394, - "hfopenllm_v2/MMLU-PRO": 0.3104 - } - }, - { - "id": "icefog72/Ice0.52-16.01-RP", - "name": "Ice0.52-16.01-RP", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4503, - "hfopenllm_v2/BBH": 0.5047, - "hfopenllm_v2/MATH Level 5": 0.0506, - "hfopenllm_v2/GPQA": 0.3087, - "hfopenllm_v2/MUSR": 0.4396, - "hfopenllm_v2/MMLU-PRO": 0.308 - } - }, - { - "id": "icefog72/Ice0.52.1-16.01-RP", - "name": "Ice0.52.1-16.01-RP", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4549, - "hfopenllm_v2/BBH": 0.5106, - "hfopenllm_v2/MATH Level 5": 0.0627, - "hfopenllm_v2/GPQA": 0.3037, - "hfopenllm_v2/MUSR": 0.4394, - "hfopenllm_v2/MMLU-PRO": 0.3105 - } - }, - { - "id": "icefog72/Ice0.53-16.01-RP", - "name": "Ice0.53-16.01-RP", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4741, - "hfopenllm_v2/BBH": 0.5102, - "hfopenllm_v2/MATH Level 5": 0.0634, - "hfopenllm_v2/GPQA": 0.3087, - "hfopenllm_v2/MUSR": 0.4327, - "hfopenllm_v2/MMLU-PRO": 0.313 - } - }, - { - "id": "icefog72/Ice0.54-17.01-RP", - "name": "Ice0.54-17.01-RP", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4379, - "hfopenllm_v2/BBH": 0.4853, - "hfopenllm_v2/MATH Level 5": 0.0408, - "hfopenllm_v2/GPQA": 0.3096, - "hfopenllm_v2/MUSR": 0.4874, - "hfopenllm_v2/MMLU-PRO": 0.2326 - } - }, - { - "id": "icefog72/Ice0.55-17.01-RP", - "name": "Ice0.55-17.01-RP", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4961, - "hfopenllm_v2/BBH": 0.5077, - "hfopenllm_v2/MATH Level 5": 0.0604, - "hfopenllm_v2/GPQA": 0.2869, - "hfopenllm_v2/MUSR": 0.4725, - "hfopenllm_v2/MMLU-PRO": 0.2658 - } - }, - { - "id": "icefog72/Ice0.57-17.01-RP", - "name": "Ice0.57-17.01-RP", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5152, - "hfopenllm_v2/BBH": 0.5064, - "hfopenllm_v2/MATH Level 5": 0.0514, - "hfopenllm_v2/GPQA": 0.297, - "hfopenllm_v2/MUSR": 0.4686, - "hfopenllm_v2/MMLU-PRO": 0.2651 - } - }, - { - "id": "icefog72/Ice0.60-18.01-RP", - "name": "Ice0.60-18.01-RP", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5374, - "hfopenllm_v2/BBH": 0.5094, - "hfopenllm_v2/MATH Level 5": 0.0536, - "hfopenllm_v2/GPQA": 0.3045, - "hfopenllm_v2/MUSR": 0.467, - "hfopenllm_v2/MMLU-PRO": 0.2837 - } - }, - { - "id": "icefog72/Ice0.60.1-18.01-RP", - "name": "Ice0.60.1-18.01-RP", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5188, - "hfopenllm_v2/BBH": 0.512, - "hfopenllm_v2/MATH Level 5": 0.0461, - "hfopenllm_v2/GPQA": 0.302, - "hfopenllm_v2/MUSR": 0.4498, - "hfopenllm_v2/MMLU-PRO": 0.2914 - } - }, - { - "id": "icefog72/Ice0.61-18.01-RP", - "name": "Ice0.61-18.01-RP", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5441, - "hfopenllm_v2/BBH": 0.5105, - "hfopenllm_v2/MATH Level 5": 0.0468, - "hfopenllm_v2/GPQA": 0.2987, - "hfopenllm_v2/MUSR": 0.4697, - "hfopenllm_v2/MMLU-PRO": 0.2709 - } - }, - { - "id": "icefog72/Ice0.62-18.01-RP", - "name": "Ice0.62-18.01-RP", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5367, - "hfopenllm_v2/BBH": 0.5103, - "hfopenllm_v2/MATH Level 5": 0.0574, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.4538, - "hfopenllm_v2/MMLU-PRO": 0.2877 - } - }, - { - "id": "icefog72/Ice0.62.1-24.01-RP", - "name": "Ice0.62.1-24.01-RP", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5182, - "hfopenllm_v2/BBH": 0.5109, - "hfopenllm_v2/MATH Level 5": 0.0559, - "hfopenllm_v2/GPQA": 0.3003, - "hfopenllm_v2/MUSR": 0.4551, - "hfopenllm_v2/MMLU-PRO": 0.2871 - } - }, - { - "id": "icefog72/Ice0.64-24.01-RP", - "name": "Ice0.64-24.01-RP", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5441, - "hfopenllm_v2/BBH": 0.506, - "hfopenllm_v2/MATH Level 5": 0.0627, - "hfopenllm_v2/GPQA": 0.3037, - "hfopenllm_v2/MUSR": 0.462, - "hfopenllm_v2/MMLU-PRO": 0.2933 - } - }, - { - "id": "icefog72/Ice0.64.1-24.01-RP", - "name": "Ice0.64.1-24.01-RP", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5447, - "hfopenllm_v2/BBH": 0.506, - "hfopenllm_v2/MATH Level 5": 0.0627, - "hfopenllm_v2/GPQA": 0.3037, - "hfopenllm_v2/MUSR": 0.462, - "hfopenllm_v2/MMLU-PRO": 0.2933 - } - }, - { - "id": "icefog72/Ice0.65-25.01-RP", - "name": "Ice0.65-25.01-RP", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5029, - "hfopenllm_v2/BBH": 0.5096, - "hfopenllm_v2/MATH Level 5": 0.065, - "hfopenllm_v2/GPQA": 0.3045, - "hfopenllm_v2/MUSR": 0.434, - "hfopenllm_v2/MMLU-PRO": 0.2997 - } - }, - { - "id": "icefog72/Ice0.66-25.01-RP", - "name": "Ice0.66-25.01-RP", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5325, - "hfopenllm_v2/BBH": 0.5129, - "hfopenllm_v2/MATH Level 5": 0.0604, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.4434, - "hfopenllm_v2/MMLU-PRO": 0.3039 - } - }, - { - "id": "icefog72/Ice0.67-25.01-RP", - "name": "Ice0.67-25.01-RP", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5361, - "hfopenllm_v2/BBH": 0.5113, - "hfopenllm_v2/MATH Level 5": 0.0748, - "hfopenllm_v2/GPQA": 0.2852, - "hfopenllm_v2/MUSR": 0.4279, - "hfopenllm_v2/MMLU-PRO": 0.3097 - } - }, - { - "id": "icefog72/Ice0.68-25.01-RP", - "name": "Ice0.68-25.01-RP", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5514, - "hfopenllm_v2/BBH": 0.513, - "hfopenllm_v2/MATH Level 5": 0.0725, - "hfopenllm_v2/GPQA": 0.3104, - "hfopenllm_v2/MUSR": 0.4446, - "hfopenllm_v2/MMLU-PRO": 0.3012 - } - }, - { - "id": "icefog72/Ice0.69-25.01-RP", - "name": "Ice0.69-25.01-RP", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5438, - "hfopenllm_v2/BBH": 0.5098, - "hfopenllm_v2/MATH Level 5": 0.0566, - "hfopenllm_v2/GPQA": 0.3129, - "hfopenllm_v2/MUSR": 0.4486, - "hfopenllm_v2/MMLU-PRO": 0.2965 - } - }, - { - "id": "icefog72/Ice0.7-29.09-RP", - "name": "Ice0.7-29.09-RP", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5176, - "hfopenllm_v2/BBH": 0.5048, - "hfopenllm_v2/MATH Level 5": 0.0665, - "hfopenllm_v2/GPQA": 0.2878, - "hfopenllm_v2/MUSR": 0.4238, - "hfopenllm_v2/MMLU-PRO": 0.3127 - } - }, - { - "id": "icefog72/Ice0.70-25.01-RP", - "name": "Ice0.70-25.01-RP", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5498, - "hfopenllm_v2/BBH": 0.5136, - "hfopenllm_v2/MATH Level 5": 0.0597, - "hfopenllm_v2/GPQA": 0.3079, - "hfopenllm_v2/MUSR": 0.4512, - "hfopenllm_v2/MMLU-PRO": 0.2996 - } - }, - { - "id": "icefog72/Ice0.70.1-01.02-RP", - "name": "Ice0.70.1-01.02-RP", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.507, - "hfopenllm_v2/BBH": 0.506, - "hfopenllm_v2/MATH Level 5": 0.034, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.4599, - "hfopenllm_v2/MMLU-PRO": 0.2749 - } - }, - { - "id": "icefog72/Ice0.73-01.02-RP", - "name": "Ice0.73-01.02-RP", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5292, - "hfopenllm_v2/BBH": 0.5103, - "hfopenllm_v2/MATH Level 5": 0.0385, - "hfopenllm_v2/GPQA": 0.2911, - "hfopenllm_v2/MUSR": 0.4664, - "hfopenllm_v2/MMLU-PRO": 0.2702 - } - }, - { - "id": "icefog72/Ice0.74-02.02-RP", - "name": "Ice0.74-02.02-RP", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2935, - "hfopenllm_v2/BBH": 0.4646, - "hfopenllm_v2/MATH Level 5": 0.0015, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.428, - "hfopenllm_v2/MMLU-PRO": 0.2143 - } - }, - { - "id": "icefog72/Ice0.76-02.02-RP", - "name": "Ice0.76-02.02-RP", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4529, - "hfopenllm_v2/BBH": 0.5086, - "hfopenllm_v2/MATH Level 5": 0.0144, - "hfopenllm_v2/GPQA": 0.2869, - "hfopenllm_v2/MUSR": 0.4362, - "hfopenllm_v2/MMLU-PRO": 0.2652 - } - }, - { - "id": "icefog72/Ice0.77-02.02-RP", - "name": "Ice0.77-02.02-RP", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.531, - "hfopenllm_v2/BBH": 0.5109, - "hfopenllm_v2/MATH Level 5": 0.0393, - "hfopenllm_v2/GPQA": 0.2852, - "hfopenllm_v2/MUSR": 0.4765, - "hfopenllm_v2/MMLU-PRO": 0.2999 - } - }, - { - "id": "icefog72/Ice0.78-02.02-RP", - "name": "Ice0.78-02.02-RP", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4053, - "hfopenllm_v2/BBH": 0.5002, - "hfopenllm_v2/MATH Level 5": 0.0438, - "hfopenllm_v2/GPQA": 0.2936, - "hfopenllm_v2/MUSR": 0.4686, - "hfopenllm_v2/MMLU-PRO": 0.2955 - } - }, - { - "id": "icefog72/Ice0.80-03.02-RP", - "name": "Ice0.80-03.02-RP", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5516, - "hfopenllm_v2/BBH": 0.5098, - "hfopenllm_v2/MATH Level 5": 0.0559, - "hfopenllm_v2/GPQA": 0.2785, - "hfopenllm_v2/MUSR": 0.4923, - "hfopenllm_v2/MMLU-PRO": 0.2912 - } - }, - { - "id": "icefog72/IceCocoaRP-7b", - "name": "IceCocoaRP-7b", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4962, - "hfopenllm_v2/BBH": 0.4938, - "hfopenllm_v2/MATH Level 5": 0.0574, - "hfopenllm_v2/GPQA": 0.2953, - "hfopenllm_v2/MUSR": 0.4198, - "hfopenllm_v2/MMLU-PRO": 0.3098 - } - }, - { - "id": "icefog72/IceCoffeeRP-7b", - "name": "IceCoffeeRP-7b", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4959, - "hfopenllm_v2/BBH": 0.4889, - "hfopenllm_v2/MATH Level 5": 0.0544, - "hfopenllm_v2/GPQA": 0.2852, - "hfopenllm_v2/MUSR": 0.416, - "hfopenllm_v2/MMLU-PRO": 0.2975 - } - }, - { - "id": "icefog72/IceDrinkByFrankensteinV3RP", - "name": "IceDrinkByFrankensteinV3RP", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4975, - "hfopenllm_v2/BBH": 0.4833, - "hfopenllm_v2/MATH Level 5": 0.0506, - "hfopenllm_v2/GPQA": 0.2617, - "hfopenllm_v2/MUSR": 0.4253, - "hfopenllm_v2/MMLU-PRO": 0.2927 - } - }, - { - "id": "icefog72/IceDrinkNameGoesHereRP-7b-Model_Stock", - "name": "IceDrinkNameGoesHereRP-7b-Model_Stock", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4968, - "hfopenllm_v2/BBH": 0.4658, - "hfopenllm_v2/MATH Level 5": 0.0408, - "hfopenllm_v2/GPQA": 0.2685, - "hfopenllm_v2/MUSR": 0.4067, - "hfopenllm_v2/MMLU-PRO": 0.2817 - } - }, - { - "id": "icefog72/IceDrinkNameNotFoundRP-7b-Model_Stock", - "name": "IceDrinkNameNotFoundRP-7b-Model_Stock", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.513, - "hfopenllm_v2/BBH": 0.5026, - "hfopenllm_v2/MATH Level 5": 0.0604, - "hfopenllm_v2/GPQA": 0.2777, - "hfopenllm_v2/MUSR": 0.4372, - "hfopenllm_v2/MMLU-PRO": 0.3064 - } - }, - { - "id": "icefog72/IceDrunkCherryRP-7b", - "name": "IceDrunkCherryRP-7b", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4898, - "hfopenllm_v2/BBH": 0.4847, - "hfopenllm_v2/MATH Level 5": 0.0612, - "hfopenllm_v2/GPQA": 0.2768, - "hfopenllm_v2/MUSR": 0.4292, - "hfopenllm_v2/MMLU-PRO": 0.3009 - } - }, - { - "id": "icefog72/IceDrunkenCherryRP-7b", - "name": "IceDrunkenCherryRP-7b", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4763, - "hfopenllm_v2/BBH": 0.5093, - "hfopenllm_v2/MATH Level 5": 0.0642, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.4446, - "hfopenllm_v2/MMLU-PRO": 0.3099 - } - }, - { - "id": "icefog72/IceEspressoRPv2-7b", - "name": "IceEspressoRPv2-7b", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4977, - "hfopenllm_v2/BBH": 0.5055, - "hfopenllm_v2/MATH Level 5": 0.0619, - "hfopenllm_v2/GPQA": 0.2894, - "hfopenllm_v2/MUSR": 0.4331, - "hfopenllm_v2/MMLU-PRO": 0.3061 - } - }, - { - "id": "icefog72/IceLemonTeaRP-32k-7b", - "name": "IceLemonTeaRP-32k-7b", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5212, - "hfopenllm_v2/BBH": 0.4997, - "hfopenllm_v2/MATH Level 5": 0.0544, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.429, - "hfopenllm_v2/MMLU-PRO": 0.3068 - } - }, - { - "id": "icefog72/IceMartiniRP-7b", - "name": "IceMartiniRP-7b", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5045, - "hfopenllm_v2/BBH": 0.4972, - "hfopenllm_v2/MATH Level 5": 0.0665, - "hfopenllm_v2/GPQA": 0.2794, - "hfopenllm_v2/MUSR": 0.4345, - "hfopenllm_v2/MMLU-PRO": 0.3073 - } - }, - { - "id": "icefog72/IceNalyvkaRP-7b", - "name": "IceNalyvkaRP-7b", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5498, - "hfopenllm_v2/BBH": 0.5136, - "hfopenllm_v2/MATH Level 5": 0.0597, - "hfopenllm_v2/GPQA": 0.3079, - "hfopenllm_v2/MUSR": 0.4512, - "hfopenllm_v2/MMLU-PRO": 0.2996 - } - }, - { - "id": "icefog72/IceSakeRP-7b", - "name": "IceSakeRP-7b", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5228, - "hfopenllm_v2/BBH": 0.5119, - "hfopenllm_v2/MATH Level 5": 0.0634, - "hfopenllm_v2/GPQA": 0.2852, - "hfopenllm_v2/MUSR": 0.413, - "hfopenllm_v2/MMLU-PRO": 0.3177 - } - }, - { - "id": "icefog72/IceSakeV4RP-7b", - "name": "IceSakeV4RP-7b", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4634, - "hfopenllm_v2/BBH": 0.493, - "hfopenllm_v2/MATH Level 5": 0.0559, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.4082, - "hfopenllm_v2/MMLU-PRO": 0.3103 - } - }, - { - "id": "icefog72/IceSakeV6RP-7b", - "name": "IceSakeV6RP-7b", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5033, - "hfopenllm_v2/BBH": 0.4976, - "hfopenllm_v2/MATH Level 5": 0.0619, - "hfopenllm_v2/GPQA": 0.2911, - "hfopenllm_v2/MUSR": 0.42, - "hfopenllm_v2/MMLU-PRO": 0.3093 - } - }, - { - "id": "icefog72/IceSakeV8RP-7b", - "name": "IceSakeV8RP-7b", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6086, - "hfopenllm_v2/BBH": 0.4885, - "hfopenllm_v2/MATH Level 5": 0.0597, - "hfopenllm_v2/GPQA": 0.276, - "hfopenllm_v2/MUSR": 0.3993, - "hfopenllm_v2/MMLU-PRO": 0.301 - } - }, - { - "id": "icefog72/IceTea21EnergyDrinkRPV13-DPOv3", - "name": "IceTea21EnergyDrinkRPV13-DPOv3", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5263, - "hfopenllm_v2/BBH": 0.502, - "hfopenllm_v2/MATH Level 5": 0.0582, - "hfopenllm_v2/GPQA": 0.2836, - "hfopenllm_v2/MUSR": 0.4372, - "hfopenllm_v2/MMLU-PRO": 0.3056 - } - }, - { - "id": "icefog72/IceTea21EnergyDrinkRPV13-DPOv3.5", - "name": "IceTea21EnergyDrinkRPV13-DPOv3.5", - "developer": "icefog72", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4871, - "hfopenllm_v2/BBH": 0.44, - "hfopenllm_v2/MATH Level 5": 0.0363, - "hfopenllm_v2/GPQA": 0.2844, - "hfopenllm_v2/MUSR": 0.3964, - "hfopenllm_v2/MMLU-PRO": 0.2498 - } - }, - { - "id": "IDEA-CCNL/Ziya-LLaMA-13B-v1", - "name": "Ziya-LLaMA-13B-v1", - "developer": "IDEA-CCNL", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1697, - "hfopenllm_v2/BBH": 0.2877, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2492, - "hfopenllm_v2/MUSR": 0.3751, - "hfopenllm_v2/MMLU-PRO": 0.1101 - } - }, - { - "id": "IDEA-CCNL/Ziya-LLaMA-7B-Reward", - "name": "IDEA-CCNL/Ziya-LLaMA-7B-Reward", - "developer": "IDEA-CCNL", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6378, - "reward-bench/Chat": 0.8687, - "reward-bench/Chat Hard": 0.4605, - "reward-bench/Safety": 0.6405, - "reward-bench/Reasoning": 0.5775, - "reward-bench/Prior Sets (0.5 weight)": 0.6461 - } - }, - { - "id": "ifable/gemma-2-Ifable-9B", - "name": "gemma-2-Ifable-9B", - "developer": "ifable", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2984, - "hfopenllm_v2/BBH": 0.5866, - "hfopenllm_v2/MATH Level 5": 0.1397, - "hfopenllm_v2/GPQA": 0.3414, - "hfopenllm_v2/MUSR": 0.4053, - "hfopenllm_v2/MMLU-PRO": 0.4226 - } - }, - { - "id": "iFaz/llama31_8B_en_emo_v4", - "name": "llama31_8B_en_emo_v4", - "developer": "iFaz", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3043, - "hfopenllm_v2/BBH": 0.4916, - "hfopenllm_v2/MATH Level 5": 0.0884, - "hfopenllm_v2/GPQA": 0.297, - "hfopenllm_v2/MUSR": 0.3643, - "hfopenllm_v2/MMLU-PRO": 0.3049 - } - }, - { - "id": "iFaz/llama32_1B_en_emo_v1", - "name": "llama32_1B_en_emo_v1", - "developer": "iFaz", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4408, - "hfopenllm_v2/BBH": 0.338, - "hfopenllm_v2/MATH Level 5": 0.0378, - "hfopenllm_v2/GPQA": 0.25, - "hfopenllm_v2/MUSR": 0.3489, - "hfopenllm_v2/MMLU-PRO": 0.1761 - } - }, - { - "id": "iFaz/llama32_3B_en_emo_1000_stp", - "name": "llama32_3B_en_emo_1000_stp", - "developer": "iFaz", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7295, - "hfopenllm_v2/BBH": 0.4522, - "hfopenllm_v2/MATH Level 5": 0.1465, - "hfopenllm_v2/GPQA": 0.2777, - "hfopenllm_v2/MUSR": 0.3621, - "hfopenllm_v2/MMLU-PRO": 0.3123 - } - }, - { - "id": "iFaz/llama32_3B_en_emo_2000_stp", - "name": "llama32_3B_en_emo_2000_stp", - "developer": "iFaz", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7369, - "hfopenllm_v2/BBH": 0.4535, - "hfopenllm_v2/MATH Level 5": 0.1533, - "hfopenllm_v2/GPQA": 0.2836, - "hfopenllm_v2/MUSR": 0.3527, - "hfopenllm_v2/MMLU-PRO": 0.3098 - } - }, - { - "id": "iFaz/llama32_3B_en_emo_300_stp", - "name": "llama32_3B_en_emo_300_stp", - "developer": "iFaz", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7256, - "hfopenllm_v2/BBH": 0.4505, - "hfopenllm_v2/MATH Level 5": 0.1601, - "hfopenllm_v2/GPQA": 0.2743, - "hfopenllm_v2/MUSR": 0.3621, - "hfopenllm_v2/MMLU-PRO": 0.3148 - } - }, - { - "id": "iFaz/llama32_3B_en_emo_5000_stp", - "name": "llama32_3B_en_emo_5000_stp", - "developer": "iFaz", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.71, - "hfopenllm_v2/BBH": 0.4568, - "hfopenllm_v2/MATH Level 5": 0.1292, - "hfopenllm_v2/GPQA": 0.2794, - "hfopenllm_v2/MUSR": 0.3446, - "hfopenllm_v2/MMLU-PRO": 0.3067 - } - }, - { - "id": "iFaz/llama32_3B_en_emo_v2", - "name": "llama32_3B_en_emo_v2", - "developer": "iFaz", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5454, - "hfopenllm_v2/BBH": 0.4284, - "hfopenllm_v2/MATH Level 5": 0.1088, - "hfopenllm_v2/GPQA": 0.2676, - "hfopenllm_v2/MUSR": 0.3482, - "hfopenllm_v2/MMLU-PRO": 0.3004 - } - }, - { - "id": "iFaz/llama32_3B_en_emo_v3", - "name": "llama32_3B_en_emo_v3", - "developer": "iFaz", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5759, - "hfopenllm_v2/BBH": 0.4301, - "hfopenllm_v2/MATH Level 5": 0.068, - "hfopenllm_v2/GPQA": 0.2676, - "hfopenllm_v2/MUSR": 0.3553, - "hfopenllm_v2/MMLU-PRO": 0.271 - } - }, - { - "id": "ilsp/Llama-Krikri-8B-Instruct", - "name": "Llama-Krikri-8B-Instruct", - "developer": "ilsp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6079, - "hfopenllm_v2/BBH": 0.5047, - "hfopenllm_v2/MATH Level 5": 0.1178, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.408, - "hfopenllm_v2/MMLU-PRO": 0.3313 - } - }, - { - "id": "IlyaGusev/gemma-2-2b-it-abliterated", - "name": "gemma-2-2b-it-abliterated", - "developer": "IlyaGusev", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5331, - "hfopenllm_v2/BBH": 0.4119, - "hfopenllm_v2/MATH Level 5": 0.0612, - "hfopenllm_v2/GPQA": 0.2651, - "hfopenllm_v2/MUSR": 0.3782, - "hfopenllm_v2/MMLU-PRO": 0.2538 - } - }, - { - "id": "IlyaGusev/gemma-2-9b-it-abliterated", - "name": "gemma-2-9b-it-abliterated", - "developer": "IlyaGusev", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7473, - "hfopenllm_v2/BBH": 0.5906, - "hfopenllm_v2/MATH Level 5": 0.1775, - "hfopenllm_v2/GPQA": 0.3456, - "hfopenllm_v2/MUSR": 0.4034, - "hfopenllm_v2/MMLU-PRO": 0.3915 - } - }, - { - "id": "Infinirc/Infinirc-Llama3-8B-2G-Release-v1.0", - "name": "Infinirc-Llama3-8B-2G-Release-v1.0", - "developer": "Infinirc", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2024, - "hfopenllm_v2/BBH": 0.4351, - "hfopenllm_v2/MATH Level 5": 0.0166, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.4609, - "hfopenllm_v2/MMLU-PRO": 0.216 - } - }, - { - "id": "inflatebot/MN-12B-Mag-Mell-R1", - "name": "MN-12B-Mag-Mell-R1", - "developer": "inflatebot", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4613, - "hfopenllm_v2/BBH": 0.5304, - "hfopenllm_v2/MATH Level 5": 0.1299, - "hfopenllm_v2/GPQA": 0.3163, - "hfopenllm_v2/MUSR": 0.4002, - "hfopenllm_v2/MMLU-PRO": 0.3438 - } - }, - { - "id": "infly/INF-ORM-Llama3.1-70B", - "name": "infly/INF-ORM-Llama3.1-70B", - "developer": "infly", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7648, - "reward-bench/Chat": 0.9665, - "reward-bench/Chat Hard": 0.9101, - "reward-bench/Safety": 0.9644, - "reward-bench/Reasoning": 0.9912, - "reward-bench/Factuality": 0.7411, - "reward-bench/Precise IF": 0.4188, - "reward-bench/Math": 0.6995, - "reward-bench/Focus": 0.903, - "reward-bench/Ties": 0.8622 - } - }, - { - "id": "informatiker/Qwen2-7B-Instruct-abliterated", - "name": "Qwen2-7B-Instruct-abliterated", - "developer": "informatiker", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5822, - "hfopenllm_v2/BBH": 0.5534, - "hfopenllm_v2/MATH Level 5": 0.2636, - "hfopenllm_v2/GPQA": 0.3012, - "hfopenllm_v2/MUSR": 0.3888, - "hfopenllm_v2/MMLU-PRO": 0.3873 - } - }, - { - "id": "INSAIT-Institute/BgGPT-Gemma-2-27B-IT-v1.0", - "name": "BgGPT-Gemma-2-27B-IT-v1.0", - "developer": "INSAIT-Institute", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0, - "hfopenllm_v2/BBH": 0.2912, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3575, - "hfopenllm_v2/MMLU-PRO": 0.1167 - } - }, - { - "id": "insightfactory/Llama-3.2-3B-Instruct-unsloth-bnb-4bitlora_model", - "name": "Llama-3.2-3B-Instruct-unsloth-bnb-4bitlora_model", - "developer": "insightfactory", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4588, - "hfopenllm_v2/BBH": 0.4146, - "hfopenllm_v2/MATH Level 5": 0.105, - "hfopenllm_v2/GPQA": 0.2718, - "hfopenllm_v2/MUSR": 0.3499, - "hfopenllm_v2/MMLU-PRO": 0.296 - } - }, - { - "id": "instruction-pretrain/InstructLM-500M", - "name": "InstructLM-500M", - "developer": "instruction-pretrain", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1028, - "hfopenllm_v2/BBH": 0.2941, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2567, - "hfopenllm_v2/MUSR": 0.3528, - "hfopenllm_v2/MMLU-PRO": 0.1141 - } - }, - { - "id": "Intel/neural-chat-7b-v3", - "name": "neural-chat-7b-v3", - "developer": "Intel", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2778, - "hfopenllm_v2/BBH": 0.5048, - "hfopenllm_v2/MATH Level 5": 0.0295, - "hfopenllm_v2/GPQA": 0.2919, - "hfopenllm_v2/MUSR": 0.5055, - "hfopenllm_v2/MMLU-PRO": 0.2699 - } - }, - { - "id": "Intel/neural-chat-7b-v3-1", - "name": "neural-chat-7b-v3-1", - "developer": "Intel", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4687, - "hfopenllm_v2/BBH": 0.5052, - "hfopenllm_v2/MATH Level 5": 0.0355, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.4979, - "hfopenllm_v2/MMLU-PRO": 0.2678 - } - }, - { - "id": "Intel/neural-chat-7b-v3-2", - "name": "neural-chat-7b-v3-2", - "developer": "Intel", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4988, - "hfopenllm_v2/BBH": 0.5032, - "hfopenllm_v2/MATH Level 5": 0.0476, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.4895, - "hfopenllm_v2/MMLU-PRO": 0.2667 - } - }, - { - "id": "Intel/neural-chat-7b-v3-3", - "name": "neural-chat-7b-v3-3", - "developer": "Intel", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4763, - "hfopenllm_v2/BBH": 0.4877, - "hfopenllm_v2/MATH Level 5": 0.0408, - "hfopenllm_v2/GPQA": 0.2894, - "hfopenllm_v2/MUSR": 0.486, - "hfopenllm_v2/MMLU-PRO": 0.2625 - } - }, - { - "id": "internlm/internlm2-1_8b", - "name": "internlm2-1_8b", - "developer": "internlm", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2198, - "hfopenllm_v2/BBH": 0.388, - "hfopenllm_v2/MATH Level 5": 0.0211, - "hfopenllm_v2/GPQA": 0.2483, - "hfopenllm_v2/MUSR": 0.3813, - "hfopenllm_v2/MMLU-PRO": 0.1588 - } - }, - { - "id": "internlm/internlm2-1_8b-reward", - "name": "internlm/internlm2-1_8b-reward", - "developer": "internlm", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.3902, - "reward-bench/Chat": 0.9358, - "reward-bench/Chat Hard": 0.6623, - "reward-bench/Safety": 0.4711, - "reward-bench/Reasoning": 0.8724, - "reward-bench/Factuality": 0.2758, - "reward-bench/Precise IF": 0.3625, - "reward-bench/Math": 0.4426, - "reward-bench/Focus": 0.596, - "reward-bench/Ties": 0.1934 - } - }, - { - "id": "internlm/internlm2-20b-reward", - "name": "internlm/internlm2-20b-reward", - "developer": "internlm", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.9016, - "reward-bench/Factuality": 0.5558, - "reward-bench/Precise IF": 0.3625, - "reward-bench/Math": 0.5738, - "reward-bench/Safety": 0.8946, - "reward-bench/Focus": 0.7253, - "reward-bench/Ties": 0.5483, - "reward-bench/Chat": 0.9888, - "reward-bench/Chat Hard": 0.7654, - "reward-bench/Reasoning": 0.9576 - } - }, - { - "id": "internlm/internlm2-7b", - "name": "internlm2-7b", - "developer": "internlm", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.228, - "hfopenllm_v2/BBH": 0.5825, - "hfopenllm_v2/MATH Level 5": 0.0857, - "hfopenllm_v2/GPQA": 0.3367, - "hfopenllm_v2/MUSR": 0.44, - "hfopenllm_v2/MMLU-PRO": 0.19 - } - }, - { - "id": "internlm/internlm2-7b-reward", - "name": "internlm/internlm2-7b-reward", - "developer": "internlm", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5335, - "reward-bench/Chat": 0.9916, - "reward-bench/Chat Hard": 0.6952, - "reward-bench/Safety": 0.5956, - "reward-bench/Reasoning": 0.9453, - "reward-bench/Factuality": 0.4211, - "reward-bench/Precise IF": 0.4, - "reward-bench/Math": 0.5628, - "reward-bench/Focus": 0.7051, - "reward-bench/Ties": 0.5164 - } - }, - { - "id": "internlm/internlm2-chat-1_8b", - "name": "internlm2-chat-1_8b", - "developer": "internlm", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2387, - "hfopenllm_v2/BBH": 0.4452, - "hfopenllm_v2/MATH Level 5": 0.0325, - "hfopenllm_v2/GPQA": 0.2659, - "hfopenllm_v2/MUSR": 0.3631, - "hfopenllm_v2/MMLU-PRO": 0.1839 - } - }, - { - "id": "internlm/internlm2_5-1_8b-chat", - "name": "internlm2_5-1_8b-chat", - "developer": "internlm", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3849, - "hfopenllm_v2/BBH": 0.4489, - "hfopenllm_v2/MATH Level 5": 0.1586, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.3594, - "hfopenllm_v2/MMLU-PRO": 0.1299 - } - }, - { - "id": "internlm/internlm2_5-20b-chat", - "name": "internlm2_5-20b-chat", - "developer": "internlm", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.701, - "hfopenllm_v2/BBH": 0.7474, - "hfopenllm_v2/MATH Level 5": 0.4079, - "hfopenllm_v2/GPQA": 0.3213, - "hfopenllm_v2/MUSR": 0.4558, - "hfopenllm_v2/MMLU-PRO": 0.3998 - } - }, - { - "id": "internlm/internlm2_5-7b-chat", - "name": "internlm2_5-7b-chat", - "developer": "internlm", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5539, - "hfopenllm_v2/BBH": 0.7073, - "hfopenllm_v2/MATH Level 5": 0.253, - "hfopenllm_v2/GPQA": 0.3473, - "hfopenllm_v2/MUSR": 0.4594, - "hfopenllm_v2/MMLU-PRO": 0.3777 - } - }, - { - "id": "intervitens/mini-magnum-12b-v1.1", - "name": "mini-magnum-12b-v1.1", - "developer": "intervitens", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5156, - "hfopenllm_v2/BBH": 0.5062, - "hfopenllm_v2/MATH Level 5": 0.0619, - "hfopenllm_v2/GPQA": 0.2886, - "hfopenllm_v2/MUSR": 0.4004, - "hfopenllm_v2/MMLU-PRO": 0.3291 - } - }, - { - "id": "IntervitensInc/internlm2_5-20b-llamafied", - "name": "internlm2_5-20b-llamafied", - "developer": "IntervitensInc", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.341, - "hfopenllm_v2/BBH": 0.7478, - "hfopenllm_v2/MATH Level 5": 0.1715, - "hfopenllm_v2/GPQA": 0.3381, - "hfopenllm_v2/MUSR": 0.4475, - "hfopenllm_v2/MMLU-PRO": 0.4051 - } - }, - { - "id": "inumulaisk/eval_model", - "name": "eval_model", - "developer": "inumulaisk", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1931, - "hfopenllm_v2/BBH": 0.3512, - "hfopenllm_v2/MATH Level 5": 0.2976, - "hfopenllm_v2/GPQA": 0.2794, - "hfopenllm_v2/MUSR": 0.358, - "hfopenllm_v2/MMLU-PRO": 0.1664 - } - }, - { - "id": "invalid-coder/Sakura-SOLAR-Instruct-CarbonVillain-en-10.7B-v2-slerp", - "name": "Sakura-SOLAR-Instruct-CarbonVillain-en-10.7B-v2-slerp", - "developer": "invalid-coder", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4555, - "hfopenllm_v2/BBH": 0.5158, - "hfopenllm_v2/MATH Level 5": 0.0491, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.3992, - "hfopenllm_v2/MMLU-PRO": 0.3146 - } - }, - { - "id": "Invalid-Null/PeiYangMe-0.5", - "name": "PeiYangMe-0.5", - "developer": "Invalid-Null", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1409, - "hfopenllm_v2/BBH": 0.2791, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2441, - "hfopenllm_v2/MUSR": 0.3738, - "hfopenllm_v2/MMLU-PRO": 0.1109 - } - }, - { - "id": "Invalid-Null/PeiYangMe-0.7", - "name": "PeiYangMe-0.7", - "developer": "Invalid-Null", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1491, - "hfopenllm_v2/BBH": 0.3028, - "hfopenllm_v2/MATH Level 5": 0.0113, - "hfopenllm_v2/GPQA": 0.2332, - "hfopenllm_v2/MUSR": 0.3857, - "hfopenllm_v2/MMLU-PRO": 0.1101 - } - }, - { - "id": "invisietch/EtherealRainbow-v0.2-8B", - "name": "EtherealRainbow-v0.2-8B", - "developer": "invisietch", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3903, - "hfopenllm_v2/BBH": 0.5102, - "hfopenllm_v2/MATH Level 5": 0.0823, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.3827, - "hfopenllm_v2/MMLU-PRO": 0.3653 - } - }, - { - "id": "invisietch/EtherealRainbow-v0.3-8B", - "name": "EtherealRainbow-v0.3-8B", - "developer": "invisietch", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3682, - "hfopenllm_v2/BBH": 0.5097, - "hfopenllm_v2/MATH Level 5": 0.0763, - "hfopenllm_v2/GPQA": 0.3045, - "hfopenllm_v2/MUSR": 0.3904, - "hfopenllm_v2/MMLU-PRO": 0.3626 - } - }, - { - "id": "invisietch/MiS-Firefly-v0.2-22B", - "name": "MiS-Firefly-v0.2-22B", - "developer": "invisietch", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5371, - "hfopenllm_v2/BBH": 0.5514, - "hfopenllm_v2/MATH Level 5": 0.1654, - "hfopenllm_v2/GPQA": 0.3045, - "hfopenllm_v2/MUSR": 0.4694, - "hfopenllm_v2/MMLU-PRO": 0.362 - } - }, - { - "id": "invisietch/Nimbus-Miqu-v0.1-70B", - "name": "Nimbus-Miqu-v0.1-70B", - "developer": "invisietch", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4647, - "hfopenllm_v2/BBH": 0.601, - "hfopenllm_v2/MATH Level 5": 0.0604, - "hfopenllm_v2/GPQA": 0.3389, - "hfopenllm_v2/MUSR": 0.4133, - "hfopenllm_v2/MMLU-PRO": 0.3853 - } - }, - { - "id": "irahulpandey/mistralai-7B-slerp-v0.1", - "name": "mistralai-7B-slerp-v0.1", - "developer": "irahulpandey", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4966, - "hfopenllm_v2/BBH": 0.5011, - "hfopenllm_v2/MATH Level 5": 0.0514, - "hfopenllm_v2/GPQA": 0.3037, - "hfopenllm_v2/MUSR": 0.455, - "hfopenllm_v2/MMLU-PRO": 0.2951 - } - }, - { - "id": "iRyanBell/ARC1", - "name": "ARC1", - "developer": "iRyanBell", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4411, - "hfopenllm_v2/BBH": 0.4903, - "hfopenllm_v2/MATH Level 5": 0.0687, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.3991, - "hfopenllm_v2/MMLU-PRO": 0.3371 - } - }, - { - "id": "iRyanBell/ARC1-II", - "name": "ARC1-II", - "developer": "iRyanBell", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1708, - "hfopenllm_v2/BBH": 0.3382, - "hfopenllm_v2/MATH Level 5": 0.0219, - "hfopenllm_v2/GPQA": 0.2718, - "hfopenllm_v2/MUSR": 0.4913, - "hfopenllm_v2/MMLU-PRO": 0.1686 - } - }, - { - "id": "Isaak-Carter/Josiefied-Qwen2.5-7B-Instruct-abliterated", - "name": "Josiefied-Qwen2.5-7B-Instruct-abliterated", - "developer": "Isaak-Carter", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7317, - "hfopenllm_v2/BBH": 0.5396, - "hfopenllm_v2/MATH Level 5": 0.4924, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.4087, - "hfopenllm_v2/MMLU-PRO": 0.4276 - } - }, - { - "id": "Isaak-Carter/Josiefied-Qwen2.5-7B-Instruct-abliterated-v2", - "name": "Josiefied-Qwen2.5-7B-Instruct-abliterated-v2", - "developer": "Isaak-Carter", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7841, - "hfopenllm_v2/BBH": 0.5311, - "hfopenllm_v2/MATH Level 5": 0.4721, - "hfopenllm_v2/GPQA": 0.2987, - "hfopenllm_v2/MUSR": 0.4354, - "hfopenllm_v2/MMLU-PRO": 0.4128 - } - }, - { - "id": "Isaak-Carter/JOSIEv4o-8b-stage1-v4", - "name": "JOSIEv4o-8b-stage1-v4", - "developer": "Isaak-Carter", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2477, - "hfopenllm_v2/BBH": 0.4758, - "hfopenllm_v2/MATH Level 5": 0.0453, - "hfopenllm_v2/GPQA": 0.2911, - "hfopenllm_v2/MUSR": 0.3641, - "hfopenllm_v2/MMLU-PRO": 0.3292 - } - }, - { - "id": "J-LAB/Thynk_orpo", - "name": "Thynk_orpo", - "developer": "J-LAB", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2102, - "hfopenllm_v2/BBH": 0.4463, - "hfopenllm_v2/MATH Level 5": 0.148, - "hfopenllm_v2/GPQA": 0.2928, - "hfopenllm_v2/MUSR": 0.4515, - "hfopenllm_v2/MMLU-PRO": 0.3231 - } - }, - { - "id": "JackFram/llama-160m", - "name": "llama-160m", - "developer": "JackFram", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1791, - "hfopenllm_v2/BBH": 0.2888, - "hfopenllm_v2/MATH Level 5": 0.0083, - "hfopenllm_v2/GPQA": 0.2617, - "hfopenllm_v2/MUSR": 0.3792, - "hfopenllm_v2/MMLU-PRO": 0.1128 - } - }, - { - "id": "JackFram/llama-68m", - "name": "llama-68m", - "developer": "JackFram", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1726, - "hfopenllm_v2/BBH": 0.2936, - "hfopenllm_v2/MATH Level 5": 0.006, - "hfopenllm_v2/GPQA": 0.2584, - "hfopenllm_v2/MUSR": 0.391, - "hfopenllm_v2/MMLU-PRO": 0.1144 - } - }, - { - "id": "Jacoby746/Casual-Magnum-34B", - "name": "Casual-Magnum-34B", - "developer": "Jacoby746", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.193, - "hfopenllm_v2/BBH": 0.6032, - "hfopenllm_v2/MATH Level 5": 0.0921, - "hfopenllm_v2/GPQA": 0.3725, - "hfopenllm_v2/MUSR": 0.4078, - "hfopenllm_v2/MMLU-PRO": 0.5184 - } - }, - { - "id": "Jacoby746/Inf-Silent-Kunoichi-v0.1-2x7B", - "name": "Inf-Silent-Kunoichi-v0.1-2x7B", - "developer": "Jacoby746", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.388, - "hfopenllm_v2/BBH": 0.5185, - "hfopenllm_v2/MATH Level 5": 0.071, - "hfopenllm_v2/GPQA": 0.2894, - "hfopenllm_v2/MUSR": 0.428, - "hfopenllm_v2/MMLU-PRO": 0.3271 - } - }, - { - "id": "Jacoby746/Inf-Silent-Kunoichi-v0.2-2x7B", - "name": "Inf-Silent-Kunoichi-v0.2-2x7B", - "developer": "Jacoby746", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3636, - "hfopenllm_v2/BBH": 0.5209, - "hfopenllm_v2/MATH Level 5": 0.0627, - "hfopenllm_v2/GPQA": 0.3003, - "hfopenllm_v2/MUSR": 0.432, - "hfopenllm_v2/MMLU-PRO": 0.3272 - } - }, - { - "id": "Jacoby746/Proto-Athena-4x7B", - "name": "Proto-Athena-4x7B", - "developer": "Jacoby746", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3703, - "hfopenllm_v2/BBH": 0.5107, - "hfopenllm_v2/MATH Level 5": 0.065, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.4348, - "hfopenllm_v2/MMLU-PRO": 0.3206 - } - }, - { - "id": "Jacoby746/Proto-Athena-v0.2-4x7B", - "name": "Proto-Athena-v0.2-4x7B", - "developer": "Jacoby746", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3752, - "hfopenllm_v2/BBH": 0.5068, - "hfopenllm_v2/MATH Level 5": 0.0634, - "hfopenllm_v2/GPQA": 0.2987, - "hfopenllm_v2/MUSR": 0.4213, - "hfopenllm_v2/MMLU-PRO": 0.3197 - } - }, - { - "id": "Jacoby746/Proto-Harpy-Blazing-Light-v0.1-2x7B", - "name": "Proto-Harpy-Blazing-Light-v0.1-2x7B", - "developer": "Jacoby746", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4905, - "hfopenllm_v2/BBH": 0.5187, - "hfopenllm_v2/MATH Level 5": 0.0748, - "hfopenllm_v2/GPQA": 0.2953, - "hfopenllm_v2/MUSR": 0.445, - "hfopenllm_v2/MMLU-PRO": 0.3301 - } - }, - { - "id": "Jacoby746/Proto-Harpy-Spark-v0.1-7B", - "name": "Proto-Harpy-Spark-v0.1-7B", - "developer": "Jacoby746", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4333, - "hfopenllm_v2/BBH": 0.4736, - "hfopenllm_v2/MATH Level 5": 0.0619, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.4317, - "hfopenllm_v2/MMLU-PRO": 0.3069 - } - }, - { - "id": "jaredjoss/pythia-410m-roberta-lr_8e7-kl_01-steps_12000-rlhf-model", - "name": "pythia-410m-roberta-lr_8e7-kl_01-steps_12000-rlhf-model", - "developer": "jaredjoss", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1572, - "hfopenllm_v2/BBH": 0.2863, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3607, - "hfopenllm_v2/MMLU-PRO": 0.1169 - } - }, - { - "id": "jaspionjader/Auro-Kosmos-EVAA-v2-8B", - "name": "Auro-Kosmos-EVAA-v2-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4778, - "hfopenllm_v2/BBH": 0.5447, - "hfopenllm_v2/MATH Level 5": 0.1412, - "hfopenllm_v2/GPQA": 0.3154, - "hfopenllm_v2/MUSR": 0.425, - "hfopenllm_v2/MMLU-PRO": 0.3858 - } - }, - { - "id": "jaspionjader/Auro-Kosmos-EVAA-v2.1-8B", - "name": "Auro-Kosmos-EVAA-v2.1-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4666, - "hfopenllm_v2/BBH": 0.5444, - "hfopenllm_v2/MATH Level 5": 0.1458, - "hfopenllm_v2/GPQA": 0.3087, - "hfopenllm_v2/MUSR": 0.4317, - "hfopenllm_v2/MMLU-PRO": 0.3826 - } - }, - { - "id": "jaspionjader/Auro-Kosmos-EVAA-v2.2-8B", - "name": "Auro-Kosmos-EVAA-v2.2-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4268, - "hfopenllm_v2/BBH": 0.5431, - "hfopenllm_v2/MATH Level 5": 0.1412, - "hfopenllm_v2/GPQA": 0.3104, - "hfopenllm_v2/MUSR": 0.4251, - "hfopenllm_v2/MMLU-PRO": 0.3798 - } - }, - { - "id": "jaspionjader/Auro-Kosmos-EVAA-v2.3-8B", - "name": "Auro-Kosmos-EVAA-v2.3-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4271, - "hfopenllm_v2/BBH": 0.5441, - "hfopenllm_v2/MATH Level 5": 0.1344, - "hfopenllm_v2/GPQA": 0.3121, - "hfopenllm_v2/MUSR": 0.4278, - "hfopenllm_v2/MMLU-PRO": 0.3784 - } - }, - { - "id": "jaspionjader/bbb-1", - "name": "bbb-1", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4864, - "hfopenllm_v2/BBH": 0.5376, - "hfopenllm_v2/MATH Level 5": 0.1367, - "hfopenllm_v2/GPQA": 0.3138, - "hfopenllm_v2/MUSR": 0.4171, - "hfopenllm_v2/MMLU-PRO": 0.3897 - } - }, - { - "id": "jaspionjader/bbb-2", - "name": "bbb-2", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4077, - "hfopenllm_v2/BBH": 0.5067, - "hfopenllm_v2/MATH Level 5": 0.1125, - "hfopenllm_v2/GPQA": 0.302, - "hfopenllm_v2/MUSR": 0.4145, - "hfopenllm_v2/MMLU-PRO": 0.3635 - } - }, - { - "id": "jaspionjader/bbb-3", - "name": "bbb-3", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4168, - "hfopenllm_v2/BBH": 0.5158, - "hfopenllm_v2/MATH Level 5": 0.1405, - "hfopenllm_v2/GPQA": 0.3112, - "hfopenllm_v2/MUSR": 0.4265, - "hfopenllm_v2/MMLU-PRO": 0.3856 - } - }, - { - "id": "jaspionjader/bbb-4", - "name": "bbb-4", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4768, - "hfopenllm_v2/BBH": 0.5212, - "hfopenllm_v2/MATH Level 5": 0.1276, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.4092, - "hfopenllm_v2/MMLU-PRO": 0.3773 - } - }, - { - "id": "jaspionjader/bbb-5", - "name": "bbb-5", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4703, - "hfopenllm_v2/BBH": 0.5207, - "hfopenllm_v2/MATH Level 5": 0.1397, - "hfopenllm_v2/GPQA": 0.3045, - "hfopenllm_v2/MUSR": 0.3998, - "hfopenllm_v2/MMLU-PRO": 0.3834 - } - }, - { - "id": "jaspionjader/bbb-6", - "name": "bbb-6", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.488, - "hfopenllm_v2/BBH": 0.5211, - "hfopenllm_v2/MATH Level 5": 0.139, - "hfopenllm_v2/GPQA": 0.3104, - "hfopenllm_v2/MUSR": 0.4052, - "hfopenllm_v2/MMLU-PRO": 0.3871 - } - }, - { - "id": "jaspionjader/bbb-7", - "name": "bbb-7", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4828, - "hfopenllm_v2/BBH": 0.5211, - "hfopenllm_v2/MATH Level 5": 0.1367, - "hfopenllm_v2/GPQA": 0.3104, - "hfopenllm_v2/MUSR": 0.4038, - "hfopenllm_v2/MMLU-PRO": 0.386 - } - }, - { - "id": "jaspionjader/bh-1", - "name": "bh-1", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4284, - "hfopenllm_v2/BBH": 0.589, - "hfopenllm_v2/MATH Level 5": 0.0536, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.4441, - "hfopenllm_v2/MMLU-PRO": 0.3449 - } - }, - { - "id": "jaspionjader/bh-10", - "name": "bh-10", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4618, - "hfopenllm_v2/BBH": 0.5856, - "hfopenllm_v2/MATH Level 5": 0.1103, - "hfopenllm_v2/GPQA": 0.3003, - "hfopenllm_v2/MUSR": 0.4199, - "hfopenllm_v2/MMLU-PRO": 0.3708 - } - }, - { - "id": "jaspionjader/bh-11", - "name": "bh-11", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4575, - "hfopenllm_v2/BBH": 0.5851, - "hfopenllm_v2/MATH Level 5": 0.1178, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.4146, - "hfopenllm_v2/MMLU-PRO": 0.3738 - } - }, - { - "id": "jaspionjader/bh-12", - "name": "bh-12", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4734, - "hfopenllm_v2/BBH": 0.5802, - "hfopenllm_v2/MATH Level 5": 0.1186, - "hfopenllm_v2/GPQA": 0.3003, - "hfopenllm_v2/MUSR": 0.4145, - "hfopenllm_v2/MMLU-PRO": 0.3737 - } - }, - { - "id": "jaspionjader/bh-13", - "name": "bh-13", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4698, - "hfopenllm_v2/BBH": 0.5778, - "hfopenllm_v2/MATH Level 5": 0.1125, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.4159, - "hfopenllm_v2/MMLU-PRO": 0.373 - } - }, - { - "id": "jaspionjader/bh-15", - "name": "bh-15", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4745, - "hfopenllm_v2/BBH": 0.5819, - "hfopenllm_v2/MATH Level 5": 0.1246, - "hfopenllm_v2/GPQA": 0.2987, - "hfopenllm_v2/MUSR": 0.4105, - "hfopenllm_v2/MMLU-PRO": 0.3767 - } - }, - { - "id": "jaspionjader/bh-16", - "name": "bh-16", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4731, - "hfopenllm_v2/BBH": 0.5783, - "hfopenllm_v2/MATH Level 5": 0.1193, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.4159, - "hfopenllm_v2/MMLU-PRO": 0.3776 - } - }, - { - "id": "jaspionjader/bh-17", - "name": "bh-17", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4722, - "hfopenllm_v2/BBH": 0.5776, - "hfopenllm_v2/MATH Level 5": 0.1133, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.4158, - "hfopenllm_v2/MMLU-PRO": 0.3757 - } - }, - { - "id": "jaspionjader/bh-18", - "name": "bh-18", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4725, - "hfopenllm_v2/BBH": 0.5824, - "hfopenllm_v2/MATH Level 5": 0.1186, - "hfopenllm_v2/GPQA": 0.3003, - "hfopenllm_v2/MUSR": 0.4185, - "hfopenllm_v2/MMLU-PRO": 0.3757 - } - }, - { - "id": "jaspionjader/bh-19", - "name": "bh-19", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4584, - "hfopenllm_v2/BBH": 0.5766, - "hfopenllm_v2/MATH Level 5": 0.1193, - "hfopenllm_v2/GPQA": 0.297, - "hfopenllm_v2/MUSR": 0.4171, - "hfopenllm_v2/MMLU-PRO": 0.3775 - } - }, - { - "id": "jaspionjader/bh-2", - "name": "bh-2", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4579, - "hfopenllm_v2/BBH": 0.5937, - "hfopenllm_v2/MATH Level 5": 0.1027, - "hfopenllm_v2/GPQA": 0.3012, - "hfopenllm_v2/MUSR": 0.4186, - "hfopenllm_v2/MMLU-PRO": 0.3695 - } - }, - { - "id": "jaspionjader/bh-20", - "name": "bh-20", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4727, - "hfopenllm_v2/BBH": 0.575, - "hfopenllm_v2/MATH Level 5": 0.1201, - "hfopenllm_v2/GPQA": 0.2878, - "hfopenllm_v2/MUSR": 0.4105, - "hfopenllm_v2/MMLU-PRO": 0.3768 - } - }, - { - "id": "jaspionjader/bh-21", - "name": "bh-21", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.47, - "hfopenllm_v2/BBH": 0.5738, - "hfopenllm_v2/MATH Level 5": 0.1216, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.4158, - "hfopenllm_v2/MMLU-PRO": 0.3776 - } - }, - { - "id": "jaspionjader/bh-22", - "name": "bh-22", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.46, - "hfopenllm_v2/BBH": 0.5793, - "hfopenllm_v2/MATH Level 5": 0.1186, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.4172, - "hfopenllm_v2/MMLU-PRO": 0.3764 - } - }, - { - "id": "jaspionjader/bh-23", - "name": "bh-23", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4658, - "hfopenllm_v2/BBH": 0.57, - "hfopenllm_v2/MATH Level 5": 0.1201, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.4197, - "hfopenllm_v2/MMLU-PRO": 0.3796 - } - }, - { - "id": "jaspionjader/bh-24", - "name": "bh-24", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4715, - "hfopenllm_v2/BBH": 0.5717, - "hfopenllm_v2/MATH Level 5": 0.1269, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.4158, - "hfopenllm_v2/MMLU-PRO": 0.3809 - } - }, - { - "id": "jaspionjader/bh-25", - "name": "bh-25", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4752, - "hfopenllm_v2/BBH": 0.5706, - "hfopenllm_v2/MATH Level 5": 0.1133, - "hfopenllm_v2/GPQA": 0.2911, - "hfopenllm_v2/MUSR": 0.4118, - "hfopenllm_v2/MMLU-PRO": 0.3782 - } - }, - { - "id": "jaspionjader/bh-26", - "name": "bh-26", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4691, - "hfopenllm_v2/BBH": 0.5735, - "hfopenllm_v2/MATH Level 5": 0.1163, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.4277, - "hfopenllm_v2/MMLU-PRO": 0.3772 - } - }, - { - "id": "jaspionjader/bh-27", - "name": "bh-27", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4819, - "hfopenllm_v2/BBH": 0.5714, - "hfopenllm_v2/MATH Level 5": 0.1276, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.4091, - "hfopenllm_v2/MMLU-PRO": 0.3799 - } - }, - { - "id": "jaspionjader/bh-28", - "name": "bh-28", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4785, - "hfopenllm_v2/BBH": 0.5703, - "hfopenllm_v2/MATH Level 5": 0.1231, - "hfopenllm_v2/GPQA": 0.2987, - "hfopenllm_v2/MUSR": 0.4131, - "hfopenllm_v2/MMLU-PRO": 0.3812 - } - }, - { - "id": "jaspionjader/bh-29", - "name": "bh-29", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4688, - "hfopenllm_v2/BBH": 0.567, - "hfopenllm_v2/MATH Level 5": 0.1208, - "hfopenllm_v2/GPQA": 0.2953, - "hfopenllm_v2/MUSR": 0.4237, - "hfopenllm_v2/MMLU-PRO": 0.3819 - } - }, - { - "id": "jaspionjader/bh-3", - "name": "bh-3", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4664, - "hfopenllm_v2/BBH": 0.5891, - "hfopenllm_v2/MATH Level 5": 0.1148, - "hfopenllm_v2/GPQA": 0.302, - "hfopenllm_v2/MUSR": 0.4173, - "hfopenllm_v2/MMLU-PRO": 0.3702 - } - }, - { - "id": "jaspionjader/bh-30", - "name": "bh-30", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4666, - "hfopenllm_v2/BBH": 0.5706, - "hfopenllm_v2/MATH Level 5": 0.1231, - "hfopenllm_v2/GPQA": 0.2928, - "hfopenllm_v2/MUSR": 0.4144, - "hfopenllm_v2/MMLU-PRO": 0.3782 - } - }, - { - "id": "jaspionjader/bh-31", - "name": "bh-31", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4727, - "hfopenllm_v2/BBH": 0.5665, - "hfopenllm_v2/MATH Level 5": 0.1284, - "hfopenllm_v2/GPQA": 0.2936, - "hfopenllm_v2/MUSR": 0.4104, - "hfopenllm_v2/MMLU-PRO": 0.382 - } - }, - { - "id": "jaspionjader/bh-32", - "name": "bh-32", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4636, - "hfopenllm_v2/BBH": 0.5662, - "hfopenllm_v2/MATH Level 5": 0.1246, - "hfopenllm_v2/GPQA": 0.297, - "hfopenllm_v2/MUSR": 0.4157, - "hfopenllm_v2/MMLU-PRO": 0.3812 - } - }, - { - "id": "jaspionjader/bh-33", - "name": "bh-33", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4685, - "hfopenllm_v2/BBH": 0.5653, - "hfopenllm_v2/MATH Level 5": 0.1178, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.4157, - "hfopenllm_v2/MMLU-PRO": 0.3808 - } - }, - { - "id": "jaspionjader/bh-34", - "name": "bh-34", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4624, - "hfopenllm_v2/BBH": 0.5681, - "hfopenllm_v2/MATH Level 5": 0.1208, - "hfopenllm_v2/GPQA": 0.2919, - "hfopenllm_v2/MUSR": 0.4185, - "hfopenllm_v2/MMLU-PRO": 0.3804 - } - }, - { - "id": "jaspionjader/bh-35", - "name": "bh-35", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4721, - "hfopenllm_v2/BBH": 0.564, - "hfopenllm_v2/MATH Level 5": 0.1246, - "hfopenllm_v2/GPQA": 0.2953, - "hfopenllm_v2/MUSR": 0.4183, - "hfopenllm_v2/MMLU-PRO": 0.383 - } - }, - { - "id": "jaspionjader/bh-36", - "name": "bh-36", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4666, - "hfopenllm_v2/BBH": 0.5664, - "hfopenllm_v2/MATH Level 5": 0.1239, - "hfopenllm_v2/GPQA": 0.302, - "hfopenllm_v2/MUSR": 0.4196, - "hfopenllm_v2/MMLU-PRO": 0.3831 - } - }, - { - "id": "jaspionjader/bh-37", - "name": "bh-37", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.488, - "hfopenllm_v2/BBH": 0.5625, - "hfopenllm_v2/MATH Level 5": 0.1216, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.4156, - "hfopenllm_v2/MMLU-PRO": 0.3828 - } - }, - { - "id": "jaspionjader/bh-38", - "name": "bh-38", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4618, - "hfopenllm_v2/BBH": 0.5658, - "hfopenllm_v2/MATH Level 5": 0.1239, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.4117, - "hfopenllm_v2/MMLU-PRO": 0.3811 - } - }, - { - "id": "jaspionjader/bh-39", - "name": "bh-39", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4576, - "hfopenllm_v2/BBH": 0.5633, - "hfopenllm_v2/MATH Level 5": 0.1254, - "hfopenllm_v2/GPQA": 0.3003, - "hfopenllm_v2/MUSR": 0.4262, - "hfopenllm_v2/MMLU-PRO": 0.3831 - } - }, - { - "id": "jaspionjader/bh-4", - "name": "bh-4", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4673, - "hfopenllm_v2/BBH": 0.5892, - "hfopenllm_v2/MATH Level 5": 0.1095, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.4173, - "hfopenllm_v2/MMLU-PRO": 0.3705 - } - }, - { - "id": "jaspionjader/bh-40", - "name": "bh-40", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4536, - "hfopenllm_v2/BBH": 0.5634, - "hfopenllm_v2/MATH Level 5": 0.1246, - "hfopenllm_v2/GPQA": 0.2987, - "hfopenllm_v2/MUSR": 0.4236, - "hfopenllm_v2/MMLU-PRO": 0.3835 - } - }, - { - "id": "jaspionjader/bh-41", - "name": "bh-41", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.474, - "hfopenllm_v2/BBH": 0.5614, - "hfopenllm_v2/MATH Level 5": 0.1254, - "hfopenllm_v2/GPQA": 0.2928, - "hfopenllm_v2/MUSR": 0.4183, - "hfopenllm_v2/MMLU-PRO": 0.3825 - } - }, - { - "id": "jaspionjader/bh-42", - "name": "bh-42", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.466, - "hfopenllm_v2/BBH": 0.5646, - "hfopenllm_v2/MATH Level 5": 0.1269, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.421, - "hfopenllm_v2/MMLU-PRO": 0.3812 - } - }, - { - "id": "jaspionjader/bh-43", - "name": "bh-43", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.46, - "hfopenllm_v2/BBH": 0.5635, - "hfopenllm_v2/MATH Level 5": 0.1239, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.4156, - "hfopenllm_v2/MMLU-PRO": 0.382 - } - }, - { - "id": "jaspionjader/bh-44", - "name": "bh-44", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4706, - "hfopenllm_v2/BBH": 0.5643, - "hfopenllm_v2/MATH Level 5": 0.1216, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.4249, - "hfopenllm_v2/MMLU-PRO": 0.3834 - } - }, - { - "id": "jaspionjader/bh-46", - "name": "bh-46", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4727, - "hfopenllm_v2/BBH": 0.5632, - "hfopenllm_v2/MATH Level 5": 0.1276, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.4262, - "hfopenllm_v2/MMLU-PRO": 0.3822 - } - }, - { - "id": "jaspionjader/bh-47", - "name": "bh-47", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4652, - "hfopenllm_v2/BBH": 0.5546, - "hfopenllm_v2/MATH Level 5": 0.1276, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.4156, - "hfopenllm_v2/MMLU-PRO": 0.3855 - } - }, - { - "id": "jaspionjader/bh-48", - "name": "bh-48", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4688, - "hfopenllm_v2/BBH": 0.5541, - "hfopenllm_v2/MATH Level 5": 0.1254, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.4209, - "hfopenllm_v2/MMLU-PRO": 0.386 - } - }, - { - "id": "jaspionjader/bh-49", - "name": "bh-49", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4725, - "hfopenllm_v2/BBH": 0.554, - "hfopenllm_v2/MATH Level 5": 0.1201, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.4129, - "hfopenllm_v2/MMLU-PRO": 0.3808 - } - }, - { - "id": "jaspionjader/bh-5", - "name": "bh-5", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4652, - "hfopenllm_v2/BBH": 0.5882, - "hfopenllm_v2/MATH Level 5": 0.1057, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.4186, - "hfopenllm_v2/MMLU-PRO": 0.3702 - } - }, - { - "id": "jaspionjader/bh-50", - "name": "bh-50", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4725, - "hfopenllm_v2/BBH": 0.5553, - "hfopenllm_v2/MATH Level 5": 0.1208, - "hfopenllm_v2/GPQA": 0.2936, - "hfopenllm_v2/MUSR": 0.4169, - "hfopenllm_v2/MMLU-PRO": 0.3842 - } - }, - { - "id": "jaspionjader/bh-51", - "name": "bh-51", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.463, - "hfopenllm_v2/BBH": 0.5557, - "hfopenllm_v2/MATH Level 5": 0.1239, - "hfopenllm_v2/GPQA": 0.2928, - "hfopenllm_v2/MUSR": 0.4168, - "hfopenllm_v2/MMLU-PRO": 0.3831 - } - }, - { - "id": "jaspionjader/bh-52", - "name": "bh-52", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4536, - "hfopenllm_v2/BBH": 0.5444, - "hfopenllm_v2/MATH Level 5": 0.1201, - "hfopenllm_v2/GPQA": 0.2919, - "hfopenllm_v2/MUSR": 0.4169, - "hfopenllm_v2/MMLU-PRO": 0.3843 - } - }, - { - "id": "jaspionjader/bh-53", - "name": "bh-53", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.478, - "hfopenllm_v2/BBH": 0.5494, - "hfopenllm_v2/MATH Level 5": 0.1269, - "hfopenllm_v2/GPQA": 0.2987, - "hfopenllm_v2/MUSR": 0.4196, - "hfopenllm_v2/MMLU-PRO": 0.3858 - } - }, - { - "id": "jaspionjader/bh-54", - "name": "bh-54", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4841, - "hfopenllm_v2/BBH": 0.5548, - "hfopenllm_v2/MATH Level 5": 0.1292, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.4155, - "hfopenllm_v2/MMLU-PRO": 0.3825 - } - }, - { - "id": "jaspionjader/bh-55", - "name": "bh-55", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4709, - "hfopenllm_v2/BBH": 0.555, - "hfopenllm_v2/MATH Level 5": 0.1284, - "hfopenllm_v2/GPQA": 0.3062, - "hfopenllm_v2/MUSR": 0.4222, - "hfopenllm_v2/MMLU-PRO": 0.3846 - } - }, - { - "id": "jaspionjader/bh-56", - "name": "bh-56", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.46, - "hfopenllm_v2/BBH": 0.5447, - "hfopenllm_v2/MATH Level 5": 0.1231, - "hfopenllm_v2/GPQA": 0.3003, - "hfopenllm_v2/MUSR": 0.4116, - "hfopenllm_v2/MMLU-PRO": 0.3844 - } - }, - { - "id": "jaspionjader/bh-57", - "name": "bh-57", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4405, - "hfopenllm_v2/BBH": 0.5425, - "hfopenllm_v2/MATH Level 5": 0.1261, - "hfopenllm_v2/GPQA": 0.3037, - "hfopenllm_v2/MUSR": 0.421, - "hfopenllm_v2/MMLU-PRO": 0.3896 - } - }, - { - "id": "jaspionjader/bh-58", - "name": "bh-58", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.463, - "hfopenllm_v2/BBH": 0.5446, - "hfopenllm_v2/MATH Level 5": 0.1322, - "hfopenllm_v2/GPQA": 0.3062, - "hfopenllm_v2/MUSR": 0.4183, - "hfopenllm_v2/MMLU-PRO": 0.3896 - } - }, - { - "id": "jaspionjader/bh-59", - "name": "bh-59", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4341, - "hfopenllm_v2/BBH": 0.5512, - "hfopenllm_v2/MATH Level 5": 0.1541, - "hfopenllm_v2/GPQA": 0.3154, - "hfopenllm_v2/MUSR": 0.417, - "hfopenllm_v2/MMLU-PRO": 0.3838 - } - }, - { - "id": "jaspionjader/bh-6", - "name": "bh-6", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4621, - "hfopenllm_v2/BBH": 0.5891, - "hfopenllm_v2/MATH Level 5": 0.1088, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.4199, - "hfopenllm_v2/MMLU-PRO": 0.3698 - } - }, - { - "id": "jaspionjader/bh-60", - "name": "bh-60", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4207, - "hfopenllm_v2/BBH": 0.5369, - "hfopenllm_v2/MATH Level 5": 0.1579, - "hfopenllm_v2/GPQA": 0.3255, - "hfopenllm_v2/MUSR": 0.4289, - "hfopenllm_v2/MMLU-PRO": 0.3689 - } - }, - { - "id": "jaspionjader/bh-61", - "name": "bh-61", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4247, - "hfopenllm_v2/BBH": 0.5271, - "hfopenllm_v2/MATH Level 5": 0.1707, - "hfopenllm_v2/GPQA": 0.3188, - "hfopenllm_v2/MUSR": 0.4356, - "hfopenllm_v2/MMLU-PRO": 0.3679 - } - }, - { - "id": "jaspionjader/bh-62", - "name": "bh-62", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.415, - "hfopenllm_v2/BBH": 0.5379, - "hfopenllm_v2/MATH Level 5": 0.1624, - "hfopenllm_v2/GPQA": 0.3205, - "hfopenllm_v2/MUSR": 0.4289, - "hfopenllm_v2/MMLU-PRO": 0.3719 - } - }, - { - "id": "jaspionjader/bh-63", - "name": "bh-63", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4308, - "hfopenllm_v2/BBH": 0.4917, - "hfopenllm_v2/MATH Level 5": 0.111, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.4313, - "hfopenllm_v2/MMLU-PRO": 0.3248 - } - }, - { - "id": "jaspionjader/bh-64", - "name": "bh-64", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.414, - "hfopenllm_v2/BBH": 0.536, - "hfopenllm_v2/MATH Level 5": 0.1548, - "hfopenllm_v2/GPQA": 0.3213, - "hfopenllm_v2/MUSR": 0.4355, - "hfopenllm_v2/MMLU-PRO": 0.3693 - } - }, - { - "id": "jaspionjader/bh-7", - "name": "bh-7", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4624, - "hfopenllm_v2/BBH": 0.5861, - "hfopenllm_v2/MATH Level 5": 0.114, - "hfopenllm_v2/GPQA": 0.3037, - "hfopenllm_v2/MUSR": 0.4119, - "hfopenllm_v2/MMLU-PRO": 0.3715 - } - }, - { - "id": "jaspionjader/bh-8", - "name": "bh-8", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4597, - "hfopenllm_v2/BBH": 0.59, - "hfopenllm_v2/MATH Level 5": 0.1178, - "hfopenllm_v2/GPQA": 0.3012, - "hfopenllm_v2/MUSR": 0.4265, - "hfopenllm_v2/MMLU-PRO": 0.372 - } - }, - { - "id": "jaspionjader/bh-9", - "name": "bh-9", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4509, - "hfopenllm_v2/BBH": 0.585, - "hfopenllm_v2/MATH Level 5": 0.1156, - "hfopenllm_v2/GPQA": 0.302, - "hfopenllm_v2/MUSR": 0.4146, - "hfopenllm_v2/MMLU-PRO": 0.3703 - } - }, - { - "id": "jaspionjader/dp-6-8b", - "name": "dp-6-8b", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4806, - "hfopenllm_v2/BBH": 0.53, - "hfopenllm_v2/MATH Level 5": 0.1329, - "hfopenllm_v2/GPQA": 0.3079, - "hfopenllm_v2/MUSR": 0.4434, - "hfopenllm_v2/MMLU-PRO": 0.3897 - } - }, - { - "id": "jaspionjader/dp-7-8b", - "name": "dp-7-8b", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4498, - "hfopenllm_v2/BBH": 0.5291, - "hfopenllm_v2/MATH Level 5": 0.1261, - "hfopenllm_v2/GPQA": 0.3062, - "hfopenllm_v2/MUSR": 0.4407, - "hfopenllm_v2/MMLU-PRO": 0.3934 - } - }, - { - "id": "jaspionjader/ek-6", - "name": "ek-6", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4642, - "hfopenllm_v2/BBH": 0.5219, - "hfopenllm_v2/MATH Level 5": 0.1322, - "hfopenllm_v2/GPQA": 0.3087, - "hfopenllm_v2/MUSR": 0.4144, - "hfopenllm_v2/MMLU-PRO": 0.3861 - } - }, - { - "id": "jaspionjader/ek-7", - "name": "ek-7", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4767, - "hfopenllm_v2/BBH": 0.5194, - "hfopenllm_v2/MATH Level 5": 0.1329, - "hfopenllm_v2/GPQA": 0.3163, - "hfopenllm_v2/MUSR": 0.4171, - "hfopenllm_v2/MMLU-PRO": 0.3887 - } - }, - { - "id": "jaspionjader/f-1-8b", - "name": "f-1-8b", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4983, - "hfopenllm_v2/BBH": 0.5141, - "hfopenllm_v2/MATH Level 5": 0.1284, - "hfopenllm_v2/GPQA": 0.3087, - "hfopenllm_v2/MUSR": 0.4527, - "hfopenllm_v2/MMLU-PRO": 0.3907 - } - }, - { - "id": "jaspionjader/f-2-8b", - "name": "f-2-8b", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4824, - "hfopenllm_v2/BBH": 0.5294, - "hfopenllm_v2/MATH Level 5": 0.1171, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.4501, - "hfopenllm_v2/MMLU-PRO": 0.3962 - } - }, - { - "id": "jaspionjader/f-3-8b", - "name": "f-3-8b", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4803, - "hfopenllm_v2/BBH": 0.5275, - "hfopenllm_v2/MATH Level 5": 0.1216, - "hfopenllm_v2/GPQA": 0.3138, - "hfopenllm_v2/MUSR": 0.4421, - "hfopenllm_v2/MMLU-PRO": 0.3954 - } - }, - { - "id": "jaspionjader/f-4-8b", - "name": "f-4-8b", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4797, - "hfopenllm_v2/BBH": 0.5289, - "hfopenllm_v2/MATH Level 5": 0.1148, - "hfopenllm_v2/GPQA": 0.3087, - "hfopenllm_v2/MUSR": 0.4514, - "hfopenllm_v2/MMLU-PRO": 0.3956 - } - }, - { - "id": "jaspionjader/f-5-8b", - "name": "f-5-8b", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5044, - "hfopenllm_v2/BBH": 0.5313, - "hfopenllm_v2/MATH Level 5": 0.1239, - "hfopenllm_v2/GPQA": 0.3087, - "hfopenllm_v2/MUSR": 0.4461, - "hfopenllm_v2/MMLU-PRO": 0.3949 - } - }, - { - "id": "jaspionjader/f-6-8b", - "name": "f-6-8b", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4846, - "hfopenllm_v2/BBH": 0.5241, - "hfopenllm_v2/MATH Level 5": 0.1193, - "hfopenllm_v2/GPQA": 0.3079, - "hfopenllm_v2/MUSR": 0.4474, - "hfopenllm_v2/MMLU-PRO": 0.3939 - } - }, - { - "id": "jaspionjader/f-7-8b", - "name": "f-7-8b", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4462, - "hfopenllm_v2/BBH": 0.5277, - "hfopenllm_v2/MATH Level 5": 0.1239, - "hfopenllm_v2/GPQA": 0.3129, - "hfopenllm_v2/MUSR": 0.4315, - "hfopenllm_v2/MMLU-PRO": 0.3936 - } - }, - { - "id": "jaspionjader/f-8-8b", - "name": "f-8-8b", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4739, - "hfopenllm_v2/BBH": 0.5259, - "hfopenllm_v2/MATH Level 5": 0.1224, - "hfopenllm_v2/GPQA": 0.3096, - "hfopenllm_v2/MUSR": 0.4354, - "hfopenllm_v2/MMLU-PRO": 0.394 - } - }, - { - "id": "jaspionjader/f-9-8b", - "name": "f-9-8b", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4602, - "hfopenllm_v2/BBH": 0.5292, - "hfopenllm_v2/MATH Level 5": 0.1299, - "hfopenllm_v2/GPQA": 0.3062, - "hfopenllm_v2/MUSR": 0.4461, - "hfopenllm_v2/MMLU-PRO": 0.3944 - } - }, - { - "id": "jaspionjader/fct-14-8b", - "name": "fct-14-8b", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4129, - "hfopenllm_v2/BBH": 0.5206, - "hfopenllm_v2/MATH Level 5": 0.1201, - "hfopenllm_v2/GPQA": 0.3163, - "hfopenllm_v2/MUSR": 0.4186, - "hfopenllm_v2/MMLU-PRO": 0.3875 - } - }, - { - "id": "jaspionjader/fct-9-8b", - "name": "fct-9-8b", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4354, - "hfopenllm_v2/BBH": 0.5205, - "hfopenllm_v2/MATH Level 5": 0.1193, - "hfopenllm_v2/GPQA": 0.3079, - "hfopenllm_v2/MUSR": 0.4291, - "hfopenllm_v2/MMLU-PRO": 0.3932 - } - }, - { - "id": "jaspionjader/fr-1-8b", - "name": "fr-1-8b", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4211, - "hfopenllm_v2/BBH": 0.5142, - "hfopenllm_v2/MATH Level 5": 0.1118, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.4277, - "hfopenllm_v2/MMLU-PRO": 0.361 - } - }, - { - "id": "jaspionjader/fr-10-8b", - "name": "fr-10-8b", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4402, - "hfopenllm_v2/BBH": 0.5207, - "hfopenllm_v2/MATH Level 5": 0.1224, - "hfopenllm_v2/GPQA": 0.3171, - "hfopenllm_v2/MUSR": 0.4119, - "hfopenllm_v2/MMLU-PRO": 0.3863 - } - }, - { - "id": "jaspionjader/fr-3-8b", - "name": "fr-3-8b", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4326, - "hfopenllm_v2/BBH": 0.5255, - "hfopenllm_v2/MATH Level 5": 0.1133, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.4198, - "hfopenllm_v2/MMLU-PRO": 0.3863 - } - }, - { - "id": "jaspionjader/gamma-Kosmos-EVAA-8B", - "name": "gamma-Kosmos-EVAA-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.425, - "hfopenllm_v2/BBH": 0.5253, - "hfopenllm_v2/MATH Level 5": 0.0899, - "hfopenllm_v2/GPQA": 0.3138, - "hfopenllm_v2/MUSR": 0.4412, - "hfopenllm_v2/MMLU-PRO": 0.3776 - } - }, - { - "id": "jaspionjader/gamma-Kosmos-EVAA-v2-8B", - "name": "gamma-Kosmos-EVAA-v2-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4233, - "hfopenllm_v2/BBH": 0.5262, - "hfopenllm_v2/MATH Level 5": 0.1057, - "hfopenllm_v2/GPQA": 0.3205, - "hfopenllm_v2/MUSR": 0.4344, - "hfopenllm_v2/MMLU-PRO": 0.3756 - } - }, - { - "id": "jaspionjader/gamma-Kosmos-EVAA-v3-8B", - "name": "gamma-Kosmos-EVAA-v3-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4333, - "hfopenllm_v2/BBH": 0.5278, - "hfopenllm_v2/MATH Level 5": 0.111, - "hfopenllm_v2/GPQA": 0.3129, - "hfopenllm_v2/MUSR": 0.4263, - "hfopenllm_v2/MMLU-PRO": 0.3898 - } - }, - { - "id": "jaspionjader/knf-2-8b", - "name": "knf-2-8b", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.425, - "hfopenllm_v2/BBH": 0.5207, - "hfopenllm_v2/MATH Level 5": 0.1201, - "hfopenllm_v2/GPQA": 0.3104, - "hfopenllm_v2/MUSR": 0.4185, - "hfopenllm_v2/MMLU-PRO": 0.3875 - } - }, - { - "id": "jaspionjader/knfp-2-8b", - "name": "knfp-2-8b", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5327, - "hfopenllm_v2/BBH": 0.5305, - "hfopenllm_v2/MATH Level 5": 0.1427, - "hfopenllm_v2/GPQA": 0.2928, - "hfopenllm_v2/MUSR": 0.4185, - "hfopenllm_v2/MMLU-PRO": 0.3726 - } - }, - { - "id": "jaspionjader/knfp-3-8b", - "name": "knfp-3-8b", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4946, - "hfopenllm_v2/BBH": 0.52, - "hfopenllm_v2/MATH Level 5": 0.1224, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.4171, - "hfopenllm_v2/MMLU-PRO": 0.3881 - } - }, - { - "id": "jaspionjader/Kosmos-Aurora_faustus-8B", - "name": "Kosmos-Aurora_faustus-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4432, - "hfopenllm_v2/BBH": 0.526, - "hfopenllm_v2/MATH Level 5": 0.1125, - "hfopenllm_v2/GPQA": 0.2953, - "hfopenllm_v2/MUSR": 0.4117, - "hfopenllm_v2/MMLU-PRO": 0.3813 - } - }, - { - "id": "jaspionjader/Kosmos-Elusive-8b", - "name": "Kosmos-Elusive-8b", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4169, - "hfopenllm_v2/BBH": 0.5339, - "hfopenllm_v2/MATH Level 5": 0.1261, - "hfopenllm_v2/GPQA": 0.3079, - "hfopenllm_v2/MUSR": 0.4078, - "hfopenllm_v2/MMLU-PRO": 0.376 - } - }, - { - "id": "jaspionjader/Kosmos-Elusive-VENN-8B", - "name": "Kosmos-Elusive-VENN-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4233, - "hfopenllm_v2/BBH": 0.5356, - "hfopenllm_v2/MATH Level 5": 0.1246, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.4157, - "hfopenllm_v2/MMLU-PRO": 0.3797 - } - }, - { - "id": "jaspionjader/Kosmos-Elusive-VENN-Asymmetric-8B", - "name": "Kosmos-Elusive-VENN-Asymmetric-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4542, - "hfopenllm_v2/BBH": 0.5313, - "hfopenllm_v2/MATH Level 5": 0.1344, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.4251, - "hfopenllm_v2/MMLU-PRO": 0.3842 - } - }, - { - "id": "jaspionjader/Kosmos-Elusive-VENN-Aurora_faustus-8B", - "name": "Kosmos-Elusive-VENN-Aurora_faustus-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4335, - "hfopenllm_v2/BBH": 0.5304, - "hfopenllm_v2/MATH Level 5": 0.1125, - "hfopenllm_v2/GPQA": 0.2953, - "hfopenllm_v2/MUSR": 0.417, - "hfopenllm_v2/MMLU-PRO": 0.3795 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-8B", - "name": "Kosmos-EVAA-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4405, - "hfopenllm_v2/BBH": 0.5312, - "hfopenllm_v2/MATH Level 5": 0.1178, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.4237, - "hfopenllm_v2/MMLU-PRO": 0.3818 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-Franken-Immersive-v39-8B", - "name": "Kosmos-EVAA-Franken-Immersive-v39-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4378, - "hfopenllm_v2/BBH": 0.519, - "hfopenllm_v2/MATH Level 5": 0.1292, - "hfopenllm_v2/GPQA": 0.3154, - "hfopenllm_v2/MUSR": 0.4236, - "hfopenllm_v2/MMLU-PRO": 0.39 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-Franken-v38-8B", - "name": "Kosmos-EVAA-Franken-v38-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4356, - "hfopenllm_v2/BBH": 0.523, - "hfopenllm_v2/MATH Level 5": 0.1292, - "hfopenllm_v2/GPQA": 0.3087, - "hfopenllm_v2/MUSR": 0.4212, - "hfopenllm_v2/MMLU-PRO": 0.389 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-Fusion-8B", - "name": "Kosmos-EVAA-Fusion-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4418, - "hfopenllm_v2/BBH": 0.5406, - "hfopenllm_v2/MATH Level 5": 0.1352, - "hfopenllm_v2/GPQA": 0.3062, - "hfopenllm_v2/MUSR": 0.4277, - "hfopenllm_v2/MMLU-PRO": 0.386 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-gamma-8B", - "name": "Kosmos-EVAA-gamma-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4572, - "hfopenllm_v2/BBH": 0.5322, - "hfopenllm_v2/MATH Level 5": 0.105, - "hfopenllm_v2/GPQA": 0.3188, - "hfopenllm_v2/MUSR": 0.4306, - "hfopenllm_v2/MMLU-PRO": 0.3901 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-gamma-alt-8B", - "name": "Kosmos-EVAA-gamma-alt-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4542, - "hfopenllm_v2/BBH": 0.5298, - "hfopenllm_v2/MATH Level 5": 0.1095, - "hfopenllm_v2/GPQA": 0.3247, - "hfopenllm_v2/MUSR": 0.4292, - "hfopenllm_v2/MMLU-PRO": 0.3896 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-gamma-light-8B", - "name": "Kosmos-EVAA-gamma-light-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4581, - "hfopenllm_v2/BBH": 0.5376, - "hfopenllm_v2/MATH Level 5": 0.1103, - "hfopenllm_v2/GPQA": 0.3163, - "hfopenllm_v2/MUSR": 0.4291, - "hfopenllm_v2/MMLU-PRO": 0.3943 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-gamma-light-alt-8B", - "name": "Kosmos-EVAA-gamma-light-alt-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4454, - "hfopenllm_v2/BBH": 0.5327, - "hfopenllm_v2/MATH Level 5": 0.1133, - "hfopenllm_v2/GPQA": 0.3138, - "hfopenllm_v2/MUSR": 0.4305, - "hfopenllm_v2/MMLU-PRO": 0.3923 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-gamma-ultra-light-8B", - "name": "Kosmos-EVAA-gamma-ultra-light-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4563, - "hfopenllm_v2/BBH": 0.5316, - "hfopenllm_v2/MATH Level 5": 0.1178, - "hfopenllm_v2/GPQA": 0.3163, - "hfopenllm_v2/MUSR": 0.4197, - "hfopenllm_v2/MMLU-PRO": 0.3915 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-gamma-v13-8B", - "name": "Kosmos-EVAA-gamma-v13-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4429, - "hfopenllm_v2/BBH": 0.5359, - "hfopenllm_v2/MATH Level 5": 0.1118, - "hfopenllm_v2/GPQA": 0.3138, - "hfopenllm_v2/MUSR": 0.4278, - "hfopenllm_v2/MMLU-PRO": 0.393 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-gamma-v14-8B", - "name": "Kosmos-EVAA-gamma-v14-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.438, - "hfopenllm_v2/BBH": 0.5363, - "hfopenllm_v2/MATH Level 5": 0.1103, - "hfopenllm_v2/GPQA": 0.3129, - "hfopenllm_v2/MUSR": 0.4277, - "hfopenllm_v2/MMLU-PRO": 0.3931 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-gamma-v15-8B", - "name": "Kosmos-EVAA-gamma-v15-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4654, - "hfopenllm_v2/BBH": 0.5343, - "hfopenllm_v2/MATH Level 5": 0.111, - "hfopenllm_v2/GPQA": 0.3112, - "hfopenllm_v2/MUSR": 0.4277, - "hfopenllm_v2/MMLU-PRO": 0.3941 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-gamma-v16-8B", - "name": "Kosmos-EVAA-gamma-v16-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4557, - "hfopenllm_v2/BBH": 0.5344, - "hfopenllm_v2/MATH Level 5": 0.1171, - "hfopenllm_v2/GPQA": 0.3154, - "hfopenllm_v2/MUSR": 0.4264, - "hfopenllm_v2/MMLU-PRO": 0.3917 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-gamma-v17-8B", - "name": "Kosmos-EVAA-gamma-v17-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4462, - "hfopenllm_v2/BBH": 0.5347, - "hfopenllm_v2/MATH Level 5": 0.111, - "hfopenllm_v2/GPQA": 0.3112, - "hfopenllm_v2/MUSR": 0.4291, - "hfopenllm_v2/MMLU-PRO": 0.3923 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-gamma-v18-8B", - "name": "Kosmos-EVAA-gamma-v18-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4341, - "hfopenllm_v2/BBH": 0.5339, - "hfopenllm_v2/MATH Level 5": 0.111, - "hfopenllm_v2/GPQA": 0.3112, - "hfopenllm_v2/MUSR": 0.4317, - "hfopenllm_v2/MMLU-PRO": 0.3905 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-immersive-sof-v44-8B", - "name": "Kosmos-EVAA-immersive-sof-v44-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4408, - "hfopenllm_v2/BBH": 0.5215, - "hfopenllm_v2/MATH Level 5": 0.1186, - "hfopenllm_v2/GPQA": 0.3096, - "hfopenllm_v2/MUSR": 0.4144, - "hfopenllm_v2/MMLU-PRO": 0.3888 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-PRP-8B", - "name": "Kosmos-EVAA-PRP-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3405, - "hfopenllm_v2/BBH": 0.5196, - "hfopenllm_v2/MATH Level 5": 0.0884, - "hfopenllm_v2/GPQA": 0.3129, - "hfopenllm_v2/MUSR": 0.4301, - "hfopenllm_v2/MMLU-PRO": 0.3647 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-PRP-light-8B", - "name": "Kosmos-EVAA-PRP-light-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3824, - "hfopenllm_v2/BBH": 0.5271, - "hfopenllm_v2/MATH Level 5": 0.1103, - "hfopenllm_v2/GPQA": 0.3121, - "hfopenllm_v2/MUSR": 0.4249, - "hfopenllm_v2/MMLU-PRO": 0.3782 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-PRP-v23-8B", - "name": "Kosmos-EVAA-PRP-v23-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4041, - "hfopenllm_v2/BBH": 0.529, - "hfopenllm_v2/MATH Level 5": 0.1156, - "hfopenllm_v2/GPQA": 0.3087, - "hfopenllm_v2/MUSR": 0.4368, - "hfopenllm_v2/MMLU-PRO": 0.3706 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-PRP-v24-8B", - "name": "Kosmos-EVAA-PRP-v24-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4259, - "hfopenllm_v2/BBH": 0.5276, - "hfopenllm_v2/MATH Level 5": 0.1103, - "hfopenllm_v2/GPQA": 0.3104, - "hfopenllm_v2/MUSR": 0.429, - "hfopenllm_v2/MMLU-PRO": 0.3779 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-PRP-v25-8B", - "name": "Kosmos-EVAA-PRP-v25-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4421, - "hfopenllm_v2/BBH": 0.5291, - "hfopenllm_v2/MATH Level 5": 0.1186, - "hfopenllm_v2/GPQA": 0.318, - "hfopenllm_v2/MUSR": 0.4303, - "hfopenllm_v2/MMLU-PRO": 0.3716 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-PRP-v26-8B", - "name": "Kosmos-EVAA-PRP-v26-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4414, - "hfopenllm_v2/BBH": 0.5271, - "hfopenllm_v2/MATH Level 5": 0.1133, - "hfopenllm_v2/GPQA": 0.3045, - "hfopenllm_v2/MUSR": 0.4264, - "hfopenllm_v2/MMLU-PRO": 0.3793 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-PRP-v27-8B", - "name": "Kosmos-EVAA-PRP-v27-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4378, - "hfopenllm_v2/BBH": 0.529, - "hfopenllm_v2/MATH Level 5": 0.1193, - "hfopenllm_v2/GPQA": 0.3087, - "hfopenllm_v2/MUSR": 0.4343, - "hfopenllm_v2/MMLU-PRO": 0.3755 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-PRP-v28-8B", - "name": "Kosmos-EVAA-PRP-v28-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4366, - "hfopenllm_v2/BBH": 0.5295, - "hfopenllm_v2/MATH Level 5": 0.1171, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.433, - "hfopenllm_v2/MMLU-PRO": 0.375 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-PRP-v29-8B", - "name": "Kosmos-EVAA-PRP-v29-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4487, - "hfopenllm_v2/BBH": 0.5275, - "hfopenllm_v2/MATH Level 5": 0.1201, - "hfopenllm_v2/GPQA": 0.3104, - "hfopenllm_v2/MUSR": 0.4237, - "hfopenllm_v2/MMLU-PRO": 0.3765 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-PRP-v30-8B", - "name": "Kosmos-EVAA-PRP-v30-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4295, - "hfopenllm_v2/BBH": 0.5328, - "hfopenllm_v2/MATH Level 5": 0.1178, - "hfopenllm_v2/GPQA": 0.3045, - "hfopenllm_v2/MUSR": 0.4263, - "hfopenllm_v2/MMLU-PRO": 0.3938 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-PRP-v31-8B", - "name": "Kosmos-EVAA-PRP-v31-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4399, - "hfopenllm_v2/BBH": 0.5315, - "hfopenllm_v2/MATH Level 5": 0.1133, - "hfopenllm_v2/GPQA": 0.3138, - "hfopenllm_v2/MUSR": 0.4251, - "hfopenllm_v2/MMLU-PRO": 0.3935 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-PRP-v32-8B", - "name": "Kosmos-EVAA-PRP-v32-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4487, - "hfopenllm_v2/BBH": 0.5293, - "hfopenllm_v2/MATH Level 5": 0.1148, - "hfopenllm_v2/GPQA": 0.3163, - "hfopenllm_v2/MUSR": 0.4211, - "hfopenllm_v2/MMLU-PRO": 0.3777 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-PRP-v33-8B", - "name": "Kosmos-EVAA-PRP-v33-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4302, - "hfopenllm_v2/BBH": 0.5321, - "hfopenllm_v2/MATH Level 5": 0.1178, - "hfopenllm_v2/GPQA": 0.3129, - "hfopenllm_v2/MUSR": 0.4184, - "hfopenllm_v2/MMLU-PRO": 0.3909 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-PRP-v34-8B", - "name": "Kosmos-EVAA-PRP-v34-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4563, - "hfopenllm_v2/BBH": 0.5333, - "hfopenllm_v2/MATH Level 5": 0.1125, - "hfopenllm_v2/GPQA": 0.3112, - "hfopenllm_v2/MUSR": 0.4237, - "hfopenllm_v2/MMLU-PRO": 0.3927 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-TSN-8B", - "name": "Kosmos-EVAA-TSN-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4721, - "hfopenllm_v2/BBH": 0.5177, - "hfopenllm_v2/MATH Level 5": 0.1344, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.4329, - "hfopenllm_v2/MMLU-PRO": 0.3816 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-TSN-light-8B", - "name": "Kosmos-EVAA-TSN-light-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4685, - "hfopenllm_v2/BBH": 0.5235, - "hfopenllm_v2/MATH Level 5": 0.1216, - "hfopenllm_v2/GPQA": 0.3045, - "hfopenllm_v2/MUSR": 0.4289, - "hfopenllm_v2/MMLU-PRO": 0.3806 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-TSN-v19-8B", - "name": "Kosmos-EVAA-TSN-v19-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4564, - "hfopenllm_v2/BBH": 0.5316, - "hfopenllm_v2/MATH Level 5": 0.1156, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.4277, - "hfopenllm_v2/MMLU-PRO": 0.379 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-TSN-v20-8B", - "name": "Kosmos-EVAA-TSN-v20-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4423, - "hfopenllm_v2/BBH": 0.525, - "hfopenllm_v2/MATH Level 5": 0.1246, - "hfopenllm_v2/GPQA": 0.3138, - "hfopenllm_v2/MUSR": 0.421, - "hfopenllm_v2/MMLU-PRO": 0.3936 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-TSN-v21-8B", - "name": "Kosmos-EVAA-TSN-v21-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.467, - "hfopenllm_v2/BBH": 0.5248, - "hfopenllm_v2/MATH Level 5": 0.1193, - "hfopenllm_v2/GPQA": 0.3121, - "hfopenllm_v2/MUSR": 0.4343, - "hfopenllm_v2/MMLU-PRO": 0.3816 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-TSN-v22-8B", - "name": "Kosmos-EVAA-TSN-v22-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4673, - "hfopenllm_v2/BBH": 0.5246, - "hfopenllm_v2/MATH Level 5": 0.1133, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.4303, - "hfopenllm_v2/MMLU-PRO": 0.3812 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-v10-8B", - "name": "Kosmos-EVAA-v10-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4262, - "hfopenllm_v2/BBH": 0.5376, - "hfopenllm_v2/MATH Level 5": 0.1246, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.4224, - "hfopenllm_v2/MMLU-PRO": 0.3831 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-v11-8B", - "name": "Kosmos-EVAA-v11-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4426, - "hfopenllm_v2/BBH": 0.5359, - "hfopenllm_v2/MATH Level 5": 0.1322, - "hfopenllm_v2/GPQA": 0.3154, - "hfopenllm_v2/MUSR": 0.4184, - "hfopenllm_v2/MMLU-PRO": 0.3836 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-v12-8B", - "name": "Kosmos-EVAA-v12-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4378, - "hfopenllm_v2/BBH": 0.5349, - "hfopenllm_v2/MATH Level 5": 0.1367, - "hfopenllm_v2/GPQA": 0.3096, - "hfopenllm_v2/MUSR": 0.4211, - "hfopenllm_v2/MMLU-PRO": 0.3836 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-v2-8B", - "name": "Kosmos-EVAA-v2-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4396, - "hfopenllm_v2/BBH": 0.5341, - "hfopenllm_v2/MATH Level 5": 0.1322, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.4211, - "hfopenllm_v2/MMLU-PRO": 0.3826 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-v3-8B", - "name": "Kosmos-EVAA-v3-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4411, - "hfopenllm_v2/BBH": 0.5331, - "hfopenllm_v2/MATH Level 5": 0.1329, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.4224, - "hfopenllm_v2/MMLU-PRO": 0.3821 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-v4-8B", - "name": "Kosmos-EVAA-v4-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4289, - "hfopenllm_v2/BBH": 0.5337, - "hfopenllm_v2/MATH Level 5": 0.1254, - "hfopenllm_v2/GPQA": 0.3003, - "hfopenllm_v2/MUSR": 0.4197, - "hfopenllm_v2/MMLU-PRO": 0.3817 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-v5-8B", - "name": "Kosmos-EVAA-v5-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.446, - "hfopenllm_v2/BBH": 0.5345, - "hfopenllm_v2/MATH Level 5": 0.1261, - "hfopenllm_v2/GPQA": 0.3037, - "hfopenllm_v2/MUSR": 0.4224, - "hfopenllm_v2/MMLU-PRO": 0.3821 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-v6-8B", - "name": "Kosmos-EVAA-v6-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4396, - "hfopenllm_v2/BBH": 0.538, - "hfopenllm_v2/MATH Level 5": 0.1292, - "hfopenllm_v2/GPQA": 0.302, - "hfopenllm_v2/MUSR": 0.4184, - "hfopenllm_v2/MMLU-PRO": 0.3821 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-v7-8B", - "name": "Kosmos-EVAA-v7-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4277, - "hfopenllm_v2/BBH": 0.5335, - "hfopenllm_v2/MATH Level 5": 0.1337, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.4171, - "hfopenllm_v2/MMLU-PRO": 0.3836 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-v8-8B", - "name": "Kosmos-EVAA-v8-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4383, - "hfopenllm_v2/BBH": 0.5359, - "hfopenllm_v2/MATH Level 5": 0.1307, - "hfopenllm_v2/GPQA": 0.3037, - "hfopenllm_v2/MUSR": 0.421, - "hfopenllm_v2/MMLU-PRO": 0.3827 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-v9-8B", - "name": "Kosmos-EVAA-v9-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4369, - "hfopenllm_v2/BBH": 0.5361, - "hfopenllm_v2/MATH Level 5": 0.1276, - "hfopenllm_v2/GPQA": 0.3062, - "hfopenllm_v2/MUSR": 0.4184, - "hfopenllm_v2/MMLU-PRO": 0.382 - } - }, - { - "id": "jaspionjader/Kosmos-EVAA-v9-TitanFusion-Mix-8B", - "name": "Kosmos-EVAA-v9-TitanFusion-Mix-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4284, - "hfopenllm_v2/BBH": 0.554, - "hfopenllm_v2/MATH Level 5": 0.1148, - "hfopenllm_v2/GPQA": 0.2878, - "hfopenllm_v2/MUSR": 0.4354, - "hfopenllm_v2/MMLU-PRO": 0.3836 - } - }, - { - "id": "jaspionjader/Kosmos-VENN-8B", - "name": "Kosmos-VENN-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4332, - "hfopenllm_v2/BBH": 0.5318, - "hfopenllm_v2/MATH Level 5": 0.1412, - "hfopenllm_v2/GPQA": 0.2928, - "hfopenllm_v2/MUSR": 0.4211, - "hfopenllm_v2/MMLU-PRO": 0.3801 - } - }, - { - "id": "jaspionjader/kstc-1-8b", - "name": "kstc-1-8b", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4643, - "hfopenllm_v2/BBH": 0.5209, - "hfopenllm_v2/MATH Level 5": 0.1171, - "hfopenllm_v2/GPQA": 0.3171, - "hfopenllm_v2/MUSR": 0.4158, - "hfopenllm_v2/MMLU-PRO": 0.3892 - } - }, - { - "id": "jaspionjader/kstc-11-8b", - "name": "kstc-11-8b", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4757, - "hfopenllm_v2/BBH": 0.5189, - "hfopenllm_v2/MATH Level 5": 0.1201, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.4118, - "hfopenllm_v2/MMLU-PRO": 0.3879 - } - }, - { - "id": "jaspionjader/kstc-4-8b", - "name": "kstc-4-8b", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.477, - "hfopenllm_v2/BBH": 0.5216, - "hfopenllm_v2/MATH Level 5": 0.1239, - "hfopenllm_v2/GPQA": 0.3037, - "hfopenllm_v2/MUSR": 0.4118, - "hfopenllm_v2/MMLU-PRO": 0.3869 - } - }, - { - "id": "jaspionjader/kstc-5-8b", - "name": "kstc-5-8b", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4721, - "hfopenllm_v2/BBH": 0.5211, - "hfopenllm_v2/MATH Level 5": 0.1299, - "hfopenllm_v2/GPQA": 0.3154, - "hfopenllm_v2/MUSR": 0.4224, - "hfopenllm_v2/MMLU-PRO": 0.3892 - } - }, - { - "id": "jaspionjader/kstc-6-8b", - "name": "kstc-6-8b", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4944, - "hfopenllm_v2/BBH": 0.5231, - "hfopenllm_v2/MATH Level 5": 0.1246, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.4105, - "hfopenllm_v2/MMLU-PRO": 0.3857 - } - }, - { - "id": "jaspionjader/kstc-8-8b", - "name": "kstc-8-8b", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.491, - "hfopenllm_v2/BBH": 0.5239, - "hfopenllm_v2/MATH Level 5": 0.1307, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.4211, - "hfopenllm_v2/MMLU-PRO": 0.3889 - } - }, - { - "id": "jaspionjader/kstc-9-8b", - "name": "kstc-9-8b", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4861, - "hfopenllm_v2/BBH": 0.5238, - "hfopenllm_v2/MATH Level 5": 0.136, - "hfopenllm_v2/GPQA": 0.3012, - "hfopenllm_v2/MUSR": 0.4118, - "hfopenllm_v2/MMLU-PRO": 0.3872 - } - }, - { - "id": "jaspionjader/PRP-Kosmos-EVAA-8B", - "name": "PRP-Kosmos-EVAA-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3633, - "hfopenllm_v2/BBH": 0.5237, - "hfopenllm_v2/MATH Level 5": 0.0959, - "hfopenllm_v2/GPQA": 0.3096, - "hfopenllm_v2/MUSR": 0.425, - "hfopenllm_v2/MMLU-PRO": 0.3766 - } - }, - { - "id": "jaspionjader/PRP-Kosmos-EVAA-light-8B", - "name": "PRP-Kosmos-EVAA-light-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4321, - "hfopenllm_v2/BBH": 0.5275, - "hfopenllm_v2/MATH Level 5": 0.1103, - "hfopenllm_v2/GPQA": 0.3221, - "hfopenllm_v2/MUSR": 0.4235, - "hfopenllm_v2/MMLU-PRO": 0.3631 - } - }, - { - "id": "jaspionjader/slu-10", - "name": "slu-10", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.436, - "hfopenllm_v2/BBH": 0.5096, - "hfopenllm_v2/MATH Level 5": 0.0974, - "hfopenllm_v2/GPQA": 0.3138, - "hfopenllm_v2/MUSR": 0.392, - "hfopenllm_v2/MMLU-PRO": 0.3664 - } - }, - { - "id": "jaspionjader/slu-11", - "name": "slu-11", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3725, - "hfopenllm_v2/BBH": 0.489, - "hfopenllm_v2/MATH Level 5": 0.0559, - "hfopenllm_v2/GPQA": 0.3037, - "hfopenllm_v2/MUSR": 0.3919, - "hfopenllm_v2/MMLU-PRO": 0.3382 - } - }, - { - "id": "jaspionjader/slu-13", - "name": "slu-13", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4378, - "hfopenllm_v2/BBH": 0.5097, - "hfopenllm_v2/MATH Level 5": 0.0808, - "hfopenllm_v2/GPQA": 0.3079, - "hfopenllm_v2/MUSR": 0.3814, - "hfopenllm_v2/MMLU-PRO": 0.358 - } - }, - { - "id": "jaspionjader/slu-14", - "name": "slu-14", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4107, - "hfopenllm_v2/BBH": 0.5089, - "hfopenllm_v2/MATH Level 5": 0.0974, - "hfopenllm_v2/GPQA": 0.3079, - "hfopenllm_v2/MUSR": 0.396, - "hfopenllm_v2/MMLU-PRO": 0.3627 - } - }, - { - "id": "jaspionjader/slu-17", - "name": "slu-17", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4217, - "hfopenllm_v2/BBH": 0.5071, - "hfopenllm_v2/MATH Level 5": 0.0853, - "hfopenllm_v2/GPQA": 0.3087, - "hfopenllm_v2/MUSR": 0.3761, - "hfopenllm_v2/MMLU-PRO": 0.3619 - } - }, - { - "id": "jaspionjader/slu-2", - "name": "slu-2", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4016, - "hfopenllm_v2/BBH": 0.5008, - "hfopenllm_v2/MATH Level 5": 0.0634, - "hfopenllm_v2/GPQA": 0.2987, - "hfopenllm_v2/MUSR": 0.3959, - "hfopenllm_v2/MMLU-PRO": 0.3506 - } - }, - { - "id": "jaspionjader/slu-20", - "name": "slu-20", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4393, - "hfopenllm_v2/BBH": 0.5061, - "hfopenllm_v2/MATH Level 5": 0.0869, - "hfopenllm_v2/GPQA": 0.3087, - "hfopenllm_v2/MUSR": 0.3933, - "hfopenllm_v2/MMLU-PRO": 0.3665 - } - }, - { - "id": "jaspionjader/slu-22", - "name": "slu-22", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4321, - "hfopenllm_v2/BBH": 0.5082, - "hfopenllm_v2/MATH Level 5": 0.0793, - "hfopenllm_v2/GPQA": 0.3163, - "hfopenllm_v2/MUSR": 0.3893, - "hfopenllm_v2/MMLU-PRO": 0.365 - } - }, - { - "id": "jaspionjader/slu-23", - "name": "slu-23", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4478, - "hfopenllm_v2/BBH": 0.5132, - "hfopenllm_v2/MATH Level 5": 0.0944, - "hfopenllm_v2/GPQA": 0.3045, - "hfopenllm_v2/MUSR": 0.4092, - "hfopenllm_v2/MMLU-PRO": 0.3725 - } - }, - { - "id": "jaspionjader/slu-25", - "name": "slu-25", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.45, - "hfopenllm_v2/BBH": 0.5095, - "hfopenllm_v2/MATH Level 5": 0.0838, - "hfopenllm_v2/GPQA": 0.3087, - "hfopenllm_v2/MUSR": 0.3946, - "hfopenllm_v2/MMLU-PRO": 0.3684 - } - }, - { - "id": "jaspionjader/slu-29", - "name": "slu-29", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4431, - "hfopenllm_v2/BBH": 0.5096, - "hfopenllm_v2/MATH Level 5": 0.0869, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.3933, - "hfopenllm_v2/MMLU-PRO": 0.3669 - } - }, - { - "id": "jaspionjader/slu-32", - "name": "slu-32", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4516, - "hfopenllm_v2/BBH": 0.5167, - "hfopenllm_v2/MATH Level 5": 0.1073, - "hfopenllm_v2/GPQA": 0.302, - "hfopenllm_v2/MUSR": 0.4039, - "hfopenllm_v2/MMLU-PRO": 0.3766 - } - }, - { - "id": "jaspionjader/slu-33", - "name": "slu-33", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4457, - "hfopenllm_v2/BBH": 0.5081, - "hfopenllm_v2/MATH Level 5": 0.0997, - "hfopenllm_v2/GPQA": 0.3121, - "hfopenllm_v2/MUSR": 0.3867, - "hfopenllm_v2/MMLU-PRO": 0.3679 - } - }, - { - "id": "jaspionjader/slu-34", - "name": "slu-34", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4351, - "hfopenllm_v2/BBH": 0.5077, - "hfopenllm_v2/MATH Level 5": 0.0997, - "hfopenllm_v2/GPQA": 0.3079, - "hfopenllm_v2/MUSR": 0.388, - "hfopenllm_v2/MMLU-PRO": 0.372 - } - }, - { - "id": "jaspionjader/slu-35", - "name": "slu-35", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4242, - "hfopenllm_v2/BBH": 0.5103, - "hfopenllm_v2/MATH Level 5": 0.1012, - "hfopenllm_v2/GPQA": 0.3121, - "hfopenllm_v2/MUSR": 0.3946, - "hfopenllm_v2/MMLU-PRO": 0.3676 - } - }, - { - "id": "jaspionjader/slu-36", - "name": "slu-36", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4518, - "hfopenllm_v2/BBH": 0.5087, - "hfopenllm_v2/MATH Level 5": 0.0906, - "hfopenllm_v2/GPQA": 0.3121, - "hfopenllm_v2/MUSR": 0.3933, - "hfopenllm_v2/MMLU-PRO": 0.3711 - } - }, - { - "id": "jaspionjader/slu-37", - "name": "slu-37", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4534, - "hfopenllm_v2/BBH": 0.51, - "hfopenllm_v2/MATH Level 5": 0.0974, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.3946, - "hfopenllm_v2/MMLU-PRO": 0.3695 - } - }, - { - "id": "jaspionjader/slu-6", - "name": "slu-6", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4117, - "hfopenllm_v2/BBH": 0.5099, - "hfopenllm_v2/MATH Level 5": 0.0944, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.4066, - "hfopenllm_v2/MMLU-PRO": 0.3611 - } - }, - { - "id": "jaspionjader/slu-mix-1", - "name": "slu-mix-1", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4569, - "hfopenllm_v2/BBH": 0.524, - "hfopenllm_v2/MATH Level 5": 0.1118, - "hfopenllm_v2/GPQA": 0.3003, - "hfopenllm_v2/MUSR": 0.4277, - "hfopenllm_v2/MMLU-PRO": 0.393 - } - }, - { - "id": "jaspionjader/sof-1", - "name": "sof-1", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4314, - "hfopenllm_v2/BBH": 0.501, - "hfopenllm_v2/MATH Level 5": 0.114, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.4082, - "hfopenllm_v2/MMLU-PRO": 0.3674 - } - }, - { - "id": "jaspionjader/sof-10", - "name": "sof-10", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4648, - "hfopenllm_v2/BBH": 0.5197, - "hfopenllm_v2/MATH Level 5": 0.1239, - "hfopenllm_v2/GPQA": 0.3062, - "hfopenllm_v2/MUSR": 0.4091, - "hfopenllm_v2/MMLU-PRO": 0.3874 - } - }, - { - "id": "jaspionjader/sof-3", - "name": "sof-3", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4637, - "hfopenllm_v2/BBH": 0.5206, - "hfopenllm_v2/MATH Level 5": 0.1276, - "hfopenllm_v2/GPQA": 0.297, - "hfopenllm_v2/MUSR": 0.4131, - "hfopenllm_v2/MMLU-PRO": 0.3812 - } - }, - { - "id": "jaspionjader/sof-6", - "name": "sof-6", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4354, - "hfopenllm_v2/BBH": 0.5209, - "hfopenllm_v2/MATH Level 5": 0.1299, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.4171, - "hfopenllm_v2/MMLU-PRO": 0.3844 - } - }, - { - "id": "jaspionjader/test-10", - "name": "test-10", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4578, - "hfopenllm_v2/BBH": 0.5316, - "hfopenllm_v2/MATH Level 5": 0.114, - "hfopenllm_v2/GPQA": 0.3196, - "hfopenllm_v2/MUSR": 0.4251, - "hfopenllm_v2/MMLU-PRO": 0.3936 - } - }, - { - "id": "jaspionjader/test-11", - "name": "test-11", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4541, - "hfopenllm_v2/BBH": 0.535, - "hfopenllm_v2/MATH Level 5": 0.1201, - "hfopenllm_v2/GPQA": 0.3238, - "hfopenllm_v2/MUSR": 0.429, - "hfopenllm_v2/MMLU-PRO": 0.3939 - } - }, - { - "id": "jaspionjader/test-12", - "name": "test-12", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4368, - "hfopenllm_v2/BBH": 0.5347, - "hfopenllm_v2/MATH Level 5": 0.108, - "hfopenllm_v2/GPQA": 0.3188, - "hfopenllm_v2/MUSR": 0.425, - "hfopenllm_v2/MMLU-PRO": 0.3935 - } - }, - { - "id": "jaspionjader/test-13", - "name": "test-13", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4581, - "hfopenllm_v2/BBH": 0.5318, - "hfopenllm_v2/MATH Level 5": 0.1057, - "hfopenllm_v2/GPQA": 0.3163, - "hfopenllm_v2/MUSR": 0.4264, - "hfopenllm_v2/MMLU-PRO": 0.3935 - } - }, - { - "id": "jaspionjader/test-14", - "name": "test-14", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4444, - "hfopenllm_v2/BBH": 0.5323, - "hfopenllm_v2/MATH Level 5": 0.1103, - "hfopenllm_v2/GPQA": 0.3146, - "hfopenllm_v2/MUSR": 0.4317, - "hfopenllm_v2/MMLU-PRO": 0.393 - } - }, - { - "id": "jaspionjader/test-15", - "name": "test-15", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4365, - "hfopenllm_v2/BBH": 0.5328, - "hfopenllm_v2/MATH Level 5": 0.1118, - "hfopenllm_v2/GPQA": 0.3121, - "hfopenllm_v2/MUSR": 0.4264, - "hfopenllm_v2/MMLU-PRO": 0.393 - } - }, - { - "id": "jaspionjader/test-16", - "name": "test-16", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4599, - "hfopenllm_v2/BBH": 0.533, - "hfopenllm_v2/MATH Level 5": 0.1095, - "hfopenllm_v2/GPQA": 0.3138, - "hfopenllm_v2/MUSR": 0.4225, - "hfopenllm_v2/MMLU-PRO": 0.393 - } - }, - { - "id": "jaspionjader/test-17", - "name": "test-17", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4267, - "hfopenllm_v2/BBH": 0.5329, - "hfopenllm_v2/MATH Level 5": 0.1103, - "hfopenllm_v2/GPQA": 0.3129, - "hfopenllm_v2/MUSR": 0.429, - "hfopenllm_v2/MMLU-PRO": 0.3929 - } - }, - { - "id": "jaspionjader/test-18", - "name": "test-18", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4392, - "hfopenllm_v2/BBH": 0.5317, - "hfopenllm_v2/MATH Level 5": 0.1148, - "hfopenllm_v2/GPQA": 0.3121, - "hfopenllm_v2/MUSR": 0.4251, - "hfopenllm_v2/MMLU-PRO": 0.393 - } - }, - { - "id": "jaspionjader/test-19", - "name": "test-19", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4401, - "hfopenllm_v2/BBH": 0.5319, - "hfopenllm_v2/MATH Level 5": 0.1095, - "hfopenllm_v2/GPQA": 0.3096, - "hfopenllm_v2/MUSR": 0.4264, - "hfopenllm_v2/MMLU-PRO": 0.3929 - } - }, - { - "id": "jaspionjader/test-20", - "name": "test-20", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4529, - "hfopenllm_v2/BBH": 0.5327, - "hfopenllm_v2/MATH Level 5": 0.1118, - "hfopenllm_v2/GPQA": 0.3138, - "hfopenllm_v2/MUSR": 0.4251, - "hfopenllm_v2/MMLU-PRO": 0.392 - } - }, - { - "id": "jaspionjader/TSN-Kosmos-EVAA-8B", - "name": "TSN-Kosmos-EVAA-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4903, - "hfopenllm_v2/BBH": 0.5347, - "hfopenllm_v2/MATH Level 5": 0.145, - "hfopenllm_v2/GPQA": 0.3205, - "hfopenllm_v2/MUSR": 0.4173, - "hfopenllm_v2/MMLU-PRO": 0.3831 - } - }, - { - "id": "jaspionjader/TSN-Kosmos-EVAA-v2-8B", - "name": "TSN-Kosmos-EVAA-v2-8B", - "developer": "jaspionjader", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4667, - "hfopenllm_v2/BBH": 0.5343, - "hfopenllm_v2/MATH Level 5": 0.108, - "hfopenllm_v2/GPQA": 0.3221, - "hfopenllm_v2/MUSR": 0.4186, - "hfopenllm_v2/MMLU-PRO": 0.3762 - } - }, - { - "id": "jayasuryajsk/Qwen2.5-3B-reasoner", - "name": "Qwen2.5-3B-reasoner", - "developer": "jayasuryajsk", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.416, - "hfopenllm_v2/BBH": 0.4651, - "hfopenllm_v2/MATH Level 5": 0.2085, - "hfopenllm_v2/GPQA": 0.302, - "hfopenllm_v2/MUSR": 0.4123, - "hfopenllm_v2/MMLU-PRO": 0.3482 - } - }, - { - "id": "JayHyeon/Qwen-0.5B-DPO-1epoch", - "name": "Qwen-0.5B-DPO-1epoch", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2647, - "hfopenllm_v2/BBH": 0.3191, - "hfopenllm_v2/MATH Level 5": 0.0287, - "hfopenllm_v2/GPQA": 0.2525, - "hfopenllm_v2/MUSR": 0.3352, - "hfopenllm_v2/MMLU-PRO": 0.1558 - } - }, - { - "id": "JayHyeon/Qwen-0.5B-DPO-5epoch", - "name": "Qwen-0.5B-DPO-5epoch", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.257, - "hfopenllm_v2/BBH": 0.3112, - "hfopenllm_v2/MATH Level 5": 0.04, - "hfopenllm_v2/GPQA": 0.2433, - "hfopenllm_v2/MUSR": 0.338, - "hfopenllm_v2/MMLU-PRO": 0.1533 - } - }, - { - "id": "JayHyeon/Qwen-0.5B-eDPO-1epoch", - "name": "Qwen-0.5B-eDPO-1epoch", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2623, - "hfopenllm_v2/BBH": 0.3181, - "hfopenllm_v2/MATH Level 5": 0.0347, - "hfopenllm_v2/GPQA": 0.2424, - "hfopenllm_v2/MUSR": 0.3327, - "hfopenllm_v2/MMLU-PRO": 0.1553 - } - }, - { - "id": "JayHyeon/Qwen-0.5B-eDPO-5epoch", - "name": "Qwen-0.5B-eDPO-5epoch", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2477, - "hfopenllm_v2/BBH": 0.3096, - "hfopenllm_v2/MATH Level 5": 0.0234, - "hfopenllm_v2/GPQA": 0.2492, - "hfopenllm_v2/MUSR": 0.3326, - "hfopenllm_v2/MMLU-PRO": 0.1523 - } - }, - { - "id": "JayHyeon/Qwen-0.5B-IRPO-1epoch", - "name": "Qwen-0.5B-IRPO-1epoch", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2589, - "hfopenllm_v2/BBH": 0.3164, - "hfopenllm_v2/MATH Level 5": 0.0317, - "hfopenllm_v2/GPQA": 0.2466, - "hfopenllm_v2/MUSR": 0.3286, - "hfopenllm_v2/MMLU-PRO": 0.15 - } - }, - { - "id": "JayHyeon/Qwen-0.5B-IRPO-5epoch", - "name": "Qwen-0.5B-IRPO-5epoch", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2487, - "hfopenllm_v2/BBH": 0.3189, - "hfopenllm_v2/MATH Level 5": 0.0325, - "hfopenllm_v2/GPQA": 0.2399, - "hfopenllm_v2/MUSR": 0.3287, - "hfopenllm_v2/MMLU-PRO": 0.1507 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-Instruct-SFT", - "name": "Qwen2.5-0.5B-Instruct-SFT", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2768, - "hfopenllm_v2/BBH": 0.3254, - "hfopenllm_v2/MATH Level 5": 0.0393, - "hfopenllm_v2/GPQA": 0.2827, - "hfopenllm_v2/MUSR": 0.3342, - "hfopenllm_v2/MMLU-PRO": 0.152 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-Instruct-SFT-DPO-1epoch_v1", - "name": "Qwen2.5-0.5B-Instruct-SFT-DPO-1epoch_v1", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2469, - "hfopenllm_v2/BBH": 0.326, - "hfopenllm_v2/MATH Level 5": 0.065, - "hfopenllm_v2/GPQA": 0.2727, - "hfopenllm_v2/MUSR": 0.3434, - "hfopenllm_v2/MMLU-PRO": 0.1575 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-Instruct-SFT-IRPO-1epoch_v1", - "name": "Qwen2.5-0.5B-Instruct-SFT-IRPO-1epoch_v1", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2606, - "hfopenllm_v2/BBH": 0.3308, - "hfopenllm_v2/MATH Level 5": 0.0498, - "hfopenllm_v2/GPQA": 0.2802, - "hfopenllm_v2/MUSR": 0.3288, - "hfopenllm_v2/MMLU-PRO": 0.1626 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-Instruct-SFT-MDPO-1epoch_v1", - "name": "Qwen2.5-0.5B-Instruct-SFT-MDPO-1epoch_v1", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2529, - "hfopenllm_v2/BBH": 0.3262, - "hfopenllm_v2/MATH Level 5": 0.0566, - "hfopenllm_v2/GPQA": 0.2685, - "hfopenllm_v2/MUSR": 0.3301, - "hfopenllm_v2/MMLU-PRO": 0.1576 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT", - "name": "Qwen2.5-0.5B-SFT", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1964, - "hfopenllm_v2/BBH": 0.3121, - "hfopenllm_v2/MATH Level 5": 0.0272, - "hfopenllm_v2/GPQA": 0.2785, - "hfopenllm_v2/MUSR": 0.3394, - "hfopenllm_v2/MMLU-PRO": 0.1673 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-1e-4", - "name": "Qwen2.5-0.5B-SFT-1e-4", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.202, - "hfopenllm_v2/BBH": 0.3017, - "hfopenllm_v2/MATH Level 5": 0.0189, - "hfopenllm_v2/GPQA": 0.2508, - "hfopenllm_v2/MUSR": 0.3446, - "hfopenllm_v2/MMLU-PRO": 0.1619 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-1e-4-2ep", - "name": "Qwen2.5-0.5B-SFT-1e-4-2ep", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.214, - "hfopenllm_v2/BBH": 0.3172, - "hfopenllm_v2/MATH Level 5": 0.0264, - "hfopenllm_v2/GPQA": 0.2466, - "hfopenllm_v2/MUSR": 0.3473, - "hfopenllm_v2/MMLU-PRO": 0.1537 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-1e-4-3ep", - "name": "Qwen2.5-0.5B-SFT-1e-4-3ep", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2257, - "hfopenllm_v2/BBH": 0.3064, - "hfopenllm_v2/MATH Level 5": 0.0264, - "hfopenllm_v2/GPQA": 0.2483, - "hfopenllm_v2/MUSR": 0.3661, - "hfopenllm_v2/MMLU-PRO": 0.1532 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-1e-4-5ep", - "name": "Qwen2.5-0.5B-SFT-1e-4-5ep", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1987, - "hfopenllm_v2/BBH": 0.3104, - "hfopenllm_v2/MATH Level 5": 0.0196, - "hfopenllm_v2/GPQA": 0.2534, - "hfopenllm_v2/MUSR": 0.3407, - "hfopenllm_v2/MMLU-PRO": 0.1558 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-1e-5", - "name": "Qwen2.5-0.5B-SFT-1e-5", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1986, - "hfopenllm_v2/BBH": 0.314, - "hfopenllm_v2/MATH Level 5": 0.0378, - "hfopenllm_v2/GPQA": 0.2685, - "hfopenllm_v2/MUSR": 0.346, - "hfopenllm_v2/MMLU-PRO": 0.1698 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-1e-5-2ep", - "name": "Qwen2.5-0.5B-SFT-1e-5-2ep", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1971, - "hfopenllm_v2/BBH": 0.3225, - "hfopenllm_v2/MATH Level 5": 0.0529, - "hfopenllm_v2/GPQA": 0.2693, - "hfopenllm_v2/MUSR": 0.3368, - "hfopenllm_v2/MMLU-PRO": 0.1651 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-1e-5-3ep", - "name": "Qwen2.5-0.5B-SFT-1e-5-3ep", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2241, - "hfopenllm_v2/BBH": 0.3247, - "hfopenllm_v2/MATH Level 5": 0.0536, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.3353, - "hfopenllm_v2/MMLU-PRO": 0.1689 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-1e-5-5ep", - "name": "Qwen2.5-0.5B-SFT-1e-5-5ep", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2292, - "hfopenllm_v2/BBH": 0.3259, - "hfopenllm_v2/MATH Level 5": 0.0521, - "hfopenllm_v2/GPQA": 0.2794, - "hfopenllm_v2/MUSR": 0.3235, - "hfopenllm_v2/MMLU-PRO": 0.1688 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-4", - "name": "Qwen2.5-0.5B-SFT-2e-4", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2034, - "hfopenllm_v2/BBH": 0.2936, - "hfopenllm_v2/MATH Level 5": 0.0242, - "hfopenllm_v2/GPQA": 0.2576, - "hfopenllm_v2/MUSR": 0.3434, - "hfopenllm_v2/MMLU-PRO": 0.1413 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-4-2ep", - "name": "Qwen2.5-0.5B-SFT-2e-4-2ep", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1831, - "hfopenllm_v2/BBH": 0.2984, - "hfopenllm_v2/MATH Level 5": 0.0249, - "hfopenllm_v2/GPQA": 0.2424, - "hfopenllm_v2/MUSR": 0.3568, - "hfopenllm_v2/MMLU-PRO": 0.1484 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-4-3ep", - "name": "Qwen2.5-0.5B-SFT-2e-4-3ep", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.199, - "hfopenllm_v2/BBH": 0.311, - "hfopenllm_v2/MATH Level 5": 0.0151, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.3449, - "hfopenllm_v2/MMLU-PRO": 0.1416 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-4-5ep", - "name": "Qwen2.5-0.5B-SFT-2e-4-5ep", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1897, - "hfopenllm_v2/BBH": 0.2936, - "hfopenllm_v2/MATH Level 5": 0.0181, - "hfopenllm_v2/GPQA": 0.2693, - "hfopenllm_v2/MUSR": 0.3874, - "hfopenllm_v2/MMLU-PRO": 0.1336 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5", - "name": "Qwen2.5-0.5B-SFT-2e-5", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2068, - "hfopenllm_v2/BBH": 0.3204, - "hfopenllm_v2/MATH Level 5": 0.037, - "hfopenllm_v2/GPQA": 0.2693, - "hfopenllm_v2/MUSR": 0.3487, - "hfopenllm_v2/MMLU-PRO": 0.1678 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2201, - "hfopenllm_v2/BBH": 0.3217, - "hfopenllm_v2/MATH Level 5": 0.0408, - "hfopenllm_v2/GPQA": 0.2777, - "hfopenllm_v2/MUSR": 0.3367, - "hfopenllm_v2/MMLU-PRO": 0.171 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_1e-6-1ep_0alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_1e-6-1ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2542, - "hfopenllm_v2/BBH": 0.3167, - "hfopenllm_v2/MATH Level 5": 0.0408, - "hfopenllm_v2/GPQA": 0.2718, - "hfopenllm_v2/MUSR": 0.3289, - "hfopenllm_v2/MMLU-PRO": 0.158 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_1e-6-2ep_0alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_1e-6-2ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2451, - "hfopenllm_v2/BBH": 0.316, - "hfopenllm_v2/MATH Level 5": 0.0408, - "hfopenllm_v2/GPQA": 0.2743, - "hfopenllm_v2/MUSR": 0.3302, - "hfopenllm_v2/MMLU-PRO": 0.1561 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_1e-6-3ep_0alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_1e-6-3ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2557, - "hfopenllm_v2/BBH": 0.3142, - "hfopenllm_v2/MATH Level 5": 0.04, - "hfopenllm_v2/GPQA": 0.2743, - "hfopenllm_v2/MUSR": 0.3315, - "hfopenllm_v2/MMLU-PRO": 0.1575 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_1e-7-2ep_0alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_1e-7-2ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2605, - "hfopenllm_v2/BBH": 0.3167, - "hfopenllm_v2/MATH Level 5": 0.0363, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.3341, - "hfopenllm_v2/MMLU-PRO": 0.1577 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_1e-7-3ep_0alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_1e-7-3ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2578, - "hfopenllm_v2/BBH": 0.3173, - "hfopenllm_v2/MATH Level 5": 0.0355, - "hfopenllm_v2/GPQA": 0.2634, - "hfopenllm_v2/MUSR": 0.3288, - "hfopenllm_v2/MMLU-PRO": 0.1583 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_2e-6-1ep_0alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_2e-6-1ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2335, - "hfopenllm_v2/BBH": 0.3198, - "hfopenllm_v2/MATH Level 5": 0.0385, - "hfopenllm_v2/GPQA": 0.2752, - "hfopenllm_v2/MUSR": 0.3276, - "hfopenllm_v2/MMLU-PRO": 0.1581 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_2e-6-2ep_0alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_2e-6-2ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2472, - "hfopenllm_v2/BBH": 0.3226, - "hfopenllm_v2/MATH Level 5": 0.0506, - "hfopenllm_v2/GPQA": 0.276, - "hfopenllm_v2/MUSR": 0.3262, - "hfopenllm_v2/MMLU-PRO": 0.1538 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_2e-6-3ep_0alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_2e-6-3ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2474, - "hfopenllm_v2/BBH": 0.3229, - "hfopenllm_v2/MATH Level 5": 0.0415, - "hfopenllm_v2/GPQA": 0.2727, - "hfopenllm_v2/MUSR": 0.3275, - "hfopenllm_v2/MMLU-PRO": 0.1539 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_3e-6-1ep_0alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_3e-6-1ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2403, - "hfopenllm_v2/BBH": 0.3245, - "hfopenllm_v2/MATH Level 5": 0.0431, - "hfopenllm_v2/GPQA": 0.2819, - "hfopenllm_v2/MUSR": 0.3262, - "hfopenllm_v2/MMLU-PRO": 0.1573 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_3e-6-2ep_0alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_3e-6-2ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2368, - "hfopenllm_v2/BBH": 0.3224, - "hfopenllm_v2/MATH Level 5": 0.0461, - "hfopenllm_v2/GPQA": 0.2743, - "hfopenllm_v2/MUSR": 0.3355, - "hfopenllm_v2/MMLU-PRO": 0.1516 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_3e-6-3ep_0alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_3e-6-3ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2372, - "hfopenllm_v2/BBH": 0.3248, - "hfopenllm_v2/MATH Level 5": 0.0476, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.3394, - "hfopenllm_v2/MMLU-PRO": 0.155 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_3e-7-3ep_0alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_3e-7-3ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2499, - "hfopenllm_v2/BBH": 0.3181, - "hfopenllm_v2/MATH Level 5": 0.0415, - "hfopenllm_v2/GPQA": 0.2651, - "hfopenllm_v2/MUSR": 0.3288, - "hfopenllm_v2/MMLU-PRO": 0.1574 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_5e-6-1ep_0alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_5e-6-1ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2381, - "hfopenllm_v2/BBH": 0.3242, - "hfopenllm_v2/MATH Level 5": 0.0498, - "hfopenllm_v2/GPQA": 0.2743, - "hfopenllm_v2/MUSR": 0.3328, - "hfopenllm_v2/MMLU-PRO": 0.1572 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_5e-6-2ep_0alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_5e-6-2ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2421, - "hfopenllm_v2/BBH": 0.3225, - "hfopenllm_v2/MATH Level 5": 0.04, - "hfopenllm_v2/GPQA": 0.2802, - "hfopenllm_v2/MUSR": 0.3408, - "hfopenllm_v2/MMLU-PRO": 0.1496 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_5e-6-3ep_0alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_5e-6-3ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2381, - "hfopenllm_v2/BBH": 0.3265, - "hfopenllm_v2/MATH Level 5": 0.0446, - "hfopenllm_v2/GPQA": 0.276, - "hfopenllm_v2/MUSR": 0.3408, - "hfopenllm_v2/MMLU-PRO": 0.1499 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_5e-7_1ep_0alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_5e-7_1ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2526, - "hfopenllm_v2/BBH": 0.3177, - "hfopenllm_v2/MATH Level 5": 0.0438, - "hfopenllm_v2/GPQA": 0.2735, - "hfopenllm_v2/MUSR": 0.3342, - "hfopenllm_v2/MMLU-PRO": 0.1572 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_5e-7_2ep_0alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_5e-7_2ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2457, - "hfopenllm_v2/BBH": 0.316, - "hfopenllm_v2/MATH Level 5": 0.0446, - "hfopenllm_v2/GPQA": 0.2727, - "hfopenllm_v2/MUSR": 0.3302, - "hfopenllm_v2/MMLU-PRO": 0.1572 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_5e-7_3ep_0alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_5e-7_3ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2442, - "hfopenllm_v2/BBH": 0.3194, - "hfopenllm_v2/MATH Level 5": 0.0483, - "hfopenllm_v2/GPQA": 0.2735, - "hfopenllm_v2/MUSR": 0.3315, - "hfopenllm_v2/MMLU-PRO": 0.1567 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_7e-7_1ep_0alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_7e-7_1ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2604, - "hfopenllm_v2/BBH": 0.3178, - "hfopenllm_v2/MATH Level 5": 0.0355, - "hfopenllm_v2/GPQA": 0.276, - "hfopenllm_v2/MUSR": 0.3288, - "hfopenllm_v2/MMLU-PRO": 0.1567 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_7e-7_2ep_0alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_7e-7_2ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.249, - "hfopenllm_v2/BBH": 0.3173, - "hfopenllm_v2/MATH Level 5": 0.0393, - "hfopenllm_v2/GPQA": 0.271, - "hfopenllm_v2/MUSR": 0.3302, - "hfopenllm_v2/MMLU-PRO": 0.1569 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_7e-7_3ep_0alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_7e-7_3ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2604, - "hfopenllm_v2/BBH": 0.315, - "hfopenllm_v2/MATH Level 5": 0.0378, - "hfopenllm_v2/GPQA": 0.2743, - "hfopenllm_v2/MUSR": 0.3342, - "hfopenllm_v2/MMLU-PRO": 0.1566 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-DPOP_3e-7-3ep_0alp_5lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-DPOP_3e-7-3ep_0alp_5lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2411, - "hfopenllm_v2/BBH": 0.3167, - "hfopenllm_v2/MATH Level 5": 0.0347, - "hfopenllm_v2/GPQA": 0.271, - "hfopenllm_v2/MUSR": 0.3301, - "hfopenllm_v2/MMLU-PRO": 0.1562 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-DPOP_5e-6-1ep_0alp_5lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-DPOP_5e-6-1ep_0alp_5lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2369, - "hfopenllm_v2/BBH": 0.326, - "hfopenllm_v2/MATH Level 5": 0.0453, - "hfopenllm_v2/GPQA": 0.276, - "hfopenllm_v2/MUSR": 0.3355, - "hfopenllm_v2/MMLU-PRO": 0.157 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-DPOP_5e-6-2ep_0alp_5lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-DPOP_5e-6-2ep_0alp_5lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2262, - "hfopenllm_v2/BBH": 0.3262, - "hfopenllm_v2/MATH Level 5": 0.0347, - "hfopenllm_v2/GPQA": 0.2794, - "hfopenllm_v2/MUSR": 0.3408, - "hfopenllm_v2/MMLU-PRO": 0.1541 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-DPOP_5e-6-3ep_0alp_5lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-DPOP_5e-6-3ep_0alp_5lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2508, - "hfopenllm_v2/BBH": 0.3199, - "hfopenllm_v2/MATH Level 5": 0.0408, - "hfopenllm_v2/GPQA": 0.276, - "hfopenllm_v2/MUSR": 0.3355, - "hfopenllm_v2/MMLU-PRO": 0.1555 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-DPOP_5e-7-1ep_0alp_5lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-DPOP_5e-7-1ep_0alp_5lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.239, - "hfopenllm_v2/BBH": 0.3182, - "hfopenllm_v2/MATH Level 5": 0.04, - "hfopenllm_v2/GPQA": 0.2676, - "hfopenllm_v2/MUSR": 0.3328, - "hfopenllm_v2/MMLU-PRO": 0.156 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-DPOP_5e-7-2ep_0alp_5lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-DPOP_5e-7-2ep_0alp_5lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2423, - "hfopenllm_v2/BBH": 0.3154, - "hfopenllm_v2/MATH Level 5": 0.0347, - "hfopenllm_v2/GPQA": 0.2676, - "hfopenllm_v2/MUSR": 0.3328, - "hfopenllm_v2/MMLU-PRO": 0.1548 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-DPOP_5e-7-3ep_0alp_5lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-DPOP_5e-7-3ep_0alp_5lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2493, - "hfopenllm_v2/BBH": 0.319, - "hfopenllm_v2/MATH Level 5": 0.0438, - "hfopenllm_v2/GPQA": 0.2651, - "hfopenllm_v2/MUSR": 0.3341, - "hfopenllm_v2/MMLU-PRO": 0.1561 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-IRPO_1e-7-1ep_1alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-IRPO_1e-7-1ep_1alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.255, - "hfopenllm_v2/BBH": 0.3211, - "hfopenllm_v2/MATH Level 5": 0.0491, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.3288, - "hfopenllm_v2/MMLU-PRO": 0.1571 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-IRPO_1e-7-2ep_1alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-IRPO_1e-7-2ep_1alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2478, - "hfopenllm_v2/BBH": 0.3198, - "hfopenllm_v2/MATH Level 5": 0.0423, - "hfopenllm_v2/GPQA": 0.2668, - "hfopenllm_v2/MUSR": 0.3315, - "hfopenllm_v2/MMLU-PRO": 0.1587 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-IRPO_1e-7-3ep_1alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-IRPO_1e-7-3ep_1alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2475, - "hfopenllm_v2/BBH": 0.3225, - "hfopenllm_v2/MATH Level 5": 0.04, - "hfopenllm_v2/GPQA": 0.271, - "hfopenllm_v2/MUSR": 0.3301, - "hfopenllm_v2/MMLU-PRO": 0.1556 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-IRPO_3e-7-3ep_1alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-IRPO_3e-7-3ep_1alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.259, - "hfopenllm_v2/BBH": 0.3185, - "hfopenllm_v2/MATH Level 5": 0.0363, - "hfopenllm_v2/GPQA": 0.2727, - "hfopenllm_v2/MUSR": 0.3275, - "hfopenllm_v2/MMLU-PRO": 0.1586 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-IRPO_5e-6-1ep_1alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-IRPO_5e-6-1ep_1alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2323, - "hfopenllm_v2/BBH": 0.3179, - "hfopenllm_v2/MATH Level 5": 0.0453, - "hfopenllm_v2/GPQA": 0.2827, - "hfopenllm_v2/MUSR": 0.3262, - "hfopenllm_v2/MMLU-PRO": 0.1548 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-IRPO_5e-6-2ep_1alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-IRPO_5e-6-2ep_1alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2315, - "hfopenllm_v2/BBH": 0.326, - "hfopenllm_v2/MATH Level 5": 0.0415, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.3383, - "hfopenllm_v2/MMLU-PRO": 0.1521 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-IRPO_5e-6-3ep_1alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-IRPO_5e-6-3ep_1alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2298, - "hfopenllm_v2/BBH": 0.332, - "hfopenllm_v2/MATH Level 5": 0.0431, - "hfopenllm_v2/GPQA": 0.2659, - "hfopenllm_v2/MUSR": 0.3329, - "hfopenllm_v2/MMLU-PRO": 0.1567 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-IRPO_5e-7-1ep_1alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-IRPO_5e-7-1ep_1alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2469, - "hfopenllm_v2/BBH": 0.3179, - "hfopenllm_v2/MATH Level 5": 0.0415, - "hfopenllm_v2/GPQA": 0.2794, - "hfopenllm_v2/MUSR": 0.3302, - "hfopenllm_v2/MMLU-PRO": 0.1575 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-IRPO_5e-7-2ep_1alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-IRPO_5e-7-2ep_1alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.252, - "hfopenllm_v2/BBH": 0.3168, - "hfopenllm_v2/MATH Level 5": 0.037, - "hfopenllm_v2/GPQA": 0.2752, - "hfopenllm_v2/MUSR": 0.3328, - "hfopenllm_v2/MMLU-PRO": 0.1576 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-IRPO_5e-7-3ep_1alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-IRPO_5e-7-3ep_1alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2666, - "hfopenllm_v2/BBH": 0.3191, - "hfopenllm_v2/MATH Level 5": 0.0347, - "hfopenllm_v2/GPQA": 0.2718, - "hfopenllm_v2/MUSR": 0.3289, - "hfopenllm_v2/MMLU-PRO": 0.1567 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_0.5_1e-7-1ep_0alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_0.5_1e-7-1ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2499, - "hfopenllm_v2/BBH": 0.3178, - "hfopenllm_v2/MATH Level 5": 0.037, - "hfopenllm_v2/GPQA": 0.2651, - "hfopenllm_v2/MUSR": 0.3341, - "hfopenllm_v2/MMLU-PRO": 0.1562 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_0.5_1e-7-2ep_0alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_0.5_1e-7-2ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2417, - "hfopenllm_v2/BBH": 0.3178, - "hfopenllm_v2/MATH Level 5": 0.04, - "hfopenllm_v2/GPQA": 0.2685, - "hfopenllm_v2/MUSR": 0.3328, - "hfopenllm_v2/MMLU-PRO": 0.1575 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_0.5_1e-7-3ep_0alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_0.5_1e-7-3ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2562, - "hfopenllm_v2/BBH": 0.319, - "hfopenllm_v2/MATH Level 5": 0.0423, - "hfopenllm_v2/GPQA": 0.2659, - "hfopenllm_v2/MUSR": 0.3341, - "hfopenllm_v2/MMLU-PRO": 0.1576 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_1e-6-3ep_0alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_1e-6-3ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2408, - "hfopenllm_v2/BBH": 0.3165, - "hfopenllm_v2/MATH Level 5": 0.0431, - "hfopenllm_v2/GPQA": 0.2735, - "hfopenllm_v2/MUSR": 0.3315, - "hfopenllm_v2/MMLU-PRO": 0.1557 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_1e-6_1ep_0alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_1e-6_1ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2481, - "hfopenllm_v2/BBH": 0.3204, - "hfopenllm_v2/MATH Level 5": 0.0476, - "hfopenllm_v2/GPQA": 0.276, - "hfopenllm_v2/MUSR": 0.3302, - "hfopenllm_v2/MMLU-PRO": 0.1592 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_1e-6_2ep_0alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_1e-6_2ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2545, - "hfopenllm_v2/BBH": 0.3186, - "hfopenllm_v2/MATH Level 5": 0.0498, - "hfopenllm_v2/GPQA": 0.2718, - "hfopenllm_v2/MUSR": 0.3289, - "hfopenllm_v2/MMLU-PRO": 0.1561 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_2e-6-3ep_0alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_2e-6-3ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.252, - "hfopenllm_v2/BBH": 0.3204, - "hfopenllm_v2/MATH Level 5": 0.0393, - "hfopenllm_v2/GPQA": 0.2727, - "hfopenllm_v2/MUSR": 0.3262, - "hfopenllm_v2/MMLU-PRO": 0.1538 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_2e-6_1ep_0alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_2e-6_1ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2315, - "hfopenllm_v2/BBH": 0.3213, - "hfopenllm_v2/MATH Level 5": 0.0453, - "hfopenllm_v2/GPQA": 0.2802, - "hfopenllm_v2/MUSR": 0.3222, - "hfopenllm_v2/MMLU-PRO": 0.1582 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_2e-6_2ep_0alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_2e-6_2ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2515, - "hfopenllm_v2/BBH": 0.3187, - "hfopenllm_v2/MATH Level 5": 0.0431, - "hfopenllm_v2/GPQA": 0.2718, - "hfopenllm_v2/MUSR": 0.3289, - "hfopenllm_v2/MMLU-PRO": 0.1539 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_3e-6-1ep_0alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_3e-6-1ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2472, - "hfopenllm_v2/BBH": 0.3213, - "hfopenllm_v2/MATH Level 5": 0.0347, - "hfopenllm_v2/GPQA": 0.2727, - "hfopenllm_v2/MUSR": 0.3262, - "hfopenllm_v2/MMLU-PRO": 0.1588 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_3e-6-2ep_0alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_3e-6-2ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.246, - "hfopenllm_v2/BBH": 0.3234, - "hfopenllm_v2/MATH Level 5": 0.0378, - "hfopenllm_v2/GPQA": 0.2794, - "hfopenllm_v2/MUSR": 0.3302, - "hfopenllm_v2/MMLU-PRO": 0.1533 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_3e-6-3ep_0alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_3e-6-3ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2524, - "hfopenllm_v2/BBH": 0.3256, - "hfopenllm_v2/MATH Level 5": 0.0536, - "hfopenllm_v2/GPQA": 0.2777, - "hfopenllm_v2/MUSR": 0.3368, - "hfopenllm_v2/MMLU-PRO": 0.1531 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_5e-6-1ep_0alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_5e-6-1ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2265, - "hfopenllm_v2/BBH": 0.3252, - "hfopenllm_v2/MATH Level 5": 0.0476, - "hfopenllm_v2/GPQA": 0.2735, - "hfopenllm_v2/MUSR": 0.3262, - "hfopenllm_v2/MMLU-PRO": 0.1568 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_5e-6-2ep_0alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_5e-6-2ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2302, - "hfopenllm_v2/BBH": 0.3224, - "hfopenllm_v2/MATH Level 5": 0.0438, - "hfopenllm_v2/GPQA": 0.2768, - "hfopenllm_v2/MUSR": 0.3408, - "hfopenllm_v2/MMLU-PRO": 0.15 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_5e-6-3ep_0alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_5e-6-3ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2524, - "hfopenllm_v2/BBH": 0.3278, - "hfopenllm_v2/MATH Level 5": 0.0408, - "hfopenllm_v2/GPQA": 0.2777, - "hfopenllm_v2/MUSR": 0.3395, - "hfopenllm_v2/MMLU-PRO": 0.1521 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_5e-7-3ep_0alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_5e-7-3ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2658, - "hfopenllm_v2/BBH": 0.3175, - "hfopenllm_v2/MATH Level 5": 0.0363, - "hfopenllm_v2/GPQA": 0.2617, - "hfopenllm_v2/MUSR": 0.3302, - "hfopenllm_v2/MMLU-PRO": 0.1575 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_5e-7_1ep_0alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_5e-7_1ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2487, - "hfopenllm_v2/BBH": 0.3189, - "hfopenllm_v2/MATH Level 5": 0.0378, - "hfopenllm_v2/GPQA": 0.2718, - "hfopenllm_v2/MUSR": 0.3275, - "hfopenllm_v2/MMLU-PRO": 0.1595 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_5e-7_2ep_0alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_5e-7_2ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.256, - "hfopenllm_v2/BBH": 0.3159, - "hfopenllm_v2/MATH Level 5": 0.0378, - "hfopenllm_v2/GPQA": 0.2768, - "hfopenllm_v2/MUSR": 0.3275, - "hfopenllm_v2/MMLU-PRO": 0.1562 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_7e-7-3ep_0alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_7e-7-3ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2499, - "hfopenllm_v2/BBH": 0.3156, - "hfopenllm_v2/MATH Level 5": 0.04, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.3302, - "hfopenllm_v2/MMLU-PRO": 0.1556 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_7e-7_1ep_0alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_7e-7_1ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2496, - "hfopenllm_v2/BBH": 0.3177, - "hfopenllm_v2/MATH Level 5": 0.0453, - "hfopenllm_v2/GPQA": 0.2626, - "hfopenllm_v2/MUSR": 0.3315, - "hfopenllm_v2/MMLU-PRO": 0.1567 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_7e-7_2ep_0alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_7e-7_2ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2515, - "hfopenllm_v2/BBH": 0.3172, - "hfopenllm_v2/MATH Level 5": 0.0438, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.3275, - "hfopenllm_v2/MMLU-PRO": 0.1553 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-3ep", - "name": "Qwen2.5-0.5B-SFT-2e-5-3ep", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2281, - "hfopenllm_v2/BBH": 0.324, - "hfopenllm_v2/MATH Level 5": 0.0453, - "hfopenllm_v2/GPQA": 0.2617, - "hfopenllm_v2/MUSR": 0.3301, - "hfopenllm_v2/MMLU-PRO": 0.1746 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-5ep", - "name": "Qwen2.5-0.5B-SFT-2e-5-5ep", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2348, - "hfopenllm_v2/BBH": 0.3308, - "hfopenllm_v2/MATH Level 5": 0.0506, - "hfopenllm_v2/GPQA": 0.2643, - "hfopenllm_v2/MUSR": 0.3409, - "hfopenllm_v2/MMLU-PRO": 0.1695 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-5ep-MDPO_5e-7_3ep_0alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-5ep-MDPO_5e-7_3ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2526, - "hfopenllm_v2/BBH": 0.3238, - "hfopenllm_v2/MATH Level 5": 0.0393, - "hfopenllm_v2/GPQA": 0.2676, - "hfopenllm_v2/MUSR": 0.3528, - "hfopenllm_v2/MMLU-PRO": 0.1574 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-5ep-MDPO_5e-7_3ep_0alp_0lam_1ep", - "name": "Qwen2.5-0.5B-SFT-2e-5-5ep-MDPO_5e-7_3ep_0alp_0lam_1ep", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2481, - "hfopenllm_v2/BBH": 0.3175, - "hfopenllm_v2/MATH Level 5": 0.0385, - "hfopenllm_v2/GPQA": 0.2626, - "hfopenllm_v2/MUSR": 0.3475, - "hfopenllm_v2/MMLU-PRO": 0.1597 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-5ep-MDPO_5e-7_3ep_0alp_0lam_2ep", - "name": "Qwen2.5-0.5B-SFT-2e-5-5ep-MDPO_5e-7_3ep_0alp_0lam_2ep", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2548, - "hfopenllm_v2/BBH": 0.3199, - "hfopenllm_v2/MATH Level 5": 0.0385, - "hfopenllm_v2/GPQA": 0.2651, - "hfopenllm_v2/MUSR": 0.3435, - "hfopenllm_v2/MMLU-PRO": 0.1562 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-5ep-MDPO_7e-7_3ep_0alp_0lam", - "name": "Qwen2.5-0.5B-SFT-2e-5-5ep-MDPO_7e-7_3ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2423, - "hfopenllm_v2/BBH": 0.3219, - "hfopenllm_v2/MATH Level 5": 0.034, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.3515, - "hfopenllm_v2/MMLU-PRO": 0.1563 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-5ep-MDPO_7e-7_3ep_0alp_0lam_1ep", - "name": "Qwen2.5-0.5B-SFT-2e-5-5ep-MDPO_7e-7_3ep_0alp_0lam_1ep", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2493, - "hfopenllm_v2/BBH": 0.3191, - "hfopenllm_v2/MATH Level 5": 0.0393, - "hfopenllm_v2/GPQA": 0.2685, - "hfopenllm_v2/MUSR": 0.3475, - "hfopenllm_v2/MMLU-PRO": 0.1592 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-5ep-MDPO_7e-7_3ep_0alp_0lam_2ep", - "name": "Qwen2.5-0.5B-SFT-2e-5-5ep-MDPO_7e-7_3ep_0alp_0lam_2ep", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2478, - "hfopenllm_v2/BBH": 0.3218, - "hfopenllm_v2/MATH Level 5": 0.0415, - "hfopenllm_v2/GPQA": 0.2693, - "hfopenllm_v2/MUSR": 0.3515, - "hfopenllm_v2/MMLU-PRO": 0.1556 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-5e-5", - "name": "Qwen2.5-0.5B-SFT-5e-5", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.201, - "hfopenllm_v2/BBH": 0.3109, - "hfopenllm_v2/MATH Level 5": 0.034, - "hfopenllm_v2/GPQA": 0.2676, - "hfopenllm_v2/MUSR": 0.3381, - "hfopenllm_v2/MMLU-PRO": 0.1672 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-5e-5-2ep", - "name": "Qwen2.5-0.5B-SFT-5e-5-2ep", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2175, - "hfopenllm_v2/BBH": 0.318, - "hfopenllm_v2/MATH Level 5": 0.0378, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3368, - "hfopenllm_v2/MMLU-PRO": 0.1627 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-5e-5-3ep", - "name": "Qwen2.5-0.5B-SFT-5e-5-3ep", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2199, - "hfopenllm_v2/BBH": 0.3297, - "hfopenllm_v2/MATH Level 5": 0.0302, - "hfopenllm_v2/GPQA": 0.2534, - "hfopenllm_v2/MUSR": 0.3593, - "hfopenllm_v2/MMLU-PRO": 0.1651 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-5e-5-5ep", - "name": "Qwen2.5-0.5B-SFT-5e-5-5ep", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2077, - "hfopenllm_v2/BBH": 0.3276, - "hfopenllm_v2/MATH Level 5": 0.0272, - "hfopenllm_v2/GPQA": 0.2685, - "hfopenllm_v2/MUSR": 0.3766, - "hfopenllm_v2/MMLU-PRO": 0.1587 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-7e-5", - "name": "Qwen2.5-0.5B-SFT-7e-5", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2093, - "hfopenllm_v2/BBH": 0.3158, - "hfopenllm_v2/MATH Level 5": 0.0302, - "hfopenllm_v2/GPQA": 0.2567, - "hfopenllm_v2/MUSR": 0.3367, - "hfopenllm_v2/MMLU-PRO": 0.1622 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-7e-5-2ep", - "name": "Qwen2.5-0.5B-SFT-7e-5-2ep", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2156, - "hfopenllm_v2/BBH": 0.31, - "hfopenllm_v2/MATH Level 5": 0.0393, - "hfopenllm_v2/GPQA": 0.2424, - "hfopenllm_v2/MUSR": 0.3367, - "hfopenllm_v2/MMLU-PRO": 0.1567 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-7e-5-3ep", - "name": "Qwen2.5-0.5B-SFT-7e-5-3ep", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2381, - "hfopenllm_v2/BBH": 0.3199, - "hfopenllm_v2/MATH Level 5": 0.0332, - "hfopenllm_v2/GPQA": 0.2366, - "hfopenllm_v2/MUSR": 0.3554, - "hfopenllm_v2/MMLU-PRO": 0.1522 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-7e-5-5ep", - "name": "Qwen2.5-0.5B-SFT-7e-5-5ep", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.212, - "hfopenllm_v2/BBH": 0.32, - "hfopenllm_v2/MATH Level 5": 0.0219, - "hfopenllm_v2/GPQA": 0.2458, - "hfopenllm_v2/MUSR": 0.3713, - "hfopenllm_v2/MMLU-PRO": 0.1628 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-DPO-1epoch_v1", - "name": "Qwen2.5-0.5B-SFT-DPO-1epoch_v1", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2025, - "hfopenllm_v2/BBH": 0.3268, - "hfopenllm_v2/MATH Level 5": 0.0363, - "hfopenllm_v2/GPQA": 0.2727, - "hfopenllm_v2/MUSR": 0.3209, - "hfopenllm_v2/MMLU-PRO": 0.133 - } - }, - { - "id": "JayHyeon/Qwen2.5-0.5B-SFT-MDPO-1epoch_v1", - "name": "Qwen2.5-0.5B-SFT-MDPO-1epoch_v1", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1964, - "hfopenllm_v2/BBH": 0.3293, - "hfopenllm_v2/MATH Level 5": 0.0468, - "hfopenllm_v2/GPQA": 0.276, - "hfopenllm_v2/MUSR": 0.3262, - "hfopenllm_v2/MMLU-PRO": 0.1337 - } - }, - { - "id": "JayHyeon/Qwen_0.5-cDPO_5e-7-3ep_0vpo_const_0.1", - "name": "Qwen_0.5-cDPO_5e-7-3ep_0vpo_const_0.1", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2393, - "hfopenllm_v2/BBH": 0.3244, - "hfopenllm_v2/MATH Level 5": 0.0514, - "hfopenllm_v2/GPQA": 0.2777, - "hfopenllm_v2/MUSR": 0.3222, - "hfopenllm_v2/MMLU-PRO": 0.1573 - } - }, - { - "id": "JayHyeon/Qwen_0.5-cDPO_5e-7-3ep_0vpo_const_0.3", - "name": "Qwen_0.5-cDPO_5e-7-3ep_0vpo_const_0.3", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2475, - "hfopenllm_v2/BBH": 0.3209, - "hfopenllm_v2/MATH Level 5": 0.0461, - "hfopenllm_v2/GPQA": 0.281, - "hfopenllm_v2/MUSR": 0.3275, - "hfopenllm_v2/MMLU-PRO": 0.1567 - } - }, - { - "id": "JayHyeon/Qwen_0.5-DPO_1e-6-3ep_0alp_0lam", - "name": "Qwen_0.5-DPO_1e-6-3ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2316, - "hfopenllm_v2/BBH": 0.3258, - "hfopenllm_v2/MATH Level 5": 0.0529, - "hfopenllm_v2/GPQA": 0.2693, - "hfopenllm_v2/MUSR": 0.3221, - "hfopenllm_v2/MMLU-PRO": 0.158 - } - }, - { - "id": "JayHyeon/Qwen_0.5-DPO_1e-7-3ep_0alp_0lam", - "name": "Qwen_0.5-DPO_1e-7-3ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.236, - "hfopenllm_v2/BBH": 0.3225, - "hfopenllm_v2/MATH Level 5": 0.0438, - "hfopenllm_v2/GPQA": 0.271, - "hfopenllm_v2/MUSR": 0.3222, - "hfopenllm_v2/MMLU-PRO": 0.1596 - } - }, - { - "id": "JayHyeon/Qwen_0.5-DPO_3e-6-1ep_0alp_0lam", - "name": "Qwen_0.5-DPO_3e-6-1ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2337, - "hfopenllm_v2/BBH": 0.3132, - "hfopenllm_v2/MATH Level 5": 0.0347, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.3235, - "hfopenllm_v2/MMLU-PRO": 0.1533 - } - }, - { - "id": "JayHyeon/Qwen_0.5-DPO_3e-6-2ep_0alp_0lam", - "name": "Qwen_0.5-DPO_3e-6-2ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2569, - "hfopenllm_v2/BBH": 0.3276, - "hfopenllm_v2/MATH Level 5": 0.0544, - "hfopenllm_v2/GPQA": 0.2718, - "hfopenllm_v2/MUSR": 0.3156, - "hfopenllm_v2/MMLU-PRO": 0.1565 - } - }, - { - "id": "JayHyeon/Qwen_0.5-DPO_3e-6-3ep_0alp_0lam", - "name": "Qwen_0.5-DPO_3e-6-3ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.246, - "hfopenllm_v2/BBH": 0.3267, - "hfopenllm_v2/MATH Level 5": 0.0431, - "hfopenllm_v2/GPQA": 0.2685, - "hfopenllm_v2/MUSR": 0.3209, - "hfopenllm_v2/MMLU-PRO": 0.1543 - } - }, - { - "id": "JayHyeon/Qwen_0.5-DPO_3e-7-1ep_0alp_0lam", - "name": "Qwen_0.5-DPO_3e-7-1ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2529, - "hfopenllm_v2/BBH": 0.3229, - "hfopenllm_v2/MATH Level 5": 0.0551, - "hfopenllm_v2/GPQA": 0.2676, - "hfopenllm_v2/MUSR": 0.3195, - "hfopenllm_v2/MMLU-PRO": 0.1597 - } - }, - { - "id": "JayHyeon/Qwen_0.5-DPO_3e-7-2ep_0alp_0lam", - "name": "Qwen_0.5-DPO_3e-7-2ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2505, - "hfopenllm_v2/BBH": 0.3256, - "hfopenllm_v2/MATH Level 5": 0.0476, - "hfopenllm_v2/GPQA": 0.2718, - "hfopenllm_v2/MUSR": 0.3195, - "hfopenllm_v2/MMLU-PRO": 0.1599 - } - }, - { - "id": "JayHyeon/Qwen_0.5-DPO_3e-7-3ep_0alp_0lam", - "name": "Qwen_0.5-DPO_3e-7-3ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2387, - "hfopenllm_v2/BBH": 0.3258, - "hfopenllm_v2/MATH Level 5": 0.0446, - "hfopenllm_v2/GPQA": 0.2743, - "hfopenllm_v2/MUSR": 0.3169, - "hfopenllm_v2/MMLU-PRO": 0.1589 - } - }, - { - "id": "JayHyeon/Qwen_0.5-DPO_5e-7-1ep_0alp_0lam", - "name": "Qwen_0.5-DPO_5e-7-1ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2532, - "hfopenllm_v2/BBH": 0.3218, - "hfopenllm_v2/MATH Level 5": 0.0634, - "hfopenllm_v2/GPQA": 0.2685, - "hfopenllm_v2/MUSR": 0.3209, - "hfopenllm_v2/MMLU-PRO": 0.1593 - } - }, - { - "id": "JayHyeon/Qwen_0.5-DPO_5e-7-2ep_0alp_0lam", - "name": "Qwen_0.5-DPO_5e-7-2ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2456, - "hfopenllm_v2/BBH": 0.3299, - "hfopenllm_v2/MATH Level 5": 0.0536, - "hfopenllm_v2/GPQA": 0.271, - "hfopenllm_v2/MUSR": 0.3181, - "hfopenllm_v2/MMLU-PRO": 0.1602 - } - }, - { - "id": "JayHyeon/Qwen_0.5-DPO_5e-7-3ep_0alp_0lam", - "name": "Qwen_0.5-DPO_5e-7-3ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2423, - "hfopenllm_v2/BBH": 0.3271, - "hfopenllm_v2/MATH Level 5": 0.0514, - "hfopenllm_v2/GPQA": 0.2743, - "hfopenllm_v2/MUSR": 0.3181, - "hfopenllm_v2/MMLU-PRO": 0.1595 - } - }, - { - "id": "JayHyeon/Qwen_0.5-DPOP_1e-6-3ep_0alp_5lam", - "name": "Qwen_0.5-DPOP_1e-6-3ep_0alp_5lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2532, - "hfopenllm_v2/BBH": 0.314, - "hfopenllm_v2/MATH Level 5": 0.0491, - "hfopenllm_v2/GPQA": 0.2743, - "hfopenllm_v2/MUSR": 0.3315, - "hfopenllm_v2/MMLU-PRO": 0.1566 - } - }, - { - "id": "JayHyeon/Qwen_0.5-DPOP_1e-7-3ep_0alp_5lam", - "name": "Qwen_0.5-DPOP_1e-7-3ep_0alp_5lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.267, - "hfopenllm_v2/BBH": 0.3189, - "hfopenllm_v2/MATH Level 5": 0.0408, - "hfopenllm_v2/GPQA": 0.2668, - "hfopenllm_v2/MUSR": 0.3288, - "hfopenllm_v2/MMLU-PRO": 0.1562 - } - }, - { - "id": "JayHyeon/Qwen_0.5-DPOP_3e-6-1ep_0alp_5lam", - "name": "Qwen_0.5-DPOP_3e-6-1ep_0alp_5lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2481, - "hfopenllm_v2/BBH": 0.3261, - "hfopenllm_v2/MATH Level 5": 0.0438, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3368, - "hfopenllm_v2/MMLU-PRO": 0.1565 - } - }, - { - "id": "JayHyeon/Qwen_0.5-DPOP_3e-6-2ep_0alp_5lam", - "name": "Qwen_0.5-DPOP_3e-6-2ep_0alp_5lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2383, - "hfopenllm_v2/BBH": 0.3218, - "hfopenllm_v2/MATH Level 5": 0.0431, - "hfopenllm_v2/GPQA": 0.2794, - "hfopenllm_v2/MUSR": 0.3342, - "hfopenllm_v2/MMLU-PRO": 0.1503 - } - }, - { - "id": "JayHyeon/Qwen_0.5-DPOP_3e-6-3ep_0alp_5lam", - "name": "Qwen_0.5-DPOP_3e-6-3ep_0alp_5lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2471, - "hfopenllm_v2/BBH": 0.3224, - "hfopenllm_v2/MATH Level 5": 0.04, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.3328, - "hfopenllm_v2/MMLU-PRO": 0.1533 - } - }, - { - "id": "JayHyeon/Qwen_0.5-DPOP_3e-7-1ep_0alp_5lam", - "name": "Qwen_0.5-DPOP_3e-7-1ep_0alp_5lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2447, - "hfopenllm_v2/BBH": 0.3181, - "hfopenllm_v2/MATH Level 5": 0.0438, - "hfopenllm_v2/GPQA": 0.2617, - "hfopenllm_v2/MUSR": 0.3341, - "hfopenllm_v2/MMLU-PRO": 0.1565 - } - }, - { - "id": "JayHyeon/Qwen_0.5-DPOP_3e-7-2ep_0alp_5lam", - "name": "Qwen_0.5-DPOP_3e-7-2ep_0alp_5lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2551, - "hfopenllm_v2/BBH": 0.3194, - "hfopenllm_v2/MATH Level 5": 0.0446, - "hfopenllm_v2/GPQA": 0.2617, - "hfopenllm_v2/MUSR": 0.3262, - "hfopenllm_v2/MMLU-PRO": 0.1567 - } - }, - { - "id": "JayHyeon/Qwen_0.5-DPOP_3e-7-3ep_0alp_5lam", - "name": "Qwen_0.5-DPOP_3e-7-3ep_0alp_5lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2538, - "hfopenllm_v2/BBH": 0.3153, - "hfopenllm_v2/MATH Level 5": 0.0415, - "hfopenllm_v2/GPQA": 0.2676, - "hfopenllm_v2/MUSR": 0.3261, - "hfopenllm_v2/MMLU-PRO": 0.1583 - } - }, - { - "id": "JayHyeon/Qwen_0.5-DPOP_5e-7-1ep_0alp_5lam", - "name": "Qwen_0.5-DPOP_5e-7-1ep_0alp_5lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2402, - "hfopenllm_v2/BBH": 0.3168, - "hfopenllm_v2/MATH Level 5": 0.0378, - "hfopenllm_v2/GPQA": 0.2718, - "hfopenllm_v2/MUSR": 0.3328, - "hfopenllm_v2/MMLU-PRO": 0.1568 - } - }, - { - "id": "JayHyeon/Qwen_0.5-DPOP_5e-7-2ep_0alp_5lam", - "name": "Qwen_0.5-DPOP_5e-7-2ep_0alp_5lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2484, - "hfopenllm_v2/BBH": 0.3211, - "hfopenllm_v2/MATH Level 5": 0.0438, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.3288, - "hfopenllm_v2/MMLU-PRO": 0.1573 - } - }, - { - "id": "JayHyeon/Qwen_0.5-DPOP_5e-7-3ep_0alp_5lam", - "name": "Qwen_0.5-DPOP_5e-7-3ep_0alp_5lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2578, - "hfopenllm_v2/BBH": 0.3203, - "hfopenllm_v2/MATH Level 5": 0.0423, - "hfopenllm_v2/GPQA": 0.271, - "hfopenllm_v2/MUSR": 0.3289, - "hfopenllm_v2/MMLU-PRO": 0.1583 - } - }, - { - "id": "JayHyeon/Qwen_0.5-IPO_5e-7-1ep_0alp_0lam", - "name": "Qwen_0.5-IPO_5e-7-1ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2574, - "hfopenllm_v2/BBH": 0.3279, - "hfopenllm_v2/MATH Level 5": 0.0559, - "hfopenllm_v2/GPQA": 0.2693, - "hfopenllm_v2/MUSR": 0.3169, - "hfopenllm_v2/MMLU-PRO": 0.1651 - } - }, - { - "id": "JayHyeon/Qwen_0.5-IPO_5e-7-3ep_0alp_0lam", - "name": "Qwen_0.5-IPO_5e-7-3ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3072, - "hfopenllm_v2/BBH": 0.3264, - "hfopenllm_v2/MATH Level 5": 0.0582, - "hfopenllm_v2/GPQA": 0.2567, - "hfopenllm_v2/MUSR": 0.3156, - "hfopenllm_v2/MMLU-PRO": 0.1624 - } - }, - { - "id": "JayHyeon/Qwen_0.5-IRPO_1e-6-3ep_1alp_0lam", - "name": "Qwen_0.5-IRPO_1e-6-3ep_1alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2551, - "hfopenllm_v2/BBH": 0.3242, - "hfopenllm_v2/MATH Level 5": 0.0468, - "hfopenllm_v2/GPQA": 0.2668, - "hfopenllm_v2/MUSR": 0.3182, - "hfopenllm_v2/MMLU-PRO": 0.1574 - } - }, - { - "id": "JayHyeon/Qwen_0.5-IRPO_1e-7-3ep_1alp_0lam", - "name": "Qwen_0.5-IRPO_1e-7-3ep_1alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2636, - "hfopenllm_v2/BBH": 0.3198, - "hfopenllm_v2/MATH Level 5": 0.0514, - "hfopenllm_v2/GPQA": 0.276, - "hfopenllm_v2/MUSR": 0.3262, - "hfopenllm_v2/MMLU-PRO": 0.1586 - } - }, - { - "id": "JayHyeon/Qwen_0.5-IRPO_3e-6-1ep_1alp_0lam", - "name": "Qwen_0.5-IRPO_3e-6-1ep_1alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2323, - "hfopenllm_v2/BBH": 0.3255, - "hfopenllm_v2/MATH Level 5": 0.037, - "hfopenllm_v2/GPQA": 0.2508, - "hfopenllm_v2/MUSR": 0.3169, - "hfopenllm_v2/MMLU-PRO": 0.1612 - } - }, - { - "id": "JayHyeon/Qwen_0.5-IRPO_3e-6-2ep_1alp_0lam", - "name": "Qwen_0.5-IRPO_3e-6-2ep_1alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2414, - "hfopenllm_v2/BBH": 0.3314, - "hfopenllm_v2/MATH Level 5": 0.0347, - "hfopenllm_v2/GPQA": 0.2517, - "hfopenllm_v2/MUSR": 0.3342, - "hfopenllm_v2/MMLU-PRO": 0.1532 - } - }, - { - "id": "JayHyeon/Qwen_0.5-IRPO_3e-6-3ep_1alp_0lam", - "name": "Qwen_0.5-IRPO_3e-6-3ep_1alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2678, - "hfopenllm_v2/BBH": 0.3362, - "hfopenllm_v2/MATH Level 5": 0.0514, - "hfopenllm_v2/GPQA": 0.2542, - "hfopenllm_v2/MUSR": 0.3382, - "hfopenllm_v2/MMLU-PRO": 0.1561 - } - }, - { - "id": "JayHyeon/Qwen_0.5-IRPO_3e-7-1ep_1alp_0lam", - "name": "Qwen_0.5-IRPO_3e-7-1ep_1alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2561, - "hfopenllm_v2/BBH": 0.3231, - "hfopenllm_v2/MATH Level 5": 0.0536, - "hfopenllm_v2/GPQA": 0.2718, - "hfopenllm_v2/MUSR": 0.3196, - "hfopenllm_v2/MMLU-PRO": 0.1589 - } - }, - { - "id": "JayHyeon/Qwen_0.5-IRPO_3e-7-3ep_1alp_0lam", - "name": "Qwen_0.5-IRPO_3e-7-3ep_1alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2639, - "hfopenllm_v2/BBH": 0.3257, - "hfopenllm_v2/MATH Level 5": 0.0476, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.3209, - "hfopenllm_v2/MMLU-PRO": 0.1587 - } - }, - { - "id": "JayHyeon/Qwen_0.5-IRPO_5e-7-1ep_1alp_0lam", - "name": "Qwen_0.5-IRPO_5e-7-1ep_1alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2518, - "hfopenllm_v2/BBH": 0.3214, - "hfopenllm_v2/MATH Level 5": 0.0574, - "hfopenllm_v2/GPQA": 0.2735, - "hfopenllm_v2/MUSR": 0.3169, - "hfopenllm_v2/MMLU-PRO": 0.1585 - } - }, - { - "id": "JayHyeon/Qwen_0.5-IRPO_5e-7-2ep_1alp_0lam", - "name": "Qwen_0.5-IRPO_5e-7-2ep_1alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2438, - "hfopenllm_v2/BBH": 0.3266, - "hfopenllm_v2/MATH Level 5": 0.0619, - "hfopenllm_v2/GPQA": 0.2727, - "hfopenllm_v2/MUSR": 0.3196, - "hfopenllm_v2/MMLU-PRO": 0.1554 - } - }, - { - "id": "JayHyeon/Qwen_0.5-IRPO_5e-7-3ep_1alp_0lam", - "name": "Qwen_0.5-IRPO_5e-7-3ep_1alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2465, - "hfopenllm_v2/BBH": 0.3246, - "hfopenllm_v2/MATH Level 5": 0.0529, - "hfopenllm_v2/GPQA": 0.2718, - "hfopenllm_v2/MUSR": 0.3182, - "hfopenllm_v2/MMLU-PRO": 0.1563 - } - }, - { - "id": "JayHyeon/Qwen_0.5-MDPO_0.1_3e-6-3ep_0alp_0lam", - "name": "Qwen_0.5-MDPO_0.1_3e-6-3ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2506, - "hfopenllm_v2/BBH": 0.3261, - "hfopenllm_v2/MATH Level 5": 0.0498, - "hfopenllm_v2/GPQA": 0.2819, - "hfopenllm_v2/MUSR": 0.3382, - "hfopenllm_v2/MMLU-PRO": 0.1522 - } - }, - { - "id": "JayHyeon/Qwen_0.5-MDPO_0.1_5e-7-3ep_0alp_0lam", - "name": "Qwen_0.5-MDPO_0.1_5e-7-3ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2457, - "hfopenllm_v2/BBH": 0.318, - "hfopenllm_v2/MATH Level 5": 0.0347, - "hfopenllm_v2/GPQA": 0.2634, - "hfopenllm_v2/MUSR": 0.3315, - "hfopenllm_v2/MMLU-PRO": 0.1566 - } - }, - { - "id": "JayHyeon/Qwen_0.5-MDPO_0.3_3e-6-3ep_0alp_0lam", - "name": "Qwen_0.5-MDPO_0.3_3e-6-3ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2454, - "hfopenllm_v2/BBH": 0.3216, - "hfopenllm_v2/MATH Level 5": 0.0506, - "hfopenllm_v2/GPQA": 0.2802, - "hfopenllm_v2/MUSR": 0.3382, - "hfopenllm_v2/MMLU-PRO": 0.1544 - } - }, - { - "id": "JayHyeon/Qwen_0.5-MDPO_0.3_5e-7-3ep_0alp_0lam", - "name": "Qwen_0.5-MDPO_0.3_5e-7-3ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2342, - "hfopenllm_v2/BBH": 0.3189, - "hfopenllm_v2/MATH Level 5": 0.04, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.3302, - "hfopenllm_v2/MMLU-PRO": 0.158 - } - }, - { - "id": "JayHyeon/Qwen_0.5-MDPO_0.5_1e-5-3ep_0alp_0lam", - "name": "Qwen_0.5-MDPO_0.5_1e-5-3ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.232, - "hfopenllm_v2/BBH": 0.3234, - "hfopenllm_v2/MATH Level 5": 0.0393, - "hfopenllm_v2/GPQA": 0.2743, - "hfopenllm_v2/MUSR": 0.3369, - "hfopenllm_v2/MMLU-PRO": 0.1543 - } - }, - { - "id": "JayHyeon/Qwen_0.5-MDPO_0.5_3e-7-1ep_0alp_0lam", - "name": "Qwen_0.5-MDPO_0.5_3e-7-1ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2418, - "hfopenllm_v2/BBH": 0.3175, - "hfopenllm_v2/MATH Level 5": 0.0423, - "hfopenllm_v2/GPQA": 0.2626, - "hfopenllm_v2/MUSR": 0.3288, - "hfopenllm_v2/MMLU-PRO": 0.158 - } - }, - { - "id": "JayHyeon/Qwen_0.5-MDPO_0.5_3e-7-2ep_0alp_0lam", - "name": "Qwen_0.5-MDPO_0.5_3e-7-2ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2493, - "hfopenllm_v2/BBH": 0.3197, - "hfopenllm_v2/MATH Level 5": 0.0423, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.3315, - "hfopenllm_v2/MMLU-PRO": 0.1571 - } - }, - { - "id": "JayHyeon/Qwen_0.5-MDPO_0.5_3e-7-3ep_0alp_0lam", - "name": "Qwen_0.5-MDPO_0.5_3e-7-3ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.252, - "hfopenllm_v2/BBH": 0.3198, - "hfopenllm_v2/MATH Level 5": 0.0423, - "hfopenllm_v2/GPQA": 0.2634, - "hfopenllm_v2/MUSR": 0.3262, - "hfopenllm_v2/MMLU-PRO": 0.1551 - } - }, - { - "id": "JayHyeon/Qwen_0.5-MDPO_0.5_4e-6-3ep_0alp_0lam", - "name": "Qwen_0.5-MDPO_0.5_4e-6-3ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.258, - "hfopenllm_v2/BBH": 0.3248, - "hfopenllm_v2/MATH Level 5": 0.0476, - "hfopenllm_v2/GPQA": 0.2752, - "hfopenllm_v2/MUSR": 0.3422, - "hfopenllm_v2/MMLU-PRO": 0.1539 - } - }, - { - "id": "JayHyeon/Qwen_0.5-MDPO_0.5_6e-6-3ep_0alp_0lam", - "name": "Qwen_0.5-MDPO_0.5_6e-6-3ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.232, - "hfopenllm_v2/BBH": 0.3265, - "hfopenllm_v2/MATH Level 5": 0.0385, - "hfopenllm_v2/GPQA": 0.271, - "hfopenllm_v2/MUSR": 0.3395, - "hfopenllm_v2/MMLU-PRO": 0.1537 - } - }, - { - "id": "JayHyeon/Qwen_0.5-MDPO_0.5_7e-6-3ep_0alp_0lam", - "name": "Qwen_0.5-MDPO_0.5_7e-6-3ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2488, - "hfopenllm_v2/BBH": 0.3273, - "hfopenllm_v2/MATH Level 5": 0.0461, - "hfopenllm_v2/GPQA": 0.2718, - "hfopenllm_v2/MUSR": 0.3342, - "hfopenllm_v2/MMLU-PRO": 0.1531 - } - }, - { - "id": "JayHyeon/Qwen_0.5-MDPO_0.5_7e-7-3ep_0alp_0lam", - "name": "Qwen_0.5-MDPO_0.5_7e-7-3ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2524, - "hfopenllm_v2/BBH": 0.313, - "hfopenllm_v2/MATH Level 5": 0.0446, - "hfopenllm_v2/GPQA": 0.271, - "hfopenllm_v2/MUSR": 0.3289, - "hfopenllm_v2/MMLU-PRO": 0.1564 - } - }, - { - "id": "JayHyeon/Qwen_0.5-MDPO_0.7_3e-6-3ep_0alp_0lam", - "name": "Qwen_0.5-MDPO_0.7_3e-6-3ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2514, - "hfopenllm_v2/BBH": 0.3221, - "hfopenllm_v2/MATH Level 5": 0.0438, - "hfopenllm_v2/GPQA": 0.2752, - "hfopenllm_v2/MUSR": 0.3315, - "hfopenllm_v2/MMLU-PRO": 0.1538 - } - }, - { - "id": "JayHyeon/Qwen_0.5-MDPO_0.7_5e-7-3ep_0alp_0lam", - "name": "Qwen_0.5-MDPO_0.7_5e-7-3ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2457, - "hfopenllm_v2/BBH": 0.318, - "hfopenllm_v2/MATH Level 5": 0.0385, - "hfopenllm_v2/GPQA": 0.2668, - "hfopenllm_v2/MUSR": 0.3275, - "hfopenllm_v2/MMLU-PRO": 0.1572 - } - }, - { - "id": "JayHyeon/Qwen_0.5-MDPO_0.9_5e-7-3ep_0alp_0lam", - "name": "Qwen_0.5-MDPO_0.9_5e-7-3ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2636, - "hfopenllm_v2/BBH": 0.3181, - "hfopenllm_v2/MATH Level 5": 0.0476, - "hfopenllm_v2/GPQA": 0.2659, - "hfopenllm_v2/MUSR": 0.3235, - "hfopenllm_v2/MMLU-PRO": 0.1574 - } - }, - { - "id": "JayHyeon/Qwen_0.5-rDPO_3e-6-1ep_0vpo_const_0.1", - "name": "Qwen_0.5-rDPO_3e-6-1ep_0vpo_const_0.1", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2321, - "hfopenllm_v2/BBH": 0.3278, - "hfopenllm_v2/MATH Level 5": 0.0476, - "hfopenllm_v2/GPQA": 0.2576, - "hfopenllm_v2/MUSR": 0.3022, - "hfopenllm_v2/MMLU-PRO": 0.1496 - } - }, - { - "id": "JayHyeon/Qwen_0.5-rDPO_5e-7-3ep_0vpo_const_0.1", - "name": "Qwen_0.5-rDPO_5e-7-3ep_0vpo_const_0.1", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2542, - "hfopenllm_v2/BBH": 0.3253, - "hfopenllm_v2/MATH Level 5": 0.0529, - "hfopenllm_v2/GPQA": 0.271, - "hfopenllm_v2/MUSR": 0.3181, - "hfopenllm_v2/MMLU-PRO": 0.1609 - } - }, - { - "id": "JayHyeon/Qwen_0.5-rDPO_5e-7-3ep_0vpo_const_0.3", - "name": "Qwen_0.5-rDPO_5e-7-3ep_0vpo_const_0.3", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2739, - "hfopenllm_v2/BBH": 0.3245, - "hfopenllm_v2/MATH Level 5": 0.0461, - "hfopenllm_v2/GPQA": 0.2508, - "hfopenllm_v2/MUSR": 0.3089, - "hfopenllm_v2/MMLU-PRO": 0.1597 - } - }, - { - "id": "JayHyeon/Qwen_0.5-VDPO_3e-6-1ep_3vpo_const", - "name": "Qwen_0.5-VDPO_3e-6-1ep_3vpo_const", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2483, - "hfopenllm_v2/BBH": 0.3174, - "hfopenllm_v2/MATH Level 5": 0.0378, - "hfopenllm_v2/GPQA": 0.2542, - "hfopenllm_v2/MUSR": 0.3328, - "hfopenllm_v2/MMLU-PRO": 0.1558 - } - }, - { - "id": "JayHyeon/Qwen_0.5-VDPO_5e-7-1ep_0alp_0lam", - "name": "Qwen_0.5-VDPO_5e-7-1ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2518, - "hfopenllm_v2/BBH": 0.3218, - "hfopenllm_v2/MATH Level 5": 0.0529, - "hfopenllm_v2/GPQA": 0.2718, - "hfopenllm_v2/MUSR": 0.3235, - "hfopenllm_v2/MMLU-PRO": 0.1595 - } - }, - { - "id": "JayHyeon/Qwen_0.5-VDPO_5e-7-1ep_10vpo_const", - "name": "Qwen_0.5-VDPO_5e-7-1ep_10vpo_const", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2536, - "hfopenllm_v2/BBH": 0.3234, - "hfopenllm_v2/MATH Level 5": 0.0491, - "hfopenllm_v2/GPQA": 0.276, - "hfopenllm_v2/MUSR": 0.3236, - "hfopenllm_v2/MMLU-PRO": 0.1597 - } - }, - { - "id": "JayHyeon/Qwen_0.5-VDPO_5e-7-1ep_1vpo_const", - "name": "Qwen_0.5-VDPO_5e-7-1ep_1vpo_const", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2448, - "hfopenllm_v2/BBH": 0.324, - "hfopenllm_v2/MATH Level 5": 0.0604, - "hfopenllm_v2/GPQA": 0.2752, - "hfopenllm_v2/MUSR": 0.3249, - "hfopenllm_v2/MMLU-PRO": 0.1587 - } - }, - { - "id": "JayHyeon/Qwen_0.5-VDPO_5e-7-1ep_3vpo_const", - "name": "Qwen_0.5-VDPO_5e-7-1ep_3vpo_const", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2505, - "hfopenllm_v2/BBH": 0.3227, - "hfopenllm_v2/MATH Level 5": 0.0468, - "hfopenllm_v2/GPQA": 0.271, - "hfopenllm_v2/MUSR": 0.3209, - "hfopenllm_v2/MMLU-PRO": 0.1589 - } - }, - { - "id": "JayHyeon/Qwen_0.5-VDPO_5e-7-3ep_0alp_0lam", - "name": "Qwen_0.5-VDPO_5e-7-3ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2472, - "hfopenllm_v2/BBH": 0.3255, - "hfopenllm_v2/MATH Level 5": 0.0498, - "hfopenllm_v2/GPQA": 0.2752, - "hfopenllm_v2/MUSR": 0.3208, - "hfopenllm_v2/MMLU-PRO": 0.1587 - } - }, - { - "id": "JayHyeon/Qwen_0.5-VDPO_5e-7-3ep_1vpo_const", - "name": "Qwen_0.5-VDPO_5e-7-3ep_1vpo_const", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2417, - "hfopenllm_v2/BBH": 0.3256, - "hfopenllm_v2/MATH Level 5": 0.0582, - "hfopenllm_v2/GPQA": 0.2727, - "hfopenllm_v2/MUSR": 0.3275, - "hfopenllm_v2/MMLU-PRO": 0.1562 - } - }, - { - "id": "JayHyeon/Qwen_0.5-VDPO_5e-7-3ep_3vpo_const", - "name": "Qwen_0.5-VDPO_5e-7-3ep_3vpo_const", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2527, - "hfopenllm_v2/BBH": 0.3235, - "hfopenllm_v2/MATH Level 5": 0.0536, - "hfopenllm_v2/GPQA": 0.2785, - "hfopenllm_v2/MUSR": 0.3235, - "hfopenllm_v2/MMLU-PRO": 0.158 - } - }, - { - "id": "JayHyeon/Qwen_0.5-VIPO_5e-7-1ep_0alp_0lam", - "name": "Qwen_0.5-VIPO_5e-7-1ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2669, - "hfopenllm_v2/BBH": 0.3314, - "hfopenllm_v2/MATH Level 5": 0.071, - "hfopenllm_v2/GPQA": 0.2676, - "hfopenllm_v2/MUSR": 0.3168, - "hfopenllm_v2/MMLU-PRO": 0.1634 - } - }, - { - "id": "JayHyeon/Qwen_0.5-VIPO_5e-7-1ep_10vpo_const", - "name": "Qwen_0.5-VIPO_5e-7-1ep_10vpo_const", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2702, - "hfopenllm_v2/BBH": 0.33, - "hfopenllm_v2/MATH Level 5": 0.074, - "hfopenllm_v2/GPQA": 0.2752, - "hfopenllm_v2/MUSR": 0.3208, - "hfopenllm_v2/MMLU-PRO": 0.1635 - } - }, - { - "id": "JayHyeon/Qwen_0.5-VIPO_5e-7-1ep_1vpo_const", - "name": "Qwen_0.5-VIPO_5e-7-1ep_1vpo_const", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.248, - "hfopenllm_v2/BBH": 0.3309, - "hfopenllm_v2/MATH Level 5": 0.068, - "hfopenllm_v2/GPQA": 0.2643, - "hfopenllm_v2/MUSR": 0.3208, - "hfopenllm_v2/MMLU-PRO": 0.1649 - } - }, - { - "id": "JayHyeon/Qwen_0.5-VIPO_5e-7-1ep_30vpo_const", - "name": "Qwen_0.5-VIPO_5e-7-1ep_30vpo_const", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2622, - "hfopenllm_v2/BBH": 0.3282, - "hfopenllm_v2/MATH Level 5": 0.074, - "hfopenllm_v2/GPQA": 0.2693, - "hfopenllm_v2/MUSR": 0.3221, - "hfopenllm_v2/MMLU-PRO": 0.1634 - } - }, - { - "id": "JayHyeon/Qwen_0.5-VIPO_5e-7-1ep_3vpo_const", - "name": "Qwen_0.5-VIPO_5e-7-1ep_3vpo_const", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2609, - "hfopenllm_v2/BBH": 0.3298, - "hfopenllm_v2/MATH Level 5": 0.065, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.3168, - "hfopenllm_v2/MMLU-PRO": 0.1651 - } - }, - { - "id": "JayHyeon/Qwen_0.5-VIPO_5e-7-3ep_0alp_0lam", - "name": "Qwen_0.5-VIPO_5e-7-3ep_0alp_0lam", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.293, - "hfopenllm_v2/BBH": 0.322, - "hfopenllm_v2/MATH Level 5": 0.0627, - "hfopenllm_v2/GPQA": 0.2685, - "hfopenllm_v2/MUSR": 0.3116, - "hfopenllm_v2/MMLU-PRO": 0.1591 - } - }, - { - "id": "JayHyeon/Qwen_0.5-VIPO_5e-7-3ep_10vpo_const", - "name": "Qwen_0.5-VIPO_5e-7-3ep_10vpo_const", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2881, - "hfopenllm_v2/BBH": 0.3255, - "hfopenllm_v2/MATH Level 5": 0.0725, - "hfopenllm_v2/GPQA": 0.2752, - "hfopenllm_v2/MUSR": 0.3102, - "hfopenllm_v2/MMLU-PRO": 0.1582 - } - }, - { - "id": "JayHyeon/Qwen_0.5-VIPO_5e-7-3ep_1vpo_const", - "name": "Qwen_0.5-VIPO_5e-7-3ep_1vpo_const", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2887, - "hfopenllm_v2/BBH": 0.3237, - "hfopenllm_v2/MATH Level 5": 0.0748, - "hfopenllm_v2/GPQA": 0.2802, - "hfopenllm_v2/MUSR": 0.3142, - "hfopenllm_v2/MMLU-PRO": 0.1609 - } - }, - { - "id": "JayHyeon/Qwen_0.5-VIPO_5e-7-3ep_30vpo_const", - "name": "Qwen_0.5-VIPO_5e-7-3ep_30vpo_const", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2905, - "hfopenllm_v2/BBH": 0.3254, - "hfopenllm_v2/MATH Level 5": 0.077, - "hfopenllm_v2/GPQA": 0.2735, - "hfopenllm_v2/MUSR": 0.3129, - "hfopenllm_v2/MMLU-PRO": 0.1574 - } - }, - { - "id": "JayHyeon/Qwen_0.5-VIPO_5e-7-3ep_3vpo_const", - "name": "Qwen_0.5-VIPO_5e-7-3ep_3vpo_const", - "developer": "JayHyeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2905, - "hfopenllm_v2/BBH": 0.3238, - "hfopenllm_v2/MATH Level 5": 0.0702, - "hfopenllm_v2/GPQA": 0.2735, - "hfopenllm_v2/MUSR": 0.3089, - "hfopenllm_v2/MMLU-PRO": 0.1592 - } - }, - { - "id": "jeanmichela/o-distil-qwen", - "name": "o-distil-qwen", - "developer": "jeanmichela", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4482, - "hfopenllm_v2/BBH": 0.59, - "hfopenllm_v2/MATH Level 5": 0.565, - "hfopenllm_v2/GPQA": 0.3935, - "hfopenllm_v2/MUSR": 0.534, - "hfopenllm_v2/MMLU-PRO": 0.4658 - } - }, - { - "id": "jebcarter/psyonic-cetacean-20B", - "name": "psyonic-cetacean-20B", - "developer": "jebcarter", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2544, - "hfopenllm_v2/BBH": 0.4907, - "hfopenllm_v2/MATH Level 5": 0.0181, - "hfopenllm_v2/GPQA": 0.2735, - "hfopenllm_v2/MUSR": 0.4661, - "hfopenllm_v2/MMLU-PRO": 0.2886 - } - }, - { - "id": "jebish7/aya-expanse-8b", - "name": "aya-expanse-8b", - "developer": "jebish7", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3791, - "hfopenllm_v2/BBH": 0.4969, - "hfopenllm_v2/MATH Level 5": 0.0816, - "hfopenllm_v2/GPQA": 0.2836, - "hfopenllm_v2/MUSR": 0.3869, - "hfopenllm_v2/MMLU-PRO": 0.3103 - } - }, - { - "id": "jebish7/gemma-2-2b-it", - "name": "gemma-2-2b-it", - "developer": "jebish7", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1272, - "hfopenllm_v2/BBH": 0.4395, - "hfopenllm_v2/MATH Level 5": 0.034, - "hfopenllm_v2/GPQA": 0.297, - "hfopenllm_v2/MUSR": 0.4244, - "hfopenllm_v2/MMLU-PRO": 0.2715 - } - }, - { - "id": "jebish7/gemma-2-9b-it", - "name": "gemma-2-9b-it", - "developer": "jebish7", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1557, - "hfopenllm_v2/BBH": 0.5949, - "hfopenllm_v2/MATH Level 5": 0.0846, - "hfopenllm_v2/GPQA": 0.3473, - "hfopenllm_v2/MUSR": 0.4554, - "hfopenllm_v2/MMLU-PRO": 0.4143 - } - }, - { - "id": "jebish7/Llama-3-Nanda-10B-Chat", - "name": "Llama-3-Nanda-10B-Chat", - "developer": "jebish7", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2953, - "hfopenllm_v2/BBH": 0.4959, - "hfopenllm_v2/MATH Level 5": 0.0559, - "hfopenllm_v2/GPQA": 0.3012, - "hfopenllm_v2/MUSR": 0.4356, - "hfopenllm_v2/MMLU-PRO": 0.3157 - } - }, - { - "id": "jebish7/Llama-3.1-8B-Instruct", - "name": "Llama-3.1-8B-Instruct", - "developer": "jebish7", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5058, - "hfopenllm_v2/BBH": 0.5088, - "hfopenllm_v2/MATH Level 5": 0.1548, - "hfopenllm_v2/GPQA": 0.3213, - "hfopenllm_v2/MUSR": 0.3998, - "hfopenllm_v2/MMLU-PRO": 0.3777 - } - }, - { - "id": "jebish7/Nemotron-4-Mini-Hindi-4B-Base", - "name": "Nemotron-4-Mini-Hindi-4B-Base", - "developer": "jebish7", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2285, - "hfopenllm_v2/BBH": 0.3924, - "hfopenllm_v2/MATH Level 5": 0.0272, - "hfopenllm_v2/GPQA": 0.2836, - "hfopenllm_v2/MUSR": 0.4249, - "hfopenllm_v2/MMLU-PRO": 0.2503 - } - }, - { - "id": "jebish7/Nemotron-4-Mini-Hindi-4B-Instruct", - "name": "Nemotron-4-Mini-Hindi-4B-Instruct", - "developer": "jebish7", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3345, - "hfopenllm_v2/BBH": 0.4041, - "hfopenllm_v2/MATH Level 5": 0.0287, - "hfopenllm_v2/GPQA": 0.3087, - "hfopenllm_v2/MUSR": 0.4153, - "hfopenllm_v2/MMLU-PRO": 0.2595 - } - }, - { - "id": "jebish7/Nemotron-Mini-4B-Instruct", - "name": "Nemotron-Mini-4B-Instruct", - "developer": "jebish7", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3709, - "hfopenllm_v2/BBH": 0.4244, - "hfopenllm_v2/MATH Level 5": 0.0325, - "hfopenllm_v2/GPQA": 0.276, - "hfopenllm_v2/MUSR": 0.4727, - "hfopenllm_v2/MMLU-PRO": 0.2783 - } - }, - { - "id": "jebish7/qwen2.5-0.5B-IHA-Hin", - "name": "qwen2.5-0.5B-IHA-Hin", - "developer": "jebish7", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1416, - "hfopenllm_v2/BBH": 0.2989, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2525, - "hfopenllm_v2/MUSR": 0.3475, - "hfopenllm_v2/MMLU-PRO": 0.1094 - } - }, - { - "id": "jeffmeloy/jeffmeloy_Qwen2.5-7B-minperplexity-1", - "name": "jeffmeloy_Qwen2.5-7B-minperplexity-1", - "developer": "jeffmeloy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3757, - "hfopenllm_v2/BBH": 0.5582, - "hfopenllm_v2/MATH Level 5": 0.2915, - "hfopenllm_v2/GPQA": 0.3322, - "hfopenllm_v2/MUSR": 0.429, - "hfopenllm_v2/MMLU-PRO": 0.4368 - } - }, - { - "id": "jeffmeloy/Qwen-7B-nerd-uncensored-v1.0", - "name": "Qwen-7B-nerd-uncensored-v1.0", - "developer": "jeffmeloy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6136, - "hfopenllm_v2/BBH": 0.5421, - "hfopenllm_v2/MATH Level 5": 0.287, - "hfopenllm_v2/GPQA": 0.328, - "hfopenllm_v2/MUSR": 0.4793, - "hfopenllm_v2/MMLU-PRO": 0.4363 - } - }, - { - "id": "jeffmeloy/Qwen2.5-7B-minperplexity-2", - "name": "Qwen2.5-7B-minperplexity-2", - "developer": "jeffmeloy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5097, - "hfopenllm_v2/BBH": 0.5524, - "hfopenllm_v2/MATH Level 5": 0.3014, - "hfopenllm_v2/GPQA": 0.3112, - "hfopenllm_v2/MUSR": 0.4625, - "hfopenllm_v2/MMLU-PRO": 0.4346 - } - }, - { - "id": "jeffmeloy/Qwen2.5-7B-nerd-uncensored-v0.9", - "name": "Qwen2.5-7B-nerd-uncensored-v0.9", - "developer": "jeffmeloy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6048, - "hfopenllm_v2/BBH": 0.547, - "hfopenllm_v2/MATH Level 5": 0.2946, - "hfopenllm_v2/GPQA": 0.323, - "hfopenllm_v2/MUSR": 0.482, - "hfopenllm_v2/MMLU-PRO": 0.4363 - } - }, - { - "id": "jeffmeloy/Qwen2.5-7B-nerd-uncensored-v1.0", - "name": "Qwen2.5-7B-nerd-uncensored-v1.0", - "developer": "jeffmeloy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7695, - "hfopenllm_v2/BBH": 0.5418, - "hfopenllm_v2/MATH Level 5": 0.4713, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.4551, - "hfopenllm_v2/MMLU-PRO": 0.4254 - } - }, - { - "id": "jeffmeloy/Qwen2.5-7B-nerd-uncensored-v1.1", - "name": "Qwen2.5-7B-nerd-uncensored-v1.1", - "developer": "jeffmeloy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6626, - "hfopenllm_v2/BBH": 0.4864, - "hfopenllm_v2/MATH Level 5": 0.1329, - "hfopenllm_v2/GPQA": 0.2869, - "hfopenllm_v2/MUSR": 0.3843, - "hfopenllm_v2/MMLU-PRO": 0.385 - } - }, - { - "id": "jeffmeloy/Qwen2.5-7B-nerd-uncensored-v1.2", - "name": "Qwen2.5-7B-nerd-uncensored-v1.2", - "developer": "jeffmeloy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4965, - "hfopenllm_v2/BBH": 0.4946, - "hfopenllm_v2/MATH Level 5": 0.1208, - "hfopenllm_v2/GPQA": 0.3037, - "hfopenllm_v2/MUSR": 0.4172, - "hfopenllm_v2/MMLU-PRO": 0.3969 - } - }, - { - "id": "jeffmeloy/Qwen2.5-7B-nerd-uncensored-v1.3", - "name": "Qwen2.5-7B-nerd-uncensored-v1.3", - "developer": "jeffmeloy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4995, - "hfopenllm_v2/BBH": 0.5026, - "hfopenllm_v2/MATH Level 5": 0.1231, - "hfopenllm_v2/GPQA": 0.3129, - "hfopenllm_v2/MUSR": 0.4187, - "hfopenllm_v2/MMLU-PRO": 0.4016 - } - }, - { - "id": "jeffmeloy/Qwen2.5-7B-nerd-uncensored-v1.4", - "name": "Qwen2.5-7B-nerd-uncensored-v1.4", - "developer": "jeffmeloy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6079, - "hfopenllm_v2/BBH": 0.5467, - "hfopenllm_v2/MATH Level 5": 0.281, - "hfopenllm_v2/GPQA": 0.3238, - "hfopenllm_v2/MUSR": 0.4714, - "hfopenllm_v2/MMLU-PRO": 0.4419 - } - }, - { - "id": "jeffmeloy/Qwen2.5-7B-nerd-uncensored-v1.5", - "name": "Qwen2.5-7B-nerd-uncensored-v1.5", - "developer": "jeffmeloy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.565, - "hfopenllm_v2/BBH": 0.5523, - "hfopenllm_v2/MATH Level 5": 0.2757, - "hfopenllm_v2/GPQA": 0.3272, - "hfopenllm_v2/MUSR": 0.4982, - "hfopenllm_v2/MMLU-PRO": 0.4448 - } - }, - { - "id": "jeffmeloy/Qwen2.5-7B-nerd-uncensored-v1.7", - "name": "Qwen2.5-7B-nerd-uncensored-v1.7", - "developer": "jeffmeloy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4202, - "hfopenllm_v2/BBH": 0.5392, - "hfopenllm_v2/MATH Level 5": 0.2915, - "hfopenllm_v2/GPQA": 0.3238, - "hfopenllm_v2/MUSR": 0.4848, - "hfopenllm_v2/MMLU-PRO": 0.428 - } - }, - { - "id": "jeffmeloy/Qwen2.5-7B-nerd-uncensored-v1.8", - "name": "Qwen2.5-7B-nerd-uncensored-v1.8", - "developer": "jeffmeloy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6256, - "hfopenllm_v2/BBH": 0.5447, - "hfopenllm_v2/MATH Level 5": 0.2704, - "hfopenllm_v2/GPQA": 0.3238, - "hfopenllm_v2/MUSR": 0.4767, - "hfopenllm_v2/MMLU-PRO": 0.4343 - } - }, - { - "id": "jeffmeloy/Qwen2.5-7B-olm-v1.0", - "name": "Qwen2.5-7B-olm-v1.0", - "developer": "jeffmeloy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5331, - "hfopenllm_v2/BBH": 0.566, - "hfopenllm_v2/MATH Level 5": 0.2863, - "hfopenllm_v2/GPQA": 0.3205, - "hfopenllm_v2/MUSR": 0.4278, - "hfopenllm_v2/MMLU-PRO": 0.4566 - } - }, - { - "id": "jeffmeloy/Qwen2.5-7B-olm-v1.1", - "name": "Qwen2.5-7B-olm-v1.1", - "developer": "jeffmeloy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4329, - "hfopenllm_v2/BBH": 0.5478, - "hfopenllm_v2/MATH Level 5": 0.3829, - "hfopenllm_v2/GPQA": 0.3087, - "hfopenllm_v2/MUSR": 0.4808, - "hfopenllm_v2/MMLU-PRO": 0.4354 - } - }, - { - "id": "jeffmeloy/Qwen2.5-7B-olm-v1.2", - "name": "Qwen2.5-7B-olm-v1.2", - "developer": "jeffmeloy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4203, - "hfopenllm_v2/BBH": 0.5533, - "hfopenllm_v2/MATH Level 5": 0.2847, - "hfopenllm_v2/GPQA": 0.3171, - "hfopenllm_v2/MUSR": 0.4688, - "hfopenllm_v2/MMLU-PRO": 0.4387 - } - }, - { - "id": "jeffmeloy/Qwen2.5-7B-olm-v1.3", - "name": "Qwen2.5-7B-olm-v1.3", - "developer": "jeffmeloy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4219, - "hfopenllm_v2/BBH": 0.5532, - "hfopenllm_v2/MATH Level 5": 0.3104, - "hfopenllm_v2/GPQA": 0.3213, - "hfopenllm_v2/MUSR": 0.4701, - "hfopenllm_v2/MMLU-PRO": 0.447 - } - }, - { - "id": "jeffmeloy/Qwen2.5-7B-olm-v1.4", - "name": "Qwen2.5-7B-olm-v1.4", - "developer": "jeffmeloy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4545, - "hfopenllm_v2/BBH": 0.5582, - "hfopenllm_v2/MATH Level 5": 0.2923, - "hfopenllm_v2/GPQA": 0.3121, - "hfopenllm_v2/MUSR": 0.4622, - "hfopenllm_v2/MMLU-PRO": 0.4457 - } - }, - { - "id": "jeffmeloy/Qwen2.5-7B-olm-v1.5", - "name": "Qwen2.5-7B-olm-v1.5", - "developer": "jeffmeloy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4547, - "hfopenllm_v2/BBH": 0.5544, - "hfopenllm_v2/MATH Level 5": 0.2817, - "hfopenllm_v2/GPQA": 0.3398, - "hfopenllm_v2/MUSR": 0.4539, - "hfopenllm_v2/MMLU-PRO": 0.4399 - } - }, - { - "id": "jeonsworld/CarbonVillain-en-10.7B-v4", - "name": "CarbonVillain-en-10.7B-v4", - "developer": "jeonsworld", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4579, - "hfopenllm_v2/BBH": 0.5168, - "hfopenllm_v2/MATH Level 5": 0.0468, - "hfopenllm_v2/GPQA": 0.3062, - "hfopenllm_v2/MUSR": 0.3965, - "hfopenllm_v2/MMLU-PRO": 0.3142 - } - }, - { - "id": "jiangxinyang-shanda/Homer-LLama3-8B", - "name": "Homer-LLama3-8B", - "developer": "jiangxinyang-shanda", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3992, - "hfopenllm_v2/BBH": 0.5173, - "hfopenllm_v2/MATH Level 5": 0.0861, - "hfopenllm_v2/GPQA": 0.297, - "hfopenllm_v2/MUSR": 0.4056, - "hfopenllm_v2/MMLU-PRO": 0.3139 - } - }, - { - "id": "jieliu/Storm-7B", - "name": "Storm-7B", - "developer": "jieliu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3424, - "hfopenllm_v2/BBH": 0.5187, - "hfopenllm_v2/MATH Level 5": 0.0612, - "hfopenllm_v2/GPQA": 0.3079, - "hfopenllm_v2/MUSR": 0.4429, - "hfopenllm_v2/MMLU-PRO": 0.3119 - } - }, - { - "id": "Jimmy19991222/llama-3-8b-instruct-gapo-v2-bert-f1-beta10-gamma0.3-lr1.0e-6-1minus-rerun", - "name": "llama-3-8b-instruct-gapo-v2-bert-f1-beta10-gamma0.3-lr1.0e-6-1minus-rerun", - "developer": "Jimmy19991222", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6717, - "hfopenllm_v2/BBH": 0.488, - "hfopenllm_v2/MATH Level 5": 0.0604, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.4041, - "hfopenllm_v2/MMLU-PRO": 0.3634 - } - }, - { - "id": "Jimmy19991222/llama-3-8b-instruct-gapo-v2-bert_f1-beta10-gamma0.3-lr1.0e-6-scale-log", - "name": "llama-3-8b-instruct-gapo-v2-bert_f1-beta10-gamma0.3-lr1.0e-6-scale-log", - "developer": "Jimmy19991222", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6556, - "hfopenllm_v2/BBH": 0.4935, - "hfopenllm_v2/MATH Level 5": 0.0544, - "hfopenllm_v2/GPQA": 0.3045, - "hfopenllm_v2/MUSR": 0.4, - "hfopenllm_v2/MMLU-PRO": 0.3658 - } - }, - { - "id": "Jimmy19991222/llama-3-8b-instruct-gapo-v2-bert_p-beta10-gamma0.3-lr1.0e-6-scale-log", - "name": "llama-3-8b-instruct-gapo-v2-bert_p-beta10-gamma0.3-lr1.0e-6-scale-log", - "developer": "Jimmy19991222", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6315, - "hfopenllm_v2/BBH": 0.4916, - "hfopenllm_v2/MATH Level 5": 0.065, - "hfopenllm_v2/GPQA": 0.2861, - "hfopenllm_v2/MUSR": 0.3935, - "hfopenllm_v2/MMLU-PRO": 0.3611 - } - }, - { - "id": "Jimmy19991222/llama-3-8b-instruct-gapo-v2-bleu-beta0.1-no-length-scale-gamma0.4", - "name": "llama-3-8b-instruct-gapo-v2-bleu-beta0.1-no-length-scale-gamma0.4", - "developer": "Jimmy19991222", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6285, - "hfopenllm_v2/BBH": 0.4986, - "hfopenllm_v2/MATH Level 5": 0.0514, - "hfopenllm_v2/GPQA": 0.2928, - "hfopenllm_v2/MUSR": 0.4014, - "hfopenllm_v2/MMLU-PRO": 0.3545 - } - }, - { - "id": "Jimmy19991222/llama-3-8b-instruct-gapo-v2-rouge2-beta10-1minus-gamma0.3-rerun", - "name": "llama-3-8b-instruct-gapo-v2-rouge2-beta10-1minus-gamma0.3-rerun", - "developer": "Jimmy19991222", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6678, - "hfopenllm_v2/BBH": 0.494, - "hfopenllm_v2/MATH Level 5": 0.0612, - "hfopenllm_v2/GPQA": 0.3062, - "hfopenllm_v2/MUSR": 0.3987, - "hfopenllm_v2/MMLU-PRO": 0.3658 - } - }, - { - "id": "Jimmy19991222/llama-3-8b-instruct-gapo-v2-rouge2-beta10-gamma0.3-lr1.0e-6-scale-log", - "name": "llama-3-8b-instruct-gapo-v2-rouge2-beta10-gamma0.3-lr1.0e-6-scale-log", - "developer": "Jimmy19991222", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6605, - "hfopenllm_v2/BBH": 0.4916, - "hfopenllm_v2/MATH Level 5": 0.0657, - "hfopenllm_v2/GPQA": 0.3037, - "hfopenllm_v2/MUSR": 0.4, - "hfopenllm_v2/MMLU-PRO": 0.3664 - } - }, - { - "id": "Jimmy19991222/llama-3-8b-instruct-gapo-v2-rougeL-beta10-gamma0.3-lr1.0e-6-scale-log", - "name": "llama-3-8b-instruct-gapo-v2-rougeL-beta10-gamma0.3-lr1.0e-6-scale-log", - "developer": "Jimmy19991222", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6492, - "hfopenllm_v2/BBH": 0.4952, - "hfopenllm_v2/MATH Level 5": 0.0642, - "hfopenllm_v2/GPQA": 0.302, - "hfopenllm_v2/MUSR": 0.3961, - "hfopenllm_v2/MMLU-PRO": 0.3711 - } - }, - { - "id": "Jimmy19991222/Llama-3-Instruct-8B-SimPO-v0.2", - "name": "Llama-3-Instruct-8B-SimPO-v0.2", - "developer": "Jimmy19991222", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.654, - "hfopenllm_v2/BBH": 0.4984, - "hfopenllm_v2/MATH Level 5": 0.0619, - "hfopenllm_v2/GPQA": 0.3146, - "hfopenllm_v2/MUSR": 0.4013, - "hfopenllm_v2/MMLU-PRO": 0.3686 - } - }, - { - "id": "jiviai/medX_v2", - "name": "medX_v2", - "developer": "jiviai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3743, - "hfopenllm_v2/BBH": 0.4509, - "hfopenllm_v2/MATH Level 5": 0.0544, - "hfopenllm_v2/GPQA": 0.323, - "hfopenllm_v2/MUSR": 0.3498, - "hfopenllm_v2/MMLU-PRO": 0.3428 - } - }, - { - "id": "jlzhou/Qwen2.5-3B-Infinity-Instruct-0625", - "name": "Qwen2.5-3B-Infinity-Instruct-0625", - "developer": "jlzhou", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3558, - "hfopenllm_v2/BBH": 0.4774, - "hfopenllm_v2/MATH Level 5": 0.1367, - "hfopenllm_v2/GPQA": 0.2693, - "hfopenllm_v2/MUSR": 0.3981, - "hfopenllm_v2/MMLU-PRO": 0.3199 - } - }, - { - "id": "johnsutor/Llama-3-8B-Instruct_breadcrumbs-density-0.1-gamma-0.01", - "name": "Llama-3-8B-Instruct_breadcrumbs-density-0.1-gamma-0.01", - "developer": "johnsutor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4271, - "hfopenllm_v2/BBH": 0.5036, - "hfopenllm_v2/MATH Level 5": 0.0453, - "hfopenllm_v2/GPQA": 0.3221, - "hfopenllm_v2/MUSR": 0.4638, - "hfopenllm_v2/MMLU-PRO": 0.3739 - } - }, - { - "id": "johnsutor/Llama-3-8B-Instruct_breadcrumbs-density-0.1-gamma-0.1", - "name": "Llama-3-8B-Instruct_breadcrumbs-density-0.1-gamma-0.1", - "developer": "johnsutor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4253, - "hfopenllm_v2/BBH": 0.5019, - "hfopenllm_v2/MATH Level 5": 0.0967, - "hfopenllm_v2/GPQA": 0.3012, - "hfopenllm_v2/MUSR": 0.415, - "hfopenllm_v2/MMLU-PRO": 0.3724 - } - }, - { - "id": "johnsutor/Llama-3-8B-Instruct_breadcrumbs-density-0.3-gamma-0.01", - "name": "Llama-3-8B-Instruct_breadcrumbs-density-0.3-gamma-0.01", - "developer": "johnsutor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3377, - "hfopenllm_v2/BBH": 0.4917, - "hfopenllm_v2/MATH Level 5": 0.0106, - "hfopenllm_v2/GPQA": 0.3121, - "hfopenllm_v2/MUSR": 0.5018, - "hfopenllm_v2/MMLU-PRO": 0.3533 - } - }, - { - "id": "johnsutor/Llama-3-8B-Instruct_breadcrumbs-density-0.3-gamma-0.1", - "name": "Llama-3-8B-Instruct_breadcrumbs-density-0.3-gamma-0.1", - "developer": "johnsutor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4274, - "hfopenllm_v2/BBH": 0.5126, - "hfopenllm_v2/MATH Level 5": 0.0808, - "hfopenllm_v2/GPQA": 0.3087, - "hfopenllm_v2/MUSR": 0.4226, - "hfopenllm_v2/MMLU-PRO": 0.3739 - } - }, - { - "id": "johnsutor/Llama-3-8B-Instruct_breadcrumbs-density-0.5-gamma-0.01", - "name": "Llama-3-8B-Instruct_breadcrumbs-density-0.5-gamma-0.01", - "developer": "johnsutor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3204, - "hfopenllm_v2/BBH": 0.4884, - "hfopenllm_v2/MATH Level 5": 0.0038, - "hfopenllm_v2/GPQA": 0.302, - "hfopenllm_v2/MUSR": 0.5098, - "hfopenllm_v2/MMLU-PRO": 0.3344 - } - }, - { - "id": "johnsutor/Llama-3-8B-Instruct_breadcrumbs-density-0.5-gamma-0.1", - "name": "Llama-3-8B-Instruct_breadcrumbs-density-0.5-gamma-0.1", - "developer": "johnsutor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4396, - "hfopenllm_v2/BBH": 0.514, - "hfopenllm_v2/MATH Level 5": 0.0801, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.4398, - "hfopenllm_v2/MMLU-PRO": 0.3696 - } - }, - { - "id": "johnsutor/Llama-3-8B-Instruct_breadcrumbs-density-0.7-gamma-0.01", - "name": "Llama-3-8B-Instruct_breadcrumbs-density-0.7-gamma-0.01", - "developer": "johnsutor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2814, - "hfopenllm_v2/BBH": 0.4854, - "hfopenllm_v2/MATH Level 5": 0.0023, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.5163, - "hfopenllm_v2/MMLU-PRO": 0.3295 - } - }, - { - "id": "johnsutor/Llama-3-8B-Instruct_breadcrumbs-density-0.7-gamma-0.1", - "name": "Llama-3-8B-Instruct_breadcrumbs-density-0.7-gamma-0.1", - "developer": "johnsutor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4302, - "hfopenllm_v2/BBH": 0.5157, - "hfopenllm_v2/MATH Level 5": 0.0627, - "hfopenllm_v2/GPQA": 0.3079, - "hfopenllm_v2/MUSR": 0.4332, - "hfopenllm_v2/MMLU-PRO": 0.3663 - } - }, - { - "id": "johnsutor/Llama-3-8B-Instruct_breadcrumbs-density-0.9-gamma-0.01", - "name": "Llama-3-8B-Instruct_breadcrumbs-density-0.9-gamma-0.01", - "developer": "johnsutor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.279, - "hfopenllm_v2/BBH": 0.4861, - "hfopenllm_v2/MATH Level 5": 0.0015, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.515, - "hfopenllm_v2/MMLU-PRO": 0.3305 - } - }, - { - "id": "johnsutor/Llama-3-8B-Instruct_breadcrumbs-density-0.9-gamma-0.1", - "name": "Llama-3-8B-Instruct_breadcrumbs-density-0.9-gamma-0.1", - "developer": "johnsutor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4223, - "hfopenllm_v2/BBH": 0.5154, - "hfopenllm_v2/MATH Level 5": 0.074, - "hfopenllm_v2/GPQA": 0.3079, - "hfopenllm_v2/MUSR": 0.4384, - "hfopenllm_v2/MMLU-PRO": 0.365 - } - }, - { - "id": "johnsutor/Llama-3-8B-Instruct_breadcrumbs_ties-density-0.1-gamma-0.01", - "name": "Llama-3-8B-Instruct_breadcrumbs_ties-density-0.1-gamma-0.01", - "developer": "johnsutor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4359, - "hfopenllm_v2/BBH": 0.5041, - "hfopenllm_v2/MATH Level 5": 0.0483, - "hfopenllm_v2/GPQA": 0.3104, - "hfopenllm_v2/MUSR": 0.4532, - "hfopenllm_v2/MMLU-PRO": 0.3762 - } - }, - { - "id": "johnsutor/Llama-3-8B-Instruct_breadcrumbs_ties-density-0.1-gamma-0.1", - "name": "Llama-3-8B-Instruct_breadcrumbs_ties-density-0.1-gamma-0.1", - "developer": "johnsutor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4202, - "hfopenllm_v2/BBH": 0.5011, - "hfopenllm_v2/MATH Level 5": 0.0982, - "hfopenllm_v2/GPQA": 0.3003, - "hfopenllm_v2/MUSR": 0.415, - "hfopenllm_v2/MMLU-PRO": 0.3699 - } - }, - { - "id": "johnsutor/Llama-3-8B-Instruct_breadcrumbs_ties-density-0.3-gamma-0.01", - "name": "Llama-3-8B-Instruct_breadcrumbs_ties-density-0.3-gamma-0.01", - "developer": "johnsutor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3518, - "hfopenllm_v2/BBH": 0.4999, - "hfopenllm_v2/MATH Level 5": 0.0234, - "hfopenllm_v2/GPQA": 0.3062, - "hfopenllm_v2/MUSR": 0.4871, - "hfopenllm_v2/MMLU-PRO": 0.3611 - } - }, - { - "id": "johnsutor/Llama-3-8B-Instruct_breadcrumbs_ties-density-0.3-gamma-0.1", - "name": "Llama-3-8B-Instruct_breadcrumbs_ties-density-0.3-gamma-0.1", - "developer": "johnsutor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4204, - "hfopenllm_v2/BBH": 0.5107, - "hfopenllm_v2/MATH Level 5": 0.0876, - "hfopenllm_v2/GPQA": 0.3045, - "hfopenllm_v2/MUSR": 0.4279, - "hfopenllm_v2/MMLU-PRO": 0.371 - } - }, - { - "id": "johnsutor/Llama-3-8B-Instruct_breadcrumbs_ties-density-0.5-gamma-0.01", - "name": "Llama-3-8B-Instruct_breadcrumbs_ties-density-0.5-gamma-0.01", - "developer": "johnsutor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3454, - "hfopenllm_v2/BBH": 0.4984, - "hfopenllm_v2/MATH Level 5": 0.0219, - "hfopenllm_v2/GPQA": 0.297, - "hfopenllm_v2/MUSR": 0.4911, - "hfopenllm_v2/MMLU-PRO": 0.3531 - } - }, - { - "id": "johnsutor/Llama-3-8B-Instruct_breadcrumbs_ties-density-0.5-gamma-0.1", - "name": "Llama-3-8B-Instruct_breadcrumbs_ties-density-0.5-gamma-0.1", - "developer": "johnsutor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4092, - "hfopenllm_v2/BBH": 0.5137, - "hfopenllm_v2/MATH Level 5": 0.0808, - "hfopenllm_v2/GPQA": 0.2953, - "hfopenllm_v2/MUSR": 0.4357, - "hfopenllm_v2/MMLU-PRO": 0.3669 - } - }, - { - "id": "johnsutor/Llama-3-8B-Instruct_breadcrumbs_ties-density-0.7-gamma-0.01", - "name": "Llama-3-8B-Instruct_breadcrumbs_ties-density-0.7-gamma-0.01", - "developer": "johnsutor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2904, - "hfopenllm_v2/BBH": 0.4967, - "hfopenllm_v2/MATH Level 5": 0.0144, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.4991, - "hfopenllm_v2/MMLU-PRO": 0.349 - } - }, - { - "id": "johnsutor/Llama-3-8B-Instruct_breadcrumbs_ties-density-0.7-gamma-0.1", - "name": "Llama-3-8B-Instruct_breadcrumbs_ties-density-0.7-gamma-0.1", - "developer": "johnsutor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4199, - "hfopenllm_v2/BBH": 0.5147, - "hfopenllm_v2/MATH Level 5": 0.0808, - "hfopenllm_v2/GPQA": 0.2987, - "hfopenllm_v2/MUSR": 0.4358, - "hfopenllm_v2/MMLU-PRO": 0.3615 - } - }, - { - "id": "johnsutor/Llama-3-8B-Instruct_breadcrumbs_ties-density-0.9-gamma-0.01", - "name": "Llama-3-8B-Instruct_breadcrumbs_ties-density-0.9-gamma-0.01", - "developer": "johnsutor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2913, - "hfopenllm_v2/BBH": 0.4918, - "hfopenllm_v2/MATH Level 5": 0.0106, - "hfopenllm_v2/GPQA": 0.3003, - "hfopenllm_v2/MUSR": 0.4977, - "hfopenllm_v2/MMLU-PRO": 0.3454 - } - }, - { - "id": "johnsutor/Llama-3-8B-Instruct_breadcrumbs_ties-density-0.9-gamma-0.1", - "name": "Llama-3-8B-Instruct_breadcrumbs_ties-density-0.9-gamma-0.1", - "developer": "johnsutor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4162, - "hfopenllm_v2/BBH": 0.5139, - "hfopenllm_v2/MATH Level 5": 0.0778, - "hfopenllm_v2/GPQA": 0.297, - "hfopenllm_v2/MUSR": 0.4317, - "hfopenllm_v2/MMLU-PRO": 0.3625 - } - }, - { - "id": "johnsutor/Llama-3-8B-Instruct_dare_linear", - "name": "Llama-3-8B-Instruct_dare_linear", - "developer": "johnsutor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2145, - "hfopenllm_v2/BBH": 0.4283, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.4979, - "hfopenllm_v2/MMLU-PRO": 0.2414 - } - }, - { - "id": "johnsutor/Llama-3-8B-Instruct_dare_ties-density-0.1", - "name": "Llama-3-8B-Instruct_dare_ties-density-0.1", - "developer": "johnsutor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1891, - "hfopenllm_v2/BBH": 0.4119, - "hfopenllm_v2/MATH Level 5": 0.0008, - "hfopenllm_v2/GPQA": 0.2718, - "hfopenllm_v2/MUSR": 0.4658, - "hfopenllm_v2/MMLU-PRO": 0.2265 - } - }, - { - "id": "johnsutor/Llama-3-8B-Instruct_dare_ties-density-0.3", - "name": "Llama-3-8B-Instruct_dare_ties-density-0.3", - "developer": "johnsutor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2113, - "hfopenllm_v2/BBH": 0.4559, - "hfopenllm_v2/MATH Level 5": 0.0015, - "hfopenllm_v2/GPQA": 0.297, - "hfopenllm_v2/MUSR": 0.5069, - "hfopenllm_v2/MMLU-PRO": 0.304 - } - }, - { - "id": "johnsutor/Llama-3-8B-Instruct_dare_ties-density-0.7", - "name": "Llama-3-8B-Instruct_dare_ties-density-0.7", - "developer": "johnsutor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2034, - "hfopenllm_v2/BBH": 0.4723, - "hfopenllm_v2/MATH Level 5": 0.003, - "hfopenllm_v2/GPQA": 0.3037, - "hfopenllm_v2/MUSR": 0.511, - "hfopenllm_v2/MMLU-PRO": 0.3148 - } - }, - { - "id": "johnsutor/Llama-3-8B-Instruct_dare_ties-density-0.9", - "name": "Llama-3-8B-Instruct_dare_ties-density-0.9", - "developer": "johnsutor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2161, - "hfopenllm_v2/BBH": 0.4664, - "hfopenllm_v2/MATH Level 5": 0.0015, - "hfopenllm_v2/GPQA": 0.3079, - "hfopenllm_v2/MUSR": 0.523, - "hfopenllm_v2/MMLU-PRO": 0.3143 - } - }, - { - "id": "johnsutor/Llama-3-8B-Instruct_linear", - "name": "Llama-3-8B-Instruct_linear", - "developer": "johnsutor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4308, - "hfopenllm_v2/BBH": 0.5031, - "hfopenllm_v2/MATH Level 5": 0.1005, - "hfopenllm_v2/GPQA": 0.2953, - "hfopenllm_v2/MUSR": 0.4097, - "hfopenllm_v2/MMLU-PRO": 0.3712 - } - }, - { - "id": "johnsutor/Llama-3-8B-Instruct_ties-density-0.1", - "name": "Llama-3-8B-Instruct_ties-density-0.1", - "developer": "johnsutor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4116, - "hfopenllm_v2/BBH": 0.5021, - "hfopenllm_v2/MATH Level 5": 0.0793, - "hfopenllm_v2/GPQA": 0.2886, - "hfopenllm_v2/MUSR": 0.4174, - "hfopenllm_v2/MMLU-PRO": 0.36 - } - }, - { - "id": "johnsutor/Llama-3-8B-Instruct_ties-density-0.3", - "name": "Llama-3-8B-Instruct_ties-density-0.3", - "developer": "johnsutor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3626, - "hfopenllm_v2/BBH": 0.4906, - "hfopenllm_v2/MATH Level 5": 0.0672, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.4025, - "hfopenllm_v2/MMLU-PRO": 0.3321 - } - }, - { - "id": "johnsutor/Llama-3-8B-Instruct_ties-density-0.5", - "name": "Llama-3-8B-Instruct_ties-density-0.5", - "developer": "johnsutor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3797, - "hfopenllm_v2/BBH": 0.4793, - "hfopenllm_v2/MATH Level 5": 0.0612, - "hfopenllm_v2/GPQA": 0.3045, - "hfopenllm_v2/MUSR": 0.388, - "hfopenllm_v2/MMLU-PRO": 0.3175 - } - }, - { - "id": "johnsutor/Llama-3-8B-Instruct_ties-density-0.7", - "name": "Llama-3-8B-Instruct_ties-density-0.7", - "developer": "johnsutor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3681, - "hfopenllm_v2/BBH": 0.4738, - "hfopenllm_v2/MATH Level 5": 0.0672, - "hfopenllm_v2/GPQA": 0.3096, - "hfopenllm_v2/MUSR": 0.3881, - "hfopenllm_v2/MMLU-PRO": 0.3152 - } - }, - { - "id": "johnsutor/Llama-3-8B-Instruct_ties-density-0.9", - "name": "Llama-3-8B-Instruct_ties-density-0.9", - "developer": "johnsutor", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3858, - "hfopenllm_v2/BBH": 0.4735, - "hfopenllm_v2/MATH Level 5": 0.0619, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.388, - "hfopenllm_v2/MMLU-PRO": 0.3182 - } - }, - { - "id": "jondurbin/bagel-dpo-34b-v0.5", - "name": "jondurbin/bagel-dpo-34b-v0.5", - "developer": "jondurbin", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7215, - "reward-bench/Chat": 0.9385, - "reward-bench/Chat Hard": 0.5504, - "reward-bench/Safety": 0.6446, - "reward-bench/Reasoning": 0.8889, - "reward-bench/Prior Sets (0.5 weight)": 0.4487 - } - }, - { - "id": "Joseph717171/Hermes-3-Llama-3.1-8B_TIES_with_Base_Embeds_Initialized_to_Special_Instruct_Toks_dtypeF32", - "name": "Hermes-3-Llama-3.1-8B_TIES_with_Base_Embeds_Initialized_to_Special_Instruct_Toks_dtypeF32", - "developer": "Joseph717171", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6185, - "hfopenllm_v2/BBH": 0.5177, - "hfopenllm_v2/MATH Level 5": 0.0514, - "hfopenllm_v2/GPQA": 0.2827, - "hfopenllm_v2/MUSR": 0.4369, - "hfopenllm_v2/MMLU-PRO": 0.3144 - } - }, - { - "id": "Joseph717171/Llama-3.1-SuperNova-8B-Lite_TIES_with_Base", - "name": "Llama-3.1-SuperNova-8B-Lite_TIES_with_Base", - "developer": "Joseph717171", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8096, - "hfopenllm_v2/BBH": 0.5147, - "hfopenllm_v2/MATH Level 5": 0.1835, - "hfopenllm_v2/GPQA": 0.3096, - "hfopenllm_v2/MUSR": 0.411, - "hfopenllm_v2/MMLU-PRO": 0.388 - } - }, - { - "id": "Josephgflowers/Cinder-Phi-2-V1-F16-gguf", - "name": "Cinder-Phi-2-V1-F16-gguf", - "developer": "Josephgflowers", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2357, - "hfopenllm_v2/BBH": 0.4397, - "hfopenllm_v2/MATH Level 5": 0.0242, - "hfopenllm_v2/GPQA": 0.2819, - "hfopenllm_v2/MUSR": 0.3435, - "hfopenllm_v2/MMLU-PRO": 0.2161 - } - }, - { - "id": "Josephgflowers/Differential-Attention-Liquid-Metal-Tinyllama", - "name": "Differential-Attention-Liquid-Metal-Tinyllama", - "developer": "Josephgflowers", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2227, - "hfopenllm_v2/BBH": 0.2926, - "hfopenllm_v2/MATH Level 5": 0.0325, - "hfopenllm_v2/GPQA": 0.2508, - "hfopenllm_v2/MUSR": 0.3356, - "hfopenllm_v2/MMLU-PRO": 0.1214 - } - }, - { - "id": "Josephgflowers/TinyLlama-Cinder-Agent-v1", - "name": "TinyLlama-Cinder-Agent-v1", - "developer": "Josephgflowers", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.267, - "hfopenllm_v2/BBH": 0.3116, - "hfopenllm_v2/MATH Level 5": 0.0347, - "hfopenllm_v2/GPQA": 0.2441, - "hfopenllm_v2/MUSR": 0.3395, - "hfopenllm_v2/MMLU-PRO": 0.1161 - } - }, - { - "id": "Josephgflowers/Tinyllama-r1", - "name": "Tinyllama-r1", - "developer": "Josephgflowers", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2119, - "hfopenllm_v2/BBH": 0.3015, - "hfopenllm_v2/MATH Level 5": 0.0325, - "hfopenllm_v2/GPQA": 0.2567, - "hfopenllm_v2/MUSR": 0.3315, - "hfopenllm_v2/MMLU-PRO": 0.1134 - } - }, - { - "id": "Josephgflowers/Tinyllama-STEM-Cinder-Agent-v1", - "name": "Tinyllama-STEM-Cinder-Agent-v1", - "developer": "Josephgflowers", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2126, - "hfopenllm_v2/BBH": 0.3084, - "hfopenllm_v2/MATH Level 5": 0.0672, - "hfopenllm_v2/GPQA": 0.2349, - "hfopenllm_v2/MUSR": 0.3341, - "hfopenllm_v2/MMLU-PRO": 0.1086 - } - }, - { - "id": "Josephgflowers/TinyLlama-v1.1-Cinders-World", - "name": "TinyLlama-v1.1-Cinders-World", - "developer": "Josephgflowers", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2469, - "hfopenllm_v2/BBH": 0.2998, - "hfopenllm_v2/MATH Level 5": 0.0347, - "hfopenllm_v2/GPQA": 0.2441, - "hfopenllm_v2/MUSR": 0.3356, - "hfopenllm_v2/MMLU-PRO": 0.1198 - } - }, - { - "id": "Josephgflowers/TinyLlama_v1.1_math_code-world-test-1", - "name": "TinyLlama_v1.1_math_code-world-test-1", - "developer": "Josephgflowers", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0078, - "hfopenllm_v2/BBH": 0.3146, - "hfopenllm_v2/MATH Level 5": 0.0196, - "hfopenllm_v2/GPQA": 0.2341, - "hfopenllm_v2/MUSR": 0.3499, - "hfopenllm_v2/MMLU-PRO": 0.1132 - } - }, - { - "id": "jpacifico/Chocolatine-14B-Instruct-4k-DPO", - "name": "Chocolatine-14B-Instruct-4k-DPO", - "developer": "jpacifico", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4689, - "hfopenllm_v2/BBH": 0.63, - "hfopenllm_v2/MATH Level 5": 0.1782, - "hfopenllm_v2/GPQA": 0.3414, - "hfopenllm_v2/MUSR": 0.4439, - "hfopenllm_v2/MMLU-PRO": 0.4764 - } - }, - { - "id": "jpacifico/Chocolatine-14B-Instruct-DPO-v1.2", - "name": "Chocolatine-14B-Instruct-DPO-v1.2", - "developer": "jpacifico", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6852, - "hfopenllm_v2/BBH": 0.6438, - "hfopenllm_v2/MATH Level 5": 0.2092, - "hfopenllm_v2/GPQA": 0.3255, - "hfopenllm_v2/MUSR": 0.4268, - "hfopenllm_v2/MMLU-PRO": 0.4697 - } - }, - { - "id": "jpacifico/Chocolatine-14B-Instruct-DPO-v1.3", - "name": "Chocolatine-14B-Instruct-DPO-v1.3", - "developer": "jpacifico", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.704, - "hfopenllm_v2/BBH": 0.6846, - "hfopenllm_v2/MATH Level 5": 0.5619, - "hfopenllm_v2/GPQA": 0.3414, - "hfopenllm_v2/MUSR": 0.4234, - "hfopenllm_v2/MMLU-PRO": 0.5374 - } - }, - { - "id": "jpacifico/Chocolatine-2-14B-Instruct-DPO-v2.0b1", - "name": "Chocolatine-2-14B-Instruct-DPO-v2.0b1", - "developer": "jpacifico", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1033, - "hfopenllm_v2/BBH": 0.6696, - "hfopenllm_v2/MATH Level 5": 0.2757, - "hfopenllm_v2/GPQA": 0.3758, - "hfopenllm_v2/MUSR": 0.4467, - "hfopenllm_v2/MMLU-PRO": 0.5124 - } - }, - { - "id": "jpacifico/Chocolatine-2-14B-Instruct-v2.0", - "name": "Chocolatine-2-14B-Instruct-v2.0", - "developer": "jpacifico", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0885, - "hfopenllm_v2/BBH": 0.677, - "hfopenllm_v2/MATH Level 5": 0.4804, - "hfopenllm_v2/GPQA": 0.3876, - "hfopenllm_v2/MUSR": 0.5021, - "hfopenllm_v2/MMLU-PRO": 0.5302 - } - }, - { - "id": "jpacifico/Chocolatine-2-14B-Instruct-v2.0.1", - "name": "Chocolatine-2-14B-Instruct-v2.0.1", - "developer": "jpacifico", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0742, - "hfopenllm_v2/BBH": 0.6736, - "hfopenllm_v2/MATH Level 5": 0.4796, - "hfopenllm_v2/GPQA": 0.3918, - "hfopenllm_v2/MUSR": 0.5008, - "hfopenllm_v2/MMLU-PRO": 0.5299 - } - }, - { - "id": "jpacifico/Chocolatine-2-14B-Instruct-v2.0.3", - "name": "Chocolatine-2-14B-Instruct-v2.0.3", - "developer": "jpacifico", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7037, - "hfopenllm_v2/BBH": 0.6548, - "hfopenllm_v2/MATH Level 5": 0.4207, - "hfopenllm_v2/GPQA": 0.3792, - "hfopenllm_v2/MUSR": 0.4768, - "hfopenllm_v2/MMLU-PRO": 0.5374 - } - }, - { - "id": "jpacifico/Chocolatine-2-14B-Instruct-v2.0b2", - "name": "Chocolatine-2-14B-Instruct-v2.0b2", - "developer": "jpacifico", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7241, - "hfopenllm_v2/BBH": 0.6476, - "hfopenllm_v2/MATH Level 5": 0.395, - "hfopenllm_v2/GPQA": 0.3834, - "hfopenllm_v2/MUSR": 0.4808, - "hfopenllm_v2/MMLU-PRO": 0.5369 - } - }, - { - "id": "jpacifico/Chocolatine-2-14B-Instruct-v2.0b3", - "name": "Chocolatine-2-14B-Instruct-v2.0b3", - "developer": "jpacifico", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7323, - "hfopenllm_v2/BBH": 0.6469, - "hfopenllm_v2/MATH Level 5": 0.4109, - "hfopenllm_v2/GPQA": 0.3792, - "hfopenllm_v2/MUSR": 0.4781, - "hfopenllm_v2/MMLU-PRO": 0.5337 - } - }, - { - "id": "jpacifico/Chocolatine-3B-Instruct-DPO-Revised", - "name": "Chocolatine-3B-Instruct-DPO-Revised", - "developer": "jpacifico", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5623, - "hfopenllm_v2/BBH": 0.554, - "hfopenllm_v2/MATH Level 5": 0.1805, - "hfopenllm_v2/GPQA": 0.3221, - "hfopenllm_v2/MUSR": 0.4453, - "hfopenllm_v2/MMLU-PRO": 0.3989 - } - }, - { - "id": "jpacifico/Chocolatine-3B-Instruct-DPO-v1.0", - "name": "Chocolatine-3B-Instruct-DPO-v1.0", - "developer": "jpacifico", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3737, - "hfopenllm_v2/BBH": 0.5471, - "hfopenllm_v2/MATH Level 5": 0.1782, - "hfopenllm_v2/GPQA": 0.3154, - "hfopenllm_v2/MUSR": 0.4755, - "hfopenllm_v2/MMLU-PRO": 0.3937 - } - }, - { - "id": "jpacifico/Chocolatine-3B-Instruct-DPO-v1.2", - "name": "Chocolatine-3B-Instruct-DPO-v1.2", - "developer": "jpacifico", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5455, - "hfopenllm_v2/BBH": 0.5487, - "hfopenllm_v2/MATH Level 5": 0.2047, - "hfopenllm_v2/GPQA": 0.3389, - "hfopenllm_v2/MUSR": 0.4154, - "hfopenllm_v2/MMLU-PRO": 0.3877 - } - }, - { - "id": "jpacifico/Distilucie-7B-Math-Instruct-DPO-v0.1", - "name": "Distilucie-7B-Math-Instruct-DPO-v0.1", - "developer": "jpacifico", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3048, - "hfopenllm_v2/BBH": 0.3835, - "hfopenllm_v2/MATH Level 5": 0.0257, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.3644, - "hfopenllm_v2/MMLU-PRO": 0.1809 - } - }, - { - "id": "jpacifico/Lucie-7B-Instruct-DPO-v1.1", - "name": "Lucie-7B-Instruct-DPO-v1.1", - "developer": "jpacifico", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3121, - "hfopenllm_v2/BBH": 0.3781, - "hfopenllm_v2/MATH Level 5": 0.0234, - "hfopenllm_v2/GPQA": 0.2878, - "hfopenllm_v2/MUSR": 0.4016, - "hfopenllm_v2/MMLU-PRO": 0.1838 - } - }, - { - "id": "jpacifico/Lucie-7B-Instruct-DPO-v1.1.3", - "name": "Lucie-7B-Instruct-DPO-v1.1.3", - "developer": "jpacifico", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3045, - "hfopenllm_v2/BBH": 0.3819, - "hfopenllm_v2/MATH Level 5": 0.0242, - "hfopenllm_v2/GPQA": 0.2861, - "hfopenllm_v2/MUSR": 0.3818, - "hfopenllm_v2/MMLU-PRO": 0.1764 - } - }, - { - "id": "jpacifico/Lucie-7B-Instruct-Merged-Model_Stock-v1.0", - "name": "Lucie-7B-Instruct-Merged-Model_Stock-v1.0", - "developer": "jpacifico", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3234, - "hfopenllm_v2/BBH": 0.3802, - "hfopenllm_v2/MATH Level 5": 0.0242, - "hfopenllm_v2/GPQA": 0.2886, - "hfopenllm_v2/MUSR": 0.3844, - "hfopenllm_v2/MMLU-PRO": 0.1871 - } - }, - { - "id": "jpacifico/Lucie-7B-Instruct-Merged-Model_Stock-v1.1", - "name": "Lucie-7B-Instruct-Merged-Model_Stock-v1.1", - "developer": "jpacifico", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3014, - "hfopenllm_v2/BBH": 0.3808, - "hfopenllm_v2/MATH Level 5": 0.0279, - "hfopenllm_v2/GPQA": 0.2827, - "hfopenllm_v2/MUSR": 0.375, - "hfopenllm_v2/MMLU-PRO": 0.1862 - } - }, - { - "id": "jpacifico/Lucie-Boosted-7B-Instruct", - "name": "Lucie-Boosted-7B-Instruct", - "developer": "jpacifico", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2566, - "hfopenllm_v2/BBH": 0.3465, - "hfopenllm_v2/MATH Level 5": 0.0128, - "hfopenllm_v2/GPQA": 0.2668, - "hfopenllm_v2/MUSR": 0.3699, - "hfopenllm_v2/MMLU-PRO": 0.163 - } - }, - { - "id": "jsfs11/L3-8B-Stheno-slerp", - "name": "L3-8B-Stheno-slerp", - "developer": "jsfs11", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6752, - "hfopenllm_v2/BBH": 0.5326, - "hfopenllm_v2/MATH Level 5": 0.0989, - "hfopenllm_v2/GPQA": 0.2852, - "hfopenllm_v2/MUSR": 0.3725, - "hfopenllm_v2/MMLU-PRO": 0.3649 - } - }, - { - "id": "jsfs11/MixtureofMerges-MoE-4x7b-v4", - "name": "MixtureofMerges-MoE-4x7b-v4", - "developer": "jsfs11", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.403, - "hfopenllm_v2/BBH": 0.5169, - "hfopenllm_v2/MATH Level 5": 0.0634, - "hfopenllm_v2/GPQA": 0.2861, - "hfopenllm_v2/MUSR": 0.4386, - "hfopenllm_v2/MMLU-PRO": 0.3032 - } - }, - { - "id": "jsfs11/MixtureofMerges-MoE-4x7b-v5", - "name": "MixtureofMerges-MoE-4x7b-v5", - "developer": "jsfs11", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4199, - "hfopenllm_v2/BBH": 0.5198, - "hfopenllm_v2/MATH Level 5": 0.0755, - "hfopenllm_v2/GPQA": 0.2844, - "hfopenllm_v2/MUSR": 0.4305, - "hfopenllm_v2/MMLU-PRO": 0.3098 - } - }, - { - "id": "JungZoona/T3Q-Qwen2.5-14B-Instruct-1M-e3", - "name": "T3Q-Qwen2.5-14B-Instruct-1M-e3", - "developer": "JungZoona", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7324, - "hfopenllm_v2/BBH": 0.7586, - "hfopenllm_v2/MATH Level 5": 0.2863, - "hfopenllm_v2/GPQA": 0.4169, - "hfopenllm_v2/MUSR": 0.5911, - "hfopenllm_v2/MMLU-PRO": 0.5884 - } - }, - { - "id": "JungZoona/T3Q-qwen2.5-14b-v1.0-e3", - "name": "T3Q-qwen2.5-14b-v1.0-e3", - "developer": "JungZoona", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7324, - "hfopenllm_v2/BBH": 0.7586, - "hfopenllm_v2/MATH Level 5": 0.2863, - "hfopenllm_v2/GPQA": 0.4169, - "hfopenllm_v2/MUSR": 0.5911, - "hfopenllm_v2/MMLU-PRO": 0.5884 - } - }, - { - "id": "Junhoee/Qwen-Megumin", - "name": "Qwen-Megumin", - "developer": "Junhoee", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7141, - "hfopenllm_v2/BBH": 0.5285, - "hfopenllm_v2/MATH Level 5": 0.4902, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.398, - "hfopenllm_v2/MMLU-PRO": 0.4199 - } - }, - { - "id": "kaist-ai/janus-7b", - "name": "janus-7b", - "developer": "kaist-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3775, - "hfopenllm_v2/BBH": 0.4694, - "hfopenllm_v2/MATH Level 5": 0.0408, - "hfopenllm_v2/GPQA": 0.2727, - "hfopenllm_v2/MUSR": 0.4401, - "hfopenllm_v2/MMLU-PRO": 0.2874 - } - }, - { - "id": "kaist-ai/janus-dpo-7b", - "name": "janus-dpo-7b", - "developer": "kaist-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4003, - "hfopenllm_v2/BBH": 0.4773, - "hfopenllm_v2/MATH Level 5": 0.0415, - "hfopenllm_v2/GPQA": 0.2819, - "hfopenllm_v2/MUSR": 0.4387, - "hfopenllm_v2/MMLU-PRO": 0.2976 - } - }, - { - "id": "kaist-ai/janus-rm-7b", - "name": "janus-rm-7b", - "developer": "kaist-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1778, - "hfopenllm_v2/BBH": 0.3056, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2517, - "hfopenllm_v2/MUSR": 0.3883, - "hfopenllm_v2/MMLU-PRO": 0.1126 - } - }, - { - "id": "kaist-ai/mistral-orpo-capybara-7k", - "name": "mistral-orpo-capybara-7k", - "developer": "kaist-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5367, - "hfopenllm_v2/BBH": 0.4489, - "hfopenllm_v2/MATH Level 5": 0.0393, - "hfopenllm_v2/GPQA": 0.2861, - "hfopenllm_v2/MUSR": 0.3964, - "hfopenllm_v2/MMLU-PRO": 0.2971 - } - }, - { - "id": "katanemo/arch-agent-1-5b", - "name": "Arch-Agent-1.5B", - "developer": "katanemo", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 60.0, - "bfcl/bfcl.overall.overall_accuracy": 32.14, - "bfcl/bfcl.overall.total_cost_usd": 2.45, - "bfcl/bfcl.overall.latency_mean_s": 2.38, - "bfcl/bfcl.overall.latency_std_s": 4.01, - "bfcl/bfcl.overall.latency_p95_s": 5.3, - "bfcl/bfcl.non_live.ast_accuracy": 82.67, - "bfcl/bfcl.non_live.simple_ast_accuracy": 72.17, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 92.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 85.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 81.0, - "bfcl/bfcl.live.live_accuracy": 67.73, - "bfcl/bfcl.live.live_simple_ast_accuracy": 70.54, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 67.81, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 31.25, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 58.33, - "bfcl/bfcl.multi_turn.accuracy": 26.62, - "bfcl/bfcl.multi_turn.base_accuracy": 35.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 27.5, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 21.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 22.0, - "bfcl/bfcl.web_search.accuracy": 0.0, - "bfcl/bfcl.web_search.base_accuracy": 0.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 0.0, - "bfcl/bfcl.memory.accuracy": 8.17, - "bfcl/bfcl.memory.kv_accuracy": 5.81, - "bfcl/bfcl.memory.vector_accuracy": 5.81, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 12.9, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 75.0, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 74.83 - } - }, - { - "id": "katanemo/arch-agent-32b", - "name": "Arch-Agent-32B", - "developer": "katanemo", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 37.0, - "bfcl/bfcl.overall.overall_accuracy": 45.37, - "bfcl/bfcl.overall.total_cost_usd": 8.87, - "bfcl/bfcl.overall.latency_mean_s": 9.44, - "bfcl/bfcl.overall.latency_std_s": 21.44, - "bfcl/bfcl.overall.latency_p95_s": 24.87, - "bfcl/bfcl.non_live.ast_accuracy": 88.92, - "bfcl/bfcl.non_live.simple_ast_accuracy": 76.67, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 94.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 96.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 89.0, - "bfcl/bfcl.live.live_accuracy": 80.68, - "bfcl/bfcl.live.live_simple_ast_accuracy": 86.43, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 79.11, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 93.75, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 79.17, - "bfcl/bfcl.multi_turn.accuracy": 54.25, - "bfcl/bfcl.multi_turn.base_accuracy": 64.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 58.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 53.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 41.5, - "bfcl/bfcl.web_search.accuracy": 5.0, - "bfcl/bfcl.web_search.base_accuracy": 4.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 6.0, - "bfcl/bfcl.memory.accuracy": 14.62, - "bfcl/bfcl.memory.kv_accuracy": 5.81, - "bfcl/bfcl.memory.vector_accuracy": 9.03, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 29.03, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 81.25, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 82.15 - } - }, - { - "id": "katanemo/arch-agent-3b", - "name": "Arch-Agent-3B", - "developer": "katanemo", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 56.0, - "bfcl/bfcl.overall.overall_accuracy": 35.36, - "bfcl/bfcl.overall.total_cost_usd": 3.7, - "bfcl/bfcl.overall.latency_mean_s": 3.56, - "bfcl/bfcl.overall.latency_std_s": 6.65, - "bfcl/bfcl.overall.latency_p95_s": 8.19, - "bfcl/bfcl.non_live.ast_accuracy": 86.67, - "bfcl/bfcl.non_live.simple_ast_accuracy": 78.67, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 94.5, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 91.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 82.5, - "bfcl/bfcl.live.live_accuracy": 72.91, - "bfcl/bfcl.live.live_simple_ast_accuracy": 75.58, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 72.27, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 68.75, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 75.0, - "bfcl/bfcl.multi_turn.accuracy": 34.88, - "bfcl/bfcl.multi_turn.base_accuracy": 42.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 37.5, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 31.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 29.0, - "bfcl/bfcl.web_search.accuracy": 0.5, - "bfcl/bfcl.web_search.base_accuracy": 1.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 0.0, - "bfcl/bfcl.memory.accuracy": 6.88, - "bfcl/bfcl.memory.kv_accuracy": 5.16, - "bfcl/bfcl.memory.vector_accuracy": 5.81, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 9.68, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 68.75, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 74.67 - } - }, - { - "id": "kavonalds/BunderMaxx-0710", - "name": "BunderMaxx-0710", - "developer": "kavonalds", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2701, - "hfopenllm_v2/BBH": 0.5566, - "hfopenllm_v2/MATH Level 5": 0.068, - "hfopenllm_v2/GPQA": 0.2802, - "hfopenllm_v2/MUSR": 0.3682, - "hfopenllm_v2/MMLU-PRO": 0.1449 - } - }, - { - "id": "kavonalds/BunderMaxx-1010", - "name": "BunderMaxx-1010", - "developer": "kavonalds", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2981, - "hfopenllm_v2/BBH": 0.702, - "hfopenllm_v2/MATH Level 5": 0.105, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.3484, - "hfopenllm_v2/MMLU-PRO": 0.1224 - } - }, - { - "id": "kavonalds/Lancer-1-1b-Instruct", - "name": "Lancer-1-1b-Instruct", - "developer": "kavonalds", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5546, - "hfopenllm_v2/BBH": 0.3253, - "hfopenllm_v2/MATH Level 5": 0.0393, - "hfopenllm_v2/GPQA": 0.2617, - "hfopenllm_v2/MUSR": 0.3144, - "hfopenllm_v2/MMLU-PRO": 0.1568 - } - }, - { - "id": "kayfour/T3Q-Qwen2.5-7B-it-KOR-Safe", - "name": "T3Q-Qwen2.5-7B-it-KOR-Safe", - "developer": "kayfour", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6081, - "hfopenllm_v2/BBH": 0.555, - "hfopenllm_v2/MATH Level 5": 0.3761, - "hfopenllm_v2/GPQA": 0.3213, - "hfopenllm_v2/MUSR": 0.4277, - "hfopenllm_v2/MMLU-PRO": 0.4464 - } - }, - { - "id": "keeeeenw/MicroLlama", - "name": "MicroLlama", - "developer": "keeeeenw", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1985, - "hfopenllm_v2/BBH": 0.3007, - "hfopenllm_v2/MATH Level 5": 0.0113, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.3698, - "hfopenllm_v2/MMLU-PRO": 0.1138 - } - }, - { - "id": "kekmodel/StopCarbon-10.7B-v5", - "name": "StopCarbon-10.7B-v5", - "developer": "kekmodel", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4728, - "hfopenllm_v2/BBH": 0.5178, - "hfopenllm_v2/MATH Level 5": 0.0559, - "hfopenllm_v2/GPQA": 0.3062, - "hfopenllm_v2/MUSR": 0.4019, - "hfopenllm_v2/MMLU-PRO": 0.3157 - } - }, - { - "id": "kevin009/llamaRAGdrama", - "name": "llamaRAGdrama", - "developer": "kevin009", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2598, - "hfopenllm_v2/BBH": 0.4007, - "hfopenllm_v2/MATH Level 5": 0.0431, - "hfopenllm_v2/GPQA": 0.2643, - "hfopenllm_v2/MUSR": 0.4316, - "hfopenllm_v2/MMLU-PRO": 0.2724 - } - }, - { - "id": "Khetterman/DarkAtom-12B-v3", - "name": "DarkAtom-12B-v3", - "developer": "Khetterman", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6173, - "hfopenllm_v2/BBH": 0.5154, - "hfopenllm_v2/MATH Level 5": 0.111, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.4468, - "hfopenllm_v2/MMLU-PRO": 0.3546 - } - }, - { - "id": "Khetterman/Kosmos-8B-v1", - "name": "Kosmos-8B-v1", - "developer": "Khetterman", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4129, - "hfopenllm_v2/BBH": 0.5234, - "hfopenllm_v2/MATH Level 5": 0.0989, - "hfopenllm_v2/GPQA": 0.2987, - "hfopenllm_v2/MUSR": 0.3919, - "hfopenllm_v2/MMLU-PRO": 0.3669 - } - }, - { - "id": "khoantap/cheap-moe-merge", - "name": "cheap-moe-merge", - "developer": "khoantap", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4557, - "hfopenllm_v2/BBH": 0.5131, - "hfopenllm_v2/MATH Level 5": 0.0921, - "hfopenllm_v2/GPQA": 0.2953, - "hfopenllm_v2/MUSR": 0.4103, - "hfopenllm_v2/MMLU-PRO": 0.3339 - } - }, - { - "id": "khoantap/llama-3-8b-stock-merge", - "name": "llama-3-8b-stock-merge", - "developer": "khoantap", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4812, - "hfopenllm_v2/BBH": 0.5162, - "hfopenllm_v2/MATH Level 5": 0.1616, - "hfopenllm_v2/GPQA": 0.318, - "hfopenllm_v2/MUSR": 0.3946, - "hfopenllm_v2/MMLU-PRO": 0.38 - } - }, - { - "id": "khoantap/llama-breadcrumbs-ties-merge", - "name": "llama-breadcrumbs-ties-merge", - "developer": "khoantap", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2205, - "hfopenllm_v2/BBH": 0.5416, - "hfopenllm_v2/MATH Level 5": 0.1125, - "hfopenllm_v2/GPQA": 0.2659, - "hfopenllm_v2/MUSR": 0.4434, - "hfopenllm_v2/MMLU-PRO": 0.3172 - } - }, - { - "id": "khoantap/llama-evolve-ties-best-merge", - "name": "llama-evolve-ties-best-merge", - "developer": "khoantap", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6744, - "hfopenllm_v2/BBH": 0.5414, - "hfopenllm_v2/MATH Level 5": 0.1563, - "hfopenllm_v2/GPQA": 0.3171, - "hfopenllm_v2/MUSR": 0.3946, - "hfopenllm_v2/MMLU-PRO": 0.386 - } - }, - { - "id": "khoantap/llama-linear-0.5-0.5-1-merge", - "name": "llama-linear-0.5-0.5-1-merge", - "developer": "khoantap", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4812, - "hfopenllm_v2/BBH": 0.5643, - "hfopenllm_v2/MATH Level 5": 0.2054, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.4143, - "hfopenllm_v2/MMLU-PRO": 0.3833 - } - }, - { - "id": "khoantap/llama-linear-0.5-1-0.5-merge", - "name": "llama-linear-0.5-1-0.5-merge", - "developer": "khoantap", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5032, - "hfopenllm_v2/BBH": 0.5951, - "hfopenllm_v2/MATH Level 5": 0.148, - "hfopenllm_v2/GPQA": 0.2936, - "hfopenllm_v2/MUSR": 0.4172, - "hfopenllm_v2/MMLU-PRO": 0.369 - } - }, - { - "id": "khoantap/llama-linear-1-0.5-0.5-merge", - "name": "llama-linear-1-0.5-0.5-merge", - "developer": "khoantap", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4515, - "hfopenllm_v2/BBH": 0.5526, - "hfopenllm_v2/MATH Level 5": 0.2477, - "hfopenllm_v2/GPQA": 0.2928, - "hfopenllm_v2/MUSR": 0.4118, - "hfopenllm_v2/MMLU-PRO": 0.3635 - } - }, - { - "id": "khoantap/llama-slerp-merge", - "name": "llama-slerp-merge", - "developer": "khoantap", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.498, - "hfopenllm_v2/BBH": 0.5783, - "hfopenllm_v2/MATH Level 5": 0.0831, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.4053, - "hfopenllm_v2/MMLU-PRO": 0.3678 - } - }, - { - "id": "khoantap/moe-out-merge", - "name": "moe-out-merge", - "developer": "khoantap", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4505, - "hfopenllm_v2/BBH": 0.5151, - "hfopenllm_v2/MATH Level 5": 0.0929, - "hfopenllm_v2/GPQA": 0.2886, - "hfopenllm_v2/MUSR": 0.4063, - "hfopenllm_v2/MMLU-PRO": 0.3348 - } - }, - { - "id": "khulaifi95/Llama-3.1-8B-Reason-Blend-888k", - "name": "Llama-3.1-8B-Reason-Blend-888k", - "developer": "khulaifi95", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5832, - "hfopenllm_v2/BBH": 0.479, - "hfopenllm_v2/MATH Level 5": 0.1156, - "hfopenllm_v2/GPQA": 0.2794, - "hfopenllm_v2/MUSR": 0.3379, - "hfopenllm_v2/MMLU-PRO": 0.31 - } - }, - { - "id": "Kimargin/GPT-NEO-1.3B-wiki", - "name": "GPT-NEO-1.3B-wiki", - "developer": "Kimargin", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1921, - "hfopenllm_v2/BBH": 0.3026, - "hfopenllm_v2/MATH Level 5": 0.0144, - "hfopenllm_v2/GPQA": 0.245, - "hfopenllm_v2/MUSR": 0.3883, - "hfopenllm_v2/MMLU-PRO": 0.1099 - } - }, - { - "id": "KingNish/qwen-1b-continued", - "name": "qwen-1b-continued", - "developer": "KingNish", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1255, - "hfopenllm_v2/BBH": 0.2991, - "hfopenllm_v2/MATH Level 5": 0.0091, - "hfopenllm_v2/GPQA": 0.2676, - "hfopenllm_v2/MUSR": 0.3859, - "hfopenllm_v2/MMLU-PRO": 0.1261 - } - }, - { - "id": "KingNish/qwen-1b-continued-v2", - "name": "qwen-1b-continued-v2", - "developer": "KingNish", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1579, - "hfopenllm_v2/BBH": 0.3119, - "hfopenllm_v2/MATH Level 5": 0.0106, - "hfopenllm_v2/GPQA": 0.25, - "hfopenllm_v2/MUSR": 0.3393, - "hfopenllm_v2/MMLU-PRO": 0.1193 - } - }, - { - "id": "KingNish/qwen-1b-continued-v2.1", - "name": "qwen-1b-continued-v2.1", - "developer": "KingNish", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1127, - "hfopenllm_v2/BBH": 0.3042, - "hfopenllm_v2/MATH Level 5": 0.0091, - "hfopenllm_v2/GPQA": 0.2676, - "hfopenllm_v2/MUSR": 0.4154, - "hfopenllm_v2/MMLU-PRO": 0.1278 - } - }, - { - "id": "KingNish/qwen-1b-continued-v2.2", - "name": "qwen-1b-continued-v2.2", - "developer": "KingNish", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1413, - "hfopenllm_v2/BBH": 0.3059, - "hfopenllm_v2/MATH Level 5": 0.0151, - "hfopenllm_v2/GPQA": 0.2567, - "hfopenllm_v2/MUSR": 0.3513, - "hfopenllm_v2/MMLU-PRO": 0.1262 - } - }, - { - "id": "KingNish/Qwen2.5-0.5b-Test-ft", - "name": "Qwen2.5-0.5b-Test-ft", - "developer": "KingNish", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2671, - "hfopenllm_v2/BBH": 0.3232, - "hfopenllm_v2/MATH Level 5": 0.0355, - "hfopenllm_v2/GPQA": 0.2634, - "hfopenllm_v2/MUSR": 0.3421, - "hfopenllm_v2/MMLU-PRO": 0.1689 - } - }, - { - "id": "KingNish/Reasoning-0.5b", - "name": "Reasoning-0.5b", - "developer": "KingNish", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2174, - "hfopenllm_v2/BBH": 0.3354, - "hfopenllm_v2/MATH Level 5": 0.0219, - "hfopenllm_v2/GPQA": 0.2676, - "hfopenllm_v2/MUSR": 0.3513, - "hfopenllm_v2/MMLU-PRO": 0.1641 - } - }, - { - "id": "KingNish/Reasoning-Llama-3b-v0.1", - "name": "Reasoning-Llama-3b-v0.1", - "developer": "KingNish", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6225, - "hfopenllm_v2/BBH": 0.4343, - "hfopenllm_v2/MATH Level 5": 0.1299, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3168, - "hfopenllm_v2/MMLU-PRO": 0.3029 - } - }, - { - "id": "kms7530/chemeng_llama-3-8b-Instruct-bnb-4bit_24_1_100_1", - "name": "chemeng_llama-3-8b-Instruct-bnb-4bit_24_1_100_1", - "developer": "kms7530", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5455, - "hfopenllm_v2/BBH": 0.4289, - "hfopenllm_v2/MATH Level 5": 0.0619, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.3821, - "hfopenllm_v2/MMLU-PRO": 0.2798 - } - }, - { - "id": "kms7530/chemeng_phi-3-mini-4k-instruct-bnb-4bit_16_4_100_1_nonmath", - "name": "chemeng_phi-3-mini-4k-instruct-bnb-4bit_16_4_100_1_nonmath", - "developer": "kms7530", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4863, - "hfopenllm_v2/BBH": 0.4987, - "hfopenllm_v2/MATH Level 5": 0.108, - "hfopenllm_v2/GPQA": 0.3104, - "hfopenllm_v2/MUSR": 0.3983, - "hfopenllm_v2/MMLU-PRO": 0.3481 - } - }, - { - "id": "kms7530/chemeng_qwen-math-7b_24_1_100_1", - "name": "chemeng_qwen-math-7b_24_1_100_1", - "developer": "kms7530", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2111, - "hfopenllm_v2/BBH": 0.3578, - "hfopenllm_v2/MATH Level 5": 0.2243, - "hfopenllm_v2/GPQA": 0.2441, - "hfopenllm_v2/MUSR": 0.3687, - "hfopenllm_v2/MMLU-PRO": 0.2158 - } - }, - { - "id": "kms7530/chemeng_qwen-math-7b_24_1_100_1_nonmath", - "name": "chemeng_qwen-math-7b_24_1_100_1_nonmath", - "developer": "kms7530", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2584, - "hfopenllm_v2/BBH": 0.3893, - "hfopenllm_v2/MATH Level 5": 0.3097, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.4087, - "hfopenllm_v2/MMLU-PRO": 0.2452 - } - }, - { - "id": "kno10/ende-chat-0.0.5", - "name": "ende-chat-0.0.5", - "developer": "kno10", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3404, - "hfopenllm_v2/BBH": 0.3604, - "hfopenllm_v2/MATH Level 5": 0.0204, - "hfopenllm_v2/GPQA": 0.2651, - "hfopenllm_v2/MUSR": 0.3938, - "hfopenllm_v2/MMLU-PRO": 0.179 - } - }, - { - "id": "kno10/ende-chat-0.0.7", - "name": "ende-chat-0.0.7", - "developer": "kno10", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4401, - "hfopenllm_v2/BBH": 0.3792, - "hfopenllm_v2/MATH Level 5": 0.0174, - "hfopenllm_v2/GPQA": 0.281, - "hfopenllm_v2/MUSR": 0.3861, - "hfopenllm_v2/MMLU-PRO": 0.1966 - } - }, - { - "id": "Kquant03/CognitiveFusion2-4x7B-BF16", - "name": "CognitiveFusion2-4x7B-BF16", - "developer": "Kquant03", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3567, - "hfopenllm_v2/BBH": 0.4108, - "hfopenllm_v2/MATH Level 5": 0.0574, - "hfopenllm_v2/GPQA": 0.2861, - "hfopenllm_v2/MUSR": 0.4146, - "hfopenllm_v2/MMLU-PRO": 0.2793 - } - }, - { - "id": "Kquant03/L3-Pneuma-8B", - "name": "L3-Pneuma-8B", - "developer": "Kquant03", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2374, - "hfopenllm_v2/BBH": 0.4955, - "hfopenllm_v2/MATH Level 5": 0.0506, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.4172, - "hfopenllm_v2/MMLU-PRO": 0.3184 - } - }, - { - "id": "Krystalan/DRT-o1-14B", - "name": "DRT-o1-14B", - "developer": "Krystalan", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4068, - "hfopenllm_v2/BBH": 0.6379, - "hfopenllm_v2/MATH Level 5": 0.4826, - "hfopenllm_v2/GPQA": 0.3523, - "hfopenllm_v2/MUSR": 0.4795, - "hfopenllm_v2/MMLU-PRO": 0.5179 - } - }, - { - "id": "Krystalan/DRT-o1-7B", - "name": "DRT-o1-7B", - "developer": "Krystalan", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3928, - "hfopenllm_v2/BBH": 0.5468, - "hfopenllm_v2/MATH Level 5": 0.4479, - "hfopenllm_v2/GPQA": 0.3213, - "hfopenllm_v2/MUSR": 0.5087, - "hfopenllm_v2/MMLU-PRO": 0.4151 - } - }, - { - "id": "KSU-HW-SEC/Llama3-70b-SVA-FT-1415", - "name": "Llama3-70b-SVA-FT-1415", - "developer": "KSU-HW-SEC", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.618, - "hfopenllm_v2/BBH": 0.665, - "hfopenllm_v2/MATH Level 5": 0.2198, - "hfopenllm_v2/GPQA": 0.375, - "hfopenllm_v2/MUSR": 0.4565, - "hfopenllm_v2/MMLU-PRO": 0.5243 - } - }, - { - "id": "KSU-HW-SEC/Llama3-70b-SVA-FT-500", - "name": "Llama3-70b-SVA-FT-500", - "developer": "KSU-HW-SEC", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6105, - "hfopenllm_v2/BBH": 0.6692, - "hfopenllm_v2/MATH Level 5": 0.2137, - "hfopenllm_v2/GPQA": 0.3809, - "hfopenllm_v2/MUSR": 0.4511, - "hfopenllm_v2/MMLU-PRO": 0.5227 - } - }, - { - "id": "KSU-HW-SEC/Llama3-70b-SVA-FT-final", - "name": "Llama3-70b-SVA-FT-final", - "developer": "KSU-HW-SEC", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6165, - "hfopenllm_v2/BBH": 0.665, - "hfopenllm_v2/MATH Level 5": 0.2198, - "hfopenllm_v2/GPQA": 0.375, - "hfopenllm_v2/MUSR": 0.4565, - "hfopenllm_v2/MMLU-PRO": 0.5243 - } - }, - { - "id": "KSU-HW-SEC/Llama3.1-70b-SVA-FT-1000step", - "name": "Llama3.1-70b-SVA-FT-1000step", - "developer": "KSU-HW-SEC", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7238, - "hfopenllm_v2/BBH": 0.6903, - "hfopenllm_v2/MATH Level 5": 0.321, - "hfopenllm_v2/GPQA": 0.396, - "hfopenllm_v2/MUSR": 0.4592, - "hfopenllm_v2/MMLU-PRO": 0.5252 - } - }, - { - "id": "kuaishou/kwaipilot-40b-0604", - "name": "kwaipilot-40b-0604", - "developer": "Kuaishou", - "evaluator_relationship": null, - "benchmark_scores": { - "livecodebenchpro/Hard Problems": 0.0, - "livecodebenchpro/Medium Problems": 0.07042253521126761, - "livecodebenchpro/Easy Problems": 0.056338028169014086 - } - }, - { - "id": "Kukedlc/NeuralExperiment-7b-MagicCoder-v7.5", - "name": "NeuralExperiment-7b-MagicCoder-v7.5", - "developer": "Kukedlc", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4553, - "hfopenllm_v2/BBH": 0.3988, - "hfopenllm_v2/MATH Level 5": 0.0665, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.4282, - "hfopenllm_v2/MMLU-PRO": 0.2824 - } - }, - { - "id": "Kukedlc/NeuralLLaMa-3-8b-DT-v0.1", - "name": "NeuralLLaMa-3-8b-DT-v0.1", - "developer": "Kukedlc", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4371, - "hfopenllm_v2/BBH": 0.4987, - "hfopenllm_v2/MATH Level 5": 0.0808, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.4071, - "hfopenllm_v2/MMLU-PRO": 0.3792 - } - }, - { - "id": "Kukedlc/NeuralLLaMa-3-8b-ORPO-v0.3", - "name": "NeuralLLaMa-3-8b-ORPO-v0.3", - "developer": "Kukedlc", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5276, - "hfopenllm_v2/BBH": 0.4557, - "hfopenllm_v2/MATH Level 5": 0.0483, - "hfopenllm_v2/GPQA": 0.2391, - "hfopenllm_v2/MUSR": 0.37, - "hfopenllm_v2/MMLU-PRO": 0.3057 - } - }, - { - "id": "Kukedlc/NeuralSynthesis-7B-v0.1", - "name": "NeuralSynthesis-7B-v0.1", - "developer": "Kukedlc", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4185, - "hfopenllm_v2/BBH": 0.5145, - "hfopenllm_v2/MATH Level 5": 0.0634, - "hfopenllm_v2/GPQA": 0.281, - "hfopenllm_v2/MUSR": 0.4333, - "hfopenllm_v2/MMLU-PRO": 0.3049 - } - }, - { - "id": "Kukedlc/NeuralSynthesis-7B-v0.3", - "name": "NeuralSynthesis-7B-v0.3", - "developer": "Kukedlc", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4078, - "hfopenllm_v2/BBH": 0.5138, - "hfopenllm_v2/MATH Level 5": 0.0778, - "hfopenllm_v2/GPQA": 0.2802, - "hfopenllm_v2/MUSR": 0.4346, - "hfopenllm_v2/MMLU-PRO": 0.305 - } - }, - { - "id": "Kukedlc/NeuralSynthesis-7b-v0.4-slerp", - "name": "NeuralSynthesis-7b-v0.4-slerp", - "developer": "Kukedlc", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3947, - "hfopenllm_v2/BBH": 0.5143, - "hfopenllm_v2/MATH Level 5": 0.0627, - "hfopenllm_v2/GPQA": 0.2777, - "hfopenllm_v2/MUSR": 0.4332, - "hfopenllm_v2/MMLU-PRO": 0.3043 - } - }, - { - "id": "Kukedlc/Qwen-2.5-7b-Spanish-o1-CoT", - "name": "Qwen-2.5-7b-Spanish-o1-CoT", - "developer": "Kukedlc", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.421, - "hfopenllm_v2/BBH": 0.5602, - "hfopenllm_v2/MATH Level 5": 0.2727, - "hfopenllm_v2/GPQA": 0.3205, - "hfopenllm_v2/MUSR": 0.4777, - "hfopenllm_v2/MMLU-PRO": 0.4363 - } - }, - { - "id": "Kumar955/Hemanth-llm", - "name": "Hemanth-llm", - "developer": "Kumar955", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5045, - "hfopenllm_v2/BBH": 0.5225, - "hfopenllm_v2/MATH Level 5": 0.0702, - "hfopenllm_v2/GPQA": 0.2827, - "hfopenllm_v2/MUSR": 0.4486, - "hfopenllm_v2/MMLU-PRO": 0.3113 - } - }, - { - "id": "kyutai/helium-1-preview-2b", - "name": "helium-1-preview-2b", - "developer": "kyutai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2614, - "hfopenllm_v2/BBH": 0.3638, - "hfopenllm_v2/MATH Level 5": 0.0136, - "hfopenllm_v2/GPQA": 0.2785, - "hfopenllm_v2/MUSR": 0.355, - "hfopenllm_v2/MMLU-PRO": 0.1873 - } - }, - { - "id": "kz919/QwQ-0.5B-Distilled-SFT", - "name": "QwQ-0.5B-Distilled-SFT", - "developer": "kz919", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3077, - "hfopenllm_v2/BBH": 0.3256, - "hfopenllm_v2/MATH Level 5": 0.074, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.3409, - "hfopenllm_v2/MMLU-PRO": 0.1587 - } - }, - { - "id": "L-RAGE/3_PRYMMAL-ECE-7B-SLERP-V1", - "name": "3_PRYMMAL-ECE-7B-SLERP-V1", - "developer": "L-RAGE", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2742, - "hfopenllm_v2/BBH": 0.4228, - "hfopenllm_v2/MATH Level 5": 0.108, - "hfopenllm_v2/GPQA": 0.2819, - "hfopenllm_v2/MUSR": 0.3841, - "hfopenllm_v2/MMLU-PRO": 0.2925 - } - }, - { - "id": "ladydaina/ECE-FDF", - "name": "ECE-FDF", - "developer": "ladydaina", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3728, - "hfopenllm_v2/BBH": 0.515, - "hfopenllm_v2/MATH Level 5": 0.0816, - "hfopenllm_v2/GPQA": 0.2827, - "hfopenllm_v2/MUSR": 0.4504, - "hfopenllm_v2/MMLU-PRO": 0.3007 - } - }, - { - "id": "laislemke/LLaMA-2-vicuna-7b-slerp", - "name": "LLaMA-2-vicuna-7b-slerp", - "developer": "laislemke", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2932, - "hfopenllm_v2/BBH": 0.2986, - "hfopenllm_v2/MATH Level 5": 0.0113, - "hfopenllm_v2/GPQA": 0.2735, - "hfopenllm_v2/MUSR": 0.3833, - "hfopenllm_v2/MMLU-PRO": 0.1342 - } - }, - { - "id": "lalainy/ECE-PRYMMAL-0.5B-FT-V5-MUSR", - "name": "ECE-PRYMMAL-0.5B-FT-V5-MUSR", - "developer": "lalainy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2138, - "hfopenllm_v2/BBH": 0.3269, - "hfopenllm_v2/MATH Level 5": 0.0453, - "hfopenllm_v2/GPQA": 0.2743, - "hfopenllm_v2/MUSR": 0.3262, - "hfopenllm_v2/MMLU-PRO": 0.1533 - } - }, - { - "id": "lalainy/ECE-PRYMMAL-0.5B-SLERP-V4", - "name": "ECE-PRYMMAL-0.5B-SLERP-V4", - "developer": "lalainy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1564, - "hfopenllm_v2/BBH": 0.2894, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2626, - "hfopenllm_v2/MUSR": 0.3789, - "hfopenllm_v2/MMLU-PRO": 0.1169 - } - }, - { - "id": "lalainy/ECE-PRYMMAL-YL-0.5B-SLERP-BIS-V1", - "name": "ECE-PRYMMAL-YL-0.5B-SLERP-BIS-V1", - "developer": "lalainy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1437, - "hfopenllm_v2/BBH": 0.3032, - "hfopenllm_v2/MATH Level 5": 0.0008, - "hfopenllm_v2/GPQA": 0.2349, - "hfopenllm_v2/MUSR": 0.3646, - "hfopenllm_v2/MMLU-PRO": 0.1121 - } - }, - { - "id": "lalainy/ECE-PRYMMAL-YL-1B-SLERP-V3", - "name": "ECE-PRYMMAL-YL-1B-SLERP-V3", - "developer": "lalainy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.325, - "hfopenllm_v2/BBH": 0.4225, - "hfopenllm_v2/MATH Level 5": 0.0974, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.4213, - "hfopenllm_v2/MMLU-PRO": 0.2931 - } - }, - { - "id": "lalainy/ECE-PRYMMAL-YL-1B-SLERP-V4", - "name": "ECE-PRYMMAL-YL-1B-SLERP-V4", - "developer": "lalainy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3324, - "hfopenllm_v2/BBH": 0.4171, - "hfopenllm_v2/MATH Level 5": 0.1005, - "hfopenllm_v2/GPQA": 0.2861, - "hfopenllm_v2/MUSR": 0.4306, - "hfopenllm_v2/MMLU-PRO": 0.2893 - } - }, - { - "id": "lalainy/ECE-PRYMMAL-YL-6B-SLERP-V1", - "name": "ECE-PRYMMAL-YL-6B-SLERP-V1", - "developer": "lalainy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3264, - "hfopenllm_v2/BBH": 0.4629, - "hfopenllm_v2/MATH Level 5": 0.1269, - "hfopenllm_v2/GPQA": 0.2886, - "hfopenllm_v2/MUSR": 0.4864, - "hfopenllm_v2/MMLU-PRO": 0.3214 - } - }, - { - "id": "lalainy/ECE-PRYMMAL-YL-6B-SLERP-V2", - "name": "ECE-PRYMMAL-YL-6B-SLERP-V2", - "developer": "lalainy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3249, - "hfopenllm_v2/BBH": 0.4629, - "hfopenllm_v2/MATH Level 5": 0.1269, - "hfopenllm_v2/GPQA": 0.2886, - "hfopenllm_v2/MUSR": 0.4864, - "hfopenllm_v2/MMLU-PRO": 0.3214 - } - }, - { - "id": "Lambent/qwen2.5-reinstruct-alternate-lumen-14B", - "name": "qwen2.5-reinstruct-alternate-lumen-14B", - "developer": "Lambent", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4794, - "hfopenllm_v2/BBH": 0.6459, - "hfopenllm_v2/MATH Level 5": 0.4622, - "hfopenllm_v2/GPQA": 0.3767, - "hfopenllm_v2/MUSR": 0.477, - "hfopenllm_v2/MMLU-PRO": 0.5388 - } - }, - { - "id": "Langboat/Mengzi3-8B-Chat", - "name": "Mengzi3-8B-Chat", - "developer": "Langboat", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.514, - "hfopenllm_v2/BBH": 0.4684, - "hfopenllm_v2/MATH Level 5": 0.0906, - "hfopenllm_v2/GPQA": 0.2743, - "hfopenllm_v2/MUSR": 0.4078, - "hfopenllm_v2/MMLU-PRO": 0.3142 - } - }, - { - "id": "langgptai/Qwen-las-v0.1", - "name": "Qwen-las-v0.1", - "developer": "langgptai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3301, - "hfopenllm_v2/BBH": 0.3893, - "hfopenllm_v2/MATH Level 5": 0.037, - "hfopenllm_v2/GPQA": 0.2466, - "hfopenllm_v2/MUSR": 0.3701, - "hfopenllm_v2/MMLU-PRO": 0.2325 - } - }, - { - "id": "langgptai/qwen1.5-7b-chat-sa-v0.1", - "name": "qwen1.5-7b-chat-sa-v0.1", - "developer": "langgptai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4268, - "hfopenllm_v2/BBH": 0.4325, - "hfopenllm_v2/MATH Level 5": 0.0302, - "hfopenllm_v2/GPQA": 0.3121, - "hfopenllm_v2/MUSR": 0.3551, - "hfopenllm_v2/MMLU-PRO": 0.2993 - } - }, - { - "id": "lars1234/Mistral-Small-24B-Instruct-2501-writer", - "name": "Mistral-Small-24B-Instruct-2501-writer", - "developer": "lars1234", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6565, - "hfopenllm_v2/BBH": 0.6733, - "hfopenllm_v2/MATH Level 5": 0.3557, - "hfopenllm_v2/GPQA": 0.3893, - "hfopenllm_v2/MUSR": 0.4645, - "hfopenllm_v2/MMLU-PRO": 0.5448 - } - }, - { - "id": "Lawnakk/BBA100", - "name": "BBA100", - "developer": "Lawnakk", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2076, - "hfopenllm_v2/BBH": 0.2826, - "hfopenllm_v2/MATH Level 5": 0.0098, - "hfopenllm_v2/GPQA": 0.2441, - "hfopenllm_v2/MUSR": 0.402, - "hfopenllm_v2/MMLU-PRO": 0.1122 - } - }, - { - "id": "Lawnakk/BBALAW1", - "name": "BBALAW1", - "developer": "Lawnakk", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1905, - "hfopenllm_v2/BBH": 0.2872, - "hfopenllm_v2/MATH Level 5": 0.0098, - "hfopenllm_v2/GPQA": 0.2433, - "hfopenllm_v2/MUSR": 0.4153, - "hfopenllm_v2/MMLU-PRO": 0.1121 - } - }, - { - "id": "Lawnakk/BBALAW1.0", - "name": "BBALAW1.0", - "developer": "Lawnakk", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1351, - "hfopenllm_v2/BBH": 0.2828, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2559, - "hfopenllm_v2/MUSR": 0.3526, - "hfopenllm_v2/MMLU-PRO": 0.1128 - } - }, - { - "id": "Lawnakk/BBALAW1.2", - "name": "BBALAW1.2", - "developer": "Lawnakk", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1354, - "hfopenllm_v2/BBH": 0.2811, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2643, - "hfopenllm_v2/MUSR": 0.3579, - "hfopenllm_v2/MMLU-PRO": 0.1105 - } - }, - { - "id": "Lawnakk/BBALAW1.3", - "name": "BBALAW1.3", - "developer": "Lawnakk", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1354, - "hfopenllm_v2/BBH": 0.2827, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.3619, - "hfopenllm_v2/MMLU-PRO": 0.1094 - } - }, - { - "id": "Lawnakk/BBALAW1.6", - "name": "BBALAW1.6", - "developer": "Lawnakk", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5245, - "hfopenllm_v2/BBH": 0.5554, - "hfopenllm_v2/MATH Level 5": 0.3603, - "hfopenllm_v2/GPQA": 0.3238, - "hfopenllm_v2/MUSR": 0.4368, - "hfopenllm_v2/MMLU-PRO": 0.4507 - } - }, - { - "id": "Lawnakk/BBALAW1.61", - "name": "BBALAW1.61", - "developer": "Lawnakk", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5771, - "hfopenllm_v2/BBH": 0.5549, - "hfopenllm_v2/MATH Level 5": 0.3663, - "hfopenllm_v2/GPQA": 0.3171, - "hfopenllm_v2/MUSR": 0.4355, - "hfopenllm_v2/MMLU-PRO": 0.4471 - } - }, - { - "id": "Lawnakk/BBALAW1.62", - "name": "BBALAW1.62", - "developer": "Lawnakk", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5046, - "hfopenllm_v2/BBH": 0.5581, - "hfopenllm_v2/MATH Level 5": 0.2825, - "hfopenllm_v2/GPQA": 0.3196, - "hfopenllm_v2/MUSR": 0.4343, - "hfopenllm_v2/MMLU-PRO": 0.4545 - } - }, - { - "id": "Lawnakk/BBALAW1.63", - "name": "BBALAW1.63", - "developer": "Lawnakk", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4407, - "hfopenllm_v2/BBH": 0.5541, - "hfopenllm_v2/MATH Level 5": 0.3701, - "hfopenllm_v2/GPQA": 0.3121, - "hfopenllm_v2/MUSR": 0.4303, - "hfopenllm_v2/MMLU-PRO": 0.4471 - } - }, - { - "id": "Lawnakk/BBALAW1.64", - "name": "BBALAW1.64", - "developer": "Lawnakk", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1395, - "hfopenllm_v2/BBH": 0.2779, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2483, - "hfopenllm_v2/MUSR": 0.3447, - "hfopenllm_v2/MMLU-PRO": 0.1115 - } - }, - { - "id": "leafspark/Llama-3.1-8B-MultiReflection-Instruct", - "name": "Llama-3.1-8B-MultiReflection-Instruct", - "developer": "leafspark", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7125, - "hfopenllm_v2/BBH": 0.5009, - "hfopenllm_v2/MATH Level 5": 0.1707, - "hfopenllm_v2/GPQA": 0.2928, - "hfopenllm_v2/MUSR": 0.3682, - "hfopenllm_v2/MMLU-PRO": 0.3724 - } - }, - { - "id": "LEESM/llama-2-7b-hf-lora-oki100p", - "name": "llama-2-7b-hf-lora-oki100p", - "developer": "LEESM", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2513, - "hfopenllm_v2/BBH": 0.3492, - "hfopenllm_v2/MATH Level 5": 0.0166, - "hfopenllm_v2/GPQA": 0.2693, - "hfopenllm_v2/MUSR": 0.3687, - "hfopenllm_v2/MMLU-PRO": 0.1856 - } - }, - { - "id": "LEESM/llama-2-7b-hf-lora-oki10p", - "name": "llama-2-7b-hf-lora-oki10p", - "developer": "LEESM", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.227, - "hfopenllm_v2/BBH": 0.3531, - "hfopenllm_v2/MATH Level 5": 0.0166, - "hfopenllm_v2/GPQA": 0.2542, - "hfopenllm_v2/MUSR": 0.3475, - "hfopenllm_v2/MMLU-PRO": 0.1679 - } - }, - { - "id": "LEESM/llama-3-8b-bnb-4b-kowiki231101", - "name": "llama-3-8b-bnb-4b-kowiki231101", - "developer": "LEESM", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1685, - "hfopenllm_v2/BBH": 0.4131, - "hfopenllm_v2/MATH Level 5": 0.0136, - "hfopenllm_v2/GPQA": 0.271, - "hfopenllm_v2/MUSR": 0.3551, - "hfopenllm_v2/MMLU-PRO": 0.2425 - } - }, - { - "id": "LEESM/llama-3-Korean-Bllossom-8B-trexlab-oki10p", - "name": "llama-3-Korean-Bllossom-8B-trexlab-oki10p", - "developer": "LEESM", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2137, - "hfopenllm_v2/BBH": 0.4343, - "hfopenllm_v2/MATH Level 5": 0.0468, - "hfopenllm_v2/GPQA": 0.2752, - "hfopenllm_v2/MUSR": 0.3869, - "hfopenllm_v2/MMLU-PRO": 0.3177 - } - }, - { - "id": "lemon07r/Gemma-2-Ataraxy-9B", - "name": "Gemma-2-Ataraxy-9B", - "developer": "lemon07r", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3009, - "hfopenllm_v2/BBH": 0.5931, - "hfopenllm_v2/MATH Level 5": 0.0853, - "hfopenllm_v2/GPQA": 0.3347, - "hfopenllm_v2/MUSR": 0.4424, - "hfopenllm_v2/MMLU-PRO": 0.4226 - } - }, - { - "id": "lemon07r/Gemma-2-Ataraxy-Advanced-9B", - "name": "Gemma-2-Ataraxy-Advanced-9B", - "developer": "lemon07r", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5516, - "hfopenllm_v2/BBH": 0.5889, - "hfopenllm_v2/MATH Level 5": 0.1979, - "hfopenllm_v2/GPQA": 0.3356, - "hfopenllm_v2/MUSR": 0.3761, - "hfopenllm_v2/MMLU-PRO": 0.4244 - } - }, - { - "id": "lemon07r/Gemma-2-Ataraxy-Remix-9B", - "name": "Gemma-2-Ataraxy-Remix-9B", - "developer": "lemon07r", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7083, - "hfopenllm_v2/BBH": 0.5892, - "hfopenllm_v2/MATH Level 5": 0.2017, - "hfopenllm_v2/GPQA": 0.3389, - "hfopenllm_v2/MUSR": 0.4372, - "hfopenllm_v2/MMLU-PRO": 0.4239 - } - }, - { - "id": "lemon07r/Gemma-2-Ataraxy-v2-9B", - "name": "Gemma-2-Ataraxy-v2-9B", - "developer": "lemon07r", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2136, - "hfopenllm_v2/BBH": 0.5766, - "hfopenllm_v2/MATH Level 5": 0.0846, - "hfopenllm_v2/GPQA": 0.3423, - "hfopenllm_v2/MUSR": 0.3484, - "hfopenllm_v2/MMLU-PRO": 0.4221 - } - }, - { - "id": "lemon07r/Gemma-2-Ataraxy-v2a-9B", - "name": "Gemma-2-Ataraxy-v2a-9B", - "developer": "lemon07r", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1595, - "hfopenllm_v2/BBH": 0.5182, - "hfopenllm_v2/MATH Level 5": 0.0612, - "hfopenllm_v2/GPQA": 0.3398, - "hfopenllm_v2/MUSR": 0.3165, - "hfopenllm_v2/MMLU-PRO": 0.3515 - } - }, - { - "id": "lemon07r/Gemma-2-Ataraxy-v2f-9B", - "name": "Gemma-2-Ataraxy-v2f-9B", - "developer": "lemon07r", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3791, - "hfopenllm_v2/BBH": 0.5193, - "hfopenllm_v2/MATH Level 5": 0.1163, - "hfopenllm_v2/GPQA": 0.3389, - "hfopenllm_v2/MUSR": 0.3231, - "hfopenllm_v2/MMLU-PRO": 0.3503 - } - }, - { - "id": "lemon07r/Gemma-2-Ataraxy-v3-Advanced-9B", - "name": "Gemma-2-Ataraxy-v3-Advanced-9B", - "developer": "lemon07r", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6602, - "hfopenllm_v2/BBH": 0.5935, - "hfopenllm_v2/MATH Level 5": 0.1873, - "hfopenllm_v2/GPQA": 0.3364, - "hfopenllm_v2/MUSR": 0.445, - "hfopenllm_v2/MMLU-PRO": 0.4196 - } - }, - { - "id": "lemon07r/Gemma-2-Ataraxy-v3b-9B", - "name": "Gemma-2-Ataraxy-v3b-9B", - "developer": "lemon07r", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6809, - "hfopenllm_v2/BBH": 0.5908, - "hfopenllm_v2/MATH Level 5": 0.2153, - "hfopenllm_v2/GPQA": 0.3331, - "hfopenllm_v2/MUSR": 0.4489, - "hfopenllm_v2/MMLU-PRO": 0.4205 - } - }, - { - "id": "lemon07r/Gemma-2-Ataraxy-v3i-9B", - "name": "Gemma-2-Ataraxy-v3i-9B", - "developer": "lemon07r", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4203, - "hfopenllm_v2/BBH": 0.5626, - "hfopenllm_v2/MATH Level 5": 0.1533, - "hfopenllm_v2/GPQA": 0.328, - "hfopenllm_v2/MUSR": 0.3181, - "hfopenllm_v2/MMLU-PRO": 0.4166 - } - }, - { - "id": "lemon07r/Gemma-2-Ataraxy-v3j-9B", - "name": "Gemma-2-Ataraxy-v3j-9B", - "developer": "lemon07r", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4169, - "hfopenllm_v2/BBH": 0.5632, - "hfopenllm_v2/MATH Level 5": 0.1692, - "hfopenllm_v2/GPQA": 0.328, - "hfopenllm_v2/MUSR": 0.318, - "hfopenllm_v2/MMLU-PRO": 0.4134 - } - }, - { - "id": "lemon07r/Gemma-2-Ataraxy-v4-Advanced-9B", - "name": "Gemma-2-Ataraxy-v4-Advanced-9B", - "developer": "lemon07r", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7015, - "hfopenllm_v2/BBH": 0.6024, - "hfopenllm_v2/MATH Level 5": 0.2153, - "hfopenllm_v2/GPQA": 0.3389, - "hfopenllm_v2/MUSR": 0.4581, - "hfopenllm_v2/MMLU-PRO": 0.4367 - } - }, - { - "id": "lemon07r/Gemma-2-Ataraxy-v4a-Advanced-9B", - "name": "Gemma-2-Ataraxy-v4a-Advanced-9B", - "developer": "lemon07r", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7135, - "hfopenllm_v2/BBH": 0.5988, - "hfopenllm_v2/MATH Level 5": 0.2115, - "hfopenllm_v2/GPQA": 0.344, - "hfopenllm_v2/MUSR": 0.4489, - "hfopenllm_v2/MMLU-PRO": 0.4309 - } - }, - { - "id": "lemon07r/Gemma-2-Ataraxy-v4b-9B", - "name": "Gemma-2-Ataraxy-v4b-9B", - "developer": "lemon07r", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6878, - "hfopenllm_v2/BBH": 0.6039, - "hfopenllm_v2/MATH Level 5": 0.2334, - "hfopenllm_v2/GPQA": 0.3406, - "hfopenllm_v2/MUSR": 0.4555, - "hfopenllm_v2/MMLU-PRO": 0.4357 - } - }, - { - "id": "lemon07r/Gemma-2-Ataraxy-v4c-9B", - "name": "Gemma-2-Ataraxy-v4c-9B", - "developer": "lemon07r", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6945, - "hfopenllm_v2/BBH": 0.6084, - "hfopenllm_v2/MATH Level 5": 0.2266, - "hfopenllm_v2/GPQA": 0.3339, - "hfopenllm_v2/MUSR": 0.4528, - "hfopenllm_v2/MMLU-PRO": 0.4395 - } - }, - { - "id": "lemon07r/Gemma-2-Ataraxy-v4d-9B", - "name": "Gemma-2-Ataraxy-v4d-9B", - "developer": "lemon07r", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.725, - "hfopenllm_v2/BBH": 0.6054, - "hfopenllm_v2/MATH Level 5": 0.2334, - "hfopenllm_v2/GPQA": 0.3473, - "hfopenllm_v2/MUSR": 0.4541, - "hfopenllm_v2/MMLU-PRO": 0.4346 - } - }, - { - "id": "lemon07r/llama-3-NeuralMahou-8b", - "name": "llama-3-NeuralMahou-8b", - "developer": "lemon07r", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4901, - "hfopenllm_v2/BBH": 0.4184, - "hfopenllm_v2/MATH Level 5": 0.102, - "hfopenllm_v2/GPQA": 0.2886, - "hfopenllm_v2/MUSR": 0.3873, - "hfopenllm_v2/MMLU-PRO": 0.369 - } - }, - { - "id": "lemon07r/Llama-3-RedMagic4-8B", - "name": "Llama-3-RedMagic4-8B", - "developer": "lemon07r", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4864, - "hfopenllm_v2/BBH": 0.4256, - "hfopenllm_v2/MATH Level 5": 0.0899, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.3766, - "hfopenllm_v2/MMLU-PRO": 0.3676 - } - }, - { - "id": "LenguajeNaturalAI/leniachat-gemma-2b-v0", - "name": "leniachat-gemma-2b-v0", - "developer": "LenguajeNaturalAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.215, - "hfopenllm_v2/BBH": 0.3074, - "hfopenllm_v2/MATH Level 5": 0.0113, - "hfopenllm_v2/GPQA": 0.2659, - "hfopenllm_v2/MUSR": 0.3659, - "hfopenllm_v2/MMLU-PRO": 0.117 - } - }, - { - "id": "LenguajeNaturalAI/leniachat-qwen2-1.5B-v0", - "name": "leniachat-qwen2-1.5B-v0", - "developer": "LenguajeNaturalAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2221, - "hfopenllm_v2/BBH": 0.3684, - "hfopenllm_v2/MATH Level 5": 0.0128, - "hfopenllm_v2/GPQA": 0.2617, - "hfopenllm_v2/MUSR": 0.375, - "hfopenllm_v2/MMLU-PRO": 0.188 - } - }, - { - "id": "LeroyDyer/_Spydaz_Web_AI_12", - "name": "_Spydaz_Web_AI_12", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2765, - "hfopenllm_v2/BBH": 0.3163, - "hfopenllm_v2/MATH Level 5": 0.0136, - "hfopenllm_v2/GPQA": 0.2685, - "hfopenllm_v2/MUSR": 0.3582, - "hfopenllm_v2/MMLU-PRO": 0.1137 - } - }, - { - "id": "LeroyDyer/_Spydaz_Web_AI_14", - "name": "_Spydaz_Web_AI_14", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1812, - "hfopenllm_v2/BBH": 0.2989, - "hfopenllm_v2/MATH Level 5": 0.0121, - "hfopenllm_v2/GPQA": 0.2659, - "hfopenllm_v2/MUSR": 0.3395, - "hfopenllm_v2/MMLU-PRO": 0.1139 - } - }, - { - "id": "LeroyDyer/_Spydaz_Web_AI_AGI_R1_001", - "name": "_Spydaz_Web_AI_AGI_R1_001", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4505, - "hfopenllm_v2/BBH": 0.4609, - "hfopenllm_v2/MATH Level 5": 0.0634, - "hfopenllm_v2/GPQA": 0.2676, - "hfopenllm_v2/MUSR": 0.4256, - "hfopenllm_v2/MMLU-PRO": 0.2734 - } - }, - { - "id": "LeroyDyer/_Spydaz_Web_AI_AGI_R1_002", - "name": "_Spydaz_Web_AI_AGI_R1_002", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5307, - "hfopenllm_v2/BBH": 0.4683, - "hfopenllm_v2/MATH Level 5": 0.0582, - "hfopenllm_v2/GPQA": 0.2685, - "hfopenllm_v2/MUSR": 0.4255, - "hfopenllm_v2/MMLU-PRO": 0.2894 - } - }, - { - "id": "LeroyDyer/_Spydaz_Web_AI_AGI_R1_MasterCoder", - "name": "_Spydaz_Web_AI_AGI_R1_MasterCoder", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4143, - "hfopenllm_v2/BBH": 0.4689, - "hfopenllm_v2/MATH Level 5": 0.0612, - "hfopenllm_v2/GPQA": 0.276, - "hfopenllm_v2/MUSR": 0.472, - "hfopenllm_v2/MMLU-PRO": 0.2719 - } - }, - { - "id": "LeroyDyer/_Spydaz_Web_AI_AGI_R1_Math_001", - "name": "_Spydaz_Web_AI_AGI_R1_Math_001", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4571, - "hfopenllm_v2/BBH": 0.4818, - "hfopenllm_v2/MATH Level 5": 0.0695, - "hfopenllm_v2/GPQA": 0.2768, - "hfopenllm_v2/MUSR": 0.4778, - "hfopenllm_v2/MMLU-PRO": 0.2681 - } - }, - { - "id": "LeroyDyer/_Spydaz_Web_AI_AGI_R1_Math_003", - "name": "_Spydaz_Web_AI_AGI_R1_Math_003", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.62, - "hfopenllm_v2/BBH": 0.4756, - "hfopenllm_v2/MATH Level 5": 0.0695, - "hfopenllm_v2/GPQA": 0.281, - "hfopenllm_v2/MUSR": 0.4202, - "hfopenllm_v2/MMLU-PRO": 0.2999 - } - }, - { - "id": "LeroyDyer/_Spydaz_Web_AI_AGI_R1_Math_AdvancedStudent", - "name": "_Spydaz_Web_AI_AGI_R1_Math_AdvancedStudent", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5951, - "hfopenllm_v2/BBH": 0.4927, - "hfopenllm_v2/MATH Level 5": 0.0544, - "hfopenllm_v2/GPQA": 0.2919, - "hfopenllm_v2/MUSR": 0.5198, - "hfopenllm_v2/MMLU-PRO": 0.3 - } - }, - { - "id": "LeroyDyer/_Spydaz_Web_AI_AGI_R1_Math_Student", - "name": "_Spydaz_Web_AI_AGI_R1_Math_Student", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5736, - "hfopenllm_v2/BBH": 0.4881, - "hfopenllm_v2/MATH Level 5": 0.0514, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.5098, - "hfopenllm_v2/MMLU-PRO": 0.2927 - } - }, - { - "id": "LeroyDyer/_Spydaz_Web_AI_AGI_R1_Math_Teacher", - "name": "_Spydaz_Web_AI_AGI_R1_Math_Teacher", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5772, - "hfopenllm_v2/BBH": 0.4805, - "hfopenllm_v2/MATH Level 5": 0.0544, - "hfopenllm_v2/GPQA": 0.2861, - "hfopenllm_v2/MUSR": 0.5222, - "hfopenllm_v2/MMLU-PRO": 0.2956 - } - }, - { - "id": "LeroyDyer/_Spydaz_Web_AI_AGI_R1_MUSR", - "name": "_Spydaz_Web_AI_AGI_R1_MUSR", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4786, - "hfopenllm_v2/BBH": 0.4672, - "hfopenllm_v2/MATH Level 5": 0.0604, - "hfopenllm_v2/GPQA": 0.2844, - "hfopenllm_v2/MUSR": 0.4869, - "hfopenllm_v2/MMLU-PRO": 0.2828 - } - }, - { - "id": "LeroyDyer/_Spydaz_Web_AI_AGI_R1_OmG_001", - "name": "_Spydaz_Web_AI_AGI_R1_OmG_001", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5818, - "hfopenllm_v2/BBH": 0.4908, - "hfopenllm_v2/MATH Level 5": 0.0506, - "hfopenllm_v2/GPQA": 0.3003, - "hfopenllm_v2/MUSR": 0.4486, - "hfopenllm_v2/MMLU-PRO": 0.2906 - } - }, - { - "id": "LeroyDyer/_Spydaz_Web_AI_AGI_R1_OmG_002", - "name": "_Spydaz_Web_AI_AGI_R1_OmG_002", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5462, - "hfopenllm_v2/BBH": 0.4655, - "hfopenllm_v2/MATH Level 5": 0.0498, - "hfopenllm_v2/GPQA": 0.2785, - "hfopenllm_v2/MUSR": 0.4511, - "hfopenllm_v2/MMLU-PRO": 0.2867 - } - }, - { - "id": "LeroyDyer/_Spydaz_Web_AI_AGI_R1_OmG_Coder", - "name": "_Spydaz_Web_AI_AGI_R1_OmG_Coder", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4924, - "hfopenllm_v2/BBH": 0.4638, - "hfopenllm_v2/MATH Level 5": 0.0544, - "hfopenllm_v2/GPQA": 0.2735, - "hfopenllm_v2/MUSR": 0.5625, - "hfopenllm_v2/MMLU-PRO": 0.289 - } - }, - { - "id": "LeroyDyer/_Spydaz_Web_AI_AGI_R1_OmG_Math", - "name": "_Spydaz_Web_AI_AGI_R1_OmG_Math", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5033, - "hfopenllm_v2/BBH": 0.4677, - "hfopenllm_v2/MATH Level 5": 0.0476, - "hfopenllm_v2/GPQA": 0.2827, - "hfopenllm_v2/MUSR": 0.4326, - "hfopenllm_v2/MMLU-PRO": 0.2913 - } - }, - { - "id": "LeroyDyer/_Spydaz_Web_AI_AGI_R1_OmG_MathMaster", - "name": "_Spydaz_Web_AI_AGI_R1_OmG_MathMaster", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5558, - "hfopenllm_v2/BBH": 0.4742, - "hfopenllm_v2/MATH Level 5": 0.0536, - "hfopenllm_v2/GPQA": 0.2878, - "hfopenllm_v2/MUSR": 0.451, - "hfopenllm_v2/MMLU-PRO": 0.2672 - } - }, - { - "id": "LeroyDyer/_Spydaz_Web_AI_AGI_R1_Student_Coder", - "name": "_Spydaz_Web_AI_AGI_R1_Student_Coder", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.545, - "hfopenllm_v2/BBH": 0.4651, - "hfopenllm_v2/MATH Level 5": 0.0657, - "hfopenllm_v2/GPQA": 0.2844, - "hfopenllm_v2/MUSR": 0.4388, - "hfopenllm_v2/MMLU-PRO": 0.2768 - } - }, - { - "id": "LeroyDyer/_Spydaz_Web_AI_AGI_R1_Teacher_Coder", - "name": "_Spydaz_Web_AI_AGI_R1_Teacher_Coder", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5082, - "hfopenllm_v2/BBH": 0.4797, - "hfopenllm_v2/MATH Level 5": 0.065, - "hfopenllm_v2/GPQA": 0.2911, - "hfopenllm_v2/MUSR": 0.4338, - "hfopenllm_v2/MMLU-PRO": 0.2845 - } - }, - { - "id": "LeroyDyer/_Spydaz_Web_AI_AGI_R1_Top_Student", - "name": "_Spydaz_Web_AI_AGI_R1_Top_Student", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.604, - "hfopenllm_v2/BBH": 0.4988, - "hfopenllm_v2/MATH Level 5": 0.0725, - "hfopenllm_v2/GPQA": 0.2727, - "hfopenllm_v2/MUSR": 0.5398, - "hfopenllm_v2/MMLU-PRO": 0.3024 - } - }, - { - "id": "LeroyDyer/_Spydaz_Web_AI_AGI_R1_X1", - "name": "_Spydaz_Web_AI_AGI_R1_X1", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4273, - "hfopenllm_v2/BBH": 0.4759, - "hfopenllm_v2/MATH Level 5": 0.0566, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.4232, - "hfopenllm_v2/MMLU-PRO": 0.2891 - } - }, - { - "id": "LeroyDyer/_Spydaz_Web_AI_AGI_R1_X2", - "name": "_Spydaz_Web_AI_AGI_R1_X2", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5434, - "hfopenllm_v2/BBH": 0.4786, - "hfopenllm_v2/MATH Level 5": 0.0612, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.4695, - "hfopenllm_v2/MMLU-PRO": 0.2921 - } - }, - { - "id": "LeroyDyer/_Spydaz_Web_AI_AGI_RP_R1", - "name": "_Spydaz_Web_AI_AGI_RP_R1", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5426, - "hfopenllm_v2/BBH": 0.4701, - "hfopenllm_v2/MATH Level 5": 0.0604, - "hfopenllm_v2/GPQA": 0.2693, - "hfopenllm_v2/MUSR": 0.4201, - "hfopenllm_v2/MMLU-PRO": 0.2894 - } - }, - { - "id": "LeroyDyer/_Spydaz_Web_AI_BIBLE_002", - "name": "_Spydaz_Web_AI_BIBLE_002", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2195, - "hfopenllm_v2/BBH": 0.3289, - "hfopenllm_v2/MATH Level 5": 0.0174, - "hfopenllm_v2/GPQA": 0.2844, - "hfopenllm_v2/MUSR": 0.3407, - "hfopenllm_v2/MMLU-PRO": 0.1368 - } - }, - { - "id": "LeroyDyer/_Spydaz_Web_AI_ChatML_002", - "name": "_Spydaz_Web_AI_ChatML_002", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2412, - "hfopenllm_v2/BBH": 0.3106, - "hfopenllm_v2/MATH Level 5": 0.0113, - "hfopenllm_v2/GPQA": 0.2576, - "hfopenllm_v2/MUSR": 0.3623, - "hfopenllm_v2/MMLU-PRO": 0.1095 - } - }, - { - "id": "LeroyDyer/_Spydaz_Web_AI_ChatQA", - "name": "_Spydaz_Web_AI_ChatQA", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1415, - "hfopenllm_v2/BBH": 0.3236, - "hfopenllm_v2/MATH Level 5": 0.0098, - "hfopenllm_v2/GPQA": 0.2659, - "hfopenllm_v2/MUSR": 0.3447, - "hfopenllm_v2/MMLU-PRO": 0.1475 - } - }, - { - "id": "LeroyDyer/_Spydaz_Web_AI_ChatQA_003", - "name": "_Spydaz_Web_AI_ChatQA_003", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2209, - "hfopenllm_v2/BBH": 0.3172, - "hfopenllm_v2/MATH Level 5": 0.0106, - "hfopenllm_v2/GPQA": 0.271, - "hfopenllm_v2/MUSR": 0.3818, - "hfopenllm_v2/MMLU-PRO": 0.1133 - } - }, - { - "id": "LeroyDyer/_Spydaz_Web_AI_TEMP_", - "name": "_Spydaz_Web_AI_TEMP_", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4795, - "hfopenllm_v2/BBH": 0.4957, - "hfopenllm_v2/MATH Level 5": 0.1239, - "hfopenllm_v2/GPQA": 0.2794, - "hfopenllm_v2/MUSR": 0.4218, - "hfopenllm_v2/MMLU-PRO": 0.3121 - } - }, - { - "id": "LeroyDyer/_Spydaz_Web_AI_Top_Teacher_", - "name": "_Spydaz_Web_AI_Top_Teacher_", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4404, - "hfopenllm_v2/BBH": 0.4891, - "hfopenllm_v2/MATH Level 5": 0.1156, - "hfopenllm_v2/GPQA": 0.2777, - "hfopenllm_v2/MUSR": 0.4366, - "hfopenllm_v2/MMLU-PRO": 0.315 - } - }, - { - "id": "LeroyDyer/CheckPoint_A", - "name": "CheckPoint_A", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4513, - "hfopenllm_v2/BBH": 0.4748, - "hfopenllm_v2/MATH Level 5": 0.0589, - "hfopenllm_v2/GPQA": 0.2836, - "hfopenllm_v2/MUSR": 0.4231, - "hfopenllm_v2/MMLU-PRO": 0.288 - } - }, - { - "id": "LeroyDyer/CheckPoint_B", - "name": "CheckPoint_B", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.444, - "hfopenllm_v2/BBH": 0.478, - "hfopenllm_v2/MATH Level 5": 0.0718, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.3898, - "hfopenllm_v2/MMLU-PRO": 0.2907 - } - }, - { - "id": "LeroyDyer/CheckPoint_C", - "name": "CheckPoint_C", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3477, - "hfopenllm_v2/BBH": 0.4586, - "hfopenllm_v2/MATH Level 5": 0.0551, - "hfopenllm_v2/GPQA": 0.271, - "hfopenllm_v2/MUSR": 0.4346, - "hfopenllm_v2/MMLU-PRO": 0.3021 - } - }, - { - "id": "LeroyDyer/CheckPoint_R1", - "name": "CheckPoint_R1", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1728, - "hfopenllm_v2/BBH": 0.4225, - "hfopenllm_v2/MATH Level 5": 0.0431, - "hfopenllm_v2/GPQA": 0.2743, - "hfopenllm_v2/MUSR": 0.4031, - "hfopenllm_v2/MMLU-PRO": 0.2205 - } - }, - { - "id": "LeroyDyer/LCARS_AI_001", - "name": "LCARS_AI_001", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3109, - "hfopenllm_v2/BBH": 0.4258, - "hfopenllm_v2/MATH Level 5": 0.0234, - "hfopenllm_v2/GPQA": 0.2634, - "hfopenllm_v2/MUSR": 0.4384, - "hfopenllm_v2/MMLU-PRO": 0.267 - } - }, - { - "id": "LeroyDyer/LCARS_AI_1x4_003_SuperAI", - "name": "LCARS_AI_1x4_003_SuperAI", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4111, - "hfopenllm_v2/BBH": 0.492, - "hfopenllm_v2/MATH Level 5": 0.0574, - "hfopenllm_v2/GPQA": 0.2827, - "hfopenllm_v2/MUSR": 0.4506, - "hfopenllm_v2/MMLU-PRO": 0.2972 - } - }, - { - "id": "LeroyDyer/LCARS_AI_StarTrek_Computer", - "name": "LCARS_AI_StarTrek_Computer", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3583, - "hfopenllm_v2/BBH": 0.4446, - "hfopenllm_v2/MATH Level 5": 0.0408, - "hfopenllm_v2/GPQA": 0.2676, - "hfopenllm_v2/MUSR": 0.395, - "hfopenllm_v2/MMLU-PRO": 0.2458 - } - }, - { - "id": "LeroyDyer/LCARS_TOP_SCORE", - "name": "LCARS_TOP_SCORE", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4371, - "hfopenllm_v2/BBH": 0.5127, - "hfopenllm_v2/MATH Level 5": 0.0672, - "hfopenllm_v2/GPQA": 0.2861, - "hfopenllm_v2/MUSR": 0.4293, - "hfopenllm_v2/MMLU-PRO": 0.3031 - } - }, - { - "id": "LeroyDyer/Mixtral_AI_SwahiliTron_7b", - "name": "Mixtral_AI_SwahiliTron_7b", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1534, - "hfopenllm_v2/BBH": 0.3055, - "hfopenllm_v2/MATH Level 5": 0.0136, - "hfopenllm_v2/GPQA": 0.2651, - "hfopenllm_v2/MUSR": 0.342, - "hfopenllm_v2/MMLU-PRO": 0.1208 - } - }, - { - "id": "LeroyDyer/SpydazWeb_AI_CyberTron_Ultra_7b", - "name": "SpydazWeb_AI_CyberTron_Ultra_7b", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1556, - "hfopenllm_v2/BBH": 0.4811, - "hfopenllm_v2/MATH Level 5": 0.0136, - "hfopenllm_v2/GPQA": 0.2928, - "hfopenllm_v2/MUSR": 0.4136, - "hfopenllm_v2/MMLU-PRO": 0.2866 - } - }, - { - "id": "LeroyDyer/SpydazWeb_AI_HumanAGI_001_M2", - "name": "SpydazWeb_AI_HumanAGI_001_M2", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.394, - "hfopenllm_v2/BBH": 0.4888, - "hfopenllm_v2/MATH Level 5": 0.0385, - "hfopenllm_v2/GPQA": 0.2894, - "hfopenllm_v2/MUSR": 0.4503, - "hfopenllm_v2/MMLU-PRO": 0.3005 - } - }, - { - "id": "LeroyDyer/SpydazWeb_AI_HumanAGI_002", - "name": "SpydazWeb_AI_HumanAGI_002", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4088, - "hfopenllm_v2/BBH": 0.5044, - "hfopenllm_v2/MATH Level 5": 0.0665, - "hfopenllm_v2/GPQA": 0.2869, - "hfopenllm_v2/MUSR": 0.4865, - "hfopenllm_v2/MMLU-PRO": 0.3059 - } - }, - { - "id": "LeroyDyer/SpydazWeb_AI_HumanAI_001", - "name": "SpydazWeb_AI_HumanAI_001", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2252, - "hfopenllm_v2/BBH": 0.3344, - "hfopenllm_v2/MATH Level 5": 0.0166, - "hfopenllm_v2/GPQA": 0.2886, - "hfopenllm_v2/MUSR": 0.386, - "hfopenllm_v2/MMLU-PRO": 0.1271 - } - }, - { - "id": "LeroyDyer/SpydazWeb_AI_HumanAI_006", - "name": "SpydazWeb_AI_HumanAI_006", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.143, - "hfopenllm_v2/BBH": 0.3302, - "hfopenllm_v2/MATH Level 5": 0.0106, - "hfopenllm_v2/GPQA": 0.2802, - "hfopenllm_v2/MUSR": 0.3568, - "hfopenllm_v2/MMLU-PRO": 0.1135 - } - }, - { - "id": "LeroyDyer/SpydazWeb_AI_HumanAI_007", - "name": "SpydazWeb_AI_HumanAI_007", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3352, - "hfopenllm_v2/BBH": 0.3416, - "hfopenllm_v2/MATH Level 5": 0.0227, - "hfopenllm_v2/GPQA": 0.2886, - "hfopenllm_v2/MUSR": 0.4096, - "hfopenllm_v2/MMLU-PRO": 0.1352 - } - }, - { - "id": "LeroyDyer/SpydazWeb_AI_HumanAI_009_CHAT", - "name": "SpydazWeb_AI_HumanAI_009_CHAT", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2973, - "hfopenllm_v2/BBH": 0.3307, - "hfopenllm_v2/MATH Level 5": 0.0166, - "hfopenllm_v2/GPQA": 0.281, - "hfopenllm_v2/MUSR": 0.4138, - "hfopenllm_v2/MMLU-PRO": 0.1433 - } - }, - { - "id": "LeroyDyer/SpydazWeb_AI_HumanAI_010_CHAT", - "name": "SpydazWeb_AI_HumanAI_010_CHAT", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2507, - "hfopenllm_v2/BBH": 0.3336, - "hfopenllm_v2/MATH Level 5": 0.0181, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.4137, - "hfopenllm_v2/MMLU-PRO": 0.143 - } - }, - { - "id": "LeroyDyer/SpydazWeb_AI_HumanAI_011_INSTRUCT", - "name": "SpydazWeb_AI_HumanAI_011_INSTRUCT", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3149, - "hfopenllm_v2/BBH": 0.3523, - "hfopenllm_v2/MATH Level 5": 0.0144, - "hfopenllm_v2/GPQA": 0.2794, - "hfopenllm_v2/MUSR": 0.3831, - "hfopenllm_v2/MMLU-PRO": 0.1595 - } - }, - { - "id": "LeroyDyer/SpydazWeb_AI_HumanAI_011_INSTRUCT_ML", - "name": "SpydazWeb_AI_HumanAI_011_INSTRUCT_ML", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3752, - "hfopenllm_v2/BBH": 0.3984, - "hfopenllm_v2/MATH Level 5": 0.0257, - "hfopenllm_v2/GPQA": 0.2928, - "hfopenllm_v2/MUSR": 0.4239, - "hfopenllm_v2/MMLU-PRO": 0.2019 - } - }, - { - "id": "LeroyDyer/SpydazWeb_AI_HumanAI_011_INSTRUCT_ML_r1", - "name": "SpydazWeb_AI_HumanAI_011_INSTRUCT_ML_r1", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.405, - "hfopenllm_v2/BBH": 0.4858, - "hfopenllm_v2/MATH Level 5": 0.0551, - "hfopenllm_v2/GPQA": 0.2928, - "hfopenllm_v2/MUSR": 0.3921, - "hfopenllm_v2/MMLU-PRO": 0.2956 - } - }, - { - "id": "LeroyDyer/SpydazWeb_AI_HumanAI_012_INSTRUCT_IA", - "name": "SpydazWeb_AI_HumanAI_012_INSTRUCT_IA", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3036, - "hfopenllm_v2/BBH": 0.4575, - "hfopenllm_v2/MATH Level 5": 0.0446, - "hfopenllm_v2/GPQA": 0.3012, - "hfopenllm_v2/MUSR": 0.4253, - "hfopenllm_v2/MMLU-PRO": 0.2329 - } - }, - { - "id": "LeroyDyer/SpydazWeb_AI_HumanAI_012_INSTRUCT_MX", - "name": "SpydazWeb_AI_HumanAI_012_INSTRUCT_MX", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3066, - "hfopenllm_v2/BBH": 0.3158, - "hfopenllm_v2/MATH Level 5": 0.0151, - "hfopenllm_v2/GPQA": 0.2911, - "hfopenllm_v2/MUSR": 0.3444, - "hfopenllm_v2/MMLU-PRO": 0.1107 - } - }, - { - "id": "LeroyDyer/SpydazWeb_AI_HumanAI_012_INSTRUCT_XA", - "name": "SpydazWeb_AI_HumanAI_012_INSTRUCT_XA", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3579, - "hfopenllm_v2/BBH": 0.4477, - "hfopenllm_v2/MATH Level 5": 0.0423, - "hfopenllm_v2/GPQA": 0.3096, - "hfopenllm_v2/MUSR": 0.4134, - "hfopenllm_v2/MMLU-PRO": 0.2376 - } - }, - { - "id": "LeroyDyer/SpydazWeb_AI_HumanAI_RP", - "name": "SpydazWeb_AI_HumanAI_RP", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2541, - "hfopenllm_v2/BBH": 0.3323, - "hfopenllm_v2/MATH Level 5": 0.0128, - "hfopenllm_v2/GPQA": 0.2752, - "hfopenllm_v2/MUSR": 0.3883, - "hfopenllm_v2/MMLU-PRO": 0.1324 - } - }, - { - "id": "LeroyDyer/SpydazWeb_AI_HumanAI_TextVision", - "name": "SpydazWeb_AI_HumanAI_TextVision", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3063, - "hfopenllm_v2/BBH": 0.3354, - "hfopenllm_v2/MATH Level 5": 0.0144, - "hfopenllm_v2/GPQA": 0.2919, - "hfopenllm_v2/MUSR": 0.3938, - "hfopenllm_v2/MMLU-PRO": 0.1387 - } - }, - { - "id": "LeroyDyer/SpydazWeb_HumanAI_M1", - "name": "SpydazWeb_HumanAI_M1", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3582, - "hfopenllm_v2/BBH": 0.3563, - "hfopenllm_v2/MATH Level 5": 0.0249, - "hfopenllm_v2/GPQA": 0.2676, - "hfopenllm_v2/MUSR": 0.3671, - "hfopenllm_v2/MMLU-PRO": 0.1663 - } - }, - { - "id": "LeroyDyer/SpydazWeb_HumanAI_M2", - "name": "SpydazWeb_HumanAI_M2", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.375, - "hfopenllm_v2/BBH": 0.3931, - "hfopenllm_v2/MATH Level 5": 0.0287, - "hfopenllm_v2/GPQA": 0.2794, - "hfopenllm_v2/MUSR": 0.3751, - "hfopenllm_v2/MMLU-PRO": 0.201 - } - }, - { - "id": "LeroyDyer/SpydazWeb_HumanAI_M3", - "name": "SpydazWeb_HumanAI_M3", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1579, - "hfopenllm_v2/BBH": 0.3127, - "hfopenllm_v2/MATH Level 5": 0.0091, - "hfopenllm_v2/GPQA": 0.271, - "hfopenllm_v2/MUSR": 0.3914, - "hfopenllm_v2/MMLU-PRO": 0.1149 - } - }, - { - "id": "LeroyDyer/SpydazWebAI_Human_AGI", - "name": "SpydazWebAI_Human_AGI", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3388, - "hfopenllm_v2/BBH": 0.3375, - "hfopenllm_v2/MATH Level 5": 0.0144, - "hfopenllm_v2/GPQA": 0.2827, - "hfopenllm_v2/MUSR": 0.3966, - "hfopenllm_v2/MMLU-PRO": 0.1479 - } - }, - { - "id": "LeroyDyer/SpydazWebAI_Human_AGI_001", - "name": "SpydazWebAI_Human_AGI_001", - "developer": "LeroyDyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3118, - "hfopenllm_v2/BBH": 0.3433, - "hfopenllm_v2/MATH Level 5": 0.0196, - "hfopenllm_v2/GPQA": 0.2987, - "hfopenllm_v2/MUSR": 0.3994, - "hfopenllm_v2/MMLU-PRO": 0.1426 - } - }, - { - "id": "lesubra/ECE-EIFFEL-3B", - "name": "ECE-EIFFEL-3B", - "developer": "lesubra", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3469, - "hfopenllm_v2/BBH": 0.5102, - "hfopenllm_v2/MATH Level 5": 0.1216, - "hfopenllm_v2/GPQA": 0.3314, - "hfopenllm_v2/MUSR": 0.4362, - "hfopenllm_v2/MMLU-PRO": 0.3821 - } - }, - { - "id": "lesubra/ECE-EIFFEL-3Bv2", - "name": "ECE-EIFFEL-3Bv2", - "developer": "lesubra", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3013, - "hfopenllm_v2/BBH": 0.5424, - "hfopenllm_v2/MATH Level 5": 0.1186, - "hfopenllm_v2/GPQA": 0.3356, - "hfopenllm_v2/MUSR": 0.4443, - "hfopenllm_v2/MMLU-PRO": 0.3999 - } - }, - { - "id": "lesubra/ECE-EIFFEL-3Bv3", - "name": "ECE-EIFFEL-3Bv3", - "developer": "lesubra", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3786, - "hfopenllm_v2/BBH": 0.5469, - "hfopenllm_v2/MATH Level 5": 0.1669, - "hfopenllm_v2/GPQA": 0.3297, - "hfopenllm_v2/MUSR": 0.4675, - "hfopenllm_v2/MMLU-PRO": 0.3975 - } - }, - { - "id": "lesubra/ECE-PRYMMAL-3B-SLERP-V1", - "name": "ECE-PRYMMAL-3B-SLERP-V1", - "developer": "lesubra", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2933, - "hfopenllm_v2/BBH": 0.5341, - "hfopenllm_v2/MATH Level 5": 0.1662, - "hfopenllm_v2/GPQA": 0.3171, - "hfopenllm_v2/MUSR": 0.4595, - "hfopenllm_v2/MMLU-PRO": 0.39 - } - }, - { - "id": "lesubra/ECE-PRYMMAL-3B-SLERP-V2", - "name": "ECE-PRYMMAL-3B-SLERP-V2", - "developer": "lesubra", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2933, - "hfopenllm_v2/BBH": 0.5341, - "hfopenllm_v2/MATH Level 5": 0.1662, - "hfopenllm_v2/GPQA": 0.3171, - "hfopenllm_v2/MUSR": 0.4595, - "hfopenllm_v2/MMLU-PRO": 0.39 - } - }, - { - "id": "lesubra/ECE-PRYMMAL-3B-SLERP_2-V1", - "name": "ECE-PRYMMAL-3B-SLERP_2-V1", - "developer": "lesubra", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3649, - "hfopenllm_v2/BBH": 0.5411, - "hfopenllm_v2/MATH Level 5": 0.1677, - "hfopenllm_v2/GPQA": 0.3213, - "hfopenllm_v2/MUSR": 0.4661, - "hfopenllm_v2/MMLU-PRO": 0.399 - } - }, - { - "id": "lesubra/ECE-PRYMMAL-3B-SLERP_2-V2", - "name": "ECE-PRYMMAL-3B-SLERP_2-V2", - "developer": "lesubra", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3664, - "hfopenllm_v2/BBH": 0.5411, - "hfopenllm_v2/MATH Level 5": 0.1677, - "hfopenllm_v2/GPQA": 0.3213, - "hfopenllm_v2/MUSR": 0.4661, - "hfopenllm_v2/MMLU-PRO": 0.399 - } - }, - { - "id": "lesubra/merge-test", - "name": "merge-test", - "developer": "lesubra", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5383, - "hfopenllm_v2/BBH": 0.524, - "hfopenllm_v2/MATH Level 5": 0.1208, - "hfopenllm_v2/GPQA": 0.3221, - "hfopenllm_v2/MUSR": 0.4419, - "hfopenllm_v2/MMLU-PRO": 0.3874 - } - }, - { - "id": "LGAI-EXAONE/EXAONE-3.0-7.8B-Instruct", - "name": "EXAONE-3.0-7.8B-Instruct", - "developer": "LGAI-EXAONE", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7193, - "hfopenllm_v2/BBH": 0.4174, - "hfopenllm_v2/MATH Level 5": 0.3044, - "hfopenllm_v2/GPQA": 0.2659, - "hfopenllm_v2/MUSR": 0.3661, - "hfopenllm_v2/MMLU-PRO": 0.3577 - } - }, - { - "id": "LGAI-EXAONE/EXAONE-3.5-2.4B-Instruct", - "name": "EXAONE-3.5-2.4B-Instruct", - "developer": "LGAI-EXAONE", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.795, - "hfopenllm_v2/BBH": 0.4092, - "hfopenllm_v2/MATH Level 5": 0.3678, - "hfopenllm_v2/GPQA": 0.2659, - "hfopenllm_v2/MUSR": 0.3661, - "hfopenllm_v2/MMLU-PRO": 0.328 - } - }, - { - "id": "LGAI-EXAONE/EXAONE-3.5-32B-Instruct", - "name": "EXAONE-3.5-32B-Instruct", - "developer": "LGAI-EXAONE", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8392, - "hfopenllm_v2/BBH": 0.5761, - "hfopenllm_v2/MATH Level 5": 0.5128, - "hfopenllm_v2/GPQA": 0.2878, - "hfopenllm_v2/MUSR": 0.3807, - "hfopenllm_v2/MMLU-PRO": 0.4637 - } - }, - { - "id": "LGAI-EXAONE/EXAONE-3.5-7.8B-Instruct", - "name": "EXAONE-3.5-7.8B-Instruct", - "developer": "LGAI-EXAONE", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8136, - "hfopenllm_v2/BBH": 0.4728, - "hfopenllm_v2/MATH Level 5": 0.4751, - "hfopenllm_v2/GPQA": 0.2576, - "hfopenllm_v2/MUSR": 0.3779, - "hfopenllm_v2/MMLU-PRO": 0.4133 - } - }, - { - "id": "lightblue/suzume-llama-3-8B-multilingual", - "name": "suzume-llama-3-8B-multilingual", - "developer": "lightblue", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6678, - "hfopenllm_v2/BBH": 0.495, - "hfopenllm_v2/MATH Level 5": 0.0944, - "hfopenllm_v2/GPQA": 0.2836, - "hfopenllm_v2/MUSR": 0.3977, - "hfopenllm_v2/MMLU-PRO": 0.3383 - } - }, - { - "id": "lightblue/suzume-llama-3-8B-multilingual-orpo-borda-full", - "name": "suzume-llama-3-8B-multilingual-orpo-borda-full", - "developer": "lightblue", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5817, - "hfopenllm_v2/BBH": 0.4714, - "hfopenllm_v2/MATH Level 5": 0.0763, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3222, - "hfopenllm_v2/MMLU-PRO": 0.331 - } - }, - { - "id": "lightblue/suzume-llama-3-8B-multilingual-orpo-borda-half", - "name": "suzume-llama-3-8B-multilingual-orpo-borda-half", - "developer": "lightblue", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6249, - "hfopenllm_v2/BBH": 0.4707, - "hfopenllm_v2/MATH Level 5": 0.0906, - "hfopenllm_v2/GPQA": 0.245, - "hfopenllm_v2/MUSR": 0.3516, - "hfopenllm_v2/MMLU-PRO": 0.3614 - } - }, - { - "id": "lightblue/suzume-llama-3-8B-multilingual-orpo-borda-top25", - "name": "suzume-llama-3-8B-multilingual-orpo-borda-top25", - "developer": "lightblue", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6637, - "hfopenllm_v2/BBH": 0.4865, - "hfopenllm_v2/MATH Level 5": 0.1042, - "hfopenllm_v2/GPQA": 0.2727, - "hfopenllm_v2/MUSR": 0.3566, - "hfopenllm_v2/MMLU-PRO": 0.3684 - } - }, - { - "id": "lightblue/suzume-llama-3-8B-multilingual-orpo-borda-top75", - "name": "suzume-llama-3-8B-multilingual-orpo-borda-top75", - "developer": "lightblue", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6687, - "hfopenllm_v2/BBH": 0.4833, - "hfopenllm_v2/MATH Level 5": 0.0785, - "hfopenllm_v2/GPQA": 0.2727, - "hfopenllm_v2/MUSR": 0.3817, - "hfopenllm_v2/MMLU-PRO": 0.3769 - } - }, - { - "id": "LightningRodLabs/Flashlight-v1.0", - "name": "Flashlight-v1.0", - "developer": "LightningRodLabs", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6745, - "hfopenllm_v2/BBH": 0.6877, - "hfopenllm_v2/MATH Level 5": 0.497, - "hfopenllm_v2/GPQA": 0.3423, - "hfopenllm_v2/MUSR": 0.4101, - "hfopenllm_v2/MMLU-PRO": 0.5402 - } - }, - { - "id": "LightningRodLabs/Flashlight-v1.1", - "name": "Flashlight-v1.1", - "developer": "LightningRodLabs", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6721, - "hfopenllm_v2/BBH": 0.6901, - "hfopenllm_v2/MATH Level 5": 0.5325, - "hfopenllm_v2/GPQA": 0.3398, - "hfopenllm_v2/MUSR": 0.4048, - "hfopenllm_v2/MMLU-PRO": 0.5416 - } - }, - { - "id": "LightningRodLabs/Flashlight-v1.2", - "name": "Flashlight-v1.2", - "developer": "LightningRodLabs", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.436, - "hfopenllm_v2/BBH": 0.3265, - "hfopenllm_v2/MATH Level 5": 0.1556, - "hfopenllm_v2/GPQA": 0.2357, - "hfopenllm_v2/MUSR": 0.4554, - "hfopenllm_v2/MMLU-PRO": 0.2485 - } - }, - { - "id": "Lil-R/2_PRYMMAL-ECE-2B-SLERP-V1", - "name": "2_PRYMMAL-ECE-2B-SLERP-V1", - "developer": "Lil-R", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5823, - "hfopenllm_v2/BBH": 0.4287, - "hfopenllm_v2/MATH Level 5": 0.0914, - "hfopenllm_v2/GPQA": 0.3062, - "hfopenllm_v2/MUSR": 0.4375, - "hfopenllm_v2/MMLU-PRO": 0.2678 - } - }, - { - "id": "Lil-R/2_PRYMMAL-ECE-2B-SLERP-V2", - "name": "2_PRYMMAL-ECE-2B-SLERP-V2", - "developer": "Lil-R", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5543, - "hfopenllm_v2/BBH": 0.4376, - "hfopenllm_v2/MATH Level 5": 0.0944, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.4482, - "hfopenllm_v2/MMLU-PRO": 0.2744 - } - }, - { - "id": "Lil-R/2_PRYMMAL-ECE-7B-SLERP", - "name": "2_PRYMMAL-ECE-7B-SLERP", - "developer": "Lil-R", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5577, - "hfopenllm_v2/BBH": 0.5557, - "hfopenllm_v2/MATH Level 5": 0.3633, - "hfopenllm_v2/GPQA": 0.3104, - "hfopenllm_v2/MUSR": 0.4396, - "hfopenllm_v2/MMLU-PRO": 0.4507 - } - }, - { - "id": "Lil-R/2_PRYMMAL-ECE-7B-SLERP-V1", - "name": "2_PRYMMAL-ECE-7B-SLERP-V1", - "developer": "Lil-R", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1073, - "hfopenllm_v2/BBH": 0.3053, - "hfopenllm_v2/MATH Level 5": 0.0008, - "hfopenllm_v2/GPQA": 0.2508, - "hfopenllm_v2/MUSR": 0.3911, - "hfopenllm_v2/MMLU-PRO": 0.1124 - } - }, - { - "id": "Lil-R/2_PRYMMAL-ECE-7B-SLERP-V2", - "name": "2_PRYMMAL-ECE-7B-SLERP-V2", - "developer": "Lil-R", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1073, - "hfopenllm_v2/BBH": 0.3053, - "hfopenllm_v2/MATH Level 5": 0.0008, - "hfopenllm_v2/GPQA": 0.2508, - "hfopenllm_v2/MUSR": 0.3911, - "hfopenllm_v2/MMLU-PRO": 0.1124 - } - }, - { - "id": "Lil-R/2_PRYMMAL-ECE-7B-SLERP-V3", - "name": "2_PRYMMAL-ECE-7B-SLERP-V3", - "developer": "Lil-R", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2235, - "hfopenllm_v2/BBH": 0.3578, - "hfopenllm_v2/MATH Level 5": 0.006, - "hfopenllm_v2/GPQA": 0.2567, - "hfopenllm_v2/MUSR": 0.4107, - "hfopenllm_v2/MMLU-PRO": 0.1817 - } - }, - { - "id": "Lil-R/PRYMMAL-ECE-1B-SLERP-V1", - "name": "PRYMMAL-ECE-1B-SLERP-V1", - "developer": "Lil-R", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2874, - "hfopenllm_v2/BBH": 0.419, - "hfopenllm_v2/MATH Level 5": 0.1035, - "hfopenllm_v2/GPQA": 0.276, - "hfopenllm_v2/MUSR": 0.3974, - "hfopenllm_v2/MMLU-PRO": 0.2926 - } - }, - { - "id": "Lil-R/PRYMMAL-ECE-7B-SLERP-V8", - "name": "PRYMMAL-ECE-7B-SLERP-V8", - "developer": "Lil-R", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1258, - "hfopenllm_v2/BBH": 0.2955, - "hfopenllm_v2/MATH Level 5": 0.0098, - "hfopenllm_v2/GPQA": 0.25, - "hfopenllm_v2/MUSR": 0.3631, - "hfopenllm_v2/MMLU-PRO": 0.1128 - } - }, - { - "id": "LilRg/10PRYMMAL-3B-slerp", - "name": "10PRYMMAL-3B-slerp", - "developer": "LilRg", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1946, - "hfopenllm_v2/BBH": 0.532, - "hfopenllm_v2/MATH Level 5": 0.1495, - "hfopenllm_v2/GPQA": 0.3213, - "hfopenllm_v2/MUSR": 0.4529, - "hfopenllm_v2/MMLU-PRO": 0.3881 - } - }, - { - "id": "LilRg/ECE-1B-merge-PRYMMAL", - "name": "ECE-1B-merge-PRYMMAL", - "developer": "LilRg", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2712, - "hfopenllm_v2/BBH": 0.4235, - "hfopenllm_v2/MATH Level 5": 0.1012, - "hfopenllm_v2/GPQA": 0.281, - "hfopenllm_v2/MUSR": 0.3801, - "hfopenllm_v2/MMLU-PRO": 0.2906 - } - }, - { - "id": "LilRg/ECE_Finetunning", - "name": "ECE_Finetunning", - "developer": "LilRg", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0445, - "hfopenllm_v2/BBH": 0.4732, - "hfopenllm_v2/MATH Level 5": 0.0453, - "hfopenllm_v2/GPQA": 0.2827, - "hfopenllm_v2/MUSR": 0.3839, - "hfopenllm_v2/MMLU-PRO": 0.3191 - } - }, - { - "id": "LilRg/PRYMMAL-6B-slerp", - "name": "PRYMMAL-6B-slerp", - "developer": "LilRg", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1153, - "hfopenllm_v2/BBH": 0.2868, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2458, - "hfopenllm_v2/MUSR": 0.3698, - "hfopenllm_v2/MMLU-PRO": 0.1108 - } - }, - { - "id": "LilRg/PRYMMAL-ECE-7B-SLERP-V3", - "name": "PRYMMAL-ECE-7B-SLERP-V3", - "developer": "LilRg", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1243, - "hfopenllm_v2/BBH": 0.2957, - "hfopenllm_v2/MATH Level 5": 0.0098, - "hfopenllm_v2/GPQA": 0.2567, - "hfopenllm_v2/MUSR": 0.3671, - "hfopenllm_v2/MMLU-PRO": 0.1127 - } - }, - { - "id": "LilRg/PRYMMAL-ECE-7B-SLERP-V4", - "name": "PRYMMAL-ECE-7B-SLERP-V4", - "developer": "LilRg", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1249, - "hfopenllm_v2/BBH": 0.2957, - "hfopenllm_v2/MATH Level 5": 0.0098, - "hfopenllm_v2/GPQA": 0.2567, - "hfopenllm_v2/MUSR": 0.3671, - "hfopenllm_v2/MMLU-PRO": 0.1127 - } - }, - { - "id": "LilRg/PRYMMAL-ECE-7B-SLERP-V5", - "name": "PRYMMAL-ECE-7B-SLERP-V5", - "developer": "LilRg", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1249, - "hfopenllm_v2/BBH": 0.2957, - "hfopenllm_v2/MATH Level 5": 0.0098, - "hfopenllm_v2/GPQA": 0.2567, - "hfopenllm_v2/MUSR": 0.3671, - "hfopenllm_v2/MMLU-PRO": 0.1127 - } - }, - { - "id": "LilRg/PRYMMAL-ECE-7B-SLERP-V6", - "name": "PRYMMAL-ECE-7B-SLERP-V6", - "developer": "LilRg", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1243, - "hfopenllm_v2/BBH": 0.2957, - "hfopenllm_v2/MATH Level 5": 0.0098, - "hfopenllm_v2/GPQA": 0.2567, - "hfopenllm_v2/MUSR": 0.3671, - "hfopenllm_v2/MMLU-PRO": 0.1127 - } - }, - { - "id": "LilRg/PRYMMAL-ECE-7B-SLERP-V7", - "name": "PRYMMAL-ECE-7B-SLERP-V7", - "developer": "LilRg", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1249, - "hfopenllm_v2/BBH": 0.2957, - "hfopenllm_v2/MATH Level 5": 0.0098, - "hfopenllm_v2/GPQA": 0.2567, - "hfopenllm_v2/MUSR": 0.3671, - "hfopenllm_v2/MMLU-PRO": 0.1127 - } - }, - { - "id": "LilRg/PRYMMAL-slerp-Merge", - "name": "PRYMMAL-slerp-Merge", - "developer": "LilRg", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3044, - "hfopenllm_v2/BBH": 0.5364, - "hfopenllm_v2/MATH Level 5": 0.1616, - "hfopenllm_v2/GPQA": 0.3205, - "hfopenllm_v2/MUSR": 0.4635, - "hfopenllm_v2/MMLU-PRO": 0.3863 - } - }, - { - "id": "LimYeri/CodeMind-Llama3-8B-unsloth_v2-merged", - "name": "CodeMind-Llama3-8B-unsloth_v2-merged", - "developer": "LimYeri", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6946, - "hfopenllm_v2/BBH": 0.486, - "hfopenllm_v2/MATH Level 5": 0.0665, - "hfopenllm_v2/GPQA": 0.2651, - "hfopenllm_v2/MUSR": 0.3316, - "hfopenllm_v2/MMLU-PRO": 0.3506 - } - }, - { - "id": "LimYeri/CodeMind-Llama3-8B-unsloth_v3-merged", - "name": "CodeMind-Llama3-8B-unsloth_v3-merged", - "developer": "LimYeri", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6763, - "hfopenllm_v2/BBH": 0.4908, - "hfopenllm_v2/MATH Level 5": 0.068, - "hfopenllm_v2/GPQA": 0.2584, - "hfopenllm_v2/MUSR": 0.3356, - "hfopenllm_v2/MMLU-PRO": 0.3496 - } - }, - { - "id": "LimYeri/CodeMind-Llama3-8B-unsloth_v4-one-DPO-merged", - "name": "CodeMind-Llama3-8B-unsloth_v4-one-DPO-merged", - "developer": "LimYeri", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6492, - "hfopenllm_v2/BBH": 0.4853, - "hfopenllm_v2/MATH Level 5": 0.068, - "hfopenllm_v2/GPQA": 0.2685, - "hfopenllm_v2/MUSR": 0.3608, - "hfopenllm_v2/MMLU-PRO": 0.3354 - } - }, - { - "id": "LimYeri/CodeMind-Llama3-8B-unsloth_v4-one-merged", - "name": "CodeMind-Llama3-8B-unsloth_v4-one-merged", - "developer": "LimYeri", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3211, - "hfopenllm_v2/BBH": 0.4739, - "hfopenllm_v2/MATH Level 5": 0.0551, - "hfopenllm_v2/GPQA": 0.3096, - "hfopenllm_v2/MUSR": 0.4069, - "hfopenllm_v2/MMLU-PRO": 0.3353 - } - }, - { - "id": "LimYeri/CodeMind-Llama3.1-8B-unsloth-merged", - "name": "CodeMind-Llama3.1-8B-unsloth-merged", - "developer": "LimYeri", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.649, - "hfopenllm_v2/BBH": 0.4695, - "hfopenllm_v2/MATH Level 5": 0.1088, - "hfopenllm_v2/GPQA": 0.2643, - "hfopenllm_v2/MUSR": 0.3752, - "hfopenllm_v2/MMLU-PRO": 0.334 - } - }, - { - "id": "lkoenig/BBAI_145_", - "name": "BBAI_145_", - "developer": "lkoenig", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.445, - "hfopenllm_v2/BBH": 0.5567, - "hfopenllm_v2/MATH Level 5": 0.361, - "hfopenllm_v2/GPQA": 0.3163, - "hfopenllm_v2/MUSR": 0.4382, - "hfopenllm_v2/MMLU-PRO": 0.449 - } - }, - { - "id": "lkoenig/BBAI_200_Gemma", - "name": "BBAI_200_Gemma", - "developer": "lkoenig", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0705, - "hfopenllm_v2/BBH": 0.3449, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2668, - "hfopenllm_v2/MUSR": 0.3631, - "hfopenllm_v2/MMLU-PRO": 0.1679 - } - }, - { - "id": "lkoenig/BBAI_212_Qwencore", - "name": "BBAI_212_Qwencore", - "developer": "lkoenig", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4384, - "hfopenllm_v2/BBH": 0.5569, - "hfopenllm_v2/MATH Level 5": 0.3489, - "hfopenllm_v2/GPQA": 0.3163, - "hfopenllm_v2/MUSR": 0.4343, - "hfopenllm_v2/MMLU-PRO": 0.449 - } - }, - { - "id": "lkoenig/BBAI_212_QwenLawLo", - "name": "BBAI_212_QwenLawLo", - "developer": "lkoenig", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4566, - "hfopenllm_v2/BBH": 0.5574, - "hfopenllm_v2/MATH Level 5": 0.3603, - "hfopenllm_v2/GPQA": 0.3163, - "hfopenllm_v2/MUSR": 0.437, - "hfopenllm_v2/MMLU-PRO": 0.4489 - } - }, - { - "id": "lkoenig/BBAI_230_Xiaqwen", - "name": "BBAI_230_Xiaqwen", - "developer": "lkoenig", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4649, - "hfopenllm_v2/BBH": 0.5578, - "hfopenllm_v2/MATH Level 5": 0.3663, - "hfopenllm_v2/GPQA": 0.3138, - "hfopenllm_v2/MUSR": 0.4422, - "hfopenllm_v2/MMLU-PRO": 0.4481 - } - }, - { - "id": "lkoenig/BBAI_375_QwenDyancabs", - "name": "BBAI_375_QwenDyancabs", - "developer": "lkoenig", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4566, - "hfopenllm_v2/BBH": 0.5571, - "hfopenllm_v2/MATH Level 5": 0.3776, - "hfopenllm_v2/GPQA": 0.3129, - "hfopenllm_v2/MUSR": 0.4462, - "hfopenllm_v2/MMLU-PRO": 0.4476 - } - }, - { - "id": "lkoenig/BBAI_456_QwenKoen", - "name": "BBAI_456_QwenKoen", - "developer": "lkoenig", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4529, - "hfopenllm_v2/BBH": 0.5553, - "hfopenllm_v2/MATH Level 5": 0.3686, - "hfopenllm_v2/GPQA": 0.3129, - "hfopenllm_v2/MUSR": 0.4395, - "hfopenllm_v2/MMLU-PRO": 0.4469 - } - }, - { - "id": "lkoenig/BBAI_7B_KoenQwenDyan", - "name": "BBAI_7B_KoenQwenDyan", - "developer": "lkoenig", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5807, - "hfopenllm_v2/BBH": 0.5537, - "hfopenllm_v2/MATH Level 5": 0.3739, - "hfopenllm_v2/GPQA": 0.318, - "hfopenllm_v2/MUSR": 0.4369, - "hfopenllm_v2/MMLU-PRO": 0.446 - } - }, - { - "id": "lkoenig/BBAI_7B_Qwen2.5koen", - "name": "BBAI_7B_Qwen2.5koen", - "developer": "lkoenig", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.46, - "hfopenllm_v2/BBH": 0.5544, - "hfopenllm_v2/MATH Level 5": 0.3656, - "hfopenllm_v2/GPQA": 0.3129, - "hfopenllm_v2/MUSR": 0.4369, - "hfopenllm_v2/MMLU-PRO": 0.4485 - } - }, - { - "id": "lkoenig/BBAI_7B_QwenDyancabsLAW", - "name": "BBAI_7B_QwenDyancabsLAW", - "developer": "lkoenig", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.555, - "hfopenllm_v2/BBH": 0.5579, - "hfopenllm_v2/MATH Level 5": 0.3678, - "hfopenllm_v2/GPQA": 0.3188, - "hfopenllm_v2/MUSR": 0.4461, - "hfopenllm_v2/MMLU-PRO": 0.4471 - } - }, - { - "id": "lkoenig/BBAI_7B_QwenDyanKoenLo", - "name": "BBAI_7B_QwenDyanKoenLo", - "developer": "lkoenig", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4663, - "hfopenllm_v2/BBH": 0.5562, - "hfopenllm_v2/MATH Level 5": 0.364, - "hfopenllm_v2/GPQA": 0.3188, - "hfopenllm_v2/MUSR": 0.4343, - "hfopenllm_v2/MMLU-PRO": 0.4465 - } - }, - { - "id": "llm-blender/PairRM-hf", - "name": "llm-blender/PairRM-hf", - "developer": "llm-blender", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6087, - "reward-bench/Chat": 0.9022, - "reward-bench/Chat Hard": 0.5219, - "reward-bench/Safety": 0.477, - "reward-bench/Reasoning": 0.4898, - "reward-bench/Prior Sets (0.5 weight)": 0.6961 - } - }, - { - "id": "LLM360/K2", - "name": "K2", - "developer": "LLM360", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2252, - "hfopenllm_v2/BBH": 0.4972, - "hfopenllm_v2/MATH Level 5": 0.0272, - "hfopenllm_v2/GPQA": 0.2768, - "hfopenllm_v2/MUSR": 0.398, - "hfopenllm_v2/MMLU-PRO": 0.3004 - } - }, - { - "id": "LLM360/K2-Chat", - "name": "K2-Chat", - "developer": "LLM360", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5152, - "hfopenllm_v2/BBH": 0.5358, - "hfopenllm_v2/MATH Level 5": 0.1035, - "hfopenllm_v2/GPQA": 0.3062, - "hfopenllm_v2/MUSR": 0.457, - "hfopenllm_v2/MMLU-PRO": 0.3371 - } - }, - { - "id": "LLM4Binary/llm4decompile-1.3b-v2", - "name": "llm4decompile-1.3b-v2", - "developer": "LLM4Binary", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2268, - "hfopenllm_v2/BBH": 0.3272, - "hfopenllm_v2/MATH Level 5": 0.0128, - "hfopenllm_v2/GPQA": 0.2357, - "hfopenllm_v2/MUSR": 0.4072, - "hfopenllm_v2/MMLU-PRO": 0.1209 - } - }, - { - "id": "llmat/Mistral-v0.3-7B-ORPO", - "name": "Mistral-v0.3-7B-ORPO", - "developer": "llmat", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.377, - "hfopenllm_v2/BBH": 0.3978, - "hfopenllm_v2/MATH Level 5": 0.0242, - "hfopenllm_v2/GPQA": 0.2668, - "hfopenllm_v2/MUSR": 0.3555, - "hfopenllm_v2/MMLU-PRO": 0.2278 - } - }, - { - "id": "llnYou/ECE-PRYMMAL-YL-1B-SLERP-V5", - "name": "ECE-PRYMMAL-YL-1B-SLERP-V5", - "developer": "llnYou", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3313, - "hfopenllm_v2/BBH": 0.4233, - "hfopenllm_v2/MATH Level 5": 0.111, - "hfopenllm_v2/GPQA": 0.2861, - "hfopenllm_v2/MUSR": 0.3868, - "hfopenllm_v2/MMLU-PRO": 0.2931 - } - }, - { - "id": "llnYou/ECE-PRYMMAL-YL-1B-SLERP-V6", - "name": "ECE-PRYMMAL-YL-1B-SLERP-V6", - "developer": "llnYou", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1388, - "hfopenllm_v2/BBH": 0.3944, - "hfopenllm_v2/MATH Level 5": 0.0023, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.3928, - "hfopenllm_v2/MMLU-PRO": 0.235 - } - }, - { - "id": "llnYou/ECE-PRYMMAL-YL-3B-SLERP-V1", - "name": "ECE-PRYMMAL-YL-3B-SLERP-V1", - "developer": "llnYou", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2346, - "hfopenllm_v2/BBH": 0.4018, - "hfopenllm_v2/MATH Level 5": 0.0091, - "hfopenllm_v2/GPQA": 0.2936, - "hfopenllm_v2/MUSR": 0.3364, - "hfopenllm_v2/MMLU-PRO": 0.285 - } - }, - { - "id": "llnYou/ECE-PRYMMAL-YL-3B-SLERP-V2", - "name": "ECE-PRYMMAL-YL-3B-SLERP-V2", - "developer": "llnYou", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2309, - "hfopenllm_v2/BBH": 0.399, - "hfopenllm_v2/MATH Level 5": 0.0128, - "hfopenllm_v2/GPQA": 0.2768, - "hfopenllm_v2/MUSR": 0.3588, - "hfopenllm_v2/MMLU-PRO": 0.29 - } - }, - { - "id": "llnYou/ECE-PRYMMAL-YL-3B-SLERP-V3", - "name": "ECE-PRYMMAL-YL-3B-SLERP-V3", - "developer": "llnYou", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3581, - "hfopenllm_v2/BBH": 0.5473, - "hfopenllm_v2/MATH Level 5": 0.1299, - "hfopenllm_v2/GPQA": 0.3045, - "hfopenllm_v2/MUSR": 0.4361, - "hfopenllm_v2/MMLU-PRO": 0.4043 - } - }, - { - "id": "lmsys/vicuna-13b-v1.3", - "name": "vicuna-13b-v1.3", - "developer": "lmsys", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3344, - "hfopenllm_v2/BBH": 0.3384, - "hfopenllm_v2/MATH Level 5": 0.0144, - "hfopenllm_v2/GPQA": 0.2676, - "hfopenllm_v2/MUSR": 0.3727, - "hfopenllm_v2/MMLU-PRO": 0.2243 - } - }, - { - "id": "lmsys/vicuna-7b-v1.3", - "name": "vicuna-7b-v1.3", - "developer": "lmsys", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2909, - "hfopenllm_v2/BBH": 0.3298, - "hfopenllm_v2/MATH Level 5": 0.0128, - "hfopenllm_v2/GPQA": 0.2424, - "hfopenllm_v2/MUSR": 0.3793, - "hfopenllm_v2/MMLU-PRO": 0.1838 - } - }, - { - "id": "lmsys/vicuna-7b-v1.5", - "name": "vicuna-7b-v1.5", - "developer": "lmsys", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2352, - "hfopenllm_v2/BBH": 0.3947, - "hfopenllm_v2/MATH Level 5": 0.0136, - "hfopenllm_v2/GPQA": 0.2584, - "hfopenllm_v2/MUSR": 0.4231, - "hfopenllm_v2/MMLU-PRO": 0.2147 - } - }, - { - "id": "lmsys/Vicuna-v1.3-13B", - "name": "Vicuna v1.3 13B", - "developer": "lmsys", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.706, - "helm_classic/MMLU": 0.462, - "helm_classic/BoolQ": 0.808, - "helm_classic/NarrativeQA": 0.691, - "helm_classic/NaturalQuestions (open-book)": 0.686, - "helm_classic/QuAC": 0.403, - "helm_classic/HellaSwag": -1.0, - "helm_classic/OpenbookQA": -1.0, - "helm_classic/TruthfulQA": 0.385, - "helm_classic/MS MARCO (TREC)": -1.0, - "helm_classic/CNN/DailyMail": -1.0, - "helm_classic/XSUM": -1.0, - "helm_classic/IMDB": 0.762, - "helm_classic/CivilComments": 0.645, - "helm_classic/RAFT": 0.657 - } - }, - { - "id": "lmsys/Vicuna-v1.3-7B", - "name": "Vicuna v1.3 7B", - "developer": "lmsys", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.625, - "helm_classic/MMLU": 0.434, - "helm_classic/BoolQ": 0.76, - "helm_classic/NarrativeQA": 0.643, - "helm_classic/NaturalQuestions (open-book)": 0.634, - "helm_classic/QuAC": 0.392, - "helm_classic/HellaSwag": -1.0, - "helm_classic/OpenbookQA": -1.0, - "helm_classic/TruthfulQA": 0.292, - "helm_classic/MS MARCO (TREC)": -1.0, - "helm_classic/CNN/DailyMail": -1.0, - "helm_classic/XSUM": -1.0, - "helm_classic/IMDB": 0.916, - "helm_classic/CivilComments": 0.62, - "helm_classic/RAFT": 0.693 - } - }, - { - "id": "Locutusque/CollectiveLM-Falcon-3-7B", - "name": "CollectiveLM-Falcon-3-7B", - "developer": "Locutusque", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3918, - "hfopenllm_v2/BBH": 0.5105, - "hfopenllm_v2/MATH Level 5": 0.2183, - "hfopenllm_v2/GPQA": 0.3255, - "hfopenllm_v2/MUSR": 0.3887, - "hfopenllm_v2/MMLU-PRO": 0.3599 - } - }, - { - "id": "Locutusque/Hercules-6.0-Llama-3.1-8B", - "name": "Hercules-6.0-Llama-3.1-8B", - "developer": "Locutusque", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.663, - "hfopenllm_v2/BBH": 0.4813, - "hfopenllm_v2/MATH Level 5": 0.1669, - "hfopenllm_v2/GPQA": 0.2643, - "hfopenllm_v2/MUSR": 0.3621, - "hfopenllm_v2/MMLU-PRO": 0.3615 - } - }, - { - "id": "Locutusque/Hercules-6.1-Llama-3.1-8B", - "name": "Hercules-6.1-Llama-3.1-8B", - "developer": "Locutusque", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6007, - "hfopenllm_v2/BBH": 0.4656, - "hfopenllm_v2/MATH Level 5": 0.176, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.3553, - "hfopenllm_v2/MMLU-PRO": 0.3669 - } - }, - { - "id": "Locutusque/Llama-3-NeuralHercules-5.0-8B", - "name": "Llama-3-NeuralHercules-5.0-8B", - "developer": "Locutusque", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4489, - "hfopenllm_v2/BBH": 0.394, - "hfopenllm_v2/MATH Level 5": 0.0431, - "hfopenllm_v2/GPQA": 0.2685, - "hfopenllm_v2/MUSR": 0.3881, - "hfopenllm_v2/MMLU-PRO": 0.2933 - } - }, - { - "id": "Locutusque/Llama-3-Yggdrasil-2.0-8B", - "name": "Llama-3-Yggdrasil-2.0-8B", - "developer": "Locutusque", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5371, - "hfopenllm_v2/BBH": 0.4772, - "hfopenllm_v2/MATH Level 5": 0.0831, - "hfopenllm_v2/GPQA": 0.2626, - "hfopenllm_v2/MUSR": 0.3977, - "hfopenllm_v2/MMLU-PRO": 0.3167 - } - }, - { - "id": "Locutusque/TinyMistral-248M-v2.5", - "name": "TinyMistral-248M-v2.5", - "developer": "Locutusque", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1336, - "hfopenllm_v2/BBH": 0.3039, - "hfopenllm_v2/MATH Level 5": 0.0098, - "hfopenllm_v2/GPQA": 0.2508, - "hfopenllm_v2/MUSR": 0.3782, - "hfopenllm_v2/MMLU-PRO": 0.1135 - } - }, - { - "id": "lodrick-the-lafted/llama-3.1-8b-instruct-ortho-v7", - "name": "llama-3.1-8b-instruct-ortho-v7", - "developer": "lodrick-the-lafted", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3515, - "hfopenllm_v2/BBH": 0.3907, - "hfopenllm_v2/MATH Level 5": 0.0272, - "hfopenllm_v2/GPQA": 0.2727, - "hfopenllm_v2/MUSR": 0.3616, - "hfopenllm_v2/MMLU-PRO": 0.1974 - } - }, - { - "id": "lordjia/Llama-3-Cantonese-8B-Instruct", - "name": "Llama-3-Cantonese-8B-Instruct", - "developer": "lordjia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6669, - "hfopenllm_v2/BBH": 0.4814, - "hfopenllm_v2/MATH Level 5": 0.0891, - "hfopenllm_v2/GPQA": 0.2936, - "hfopenllm_v2/MUSR": 0.4046, - "hfopenllm_v2/MMLU-PRO": 0.3515 - } - }, - { - "id": "lordjia/Qwen2-Cantonese-7B-Instruct", - "name": "Qwen2-Cantonese-7B-Instruct", - "developer": "lordjia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5435, - "hfopenllm_v2/BBH": 0.5215, - "hfopenllm_v2/MATH Level 5": 0.256, - "hfopenllm_v2/GPQA": 0.2953, - "hfopenllm_v2/MUSR": 0.4004, - "hfopenllm_v2/MMLU-PRO": 0.3843 - } - }, - { - "id": "lt-asset/nova-1.3b", - "name": "nova-1.3b", - "developer": "lt-asset", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1214, - "hfopenllm_v2/BBH": 0.317, - "hfopenllm_v2/MATH Level 5": 0.0121, - "hfopenllm_v2/GPQA": 0.2492, - "hfopenllm_v2/MUSR": 0.3698, - "hfopenllm_v2/MMLU-PRO": 0.1142 - } - }, - { - "id": "lunahr/thea-3b-50r-u1", - "name": "thea-3b-50r-u1", - "developer": "lunahr", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.603, - "hfopenllm_v2/BBH": 0.4105, - "hfopenllm_v2/MATH Level 5": 0.1042, - "hfopenllm_v2/GPQA": 0.2836, - "hfopenllm_v2/MUSR": 0.3182, - "hfopenllm_v2/MMLU-PRO": 0.2808 - } - }, - { - "id": "lunahr/thea-v2-3b-50r", - "name": "thea-v2-3b-50r", - "developer": "lunahr", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3704, - "hfopenllm_v2/BBH": 0.4194, - "hfopenllm_v2/MATH Level 5": 0.0242, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.3222, - "hfopenllm_v2/MMLU-PRO": 0.2409 - } - }, - { - "id": "Luni/StarDust-12b-v1", - "name": "StarDust-12b-v1", - "developer": "Luni", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5459, - "hfopenllm_v2/BBH": 0.5366, - "hfopenllm_v2/MATH Level 5": 0.0763, - "hfopenllm_v2/GPQA": 0.276, - "hfopenllm_v2/MUSR": 0.4324, - "hfopenllm_v2/MMLU-PRO": 0.3412 - } - }, - { - "id": "Luni/StarDust-12b-v2", - "name": "StarDust-12b-v2", - "developer": "Luni", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5629, - "hfopenllm_v2/BBH": 0.5419, - "hfopenllm_v2/MATH Level 5": 0.0687, - "hfopenllm_v2/GPQA": 0.2936, - "hfopenllm_v2/MUSR": 0.4338, - "hfopenllm_v2/MMLU-PRO": 0.3439 - } - }, - { - "id": "Lunzima/NQLSG-Qwen2.5-14B-MegaFusion-v3", - "name": "NQLSG-Qwen2.5-14B-MegaFusion-v3", - "developer": "Lunzima", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7049, - "hfopenllm_v2/BBH": 0.6478, - "hfopenllm_v2/MATH Level 5": 0.4162, - "hfopenllm_v2/GPQA": 0.3817, - "hfopenllm_v2/MUSR": 0.4808, - "hfopenllm_v2/MMLU-PRO": 0.5394 - } - }, - { - "id": "Lunzima/NQLSG-Qwen2.5-14B-MegaFusion-v4", - "name": "NQLSG-Qwen2.5-14B-MegaFusion-v4", - "developer": "Lunzima", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6943, - "hfopenllm_v2/BBH": 0.642, - "hfopenllm_v2/MATH Level 5": 0.3467, - "hfopenllm_v2/GPQA": 0.3716, - "hfopenllm_v2/MUSR": 0.4769, - "hfopenllm_v2/MMLU-PRO": 0.5252 - } - }, - { - "id": "Lunzima/NQLSG-Qwen2.5-14B-MegaFusion-v5", - "name": "NQLSG-Qwen2.5-14B-MegaFusion-v5", - "developer": "Lunzima", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7485, - "hfopenllm_v2/BBH": 0.6467, - "hfopenllm_v2/MATH Level 5": 0.4358, - "hfopenllm_v2/GPQA": 0.3624, - "hfopenllm_v2/MUSR": 0.4473, - "hfopenllm_v2/MMLU-PRO": 0.514 - } - }, - { - "id": "Lunzima/NQLSG-Qwen2.5-14B-MegaFusion-v6", - "name": "NQLSG-Qwen2.5-14B-MegaFusion-v6", - "developer": "Lunzima", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7043, - "hfopenllm_v2/BBH": 0.6458, - "hfopenllm_v2/MATH Level 5": 0.3958, - "hfopenllm_v2/GPQA": 0.3775, - "hfopenllm_v2/MUSR": 0.4768, - "hfopenllm_v2/MMLU-PRO": 0.5392 - } - }, - { - "id": "Lunzima/NQLSG-Qwen2.5-14B-MegaFusion-v6-cpt", - "name": "NQLSG-Qwen2.5-14B-MegaFusion-v6-cpt", - "developer": "Lunzima", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4663, - "hfopenllm_v2/BBH": 0.6215, - "hfopenllm_v2/MATH Level 5": 0.3316, - "hfopenllm_v2/GPQA": 0.3758, - "hfopenllm_v2/MUSR": 0.4937, - "hfopenllm_v2/MMLU-PRO": 0.5204 - } - }, - { - "id": "Lunzima/NQLSG-Qwen2.5-14B-MegaFusion-v7", - "name": "NQLSG-Qwen2.5-14B-MegaFusion-v7", - "developer": "Lunzima", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6794, - "hfopenllm_v2/BBH": 0.6531, - "hfopenllm_v2/MATH Level 5": 0.4101, - "hfopenllm_v2/GPQA": 0.3792, - "hfopenllm_v2/MUSR": 0.4834, - "hfopenllm_v2/MMLU-PRO": 0.5376 - } - }, - { - "id": "Lunzima/NQLSG-Qwen2.5-14B-MegaFusion-v7-rebase", - "name": "NQLSG-Qwen2.5-14B-MegaFusion-v7-rebase", - "developer": "Lunzima", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6931, - "hfopenllm_v2/BBH": 0.6423, - "hfopenllm_v2/MATH Level 5": 0.3406, - "hfopenllm_v2/GPQA": 0.375, - "hfopenllm_v2/MUSR": 0.4888, - "hfopenllm_v2/MMLU-PRO": 0.5277 - } - }, - { - "id": "Lunzima/NQLSG-Qwen2.5-14B-MegaFusion-v8", - "name": "NQLSG-Qwen2.5-14B-MegaFusion-v8", - "developer": "Lunzima", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7875, - "hfopenllm_v2/BBH": 0.6419, - "hfopenllm_v2/MATH Level 5": 0.5559, - "hfopenllm_v2/GPQA": 0.3356, - "hfopenllm_v2/MUSR": 0.4394, - "hfopenllm_v2/MMLU-PRO": 0.5206 - } - }, - { - "id": "Lunzima/NQLSG-Qwen2.5-14B-MegaFusion-v8.5", - "name": "NQLSG-Qwen2.5-14B-MegaFusion-v8.5", - "developer": "Lunzima", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5929, - "hfopenllm_v2/BBH": 0.6451, - "hfopenllm_v2/MATH Level 5": 0.3656, - "hfopenllm_v2/GPQA": 0.38, - "hfopenllm_v2/MUSR": 0.477, - "hfopenllm_v2/MMLU-PRO": 0.529 - } - }, - { - "id": "Lunzima/NQLSG-Qwen2.5-14B-MegaFusion-v8.6", - "name": "NQLSG-Qwen2.5-14B-MegaFusion-v8.6", - "developer": "Lunzima", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5919, - "hfopenllm_v2/BBH": 0.6457, - "hfopenllm_v2/MATH Level 5": 0.4071, - "hfopenllm_v2/GPQA": 0.3842, - "hfopenllm_v2/MUSR": 0.4953, - "hfopenllm_v2/MMLU-PRO": 0.54 - } - }, - { - "id": "Lunzima/NQLSG-Qwen2.5-14B-MegaFusion-v8.7", - "name": "NQLSG-Qwen2.5-14B-MegaFusion-v8.7", - "developer": "Lunzima", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7875, - "hfopenllm_v2/BBH": 0.6483, - "hfopenllm_v2/MATH Level 5": 0.5408, - "hfopenllm_v2/GPQA": 0.3515, - "hfopenllm_v2/MUSR": 0.4381, - "hfopenllm_v2/MMLU-PRO": 0.5242 - } - }, - { - "id": "Lunzima/NQLSG-Qwen2.5-14B-MegaFusion-v8.8", - "name": "NQLSG-Qwen2.5-14B-MegaFusion-v8.8", - "developer": "Lunzima", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7028, - "hfopenllm_v2/BBH": 0.6566, - "hfopenllm_v2/MATH Level 5": 0.4237, - "hfopenllm_v2/GPQA": 0.3758, - "hfopenllm_v2/MUSR": 0.4912, - "hfopenllm_v2/MMLU-PRO": 0.5323 - } - }, - { - "id": "Lunzima/NQLSG-Qwen2.5-14B-MegaFusion-v8.9", - "name": "NQLSG-Qwen2.5-14B-MegaFusion-v8.9", - "developer": "Lunzima", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7993, - "hfopenllm_v2/BBH": 0.6483, - "hfopenllm_v2/MATH Level 5": 0.537, - "hfopenllm_v2/GPQA": 0.3297, - "hfopenllm_v2/MUSR": 0.4328, - "hfopenllm_v2/MMLU-PRO": 0.5199 - } - }, - { - "id": "Lunzima/NQLSG-Qwen2.5-14B-MegaFusion-v9", - "name": "NQLSG-Qwen2.5-14B-MegaFusion-v9", - "developer": "Lunzima", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5235, - "hfopenllm_v2/BBH": 0.6546, - "hfopenllm_v2/MATH Level 5": 0.4366, - "hfopenllm_v2/GPQA": 0.3884, - "hfopenllm_v2/MUSR": 0.4806, - "hfopenllm_v2/MMLU-PRO": 0.5422 - } - }, - { - "id": "Lunzima/NQLSG-Qwen2.5-14B-MegaFusion-v9-stock", - "name": "NQLSG-Qwen2.5-14B-MegaFusion-v9-stock", - "developer": "Lunzima", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6514, - "hfopenllm_v2/BBH": 0.6571, - "hfopenllm_v2/MATH Level 5": 0.4184, - "hfopenllm_v2/GPQA": 0.3842, - "hfopenllm_v2/MUSR": 0.482, - "hfopenllm_v2/MMLU-PRO": 0.5412 - } - }, - { - "id": "Lunzima/NQLSG-Qwen2.5-14B-MegaFusion-v9.1", - "name": "NQLSG-Qwen2.5-14B-MegaFusion-v9.1", - "developer": "Lunzima", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8003, - "hfopenllm_v2/BBH": 0.6555, - "hfopenllm_v2/MATH Level 5": 0.5468, - "hfopenllm_v2/GPQA": 0.3431, - "hfopenllm_v2/MUSR": 0.4354, - "hfopenllm_v2/MMLU-PRO": 0.5251 - } - }, - { - "id": "Lunzima/NQLSG-Qwen2.5-14B-MegaFusion-v9.2", - "name": "NQLSG-Qwen2.5-14B-MegaFusion-v9.2", - "developer": "Lunzima", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7862, - "hfopenllm_v2/BBH": 0.6538, - "hfopenllm_v2/MATH Level 5": 0.5332, - "hfopenllm_v2/GPQA": 0.3557, - "hfopenllm_v2/MUSR": 0.4381, - "hfopenllm_v2/MMLU-PRO": 0.5283 - } - }, - { - "id": "Lunzima/NQLSG-Qwen2.5-14B-OriginalFusion", - "name": "NQLSG-Qwen2.5-14B-OriginalFusion", - "developer": "Lunzima", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6142, - "hfopenllm_v2/BBH": 0.6592, - "hfopenllm_v2/MATH Level 5": 0.4275, - "hfopenllm_v2/GPQA": 0.3809, - "hfopenllm_v2/MUSR": 0.5122, - "hfopenllm_v2/MMLU-PRO": 0.5239 - } - }, - { - "id": "LxzGordon/URM-LLaMa-3-8B", - "name": "LxzGordon/URM-LLaMa-3-8B", - "developer": "LxzGordon", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.8991, - "reward-bench/Chat": 0.9693, - "reward-bench/Chat Hard": 0.7873, - "reward-bench/Safety": 0.8824, - "reward-bench/Reasoning": 0.9574 - } - }, - { - "id": "LxzGordon/URM-LLaMa-3.1-8B", - "name": "LxzGordon/URM-LLaMa-3.1-8B", - "developer": "LxzGordon", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7394, - "reward-bench/Chat": 0.9553, - "reward-bench/Chat Hard": 0.8816, - "reward-bench/Safety": 0.9178, - "reward-bench/Reasoning": 0.9698, - "reward-bench/Factuality": 0.6884, - "reward-bench/Precise IF": 0.45, - "reward-bench/Math": 0.6393, - "reward-bench/Focus": 0.9758, - "reward-bench/Ties": 0.7653 - } - }, - { - "id": "Lyte/Llama-3.1-8B-Instruct-Reasoner-1o1_v0.3", - "name": "Llama-3.1-8B-Instruct-Reasoner-1o1_v0.3", - "developer": "Lyte", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7098, - "hfopenllm_v2/BBH": 0.495, - "hfopenllm_v2/MATH Level 5": 0.1903, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.3461, - "hfopenllm_v2/MMLU-PRO": 0.3618 - } - }, - { - "id": "Lyte/Llama-3.2-1B-Instruct-COT-RL-Expriement1-EP04", - "name": "Llama-3.2-1B-Instruct-COT-RL-Expriement1-EP04", - "developer": "Lyte", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5774, - "hfopenllm_v2/BBH": 0.3515, - "hfopenllm_v2/MATH Level 5": 0.0801, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3236, - "hfopenllm_v2/MMLU-PRO": 0.1843 - } - }, - { - "id": "Lyte/Llama-3.2-3B-Overthinker", - "name": "Llama-3.2-3B-Overthinker", - "developer": "Lyte", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6408, - "hfopenllm_v2/BBH": 0.432, - "hfopenllm_v2/MATH Level 5": 0.1563, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3419, - "hfopenllm_v2/MMLU-PRO": 0.2985 - } - }, - { - "id": "M4-ai/TinyMistral-248M-v3", - "name": "TinyMistral-248M-v3", - "developer": "M4-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1639, - "hfopenllm_v2/BBH": 0.2885, - "hfopenllm_v2/MATH Level 5": 0.0045, - "hfopenllm_v2/GPQA": 0.2408, - "hfopenllm_v2/MUSR": 0.3793, - "hfopenllm_v2/MMLU-PRO": 0.1132 - } - }, - { - "id": "m42-health/Llama3-Med42-70B", - "name": "Llama3-Med42-70B", - "developer": "m42-health", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6291, - "hfopenllm_v2/BBH": 0.6688, - "hfopenllm_v2/MATH Level 5": 0.2258, - "hfopenllm_v2/GPQA": 0.3473, - "hfopenllm_v2/MUSR": 0.4629, - "hfopenllm_v2/MMLU-PRO": 0.4963 - } - }, - { - "id": "macadeliccc/magistrate-3.2-3b-base", - "name": "magistrate-3.2-3b-base", - "developer": "macadeliccc", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1159, - "hfopenllm_v2/BBH": 0.3343, - "hfopenllm_v2/MATH Level 5": 0.0113, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.3976, - "hfopenllm_v2/MMLU-PRO": 0.1689 - } - }, - { - "id": "macadeliccc/magistrate-3.2-3b-it", - "name": "magistrate-3.2-3b-it", - "developer": "macadeliccc", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2292, - "hfopenllm_v2/BBH": 0.3257, - "hfopenllm_v2/MATH Level 5": 0.0196, - "hfopenllm_v2/GPQA": 0.2475, - "hfopenllm_v2/MUSR": 0.3763, - "hfopenllm_v2/MMLU-PRO": 0.1592 - } - }, - { - "id": "macadeliccc/Samantha-Qwen-2-7B", - "name": "Samantha-Qwen-2-7B", - "developer": "macadeliccc", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4377, - "hfopenllm_v2/BBH": 0.5082, - "hfopenllm_v2/MATH Level 5": 0.2115, - "hfopenllm_v2/GPQA": 0.2727, - "hfopenllm_v2/MUSR": 0.4799, - "hfopenllm_v2/MMLU-PRO": 0.3779 - } - }, - { - "id": "madeagents/hammer2-1-0-5b-fc", - "name": "Hammer2.1-0.5b (FC)", - "developer": "madeagents", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 100.0, - "bfcl/bfcl.overall.overall_accuracy": 21.22, - "bfcl/bfcl.overall.total_cost_usd": 2.82, - "bfcl/bfcl.overall.latency_mean_s": 2.79, - "bfcl/bfcl.overall.latency_std_s": 3.17, - "bfcl/bfcl.overall.latency_p95_s": 9.86, - "bfcl/bfcl.non_live.ast_accuracy": 65.98, - "bfcl/bfcl.non_live.simple_ast_accuracy": 62.42, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 81.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 69.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 51.5, - "bfcl/bfcl.live.live_accuracy": 54.63, - "bfcl/bfcl.live.live_simple_ast_accuracy": 56.59, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 54.42, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 62.5, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 37.5, - "bfcl/bfcl.multi_turn.accuracy": 2.88, - "bfcl/bfcl.multi_turn.base_accuracy": 4.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 0.5, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 4.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 2.5, - "bfcl/bfcl.web_search.accuracy": 0.0, - "bfcl/bfcl.web_search.base_accuracy": 0.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 0.0, - "bfcl/bfcl.memory.accuracy": 1.08, - "bfcl/bfcl.memory.kv_accuracy": 0.65, - "bfcl/bfcl.memory.vector_accuracy": 1.94, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 0.65, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 68.75, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 80.79 - } - }, - { - "id": "madeagents/hammer2-1-1-5b-fc", - "name": "Hammer2.1-1.5b (FC)", - "developer": "madeagents", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 75.0, - "bfcl/bfcl.overall.overall_accuracy": 27.88, - "bfcl/bfcl.overall.total_cost_usd": 6.83, - "bfcl/bfcl.overall.latency_mean_s": 6.28, - "bfcl/bfcl.overall.latency_std_s": 8.79, - "bfcl/bfcl.overall.latency_p95_s": 30.72, - "bfcl/bfcl.non_live.ast_accuracy": 82.98, - "bfcl/bfcl.non_live.simple_ast_accuracy": 73.92, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 92.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 85.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 80.5, - "bfcl/bfcl.live.live_accuracy": 69.5, - "bfcl/bfcl.live.live_simple_ast_accuracy": 72.09, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 69.33, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 50.0, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 62.5, - "bfcl/bfcl.multi_turn.accuracy": 15.62, - "bfcl/bfcl.multi_turn.base_accuracy": 20.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 16.5, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 9.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 16.5, - "bfcl/bfcl.web_search.accuracy": 0.0, - "bfcl/bfcl.web_search.base_accuracy": 0.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 0.0, - "bfcl/bfcl.memory.accuracy": 0.0, - "bfcl/bfcl.memory.kv_accuracy": 0.0, - "bfcl/bfcl.memory.vector_accuracy": 0.0, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 0.0, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 75.0, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 79.4 - } - }, - { - "id": "madeagents/hammer2-1-3b-fc", - "name": "Hammer2.1-3b (FC)", - "developer": "madeagents", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 68.0, - "bfcl/bfcl.overall.overall_accuracy": 29.71, - "bfcl/bfcl.overall.total_cost_usd": 10.89, - "bfcl/bfcl.overall.latency_mean_s": 11.24, - "bfcl/bfcl.overall.latency_std_s": 15.81, - "bfcl/bfcl.overall.latency_p95_s": 47.44, - "bfcl/bfcl.non_live.ast_accuracy": 84.96, - "bfcl/bfcl.non_live.simple_ast_accuracy": 79.33, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 93.5, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 86.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 80.5, - "bfcl/bfcl.live.live_accuracy": 70.54, - "bfcl/bfcl.live.live_simple_ast_accuracy": 68.22, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 71.32, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 62.5, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 66.67, - "bfcl/bfcl.multi_turn.accuracy": 16.5, - "bfcl/bfcl.multi_turn.base_accuracy": 22.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 12.5, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 16.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 15.5, - "bfcl/bfcl.web_search.accuracy": 0.0, - "bfcl/bfcl.web_search.base_accuracy": 0.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 0.0, - "bfcl/bfcl.memory.accuracy": 3.01, - "bfcl/bfcl.memory.kv_accuracy": 2.58, - "bfcl/bfcl.memory.vector_accuracy": 3.87, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 2.58, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 56.25, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 86.12 - } - }, - { - "id": "madeagents/hammer2-1-7b-fc", - "name": "Hammer2.1-7b (FC)", - "developer": "madeagents", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 64.0, - "bfcl/bfcl.overall.overall_accuracy": 31.67, - "bfcl/bfcl.overall.total_cost_usd": 4.99, - "bfcl/bfcl.overall.latency_mean_s": 5.77, - "bfcl/bfcl.overall.latency_std_s": 10.29, - "bfcl/bfcl.overall.latency_p95_s": 31.26, - "bfcl/bfcl.non_live.ast_accuracy": 85.5, - "bfcl/bfcl.non_live.simple_ast_accuracy": 72.5, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 92.5, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 91.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 86.0, - "bfcl/bfcl.live.live_accuracy": 69.5, - "bfcl/bfcl.live.live_simple_ast_accuracy": 66.67, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 69.99, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 75.0, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 75.0, - "bfcl/bfcl.multi_turn.accuracy": 23.87, - "bfcl/bfcl.multi_turn.base_accuracy": 24.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 28.5, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 21.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 21.5, - "bfcl/bfcl.web_search.accuracy": 0.0, - "bfcl/bfcl.web_search.base_accuracy": 0.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 0.0, - "bfcl/bfcl.memory.accuracy": 0.0, - "bfcl/bfcl.memory.kv_accuracy": 0.0, - "bfcl/bfcl.memory.vector_accuracy": 0.0, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 0.0, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 50.0, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 90.12 - } - }, - { - "id": "magnifi/Phi3_intent_v56_3_w_unknown_5_lr_0.002", - "name": "Phi3_intent_v56_3_w_unknown_5_lr_0.002", - "developer": "magnifi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2018, - "hfopenllm_v2/BBH": 0.3282, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2643, - "hfopenllm_v2/MUSR": 0.4123, - "hfopenllm_v2/MMLU-PRO": 0.1472 - } - }, - { - "id": "Magpie-Align/Llama-3-8B-Magpie-Align-SFT-v0.1", - "name": "Llama-3-8B-Magpie-Align-SFT-v0.1", - "developer": "Magpie-Align", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4361, - "hfopenllm_v2/BBH": 0.4615, - "hfopenllm_v2/MATH Level 5": 0.0574, - "hfopenllm_v2/GPQA": 0.2626, - "hfopenllm_v2/MUSR": 0.3277, - "hfopenllm_v2/MMLU-PRO": 0.2863 - } - }, - { - "id": "Magpie-Align/Llama-3-8B-Magpie-Align-SFT-v0.3", - "name": "Llama-3-8B-Magpie-Align-SFT-v0.3", - "developer": "Magpie-Align", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5064, - "hfopenllm_v2/BBH": 0.4572, - "hfopenllm_v2/MATH Level 5": 0.0733, - "hfopenllm_v2/GPQA": 0.2659, - "hfopenllm_v2/MUSR": 0.3424, - "hfopenllm_v2/MMLU-PRO": 0.2902 - } - }, - { - "id": "Magpie-Align/Llama-3-8B-Magpie-Align-v0.1", - "name": "Llama-3-8B-Magpie-Align-v0.1", - "developer": "Magpie-Align", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4027, - "hfopenllm_v2/BBH": 0.4789, - "hfopenllm_v2/MATH Level 5": 0.0461, - "hfopenllm_v2/GPQA": 0.2768, - "hfopenllm_v2/MUSR": 0.3087, - "hfopenllm_v2/MMLU-PRO": 0.3001 - } - }, - { - "id": "Magpie-Align/Llama-3-8B-Magpie-Align-v0.3", - "name": "Llama-3-8B-Magpie-Align-v0.3", - "developer": "Magpie-Align", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4497, - "hfopenllm_v2/BBH": 0.457, - "hfopenllm_v2/MATH Level 5": 0.0566, - "hfopenllm_v2/GPQA": 0.2651, - "hfopenllm_v2/MUSR": 0.3406, - "hfopenllm_v2/MMLU-PRO": 0.3134 - } - }, - { - "id": "Magpie-Align/Llama-3.1-8B-Magpie-Align-SFT-v0.1", - "name": "Llama-3.1-8B-Magpie-Align-SFT-v0.1", - "developer": "Magpie-Align", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4782, - "hfopenllm_v2/BBH": 0.4764, - "hfopenllm_v2/MATH Level 5": 0.0899, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.3397, - "hfopenllm_v2/MMLU-PRO": 0.2943 - } - }, - { - "id": "Magpie-Align/Llama-3.1-8B-Magpie-Align-v0.1", - "name": "Llama-3.1-8B-Magpie-Align-v0.1", - "developer": "Magpie-Align", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4458, - "hfopenllm_v2/BBH": 0.4622, - "hfopenllm_v2/MATH Level 5": 0.0665, - "hfopenllm_v2/GPQA": 0.2634, - "hfopenllm_v2/MUSR": 0.3141, - "hfopenllm_v2/MMLU-PRO": 0.3262 - } - }, - { - "id": "Magpie-Align/MagpieLM-8B-Chat-v0.1", - "name": "MagpieLM-8B-Chat-v0.1", - "developer": "Magpie-Align", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3701, - "hfopenllm_v2/BBH": 0.4172, - "hfopenllm_v2/MATH Level 5": 0.0612, - "hfopenllm_v2/GPQA": 0.2617, - "hfopenllm_v2/MUSR": 0.3501, - "hfopenllm_v2/MMLU-PRO": 0.3195 - } - }, - { - "id": "Magpie-Align/MagpieLM-8B-SFT-v0.1", - "name": "MagpieLM-8B-SFT-v0.1", - "developer": "Magpie-Align", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4721, - "hfopenllm_v2/BBH": 0.4553, - "hfopenllm_v2/MATH Level 5": 0.0755, - "hfopenllm_v2/GPQA": 0.2676, - "hfopenllm_v2/MUSR": 0.3649, - "hfopenllm_v2/MMLU-PRO": 0.299 - } - }, - { - "id": "MagusCorp/grpo_lora_enem_llama3_7b", - "name": "grpo_lora_enem_llama3_7b", - "developer": "MagusCorp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4724, - "hfopenllm_v2/BBH": 0.4801, - "hfopenllm_v2/MATH Level 5": 0.1216, - "hfopenllm_v2/GPQA": 0.3096, - "hfopenllm_v2/MUSR": 0.3971, - "hfopenllm_v2/MMLU-PRO": 0.3574 - } - }, - { - "id": "maldv/Awqward2.5-32B-Instruct", - "name": "Awqward2.5-32B-Instruct", - "developer": "maldv", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8255, - "hfopenllm_v2/BBH": 0.6974, - "hfopenllm_v2/MATH Level 5": 0.6231, - "hfopenllm_v2/GPQA": 0.3406, - "hfopenllm_v2/MUSR": 0.4275, - "hfopenllm_v2/MMLU-PRO": 0.5723 - } - }, - { - "id": "maldv/badger-kappa-llama-3-8b", - "name": "badger-kappa-llama-3-8b", - "developer": "maldv", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4695, - "hfopenllm_v2/BBH": 0.5085, - "hfopenllm_v2/MATH Level 5": 0.0861, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.3765, - "hfopenllm_v2/MMLU-PRO": 0.3695 - } - }, - { - "id": "maldv/badger-lambda-llama-3-8b", - "name": "badger-lambda-llama-3-8b", - "developer": "maldv", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4861, - "hfopenllm_v2/BBH": 0.4963, - "hfopenllm_v2/MATH Level 5": 0.0944, - "hfopenllm_v2/GPQA": 0.2819, - "hfopenllm_v2/MUSR": 0.3754, - "hfopenllm_v2/MMLU-PRO": 0.3767 - } - }, - { - "id": "maldv/badger-mu-llama-3-8b", - "name": "badger-mu-llama-3-8b", - "developer": "maldv", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4919, - "hfopenllm_v2/BBH": 0.5143, - "hfopenllm_v2/MATH Level 5": 0.0559, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3555, - "hfopenllm_v2/MMLU-PRO": 0.3674 - } - }, - { - "id": "maldv/badger-writer-llama-3-8b", - "name": "badger-writer-llama-3-8b", - "developer": "maldv", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5303, - "hfopenllm_v2/BBH": 0.4864, - "hfopenllm_v2/MATH Level 5": 0.0755, - "hfopenllm_v2/GPQA": 0.2894, - "hfopenllm_v2/MUSR": 0.3581, - "hfopenllm_v2/MMLU-PRO": 0.376 - } - }, - { - "id": "maldv/Lytta2.5-32B-Instruct", - "name": "Lytta2.5-32B-Instruct", - "developer": "maldv", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2508, - "hfopenllm_v2/BBH": 0.56, - "hfopenllm_v2/MATH Level 5": 0.3444, - "hfopenllm_v2/GPQA": 0.2668, - "hfopenllm_v2/MUSR": 0.3769, - "hfopenllm_v2/MMLU-PRO": 0.5048 - } - }, - { - "id": "maldv/Qwentile2.5-32B-Instruct", - "name": "Qwentile2.5-32B-Instruct", - "developer": "maldv", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7393, - "hfopenllm_v2/BBH": 0.6963, - "hfopenllm_v2/MATH Level 5": 0.5219, - "hfopenllm_v2/GPQA": 0.3842, - "hfopenllm_v2/MUSR": 0.4682, - "hfopenllm_v2/MMLU-PRO": 0.5879 - } - }, - { - "id": "ManoloPueblo/ContentCuisine_1-7B-slerp", - "name": "ContentCuisine_1-7B-slerp", - "developer": "ManoloPueblo", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3907, - "hfopenllm_v2/BBH": 0.5188, - "hfopenllm_v2/MATH Level 5": 0.0733, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.4672, - "hfopenllm_v2/MMLU-PRO": 0.3054 - } - }, - { - "id": "ManoloPueblo/LLM_MERGE_CC2", - "name": "LLM_MERGE_CC2", - "developer": "ManoloPueblo", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3853, - "hfopenllm_v2/BBH": 0.5209, - "hfopenllm_v2/MATH Level 5": 0.0642, - "hfopenllm_v2/GPQA": 0.3045, - "hfopenllm_v2/MUSR": 0.4593, - "hfopenllm_v2/MMLU-PRO": 0.3032 - } - }, - { - "id": "ManoloPueblo/LLM_MERGE_CC3", - "name": "LLM_MERGE_CC3", - "developer": "ManoloPueblo", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3959, - "hfopenllm_v2/BBH": 0.5246, - "hfopenllm_v2/MATH Level 5": 0.0793, - "hfopenllm_v2/GPQA": 0.3096, - "hfopenllm_v2/MUSR": 0.4672, - "hfopenllm_v2/MMLU-PRO": 0.3156 - } - }, - { - "id": "marcuscedricridia/absolute-o1-7b", - "name": "absolute-o1-7b", - "developer": "marcuscedricridia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7516, - "hfopenllm_v2/BBH": 0.5469, - "hfopenllm_v2/MATH Level 5": 0.5083, - "hfopenllm_v2/GPQA": 0.3196, - "hfopenllm_v2/MUSR": 0.4114, - "hfopenllm_v2/MMLU-PRO": 0.4413 - } - }, - { - "id": "marcuscedricridia/Cheng-1", - "name": "Cheng-1", - "developer": "marcuscedricridia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7789, - "hfopenllm_v2/BBH": 0.5525, - "hfopenllm_v2/MATH Level 5": 0.4894, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.4073, - "hfopenllm_v2/MMLU-PRO": 0.4349 - } - }, - { - "id": "marcuscedricridia/Cheng-2", - "name": "Cheng-2", - "developer": "marcuscedricridia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8337, - "hfopenllm_v2/BBH": 0.6499, - "hfopenllm_v2/MATH Level 5": 0.5438, - "hfopenllm_v2/GPQA": 0.3456, - "hfopenllm_v2/MUSR": 0.4193, - "hfopenllm_v2/MMLU-PRO": 0.5013 - } - }, - { - "id": "marcuscedricridia/Cheng-2-v1.1", - "name": "Cheng-2-v1.1", - "developer": "marcuscedricridia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.827, - "hfopenllm_v2/BBH": 0.651, - "hfopenllm_v2/MATH Level 5": 0.5393, - "hfopenllm_v2/GPQA": 0.3431, - "hfopenllm_v2/MUSR": 0.4167, - "hfopenllm_v2/MMLU-PRO": 0.5076 - } - }, - { - "id": "marcuscedricridia/cursa-o1-7b", - "name": "cursa-o1-7b", - "developer": "marcuscedricridia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7628, - "hfopenllm_v2/BBH": 0.5466, - "hfopenllm_v2/MATH Level 5": 0.4955, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.4301, - "hfopenllm_v2/MMLU-PRO": 0.4392 - } - }, - { - "id": "marcuscedricridia/cursa-o1-7b-2-28-2025", - "name": "cursa-o1-7b-2-28-2025", - "developer": "marcuscedricridia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7467, - "hfopenllm_v2/BBH": 0.5384, - "hfopenllm_v2/MATH Level 5": 0.4811, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.4273, - "hfopenllm_v2/MMLU-PRO": 0.4365 - } - }, - { - "id": "marcuscedricridia/cursa-o1-7b-v1.1", - "name": "cursa-o1-7b-v1.1", - "developer": "marcuscedricridia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7528, - "hfopenllm_v2/BBH": 0.5493, - "hfopenllm_v2/MATH Level 5": 0.4985, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.4259, - "hfopenllm_v2/MMLU-PRO": 0.4392 - } - }, - { - "id": "marcuscedricridia/cursa-o1-7b-v1.2-normalize-false", - "name": "cursa-o1-7b-v1.2-normalize-false", - "developer": "marcuscedricridia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7616, - "hfopenllm_v2/BBH": 0.5492, - "hfopenllm_v2/MATH Level 5": 0.4992, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.4273, - "hfopenllm_v2/MMLU-PRO": 0.4436 - } - }, - { - "id": "marcuscedricridia/cursor-o1-7b", - "name": "cursor-o1-7b", - "developer": "marcuscedricridia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4107, - "hfopenllm_v2/BBH": 0.5007, - "hfopenllm_v2/MATH Level 5": 0.1412, - "hfopenllm_v2/GPQA": 0.281, - "hfopenllm_v2/MUSR": 0.4101, - "hfopenllm_v2/MMLU-PRO": 0.3251 - } - }, - { - "id": "marcuscedricridia/cursorr-o1.2-7b", - "name": "cursorr-o1.2-7b", - "developer": "marcuscedricridia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.166, - "hfopenllm_v2/BBH": 0.3068, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2542, - "hfopenllm_v2/MUSR": 0.3538, - "hfopenllm_v2/MMLU-PRO": 0.108 - } - }, - { - "id": "marcuscedricridia/etr1o-explicit-v1.1", - "name": "etr1o-explicit-v1.1", - "developer": "marcuscedricridia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.288, - "hfopenllm_v2/BBH": 0.3132, - "hfopenllm_v2/MATH Level 5": 0.0045, - "hfopenllm_v2/GPQA": 0.2777, - "hfopenllm_v2/MUSR": 0.4111, - "hfopenllm_v2/MMLU-PRO": 0.1195 - } - }, - { - "id": "marcuscedricridia/etr1o-explicit-v1.2", - "name": "etr1o-explicit-v1.2", - "developer": "marcuscedricridia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1504, - "hfopenllm_v2/BBH": 0.295, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.4031, - "hfopenllm_v2/MMLU-PRO": 0.1126 - } - }, - { - "id": "marcuscedricridia/etr1o-v1.1", - "name": "etr1o-v1.1", - "developer": "marcuscedricridia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1597, - "hfopenllm_v2/BBH": 0.31, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2567, - "hfopenllm_v2/MUSR": 0.4017, - "hfopenllm_v2/MMLU-PRO": 0.1157 - } - }, - { - "id": "marcuscedricridia/etr1o-v1.2", - "name": "etr1o-v1.2", - "developer": "marcuscedricridia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7287, - "hfopenllm_v2/BBH": 0.6349, - "hfopenllm_v2/MATH Level 5": 0.3588, - "hfopenllm_v2/GPQA": 0.3758, - "hfopenllm_v2/MUSR": 0.4714, - "hfopenllm_v2/MMLU-PRO": 0.5316 - } - }, - { - "id": "marcuscedricridia/fan-o1-7b", - "name": "fan-o1-7b", - "developer": "marcuscedricridia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4456, - "hfopenllm_v2/BBH": 0.4849, - "hfopenllm_v2/MATH Level 5": 0.1616, - "hfopenllm_v2/GPQA": 0.2844, - "hfopenllm_v2/MUSR": 0.3834, - "hfopenllm_v2/MMLU-PRO": 0.3274 - } - }, - { - "id": "marcuscedricridia/Hush-Qwen2.5-7B-MST", - "name": "Hush-Qwen2.5-7B-MST", - "developer": "marcuscedricridia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7488, - "hfopenllm_v2/BBH": 0.5458, - "hfopenllm_v2/MATH Level 5": 0.4245, - "hfopenllm_v2/GPQA": 0.3037, - "hfopenllm_v2/MUSR": 0.3914, - "hfopenllm_v2/MMLU-PRO": 0.4163 - } - }, - { - "id": "marcuscedricridia/Hush-Qwen2.5-7B-MST-v1.1", - "name": "Hush-Qwen2.5-7B-MST-v1.1", - "developer": "marcuscedricridia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7445, - "hfopenllm_v2/BBH": 0.5559, - "hfopenllm_v2/MATH Level 5": 0.4653, - "hfopenllm_v2/GPQA": 0.3062, - "hfopenllm_v2/MUSR": 0.4073, - "hfopenllm_v2/MMLU-PRO": 0.4299 - } - }, - { - "id": "marcuscedricridia/Hush-Qwen2.5-7B-MST-v1.3", - "name": "Hush-Qwen2.5-7B-MST-v1.3", - "developer": "marcuscedricridia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7043, - "hfopenllm_v2/BBH": 0.5516, - "hfopenllm_v2/MATH Level 5": 0.4758, - "hfopenllm_v2/GPQA": 0.3146, - "hfopenllm_v2/MUSR": 0.4311, - "hfopenllm_v2/MMLU-PRO": 0.444 - } - }, - { - "id": "marcuscedricridia/Hush-Qwen2.5-7B-Preview", - "name": "Hush-Qwen2.5-7B-Preview", - "developer": "marcuscedricridia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7962, - "hfopenllm_v2/BBH": 0.5431, - "hfopenllm_v2/MATH Level 5": 0.3754, - "hfopenllm_v2/GPQA": 0.3112, - "hfopenllm_v2/MUSR": 0.4298, - "hfopenllm_v2/MMLU-PRO": 0.4364 - } - }, - { - "id": "marcuscedricridia/Hush-Qwen2.5-7B-RP-v1.4-1M", - "name": "Hush-Qwen2.5-7B-RP-v1.4-1M", - "developer": "marcuscedricridia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7728, - "hfopenllm_v2/BBH": 0.5295, - "hfopenllm_v2/MATH Level 5": 0.3369, - "hfopenllm_v2/GPQA": 0.2987, - "hfopenllm_v2/MUSR": 0.4433, - "hfopenllm_v2/MMLU-PRO": 0.4135 - } - }, - { - "id": "marcuscedricridia/Hush-Qwen2.5-7B-v1.1", - "name": "Hush-Qwen2.5-7B-v1.1", - "developer": "marcuscedricridia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7889, - "hfopenllm_v2/BBH": 0.5384, - "hfopenllm_v2/MATH Level 5": 0.4381, - "hfopenllm_v2/GPQA": 0.3163, - "hfopenllm_v2/MUSR": 0.4179, - "hfopenllm_v2/MMLU-PRO": 0.4227 - } - }, - { - "id": "marcuscedricridia/Hush-Qwen2.5-7B-v1.2", - "name": "Hush-Qwen2.5-7B-v1.2", - "developer": "marcuscedricridia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7865, - "hfopenllm_v2/BBH": 0.5403, - "hfopenllm_v2/MATH Level 5": 0.4403, - "hfopenllm_v2/GPQA": 0.3146, - "hfopenllm_v2/MUSR": 0.4219, - "hfopenllm_v2/MMLU-PRO": 0.4197 - } - }, - { - "id": "marcuscedricridia/Hush-Qwen2.5-7B-v1.3", - "name": "Hush-Qwen2.5-7B-v1.3", - "developer": "marcuscedricridia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7856, - "hfopenllm_v2/BBH": 0.5327, - "hfopenllm_v2/MATH Level 5": 0.3323, - "hfopenllm_v2/GPQA": 0.3121, - "hfopenllm_v2/MUSR": 0.4246, - "hfopenllm_v2/MMLU-PRO": 0.4345 - } - }, - { - "id": "marcuscedricridia/Hush-Qwen2.5-7B-v1.4", - "name": "Hush-Qwen2.5-7B-v1.4", - "developer": "marcuscedricridia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7835, - "hfopenllm_v2/BBH": 0.5423, - "hfopenllm_v2/MATH Level 5": 0.426, - "hfopenllm_v2/GPQA": 0.3112, - "hfopenllm_v2/MUSR": 0.4232, - "hfopenllm_v2/MMLU-PRO": 0.4195 - } - }, - { - "id": "marcuscedricridia/olmner-7b", - "name": "olmner-7b", - "developer": "marcuscedricridia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7254, - "hfopenllm_v2/BBH": 0.5472, - "hfopenllm_v2/MATH Level 5": 0.463, - "hfopenllm_v2/GPQA": 0.3079, - "hfopenllm_v2/MUSR": 0.438, - "hfopenllm_v2/MMLU-PRO": 0.4309 - } - }, - { - "id": "marcuscedricridia/olmner-della-7b", - "name": "olmner-della-7b", - "developer": "marcuscedricridia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7637, - "hfopenllm_v2/BBH": 0.5491, - "hfopenllm_v2/MATH Level 5": 0.4962, - "hfopenllm_v2/GPQA": 0.3012, - "hfopenllm_v2/MUSR": 0.4208, - "hfopenllm_v2/MMLU-PRO": 0.4386 - } - }, - { - "id": "marcuscedricridia/olmner-o1-7b", - "name": "olmner-o1-7b", - "developer": "marcuscedricridia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7528, - "hfopenllm_v2/BBH": 0.5481, - "hfopenllm_v2/MATH Level 5": 0.4924, - "hfopenllm_v2/GPQA": 0.3012, - "hfopenllm_v2/MUSR": 0.4299, - "hfopenllm_v2/MMLU-PRO": 0.4386 - } - }, - { - "id": "marcuscedricridia/olmner-sbr-7b", - "name": "olmner-sbr-7b", - "developer": "marcuscedricridia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.76, - "hfopenllm_v2/BBH": 0.5462, - "hfopenllm_v2/MATH Level 5": 0.4947, - "hfopenllm_v2/GPQA": 0.3087, - "hfopenllm_v2/MUSR": 0.4154, - "hfopenllm_v2/MMLU-PRO": 0.4412 - } - }, - { - "id": "marcuscedricridia/post-cursa-o1", - "name": "post-cursa-o1", - "developer": "marcuscedricridia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7628, - "hfopenllm_v2/BBH": 0.548, - "hfopenllm_v2/MATH Level 5": 0.4872, - "hfopenllm_v2/GPQA": 0.3096, - "hfopenllm_v2/MUSR": 0.4351, - "hfopenllm_v2/MMLU-PRO": 0.4361 - } - }, - { - "id": "marcuscedricridia/pre-cursa-o1", - "name": "pre-cursa-o1", - "developer": "marcuscedricridia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7409, - "hfopenllm_v2/BBH": 0.5462, - "hfopenllm_v2/MATH Level 5": 0.5038, - "hfopenllm_v2/GPQA": 0.3096, - "hfopenllm_v2/MUSR": 0.426, - "hfopenllm_v2/MMLU-PRO": 0.4424 - } - }, - { - "id": "marcuscedricridia/pre-cursa-o1-v1.2", - "name": "pre-cursa-o1-v1.2", - "developer": "marcuscedricridia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7549, - "hfopenllm_v2/BBH": 0.5487, - "hfopenllm_v2/MATH Level 5": 0.5068, - "hfopenllm_v2/GPQA": 0.3129, - "hfopenllm_v2/MUSR": 0.4272, - "hfopenllm_v2/MMLU-PRO": 0.4402 - } - }, - { - "id": "marcuscedricridia/pre-cursa-o1-v1.3", - "name": "pre-cursa-o1-v1.3", - "developer": "marcuscedricridia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7507, - "hfopenllm_v2/BBH": 0.5455, - "hfopenllm_v2/MATH Level 5": 0.5076, - "hfopenllm_v2/GPQA": 0.3129, - "hfopenllm_v2/MUSR": 0.4271, - "hfopenllm_v2/MMLU-PRO": 0.442 - } - }, - { - "id": "marcuscedricridia/pre-cursa-o1-v1.4", - "name": "pre-cursa-o1-v1.4", - "developer": "marcuscedricridia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7488, - "hfopenllm_v2/BBH": 0.5493, - "hfopenllm_v2/MATH Level 5": 0.4834, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.4285, - "hfopenllm_v2/MMLU-PRO": 0.4436 - } - }, - { - "id": "marcuscedricridia/pre-cursa-o1-v1.6", - "name": "pre-cursa-o1-v1.6", - "developer": "marcuscedricridia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7528, - "hfopenllm_v2/BBH": 0.5473, - "hfopenllm_v2/MATH Level 5": 0.5, - "hfopenllm_v2/GPQA": 0.3205, - "hfopenllm_v2/MUSR": 0.4234, - "hfopenllm_v2/MMLU-PRO": 0.4413 - } - }, - { - "id": "marcuscedricridia/Qwen2.5-7B-Preview", - "name": "Qwen2.5-7B-Preview", - "developer": "marcuscedricridia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7679, - "hfopenllm_v2/BBH": 0.536, - "hfopenllm_v2/MATH Level 5": 0.3444, - "hfopenllm_v2/GPQA": 0.3238, - "hfopenllm_v2/MUSR": 0.414, - "hfopenllm_v2/MMLU-PRO": 0.4258 - } - }, - { - "id": "marcuscedricridia/r1o-et", - "name": "r1o-et", - "developer": "marcuscedricridia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3597, - "hfopenllm_v2/BBH": 0.4209, - "hfopenllm_v2/MATH Level 5": 0.0793, - "hfopenllm_v2/GPQA": 0.2727, - "hfopenllm_v2/MUSR": 0.3579, - "hfopenllm_v2/MMLU-PRO": 0.258 - } - }, - { - "id": "marcuscedricridia/sbr-o1-7b", - "name": "sbr-o1-7b", - "developer": "marcuscedricridia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7455, - "hfopenllm_v2/BBH": 0.5479, - "hfopenllm_v2/MATH Level 5": 0.4985, - "hfopenllm_v2/GPQA": 0.3104, - "hfopenllm_v2/MUSR": 0.4404, - "hfopenllm_v2/MMLU-PRO": 0.4355 - } - }, - { - "id": "marcuscedricridia/stray-r1o-et", - "name": "stray-r1o-et", - "developer": "marcuscedricridia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1562, - "hfopenllm_v2/BBH": 0.2967, - "hfopenllm_v2/MATH Level 5": 0.0045, - "hfopenllm_v2/GPQA": 0.2617, - "hfopenllm_v2/MUSR": 0.4086, - "hfopenllm_v2/MMLU-PRO": 0.1094 - } - }, - { - "id": "marcuscedricridia/Yell-Qwen2.5-7B-Preview", - "name": "Yell-Qwen2.5-7B-Preview", - "developer": "marcuscedricridia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5839, - "hfopenllm_v2/BBH": 0.5371, - "hfopenllm_v2/MATH Level 5": 0.1926, - "hfopenllm_v2/GPQA": 0.281, - "hfopenllm_v2/MUSR": 0.4046, - "hfopenllm_v2/MMLU-PRO": 0.3798 - } - }, - { - "id": "marcuscedricridia/Yell-Qwen2.5-7B-Preview-v1.1", - "name": "Yell-Qwen2.5-7B-Preview-v1.1", - "developer": "marcuscedricridia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5757, - "hfopenllm_v2/BBH": 0.5348, - "hfopenllm_v2/MATH Level 5": 0.1896, - "hfopenllm_v2/GPQA": 0.2861, - "hfopenllm_v2/MUSR": 0.4059, - "hfopenllm_v2/MMLU-PRO": 0.3831 - } - }, - { - "id": "marin-community/marin-8b-instruct", - "name": "Marin 8B Instruct", - "developer": "marin-community", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.325, - "helm_capabilities/MMLU-Pro": 0.188, - "helm_capabilities/GPQA": 0.168, - "helm_capabilities/IFEval": 0.632, - "helm_capabilities/WildBench": 0.477, - "helm_capabilities/Omni-MATH": 0.16 - } - }, - { - "id": "MarinaraSpaghetti/Nemomix-v4.0-12B", - "name": "Nemomix-v4.0-12B", - "developer": "MarinaraSpaghetti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5575, - "hfopenllm_v2/BBH": 0.5275, - "hfopenllm_v2/MATH Level 5": 0.108, - "hfopenllm_v2/GPQA": 0.2919, - "hfopenllm_v2/MUSR": 0.4244, - "hfopenllm_v2/MMLU-PRO": 0.3613 - } - }, - { - "id": "MarinaraSpaghetti/NemoReRemix-12B", - "name": "NemoReRemix-12B", - "developer": "MarinaraSpaghetti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3343, - "hfopenllm_v2/BBH": 0.5537, - "hfopenllm_v2/MATH Level 5": 0.0906, - "hfopenllm_v2/GPQA": 0.318, - "hfopenllm_v2/MUSR": 0.4501, - "hfopenllm_v2/MMLU-PRO": 0.3598 - } - }, - { - "id": "Marsouuu/general3B-ECE-PRYMMAL-Martial", - "name": "general3B-ECE-PRYMMAL-Martial", - "developer": "Marsouuu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2722, - "hfopenllm_v2/BBH": 0.5394, - "hfopenllm_v2/MATH Level 5": 0.1548, - "hfopenllm_v2/GPQA": 0.3196, - "hfopenllm_v2/MUSR": 0.4701, - "hfopenllm_v2/MMLU-PRO": 0.3876 - } - }, - { - "id": "Marsouuu/general3Bv2-ECE-PRYMMAL-Martial", - "name": "general3Bv2-ECE-PRYMMAL-Martial", - "developer": "Marsouuu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5693, - "hfopenllm_v2/BBH": 0.5637, - "hfopenllm_v2/MATH Level 5": 0.3671, - "hfopenllm_v2/GPQA": 0.3104, - "hfopenllm_v2/MUSR": 0.4396, - "hfopenllm_v2/MMLU-PRO": 0.4498 - } - }, - { - "id": "Marsouuu/lareneg1_78B-ECE-PRYMMAL-Martial", - "name": "lareneg1_78B-ECE-PRYMMAL-Martial", - "developer": "Marsouuu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2795, - "hfopenllm_v2/BBH": 0.423, - "hfopenllm_v2/MATH Level 5": 0.114, - "hfopenllm_v2/GPQA": 0.2819, - "hfopenllm_v2/MUSR": 0.3867, - "hfopenllm_v2/MMLU-PRO": 0.2922 - } - }, - { - "id": "Marsouuu/lareneg3B-ECE-PRYMMAL-Martial", - "name": "lareneg3B-ECE-PRYMMAL-Martial", - "developer": "Marsouuu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3303, - "hfopenllm_v2/BBH": 0.5453, - "hfopenllm_v2/MATH Level 5": 0.1518, - "hfopenllm_v2/GPQA": 0.3247, - "hfopenllm_v2/MUSR": 0.4725, - "hfopenllm_v2/MMLU-PRO": 0.3767 - } - }, - { - "id": "Marsouuu/lareneg3Bv2-ECE-PRYMMAL-Martial", - "name": "lareneg3Bv2-ECE-PRYMMAL-Martial", - "developer": "Marsouuu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5753, - "hfopenllm_v2/BBH": 0.5623, - "hfopenllm_v2/MATH Level 5": 0.3656, - "hfopenllm_v2/GPQA": 0.3196, - "hfopenllm_v2/MUSR": 0.4369, - "hfopenllm_v2/MMLU-PRO": 0.4511 - } - }, - { - "id": "Marsouuu/MiniMathExpert-2_61B-ECE-PRYMMAL-Martial", - "name": "MiniMathExpert-2_61B-ECE-PRYMMAL-Martial", - "developer": "Marsouuu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2548, - "hfopenllm_v2/BBH": 0.3953, - "hfopenllm_v2/MATH Level 5": 0.074, - "hfopenllm_v2/GPQA": 0.2752, - "hfopenllm_v2/MUSR": 0.4083, - "hfopenllm_v2/MMLU-PRO": 0.2274 - } - }, - { - "id": "Marsouuu/MiniQwenMathExpert-ECE-PRYMMAL-Martial", - "name": "MiniQwenMathExpert-ECE-PRYMMAL-Martial", - "developer": "Marsouuu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2795, - "hfopenllm_v2/BBH": 0.423, - "hfopenllm_v2/MATH Level 5": 0.114, - "hfopenllm_v2/GPQA": 0.2819, - "hfopenllm_v2/MUSR": 0.3867, - "hfopenllm_v2/MMLU-PRO": 0.2922 - } - }, - { - "id": "Marsouuu/MistralBase-4x7B-MoE-ECE-PRYMMAL-Martial", - "name": "MistralBase-4x7B-MoE-ECE-PRYMMAL-Martial", - "developer": "Marsouuu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1697, - "hfopenllm_v2/BBH": 0.3464, - "hfopenllm_v2/MATH Level 5": 0.0144, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3991, - "hfopenllm_v2/MMLU-PRO": 0.1379 - } - }, - { - "id": "matouLeLoup/ECE-PRYMMAL-0.5B-FT-EnhancedMUSREnsembleV3", - "name": "ECE-PRYMMAL-0.5B-FT-EnhancedMUSREnsembleV3", - "developer": "matouLeLoup", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1873, - "hfopenllm_v2/BBH": 0.3239, - "hfopenllm_v2/MATH Level 5": 0.0264, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.3752, - "hfopenllm_v2/MMLU-PRO": 0.172 - } - }, - { - "id": "matouLeLoup/ECE-PRYMMAL-0.5B-FT-MUSR-ENSEMBLE-V2Mathis", - "name": "ECE-PRYMMAL-0.5B-FT-MUSR-ENSEMBLE-V2Mathis", - "developer": "matouLeLoup", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1873, - "hfopenllm_v2/BBH": 0.3239, - "hfopenllm_v2/MATH Level 5": 0.0264, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.3752, - "hfopenllm_v2/MMLU-PRO": 0.172 - } - }, - { - "id": "matouLeLoup/ECE-PRYMMAL-0.5B-FT-V4-MUSR-ENSEMBLE-Mathis", - "name": "ECE-PRYMMAL-0.5B-FT-V4-MUSR-ENSEMBLE-Mathis", - "developer": "matouLeLoup", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1873, - "hfopenllm_v2/BBH": 0.3239, - "hfopenllm_v2/MATH Level 5": 0.0264, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.3752, - "hfopenllm_v2/MMLU-PRO": 0.172 - } - }, - { - "id": "matouLeLoup/ECE-PRYMMAL-0.5B-FT-V4-MUSR-Mathis", - "name": "ECE-PRYMMAL-0.5B-FT-V4-MUSR-Mathis", - "developer": "matouLeLoup", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1882, - "hfopenllm_v2/BBH": 0.3233, - "hfopenllm_v2/MATH Level 5": 0.0272, - "hfopenllm_v2/GPQA": 0.2634, - "hfopenllm_v2/MUSR": 0.3685, - "hfopenllm_v2/MMLU-PRO": 0.172 - } - }, - { - "id": "matouLeLoup/ECE-PRYMMAL-0.5B-FT-V5-MUSR-Mathis", - "name": "ECE-PRYMMAL-0.5B-FT-V5-MUSR-Mathis", - "developer": "matouLeLoup", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1652, - "hfopenllm_v2/BBH": 0.3024, - "hfopenllm_v2/MATH Level 5": 0.0189, - "hfopenllm_v2/GPQA": 0.2567, - "hfopenllm_v2/MUSR": 0.4273, - "hfopenllm_v2/MMLU-PRO": 0.1116 - } - }, - { - "id": "mattshumer/ref_70_e3", - "name": "ref_70_e3", - "developer": "mattshumer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6294, - "hfopenllm_v2/BBH": 0.6501, - "hfopenllm_v2/MATH Level 5": 0.2795, - "hfopenllm_v2/GPQA": 0.3356, - "hfopenllm_v2/MUSR": 0.4328, - "hfopenllm_v2/MMLU-PRO": 0.5303 - } - }, - { - "id": "mattshumer/Reflection-70B", - "name": "mattshumer/Reflection-70B", - "developer": "mattshumer", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.8422, - "reward-bench/Chat": 0.9749, - "reward-bench/Chat Hard": 0.7061, - "reward-bench/Safety": 0.8318, - "reward-bench/Reasoning": 0.8562 - } - }, - { - "id": "mattshumer/Reflection-Llama-3.1-70B", - "name": "Reflection-Llama-3.1-70B", - "developer": "mattshumer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0045, - "hfopenllm_v2/BBH": 0.645, - "hfopenllm_v2/MATH Level 5": 0.2145, - "hfopenllm_v2/GPQA": 0.3633, - "hfopenllm_v2/MUSR": 0.4577, - "hfopenllm_v2/MMLU-PRO": 0.4955 - } - }, - { - "id": "maywell/Qwen2-7B-Multilingual-RP", - "name": "Qwen2-7B-Multilingual-RP", - "developer": "maywell", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4347, - "hfopenllm_v2/BBH": 0.5062, - "hfopenllm_v2/MATH Level 5": 0.2243, - "hfopenllm_v2/GPQA": 0.297, - "hfopenllm_v2/MUSR": 0.3696, - "hfopenllm_v2/MMLU-PRO": 0.3859 - } - }, - { - "id": "MaziyarPanahi/calme-2.1-llama3.1-70b", - "name": "calme-2.1-llama3.1-70b", - "developer": "MaziyarPanahi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8434, - "hfopenllm_v2/BBH": 0.6448, - "hfopenllm_v2/MATH Level 5": 0.4101, - "hfopenllm_v2/GPQA": 0.328, - "hfopenllm_v2/MUSR": 0.438, - "hfopenllm_v2/MMLU-PRO": 0.5283 - } - }, - { - "id": "MaziyarPanahi/calme-2.1-phi3-4b", - "name": "calme-2.1-phi3-4b", - "developer": "MaziyarPanahi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5525, - "hfopenllm_v2/BBH": 0.5595, - "hfopenllm_v2/MATH Level 5": 0.1314, - "hfopenllm_v2/GPQA": 0.3297, - "hfopenllm_v2/MUSR": 0.4015, - "hfopenllm_v2/MMLU-PRO": 0.3746 - } - }, - { - "id": "MaziyarPanahi/calme-2.1-phi3.5-4b", - "name": "calme-2.1-phi3.5-4b", - "developer": "MaziyarPanahi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5659, - "hfopenllm_v2/BBH": 0.5484, - "hfopenllm_v2/MATH Level 5": 0.2039, - "hfopenllm_v2/GPQA": 0.344, - "hfopenllm_v2/MUSR": 0.3995, - "hfopenllm_v2/MMLU-PRO": 0.3935 - } - }, - { - "id": "MaziyarPanahi/calme-2.1-qwen2-72b", - "name": "calme-2.1-qwen2-72b", - "developer": "MaziyarPanahi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8163, - "hfopenllm_v2/BBH": 0.6966, - "hfopenllm_v2/MATH Level 5": 0.4079, - "hfopenllm_v2/GPQA": 0.3809, - "hfopenllm_v2/MUSR": 0.4732, - "hfopenllm_v2/MMLU-PRO": 0.5415 - } - }, - { - "id": "MaziyarPanahi/calme-2.1-qwen2-7b", - "name": "calme-2.1-qwen2-7b", - "developer": "MaziyarPanahi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3816, - "hfopenllm_v2/BBH": 0.5046, - "hfopenllm_v2/MATH Level 5": 0.2311, - "hfopenllm_v2/GPQA": 0.2894, - "hfopenllm_v2/MUSR": 0.4437, - "hfopenllm_v2/MMLU-PRO": 0.3693 - } - }, - { - "id": "MaziyarPanahi/calme-2.1-qwen2.5-72b", - "name": "calme-2.1-qwen2.5-72b", - "developer": "MaziyarPanahi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8662, - "hfopenllm_v2/BBH": 0.7262, - "hfopenllm_v2/MATH Level 5": 0.5914, - "hfopenllm_v2/GPQA": 0.3633, - "hfopenllm_v2/MUSR": 0.4298, - "hfopenllm_v2/MMLU-PRO": 0.5619 - } - }, - { - "id": "MaziyarPanahi/calme-2.1-rys-78b", - "name": "calme-2.1-rys-78b", - "developer": "MaziyarPanahi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8136, - "hfopenllm_v2/BBH": 0.7098, - "hfopenllm_v2/MATH Level 5": 0.3943, - "hfopenllm_v2/GPQA": 0.3943, - "hfopenllm_v2/MUSR": 0.4693, - "hfopenllm_v2/MMLU-PRO": 0.5444 - } - }, - { - "id": "MaziyarPanahi/calme-2.2-llama3-70b", - "name": "calme-2.2-llama3-70b", - "developer": "MaziyarPanahi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8208, - "hfopenllm_v2/BBH": 0.6435, - "hfopenllm_v2/MATH Level 5": 0.2394, - "hfopenllm_v2/GPQA": 0.3414, - "hfopenllm_v2/MUSR": 0.4446, - "hfopenllm_v2/MMLU-PRO": 0.5207 - } - }, - { - "id": "MaziyarPanahi/calme-2.2-llama3.1-70b", - "name": "calme-2.2-llama3.1-70b", - "developer": "MaziyarPanahi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8593, - "hfopenllm_v2/BBH": 0.6793, - "hfopenllm_v2/MATH Level 5": 0.4366, - "hfopenllm_v2/GPQA": 0.3247, - "hfopenllm_v2/MUSR": 0.4542, - "hfopenllm_v2/MMLU-PRO": 0.5415 - } - }, - { - "id": "MaziyarPanahi/calme-2.2-phi3-4b", - "name": "calme-2.2-phi3-4b", - "developer": "MaziyarPanahi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5069, - "hfopenllm_v2/BBH": 0.553, - "hfopenllm_v2/MATH Level 5": 0.145, - "hfopenllm_v2/GPQA": 0.3213, - "hfopenllm_v2/MUSR": 0.3976, - "hfopenllm_v2/MMLU-PRO": 0.3814 - } - }, - { - "id": "MaziyarPanahi/calme-2.2-qwen2-72b", - "name": "calme-2.2-qwen2-72b", - "developer": "MaziyarPanahi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8008, - "hfopenllm_v2/BBH": 0.694, - "hfopenllm_v2/MATH Level 5": 0.4532, - "hfopenllm_v2/GPQA": 0.3742, - "hfopenllm_v2/MUSR": 0.4508, - "hfopenllm_v2/MMLU-PRO": 0.5435 - } - }, - { - "id": "MaziyarPanahi/calme-2.2-qwen2-7b", - "name": "calme-2.2-qwen2-7b", - "developer": "MaziyarPanahi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3597, - "hfopenllm_v2/BBH": 0.5215, - "hfopenllm_v2/MATH Level 5": 0.2145, - "hfopenllm_v2/GPQA": 0.2911, - "hfopenllm_v2/MUSR": 0.4358, - "hfopenllm_v2/MMLU-PRO": 0.3899 - } - }, - { - "id": "MaziyarPanahi/calme-2.2-qwen2.5-72b", - "name": "calme-2.2-qwen2.5-72b", - "developer": "MaziyarPanahi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8477, - "hfopenllm_v2/BBH": 0.7276, - "hfopenllm_v2/MATH Level 5": 0.5891, - "hfopenllm_v2/GPQA": 0.3591, - "hfopenllm_v2/MUSR": 0.4207, - "hfopenllm_v2/MMLU-PRO": 0.5618 - } - }, - { - "id": "MaziyarPanahi/calme-2.2-rys-78b", - "name": "calme-2.2-rys-78b", - "developer": "MaziyarPanahi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7986, - "hfopenllm_v2/BBH": 0.7081, - "hfopenllm_v2/MATH Level 5": 0.4071, - "hfopenllm_v2/GPQA": 0.4069, - "hfopenllm_v2/MUSR": 0.4536, - "hfopenllm_v2/MMLU-PRO": 0.5386 - } - }, - { - "id": "MaziyarPanahi/calme-2.3-llama3-70b", - "name": "calme-2.3-llama3-70b", - "developer": "MaziyarPanahi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.801, - "hfopenllm_v2/BBH": 0.6399, - "hfopenllm_v2/MATH Level 5": 0.2326, - "hfopenllm_v2/GPQA": 0.3381, - "hfopenllm_v2/MUSR": 0.4261, - "hfopenllm_v2/MMLU-PRO": 0.5204 - } - }, - { - "id": "MaziyarPanahi/calme-2.3-llama3.1-70b", - "name": "calme-2.3-llama3.1-70b", - "developer": "MaziyarPanahi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8605, - "hfopenllm_v2/BBH": 0.6872, - "hfopenllm_v2/MATH Level 5": 0.3927, - "hfopenllm_v2/GPQA": 0.344, - "hfopenllm_v2/MUSR": 0.4568, - "hfopenllm_v2/MMLU-PRO": 0.5363 - } - }, - { - "id": "MaziyarPanahi/calme-2.3-phi3-4b", - "name": "calme-2.3-phi3-4b", - "developer": "MaziyarPanahi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4926, - "hfopenllm_v2/BBH": 0.5538, - "hfopenllm_v2/MATH Level 5": 0.1473, - "hfopenllm_v2/GPQA": 0.318, - "hfopenllm_v2/MUSR": 0.3988, - "hfopenllm_v2/MMLU-PRO": 0.3828 - } - }, - { - "id": "MaziyarPanahi/calme-2.3-qwen2-72b", - "name": "calme-2.3-qwen2-72b", - "developer": "MaziyarPanahi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.385, - "hfopenllm_v2/BBH": 0.6576, - "hfopenllm_v2/MATH Level 5": 0.3172, - "hfopenllm_v2/GPQA": 0.3716, - "hfopenllm_v2/MUSR": 0.4112, - "hfopenllm_v2/MMLU-PRO": 0.5419 - } - }, - { - "id": "MaziyarPanahi/calme-2.3-qwen2-7b", - "name": "calme-2.3-qwen2-7b", - "developer": "MaziyarPanahi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3825, - "hfopenllm_v2/BBH": 0.5064, - "hfopenllm_v2/MATH Level 5": 0.2069, - "hfopenllm_v2/GPQA": 0.297, - "hfopenllm_v2/MUSR": 0.4422, - "hfopenllm_v2/MMLU-PRO": 0.3611 - } - }, - { - "id": "MaziyarPanahi/calme-2.3-rys-78b", - "name": "calme-2.3-rys-78b", - "developer": "MaziyarPanahi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8066, - "hfopenllm_v2/BBH": 0.7108, - "hfopenllm_v2/MATH Level 5": 0.398, - "hfopenllm_v2/GPQA": 0.4044, - "hfopenllm_v2/MUSR": 0.4549, - "hfopenllm_v2/MMLU-PRO": 0.5475 - } - }, - { - "id": "MaziyarPanahi/calme-2.4-llama3-70b", - "name": "calme-2.4-llama3-70b", - "developer": "MaziyarPanahi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5027, - "hfopenllm_v2/BBH": 0.6418, - "hfopenllm_v2/MATH Level 5": 0.2447, - "hfopenllm_v2/GPQA": 0.3398, - "hfopenllm_v2/MUSR": 0.4288, - "hfopenllm_v2/MMLU-PRO": 0.5204 - } - }, - { - "id": "MaziyarPanahi/calme-2.4-qwen2-7b", - "name": "calme-2.4-qwen2-7b", - "developer": "MaziyarPanahi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.33, - "hfopenllm_v2/BBH": 0.5101, - "hfopenllm_v2/MATH Level 5": 0.2032, - "hfopenllm_v2/GPQA": 0.2836, - "hfopenllm_v2/MUSR": 0.4453, - "hfopenllm_v2/MMLU-PRO": 0.3977 - } - }, - { - "id": "MaziyarPanahi/calme-2.4-rys-78b", - "name": "calme-2.4-rys-78b", - "developer": "MaziyarPanahi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8011, - "hfopenllm_v2/BBH": 0.728, - "hfopenllm_v2/MATH Level 5": 0.4071, - "hfopenllm_v2/GPQA": 0.4027, - "hfopenllm_v2/MUSR": 0.5771, - "hfopenllm_v2/MMLU-PRO": 0.7002 - } - }, - { - "id": "MaziyarPanahi/calme-2.5-qwen2-7b", - "name": "calme-2.5-qwen2-7b", - "developer": "MaziyarPanahi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3145, - "hfopenllm_v2/BBH": 0.4887, - "hfopenllm_v2/MATH Level 5": 0.2258, - "hfopenllm_v2/GPQA": 0.3104, - "hfopenllm_v2/MUSR": 0.4565, - "hfopenllm_v2/MMLU-PRO": 0.3682 - } - }, - { - "id": "MaziyarPanahi/calme-2.6-qwen2-7b", - "name": "calme-2.6-qwen2-7b", - "developer": "MaziyarPanahi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3443, - "hfopenllm_v2/BBH": 0.493, - "hfopenllm_v2/MATH Level 5": 0.1216, - "hfopenllm_v2/GPQA": 0.2844, - "hfopenllm_v2/MUSR": 0.4586, - "hfopenllm_v2/MMLU-PRO": 0.3732 - } - }, - { - "id": "MaziyarPanahi/calme-2.7-qwen2-7b", - "name": "calme-2.7-qwen2-7b", - "developer": "MaziyarPanahi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3592, - "hfopenllm_v2/BBH": 0.4883, - "hfopenllm_v2/MATH Level 5": 0.1382, - "hfopenllm_v2/GPQA": 0.2911, - "hfopenllm_v2/MUSR": 0.4824, - "hfopenllm_v2/MMLU-PRO": 0.3705 - } - }, - { - "id": "MaziyarPanahi/calme-3.1-baguette-3b", - "name": "calme-3.1-baguette-3b", - "developer": "MaziyarPanahi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6234, - "hfopenllm_v2/BBH": 0.4683, - "hfopenllm_v2/MATH Level 5": 0.256, - "hfopenllm_v2/GPQA": 0.2861, - "hfopenllm_v2/MUSR": 0.4008, - "hfopenllm_v2/MMLU-PRO": 0.3399 - } - }, - { - "id": "MaziyarPanahi/calme-3.1-instruct-3b", - "name": "calme-3.1-instruct-3b", - "developer": "MaziyarPanahi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4336, - "hfopenllm_v2/BBH": 0.4813, - "hfopenllm_v2/MATH Level 5": 0.1775, - "hfopenllm_v2/GPQA": 0.2861, - "hfopenllm_v2/MUSR": 0.3952, - "hfopenllm_v2/MMLU-PRO": 0.3557 - } - }, - { - "id": "MaziyarPanahi/calme-3.1-instruct-78b", - "name": "calme-3.1-instruct-78b", - "developer": "MaziyarPanahi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8136, - "hfopenllm_v2/BBH": 0.7305, - "hfopenllm_v2/MATH Level 5": 0.3927, - "hfopenllm_v2/GPQA": 0.396, - "hfopenllm_v2/MUSR": 0.5891, - "hfopenllm_v2/MMLU-PRO": 0.7185 - } - }, - { - "id": "MaziyarPanahi/calme-3.1-llamaloi-3b", - "name": "calme-3.1-llamaloi-3b", - "developer": "MaziyarPanahi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7375, - "hfopenllm_v2/BBH": 0.4587, - "hfopenllm_v2/MATH Level 5": 0.173, - "hfopenllm_v2/GPQA": 0.281, - "hfopenllm_v2/MUSR": 0.3515, - "hfopenllm_v2/MMLU-PRO": 0.3205 - } - }, - { - "id": "MaziyarPanahi/calme-3.2-baguette-3b", - "name": "calme-3.2-baguette-3b", - "developer": "MaziyarPanahi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6338, - "hfopenllm_v2/BBH": 0.4709, - "hfopenllm_v2/MATH Level 5": 0.2825, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.4021, - "hfopenllm_v2/MMLU-PRO": 0.3338 - } - }, - { - "id": "MaziyarPanahi/calme-3.2-instruct-3b", - "name": "calme-3.2-instruct-3b", - "developer": "MaziyarPanahi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5533, - "hfopenllm_v2/BBH": 0.4866, - "hfopenllm_v2/MATH Level 5": 0.2168, - "hfopenllm_v2/GPQA": 0.2836, - "hfopenllm_v2/MUSR": 0.4047, - "hfopenllm_v2/MMLU-PRO": 0.3653 - } - }, - { - "id": "MaziyarPanahi/calme-3.2-instruct-78b", - "name": "calme-3.2-instruct-78b", - "developer": "MaziyarPanahi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8063, - "hfopenllm_v2/BBH": 0.7319, - "hfopenllm_v2/MATH Level 5": 0.4033, - "hfopenllm_v2/GPQA": 0.4027, - "hfopenllm_v2/MUSR": 0.6024, - "hfopenllm_v2/MMLU-PRO": 0.7303 - } - }, - { - "id": "MaziyarPanahi/calme-3.3-baguette-3b", - "name": "calme-3.3-baguette-3b", - "developer": "MaziyarPanahi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.636, - "hfopenllm_v2/BBH": 0.4678, - "hfopenllm_v2/MATH Level 5": 0.3807, - "hfopenllm_v2/GPQA": 0.2802, - "hfopenllm_v2/MUSR": 0.3928, - "hfopenllm_v2/MMLU-PRO": 0.3342 - } - }, - { - "id": "MaziyarPanahi/calme-3.3-instruct-3b", - "name": "calme-3.3-instruct-3b", - "developer": "MaziyarPanahi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6423, - "hfopenllm_v2/BBH": 0.4693, - "hfopenllm_v2/MATH Level 5": 0.3739, - "hfopenllm_v2/GPQA": 0.2827, - "hfopenllm_v2/MUSR": 0.4074, - "hfopenllm_v2/MMLU-PRO": 0.3305 - } - }, - { - "id": "MaziyarPanahi/Calme-4x7B-MoE-v0.1", - "name": "Calme-4x7B-MoE-v0.1", - "developer": "MaziyarPanahi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4315, - "hfopenllm_v2/BBH": 0.5103, - "hfopenllm_v2/MATH Level 5": 0.0801, - "hfopenllm_v2/GPQA": 0.2819, - "hfopenllm_v2/MUSR": 0.4199, - "hfopenllm_v2/MMLU-PRO": 0.3057 - } - }, - { - "id": "MaziyarPanahi/Calme-4x7B-MoE-v0.2", - "name": "Calme-4x7B-MoE-v0.2", - "developer": "MaziyarPanahi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4294, - "hfopenllm_v2/BBH": 0.5111, - "hfopenllm_v2/MATH Level 5": 0.074, - "hfopenllm_v2/GPQA": 0.2794, - "hfopenllm_v2/MUSR": 0.4318, - "hfopenllm_v2/MMLU-PRO": 0.3058 - } - }, - { - "id": "MaziyarPanahi/Llama-3-70B-Instruct-v0.1", - "name": "Llama-3-70B-Instruct-v0.1", - "developer": "MaziyarPanahi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4714, - "hfopenllm_v2/BBH": 0.5366, - "hfopenllm_v2/MATH Level 5": 0.1805, - "hfopenllm_v2/GPQA": 0.2844, - "hfopenllm_v2/MUSR": 0.4433, - "hfopenllm_v2/MMLU-PRO": 0.4618 - } - }, - { - "id": "MaziyarPanahi/Llama-3-8B-Instruct-v0.10", - "name": "Llama-3-8B-Instruct-v0.10", - "developer": "MaziyarPanahi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7667, - "hfopenllm_v2/BBH": 0.4924, - "hfopenllm_v2/MATH Level 5": 0.0574, - "hfopenllm_v2/GPQA": 0.3087, - "hfopenllm_v2/MUSR": 0.4214, - "hfopenllm_v2/MMLU-PRO": 0.3862 - } - }, - { - "id": "MaziyarPanahi/Llama-3-8B-Instruct-v0.8", - "name": "Llama-3-8B-Instruct-v0.8", - "developer": "MaziyarPanahi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7528, - "hfopenllm_v2/BBH": 0.4963, - "hfopenllm_v2/MATH Level 5": 0.0778, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.4202, - "hfopenllm_v2/MMLU-PRO": 0.3853 - } - }, - { - "id": "MaziyarPanahi/Llama-3-8B-Instruct-v0.9", - "name": "Llama-3-8B-Instruct-v0.9", - "developer": "MaziyarPanahi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.763, - "hfopenllm_v2/BBH": 0.4936, - "hfopenllm_v2/MATH Level 5": 0.0733, - "hfopenllm_v2/GPQA": 0.3079, - "hfopenllm_v2/MUSR": 0.4148, - "hfopenllm_v2/MMLU-PRO": 0.3846 - } - }, - { - "id": "MaziyarPanahi/Qwen1.5-MoE-A2.7B-Wikihow", - "name": "Qwen1.5-MoE-A2.7B-Wikihow", - "developer": "MaziyarPanahi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2954, - "hfopenllm_v2/BBH": 0.392, - "hfopenllm_v2/MATH Level 5": 0.0823, - "hfopenllm_v2/GPQA": 0.2752, - "hfopenllm_v2/MUSR": 0.3502, - "hfopenllm_v2/MMLU-PRO": 0.238 - } - }, - { - "id": "MaziyarPanahi/Qwen2-7B-Instruct-v0.1", - "name": "Qwen2-7B-Instruct-v0.1", - "developer": "MaziyarPanahi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3352, - "hfopenllm_v2/BBH": 0.5123, - "hfopenllm_v2/MATH Level 5": 0.2213, - "hfopenllm_v2/GPQA": 0.2852, - "hfopenllm_v2/MUSR": 0.4435, - "hfopenllm_v2/MMLU-PRO": 0.3857 - } - }, - { - "id": "MaziyarPanahi/Qwen2-7B-Instruct-v0.8", - "name": "Qwen2-7B-Instruct-v0.8", - "developer": "MaziyarPanahi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2775, - "hfopenllm_v2/BBH": 0.4637, - "hfopenllm_v2/MATH Level 5": 0.1767, - "hfopenllm_v2/GPQA": 0.2936, - "hfopenllm_v2/MUSR": 0.4293, - "hfopenllm_v2/MMLU-PRO": 0.3566 - } - }, - { - "id": "meditsolutions/Llama-3.1-MedIT-SUN-8B", - "name": "Llama-3.1-MedIT-SUN-8B", - "developer": "meditsolutions", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7837, - "hfopenllm_v2/BBH": 0.5187, - "hfopenllm_v2/MATH Level 5": 0.2092, - "hfopenllm_v2/GPQA": 0.3087, - "hfopenllm_v2/MUSR": 0.4056, - "hfopenllm_v2/MMLU-PRO": 0.3916 - } - }, - { - "id": "meditsolutions/Llama-3.2-SUN-1B-chat", - "name": "Llama-3.2-SUN-1B-chat", - "developer": "meditsolutions", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5482, - "hfopenllm_v2/BBH": 0.3514, - "hfopenllm_v2/MATH Level 5": 0.0642, - "hfopenllm_v2/GPQA": 0.2617, - "hfopenllm_v2/MUSR": 0.3249, - "hfopenllm_v2/MMLU-PRO": 0.1838 - } - }, - { - "id": "meditsolutions/Llama-3.2-SUN-1B-Instruct", - "name": "Llama-3.2-SUN-1B-Instruct", - "developer": "meditsolutions", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6413, - "hfopenllm_v2/BBH": 0.3474, - "hfopenllm_v2/MATH Level 5": 0.071, - "hfopenllm_v2/GPQA": 0.2424, - "hfopenllm_v2/MUSR": 0.3514, - "hfopenllm_v2/MMLU-PRO": 0.1781 - } - }, - { - "id": "meditsolutions/Llama-3.2-SUN-2.4B-checkpoint-26000", - "name": "Llama-3.2-SUN-2.4B-checkpoint-26000", - "developer": "meditsolutions", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2814, - "hfopenllm_v2/BBH": 0.3018, - "hfopenllm_v2/MATH Level 5": 0.0181, - "hfopenllm_v2/GPQA": 0.2777, - "hfopenllm_v2/MUSR": 0.4103, - "hfopenllm_v2/MMLU-PRO": 0.1345 - } - }, - { - "id": "meditsolutions/Llama-3.2-SUN-2.4B-checkpoint-34800", - "name": "Llama-3.2-SUN-2.4B-checkpoint-34800", - "developer": "meditsolutions", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2501, - "hfopenllm_v2/BBH": 0.3161, - "hfopenllm_v2/MATH Level 5": 0.0106, - "hfopenllm_v2/GPQA": 0.2861, - "hfopenllm_v2/MUSR": 0.4022, - "hfopenllm_v2/MMLU-PRO": 0.1357 - } - }, - { - "id": "meditsolutions/Llama-3.2-SUN-2.4B-v1.0.0", - "name": "Llama-3.2-SUN-2.4B-v1.0.0", - "developer": "meditsolutions", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5637, - "hfopenllm_v2/BBH": 0.3391, - "hfopenllm_v2/MATH Level 5": 0.0627, - "hfopenllm_v2/GPQA": 0.2576, - "hfopenllm_v2/MUSR": 0.3209, - "hfopenllm_v2/MMLU-PRO": 0.1543 - } - }, - { - "id": "meditsolutions/Llama-3.2-SUN-2.5B-chat", - "name": "Llama-3.2-SUN-2.5B-chat", - "developer": "meditsolutions", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5604, - "hfopenllm_v2/BBH": 0.3575, - "hfopenllm_v2/MATH Level 5": 0.071, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3155, - "hfopenllm_v2/MMLU-PRO": 0.1813 - } - }, - { - "id": "meditsolutions/Llama-3.2-SUN-HDIC-1B-Instruct", - "name": "Llama-3.2-SUN-HDIC-1B-Instruct", - "developer": "meditsolutions", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6827, - "hfopenllm_v2/BBH": 0.3508, - "hfopenllm_v2/MATH Level 5": 0.0619, - "hfopenllm_v2/GPQA": 0.2366, - "hfopenllm_v2/MUSR": 0.3594, - "hfopenllm_v2/MMLU-PRO": 0.1687 - } - }, - { - "id": "meditsolutions/MedIT-Mesh-3B-Instruct", - "name": "MedIT-Mesh-3B-Instruct", - "developer": "meditsolutions", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5814, - "hfopenllm_v2/BBH": 0.5576, - "hfopenllm_v2/MATH Level 5": 0.2032, - "hfopenllm_v2/GPQA": 0.3238, - "hfopenllm_v2/MUSR": 0.4048, - "hfopenllm_v2/MMLU-PRO": 0.4012 - } - }, - { - "id": "meditsolutions/MSH-Lite-7B-v1-Bielik-v2.3-Instruct-Llama-Prune", - "name": "MSH-Lite-7B-v1-Bielik-v2.3-Instruct-Llama-Prune", - "developer": "meditsolutions", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3655, - "hfopenllm_v2/BBH": 0.4035, - "hfopenllm_v2/MATH Level 5": 0.0264, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.4253, - "hfopenllm_v2/MMLU-PRO": 0.219 - } - }, - { - "id": "meditsolutions/MSH-v1-Bielik-v2.3-Instruct-MedIT-merge", - "name": "MSH-v1-Bielik-v2.3-Instruct-MedIT-merge", - "developer": "meditsolutions", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5814, - "hfopenllm_v2/BBH": 0.5672, - "hfopenllm_v2/MATH Level 5": 0.2077, - "hfopenllm_v2/GPQA": 0.3456, - "hfopenllm_v2/MUSR": 0.4385, - "hfopenllm_v2/MMLU-PRO": 0.35 - } - }, - { - "id": "meditsolutions/SmolLM2-MedIT-Upscale-2B", - "name": "SmolLM2-MedIT-Upscale-2B", - "developer": "meditsolutions", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6429, - "hfopenllm_v2/BBH": 0.3551, - "hfopenllm_v2/MATH Level 5": 0.0559, - "hfopenllm_v2/GPQA": 0.2643, - "hfopenllm_v2/MUSR": 0.3314, - "hfopenllm_v2/MMLU-PRO": 0.1971 - } - }, - { - "id": "meetkai/functionary-small-v3.1", - "name": "functionary-small-v3.1", - "developer": "meetkai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6275, - "hfopenllm_v2/BBH": 0.4982, - "hfopenllm_v2/MATH Level 5": 0.1571, - "hfopenllm_v2/GPQA": 0.2886, - "hfopenllm_v2/MUSR": 0.3834, - "hfopenllm_v2/MMLU-PRO": 0.3349 - } - }, - { - "id": "meraGPT/mera-mix-4x7B", - "name": "mera-mix-4x7B", - "developer": "meraGPT", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4832, - "hfopenllm_v2/BBH": 0.4019, - "hfopenllm_v2/MATH Level 5": 0.0536, - "hfopenllm_v2/GPQA": 0.3045, - "hfopenllm_v2/MUSR": 0.4057, - "hfopenllm_v2/MMLU-PRO": 0.2748 - } - }, - { - "id": "mergekit-community/diabolic6045_ELN-AOC-CAIN", - "name": "diabolic6045_ELN-AOC-CAIN", - "developer": "mergekit-community", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0862, - "hfopenllm_v2/BBH": 0.3126, - "hfopenllm_v2/MATH Level 5": 0.0121, - "hfopenllm_v2/GPQA": 0.2634, - "hfopenllm_v2/MUSR": 0.3658, - "hfopenllm_v2/MMLU-PRO": 0.1191 - } - }, - { - "id": "mergekit-community/JAJUKA-WEWILLNEVERFORGETYOU-3B", - "name": "JAJUKA-WEWILLNEVERFORGETYOU-3B", - "developer": "mergekit-community", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4941, - "hfopenllm_v2/BBH": 0.437, - "hfopenllm_v2/MATH Level 5": 0.1246, - "hfopenllm_v2/GPQA": 0.2928, - "hfopenllm_v2/MUSR": 0.3656, - "hfopenllm_v2/MMLU-PRO": 0.3033 - } - }, - { - "id": "mergekit-community/mergekit-dare_ties-ajgjgea", - "name": "mergekit-dare_ties-ajgjgea", - "developer": "mergekit-community", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5263, - "hfopenllm_v2/BBH": 0.3495, - "hfopenllm_v2/MATH Level 5": 0.0642, - "hfopenllm_v2/GPQA": 0.2643, - "hfopenllm_v2/MUSR": 0.3289, - "hfopenllm_v2/MMLU-PRO": 0.1744 - } - }, - { - "id": "mergekit-community/mergekit-della-zgowfmf", - "name": "mergekit-della-zgowfmf", - "developer": "mergekit-community", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4828, - "hfopenllm_v2/BBH": 0.6591, - "hfopenllm_v2/MATH Level 5": 0.3618, - "hfopenllm_v2/GPQA": 0.3901, - "hfopenllm_v2/MUSR": 0.4834, - "hfopenllm_v2/MMLU-PRO": 0.5415 - } - }, - { - "id": "mergekit-community/mergekit-model_stock-azgztvm", - "name": "mergekit-model_stock-azgztvm", - "developer": "mergekit-community", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5062, - "hfopenllm_v2/BBH": 0.6543, - "hfopenllm_v2/MATH Level 5": 0.4373, - "hfopenllm_v2/GPQA": 0.3817, - "hfopenllm_v2/MUSR": 0.473, - "hfopenllm_v2/MMLU-PRO": 0.5406 - } - }, - { - "id": "mergekit-community/mergekit-slerp-fmrazcr", - "name": "mergekit-slerp-fmrazcr", - "developer": "mergekit-community", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4174, - "hfopenllm_v2/BBH": 0.5342, - "hfopenllm_v2/MATH Level 5": 0.1193, - "hfopenllm_v2/GPQA": 0.3112, - "hfopenllm_v2/MUSR": 0.4105, - "hfopenllm_v2/MMLU-PRO": 0.3777 - } - }, - { - "id": "mergekit-community/mergekit-ties-rraxdhv", - "name": "mergekit-ties-rraxdhv", - "developer": "mergekit-community", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1123, - "hfopenllm_v2/BBH": 0.5184, - "hfopenllm_v2/MATH Level 5": 0.04, - "hfopenllm_v2/GPQA": 0.3079, - "hfopenllm_v2/MUSR": 0.4202, - "hfopenllm_v2/MMLU-PRO": 0.391 - } - }, - { - "id": "mergekit-community/mergekit-ties-ykqemwr", - "name": "mergekit-ties-ykqemwr", - "developer": "mergekit-community", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.36, - "hfopenllm_v2/BBH": 0.5455, - "hfopenllm_v2/MATH Level 5": 0.1224, - "hfopenllm_v2/GPQA": 0.3221, - "hfopenllm_v2/MUSR": 0.4198, - "hfopenllm_v2/MMLU-PRO": 0.3734 - } - }, - { - "id": "mergekit-community/sexeh_time_testing", - "name": "sexeh_time_testing", - "developer": "mergekit-community", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7329, - "hfopenllm_v2/BBH": 0.5241, - "hfopenllm_v2/MATH Level 5": 0.0899, - "hfopenllm_v2/GPQA": 0.2911, - "hfopenllm_v2/MUSR": 0.3619, - "hfopenllm_v2/MMLU-PRO": 0.3667 - } - }, - { - "id": "mergekit-community/SuperQwen-2.5-1.5B", - "name": "SuperQwen-2.5-1.5B", - "developer": "mergekit-community", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1336, - "hfopenllm_v2/BBH": 0.2907, - "hfopenllm_v2/MATH Level 5": 0.0196, - "hfopenllm_v2/GPQA": 0.2542, - "hfopenllm_v2/MUSR": 0.3355, - "hfopenllm_v2/MMLU-PRO": 0.1075 - } - }, - { - "id": "mergekit-community/VirtuosoSmall-InstructModelStock", - "name": "VirtuosoSmall-InstructModelStock", - "developer": "mergekit-community", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5238, - "hfopenllm_v2/BBH": 0.6518, - "hfopenllm_v2/MATH Level 5": 0.4094, - "hfopenllm_v2/GPQA": 0.3826, - "hfopenllm_v2/MUSR": 0.4756, - "hfopenllm_v2/MMLU-PRO": 0.5421 - } - }, - { - "id": "MEscriva/ECE-PRYMMAL-0.5B-FT-V5-MUSR-Mathis", - "name": "ECE-PRYMMAL-0.5B-FT-V5-MUSR-Mathis", - "developer": "MEscriva", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0866, - "hfopenllm_v2/BBH": 0.3057, - "hfopenllm_v2/MATH Level 5": 0.0106, - "hfopenllm_v2/GPQA": 0.2517, - "hfopenllm_v2/MUSR": 0.4017, - "hfopenllm_v2/MMLU-PRO": 0.1154 - } - }, - { - "id": "meta-llama/Llama-2-13b-chat-hf", - "name": "Llama-2-13b-chat-hf", - "developer": "meta-llama", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3985, - "hfopenllm_v2/BBH": 0.3343, - "hfopenllm_v2/MATH Level 5": 0.0136, - "hfopenllm_v2/GPQA": 0.2315, - "hfopenllm_v2/MUSR": 0.4007, - "hfopenllm_v2/MMLU-PRO": 0.1923 - } - }, - { - "id": "meta-llama/Llama-2-13b-hf", - "name": "Llama-2-13b-hf", - "developer": "meta-llama", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2482, - "hfopenllm_v2/BBH": 0.4126, - "hfopenllm_v2/MATH Level 5": 0.0151, - "hfopenllm_v2/GPQA": 0.281, - "hfopenllm_v2/MUSR": 0.3538, - "hfopenllm_v2/MMLU-PRO": 0.2378 - } - }, - { - "id": "meta-llama/Llama-2-70b-chat-hf", - "name": "Llama-2-70b-chat-hf", - "developer": "meta-llama", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4958, - "hfopenllm_v2/BBH": 0.3042, - "hfopenllm_v2/MATH Level 5": 0.0295, - "hfopenllm_v2/GPQA": 0.2643, - "hfopenllm_v2/MUSR": 0.3687, - "hfopenllm_v2/MMLU-PRO": 0.2433 - } - }, - { - "id": "meta-llama/Llama-2-70b-hf", - "name": "Llama-2-70b-hf", - "developer": "meta-llama", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2407, - "hfopenllm_v2/BBH": 0.5473, - "hfopenllm_v2/MATH Level 5": 0.0325, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.4124, - "hfopenllm_v2/MMLU-PRO": 0.3718 - } - }, - { - "id": "meta-llama/Llama-2-7b-chat-hf", - "name": "Llama-2-7b-chat-hf", - "developer": "meta-llama", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3986, - "hfopenllm_v2/BBH": 0.3114, - "hfopenllm_v2/MATH Level 5": 0.0196, - "hfopenllm_v2/GPQA": 0.2534, - "hfopenllm_v2/MUSR": 0.3676, - "hfopenllm_v2/MMLU-PRO": 0.1688 - } - }, - { - "id": "meta-llama/Llama-2-7b-hf", - "name": "Llama-2-7b-hf", - "developer": "meta-llama", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2519, - "hfopenllm_v2/BBH": 0.3496, - "hfopenllm_v2/MATH Level 5": 0.0174, - "hfopenllm_v2/GPQA": 0.2668, - "hfopenllm_v2/MUSR": 0.3701, - "hfopenllm_v2/MMLU-PRO": 0.1861 - } - }, - { - "id": "meta-llama/Llama-3.1-70B", - "name": "Llama-3.1-70B", - "developer": "meta-llama", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1684, - "hfopenllm_v2/BBH": 0.626, - "hfopenllm_v2/MATH Level 5": 0.1843, - "hfopenllm_v2/GPQA": 0.3876, - "hfopenllm_v2/MUSR": 0.4572, - "hfopenllm_v2/MMLU-PRO": 0.4654 - } - }, - { - "id": "meta-llama/Llama-3.1-70B-Instruct", - "name": "Llama-3.1-70B-Instruct", - "developer": "meta-llama", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8669, - "hfopenllm_v2/BBH": 0.6917, - "hfopenllm_v2/MATH Level 5": 0.3807, - "hfopenllm_v2/GPQA": 0.3565, - "hfopenllm_v2/MUSR": 0.4581, - "hfopenllm_v2/MMLU-PRO": 0.5309 - } - }, - { - "id": "meta-llama/Llama-3.1-8B", - "name": "Llama-3.1-8B", - "developer": "meta-llama", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1246, - "hfopenllm_v2/BBH": 0.466, - "hfopenllm_v2/MATH Level 5": 0.0657, - "hfopenllm_v2/GPQA": 0.3104, - "hfopenllm_v2/MUSR": 0.3812, - "hfopenllm_v2/MMLU-PRO": 0.3288 - } - }, - { - "id": "meta-llama/Llama-3.1-8B-Instruct", - "name": "Llama-3.1-8B-Instruct", - "developer": "meta-llama", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4922, - "hfopenllm_v2/BBH": 0.5087, - "hfopenllm_v2/MATH Level 5": 0.1556, - "hfopenllm_v2/GPQA": 0.3154, - "hfopenllm_v2/MUSR": 0.3972, - "hfopenllm_v2/MMLU-PRO": 0.3798 - } - }, - { - "id": "meta-llama/Llama-3.2-1B", - "name": "Llama-3.2-1B", - "developer": "meta-llama", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1478, - "hfopenllm_v2/BBH": 0.3115, - "hfopenllm_v2/MATH Level 5": 0.0121, - "hfopenllm_v2/GPQA": 0.2282, - "hfopenllm_v2/MUSR": 0.3447, - "hfopenllm_v2/MMLU-PRO": 0.1203 - } - }, - { - "id": "meta-llama/Llama-3.2-1B-Instruct", - "name": "Llama-3.2-1B-Instruct", - "developer": "meta-llama", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5698, - "hfopenllm_v2/BBH": 0.3497, - "hfopenllm_v2/MATH Level 5": 0.0702, - "hfopenllm_v2/GPQA": 0.2752, - "hfopenllm_v2/MUSR": 0.3329, - "hfopenllm_v2/MMLU-PRO": 0.1682 - } - }, - { - "id": "meta-llama/Llama-3.2-3B", - "name": "Llama-3.2-3B", - "developer": "meta-llama", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1337, - "hfopenllm_v2/BBH": 0.3905, - "hfopenllm_v2/MATH Level 5": 0.0189, - "hfopenllm_v2/GPQA": 0.2676, - "hfopenllm_v2/MUSR": 0.3577, - "hfopenllm_v2/MMLU-PRO": 0.2488 - } - }, - { - "id": "meta-llama/Llama-3.2-3B-Instruct", - "name": "Llama-3.2-3B-Instruct", - "developer": "meta-llama", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7393, - "hfopenllm_v2/BBH": 0.461, - "hfopenllm_v2/MATH Level 5": 0.1767, - "hfopenllm_v2/GPQA": 0.2785, - "hfopenllm_v2/MUSR": 0.3529, - "hfopenllm_v2/MMLU-PRO": 0.3195 - } - }, - { - "id": "meta-llama/Llama-3.3-70B-Instruct", - "name": "Llama-3.3-70B-Instruct", - "developer": "meta-llama", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8998, - "hfopenllm_v2/BBH": 0.6919, - "hfopenllm_v2/MATH Level 5": 0.4834, - "hfopenllm_v2/GPQA": 0.3289, - "hfopenllm_v2/MUSR": 0.4461, - "hfopenllm_v2/MMLU-PRO": 0.5332 - } - }, - { - "id": "meta-llama/Meta-Llama-3-70B", - "name": "Meta-Llama-3-70B", - "developer": "meta-llama", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1603, - "hfopenllm_v2/BBH": 0.6461, - "hfopenllm_v2/MATH Level 5": 0.1858, - "hfopenllm_v2/GPQA": 0.3977, - "hfopenllm_v2/MUSR": 0.4518, - "hfopenllm_v2/MMLU-PRO": 0.4709 - } - }, - { - "id": "meta-llama/Meta-Llama-3-70B-Instruct", - "name": "Meta-Llama-3-70B-Instruct", - "developer": "meta-llama", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8099, - "hfopenllm_v2/BBH": 0.6547, - "hfopenllm_v2/MATH Level 5": 0.2447, - "hfopenllm_v2/GPQA": 0.2869, - "hfopenllm_v2/MUSR": 0.4154, - "hfopenllm_v2/MMLU-PRO": 0.5207, - "reward-bench/Score": 0.7627, - "reward-bench/Chat": 0.9763, - "reward-bench/Chat Hard": 0.5888, - "reward-bench/Safety": 0.7297, - "reward-bench/Reasoning": 0.7854, - "reward-bench/Prior Sets (0.5 weight)": 0.7035 - } - }, - { - "id": "meta-llama/Meta-Llama-3-8B", - "name": "Meta-Llama-3-8B", - "developer": "meta-llama", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1455, - "hfopenllm_v2/BBH": 0.4598, - "hfopenllm_v2/MATH Level 5": 0.0453, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.3614, - "hfopenllm_v2/MMLU-PRO": 0.321 - } - }, - { - "id": "meta-llama/Meta-Llama-3-8B-Instruct", - "name": "Meta-Llama-3-8B-Instruct", - "developer": "meta-llama", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7408, - "hfopenllm_v2/BBH": 0.4989, - "hfopenllm_v2/MATH Level 5": 0.0869, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3568, - "hfopenllm_v2/MMLU-PRO": 0.3664, - "reward-bench/Score": 0.645, - "reward-bench/Chat": 0.8547, - "reward-bench/Chat Hard": 0.4156, - "reward-bench/Safety": 0.6797, - "reward-bench/Reasoning": 0.6482, - "reward-bench/Prior Sets (0.5 weight)": 0.6082 - } - }, - { - "id": "meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo", - "name": "meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo", - "developer": "meta-llama", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.8412, - "reward-bench/Chat": 0.9721, - "reward-bench/Chat Hard": 0.7456, - "reward-bench/Safety": 0.7757, - "reward-bench/Reasoning": 0.8715 - } - }, - { - "id": "meta-llama/Meta-Llama-3.1-70B-Instruct", - "name": "meta-llama/Meta-Llama-3.1-70B-Instruct", - "developer": "meta-llama", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.8405, - "reward-bench/Chat": 0.9721, - "reward-bench/Chat Hard": 0.7018, - "reward-bench/Safety": 0.8284, - "reward-bench/Reasoning": 0.8599 - } - }, - { - "id": "meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo", - "name": "meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo", - "developer": "meta-llama", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7808, - "reward-bench/Chat": 0.8757, - "reward-bench/Chat Hard": 0.6689, - "reward-bench/Safety": 0.7507, - "reward-bench/Reasoning": 0.828 - } - }, - { - "id": "meta-llama/Meta-Llama-3.1-8B", - "name": "Meta Llama 3.1 8B", - "developer": "unknown", - "evaluator_relationship": null, - "benchmark_scores": { - "la_leaderboard/la_leaderboard": 27.04 - } - }, - { - "id": "meta-llama/Meta-Llama-3.1-8B-Instruct", - "name": "Meta Llama 3.1 8B Instruct", - "developer": "unknown", - "evaluator_relationship": null, - "benchmark_scores": { - "la_leaderboard/la_leaderboard": 30.23 - } - }, - { - "id": "meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo", - "name": "meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo", - "developer": "meta-llama", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6565, - "reward-bench/Chat": 0.8073, - "reward-bench/Chat Hard": 0.4978, - "reward-bench/Safety": 0.6399, - "reward-bench/Reasoning": 0.6811 - } - }, - { - "id": "meta-metrics/MetaMetrics-RM-v1.0", - "name": "meta-metrics/MetaMetrics-RM-v1.0", - "developer": "meta-metrics", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.9342, - "reward-bench/Chat": 0.9832, - "reward-bench/Chat Hard": 0.864, - "reward-bench/Safety": 0.9081, - "reward-bench/Reasoning": 0.9816 - } - }, - { - "id": "meta/LLaMA-13B", - "name": "LLaMA 13B", - "developer": "Meta", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.595, - "helm_classic/MMLU": 0.422, - "helm_classic/BoolQ": 0.714, - "helm_classic/NarrativeQA": 0.711, - "helm_classic/NaturalQuestions (open-book)": 0.614, - "helm_classic/QuAC": 0.347, - "helm_classic/HellaSwag": -1.0, - "helm_classic/OpenbookQA": -1.0, - "helm_classic/TruthfulQA": 0.324, - "helm_classic/MS MARCO (TREC)": -1.0, - "helm_classic/CNN/DailyMail": -1.0, - "helm_classic/XSUM": -1.0, - "helm_classic/IMDB": 0.928, - "helm_classic/CivilComments": 0.6, - "helm_classic/RAFT": 0.643 - } - }, - { - "id": "meta/llama-2-13b", - "name": "Llama 2 13B", - "developer": "Meta", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.823, - "helm_classic/MMLU": 0.507, - "helm_classic/BoolQ": 0.811, - "helm_classic/NarrativeQA": 0.744, - "helm_classic/NaturalQuestions (open-book)": 0.637, - "helm_classic/QuAC": 0.424, - "helm_classic/HellaSwag": -1.0, - "helm_classic/OpenbookQA": -1.0, - "helm_classic/TruthfulQA": 0.33, - "helm_classic/MS MARCO (TREC)": -1.0, - "helm_classic/CNN/DailyMail": -1.0, - "helm_classic/XSUM": -1.0, - "helm_classic/IMDB": 0.962, - "helm_classic/CivilComments": 0.588, - "helm_classic/RAFT": 0.707, - "helm_lite/Mean win rate": 0.233, - "helm_lite/NarrativeQA": 0.741, - "helm_lite/NaturalQuestions (closed-book)": 0.371, - "helm_lite/OpenbookQA": 0.634, - "helm_lite/MMLU": 0.505, - "helm_lite/MATH": 0.102, - "helm_lite/GSM8K": 0.266, - "helm_lite/LegalBench": 0.591, - "helm_lite/MedQA": 0.392, - "helm_lite/WMT 2014": 0.167, - "helm_mmlu/MMLU All Subjects": 0.554, - "helm_mmlu/Abstract Algebra": 0.27, - "helm_mmlu/Anatomy": 0.496, - "helm_mmlu/College Physics": 0.235, - "helm_mmlu/Computer Security": 0.69, - "helm_mmlu/Econometrics": 0.307, - "helm_mmlu/Global Facts": 0.38, - "helm_mmlu/Jurisprudence": 0.704, - "helm_mmlu/Philosophy": 0.672, - "helm_mmlu/Professional Psychology": 0.567, - "helm_mmlu/Us Foreign Policy": 0.83, - "helm_mmlu/Astronomy": 0.546, - "helm_mmlu/Business Ethics": 0.55, - "helm_mmlu/Clinical Knowledge": 0.592, - "helm_mmlu/Conceptual Physics": 0.413, - "helm_mmlu/Electrical Engineering": 0.49, - "helm_mmlu/Elementary Mathematics": 0.307, - "helm_mmlu/Formal Logic": 0.381, - "helm_mmlu/High School World History": 0.705, - "helm_mmlu/Human Sexuality": 0.618, - "helm_mmlu/International Law": 0.752, - "helm_mmlu/Logical Fallacies": 0.687, - "helm_mmlu/Machine Learning": 0.286, - "helm_mmlu/Management": 0.738, - "helm_mmlu/Marketing": 0.786, - "helm_mmlu/Medical Genetics": 0.57, - "helm_mmlu/Miscellaneous": 0.748, - "helm_mmlu/Moral Scenarios": 0.407, - "helm_mmlu/Nutrition": 0.627, - "helm_mmlu/Prehistory": 0.654, - "helm_mmlu/Public Relations": 0.6, - "helm_mmlu/Security Studies": 0.608, - "helm_mmlu/Sociology": 0.761, - "helm_mmlu/Virology": 0.476, - "helm_mmlu/World Religions": 0.76, - "helm_mmlu/Mean win rate": 0.502 - } - }, - { - "id": "meta/llama-2-70b", - "name": "Llama 2 70B", - "developer": "Meta", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.944, - "helm_classic/MMLU": 0.582, - "helm_classic/BoolQ": 0.886, - "helm_classic/NarrativeQA": 0.77, - "helm_classic/NaturalQuestions (open-book)": 0.674, - "helm_classic/QuAC": 0.484, - "helm_classic/HellaSwag": -1.0, - "helm_classic/OpenbookQA": -1.0, - "helm_classic/TruthfulQA": 0.554, - "helm_classic/MS MARCO (TREC)": -1.0, - "helm_classic/CNN/DailyMail": -1.0, - "helm_classic/XSUM": -1.0, - "helm_classic/IMDB": 0.961, - "helm_classic/CivilComments": 0.652, - "helm_classic/RAFT": 0.727, - "helm_lite/Mean win rate": 0.482, - "helm_lite/NarrativeQA": 0.763, - "helm_lite/NaturalQuestions (closed-book)": 0.46, - "helm_lite/OpenbookQA": 0.838, - "helm_lite/MMLU": 0.58, - "helm_lite/MATH": 0.323, - "helm_lite/GSM8K": 0.567, - "helm_lite/LegalBench": 0.673, - "helm_lite/MedQA": 0.618, - "helm_lite/WMT 2014": 0.196, - "helm_mmlu/MMLU All Subjects": 0.695, - "helm_mmlu/Abstract Algebra": 0.31, - "helm_mmlu/Anatomy": 0.607, - "helm_mmlu/College Physics": 0.363, - "helm_mmlu/Computer Security": 0.77, - "helm_mmlu/Econometrics": 0.43, - "helm_mmlu/Global Facts": 0.47, - "helm_mmlu/Jurisprudence": 0.824, - "helm_mmlu/Philosophy": 0.791, - "helm_mmlu/Professional Psychology": 0.76, - "helm_mmlu/Us Foreign Policy": 0.92, - "helm_mmlu/Astronomy": 0.829, - "helm_mmlu/Business Ethics": 0.73, - "helm_mmlu/Clinical Knowledge": 0.717, - "helm_mmlu/Conceptual Physics": 0.668, - "helm_mmlu/Electrical Engineering": 0.634, - "helm_mmlu/Elementary Mathematics": 0.421, - "helm_mmlu/Formal Logic": 0.468, - "helm_mmlu/High School World History": 0.882, - "helm_mmlu/Human Sexuality": 0.84, - "helm_mmlu/International Law": 0.868, - "helm_mmlu/Logical Fallacies": 0.791, - "helm_mmlu/Machine Learning": 0.491, - "helm_mmlu/Management": 0.845, - "helm_mmlu/Marketing": 0.889, - "helm_mmlu/Medical Genetics": 0.72, - "helm_mmlu/Miscellaneous": 0.857, - "helm_mmlu/Moral Scenarios": 0.45, - "helm_mmlu/Nutrition": 0.758, - "helm_mmlu/Prehistory": 0.84, - "helm_mmlu/Public Relations": 0.745, - "helm_mmlu/Security Studies": 0.796, - "helm_mmlu/Sociology": 0.9, - "helm_mmlu/Virology": 0.53, - "helm_mmlu/World Religions": 0.854, - "helm_mmlu/Mean win rate": 0.508 - } - }, - { - "id": "meta/llama-2-7b", - "name": "Llama 2 7B", - "developer": "Meta", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.607, - "helm_classic/MMLU": 0.431, - "helm_classic/BoolQ": 0.762, - "helm_classic/NarrativeQA": 0.691, - "helm_classic/NaturalQuestions (open-book)": 0.611, - "helm_classic/QuAC": 0.406, - "helm_classic/HellaSwag": -1.0, - "helm_classic/OpenbookQA": -1.0, - "helm_classic/TruthfulQA": 0.272, - "helm_classic/MS MARCO (TREC)": -1.0, - "helm_classic/CNN/DailyMail": -1.0, - "helm_classic/XSUM": -1.0, - "helm_classic/IMDB": 0.907, - "helm_classic/CivilComments": 0.562, - "helm_classic/RAFT": 0.643, - "helm_lite/Mean win rate": 0.152, - "helm_lite/NarrativeQA": 0.686, - "helm_lite/NaturalQuestions (closed-book)": 0.333, - "helm_lite/OpenbookQA": 0.544, - "helm_lite/MMLU": 0.425, - "helm_lite/MATH": 0.097, - "helm_lite/GSM8K": 0.154, - "helm_lite/LegalBench": 0.502, - "helm_lite/MedQA": 0.392, - "helm_lite/WMT 2014": 0.144, - "helm_mmlu/MMLU All Subjects": 0.458, - "helm_mmlu/Abstract Algebra": 0.29, - "helm_mmlu/Anatomy": 0.452, - "helm_mmlu/College Physics": 0.196, - "helm_mmlu/Computer Security": 0.59, - "helm_mmlu/Econometrics": 0.316, - "helm_mmlu/Global Facts": 0.29, - "helm_mmlu/Jurisprudence": 0.519, - "helm_mmlu/Philosophy": 0.592, - "helm_mmlu/Professional Psychology": 0.459, - "helm_mmlu/Us Foreign Policy": 0.64, - "helm_mmlu/Astronomy": 0.408, - "helm_mmlu/Business Ethics": 0.48, - "helm_mmlu/Clinical Knowledge": 0.453, - "helm_mmlu/Conceptual Physics": 0.434, - "helm_mmlu/Electrical Engineering": 0.407, - "helm_mmlu/Elementary Mathematics": 0.254, - "helm_mmlu/Formal Logic": 0.27, - "helm_mmlu/High School World History": 0.662, - "helm_mmlu/Human Sexuality": 0.557, - "helm_mmlu/International Law": 0.628, - "helm_mmlu/Logical Fallacies": 0.466, - "helm_mmlu/Machine Learning": 0.402, - "helm_mmlu/Management": 0.563, - "helm_mmlu/Marketing": 0.697, - "helm_mmlu/Medical Genetics": 0.53, - "helm_mmlu/Miscellaneous": 0.632, - "helm_mmlu/Moral Scenarios": 0.238, - "helm_mmlu/Nutrition": 0.497, - "helm_mmlu/Prehistory": 0.503, - "helm_mmlu/Public Relations": 0.509, - "helm_mmlu/Security Studies": 0.433, - "helm_mmlu/Sociology": 0.617, - "helm_mmlu/Virology": 0.392, - "helm_mmlu/World Religions": 0.713, - "helm_mmlu/Mean win rate": 0.681 - } - }, - { - "id": "meta/llama-3-1-8b-instruct-prompt", - "name": "Llama-3.1-8B-Instruct (Prompt)", - "developer": "Meta", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 85.0, - "bfcl/bfcl.overall.overall_accuracy": 25.83, - "bfcl/bfcl.overall.total_cost_usd": 7.49, - "bfcl/bfcl.overall.latency_mean_s": 5.6, - "bfcl/bfcl.overall.latency_std_s": 19.37, - "bfcl/bfcl.overall.latency_p95_s": 22.6, - "bfcl/bfcl.non_live.ast_accuracy": 84.0, - "bfcl/bfcl.non_live.simple_ast_accuracy": 71.0, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 95.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 87.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 82.5, - "bfcl/bfcl.live.live_accuracy": 70.76, - "bfcl/bfcl.live.live_simple_ast_accuracy": 72.87, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 71.13, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 50.0, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 45.83, - "bfcl/bfcl.multi_turn.accuracy": 11.12, - "bfcl/bfcl.multi_turn.base_accuracy": 13.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 9.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 9.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 13.0, - "bfcl/bfcl.web_search.accuracy": 3.0, - "bfcl/bfcl.web_search.base_accuracy": 6.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 0.0, - "bfcl/bfcl.memory.accuracy": 10.75, - "bfcl/bfcl.memory.kv_accuracy": 7.74, - "bfcl/bfcl.memory.vector_accuracy": 5.81, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 18.71, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 93.75, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 42.7, - "bfcl/bfcl.format_sensitivity.max_delta": 74.5, - "bfcl/bfcl.format_sensitivity.stddev": 29.1 - } - }, - { - "id": "meta/llama-3-2-1b-instruct-fc", - "name": "Llama-3.2-1B-Instruct (FC)", - "developer": "Meta", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 107.0, - "bfcl/bfcl.overall.overall_accuracy": 10.82, - "bfcl/bfcl.overall.total_cost_usd": 1.64, - "bfcl/bfcl.overall.latency_mean_s": 3.21, - "bfcl/bfcl.overall.latency_std_s": 10.04, - "bfcl/bfcl.overall.latency_p95_s": 9.77, - "bfcl/bfcl.non_live.ast_accuracy": 38.38, - "bfcl/bfcl.non_live.simple_ast_accuracy": 44.0, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 50.5, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 44.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 15.0, - "bfcl/bfcl.live.live_accuracy": 11.77, - "bfcl/bfcl.live.live_simple_ast_accuracy": 31.78, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 7.31, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 0.0, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 0.0, - "bfcl/bfcl.multi_turn.accuracy": 0.0, - "bfcl/bfcl.multi_turn.base_accuracy": 0.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 0.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 0.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 0.0, - "bfcl/bfcl.web_search.accuracy": 0.0, - "bfcl/bfcl.web_search.base_accuracy": 0.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 0.0, - "bfcl/bfcl.memory.accuracy": 3.23, - "bfcl/bfcl.memory.kv_accuracy": 2.58, - "bfcl/bfcl.memory.vector_accuracy": 2.58, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 4.52, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 43.75, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 51.57 - } - }, - { - "id": "meta/llama-3-2-3b-instruct-fc", - "name": "Llama-3.2-3B-Instruct (FC)", - "developer": "Meta", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 98.0, - "bfcl/bfcl.overall.overall_accuracy": 21.95, - "bfcl/bfcl.overall.total_cost_usd": 6.2, - "bfcl/bfcl.overall.latency_mean_s": 6.1, - "bfcl/bfcl.overall.latency_std_s": 20.07, - "bfcl/bfcl.overall.latency_p95_s": 17.27, - "bfcl/bfcl.non_live.ast_accuracy": 82.67, - "bfcl/bfcl.non_live.simple_ast_accuracy": 70.67, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 92.5, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 88.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 79.0, - "bfcl/bfcl.live.live_accuracy": 58.33, - "bfcl/bfcl.live.live_simple_ast_accuracy": 65.12, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 57.64, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 25.0, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 37.5, - "bfcl/bfcl.multi_turn.accuracy": 4.0, - "bfcl/bfcl.multi_turn.base_accuracy": 5.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 3.5, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 4.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 3.5, - "bfcl/bfcl.web_search.accuracy": 1.0, - "bfcl/bfcl.web_search.base_accuracy": 2.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 0.0, - "bfcl/bfcl.memory.accuracy": 6.24, - "bfcl/bfcl.memory.kv_accuracy": 3.23, - "bfcl/bfcl.memory.vector_accuracy": 3.23, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 12.26, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 87.5, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 52.06 - } - }, - { - "id": "meta/llama-3-3-70b-instruct-fc", - "name": "Llama-3.3-70B-Instruct (FC)", - "developer": "Meta", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 62.0, - "bfcl/bfcl.overall.overall_accuracy": 31.9, - "bfcl/bfcl.overall.total_cost_usd": 29.54, - "bfcl/bfcl.overall.latency_mean_s": 26.11, - "bfcl/bfcl.overall.latency_std_s": 93.22, - "bfcl/bfcl.overall.latency_p95_s": 187.93, - "bfcl/bfcl.non_live.ast_accuracy": 88.02, - "bfcl/bfcl.non_live.simple_ast_accuracy": 76.08, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 95.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 90.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 91.0, - "bfcl/bfcl.live.live_accuracy": 76.61, - "bfcl/bfcl.live.live_simple_ast_accuracy": 81.4, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 75.5, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 81.25, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 70.83, - "bfcl/bfcl.multi_turn.accuracy": 21.5, - "bfcl/bfcl.multi_turn.base_accuracy": 26.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 19.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 14.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 26.5, - "bfcl/bfcl.web_search.accuracy": 10.0, - "bfcl/bfcl.web_search.base_accuracy": 14.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 6.0, - "bfcl/bfcl.memory.accuracy": 8.17, - "bfcl/bfcl.memory.kv_accuracy": 4.52, - "bfcl/bfcl.memory.vector_accuracy": 8.39, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 11.61, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 100.0, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 53.53 - } - }, - { - "id": "meta/llama-3-70b", - "name": "Llama 3 70B", - "developer": "Meta", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.793, - "helm_lite/NarrativeQA": 0.798, - "helm_lite/NaturalQuestions (closed-book)": 0.475, - "helm_lite/OpenbookQA": 0.934, - "helm_lite/MMLU": 0.695, - "helm_lite/MATH": 0.663, - "helm_lite/GSM8K": 0.805, - "helm_lite/LegalBench": 0.733, - "helm_lite/MedQA": 0.777, - "helm_lite/WMT 2014": 0.225, - "helm_mmlu/MMLU All Subjects": 0.793, - "helm_mmlu/Abstract Algebra": 0.43, - "helm_mmlu/Anatomy": 0.785, - "helm_mmlu/College Physics": 0.529, - "helm_mmlu/Computer Security": 0.85, - "helm_mmlu/Econometrics": 0.693, - "helm_mmlu/Global Facts": 0.49, - "helm_mmlu/Jurisprudence": 0.861, - "helm_mmlu/Philosophy": 0.865, - "helm_mmlu/Professional Psychology": 0.871, - "helm_mmlu/Us Foreign Policy": 0.94, - "helm_mmlu/Astronomy": 0.921, - "helm_mmlu/Business Ethics": 0.83, - "helm_mmlu/Clinical Knowledge": 0.845, - "helm_mmlu/Conceptual Physics": 0.838, - "helm_mmlu/Electrical Engineering": 0.766, - "helm_mmlu/Elementary Mathematics": 0.632, - "helm_mmlu/Formal Logic": 0.651, - "helm_mmlu/High School World History": 0.941, - "helm_mmlu/Human Sexuality": 0.878, - "helm_mmlu/International Law": 0.901, - "helm_mmlu/Logical Fallacies": 0.865, - "helm_mmlu/Machine Learning": 0.714, - "helm_mmlu/Management": 0.913, - "helm_mmlu/Marketing": 0.94, - "helm_mmlu/Medical Genetics": 0.89, - "helm_mmlu/Miscellaneous": 0.917, - "helm_mmlu/Moral Scenarios": 0.598, - "helm_mmlu/Nutrition": 0.876, - "helm_mmlu/Prehistory": 0.91, - "helm_mmlu/Public Relations": 0.727, - "helm_mmlu/Security Studies": 0.833, - "helm_mmlu/Sociology": 0.93, - "helm_mmlu/Virology": 0.59, - "helm_mmlu/World Religions": 0.906, - "helm_mmlu/Mean win rate": 0.524 - } - }, - { - "id": "meta/llama-3-8b", - "name": "Llama 3 8B", - "developer": "Meta", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.387, - "helm_lite/NarrativeQA": 0.754, - "helm_lite/NaturalQuestions (closed-book)": 0.378, - "helm_lite/OpenbookQA": 0.766, - "helm_lite/MMLU": 0.602, - "helm_lite/MATH": 0.391, - "helm_lite/GSM8K": 0.499, - "helm_lite/LegalBench": 0.637, - "helm_lite/MedQA": 0.581, - "helm_lite/WMT 2014": 0.183, - "helm_mmlu/MMLU All Subjects": 0.668, - "helm_mmlu/Abstract Algebra": 0.33, - "helm_mmlu/Anatomy": 0.696, - "helm_mmlu/College Physics": 0.451, - "helm_mmlu/Computer Security": 0.8, - "helm_mmlu/Econometrics": 0.518, - "helm_mmlu/Global Facts": 0.34, - "helm_mmlu/Jurisprudence": 0.741, - "helm_mmlu/Philosophy": 0.743, - "helm_mmlu/Professional Psychology": 0.711, - "helm_mmlu/Us Foreign Policy": 0.88, - "helm_mmlu/Astronomy": 0.711, - "helm_mmlu/Business Ethics": 0.65, - "helm_mmlu/Clinical Knowledge": 0.751, - "helm_mmlu/Conceptual Physics": 0.557, - "helm_mmlu/Electrical Engineering": 0.669, - "helm_mmlu/Elementary Mathematics": 0.426, - "helm_mmlu/Formal Logic": 0.468, - "helm_mmlu/High School World History": 0.823, - "helm_mmlu/Human Sexuality": 0.748, - "helm_mmlu/International Law": 0.843, - "helm_mmlu/Logical Fallacies": 0.755, - "helm_mmlu/Machine Learning": 0.545, - "helm_mmlu/Management": 0.874, - "helm_mmlu/Marketing": 0.885, - "helm_mmlu/Medical Genetics": 0.83, - "helm_mmlu/Miscellaneous": 0.831, - "helm_mmlu/Moral Scenarios": 0.416, - "helm_mmlu/Nutrition": 0.761, - "helm_mmlu/Prehistory": 0.738, - "helm_mmlu/Public Relations": 0.736, - "helm_mmlu/Security Studies": 0.771, - "helm_mmlu/Sociology": 0.866, - "helm_mmlu/Virology": 0.566, - "helm_mmlu/World Religions": 0.819, - "helm_mmlu/Mean win rate": 0.733 - } - }, - { - "id": "meta/llama-3.1-405b-instruct-turbo", - "name": "Llama 3.1 Instruct Turbo 405B", - "developer": "Meta", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.618, - "helm_capabilities/MMLU-Pro": 0.723, - "helm_capabilities/GPQA": 0.522, - "helm_capabilities/IFEval": 0.811, - "helm_capabilities/WildBench": 0.783, - "helm_capabilities/Omni-MATH": 0.249, - "helm_lite/Mean win rate": 0.854, - "helm_lite/NarrativeQA": 0.749, - "helm_lite/NaturalQuestions (closed-book)": 0.456, - "helm_lite/OpenbookQA": 0.94, - "helm_lite/MMLU": 0.759, - "helm_lite/MATH": 0.827, - "helm_lite/GSM8K": 0.949, - "helm_lite/LegalBench": 0.707, - "helm_lite/MedQA": 0.805, - "helm_lite/WMT 2014": 0.238, - "helm_mmlu/MMLU All Subjects": 0.845, - "helm_mmlu/Abstract Algebra": 0.7, - "helm_mmlu/Anatomy": 0.822, - "helm_mmlu/College Physics": 0.696, - "helm_mmlu/Computer Security": 0.81, - "helm_mmlu/Econometrics": 0.746, - "helm_mmlu/Global Facts": 0.71, - "helm_mmlu/Jurisprudence": 0.87, - "helm_mmlu/Philosophy": 0.878, - "helm_mmlu/Professional Psychology": 0.861, - "helm_mmlu/Us Foreign Policy": 0.94, - "helm_mmlu/Astronomy": 0.921, - "helm_mmlu/Business Ethics": 0.81, - "helm_mmlu/Clinical Knowledge": 0.879, - "helm_mmlu/Conceptual Physics": 0.877, - "helm_mmlu/Electrical Engineering": 0.821, - "helm_mmlu/Elementary Mathematics": 0.828, - "helm_mmlu/Formal Logic": 0.698, - "helm_mmlu/High School World History": 0.941, - "helm_mmlu/Human Sexuality": 0.855, - "helm_mmlu/International Law": 0.95, - "helm_mmlu/Logical Fallacies": 0.92, - "helm_mmlu/Machine Learning": 0.795, - "helm_mmlu/Management": 0.893, - "helm_mmlu/Marketing": 0.962, - "helm_mmlu/Medical Genetics": 0.93, - "helm_mmlu/Miscellaneous": 0.939, - "helm_mmlu/Moral Scenarios": 0.876, - "helm_mmlu/Nutrition": 0.928, - "helm_mmlu/Prehistory": 0.929, - "helm_mmlu/Public Relations": 0.818, - "helm_mmlu/Security Studies": 0.857, - "helm_mmlu/Sociology": 0.94, - "helm_mmlu/Virology": 0.572, - "helm_mmlu/World Religions": 0.906, - "helm_mmlu/Mean win rate": 0.33 - } - }, - { - "id": "meta/llama-3.1-70b-instruct-turbo", - "name": "Llama 3.1 Instruct Turbo 70B", - "developer": "Meta", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.574, - "helm_capabilities/MMLU-Pro": 0.653, - "helm_capabilities/GPQA": 0.426, - "helm_capabilities/IFEval": 0.821, - "helm_capabilities/WildBench": 0.758, - "helm_capabilities/Omni-MATH": 0.21, - "helm_lite/Mean win rate": 0.808, - "helm_lite/NarrativeQA": 0.772, - "helm_lite/NaturalQuestions (closed-book)": 0.452, - "helm_lite/OpenbookQA": 0.938, - "helm_lite/MMLU": 0.709, - "helm_lite/MATH": 0.783, - "helm_lite/GSM8K": 0.938, - "helm_lite/LegalBench": 0.687, - "helm_lite/MedQA": 0.769, - "helm_lite/WMT 2014": 0.223, - "helm_mmlu/MMLU All Subjects": 0.801, - "helm_mmlu/Abstract Algebra": 0.55, - "helm_mmlu/Anatomy": 0.8, - "helm_mmlu/College Physics": 0.559, - "helm_mmlu/Computer Security": 0.8, - "helm_mmlu/Econometrics": 0.675, - "helm_mmlu/Global Facts": 0.61, - "helm_mmlu/Jurisprudence": 0.889, - "helm_mmlu/Philosophy": 0.833, - "helm_mmlu/Professional Psychology": 0.846, - "helm_mmlu/Us Foreign Policy": 0.93, - "helm_mmlu/Astronomy": 0.908, - "helm_mmlu/Business Ethics": 0.72, - "helm_mmlu/Clinical Knowledge": 0.845, - "helm_mmlu/Conceptual Physics": 0.834, - "helm_mmlu/Electrical Engineering": 0.745, - "helm_mmlu/Elementary Mathematics": 0.701, - "helm_mmlu/Formal Logic": 0.675, - "helm_mmlu/High School World History": 0.937, - "helm_mmlu/Human Sexuality": 0.855, - "helm_mmlu/International Law": 0.926, - "helm_mmlu/Logical Fallacies": 0.84, - "helm_mmlu/Machine Learning": 0.696, - "helm_mmlu/Management": 0.913, - "helm_mmlu/Marketing": 0.936, - "helm_mmlu/Medical Genetics": 0.93, - "helm_mmlu/Miscellaneous": 0.913, - "helm_mmlu/Moral Scenarios": 0.834, - "helm_mmlu/Nutrition": 0.889, - "helm_mmlu/Prehistory": 0.88, - "helm_mmlu/Public Relations": 0.709, - "helm_mmlu/Security Studies": 0.849, - "helm_mmlu/Sociology": 0.92, - "helm_mmlu/Virology": 0.578, - "helm_mmlu/World Religions": 0.895, - "helm_mmlu/Mean win rate": 0.021 - } - }, - { - "id": "meta/llama-3.1-8b-instruct-turbo", - "name": "Llama 3.1 Instruct Turbo 8B", - "developer": "Meta", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.444, - "helm_capabilities/MMLU-Pro": 0.406, - "helm_capabilities/GPQA": 0.247, - "helm_capabilities/IFEval": 0.743, - "helm_capabilities/WildBench": 0.686, - "helm_capabilities/Omni-MATH": 0.137, - "helm_lite/Mean win rate": 0.303, - "helm_lite/NarrativeQA": 0.756, - "helm_lite/NaturalQuestions (closed-book)": 0.209, - "helm_lite/OpenbookQA": 0.74, - "helm_lite/MMLU": 0.5, - "helm_lite/MATH": 0.703, - "helm_lite/GSM8K": 0.798, - "helm_lite/LegalBench": 0.342, - "helm_lite/MedQA": 0.245, - "helm_lite/WMT 2014": 0.181, - "helm_mmlu/MMLU All Subjects": 0.561, - "helm_mmlu/Abstract Algebra": 0.26, - "helm_mmlu/Anatomy": 0.459, - "helm_mmlu/College Physics": 0.363, - "helm_mmlu/Computer Security": 0.71, - "helm_mmlu/Econometrics": 0.351, - "helm_mmlu/Global Facts": 0.26, - "helm_mmlu/Jurisprudence": 0.731, - "helm_mmlu/Philosophy": 0.64, - "helm_mmlu/Professional Psychology": 0.649, - "helm_mmlu/Us Foreign Policy": 0.79, - "helm_mmlu/Astronomy": 0.645, - "helm_mmlu/Business Ethics": 0.65, - "helm_mmlu/Clinical Knowledge": 0.615, - "helm_mmlu/Conceptual Physics": 0.528, - "helm_mmlu/Electrical Engineering": 0.441, - "helm_mmlu/Elementary Mathematics": 0.429, - "helm_mmlu/Formal Logic": 0.444, - "helm_mmlu/High School World History": 0.515, - "helm_mmlu/Human Sexuality": 0.733, - "helm_mmlu/International Law": 0.694, - "helm_mmlu/Logical Fallacies": 0.742, - "helm_mmlu/Machine Learning": 0.384, - "helm_mmlu/Management": 0.709, - "helm_mmlu/Marketing": 0.833, - "helm_mmlu/Medical Genetics": 0.66, - "helm_mmlu/Miscellaneous": 0.653, - "helm_mmlu/Moral Scenarios": 0.368, - "helm_mmlu/Nutrition": 0.712, - "helm_mmlu/Prehistory": 0.728, - "helm_mmlu/Public Relations": 0.664, - "helm_mmlu/Security Studies": 0.576, - "helm_mmlu/Sociology": 0.701, - "helm_mmlu/Virology": 0.446, - "helm_mmlu/World Religions": 0.789, - "helm_mmlu/Mean win rate": 0.475 - } - }, - { - "id": "meta/llama-3.2-11b-vision-instruct-turbo", - "name": "Llama 3.2 Vision Instruct Turbo 11B", - "developer": "Meta", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.325, - "helm_lite/NarrativeQA": 0.756, - "helm_lite/NaturalQuestions (closed-book)": 0.234, - "helm_lite/OpenbookQA": 0.724, - "helm_lite/MMLU": 0.511, - "helm_lite/MATH": 0.739, - "helm_lite/GSM8K": 0.823, - "helm_lite/LegalBench": 0.435, - "helm_lite/MedQA": 0.27, - "helm_lite/WMT 2014": 0.179, - "helm_mmlu/MMLU All Subjects": 0.565, - "helm_mmlu/Abstract Algebra": 0.28, - "helm_mmlu/Anatomy": 0.533, - "helm_mmlu/College Physics": 0.333, - "helm_mmlu/Computer Security": 0.71, - "helm_mmlu/Econometrics": 0.395, - "helm_mmlu/Global Facts": 0.25, - "helm_mmlu/Jurisprudence": 0.722, - "helm_mmlu/Philosophy": 0.646, - "helm_mmlu/Professional Psychology": 0.649, - "helm_mmlu/Us Foreign Policy": 0.78, - "helm_mmlu/Astronomy": 0.671, - "helm_mmlu/Business Ethics": 0.64, - "helm_mmlu/Clinical Knowledge": 0.638, - "helm_mmlu/Conceptual Physics": 0.536, - "helm_mmlu/Electrical Engineering": 0.51, - "helm_mmlu/Elementary Mathematics": 0.458, - "helm_mmlu/Formal Logic": 0.46, - "helm_mmlu/High School World History": 0.502, - "helm_mmlu/Human Sexuality": 0.763, - "helm_mmlu/International Law": 0.711, - "helm_mmlu/Logical Fallacies": 0.742, - "helm_mmlu/Machine Learning": 0.375, - "helm_mmlu/Management": 0.728, - "helm_mmlu/Marketing": 0.838, - "helm_mmlu/Medical Genetics": 0.7, - "helm_mmlu/Miscellaneous": 0.644, - "helm_mmlu/Moral Scenarios": 0.328, - "helm_mmlu/Nutrition": 0.752, - "helm_mmlu/Prehistory": 0.744, - "helm_mmlu/Public Relations": 0.645, - "helm_mmlu/Security Studies": 0.567, - "helm_mmlu/Sociology": 0.627, - "helm_mmlu/Virology": 0.446, - "helm_mmlu/World Religions": 0.696, - "helm_mmlu/Mean win rate": 0.897 - } - }, - { - "id": "meta/llama-3.2-90b-vision-instruct-turbo", - "name": "Llama 3.2 Vision Instruct Turbo 90B", - "developer": "Meta", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.819, - "helm_lite/NarrativeQA": 0.777, - "helm_lite/NaturalQuestions (closed-book)": 0.457, - "helm_lite/OpenbookQA": 0.942, - "helm_lite/MMLU": 0.703, - "helm_lite/MATH": 0.791, - "helm_lite/GSM8K": 0.936, - "helm_lite/LegalBench": 0.68, - "helm_lite/MedQA": 0.769, - "helm_lite/WMT 2014": 0.224, - "helm_mmlu/MMLU All Subjects": 0.803, - "helm_mmlu/Abstract Algebra": 0.52, - "helm_mmlu/Anatomy": 0.8, - "helm_mmlu/College Physics": 0.539, - "helm_mmlu/Computer Security": 0.81, - "helm_mmlu/Econometrics": 0.684, - "helm_mmlu/Global Facts": 0.6, - "helm_mmlu/Jurisprudence": 0.88, - "helm_mmlu/Philosophy": 0.839, - "helm_mmlu/Professional Psychology": 0.843, - "helm_mmlu/Us Foreign Policy": 0.93, - "helm_mmlu/Astronomy": 0.921, - "helm_mmlu/Business Ethics": 0.76, - "helm_mmlu/Clinical Knowledge": 0.845, - "helm_mmlu/Conceptual Physics": 0.826, - "helm_mmlu/Electrical Engineering": 0.759, - "helm_mmlu/Elementary Mathematics": 0.688, - "helm_mmlu/Formal Logic": 0.683, - "helm_mmlu/High School World History": 0.941, - "helm_mmlu/Human Sexuality": 0.87, - "helm_mmlu/International Law": 0.934, - "helm_mmlu/Logical Fallacies": 0.834, - "helm_mmlu/Machine Learning": 0.688, - "helm_mmlu/Management": 0.913, - "helm_mmlu/Marketing": 0.944, - "helm_mmlu/Medical Genetics": 0.92, - "helm_mmlu/Miscellaneous": 0.913, - "helm_mmlu/Moral Scenarios": 0.841, - "helm_mmlu/Nutrition": 0.889, - "helm_mmlu/Prehistory": 0.886, - "helm_mmlu/Public Relations": 0.718, - "helm_mmlu/Security Studies": 0.853, - "helm_mmlu/Sociology": 0.92, - "helm_mmlu/Virology": 0.584, - "helm_mmlu/World Religions": 0.901, - "helm_mmlu/Mean win rate": 0.773 - } - }, - { - "id": "meta/llama-3.3-70b-instruct-turbo", - "name": "Llama 3.3 Instruct Turbo 70B", - "developer": "Meta", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.812, - "helm_lite/NarrativeQA": 0.791, - "helm_lite/NaturalQuestions (closed-book)": 0.431, - "helm_lite/OpenbookQA": 0.928, - "helm_lite/MMLU": 0.7, - "helm_lite/MATH": 0.808, - "helm_lite/GSM8K": 0.942, - "helm_lite/LegalBench": 0.725, - "helm_lite/MedQA": 0.761, - "helm_lite/WMT 2014": 0.219, - "helm_mmlu/MMLU All Subjects": 0.791, - "helm_mmlu/Abstract Algebra": 0.5, - "helm_mmlu/Anatomy": 0.778, - "helm_mmlu/College Physics": 0.52, - "helm_mmlu/Computer Security": 0.8, - "helm_mmlu/Econometrics": 0.719, - "helm_mmlu/Global Facts": 0.58, - "helm_mmlu/Jurisprudence": 0.87, - "helm_mmlu/Philosophy": 0.83, - "helm_mmlu/Professional Psychology": 0.845, - "helm_mmlu/Us Foreign Policy": 0.93, - "helm_mmlu/Astronomy": 0.888, - "helm_mmlu/Business Ethics": 0.8, - "helm_mmlu/Clinical Knowledge": 0.83, - "helm_mmlu/Conceptual Physics": 0.821, - "helm_mmlu/Electrical Engineering": 0.745, - "helm_mmlu/Elementary Mathematics": 0.672, - "helm_mmlu/Formal Logic": 0.675, - "helm_mmlu/High School World History": 0.907, - "helm_mmlu/Human Sexuality": 0.855, - "helm_mmlu/International Law": 0.884, - "helm_mmlu/Logical Fallacies": 0.816, - "helm_mmlu/Machine Learning": 0.714, - "helm_mmlu/Management": 0.903, - "helm_mmlu/Marketing": 0.927, - "helm_mmlu/Medical Genetics": 0.9, - "helm_mmlu/Miscellaneous": 0.914, - "helm_mmlu/Moral Scenarios": 0.698, - "helm_mmlu/Nutrition": 0.882, - "helm_mmlu/Prehistory": 0.895, - "helm_mmlu/Public Relations": 0.727, - "helm_mmlu/Security Studies": 0.845, - "helm_mmlu/Sociology": 0.92, - "helm_mmlu/Virology": 0.566, - "helm_mmlu/World Religions": 0.883, - "helm_mmlu/Mean win rate": 0.722 - } - }, - { - "id": "meta/LLaMA-30B", - "name": "LLaMA 30B", - "developer": "Meta", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.781, - "helm_classic/MMLU": 0.531, - "helm_classic/BoolQ": 0.861, - "helm_classic/NarrativeQA": 0.752, - "helm_classic/NaturalQuestions (open-book)": 0.666, - "helm_classic/QuAC": 0.39, - "helm_classic/HellaSwag": -1.0, - "helm_classic/OpenbookQA": -1.0, - "helm_classic/TruthfulQA": 0.344, - "helm_classic/MS MARCO (TREC)": -1.0, - "helm_classic/CNN/DailyMail": -1.0, - "helm_classic/XSUM": -1.0, - "helm_classic/IMDB": 0.927, - "helm_classic/CivilComments": 0.549, - "helm_classic/RAFT": 0.752 - } - }, - { - "id": "meta/llama-4-maverick", - "name": "meta/llama-4-maverick", - "developer": "Meta", - "evaluator_relationship": null, - "benchmark_scores": { - "livecodebenchpro/Hard Problems": 0.0, - "livecodebenchpro/Medium Problems": 0.0, - "livecodebenchpro/Easy Problems": 0.09859154929577464 - } - }, - { - "id": "meta/llama-4-maverick-17b-128e-instruct-fp8", - "name": "Llama 4 Maverick 17Bx128E Instruct FP8", - "developer": "Meta", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.718, - "helm_capabilities/MMLU-Pro": 0.81, - "helm_capabilities/GPQA": 0.65, - "helm_capabilities/IFEval": 0.908, - "helm_capabilities/WildBench": 0.8, - "helm_capabilities/Omni-MATH": 0.422 - } - }, - { - "id": "meta/llama-4-maverick-17b-128e-instruct-fp8-fc", - "name": "Llama-4-Maverick-17B-128E-Instruct-FP8 (FC)", - "developer": "Meta", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 50.0, - "bfcl/bfcl.overall.overall_accuracy": 37.29, - "bfcl/bfcl.overall.total_cost_usd": 18.25, - "bfcl/bfcl.overall.latency_mean_s": 18.43, - "bfcl/bfcl.overall.latency_std_s": 34.11, - "bfcl/bfcl.overall.latency_p95_s": 102.75, - "bfcl/bfcl.non_live.ast_accuracy": 88.65, - "bfcl/bfcl.non_live.simple_ast_accuracy": 77.08, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 95.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 94.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 88.5, - "bfcl/bfcl.live.live_accuracy": 73.65, - "bfcl/bfcl.live.live_simple_ast_accuracy": 84.5, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 71.04, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 75.0, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 70.83, - "bfcl/bfcl.multi_turn.accuracy": 20.25, - "bfcl/bfcl.multi_turn.base_accuracy": 27.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 22.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 14.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 18.0, - "bfcl/bfcl.web_search.accuracy": 28.0, - "bfcl/bfcl.web_search.base_accuracy": 39.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 17.0, - "bfcl/bfcl.memory.accuracy": 18.92, - "bfcl/bfcl.memory.kv_accuracy": 8.39, - "bfcl/bfcl.memory.vector_accuracy": 32.9, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 15.48, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 100.0, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 55.97 - } - }, - { - "id": "meta/llama-4-scout-17b-16e-instruct", - "name": "Llama 4 Scout 17Bx16E Instruct", - "developer": "Meta", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.644, - "helm_capabilities/MMLU-Pro": 0.742, - "helm_capabilities/GPQA": 0.507, - "helm_capabilities/IFEval": 0.818, - "helm_capabilities/WildBench": 0.779, - "helm_capabilities/Omni-MATH": 0.373 - } - }, - { - "id": "meta/llama-4-scout-17b-16e-instruct-fc", - "name": "Llama-4-Scout-17B-16E-Instruct (FC)", - "developer": "Meta", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 72.0, - "bfcl/bfcl.overall.overall_accuracy": 28.13, - "bfcl/bfcl.overall.total_cost_usd": 24.68, - "bfcl/bfcl.overall.latency_mean_s": 17.86, - "bfcl/bfcl.overall.latency_std_s": 50.68, - "bfcl/bfcl.overall.latency_p95_s": 166.2, - "bfcl/bfcl.non_live.ast_accuracy": 89.38, - "bfcl/bfcl.non_live.simple_ast_accuracy": 79.0, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 94.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 94.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 90.5, - "bfcl/bfcl.live.live_accuracy": 74.69, - "bfcl/bfcl.live.live_simple_ast_accuracy": 81.78, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 72.74, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 81.25, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 79.17, - "bfcl/bfcl.multi_turn.accuracy": 9.0, - "bfcl/bfcl.multi_turn.base_accuracy": 12.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 7.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 7.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 9.5, - "bfcl/bfcl.web_search.accuracy": 14.5, - "bfcl/bfcl.web_search.base_accuracy": 18.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 11.0, - "bfcl/bfcl.memory.accuracy": 8.17, - "bfcl/bfcl.memory.kv_accuracy": 2.58, - "bfcl/bfcl.memory.vector_accuracy": 2.58, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 19.35, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 100.0, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 44.92 - } - }, - { - "id": "meta/LLaMA-65B", - "name": "LLaMA 65B", - "developer": "Meta", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.908, - "helm_classic/MMLU": 0.584, - "helm_classic/BoolQ": 0.871, - "helm_classic/NarrativeQA": 0.755, - "helm_classic/NaturalQuestions (open-book)": 0.672, - "helm_classic/QuAC": 0.401, - "helm_classic/HellaSwag": -1.0, - "helm_classic/OpenbookQA": -1.0, - "helm_classic/TruthfulQA": 0.508, - "helm_classic/MS MARCO (TREC)": -1.0, - "helm_classic/CNN/DailyMail": -1.0, - "helm_classic/XSUM": -1.0, - "helm_classic/IMDB": 0.962, - "helm_classic/CivilComments": 0.655, - "helm_classic/RAFT": 0.702, - "helm_lite/Mean win rate": 0.345, - "helm_lite/NarrativeQA": 0.755, - "helm_lite/NaturalQuestions (closed-book)": 0.433, - "helm_lite/OpenbookQA": 0.754, - "helm_lite/MMLU": 0.584, - "helm_lite/MATH": 0.257, - "helm_lite/GSM8K": 0.489, - "helm_lite/LegalBench": 0.48, - "helm_lite/MedQA": 0.507, - "helm_lite/WMT 2014": 0.189 - } - }, - { - "id": "meta/LLaMA-7B", - "name": "LLaMA 7B", - "developer": "Meta", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.533, - "helm_classic/MMLU": 0.321, - "helm_classic/BoolQ": 0.756, - "helm_classic/NarrativeQA": 0.669, - "helm_classic/NaturalQuestions (open-book)": 0.589, - "helm_classic/QuAC": 0.338, - "helm_classic/HellaSwag": -1.0, - "helm_classic/OpenbookQA": -1.0, - "helm_classic/TruthfulQA": 0.28, - "helm_classic/MS MARCO (TREC)": -1.0, - "helm_classic/CNN/DailyMail": -1.0, - "helm_classic/XSUM": -1.0, - "helm_classic/IMDB": 0.947, - "helm_classic/CivilComments": 0.563, - "helm_classic/RAFT": 0.573 - } - }, - { - "id": "meta/OPT-175B", - "name": "OPT 175B", - "developer": "Meta", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.609, - "helm_classic/MMLU": 0.318, - "helm_classic/BoolQ": 0.793, - "helm_classic/NarrativeQA": 0.671, - "helm_classic/NaturalQuestions (open-book)": 0.615, - "helm_classic/QuAC": 0.36, - "helm_classic/HellaSwag": 0.791, - "helm_classic/OpenbookQA": 0.586, - "helm_classic/TruthfulQA": 0.25, - "helm_classic/MS MARCO (TREC)": 0.448, - "helm_classic/CNN/DailyMail": 0.146, - "helm_classic/XSUM": 0.155, - "helm_classic/IMDB": 0.947, - "helm_classic/CivilComments": 0.505, - "helm_classic/RAFT": 0.606 - } - }, - { - "id": "meta/OPT-66B", - "name": "OPT 66B", - "developer": "Meta", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.448, - "helm_classic/MMLU": 0.276, - "helm_classic/BoolQ": 0.76, - "helm_classic/NarrativeQA": 0.638, - "helm_classic/NaturalQuestions (open-book)": 0.596, - "helm_classic/QuAC": 0.357, - "helm_classic/HellaSwag": 0.745, - "helm_classic/OpenbookQA": 0.534, - "helm_classic/TruthfulQA": 0.201, - "helm_classic/MS MARCO (TREC)": 0.482, - "helm_classic/CNN/DailyMail": 0.136, - "helm_classic/XSUM": 0.126, - "helm_classic/IMDB": 0.917, - "helm_classic/CivilComments": 0.506, - "helm_classic/RAFT": 0.557 - } - }, - { - "id": "mhl1/Qwen2.5-0.5B-cinstruct-stage1", - "name": "Qwen2.5-0.5B-cinstruct-stage1", - "developer": "mhl1", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1482, - "hfopenllm_v2/BBH": 0.3256, - "hfopenllm_v2/MATH Level 5": 0.0128, - "hfopenllm_v2/GPQA": 0.2651, - "hfopenllm_v2/MUSR": 0.35, - "hfopenllm_v2/MMLU-PRO": 0.1139 - } - }, - { - "id": "microsoft/DialoGPT-medium", - "name": "DialoGPT-medium", - "developer": "microsoft", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1479, - "hfopenllm_v2/BBH": 0.3014, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2542, - "hfopenllm_v2/MUSR": 0.4287, - "hfopenllm_v2/MMLU-PRO": 0.1119 - } - }, - { - "id": "microsoft/Orca-2-13b", - "name": "Orca-2-13b", - "developer": "microsoft", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3128, - "hfopenllm_v2/BBH": 0.4884, - "hfopenllm_v2/MATH Level 5": 0.0317, - "hfopenllm_v2/GPQA": 0.2802, - "hfopenllm_v2/MUSR": 0.513, - "hfopenllm_v2/MMLU-PRO": 0.2749 - } - }, - { - "id": "microsoft/Orca-2-7b", - "name": "Orca-2-7b", - "developer": "microsoft", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2183, - "hfopenllm_v2/BBH": 0.4452, - "hfopenllm_v2/MATH Level 5": 0.0196, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.5026, - "hfopenllm_v2/MMLU-PRO": 0.2319 - } - }, - { - "id": "microsoft/phi-1", - "name": "phi-1", - "developer": "microsoft", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2068, - "hfopenllm_v2/BBH": 0.3139, - "hfopenllm_v2/MATH Level 5": 0.0098, - "hfopenllm_v2/GPQA": 0.2651, - "hfopenllm_v2/MUSR": 0.3525, - "hfopenllm_v2/MMLU-PRO": 0.1162 - } - }, - { - "id": "microsoft/phi-1_5", - "name": "phi-1_5", - "developer": "microsoft", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2033, - "hfopenllm_v2/BBH": 0.336, - "hfopenllm_v2/MATH Level 5": 0.0181, - "hfopenllm_v2/GPQA": 0.2676, - "hfopenllm_v2/MUSR": 0.3404, - "hfopenllm_v2/MMLU-PRO": 0.1691 - } - }, - { - "id": "microsoft/phi-2", - "name": "Phi-2", - "developer": "microsoft", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.169, - "helm_lite/NarrativeQA": 0.703, - "helm_lite/NaturalQuestions (closed-book)": 0.155, - "helm_lite/OpenbookQA": 0.798, - "helm_lite/MMLU": 0.518, - "helm_lite/MATH": 0.255, - "helm_lite/GSM8K": 0.581, - "helm_lite/LegalBench": 0.334, - "helm_lite/MedQA": 0.41, - "helm_lite/WMT 2014": 0.038, - "helm_mmlu/MMLU All Subjects": 0.584, - "helm_mmlu/Abstract Algebra": 0.31, - "helm_mmlu/Anatomy": 0.437, - "helm_mmlu/College Physics": 0.382, - "helm_mmlu/Computer Security": 0.73, - "helm_mmlu/Econometrics": 0.342, - "helm_mmlu/Global Facts": 0.35, - "helm_mmlu/Jurisprudence": 0.694, - "helm_mmlu/Philosophy": 0.598, - "helm_mmlu/Professional Psychology": 0.572, - "helm_mmlu/Us Foreign Policy": 0.78, - "helm_mmlu/Astronomy": 0.605, - "helm_mmlu/Business Ethics": 0.59, - "helm_mmlu/Clinical Knowledge": 0.619, - "helm_mmlu/Conceptual Physics": 0.519, - "helm_mmlu/Electrical Engineering": 0.545, - "helm_mmlu/Elementary Mathematics": 0.463, - "helm_mmlu/Formal Logic": 0.389, - "helm_mmlu/High School World History": 0.73, - "helm_mmlu/Human Sexuality": 0.733, - "helm_mmlu/International Law": 0.752, - "helm_mmlu/Logical Fallacies": 0.767, - "helm_mmlu/Machine Learning": 0.5, - "helm_mmlu/Management": 0.748, - "helm_mmlu/Marketing": 0.833, - "helm_mmlu/Medical Genetics": 0.62, - "helm_mmlu/Miscellaneous": 0.688, - "helm_mmlu/Moral Scenarios": 0.231, - "helm_mmlu/Nutrition": 0.627, - "helm_mmlu/Prehistory": 0.605, - "helm_mmlu/Public Relations": 0.673, - "helm_mmlu/Security Studies": 0.702, - "helm_mmlu/Sociology": 0.816, - "helm_mmlu/Virology": 0.47, - "helm_mmlu/World Religions": 0.702, - "helm_mmlu/Mean win rate": 0.824, - "hfopenllm_v2/IFEval": 0.2739, - "hfopenllm_v2/BBH": 0.4881, - "hfopenllm_v2/MATH Level 5": 0.0295, - "hfopenllm_v2/GPQA": 0.2718, - "hfopenllm_v2/MUSR": 0.4099, - "hfopenllm_v2/MMLU-PRO": 0.2628 - } - }, - { - "id": "microsoft/Phi-3-medium-128k-instruct", - "name": "Phi-3-medium-128k-instruct", - "developer": "microsoft", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.604, - "hfopenllm_v2/BBH": 0.6382, - "hfopenllm_v2/MATH Level 5": 0.1918, - "hfopenllm_v2/GPQA": 0.3364, - "hfopenllm_v2/MUSR": 0.4129, - "hfopenllm_v2/MMLU-PRO": 0.4712 - } - }, - { - "id": "microsoft/phi-3-medium-4k-instruct", - "name": "Phi-3 14B", - "developer": "microsoft", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.509, - "helm_lite/NarrativeQA": 0.724, - "helm_lite/NaturalQuestions (closed-book)": 0.278, - "helm_lite/OpenbookQA": 0.916, - "helm_lite/MMLU": 0.675, - "helm_lite/MATH": 0.611, - "helm_lite/GSM8K": 0.878, - "helm_lite/LegalBench": 0.593, - "helm_lite/MedQA": 0.696, - "helm_lite/WMT 2014": 0.17, - "helm_mmlu/MMLU All Subjects": 0.775, - "helm_mmlu/Abstract Algebra": 0.5, - "helm_mmlu/Anatomy": 0.719, - "helm_mmlu/College Physics": 0.529, - "helm_mmlu/Computer Security": 0.79, - "helm_mmlu/Econometrics": 0.614, - "helm_mmlu/Global Facts": 0.5, - "helm_mmlu/Jurisprudence": 0.88, - "helm_mmlu/Philosophy": 0.804, - "helm_mmlu/Professional Psychology": 0.835, - "helm_mmlu/Us Foreign Policy": 0.95, - "helm_mmlu/Astronomy": 0.849, - "helm_mmlu/Business Ethics": 0.8, - "helm_mmlu/Clinical Knowledge": 0.826, - "helm_mmlu/Conceptual Physics": 0.809, - "helm_mmlu/Electrical Engineering": 0.683, - "helm_mmlu/Elementary Mathematics": 0.709, - "helm_mmlu/Formal Logic": 0.587, - "helm_mmlu/High School World History": 0.903, - "helm_mmlu/Human Sexuality": 0.863, - "helm_mmlu/International Law": 0.934, - "helm_mmlu/Logical Fallacies": 0.828, - "helm_mmlu/Machine Learning": 0.696, - "helm_mmlu/Management": 0.864, - "helm_mmlu/Marketing": 0.919, - "helm_mmlu/Medical Genetics": 0.91, - "helm_mmlu/Miscellaneous": 0.894, - "helm_mmlu/Moral Scenarios": 0.639, - "helm_mmlu/Nutrition": 0.837, - "helm_mmlu/Prehistory": 0.867, - "helm_mmlu/Public Relations": 0.755, - "helm_mmlu/Security Studies": 0.829, - "helm_mmlu/Sociology": 0.891, - "helm_mmlu/Virology": 0.554, - "helm_mmlu/World Religions": 0.865, - "helm_mmlu/Mean win rate": 0.015, - "hfopenllm_v2/IFEval": 0.6423, - "hfopenllm_v2/BBH": 0.6412, - "hfopenllm_v2/MATH Level 5": 0.1956, - "hfopenllm_v2/GPQA": 0.3364, - "hfopenllm_v2/MUSR": 0.4258, - "hfopenllm_v2/MMLU-PRO": 0.4676 - } - }, - { - "id": "microsoft/Phi-3-mini-128k-instruct", - "name": "Phi-3-mini-128k-instruct", - "developer": "microsoft", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5976, - "hfopenllm_v2/BBH": 0.5575, - "hfopenllm_v2/MATH Level 5": 0.1405, - "hfopenllm_v2/GPQA": 0.318, - "hfopenllm_v2/MUSR": 0.3937, - "hfopenllm_v2/MMLU-PRO": 0.3734 - } - }, - { - "id": "microsoft/Phi-3-mini-4k-instruct", - "name": "Phi-3-mini-4k-instruct", - "developer": "microsoft", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5613, - "hfopenllm_v2/BBH": 0.5676, - "hfopenllm_v2/MATH Level 5": 0.1163, - "hfopenllm_v2/GPQA": 0.3196, - "hfopenllm_v2/MUSR": 0.395, - "hfopenllm_v2/MMLU-PRO": 0.3866 - } - }, - { - "id": "microsoft/Phi-3-small-128k-instruct", - "name": "Phi-3-small-128k-instruct", - "developer": "microsoft", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6368, - "hfopenllm_v2/BBH": 0.6202, - "hfopenllm_v2/MATH Level 5": 0.2026, - "hfopenllm_v2/GPQA": 0.3171, - "hfopenllm_v2/MUSR": 0.4378, - "hfopenllm_v2/MMLU-PRO": 0.4491 - } - }, - { - "id": "microsoft/phi-3-small-8k-instruct", - "name": "Phi-3 7B", - "developer": "microsoft", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.473, - "helm_lite/NarrativeQA": 0.754, - "helm_lite/NaturalQuestions (closed-book)": 0.324, - "helm_lite/OpenbookQA": 0.912, - "helm_lite/MMLU": 0.659, - "helm_lite/MATH": 0.703, - "helm_lite/GSM8K": -1.0, - "helm_lite/LegalBench": 0.584, - "helm_lite/MedQA": 0.672, - "helm_lite/WMT 2014": 0.154, - "helm_mmlu/MMLU All Subjects": 0.757, - "helm_mmlu/Abstract Algebra": 0.44, - "helm_mmlu/Anatomy": 0.726, - "helm_mmlu/College Physics": 0.559, - "helm_mmlu/Computer Security": 0.77, - "helm_mmlu/Econometrics": 0.596, - "helm_mmlu/Global Facts": 0.52, - "helm_mmlu/Jurisprudence": 0.843, - "helm_mmlu/Philosophy": 0.82, - "helm_mmlu/Professional Psychology": 0.835, - "helm_mmlu/Us Foreign Policy": 0.95, - "helm_mmlu/Astronomy": 0.849, - "helm_mmlu/Business Ethics": 0.77, - "helm_mmlu/Clinical Knowledge": 0.83, - "helm_mmlu/Conceptual Physics": 0.779, - "helm_mmlu/Electrical Engineering": 0.69, - "helm_mmlu/Elementary Mathematics": 0.619, - "helm_mmlu/Formal Logic": 0.595, - "helm_mmlu/High School World History": 0.848, - "helm_mmlu/Human Sexuality": 0.817, - "helm_mmlu/International Law": 0.851, - "helm_mmlu/Logical Fallacies": 0.81, - "helm_mmlu/Machine Learning": 0.652, - "helm_mmlu/Management": 0.903, - "helm_mmlu/Marketing": 0.897, - "helm_mmlu/Medical Genetics": 0.84, - "helm_mmlu/Miscellaneous": 0.871, - "helm_mmlu/Moral Scenarios": 0.711, - "helm_mmlu/Nutrition": 0.833, - "helm_mmlu/Prehistory": 0.858, - "helm_mmlu/Public Relations": 0.727, - "helm_mmlu/Security Studies": 0.804, - "helm_mmlu/Sociology": 0.886, - "helm_mmlu/Virology": 0.548, - "helm_mmlu/World Religions": 0.825, - "helm_mmlu/Mean win rate": 0.708, - "hfopenllm_v2/IFEval": 0.6497, - "hfopenllm_v2/BBH": 0.6208, - "hfopenllm_v2/MATH Level 5": 0.1887, - "hfopenllm_v2/GPQA": 0.3121, - "hfopenllm_v2/MUSR": 0.4558, - "hfopenllm_v2/MMLU-PRO": 0.4506 - } - }, - { - "id": "microsoft/Phi-3.5-mini-instruct", - "name": "Phi-3.5-mini-instruct", - "developer": "microsoft", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5775, - "hfopenllm_v2/BBH": 0.5518, - "hfopenllm_v2/MATH Level 5": 0.1964, - "hfopenllm_v2/GPQA": 0.3398, - "hfopenllm_v2/MUSR": 0.4021, - "hfopenllm_v2/MMLU-PRO": 0.3962 - } - }, - { - "id": "microsoft/Phi-3.5-MoE-instruct", - "name": "Phi-3.5-MoE-instruct", - "developer": "microsoft", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6925, - "hfopenllm_v2/BBH": 0.6408, - "hfopenllm_v2/MATH Level 5": 0.3119, - "hfopenllm_v2/GPQA": 0.3557, - "hfopenllm_v2/MUSR": 0.4565, - "hfopenllm_v2/MMLU-PRO": 0.4658 - } - }, - { - "id": "microsoft/phi-4", - "name": "phi-4", - "developer": "microsoft", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0585, - "hfopenllm_v2/BBH": 0.6691, - "hfopenllm_v2/MATH Level 5": 0.3165, - "hfopenllm_v2/GPQA": 0.406, - "hfopenllm_v2/MUSR": 0.5034, - "hfopenllm_v2/MMLU-PRO": 0.5287 - } - }, - { - "id": "microsoft/Phi-4-mini-instruct", - "name": "Phi-4-mini-instruct", - "developer": "microsoft", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7378, - "hfopenllm_v2/BBH": 0.5689, - "hfopenllm_v2/MATH Level 5": 0.1699, - "hfopenllm_v2/GPQA": 0.3096, - "hfopenllm_v2/MUSR": 0.3873, - "hfopenllm_v2/MMLU-PRO": 0.3932 - } - }, - { - "id": "microsoft/phi-4-prompt", - "name": "Phi-4 (Prompt)", - "developer": "microsoft", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 70.0, - "bfcl/bfcl.overall.overall_accuracy": 28.79, - "bfcl/bfcl.overall.total_cost_usd": 8.72, - "bfcl/bfcl.overall.latency_mean_s": 9.49, - "bfcl/bfcl.overall.latency_std_s": 26.73, - "bfcl/bfcl.overall.latency_p95_s": 23.02, - "bfcl/bfcl.non_live.ast_accuracy": 69.56, - "bfcl/bfcl.non_live.simple_ast_accuracy": 74.25, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 89.5, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 65.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 49.5, - "bfcl/bfcl.live.live_accuracy": 60.7, - "bfcl/bfcl.live.live_simple_ast_accuracy": 65.5, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 59.64, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 81.25, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 41.67, - "bfcl/bfcl.multi_turn.accuracy": 3.88, - "bfcl/bfcl.multi_turn.base_accuracy": 9.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 0.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 3.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 3.0, - "bfcl/bfcl.web_search.accuracy": 4.5, - "bfcl/bfcl.web_search.base_accuracy": 4.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 5.0, - "bfcl/bfcl.memory.accuracy": 24.73, - "bfcl/bfcl.memory.kv_accuracy": 17.42, - "bfcl/bfcl.memory.vector_accuracy": 25.16, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 31.61, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 50.0, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 87.55, - "bfcl/bfcl.format_sensitivity.max_delta": 81.5, - "bfcl/bfcl.format_sensitivity.stddev": 23.34 - } - }, - { - "id": "microsoft/TNLG-v2-530B", - "name": "TNLG v2 530B", - "developer": "microsoft", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.787, - "helm_classic/MMLU": 0.469, - "helm_classic/BoolQ": 0.809, - "helm_classic/NarrativeQA": 0.722, - "helm_classic/NaturalQuestions (open-book)": 0.642, - "helm_classic/QuAC": 0.39, - "helm_classic/HellaSwag": 0.799, - "helm_classic/OpenbookQA": 0.562, - "helm_classic/TruthfulQA": 0.251, - "helm_classic/MS MARCO (TREC)": 0.643, - "helm_classic/CNN/DailyMail": 0.161, - "helm_classic/XSUM": 0.169, - "helm_classic/IMDB": 0.941, - "helm_classic/CivilComments": 0.601, - "helm_classic/RAFT": 0.679 - } - }, - { - "id": "microsoft/TNLG-v2-6.7B", - "name": "TNLG v2 6.7B", - "developer": "microsoft", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.309, - "helm_classic/MMLU": 0.242, - "helm_classic/BoolQ": 0.698, - "helm_classic/NarrativeQA": 0.631, - "helm_classic/NaturalQuestions (open-book)": 0.561, - "helm_classic/QuAC": 0.345, - "helm_classic/HellaSwag": 0.704, - "helm_classic/OpenbookQA": 0.478, - "helm_classic/TruthfulQA": 0.167, - "helm_classic/MS MARCO (TREC)": 0.332, - "helm_classic/CNN/DailyMail": 0.146, - "helm_classic/XSUM": 0.11, - "helm_classic/IMDB": 0.927, - "helm_classic/CivilComments": 0.532, - "helm_classic/RAFT": 0.525 - } - }, - { - "id": "mightbe/Better-PairRM", - "name": "mightbe/Better-PairRM", - "developer": "mightbe", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.673, - "reward-bench/Chat": 0.9553, - "reward-bench/Chat Hard": 0.3925, - "reward-bench/Safety": 0.8203, - "reward-bench/Reasoning": 0.4983, - "reward-bench/Prior Sets (0.5 weight)": 0.724 - } - }, - { - "id": "migtissera/Llama-3-70B-Synthia-v3.5", - "name": "Llama-3-70B-Synthia-v3.5", - "developer": "migtissera", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6076, - "hfopenllm_v2/BBH": 0.6489, - "hfopenllm_v2/MATH Level 5": 0.2115, - "hfopenllm_v2/GPQA": 0.3876, - "hfopenllm_v2/MUSR": 0.4922, - "hfopenllm_v2/MMLU-PRO": 0.4658 - } - }, - { - "id": "migtissera/Llama-3-8B-Synthia-v3.5", - "name": "Llama-3-8B-Synthia-v3.5", - "developer": "migtissera", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.507, - "hfopenllm_v2/BBH": 0.4888, - "hfopenllm_v2/MATH Level 5": 0.0657, - "hfopenllm_v2/GPQA": 0.2718, - "hfopenllm_v2/MUSR": 0.4044, - "hfopenllm_v2/MMLU-PRO": 0.303 - } - }, - { - "id": "migtissera/Tess-3-7B-SFT", - "name": "Tess-3-7B-SFT", - "developer": "migtissera", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3946, - "hfopenllm_v2/BBH": 0.4607, - "hfopenllm_v2/MATH Level 5": 0.04, - "hfopenllm_v2/GPQA": 0.271, - "hfopenllm_v2/MUSR": 0.4113, - "hfopenllm_v2/MMLU-PRO": 0.3034 - } - }, - { - "id": "migtissera/Tess-3-Mistral-Nemo-12B", - "name": "Tess-3-Mistral-Nemo-12B", - "developer": "migtissera", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3355, - "hfopenllm_v2/BBH": 0.4899, - "hfopenllm_v2/MATH Level 5": 0.0574, - "hfopenllm_v2/GPQA": 0.2508, - "hfopenllm_v2/MUSR": 0.4458, - "hfopenllm_v2/MMLU-PRO": 0.2565 - } - }, - { - "id": "migtissera/Tess-v2.5-Phi-3-medium-128k-14B", - "name": "Tess-v2.5-Phi-3-medium-128k-14B", - "developer": "migtissera", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4539, - "hfopenllm_v2/BBH": 0.6207, - "hfopenllm_v2/MATH Level 5": 0.0506, - "hfopenllm_v2/GPQA": 0.3079, - "hfopenllm_v2/MUSR": 0.4113, - "hfopenllm_v2/MMLU-PRO": 0.3732 - } - }, - { - "id": "migtissera/Tess-v2.5.2-Qwen2-72B", - "name": "Tess-v2.5.2-Qwen2-72B", - "developer": "migtissera", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4494, - "hfopenllm_v2/BBH": 0.6647, - "hfopenllm_v2/MATH Level 5": 0.2938, - "hfopenllm_v2/GPQA": 0.3507, - "hfopenllm_v2/MUSR": 0.4188, - "hfopenllm_v2/MMLU-PRO": 0.5561 - } - }, - { - "id": "migtissera/Trinity-2-Codestral-22B", - "name": "Trinity-2-Codestral-22B", - "developer": "migtissera", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4202, - "hfopenllm_v2/BBH": 0.5593, - "hfopenllm_v2/MATH Level 5": 0.0967, - "hfopenllm_v2/GPQA": 0.3146, - "hfopenllm_v2/MUSR": 0.4111, - "hfopenllm_v2/MMLU-PRO": 0.3308 - } - }, - { - "id": "migtissera/Trinity-2-Codestral-22B-v0.2", - "name": "Trinity-2-Codestral-22B-v0.2", - "developer": "migtissera", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.443, - "hfopenllm_v2/BBH": 0.5706, - "hfopenllm_v2/MATH Level 5": 0.0869, - "hfopenllm_v2/GPQA": 0.3079, - "hfopenllm_v2/MUSR": 0.4031, - "hfopenllm_v2/MMLU-PRO": 0.3354 - } - }, - { - "id": "Minami-su/Amara-o1-7B-Qwen", - "name": "Amara-o1-7B-Qwen", - "developer": "Minami-su", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.739, - "hfopenllm_v2/BBH": 0.5199, - "hfopenllm_v2/MATH Level 5": 0.5181, - "hfopenllm_v2/GPQA": 0.2936, - "hfopenllm_v2/MUSR": 0.4007, - "hfopenllm_v2/MMLU-PRO": 0.4083 - } - }, - { - "id": "Minami-su/Amara-o2-7B-Qwen", - "name": "Amara-o2-7B-Qwen", - "developer": "Minami-su", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7147, - "hfopenllm_v2/BBH": 0.5173, - "hfopenllm_v2/MATH Level 5": 0.4086, - "hfopenllm_v2/GPQA": 0.2634, - "hfopenllm_v2/MUSR": 0.3781, - "hfopenllm_v2/MMLU-PRO": 0.4165 - } - }, - { - "id": "Minami-su/test-7B-00", - "name": "test-7B-00", - "developer": "Minami-su", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.669, - "hfopenllm_v2/BBH": 0.4466, - "hfopenllm_v2/MATH Level 5": 0.4517, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.4126, - "hfopenllm_v2/MMLU-PRO": 0.3588 - } - }, - { - "id": "Minami-su/test-7B-01", - "name": "test-7B-01", - "developer": "Minami-su", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6736, - "hfopenllm_v2/BBH": 0.4422, - "hfopenllm_v2/MATH Level 5": 0.4554, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.4153, - "hfopenllm_v2/MMLU-PRO": 0.3536 - } - }, - { - "id": "Minami-su/test-v2-7B-00", - "name": "test-v2-7B-00", - "developer": "Minami-su", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6747, - "hfopenllm_v2/BBH": 0.4416, - "hfopenllm_v2/MATH Level 5": 0.4418, - "hfopenllm_v2/GPQA": 0.2919, - "hfopenllm_v2/MUSR": 0.4154, - "hfopenllm_v2/MMLU-PRO": 0.3472 - } - }, - { - "id": "mindw96/DeepSeek-llama3.3-Bllossom-8B-DACON-LLM3", - "name": "DeepSeek-llama3.3-Bllossom-8B-DACON-LLM3", - "developer": "mindw96", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1388, - "hfopenllm_v2/BBH": 0.3068, - "hfopenllm_v2/MATH Level 5": 0.0083, - "hfopenllm_v2/GPQA": 0.2508, - "hfopenllm_v2/MUSR": 0.3792, - "hfopenllm_v2/MMLU-PRO": 0.1106 - } - }, - { - "id": "minghaowu/Qwen1.5-1.8B-OpenHermes-2.5", - "name": "Qwen1.5-1.8B-OpenHermes-2.5", - "developer": "minghaowu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2778, - "hfopenllm_v2/BBH": 0.3375, - "hfopenllm_v2/MATH Level 5": 0.0242, - "hfopenllm_v2/GPQA": 0.2836, - "hfopenllm_v2/MUSR": 0.3529, - "hfopenllm_v2/MMLU-PRO": 0.1792 - } - }, - { - "id": "minimax/Minimax-2.5", - "name": "Minimax-2.5", - "developer": "minimax", - "evaluator_relationship": null, - "benchmark_scores": { - "apex-agents/Corporate Lawyer Mean Score": 0.339 - } - }, - { - "id": "minimax/minimax-m2", - "name": "MiniMax M2", - "developer": "MiniMax", - "evaluator_relationship": null, - "benchmark_scores": { - "terminal-bench-2.0/terminal-bench-2.0": 30.0 - } - }, - { - "id": "minimax/minimax-m2.1", - "name": "MiniMax M2.1", - "developer": "MiniMax", - "evaluator_relationship": null, - "benchmark_scores": { - "terminal-bench-2.0/terminal-bench-2.0": 29.2 - } - }, - { - "id": "minimax/minimax-m2.5", - "name": "Minimax m2.5", - "developer": "Minimax", - "evaluator_relationship": null, - "benchmark_scores": { - "terminal-bench-2.0/terminal-bench-2.0": 42.2 - } - }, - { - "id": "ministral/Ministral-3b-instruct", - "name": "Ministral-3b-instruct", - "developer": "ministral", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1358, - "hfopenllm_v2/BBH": 0.3192, - "hfopenllm_v2/MATH Level 5": 0.0083, - "hfopenllm_v2/GPQA": 0.2517, - "hfopenllm_v2/MUSR": 0.3382, - "hfopenllm_v2/MMLU-PRO": 0.1093 - } - }, - { - "id": "mistral-community/Mistral-7B-v0.2", - "name": "Mistral-7B-v0.2", - "developer": "mistral-community", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2266, - "hfopenllm_v2/BBH": 0.451, - "hfopenllm_v2/MATH Level 5": 0.0302, - "hfopenllm_v2/GPQA": 0.2919, - "hfopenllm_v2/MUSR": 0.4032, - "hfopenllm_v2/MMLU-PRO": 0.2953 - } - }, - { - "id": "mistral-community/Mixtral-8x22B-v0.1", - "name": "Mixtral-8x22B-v0.1", - "developer": "mistral-community", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3167, - "hfopenllm_v2/BBH": 0.38, - "hfopenllm_v2/MATH Level 5": 0.1543, - "hfopenllm_v2/GPQA": 0.33, - "hfopenllm_v2/MUSR": 0.3533, - "hfopenllm_v2/MMLU-PRO": 0.36 - } - }, - { - "id": "mistral-community/mixtral-8x22B-v0.3", - "name": "mixtral-8x22B-v0.3", - "developer": "mistral-community", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2583, - "hfopenllm_v2/BBH": 0.625, - "hfopenllm_v2/MATH Level 5": 0.1835, - "hfopenllm_v2/GPQA": 0.3775, - "hfopenllm_v2/MUSR": 0.4037, - "hfopenllm_v2/MMLU-PRO": 0.4639 - } - }, - { - "id": "mistralai/Codestral-22B-v0.1", - "name": "Codestral-22B-v0.1", - "developer": "mistralai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5772, - "hfopenllm_v2/BBH": 0.5139, - "hfopenllm_v2/MATH Level 5": 0.1005, - "hfopenllm_v2/GPQA": 0.2987, - "hfopenllm_v2/MUSR": 0.4187, - "hfopenllm_v2/MMLU-PRO": 0.3156 - } - }, - { - "id": "mistralai/Ministral-8B-Instruct-2410", - "name": "Ministral-8B-Instruct-2410", - "developer": "mistralai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5896, - "hfopenllm_v2/BBH": 0.4762, - "hfopenllm_v2/MATH Level 5": 0.1956, - "hfopenllm_v2/GPQA": 0.2844, - "hfopenllm_v2/MUSR": 0.4138, - "hfopenllm_v2/MMLU-PRO": 0.3291 - } - }, - { - "id": "mistralai/ministral-8b-instruct-2410-fc", - "name": "Ministral-8B-Instruct-2410 (FC)", - "developer": "mistralai", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 105.0, - "bfcl/bfcl.overall.overall_accuracy": 11.1, - "bfcl/bfcl.overall.total_cost_usd": 70.01, - "bfcl/bfcl.overall.latency_mean_s": 82.07, - "bfcl/bfcl.overall.latency_std_s": 212.99, - "bfcl/bfcl.overall.latency_p95_s": 568.59, - "bfcl/bfcl.non_live.ast_accuracy": 0.0, - "bfcl/bfcl.non_live.simple_ast_accuracy": 0.0, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 0.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 0.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 0.0, - "bfcl/bfcl.live.live_accuracy": 0.0, - "bfcl/bfcl.live.live_simple_ast_accuracy": 0.0, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 0.0, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 0.0, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 0.0, - "bfcl/bfcl.multi_turn.accuracy": 0.0, - "bfcl/bfcl.multi_turn.base_accuracy": 0.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 0.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 0.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 0.0, - "bfcl/bfcl.web_search.accuracy": 1.0, - "bfcl/bfcl.web_search.base_accuracy": 2.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 0.0, - "bfcl/bfcl.memory.accuracy": 4.52, - "bfcl/bfcl.memory.kv_accuracy": 3.87, - "bfcl/bfcl.memory.vector_accuracy": 7.1, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 2.58, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 0.0, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 100.0, - "bfcl/bfcl.format_sensitivity.max_delta": 0.0, - "bfcl/bfcl.format_sensitivity.stddev": 0.0 - } - }, - { - "id": "mistralai/Mistral-7B-Instruct-v0.1", - "name": "Mistral-7B-Instruct-v0.1", - "developer": "mistralai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4487, - "hfopenllm_v2/BBH": 0.3355, - "hfopenllm_v2/MATH Level 5": 0.0227, - "hfopenllm_v2/GPQA": 0.25, - "hfopenllm_v2/MUSR": 0.3848, - "hfopenllm_v2/MMLU-PRO": 0.2414 - } - }, - { - "id": "mistralai/Mistral-7B-Instruct-v0.2", - "name": "Mistral-7B-Instruct-v0.2", - "developer": "mistralai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5496, - "hfopenllm_v2/BBH": 0.446, - "hfopenllm_v2/MATH Level 5": 0.0302, - "hfopenllm_v2/GPQA": 0.276, - "hfopenllm_v2/MUSR": 0.3966, - "hfopenllm_v2/MMLU-PRO": 0.2717 - } - }, - { - "id": "mistralai/mistral-7b-instruct-v0.3", - "name": "Mistral Instruct v0.3 7B", - "developer": "mistralai", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.376, - "helm_capabilities/MMLU-Pro": 0.277, - "helm_capabilities/GPQA": 0.303, - "helm_capabilities/IFEval": 0.567, - "helm_capabilities/WildBench": 0.66, - "helm_capabilities/Omni-MATH": 0.072, - "helm_lite/Mean win rate": 0.196, - "helm_lite/NarrativeQA": 0.716, - "helm_lite/NaturalQuestions (closed-book)": 0.253, - "helm_lite/OpenbookQA": 0.79, - "helm_lite/MMLU": 0.51, - "helm_lite/MATH": 0.289, - "helm_lite/GSM8K": 0.538, - "helm_lite/LegalBench": 0.331, - "helm_lite/MedQA": 0.517, - "helm_lite/WMT 2014": 0.142, - "helm_mmlu/MMLU All Subjects": 0.599, - "helm_mmlu/Abstract Algebra": 0.27, - "helm_mmlu/Anatomy": 0.585, - "helm_mmlu/College Physics": 0.343, - "helm_mmlu/Computer Security": 0.7, - "helm_mmlu/Econometrics": 0.421, - "helm_mmlu/Global Facts": 0.33, - "helm_mmlu/Jurisprudence": 0.713, - "helm_mmlu/Philosophy": 0.659, - "helm_mmlu/Professional Psychology": 0.641, - "helm_mmlu/Us Foreign Policy": 0.79, - "helm_mmlu/Astronomy": 0.638, - "helm_mmlu/Business Ethics": 0.57, - "helm_mmlu/Clinical Knowledge": 0.687, - "helm_mmlu/Conceptual Physics": 0.549, - "helm_mmlu/Electrical Engineering": 0.572, - "helm_mmlu/Elementary Mathematics": 0.402, - "helm_mmlu/Formal Logic": 0.397, - "helm_mmlu/High School World History": 0.759, - "helm_mmlu/Human Sexuality": 0.702, - "helm_mmlu/International Law": 0.76, - "helm_mmlu/Logical Fallacies": 0.712, - "helm_mmlu/Machine Learning": 0.455, - "helm_mmlu/Management": 0.767, - "helm_mmlu/Marketing": 0.842, - "helm_mmlu/Medical Genetics": 0.75, - "helm_mmlu/Miscellaneous": 0.785, - "helm_mmlu/Moral Scenarios": 0.393, - "helm_mmlu/Nutrition": 0.676, - "helm_mmlu/Prehistory": 0.673, - "helm_mmlu/Public Relations": 0.636, - "helm_mmlu/Security Studies": 0.682, - "helm_mmlu/Sociology": 0.806, - "helm_mmlu/Virology": 0.47, - "helm_mmlu/World Religions": 0.825, - "helm_mmlu/Mean win rate": 0.509, - "hfopenllm_v2/IFEval": 0.5465, - "hfopenllm_v2/BBH": 0.4722, - "hfopenllm_v2/MATH Level 5": 0.0385, - "hfopenllm_v2/GPQA": 0.2794, - "hfopenllm_v2/MUSR": 0.3739, - "hfopenllm_v2/MMLU-PRO": 0.3075 - } - }, - { - "id": "mistralai/mistral-7b-v0.1", - "name": "Mistral v0.1 7B", - "developer": "mistralai", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.292, - "helm_lite/NarrativeQA": 0.716, - "helm_lite/NaturalQuestions (closed-book)": 0.367, - "helm_lite/OpenbookQA": 0.776, - "helm_lite/MMLU": 0.584, - "helm_lite/MATH": 0.297, - "helm_lite/GSM8K": 0.377, - "helm_lite/LegalBench": 0.58, - "helm_lite/MedQA": 0.525, - "helm_lite/WMT 2014": 0.16, - "helm_mmlu/MMLU All Subjects": 0.566, - "helm_mmlu/Abstract Algebra": 0.25, - "helm_mmlu/Anatomy": 0.467, - "helm_mmlu/College Physics": 0.314, - "helm_mmlu/Computer Security": 0.69, - "helm_mmlu/Econometrics": 0.351, - "helm_mmlu/Global Facts": 0.29, - "helm_mmlu/Jurisprudence": 0.667, - "helm_mmlu/Philosophy": 0.63, - "helm_mmlu/Professional Psychology": 0.578, - "helm_mmlu/Us Foreign Policy": 0.79, - "helm_mmlu/Astronomy": 0.599, - "helm_mmlu/Business Ethics": 0.56, - "helm_mmlu/Clinical Knowledge": 0.653, - "helm_mmlu/Conceptual Physics": 0.451, - "helm_mmlu/Electrical Engineering": 0.538, - "helm_mmlu/Elementary Mathematics": 0.32, - "helm_mmlu/Formal Logic": 0.365, - "helm_mmlu/High School World History": 0.726, - "helm_mmlu/Human Sexuality": 0.702, - "helm_mmlu/International Law": 0.76, - "helm_mmlu/Logical Fallacies": 0.693, - "helm_mmlu/Machine Learning": 0.438, - "helm_mmlu/Management": 0.709, - "helm_mmlu/Marketing": 0.833, - "helm_mmlu/Medical Genetics": 0.68, - "helm_mmlu/Miscellaneous": 0.72, - "helm_mmlu/Moral Scenarios": 0.33, - "helm_mmlu/Nutrition": 0.657, - "helm_mmlu/Prehistory": 0.642, - "helm_mmlu/Public Relations": 0.6, - "helm_mmlu/Security Studies": 0.731, - "helm_mmlu/Sociology": 0.831, - "helm_mmlu/Virology": 0.44, - "helm_mmlu/World Religions": 0.789, - "helm_mmlu/Mean win rate": 0.213, - "hfopenllm_v2/IFEval": 0.2386, - "hfopenllm_v2/BBH": 0.4419, - "hfopenllm_v2/MATH Level 5": 0.0295, - "hfopenllm_v2/GPQA": 0.2919, - "hfopenllm_v2/MUSR": 0.4139, - "hfopenllm_v2/MMLU-PRO": 0.3013 - } - }, - { - "id": "mistralai/Mistral-7B-v0.3", - "name": "Mistral-7B-v0.3", - "developer": "mistralai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2266, - "hfopenllm_v2/BBH": 0.4517, - "hfopenllm_v2/MATH Level 5": 0.0302, - "hfopenllm_v2/GPQA": 0.2919, - "hfopenllm_v2/MUSR": 0.4032, - "hfopenllm_v2/MMLU-PRO": 0.2953 - } - }, - { - "id": "mistralai/mistral-large-2402", - "name": "Mistral Large 2402", - "developer": "mistralai", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.328, - "helm_lite/NarrativeQA": 0.454, - "helm_lite/NaturalQuestions (closed-book)": 0.311, - "helm_lite/OpenbookQA": 0.894, - "helm_lite/MMLU": 0.638, - "helm_lite/MATH": 0.75, - "helm_lite/GSM8K": 0.694, - "helm_lite/LegalBench": 0.479, - "helm_lite/MedQA": 0.499, - "helm_lite/WMT 2014": 0.182, - "helm_mmlu/MMLU All Subjects": 0.688, - "helm_mmlu/Abstract Algebra": 0.45, - "helm_mmlu/Anatomy": 0.674, - "helm_mmlu/College Physics": 0.373, - "helm_mmlu/Computer Security": 0.8, - "helm_mmlu/Econometrics": 0.64, - "helm_mmlu/Global Facts": 0.34, - "helm_mmlu/Jurisprudence": 0.815, - "helm_mmlu/Philosophy": 0.794, - "helm_mmlu/Professional Psychology": 0.809, - "helm_mmlu/Us Foreign Policy": 0.92, - "helm_mmlu/Astronomy": 0.842, - "helm_mmlu/Business Ethics": 0.67, - "helm_mmlu/Clinical Knowledge": 0.751, - "helm_mmlu/Conceptual Physics": 0.574, - "helm_mmlu/Electrical Engineering": 0.545, - "helm_mmlu/Elementary Mathematics": 0.508, - "helm_mmlu/Formal Logic": 0.532, - "helm_mmlu/High School World History": 0.886, - "helm_mmlu/Human Sexuality": 0.847, - "helm_mmlu/International Law": 0.868, - "helm_mmlu/Logical Fallacies": 0.81, - "helm_mmlu/Machine Learning": 0.562, - "helm_mmlu/Management": 0.854, - "helm_mmlu/Marketing": 0.897, - "helm_mmlu/Medical Genetics": 0.74, - "helm_mmlu/Miscellaneous": 0.9, - "helm_mmlu/Moral Scenarios": 0.579, - "helm_mmlu/Nutrition": 0.791, - "helm_mmlu/Prehistory": 0.904, - "helm_mmlu/Public Relations": 0.709, - "helm_mmlu/Security Studies": 0.824, - "helm_mmlu/Sociology": 0.93, - "helm_mmlu/Virology": 0.554, - "helm_mmlu/World Religions": 0.883, - "helm_mmlu/Mean win rate": 0.464 - } - }, - { - "id": "mistralai/mistral-large-2407", - "name": "Mistral Large 2 2407", - "developer": "mistralai", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.744, - "helm_lite/NarrativeQA": 0.779, - "helm_lite/NaturalQuestions (closed-book)": 0.453, - "helm_lite/OpenbookQA": 0.932, - "helm_lite/MMLU": 0.725, - "helm_lite/MATH": 0.677, - "helm_lite/GSM8K": 0.912, - "helm_lite/LegalBench": 0.646, - "helm_lite/MedQA": 0.775, - "helm_lite/WMT 2014": 0.192, - "helm_mmlu/MMLU All Subjects": 0.8, - "helm_mmlu/Abstract Algebra": 0.7, - "helm_mmlu/Anatomy": 0.785, - "helm_mmlu/College Physics": 0.559, - "helm_mmlu/Computer Security": 0.81, - "helm_mmlu/Econometrics": 0.693, - "helm_mmlu/Global Facts": 0.56, - "helm_mmlu/Jurisprudence": 0.861, - "helm_mmlu/Philosophy": 0.826, - "helm_mmlu/Professional Psychology": 0.861, - "helm_mmlu/Us Foreign Policy": 0.9, - "helm_mmlu/Astronomy": 0.921, - "helm_mmlu/Business Ethics": 0.79, - "helm_mmlu/Clinical Knowledge": 0.864, - "helm_mmlu/Conceptual Physics": 0.864, - "helm_mmlu/Electrical Engineering": 0.793, - "helm_mmlu/Elementary Mathematics": 0.799, - "helm_mmlu/Formal Logic": 0.579, - "helm_mmlu/High School World History": 0.92, - "helm_mmlu/Human Sexuality": 0.924, - "helm_mmlu/International Law": 0.926, - "helm_mmlu/Logical Fallacies": 0.847, - "helm_mmlu/Machine Learning": 0.661, - "helm_mmlu/Management": 0.883, - "helm_mmlu/Marketing": 0.94, - "helm_mmlu/Medical Genetics": 0.9, - "helm_mmlu/Miscellaneous": 0.936, - "helm_mmlu/Moral Scenarios": 0.839, - "helm_mmlu/Nutrition": 0.827, - "helm_mmlu/Prehistory": 0.92, - "helm_mmlu/Public Relations": 0.764, - "helm_mmlu/Security Studies": 0.865, - "helm_mmlu/Sociology": 0.91, - "helm_mmlu/Virology": 0.59, - "helm_mmlu/World Religions": 0.865, - "helm_mmlu/Mean win rate": 0.24 - } - }, - { - "id": "mistralai/mistral-large-2411", - "name": "Mistral Large 2411", - "developer": "mistralai", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.598, - "helm_capabilities/MMLU-Pro": 0.599, - "helm_capabilities/GPQA": 0.435, - "helm_capabilities/IFEval": 0.876, - "helm_capabilities/WildBench": 0.801, - "helm_capabilities/Omni-MATH": 0.281 - } - }, - { - "id": "mistralai/mistral-large-2411-fc", - "name": "mistral-large-2411 (FC)", - "developer": "mistralai", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 46.0, - "bfcl/bfcl.overall.overall_accuracy": 38.37, - "bfcl/bfcl.overall.total_cost_usd": 115.98, - "bfcl/bfcl.overall.latency_mean_s": 2.04, - "bfcl/bfcl.overall.latency_std_s": 4.02, - "bfcl/bfcl.overall.latency_p95_s": 4.68, - "bfcl/bfcl.non_live.ast_accuracy": 84.65, - "bfcl/bfcl.non_live.simple_ast_accuracy": 72.08, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 93.5, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 89.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 83.5, - "bfcl/bfcl.live.live_accuracy": 81.87, - "bfcl/bfcl.live.live_simple_ast_accuracy": 87.21, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 80.72, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 81.25, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 75.0, - "bfcl/bfcl.multi_turn.accuracy": 14.12, - "bfcl/bfcl.multi_turn.base_accuracy": 18.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 11.5, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 13.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 13.5, - "bfcl/bfcl.web_search.accuracy": 28.0, - "bfcl/bfcl.web_search.base_accuracy": 41.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 15.0, - "bfcl/bfcl.memory.accuracy": 24.95, - "bfcl/bfcl.memory.kv_accuracy": 18.71, - "bfcl/bfcl.memory.vector_accuracy": 29.03, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 27.1, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 93.75, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 68.92 - } - }, - { - "id": "mistralai/mistral-large-2411-prompt", - "name": "mistral-large-2411 (Prompt)", - "developer": "mistralai", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 63.0, - "bfcl/bfcl.overall.overall_accuracy": 31.84, - "bfcl/bfcl.overall.total_cost_usd": 232.42, - "bfcl/bfcl.overall.latency_mean_s": 1.82, - "bfcl/bfcl.overall.latency_std_s": 7.15, - "bfcl/bfcl.overall.latency_p95_s": 4.08, - "bfcl/bfcl.non_live.ast_accuracy": 83.0, - "bfcl/bfcl.non_live.simple_ast_accuracy": 75.5, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 89.5, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 87.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 80.0, - "bfcl/bfcl.live.live_accuracy": 68.1, - "bfcl/bfcl.live.live_simple_ast_accuracy": 83.72, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 64.01, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 93.75, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 62.5, - "bfcl/bfcl.multi_turn.accuracy": 13.75, - "bfcl/bfcl.multi_turn.base_accuracy": 20.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 5.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 11.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 19.0, - "bfcl/bfcl.web_search.accuracy": 20.0, - "bfcl/bfcl.web_search.base_accuracy": 28.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 12.0, - "bfcl/bfcl.memory.accuracy": 23.66, - "bfcl/bfcl.memory.kv_accuracy": 16.77, - "bfcl/bfcl.memory.vector_accuracy": 30.97, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 23.23, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 93.75, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 38.77, - "bfcl/bfcl.format_sensitivity.max_delta": 13.5, - "bfcl/bfcl.format_sensitivity.stddev": 3.91 - } - }, - { - "id": "mistralai/Mistral-Large-Instruct-2411", - "name": "Mistral-Large-Instruct-2411", - "developer": "mistralai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8401, - "hfopenllm_v2/BBH": 0.6747, - "hfopenllm_v2/MATH Level 5": 0.4955, - "hfopenllm_v2/GPQA": 0.4371, - "hfopenllm_v2/MUSR": 0.454, - "hfopenllm_v2/MMLU-PRO": 0.5562 - } - }, - { - "id": "mistralai/mistral-medium-2312", - "name": "Mistral Medium 2312", - "developer": "mistralai", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.268, - "helm_lite/NarrativeQA": 0.449, - "helm_lite/NaturalQuestions (closed-book)": 0.29, - "helm_lite/OpenbookQA": 0.83, - "helm_lite/MMLU": 0.618, - "helm_lite/MATH": 0.565, - "helm_lite/GSM8K": 0.706, - "helm_lite/LegalBench": 0.452, - "helm_lite/MedQA": 0.61, - "helm_lite/WMT 2014": 0.169 - } - }, - { - "id": "mistralai/mistral-medium-2505", - "name": "Mistral-Medium-2505", - "developer": "mistralai", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 48.0, - "bfcl/bfcl.overall.overall_accuracy": 37.69, - "bfcl/bfcl.overall.total_cost_usd": 36.51, - "bfcl/bfcl.overall.latency_mean_s": 1.21, - "bfcl/bfcl.overall.latency_std_s": 3.5, - "bfcl/bfcl.overall.latency_p95_s": 2.86, - "bfcl/bfcl.non_live.ast_accuracy": 85.33, - "bfcl/bfcl.non_live.simple_ast_accuracy": 76.33, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 91.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 88.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 85.5, - "bfcl/bfcl.live.live_accuracy": 66.03, - "bfcl/bfcl.live.live_simple_ast_accuracy": 80.23, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 62.39, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 81.25, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 62.5, - "bfcl/bfcl.multi_turn.accuracy": 9.88, - "bfcl/bfcl.multi_turn.base_accuracy": 13.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 6.5, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 6.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 13.5, - "bfcl/bfcl.web_search.accuracy": 39.0, - "bfcl/bfcl.web_search.base_accuracy": 41.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 37.0, - "bfcl/bfcl.memory.accuracy": 21.72, - "bfcl/bfcl.memory.kv_accuracy": 16.13, - "bfcl/bfcl.memory.vector_accuracy": 14.84, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 34.19, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 75.0, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 74.49, - "bfcl/bfcl.format_sensitivity.max_delta": 21.5, - "bfcl/bfcl.format_sensitivity.stddev": 5.02 - } - }, - { - "id": "mistralai/mistral-medium-2505-fc", - "name": "Mistral-Medium-2505 (FC)", - "developer": "mistralai", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 49.0, - "bfcl/bfcl.overall.overall_accuracy": 37.56, - "bfcl/bfcl.overall.total_cost_usd": 18.8, - "bfcl/bfcl.overall.latency_mean_s": 1.6, - "bfcl/bfcl.overall.latency_std_s": 4.44, - "bfcl/bfcl.overall.latency_p95_s": 4.19, - "bfcl/bfcl.non_live.ast_accuracy": 67.44, - "bfcl/bfcl.non_live.simple_ast_accuracy": 39.75, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 78.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 83.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 69.0, - "bfcl/bfcl.live.live_accuracy": 67.95, - "bfcl/bfcl.live.live_simple_ast_accuracy": 67.05, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 68.09, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 81.25, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 62.5, - "bfcl/bfcl.multi_turn.accuracy": 10.75, - "bfcl/bfcl.multi_turn.base_accuracy": 15.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 7.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 7.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 13.0, - "bfcl/bfcl.web_search.accuracy": 35.0, - "bfcl/bfcl.web_search.base_accuracy": 36.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 34.0, - "bfcl/bfcl.memory.accuracy": 23.01, - "bfcl/bfcl.memory.kv_accuracy": 15.48, - "bfcl/bfcl.memory.vector_accuracy": 20.0, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 33.55, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 62.5, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 91.95 - } - }, - { - "id": "mistralai/mistral-medium-3", - "name": "mistral-medium-3", - "developer": "mistralai", - "evaluator_relationship": null, - "benchmark_scores": { - "global-mmlu-lite/Global MMLU Lite": 0.5511, - "global-mmlu-lite/Culturally Sensitive": 0.5391, - "global-mmlu-lite/Culturally Agnostic": 0.5631, - "global-mmlu-lite/Arabic": 0.455, - "global-mmlu-lite/English": 0.38, - "global-mmlu-lite/Bengali": 0.5175, - "global-mmlu-lite/German": 0.4775, - "global-mmlu-lite/French": 0.41, - "global-mmlu-lite/Hindi": 0.555, - "global-mmlu-lite/Indonesian": 0.515, - "global-mmlu-lite/Italian": 0.535, - "global-mmlu-lite/Japanese": 0.58, - "global-mmlu-lite/Korean": 0.595, - "global-mmlu-lite/Portuguese": 0.5175, - "global-mmlu-lite/Spanish": 0.5375, - "global-mmlu-lite/Swahili": 0.7075, - "global-mmlu-lite/Yoruba": 0.7675, - "global-mmlu-lite/Chinese": 0.535, - "global-mmlu-lite/Burmese": 0.7325 - } - }, - { - "id": "mistralai/Mistral-Nemo-Base-2407", - "name": "Mistral-Nemo-Base-2407", - "developer": "mistralai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.163, - "hfopenllm_v2/BBH": 0.5035, - "hfopenllm_v2/MATH Level 5": 0.0597, - "hfopenllm_v2/GPQA": 0.2936, - "hfopenllm_v2/MUSR": 0.3921, - "hfopenllm_v2/MMLU-PRO": 0.3472 - } - }, - { - "id": "mistralai/Mistral-Nemo-Instruct-2407", - "name": "Mistral-Nemo-Instruct-2407", - "developer": "mistralai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.638, - "hfopenllm_v2/BBH": 0.5037, - "hfopenllm_v2/MATH Level 5": 0.1269, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.39, - "hfopenllm_v2/MMLU-PRO": 0.3517 - } - }, - { - "id": "mistralai/mistral-small-2402", - "name": "Mistral Small 2402", - "developer": "mistralai", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.288, - "helm_lite/NarrativeQA": 0.519, - "helm_lite/NaturalQuestions (closed-book)": 0.304, - "helm_lite/OpenbookQA": 0.862, - "helm_lite/MMLU": 0.593, - "helm_lite/MATH": 0.621, - "helm_lite/GSM8K": 0.734, - "helm_lite/LegalBench": 0.389, - "helm_lite/MedQA": 0.616, - "helm_lite/WMT 2014": 0.169, - "helm_mmlu/MMLU All Subjects": 0.687, - "helm_mmlu/Abstract Algebra": 0.26, - "helm_mmlu/Anatomy": 0.674, - "helm_mmlu/College Physics": 0.402, - "helm_mmlu/Computer Security": 0.77, - "helm_mmlu/Econometrics": 0.614, - "helm_mmlu/Global Facts": 0.45, - "helm_mmlu/Jurisprudence": 0.833, - "helm_mmlu/Philosophy": 0.765, - "helm_mmlu/Professional Psychology": 0.768, - "helm_mmlu/Us Foreign Policy": 0.89, - "helm_mmlu/Astronomy": 0.77, - "helm_mmlu/Business Ethics": 0.71, - "helm_mmlu/Clinical Knowledge": 0.766, - "helm_mmlu/Conceptual Physics": 0.685, - "helm_mmlu/Electrical Engineering": 0.628, - "helm_mmlu/Elementary Mathematics": 0.415, - "helm_mmlu/Formal Logic": 0.516, - "helm_mmlu/High School World History": 0.857, - "helm_mmlu/Human Sexuality": 0.824, - "helm_mmlu/International Law": 0.826, - "helm_mmlu/Logical Fallacies": 0.804, - "helm_mmlu/Machine Learning": 0.562, - "helm_mmlu/Management": 0.786, - "helm_mmlu/Marketing": 0.906, - "helm_mmlu/Medical Genetics": 0.75, - "helm_mmlu/Miscellaneous": 0.844, - "helm_mmlu/Moral Scenarios": 0.575, - "helm_mmlu/Nutrition": 0.761, - "helm_mmlu/Prehistory": 0.802, - "helm_mmlu/Public Relations": 0.773, - "helm_mmlu/Security Studies": 0.788, - "helm_mmlu/Sociology": 0.871, - "helm_mmlu/Virology": 0.542, - "helm_mmlu/World Religions": 0.848, - "helm_mmlu/Mean win rate": 0.54 - } - }, - { - "id": "mistralai/Mistral-Small-24B-Base-2501", - "name": "Mistral-Small-24B-Base-2501", - "developer": "mistralai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1672, - "hfopenllm_v2/BBH": 0.6442, - "hfopenllm_v2/MATH Level 5": 0.1971, - "hfopenllm_v2/GPQA": 0.3876, - "hfopenllm_v2/MUSR": 0.4237, - "hfopenllm_v2/MMLU-PRO": 0.5406 - } - }, - { - "id": "mistralai/mistral-small-2503", - "name": "mistral-small-2503", - "developer": "mistralai", - "evaluator_relationship": null, - "benchmark_scores": { - "global-mmlu-lite/Global MMLU Lite": 0.7852, - "global-mmlu-lite/Culturally Sensitive": 0.7537, - "global-mmlu-lite/Culturally Agnostic": 0.8166, - "global-mmlu-lite/Arabic": 0.7875, - "global-mmlu-lite/English": 0.8, - "global-mmlu-lite/Bengali": 0.7725, - "global-mmlu-lite/German": 0.7975, - "global-mmlu-lite/French": 0.8, - "global-mmlu-lite/Hindi": 0.795, - "global-mmlu-lite/Indonesian": 0.785, - "global-mmlu-lite/Italian": 0.805, - "global-mmlu-lite/Japanese": 0.77, - "global-mmlu-lite/Korean": 0.79, - "global-mmlu-lite/Portuguese": 0.7925, - "global-mmlu-lite/Spanish": 0.7825, - "global-mmlu-lite/Swahili": 0.775, - "global-mmlu-lite/Yoruba": 0.735, - "global-mmlu-lite/Chinese": 0.7925, - "global-mmlu-lite/Burmese": 0.7825, - "helm_capabilities/Mean score": 0.558, - "helm_capabilities/MMLU-Pro": 0.61, - "helm_capabilities/GPQA": 0.392, - "helm_capabilities/IFEval": 0.75, - "helm_capabilities/WildBench": 0.788, - "helm_capabilities/Omni-MATH": 0.248 - } - }, - { - "id": "mistralai/mistral-small-2506-fc", - "name": "Mistral-small-2506 (FC)", - "developer": "mistralai", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 51.0, - "bfcl/bfcl.overall.overall_accuracy": 37.15, - "bfcl/bfcl.overall.total_cost_usd": 5.2, - "bfcl/bfcl.overall.latency_mean_s": 1.48, - "bfcl/bfcl.overall.latency_std_s": 18.25, - "bfcl/bfcl.overall.latency_p95_s": 2.5, - "bfcl/bfcl.non_live.ast_accuracy": 73.6, - "bfcl/bfcl.non_live.simple_ast_accuracy": 38.92, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 93.5, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 83.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 78.5, - "bfcl/bfcl.live.live_accuracy": 77.28, - "bfcl/bfcl.live.live_simple_ast_accuracy": 69.38, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 79.39, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 75.0, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 70.83, - "bfcl/bfcl.multi_turn.accuracy": 11.5, - "bfcl/bfcl.multi_turn.base_accuracy": 17.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 6.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 10.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 12.0, - "bfcl/bfcl.web_search.accuracy": 31.0, - "bfcl/bfcl.web_search.base_accuracy": 37.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 25.0, - "bfcl/bfcl.memory.accuracy": 18.06, - "bfcl/bfcl.memory.kv_accuracy": 8.39, - "bfcl/bfcl.memory.vector_accuracy": 14.19, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 31.61, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 87.5, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 87.94 - } - }, - { - "id": "mistralai/mistral-small-2506-prompt", - "name": "Mistral-Small-2506 (Prompt)", - "developer": "mistralai", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 59.0, - "bfcl/bfcl.overall.overall_accuracy": 32.38, - "bfcl/bfcl.overall.total_cost_usd": 6.91, - "bfcl/bfcl.overall.latency_mean_s": 0.92, - "bfcl/bfcl.overall.latency_std_s": 6.79, - "bfcl/bfcl.overall.latency_p95_s": 2.02, - "bfcl/bfcl.non_live.ast_accuracy": 89.69, - "bfcl/bfcl.non_live.simple_ast_accuracy": 78.75, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 96.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 92.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 91.5, - "bfcl/bfcl.live.live_accuracy": 79.05, - "bfcl/bfcl.live.live_simple_ast_accuracy": 81.4, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 78.54, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 93.75, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 66.67, - "bfcl/bfcl.multi_turn.accuracy": 14.75, - "bfcl/bfcl.multi_turn.base_accuracy": 20.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 17.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 9.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 12.0, - "bfcl/bfcl.web_search.accuracy": 7.5, - "bfcl/bfcl.web_search.base_accuracy": 9.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 6.0, - "bfcl/bfcl.memory.accuracy": 15.05, - "bfcl/bfcl.memory.kv_accuracy": 2.58, - "bfcl/bfcl.memory.vector_accuracy": 11.61, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 30.97, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 93.75, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 65.73, - "bfcl/bfcl.format_sensitivity.max_delta": 50.0, - "bfcl/bfcl.format_sensitivity.stddev": 13.57 - } - }, - { - "id": "mistralai/Mistral-Small-Instruct-2409", - "name": "Mistral-Small-Instruct-2409", - "developer": "mistralai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.667, - "hfopenllm_v2/BBH": 0.5213, - "hfopenllm_v2/MATH Level 5": 0.1435, - "hfopenllm_v2/GPQA": 0.3238, - "hfopenllm_v2/MUSR": 0.3632, - "hfopenllm_v2/MMLU-PRO": 0.396 - } - }, - { - "id": "mistralai/Mistral-v0.1-7B", - "name": "Mistral v0.1 7B", - "developer": "mistralai", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.884, - "helm_classic/MMLU": 0.572, - "helm_classic/BoolQ": 0.874, - "helm_classic/NarrativeQA": 0.716, - "helm_classic/NaturalQuestions (open-book)": 0.687, - "helm_classic/QuAC": 0.423, - "helm_classic/HellaSwag": -1.0, - "helm_classic/OpenbookQA": -1.0, - "helm_classic/TruthfulQA": 0.422, - "helm_classic/MS MARCO (TREC)": -1.0, - "helm_classic/CNN/DailyMail": -1.0, - "helm_classic/XSUM": -1.0, - "helm_classic/IMDB": 0.962, - "helm_classic/CivilComments": 0.624, - "helm_classic/RAFT": 0.707 - } - }, - { - "id": "mistralai/mixtral-8x22b", - "name": "Mixtral 8x22B", - "developer": "mistralai", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.705, - "helm_lite/NarrativeQA": 0.779, - "helm_lite/NaturalQuestions (closed-book)": 0.478, - "helm_lite/OpenbookQA": 0.882, - "helm_lite/MMLU": 0.701, - "helm_lite/MATH": 0.656, - "helm_lite/GSM8K": 0.8, - "helm_lite/LegalBench": 0.708, - "helm_lite/MedQA": 0.704, - "helm_lite/WMT 2014": 0.209, - "helm_mmlu/MMLU All Subjects": 0.778, - "helm_mmlu/Abstract Algebra": 0.48, - "helm_mmlu/Anatomy": 0.741, - "helm_mmlu/College Physics": 0.569, - "helm_mmlu/Computer Security": 0.84, - "helm_mmlu/Econometrics": 0.667, - "helm_mmlu/Global Facts": 0.56, - "helm_mmlu/Jurisprudence": 0.852, - "helm_mmlu/Philosophy": 0.842, - "helm_mmlu/Professional Psychology": 0.845, - "helm_mmlu/Us Foreign Policy": 0.95, - "helm_mmlu/Astronomy": 0.882, - "helm_mmlu/Business Ethics": 0.74, - "helm_mmlu/Clinical Knowledge": 0.819, - "helm_mmlu/Conceptual Physics": 0.796, - "helm_mmlu/Electrical Engineering": 0.766, - "helm_mmlu/Elementary Mathematics": 0.622, - "helm_mmlu/Formal Logic": 0.627, - "helm_mmlu/High School World History": 0.895, - "helm_mmlu/Human Sexuality": 0.885, - "helm_mmlu/International Law": 0.917, - "helm_mmlu/Logical Fallacies": 0.877, - "helm_mmlu/Machine Learning": 0.661, - "helm_mmlu/Management": 0.883, - "helm_mmlu/Marketing": 0.915, - "helm_mmlu/Medical Genetics": 0.85, - "helm_mmlu/Miscellaneous": 0.899, - "helm_mmlu/Moral Scenarios": 0.646, - "helm_mmlu/Nutrition": 0.866, - "helm_mmlu/Prehistory": 0.87, - "helm_mmlu/Public Relations": 0.755, - "helm_mmlu/Security Studies": 0.865, - "helm_mmlu/Sociology": 0.92, - "helm_mmlu/Virology": 0.596, - "helm_mmlu/World Religions": 0.901, - "helm_mmlu/Mean win rate": 0.598 - } - }, - { - "id": "mistralai/Mixtral-8x22B-Instruct-v0.1", - "name": "Mixtral-8x22B-Instruct-v0.1", - "developer": "mistralai", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.478, - "helm_capabilities/MMLU-Pro": 0.46, - "helm_capabilities/GPQA": 0.334, - "helm_capabilities/IFEval": 0.724, - "helm_capabilities/WildBench": 0.711, - "helm_capabilities/Omni-MATH": 0.163, - "hfopenllm_v2/IFEval": 0.7184, - "hfopenllm_v2/BBH": 0.6125, - "hfopenllm_v2/MATH Level 5": 0.1873, - "hfopenllm_v2/GPQA": 0.3733, - "hfopenllm_v2/MUSR": 0.4311, - "hfopenllm_v2/MMLU-PRO": 0.4483 - } - }, - { - "id": "mistralai/Mixtral-8x22B-v0.1", - "name": "Mixtral-8x22B-v0.1", - "developer": "mistralai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2583, - "hfopenllm_v2/BBH": 0.624, - "hfopenllm_v2/MATH Level 5": 0.1835, - "hfopenllm_v2/GPQA": 0.3758, - "hfopenllm_v2/MUSR": 0.4037, - "hfopenllm_v2/MMLU-PRO": 0.4639 - } - }, - { - "id": "mistralai/mixtral-8x7b-32kseqlen", - "name": "Mixtral 8x7B 32K seqlen", - "developer": "mistralai", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.51, - "helm_lite/NarrativeQA": 0.767, - "helm_lite/NaturalQuestions (closed-book)": 0.427, - "helm_lite/OpenbookQA": 0.868, - "helm_lite/MMLU": 0.649, - "helm_lite/MATH": 0.494, - "helm_lite/GSM8K": 0.622, - "helm_lite/LegalBench": 0.63, - "helm_lite/MedQA": 0.652, - "helm_lite/WMT 2014": 0.19, - "helm_mmlu/MMLU All Subjects": 0.717, - "helm_mmlu/Abstract Algebra": 0.38, - "helm_mmlu/Anatomy": 0.696, - "helm_mmlu/College Physics": 0.51, - "helm_mmlu/Computer Security": 0.81, - "helm_mmlu/Econometrics": 0.605, - "helm_mmlu/Global Facts": 0.46, - "helm_mmlu/Jurisprudence": 0.833, - "helm_mmlu/Philosophy": 0.797, - "helm_mmlu/Professional Psychology": 0.779, - "helm_mmlu/Us Foreign Policy": 0.93, - "helm_mmlu/Astronomy": 0.829, - "helm_mmlu/Business Ethics": 0.72, - "helm_mmlu/Clinical Knowledge": 0.785, - "helm_mmlu/Conceptual Physics": 0.681, - "helm_mmlu/Electrical Engineering": 0.676, - "helm_mmlu/Elementary Mathematics": 0.476, - "helm_mmlu/Formal Logic": 0.532, - "helm_mmlu/High School World History": 0.886, - "helm_mmlu/Human Sexuality": 0.87, - "helm_mmlu/International Law": 0.86, - "helm_mmlu/Logical Fallacies": 0.767, - "helm_mmlu/Machine Learning": 0.509, - "helm_mmlu/Management": 0.845, - "helm_mmlu/Marketing": 0.923, - "helm_mmlu/Medical Genetics": 0.76, - "helm_mmlu/Miscellaneous": 0.881, - "helm_mmlu/Moral Scenarios": 0.444, - "helm_mmlu/Nutrition": 0.83, - "helm_mmlu/Prehistory": 0.849, - "helm_mmlu/Public Relations": 0.682, - "helm_mmlu/Security Studies": 0.792, - "helm_mmlu/Sociology": 0.871, - "helm_mmlu/Virology": 0.506, - "helm_mmlu/World Religions": 0.871, - "helm_mmlu/Mean win rate": 0.689 - } - }, - { - "id": "mistralai/Mixtral-8x7B-Instruct-v0.1", - "name": "Mixtral-8x7B-Instruct-v0.1", - "developer": "mistralai", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.397, - "helm_capabilities/MMLU-Pro": 0.335, - "helm_capabilities/GPQA": 0.296, - "helm_capabilities/IFEval": 0.575, - "helm_capabilities/WildBench": 0.673, - "helm_capabilities/Omni-MATH": 0.105, - "hfopenllm_v2/IFEval": 0.5599, - "hfopenllm_v2/BBH": 0.4962, - "hfopenllm_v2/MATH Level 5": 0.0914, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.4203, - "hfopenllm_v2/MMLU-PRO": 0.3692, - "reward-bench/Score": 0.7455, - "reward-bench/Chat": 0.9497, - "reward-bench/Chat Hard": 0.6404, - "reward-bench/Safety": 0.7257, - "reward-bench/Reasoning": 0.7872, - "reward-bench/Prior Sets (0.5 weight)": 0.5033 - } - }, - { - "id": "mistralai/Mixtral-8x7B-v0.1", - "name": "Mixtral-8x7B-v0.1", - "developer": "mistralai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2326, - "hfopenllm_v2/BBH": 0.5098, - "hfopenllm_v2/MATH Level 5": 0.0937, - "hfopenllm_v2/GPQA": 0.3205, - "hfopenllm_v2/MUSR": 0.4413, - "hfopenllm_v2/MMLU-PRO": 0.3871 - } - }, - { - "id": "mistralai/open-mistral-nemo-2407", - "name": "Mistral NeMo 2402", - "developer": "mistralai", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.333, - "helm_lite/NarrativeQA": 0.731, - "helm_lite/NaturalQuestions (closed-book)": 0.265, - "helm_lite/OpenbookQA": 0.822, - "helm_lite/MMLU": 0.604, - "helm_lite/MATH": 0.668, - "helm_lite/GSM8K": 0.782, - "helm_lite/LegalBench": 0.415, - "helm_lite/MedQA": 0.59, - "helm_lite/WMT 2014": 0.177, - "helm_mmlu/MMLU All Subjects": 0.653, - "helm_mmlu/Abstract Algebra": 0.29, - "helm_mmlu/Anatomy": 0.607, - "helm_mmlu/College Physics": 0.373, - "helm_mmlu/Computer Security": 0.81, - "helm_mmlu/Econometrics": 0.561, - "helm_mmlu/Global Facts": 0.4, - "helm_mmlu/Jurisprudence": 0.796, - "helm_mmlu/Philosophy": 0.733, - "helm_mmlu/Professional Psychology": 0.588, - "helm_mmlu/Us Foreign Policy": 0.89, - "helm_mmlu/Astronomy": 0.691, - "helm_mmlu/Business Ethics": 0.49, - "helm_mmlu/Clinical Knowledge": 0.736, - "helm_mmlu/Conceptual Physics": 0.647, - "helm_mmlu/Electrical Engineering": 0.531, - "helm_mmlu/Elementary Mathematics": 0.439, - "helm_mmlu/Formal Logic": 0.405, - "helm_mmlu/High School World History": 0.848, - "helm_mmlu/Human Sexuality": 0.702, - "helm_mmlu/International Law": 0.769, - "helm_mmlu/Logical Fallacies": 0.791, - "helm_mmlu/Machine Learning": 0.402, - "helm_mmlu/Management": 0.796, - "helm_mmlu/Marketing": 0.889, - "helm_mmlu/Medical Genetics": 0.78, - "helm_mmlu/Miscellaneous": 0.861, - "helm_mmlu/Moral Scenarios": 0.381, - "helm_mmlu/Nutrition": 0.709, - "helm_mmlu/Prehistory": 0.765, - "helm_mmlu/Public Relations": 0.718, - "helm_mmlu/Security Studies": 0.771, - "helm_mmlu/Sociology": 0.726, - "helm_mmlu/Virology": 0.56, - "helm_mmlu/World Religions": 0.789, - "helm_mmlu/Mean win rate": 0.215 - } - }, - { - "id": "mistralai/open-mistral-nemo-2407-fc", - "name": "Open-Mistral-Nemo-2407 (FC)", - "developer": "mistralai", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 78.0, - "bfcl/bfcl.overall.overall_accuracy": 27.63, - "bfcl/bfcl.overall.total_cost_usd": 8.12, - "bfcl/bfcl.overall.latency_mean_s": 1.07, - "bfcl/bfcl.overall.latency_std_s": 11.93, - "bfcl/bfcl.overall.latency_p95_s": 1.39, - "bfcl/bfcl.non_live.ast_accuracy": 82.81, - "bfcl/bfcl.non_live.simple_ast_accuracy": 65.75, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 92.5, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 88.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 84.5, - "bfcl/bfcl.live.live_accuracy": 73.8, - "bfcl/bfcl.live.live_simple_ast_accuracy": 78.68, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 72.84, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 68.75, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 66.67, - "bfcl/bfcl.multi_turn.accuracy": 7.75, - "bfcl/bfcl.multi_turn.base_accuracy": 12.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 6.5, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 7.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 4.5, - "bfcl/bfcl.web_search.accuracy": 7.0, - "bfcl/bfcl.web_search.base_accuracy": 9.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 5.0, - "bfcl/bfcl.memory.accuracy": 10.32, - "bfcl/bfcl.memory.kv_accuracy": 8.39, - "bfcl/bfcl.memory.vector_accuracy": 9.68, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 12.9, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 81.25, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 61.77 - } - }, - { - "id": "mistralai/open-mistral-nemo-2407-prompt", - "name": "Open-Mistral-Nemo-2407 (Prompt)", - "developer": "mistralai", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 102.0, - "bfcl/bfcl.overall.overall_accuracy": 19.31, - "bfcl/bfcl.overall.total_cost_usd": 13.8, - "bfcl/bfcl.overall.latency_mean_s": 0.84, - "bfcl/bfcl.overall.latency_std_s": 7.05, - "bfcl/bfcl.overall.latency_p95_s": 1.32, - "bfcl/bfcl.non_live.ast_accuracy": 88.46, - "bfcl/bfcl.non_live.simple_ast_accuracy": 79.33, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 92.5, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 90.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 91.5, - "bfcl/bfcl.live.live_accuracy": 73.95, - "bfcl/bfcl.live.live_simple_ast_accuracy": 78.29, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 73.03, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 87.5, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 58.33, - "bfcl/bfcl.multi_turn.accuracy": 0.75, - "bfcl/bfcl.multi_turn.base_accuracy": 0.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 1.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 0.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 1.5, - "bfcl/bfcl.web_search.accuracy": 2.5, - "bfcl/bfcl.web_search.base_accuracy": 3.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 2.0, - "bfcl/bfcl.memory.accuracy": 8.6, - "bfcl/bfcl.memory.kv_accuracy": 9.68, - "bfcl/bfcl.memory.vector_accuracy": 9.68, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 6.45, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 93.75, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 6.28, - "bfcl/bfcl.format_sensitivity.max_delta": 14.5, - "bfcl/bfcl.format_sensitivity.stddev": 4.6 - } - }, - { - "id": "mixtao/MixTAO-7Bx2-MoE-v8.1", - "name": "MixTAO-7Bx2-MoE-v8.1", - "developer": "mixtao", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4162, - "hfopenllm_v2/BBH": 0.5189, - "hfopenllm_v2/MATH Level 5": 0.0906, - "hfopenllm_v2/GPQA": 0.2844, - "hfopenllm_v2/MUSR": 0.4463, - "hfopenllm_v2/MMLU-PRO": 0.3123 - } - }, - { - "id": "mkurman/llama-3.2-MEDIT-3B-o1", - "name": "llama-3.2-MEDIT-3B-o1", - "developer": "mkurman", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4382, - "hfopenllm_v2/BBH": 0.44, - "hfopenllm_v2/MATH Level 5": 0.1307, - "hfopenllm_v2/GPQA": 0.2659, - "hfopenllm_v2/MUSR": 0.3565, - "hfopenllm_v2/MMLU-PRO": 0.2741 - } - }, - { - "id": "mkurman/phi-4-MedIT-11B-exp-1", - "name": "phi-4-MedIT-11B-exp-1", - "developer": "mkurman", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5948, - "hfopenllm_v2/BBH": 0.5414, - "hfopenllm_v2/MATH Level 5": 0.0899, - "hfopenllm_v2/GPQA": 0.3012, - "hfopenllm_v2/MUSR": 0.3848, - "hfopenllm_v2/MMLU-PRO": 0.3825 - } - }, - { - "id": "mkurman/phi4-MedIT-10B-o1", - "name": "phi4-MedIT-10B-o1", - "developer": "mkurman", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3463, - "hfopenllm_v2/BBH": 0.5198, - "hfopenllm_v2/MATH Level 5": 0.1148, - "hfopenllm_v2/GPQA": 0.2458, - "hfopenllm_v2/MUSR": 0.3968, - "hfopenllm_v2/MMLU-PRO": 0.3507 - } - }, - { - "id": "mkxu/llama-3-8b-instruct-fpo", - "name": "llama-3-8b-instruct-fpo", - "developer": "mkxu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.679, - "hfopenllm_v2/BBH": 0.4959, - "hfopenllm_v2/MATH Level 5": 0.0733, - "hfopenllm_v2/GPQA": 0.2777, - "hfopenllm_v2/MUSR": 0.3658, - "hfopenllm_v2/MMLU-PRO": 0.3605 - } - }, - { - "id": "mkxu/llama-3-8b-po1", - "name": "llama-3-8b-po1", - "developer": "mkxu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4081, - "hfopenllm_v2/BBH": 0.4976, - "hfopenllm_v2/MATH Level 5": 0.0702, - "hfopenllm_v2/GPQA": 0.297, - "hfopenllm_v2/MUSR": 0.3804, - "hfopenllm_v2/MMLU-PRO": 0.3562 - } - }, - { - "id": "mlabonne/AlphaMonarch-7B", - "name": "AlphaMonarch-7B", - "developer": "mlabonne", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4939, - "hfopenllm_v2/BBH": 0.4626, - "hfopenllm_v2/MATH Level 5": 0.0408, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.4121, - "hfopenllm_v2/MMLU-PRO": 0.2473 - } - }, - { - "id": "mlabonne/Beyonder-4x7B-v3", - "name": "Beyonder-4x7B-v3", - "developer": "mlabonne", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5608, - "hfopenllm_v2/BBH": 0.4671, - "hfopenllm_v2/MATH Level 5": 0.0536, - "hfopenllm_v2/GPQA": 0.2852, - "hfopenllm_v2/MUSR": 0.4045, - "hfopenllm_v2/MMLU-PRO": 0.2512 - } - }, - { - "id": "mlabonne/BigQwen2.5-52B-Instruct", - "name": "BigQwen2.5-52B-Instruct", - "developer": "mlabonne", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7913, - "hfopenllm_v2/BBH": 0.7121, - "hfopenllm_v2/MATH Level 5": 0.5476, - "hfopenllm_v2/GPQA": 0.302, - "hfopenllm_v2/MUSR": 0.4113, - "hfopenllm_v2/MMLU-PRO": 0.5519 - } - }, - { - "id": "mlabonne/BigQwen2.5-Echo-47B-Instruct", - "name": "BigQwen2.5-Echo-47B-Instruct", - "developer": "mlabonne", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7357, - "hfopenllm_v2/BBH": 0.6125, - "hfopenllm_v2/MATH Level 5": 0.4381, - "hfopenllm_v2/GPQA": 0.3146, - "hfopenllm_v2/MUSR": 0.4125, - "hfopenllm_v2/MMLU-PRO": 0.4734 - } - }, - { - "id": "mlabonne/ChimeraLlama-3-8B-v2", - "name": "ChimeraLlama-3-8B-v2", - "developer": "mlabonne", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4469, - "hfopenllm_v2/BBH": 0.5046, - "hfopenllm_v2/MATH Level 5": 0.0906, - "hfopenllm_v2/GPQA": 0.2852, - "hfopenllm_v2/MUSR": 0.3791, - "hfopenllm_v2/MMLU-PRO": 0.3569 - } - }, - { - "id": "mlabonne/ChimeraLlama-3-8B-v3", - "name": "ChimeraLlama-3-8B-v3", - "developer": "mlabonne", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4408, - "hfopenllm_v2/BBH": 0.4978, - "hfopenllm_v2/MATH Level 5": 0.0884, - "hfopenllm_v2/GPQA": 0.2919, - "hfopenllm_v2/MUSR": 0.4004, - "hfopenllm_v2/MMLU-PRO": 0.3669 - } - }, - { - "id": "mlabonne/Daredevil-8B", - "name": "Daredevil-8B", - "developer": "mlabonne", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4548, - "hfopenllm_v2/BBH": 0.5194, - "hfopenllm_v2/MATH Level 5": 0.1065, - "hfopenllm_v2/GPQA": 0.3079, - "hfopenllm_v2/MUSR": 0.3939, - "hfopenllm_v2/MMLU-PRO": 0.3831 - } - }, - { - "id": "mlabonne/Daredevil-8B-abliterated", - "name": "Daredevil-8B-abliterated", - "developer": "mlabonne", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4426, - "hfopenllm_v2/BBH": 0.4254, - "hfopenllm_v2/MATH Level 5": 0.0944, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.407, - "hfopenllm_v2/MMLU-PRO": 0.3701 - } - }, - { - "id": "mlabonne/Hermes-3-Llama-3.1-70B-lorablated", - "name": "Hermes-3-Llama-3.1-70B-lorablated", - "developer": "mlabonne", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3424, - "hfopenllm_v2/BBH": 0.6693, - "hfopenllm_v2/MATH Level 5": 0.2243, - "hfopenllm_v2/GPQA": 0.3658, - "hfopenllm_v2/MUSR": 0.5029, - "hfopenllm_v2/MMLU-PRO": 0.4679 - } - }, - { - "id": "mlabonne/Meta-Llama-3.1-8B-Instruct-abliterated", - "name": "Meta-Llama-3.1-8B-Instruct-abliterated", - "developer": "mlabonne", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7329, - "hfopenllm_v2/BBH": 0.4874, - "hfopenllm_v2/MATH Level 5": 0.0687, - "hfopenllm_v2/GPQA": 0.2567, - "hfopenllm_v2/MUSR": 0.3649, - "hfopenllm_v2/MMLU-PRO": 0.3503 - } - }, - { - "id": "mlabonne/NeuralBeagle14-7B", - "name": "NeuralBeagle14-7B", - "developer": "mlabonne", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4935, - "hfopenllm_v2/BBH": 0.4628, - "hfopenllm_v2/MATH Level 5": 0.0521, - "hfopenllm_v2/GPQA": 0.2819, - "hfopenllm_v2/MUSR": 0.4319, - "hfopenllm_v2/MMLU-PRO": 0.2601 - } - }, - { - "id": "mlabonne/NeuralDaredevil-8B-abliterated", - "name": "NeuralDaredevil-8B-abliterated", - "developer": "mlabonne", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7561, - "hfopenllm_v2/BBH": 0.5111, - "hfopenllm_v2/MATH Level 5": 0.0906, - "hfopenllm_v2/GPQA": 0.3062, - "hfopenllm_v2/MUSR": 0.4019, - "hfopenllm_v2/MMLU-PRO": 0.3841 - } - }, - { - "id": "mlabonne/OrpoLlama-3-8B", - "name": "OrpoLlama-3-8B", - "developer": "mlabonne", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3653, - "hfopenllm_v2/BBH": 0.4424, - "hfopenllm_v2/MATH Level 5": 0.0559, - "hfopenllm_v2/GPQA": 0.2794, - "hfopenllm_v2/MUSR": 0.3579, - "hfopenllm_v2/MMLU-PRO": 0.2705 - } - }, - { - "id": "mlabonne/phixtral-2x2_8", - "name": "phixtral-2x2_8", - "developer": "mlabonne", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3431, - "hfopenllm_v2/BBH": 0.4889, - "hfopenllm_v2/MATH Level 5": 0.0355, - "hfopenllm_v2/GPQA": 0.2651, - "hfopenllm_v2/MUSR": 0.3644, - "hfopenllm_v2/MMLU-PRO": 0.2551 - } - }, - { - "id": "MLP-KTLim/llama-3-Korean-Bllossom-8B", - "name": "llama-3-Korean-Bllossom-8B", - "developer": "MLP-KTLim", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5113, - "hfopenllm_v2/BBH": 0.49, - "hfopenllm_v2/MATH Level 5": 0.102, - "hfopenllm_v2/GPQA": 0.2626, - "hfopenllm_v2/MUSR": 0.3675, - "hfopenllm_v2/MMLU-PRO": 0.3594 - } - }, - { - "id": "mlx-community/Josiefied-Qwen2.5-0.5B-Instruct-abliterated-v1-float32", - "name": "Josiefied-Qwen2.5-0.5B-Instruct-abliterated-v1-float32", - "developer": "mlx-community", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3369, - "hfopenllm_v2/BBH": 0.3292, - "hfopenllm_v2/MATH Level 5": 0.0846, - "hfopenllm_v2/GPQA": 0.2576, - "hfopenllm_v2/MUSR": 0.3249, - "hfopenllm_v2/MMLU-PRO": 0.1638 - } - }, - { - "id": "mlx-community/Mistral-Small-24B-Instruct-2501-bf16", - "name": "Mistral-Small-24B-Instruct-2501-bf16", - "developer": "mlx-community", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6283, - "hfopenllm_v2/BBH": 0.6713, - "hfopenllm_v2/MATH Level 5": 0.3225, - "hfopenllm_v2/GPQA": 0.3951, - "hfopenllm_v2/MUSR": 0.4618, - "hfopenllm_v2/MMLU-PRO": 0.5395 - } - }, - { - "id": "mmnga/Llama-3-70B-japanese-suzume-vector-v0.1", - "name": "Llama-3-70B-japanese-suzume-vector-v0.1", - "developer": "mmnga", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4649, - "hfopenllm_v2/BBH": 0.6542, - "hfopenllm_v2/MATH Level 5": 0.2326, - "hfopenllm_v2/GPQA": 0.2861, - "hfopenllm_v2/MUSR": 0.4141, - "hfopenllm_v2/MMLU-PRO": 0.5224 - } - }, - { - "id": "mobiuslabsgmbh/DeepSeek-R1-ReDistill-Llama3-8B-v1.1", - "name": "DeepSeek-R1-ReDistill-Llama3-8B-v1.1", - "developer": "mobiuslabsgmbh", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3704, - "hfopenllm_v2/BBH": 0.3473, - "hfopenllm_v2/MATH Level 5": 0.3285, - "hfopenllm_v2/GPQA": 0.271, - "hfopenllm_v2/MUSR": 0.3396, - "hfopenllm_v2/MMLU-PRO": 0.2198 - } - }, - { - "id": "mobiuslabsgmbh/DeepSeek-R1-ReDistill-Qwen-7B-v1.1", - "name": "DeepSeek-R1-ReDistill-Qwen-7B-v1.1", - "developer": "mobiuslabsgmbh", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3473, - "hfopenllm_v2/BBH": 0.3698, - "hfopenllm_v2/MATH Level 5": 0.3497, - "hfopenllm_v2/GPQA": 0.2651, - "hfopenllm_v2/MUSR": 0.4009, - "hfopenllm_v2/MMLU-PRO": 0.2326 - } - }, - { - "id": "ModelCloud/Llama-3.2-1B-Instruct-gptqmodel-4bit-vortex-v1", - "name": "Llama-3.2-1B-Instruct-gptqmodel-4bit-vortex-v1", - "developer": "ModelCloud", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5269, - "hfopenllm_v2/BBH": 0.3253, - "hfopenllm_v2/MATH Level 5": 0.0604, - "hfopenllm_v2/GPQA": 0.2534, - "hfopenllm_v2/MUSR": 0.3249, - "hfopenllm_v2/MMLU-PRO": 0.1764 - } - }, - { - "id": "ModelSpace/GemmaX2-28-9B-v0.1", - "name": "GemmaX2-28-9B-v0.1", - "developer": "ModelSpace", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0039, - "hfopenllm_v2/BBH": 0.3687, - "hfopenllm_v2/MATH Level 5": 0.0272, - "hfopenllm_v2/GPQA": 0.2768, - "hfopenllm_v2/MUSR": 0.3537, - "hfopenllm_v2/MMLU-PRO": 0.2231 - } - }, - { - "id": "moeru-ai/L3.1-Moe-2x8B-v0.2", - "name": "L3.1-Moe-2x8B-v0.2", - "developer": "moeru-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7348, - "hfopenllm_v2/BBH": 0.5256, - "hfopenllm_v2/MATH Level 5": 0.1699, - "hfopenllm_v2/GPQA": 0.3003, - "hfopenllm_v2/MUSR": 0.4199, - "hfopenllm_v2/MMLU-PRO": 0.3858 - } - }, - { - "id": "moeru-ai/L3.1-Moe-4x8B-v0.1", - "name": "L3.1-Moe-4x8B-v0.1", - "developer": "moeru-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4332, - "hfopenllm_v2/BBH": 0.4939, - "hfopenllm_v2/MATH Level 5": 0.1299, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3609, - "hfopenllm_v2/MMLU-PRO": 0.3454 - } - }, - { - "id": "moeru-ai/L3.1-Moe-4x8B-v0.2", - "name": "L3.1-Moe-4x8B-v0.2", - "developer": "moeru-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5407, - "hfopenllm_v2/BBH": 0.4466, - "hfopenllm_v2/MATH Level 5": 0.1035, - "hfopenllm_v2/GPQA": 0.2668, - "hfopenllm_v2/MUSR": 0.3234, - "hfopenllm_v2/MMLU-PRO": 0.2763 - } - }, - { - "id": "monsterapi/gemma-2-2b-LoRA-MonsterInstruct", - "name": "gemma-2-2b-LoRA-MonsterInstruct", - "developer": "monsterapi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3903, - "hfopenllm_v2/BBH": 0.365, - "hfopenllm_v2/MATH Level 5": 0.0506, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.3644, - "hfopenllm_v2/MMLU-PRO": 0.1987 - } - }, - { - "id": "monsterapi/Llama-3_1-8B-Instruct-orca-ORPO", - "name": "Llama-3_1-8B-Instruct-orca-ORPO", - "developer": "monsterapi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2273, - "hfopenllm_v2/BBH": 0.2865, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2492, - "hfopenllm_v2/MUSR": 0.3445, - "hfopenllm_v2/MMLU-PRO": 0.1168 - } - }, - { - "id": "MoonRide/Llama-3.2-3B-Khelavaster", - "name": "Llama-3.2-3B-Khelavaster", - "developer": "MoonRide", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4925, - "hfopenllm_v2/BBH": 0.4516, - "hfopenllm_v2/MATH Level 5": 0.1616, - "hfopenllm_v2/GPQA": 0.2777, - "hfopenllm_v2/MUSR": 0.3699, - "hfopenllm_v2/MMLU-PRO": 0.3122 - } - }, - { - "id": "moonshot-ai/kimi-k2-instruct", - "name": "Kimi K2 Instruct", - "developer": "Moonshot AI", - "evaluator_relationship": null, - "benchmark_scores": { - "terminal-bench-2.0/terminal-bench-2.0": 27.8 - } - }, - { - "id": "moonshot-ai/kimi-k2-thinking", - "name": "Kimi K2 Thinking", - "developer": "Moonshot AI", - "evaluator_relationship": null, - "benchmark_scores": { - "terminal-bench-2.0/terminal-bench-2.0": 35.7 - } - }, - { - "id": "moonshot-ai/kimi-k2.5", - "name": "Kimi K2.5", - "developer": "Kimi", - "evaluator_relationship": null, - "benchmark_scores": { - "terminal-bench-2.0/terminal-bench-2.0": 43.2 - } - }, - { - "id": "moonshot/Kimi K2 Thinking", - "name": "Kimi K2 Thinking", - "developer": "moonshot", - "evaluator_relationship": null, - "benchmark_scores": { - "apex-agents/Overall Pass@1": 0.04, - "apex-agents/Overall Pass@8": 0.144, - "apex-agents/Overall Mean Score": 0.115, - "apex-agents/Investment Banking Pass@1": 0.012, - "apex-agents/Management Consulting Pass@1": 0.029, - "apex-agents/Corporate Law Pass@1": 0.08, - "apex-agents/Corporate Lawyer Mean Score": 0.223 - } - }, - { - "id": "moonshot/Kimi K2.5", - "name": "Kimi K2.5", - "developer": "moonshot", - "evaluator_relationship": null, - "benchmark_scores": { - "apex-agents/Corporate Lawyer Mean Score": 0.402 - } - }, - { - "id": "moonshotai/kimi-k2-instruct", - "name": "Kimi K2 Instruct", - "developer": "moonshotai", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.768, - "helm_capabilities/MMLU-Pro": 0.819, - "helm_capabilities/GPQA": 0.652, - "helm_capabilities/IFEval": 0.85, - "helm_capabilities/WildBench": 0.862, - "helm_capabilities/Omni-MATH": 0.654 - } - }, - { - "id": "moonshotai/moonshotai-kimi-k2-instruct-fc", - "name": "Moonshotai-Kimi-K2-Instruct (FC)", - "developer": "moonshotai", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 11.0, - "bfcl/bfcl.overall.overall_accuracy": 59.06, - "bfcl/bfcl.overall.total_cost_usd": 6.19, - "bfcl/bfcl.overall.latency_mean_s": 6.4, - "bfcl/bfcl.overall.latency_std_s": 9.38, - "bfcl/bfcl.overall.latency_p95_s": 13.78, - "bfcl/bfcl.non_live.ast_accuracy": 81.6, - "bfcl/bfcl.non_live.simple_ast_accuracy": 69.42, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 92.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 82.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 83.0, - "bfcl/bfcl.live.live_accuracy": 78.68, - "bfcl/bfcl.live.live_simple_ast_accuracy": 81.78, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 78.06, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 87.5, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 66.67, - "bfcl/bfcl.multi_turn.accuracy": 50.63, - "bfcl/bfcl.multi_turn.base_accuracy": 62.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 41.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 44.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 55.0, - "bfcl/bfcl.web_search.accuracy": 66.5, - "bfcl/bfcl.web_search.base_accuracy": 72.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 61.0, - "bfcl/bfcl.memory.accuracy": 29.03, - "bfcl/bfcl.memory.kv_accuracy": 21.94, - "bfcl/bfcl.memory.vector_accuracy": 20.0, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 45.16, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 75.0, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 87.34 - } - }, - { - "id": "mosaicml/MPT-30B", - "name": "MPT 30B", - "developer": "mosaicml", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.714, - "helm_classic/MMLU": 0.437, - "helm_classic/BoolQ": 0.704, - "helm_classic/NarrativeQA": 0.732, - "helm_classic/NaturalQuestions (open-book)": 0.673, - "helm_classic/QuAC": 0.393, - "helm_classic/HellaSwag": -1.0, - "helm_classic/OpenbookQA": -1.0, - "helm_classic/TruthfulQA": 0.231, - "helm_classic/MS MARCO (TREC)": -1.0, - "helm_classic/CNN/DailyMail": -1.0, - "helm_classic/XSUM": -1.0, - "helm_classic/IMDB": 0.959, - "helm_classic/CivilComments": 0.599, - "helm_classic/RAFT": 0.723 - } - }, - { - "id": "mosaicml/mpt-7b", - "name": "mpt-7b", - "developer": "mosaicml", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2152, - "hfopenllm_v2/BBH": 0.33, - "hfopenllm_v2/MATH Level 5": 0.0159, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3672, - "hfopenllm_v2/MMLU-PRO": 0.1206 - } - }, - { - "id": "mosaicml/MPT-Instruct-30B", - "name": "MPT-Instruct 30B", - "developer": "mosaicml", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.716, - "helm_classic/MMLU": 0.444, - "helm_classic/BoolQ": 0.85, - "helm_classic/NarrativeQA": 0.733, - "helm_classic/NaturalQuestions (open-book)": 0.697, - "helm_classic/QuAC": 0.327, - "helm_classic/HellaSwag": -1.0, - "helm_classic/OpenbookQA": -1.0, - "helm_classic/TruthfulQA": 0.234, - "helm_classic/MS MARCO (TREC)": -1.0, - "helm_classic/CNN/DailyMail": -1.0, - "helm_classic/XSUM": -1.0, - "helm_classic/IMDB": 0.956, - "helm_classic/CivilComments": 0.573, - "helm_classic/RAFT": 0.68 - } - }, - { - "id": "mosama/Qwen2.5-1.5B-Instruct-CoT-Reflection", - "name": "Qwen2.5-1.5B-Instruct-CoT-Reflection", - "developer": "mosama", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.287, - "hfopenllm_v2/BBH": 0.4109, - "hfopenllm_v2/MATH Level 5": 0.0272, - "hfopenllm_v2/GPQA": 0.2617, - "hfopenllm_v2/MUSR": 0.3212, - "hfopenllm_v2/MMLU-PRO": 0.2651 - } - }, - { - "id": "Mostafa8Mehrabi/llama-3.2-1b-Insomnia-ChatBot-merged", - "name": "llama-3.2-1b-Insomnia-ChatBot-merged", - "developer": "Mostafa8Mehrabi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1321, - "hfopenllm_v2/BBH": 0.3004, - "hfopenllm_v2/MATH Level 5": 0.0076, - "hfopenllm_v2/GPQA": 0.2366, - "hfopenllm_v2/MUSR": 0.3382, - "hfopenllm_v2/MMLU-PRO": 0.1131 - } - }, - { - "id": "mrdayl/OpenCogito", - "name": "OpenCogito", - "developer": "mrdayl", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3934, - "hfopenllm_v2/BBH": 0.472, - "hfopenllm_v2/MATH Level 5": 0.2183, - "hfopenllm_v2/GPQA": 0.3003, - "hfopenllm_v2/MUSR": 0.424, - "hfopenllm_v2/MMLU-PRO": 0.3452 - } - }, - { - "id": "mrdayl/OpenCognito", - "name": "OpenCognito", - "developer": "mrdayl", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4062, - "hfopenllm_v2/BBH": 0.4706, - "hfopenllm_v2/MATH Level 5": 0.2115, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.4293, - "hfopenllm_v2/MMLU-PRO": 0.3443 - } - }, - { - "id": "mrdayl/OpenCognito-r1", - "name": "OpenCognito-r1", - "developer": "mrdayl", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4241, - "hfopenllm_v2/BBH": 0.4673, - "hfopenllm_v2/MATH Level 5": 0.1903, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.4241, - "hfopenllm_v2/MMLU-PRO": 0.3475 - } - }, - { - "id": "mrdayl/OpenCognito-r2", - "name": "OpenCognito-r2", - "developer": "mrdayl", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3959, - "hfopenllm_v2/BBH": 0.4688, - "hfopenllm_v2/MATH Level 5": 0.2024, - "hfopenllm_v2/GPQA": 0.3062, - "hfopenllm_v2/MUSR": 0.4202, - "hfopenllm_v2/MMLU-PRO": 0.3462 - } - }, - { - "id": "mrdayl/OpenThink", - "name": "OpenThink", - "developer": "mrdayl", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2054, - "hfopenllm_v2/BBH": 0.346, - "hfopenllm_v2/MATH Level 5": 0.2885, - "hfopenllm_v2/GPQA": 0.2827, - "hfopenllm_v2/MUSR": 0.3289, - "hfopenllm_v2/MMLU-PRO": 0.185 - } - }, - { - "id": "mrm8488/phi-4-14B-grpo-gsm8k-3e", - "name": "phi-4-14B-grpo-gsm8k-3e", - "developer": "mrm8488", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6885, - "hfopenllm_v2/BBH": 0.6805, - "hfopenllm_v2/MATH Level 5": 0.4524, - "hfopenllm_v2/GPQA": 0.3356, - "hfopenllm_v2/MUSR": 0.3994, - "hfopenllm_v2/MMLU-PRO": 0.5268 - } - }, - { - "id": "mrm8488/phi-4-14B-grpo-limo", - "name": "phi-4-14B-grpo-limo", - "developer": "mrm8488", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6812, - "hfopenllm_v2/BBH": 0.6785, - "hfopenllm_v2/MATH Level 5": 0.4569, - "hfopenllm_v2/GPQA": 0.3364, - "hfopenllm_v2/MUSR": 0.3981, - "hfopenllm_v2/MMLU-PRO": 0.5261 - } - }, - { - "id": "MrRobotoAI/MrRoboto-ProLong-8b-v4i", - "name": "MrRoboto-ProLong-8b-v4i", - "developer": "MrRobotoAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3835, - "hfopenllm_v2/BBH": 0.4585, - "hfopenllm_v2/MATH Level 5": 0.0551, - "hfopenllm_v2/GPQA": 0.2894, - "hfopenllm_v2/MUSR": 0.4014, - "hfopenllm_v2/MMLU-PRO": 0.3068 - } - }, - { - "id": "MrRobotoAI/MrRoboto-ProLongBASE-pt8-unaligned-8b", - "name": "MrRoboto-ProLongBASE-pt8-unaligned-8b", - "developer": "MrRobotoAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3475, - "hfopenllm_v2/BBH": 0.4515, - "hfopenllm_v2/MATH Level 5": 0.0423, - "hfopenllm_v2/GPQA": 0.281, - "hfopenllm_v2/MUSR": 0.4279, - "hfopenllm_v2/MMLU-PRO": 0.2566 - } - }, - { - "id": "MTSAIR/Cotype-Nano", - "name": "Cotype-Nano", - "developer": "MTSAIR", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3748, - "hfopenllm_v2/BBH": 0.3865, - "hfopenllm_v2/MATH Level 5": 0.0974, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.3289, - "hfopenllm_v2/MMLU-PRO": 0.2477 - } - }, - { - "id": "MTSAIR/MultiVerse_70B", - "name": "MultiVerse_70B", - "developer": "MTSAIR", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5249, - "hfopenllm_v2/BBH": 0.6183, - "hfopenllm_v2/MATH Level 5": 0.1926, - "hfopenllm_v2/GPQA": 0.354, - "hfopenllm_v2/MUSR": 0.474, - "hfopenllm_v2/MMLU-PRO": 0.486 - } - }, - { - "id": "mukaj/Llama-3.1-Hawkish-8B", - "name": "Llama-3.1-Hawkish-8B", - "developer": "mukaj", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.672, - "hfopenllm_v2/BBH": 0.4884, - "hfopenllm_v2/MATH Level 5": 0.2432, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.3967, - "hfopenllm_v2/MMLU-PRO": 0.3331 - } - }, - { - "id": "multiple/multiple", - "name": "Multiple", - "developer": "Multiple", - "evaluator_relationship": null, - "benchmark_scores": { - "terminal-bench-2.0/terminal-bench-2.0": 59.1 - } - }, - { - "id": "MultivexAI/Gladiator-Mini-Exp-1211-3B", - "name": "Gladiator-Mini-Exp-1211-3B", - "developer": "MultivexAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6876, - "hfopenllm_v2/BBH": 0.4484, - "hfopenllm_v2/MATH Level 5": 0.1375, - "hfopenllm_v2/GPQA": 0.2727, - "hfopenllm_v2/MUSR": 0.326, - "hfopenllm_v2/MMLU-PRO": 0.3152 - } - }, - { - "id": "MultivexAI/Gladiator-Mini-Exp-1221-3B-Instruct", - "name": "Gladiator-Mini-Exp-1221-3B-Instruct", - "developer": "MultivexAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6079, - "hfopenllm_v2/BBH": 0.437, - "hfopenllm_v2/MATH Level 5": 0.1352, - "hfopenllm_v2/GPQA": 0.2634, - "hfopenllm_v2/MUSR": 0.3115, - "hfopenllm_v2/MMLU-PRO": 0.3049 - } - }, - { - "id": "MultivexAI/Gladiator-Mini-Exp-1221-3B-Instruct-V2", - "name": "Gladiator-Mini-Exp-1221-3B-Instruct-V2", - "developer": "MultivexAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6215, - "hfopenllm_v2/BBH": 0.4389, - "hfopenllm_v2/MATH Level 5": 0.1412, - "hfopenllm_v2/GPQA": 0.2634, - "hfopenllm_v2/MUSR": 0.3008, - "hfopenllm_v2/MMLU-PRO": 0.3025 - } - }, - { - "id": "MultivexAI/Gladiator-Mini-Exp-1222-3B-Instruct", - "name": "Gladiator-Mini-Exp-1222-3B-Instruct", - "developer": "MultivexAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6163, - "hfopenllm_v2/BBH": 0.4373, - "hfopenllm_v2/MATH Level 5": 0.1412, - "hfopenllm_v2/GPQA": 0.2634, - "hfopenllm_v2/MUSR": 0.3128, - "hfopenllm_v2/MMLU-PRO": 0.3017 - } - }, - { - "id": "MultivexAI/Phi-3.5-Mini-Instruct-MultiVex-v0.25-GGUF", - "name": "Phi-3.5-Mini-Instruct-MultiVex-v0.25-GGUF", - "developer": "MultivexAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.144, - "hfopenllm_v2/BBH": 0.2908, - "hfopenllm_v2/MATH Level 5": 0.006, - "hfopenllm_v2/GPQA": 0.255, - "hfopenllm_v2/MUSR": 0.3642, - "hfopenllm_v2/MMLU-PRO": 0.1109 - } - }, - { - "id": "Mxode/NanoLM-0.3B-Instruct-v1", - "name": "NanoLM-0.3B-Instruct-v1", - "developer": "Mxode", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1537, - "hfopenllm_v2/BBH": 0.3028, - "hfopenllm_v2/MATH Level 5": 0.0144, - "hfopenllm_v2/GPQA": 0.2718, - "hfopenllm_v2/MUSR": 0.4155, - "hfopenllm_v2/MMLU-PRO": 0.1105 - } - }, - { - "id": "Mxode/NanoLM-0.3B-Instruct-v1.1", - "name": "NanoLM-0.3B-Instruct-v1.1", - "developer": "Mxode", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1783, - "hfopenllm_v2/BBH": 0.3014, - "hfopenllm_v2/MATH Level 5": 0.0136, - "hfopenllm_v2/GPQA": 0.25, - "hfopenllm_v2/MUSR": 0.4273, - "hfopenllm_v2/MMLU-PRO": 0.1121 - } - }, - { - "id": "Mxode/NanoLM-0.3B-Instruct-v2", - "name": "NanoLM-0.3B-Instruct-v2", - "developer": "Mxode", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1668, - "hfopenllm_v2/BBH": 0.2921, - "hfopenllm_v2/MATH Level 5": 0.0068, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.3955, - "hfopenllm_v2/MMLU-PRO": 0.1134 - } - }, - { - "id": "Mxode/NanoLM-1B-Instruct-v1.1", - "name": "NanoLM-1B-Instruct-v1.1", - "developer": "Mxode", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2395, - "hfopenllm_v2/BBH": 0.3184, - "hfopenllm_v2/MATH Level 5": 0.0363, - "hfopenllm_v2/GPQA": 0.2634, - "hfopenllm_v2/MUSR": 0.3433, - "hfopenllm_v2/MMLU-PRO": 0.1215 - } - }, - { - "id": "Mxode/NanoLM-1B-Instruct-v2", - "name": "NanoLM-1B-Instruct-v2", - "developer": "Mxode", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.263, - "hfopenllm_v2/BBH": 0.3123, - "hfopenllm_v2/MATH Level 5": 0.0415, - "hfopenllm_v2/GPQA": 0.2634, - "hfopenllm_v2/MUSR": 0.3552, - "hfopenllm_v2/MMLU-PRO": 0.1238 - } - }, - { - "id": "my_model/", - "name": "my_model/", - "developer": "my_model", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5267, - "reward-bench/Chat": 0.4553, - "reward-bench/Chat Hard": 0.5592, - "reward-bench/Safety": 0.4392, - "reward-bench/Reasoning": 0.6532 - } - }, - { - "id": "nanbeige/nanbeige3-5-pro-thinking-fc", - "name": "Nanbeige3.5-Pro-Thinking (FC)", - "developer": "nanbeige", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 32.0, - "bfcl/bfcl.overall.overall_accuracy": 47.68, - "bfcl/bfcl.overall.total_cost_usd": 23.46, - "bfcl/bfcl.overall.latency_mean_s": 21.12, - "bfcl/bfcl.overall.latency_std_s": 28.61, - "bfcl/bfcl.overall.latency_p95_s": 63.29, - "bfcl/bfcl.non_live.ast_accuracy": 38.35, - "bfcl/bfcl.non_live.simple_ast_accuracy": 43.92, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 36.5, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 53.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 20.0, - "bfcl/bfcl.live.live_accuracy": 69.95, - "bfcl/bfcl.live.live_simple_ast_accuracy": 63.18, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 71.42, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 87.5, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 66.67, - "bfcl/bfcl.multi_turn.accuracy": 40.0, - "bfcl/bfcl.multi_turn.base_accuracy": 56.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 34.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 29.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 41.0, - "bfcl/bfcl.web_search.accuracy": 42.0, - "bfcl/bfcl.web_search.base_accuracy": 47.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 37.0, - "bfcl/bfcl.memory.accuracy": 45.16, - "bfcl/bfcl.memory.kv_accuracy": 38.06, - "bfcl/bfcl.memory.vector_accuracy": 58.06, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 39.35, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 100.0, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 74.2 - } - }, - { - "id": "nanbeige/nanbeige4-3b-thinking-2511-fc", - "name": "Nanbeige4-3B-Thinking-2511 (FC)", - "developer": "nanbeige", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 25.0, - "bfcl/bfcl.overall.overall_accuracy": 51.4, - "bfcl/bfcl.overall.total_cost_usd": 14.14, - "bfcl/bfcl.overall.latency_mean_s": 13.46, - "bfcl/bfcl.overall.latency_std_s": 26.41, - "bfcl/bfcl.overall.latency_p95_s": 37.45, - "bfcl/bfcl.non_live.ast_accuracy": 81.58, - "bfcl/bfcl.non_live.simple_ast_accuracy": 63.83, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 93.5, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 84.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 84.5, - "bfcl/bfcl.live.live_accuracy": 79.42, - "bfcl/bfcl.live.live_simple_ast_accuracy": 86.05, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 78.06, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 75.0, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 70.83, - "bfcl/bfcl.multi_turn.accuracy": 51.12, - "bfcl/bfcl.multi_turn.base_accuracy": 58.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 54.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 45.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 47.0, - "bfcl/bfcl.web_search.accuracy": 21.5, - "bfcl/bfcl.web_search.base_accuracy": 31.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 12.0, - "bfcl/bfcl.memory.accuracy": 36.77, - "bfcl/bfcl.memory.kv_accuracy": 31.61, - "bfcl/bfcl.memory.vector_accuracy": 34.19, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 44.52, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 75.0, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 83.09 - } - }, - { - "id": "NAPS-ai/naps-gemma-2-27b-v-0.1.0", - "name": "naps-gemma-2-27b-v-0.1.0", - "developer": "NAPS-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0, - "hfopenllm_v2/BBH": 0.2912, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3575, - "hfopenllm_v2/MMLU-PRO": 0.1168 - } - }, - { - "id": "NAPS-ai/naps-gemma-2-27b-v0.1.0", - "name": "naps-gemma-2-27b-v0.1.0", - "developer": "NAPS-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0, - "hfopenllm_v2/BBH": 0.2912, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3575, - "hfopenllm_v2/MMLU-PRO": 0.1168 - } - }, - { - "id": "NAPS-ai/naps-llama-3_1-8b-instruct-v0.3", - "name": "naps-llama-3_1-8b-instruct-v0.3", - "developer": "NAPS-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5391, - "hfopenllm_v2/BBH": 0.4901, - "hfopenllm_v2/MATH Level 5": 0.1903, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.3787, - "hfopenllm_v2/MMLU-PRO": 0.3398 - } - }, - { - "id": "NAPS-ai/naps-llama-3_1-8b-instruct-v0.4", - "name": "naps-llama-3_1-8b-instruct-v0.4", - "developer": "NAPS-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7344, - "hfopenllm_v2/BBH": 0.4862, - "hfopenllm_v2/MATH Level 5": 0.1964, - "hfopenllm_v2/GPQA": 0.2794, - "hfopenllm_v2/MUSR": 0.4421, - "hfopenllm_v2/MMLU-PRO": 0.3475 - } - }, - { - "id": "NAPS-ai/naps-llama-3_1-instruct-v0.5.0", - "name": "naps-llama-3_1-instruct-v0.5.0", - "developer": "NAPS-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.502, - "hfopenllm_v2/BBH": 0.4148, - "hfopenllm_v2/MATH Level 5": 0.0363, - "hfopenllm_v2/GPQA": 0.2685, - "hfopenllm_v2/MUSR": 0.3713, - "hfopenllm_v2/MMLU-PRO": 0.2614 - } - }, - { - "id": "NAPS-ai/naps-llama-3_1_instruct-v0.6.0", - "name": "naps-llama-3_1_instruct-v0.6.0", - "developer": "NAPS-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.328, - "hfopenllm_v2/BBH": 0.4528, - "hfopenllm_v2/MATH Level 5": 0.0642, - "hfopenllm_v2/GPQA": 0.2819, - "hfopenllm_v2/MUSR": 0.3739, - "hfopenllm_v2/MMLU-PRO": 0.3241 - } - }, - { - "id": "NAPS-ai/naps-llama3.1-70B-v0.2-fp16", - "name": "naps-llama3.1-70B-v0.2-fp16", - "developer": "NAPS-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1845, - "hfopenllm_v2/BBH": 0.3041, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2391, - "hfopenllm_v2/MUSR": 0.3486, - "hfopenllm_v2/MMLU-PRO": 0.1099 - } - }, - { - "id": "natong19/Mistral-Nemo-Instruct-2407-abliterated", - "name": "Mistral-Nemo-Instruct-2407-abliterated", - "developer": "natong19", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6392, - "hfopenllm_v2/BBH": 0.5048, - "hfopenllm_v2/MATH Level 5": 0.1322, - "hfopenllm_v2/GPQA": 0.2869, - "hfopenllm_v2/MUSR": 0.4033, - "hfopenllm_v2/MMLU-PRO": 0.3518 - } - }, - { - "id": "natong19/Qwen2-7B-Instruct-abliterated", - "name": "Qwen2-7B-Instruct-abliterated", - "developer": "natong19", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5837, - "hfopenllm_v2/BBH": 0.5553, - "hfopenllm_v2/MATH Level 5": 0.2764, - "hfopenllm_v2/GPQA": 0.3012, - "hfopenllm_v2/MUSR": 0.4034, - "hfopenllm_v2/MMLU-PRO": 0.3842 - } - }, - { - "id": "Naveenpoliasetty/llama3-8B-V2", - "name": "llama3-8B-V2", - "developer": "Naveenpoliasetty", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4123, - "hfopenllm_v2/BBH": 0.5189, - "hfopenllm_v2/MATH Level 5": 0.0785, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.4081, - "hfopenllm_v2/MMLU-PRO": 0.3738 - } - }, - { - "id": "nazimali/Mistral-Nemo-Kurdish", - "name": "Mistral-Nemo-Kurdish", - "developer": "nazimali", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3401, - "hfopenllm_v2/BBH": 0.5133, - "hfopenllm_v2/MATH Level 5": 0.0959, - "hfopenllm_v2/GPQA": 0.3012, - "hfopenllm_v2/MUSR": 0.4116, - "hfopenllm_v2/MMLU-PRO": 0.3235 - } - }, - { - "id": "nazimali/Mistral-Nemo-Kurdish-Instruct", - "name": "Mistral-Nemo-Kurdish-Instruct", - "developer": "nazimali", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.486, - "hfopenllm_v2/BBH": 0.4721, - "hfopenllm_v2/MATH Level 5": 0.0846, - "hfopenllm_v2/GPQA": 0.2844, - "hfopenllm_v2/MUSR": 0.4006, - "hfopenllm_v2/MMLU-PRO": 0.3087 - } - }, - { - "id": "NbAiLab/nb-llama-3.1-8B-Instruct", - "name": "nb-llama-3.1-8B-Instruct", - "developer": "NbAiLab", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3625, - "hfopenllm_v2/BBH": 0.3247, - "hfopenllm_v2/MATH Level 5": 0.0227, - "hfopenllm_v2/GPQA": 0.2735, - "hfopenllm_v2/MUSR": 0.3208, - "hfopenllm_v2/MMLU-PRO": 0.1197 - } - }, - { - "id": "NbAiLab/nb-llama-3.1-8B-sft", - "name": "nb-llama-3.1-8B-sft", - "developer": "NbAiLab", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3616, - "hfopenllm_v2/BBH": 0.3282, - "hfopenllm_v2/MATH Level 5": 0.0219, - "hfopenllm_v2/GPQA": 0.2542, - "hfopenllm_v2/MUSR": 0.3287, - "hfopenllm_v2/MMLU-PRO": 0.1222 - } - }, - { - "id": "nbeerbower/BigKartoffel-mistral-nemo-20B", - "name": "BigKartoffel-mistral-nemo-20B", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5857, - "hfopenllm_v2/BBH": 0.5515, - "hfopenllm_v2/MATH Level 5": 0.0264, - "hfopenllm_v2/GPQA": 0.2869, - "hfopenllm_v2/MUSR": 0.428, - "hfopenllm_v2/MMLU-PRO": 0.353 - } - }, - { - "id": "nbeerbower/DoppelKartoffel-Mistral-Nemo-23B", - "name": "DoppelKartoffel-Mistral-Nemo-23B", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5191, - "hfopenllm_v2/BBH": 0.5218, - "hfopenllm_v2/MATH Level 5": 0.031, - "hfopenllm_v2/GPQA": 0.2752, - "hfopenllm_v2/MUSR": 0.3795, - "hfopenllm_v2/MMLU-PRO": 0.308 - } - }, - { - "id": "nbeerbower/DoublePotato-Mistral-Nemo-13B", - "name": "DoublePotato-Mistral-Nemo-13B", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6796, - "hfopenllm_v2/BBH": 0.5438, - "hfopenllm_v2/MATH Level 5": 0.04, - "hfopenllm_v2/GPQA": 0.3012, - "hfopenllm_v2/MUSR": 0.46, - "hfopenllm_v2/MMLU-PRO": 0.3596 - } - }, - { - "id": "nbeerbower/Dumpling-Qwen2.5-1.5B", - "name": "Dumpling-Qwen2.5-1.5B", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3699, - "hfopenllm_v2/BBH": 0.416, - "hfopenllm_v2/MATH Level 5": 0.1171, - "hfopenllm_v2/GPQA": 0.2685, - "hfopenllm_v2/MUSR": 0.3728, - "hfopenllm_v2/MMLU-PRO": 0.2772 - } - }, - { - "id": "nbeerbower/Dumpling-Qwen2.5-14B", - "name": "Dumpling-Qwen2.5-14B", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6064, - "hfopenllm_v2/BBH": 0.6451, - "hfopenllm_v2/MATH Level 5": 0.3097, - "hfopenllm_v2/GPQA": 0.3012, - "hfopenllm_v2/MUSR": 0.4354, - "hfopenllm_v2/MMLU-PRO": 0.517 - } - }, - { - "id": "nbeerbower/Dumpling-Qwen2.5-7B-1k-r16", - "name": "Dumpling-Qwen2.5-7B-1k-r16", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.486, - "hfopenllm_v2/BBH": 0.5214, - "hfopenllm_v2/MATH Level 5": 0.2364, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.423, - "hfopenllm_v2/MMLU-PRO": 0.3959 - } - }, - { - "id": "nbeerbower/Dumpling-Qwen2.5-7B-1k-r64-2e-5", - "name": "Dumpling-Qwen2.5-7B-1k-r64-2e-5", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4179, - "hfopenllm_v2/BBH": 0.5301, - "hfopenllm_v2/MATH Level 5": 0.2115, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.4486, - "hfopenllm_v2/MMLU-PRO": 0.4122 - } - }, - { - "id": "nbeerbower/EVA-abliterated-TIES-Qwen2.5-1.5B", - "name": "EVA-abliterated-TIES-Qwen2.5-1.5B", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4115, - "hfopenllm_v2/BBH": 0.3997, - "hfopenllm_v2/MATH Level 5": 0.1375, - "hfopenllm_v2/GPQA": 0.2651, - "hfopenllm_v2/MUSR": 0.3502, - "hfopenllm_v2/MMLU-PRO": 0.2712 - } - }, - { - "id": "nbeerbower/EVA-abliterated-TIES-Qwen2.5-14B", - "name": "EVA-abliterated-TIES-Qwen2.5-14B", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7836, - "hfopenllm_v2/BBH": 0.6372, - "hfopenllm_v2/MATH Level 5": 0.5045, - "hfopenllm_v2/GPQA": 0.3549, - "hfopenllm_v2/MUSR": 0.4407, - "hfopenllm_v2/MMLU-PRO": 0.5211 - } - }, - { - "id": "nbeerbower/Flammades-Mistral-Nemo-12B", - "name": "Flammades-Mistral-Nemo-12B", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3842, - "hfopenllm_v2/BBH": 0.53, - "hfopenllm_v2/MATH Level 5": 0.0755, - "hfopenllm_v2/GPQA": 0.3037, - "hfopenllm_v2/MUSR": 0.4806, - "hfopenllm_v2/MMLU-PRO": 0.3661 - } - }, - { - "id": "nbeerbower/gemma2-gutenberg-27B", - "name": "gemma2-gutenberg-27B", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2947, - "hfopenllm_v2/BBH": 0.3797, - "hfopenllm_v2/MATH Level 5": 0.0189, - "hfopenllm_v2/GPQA": 0.2727, - "hfopenllm_v2/MUSR": 0.3727, - "hfopenllm_v2/MMLU-PRO": 0.1982 - } - }, - { - "id": "nbeerbower/gemma2-gutenberg-9B", - "name": "gemma2-gutenberg-9B", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2796, - "hfopenllm_v2/BBH": 0.5951, - "hfopenllm_v2/MATH Level 5": 0.0808, - "hfopenllm_v2/GPQA": 0.3381, - "hfopenllm_v2/MUSR": 0.4595, - "hfopenllm_v2/MMLU-PRO": 0.4192 - } - }, - { - "id": "nbeerbower/Gemma2-Gutenberg-Doppel-9B", - "name": "Gemma2-Gutenberg-Doppel-9B", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7171, - "hfopenllm_v2/BBH": 0.587, - "hfopenllm_v2/MATH Level 5": 0.1979, - "hfopenllm_v2/GPQA": 0.3297, - "hfopenllm_v2/MUSR": 0.4608, - "hfopenllm_v2/MMLU-PRO": 0.4127 - } - }, - { - "id": "nbeerbower/Gutensuppe-mistral-nemo-12B", - "name": "Gutensuppe-mistral-nemo-12B", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2916, - "hfopenllm_v2/BBH": 0.5487, - "hfopenllm_v2/MATH Level 5": 0.1329, - "hfopenllm_v2/GPQA": 0.3372, - "hfopenllm_v2/MUSR": 0.429, - "hfopenllm_v2/MMLU-PRO": 0.368 - } - }, - { - "id": "nbeerbower/Hermes2-Gutenberg2-Mistral-7B", - "name": "Hermes2-Gutenberg2-Mistral-7B", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3721, - "hfopenllm_v2/BBH": 0.4981, - "hfopenllm_v2/MATH Level 5": 0.0574, - "hfopenllm_v2/GPQA": 0.2894, - "hfopenllm_v2/MUSR": 0.4623, - "hfopenllm_v2/MMLU-PRO": 0.2993 - } - }, - { - "id": "nbeerbower/Kartoffel-Deepfry-12B", - "name": "Kartoffel-Deepfry-12B", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5022, - "hfopenllm_v2/BBH": 0.5365, - "hfopenllm_v2/MATH Level 5": 0.0604, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.4792, - "hfopenllm_v2/MMLU-PRO": 0.3582 - } - }, - { - "id": "nbeerbower/llama-3-gutenberg-8B", - "name": "llama-3-gutenberg-8B", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4372, - "hfopenllm_v2/BBH": 0.4994, - "hfopenllm_v2/MATH Level 5": 0.0785, - "hfopenllm_v2/GPQA": 0.3012, - "hfopenllm_v2/MUSR": 0.4073, - "hfopenllm_v2/MMLU-PRO": 0.3831 - } - }, - { - "id": "nbeerbower/Llama-3.1-Nemotron-lorablated-70B", - "name": "Llama-3.1-Nemotron-lorablated-70B", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7229, - "hfopenllm_v2/BBH": 0.6825, - "hfopenllm_v2/MATH Level 5": 0.3338, - "hfopenllm_v2/GPQA": 0.3909, - "hfopenllm_v2/MUSR": 0.4682, - "hfopenllm_v2/MMLU-PRO": 0.5343 - } - }, - { - "id": "nbeerbower/llama3.1-cc-8B", - "name": "llama3.1-cc-8B", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5068, - "hfopenllm_v2/BBH": 0.4871, - "hfopenllm_v2/MATH Level 5": 0.071, - "hfopenllm_v2/GPQA": 0.2852, - "hfopenllm_v2/MUSR": 0.3885, - "hfopenllm_v2/MMLU-PRO": 0.3347 - } - }, - { - "id": "nbeerbower/Llama3.1-Gutenberg-Doppel-70B", - "name": "Llama3.1-Gutenberg-Doppel-70B", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7092, - "hfopenllm_v2/BBH": 0.6661, - "hfopenllm_v2/MATH Level 5": 0.2122, - "hfopenllm_v2/GPQA": 0.3448, - "hfopenllm_v2/MUSR": 0.4897, - "hfopenllm_v2/MMLU-PRO": 0.4737 - } - }, - { - "id": "nbeerbower/llama3.1-kartoffeldes-70B", - "name": "llama3.1-kartoffeldes-70B", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.823, - "hfopenllm_v2/BBH": 0.6894, - "hfopenllm_v2/MATH Level 5": 0.3218, - "hfopenllm_v2/GPQA": 0.3515, - "hfopenllm_v2/MUSR": 0.4646, - "hfopenllm_v2/MMLU-PRO": 0.4988 - } - }, - { - "id": "nbeerbower/Lyra-Gutenberg-mistral-nemo-12B", - "name": "Lyra-Gutenberg-mistral-nemo-12B", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3495, - "hfopenllm_v2/BBH": 0.5586, - "hfopenllm_v2/MATH Level 5": 0.1012, - "hfopenllm_v2/GPQA": 0.3339, - "hfopenllm_v2/MUSR": 0.4357, - "hfopenllm_v2/MMLU-PRO": 0.3628 - } - }, - { - "id": "nbeerbower/Lyra4-Gutenberg-12B", - "name": "Lyra4-Gutenberg-12B", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2212, - "hfopenllm_v2/BBH": 0.5387, - "hfopenllm_v2/MATH Level 5": 0.1299, - "hfopenllm_v2/GPQA": 0.3188, - "hfopenllm_v2/MUSR": 0.4038, - "hfopenllm_v2/MMLU-PRO": 0.3571 - } - }, - { - "id": "nbeerbower/Lyra4-Gutenberg2-12B", - "name": "Lyra4-Gutenberg2-12B", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2585, - "hfopenllm_v2/BBH": 0.5345, - "hfopenllm_v2/MATH Level 5": 0.1171, - "hfopenllm_v2/GPQA": 0.3129, - "hfopenllm_v2/MUSR": 0.3972, - "hfopenllm_v2/MMLU-PRO": 0.3565 - } - }, - { - "id": "nbeerbower/Mahou-1.5-mistral-nemo-12B-lorablated", - "name": "Mahou-1.5-mistral-nemo-12B-lorablated", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6825, - "hfopenllm_v2/BBH": 0.5496, - "hfopenllm_v2/MATH Level 5": 0.0891, - "hfopenllm_v2/GPQA": 0.2794, - "hfopenllm_v2/MUSR": 0.4522, - "hfopenllm_v2/MMLU-PRO": 0.3574 - } - }, - { - "id": "nbeerbower/Mistral-Gutenberg-Doppel-7B-FFT", - "name": "Mistral-Gutenberg-Doppel-7B-FFT", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5717, - "hfopenllm_v2/BBH": 0.4076, - "hfopenllm_v2/MATH Level 5": 0.0249, - "hfopenllm_v2/GPQA": 0.2836, - "hfopenllm_v2/MUSR": 0.4059, - "hfopenllm_v2/MMLU-PRO": 0.2729 - } - }, - { - "id": "nbeerbower/mistral-nemo-bophades-12B", - "name": "mistral-nemo-bophades-12B", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6794, - "hfopenllm_v2/BBH": 0.4988, - "hfopenllm_v2/MATH Level 5": 0.1231, - "hfopenllm_v2/GPQA": 0.2852, - "hfopenllm_v2/MUSR": 0.4178, - "hfopenllm_v2/MMLU-PRO": 0.3501 - } - }, - { - "id": "nbeerbower/mistral-nemo-bophades3-12B", - "name": "mistral-nemo-bophades3-12B", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6578, - "hfopenllm_v2/BBH": 0.5449, - "hfopenllm_v2/MATH Level 5": 0.0846, - "hfopenllm_v2/GPQA": 0.3121, - "hfopenllm_v2/MUSR": 0.4604, - "hfopenllm_v2/MMLU-PRO": 0.3371 - } - }, - { - "id": "nbeerbower/mistral-nemo-cc-12B", - "name": "mistral-nemo-cc-12B", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1435, - "hfopenllm_v2/BBH": 0.5399, - "hfopenllm_v2/MATH Level 5": 0.0257, - "hfopenllm_v2/GPQA": 0.3154, - "hfopenllm_v2/MUSR": 0.4424, - "hfopenllm_v2/MMLU-PRO": 0.3598 - } - }, - { - "id": "nbeerbower/mistral-nemo-gutades-12B", - "name": "mistral-nemo-gutades-12B", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3425, - "hfopenllm_v2/BBH": 0.5407, - "hfopenllm_v2/MATH Level 5": 0.1178, - "hfopenllm_v2/GPQA": 0.3154, - "hfopenllm_v2/MUSR": 0.404, - "hfopenllm_v2/MMLU-PRO": 0.3561 - } - }, - { - "id": "nbeerbower/mistral-nemo-gutenberg-12B", - "name": "mistral-nemo-gutenberg-12B", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3504, - "hfopenllm_v2/BBH": 0.5281, - "hfopenllm_v2/MATH Level 5": 0.1163, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.4171, - "hfopenllm_v2/MMLU-PRO": 0.3562 - } - }, - { - "id": "nbeerbower/mistral-nemo-gutenberg-12B-v2", - "name": "mistral-nemo-gutenberg-12B-v2", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6203, - "hfopenllm_v2/BBH": 0.5397, - "hfopenllm_v2/MATH Level 5": 0.1088, - "hfopenllm_v2/GPQA": 0.2777, - "hfopenllm_v2/MUSR": 0.4287, - "hfopenllm_v2/MMLU-PRO": 0.3499 - } - }, - { - "id": "nbeerbower/mistral-nemo-gutenberg-12B-v3", - "name": "mistral-nemo-gutenberg-12B-v3", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2183, - "hfopenllm_v2/BBH": 0.5441, - "hfopenllm_v2/MATH Level 5": 0.0597, - "hfopenllm_v2/GPQA": 0.3146, - "hfopenllm_v2/MUSR": 0.445, - "hfopenllm_v2/MMLU-PRO": 0.3644 - } - }, - { - "id": "nbeerbower/mistral-nemo-gutenberg-12B-v4", - "name": "mistral-nemo-gutenberg-12B-v4", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2379, - "hfopenllm_v2/BBH": 0.5269, - "hfopenllm_v2/MATH Level 5": 0.1261, - "hfopenllm_v2/GPQA": 0.3163, - "hfopenllm_v2/MUSR": 0.4104, - "hfopenllm_v2/MMLU-PRO": 0.3575 - } - }, - { - "id": "nbeerbower/Mistral-Nemo-Gutenberg-Doppel-12B", - "name": "Mistral-Nemo-Gutenberg-Doppel-12B", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3567, - "hfopenllm_v2/BBH": 0.5275, - "hfopenllm_v2/MATH Level 5": 0.1216, - "hfopenllm_v2/GPQA": 0.3163, - "hfopenllm_v2/MUSR": 0.4132, - "hfopenllm_v2/MMLU-PRO": 0.3579 - } - }, - { - "id": "nbeerbower/Mistral-Nemo-Gutenberg-Doppel-12B-v2", - "name": "Mistral-Nemo-Gutenberg-Doppel-12B-v2", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6536, - "hfopenllm_v2/BBH": 0.5374, - "hfopenllm_v2/MATH Level 5": 0.1156, - "hfopenllm_v2/GPQA": 0.271, - "hfopenllm_v2/MUSR": 0.4233, - "hfopenllm_v2/MMLU-PRO": 0.3546 - } - }, - { - "id": "nbeerbower/mistral-nemo-gutenberg2-12B-test", - "name": "mistral-nemo-gutenberg2-12B-test", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3385, - "hfopenllm_v2/BBH": 0.5255, - "hfopenllm_v2/MATH Level 5": 0.1163, - "hfopenllm_v2/GPQA": 0.3171, - "hfopenllm_v2/MUSR": 0.4157, - "hfopenllm_v2/MMLU-PRO": 0.3555 - } - }, - { - "id": "nbeerbower/mistral-nemo-kartoffel-12B", - "name": "mistral-nemo-kartoffel-12B", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7032, - "hfopenllm_v2/BBH": 0.5484, - "hfopenllm_v2/MATH Level 5": 0.0853, - "hfopenllm_v2/GPQA": 0.3045, - "hfopenllm_v2/MUSR": 0.4653, - "hfopenllm_v2/MMLU-PRO": 0.3585 - } - }, - { - "id": "nbeerbower/Mistral-Nemo-Moderne-12B-FFT-experimental", - "name": "Mistral-Nemo-Moderne-12B-FFT-experimental", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3352, - "hfopenllm_v2/BBH": 0.5234, - "hfopenllm_v2/MATH Level 5": 0.077, - "hfopenllm_v2/GPQA": 0.281, - "hfopenllm_v2/MUSR": 0.3715, - "hfopenllm_v2/MMLU-PRO": 0.3455 - } - }, - { - "id": "nbeerbower/mistral-nemo-narwhal-12B", - "name": "mistral-nemo-narwhal-12B", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5549, - "hfopenllm_v2/BBH": 0.5057, - "hfopenllm_v2/MATH Level 5": 0.0582, - "hfopenllm_v2/GPQA": 0.271, - "hfopenllm_v2/MUSR": 0.3847, - "hfopenllm_v2/MMLU-PRO": 0.3483 - } - }, - { - "id": "nbeerbower/Mistral-Nemo-Prism-12B", - "name": "Mistral-Nemo-Prism-12B", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6858, - "hfopenllm_v2/BBH": 0.5475, - "hfopenllm_v2/MATH Level 5": 0.0869, - "hfopenllm_v2/GPQA": 0.3079, - "hfopenllm_v2/MUSR": 0.4626, - "hfopenllm_v2/MMLU-PRO": 0.3581 - } - }, - { - "id": "nbeerbower/Mistral-Nemo-Prism-12B-v2", - "name": "Mistral-Nemo-Prism-12B-v2", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6974, - "hfopenllm_v2/BBH": 0.5492, - "hfopenllm_v2/MATH Level 5": 0.0891, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.46, - "hfopenllm_v2/MMLU-PRO": 0.3567 - } - }, - { - "id": "nbeerbower/Mistral-Nemo-Prism-12B-v7", - "name": "Mistral-Nemo-Prism-12B-v7", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6962, - "hfopenllm_v2/BBH": 0.5521, - "hfopenllm_v2/MATH Level 5": 0.0869, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.4639, - "hfopenllm_v2/MMLU-PRO": 0.359 - } - }, - { - "id": "nbeerbower/mistral-nemo-wissenschaft-12B", - "name": "mistral-nemo-wissenschaft-12B", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.652, - "hfopenllm_v2/BBH": 0.504, - "hfopenllm_v2/MATH Level 5": 0.1216, - "hfopenllm_v2/GPQA": 0.2928, - "hfopenllm_v2/MUSR": 0.4178, - "hfopenllm_v2/MMLU-PRO": 0.3532 - } - }, - { - "id": "nbeerbower/Mistral-Small-Drummer-22B", - "name": "Mistral-Small-Drummer-22B", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6331, - "hfopenllm_v2/BBH": 0.5793, - "hfopenllm_v2/MATH Level 5": 0.1888, - "hfopenllm_v2/GPQA": 0.3431, - "hfopenllm_v2/MUSR": 0.4064, - "hfopenllm_v2/MMLU-PRO": 0.4095 - } - }, - { - "id": "nbeerbower/Mistral-Small-Gutenberg-Doppel-22B", - "name": "Mistral-Small-Gutenberg-Doppel-22B", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4893, - "hfopenllm_v2/BBH": 0.5859, - "hfopenllm_v2/MATH Level 5": 0.2183, - "hfopenllm_v2/GPQA": 0.3465, - "hfopenllm_v2/MUSR": 0.3971, - "hfopenllm_v2/MMLU-PRO": 0.4124 - } - }, - { - "id": "nbeerbower/Nemo-Loony-12B-experimental", - "name": "Nemo-Loony-12B-experimental", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3734, - "hfopenllm_v2/BBH": 0.3822, - "hfopenllm_v2/MATH Level 5": 0.0151, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.3341, - "hfopenllm_v2/MMLU-PRO": 0.1589 - } - }, - { - "id": "nbeerbower/Nemoties-ChatML-12B", - "name": "Nemoties-ChatML-12B", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6382, - "hfopenllm_v2/BBH": 0.547, - "hfopenllm_v2/MATH Level 5": 0.0785, - "hfopenllm_v2/GPQA": 0.297, - "hfopenllm_v2/MUSR": 0.4509, - "hfopenllm_v2/MMLU-PRO": 0.3551 - } - }, - { - "id": "nbeerbower/Qwen2.5-Gutenberg-Doppel-14B", - "name": "Qwen2.5-Gutenberg-Doppel-14B", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8091, - "hfopenllm_v2/BBH": 0.6382, - "hfopenllm_v2/MATH Level 5": 0.5415, - "hfopenllm_v2/GPQA": 0.3331, - "hfopenllm_v2/MUSR": 0.4101, - "hfopenllm_v2/MMLU-PRO": 0.4921 - } - }, - { - "id": "nbeerbower/SmolNemo-12B-FFT-experimental", - "name": "SmolNemo-12B-FFT-experimental", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3348, - "hfopenllm_v2/BBH": 0.3336, - "hfopenllm_v2/MATH Level 5": 0.0128, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3847, - "hfopenllm_v2/MMLU-PRO": 0.1217 - } - }, - { - "id": "nbeerbower/Stella-mistral-nemo-12B-v2", - "name": "Stella-mistral-nemo-12B-v2", - "developer": "nbeerbower", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3274, - "hfopenllm_v2/BBH": 0.5484, - "hfopenllm_v2/MATH Level 5": 0.1163, - "hfopenllm_v2/GPQA": 0.3322, - "hfopenllm_v2/MUSR": 0.4304, - "hfopenllm_v2/MMLU-PRO": 0.3684 - } - }, - { - "id": "nbrahme/IndusQ", - "name": "IndusQ", - "developer": "nbrahme", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.244, - "hfopenllm_v2/BBH": 0.3062, - "hfopenllm_v2/MATH Level 5": 0.0008, - "hfopenllm_v2/GPQA": 0.2651, - "hfopenllm_v2/MUSR": 0.3366, - "hfopenllm_v2/MMLU-PRO": 0.112 - } - }, - { - "id": "NCSOFT/Llama-3-OffsetBias-8B", - "name": "NCSOFT/Llama-3-OffsetBias-8B", - "developer": "NCSOFT", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.8397, - "reward-bench/Chat": 0.9246, - "reward-bench/Chat Hard": 0.8026, - "reward-bench/Safety": 0.8676, - "reward-bench/Reasoning": 0.7639 - } - }, - { - "id": "NCSOFT/Llama-3-OffsetBias-RM-8B", - "name": "NCSOFT/Llama-3-OffsetBias-RM-8B", - "developer": "NCSOFT", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.8942, - "reward-bench/Factuality": 0.6084, - "reward-bench/Precise IF": 0.4, - "reward-bench/Math": 0.5191, - "reward-bench/Safety": 0.8676, - "reward-bench/Focus": 0.9596, - "reward-bench/Ties": 0.6786, - "reward-bench/Chat": 0.9721, - "reward-bench/Chat Hard": 0.818, - "reward-bench/Reasoning": 0.9192 - } - }, - { - "id": "NCSOFT/Llama-VARCO-8B-Instruct", - "name": "Llama-VARCO-8B-Instruct", - "developer": "NCSOFT", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.447, - "hfopenllm_v2/BBH": 0.5023, - "hfopenllm_v2/MATH Level 5": 0.1065, - "hfopenllm_v2/GPQA": 0.297, - "hfopenllm_v2/MUSR": 0.3841, - "hfopenllm_v2/MMLU-PRO": 0.319 - } - }, - { - "id": "necva/IE-cont-Llama3.1-8B", - "name": "IE-cont-Llama3.1-8B", - "developer": "necva", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2049, - "hfopenllm_v2/BBH": 0.2912, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3575, - "hfopenllm_v2/MMLU-PRO": 0.1167 - } - }, - { - "id": "necva/replica-IEPile", - "name": "replica-IEPile", - "developer": "necva", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4678, - "hfopenllm_v2/BBH": 0.4779, - "hfopenllm_v2/MATH Level 5": 0.1239, - "hfopenllm_v2/GPQA": 0.3062, - "hfopenllm_v2/MUSR": 0.3998, - "hfopenllm_v2/MMLU-PRO": 0.3561 - } - }, - { - "id": "Nekochu/Llama-3.1-8B-french-DPO", - "name": "Llama-3.1-8B-french-DPO", - "developer": "Nekochu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4656, - "hfopenllm_v2/BBH": 0.5111, - "hfopenllm_v2/MATH Level 5": 0.0974, - "hfopenllm_v2/GPQA": 0.2911, - "hfopenllm_v2/MUSR": 0.4216, - "hfopenllm_v2/MMLU-PRO": 0.3414 - } - }, - { - "id": "Nekochu/Llama-3.1-8B-German-ORPO", - "name": "Llama-3.1-8B-German-ORPO", - "developer": "Nekochu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4611, - "hfopenllm_v2/BBH": 0.4983, - "hfopenllm_v2/MATH Level 5": 0.1171, - "hfopenllm_v2/GPQA": 0.3163, - "hfopenllm_v2/MUSR": 0.4647, - "hfopenllm_v2/MMLU-PRO": 0.3393 - } - }, - { - "id": "Nekochu/Luminia-13B-v3", - "name": "Luminia-13B-v3", - "developer": "Nekochu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2523, - "hfopenllm_v2/BBH": 0.4112, - "hfopenllm_v2/MATH Level 5": 0.0181, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.3983, - "hfopenllm_v2/MMLU-PRO": 0.2215 - } - }, - { - "id": "Nekochu/Luminia-8B-RP", - "name": "Luminia-8B-RP", - "developer": "Nekochu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5574, - "hfopenllm_v2/BBH": 0.5218, - "hfopenllm_v2/MATH Level 5": 0.136, - "hfopenllm_v2/GPQA": 0.297, - "hfopenllm_v2/MUSR": 0.3998, - "hfopenllm_v2/MMLU-PRO": 0.3631 - } - }, - { - "id": "neopolita/jessi-v0.1-bf16-falcon3-7b-instruct", - "name": "jessi-v0.1-bf16-falcon3-7b-instruct", - "developer": "neopolita", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7527, - "hfopenllm_v2/BBH": 0.5516, - "hfopenllm_v2/MATH Level 5": 0.3807, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.4825, - "hfopenllm_v2/MMLU-PRO": 0.3924 - } - }, - { - "id": "neopolita/jessi-v0.1-falcon3-10b-instruct", - "name": "jessi-v0.1-falcon3-10b-instruct", - "developer": "neopolita", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7552, - "hfopenllm_v2/BBH": 0.5953, - "hfopenllm_v2/MATH Level 5": 0.2002, - "hfopenllm_v2/GPQA": 0.3188, - "hfopenllm_v2/MUSR": 0.4279, - "hfopenllm_v2/MMLU-PRO": 0.4188 - } - }, - { - "id": "neopolita/jessi-v0.1-qwen2.5-7b-instruct", - "name": "jessi-v0.1-qwen2.5-7b-instruct", - "developer": "neopolita", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7327, - "hfopenllm_v2/BBH": 0.5292, - "hfopenllm_v2/MATH Level 5": 0.4086, - "hfopenllm_v2/GPQA": 0.297, - "hfopenllm_v2/MUSR": 0.3914, - "hfopenllm_v2/MMLU-PRO": 0.4228 - } - }, - { - "id": "neopolita/jessi-v0.1-virtuoso-small", - "name": "jessi-v0.1-virtuoso-small", - "developer": "neopolita", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7959, - "hfopenllm_v2/BBH": 0.6443, - "hfopenllm_v2/MATH Level 5": 0.3399, - "hfopenllm_v2/GPQA": 0.3305, - "hfopenllm_v2/MUSR": 0.4362, - "hfopenllm_v2/MMLU-PRO": 0.513 - } - }, - { - "id": "neopolita/jessi-v0.2-falcon3-10b-instruct", - "name": "jessi-v0.2-falcon3-10b-instruct", - "developer": "neopolita", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7768, - "hfopenllm_v2/BBH": 0.6205, - "hfopenllm_v2/MATH Level 5": 0.2122, - "hfopenllm_v2/GPQA": 0.3289, - "hfopenllm_v2/MUSR": 0.4281, - "hfopenllm_v2/MMLU-PRO": 0.4354 - } - }, - { - "id": "neopolita/jessi-v0.2-falcon3-7b-instruct", - "name": "jessi-v0.2-falcon3-7b-instruct", - "developer": "neopolita", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5771, - "hfopenllm_v2/BBH": 0.5363, - "hfopenllm_v2/MATH Level 5": 0.2538, - "hfopenllm_v2/GPQA": 0.3171, - "hfopenllm_v2/MUSR": 0.4479, - "hfopenllm_v2/MMLU-PRO": 0.3905 - } - }, - { - "id": "neopolita/jessi-v0.3-falcon3-7b-instruct", - "name": "jessi-v0.3-falcon3-7b-instruct", - "developer": "neopolita", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7509, - "hfopenllm_v2/BBH": 0.5388, - "hfopenllm_v2/MATH Level 5": 0.1888, - "hfopenllm_v2/GPQA": 0.3196, - "hfopenllm_v2/MUSR": 0.4692, - "hfopenllm_v2/MMLU-PRO": 0.397 - } - }, - { - "id": "neopolita/jessi-v0.4-falcon3-7b-instruct", - "name": "jessi-v0.4-falcon3-7b-instruct", - "developer": "neopolita", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7604, - "hfopenllm_v2/BBH": 0.5522, - "hfopenllm_v2/MATH Level 5": 0.3769, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.4971, - "hfopenllm_v2/MMLU-PRO": 0.4004 - } - }, - { - "id": "neopolita/jessi-v0.5-falcon3-7b-instruct", - "name": "jessi-v0.5-falcon3-7b-instruct", - "developer": "neopolita", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7412, - "hfopenllm_v2/BBH": 0.559, - "hfopenllm_v2/MATH Level 5": 0.3739, - "hfopenllm_v2/GPQA": 0.3112, - "hfopenllm_v2/MUSR": 0.4865, - "hfopenllm_v2/MMLU-PRO": 0.3966 - } - }, - { - "id": "neopolita/jessi-v0.6-falcon3-7b-instruct", - "name": "jessi-v0.6-falcon3-7b-instruct", - "developer": "neopolita", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7402, - "hfopenllm_v2/BBH": 0.5509, - "hfopenllm_v2/MATH Level 5": 0.3565, - "hfopenllm_v2/GPQA": 0.3003, - "hfopenllm_v2/MUSR": 0.4904, - "hfopenllm_v2/MMLU-PRO": 0.3957 - } - }, - { - "id": "neopolita/loki-v0.1-virtuoso", - "name": "loki-v0.1-virtuoso", - "developer": "neopolita", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7819, - "hfopenllm_v2/BBH": 0.6467, - "hfopenllm_v2/MATH Level 5": 0.3391, - "hfopenllm_v2/GPQA": 0.3507, - "hfopenllm_v2/MUSR": 0.4375, - "hfopenllm_v2/MMLU-PRO": 0.5129 - } - }, - { - "id": "netcat420/DeepSeek-R1-Distill-Qwen-MFANN-Slerp-7b", - "name": "DeepSeek-R1-Distill-Qwen-MFANN-Slerp-7b", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.115, - "hfopenllm_v2/BBH": 0.2877, - "hfopenllm_v2/MATH Level 5": 0.0015, - "hfopenllm_v2/GPQA": 0.2643, - "hfopenllm_v2/MUSR": 0.3724, - "hfopenllm_v2/MMLU-PRO": 0.109 - } - }, - { - "id": "netcat420/DeepSeek-R1-MFANN-TIES-unretrained-7b", - "name": "DeepSeek-R1-MFANN-TIES-unretrained-7b", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2587, - "hfopenllm_v2/BBH": 0.3086, - "hfopenllm_v2/MATH Level 5": 0.0121, - "hfopenllm_v2/GPQA": 0.255, - "hfopenllm_v2/MUSR": 0.3527, - "hfopenllm_v2/MMLU-PRO": 0.1145 - } - }, - { - "id": "netcat420/Llama3.1-MFANN-8b", - "name": "Llama3.1-MFANN-8b", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.297, - "hfopenllm_v2/BBH": 0.4281, - "hfopenllm_v2/MATH Level 5": 0.0295, - "hfopenllm_v2/GPQA": 0.2878, - "hfopenllm_v2/MUSR": 0.3379, - "hfopenllm_v2/MMLU-PRO": 0.2725 - } - }, - { - "id": "netcat420/MFANN-abliterated-phi2-merge-unretrained", - "name": "MFANN-abliterated-phi2-merge-unretrained", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3005, - "hfopenllm_v2/BBH": 0.4104, - "hfopenllm_v2/MATH Level 5": 0.0287, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.3183, - "hfopenllm_v2/MMLU-PRO": 0.1478 - } - }, - { - "id": "netcat420/MFANN-llama3.1-Abliterated-SLERP", - "name": "MFANN-llama3.1-Abliterated-SLERP", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2591, - "hfopenllm_v2/BBH": 0.4574, - "hfopenllm_v2/MATH Level 5": 0.0483, - "hfopenllm_v2/GPQA": 0.2735, - "hfopenllm_v2/MUSR": 0.3809, - "hfopenllm_v2/MMLU-PRO": 0.2928 - } - }, - { - "id": "netcat420/MFANN-Llama3.1-Abliterated-Slerp-TIES", - "name": "MFANN-Llama3.1-Abliterated-Slerp-TIES", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4293, - "hfopenllm_v2/BBH": 0.4968, - "hfopenllm_v2/MATH Level 5": 0.0665, - "hfopenllm_v2/GPQA": 0.2919, - "hfopenllm_v2/MUSR": 0.3687, - "hfopenllm_v2/MMLU-PRO": 0.3531 - } - }, - { - "id": "netcat420/MFANN-Llama3.1-Abliterated-SLERP-TIES-V2", - "name": "MFANN-Llama3.1-Abliterated-SLERP-TIES-V2", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.421, - "hfopenllm_v2/BBH": 0.4924, - "hfopenllm_v2/MATH Level 5": 0.0763, - "hfopenllm_v2/GPQA": 0.297, - "hfopenllm_v2/MUSR": 0.3728, - "hfopenllm_v2/MMLU-PRO": 0.3522 - } - }, - { - "id": "netcat420/MFANN-Llama3.1-Abliterated-SLERP-TIES-V3", - "name": "MFANN-Llama3.1-Abliterated-SLERP-TIES-V3", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4238, - "hfopenllm_v2/BBH": 0.4914, - "hfopenllm_v2/MATH Level 5": 0.0755, - "hfopenllm_v2/GPQA": 0.297, - "hfopenllm_v2/MUSR": 0.3741, - "hfopenllm_v2/MMLU-PRO": 0.349 - } - }, - { - "id": "netcat420/MFANN-llama3.1-abliterated-SLERP-v3", - "name": "MFANN-llama3.1-abliterated-SLERP-v3", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3799, - "hfopenllm_v2/BBH": 0.4931, - "hfopenllm_v2/MATH Level 5": 0.0642, - "hfopenllm_v2/GPQA": 0.2911, - "hfopenllm_v2/MUSR": 0.366, - "hfopenllm_v2/MMLU-PRO": 0.3531 - } - }, - { - "id": "netcat420/MFANN-llama3.1-abliterated-SLERP-v3.1", - "name": "MFANN-llama3.1-abliterated-SLERP-v3.1", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4202, - "hfopenllm_v2/BBH": 0.4921, - "hfopenllm_v2/MATH Level 5": 0.0695, - "hfopenllm_v2/GPQA": 0.2928, - "hfopenllm_v2/MUSR": 0.3686, - "hfopenllm_v2/MMLU-PRO": 0.3543 - } - }, - { - "id": "netcat420/MFANN-Llama3.1-Abliterated-Slerp-V3.2", - "name": "MFANN-Llama3.1-Abliterated-Slerp-V3.2", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4128, - "hfopenllm_v2/BBH": 0.4978, - "hfopenllm_v2/MATH Level 5": 0.0702, - "hfopenllm_v2/GPQA": 0.2878, - "hfopenllm_v2/MUSR": 0.3754, - "hfopenllm_v2/MMLU-PRO": 0.3527 - } - }, - { - "id": "netcat420/MFANN-Llama3.1-Abliterated-SLERP-V4", - "name": "MFANN-Llama3.1-Abliterated-SLERP-V4", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4169, - "hfopenllm_v2/BBH": 0.4909, - "hfopenllm_v2/MATH Level 5": 0.068, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.3821, - "hfopenllm_v2/MMLU-PRO": 0.3516 - } - }, - { - "id": "netcat420/MFANN-Llama3.1-Abliterated-SLERP-V5", - "name": "MFANN-Llama3.1-Abliterated-SLERP-V5", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4329, - "hfopenllm_v2/BBH": 0.4952, - "hfopenllm_v2/MATH Level 5": 0.0816, - "hfopenllm_v2/GPQA": 0.2936, - "hfopenllm_v2/MUSR": 0.3781, - "hfopenllm_v2/MMLU-PRO": 0.3445 - } - }, - { - "id": "netcat420/MFANN-llama3.1-abliterated-v2", - "name": "MFANN-llama3.1-abliterated-v2", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4429, - "hfopenllm_v2/BBH": 0.4941, - "hfopenllm_v2/MATH Level 5": 0.074, - "hfopenllm_v2/GPQA": 0.2928, - "hfopenllm_v2/MUSR": 0.3845, - "hfopenllm_v2/MMLU-PRO": 0.3491 - } - }, - { - "id": "netcat420/MFANN-phigments-slerp-V2", - "name": "MFANN-phigments-slerp-V2", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3232, - "hfopenllm_v2/BBH": 0.4827, - "hfopenllm_v2/MATH Level 5": 0.0317, - "hfopenllm_v2/GPQA": 0.2727, - "hfopenllm_v2/MUSR": 0.4037, - "hfopenllm_v2/MMLU-PRO": 0.2717 - } - }, - { - "id": "netcat420/MFANN-phigments-slerp-V3.2", - "name": "MFANN-phigments-slerp-V3.2", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3524, - "hfopenllm_v2/BBH": 0.4809, - "hfopenllm_v2/MATH Level 5": 0.0332, - "hfopenllm_v2/GPQA": 0.2836, - "hfopenllm_v2/MUSR": 0.3708, - "hfopenllm_v2/MMLU-PRO": 0.2705 - } - }, - { - "id": "netcat420/MFANN-phigments-slerp-V3.3", - "name": "MFANN-phigments-slerp-V3.3", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3691, - "hfopenllm_v2/BBH": 0.4895, - "hfopenllm_v2/MATH Level 5": 0.0332, - "hfopenllm_v2/GPQA": 0.2752, - "hfopenllm_v2/MUSR": 0.3892, - "hfopenllm_v2/MMLU-PRO": 0.2803 - } - }, - { - "id": "netcat420/MFANN-SFT", - "name": "MFANN-SFT", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3682, - "hfopenllm_v2/BBH": 0.4852, - "hfopenllm_v2/MATH Level 5": 0.0597, - "hfopenllm_v2/GPQA": 0.3163, - "hfopenllm_v2/MUSR": 0.3725, - "hfopenllm_v2/MMLU-PRO": 0.3336 - } - }, - { - "id": "netcat420/MFANN3b", - "name": "MFANN3b", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2524, - "hfopenllm_v2/BBH": 0.4433, - "hfopenllm_v2/MATH Level 5": 0.0219, - "hfopenllm_v2/GPQA": 0.2919, - "hfopenllm_v2/MUSR": 0.3606, - "hfopenllm_v2/MMLU-PRO": 0.2306 - } - }, - { - "id": "netcat420/MFANN3bv0.15", - "name": "MFANN3bv0.15", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2012, - "hfopenllm_v2/BBH": 0.4539, - "hfopenllm_v2/MATH Level 5": 0.0264, - "hfopenllm_v2/GPQA": 0.2517, - "hfopenllm_v2/MUSR": 0.3958, - "hfopenllm_v2/MMLU-PRO": 0.2468 - } - }, - { - "id": "netcat420/MFANN3bv0.18", - "name": "MFANN3bv0.18", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2206, - "hfopenllm_v2/BBH": 0.4514, - "hfopenllm_v2/MATH Level 5": 0.0249, - "hfopenllm_v2/GPQA": 0.2576, - "hfopenllm_v2/MUSR": 0.4024, - "hfopenllm_v2/MMLU-PRO": 0.25 - } - }, - { - "id": "netcat420/MFANN3bv0.19", - "name": "MFANN3bv0.19", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2258, - "hfopenllm_v2/BBH": 0.4516, - "hfopenllm_v2/MATH Level 5": 0.0227, - "hfopenllm_v2/GPQA": 0.2576, - "hfopenllm_v2/MUSR": 0.4024, - "hfopenllm_v2/MMLU-PRO": 0.252 - } - }, - { - "id": "netcat420/MFANN3bv0.20", - "name": "MFANN3bv0.20", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2193, - "hfopenllm_v2/BBH": 0.4493, - "hfopenllm_v2/MATH Level 5": 0.0264, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.4077, - "hfopenllm_v2/MMLU-PRO": 0.25 - } - }, - { - "id": "netcat420/MFANN3bv0.21", - "name": "MFANN3bv0.21", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1909, - "hfopenllm_v2/BBH": 0.447, - "hfopenllm_v2/MATH Level 5": 0.0317, - "hfopenllm_v2/GPQA": 0.2643, - "hfopenllm_v2/MUSR": 0.3759, - "hfopenllm_v2/MMLU-PRO": 0.2393 - } - }, - { - "id": "netcat420/MFANN3bv0.22", - "name": "MFANN3bv0.22", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1979, - "hfopenllm_v2/BBH": 0.4485, - "hfopenllm_v2/MATH Level 5": 0.0264, - "hfopenllm_v2/GPQA": 0.2617, - "hfopenllm_v2/MUSR": 0.3521, - "hfopenllm_v2/MMLU-PRO": 0.2517 - } - }, - { - "id": "netcat420/MFANN3bv0.23", - "name": "MFANN3bv0.23", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2048, - "hfopenllm_v2/BBH": 0.4495, - "hfopenllm_v2/MATH Level 5": 0.0249, - "hfopenllm_v2/GPQA": 0.2517, - "hfopenllm_v2/MUSR": 0.3427, - "hfopenllm_v2/MMLU-PRO": 0.2418 - } - }, - { - "id": "netcat420/MFANN3bv0.24", - "name": "MFANN3bv0.24", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.22, - "hfopenllm_v2/BBH": 0.4407, - "hfopenllm_v2/MATH Level 5": 0.0279, - "hfopenllm_v2/GPQA": 0.2584, - "hfopenllm_v2/MUSR": 0.3521, - "hfopenllm_v2/MMLU-PRO": 0.2352 - } - }, - { - "id": "netcat420/MFANN3bv1.1", - "name": "MFANN3bv1.1", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2507, - "hfopenllm_v2/BBH": 0.3397, - "hfopenllm_v2/MATH Level 5": 0.0204, - "hfopenllm_v2/GPQA": 0.2668, - "hfopenllm_v2/MUSR": 0.3223, - "hfopenllm_v2/MMLU-PRO": 0.1159 - } - }, - { - "id": "netcat420/MFANN3bv1.2", - "name": "MFANN3bv1.2", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2686, - "hfopenllm_v2/BBH": 0.366, - "hfopenllm_v2/MATH Level 5": 0.0264, - "hfopenllm_v2/GPQA": 0.2634, - "hfopenllm_v2/MUSR": 0.3156, - "hfopenllm_v2/MMLU-PRO": 0.145 - } - }, - { - "id": "netcat420/MFANN3bv1.3", - "name": "MFANN3bv1.3", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2547, - "hfopenllm_v2/BBH": 0.4456, - "hfopenllm_v2/MATH Level 5": 0.0211, - "hfopenllm_v2/GPQA": 0.2576, - "hfopenllm_v2/MUSR": 0.3299, - "hfopenllm_v2/MMLU-PRO": 0.2276 - } - }, - { - "id": "netcat420/MFANN3bv1.4", - "name": "MFANN3bv1.4", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3524, - "hfopenllm_v2/BBH": 0.4809, - "hfopenllm_v2/MATH Level 5": 0.037, - "hfopenllm_v2/GPQA": 0.2827, - "hfopenllm_v2/MUSR": 0.3708, - "hfopenllm_v2/MMLU-PRO": 0.2705 - } - }, - { - "id": "netcat420/MFANNv0.19", - "name": "MFANNv0.19", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3057, - "hfopenllm_v2/BBH": 0.4731, - "hfopenllm_v2/MATH Level 5": 0.0415, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.3527, - "hfopenllm_v2/MMLU-PRO": 0.2473 - } - }, - { - "id": "netcat420/MFANNv0.20", - "name": "MFANNv0.20", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3479, - "hfopenllm_v2/BBH": 0.4574, - "hfopenllm_v2/MATH Level 5": 0.0498, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.3874, - "hfopenllm_v2/MMLU-PRO": 0.3202 - } - }, - { - "id": "netcat420/MFANNv0.21", - "name": "MFANNv0.21", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3233, - "hfopenllm_v2/BBH": 0.4576, - "hfopenllm_v2/MATH Level 5": 0.0574, - "hfopenllm_v2/GPQA": 0.2785, - "hfopenllm_v2/MUSR": 0.3993, - "hfopenllm_v2/MMLU-PRO": 0.3031 - } - }, - { - "id": "netcat420/MFANNv0.22.1", - "name": "MFANNv0.22.1", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3089, - "hfopenllm_v2/BBH": 0.4661, - "hfopenllm_v2/MATH Level 5": 0.0536, - "hfopenllm_v2/GPQA": 0.276, - "hfopenllm_v2/MUSR": 0.3753, - "hfopenllm_v2/MMLU-PRO": 0.3343 - } - }, - { - "id": "netcat420/MFANNv0.23", - "name": "MFANNv0.23", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3127, - "hfopenllm_v2/BBH": 0.4898, - "hfopenllm_v2/MATH Level 5": 0.0498, - "hfopenllm_v2/GPQA": 0.2844, - "hfopenllm_v2/MUSR": 0.3768, - "hfopenllm_v2/MMLU-PRO": 0.3388 - } - }, - { - "id": "netcat420/MFANNv0.24", - "name": "MFANNv0.24", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3162, - "hfopenllm_v2/BBH": 0.479, - "hfopenllm_v2/MATH Level 5": 0.0612, - "hfopenllm_v2/GPQA": 0.2844, - "hfopenllm_v2/MUSR": 0.3754, - "hfopenllm_v2/MMLU-PRO": 0.3348 - } - }, - { - "id": "netcat420/MFANNv0.25", - "name": "MFANNv0.25", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3467, - "hfopenllm_v2/BBH": 0.4794, - "hfopenllm_v2/MATH Level 5": 0.0582, - "hfopenllm_v2/GPQA": 0.2802, - "hfopenllm_v2/MUSR": 0.3688, - "hfopenllm_v2/MMLU-PRO": 0.3343 - } - }, - { - "id": "netcat420/Qwen2.5-7b-MFANN-slerp", - "name": "Qwen2.5-7b-MFANN-slerp", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6532, - "hfopenllm_v2/BBH": 0.5089, - "hfopenllm_v2/MATH Level 5": 0.287, - "hfopenllm_v2/GPQA": 0.2953, - "hfopenllm_v2/MUSR": 0.4073, - "hfopenllm_v2/MMLU-PRO": 0.3417 - } - }, - { - "id": "netcat420/Qwen2.5-7b-nerd-uncensored-MFANN-slerp", - "name": "Qwen2.5-7b-nerd-uncensored-MFANN-slerp", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1564, - "hfopenllm_v2/BBH": 0.292, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3792, - "hfopenllm_v2/MMLU-PRO": 0.11 - } - }, - { - "id": "netcat420/Qwen2.5-7B-nerd-uncensored-v0.9-MFANN", - "name": "Qwen2.5-7B-nerd-uncensored-v0.9-MFANN", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5878, - "hfopenllm_v2/BBH": 0.5237, - "hfopenllm_v2/MATH Level 5": 0.3376, - "hfopenllm_v2/GPQA": 0.281, - "hfopenllm_v2/MUSR": 0.3926, - "hfopenllm_v2/MMLU-PRO": 0.3904 - } - }, - { - "id": "netcat420/Qwen2.5-Coder-Scholar-7B-Abliterated-MFANN", - "name": "Qwen2.5-Coder-Scholar-7B-Abliterated-MFANN", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5742, - "hfopenllm_v2/BBH": 0.5071, - "hfopenllm_v2/MATH Level 5": 0.2568, - "hfopenllm_v2/GPQA": 0.2928, - "hfopenllm_v2/MUSR": 0.4058, - "hfopenllm_v2/MMLU-PRO": 0.3157 - } - }, - { - "id": "netcat420/Qwen2.5-Coder-Scholar-7B-Abliterated-MFANN-Slerp-Unretrained", - "name": "Qwen2.5-Coder-Scholar-7B-Abliterated-MFANN-Slerp-Unretrained", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6486, - "hfopenllm_v2/BBH": 0.5066, - "hfopenllm_v2/MATH Level 5": 0.2991, - "hfopenllm_v2/GPQA": 0.2987, - "hfopenllm_v2/MUSR": 0.4152, - "hfopenllm_v2/MMLU-PRO": 0.3432 - } - }, - { - "id": "netcat420/Qwen2.5-DeepSeek-R1-MFANN-Slerp-7b", - "name": "Qwen2.5-DeepSeek-R1-MFANN-Slerp-7b", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2676, - "hfopenllm_v2/BBH": 0.3789, - "hfopenllm_v2/MATH Level 5": 0.0181, - "hfopenllm_v2/GPQA": 0.2324, - "hfopenllm_v2/MUSR": 0.3528, - "hfopenllm_v2/MMLU-PRO": 0.1677 - } - }, - { - "id": "netcat420/Qwen2.5-MFANN-7b", - "name": "Qwen2.5-MFANN-7b", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6097, - "hfopenllm_v2/BBH": 0.5054, - "hfopenllm_v2/MATH Level 5": 0.2787, - "hfopenllm_v2/GPQA": 0.2861, - "hfopenllm_v2/MUSR": 0.4021, - "hfopenllm_v2/MMLU-PRO": 0.3233 - } - }, - { - "id": "netcat420/qwen2.5-MFANN-7b-SLERP-V1.2", - "name": "qwen2.5-MFANN-7b-SLERP-V1.2", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6606, - "hfopenllm_v2/BBH": 0.5111, - "hfopenllm_v2/MATH Level 5": 0.287, - "hfopenllm_v2/GPQA": 0.297, - "hfopenllm_v2/MUSR": 0.4259, - "hfopenllm_v2/MMLU-PRO": 0.3438 - } - }, - { - "id": "netcat420/qwen2.5-MFANN-7b-SLERPv1.1", - "name": "qwen2.5-MFANN-7b-SLERPv1.1", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6555, - "hfopenllm_v2/BBH": 0.5075, - "hfopenllm_v2/MATH Level 5": 0.2968, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.4126, - "hfopenllm_v2/MMLU-PRO": 0.3448 - } - }, - { - "id": "netcat420/qwen2.5-MFANN-7b-v1.1", - "name": "qwen2.5-MFANN-7b-v1.1", - "developer": "netcat420", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6088, - "hfopenllm_v2/BBH": 0.4967, - "hfopenllm_v2/MATH Level 5": 0.2825, - "hfopenllm_v2/GPQA": 0.276, - "hfopenllm_v2/MUSR": 0.4114, - "hfopenllm_v2/MMLU-PRO": 0.3248 - } - }, - { - "id": "netease-youdao/Confucius-o1-14B", - "name": "Confucius-o1-14B", - "developer": "netease-youdao", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6378, - "hfopenllm_v2/BBH": 0.63, - "hfopenllm_v2/MATH Level 5": 0.4313, - "hfopenllm_v2/GPQA": 0.3649, - "hfopenllm_v2/MUSR": 0.4338, - "hfopenllm_v2/MMLU-PRO": 0.5265 - } - }, - { - "id": "NeverSleep/Lumimaid-v0.2-12B", - "name": "Lumimaid-v0.2-12B", - "developer": "NeverSleep", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1099, - "hfopenllm_v2/BBH": 0.5396, - "hfopenllm_v2/MATH Level 5": 0.0566, - "hfopenllm_v2/GPQA": 0.3146, - "hfopenllm_v2/MUSR": 0.4821, - "hfopenllm_v2/MMLU-PRO": 0.3511 - } - }, - { - "id": "NeverSleep/Lumimaid-v0.2-8B", - "name": "Lumimaid-v0.2-8B", - "developer": "NeverSleep", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5038, - "hfopenllm_v2/BBH": 0.5238, - "hfopenllm_v2/MATH Level 5": 0.1435, - "hfopenllm_v2/GPQA": 0.3112, - "hfopenllm_v2/MUSR": 0.4303, - "hfopenllm_v2/MMLU-PRO": 0.3636 - } - }, - { - "id": "newsbang/Homer-7B-v0.1", - "name": "Homer-7B-v0.1", - "developer": "newsbang", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6109, - "hfopenllm_v2/BBH": 0.5601, - "hfopenllm_v2/MATH Level 5": 0.386, - "hfopenllm_v2/GPQA": 0.3247, - "hfopenllm_v2/MUSR": 0.4357, - "hfopenllm_v2/MMLU-PRO": 0.4475 - } - }, - { - "id": "newsbang/Homer-7B-v0.2", - "name": "Homer-7B-v0.2", - "developer": "newsbang", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7494, - "hfopenllm_v2/BBH": 0.5517, - "hfopenllm_v2/MATH Level 5": 0.2477, - "hfopenllm_v2/GPQA": 0.3322, - "hfopenllm_v2/MUSR": 0.4298, - "hfopenllm_v2/MMLU-PRO": 0.441 - } - }, - { - "id": "newsbang/Homer-v0.3-Qwen2.5-7B", - "name": "Homer-v0.3-Qwen2.5-7B", - "developer": "newsbang", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5154, - "hfopenllm_v2/BBH": 0.5481, - "hfopenllm_v2/MATH Level 5": 0.3089, - "hfopenllm_v2/GPQA": 0.3339, - "hfopenllm_v2/MUSR": 0.4744, - "hfopenllm_v2/MMLU-PRO": 0.4456 - } - }, - { - "id": "newsbang/Homer-v0.4-Qwen2.5-7B", - "name": "Homer-v0.4-Qwen2.5-7B", - "developer": "newsbang", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7999, - "hfopenllm_v2/BBH": 0.5533, - "hfopenllm_v2/MATH Level 5": 0.2779, - "hfopenllm_v2/GPQA": 0.3154, - "hfopenllm_v2/MUSR": 0.4311, - "hfopenllm_v2/MMLU-PRO": 0.4363 - } - }, - { - "id": "newsbang/Homer-v0.5-Qwen2.5-7B", - "name": "Homer-v0.5-Qwen2.5-7B", - "developer": "newsbang", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7881, - "hfopenllm_v2/BBH": 0.554, - "hfopenllm_v2/MATH Level 5": 0.3724, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.4193, - "hfopenllm_v2/MMLU-PRO": 0.4369 - } - }, - { - "id": "newsbang/Homer-v1.0-Qwen2.5-72B", - "name": "Homer-v1.0-Qwen2.5-72B", - "developer": "newsbang", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7628, - "hfopenllm_v2/BBH": 0.731, - "hfopenllm_v2/MATH Level 5": 0.4902, - "hfopenllm_v2/GPQA": 0.4161, - "hfopenllm_v2/MUSR": 0.4677, - "hfopenllm_v2/MMLU-PRO": 0.6145 - } - }, - { - "id": "newsbang/Homer-v1.0-Qwen2.5-7B", - "name": "Homer-v1.0-Qwen2.5-7B", - "developer": "newsbang", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6393, - "hfopenllm_v2/BBH": 0.5655, - "hfopenllm_v2/MATH Level 5": 0.3323, - "hfopenllm_v2/GPQA": 0.3221, - "hfopenllm_v2/MUSR": 0.4278, - "hfopenllm_v2/MMLU-PRO": 0.4535 - } - }, - { - "id": "Nexesenex/Dolphin3.0-Llama3.1-1B-abliterated", - "name": "Dolphin3.0-Llama3.1-1B-abliterated", - "developer": "Nexesenex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5312, - "hfopenllm_v2/BBH": 0.3241, - "hfopenllm_v2/MATH Level 5": 0.0385, - "hfopenllm_v2/GPQA": 0.2408, - "hfopenllm_v2/MUSR": 0.3237, - "hfopenllm_v2/MMLU-PRO": 0.1373 - } - }, - { - "id": "Nexesenex/Llama_3.1_8b_DeepDive_3_Prev_v1.0", - "name": "Llama_3.1_8b_DeepDive_3_Prev_v1.0", - "developer": "Nexesenex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6809, - "hfopenllm_v2/BBH": 0.5155, - "hfopenllm_v2/MATH Level 5": 0.1866, - "hfopenllm_v2/GPQA": 0.2911, - "hfopenllm_v2/MUSR": 0.3666, - "hfopenllm_v2/MMLU-PRO": 0.3438 - } - }, - { - "id": "Nexesenex/Llama_3.1_8b_DeepDive_3_R1_Prev_v1.0", - "name": "Llama_3.1_8b_DeepDive_3_R1_Prev_v1.0", - "developer": "Nexesenex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7101, - "hfopenllm_v2/BBH": 0.512, - "hfopenllm_v2/MATH Level 5": 0.1926, - "hfopenllm_v2/GPQA": 0.3003, - "hfopenllm_v2/MUSR": 0.3758, - "hfopenllm_v2/MMLU-PRO": 0.3441 - } - }, - { - "id": "Nexesenex/Llama_3.1_8b_DoberWild_v2.01", - "name": "Llama_3.1_8b_DoberWild_v2.01", - "developer": "Nexesenex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7996, - "hfopenllm_v2/BBH": 0.5251, - "hfopenllm_v2/MATH Level 5": 0.2002, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.4012, - "hfopenllm_v2/MMLU-PRO": 0.3791 - } - }, - { - "id": "Nexesenex/Llama_3.1_8b_DoberWild_v2.03", - "name": "Llama_3.1_8b_DoberWild_v2.03", - "developer": "Nexesenex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7764, - "hfopenllm_v2/BBH": 0.5294, - "hfopenllm_v2/MATH Level 5": 0.2077, - "hfopenllm_v2/GPQA": 0.3045, - "hfopenllm_v2/MUSR": 0.3906, - "hfopenllm_v2/MMLU-PRO": 0.3722 - } - }, - { - "id": "Nexesenex/Llama_3.1_8b_DobHerWild_R1_v1.1R", - "name": "Llama_3.1_8b_DobHerWild_R1_v1.1R", - "developer": "Nexesenex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.76, - "hfopenllm_v2/BBH": 0.5257, - "hfopenllm_v2/MATH Level 5": 0.2319, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.3852, - "hfopenllm_v2/MMLU-PRO": 0.3688 - } - }, - { - "id": "Nexesenex/Llama_3.1_8b_DodoWild_v2.01", - "name": "Llama_3.1_8b_DodoWild_v2.01", - "developer": "Nexesenex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7978, - "hfopenllm_v2/BBH": 0.5253, - "hfopenllm_v2/MATH Level 5": 0.1986, - "hfopenllm_v2/GPQA": 0.3037, - "hfopenllm_v2/MUSR": 0.409, - "hfopenllm_v2/MMLU-PRO": 0.3738 - } - }, - { - "id": "Nexesenex/Llama_3.1_8b_DodoWild_v2.02", - "name": "Llama_3.1_8b_DodoWild_v2.02", - "developer": "Nexesenex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8017, - "hfopenllm_v2/BBH": 0.5262, - "hfopenllm_v2/MATH Level 5": 0.2273, - "hfopenllm_v2/GPQA": 0.3045, - "hfopenllm_v2/MUSR": 0.3971, - "hfopenllm_v2/MMLU-PRO": 0.3761 - } - }, - { - "id": "Nexesenex/Llama_3.1_8b_DodoWild_v2.03", - "name": "Llama_3.1_8b_DodoWild_v2.03", - "developer": "Nexesenex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7941, - "hfopenllm_v2/BBH": 0.5308, - "hfopenllm_v2/MATH Level 5": 0.2221, - "hfopenllm_v2/GPQA": 0.3079, - "hfopenllm_v2/MUSR": 0.3959, - "hfopenllm_v2/MMLU-PRO": 0.3786 - } - }, - { - "id": "Nexesenex/Llama_3.1_8b_DodoWild_v2.10", - "name": "Llama_3.1_8b_DodoWild_v2.10", - "developer": "Nexesenex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8054, - "hfopenllm_v2/BBH": 0.5278, - "hfopenllm_v2/MATH Level 5": 0.1971, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.4157, - "hfopenllm_v2/MMLU-PRO": 0.3855 - } - }, - { - "id": "Nexesenex/Llama_3.1_8b_Dolermed_R1_V1.01", - "name": "Llama_3.1_8b_Dolermed_R1_V1.01", - "developer": "Nexesenex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7534, - "hfopenllm_v2/BBH": 0.5312, - "hfopenllm_v2/MATH Level 5": 0.2017, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.3747, - "hfopenllm_v2/MMLU-PRO": 0.3733 - } - }, - { - "id": "Nexesenex/Llama_3.1_8b_Dolermed_R1_V1.03", - "name": "Llama_3.1_8b_Dolermed_R1_V1.03", - "developer": "Nexesenex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7564, - "hfopenllm_v2/BBH": 0.5316, - "hfopenllm_v2/MATH Level 5": 0.2092, - "hfopenllm_v2/GPQA": 0.318, - "hfopenllm_v2/MUSR": 0.38, - "hfopenllm_v2/MMLU-PRO": 0.372 - } - }, - { - "id": "Nexesenex/Llama_3.1_8b_Dolermed_V1.01", - "name": "Llama_3.1_8b_Dolermed_V1.01", - "developer": "Nexesenex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5087, - "hfopenllm_v2/BBH": 0.5194, - "hfopenllm_v2/MATH Level 5": 0.1344, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.3945, - "hfopenllm_v2/MMLU-PRO": 0.357 - } - }, - { - "id": "Nexesenex/Llama_3.1_8b_Dolerstormed_V1.04", - "name": "Llama_3.1_8b_Dolerstormed_V1.04", - "developer": "Nexesenex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7889, - "hfopenllm_v2/BBH": 0.5195, - "hfopenllm_v2/MATH Level 5": 0.1926, - "hfopenllm_v2/GPQA": 0.3221, - "hfopenllm_v2/MUSR": 0.403, - "hfopenllm_v2/MMLU-PRO": 0.3889 - } - }, - { - "id": "Nexesenex/Llama_3.1_8b_Hermedash_R1_V1.04", - "name": "Llama_3.1_8b_Hermedash_R1_V1.04", - "developer": "Nexesenex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7872, - "hfopenllm_v2/BBH": 0.5192, - "hfopenllm_v2/MATH Level 5": 0.1866, - "hfopenllm_v2/GPQA": 0.323, - "hfopenllm_v2/MUSR": 0.4111, - "hfopenllm_v2/MMLU-PRO": 0.3882 - } - }, - { - "id": "Nexesenex/Llama_3.1_8b_Hermedive_R1_V1.01", - "name": "Llama_3.1_8b_Hermedive_R1_V1.01", - "developer": "Nexesenex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5001, - "hfopenllm_v2/BBH": 0.5171, - "hfopenllm_v2/MATH Level 5": 0.1775, - "hfopenllm_v2/GPQA": 0.2827, - "hfopenllm_v2/MUSR": 0.4008, - "hfopenllm_v2/MMLU-PRO": 0.3427 - } - }, - { - "id": "Nexesenex/Llama_3.1_8b_Hermedive_R1_V1.03", - "name": "Llama_3.1_8b_Hermedive_R1_V1.03", - "developer": "Nexesenex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6648, - "hfopenllm_v2/BBH": 0.5141, - "hfopenllm_v2/MATH Level 5": 0.1858, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.3613, - "hfopenllm_v2/MMLU-PRO": 0.3488 - } - }, - { - "id": "Nexesenex/Llama_3.1_8b_Hermedive_V1.01", - "name": "Llama_3.1_8b_Hermedive_V1.01", - "developer": "Nexesenex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5062, - "hfopenllm_v2/BBH": 0.4918, - "hfopenllm_v2/MATH Level 5": 0.1647, - "hfopenllm_v2/GPQA": 0.2894, - "hfopenllm_v2/MUSR": 0.3697, - "hfopenllm_v2/MMLU-PRO": 0.3551 - } - }, - { - "id": "Nexesenex/Llama_3.1_8b_Mediver_V1.01", - "name": "Llama_3.1_8b_Mediver_V1.01", - "developer": "Nexesenex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1885, - "hfopenllm_v2/BBH": 0.4415, - "hfopenllm_v2/MATH Level 5": 0.0015, - "hfopenllm_v2/GPQA": 0.2777, - "hfopenllm_v2/MUSR": 0.3898, - "hfopenllm_v2/MMLU-PRO": 0.2994 - } - }, - { - "id": "Nexesenex/Llama_3.1_8b_Medusa_v1.01", - "name": "Llama_3.1_8b_Medusa_v1.01", - "developer": "Nexesenex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7685, - "hfopenllm_v2/BBH": 0.5018, - "hfopenllm_v2/MATH Level 5": 0.1465, - "hfopenllm_v2/GPQA": 0.2919, - "hfopenllm_v2/MUSR": 0.4067, - "hfopenllm_v2/MMLU-PRO": 0.3531 - } - }, - { - "id": "Nexesenex/Llama_3.1_8b_Smarteaz_0.2_R1", - "name": "Llama_3.1_8b_Smarteaz_0.2_R1", - "developer": "Nexesenex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6346, - "hfopenllm_v2/BBH": 0.5113, - "hfopenllm_v2/MATH Level 5": 0.2606, - "hfopenllm_v2/GPQA": 0.3003, - "hfopenllm_v2/MUSR": 0.4188, - "hfopenllm_v2/MMLU-PRO": 0.3645 - } - }, - { - "id": "Nexesenex/Llama_3.1_8b_Smarteaz_V1.01", - "name": "Llama_3.1_8b_Smarteaz_V1.01", - "developer": "Nexesenex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8151, - "hfopenllm_v2/BBH": 0.5241, - "hfopenllm_v2/MATH Level 5": 0.2341, - "hfopenllm_v2/GPQA": 0.3096, - "hfopenllm_v2/MUSR": 0.3789, - "hfopenllm_v2/MMLU-PRO": 0.3736 - } - }, - { - "id": "Nexesenex/Llama_3.1_8b_Stormeder_v1.04", - "name": "Llama_3.1_8b_Stormeder_v1.04", - "developer": "Nexesenex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7853, - "hfopenllm_v2/BBH": 0.5207, - "hfopenllm_v2/MATH Level 5": 0.185, - "hfopenllm_v2/GPQA": 0.3205, - "hfopenllm_v2/MUSR": 0.3949, - "hfopenllm_v2/MMLU-PRO": 0.3852 - } - }, - { - "id": "Nexesenex/Llama_3.1_8b_Typhoon_v1.03", - "name": "Llama_3.1_8b_Typhoon_v1.03", - "developer": "Nexesenex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8078, - "hfopenllm_v2/BBH": 0.5314, - "hfopenllm_v2/MATH Level 5": 0.2273, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.3815, - "hfopenllm_v2/MMLU-PRO": 0.3842 - } - }, - { - "id": "Nexesenex/Llama_3.2_1b_AquaSyn_0.1", - "name": "Llama_3.2_1b_AquaSyn_0.1", - "developer": "Nexesenex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2741, - "hfopenllm_v2/BBH": 0.3284, - "hfopenllm_v2/MATH Level 5": 0.0219, - "hfopenllm_v2/GPQA": 0.2483, - "hfopenllm_v2/MUSR": 0.346, - "hfopenllm_v2/MMLU-PRO": 0.1378 - } - }, - { - "id": "Nexesenex/Llama_3.2_1b_AquaSyn_0.11", - "name": "Llama_3.2_1b_AquaSyn_0.11", - "developer": "Nexesenex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2431, - "hfopenllm_v2/BBH": 0.3112, - "hfopenllm_v2/MATH Level 5": 0.0234, - "hfopenllm_v2/GPQA": 0.2651, - "hfopenllm_v2/MUSR": 0.3368, - "hfopenllm_v2/MMLU-PRO": 0.1116 - } - }, - { - "id": "Nexesenex/Llama_3.2_1b_Dolto_0.1", - "name": "Llama_3.2_1b_Dolto_0.1", - "developer": "Nexesenex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5434, - "hfopenllm_v2/BBH": 0.335, - "hfopenllm_v2/MATH Level 5": 0.037, - "hfopenllm_v2/GPQA": 0.2374, - "hfopenllm_v2/MUSR": 0.3421, - "hfopenllm_v2/MMLU-PRO": 0.1364 - } - }, - { - "id": "Nexesenex/Llama_3.2_1b_Odyssea_V1", - "name": "Llama_3.2_1b_Odyssea_V1", - "developer": "Nexesenex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2553, - "hfopenllm_v2/BBH": 0.301, - "hfopenllm_v2/MATH Level 5": 0.0144, - "hfopenllm_v2/GPQA": 0.2584, - "hfopenllm_v2/MUSR": 0.3394, - "hfopenllm_v2/MMLU-PRO": 0.1153 - } - }, - { - "id": "Nexesenex/Llama_3.2_1b_Odyssea_V1.01", - "name": "Llama_3.2_1b_Odyssea_V1.01", - "developer": "Nexesenex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2495, - "hfopenllm_v2/BBH": 0.3045, - "hfopenllm_v2/MATH Level 5": 0.0174, - "hfopenllm_v2/GPQA": 0.2559, - "hfopenllm_v2/MUSR": 0.342, - "hfopenllm_v2/MMLU-PRO": 0.1152 - } - }, - { - "id": "Nexesenex/Llama_3.2_1b_OpenTree_R1_0.1", - "name": "Llama_3.2_1b_OpenTree_R1_0.1", - "developer": "Nexesenex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5366, - "hfopenllm_v2/BBH": 0.328, - "hfopenllm_v2/MATH Level 5": 0.0476, - "hfopenllm_v2/GPQA": 0.2525, - "hfopenllm_v2/MUSR": 0.3131, - "hfopenllm_v2/MMLU-PRO": 0.1675 - } - }, - { - "id": "Nexesenex/Llama_3.2_1b_OrcaSun_V1", - "name": "Llama_3.2_1b_OrcaSun_V1", - "developer": "Nexesenex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5949, - "hfopenllm_v2/BBH": 0.355, - "hfopenllm_v2/MATH Level 5": 0.0597, - "hfopenllm_v2/GPQA": 0.2366, - "hfopenllm_v2/MUSR": 0.338, - "hfopenllm_v2/MMLU-PRO": 0.1904 - } - }, - { - "id": "Nexesenex/Llama_3.2_1b_RandomLego_RP_R1_0.1", - "name": "Llama_3.2_1b_RandomLego_RP_R1_0.1", - "developer": "Nexesenex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5543, - "hfopenllm_v2/BBH": 0.3428, - "hfopenllm_v2/MATH Level 5": 0.0566, - "hfopenllm_v2/GPQA": 0.25, - "hfopenllm_v2/MUSR": 0.3249, - "hfopenllm_v2/MMLU-PRO": 0.1563 - } - }, - { - "id": "Nexesenex/Llama_3.2_1b_SunOrca_V1", - "name": "Llama_3.2_1b_SunOrca_V1", - "developer": "Nexesenex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.543, - "hfopenllm_v2/BBH": 0.3431, - "hfopenllm_v2/MATH Level 5": 0.0672, - "hfopenllm_v2/GPQA": 0.2743, - "hfopenllm_v2/MUSR": 0.3262, - "hfopenllm_v2/MMLU-PRO": 0.1884 - } - }, - { - "id": "Nexesenex/Llama_3.2_1b_Sydonia_0.1", - "name": "Llama_3.2_1b_Sydonia_0.1", - "developer": "Nexesenex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2197, - "hfopenllm_v2/BBH": 0.3121, - "hfopenllm_v2/MATH Level 5": 0.0204, - "hfopenllm_v2/GPQA": 0.2282, - "hfopenllm_v2/MUSR": 0.3382, - "hfopenllm_v2/MMLU-PRO": 0.1224 - } - }, - { - "id": "Nexesenex/Llama_3.2_1b_Syneridol_0.2", - "name": "Llama_3.2_1b_Syneridol_0.2", - "developer": "Nexesenex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2157, - "hfopenllm_v2/BBH": 0.3139, - "hfopenllm_v2/MATH Level 5": 0.0219, - "hfopenllm_v2/GPQA": 0.2349, - "hfopenllm_v2/MUSR": 0.3343, - "hfopenllm_v2/MMLU-PRO": 0.1227 - } - }, - { - "id": "Nexesenex/Llama_3.2_1b_Synopsys_0.1", - "name": "Llama_3.2_1b_Synopsys_0.1", - "developer": "Nexesenex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1764, - "hfopenllm_v2/BBH": 0.3162, - "hfopenllm_v2/MATH Level 5": 0.0166, - "hfopenllm_v2/GPQA": 0.2391, - "hfopenllm_v2/MUSR": 0.3461, - "hfopenllm_v2/MMLU-PRO": 0.1231 - } - }, - { - "id": "Nexesenex/Llama_3.2_1b_Synopsys_0.11", - "name": "Llama_3.2_1b_Synopsys_0.11", - "developer": "Nexesenex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2842, - "hfopenllm_v2/BBH": 0.3102, - "hfopenllm_v2/MATH Level 5": 0.0128, - "hfopenllm_v2/GPQA": 0.2626, - "hfopenllm_v2/MUSR": 0.3513, - "hfopenllm_v2/MMLU-PRO": 0.1123 - } - }, - { - "id": "Nexesenex/Llama_3.2_3b_Kermes_v1", - "name": "Llama_3.2_3b_Kermes_v1", - "developer": "Nexesenex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4852, - "hfopenllm_v2/BBH": 0.441, - "hfopenllm_v2/MATH Level 5": 0.031, - "hfopenllm_v2/GPQA": 0.2735, - "hfopenllm_v2/MUSR": 0.407, - "hfopenllm_v2/MMLU-PRO": 0.2547 - } - }, - { - "id": "Nexesenex/Llama_3.2_3b_Kermes_v2", - "name": "Llama_3.2_3b_Kermes_v2", - "developer": "Nexesenex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5754, - "hfopenllm_v2/BBH": 0.4455, - "hfopenllm_v2/MATH Level 5": 0.0544, - "hfopenllm_v2/GPQA": 0.2651, - "hfopenllm_v2/MUSR": 0.3778, - "hfopenllm_v2/MMLU-PRO": 0.2734 - } - }, - { - "id": "Nexesenex/Llama_3.2_3b_Kermes_v2.1", - "name": "Llama_3.2_3b_Kermes_v2.1", - "developer": "Nexesenex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5584, - "hfopenllm_v2/BBH": 0.4464, - "hfopenllm_v2/MATH Level 5": 0.0521, - "hfopenllm_v2/GPQA": 0.2794, - "hfopenllm_v2/MUSR": 0.3964, - "hfopenllm_v2/MMLU-PRO": 0.2692 - } - }, - { - "id": "Nexesenex/Nemotron_W_4b_Halo_0.1", - "name": "Nemotron_W_4b_Halo_0.1", - "developer": "Nexesenex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3627, - "hfopenllm_v2/BBH": 0.4135, - "hfopenllm_v2/MATH Level 5": 0.0423, - "hfopenllm_v2/GPQA": 0.2802, - "hfopenllm_v2/MUSR": 0.4165, - "hfopenllm_v2/MMLU-PRO": 0.2505 - } - }, - { - "id": "Nexesenex/Nemotron_W_4b_MagLight_0.1", - "name": "Nemotron_W_4b_MagLight_0.1", - "developer": "Nexesenex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.423, - "hfopenllm_v2/BBH": 0.4231, - "hfopenllm_v2/MATH Level 5": 0.04, - "hfopenllm_v2/GPQA": 0.2836, - "hfopenllm_v2/MUSR": 0.4112, - "hfopenllm_v2/MMLU-PRO": 0.2545 - } - }, - { - "id": "Nexesenex/pankajmathur_orca_mini_v9_6_1B-instruct-Abliterated-LPL", - "name": "pankajmathur_orca_mini_v9_6_1B-instruct-Abliterated-LPL", - "developer": "Nexesenex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.589, - "hfopenllm_v2/BBH": 0.3562, - "hfopenllm_v2/MATH Level 5": 0.0748, - "hfopenllm_v2/GPQA": 0.2668, - "hfopenllm_v2/MUSR": 0.3396, - "hfopenllm_v2/MMLU-PRO": 0.1803 - } - }, - { - "id": "Nexesenex/Qwen_2.5_3b_Smarteaz_0.01a", - "name": "Qwen_2.5_3b_Smarteaz_0.01a", - "developer": "Nexesenex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4012, - "hfopenllm_v2/BBH": 0.4637, - "hfopenllm_v2/MATH Level 5": 0.1805, - "hfopenllm_v2/GPQA": 0.2777, - "hfopenllm_v2/MUSR": 0.432, - "hfopenllm_v2/MMLU-PRO": 0.286 - } - }, - { - "id": "Nexusflow/NexusRaven-V2-13B", - "name": "NexusRaven-V2-13B", - "developer": "Nexusflow", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1791, - "hfopenllm_v2/BBH": 0.3949, - "hfopenllm_v2/MATH Level 5": 0.0295, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3737, - "hfopenllm_v2/MMLU-PRO": 0.1872 - } - }, - { - "id": "Nexusflow/Starling-RM-34B", - "name": "Nexusflow/Starling-RM-34B", - "developer": "Nexusflow", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.8133, - "reward-bench/Factuality": 0.4589, - "reward-bench/Precise IF": 0.3187, - "reward-bench/Math": 0.6175, - "reward-bench/Safety": 0.877, - "reward-bench/Focus": 0.4808, - "reward-bench/Ties": 0.1004, - "reward-bench/Chat": 0.9693, - "reward-bench/Chat Hard": 0.5724, - "reward-bench/Reasoning": 0.8845, - "reward-bench/Prior Sets (0.5 weight)": 0.7137 - } - }, - { - "id": "nguyentd/FinancialAdvice-Qwen2.5-7B", - "name": "FinancialAdvice-Qwen2.5-7B", - "developer": "nguyentd", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4496, - "hfopenllm_v2/BBH": 0.4731, - "hfopenllm_v2/MATH Level 5": 0.1148, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.4025, - "hfopenllm_v2/MMLU-PRO": 0.3752 - } - }, - { - "id": "ngxson/MiniThinky-1B-Llama-3.2", - "name": "MiniThinky-1B-Llama-3.2", - "developer": "ngxson", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2771, - "hfopenllm_v2/BBH": 0.3142, - "hfopenllm_v2/MATH Level 5": 0.0574, - "hfopenllm_v2/GPQA": 0.2391, - "hfopenllm_v2/MUSR": 0.3434, - "hfopenllm_v2/MMLU-PRO": 0.1147 - } - }, - { - "id": "ngxson/MiniThinky-v2-1B-Llama-3.2", - "name": "MiniThinky-v2-1B-Llama-3.2", - "developer": "ngxson", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2963, - "hfopenllm_v2/BBH": 0.3205, - "hfopenllm_v2/MATH Level 5": 0.0287, - "hfopenllm_v2/GPQA": 0.2399, - "hfopenllm_v2/MUSR": 0.3356, - "hfopenllm_v2/MMLU-PRO": 0.1116 - } - }, - { - "id": "nhyha/merge_Qwen2.5-7B-Instruct_20241023_0314", - "name": "merge_Qwen2.5-7B-Instruct_20241023_0314", - "developer": "nhyha", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5695, - "hfopenllm_v2/BBH": 0.5559, - "hfopenllm_v2/MATH Level 5": 0.3542, - "hfopenllm_v2/GPQA": 0.3213, - "hfopenllm_v2/MUSR": 0.4251, - "hfopenllm_v2/MMLU-PRO": 0.4542 - } - }, - { - "id": "nhyha/N3N_Delirium-v1_1030_0227", - "name": "N3N_Delirium-v1_1030_0227", - "developer": "nhyha", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8023, - "hfopenllm_v2/BBH": 0.5891, - "hfopenllm_v2/MATH Level 5": 0.2107, - "hfopenllm_v2/GPQA": 0.3372, - "hfopenllm_v2/MUSR": 0.4098, - "hfopenllm_v2/MMLU-PRO": 0.415 - } - }, - { - "id": "nhyha/N3N_gemma-2-9b-it_20241029_1532", - "name": "N3N_gemma-2-9b-it_20241029_1532", - "developer": "nhyha", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6752, - "hfopenllm_v2/BBH": 0.5863, - "hfopenllm_v2/MATH Level 5": 0.2122, - "hfopenllm_v2/GPQA": 0.3406, - "hfopenllm_v2/MUSR": 0.4594, - "hfopenllm_v2/MMLU-PRO": 0.4122 - } - }, - { - "id": "nhyha/N3N_gemma-2-9b-it_20241110_2026", - "name": "N3N_gemma-2-9b-it_20241110_2026", - "developer": "nhyha", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6283, - "hfopenllm_v2/BBH": 0.5867, - "hfopenllm_v2/MATH Level 5": 0.1609, - "hfopenllm_v2/GPQA": 0.3364, - "hfopenllm_v2/MUSR": 0.4073, - "hfopenllm_v2/MMLU-PRO": 0.402 - } - }, - { - "id": "nhyha/N3N_Llama-3.1-8B-Instruct_1028_0216", - "name": "N3N_Llama-3.1-8B-Instruct_1028_0216", - "developer": "nhyha", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4796, - "hfopenllm_v2/BBH": 0.5054, - "hfopenllm_v2/MATH Level 5": 0.1707, - "hfopenllm_v2/GPQA": 0.3062, - "hfopenllm_v2/MUSR": 0.405, - "hfopenllm_v2/MMLU-PRO": 0.3638 - } - }, - { - "id": "nicolinho/QRM-Gemma-2-27B", - "name": "nicolinho/QRM-Gemma-2-27B", - "developer": "nicolinho", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.9444, - "reward-bench/Factuality": 0.7853, - "reward-bench/Precise IF": 0.3719, - "reward-bench/Math": 0.6995, - "reward-bench/Safety": 0.927, - "reward-bench/Focus": 0.9535, - "reward-bench/Ties": 0.8321, - "reward-bench/Chat": 0.9665, - "reward-bench/Chat Hard": 0.9013, - "reward-bench/Reasoning": 0.9826 - } - }, - { - "id": "nicolinho/QRM-Llama3-8B", - "name": "nicolinho/QRM-Llama3-8B", - "developer": "nicolinho", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.911, - "reward-bench/Chat": 0.9581, - "reward-bench/Chat Hard": 0.8114, - "reward-bench/Safety": 0.8986, - "reward-bench/Reasoning": 0.9758 - } - }, - { - "id": "nicolinho/QRM-Llama3.1-8B", - "name": "nicolinho/QRM-Llama3.1-8B", - "developer": "nicolinho", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.9306, - "reward-bench/Chat": 0.9441, - "reward-bench/Chat Hard": 0.8969, - "reward-bench/Safety": 0.923, - "reward-bench/Reasoning": 0.9583 - } - }, - { - "id": "nicolinho/QRM-Llama3.1-8B-v2", - "name": "nicolinho/QRM-Llama3.1-8B-v2", - "developer": "nicolinho", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7074, - "reward-bench/Chat": 0.9637, - "reward-bench/Chat Hard": 0.8684, - "reward-bench/Safety": 0.9467, - "reward-bench/Reasoning": 0.9677, - "reward-bench/Factuality": 0.6653, - "reward-bench/Precise IF": 0.4062, - "reward-bench/Math": 0.612, - "reward-bench/Focus": 0.8909, - "reward-bench/Ties": 0.7234 - } - }, - { - "id": "nidum/Nidum-Limitless-Gemma-2B", - "name": "Nidum-Limitless-Gemma-2B", - "developer": "nidum", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2424, - "hfopenllm_v2/BBH": 0.3079, - "hfopenllm_v2/MATH Level 5": 0.0136, - "hfopenllm_v2/GPQA": 0.2643, - "hfopenllm_v2/MUSR": 0.374, - "hfopenllm_v2/MMLU-PRO": 0.1174 - } - }, - { - "id": "NikolaSigmoid/AceMath-1.5B-Instruct-1epoch", - "name": "AceMath-1.5B-Instruct-1epoch", - "developer": "NikolaSigmoid", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2849, - "hfopenllm_v2/BBH": 0.4263, - "hfopenllm_v2/MATH Level 5": 0.3051, - "hfopenllm_v2/GPQA": 0.2777, - "hfopenllm_v2/MUSR": 0.3925, - "hfopenllm_v2/MMLU-PRO": 0.2376 - } - }, - { - "id": "NikolaSigmoid/AceMath-1.5B-Instruct-dolphin-r1-200", - "name": "AceMath-1.5B-Instruct-dolphin-r1-200", - "developer": "NikolaSigmoid", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1808, - "hfopenllm_v2/BBH": 0.2815, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2559, - "hfopenllm_v2/MUSR": 0.375, - "hfopenllm_v2/MMLU-PRO": 0.1143 - } - }, - { - "id": "NikolaSigmoid/acemath-200", - "name": "acemath-200", - "developer": "NikolaSigmoid", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2849, - "hfopenllm_v2/BBH": 0.4263, - "hfopenllm_v2/MATH Level 5": 0.3051, - "hfopenllm_v2/GPQA": 0.2777, - "hfopenllm_v2/MUSR": 0.3925, - "hfopenllm_v2/MMLU-PRO": 0.2376 - } - }, - { - "id": "NikolaSigmoid/DeepSeek-R1-Distill-Qwen-1.5B-500", - "name": "DeepSeek-R1-Distill-Qwen-1.5B-500", - "developer": "NikolaSigmoid", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1749, - "hfopenllm_v2/BBH": 0.2602, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2458, - "hfopenllm_v2/MUSR": 0.338, - "hfopenllm_v2/MMLU-PRO": 0.1125 - } - }, - { - "id": "NikolaSigmoid/phi-4-14b", - "name": "phi-4-14b", - "developer": "NikolaSigmoid", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0561, - "hfopenllm_v2/BBH": 0.6695, - "hfopenllm_v2/MATH Level 5": 0.2938, - "hfopenllm_v2/GPQA": 0.4035, - "hfopenllm_v2/MUSR": 0.5047, - "hfopenllm_v2/MMLU-PRO": 0.5278 - } - }, - { - "id": "NikolaSigmoid/phi-4-1steps", - "name": "phi-4-1steps", - "developer": "NikolaSigmoid", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0528, - "hfopenllm_v2/BBH": 0.6707, - "hfopenllm_v2/MATH Level 5": 0.2983, - "hfopenllm_v2/GPQA": 0.4018, - "hfopenllm_v2/MUSR": 0.5021, - "hfopenllm_v2/MMLU-PRO": 0.5273 - } - }, - { - "id": "NikolaSigmoid/phi-4-300steps", - "name": "phi-4-300steps", - "developer": "NikolaSigmoid", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0561, - "hfopenllm_v2/BBH": 0.6701, - "hfopenllm_v2/MATH Level 5": 0.2946, - "hfopenllm_v2/GPQA": 0.4052, - "hfopenllm_v2/MUSR": 0.5034, - "hfopenllm_v2/MMLU-PRO": 0.5288 - } - }, - { - "id": "nisten/franqwenstein-35b", - "name": "franqwenstein-35b", - "developer": "nisten", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3914, - "hfopenllm_v2/BBH": 0.6591, - "hfopenllm_v2/MATH Level 5": 0.3044, - "hfopenllm_v2/GPQA": 0.3591, - "hfopenllm_v2/MUSR": 0.4681, - "hfopenllm_v2/MMLU-PRO": 0.5611 - } - }, - { - "id": "nisten/tqwendo-36b", - "name": "tqwendo-36b", - "developer": "nisten", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6778, - "hfopenllm_v2/BBH": 0.6432, - "hfopenllm_v2/MATH Level 5": 0.4154, - "hfopenllm_v2/GPQA": 0.3314, - "hfopenllm_v2/MUSR": 0.443, - "hfopenllm_v2/MMLU-PRO": 0.4381 - } - }, - { - "id": "Nitral-AI/Captain-Eris-BMO_Violent-GRPO-v0.420", - "name": "Captain-Eris-BMO_Violent-GRPO-v0.420", - "developer": "Nitral-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6313, - "hfopenllm_v2/BBH": 0.5079, - "hfopenllm_v2/MATH Level 5": 0.1314, - "hfopenllm_v2/GPQA": 0.3096, - "hfopenllm_v2/MUSR": 0.4228, - "hfopenllm_v2/MMLU-PRO": 0.3596 - } - }, - { - "id": "Nitral-AI/Captain-Eris_BMO-Violent-12B", - "name": "Captain-Eris_BMO-Violent-12B", - "developer": "Nitral-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6152, - "hfopenllm_v2/BBH": 0.5104, - "hfopenllm_v2/MATH Level 5": 0.1367, - "hfopenllm_v2/GPQA": 0.3096, - "hfopenllm_v2/MUSR": 0.4255, - "hfopenllm_v2/MMLU-PRO": 0.3571 - } - }, - { - "id": "Nitral-AI/Captain-Eris_Violet-GRPO-v0.420", - "name": "Captain-Eris_Violet-GRPO-v0.420", - "developer": "Nitral-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6262, - "hfopenllm_v2/BBH": 0.5159, - "hfopenllm_v2/MATH Level 5": 0.108, - "hfopenllm_v2/GPQA": 0.2987, - "hfopenllm_v2/MUSR": 0.4279, - "hfopenllm_v2/MMLU-PRO": 0.3535 - } - }, - { - "id": "Nitral-AI/Captain-Eris_Violet-V0.420-12B", - "name": "Captain-Eris_Violet-V0.420-12B", - "developer": "Nitral-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4339, - "hfopenllm_v2/BBH": 0.5478, - "hfopenllm_v2/MATH Level 5": 0.1073, - "hfopenllm_v2/GPQA": 0.3112, - "hfopenllm_v2/MUSR": 0.4331, - "hfopenllm_v2/MMLU-PRO": 0.3723 - } - }, - { - "id": "Nitral-AI/Captain_BMO-12B", - "name": "Captain_BMO-12B", - "developer": "Nitral-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4751, - "hfopenllm_v2/BBH": 0.5286, - "hfopenllm_v2/MATH Level 5": 0.1397, - "hfopenllm_v2/GPQA": 0.3196, - "hfopenllm_v2/MUSR": 0.3748, - "hfopenllm_v2/MMLU-PRO": 0.3569 - } - }, - { - "id": "Nitral-AI/Hathor_Stable-v0.2-L3-8B", - "name": "Hathor_Stable-v0.2-L3-8B", - "developer": "Nitral-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7175, - "hfopenllm_v2/BBH": 0.5286, - "hfopenllm_v2/MATH Level 5": 0.105, - "hfopenllm_v2/GPQA": 0.2869, - "hfopenllm_v2/MUSR": 0.3781, - "hfopenllm_v2/MMLU-PRO": 0.3696 - } - }, - { - "id": "Nitral-AI/Hathor_Tahsin-L3-8B-v0.85", - "name": "Hathor_Tahsin-L3-8B-v0.85", - "developer": "Nitral-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.711, - "hfopenllm_v2/BBH": 0.5279, - "hfopenllm_v2/MATH Level 5": 0.1005, - "hfopenllm_v2/GPQA": 0.2852, - "hfopenllm_v2/MUSR": 0.3647, - "hfopenllm_v2/MMLU-PRO": 0.372 - } - }, - { - "id": "Nitral-AI/Nera_Noctis-12B", - "name": "Nera_Noctis-12B", - "developer": "Nitral-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4562, - "hfopenllm_v2/BBH": 0.5194, - "hfopenllm_v2/MATH Level 5": 0.0876, - "hfopenllm_v2/GPQA": 0.2634, - "hfopenllm_v2/MUSR": 0.3979, - "hfopenllm_v2/MMLU-PRO": 0.3468 - } - }, - { - "id": "NJS26/NJS_777", - "name": "NJS_777", - "developer": "NJS26", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1881, - "hfopenllm_v2/BBH": 0.2178, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2064, - "hfopenllm_v2/MUSR": 0.3538, - "hfopenllm_v2/MMLU-PRO": 0.1163 - } - }, - { - "id": "NLPark/AnFeng_v3.1-Avocet", - "name": "AnFeng_v3.1-Avocet", - "developer": "NLPark", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5096, - "hfopenllm_v2/BBH": 0.5829, - "hfopenllm_v2/MATH Level 5": 0.1594, - "hfopenllm_v2/GPQA": 0.3247, - "hfopenllm_v2/MUSR": 0.4476, - "hfopenllm_v2/MMLU-PRO": 0.4438 - } - }, - { - "id": "NLPark/B-and-W_Flycatcher-3AD1E", - "name": "B-and-W_Flycatcher-3AD1E", - "developer": "NLPark", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4908, - "hfopenllm_v2/BBH": 0.6065, - "hfopenllm_v2/MATH Level 5": 0.2379, - "hfopenllm_v2/GPQA": 0.3305, - "hfopenllm_v2/MUSR": 0.4423, - "hfopenllm_v2/MMLU-PRO": 0.4741 - } - }, - { - "id": "NLPark/Shi-Ci-Robin-Test_3AD80", - "name": "Shi-Ci-Robin-Test_3AD80", - "developer": "NLPark", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7227, - "hfopenllm_v2/BBH": 0.6705, - "hfopenllm_v2/MATH Level 5": 0.3157, - "hfopenllm_v2/GPQA": 0.3599, - "hfopenllm_v2/MUSR": 0.4696, - "hfopenllm_v2/MMLU-PRO": 0.5121 - } - }, - { - "id": "nlpguy/Lion-Lamarck-v.1.0.8", - "name": "Lion-Lamarck-v.1.0.8", - "developer": "nlpguy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4509, - "hfopenllm_v2/BBH": 0.5869, - "hfopenllm_v2/MATH Level 5": 0.5544, - "hfopenllm_v2/GPQA": 0.3582, - "hfopenllm_v2/MUSR": 0.4673, - "hfopenllm_v2/MMLU-PRO": 0.4643 - } - }, - { - "id": "nlpguy/Lion-Lamarck-v.1.0.9", - "name": "Lion-Lamarck-v.1.0.9", - "developer": "nlpguy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3409, - "hfopenllm_v2/BBH": 0.5918, - "hfopenllm_v2/MATH Level 5": 0.5642, - "hfopenllm_v2/GPQA": 0.3901, - "hfopenllm_v2/MUSR": 0.53, - "hfopenllm_v2/MMLU-PRO": 0.4704 - } - }, - { - "id": "nlpguy/Lion-Lamarck-v.1.1.0", - "name": "Lion-Lamarck-v.1.1.0", - "developer": "nlpguy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3658, - "hfopenllm_v2/BBH": 0.5962, - "hfopenllm_v2/MATH Level 5": 0.5755, - "hfopenllm_v2/GPQA": 0.3926, - "hfopenllm_v2/MUSR": 0.5325, - "hfopenllm_v2/MMLU-PRO": 0.4631 - } - }, - { - "id": "nlpguy/Miisce-one", - "name": "Miisce-one", - "developer": "nlpguy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6066, - "hfopenllm_v2/BBH": 0.6505, - "hfopenllm_v2/MATH Level 5": 0.4169, - "hfopenllm_v2/GPQA": 0.3859, - "hfopenllm_v2/MUSR": 0.482, - "hfopenllm_v2/MMLU-PRO": 0.5412 - } - }, - { - "id": "nlpguy/Mistral-NeMo-Minitron-Upscale-v1", - "name": "Mistral-NeMo-Minitron-Upscale-v1", - "developer": "nlpguy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1648, - "hfopenllm_v2/BBH": 0.4468, - "hfopenllm_v2/MATH Level 5": 0.0144, - "hfopenllm_v2/GPQA": 0.2802, - "hfopenllm_v2/MUSR": 0.3804, - "hfopenllm_v2/MMLU-PRO": 0.2537 - } - }, - { - "id": "nlpguy/Mistral-NeMo-Minitron-Upscale-v2", - "name": "Mistral-NeMo-Minitron-Upscale-v2", - "developer": "nlpguy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1573, - "hfopenllm_v2/BBH": 0.395, - "hfopenllm_v2/MATH Level 5": 0.0128, - "hfopenllm_v2/GPQA": 0.2735, - "hfopenllm_v2/MUSR": 0.3791, - "hfopenllm_v2/MMLU-PRO": 0.1927 - } - }, - { - "id": "nlpguy/Mistral-NeMo-Minitron-Upscale-v3", - "name": "Mistral-NeMo-Minitron-Upscale-v3", - "developer": "nlpguy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1412, - "hfopenllm_v2/BBH": 0.3052, - "hfopenllm_v2/MATH Level 5": 0.0113, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.4098, - "hfopenllm_v2/MMLU-PRO": 0.1171 - } - }, - { - "id": "nlpguy/StableProse", - "name": "StableProse", - "developer": "nlpguy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1972, - "hfopenllm_v2/BBH": 0.5117, - "hfopenllm_v2/MATH Level 5": 0.065, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.4067, - "hfopenllm_v2/MMLU-PRO": 0.3468 - } - }, - { - "id": "nlpguy/StarFusion-alpha1", - "name": "StarFusion-alpha1", - "developer": "nlpguy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.566, - "hfopenllm_v2/BBH": 0.4429, - "hfopenllm_v2/MATH Level 5": 0.0718, - "hfopenllm_v2/GPQA": 0.2953, - "hfopenllm_v2/MUSR": 0.4081, - "hfopenllm_v2/MMLU-PRO": 0.3191 - } - }, - { - "id": "Nohobby/MS-Schisandra-22B-v0.1", - "name": "MS-Schisandra-22B-v0.1", - "developer": "Nohobby", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6331, - "hfopenllm_v2/BBH": 0.579, - "hfopenllm_v2/MATH Level 5": 0.2228, - "hfopenllm_v2/GPQA": 0.3322, - "hfopenllm_v2/MUSR": 0.3928, - "hfopenllm_v2/MMLU-PRO": 0.4096 - } - }, - { - "id": "Nohobby/MS-Schisandra-22B-v0.2", - "name": "MS-Schisandra-22B-v0.2", - "developer": "Nohobby", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6383, - "hfopenllm_v2/BBH": 0.5841, - "hfopenllm_v2/MATH Level 5": 0.2032, - "hfopenllm_v2/GPQA": 0.3356, - "hfopenllm_v2/MUSR": 0.4075, - "hfopenllm_v2/MMLU-PRO": 0.4136 - } - }, - { - "id": "noname0202/gemma-2-2b-it-ties", - "name": "gemma-2-2b-it-ties", - "developer": "noname0202", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1266, - "hfopenllm_v2/BBH": 0.4206, - "hfopenllm_v2/MATH Level 5": 0.0242, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.3929, - "hfopenllm_v2/MMLU-PRO": 0.2561 - } - }, - { - "id": "noname0202/gemma-2-9b-sft-jp-en-zh-v1", - "name": "gemma-2-9b-sft-jp-en-zh-v1", - "developer": "noname0202", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2988, - "hfopenllm_v2/BBH": 0.4519, - "hfopenllm_v2/MATH Level 5": 0.0891, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.408, - "hfopenllm_v2/MMLU-PRO": 0.3125 - } - }, - { - "id": "noname0202/gemma-2-9b-sft-jp-en-zh-v2", - "name": "gemma-2-9b-sft-jp-en-zh-v2", - "developer": "noname0202", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3993, - "hfopenllm_v2/BBH": 0.4515, - "hfopenllm_v2/MATH Level 5": 0.1042, - "hfopenllm_v2/GPQA": 0.2878, - "hfopenllm_v2/MUSR": 0.3612, - "hfopenllm_v2/MMLU-PRO": 0.3675 - } - }, - { - "id": "noname0202/Llama-3.2-4x3B-Instruct", - "name": "Llama-3.2-4x3B-Instruct", - "developer": "noname0202", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7067, - "hfopenllm_v2/BBH": 0.4647, - "hfopenllm_v2/MATH Level 5": 0.1586, - "hfopenllm_v2/GPQA": 0.2727, - "hfopenllm_v2/MUSR": 0.3674, - "hfopenllm_v2/MMLU-PRO": 0.3285 - } - }, - { - "id": "noname0202/llama-math-1b-r16-0to512tokens-test", - "name": "llama-math-1b-r16-0to512tokens-test", - "developer": "noname0202", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.547, - "hfopenllm_v2/BBH": 0.3488, - "hfopenllm_v2/MATH Level 5": 0.0816, - "hfopenllm_v2/GPQA": 0.2668, - "hfopenllm_v2/MUSR": 0.3143, - "hfopenllm_v2/MMLU-PRO": 0.1728 - } - }, - { - "id": "noname0202/llama-math-1b-r32-0to512tokens-test", - "name": "llama-math-1b-r32-0to512tokens-test", - "developer": "noname0202", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5683, - "hfopenllm_v2/BBH": 0.3495, - "hfopenllm_v2/MATH Level 5": 0.0906, - "hfopenllm_v2/GPQA": 0.2651, - "hfopenllm_v2/MUSR": 0.3209, - "hfopenllm_v2/MMLU-PRO": 0.176 - } - }, - { - "id": "noname0202/llama-math-1b-r32-test", - "name": "llama-math-1b-r32-test", - "developer": "noname0202", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5819, - "hfopenllm_v2/BBH": 0.3486, - "hfopenllm_v2/MATH Level 5": 0.0725, - "hfopenllm_v2/GPQA": 0.2617, - "hfopenllm_v2/MUSR": 0.3156, - "hfopenllm_v2/MMLU-PRO": 0.1781 - } - }, - { - "id": "noname0202/llama-math-1b-r8-512tokens-test", - "name": "llama-math-1b-r8-512tokens-test", - "developer": "noname0202", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5792, - "hfopenllm_v2/BBH": 0.3496, - "hfopenllm_v2/MATH Level 5": 0.0816, - "hfopenllm_v2/GPQA": 0.2685, - "hfopenllm_v2/MUSR": 0.3169, - "hfopenllm_v2/MMLU-PRO": 0.1753 - } - }, - { - "id": "Norquinal/Alpha", - "name": "Alpha", - "developer": "Norquinal", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2803, - "hfopenllm_v2/BBH": 0.3374, - "hfopenllm_v2/MATH Level 5": 0.0574, - "hfopenllm_v2/GPQA": 0.2651, - "hfopenllm_v2/MUSR": 0.3631, - "hfopenllm_v2/MMLU-PRO": 0.3003 - } - }, - { - "id": "Norquinal/Bravo", - "name": "Bravo", - "developer": "Norquinal", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3025, - "hfopenllm_v2/BBH": 0.3558, - "hfopenllm_v2/MATH Level 5": 0.0574, - "hfopenllm_v2/GPQA": 0.2819, - "hfopenllm_v2/MUSR": 0.3869, - "hfopenllm_v2/MMLU-PRO": 0.3127 - } - }, - { - "id": "Norquinal/Charlie", - "name": "Charlie", - "developer": "Norquinal", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3061, - "hfopenllm_v2/BBH": 0.3515, - "hfopenllm_v2/MATH Level 5": 0.0582, - "hfopenllm_v2/GPQA": 0.271, - "hfopenllm_v2/MUSR": 0.3737, - "hfopenllm_v2/MMLU-PRO": 0.3093 - } - }, - { - "id": "Norquinal/Delta", - "name": "Delta", - "developer": "Norquinal", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2538, - "hfopenllm_v2/BBH": 0.3435, - "hfopenllm_v2/MATH Level 5": 0.0612, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.3777, - "hfopenllm_v2/MMLU-PRO": 0.2959 - } - }, - { - "id": "Norquinal/Echo", - "name": "Echo", - "developer": "Norquinal", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3158, - "hfopenllm_v2/BBH": 0.353, - "hfopenllm_v2/MATH Level 5": 0.0574, - "hfopenllm_v2/GPQA": 0.2794, - "hfopenllm_v2/MUSR": 0.3804, - "hfopenllm_v2/MMLU-PRO": 0.3095 - } - }, - { - "id": "Norquinal/Foxtrot", - "name": "Foxtrot", - "developer": "Norquinal", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3012, - "hfopenllm_v2/BBH": 0.3558, - "hfopenllm_v2/MATH Level 5": 0.0582, - "hfopenllm_v2/GPQA": 0.2869, - "hfopenllm_v2/MUSR": 0.3804, - "hfopenllm_v2/MMLU-PRO": 0.305 - } - }, - { - "id": "Norquinal/Golf", - "name": "Golf", - "developer": "Norquinal", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3534, - "hfopenllm_v2/BBH": 0.3533, - "hfopenllm_v2/MATH Level 5": 0.0536, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.338, - "hfopenllm_v2/MMLU-PRO": 0.3056 - } - }, - { - "id": "Norquinal/Hotel", - "name": "Hotel", - "developer": "Norquinal", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3215, - "hfopenllm_v2/BBH": 0.3679, - "hfopenllm_v2/MATH Level 5": 0.0529, - "hfopenllm_v2/GPQA": 0.2794, - "hfopenllm_v2/MUSR": 0.3288, - "hfopenllm_v2/MMLU-PRO": 0.3157 - } - }, - { - "id": "NotASI/FineTome-Llama3.2-1B-0929", - "name": "FineTome-Llama3.2-1B-0929", - "developer": "NotASI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3991, - "hfopenllm_v2/BBH": 0.3246, - "hfopenllm_v2/MATH Level 5": 0.0363, - "hfopenllm_v2/GPQA": 0.2727, - "hfopenllm_v2/MUSR": 0.3488, - "hfopenllm_v2/MMLU-PRO": 0.1429 - } - }, - { - "id": "NotASI/FineTome-Llama3.2-3B-1002", - "name": "FineTome-Llama3.2-3B-1002", - "developer": "NotASI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5474, - "hfopenllm_v2/BBH": 0.4319, - "hfopenllm_v2/MATH Level 5": 0.0627, - "hfopenllm_v2/GPQA": 0.2508, - "hfopenllm_v2/MUSR": 0.3685, - "hfopenllm_v2/MMLU-PRO": 0.2437 - } - }, - { - "id": "NotASI/FineTome-v1.5-Llama3.2-1B-1007", - "name": "FineTome-v1.5-Llama3.2-1B-1007", - "developer": "NotASI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3924, - "hfopenllm_v2/BBH": 0.3241, - "hfopenllm_v2/MATH Level 5": 0.0317, - "hfopenllm_v2/GPQA": 0.25, - "hfopenllm_v2/MUSR": 0.3475, - "hfopenllm_v2/MMLU-PRO": 0.1427 - } - }, - { - "id": "NotASI/FineTome-v1.5-Llama3.2-3B-1007", - "name": "FineTome-v1.5-Llama3.2-3B-1007", - "developer": "NotASI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5508, - "hfopenllm_v2/BBH": 0.4312, - "hfopenllm_v2/MATH Level 5": 0.0642, - "hfopenllm_v2/GPQA": 0.2617, - "hfopenllm_v2/MUSR": 0.3645, - "hfopenllm_v2/MMLU-PRO": 0.2448 - } - }, - { - "id": "notbdq/Qwen2.5-14B-Instruct-1M-GRPO-Reasoning", - "name": "Qwen2.5-14B-Instruct-1M-GRPO-Reasoning", - "developer": "notbdq", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8414, - "hfopenllm_v2/BBH": 0.6198, - "hfopenllm_v2/MATH Level 5": 0.5302, - "hfopenllm_v2/GPQA": 0.3431, - "hfopenllm_v2/MUSR": 0.418, - "hfopenllm_v2/MMLU-PRO": 0.485 - } - }, - { - "id": "nothingiisreal/L3.1-8B-Celeste-V1.5", - "name": "L3.1-8B-Celeste-V1.5", - "developer": "nothingiisreal", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7327, - "hfopenllm_v2/BBH": 0.5012, - "hfopenllm_v2/MATH Level 5": 0.1465, - "hfopenllm_v2/GPQA": 0.2844, - "hfopenllm_v2/MUSR": 0.3749, - "hfopenllm_v2/MMLU-PRO": 0.3704 - } - }, - { - "id": "nothingiisreal/MN-12B-Starcannon-v2", - "name": "MN-12B-Starcannon-v2", - "developer": "nothingiisreal", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3925, - "hfopenllm_v2/BBH": 0.5004, - "hfopenllm_v2/MATH Level 5": 0.0597, - "hfopenllm_v2/GPQA": 0.2785, - "hfopenllm_v2/MUSR": 0.3978, - "hfopenllm_v2/MMLU-PRO": 0.3128 - } - }, - { - "id": "nothingiisreal/MN-12B-Starcannon-v3", - "name": "MN-12B-Starcannon-v3", - "developer": "nothingiisreal", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3807, - "hfopenllm_v2/BBH": 0.5171, - "hfopenllm_v2/MATH Level 5": 0.0778, - "hfopenllm_v2/GPQA": 0.2735, - "hfopenllm_v2/MUSR": 0.4046, - "hfopenllm_v2/MMLU-PRO": 0.3265 - } - }, - { - "id": "NousResearch/DeepHermes-3-Mistral-24B-Preview", - "name": "DeepHermes-3-Mistral-24B-Preview", - "developer": "NousResearch", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4536, - "hfopenllm_v2/BBH": 0.6488, - "hfopenllm_v2/MATH Level 5": 0.2576, - "hfopenllm_v2/GPQA": 0.37, - "hfopenllm_v2/MUSR": 0.4503, - "hfopenllm_v2/MMLU-PRO": 0.459 - } - }, - { - "id": "NousResearch/Hermes-2-Pro-Llama-3-8B", - "name": "Hermes-2-Pro-Llama-3-8B", - "developer": "NousResearch", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5362, - "hfopenllm_v2/BBH": 0.5071, - "hfopenllm_v2/MATH Level 5": 0.0838, - "hfopenllm_v2/GPQA": 0.2928, - "hfopenllm_v2/MUSR": 0.4262, - "hfopenllm_v2/MMLU-PRO": 0.3052 - } - }, - { - "id": "NousResearch/Hermes-2-Pro-Mistral-7B", - "name": "Hermes-2-Pro-Mistral-7B", - "developer": "NousResearch", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5668, - "hfopenllm_v2/BBH": 0.4995, - "hfopenllm_v2/MATH Level 5": 0.0604, - "hfopenllm_v2/GPQA": 0.2735, - "hfopenllm_v2/MUSR": 0.4376, - "hfopenllm_v2/MMLU-PRO": 0.2946 - } - }, - { - "id": "NousResearch/Hermes-2-Theta-Llama-3-8B", - "name": "Hermes-2-Theta-Llama-3-8B", - "developer": "NousResearch", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6518, - "hfopenllm_v2/BBH": 0.5207, - "hfopenllm_v2/MATH Level 5": 0.0967, - "hfopenllm_v2/GPQA": 0.3037, - "hfopenllm_v2/MUSR": 0.3949, - "hfopenllm_v2/MMLU-PRO": 0.3369 - } - }, - { - "id": "NousResearch/Hermes-3-Llama-3.1-70B", - "name": "NousResearch/Hermes-3-Llama-3.1-70B", - "developer": "NousResearch", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7661, - "hfopenllm_v2/BBH": 0.6756, - "hfopenllm_v2/MATH Level 5": 0.21, - "hfopenllm_v2/GPQA": 0.3616, - "hfopenllm_v2/MUSR": 0.4949, - "hfopenllm_v2/MMLU-PRO": 0.4727, - "reward-bench/Score": 0.7847, - "reward-bench/Chat": 0.9623, - "reward-bench/Chat Hard": 0.5669, - "reward-bench/Safety": 0.823, - "reward-bench/Reasoning": 0.7867 - } - }, - { - "id": "NousResearch/Hermes-3-Llama-3.1-8B", - "name": "Hermes-3-Llama-3.1-8B", - "developer": "NousResearch", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.617, - "hfopenllm_v2/BBH": 0.5177, - "hfopenllm_v2/MATH Level 5": 0.0476, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.4369, - "hfopenllm_v2/MMLU-PRO": 0.3139 - } - }, - { - "id": "NousResearch/Hermes-3-Llama-3.2-3B", - "name": "Hermes-3-Llama-3.2-3B", - "developer": "NousResearch", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3825, - "hfopenllm_v2/BBH": 0.4352, - "hfopenllm_v2/MATH Level 5": 0.0393, - "hfopenllm_v2/GPQA": 0.2752, - "hfopenllm_v2/MUSR": 0.403, - "hfopenllm_v2/MMLU-PRO": 0.2544 - } - }, - { - "id": "NousResearch/Nous-Hermes-2-Mistral-7B-DPO", - "name": "NousResearch/Nous-Hermes-2-Mistral-7B-DPO", - "developer": "NousResearch", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5763, - "hfopenllm_v2/BBH": 0.4853, - "hfopenllm_v2/MATH Level 5": 0.0476, - "hfopenllm_v2/GPQA": 0.2928, - "hfopenllm_v2/MUSR": 0.4, - "hfopenllm_v2/MMLU-PRO": 0.3015, - "reward-bench/Score": 0.7481, - "reward-bench/Chat": 0.9218, - "reward-bench/Chat Hard": 0.6053, - "reward-bench/Safety": 0.8243, - "reward-bench/Reasoning": 0.7375, - "reward-bench/Prior Sets (0.5 weight)": 0.555 - } - }, - { - "id": "NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO", - "name": "NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO", - "developer": "NousResearch", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5897, - "hfopenllm_v2/BBH": 0.5539, - "hfopenllm_v2/MATH Level 5": 0.1224, - "hfopenllm_v2/GPQA": 0.3213, - "hfopenllm_v2/MUSR": 0.4595, - "hfopenllm_v2/MMLU-PRO": 0.3666, - "reward-bench/Score": 0.7138, - "reward-bench/Chat": 0.9162, - "reward-bench/Chat Hard": 0.6053, - "reward-bench/Safety": 0.8149, - "reward-bench/Reasoning": 0.6126, - "reward-bench/Prior Sets (0.5 weight)": 0.5266 - } - }, - { - "id": "NousResearch/Nous-Hermes-2-Mixtral-8x7B-SFT", - "name": "Nous-Hermes-2-Mixtral-8x7B-SFT", - "developer": "NousResearch", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5731, - "hfopenllm_v2/BBH": 0.5058, - "hfopenllm_v2/MATH Level 5": 0.0211, - "hfopenllm_v2/GPQA": 0.302, - "hfopenllm_v2/MUSR": 0.4214, - "hfopenllm_v2/MMLU-PRO": 0.3066 - } - }, - { - "id": "NousResearch/Nous-Hermes-2-SOLAR-10.7B", - "name": "Nous-Hermes-2-SOLAR-10.7B", - "developer": "NousResearch", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5279, - "hfopenllm_v2/BBH": 0.5414, - "hfopenllm_v2/MATH Level 5": 0.0574, - "hfopenllm_v2/GPQA": 0.2936, - "hfopenllm_v2/MUSR": 0.4373, - "hfopenllm_v2/MMLU-PRO": 0.3458 - } - }, - { - "id": "NousResearch/Nous-Hermes-llama-2-7b", - "name": "Nous-Hermes-llama-2-7b", - "developer": "NousResearch", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1729, - "hfopenllm_v2/BBH": 0.3824, - "hfopenllm_v2/MATH Level 5": 0.0091, - "hfopenllm_v2/GPQA": 0.2634, - "hfopenllm_v2/MUSR": 0.4257, - "hfopenllm_v2/MMLU-PRO": 0.194 - } - }, - { - "id": "NousResearch/Yarn-Llama-2-13b-128k", - "name": "Yarn-Llama-2-13b-128k", - "developer": "NousResearch", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1655, - "hfopenllm_v2/BBH": 0.3827, - "hfopenllm_v2/MATH Level 5": 0.0174, - "hfopenllm_v2/GPQA": 0.2584, - "hfopenllm_v2/MUSR": 0.3458, - "hfopenllm_v2/MMLU-PRO": 0.232 - } - }, - { - "id": "NousResearch/Yarn-Llama-2-7b-128k", - "name": "Yarn-Llama-2-7b-128k", - "developer": "NousResearch", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1485, - "hfopenllm_v2/BBH": 0.3248, - "hfopenllm_v2/MATH Level 5": 0.0151, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3967, - "hfopenllm_v2/MMLU-PRO": 0.1791 - } - }, - { - "id": "NousResearch/Yarn-Llama-2-7b-64k", - "name": "Yarn-Llama-2-7b-64k", - "developer": "NousResearch", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.17, - "hfopenllm_v2/BBH": 0.3326, - "hfopenllm_v2/MATH Level 5": 0.0159, - "hfopenllm_v2/GPQA": 0.2643, - "hfopenllm_v2/MUSR": 0.3939, - "hfopenllm_v2/MMLU-PRO": 0.1799 - } - }, - { - "id": "NousResearch/Yarn-Mistral-7b-128k", - "name": "Yarn-Mistral-7b-128k", - "developer": "NousResearch", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1934, - "hfopenllm_v2/BBH": 0.4314, - "hfopenllm_v2/MATH Level 5": 0.0317, - "hfopenllm_v2/GPQA": 0.2987, - "hfopenllm_v2/MUSR": 0.4071, - "hfopenllm_v2/MMLU-PRO": 0.2893 - } - }, - { - "id": "NousResearch/Yarn-Mistral-7b-64k", - "name": "Yarn-Mistral-7b-64k", - "developer": "NousResearch", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.208, - "hfopenllm_v2/BBH": 0.4293, - "hfopenllm_v2/MATH Level 5": 0.037, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.4124, - "hfopenllm_v2/MMLU-PRO": 0.2914 - } - }, - { - "id": "NousResearch/Yarn-Solar-10b-32k", - "name": "Yarn-Solar-10b-32k", - "developer": "NousResearch", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1942, - "hfopenllm_v2/BBH": 0.4987, - "hfopenllm_v2/MATH Level 5": 0.0302, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.4146, - "hfopenllm_v2/MMLU-PRO": 0.3272 - } - }, - { - "id": "NousResearch/Yarn-Solar-10b-64k", - "name": "Yarn-Solar-10b-64k", - "developer": "NousResearch", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1989, - "hfopenllm_v2/BBH": 0.4922, - "hfopenllm_v2/MATH Level 5": 0.0287, - "hfopenllm_v2/GPQA": 0.302, - "hfopenllm_v2/MUSR": 0.4014, - "hfopenllm_v2/MMLU-PRO": 0.3148 - } - }, - { - "id": "Novaciano/ASTAROTH-3.2-1B", - "name": "ASTAROTH-3.2-1B", - "developer": "Novaciano", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5613, - "hfopenllm_v2/BBH": 0.3543, - "hfopenllm_v2/MATH Level 5": 0.0733, - "hfopenllm_v2/GPQA": 0.2559, - "hfopenllm_v2/MUSR": 0.3142, - "hfopenllm_v2/MMLU-PRO": 0.1909 - } - }, - { - "id": "Novaciano/BLAST_PROCESSING-3.2-1B", - "name": "BLAST_PROCESSING-3.2-1B", - "developer": "Novaciano", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3922, - "hfopenllm_v2/BBH": 0.346, - "hfopenllm_v2/MATH Level 5": 0.0748, - "hfopenllm_v2/GPQA": 0.2659, - "hfopenllm_v2/MUSR": 0.3351, - "hfopenllm_v2/MMLU-PRO": 0.1941 - } - }, - { - "id": "Novaciano/Cerberus-3.2-1B", - "name": "Cerberus-3.2-1B", - "developer": "Novaciano", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5017, - "hfopenllm_v2/BBH": 0.4165, - "hfopenllm_v2/MATH Level 5": 0.0582, - "hfopenllm_v2/GPQA": 0.2584, - "hfopenllm_v2/MUSR": 0.3289, - "hfopenllm_v2/MMLU-PRO": 0.1663 - } - }, - { - "id": "Novaciano/Cultist-3.2-1B", - "name": "Cultist-3.2-1B", - "developer": "Novaciano", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5295, - "hfopenllm_v2/BBH": 0.3399, - "hfopenllm_v2/MATH Level 5": 0.0589, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.333, - "hfopenllm_v2/MMLU-PRO": 0.1714 - } - }, - { - "id": "Novaciano/FuseChat-3.2-1B-GRPO_Creative_RP", - "name": "FuseChat-3.2-1B-GRPO_Creative_RP", - "developer": "Novaciano", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5598, - "hfopenllm_v2/BBH": 0.3488, - "hfopenllm_v2/MATH Level 5": 0.0801, - "hfopenllm_v2/GPQA": 0.2559, - "hfopenllm_v2/MUSR": 0.3329, - "hfopenllm_v2/MMLU-PRO": 0.1735 - } - }, - { - "id": "Novaciano/Fusetrix-3.2-1B-GRPO_RP_Creative", - "name": "Fusetrix-3.2-1B-GRPO_RP_Creative", - "developer": "Novaciano", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5366, - "hfopenllm_v2/BBH": 0.3435, - "hfopenllm_v2/MATH Level 5": 0.1148, - "hfopenllm_v2/GPQA": 0.25, - "hfopenllm_v2/MUSR": 0.3209, - "hfopenllm_v2/MMLU-PRO": 0.1758 - } - }, - { - "id": "Novaciano/Fusetrix-Dolphin-3.2-1B-GRPO_Creative_RP", - "name": "Fusetrix-Dolphin-3.2-1B-GRPO_Creative_RP", - "developer": "Novaciano", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5343, - "hfopenllm_v2/BBH": 0.3502, - "hfopenllm_v2/MATH Level 5": 0.105, - "hfopenllm_v2/GPQA": 0.2685, - "hfopenllm_v2/MUSR": 0.3183, - "hfopenllm_v2/MMLU-PRO": 0.1823 - } - }, - { - "id": "Novaciano/HarmfulProject-3.2-1B", - "name": "HarmfulProject-3.2-1B", - "developer": "Novaciano", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3874, - "hfopenllm_v2/BBH": 0.3274, - "hfopenllm_v2/MATH Level 5": 0.0476, - "hfopenllm_v2/GPQA": 0.2668, - "hfopenllm_v2/MUSR": 0.3419, - "hfopenllm_v2/MMLU-PRO": 0.1823 - } - }, - { - "id": "Novaciano/La_Mejor_Mezcla-3.2-1B", - "name": "La_Mejor_Mezcla-3.2-1B", - "developer": "Novaciano", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.551, - "hfopenllm_v2/BBH": 0.3488, - "hfopenllm_v2/MATH Level 5": 0.0899, - "hfopenllm_v2/GPQA": 0.2576, - "hfopenllm_v2/MUSR": 0.3196, - "hfopenllm_v2/MMLU-PRO": 0.1829 - } - }, - { - "id": "Novaciano/LEWD-Mental-Cultist-3.2-1B", - "name": "LEWD-Mental-Cultist-3.2-1B", - "developer": "Novaciano", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5309, - "hfopenllm_v2/BBH": 0.3513, - "hfopenllm_v2/MATH Level 5": 0.0529, - "hfopenllm_v2/GPQA": 0.2567, - "hfopenllm_v2/MUSR": 0.3223, - "hfopenllm_v2/MMLU-PRO": 0.1769 - } - }, - { - "id": "Novaciano/Sigil-Of-Satan-3.2-1B", - "name": "Sigil-Of-Satan-3.2-1B", - "developer": "Novaciano", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5494, - "hfopenllm_v2/BBH": 0.3546, - "hfopenllm_v2/MATH Level 5": 0.0544, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.3276, - "hfopenllm_v2/MMLU-PRO": 0.1855 - } - }, - { - "id": "NTQAI/Nxcode-CQ-7B-orpo", - "name": "Nxcode-CQ-7B-orpo", - "developer": "NTQAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4007, - "hfopenllm_v2/BBH": 0.4143, - "hfopenllm_v2/MATH Level 5": 0.0219, - "hfopenllm_v2/GPQA": 0.2542, - "hfopenllm_v2/MUSR": 0.394, - "hfopenllm_v2/MMLU-PRO": 0.1612 - } - }, - { - "id": "NTQAI/NxMobileLM-1.5B-SFT", - "name": "NxMobileLM-1.5B-SFT", - "developer": "NTQAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6392, - "hfopenllm_v2/BBH": 0.3957, - "hfopenllm_v2/MATH Level 5": 0.0846, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3555, - "hfopenllm_v2/MMLU-PRO": 0.2817 - } - }, - { - "id": "NucleusAI/nucleus-22B-token-500B", - "name": "nucleus-22B-token-500B", - "developer": "NucleusAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0257, - "hfopenllm_v2/BBH": 0.292, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.25, - "hfopenllm_v2/MUSR": 0.3511, - "hfopenllm_v2/MMLU-PRO": 0.1162 - } - }, - { - "id": "nvidia/AceInstruct-1.5B", - "name": "AceInstruct-1.5B", - "developer": "nvidia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3948, - "hfopenllm_v2/BBH": 0.3932, - "hfopenllm_v2/MATH Level 5": 0.3127, - "hfopenllm_v2/GPQA": 0.2718, - "hfopenllm_v2/MUSR": 0.346, - "hfopenllm_v2/MMLU-PRO": 0.2574 - } - }, - { - "id": "nvidia/AceInstruct-72B", - "name": "AceInstruct-72B", - "developer": "nvidia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7119, - "hfopenllm_v2/BBH": 0.6139, - "hfopenllm_v2/MATH Level 5": 0.6261, - "hfopenllm_v2/GPQA": 0.3213, - "hfopenllm_v2/MUSR": 0.4206, - "hfopenllm_v2/MMLU-PRO": 0.4874 - } - }, - { - "id": "nvidia/AceInstruct-7B", - "name": "AceInstruct-7B", - "developer": "nvidia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5422, - "hfopenllm_v2/BBH": 0.5501, - "hfopenllm_v2/MATH Level 5": 0.5295, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.4255, - "hfopenllm_v2/MMLU-PRO": 0.4177 - } - }, - { - "id": "nvidia/AceMath-1.5B-Instruct", - "name": "AceMath-1.5B-Instruct", - "developer": "nvidia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3212, - "hfopenllm_v2/BBH": 0.4024, - "hfopenllm_v2/MATH Level 5": 0.5287, - "hfopenllm_v2/GPQA": 0.2743, - "hfopenllm_v2/MUSR": 0.3607, - "hfopenllm_v2/MMLU-PRO": 0.2064 - } - }, - { - "id": "nvidia/AceMath-72B-Instruct", - "name": "AceMath-72B-Instruct", - "developer": "nvidia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.495, - "hfopenllm_v2/BBH": 0.6402, - "hfopenllm_v2/MATH Level 5": 0.7145, - "hfopenllm_v2/GPQA": 0.271, - "hfopenllm_v2/MUSR": 0.4062, - "hfopenllm_v2/MMLU-PRO": 0.4411 - } - }, - { - "id": "nvidia/AceMath-72B-RM", - "name": "AceMath-72B-RM", - "developer": "nvidia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1413, - "hfopenllm_v2/BBH": 0.2717, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2341, - "hfopenllm_v2/MUSR": 0.3351, - "hfopenllm_v2/MMLU-PRO": 0.1179 - } - }, - { - "id": "nvidia/AceMath-7B-Instruct", - "name": "AceMath-7B-Instruct", - "developer": "nvidia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4532, - "hfopenllm_v2/BBH": 0.4994, - "hfopenllm_v2/MATH Level 5": 0.6337, - "hfopenllm_v2/GPQA": 0.2919, - "hfopenllm_v2/MUSR": 0.4193, - "hfopenllm_v2/MMLU-PRO": 0.3383 - } - }, - { - "id": "nvidia/AceMath-7B-RM", - "name": "AceMath-7B-RM", - "developer": "nvidia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1494, - "hfopenllm_v2/BBH": 0.2423, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2458, - "hfopenllm_v2/MUSR": 0.358, - "hfopenllm_v2/MMLU-PRO": 0.1139 - } - }, - { - "id": "nvidia/Hymba-1.5B-Base", - "name": "Hymba-1.5B-Base", - "developer": "nvidia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2295, - "hfopenllm_v2/BBH": 0.3256, - "hfopenllm_v2/MATH Level 5": 0.0136, - "hfopenllm_v2/GPQA": 0.2559, - "hfopenllm_v2/MUSR": 0.3566, - "hfopenllm_v2/MMLU-PRO": 0.1922 - } - }, - { - "id": "nvidia/Hymba-1.5B-Instruct", - "name": "Hymba-1.5B-Instruct", - "developer": "nvidia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6009, - "hfopenllm_v2/BBH": 0.3067, - "hfopenllm_v2/MATH Level 5": 0.0272, - "hfopenllm_v2/GPQA": 0.2886, - "hfopenllm_v2/MUSR": 0.3316, - "hfopenllm_v2/MMLU-PRO": 0.204 - } - }, - { - "id": "nvidia/llama-3-1-nemotron-ultra-253b-v1-fc", - "name": "Llama-3.1-Nemotron-Ultra-253B-v1 (FC)", - "developer": "nvidia", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 108.0, - "bfcl/bfcl.overall.overall_accuracy": 10.0, - "bfcl/bfcl.overall.total_cost_usd": 0.72, - "bfcl/bfcl.overall.latency_mean_s": 1.42, - "bfcl/bfcl.overall.latency_std_s": 1.84, - "bfcl/bfcl.overall.latency_p95_s": 2.4, - "bfcl/bfcl.non_live.ast_accuracy": 0.0, - "bfcl/bfcl.non_live.simple_ast_accuracy": 0.0, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 0.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 0.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 0.0, - "bfcl/bfcl.live.live_accuracy": 0.0, - "bfcl/bfcl.live.live_simple_ast_accuracy": 0.0, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 0.0, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 0.0, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 0.0, - "bfcl/bfcl.multi_turn.accuracy": 0.0, - "bfcl/bfcl.multi_turn.base_accuracy": 0.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 0.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 0.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 0.0, - "bfcl/bfcl.web_search.accuracy": 0.0, - "bfcl/bfcl.web_search.base_accuracy": 0.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 0.0, - "bfcl/bfcl.memory.accuracy": 0.0, - "bfcl/bfcl.memory.kv_accuracy": 0.0, - "bfcl/bfcl.memory.vector_accuracy": 0.0, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 0.0, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 0.0, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 100.0 - } - }, - { - "id": "nvidia/Llama-3.1-Minitron-4B-Depth-Base", - "name": "Llama-3.1-Minitron-4B-Depth-Base", - "developer": "nvidia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1607, - "hfopenllm_v2/BBH": 0.4171, - "hfopenllm_v2/MATH Level 5": 0.0196, - "hfopenllm_v2/GPQA": 0.2634, - "hfopenllm_v2/MUSR": 0.4011, - "hfopenllm_v2/MMLU-PRO": 0.2798 - } - }, - { - "id": "nvidia/Llama-3.1-Nemotron-70B-Instruct-HF", - "name": "Llama-3.1-Nemotron-70B-Instruct-HF", - "developer": "nvidia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7381, - "hfopenllm_v2/BBH": 0.6316, - "hfopenllm_v2/MATH Level 5": 0.4267, - "hfopenllm_v2/GPQA": 0.2584, - "hfopenllm_v2/MUSR": 0.4328, - "hfopenllm_v2/MMLU-PRO": 0.4919 - } - }, - { - "id": "nvidia/Llama-3.1-Nemotron-70B-Reward", - "name": "nvidia/Llama-3.1-Nemotron-70B-Reward", - "developer": "nvidia", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.9411, - "reward-bench/Chat": 0.9749, - "reward-bench/Chat Hard": 0.8575, - "reward-bench/Safety": 0.9514, - "reward-bench/Reasoning": 0.9807 - } - }, - { - "id": "nvidia/Llama3-70B-SteerLM-RM", - "name": "nvidia/Llama3-70B-SteerLM-RM", - "developer": "nvidia", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.8877, - "reward-bench/Chat": 0.9134, - "reward-bench/Chat Hard": 0.8026, - "reward-bench/Safety": 0.9284, - "reward-bench/Reasoning": 0.9064 - } - }, - { - "id": "nvidia/Minitron-4B-Base", - "name": "Minitron-4B-Base", - "developer": "nvidia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2218, - "hfopenllm_v2/BBH": 0.4084, - "hfopenllm_v2/MATH Level 5": 0.0196, - "hfopenllm_v2/GPQA": 0.2693, - "hfopenllm_v2/MUSR": 0.4134, - "hfopenllm_v2/MMLU-PRO": 0.262 - } - }, - { - "id": "nvidia/Minitron-8B-Base", - "name": "Minitron-8B-Base", - "developer": "nvidia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2424, - "hfopenllm_v2/BBH": 0.4395, - "hfopenllm_v2/MATH Level 5": 0.0257, - "hfopenllm_v2/GPQA": 0.2735, - "hfopenllm_v2/MUSR": 0.4026, - "hfopenllm_v2/MMLU-PRO": 0.3181 - } - }, - { - "id": "nvidia/Mistral-NeMo-Minitron-8B-Base", - "name": "Mistral-NeMo-Minitron-8B-Base", - "developer": "nvidia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1946, - "hfopenllm_v2/BBH": 0.5219, - "hfopenllm_v2/MATH Level 5": 0.0483, - "hfopenllm_v2/GPQA": 0.3255, - "hfopenllm_v2/MUSR": 0.4092, - "hfopenllm_v2/MMLU-PRO": 0.3796 - } - }, - { - "id": "nvidia/Mistral-NeMo-Minitron-8B-Instruct", - "name": "Mistral-NeMo-Minitron-8B-Instruct", - "developer": "nvidia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5004, - "hfopenllm_v2/BBH": 0.5321, - "hfopenllm_v2/MATH Level 5": 0.1163, - "hfopenllm_v2/GPQA": 0.2878, - "hfopenllm_v2/MUSR": 0.3886, - "hfopenllm_v2/MMLU-PRO": 0.3991 - } - }, - { - "id": "nvidia/Nemotron-4-340B-Reward", - "name": "nvidia/Nemotron-4-340B-Reward", - "developer": "nvidia", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.92, - "reward-bench/Chat": 0.9581, - "reward-bench/Chat Hard": 0.8706, - "reward-bench/Safety": 0.9149, - "reward-bench/Reasoning": 0.9363 - } - }, - { - "id": "nvidia/Nemotron-Mini-4B-Instruct", - "name": "Nemotron-Mini-4B-Instruct", - "developer": "nvidia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6669, - "hfopenllm_v2/BBH": 0.3865, - "hfopenllm_v2/MATH Level 5": 0.0257, - "hfopenllm_v2/GPQA": 0.2802, - "hfopenllm_v2/MUSR": 0.3767, - "hfopenllm_v2/MMLU-PRO": 0.2626 - } - }, - { - "id": "nvidia/OpenMath2-Llama3.1-8B", - "name": "OpenMath2-Llama3.1-8B", - "developer": "nvidia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2331, - "hfopenllm_v2/BBH": 0.4096, - "hfopenllm_v2/MATH Level 5": 0.2674, - "hfopenllm_v2/GPQA": 0.2651, - "hfopenllm_v2/MUSR": 0.3436, - "hfopenllm_v2/MMLU-PRO": 0.1553 - } - }, - { - "id": "nxmwxm/Beast-Soul-new", - "name": "Beast-Soul-new", - "developer": "nxmwxm", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4869, - "hfopenllm_v2/BBH": 0.5227, - "hfopenllm_v2/MATH Level 5": 0.074, - "hfopenllm_v2/GPQA": 0.2819, - "hfopenllm_v2/MUSR": 0.4459, - "hfopenllm_v2/MMLU-PRO": 0.3102 - } - }, - { - "id": "NYTK/PULI-GPTrio", - "name": "PULI-GPTrio", - "developer": "NYTK", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.218, - "hfopenllm_v2/BBH": 0.306, - "hfopenllm_v2/MATH Level 5": 0.0121, - "hfopenllm_v2/GPQA": 0.2659, - "hfopenllm_v2/MUSR": 0.3819, - "hfopenllm_v2/MMLU-PRO": 0.1137 - } - }, - { - "id": "NYTK/PULI-LlumiX-32K", - "name": "PULI-LlumiX-32K", - "developer": "NYTK", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.17, - "hfopenllm_v2/BBH": 0.3189, - "hfopenllm_v2/MATH Level 5": 0.0128, - "hfopenllm_v2/GPQA": 0.2534, - "hfopenllm_v2/MUSR": 0.3964, - "hfopenllm_v2/MMLU-PRO": 0.1681 - } - }, - { - "id": "NyxKrage/Microsoft_Phi-4", - "name": "Microsoft_Phi-4", - "developer": "NyxKrage", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0585, - "hfopenllm_v2/BBH": 0.6691, - "hfopenllm_v2/MATH Level 5": 0.2991, - "hfopenllm_v2/GPQA": 0.406, - "hfopenllm_v2/MUSR": 0.5034, - "hfopenllm_v2/MMLU-PRO": 0.5287 - } - }, - { - "id": "occiglot/occiglot-7b-es-en-instruct", - "name": "occiglot-7b-es-en-instruct", - "developer": "occiglot", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3485, - "hfopenllm_v2/BBH": 0.4111, - "hfopenllm_v2/MATH Level 5": 0.0242, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3738, - "hfopenllm_v2/MMLU-PRO": 0.2311 - } - }, - { - "id": "odyssey-labs/Astral-1-10B", - "name": "Astral-1-10B", - "developer": "odyssey-labs", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3878, - "hfopenllm_v2/BBH": 0.4873, - "hfopenllm_v2/MATH Level 5": 0.0347, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.428, - "hfopenllm_v2/MMLU-PRO": 0.2985 - } - }, - { - "id": "OEvortex/Emotional-llama-8B", - "name": "Emotional-llama-8B", - "developer": "OEvortex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3516, - "hfopenllm_v2/BBH": 0.4839, - "hfopenllm_v2/MATH Level 5": 0.0816, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.3659, - "hfopenllm_v2/MMLU-PRO": 0.3535 - } - }, - { - "id": "OEvortex/HelpingAI-15B", - "name": "HelpingAI-15B", - "developer": "OEvortex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.203, - "hfopenllm_v2/BBH": 0.2936, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2576, - "hfopenllm_v2/MUSR": 0.3619, - "hfopenllm_v2/MMLU-PRO": 0.1111 - } - }, - { - "id": "OEvortex/HelpingAI-3B-reloaded", - "name": "HelpingAI-3B-reloaded", - "developer": "OEvortex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4647, - "hfopenllm_v2/BBH": 0.4129, - "hfopenllm_v2/MATH Level 5": 0.0136, - "hfopenllm_v2/GPQA": 0.2634, - "hfopenllm_v2/MUSR": 0.3524, - "hfopenllm_v2/MMLU-PRO": 0.2595 - } - }, - { - "id": "OEvortex/HelpingAI2-9B", - "name": "HelpingAI2-9B", - "developer": "OEvortex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4413, - "hfopenllm_v2/BBH": 0.4845, - "hfopenllm_v2/MATH Level 5": 0.0589, - "hfopenllm_v2/GPQA": 0.2584, - "hfopenllm_v2/MUSR": 0.3711, - "hfopenllm_v2/MMLU-PRO": 0.29 - } - }, - { - "id": "OEvortex/HelpingAI2.5-10B", - "name": "HelpingAI2.5-10B", - "developer": "OEvortex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3277, - "hfopenllm_v2/BBH": 0.4496, - "hfopenllm_v2/MATH Level 5": 0.0204, - "hfopenllm_v2/GPQA": 0.2693, - "hfopenllm_v2/MUSR": 0.3738, - "hfopenllm_v2/MMLU-PRO": 0.2575 - } - }, - { - "id": "olabs-ai/reflection_model", - "name": "reflection_model", - "developer": "olabs-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1599, - "hfopenllm_v2/BBH": 0.4713, - "hfopenllm_v2/MATH Level 5": 0.0514, - "hfopenllm_v2/GPQA": 0.3003, - "hfopenllm_v2/MUSR": 0.3508, - "hfopenllm_v2/MMLU-PRO": 0.3311 - } - }, - { - "id": "OliveiraJLT/Sagui-7B-Instruct-v0.1", - "name": "Sagui-7B-Instruct-v0.1", - "developer": "OliveiraJLT", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2892, - "hfopenllm_v2/BBH": 0.3111, - "hfopenllm_v2/MATH Level 5": 0.0151, - "hfopenllm_v2/GPQA": 0.2424, - "hfopenllm_v2/MUSR": 0.4191, - "hfopenllm_v2/MMLU-PRO": 0.1485 - } - }, - { - "id": "Omkar1102/code-yi", - "name": "code-yi", - "developer": "Omkar1102", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2254, - "hfopenllm_v2/BBH": 0.275, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2576, - "hfopenllm_v2/MUSR": 0.3762, - "hfopenllm_v2/MMLU-PRO": 0.1123 - } - }, - { - "id": "OmnicromsBrain/NeuralStar_FusionWriter_4x7b", - "name": "NeuralStar_FusionWriter_4x7b", - "developer": "OmnicromsBrain", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5964, - "hfopenllm_v2/BBH": 0.4776, - "hfopenllm_v2/MATH Level 5": 0.0491, - "hfopenllm_v2/GPQA": 0.2785, - "hfopenllm_v2/MUSR": 0.4019, - "hfopenllm_v2/MMLU-PRO": 0.2606 - } - }, - { - "id": "OnlyCheeini/greesychat-turbo", - "name": "greesychat-turbo", - "developer": "OnlyCheeini", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0233, - "hfopenllm_v2/BBH": 0.3092, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3314, - "hfopenllm_v2/MMLU-PRO": 0.1138 - } - }, - { - "id": "ontocord/Llama_3.2_1b-autoredteam_helpfulness-train", - "name": "Llama_3.2_1b-autoredteam_helpfulness-train", - "developer": "ontocord", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2765, - "hfopenllm_v2/BBH": 0.3115, - "hfopenllm_v2/MATH Level 5": 0.0166, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3459, - "hfopenllm_v2/MMLU-PRO": 0.1132 - } - }, - { - "id": "ontocord/merged_0.2_expert_0.8", - "name": "merged_0.2_expert_0.8", - "developer": "ontocord", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1743, - "hfopenllm_v2/BBH": 0.3046, - "hfopenllm_v2/MATH Level 5": 0.0264, - "hfopenllm_v2/GPQA": 0.2617, - "hfopenllm_v2/MUSR": 0.3621, - "hfopenllm_v2/MMLU-PRO": 0.1111 - } - }, - { - "id": "ontocord/merged_0.2_expert_0.8-stack_2x", - "name": "merged_0.2_expert_0.8-stack_2x", - "developer": "ontocord", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1796, - "hfopenllm_v2/BBH": 0.3006, - "hfopenllm_v2/MATH Level 5": 0.0249, - "hfopenllm_v2/GPQA": 0.2626, - "hfopenllm_v2/MUSR": 0.3541, - "hfopenllm_v2/MMLU-PRO": 0.1103 - } - }, - { - "id": "ontocord/merged_0.5_expert_0.5", - "name": "merged_0.5_expert_0.5", - "developer": "ontocord", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1787, - "hfopenllm_v2/BBH": 0.3017, - "hfopenllm_v2/MATH Level 5": 0.0196, - "hfopenllm_v2/GPQA": 0.2643, - "hfopenllm_v2/MUSR": 0.3542, - "hfopenllm_v2/MMLU-PRO": 0.1108 - } - }, - { - "id": "ontocord/ontocord_wide_3b-stage1_shuf_sample1_jsonl-pretrained-autoredteam_helpful-0.25_helpful", - "name": "ontocord_wide_3b-stage1_shuf_sample1_jsonl-pretrained-autoredteam_helpful-0.25_helpful", - "developer": "ontocord", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1318, - "hfopenllm_v2/BBH": 0.3004, - "hfopenllm_v2/MATH Level 5": 0.0106, - "hfopenllm_v2/GPQA": 0.2676, - "hfopenllm_v2/MUSR": 0.3631, - "hfopenllm_v2/MMLU-PRO": 0.1142 - } - }, - { - "id": "ontocord/ontocord_wide_7b-stacked-stage1", - "name": "ontocord_wide_7b-stacked-stage1", - "developer": "ontocord", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1485, - "hfopenllm_v2/BBH": 0.2897, - "hfopenllm_v2/MATH Level 5": 0.0091, - "hfopenllm_v2/GPQA": 0.2534, - "hfopenllm_v2/MUSR": 0.3604, - "hfopenllm_v2/MMLU-PRO": 0.1105 - } - }, - { - "id": "ontocord/ontocord_wide_7b-stacked-stage1-instruct", - "name": "ontocord_wide_7b-stacked-stage1-instruct", - "developer": "ontocord", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.153, - "hfopenllm_v2/BBH": 0.2854, - "hfopenllm_v2/MATH Level 5": 0.0068, - "hfopenllm_v2/GPQA": 0.2466, - "hfopenllm_v2/MUSR": 0.3538, - "hfopenllm_v2/MMLU-PRO": 0.1117 - } - }, - { - "id": "ontocord/RedPajama-3B-v1-AutoRedteam", - "name": "RedPajama-3B-v1-AutoRedteam", - "developer": "ontocord", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1343, - "hfopenllm_v2/BBH": 0.3026, - "hfopenllm_v2/MATH Level 5": 0.0091, - "hfopenllm_v2/GPQA": 0.2424, - "hfopenllm_v2/MUSR": 0.3661, - "hfopenllm_v2/MMLU-PRO": 0.1108 - } - }, - { - "id": "ontocord/RedPajama-3B-v1-AutoRedteam-Harmless-only", - "name": "RedPajama-3B-v1-AutoRedteam-Harmless-only", - "developer": "ontocord", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1525, - "hfopenllm_v2/BBH": 0.3124, - "hfopenllm_v2/MATH Level 5": 0.006, - "hfopenllm_v2/GPQA": 0.2315, - "hfopenllm_v2/MUSR": 0.3661, - "hfopenllm_v2/MMLU-PRO": 0.11 - } - }, - { - "id": "ontocord/RedPajama3b_v1-autoredteam_helpfulness-train", - "name": "RedPajama3b_v1-autoredteam_helpfulness-train", - "developer": "ontocord", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2848, - "hfopenllm_v2/BBH": 0.3093, - "hfopenllm_v2/MATH Level 5": 0.0068, - "hfopenllm_v2/GPQA": 0.2458, - "hfopenllm_v2/MUSR": 0.358, - "hfopenllm_v2/MMLU-PRO": 0.1107 - } - }, - { - "id": "ontocord/starcoder2-29b-ls", - "name": "starcoder2-29b-ls", - "developer": "ontocord", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2149, - "hfopenllm_v2/BBH": 0.3735, - "hfopenllm_v2/MATH Level 5": 0.0189, - "hfopenllm_v2/GPQA": 0.2735, - "hfopenllm_v2/MUSR": 0.37, - "hfopenllm_v2/MMLU-PRO": 0.1869 - } - }, - { - "id": "ontocord/starcoder2_3b-AutoRedteam", - "name": "starcoder2_3b-AutoRedteam", - "developer": "ontocord", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1574, - "hfopenllm_v2/BBH": 0.3498, - "hfopenllm_v2/MATH Level 5": 0.0106, - "hfopenllm_v2/GPQA": 0.2517, - "hfopenllm_v2/MUSR": 0.3646, - "hfopenllm_v2/MMLU-PRO": 0.1336 - } - }, - { - "id": "ontocord/wide_3b-merge_test", - "name": "wide_3b-merge_test", - "developer": "ontocord", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1763, - "hfopenllm_v2/BBH": 0.3011, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2399, - "hfopenllm_v2/MUSR": 0.342, - "hfopenllm_v2/MMLU-PRO": 0.1066 - } - }, - { - "id": "ontocord/wide_3b-stage1_shuf_sample1_jsonl-pretrained", - "name": "wide_3b-stage1_shuf_sample1_jsonl-pretrained", - "developer": "ontocord", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1395, - "hfopenllm_v2/BBH": 0.3004, - "hfopenllm_v2/MATH Level 5": 0.0166, - "hfopenllm_v2/GPQA": 0.2659, - "hfopenllm_v2/MUSR": 0.3632, - "hfopenllm_v2/MMLU-PRO": 0.114 - } - }, - { - "id": "ontocord/wide_3b_sft_stag1.2-lyrical_law_news_software_howto_formattedtext_math_wiki-merge", - "name": "wide_3b_sft_stag1.2-lyrical_law_news_software_howto_formattedtext_math_wiki-merge", - "developer": "ontocord", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1664, - "hfopenllm_v2/BBH": 0.3031, - "hfopenllm_v2/MATH Level 5": 0.0113, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3845, - "hfopenllm_v2/MMLU-PRO": 0.1111 - } - }, - { - "id": "ontocord/wide_3b_sft_stag1.2-lyrical_news_software_howto_formattedtext-merge", - "name": "wide_3b_sft_stag1.2-lyrical_news_software_howto_formattedtext-merge", - "developer": "ontocord", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1697, - "hfopenllm_v2/BBH": 0.2975, - "hfopenllm_v2/MATH Level 5": 0.0136, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3778, - "hfopenllm_v2/MMLU-PRO": 0.1125 - } - }, - { - "id": "ontocord/wide_3b_sft_stage1.1-ss1-no_redteam_skg_poem.no_issue", - "name": "wide_3b_sft_stage1.1-ss1-no_redteam_skg_poem.no_issue", - "developer": "ontocord", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.148, - "hfopenllm_v2/BBH": 0.3095, - "hfopenllm_v2/MATH Level 5": 0.0204, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.3579, - "hfopenllm_v2/MMLU-PRO": 0.1108 - } - }, - { - "id": "ontocord/wide_3b_sft_stage1.1-ss1-with_generics_intr.no_issue", - "name": "wide_3b_sft_stage1.1-ss1-with_generics_intr.no_issue", - "developer": "ontocord", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1237, - "hfopenllm_v2/BBH": 0.306, - "hfopenllm_v2/MATH Level 5": 0.0106, - "hfopenllm_v2/GPQA": 0.2743, - "hfopenllm_v2/MUSR": 0.3673, - "hfopenllm_v2/MMLU-PRO": 0.1111 - } - }, - { - "id": "ontocord/wide_3b_sft_stage1.1-ss1-with_generics_intr_math.no_issue", - "name": "wide_3b_sft_stage1.1-ss1-with_generics_intr_math.no_issue", - "developer": "ontocord", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1192, - "hfopenllm_v2/BBH": 0.2956, - "hfopenllm_v2/MATH Level 5": 0.0068, - "hfopenllm_v2/GPQA": 0.2643, - "hfopenllm_v2/MUSR": 0.3553, - "hfopenllm_v2/MMLU-PRO": 0.1183 - } - }, - { - "id": "ontocord/wide_3b_sft_stage1.1-ss1-with_generics_intr_math_stories.no_issue", - "name": "wide_3b_sft_stage1.1-ss1-with_generics_intr_math_stories.no_issue", - "developer": "ontocord", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1128, - "hfopenllm_v2/BBH": 0.3171, - "hfopenllm_v2/MATH Level 5": 0.0113, - "hfopenllm_v2/GPQA": 0.2685, - "hfopenllm_v2/MUSR": 0.346, - "hfopenllm_v2/MMLU-PRO": 0.1129 - } - }, - { - "id": "ontocord/wide_3b_sft_stage1.1-ss1-with_generics_intr_math_stories_no_orig_instr.no_issue", - "name": "wide_3b_sft_stage1.1-ss1-with_generics_intr_math_stories_no_orig_instr.no_issue", - "developer": "ontocord", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1317, - "hfopenllm_v2/BBH": 0.3064, - "hfopenllm_v2/MATH Level 5": 0.0091, - "hfopenllm_v2/GPQA": 0.2651, - "hfopenllm_v2/MUSR": 0.3446, - "hfopenllm_v2/MMLU-PRO": 0.1144 - } - }, - { - "id": "ontocord/wide_3b_sft_stage1.1-ss1-with_generics_intr_stories.no_issue", - "name": "wide_3b_sft_stage1.1-ss1-with_generics_intr_stories.no_issue", - "developer": "ontocord", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1182, - "hfopenllm_v2/BBH": 0.3037, - "hfopenllm_v2/MATH Level 5": 0.0083, - "hfopenllm_v2/GPQA": 0.2659, - "hfopenllm_v2/MUSR": 0.3567, - "hfopenllm_v2/MMLU-PRO": 0.1162 - } - }, - { - "id": "ontocord/wide_3b_sft_stage1.1-ss1-with_generics_math.no_issue", - "name": "wide_3b_sft_stage1.1-ss1-with_generics_math.no_issue", - "developer": "ontocord", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.124, - "hfopenllm_v2/BBH": 0.3032, - "hfopenllm_v2/MATH Level 5": 0.0076, - "hfopenllm_v2/GPQA": 0.2584, - "hfopenllm_v2/MUSR": 0.3487, - "hfopenllm_v2/MMLU-PRO": 0.1128 - } - }, - { - "id": "ontocord/wide_3b_sft_stage1.1-ss1-with_math.no_issue", - "name": "wide_3b_sft_stage1.1-ss1-with_math.no_issue", - "developer": "ontocord", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1298, - "hfopenllm_v2/BBH": 0.3052, - "hfopenllm_v2/MATH Level 5": 0.0159, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3928, - "hfopenllm_v2/MMLU-PRO": 0.1147 - } - }, - { - "id": "ontocord/wide_3b_sft_stage1.1-ss1-with_r1_generics_intr_math_stories.no_issue", - "name": "wide_3b_sft_stage1.1-ss1-with_r1_generics_intr_math_stories.no_issue", - "developer": "ontocord", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2049, - "hfopenllm_v2/BBH": 0.2912, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3575, - "hfopenllm_v2/MMLU-PRO": 0.1167 - } - }, - { - "id": "ontocord/wide_3b_sft_stage1.2-ss1-expert_fictional_lyrical", - "name": "wide_3b_sft_stage1.2-ss1-expert_fictional_lyrical", - "developer": "ontocord", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1461, - "hfopenllm_v2/BBH": 0.2998, - "hfopenllm_v2/MATH Level 5": 0.0136, - "hfopenllm_v2/GPQA": 0.2643, - "hfopenllm_v2/MUSR": 0.3926, - "hfopenllm_v2/MMLU-PRO": 0.1141 - } - }, - { - "id": "ontocord/wide_3b_sft_stage1.2-ss1-expert_formatted_text", - "name": "wide_3b_sft_stage1.2-ss1-expert_formatted_text", - "developer": "ontocord", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1487, - "hfopenllm_v2/BBH": 0.3069, - "hfopenllm_v2/MATH Level 5": 0.0121, - "hfopenllm_v2/GPQA": 0.2617, - "hfopenllm_v2/MUSR": 0.3474, - "hfopenllm_v2/MMLU-PRO": 0.1146 - } - }, - { - "id": "ontocord/wide_3b_sft_stage1.2-ss1-expert_how-to", - "name": "wide_3b_sft_stage1.2-ss1-expert_how-to", - "developer": "ontocord", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1245, - "hfopenllm_v2/BBH": 0.3047, - "hfopenllm_v2/MATH Level 5": 0.0144, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3658, - "hfopenllm_v2/MMLU-PRO": 0.1153 - } - }, - { - "id": "ontocord/wide_3b_sft_stage1.2-ss1-expert_math", - "name": "wide_3b_sft_stage1.2-ss1-expert_math", - "developer": "ontocord", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1915, - "hfopenllm_v2/BBH": 0.306, - "hfopenllm_v2/MATH Level 5": 0.0279, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.37, - "hfopenllm_v2/MMLU-PRO": 0.1092 - } - }, - { - "id": "ontocord/wide_3b_sft_stage1.2-ss1-expert_news", - "name": "wide_3b_sft_stage1.2-ss1-expert_news", - "developer": "ontocord", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1658, - "hfopenllm_v2/BBH": 0.2926, - "hfopenllm_v2/MATH Level 5": 0.0166, - "hfopenllm_v2/GPQA": 0.2676, - "hfopenllm_v2/MUSR": 0.3621, - "hfopenllm_v2/MMLU-PRO": 0.1111 - } - }, - { - "id": "ontocord/wide_3b_sft_stage1.2-ss1-expert_software", - "name": "wide_3b_sft_stage1.2-ss1-expert_software", - "developer": "ontocord", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1734, - "hfopenllm_v2/BBH": 0.298, - "hfopenllm_v2/MATH Level 5": 0.0159, - "hfopenllm_v2/GPQA": 0.2584, - "hfopenllm_v2/MUSR": 0.3569, - "hfopenllm_v2/MMLU-PRO": 0.114 - } - }, - { - "id": "ontocord/wide_6.6b_sft_stag1.2-lyrical_law_news_software_howto_formattedtext_math_wiki-merge-stacked", - "name": "wide_6.6b_sft_stag1.2-lyrical_law_news_software_howto_formattedtext_math_wiki-merge-stacked", - "developer": "ontocord", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1244, - "hfopenllm_v2/BBH": 0.3026, - "hfopenllm_v2/MATH Level 5": 0.0144, - "hfopenllm_v2/GPQA": 0.2659, - "hfopenllm_v2/MUSR": 0.3686, - "hfopenllm_v2/MMLU-PRO": 0.1115 - } - }, - { - "id": "oobabooga/CodeBooga-34B-v0.1", - "name": "CodeBooga-34B-v0.1", - "developer": "oobabooga", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.525, - "hfopenllm_v2/BBH": 0.3427, - "hfopenllm_v2/MATH Level 5": 0.0393, - "hfopenllm_v2/GPQA": 0.2567, - "hfopenllm_v2/MUSR": 0.431, - "hfopenllm_v2/MMLU-PRO": 0.236 - } - }, - { - "id": "oopere/Llama-FinSent-S", - "name": "Llama-FinSent-S", - "developer": "oopere", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2164, - "hfopenllm_v2/BBH": 0.3169, - "hfopenllm_v2/MATH Level 5": 0.0128, - "hfopenllm_v2/GPQA": 0.2584, - "hfopenllm_v2/MUSR": 0.3832, - "hfopenllm_v2/MMLU-PRO": 0.1134 - } - }, - { - "id": "oopere/pruned10-llama-3.2-3B", - "name": "pruned10-llama-3.2-3B", - "developer": "oopere", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1776, - "hfopenllm_v2/BBH": 0.334, - "hfopenllm_v2/MATH Level 5": 0.0196, - "hfopenllm_v2/GPQA": 0.2668, - "hfopenllm_v2/MUSR": 0.3722, - "hfopenllm_v2/MMLU-PRO": 0.164 - } - }, - { - "id": "oopere/pruned20-llama-1b", - "name": "pruned20-llama-1b", - "developer": "oopere", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1994, - "hfopenllm_v2/BBH": 0.3031, - "hfopenllm_v2/MATH Level 5": 0.0106, - "hfopenllm_v2/GPQA": 0.25, - "hfopenllm_v2/MUSR": 0.3631, - "hfopenllm_v2/MMLU-PRO": 0.1123 - } - }, - { - "id": "oopere/pruned20-llama-3.2-3b", - "name": "pruned20-llama-3.2-3b", - "developer": "oopere", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1789, - "hfopenllm_v2/BBH": 0.3248, - "hfopenllm_v2/MATH Level 5": 0.0159, - "hfopenllm_v2/GPQA": 0.2659, - "hfopenllm_v2/MUSR": 0.3418, - "hfopenllm_v2/MMLU-PRO": 0.128 - } - }, - { - "id": "oopere/pruned40-llama-1b", - "name": "pruned40-llama-1b", - "developer": "oopere", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2284, - "hfopenllm_v2/BBH": 0.2969, - "hfopenllm_v2/MATH Level 5": 0.0076, - "hfopenllm_v2/GPQA": 0.2433, - "hfopenllm_v2/MUSR": 0.4287, - "hfopenllm_v2/MMLU-PRO": 0.1082 - } - }, - { - "id": "oopere/pruned40-llama-3.2-1B", - "name": "pruned40-llama-3.2-1B", - "developer": "oopere", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2266, - "hfopenllm_v2/BBH": 0.2982, - "hfopenllm_v2/MATH Level 5": 0.0083, - "hfopenllm_v2/GPQA": 0.2542, - "hfopenllm_v2/MUSR": 0.4352, - "hfopenllm_v2/MMLU-PRO": 0.1115 - } - }, - { - "id": "oopere/pruned40-llama-3.2-3b", - "name": "pruned40-llama-3.2-3b", - "developer": "oopere", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2183, - "hfopenllm_v2/BBH": 0.3167, - "hfopenllm_v2/MATH Level 5": 0.0128, - "hfopenllm_v2/GPQA": 0.2299, - "hfopenllm_v2/MUSR": 0.3539, - "hfopenllm_v2/MMLU-PRO": 0.1177 - } - }, - { - "id": "oopere/pruned60-llama-1b", - "name": "pruned60-llama-1b", - "developer": "oopere", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1829, - "hfopenllm_v2/BBH": 0.3016, - "hfopenllm_v2/MATH Level 5": 0.0023, - "hfopenllm_v2/GPQA": 0.2492, - "hfopenllm_v2/MUSR": 0.4088, - "hfopenllm_v2/MMLU-PRO": 0.1173 - } - }, - { - "id": "oopere/pruned60-llama-3.2-3b", - "name": "pruned60-llama-3.2-3b", - "developer": "oopere", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1825, - "hfopenllm_v2/BBH": 0.3166, - "hfopenllm_v2/MATH Level 5": 0.0038, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.3633, - "hfopenllm_v2/MMLU-PRO": 0.1131 - } - }, - { - "id": "open-atlas/Atlas-Flash-1.5B-Preview", - "name": "Atlas-Flash-1.5B-Preview", - "developer": "open-atlas", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.327, - "hfopenllm_v2/BBH": 0.3215, - "hfopenllm_v2/MATH Level 5": 0.2213, - "hfopenllm_v2/GPQA": 0.2525, - "hfopenllm_v2/MUSR": 0.3488, - "hfopenllm_v2/MMLU-PRO": 0.1374 - } - }, - { - "id": "open-atlas/Atlas-Flash-7B-Preview", - "name": "Atlas-Flash-7B-Preview", - "developer": "open-atlas", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3908, - "hfopenllm_v2/BBH": 0.3542, - "hfopenllm_v2/MATH Level 5": 0.2576, - "hfopenllm_v2/GPQA": 0.2886, - "hfopenllm_v2/MUSR": 0.3836, - "hfopenllm_v2/MMLU-PRO": 0.2784 - } - }, - { - "id": "open-neo/Kyro-n1-3B", - "name": "Kyro-n1-3B", - "developer": "open-neo", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4595, - "hfopenllm_v2/BBH": 0.4685, - "hfopenllm_v2/MATH Level 5": 0.2855, - "hfopenllm_v2/GPQA": 0.2819, - "hfopenllm_v2/MUSR": 0.4088, - "hfopenllm_v2/MMLU-PRO": 0.3423 - } - }, - { - "id": "open-neo/Kyro-n1-7B", - "name": "Kyro-n1-7B", - "developer": "open-neo", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5573, - "hfopenllm_v2/BBH": 0.5387, - "hfopenllm_v2/MATH Level 5": 0.3897, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.3884, - "hfopenllm_v2/MMLU-PRO": 0.4333 - } - }, - { - "id": "Open-Orca/Mistral-7B-OpenOrca", - "name": "Mistral-7B-OpenOrca", - "developer": "Open-Orca", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4978, - "hfopenllm_v2/BBH": 0.4768, - "hfopenllm_v2/MATH Level 5": 0.0355, - "hfopenllm_v2/GPQA": 0.2718, - "hfopenllm_v2/MUSR": 0.3858, - "hfopenllm_v2/MMLU-PRO": 0.2653 - } - }, - { - "id": "open-thoughts/OpenThinker-7B", - "name": "OpenThinker-7B", - "developer": "open-thoughts", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4089, - "hfopenllm_v2/BBH": 0.5343, - "hfopenllm_v2/MATH Level 5": 0.426, - "hfopenllm_v2/GPQA": 0.2567, - "hfopenllm_v2/MUSR": 0.382, - "hfopenllm_v2/MMLU-PRO": 0.4165 - } - }, - { - "id": "openai-community/gpt2", - "name": "gpt2", - "developer": "openai-community", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.178, - "hfopenllm_v2/BBH": 0.3017, - "hfopenllm_v2/MATH Level 5": 0.0053, - "hfopenllm_v2/GPQA": 0.2584, - "hfopenllm_v2/MUSR": 0.439, - "hfopenllm_v2/MMLU-PRO": 0.1165 - } - }, - { - "id": "openai-community/gpt2-large", - "name": "gpt2-large", - "developer": "openai-community", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2048, - "hfopenllm_v2/BBH": 0.3069, - "hfopenllm_v2/MATH Level 5": 0.0121, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3789, - "hfopenllm_v2/MMLU-PRO": 0.1142 - } - }, - { - "id": "openai-community/gpt2-medium", - "name": "gpt2-medium", - "developer": "openai-community", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2208, - "hfopenllm_v2/BBH": 0.305, - "hfopenllm_v2/MATH Level 5": 0.0076, - "hfopenllm_v2/GPQA": 0.2626, - "hfopenllm_v2/MUSR": 0.3884, - "hfopenllm_v2/MMLU-PRO": 0.1182 - } - }, - { - "id": "openai-community/gpt2-xl", - "name": "gpt2-xl", - "developer": "openai-community", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2039, - "hfopenllm_v2/BBH": 0.3009, - "hfopenllm_v2/MATH Level 5": 0.0098, - "hfopenllm_v2/GPQA": 0.2584, - "hfopenllm_v2/MUSR": 0.371, - "hfopenllm_v2/MMLU-PRO": 0.1131 - } - }, - { - "id": "openai/ada-350M", - "name": "ada 350M", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.108, - "helm_classic/MMLU": 0.243, - "helm_classic/BoolQ": 0.581, - "helm_classic/NarrativeQA": 0.326, - "helm_classic/NaturalQuestions (open-book)": 0.365, - "helm_classic/QuAC": 0.242, - "helm_classic/HellaSwag": 0.435, - "helm_classic/OpenbookQA": 0.38, - "helm_classic/TruthfulQA": 0.215, - "helm_classic/MS MARCO (TREC)": 0.29, - "helm_classic/CNN/DailyMail": 0.09, - "helm_classic/XSUM": 0.022, - "helm_classic/IMDB": 0.849, - "helm_classic/CivilComments": 0.517, - "helm_classic/RAFT": 0.423 - } - }, - { - "id": "openai/babbage-1.3B", - "name": "babbage 1.3B", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.114, - "helm_classic/MMLU": 0.235, - "helm_classic/BoolQ": 0.574, - "helm_classic/NarrativeQA": 0.491, - "helm_classic/NaturalQuestions (open-book)": 0.451, - "helm_classic/QuAC": 0.273, - "helm_classic/HellaSwag": 0.555, - "helm_classic/OpenbookQA": 0.438, - "helm_classic/TruthfulQA": 0.188, - "helm_classic/MS MARCO (TREC)": 0.317, - "helm_classic/CNN/DailyMail": 0.079, - "helm_classic/XSUM": 0.045, - "helm_classic/IMDB": 0.597, - "helm_classic/CivilComments": 0.519, - "helm_classic/RAFT": 0.455 - } - }, - { - "id": "openai/curie-6.7B", - "name": "curie 6.7B", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.247, - "helm_classic/MMLU": 0.243, - "helm_classic/BoolQ": 0.656, - "helm_classic/NarrativeQA": 0.604, - "helm_classic/NaturalQuestions (open-book)": 0.552, - "helm_classic/QuAC": 0.321, - "helm_classic/HellaSwag": 0.682, - "helm_classic/OpenbookQA": 0.502, - "helm_classic/TruthfulQA": 0.232, - "helm_classic/MS MARCO (TREC)": 0.3, - "helm_classic/CNN/DailyMail": 0.113, - "helm_classic/XSUM": 0.091, - "helm_classic/IMDB": 0.889, - "helm_classic/CivilComments": 0.539, - "helm_classic/RAFT": 0.49 - } - }, - { - "id": "openai/davinci-175B", - "name": "davinci 175B", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.538, - "helm_classic/MMLU": 0.422, - "helm_classic/BoolQ": 0.722, - "helm_classic/NarrativeQA": 0.687, - "helm_classic/NaturalQuestions (open-book)": 0.625, - "helm_classic/QuAC": 0.36, - "helm_classic/HellaSwag": 0.775, - "helm_classic/OpenbookQA": 0.586, - "helm_classic/TruthfulQA": 0.194, - "helm_classic/MS MARCO (TREC)": 0.378, - "helm_classic/CNN/DailyMail": 0.127, - "helm_classic/XSUM": 0.126, - "helm_classic/IMDB": 0.933, - "helm_classic/CivilComments": 0.532, - "helm_classic/RAFT": 0.642 - } - }, - { - "id": "openai/GPT 4o", - "name": "GPT 4o", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "apex-v1/Overall Score": 0.359 - } - }, - { - "id": "openai/GPT 5", - "name": "GPT 5", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "apex-agents/Overall Pass@1": 0.183, - "apex-agents/Overall Pass@8": 0.31, - "apex-agents/Overall Mean Score": 0.329, - "apex-agents/Investment Banking Pass@1": 0.273, - "apex-agents/Management Consulting Pass@1": 0.123, - "apex-agents/Corporate Law Pass@1": 0.153, - "apex-agents/Corporate Lawyer Mean Score": 0.382, - "ace/Overall Score": 0.561, - "ace/DIY Score": 0.55, - "ace/Food Score": 0.7, - "ace/Gaming Score": 0.575, - "apex-v1/Overall Score": 0.67, - "apex-v1/Big Law Score": 0.78, - "apex-v1/Medicine (MD) Score": 0.66, - "apex-v1/Investment Banking Score": 0.61 - } - }, - { - "id": "openai/GPT 5 Codex", - "name": "GPT 5 Codex", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "apex-agents/Corporate Lawyer Mean Score": 0.362 - } - }, - { - "id": "openai/GPT 5.1", - "name": "GPT 5.1", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "apex-agents/Corporate Lawyer Mean Score": 0.376, - "ace/Overall Score": 0.551, - "ace/DIY Score": 0.56, - "ace/Gaming Score": 0.61, - "ace/Shopping Score": 0.45, - "apex-v1/Big Law Score": 0.77 - } - }, - { - "id": "openai/GPT 5.1 Codex", - "name": "GPT 5.1 Codex", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "apex-agents/Corporate Lawyer Mean Score": 0.366 - } - }, - { - "id": "openai/GPT 5.2", - "name": "GPT 5.2", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "ace/Overall Score": 0.515, - "ace/Food Score": 0.65, - "ace/Gaming Score": 0.578, - "apex-agents/Overall Pass@1": 0.23, - "apex-agents/Overall Pass@8": 0.4, - "apex-agents/Overall Mean Score": 0.387, - "apex-agents/Investment Banking Pass@1": 0.273, - "apex-agents/Management Consulting Pass@1": 0.227, - "apex-agents/Corporate Law Pass@1": 0.189, - "apex-agents/Corporate Lawyer Mean Score": 0.443 - } - }, - { - "id": "openai/GPT 5.2 Codex", - "name": "GPT 5.2 Codex", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "apex-agents/Overall Pass@1": 0.276, - "apex-agents/Corporate Lawyer Mean Score": 0.394 - } - }, - { - "id": "openai/GPT 5.2 Pro", - "name": "GPT 5.2 Pro", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "apex-v1/Overall Score": 0.668, - "apex-v1/Consulting Score": 0.64, - "apex-v1/Medicine (MD) Score": 0.65, - "apex-v1/Investment Banking Score": 0.64 - } - }, - { - "id": "openai/GPT 5.3 Codex", - "name": "GPT 5.3 Codex", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "apex-agents/Overall Pass@1": 0.317 - } - }, - { - "id": "openai/GPT OSS 120B", - "name": "GPT OSS 120B", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "apex-agents/Overall Pass@1": 0.047, - "apex-agents/Overall Pass@8": 0.115, - "apex-agents/Overall Mean Score": 0.145, - "apex-agents/Investment Banking Pass@1": 0.027, - "apex-agents/Management Consulting Pass@1": 0.035, - "apex-agents/Corporate Law Pass@1": 0.078, - "apex-agents/Corporate Lawyer Mean Score": 0.269 - } - }, - { - "id": "openai/gpt-3.5-turbo-0125", - "name": "GPT-3.5 Turbo 0125", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_mmlu/MMLU All Subjects": 0.673, - "helm_mmlu/Abstract Algebra": 0.31, - "helm_mmlu/Anatomy": 0.696, - "helm_mmlu/College Physics": 0.471, - "helm_mmlu/Computer Security": 0.78, - "helm_mmlu/Econometrics": 0.474, - "helm_mmlu/Global Facts": 0.39, - "helm_mmlu/Jurisprudence": 0.806, - "helm_mmlu/Philosophy": 0.746, - "helm_mmlu/Professional Psychology": 0.722, - "helm_mmlu/Us Foreign Policy": 0.89, - "helm_mmlu/Astronomy": 0.75, - "helm_mmlu/Business Ethics": 0.75, - "helm_mmlu/Clinical Knowledge": 0.755, - "helm_mmlu/Conceptual Physics": 0.634, - "helm_mmlu/Electrical Engineering": 0.669, - "helm_mmlu/Elementary Mathematics": 0.534, - "helm_mmlu/Formal Logic": 0.444, - "helm_mmlu/High School World History": 0.819, - "helm_mmlu/Human Sexuality": 0.779, - "helm_mmlu/International Law": 0.81, - "helm_mmlu/Logical Fallacies": 0.779, - "helm_mmlu/Machine Learning": 0.455, - "helm_mmlu/Management": 0.835, - "helm_mmlu/Marketing": 0.91, - "helm_mmlu/Medical Genetics": 0.73, - "helm_mmlu/Miscellaneous": 0.89, - "helm_mmlu/Moral Scenarios": 0.355, - "helm_mmlu/Nutrition": 0.748, - "helm_mmlu/Prehistory": 0.735, - "helm_mmlu/Public Relations": 0.727, - "helm_mmlu/Security Studies": 0.751, - "helm_mmlu/Sociology": 0.861, - "helm_mmlu/Virology": 0.536, - "helm_mmlu/World Religions": 0.842, - "helm_mmlu/Mean win rate": 0.493, - "reward-bench/Score": 0.6534, - "reward-bench/Chat": 0.9218, - "reward-bench/Chat Hard": 0.4452, - "reward-bench/Safety": 0.6547, - "reward-bench/Reasoning": 0.5912, - "reward-bench/Prior Sets (0.5 weight)": 0.6548 - } - }, - { - "id": "openai/gpt-3.5-turbo-0301", - "name": "gpt-3.5-turbo-0301", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.76, - "helm_classic/MMLU": 0.59, - "helm_classic/BoolQ": 0.74, - "helm_classic/NarrativeQA": 0.663, - "helm_classic/NaturalQuestions (open-book)": 0.624, - "helm_classic/QuAC": 0.512, - "helm_classic/HellaSwag": -1.0, - "helm_classic/OpenbookQA": -1.0, - "helm_classic/TruthfulQA": 0.609, - "helm_classic/MS MARCO (TREC)": -1.0, - "helm_classic/CNN/DailyMail": -1.0, - "helm_classic/XSUM": -1.0, - "helm_classic/IMDB": 0.899, - "helm_classic/CivilComments": 0.674, - "helm_classic/RAFT": 0.768 - } - }, - { - "id": "openai/gpt-3.5-turbo-0613", - "name": "GPT-3.5 Turbo 0613", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.783, - "helm_classic/MMLU": 0.391, - "helm_classic/BoolQ": 0.87, - "helm_classic/NarrativeQA": 0.625, - "helm_classic/NaturalQuestions (open-book)": 0.675, - "helm_classic/QuAC": 0.485, - "helm_classic/HellaSwag": -1.0, - "helm_classic/OpenbookQA": -1.0, - "helm_classic/TruthfulQA": 0.339, - "helm_classic/MS MARCO (TREC)": -1.0, - "helm_classic/CNN/DailyMail": -1.0, - "helm_classic/XSUM": -1.0, - "helm_classic/IMDB": 0.943, - "helm_classic/CivilComments": 0.696, - "helm_classic/RAFT": 0.748, - "helm_instruct/Mean win rate": 0.689, - "helm_instruct/Anthropic RLHF dataset": 4.964, - "helm_instruct/Best ChatGPT Prompts": 4.986, - "helm_instruct/Koala test dataset": 4.987, - "helm_instruct/Open Assistant": 4.987, - "helm_instruct/Self Instruct": 4.99, - "helm_instruct/Vicuna": 4.992, - "helm_lite/Mean win rate": 0.358, - "helm_lite/NarrativeQA": 0.655, - "helm_lite/NaturalQuestions (closed-book)": 0.335, - "helm_lite/OpenbookQA": 0.838, - "helm_lite/MMLU": 0.614, - "helm_lite/MATH": 0.667, - "helm_lite/GSM8K": 0.501, - "helm_lite/LegalBench": 0.528, - "helm_lite/MedQA": 0.622, - "helm_lite/WMT 2014": 0.187, - "helm_mmlu/MMLU All Subjects": 0.689, - "helm_mmlu/Abstract Algebra": 0.38, - "helm_mmlu/Anatomy": 0.659, - "helm_mmlu/College Physics": 0.461, - "helm_mmlu/Computer Security": 0.81, - "helm_mmlu/Econometrics": 0.5, - "helm_mmlu/Global Facts": 0.37, - "helm_mmlu/Jurisprudence": 0.806, - "helm_mmlu/Philosophy": 0.759, - "helm_mmlu/Professional Psychology": 0.732, - "helm_mmlu/Us Foreign Policy": 0.88, - "helm_mmlu/Astronomy": 0.763, - "helm_mmlu/Business Ethics": 0.75, - "helm_mmlu/Clinical Knowledge": 0.777, - "helm_mmlu/Conceptual Physics": 0.613, - "helm_mmlu/Electrical Engineering": 0.648, - "helm_mmlu/Elementary Mathematics": 0.5, - "helm_mmlu/Formal Logic": 0.397, - "helm_mmlu/High School World History": 0.857, - "helm_mmlu/Human Sexuality": 0.786, - "helm_mmlu/International Law": 0.843, - "helm_mmlu/Logical Fallacies": 0.791, - "helm_mmlu/Machine Learning": 0.455, - "helm_mmlu/Management": 0.845, - "helm_mmlu/Marketing": 0.91, - "helm_mmlu/Medical Genetics": 0.8, - "helm_mmlu/Miscellaneous": 0.893, - "helm_mmlu/Moral Scenarios": 0.404, - "helm_mmlu/Nutrition": 0.758, - "helm_mmlu/Prehistory": 0.787, - "helm_mmlu/Public Relations": 0.745, - "helm_mmlu/Security Studies": 0.8, - "helm_mmlu/Sociology": 0.871, - "helm_mmlu/Virology": 0.542, - "helm_mmlu/World Religions": 0.836, - "helm_mmlu/Mean win rate": 0.589 - } - }, - { - "id": "openai/gpt-4-0125-preview", - "name": "openai/gpt-4-0125-preview", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.8434, - "reward-bench/Chat": 0.9525, - "reward-bench/Chat Hard": 0.7434, - "reward-bench/Safety": 0.8757, - "reward-bench/Reasoning": 0.8692, - "reward-bench/Prior Sets (0.5 weight)": 0.7085 - } - }, - { - "id": "openai/gpt-4-0314", - "name": "GPT-4 0314", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_instruct/Mean win rate": 0.611, - "helm_instruct/Anthropic RLHF dataset": 4.934, - "helm_instruct/Best ChatGPT Prompts": 4.973, - "helm_instruct/Koala test dataset": 4.966, - "helm_instruct/Open Assistant": 4.986, - "helm_instruct/Self Instruct": 4.976, - "helm_instruct/Vicuna": 4.995 - } - }, - { - "id": "openai/gpt-4-0613", - "name": "GPT-4 0613", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.867, - "helm_lite/NarrativeQA": 0.768, - "helm_lite/NaturalQuestions (closed-book)": 0.457, - "helm_lite/OpenbookQA": 0.96, - "helm_lite/MMLU": 0.735, - "helm_lite/MATH": 0.802, - "helm_lite/GSM8K": 0.932, - "helm_lite/LegalBench": 0.713, - "helm_lite/MedQA": 0.815, - "helm_lite/WMT 2014": 0.211, - "helm_mmlu/MMLU All Subjects": 0.824, - "helm_mmlu/Abstract Algebra": 0.63, - "helm_mmlu/Anatomy": 0.8, - "helm_mmlu/College Physics": 0.627, - "helm_mmlu/Computer Security": 0.86, - "helm_mmlu/Econometrics": 0.684, - "helm_mmlu/Global Facts": 0.62, - "helm_mmlu/Jurisprudence": 0.889, - "helm_mmlu/Philosophy": 0.859, - "helm_mmlu/Professional Psychology": 0.891, - "helm_mmlu/Us Foreign Policy": 0.95, - "helm_mmlu/Astronomy": 0.934, - "helm_mmlu/Business Ethics": 0.79, - "helm_mmlu/Clinical Knowledge": 0.845, - "helm_mmlu/Conceptual Physics": 0.868, - "helm_mmlu/Electrical Engineering": 0.786, - "helm_mmlu/Elementary Mathematics": 0.807, - "helm_mmlu/Formal Logic": 0.643, - "helm_mmlu/High School World History": 0.945, - "helm_mmlu/Human Sexuality": 0.908, - "helm_mmlu/International Law": 0.917, - "helm_mmlu/Logical Fallacies": 0.871, - "helm_mmlu/Machine Learning": 0.759, - "helm_mmlu/Management": 0.932, - "helm_mmlu/Marketing": 0.962, - "helm_mmlu/Medical Genetics": 0.94, - "helm_mmlu/Miscellaneous": 0.949, - "helm_mmlu/Moral Scenarios": 0.902, - "helm_mmlu/Nutrition": 0.892, - "helm_mmlu/Prehistory": 0.926, - "helm_mmlu/Public Relations": 0.745, - "helm_mmlu/Security Studies": 0.861, - "helm_mmlu/Sociology": 0.93, - "helm_mmlu/Virology": 0.596, - "helm_mmlu/World Religions": 0.877, - "helm_mmlu/Mean win rate": 0.517 - } - }, - { - "id": "openai/gpt-4-1-2025-04-14-fc", - "name": "GPT-4.1-2025-04-14 (FC)", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 20.0, - "bfcl/bfcl.overall.overall_accuracy": 53.96, - "bfcl/bfcl.overall.total_cost_usd": 100.75, - "bfcl/bfcl.overall.latency_mean_s": 1.63, - "bfcl/bfcl.overall.latency_std_s": 3.05, - "bfcl/bfcl.overall.latency_p95_s": 4.01, - "bfcl/bfcl.non_live.ast_accuracy": 82.79, - "bfcl/bfcl.non_live.simple_ast_accuracy": 72.67, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 89.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 88.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 81.5, - "bfcl/bfcl.live.live_accuracy": 69.95, - "bfcl/bfcl.live.live_simple_ast_accuracy": 69.38, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 70.28, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 56.25, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 70.83, - "bfcl/bfcl.multi_turn.accuracy": 38.88, - "bfcl/bfcl.multi_turn.base_accuracy": 47.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 32.5, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 32.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 43.0, - "bfcl/bfcl.web_search.accuracy": 68.0, - "bfcl/bfcl.web_search.base_accuracy": 67.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 69.0, - "bfcl/bfcl.memory.accuracy": 23.87, - "bfcl/bfcl.memory.kv_accuracy": 16.13, - "bfcl/bfcl.memory.vector_accuracy": 18.06, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 37.42, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 87.5, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 86.52 - } - }, - { - "id": "openai/gpt-4-1-2025-04-14-prompt", - "name": "GPT-4.1-2025-04-14 (Prompt)", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 45.0, - "bfcl/bfcl.overall.overall_accuracy": 39.38, - "bfcl/bfcl.overall.total_cost_usd": 145.85, - "bfcl/bfcl.overall.latency_mean_s": 1.2, - "bfcl/bfcl.overall.latency_std_s": 3.23, - "bfcl/bfcl.overall.latency_p95_s": 2.53, - "bfcl/bfcl.non_live.ast_accuracy": 88.69, - "bfcl/bfcl.non_live.simple_ast_accuracy": 78.25, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 93.5, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 94.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 89.0, - "bfcl/bfcl.live.live_accuracy": 78.9, - "bfcl/bfcl.live.live_simple_ast_accuracy": 84.88, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 77.4, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 87.5, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 75.0, - "bfcl/bfcl.multi_turn.accuracy": 9.75, - "bfcl/bfcl.multi_turn.base_accuracy": 10.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 11.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 8.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 9.5, - "bfcl/bfcl.web_search.accuracy": 35.0, - "bfcl/bfcl.web_search.base_accuracy": 40.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 30.0, - "bfcl/bfcl.memory.accuracy": 21.51, - "bfcl/bfcl.memory.kv_accuracy": 9.68, - "bfcl/bfcl.memory.vector_accuracy": 19.35, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 35.48, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 100.0, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 83.99, - "bfcl/bfcl.format_sensitivity.max_delta": 23.5, - "bfcl/bfcl.format_sensitivity.stddev": 6.18 - } - }, - { - "id": "openai/gpt-4-1-mini-2025-04-14-fc", - "name": "GPT-4.1-mini-2025-04-14 (FC)", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 27.0, - "bfcl/bfcl.overall.overall_accuracy": 50.45, - "bfcl/bfcl.overall.total_cost_usd": 19.25, - "bfcl/bfcl.overall.latency_mean_s": 1.32, - "bfcl/bfcl.overall.latency_std_s": 3.65, - "bfcl/bfcl.overall.latency_p95_s": 2.4, - "bfcl/bfcl.non_live.ast_accuracy": 83.83, - "bfcl/bfcl.non_live.simple_ast_accuracy": 73.33, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 89.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 91.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 82.0, - "bfcl/bfcl.live.live_accuracy": 68.84, - "bfcl/bfcl.live.live_simple_ast_accuracy": 67.05, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 69.8, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 43.75, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 62.5, - "bfcl/bfcl.multi_turn.accuracy": 34.13, - "bfcl/bfcl.multi_turn.base_accuracy": 43.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 22.5, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 30.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 40.0, - "bfcl/bfcl.web_search.accuracy": 57.0, - "bfcl/bfcl.web_search.base_accuracy": 62.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 52.0, - "bfcl/bfcl.memory.accuracy": 26.88, - "bfcl/bfcl.memory.kv_accuracy": 22.58, - "bfcl/bfcl.memory.vector_accuracy": 16.13, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 41.94, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 81.25, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 81.69 - } - }, - { - "id": "openai/gpt-4-1-mini-2025-04-14-prompt", - "name": "GPT-4.1-mini-2025-04-14 (Prompt)", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 67.0, - "bfcl/bfcl.overall.overall_accuracy": 29.73, - "bfcl/bfcl.overall.total_cost_usd": 20.52, - "bfcl/bfcl.overall.latency_mean_s": 1.36, - "bfcl/bfcl.overall.latency_std_s": 4.5, - "bfcl/bfcl.overall.latency_p95_s": 3.38, - "bfcl/bfcl.non_live.ast_accuracy": 84.6, - "bfcl/bfcl.non_live.simple_ast_accuracy": 74.92, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 92.5, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 87.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 83.5, - "bfcl/bfcl.live.live_accuracy": 74.76, - "bfcl/bfcl.live.live_simple_ast_accuracy": 80.62, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 73.31, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 81.25, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 70.83, - "bfcl/bfcl.multi_turn.accuracy": 2.5, - "bfcl/bfcl.multi_turn.base_accuracy": 1.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 4.5, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 2.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 1.5, - "bfcl/bfcl.web_search.accuracy": 4.0, - "bfcl/bfcl.web_search.base_accuracy": 7.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 1.0, - "bfcl/bfcl.memory.accuracy": 24.3, - "bfcl/bfcl.memory.kv_accuracy": 20.65, - "bfcl/bfcl.memory.vector_accuracy": 13.55, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 38.71, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 87.5, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 73.88, - "bfcl/bfcl.format_sensitivity.max_delta": 45.0, - "bfcl/bfcl.format_sensitivity.stddev": 13.33 - } - }, - { - "id": "openai/gpt-4-1-nano-2025-04-14-fc", - "name": "GPT-4.1-nano-2025-04-14 (FC)", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 58.0, - "bfcl/bfcl.overall.overall_accuracy": 33.05, - "bfcl/bfcl.overall.total_cost_usd": 5.66, - "bfcl/bfcl.overall.latency_mean_s": 1.44, - "bfcl/bfcl.overall.latency_std_s": 10.84, - "bfcl/bfcl.overall.latency_p95_s": 2.26, - "bfcl/bfcl.non_live.ast_accuracy": 72.98, - "bfcl/bfcl.non_live.simple_ast_accuracy": 59.92, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 79.5, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 84.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 68.5, - "bfcl/bfcl.live.live_accuracy": 60.77, - "bfcl/bfcl.live.live_simple_ast_accuracy": 58.14, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 61.44, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 68.75, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 54.17, - "bfcl/bfcl.multi_turn.accuracy": 23.62, - "bfcl/bfcl.multi_turn.base_accuracy": 39.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 7.5, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 17.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 30.0, - "bfcl/bfcl.web_search.accuracy": 11.0, - "bfcl/bfcl.web_search.base_accuracy": 13.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 9.0, - "bfcl/bfcl.memory.accuracy": 18.92, - "bfcl/bfcl.memory.kv_accuracy": 10.32, - "bfcl/bfcl.memory.vector_accuracy": 19.35, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 27.1, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 93.75, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 66.0 - } - }, - { - "id": "openai/gpt-4-1-nano-2025-04-14-prompt", - "name": "GPT-4.1-nano-2025-04-14 (Prompt)", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 90.0, - "bfcl/bfcl.overall.overall_accuracy": 24.88, - "bfcl/bfcl.overall.total_cost_usd": 7.42, - "bfcl/bfcl.overall.latency_mean_s": 1.02, - "bfcl/bfcl.overall.latency_std_s": 7.3, - "bfcl/bfcl.overall.latency_p95_s": 1.88, - "bfcl/bfcl.non_live.ast_accuracy": 72.44, - "bfcl/bfcl.non_live.simple_ast_accuracy": 68.75, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 63.5, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 85.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 72.5, - "bfcl/bfcl.live.live_accuracy": 50.33, - "bfcl/bfcl.live.live_simple_ast_accuracy": 63.18, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 46.53, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 87.5, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 54.17, - "bfcl/bfcl.multi_turn.accuracy": 2.0, - "bfcl/bfcl.multi_turn.base_accuracy": 2.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 1.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 2.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 2.0, - "bfcl/bfcl.web_search.accuracy": 1.5, - "bfcl/bfcl.web_search.base_accuracy": 2.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 1.0, - "bfcl/bfcl.memory.accuracy": 16.77, - "bfcl/bfcl.memory.kv_accuracy": 9.03, - "bfcl/bfcl.memory.vector_accuracy": 14.19, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 27.1, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 68.75, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 83.44, - "bfcl/bfcl.format_sensitivity.max_delta": 73.0, - "bfcl/bfcl.format_sensitivity.stddev": 17.08 - } - }, - { - "id": "openai/gpt-4-1106-preview", - "name": "GPT-4 Turbo 1106 preview", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.698, - "helm_lite/NarrativeQA": 0.727, - "helm_lite/NaturalQuestions (closed-book)": 0.435, - "helm_lite/OpenbookQA": 0.95, - "helm_lite/MMLU": 0.699, - "helm_lite/MATH": 0.857, - "helm_lite/GSM8K": 0.668, - "helm_lite/LegalBench": 0.626, - "helm_lite/MedQA": 0.817, - "helm_lite/WMT 2014": 0.205, - "helm_mmlu/MMLU All Subjects": 0.796, - "helm_mmlu/Abstract Algebra": 0.53, - "helm_mmlu/Anatomy": 0.807, - "helm_mmlu/College Physics": 0.402, - "helm_mmlu/Computer Security": 0.86, - "helm_mmlu/Econometrics": 0.675, - "helm_mmlu/Global Facts": 0.58, - "helm_mmlu/Jurisprudence": 0.889, - "helm_mmlu/Philosophy": 0.852, - "helm_mmlu/Professional Psychology": 0.887, - "helm_mmlu/Us Foreign Policy": 0.96, - "helm_mmlu/Astronomy": 0.941, - "helm_mmlu/Business Ethics": 0.78, - "helm_mmlu/Clinical Knowledge": 0.864, - "helm_mmlu/Conceptual Physics": 0.894, - "helm_mmlu/Electrical Engineering": 0.772, - "helm_mmlu/Elementary Mathematics": 0.638, - "helm_mmlu/Formal Logic": 0.651, - "helm_mmlu/High School World History": 0.958, - "helm_mmlu/Human Sexuality": 0.908, - "helm_mmlu/International Law": 0.926, - "helm_mmlu/Logical Fallacies": 0.865, - "helm_mmlu/Machine Learning": 0.723, - "helm_mmlu/Management": 0.913, - "helm_mmlu/Marketing": 0.932, - "helm_mmlu/Medical Genetics": 0.93, - "helm_mmlu/Miscellaneous": 0.946, - "helm_mmlu/Moral Scenarios": 0.816, - "helm_mmlu/Nutrition": 0.879, - "helm_mmlu/Prehistory": 0.917, - "helm_mmlu/Public Relations": 0.782, - "helm_mmlu/Security Studies": 0.841, - "helm_mmlu/Sociology": 0.925, - "helm_mmlu/Virology": 0.59, - "helm_mmlu/World Religions": 0.854, - "helm_mmlu/Mean win rate": 0.416 - } - }, - { - "id": "openai/gpt-4-turbo-2024-04-09", - "name": "GPT-4 Turbo 2024-04-09", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.864, - "helm_lite/NarrativeQA": 0.761, - "helm_lite/NaturalQuestions (closed-book)": 0.482, - "helm_lite/OpenbookQA": 0.97, - "helm_lite/MMLU": 0.711, - "helm_lite/MATH": 0.833, - "helm_lite/GSM8K": 0.824, - "helm_lite/LegalBench": 0.727, - "helm_lite/MedQA": 0.783, - "helm_lite/WMT 2014": 0.218, - "helm_mmlu/MMLU All Subjects": 0.813, - "helm_mmlu/Abstract Algebra": 0.56, - "helm_mmlu/Anatomy": 0.822, - "helm_mmlu/College Physics": 0.539, - "helm_mmlu/Computer Security": 0.83, - "helm_mmlu/Econometrics": 0.675, - "helm_mmlu/Global Facts": 0.58, - "helm_mmlu/Jurisprudence": 0.88, - "helm_mmlu/Philosophy": 0.868, - "helm_mmlu/Professional Psychology": 0.873, - "helm_mmlu/Us Foreign Policy": 0.96, - "helm_mmlu/Astronomy": 0.941, - "helm_mmlu/Business Ethics": 0.82, - "helm_mmlu/Clinical Knowledge": 0.83, - "helm_mmlu/Conceptual Physics": 0.894, - "helm_mmlu/Electrical Engineering": 0.752, - "helm_mmlu/Elementary Mathematics": 0.72, - "helm_mmlu/Formal Logic": 0.706, - "helm_mmlu/High School World History": 0.941, - "helm_mmlu/Human Sexuality": 0.901, - "helm_mmlu/International Law": 0.942, - "helm_mmlu/Logical Fallacies": 0.871, - "helm_mmlu/Machine Learning": 0.741, - "helm_mmlu/Management": 0.883, - "helm_mmlu/Marketing": 0.949, - "helm_mmlu/Medical Genetics": 0.92, - "helm_mmlu/Miscellaneous": 0.945, - "helm_mmlu/Moral Scenarios": 0.803, - "helm_mmlu/Nutrition": 0.892, - "helm_mmlu/Prehistory": 0.92, - "helm_mmlu/Public Relations": 0.755, - "helm_mmlu/Security Studies": 0.8, - "helm_mmlu/Sociology": 0.915, - "helm_mmlu/Virology": 0.602, - "helm_mmlu/World Religions": 0.848, - "helm_mmlu/Mean win rate": 0.351, - "reward-bench/Score": 0.8395, - "reward-bench/Chat": 0.9525, - "reward-bench/Chat Hard": 0.7544, - "reward-bench/Safety": 0.8757, - "reward-bench/Reasoning": 0.827, - "reward-bench/Prior Sets (0.5 weight)": 0.7363 - } - }, - { - "id": "openai/gpt-4.1", - "name": "openai/gpt-4.1", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "livecodebenchpro/Hard Problems": 0.0, - "livecodebenchpro/Medium Problems": 0.0, - "livecodebenchpro/Easy Problems": 0.19718309859154928 - } - }, - { - "id": "openai/gpt-4.1-2025-04-14", - "name": "gpt-4.1-2025-04-14", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "global-mmlu-lite/Global MMLU Lite": 0.8755, - "global-mmlu-lite/Culturally Sensitive": 0.8541, - "global-mmlu-lite/Culturally Agnostic": 0.8969, - "global-mmlu-lite/Arabic": 0.88, - "global-mmlu-lite/English": 0.8825, - "global-mmlu-lite/Bengali": 0.8625, - "global-mmlu-lite/German": 0.875, - "global-mmlu-lite/French": 0.8875, - "global-mmlu-lite/Hindi": 0.8775, - "global-mmlu-lite/Indonesian": 0.885, - "global-mmlu-lite/Italian": 0.88, - "global-mmlu-lite/Japanese": 0.8725, - "global-mmlu-lite/Korean": 0.87, - "global-mmlu-lite/Portuguese": 0.875, - "global-mmlu-lite/Spanish": 0.885, - "global-mmlu-lite/Swahili": 0.8725, - "global-mmlu-lite/Yoruba": 0.875, - "global-mmlu-lite/Chinese": 0.87, - "global-mmlu-lite/Burmese": 0.8575, - "helm_capabilities/Mean score": 0.727, - "helm_capabilities/MMLU-Pro": 0.811, - "helm_capabilities/GPQA": 0.659, - "helm_capabilities/IFEval": 0.838, - "helm_capabilities/WildBench": 0.854, - "helm_capabilities/Omni-MATH": 0.471, - "reward-bench/Score": 0.7232, - "reward-bench/Factuality": 0.8289, - "reward-bench/Precise IF": 0.3974, - "reward-bench/Math": 0.6521, - "reward-bench/Safety": 0.8726, - "reward-bench/Focus": 0.7338, - "reward-bench/Ties": 0.8542 - } - }, - { - "id": "openai/gpt-4.1-mini-2025-04-14", - "name": "GPT-4.1 mini 2025-04-14", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.726, - "helm_capabilities/MMLU-Pro": 0.783, - "helm_capabilities/GPQA": 0.614, - "helm_capabilities/IFEval": 0.904, - "helm_capabilities/WildBench": 0.838, - "helm_capabilities/Omni-MATH": 0.491, - "reward-bench/Score": 0.6573, - "reward-bench/Factuality": 0.6084, - "reward-bench/Precise IF": 0.4125, - "reward-bench/Math": 0.7213, - "reward-bench/Safety": 0.7265, - "reward-bench/Focus": 0.7354, - "reward-bench/Ties": 0.74 - } - }, - { - "id": "openai/gpt-4.1-nano-2025-04-14", - "name": "GPT-4.1 nano 2025-04-14", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.616, - "helm_capabilities/MMLU-Pro": 0.55, - "helm_capabilities/GPQA": 0.507, - "helm_capabilities/IFEval": 0.843, - "helm_capabilities/WildBench": 0.811, - "helm_capabilities/Omni-MATH": 0.367, - "reward-bench/Score": 0.4849, - "reward-bench/Factuality": 0.4646, - "reward-bench/Precise IF": 0.2578, - "reward-bench/Math": 0.5041, - "reward-bench/Safety": 0.7156, - "reward-bench/Focus": 0.466, - "reward-bench/Ties": 0.5015 - } - }, - { - "id": "openai/gpt-4o-2024-05-13", - "name": "GPT-4o 2024-05-13", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.938, - "helm_lite/NarrativeQA": 0.804, - "helm_lite/NaturalQuestions (closed-book)": 0.501, - "helm_lite/OpenbookQA": 0.966, - "helm_lite/MMLU": 0.748, - "helm_lite/MATH": 0.829, - "helm_lite/GSM8K": 0.905, - "helm_lite/LegalBench": 0.733, - "helm_lite/MedQA": 0.857, - "helm_lite/WMT 2014": 0.231, - "helm_mmlu/MMLU All Subjects": 0.842, - "helm_mmlu/Abstract Algebra": 0.66, - "helm_mmlu/Anatomy": 0.911, - "helm_mmlu/College Physics": 0.686, - "helm_mmlu/Computer Security": 0.85, - "helm_mmlu/Econometrics": 0.693, - "helm_mmlu/Global Facts": 0.64, - "helm_mmlu/Jurisprudence": 0.898, - "helm_mmlu/Philosophy": 0.9, - "helm_mmlu/Professional Psychology": 0.905, - "helm_mmlu/Us Foreign Policy": 0.96, - "helm_mmlu/Astronomy": 0.941, - "helm_mmlu/Business Ethics": 0.85, - "helm_mmlu/Clinical Knowledge": 0.894, - "helm_mmlu/Conceptual Physics": 0.911, - "helm_mmlu/Electrical Engineering": 0.807, - "helm_mmlu/Elementary Mathematics": 0.741, - "helm_mmlu/Formal Logic": 0.683, - "helm_mmlu/High School World History": 0.945, - "helm_mmlu/Human Sexuality": 0.908, - "helm_mmlu/International Law": 0.934, - "helm_mmlu/Logical Fallacies": 0.883, - "helm_mmlu/Machine Learning": 0.768, - "helm_mmlu/Management": 0.942, - "helm_mmlu/Marketing": 0.936, - "helm_mmlu/Medical Genetics": 0.96, - "helm_mmlu/Miscellaneous": 0.954, - "helm_mmlu/Moral Scenarios": 0.841, - "helm_mmlu/Nutrition": 0.899, - "helm_mmlu/Prehistory": 0.938, - "helm_mmlu/Public Relations": 0.809, - "helm_mmlu/Security Studies": 0.837, - "helm_mmlu/Sociology": 0.94, - "helm_mmlu/Virology": 0.596, - "helm_mmlu/World Religions": 0.889, - "helm_mmlu/Mean win rate": 0.671, - "reward-bench/Score": 0.8327, - "reward-bench/Chat": 0.9665, - "reward-bench/Chat Hard": 0.7039, - "reward-bench/Safety": 0.8649, - "reward-bench/Reasoning": 0.8487, - "reward-bench/Prior Sets (0.5 weight)": 0.7262 - } - }, - { - "id": "openai/gpt-4o-2024-08-06", - "name": "GPT-4o 2024-08-06", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.928, - "helm_lite/NarrativeQA": 0.795, - "helm_lite/NaturalQuestions (closed-book)": 0.496, - "helm_lite/OpenbookQA": 0.968, - "helm_lite/MMLU": 0.738, - "helm_lite/MATH": 0.853, - "helm_lite/GSM8K": 0.909, - "helm_lite/LegalBench": 0.721, - "helm_lite/MedQA": 0.863, - "helm_lite/WMT 2014": 0.225, - "helm_mmlu/MMLU All Subjects": 0.843, - "helm_mmlu/Abstract Algebra": 0.58, - "helm_mmlu/Anatomy": 0.911, - "helm_mmlu/College Physics": 0.686, - "helm_mmlu/Computer Security": 0.85, - "helm_mmlu/Econometrics": 0.711, - "helm_mmlu/Global Facts": 0.69, - "helm_mmlu/Jurisprudence": 0.907, - "helm_mmlu/Philosophy": 0.894, - "helm_mmlu/Professional Psychology": 0.899, - "helm_mmlu/Us Foreign Policy": 0.95, - "helm_mmlu/Astronomy": 0.947, - "helm_mmlu/Business Ethics": 0.89, - "helm_mmlu/Clinical Knowledge": 0.894, - "helm_mmlu/Conceptual Physics": 0.923, - "helm_mmlu/Electrical Engineering": 0.793, - "helm_mmlu/Elementary Mathematics": 0.775, - "helm_mmlu/Formal Logic": 0.675, - "helm_mmlu/High School World History": 0.941, - "helm_mmlu/Human Sexuality": 0.901, - "helm_mmlu/International Law": 0.942, - "helm_mmlu/Logical Fallacies": 0.902, - "helm_mmlu/Machine Learning": 0.777, - "helm_mmlu/Management": 0.913, - "helm_mmlu/Marketing": 0.94, - "helm_mmlu/Medical Genetics": 0.98, - "helm_mmlu/Miscellaneous": 0.958, - "helm_mmlu/Moral Scenarios": 0.802, - "helm_mmlu/Nutrition": 0.905, - "helm_mmlu/Prehistory": 0.935, - "helm_mmlu/Public Relations": 0.782, - "helm_mmlu/Security Studies": 0.833, - "helm_mmlu/Sociology": 0.945, - "helm_mmlu/Virology": 0.578, - "helm_mmlu/World Religions": 0.883, - "helm_mmlu/Mean win rate": 0.52, - "reward-bench/Score": 0.6493, - "reward-bench/Chat": 0.9609, - "reward-bench/Chat Hard": 0.761, - "reward-bench/Safety": 0.8619, - "reward-bench/Reasoning": 0.8661, - "reward-bench/Factuality": 0.5684, - "reward-bench/Precise IF": 0.3312, - "reward-bench/Math": 0.623, - "reward-bench/Focus": 0.7293, - "reward-bench/Ties": 0.7819 - } - }, - { - "id": "openai/gpt-4o-2024-11-20", - "name": "GPT-4o 2024-11-20", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.634, - "helm_capabilities/MMLU-Pro": 0.713, - "helm_capabilities/GPQA": 0.52, - "helm_capabilities/IFEval": 0.817, - "helm_capabilities/WildBench": 0.828, - "helm_capabilities/Omni-MATH": 0.293, - "livecodebenchpro/Hard Problems": 0.0, - "livecodebenchpro/Medium Problems": 0.0, - "livecodebenchpro/Easy Problems": 0.07042253521126761 - } - }, - { - "id": "openai/gpt-4o-mini-2024-07-18", - "name": "GPT-4o mini 2024-07-18", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.565, - "helm_capabilities/MMLU-Pro": 0.603, - "helm_capabilities/GPQA": 0.368, - "helm_capabilities/IFEval": 0.782, - "helm_capabilities/WildBench": 0.791, - "helm_capabilities/Omni-MATH": 0.28, - "helm_lite/Mean win rate": 0.701, - "helm_lite/NarrativeQA": 0.768, - "helm_lite/NaturalQuestions (closed-book)": 0.386, - "helm_lite/OpenbookQA": 0.92, - "helm_lite/MMLU": 0.668, - "helm_lite/MATH": 0.802, - "helm_lite/GSM8K": 0.843, - "helm_lite/LegalBench": 0.653, - "helm_lite/MedQA": 0.748, - "helm_lite/WMT 2014": 0.206, - "helm_mmlu/MMLU All Subjects": 0.767, - "helm_mmlu/Abstract Algebra": 0.42, - "helm_mmlu/Anatomy": 0.77, - "helm_mmlu/College Physics": 0.559, - "helm_mmlu/Computer Security": 0.85, - "helm_mmlu/Econometrics": 0.649, - "helm_mmlu/Global Facts": 0.45, - "helm_mmlu/Jurisprudence": 0.87, - "helm_mmlu/Philosophy": 0.772, - "helm_mmlu/Professional Psychology": 0.833, - "helm_mmlu/Us Foreign Policy": 0.91, - "helm_mmlu/Astronomy": 0.849, - "helm_mmlu/Business Ethics": 0.79, - "helm_mmlu/Clinical Knowledge": 0.845, - "helm_mmlu/Conceptual Physics": 0.791, - "helm_mmlu/Electrical Engineering": 0.731, - "helm_mmlu/Elementary Mathematics": 0.651, - "helm_mmlu/Formal Logic": 0.556, - "helm_mmlu/High School World History": 0.903, - "helm_mmlu/Human Sexuality": 0.863, - "helm_mmlu/International Law": 0.926, - "helm_mmlu/Logical Fallacies": 0.871, - "helm_mmlu/Machine Learning": 0.616, - "helm_mmlu/Management": 0.845, - "helm_mmlu/Marketing": 0.927, - "helm_mmlu/Medical Genetics": 0.89, - "helm_mmlu/Miscellaneous": 0.913, - "helm_mmlu/Moral Scenarios": 0.485, - "helm_mmlu/Nutrition": 0.827, - "helm_mmlu/Prehistory": 0.833, - "helm_mmlu/Public Relations": 0.791, - "helm_mmlu/Security Studies": 0.788, - "helm_mmlu/Sociology": 0.9, - "helm_mmlu/Virology": 0.536, - "helm_mmlu/World Religions": 0.86, - "helm_mmlu/Mean win rate": 0.774, - "reward-bench/Score": 0.8007, - "reward-bench/Factuality": 0.4105, - "reward-bench/Precise IF": 0.3438, - "reward-bench/Math": 0.5191, - "reward-bench/Safety": 0.8081, - "reward-bench/Focus": 0.7414, - "reward-bench/Ties": 0.6962, - "reward-bench/Chat": 0.9497, - "reward-bench/Chat Hard": 0.6075, - "reward-bench/Reasoning": 0.8374 - } - }, - { - "id": "openai/gpt-5", - "name": "GPT-5", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "terminal-bench-2.0/terminal-bench-2.0": 49.6 - } - }, - { - "id": "openai/gpt-5-2-2025-12-11-fc", - "name": "GPT-5.2-2025-12-11 (FC)", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 16.0, - "bfcl/bfcl.overall.overall_accuracy": 55.87, - "bfcl/bfcl.overall.total_cost_usd": 85.65, - "bfcl/bfcl.overall.latency_mean_s": 2.23, - "bfcl/bfcl.overall.latency_std_s": 9.75, - "bfcl/bfcl.overall.latency_p95_s": 5.26, - "bfcl/bfcl.non_live.ast_accuracy": 81.85, - "bfcl/bfcl.non_live.simple_ast_accuracy": 72.92, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 88.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 89.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 77.5, - "bfcl/bfcl.live.live_accuracy": 70.39, - "bfcl/bfcl.live.live_simple_ast_accuracy": 71.71, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 70.37, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 68.75, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 58.33, - "bfcl/bfcl.multi_turn.accuracy": 28.12, - "bfcl/bfcl.multi_turn.base_accuracy": 36.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 18.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 27.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 30.5, - "bfcl/bfcl.web_search.accuracy": 75.5, - "bfcl/bfcl.web_search.base_accuracy": 78.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 73.0, - "bfcl/bfcl.memory.accuracy": 45.81, - "bfcl/bfcl.memory.kv_accuracy": 33.55, - "bfcl/bfcl.memory.vector_accuracy": 43.23, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 60.65, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 75.0, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 79.42 - } - }, - { - "id": "openai/gpt-5-2-2025-12-11-prompt", - "name": "GPT-5.2-2025-12-11 (Prompt)", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 38.0, - "bfcl/bfcl.overall.overall_accuracy": 45.27, - "bfcl/bfcl.overall.total_cost_usd": 164.58, - "bfcl/bfcl.overall.latency_mean_s": 4.21, - "bfcl/bfcl.overall.latency_std_s": 20.93, - "bfcl/bfcl.overall.latency_p95_s": 10.58, - "bfcl/bfcl.non_live.ast_accuracy": 78.29, - "bfcl/bfcl.non_live.simple_ast_accuracy": 71.17, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 83.5, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 84.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 74.5, - "bfcl/bfcl.live.live_accuracy": 67.14, - "bfcl/bfcl.live.live_simple_ast_accuracy": 77.91, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 64.58, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 75.0, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 58.33, - "bfcl/bfcl.multi_turn.accuracy": 43.75, - "bfcl/bfcl.multi_turn.base_accuracy": 54.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 40.5, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 33.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 46.5, - "bfcl/bfcl.web_search.accuracy": 40.5, - "bfcl/bfcl.web_search.base_accuracy": 45.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 36.0, - "bfcl/bfcl.memory.accuracy": 3.87, - "bfcl/bfcl.memory.kv_accuracy": 2.58, - "bfcl/bfcl.memory.vector_accuracy": 1.94, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 7.1, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 75.0, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 87.26, - "bfcl/bfcl.format_sensitivity.max_delta": 13.0, - "bfcl/bfcl.format_sensitivity.stddev": 3.25 - } - }, - { - "id": "openai/gpt-5-2025-08-07", - "name": "gpt-5-2025-08-07", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "global-mmlu-lite/Global MMLU Lite": 0.8895, - "global-mmlu-lite/Culturally Sensitive": 0.8913, - "global-mmlu-lite/Culturally Agnostic": 0.8878, - "global-mmlu-lite/Arabic": 0.8925, - "global-mmlu-lite/English": 0.8725, - "global-mmlu-lite/Bengali": 0.9, - "global-mmlu-lite/German": 0.91, - "global-mmlu-lite/French": 0.9075, - "global-mmlu-lite/Hindi": 0.865, - "global-mmlu-lite/Indonesian": 0.795, - "global-mmlu-lite/Italian": 0.9075, - "global-mmlu-lite/Japanese": 0.8875, - "global-mmlu-lite/Korean": 0.915, - "global-mmlu-lite/Portuguese": 0.8875, - "global-mmlu-lite/Spanish": 0.905, - "global-mmlu-lite/Swahili": 0.865, - "global-mmlu-lite/Yoruba": 0.9125, - "global-mmlu-lite/Chinese": 0.895, - "global-mmlu-lite/Burmese": 0.915, - "helm_capabilities/Mean score": 0.807, - "helm_capabilities/MMLU-Pro": 0.863, - "helm_capabilities/GPQA": 0.791, - "helm_capabilities/IFEval": 0.875, - "helm_capabilities/WildBench": 0.857, - "helm_capabilities/Omni-MATH": 0.647, - "livecodebenchpro/Hard Problems": 0.0423, - "livecodebenchpro/Medium Problems": 0.4085, - "livecodebenchpro/Easy Problems": 0.9014 - } - }, - { - "id": "openai/gpt-5-codex", - "name": "GPT-5-Codex", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "terminal-bench-2.0/terminal-bench-2.0": 44.3 - } - }, - { - "id": "openai/gpt-5-mini", - "name": "GPT-5-Mini", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "terminal-bench-2.0/terminal-bench-2.0": 31.9 - } - }, - { - "id": "openai/gpt-5-mini-2025-08-07", - "name": "GPT-5 mini 2025-08-07", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.819, - "helm_capabilities/MMLU-Pro": 0.835, - "helm_capabilities/GPQA": 0.756, - "helm_capabilities/IFEval": 0.927, - "helm_capabilities/WildBench": 0.855, - "helm_capabilities/Omni-MATH": 0.722 - } - }, - { - "id": "openai/gpt-5-mini-2025-08-07-fc", - "name": "GPT-5-mini-2025-08-07 (FC)", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 17.0, - "bfcl/bfcl.overall.overall_accuracy": 55.46, - "bfcl/bfcl.overall.total_cost_usd": 22.18, - "bfcl/bfcl.overall.latency_mean_s": 8.32, - "bfcl/bfcl.overall.latency_std_s": 17.35, - "bfcl/bfcl.overall.latency_p95_s": 19.8, - "bfcl/bfcl.non_live.ast_accuracy": 69.85, - "bfcl/bfcl.non_live.simple_ast_accuracy": 59.92, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 69.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 80.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 70.5, - "bfcl/bfcl.live.live_accuracy": 58.62, - "bfcl/bfcl.live.live_simple_ast_accuracy": 62.02, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 58.02, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 62.5, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 45.83, - "bfcl/bfcl.multi_turn.accuracy": 27.5, - "bfcl/bfcl.multi_turn.base_accuracy": 36.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 17.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 23.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 33.0, - "bfcl/bfcl.web_search.accuracy": 82.0, - "bfcl/bfcl.web_search.base_accuracy": 87.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 77.0, - "bfcl/bfcl.memory.accuracy": 44.3, - "bfcl/bfcl.memory.kv_accuracy": 36.77, - "bfcl/bfcl.memory.vector_accuracy": 43.87, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 52.26, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 62.5, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 91.01 - } - }, - { - "id": "openai/gpt-5-mini-2025-08-07-prompt", - "name": "GPT-5-mini-2025-08-07 (Prompt)", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 77.0, - "bfcl/bfcl.overall.overall_accuracy": 27.83, - "bfcl/bfcl.overall.total_cost_usd": 82.74, - "bfcl/bfcl.overall.latency_mean_s": 8.89, - "bfcl/bfcl.overall.latency_std_s": 11.08, - "bfcl/bfcl.overall.latency_p95_s": 19.72, - "bfcl/bfcl.non_live.ast_accuracy": 68.04, - "bfcl/bfcl.non_live.simple_ast_accuracy": 59.17, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 72.5, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 71.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 69.0, - "bfcl/bfcl.live.live_accuracy": 62.55, - "bfcl/bfcl.live.live_simple_ast_accuracy": 69.77, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 61.16, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 75.0, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 37.5, - "bfcl/bfcl.multi_turn.accuracy": 5.5, - "bfcl/bfcl.multi_turn.base_accuracy": 5.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 5.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 4.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 7.0, - "bfcl/bfcl.web_search.accuracy": 8.5, - "bfcl/bfcl.web_search.base_accuracy": 11.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 6.0, - "bfcl/bfcl.memory.accuracy": 29.25, - "bfcl/bfcl.memory.kv_accuracy": 19.35, - "bfcl/bfcl.memory.vector_accuracy": 29.68, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 38.71, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 93.75, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 55.71, - "bfcl/bfcl.format_sensitivity.max_delta": 16.0, - "bfcl/bfcl.format_sensitivity.stddev": 3.78 - } - }, - { - "id": "openai/gpt-5-nano", - "name": "GPT-5-Nano", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "terminal-bench-2.0/terminal-bench-2.0": 7.0 - } - }, - { - "id": "openai/gpt-5-nano-2025-08-07", - "name": "GPT-5 nano 2025-08-07", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.748, - "helm_capabilities/MMLU-Pro": 0.778, - "helm_capabilities/GPQA": 0.679, - "helm_capabilities/IFEval": 0.932, - "helm_capabilities/WildBench": 0.806, - "helm_capabilities/Omni-MATH": 0.547 - } - }, - { - "id": "openai/gpt-5-nano-2025-08-07-fc", - "name": "GPT-5-nano-2025-08-07 (FC)", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 24.0, - "bfcl/bfcl.overall.overall_accuracy": 51.45, - "bfcl/bfcl.overall.total_cost_usd": 8.79, - "bfcl/bfcl.overall.latency_mean_s": 10.36, - "bfcl/bfcl.overall.latency_std_s": 10.37, - "bfcl/bfcl.overall.latency_p95_s": 23.56, - "bfcl/bfcl.non_live.ast_accuracy": 68.0, - "bfcl/bfcl.non_live.simple_ast_accuracy": 57.0, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 64.5, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 79.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 71.5, - "bfcl/bfcl.live.live_accuracy": 59.44, - "bfcl/bfcl.live.live_simple_ast_accuracy": 58.91, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 59.83, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 50.0, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 54.17, - "bfcl/bfcl.multi_turn.accuracy": 34.5, - "bfcl/bfcl.multi_turn.base_accuracy": 44.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 23.5, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 32.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 38.0, - "bfcl/bfcl.web_search.accuracy": 72.5, - "bfcl/bfcl.web_search.base_accuracy": 74.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 71.0, - "bfcl/bfcl.memory.accuracy": 24.73, - "bfcl/bfcl.memory.kv_accuracy": 18.06, - "bfcl/bfcl.memory.vector_accuracy": 27.1, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 29.03, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 75.0, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 89.1 - } - }, - { - "id": "openai/gpt-5-nano-2025-08-07-prompt", - "name": "GPT-5-nano-2025-08-07 (Prompt)", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 79.0, - "bfcl/bfcl.overall.overall_accuracy": 27.55, - "bfcl/bfcl.overall.total_cost_usd": 21.47, - "bfcl/bfcl.overall.latency_mean_s": 10.67, - "bfcl/bfcl.overall.latency_std_s": 7.68, - "bfcl/bfcl.overall.latency_p95_s": 23.28, - "bfcl/bfcl.non_live.ast_accuracy": 80.81, - "bfcl/bfcl.non_live.simple_ast_accuracy": 69.25, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 86.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 87.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 80.5, - "bfcl/bfcl.live.live_accuracy": 70.69, - "bfcl/bfcl.live.live_simple_ast_accuracy": 76.36, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 69.71, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 68.75, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 54.17, - "bfcl/bfcl.multi_turn.accuracy": 0.75, - "bfcl/bfcl.multi_turn.base_accuracy": 1.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 1.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 0.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 1.0, - "bfcl/bfcl.web_search.accuracy": 13.5, - "bfcl/bfcl.web_search.base_accuracy": 10.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 17.0, - "bfcl/bfcl.memory.accuracy": 24.52, - "bfcl/bfcl.memory.kv_accuracy": 20.65, - "bfcl/bfcl.memory.vector_accuracy": 31.61, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 21.29, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 93.75, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 45.75, - "bfcl/bfcl.format_sensitivity.max_delta": 8.5, - "bfcl/bfcl.format_sensitivity.stddev": 2.57 - } - }, - { - "id": "openai/gpt-5.1", - "name": "GPT-5.1", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "terminal-bench-2.0/terminal-bench-2.0": 47.6 - } - }, - { - "id": "openai/gpt-5.1-codex", - "name": "GPT-5.1-Codex", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "terminal-bench-2.0/terminal-bench-2.0": 53.5 - } - }, - { - "id": "openai/gpt-5.1-codex-max", - "name": "GPT-5.1-Codex-Max", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "terminal-bench-2.0/terminal-bench-2.0": 60.4 - } - }, - { - "id": "openai/gpt-5.1-codex-mini", - "name": "GPT-5.1-Codex-Mini", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "terminal-bench-2.0/terminal-bench-2.0": 43.1 - } - }, - { - "id": "openai/gpt-5.2", - "name": "GPT-5.2", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "terminal-bench-2.0/terminal-bench-2.0": 64.9 - } - }, - { - "id": "openai/gpt-5.2-2025-12-11", - "name": "gpt-5.2-2025-12-11", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "appworld_test_normal/appworld/test_normal": 0.0, - "browsecompplus/browsecompplus": 0.26, - "livecodebenchpro/Hard Problems": 0.1594, - "livecodebenchpro/Medium Problems": 0.5211, - "livecodebenchpro/Easy Problems": 0.9014, - "swe-bench/swe-bench": 0.57, - "tau-bench-2_airline/tau-bench-2/airline": 0.54, - "tau-bench-2_retail/tau-bench-2/retail": 0.68, - "tau-bench-2_telecom/tau-bench-2/telecom": 0.5354 - } - }, - { - "id": "openai/gpt-5.2-codex", - "name": "GPT-5.2-Codex", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "terminal-bench-2.0/terminal-bench-2.0": 66.5 - } - }, - { - "id": "openai/gpt-5.3-codex", - "name": "GPT-5.3-Codex", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "terminal-bench-2.0/terminal-bench-2.0": 74.6 - } - }, - { - "id": "openai/GPT-J-6B", - "name": "GPT-J 6B", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.273, - "helm_classic/MMLU": 0.249, - "helm_classic/BoolQ": 0.649, - "helm_classic/NarrativeQA": 0.545, - "helm_classic/NaturalQuestions (open-book)": 0.559, - "helm_classic/QuAC": 0.33, - "helm_classic/HellaSwag": 0.663, - "helm_classic/OpenbookQA": 0.514, - "helm_classic/TruthfulQA": 0.199, - "helm_classic/MS MARCO (TREC)": 0.345, - "helm_classic/CNN/DailyMail": 0.131, - "helm_classic/XSUM": 0.096, - "helm_classic/IMDB": 0.939, - "helm_classic/CivilComments": 0.52, - "helm_classic/RAFT": 0.619 - } - }, - { - "id": "openai/GPT-NeoX-20B", - "name": "GPT-NeoX 20B", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.351, - "helm_classic/MMLU": 0.276, - "helm_classic/BoolQ": 0.683, - "helm_classic/NarrativeQA": 0.599, - "helm_classic/NaturalQuestions (open-book)": 0.596, - "helm_classic/QuAC": 0.326, - "helm_classic/HellaSwag": 0.718, - "helm_classic/OpenbookQA": 0.524, - "helm_classic/TruthfulQA": 0.216, - "helm_classic/MS MARCO (TREC)": 0.398, - "helm_classic/CNN/DailyMail": 0.123, - "helm_classic/XSUM": 0.102, - "helm_classic/IMDB": 0.948, - "helm_classic/CivilComments": 0.516, - "helm_classic/RAFT": 0.505 - } - }, - { - "id": "openai/gpt-oss-120b", - "name": "GPT-OSS-120B", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.77, - "helm_capabilities/MMLU-Pro": 0.795, - "helm_capabilities/GPQA": 0.684, - "helm_capabilities/IFEval": 0.836, - "helm_capabilities/WildBench": 0.845, - "helm_capabilities/Omni-MATH": 0.688, - "livecodebenchpro/Hard Problems": 0.0, - "livecodebenchpro/Medium Problems": 0.11267605633802817, - "livecodebenchpro/Easy Problems": 0.6619718309859155, - "terminal-bench-2.0/terminal-bench-2.0": 18.7 - } - }, - { - "id": "openai/gpt-oss-20b", - "name": "GPT-OSS-20B", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.674, - "helm_capabilities/MMLU-Pro": 0.74, - "helm_capabilities/GPQA": 0.594, - "helm_capabilities/IFEval": 0.732, - "helm_capabilities/WildBench": 0.737, - "helm_capabilities/Omni-MATH": 0.565, - "livecodebenchpro/Hard Problems": 0.0, - "livecodebenchpro/Medium Problems": 0.056338028169014086, - "livecodebenchpro/Easy Problems": 0.5070422535211268, - "terminal-bench-2.0/terminal-bench-2.0": 3.4 - } - }, - { - "id": "openai/o3", - "name": "o3", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "ace/Overall Score": 0.529, - "ace/Gaming Score": 0.585, - "ace/Shopping Score": 0.45, - "apex-v1/Big Law Score": 0.76 - } - }, - { - "id": "openai/o3 Pro", - "name": "o3 Pro", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "ace/Overall Score": 0.552, - "ace/DIY Score": 0.54, - "ace/Food Score": 0.6, - "ace/Gaming Score": 0.613, - "ace/Shopping Score": 0.45 - } - }, - { - "id": "openai/o3-2025-04-16", - "name": "o3-2025-04-16", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.811, - "helm_capabilities/MMLU-Pro": 0.859, - "helm_capabilities/GPQA": 0.753, - "helm_capabilities/IFEval": 0.869, - "helm_capabilities/WildBench": 0.861, - "helm_capabilities/Omni-MATH": 0.714, - "livecodebenchpro/Hard Problems": 0.0, - "livecodebenchpro/Medium Problems": 0.22535211267605634, - "livecodebenchpro/Easy Problems": 0.7183098591549296 - } - }, - { - "id": "openai/o3-2025-04-16-fc", - "name": "o3-2025-04-16 (FC)", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 30.0, - "bfcl/bfcl.overall.overall_accuracy": 48.56, - "bfcl/bfcl.overall.total_cost_usd": 133.45, - "bfcl/bfcl.overall.latency_mean_s": 3.5, - "bfcl/bfcl.overall.latency_std_s": 8.69, - "bfcl/bfcl.overall.latency_p95_s": 8.39, - "bfcl/bfcl.non_live.ast_accuracy": 40.38, - "bfcl/bfcl.non_live.simple_ast_accuracy": 74.5, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 87.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 0.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 0.0, - "bfcl/bfcl.live.live_accuracy": 66.17, - "bfcl/bfcl.live.live_simple_ast_accuracy": 70.54, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 67.62, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 0.0, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 0.0, - "bfcl/bfcl.multi_turn.accuracy": 14.75, - "bfcl/bfcl.multi_turn.base_accuracy": 16.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 11.5, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 14.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 16.5, - "bfcl/bfcl.web_search.accuracy": 77.0, - "bfcl/bfcl.web_search.base_accuracy": 79.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 75.0, - "bfcl/bfcl.memory.accuracy": 47.31, - "bfcl/bfcl.memory.kv_accuracy": 24.52, - "bfcl/bfcl.memory.vector_accuracy": 44.52, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 72.9, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 81.25, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 86.13 - } - }, - { - "id": "openai/o3-2025-04-16-prompt", - "name": "o3-2025-04-16 (Prompt)", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 8.0, - "bfcl/bfcl.overall.overall_accuracy": 63.05, - "bfcl/bfcl.overall.total_cost_usd": 234.64, - "bfcl/bfcl.overall.latency_mean_s": 4.83, - "bfcl/bfcl.overall.latency_std_s": 7.01, - "bfcl/bfcl.overall.latency_p95_s": 11.7, - "bfcl/bfcl.non_live.ast_accuracy": 81.94, - "bfcl/bfcl.non_live.simple_ast_accuracy": 74.25, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 89.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 86.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 78.0, - "bfcl/bfcl.live.live_accuracy": 73.21, - "bfcl/bfcl.live.live_simple_ast_accuracy": 83.33, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 70.75, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 75.0, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 70.83, - "bfcl/bfcl.multi_turn.accuracy": 62.25, - "bfcl/bfcl.multi_turn.base_accuracy": 68.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 63.5, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 54.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 63.0, - "bfcl/bfcl.web_search.accuracy": 50.5, - "bfcl/bfcl.web_search.base_accuracy": 51.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 50.0, - "bfcl/bfcl.memory.accuracy": 51.83, - "bfcl/bfcl.memory.kv_accuracy": 33.55, - "bfcl/bfcl.memory.vector_accuracy": 50.32, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 71.61, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 93.75, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 83.98, - "bfcl/bfcl.format_sensitivity.max_delta": 8.5, - "bfcl/bfcl.format_sensitivity.stddev": 2.75 - } - }, - { - "id": "openai/o3-mini-2025-01-31", - "name": "o3-mini-2025-01-31", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "global-mmlu-lite/Global MMLU Lite": 0.78, - "global-mmlu-lite/Culturally Sensitive": 0.765, - "global-mmlu-lite/Culturally Agnostic": 0.795, - "global-mmlu-lite/Arabic": 0.7725, - "global-mmlu-lite/English": 0.8025, - "global-mmlu-lite/Bengali": 0.77, - "global-mmlu-lite/German": 0.7525, - "global-mmlu-lite/French": 0.74, - "global-mmlu-lite/Hindi": 0.7525, - "global-mmlu-lite/Indonesian": 0.7425, - "global-mmlu-lite/Italian": 0.8, - "global-mmlu-lite/Japanese": 0.81, - "global-mmlu-lite/Korean": 0.8075, - "global-mmlu-lite/Portuguese": 0.7975, - "global-mmlu-lite/Spanish": 0.775, - "global-mmlu-lite/Swahili": 0.765, - "global-mmlu-lite/Yoruba": 0.7725, - "global-mmlu-lite/Chinese": 0.8125, - "global-mmlu-lite/Burmese": 0.8075 - } - }, - { - "id": "openai/o4-mini-2025-04-16", - "name": "o4-mini-2025-04-16", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "global-mmlu-lite/Global MMLU Lite": 0.8705, - "global-mmlu-lite/Culturally Sensitive": 0.8503, - "global-mmlu-lite/Culturally Agnostic": 0.8906, - "global-mmlu-lite/Arabic": 0.865, - "global-mmlu-lite/English": 0.8675, - "global-mmlu-lite/Bengali": 0.8875, - "global-mmlu-lite/German": 0.8775, - "global-mmlu-lite/French": 0.87, - "global-mmlu-lite/Hindi": 0.87, - "global-mmlu-lite/Indonesian": 0.8675, - "global-mmlu-lite/Italian": 0.855, - "global-mmlu-lite/Japanese": 0.885, - "global-mmlu-lite/Korean": 0.88, - "global-mmlu-lite/Portuguese": 0.88, - "global-mmlu-lite/Spanish": 0.855, - "global-mmlu-lite/Swahili": 0.8525, - "global-mmlu-lite/Yoruba": 0.8525, - "global-mmlu-lite/Chinese": 0.89, - "global-mmlu-lite/Burmese": 0.8725, - "helm_capabilities/Mean score": 0.812, - "helm_capabilities/MMLU-Pro": 0.82, - "helm_capabilities/GPQA": 0.735, - "helm_capabilities/IFEval": 0.929, - "helm_capabilities/WildBench": 0.854, - "helm_capabilities/Omni-MATH": 0.72, - "livecodebenchpro/Hard Problems": 0.0143, - "livecodebenchpro/Medium Problems": 0.2923, - "livecodebenchpro/Easy Problems": 0.8571 - } - }, - { - "id": "openai/o4-mini-2025-04-16-fc", - "name": "o4-mini-2025-04-16 (FC)", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 21.0, - "bfcl/bfcl.overall.overall_accuracy": 53.24, - "bfcl/bfcl.overall.total_cost_usd": 81.91, - "bfcl/bfcl.overall.latency_mean_s": 3.71, - "bfcl/bfcl.overall.latency_std_s": 7.18, - "bfcl/bfcl.overall.latency_p95_s": 9.33, - "bfcl/bfcl.non_live.ast_accuracy": 37.73, - "bfcl/bfcl.non_live.simple_ast_accuracy": 66.92, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 84.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 0.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 0.0, - "bfcl/bfcl.live.live_accuracy": 66.1, - "bfcl/bfcl.live.live_simple_ast_accuracy": 69.38, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 67.81, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 0.0, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 0.0, - "bfcl/bfcl.multi_turn.accuracy": 41.75, - "bfcl/bfcl.multi_turn.base_accuracy": 51.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 30.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 40.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 45.5, - "bfcl/bfcl.web_search.accuracy": 75.5, - "bfcl/bfcl.web_search.base_accuracy": 75.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 76.0, - "bfcl/bfcl.memory.accuracy": 34.19, - "bfcl/bfcl.memory.kv_accuracy": 19.35, - "bfcl/bfcl.memory.vector_accuracy": 24.52, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 58.71, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 81.25, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 83.91 - } - }, - { - "id": "openai/o4-mini-2025-04-16-prompt", - "name": "o4-mini-2025-04-16 (Prompt)", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 28.0, - "bfcl/bfcl.overall.overall_accuracy": 50.26, - "bfcl/bfcl.overall.total_cost_usd": 133.63, - "bfcl/bfcl.overall.latency_mean_s": 4.47, - "bfcl/bfcl.overall.latency_std_s": 5.19, - "bfcl/bfcl.overall.latency_p95_s": 10.19, - "bfcl/bfcl.non_live.ast_accuracy": 81.29, - "bfcl/bfcl.non_live.simple_ast_accuracy": 72.67, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 88.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 84.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 80.0, - "bfcl/bfcl.live.live_accuracy": 70.76, - "bfcl/bfcl.live.live_simple_ast_accuracy": 79.46, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 68.76, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 75.0, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 62.5, - "bfcl/bfcl.multi_turn.accuracy": 16.62, - "bfcl/bfcl.multi_turn.base_accuracy": 16.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 18.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 17.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 14.5, - "bfcl/bfcl.web_search.accuracy": 71.5, - "bfcl/bfcl.web_search.base_accuracy": 73.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 70.0, - "bfcl/bfcl.memory.accuracy": 35.27, - "bfcl/bfcl.memory.kv_accuracy": 22.58, - "bfcl/bfcl.memory.vector_accuracy": 25.16, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 58.06, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 81.25, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 87.16, - "bfcl/bfcl.format_sensitivity.max_delta": 9.5, - "bfcl/bfcl.format_sensitivity.stddev": 2.6 - } - }, - { - "id": "openai/text-ada-001", - "name": "text-ada-001", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.107, - "helm_classic/MMLU": 0.238, - "helm_classic/BoolQ": 0.464, - "helm_classic/NarrativeQA": 0.238, - "helm_classic/NaturalQuestions (open-book)": 0.149, - "helm_classic/QuAC": 0.176, - "helm_classic/HellaSwag": 0.429, - "helm_classic/OpenbookQA": 0.346, - "helm_classic/TruthfulQA": 0.232, - "helm_classic/MS MARCO (TREC)": 0.302, - "helm_classic/CNN/DailyMail": 0.136, - "helm_classic/XSUM": 0.034, - "helm_classic/IMDB": 0.822, - "helm_classic/CivilComments": 0.503, - "helm_classic/RAFT": 0.406 - } - }, - { - "id": "openai/text-babbage-001", - "name": "text-babbage-001", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.229, - "helm_classic/MMLU": 0.229, - "helm_classic/BoolQ": 0.451, - "helm_classic/NarrativeQA": 0.429, - "helm_classic/NaturalQuestions (open-book)": 0.33, - "helm_classic/QuAC": 0.284, - "helm_classic/HellaSwag": 0.561, - "helm_classic/OpenbookQA": 0.452, - "helm_classic/TruthfulQA": 0.233, - "helm_classic/MS MARCO (TREC)": 0.449, - "helm_classic/CNN/DailyMail": 0.151, - "helm_classic/XSUM": 0.046, - "helm_classic/IMDB": 0.913, - "helm_classic/CivilComments": 0.499, - "helm_classic/RAFT": 0.509 - } - }, - { - "id": "openai/text-curie-001", - "name": "text-curie-001", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.36, - "helm_classic/MMLU": 0.237, - "helm_classic/BoolQ": 0.62, - "helm_classic/NarrativeQA": 0.582, - "helm_classic/NaturalQuestions (open-book)": 0.571, - "helm_classic/QuAC": 0.358, - "helm_classic/HellaSwag": 0.676, - "helm_classic/OpenbookQA": 0.514, - "helm_classic/TruthfulQA": 0.257, - "helm_classic/MS MARCO (TREC)": 0.507, - "helm_classic/CNN/DailyMail": 0.152, - "helm_classic/XSUM": 0.076, - "helm_classic/IMDB": 0.923, - "helm_classic/CivilComments": 0.537, - "helm_classic/RAFT": 0.489 - } - }, - { - "id": "openai/text-davinci-002", - "name": "GPT-3.5 text-davinci-002", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.905, - "helm_classic/MMLU": 0.568, - "helm_classic/BoolQ": 0.877, - "helm_classic/NarrativeQA": 0.727, - "helm_classic/NaturalQuestions (open-book)": 0.713, - "helm_classic/QuAC": 0.445, - "helm_classic/HellaSwag": 0.815, - "helm_classic/OpenbookQA": 0.594, - "helm_classic/TruthfulQA": 0.61, - "helm_classic/MS MARCO (TREC)": 0.664, - "helm_classic/CNN/DailyMail": 0.153, - "helm_classic/XSUM": 0.144, - "helm_classic/IMDB": 0.948, - "helm_classic/CivilComments": 0.668, - "helm_classic/RAFT": 0.733, - "helm_lite/Mean win rate": 0.336, - "helm_lite/NarrativeQA": 0.719, - "helm_lite/NaturalQuestions (closed-book)": 0.394, - "helm_lite/OpenbookQA": 0.796, - "helm_lite/MMLU": 0.568, - "helm_lite/MATH": 0.428, - "helm_lite/GSM8K": 0.479, - "helm_lite/LegalBench": 0.58, - "helm_lite/MedQA": 0.525, - "helm_lite/WMT 2014": 0.174 - } - }, - { - "id": "openai/text-davinci-003", - "name": "GPT-3.5 text-davinci-003", - "developer": "OpenAI", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.872, - "helm_classic/MMLU": 0.569, - "helm_classic/BoolQ": 0.881, - "helm_classic/NarrativeQA": 0.727, - "helm_classic/NaturalQuestions (open-book)": 0.77, - "helm_classic/QuAC": 0.525, - "helm_classic/HellaSwag": 0.822, - "helm_classic/OpenbookQA": 0.646, - "helm_classic/TruthfulQA": 0.593, - "helm_classic/MS MARCO (TREC)": 0.644, - "helm_classic/CNN/DailyMail": 0.156, - "helm_classic/XSUM": 0.124, - "helm_classic/IMDB": 0.848, - "helm_classic/CivilComments": 0.684, - "helm_classic/RAFT": 0.759, - "helm_lite/Mean win rate": 0.439, - "helm_lite/NarrativeQA": 0.731, - "helm_lite/NaturalQuestions (closed-book)": 0.413, - "helm_lite/OpenbookQA": 0.828, - "helm_lite/MMLU": 0.555, - "helm_lite/MATH": 0.449, - "helm_lite/GSM8K": 0.615, - "helm_lite/LegalBench": 0.622, - "helm_lite/MedQA": 0.531, - "helm_lite/WMT 2014": 0.191 - } - }, - { - "id": "OpenAssistant/oasst-rm-2-pythia-6.9b-epoch-1", - "name": "OpenAssistant/oasst-rm-2-pythia-6.9b-epoch-1", - "developer": "OpenAssistant", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.2653, - "reward-bench/Chat": 0.9246, - "reward-bench/Chat Hard": 0.3728, - "reward-bench/Safety": 0.3289, - "reward-bench/Reasoning": 0.5855, - "reward-bench/Prior Sets (0.5 weight)": 0.6801, - "reward-bench/Factuality": 0.3979, - "reward-bench/Precise IF": 0.2875, - "reward-bench/Math": 0.377, - "reward-bench/Focus": 0.1535, - "reward-bench/Ties": 0.047 - } - }, - { - "id": "OpenAssistant/oasst-rm-2.1-pythia-1.4b-epoch-2.5", - "name": "OpenAssistant/oasst-rm-2.1-pythia-1.4b-epoch-2.5", - "developer": "OpenAssistant", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.2648, - "reward-bench/Chat": 0.8855, - "reward-bench/Chat Hard": 0.4868, - "reward-bench/Safety": 0.3244, - "reward-bench/Reasoning": 0.7752, - "reward-bench/Prior Sets (0.5 weight)": 0.6533, - "reward-bench/Factuality": 0.3179, - "reward-bench/Precise IF": 0.2625, - "reward-bench/Math": 0.3934, - "reward-bench/Focus": 0.2707, - "reward-bench/Ties": 0.0198 - } - }, - { - "id": "OpenAssistant/oasst-sft-1-pythia-12b", - "name": "oasst-sft-1-pythia-12b", - "developer": "OpenAssistant", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1055, - "hfopenllm_v2/BBH": 0.3147, - "hfopenllm_v2/MATH Level 5": 0.0151, - "hfopenllm_v2/GPQA": 0.2576, - "hfopenllm_v2/MUSR": 0.3327, - "hfopenllm_v2/MMLU-PRO": 0.1113 - } - }, - { - "id": "OpenAssistant/reward-model-deberta-v3-large-v2", - "name": "OpenAssistant/reward-model-deberta-v3-large-v2", - "developer": "OpenAssistant", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6126, - "reward-bench/Factuality": 0.3853, - "reward-bench/Precise IF": 0.2687, - "reward-bench/Math": 0.5027, - "reward-bench/Safety": 0.7338, - "reward-bench/Focus": 0.2768, - "reward-bench/Ties": 0.12, - "reward-bench/Chat": 0.8939, - "reward-bench/Chat Hard": 0.4518, - "reward-bench/Reasoning": 0.3855, - "reward-bench/Prior Sets (0.5 weight)": 0.5836 - } - }, - { - "id": "openbmb/Eurus-7b-kto", - "name": "openbmb/Eurus-7b-kto", - "developer": "openbmb", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.69, - "reward-bench/Chat": 0.9525, - "reward-bench/Chat Hard": 0.5373, - "reward-bench/Safety": 0.6054, - "reward-bench/Reasoning": 0.7467, - "reward-bench/Prior Sets (0.5 weight)": 0.5261 - } - }, - { - "id": "openbmb/Eurus-RM-7b", - "name": "openbmb/Eurus-RM-7b", - "developer": "openbmb", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.8159, - "reward-bench/Factuality": 0.6, - "reward-bench/Precise IF": 0.3438, - "reward-bench/Math": 0.5683, - "reward-bench/Safety": 0.8135, - "reward-bench/Focus": 0.7475, - "reward-bench/Ties": 0.5972, - "reward-bench/Chat": 0.9804, - "reward-bench/Chat Hard": 0.6557, - "reward-bench/Reasoning": 0.8633, - "reward-bench/Prior Sets (0.5 weight)": 0.7172 - } - }, - { - "id": "openbmb/MiniCPM-2B-dpo-fp32", - "name": "openbmb/MiniCPM-2B-dpo-fp32", - "developer": "openbmb", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.673, - "reward-bench/Chat": 0.8911, - "reward-bench/Chat Hard": 0.4934, - "reward-bench/Safety": 0.573, - "reward-bench/Reasoning": 0.8233, - "reward-bench/Prior Sets (0.5 weight)": 0.4958 - } - }, - { - "id": "openbmb/MiniCPM-S-1B-sft-llama-format", - "name": "MiniCPM-S-1B-sft-llama-format", - "developer": "openbmb", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3329, - "hfopenllm_v2/BBH": 0.3049, - "hfopenllm_v2/MATH Level 5": 0.031, - "hfopenllm_v2/GPQA": 0.271, - "hfopenllm_v2/MUSR": 0.3317, - "hfopenllm_v2/MMLU-PRO": 0.1858 - } - }, - { - "id": "openbmb/minicpm3-4b-fc-fc", - "name": "MiniCPM3-4B-FC (FC)", - "developer": "openbmb", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 86.0, - "bfcl/bfcl.overall.overall_accuracy": 25.55, - "bfcl/bfcl.overall.total_cost_usd": 54.05, - "bfcl/bfcl.overall.latency_mean_s": 118.62, - "bfcl/bfcl.overall.latency_std_s": 143.98, - "bfcl/bfcl.overall.latency_p95_s": 388.67, - "bfcl/bfcl.non_live.ast_accuracy": 81.75, - "bfcl/bfcl.non_live.simple_ast_accuracy": 70.5, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 92.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 84.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 80.5, - "bfcl/bfcl.live.live_accuracy": 65.21, - "bfcl/bfcl.live.live_simple_ast_accuracy": 73.26, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 63.53, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 50.0, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 62.5, - "bfcl/bfcl.multi_turn.accuracy": 3.88, - "bfcl/bfcl.multi_turn.base_accuracy": 6.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 2.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 4.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 2.5, - "bfcl/bfcl.web_search.accuracy": 0.0, - "bfcl/bfcl.web_search.base_accuracy": 0.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 0.0, - "bfcl/bfcl.memory.accuracy": 12.04, - "bfcl/bfcl.memory.kv_accuracy": 9.68, - "bfcl/bfcl.memory.vector_accuracy": 15.48, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 10.97, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 68.75, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 72.84 - } - }, - { - "id": "openbmb/minicpm3-4b-prompt", - "name": "MiniCPM3-4B (Prompt)", - "developer": "openbmb", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 97.0, - "bfcl/bfcl.overall.overall_accuracy": 22.08, - "bfcl/bfcl.overall.total_cost_usd": 29.83, - "bfcl/bfcl.overall.latency_mean_s": 31.18, - "bfcl/bfcl.overall.latency_std_s": 35.61, - "bfcl/bfcl.overall.latency_p95_s": 102.02, - "bfcl/bfcl.non_live.ast_accuracy": 70.54, - "bfcl/bfcl.non_live.simple_ast_accuracy": 66.17, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 77.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 70.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 69.0, - "bfcl/bfcl.live.live_accuracy": 43.15, - "bfcl/bfcl.live.live_simple_ast_accuracy": 47.67, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 42.17, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 43.75, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 37.5, - "bfcl/bfcl.multi_turn.accuracy": 3.5, - "bfcl/bfcl.multi_turn.base_accuracy": 4.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 4.5, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 2.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 3.0, - "bfcl/bfcl.web_search.accuracy": 2.0, - "bfcl/bfcl.web_search.base_accuracy": 2.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 2.0, - "bfcl/bfcl.memory.accuracy": 9.46, - "bfcl/bfcl.memory.kv_accuracy": 8.39, - "bfcl/bfcl.memory.vector_accuracy": 10.32, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 9.68, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 56.25, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 73.71, - "bfcl/bfcl.format_sensitivity.max_delta": 68.0, - "bfcl/bfcl.format_sensitivity.stddev": 16.55 - } - }, - { - "id": "openbmb/UltraRM-13b", - "name": "openbmb/UltraRM-13b", - "developer": "openbmb", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.4683, - "reward-bench/Chat": 0.9637, - "reward-bench/Chat Hard": 0.5548, - "reward-bench/Safety": 0.5089, - "reward-bench/Reasoning": 0.6244, - "reward-bench/Prior Sets (0.5 weight)": 0.7294, - "reward-bench/Factuality": 0.5063, - "reward-bench/Precise IF": 0.3312, - "reward-bench/Math": 0.5519, - "reward-bench/Focus": 0.6081, - "reward-bench/Ties": 0.3036 - } - }, - { - "id": "OpenBuddy/openbuddy-falcon3-10b-v24.2-131k", - "name": "openbuddy-falcon3-10b-v24.2-131k", - "developer": "OpenBuddy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5086, - "hfopenllm_v2/BBH": 0.6004, - "hfopenllm_v2/MATH Level 5": 0.213, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.4186, - "hfopenllm_v2/MMLU-PRO": 0.3834 - } - }, - { - "id": "OpenBuddy/openbuddy-llama3-70b-v21.2-32k", - "name": "openbuddy-llama3-70b-v21.2-32k", - "developer": "OpenBuddy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.701, - "hfopenllm_v2/BBH": 0.6507, - "hfopenllm_v2/MATH Level 5": 0.2032, - "hfopenllm_v2/GPQA": 0.3423, - "hfopenllm_v2/MUSR": 0.458, - "hfopenllm_v2/MMLU-PRO": 0.4832 - } - }, - { - "id": "OpenBuddy/openbuddy-llama3-8b-v21.1-8k", - "name": "openbuddy-llama3-8b-v21.1-8k", - "developer": "OpenBuddy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.557, - "hfopenllm_v2/BBH": 0.4788, - "hfopenllm_v2/MATH Level 5": 0.0431, - "hfopenllm_v2/GPQA": 0.271, - "hfopenllm_v2/MUSR": 0.3988, - "hfopenllm_v2/MMLU-PRO": 0.2955 - } - }, - { - "id": "OpenBuddy/openbuddy-llama3-8b-v21.2-32k", - "name": "openbuddy-llama3-8b-v21.2-32k", - "developer": "OpenBuddy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6192, - "hfopenllm_v2/BBH": 0.4856, - "hfopenllm_v2/MATH Level 5": 0.0785, - "hfopenllm_v2/GPQA": 0.2794, - "hfopenllm_v2/MUSR": 0.3779, - "hfopenllm_v2/MMLU-PRO": 0.3299 - } - }, - { - "id": "OpenBuddy/openbuddy-llama3.1-70b-v22.1-131k", - "name": "openbuddy-llama3.1-70b-v22.1-131k", - "developer": "OpenBuddy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7333, - "hfopenllm_v2/BBH": 0.6698, - "hfopenllm_v2/MATH Level 5": 0.395, - "hfopenllm_v2/GPQA": 0.375, - "hfopenllm_v2/MUSR": 0.463, - "hfopenllm_v2/MMLU-PRO": 0.5304 - } - }, - { - "id": "OpenBuddy/openbuddy-llama3.1-8b-v22.2-131k", - "name": "openbuddy-llama3.1-8b-v22.2-131k", - "developer": "OpenBuddy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6657, - "hfopenllm_v2/BBH": 0.5007, - "hfopenllm_v2/MATH Level 5": 0.1148, - "hfopenllm_v2/GPQA": 0.2794, - "hfopenllm_v2/MUSR": 0.4081, - "hfopenllm_v2/MMLU-PRO": 0.331 - } - }, - { - "id": "OpenBuddy/openbuddy-llama3.1-8b-v22.3-131k", - "name": "openbuddy-llama3.1-8b-v22.3-131k", - "developer": "OpenBuddy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5997, - "hfopenllm_v2/BBH": 0.5066, - "hfopenllm_v2/MATH Level 5": 0.1208, - "hfopenllm_v2/GPQA": 0.2794, - "hfopenllm_v2/MUSR": 0.4015, - "hfopenllm_v2/MMLU-PRO": 0.3277 - } - }, - { - "id": "OpenBuddy/openbuddy-llama3.2-1b-v23.1-131k", - "name": "openbuddy-llama3.2-1b-v23.1-131k", - "developer": "OpenBuddy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.359, - "hfopenllm_v2/BBH": 0.3267, - "hfopenllm_v2/MATH Level 5": 0.0249, - "hfopenllm_v2/GPQA": 0.2584, - "hfopenllm_v2/MUSR": 0.3342, - "hfopenllm_v2/MMLU-PRO": 0.184 - } - }, - { - "id": "OpenBuddy/openbuddy-llama3.2-3b-v23.2-131k", - "name": "openbuddy-llama3.2-3b-v23.2-131k", - "developer": "OpenBuddy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4319, - "hfopenllm_v2/BBH": 0.4073, - "hfopenllm_v2/MATH Level 5": 0.0264, - "hfopenllm_v2/GPQA": 0.276, - "hfopenllm_v2/MUSR": 0.3263, - "hfopenllm_v2/MMLU-PRO": 0.2479 - } - }, - { - "id": "OpenBuddy/openbuddy-llama3.3-70b-v24.1-131k", - "name": "openbuddy-llama3.3-70b-v24.1-131k", - "developer": "OpenBuddy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8121, - "hfopenllm_v2/BBH": 0.6858, - "hfopenllm_v2/MATH Level 5": 0.4411, - "hfopenllm_v2/GPQA": 0.4346, - "hfopenllm_v2/MUSR": 0.4869, - "hfopenllm_v2/MMLU-PRO": 0.5327 - } - }, - { - "id": "OpenBuddy/openbuddy-mixtral-7bx8-v18.1-32k", - "name": "openbuddy-mixtral-7bx8-v18.1-32k", - "developer": "OpenBuddy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5493, - "hfopenllm_v2/BBH": 0.4656, - "hfopenllm_v2/MATH Level 5": 0.108, - "hfopenllm_v2/GPQA": 0.3045, - "hfopenllm_v2/MUSR": 0.3831, - "hfopenllm_v2/MMLU-PRO": 0.3804 - } - }, - { - "id": "OpenBuddy/openbuddy-nemotron-70b-v23.1-131k", - "name": "openbuddy-nemotron-70b-v23.1-131k", - "developer": "OpenBuddy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7555, - "hfopenllm_v2/BBH": 0.6749, - "hfopenllm_v2/MATH Level 5": 0.321, - "hfopenllm_v2/GPQA": 0.3633, - "hfopenllm_v2/MUSR": 0.4538, - "hfopenllm_v2/MMLU-PRO": 0.5175 - } - }, - { - "id": "OpenBuddy/openbuddy-nemotron-70b-v23.2-131k", - "name": "openbuddy-nemotron-70b-v23.2-131k", - "developer": "OpenBuddy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7227, - "hfopenllm_v2/BBH": 0.6705, - "hfopenllm_v2/MATH Level 5": 0.3157, - "hfopenllm_v2/GPQA": 0.3599, - "hfopenllm_v2/MUSR": 0.4696, - "hfopenllm_v2/MMLU-PRO": 0.5121 - } - }, - { - "id": "OpenBuddy/openbuddy-qwen2.5llamaify-14b-v23.1-200k", - "name": "openbuddy-qwen2.5llamaify-14b-v23.1-200k", - "developer": "OpenBuddy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6309, - "hfopenllm_v2/BBH": 0.6013, - "hfopenllm_v2/MATH Level 5": 0.2538, - "hfopenllm_v2/GPQA": 0.3331, - "hfopenllm_v2/MUSR": 0.424, - "hfopenllm_v2/MMLU-PRO": 0.4673 - } - }, - { - "id": "OpenBuddy/openbuddy-qwen2.5llamaify-14b-v23.3-200k", - "name": "openbuddy-qwen2.5llamaify-14b-v23.3-200k", - "developer": "OpenBuddy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6131, - "hfopenllm_v2/BBH": 0.6081, - "hfopenllm_v2/MATH Level 5": 0.2311, - "hfopenllm_v2/GPQA": 0.3272, - "hfopenllm_v2/MUSR": 0.4346, - "hfopenllm_v2/MMLU-PRO": 0.4795 - } - }, - { - "id": "OpenBuddy/openbuddy-qwen2.5llamaify-7b-v23.1-200k", - "name": "openbuddy-qwen2.5llamaify-7b-v23.1-200k", - "developer": "OpenBuddy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5673, - "hfopenllm_v2/BBH": 0.5509, - "hfopenllm_v2/MATH Level 5": 0.1888, - "hfopenllm_v2/GPQA": 0.3146, - "hfopenllm_v2/MUSR": 0.4363, - "hfopenllm_v2/MMLU-PRO": 0.3948 - } - }, - { - "id": "OpenBuddy/openbuddy-qwq-32b-v24.1-200k", - "name": "openbuddy-qwq-32b-v24.1-200k", - "developer": "OpenBuddy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5937, - "hfopenllm_v2/BBH": 0.6798, - "hfopenllm_v2/MATH Level 5": 0.3739, - "hfopenllm_v2/GPQA": 0.3809, - "hfopenllm_v2/MUSR": 0.4849, - "hfopenllm_v2/MMLU-PRO": 0.549 - } - }, - { - "id": "OpenBuddy/openbuddy-qwq-32b-v24.2-200k", - "name": "openbuddy-qwq-32b-v24.2-200k", - "developer": "OpenBuddy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.597, - "hfopenllm_v2/BBH": 0.6772, - "hfopenllm_v2/MATH Level 5": 0.3776, - "hfopenllm_v2/GPQA": 0.3767, - "hfopenllm_v2/MUSR": 0.4718, - "hfopenllm_v2/MMLU-PRO": 0.5446 - } - }, - { - "id": "OpenBuddy/openbuddy-yi1.5-34b-v21.3-32k", - "name": "openbuddy-yi1.5-34b-v21.3-32k", - "developer": "OpenBuddy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.542, - "hfopenllm_v2/BBH": 0.6163, - "hfopenllm_v2/MATH Level 5": 0.1782, - "hfopenllm_v2/GPQA": 0.349, - "hfopenllm_v2/MUSR": 0.4439, - "hfopenllm_v2/MMLU-PRO": 0.4599 - } - }, - { - "id": "OpenBuddy/openbuddy-zero-14b-v22.3-32k", - "name": "openbuddy-zero-14b-v22.3-32k", - "developer": "OpenBuddy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3753, - "hfopenllm_v2/BBH": 0.486, - "hfopenllm_v2/MATH Level 5": 0.0937, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.4166, - "hfopenllm_v2/MMLU-PRO": 0.3187 - } - }, - { - "id": "OpenBuddy/openbuddy-zero-3b-v21.2-32k", - "name": "openbuddy-zero-3b-v21.2-32k", - "developer": "OpenBuddy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3802, - "hfopenllm_v2/BBH": 0.3935, - "hfopenllm_v2/MATH Level 5": 0.0189, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3566, - "hfopenllm_v2/MMLU-PRO": 0.2034 - } - }, - { - "id": "OpenBuddy/openbuddy-zero-56b-v21.2-32k", - "name": "openbuddy-zero-56b-v21.2-32k", - "developer": "OpenBuddy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5057, - "hfopenllm_v2/BBH": 0.6128, - "hfopenllm_v2/MATH Level 5": 0.1624, - "hfopenllm_v2/GPQA": 0.318, - "hfopenllm_v2/MUSR": 0.4305, - "hfopenllm_v2/MMLU-PRO": 0.4399 - } - }, - { - "id": "openchat/openchat-3.5-0106", - "name": "openchat-3.5-0106", - "developer": "openchat", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5967, - "hfopenllm_v2/BBH": 0.4617, - "hfopenllm_v2/MATH Level 5": 0.0763, - "hfopenllm_v2/GPQA": 0.3079, - "hfopenllm_v2/MUSR": 0.4254, - "hfopenllm_v2/MMLU-PRO": 0.3291 - } - }, - { - "id": "openchat/openchat-3.5-1210", - "name": "openchat-3.5-1210", - "developer": "openchat", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6037, - "hfopenllm_v2/BBH": 0.4535, - "hfopenllm_v2/MATH Level 5": 0.0785, - "hfopenllm_v2/GPQA": 0.3012, - "hfopenllm_v2/MUSR": 0.4414, - "hfopenllm_v2/MMLU-PRO": 0.3142 - } - }, - { - "id": "openchat/openchat-3.6-8b-20240522", - "name": "openchat-3.6-8b-20240522", - "developer": "openchat", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5343, - "hfopenllm_v2/BBH": 0.5338, - "hfopenllm_v2/MATH Level 5": 0.0997, - "hfopenllm_v2/GPQA": 0.318, - "hfopenllm_v2/MUSR": 0.3999, - "hfopenllm_v2/MMLU-PRO": 0.3229 - } - }, - { - "id": "openchat/openchat_3.5", - "name": "openchat_3.5", - "developer": "openchat", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5931, - "hfopenllm_v2/BBH": 0.4426, - "hfopenllm_v2/MATH Level 5": 0.0725, - "hfopenllm_v2/GPQA": 0.2987, - "hfopenllm_v2/MUSR": 0.4229, - "hfopenllm_v2/MMLU-PRO": 0.3153 - } - }, - { - "id": "openchat/openchat_v3.2", - "name": "openchat_v3.2", - "developer": "openchat", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2981, - "hfopenllm_v2/BBH": 0.4331, - "hfopenllm_v2/MATH Level 5": 0.0128, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.4336, - "hfopenllm_v2/MMLU-PRO": 0.2422 - } - }, - { - "id": "openchat/openchat_v3.2_super", - "name": "openchat_v3.2_super", - "developer": "openchat", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2862, - "hfopenllm_v2/BBH": 0.4221, - "hfopenllm_v2/MATH Level 5": 0.0211, - "hfopenllm_v2/GPQA": 0.2643, - "hfopenllm_v2/MUSR": 0.4161, - "hfopenllm_v2/MMLU-PRO": 0.2425 - } - }, - { - "id": "opencompass/CompassJudger-1-1.5B-Instruct", - "name": "opencompass/CompassJudger-1-1.5B-Instruct", - "developer": "opencompass", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7344, - "reward-bench/Chat": 0.9637, - "reward-bench/Chat Hard": 0.4923, - "reward-bench/Safety": 0.7818, - "reward-bench/Reasoning": 0.6999 - } - }, - { - "id": "opencompass/CompassJudger-1-14B-Instruct", - "name": "opencompass/CompassJudger-1-14B-Instruct", - "developer": "opencompass", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.8409, - "reward-bench/Chat": 0.9749, - "reward-bench/Chat Hard": 0.6228, - "reward-bench/Safety": 0.8392, - "reward-bench/Reasoning": 0.9268 - } - }, - { - "id": "opencompass/CompassJudger-1-32B-Instruct", - "name": "opencompass/CompassJudger-1-32B-Instruct", - "developer": "opencompass", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.8522, - "reward-bench/Chat": 0.9804, - "reward-bench/Chat Hard": 0.6513, - "reward-bench/Safety": 0.8527, - "reward-bench/Reasoning": 0.9244 - } - }, - { - "id": "opencompass/CompassJudger-1-7B-Instruct", - "name": "opencompass/CompassJudger-1-7B-Instruct", - "developer": "opencompass", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.8317, - "reward-bench/Chat": 0.9777, - "reward-bench/Chat Hard": 0.6096, - "reward-bench/Safety": 0.8446, - "reward-bench/Reasoning": 0.8948 - } - }, - { - "id": "OpenGenerativeAI/Bifrost", - "name": "Bifrost", - "developer": "OpenGenerativeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6348, - "hfopenllm_v2/BBH": 0.6849, - "hfopenllm_v2/MATH Level 5": 0.2545, - "hfopenllm_v2/GPQA": 0.3683, - "hfopenllm_v2/MUSR": 0.4598, - "hfopenllm_v2/MMLU-PRO": 0.516 - } - }, - { - "id": "OpenGenerativeAI/Bifrost-14B", - "name": "Bifrost-14B", - "developer": "OpenGenerativeAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6615, - "hfopenllm_v2/BBH": 0.6845, - "hfopenllm_v2/MATH Level 5": 0.2356, - "hfopenllm_v2/GPQA": 0.3792, - "hfopenllm_v2/MUSR": 0.4624, - "hfopenllm_v2/MMLU-PRO": 0.5074 - } - }, - { - "id": "OpenLeecher/llama3-8b-lima", - "name": "llama3-8b-lima", - "developer": "OpenLeecher", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4371, - "hfopenllm_v2/BBH": 0.4296, - "hfopenllm_v2/MATH Level 5": 0.0506, - "hfopenllm_v2/GPQA": 0.2383, - "hfopenllm_v2/MUSR": 0.3713, - "hfopenllm_v2/MMLU-PRO": 0.2626 - } - }, - { - "id": "OpenLLM-France/Lucie-7B", - "name": "Lucie-7B", - "developer": "OpenLLM-France", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2496, - "hfopenllm_v2/BBH": 0.3492, - "hfopenllm_v2/MATH Level 5": 0.0144, - "hfopenllm_v2/GPQA": 0.2727, - "hfopenllm_v2/MUSR": 0.3923, - "hfopenllm_v2/MMLU-PRO": 0.1498 - } - }, - { - "id": "OpenLLM-France/Lucie-7B-Instruct", - "name": "Lucie-7B-Instruct", - "developer": "OpenLLM-France", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2796, - "hfopenllm_v2/BBH": 0.3254, - "hfopenllm_v2/MATH Level 5": 0.0166, - "hfopenllm_v2/GPQA": 0.2794, - "hfopenllm_v2/MUSR": 0.3662, - "hfopenllm_v2/MMLU-PRO": 0.1556 - } - }, - { - "id": "OpenLLM-France/Lucie-7B-Instruct-human-data", - "name": "Lucie-7B-Instruct-human-data", - "developer": "OpenLLM-France", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2946, - "hfopenllm_v2/BBH": 0.3284, - "hfopenllm_v2/MATH Level 5": 0.0219, - "hfopenllm_v2/GPQA": 0.2752, - "hfopenllm_v2/MUSR": 0.3729, - "hfopenllm_v2/MMLU-PRO": 0.143 - } - }, - { - "id": "OpenLLM-France/Lucie-7B-Instruct-v1.1", - "name": "Lucie-7B-Instruct-v1.1", - "developer": "OpenLLM-France", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3039, - "hfopenllm_v2/BBH": 0.3816, - "hfopenllm_v2/MATH Level 5": 0.0317, - "hfopenllm_v2/GPQA": 0.2819, - "hfopenllm_v2/MUSR": 0.375, - "hfopenllm_v2/MMLU-PRO": 0.1864 - } - }, - { - "id": "OpenScholar/Llama-3.1_OpenScholar-8B", - "name": "Llama-3.1_OpenScholar-8B", - "developer": "OpenScholar", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6064, - "hfopenllm_v2/BBH": 0.5208, - "hfopenllm_v2/MATH Level 5": 0.1654, - "hfopenllm_v2/GPQA": 0.2819, - "hfopenllm_v2/MUSR": 0.4275, - "hfopenllm_v2/MMLU-PRO": 0.3708 - } - }, - { - "id": "orai-nlp/Llama-eus-8B", - "name": "Llama-eus-8B", - "developer": "orai-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2161, - "hfopenllm_v2/BBH": 0.4418, - "hfopenllm_v2/MATH Level 5": 0.0468, - "hfopenllm_v2/GPQA": 0.2894, - "hfopenllm_v2/MUSR": 0.3919, - "hfopenllm_v2/MMLU-PRO": 0.3058 - } - }, - { - "id": "Orenguteng/Llama-3.1-8B-Lexi-Uncensored", - "name": "Llama-3.1-8B-Lexi-Uncensored", - "developer": "Orenguteng", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7777, - "hfopenllm_v2/BBH": 0.5057, - "hfopenllm_v2/MATH Level 5": 0.1571, - "hfopenllm_v2/GPQA": 0.2718, - "hfopenllm_v2/MUSR": 0.3871, - "hfopenllm_v2/MMLU-PRO": 0.379 - } - }, - { - "id": "Orenguteng/Llama-3.1-8B-Lexi-Uncensored-V2", - "name": "Llama-3.1-8B-Lexi-Uncensored-V2", - "developer": "Orenguteng", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7792, - "hfopenllm_v2/BBH": 0.5084, - "hfopenllm_v2/MATH Level 5": 0.1971, - "hfopenllm_v2/GPQA": 0.2827, - "hfopenllm_v2/MUSR": 0.3843, - "hfopenllm_v2/MMLU-PRO": 0.3781 - } - }, - { - "id": "Orion-zhen/phi-4-abliterated", - "name": "phi-4-abliterated", - "developer": "Orion-zhen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0576, - "hfopenllm_v2/BBH": 0.6698, - "hfopenllm_v2/MATH Level 5": 0.3021, - "hfopenllm_v2/GPQA": 0.4044, - "hfopenllm_v2/MUSR": 0.5006, - "hfopenllm_v2/MMLU-PRO": 0.5292 - } - }, - { - "id": "Orion-zhen/Qwen2.5-7B-Instruct-Uncensored", - "name": "Qwen2.5-7B-Instruct-Uncensored", - "developer": "Orion-zhen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7204, - "hfopenllm_v2/BBH": 0.5474, - "hfopenllm_v2/MATH Level 5": 0.4773, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.4361, - "hfopenllm_v2/MMLU-PRO": 0.4427 - } - }, - { - "id": "oxyapi/oxy-1-small", - "name": "oxy-1-small", - "developer": "oxyapi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6245, - "hfopenllm_v2/BBH": 0.5885, - "hfopenllm_v2/MATH Level 5": 0.3603, - "hfopenllm_v2/GPQA": 0.3716, - "hfopenllm_v2/MUSR": 0.4487, - "hfopenllm_v2/MMLU-PRO": 0.5001 - } - }, - { - "id": "ozone-ai/0x-lite", - "name": "0x-lite", - "developer": "ozone-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.774, - "hfopenllm_v2/BBH": 0.6341, - "hfopenllm_v2/MATH Level 5": 0.5045, - "hfopenllm_v2/GPQA": 0.3196, - "hfopenllm_v2/MUSR": 0.4221, - "hfopenllm_v2/MMLU-PRO": 0.5184 - } - }, - { - "id": "ozone-research/Chirp-01", - "name": "Chirp-01", - "developer": "ozone-research", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6348, - "hfopenllm_v2/BBH": 0.465, - "hfopenllm_v2/MATH Level 5": 0.3467, - "hfopenllm_v2/GPQA": 0.2718, - "hfopenllm_v2/MUSR": 0.4487, - "hfopenllm_v2/MMLU-PRO": 0.3508 - } - }, - { - "id": "P0x0/Astra-v1-12B", - "name": "Astra-v1-12B", - "developer": "P0x0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2806, - "hfopenllm_v2/BBH": 0.5215, - "hfopenllm_v2/MATH Level 5": 0.1133, - "hfopenllm_v2/GPQA": 0.3138, - "hfopenllm_v2/MUSR": 0.4052, - "hfopenllm_v2/MMLU-PRO": 0.3461 - } - }, - { - "id": "paloalma/ECE-TW3-JRGL-V1", - "name": "ECE-TW3-JRGL-V1", - "developer": "paloalma", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5535, - "hfopenllm_v2/BBH": 0.6284, - "hfopenllm_v2/MATH Level 5": 0.1314, - "hfopenllm_v2/GPQA": 0.3473, - "hfopenllm_v2/MUSR": 0.4621, - "hfopenllm_v2/MMLU-PRO": 0.4221 - } - }, - { - "id": "paloalma/ECE-TW3-JRGL-V2", - "name": "ECE-TW3-JRGL-V2", - "developer": "paloalma", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2255, - "hfopenllm_v2/BBH": 0.6031, - "hfopenllm_v2/MATH Level 5": 0.185, - "hfopenllm_v2/GPQA": 0.3314, - "hfopenllm_v2/MUSR": 0.4793, - "hfopenllm_v2/MMLU-PRO": 0.4588 - } - }, - { - "id": "paloalma/ECE-TW3-JRGL-V5", - "name": "ECE-TW3-JRGL-V5", - "developer": "paloalma", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4553, - "hfopenllm_v2/BBH": 0.6025, - "hfopenllm_v2/MATH Level 5": 0.1835, - "hfopenllm_v2/GPQA": 0.3414, - "hfopenllm_v2/MUSR": 0.4621, - "hfopenllm_v2/MMLU-PRO": 0.4648 - } - }, - { - "id": "paloalma/Le_Triomphant-ECE-TW3", - "name": "Le_Triomphant-ECE-TW3", - "developer": "paloalma", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5402, - "hfopenllm_v2/BBH": 0.6112, - "hfopenllm_v2/MATH Level 5": 0.1949, - "hfopenllm_v2/GPQA": 0.349, - "hfopenllm_v2/MUSR": 0.4725, - "hfopenllm_v2/MMLU-PRO": 0.4763 - } - }, - { - "id": "paloalma/TW3-JRGL-v2", - "name": "TW3-JRGL-v2", - "developer": "paloalma", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5316, - "hfopenllm_v2/BBH": 0.6138, - "hfopenllm_v2/MATH Level 5": 0.179, - "hfopenllm_v2/GPQA": 0.3591, - "hfopenllm_v2/MUSR": 0.4858, - "hfopenllm_v2/MMLU-PRO": 0.4858 - } - }, - { - "id": "pankajmathur/Al_Dente_v1_8b", - "name": "Al_Dente_v1_8b", - "developer": "pankajmathur", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3694, - "hfopenllm_v2/BBH": 0.4835, - "hfopenllm_v2/MATH Level 5": 0.0408, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.3987, - "hfopenllm_v2/MMLU-PRO": 0.286 - } - }, - { - "id": "pankajmathur/model_007_13b_v2", - "name": "model_007_13b_v2", - "developer": "pankajmathur", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3056, - "hfopenllm_v2/BBH": 0.4702, - "hfopenllm_v2/MATH Level 5": 0.0211, - "hfopenllm_v2/GPQA": 0.2836, - "hfopenllm_v2/MUSR": 0.4611, - "hfopenllm_v2/MMLU-PRO": 0.2461 - } - }, - { - "id": "pankajmathur/orca_mini_3b", - "name": "orca_mini_3b", - "developer": "pankajmathur", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0742, - "hfopenllm_v2/BBH": 0.3196, - "hfopenllm_v2/MATH Level 5": 0.0083, - "hfopenllm_v2/GPQA": 0.2458, - "hfopenllm_v2/MUSR": 0.3349, - "hfopenllm_v2/MMLU-PRO": 0.1145 - } - }, - { - "id": "pankajmathur/orca_mini_7b", - "name": "orca_mini_7b", - "developer": "pankajmathur", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0412, - "hfopenllm_v2/BBH": 0.3332, - "hfopenllm_v2/MATH Level 5": 0.0128, - "hfopenllm_v2/GPQA": 0.2542, - "hfopenllm_v2/MUSR": 0.3698, - "hfopenllm_v2/MMLU-PRO": 0.1246 - } - }, - { - "id": "pankajmathur/orca_mini_phi-4", - "name": "orca_mini_phi-4", - "developer": "pankajmathur", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7781, - "hfopenllm_v2/BBH": 0.6856, - "hfopenllm_v2/MATH Level 5": 0.2953, - "hfopenllm_v2/GPQA": 0.3742, - "hfopenllm_v2/MUSR": 0.4703, - "hfopenllm_v2/MMLU-PRO": 0.5255 - } - }, - { - "id": "pankajmathur/orca_mini_v2_7b", - "name": "orca_mini_v2_7b", - "developer": "pankajmathur", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1358, - "hfopenllm_v2/BBH": 0.3536, - "hfopenllm_v2/MATH Level 5": 0.0113, - "hfopenllm_v2/GPQA": 0.2492, - "hfopenllm_v2/MUSR": 0.3593, - "hfopenllm_v2/MMLU-PRO": 0.1542 - } - }, - { - "id": "pankajmathur/orca_mini_v3_13b", - "name": "orca_mini_v3_13b", - "developer": "pankajmathur", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2897, - "hfopenllm_v2/BBH": 0.4711, - "hfopenllm_v2/MATH Level 5": 0.0211, - "hfopenllm_v2/GPQA": 0.2651, - "hfopenllm_v2/MUSR": 0.4598, - "hfopenllm_v2/MMLU-PRO": 0.2305 - } - }, - { - "id": "pankajmathur/orca_mini_v3_70b", - "name": "orca_mini_v3_70b", - "developer": "pankajmathur", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4015, - "hfopenllm_v2/BBH": 0.5949, - "hfopenllm_v2/MATH Level 5": 0.0385, - "hfopenllm_v2/GPQA": 0.318, - "hfopenllm_v2/MUSR": 0.5079, - "hfopenllm_v2/MMLU-PRO": 0.3757 - } - }, - { - "id": "pankajmathur/orca_mini_v3_7b", - "name": "orca_mini_v3_7b", - "developer": "pankajmathur", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2821, - "hfopenllm_v2/BBH": 0.4095, - "hfopenllm_v2/MATH Level 5": 0.0106, - "hfopenllm_v2/GPQA": 0.2466, - "hfopenllm_v2/MUSR": 0.4982, - "hfopenllm_v2/MMLU-PRO": 0.2084 - } - }, - { - "id": "pankajmathur/orca_mini_v5_8b", - "name": "orca_mini_v5_8b", - "developer": "pankajmathur", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4806, - "hfopenllm_v2/BBH": 0.5064, - "hfopenllm_v2/MATH Level 5": 0.0989, - "hfopenllm_v2/GPQA": 0.2869, - "hfopenllm_v2/MUSR": 0.4, - "hfopenllm_v2/MMLU-PRO": 0.3076 - } - }, - { - "id": "pankajmathur/orca_mini_v5_8b_dpo", - "name": "orca_mini_v5_8b_dpo", - "developer": "pankajmathur", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4896, - "hfopenllm_v2/BBH": 0.5075, - "hfopenllm_v2/MATH Level 5": 0.0974, - "hfopenllm_v2/GPQA": 0.2743, - "hfopenllm_v2/MUSR": 0.3894, - "hfopenllm_v2/MMLU-PRO": 0.3116 - } - }, - { - "id": "pankajmathur/orca_mini_v5_8b_orpo", - "name": "orca_mini_v5_8b_orpo", - "developer": "pankajmathur", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0824, - "hfopenllm_v2/BBH": 0.4964, - "hfopenllm_v2/MATH Level 5": 0.0665, - "hfopenllm_v2/GPQA": 0.2844, - "hfopenllm_v2/MUSR": 0.4131, - "hfopenllm_v2/MMLU-PRO": 0.2947 - } - }, - { - "id": "pankajmathur/orca_mini_v6_8b", - "name": "orca_mini_v6_8b", - "developer": "pankajmathur", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0111, - "hfopenllm_v2/BBH": 0.3029, - "hfopenllm_v2/MATH Level 5": 0.0038, - "hfopenllm_v2/GPQA": 0.2383, - "hfopenllm_v2/MUSR": 0.3555, - "hfopenllm_v2/MMLU-PRO": 0.1125 - } - }, - { - "id": "pankajmathur/orca_mini_v6_8b_dpo", - "name": "orca_mini_v6_8b_dpo", - "developer": "pankajmathur", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3883, - "hfopenllm_v2/BBH": 0.5203, - "hfopenllm_v2/MATH Level 5": 0.0612, - "hfopenllm_v2/GPQA": 0.3012, - "hfopenllm_v2/MUSR": 0.409, - "hfopenllm_v2/MMLU-PRO": 0.3596 - } - }, - { - "id": "pankajmathur/orca_mini_v7_72b", - "name": "orca_mini_v7_72b", - "developer": "pankajmathur", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.593, - "hfopenllm_v2/BBH": 0.6842, - "hfopenllm_v2/MATH Level 5": 0.0937, - "hfopenllm_v2/GPQA": 0.3851, - "hfopenllm_v2/MUSR": 0.507, - "hfopenllm_v2/MMLU-PRO": 0.5622 - } - }, - { - "id": "pankajmathur/orca_mini_v7_7b", - "name": "orca_mini_v7_7b", - "developer": "pankajmathur", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4388, - "hfopenllm_v2/BBH": 0.5275, - "hfopenllm_v2/MATH Level 5": 0.1208, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.436, - "hfopenllm_v2/MMLU-PRO": 0.4167 - } - }, - { - "id": "pankajmathur/orca_mini_v8_1_70b", - "name": "orca_mini_v8_1_70b", - "developer": "pankajmathur", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8571, - "hfopenllm_v2/BBH": 0.6781, - "hfopenllm_v2/MATH Level 5": 0.3527, - "hfopenllm_v2/GPQA": 0.4329, - "hfopenllm_v2/MUSR": 0.4437, - "hfopenllm_v2/MMLU-PRO": 0.4983 - } - }, - { - "id": "pankajmathur/orca_mini_v9_0_3B-Instruct", - "name": "orca_mini_v9_0_3B-Instruct", - "developer": "pankajmathur", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5754, - "hfopenllm_v2/BBH": 0.4413, - "hfopenllm_v2/MATH Level 5": 0.1465, - "hfopenllm_v2/GPQA": 0.3012, - "hfopenllm_v2/MUSR": 0.3659, - "hfopenllm_v2/MMLU-PRO": 0.2603 - } - }, - { - "id": "pankajmathur/orca_mini_v9_1_1B-Instruct", - "name": "orca_mini_v9_1_1B-Instruct", - "developer": "pankajmathur", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3629, - "hfopenllm_v2/BBH": 0.3205, - "hfopenllm_v2/MATH Level 5": 0.0461, - "hfopenllm_v2/GPQA": 0.2567, - "hfopenllm_v2/MUSR": 0.3381, - "hfopenllm_v2/MMLU-PRO": 0.1374 - } - }, - { - "id": "pankajmathur/orca_mini_v9_2_14B", - "name": "orca_mini_v9_2_14B", - "developer": "pankajmathur", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7781, - "hfopenllm_v2/BBH": 0.6856, - "hfopenllm_v2/MATH Level 5": 0.2953, - "hfopenllm_v2/GPQA": 0.3742, - "hfopenllm_v2/MUSR": 0.4703, - "hfopenllm_v2/MMLU-PRO": 0.5255 - } - }, - { - "id": "pankajmathur/orca_mini_v9_2_70b", - "name": "orca_mini_v9_2_70b", - "developer": "pankajmathur", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8383, - "hfopenllm_v2/BBH": 0.6745, - "hfopenllm_v2/MATH Level 5": 0.2938, - "hfopenllm_v2/GPQA": 0.3733, - "hfopenllm_v2/MUSR": 0.471, - "hfopenllm_v2/MMLU-PRO": 0.4821 - } - }, - { - "id": "pankajmathur/orca_mini_v9_4_70B", - "name": "orca_mini_v9_4_70B", - "developer": "pankajmathur", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8015, - "hfopenllm_v2/BBH": 0.6419, - "hfopenllm_v2/MATH Level 5": 0.3263, - "hfopenllm_v2/GPQA": 0.3658, - "hfopenllm_v2/MUSR": 0.4647, - "hfopenllm_v2/MMLU-PRO": 0.4536 - } - }, - { - "id": "pankajmathur/orca_mini_v9_5_1B-Instruct", - "name": "orca_mini_v9_5_1B-Instruct", - "developer": "pankajmathur", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4638, - "hfopenllm_v2/BBH": 0.3337, - "hfopenllm_v2/MATH Level 5": 0.0302, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.3182, - "hfopenllm_v2/MMLU-PRO": 0.137 - } - }, - { - "id": "pankajmathur/orca_mini_v9_5_1B-Instruct_preview", - "name": "orca_mini_v9_5_1B-Instruct_preview", - "developer": "pankajmathur", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3936, - "hfopenllm_v2/BBH": 0.3277, - "hfopenllm_v2/MATH Level 5": 0.0385, - "hfopenllm_v2/GPQA": 0.2634, - "hfopenllm_v2/MUSR": 0.3395, - "hfopenllm_v2/MMLU-PRO": 0.1327 - } - }, - { - "id": "pankajmathur/orca_mini_v9_5_3B-Instruct", - "name": "orca_mini_v9_5_3B-Instruct", - "developer": "pankajmathur", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7207, - "hfopenllm_v2/BBH": 0.4496, - "hfopenllm_v2/MATH Level 5": 0.1322, - "hfopenllm_v2/GPQA": 0.2869, - "hfopenllm_v2/MUSR": 0.427, - "hfopenllm_v2/MMLU-PRO": 0.2882 - } - }, - { - "id": "pankajmathur/orca_mini_v9_6_1B-Instruct", - "name": "orca_mini_v9_6_1B-Instruct", - "developer": "pankajmathur", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6086, - "hfopenllm_v2/BBH": 0.3561, - "hfopenllm_v2/MATH Level 5": 0.077, - "hfopenllm_v2/GPQA": 0.2685, - "hfopenllm_v2/MUSR": 0.3396, - "hfopenllm_v2/MMLU-PRO": 0.1809 - } - }, - { - "id": "pankajmathur/orca_mini_v9_6_3B-Instruct", - "name": "orca_mini_v9_6_3B-Instruct", - "developer": "pankajmathur", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7316, - "hfopenllm_v2/BBH": 0.4568, - "hfopenllm_v2/MATH Level 5": 0.1329, - "hfopenllm_v2/GPQA": 0.2936, - "hfopenllm_v2/MUSR": 0.4068, - "hfopenllm_v2/MMLU-PRO": 0.2851 - } - }, - { - "id": "pankajmathur/orca_mini_v9_7_1B-Instruct", - "name": "orca_mini_v9_7_1B-Instruct", - "developer": "pankajmathur", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.561, - "hfopenllm_v2/BBH": 0.3182, - "hfopenllm_v2/MATH Level 5": 0.0446, - "hfopenllm_v2/GPQA": 0.2727, - "hfopenllm_v2/MUSR": 0.3527, - "hfopenllm_v2/MMLU-PRO": 0.1345 - } - }, - { - "id": "pankajmathur/orca_mini_v9_7_3B-Instruct", - "name": "orca_mini_v9_7_3B-Instruct", - "developer": "pankajmathur", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5618, - "hfopenllm_v2/BBH": 0.3297, - "hfopenllm_v2/MATH Level 5": 0.0619, - "hfopenllm_v2/GPQA": 0.2617, - "hfopenllm_v2/MUSR": 0.3619, - "hfopenllm_v2/MMLU-PRO": 0.1375 - } - }, - { - "id": "Parissa3/test-model", - "name": "test-model", - "developer": "Parissa3", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3883, - "hfopenllm_v2/BBH": 0.5194, - "hfopenllm_v2/MATH Level 5": 0.065, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.4685, - "hfopenllm_v2/MMLU-PRO": 0.3057 - } - }, - { - "id": "paulml/ECE-ILAB-Q1", - "name": "ECE-ILAB-Q1", - "developer": "paulml", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7865, - "hfopenllm_v2/BBH": 0.6718, - "hfopenllm_v2/MATH Level 5": 0.3557, - "hfopenllm_v2/GPQA": 0.3867, - "hfopenllm_v2/MUSR": 0.4614, - "hfopenllm_v2/MMLU-PRO": 0.5505 - } - }, - { - "id": "phronetic-ai/rzn-t-prompt", - "name": "RZN-T (Prompt)", - "developer": "phronetic-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 96.0, - "bfcl/bfcl.overall.overall_accuracy": 22.25, - "bfcl/bfcl.overall.total_cost_usd": 12.31, - "bfcl/bfcl.overall.latency_mean_s": 12.32, - "bfcl/bfcl.overall.latency_std_s": 27.53, - "bfcl/bfcl.overall.latency_p95_s": 39.84, - "bfcl/bfcl.non_live.ast_accuracy": 67.94, - "bfcl/bfcl.non_live.simple_ast_accuracy": 63.75, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 75.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 69.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 63.5, - "bfcl/bfcl.live.live_accuracy": 49.74, - "bfcl/bfcl.live.live_simple_ast_accuracy": 61.24, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 47.2, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 43.75, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 41.67, - "bfcl/bfcl.multi_turn.accuracy": 2.88, - "bfcl/bfcl.multi_turn.base_accuracy": 4.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 2.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 2.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 2.5, - "bfcl/bfcl.web_search.accuracy": 0.0, - "bfcl/bfcl.web_search.base_accuracy": 0.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 0.0, - "bfcl/bfcl.memory.accuracy": 6.88, - "bfcl/bfcl.memory.kv_accuracy": 9.03, - "bfcl/bfcl.memory.vector_accuracy": 6.45, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 5.16, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 81.25, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 82.41, - "bfcl/bfcl.format_sensitivity.max_delta": 63.5, - "bfcl/bfcl.format_sensitivity.stddev": 25.53 - } - }, - { - "id": "Pinkstack/PARM-V1.5-base-QwQ-Qwen-2.5-o1-3B", - "name": "PARM-V1.5-base-QwQ-Qwen-2.5-o1-3B", - "developer": "Pinkstack", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5085, - "hfopenllm_v2/BBH": 0.4711, - "hfopenllm_v2/MATH Level 5": 0.1692, - "hfopenllm_v2/GPQA": 0.297, - "hfopenllm_v2/MUSR": 0.4479, - "hfopenllm_v2/MMLU-PRO": 0.3511 - } - }, - { - "id": "Pinkstack/SuperThoughts-CoT-14B-16k-o1-QwQ", - "name": "SuperThoughts-CoT-14B-16k-o1-QwQ", - "developer": "Pinkstack", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0515, - "hfopenllm_v2/BBH": 0.672, - "hfopenllm_v2/MATH Level 5": 0.4199, - "hfopenllm_v2/GPQA": 0.3926, - "hfopenllm_v2/MUSR": 0.4914, - "hfopenllm_v2/MMLU-PRO": 0.5268 - } - }, - { - "id": "Pinkstack/Superthoughts-lite-1.8B-experimental-o1", - "name": "Superthoughts-lite-1.8B-experimental-o1", - "developer": "Pinkstack", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0375, - "hfopenllm_v2/BBH": 0.3435, - "hfopenllm_v2/MATH Level 5": 0.0317, - "hfopenllm_v2/GPQA": 0.2752, - "hfopenllm_v2/MUSR": 0.3354, - "hfopenllm_v2/MMLU-PRO": 0.1851 - } - }, - { - "id": "Pinkstack/Superthoughts-lite-v1", - "name": "Superthoughts-lite-v1", - "developer": "Pinkstack", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1659, - "hfopenllm_v2/BBH": 0.3466, - "hfopenllm_v2/MATH Level 5": 0.0295, - "hfopenllm_v2/GPQA": 0.281, - "hfopenllm_v2/MUSR": 0.3672, - "hfopenllm_v2/MMLU-PRO": 0.1755 - } - }, - { - "id": "pints-ai/1.5-Pints-16K-v0.1", - "name": "1.5-Pints-16K-v0.1", - "developer": "pints-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1636, - "hfopenllm_v2/BBH": 0.3133, - "hfopenllm_v2/MATH Level 5": 0.0144, - "hfopenllm_v2/GPQA": 0.2357, - "hfopenllm_v2/MUSR": 0.3579, - "hfopenllm_v2/MMLU-PRO": 0.1119 - } - }, - { - "id": "pints-ai/1.5-Pints-2K-v0.1", - "name": "1.5-Pints-2K-v0.1", - "developer": "pints-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1762, - "hfopenllm_v2/BBH": 0.298, - "hfopenllm_v2/MATH Level 5": 0.0128, - "hfopenllm_v2/GPQA": 0.2483, - "hfopenllm_v2/MUSR": 0.3502, - "hfopenllm_v2/MMLU-PRO": 0.1104 - } - }, - { - "id": "piotr25691/thea-3b-25r", - "name": "thea-3b-25r", - "developer": "piotr25691", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7344, - "hfopenllm_v2/BBH": 0.4484, - "hfopenllm_v2/MATH Level 5": 0.1782, - "hfopenllm_v2/GPQA": 0.2676, - "hfopenllm_v2/MUSR": 0.3315, - "hfopenllm_v2/MMLU-PRO": 0.3182 - } - }, - { - "id": "piotr25691/thea-c-3b-25r", - "name": "thea-c-3b-25r", - "developer": "piotr25691", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7402, - "hfopenllm_v2/BBH": 0.4532, - "hfopenllm_v2/MATH Level 5": 0.1526, - "hfopenllm_v2/GPQA": 0.2651, - "hfopenllm_v2/MUSR": 0.3315, - "hfopenllm_v2/MMLU-PRO": 0.3178 - } - }, - { - "id": "piotr25691/thea-rp-3b-25r", - "name": "thea-rp-3b-25r", - "developer": "piotr25691", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6578, - "hfopenllm_v2/BBH": 0.439, - "hfopenllm_v2/MATH Level 5": 0.1322, - "hfopenllm_v2/GPQA": 0.2743, - "hfopenllm_v2/MUSR": 0.3819, - "hfopenllm_v2/MMLU-PRO": 0.306 - } - }, - { - "id": "PJMixers-Dev/L3.2-Instruct-Thinking-v0.1-1B", - "name": "L3.2-Instruct-Thinking-v0.1-1B", - "developer": "PJMixers-Dev", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4628, - "hfopenllm_v2/BBH": 0.3302, - "hfopenllm_v2/MATH Level 5": 0.0544, - "hfopenllm_v2/GPQA": 0.2576, - "hfopenllm_v2/MUSR": 0.3262, - "hfopenllm_v2/MMLU-PRO": 0.1483 - } - }, - { - "id": "PJMixers-Dev/LLaMa-3.1-Instruct-Interleaved-Zeroed-13B", - "name": "LLaMa-3.1-Instruct-Interleaved-Zeroed-13B", - "developer": "PJMixers-Dev", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7871, - "hfopenllm_v2/BBH": 0.5073, - "hfopenllm_v2/MATH Level 5": 0.2002, - "hfopenllm_v2/GPQA": 0.2919, - "hfopenllm_v2/MUSR": 0.387, - "hfopenllm_v2/MMLU-PRO": 0.3767 - } - }, - { - "id": "PJMixers-Dev/LLaMa-3.1-RomboTiesTest-8B", - "name": "LLaMa-3.1-RomboTiesTest-8B", - "developer": "PJMixers-Dev", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7825, - "hfopenllm_v2/BBH": 0.5073, - "hfopenllm_v2/MATH Level 5": 0.2002, - "hfopenllm_v2/GPQA": 0.2919, - "hfopenllm_v2/MUSR": 0.387, - "hfopenllm_v2/MMLU-PRO": 0.3767 - } - }, - { - "id": "PJMixers-Dev/LLaMa-3.1-RomboTiesTest2-8B", - "name": "LLaMa-3.1-RomboTiesTest2-8B", - "developer": "PJMixers-Dev", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7825, - "hfopenllm_v2/BBH": 0.5073, - "hfopenllm_v2/MATH Level 5": 0.2002, - "hfopenllm_v2/GPQA": 0.2919, - "hfopenllm_v2/MUSR": 0.387, - "hfopenllm_v2/MMLU-PRO": 0.3767 - } - }, - { - "id": "PJMixers-Dev/LLaMa-3.2-Instruct-JankMix-v0.1-SFT-3B", - "name": "LLaMa-3.2-Instruct-JankMix-v0.1-SFT-3B", - "developer": "PJMixers-Dev", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6931, - "hfopenllm_v2/BBH": 0.4556, - "hfopenllm_v2/MATH Level 5": 0.1216, - "hfopenllm_v2/GPQA": 0.2743, - "hfopenllm_v2/MUSR": 0.37, - "hfopenllm_v2/MMLU-PRO": 0.3127 - } - }, - { - "id": "PJMixers-Dev/LLaMa-3.2-Instruct-JankMix-v0.2-SFT-3B", - "name": "LLaMa-3.2-Instruct-JankMix-v0.2-SFT-3B", - "developer": "PJMixers-Dev", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6292, - "hfopenllm_v2/BBH": 0.4581, - "hfopenllm_v2/MATH Level 5": 0.1299, - "hfopenllm_v2/GPQA": 0.2727, - "hfopenllm_v2/MUSR": 0.3659, - "hfopenllm_v2/MMLU-PRO": 0.3115 - } - }, - { - "id": "PJMixers-Dev/LLaMa-3.2-Instruct-JankMix-v0.2-SFT-HailMary-v0.1-KTO-3B", - "name": "LLaMa-3.2-Instruct-JankMix-v0.2-SFT-HailMary-v0.1-KTO-3B", - "developer": "PJMixers-Dev", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6504, - "hfopenllm_v2/BBH": 0.4511, - "hfopenllm_v2/MATH Level 5": 0.1261, - "hfopenllm_v2/GPQA": 0.2718, - "hfopenllm_v2/MUSR": 0.3687, - "hfopenllm_v2/MMLU-PRO": 0.3108 - } - }, - { - "id": "PJMixers-Dev/LLaMa-3.2-Instruct-JankMixBread-v0.1-3B", - "name": "LLaMa-3.2-Instruct-JankMixBread-v0.1-3B", - "developer": "PJMixers-Dev", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5041, - "hfopenllm_v2/BBH": 0.4483, - "hfopenllm_v2/MATH Level 5": 0.1307, - "hfopenllm_v2/GPQA": 0.2827, - "hfopenllm_v2/MUSR": 0.3516, - "hfopenllm_v2/MMLU-PRO": 0.3083 - } - }, - { - "id": "PJMixers-Dev/Qwen2.5-RomboTiesTest-7B", - "name": "Qwen2.5-RomboTiesTest-7B", - "developer": "PJMixers-Dev", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7558, - "hfopenllm_v2/BBH": 0.5399, - "hfopenllm_v2/MATH Level 5": 0.4962, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.4034, - "hfopenllm_v2/MMLU-PRO": 0.4285 - } - }, - { - "id": "PJMixers/LLaMa-3-CursedStock-v2.0-8B", - "name": "LLaMa-3-CursedStock-v2.0-8B", - "developer": "PJMixers", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6331, - "hfopenllm_v2/BBH": 0.5271, - "hfopenllm_v2/MATH Level 5": 0.0944, - "hfopenllm_v2/GPQA": 0.2743, - "hfopenllm_v2/MUSR": 0.3856, - "hfopenllm_v2/MMLU-PRO": 0.3556 - } - }, - { - "id": "PKU-Alignment/beaver-7b-v1.0-cost", - "name": "PKU-Alignment/beaver-7b-v1.0-cost", - "developer": "PKU-Alignment", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5798, - "reward-bench/Factuality": 0.3263, - "reward-bench/Precise IF": 0.2313, - "reward-bench/Math": 0.3989, - "reward-bench/Safety": 0.7351, - "reward-bench/Focus": 0.2939, - "reward-bench/Ties": -0.01, - "reward-bench/Chat": 0.6173, - "reward-bench/Chat Hard": 0.4232, - "reward-bench/Reasoning": 0.5482, - "reward-bench/Prior Sets (0.5 weight)": 0.57 - } - }, - { - "id": "PKU-Alignment/beaver-7b-v1.0-reward", - "name": "PKU-Alignment/beaver-7b-v1.0-reward", - "developer": "PKU-Alignment", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.4727, - "reward-bench/Factuality": 0.2105, - "reward-bench/Precise IF": 0.2938, - "reward-bench/Math": 0.2623, - "reward-bench/Safety": 0.3757, - "reward-bench/Focus": 0.0646, - "reward-bench/Ties": -0.01, - "reward-bench/Chat": 0.8184, - "reward-bench/Chat Hard": 0.2873, - "reward-bench/Reasoning": 0.346, - "reward-bench/Prior Sets (0.5 weight)": 0.5993 - } - }, - { - "id": "PKU-Alignment/beaver-7b-v2.0-cost", - "name": "PKU-Alignment/beaver-7b-v2.0-cost", - "developer": "PKU-Alignment", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5957, - "reward-bench/Factuality": 0.3789, - "reward-bench/Precise IF": 0.275, - "reward-bench/Math": 0.3333, - "reward-bench/Safety": 0.7608, - "reward-bench/Focus": 0.2828, - "reward-bench/Ties": -0.01, - "reward-bench/Chat": 0.5726, - "reward-bench/Chat Hard": 0.4561, - "reward-bench/Reasoning": 0.6211, - "reward-bench/Prior Sets (0.5 weight)": 0.5397 - } - }, - { - "id": "PKU-Alignment/beaver-7b-v2.0-reward", - "name": "PKU-Alignment/beaver-7b-v2.0-reward", - "developer": "PKU-Alignment", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6366, - "reward-bench/Factuality": 0.2168, - "reward-bench/Precise IF": 0.2562, - "reward-bench/Math": 0.3825, - "reward-bench/Safety": 0.6041, - "reward-bench/Focus": 0.2606, - "reward-bench/Ties": 0.0944, - "reward-bench/Chat": 0.8994, - "reward-bench/Chat Hard": 0.364, - "reward-bench/Reasoning": 0.6887, - "reward-bench/Prior Sets (0.5 weight)": 0.6171 - } - }, - { - "id": "PocketDoc/Dans-Instruct-CoreCurriculum-12b", - "name": "Dans-Instruct-CoreCurriculum-12b", - "developer": "PocketDoc", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2191, - "hfopenllm_v2/BBH": 0.3789, - "hfopenllm_v2/MATH Level 5": 0.0544, - "hfopenllm_v2/GPQA": 0.2827, - "hfopenllm_v2/MUSR": 0.4096, - "hfopenllm_v2/MMLU-PRO": 0.1219 - } - }, - { - "id": "PocketDoc/Dans-PersonalityEngine-v1.0.0-8b", - "name": "Dans-PersonalityEngine-v1.0.0-8b", - "developer": "PocketDoc", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4982, - "hfopenllm_v2/BBH": 0.4733, - "hfopenllm_v2/MATH Level 5": 0.0816, - "hfopenllm_v2/GPQA": 0.2852, - "hfopenllm_v2/MUSR": 0.3542, - "hfopenllm_v2/MMLU-PRO": 0.3065 - } - }, - { - "id": "PocketDoc/Dans-PersonalityEngine-V1.1.0-12b", - "name": "Dans-PersonalityEngine-V1.1.0-12b", - "developer": "PocketDoc", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7075, - "hfopenllm_v2/BBH": 0.5361, - "hfopenllm_v2/MATH Level 5": 0.105, - "hfopenllm_v2/GPQA": 0.2869, - "hfopenllm_v2/MUSR": 0.4587, - "hfopenllm_v2/MMLU-PRO": 0.3262 - } - }, - { - "id": "PocketDoc/Dans-PersonalityEngine-V1.2.0-24b", - "name": "Dans-PersonalityEngine-V1.2.0-24b", - "developer": "PocketDoc", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7886, - "hfopenllm_v2/BBH": 0.6421, - "hfopenllm_v2/MATH Level 5": 0.2455, - "hfopenllm_v2/GPQA": 0.3188, - "hfopenllm_v2/MUSR": 0.43, - "hfopenllm_v2/MMLU-PRO": 0.5026 - } - }, - { - "id": "PocketDoc/Dans-SakuraKaze-V1.0.0-12b", - "name": "Dans-SakuraKaze-V1.0.0-12b", - "developer": "PocketDoc", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.652, - "hfopenllm_v2/BBH": 0.5405, - "hfopenllm_v2/MATH Level 5": 0.0929, - "hfopenllm_v2/GPQA": 0.2936, - "hfopenllm_v2/MUSR": 0.4745, - "hfopenllm_v2/MMLU-PRO": 0.356 - } - }, - { - "id": "PoLL/gpt-3.5-turbo-0125_claude-3-sonnet-2024022...", - "name": "PoLL/gpt-3.5-turbo-0125_claude-3-sonnet-2024022...", - "developer": "PoLL", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7578, - "reward-bench/Chat": 0.9525, - "reward-bench/Chat Hard": 0.5406, - "reward-bench/Safety": 0.8034, - "reward-bench/Reasoning": 0.7346 - } - }, - { - "id": "postbot/gpt2-medium-emailgen", - "name": "gpt2-medium-emailgen", - "developer": "postbot", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1492, - "hfopenllm_v2/BBH": 0.313, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3911, - "hfopenllm_v2/MMLU-PRO": 0.1147 - } - }, - { - "id": "PowerInfer/SmallThinker-3B-Preview", - "name": "SmallThinker-3B-Preview", - "developer": "PowerInfer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.62, - "hfopenllm_v2/BBH": 0.4495, - "hfopenllm_v2/MATH Level 5": 0.2779, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.3525, - "hfopenllm_v2/MMLU-PRO": 0.3018 - } - }, - { - "id": "PranavHarshan/LaMistral-V4", - "name": "LaMistral-V4", - "developer": "PranavHarshan", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6239, - "hfopenllm_v2/BBH": 0.5184, - "hfopenllm_v2/MATH Level 5": 0.0687, - "hfopenllm_v2/GPQA": 0.328, - "hfopenllm_v2/MUSR": 0.3643, - "hfopenllm_v2/MMLU-PRO": 0.3599 - } - }, - { - "id": "PranavHarshan/MedNarra-X1", - "name": "MedNarra-X1", - "developer": "PranavHarshan", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4338, - "hfopenllm_v2/BBH": 0.4637, - "hfopenllm_v2/MATH Level 5": 0.0438, - "hfopenllm_v2/GPQA": 0.3079, - "hfopenllm_v2/MUSR": 0.354, - "hfopenllm_v2/MMLU-PRO": 0.3431 - } - }, - { - "id": "Pretergeek/OpenChat-3.5-0106_10.7B_48Layers-Appended", - "name": "OpenChat-3.5-0106_10.7B_48Layers-Appended", - "developer": "Pretergeek", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5961, - "hfopenllm_v2/BBH": 0.462, - "hfopenllm_v2/MATH Level 5": 0.0793, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.4254, - "hfopenllm_v2/MMLU-PRO": 0.329 - } - }, - { - "id": "Pretergeek/OpenChat-3.5-0106_10.7B_48Layers-Interleaved", - "name": "OpenChat-3.5-0106_10.7B_48Layers-Interleaved", - "developer": "Pretergeek", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5961, - "hfopenllm_v2/BBH": 0.462, - "hfopenllm_v2/MATH Level 5": 0.0778, - "hfopenllm_v2/GPQA": 0.3045, - "hfopenllm_v2/MUSR": 0.4254, - "hfopenllm_v2/MMLU-PRO": 0.3299 - } - }, - { - "id": "Pretergeek/OpenChat-3.5-0106_32K-PoSE", - "name": "OpenChat-3.5-0106_32K-PoSE", - "developer": "Pretergeek", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3969, - "hfopenllm_v2/BBH": 0.3471, - "hfopenllm_v2/MATH Level 5": 0.0264, - "hfopenllm_v2/GPQA": 0.276, - "hfopenllm_v2/MUSR": 0.4205, - "hfopenllm_v2/MMLU-PRO": 0.2031 - } - }, - { - "id": "Pretergeek/OpenChat-3.5-0106_8.11B_36Layers-Appended", - "name": "OpenChat-3.5-0106_8.11B_36Layers-Appended", - "developer": "Pretergeek", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5976, - "hfopenllm_v2/BBH": 0.462, - "hfopenllm_v2/MATH Level 5": 0.0793, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.4254, - "hfopenllm_v2/MMLU-PRO": 0.329 - } - }, - { - "id": "Pretergeek/OpenChat-3.5-0106_8.11B_36Layers-Interleaved", - "name": "OpenChat-3.5-0106_8.11B_36Layers-Interleaved", - "developer": "Pretergeek", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5961, - "hfopenllm_v2/BBH": 0.4621, - "hfopenllm_v2/MATH Level 5": 0.0778, - "hfopenllm_v2/GPQA": 0.3045, - "hfopenllm_v2/MUSR": 0.4241, - "hfopenllm_v2/MMLU-PRO": 0.3299 - } - }, - { - "id": "Pretergeek/OpenChat-3.5-0106_8.99B_40Layers-Appended", - "name": "OpenChat-3.5-0106_8.99B_40Layers-Appended", - "developer": "Pretergeek", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5961, - "hfopenllm_v2/BBH": 0.462, - "hfopenllm_v2/MATH Level 5": 0.0793, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.4254, - "hfopenllm_v2/MMLU-PRO": 0.329 - } - }, - { - "id": "Pretergeek/OpenChat-3.5-0106_8.99B_40Layers-Interleaved", - "name": "OpenChat-3.5-0106_8.99B_40Layers-Interleaved", - "developer": "Pretergeek", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5976, - "hfopenllm_v2/BBH": 0.4621, - "hfopenllm_v2/MATH Level 5": 0.0778, - "hfopenllm_v2/GPQA": 0.3045, - "hfopenllm_v2/MUSR": 0.4241, - "hfopenllm_v2/MMLU-PRO": 0.3299 - } - }, - { - "id": "Pretergeek/OpenChat-3.5-0106_9.86B_44Layers-Appended", - "name": "OpenChat-3.5-0106_9.86B_44Layers-Appended", - "developer": "Pretergeek", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5961, - "hfopenllm_v2/BBH": 0.462, - "hfopenllm_v2/MATH Level 5": 0.0793, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.4254, - "hfopenllm_v2/MMLU-PRO": 0.329 - } - }, - { - "id": "Pretergeek/openchat-3.5-0106_Rebased_Mistral-7B-v0.2", - "name": "openchat-3.5-0106_Rebased_Mistral-7B-v0.2", - "developer": "Pretergeek", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3706, - "hfopenllm_v2/BBH": 0.3627, - "hfopenllm_v2/MATH Level 5": 0.0453, - "hfopenllm_v2/GPQA": 0.2718, - "hfopenllm_v2/MUSR": 0.484, - "hfopenllm_v2/MMLU-PRO": 0.283 - } - }, - { - "id": "PrimeIntellect/INTELLECT-1", - "name": "INTELLECT-1", - "developer": "PrimeIntellect", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1757, - "hfopenllm_v2/BBH": 0.274, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.25, - "hfopenllm_v2/MUSR": 0.3753, - "hfopenllm_v2/MMLU-PRO": 0.112 - } - }, - { - "id": "PrimeIntellect/INTELLECT-1-Instruct", - "name": "INTELLECT-1-Instruct", - "developer": "PrimeIntellect", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0, - "hfopenllm_v2/BBH": 0.287, - "hfopenllm_v2/MATH Level 5": 0.0227, - "hfopenllm_v2/GPQA": 0.2483, - "hfopenllm_v2/MUSR": 0.3577, - "hfopenllm_v2/MMLU-PRO": 0.1064 - } - }, - { - "id": "prince-canuma/Ministral-8B-Instruct-2410-HF", - "name": "Ministral-8B-Instruct-2410-HF", - "developer": "prince-canuma", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5912, - "hfopenllm_v2/BBH": 0.4586, - "hfopenllm_v2/MATH Level 5": 0.1918, - "hfopenllm_v2/GPQA": 0.281, - "hfopenllm_v2/MUSR": 0.4138, - "hfopenllm_v2/MMLU-PRO": 0.3298 - } - }, - { - "id": "princeton-nlp/gemma-2-9b-it-DPO", - "name": "gemma-2-9b-it-DPO", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2769, - "hfopenllm_v2/BBH": 0.5941, - "hfopenllm_v2/MATH Level 5": 0.0831, - "hfopenllm_v2/GPQA": 0.3356, - "hfopenllm_v2/MUSR": 0.382, - "hfopenllm_v2/MMLU-PRO": 0.3723 - } - }, - { - "id": "princeton-nlp/gemma-2-9b-it-SimPO", - "name": "gemma-2-9b-it-SimPO", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3207, - "hfopenllm_v2/BBH": 0.5839, - "hfopenllm_v2/MATH Level 5": 0.071, - "hfopenllm_v2/GPQA": 0.3356, - "hfopenllm_v2/MUSR": 0.4123, - "hfopenllm_v2/MMLU-PRO": 0.3975 - } - }, - { - "id": "princeton-nlp/Llama-3-8B-ProLong-512k-Base", - "name": "Llama-3-8B-ProLong-512k-Base", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5322, - "hfopenllm_v2/BBH": 0.5033, - "hfopenllm_v2/MATH Level 5": 0.0687, - "hfopenllm_v2/GPQA": 0.2617, - "hfopenllm_v2/MUSR": 0.4223, - "hfopenllm_v2/MMLU-PRO": 0.3329 - } - }, - { - "id": "princeton-nlp/Llama-3-8B-ProLong-512k-Instruct", - "name": "Llama-3-8B-ProLong-512k-Instruct", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5508, - "hfopenllm_v2/BBH": 0.5028, - "hfopenllm_v2/MATH Level 5": 0.0529, - "hfopenllm_v2/GPQA": 0.2861, - "hfopenllm_v2/MUSR": 0.4266, - "hfopenllm_v2/MMLU-PRO": 0.3231 - } - }, - { - "id": "princeton-nlp/Llama-3-8B-ProLong-64k-Base", - "name": "Llama-3-8B-ProLong-64k-Base", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5201, - "hfopenllm_v2/BBH": 0.4927, - "hfopenllm_v2/MATH Level 5": 0.065, - "hfopenllm_v2/GPQA": 0.2651, - "hfopenllm_v2/MUSR": 0.4341, - "hfopenllm_v2/MMLU-PRO": 0.3348 - } - }, - { - "id": "princeton-nlp/Llama-3-8B-ProLong-64k-Instruct", - "name": "Llama-3-8B-ProLong-64k-Instruct", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5563, - "hfopenllm_v2/BBH": 0.5083, - "hfopenllm_v2/MATH Level 5": 0.065, - "hfopenllm_v2/GPQA": 0.2953, - "hfopenllm_v2/MUSR": 0.4397, - "hfopenllm_v2/MMLU-PRO": 0.3275 - } - }, - { - "id": "princeton-nlp/Llama-3-Base-8B-SFT", - "name": "Llama-3-Base-8B-SFT", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2796, - "hfopenllm_v2/BBH": 0.4643, - "hfopenllm_v2/MATH Level 5": 0.04, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.4118, - "hfopenllm_v2/MMLU-PRO": 0.3093 - } - }, - { - "id": "princeton-nlp/Llama-3-Base-8B-SFT-CPO", - "name": "Llama-3-Base-8B-SFT-CPO", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3703, - "hfopenllm_v2/BBH": 0.4595, - "hfopenllm_v2/MATH Level 5": 0.0544, - "hfopenllm_v2/GPQA": 0.2743, - "hfopenllm_v2/MUSR": 0.3609, - "hfopenllm_v2/MMLU-PRO": 0.2976 - } - }, - { - "id": "princeton-nlp/Llama-3-Base-8B-SFT-DPO", - "name": "Llama-3-Base-8B-SFT-DPO", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4111, - "hfopenllm_v2/BBH": 0.4666, - "hfopenllm_v2/MATH Level 5": 0.0415, - "hfopenllm_v2/GPQA": 0.3104, - "hfopenllm_v2/MUSR": 0.3867, - "hfopenllm_v2/MMLU-PRO": 0.3078 - } - }, - { - "id": "princeton-nlp/Llama-3-Base-8B-SFT-IPO", - "name": "Llama-3-Base-8B-SFT-IPO", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4487, - "hfopenllm_v2/BBH": 0.469, - "hfopenllm_v2/MATH Level 5": 0.0393, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.3919, - "hfopenllm_v2/MMLU-PRO": 0.3115 - } - }, - { - "id": "princeton-nlp/Llama-3-Base-8B-SFT-KTO", - "name": "Llama-3-Base-8B-SFT-KTO", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4523, - "hfopenllm_v2/BBH": 0.4693, - "hfopenllm_v2/MATH Level 5": 0.0529, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.3842, - "hfopenllm_v2/MMLU-PRO": 0.3054 - } - }, - { - "id": "princeton-nlp/Llama-3-Base-8B-SFT-ORPO", - "name": "Llama-3-Base-8B-SFT-ORPO", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4517, - "hfopenllm_v2/BBH": 0.4734, - "hfopenllm_v2/MATH Level 5": 0.0468, - "hfopenllm_v2/GPQA": 0.3138, - "hfopenllm_v2/MUSR": 0.3707, - "hfopenllm_v2/MMLU-PRO": 0.3083 - } - }, - { - "id": "princeton-nlp/Llama-3-Base-8B-SFT-RDPO", - "name": "Llama-3-Base-8B-SFT-RDPO", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.448, - "hfopenllm_v2/BBH": 0.4662, - "hfopenllm_v2/MATH Level 5": 0.0574, - "hfopenllm_v2/GPQA": 0.3062, - "hfopenllm_v2/MUSR": 0.4027, - "hfopenllm_v2/MMLU-PRO": 0.3014 - } - }, - { - "id": "princeton-nlp/Llama-3-Base-8B-SFT-RRHF", - "name": "Llama-3-Base-8B-SFT-RRHF", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3357, - "hfopenllm_v2/BBH": 0.452, - "hfopenllm_v2/MATH Level 5": 0.0453, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.3722, - "hfopenllm_v2/MMLU-PRO": 0.2889 - } - }, - { - "id": "princeton-nlp/Llama-3-Base-8B-SFT-SimPO", - "name": "Llama-3-Base-8B-SFT-SimPO", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4685, - "hfopenllm_v2/BBH": 0.4741, - "hfopenllm_v2/MATH Level 5": 0.0551, - "hfopenllm_v2/GPQA": 0.2886, - "hfopenllm_v2/MUSR": 0.4127, - "hfopenllm_v2/MMLU-PRO": 0.3105 - } - }, - { - "id": "princeton-nlp/Llama-3-Base-8B-SFT-SLiC-HF", - "name": "Llama-3-Base-8B-SFT-SLiC-HF", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.489, - "hfopenllm_v2/BBH": 0.4704, - "hfopenllm_v2/MATH Level 5": 0.0506, - "hfopenllm_v2/GPQA": 0.2869, - "hfopenllm_v2/MUSR": 0.4091, - "hfopenllm_v2/MMLU-PRO": 0.3063 - } - }, - { - "id": "princeton-nlp/Llama-3-Instruct-8B-CPO", - "name": "Llama-3-Instruct-8B-CPO", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7293, - "hfopenllm_v2/BBH": 0.4999, - "hfopenllm_v2/MATH Level 5": 0.0989, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3514, - "hfopenllm_v2/MMLU-PRO": 0.3652 - } - }, - { - "id": "princeton-nlp/Llama-3-Instruct-8B-CPO-v0.2", - "name": "Llama-3-Instruct-8B-CPO-v0.2", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7506, - "hfopenllm_v2/BBH": 0.5027, - "hfopenllm_v2/MATH Level 5": 0.108, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.3619, - "hfopenllm_v2/MMLU-PRO": 0.3706 - } - }, - { - "id": "princeton-nlp/Llama-3-Instruct-8B-DPO", - "name": "Llama-3-Instruct-8B-DPO", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6757, - "hfopenllm_v2/BBH": 0.4991, - "hfopenllm_v2/MATH Level 5": 0.0846, - "hfopenllm_v2/GPQA": 0.2718, - "hfopenllm_v2/MUSR": 0.3738, - "hfopenllm_v2/MMLU-PRO": 0.3665 - } - }, - { - "id": "princeton-nlp/Llama-3-Instruct-8B-DPO-v0.2", - "name": "Llama-3-Instruct-8B-DPO-v0.2", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7208, - "hfopenllm_v2/BBH": 0.5056, - "hfopenllm_v2/MATH Level 5": 0.0899, - "hfopenllm_v2/GPQA": 0.2869, - "hfopenllm_v2/MUSR": 0.3844, - "hfopenllm_v2/MMLU-PRO": 0.3769 - } - }, - { - "id": "princeton-nlp/Llama-3-Instruct-8B-KTO", - "name": "Llama-3-Instruct-8B-KTO", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6864, - "hfopenllm_v2/BBH": 0.4982, - "hfopenllm_v2/MATH Level 5": 0.0725, - "hfopenllm_v2/GPQA": 0.276, - "hfopenllm_v2/MUSR": 0.3698, - "hfopenllm_v2/MMLU-PRO": 0.3599 - } - }, - { - "id": "princeton-nlp/Llama-3-Instruct-8B-KTO-v0.2", - "name": "Llama-3-Instruct-8B-KTO-v0.2", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.729, - "hfopenllm_v2/BBH": 0.508, - "hfopenllm_v2/MATH Level 5": 0.0997, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3777, - "hfopenllm_v2/MMLU-PRO": 0.3668 - } - }, - { - "id": "princeton-nlp/Llama-3-Instruct-8B-ORPO", - "name": "Llama-3-Instruct-8B-ORPO", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7128, - "hfopenllm_v2/BBH": 0.5001, - "hfopenllm_v2/MATH Level 5": 0.0785, - "hfopenllm_v2/GPQA": 0.2584, - "hfopenllm_v2/MUSR": 0.3502, - "hfopenllm_v2/MMLU-PRO": 0.3646 - } - }, - { - "id": "princeton-nlp/Llama-3-Instruct-8B-ORPO-v0.2", - "name": "Llama-3-Instruct-8B-ORPO-v0.2", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7633, - "hfopenllm_v2/BBH": 0.5078, - "hfopenllm_v2/MATH Level 5": 0.102, - "hfopenllm_v2/GPQA": 0.2836, - "hfopenllm_v2/MUSR": 0.378, - "hfopenllm_v2/MMLU-PRO": 0.3731 - } - }, - { - "id": "princeton-nlp/Llama-3-Instruct-8B-RDPO", - "name": "Llama-3-Instruct-8B-RDPO", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.666, - "hfopenllm_v2/BBH": 0.5034, - "hfopenllm_v2/MATH Level 5": 0.0846, - "hfopenllm_v2/GPQA": 0.2827, - "hfopenllm_v2/MUSR": 0.3752, - "hfopenllm_v2/MMLU-PRO": 0.3607 - } - }, - { - "id": "princeton-nlp/Llama-3-Instruct-8B-RDPO-v0.2", - "name": "Llama-3-Instruct-8B-RDPO-v0.2", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7077, - "hfopenllm_v2/BBH": 0.5049, - "hfopenllm_v2/MATH Level 5": 0.0869, - "hfopenllm_v2/GPQA": 0.2928, - "hfopenllm_v2/MUSR": 0.3804, - "hfopenllm_v2/MMLU-PRO": 0.3774 - } - }, - { - "id": "princeton-nlp/Llama-3-Instruct-8B-RRHF", - "name": "Llama-3-Instruct-8B-RRHF", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7275, - "hfopenllm_v2/BBH": 0.4911, - "hfopenllm_v2/MATH Level 5": 0.0967, - "hfopenllm_v2/GPQA": 0.2802, - "hfopenllm_v2/MUSR": 0.3476, - "hfopenllm_v2/MMLU-PRO": 0.3644 - } - }, - { - "id": "princeton-nlp/Llama-3-Instruct-8B-RRHF-v0.2", - "name": "Llama-3-Instruct-8B-RRHF-v0.2", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7125, - "hfopenllm_v2/BBH": 0.4984, - "hfopenllm_v2/MATH Level 5": 0.0876, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3738, - "hfopenllm_v2/MMLU-PRO": 0.3482 - } - }, - { - "id": "princeton-nlp/Llama-3-Instruct-8B-SimPO", - "name": "Llama-3-Instruct-8B-SimPO", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6504, - "hfopenllm_v2/BBH": 0.4845, - "hfopenllm_v2/MATH Level 5": 0.0861, - "hfopenllm_v2/GPQA": 0.2936, - "hfopenllm_v2/MUSR": 0.3948, - "hfopenllm_v2/MMLU-PRO": 0.3489 - } - }, - { - "id": "princeton-nlp/Llama-3-Instruct-8B-SimPO-v0.2", - "name": "Llama-3-Instruct-8B-SimPO-v0.2", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6809, - "hfopenllm_v2/BBH": 0.5038, - "hfopenllm_v2/MATH Level 5": 0.074, - "hfopenllm_v2/GPQA": 0.3012, - "hfopenllm_v2/MUSR": 0.3988, - "hfopenllm_v2/MMLU-PRO": 0.3622 - } - }, - { - "id": "princeton-nlp/Llama-3-Instruct-8B-SLiC-HF", - "name": "Llama-3-Instruct-8B-SLiC-HF", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.74, - "hfopenllm_v2/BBH": 0.5029, - "hfopenllm_v2/MATH Level 5": 0.0974, - "hfopenllm_v2/GPQA": 0.2861, - "hfopenllm_v2/MUSR": 0.3723, - "hfopenllm_v2/MMLU-PRO": 0.3585 - } - }, - { - "id": "princeton-nlp/Llama-3-Instruct-8B-SLiC-HF-v0.2", - "name": "Llama-3-Instruct-8B-SLiC-HF-v0.2", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.711, - "hfopenllm_v2/BBH": 0.4984, - "hfopenllm_v2/MATH Level 5": 0.0876, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3738, - "hfopenllm_v2/MMLU-PRO": 0.3482 - } - }, - { - "id": "princeton-nlp/Mistral-7B-Base-SFT-CPO", - "name": "Mistral-7B-Base-SFT-CPO", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4655, - "hfopenllm_v2/BBH": 0.4382, - "hfopenllm_v2/MATH Level 5": 0.0279, - "hfopenllm_v2/GPQA": 0.2919, - "hfopenllm_v2/MUSR": 0.4071, - "hfopenllm_v2/MMLU-PRO": 0.2651 - } - }, - { - "id": "princeton-nlp/Mistral-7B-Base-SFT-DPO", - "name": "Mistral-7B-Base-SFT-DPO", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4403, - "hfopenllm_v2/BBH": 0.435, - "hfopenllm_v2/MATH Level 5": 0.0211, - "hfopenllm_v2/GPQA": 0.2727, - "hfopenllm_v2/MUSR": 0.4122, - "hfopenllm_v2/MMLU-PRO": 0.2645 - } - }, - { - "id": "princeton-nlp/Mistral-7B-Base-SFT-IPO", - "name": "Mistral-7B-Base-SFT-IPO", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.483, - "hfopenllm_v2/BBH": 0.4458, - "hfopenllm_v2/MATH Level 5": 0.0287, - "hfopenllm_v2/GPQA": 0.2802, - "hfopenllm_v2/MUSR": 0.3776, - "hfopenllm_v2/MMLU-PRO": 0.2792 - } - }, - { - "id": "princeton-nlp/Mistral-7B-Base-SFT-KTO", - "name": "Mistral-7B-Base-SFT-KTO", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4785, - "hfopenllm_v2/BBH": 0.4476, - "hfopenllm_v2/MATH Level 5": 0.0393, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.4368, - "hfopenllm_v2/MMLU-PRO": 0.2872 - } - }, - { - "id": "princeton-nlp/Mistral-7B-Base-SFT-RDPO", - "name": "Mistral-7B-Base-SFT-RDPO", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4606, - "hfopenllm_v2/BBH": 0.444, - "hfopenllm_v2/MATH Level 5": 0.0219, - "hfopenllm_v2/GPQA": 0.2777, - "hfopenllm_v2/MUSR": 0.3579, - "hfopenllm_v2/MMLU-PRO": 0.2777 - } - }, - { - "id": "princeton-nlp/Mistral-7B-Base-SFT-RRHF", - "name": "Mistral-7B-Base-SFT-RRHF", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4407, - "hfopenllm_v2/BBH": 0.4281, - "hfopenllm_v2/MATH Level 5": 0.0249, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.4187, - "hfopenllm_v2/MMLU-PRO": 0.2398 - } - }, - { - "id": "princeton-nlp/Mistral-7B-Base-SFT-SimPO", - "name": "Mistral-7B-Base-SFT-SimPO", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4701, - "hfopenllm_v2/BBH": 0.4398, - "hfopenllm_v2/MATH Level 5": 0.0144, - "hfopenllm_v2/GPQA": 0.2836, - "hfopenllm_v2/MUSR": 0.3971, - "hfopenllm_v2/MMLU-PRO": 0.2702 - } - }, - { - "id": "princeton-nlp/Mistral-7B-Base-SFT-SLiC-HF", - "name": "Mistral-7B-Base-SFT-SLiC-HF", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5127, - "hfopenllm_v2/BBH": 0.4422, - "hfopenllm_v2/MATH Level 5": 0.0355, - "hfopenllm_v2/GPQA": 0.2919, - "hfopenllm_v2/MUSR": 0.4261, - "hfopenllm_v2/MMLU-PRO": 0.2781 - } - }, - { - "id": "princeton-nlp/Mistral-7B-Instruct-CPO", - "name": "Mistral-7B-Instruct-CPO", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4203, - "hfopenllm_v2/BBH": 0.4069, - "hfopenllm_v2/MATH Level 5": 0.0204, - "hfopenllm_v2/GPQA": 0.2659, - "hfopenllm_v2/MUSR": 0.4178, - "hfopenllm_v2/MMLU-PRO": 0.2701 - } - }, - { - "id": "princeton-nlp/Mistral-7B-Instruct-DPO", - "name": "Mistral-7B-Instruct-DPO", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5176, - "hfopenllm_v2/BBH": 0.406, - "hfopenllm_v2/MATH Level 5": 0.031, - "hfopenllm_v2/GPQA": 0.2685, - "hfopenllm_v2/MUSR": 0.3833, - "hfopenllm_v2/MMLU-PRO": 0.2749 - } - }, - { - "id": "princeton-nlp/Mistral-7B-Instruct-IPO", - "name": "Mistral-7B-Instruct-IPO", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4929, - "hfopenllm_v2/BBH": 0.4322, - "hfopenllm_v2/MATH Level 5": 0.0204, - "hfopenllm_v2/GPQA": 0.2735, - "hfopenllm_v2/MUSR": 0.4324, - "hfopenllm_v2/MMLU-PRO": 0.2708 - } - }, - { - "id": "princeton-nlp/Mistral-7B-Instruct-KTO", - "name": "Mistral-7B-Instruct-KTO", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4908, - "hfopenllm_v2/BBH": 0.414, - "hfopenllm_v2/MATH Level 5": 0.0264, - "hfopenllm_v2/GPQA": 0.2735, - "hfopenllm_v2/MUSR": 0.3953, - "hfopenllm_v2/MMLU-PRO": 0.2812 - } - }, - { - "id": "princeton-nlp/Mistral-7B-Instruct-ORPO", - "name": "Mistral-7B-Instruct-ORPO", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.472, - "hfopenllm_v2/BBH": 0.4104, - "hfopenllm_v2/MATH Level 5": 0.0295, - "hfopenllm_v2/GPQA": 0.2743, - "hfopenllm_v2/MUSR": 0.3912, - "hfopenllm_v2/MMLU-PRO": 0.2662 - } - }, - { - "id": "princeton-nlp/Mistral-7B-Instruct-RDPO", - "name": "Mistral-7B-Instruct-RDPO", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4887, - "hfopenllm_v2/BBH": 0.405, - "hfopenllm_v2/MATH Level 5": 0.0249, - "hfopenllm_v2/GPQA": 0.2802, - "hfopenllm_v2/MUSR": 0.3873, - "hfopenllm_v2/MMLU-PRO": 0.2777 - } - }, - { - "id": "princeton-nlp/Mistral-7B-Instruct-RRHF", - "name": "Mistral-7B-Instruct-RRHF", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.496, - "hfopenllm_v2/BBH": 0.419, - "hfopenllm_v2/MATH Level 5": 0.0279, - "hfopenllm_v2/GPQA": 0.276, - "hfopenllm_v2/MUSR": 0.3979, - "hfopenllm_v2/MMLU-PRO": 0.2651 - } - }, - { - "id": "princeton-nlp/Mistral-7B-Instruct-SimPO", - "name": "Mistral-7B-Instruct-SimPO", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4687, - "hfopenllm_v2/BBH": 0.4507, - "hfopenllm_v2/MATH Level 5": 0.0287, - "hfopenllm_v2/GPQA": 0.2785, - "hfopenllm_v2/MUSR": 0.4098, - "hfopenllm_v2/MMLU-PRO": 0.2797 - } - }, - { - "id": "princeton-nlp/Mistral-7B-Instruct-SLiC-HF", - "name": "Mistral-7B-Instruct-SLiC-HF", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5115, - "hfopenllm_v2/BBH": 0.404, - "hfopenllm_v2/MATH Level 5": 0.0174, - "hfopenllm_v2/GPQA": 0.2727, - "hfopenllm_v2/MUSR": 0.3913, - "hfopenllm_v2/MMLU-PRO": 0.2715 - } - }, - { - "id": "princeton-nlp/Sheared-LLaMA-1.3B", - "name": "Sheared-LLaMA-1.3B", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2198, - "hfopenllm_v2/BBH": 0.3197, - "hfopenllm_v2/MATH Level 5": 0.0128, - "hfopenllm_v2/GPQA": 0.2399, - "hfopenllm_v2/MUSR": 0.3713, - "hfopenllm_v2/MMLU-PRO": 0.1171 - } - }, - { - "id": "princeton-nlp/Sheared-LLaMA-2.7B", - "name": "Sheared-LLaMA-2.7B", - "developer": "princeton-nlp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2417, - "hfopenllm_v2/BBH": 0.3259, - "hfopenllm_v2/MATH Level 5": 0.0128, - "hfopenllm_v2/GPQA": 0.2752, - "hfopenllm_v2/MUSR": 0.3567, - "hfopenllm_v2/MMLU-PRO": 0.1187 - } - }, - { - "id": "prithivMLmods/Bellatrix-1.5B-xElite", - "name": "Bellatrix-1.5B-xElite", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1964, - "hfopenllm_v2/BBH": 0.3501, - "hfopenllm_v2/MATH Level 5": 0.287, - "hfopenllm_v2/GPQA": 0.2785, - "hfopenllm_v2/MUSR": 0.3619, - "hfopenllm_v2/MMLU-PRO": 0.1657 - } - }, - { - "id": "prithivMLmods/Bellatrix-Tiny-1.5B-R1", - "name": "Bellatrix-Tiny-1.5B-R1", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3352, - "hfopenllm_v2/BBH": 0.4022, - "hfopenllm_v2/MATH Level 5": 0.0604, - "hfopenllm_v2/GPQA": 0.2987, - "hfopenllm_v2/MUSR": 0.3683, - "hfopenllm_v2/MMLU-PRO": 0.2751 - } - }, - { - "id": "prithivMLmods/Bellatrix-Tiny-1B-v2", - "name": "Bellatrix-Tiny-1B-v2", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.151, - "hfopenllm_v2/BBH": 0.3268, - "hfopenllm_v2/MATH Level 5": 0.0287, - "hfopenllm_v2/GPQA": 0.2727, - "hfopenllm_v2/MUSR": 0.343, - "hfopenllm_v2/MMLU-PRO": 0.1493 - } - }, - { - "id": "prithivMLmods/Blaze-14B-xElite", - "name": "Blaze-14B-xElite", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0363, - "hfopenllm_v2/BBH": 0.6628, - "hfopenllm_v2/MATH Level 5": 0.3693, - "hfopenllm_v2/GPQA": 0.3943, - "hfopenllm_v2/MUSR": 0.4625, - "hfopenllm_v2/MMLU-PRO": 0.5111 - } - }, - { - "id": "prithivMLmods/Calcium-Opus-14B-Elite", - "name": "Calcium-Opus-14B-Elite", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6052, - "hfopenllm_v2/BBH": 0.6317, - "hfopenllm_v2/MATH Level 5": 0.4789, - "hfopenllm_v2/GPQA": 0.3742, - "hfopenllm_v2/MUSR": 0.486, - "hfopenllm_v2/MMLU-PRO": 0.5302 - } - }, - { - "id": "prithivMLmods/Calcium-Opus-14B-Elite-1M", - "name": "Calcium-Opus-14B-Elite-1M", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5613, - "hfopenllm_v2/BBH": 0.6329, - "hfopenllm_v2/MATH Level 5": 0.4456, - "hfopenllm_v2/GPQA": 0.3523, - "hfopenllm_v2/MUSR": 0.4676, - "hfopenllm_v2/MMLU-PRO": 0.5152 - } - }, - { - "id": "prithivMLmods/Calcium-Opus-14B-Elite-Stock", - "name": "Calcium-Opus-14B-Elite-Stock", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6143, - "hfopenllm_v2/BBH": 0.6329, - "hfopenllm_v2/MATH Level 5": 0.4668, - "hfopenllm_v2/GPQA": 0.3683, - "hfopenllm_v2/MUSR": 0.4808, - "hfopenllm_v2/MMLU-PRO": 0.5284 - } - }, - { - "id": "prithivMLmods/Calcium-Opus-14B-Elite2", - "name": "Calcium-Opus-14B-Elite2", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6176, - "hfopenllm_v2/BBH": 0.6318, - "hfopenllm_v2/MATH Level 5": 0.469, - "hfopenllm_v2/GPQA": 0.37, - "hfopenllm_v2/MUSR": 0.494, - "hfopenllm_v2/MMLU-PRO": 0.5301 - } - }, - { - "id": "prithivMLmods/Calcium-Opus-14B-Elite2-R1", - "name": "Calcium-Opus-14B-Elite2-R1", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6326, - "hfopenllm_v2/BBH": 0.6362, - "hfopenllm_v2/MATH Level 5": 0.3338, - "hfopenllm_v2/GPQA": 0.3909, - "hfopenllm_v2/MUSR": 0.49, - "hfopenllm_v2/MMLU-PRO": 0.5248 - } - }, - { - "id": "prithivMLmods/Calcium-Opus-14B-Elite3", - "name": "Calcium-Opus-14B-Elite3", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5428, - "hfopenllm_v2/BBH": 0.635, - "hfopenllm_v2/MATH Level 5": 0.4705, - "hfopenllm_v2/GPQA": 0.3708, - "hfopenllm_v2/MUSR": 0.4795, - "hfopenllm_v2/MMLU-PRO": 0.5335 - } - }, - { - "id": "prithivMLmods/Calcium-Opus-14B-Elite4", - "name": "Calcium-Opus-14B-Elite4", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6112, - "hfopenllm_v2/BBH": 0.6195, - "hfopenllm_v2/MATH Level 5": 0.3625, - "hfopenllm_v2/GPQA": 0.3557, - "hfopenllm_v2/MUSR": 0.4687, - "hfopenllm_v2/MMLU-PRO": 0.5149 - } - }, - { - "id": "prithivMLmods/Calcium-Opus-14B-Merge", - "name": "Calcium-Opus-14B-Merge", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4949, - "hfopenllm_v2/BBH": 0.6319, - "hfopenllm_v2/MATH Level 5": 0.4637, - "hfopenllm_v2/GPQA": 0.3708, - "hfopenllm_v2/MUSR": 0.4861, - "hfopenllm_v2/MMLU-PRO": 0.5356 - } - }, - { - "id": "prithivMLmods/Calcium-Opus-20B-v1", - "name": "Calcium-Opus-20B-v1", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3093, - "hfopenllm_v2/BBH": 0.599, - "hfopenllm_v2/MATH Level 5": 0.3618, - "hfopenllm_v2/GPQA": 0.3532, - "hfopenllm_v2/MUSR": 0.4943, - "hfopenllm_v2/MMLU-PRO": 0.4734 - } - }, - { - "id": "prithivMLmods/COCO-7B-Instruct-1M", - "name": "COCO-7B-Instruct-1M", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4743, - "hfopenllm_v2/BBH": 0.541, - "hfopenllm_v2/MATH Level 5": 0.3497, - "hfopenllm_v2/GPQA": 0.3079, - "hfopenllm_v2/MUSR": 0.4382, - "hfopenllm_v2/MMLU-PRO": 0.4186 - } - }, - { - "id": "prithivMLmods/Codepy-Deepthink-3B", - "name": "Codepy-Deepthink-3B", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4327, - "hfopenllm_v2/BBH": 0.4259, - "hfopenllm_v2/MATH Level 5": 0.1156, - "hfopenllm_v2/GPQA": 0.2794, - "hfopenllm_v2/MUSR": 0.331, - "hfopenllm_v2/MMLU-PRO": 0.309 - } - }, - { - "id": "prithivMLmods/Coma-II-14B", - "name": "Coma-II-14B", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4168, - "hfopenllm_v2/BBH": 0.6321, - "hfopenllm_v2/MATH Level 5": 0.5514, - "hfopenllm_v2/GPQA": 0.4002, - "hfopenllm_v2/MUSR": 0.5351, - "hfopenllm_v2/MMLU-PRO": 0.504 - } - }, - { - "id": "prithivMLmods/Condor-Opus-14B-Exp", - "name": "Condor-Opus-14B-Exp", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4043, - "hfopenllm_v2/BBH": 0.6154, - "hfopenllm_v2/MATH Level 5": 0.5227, - "hfopenllm_v2/GPQA": 0.3918, - "hfopenllm_v2/MUSR": 0.5194, - "hfopenllm_v2/MMLU-PRO": 0.5014 - } - }, - { - "id": "prithivMLmods/Cygnus-II-14B", - "name": "Cygnus-II-14B", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6184, - "hfopenllm_v2/BBH": 0.6661, - "hfopenllm_v2/MATH Level 5": 0.4396, - "hfopenllm_v2/GPQA": 0.3876, - "hfopenllm_v2/MUSR": 0.4688, - "hfopenllm_v2/MMLU-PRO": 0.5391 - } - }, - { - "id": "prithivMLmods/Deepthink-Llama-3-8B-Preview", - "name": "Deepthink-Llama-3-8B-Preview", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2955, - "hfopenllm_v2/BBH": 0.4665, - "hfopenllm_v2/MATH Level 5": 0.355, - "hfopenllm_v2/GPQA": 0.3163, - "hfopenllm_v2/MUSR": 0.3707, - "hfopenllm_v2/MMLU-PRO": 0.2739 - } - }, - { - "id": "prithivMLmods/Deepthink-Reasoning-14B", - "name": "Deepthink-Reasoning-14B", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5424, - "hfopenllm_v2/BBH": 0.6334, - "hfopenllm_v2/MATH Level 5": 0.423, - "hfopenllm_v2/GPQA": 0.3666, - "hfopenllm_v2/MUSR": 0.4732, - "hfopenllm_v2/MMLU-PRO": 0.5296 - } - }, - { - "id": "prithivMLmods/Deepthink-Reasoning-7B", - "name": "Deepthink-Reasoning-7B", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.484, - "hfopenllm_v2/BBH": 0.5505, - "hfopenllm_v2/MATH Level 5": 0.3346, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.4432, - "hfopenllm_v2/MMLU-PRO": 0.4349 - } - }, - { - "id": "prithivMLmods/Dinobot-Opus-14B-Exp", - "name": "Dinobot-Opus-14B-Exp", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.824, - "hfopenllm_v2/BBH": 0.637, - "hfopenllm_v2/MATH Level 5": 0.5317, - "hfopenllm_v2/GPQA": 0.3247, - "hfopenllm_v2/MUSR": 0.426, - "hfopenllm_v2/MMLU-PRO": 0.4979 - } - }, - { - "id": "prithivMLmods/Elita-0.1-Distilled-R1-abliterated", - "name": "Elita-0.1-Distilled-R1-abliterated", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3542, - "hfopenllm_v2/BBH": 0.3828, - "hfopenllm_v2/MATH Level 5": 0.3066, - "hfopenllm_v2/GPQA": 0.2659, - "hfopenllm_v2/MUSR": 0.366, - "hfopenllm_v2/MMLU-PRO": 0.2758 - } - }, - { - "id": "prithivMLmods/Elita-1", - "name": "Elita-1", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4906, - "hfopenllm_v2/BBH": 0.652, - "hfopenllm_v2/MATH Level 5": 0.3429, - "hfopenllm_v2/GPQA": 0.3758, - "hfopenllm_v2/MUSR": 0.4834, - "hfopenllm_v2/MMLU-PRO": 0.5381 - } - }, - { - "id": "prithivMLmods/Epimetheus-14B-Axo", - "name": "Epimetheus-14B-Axo", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5546, - "hfopenllm_v2/BBH": 0.6613, - "hfopenllm_v2/MATH Level 5": 0.4101, - "hfopenllm_v2/GPQA": 0.3926, - "hfopenllm_v2/MUSR": 0.482, - "hfopenllm_v2/MMLU-PRO": 0.5304 - } - }, - { - "id": "prithivMLmods/Equuleus-Opus-14B-Exp", - "name": "Equuleus-Opus-14B-Exp", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7001, - "hfopenllm_v2/BBH": 0.6434, - "hfopenllm_v2/MATH Level 5": 0.4585, - "hfopenllm_v2/GPQA": 0.3867, - "hfopenllm_v2/MUSR": 0.4952, - "hfopenllm_v2/MMLU-PRO": 0.5374 - } - }, - { - "id": "prithivMLmods/Eridanus-Opus-14B-r999", - "name": "Eridanus-Opus-14B-r999", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6386, - "hfopenllm_v2/BBH": 0.6584, - "hfopenllm_v2/MATH Level 5": 0.386, - "hfopenllm_v2/GPQA": 0.3943, - "hfopenllm_v2/MUSR": 0.4769, - "hfopenllm_v2/MMLU-PRO": 0.5362 - } - }, - { - "id": "prithivMLmods/Evac-Opus-14B-Exp", - "name": "Evac-Opus-14B-Exp", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5916, - "hfopenllm_v2/BBH": 0.6475, - "hfopenllm_v2/MATH Level 5": 0.4215, - "hfopenllm_v2/GPQA": 0.3884, - "hfopenllm_v2/MUSR": 0.4728, - "hfopenllm_v2/MMLU-PRO": 0.5317 - } - }, - { - "id": "prithivMLmods/FastThink-0.5B-Tiny", - "name": "FastThink-0.5B-Tiny", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.258, - "hfopenllm_v2/BBH": 0.3206, - "hfopenllm_v2/MATH Level 5": 0.0204, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.3566, - "hfopenllm_v2/MMLU-PRO": 0.1649 - } - }, - { - "id": "prithivMLmods/Gaea-Opus-14B-Exp", - "name": "Gaea-Opus-14B-Exp", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5956, - "hfopenllm_v2/BBH": 0.656, - "hfopenllm_v2/MATH Level 5": 0.4275, - "hfopenllm_v2/GPQA": 0.3909, - "hfopenllm_v2/MUSR": 0.4859, - "hfopenllm_v2/MMLU-PRO": 0.5401 - } - }, - { - "id": "prithivMLmods/Galactic-Qwen-14B-Exp1", - "name": "Galactic-Qwen-14B-Exp1", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5832, - "hfopenllm_v2/BBH": 0.6582, - "hfopenllm_v2/MATH Level 5": 0.4018, - "hfopenllm_v2/GPQA": 0.3935, - "hfopenllm_v2/MUSR": 0.4781, - "hfopenllm_v2/MMLU-PRO": 0.5396 - } - }, - { - "id": "prithivMLmods/Galactic-Qwen-14B-Exp2", - "name": "Galactic-Qwen-14B-Exp2", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.662, - "hfopenllm_v2/BBH": 0.7203, - "hfopenllm_v2/MATH Level 5": 0.3474, - "hfopenllm_v2/GPQA": 0.3993, - "hfopenllm_v2/MUSR": 0.5354, - "hfopenllm_v2/MMLU-PRO": 0.5691 - } - }, - { - "id": "prithivMLmods/Gauss-Opus-14B-R999", - "name": "Gauss-Opus-14B-R999", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3907, - "hfopenllm_v2/BBH": 0.6228, - "hfopenllm_v2/MATH Level 5": 0.5755, - "hfopenllm_v2/GPQA": 0.3918, - "hfopenllm_v2/MUSR": 0.5338, - "hfopenllm_v2/MMLU-PRO": 0.5007 - } - }, - { - "id": "prithivMLmods/GWQ-9B-Preview", - "name": "GWQ-9B-Preview", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5066, - "hfopenllm_v2/BBH": 0.5806, - "hfopenllm_v2/MATH Level 5": 0.2266, - "hfopenllm_v2/GPQA": 0.3398, - "hfopenllm_v2/MUSR": 0.4951, - "hfopenllm_v2/MMLU-PRO": 0.3984 - } - }, - { - "id": "prithivMLmods/GWQ-9B-Preview2", - "name": "GWQ-9B-Preview2", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5209, - "hfopenllm_v2/BBH": 0.5797, - "hfopenllm_v2/MATH Level 5": 0.2372, - "hfopenllm_v2/GPQA": 0.3263, - "hfopenllm_v2/MUSR": 0.486, - "hfopenllm_v2/MMLU-PRO": 0.3997 - } - }, - { - "id": "prithivMLmods/GWQ2b", - "name": "GWQ2b", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4115, - "hfopenllm_v2/BBH": 0.4143, - "hfopenllm_v2/MATH Level 5": 0.0627, - "hfopenllm_v2/GPQA": 0.2827, - "hfopenllm_v2/MUSR": 0.4311, - "hfopenllm_v2/MMLU-PRO": 0.2473 - } - }, - { - "id": "prithivMLmods/Jolt-v0.1", - "name": "Jolt-v0.1", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5092, - "hfopenllm_v2/BBH": 0.6521, - "hfopenllm_v2/MATH Level 5": 0.3565, - "hfopenllm_v2/GPQA": 0.38, - "hfopenllm_v2/MUSR": 0.4847, - "hfopenllm_v2/MMLU-PRO": 0.5386 - } - }, - { - "id": "prithivMLmods/Lacerta-Opus-14B-Elite8", - "name": "Lacerta-Opus-14B-Elite8", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6141, - "hfopenllm_v2/BBH": 0.6401, - "hfopenllm_v2/MATH Level 5": 0.3648, - "hfopenllm_v2/GPQA": 0.3784, - "hfopenllm_v2/MUSR": 0.4635, - "hfopenllm_v2/MMLU-PRO": 0.5322 - } - }, - { - "id": "prithivMLmods/Llama-3.1-5B-Instruct", - "name": "Llama-3.1-5B-Instruct", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1407, - "hfopenllm_v2/BBH": 0.3051, - "hfopenllm_v2/MATH Level 5": 0.0151, - "hfopenllm_v2/GPQA": 0.2643, - "hfopenllm_v2/MUSR": 0.354, - "hfopenllm_v2/MMLU-PRO": 0.1184 - } - }, - { - "id": "prithivMLmods/Llama-3.1-8B-Open-SFT", - "name": "Llama-3.1-8B-Open-SFT", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4123, - "hfopenllm_v2/BBH": 0.4968, - "hfopenllm_v2/MATH Level 5": 0.1216, - "hfopenllm_v2/GPQA": 0.3096, - "hfopenllm_v2/MUSR": 0.3904, - "hfopenllm_v2/MMLU-PRO": 0.3522 - } - }, - { - "id": "prithivMLmods/Llama-3.2-3B-Math-Oct", - "name": "Llama-3.2-3B-Math-Oct", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4585, - "hfopenllm_v2/BBH": 0.4372, - "hfopenllm_v2/MATH Level 5": 0.1156, - "hfopenllm_v2/GPQA": 0.2584, - "hfopenllm_v2/MUSR": 0.347, - "hfopenllm_v2/MMLU-PRO": 0.2911 - } - }, - { - "id": "prithivMLmods/Llama-3.2-6B-AlgoCode", - "name": "Llama-3.2-6B-AlgoCode", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2136, - "hfopenllm_v2/BBH": 0.3748, - "hfopenllm_v2/MATH Level 5": 0.0136, - "hfopenllm_v2/GPQA": 0.2869, - "hfopenllm_v2/MUSR": 0.4013, - "hfopenllm_v2/MMLU-PRO": 0.1798 - } - }, - { - "id": "prithivMLmods/Llama-8B-Distill-CoT", - "name": "Llama-8B-Distill-CoT", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3342, - "hfopenllm_v2/BBH": 0.4298, - "hfopenllm_v2/MATH Level 5": 0.4003, - "hfopenllm_v2/GPQA": 0.2894, - "hfopenllm_v2/MUSR": 0.372, - "hfopenllm_v2/MMLU-PRO": 0.2732 - } - }, - { - "id": "prithivMLmods/Llama-Deepsync-1B", - "name": "Llama-Deepsync-1B", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.357, - "hfopenllm_v2/BBH": 0.3386, - "hfopenllm_v2/MATH Level 5": 0.0438, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3565, - "hfopenllm_v2/MMLU-PRO": 0.1738 - } - }, - { - "id": "prithivMLmods/Llama-Deepsync-3B", - "name": "Llama-Deepsync-3B", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4302, - "hfopenllm_v2/BBH": 0.4292, - "hfopenllm_v2/MATH Level 5": 0.1178, - "hfopenllm_v2/GPQA": 0.2718, - "hfopenllm_v2/MUSR": 0.3324, - "hfopenllm_v2/MMLU-PRO": 0.3031 - } - }, - { - "id": "prithivMLmods/Llama-Express.1-Math", - "name": "Llama-Express.1-Math", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5084, - "hfopenllm_v2/BBH": 0.3364, - "hfopenllm_v2/MATH Level 5": 0.0559, - "hfopenllm_v2/GPQA": 0.2634, - "hfopenllm_v2/MUSR": 0.3143, - "hfopenllm_v2/MMLU-PRO": 0.161 - } - }, - { - "id": "prithivMLmods/LwQ-10B-Instruct", - "name": "LwQ-10B-Instruct", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3935, - "hfopenllm_v2/BBH": 0.5122, - "hfopenllm_v2/MATH Level 5": 0.04, - "hfopenllm_v2/GPQA": 0.3121, - "hfopenllm_v2/MUSR": 0.4544, - "hfopenllm_v2/MMLU-PRO": 0.3318 - } - }, - { - "id": "prithivMLmods/LwQ-Reasoner-10B", - "name": "LwQ-Reasoner-10B", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2941, - "hfopenllm_v2/BBH": 0.5866, - "hfopenllm_v2/MATH Level 5": 0.358, - "hfopenllm_v2/GPQA": 0.3465, - "hfopenllm_v2/MUSR": 0.4079, - "hfopenllm_v2/MMLU-PRO": 0.4147 - } - }, - { - "id": "prithivMLmods/Magellanic-Opus-14B-Exp", - "name": "Magellanic-Opus-14B-Exp", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6866, - "hfopenllm_v2/BBH": 0.6383, - "hfopenllm_v2/MATH Level 5": 0.3799, - "hfopenllm_v2/GPQA": 0.3742, - "hfopenllm_v2/MUSR": 0.4926, - "hfopenllm_v2/MMLU-PRO": 0.5273 - } - }, - { - "id": "prithivMLmods/Magellanic-Qwen-25B-R999", - "name": "Magellanic-Qwen-25B-R999", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1873, - "hfopenllm_v2/BBH": 0.2608, - "hfopenllm_v2/MATH Level 5": 0.0053, - "hfopenllm_v2/GPQA": 0.2508, - "hfopenllm_v2/MUSR": 0.3831, - "hfopenllm_v2/MMLU-PRO": 0.13 - } - }, - { - "id": "prithivMLmods/Megatron-Corpus-14B-Exp", - "name": "Megatron-Corpus-14B-Exp", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4983, - "hfopenllm_v2/BBH": 0.6355, - "hfopenllm_v2/MATH Level 5": 0.3429, - "hfopenllm_v2/GPQA": 0.3633, - "hfopenllm_v2/MUSR": 0.4767, - "hfopenllm_v2/MMLU-PRO": 0.526 - } - }, - { - "id": "prithivMLmods/Megatron-Corpus-14B-Exp.v2", - "name": "Megatron-Corpus-14B-Exp.v2", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.487, - "hfopenllm_v2/BBH": 0.6321, - "hfopenllm_v2/MATH Level 5": 0.2591, - "hfopenllm_v2/GPQA": 0.3423, - "hfopenllm_v2/MUSR": 0.449, - "hfopenllm_v2/MMLU-PRO": 0.481 - } - }, - { - "id": "prithivMLmods/Megatron-Opus-14B-2.0", - "name": "Megatron-Opus-14B-2.0", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6694, - "hfopenllm_v2/BBH": 0.6871, - "hfopenllm_v2/MATH Level 5": 0.2779, - "hfopenllm_v2/GPQA": 0.3591, - "hfopenllm_v2/MUSR": 0.414, - "hfopenllm_v2/MMLU-PRO": 0.517 - } - }, - { - "id": "prithivMLmods/Megatron-Opus-14B-2.1", - "name": "Megatron-Opus-14B-2.1", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0246, - "hfopenllm_v2/BBH": 0.6727, - "hfopenllm_v2/MATH Level 5": 0.2998, - "hfopenllm_v2/GPQA": 0.3834, - "hfopenllm_v2/MUSR": 0.4928, - "hfopenllm_v2/MMLU-PRO": 0.5174 - } - }, - { - "id": "prithivMLmods/Megatron-Opus-14B-Exp", - "name": "Megatron-Opus-14B-Exp", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4979, - "hfopenllm_v2/BBH": 0.6516, - "hfopenllm_v2/MATH Level 5": 0.3535, - "hfopenllm_v2/GPQA": 0.375, - "hfopenllm_v2/MUSR": 0.4887, - "hfopenllm_v2/MMLU-PRO": 0.5401 - } - }, - { - "id": "prithivMLmods/Megatron-Opus-14B-Stock", - "name": "Megatron-Opus-14B-Stock", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5174, - "hfopenllm_v2/BBH": 0.6412, - "hfopenllm_v2/MATH Level 5": 0.3346, - "hfopenllm_v2/GPQA": 0.375, - "hfopenllm_v2/MUSR": 0.482, - "hfopenllm_v2/MMLU-PRO": 0.5293 - } - }, - { - "id": "prithivMLmods/Megatron-Opus-7B-Exp", - "name": "Megatron-Opus-7B-Exp", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6017, - "hfopenllm_v2/BBH": 0.5367, - "hfopenllm_v2/MATH Level 5": 0.1971, - "hfopenllm_v2/GPQA": 0.3112, - "hfopenllm_v2/MUSR": 0.4186, - "hfopenllm_v2/MMLU-PRO": 0.39 - } - }, - { - "id": "prithivMLmods/Messier-Opus-14B-Elite7", - "name": "Messier-Opus-14B-Elite7", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7113, - "hfopenllm_v2/BBH": 0.6499, - "hfopenllm_v2/MATH Level 5": 0.4071, - "hfopenllm_v2/GPQA": 0.3909, - "hfopenllm_v2/MUSR": 0.4886, - "hfopenllm_v2/MMLU-PRO": 0.5404 - } - }, - { - "id": "prithivMLmods/Omni-Reasoner-Merged", - "name": "Omni-Reasoner-Merged", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4599, - "hfopenllm_v2/BBH": 0.5508, - "hfopenllm_v2/MATH Level 5": 0.3331, - "hfopenllm_v2/GPQA": 0.3037, - "hfopenllm_v2/MUSR": 0.4616, - "hfopenllm_v2/MMLU-PRO": 0.4364 - } - }, - { - "id": "prithivMLmods/Omni-Reasoner3-Merged", - "name": "Omni-Reasoner3-Merged", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4935, - "hfopenllm_v2/BBH": 0.4388, - "hfopenllm_v2/MATH Level 5": 0.1088, - "hfopenllm_v2/GPQA": 0.2643, - "hfopenllm_v2/MUSR": 0.3522, - "hfopenllm_v2/MMLU-PRO": 0.295 - } - }, - { - "id": "prithivMLmods/Pegasus-Opus-14B-Exp", - "name": "Pegasus-Opus-14B-Exp", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6982, - "hfopenllm_v2/BBH": 0.6548, - "hfopenllm_v2/MATH Level 5": 0.4086, - "hfopenllm_v2/GPQA": 0.3951, - "hfopenllm_v2/MUSR": 0.486, - "hfopenllm_v2/MMLU-PRO": 0.5412 - } - }, - { - "id": "prithivMLmods/Phi-4-Empathetic", - "name": "Phi-4-Empathetic", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0497, - "hfopenllm_v2/BBH": 0.6727, - "hfopenllm_v2/MATH Level 5": 0.2621, - "hfopenllm_v2/GPQA": 0.38, - "hfopenllm_v2/MUSR": 0.4991, - "hfopenllm_v2/MMLU-PRO": 0.5066 - } - }, - { - "id": "prithivMLmods/Phi-4-Math-IO", - "name": "Phi-4-Math-IO", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.059, - "hfopenllm_v2/BBH": 0.6668, - "hfopenllm_v2/MATH Level 5": 0.4577, - "hfopenllm_v2/GPQA": 0.3985, - "hfopenllm_v2/MUSR": 0.4873, - "hfopenllm_v2/MMLU-PRO": 0.5205 - } - }, - { - "id": "prithivMLmods/Phi-4-o1", - "name": "Phi-4-o1", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.029, - "hfopenllm_v2/BBH": 0.6689, - "hfopenllm_v2/MATH Level 5": 0.3995, - "hfopenllm_v2/GPQA": 0.3826, - "hfopenllm_v2/MUSR": 0.4978, - "hfopenllm_v2/MMLU-PRO": 0.5174 - } - }, - { - "id": "prithivMLmods/Phi-4-QwQ", - "name": "Phi-4-QwQ", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0559, - "hfopenllm_v2/BBH": 0.6696, - "hfopenllm_v2/MATH Level 5": 0.4577, - "hfopenllm_v2/GPQA": 0.3909, - "hfopenllm_v2/MUSR": 0.4651, - "hfopenllm_v2/MMLU-PRO": 0.5275 - } - }, - { - "id": "prithivMLmods/Phi-4-Super", - "name": "Phi-4-Super", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0481, - "hfopenllm_v2/BBH": 0.672, - "hfopenllm_v2/MATH Level 5": 0.3489, - "hfopenllm_v2/GPQA": 0.3943, - "hfopenllm_v2/MUSR": 0.5044, - "hfopenllm_v2/MMLU-PRO": 0.5266 - } - }, - { - "id": "prithivMLmods/Phi-4-Super-1", - "name": "Phi-4-Super-1", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0418, - "hfopenllm_v2/BBH": 0.6729, - "hfopenllm_v2/MATH Level 5": 0.352, - "hfopenllm_v2/GPQA": 0.3935, - "hfopenllm_v2/MUSR": 0.5017, - "hfopenllm_v2/MMLU-PRO": 0.5235 - } - }, - { - "id": "prithivMLmods/Phi-4-Super-o1", - "name": "Phi-4-Super-o1", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0418, - "hfopenllm_v2/BBH": 0.6729, - "hfopenllm_v2/MATH Level 5": 0.352, - "hfopenllm_v2/GPQA": 0.3935, - "hfopenllm_v2/MUSR": 0.5017, - "hfopenllm_v2/MMLU-PRO": 0.5235 - } - }, - { - "id": "prithivMLmods/Phi4-Super", - "name": "Phi4-Super", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0481, - "hfopenllm_v2/BBH": 0.672, - "hfopenllm_v2/MATH Level 5": 0.3489, - "hfopenllm_v2/GPQA": 0.3943, - "hfopenllm_v2/MUSR": 0.5044, - "hfopenllm_v2/MMLU-PRO": 0.5266 - } - }, - { - "id": "prithivMLmods/Porpoise-Opus-14B-Exp", - "name": "Porpoise-Opus-14B-Exp", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7098, - "hfopenllm_v2/BBH": 0.6519, - "hfopenllm_v2/MATH Level 5": 0.4041, - "hfopenllm_v2/GPQA": 0.3935, - "hfopenllm_v2/MUSR": 0.4926, - "hfopenllm_v2/MMLU-PRO": 0.5396 - } - }, - { - "id": "prithivMLmods/Primal-Opus-14B-Optimus-v1", - "name": "Primal-Opus-14B-Optimus-v1", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5013, - "hfopenllm_v2/BBH": 0.6419, - "hfopenllm_v2/MATH Level 5": 0.3384, - "hfopenllm_v2/GPQA": 0.3725, - "hfopenllm_v2/MUSR": 0.4847, - "hfopenllm_v2/MMLU-PRO": 0.5259 - } - }, - { - "id": "prithivMLmods/Primal-Opus-14B-Optimus-v2", - "name": "Primal-Opus-14B-Optimus-v2", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6404, - "hfopenllm_v2/BBH": 0.6544, - "hfopenllm_v2/MATH Level 5": 0.4207, - "hfopenllm_v2/GPQA": 0.3918, - "hfopenllm_v2/MUSR": 0.49, - "hfopenllm_v2/MMLU-PRO": 0.5422 - } - }, - { - "id": "prithivMLmods/Qwen-7B-Distill-Reasoner", - "name": "Qwen-7B-Distill-Reasoner", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3396, - "hfopenllm_v2/BBH": 0.4409, - "hfopenllm_v2/MATH Level 5": 0.395, - "hfopenllm_v2/GPQA": 0.3272, - "hfopenllm_v2/MUSR": 0.366, - "hfopenllm_v2/MMLU-PRO": 0.2818 - } - }, - { - "id": "prithivMLmods/Qwen2.5-1.5B-DeepSeek-R1-Instruct", - "name": "Qwen2.5-1.5B-DeepSeek-R1-Instruct", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1397, - "hfopenllm_v2/BBH": 0.2824, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.276, - "hfopenllm_v2/MUSR": 0.3724, - "hfopenllm_v2/MMLU-PRO": 0.1123 - } - }, - { - "id": "prithivMLmods/Qwen2.5-14B-DeepSeek-R1-1M", - "name": "Qwen2.5-14B-DeepSeek-R1-1M", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4193, - "hfopenllm_v2/BBH": 0.5935, - "hfopenllm_v2/MATH Level 5": 0.5128, - "hfopenllm_v2/GPQA": 0.3322, - "hfopenllm_v2/MUSR": 0.4606, - "hfopenllm_v2/MMLU-PRO": 0.4899 - } - }, - { - "id": "prithivMLmods/Qwen2.5-7B-DeepSeek-R1-1M", - "name": "Qwen2.5-7B-DeepSeek-R1-1M", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1861, - "hfopenllm_v2/BBH": 0.3126, - "hfopenllm_v2/MATH Level 5": 0.0151, - "hfopenllm_v2/GPQA": 0.2617, - "hfopenllm_v2/MUSR": 0.3417, - "hfopenllm_v2/MMLU-PRO": 0.1201 - } - }, - { - "id": "prithivMLmods/QwQ-LCoT-14B-Conversational", - "name": "QwQ-LCoT-14B-Conversational", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4047, - "hfopenllm_v2/BBH": 0.624, - "hfopenllm_v2/MATH Level 5": 0.4653, - "hfopenllm_v2/GPQA": 0.3498, - "hfopenllm_v2/MUSR": 0.4847, - "hfopenllm_v2/MMLU-PRO": 0.5278 - } - }, - { - "id": "prithivMLmods/QwQ-LCoT-3B-Instruct", - "name": "QwQ-LCoT-3B-Instruct", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4354, - "hfopenllm_v2/BBH": 0.4763, - "hfopenllm_v2/MATH Level 5": 0.2825, - "hfopenllm_v2/GPQA": 0.2819, - "hfopenllm_v2/MUSR": 0.4358, - "hfopenllm_v2/MMLU-PRO": 0.3582 - } - }, - { - "id": "prithivMLmods/QwQ-LCoT-7B-Instruct", - "name": "QwQ-LCoT-7B-Instruct", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4987, - "hfopenllm_v2/BBH": 0.5466, - "hfopenllm_v2/MATH Level 5": 0.3716, - "hfopenllm_v2/GPQA": 0.302, - "hfopenllm_v2/MUSR": 0.4802, - "hfopenllm_v2/MMLU-PRO": 0.4334 - } - }, - { - "id": "prithivMLmods/QwQ-LCoT1-Merged", - "name": "QwQ-LCoT1-Merged", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4751, - "hfopenllm_v2/BBH": 0.5481, - "hfopenllm_v2/MATH Level 5": 0.3731, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.4696, - "hfopenllm_v2/MMLU-PRO": 0.4358 - } - }, - { - "id": "prithivMLmods/QwQ-LCoT2-7B-Instruct", - "name": "QwQ-LCoT2-7B-Instruct", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5561, - "hfopenllm_v2/BBH": 0.5425, - "hfopenllm_v2/MATH Level 5": 0.327, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.4564, - "hfopenllm_v2/MMLU-PRO": 0.4342 - } - }, - { - "id": "prithivMLmods/QwQ-MathOct-7B", - "name": "QwQ-MathOct-7B", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4684, - "hfopenllm_v2/BBH": 0.5486, - "hfopenllm_v2/MATH Level 5": 0.2953, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.4601, - "hfopenllm_v2/MMLU-PRO": 0.433 - } - }, - { - "id": "prithivMLmods/QwQ-R1-Distill-1.5B-CoT", - "name": "QwQ-R1-Distill-1.5B-CoT", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2194, - "hfopenllm_v2/BBH": 0.3666, - "hfopenllm_v2/MATH Level 5": 0.3346, - "hfopenllm_v2/GPQA": 0.2861, - "hfopenllm_v2/MUSR": 0.3434, - "hfopenllm_v2/MMLU-PRO": 0.1913 - } - }, - { - "id": "prithivMLmods/QwQ-R1-Distill-7B-CoT", - "name": "QwQ-R1-Distill-7B-CoT", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.35, - "hfopenllm_v2/BBH": 0.4388, - "hfopenllm_v2/MATH Level 5": 0.4683, - "hfopenllm_v2/GPQA": 0.2936, - "hfopenllm_v2/MUSR": 0.3779, - "hfopenllm_v2/MMLU-PRO": 0.2804 - } - }, - { - "id": "prithivMLmods/SmolLM2-CoT-360M", - "name": "SmolLM2-CoT-360M", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2216, - "hfopenllm_v2/BBH": 0.3135, - "hfopenllm_v2/MATH Level 5": 0.0204, - "hfopenllm_v2/GPQA": 0.2366, - "hfopenllm_v2/MUSR": 0.3794, - "hfopenllm_v2/MMLU-PRO": 0.1085 - } - }, - { - "id": "prithivMLmods/Sombrero-Opus-14B-Elite5", - "name": "Sombrero-Opus-14B-Elite5", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7881, - "hfopenllm_v2/BBH": 0.6502, - "hfopenllm_v2/MATH Level 5": 0.5355, - "hfopenllm_v2/GPQA": 0.3364, - "hfopenllm_v2/MUSR": 0.4287, - "hfopenllm_v2/MMLU-PRO": 0.52 - } - }, - { - "id": "prithivMLmods/Sombrero-Opus-14B-Elite6", - "name": "Sombrero-Opus-14B-Elite6", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7226, - "hfopenllm_v2/BBH": 0.6488, - "hfopenllm_v2/MATH Level 5": 0.4079, - "hfopenllm_v2/GPQA": 0.3935, - "hfopenllm_v2/MUSR": 0.4886, - "hfopenllm_v2/MMLU-PRO": 0.539 - } - }, - { - "id": "prithivMLmods/Sombrero-Opus-14B-Sm1", - "name": "Sombrero-Opus-14B-Sm1", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3813, - "hfopenllm_v2/BBH": 0.6355, - "hfopenllm_v2/MATH Level 5": 0.5665, - "hfopenllm_v2/GPQA": 0.4035, - "hfopenllm_v2/MUSR": 0.5299, - "hfopenllm_v2/MMLU-PRO": 0.5125 - } - }, - { - "id": "prithivMLmods/Sombrero-Opus-14B-Sm2", - "name": "Sombrero-Opus-14B-Sm2", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4272, - "hfopenllm_v2/BBH": 0.6609, - "hfopenllm_v2/MATH Level 5": 0.4864, - "hfopenllm_v2/GPQA": 0.3884, - "hfopenllm_v2/MUSR": 0.5088, - "hfopenllm_v2/MMLU-PRO": 0.5345 - } - }, - { - "id": "prithivMLmods/Sombrero-Opus-14B-Sm4", - "name": "Sombrero-Opus-14B-Sm4", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4347, - "hfopenllm_v2/BBH": 0.6613, - "hfopenllm_v2/MATH Level 5": 0.4879, - "hfopenllm_v2/GPQA": 0.3951, - "hfopenllm_v2/MUSR": 0.5192, - "hfopenllm_v2/MMLU-PRO": 0.53 - } - }, - { - "id": "prithivMLmods/Sombrero-Opus-14B-Sm5", - "name": "Sombrero-Opus-14B-Sm5", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6852, - "hfopenllm_v2/BBH": 0.6564, - "hfopenllm_v2/MATH Level 5": 0.4094, - "hfopenllm_v2/GPQA": 0.3867, - "hfopenllm_v2/MUSR": 0.4806, - "hfopenllm_v2/MMLU-PRO": 0.54 - } - }, - { - "id": "prithivMLmods/Sqweeks-7B-Instruct", - "name": "Sqweeks-7B-Instruct", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2158, - "hfopenllm_v2/BBH": 0.4667, - "hfopenllm_v2/MATH Level 5": 0.5144, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.4476, - "hfopenllm_v2/MMLU-PRO": 0.3133 - } - }, - { - "id": "prithivMLmods/Tadpole-Opus-14B-Exp", - "name": "Tadpole-Opus-14B-Exp", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.575, - "hfopenllm_v2/BBH": 0.6369, - "hfopenllm_v2/MATH Level 5": 0.3134, - "hfopenllm_v2/GPQA": 0.3859, - "hfopenllm_v2/MUSR": 0.4728, - "hfopenllm_v2/MMLU-PRO": 0.5322 - } - }, - { - "id": "prithivMLmods/Taurus-Opus-7B", - "name": "Taurus-Opus-7B", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4223, - "hfopenllm_v2/BBH": 0.5367, - "hfopenllm_v2/MATH Level 5": 0.2168, - "hfopenllm_v2/GPQA": 0.3263, - "hfopenllm_v2/MUSR": 0.4399, - "hfopenllm_v2/MMLU-PRO": 0.3951 - } - }, - { - "id": "prithivMLmods/Triangulum-10B", - "name": "Triangulum-10B", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3229, - "hfopenllm_v2/BBH": 0.5968, - "hfopenllm_v2/MATH Level 5": 0.355, - "hfopenllm_v2/GPQA": 0.354, - "hfopenllm_v2/MUSR": 0.4172, - "hfopenllm_v2/MMLU-PRO": 0.4178 - } - }, - { - "id": "prithivMLmods/Triangulum-5B", - "name": "Triangulum-5B", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1283, - "hfopenllm_v2/BBH": 0.3124, - "hfopenllm_v2/MATH Level 5": 0.0106, - "hfopenllm_v2/GPQA": 0.255, - "hfopenllm_v2/MUSR": 0.3445, - "hfopenllm_v2/MMLU-PRO": 0.1223 - } - }, - { - "id": "prithivMLmods/Triangulum-v2-10B", - "name": "Triangulum-v2-10B", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6705, - "hfopenllm_v2/BBH": 0.6065, - "hfopenllm_v2/MATH Level 5": 0.2447, - "hfopenllm_v2/GPQA": 0.3372, - "hfopenllm_v2/MUSR": 0.4281, - "hfopenllm_v2/MMLU-PRO": 0.4466 - } - }, - { - "id": "prithivMLmods/Tucana-Opus-14B-r999", - "name": "Tucana-Opus-14B-r999", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6067, - "hfopenllm_v2/BBH": 0.6557, - "hfopenllm_v2/MATH Level 5": 0.4063, - "hfopenllm_v2/GPQA": 0.3918, - "hfopenllm_v2/MUSR": 0.473, - "hfopenllm_v2/MMLU-PRO": 0.5384 - } - }, - { - "id": "prithivMLmods/Tulu-MathLingo-8B", - "name": "Tulu-MathLingo-8B", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5589, - "hfopenllm_v2/BBH": 0.4659, - "hfopenllm_v2/MATH Level 5": 0.145, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.3864, - "hfopenllm_v2/MMLU-PRO": 0.3044 - } - }, - { - "id": "prithivMLmods/Viper-Coder-7B-Elite14", - "name": "Viper-Coder-7B-Elite14", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1488, - "hfopenllm_v2/BBH": 0.2829, - "hfopenllm_v2/MATH Level 5": 0.0106, - "hfopenllm_v2/GPQA": 0.255, - "hfopenllm_v2/MUSR": 0.3422, - "hfopenllm_v2/MMLU-PRO": 0.1089 - } - }, - { - "id": "prithivMLmods/Viper-Coder-Hybrid-v1.2", - "name": "Viper-Coder-Hybrid-v1.2", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6736, - "hfopenllm_v2/BBH": 0.6391, - "hfopenllm_v2/MATH Level 5": 0.3331, - "hfopenllm_v2/GPQA": 0.3742, - "hfopenllm_v2/MUSR": 0.4822, - "hfopenllm_v2/MMLU-PRO": 0.5243 - } - }, - { - "id": "prithivMLmods/Viper-Coder-Hybrid-v1.3", - "name": "Viper-Coder-Hybrid-v1.3", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7555, - "hfopenllm_v2/BBH": 0.6471, - "hfopenllm_v2/MATH Level 5": 0.4517, - "hfopenllm_v2/GPQA": 0.3381, - "hfopenllm_v2/MUSR": 0.4403, - "hfopenllm_v2/MMLU-PRO": 0.5097 - } - }, - { - "id": "prithivMLmods/Viper-Coder-HybridMini-v1.3", - "name": "Viper-Coder-HybridMini-v1.3", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6104, - "hfopenllm_v2/BBH": 0.5365, - "hfopenllm_v2/MATH Level 5": 0.463, - "hfopenllm_v2/GPQA": 0.3171, - "hfopenllm_v2/MUSR": 0.4505, - "hfopenllm_v2/MMLU-PRO": 0.4352 - } - }, - { - "id": "prithivMLmods/Viper-Coder-v0.1", - "name": "Viper-Coder-v0.1", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5521, - "hfopenllm_v2/BBH": 0.6143, - "hfopenllm_v2/MATH Level 5": 0.327, - "hfopenllm_v2/GPQA": 0.354, - "hfopenllm_v2/MUSR": 0.4394, - "hfopenllm_v2/MMLU-PRO": 0.3928 - } - }, - { - "id": "prithivMLmods/Viper-Coder-v1.1", - "name": "Viper-Coder-v1.1", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4432, - "hfopenllm_v2/BBH": 0.6492, - "hfopenllm_v2/MATH Level 5": 0.5461, - "hfopenllm_v2/GPQA": 0.401, - "hfopenllm_v2/MUSR": 0.5219, - "hfopenllm_v2/MMLU-PRO": 0.5232 - } - }, - { - "id": "prithivMLmods/Viper-Coder-v1.6-r999", - "name": "Viper-Coder-v1.6-r999", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4433, - "hfopenllm_v2/BBH": 0.6492, - "hfopenllm_v2/MATH Level 5": 0.5657, - "hfopenllm_v2/GPQA": 0.401, - "hfopenllm_v2/MUSR": 0.5219, - "hfopenllm_v2/MMLU-PRO": 0.5232 - } - }, - { - "id": "prithivMLmods/Viper-Coder-v1.7-Vsm6", - "name": "Viper-Coder-v1.7-Vsm6", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5004, - "hfopenllm_v2/BBH": 0.6502, - "hfopenllm_v2/MATH Level 5": 0.4645, - "hfopenllm_v2/GPQA": 0.3968, - "hfopenllm_v2/MUSR": 0.4768, - "hfopenllm_v2/MMLU-PRO": 0.5288 - } - }, - { - "id": "prithivMLmods/Viper-OneCoder-UIGEN", - "name": "Viper-OneCoder-UIGEN", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4692, - "hfopenllm_v2/BBH": 0.6047, - "hfopenllm_v2/MATH Level 5": 0.3867, - "hfopenllm_v2/GPQA": 0.3423, - "hfopenllm_v2/MUSR": 0.4514, - "hfopenllm_v2/MMLU-PRO": 0.3904 - } - }, - { - "id": "prithivMLmods/Volans-Opus-14B-Exp", - "name": "Volans-Opus-14B-Exp", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5868, - "hfopenllm_v2/BBH": 0.6521, - "hfopenllm_v2/MATH Level 5": 0.4252, - "hfopenllm_v2/GPQA": 0.3851, - "hfopenllm_v2/MUSR": 0.4872, - "hfopenllm_v2/MMLU-PRO": 0.5385 - } - }, - { - "id": "prithivMLmods/WebMind-7B-v0.1", - "name": "WebMind-7B-v0.1", - "developer": "prithivMLmods", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5278, - "hfopenllm_v2/BBH": 0.5434, - "hfopenllm_v2/MATH Level 5": 0.3648, - "hfopenllm_v2/GPQA": 0.3171, - "hfopenllm_v2/MUSR": 0.4537, - "hfopenllm_v2/MMLU-PRO": 0.4279 - } - }, - { - "id": "prometheus-eval/prometheus-7b-v2.0", - "name": "prometheus-eval/prometheus-7b-v2.0", - "developer": "prometheus-eval", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7204, - "reward-bench/Chat": 0.8547, - "reward-bench/Chat Hard": 0.4912, - "reward-bench/Safety": 0.7709, - "reward-bench/Reasoning": 0.7648 - } - }, - { - "id": "prometheus-eval/prometheus-8x7b-v2.0", - "name": "prometheus-eval/prometheus-8x7b-v2.0", - "developer": "prometheus-eval", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7451, - "reward-bench/Chat": 0.9302, - "reward-bench/Chat Hard": 0.4715, - "reward-bench/Safety": 0.8047, - "reward-bench/Reasoning": 0.774 - } - }, - { - "id": "pszemraj/Llama-3-6.3b-v0.1", - "name": "Llama-3-6.3b-v0.1", - "developer": "pszemraj", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1044, - "hfopenllm_v2/BBH": 0.4197, - "hfopenllm_v2/MATH Level 5": 0.0211, - "hfopenllm_v2/GPQA": 0.2836, - "hfopenllm_v2/MUSR": 0.3908, - "hfopenllm_v2/MMLU-PRO": 0.284 - } - }, - { - "id": "pszemraj/Mistral-v0.3-6B", - "name": "Mistral-v0.3-6B", - "developer": "pszemraj", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2454, - "hfopenllm_v2/BBH": 0.3774, - "hfopenllm_v2/MATH Level 5": 0.0136, - "hfopenllm_v2/GPQA": 0.2651, - "hfopenllm_v2/MUSR": 0.3908, - "hfopenllm_v2/MMLU-PRO": 0.2143 - } - }, - { - "id": "PuxAI/LUA_model", - "name": "LUA_model", - "developer": "PuxAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2282, - "hfopenllm_v2/BBH": 0.2877, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3484, - "hfopenllm_v2/MMLU-PRO": 0.1123 - } - }, - { - "id": "PygmalionAI/pygmalion-6b", - "name": "pygmalion-6b", - "developer": "PygmalionAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2091, - "hfopenllm_v2/BBH": 0.3199, - "hfopenllm_v2/MATH Level 5": 0.0083, - "hfopenllm_v2/GPQA": 0.2492, - "hfopenllm_v2/MUSR": 0.3684, - "hfopenllm_v2/MMLU-PRO": 0.1184 - } - }, - { - "id": "Q-bert/MetaMath-1B", - "name": "MetaMath-1B", - "developer": "Q-bert", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.53, - "hfopenllm_v2/BBH": 0.3451, - "hfopenllm_v2/MATH Level 5": 0.0627, - "hfopenllm_v2/GPQA": 0.2517, - "hfopenllm_v2/MUSR": 0.3289, - "hfopenllm_v2/MMLU-PRO": 0.1495 - } - }, - { - "id": "qingy2019/LLaMa_3.2_3B_Catalysts", - "name": "LLaMa_3.2_3B_Catalysts", - "developer": "qingy2019", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4992, - "hfopenllm_v2/BBH": 0.4468, - "hfopenllm_v2/MATH Level 5": 0.1292, - "hfopenllm_v2/GPQA": 0.2886, - "hfopenllm_v2/MUSR": 0.3788, - "hfopenllm_v2/MMLU-PRO": 0.3008 - } - }, - { - "id": "qingy2019/OpenMath2-Llama3.1-8B", - "name": "OpenMath2-Llama3.1-8B", - "developer": "qingy2019", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2331, - "hfopenllm_v2/BBH": 0.4096, - "hfopenllm_v2/MATH Level 5": 0.2674, - "hfopenllm_v2/GPQA": 0.2651, - "hfopenllm_v2/MUSR": 0.3436, - "hfopenllm_v2/MMLU-PRO": 0.1553 - } - }, - { - "id": "qingy2019/Oracle-14B", - "name": "Oracle-14B", - "developer": "qingy2019", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2358, - "hfopenllm_v2/BBH": 0.4612, - "hfopenllm_v2/MATH Level 5": 0.0642, - "hfopenllm_v2/GPQA": 0.2576, - "hfopenllm_v2/MUSR": 0.3717, - "hfopenllm_v2/MMLU-PRO": 0.2382 - } - }, - { - "id": "qingy2019/Qwen2.5-Math-14B-Instruct", - "name": "Qwen2.5-Math-14B-Instruct", - "developer": "qingy2019", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6066, - "hfopenllm_v2/BBH": 0.635, - "hfopenllm_v2/MATH Level 5": 0.3716, - "hfopenllm_v2/GPQA": 0.3725, - "hfopenllm_v2/MUSR": 0.4757, - "hfopenllm_v2/MMLU-PRO": 0.5331 - } - }, - { - "id": "qingy2019/Qwen2.5-Math-14B-Instruct-Alpha", - "name": "Qwen2.5-Math-14B-Instruct-Alpha", - "developer": "qingy2019", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5981, - "hfopenllm_v2/BBH": 0.6375, - "hfopenllm_v2/MATH Level 5": 0.3142, - "hfopenllm_v2/GPQA": 0.37, - "hfopenllm_v2/MUSR": 0.4649, - "hfopenllm_v2/MMLU-PRO": 0.5331 - } - }, - { - "id": "qingy2019/Qwen2.5-Math-14B-Instruct-Pro", - "name": "Qwen2.5-Math-14B-Instruct-Pro", - "developer": "qingy2019", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1922, - "hfopenllm_v2/BBH": 0.5319, - "hfopenllm_v2/MATH Level 5": 0.284, - "hfopenllm_v2/GPQA": 0.3112, - "hfopenllm_v2/MUSR": 0.374, - "hfopenllm_v2/MMLU-PRO": 0.3558 - } - }, - { - "id": "qingy2019/Qwen2.5-Ultimate-14B-Instruct", - "name": "Qwen2.5-Ultimate-14B-Instruct", - "developer": "qingy2019", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3938, - "hfopenllm_v2/BBH": 0.5842, - "hfopenllm_v2/MATH Level 5": 0.2893, - "hfopenllm_v2/GPQA": 0.3565, - "hfopenllm_v2/MUSR": 0.4135, - "hfopenllm_v2/MMLU-PRO": 0.4929 - } - }, - { - "id": "qingy2024/Benchmaxx-Llama-3.2-1B-Instruct", - "name": "Benchmaxx-Llama-3.2-1B-Instruct", - "developer": "qingy2024", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2014, - "hfopenllm_v2/BBH": 0.8269, - "hfopenllm_v2/MATH Level 5": 0.4804, - "hfopenllm_v2/GPQA": 0.2836, - "hfopenllm_v2/MUSR": 0.3446, - "hfopenllm_v2/MMLU-PRO": 0.1113 - } - }, - { - "id": "qingy2024/Eyas-17B-Instruct", - "name": "Eyas-17B-Instruct", - "developer": "qingy2024", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6575, - "hfopenllm_v2/BBH": 0.6085, - "hfopenllm_v2/MATH Level 5": 0.247, - "hfopenllm_v2/GPQA": 0.3146, - "hfopenllm_v2/MUSR": 0.4522, - "hfopenllm_v2/MMLU-PRO": 0.4343 - } - }, - { - "id": "qingy2024/Falcon3-2x10B-MoE-Instruct", - "name": "Falcon3-2x10B-MoE-Instruct", - "developer": "qingy2024", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.785, - "hfopenllm_v2/BBH": 0.6185, - "hfopenllm_v2/MATH Level 5": 0.2795, - "hfopenllm_v2/GPQA": 0.3305, - "hfopenllm_v2/MUSR": 0.4284, - "hfopenllm_v2/MMLU-PRO": 0.4423 - } - }, - { - "id": "qingy2024/Fusion-14B-Instruct", - "name": "Fusion-14B-Instruct", - "developer": "qingy2024", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.726, - "hfopenllm_v2/BBH": 0.6396, - "hfopenllm_v2/MATH Level 5": 0.3369, - "hfopenllm_v2/GPQA": 0.3549, - "hfopenllm_v2/MUSR": 0.44, - "hfopenllm_v2/MMLU-PRO": 0.5044 - } - }, - { - "id": "qingy2024/Fusion2-14B-Instruct", - "name": "Fusion2-14B-Instruct", - "developer": "qingy2024", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6064, - "hfopenllm_v2/BBH": 0.6119, - "hfopenllm_v2/MATH Level 5": 0.3127, - "hfopenllm_v2/GPQA": 0.3448, - "hfopenllm_v2/MUSR": 0.4634, - "hfopenllm_v2/MMLU-PRO": 0.5051 - } - }, - { - "id": "qingy2024/Fusion4-14B-Instruct", - "name": "Fusion4-14B-Instruct", - "developer": "qingy2024", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7649, - "hfopenllm_v2/BBH": 0.6543, - "hfopenllm_v2/MATH Level 5": 0.3882, - "hfopenllm_v2/GPQA": 0.3305, - "hfopenllm_v2/MUSR": 0.4326, - "hfopenllm_v2/MMLU-PRO": 0.5194 - } - }, - { - "id": "qingy2024/OwO-14B-Instruct", - "name": "OwO-14B-Instruct", - "developer": "qingy2024", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1383, - "hfopenllm_v2/BBH": 0.6165, - "hfopenllm_v2/MATH Level 5": 0.4162, - "hfopenllm_v2/GPQA": 0.3641, - "hfopenllm_v2/MUSR": 0.4407, - "hfopenllm_v2/MMLU-PRO": 0.5181 - } - }, - { - "id": "qingy2024/Qwarkstar-4B", - "name": "Qwarkstar-4B", - "developer": "qingy2024", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1994, - "hfopenllm_v2/BBH": 0.4015, - "hfopenllm_v2/MATH Level 5": 0.0861, - "hfopenllm_v2/GPQA": 0.3247, - "hfopenllm_v2/MUSR": 0.4428, - "hfopenllm_v2/MMLU-PRO": 0.2425 - } - }, - { - "id": "qingy2024/Qwarkstar-4B-Instruct-Preview", - "name": "Qwarkstar-4B-Instruct-Preview", - "developer": "qingy2024", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5324, - "hfopenllm_v2/BBH": 0.4358, - "hfopenllm_v2/MATH Level 5": 0.1284, - "hfopenllm_v2/GPQA": 0.2802, - "hfopenllm_v2/MUSR": 0.3896, - "hfopenllm_v2/MMLU-PRO": 0.2502 - } - }, - { - "id": "qingy2024/Qwen2.5-4B", - "name": "Qwen2.5-4B", - "developer": "qingy2024", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2158, - "hfopenllm_v2/BBH": 0.4269, - "hfopenllm_v2/MATH Level 5": 0.0514, - "hfopenllm_v2/GPQA": 0.2911, - "hfopenllm_v2/MUSR": 0.461, - "hfopenllm_v2/MMLU-PRO": 0.2525 - } - }, - { - "id": "qingy2024/Qwen2.5-Coder-Draft-1.5B-Instruct", - "name": "Qwen2.5-Coder-Draft-1.5B-Instruct", - "developer": "qingy2024", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4125, - "hfopenllm_v2/BBH": 0.3837, - "hfopenllm_v2/MATH Level 5": 0.1579, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.358, - "hfopenllm_v2/MMLU-PRO": 0.2244 - } - }, - { - "id": "qingy2024/Qwen2.5-Math-14B-Instruct-Alpha", - "name": "Qwen2.5-Math-14B-Instruct-Alpha", - "developer": "qingy2024", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7704, - "hfopenllm_v2/BBH": 0.6465, - "hfopenllm_v2/MATH Level 5": 0.429, - "hfopenllm_v2/GPQA": 0.349, - "hfopenllm_v2/MUSR": 0.4021, - "hfopenllm_v2/MMLU-PRO": 0.4966 - } - }, - { - "id": "qingy2024/Qwen2.5-Math-14B-Instruct-Preview", - "name": "Qwen2.5-Math-14B-Instruct-Preview", - "developer": "qingy2024", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7826, - "hfopenllm_v2/BBH": 0.6294, - "hfopenllm_v2/MATH Level 5": 0.4758, - "hfopenllm_v2/GPQA": 0.3406, - "hfopenllm_v2/MUSR": 0.4115, - "hfopenllm_v2/MMLU-PRO": 0.4993 - } - }, - { - "id": "qingy2024/Qwen2.6-14B-Instruct", - "name": "Qwen2.6-14B-Instruct", - "developer": "qingy2024", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5811, - "hfopenllm_v2/BBH": 0.6394, - "hfopenllm_v2/MATH Level 5": 0.3051, - "hfopenllm_v2/GPQA": 0.3792, - "hfopenllm_v2/MUSR": 0.4569, - "hfopenllm_v2/MMLU-PRO": 0.5285 - } - }, - { - "id": "qingy2024/Qwen2.6-Math-14B-Instruct", - "name": "Qwen2.6-Math-14B-Instruct", - "developer": "qingy2024", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3862, - "hfopenllm_v2/BBH": 0.6324, - "hfopenllm_v2/MATH Level 5": 0.429, - "hfopenllm_v2/GPQA": 0.37, - "hfopenllm_v2/MUSR": 0.4759, - "hfopenllm_v2/MMLU-PRO": 0.5241 - } - }, - { - "id": "qingy2024/QwEnlarge-16B-Instruct", - "name": "QwEnlarge-16B-Instruct", - "developer": "qingy2024", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7802, - "hfopenllm_v2/BBH": 0.5949, - "hfopenllm_v2/MATH Level 5": 0.46, - "hfopenllm_v2/GPQA": 0.3331, - "hfopenllm_v2/MUSR": 0.4101, - "hfopenllm_v2/MMLU-PRO": 0.4476 - } - }, - { - "id": "qingy2024/QwQ-14B-Math-v0.2", - "name": "QwQ-14B-Math-v0.2", - "developer": "qingy2024", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3391, - "hfopenllm_v2/BBH": 0.5731, - "hfopenllm_v2/MATH Level 5": 0.4811, - "hfopenllm_v2/GPQA": 0.2626, - "hfopenllm_v2/MUSR": 0.4021, - "hfopenllm_v2/MMLU-PRO": 0.48 - } - }, - { - "id": "qq8933/OpenLongCoT-Base-Gemma2-2B", - "name": "OpenLongCoT-Base-Gemma2-2B", - "developer": "qq8933", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1965, - "hfopenllm_v2/BBH": 0.3106, - "hfopenllm_v2/MATH Level 5": 0.0234, - "hfopenllm_v2/GPQA": 0.2626, - "hfopenllm_v2/MUSR": 0.3222, - "hfopenllm_v2/MMLU-PRO": 0.1316 - } - }, - { - "id": "Quazim0t0/1up-14b", - "name": "1up-14b", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6888, - "hfopenllm_v2/BBH": 0.6921, - "hfopenllm_v2/MATH Level 5": 0.4162, - "hfopenllm_v2/GPQA": 0.3624, - "hfopenllm_v2/MUSR": 0.4583, - "hfopenllm_v2/MMLU-PRO": 0.5406 - } - }, - { - "id": "Quazim0t0/Adamant-14B-sce", - "name": "Adamant-14B-sce", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6858, - "hfopenllm_v2/BBH": 0.6859, - "hfopenllm_v2/MATH Level 5": 0.3988, - "hfopenllm_v2/GPQA": 0.3507, - "hfopenllm_v2/MUSR": 0.4558, - "hfopenllm_v2/MMLU-PRO": 0.5372 - } - }, - { - "id": "Quazim0t0/Alice-14B", - "name": "Alice-14B", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6836, - "hfopenllm_v2/BBH": 0.6938, - "hfopenllm_v2/MATH Level 5": 0.4569, - "hfopenllm_v2/GPQA": 0.3515, - "hfopenllm_v2/MUSR": 0.4479, - "hfopenllm_v2/MMLU-PRO": 0.5419 - } - }, - { - "id": "Quazim0t0/Alien-CoT-14B-sce", - "name": "Alien-CoT-14B-sce", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0749, - "hfopenllm_v2/BBH": 0.6395, - "hfopenllm_v2/MATH Level 5": 0.5204, - "hfopenllm_v2/GPQA": 0.3918, - "hfopenllm_v2/MUSR": 0.4785, - "hfopenllm_v2/MMLU-PRO": 0.517 - } - }, - { - "id": "Quazim0t0/Aura-8B-Linear", - "name": "Aura-8B-Linear", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7948, - "hfopenllm_v2/BBH": 0.5074, - "hfopenllm_v2/MATH Level 5": 0.1805, - "hfopenllm_v2/GPQA": 0.2693, - "hfopenllm_v2/MUSR": 0.3687, - "hfopenllm_v2/MMLU-PRO": 0.3801 - } - }, - { - "id": "Quazim0t0/bloom-14b-stock", - "name": "bloom-14b-stock", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6575, - "hfopenllm_v2/BBH": 0.6878, - "hfopenllm_v2/MATH Level 5": 0.4811, - "hfopenllm_v2/GPQA": 0.3314, - "hfopenllm_v2/MUSR": 0.431, - "hfopenllm_v2/MMLU-PRO": 0.5373 - } - }, - { - "id": "Quazim0t0/caramel-14B", - "name": "caramel-14B", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6745, - "hfopenllm_v2/BBH": 0.6919, - "hfopenllm_v2/MATH Level 5": 0.4713, - "hfopenllm_v2/GPQA": 0.3448, - "hfopenllm_v2/MUSR": 0.4454, - "hfopenllm_v2/MMLU-PRO": 0.5436 - } - }, - { - "id": "Quazim0t0/Casa-14b-sce", - "name": "Casa-14b-sce", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6654, - "hfopenllm_v2/BBH": 0.6901, - "hfopenllm_v2/MATH Level 5": 0.4698, - "hfopenllm_v2/GPQA": 0.3331, - "hfopenllm_v2/MUSR": 0.431, - "hfopenllm_v2/MMLU-PRO": 0.5426 - } - }, - { - "id": "Quazim0t0/Charlie-8B-Linear", - "name": "Charlie-8B-Linear", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7381, - "hfopenllm_v2/BBH": 0.5141, - "hfopenllm_v2/MATH Level 5": 0.2651, - "hfopenllm_v2/GPQA": 0.271, - "hfopenllm_v2/MUSR": 0.3485, - "hfopenllm_v2/MMLU-PRO": 0.3573 - } - }, - { - "id": "Quazim0t0/Chromatic-8b-sce", - "name": "Chromatic-8b-sce", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5085, - "hfopenllm_v2/BBH": 0.5063, - "hfopenllm_v2/MATH Level 5": 0.1556, - "hfopenllm_v2/GPQA": 0.3196, - "hfopenllm_v2/MUSR": 0.4051, - "hfopenllm_v2/MMLU-PRO": 0.3755 - } - }, - { - "id": "Quazim0t0/CoT_Phi", - "name": "CoT_Phi", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6159, - "hfopenllm_v2/BBH": 0.6751, - "hfopenllm_v2/MATH Level 5": 0.3308, - "hfopenllm_v2/GPQA": 0.3582, - "hfopenllm_v2/MUSR": 0.4244, - "hfopenllm_v2/MMLU-PRO": 0.4901 - } - }, - { - "id": "Quazim0t0/Dyson-14b", - "name": "Dyson-14b", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5857, - "hfopenllm_v2/BBH": 0.6863, - "hfopenllm_v2/MATH Level 5": 0.5393, - "hfopenllm_v2/GPQA": 0.3138, - "hfopenllm_v2/MUSR": 0.4259, - "hfopenllm_v2/MMLU-PRO": 0.5399 - } - }, - { - "id": "Quazim0t0/Edu-14B-Linear", - "name": "Edu-14B-Linear", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6158, - "hfopenllm_v2/BBH": 0.6758, - "hfopenllm_v2/MATH Level 5": 0.2447, - "hfopenllm_v2/GPQA": 0.3171, - "hfopenllm_v2/MUSR": 0.4378, - "hfopenllm_v2/MMLU-PRO": 0.5086 - } - }, - { - "id": "Quazim0t0/Fugazi14b", - "name": "Fugazi14b", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6998, - "hfopenllm_v2/BBH": 0.6941, - "hfopenllm_v2/MATH Level 5": 0.4653, - "hfopenllm_v2/GPQA": 0.3515, - "hfopenllm_v2/MUSR": 0.4546, - "hfopenllm_v2/MMLU-PRO": 0.5417 - } - }, - { - "id": "Quazim0t0/Geedorah-14B", - "name": "Geedorah-14B", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6873, - "hfopenllm_v2/BBH": 0.6964, - "hfopenllm_v2/MATH Level 5": 0.4449, - "hfopenllm_v2/GPQA": 0.3473, - "hfopenllm_v2/MUSR": 0.4547, - "hfopenllm_v2/MMLU-PRO": 0.5421 - } - }, - { - "id": "Quazim0t0/GivingTree-8b-sce", - "name": "GivingTree-8b-sce", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5006, - "hfopenllm_v2/BBH": 0.504, - "hfopenllm_v2/MATH Level 5": 0.1526, - "hfopenllm_v2/GPQA": 0.3221, - "hfopenllm_v2/MUSR": 0.4051, - "hfopenllm_v2/MMLU-PRO": 0.3761 - } - }, - { - "id": "Quazim0t0/graphite-14b-sce", - "name": "graphite-14b-sce", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3217, - "hfopenllm_v2/BBH": 0.6631, - "hfopenllm_v2/MATH Level 5": 0.3006, - "hfopenllm_v2/GPQA": 0.2894, - "hfopenllm_v2/MUSR": 0.3981, - "hfopenllm_v2/MMLU-PRO": 0.528 - } - }, - { - "id": "Quazim0t0/GuiltySpark-14B-ties", - "name": "GuiltySpark-14B-ties", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6854, - "hfopenllm_v2/BBH": 0.6914, - "hfopenllm_v2/MATH Level 5": 0.3837, - "hfopenllm_v2/GPQA": 0.3649, - "hfopenllm_v2/MUSR": 0.4557, - "hfopenllm_v2/MMLU-PRO": 0.54 - } - }, - { - "id": "Quazim0t0/GZA-14B-sce", - "name": "GZA-14B-sce", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6274, - "hfopenllm_v2/BBH": 0.6687, - "hfopenllm_v2/MATH Level 5": 0.4721, - "hfopenllm_v2/GPQA": 0.302, - "hfopenllm_v2/MUSR": 0.4285, - "hfopenllm_v2/MMLU-PRO": 0.5232 - } - }, - { - "id": "Quazim0t0/Halo-14B-sce", - "name": "Halo-14B-sce", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6754, - "hfopenllm_v2/BBH": 0.6876, - "hfopenllm_v2/MATH Level 5": 0.429, - "hfopenllm_v2/GPQA": 0.3473, - "hfopenllm_v2/MUSR": 0.4401, - "hfopenllm_v2/MMLU-PRO": 0.5376 - } - }, - { - "id": "Quazim0t0/Heretic1.5b", - "name": "Heretic1.5b", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2062, - "hfopenllm_v2/BBH": 0.3529, - "hfopenllm_v2/MATH Level 5": 0.244, - "hfopenllm_v2/GPQA": 0.2685, - "hfopenllm_v2/MUSR": 0.3511, - "hfopenllm_v2/MMLU-PRO": 0.1728 - } - }, - { - "id": "Quazim0t0/Hyde-14b-sce", - "name": "Hyde-14b-sce", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6715, - "hfopenllm_v2/BBH": 0.6885, - "hfopenllm_v2/MATH Level 5": 0.2734, - "hfopenllm_v2/GPQA": 0.3414, - "hfopenllm_v2/MUSR": 0.4141, - "hfopenllm_v2/MMLU-PRO": 0.53 - } - }, - { - "id": "Quazim0t0/Imagine-v0.5-16bit", - "name": "Imagine-v0.5-16bit", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2759, - "hfopenllm_v2/BBH": 0.6769, - "hfopenllm_v2/MATH Level 5": 0.1397, - "hfopenllm_v2/GPQA": 0.3649, - "hfopenllm_v2/MUSR": 0.4349, - "hfopenllm_v2/MMLU-PRO": 0.5354 - } - }, - { - "id": "Quazim0t0/Imbue-14b", - "name": "Imbue-14b", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.52, - "hfopenllm_v2/BBH": 0.6845, - "hfopenllm_v2/MATH Level 5": 0.5317, - "hfopenllm_v2/GPQA": 0.3129, - "hfopenllm_v2/MUSR": 0.4167, - "hfopenllm_v2/MMLU-PRO": 0.5402 - } - }, - { - "id": "Quazim0t0/Insom", - "name": "Insom", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6818, - "hfopenllm_v2/BBH": 0.6881, - "hfopenllm_v2/MATH Level 5": 0.3852, - "hfopenllm_v2/GPQA": 0.3498, - "hfopenllm_v2/MUSR": 0.4311, - "hfopenllm_v2/MMLU-PRO": 0.5352 - } - }, - { - "id": "Quazim0t0/InspectorDeck-14B-sce", - "name": "InspectorDeck-14B-sce", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3241, - "hfopenllm_v2/BBH": 0.6668, - "hfopenllm_v2/MATH Level 5": 0.3165, - "hfopenllm_v2/GPQA": 0.297, - "hfopenllm_v2/MUSR": 0.3982, - "hfopenllm_v2/MMLU-PRO": 0.5261 - } - }, - { - "id": "Quazim0t0/Jekyl-8b-sce", - "name": "Jekyl-8b-sce", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4697, - "hfopenllm_v2/BBH": 0.4994, - "hfopenllm_v2/MATH Level 5": 0.1616, - "hfopenllm_v2/GPQA": 0.3381, - "hfopenllm_v2/MUSR": 0.4197, - "hfopenllm_v2/MMLU-PRO": 0.3686 - } - }, - { - "id": "Quazim0t0/Jigsaw-14B-Linear", - "name": "Jigsaw-14B-Linear", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.648, - "hfopenllm_v2/BBH": 0.6865, - "hfopenllm_v2/MATH Level 5": 0.2651, - "hfopenllm_v2/GPQA": 0.3406, - "hfopenllm_v2/MUSR": 0.4483, - "hfopenllm_v2/MMLU-PRO": 0.5234 - } - }, - { - "id": "Quazim0t0/Katana-8b-sce", - "name": "Katana-8b-sce", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5107, - "hfopenllm_v2/BBH": 0.5075, - "hfopenllm_v2/MATH Level 5": 0.1511, - "hfopenllm_v2/GPQA": 0.3247, - "hfopenllm_v2/MUSR": 0.4038, - "hfopenllm_v2/MMLU-PRO": 0.3771 - } - }, - { - "id": "Quazim0t0/Knot-CoT-14B-sce", - "name": "Knot-CoT-14B-sce", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4832, - "hfopenllm_v2/BBH": 0.6616, - "hfopenllm_v2/MATH Level 5": 0.3995, - "hfopenllm_v2/GPQA": 0.2936, - "hfopenllm_v2/MUSR": 0.414, - "hfopenllm_v2/MMLU-PRO": 0.5154 - } - }, - { - "id": "Quazim0t0/Lineage-14B", - "name": "Lineage-14B", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.707, - "hfopenllm_v2/BBH": 0.6934, - "hfopenllm_v2/MATH Level 5": 0.4245, - "hfopenllm_v2/GPQA": 0.3599, - "hfopenllm_v2/MUSR": 0.4597, - "hfopenllm_v2/MMLU-PRO": 0.5411 - } - }, - { - "id": "Quazim0t0/Lo-Phi-14b", - "name": "Lo-Phi-14b", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4941, - "hfopenllm_v2/BBH": 0.6852, - "hfopenllm_v2/MATH Level 5": 0.5196, - "hfopenllm_v2/GPQA": 0.328, - "hfopenllm_v2/MUSR": 0.4232, - "hfopenllm_v2/MMLU-PRO": 0.5369 - } - }, - { - "id": "Quazim0t0/Loke-14B-sce", - "name": "Loke-14B-sce", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6848, - "hfopenllm_v2/BBH": 0.6924, - "hfopenllm_v2/MATH Level 5": 0.3905, - "hfopenllm_v2/GPQA": 0.3649, - "hfopenllm_v2/MUSR": 0.4637, - "hfopenllm_v2/MMLU-PRO": 0.5401 - } - }, - { - "id": "Quazim0t0/Math_Phi4_Reason", - "name": "Math_Phi4_Reason", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.322, - "hfopenllm_v2/BBH": 0.624, - "hfopenllm_v2/MATH Level 5": 0.3278, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.4034, - "hfopenllm_v2/MMLU-PRO": 0.503 - } - }, - { - "id": "Quazim0t0/MFDOOM-14B", - "name": "MFDOOM-14B", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6736, - "hfopenllm_v2/BBH": 0.6916, - "hfopenllm_v2/MATH Level 5": 0.5264, - "hfopenllm_v2/GPQA": 0.323, - "hfopenllm_v2/MUSR": 0.4377, - "hfopenllm_v2/MMLU-PRO": 0.5426 - } - }, - { - "id": "Quazim0t0/MFGRIMM-14B", - "name": "MFGRIMM-14B", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6894, - "hfopenllm_v2/BBH": 0.6909, - "hfopenllm_v2/MATH Level 5": 0.506, - "hfopenllm_v2/GPQA": 0.3339, - "hfopenllm_v2/MUSR": 0.4361, - "hfopenllm_v2/MMLU-PRO": 0.5416 - } - }, - { - "id": "Quazim0t0/Mithril-14B-sce", - "name": "Mithril-14B-sce", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6958, - "hfopenllm_v2/BBH": 0.6926, - "hfopenllm_v2/MATH Level 5": 0.3822, - "hfopenllm_v2/GPQA": 0.3691, - "hfopenllm_v2/MUSR": 0.4611, - "hfopenllm_v2/MMLU-PRO": 0.5403 - } - }, - { - "id": "Quazim0t0/mocha-14B", - "name": "mocha-14B", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5893, - "hfopenllm_v2/BBH": 0.6895, - "hfopenllm_v2/MATH Level 5": 0.5264, - "hfopenllm_v2/GPQA": 0.3305, - "hfopenllm_v2/MUSR": 0.4272, - "hfopenllm_v2/MMLU-PRO": 0.5384 - } - }, - { - "id": "Quazim0t0/Mononoke-14B-sce", - "name": "Mononoke-14B-sce", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3502, - "hfopenllm_v2/BBH": 0.6744, - "hfopenllm_v2/MATH Level 5": 0.4698, - "hfopenllm_v2/GPQA": 0.323, - "hfopenllm_v2/MUSR": 0.4155, - "hfopenllm_v2/MMLU-PRO": 0.5298 - } - }, - { - "id": "Quazim0t0/mosaic-14b-sce", - "name": "mosaic-14b-sce", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6876, - "hfopenllm_v2/BBH": 0.6907, - "hfopenllm_v2/MATH Level 5": 0.4026, - "hfopenllm_v2/GPQA": 0.3624, - "hfopenllm_v2/MUSR": 0.4558, - "hfopenllm_v2/MMLU-PRO": 0.5396 - } - }, - { - "id": "Quazim0t0/Motion-8B-Linear", - "name": "Motion-8B-Linear", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7686, - "hfopenllm_v2/BBH": 0.5084, - "hfopenllm_v2/MATH Level 5": 0.1888, - "hfopenllm_v2/GPQA": 0.271, - "hfopenllm_v2/MUSR": 0.3606, - "hfopenllm_v2/MMLU-PRO": 0.3785 - } - }, - { - "id": "Quazim0t0/Mouse-9B", - "name": "Mouse-9B", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1325, - "hfopenllm_v2/BBH": 0.2979, - "hfopenllm_v2/MATH Level 5": 0.0053, - "hfopenllm_v2/GPQA": 0.2542, - "hfopenllm_v2/MUSR": 0.347, - "hfopenllm_v2/MMLU-PRO": 0.1139 - } - }, - { - "id": "Quazim0t0/Nova-14b-sce", - "name": "Nova-14b-sce", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7022, - "hfopenllm_v2/BBH": 0.6935, - "hfopenllm_v2/MATH Level 5": 0.4162, - "hfopenllm_v2/GPQA": 0.3633, - "hfopenllm_v2/MUSR": 0.4571, - "hfopenllm_v2/MMLU-PRO": 0.5413 - } - }, - { - "id": "Quazim0t0/NovaScotia-14b-stock", - "name": "NovaScotia-14b-stock", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6787, - "hfopenllm_v2/BBH": 0.6935, - "hfopenllm_v2/MATH Level 5": 0.463, - "hfopenllm_v2/GPQA": 0.349, - "hfopenllm_v2/MUSR": 0.4493, - "hfopenllm_v2/MMLU-PRO": 0.5409 - } - }, - { - "id": "Quazim0t0/Oasis-14B-ties", - "name": "Oasis-14B-ties", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6937, - "hfopenllm_v2/BBH": 0.6915, - "hfopenllm_v2/MATH Level 5": 0.3754, - "hfopenllm_v2/GPQA": 0.3649, - "hfopenllm_v2/MUSR": 0.4571, - "hfopenllm_v2/MMLU-PRO": 0.5405 - } - }, - { - "id": "Quazim0t0/ODB-14B-sce", - "name": "ODB-14B-sce", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2922, - "hfopenllm_v2/BBH": 0.6559, - "hfopenllm_v2/MATH Level 5": 0.2545, - "hfopenllm_v2/GPQA": 0.2659, - "hfopenllm_v2/MUSR": 0.3929, - "hfopenllm_v2/MMLU-PRO": 0.5207 - } - }, - { - "id": "Quazim0t0/Origami-14B-sce", - "name": "Origami-14B-sce", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3259, - "hfopenllm_v2/BBH": 0.662, - "hfopenllm_v2/MATH Level 5": 0.2915, - "hfopenllm_v2/GPQA": 0.2836, - "hfopenllm_v2/MUSR": 0.4035, - "hfopenllm_v2/MMLU-PRO": 0.5244 - } - }, - { - "id": "Quazim0t0/Phi4.Turn.R1Distill.16bit", - "name": "Phi4.Turn.R1Distill.16bit", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3126, - "hfopenllm_v2/BBH": 0.6563, - "hfopenllm_v2/MATH Level 5": 0.2311, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.3902, - "hfopenllm_v2/MMLU-PRO": 0.5257 - } - }, - { - "id": "Quazim0t0/Phi4.Turn.R1Distill_v1.5.1-Tensors", - "name": "Phi4.Turn.R1Distill_v1.5.1-Tensors", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2995, - "hfopenllm_v2/BBH": 0.6456, - "hfopenllm_v2/MATH Level 5": 0.219, - "hfopenllm_v2/GPQA": 0.2685, - "hfopenllm_v2/MUSR": 0.3929, - "hfopenllm_v2/MMLU-PRO": 0.5117 - } - }, - { - "id": "Quazim0t0/Phi4Basis-14B-sce", - "name": "Phi4Basis-14B-sce", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6502, - "hfopenllm_v2/BBH": 0.6909, - "hfopenllm_v2/MATH Level 5": 0.4789, - "hfopenllm_v2/GPQA": 0.3289, - "hfopenllm_v2/MUSR": 0.4338, - "hfopenllm_v2/MMLU-PRO": 0.539 - } - }, - { - "id": "Quazim0t0/Ponder-14B-linear", - "name": "Ponder-14B-linear", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6906, - "hfopenllm_v2/BBH": 0.6943, - "hfopenllm_v2/MATH Level 5": 0.4282, - "hfopenllm_v2/GPQA": 0.3582, - "hfopenllm_v2/MUSR": 0.4558, - "hfopenllm_v2/MMLU-PRO": 0.5408 - } - }, - { - "id": "Quazim0t0/Rosemary-14b", - "name": "Rosemary-14b", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6915, - "hfopenllm_v2/BBH": 0.6955, - "hfopenllm_v2/MATH Level 5": 0.4388, - "hfopenllm_v2/GPQA": 0.3565, - "hfopenllm_v2/MUSR": 0.4492, - "hfopenllm_v2/MMLU-PRO": 0.5396 - } - }, - { - "id": "Quazim0t0/Rune-14b", - "name": "Rune-14b", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7016, - "hfopenllm_v2/BBH": 0.6937, - "hfopenllm_v2/MATH Level 5": 0.4585, - "hfopenllm_v2/GPQA": 0.3515, - "hfopenllm_v2/MUSR": 0.4533, - "hfopenllm_v2/MMLU-PRO": 0.5411 - } - }, - { - "id": "Quazim0t0/RZA-14B-sce", - "name": "RZA-14B-sce", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4774, - "hfopenllm_v2/BBH": 0.6686, - "hfopenllm_v2/MATH Level 5": 0.5189, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.4113, - "hfopenllm_v2/MMLU-PRO": 0.5383 - } - }, - { - "id": "Quazim0t0/Sake-20b", - "name": "Sake-20b", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6693, - "hfopenllm_v2/BBH": 0.677, - "hfopenllm_v2/MATH Level 5": 0.4653, - "hfopenllm_v2/GPQA": 0.3188, - "hfopenllm_v2/MUSR": 0.4494, - "hfopenllm_v2/MMLU-PRO": 0.5391 - } - }, - { - "id": "Quazim0t0/Spok-14b-sce", - "name": "Spok-14b-sce", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6682, - "hfopenllm_v2/BBH": 0.6899, - "hfopenllm_v2/MATH Level 5": 0.2719, - "hfopenllm_v2/GPQA": 0.3456, - "hfopenllm_v2/MUSR": 0.4141, - "hfopenllm_v2/MMLU-PRO": 0.5298 - } - }, - { - "id": "Quazim0t0/Sumatra-20b", - "name": "Sumatra-20b", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6738, - "hfopenllm_v2/BBH": 0.6855, - "hfopenllm_v2/MATH Level 5": 0.3671, - "hfopenllm_v2/GPQA": 0.3263, - "hfopenllm_v2/MUSR": 0.456, - "hfopenllm_v2/MMLU-PRO": 0.5415 - } - }, - { - "id": "Quazim0t0/SuperNova14b", - "name": "SuperNova14b", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7076, - "hfopenllm_v2/BBH": 0.6937, - "hfopenllm_v2/MATH Level 5": 0.4396, - "hfopenllm_v2/GPQA": 0.3523, - "hfopenllm_v2/MUSR": 0.4545, - "hfopenllm_v2/MMLU-PRO": 0.5435 - } - }, - { - "id": "Quazim0t0/SZA-14B-sce", - "name": "SZA-14B-sce", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5659, - "hfopenllm_v2/BBH": 0.6889, - "hfopenllm_v2/MATH Level 5": 0.5242, - "hfopenllm_v2/GPQA": 0.3305, - "hfopenllm_v2/MUSR": 0.4339, - "hfopenllm_v2/MMLU-PRO": 0.5353 - } - }, - { - "id": "Quazim0t0/TB0-8B-sce", - "name": "TB0-8B-sce", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5107, - "hfopenllm_v2/BBH": 0.5075, - "hfopenllm_v2/MATH Level 5": 0.1511, - "hfopenllm_v2/GPQA": 0.3247, - "hfopenllm_v2/MUSR": 0.4038, - "hfopenllm_v2/MMLU-PRO": 0.3771 - } - }, - { - "id": "Quazim0t0/TBL-8B-sce", - "name": "TBL-8B-sce", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4581, - "hfopenllm_v2/BBH": 0.5008, - "hfopenllm_v2/MATH Level 5": 0.1533, - "hfopenllm_v2/GPQA": 0.3339, - "hfopenllm_v2/MUSR": 0.4236, - "hfopenllm_v2/MMLU-PRO": 0.3689 - } - }, - { - "id": "Quazim0t0/tesseract-14b-stock", - "name": "tesseract-14b-stock", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5848, - "hfopenllm_v2/BBH": 0.688, - "hfopenllm_v2/MATH Level 5": 0.5144, - "hfopenllm_v2/GPQA": 0.3272, - "hfopenllm_v2/MUSR": 0.4232, - "hfopenllm_v2/MMLU-PRO": 0.5389 - } - }, - { - "id": "Quazim0t0/ThinkPhi1.1-Tensors", - "name": "ThinkPhi1.1-Tensors", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3908, - "hfopenllm_v2/BBH": 0.6449, - "hfopenllm_v2/MATH Level 5": 0.182, - "hfopenllm_v2/GPQA": 0.2987, - "hfopenllm_v2/MUSR": 0.418, - "hfopenllm_v2/MMLU-PRO": 0.4908 - } - }, - { - "id": "Quazim0t0/time-14b-stock", - "name": "time-14b-stock", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6699, - "hfopenllm_v2/BBH": 0.6897, - "hfopenllm_v2/MATH Level 5": 0.5083, - "hfopenllm_v2/GPQA": 0.3347, - "hfopenllm_v2/MUSR": 0.4323, - "hfopenllm_v2/MMLU-PRO": 0.5419 - } - }, - { - "id": "Quazim0t0/Venti-20b", - "name": "Venti-20b", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6641, - "hfopenllm_v2/BBH": 0.6901, - "hfopenllm_v2/MATH Level 5": 0.3391, - "hfopenllm_v2/GPQA": 0.3322, - "hfopenllm_v2/MUSR": 0.448, - "hfopenllm_v2/MMLU-PRO": 0.5386 - } - }, - { - "id": "Quazim0t0/Venti-Blend-sce", - "name": "Venti-Blend-sce", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6879, - "hfopenllm_v2/BBH": 0.6843, - "hfopenllm_v2/MATH Level 5": 0.4056, - "hfopenllm_v2/GPQA": 0.3163, - "hfopenllm_v2/MUSR": 0.4389, - "hfopenllm_v2/MMLU-PRO": 0.5414 - } - }, - { - "id": "Quazim0t0/Vine-14b-sce", - "name": "Vine-14b-sce", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6733, - "hfopenllm_v2/BBH": 0.6891, - "hfopenllm_v2/MATH Level 5": 0.5008, - "hfopenllm_v2/GPQA": 0.3339, - "hfopenllm_v2/MUSR": 0.4323, - "hfopenllm_v2/MMLU-PRO": 0.5408 - } - }, - { - "id": "Quazim0t0/Wendy-14B", - "name": "Wendy-14B", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6772, - "hfopenllm_v2/BBH": 0.6958, - "hfopenllm_v2/MATH Level 5": 0.4834, - "hfopenllm_v2/GPQA": 0.3322, - "hfopenllm_v2/MUSR": 0.4428, - "hfopenllm_v2/MMLU-PRO": 0.5435 - } - }, - { - "id": "Quazim0t0/Wu-14b-sce", - "name": "Wu-14b-sce", - "developer": "Quazim0t0", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6718, - "hfopenllm_v2/BBH": 0.6885, - "hfopenllm_v2/MATH Level 5": 0.2613, - "hfopenllm_v2/GPQA": 0.3465, - "hfopenllm_v2/MUSR": 0.4114, - "hfopenllm_v2/MMLU-PRO": 0.5293 - } - }, - { - "id": "Qwen/Qwen1.5-0.5B", - "name": "Qwen1.5-0.5B", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1706, - "hfopenllm_v2/BBH": 0.3154, - "hfopenllm_v2/MATH Level 5": 0.0174, - "hfopenllm_v2/GPQA": 0.2542, - "hfopenllm_v2/MUSR": 0.3616, - "hfopenllm_v2/MMLU-PRO": 0.1307 - } - }, - { - "id": "Qwen/Qwen1.5-0.5B-Chat", - "name": "Qwen/Qwen1.5-0.5B-Chat", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1807, - "hfopenllm_v2/BBH": 0.3167, - "hfopenllm_v2/MATH Level 5": 0.0068, - "hfopenllm_v2/GPQA": 0.2693, - "hfopenllm_v2/MUSR": 0.3837, - "hfopenllm_v2/MMLU-PRO": 0.1213, - "reward-bench/Score": 0.5298, - "reward-bench/Chat": 0.3547, - "reward-bench/Chat Hard": 0.6294, - "reward-bench/Safety": 0.5703, - "reward-bench/Reasoning": 0.5984, - "reward-bench/Prior Sets (0.5 weight)": 0.4629 - } - }, - { - "id": "Qwen/Qwen1.5-1.8B", - "name": "Qwen1.5-1.8B", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2154, - "hfopenllm_v2/BBH": 0.3476, - "hfopenllm_v2/MATH Level 5": 0.0317, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.3605, - "hfopenllm_v2/MMLU-PRO": 0.1882 - } - }, - { - "id": "Qwen/Qwen1.5-1.8B-Chat", - "name": "Qwen/Qwen1.5-1.8B-Chat", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2019, - "hfopenllm_v2/BBH": 0.3256, - "hfopenllm_v2/MATH Level 5": 0.0196, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.426, - "hfopenllm_v2/MMLU-PRO": 0.1804, - "reward-bench/Score": 0.589, - "reward-bench/Chat": 0.5615, - "reward-bench/Chat Hard": 0.6031, - "reward-bench/Safety": 0.4838, - "reward-bench/Reasoning": 0.7793, - "reward-bench/Prior Sets (0.5 weight)": 0.4453 - } - }, - { - "id": "Qwen/Qwen1.5-110B", - "name": "Qwen1.5-110B", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3422, - "hfopenllm_v2/BBH": 0.61, - "hfopenllm_v2/MATH Level 5": 0.247, - "hfopenllm_v2/GPQA": 0.3523, - "hfopenllm_v2/MUSR": 0.4408, - "hfopenllm_v2/MMLU-PRO": 0.5361 - } - }, - { - "id": "qwen/qwen1.5-110b-chat", - "name": "Qwen1.5 Chat 110B", - "developer": "qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.55, - "helm_lite/NarrativeQA": 0.721, - "helm_lite/NaturalQuestions (closed-book)": 0.35, - "helm_lite/OpenbookQA": 0.922, - "helm_lite/MMLU": 0.704, - "helm_lite/MATH": 0.568, - "helm_lite/GSM8K": 0.815, - "helm_lite/LegalBench": 0.624, - "helm_lite/MedQA": 0.64, - "helm_lite/WMT 2014": 0.192, - "helm_mmlu/MMLU All Subjects": 0.768, - "helm_mmlu/Abstract Algebra": 0.57, - "helm_mmlu/Anatomy": 0.696, - "helm_mmlu/College Physics": 0.51, - "helm_mmlu/Computer Security": 0.82, - "helm_mmlu/Econometrics": 0.64, - "helm_mmlu/Global Facts": 0.51, - "helm_mmlu/Jurisprudence": 0.833, - "helm_mmlu/Philosophy": 0.823, - "helm_mmlu/Professional Psychology": 0.82, - "helm_mmlu/Us Foreign Policy": 0.87, - "helm_mmlu/Astronomy": 0.901, - "helm_mmlu/Business Ethics": 0.8, - "helm_mmlu/Clinical Knowledge": 0.766, - "helm_mmlu/Conceptual Physics": 0.838, - "helm_mmlu/Electrical Engineering": 0.752, - "helm_mmlu/Elementary Mathematics": 0.669, - "helm_mmlu/Formal Logic": 0.643, - "helm_mmlu/High School World History": 0.903, - "helm_mmlu/Human Sexuality": 0.855, - "helm_mmlu/International Law": 0.876, - "helm_mmlu/Logical Fallacies": 0.828, - "helm_mmlu/Machine Learning": 0.634, - "helm_mmlu/Management": 0.835, - "helm_mmlu/Marketing": 0.919, - "helm_mmlu/Medical Genetics": 0.85, - "helm_mmlu/Miscellaneous": 0.934, - "helm_mmlu/Moral Scenarios": 0.783, - "helm_mmlu/Nutrition": 0.804, - "helm_mmlu/Prehistory": 0.867, - "helm_mmlu/Public Relations": 0.773, - "helm_mmlu/Security Studies": 0.735, - "helm_mmlu/Sociology": 0.866, - "helm_mmlu/Virology": 0.542, - "helm_mmlu/World Religions": 0.871, - "helm_mmlu/Mean win rate": 0.875, - "hfopenllm_v2/IFEval": 0.5939, - "hfopenllm_v2/BBH": 0.6184, - "hfopenllm_v2/MATH Level 5": 0.2341, - "hfopenllm_v2/GPQA": 0.3414, - "hfopenllm_v2/MUSR": 0.4522, - "hfopenllm_v2/MMLU-PRO": 0.4825 - } - }, - { - "id": "qwen/qwen1.5-14b", - "name": "Qwen1.5 14B", - "developer": "qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.425, - "helm_lite/NarrativeQA": 0.711, - "helm_lite/NaturalQuestions (closed-book)": 0.3, - "helm_lite/OpenbookQA": 0.862, - "helm_lite/MMLU": 0.626, - "helm_lite/MATH": 0.686, - "helm_lite/GSM8K": 0.693, - "helm_lite/LegalBench": 0.593, - "helm_lite/MedQA": 0.515, - "helm_lite/WMT 2014": 0.178, - "helm_mmlu/MMLU All Subjects": 0.686, - "helm_mmlu/Abstract Algebra": 0.4, - "helm_mmlu/Anatomy": 0.637, - "helm_mmlu/College Physics": 0.48, - "helm_mmlu/Computer Security": 0.84, - "helm_mmlu/Econometrics": 0.561, - "helm_mmlu/Global Facts": 0.49, - "helm_mmlu/Jurisprudence": 0.769, - "helm_mmlu/Philosophy": 0.717, - "helm_mmlu/Professional Psychology": 0.699, - "helm_mmlu/Us Foreign Policy": 0.87, - "helm_mmlu/Astronomy": 0.724, - "helm_mmlu/Business Ethics": 0.75, - "helm_mmlu/Clinical Knowledge": 0.736, - "helm_mmlu/Conceptual Physics": 0.694, - "helm_mmlu/Electrical Engineering": 0.683, - "helm_mmlu/Elementary Mathematics": 0.603, - "helm_mmlu/Formal Logic": 0.492, - "helm_mmlu/High School World History": 0.84, - "helm_mmlu/Human Sexuality": 0.756, - "helm_mmlu/International Law": 0.826, - "helm_mmlu/Logical Fallacies": 0.736, - "helm_mmlu/Machine Learning": 0.509, - "helm_mmlu/Management": 0.816, - "helm_mmlu/Marketing": 0.893, - "helm_mmlu/Medical Genetics": 0.76, - "helm_mmlu/Miscellaneous": 0.835, - "helm_mmlu/Moral Scenarios": 0.368, - "helm_mmlu/Nutrition": 0.742, - "helm_mmlu/Prehistory": 0.71, - "helm_mmlu/Public Relations": 0.655, - "helm_mmlu/Security Studies": 0.8, - "helm_mmlu/Sociology": 0.841, - "helm_mmlu/Virology": 0.458, - "helm_mmlu/World Religions": 0.842, - "helm_mmlu/Mean win rate": 0.796, - "hfopenllm_v2/IFEval": 0.2905, - "hfopenllm_v2/BBH": 0.508, - "hfopenllm_v2/MATH Level 5": 0.2024, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.4186, - "hfopenllm_v2/MMLU-PRO": 0.3644 - } - }, - { - "id": "Qwen/Qwen1.5-14B-Chat", - "name": "Qwen/Qwen1.5-14B-Chat", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4768, - "hfopenllm_v2/BBH": 0.5229, - "hfopenllm_v2/MATH Level 5": 0.1526, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.44, - "hfopenllm_v2/MMLU-PRO": 0.3618, - "reward-bench/Score": 0.6864, - "reward-bench/Chat": 0.5726, - "reward-bench/Chat Hard": 0.7018, - "reward-bench/Safety": 0.7122, - "reward-bench/Reasoning": 0.8961, - "reward-bench/Prior Sets (0.5 weight)": 0.4123 - } - }, - { - "id": "qwen/qwen1.5-32b", - "name": "Qwen1.5 32B", - "developer": "qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.546, - "helm_lite/NarrativeQA": 0.589, - "helm_lite/NaturalQuestions (closed-book)": 0.353, - "helm_lite/OpenbookQA": 0.932, - "helm_lite/MMLU": 0.628, - "helm_lite/MATH": 0.733, - "helm_lite/GSM8K": 0.773, - "helm_lite/LegalBench": 0.636, - "helm_lite/MedQA": 0.656, - "helm_lite/WMT 2014": 0.193, - "helm_mmlu/MMLU All Subjects": 0.744, - "helm_mmlu/Abstract Algebra": 0.4, - "helm_mmlu/Anatomy": 0.644, - "helm_mmlu/College Physics": 0.51, - "helm_mmlu/Computer Security": 0.77, - "helm_mmlu/Econometrics": 0.561, - "helm_mmlu/Global Facts": 0.47, - "helm_mmlu/Jurisprudence": 0.843, - "helm_mmlu/Philosophy": 0.826, - "helm_mmlu/Professional Psychology": 0.75, - "helm_mmlu/Us Foreign Policy": 0.91, - "helm_mmlu/Astronomy": 0.855, - "helm_mmlu/Business Ethics": 0.77, - "helm_mmlu/Clinical Knowledge": 0.781, - "helm_mmlu/Conceptual Physics": 0.766, - "helm_mmlu/Electrical Engineering": 0.731, - "helm_mmlu/Elementary Mathematics": 0.685, - "helm_mmlu/Formal Logic": 0.524, - "helm_mmlu/High School World History": 0.869, - "helm_mmlu/Human Sexuality": 0.847, - "helm_mmlu/International Law": 0.884, - "helm_mmlu/Logical Fallacies": 0.822, - "helm_mmlu/Machine Learning": 0.616, - "helm_mmlu/Management": 0.874, - "helm_mmlu/Marketing": 0.936, - "helm_mmlu/Medical Genetics": 0.85, - "helm_mmlu/Miscellaneous": 0.884, - "helm_mmlu/Moral Scenarios": 0.545, - "helm_mmlu/Nutrition": 0.81, - "helm_mmlu/Prehistory": 0.83, - "helm_mmlu/Public Relations": 0.664, - "helm_mmlu/Security Studies": 0.829, - "helm_mmlu/Sociology": 0.881, - "helm_mmlu/Virology": 0.578, - "helm_mmlu/World Religions": 0.854, - "helm_mmlu/Mean win rate": 0.624, - "hfopenllm_v2/IFEval": 0.3297, - "hfopenllm_v2/BBH": 0.5715, - "hfopenllm_v2/MATH Level 5": 0.3029, - "hfopenllm_v2/GPQA": 0.3297, - "hfopenllm_v2/MUSR": 0.4278, - "hfopenllm_v2/MMLU-PRO": 0.45 - } - }, - { - "id": "Qwen/Qwen1.5-32B-Chat", - "name": "Qwen1.5-32B-Chat", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5532, - "hfopenllm_v2/BBH": 0.6067, - "hfopenllm_v2/MATH Level 5": 0.1956, - "hfopenllm_v2/GPQA": 0.3062, - "hfopenllm_v2/MUSR": 0.416, - "hfopenllm_v2/MMLU-PRO": 0.4457 - } - }, - { - "id": "Qwen/Qwen1.5-4B", - "name": "Qwen1.5-4B", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2445, - "hfopenllm_v2/BBH": 0.4054, - "hfopenllm_v2/MATH Level 5": 0.0529, - "hfopenllm_v2/GPQA": 0.2768, - "hfopenllm_v2/MUSR": 0.3604, - "hfopenllm_v2/MMLU-PRO": 0.246 - } - }, - { - "id": "Qwen/Qwen1.5-4B-Chat", - "name": "Qwen/Qwen1.5-4B-Chat", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3157, - "hfopenllm_v2/BBH": 0.4006, - "hfopenllm_v2/MATH Level 5": 0.0279, - "hfopenllm_v2/GPQA": 0.2668, - "hfopenllm_v2/MUSR": 0.3978, - "hfopenllm_v2/MMLU-PRO": 0.2396, - "reward-bench/Score": 0.5477, - "reward-bench/Chat": 0.3883, - "reward-bench/Chat Hard": 0.6272, - "reward-bench/Safety": 0.5568, - "reward-bench/Reasoning": 0.6689, - "reward-bench/Prior Sets (0.5 weight)": 0.447 - } - }, - { - "id": "qwen/qwen1.5-72b", - "name": "Qwen1.5 72B", - "developer": "qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.608, - "helm_lite/NarrativeQA": 0.601, - "helm_lite/NaturalQuestions (closed-book)": 0.417, - "helm_lite/OpenbookQA": 0.93, - "helm_lite/MMLU": 0.647, - "helm_lite/MATH": 0.683, - "helm_lite/GSM8K": 0.799, - "helm_lite/LegalBench": 0.694, - "helm_lite/MedQA": 0.67, - "helm_lite/WMT 2014": 0.201, - "helm_mmlu/MMLU All Subjects": 0.774, - "helm_mmlu/Abstract Algebra": 0.44, - "helm_mmlu/Anatomy": 0.733, - "helm_mmlu/College Physics": 0.559, - "helm_mmlu/Computer Security": 0.81, - "helm_mmlu/Econometrics": 0.544, - "helm_mmlu/Global Facts": 0.56, - "helm_mmlu/Jurisprudence": 0.824, - "helm_mmlu/Philosophy": 0.83, - "helm_mmlu/Professional Psychology": 0.809, - "helm_mmlu/Us Foreign Policy": 0.94, - "helm_mmlu/Astronomy": 0.868, - "helm_mmlu/Business Ethics": 0.79, - "helm_mmlu/Clinical Knowledge": 0.834, - "helm_mmlu/Conceptual Physics": 0.821, - "helm_mmlu/Electrical Engineering": 0.779, - "helm_mmlu/Elementary Mathematics": 0.696, - "helm_mmlu/Formal Logic": 0.556, - "helm_mmlu/High School World History": 0.899, - "helm_mmlu/Human Sexuality": 0.878, - "helm_mmlu/International Law": 0.909, - "helm_mmlu/Logical Fallacies": 0.853, - "helm_mmlu/Machine Learning": 0.67, - "helm_mmlu/Management": 0.854, - "helm_mmlu/Marketing": 0.949, - "helm_mmlu/Medical Genetics": 0.87, - "helm_mmlu/Miscellaneous": 0.921, - "helm_mmlu/Moral Scenarios": 0.669, - "helm_mmlu/Nutrition": 0.859, - "helm_mmlu/Prehistory": 0.88, - "helm_mmlu/Public Relations": 0.755, - "helm_mmlu/Security Studies": 0.824, - "helm_mmlu/Sociology": 0.9, - "helm_mmlu/Virology": 0.584, - "helm_mmlu/World Religions": 0.883, - "helm_mmlu/Mean win rate": 0.65 - } - }, - { - "id": "Qwen/Qwen1.5-72B-Chat", - "name": "Qwen/Qwen1.5-72B-Chat", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6723, - "reward-bench/Chat": 0.6229, - "reward-bench/Chat Hard": 0.6601, - "reward-bench/Safety": 0.6757, - "reward-bench/Reasoning": 0.8554, - "reward-bench/Prior Sets (0.5 weight)": 0.4226 - } - }, - { - "id": "qwen/qwen1.5-7b", - "name": "Qwen1.5 7B", - "developer": "qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.275, - "helm_lite/NarrativeQA": 0.448, - "helm_lite/NaturalQuestions (closed-book)": 0.27, - "helm_lite/OpenbookQA": 0.806, - "helm_lite/MMLU": 0.569, - "helm_lite/MATH": 0.561, - "helm_lite/GSM8K": 0.6, - "helm_lite/LegalBench": 0.523, - "helm_lite/MedQA": 0.479, - "helm_lite/WMT 2014": 0.153, - "helm_mmlu/MMLU All Subjects": 0.626, - "helm_mmlu/Abstract Algebra": 0.39, - "helm_mmlu/Anatomy": 0.526, - "helm_mmlu/College Physics": 0.471, - "helm_mmlu/Computer Security": 0.76, - "helm_mmlu/Econometrics": 0.447, - "helm_mmlu/Global Facts": 0.4, - "helm_mmlu/Jurisprudence": 0.778, - "helm_mmlu/Philosophy": 0.691, - "helm_mmlu/Professional Psychology": 0.603, - "helm_mmlu/Us Foreign Policy": 0.84, - "helm_mmlu/Astronomy": 0.671, - "helm_mmlu/Business Ethics": 0.69, - "helm_mmlu/Clinical Knowledge": 0.691, - "helm_mmlu/Conceptual Physics": 0.579, - "helm_mmlu/Electrical Engineering": 0.572, - "helm_mmlu/Elementary Mathematics": 0.5, - "helm_mmlu/Formal Logic": 0.397, - "helm_mmlu/High School World History": 0.789, - "helm_mmlu/Human Sexuality": 0.695, - "helm_mmlu/International Law": 0.76, - "helm_mmlu/Logical Fallacies": 0.706, - "helm_mmlu/Machine Learning": 0.411, - "helm_mmlu/Management": 0.816, - "helm_mmlu/Marketing": 0.863, - "helm_mmlu/Medical Genetics": 0.69, - "helm_mmlu/Miscellaneous": 0.765, - "helm_mmlu/Moral Scenarios": 0.372, - "helm_mmlu/Nutrition": 0.696, - "helm_mmlu/Prehistory": 0.688, - "helm_mmlu/Public Relations": 0.627, - "helm_mmlu/Security Studies": 0.727, - "helm_mmlu/Sociology": 0.836, - "helm_mmlu/Virology": 0.488, - "helm_mmlu/World Religions": 0.778, - "helm_mmlu/Mean win rate": 0.843, - "hfopenllm_v2/IFEval": 0.2684, - "hfopenllm_v2/BBH": 0.456, - "hfopenllm_v2/MATH Level 5": 0.0929, - "hfopenllm_v2/GPQA": 0.2987, - "hfopenllm_v2/MUSR": 0.4103, - "hfopenllm_v2/MMLU-PRO": 0.2916 - } - }, - { - "id": "Qwen/Qwen1.5-7B-Chat", - "name": "Qwen/Qwen1.5-7B-Chat", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4371, - "hfopenllm_v2/BBH": 0.451, - "hfopenllm_v2/MATH Level 5": 0.0627, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.3779, - "hfopenllm_v2/MMLU-PRO": 0.2951, - "reward-bench/Score": 0.675, - "reward-bench/Chat": 0.5363, - "reward-bench/Chat Hard": 0.6908, - "reward-bench/Safety": 0.6919, - "reward-bench/Reasoning": 0.9041, - "reward-bench/Prior Sets (0.5 weight)": 0.4288 - } - }, - { - "id": "Qwen/Qwen1.5-MoE-A2.7B", - "name": "Qwen1.5-MoE-A2.7B", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.266, - "hfopenllm_v2/BBH": 0.4114, - "hfopenllm_v2/MATH Level 5": 0.0929, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.4013, - "hfopenllm_v2/MMLU-PRO": 0.2778 - } - }, - { - "id": "Qwen/Qwen1.5-MoE-A2.7B-Chat", - "name": "Qwen/Qwen1.5-MoE-A2.7B-Chat", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3795, - "hfopenllm_v2/BBH": 0.4272, - "hfopenllm_v2/MATH Level 5": 0.0634, - "hfopenllm_v2/GPQA": 0.2743, - "hfopenllm_v2/MUSR": 0.3899, - "hfopenllm_v2/MMLU-PRO": 0.2923, - "reward-bench/Score": 0.6644, - "reward-bench/Chat": 0.7291, - "reward-bench/Chat Hard": 0.6316, - "reward-bench/Safety": 0.6284, - "reward-bench/Reasoning": 0.774, - "reward-bench/Prior Sets (0.5 weight)": 0.4536 - } - }, - { - "id": "Qwen/Qwen2-0.5B", - "name": "Qwen2-0.5B", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1873, - "hfopenllm_v2/BBH": 0.3239, - "hfopenllm_v2/MATH Level 5": 0.0264, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.3752, - "hfopenllm_v2/MMLU-PRO": 0.172 - } - }, - { - "id": "Qwen/Qwen2-0.5B-Instruct", - "name": "Qwen2-0.5B-Instruct", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2247, - "hfopenllm_v2/BBH": 0.3173, - "hfopenllm_v2/MATH Level 5": 0.0287, - "hfopenllm_v2/GPQA": 0.2466, - "hfopenllm_v2/MUSR": 0.3353, - "hfopenllm_v2/MMLU-PRO": 0.1531 - } - }, - { - "id": "Qwen/Qwen2-1.5B", - "name": "Qwen2-1.5B", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2113, - "hfopenllm_v2/BBH": 0.3575, - "hfopenllm_v2/MATH Level 5": 0.0702, - "hfopenllm_v2/GPQA": 0.2643, - "hfopenllm_v2/MUSR": 0.3658, - "hfopenllm_v2/MMLU-PRO": 0.2552 - } - }, - { - "id": "Qwen/Qwen2-1.5B-Instruct", - "name": "Qwen2-1.5B-Instruct", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3371, - "hfopenllm_v2/BBH": 0.3852, - "hfopenllm_v2/MATH Level 5": 0.0718, - "hfopenllm_v2/GPQA": 0.2617, - "hfopenllm_v2/MUSR": 0.4293, - "hfopenllm_v2/MMLU-PRO": 0.2501 - } - }, - { - "id": "Qwen/Qwen2-57B-A14B", - "name": "Qwen2-57B-A14B", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3113, - "hfopenllm_v2/BBH": 0.5618, - "hfopenllm_v2/MATH Level 5": 0.1866, - "hfopenllm_v2/GPQA": 0.3062, - "hfopenllm_v2/MUSR": 0.4174, - "hfopenllm_v2/MMLU-PRO": 0.4916 - } - }, - { - "id": "Qwen/Qwen2-57B-A14B-Instruct", - "name": "Qwen2-57B-A14B-Instruct", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6338, - "hfopenllm_v2/BBH": 0.5888, - "hfopenllm_v2/MATH Level 5": 0.2817, - "hfopenllm_v2/GPQA": 0.3314, - "hfopenllm_v2/MUSR": 0.4361, - "hfopenllm_v2/MMLU-PRO": 0.4575 - } - }, - { - "id": "Qwen/Qwen2-72B", - "name": "Qwen2-72B", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3824, - "hfopenllm_v2/BBH": 0.6617, - "hfopenllm_v2/MATH Level 5": 0.3112, - "hfopenllm_v2/GPQA": 0.3943, - "hfopenllm_v2/MUSR": 0.4704, - "hfopenllm_v2/MMLU-PRO": 0.5731 - } - }, - { - "id": "qwen/qwen2-72b-instruct", - "name": "Qwen2 Instruct 72B", - "developer": "qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.77, - "helm_lite/NarrativeQA": 0.727, - "helm_lite/NaturalQuestions (closed-book)": 0.39, - "helm_lite/OpenbookQA": 0.954, - "helm_lite/MMLU": 0.769, - "helm_lite/MATH": 0.79, - "helm_lite/GSM8K": 0.92, - "helm_lite/LegalBench": 0.712, - "helm_lite/MedQA": 0.746, - "helm_lite/WMT 2014": 0.207, - "helm_mmlu/MMLU All Subjects": 0.824, - "helm_mmlu/Abstract Algebra": 0.67, - "helm_mmlu/Anatomy": 0.793, - "helm_mmlu/College Physics": 0.598, - "helm_mmlu/Computer Security": 0.85, - "helm_mmlu/Econometrics": 0.737, - "helm_mmlu/Global Facts": 0.58, - "helm_mmlu/Jurisprudence": 0.87, - "helm_mmlu/Philosophy": 0.859, - "helm_mmlu/Professional Psychology": 0.886, - "helm_mmlu/Us Foreign Policy": 0.94, - "helm_mmlu/Astronomy": 0.934, - "helm_mmlu/Business Ethics": 0.82, - "helm_mmlu/Clinical Knowledge": 0.868, - "helm_mmlu/Conceptual Physics": 0.872, - "helm_mmlu/Electrical Engineering": 0.793, - "helm_mmlu/Elementary Mathematics": 0.825, - "helm_mmlu/Formal Logic": 0.667, - "helm_mmlu/High School World History": 0.932, - "helm_mmlu/Human Sexuality": 0.893, - "helm_mmlu/International Law": 0.893, - "helm_mmlu/Logical Fallacies": 0.914, - "helm_mmlu/Machine Learning": 0.768, - "helm_mmlu/Management": 0.903, - "helm_mmlu/Marketing": 0.953, - "helm_mmlu/Medical Genetics": 0.9, - "helm_mmlu/Miscellaneous": 0.943, - "helm_mmlu/Moral Scenarios": 0.815, - "helm_mmlu/Nutrition": 0.902, - "helm_mmlu/Prehistory": 0.914, - "helm_mmlu/Public Relations": 0.745, - "helm_mmlu/Security Studies": 0.837, - "helm_mmlu/Sociology": 0.935, - "helm_mmlu/Virology": 0.56, - "helm_mmlu/World Religions": 0.848, - "helm_mmlu/Mean win rate": 0.826, - "hfopenllm_v2/IFEval": 0.7989, - "hfopenllm_v2/BBH": 0.6977, - "hfopenllm_v2/MATH Level 5": 0.4177, - "hfopenllm_v2/GPQA": 0.3725, - "hfopenllm_v2/MUSR": 0.456, - "hfopenllm_v2/MMLU-PRO": 0.5403 - } - }, - { - "id": "Qwen/Qwen2-7B", - "name": "Qwen2-7B", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3149, - "hfopenllm_v2/BBH": 0.5315, - "hfopenllm_v2/MATH Level 5": 0.2039, - "hfopenllm_v2/GPQA": 0.3045, - "hfopenllm_v2/MUSR": 0.4439, - "hfopenllm_v2/MMLU-PRO": 0.4183 - } - }, - { - "id": "Qwen/Qwen2-7B-Instruct", - "name": "Qwen2-7B-Instruct", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5679, - "hfopenllm_v2/BBH": 0.5545, - "hfopenllm_v2/MATH Level 5": 0.2764, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.3928, - "hfopenllm_v2/MMLU-PRO": 0.3847 - } - }, - { - "id": "Qwen/Qwen2-Math-72B-Instruct", - "name": "Qwen2-Math-72B-Instruct", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5694, - "hfopenllm_v2/BBH": 0.6343, - "hfopenllm_v2/MATH Level 5": 0.5536, - "hfopenllm_v2/GPQA": 0.3683, - "hfopenllm_v2/MUSR": 0.4517, - "hfopenllm_v2/MMLU-PRO": 0.4273 - } - }, - { - "id": "Qwen/Qwen2-Math-7B", - "name": "Qwen2-Math-7B", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2687, - "hfopenllm_v2/BBH": 0.387, - "hfopenllm_v2/MATH Level 5": 0.2477, - "hfopenllm_v2/GPQA": 0.2634, - "hfopenllm_v2/MUSR": 0.3593, - "hfopenllm_v2/MMLU-PRO": 0.1197 - } - }, - { - "id": "Qwen/Qwen2-VL-72B-Instruct", - "name": "Qwen2-VL-72B-Instruct", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5982, - "hfopenllm_v2/BBH": 0.6946, - "hfopenllm_v2/MATH Level 5": 0.3444, - "hfopenllm_v2/GPQA": 0.3876, - "hfopenllm_v2/MUSR": 0.4492, - "hfopenllm_v2/MMLU-PRO": 0.5717 - } - }, - { - "id": "Qwen/Qwen2-VL-7B-Instruct", - "name": "Qwen2-VL-7B-Instruct", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4599, - "hfopenllm_v2/BBH": 0.5465, - "hfopenllm_v2/MATH Level 5": 0.1986, - "hfopenllm_v2/GPQA": 0.3196, - "hfopenllm_v2/MUSR": 0.4375, - "hfopenllm_v2/MMLU-PRO": 0.4095 - } - }, - { - "id": "Qwen/Qwen2.5-0.5B", - "name": "Qwen2.5-0.5B", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1627, - "hfopenllm_v2/BBH": 0.3275, - "hfopenllm_v2/MATH Level 5": 0.0393, - "hfopenllm_v2/GPQA": 0.2466, - "hfopenllm_v2/MUSR": 0.3433, - "hfopenllm_v2/MMLU-PRO": 0.1906 - } - }, - { - "id": "Qwen/Qwen2.5-0.5B-Instruct", - "name": "Qwen2.5-0.5B-Instruct", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3071, - "hfopenllm_v2/BBH": 0.3341, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2576, - "hfopenllm_v2/MUSR": 0.3329, - "hfopenllm_v2/MMLU-PRO": 0.1697 - } - }, - { - "id": "Qwen/Qwen2.5-1.5B", - "name": "Qwen2.5-1.5B", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2674, - "hfopenllm_v2/BBH": 0.4078, - "hfopenllm_v2/MATH Level 5": 0.0914, - "hfopenllm_v2/GPQA": 0.2852, - "hfopenllm_v2/MUSR": 0.3576, - "hfopenllm_v2/MMLU-PRO": 0.2855 - } - }, - { - "id": "Qwen/Qwen2.5-1.5B-Instruct", - "name": "Qwen2.5-1.5B-Instruct", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4476, - "hfopenllm_v2/BBH": 0.4289, - "hfopenllm_v2/MATH Level 5": 0.2205, - "hfopenllm_v2/GPQA": 0.2559, - "hfopenllm_v2/MUSR": 0.3663, - "hfopenllm_v2/MMLU-PRO": 0.2799 - } - }, - { - "id": "Qwen/Qwen2.5-14B", - "name": "Qwen2.5-14B", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3694, - "hfopenllm_v2/BBH": 0.6161, - "hfopenllm_v2/MATH Level 5": 0.29, - "hfopenllm_v2/GPQA": 0.3817, - "hfopenllm_v2/MUSR": 0.4502, - "hfopenllm_v2/MMLU-PRO": 0.5249 - } - }, - { - "id": "Qwen/Qwen2.5-14B-Instruct", - "name": "Qwen2.5-14B-Instruct", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8158, - "hfopenllm_v2/BBH": 0.639, - "hfopenllm_v2/MATH Level 5": 0.5476, - "hfopenllm_v2/GPQA": 0.3221, - "hfopenllm_v2/MUSR": 0.4101, - "hfopenllm_v2/MMLU-PRO": 0.4904 - } - }, - { - "id": "Qwen/Qwen2.5-14B-Instruct-1M", - "name": "Qwen2.5-14B-Instruct-1M", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8414, - "hfopenllm_v2/BBH": 0.6198, - "hfopenllm_v2/MATH Level 5": 0.5302, - "hfopenllm_v2/GPQA": 0.3431, - "hfopenllm_v2/MUSR": 0.418, - "hfopenllm_v2/MMLU-PRO": 0.485 - } - }, - { - "id": "Qwen/Qwen2.5-32B", - "name": "Qwen2.5-32B", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4077, - "hfopenllm_v2/BBH": 0.6771, - "hfopenllm_v2/MATH Level 5": 0.3565, - "hfopenllm_v2/GPQA": 0.4119, - "hfopenllm_v2/MUSR": 0.4978, - "hfopenllm_v2/MMLU-PRO": 0.5805 - } - }, - { - "id": "Qwen/Qwen2.5-32B-Instruct", - "name": "Qwen2.5-32B-Instruct", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8346, - "hfopenllm_v2/BBH": 0.6913, - "hfopenllm_v2/MATH Level 5": 0.6254, - "hfopenllm_v2/GPQA": 0.3381, - "hfopenllm_v2/MUSR": 0.4261, - "hfopenllm_v2/MMLU-PRO": 0.5667 - } - }, - { - "id": "Qwen/Qwen2.5-3B", - "name": "Qwen2.5-3B", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.269, - "hfopenllm_v2/BBH": 0.4612, - "hfopenllm_v2/MATH Level 5": 0.148, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.4303, - "hfopenllm_v2/MMLU-PRO": 0.3203 - } - }, - { - "id": "Qwen/Qwen2.5-3B-Instruct", - "name": "Qwen2.5-3B-Instruct", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6475, - "hfopenllm_v2/BBH": 0.4693, - "hfopenllm_v2/MATH Level 5": 0.3678, - "hfopenllm_v2/GPQA": 0.2727, - "hfopenllm_v2/MUSR": 0.3968, - "hfopenllm_v2/MMLU-PRO": 0.3255, - "theory_of_mind/accuracy on theory_of_mind for scorer model_graded_fact": 0.78 - } - }, - { - "id": "Qwen/Qwen2.5-72B", - "name": "Qwen2.5-72B", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4137, - "hfopenllm_v2/BBH": 0.6797, - "hfopenllm_v2/MATH Level 5": 0.3912, - "hfopenllm_v2/GPQA": 0.4052, - "hfopenllm_v2/MUSR": 0.4771, - "hfopenllm_v2/MMLU-PRO": 0.5968 - } - }, - { - "id": "Qwen/Qwen2.5-72B-Instruct", - "name": "Qwen2.5-72B-Instruct", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8638, - "hfopenllm_v2/BBH": 0.7273, - "hfopenllm_v2/MATH Level 5": 0.5982, - "hfopenllm_v2/GPQA": 0.375, - "hfopenllm_v2/MUSR": 0.4206, - "hfopenllm_v2/MMLU-PRO": 0.5626 - } - }, - { - "id": "qwen/qwen2.5-72b-instruct-turbo", - "name": "Qwen2.5 Instruct Turbo 72B", - "developer": "qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.599, - "helm_capabilities/MMLU-Pro": 0.631, - "helm_capabilities/GPQA": 0.426, - "helm_capabilities/IFEval": 0.806, - "helm_capabilities/WildBench": 0.802, - "helm_capabilities/Omni-MATH": 0.33, - "helm_lite/Mean win rate": 0.745, - "helm_lite/NarrativeQA": 0.745, - "helm_lite/NaturalQuestions (closed-book)": 0.359, - "helm_lite/OpenbookQA": 0.962, - "helm_lite/MMLU": 0.77, - "helm_lite/MATH": 0.884, - "helm_lite/GSM8K": 0.9, - "helm_lite/LegalBench": 0.74, - "helm_lite/MedQA": 0.753, - "helm_lite/WMT 2014": 0.207, - "helm_mmlu/MMLU All Subjects": 0.834, - "helm_mmlu/Abstract Algebra": 0.68, - "helm_mmlu/Anatomy": 0.822, - "helm_mmlu/College Physics": 0.588, - "helm_mmlu/Computer Security": 0.86, - "helm_mmlu/Econometrics": 0.728, - "helm_mmlu/Global Facts": 0.61, - "helm_mmlu/Jurisprudence": 0.87, - "helm_mmlu/Philosophy": 0.839, - "helm_mmlu/Professional Psychology": 0.864, - "helm_mmlu/Us Foreign Policy": 0.96, - "helm_mmlu/Astronomy": 0.934, - "helm_mmlu/Business Ethics": 0.85, - "helm_mmlu/Clinical Knowledge": 0.872, - "helm_mmlu/Conceptual Physics": 0.885, - "helm_mmlu/Electrical Engineering": 0.8, - "helm_mmlu/Elementary Mathematics": 0.87, - "helm_mmlu/Formal Logic": 0.73, - "helm_mmlu/High School World History": 0.92, - "helm_mmlu/Human Sexuality": 0.878, - "helm_mmlu/International Law": 0.893, - "helm_mmlu/Logical Fallacies": 0.89, - "helm_mmlu/Machine Learning": 0.777, - "helm_mmlu/Management": 0.913, - "helm_mmlu/Marketing": 0.953, - "helm_mmlu/Medical Genetics": 0.92, - "helm_mmlu/Miscellaneous": 0.932, - "helm_mmlu/Moral Scenarios": 0.787, - "helm_mmlu/Nutrition": 0.886, - "helm_mmlu/Prehistory": 0.91, - "helm_mmlu/Public Relations": 0.782, - "helm_mmlu/Security Studies": 0.849, - "helm_mmlu/Sociology": 0.925, - "helm_mmlu/Virology": 0.584, - "helm_mmlu/World Religions": 0.901, - "helm_mmlu/Mean win rate": 0.548 - } - }, - { - "id": "Qwen/Qwen2.5-7B", - "name": "Qwen2.5-7B", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3374, - "hfopenllm_v2/BBH": 0.5416, - "hfopenllm_v2/MATH Level 5": 0.2508, - "hfopenllm_v2/GPQA": 0.3247, - "hfopenllm_v2/MUSR": 0.4424, - "hfopenllm_v2/MMLU-PRO": 0.4365, - "la_leaderboard/la_leaderboard": 27.61 - } - }, - { - "id": "Qwen/Qwen2.5-7B-Instruct", - "name": "Qwen2.5-7B-Instruct", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7585, - "hfopenllm_v2/BBH": 0.5394, - "hfopenllm_v2/MATH Level 5": 0.5, - "hfopenllm_v2/GPQA": 0.2911, - "hfopenllm_v2/MUSR": 0.402, - "hfopenllm_v2/MMLU-PRO": 0.4287 - } - }, - { - "id": "Qwen/Qwen2.5-7B-Instruct-1M", - "name": "Qwen2.5-7B-Instruct-1M", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7448, - "hfopenllm_v2/BBH": 0.5404, - "hfopenllm_v2/MATH Level 5": 0.4335, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.4087, - "hfopenllm_v2/MMLU-PRO": 0.3505 - } - }, - { - "id": "qwen/qwen2.5-7b-instruct-turbo", - "name": "Qwen2.5 Instruct Turbo 7B", - "developer": "qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.529, - "helm_capabilities/MMLU-Pro": 0.539, - "helm_capabilities/GPQA": 0.341, - "helm_capabilities/IFEval": 0.741, - "helm_capabilities/WildBench": 0.731, - "helm_capabilities/Omni-MATH": 0.294, - "helm_lite/Mean win rate": 0.488, - "helm_lite/NarrativeQA": 0.742, - "helm_lite/NaturalQuestions (closed-book)": 0.205, - "helm_lite/OpenbookQA": 0.862, - "helm_lite/MMLU": 0.658, - "helm_lite/MATH": 0.835, - "helm_lite/GSM8K": 0.83, - "helm_lite/LegalBench": 0.632, - "helm_lite/MedQA": 0.6, - "helm_lite/WMT 2014": 0.155, - "helm_mmlu/MMLU All Subjects": 0.729, - "helm_mmlu/Abstract Algebra": 0.49, - "helm_mmlu/Anatomy": 0.689, - "helm_mmlu/College Physics": 0.51, - "helm_mmlu/Computer Security": 0.79, - "helm_mmlu/Econometrics": 0.64, - "helm_mmlu/Global Facts": 0.42, - "helm_mmlu/Jurisprudence": 0.796, - "helm_mmlu/Philosophy": 0.746, - "helm_mmlu/Professional Psychology": 0.757, - "helm_mmlu/Us Foreign Policy": 0.86, - "helm_mmlu/Astronomy": 0.836, - "helm_mmlu/Business Ethics": 0.82, - "helm_mmlu/Clinical Knowledge": 0.785, - "helm_mmlu/Conceptual Physics": 0.736, - "helm_mmlu/Electrical Engineering": 0.717, - "helm_mmlu/Elementary Mathematics": 0.643, - "helm_mmlu/Formal Logic": 0.587, - "helm_mmlu/High School World History": 0.878, - "helm_mmlu/Human Sexuality": 0.794, - "helm_mmlu/International Law": 0.86, - "helm_mmlu/Logical Fallacies": 0.773, - "helm_mmlu/Machine Learning": 0.554, - "helm_mmlu/Management": 0.845, - "helm_mmlu/Marketing": 0.919, - "helm_mmlu/Medical Genetics": 0.85, - "helm_mmlu/Miscellaneous": 0.852, - "helm_mmlu/Moral Scenarios": 0.511, - "helm_mmlu/Nutrition": 0.778, - "helm_mmlu/Prehistory": 0.836, - "helm_mmlu/Public Relations": 0.709, - "helm_mmlu/Security Studies": 0.682, - "helm_mmlu/Sociology": 0.861, - "helm_mmlu/Virology": 0.578, - "helm_mmlu/World Religions": 0.83, - "helm_mmlu/Mean win rate": 0.887 - } - }, - { - "id": "Qwen/Qwen2.5-Coder-14B", - "name": "Qwen2.5-Coder-14B", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3473, - "hfopenllm_v2/BBH": 0.5865, - "hfopenllm_v2/MATH Level 5": 0.2251, - "hfopenllm_v2/GPQA": 0.2928, - "hfopenllm_v2/MUSR": 0.3874, - "hfopenllm_v2/MMLU-PRO": 0.4521 - } - }, - { - "id": "Qwen/Qwen2.5-Coder-14B-Instruct", - "name": "Qwen2.5-Coder-14B-Instruct", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6908, - "hfopenllm_v2/BBH": 0.614, - "hfopenllm_v2/MATH Level 5": 0.3248, - "hfopenllm_v2/GPQA": 0.3045, - "hfopenllm_v2/MUSR": 0.3915, - "hfopenllm_v2/MMLU-PRO": 0.3939 - } - }, - { - "id": "Qwen/Qwen2.5-Coder-32B", - "name": "Qwen2.5-Coder-32B", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4363, - "hfopenllm_v2/BBH": 0.6404, - "hfopenllm_v2/MATH Level 5": 0.3089, - "hfopenllm_v2/GPQA": 0.3465, - "hfopenllm_v2/MUSR": 0.4528, - "hfopenllm_v2/MMLU-PRO": 0.5303 - } - }, - { - "id": "Qwen/Qwen2.5-Coder-32B-Instruct", - "name": "Qwen2.5-Coder-32B-Instruct", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7265, - "hfopenllm_v2/BBH": 0.6625, - "hfopenllm_v2/MATH Level 5": 0.4955, - "hfopenllm_v2/GPQA": 0.349, - "hfopenllm_v2/MUSR": 0.4386, - "hfopenllm_v2/MMLU-PRO": 0.4413 - } - }, - { - "id": "Qwen/Qwen2.5-Coder-7B", - "name": "Qwen2.5-Coder-7B", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3446, - "hfopenllm_v2/BBH": 0.4856, - "hfopenllm_v2/MATH Level 5": 0.1918, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3449, - "hfopenllm_v2/MMLU-PRO": 0.3679 - } - }, - { - "id": "Qwen/Qwen2.5-Coder-7B-Instruct", - "name": "Qwen2.5-Coder-7B-Instruct", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6147, - "hfopenllm_v2/BBH": 0.4999, - "hfopenllm_v2/MATH Level 5": 0.031, - "hfopenllm_v2/GPQA": 0.2936, - "hfopenllm_v2/MUSR": 0.4099, - "hfopenllm_v2/MMLU-PRO": 0.3354 - } - }, - { - "id": "Qwen/Qwen2.5-Math-1.5B-Instruct", - "name": "Qwen2.5-Math-1.5B-Instruct", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1856, - "hfopenllm_v2/BBH": 0.3752, - "hfopenllm_v2/MATH Level 5": 0.2628, - "hfopenllm_v2/GPQA": 0.2651, - "hfopenllm_v2/MUSR": 0.3685, - "hfopenllm_v2/MMLU-PRO": 0.1801 - } - }, - { - "id": "Qwen/Qwen2.5-Math-72B-Instruct", - "name": "Qwen2.5-Math-72B-Instruct", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4003, - "hfopenllm_v2/BBH": 0.6452, - "hfopenllm_v2/MATH Level 5": 0.6239, - "hfopenllm_v2/GPQA": 0.3314, - "hfopenllm_v2/MUSR": 0.4473, - "hfopenllm_v2/MMLU-PRO": 0.4812 - } - }, - { - "id": "Qwen/Qwen2.5-Math-7B", - "name": "Qwen2.5-Math-7B", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.246, - "hfopenllm_v2/BBH": 0.4455, - "hfopenllm_v2/MATH Level 5": 0.3051, - "hfopenllm_v2/GPQA": 0.2936, - "hfopenllm_v2/MUSR": 0.3781, - "hfopenllm_v2/MMLU-PRO": 0.2718 - } - }, - { - "id": "Qwen/Qwen2.5-Math-7B-Instruct", - "name": "Qwen2.5-Math-7B-Instruct", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2636, - "hfopenllm_v2/BBH": 0.4388, - "hfopenllm_v2/MATH Level 5": 0.5808, - "hfopenllm_v2/GPQA": 0.2617, - "hfopenllm_v2/MUSR": 0.3647, - "hfopenllm_v2/MMLU-PRO": 0.282 - } - }, - { - "id": "qwen/qwen3-0-6b-fc", - "name": "Qwen3-0.6B (FC)", - "developer": "qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 92.0, - "bfcl/bfcl.overall.overall_accuracy": 23.93, - "bfcl/bfcl.overall.total_cost_usd": 0.46, - "bfcl/bfcl.overall.latency_mean_s": 0.68, - "bfcl/bfcl.overall.latency_std_s": 8.45, - "bfcl/bfcl.overall.latency_p95_s": 0.96, - "bfcl/bfcl.non_live.ast_accuracy": 71.79, - "bfcl/bfcl.non_live.simple_ast_accuracy": 64.17, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 86.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 67.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 69.5, - "bfcl/bfcl.live.live_accuracy": 56.62, - "bfcl/bfcl.live.live_simple_ast_accuracy": 61.24, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 56.13, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 43.75, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 37.5, - "bfcl/bfcl.multi_turn.accuracy": 3.62, - "bfcl/bfcl.multi_turn.base_accuracy": 5.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 2.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 3.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 4.0, - "bfcl/bfcl.web_search.accuracy": 1.0, - "bfcl/bfcl.web_search.base_accuracy": 1.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 1.0, - "bfcl/bfcl.memory.accuracy": 8.6, - "bfcl/bfcl.memory.kv_accuracy": 2.58, - "bfcl/bfcl.memory.vector_accuracy": 1.94, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 21.29, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 75.0, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 80.84 - } - }, - { - "id": "qwen/qwen3-0-6b-prompt", - "name": "Qwen3-0.6B (Prompt)", - "developer": "qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 94.0, - "bfcl/bfcl.overall.overall_accuracy": 22.38, - "bfcl/bfcl.overall.total_cost_usd": 3.65, - "bfcl/bfcl.overall.latency_mean_s": 3.1, - "bfcl/bfcl.overall.latency_std_s": 4.32, - "bfcl/bfcl.overall.latency_p95_s": 10.31, - "bfcl/bfcl.non_live.ast_accuracy": 70.0, - "bfcl/bfcl.non_live.simple_ast_accuracy": 64.0, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 78.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 75.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 63.0, - "bfcl/bfcl.live.live_accuracy": 49.37, - "bfcl/bfcl.live.live_simple_ast_accuracy": 57.75, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 47.77, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 37.5, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 37.5, - "bfcl/bfcl.multi_turn.accuracy": 1.38, - "bfcl/bfcl.multi_turn.base_accuracy": 1.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 1.5, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 1.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 1.0, - "bfcl/bfcl.web_search.accuracy": 0.5, - "bfcl/bfcl.web_search.base_accuracy": 1.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 0.0, - "bfcl/bfcl.memory.accuracy": 8.39, - "bfcl/bfcl.memory.kv_accuracy": 1.29, - "bfcl/bfcl.memory.vector_accuracy": 2.58, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 21.29, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 75.0, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 82.5, - "bfcl/bfcl.format_sensitivity.max_delta": 60.5, - "bfcl/bfcl.format_sensitivity.stddev": 24.35 - } - }, - { - "id": "qwen/qwen3-1-7b-fc", - "name": "Qwen3-1.7B (FC)", - "developer": "qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 71.0, - "bfcl/bfcl.overall.overall_accuracy": 28.41, - "bfcl/bfcl.overall.total_cost_usd": 4.33, - "bfcl/bfcl.overall.latency_mean_s": 5.12, - "bfcl/bfcl.overall.latency_std_s": 7.37, - "bfcl/bfcl.overall.latency_p95_s": 13.35, - "bfcl/bfcl.non_live.ast_accuracy": 82.92, - "bfcl/bfcl.non_live.simple_ast_accuracy": 70.67, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 92.5, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 88.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 80.0, - "bfcl/bfcl.live.live_accuracy": 74.61, - "bfcl/bfcl.live.live_simple_ast_accuracy": 76.74, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 74.26, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 62.5, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 75.0, - "bfcl/bfcl.multi_turn.accuracy": 11.0, - "bfcl/bfcl.multi_turn.base_accuracy": 15.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 6.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 12.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 11.0, - "bfcl/bfcl.web_search.accuracy": 2.5, - "bfcl/bfcl.web_search.base_accuracy": 3.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 2.0, - "bfcl/bfcl.memory.accuracy": 6.02, - "bfcl/bfcl.memory.kv_accuracy": 4.52, - "bfcl/bfcl.memory.vector_accuracy": 7.74, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 5.81, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 81.25, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 76.54 - } - }, - { - "id": "qwen/qwen3-14b-fc", - "name": "Qwen3-14B (FC)", - "developer": "qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 43.0, - "bfcl/bfcl.overall.overall_accuracy": 41.03, - "bfcl/bfcl.overall.total_cost_usd": 3.38, - "bfcl/bfcl.overall.latency_mean_s": 4.5, - "bfcl/bfcl.overall.latency_std_s": 18.84, - "bfcl/bfcl.overall.latency_p95_s": 13.34, - "bfcl/bfcl.non_live.ast_accuracy": 84.94, - "bfcl/bfcl.non_live.simple_ast_accuracy": 74.75, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 93.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 80.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 92.0, - "bfcl/bfcl.live.live_accuracy": 80.01, - "bfcl/bfcl.live.live_simple_ast_accuracy": 85.66, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 79.01, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 68.75, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 70.83, - "bfcl/bfcl.multi_turn.accuracy": 34.75, - "bfcl/bfcl.multi_turn.base_accuracy": 39.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 34.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 33.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 32.5, - "bfcl/bfcl.web_search.accuracy": 10.0, - "bfcl/bfcl.web_search.base_accuracy": 8.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 12.0, - "bfcl/bfcl.memory.accuracy": 19.57, - "bfcl/bfcl.memory.kv_accuracy": 7.1, - "bfcl/bfcl.memory.vector_accuracy": 16.77, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 34.84, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 87.5, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 81.94 - } - }, - { - "id": "qwen/qwen3-14b-prompt", - "name": "Qwen3-14B (Prompt)", - "developer": "qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 47.0, - "bfcl/bfcl.overall.overall_accuracy": 37.77, - "bfcl/bfcl.overall.total_cost_usd": 1.35, - "bfcl/bfcl.overall.latency_mean_s": 1.2, - "bfcl/bfcl.overall.latency_std_s": 8.5, - "bfcl/bfcl.overall.latency_p95_s": 2.3, - "bfcl/bfcl.non_live.ast_accuracy": 89.46, - "bfcl/bfcl.non_live.simple_ast_accuracy": 76.83, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 93.5, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 95.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 92.0, - "bfcl/bfcl.live.live_accuracy": 79.35, - "bfcl/bfcl.live.live_simple_ast_accuracy": 84.5, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 78.06, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 87.5, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 75.0, - "bfcl/bfcl.multi_turn.accuracy": 26.13, - "bfcl/bfcl.multi_turn.base_accuracy": 16.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 37.5, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 31.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 19.5, - "bfcl/bfcl.web_search.accuracy": 10.5, - "bfcl/bfcl.web_search.base_accuracy": 6.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 15.0, - "bfcl/bfcl.memory.accuracy": 11.18, - "bfcl/bfcl.memory.kv_accuracy": 4.52, - "bfcl/bfcl.memory.vector_accuracy": 6.45, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 22.58, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 81.25, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 87.18, - "bfcl/bfcl.format_sensitivity.max_delta": 14.0, - "bfcl/bfcl.format_sensitivity.stddev": 3.97 - } - }, - { - "id": "qwen/qwen3-235b-a22b-fp8-tput", - "name": "Qwen3 235B A22B FP8 Throughput", - "developer": "qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.726, - "helm_capabilities/MMLU-Pro": 0.817, - "helm_capabilities/GPQA": 0.623, - "helm_capabilities/IFEval": 0.816, - "helm_capabilities/WildBench": 0.828, - "helm_capabilities/Omni-MATH": 0.548 - } - }, - { - "id": "qwen/qwen3-235b-a22b-instruct-2507-fc", - "name": "Qwen3-235B-A22B-Instruct-2507 (FC)", - "developer": "qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 31.0, - "bfcl/bfcl.overall.overall_accuracy": 47.99, - "bfcl/bfcl.overall.total_cost_usd": 2.5, - "bfcl/bfcl.overall.latency_mean_s": 2.57, - "bfcl/bfcl.overall.latency_std_s": 2.44, - "bfcl/bfcl.overall.latency_p95_s": 6.27, - "bfcl/bfcl.non_live.ast_accuracy": 37.4, - "bfcl/bfcl.non_live.simple_ast_accuracy": 40.58, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 36.5, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 53.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 19.5, - "bfcl/bfcl.live.live_accuracy": 68.91, - "bfcl/bfcl.live.live_simple_ast_accuracy": 58.53, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 71.6, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 68.75, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 62.5, - "bfcl/bfcl.multi_turn.accuracy": 45.38, - "bfcl/bfcl.multi_turn.base_accuracy": 57.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 35.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 33.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 55.5, - "bfcl/bfcl.web_search.accuracy": 54.0, - "bfcl/bfcl.web_search.base_accuracy": 57.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 51.0, - "bfcl/bfcl.memory.accuracy": 23.87, - "bfcl/bfcl.memory.kv_accuracy": 7.1, - "bfcl/bfcl.memory.vector_accuracy": 18.71, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 45.81, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 87.5, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 81.73 - } - }, - { - "id": "qwen/qwen3-235b-a22b-instruct-2507-fp8", - "name": "Qwen3 235B A22B Instruct 2507 FP8", - "developer": "qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.798, - "helm_capabilities/MMLU-Pro": 0.844, - "helm_capabilities/GPQA": 0.726, - "helm_capabilities/IFEval": 0.835, - "helm_capabilities/WildBench": 0.866, - "helm_capabilities/Omni-MATH": 0.718 - } - }, - { - "id": "qwen/qwen3-235b-a22b-instruct-2507-prompt", - "name": "Qwen3-235B-A22B-Instruct-2507 (Prompt)", - "developer": "qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 23.0, - "bfcl/bfcl.overall.overall_accuracy": 52.15, - "bfcl/bfcl.overall.total_cost_usd": 3.12, - "bfcl/bfcl.overall.latency_mean_s": 2.56, - "bfcl/bfcl.overall.latency_std_s": 2.75, - "bfcl/bfcl.overall.latency_p95_s": 7.61, - "bfcl/bfcl.non_live.ast_accuracy": 90.33, - "bfcl/bfcl.non_live.simple_ast_accuracy": 79.83, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 95.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 95.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 91.0, - "bfcl/bfcl.live.live_accuracy": 78.68, - "bfcl/bfcl.live.live_simple_ast_accuracy": 82.95, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 77.78, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 81.25, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 70.83, - "bfcl/bfcl.multi_turn.accuracy": 44.62, - "bfcl/bfcl.multi_turn.base_accuracy": 54.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 42.5, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 31.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 50.5, - "bfcl/bfcl.web_search.accuracy": 50.5, - "bfcl/bfcl.web_search.base_accuracy": 56.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 45.0, - "bfcl/bfcl.memory.accuracy": 19.35, - "bfcl/bfcl.memory.kv_accuracy": 12.9, - "bfcl/bfcl.memory.vector_accuracy": 11.61, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 33.55, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 93.75, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 78.89, - "bfcl/bfcl.format_sensitivity.max_delta": 8.0, - "bfcl/bfcl.format_sensitivity.stddev": 1.95 - } - }, - { - "id": "qwen/qwen3-30b-a3b-instruct-2507-fc", - "name": "Qwen3-30B-A3B-Instruct-2507 (FC)", - "developer": "qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 41.0, - "bfcl/bfcl.overall.overall_accuracy": 41.39, - "bfcl/bfcl.overall.total_cost_usd": 5.62, - "bfcl/bfcl.overall.latency_mean_s": 5.95, - "bfcl/bfcl.overall.latency_std_s": 25.48, - "bfcl/bfcl.overall.latency_p95_s": 12.7, - "bfcl/bfcl.non_live.ast_accuracy": 85.77, - "bfcl/bfcl.non_live.simple_ast_accuracy": 68.58, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 94.5, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 91.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 88.5, - "bfcl/bfcl.live.live_accuracy": 77.94, - "bfcl/bfcl.live.live_simple_ast_accuracy": 83.33, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 76.83, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 68.75, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 75.0, - "bfcl/bfcl.multi_turn.accuracy": 30.0, - "bfcl/bfcl.multi_turn.base_accuracy": 43.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 10.5, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 25.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 41.0, - "bfcl/bfcl.web_search.accuracy": 22.5, - "bfcl/bfcl.web_search.base_accuracy": 21.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 24.0, - "bfcl/bfcl.memory.accuracy": 17.63, - "bfcl/bfcl.memory.kv_accuracy": 9.03, - "bfcl/bfcl.memory.vector_accuracy": 9.03, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 34.84, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 81.25, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 79.9 - } - }, - { - "id": "qwen/qwen3-30b-a3b-instruct-2507-prompt", - "name": "Qwen3-30B-A3B-Instruct-2507 (Prompt)", - "developer": "qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 53.0, - "bfcl/bfcl.overall.overall_accuracy": 36.7, - "bfcl/bfcl.overall.total_cost_usd": 1.56, - "bfcl/bfcl.overall.latency_mean_s": 1.24, - "bfcl/bfcl.overall.latency_std_s": 7.9, - "bfcl/bfcl.overall.latency_p95_s": 2.84, - "bfcl/bfcl.non_live.ast_accuracy": 88.92, - "bfcl/bfcl.non_live.simple_ast_accuracy": 80.67, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 93.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 94.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 88.0, - "bfcl/bfcl.live.live_accuracy": 78.39, - "bfcl/bfcl.live.live_simple_ast_accuracy": 82.56, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 77.49, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 87.5, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 66.67, - "bfcl/bfcl.multi_turn.accuracy": 23.5, - "bfcl/bfcl.multi_turn.base_accuracy": 33.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 16.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 16.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 29.0, - "bfcl/bfcl.web_search.accuracy": 17.5, - "bfcl/bfcl.web_search.base_accuracy": 15.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 20.0, - "bfcl/bfcl.memory.accuracy": 9.68, - "bfcl/bfcl.memory.kv_accuracy": 5.81, - "bfcl/bfcl.memory.vector_accuracy": 6.45, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 16.77, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 93.75, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 74.85, - "bfcl/bfcl.format_sensitivity.max_delta": 16.0, - "bfcl/bfcl.format_sensitivity.stddev": 4.13 - } - }, - { - "id": "qwen/qwen3-32b-fc", - "name": "Qwen3-32B (FC)", - "developer": "qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 29.0, - "bfcl/bfcl.overall.overall_accuracy": 48.71, - "bfcl/bfcl.overall.total_cost_usd": 153.08, - "bfcl/bfcl.overall.latency_mean_s": 169.87, - "bfcl/bfcl.overall.latency_std_s": 164.27, - "bfcl/bfcl.overall.latency_p95_s": 473.49, - "bfcl/bfcl.non_live.ast_accuracy": 88.77, - "bfcl/bfcl.non_live.simple_ast_accuracy": 75.58, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 94.5, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 93.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 91.5, - "bfcl/bfcl.live.live_accuracy": 82.01, - "bfcl/bfcl.live.live_simple_ast_accuracy": 89.53, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 80.91, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 81.25, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 50.0, - "bfcl/bfcl.multi_turn.accuracy": 47.87, - "bfcl/bfcl.multi_turn.base_accuracy": 56.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 52.5, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 40.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 43.0, - "bfcl/bfcl.web_search.accuracy": 21.5, - "bfcl/bfcl.web_search.base_accuracy": 25.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 18.0, - "bfcl/bfcl.memory.accuracy": 26.67, - "bfcl/bfcl.memory.kv_accuracy": 12.26, - "bfcl/bfcl.memory.vector_accuracy": 25.81, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 41.94, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 93.75, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 76.37 - } - }, - { - "id": "qwen/qwen3-32b-prompt", - "name": "Qwen3-32B (Prompt)", - "developer": "qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 33.0, - "bfcl/bfcl.overall.overall_accuracy": 46.78, - "bfcl/bfcl.overall.total_cost_usd": 199.47, - "bfcl/bfcl.overall.latency_mean_s": 167.54, - "bfcl/bfcl.overall.latency_std_s": 160.5, - "bfcl/bfcl.overall.latency_p95_s": 457.87, - "bfcl/bfcl.non_live.ast_accuracy": 90.27, - "bfcl/bfcl.non_live.simple_ast_accuracy": 79.08, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 97.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 93.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 91.5, - "bfcl/bfcl.live.live_accuracy": 82.01, - "bfcl/bfcl.live.live_simple_ast_accuracy": 87.21, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 81.2, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 81.25, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 62.5, - "bfcl/bfcl.multi_turn.accuracy": 43.25, - "bfcl/bfcl.multi_turn.base_accuracy": 54.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 46.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 36.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 36.5, - "bfcl/bfcl.web_search.accuracy": 26.0, - "bfcl/bfcl.web_search.base_accuracy": 34.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 18.0, - "bfcl/bfcl.memory.accuracy": 15.7, - "bfcl/bfcl.memory.kv_accuracy": 13.55, - "bfcl/bfcl.memory.vector_accuracy": 14.19, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 19.35, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 81.25, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 82.39, - "bfcl/bfcl.format_sensitivity.max_delta": 15.5, - "bfcl/bfcl.format_sensitivity.stddev": 3.75 - } - }, - { - "id": "qwen/qwen3-4b-instruct-2507-fc", - "name": "Qwen3-4B-Instruct-2507 (FC)", - "developer": "qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 54.0, - "bfcl/bfcl.overall.overall_accuracy": 35.68, - "bfcl/bfcl.overall.total_cost_usd": 6.37, - "bfcl/bfcl.overall.latency_mean_s": 7.61, - "bfcl/bfcl.overall.latency_std_s": 20.36, - "bfcl/bfcl.overall.latency_p95_s": 49.18, - "bfcl/bfcl.non_live.ast_accuracy": 87.88, - "bfcl/bfcl.non_live.simple_ast_accuracy": 75.5, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 93.5, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 92.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 90.0, - "bfcl/bfcl.live.live_accuracy": 76.39, - "bfcl/bfcl.live.live_simple_ast_accuracy": 79.07, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 76.16, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 62.5, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 66.67, - "bfcl/bfcl.multi_turn.accuracy": 22.12, - "bfcl/bfcl.multi_turn.base_accuracy": 26.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 21.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 15.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 25.5, - "bfcl/bfcl.web_search.accuracy": 3.0, - "bfcl/bfcl.web_search.base_accuracy": 4.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 2.0, - "bfcl/bfcl.memory.accuracy": 17.63, - "bfcl/bfcl.memory.kv_accuracy": 16.13, - "bfcl/bfcl.memory.vector_accuracy": 12.26, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 24.52, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 87.5, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 84.93 - } - }, - { - "id": "qwen/qwen3-4b-instruct-2507-prompt", - "name": "Qwen3-4B-Instruct-2507 (Prompt)", - "developer": "qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 55.0, - "bfcl/bfcl.overall.overall_accuracy": 35.52, - "bfcl/bfcl.overall.total_cost_usd": 53.66, - "bfcl/bfcl.overall.latency_mean_s": 44.7, - "bfcl/bfcl.overall.latency_std_s": 163.79, - "bfcl/bfcl.overall.latency_p95_s": 208.06, - "bfcl/bfcl.non_live.ast_accuracy": 86.44, - "bfcl/bfcl.non_live.simple_ast_accuracy": 77.25, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 91.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 88.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 89.5, - "bfcl/bfcl.live.live_accuracy": 74.69, - "bfcl/bfcl.live.live_simple_ast_accuracy": 77.91, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 74.17, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 81.25, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 58.33, - "bfcl/bfcl.multi_turn.accuracy": 20.5, - "bfcl/bfcl.multi_turn.base_accuracy": 24.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 21.5, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 16.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 20.0, - "bfcl/bfcl.web_search.accuracy": 4.5, - "bfcl/bfcl.web_search.base_accuracy": 4.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 5.0, - "bfcl/bfcl.memory.accuracy": 23.87, - "bfcl/bfcl.memory.kv_accuracy": 12.9, - "bfcl/bfcl.memory.vector_accuracy": 14.19, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 44.52, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 87.5, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 75.87, - "bfcl/bfcl.format_sensitivity.max_delta": 18.0, - "bfcl/bfcl.format_sensitivity.stddev": 5.22 - } - }, - { - "id": "qwen/qwen3-8b-fc", - "name": "Qwen3-8B (FC)", - "developer": "qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 39.0, - "bfcl/bfcl.overall.overall_accuracy": 42.57, - "bfcl/bfcl.overall.total_cost_usd": 43.32, - "bfcl/bfcl.overall.latency_mean_s": 51.36, - "bfcl/bfcl.overall.latency_std_s": 76.14, - "bfcl/bfcl.overall.latency_p95_s": 188.98, - "bfcl/bfcl.non_live.ast_accuracy": 87.58, - "bfcl/bfcl.non_live.simple_ast_accuracy": 72.83, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 96.5, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 92.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 89.0, - "bfcl/bfcl.live.live_accuracy": 80.53, - "bfcl/bfcl.live.live_simple_ast_accuracy": 84.5, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 79.68, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 75.0, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 79.17, - "bfcl/bfcl.multi_turn.accuracy": 41.75, - "bfcl/bfcl.multi_turn.base_accuracy": 50.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 42.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 40.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 34.5, - "bfcl/bfcl.web_search.accuracy": 12.0, - "bfcl/bfcl.web_search.base_accuracy": 15.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 9.0, - "bfcl/bfcl.memory.accuracy": 14.62, - "bfcl/bfcl.memory.kv_accuracy": 5.16, - "bfcl/bfcl.memory.vector_accuracy": 7.1, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 31.61, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 93.75, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 79.07 - } - }, - { - "id": "qwen/qwen3-8b-prompt", - "name": "Qwen3-8B (Prompt)", - "developer": "qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 44.0, - "bfcl/bfcl.overall.overall_accuracy": 40.43, - "bfcl/bfcl.overall.total_cost_usd": 63.95, - "bfcl/bfcl.overall.latency_mean_s": 54.17, - "bfcl/bfcl.overall.latency_std_s": 79.9, - "bfcl/bfcl.overall.latency_p95_s": 194.15, - "bfcl/bfcl.non_live.ast_accuracy": 88.56, - "bfcl/bfcl.non_live.simple_ast_accuracy": 75.25, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 95.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 94.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 89.5, - "bfcl/bfcl.live.live_accuracy": 80.09, - "bfcl/bfcl.live.live_simple_ast_accuracy": 84.5, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 78.92, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 93.75, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 75.0, - "bfcl/bfcl.multi_turn.accuracy": 33.38, - "bfcl/bfcl.multi_turn.base_accuracy": 41.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 38.5, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 27.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 26.5, - "bfcl/bfcl.web_search.accuracy": 13.5, - "bfcl/bfcl.web_search.base_accuracy": 19.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 8.0, - "bfcl/bfcl.memory.accuracy": 13.12, - "bfcl/bfcl.memory.kv_accuracy": 3.87, - "bfcl/bfcl.memory.vector_accuracy": 10.32, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 25.16, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 75.0, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 82.27, - "bfcl/bfcl.format_sensitivity.max_delta": 16.5, - "bfcl/bfcl.format_sensitivity.stddev": 5.09 - } - }, - { - "id": "Qwen/QwQ-32B", - "name": "QwQ-32B", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3977, - "hfopenllm_v2/BBH": 0.2983, - "hfopenllm_v2/MATH Level 5": 0.1609, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.4206, - "hfopenllm_v2/MMLU-PRO": 0.1196 - } - }, - { - "id": "Qwen/QwQ-32B-Preview", - "name": "QwQ-32B-Preview", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4035, - "hfopenllm_v2/BBH": 0.6691, - "hfopenllm_v2/MATH Level 5": 0.4494, - "hfopenllm_v2/GPQA": 0.2819, - "hfopenllm_v2/MUSR": 0.411, - "hfopenllm_v2/MMLU-PRO": 0.5678 - } - }, - { - "id": "Qwen/WorldPM-72B", - "name": "Qwen/WorldPM-72B", - "developer": "Qwen", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6333, - "reward-bench/Factuality": 0.7074, - "reward-bench/Precise IF": 0.3125, - "reward-bench/Math": 0.6557, - "reward-bench/Safety": 0.8533, - "reward-bench/Focus": 0.9172, - "reward-bench/Ties": 0.3535 - } - }, - { - "id": "R-I-S-E/RISE-Judge-Qwen2.5-32B", - "name": "R-I-S-E/RISE-Judge-Qwen2.5-32B", - "developer": "R-I-S-E", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.9266, - "reward-bench/Chat": 0.9665, - "reward-bench/Chat Hard": 0.8333, - "reward-bench/Safety": 0.9189, - "reward-bench/Reasoning": 0.9877 - } - }, - { - "id": "R-I-S-E/RISE-Judge-Qwen2.5-7B", - "name": "R-I-S-E/RISE-Judge-Qwen2.5-7B", - "developer": "R-I-S-E", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.8819, - "reward-bench/Chat": 0.9218, - "reward-bench/Chat Hard": 0.7654, - "reward-bench/Safety": 0.8797, - "reward-bench/Reasoning": 0.9608 - } - }, - { - "id": "Rakuten/RakutenAI-2.0-mini-instruct", - "name": "RakutenAI-2.0-mini-instruct", - "developer": "Rakuten", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6794, - "hfopenllm_v2/BBH": 0.2867, - "hfopenllm_v2/MATH Level 5": 0.0521, - "hfopenllm_v2/GPQA": 0.2668, - "hfopenllm_v2/MUSR": 0.3249, - "hfopenllm_v2/MMLU-PRO": 0.1118 - } - }, - { - "id": "Rakuten/RakutenAI-7B", - "name": "RakutenAI-7B", - "developer": "Rakuten", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1556, - "hfopenllm_v2/BBH": 0.4315, - "hfopenllm_v2/MATH Level 5": 0.0196, - "hfopenllm_v2/GPQA": 0.2894, - "hfopenllm_v2/MUSR": 0.3738, - "hfopenllm_v2/MMLU-PRO": 0.2877 - } - }, - { - "id": "Rakuten/RakutenAI-7B-chat", - "name": "RakutenAI-7B-chat", - "developer": "Rakuten", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2686, - "hfopenllm_v2/BBH": 0.4316, - "hfopenllm_v2/MATH Level 5": 0.0295, - "hfopenllm_v2/GPQA": 0.2567, - "hfopenllm_v2/MUSR": 0.379, - "hfopenllm_v2/MMLU-PRO": 0.2798 - } - }, - { - "id": "raphgg/test-2.5-72B", - "name": "test-2.5-72B", - "developer": "raphgg", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8437, - "hfopenllm_v2/BBH": 0.7266, - "hfopenllm_v2/MATH Level 5": 0.4109, - "hfopenllm_v2/GPQA": 0.3893, - "hfopenllm_v2/MUSR": 0.4812, - "hfopenllm_v2/MMLU-PRO": 0.5837 - } - }, - { - "id": "rasyosef/Mistral-NeMo-Minitron-8B-Chat", - "name": "Mistral-NeMo-Minitron-8B-Chat", - "developer": "rasyosef", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4452, - "hfopenllm_v2/BBH": 0.4759, - "hfopenllm_v2/MATH Level 5": 0.0272, - "hfopenllm_v2/GPQA": 0.276, - "hfopenllm_v2/MUSR": 0.4304, - "hfopenllm_v2/MMLU-PRO": 0.2404 - } - }, - { - "id": "rasyosef/Phi-1_5-Instruct-v0.1", - "name": "Phi-1_5-Instruct-v0.1", - "developer": "rasyosef", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2402, - "hfopenllm_v2/BBH": 0.3118, - "hfopenllm_v2/MATH Level 5": 0.0136, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3422, - "hfopenllm_v2/MMLU-PRO": 0.1562 - } - }, - { - "id": "rasyosef/phi-2-instruct-apo", - "name": "phi-2-instruct-apo", - "developer": "rasyosef", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3146, - "hfopenllm_v2/BBH": 0.4445, - "hfopenllm_v2/MATH Level 5": 0.0302, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.3342, - "hfopenllm_v2/MMLU-PRO": 0.2155 - } - }, - { - "id": "rasyosef/phi-2-instruct-v0.1", - "name": "phi-2-instruct-v0.1", - "developer": "rasyosef", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3681, - "hfopenllm_v2/BBH": 0.4726, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2743, - "hfopenllm_v2/MUSR": 0.3524, - "hfopenllm_v2/MMLU-PRO": 0.2247 - } - }, - { - "id": "Ray2333/Gemma-2B-rewardmodel-baseline", - "name": "Ray2333/Gemma-2B-rewardmodel-baseline", - "developer": "Ray2333", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.729, - "reward-bench/Chat": 0.9413, - "reward-bench/Chat Hard": 0.4693, - "reward-bench/Safety": 0.7865, - "reward-bench/Reasoning": 0.7384, - "reward-bench/Prior Sets (0.5 weight)": 0.6897 - } - }, - { - "id": "Ray2333/Gemma-2B-rewardmodel-ft", - "name": "Ray2333/Gemma-2B-rewardmodel-ft", - "developer": "Ray2333", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.8048, - "reward-bench/Chat": 0.7793, - "reward-bench/Chat Hard": 0.7478, - "reward-bench/Safety": 0.8527, - "reward-bench/Reasoning": 0.8393 - } - }, - { - "id": "Ray2333/GRM-Gemma-2B-rewardmodel-ft", - "name": "Ray2333/GRM-Gemma-2B-rewardmodel-ft", - "developer": "Ray2333", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.8447, - "reward-bench/Chat": 0.8939, - "reward-bench/Chat Hard": 0.7522, - "reward-bench/Safety": 0.8446, - "reward-bench/Reasoning": 0.8881 - } - }, - { - "id": "Ray2333/GRM-Gemma-2B-sftreg", - "name": "Ray2333/GRM-Gemma-2B-sftreg", - "developer": "Ray2333", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7451, - "reward-bench/Chat": 0.9553, - "reward-bench/Chat Hard": 0.4868, - "reward-bench/Safety": 0.7932, - "reward-bench/Reasoning": 0.7684, - "reward-bench/Prior Sets (0.5 weight)": 0.6983 - } - }, - { - "id": "Ray2333/GRM-gemma2-2B-rewardmodel-ft", - "name": "Ray2333/GRM-gemma2-2B-rewardmodel-ft", - "developer": "Ray2333", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5966, - "reward-bench/Chat": 0.9302, - "reward-bench/Chat Hard": 0.7719, - "reward-bench/Safety": 0.9222, - "reward-bench/Reasoning": 0.912, - "reward-bench/Factuality": 0.5305, - "reward-bench/Precise IF": 0.3125, - "reward-bench/Math": 0.5902, - "reward-bench/Focus": 0.7455, - "reward-bench/Ties": 0.4788 - } - }, - { - "id": "Ray2333/GRM-llama3-8B-distill", - "name": "Ray2333/GRM-llama3-8B-distill", - "developer": "Ray2333", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.589, - "reward-bench/Chat": 0.9832, - "reward-bench/Chat Hard": 0.6842, - "reward-bench/Safety": 0.7222, - "reward-bench/Reasoning": 0.9133, - "reward-bench/Prior Sets (0.5 weight)": 0.7209, - "reward-bench/Factuality": 0.5874, - "reward-bench/Precise IF": 0.3875, - "reward-bench/Math": 0.5902, - "reward-bench/Focus": 0.6727, - "reward-bench/Ties": 0.5743 - } - }, - { - "id": "Ray2333/GRM-Llama3-8B-rewardmodel-ft", - "name": "Ray2333/GRM-Llama3-8B-rewardmodel-ft", - "developer": "Ray2333", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6766, - "reward-bench/Chat": 0.9553, - "reward-bench/Chat Hard": 0.8618, - "reward-bench/Safety": 0.9222, - "reward-bench/Reasoning": 0.9362, - "reward-bench/Factuality": 0.6274, - "reward-bench/Precise IF": 0.35, - "reward-bench/Math": 0.5847, - "reward-bench/Focus": 0.8929, - "reward-bench/Ties": 0.6824 - } - }, - { - "id": "Ray2333/GRM-llama3-8B-sftreg", - "name": "Ray2333/GRM-llama3-8B-sftreg", - "developer": "Ray2333", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6089, - "reward-bench/Chat": 0.986, - "reward-bench/Chat Hard": 0.6776, - "reward-bench/Safety": 0.7867, - "reward-bench/Reasoning": 0.9229, - "reward-bench/Prior Sets (0.5 weight)": 0.7309, - "reward-bench/Factuality": 0.6189, - "reward-bench/Precise IF": 0.3875, - "reward-bench/Math": 0.5792, - "reward-bench/Focus": 0.6828, - "reward-bench/Ties": 0.5981 - } - }, - { - "id": "Ray2333/GRM-llama3.2-3B-rewardmodel-ft", - "name": "Ray2333/GRM-llama3.2-3B-rewardmodel-ft", - "developer": "Ray2333", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.9092, - "reward-bench/Chat": 0.9162, - "reward-bench/Chat Hard": 0.8487, - "reward-bench/Safety": 0.927, - "reward-bench/Reasoning": 0.945 - } - }, - { - "id": "Ray2333/reward-model-Mistral-7B-instruct-Unifie...", - "name": "Ray2333/reward-model-Mistral-7B-instruct-Unifie...", - "developer": "Ray2333", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7661, - "reward-bench/Chat": 0.9777, - "reward-bench/Chat Hard": 0.5066, - "reward-bench/Safety": 0.8527, - "reward-bench/Reasoning": 0.7389, - "reward-bench/Prior Sets (0.5 weight)": 0.7434 - } - }, - { - "id": "RDson/WomboCombo-R1-Coder-14B-Preview", - "name": "WomboCombo-R1-Coder-14B-Preview", - "developer": "RDson", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6286, - "hfopenllm_v2/BBH": 0.6392, - "hfopenllm_v2/MATH Level 5": 0.5989, - "hfopenllm_v2/GPQA": 0.3213, - "hfopenllm_v2/MUSR": 0.4844, - "hfopenllm_v2/MMLU-PRO": 0.5168 - } - }, - { - "id": "realtreetune/rho-1b-sft-MATH", - "name": "rho-1b-sft-MATH", - "developer": "realtreetune", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2121, - "hfopenllm_v2/BBH": 0.3144, - "hfopenllm_v2/MATH Level 5": 0.0347, - "hfopenllm_v2/GPQA": 0.2525, - "hfopenllm_v2/MUSR": 0.3458, - "hfopenllm_v2/MMLU-PRO": 0.1117 - } - }, - { - "id": "recoilme/Gemma-2-Ataraxy-Gemmasutra-9B-slerp", - "name": "Gemma-2-Ataraxy-Gemmasutra-9B-slerp", - "developer": "recoilme", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7649, - "hfopenllm_v2/BBH": 0.5974, - "hfopenllm_v2/MATH Level 5": 0.0174, - "hfopenllm_v2/GPQA": 0.3305, - "hfopenllm_v2/MUSR": 0.4245, - "hfopenllm_v2/MMLU-PRO": 0.4207 - } - }, - { - "id": "recoilme/recoilme-gemma-2-9B-v0.1", - "name": "recoilme-gemma-2-9B-v0.1", - "developer": "recoilme", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7515, - "hfopenllm_v2/BBH": 0.5995, - "hfopenllm_v2/MATH Level 5": 0.2039, - "hfopenllm_v2/GPQA": 0.3389, - "hfopenllm_v2/MUSR": 0.4191, - "hfopenllm_v2/MMLU-PRO": 0.4159 - } - }, - { - "id": "recoilme/recoilme-gemma-2-9B-v0.2", - "name": "recoilme-gemma-2-9B-v0.2", - "developer": "recoilme", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2747, - "hfopenllm_v2/BBH": 0.6031, - "hfopenllm_v2/MATH Level 5": 0.0831, - "hfopenllm_v2/GPQA": 0.3305, - "hfopenllm_v2/MUSR": 0.4686, - "hfopenllm_v2/MMLU-PRO": 0.4122 - } - }, - { - "id": "recoilme/recoilme-gemma-2-9B-v0.3", - "name": "recoilme-gemma-2-9B-v0.3", - "developer": "recoilme", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5761, - "hfopenllm_v2/BBH": 0.602, - "hfopenllm_v2/MATH Level 5": 0.1888, - "hfopenllm_v2/GPQA": 0.3372, - "hfopenllm_v2/MUSR": 0.4632, - "hfopenllm_v2/MMLU-PRO": 0.4039 - } - }, - { - "id": "recoilme/recoilme-gemma-2-9B-v0.4", - "name": "recoilme-gemma-2-9B-v0.4", - "developer": "recoilme", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2562, - "hfopenllm_v2/BBH": 0.5967, - "hfopenllm_v2/MATH Level 5": 0.0846, - "hfopenllm_v2/GPQA": 0.3406, - "hfopenllm_v2/MUSR": 0.4727, - "hfopenllm_v2/MMLU-PRO": 0.4406 - } - }, - { - "id": "recoilme/recoilme-gemma-2-9B-v0.5", - "name": "recoilme-gemma-2-9B-v0.5", - "developer": "recoilme", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7664, - "hfopenllm_v2/BBH": 0.5981, - "hfopenllm_v2/MATH Level 5": 0.2115, - "hfopenllm_v2/GPQA": 0.3364, - "hfopenllm_v2/MUSR": 0.4232, - "hfopenllm_v2/MMLU-PRO": 0.42 - } - }, - { - "id": "redrix/AngelSlayer-12B-Unslop-Mell-RPMax-DARKNESS", - "name": "AngelSlayer-12B-Unslop-Mell-RPMax-DARKNESS", - "developer": "redrix", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.536, - "hfopenllm_v2/BBH": 0.5129, - "hfopenllm_v2/MATH Level 5": 0.1133, - "hfopenllm_v2/GPQA": 0.3154, - "hfopenllm_v2/MUSR": 0.3818, - "hfopenllm_v2/MMLU-PRO": 0.318 - } - }, - { - "id": "redrix/patricide-12B-Unslop-Mell", - "name": "patricide-12B-Unslop-Mell", - "developer": "redrix", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4074, - "hfopenllm_v2/BBH": 0.5399, - "hfopenllm_v2/MATH Level 5": 0.1314, - "hfopenllm_v2/GPQA": 0.3238, - "hfopenllm_v2/MUSR": 0.4026, - "hfopenllm_v2/MMLU-PRO": 0.357 - } - }, - { - "id": "refuelai/Llama-3-Refueled", - "name": "Llama-3-Refueled", - "developer": "refuelai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.462, - "hfopenllm_v2/BBH": 0.5871, - "hfopenllm_v2/MATH Level 5": 0.0665, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.4454, - "hfopenllm_v2/MMLU-PRO": 0.3095 - } - }, - { - "id": "Replete-AI/L3-Pneuma-8B", - "name": "L3-Pneuma-8B", - "developer": "Replete-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2413, - "hfopenllm_v2/BBH": 0.4909, - "hfopenllm_v2/MATH Level 5": 0.0544, - "hfopenllm_v2/GPQA": 0.318, - "hfopenllm_v2/MUSR": 0.4105, - "hfopenllm_v2/MMLU-PRO": 0.3176 - } - }, - { - "id": "Replete-AI/L3.1-Pneuma-8B", - "name": "L3.1-Pneuma-8B", - "developer": "Replete-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7076, - "hfopenllm_v2/BBH": 0.505, - "hfopenllm_v2/MATH Level 5": 0.2198, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.3871, - "hfopenllm_v2/MMLU-PRO": 0.3691 - } - }, - { - "id": "Replete-AI/Llama3-8B-Instruct-Replete-Adapted", - "name": "Llama3-8B-Instruct-Replete-Adapted", - "developer": "Replete-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6915, - "hfopenllm_v2/BBH": 0.487, - "hfopenllm_v2/MATH Level 5": 0.071, - "hfopenllm_v2/GPQA": 0.281, - "hfopenllm_v2/MUSR": 0.3634, - "hfopenllm_v2/MMLU-PRO": 0.3391 - } - }, - { - "id": "Replete-AI/Replete-Coder-Instruct-8b-Merged", - "name": "Replete-Coder-Instruct-8b-Merged", - "developer": "Replete-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5388, - "hfopenllm_v2/BBH": 0.4462, - "hfopenllm_v2/MATH Level 5": 0.0778, - "hfopenllm_v2/GPQA": 0.2693, - "hfopenllm_v2/MUSR": 0.366, - "hfopenllm_v2/MMLU-PRO": 0.1805 - } - }, - { - "id": "Replete-AI/Replete-Coder-Llama3-8B", - "name": "Replete-Coder-Llama3-8B", - "developer": "Replete-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4729, - "hfopenllm_v2/BBH": 0.3271, - "hfopenllm_v2/MATH Level 5": 0.0476, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.3953, - "hfopenllm_v2/MMLU-PRO": 0.1331 - } - }, - { - "id": "Replete-AI/Replete-Coder-Qwen2-1.5b", - "name": "Replete-Coder-Qwen2-1.5b", - "developer": "Replete-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3014, - "hfopenllm_v2/BBH": 0.3475, - "hfopenllm_v2/MATH Level 5": 0.0385, - "hfopenllm_v2/GPQA": 0.2685, - "hfopenllm_v2/MUSR": 0.4073, - "hfopenllm_v2/MMLU-PRO": 0.2147 - } - }, - { - "id": "Replete-AI/Replete-LLM-Qwen2-7b", - "name": "Replete-LLM-Qwen2-7b", - "developer": "Replete-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0932, - "hfopenllm_v2/BBH": 0.2977, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2475, - "hfopenllm_v2/MUSR": 0.3941, - "hfopenllm_v2/MMLU-PRO": 0.1157 - } - }, - { - "id": "Replete-AI/Replete-LLM-Qwen2-7b_Beta-Preview", - "name": "Replete-LLM-Qwen2-7b_Beta-Preview", - "developer": "Replete-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0858, - "hfopenllm_v2/BBH": 0.2929, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2483, - "hfopenllm_v2/MUSR": 0.3981, - "hfopenllm_v2/MMLU-PRO": 0.1285 - } - }, - { - "id": "Replete-AI/Replete-LLM-V2-Llama-3.1-8b", - "name": "Replete-LLM-V2-Llama-3.1-8b", - "developer": "Replete-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5515, - "hfopenllm_v2/BBH": 0.5339, - "hfopenllm_v2/MATH Level 5": 0.1405, - "hfopenllm_v2/GPQA": 0.3138, - "hfopenllm_v2/MUSR": 0.4001, - "hfopenllm_v2/MMLU-PRO": 0.3753 - } - }, - { - "id": "RESMPDEV/EVA-Qwen2.5-1.5B-FRFR", - "name": "EVA-Qwen2.5-1.5B-FRFR", - "developer": "RESMPDEV", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3082, - "hfopenllm_v2/BBH": 0.3932, - "hfopenllm_v2/MATH Level 5": 0.1027, - "hfopenllm_v2/GPQA": 0.2794, - "hfopenllm_v2/MUSR": 0.3539, - "hfopenllm_v2/MMLU-PRO": 0.277 - } - }, - { - "id": "RESMPDEV/Qwen2-Wukong-0.5B", - "name": "Qwen2-Wukong-0.5B", - "developer": "RESMPDEV", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1854, - "hfopenllm_v2/BBH": 0.3085, - "hfopenllm_v2/MATH Level 5": 0.0015, - "hfopenllm_v2/GPQA": 0.2366, - "hfopenllm_v2/MUSR": 0.3525, - "hfopenllm_v2/MMLU-PRO": 0.1327 - } - }, - { - "id": "RezVortex/Jajuka-3b", - "name": "Jajuka-3b", - "developer": "RezVortex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6925, - "hfopenllm_v2/BBH": 0.4594, - "hfopenllm_v2/MATH Level 5": 0.1594, - "hfopenllm_v2/GPQA": 0.2659, - "hfopenllm_v2/MUSR": 0.3671, - "hfopenllm_v2/MMLU-PRO": 0.3137 - } - }, - { - "id": "RezVortex/JAJUKA-WEWILLNEVERFORGETYOU-3B", - "name": "JAJUKA-WEWILLNEVERFORGETYOU-3B", - "developer": "RezVortex", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6858, - "hfopenllm_v2/BBH": 0.4619, - "hfopenllm_v2/MATH Level 5": 0.1548, - "hfopenllm_v2/GPQA": 0.2576, - "hfopenllm_v2/MUSR": 0.363, - "hfopenllm_v2/MMLU-PRO": 0.3143 - } - }, - { - "id": "rhplus0831/maid-yuzu-v7", - "name": "maid-yuzu-v7", - "developer": "rhplus0831", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6462, - "hfopenllm_v2/BBH": 0.4805, - "hfopenllm_v2/MATH Level 5": 0.102, - "hfopenllm_v2/GPQA": 0.3096, - "hfopenllm_v2/MUSR": 0.4136, - "hfopenllm_v2/MMLU-PRO": 0.354 - } - }, - { - "id": "rhymes-ai/Aria", - "name": "Aria", - "developer": "rhymes-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4773, - "hfopenllm_v2/BBH": 0.5695, - "hfopenllm_v2/MATH Level 5": 0.1934, - "hfopenllm_v2/GPQA": 0.3624, - "hfopenllm_v2/MUSR": 0.4338, - "hfopenllm_v2/MMLU-PRO": 0.4405 - } - }, - { - "id": "rhysjones/phi-2-orange-v2", - "name": "phi-2-orange-v2", - "developer": "rhysjones", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.367, - "hfopenllm_v2/BBH": 0.477, - "hfopenllm_v2/MATH Level 5": 0.0408, - "hfopenllm_v2/GPQA": 0.2617, - "hfopenllm_v2/MUSR": 0.363, - "hfopenllm_v2/MMLU-PRO": 0.2532 - } - }, - { - "id": "riaz/FineLlama-3.1-8B", - "name": "FineLlama-3.1-8B", - "developer": "riaz", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4137, - "hfopenllm_v2/BBH": 0.4565, - "hfopenllm_v2/MATH Level 5": 0.0453, - "hfopenllm_v2/GPQA": 0.276, - "hfopenllm_v2/MUSR": 0.3776, - "hfopenllm_v2/MMLU-PRO": 0.2978 - } - }, - { - "id": "RLHFlow/ArmoRM-Llama3-8B-v0.1", - "name": "RLHFlow/ArmoRM-Llama3-8B-v0.1", - "developer": "RLHFlow", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1897, - "hfopenllm_v2/BBH": 0.2876, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2492, - "hfopenllm_v2/MUSR": 0.3948, - "hfopenllm_v2/MMLU-PRO": 0.1078, - "reward-bench/Score": 0.886, - "reward-bench/Factuality": 0.6568, - "reward-bench/Precise IF": 0.4188, - "reward-bench/Math": 0.6612, - "reward-bench/Safety": 0.9054, - "reward-bench/Focus": 0.7657, - "reward-bench/Ties": 0.6629, - "reward-bench/Chat": 0.9693, - "reward-bench/Chat Hard": 0.7675, - "reward-bench/Reasoning": 0.9735, - "reward-bench/Prior Sets (0.5 weight)": 0.7429 - } - }, - { - "id": "RLHFlow/LLaMA3-iterative-DPO-final", - "name": "RLHFlow/LLaMA3-iterative-DPO-final", - "developer": "RLHFlow", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.534, - "hfopenllm_v2/BBH": 0.5058, - "hfopenllm_v2/MATH Level 5": 0.0884, - "hfopenllm_v2/GPQA": 0.2836, - "hfopenllm_v2/MUSR": 0.3673, - "hfopenllm_v2/MMLU-PRO": 0.3257, - "reward-bench/Score": 0.6783, - "reward-bench/Chat": 0.838, - "reward-bench/Chat Hard": 0.5921, - "reward-bench/Safety": 0.7865, - "reward-bench/Reasoning": 0.6161, - "reward-bench/Prior Sets (0.5 weight)": 0.4392 - } - }, - { - "id": "RLHFlow/pair-preference-model-LLaMA3-8B", - "name": "RLHFlow/pair-preference-model-LLaMA3-8B", - "developer": "RLHFlow", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.8575, - "reward-bench/Chat": 0.9832, - "reward-bench/Chat Hard": 0.6579, - "reward-bench/Safety": 0.8973, - "reward-bench/Reasoning": 0.9473, - "reward-bench/Prior Sets (0.5 weight)": 0.7458 - } - }, - { - "id": "RLHFlow/RewardModel-Mistral-7B-for-DPA-v1", - "name": "RLHFlow/RewardModel-Mistral-7B-for-DPA-v1", - "developer": "RLHFlow", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6633, - "reward-bench/Chat": 0.8799, - "reward-bench/Chat Hard": 0.4978, - "reward-bench/Safety": 0.7068, - "reward-bench/Reasoning": 0.5971, - "reward-bench/Prior Sets (0.5 weight)": 0.6068 - } - }, - { - "id": "rmdhirr/Gluon-8B", - "name": "Gluon-8B", - "developer": "rmdhirr", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5053, - "hfopenllm_v2/BBH": 0.5153, - "hfopenllm_v2/MATH Level 5": 0.1443, - "hfopenllm_v2/GPQA": 0.3121, - "hfopenllm_v2/MUSR": 0.4039, - "hfopenllm_v2/MMLU-PRO": 0.3808 - } - }, - { - "id": "Ro-xe/FMixIA-7B-DARE-0", - "name": "FMixIA-7B-DARE-0", - "developer": "Ro-xe", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3341, - "hfopenllm_v2/BBH": 0.5035, - "hfopenllm_v2/MATH Level 5": 0.0529, - "hfopenllm_v2/GPQA": 0.2894, - "hfopenllm_v2/MUSR": 0.4545, - "hfopenllm_v2/MMLU-PRO": 0.3016 - } - }, - { - "id": "Ro-xe/FMixIA-7B-SLERP-27", - "name": "FMixIA-7B-SLERP-27", - "developer": "Ro-xe", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3765, - "hfopenllm_v2/BBH": 0.5151, - "hfopenllm_v2/MATH Level 5": 0.0634, - "hfopenllm_v2/GPQA": 0.2953, - "hfopenllm_v2/MUSR": 0.4412, - "hfopenllm_v2/MMLU-PRO": 0.3008 - } - }, - { - "id": "Ro-xe/FMixIA-7B-TIES-1", - "name": "FMixIA-7B-TIES-1", - "developer": "Ro-xe", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3453, - "hfopenllm_v2/BBH": 0.5092, - "hfopenllm_v2/MATH Level 5": 0.0566, - "hfopenllm_v2/GPQA": 0.2886, - "hfopenllm_v2/MUSR": 0.4689, - "hfopenllm_v2/MMLU-PRO": 0.2992 - } - }, - { - "id": "Ro-xe/FMixIA-FrankenMerge-9.5B-PT-9", - "name": "FMixIA-FrankenMerge-9.5B-PT-9", - "developer": "Ro-xe", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.194, - "hfopenllm_v2/BBH": 0.5088, - "hfopenllm_v2/MATH Level 5": 0.003, - "hfopenllm_v2/GPQA": 0.3079, - "hfopenllm_v2/MUSR": 0.417, - "hfopenllm_v2/MMLU-PRO": 0.3657 - } - }, - { - "id": "Rombo-Org/Rombo-LLM-V2.5-Qwen-7b", - "name": "Rombo-LLM-V2.5-Qwen-7b", - "developer": "Rombo-Org", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7482, - "hfopenllm_v2/BBH": 0.54, - "hfopenllm_v2/MATH Level 5": 0.5068, - "hfopenllm_v2/GPQA": 0.3012, - "hfopenllm_v2/MUSR": 0.398, - "hfopenllm_v2/MMLU-PRO": 0.4283 - } - }, - { - "id": "rombodawg/Rombos-Coder-V2.5-Qwen-14b", - "name": "Rombos-Coder-V2.5-Qwen-14b", - "developer": "rombodawg", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7047, - "hfopenllm_v2/BBH": 0.6165, - "hfopenllm_v2/MATH Level 5": 0.3301, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.3915, - "hfopenllm_v2/MMLU-PRO": 0.3939 - } - }, - { - "id": "rombodawg/Rombos-Coder-V2.5-Qwen-7b", - "name": "Rombos-Coder-V2.5-Qwen-7b", - "developer": "rombodawg", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.621, - "hfopenllm_v2/BBH": 0.5077, - "hfopenllm_v2/MATH Level 5": 0.3338, - "hfopenllm_v2/GPQA": 0.2836, - "hfopenllm_v2/MUSR": 0.3979, - "hfopenllm_v2/MMLU-PRO": 0.3398 - } - }, - { - "id": "rombodawg/Rombos-LLM-V2.5-Qwen-0.5b", - "name": "Rombos-LLM-V2.5-Qwen-0.5b", - "developer": "rombodawg", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2847, - "hfopenllm_v2/BBH": 0.3294, - "hfopenllm_v2/MATH Level 5": 0.068, - "hfopenllm_v2/GPQA": 0.2668, - "hfopenllm_v2/MUSR": 0.3236, - "hfopenllm_v2/MMLU-PRO": 0.1866 - } - }, - { - "id": "rombodawg/Rombos-LLM-V2.5-Qwen-1.5b", - "name": "Rombos-LLM-V2.5-Qwen-1.5b", - "developer": "rombodawg", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3402, - "hfopenllm_v2/BBH": 0.4257, - "hfopenllm_v2/MATH Level 5": 0.0853, - "hfopenllm_v2/GPQA": 0.2886, - "hfopenllm_v2/MUSR": 0.4186, - "hfopenllm_v2/MMLU-PRO": 0.2922 - } - }, - { - "id": "rombodawg/Rombos-LLM-V2.5-Qwen-14b", - "name": "Rombos-LLM-V2.5-Qwen-14b", - "developer": "rombodawg", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.584, - "hfopenllm_v2/BBH": 0.6481, - "hfopenllm_v2/MATH Level 5": 0.4554, - "hfopenllm_v2/GPQA": 0.3716, - "hfopenllm_v2/MUSR": 0.4717, - "hfopenllm_v2/MMLU-PRO": 0.5376 - } - }, - { - "id": "rombodawg/Rombos-LLM-V2.5-Qwen-32b", - "name": "Rombos-LLM-V2.5-Qwen-32b", - "developer": "rombodawg", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6827, - "hfopenllm_v2/BBH": 0.7046, - "hfopenllm_v2/MATH Level 5": 0.4955, - "hfopenllm_v2/GPQA": 0.3968, - "hfopenllm_v2/MUSR": 0.5034, - "hfopenllm_v2/MMLU-PRO": 0.5916 - } - }, - { - "id": "rombodawg/Rombos-LLM-V2.5-Qwen-3b", - "name": "Rombos-LLM-V2.5-Qwen-3b", - "developer": "rombodawg", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5342, - "hfopenllm_v2/BBH": 0.4809, - "hfopenllm_v2/MATH Level 5": 0.2795, - "hfopenllm_v2/GPQA": 0.3079, - "hfopenllm_v2/MUSR": 0.4042, - "hfopenllm_v2/MMLU-PRO": 0.3761 - } - }, - { - "id": "rombodawg/Rombos-LLM-V2.5-Qwen-72b", - "name": "Rombos-LLM-V2.5-Qwen-72b", - "developer": "rombodawg", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7155, - "hfopenllm_v2/BBH": 0.723, - "hfopenllm_v2/MATH Level 5": 0.5423, - "hfopenllm_v2/GPQA": 0.3985, - "hfopenllm_v2/MUSR": 0.4599, - "hfopenllm_v2/MMLU-PRO": 0.5935 - } - }, - { - "id": "rombodawg/Rombos-LLM-V2.5-Qwen-7b", - "name": "Rombos-LLM-V2.5-Qwen-7b", - "developer": "rombodawg", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6237, - "hfopenllm_v2/BBH": 0.5544, - "hfopenllm_v2/MATH Level 5": 0.3814, - "hfopenllm_v2/GPQA": 0.318, - "hfopenllm_v2/MUSR": 0.4291, - "hfopenllm_v2/MMLU-PRO": 0.4469 - } - }, - { - "id": "rombodawg/Rombos-LLM-V2.5.1-Qwen-3b", - "name": "Rombos-LLM-V2.5.1-Qwen-3b", - "developer": "rombodawg", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2566, - "hfopenllm_v2/BBH": 0.39, - "hfopenllm_v2/MATH Level 5": 0.1208, - "hfopenllm_v2/GPQA": 0.2626, - "hfopenllm_v2/MUSR": 0.3991, - "hfopenllm_v2/MMLU-PRO": 0.2741 - } - }, - { - "id": "rombodawg/Rombos-LLM-V2.6-Nemotron-70b", - "name": "Rombos-LLM-V2.6-Nemotron-70b", - "developer": "rombodawg", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7527, - "hfopenllm_v2/BBH": 0.6938, - "hfopenllm_v2/MATH Level 5": 0.3331, - "hfopenllm_v2/GPQA": 0.406, - "hfopenllm_v2/MUSR": 0.4669, - "hfopenllm_v2/MMLU-PRO": 0.5329 - } - }, - { - "id": "rombodawg/Rombos-LLM-V2.6-Qwen-14b", - "name": "Rombos-LLM-V2.6-Qwen-14b", - "developer": "rombodawg", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8432, - "hfopenllm_v2/BBH": 0.6442, - "hfopenllm_v2/MATH Level 5": 0.5211, - "hfopenllm_v2/GPQA": 0.3339, - "hfopenllm_v2/MUSR": 0.4221, - "hfopenllm_v2/MMLU-PRO": 0.4961 - } - }, - { - "id": "rombodawg/rombos_Replete-Coder-Instruct-8b-Merged", - "name": "rombos_Replete-Coder-Instruct-8b-Merged", - "developer": "rombodawg", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5388, - "hfopenllm_v2/BBH": 0.4462, - "hfopenllm_v2/MATH Level 5": 0.0778, - "hfopenllm_v2/GPQA": 0.2693, - "hfopenllm_v2/MUSR": 0.366, - "hfopenllm_v2/MMLU-PRO": 0.1809 - } - }, - { - "id": "rombodawg/rombos_Replete-Coder-Llama3-8B", - "name": "rombos_Replete-Coder-Llama3-8B", - "developer": "rombodawg", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4714, - "hfopenllm_v2/BBH": 0.3276, - "hfopenllm_v2/MATH Level 5": 0.0393, - "hfopenllm_v2/GPQA": 0.2668, - "hfopenllm_v2/MUSR": 0.3966, - "hfopenllm_v2/MMLU-PRO": 0.1335 - } - }, - { - "id": "rootxhacker/Apollo-70B", - "name": "Apollo-70B", - "developer": "rootxhacker", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5099, - "hfopenllm_v2/BBH": 0.6804, - "hfopenllm_v2/MATH Level 5": 0.5612, - "hfopenllm_v2/GPQA": 0.4572, - "hfopenllm_v2/MUSR": 0.4948, - "hfopenllm_v2/MMLU-PRO": 0.5279 - } - }, - { - "id": "rootxhacker/apollo-7B", - "name": "apollo-7B", - "developer": "rootxhacker", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2953, - "hfopenllm_v2/BBH": 0.3636, - "hfopenllm_v2/MATH Level 5": 0.0257, - "hfopenllm_v2/GPQA": 0.2785, - "hfopenllm_v2/MUSR": 0.4131, - "hfopenllm_v2/MMLU-PRO": 0.1748 - } - }, - { - "id": "rootxhacker/Apollo_v2-32B", - "name": "Apollo_v2-32B", - "developer": "rootxhacker", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.428, - "hfopenllm_v2/BBH": 0.7072, - "hfopenllm_v2/MATH Level 5": 0.4275, - "hfopenllm_v2/GPQA": 0.3784, - "hfopenllm_v2/MUSR": 0.4994, - "hfopenllm_v2/MMLU-PRO": 0.5869 - } - }, - { - "id": "rsh345/mistral-ft-optimized-1218-NeuralHermes-2.5-Mistral-7B", - "name": "mistral-ft-optimized-1218-NeuralHermes-2.5-Mistral-7B", - "developer": "rsh345", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3892, - "hfopenllm_v2/BBH": 0.5188, - "hfopenllm_v2/MATH Level 5": 0.0733, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.4672, - "hfopenllm_v2/MMLU-PRO": 0.3054 - } - }, - { - "id": "rubenroy/Geneva-12B-GCv2-5m", - "name": "Geneva-12B-GCv2-5m", - "developer": "rubenroy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2586, - "hfopenllm_v2/BBH": 0.5278, - "hfopenllm_v2/MATH Level 5": 0.0801, - "hfopenllm_v2/GPQA": 0.2878, - "hfopenllm_v2/MUSR": 0.3525, - "hfopenllm_v2/MMLU-PRO": 0.325 - } - }, - { - "id": "rubenroy/Gilgamesh-72B", - "name": "Gilgamesh-72B", - "developer": "rubenroy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8486, - "hfopenllm_v2/BBH": 0.7253, - "hfopenllm_v2/MATH Level 5": 0.4381, - "hfopenllm_v2/GPQA": 0.3943, - "hfopenllm_v2/MUSR": 0.4626, - "hfopenllm_v2/MMLU-PRO": 0.5802 - } - }, - { - "id": "rubenroy/Zurich-14B-GCv2-5m", - "name": "Zurich-14B-GCv2-5m", - "developer": "rubenroy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6164, - "hfopenllm_v2/BBH": 0.6308, - "hfopenllm_v2/MATH Level 5": 0.3074, - "hfopenllm_v2/GPQA": 0.3616, - "hfopenllm_v2/MUSR": 0.4874, - "hfopenllm_v2/MMLU-PRO": 0.5233 - } - }, - { - "id": "RubielLabarta/LogoS-7Bx2-MoE-13B-v0.2", - "name": "LogoS-7Bx2-MoE-13B-v0.2", - "developer": "RubielLabarta", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4379, - "hfopenllm_v2/BBH": 0.5207, - "hfopenllm_v2/MATH Level 5": 0.0574, - "hfopenllm_v2/GPQA": 0.2777, - "hfopenllm_v2/MUSR": 0.4226, - "hfopenllm_v2/MMLU-PRO": 0.3088 - } - }, - { - "id": "ruizhe1217/sft-s1-qwen-0.5b", - "name": "sft-s1-qwen-0.5b", - "developer": "ruizhe1217", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2749, - "hfopenllm_v2/BBH": 0.3301, - "hfopenllm_v2/MATH Level 5": 0.0619, - "hfopenllm_v2/GPQA": 0.271, - "hfopenllm_v2/MUSR": 0.3196, - "hfopenllm_v2/MMLU-PRO": 0.1892 - } - }, - { - "id": "rwitz/go-bruins-v2", - "name": "go-bruins-v2", - "developer": "rwitz", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4096, - "hfopenllm_v2/BBH": 0.3799, - "hfopenllm_v2/MATH Level 5": 0.0672, - "hfopenllm_v2/GPQA": 0.2626, - "hfopenllm_v2/MUSR": 0.4138, - "hfopenllm_v2/MMLU-PRO": 0.2761 - } - }, - { - "id": "RWKV/rwkv-raven-14b", - "name": "rwkv-raven-14b", - "developer": "RWKV", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0768, - "hfopenllm_v2/BBH": 0.3307, - "hfopenllm_v2/MATH Level 5": 0.0045, - "hfopenllm_v2/GPQA": 0.229, - "hfopenllm_v2/MUSR": 0.3951, - "hfopenllm_v2/MMLU-PRO": 0.115 - } - }, - { - "id": "sabersaleh/Llama2-7B-CPO", - "name": "Llama2-7B-CPO", - "developer": "sabersaleh", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1545, - "hfopenllm_v2/BBH": 0.3458, - "hfopenllm_v2/MATH Level 5": 0.0136, - "hfopenllm_v2/GPQA": 0.2676, - "hfopenllm_v2/MUSR": 0.4048, - "hfopenllm_v2/MMLU-PRO": 0.1606 - } - }, - { - "id": "sabersaleh/Llama2-7B-DPO", - "name": "Llama2-7B-DPO", - "developer": "sabersaleh", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1453, - "hfopenllm_v2/BBH": 0.3512, - "hfopenllm_v2/MATH Level 5": 0.0159, - "hfopenllm_v2/GPQA": 0.2685, - "hfopenllm_v2/MUSR": 0.4114, - "hfopenllm_v2/MMLU-PRO": 0.1626 - } - }, - { - "id": "sabersaleh/Llama2-7B-IPO", - "name": "Llama2-7B-IPO", - "developer": "sabersaleh", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1769, - "hfopenllm_v2/BBH": 0.3475, - "hfopenllm_v2/MATH Level 5": 0.0159, - "hfopenllm_v2/GPQA": 0.2676, - "hfopenllm_v2/MUSR": 0.4048, - "hfopenllm_v2/MMLU-PRO": 0.1617 - } - }, - { - "id": "sabersaleh/Llama2-7B-KTO", - "name": "Llama2-7B-KTO", - "developer": "sabersaleh", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1528, - "hfopenllm_v2/BBH": 0.3501, - "hfopenllm_v2/MATH Level 5": 0.0189, - "hfopenllm_v2/GPQA": 0.2676, - "hfopenllm_v2/MUSR": 0.4167, - "hfopenllm_v2/MMLU-PRO": 0.1636 - } - }, - { - "id": "sabersaleh/Llama2-7B-SimPO", - "name": "Llama2-7B-SimPO", - "developer": "sabersaleh", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1659, - "hfopenllm_v2/BBH": 0.3489, - "hfopenllm_v2/MATH Level 5": 0.0159, - "hfopenllm_v2/GPQA": 0.271, - "hfopenllm_v2/MUSR": 0.4007, - "hfopenllm_v2/MMLU-PRO": 0.1641 - } - }, - { - "id": "sabersaleh/Llama2-7B-SPO", - "name": "Llama2-7B-SPO", - "developer": "sabersaleh", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1567, - "hfopenllm_v2/BBH": 0.3383, - "hfopenllm_v2/MATH Level 5": 0.0196, - "hfopenllm_v2/GPQA": 0.2768, - "hfopenllm_v2/MUSR": 0.3874, - "hfopenllm_v2/MMLU-PRO": 0.1757 - } - }, - { - "id": "sabersaleh/Llama3", - "name": "Llama3", - "developer": "sabersaleh", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3321, - "hfopenllm_v2/BBH": 0.4782, - "hfopenllm_v2/MATH Level 5": 0.0566, - "hfopenllm_v2/GPQA": 0.3104, - "hfopenllm_v2/MUSR": 0.3933, - "hfopenllm_v2/MMLU-PRO": 0.3162 - } - }, - { - "id": "sabersalehk/Llama3-001-300", - "name": "Llama3-001-300", - "developer": "sabersalehk", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3179, - "hfopenllm_v2/BBH": 0.4745, - "hfopenllm_v2/MATH Level 5": 0.0529, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.4064, - "hfopenllm_v2/MMLU-PRO": 0.3158 - } - }, - { - "id": "sabersalehk/Llama3-SimPO", - "name": "Llama3-SimPO", - "developer": "sabersalehk", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3642, - "hfopenllm_v2/BBH": 0.4874, - "hfopenllm_v2/MATH Level 5": 0.0574, - "hfopenllm_v2/GPQA": 0.3079, - "hfopenllm_v2/MUSR": 0.4046, - "hfopenllm_v2/MMLU-PRO": 0.3157 - } - }, - { - "id": "sabersalehk/Llama3_001_200", - "name": "Llama3_001_200", - "developer": "sabersalehk", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3218, - "hfopenllm_v2/BBH": 0.4728, - "hfopenllm_v2/MATH Level 5": 0.0514, - "hfopenllm_v2/GPQA": 0.3037, - "hfopenllm_v2/MUSR": 0.4037, - "hfopenllm_v2/MMLU-PRO": 0.3183 - } - }, - { - "id": "sabersalehk/Llama3_01_300", - "name": "Llama3_01_300", - "developer": "sabersalehk", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2959, - "hfopenllm_v2/BBH": 0.4691, - "hfopenllm_v2/MATH Level 5": 0.0498, - "hfopenllm_v2/GPQA": 0.3079, - "hfopenllm_v2/MUSR": 0.4065, - "hfopenllm_v2/MMLU-PRO": 0.3124 - } - }, - { - "id": "SaisExperiments/Evil-Alpaca-3B-L3.2", - "name": "Evil-Alpaca-3B-L3.2", - "developer": "SaisExperiments", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3251, - "hfopenllm_v2/BBH": 0.4341, - "hfopenllm_v2/MATH Level 5": 0.0702, - "hfopenllm_v2/GPQA": 0.2634, - "hfopenllm_v2/MUSR": 0.4198, - "hfopenllm_v2/MMLU-PRO": 0.2621 - } - }, - { - "id": "SaisExperiments/Gemma-2-2B-Opus-Instruct", - "name": "Gemma-2-2B-Opus-Instruct", - "developer": "SaisExperiments", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.475, - "hfopenllm_v2/BBH": 0.4293, - "hfopenllm_v2/MATH Level 5": 0.0506, - "hfopenllm_v2/GPQA": 0.2836, - "hfopenllm_v2/MUSR": 0.4057, - "hfopenllm_v2/MMLU-PRO": 0.265 - } - }, - { - "id": "SaisExperiments/Gemma-2-2B-Stheno-Filtered", - "name": "Gemma-2-2B-Stheno-Filtered", - "developer": "SaisExperiments", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4197, - "hfopenllm_v2/BBH": 0.4149, - "hfopenllm_v2/MATH Level 5": 0.0461, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.4003, - "hfopenllm_v2/MMLU-PRO": 0.263 - } - }, - { - "id": "SaisExperiments/Not-So-Small-Alpaca-24B", - "name": "Not-So-Small-Alpaca-24B", - "developer": "SaisExperiments", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6244, - "hfopenllm_v2/BBH": 0.5339, - "hfopenllm_v2/MATH Level 5": 0.1828, - "hfopenllm_v2/GPQA": 0.3591, - "hfopenllm_v2/MUSR": 0.4282, - "hfopenllm_v2/MMLU-PRO": 0.3694 - } - }, - { - "id": "SaisExperiments/QwOwO-7B-V1", - "name": "QwOwO-7B-V1", - "developer": "SaisExperiments", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4556, - "hfopenllm_v2/BBH": 0.5431, - "hfopenllm_v2/MATH Level 5": 0.386, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3835, - "hfopenllm_v2/MMLU-PRO": 0.4224 - } - }, - { - "id": "SaisExperiments/RightSheep-Llama3.2-3B", - "name": "RightSheep-Llama3.2-3B", - "developer": "SaisExperiments", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4156, - "hfopenllm_v2/BBH": 0.4241, - "hfopenllm_v2/MATH Level 5": 0.0808, - "hfopenllm_v2/GPQA": 0.2869, - "hfopenllm_v2/MUSR": 0.3767, - "hfopenllm_v2/MMLU-PRO": 0.254 - } - }, - { - "id": "saishf/Fimbulvetr-Kuro-Lotus-10.7B", - "name": "Fimbulvetr-Kuro-Lotus-10.7B", - "developer": "saishf", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4939, - "hfopenllm_v2/BBH": 0.4342, - "hfopenllm_v2/MATH Level 5": 0.0536, - "hfopenllm_v2/GPQA": 0.3012, - "hfopenllm_v2/MUSR": 0.4445, - "hfopenllm_v2/MMLU-PRO": 0.3389 - } - }, - { - "id": "saishf/Neural-SOVLish-Devil-8B-L3", - "name": "Neural-SOVLish-Devil-8B-L3", - "developer": "saishf", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4199, - "hfopenllm_v2/BBH": 0.5142, - "hfopenllm_v2/MATH Level 5": 0.0891, - "hfopenllm_v2/GPQA": 0.3079, - "hfopenllm_v2/MUSR": 0.411, - "hfopenllm_v2/MMLU-PRO": 0.3807 - } - }, - { - "id": "saishshinde15/TethysAI_Base_Reasoning", - "name": "TethysAI_Base_Reasoning", - "developer": "saishshinde15", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6369, - "hfopenllm_v2/BBH": 0.4519, - "hfopenllm_v2/MATH Level 5": 0.3142, - "hfopenllm_v2/GPQA": 0.2861, - "hfopenllm_v2/MUSR": 0.4075, - "hfopenllm_v2/MMLU-PRO": 0.3236 - } - }, - { - "id": "saishshinde15/TethysAI_Vortex", - "name": "TethysAI_Vortex", - "developer": "saishshinde15", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4298, - "hfopenllm_v2/BBH": 0.4749, - "hfopenllm_v2/MATH Level 5": 0.315, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.4458, - "hfopenllm_v2/MMLU-PRO": 0.3241 - } - }, - { - "id": "saishshinde15/TethysAI_Vortex_Reasoning", - "name": "TethysAI_Vortex_Reasoning", - "developer": "saishshinde15", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4021, - "hfopenllm_v2/BBH": 0.4694, - "hfopenllm_v2/MATH Level 5": 0.2145, - "hfopenllm_v2/GPQA": 0.3045, - "hfopenllm_v2/MUSR": 0.4084, - "hfopenllm_v2/MMLU-PRO": 0.3381 - } - }, - { - "id": "sakaltcommunity/novablast-preview", - "name": "novablast-preview", - "developer": "sakaltcommunity", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.453, - "hfopenllm_v2/BBH": 0.7043, - "hfopenllm_v2/MATH Level 5": 0.4894, - "hfopenllm_v2/GPQA": 0.3817, - "hfopenllm_v2/MUSR": 0.5021, - "hfopenllm_v2/MMLU-PRO": 0.5915 - } - }, - { - "id": "sakaltcommunity/sakaltum-7b", - "name": "sakaltum-7b", - "developer": "sakaltcommunity", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2604, - "hfopenllm_v2/BBH": 0.4575, - "hfopenllm_v2/MATH Level 5": 0.0295, - "hfopenllm_v2/GPQA": 0.2727, - "hfopenllm_v2/MUSR": 0.3775, - "hfopenllm_v2/MMLU-PRO": 0.2769 - } - }, - { - "id": "Sakalti/Anemoi-3B", - "name": "Anemoi-3B", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3804, - "hfopenllm_v2/BBH": 0.4922, - "hfopenllm_v2/MATH Level 5": 0.1775, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.4371, - "hfopenllm_v2/MMLU-PRO": 0.3766 - } - }, - { - "id": "Sakalti/Euphrates-14B", - "name": "Euphrates-14B", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2647, - "hfopenllm_v2/BBH": 0.6138, - "hfopenllm_v2/MATH Level 5": 0.3051, - "hfopenllm_v2/GPQA": 0.3935, - "hfopenllm_v2/MUSR": 0.4516, - "hfopenllm_v2/MMLU-PRO": 0.5255 - } - }, - { - "id": "Sakalti/light-1.1-3B", - "name": "light-1.1-3B", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2735, - "hfopenllm_v2/BBH": 0.2803, - "hfopenllm_v2/MATH Level 5": 0.0113, - "hfopenllm_v2/GPQA": 0.2617, - "hfopenllm_v2/MUSR": 0.3901, - "hfopenllm_v2/MMLU-PRO": 0.1209 - } - }, - { - "id": "Sakalti/light-3B", - "name": "light-3B", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5337, - "hfopenllm_v2/BBH": 0.4831, - "hfopenllm_v2/MATH Level 5": 0.2591, - "hfopenllm_v2/GPQA": 0.2953, - "hfopenllm_v2/MUSR": 0.4015, - "hfopenllm_v2/MMLU-PRO": 0.3775 - } - }, - { - "id": "Sakalti/light-3b-beta", - "name": "light-3b-beta", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5485, - "hfopenllm_v2/BBH": 0.4815, - "hfopenllm_v2/MATH Level 5": 0.2772, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.4015, - "hfopenllm_v2/MMLU-PRO": 0.3758 - } - }, - { - "id": "Sakalti/light-7b-beta", - "name": "light-7b-beta", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6234, - "hfopenllm_v2/BBH": 0.5548, - "hfopenllm_v2/MATH Level 5": 0.3769, - "hfopenllm_v2/GPQA": 0.3213, - "hfopenllm_v2/MUSR": 0.4291, - "hfopenllm_v2/MMLU-PRO": 0.4456 - } - }, - { - "id": "Sakalti/llama-3-yanyuedao-8b-instruct", - "name": "llama-3-yanyuedao-8b-instruct", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2186, - "hfopenllm_v2/BBH": 0.435, - "hfopenllm_v2/MATH Level 5": 0.0385, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.4199, - "hfopenllm_v2/MMLU-PRO": 0.2911 - } - }, - { - "id": "Sakalti/Llama3.2-3B-Uranus-1", - "name": "Llama3.2-3B-Uranus-1", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5335, - "hfopenllm_v2/BBH": 0.4437, - "hfopenllm_v2/MATH Level 5": 0.1495, - "hfopenllm_v2/GPQA": 0.297, - "hfopenllm_v2/MUSR": 0.3669, - "hfopenllm_v2/MMLU-PRO": 0.3094 - } - }, - { - "id": "Sakalti/magro-7B", - "name": "magro-7B", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1344, - "hfopenllm_v2/BBH": 0.4186, - "hfopenllm_v2/MATH Level 5": 0.0204, - "hfopenllm_v2/GPQA": 0.2953, - "hfopenllm_v2/MUSR": 0.446, - "hfopenllm_v2/MMLU-PRO": 0.2765 - } - }, - { - "id": "Sakalti/Magro-7B-v1.1", - "name": "Magro-7B-v1.1", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1204, - "hfopenllm_v2/BBH": 0.4179, - "hfopenllm_v2/MATH Level 5": 0.0249, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.4433, - "hfopenllm_v2/MMLU-PRO": 0.2764 - } - }, - { - "id": "Sakalti/mergekit-01", - "name": "mergekit-01", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6234, - "hfopenllm_v2/BBH": 0.5548, - "hfopenllm_v2/MATH Level 5": 0.3769, - "hfopenllm_v2/GPQA": 0.3213, - "hfopenllm_v2/MUSR": 0.4291, - "hfopenllm_v2/MMLU-PRO": 0.4456 - } - }, - { - "id": "Sakalti/mergekit-della_linear-vmeykci", - "name": "mergekit-della_linear-vmeykci", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1126, - "hfopenllm_v2/BBH": 0.2816, - "hfopenllm_v2/MATH Level 5": 0.0106, - "hfopenllm_v2/GPQA": 0.2634, - "hfopenllm_v2/MUSR": 0.3897, - "hfopenllm_v2/MMLU-PRO": 0.1089 - } - }, - { - "id": "Sakalti/model-3", - "name": "model-3", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6264, - "hfopenllm_v2/BBH": 0.5542, - "hfopenllm_v2/MATH Level 5": 0.3708, - "hfopenllm_v2/GPQA": 0.3213, - "hfopenllm_v2/MUSR": 0.4264, - "hfopenllm_v2/MMLU-PRO": 0.4455 - } - }, - { - "id": "Sakalti/Neptuno-3B", - "name": "Neptuno-3B", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4296, - "hfopenllm_v2/BBH": 0.4834, - "hfopenllm_v2/MATH Level 5": 0.2553, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.4002, - "hfopenllm_v2/MMLU-PRO": 0.3773 - } - }, - { - "id": "Sakalti/Neptuno-Alpha", - "name": "Neptuno-Alpha", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.378, - "hfopenllm_v2/BBH": 0.4925, - "hfopenllm_v2/MATH Level 5": 0.1835, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.4371, - "hfopenllm_v2/MMLU-PRO": 0.3767 - } - }, - { - "id": "Sakalti/Oxyge1-33B", - "name": "Oxyge1-33B", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4548, - "hfopenllm_v2/BBH": 0.7033, - "hfopenllm_v2/MATH Level 5": 0.4962, - "hfopenllm_v2/GPQA": 0.3826, - "hfopenllm_v2/MUSR": 0.5008, - "hfopenllm_v2/MMLU-PRO": 0.5909 - } - }, - { - "id": "Sakalti/Phi3.5-Comets-3.8B", - "name": "Phi3.5-Comets-3.8B", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2094, - "hfopenllm_v2/BBH": 0.3335, - "hfopenllm_v2/MATH Level 5": 0.0008, - "hfopenllm_v2/GPQA": 0.2492, - "hfopenllm_v2/MUSR": 0.3764, - "hfopenllm_v2/MMLU-PRO": 0.1153 - } - }, - { - "id": "Sakalti/Qwen2.5-1B-Instruct", - "name": "Qwen2.5-1B-Instruct", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1751, - "hfopenllm_v2/BBH": 0.3027, - "hfopenllm_v2/MATH Level 5": 0.006, - "hfopenllm_v2/GPQA": 0.2559, - "hfopenllm_v2/MUSR": 0.3369, - "hfopenllm_v2/MMLU-PRO": 0.1213 - } - }, - { - "id": "Sakalti/qwen2.5-2.3B", - "name": "qwen2.5-2.3B", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1288, - "hfopenllm_v2/BBH": 0.2849, - "hfopenllm_v2/MATH Level 5": 0.0053, - "hfopenllm_v2/GPQA": 0.2517, - "hfopenllm_v2/MUSR": 0.3857, - "hfopenllm_v2/MMLU-PRO": 0.1173 - } - }, - { - "id": "Sakalti/QwenTest-7", - "name": "QwenTest-7", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1672, - "hfopenllm_v2/BBH": 0.3063, - "hfopenllm_v2/MATH Level 5": 0.0038, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3422, - "hfopenllm_v2/MMLU-PRO": 0.1212 - } - }, - { - "id": "Sakalti/Saba-Passthrough-2", - "name": "Saba-Passthrough-2", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1691, - "hfopenllm_v2/BBH": 0.3672, - "hfopenllm_v2/MATH Level 5": 0.0008, - "hfopenllm_v2/GPQA": 0.2634, - "hfopenllm_v2/MUSR": 0.3844, - "hfopenllm_v2/MMLU-PRO": 0.2077 - } - }, - { - "id": "Sakalti/Saba1-1.8B", - "name": "Saba1-1.8B", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3333, - "hfopenllm_v2/BBH": 0.4147, - "hfopenllm_v2/MATH Level 5": 0.1541, - "hfopenllm_v2/GPQA": 0.2827, - "hfopenllm_v2/MUSR": 0.4239, - "hfopenllm_v2/MMLU-PRO": 0.2926 - } - }, - { - "id": "Sakalti/Saba1-7B", - "name": "Saba1-7B", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4585, - "hfopenllm_v2/BBH": 0.5489, - "hfopenllm_v2/MATH Level 5": 0.3663, - "hfopenllm_v2/GPQA": 0.3163, - "hfopenllm_v2/MUSR": 0.4793, - "hfopenllm_v2/MMLU-PRO": 0.4376 - } - }, - { - "id": "Sakalti/Saba1.5-1.5B", - "name": "Saba1.5-1.5B", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3333, - "hfopenllm_v2/BBH": 0.4147, - "hfopenllm_v2/MATH Level 5": 0.1541, - "hfopenllm_v2/GPQA": 0.2827, - "hfopenllm_v2/MUSR": 0.4239, - "hfopenllm_v2/MMLU-PRO": 0.2926 - } - }, - { - "id": "Sakalti/Saba1.5-Pro-3B", - "name": "Saba1.5-Pro-3B", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2386, - "hfopenllm_v2/BBH": 0.3623, - "hfopenllm_v2/MATH Level 5": 0.0272, - "hfopenllm_v2/GPQA": 0.2685, - "hfopenllm_v2/MUSR": 0.4405, - "hfopenllm_v2/MMLU-PRO": 0.1958 - } - }, - { - "id": "Sakalti/Saba2-14B-Preview", - "name": "Saba2-14B-Preview", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4722, - "hfopenllm_v2/BBH": 0.6496, - "hfopenllm_v2/MATH Level 5": 0.3127, - "hfopenllm_v2/GPQA": 0.3826, - "hfopenllm_v2/MUSR": 0.4781, - "hfopenllm_v2/MMLU-PRO": 0.5384 - } - }, - { - "id": "Sakalti/Saba2-3B", - "name": "Saba2-3B", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2865, - "hfopenllm_v2/BBH": 0.2801, - "hfopenllm_v2/MATH Level 5": 0.006, - "hfopenllm_v2/GPQA": 0.2617, - "hfopenllm_v2/MUSR": 0.3927, - "hfopenllm_v2/MMLU-PRO": 0.121 - } - }, - { - "id": "Sakalti/Sailor-japanese", - "name": "Sailor-japanese", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1605, - "hfopenllm_v2/BBH": 0.2913, - "hfopenllm_v2/MATH Level 5": 0.003, - "hfopenllm_v2/GPQA": 0.2534, - "hfopenllm_v2/MUSR": 0.3912, - "hfopenllm_v2/MMLU-PRO": 0.1164 - } - }, - { - "id": "Sakalti/Saka-1.5B", - "name": "Saka-1.5B", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2726, - "hfopenllm_v2/BBH": 0.3988, - "hfopenllm_v2/MATH Level 5": 0.0801, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.3739, - "hfopenllm_v2/MMLU-PRO": 0.2415 - } - }, - { - "id": "Sakalti/Saka-14B", - "name": "Saka-14B", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7174, - "hfopenllm_v2/BBH": 0.6497, - "hfopenllm_v2/MATH Level 5": 0.4094, - "hfopenllm_v2/GPQA": 0.396, - "hfopenllm_v2/MUSR": 0.4886, - "hfopenllm_v2/MMLU-PRO": 0.5396 - } - }, - { - "id": "Sakalti/Saka-24B", - "name": "Saka-24B", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3819, - "hfopenllm_v2/BBH": 0.6072, - "hfopenllm_v2/MATH Level 5": 0.1805, - "hfopenllm_v2/GPQA": 0.3423, - "hfopenllm_v2/MUSR": 0.4541, - "hfopenllm_v2/MMLU-PRO": 0.4766 - } - }, - { - "id": "Sakalti/Saka-7.2B", - "name": "Saka-7.2B", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1545, - "hfopenllm_v2/BBH": 0.2945, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2391, - "hfopenllm_v2/MUSR": 0.3711, - "hfopenllm_v2/MMLU-PRO": 0.116 - } - }, - { - "id": "Sakalti/Saka-7.6B", - "name": "Saka-7.6B", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4524, - "hfopenllm_v2/BBH": 0.5655, - "hfopenllm_v2/MATH Level 5": 0.3255, - "hfopenllm_v2/GPQA": 0.3163, - "hfopenllm_v2/MUSR": 0.4489, - "hfopenllm_v2/MMLU-PRO": 0.454 - } - }, - { - "id": "Sakalti/SakalFusion-7B-Alpha", - "name": "SakalFusion-7B-Alpha", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.529, - "hfopenllm_v2/BBH": 0.5591, - "hfopenllm_v2/MATH Level 5": 0.3844, - "hfopenllm_v2/GPQA": 0.3255, - "hfopenllm_v2/MUSR": 0.4581, - "hfopenllm_v2/MMLU-PRO": 0.4474 - } - }, - { - "id": "Sakalti/SakalFusion-7B-Beta", - "name": "SakalFusion-7B-Beta", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1809, - "hfopenllm_v2/BBH": 0.2881, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2433, - "hfopenllm_v2/MUSR": 0.3872, - "hfopenllm_v2/MMLU-PRO": 0.109 - } - }, - { - "id": "Sakalti/SakaMoe-3x1.6B-Instruct", - "name": "SakaMoe-3x1.6B-Instruct", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2371, - "hfopenllm_v2/BBH": 0.3282, - "hfopenllm_v2/MATH Level 5": 0.0544, - "hfopenllm_v2/GPQA": 0.2668, - "hfopenllm_v2/MUSR": 0.3342, - "hfopenllm_v2/MMLU-PRO": 0.1882 - } - }, - { - "id": "Sakalti/SJT-0.5B", - "name": "SJT-0.5B", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2425, - "hfopenllm_v2/BBH": 0.3306, - "hfopenllm_v2/MATH Level 5": 0.0521, - "hfopenllm_v2/GPQA": 0.2718, - "hfopenllm_v2/MUSR": 0.3196, - "hfopenllm_v2/MMLU-PRO": 0.1891 - } - }, - { - "id": "Sakalti/SJT-1.5B-Alpha", - "name": "SJT-1.5B-Alpha", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3449, - "hfopenllm_v2/BBH": 0.4241, - "hfopenllm_v2/MATH Level 5": 0.0997, - "hfopenllm_v2/GPQA": 0.2919, - "hfopenllm_v2/MUSR": 0.4226, - "hfopenllm_v2/MMLU-PRO": 0.2961 - } - }, - { - "id": "Sakalti/SJT-1.5B-Alpha-1.1", - "name": "SJT-1.5B-Alpha-1.1", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3439, - "hfopenllm_v2/BBH": 0.4243, - "hfopenllm_v2/MATH Level 5": 0.0959, - "hfopenllm_v2/GPQA": 0.2894, - "hfopenllm_v2/MUSR": 0.4239, - "hfopenllm_v2/MMLU-PRO": 0.2966 - } - }, - { - "id": "Sakalti/SJT-1.7B", - "name": "SJT-1.7B", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1776, - "hfopenllm_v2/BBH": 0.2934, - "hfopenllm_v2/MATH Level 5": 0.0015, - "hfopenllm_v2/GPQA": 0.2416, - "hfopenllm_v2/MUSR": 0.3964, - "hfopenllm_v2/MMLU-PRO": 0.1133 - } - }, - { - "id": "Sakalti/SJT-14B", - "name": "SJT-14B", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5494, - "hfopenllm_v2/BBH": 0.6536, - "hfopenllm_v2/MATH Level 5": 0.3844, - "hfopenllm_v2/GPQA": 0.3867, - "hfopenllm_v2/MUSR": 0.4766, - "hfopenllm_v2/MMLU-PRO": 0.5381 - } - }, - { - "id": "Sakalti/SJT-2.4B", - "name": "SJT-2.4B", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2804, - "hfopenllm_v2/BBH": 0.349, - "hfopenllm_v2/MATH Level 5": 0.0219, - "hfopenllm_v2/GPQA": 0.2559, - "hfopenllm_v2/MUSR": 0.3699, - "hfopenllm_v2/MMLU-PRO": 0.1858 - } - }, - { - "id": "Sakalti/SJT-24B-Alpha", - "name": "SJT-24B-Alpha", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3206, - "hfopenllm_v2/BBH": 0.6081, - "hfopenllm_v2/MATH Level 5": 0.253, - "hfopenllm_v2/GPQA": 0.3809, - "hfopenllm_v2/MUSR": 0.4595, - "hfopenllm_v2/MMLU-PRO": 0.4857 - } - }, - { - "id": "Sakalti/SJT-2B", - "name": "SJT-2B", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2151, - "hfopenllm_v2/BBH": 0.2936, - "hfopenllm_v2/MATH Level 5": 0.0008, - "hfopenllm_v2/GPQA": 0.2416, - "hfopenllm_v2/MUSR": 0.3564, - "hfopenllm_v2/MMLU-PRO": 0.1187 - } - }, - { - "id": "Sakalti/SJT-2B-V1.1", - "name": "SJT-2B-V1.1", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3977, - "hfopenllm_v2/BBH": 0.3984, - "hfopenllm_v2/MATH Level 5": 0.0483, - "hfopenllm_v2/GPQA": 0.2676, - "hfopenllm_v2/MUSR": 0.4299, - "hfopenllm_v2/MMLU-PRO": 0.2124 - } - }, - { - "id": "Sakalti/SJT-3.7B", - "name": "SJT-3.7B", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1078, - "hfopenllm_v2/BBH": 0.3393, - "hfopenllm_v2/MATH Level 5": 0.0121, - "hfopenllm_v2/GPQA": 0.2559, - "hfopenllm_v2/MUSR": 0.3617, - "hfopenllm_v2/MMLU-PRO": 0.1505 - } - }, - { - "id": "Sakalti/SJT-4B", - "name": "SJT-4B", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4077, - "hfopenllm_v2/BBH": 0.4886, - "hfopenllm_v2/MATH Level 5": 0.1156, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.478, - "hfopenllm_v2/MMLU-PRO": 0.3281 - } - }, - { - "id": "Sakalti/SJT-7.5B", - "name": "SJT-7.5B", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4223, - "hfopenllm_v2/BBH": 0.5367, - "hfopenllm_v2/MATH Level 5": 0.2168, - "hfopenllm_v2/GPQA": 0.3263, - "hfopenllm_v2/MUSR": 0.4399, - "hfopenllm_v2/MMLU-PRO": 0.3951 - } - }, - { - "id": "Sakalti/SJT-7B-V1.1", - "name": "SJT-7B-V1.1", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4703, - "hfopenllm_v2/BBH": 0.5419, - "hfopenllm_v2/MATH Level 5": 0.2432, - "hfopenllm_v2/GPQA": 0.3339, - "hfopenllm_v2/MUSR": 0.4411, - "hfopenllm_v2/MMLU-PRO": 0.4412 - } - }, - { - "id": "Sakalti/SJT-7B-V1.1-Multilingal", - "name": "SJT-7B-V1.1-Multilingal", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1949, - "hfopenllm_v2/BBH": 0.292, - "hfopenllm_v2/MATH Level 5": 0.0045, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3621, - "hfopenllm_v2/MMLU-PRO": 0.1137 - } - }, - { - "id": "Sakalti/SJT-8B", - "name": "SJT-8B", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6535, - "hfopenllm_v2/BBH": 0.5282, - "hfopenllm_v2/MATH Level 5": 0.2538, - "hfopenllm_v2/GPQA": 0.3297, - "hfopenllm_v2/MUSR": 0.408, - "hfopenllm_v2/MMLU-PRO": 0.4266 - } - }, - { - "id": "Sakalti/SJT-8B-V1.1", - "name": "SJT-8B-V1.1", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4621, - "hfopenllm_v2/BBH": 0.5121, - "hfopenllm_v2/MATH Level 5": 0.2069, - "hfopenllm_v2/GPQA": 0.3364, - "hfopenllm_v2/MUSR": 0.4266, - "hfopenllm_v2/MMLU-PRO": 0.4231 - } - }, - { - "id": "Sakalti/SJT-900M", - "name": "SJT-900M", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.241, - "hfopenllm_v2/BBH": 0.3169, - "hfopenllm_v2/MATH Level 5": 0.0136, - "hfopenllm_v2/GPQA": 0.2534, - "hfopenllm_v2/MUSR": 0.3595, - "hfopenllm_v2/MMLU-PRO": 0.1142 - } - }, - { - "id": "Sakalti/SJT-Moe2x7.5B", - "name": "SJT-Moe2x7.5B", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4117, - "hfopenllm_v2/BBH": 0.5371, - "hfopenllm_v2/MATH Level 5": 0.2145, - "hfopenllm_v2/GPQA": 0.3263, - "hfopenllm_v2/MUSR": 0.4399, - "hfopenllm_v2/MMLU-PRO": 0.3954 - } - }, - { - "id": "Sakalti/SJTPass-2", - "name": "SJTPass-2", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.24, - "hfopenllm_v2/BBH": 0.3302, - "hfopenllm_v2/MATH Level 5": 0.0529, - "hfopenllm_v2/GPQA": 0.2727, - "hfopenllm_v2/MUSR": 0.3222, - "hfopenllm_v2/MMLU-PRO": 0.1902 - } - }, - { - "id": "Sakalti/SJTPass-4", - "name": "SJTPass-4", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1913, - "hfopenllm_v2/BBH": 0.2964, - "hfopenllm_v2/MATH Level 5": 0.0023, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3898, - "hfopenllm_v2/MMLU-PRO": 0.1083 - } - }, - { - "id": "Sakalti/SJTPass-5", - "name": "SJTPass-5", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2425, - "hfopenllm_v2/BBH": 0.3103, - "hfopenllm_v2/MATH Level 5": 0.0159, - "hfopenllm_v2/GPQA": 0.2668, - "hfopenllm_v2/MUSR": 0.3794, - "hfopenllm_v2/MMLU-PRO": 0.1327 - } - }, - { - "id": "Sakalti/tara-3.8B", - "name": "tara-3.8B", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4077, - "hfopenllm_v2/BBH": 0.4886, - "hfopenllm_v2/MATH Level 5": 0.1156, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.478, - "hfopenllm_v2/MMLU-PRO": 0.3281 - } - }, - { - "id": "Sakalti/Tara-3.8B-v1.1", - "name": "Tara-3.8B-v1.1", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4062, - "hfopenllm_v2/BBH": 0.4886, - "hfopenllm_v2/MATH Level 5": 0.1156, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.478, - "hfopenllm_v2/MMLU-PRO": 0.3281 - } - }, - { - "id": "Sakalti/ultiima-14B", - "name": "ultiima-14B", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5701, - "hfopenllm_v2/BBH": 0.6491, - "hfopenllm_v2/MATH Level 5": 0.4698, - "hfopenllm_v2/GPQA": 0.3742, - "hfopenllm_v2/MUSR": 0.4718, - "hfopenllm_v2/MMLU-PRO": 0.5381 - } - }, - { - "id": "Sakalti/ultiima-14B-v0.2", - "name": "ultiima-14B-v0.2", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.707, - "hfopenllm_v2/BBH": 0.6472, - "hfopenllm_v2/MATH Level 5": 0.3995, - "hfopenllm_v2/GPQA": 0.3826, - "hfopenllm_v2/MUSR": 0.4794, - "hfopenllm_v2/MMLU-PRO": 0.5387 - } - }, - { - "id": "Sakalti/ultiima-14B-v0.3", - "name": "ultiima-14B-v0.3", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.704, - "hfopenllm_v2/BBH": 0.6398, - "hfopenllm_v2/MATH Level 5": 0.3965, - "hfopenllm_v2/GPQA": 0.3767, - "hfopenllm_v2/MUSR": 0.4754, - "hfopenllm_v2/MMLU-PRO": 0.5337 - } - }, - { - "id": "Sakalti/ultiima-14B-v0.4", - "name": "ultiima-14B-v0.4", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3008, - "hfopenllm_v2/BBH": 0.642, - "hfopenllm_v2/MATH Level 5": 0.3535, - "hfopenllm_v2/GPQA": 0.396, - "hfopenllm_v2/MUSR": 0.4886, - "hfopenllm_v2/MMLU-PRO": 0.5278 - } - }, - { - "id": "Sakalti/ultiima-32B", - "name": "ultiima-32B", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6854, - "hfopenllm_v2/BBH": 0.7037, - "hfopenllm_v2/MATH Level 5": 0.4962, - "hfopenllm_v2/GPQA": 0.3809, - "hfopenllm_v2/MUSR": 0.4995, - "hfopenllm_v2/MMLU-PRO": 0.591 - } - }, - { - "id": "Sakalti/ultiima-72B", - "name": "ultiima-72B", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.714, - "hfopenllm_v2/BBH": 0.7218, - "hfopenllm_v2/MATH Level 5": 0.5355, - "hfopenllm_v2/GPQA": 0.4144, - "hfopenllm_v2/MUSR": 0.4652, - "hfopenllm_v2/MMLU-PRO": 0.5906 - } - }, - { - "id": "Sakalti/ultiima-72B-v1.5", - "name": "ultiima-72B-v1.5", - "developer": "Sakalti", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.655, - "hfopenllm_v2/BBH": 0.7392, - "hfopenllm_v2/MATH Level 5": 0.4396, - "hfopenllm_v2/GPQA": 0.4136, - "hfopenllm_v2/MUSR": 0.4691, - "hfopenllm_v2/MMLU-PRO": 0.6054 - } - }, - { - "id": "sakhan10/quantized_open_llama_3b_v2", - "name": "quantized_open_llama_3b_v2", - "developer": "sakhan10", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1872, - "hfopenllm_v2/BBH": 0.302, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2768, - "hfopenllm_v2/MUSR": 0.3682, - "hfopenllm_v2/MMLU-PRO": 0.1095 - } - }, - { - "id": "Salesforce/LLaMA-3-8B-SFR-Iterative-DPO-R", - "name": "LLaMA-3-8B-SFR-Iterative-DPO-R", - "developer": "Salesforce", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3816, - "hfopenllm_v2/BBH": 0.5012, - "hfopenllm_v2/MATH Level 5": 0.0914, - "hfopenllm_v2/GPQA": 0.2878, - "hfopenllm_v2/MUSR": 0.3633, - "hfopenllm_v2/MMLU-PRO": 0.3172 - } - }, - { - "id": "Salesforce/SFR-LLaMa-3.1-70B-Judge-r", - "name": "Salesforce/SFR-LLaMa-3.1-70B-Judge-r", - "developer": "Salesforce", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.9272, - "reward-bench/Chat": 0.9693, - "reward-bench/Chat Hard": 0.8476, - "reward-bench/Safety": 0.9162, - "reward-bench/Reasoning": 0.9757 - } - }, - { - "id": "Salesforce/SFR-LLaMa-3.1-8B-Judge-r", - "name": "Salesforce/SFR-LLaMa-3.1-8B-Judge-r", - "developer": "Salesforce", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.8865, - "reward-bench/Chat": 0.9553, - "reward-bench/Chat Hard": 0.7774, - "reward-bench/Safety": 0.8622, - "reward-bench/Reasoning": 0.9513 - } - }, - { - "id": "Salesforce/SFR-nemo-12B-Judge-r", - "name": "Salesforce/SFR-nemo-12B-Judge-r", - "developer": "Salesforce", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.9027, - "reward-bench/Chat": 0.9721, - "reward-bench/Chat Hard": 0.8224, - "reward-bench/Safety": 0.8649, - "reward-bench/Reasoning": 0.9513 - } - }, - { - "id": "salesforce/xlam-2-1b-fc-r-fc", - "name": "xLAM-2-1b-fc-r (FC)", - "developer": "salesforce", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 65.0, - "bfcl/bfcl.overall.overall_accuracy": 30.44, - "bfcl/bfcl.overall.total_cost_usd": 2.79, - "bfcl/bfcl.overall.latency_mean_s": 2.84, - "bfcl/bfcl.overall.latency_std_s": 2.35, - "bfcl/bfcl.overall.latency_p95_s": 6.52, - "bfcl/bfcl.non_live.ast_accuracy": 69.04, - "bfcl/bfcl.non_live.simple_ast_accuracy": 64.17, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 82.5, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 73.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 56.0, - "bfcl/bfcl.live.live_accuracy": 55.14, - "bfcl/bfcl.live.live_simple_ast_accuracy": 68.22, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 52.8, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 43.75, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 25.0, - "bfcl/bfcl.multi_turn.accuracy": 36.0, - "bfcl/bfcl.multi_turn.base_accuracy": 45.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 36.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 37.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 25.5, - "bfcl/bfcl.web_search.accuracy": 0.0, - "bfcl/bfcl.web_search.base_accuracy": 0.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 0.0, - "bfcl/bfcl.memory.accuracy": 3.87, - "bfcl/bfcl.memory.kv_accuracy": 3.87, - "bfcl/bfcl.memory.vector_accuracy": 3.87, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 3.87, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 87.5, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 64.47 - } - }, - { - "id": "salesforce/xlam-2-32b-fc-r-fc", - "name": "xLAM-2-32b-fc-r (FC)", - "developer": "salesforce", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 18.0, - "bfcl/bfcl.overall.overall_accuracy": 54.66, - "bfcl/bfcl.overall.total_cost_usd": 6.0, - "bfcl/bfcl.overall.latency_mean_s": 6.94, - "bfcl/bfcl.overall.latency_std_s": 8.21, - "bfcl/bfcl.overall.latency_p95_s": 17.66, - "bfcl/bfcl.non_live.ast_accuracy": 89.6, - "bfcl/bfcl.non_live.simple_ast_accuracy": 80.42, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 94.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 93.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 91.0, - "bfcl/bfcl.live.live_accuracy": 75.5, - "bfcl/bfcl.live.live_simple_ast_accuracy": 82.17, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 74.64, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 50.0, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 58.33, - "bfcl/bfcl.multi_turn.accuracy": 69.5, - "bfcl/bfcl.multi_turn.base_accuracy": 81.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 72.5, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 67.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 56.5, - "bfcl/bfcl.web_search.accuracy": 25.5, - "bfcl/bfcl.web_search.base_accuracy": 37.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 14.0, - "bfcl/bfcl.memory.accuracy": 20.86, - "bfcl/bfcl.memory.kv_accuracy": 6.45, - "bfcl/bfcl.memory.vector_accuracy": 10.32, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 45.81, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 81.25, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 80.23 - } - }, - { - "id": "salesforce/xlam-2-3b-fc-r-fc", - "name": "xLAM-2-3b-fc-r (FC)", - "developer": "salesforce", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 42.0, - "bfcl/bfcl.overall.overall_accuracy": 41.22, - "bfcl/bfcl.overall.total_cost_usd": 3.36, - "bfcl/bfcl.overall.latency_mean_s": 3.8, - "bfcl/bfcl.overall.latency_std_s": 3.59, - "bfcl/bfcl.overall.latency_p95_s": 8.79, - "bfcl/bfcl.non_live.ast_accuracy": 82.96, - "bfcl/bfcl.non_live.simple_ast_accuracy": 75.33, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 91.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 86.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 79.0, - "bfcl/bfcl.live.live_accuracy": 62.92, - "bfcl/bfcl.live.live_simple_ast_accuracy": 73.26, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 60.68, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 62.5, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 50.0, - "bfcl/bfcl.multi_turn.accuracy": 58.38, - "bfcl/bfcl.multi_turn.base_accuracy": 71.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 59.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 57.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 45.5, - "bfcl/bfcl.web_search.accuracy": 2.5, - "bfcl/bfcl.web_search.base_accuracy": 3.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 2.0, - "bfcl/bfcl.memory.accuracy": 11.4, - "bfcl/bfcl.memory.kv_accuracy": 5.81, - "bfcl/bfcl.memory.vector_accuracy": 5.81, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 22.58, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 87.5, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 63.45 - } - }, - { - "id": "salesforce/xlam-2-70b-fc-r-fc", - "name": "xLAM-2-70b-fc-r (FC)", - "developer": "salesforce", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 22.0, - "bfcl/bfcl.overall.overall_accuracy": 53.07, - "bfcl/bfcl.overall.total_cost_usd": 25.1, - "bfcl/bfcl.overall.latency_mean_s": 28.06, - "bfcl/bfcl.overall.latency_std_s": 68.77, - "bfcl/bfcl.overall.latency_p95_s": 91.21, - "bfcl/bfcl.non_live.ast_accuracy": 88.44, - "bfcl/bfcl.non_live.simple_ast_accuracy": 78.25, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 94.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 92.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 89.5, - "bfcl/bfcl.live.live_accuracy": 72.17, - "bfcl/bfcl.live.live_simple_ast_accuracy": 77.91, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 71.13, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 68.75, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 58.33, - "bfcl/bfcl.multi_turn.accuracy": 77.38, - "bfcl/bfcl.multi_turn.base_accuracy": 82.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 77.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 74.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 76.0, - "bfcl/bfcl.web_search.accuracy": 15.0, - "bfcl/bfcl.web_search.base_accuracy": 17.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 13.0, - "bfcl/bfcl.memory.accuracy": 14.41, - "bfcl/bfcl.memory.kv_accuracy": 2.58, - "bfcl/bfcl.memory.vector_accuracy": 10.97, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 29.68, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 75.0, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 79.11 - } - }, - { - "id": "salesforce/xlam-2-8b-fc-r-fc", - "name": "xLAM-2-8b-fc-r (FC)", - "developer": "salesforce", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 34.0, - "bfcl/bfcl.overall.overall_accuracy": 46.68, - "bfcl/bfcl.overall.total_cost_usd": 20.92, - "bfcl/bfcl.overall.latency_mean_s": 22.65, - "bfcl/bfcl.overall.latency_std_s": 46.92, - "bfcl/bfcl.overall.latency_p95_s": 108.81, - "bfcl/bfcl.non_live.ast_accuracy": 84.58, - "bfcl/bfcl.non_live.simple_ast_accuracy": 73.83, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 93.5, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 87.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 83.5, - "bfcl/bfcl.live.live_accuracy": 67.95, - "bfcl/bfcl.live.live_simple_ast_accuracy": 75.58, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 66.57, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 56.25, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 54.17, - "bfcl/bfcl.multi_turn.accuracy": 70.0, - "bfcl/bfcl.multi_turn.base_accuracy": 76.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 72.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 65.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 67.0, - "bfcl/bfcl.web_search.accuracy": 6.5, - "bfcl/bfcl.web_search.base_accuracy": 11.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 2.0, - "bfcl/bfcl.memory.accuracy": 13.98, - "bfcl/bfcl.memory.kv_accuracy": 5.81, - "bfcl/bfcl.memory.vector_accuracy": 15.48, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 20.65, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 87.5, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 63.28 - } - }, - { - "id": "saltlux/luxia-21.4b-alignment-v1.0", - "name": "luxia-21.4b-alignment-v1.0", - "developer": "saltlux", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3693, - "hfopenllm_v2/BBH": 0.6373, - "hfopenllm_v2/MATH Level 5": 0.0974, - "hfopenllm_v2/GPQA": 0.3012, - "hfopenllm_v2/MUSR": 0.4328, - "hfopenllm_v2/MMLU-PRO": 0.3403 - } - }, - { - "id": "saltlux/luxia-21.4b-alignment-v1.2", - "name": "luxia-21.4b-alignment-v1.2", - "developer": "saltlux", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4115, - "hfopenllm_v2/BBH": 0.6371, - "hfopenllm_v2/MATH Level 5": 0.0846, - "hfopenllm_v2/GPQA": 0.3079, - "hfopenllm_v2/MUSR": 0.4459, - "hfopenllm_v2/MMLU-PRO": 0.3473 - } - }, - { - "id": "sam-paech/Darkest-muse-v1", - "name": "Darkest-muse-v1", - "developer": "sam-paech", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7344, - "hfopenllm_v2/BBH": 0.5968, - "hfopenllm_v2/MATH Level 5": 0.2145, - "hfopenllm_v2/GPQA": 0.344, - "hfopenllm_v2/MUSR": 0.4502, - "hfopenllm_v2/MMLU-PRO": 0.4184 - } - }, - { - "id": "sam-paech/Delirium-v1", - "name": "Delirium-v1", - "developer": "sam-paech", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7208, - "hfopenllm_v2/BBH": 0.5962, - "hfopenllm_v2/MATH Level 5": 0.2107, - "hfopenllm_v2/GPQA": 0.3431, - "hfopenllm_v2/MUSR": 0.4514, - "hfopenllm_v2/MMLU-PRO": 0.419 - } - }, - { - "id": "sam-paech/Quill-v1", - "name": "Quill-v1", - "developer": "sam-paech", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7122, - "hfopenllm_v2/BBH": 0.5969, - "hfopenllm_v2/MATH Level 5": 0.2122, - "hfopenllm_v2/GPQA": 0.3398, - "hfopenllm_v2/MUSR": 0.4555, - "hfopenllm_v2/MMLU-PRO": 0.4171 - } - }, - { - "id": "SanjiWatsuki/Kunoichi-DPO-v2-7B", - "name": "Kunoichi-DPO-v2-7B", - "developer": "SanjiWatsuki", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5431, - "hfopenllm_v2/BBH": 0.4416, - "hfopenllm_v2/MATH Level 5": 0.0763, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.4188, - "hfopenllm_v2/MMLU-PRO": 0.3107 - } - }, - { - "id": "SanjiWatsuki/Silicon-Maid-7B", - "name": "Silicon-Maid-7B", - "developer": "SanjiWatsuki", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5368, - "hfopenllm_v2/BBH": 0.4128, - "hfopenllm_v2/MATH Level 5": 0.065, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.4188, - "hfopenllm_v2/MMLU-PRO": 0.3083 - } - }, - { - "id": "Sao10K/70B-L3.3-Cirrus-x1", - "name": "70B-L3.3-Cirrus-x1", - "developer": "Sao10K", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6681, - "hfopenllm_v2/BBH": 0.7029, - "hfopenllm_v2/MATH Level 5": 0.3739, - "hfopenllm_v2/GPQA": 0.4497, - "hfopenllm_v2/MUSR": 0.4842, - "hfopenllm_v2/MMLU-PRO": 0.5378 - } - }, - { - "id": "Sao10K/Fimbulvetr-11B-v2", - "name": "Fimbulvetr-11B-v2", - "developer": "Sao10K", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.51, - "hfopenllm_v2/BBH": 0.4544, - "hfopenllm_v2/MATH Level 5": 0.068, - "hfopenllm_v2/GPQA": 0.2919, - "hfopenllm_v2/MUSR": 0.4354, - "hfopenllm_v2/MMLU-PRO": 0.3301 - } - }, - { - "id": "Sao10K/L3-70B-Euryale-v2.1", - "name": "L3-70B-Euryale-v2.1", - "developer": "Sao10K", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7281, - "hfopenllm_v2/BBH": 0.6503, - "hfopenllm_v2/MATH Level 5": 0.2243, - "hfopenllm_v2/GPQA": 0.3314, - "hfopenllm_v2/MUSR": 0.4196, - "hfopenllm_v2/MMLU-PRO": 0.5096 - } - }, - { - "id": "Sao10K/L3-8B-Lunaris-v1", - "name": "L3-8B-Lunaris-v1", - "developer": "Sao10K", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6895, - "hfopenllm_v2/BBH": 0.5235, - "hfopenllm_v2/MATH Level 5": 0.0906, - "hfopenllm_v2/GPQA": 0.3012, - "hfopenllm_v2/MUSR": 0.3727, - "hfopenllm_v2/MMLU-PRO": 0.3787 - } - }, - { - "id": "Sao10K/L3-8B-Niitama-v1", - "name": "L3-8B-Niitama-v1", - "developer": "Sao10K", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6791, - "hfopenllm_v2/BBH": 0.5303, - "hfopenllm_v2/MATH Level 5": 0.0982, - "hfopenllm_v2/GPQA": 0.3079, - "hfopenllm_v2/MUSR": 0.3807, - "hfopenllm_v2/MMLU-PRO": 0.3701 - } - }, - { - "id": "Sao10K/L3-8B-Stheno-v3.2", - "name": "L3-8B-Stheno-v3.2", - "developer": "Sao10K", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6873, - "hfopenllm_v2/BBH": 0.5228, - "hfopenllm_v2/MATH Level 5": 0.0929, - "hfopenllm_v2/GPQA": 0.3104, - "hfopenllm_v2/MUSR": 0.3794, - "hfopenllm_v2/MMLU-PRO": 0.3768 - } - }, - { - "id": "Sao10K/L3-8B-Stheno-v3.3-32K", - "name": "L3-8B-Stheno-v3.3-32K", - "developer": "Sao10K", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4604, - "hfopenllm_v2/BBH": 0.3844, - "hfopenllm_v2/MATH Level 5": 0.0144, - "hfopenllm_v2/GPQA": 0.2567, - "hfopenllm_v2/MUSR": 0.3725, - "hfopenllm_v2/MMLU-PRO": 0.1896 - } - }, - { - "id": "Sao10K/MN-12B-Lyra-v3", - "name": "MN-12B-Lyra-v3", - "developer": "Sao10K", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4486, - "hfopenllm_v2/BBH": 0.4804, - "hfopenllm_v2/MATH Level 5": 0.0937, - "hfopenllm_v2/GPQA": 0.2777, - "hfopenllm_v2/MUSR": 0.4019, - "hfopenllm_v2/MMLU-PRO": 0.3249 - } - }, - { - "id": "sarvamai/OpenHathi-7B-Hi-v0.1-Base", - "name": "OpenHathi-7B-Hi-v0.1-Base", - "developer": "sarvamai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1804, - "hfopenllm_v2/BBH": 0.3354, - "hfopenllm_v2/MATH Level 5": 0.0083, - "hfopenllm_v2/GPQA": 0.2534, - "hfopenllm_v2/MUSR": 0.3658, - "hfopenllm_v2/MMLU-PRO": 0.1543 - } - }, - { - "id": "Saxo/Linkbricks-Horizon-AI-Avengers-V1-32B", - "name": "Linkbricks-Horizon-AI-Avengers-V1-32B", - "developer": "Saxo", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7972, - "hfopenllm_v2/BBH": 0.7001, - "hfopenllm_v2/MATH Level 5": 0.6027, - "hfopenllm_v2/GPQA": 0.3624, - "hfopenllm_v2/MUSR": 0.4538, - "hfopenllm_v2/MMLU-PRO": 0.5793 - } - }, - { - "id": "Saxo/Linkbricks-Horizon-AI-Avengers-V2-32B", - "name": "Linkbricks-Horizon-AI-Avengers-V2-32B", - "developer": "Saxo", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7956, - "hfopenllm_v2/BBH": 0.7023, - "hfopenllm_v2/MATH Level 5": 0.5665, - "hfopenllm_v2/GPQA": 0.2659, - "hfopenllm_v2/MUSR": 0.4166, - "hfopenllm_v2/MMLU-PRO": 0.572 - } - }, - { - "id": "Saxo/Linkbricks-Horizon-AI-Avengers-V3-32B", - "name": "Linkbricks-Horizon-AI-Avengers-V3-32B", - "developer": "Saxo", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8249, - "hfopenllm_v2/BBH": 0.6913, - "hfopenllm_v2/MATH Level 5": 0.6178, - "hfopenllm_v2/GPQA": 0.3381, - "hfopenllm_v2/MUSR": 0.4275, - "hfopenllm_v2/MMLU-PRO": 0.5664 - } - }, - { - "id": "Saxo/Linkbricks-Horizon-AI-Avengers-V4-32B", - "name": "Linkbricks-Horizon-AI-Avengers-V4-32B", - "developer": "Saxo", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7631, - "hfopenllm_v2/BBH": 0.692, - "hfopenllm_v2/MATH Level 5": 0.5363, - "hfopenllm_v2/GPQA": 0.3616, - "hfopenllm_v2/MUSR": 0.4643, - "hfopenllm_v2/MMLU-PRO": 0.5752 - } - }, - { - "id": "Saxo/Linkbricks-Horizon-AI-Avengers-V5-32B", - "name": "Linkbricks-Horizon-AI-Avengers-V5-32B", - "developer": "Saxo", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7516, - "hfopenllm_v2/BBH": 0.6929, - "hfopenllm_v2/MATH Level 5": 0.5461, - "hfopenllm_v2/GPQA": 0.3557, - "hfopenllm_v2/MUSR": 0.4709, - "hfopenllm_v2/MMLU-PRO": 0.5762 - } - }, - { - "id": "Saxo/Linkbricks-Horizon-AI-Avengers-V6-32B", - "name": "Linkbricks-Horizon-AI-Avengers-V6-32B", - "developer": "Saxo", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8209, - "hfopenllm_v2/BBH": 0.689, - "hfopenllm_v2/MATH Level 5": 0.6224, - "hfopenllm_v2/GPQA": 0.3347, - "hfopenllm_v2/MUSR": 0.4274, - "hfopenllm_v2/MMLU-PRO": 0.5672 - } - }, - { - "id": "Saxo/Linkbricks-Horizon-AI-Korean-Avengers-V2-27B", - "name": "Linkbricks-Horizon-AI-Korean-Avengers-V2-27B", - "developer": "Saxo", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8146, - "hfopenllm_v2/BBH": 0.6463, - "hfopenllm_v2/MATH Level 5": 0.2802, - "hfopenllm_v2/GPQA": 0.3473, - "hfopenllm_v2/MUSR": 0.4139, - "hfopenllm_v2/MMLU-PRO": 0.4599 - } - }, - { - "id": "Saxo/Linkbricks-Horizon-AI-Korean-Avengers-V3-27B", - "name": "Linkbricks-Horizon-AI-Korean-Avengers-V3-27B", - "developer": "Saxo", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8142, - "hfopenllm_v2/BBH": 0.6404, - "hfopenllm_v2/MATH Level 5": 0.2492, - "hfopenllm_v2/GPQA": 0.3591, - "hfopenllm_v2/MUSR": 0.4467, - "hfopenllm_v2/MMLU-PRO": 0.4524 - } - }, - { - "id": "Saxo/Linkbricks-Horizon-AI-Korean-Superb-22B", - "name": "Linkbricks-Horizon-AI-Korean-Superb-22B", - "developer": "Saxo", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6767, - "hfopenllm_v2/BBH": 0.5626, - "hfopenllm_v2/MATH Level 5": 0.2372, - "hfopenllm_v2/GPQA": 0.3263, - "hfopenllm_v2/MUSR": 0.3908, - "hfopenllm_v2/MMLU-PRO": 0.3871 - } - }, - { - "id": "Saxo/Linkbricks-Horizon-AI-Korean-Superb-27B", - "name": "Linkbricks-Horizon-AI-Korean-Superb-27B", - "developer": "Saxo", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7768, - "hfopenllm_v2/BBH": 0.6518, - "hfopenllm_v2/MATH Level 5": 0.2719, - "hfopenllm_v2/GPQA": 0.3599, - "hfopenllm_v2/MUSR": 0.4791, - "hfopenllm_v2/MMLU-PRO": 0.4647 - } - }, - { - "id": "Saxo/Linkbricks-Horizon-AI-Superb-27B", - "name": "Linkbricks-Horizon-AI-Superb-27B", - "developer": "Saxo", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7302, - "hfopenllm_v2/BBH": 0.6186, - "hfopenllm_v2/MATH Level 5": 0.2221, - "hfopenllm_v2/GPQA": 0.3574, - "hfopenllm_v2/MUSR": 0.465, - "hfopenllm_v2/MMLU-PRO": 0.406 - } - }, - { - "id": "schnapss/testmerge-7b", - "name": "testmerge-7b", - "developer": "schnapss", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3922, - "hfopenllm_v2/BBH": 0.5187, - "hfopenllm_v2/MATH Level 5": 0.0687, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.4686, - "hfopenllm_v2/MMLU-PRO": 0.306 - } - }, - { - "id": "Schrieffer/Llama-SARM-4B", - "name": "Schrieffer/Llama-SARM-4B", - "developer": "Schrieffer", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7379, - "reward-bench/Factuality": 0.6874, - "reward-bench/Precise IF": 0.4281, - "reward-bench/Math": 0.6448, - "reward-bench/Safety": 0.9178, - "reward-bench/Focus": 0.9556, - "reward-bench/Ties": 0.7939 - } - }, - { - "id": "sci-m-wang/deepseek-llm-7b-chat-sa-v0.1", - "name": "deepseek-llm-7b-chat-sa-v0.1", - "developer": "sci-m-wang", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4036, - "hfopenllm_v2/BBH": 0.3718, - "hfopenllm_v2/MATH Level 5": 0.0264, - "hfopenllm_v2/GPQA": 0.2567, - "hfopenllm_v2/MUSR": 0.4173, - "hfopenllm_v2/MMLU-PRO": 0.2209 - } - }, - { - "id": "sci-m-wang/Mistral-7B-Instruct-sa-v0.1", - "name": "Mistral-7B-Instruct-sa-v0.1", - "developer": "sci-m-wang", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4335, - "hfopenllm_v2/BBH": 0.3273, - "hfopenllm_v2/MATH Level 5": 0.0144, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.39, - "hfopenllm_v2/MMLU-PRO": 0.2362 - } - }, - { - "id": "sci-m-wang/Phi-3-mini-4k-instruct-sa-v0.1", - "name": "Phi-3-mini-4k-instruct-sa-v0.1", - "developer": "sci-m-wang", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5021, - "hfopenllm_v2/BBH": 0.5502, - "hfopenllm_v2/MATH Level 5": 0.148, - "hfopenllm_v2/GPQA": 0.3289, - "hfopenllm_v2/MUSR": 0.4073, - "hfopenllm_v2/MMLU-PRO": 0.3985 - } - }, - { - "id": "SeaLLMs/SeaLLM-7B-v2", - "name": "SeaLLM-7B-v2", - "developer": "SeaLLMs", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3671, - "hfopenllm_v2/BBH": 0.4902, - "hfopenllm_v2/MATH Level 5": 0.0853, - "hfopenllm_v2/GPQA": 0.2785, - "hfopenllm_v2/MUSR": 0.407, - "hfopenllm_v2/MMLU-PRO": 0.3083 - } - }, - { - "id": "SeaLLMs/SeaLLM-7B-v2.5", - "name": "SeaLLM-7B-v2.5", - "developer": "SeaLLMs", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4522, - "hfopenllm_v2/BBH": 0.498, - "hfopenllm_v2/MATH Level 5": 0.1088, - "hfopenllm_v2/GPQA": 0.276, - "hfopenllm_v2/MUSR": 0.4203, - "hfopenllm_v2/MMLU-PRO": 0.3203 - } - }, - { - "id": "SeaLLMs/SeaLLMs-v3-7B-Chat", - "name": "SeaLLMs-v3-7B-Chat", - "developer": "SeaLLMs", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4377, - "hfopenllm_v2/BBH": 0.5266, - "hfopenllm_v2/MATH Level 5": 0.1858, - "hfopenllm_v2/GPQA": 0.2987, - "hfopenllm_v2/MUSR": 0.4174, - "hfopenllm_v2/MMLU-PRO": 0.3895 - } - }, - { - "id": "securin/Securin-LLM-V2.5-Qwen-1.5B", - "name": "Securin-LLM-V2.5-Qwen-1.5B", - "developer": "securin", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1492, - "hfopenllm_v2/BBH": 0.3158, - "hfopenllm_v2/MATH Level 5": 0.0249, - "hfopenllm_v2/GPQA": 0.25, - "hfopenllm_v2/MUSR": 0.3606, - "hfopenllm_v2/MMLU-PRO": 0.1615 - } - }, - { - "id": "senseable/WestLake-7B-v2", - "name": "WestLake-7B-v2", - "developer": "senseable", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4419, - "hfopenllm_v2/BBH": 0.4073, - "hfopenllm_v2/MATH Level 5": 0.0483, - "hfopenllm_v2/GPQA": 0.2768, - "hfopenllm_v2/MUSR": 0.3937, - "hfopenllm_v2/MMLU-PRO": 0.2764 - } - }, - { - "id": "SenseLLM/ReflectionCoder-CL-34B", - "name": "ReflectionCoder-CL-34B", - "developer": "SenseLLM", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4008, - "hfopenllm_v2/BBH": 0.3953, - "hfopenllm_v2/MATH Level 5": 0.0332, - "hfopenllm_v2/GPQA": 0.2508, - "hfopenllm_v2/MUSR": 0.4155, - "hfopenllm_v2/MMLU-PRO": 0.1424 - } - }, - { - "id": "SenseLLM/ReflectionCoder-DS-33B", - "name": "ReflectionCoder-DS-33B", - "developer": "SenseLLM", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3787, - "hfopenllm_v2/BBH": 0.3449, - "hfopenllm_v2/MATH Level 5": 0.0302, - "hfopenllm_v2/GPQA": 0.2743, - "hfopenllm_v2/MUSR": 0.3343, - "hfopenllm_v2/MMLU-PRO": 0.1202 - } - }, - { - "id": "SentientAGI/Dobby-Mini-Leashed-Llama-3.1-8B", - "name": "Dobby-Mini-Leashed-Llama-3.1-8B", - "developer": "SentientAGI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7847, - "hfopenllm_v2/BBH": 0.5138, - "hfopenllm_v2/MATH Level 5": 0.1858, - "hfopenllm_v2/GPQA": 0.302, - "hfopenllm_v2/MUSR": 0.4254, - "hfopenllm_v2/MMLU-PRO": 0.3694 - } - }, - { - "id": "SentientAGI/Dobby-Mini-Unhinged-Llama-3.1-8B", - "name": "Dobby-Mini-Unhinged-Llama-3.1-8B", - "developer": "SentientAGI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7457, - "hfopenllm_v2/BBH": 0.5142, - "hfopenllm_v2/MATH Level 5": 0.1563, - "hfopenllm_v2/GPQA": 0.3062, - "hfopenllm_v2/MUSR": 0.4013, - "hfopenllm_v2/MMLU-PRO": 0.3585 - } - }, - { - "id": "SeppeV/SmolLM_pretrained_with_sft_trained_with_1pc_data_on_a_preference_dpo", - "name": "SmolLM_pretrained_with_sft_trained_with_1pc_data_on_a_preference_dpo", - "developer": "SeppeV", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0955, - "hfopenllm_v2/BBH": 0.3073, - "hfopenllm_v2/MATH Level 5": 0.0121, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.4032, - "hfopenllm_v2/MMLU-PRO": 0.1161 - } - }, - { - "id": "sequelbox/gemma-2-9B-MOTH", - "name": "gemma-2-9B-MOTH", - "developer": "sequelbox", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2059, - "hfopenllm_v2/BBH": 0.308, - "hfopenllm_v2/MATH Level 5": 0.0106, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3409, - "hfopenllm_v2/MMLU-PRO": 0.114 - } - }, - { - "id": "sequelbox/Llama3.1-70B-PlumChat", - "name": "Llama3.1-70B-PlumChat", - "developer": "sequelbox", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5616, - "hfopenllm_v2/BBH": 0.6753, - "hfopenllm_v2/MATH Level 5": 0.3029, - "hfopenllm_v2/GPQA": 0.3909, - "hfopenllm_v2/MUSR": 0.4774, - "hfopenllm_v2/MMLU-PRO": 0.5164 - } - }, - { - "id": "sequelbox/Llama3.1-8B-MOTH", - "name": "Llama3.1-8B-MOTH", - "developer": "sequelbox", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5245, - "hfopenllm_v2/BBH": 0.4902, - "hfopenllm_v2/MATH Level 5": 0.1216, - "hfopenllm_v2/GPQA": 0.2685, - "hfopenllm_v2/MUSR": 0.3689, - "hfopenllm_v2/MMLU-PRO": 0.3339 - } - }, - { - "id": "sequelbox/Llama3.1-8B-PlumChat", - "name": "Llama3.1-8B-PlumChat", - "developer": "sequelbox", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4243, - "hfopenllm_v2/BBH": 0.3873, - "hfopenllm_v2/MATH Level 5": 0.0363, - "hfopenllm_v2/GPQA": 0.2651, - "hfopenllm_v2/MUSR": 0.3755, - "hfopenllm_v2/MMLU-PRO": 0.2127 - } - }, - { - "id": "sequelbox/Llama3.1-8B-PlumCode", - "name": "Llama3.1-8B-PlumCode", - "developer": "sequelbox", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2045, - "hfopenllm_v2/BBH": 0.3368, - "hfopenllm_v2/MATH Level 5": 0.0272, - "hfopenllm_v2/GPQA": 0.276, - "hfopenllm_v2/MUSR": 0.3773, - "hfopenllm_v2/MMLU-PRO": 0.2335 - } - }, - { - "id": "sequelbox/Llama3.1-8B-PlumMath", - "name": "Llama3.1-8B-PlumMath", - "developer": "sequelbox", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2242, - "hfopenllm_v2/BBH": 0.4032, - "hfopenllm_v2/MATH Level 5": 0.0476, - "hfopenllm_v2/GPQA": 0.318, - "hfopenllm_v2/MUSR": 0.3919, - "hfopenllm_v2/MMLU-PRO": 0.2975 - } - }, - { - "id": "sethuiyer/Llama-3.1-8B-Experimental-1206-Instruct", - "name": "Llama-3.1-8B-Experimental-1206-Instruct", - "developer": "sethuiyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6967, - "hfopenllm_v2/BBH": 0.5104, - "hfopenllm_v2/MATH Level 5": 0.1118, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.3966, - "hfopenllm_v2/MMLU-PRO": 0.3529 - } - }, - { - "id": "sethuiyer/Llama-3.1-8B-Experimental-1208-Instruct", - "name": "Llama-3.1-8B-Experimental-1208-Instruct", - "developer": "sethuiyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.61, - "hfopenllm_v2/BBH": 0.4964, - "hfopenllm_v2/MATH Level 5": 0.0891, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.379, - "hfopenllm_v2/MMLU-PRO": 0.3511 - } - }, - { - "id": "sethuiyer/Llamaverse-3.1-8B-Instruct", - "name": "Llamaverse-3.1-8B-Instruct", - "developer": "sethuiyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6185, - "hfopenllm_v2/BBH": 0.5414, - "hfopenllm_v2/MATH Level 5": 0.1858, - "hfopenllm_v2/GPQA": 0.2911, - "hfopenllm_v2/MUSR": 0.3762, - "hfopenllm_v2/MMLU-PRO": 0.3523 - } - }, - { - "id": "sethuiyer/LlamaZero-3.1-8B-Experimental-1208", - "name": "LlamaZero-3.1-8B-Experimental-1208", - "developer": "sethuiyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6051, - "hfopenllm_v2/BBH": 0.4981, - "hfopenllm_v2/MATH Level 5": 0.108, - "hfopenllm_v2/GPQA": 0.2685, - "hfopenllm_v2/MUSR": 0.382, - "hfopenllm_v2/MMLU-PRO": 0.3 - } - }, - { - "id": "sethuiyer/Llamazing-3.1-8B-Instruct", - "name": "Llamazing-3.1-8B-Instruct", - "developer": "sethuiyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5711, - "hfopenllm_v2/BBH": 0.5291, - "hfopenllm_v2/MATH Level 5": 0.0544, - "hfopenllm_v2/GPQA": 0.3121, - "hfopenllm_v2/MUSR": 0.3976, - "hfopenllm_v2/MMLU-PRO": 0.3606 - } - }, - { - "id": "sethuiyer/Qwen2.5-7B-Anvita", - "name": "Qwen2.5-7B-Anvita", - "developer": "sethuiyer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.648, - "hfopenllm_v2/BBH": 0.5466, - "hfopenllm_v2/MATH Level 5": 0.2017, - "hfopenllm_v2/GPQA": 0.3272, - "hfopenllm_v2/MUSR": 0.4337, - "hfopenllm_v2/MMLU-PRO": 0.4166 - } - }, - { - "id": "SF-Foundation/TextEval-Llama3.1-70B", - "name": "SF-Foundation/TextEval-Llama3.1-70B", - "developer": "SF-Foundation", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.9348, - "reward-bench/Chat": 0.9413, - "reward-bench/Chat Hard": 0.9013, - "reward-bench/Safety": 0.9324, - "reward-bench/Reasoning": 0.9641 - } - }, - { - "id": "SF-Foundation/TextEval-OffsetBias-12B", - "name": "SF-Foundation/TextEval-OffsetBias-12B", - "developer": "SF-Foundation", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.9105, - "reward-bench/Chat": 0.919, - "reward-bench/Chat Hard": 0.8662, - "reward-bench/Safety": 0.9203, - "reward-bench/Reasoning": 0.9365 - } - }, - { - "id": "sfairXC/FsfairX-LLaMA3-RM-v0.1", - "name": "sfairXC/FsfairX-LLaMA3-RM-v0.1", - "developer": "sfairXC", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6292, - "reward-bench/Chat": 0.9944, - "reward-bench/Chat Hard": 0.6513, - "reward-bench/Safety": 0.7667, - "reward-bench/Reasoning": 0.8644, - "reward-bench/Prior Sets (0.5 weight)": 0.7492, - "reward-bench/Factuality": 0.5916, - "reward-bench/Precise IF": 0.4188, - "reward-bench/Math": 0.6284, - "reward-bench/Focus": 0.7051, - "reward-bench/Ties": 0.6647 - } - }, - { - "id": "shadowml/BeagSake-7B", - "name": "BeagSake-7B", - "developer": "shadowml", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5216, - "hfopenllm_v2/BBH": 0.4711, - "hfopenllm_v2/MATH Level 5": 0.0506, - "hfopenllm_v2/GPQA": 0.281, - "hfopenllm_v2/MUSR": 0.4124, - "hfopenllm_v2/MMLU-PRO": 0.2585 - } - }, - { - "id": "shadowml/Mixolar-4x7b", - "name": "Mixolar-4x7b", - "developer": "shadowml", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3893, - "hfopenllm_v2/BBH": 0.5216, - "hfopenllm_v2/MATH Level 5": 0.0582, - "hfopenllm_v2/GPQA": 0.2928, - "hfopenllm_v2/MUSR": 0.4258, - "hfopenllm_v2/MMLU-PRO": 0.3305 - } - }, - { - "id": "Sharathhebbar24/chat_gpt2_dpo", - "name": "chat_gpt2_dpo", - "developer": "Sharathhebbar24", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0986, - "hfopenllm_v2/BBH": 0.2902, - "hfopenllm_v2/MATH Level 5": 0.0053, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3818, - "hfopenllm_v2/MMLU-PRO": 0.1142 - } - }, - { - "id": "Sharathhebbar24/SSH_355M", - "name": "SSH_355M", - "developer": "Sharathhebbar24", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1424, - "hfopenllm_v2/BBH": 0.3099, - "hfopenllm_v2/MATH Level 5": 0.0091, - "hfopenllm_v2/GPQA": 0.2584, - "hfopenllm_v2/MUSR": 0.4178, - "hfopenllm_v2/MMLU-PRO": 0.1176 - } - }, - { - "id": "shastraai/Shastra-LLAMA2-Math-Commonsense-SFT", - "name": "Shastra-LLAMA2-Math-Commonsense-SFT", - "developer": "shastraai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3042, - "hfopenllm_v2/BBH": 0.3843, - "hfopenllm_v2/MATH Level 5": 0.0174, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3604, - "hfopenllm_v2/MMLU-PRO": 0.1997 - } - }, - { - "id": "ShikaiChen/LDL-Reward-Gemma-2-27B-v0.1", - "name": "ShikaiChen/LDL-Reward-Gemma-2-27B-v0.1", - "developer": "ShikaiChen", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.9499, - "reward-bench/Factuality": 0.7558, - "reward-bench/Precise IF": 0.35, - "reward-bench/Math": 0.6448, - "reward-bench/Safety": 0.9378, - "reward-bench/Focus": 0.9131, - "reward-bench/Ties": 0.7633, - "reward-bench/Chat": 0.9637, - "reward-bench/Chat Hard": 0.9079, - "reward-bench/Reasoning": 0.9903 - } - }, - { - "id": "shivam9980/mistral-7b-news-cnn-merged", - "name": "mistral-7b-news-cnn-merged", - "developer": "shivam9980", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4634, - "hfopenllm_v2/BBH": 0.3635, - "hfopenllm_v2/MATH Level 5": 0.0189, - "hfopenllm_v2/GPQA": 0.3087, - "hfopenllm_v2/MUSR": 0.4523, - "hfopenllm_v2/MMLU-PRO": 0.2827 - } - }, - { - "id": "shivam9980/NEPALI-LLM", - "name": "NEPALI-LLM", - "developer": "shivam9980", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0417, - "hfopenllm_v2/BBH": 0.3828, - "hfopenllm_v2/MATH Level 5": 0.0091, - "hfopenllm_v2/GPQA": 0.2617, - "hfopenllm_v2/MUSR": 0.4122, - "hfopenllm_v2/MMLU-PRO": 0.2064 - } - }, - { - "id": "shivank21/mistral_dpo_self", - "name": "mistral_dpo_self", - "developer": "shivank21", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3403, - "hfopenllm_v2/BBH": 0.3216, - "hfopenllm_v2/MATH Level 5": 0.0219, - "hfopenllm_v2/GPQA": 0.2408, - "hfopenllm_v2/MUSR": 0.3247, - "hfopenllm_v2/MMLU-PRO": 0.2214 - } - }, - { - "id": "Shreyash2010/Uma-4x4B-Instruct-v0.1", - "name": "Uma-4x4B-Instruct-v0.1", - "developer": "Shreyash2010", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5517, - "hfopenllm_v2/BBH": 0.5512, - "hfopenllm_v2/MATH Level 5": 0.1775, - "hfopenllm_v2/GPQA": 0.3347, - "hfopenllm_v2/MUSR": 0.4441, - "hfopenllm_v2/MMLU-PRO": 0.387 - } - }, - { - "id": "shuttleai/shuttle-3", - "name": "shuttle-3", - "developer": "shuttleai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8154, - "hfopenllm_v2/BBH": 0.742, - "hfopenllm_v2/MATH Level 5": 0.46, - "hfopenllm_v2/GPQA": 0.4119, - "hfopenllm_v2/MUSR": 0.4377, - "hfopenllm_v2/MMLU-PRO": 0.5716 - } - }, - { - "id": "shyamieee/Padma-v7.0", - "name": "Padma-v7.0", - "developer": "shyamieee", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3841, - "hfopenllm_v2/BBH": 0.5119, - "hfopenllm_v2/MATH Level 5": 0.0702, - "hfopenllm_v2/GPQA": 0.2861, - "hfopenllm_v2/MUSR": 0.4386, - "hfopenllm_v2/MMLU-PRO": 0.3029 - } - }, - { - "id": "Sicarius-Prototyping/bacon_and_food", - "name": "bacon_and_food", - "developer": "Sicarius-Prototyping", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.586, - "hfopenllm_v2/BBH": 0.4725, - "hfopenllm_v2/MATH Level 5": 0.0982, - "hfopenllm_v2/GPQA": 0.3096, - "hfopenllm_v2/MUSR": 0.3884, - "hfopenllm_v2/MMLU-PRO": 0.3263 - } - }, - { - "id": "Sicarius-Prototyping/Brainy_LLAMA", - "name": "Brainy_LLAMA", - "developer": "Sicarius-Prototyping", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5204, - "hfopenllm_v2/BBH": 0.5117, - "hfopenllm_v2/MATH Level 5": 0.1337, - "hfopenllm_v2/GPQA": 0.3138, - "hfopenllm_v2/MUSR": 0.4143, - "hfopenllm_v2/MMLU-PRO": 0.3849 - } - }, - { - "id": "Sicarius-Prototyping/Micropenis_1B", - "name": "Micropenis_1B", - "developer": "Sicarius-Prototyping", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3461, - "hfopenllm_v2/BBH": 0.3372, - "hfopenllm_v2/MATH Level 5": 0.0461, - "hfopenllm_v2/GPQA": 0.2626, - "hfopenllm_v2/MUSR": 0.3325, - "hfopenllm_v2/MMLU-PRO": 0.186 - } - }, - { - "id": "SicariusSicariiStuff/2B-ad", - "name": "2B-ad", - "developer": "SicariusSicariiStuff", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4379, - "hfopenllm_v2/BBH": 0.4092, - "hfopenllm_v2/MATH Level 5": 0.0506, - "hfopenllm_v2/GPQA": 0.281, - "hfopenllm_v2/MUSR": 0.4015, - "hfopenllm_v2/MMLU-PRO": 0.2662 - } - }, - { - "id": "SicariusSicariiStuff/2B_or_not_2B", - "name": "2B_or_not_2B", - "developer": "SicariusSicariiStuff", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2062, - "hfopenllm_v2/BBH": 0.3416, - "hfopenllm_v2/MATH Level 5": 0.0196, - "hfopenllm_v2/GPQA": 0.2475, - "hfopenllm_v2/MUSR": 0.3791, - "hfopenllm_v2/MMLU-PRO": 0.1399 - } - }, - { - "id": "SicariusSicariiStuff/dn_ep02", - "name": "dn_ep02", - "developer": "SicariusSicariiStuff", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5064, - "hfopenllm_v2/BBH": 0.5266, - "hfopenllm_v2/MATH Level 5": 0.142, - "hfopenllm_v2/GPQA": 0.3154, - "hfopenllm_v2/MUSR": 0.4316, - "hfopenllm_v2/MMLU-PRO": 0.3998 - } - }, - { - "id": "SicariusSicariiStuff/Dusk_Rainbow", - "name": "Dusk_Rainbow", - "developer": "SicariusSicariiStuff", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3588, - "hfopenllm_v2/BBH": 0.4772, - "hfopenllm_v2/MATH Level 5": 0.0748, - "hfopenllm_v2/GPQA": 0.3087, - "hfopenllm_v2/MUSR": 0.4025, - "hfopenllm_v2/MMLU-PRO": 0.3443 - } - }, - { - "id": "SicariusSicariiStuff/Eximius_Persona_5B", - "name": "Eximius_Persona_5B", - "developer": "SicariusSicariiStuff", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.656, - "hfopenllm_v2/BBH": 0.4512, - "hfopenllm_v2/MATH Level 5": 0.102, - "hfopenllm_v2/GPQA": 0.2643, - "hfopenllm_v2/MUSR": 0.3818, - "hfopenllm_v2/MMLU-PRO": 0.314 - } - }, - { - "id": "SicariusSicariiStuff/Impish_LLAMA_3B", - "name": "Impish_LLAMA_3B", - "developer": "SicariusSicariiStuff", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.463, - "hfopenllm_v2/BBH": 0.4091, - "hfopenllm_v2/MATH Level 5": 0.1125, - "hfopenllm_v2/GPQA": 0.2878, - "hfopenllm_v2/MUSR": 0.3673, - "hfopenllm_v2/MMLU-PRO": 0.2941 - } - }, - { - "id": "SicariusSicariiStuff/Impish_Mind_8B", - "name": "Impish_Mind_8B", - "developer": "SicariusSicariiStuff", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3179, - "hfopenllm_v2/BBH": 0.4674, - "hfopenllm_v2/MATH Level 5": 0.105, - "hfopenllm_v2/GPQA": 0.3045, - "hfopenllm_v2/MUSR": 0.407, - "hfopenllm_v2/MMLU-PRO": 0.3309 - } - }, - { - "id": "SicariusSicariiStuff/Impish_QWEN_14B-1M", - "name": "Impish_QWEN_14B-1M", - "developer": "SicariusSicariiStuff", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7868, - "hfopenllm_v2/BBH": 0.6283, - "hfopenllm_v2/MATH Level 5": 0.3965, - "hfopenllm_v2/GPQA": 0.3507, - "hfopenllm_v2/MUSR": 0.4615, - "hfopenllm_v2/MMLU-PRO": 0.5044 - } - }, - { - "id": "SicariusSicariiStuff/Impish_QWEN_7B-1M", - "name": "Impish_QWEN_7B-1M", - "developer": "SicariusSicariiStuff", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6382, - "hfopenllm_v2/BBH": 0.5372, - "hfopenllm_v2/MATH Level 5": 0.3089, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.4074, - "hfopenllm_v2/MMLU-PRO": 0.4265 - } - }, - { - "id": "SicariusSicariiStuff/LLAMA-3_8B_Unaligned_BETA", - "name": "LLAMA-3_8B_Unaligned_BETA", - "developer": "SicariusSicariiStuff", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3713, - "hfopenllm_v2/BBH": 0.4717, - "hfopenllm_v2/MATH Level 5": 0.0838, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.4119, - "hfopenllm_v2/MMLU-PRO": 0.3465 - } - }, - { - "id": "SicariusSicariiStuff/Phi-Line_14B", - "name": "Phi-Line_14B", - "developer": "SicariusSicariiStuff", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6496, - "hfopenllm_v2/BBH": 0.6154, - "hfopenllm_v2/MATH Level 5": 0.386, - "hfopenllm_v2/GPQA": 0.3532, - "hfopenllm_v2/MUSR": 0.4479, - "hfopenllm_v2/MMLU-PRO": 0.5454 - } - }, - { - "id": "SicariusSicariiStuff/Phi-lthy4", - "name": "Phi-lthy4", - "developer": "SicariusSicariiStuff", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7679, - "hfopenllm_v2/BBH": 0.5879, - "hfopenllm_v2/MATH Level 5": 0.1367, - "hfopenllm_v2/GPQA": 0.2869, - "hfopenllm_v2/MUSR": 0.4083, - "hfopenllm_v2/MMLU-PRO": 0.4333 - } - }, - { - "id": "SicariusSicariiStuff/Qwen2.5-14B_Uncencored", - "name": "Qwen2.5-14B_Uncencored", - "developer": "SicariusSicariiStuff", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3158, - "hfopenllm_v2/BBH": 0.6309, - "hfopenllm_v2/MATH Level 5": 0.318, - "hfopenllm_v2/GPQA": 0.3817, - "hfopenllm_v2/MUSR": 0.4517, - "hfopenllm_v2/MMLU-PRO": 0.5266 - } - }, - { - "id": "SicariusSicariiStuff/Qwen2.5-14B_Uncensored", - "name": "Qwen2.5-14B_Uncensored", - "developer": "SicariusSicariiStuff", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3173, - "hfopenllm_v2/BBH": 0.6309, - "hfopenllm_v2/MATH Level 5": 0.318, - "hfopenllm_v2/GPQA": 0.3817, - "hfopenllm_v2/MUSR": 0.4517, - "hfopenllm_v2/MMLU-PRO": 0.5266 - } - }, - { - "id": "SicariusSicariiStuff/Qwen2.5-14B_Uncensored_Instruct", - "name": "Qwen2.5-14B_Uncensored_Instruct", - "developer": "SicariusSicariiStuff", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3789, - "hfopenllm_v2/BBH": 0.5937, - "hfopenllm_v2/MATH Level 5": 0.3285, - "hfopenllm_v2/GPQA": 0.3297, - "hfopenllm_v2/MUSR": 0.3697, - "hfopenllm_v2/MMLU-PRO": 0.5127 - } - }, - { - "id": "SicariusSicariiStuff/Redemption_Wind_24B", - "name": "Redemption_Wind_24B", - "developer": "SicariusSicariiStuff", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2501, - "hfopenllm_v2/BBH": 0.6428, - "hfopenllm_v2/MATH Level 5": 0.1858, - "hfopenllm_v2/GPQA": 0.3834, - "hfopenllm_v2/MUSR": 0.4262, - "hfopenllm_v2/MMLU-PRO": 0.5432 - } - }, - { - "id": "SicariusSicariiStuff/Winged_Imp_8B", - "name": "Winged_Imp_8B", - "developer": "SicariusSicariiStuff", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.743, - "hfopenllm_v2/BBH": 0.512, - "hfopenllm_v2/MATH Level 5": 0.1201, - "hfopenllm_v2/GPQA": 0.2827, - "hfopenllm_v2/MUSR": 0.4148, - "hfopenllm_v2/MMLU-PRO": 0.3639 - } - }, - { - "id": "SicariusSicariiStuff/Wingless_Imp_8B", - "name": "Wingless_Imp_8B", - "developer": "SicariusSicariiStuff", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.743, - "hfopenllm_v2/BBH": 0.512, - "hfopenllm_v2/MATH Level 5": 0.1201, - "hfopenllm_v2/GPQA": 0.2827, - "hfopenllm_v2/MUSR": 0.4148, - "hfopenllm_v2/MMLU-PRO": 0.3639 - } - }, - { - "id": "SicariusSicariiStuff/Zion_Alpha", - "name": "Zion_Alpha", - "developer": "SicariusSicariiStuff", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3324, - "hfopenllm_v2/BBH": 0.4932, - "hfopenllm_v2/MATH Level 5": 0.0521, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.4727, - "hfopenllm_v2/MMLU-PRO": 0.3132 - } - }, - { - "id": "silma-ai/SILMA-9B-Instruct-v1.0", - "name": "SILMA-9B-Instruct-v1.0", - "developer": "silma-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5842, - "hfopenllm_v2/BBH": 0.5219, - "hfopenllm_v2/MATH Level 5": 0.1163, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.4637, - "hfopenllm_v2/MMLU-PRO": 0.392 - } - }, - { - "id": "silma-ai/SILMA-Kashif-2B-Instruct-v1.0", - "name": "SILMA-Kashif-2B-Instruct-v1.0", - "developer": "silma-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1181, - "hfopenllm_v2/BBH": 0.3793, - "hfopenllm_v2/MATH Level 5": 0.0113, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.4043, - "hfopenllm_v2/MMLU-PRO": 0.2258 - } - }, - { - "id": "siqi00/Mistral-7B-DFT", - "name": "Mistral-7B-DFT", - "developer": "siqi00", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5569, - "hfopenllm_v2/BBH": 0.4665, - "hfopenllm_v2/MATH Level 5": 0.0378, - "hfopenllm_v2/GPQA": 0.3045, - "hfopenllm_v2/MUSR": 0.4191, - "hfopenllm_v2/MMLU-PRO": 0.2963 - } - }, - { - "id": "siqi00/Mistral-7B-DFT2", - "name": "Mistral-7B-DFT2", - "developer": "siqi00", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5804, - "hfopenllm_v2/BBH": 0.3968, - "hfopenllm_v2/MATH Level 5": 0.0453, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.4401, - "hfopenllm_v2/MMLU-PRO": 0.2852 - } - }, - { - "id": "skumar9/Llama-medx_v2", - "name": "Llama-medx_v2", - "developer": "skumar9", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4462, - "hfopenllm_v2/BBH": 0.4909, - "hfopenllm_v2/MATH Level 5": 0.0914, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.3661, - "hfopenllm_v2/MMLU-PRO": 0.3463 - } - }, - { - "id": "skymizer/Llama2-7b-sft-chat-custom-template-dpo", - "name": "Llama2-7b-sft-chat-custom-template-dpo", - "developer": "skymizer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2353, - "hfopenllm_v2/BBH": 0.3688, - "hfopenllm_v2/MATH Level 5": 0.0144, - "hfopenllm_v2/GPQA": 0.2391, - "hfopenllm_v2/MUSR": 0.4429, - "hfopenllm_v2/MMLU-PRO": 0.1946 - } - }, - { - "id": "SkyOrbis/SKY-Ko-Llama3.1-8B-lora", - "name": "SKY-Ko-Llama3.1-8B-lora", - "developer": "SkyOrbis", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5058, - "hfopenllm_v2/BBH": 0.5088, - "hfopenllm_v2/MATH Level 5": 0.1548, - "hfopenllm_v2/GPQA": 0.3213, - "hfopenllm_v2/MUSR": 0.3998, - "hfopenllm_v2/MMLU-PRO": 0.3777 - } - }, - { - "id": "SkyOrbis/SKY-Ko-Llama3.1-8B-lora-epoch1", - "name": "SKY-Ko-Llama3.1-8B-lora-epoch1", - "developer": "SkyOrbis", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5058, - "hfopenllm_v2/BBH": 0.5088, - "hfopenllm_v2/MATH Level 5": 0.1548, - "hfopenllm_v2/GPQA": 0.3213, - "hfopenllm_v2/MUSR": 0.3998, - "hfopenllm_v2/MMLU-PRO": 0.3777 - } - }, - { - "id": "SkyOrbis/SKY-Ko-Llama3.2-1B-lora-epoch3", - "name": "SKY-Ko-Llama3.2-1B-lora-epoch3", - "developer": "SkyOrbis", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3247, - "hfopenllm_v2/BBH": 0.3167, - "hfopenllm_v2/MATH Level 5": 0.0272, - "hfopenllm_v2/GPQA": 0.2517, - "hfopenllm_v2/MUSR": 0.3382, - "hfopenllm_v2/MMLU-PRO": 0.1279 - } - }, - { - "id": "SkyOrbis/SKY-Ko-Llama3.2-1B-lora-epoch5", - "name": "SKY-Ko-Llama3.2-1B-lora-epoch5", - "developer": "SkyOrbis", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.436, - "hfopenllm_v2/BBH": 0.3406, - "hfopenllm_v2/MATH Level 5": 0.0521, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3471, - "hfopenllm_v2/MMLU-PRO": 0.1946 - } - }, - { - "id": "SkyOrbis/SKY-Ko-Llama3.2-1B-lora-v2-epoch3", - "name": "SKY-Ko-Llama3.2-1B-lora-v2-epoch3", - "developer": "SkyOrbis", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.436, - "hfopenllm_v2/BBH": 0.3406, - "hfopenllm_v2/MATH Level 5": 0.0521, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3471, - "hfopenllm_v2/MMLU-PRO": 0.1946 - } - }, - { - "id": "SkyOrbis/SKY-Ko-Llama3.2-1B-lora-v2-epoch5", - "name": "SKY-Ko-Llama3.2-1B-lora-v2-epoch5", - "developer": "SkyOrbis", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4247, - "hfopenllm_v2/BBH": 0.3397, - "hfopenllm_v2/MATH Level 5": 0.0506, - "hfopenllm_v2/GPQA": 0.2542, - "hfopenllm_v2/MUSR": 0.3458, - "hfopenllm_v2/MMLU-PRO": 0.1946 - } - }, - { - "id": "SkyOrbis/SKY-Ko-Llama3.2-3B-lora-epoch1", - "name": "SKY-Ko-Llama3.2-3B-lora-epoch1", - "developer": "SkyOrbis", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5331, - "hfopenllm_v2/BBH": 0.44, - "hfopenllm_v2/MATH Level 5": 0.1458, - "hfopenllm_v2/GPQA": 0.2919, - "hfopenllm_v2/MUSR": 0.3522, - "hfopenllm_v2/MMLU-PRO": 0.3004 - } - }, - { - "id": "SkyOrbis/SKY-Ko-Llama3.2-3B-lora-epoch2", - "name": "SKY-Ko-Llama3.2-3B-lora-epoch2", - "developer": "SkyOrbis", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5331, - "hfopenllm_v2/BBH": 0.44, - "hfopenllm_v2/MATH Level 5": 0.1458, - "hfopenllm_v2/GPQA": 0.2919, - "hfopenllm_v2/MUSR": 0.3522, - "hfopenllm_v2/MMLU-PRO": 0.3004 - } - }, - { - "id": "SkyOrbis/SKY-Ko-Llama3.2-3B-lora-epoch3", - "name": "SKY-Ko-Llama3.2-3B-lora-epoch3", - "developer": "SkyOrbis", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5331, - "hfopenllm_v2/BBH": 0.44, - "hfopenllm_v2/MATH Level 5": 0.1458, - "hfopenllm_v2/GPQA": 0.2919, - "hfopenllm_v2/MUSR": 0.3522, - "hfopenllm_v2/MMLU-PRO": 0.3004 - } - }, - { - "id": "SkyOrbis/SKY-Ko-Qwen2.5-3B-Instruct", - "name": "SKY-Ko-Qwen2.5-3B-Instruct", - "developer": "SkyOrbis", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3534, - "hfopenllm_v2/BBH": 0.4265, - "hfopenllm_v2/MATH Level 5": 0.0695, - "hfopenllm_v2/GPQA": 0.2794, - "hfopenllm_v2/MUSR": 0.4024, - "hfopenllm_v2/MMLU-PRO": 0.2812 - } - }, - { - "id": "SkyOrbis/SKY-Ko-Qwen2.5-7B-Instruct-SFT-step-15000", - "name": "SKY-Ko-Qwen2.5-7B-Instruct-SFT-step-15000", - "developer": "SkyOrbis", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3819, - "hfopenllm_v2/BBH": 0.5078, - "hfopenllm_v2/MATH Level 5": 0.1866, - "hfopenllm_v2/GPQA": 0.3272, - "hfopenllm_v2/MUSR": 0.4436, - "hfopenllm_v2/MMLU-PRO": 0.3914 - } - }, - { - "id": "SkyOrbis/SKY-Ko-Qwen2.5-7B-Instruct-SFT-step-5000", - "name": "SKY-Ko-Qwen2.5-7B-Instruct-SFT-step-5000", - "developer": "SkyOrbis", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3812, - "hfopenllm_v2/BBH": 0.539, - "hfopenllm_v2/MATH Level 5": 0.21, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.4238, - "hfopenllm_v2/MMLU-PRO": 0.4238 - } - }, - { - "id": "Skywork/Skywork-Critic-Llama-3.1-70B", - "name": "Skywork/Skywork-Critic-Llama-3.1-70B", - "developer": "Skywork", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.9331, - "reward-bench/Chat": 0.9665, - "reward-bench/Chat Hard": 0.8794, - "reward-bench/Safety": 0.9311, - "reward-bench/Reasoning": 0.9554 - } - }, - { - "id": "Skywork/Skywork-Critic-Llama-3.1-8B", - "name": "Skywork/Skywork-Critic-Llama-3.1-8B", - "developer": "Skywork", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.8896, - "reward-bench/Chat": 0.9358, - "reward-bench/Chat Hard": 0.8136, - "reward-bench/Safety": 0.9108, - "reward-bench/Reasoning": 0.898 - } - }, - { - "id": "Skywork/Skywork-o1-Open-Llama-3.1-8B", - "name": "Skywork-o1-Open-Llama-3.1-8B", - "developer": "Skywork", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3518, - "hfopenllm_v2/BBH": 0.4516, - "hfopenllm_v2/MATH Level 5": 0.5211, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3156, - "hfopenllm_v2/MMLU-PRO": 0.203 - } - }, - { - "id": "Skywork/Skywork-Reward-Gemma-2-27B", - "name": "Skywork/Skywork-Reward-Gemma-2-27B", - "developer": "Skywork", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.938, - "reward-bench/Factuality": 0.7368, - "reward-bench/Precise IF": 0.4031, - "reward-bench/Math": 0.7049, - "reward-bench/Safety": 0.9189, - "reward-bench/Focus": 0.9323, - "reward-bench/Ties": 0.8261, - "reward-bench/Chat": 0.9581, - "reward-bench/Chat Hard": 0.9145, - "reward-bench/Reasoning": 0.9606 - } - }, - { - "id": "Skywork/Skywork-Reward-Gemma-2-27B-v0.2", - "name": "Skywork/Skywork-Reward-Gemma-2-27B-v0.2", - "developer": "Skywork", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7807, - "hfopenllm_v2/BBH": 0.636, - "hfopenllm_v2/MATH Level 5": 0.2273, - "hfopenllm_v2/GPQA": 0.344, - "hfopenllm_v2/MUSR": 0.4231, - "hfopenllm_v2/MMLU-PRO": 0.4103, - "reward-bench/Score": 0.9426, - "reward-bench/Factuality": 0.7674, - "reward-bench/Precise IF": 0.375, - "reward-bench/Math": 0.6721, - "reward-bench/Safety": 0.9297, - "reward-bench/Focus": 0.9172, - "reward-bench/Ties": 0.8182, - "reward-bench/Chat": 0.9609, - "reward-bench/Chat Hard": 0.8991, - "reward-bench/Reasoning": 0.9807 - } - }, - { - "id": "Skywork/Skywork-Reward-Llama-3.1-8B", - "name": "Skywork/Skywork-Reward-Llama-3.1-8B", - "developer": "Skywork", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7314, - "reward-bench/Chat": 0.9581, - "reward-bench/Chat Hard": 0.8728, - "reward-bench/Safety": 0.9333, - "reward-bench/Reasoning": 0.962, - "reward-bench/Factuality": 0.6989, - "reward-bench/Precise IF": 0.425, - "reward-bench/Math": 0.6284, - "reward-bench/Focus": 0.9616, - "reward-bench/Ties": 0.741 - } - }, - { - "id": "Skywork/Skywork-Reward-Llama-3.1-8B-v0.2", - "name": "Skywork/Skywork-Reward-Llama-3.1-8B-v0.2", - "developer": "Skywork", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7175, - "reward-bench/Chat": 0.9469, - "reward-bench/Chat Hard": 0.8838, - "reward-bench/Safety": 0.9422, - "reward-bench/Reasoning": 0.9675, - "reward-bench/Factuality": 0.6968, - "reward-bench/Precise IF": 0.4062, - "reward-bench/Math": 0.6011, - "reward-bench/Focus": 0.9414, - "reward-bench/Ties": 0.7169 - } - }, - { - "id": "Skywork/Skywork-Reward-V2-Llama-3.1-8B", - "name": "Skywork/Skywork-Reward-V2-Llama-3.1-8B", - "developer": "Skywork", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.8413, - "reward-bench/Factuality": 0.8463, - "reward-bench/Precise IF": 0.6625, - "reward-bench/Math": 0.776, - "reward-bench/Safety": 0.9667, - "reward-bench/Focus": 0.9838, - "reward-bench/Ties": 0.8124 - } - }, - { - "id": "Skywork/Skywork-Reward-V2-Llama-3.2-1B", - "name": "Skywork/Skywork-Reward-V2-Llama-3.2-1B", - "developer": "Skywork", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6438, - "reward-bench/Factuality": 0.6084, - "reward-bench/Precise IF": 0.4562, - "reward-bench/Math": 0.6011, - "reward-bench/Safety": 0.8733, - "reward-bench/Focus": 0.8929, - "reward-bench/Ties": 0.4306 - } - }, - { - "id": "Skywork/Skywork-Reward-V2-Llama-3.2-3B", - "name": "Skywork/Skywork-Reward-V2-Llama-3.2-3B", - "developer": "Skywork", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7466, - "reward-bench/Factuality": 0.7621, - "reward-bench/Precise IF": 0.4562, - "reward-bench/Math": 0.694, - "reward-bench/Safety": 0.9311, - "reward-bench/Focus": 0.9596, - "reward-bench/Ties": 0.6768 - } - }, - { - "id": "Skywork/Skywork-Reward-V2-Qwen3-0.6B", - "name": "Skywork/Skywork-Reward-V2-Qwen3-0.6B", - "developer": "Skywork", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6125, - "reward-bench/Factuality": 0.58, - "reward-bench/Precise IF": 0.4, - "reward-bench/Math": 0.7158, - "reward-bench/Safety": 0.8444, - "reward-bench/Focus": 0.7949, - "reward-bench/Ties": 0.3397 - } - }, - { - "id": "Skywork/Skywork-Reward-V2-Qwen3-1.7B", - "name": "Skywork/Skywork-Reward-V2-Qwen3-1.7B", - "developer": "Skywork", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6818, - "reward-bench/Factuality": 0.6568, - "reward-bench/Precise IF": 0.4437, - "reward-bench/Math": 0.7268, - "reward-bench/Safety": 0.8911, - "reward-bench/Focus": 0.8848, - "reward-bench/Ties": 0.4872 - } - }, - { - "id": "Skywork/Skywork-Reward-V2-Qwen3-4B", - "name": "Skywork/Skywork-Reward-V2-Qwen3-4B", - "developer": "Skywork", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7551, - "reward-bench/Factuality": 0.7737, - "reward-bench/Precise IF": 0.4625, - "reward-bench/Math": 0.7322, - "reward-bench/Safety": 0.9222, - "reward-bench/Focus": 0.9657, - "reward-bench/Ties": 0.6743 - } - }, - { - "id": "Skywork/Skywork-Reward-V2-Qwen3-8B", - "name": "Skywork/Skywork-Reward-V2-Qwen3-8B", - "developer": "Skywork", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7837, - "reward-bench/Factuality": 0.7989, - "reward-bench/Precise IF": 0.5, - "reward-bench/Math": 0.7705, - "reward-bench/Safety": 0.94, - "reward-bench/Focus": 0.9636, - "reward-bench/Ties": 0.7294 - } - }, - { - "id": "Skywork/Skywork-VL-Reward-7B", - "name": "Skywork/Skywork-VL-Reward-7B", - "developer": "Skywork", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.9007, - "reward-bench/Factuality": 0.6063, - "reward-bench/Precise IF": 0.35, - "reward-bench/Math": 0.6339, - "reward-bench/Safety": 0.9108, - "reward-bench/Focus": 0.8909, - "reward-bench/Ties": 0.7586, - "reward-bench/Chat": 0.8994, - "reward-bench/Chat Hard": 0.875, - "reward-bench/Reasoning": 0.9176 - } - }, - { - "id": "snowflake/snowflake-arctic-instruct", - "name": "Arctic Instruct", - "developer": "snowflake", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.338, - "helm_lite/NarrativeQA": 0.654, - "helm_lite/NaturalQuestions (closed-book)": 0.39, - "helm_lite/OpenbookQA": 0.828, - "helm_lite/MMLU": 0.575, - "helm_lite/MATH": 0.519, - "helm_lite/GSM8K": 0.768, - "helm_lite/LegalBench": 0.588, - "helm_lite/MedQA": 0.581, - "helm_lite/WMT 2014": 0.172, - "helm_mmlu/MMLU All Subjects": 0.677, - "helm_mmlu/Abstract Algebra": 0.35, - "helm_mmlu/Anatomy": 0.652, - "helm_mmlu/College Physics": 0.461, - "helm_mmlu/Computer Security": 0.84, - "helm_mmlu/Econometrics": 0.5, - "helm_mmlu/Global Facts": 0.39, - "helm_mmlu/Jurisprudence": 0.741, - "helm_mmlu/Philosophy": 0.752, - "helm_mmlu/Professional Psychology": 0.724, - "helm_mmlu/Us Foreign Policy": 0.88, - "helm_mmlu/Astronomy": 0.763, - "helm_mmlu/Business Ethics": 0.69, - "helm_mmlu/Clinical Knowledge": 0.781, - "helm_mmlu/Conceptual Physics": 0.634, - "helm_mmlu/Electrical Engineering": 0.662, - "helm_mmlu/Elementary Mathematics": 0.481, - "helm_mmlu/Formal Logic": 0.444, - "helm_mmlu/High School World History": 0.827, - "helm_mmlu/Human Sexuality": 0.847, - "helm_mmlu/International Law": 0.826, - "helm_mmlu/Logical Fallacies": 0.779, - "helm_mmlu/Machine Learning": 0.473, - "helm_mmlu/Management": 0.796, - "helm_mmlu/Marketing": 0.902, - "helm_mmlu/Medical Genetics": 0.76, - "helm_mmlu/Miscellaneous": 0.875, - "helm_mmlu/Moral Scenarios": 0.28, - "helm_mmlu/Nutrition": 0.725, - "helm_mmlu/Prehistory": 0.79, - "helm_mmlu/Public Relations": 0.664, - "helm_mmlu/Security Studies": 0.78, - "helm_mmlu/Sociology": 0.891, - "helm_mmlu/Virology": 0.536, - "helm_mmlu/World Religions": 0.854, - "helm_mmlu/Mean win rate": 0.565 - } - }, - { - "id": "Solshine/Brimful-merged-replete", - "name": "Brimful-merged-replete", - "developer": "Solshine", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1761, - "hfopenllm_v2/BBH": 0.2883, - "hfopenllm_v2/MATH Level 5": 0.003, - "hfopenllm_v2/GPQA": 0.2576, - "hfopenllm_v2/MUSR": 0.3421, - "hfopenllm_v2/MMLU-PRO": 0.1085 - } - }, - { - "id": "Solshine/Llama-3-1-big-thoughtful-passthrough-merge-2", - "name": "Llama-3-1-big-thoughtful-passthrough-merge-2", - "developer": "Solshine", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2547, - "hfopenllm_v2/BBH": 0.3209, - "hfopenllm_v2/MATH Level 5": 0.0106, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3889, - "hfopenllm_v2/MMLU-PRO": 0.1185 - } - }, - { - "id": "someon98/qwen-CoMa-0.5b", - "name": "qwen-CoMa-0.5b", - "developer": "someon98", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2277, - "hfopenllm_v2/BBH": 0.2953, - "hfopenllm_v2/MATH Level 5": 0.0045, - "hfopenllm_v2/GPQA": 0.2399, - "hfopenllm_v2/MUSR": 0.4046, - "hfopenllm_v2/MMLU-PRO": 0.1099 - } - }, - { - "id": "sometimesanotion/ChocoTrio-14B-v1", - "name": "ChocoTrio-14B-v1", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7089, - "hfopenllm_v2/BBH": 0.6506, - "hfopenllm_v2/MATH Level 5": 0.3973, - "hfopenllm_v2/GPQA": 0.3851, - "hfopenllm_v2/MUSR": 0.4821, - "hfopenllm_v2/MMLU-PRO": 0.537 - } - }, - { - "id": "sometimesanotion/IF-reasoning-experiment-40", - "name": "IF-reasoning-experiment-40", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.633, - "hfopenllm_v2/BBH": 0.6112, - "hfopenllm_v2/MATH Level 5": 0.3716, - "hfopenllm_v2/GPQA": 0.38, - "hfopenllm_v2/MUSR": 0.5194, - "hfopenllm_v2/MMLU-PRO": 0.5025 - } - }, - { - "id": "sometimesanotion/IF-reasoning-experiment-80", - "name": "IF-reasoning-experiment-80", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5463, - "hfopenllm_v2/BBH": 0.421, - "hfopenllm_v2/MATH Level 5": 0.0989, - "hfopenllm_v2/GPQA": 0.2844, - "hfopenllm_v2/MUSR": 0.5025, - "hfopenllm_v2/MMLU-PRO": 0.3368 - } - }, - { - "id": "sometimesanotion/KytheraMix-7B-v0.2", - "name": "KytheraMix-7B-v0.2", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6129, - "hfopenllm_v2/BBH": 0.5635, - "hfopenllm_v2/MATH Level 5": 0.2923, - "hfopenllm_v2/GPQA": 0.3356, - "hfopenllm_v2/MUSR": 0.4594, - "hfopenllm_v2/MMLU-PRO": 0.4505 - } - }, - { - "id": "sometimesanotion/lamarck-14b-prose-model_stock", - "name": "lamarck-14b-prose-model_stock", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4276, - "hfopenllm_v2/BBH": 0.6488, - "hfopenllm_v2/MATH Level 5": 0.3414, - "hfopenllm_v2/GPQA": 0.3935, - "hfopenllm_v2/MUSR": 0.4846, - "hfopenllm_v2/MMLU-PRO": 0.5354 - } - }, - { - "id": "sometimesanotion/lamarck-14b-reason-model_stock", - "name": "lamarck-14b-reason-model_stock", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4965, - "hfopenllm_v2/BBH": 0.6569, - "hfopenllm_v2/MATH Level 5": 0.358, - "hfopenllm_v2/GPQA": 0.3842, - "hfopenllm_v2/MUSR": 0.4741, - "hfopenllm_v2/MMLU-PRO": 0.5402 - } - }, - { - "id": "sometimesanotion/Lamarck-14B-v0.1-experimental", - "name": "Lamarck-14B-v0.1-experimental", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5354, - "hfopenllm_v2/BBH": 0.6583, - "hfopenllm_v2/MATH Level 5": 0.358, - "hfopenllm_v2/GPQA": 0.3817, - "hfopenllm_v2/MUSR": 0.4728, - "hfopenllm_v2/MMLU-PRO": 0.5408 - } - }, - { - "id": "sometimesanotion/Lamarck-14B-v0.3", - "name": "Lamarck-14B-v0.3", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5032, - "hfopenllm_v2/BBH": 0.6611, - "hfopenllm_v2/MATH Level 5": 0.3406, - "hfopenllm_v2/GPQA": 0.3884, - "hfopenllm_v2/MUSR": 0.4688, - "hfopenllm_v2/MMLU-PRO": 0.5411 - } - }, - { - "id": "sometimesanotion/Lamarck-14B-v0.4-Qwenvergence", - "name": "Lamarck-14B-v0.4-Qwenvergence", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4906, - "hfopenllm_v2/BBH": 0.6535, - "hfopenllm_v2/MATH Level 5": 0.3399, - "hfopenllm_v2/GPQA": 0.3784, - "hfopenllm_v2/MUSR": 0.4847, - "hfopenllm_v2/MMLU-PRO": 0.5406 - } - }, - { - "id": "sometimesanotion/Lamarck-14B-v0.6", - "name": "Lamarck-14B-v0.6", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6973, - "hfopenllm_v2/BBH": 0.646, - "hfopenllm_v2/MATH Level 5": 0.4041, - "hfopenllm_v2/GPQA": 0.3893, - "hfopenllm_v2/MUSR": 0.4847, - "hfopenllm_v2/MMLU-PRO": 0.54 - } - }, - { - "id": "sometimesanotion/Lamarck-14B-v0.6-002-model_stock", - "name": "Lamarck-14B-v0.6-002-model_stock", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6692, - "hfopenllm_v2/BBH": 0.6143, - "hfopenllm_v2/MATH Level 5": 0.3776, - "hfopenllm_v2/GPQA": 0.3742, - "hfopenllm_v2/MUSR": 0.518, - "hfopenllm_v2/MMLU-PRO": 0.5054 - } - }, - { - "id": "sometimesanotion/Lamarck-14B-v0.6-model_stock", - "name": "Lamarck-14B-v0.6-model_stock", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.679, - "hfopenllm_v2/BBH": 0.6269, - "hfopenllm_v2/MATH Level 5": 0.4245, - "hfopenllm_v2/GPQA": 0.3842, - "hfopenllm_v2/MUSR": 0.5007, - "hfopenllm_v2/MMLU-PRO": 0.5198 - } - }, - { - "id": "sometimesanotion/Lamarck-14B-v0.7-Fusion", - "name": "Lamarck-14B-v0.7-Fusion", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6821, - "hfopenllm_v2/BBH": 0.6544, - "hfopenllm_v2/MATH Level 5": 0.4041, - "hfopenllm_v2/GPQA": 0.401, - "hfopenllm_v2/MUSR": 0.4991, - "hfopenllm_v2/MMLU-PRO": 0.5391 - } - }, - { - "id": "sometimesanotion/Lamarck-14B-v0.7-rc1", - "name": "Lamarck-14B-v0.7-rc1", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7305, - "hfopenllm_v2/BBH": 0.6486, - "hfopenllm_v2/MATH Level 5": 0.3852, - "hfopenllm_v2/GPQA": 0.3893, - "hfopenllm_v2/MUSR": 0.4715, - "hfopenllm_v2/MMLU-PRO": 0.5416 - } - }, - { - "id": "sometimesanotion/Lamarck-14B-v0.7-rc4", - "name": "Lamarck-14B-v0.7-rc4", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7211, - "hfopenllm_v2/BBH": 0.651, - "hfopenllm_v2/MATH Level 5": 0.4026, - "hfopenllm_v2/GPQA": 0.3893, - "hfopenllm_v2/MUSR": 0.4912, - "hfopenllm_v2/MMLU-PRO": 0.54 - } - }, - { - "id": "sometimesanotion/LamarckInfusion-14B-v1", - "name": "LamarckInfusion-14B-v1", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7198, - "hfopenllm_v2/BBH": 0.6539, - "hfopenllm_v2/MATH Level 5": 0.4169, - "hfopenllm_v2/GPQA": 0.3909, - "hfopenllm_v2/MUSR": 0.4899, - "hfopenllm_v2/MMLU-PRO": 0.5376 - } - }, - { - "id": "sometimesanotion/LamarckInfusion-14B-v2", - "name": "LamarckInfusion-14B-v2", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6812, - "hfopenllm_v2/BBH": 0.6564, - "hfopenllm_v2/MATH Level 5": 0.4388, - "hfopenllm_v2/GPQA": 0.3876, - "hfopenllm_v2/MUSR": 0.4993, - "hfopenllm_v2/MMLU-PRO": 0.5416 - } - }, - { - "id": "sometimesanotion/LamarckInfusion-14B-v2-hi", - "name": "LamarckInfusion-14B-v2-hi", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6855, - "hfopenllm_v2/BBH": 0.6555, - "hfopenllm_v2/MATH Level 5": 0.423, - "hfopenllm_v2/GPQA": 0.3884, - "hfopenllm_v2/MUSR": 0.4847, - "hfopenllm_v2/MMLU-PRO": 0.5405 - } - }, - { - "id": "sometimesanotion/LamarckInfusion-14B-v2-lo", - "name": "LamarckInfusion-14B-v2-lo", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6788, - "hfopenllm_v2/BBH": 0.6528, - "hfopenllm_v2/MATH Level 5": 0.4237, - "hfopenllm_v2/GPQA": 0.3859, - "hfopenllm_v2/MUSR": 0.4991, - "hfopenllm_v2/MMLU-PRO": 0.5397 - } - }, - { - "id": "sometimesanotion/LamarckInfusion-14B-v3", - "name": "LamarckInfusion-14B-v3", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7131, - "hfopenllm_v2/BBH": 0.6518, - "hfopenllm_v2/MATH Level 5": 0.4124, - "hfopenllm_v2/GPQA": 0.3867, - "hfopenllm_v2/MUSR": 0.482, - "hfopenllm_v2/MMLU-PRO": 0.5407 - } - }, - { - "id": "sometimesanotion/Qwen-14B-ProseStock-v4", - "name": "Qwen-14B-ProseStock-v4", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4942, - "hfopenllm_v2/BBH": 0.6498, - "hfopenllm_v2/MATH Level 5": 0.364, - "hfopenllm_v2/GPQA": 0.3884, - "hfopenllm_v2/MUSR": 0.4938, - "hfopenllm_v2/MMLU-PRO": 0.5386 - } - }, - { - "id": "sometimesanotion/Qwen-2.5-14B-Virmarckeoso", - "name": "Qwen-2.5-14B-Virmarckeoso", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4813, - "hfopenllm_v2/BBH": 0.657, - "hfopenllm_v2/MATH Level 5": 0.3565, - "hfopenllm_v2/GPQA": 0.3792, - "hfopenllm_v2/MUSR": 0.4794, - "hfopenllm_v2/MMLU-PRO": 0.5377 - } - }, - { - "id": "sometimesanotion/Qwen2.5-14B-Vimarckoso", - "name": "Qwen2.5-14B-Vimarckoso", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4574, - "hfopenllm_v2/BBH": 0.6446, - "hfopenllm_v2/MATH Level 5": 0.3384, - "hfopenllm_v2/GPQA": 0.3926, - "hfopenllm_v2/MUSR": 0.4859, - "hfopenllm_v2/MMLU-PRO": 0.5329 - } - }, - { - "id": "sometimesanotion/Qwen2.5-14B-Vimarckoso-v2", - "name": "Qwen2.5-14B-Vimarckoso-v2", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4505, - "hfopenllm_v2/BBH": 0.655, - "hfopenllm_v2/MATH Level 5": 0.358, - "hfopenllm_v2/GPQA": 0.3826, - "hfopenllm_v2/MUSR": 0.4819, - "hfopenllm_v2/MMLU-PRO": 0.538 - } - }, - { - "id": "sometimesanotion/Qwen2.5-14B-Vimarckoso-v3", - "name": "Qwen2.5-14B-Vimarckoso-v3", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7257, - "hfopenllm_v2/BBH": 0.6415, - "hfopenllm_v2/MATH Level 5": 0.4003, - "hfopenllm_v2/GPQA": 0.38, - "hfopenllm_v2/MUSR": 0.4807, - "hfopenllm_v2/MMLU-PRO": 0.5343 - } - }, - { - "id": "sometimesanotion/Qwen2.5-14B-Vimarckoso-v3-IF-Variant", - "name": "Qwen2.5-14B-Vimarckoso-v3-IF-Variant", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6413, - "hfopenllm_v2/BBH": 0.5521, - "hfopenllm_v2/MATH Level 5": 0.2545, - "hfopenllm_v2/GPQA": 0.3473, - "hfopenllm_v2/MUSR": 0.5319, - "hfopenllm_v2/MMLU-PRO": 0.4589 - } - }, - { - "id": "sometimesanotion/Qwen2.5-14B-Vimarckoso-v3-model_stock", - "name": "Qwen2.5-14B-Vimarckoso-v3-model_stock", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7162, - "hfopenllm_v2/BBH": 0.6421, - "hfopenllm_v2/MATH Level 5": 0.4245, - "hfopenllm_v2/GPQA": 0.38, - "hfopenllm_v2/MUSR": 0.4781, - "hfopenllm_v2/MMLU-PRO": 0.5316 - } - }, - { - "id": "sometimesanotion/Qwen2.5-14B-Vimarckoso-v3-Prose01", - "name": "Qwen2.5-14B-Vimarckoso-v3-Prose01", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6872, - "hfopenllm_v2/BBH": 0.6359, - "hfopenllm_v2/MATH Level 5": 0.3995, - "hfopenllm_v2/GPQA": 0.3867, - "hfopenllm_v2/MUSR": 0.4807, - "hfopenllm_v2/MMLU-PRO": 0.5275 - } - }, - { - "id": "sometimesanotion/Qwen2.5-7B-Gordion-v0.1", - "name": "Qwen2.5-7B-Gordion-v0.1", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7482, - "hfopenllm_v2/BBH": 0.5524, - "hfopenllm_v2/MATH Level 5": 0.2915, - "hfopenllm_v2/GPQA": 0.3079, - "hfopenllm_v2/MUSR": 0.4016, - "hfopenllm_v2/MMLU-PRO": 0.43 - } - }, - { - "id": "sometimesanotion/Qwen2.5-7B-Gordion-v0.1-Prose", - "name": "Qwen2.5-7B-Gordion-v0.1-Prose", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5347, - "hfopenllm_v2/BBH": 0.5599, - "hfopenllm_v2/MATH Level 5": 0.2893, - "hfopenllm_v2/GPQA": 0.3205, - "hfopenllm_v2/MUSR": 0.4502, - "hfopenllm_v2/MMLU-PRO": 0.4525 - } - }, - { - "id": "sometimesanotion/Qwen2.5-7B-Gordion-v0.1-Reason", - "name": "Qwen2.5-7B-Gordion-v0.1-Reason", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4917, - "hfopenllm_v2/BBH": 0.5498, - "hfopenllm_v2/MATH Level 5": 0.2621, - "hfopenllm_v2/GPQA": 0.3406, - "hfopenllm_v2/MUSR": 0.4434, - "hfopenllm_v2/MMLU-PRO": 0.4307 - } - }, - { - "id": "sometimesanotion/Qwentessential-14B-v1", - "name": "Qwentessential-14B-v1", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6279, - "hfopenllm_v2/BBH": 0.6545, - "hfopenllm_v2/MATH Level 5": 0.4071, - "hfopenllm_v2/GPQA": 0.3876, - "hfopenllm_v2/MUSR": 0.4873, - "hfopenllm_v2/MMLU-PRO": 0.5381 - } - }, - { - "id": "sometimesanotion/Qwentinuum-14B-v013", - "name": "Qwentinuum-14B-v013", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6711, - "hfopenllm_v2/BBH": 0.6087, - "hfopenllm_v2/MATH Level 5": 0.3708, - "hfopenllm_v2/GPQA": 0.3574, - "hfopenllm_v2/MUSR": 0.5154, - "hfopenllm_v2/MMLU-PRO": 0.4991 - } - }, - { - "id": "sometimesanotion/Qwentinuum-14B-v1", - "name": "Qwentinuum-14B-v1", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5032, - "hfopenllm_v2/BBH": 0.6573, - "hfopenllm_v2/MATH Level 5": 0.3603, - "hfopenllm_v2/GPQA": 0.3826, - "hfopenllm_v2/MUSR": 0.4781, - "hfopenllm_v2/MMLU-PRO": 0.541 - } - }, - { - "id": "sometimesanotion/Qwentinuum-14B-v2", - "name": "Qwentinuum-14B-v2", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5378, - "hfopenllm_v2/BBH": 0.6555, - "hfopenllm_v2/MATH Level 5": 0.3754, - "hfopenllm_v2/GPQA": 0.3884, - "hfopenllm_v2/MUSR": 0.4714, - "hfopenllm_v2/MMLU-PRO": 0.5409 - } - }, - { - "id": "sometimesanotion/Qwentinuum-14B-v3", - "name": "Qwentinuum-14B-v3", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6158, - "hfopenllm_v2/BBH": 0.6539, - "hfopenllm_v2/MATH Level 5": 0.3535, - "hfopenllm_v2/GPQA": 0.3876, - "hfopenllm_v2/MUSR": 0.486, - "hfopenllm_v2/MMLU-PRO": 0.5413 - } - }, - { - "id": "sometimesanotion/Qwentinuum-14B-v5", - "name": "Qwentinuum-14B-v5", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6286, - "hfopenllm_v2/BBH": 0.655, - "hfopenllm_v2/MATH Level 5": 0.3444, - "hfopenllm_v2/GPQA": 0.3876, - "hfopenllm_v2/MUSR": 0.4874, - "hfopenllm_v2/MMLU-PRO": 0.5418 - } - }, - { - "id": "sometimesanotion/Qwentinuum-14B-v6", - "name": "Qwentinuum-14B-v6", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6304, - "hfopenllm_v2/BBH": 0.6545, - "hfopenllm_v2/MATH Level 5": 0.3603, - "hfopenllm_v2/GPQA": 0.3867, - "hfopenllm_v2/MUSR": 0.49, - "hfopenllm_v2/MMLU-PRO": 0.54 - } - }, - { - "id": "sometimesanotion/Qwentinuum-14B-v6-Prose", - "name": "Qwentinuum-14B-v6-Prose", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5643, - "hfopenllm_v2/BBH": 0.6545, - "hfopenllm_v2/MATH Level 5": 0.3701, - "hfopenllm_v2/GPQA": 0.3884, - "hfopenllm_v2/MUSR": 0.4913, - "hfopenllm_v2/MMLU-PRO": 0.5392 - } - }, - { - "id": "sometimesanotion/Qwentinuum-14B-v7", - "name": "Qwentinuum-14B-v7", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6109, - "hfopenllm_v2/BBH": 0.6551, - "hfopenllm_v2/MATH Level 5": 0.3573, - "hfopenllm_v2/GPQA": 0.3909, - "hfopenllm_v2/MUSR": 0.482, - "hfopenllm_v2/MMLU-PRO": 0.541 - } - }, - { - "id": "sometimesanotion/Qwentinuum-14B-v8", - "name": "Qwentinuum-14B-v8", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5412, - "hfopenllm_v2/BBH": 0.6534, - "hfopenllm_v2/MATH Level 5": 0.3912, - "hfopenllm_v2/GPQA": 0.3834, - "hfopenllm_v2/MUSR": 0.4873, - "hfopenllm_v2/MMLU-PRO": 0.5412 - } - }, - { - "id": "sometimesanotion/Qwentinuum-14B-v9", - "name": "Qwentinuum-14B-v9", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5107, - "hfopenllm_v2/BBH": 0.658, - "hfopenllm_v2/MATH Level 5": 0.3482, - "hfopenllm_v2/GPQA": 0.3859, - "hfopenllm_v2/MUSR": 0.4781, - "hfopenllm_v2/MMLU-PRO": 0.5421 - } - }, - { - "id": "sometimesanotion/Qwenvergence-14B-qv256", - "name": "Qwenvergence-14B-qv256", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7006, - "hfopenllm_v2/BBH": 0.6312, - "hfopenllm_v2/MATH Level 5": 0.3897, - "hfopenllm_v2/GPQA": 0.3784, - "hfopenllm_v2/MUSR": 0.4926, - "hfopenllm_v2/MMLU-PRO": 0.5178 - } - }, - { - "id": "sometimesanotion/Qwenvergence-14B-v0.6-004-model_stock", - "name": "Qwenvergence-14B-v0.6-004-model_stock", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.686, - "hfopenllm_v2/BBH": 0.6249, - "hfopenllm_v2/MATH Level 5": 0.4094, - "hfopenllm_v2/GPQA": 0.3834, - "hfopenllm_v2/MUSR": 0.5033, - "hfopenllm_v2/MMLU-PRO": 0.5193 - } - }, - { - "id": "sometimesanotion/Qwenvergence-14B-v10", - "name": "Qwenvergence-14B-v10", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6757, - "hfopenllm_v2/BBH": 0.6316, - "hfopenllm_v2/MATH Level 5": 0.4789, - "hfopenllm_v2/GPQA": 0.3792, - "hfopenllm_v2/MUSR": 0.4991, - "hfopenllm_v2/MMLU-PRO": 0.5239 - } - }, - { - "id": "sometimesanotion/Qwenvergence-14B-v11", - "name": "Qwenvergence-14B-v11", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7192, - "hfopenllm_v2/BBH": 0.6368, - "hfopenllm_v2/MATH Level 5": 0.4645, - "hfopenllm_v2/GPQA": 0.3725, - "hfopenllm_v2/MUSR": 0.4754, - "hfopenllm_v2/MMLU-PRO": 0.5327 - } - }, - { - "id": "sometimesanotion/Qwenvergence-14B-v12-Prose", - "name": "Qwenvergence-14B-v12-Prose", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5412, - "hfopenllm_v2/BBH": 0.6504, - "hfopenllm_v2/MATH Level 5": 0.3535, - "hfopenllm_v2/GPQA": 0.3867, - "hfopenllm_v2/MUSR": 0.4991, - "hfopenllm_v2/MMLU-PRO": 0.5381 - } - }, - { - "id": "sometimesanotion/Qwenvergence-14B-v12-Prose-DS", - "name": "Qwenvergence-14B-v12-Prose-DS", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6173, - "hfopenllm_v2/BBH": 0.6507, - "hfopenllm_v2/MATH Level 5": 0.4305, - "hfopenllm_v2/GPQA": 0.3943, - "hfopenllm_v2/MUSR": 0.5151, - "hfopenllm_v2/MMLU-PRO": 0.5369 - } - }, - { - "id": "sometimesanotion/Qwenvergence-14B-v13-Prose-DS", - "name": "Qwenvergence-14B-v13-Prose-DS", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7178, - "hfopenllm_v2/BBH": 0.6405, - "hfopenllm_v2/MATH Level 5": 0.386, - "hfopenllm_v2/GPQA": 0.3834, - "hfopenllm_v2/MUSR": 0.4927, - "hfopenllm_v2/MMLU-PRO": 0.5349 - } - }, - { - "id": "sometimesanotion/Qwenvergence-14B-v15-Prose-MS", - "name": "Qwenvergence-14B-v15-Prose-MS", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5032, - "hfopenllm_v2/BBH": 0.655, - "hfopenllm_v2/MATH Level 5": 0.3633, - "hfopenllm_v2/GPQA": 0.3951, - "hfopenllm_v2/MUSR": 0.4913, - "hfopenllm_v2/MMLU-PRO": 0.5393 - } - }, - { - "id": "sometimesanotion/Qwenvergence-14B-v2-Prose", - "name": "Qwenvergence-14B-v2-Prose", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4705, - "hfopenllm_v2/BBH": 0.6519, - "hfopenllm_v2/MATH Level 5": 0.3557, - "hfopenllm_v2/GPQA": 0.3935, - "hfopenllm_v2/MUSR": 0.4926, - "hfopenllm_v2/MMLU-PRO": 0.5372 - } - }, - { - "id": "sometimesanotion/Qwenvergence-14B-v3", - "name": "Qwenvergence-14B-v3", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5044, - "hfopenllm_v2/BBH": 0.6548, - "hfopenllm_v2/MATH Level 5": 0.3693, - "hfopenllm_v2/GPQA": 0.3842, - "hfopenllm_v2/MUSR": 0.4886, - "hfopenllm_v2/MMLU-PRO": 0.5386 - } - }, - { - "id": "sometimesanotion/Qwenvergence-14B-v3-Prose", - "name": "Qwenvergence-14B-v3-Prose", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4918, - "hfopenllm_v2/BBH": 0.6513, - "hfopenllm_v2/MATH Level 5": 0.3648, - "hfopenllm_v2/GPQA": 0.3951, - "hfopenllm_v2/MUSR": 0.4939, - "hfopenllm_v2/MMLU-PRO": 0.537 - } - }, - { - "id": "sometimesanotion/Qwenvergence-14B-v3-Reason", - "name": "Qwenvergence-14B-v3-Reason", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5278, - "hfopenllm_v2/BBH": 0.6557, - "hfopenllm_v2/MATH Level 5": 0.3119, - "hfopenllm_v2/GPQA": 0.3842, - "hfopenllm_v2/MUSR": 0.4754, - "hfopenllm_v2/MMLU-PRO": 0.5396 - } - }, - { - "id": "sometimesanotion/Qwenvergence-14B-v6-Prose", - "name": "Qwenvergence-14B-v6-Prose", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.599, - "hfopenllm_v2/BBH": 0.6544, - "hfopenllm_v2/MATH Level 5": 0.3565, - "hfopenllm_v2/GPQA": 0.3884, - "hfopenllm_v2/MUSR": 0.4887, - "hfopenllm_v2/MMLU-PRO": 0.5371 - } - }, - { - "id": "sometimesanotion/Qwenvergence-14B-v6-Prose-model_stock", - "name": "Qwenvergence-14B-v6-Prose-model_stock", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4811, - "hfopenllm_v2/BBH": 0.653, - "hfopenllm_v2/MATH Level 5": 0.3603, - "hfopenllm_v2/GPQA": 0.3935, - "hfopenllm_v2/MUSR": 0.4899, - "hfopenllm_v2/MMLU-PRO": 0.5387 - } - }, - { - "id": "sometimesanotion/Qwenvergence-14B-v8", - "name": "Qwenvergence-14B-v8", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5913, - "hfopenllm_v2/BBH": 0.6522, - "hfopenllm_v2/MATH Level 5": 0.4048, - "hfopenllm_v2/GPQA": 0.3809, - "hfopenllm_v2/MUSR": 0.4768, - "hfopenllm_v2/MMLU-PRO": 0.5435 - } - }, - { - "id": "sometimesanotion/Qwenvergence-14B-v9", - "name": "Qwenvergence-14B-v9", - "developer": "sometimesanotion", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6598, - "hfopenllm_v2/BBH": 0.6166, - "hfopenllm_v2/MATH Level 5": 0.4139, - "hfopenllm_v2/GPQA": 0.3683, - "hfopenllm_v2/MUSR": 0.5141, - "hfopenllm_v2/MMLU-PRO": 0.5111 - } - }, - { - "id": "sonthenguyen/ft-unsloth-zephyr-sft-bnb-4bit-20241014-161415", - "name": "ft-unsloth-zephyr-sft-bnb-4bit-20241014-161415", - "developer": "sonthenguyen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2893, - "hfopenllm_v2/BBH": 0.3804, - "hfopenllm_v2/MATH Level 5": 0.0113, - "hfopenllm_v2/GPQA": 0.2466, - "hfopenllm_v2/MUSR": 0.3861, - "hfopenllm_v2/MMLU-PRO": 0.1401 - } - }, - { - "id": "sonthenguyen/ft-unsloth-zephyr-sft-bnb-4bit-20241014-164205", - "name": "ft-unsloth-zephyr-sft-bnb-4bit-20241014-164205", - "developer": "sonthenguyen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3199, - "hfopenllm_v2/BBH": 0.3959, - "hfopenllm_v2/MATH Level 5": 0.0083, - "hfopenllm_v2/GPQA": 0.276, - "hfopenllm_v2/MUSR": 0.4272, - "hfopenllm_v2/MMLU-PRO": 0.2124 - } - }, - { - "id": "sonthenguyen/ft-unsloth-zephyr-sft-bnb-4bit-20241014-170522", - "name": "ft-unsloth-zephyr-sft-bnb-4bit-20241014-170522", - "developer": "sonthenguyen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3764, - "hfopenllm_v2/BBH": 0.3828, - "hfopenllm_v2/MATH Level 5": 0.0091, - "hfopenllm_v2/GPQA": 0.2651, - "hfopenllm_v2/MUSR": 0.4404, - "hfopenllm_v2/MMLU-PRO": 0.2055 - } - }, - { - "id": "sonthenguyen/zephyr-sft-bnb-4bit-DPO-mtbc-213steps", - "name": "zephyr-sft-bnb-4bit-DPO-mtbc-213steps", - "developer": "sonthenguyen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4275, - "hfopenllm_v2/BBH": 0.4197, - "hfopenllm_v2/MATH Level 5": 0.0257, - "hfopenllm_v2/GPQA": 0.2617, - "hfopenllm_v2/MUSR": 0.4086, - "hfopenllm_v2/MMLU-PRO": 0.2709 - } - }, - { - "id": "sonthenguyen/zephyr-sft-bnb-4bit-DPO-mtbo-180steps", - "name": "zephyr-sft-bnb-4bit-DPO-mtbo-180steps", - "developer": "sonthenguyen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4087, - "hfopenllm_v2/BBH": 0.4323, - "hfopenllm_v2/MATH Level 5": 0.0234, - "hfopenllm_v2/GPQA": 0.276, - "hfopenllm_v2/MUSR": 0.3885, - "hfopenllm_v2/MMLU-PRO": 0.2748 - } - }, - { - "id": "sonthenguyen/zephyr-sft-bnb-4bit-DPO-mtbr-180steps", - "name": "zephyr-sft-bnb-4bit-DPO-mtbr-180steps", - "developer": "sonthenguyen", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4032, - "hfopenllm_v2/BBH": 0.4305, - "hfopenllm_v2/MATH Level 5": 0.0249, - "hfopenllm_v2/GPQA": 0.2802, - "hfopenllm_v2/MUSR": 0.4258, - "hfopenllm_v2/MMLU-PRO": 0.2711 - } - }, - { - "id": "sophosympatheia/Midnight-Miqu-70B-v1.5", - "name": "Midnight-Miqu-70B-v1.5", - "developer": "sophosympatheia", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6118, - "hfopenllm_v2/BBH": 0.5606, - "hfopenllm_v2/MATH Level 5": 0.0702, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.4244, - "hfopenllm_v2/MMLU-PRO": 0.3825 - } - }, - { - "id": "Sorawiz/Gemma-9B-Base", - "name": "Gemma-9B-Base", - "developer": "Sorawiz", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1667, - "hfopenllm_v2/BBH": 0.593, - "hfopenllm_v2/MATH Level 5": 0.0982, - "hfopenllm_v2/GPQA": 0.3398, - "hfopenllm_v2/MUSR": 0.4045, - "hfopenllm_v2/MMLU-PRO": 0.4235 - } - }, - { - "id": "Sorawiz/Gemma-Creative-9B-Base", - "name": "Gemma-Creative-9B-Base", - "developer": "Sorawiz", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1515, - "hfopenllm_v2/BBH": 0.5459, - "hfopenllm_v2/MATH Level 5": 0.0778, - "hfopenllm_v2/GPQA": 0.3297, - "hfopenllm_v2/MUSR": 0.4019, - "hfopenllm_v2/MMLU-PRO": 0.4008 - } - }, - { - "id": "Sourjayon/DeepSeek-R1-8b-Sify", - "name": "DeepSeek-R1-8b-Sify", - "developer": "Sourjayon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3679, - "hfopenllm_v2/BBH": 0.3379, - "hfopenllm_v2/MATH Level 5": 0.2447, - "hfopenllm_v2/GPQA": 0.2525, - "hfopenllm_v2/MUSR": 0.3303, - "hfopenllm_v2/MMLU-PRO": 0.1981 - } - }, - { - "id": "Sourjayon/DeepSeek-R1-ForumNXT", - "name": "DeepSeek-R1-ForumNXT", - "developer": "Sourjayon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2603, - "hfopenllm_v2/BBH": 0.331, - "hfopenllm_v2/MATH Level 5": 0.2576, - "hfopenllm_v2/GPQA": 0.2743, - "hfopenllm_v2/MUSR": 0.3392, - "hfopenllm_v2/MMLU-PRO": 0.1648 - } - }, - { - "id": "SpaceYL/ECE_Poirot", - "name": "ECE_Poirot", - "developer": "SpaceYL", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3107, - "hfopenllm_v2/BBH": 0.4262, - "hfopenllm_v2/MATH Level 5": 0.0914, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.4026, - "hfopenllm_v2/MMLU-PRO": 0.2883 - } - }, - { - "id": "speakleash-ack-cyfronet-agh/bielik-11b-v2-3-instruct-prompt", - "name": "Bielik-11B-v2.3-Instruct (Prompt)", - "developer": "speakleash-ack-cyfronet-agh", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 99.0, - "bfcl/bfcl.overall.overall_accuracy": 21.9, - "bfcl/bfcl.overall.total_cost_usd": 22.44, - "bfcl/bfcl.overall.latency_mean_s": 23.75, - "bfcl/bfcl.overall.latency_std_s": 61.76, - "bfcl/bfcl.overall.latency_p95_s": 72.8, - "bfcl/bfcl.non_live.ast_accuracy": 81.5, - "bfcl/bfcl.non_live.simple_ast_accuracy": 73.0, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 92.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 85.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 75.5, - "bfcl/bfcl.live.live_accuracy": 67.8, - "bfcl/bfcl.live.live_simple_ast_accuracy": 75.58, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 66.19, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 62.5, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 58.33, - "bfcl/bfcl.multi_turn.accuracy": 2.62, - "bfcl/bfcl.multi_turn.base_accuracy": 4.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 0.5, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 3.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 2.5, - "bfcl/bfcl.web_search.accuracy": 1.5, - "bfcl/bfcl.web_search.base_accuracy": 1.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 2.0, - "bfcl/bfcl.memory.accuracy": 11.4, - "bfcl/bfcl.memory.kv_accuracy": 7.1, - "bfcl/bfcl.memory.vector_accuracy": 4.52, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 22.58, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 93.75, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 36.01, - "bfcl/bfcl.format_sensitivity.max_delta": 35.0, - "bfcl/bfcl.format_sensitivity.stddev": 9.74 - } - }, - { - "id": "speakleash/Bielik-11B-v2", - "name": "Bielik-11B-v2", - "developer": "speakleash", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2381, - "hfopenllm_v2/BBH": 0.4931, - "hfopenllm_v2/MATH Level 5": 0.0785, - "hfopenllm_v2/GPQA": 0.2886, - "hfopenllm_v2/MUSR": 0.3924, - "hfopenllm_v2/MMLU-PRO": 0.3137 - } - }, - { - "id": "speakleash/Bielik-11B-v2.0-Instruct", - "name": "Bielik-11B-v2.0-Instruct", - "developer": "speakleash", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5252, - "hfopenllm_v2/BBH": 0.5362, - "hfopenllm_v2/MATH Level 5": 0.1186, - "hfopenllm_v2/GPQA": 0.3171, - "hfopenllm_v2/MUSR": 0.4467, - "hfopenllm_v2/MMLU-PRO": 0.3351 - } - }, - { - "id": "speakleash/Bielik-11B-v2.1-Instruct", - "name": "Bielik-11B-v2.1-Instruct", - "developer": "speakleash", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.509, - "hfopenllm_v2/BBH": 0.553, - "hfopenllm_v2/MATH Level 5": 0.2666, - "hfopenllm_v2/GPQA": 0.3372, - "hfopenllm_v2/MUSR": 0.4185, - "hfopenllm_v2/MMLU-PRO": 0.3447 - } - }, - { - "id": "speakleash/Bielik-11B-v2.2-Instruct", - "name": "Bielik-11B-v2.2-Instruct", - "developer": "speakleash", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5552, - "hfopenllm_v2/BBH": 0.5597, - "hfopenllm_v2/MATH Level 5": 0.2681, - "hfopenllm_v2/GPQA": 0.3314, - "hfopenllm_v2/MUSR": 0.4171, - "hfopenllm_v2/MMLU-PRO": 0.3487 - } - }, - { - "id": "speakleash/Bielik-11B-v2.3-Instruct", - "name": "Bielik-11B-v2.3-Instruct", - "developer": "speakleash", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5583, - "hfopenllm_v2/BBH": 0.5663, - "hfopenllm_v2/MATH Level 5": 0.2085, - "hfopenllm_v2/GPQA": 0.3406, - "hfopenllm_v2/MUSR": 0.4518, - "hfopenllm_v2/MMLU-PRO": 0.3444 - } - }, - { - "id": "Spestly/Athena-1-3B", - "name": "Athena-1-3B", - "developer": "Spestly", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5569, - "hfopenllm_v2/BBH": 0.4702, - "hfopenllm_v2/MATH Level 5": 0.2379, - "hfopenllm_v2/GPQA": 0.2936, - "hfopenllm_v2/MUSR": 0.4362, - "hfopenllm_v2/MMLU-PRO": 0.3519 - } - }, - { - "id": "Spestly/Atlas-Pro-1.5B-Preview", - "name": "Atlas-Pro-1.5B-Preview", - "developer": "Spestly", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.243, - "hfopenllm_v2/BBH": 0.3499, - "hfopenllm_v2/MATH Level 5": 0.3195, - "hfopenllm_v2/GPQA": 0.297, - "hfopenllm_v2/MUSR": 0.3354, - "hfopenllm_v2/MMLU-PRO": 0.1925 - } - }, - { - "id": "Spestly/Atlas-Pro-7B-Preview", - "name": "Atlas-Pro-7B-Preview", - "developer": "Spestly", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3154, - "hfopenllm_v2/BBH": 0.4668, - "hfopenllm_v2/MATH Level 5": 0.5083, - "hfopenllm_v2/GPQA": 0.3372, - "hfopenllm_v2/MUSR": 0.3911, - "hfopenllm_v2/MMLU-PRO": 0.297 - } - }, - { - "id": "spmurrayzzz/Mistral-Syndicate-7B", - "name": "Mistral-Syndicate-7B", - "developer": "spmurrayzzz", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2496, - "hfopenllm_v2/BBH": 0.4245, - "hfopenllm_v2/MATH Level 5": 0.034, - "hfopenllm_v2/GPQA": 0.276, - "hfopenllm_v2/MUSR": 0.4386, - "hfopenllm_v2/MMLU-PRO": 0.2631 - } - }, - { - "id": "spow12/ChatWaifu_12B_v2.0", - "name": "ChatWaifu_12B_v2.0", - "developer": "spow12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4768, - "hfopenllm_v2/BBH": 0.5208, - "hfopenllm_v2/MATH Level 5": 0.071, - "hfopenllm_v2/GPQA": 0.2768, - "hfopenllm_v2/MUSR": 0.4432, - "hfopenllm_v2/MMLU-PRO": 0.3388 - } - }, - { - "id": "spow12/ChatWaifu_22B_v2.0_preview", - "name": "ChatWaifu_22B_v2.0_preview", - "developer": "spow12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6745, - "hfopenllm_v2/BBH": 0.617, - "hfopenllm_v2/MATH Level 5": 0.1888, - "hfopenllm_v2/GPQA": 0.3154, - "hfopenllm_v2/MUSR": 0.3685, - "hfopenllm_v2/MMLU-PRO": 0.3988 - } - }, - { - "id": "spow12/ChatWaifu_v1.4", - "name": "ChatWaifu_v1.4", - "developer": "spow12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5691, - "hfopenllm_v2/BBH": 0.5176, - "hfopenllm_v2/MATH Level 5": 0.1057, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.4743, - "hfopenllm_v2/MMLU-PRO": 0.3475 - } - }, - { - "id": "spow12/ChatWaifu_v2.0_22B", - "name": "ChatWaifu_v2.0_22B", - "developer": "spow12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6511, - "hfopenllm_v2/BBH": 0.5926, - "hfopenllm_v2/MATH Level 5": 0.1858, - "hfopenllm_v2/GPQA": 0.3247, - "hfopenllm_v2/MUSR": 0.3842, - "hfopenllm_v2/MMLU-PRO": 0.3836 - } - }, - { - "id": "ssmits/Qwen2.5-95B-Instruct", - "name": "Qwen2.5-95B-Instruct", - "developer": "ssmits", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8431, - "hfopenllm_v2/BBH": 0.7038, - "hfopenllm_v2/MATH Level 5": 0.5302, - "hfopenllm_v2/GPQA": 0.3641, - "hfopenllm_v2/MUSR": 0.4284, - "hfopenllm_v2/MMLU-PRO": 0.5217 - } - }, - { - "id": "stabilityai/stable-code-instruct-3b", - "name": "stabilityai/stable-code-instruct-3b", - "developer": "stabilityai", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6216, - "reward-bench/Chat": 0.5782, - "reward-bench/Chat Hard": 0.5855, - "reward-bench/Safety": 0.6554, - "reward-bench/Reasoning": 0.7528, - "reward-bench/Prior Sets (0.5 weight)": 0.4506 - } - }, - { - "id": "stabilityai/StableBeluga2", - "name": "StableBeluga2", - "developer": "stabilityai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3787, - "hfopenllm_v2/BBH": 0.5824, - "hfopenllm_v2/MATH Level 5": 0.0438, - "hfopenllm_v2/GPQA": 0.3163, - "hfopenllm_v2/MUSR": 0.473, - "hfopenllm_v2/MMLU-PRO": 0.3326 - } - }, - { - "id": "stabilityai/stablelm-2-12b", - "name": "stablelm-2-12b", - "developer": "stabilityai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1569, - "hfopenllm_v2/BBH": 0.4509, - "hfopenllm_v2/MATH Level 5": 0.0431, - "hfopenllm_v2/GPQA": 0.2785, - "hfopenllm_v2/MUSR": 0.4479, - "hfopenllm_v2/MMLU-PRO": 0.3072 - } - }, - { - "id": "stabilityai/stablelm-2-12b-chat", - "name": "stabilityai/stablelm-2-12b-chat", - "developer": "stabilityai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4082, - "hfopenllm_v2/BBH": 0.4672, - "hfopenllm_v2/MATH Level 5": 0.0536, - "hfopenllm_v2/GPQA": 0.2668, - "hfopenllm_v2/MUSR": 0.3914, - "hfopenllm_v2/MMLU-PRO": 0.2734, - "reward-bench/Score": 0.7642, - "reward-bench/Chat": 0.9665, - "reward-bench/Chat Hard": 0.5548, - "reward-bench/Safety": 0.7811, - "reward-bench/Reasoning": 0.8945, - "reward-bench/Prior Sets (0.5 weight)": 0.4839 - } - }, - { - "id": "stabilityai/stablelm-2-1_6b", - "name": "stablelm-2-1_6b", - "developer": "stabilityai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1157, - "hfopenllm_v2/BBH": 0.3385, - "hfopenllm_v2/MATH Level 5": 0.0076, - "hfopenllm_v2/GPQA": 0.2483, - "hfopenllm_v2/MUSR": 0.3882, - "hfopenllm_v2/MMLU-PRO": 0.1464 - } - }, - { - "id": "stabilityai/stablelm-2-1_6b-chat", - "name": "stablelm-2-1_6b-chat", - "developer": "stabilityai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.306, - "hfopenllm_v2/BBH": 0.339, - "hfopenllm_v2/MATH Level 5": 0.0249, - "hfopenllm_v2/GPQA": 0.2475, - "hfopenllm_v2/MUSR": 0.358, - "hfopenllm_v2/MMLU-PRO": 0.1622 - } - }, - { - "id": "stabilityai/stablelm-2-zephyr-1_6b", - "name": "stabilityai/stablelm-2-zephyr-1_6b", - "developer": "stabilityai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3279, - "hfopenllm_v2/BBH": 0.3352, - "hfopenllm_v2/MATH Level 5": 0.0332, - "hfopenllm_v2/GPQA": 0.2433, - "hfopenllm_v2/MUSR": 0.3511, - "hfopenllm_v2/MMLU-PRO": 0.1714, - "reward-bench/Score": 0.6574, - "reward-bench/Chat": 0.9665, - "reward-bench/Chat Hard": 0.4671, - "reward-bench/Safety": 0.6027, - "reward-bench/Reasoning": 0.6784, - "reward-bench/Prior Sets (0.5 weight)": 0.4868 - } - }, - { - "id": "stabilityai/stablelm-3b-4e1t", - "name": "stablelm-3b-4e1t", - "developer": "stabilityai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2203, - "hfopenllm_v2/BBH": 0.3504, - "hfopenllm_v2/MATH Level 5": 0.0106, - "hfopenllm_v2/GPQA": 0.2374, - "hfopenllm_v2/MUSR": 0.3778, - "hfopenllm_v2/MMLU-PRO": 0.1669 - } - }, - { - "id": "stabilityai/stablelm-zephyr-3b", - "name": "stabilityai/stablelm-zephyr-3b", - "developer": "stabilityai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3683, - "hfopenllm_v2/BBH": 0.3866, - "hfopenllm_v2/MATH Level 5": 0.0431, - "hfopenllm_v2/GPQA": 0.2391, - "hfopenllm_v2/MUSR": 0.4183, - "hfopenllm_v2/MMLU-PRO": 0.1768, - "reward-bench/Score": 0.7146, - "reward-bench/Chat": 0.8631, - "reward-bench/Chat Hard": 0.6009, - "reward-bench/Safety": 0.7405, - "reward-bench/Reasoning": 0.7573, - "reward-bench/Prior Sets (0.5 weight)": 0.5075 - } - }, - { - "id": "stanford/Alpaca-7B", - "name": "Alpaca 7B", - "developer": "stanford", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.381, - "helm_classic/MMLU": 0.385, - "helm_classic/BoolQ": 0.778, - "helm_classic/NarrativeQA": 0.396, - "helm_classic/NaturalQuestions (open-book)": 0.592, - "helm_classic/QuAC": 0.27, - "helm_classic/HellaSwag": -1.0, - "helm_classic/OpenbookQA": -1.0, - "helm_classic/TruthfulQA": 0.243, - "helm_classic/MS MARCO (TREC)": -1.0, - "helm_classic/CNN/DailyMail": -1.0, - "helm_classic/XSUM": -1.0, - "helm_classic/IMDB": 0.738, - "helm_classic/CivilComments": 0.566, - "helm_classic/RAFT": 0.486 - } - }, - { - "id": "stanfordnlp/SteamSHP-flan-t5-large", - "name": "stanfordnlp/SteamSHP-flan-t5-large", - "developer": "stanfordnlp", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.4962, - "reward-bench/Chat": 0.8575, - "reward-bench/Chat Hard": 0.3311, - "reward-bench/Safety": 0.3743, - "reward-bench/Reasoning": 0.3563, - "reward-bench/Prior Sets (0.5 weight)": 0.6273 - } - }, - { - "id": "stanfordnlp/SteamSHP-flan-t5-xl", - "name": "stanfordnlp/SteamSHP-flan-t5-xl", - "developer": "stanfordnlp", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5135, - "reward-bench/Chat": 0.8547, - "reward-bench/Chat Hard": 0.3684, - "reward-bench/Safety": 0.3784, - "reward-bench/Reasoning": 0.3841, - "reward-bench/Prior Sets (0.5 weight)": 0.6498 - } - }, - { - "id": "Stark2008/GutenLaserPi", - "name": "GutenLaserPi", - "developer": "Stark2008", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4227, - "hfopenllm_v2/BBH": 0.5212, - "hfopenllm_v2/MATH Level 5": 0.0785, - "hfopenllm_v2/GPQA": 0.2869, - "hfopenllm_v2/MUSR": 0.462, - "hfopenllm_v2/MMLU-PRO": 0.3106 - } - }, - { - "id": "Stark2008/LayleleFlamPi", - "name": "LayleleFlamPi", - "developer": "Stark2008", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4284, - "hfopenllm_v2/BBH": 0.5116, - "hfopenllm_v2/MATH Level 5": 0.0665, - "hfopenllm_v2/GPQA": 0.2852, - "hfopenllm_v2/MUSR": 0.4608, - "hfopenllm_v2/MMLU-PRO": 0.3093 - } - }, - { - "id": "Stark2008/VisFlamCat", - "name": "VisFlamCat", - "developer": "Stark2008", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4366, - "hfopenllm_v2/BBH": 0.5217, - "hfopenllm_v2/MATH Level 5": 0.0763, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.4463, - "hfopenllm_v2/MMLU-PRO": 0.3144 - } - }, - { - "id": "Steelskull/L3.3-MS-Nevoria-70b", - "name": "L3.3-MS-Nevoria-70b", - "developer": "Steelskull", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6963, - "hfopenllm_v2/BBH": 0.6998, - "hfopenllm_v2/MATH Level 5": 0.3958, - "hfopenllm_v2/GPQA": 0.4706, - "hfopenllm_v2/MUSR": 0.4682, - "hfopenllm_v2/MMLU-PRO": 0.5535 - } - }, - { - "id": "Steelskull/L3.3-Nevoria-R1-70b", - "name": "L3.3-Nevoria-R1-70b", - "developer": "Steelskull", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6024, - "hfopenllm_v2/BBH": 0.6972, - "hfopenllm_v2/MATH Level 5": 0.463, - "hfopenllm_v2/GPQA": 0.469, - "hfopenllm_v2/MUSR": 0.4775, - "hfopenllm_v2/MMLU-PRO": 0.5463 - } - }, - { - "id": "StelleX/Qwen2.5_Math_7B_Cot", - "name": "Qwen2.5_Math_7B_Cot", - "developer": "StelleX", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2143, - "hfopenllm_v2/BBH": 0.4313, - "hfopenllm_v2/MATH Level 5": 0.3263, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.3924, - "hfopenllm_v2/MMLU-PRO": 0.281 - } - }, - { - "id": "StelleX/Vorisatex-7B-preview", - "name": "Vorisatex-7B-preview", - "developer": "StelleX", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1515, - "hfopenllm_v2/BBH": 0.3112, - "hfopenllm_v2/MATH Level 5": 0.0287, - "hfopenllm_v2/GPQA": 0.2517, - "hfopenllm_v2/MUSR": 0.4192, - "hfopenllm_v2/MMLU-PRO": 0.1166 - } - }, - { - "id": "sthenno-com/miscii-14b-0130", - "name": "miscii-14b-0130", - "developer": "sthenno-com", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6647, - "hfopenllm_v2/BBH": 0.6505, - "hfopenllm_v2/MATH Level 5": 0.432, - "hfopenllm_v2/GPQA": 0.3817, - "hfopenllm_v2/MUSR": 0.4912, - "hfopenllm_v2/MMLU-PRO": 0.5363 - } - }, - { - "id": "sthenno-com/miscii-14b-0218", - "name": "miscii-14b-0218", - "developer": "sthenno-com", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7656, - "hfopenllm_v2/BBH": 0.6559, - "hfopenllm_v2/MATH Level 5": 0.5144, - "hfopenllm_v2/GPQA": 0.3834, - "hfopenllm_v2/MUSR": 0.4273, - "hfopenllm_v2/MMLU-PRO": 0.5298 - } - }, - { - "id": "sthenno-com/miscii-14b-1028", - "name": "miscii-14b-1028", - "developer": "sthenno-com", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8237, - "hfopenllm_v2/BBH": 0.6448, - "hfopenllm_v2/MATH Level 5": 0.503, - "hfopenllm_v2/GPQA": 0.3565, - "hfopenllm_v2/MUSR": 0.4182, - "hfopenllm_v2/MMLU-PRO": 0.5153 - } - }, - { - "id": "sthenno-com/miscii-14b-1225", - "name": "miscii-14b-1225", - "developer": "sthenno-com", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7878, - "hfopenllm_v2/BBH": 0.6572, - "hfopenllm_v2/MATH Level 5": 0.4517, - "hfopenllm_v2/GPQA": 0.3775, - "hfopenllm_v2/MUSR": 0.4366, - "hfopenllm_v2/MMLU-PRO": 0.5272 - } - }, - { - "id": "sthenno/tempesthenno-0120", - "name": "tempesthenno-0120", - "developer": "sthenno", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.539, - "hfopenllm_v2/BBH": 0.6373, - "hfopenllm_v2/MATH Level 5": 0.3353, - "hfopenllm_v2/GPQA": 0.3943, - "hfopenllm_v2/MUSR": 0.4633, - "hfopenllm_v2/MMLU-PRO": 0.529 - } - }, - { - "id": "sthenno/tempesthenno-fusion-0309", - "name": "tempesthenno-fusion-0309", - "developer": "sthenno", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7692, - "hfopenllm_v2/BBH": 0.6581, - "hfopenllm_v2/MATH Level 5": 0.4766, - "hfopenllm_v2/GPQA": 0.37, - "hfopenllm_v2/MUSR": 0.4325, - "hfopenllm_v2/MMLU-PRO": 0.5258 - } - }, - { - "id": "sthenno/tempesthenno-kto-0205-ckpt80", - "name": "tempesthenno-kto-0205-ckpt80", - "developer": "sthenno", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8054, - "hfopenllm_v2/BBH": 0.6543, - "hfopenllm_v2/MATH Level 5": 0.4592, - "hfopenllm_v2/GPQA": 0.3482, - "hfopenllm_v2/MUSR": 0.4248, - "hfopenllm_v2/MMLU-PRO": 0.5286 - } - }, - { - "id": "sthenno/tempesthenno-nuslerp-001", - "name": "tempesthenno-nuslerp-001", - "developer": "sthenno", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7926, - "hfopenllm_v2/BBH": 0.6578, - "hfopenllm_v2/MATH Level 5": 0.4758, - "hfopenllm_v2/GPQA": 0.3733, - "hfopenllm_v2/MUSR": 0.43, - "hfopenllm_v2/MMLU-PRO": 0.5257 - } - }, - { - "id": "sthenno/tempesthenno-nuslerp-0124", - "name": "tempesthenno-nuslerp-0124", - "developer": "sthenno", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7004, - "hfopenllm_v2/BBH": 0.6469, - "hfopenllm_v2/MATH Level 5": 0.4116, - "hfopenllm_v2/GPQA": 0.3901, - "hfopenllm_v2/MUSR": 0.4859, - "hfopenllm_v2/MMLU-PRO": 0.5352 - } - }, - { - "id": "sthenno/tempesthenno-ppo-ckpt40", - "name": "tempesthenno-ppo-ckpt40", - "developer": "sthenno", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7923, - "hfopenllm_v2/BBH": 0.655, - "hfopenllm_v2/MATH Level 5": 0.4736, - "hfopenllm_v2/GPQA": 0.3775, - "hfopenllm_v2/MUSR": 0.4352, - "hfopenllm_v2/MMLU-PRO": 0.5292 - } - }, - { - "id": "sthenno/tempesthenno-sft-0309-ckpt10", - "name": "tempesthenno-sft-0309-ckpt10", - "developer": "sthenno", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7744, - "hfopenllm_v2/BBH": 0.6552, - "hfopenllm_v2/MATH Level 5": 0.4721, - "hfopenllm_v2/GPQA": 0.3716, - "hfopenllm_v2/MUSR": 0.4364, - "hfopenllm_v2/MMLU-PRO": 0.5258 - } - }, - { - "id": "sthenno/tempesthenno-sft-0314-stage1-ckpt50", - "name": "tempesthenno-sft-0314-stage1-ckpt50", - "developer": "sthenno", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7394, - "hfopenllm_v2/BBH": 0.6601, - "hfopenllm_v2/MATH Level 5": 0.4683, - "hfopenllm_v2/GPQA": 0.3733, - "hfopenllm_v2/MUSR": 0.4429, - "hfopenllm_v2/MMLU-PRO": 0.5302 - } - }, - { - "id": "sthenno/tempestissimo-14b-0309", - "name": "tempestissimo-14b-0309", - "developer": "sthenno", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7549, - "hfopenllm_v2/BBH": 0.6587, - "hfopenllm_v2/MATH Level 5": 0.4796, - "hfopenllm_v2/GPQA": 0.3666, - "hfopenllm_v2/MUSR": 0.4312, - "hfopenllm_v2/MMLU-PRO": 0.5281 - } - }, - { - "id": "streamerbtw1002/Nexuim-R1-7B-Instruct", - "name": "Nexuim-R1-7B-Instruct", - "developer": "streamerbtw1002", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6934, - "hfopenllm_v2/BBH": 0.5175, - "hfopenllm_v2/MATH Level 5": 0.4456, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3356, - "hfopenllm_v2/MMLU-PRO": 0.4138 - } - }, - { - "id": "stupidity-ai/Llama-3-8B-Instruct-MultiMoose", - "name": "Llama-3-8B-Instruct-MultiMoose", - "developer": "stupidity-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2318, - "hfopenllm_v2/BBH": 0.2823, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2534, - "hfopenllm_v2/MUSR": 0.3485, - "hfopenllm_v2/MMLU-PRO": 0.1094 - } - }, - { - "id": "suayptalha/Clarus-7B-v0.1", - "name": "Clarus-7B-v0.1", - "developer": "suayptalha", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7454, - "hfopenllm_v2/BBH": 0.5497, - "hfopenllm_v2/MATH Level 5": 0.4924, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.443, - "hfopenllm_v2/MMLU-PRO": 0.4387 - } - }, - { - "id": "suayptalha/Clarus-7B-v0.2", - "name": "Clarus-7B-v0.2", - "developer": "suayptalha", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7679, - "hfopenllm_v2/BBH": 0.549, - "hfopenllm_v2/MATH Level 5": 0.4856, - "hfopenllm_v2/GPQA": 0.302, - "hfopenllm_v2/MUSR": 0.4417, - "hfopenllm_v2/MMLU-PRO": 0.44 - } - }, - { - "id": "suayptalha/Clarus-7B-v0.3", - "name": "Clarus-7B-v0.3", - "developer": "suayptalha", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7509, - "hfopenllm_v2/BBH": 0.5526, - "hfopenllm_v2/MATH Level 5": 0.4879, - "hfopenllm_v2/GPQA": 0.3121, - "hfopenllm_v2/MUSR": 0.4402, - "hfopenllm_v2/MMLU-PRO": 0.4385 - } - }, - { - "id": "suayptalha/DeepSeek-R1-Distill-Llama-3B", - "name": "DeepSeek-R1-Distill-Llama-3B", - "developer": "suayptalha", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7093, - "hfopenllm_v2/BBH": 0.4452, - "hfopenllm_v2/MATH Level 5": 0.2092, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.3396, - "hfopenllm_v2/MMLU-PRO": 0.2978 - } - }, - { - "id": "suayptalha/Falcon3-Jessi-v0.4-7B-Slerp", - "name": "Falcon3-Jessi-v0.4-7B-Slerp", - "developer": "suayptalha", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7676, - "hfopenllm_v2/BBH": 0.5591, - "hfopenllm_v2/MATH Level 5": 0.3965, - "hfopenllm_v2/GPQA": 0.3121, - "hfopenllm_v2/MUSR": 0.4812, - "hfopenllm_v2/MMLU-PRO": 0.406 - } - }, - { - "id": "suayptalha/HomerCreativeAnvita-Mix-Qw7B", - "name": "HomerCreativeAnvita-Mix-Qw7B", - "developer": "suayptalha", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7808, - "hfopenllm_v2/BBH": 0.5565, - "hfopenllm_v2/MATH Level 5": 0.361, - "hfopenllm_v2/GPQA": 0.3146, - "hfopenllm_v2/MUSR": 0.4416, - "hfopenllm_v2/MMLU-PRO": 0.4445 - } - }, - { - "id": "suayptalha/Komodo-Llama-3.2-3B-v2-fp16", - "name": "Komodo-Llama-3.2-3B-v2-fp16", - "developer": "suayptalha", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6341, - "hfopenllm_v2/BBH": 0.4355, - "hfopenllm_v2/MATH Level 5": 0.1065, - "hfopenllm_v2/GPQA": 0.2777, - "hfopenllm_v2/MUSR": 0.3406, - "hfopenllm_v2/MMLU-PRO": 0.2852 - } - }, - { - "id": "suayptalha/Lamarckvergence-14B", - "name": "Lamarckvergence-14B", - "developer": "suayptalha", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7656, - "hfopenllm_v2/BBH": 0.6517, - "hfopenllm_v2/MATH Level 5": 0.54, - "hfopenllm_v2/GPQA": 0.3633, - "hfopenllm_v2/MUSR": 0.4422, - "hfopenllm_v2/MMLU-PRO": 0.5283 - } - }, - { - "id": "suayptalha/Lix-14B-v0.1", - "name": "Lix-14B-v0.1", - "developer": "suayptalha", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7813, - "hfopenllm_v2/BBH": 0.6608, - "hfopenllm_v2/MATH Level 5": 0.5295, - "hfopenllm_v2/GPQA": 0.37, - "hfopenllm_v2/MUSR": 0.4338, - "hfopenllm_v2/MMLU-PRO": 0.5314 - } - }, - { - "id": "suayptalha/Luminis-phi-4", - "name": "Luminis-phi-4", - "developer": "suayptalha", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.69, - "hfopenllm_v2/BBH": 0.692, - "hfopenllm_v2/MATH Level 5": 0.4637, - "hfopenllm_v2/GPQA": 0.3515, - "hfopenllm_v2/MUSR": 0.4572, - "hfopenllm_v2/MMLU-PRO": 0.5424 - } - }, - { - "id": "suayptalha/Maestro-10B", - "name": "Maestro-10B", - "developer": "suayptalha", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7768, - "hfopenllm_v2/BBH": 0.5746, - "hfopenllm_v2/MATH Level 5": 0.1911, - "hfopenllm_v2/GPQA": 0.3331, - "hfopenllm_v2/MUSR": 0.4397, - "hfopenllm_v2/MMLU-PRO": 0.4218 - } - }, - { - "id": "suayptalha/Rombos-2.5-T.E-8.1", - "name": "Rombos-2.5-T.E-8.1", - "developer": "suayptalha", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6925, - "hfopenllm_v2/BBH": 0.5515, - "hfopenllm_v2/MATH Level 5": 0.4924, - "hfopenllm_v2/GPQA": 0.3112, - "hfopenllm_v2/MUSR": 0.4166, - "hfopenllm_v2/MMLU-PRO": 0.4446 - } - }, - { - "id": "SultanR/SmolTulu-1.7b-Instruct", - "name": "SmolTulu-1.7b-Instruct", - "developer": "SultanR", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6541, - "hfopenllm_v2/BBH": 0.3713, - "hfopenllm_v2/MATH Level 5": 0.0793, - "hfopenllm_v2/GPQA": 0.2693, - "hfopenllm_v2/MUSR": 0.354, - "hfopenllm_v2/MMLU-PRO": 0.171 - } - }, - { - "id": "SultanR/SmolTulu-1.7b-it-v0", - "name": "SmolTulu-1.7b-it-v0", - "developer": "SultanR", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6541, - "hfopenllm_v2/BBH": 0.3713, - "hfopenllm_v2/MATH Level 5": 0.0793, - "hfopenllm_v2/GPQA": 0.2693, - "hfopenllm_v2/MUSR": 0.354, - "hfopenllm_v2/MMLU-PRO": 0.171 - } - }, - { - "id": "SultanR/SmolTulu-1.7b-Reinforced", - "name": "SmolTulu-1.7b-Reinforced", - "developer": "SultanR", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6791, - "hfopenllm_v2/BBH": 0.3552, - "hfopenllm_v2/MATH Level 5": 0.0718, - "hfopenllm_v2/GPQA": 0.276, - "hfopenllm_v2/MUSR": 0.3406, - "hfopenllm_v2/MMLU-PRO": 0.1763 - } - }, - { - "id": "SultanR/SmolTulu-1.7b-RM", - "name": "SultanR/SmolTulu-1.7b-RM", - "developer": "SultanR", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.5094, - "reward-bench/Chat": 0.743, - "reward-bench/Chat Hard": 0.4408, - "reward-bench/Safety": 0.5716, - "reward-bench/Reasoning": 0.2821 - } - }, - { - "id": "sumink/bbhqwen", - "name": "bbhqwen", - "developer": "sumink", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1809, - "hfopenllm_v2/BBH": 0.3388, - "hfopenllm_v2/MATH Level 5": 0.0106, - "hfopenllm_v2/GPQA": 0.2576, - "hfopenllm_v2/MUSR": 0.4352, - "hfopenllm_v2/MMLU-PRO": 0.1617 - } - }, - { - "id": "sumink/bbhqwen2", - "name": "bbhqwen2", - "developer": "sumink", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1533, - "hfopenllm_v2/BBH": 0.3066, - "hfopenllm_v2/MATH Level 5": 0.006, - "hfopenllm_v2/GPQA": 0.2626, - "hfopenllm_v2/MUSR": 0.4431, - "hfopenllm_v2/MMLU-PRO": 0.1149 - } - }, - { - "id": "sumink/bbhqwen3", - "name": "bbhqwen3", - "developer": "sumink", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1943, - "hfopenllm_v2/BBH": 0.2951, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2576, - "hfopenllm_v2/MUSR": 0.3796, - "hfopenllm_v2/MMLU-PRO": 0.1166 - } - }, - { - "id": "sumink/bbhqwen4", - "name": "bbhqwen4", - "developer": "sumink", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1449, - "hfopenllm_v2/BBH": 0.3199, - "hfopenllm_v2/MATH Level 5": 0.006, - "hfopenllm_v2/GPQA": 0.2441, - "hfopenllm_v2/MUSR": 0.4029, - "hfopenllm_v2/MMLU-PRO": 0.1509 - } - }, - { - "id": "sumink/bbhqwen5", - "name": "bbhqwen5", - "developer": "sumink", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1522, - "hfopenllm_v2/BBH": 0.2913, - "hfopenllm_v2/MATH Level 5": 0.0023, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.4019, - "hfopenllm_v2/MMLU-PRO": 0.1131 - } - }, - { - "id": "sumink/bbhqwen6", - "name": "bbhqwen6", - "developer": "sumink", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1893, - "hfopenllm_v2/BBH": 0.2782, - "hfopenllm_v2/MATH Level 5": 0.0008, - "hfopenllm_v2/GPQA": 0.2584, - "hfopenllm_v2/MUSR": 0.358, - "hfopenllm_v2/MMLU-PRO": 0.1153 - } - }, - { - "id": "sumink/flflmillama", - "name": "flflmillama", - "developer": "sumink", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1676, - "hfopenllm_v2/BBH": 0.3851, - "hfopenllm_v2/MATH Level 5": 0.0196, - "hfopenllm_v2/GPQA": 0.2919, - "hfopenllm_v2/MUSR": 0.3591, - "hfopenllm_v2/MMLU-PRO": 0.2096 - } - }, - { - "id": "sumink/ftgpt", - "name": "ftgpt", - "developer": "sumink", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0787, - "hfopenllm_v2/BBH": 0.2919, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2643, - "hfopenllm_v2/MUSR": 0.4138, - "hfopenllm_v2/MMLU-PRO": 0.1172 - } - }, - { - "id": "sumink/llamaft", - "name": "llamaft", - "developer": "sumink", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1609, - "hfopenllm_v2/BBH": 0.3763, - "hfopenllm_v2/MATH Level 5": 0.0166, - "hfopenllm_v2/GPQA": 0.271, - "hfopenllm_v2/MUSR": 0.3498, - "hfopenllm_v2/MMLU-PRO": 0.2114 - } - }, - { - "id": "sumink/llamamerge", - "name": "llamamerge", - "developer": "sumink", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2672, - "hfopenllm_v2/BBH": 0.4632, - "hfopenllm_v2/MATH Level 5": 0.0151, - "hfopenllm_v2/GPQA": 0.2987, - "hfopenllm_v2/MUSR": 0.424, - "hfopenllm_v2/MMLU-PRO": 0.259 - } - }, - { - "id": "sumink/llftfl7", - "name": "llftfl7", - "developer": "sumink", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1714, - "hfopenllm_v2/BBH": 0.3786, - "hfopenllm_v2/MATH Level 5": 0.0106, - "hfopenllm_v2/GPQA": 0.281, - "hfopenllm_v2/MUSR": 0.3632, - "hfopenllm_v2/MMLU-PRO": 0.1743 - } - }, - { - "id": "sumink/llmer", - "name": "llmer", - "developer": "sumink", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3191, - "hfopenllm_v2/BBH": 0.4885, - "hfopenllm_v2/MATH Level 5": 0.065, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.4039, - "hfopenllm_v2/MMLU-PRO": 0.3529 - } - }, - { - "id": "sumink/Qmerft", - "name": "Qmerft", - "developer": "sumink", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1564, - "hfopenllm_v2/BBH": 0.2939, - "hfopenllm_v2/MATH Level 5": 0.0023, - "hfopenllm_v2/GPQA": 0.2525, - "hfopenllm_v2/MUSR": 0.3688, - "hfopenllm_v2/MMLU-PRO": 0.1157 - } - }, - { - "id": "sumink/Qwenftmodel", - "name": "Qwenftmodel", - "developer": "sumink", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1729, - "hfopenllm_v2/BBH": 0.3823, - "hfopenllm_v2/MATH Level 5": 0.0891, - "hfopenllm_v2/GPQA": 0.2567, - "hfopenllm_v2/MUSR": 0.3617, - "hfopenllm_v2/MMLU-PRO": 0.2339 - } - }, - { - "id": "sumink/Qwenmplus", - "name": "Qwenmplus", - "developer": "sumink", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.204, - "hfopenllm_v2/BBH": 0.3676, - "hfopenllm_v2/MATH Level 5": 0.0249, - "hfopenllm_v2/GPQA": 0.2852, - "hfopenllm_v2/MUSR": 0.3828, - "hfopenllm_v2/MMLU-PRO": 0.1992 - } - }, - { - "id": "sumink/Qwensci", - "name": "Qwensci", - "developer": "sumink", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.174, - "hfopenllm_v2/BBH": 0.3282, - "hfopenllm_v2/MATH Level 5": 0.0204, - "hfopenllm_v2/GPQA": 0.2584, - "hfopenllm_v2/MUSR": 0.3609, - "hfopenllm_v2/MMLU-PRO": 0.126 - } - }, - { - "id": "sumink/qwft", - "name": "qwft", - "developer": "sumink", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1197, - "hfopenllm_v2/BBH": 0.3002, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2525, - "hfopenllm_v2/MUSR": 0.3581, - "hfopenllm_v2/MMLU-PRO": 0.1129 - } - }, - { - "id": "sumink/qwmer", - "name": "qwmer", - "developer": "sumink", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2212, - "hfopenllm_v2/BBH": 0.4299, - "hfopenllm_v2/MATH Level 5": 0.0008, - "hfopenllm_v2/GPQA": 0.2869, - "hfopenllm_v2/MUSR": 0.4032, - "hfopenllm_v2/MMLU-PRO": 0.2215 - } - }, - { - "id": "sumink/solarmer3", - "name": "solarmer3", - "developer": "sumink", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3741, - "hfopenllm_v2/BBH": 0.5266, - "hfopenllm_v2/MATH Level 5": 0.0582, - "hfopenllm_v2/GPQA": 0.2911, - "hfopenllm_v2/MUSR": 0.4401, - "hfopenllm_v2/MMLU-PRO": 0.3323 - } - }, - { - "id": "sumink/somer", - "name": "somer", - "developer": "sumink", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.299, - "hfopenllm_v2/BBH": 0.5194, - "hfopenllm_v2/MATH Level 5": 0.0415, - "hfopenllm_v2/GPQA": 0.2987, - "hfopenllm_v2/MUSR": 0.465, - "hfopenllm_v2/MMLU-PRO": 0.3447 - } - }, - { - "id": "sumink/somer2", - "name": "somer2", - "developer": "sumink", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3132, - "hfopenllm_v2/BBH": 0.5167, - "hfopenllm_v2/MATH Level 5": 0.0468, - "hfopenllm_v2/GPQA": 0.3037, - "hfopenllm_v2/MUSR": 0.4663, - "hfopenllm_v2/MMLU-PRO": 0.3433 - } - }, - { - "id": "sumink/somerft", - "name": "somerft", - "developer": "sumink", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1431, - "hfopenllm_v2/BBH": 0.3093, - "hfopenllm_v2/MATH Level 5": 0.0144, - "hfopenllm_v2/GPQA": 0.2483, - "hfopenllm_v2/MUSR": 0.4045, - "hfopenllm_v2/MMLU-PRO": 0.1117 - } - }, - { - "id": "sunbaby/BrainCog-8B-0.1-Instruct", - "name": "BrainCog-8B-0.1-Instruct", - "developer": "sunbaby", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4253, - "hfopenllm_v2/BBH": 0.4618, - "hfopenllm_v2/MATH Level 5": 0.0967, - "hfopenllm_v2/GPQA": 0.3012, - "hfopenllm_v2/MUSR": 0.3656, - "hfopenllm_v2/MMLU-PRO": 0.2858 - } - }, - { - "id": "Supichi/BBA-123", - "name": "BBA-123", - "developer": "Supichi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.208, - "hfopenllm_v2/BBH": 0.292, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3499, - "hfopenllm_v2/MMLU-PRO": 0.1167 - } - }, - { - "id": "Supichi/BBA99", - "name": "BBA99", - "developer": "Supichi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1407, - "hfopenllm_v2/BBH": 0.2769, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2634, - "hfopenllm_v2/MUSR": 0.3218, - "hfopenllm_v2/MMLU-PRO": 0.1112 - } - }, - { - "id": "Supichi/BBAI_135_Gemma", - "name": "BBAI_135_Gemma", - "developer": "Supichi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0656, - "hfopenllm_v2/BBH": 0.3568, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2676, - "hfopenllm_v2/MUSR": 0.3805, - "hfopenllm_v2/MMLU-PRO": 0.1672 - } - }, - { - "id": "Supichi/BBAI_250_Xia0_gZ", - "name": "BBAI_250_Xia0_gZ", - "developer": "Supichi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4685, - "hfopenllm_v2/BBH": 0.5568, - "hfopenllm_v2/MATH Level 5": 0.364, - "hfopenllm_v2/GPQA": 0.3213, - "hfopenllm_v2/MUSR": 0.4579, - "hfopenllm_v2/MMLU-PRO": 0.4465 - } - }, - { - "id": "Supichi/BBAI_275_Tsunami_gZ", - "name": "BBAI_275_Tsunami_gZ", - "developer": "Supichi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.537, - "hfopenllm_v2/BBH": 0.5531, - "hfopenllm_v2/MATH Level 5": 0.3285, - "hfopenllm_v2/GPQA": 0.3213, - "hfopenllm_v2/MUSR": 0.4448, - "hfopenllm_v2/MMLU-PRO": 0.4492 - } - }, - { - "id": "Supichi/BBAI_525_Tsu_gZ_Xia0", - "name": "BBAI_525_Tsu_gZ_Xia0", - "developer": "Supichi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5339, - "hfopenllm_v2/BBH": 0.5562, - "hfopenllm_v2/MATH Level 5": 0.3429, - "hfopenllm_v2/GPQA": 0.3121, - "hfopenllm_v2/MUSR": 0.4474, - "hfopenllm_v2/MMLU-PRO": 0.4477 - } - }, - { - "id": "Supichi/BBAI_78B_Calme_3_1_Ties", - "name": "BBAI_78B_Calme_3_1_Ties", - "developer": "Supichi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1828, - "hfopenllm_v2/BBH": 0.2828, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.229, - "hfopenllm_v2/MUSR": 0.31, - "hfopenllm_v2/MMLU-PRO": 0.1144 - } - }, - { - "id": "Supichi/BBAI_QWEEN_V000000_LUMEN_14B", - "name": "BBAI_QWEEN_V000000_LUMEN_14B", - "developer": "Supichi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1815, - "hfopenllm_v2/BBH": 0.2297, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2315, - "hfopenllm_v2/MUSR": 0.3445, - "hfopenllm_v2/MMLU-PRO": 0.116 - } - }, - { - "id": "Supichi/BBAIK29", - "name": "BBAIK29", - "developer": "Supichi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4588, - "hfopenllm_v2/BBH": 0.559, - "hfopenllm_v2/MATH Level 5": 0.3678, - "hfopenllm_v2/GPQA": 0.3121, - "hfopenllm_v2/MUSR": 0.4501, - "hfopenllm_v2/MMLU-PRO": 0.4469 - } - }, - { - "id": "Supichi/HF_TOKEN", - "name": "HF_TOKEN", - "developer": "Supichi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.138, - "hfopenllm_v2/BBH": 0.2764, - "hfopenllm_v2/MATH Level 5": 0.0008, - "hfopenllm_v2/GPQA": 0.2634, - "hfopenllm_v2/MUSR": 0.3272, - "hfopenllm_v2/MMLU-PRO": 0.111 - } - }, - { - "id": "Supichi/NJS26", - "name": "NJS26", - "developer": "Supichi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0448, - "hfopenllm_v2/BBH": 0.478, - "hfopenllm_v2/MATH Level 5": 0.0325, - "hfopenllm_v2/GPQA": 0.318, - "hfopenllm_v2/MUSR": 0.3854, - "hfopenllm_v2/MMLU-PRO": 0.3037 - } - }, - { - "id": "Svak/MN-12B-Inferor-v0.0", - "name": "MN-12B-Inferor-v0.0", - "developer": "Svak", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5708, - "hfopenllm_v2/BBH": 0.5195, - "hfopenllm_v2/MATH Level 5": 0.102, - "hfopenllm_v2/GPQA": 0.3087, - "hfopenllm_v2/MUSR": 0.4639, - "hfopenllm_v2/MMLU-PRO": 0.3559 - } - }, - { - "id": "Svak/MN-12B-Inferor-v0.1", - "name": "MN-12B-Inferor-v0.1", - "developer": "Svak", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6347, - "hfopenllm_v2/BBH": 0.5147, - "hfopenllm_v2/MATH Level 5": 0.1261, - "hfopenllm_v2/GPQA": 0.3255, - "hfopenllm_v2/MUSR": 0.4351, - "hfopenllm_v2/MMLU-PRO": 0.3662 - } - }, - { - "id": "swap-uniba/LLaMAntino-3-ANITA-8B-Inst-DPO-ITA", - "name": "LLaMAntino-3-ANITA-8B-Inst-DPO-ITA", - "developer": "swap-uniba", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4815, - "hfopenllm_v2/BBH": 0.4936, - "hfopenllm_v2/MATH Level 5": 0.0483, - "hfopenllm_v2/GPQA": 0.2987, - "hfopenllm_v2/MUSR": 0.4387, - "hfopenllm_v2/MMLU-PRO": 0.3723 - } - }, - { - "id": "Syed-Hasan-8503/Phi-3-mini-4K-instruct-cpo-simpo", - "name": "Phi-3-mini-4K-instruct-cpo-simpo", - "developer": "Syed-Hasan-8503", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5714, - "hfopenllm_v2/BBH": 0.5682, - "hfopenllm_v2/MATH Level 5": 0.1571, - "hfopenllm_v2/GPQA": 0.3305, - "hfopenllm_v2/MUSR": 0.3964, - "hfopenllm_v2/MMLU-PRO": 0.3861 - } - }, - { - "id": "synergetic/FrankenQwen2.5-14B", - "name": "FrankenQwen2.5-14B", - "developer": "synergetic", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1869, - "hfopenllm_v2/BBH": 0.6048, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.3843, - "hfopenllm_v2/MMLU-PRO": 0.4382 - } - }, - { - "id": "T145/KRONOS-8B-V1-P1", - "name": "KRONOS-8B-V1-P1", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.785, - "hfopenllm_v2/BBH": 0.5085, - "hfopenllm_v2/MATH Level 5": 0.1979, - "hfopenllm_v2/GPQA": 0.2953, - "hfopenllm_v2/MUSR": 0.3881, - "hfopenllm_v2/MMLU-PRO": 0.376 - } - }, - { - "id": "T145/KRONOS-8B-V1-P2", - "name": "KRONOS-8B-V1-P2", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6724, - "hfopenllm_v2/BBH": 0.4772, - "hfopenllm_v2/MATH Level 5": 0.1601, - "hfopenllm_v2/GPQA": 0.2919, - "hfopenllm_v2/MUSR": 0.3568, - "hfopenllm_v2/MMLU-PRO": 0.3453 - } - }, - { - "id": "T145/KRONOS-8B-V1-P3", - "name": "KRONOS-8B-V1-P3", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7137, - "hfopenllm_v2/BBH": 0.5128, - "hfopenllm_v2/MATH Level 5": 0.1926, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3616, - "hfopenllm_v2/MMLU-PRO": 0.3405 - } - }, - { - "id": "T145/KRONOS-8B-V2", - "name": "KRONOS-8B-V2", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.518, - "hfopenllm_v2/BBH": 0.5133, - "hfopenllm_v2/MATH Level 5": 0.2266, - "hfopenllm_v2/GPQA": 0.2987, - "hfopenllm_v2/MUSR": 0.3829, - "hfopenllm_v2/MMLU-PRO": 0.3738 - } - }, - { - "id": "T145/KRONOS-8B-V3", - "name": "KRONOS-8B-V3", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5475, - "hfopenllm_v2/BBH": 0.5119, - "hfopenllm_v2/MATH Level 5": 0.2598, - "hfopenllm_v2/GPQA": 0.2886, - "hfopenllm_v2/MUSR": 0.3922, - "hfopenllm_v2/MMLU-PRO": 0.3738 - } - }, - { - "id": "T145/KRONOS-8B-V4", - "name": "KRONOS-8B-V4", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7889, - "hfopenllm_v2/BBH": 0.5092, - "hfopenllm_v2/MATH Level 5": 0.1949, - "hfopenllm_v2/GPQA": 0.2894, - "hfopenllm_v2/MUSR": 0.383, - "hfopenllm_v2/MMLU-PRO": 0.3786 - } - }, - { - "id": "T145/KRONOS-8B-V5", - "name": "KRONOS-8B-V5", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5405, - "hfopenllm_v2/BBH": 0.5089, - "hfopenllm_v2/MATH Level 5": 0.2689, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.4055, - "hfopenllm_v2/MMLU-PRO": 0.3759 - } - }, - { - "id": "T145/KRONOS-8B-V6", - "name": "KRONOS-8B-V6", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7022, - "hfopenllm_v2/BBH": 0.5034, - "hfopenllm_v2/MATH Level 5": 0.2598, - "hfopenllm_v2/GPQA": 0.2794, - "hfopenllm_v2/MUSR": 0.4121, - "hfopenllm_v2/MMLU-PRO": 0.3501 - } - }, - { - "id": "T145/KRONOS-8B-V7", - "name": "KRONOS-8B-V7", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3529, - "hfopenllm_v2/BBH": 0.4526, - "hfopenllm_v2/MATH Level 5": 0.111, - "hfopenllm_v2/GPQA": 0.2668, - "hfopenllm_v2/MUSR": 0.3671, - "hfopenllm_v2/MMLU-PRO": 0.2697 - } - }, - { - "id": "T145/KRONOS-8B-V8", - "name": "KRONOS-8B-V8", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.777, - "hfopenllm_v2/BBH": 0.5094, - "hfopenllm_v2/MATH Level 5": 0.2047, - "hfopenllm_v2/GPQA": 0.2894, - "hfopenllm_v2/MUSR": 0.3869, - "hfopenllm_v2/MMLU-PRO": 0.3782 - } - }, - { - "id": "T145/KRONOS-8B-V9", - "name": "KRONOS-8B-V9", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7856, - "hfopenllm_v2/BBH": 0.5099, - "hfopenllm_v2/MATH Level 5": 0.1986, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.3868, - "hfopenllm_v2/MMLU-PRO": 0.3752 - } - }, - { - "id": "T145/Llama-3.1-8B-Instruct-Zeus", - "name": "Llama-3.1-8B-Instruct-Zeus", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7941, - "hfopenllm_v2/BBH": 0.5174, - "hfopenllm_v2/MATH Level 5": 0.1956, - "hfopenllm_v2/GPQA": 0.3012, - "hfopenllm_v2/MUSR": 0.3976, - "hfopenllm_v2/MMLU-PRO": 0.3893 - } - }, - { - "id": "T145/Llama-3.1-8B-Zeus", - "name": "Llama-3.1-8B-Zeus", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3518, - "hfopenllm_v2/BBH": 0.3671, - "hfopenllm_v2/MATH Level 5": 0.0144, - "hfopenllm_v2/GPQA": 0.2651, - "hfopenllm_v2/MUSR": 0.3316, - "hfopenllm_v2/MMLU-PRO": 0.1332 - } - }, - { - "id": "T145/Meta-Llama-3.1-8B-Instruct-TIES", - "name": "Meta-Llama-3.1-8B-Instruct-TIES", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5424, - "hfopenllm_v2/BBH": 0.507, - "hfopenllm_v2/MATH Level 5": 0.21, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.3843, - "hfopenllm_v2/MMLU-PRO": 0.378 - } - }, - { - "id": "T145/qwen-2.5-3B-merge-test", - "name": "qwen-2.5-3B-merge-test", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5751, - "hfopenllm_v2/BBH": 0.4842, - "hfopenllm_v2/MATH Level 5": 0.3202, - "hfopenllm_v2/GPQA": 0.2852, - "hfopenllm_v2/MUSR": 0.4007, - "hfopenllm_v2/MMLU-PRO": 0.329 - } - }, - { - "id": "T145/ZEUS-8B-V10", - "name": "ZEUS-8B-V10", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7707, - "hfopenllm_v2/BBH": 0.527, - "hfopenllm_v2/MATH Level 5": 0.2115, - "hfopenllm_v2/GPQA": 0.3247, - "hfopenllm_v2/MUSR": 0.3898, - "hfopenllm_v2/MMLU-PRO": 0.3904 - } - }, - { - "id": "T145/ZEUS-8B-V11", - "name": "ZEUS-8B-V11", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.81, - "hfopenllm_v2/BBH": 0.5162, - "hfopenllm_v2/MATH Level 5": 0.1964, - "hfopenllm_v2/GPQA": 0.3146, - "hfopenllm_v2/MUSR": 0.3807, - "hfopenllm_v2/MMLU-PRO": 0.3884 - } - }, - { - "id": "T145/ZEUS-8B-V12", - "name": "ZEUS-8B-V12", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7816, - "hfopenllm_v2/BBH": 0.5254, - "hfopenllm_v2/MATH Level 5": 0.2115, - "hfopenllm_v2/GPQA": 0.3205, - "hfopenllm_v2/MUSR": 0.3858, - "hfopenllm_v2/MMLU-PRO": 0.3912 - } - }, - { - "id": "T145/ZEUS-8B-V13", - "name": "ZEUS-8B-V13", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7904, - "hfopenllm_v2/BBH": 0.5277, - "hfopenllm_v2/MATH Level 5": 0.2137, - "hfopenllm_v2/GPQA": 0.3238, - "hfopenllm_v2/MUSR": 0.3845, - "hfopenllm_v2/MMLU-PRO": 0.3911 - } - }, - { - "id": "T145/ZEUS-8B-V13-abliterated", - "name": "ZEUS-8B-V13-abliterated", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7878, - "hfopenllm_v2/BBH": 0.5198, - "hfopenllm_v2/MATH Level 5": 0.179, - "hfopenllm_v2/GPQA": 0.3112, - "hfopenllm_v2/MUSR": 0.3871, - "hfopenllm_v2/MMLU-PRO": 0.3872 - } - }, - { - "id": "T145/ZEUS-8B-V14", - "name": "ZEUS-8B-V14", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7709, - "hfopenllm_v2/BBH": 0.5275, - "hfopenllm_v2/MATH Level 5": 0.213, - "hfopenllm_v2/GPQA": 0.3205, - "hfopenllm_v2/MUSR": 0.3844, - "hfopenllm_v2/MMLU-PRO": 0.3914 - } - }, - { - "id": "T145/ZEUS-8B-V15", - "name": "ZEUS-8B-V15", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7013, - "hfopenllm_v2/BBH": 0.5538, - "hfopenllm_v2/MATH Level 5": 0.2304, - "hfopenllm_v2/GPQA": 0.276, - "hfopenllm_v2/MUSR": 0.402, - "hfopenllm_v2/MMLU-PRO": 0.4059 - } - }, - { - "id": "T145/ZEUS-8B-V16", - "name": "ZEUS-8B-V16", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7925, - "hfopenllm_v2/BBH": 0.5266, - "hfopenllm_v2/MATH Level 5": 0.2205, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.3951, - "hfopenllm_v2/MMLU-PRO": 0.3926 - } - }, - { - "id": "T145/ZEUS-8B-V17", - "name": "ZEUS-8B-V17", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7941, - "hfopenllm_v2/BBH": 0.5251, - "hfopenllm_v2/MATH Level 5": 0.2243, - "hfopenllm_v2/GPQA": 0.3221, - "hfopenllm_v2/MUSR": 0.4016, - "hfopenllm_v2/MMLU-PRO": 0.3935 - } - }, - { - "id": "T145/ZEUS-8B-V17-abliterated", - "name": "ZEUS-8B-V17-abliterated", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7576, - "hfopenllm_v2/BBH": 0.52, - "hfopenllm_v2/MATH Level 5": 0.0438, - "hfopenllm_v2/GPQA": 0.3037, - "hfopenllm_v2/MUSR": 0.4269, - "hfopenllm_v2/MMLU-PRO": 0.3622 - } - }, - { - "id": "T145/ZEUS-8B-V17-abliterated-V2", - "name": "ZEUS-8B-V17-abliterated-V2", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6532, - "hfopenllm_v2/BBH": 0.4928, - "hfopenllm_v2/MATH Level 5": 0.1118, - "hfopenllm_v2/GPQA": 0.2735, - "hfopenllm_v2/MUSR": 0.3407, - "hfopenllm_v2/MMLU-PRO": 0.3402 - } - }, - { - "id": "T145/ZEUS-8B-V17-abliterated-V4", - "name": "ZEUS-8B-V17-abliterated-V4", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7228, - "hfopenllm_v2/BBH": 0.5169, - "hfopenllm_v2/MATH Level 5": 0.0937, - "hfopenllm_v2/GPQA": 0.2836, - "hfopenllm_v2/MUSR": 0.4187, - "hfopenllm_v2/MMLU-PRO": 0.3774 - } - }, - { - "id": "T145/ZEUS-8B-V18", - "name": "ZEUS-8B-V18", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7834, - "hfopenllm_v2/BBH": 0.527, - "hfopenllm_v2/MATH Level 5": 0.2183, - "hfopenllm_v2/GPQA": 0.3213, - "hfopenllm_v2/MUSR": 0.4043, - "hfopenllm_v2/MMLU-PRO": 0.3942 - } - }, - { - "id": "T145/ZEUS-8B-V19", - "name": "ZEUS-8B-V19", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7883, - "hfopenllm_v2/BBH": 0.5276, - "hfopenllm_v2/MATH Level 5": 0.2205, - "hfopenllm_v2/GPQA": 0.3221, - "hfopenllm_v2/MUSR": 0.4043, - "hfopenllm_v2/MMLU-PRO": 0.3934 - } - }, - { - "id": "T145/ZEUS-8B-V2", - "name": "ZEUS-8B-V2", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8029, - "hfopenllm_v2/BBH": 0.5194, - "hfopenllm_v2/MATH Level 5": 0.216, - "hfopenllm_v2/GPQA": 0.302, - "hfopenllm_v2/MUSR": 0.391, - "hfopenllm_v2/MMLU-PRO": 0.3896 - } - }, - { - "id": "T145/ZEUS-8B-V2-abliterated", - "name": "ZEUS-8B-V2-abliterated", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7895, - "hfopenllm_v2/BBH": 0.5129, - "hfopenllm_v2/MATH Level 5": 0.2115, - "hfopenllm_v2/GPQA": 0.3129, - "hfopenllm_v2/MUSR": 0.3911, - "hfopenllm_v2/MMLU-PRO": 0.3825 - } - }, - { - "id": "T145/ZEUS-8B-V2-ORPO", - "name": "ZEUS-8B-V2-ORPO", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7187, - "hfopenllm_v2/BBH": 0.5075, - "hfopenllm_v2/MATH Level 5": 0.1828, - "hfopenllm_v2/GPQA": 0.3104, - "hfopenllm_v2/MUSR": 0.3935, - "hfopenllm_v2/MMLU-PRO": 0.3678 - } - }, - { - "id": "T145/ZEUS-8B-V20", - "name": "ZEUS-8B-V20", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7956, - "hfopenllm_v2/BBH": 0.5244, - "hfopenllm_v2/MATH Level 5": 0.219, - "hfopenllm_v2/GPQA": 0.323, - "hfopenllm_v2/MUSR": 0.4043, - "hfopenllm_v2/MMLU-PRO": 0.393 - } - }, - { - "id": "T145/ZEUS-8B-V21", - "name": "ZEUS-8B-V21", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3785, - "hfopenllm_v2/BBH": 0.3398, - "hfopenllm_v2/MATH Level 5": 0.1594, - "hfopenllm_v2/GPQA": 0.2643, - "hfopenllm_v2/MUSR": 0.3262, - "hfopenllm_v2/MMLU-PRO": 0.1714 - } - }, - { - "id": "T145/ZEUS-8B-V22", - "name": "ZEUS-8B-V22", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7995, - "hfopenllm_v2/BBH": 0.5245, - "hfopenllm_v2/MATH Level 5": 0.2228, - "hfopenllm_v2/GPQA": 0.328, - "hfopenllm_v2/MUSR": 0.399, - "hfopenllm_v2/MMLU-PRO": 0.3938 - } - }, - { - "id": "T145/ZEUS-8B-V23", - "name": "ZEUS-8B-V23", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7621, - "hfopenllm_v2/BBH": 0.5195, - "hfopenllm_v2/MATH Level 5": 0.182, - "hfopenllm_v2/GPQA": 0.3096, - "hfopenllm_v2/MUSR": 0.3922, - "hfopenllm_v2/MMLU-PRO": 0.3666 - } - }, - { - "id": "T145/ZEUS-8B-V24", - "name": "ZEUS-8B-V24", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6, - "hfopenllm_v2/BBH": 0.4778, - "hfopenllm_v2/MATH Level 5": 0.1458, - "hfopenllm_v2/GPQA": 0.2617, - "hfopenllm_v2/MUSR": 0.3729, - "hfopenllm_v2/MMLU-PRO": 0.3285 - } - }, - { - "id": "T145/ZEUS-8B-V25", - "name": "ZEUS-8B-V25", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.332, - "hfopenllm_v2/BBH": 0.4547, - "hfopenllm_v2/MATH Level 5": 0.2039, - "hfopenllm_v2/GPQA": 0.2643, - "hfopenllm_v2/MUSR": 0.3488, - "hfopenllm_v2/MMLU-PRO": 0.2885 - } - }, - { - "id": "T145/ZEUS-8B-V26", - "name": "ZEUS-8B-V26", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6708, - "hfopenllm_v2/BBH": 0.5232, - "hfopenllm_v2/MATH Level 5": 0.1246, - "hfopenllm_v2/GPQA": 0.2953, - "hfopenllm_v2/MUSR": 0.4016, - "hfopenllm_v2/MMLU-PRO": 0.3907 - } - }, - { - "id": "T145/ZEUS-8B-V27", - "name": "ZEUS-8B-V27", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6544, - "hfopenllm_v2/BBH": 0.523, - "hfopenllm_v2/MATH Level 5": 0.1344, - "hfopenllm_v2/GPQA": 0.3079, - "hfopenllm_v2/MUSR": 0.3977, - "hfopenllm_v2/MMLU-PRO": 0.3902 - } - }, - { - "id": "T145/ZEUS-8B-V28", - "name": "ZEUS-8B-V28", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6353, - "hfopenllm_v2/BBH": 0.5254, - "hfopenllm_v2/MATH Level 5": 0.1269, - "hfopenllm_v2/GPQA": 0.3037, - "hfopenllm_v2/MUSR": 0.3896, - "hfopenllm_v2/MMLU-PRO": 0.3902 - } - }, - { - "id": "T145/ZEUS-8B-V29", - "name": "ZEUS-8B-V29", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7418, - "hfopenllm_v2/BBH": 0.5253, - "hfopenllm_v2/MATH Level 5": 0.1601, - "hfopenllm_v2/GPQA": 0.3263, - "hfopenllm_v2/MUSR": 0.4003, - "hfopenllm_v2/MMLU-PRO": 0.392 - } - }, - { - "id": "T145/ZEUS-8B-V2L1", - "name": "ZEUS-8B-V2L1", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3192, - "hfopenllm_v2/BBH": 0.5013, - "hfopenllm_v2/MATH Level 5": 0.1239, - "hfopenllm_v2/GPQA": 0.3129, - "hfopenllm_v2/MUSR": 0.3882, - "hfopenllm_v2/MMLU-PRO": 0.3638 - } - }, - { - "id": "T145/ZEUS-8B-V2L2", - "name": "ZEUS-8B-V2L2", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8021, - "hfopenllm_v2/BBH": 0.5203, - "hfopenllm_v2/MATH Level 5": 0.2017, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.3975, - "hfopenllm_v2/MMLU-PRO": 0.3884 - } - }, - { - "id": "T145/ZEUS-8B-V3", - "name": "ZEUS-8B-V3", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7887, - "hfopenllm_v2/BBH": 0.5265, - "hfopenllm_v2/MATH Level 5": 0.1677, - "hfopenllm_v2/GPQA": 0.3221, - "hfopenllm_v2/MUSR": 0.4017, - "hfopenllm_v2/MMLU-PRO": 0.3804 - } - }, - { - "id": "T145/ZEUS-8B-V30", - "name": "ZEUS-8B-V30", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7436, - "hfopenllm_v2/BBH": 0.5243, - "hfopenllm_v2/MATH Level 5": 0.1586, - "hfopenllm_v2/GPQA": 0.3205, - "hfopenllm_v2/MUSR": 0.4029, - "hfopenllm_v2/MMLU-PRO": 0.3944 - } - }, - { - "id": "T145/ZEUS-8B-V4", - "name": "ZEUS-8B-V4", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7807, - "hfopenllm_v2/BBH": 0.5246, - "hfopenllm_v2/MATH Level 5": 0.1926, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.4029, - "hfopenllm_v2/MMLU-PRO": 0.3788 - } - }, - { - "id": "T145/ZEUS-8B-V6", - "name": "ZEUS-8B-V6", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7838, - "hfopenllm_v2/BBH": 0.524, - "hfopenllm_v2/MATH Level 5": 0.2024, - "hfopenllm_v2/GPQA": 0.3045, - "hfopenllm_v2/MUSR": 0.4068, - "hfopenllm_v2/MMLU-PRO": 0.3759 - } - }, - { - "id": "T145/ZEUS-8B-V7", - "name": "ZEUS-8B-V7", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7786, - "hfopenllm_v2/BBH": 0.507, - "hfopenllm_v2/MATH Level 5": 0.148, - "hfopenllm_v2/GPQA": 0.297, - "hfopenllm_v2/MUSR": 0.4162, - "hfopenllm_v2/MMLU-PRO": 0.3812 - } - }, - { - "id": "T145/ZEUS-8B-V8", - "name": "ZEUS-8B-V8", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7914, - "hfopenllm_v2/BBH": 0.5065, - "hfopenllm_v2/MATH Level 5": 0.1329, - "hfopenllm_v2/GPQA": 0.2878, - "hfopenllm_v2/MUSR": 0.4214, - "hfopenllm_v2/MMLU-PRO": 0.3761 - } - }, - { - "id": "T145/ZEUS-8B-V9", - "name": "ZEUS-8B-V9", - "developer": "T145", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5551, - "hfopenllm_v2/BBH": 0.5207, - "hfopenllm_v2/MATH Level 5": 0.2137, - "hfopenllm_v2/GPQA": 0.2911, - "hfopenllm_v2/MUSR": 0.3949, - "hfopenllm_v2/MMLU-PRO": 0.3901 - } - }, - { - "id": "talha2001/Beast-Soul-new", - "name": "Beast-Soul-new", - "developer": "talha2001", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4854, - "hfopenllm_v2/BBH": 0.5227, - "hfopenllm_v2/MATH Level 5": 0.074, - "hfopenllm_v2/GPQA": 0.2819, - "hfopenllm_v2/MUSR": 0.4459, - "hfopenllm_v2/MMLU-PRO": 0.3102 - } - }, - { - "id": "tangledgroup/tangled-llama-pints-1.5b-v0.1-instruct", - "name": "tangled-llama-pints-1.5b-v0.1-instruct", - "developer": "tangledgroup", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1509, - "hfopenllm_v2/BBH": 0.3143, - "hfopenllm_v2/MATH Level 5": 0.0121, - "hfopenllm_v2/GPQA": 0.2399, - "hfopenllm_v2/MUSR": 0.3761, - "hfopenllm_v2/MMLU-PRO": 0.1109 - } - }, - { - "id": "tangledgroup/tangled-llama-pints-1.5b-v0.2-instruct", - "name": "tangled-llama-pints-1.5b-v0.2-instruct", - "developer": "tangledgroup", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1724, - "hfopenllm_v2/BBH": 0.3158, - "hfopenllm_v2/MATH Level 5": 0.0128, - "hfopenllm_v2/GPQA": 0.2416, - "hfopenllm_v2/MUSR": 0.3643, - "hfopenllm_v2/MMLU-PRO": 0.1117 - } - }, - { - "id": "tanliboy/lambda-gemma-2-9b-dpo", - "name": "lambda-gemma-2-9b-dpo", - "developer": "tanliboy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4501, - "hfopenllm_v2/BBH": 0.5472, - "hfopenllm_v2/MATH Level 5": 0.0944, - "hfopenllm_v2/GPQA": 0.3138, - "hfopenllm_v2/MUSR": 0.4017, - "hfopenllm_v2/MMLU-PRO": 0.3792 - } - }, - { - "id": "tanliboy/lambda-qwen2.5-14b-dpo-test", - "name": "lambda-qwen2.5-14b-dpo-test", - "developer": "tanliboy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8231, - "hfopenllm_v2/BBH": 0.6394, - "hfopenllm_v2/MATH Level 5": 0.5461, - "hfopenllm_v2/GPQA": 0.3624, - "hfopenllm_v2/MUSR": 0.426, - "hfopenllm_v2/MMLU-PRO": 0.4848 - } - }, - { - "id": "tanliboy/lambda-qwen2.5-32b-dpo-test", - "name": "lambda-qwen2.5-32b-dpo-test", - "developer": "tanliboy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8084, - "hfopenllm_v2/BBH": 0.6764, - "hfopenllm_v2/MATH Level 5": 0.6103, - "hfopenllm_v2/GPQA": 0.3565, - "hfopenllm_v2/MUSR": 0.4274, - "hfopenllm_v2/MMLU-PRO": 0.5657 - } - }, - { - "id": "tannedbum/Ellaria-9B", - "name": "Ellaria-9B", - "developer": "tannedbum", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7826, - "hfopenllm_v2/BBH": 0.5942, - "hfopenllm_v2/MATH Level 5": 0.2077, - "hfopenllm_v2/GPQA": 0.3331, - "hfopenllm_v2/MUSR": 0.4151, - "hfopenllm_v2/MMLU-PRO": 0.4205 - } - }, - { - "id": "tannedbum/L3-Nymeria-Maid-8B", - "name": "L3-Nymeria-Maid-8B", - "developer": "tannedbum", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.725, - "hfopenllm_v2/BBH": 0.5146, - "hfopenllm_v2/MATH Level 5": 0.0937, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.3751, - "hfopenllm_v2/MMLU-PRO": 0.3747 - } - }, - { - "id": "tannedbum/L3-Nymeria-v2-8B", - "name": "L3-Nymeria-v2-8B", - "developer": "tannedbum", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7168, - "hfopenllm_v2/BBH": 0.5224, - "hfopenllm_v2/MATH Level 5": 0.0921, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.3699, - "hfopenllm_v2/MMLU-PRO": 0.3753 - } - }, - { - "id": "tannedbum/L3-Rhaenys-8B", - "name": "L3-Rhaenys-8B", - "developer": "tannedbum", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7363, - "hfopenllm_v2/BBH": 0.5299, - "hfopenllm_v2/MATH Level 5": 0.0876, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.3725, - "hfopenllm_v2/MMLU-PRO": 0.3799 - } - }, - { - "id": "Tarek07/Progenitor-V1.1-LLaMa-70B", - "name": "Progenitor-V1.1-LLaMa-70B", - "developer": "Tarek07", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6906, - "hfopenllm_v2/BBH": 0.6971, - "hfopenllm_v2/MATH Level 5": 0.3573, - "hfopenllm_v2/GPQA": 0.4581, - "hfopenllm_v2/MUSR": 0.4736, - "hfopenllm_v2/MMLU-PRO": 0.5465 - } - }, - { - "id": "Tarek07/Thalassic-Alpha-LLaMa-70B", - "name": "Thalassic-Alpha-LLaMa-70B", - "developer": "Tarek07", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7003, - "hfopenllm_v2/BBH": 0.694, - "hfopenllm_v2/MATH Level 5": 0.315, - "hfopenllm_v2/GPQA": 0.4438, - "hfopenllm_v2/MUSR": 0.4802, - "hfopenllm_v2/MMLU-PRO": 0.5435 - } - }, - { - "id": "TeeZee/DoubleBagel-57B-v1.0", - "name": "DoubleBagel-57B-v1.0", - "developer": "TeeZee", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2336, - "hfopenllm_v2/BBH": 0.3251, - "hfopenllm_v2/MATH Level 5": 0.0098, - "hfopenllm_v2/GPQA": 0.276, - "hfopenllm_v2/MUSR": 0.4315, - "hfopenllm_v2/MMLU-PRO": 0.1478 - } - }, - { - "id": "teknium/CollectiveCognition-v1.1-Mistral-7B", - "name": "CollectiveCognition-v1.1-Mistral-7B", - "developer": "teknium", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.279, - "hfopenllm_v2/BBH": 0.4493, - "hfopenllm_v2/MATH Level 5": 0.031, - "hfopenllm_v2/GPQA": 0.2869, - "hfopenllm_v2/MUSR": 0.3869, - "hfopenllm_v2/MMLU-PRO": 0.2837 - } - }, - { - "id": "teknium/OpenHermes-13B", - "name": "OpenHermes-13B", - "developer": "teknium", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2668, - "hfopenllm_v2/BBH": 0.4206, - "hfopenllm_v2/MATH Level 5": 0.0121, - "hfopenllm_v2/GPQA": 0.2727, - "hfopenllm_v2/MUSR": 0.4043, - "hfopenllm_v2/MMLU-PRO": 0.2389 - } - }, - { - "id": "teknium/OpenHermes-2-Mistral-7B", - "name": "OpenHermes-2-Mistral-7B", - "developer": "teknium", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5286, - "hfopenllm_v2/BBH": 0.4948, - "hfopenllm_v2/MATH Level 5": 0.0453, - "hfopenllm_v2/GPQA": 0.2836, - "hfopenllm_v2/MUSR": 0.452, - "hfopenllm_v2/MMLU-PRO": 0.2931 - } - }, - { - "id": "teknium/OpenHermes-2.5-Mistral-7B", - "name": "OpenHermes-2.5-Mistral-7B", - "developer": "teknium", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5571, - "hfopenllm_v2/BBH": 0.487, - "hfopenllm_v2/MATH Level 5": 0.0506, - "hfopenllm_v2/GPQA": 0.2836, - "hfopenllm_v2/MUSR": 0.4242, - "hfopenllm_v2/MMLU-PRO": 0.3054 - } - }, - { - "id": "teknium/OpenHermes-7B", - "name": "OpenHermes-7B", - "developer": "teknium", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1813, - "hfopenllm_v2/BBH": 0.362, - "hfopenllm_v2/MATH Level 5": 0.0159, - "hfopenllm_v2/GPQA": 0.2693, - "hfopenllm_v2/MUSR": 0.4324, - "hfopenllm_v2/MMLU-PRO": 0.1933 - } - }, - { - "id": "Telugu-LLM-Labs/Indic-gemma-2b-finetuned-sft-Navarasa-2.0", - "name": "Indic-gemma-2b-finetuned-sft-Navarasa-2.0", - "developer": "Telugu-LLM-Labs", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2103, - "hfopenllm_v2/BBH": 0.3241, - "hfopenllm_v2/MATH Level 5": 0.0272, - "hfopenllm_v2/GPQA": 0.2433, - "hfopenllm_v2/MUSR": 0.3899, - "hfopenllm_v2/MMLU-PRO": 0.1279 - } - }, - { - "id": "Telugu-LLM-Labs/Indic-gemma-7b-finetuned-sft-Navarasa-2.0", - "name": "Indic-gemma-7b-finetuned-sft-Navarasa-2.0", - "developer": "Telugu-LLM-Labs", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3237, - "hfopenllm_v2/BBH": 0.4023, - "hfopenllm_v2/MATH Level 5": 0.0257, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.4083, - "hfopenllm_v2/MMLU-PRO": 0.235 - } - }, - { - "id": "TencentARC/LLaMA-Pro-8B", - "name": "LLaMA-Pro-8B", - "developer": "TencentARC", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2277, - "hfopenllm_v2/BBH": 0.3484, - "hfopenllm_v2/MATH Level 5": 0.0189, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.4018, - "hfopenllm_v2/MMLU-PRO": 0.1811 - } - }, - { - "id": "TencentARC/LLaMA-Pro-8B-Instruct", - "name": "LLaMA-Pro-8B-Instruct", - "developer": "TencentARC", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4486, - "hfopenllm_v2/BBH": 0.4224, - "hfopenllm_v2/MATH Level 5": 0.0249, - "hfopenllm_v2/GPQA": 0.2743, - "hfopenllm_v2/MUSR": 0.419, - "hfopenllm_v2/MMLU-PRO": 0.1946 - } - }, - { - "id": "TencentARC/MetaMath-Mistral-Pro", - "name": "MetaMath-Mistral-Pro", - "developer": "TencentARC", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2119, - "hfopenllm_v2/BBH": 0.4413, - "hfopenllm_v2/MATH Level 5": 0.0763, - "hfopenllm_v2/GPQA": 0.2693, - "hfopenllm_v2/MUSR": 0.3524, - "hfopenllm_v2/MMLU-PRO": 0.2472 - } - }, - { - "id": "TencentARC/Mistral_Pro_8B_v0.1", - "name": "Mistral_Pro_8B_v0.1", - "developer": "TencentARC", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2115, - "hfopenllm_v2/BBH": 0.4526, - "hfopenllm_v2/MATH Level 5": 0.0566, - "hfopenllm_v2/GPQA": 0.2802, - "hfopenllm_v2/MUSR": 0.4242, - "hfopenllm_v2/MMLU-PRO": 0.2765 - } - }, - { - "id": "tensopolis/falcon3-10b-tensopolis-v1", - "name": "falcon3-10b-tensopolis-v1", - "developer": "tensopolis", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7817, - "hfopenllm_v2/BBH": 0.6182, - "hfopenllm_v2/MATH Level 5": 0.2749, - "hfopenllm_v2/GPQA": 0.3297, - "hfopenllm_v2/MUSR": 0.4375, - "hfopenllm_v2/MMLU-PRO": 0.442 - } - }, - { - "id": "tensopolis/falcon3-10b-tensopolis-v2", - "name": "falcon3-10b-tensopolis-v2", - "developer": "tensopolis", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7792, - "hfopenllm_v2/BBH": 0.6182, - "hfopenllm_v2/MATH Level 5": 0.2666, - "hfopenllm_v2/GPQA": 0.3272, - "hfopenllm_v2/MUSR": 0.4297, - "hfopenllm_v2/MMLU-PRO": 0.4424 - } - }, - { - "id": "tensopolis/lamarckvergence-14b-tensopolis-v1", - "name": "lamarckvergence-14b-tensopolis-v1", - "developer": "tensopolis", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7604, - "hfopenllm_v2/BBH": 0.6561, - "hfopenllm_v2/MATH Level 5": 0.5166, - "hfopenllm_v2/GPQA": 0.3607, - "hfopenllm_v2/MUSR": 0.4475, - "hfopenllm_v2/MMLU-PRO": 0.525 - } - }, - { - "id": "tensopolis/mistral-small-2501-tensopolis-v1", - "name": "mistral-small-2501-tensopolis-v1", - "developer": "tensopolis", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7762, - "hfopenllm_v2/BBH": 0.6475, - "hfopenllm_v2/MATH Level 5": 0.4441, - "hfopenllm_v2/GPQA": 0.3574, - "hfopenllm_v2/MUSR": 0.428, - "hfopenllm_v2/MMLU-PRO": 0.4465 - } - }, - { - "id": "tensopolis/mistral-small-r1-tensopolis", - "name": "mistral-small-r1-tensopolis", - "developer": "tensopolis", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4622, - "hfopenllm_v2/BBH": 0.5436, - "hfopenllm_v2/MATH Level 5": 0.2908, - "hfopenllm_v2/GPQA": 0.2819, - "hfopenllm_v2/MUSR": 0.3738, - "hfopenllm_v2/MMLU-PRO": 0.4035 - } - }, - { - "id": "tensopolis/phi-4-tensopolis-v1", - "name": "phi-4-tensopolis-v1", - "developer": "tensopolis", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6767, - "hfopenllm_v2/BBH": 0.6872, - "hfopenllm_v2/MATH Level 5": 0.494, - "hfopenllm_v2/GPQA": 0.3347, - "hfopenllm_v2/MUSR": 0.4141, - "hfopenllm_v2/MMLU-PRO": 0.5384 - } - }, - { - "id": "tensopolis/qwen2.5-14b-tensopolis-v1", - "name": "qwen2.5-14b-tensopolis-v1", - "developer": "tensopolis", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.799, - "hfopenllm_v2/BBH": 0.6364, - "hfopenllm_v2/MATH Level 5": 0.5295, - "hfopenllm_v2/GPQA": 0.3347, - "hfopenllm_v2/MUSR": 0.4193, - "hfopenllm_v2/MMLU-PRO": 0.4911 - } - }, - { - "id": "tensopolis/qwen2.5-3b-or1-tensopolis", - "name": "qwen2.5-3b-or1-tensopolis", - "developer": "tensopolis", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.354, - "hfopenllm_v2/BBH": 0.4421, - "hfopenllm_v2/MATH Level 5": 0.173, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.3749, - "hfopenllm_v2/MMLU-PRO": 0.3197 - } - }, - { - "id": "tensopolis/qwen2.5-7b-tensopolis-v1", - "name": "qwen2.5-7b-tensopolis-v1", - "developer": "tensopolis", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7661, - "hfopenllm_v2/BBH": 0.5379, - "hfopenllm_v2/MATH Level 5": 0.4562, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.4339, - "hfopenllm_v2/MMLU-PRO": 0.4269 - } - }, - { - "id": "tensopolis/qwen2.5-7b-tensopolis-v2", - "name": "qwen2.5-7b-tensopolis-v2", - "developer": "tensopolis", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7521, - "hfopenllm_v2/BBH": 0.5415, - "hfopenllm_v2/MATH Level 5": 0.4819, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.4246, - "hfopenllm_v2/MMLU-PRO": 0.4243 - } - }, - { - "id": "tensopolis/virtuoso-lite-tensopolis-v1", - "name": "virtuoso-lite-tensopolis-v1", - "developer": "tensopolis", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8069, - "hfopenllm_v2/BBH": 0.6102, - "hfopenllm_v2/MATH Level 5": 0.2545, - "hfopenllm_v2/GPQA": 0.3448, - "hfopenllm_v2/MUSR": 0.4582, - "hfopenllm_v2/MMLU-PRO": 0.4435 - } - }, - { - "id": "tensopolis/virtuoso-lite-tensopolis-v2", - "name": "virtuoso-lite-tensopolis-v2", - "developer": "tensopolis", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8029, - "hfopenllm_v2/BBH": 0.61, - "hfopenllm_v2/MATH Level 5": 0.25, - "hfopenllm_v2/GPQA": 0.3431, - "hfopenllm_v2/MUSR": 0.4595, - "hfopenllm_v2/MMLU-PRO": 0.444 - } - }, - { - "id": "tensopolis/virtuoso-small-tensopolis-v1", - "name": "virtuoso-small-tensopolis-v1", - "developer": "tensopolis", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7856, - "hfopenllm_v2/BBH": 0.6415, - "hfopenllm_v2/MATH Level 5": 0.3527, - "hfopenllm_v2/GPQA": 0.328, - "hfopenllm_v2/MUSR": 0.4326, - "hfopenllm_v2/MMLU-PRO": 0.4968 - } - }, - { - "id": "tensopolis/virtuoso-small-tensopolis-v2", - "name": "virtuoso-small-tensopolis-v2", - "developer": "tensopolis", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.802, - "hfopenllm_v2/BBH": 0.6516, - "hfopenllm_v2/MATH Level 5": 0.3875, - "hfopenllm_v2/GPQA": 0.3289, - "hfopenllm_v2/MUSR": 0.4352, - "hfopenllm_v2/MMLU-PRO": 0.5154 - } - }, - { - "id": "tensopolis/virtuoso-small-v2-tensopolis-v1", - "name": "virtuoso-small-v2-tensopolis-v1", - "developer": "tensopolis", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8419, - "hfopenllm_v2/BBH": 0.6545, - "hfopenllm_v2/MATH Level 5": 0.4524, - "hfopenllm_v2/GPQA": 0.3465, - "hfopenllm_v2/MUSR": 0.4509, - "hfopenllm_v2/MMLU-PRO": 0.5175 - } - }, - { - "id": "tensoropera/Fox-1-1.6B", - "name": "Fox-1-1.6B", - "developer": "tensoropera", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2766, - "hfopenllm_v2/BBH": 0.3307, - "hfopenllm_v2/MATH Level 5": 0.0174, - "hfopenllm_v2/GPQA": 0.2634, - "hfopenllm_v2/MUSR": 0.355, - "hfopenllm_v2/MMLU-PRO": 0.1371 - } - }, - { - "id": "tenyx/Llama3-TenyxChat-70B", - "name": "Llama3-TenyxChat-70B", - "developer": "tenyx", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8087, - "hfopenllm_v2/BBH": 0.6511, - "hfopenllm_v2/MATH Level 5": 0.2356, - "hfopenllm_v2/GPQA": 0.3012, - "hfopenllm_v2/MUSR": 0.426, - "hfopenllm_v2/MMLU-PRO": 0.521 - } - }, - { - "id": "TheDrummer/Cydonia-22B-v1.2", - "name": "Cydonia-22B-v1.2", - "developer": "TheDrummer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5635, - "hfopenllm_v2/BBH": 0.5809, - "hfopenllm_v2/MATH Level 5": 0.2032, - "hfopenllm_v2/GPQA": 0.3305, - "hfopenllm_v2/MUSR": 0.4022, - "hfopenllm_v2/MMLU-PRO": 0.4141 - } - }, - { - "id": "TheDrummer/Gemmasutra-9B-v1", - "name": "Gemmasutra-9B-v1", - "developer": "TheDrummer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2416, - "hfopenllm_v2/BBH": 0.5887, - "hfopenllm_v2/MATH Level 5": 0.0831, - "hfopenllm_v2/GPQA": 0.3104, - "hfopenllm_v2/MUSR": 0.4846, - "hfopenllm_v2/MMLU-PRO": 0.4045 - } - }, - { - "id": "TheDrummer/Gemmasutra-Mini-2B-v1", - "name": "Gemmasutra-Mini-2B-v1", - "developer": "TheDrummer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2549, - "hfopenllm_v2/BBH": 0.3575, - "hfopenllm_v2/MATH Level 5": 0.0378, - "hfopenllm_v2/GPQA": 0.271, - "hfopenllm_v2/MUSR": 0.349, - "hfopenllm_v2/MMLU-PRO": 0.2055 - } - }, - { - "id": "TheDrummer/Llama-3SOME-8B-v2", - "name": "Llama-3SOME-8B-v2", - "developer": "TheDrummer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4508, - "hfopenllm_v2/BBH": 0.5203, - "hfopenllm_v2/MATH Level 5": 0.0937, - "hfopenllm_v2/GPQA": 0.302, - "hfopenllm_v2/MUSR": 0.3833, - "hfopenllm_v2/MMLU-PRO": 0.3753 - } - }, - { - "id": "TheDrummer/Ministrations-8B-v1", - "name": "Ministrations-8B-v1", - "developer": "TheDrummer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2822, - "hfopenllm_v2/BBH": 0.4877, - "hfopenllm_v2/MATH Level 5": 0.1843, - "hfopenllm_v2/GPQA": 0.3247, - "hfopenllm_v2/MUSR": 0.4449, - "hfopenllm_v2/MMLU-PRO": 0.3644 - } - }, - { - "id": "TheDrummer/Rocinante-12B-v1", - "name": "Rocinante-12B-v1", - "developer": "TheDrummer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6076, - "hfopenllm_v2/BBH": 0.5065, - "hfopenllm_v2/MATH Level 5": 0.1269, - "hfopenllm_v2/GPQA": 0.2911, - "hfopenllm_v2/MUSR": 0.4017, - "hfopenllm_v2/MMLU-PRO": 0.3477 - } - }, - { - "id": "TheDrummer/Tiger-Gemma-9B-v1", - "name": "Tiger-Gemma-9B-v1", - "developer": "TheDrummer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7282, - "hfopenllm_v2/BBH": 0.5704, - "hfopenllm_v2/MATH Level 5": 0.1835, - "hfopenllm_v2/GPQA": 0.3389, - "hfopenllm_v2/MUSR": 0.4162, - "hfopenllm_v2/MMLU-PRO": 0.4118 - } - }, - { - "id": "TheDrummer/Tiger-Gemma-9B-v2", - "name": "Tiger-Gemma-9B-v2", - "developer": "TheDrummer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6986, - "hfopenllm_v2/BBH": 0.5617, - "hfopenllm_v2/MATH Level 5": 0.182, - "hfopenllm_v2/GPQA": 0.3398, - "hfopenllm_v2/MUSR": 0.4084, - "hfopenllm_v2/MMLU-PRO": 0.4112 - } - }, - { - "id": "TheDrummer/Tiger-Gemma-9B-v3", - "name": "Tiger-Gemma-9B-v3", - "developer": "TheDrummer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6821, - "hfopenllm_v2/BBH": 0.5812, - "hfopenllm_v2/MATH Level 5": 0.1624, - "hfopenllm_v2/GPQA": 0.3389, - "hfopenllm_v2/MUSR": 0.4004, - "hfopenllm_v2/MMLU-PRO": 0.4059 - } - }, - { - "id": "TheDrunkenSnail/Daughter-of-Rhodia-12B", - "name": "Daughter-of-Rhodia-12B", - "developer": "TheDrunkenSnail", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6904, - "hfopenllm_v2/BBH": 0.5179, - "hfopenllm_v2/MATH Level 5": 0.1224, - "hfopenllm_v2/GPQA": 0.3171, - "hfopenllm_v2/MUSR": 0.4348, - "hfopenllm_v2/MMLU-PRO": 0.3641 - } - }, - { - "id": "TheDrunkenSnail/Mother-of-Rhodia-12B", - "name": "Mother-of-Rhodia-12B", - "developer": "TheDrunkenSnail", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6505, - "hfopenllm_v2/BBH": 0.4948, - "hfopenllm_v2/MATH Level 5": 0.1224, - "hfopenllm_v2/GPQA": 0.2987, - "hfopenllm_v2/MUSR": 0.4124, - "hfopenllm_v2/MMLU-PRO": 0.3551 - } - }, - { - "id": "TheDrunkenSnail/Son-of-Rhodia", - "name": "Son-of-Rhodia", - "developer": "TheDrunkenSnail", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7046, - "hfopenllm_v2/BBH": 0.5097, - "hfopenllm_v2/MATH Level 5": 0.1314, - "hfopenllm_v2/GPQA": 0.3129, - "hfopenllm_v2/MUSR": 0.4203, - "hfopenllm_v2/MMLU-PRO": 0.3608 - } - }, - { - "id": "TheHierophant/Underground-Cognitive-V0.3-test", - "name": "Underground-Cognitive-V0.3-test", - "developer": "TheHierophant", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4808, - "hfopenllm_v2/BBH": 0.529, - "hfopenllm_v2/MATH Level 5": 0.0589, - "hfopenllm_v2/GPQA": 0.2987, - "hfopenllm_v2/MUSR": 0.4351, - "hfopenllm_v2/MMLU-PRO": 0.3318 - } - }, - { - "id": "theo77186/Qwen2.5-Coder-7B-Instruct-20241106", - "name": "Qwen2.5-Coder-7B-Instruct-20241106", - "developer": "theo77186", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6101, - "hfopenllm_v2/BBH": 0.5008, - "hfopenllm_v2/MATH Level 5": 0.3882, - "hfopenllm_v2/GPQA": 0.2919, - "hfopenllm_v2/MUSR": 0.4073, - "hfopenllm_v2/MMLU-PRO": 0.3353 - } - }, - { - "id": "theprint/Boptruth-Agatha-7B", - "name": "Boptruth-Agatha-7B", - "developer": "theprint", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3124, - "hfopenllm_v2/BBH": 0.4984, - "hfopenllm_v2/MATH Level 5": 0.0551, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.4277, - "hfopenllm_v2/MMLU-PRO": 0.2861 - } - }, - { - "id": "theprint/CleverBoi-7B-v2", - "name": "CleverBoi-7B-v2", - "developer": "theprint", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.217, - "hfopenllm_v2/BBH": 0.4532, - "hfopenllm_v2/MATH Level 5": 0.0264, - "hfopenllm_v2/GPQA": 0.2886, - "hfopenllm_v2/MUSR": 0.4695, - "hfopenllm_v2/MMLU-PRO": 0.2709 - } - }, - { - "id": "theprint/CleverBoi-7B-v3", - "name": "CleverBoi-7B-v3", - "developer": "theprint", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2382, - "hfopenllm_v2/BBH": 0.4414, - "hfopenllm_v2/MATH Level 5": 0.04, - "hfopenllm_v2/GPQA": 0.2659, - "hfopenllm_v2/MUSR": 0.4072, - "hfopenllm_v2/MMLU-PRO": 0.2868 - } - }, - { - "id": "theprint/CleverBoi-Llama-3.1-8B-Instruct", - "name": "CleverBoi-Llama-3.1-8B-Instruct", - "developer": "theprint", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1682, - "hfopenllm_v2/BBH": 0.456, - "hfopenllm_v2/MATH Level 5": 0.0491, - "hfopenllm_v2/GPQA": 0.3003, - "hfopenllm_v2/MUSR": 0.4014, - "hfopenllm_v2/MMLU-PRO": 0.3075 - } - }, - { - "id": "theprint/CleverBoi-Llama-3.1-8B-v2", - "name": "CleverBoi-Llama-3.1-8B-v2", - "developer": "theprint", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1961, - "hfopenllm_v2/BBH": 0.4668, - "hfopenllm_v2/MATH Level 5": 0.0529, - "hfopenllm_v2/GPQA": 0.2861, - "hfopenllm_v2/MUSR": 0.3735, - "hfopenllm_v2/MMLU-PRO": 0.3188 - } - }, - { - "id": "theprint/CleverBoi-Nemo-12B-v2", - "name": "CleverBoi-Nemo-12B-v2", - "developer": "theprint", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2046, - "hfopenllm_v2/BBH": 0.5241, - "hfopenllm_v2/MATH Level 5": 0.1035, - "hfopenllm_v2/GPQA": 0.3138, - "hfopenllm_v2/MUSR": 0.4187, - "hfopenllm_v2/MMLU-PRO": 0.3228 - } - }, - { - "id": "theprint/Code-Llama-Bagel-8B", - "name": "Code-Llama-Bagel-8B", - "developer": "theprint", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.253, - "hfopenllm_v2/BBH": 0.4697, - "hfopenllm_v2/MATH Level 5": 0.0612, - "hfopenllm_v2/GPQA": 0.276, - "hfopenllm_v2/MUSR": 0.368, - "hfopenllm_v2/MMLU-PRO": 0.2822 - } - }, - { - "id": "theprint/Conversely-Mistral-7B", - "name": "Conversely-Mistral-7B", - "developer": "theprint", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2608, - "hfopenllm_v2/BBH": 0.4672, - "hfopenllm_v2/MATH Level 5": 0.0279, - "hfopenllm_v2/GPQA": 0.2852, - "hfopenllm_v2/MUSR": 0.4189, - "hfopenllm_v2/MMLU-PRO": 0.2826 - } - }, - { - "id": "theprint/Llama-3.2-3B-VanRossum", - "name": "Llama-3.2-3B-VanRossum", - "developer": "theprint", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4783, - "hfopenllm_v2/BBH": 0.4279, - "hfopenllm_v2/MATH Level 5": 0.0974, - "hfopenllm_v2/GPQA": 0.2676, - "hfopenllm_v2/MUSR": 0.3442, - "hfopenllm_v2/MMLU-PRO": 0.277 - } - }, - { - "id": "theprint/phi-3-mini-4k-python", - "name": "phi-3-mini-4k-python", - "developer": "theprint", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2409, - "hfopenllm_v2/BBH": 0.4938, - "hfopenllm_v2/MATH Level 5": 0.105, - "hfopenllm_v2/GPQA": 0.2911, - "hfopenllm_v2/MUSR": 0.3922, - "hfopenllm_v2/MMLU-PRO": 0.3577 - } - }, - { - "id": "theprint/ReWiz-7B", - "name": "ReWiz-7B", - "developer": "theprint", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4048, - "hfopenllm_v2/BBH": 0.4564, - "hfopenllm_v2/MATH Level 5": 0.0408, - "hfopenllm_v2/GPQA": 0.2752, - "hfopenllm_v2/MUSR": 0.4612, - "hfopenllm_v2/MMLU-PRO": 0.267 - } - }, - { - "id": "theprint/ReWiz-Llama-3.1-8B-v2", - "name": "ReWiz-Llama-3.1-8B-v2", - "developer": "theprint", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2379, - "hfopenllm_v2/BBH": 0.4632, - "hfopenllm_v2/MATH Level 5": 0.0574, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.3814, - "hfopenllm_v2/MMLU-PRO": 0.331 - } - }, - { - "id": "theprint/ReWiz-Llama-3.2-3B", - "name": "ReWiz-Llama-3.2-3B", - "developer": "theprint", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4649, - "hfopenllm_v2/BBH": 0.4343, - "hfopenllm_v2/MATH Level 5": 0.1095, - "hfopenllm_v2/GPQA": 0.2836, - "hfopenllm_v2/MUSR": 0.3614, - "hfopenllm_v2/MMLU-PRO": 0.2887 - } - }, - { - "id": "theprint/ReWiz-Nemo-12B-Instruct", - "name": "ReWiz-Nemo-12B-Instruct", - "developer": "theprint", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1062, - "hfopenllm_v2/BBH": 0.5092, - "hfopenllm_v2/MATH Level 5": 0.1042, - "hfopenllm_v2/GPQA": 0.3238, - "hfopenllm_v2/MUSR": 0.4096, - "hfopenllm_v2/MMLU-PRO": 0.3339 - } - }, - { - "id": "theprint/ReWiz-Qwen-2.5-14B", - "name": "ReWiz-Qwen-2.5-14B", - "developer": "theprint", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2785, - "hfopenllm_v2/BBH": 0.6179, - "hfopenllm_v2/MATH Level 5": 0.2923, - "hfopenllm_v2/GPQA": 0.38, - "hfopenllm_v2/MUSR": 0.4539, - "hfopenllm_v2/MMLU-PRO": 0.5092 - } - }, - { - "id": "theprint/ReWiz-Worldbuilder-7B", - "name": "ReWiz-Worldbuilder-7B", - "developer": "theprint", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.251, - "hfopenllm_v2/BBH": 0.4636, - "hfopenllm_v2/MATH Level 5": 0.037, - "hfopenllm_v2/GPQA": 0.2693, - "hfopenllm_v2/MUSR": 0.4572, - "hfopenllm_v2/MMLU-PRO": 0.2971 - } - }, - { - "id": "theprint/RuDolph-Hermes-7B", - "name": "RuDolph-Hermes-7B", - "developer": "theprint", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3604, - "hfopenllm_v2/BBH": 0.5053, - "hfopenllm_v2/MATH Level 5": 0.0514, - "hfopenllm_v2/GPQA": 0.3121, - "hfopenllm_v2/MUSR": 0.4226, - "hfopenllm_v2/MMLU-PRO": 0.3073 - } - }, - { - "id": "theprint/WorldBuilder-12B", - "name": "WorldBuilder-12B", - "developer": "theprint", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1374, - "hfopenllm_v2/BBH": 0.501, - "hfopenllm_v2/MATH Level 5": 0.0446, - "hfopenllm_v2/GPQA": 0.297, - "hfopenllm_v2/MUSR": 0.4066, - "hfopenllm_v2/MMLU-PRO": 0.3192 - } - }, - { - "id": "TheTsar1209/nemo-carpmuscle-v0.1", - "name": "nemo-carpmuscle-v0.1", - "developer": "TheTsar1209", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2276, - "hfopenllm_v2/BBH": 0.5084, - "hfopenllm_v2/MATH Level 5": 0.0476, - "hfopenllm_v2/GPQA": 0.297, - "hfopenllm_v2/MUSR": 0.4135, - "hfopenllm_v2/MMLU-PRO": 0.3406 - } - }, - { - "id": "TheTsar1209/qwen-carpmuscle-r-v0.3", - "name": "qwen-carpmuscle-r-v0.3", - "developer": "TheTsar1209", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4455, - "hfopenllm_v2/BBH": 0.6227, - "hfopenllm_v2/MATH Level 5": 0.3006, - "hfopenllm_v2/GPQA": 0.3507, - "hfopenllm_v2/MUSR": 0.4278, - "hfopenllm_v2/MMLU-PRO": 0.5103 - } - }, - { - "id": "TheTsar1209/qwen-carpmuscle-v0.1", - "name": "qwen-carpmuscle-v0.1", - "developer": "TheTsar1209", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5622, - "hfopenllm_v2/BBH": 0.6434, - "hfopenllm_v2/MATH Level 5": 0.2628, - "hfopenllm_v2/GPQA": 0.344, - "hfopenllm_v2/MUSR": 0.4161, - "hfopenllm_v2/MMLU-PRO": 0.52 - } - }, - { - "id": "TheTsar1209/qwen-carpmuscle-v0.2", - "name": "qwen-carpmuscle-v0.2", - "developer": "TheTsar1209", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5257, - "hfopenllm_v2/BBH": 0.6387, - "hfopenllm_v2/MATH Level 5": 0.2832, - "hfopenllm_v2/GPQA": 0.3557, - "hfopenllm_v2/MUSR": 0.4346, - "hfopenllm_v2/MMLU-PRO": 0.5147 - } - }, - { - "id": "TheTsar1209/qwen-carpmuscle-v0.3", - "name": "qwen-carpmuscle-v0.3", - "developer": "TheTsar1209", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4476, - "hfopenllm_v2/BBH": 0.6152, - "hfopenllm_v2/MATH Level 5": 0.3134, - "hfopenllm_v2/GPQA": 0.3565, - "hfopenllm_v2/MUSR": 0.4132, - "hfopenllm_v2/MMLU-PRO": 0.5062 - } - }, - { - "id": "TheTsar1209/qwen-carpmuscle-v0.4", - "name": "qwen-carpmuscle-v0.4", - "developer": "TheTsar1209", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7202, - "hfopenllm_v2/BBH": 0.6454, - "hfopenllm_v2/MATH Level 5": 0.2772, - "hfopenllm_v2/GPQA": 0.3523, - "hfopenllm_v2/MUSR": 0.4516, - "hfopenllm_v2/MMLU-PRO": 0.5144 - } - }, - { - "id": "TheTsar1209/qwen-carpmuscle-v0.4.1", - "name": "qwen-carpmuscle-v0.4.1", - "developer": "TheTsar1209", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.736, - "hfopenllm_v2/BBH": 0.6507, - "hfopenllm_v2/MATH Level 5": 0.2779, - "hfopenllm_v2/GPQA": 0.3456, - "hfopenllm_v2/MUSR": 0.4489, - "hfopenllm_v2/MMLU-PRO": 0.5191 - } - }, - { - "id": "thinkcoder/llama3-8b-instruct-lora-8-sft", - "name": "llama3-8b-instruct-lora-8-sft", - "developer": "thinkcoder", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.648, - "hfopenllm_v2/BBH": 0.4865, - "hfopenllm_v2/MATH Level 5": 0.102, - "hfopenllm_v2/GPQA": 0.2668, - "hfopenllm_v2/MUSR": 0.3235, - "hfopenllm_v2/MMLU-PRO": 0.3476 - } - }, - { - "id": "thirdeyeai/elevate360m", - "name": "elevate360m", - "developer": "thirdeyeai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0445, - "hfopenllm_v2/BBH": 0.2963, - "hfopenllm_v2/MATH Level 5": 0.0159, - "hfopenllm_v2/GPQA": 0.2408, - "hfopenllm_v2/MUSR": 0.3462, - "hfopenllm_v2/MMLU-PRO": 0.1077 - } - }, - { - "id": "thomas-yanxin/XinYuan-Qwen2-1_5B", - "name": "XinYuan-Qwen2-1_5B", - "developer": "thomas-yanxin", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2986, - "hfopenllm_v2/BBH": 0.3635, - "hfopenllm_v2/MATH Level 5": 0.0672, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.3634, - "hfopenllm_v2/MMLU-PRO": 0.2357 - } - }, - { - "id": "thomas-yanxin/XinYuan-Qwen2-7B", - "name": "XinYuan-Qwen2-7B", - "developer": "thomas-yanxin", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4438, - "hfopenllm_v2/BBH": 0.4937, - "hfopenllm_v2/MATH Level 5": 0.1458, - "hfopenllm_v2/GPQA": 0.2911, - "hfopenllm_v2/MUSR": 0.4058, - "hfopenllm_v2/MMLU-PRO": 0.3925 - } - }, - { - "id": "thomas-yanxin/XinYuan-Qwen2-7B-0917", - "name": "XinYuan-Qwen2-7B-0917", - "developer": "thomas-yanxin", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3719, - "hfopenllm_v2/BBH": 0.5169, - "hfopenllm_v2/MATH Level 5": 0.1979, - "hfopenllm_v2/GPQA": 0.3096, - "hfopenllm_v2/MUSR": 0.4401, - "hfopenllm_v2/MMLU-PRO": 0.4245 - } - }, - { - "id": "thomas-yanxin/XinYuan-Qwen2.5-7B-0917", - "name": "XinYuan-Qwen2.5-7B-0917", - "developer": "thomas-yanxin", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3577, - "hfopenllm_v2/BBH": 0.5184, - "hfopenllm_v2/MATH Level 5": 0.1934, - "hfopenllm_v2/GPQA": 0.281, - "hfopenllm_v2/MUSR": 0.3676, - "hfopenllm_v2/MMLU-PRO": 0.3882 - } - }, - { - "id": "THUDM/glm-4-9b", - "name": "glm-4-9b", - "developer": "THUDM", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1426, - "hfopenllm_v2/BBH": 0.5528, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.3163, - "hfopenllm_v2/MUSR": 0.4386, - "hfopenllm_v2/MMLU-PRO": 0.4145 - } - }, - { - "id": "THUDM/glm-4-9b-chat", - "name": "glm-4-9b-chat", - "developer": "THUDM", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0, - "hfopenllm_v2/BBH": 0.4736, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.3138, - "hfopenllm_v2/MUSR": 0.3994, - "hfopenllm_v2/MMLU-PRO": 0.3167 - } - }, - { - "id": "THUDM/glm-4-9b-chat-1m", - "name": "glm-4-9b-chat-1m", - "developer": "THUDM", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0, - "hfopenllm_v2/BBH": 0.418, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.3037, - "hfopenllm_v2/MUSR": 0.3795, - "hfopenllm_v2/MMLU-PRO": 0.3163 - } - }, - { - "id": "THUDM/glm-4-9b-chat-1m-hf", - "name": "glm-4-9b-chat-1m-hf", - "developer": "THUDM", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5341, - "hfopenllm_v2/BBH": 0.3901, - "hfopenllm_v2/MATH Level 5": 0.0483, - "hfopenllm_v2/GPQA": 0.2919, - "hfopenllm_v2/MUSR": 0.3689, - "hfopenllm_v2/MMLU-PRO": 0.1814 - } - }, - { - "id": "THUDM/glm-4-9b-chat-hf", - "name": "glm-4-9b-chat-hf", - "developer": "THUDM", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6513, - "hfopenllm_v2/BBH": 0.4432, - "hfopenllm_v2/MATH Level 5": 0.0846, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.3593, - "hfopenllm_v2/MMLU-PRO": 0.2774 - } - }, - { - "id": "tianyil1/MistralForCausalLM_Cal_DPO", - "name": "MistralForCausalLM_Cal_DPO", - "developer": "tianyil1", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5328, - "hfopenllm_v2/BBH": 0.4381, - "hfopenllm_v2/MATH Level 5": 0.0287, - "hfopenllm_v2/GPQA": 0.276, - "hfopenllm_v2/MUSR": 0.3977, - "hfopenllm_v2/MMLU-PRO": 0.2763 - } - }, - { - "id": "TIGER-Lab/AceCoder-Qwen2.5-7B-Ins-Rule", - "name": "AceCoder-Qwen2.5-7B-Ins-Rule", - "developer": "TIGER-Lab", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7424, - "hfopenllm_v2/BBH": 0.5404, - "hfopenllm_v2/MATH Level 5": 0.4992, - "hfopenllm_v2/GPQA": 0.3012, - "hfopenllm_v2/MUSR": 0.398, - "hfopenllm_v2/MMLU-PRO": 0.4322 - } - }, - { - "id": "TIGER-Lab/AceCoder-Qwen2.5-Coder-7B-Base-Rule", - "name": "AceCoder-Qwen2.5-Coder-7B-Base-Rule", - "developer": "TIGER-Lab", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4408, - "hfopenllm_v2/BBH": 0.4902, - "hfopenllm_v2/MATH Level 5": 0.2017, - "hfopenllm_v2/GPQA": 0.2718, - "hfopenllm_v2/MUSR": 0.3449, - "hfopenllm_v2/MMLU-PRO": 0.3745 - } - }, - { - "id": "TIGER-Lab/AceCoder-Qwen2.5-Coder-7B-Ins-Rule", - "name": "AceCoder-Qwen2.5-Coder-7B-Ins-Rule", - "developer": "TIGER-Lab", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6222, - "hfopenllm_v2/BBH": 0.5089, - "hfopenllm_v2/MATH Level 5": 0.3603, - "hfopenllm_v2/GPQA": 0.2777, - "hfopenllm_v2/MUSR": 0.4046, - "hfopenllm_v2/MMLU-PRO": 0.3428 - } - }, - { - "id": "TIGER-Lab/AceCodeRM-7B", - "name": "AceCodeRM-7B", - "developer": "TIGER-Lab", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5855, - "hfopenllm_v2/BBH": 0.4773, - "hfopenllm_v2/MATH Level 5": 0.3467, - "hfopenllm_v2/GPQA": 0.3045, - "hfopenllm_v2/MUSR": 0.4192, - "hfopenllm_v2/MMLU-PRO": 0.3361 - } - }, - { - "id": "TIGER-Lab/MAmmoTH2-7B-Plus", - "name": "MAmmoTH2-7B-Plus", - "developer": "TIGER-Lab", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5575, - "hfopenllm_v2/BBH": 0.4235, - "hfopenllm_v2/MATH Level 5": 0.1858, - "hfopenllm_v2/GPQA": 0.2802, - "hfopenllm_v2/MUSR": 0.4124, - "hfopenllm_v2/MMLU-PRO": 0.3017 - } - }, - { - "id": "TIGER-Lab/Qwen2.5-Math-7B-CFT", - "name": "Qwen2.5-Math-7B-CFT", - "developer": "TIGER-Lab", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2777, - "hfopenllm_v2/BBH": 0.4637, - "hfopenllm_v2/MATH Level 5": 0.5574, - "hfopenllm_v2/GPQA": 0.2861, - "hfopenllm_v2/MUSR": 0.3887, - "hfopenllm_v2/MMLU-PRO": 0.2945 - } - }, - { - "id": "tii-uae/falcon3-10b-instruct-fc", - "name": "Falcon3-10B-Instruct (FC)", - "developer": "tii-uae", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 82.0, - "bfcl/bfcl.overall.overall_accuracy": 27.01, - "bfcl/bfcl.overall.total_cost_usd": 52.59, - "bfcl/bfcl.overall.latency_mean_s": 69.27, - "bfcl/bfcl.overall.latency_std_s": 92.22, - "bfcl/bfcl.overall.latency_p95_s": 190.96, - "bfcl/bfcl.non_live.ast_accuracy": 85.0, - "bfcl/bfcl.non_live.simple_ast_accuracy": 70.5, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 93.5, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 88.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 87.5, - "bfcl/bfcl.live.live_accuracy": 75.43, - "bfcl/bfcl.live.live_simple_ast_accuracy": 77.13, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 76.16, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 50.0, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 41.67, - "bfcl/bfcl.multi_turn.accuracy": 6.5, - "bfcl/bfcl.multi_turn.base_accuracy": 6.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 9.5, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 5.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 5.0, - "bfcl/bfcl.web_search.accuracy": 1.5, - "bfcl/bfcl.web_search.base_accuracy": 2.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 1.0, - "bfcl/bfcl.memory.accuracy": 27.53, - "bfcl/bfcl.memory.kv_accuracy": 12.26, - "bfcl/bfcl.memory.vector_accuracy": 19.35, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 50.97, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 93.75, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 32.09 - } - }, - { - "id": "tii-uae/falcon3-1b-instruct-fc", - "name": "Falcon3-1B-Instruct (FC)", - "developer": "tii-uae", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 106.0, - "bfcl/bfcl.overall.overall_accuracy": 11.08, - "bfcl/bfcl.overall.total_cost_usd": 1.72, - "bfcl/bfcl.overall.latency_mean_s": 5.23, - "bfcl/bfcl.overall.latency_std_s": 14.34, - "bfcl/bfcl.overall.latency_p95_s": 11.48, - "bfcl/bfcl.non_live.ast_accuracy": 9.02, - "bfcl/bfcl.non_live.simple_ast_accuracy": 2.58, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 6.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 18.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 9.5, - "bfcl/bfcl.live.live_accuracy": 2.89, - "bfcl/bfcl.live.live_simple_ast_accuracy": 4.26, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 2.37, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 0.0, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 12.5, - "bfcl/bfcl.multi_turn.accuracy": 0.0, - "bfcl/bfcl.multi_turn.base_accuracy": 0.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 0.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 0.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 0.0, - "bfcl/bfcl.web_search.accuracy": 0.0, - "bfcl/bfcl.web_search.base_accuracy": 0.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 0.0, - "bfcl/bfcl.memory.accuracy": 5.81, - "bfcl/bfcl.memory.kv_accuracy": 5.16, - "bfcl/bfcl.memory.vector_accuracy": 7.74, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 4.52, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 0.0, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 87.3 - } - }, - { - "id": "tii-uae/falcon3-3b-instruct-fc", - "name": "Falcon3-3B-Instruct (FC)", - "developer": "tii-uae", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 104.0, - "bfcl/bfcl.overall.overall_accuracy": 16.25, - "bfcl/bfcl.overall.total_cost_usd": 36.7, - "bfcl/bfcl.overall.latency_mean_s": 38.52, - "bfcl/bfcl.overall.latency_std_s": 107.47, - "bfcl/bfcl.overall.latency_p95_s": 103.62, - "bfcl/bfcl.non_live.ast_accuracy": 54.62, - "bfcl/bfcl.non_live.simple_ast_accuracy": 56.5, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 69.5, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 67.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 25.5, - "bfcl/bfcl.live.live_accuracy": 54.48, - "bfcl/bfcl.live.live_simple_ast_accuracy": 57.36, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 54.7, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 25.0, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 33.33, - "bfcl/bfcl.multi_turn.accuracy": 1.0, - "bfcl/bfcl.multi_turn.base_accuracy": 1.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 0.5, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 0.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 1.5, - "bfcl/bfcl.web_search.accuracy": 1.0, - "bfcl/bfcl.web_search.base_accuracy": 1.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 1.0, - "bfcl/bfcl.memory.accuracy": 7.74, - "bfcl/bfcl.memory.kv_accuracy": 6.45, - "bfcl/bfcl.memory.vector_accuracy": 8.39, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 8.39, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 81.25, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 32.92 - } - }, - { - "id": "tii-uae/falcon3-7b-instruct-fc", - "name": "Falcon3-7B-Instruct (FC)", - "developer": "tii-uae", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 91.0, - "bfcl/bfcl.overall.overall_accuracy": 24.03, - "bfcl/bfcl.overall.total_cost_usd": 73.61, - "bfcl/bfcl.overall.latency_mean_s": 93.11, - "bfcl/bfcl.overall.latency_std_s": 117.8, - "bfcl/bfcl.overall.latency_p95_s": 315.7, - "bfcl/bfcl.non_live.ast_accuracy": 82.69, - "bfcl/bfcl.non_live.simple_ast_accuracy": 65.75, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 89.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 87.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 89.0, - "bfcl/bfcl.live.live_accuracy": 68.32, - "bfcl/bfcl.live.live_simple_ast_accuracy": 74.81, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 66.76, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 75.0, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 62.5, - "bfcl/bfcl.multi_turn.accuracy": 5.0, - "bfcl/bfcl.multi_turn.base_accuracy": 7.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 4.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 5.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 4.0, - "bfcl/bfcl.web_search.accuracy": 0.5, - "bfcl/bfcl.web_search.base_accuracy": 1.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 0.0, - "bfcl/bfcl.memory.accuracy": 20.65, - "bfcl/bfcl.memory.kv_accuracy": 10.32, - "bfcl/bfcl.memory.vector_accuracy": 12.9, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 38.71, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 100.0, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 31.99 - } - }, - { - "id": "tiiuae/falcon-11B", - "name": "falcon-11B", - "developer": "tiiuae", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3261, - "hfopenllm_v2/BBH": 0.4392, - "hfopenllm_v2/MATH Level 5": 0.0279, - "hfopenllm_v2/GPQA": 0.271, - "hfopenllm_v2/MUSR": 0.3986, - "hfopenllm_v2/MMLU-PRO": 0.2389 - } - }, - { - "id": "tiiuae/falcon-40b", - "name": "Falcon 40B", - "developer": "tiiuae", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.729, - "helm_classic/MMLU": 0.509, - "helm_classic/BoolQ": 0.819, - "helm_classic/NarrativeQA": 0.673, - "helm_classic/NaturalQuestions (open-book)": 0.675, - "helm_classic/QuAC": 0.307, - "helm_classic/HellaSwag": -1.0, - "helm_classic/OpenbookQA": -1.0, - "helm_classic/TruthfulQA": 0.353, - "helm_classic/MS MARCO (TREC)": -1.0, - "helm_classic/CNN/DailyMail": -1.0, - "helm_classic/XSUM": -1.0, - "helm_classic/IMDB": 0.959, - "helm_classic/CivilComments": 0.552, - "helm_classic/RAFT": 0.661, - "helm_lite/Mean win rate": 0.217, - "helm_lite/NarrativeQA": 0.671, - "helm_lite/NaturalQuestions (closed-book)": 0.392, - "helm_lite/OpenbookQA": 0.662, - "helm_lite/MMLU": 0.507, - "helm_lite/MATH": 0.128, - "helm_lite/GSM8K": 0.267, - "helm_lite/LegalBench": 0.442, - "helm_lite/MedQA": 0.419, - "helm_lite/WMT 2014": 0.162, - "hfopenllm_v2/IFEval": 0.2496, - "hfopenllm_v2/BBH": 0.4019, - "hfopenllm_v2/MATH Level 5": 0.0181, - "hfopenllm_v2/GPQA": 0.2735, - "hfopenllm_v2/MUSR": 0.3631, - "hfopenllm_v2/MMLU-PRO": 0.2505 - } - }, - { - "id": "tiiuae/falcon-40b-instruct", - "name": "falcon-40b-instruct", - "developer": "tiiuae", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2454, - "hfopenllm_v2/BBH": 0.4054, - "hfopenllm_v2/MATH Level 5": 0.0196, - "hfopenllm_v2/GPQA": 0.25, - "hfopenllm_v2/MUSR": 0.3762, - "hfopenllm_v2/MMLU-PRO": 0.2261 - } - }, - { - "id": "tiiuae/falcon-7b", - "name": "Falcon 7B", - "developer": "tiiuae", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.378, - "helm_classic/MMLU": 0.286, - "helm_classic/BoolQ": 0.753, - "helm_classic/NarrativeQA": 0.621, - "helm_classic/NaturalQuestions (open-book)": 0.579, - "helm_classic/QuAC": 0.332, - "helm_classic/HellaSwag": -1.0, - "helm_classic/OpenbookQA": -1.0, - "helm_classic/TruthfulQA": 0.234, - "helm_classic/MS MARCO (TREC)": -1.0, - "helm_classic/CNN/DailyMail": -1.0, - "helm_classic/XSUM": -1.0, - "helm_classic/IMDB": 0.836, - "helm_classic/CivilComments": 0.514, - "helm_classic/RAFT": 0.602, - "helm_lite/Mean win rate": 0.064, - "helm_lite/NarrativeQA": 0.621, - "helm_lite/NaturalQuestions (closed-book)": 0.285, - "helm_lite/OpenbookQA": 0.26, - "helm_lite/MMLU": 0.288, - "helm_lite/MATH": 0.044, - "helm_lite/GSM8K": 0.055, - "helm_lite/LegalBench": 0.346, - "helm_lite/MedQA": 0.254, - "helm_lite/WMT 2014": 0.094, - "hfopenllm_v2/IFEval": 0.1821, - "hfopenllm_v2/BBH": 0.3285, - "hfopenllm_v2/MATH Level 5": 0.0098, - "hfopenllm_v2/GPQA": 0.245, - "hfopenllm_v2/MUSR": 0.3778, - "hfopenllm_v2/MMLU-PRO": 0.1125 - } - }, - { - "id": "tiiuae/falcon-7b-instruct", - "name": "falcon-7b-instruct", - "developer": "tiiuae", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1969, - "hfopenllm_v2/BBH": 0.3203, - "hfopenllm_v2/MATH Level 5": 0.0121, - "hfopenllm_v2/GPQA": 0.2475, - "hfopenllm_v2/MUSR": 0.3634, - "hfopenllm_v2/MMLU-PRO": 0.1155 - } - }, - { - "id": "tiiuae/Falcon-Instruct-40B", - "name": "Falcon-Instruct 40B", - "developer": "tiiuae", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.727, - "helm_classic/MMLU": 0.497, - "helm_classic/BoolQ": 0.829, - "helm_classic/NarrativeQA": 0.625, - "helm_classic/NaturalQuestions (open-book)": 0.666, - "helm_classic/QuAC": 0.371, - "helm_classic/HellaSwag": -1.0, - "helm_classic/OpenbookQA": -1.0, - "helm_classic/TruthfulQA": 0.384, - "helm_classic/MS MARCO (TREC)": -1.0, - "helm_classic/CNN/DailyMail": -1.0, - "helm_classic/XSUM": -1.0, - "helm_classic/IMDB": 0.959, - "helm_classic/CivilComments": 0.603, - "helm_classic/RAFT": 0.586 - } - }, - { - "id": "tiiuae/Falcon-Instruct-7B", - "name": "Falcon-Instruct 7B", - "developer": "tiiuae", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.244, - "helm_classic/MMLU": 0.275, - "helm_classic/BoolQ": 0.72, - "helm_classic/NarrativeQA": 0.476, - "helm_classic/NaturalQuestions (open-book)": 0.449, - "helm_classic/QuAC": 0.311, - "helm_classic/HellaSwag": -1.0, - "helm_classic/OpenbookQA": -1.0, - "helm_classic/TruthfulQA": 0.213, - "helm_classic/MS MARCO (TREC)": -1.0, - "helm_classic/CNN/DailyMail": -1.0, - "helm_classic/XSUM": -1.0, - "helm_classic/IMDB": 0.852, - "helm_classic/CivilComments": 0.511, - "helm_classic/RAFT": 0.523 - } - }, - { - "id": "tiiuae/falcon-mamba-7b", - "name": "falcon-mamba-7b", - "developer": "tiiuae", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3336, - "hfopenllm_v2/BBH": 0.4285, - "hfopenllm_v2/MATH Level 5": 0.0446, - "hfopenllm_v2/GPQA": 0.3104, - "hfopenllm_v2/MUSR": 0.421, - "hfopenllm_v2/MMLU-PRO": 0.2302 - } - }, - { - "id": "tiiuae/Falcon3-10B-Base", - "name": "Falcon3-10B-Base", - "developer": "tiiuae", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3648, - "hfopenllm_v2/BBH": 0.595, - "hfopenllm_v2/MATH Level 5": 0.2492, - "hfopenllm_v2/GPQA": 0.3456, - "hfopenllm_v2/MUSR": 0.4398, - "hfopenllm_v2/MMLU-PRO": 0.424 - } - }, - { - "id": "tiiuae/Falcon3-10B-Instruct", - "name": "Falcon3-10B-Instruct", - "developer": "tiiuae", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7817, - "hfopenllm_v2/BBH": 0.617, - "hfopenllm_v2/MATH Level 5": 0.2764, - "hfopenllm_v2/GPQA": 0.3289, - "hfopenllm_v2/MUSR": 0.4323, - "hfopenllm_v2/MMLU-PRO": 0.4429 - } - }, - { - "id": "tiiuae/Falcon3-1B-Base", - "name": "Falcon3-1B-Base", - "developer": "tiiuae", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2428, - "hfopenllm_v2/BBH": 0.3571, - "hfopenllm_v2/MATH Level 5": 0.0332, - "hfopenllm_v2/GPQA": 0.2794, - "hfopenllm_v2/MUSR": 0.4147, - "hfopenllm_v2/MMLU-PRO": 0.1608 - } - }, - { - "id": "tiiuae/Falcon3-1B-Instruct", - "name": "Falcon3-1B-Instruct", - "developer": "tiiuae", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5557, - "hfopenllm_v2/BBH": 0.3745, - "hfopenllm_v2/MATH Level 5": 0.0634, - "hfopenllm_v2/GPQA": 0.2668, - "hfopenllm_v2/MUSR": 0.4189, - "hfopenllm_v2/MMLU-PRO": 0.1838 - } - }, - { - "id": "tiiuae/Falcon3-3B-Base", - "name": "Falcon3-3B-Base", - "developer": "tiiuae", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2765, - "hfopenllm_v2/BBH": 0.4421, - "hfopenllm_v2/MATH Level 5": 0.1178, - "hfopenllm_v2/GPQA": 0.297, - "hfopenllm_v2/MUSR": 0.375, - "hfopenllm_v2/MMLU-PRO": 0.2879 - } - }, - { - "id": "tiiuae/Falcon3-3B-Instruct", - "name": "Falcon3-3B-Instruct", - "developer": "tiiuae", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6977, - "hfopenllm_v2/BBH": 0.4754, - "hfopenllm_v2/MATH Level 5": 0.25, - "hfopenllm_v2/GPQA": 0.2886, - "hfopenllm_v2/MUSR": 0.4136, - "hfopenllm_v2/MMLU-PRO": 0.3005 - } - }, - { - "id": "tiiuae/Falcon3-7B-Base", - "name": "Falcon3-7B-Base", - "developer": "tiiuae", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3416, - "hfopenllm_v2/BBH": 0.5099, - "hfopenllm_v2/MATH Level 5": 0.1941, - "hfopenllm_v2/GPQA": 0.3465, - "hfopenllm_v2/MUSR": 0.4702, - "hfopenllm_v2/MMLU-PRO": 0.391 - } - }, - { - "id": "tiiuae/Falcon3-7B-Instruct", - "name": "Falcon3-7B-Instruct", - "developer": "tiiuae", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7612, - "hfopenllm_v2/BBH": 0.5632, - "hfopenllm_v2/MATH Level 5": 0.4086, - "hfopenllm_v2/GPQA": 0.3104, - "hfopenllm_v2/MUSR": 0.4827, - "hfopenllm_v2/MMLU-PRO": 0.4087 - } - }, - { - "id": "tiiuae/Falcon3-Mamba-7B-Base", - "name": "Falcon3-Mamba-7B-Base", - "developer": "tiiuae", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2891, - "hfopenllm_v2/BBH": 0.4699, - "hfopenllm_v2/MATH Level 5": 0.1941, - "hfopenllm_v2/GPQA": 0.3096, - "hfopenllm_v2/MUSR": 0.3431, - "hfopenllm_v2/MMLU-PRO": 0.3038 - } - }, - { - "id": "tiiuae/Falcon3-Mamba-7B-Instruct", - "name": "Falcon3-Mamba-7B-Instruct", - "developer": "tiiuae", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7165, - "hfopenllm_v2/BBH": 0.4679, - "hfopenllm_v2/MATH Level 5": 0.3006, - "hfopenllm_v2/GPQA": 0.3037, - "hfopenllm_v2/MUSR": 0.3869, - "hfopenllm_v2/MMLU-PRO": 0.3369 - } - }, - { - "id": "Tijmen2/cosmosage-v3", - "name": "cosmosage-v3", - "developer": "Tijmen2", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4482, - "hfopenllm_v2/BBH": 0.4551, - "hfopenllm_v2/MATH Level 5": 0.0506, - "hfopenllm_v2/GPQA": 0.2827, - "hfopenllm_v2/MUSR": 0.4199, - "hfopenllm_v2/MMLU-PRO": 0.2486 - } - }, - { - "id": "tinycompany/BiBo-v0.3", - "name": "BiBo-v0.3", - "developer": "tinycompany", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5184, - "hfopenllm_v2/BBH": 0.4642, - "hfopenllm_v2/MATH Level 5": 0.0876, - "hfopenllm_v2/GPQA": 0.2676, - "hfopenllm_v2/MUSR": 0.395, - "hfopenllm_v2/MMLU-PRO": 0.2995 - } - }, - { - "id": "tinycompany/BiBo-v0.7", - "name": "BiBo-v0.7", - "developer": "tinycompany", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3738, - "hfopenllm_v2/BBH": 0.4311, - "hfopenllm_v2/MATH Level 5": 0.0823, - "hfopenllm_v2/GPQA": 0.2768, - "hfopenllm_v2/MUSR": 0.4044, - "hfopenllm_v2/MMLU-PRO": 0.265 - } - }, - { - "id": "tinycompany/ShawtyIsBad-bgem3", - "name": "ShawtyIsBad-bgem3", - "developer": "tinycompany", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2608, - "hfopenllm_v2/BBH": 0.3853, - "hfopenllm_v2/MATH Level 5": 0.0483, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.3695, - "hfopenllm_v2/MMLU-PRO": 0.2583 - } - }, - { - "id": "tinycompany/ShawtyIsBad-e5-large", - "name": "ShawtyIsBad-e5-large", - "developer": "tinycompany", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2468, - "hfopenllm_v2/BBH": 0.3873, - "hfopenllm_v2/MATH Level 5": 0.0453, - "hfopenllm_v2/GPQA": 0.302, - "hfopenllm_v2/MUSR": 0.372, - "hfopenllm_v2/MMLU-PRO": 0.2569 - } - }, - { - "id": "tinycompany/ShawtyIsBad-ib", - "name": "ShawtyIsBad-ib", - "developer": "tinycompany", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2565, - "hfopenllm_v2/BBH": 0.388, - "hfopenllm_v2/MATH Level 5": 0.0491, - "hfopenllm_v2/GPQA": 0.2987, - "hfopenllm_v2/MUSR": 0.3641, - "hfopenllm_v2/MMLU-PRO": 0.2581 - } - }, - { - "id": "tinycompany/ShawtyIsBad-nomic-moe", - "name": "ShawtyIsBad-nomic-moe", - "developer": "tinycompany", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2608, - "hfopenllm_v2/BBH": 0.3878, - "hfopenllm_v2/MATH Level 5": 0.0431, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.3747, - "hfopenllm_v2/MMLU-PRO": 0.2572 - } - }, - { - "id": "tinycompany/ShawtyIsBad-nomic1.5", - "name": "ShawtyIsBad-nomic1.5", - "developer": "tinycompany", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2544, - "hfopenllm_v2/BBH": 0.3874, - "hfopenllm_v2/MATH Level 5": 0.0431, - "hfopenllm_v2/GPQA": 0.3112, - "hfopenllm_v2/MUSR": 0.3628, - "hfopenllm_v2/MMLU-PRO": 0.2567 - } - }, - { - "id": "tinycompany/SigmaBoi-base", - "name": "SigmaBoi-base", - "developer": "tinycompany", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2447, - "hfopenllm_v2/BBH": 0.4314, - "hfopenllm_v2/MATH Level 5": 0.0778, - "hfopenllm_v2/GPQA": 0.2936, - "hfopenllm_v2/MUSR": 0.4343, - "hfopenllm_v2/MMLU-PRO": 0.2817 - } - }, - { - "id": "tinycompany/SigmaBoi-bge-m3", - "name": "SigmaBoi-bge-m3", - "developer": "tinycompany", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.245, - "hfopenllm_v2/BBH": 0.4351, - "hfopenllm_v2/MATH Level 5": 0.0763, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.4383, - "hfopenllm_v2/MMLU-PRO": 0.2819 - } - }, - { - "id": "tinycompany/SigmaBoi-bgem3", - "name": "SigmaBoi-bgem3", - "developer": "tinycompany", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.245, - "hfopenllm_v2/BBH": 0.4351, - "hfopenllm_v2/MATH Level 5": 0.0763, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.4383, - "hfopenllm_v2/MMLU-PRO": 0.2819 - } - }, - { - "id": "tinycompany/SigmaBoi-ib", - "name": "SigmaBoi-ib", - "developer": "tinycompany", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2477, - "hfopenllm_v2/BBH": 0.4344, - "hfopenllm_v2/MATH Level 5": 0.074, - "hfopenllm_v2/GPQA": 0.2878, - "hfopenllm_v2/MUSR": 0.429, - "hfopenllm_v2/MMLU-PRO": 0.2824 - } - }, - { - "id": "tinycompany/SigmaBoi-nomic-moe", - "name": "SigmaBoi-nomic-moe", - "developer": "tinycompany", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2474, - "hfopenllm_v2/BBH": 0.4334, - "hfopenllm_v2/MATH Level 5": 0.0718, - "hfopenllm_v2/GPQA": 0.2928, - "hfopenllm_v2/MUSR": 0.4316, - "hfopenllm_v2/MMLU-PRO": 0.2837 - } - }, - { - "id": "tinycompany/SigmaBoi-nomic1.5", - "name": "SigmaBoi-nomic1.5", - "developer": "tinycompany", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2447, - "hfopenllm_v2/BBH": 0.4371, - "hfopenllm_v2/MATH Level 5": 0.0831, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.4316, - "hfopenllm_v2/MMLU-PRO": 0.2841 - } - }, - { - "id": "tinycompany/SigmaBoi-nomic1.5-fp32", - "name": "SigmaBoi-nomic1.5-fp32", - "developer": "tinycompany", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2462, - "hfopenllm_v2/BBH": 0.4371, - "hfopenllm_v2/MATH Level 5": 0.0831, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.4316, - "hfopenllm_v2/MMLU-PRO": 0.2841 - } - }, - { - "id": "tinycompany/Tamed-Shawty", - "name": "Tamed-Shawty", - "developer": "tinycompany", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3831, - "hfopenllm_v2/BBH": 0.3837, - "hfopenllm_v2/MATH Level 5": 0.0718, - "hfopenllm_v2/GPQA": 0.2626, - "hfopenllm_v2/MUSR": 0.3501, - "hfopenllm_v2/MMLU-PRO": 0.2601 - } - }, - { - "id": "TinyLlama/TinyLlama-1.1B-Chat-v0.1", - "name": "TinyLlama-1.1B-Chat-v0.1", - "developer": "TinyLlama", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1479, - "hfopenllm_v2/BBH": 0.3084, - "hfopenllm_v2/MATH Level 5": 0.006, - "hfopenllm_v2/GPQA": 0.229, - "hfopenllm_v2/MUSR": 0.3592, - "hfopenllm_v2/MMLU-PRO": 0.1098 - } - }, - { - "id": "TinyLlama/TinyLlama-1.1B-Chat-v0.5", - "name": "TinyLlama-1.1B-Chat-v0.5", - "developer": "TinyLlama", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1634, - "hfopenllm_v2/BBH": 0.3105, - "hfopenllm_v2/MATH Level 5": 0.0038, - "hfopenllm_v2/GPQA": 0.2483, - "hfopenllm_v2/MUSR": 0.3661, - "hfopenllm_v2/MMLU-PRO": 0.1096 - } - }, - { - "id": "TinyLlama/TinyLlama-1.1B-Chat-v0.6", - "name": "TinyLlama-1.1B-Chat-v0.6", - "developer": "TinyLlama", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1574, - "hfopenllm_v2/BBH": 0.3067, - "hfopenllm_v2/MATH Level 5": 0.0159, - "hfopenllm_v2/GPQA": 0.2584, - "hfopenllm_v2/MUSR": 0.3422, - "hfopenllm_v2/MMLU-PRO": 0.1149 - } - }, - { - "id": "TinyLlama/TinyLlama-1.1B-Chat-v1.0", - "name": "TinyLlama-1.1B-Chat-v1.0", - "developer": "TinyLlama", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0596, - "hfopenllm_v2/BBH": 0.3104, - "hfopenllm_v2/MATH Level 5": 0.0151, - "hfopenllm_v2/GPQA": 0.25, - "hfopenllm_v2/MUSR": 0.3515, - "hfopenllm_v2/MMLU-PRO": 0.1101 - } - }, - { - "id": "TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T", - "name": "TinyLlama-1.1B-intermediate-step-1431k-3T", - "developer": "TinyLlama", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2277, - "hfopenllm_v2/BBH": 0.3071, - "hfopenllm_v2/MATH Level 5": 0.0121, - "hfopenllm_v2/GPQA": 0.2525, - "hfopenllm_v2/MUSR": 0.338, - "hfopenllm_v2/MMLU-PRO": 0.112 - } - }, - { - "id": "TinyLlama/TinyLlama_v1.1", - "name": "TinyLlama_v1.1", - "developer": "TinyLlama", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2001, - "hfopenllm_v2/BBH": 0.3024, - "hfopenllm_v2/MATH Level 5": 0.0121, - "hfopenllm_v2/GPQA": 0.2458, - "hfopenllm_v2/MUSR": 0.37, - "hfopenllm_v2/MMLU-PRO": 0.1049 - } - }, - { - "id": "tklohj/WindyFloLLM", - "name": "WindyFloLLM", - "developer": "tklohj", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2669, - "hfopenllm_v2/BBH": 0.4637, - "hfopenllm_v2/MATH Level 5": 0.0159, - "hfopenllm_v2/GPQA": 0.2752, - "hfopenllm_v2/MUSR": 0.4253, - "hfopenllm_v2/MMLU-PRO": 0.2581 - } - }, - { - "id": "ToastyPigeon/Sto-vo-kor-12B", - "name": "Sto-vo-kor-12B", - "developer": "ToastyPigeon", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5501, - "hfopenllm_v2/BBH": 0.5065, - "hfopenllm_v2/MATH Level 5": 0.1088, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.3938, - "hfopenllm_v2/MMLU-PRO": 0.3398 - } - }, - { - "id": "together/RedPajama-INCITE-Base-7B", - "name": "RedPajama-INCITE-Base 7B", - "developer": "together", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.378, - "helm_classic/MMLU": 0.302, - "helm_classic/BoolQ": 0.713, - "helm_classic/NarrativeQA": 0.617, - "helm_classic/NaturalQuestions (open-book)": 0.586, - "helm_classic/QuAC": 0.336, - "helm_classic/HellaSwag": -1.0, - "helm_classic/OpenbookQA": -1.0, - "helm_classic/TruthfulQA": 0.205, - "helm_classic/MS MARCO (TREC)": -1.0, - "helm_classic/CNN/DailyMail": -1.0, - "helm_classic/XSUM": -1.0, - "helm_classic/IMDB": 0.752, - "helm_classic/CivilComments": 0.547, - "helm_classic/RAFT": 0.648 - } - }, - { - "id": "together/RedPajama-INCITE-Base-v1-3B", - "name": "RedPajama-INCITE-Base-v1 3B", - "developer": "together", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.311, - "helm_classic/MMLU": 0.263, - "helm_classic/BoolQ": 0.685, - "helm_classic/NarrativeQA": 0.555, - "helm_classic/NaturalQuestions (open-book)": 0.52, - "helm_classic/QuAC": 0.309, - "helm_classic/HellaSwag": -1.0, - "helm_classic/OpenbookQA": -1.0, - "helm_classic/TruthfulQA": 0.277, - "helm_classic/MS MARCO (TREC)": -1.0, - "helm_classic/CNN/DailyMail": -1.0, - "helm_classic/XSUM": -1.0, - "helm_classic/IMDB": 0.907, - "helm_classic/CivilComments": 0.549, - "helm_classic/RAFT": 0.502 - } - }, - { - "id": "together/RedPajama-INCITE-Instruct-7B", - "name": "RedPajama-INCITE-Instruct 7B", - "developer": "together", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.524, - "helm_classic/MMLU": 0.363, - "helm_classic/BoolQ": 0.705, - "helm_classic/NarrativeQA": 0.638, - "helm_classic/NaturalQuestions (open-book)": 0.659, - "helm_classic/QuAC": 0.26, - "helm_classic/HellaSwag": -1.0, - "helm_classic/OpenbookQA": -1.0, - "helm_classic/TruthfulQA": 0.243, - "helm_classic/MS MARCO (TREC)": -1.0, - "helm_classic/CNN/DailyMail": -1.0, - "helm_classic/XSUM": -1.0, - "helm_classic/IMDB": 0.927, - "helm_classic/CivilComments": 0.664, - "helm_classic/RAFT": 0.695 - } - }, - { - "id": "together/RedPajama-INCITE-Instruct-v1-3B", - "name": "RedPajama-INCITE-Instruct-v1 3B", - "developer": "together", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.366, - "helm_classic/MMLU": 0.257, - "helm_classic/BoolQ": 0.677, - "helm_classic/NarrativeQA": 0.638, - "helm_classic/NaturalQuestions (open-book)": 0.637, - "helm_classic/QuAC": 0.259, - "helm_classic/HellaSwag": -1.0, - "helm_classic/OpenbookQA": -1.0, - "helm_classic/TruthfulQA": 0.208, - "helm_classic/MS MARCO (TREC)": -1.0, - "helm_classic/CNN/DailyMail": -1.0, - "helm_classic/XSUM": -1.0, - "helm_classic/IMDB": 0.894, - "helm_classic/CivilComments": 0.549, - "helm_classic/RAFT": 0.661 - } - }, - { - "id": "togethercomputer/GPT-JT-6B-v1", - "name": "GPT-JT-6B-v1", - "developer": "togethercomputer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2061, - "hfopenllm_v2/BBH": 0.3303, - "hfopenllm_v2/MATH Level 5": 0.0106, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.3737, - "hfopenllm_v2/MMLU-PRO": 0.1626 - } - }, - { - "id": "togethercomputer/GPT-NeoXT-Chat-Base-20B", - "name": "GPT-NeoXT-Chat-Base-20B", - "developer": "togethercomputer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.183, - "hfopenllm_v2/BBH": 0.3321, - "hfopenllm_v2/MATH Level 5": 0.0234, - "hfopenllm_v2/GPQA": 0.25, - "hfopenllm_v2/MUSR": 0.3461, - "hfopenllm_v2/MMLU-PRO": 0.1145 - } - }, - { - "id": "togethercomputer/LLaMA-2-7B-32K", - "name": "LLaMA-2-7B-32K", - "developer": "togethercomputer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1865, - "hfopenllm_v2/BBH": 0.34, - "hfopenllm_v2/MATH Level 5": 0.0144, - "hfopenllm_v2/GPQA": 0.25, - "hfopenllm_v2/MUSR": 0.3754, - "hfopenllm_v2/MMLU-PRO": 0.1768 - } - }, - { - "id": "togethercomputer/Llama-2-7B-32K-Instruct", - "name": "Llama-2-7B-32K-Instruct", - "developer": "togethercomputer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.213, - "hfopenllm_v2/BBH": 0.3443, - "hfopenllm_v2/MATH Level 5": 0.0159, - "hfopenllm_v2/GPQA": 0.2517, - "hfopenllm_v2/MUSR": 0.4056, - "hfopenllm_v2/MMLU-PRO": 0.1781 - } - }, - { - "id": "togethercomputer/RedPajama-INCITE-7B-Base", - "name": "RedPajama-INCITE-7B-Base", - "developer": "togethercomputer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2082, - "hfopenllm_v2/BBH": 0.3195, - "hfopenllm_v2/MATH Level 5": 0.0159, - "hfopenllm_v2/GPQA": 0.255, - "hfopenllm_v2/MUSR": 0.362, - "hfopenllm_v2/MMLU-PRO": 0.1197 - } - }, - { - "id": "togethercomputer/RedPajama-INCITE-7B-Chat", - "name": "RedPajama-INCITE-7B-Chat", - "developer": "togethercomputer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1558, - "hfopenllm_v2/BBH": 0.3175, - "hfopenllm_v2/MATH Level 5": 0.0068, - "hfopenllm_v2/GPQA": 0.2525, - "hfopenllm_v2/MUSR": 0.3448, - "hfopenllm_v2/MMLU-PRO": 0.1121 - } - }, - { - "id": "togethercomputer/RedPajama-INCITE-7B-Instruct", - "name": "RedPajama-INCITE-7B-Instruct", - "developer": "togethercomputer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2055, - "hfopenllm_v2/BBH": 0.3377, - "hfopenllm_v2/MATH Level 5": 0.0211, - "hfopenllm_v2/GPQA": 0.2508, - "hfopenllm_v2/MUSR": 0.3685, - "hfopenllm_v2/MMLU-PRO": 0.1272 - } - }, - { - "id": "togethercomputer/RedPajama-INCITE-Base-3B-v1", - "name": "RedPajama-INCITE-Base-3B-v1", - "developer": "togethercomputer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2294, - "hfopenllm_v2/BBH": 0.306, - "hfopenllm_v2/MATH Level 5": 0.0144, - "hfopenllm_v2/GPQA": 0.2433, - "hfopenllm_v2/MUSR": 0.3739, - "hfopenllm_v2/MMLU-PRO": 0.1111 - } - }, - { - "id": "togethercomputer/RedPajama-INCITE-Chat-3B-v1", - "name": "RedPajama-INCITE-Chat-3B-v1", - "developer": "togethercomputer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1652, - "hfopenllm_v2/BBH": 0.3217, - "hfopenllm_v2/MATH Level 5": 0.0091, - "hfopenllm_v2/GPQA": 0.2441, - "hfopenllm_v2/MUSR": 0.3684, - "hfopenllm_v2/MMLU-PRO": 0.1127 - } - }, - { - "id": "togethercomputer/RedPajama-INCITE-Instruct-3B-v1", - "name": "RedPajama-INCITE-Instruct-3B-v1", - "developer": "togethercomputer", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2124, - "hfopenllm_v2/BBH": 0.3146, - "hfopenllm_v2/MATH Level 5": 0.0128, - "hfopenllm_v2/GPQA": 0.2475, - "hfopenllm_v2/MUSR": 0.3886, - "hfopenllm_v2/MMLU-PRO": 0.111 - } - }, - { - "id": "tokyotech-llm/Llama-3-Swallow-8B-Instruct-v0.1", - "name": "Llama-3-Swallow-8B-Instruct-v0.1", - "developer": "tokyotech-llm", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5508, - "hfopenllm_v2/BBH": 0.5009, - "hfopenllm_v2/MATH Level 5": 0.0748, - "hfopenllm_v2/GPQA": 0.2894, - "hfopenllm_v2/MUSR": 0.4357, - "hfopenllm_v2/MMLU-PRO": 0.3088 - } - }, - { - "id": "tomasmcm/sky-t1-coder-32b-flash", - "name": "sky-t1-coder-32b-flash", - "developer": "tomasmcm", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.778, - "hfopenllm_v2/BBH": 0.6822, - "hfopenllm_v2/MATH Level 5": 0.5423, - "hfopenllm_v2/GPQA": 0.3683, - "hfopenllm_v2/MUSR": 0.4233, - "hfopenllm_v2/MMLU-PRO": 0.5782 - } - }, - { - "id": "Trappu/Magnum-Picaro-0.7-v2-12b", - "name": "Magnum-Picaro-0.7-v2-12b", - "developer": "Trappu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3003, - "hfopenllm_v2/BBH": 0.5507, - "hfopenllm_v2/MATH Level 5": 0.0665, - "hfopenllm_v2/GPQA": 0.323, - "hfopenllm_v2/MUSR": 0.4727, - "hfopenllm_v2/MMLU-PRO": 0.358 - } - }, - { - "id": "Trappu/Nemo-Picaro-12B", - "name": "Nemo-Picaro-12B", - "developer": "Trappu", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2577, - "hfopenllm_v2/BBH": 0.549, - "hfopenllm_v2/MATH Level 5": 0.0846, - "hfopenllm_v2/GPQA": 0.3272, - "hfopenllm_v2/MUSR": 0.4726, - "hfopenllm_v2/MMLU-PRO": 0.3605 - } - }, - { - "id": "Tremontaine/L3-12B-Lunaris-v1", - "name": "L3-12B-Lunaris-v1", - "developer": "Tremontaine", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6909, - "hfopenllm_v2/BBH": 0.523, - "hfopenllm_v2/MATH Level 5": 0.0876, - "hfopenllm_v2/GPQA": 0.3096, - "hfopenllm_v2/MUSR": 0.3674, - "hfopenllm_v2/MMLU-PRO": 0.3775 - } - }, - { - "id": "Triangle104/Annunaki-12b", - "name": "Annunaki-12b", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3872, - "hfopenllm_v2/BBH": 0.5499, - "hfopenllm_v2/MATH Level 5": 0.1216, - "hfopenllm_v2/GPQA": 0.3213, - "hfopenllm_v2/MUSR": 0.4409, - "hfopenllm_v2/MMLU-PRO": 0.3721 - } - }, - { - "id": "Triangle104/BigTalker-Lite-8B", - "name": "BigTalker-Lite-8B", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3689, - "hfopenllm_v2/BBH": 0.5308, - "hfopenllm_v2/MATH Level 5": 0.102, - "hfopenllm_v2/GPQA": 0.3104, - "hfopenllm_v2/MUSR": 0.4208, - "hfopenllm_v2/MMLU-PRO": 0.3431 - } - }, - { - "id": "Triangle104/Chatty-Harry_V2.0", - "name": "Chatty-Harry_V2.0", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3326, - "hfopenllm_v2/BBH": 0.5319, - "hfopenllm_v2/MATH Level 5": 0.139, - "hfopenllm_v2/GPQA": 0.323, - "hfopenllm_v2/MUSR": 0.4078, - "hfopenllm_v2/MMLU-PRO": 0.3683 - } - }, - { - "id": "Triangle104/Chatty-Harry_V3.0", - "name": "Chatty-Harry_V3.0", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3675, - "hfopenllm_v2/BBH": 0.5526, - "hfopenllm_v2/MATH Level 5": 0.1125, - "hfopenllm_v2/GPQA": 0.323, - "hfopenllm_v2/MUSR": 0.4408, - "hfopenllm_v2/MMLU-PRO": 0.3702 - } - }, - { - "id": "Triangle104/Chronos-Prism_V1.0", - "name": "Chronos-Prism_V1.0", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3259, - "hfopenllm_v2/BBH": 0.5554, - "hfopenllm_v2/MATH Level 5": 0.1201, - "hfopenllm_v2/GPQA": 0.3096, - "hfopenllm_v2/MUSR": 0.4263, - "hfopenllm_v2/MMLU-PRO": 0.3673 - } - }, - { - "id": "Triangle104/Dark-Chivalry_V1.0", - "name": "Dark-Chivalry_V1.0", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4326, - "hfopenllm_v2/BBH": 0.4974, - "hfopenllm_v2/MATH Level 5": 0.1314, - "hfopenllm_v2/GPQA": 0.2936, - "hfopenllm_v2/MUSR": 0.4182, - "hfopenllm_v2/MMLU-PRO": 0.3444 - } - }, - { - "id": "Triangle104/Distilled-DarkPlanet-Allades-8B", - "name": "Distilled-DarkPlanet-Allades-8B", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.346, - "hfopenllm_v2/BBH": 0.4634, - "hfopenllm_v2/MATH Level 5": 0.4003, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.3538, - "hfopenllm_v2/MMLU-PRO": 0.2901 - } - }, - { - "id": "Triangle104/Distilled-DarkPlanet-Allades-8B_TIES", - "name": "Distilled-DarkPlanet-Allades-8B_TIES", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3892, - "hfopenllm_v2/BBH": 0.5042, - "hfopenllm_v2/MATH Level 5": 0.0906, - "hfopenllm_v2/GPQA": 0.3146, - "hfopenllm_v2/MUSR": 0.3868, - "hfopenllm_v2/MMLU-PRO": 0.3401 - } - }, - { - "id": "Triangle104/Distilled-Whiskey-8b", - "name": "Distilled-Whiskey-8b", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3448, - "hfopenllm_v2/BBH": 0.5028, - "hfopenllm_v2/MATH Level 5": 0.2545, - "hfopenllm_v2/GPQA": 0.3314, - "hfopenllm_v2/MUSR": 0.4172, - "hfopenllm_v2/MMLU-PRO": 0.3367 - } - }, - { - "id": "Triangle104/Dolphin3-Llama3.2-Smart", - "name": "Dolphin3-Llama3.2-Smart", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4137, - "hfopenllm_v2/BBH": 0.3975, - "hfopenllm_v2/MATH Level 5": 0.0438, - "hfopenllm_v2/GPQA": 0.2693, - "hfopenllm_v2/MUSR": 0.3922, - "hfopenllm_v2/MMLU-PRO": 0.2195 - } - }, - { - "id": "Triangle104/DS-Distilled-Hermes-Llama-3.1", - "name": "DS-Distilled-Hermes-Llama-3.1", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3229, - "hfopenllm_v2/BBH": 0.5117, - "hfopenllm_v2/MATH Level 5": 0.2931, - "hfopenllm_v2/GPQA": 0.3188, - "hfopenllm_v2/MUSR": 0.4039, - "hfopenllm_v2/MMLU-PRO": 0.311 - } - }, - { - "id": "Triangle104/DS-Distilled-Hermes-Llama-3.1_TIES", - "name": "DS-Distilled-Hermes-Llama-3.1_TIES", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1364, - "hfopenllm_v2/BBH": 0.2928, - "hfopenllm_v2/MATH Level 5": 0.0091, - "hfopenllm_v2/GPQA": 0.245, - "hfopenllm_v2/MUSR": 0.3621, - "hfopenllm_v2/MMLU-PRO": 0.1104 - } - }, - { - "id": "Triangle104/DS-R1-Distill-Q2.5-10B-Harmony", - "name": "DS-R1-Distill-Q2.5-10B-Harmony", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1751, - "hfopenllm_v2/BBH": 0.2643, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2106, - "hfopenllm_v2/MUSR": 0.3128, - "hfopenllm_v2/MMLU-PRO": 0.1173 - } - }, - { - "id": "Triangle104/DS-R1-Distill-Q2.5-14B-Harmony_V0.1", - "name": "DS-R1-Distill-Q2.5-14B-Harmony_V0.1", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4515, - "hfopenllm_v2/BBH": 0.5783, - "hfopenllm_v2/MATH Level 5": 0.5551, - "hfopenllm_v2/GPQA": 0.3935, - "hfopenllm_v2/MUSR": 0.5567, - "hfopenllm_v2/MMLU-PRO": 0.4601 - } - }, - { - "id": "Triangle104/DS-R1-Distill-Q2.5-7B-RP", - "name": "DS-R1-Distill-Q2.5-7B-RP", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3445, - "hfopenllm_v2/BBH": 0.4383, - "hfopenllm_v2/MATH Level 5": 0.4683, - "hfopenllm_v2/GPQA": 0.3138, - "hfopenllm_v2/MUSR": 0.403, - "hfopenllm_v2/MMLU-PRO": 0.2891 - } - }, - { - "id": "Triangle104/DS-R1-Llama-8B-Harmony", - "name": "DS-R1-Llama-8B-Harmony", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3566, - "hfopenllm_v2/BBH": 0.4154, - "hfopenllm_v2/MATH Level 5": 0.4282, - "hfopenllm_v2/GPQA": 0.2919, - "hfopenllm_v2/MUSR": 0.3762, - "hfopenllm_v2/MMLU-PRO": 0.2744 - } - }, - { - "id": "Triangle104/DSR1-Distill-Llama-Lit-8B", - "name": "DSR1-Distill-Llama-Lit-8B", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1885, - "hfopenllm_v2/BBH": 0.4284, - "hfopenllm_v2/MATH Level 5": 0.352, - "hfopenllm_v2/GPQA": 0.3029, - "hfopenllm_v2/MUSR": 0.3535, - "hfopenllm_v2/MMLU-PRO": 0.2798 - } - }, - { - "id": "Triangle104/DSR1-Distill-Qwen-7B-RP", - "name": "DSR1-Distill-Qwen-7B-RP", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3609, - "hfopenllm_v2/BBH": 0.4326, - "hfopenllm_v2/MATH Level 5": 0.4804, - "hfopenllm_v2/GPQA": 0.3196, - "hfopenllm_v2/MUSR": 0.4045, - "hfopenllm_v2/MMLU-PRO": 0.3028 - } - }, - { - "id": "Triangle104/Gemmadevi-Stock-10B", - "name": "Gemmadevi-Stock-10B", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1582, - "hfopenllm_v2/BBH": 0.6066, - "hfopenllm_v2/MATH Level 5": 0.0967, - "hfopenllm_v2/GPQA": 0.3532, - "hfopenllm_v2/MUSR": 0.4621, - "hfopenllm_v2/MMLU-PRO": 0.4262 - } - }, - { - "id": "Triangle104/Hermes-Llama-3.2-CoT", - "name": "Hermes-Llama-3.2-CoT", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4178, - "hfopenllm_v2/BBH": 0.4616, - "hfopenllm_v2/MATH Level 5": 0.0952, - "hfopenllm_v2/GPQA": 0.2794, - "hfopenllm_v2/MUSR": 0.3698, - "hfopenllm_v2/MMLU-PRO": 0.2947 - } - }, - { - "id": "Triangle104/Hermes-Llama-3.2-CoT-Summary", - "name": "Hermes-Llama-3.2-CoT-Summary", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.483, - "hfopenllm_v2/BBH": 0.42, - "hfopenllm_v2/MATH Level 5": 0.0831, - "hfopenllm_v2/GPQA": 0.2559, - "hfopenllm_v2/MUSR": 0.3575, - "hfopenllm_v2/MMLU-PRO": 0.2901 - } - }, - { - "id": "Triangle104/Hermes3-L3.1-DirtyHarry-8B", - "name": "Hermes3-L3.1-DirtyHarry-8B", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3242, - "hfopenllm_v2/BBH": 0.5066, - "hfopenllm_v2/MATH Level 5": 0.0718, - "hfopenllm_v2/GPQA": 0.302, - "hfopenllm_v2/MUSR": 0.4069, - "hfopenllm_v2/MMLU-PRO": 0.3339 - } - }, - { - "id": "Triangle104/Herodotos-14B", - "name": "Herodotos-14B", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4667, - "hfopenllm_v2/BBH": 0.6435, - "hfopenllm_v2/MATH Level 5": 0.5045, - "hfopenllm_v2/GPQA": 0.3733, - "hfopenllm_v2/MUSR": 0.4795, - "hfopenllm_v2/MMLU-PRO": 0.529 - } - }, - { - "id": "Triangle104/Herodotos-14B_V0.1", - "name": "Herodotos-14B_V0.1", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1879, - "hfopenllm_v2/BBH": 0.3017, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.224, - "hfopenllm_v2/MUSR": 0.3684, - "hfopenllm_v2/MMLU-PRO": 0.1164 - } - }, - { - "id": "Triangle104/L3.1-8B-Dusky-Ink", - "name": "L3.1-8B-Dusky-Ink", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.453, - "hfopenllm_v2/BBH": 0.5098, - "hfopenllm_v2/MATH Level 5": 0.1231, - "hfopenllm_v2/GPQA": 0.2894, - "hfopenllm_v2/MUSR": 0.4224, - "hfopenllm_v2/MMLU-PRO": 0.3683 - } - }, - { - "id": "Triangle104/L3.1-8B-Dusky-Ink_v0.r1", - "name": "L3.1-8B-Dusky-Ink_v0.r1", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1985, - "hfopenllm_v2/BBH": 0.4337, - "hfopenllm_v2/MATH Level 5": 0.0431, - "hfopenllm_v2/GPQA": 0.3037, - "hfopenllm_v2/MUSR": 0.3988, - "hfopenllm_v2/MMLU-PRO": 0.3206 - } - }, - { - "id": "Triangle104/Llama3.1-Allades-Lit-8b", - "name": "Llama3.1-Allades-Lit-8b", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2461, - "hfopenllm_v2/BBH": 0.4183, - "hfopenllm_v2/MATH Level 5": 0.0023, - "hfopenllm_v2/GPQA": 0.2844, - "hfopenllm_v2/MUSR": 0.3708, - "hfopenllm_v2/MMLU-PRO": 0.2724 - } - }, - { - "id": "Triangle104/Llama3.1-cc-Lit-8b", - "name": "Llama3.1-cc-Lit-8b", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2993, - "hfopenllm_v2/BBH": 0.3848, - "hfopenllm_v2/MATH Level 5": 0.003, - "hfopenllm_v2/GPQA": 0.2777, - "hfopenllm_v2/MUSR": 0.3854, - "hfopenllm_v2/MMLU-PRO": 0.3004 - } - }, - { - "id": "Triangle104/LThreePointOne-8B-HermesBlackroot", - "name": "LThreePointOne-8B-HermesBlackroot", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1792, - "hfopenllm_v2/BBH": 0.4998, - "hfopenllm_v2/MATH Level 5": 0.0196, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.3586, - "hfopenllm_v2/MMLU-PRO": 0.3285 - } - }, - { - "id": "Triangle104/LThreePointOne-8B-HermesInk", - "name": "LThreePointOne-8B-HermesInk", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4031, - "hfopenllm_v2/BBH": 0.5223, - "hfopenllm_v2/MATH Level 5": 0.1722, - "hfopenllm_v2/GPQA": 0.323, - "hfopenllm_v2/MUSR": 0.4129, - "hfopenllm_v2/MMLU-PRO": 0.3467 - } - }, - { - "id": "Triangle104/Minerva-1.5b", - "name": "Minerva-1.5b", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2694, - "hfopenllm_v2/BBH": 0.4026, - "hfopenllm_v2/MATH Level 5": 0.1027, - "hfopenllm_v2/GPQA": 0.3104, - "hfopenllm_v2/MUSR": 0.3655, - "hfopenllm_v2/MMLU-PRO": 0.2698 - } - }, - { - "id": "Triangle104/Minerva-1.5b_V0.2", - "name": "Minerva-1.5b_V0.2", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3083, - "hfopenllm_v2/BBH": 0.3989, - "hfopenllm_v2/MATH Level 5": 0.114, - "hfopenllm_v2/GPQA": 0.2852, - "hfopenllm_v2/MUSR": 0.396, - "hfopenllm_v2/MMLU-PRO": 0.2911 - } - }, - { - "id": "Triangle104/Minerva-10b", - "name": "Minerva-10b", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1879, - "hfopenllm_v2/BBH": 0.4462, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.281, - "hfopenllm_v2/MUSR": 0.3627, - "hfopenllm_v2/MMLU-PRO": 0.2318 - } - }, - { - "id": "Triangle104/Minerva-14b", - "name": "Minerva-14b", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3468, - "hfopenllm_v2/BBH": 0.6301, - "hfopenllm_v2/MATH Level 5": 0.3051, - "hfopenllm_v2/GPQA": 0.3742, - "hfopenllm_v2/MUSR": 0.4766, - "hfopenllm_v2/MMLU-PRO": 0.5194 - } - }, - { - "id": "Triangle104/Minerva-14b-V0.1", - "name": "Minerva-14b-V0.1", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0861, - "hfopenllm_v2/BBH": 0.609, - "hfopenllm_v2/MATH Level 5": 0.3051, - "hfopenllm_v2/GPQA": 0.3658, - "hfopenllm_v2/MUSR": 0.47, - "hfopenllm_v2/MMLU-PRO": 0.5118 - } - }, - { - "id": "Triangle104/Minerva-7b", - "name": "Minerva-7b", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3724, - "hfopenllm_v2/BBH": 0.5498, - "hfopenllm_v2/MATH Level 5": 0.284, - "hfopenllm_v2/GPQA": 0.323, - "hfopenllm_v2/MUSR": 0.4143, - "hfopenllm_v2/MMLU-PRO": 0.4444 - } - }, - { - "id": "Triangle104/Minerva-8b", - "name": "Minerva-8b", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1721, - "hfopenllm_v2/BBH": 0.4669, - "hfopenllm_v2/MATH Level 5": 0.0045, - "hfopenllm_v2/GPQA": 0.3121, - "hfopenllm_v2/MUSR": 0.4273, - "hfopenllm_v2/MMLU-PRO": 0.3089 - } - }, - { - "id": "Triangle104/Mistral-Redemption-Arc", - "name": "Mistral-Redemption-Arc", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4029, - "hfopenllm_v2/BBH": 0.6255, - "hfopenllm_v2/MATH Level 5": 0.4101, - "hfopenllm_v2/GPQA": 0.3473, - "hfopenllm_v2/MUSR": 0.4595, - "hfopenllm_v2/MMLU-PRO": 0.451 - } - }, - { - "id": "Triangle104/Mistral-Small-24b-Harmony", - "name": "Mistral-Small-24b-Harmony", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1687, - "hfopenllm_v2/BBH": 0.6434, - "hfopenllm_v2/MATH Level 5": 0.1911, - "hfopenllm_v2/GPQA": 0.3842, - "hfopenllm_v2/MUSR": 0.4276, - "hfopenllm_v2/MMLU-PRO": 0.5431 - } - }, - { - "id": "Triangle104/Pans_Gutenbergum_V0.1", - "name": "Pans_Gutenbergum_V0.1", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3097, - "hfopenllm_v2/BBH": 0.5541, - "hfopenllm_v2/MATH Level 5": 0.1057, - "hfopenllm_v2/GPQA": 0.323, - "hfopenllm_v2/MUSR": 0.4528, - "hfopenllm_v2/MMLU-PRO": 0.3697 - } - }, - { - "id": "Triangle104/Pans_Gutenbergum_V0.2", - "name": "Pans_Gutenbergum_V0.2", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3215, - "hfopenllm_v2/BBH": 0.5526, - "hfopenllm_v2/MATH Level 5": 0.0687, - "hfopenllm_v2/GPQA": 0.3121, - "hfopenllm_v2/MUSR": 0.4673, - "hfopenllm_v2/MMLU-PRO": 0.3585 - } - }, - { - "id": "Triangle104/Pantheon_ChatWaifu_V0.2", - "name": "Pantheon_ChatWaifu_V0.2", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2683, - "hfopenllm_v2/BBH": 0.5532, - "hfopenllm_v2/MATH Level 5": 0.0566, - "hfopenllm_v2/GPQA": 0.318, - "hfopenllm_v2/MUSR": 0.4755, - "hfopenllm_v2/MMLU-PRO": 0.3442 - } - }, - { - "id": "Triangle104/Phi-4-AbliteratedRP", - "name": "Phi-4-AbliteratedRP", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4923, - "hfopenllm_v2/BBH": 0.6709, - "hfopenllm_v2/MATH Level 5": 0.3074, - "hfopenllm_v2/GPQA": 0.3951, - "hfopenllm_v2/MUSR": 0.5098, - "hfopenllm_v2/MMLU-PRO": 0.5308 - } - }, - { - "id": "Triangle104/Phi4-RP-o1", - "name": "Phi4-RP-o1", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.022, - "hfopenllm_v2/BBH": 0.6653, - "hfopenllm_v2/MATH Level 5": 0.3776, - "hfopenllm_v2/GPQA": 0.3733, - "hfopenllm_v2/MUSR": 0.4756, - "hfopenllm_v2/MMLU-PRO": 0.5111 - } - }, - { - "id": "Triangle104/Phi4-RP-o1-Ablit", - "name": "Phi4-RP-o1-Ablit", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0239, - "hfopenllm_v2/BBH": 0.663, - "hfopenllm_v2/MATH Level 5": 0.3882, - "hfopenllm_v2/GPQA": 0.3633, - "hfopenllm_v2/MUSR": 0.4754, - "hfopenllm_v2/MMLU-PRO": 0.5105 - } - }, - { - "id": "Triangle104/Porpoise-R1-Llama3.2-3b", - "name": "Porpoise-R1-Llama3.2-3b", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4352, - "hfopenllm_v2/BBH": 0.3824, - "hfopenllm_v2/MATH Level 5": 0.0423, - "hfopenllm_v2/GPQA": 0.2668, - "hfopenllm_v2/MUSR": 0.3576, - "hfopenllm_v2/MMLU-PRO": 0.2117 - } - }, - { - "id": "Triangle104/Q2.5-14B-Instruct-1M-Harmony", - "name": "Q2.5-14B-Instruct-1M-Harmony", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5986, - "hfopenllm_v2/BBH": 0.6339, - "hfopenllm_v2/MATH Level 5": 0.3769, - "hfopenllm_v2/GPQA": 0.375, - "hfopenllm_v2/MUSR": 0.4795, - "hfopenllm_v2/MMLU-PRO": 0.5075 - } - }, - { - "id": "Triangle104/Q2.5-AthensCOT", - "name": "Q2.5-AthensCOT", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4573, - "hfopenllm_v2/BBH": 0.5542, - "hfopenllm_v2/MATH Level 5": 0.2915, - "hfopenllm_v2/GPQA": 0.3003, - "hfopenllm_v2/MUSR": 0.4578, - "hfopenllm_v2/MMLU-PRO": 0.4379 - } - }, - { - "id": "Triangle104/Q2.5-CodeR1-3B", - "name": "Q2.5-CodeR1-3B", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3588, - "hfopenllm_v2/BBH": 0.4661, - "hfopenllm_v2/MATH Level 5": 0.1639, - "hfopenllm_v2/GPQA": 0.3037, - "hfopenllm_v2/MUSR": 0.4315, - "hfopenllm_v2/MMLU-PRO": 0.2979 - } - }, - { - "id": "Triangle104/Q2.5-EVACOT-7b", - "name": "Q2.5-EVACOT-7b", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5784, - "hfopenllm_v2/BBH": 0.5506, - "hfopenllm_v2/MATH Level 5": 0.2825, - "hfopenllm_v2/GPQA": 0.318, - "hfopenllm_v2/MUSR": 0.4499, - "hfopenllm_v2/MMLU-PRO": 0.4331 - } - }, - { - "id": "Triangle104/Q2.5-EvaHumane-RP", - "name": "Q2.5-EvaHumane-RP", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3676, - "hfopenllm_v2/BBH": 0.5328, - "hfopenllm_v2/MATH Level 5": 0.2923, - "hfopenllm_v2/GPQA": 0.3188, - "hfopenllm_v2/MUSR": 0.4276, - "hfopenllm_v2/MMLU-PRO": 0.4412 - } - }, - { - "id": "Triangle104/Q2.5-Humane-RP", - "name": "Q2.5-Humane-RP", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4412, - "hfopenllm_v2/BBH": 0.5649, - "hfopenllm_v2/MATH Level 5": 0.3391, - "hfopenllm_v2/GPQA": 0.3188, - "hfopenllm_v2/MUSR": 0.4528, - "hfopenllm_v2/MMLU-PRO": 0.4492 - } - }, - { - "id": "Triangle104/Q2.5-Instruct-1M_Harmony", - "name": "Q2.5-Instruct-1M_Harmony", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6038, - "hfopenllm_v2/BBH": 0.5373, - "hfopenllm_v2/MATH Level 5": 0.3323, - "hfopenllm_v2/GPQA": 0.323, - "hfopenllm_v2/MUSR": 0.4688, - "hfopenllm_v2/MMLU-PRO": 0.4366 - } - }, - { - "id": "Triangle104/Q2.5-R1-3B", - "name": "Q2.5-R1-3B", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4214, - "hfopenllm_v2/BBH": 0.4812, - "hfopenllm_v2/MATH Level 5": 0.2674, - "hfopenllm_v2/GPQA": 0.3096, - "hfopenllm_v2/MUSR": 0.432, - "hfopenllm_v2/MMLU-PRO": 0.3813 - } - }, - { - "id": "Triangle104/Q2.5-R1-7B", - "name": "Q2.5-R1-7B", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1346, - "hfopenllm_v2/BBH": 0.3007, - "hfopenllm_v2/MATH Level 5": 0.0166, - "hfopenllm_v2/GPQA": 0.2525, - "hfopenllm_v2/MUSR": 0.3607, - "hfopenllm_v2/MMLU-PRO": 0.118 - } - }, - { - "id": "Triangle104/Robo-Gutenberg_V1.0", - "name": "Robo-Gutenberg_V1.0", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6008, - "hfopenllm_v2/BBH": 0.6537, - "hfopenllm_v2/MATH Level 5": 0.4562, - "hfopenllm_v2/GPQA": 0.3859, - "hfopenllm_v2/MUSR": 0.4744, - "hfopenllm_v2/MMLU-PRO": 0.5391 - } - }, - { - "id": "Triangle104/Rocinante-Prism_V2.0", - "name": "Rocinante-Prism_V2.0", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2616, - "hfopenllm_v2/BBH": 0.5361, - "hfopenllm_v2/MATH Level 5": 0.111, - "hfopenllm_v2/GPQA": 0.3205, - "hfopenllm_v2/MUSR": 0.445, - "hfopenllm_v2/MMLU-PRO": 0.364 - } - }, - { - "id": "Triangle104/Rocinante-Prism_V2.1", - "name": "Rocinante-Prism_V2.1", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2558, - "hfopenllm_v2/BBH": 0.5333, - "hfopenllm_v2/MATH Level 5": 0.1125, - "hfopenllm_v2/GPQA": 0.3196, - "hfopenllm_v2/MUSR": 0.449, - "hfopenllm_v2/MMLU-PRO": 0.3651 - } - }, - { - "id": "Triangle104/RomboHermes3-R1-Llama3.2-3b", - "name": "RomboHermes3-R1-Llama3.2-3b", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3007, - "hfopenllm_v2/BBH": 0.4264, - "hfopenllm_v2/MATH Level 5": 0.0816, - "hfopenllm_v2/GPQA": 0.2836, - "hfopenllm_v2/MUSR": 0.3657, - "hfopenllm_v2/MMLU-PRO": 0.2957 - } - }, - { - "id": "Triangle104/Rombos-Novasky-7B_V1c", - "name": "Rombos-Novasky-7B_V1c", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.408, - "hfopenllm_v2/BBH": 0.4349, - "hfopenllm_v2/MATH Level 5": 0.0853, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.4465, - "hfopenllm_v2/MMLU-PRO": 0.2738 - } - }, - { - "id": "Triangle104/Set-70b", - "name": "Set-70b", - "developer": "Triangle104", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7643, - "hfopenllm_v2/BBH": 0.7014, - "hfopenllm_v2/MATH Level 5": 0.364, - "hfopenllm_v2/GPQA": 0.4463, - "hfopenllm_v2/MUSR": 0.4696, - "hfopenllm_v2/MMLU-PRO": 0.5442 - } - }, - { - "id": "trthminh1112/autotrain-llama32-1b-finetune", - "name": "autotrain-llama32-1b-finetune", - "developer": "trthminh1112", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1769, - "hfopenllm_v2/BBH": 0.2996, - "hfopenllm_v2/MATH Level 5": 0.0151, - "hfopenllm_v2/GPQA": 0.2567, - "hfopenllm_v2/MUSR": 0.3513, - "hfopenllm_v2/MMLU-PRO": 0.1099 - } - }, - { - "id": "Tsunami-th/Tsunami-0.5-7B-Instruct", - "name": "Tsunami-0.5-7B-Instruct", - "developer": "Tsunami-th", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.74, - "hfopenllm_v2/BBH": 0.5524, - "hfopenllm_v2/MATH Level 5": 0.5045, - "hfopenllm_v2/GPQA": 0.3087, - "hfopenllm_v2/MUSR": 0.4257, - "hfopenllm_v2/MMLU-PRO": 0.4413 - } - }, - { - "id": "Tsunami-th/Tsunami-0.5x-7B-Instruct", - "name": "Tsunami-0.5x-7B-Instruct", - "developer": "Tsunami-th", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7099, - "hfopenllm_v2/BBH": 0.5593, - "hfopenllm_v2/MATH Level 5": 0.4207, - "hfopenllm_v2/GPQA": 0.3146, - "hfopenllm_v2/MUSR": 0.4667, - "hfopenllm_v2/MMLU-PRO": 0.4458 - } - }, - { - "id": "Tsunami-th/Tsunami-1.0-14B-Instruct", - "name": "Tsunami-1.0-14B-Instruct", - "developer": "Tsunami-th", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7829, - "hfopenllm_v2/BBH": 0.6439, - "hfopenllm_v2/MATH Level 5": 0.4585, - "hfopenllm_v2/GPQA": 0.3565, - "hfopenllm_v2/MUSR": 0.4459, - "hfopenllm_v2/MMLU-PRO": 0.5249 - } - }, - { - "id": "Tsunami-th/Tsunami-1.0-7B-Instruct", - "name": "Tsunami-1.0-7B-Instruct", - "developer": "Tsunami-th", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7309, - "hfopenllm_v2/BBH": 0.5491, - "hfopenllm_v2/MATH Level 5": 0.4335, - "hfopenllm_v2/GPQA": 0.3129, - "hfopenllm_v2/MUSR": 0.4493, - "hfopenllm_v2/MMLU-PRO": 0.4424 - } - }, - { - "id": "TTTXXX01/Mistral-7B-Base-SimPO2-5e-7", - "name": "Mistral-7B-Base-SimPO2-5e-7", - "developer": "TTTXXX01", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4392, - "hfopenllm_v2/BBH": 0.432, - "hfopenllm_v2/MATH Level 5": 0.0264, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.3604, - "hfopenllm_v2/MMLU-PRO": 0.2766 - } - }, - { - "id": "tugstugi/Qwen2.5-7B-Instruct-QwQ-v0.1", - "name": "Qwen2.5-7B-Instruct-QwQ-v0.1", - "developer": "tugstugi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6017, - "hfopenllm_v2/BBH": 0.5101, - "hfopenllm_v2/MATH Level 5": 0.3814, - "hfopenllm_v2/GPQA": 0.2685, - "hfopenllm_v2/MUSR": 0.3794, - "hfopenllm_v2/MMLU-PRO": 0.4081 - } - }, - { - "id": "UCLA-AGI/Gemma-2-9B-It-SPPO-Iter1", - "name": "Gemma-2-9B-It-SPPO-Iter1", - "developer": "UCLA-AGI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3082, - "hfopenllm_v2/BBH": 0.5969, - "hfopenllm_v2/MATH Level 5": 0.0899, - "hfopenllm_v2/GPQA": 0.3364, - "hfopenllm_v2/MUSR": 0.4099, - "hfopenllm_v2/MMLU-PRO": 0.3907 - } - }, - { - "id": "UCLA-AGI/Gemma-2-9B-It-SPPO-Iter2", - "name": "Gemma-2-9B-It-SPPO-Iter2", - "developer": "UCLA-AGI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.31, - "hfopenllm_v2/BBH": 0.599, - "hfopenllm_v2/MATH Level 5": 0.0808, - "hfopenllm_v2/GPQA": 0.3347, - "hfopenllm_v2/MUSR": 0.4139, - "hfopenllm_v2/MMLU-PRO": 0.387 - } - }, - { - "id": "UCLA-AGI/Gemma-2-9B-It-SPPO-Iter3", - "name": "Gemma-2-9B-It-SPPO-Iter3", - "developer": "UCLA-AGI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3167, - "hfopenllm_v2/BBH": 0.6007, - "hfopenllm_v2/MATH Level 5": 0.071, - "hfopenllm_v2/GPQA": 0.3389, - "hfopenllm_v2/MUSR": 0.4166, - "hfopenllm_v2/MMLU-PRO": 0.3826 - } - }, - { - "id": "UCLA-AGI/Llama-3-Instruct-8B-SPPO-Iter1", - "name": "Llama-3-Instruct-8B-SPPO-Iter1", - "developer": "UCLA-AGI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7299, - "hfopenllm_v2/BBH": 0.5058, - "hfopenllm_v2/MATH Level 5": 0.1148, - "hfopenllm_v2/GPQA": 0.2676, - "hfopenllm_v2/MUSR": 0.3568, - "hfopenllm_v2/MMLU-PRO": 0.3711 - } - }, - { - "id": "UCLA-AGI/Llama-3-Instruct-8B-SPPO-Iter2", - "name": "Llama-3-Instruct-8B-SPPO-Iter2", - "developer": "UCLA-AGI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6989, - "hfopenllm_v2/BBH": 0.5089, - "hfopenllm_v2/MATH Level 5": 0.1035, - "hfopenllm_v2/GPQA": 0.2668, - "hfopenllm_v2/MUSR": 0.3594, - "hfopenllm_v2/MMLU-PRO": 0.3692 - } - }, - { - "id": "UCLA-AGI/Llama-3-Instruct-8B-SPPO-Iter3", - "name": "Llama-3-Instruct-8B-SPPO-Iter3", - "developer": "UCLA-AGI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6703, - "hfopenllm_v2/BBH": 0.5076, - "hfopenllm_v2/MATH Level 5": 0.0718, - "hfopenllm_v2/GPQA": 0.2651, - "hfopenllm_v2/MUSR": 0.3647, - "hfopenllm_v2/MMLU-PRO": 0.3658 - } - }, - { - "id": "UCLA-AGI/Mistral7B-PairRM-SPPO", - "name": "Mistral7B-PairRM-SPPO", - "developer": "UCLA-AGI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4355, - "hfopenllm_v2/BBH": 0.4439, - "hfopenllm_v2/MATH Level 5": 0.031, - "hfopenllm_v2/GPQA": 0.281, - "hfopenllm_v2/MUSR": 0.3965, - "hfopenllm_v2/MMLU-PRO": 0.2621 - } - }, - { - "id": "UCLA-AGI/Mistral7B-PairRM-SPPO-Iter1", - "name": "Mistral7B-PairRM-SPPO-Iter1", - "developer": "UCLA-AGI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5047, - "hfopenllm_v2/BBH": 0.4468, - "hfopenllm_v2/MATH Level 5": 0.0249, - "hfopenllm_v2/GPQA": 0.2836, - "hfopenllm_v2/MUSR": 0.3992, - "hfopenllm_v2/MMLU-PRO": 0.2695 - } - }, - { - "id": "UCLA-AGI/Mistral7B-PairRM-SPPO-Iter2", - "name": "Mistral7B-PairRM-SPPO-Iter2", - "developer": "UCLA-AGI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4446, - "hfopenllm_v2/BBH": 0.4466, - "hfopenllm_v2/MATH Level 5": 0.0219, - "hfopenllm_v2/GPQA": 0.2886, - "hfopenllm_v2/MUSR": 0.4085, - "hfopenllm_v2/MMLU-PRO": 0.2677 - } - }, - { - "id": "UCLA-AGI/Mistral7B-PairRM-SPPO-Iter3", - "name": "Mistral7B-PairRM-SPPO-Iter3", - "developer": "UCLA-AGI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4351, - "hfopenllm_v2/BBH": 0.4397, - "hfopenllm_v2/MATH Level 5": 0.0234, - "hfopenllm_v2/GPQA": 0.2752, - "hfopenllm_v2/MUSR": 0.4071, - "hfopenllm_v2/MMLU-PRO": 0.2658 - } - }, - { - "id": "uiuc-oumi/coalm-70b", - "name": "CoALM-70B", - "developer": "uiuc-oumi", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 74.0, - "bfcl/bfcl.overall.overall_accuracy": 27.99, - "bfcl/bfcl.overall.total_cost_usd": 19.89, - "bfcl/bfcl.overall.latency_mean_s": 16.22, - "bfcl/bfcl.overall.latency_std_s": 59.91, - "bfcl/bfcl.overall.latency_p95_s": 36.0, - "bfcl/bfcl.non_live.ast_accuracy": 83.44, - "bfcl/bfcl.non_live.simple_ast_accuracy": 70.25, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 92.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 88.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 83.0, - "bfcl/bfcl.live.live_accuracy": 67.28, - "bfcl/bfcl.live.live_simple_ast_accuracy": 70.54, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 66.57, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 68.75, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 62.5, - "bfcl/bfcl.multi_turn.accuracy": 10.62, - "bfcl/bfcl.multi_turn.base_accuracy": 11.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 14.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 9.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 8.5, - "bfcl/bfcl.web_search.accuracy": 0.0, - "bfcl/bfcl.web_search.base_accuracy": 0.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 0.0, - "bfcl/bfcl.memory.accuracy": 5.81, - "bfcl/bfcl.memory.kv_accuracy": 9.03, - "bfcl/bfcl.memory.vector_accuracy": 5.16, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 3.23, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 93.75, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 85.65, - "bfcl/bfcl.format_sensitivity.max_delta": 72.0, - "bfcl/bfcl.format_sensitivity.stddev": 27.76 - } - }, - { - "id": "uiuc-oumi/coalm-8b", - "name": "CoALM-8B", - "developer": "uiuc-oumi", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 84.0, - "bfcl/bfcl.overall.overall_accuracy": 26.81, - "bfcl/bfcl.overall.total_cost_usd": 25.33, - "bfcl/bfcl.overall.latency_mean_s": 20.36, - "bfcl/bfcl.overall.latency_std_s": 73.74, - "bfcl/bfcl.overall.latency_p95_s": 138.04, - "bfcl/bfcl.non_live.ast_accuracy": 84.87, - "bfcl/bfcl.non_live.simple_ast_accuracy": 69.5, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 93.5, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 88.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 88.5, - "bfcl/bfcl.live.live_accuracy": 66.77, - "bfcl/bfcl.live.live_simple_ast_accuracy": 70.54, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 66.19, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 62.5, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 54.17, - "bfcl/bfcl.multi_turn.accuracy": 8.0, - "bfcl/bfcl.multi_turn.base_accuracy": 10.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 7.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 8.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 7.0, - "bfcl/bfcl.web_search.accuracy": 0.0, - "bfcl/bfcl.web_search.base_accuracy": 0.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 0.0, - "bfcl/bfcl.memory.accuracy": 2.8, - "bfcl/bfcl.memory.kv_accuracy": 3.23, - "bfcl/bfcl.memory.vector_accuracy": 3.87, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 1.29, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 87.5, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 86.9, - "bfcl/bfcl.format_sensitivity.max_delta": 79.0, - "bfcl/bfcl.format_sensitivity.stddev": 34.18 - } - }, - { - "id": "UKzExecution/LlamaExecutor-8B-3.0.5", - "name": "LlamaExecutor-8B-3.0.5", - "developer": "UKzExecution", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7403, - "hfopenllm_v2/BBH": 0.5006, - "hfopenllm_v2/MATH Level 5": 0.102, - "hfopenllm_v2/GPQA": 0.2559, - "hfopenllm_v2/MUSR": 0.3754, - "hfopenllm_v2/MMLU-PRO": 0.3625 - } - }, - { - "id": "Unbabel/TowerInstruct-Mistral-7B-v0.2", - "name": "TowerInstruct-Mistral-7B-v0.2", - "developer": "Unbabel", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2843, - "hfopenllm_v2/BBH": 0.3882, - "hfopenllm_v2/MATH Level 5": 0.0204, - "hfopenllm_v2/GPQA": 0.2475, - "hfopenllm_v2/MUSR": 0.4522, - "hfopenllm_v2/MMLU-PRO": 0.1968 - } - }, - { - "id": "Undi95/MG-FinalMix-72B", - "name": "MG-FinalMix-72B", - "developer": "Undi95", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8014, - "hfopenllm_v2/BBH": 0.6973, - "hfopenllm_v2/MATH Level 5": 0.3973, - "hfopenllm_v2/GPQA": 0.3851, - "hfopenllm_v2/MUSR": 0.4823, - "hfopenllm_v2/MMLU-PRO": 0.5427 - } - }, - { - "id": "Undi95/Phi4-abliterated", - "name": "Phi4-abliterated", - "developer": "Undi95", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6618, - "hfopenllm_v2/BBH": 0.6809, - "hfopenllm_v2/MATH Level 5": 0.3701, - "hfopenllm_v2/GPQA": 0.3305, - "hfopenllm_v2/MUSR": 0.4034, - "hfopenllm_v2/MMLU-PRO": 0.5281 - } - }, - { - "id": "universalml/NepaliGPT-2.0", - "name": "NepaliGPT-2.0", - "developer": "universalml", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0365, - "hfopenllm_v2/BBH": 0.466, - "hfopenllm_v2/MATH Level 5": 0.0045, - "hfopenllm_v2/GPQA": 0.281, - "hfopenllm_v2/MUSR": 0.4657, - "hfopenllm_v2/MMLU-PRO": 0.33 - } - }, - { - "id": "unknown/aya-expanse-32b", - "name": "aya-expanse-32b", - "developer": "unknown", - "evaluator_relationship": null, - "benchmark_scores": { - "global-mmlu-lite/Global MMLU Lite": 0.7353, - "global-mmlu-lite/Culturally Sensitive": 0.6891, - "global-mmlu-lite/Culturally Agnostic": 0.7815, - "global-mmlu-lite/Arabic": 0.7425, - "global-mmlu-lite/English": 0.7544, - "global-mmlu-lite/Bengali": 0.7343, - "global-mmlu-lite/German": 0.7425, - "global-mmlu-lite/French": 0.7325, - "global-mmlu-lite/Hindi": 0.7375, - "global-mmlu-lite/Indonesian": 0.7594, - "global-mmlu-lite/Italian": 0.7305, - "global-mmlu-lite/Japanese": 0.7419, - "global-mmlu-lite/Korean": 0.7525, - "global-mmlu-lite/Portuguese": 0.7544, - "global-mmlu-lite/Spanish": 0.7362, - "global-mmlu-lite/Swahili": 0.7071, - "global-mmlu-lite/Yoruba": 0.6942, - "global-mmlu-lite/Chinese": 0.743, - "global-mmlu-lite/Burmese": 0.7025 - } - }, - { - "id": "unknown/granite-4.0-h-small", - "name": "granite-4.0-h-small", - "developer": "unknown", - "evaluator_relationship": null, - "benchmark_scores": { - "global-mmlu-lite/Global MMLU Lite": 0.7503, - "global-mmlu-lite/Culturally Sensitive": 0.7182, - "global-mmlu-lite/Culturally Agnostic": 0.7826, - "global-mmlu-lite/Arabic": 0.7613, - "global-mmlu-lite/English": 0.77, - "global-mmlu-lite/Bengali": 0.7613, - "global-mmlu-lite/German": 0.755, - "global-mmlu-lite/French": 0.7594, - "global-mmlu-lite/Hindi": 0.7575, - "global-mmlu-lite/Indonesian": 0.7614, - "global-mmlu-lite/Italian": 0.7525, - "global-mmlu-lite/Japanese": 0.7406, - "global-mmlu-lite/Korean": 0.7525, - "global-mmlu-lite/Portuguese": 0.757, - "global-mmlu-lite/Spanish": 0.7638, - "global-mmlu-lite/Swahili": 0.7318, - "global-mmlu-lite/Yoruba": 0.6921, - "global-mmlu-lite/Chinese": 0.7475, - "global-mmlu-lite/Burmese": 0.7419 - } - }, - { - "id": "unknown/o4-mini-2025-04-16", - "name": "o4-mini-2025-04-16", - "developer": "unknown", - "evaluator_relationship": null, - "benchmark_scores": { - "global-mmlu-lite/Global MMLU Lite": 0.8705, - "global-mmlu-lite/Culturally Sensitive": 0.8503, - "global-mmlu-lite/Culturally Agnostic": 0.8906, - "global-mmlu-lite/Arabic": 0.865, - "global-mmlu-lite/English": 0.8675, - "global-mmlu-lite/Bengali": 0.8875, - "global-mmlu-lite/German": 0.8775, - "global-mmlu-lite/French": 0.87, - "global-mmlu-lite/Hindi": 0.87, - "global-mmlu-lite/Indonesian": 0.8675, - "global-mmlu-lite/Italian": 0.855, - "global-mmlu-lite/Japanese": 0.885, - "global-mmlu-lite/Korean": 0.88, - "global-mmlu-lite/Portuguese": 0.88, - "global-mmlu-lite/Spanish": 0.855, - "global-mmlu-lite/Swahili": 0.8525, - "global-mmlu-lite/Yoruba": 0.8525, - "global-mmlu-lite/Chinese": 0.89, - "global-mmlu-lite/Burmese": 0.8725 - } - }, - { - "id": "unsloth/Llama-3.2-1B-Instruct", - "name": "Llama-3.2-1B-Instruct", - "developer": "unsloth", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.581, - "hfopenllm_v2/BBH": 0.3485, - "hfopenllm_v2/MATH Level 5": 0.0823, - "hfopenllm_v2/GPQA": 0.2676, - "hfopenllm_v2/MUSR": 0.3196, - "hfopenllm_v2/MMLU-PRO": 0.1742 - } - }, - { - "id": "unsloth/Llama-3.2-1B-Instruct-no-system-message", - "name": "Llama-3.2-1B-Instruct-no-system-message", - "developer": "unsloth", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.565, - "hfopenllm_v2/BBH": 0.3544, - "hfopenllm_v2/MATH Level 5": 0.0755, - "hfopenllm_v2/GPQA": 0.2727, - "hfopenllm_v2/MUSR": 0.3341, - "hfopenllm_v2/MMLU-PRO": 0.1669 - } - }, - { - "id": "unsloth/Phi-3-mini-4k-instruct", - "name": "Phi-3-mini-4k-instruct", - "developer": "unsloth", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.544, - "hfopenllm_v2/BBH": 0.55, - "hfopenllm_v2/MATH Level 5": 0.1639, - "hfopenllm_v2/GPQA": 0.323, - "hfopenllm_v2/MUSR": 0.4284, - "hfopenllm_v2/MMLU-PRO": 0.4031 - } - }, - { - "id": "unsloth/phi-4", - "name": "phi-4", - "developer": "unsloth", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6882, - "hfopenllm_v2/BBH": 0.6886, - "hfopenllm_v2/MATH Level 5": 0.5, - "hfopenllm_v2/GPQA": 0.3364, - "hfopenllm_v2/MUSR": 0.4114, - "hfopenllm_v2/MMLU-PRO": 0.5378 - } - }, - { - "id": "unsloth/phi-4-bnb-4bit", - "name": "phi-4-bnb-4bit", - "developer": "unsloth", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.673, - "hfopenllm_v2/BBH": 0.677, - "hfopenllm_v2/MATH Level 5": 0.4607, - "hfopenllm_v2/GPQA": 0.3381, - "hfopenllm_v2/MUSR": 0.4007, - "hfopenllm_v2/MMLU-PRO": 0.5256 - } - }, - { - "id": "unsloth/phi-4-unsloth-bnb-4bit", - "name": "phi-4-unsloth-bnb-4bit", - "developer": "unsloth", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6794, - "hfopenllm_v2/BBH": 0.6791, - "hfopenllm_v2/MATH Level 5": 0.4562, - "hfopenllm_v2/GPQA": 0.3364, - "hfopenllm_v2/MUSR": 0.4034, - "hfopenllm_v2/MMLU-PRO": 0.5286 - } - }, - { - "id": "upstage/SOLAR-10.7B-Instruct-v1.0", - "name": "SOLAR-10.7B-Instruct-v1.0", - "developer": "upstage", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4737, - "hfopenllm_v2/BBH": 0.5162, - "hfopenllm_v2/MATH Level 5": 0.0566, - "hfopenllm_v2/GPQA": 0.3087, - "hfopenllm_v2/MUSR": 0.3899, - "hfopenllm_v2/MMLU-PRO": 0.3138, - "reward-bench/Score": 0.7391, - "reward-bench/Chat": 0.8156, - "reward-bench/Chat Hard": 0.6864, - "reward-bench/Safety": 0.8514, - "reward-bench/Reasoning": 0.7252, - "reward-bench/Prior Sets (0.5 weight)": 0.4949 - } - }, - { - "id": "upstage/SOLAR-10.7B-v1.0", - "name": "SOLAR-10.7B-v1.0", - "developer": "upstage", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2421, - "hfopenllm_v2/BBH": 0.5094, - "hfopenllm_v2/MATH Level 5": 0.0264, - "hfopenllm_v2/GPQA": 0.281, - "hfopenllm_v2/MUSR": 0.4372, - "hfopenllm_v2/MMLU-PRO": 0.34 - } - }, - { - "id": "upstage/solar-pro-241126", - "name": "Solar Pro", - "developer": "upstage", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.602, - "helm_lite/NarrativeQA": 0.753, - "helm_lite/NaturalQuestions (closed-book)": 0.297, - "helm_lite/OpenbookQA": 0.922, - "helm_lite/MMLU": 0.679, - "helm_lite/MATH": 0.567, - "helm_lite/GSM8K": 0.871, - "helm_lite/LegalBench": 0.67, - "helm_lite/MedQA": 0.698, - "helm_lite/WMT 2014": 0.169, - "helm_mmlu/MMLU All Subjects": 0.776, - "helm_mmlu/Abstract Algebra": 0.46, - "helm_mmlu/Anatomy": 0.719, - "helm_mmlu/College Physics": 0.559, - "helm_mmlu/Computer Security": 0.82, - "helm_mmlu/Econometrics": 0.605, - "helm_mmlu/Global Facts": 0.5, - "helm_mmlu/Jurisprudence": 0.898, - "helm_mmlu/Philosophy": 0.817, - "helm_mmlu/Professional Psychology": 0.85, - "helm_mmlu/Us Foreign Policy": 0.97, - "helm_mmlu/Astronomy": 0.868, - "helm_mmlu/Business Ethics": 0.8, - "helm_mmlu/Clinical Knowledge": 0.808, - "helm_mmlu/Conceptual Physics": 0.826, - "helm_mmlu/Electrical Engineering": 0.697, - "helm_mmlu/Elementary Mathematics": 0.611, - "helm_mmlu/Formal Logic": 0.579, - "helm_mmlu/High School World History": 0.907, - "helm_mmlu/Human Sexuality": 0.847, - "helm_mmlu/International Law": 0.901, - "helm_mmlu/Logical Fallacies": 0.865, - "helm_mmlu/Machine Learning": 0.616, - "helm_mmlu/Management": 0.864, - "helm_mmlu/Marketing": 0.953, - "helm_mmlu/Medical Genetics": 0.91, - "helm_mmlu/Miscellaneous": 0.888, - "helm_mmlu/Moral Scenarios": 0.811, - "helm_mmlu/Nutrition": 0.859, - "helm_mmlu/Prehistory": 0.867, - "helm_mmlu/Public Relations": 0.764, - "helm_mmlu/Security Studies": 0.82, - "helm_mmlu/Sociology": 0.886, - "helm_mmlu/Virology": 0.572, - "helm_mmlu/World Religions": 0.883, - "helm_mmlu/Mean win rate": 0.462 - } - }, - { - "id": "upstage/solar-pro-preview-instruct", - "name": "solar-pro-preview-instruct", - "developer": "upstage", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8416, - "hfopenllm_v2/BBH": 0.6817, - "hfopenllm_v2/MATH Level 5": 0.2205, - "hfopenllm_v2/GPQA": 0.3708, - "hfopenllm_v2/MUSR": 0.4417, - "hfopenllm_v2/MMLU-PRO": 0.5273 - } - }, - { - "id": "utkmst/chimera-beta-test2-lora-merged", - "name": "chimera-beta-test2-lora-merged", - "developer": "utkmst", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6054, - "hfopenllm_v2/BBH": 0.4796, - "hfopenllm_v2/MATH Level 5": 0.0952, - "hfopenllm_v2/GPQA": 0.3037, - "hfopenllm_v2/MUSR": 0.4118, - "hfopenllm_v2/MMLU-PRO": 0.2992 - } - }, - { - "id": "utter-project/EuroLLM-9B", - "name": "EuroLLM 9B", - "developer": "unknown", - "evaluator_relationship": null, - "benchmark_scores": { - "la_leaderboard/la_leaderboard": 25.87 - } - }, - { - "id": "uukuguy/speechless-code-mistral-7b-v1.0", - "name": "speechless-code-mistral-7b-v1.0", - "developer": "uukuguy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3665, - "hfopenllm_v2/BBH": 0.4572, - "hfopenllm_v2/MATH Level 5": 0.0521, - "hfopenllm_v2/GPQA": 0.2844, - "hfopenllm_v2/MUSR": 0.4502, - "hfopenllm_v2/MMLU-PRO": 0.3146 - } - }, - { - "id": "uukuguy/speechless-codellama-34b-v2.0", - "name": "speechless-codellama-34b-v2.0", - "developer": "uukuguy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4604, - "hfopenllm_v2/BBH": 0.4813, - "hfopenllm_v2/MATH Level 5": 0.0431, - "hfopenllm_v2/GPQA": 0.2693, - "hfopenllm_v2/MUSR": 0.3787, - "hfopenllm_v2/MMLU-PRO": 0.2542 - } - }, - { - "id": "uukuguy/speechless-coder-ds-6.7b", - "name": "speechless-coder-ds-6.7b", - "developer": "uukuguy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2505, - "hfopenllm_v2/BBH": 0.4036, - "hfopenllm_v2/MATH Level 5": 0.0211, - "hfopenllm_v2/GPQA": 0.2643, - "hfopenllm_v2/MUSR": 0.3819, - "hfopenllm_v2/MMLU-PRO": 0.1719 - } - }, - { - "id": "uukuguy/speechless-instruct-mistral-7b-v0.2", - "name": "speechless-instruct-mistral-7b-v0.2", - "developer": "uukuguy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3261, - "hfopenllm_v2/BBH": 0.4607, - "hfopenllm_v2/MATH Level 5": 0.0491, - "hfopenllm_v2/GPQA": 0.2819, - "hfopenllm_v2/MUSR": 0.4902, - "hfopenllm_v2/MMLU-PRO": 0.2902 - } - }, - { - "id": "uukuguy/speechless-llama2-hermes-orca-platypus-wizardlm-13b", - "name": "speechless-llama2-hermes-orca-platypus-wizardlm-13b", - "developer": "uukuguy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4562, - "hfopenllm_v2/BBH": 0.4846, - "hfopenllm_v2/MATH Level 5": 0.0204, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.4655, - "hfopenllm_v2/MMLU-PRO": 0.2559 - } - }, - { - "id": "uukuguy/speechless-mistral-dolphin-orca-platypus-samantha-7b", - "name": "speechless-mistral-dolphin-orca-platypus-samantha-7b", - "developer": "uukuguy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.37, - "hfopenllm_v2/BBH": 0.4983, - "hfopenllm_v2/MATH Level 5": 0.0295, - "hfopenllm_v2/GPQA": 0.2836, - "hfopenllm_v2/MUSR": 0.4361, - "hfopenllm_v2/MMLU-PRO": 0.299 - } - }, - { - "id": "uukuguy/speechless-zephyr-code-functionary-7b", - "name": "speechless-zephyr-code-functionary-7b", - "developer": "uukuguy", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2696, - "hfopenllm_v2/BBH": 0.4664, - "hfopenllm_v2/MATH Level 5": 0.0423, - "hfopenllm_v2/GPQA": 0.3003, - "hfopenllm_v2/MUSR": 0.4268, - "hfopenllm_v2/MMLU-PRO": 0.3094 - } - }, - { - "id": "v000000/L3-8B-Stheno-v3.2-abliterated", - "name": "L3-8B-Stheno-v3.2-abliterated", - "developer": "v000000", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6718, - "hfopenllm_v2/BBH": 0.5141, - "hfopenllm_v2/MATH Level 5": 0.0695, - "hfopenllm_v2/GPQA": 0.3096, - "hfopenllm_v2/MUSR": 0.362, - "hfopenllm_v2/MMLU-PRO": 0.3604 - } - }, - { - "id": "v000000/L3.1-Niitorm-8B-DPO-t0.0001", - "name": "L3.1-Niitorm-8B-DPO-t0.0001", - "developer": "v000000", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7689, - "hfopenllm_v2/BBH": 0.5134, - "hfopenllm_v2/MATH Level 5": 0.1624, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.388, - "hfopenllm_v2/MMLU-PRO": 0.3866 - } - }, - { - "id": "v000000/L3.1-Storniitova-8B", - "name": "L3.1-Storniitova-8B", - "developer": "v000000", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7817, - "hfopenllm_v2/BBH": 0.5151, - "hfopenllm_v2/MATH Level 5": 0.1465, - "hfopenllm_v2/GPQA": 0.2894, - "hfopenllm_v2/MUSR": 0.4029, - "hfopenllm_v2/MMLU-PRO": 0.3776 - } - }, - { - "id": "v000000/Qwen2.5-14B-Gutenberg-1e-Delta", - "name": "Qwen2.5-14B-Gutenberg-1e-Delta", - "developer": "v000000", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8045, - "hfopenllm_v2/BBH": 0.6398, - "hfopenllm_v2/MATH Level 5": 0.5264, - "hfopenllm_v2/GPQA": 0.3289, - "hfopenllm_v2/MUSR": 0.4073, - "hfopenllm_v2/MMLU-PRO": 0.493 - } - }, - { - "id": "v000000/Qwen2.5-14B-Gutenberg-Instruct-Slerpeno", - "name": "Qwen2.5-14B-Gutenberg-Instruct-Slerpeno", - "developer": "v000000", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8197, - "hfopenllm_v2/BBH": 0.639, - "hfopenllm_v2/MATH Level 5": 0.5325, - "hfopenllm_v2/GPQA": 0.3314, - "hfopenllm_v2/MUSR": 0.4114, - "hfopenllm_v2/MMLU-PRO": 0.4924 - } - }, - { - "id": "v000000/Qwen2.5-Lumen-14B", - "name": "Qwen2.5-Lumen-14B", - "developer": "v000000", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8064, - "hfopenllm_v2/BBH": 0.6391, - "hfopenllm_v2/MATH Level 5": 0.5363, - "hfopenllm_v2/GPQA": 0.328, - "hfopenllm_v2/MUSR": 0.4114, - "hfopenllm_v2/MMLU-PRO": 0.4903 - } - }, - { - "id": "V3N0M/Jenna-Tiny-2.0", - "name": "Jenna-Tiny-2.0", - "developer": "V3N0M", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2309, - "hfopenllm_v2/BBH": 0.3148, - "hfopenllm_v2/MATH Level 5": 0.0121, - "hfopenllm_v2/GPQA": 0.25, - "hfopenllm_v2/MUSR": 0.3367, - "hfopenllm_v2/MMLU-PRO": 0.1147 - } - }, - { - "id": "VAGOsolutions/Llama-3-SauerkrautLM-70b-Instruct", - "name": "Llama-3-SauerkrautLM-70b-Instruct", - "developer": "VAGOsolutions", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8045, - "hfopenllm_v2/BBH": 0.6663, - "hfopenllm_v2/MATH Level 5": 0.2281, - "hfopenllm_v2/GPQA": 0.328, - "hfopenllm_v2/MUSR": 0.4339, - "hfopenllm_v2/MMLU-PRO": 0.5392 - } - }, - { - "id": "VAGOsolutions/Llama-3-SauerkrautLM-8b-Instruct", - "name": "Llama-3-SauerkrautLM-8b-Instruct", - "developer": "VAGOsolutions", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7445, - "hfopenllm_v2/BBH": 0.4943, - "hfopenllm_v2/MATH Level 5": 0.0665, - "hfopenllm_v2/GPQA": 0.3087, - "hfopenllm_v2/MUSR": 0.4241, - "hfopenllm_v2/MMLU-PRO": 0.3857 - } - }, - { - "id": "VAGOsolutions/Llama-3.1-SauerkrautLM-70b-Instruct", - "name": "Llama-3.1-SauerkrautLM-70b-Instruct", - "developer": "VAGOsolutions", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8656, - "hfopenllm_v2/BBH": 0.7006, - "hfopenllm_v2/MATH Level 5": 0.3693, - "hfopenllm_v2/GPQA": 0.3414, - "hfopenllm_v2/MUSR": 0.4711, - "hfopenllm_v2/MMLU-PRO": 0.5335 - } - }, - { - "id": "VAGOsolutions/Llama-3.1-SauerkrautLM-8b-Instruct", - "name": "Llama-3.1-SauerkrautLM-8b-Instruct", - "developer": "VAGOsolutions", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8017, - "hfopenllm_v2/BBH": 0.5115, - "hfopenllm_v2/MATH Level 5": 0.1941, - "hfopenllm_v2/GPQA": 0.2903, - "hfopenllm_v2/MUSR": 0.4148, - "hfopenllm_v2/MMLU-PRO": 0.389 - } - }, - { - "id": "VAGOsolutions/SauerkrautLM-1.5b", - "name": "SauerkrautLM-1.5b", - "developer": "VAGOsolutions", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2404, - "hfopenllm_v2/BBH": 0.3704, - "hfopenllm_v2/MATH Level 5": 0.0363, - "hfopenllm_v2/GPQA": 0.271, - "hfopenllm_v2/MUSR": 0.3739, - "hfopenllm_v2/MMLU-PRO": 0.2151 - } - }, - { - "id": "VAGOsolutions/SauerkrautLM-7b-HerO", - "name": "SauerkrautLM-7b-HerO", - "developer": "VAGOsolutions", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5346, - "hfopenllm_v2/BBH": 0.4904, - "hfopenllm_v2/MATH Level 5": 0.0393, - "hfopenllm_v2/GPQA": 0.2727, - "hfopenllm_v2/MUSR": 0.3924, - "hfopenllm_v2/MMLU-PRO": 0.3046 - } - }, - { - "id": "VAGOsolutions/SauerkrautLM-7b-LaserChat", - "name": "SauerkrautLM-7b-LaserChat", - "developer": "VAGOsolutions", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5988, - "hfopenllm_v2/BBH": 0.4543, - "hfopenllm_v2/MATH Level 5": 0.0778, - "hfopenllm_v2/GPQA": 0.3003, - "hfopenllm_v2/MUSR": 0.4148, - "hfopenllm_v2/MMLU-PRO": 0.3305 - } - }, - { - "id": "VAGOsolutions/SauerkrautLM-gemma-2-2b-it", - "name": "SauerkrautLM-gemma-2-2b-it", - "developer": "VAGOsolutions", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1321, - "hfopenllm_v2/BBH": 0.4241, - "hfopenllm_v2/MATH Level 5": 0.0219, - "hfopenllm_v2/GPQA": 0.2727, - "hfopenllm_v2/MUSR": 0.3995, - "hfopenllm_v2/MMLU-PRO": 0.2693 - } - }, - { - "id": "VAGOsolutions/SauerkrautLM-gemma-2-9b-it", - "name": "SauerkrautLM-gemma-2-9b-it", - "developer": "VAGOsolutions", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3024, - "hfopenllm_v2/BBH": 0.6073, - "hfopenllm_v2/MATH Level 5": 0.0838, - "hfopenllm_v2/GPQA": 0.3272, - "hfopenllm_v2/MUSR": 0.4318, - "hfopenllm_v2/MMLU-PRO": 0.4091 - } - }, - { - "id": "VAGOsolutions/SauerkrautLM-Gemma-2b", - "name": "SauerkrautLM-Gemma-2b", - "developer": "VAGOsolutions", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2475, - "hfopenllm_v2/BBH": 0.3416, - "hfopenllm_v2/MATH Level 5": 0.0279, - "hfopenllm_v2/GPQA": 0.2567, - "hfopenllm_v2/MUSR": 0.3676, - "hfopenllm_v2/MMLU-PRO": 0.1469 - } - }, - { - "id": "VAGOsolutions/SauerkrautLM-Gemma-7b", - "name": "SauerkrautLM-Gemma-7b", - "developer": "VAGOsolutions", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3407, - "hfopenllm_v2/BBH": 0.4188, - "hfopenllm_v2/MATH Level 5": 0.0672, - "hfopenllm_v2/GPQA": 0.2861, - "hfopenllm_v2/MUSR": 0.3594, - "hfopenllm_v2/MMLU-PRO": 0.2961 - } - }, - { - "id": "VAGOsolutions/SauerkrautLM-Mixtral-8x7B-Instruct", - "name": "SauerkrautLM-Mixtral-8x7B-Instruct", - "developer": "VAGOsolutions", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5602, - "hfopenllm_v2/BBH": 0.5277, - "hfopenllm_v2/MATH Level 5": 0.0982, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.4204, - "hfopenllm_v2/MMLU-PRO": 0.365 - } - }, - { - "id": "VAGOsolutions/SauerkrautLM-Nemo-12b-Instruct", - "name": "SauerkrautLM-Nemo-12b-Instruct", - "developer": "VAGOsolutions", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6113, - "hfopenllm_v2/BBH": 0.5214, - "hfopenllm_v2/MATH Level 5": 0.1224, - "hfopenllm_v2/GPQA": 0.3096, - "hfopenllm_v2/MUSR": 0.4469, - "hfopenllm_v2/MMLU-PRO": 0.3385 - } - }, - { - "id": "VAGOsolutions/SauerkrautLM-Phi-3-medium", - "name": "SauerkrautLM-Phi-3-medium", - "developer": "VAGOsolutions", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4409, - "hfopenllm_v2/BBH": 0.6433, - "hfopenllm_v2/MATH Level 5": 0.1601, - "hfopenllm_v2/GPQA": 0.3347, - "hfopenllm_v2/MUSR": 0.4845, - "hfopenllm_v2/MMLU-PRO": 0.4665 - } - }, - { - "id": "VAGOsolutions/SauerkrautLM-SOLAR-Instruct", - "name": "SauerkrautLM-SOLAR-Instruct", - "developer": "VAGOsolutions", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4917, - "hfopenllm_v2/BBH": 0.5169, - "hfopenllm_v2/MATH Level 5": 0.0634, - "hfopenllm_v2/GPQA": 0.3054, - "hfopenllm_v2/MUSR": 0.3965, - "hfopenllm_v2/MMLU-PRO": 0.3183 - } - }, - { - "id": "VAGOsolutions/SauerkrautLM-v2-14b-DPO", - "name": "SauerkrautLM-v2-14b-DPO", - "developer": "VAGOsolutions", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7412, - "hfopenllm_v2/BBH": 0.656, - "hfopenllm_v2/MATH Level 5": 0.3165, - "hfopenllm_v2/GPQA": 0.3196, - "hfopenllm_v2/MUSR": 0.4375, - "hfopenllm_v2/MMLU-PRO": 0.5117 - } - }, - { - "id": "VAGOsolutions/SauerkrautLM-v2-14b-SFT", - "name": "SauerkrautLM-v2-14b-SFT", - "developer": "VAGOsolutions", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6949, - "hfopenllm_v2/BBH": 0.621, - "hfopenllm_v2/MATH Level 5": 0.3285, - "hfopenllm_v2/GPQA": 0.3356, - "hfopenllm_v2/MUSR": 0.4179, - "hfopenllm_v2/MMLU-PRO": 0.5205 - } - }, - { - "id": "ValiantLabs/Llama3-70B-Fireplace", - "name": "Llama3-70B-Fireplace", - "developer": "ValiantLabs", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7774, - "hfopenllm_v2/BBH": 0.6489, - "hfopenllm_v2/MATH Level 5": 0.2145, - "hfopenllm_v2/GPQA": 0.3549, - "hfopenllm_v2/MUSR": 0.4449, - "hfopenllm_v2/MMLU-PRO": 0.4893 - } - }, - { - "id": "ValiantLabs/Llama3-70B-ShiningValiant2", - "name": "Llama3-70B-ShiningValiant2", - "developer": "ValiantLabs", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6122, - "hfopenllm_v2/BBH": 0.6338, - "hfopenllm_v2/MATH Level 5": 0.2077, - "hfopenllm_v2/GPQA": 0.3305, - "hfopenllm_v2/MUSR": 0.4326, - "hfopenllm_v2/MMLU-PRO": 0.4898 - } - }, - { - "id": "ValiantLabs/Llama3.1-70B-ShiningValiant2", - "name": "Llama3.1-70B-ShiningValiant2", - "developer": "ValiantLabs", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5355, - "hfopenllm_v2/BBH": 0.6738, - "hfopenllm_v2/MATH Level 5": 0.2915, - "hfopenllm_v2/GPQA": 0.3926, - "hfopenllm_v2/MUSR": 0.4681, - "hfopenllm_v2/MMLU-PRO": 0.5173 - } - }, - { - "id": "ValiantLabs/Llama3.1-8B-Cobalt", - "name": "Llama3.1-8B-Cobalt", - "developer": "ValiantLabs", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3496, - "hfopenllm_v2/BBH": 0.4947, - "hfopenllm_v2/MATH Level 5": 0.1269, - "hfopenllm_v2/GPQA": 0.3037, - "hfopenllm_v2/MUSR": 0.3959, - "hfopenllm_v2/MMLU-PRO": 0.3644 - } - }, - { - "id": "ValiantLabs/Llama3.1-8B-Enigma", - "name": "Llama3.1-8B-Enigma", - "developer": "ValiantLabs", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2681, - "hfopenllm_v2/BBH": 0.4478, - "hfopenllm_v2/MATH Level 5": 0.0891, - "hfopenllm_v2/GPQA": 0.2878, - "hfopenllm_v2/MUSR": 0.4196, - "hfopenllm_v2/MMLU-PRO": 0.3409 - } - }, - { - "id": "ValiantLabs/Llama3.1-8B-Esper2", - "name": "Llama3.1-8B-Esper2", - "developer": "ValiantLabs", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2567, - "hfopenllm_v2/BBH": 0.447, - "hfopenllm_v2/MATH Level 5": 0.0589, - "hfopenllm_v2/GPQA": 0.2727, - "hfopenllm_v2/MUSR": 0.3561, - "hfopenllm_v2/MMLU-PRO": 0.2904 - } - }, - { - "id": "ValiantLabs/Llama3.1-8B-Fireplace2", - "name": "Llama3.1-8B-Fireplace2", - "developer": "ValiantLabs", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5328, - "hfopenllm_v2/BBH": 0.4613, - "hfopenllm_v2/MATH Level 5": 0.0876, - "hfopenllm_v2/GPQA": 0.2894, - "hfopenllm_v2/MUSR": 0.3367, - "hfopenllm_v2/MMLU-PRO": 0.2424 - } - }, - { - "id": "ValiantLabs/Llama3.1-8B-ShiningValiant2", - "name": "Llama3.1-8B-ShiningValiant2", - "developer": "ValiantLabs", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2678, - "hfopenllm_v2/BBH": 0.4429, - "hfopenllm_v2/MATH Level 5": 0.0521, - "hfopenllm_v2/GPQA": 0.302, - "hfopenllm_v2/MUSR": 0.3959, - "hfopenllm_v2/MMLU-PRO": 0.2927 - } - }, - { - "id": "ValiantLabs/Llama3.2-3B-Enigma", - "name": "Llama3.2-3B-Enigma", - "developer": "ValiantLabs", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2786, - "hfopenllm_v2/BBH": 0.3723, - "hfopenllm_v2/MATH Level 5": 0.0438, - "hfopenllm_v2/GPQA": 0.2617, - "hfopenllm_v2/MUSR": 0.3921, - "hfopenllm_v2/MMLU-PRO": 0.2428 - } - }, - { - "id": "ValiantLabs/Llama3.2-3B-Esper2", - "name": "Llama3.2-3B-Esper2", - "developer": "ValiantLabs", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.275, - "hfopenllm_v2/BBH": 0.3808, - "hfopenllm_v2/MATH Level 5": 0.0363, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.355, - "hfopenllm_v2/MMLU-PRO": 0.2257 - } - }, - { - "id": "ValiantLabs/Llama3.2-3B-ShiningValiant2", - "name": "Llama3.2-3B-ShiningValiant2", - "developer": "ValiantLabs", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2625, - "hfopenllm_v2/BBH": 0.4226, - "hfopenllm_v2/MATH Level 5": 0.0823, - "hfopenllm_v2/GPQA": 0.2802, - "hfopenllm_v2/MUSR": 0.3866, - "hfopenllm_v2/MMLU-PRO": 0.2829 - } - }, - { - "id": "vhab10/llama-3-8b-merged-linear", - "name": "llama-3-8b-merged-linear", - "developer": "vhab10", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5917, - "hfopenllm_v2/BBH": 0.4937, - "hfopenllm_v2/MATH Level 5": 0.0816, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.4191, - "hfopenllm_v2/MMLU-PRO": 0.3704 - } - }, - { - "id": "vhab10/Llama-3.1-8B-Base-Instruct-SLERP", - "name": "Llama-3.1-8B-Base-Instruct-SLERP", - "developer": "vhab10", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2907, - "hfopenllm_v2/BBH": 0.5057, - "hfopenllm_v2/MATH Level 5": 0.1201, - "hfopenllm_v2/GPQA": 0.2961, - "hfopenllm_v2/MUSR": 0.4011, - "hfopenllm_v2/MMLU-PRO": 0.3621 - } - }, - { - "id": "vhab10/Llama-3.2-Instruct-3B-TIES", - "name": "Llama-3.2-Instruct-3B-TIES", - "developer": "vhab10", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4727, - "hfopenllm_v2/BBH": 0.4332, - "hfopenllm_v2/MATH Level 5": 0.0982, - "hfopenllm_v2/GPQA": 0.2693, - "hfopenllm_v2/MUSR": 0.3497, - "hfopenllm_v2/MMLU-PRO": 0.2916 - } - }, - { - "id": "vicgalle/CarbonBeagle-11B", - "name": "CarbonBeagle-11B", - "developer": "vicgalle", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5415, - "hfopenllm_v2/BBH": 0.5294, - "hfopenllm_v2/MATH Level 5": 0.0619, - "hfopenllm_v2/GPQA": 0.302, - "hfopenllm_v2/MUSR": 0.402, - "hfopenllm_v2/MMLU-PRO": 0.3276 - } - }, - { - "id": "vicgalle/CarbonBeagle-11B-truthy", - "name": "CarbonBeagle-11B-truthy", - "developer": "vicgalle", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5212, - "hfopenllm_v2/BBH": 0.5348, - "hfopenllm_v2/MATH Level 5": 0.0491, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.374, - "hfopenllm_v2/MMLU-PRO": 0.3357 - } - }, - { - "id": "vicgalle/Configurable-Hermes-2-Pro-Llama-3-8B", - "name": "Configurable-Hermes-2-Pro-Llama-3-8B", - "developer": "vicgalle", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5763, - "hfopenllm_v2/BBH": 0.5055, - "hfopenllm_v2/MATH Level 5": 0.0763, - "hfopenllm_v2/GPQA": 0.297, - "hfopenllm_v2/MUSR": 0.4184, - "hfopenllm_v2/MMLU-PRO": 0.3098 - } - }, - { - "id": "vicgalle/Configurable-Llama-3.1-8B-Instruct", - "name": "Configurable-Llama-3.1-8B-Instruct", - "developer": "vicgalle", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8312, - "hfopenllm_v2/BBH": 0.5045, - "hfopenllm_v2/MATH Level 5": 0.173, - "hfopenllm_v2/GPQA": 0.2743, - "hfopenllm_v2/MUSR": 0.3845, - "hfopenllm_v2/MMLU-PRO": 0.3592 - } - }, - { - "id": "vicgalle/Configurable-Yi-1.5-9B-Chat", - "name": "Configurable-Yi-1.5-9B-Chat", - "developer": "vicgalle", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4323, - "hfopenllm_v2/BBH": 0.5452, - "hfopenllm_v2/MATH Level 5": 0.2047, - "hfopenllm_v2/GPQA": 0.3431, - "hfopenllm_v2/MUSR": 0.4271, - "hfopenllm_v2/MMLU-PRO": 0.4015 - } - }, - { - "id": "vicgalle/ConfigurableBeagle-11B", - "name": "ConfigurableBeagle-11B", - "developer": "vicgalle", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5834, - "hfopenllm_v2/BBH": 0.5287, - "hfopenllm_v2/MATH Level 5": 0.0431, - "hfopenllm_v2/GPQA": 0.302, - "hfopenllm_v2/MUSR": 0.3953, - "hfopenllm_v2/MMLU-PRO": 0.3374 - } - }, - { - "id": "vicgalle/ConfigurableHermes-7B", - "name": "ConfigurableHermes-7B", - "developer": "vicgalle", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5411, - "hfopenllm_v2/BBH": 0.4573, - "hfopenllm_v2/MATH Level 5": 0.0476, - "hfopenllm_v2/GPQA": 0.2768, - "hfopenllm_v2/MUSR": 0.4057, - "hfopenllm_v2/MMLU-PRO": 0.3025 - } - }, - { - "id": "vicgalle/ConfigurableSOLAR-10.7B", - "name": "ConfigurableSOLAR-10.7B", - "developer": "vicgalle", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.51, - "hfopenllm_v2/BBH": 0.4867, - "hfopenllm_v2/MATH Level 5": 0.0665, - "hfopenllm_v2/GPQA": 0.2987, - "hfopenllm_v2/MUSR": 0.3805, - "hfopenllm_v2/MMLU-PRO": 0.3173 - } - }, - { - "id": "vicgalle/Humanish-RP-Llama-3.1-8B", - "name": "Humanish-RP-Llama-3.1-8B", - "developer": "vicgalle", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6669, - "hfopenllm_v2/BBH": 0.51, - "hfopenllm_v2/MATH Level 5": 0.1518, - "hfopenllm_v2/GPQA": 0.2869, - "hfopenllm_v2/MUSR": 0.3952, - "hfopenllm_v2/MMLU-PRO": 0.3477 - } - }, - { - "id": "vicgalle/Merge-Mistral-Prometheus-7B", - "name": "Merge-Mistral-Prometheus-7B", - "developer": "vicgalle", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4848, - "hfopenllm_v2/BBH": 0.4201, - "hfopenllm_v2/MATH Level 5": 0.0181, - "hfopenllm_v2/GPQA": 0.2634, - "hfopenllm_v2/MUSR": 0.41, - "hfopenllm_v2/MMLU-PRO": 0.2717 - } - }, - { - "id": "vicgalle/Merge-Mixtral-Prometheus-8x7B", - "name": "Merge-Mixtral-Prometheus-8x7B", - "developer": "vicgalle", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5744, - "hfopenllm_v2/BBH": 0.5351, - "hfopenllm_v2/MATH Level 5": 0.0929, - "hfopenllm_v2/GPQA": 0.3087, - "hfopenllm_v2/MUSR": 0.4098, - "hfopenllm_v2/MMLU-PRO": 0.3684 - } - }, - { - "id": "vicgalle/Roleplay-Llama-3-8B", - "name": "Roleplay-Llama-3-8B", - "developer": "vicgalle", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.732, - "hfopenllm_v2/BBH": 0.5012, - "hfopenllm_v2/MATH Level 5": 0.0914, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.3529, - "hfopenllm_v2/MMLU-PRO": 0.3708 - } - }, - { - "id": "viettelsecurity-ai/security-llama3.2-3b", - "name": "security-llama3.2-3b", - "developer": "viettelsecurity-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5909, - "hfopenllm_v2/BBH": 0.4401, - "hfopenllm_v2/MATH Level 5": 0.1261, - "hfopenllm_v2/GPQA": 0.2743, - "hfopenllm_v2/MUSR": 0.3379, - "hfopenllm_v2/MMLU-PRO": 0.2837 - } - }, - { - "id": "vihangd/smart-dan-sft-v0.1", - "name": "smart-dan-sft-v0.1", - "developer": "vihangd", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1576, - "hfopenllm_v2/BBH": 0.3062, - "hfopenllm_v2/MATH Level 5": 0.0098, - "hfopenllm_v2/GPQA": 0.255, - "hfopenllm_v2/MUSR": 0.3502, - "hfopenllm_v2/MMLU-PRO": 0.1142 - } - }, - { - "id": "Vikhrmodels/Vikhr-Llama3.1-8B-Instruct-R-21-09-24", - "name": "Vikhr-Llama3.1-8B-Instruct-R-21-09-24", - "developer": "Vikhrmodels", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6431, - "hfopenllm_v2/BBH": 0.5272, - "hfopenllm_v2/MATH Level 5": 0.2175, - "hfopenllm_v2/GPQA": 0.245, - "hfopenllm_v2/MUSR": 0.3754, - "hfopenllm_v2/MMLU-PRO": 0.3547 - } - }, - { - "id": "Vikhrmodels/Vikhr-Nemo-12B-Instruct-R-21-09-24", - "name": "Vikhr-Nemo-12B-Instruct-R-21-09-24", - "developer": "Vikhrmodels", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5999, - "hfopenllm_v2/BBH": 0.5212, - "hfopenllm_v2/MATH Level 5": 0.1715, - "hfopenllm_v2/GPQA": 0.2911, - "hfopenllm_v2/MUSR": 0.4073, - "hfopenllm_v2/MMLU-PRO": 0.3398 - } - }, - { - "id": "VIRNECT/llama-3-Korean-8B", - "name": "llama-3-Korean-8B", - "developer": "VIRNECT", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5058, - "hfopenllm_v2/BBH": 0.4908, - "hfopenllm_v2/MATH Level 5": 0.0929, - "hfopenllm_v2/GPQA": 0.271, - "hfopenllm_v2/MUSR": 0.3662, - "hfopenllm_v2/MMLU-PRO": 0.3539 - } - }, - { - "id": "VIRNECT/llama-3-Korean-8B-r-v-0.1", - "name": "llama-3-Korean-8B-r-v-0.1", - "developer": "VIRNECT", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4916, - "hfopenllm_v2/BBH": 0.4806, - "hfopenllm_v2/MATH Level 5": 0.0861, - "hfopenllm_v2/GPQA": 0.2424, - "hfopenllm_v2/MUSR": 0.3675, - "hfopenllm_v2/MMLU-PRO": 0.326 - } - }, - { - "id": "voidful/smol-360m-ft", - "name": "smol-360m-ft", - "developer": "voidful", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2013, - "hfopenllm_v2/BBH": 0.3012, - "hfopenllm_v2/MATH Level 5": 0.0083, - "hfopenllm_v2/GPQA": 0.2458, - "hfopenllm_v2/MUSR": 0.3714, - "hfopenllm_v2/MMLU-PRO": 0.1087 - } - }, - { - "id": "vonjack/MobileLLM-125M-HF", - "name": "MobileLLM-125M-HF", - "developer": "vonjack", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2107, - "hfopenllm_v2/BBH": 0.3027, - "hfopenllm_v2/MATH Level 5": 0.0091, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3782, - "hfopenllm_v2/MMLU-PRO": 0.1164 - } - }, - { - "id": "vonjack/Phi-3-mini-4k-instruct-LLaMAfied", - "name": "Phi-3-mini-4k-instruct-LLaMAfied", - "developer": "vonjack", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5787, - "hfopenllm_v2/BBH": 0.5741, - "hfopenllm_v2/MATH Level 5": 0.1382, - "hfopenllm_v2/GPQA": 0.3305, - "hfopenllm_v2/MUSR": 0.3924, - "hfopenllm_v2/MMLU-PRO": 0.3885 - } - }, - { - "id": "vonjack/Phi-3.5-mini-instruct-hermes-fc-json", - "name": "Phi-3.5-mini-instruct-hermes-fc-json", - "developer": "vonjack", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1416, - "hfopenllm_v2/BBH": 0.2975, - "hfopenllm_v2/MATH Level 5": 0.0076, - "hfopenllm_v2/GPQA": 0.2542, - "hfopenllm_v2/MUSR": 0.4041, - "hfopenllm_v2/MMLU-PRO": 0.1139 - } - }, - { - "id": "vonjack/Qwen2.5-Coder-0.5B-Merged", - "name": "Qwen2.5-Coder-0.5B-Merged", - "developer": "vonjack", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.31, - "hfopenllm_v2/BBH": 0.3076, - "hfopenllm_v2/MATH Level 5": 0.0378, - "hfopenllm_v2/GPQA": 0.2534, - "hfopenllm_v2/MUSR": 0.3303, - "hfopenllm_v2/MMLU-PRO": 0.1202 - } - }, - { - "id": "vonjack/SmolLM2-1.7B-Merged", - "name": "SmolLM2-1.7B-Merged", - "developer": "vonjack", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3698, - "hfopenllm_v2/BBH": 0.3587, - "hfopenllm_v2/MATH Level 5": 0.0627, - "hfopenllm_v2/GPQA": 0.2794, - "hfopenllm_v2/MUSR": 0.3408, - "hfopenllm_v2/MMLU-PRO": 0.2048 - } - }, - { - "id": "vonjack/SmolLM2-135M-Merged", - "name": "SmolLM2-135M-Merged", - "developer": "vonjack", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2483, - "hfopenllm_v2/BBH": 0.31, - "hfopenllm_v2/MATH Level 5": 0.0113, - "hfopenllm_v2/GPQA": 0.2383, - "hfopenllm_v2/MUSR": 0.3662, - "hfopenllm_v2/MMLU-PRO": 0.1112 - } - }, - { - "id": "vonjack/SmolLM2-360M-Merged", - "name": "SmolLM2-360M-Merged", - "developer": "vonjack", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3206, - "hfopenllm_v2/BBH": 0.3155, - "hfopenllm_v2/MATH Level 5": 0.0174, - "hfopenllm_v2/GPQA": 0.2559, - "hfopenllm_v2/MUSR": 0.3527, - "hfopenllm_v2/MMLU-PRO": 0.1098 - } - }, - { - "id": "w4r10ck/SOLAR-10.7B-Instruct-v1.0-uncensored", - "name": "SOLAR-10.7B-Instruct-v1.0-uncensored", - "developer": "w4r10ck", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3884, - "hfopenllm_v2/BBH": 0.5302, - "hfopenllm_v2/MATH Level 5": 0.0657, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.4639, - "hfopenllm_v2/MMLU-PRO": 0.3344 - } - }, - { - "id": "wanlige/li-14b-v0.4", - "name": "li-14b-v0.4", - "developer": "wanlige", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8133, - "hfopenllm_v2/BBH": 0.6544, - "hfopenllm_v2/MATH Level 5": 0.5574, - "hfopenllm_v2/GPQA": 0.3389, - "hfopenllm_v2/MUSR": 0.446, - "hfopenllm_v2/MMLU-PRO": 0.5167 - } - }, - { - "id": "wanlige/li-14b-v0.4-slerp", - "name": "li-14b-v0.4-slerp", - "developer": "wanlige", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4606, - "hfopenllm_v2/BBH": 0.6587, - "hfopenllm_v2/MATH Level 5": 0.4192, - "hfopenllm_v2/GPQA": 0.4002, - "hfopenllm_v2/MUSR": 0.4768, - "hfopenllm_v2/MMLU-PRO": 0.5372 - } - }, - { - "id": "wanlige/li-14b-v0.4-slerp0.1", - "name": "li-14b-v0.4-slerp0.1", - "developer": "wanlige", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7923, - "hfopenllm_v2/BBH": 0.6572, - "hfopenllm_v2/MATH Level 5": 0.5332, - "hfopenllm_v2/GPQA": 0.3591, - "hfopenllm_v2/MUSR": 0.4207, - "hfopenllm_v2/MMLU-PRO": 0.5294 - } - }, - { - "id": "wannaphong/KhanomTanLLM-Instruct", - "name": "KhanomTanLLM-Instruct", - "developer": "wannaphong", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1621, - "hfopenllm_v2/BBH": 0.3093, - "hfopenllm_v2/MATH Level 5": 0.0136, - "hfopenllm_v2/GPQA": 0.2634, - "hfopenllm_v2/MUSR": 0.3701, - "hfopenllm_v2/MMLU-PRO": 0.1119 - } - }, - { - "id": "waqasali1707/Beast-Soul-new", - "name": "Beast-Soul-new", - "developer": "waqasali1707", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.503, - "hfopenllm_v2/BBH": 0.5225, - "hfopenllm_v2/MATH Level 5": 0.0702, - "hfopenllm_v2/GPQA": 0.2827, - "hfopenllm_v2/MUSR": 0.4486, - "hfopenllm_v2/MMLU-PRO": 0.3108 - } - }, - { - "id": "wave-on-discord/qwent-7b", - "name": "qwent-7b", - "developer": "wave-on-discord", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2015, - "hfopenllm_v2/BBH": 0.4228, - "hfopenllm_v2/MATH Level 5": 0.0038, - "hfopenllm_v2/GPQA": 0.2651, - "hfopenllm_v2/MUSR": 0.3817, - "hfopenllm_v2/MMLU-PRO": 0.1603 - } - }, - { - "id": "weathermanj/Menda-3B-500", - "name": "Menda-3B-500", - "developer": "weathermanj", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6353, - "hfopenllm_v2/BBH": 0.4766, - "hfopenllm_v2/MATH Level 5": 0.3724, - "hfopenllm_v2/GPQA": 0.2878, - "hfopenllm_v2/MUSR": 0.3968, - "hfopenllm_v2/MMLU-PRO": 0.3475 - } - }, - { - "id": "weathermanj/Menda-3b-750", - "name": "Menda-3b-750", - "developer": "weathermanj", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6335, - "hfopenllm_v2/BBH": 0.4737, - "hfopenllm_v2/MATH Level 5": 0.3716, - "hfopenllm_v2/GPQA": 0.2878, - "hfopenllm_v2/MUSR": 0.3942, - "hfopenllm_v2/MMLU-PRO": 0.3506 - } - }, - { - "id": "weathermanj/Menda-3b-Optim-100", - "name": "Menda-3b-Optim-100", - "developer": "weathermanj", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6398, - "hfopenllm_v2/BBH": 0.4735, - "hfopenllm_v2/MATH Level 5": 0.3716, - "hfopenllm_v2/GPQA": 0.2894, - "hfopenllm_v2/MUSR": 0.3993, - "hfopenllm_v2/MMLU-PRO": 0.3461 - } - }, - { - "id": "weathermanj/Menda-3b-Optim-200", - "name": "Menda-3b-Optim-200", - "developer": "weathermanj", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6375, - "hfopenllm_v2/BBH": 0.4746, - "hfopenllm_v2/MATH Level 5": 0.3731, - "hfopenllm_v2/GPQA": 0.2827, - "hfopenllm_v2/MUSR": 0.4033, - "hfopenllm_v2/MMLU-PRO": 0.3484 - } - }, - { - "id": "wenbopan/Faro-Yi-9B-DPO", - "name": "wenbopan/Faro-Yi-9B-DPO", - "developer": "wenbopan", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6461, - "reward-bench/Chat": 0.9218, - "reward-bench/Chat Hard": 0.5307, - "reward-bench/Safety": 0.5514, - "reward-bench/Reasoning": 0.5839, - "reward-bench/Prior Sets (0.5 weight)": 0.6395 - } - }, - { - "id": "weqweasdas/hh_rlhf_rm_open_llama_3b", - "name": "weqweasdas/hh_rlhf_rm_open_llama_3b", - "developer": "weqweasdas", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.2498, - "reward-bench/Chat": 0.8184, - "reward-bench/Chat Hard": 0.3728, - "reward-bench/Safety": 0.24, - "reward-bench/Reasoning": 0.3281, - "reward-bench/Prior Sets (0.5 weight)": 0.6564, - "reward-bench/Factuality": 0.3642, - "reward-bench/Precise IF": 0.275, - "reward-bench/Math": 0.3497, - "reward-bench/Focus": 0.2384, - "reward-bench/Ties": 0.0315 - } - }, - { - "id": "weqweasdas/RM-Gemma-2B", - "name": "weqweasdas/RM-Gemma-2B", - "developer": "weqweasdas", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6549, - "reward-bench/Factuality": 0.3705, - "reward-bench/Precise IF": 0.2812, - "reward-bench/Math": 0.4317, - "reward-bench/Safety": 0.4986, - "reward-bench/Focus": 0.2343, - "reward-bench/Ties": 0.1851, - "reward-bench/Chat": 0.9441, - "reward-bench/Chat Hard": 0.4079, - "reward-bench/Reasoning": 0.7637, - "reward-bench/Prior Sets (0.5 weight)": 0.6652 - } - }, - { - "id": "weqweasdas/RM-Gemma-7B", - "name": "weqweasdas/RM-Gemma-7B", - "developer": "weqweasdas", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6967, - "reward-bench/Factuality": 0.4926, - "reward-bench/Precise IF": 0.3937, - "reward-bench/Math": 0.6066, - "reward-bench/Safety": 0.5784, - "reward-bench/Focus": 0.497, - "reward-bench/Ties": 0.4232, - "reward-bench/Chat": 0.9693, - "reward-bench/Chat Hard": 0.4978, - "reward-bench/Reasoning": 0.7362, - "reward-bench/Prior Sets (0.5 weight)": 0.7069 - } - }, - { - "id": "weqweasdas/RM-Gemma-7B-4096", - "name": "weqweasdas/RM-Gemma-7B-4096", - "developer": "weqweasdas", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.6922, - "reward-bench/Chat": 0.9497, - "reward-bench/Chat Hard": 0.5022, - "reward-bench/Safety": 0.5608, - "reward-bench/Reasoning": 0.7511, - "reward-bench/Prior Sets (0.5 weight)": 0.7024 - } - }, - { - "id": "weqweasdas/RM-Mistral-7B", - "name": "weqweasdas/RM-Mistral-7B", - "developer": "weqweasdas", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.7982, - "reward-bench/Factuality": 0.5937, - "reward-bench/Precise IF": 0.3438, - "reward-bench/Math": 0.5956, - "reward-bench/Safety": 0.8703, - "reward-bench/Focus": 0.7293, - "reward-bench/Ties": 0.6226, - "reward-bench/Chat": 0.9665, - "reward-bench/Chat Hard": 0.6053, - "reward-bench/Reasoning": 0.7736, - "reward-bench/Prior Sets (0.5 weight)": 0.753 - } - }, - { - "id": "Weyaxi/Bagel-Hermes-2x34B", - "name": "Bagel-Hermes-2x34B", - "developer": "Weyaxi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5432, - "hfopenllm_v2/BBH": 0.4917, - "hfopenllm_v2/MATH Level 5": 0.0604, - "hfopenllm_v2/GPQA": 0.328, - "hfopenllm_v2/MUSR": 0.4517, - "hfopenllm_v2/MMLU-PRO": 0.4589 - } - }, - { - "id": "Weyaxi/Bagel-Hermes-34B-Slerp", - "name": "Bagel-Hermes-34B-Slerp", - "developer": "Weyaxi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4603, - "hfopenllm_v2/BBH": 0.5922, - "hfopenllm_v2/MATH Level 5": 0.0604, - "hfopenllm_v2/GPQA": 0.3347, - "hfopenllm_v2/MUSR": 0.4622, - "hfopenllm_v2/MMLU-PRO": 0.4703 - } - }, - { - "id": "Weyaxi/Einstein-v4-7B", - "name": "Einstein-v4-7B", - "developer": "Weyaxi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4708, - "hfopenllm_v2/BBH": 0.3849, - "hfopenllm_v2/MATH Level 5": 0.0189, - "hfopenllm_v2/GPQA": 0.2819, - "hfopenllm_v2/MUSR": 0.4682, - "hfopenllm_v2/MMLU-PRO": 0.2259 - } - }, - { - "id": "Weyaxi/Einstein-v6.1-developed-by-Weyaxi-Llama3-8B", - "name": "Einstein-v6.1-developed-by-Weyaxi-Llama3-8B", - "developer": "Weyaxi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3927, - "hfopenllm_v2/BBH": 0.5044, - "hfopenllm_v2/MATH Level 5": 0.0718, - "hfopenllm_v2/GPQA": 0.2735, - "hfopenllm_v2/MUSR": 0.4332, - "hfopenllm_v2/MMLU-PRO": 0.3093 - } - }, - { - "id": "Weyaxi/Einstein-v6.1-Llama3-8B", - "name": "Einstein-v6.1-Llama3-8B", - "developer": "Weyaxi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4568, - "hfopenllm_v2/BBH": 0.5008, - "hfopenllm_v2/MATH Level 5": 0.068, - "hfopenllm_v2/GPQA": 0.2819, - "hfopenllm_v2/MUSR": 0.4213, - "hfopenllm_v2/MMLU-PRO": 0.3131 - } - }, - { - "id": "Weyaxi/Einstein-v7-Qwen2-7B", - "name": "Einstein-v7-Qwen2-7B", - "developer": "Weyaxi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.41, - "hfopenllm_v2/BBH": 0.5161, - "hfopenllm_v2/MATH Level 5": 0.1994, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.44, - "hfopenllm_v2/MMLU-PRO": 0.4096 - } - }, - { - "id": "Weyaxi/Einstein-v8-Llama3.2-1B", - "name": "Einstein-v8-Llama3.2-1B", - "developer": "Weyaxi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1862, - "hfopenllm_v2/BBH": 0.3018, - "hfopenllm_v2/MATH Level 5": 0.0008, - "hfopenllm_v2/GPQA": 0.2584, - "hfopenllm_v2/MUSR": 0.3618, - "hfopenllm_v2/MMLU-PRO": 0.1161 - } - }, - { - "id": "Weyaxi/SauerkrautLM-UNA-SOLAR-Instruct", - "name": "SauerkrautLM-UNA-SOLAR-Instruct", - "developer": "Weyaxi", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4573, - "hfopenllm_v2/BBH": 0.5166, - "hfopenllm_v2/MATH Level 5": 0.0461, - "hfopenllm_v2/GPQA": 0.3112, - "hfopenllm_v2/MUSR": 0.3979, - "hfopenllm_v2/MMLU-PRO": 0.3153 - } - }, - { - "id": "win10/ArliAI-RPMax-v1.3-merge-13.3B", - "name": "ArliAI-RPMax-v1.3-merge-13.3B", - "developer": "win10", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3038, - "hfopenllm_v2/BBH": 0.4581, - "hfopenllm_v2/MATH Level 5": 0.0393, - "hfopenllm_v2/GPQA": 0.2743, - "hfopenllm_v2/MUSR": 0.4325, - "hfopenllm_v2/MMLU-PRO": 0.32 - } - }, - { - "id": "win10/Breeze-13B-32k-Instruct-v1_0", - "name": "Breeze-13B-32k-Instruct-v1_0", - "developer": "win10", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3584, - "hfopenllm_v2/BBH": 0.4611, - "hfopenllm_v2/MATH Level 5": 0.0128, - "hfopenllm_v2/GPQA": 0.2643, - "hfopenllm_v2/MUSR": 0.4202, - "hfopenllm_v2/MMLU-PRO": 0.2568 - } - }, - { - "id": "win10/EVA-Norns-Qwen2.5-v0.1", - "name": "EVA-Norns-Qwen2.5-v0.1", - "developer": "win10", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.622, - "hfopenllm_v2/BBH": 0.5072, - "hfopenllm_v2/MATH Level 5": 0.2613, - "hfopenllm_v2/GPQA": 0.2852, - "hfopenllm_v2/MUSR": 0.4045, - "hfopenllm_v2/MMLU-PRO": 0.3425 - } - }, - { - "id": "win10/Llama-3.2-3B-Instruct-24-9-29", - "name": "Llama-3.2-3B-Instruct-24-9-29", - "developer": "win10", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7332, - "hfopenllm_v2/BBH": 0.4614, - "hfopenllm_v2/MATH Level 5": 0.1707, - "hfopenllm_v2/GPQA": 0.2743, - "hfopenllm_v2/MUSR": 0.3555, - "hfopenllm_v2/MMLU-PRO": 0.3228 - } - }, - { - "id": "win10/llama3-13.45b-Instruct", - "name": "llama3-13.45b-Instruct", - "developer": "win10", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4144, - "hfopenllm_v2/BBH": 0.4865, - "hfopenllm_v2/MATH Level 5": 0.0242, - "hfopenllm_v2/GPQA": 0.2584, - "hfopenllm_v2/MUSR": 0.3848, - "hfopenllm_v2/MMLU-PRO": 0.3345 - } - }, - { - "id": "win10/miscii-14b-1M-0128", - "name": "miscii-14b-1M-0128", - "developer": "win10", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4181, - "hfopenllm_v2/BBH": 0.5742, - "hfopenllm_v2/MATH Level 5": 0.4773, - "hfopenllm_v2/GPQA": 0.3826, - "hfopenllm_v2/MUSR": 0.5431, - "hfopenllm_v2/MMLU-PRO": 0.4491 - } - }, - { - "id": "win10/Norns-Qwen2.5-12B", - "name": "Norns-Qwen2.5-12B", - "developer": "win10", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4897, - "hfopenllm_v2/BBH": 0.4619, - "hfopenllm_v2/MATH Level 5": 0.0838, - "hfopenllm_v2/GPQA": 0.2836, - "hfopenllm_v2/MUSR": 0.3555, - "hfopenllm_v2/MMLU-PRO": 0.266 - } - }, - { - "id": "win10/Norns-Qwen2.5-7B", - "name": "Norns-Qwen2.5-7B", - "developer": "win10", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6122, - "hfopenllm_v2/BBH": 0.5073, - "hfopenllm_v2/MATH Level 5": 0.2628, - "hfopenllm_v2/GPQA": 0.2844, - "hfopenllm_v2/MUSR": 0.4085, - "hfopenllm_v2/MMLU-PRO": 0.3413 - } - }, - { - "id": "win10/Qwen2.5-2B-Instruct", - "name": "Qwen2.5-2B-Instruct", - "developer": "win10", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2273, - "hfopenllm_v2/BBH": 0.3706, - "hfopenllm_v2/MATH Level 5": 0.0227, - "hfopenllm_v2/GPQA": 0.2676, - "hfopenllm_v2/MUSR": 0.4378, - "hfopenllm_v2/MMLU-PRO": 0.1934 - } - }, - { - "id": "winglian/llama-3-8b-256k-PoSE", - "name": "llama-3-8b-256k-PoSE", - "developer": "winglian", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2909, - "hfopenllm_v2/BBH": 0.3157, - "hfopenllm_v2/MATH Level 5": 0.0196, - "hfopenllm_v2/GPQA": 0.2576, - "hfopenllm_v2/MUSR": 0.3316, - "hfopenllm_v2/MMLU-PRO": 0.1116 - } - }, - { - "id": "winglian/Llama-3-8b-64k-PoSE", - "name": "Llama-3-8b-64k-PoSE", - "developer": "winglian", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2857, - "hfopenllm_v2/BBH": 0.3702, - "hfopenllm_v2/MATH Level 5": 0.0415, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.3396, - "hfopenllm_v2/MMLU-PRO": 0.2467 - } - }, - { - "id": "WizardLMTeam/WizardLM-13B-V1.0", - "name": "WizardLM-13B-V1.0", - "developer": "WizardLMTeam", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.185, - "hfopenllm_v2/BBH": 0.2913, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3497, - "hfopenllm_v2/MMLU-PRO": 0.1166 - } - }, - { - "id": "WizardLMTeam/WizardLM-13B-V1.2", - "name": "WizardLM-13B-V1.2", - "developer": "WizardLMTeam", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3392, - "hfopenllm_v2/BBH": 0.4462, - "hfopenllm_v2/MATH Level 5": 0.0189, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.4378, - "hfopenllm_v2/MMLU-PRO": 0.2519 - } - }, - { - "id": "WizardLMTeam/WizardLM-70B-V1.0", - "name": "WizardLM-70B-V1.0", - "developer": "WizardLMTeam", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4951, - "hfopenllm_v2/BBH": 0.559, - "hfopenllm_v2/MATH Level 5": 0.0393, - "hfopenllm_v2/GPQA": 0.2659, - "hfopenllm_v2/MUSR": 0.4391, - "hfopenllm_v2/MMLU-PRO": 0.3447 - } - }, - { - "id": "Wladastic/Mini-Think-Base-1B", - "name": "Mini-Think-Base-1B", - "developer": "Wladastic", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5588, - "hfopenllm_v2/BBH": 0.3574, - "hfopenllm_v2/MATH Level 5": 0.0733, - "hfopenllm_v2/GPQA": 0.2634, - "hfopenllm_v2/MUSR": 0.3275, - "hfopenllm_v2/MMLU-PRO": 0.1772 - } - }, - { - "id": "writer/InstructPalmyra-30B", - "name": "InstructPalmyra 30B", - "developer": "writer", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.568, - "helm_classic/MMLU": 0.403, - "helm_classic/BoolQ": 0.751, - "helm_classic/NarrativeQA": 0.496, - "helm_classic/NaturalQuestions (open-book)": 0.682, - "helm_classic/QuAC": 0.433, - "helm_classic/HellaSwag": -1.0, - "helm_classic/OpenbookQA": -1.0, - "helm_classic/TruthfulQA": 0.185, - "helm_classic/MS MARCO (TREC)": -1.0, - "helm_classic/CNN/DailyMail": 0.152, - "helm_classic/XSUM": 0.104, - "helm_classic/IMDB": 0.94, - "helm_classic/CivilComments": 0.555, - "helm_classic/RAFT": 0.652 - } - }, - { - "id": "writer/palmyra-fin", - "name": "Palmyra Fin", - "developer": "writer", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.577, - "helm_capabilities/MMLU-Pro": 0.591, - "helm_capabilities/GPQA": 0.422, - "helm_capabilities/IFEval": 0.793, - "helm_capabilities/WildBench": 0.783, - "helm_capabilities/Omni-MATH": 0.295 - } - }, - { - "id": "writer/palmyra-med", - "name": "Palmyra Med", - "developer": "writer", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.476, - "helm_capabilities/MMLU-Pro": 0.411, - "helm_capabilities/GPQA": 0.368, - "helm_capabilities/IFEval": 0.767, - "helm_capabilities/WildBench": 0.676, - "helm_capabilities/Omni-MATH": 0.156 - } - }, - { - "id": "writer/palmyra-x-004", - "name": "Palmyra-X-004", - "developer": "writer", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.609, - "helm_capabilities/MMLU-Pro": 0.657, - "helm_capabilities/GPQA": 0.395, - "helm_capabilities/IFEval": 0.872, - "helm_capabilities/WildBench": 0.802, - "helm_capabilities/Omni-MATH": 0.32, - "helm_lite/Mean win rate": 0.808, - "helm_lite/NarrativeQA": 0.773, - "helm_lite/NaturalQuestions (closed-book)": 0.457, - "helm_lite/OpenbookQA": 0.926, - "helm_lite/MMLU": 0.739, - "helm_lite/MATH": 0.767, - "helm_lite/GSM8K": 0.905, - "helm_lite/LegalBench": 0.73, - "helm_lite/MedQA": 0.775, - "helm_lite/WMT 2014": 0.203, - "helm_mmlu/MMLU All Subjects": 0.813, - "helm_mmlu/Abstract Algebra": 0.75, - "helm_mmlu/Anatomy": 0.822, - "helm_mmlu/College Physics": 0.647, - "helm_mmlu/Computer Security": 0.82, - "helm_mmlu/Econometrics": 0.684, - "helm_mmlu/Global Facts": 0.62, - "helm_mmlu/Jurisprudence": 0.843, - "helm_mmlu/Philosophy": 0.83, - "helm_mmlu/Professional Psychology": 0.845, - "helm_mmlu/Us Foreign Policy": 0.92, - "helm_mmlu/Astronomy": 0.928, - "helm_mmlu/Business Ethics": 0.76, - "helm_mmlu/Clinical Knowledge": 0.879, - "helm_mmlu/Conceptual Physics": 0.885, - "helm_mmlu/Electrical Engineering": 0.793, - "helm_mmlu/Elementary Mathematics": 0.841, - "helm_mmlu/Formal Logic": 0.579, - "helm_mmlu/High School World History": 0.911, - "helm_mmlu/Human Sexuality": 0.924, - "helm_mmlu/International Law": 0.901, - "helm_mmlu/Logical Fallacies": 0.877, - "helm_mmlu/Machine Learning": 0.679, - "helm_mmlu/Management": 0.903, - "helm_mmlu/Marketing": 0.932, - "helm_mmlu/Medical Genetics": 0.87, - "helm_mmlu/Miscellaneous": 0.934, - "helm_mmlu/Moral Scenarios": 0.825, - "helm_mmlu/Nutrition": 0.869, - "helm_mmlu/Prehistory": 0.917, - "helm_mmlu/Public Relations": 0.791, - "helm_mmlu/Security Studies": 0.849, - "helm_mmlu/Sociology": 0.915, - "helm_mmlu/Virology": 0.584, - "helm_mmlu/World Religions": 0.842, - "helm_mmlu/Mean win rate": 0.629 - } - }, - { - "id": "writer/palmyra-x-004-fc", - "name": "palmyra-x-004 (FC)", - "developer": "writer", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 76.0, - "bfcl/bfcl.overall.overall_accuracy": 27.87, - "bfcl/bfcl.overall.total_cost_usd": 178.15, - "bfcl/bfcl.overall.latency_mean_s": 3.71, - "bfcl/bfcl.overall.latency_std_s": 7.62, - "bfcl/bfcl.overall.latency_p95_s": 8.04, - "bfcl/bfcl.non_live.ast_accuracy": 87.46, - "bfcl/bfcl.non_live.simple_ast_accuracy": 71.33, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 96.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 93.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 89.5, - "bfcl/bfcl.live.live_accuracy": 77.87, - "bfcl/bfcl.live.live_simple_ast_accuracy": 79.46, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 77.97, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 56.25, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 70.83, - "bfcl/bfcl.multi_turn.accuracy": 0.38, - "bfcl/bfcl.multi_turn.base_accuracy": 0.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 0.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 0.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 0.5, - "bfcl/bfcl.web_search.accuracy": 2.5, - "bfcl/bfcl.web_search.base_accuracy": 4.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 1.0, - "bfcl/bfcl.memory.accuracy": 13.12, - "bfcl/bfcl.memory.kv_accuracy": 6.45, - "bfcl/bfcl.memory.vector_accuracy": 14.19, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 18.71, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 81.25, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 80.99 - } - }, - { - "id": "writer/palmyra-x-v2", - "name": "Palmyra X V2 33B", - "developer": "writer", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.589, - "helm_lite/NarrativeQA": 0.752, - "helm_lite/NaturalQuestions (closed-book)": 0.428, - "helm_lite/OpenbookQA": 0.878, - "helm_lite/MMLU": 0.621, - "helm_lite/MATH": 0.58, - "helm_lite/GSM8K": 0.735, - "helm_lite/LegalBench": 0.644, - "helm_lite/MedQA": 0.598, - "helm_lite/WMT 2014": 0.239 - } - }, - { - "id": "writer/palmyra-x-v3", - "name": "Palmyra X V3 72B", - "developer": "writer", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_lite/Mean win rate": 0.679, - "helm_lite/NarrativeQA": 0.706, - "helm_lite/NaturalQuestions (closed-book)": 0.407, - "helm_lite/OpenbookQA": 0.938, - "helm_lite/MMLU": 0.702, - "helm_lite/MATH": 0.723, - "helm_lite/GSM8K": 0.831, - "helm_lite/LegalBench": 0.709, - "helm_lite/MedQA": 0.684, - "helm_lite/WMT 2014": 0.262, - "helm_mmlu/MMLU All Subjects": 0.786, - "helm_mmlu/Abstract Algebra": 0.53, - "helm_mmlu/Anatomy": 0.733, - "helm_mmlu/College Physics": 0.549, - "helm_mmlu/Computer Security": 0.78, - "helm_mmlu/Econometrics": 0.649, - "helm_mmlu/Global Facts": 0.53, - "helm_mmlu/Jurisprudence": 0.88, - "helm_mmlu/Philosophy": 0.836, - "helm_mmlu/Professional Psychology": 0.858, - "helm_mmlu/Us Foreign Policy": 0.96, - "helm_mmlu/Astronomy": 0.862, - "helm_mmlu/Business Ethics": 0.83, - "helm_mmlu/Clinical Knowledge": 0.804, - "helm_mmlu/Conceptual Physics": 0.809, - "helm_mmlu/Electrical Engineering": 0.772, - "helm_mmlu/Elementary Mathematics": 0.661, - "helm_mmlu/Formal Logic": 0.659, - "helm_mmlu/High School World History": 0.911, - "helm_mmlu/Human Sexuality": 0.924, - "helm_mmlu/International Law": 0.909, - "helm_mmlu/Logical Fallacies": 0.877, - "helm_mmlu/Machine Learning": 0.625, - "helm_mmlu/Management": 0.903, - "helm_mmlu/Marketing": 0.94, - "helm_mmlu/Medical Genetics": 0.83, - "helm_mmlu/Miscellaneous": 0.894, - "helm_mmlu/Moral Scenarios": 0.562, - "helm_mmlu/Nutrition": 0.856, - "helm_mmlu/Prehistory": 0.87, - "helm_mmlu/Public Relations": 0.773, - "helm_mmlu/Security Studies": 0.833, - "helm_mmlu/Sociology": 0.91, - "helm_mmlu/Virology": 0.572, - "helm_mmlu/World Religions": 0.877, - "helm_mmlu/Mean win rate": 0.325 - } - }, - { - "id": "writer/palmyra-x5", - "name": "Palmyra X5", - "developer": "writer", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.696, - "helm_capabilities/MMLU-Pro": 0.804, - "helm_capabilities/GPQA": 0.661, - "helm_capabilities/IFEval": 0.823, - "helm_capabilities/WildBench": 0.78, - "helm_capabilities/Omni-MATH": 0.414 - } - }, - { - "id": "wzhouad/gemma-2-9b-it-WPO-HB", - "name": "gemma-2-9b-it-WPO-HB", - "developer": "wzhouad", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5437, - "hfopenllm_v2/BBH": 0.5629, - "hfopenllm_v2/MATH Level 5": 0.1533, - "hfopenllm_v2/GPQA": 0.3498, - "hfopenllm_v2/MUSR": 0.3675, - "hfopenllm_v2/MMLU-PRO": 0.336 - } - }, - { - "id": "x0000001/Deepseek-Lumen-R1-Qwen2.5-14B", - "name": "Deepseek-Lumen-R1-Qwen2.5-14B", - "developer": "x0000001", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4436, - "hfopenllm_v2/BBH": 0.4569, - "hfopenllm_v2/MATH Level 5": 0.2779, - "hfopenllm_v2/GPQA": 0.2852, - "hfopenllm_v2/MUSR": 0.474, - "hfopenllm_v2/MMLU-PRO": 0.4379 - } - }, - { - "id": "xai/Grok 4", - "name": "Grok 4", - "developer": "xAI", - "evaluator_relationship": null, - "benchmark_scores": { - "apex-agents/Overall Pass@1": 0.152, - "apex-agents/Overall Pass@8": 0.329, - "apex-agents/Overall Mean Score": 0.303, - "apex-agents/Investment Banking Pass@1": 0.17, - "apex-agents/Management Consulting Pass@1": 0.12, - "apex-agents/Corporate Law Pass@1": 0.165, - "apex-agents/Corporate Lawyer Mean Score": 0.41, - "apex-v1/Overall Score": 0.635 - } - }, - { - "id": "xai/grok-3-beta", - "name": "Grok 3 Beta", - "developer": "xAI", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.727, - "helm_capabilities/MMLU-Pro": 0.788, - "helm_capabilities/GPQA": 0.65, - "helm_capabilities/IFEval": 0.884, - "helm_capabilities/WildBench": 0.849, - "helm_capabilities/Omni-MATH": 0.464 - } - }, - { - "id": "xai/grok-3-mini", - "name": "grok-3-mini", - "developer": "xAI", - "evaluator_relationship": null, - "benchmark_scores": { - "global-mmlu-lite/Global MMLU Lite": 0.673, - "global-mmlu-lite/Culturally Sensitive": 0.6717, - "global-mmlu-lite/Culturally Agnostic": 0.6743, - "global-mmlu-lite/Arabic": 0.755, - "global-mmlu-lite/English": 0.5075, - "global-mmlu-lite/Bengali": 0.7355, - "global-mmlu-lite/German": 0.6591, - "global-mmlu-lite/French": 0.485, - "global-mmlu-lite/Hindi": 0.56, - "global-mmlu-lite/Indonesian": 0.725, - "global-mmlu-lite/Italian": 0.696, - "global-mmlu-lite/Japanese": 0.6575, - "global-mmlu-lite/Korean": 0.7325, - "global-mmlu-lite/Portuguese": 0.6275, - "global-mmlu-lite/Spanish": 0.61, - "global-mmlu-lite/Swahili": 0.7625, - "global-mmlu-lite/Yoruba": 0.8296, - "global-mmlu-lite/Chinese": 0.5564, - "global-mmlu-lite/Burmese": 0.8693 - } - }, - { - "id": "xai/grok-3-mini-beta", - "name": "Grok 3 mini Beta", - "developer": "xAI", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.679, - "helm_capabilities/MMLU-Pro": 0.799, - "helm_capabilities/GPQA": 0.675, - "helm_capabilities/IFEval": 0.951, - "helm_capabilities/WildBench": 0.651, - "helm_capabilities/Omni-MATH": 0.318 - } - }, - { - "id": "xai/grok-4", - "name": "Grok 4", - "developer": "xAI", - "evaluator_relationship": null, - "benchmark_scores": { - "terminal-bench-2.0/terminal-bench-2.0": 23.1 - } - }, - { - "id": "xai/grok-4-0709", - "name": "grok-4-0709", - "developer": "xAI", - "evaluator_relationship": null, - "benchmark_scores": { - "global-mmlu-lite/Global MMLU Lite": 0.8881, - "global-mmlu-lite/Culturally Sensitive": 0.8862, - "global-mmlu-lite/Culturally Agnostic": 0.89, - "global-mmlu-lite/Arabic": 0.885, - "global-mmlu-lite/English": 0.905, - "global-mmlu-lite/Bengali": 0.8925, - "global-mmlu-lite/German": 0.8725, - "global-mmlu-lite/French": 0.875, - "global-mmlu-lite/Hindi": 0.8675, - "global-mmlu-lite/Indonesian": 0.89, - "global-mmlu-lite/Italian": 0.9025, - "global-mmlu-lite/Japanese": 0.87, - "global-mmlu-lite/Korean": 0.895, - "global-mmlu-lite/Portuguese": 0.8725, - "global-mmlu-lite/Spanish": 0.9075, - "global-mmlu-lite/Swahili": 0.91, - "global-mmlu-lite/Yoruba": 0.905, - "global-mmlu-lite/Chinese": 0.8525, - "global-mmlu-lite/Burmese": 0.9075, - "helm_capabilities/Mean score": 0.785, - "helm_capabilities/MMLU-Pro": 0.851, - "helm_capabilities/GPQA": 0.726, - "helm_capabilities/IFEval": 0.949, - "helm_capabilities/WildBench": 0.797, - "helm_capabilities/Omni-MATH": 0.603 - } - }, - { - "id": "xai/grok-4-0709-fc", - "name": "Grok-4-0709 (FC)", - "developer": "xAI", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 10.0, - "bfcl/bfcl.overall.overall_accuracy": 61.38, - "bfcl/bfcl.overall.total_cost_usd": 355.17, - "bfcl/bfcl.overall.latency_mean_s": 15.49, - "bfcl/bfcl.overall.latency_std_s": 26.22, - "bfcl/bfcl.overall.latency_p95_s": 44.28, - "bfcl/bfcl.non_live.ast_accuracy": 85.38, - "bfcl/bfcl.non_live.simple_ast_accuracy": 73.5, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 92.5, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 88.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 87.0, - "bfcl/bfcl.live.live_accuracy": 75.57, - "bfcl/bfcl.live.live_simple_ast_accuracy": 82.17, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 73.88, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 75.0, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 79.17, - "bfcl/bfcl.multi_turn.accuracy": 33.88, - "bfcl/bfcl.multi_turn.base_accuracy": 44.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 19.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 28.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 44.0, - "bfcl/bfcl.web_search.accuracy": 82.0, - "bfcl/bfcl.web_search.base_accuracy": 80.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 84.0, - "bfcl/bfcl.memory.accuracy": 55.91, - "bfcl/bfcl.memory.kv_accuracy": 57.42, - "bfcl/bfcl.memory.vector_accuracy": 58.71, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 51.61, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 87.5, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 75.4 - } - }, - { - "id": "xai/grok-4-0709-prompt", - "name": "Grok-4-0709 (Prompt)", - "developer": "xAI", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 9.0, - "bfcl/bfcl.overall.overall_accuracy": 62.97, - "bfcl/bfcl.overall.total_cost_usd": 348.19, - "bfcl/bfcl.overall.latency_mean_s": 30.38, - "bfcl/bfcl.overall.latency_std_s": 36.19, - "bfcl/bfcl.overall.latency_p95_s": 101.54, - "bfcl/bfcl.non_live.ast_accuracy": 82.75, - "bfcl/bfcl.non_live.simple_ast_accuracy": 67.0, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 93.5, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 89.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 81.5, - "bfcl/bfcl.live.live_accuracy": 72.54, - "bfcl/bfcl.live.live_simple_ast_accuracy": 81.78, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 70.18, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 81.25, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 70.83, - "bfcl/bfcl.multi_turn.accuracy": 47.0, - "bfcl/bfcl.multi_turn.base_accuracy": 55.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 46.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 36.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 50.5, - "bfcl/bfcl.web_search.accuracy": 74.0, - "bfcl/bfcl.web_search.base_accuracy": 74.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 74.0, - "bfcl/bfcl.memory.accuracy": 50.54, - "bfcl/bfcl.memory.kv_accuracy": 43.87, - "bfcl/bfcl.memory.vector_accuracy": 59.35, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 48.39, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 81.25, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 84.3, - "bfcl/bfcl.format_sensitivity.max_delta": 13.0, - "bfcl/bfcl.format_sensitivity.stddev": 2.88 - } - }, - { - "id": "xai/grok-4-1-fast-non-reasoning-fc", - "name": "Grok-4-1-fast-non-reasoning (FC)", - "developer": "xAI", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 12.0, - "bfcl/bfcl.overall.overall_accuracy": 58.29, - "bfcl/bfcl.overall.total_cost_usd": 16.27, - "bfcl/bfcl.overall.latency_mean_s": 2.29, - "bfcl/bfcl.overall.latency_std_s": 7.31, - "bfcl/bfcl.overall.latency_p95_s": 5.34, - "bfcl/bfcl.non_live.ast_accuracy": 88.13, - "bfcl/bfcl.non_live.simple_ast_accuracy": 76.0, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 93.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 93.0, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 90.5, - "bfcl/bfcl.live.live_accuracy": 77.94, - "bfcl/bfcl.live.live_simple_ast_accuracy": 82.95, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 76.92, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 75.0, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 70.83, - "bfcl/bfcl.multi_turn.accuracy": 46.75, - "bfcl/bfcl.multi_turn.base_accuracy": 58.0, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 39.5, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 37.5, - "bfcl/bfcl.multi_turn.long_context_accuracy": 52.0, - "bfcl/bfcl.web_search.accuracy": 75.0, - "bfcl/bfcl.web_search.base_accuracy": 74.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 76.0, - "bfcl/bfcl.memory.accuracy": 26.24, - "bfcl/bfcl.memory.kv_accuracy": 20.65, - "bfcl/bfcl.memory.vector_accuracy": 20.0, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 38.06, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 81.25, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 74.09 - } - }, - { - "id": "xai/grok-4-1-fast-reasoning-fc", - "name": "Grok-4-1-fast-reasoning (FC)", - "developer": "xAI", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 5.0, - "bfcl/bfcl.overall.overall_accuracy": 69.57, - "bfcl/bfcl.overall.total_cost_usd": 17.26, - "bfcl/bfcl.overall.latency_mean_s": 6.74, - "bfcl/bfcl.overall.latency_std_s": 12.78, - "bfcl/bfcl.overall.latency_p95_s": 17.57, - "bfcl/bfcl.non_live.ast_accuracy": 88.27, - "bfcl/bfcl.non_live.simple_ast_accuracy": 77.58, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 93.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 92.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 90.0, - "bfcl/bfcl.live.live_accuracy": 78.46, - "bfcl/bfcl.live.live_simple_ast_accuracy": 84.11, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 77.3, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 75.0, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 70.83, - "bfcl/bfcl.multi_turn.accuracy": 58.87, - "bfcl/bfcl.multi_turn.base_accuracy": 70.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 59.5, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 43.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 62.5, - "bfcl/bfcl.web_search.accuracy": 82.5, - "bfcl/bfcl.web_search.base_accuracy": 82.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 83.0, - "bfcl/bfcl.memory.accuracy": 53.98, - "bfcl/bfcl.memory.kv_accuracy": 41.29, - "bfcl/bfcl.memory.vector_accuracy": 57.42, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 63.23, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 81.25, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 79.43 - } - }, - { - "id": "xai/grok-code-fast-1", - "name": "Grok Code Fast 1", - "developer": "xAI", - "evaluator_relationship": null, - "benchmark_scores": { - "terminal-bench-2.0/terminal-bench-2.0": 14.2 - } - }, - { - "id": "Xclbr7/Arcanum-12b", - "name": "Arcanum-12b", - "developer": "Xclbr7", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2907, - "hfopenllm_v2/BBH": 0.5265, - "hfopenllm_v2/MATH Level 5": 0.1193, - "hfopenllm_v2/GPQA": 0.3205, - "hfopenllm_v2/MUSR": 0.417, - "hfopenllm_v2/MMLU-PRO": 0.3586 - } - }, - { - "id": "Xclbr7/caliburn-12b", - "name": "caliburn-12b", - "developer": "Xclbr7", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3576, - "hfopenllm_v2/BBH": 0.5519, - "hfopenllm_v2/MATH Level 5": 0.1125, - "hfopenllm_v2/GPQA": 0.3364, - "hfopenllm_v2/MUSR": 0.4292, - "hfopenllm_v2/MMLU-PRO": 0.3675 - } - }, - { - "id": "Xclbr7/caliburn-v2-12b", - "name": "caliburn-v2-12b", - "developer": "Xclbr7", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2967, - "hfopenllm_v2/BBH": 0.5141, - "hfopenllm_v2/MATH Level 5": 0.105, - "hfopenllm_v2/GPQA": 0.3263, - "hfopenllm_v2/MUSR": 0.437, - "hfopenllm_v2/MMLU-PRO": 0.3784 - } - }, - { - "id": "Xclbr7/Hyena-12b", - "name": "Hyena-12b", - "developer": "Xclbr7", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3404, - "hfopenllm_v2/BBH": 0.5457, - "hfopenllm_v2/MATH Level 5": 0.1133, - "hfopenllm_v2/GPQA": 0.2978, - "hfopenllm_v2/MUSR": 0.3984, - "hfopenllm_v2/MMLU-PRO": 0.3439 - } - }, - { - "id": "Xiaojian9992024/Llama3.2-1B-THREADRIPPER", - "name": "Llama3.2-1B-THREADRIPPER", - "developer": "Xiaojian9992024", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5576, - "hfopenllm_v2/BBH": 0.3544, - "hfopenllm_v2/MATH Level 5": 0.074, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.313, - "hfopenllm_v2/MMLU-PRO": 0.1763 - } - }, - { - "id": "Xiaojian9992024/Llama3.2-1B-THREADRIPPER-v0.2", - "name": "Llama3.2-1B-THREADRIPPER-v0.2", - "developer": "Xiaojian9992024", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5318, - "hfopenllm_v2/BBH": 0.3528, - "hfopenllm_v2/MATH Level 5": 0.0657, - "hfopenllm_v2/GPQA": 0.2659, - "hfopenllm_v2/MUSR": 0.3316, - "hfopenllm_v2/MMLU-PRO": 0.1745 - } - }, - { - "id": "Xiaojian9992024/Phi-4-Megatron-Empathetic", - "name": "Phi-4-Megatron-Empathetic", - "developer": "Xiaojian9992024", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.0173, - "hfopenllm_v2/BBH": 0.6673, - "hfopenllm_v2/MATH Level 5": 0.2696, - "hfopenllm_v2/GPQA": 0.3859, - "hfopenllm_v2/MUSR": 0.5071, - "hfopenllm_v2/MMLU-PRO": 0.5082 - } - }, - { - "id": "Xiaojian9992024/Phi-4-mini-UNOFFICAL", - "name": "Phi-4-mini-UNOFFICAL", - "developer": "Xiaojian9992024", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1273, - "hfopenllm_v2/BBH": 0.2944, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2408, - "hfopenllm_v2/MUSR": 0.3368, - "hfopenllm_v2/MMLU-PRO": 0.1144 - } - }, - { - "id": "Xiaojian9992024/Qwen2.5-7B-MS-Destroyer", - "name": "Qwen2.5-7B-MS-Destroyer", - "developer": "Xiaojian9992024", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7296, - "hfopenllm_v2/BBH": 0.547, - "hfopenllm_v2/MATH Level 5": 0.4592, - "hfopenllm_v2/GPQA": 0.3045, - "hfopenllm_v2/MUSR": 0.427, - "hfopenllm_v2/MMLU-PRO": 0.4412 - } - }, - { - "id": "Xiaojian9992024/Qwen2.5-Dyanka-7B-Preview", - "name": "Qwen2.5-Dyanka-7B-Preview", - "developer": "Xiaojian9992024", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.764, - "hfopenllm_v2/BBH": 0.5543, - "hfopenllm_v2/MATH Level 5": 0.4879, - "hfopenllm_v2/GPQA": 0.3171, - "hfopenllm_v2/MUSR": 0.4481, - "hfopenllm_v2/MMLU-PRO": 0.4376 - } - }, - { - "id": "Xiaojian9992024/Qwen2.5-Dyanka-7B-Preview-v0.2", - "name": "Qwen2.5-Dyanka-7B-Preview-v0.2", - "developer": "Xiaojian9992024", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6702, - "hfopenllm_v2/BBH": 0.5374, - "hfopenllm_v2/MATH Level 5": 0.4721, - "hfopenllm_v2/GPQA": 0.2936, - "hfopenllm_v2/MUSR": 0.4467, - "hfopenllm_v2/MMLU-PRO": 0.4371 - } - }, - { - "id": "Xiaojian9992024/Qwen2.5-THREADRIPPER-Medium-Censored", - "name": "Qwen2.5-THREADRIPPER-Medium-Censored", - "developer": "Xiaojian9992024", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8112, - "hfopenllm_v2/BBH": 0.6431, - "hfopenllm_v2/MATH Level 5": 0.534, - "hfopenllm_v2/GPQA": 0.3347, - "hfopenllm_v2/MUSR": 0.414, - "hfopenllm_v2/MMLU-PRO": 0.4929 - } - }, - { - "id": "Xiaojian9992024/Qwen2.5-THREADRIPPER-Small", - "name": "Qwen2.5-THREADRIPPER-Small", - "developer": "Xiaojian9992024", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7689, - "hfopenllm_v2/BBH": 0.549, - "hfopenllm_v2/MATH Level 5": 0.4736, - "hfopenllm_v2/GPQA": 0.3104, - "hfopenllm_v2/MUSR": 0.4349, - "hfopenllm_v2/MMLU-PRO": 0.4357 - } - }, - { - "id": "Xiaojian9992024/Qwen2.5-THREADRIPPER-Small-AnniversaryEdition", - "name": "Qwen2.5-THREADRIPPER-Small-AnniversaryEdition", - "developer": "Xiaojian9992024", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7404, - "hfopenllm_v2/BBH": 0.5465, - "hfopenllm_v2/MATH Level 5": 0.5076, - "hfopenllm_v2/GPQA": 0.2685, - "hfopenllm_v2/MUSR": 0.3807, - "hfopenllm_v2/MMLU-PRO": 0.4393 - } - }, - { - "id": "Xiaojian9992024/Qwen2.5-Ultra-1.5B-25.02-Exp", - "name": "Qwen2.5-Ultra-1.5B-25.02-Exp", - "developer": "Xiaojian9992024", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4073, - "hfopenllm_v2/BBH": 0.4066, - "hfopenllm_v2/MATH Level 5": 0.0831, - "hfopenllm_v2/GPQA": 0.2584, - "hfopenllm_v2/MUSR": 0.3383, - "hfopenllm_v2/MMLU-PRO": 0.2641 - } - }, - { - "id": "Xiaojian9992024/Reflection-L3.2-JametMiniMix-3B", - "name": "Reflection-L3.2-JametMiniMix-3B", - "developer": "Xiaojian9992024", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4619, - "hfopenllm_v2/BBH": 0.439, - "hfopenllm_v2/MATH Level 5": 0.1193, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.3667, - "hfopenllm_v2/MMLU-PRO": 0.2988 - } - }, - { - "id": "xinchen9/llama3-b8-ft-dis", - "name": "llama3-b8-ft-dis", - "developer": "xinchen9", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1546, - "hfopenllm_v2/BBH": 0.4626, - "hfopenllm_v2/MATH Level 5": 0.0393, - "hfopenllm_v2/GPQA": 0.3129, - "hfopenllm_v2/MUSR": 0.3654, - "hfopenllm_v2/MMLU-PRO": 0.3244 - } - }, - { - "id": "xinchen9/Llama3.1_8B_Instruct_CoT", - "name": "Llama3.1_8B_Instruct_CoT", - "developer": "xinchen9", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2974, - "hfopenllm_v2/BBH": 0.4398, - "hfopenllm_v2/MATH Level 5": 0.0604, - "hfopenllm_v2/GPQA": 0.302, - "hfopenllm_v2/MUSR": 0.4371, - "hfopenllm_v2/MMLU-PRO": 0.2879 - } - }, - { - "id": "xinchen9/Llama3.1_CoT", - "name": "Llama3.1_CoT", - "developer": "xinchen9", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2246, - "hfopenllm_v2/BBH": 0.4341, - "hfopenllm_v2/MATH Level 5": 0.0385, - "hfopenllm_v2/GPQA": 0.2886, - "hfopenllm_v2/MUSR": 0.4305, - "hfopenllm_v2/MMLU-PRO": 0.2739 - } - }, - { - "id": "xinchen9/Llama3.1_CoT_V1", - "name": "Llama3.1_CoT_V1", - "developer": "xinchen9", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2453, - "hfopenllm_v2/BBH": 0.4376, - "hfopenllm_v2/MATH Level 5": 0.0332, - "hfopenllm_v2/GPQA": 0.2794, - "hfopenllm_v2/MUSR": 0.4572, - "hfopenllm_v2/MMLU-PRO": 0.2805 - } - }, - { - "id": "xinchen9/Mistral-7B-CoT", - "name": "Mistral-7B-CoT", - "developer": "xinchen9", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2783, - "hfopenllm_v2/BBH": 0.3873, - "hfopenllm_v2/MATH Level 5": 0.0249, - "hfopenllm_v2/GPQA": 0.2492, - "hfopenllm_v2/MUSR": 0.3994, - "hfopenllm_v2/MMLU-PRO": 0.2284 - } - }, - { - "id": "Xkev/Llama-3.2V-11B-cot", - "name": "Llama-3.2V-11B-cot", - "developer": "Xkev", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4158, - "hfopenllm_v2/BBH": 0.4959, - "hfopenllm_v2/MATH Level 5": 0.1556, - "hfopenllm_v2/GPQA": 0.2953, - "hfopenllm_v2/MUSR": 0.4159, - "hfopenllm_v2/MMLU-PRO": 0.3587 - } - }, - { - "id": "xkp24/Llama-3-8B-Instruct-SPPO-Iter2_bt_2b-table", - "name": "Llama-3-8B-Instruct-SPPO-Iter2_bt_2b-table", - "developer": "xkp24", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6375, - "hfopenllm_v2/BBH": 0.4912, - "hfopenllm_v2/MATH Level 5": 0.0921, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.382, - "hfopenllm_v2/MMLU-PRO": 0.3686 - } - }, - { - "id": "xkp24/Llama-3-8B-Instruct-SPPO-Iter2_bt_8b-table", - "name": "Llama-3-8B-Instruct-SPPO-Iter2_bt_8b-table", - "developer": "xkp24", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7275, - "hfopenllm_v2/BBH": 0.5057, - "hfopenllm_v2/MATH Level 5": 0.0846, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3819, - "hfopenllm_v2/MMLU-PRO": 0.3697 - } - }, - { - "id": "xkp24/Llama-3-8B-Instruct-SPPO-Iter2_gp_2b-table", - "name": "Llama-3-8B-Instruct-SPPO-Iter2_gp_2b-table", - "developer": "xkp24", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6569, - "hfopenllm_v2/BBH": 0.4952, - "hfopenllm_v2/MATH Level 5": 0.0891, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3594, - "hfopenllm_v2/MMLU-PRO": 0.3702 - } - }, - { - "id": "xkp24/Llama-3-8B-Instruct-SPPO-Iter2_gp_8b-table", - "name": "Llama-3-8B-Instruct-SPPO-Iter2_gp_8b-table", - "developer": "xkp24", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6621, - "hfopenllm_v2/BBH": 0.5004, - "hfopenllm_v2/MATH Level 5": 0.0861, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3805, - "hfopenllm_v2/MMLU-PRO": 0.36 - } - }, - { - "id": "xkp24/Llama-3-8B-Instruct-SPPO-score-Iter2_bt_2b-table-0.001", - "name": "Llama-3-8B-Instruct-SPPO-score-Iter2_bt_2b-table-0.001", - "developer": "xkp24", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6042, - "hfopenllm_v2/BBH": 0.4936, - "hfopenllm_v2/MATH Level 5": 0.0997, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3793, - "hfopenllm_v2/MMLU-PRO": 0.3708 - } - }, - { - "id": "xkp24/Llama-3-8B-Instruct-SPPO-score-Iter2_bt_8b-table-0.002", - "name": "Llama-3-8B-Instruct-SPPO-score-Iter2_bt_8b-table-0.002", - "developer": "xkp24", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7132, - "hfopenllm_v2/BBH": 0.4996, - "hfopenllm_v2/MATH Level 5": 0.0853, - "hfopenllm_v2/GPQA": 0.2584, - "hfopenllm_v2/MUSR": 0.3872, - "hfopenllm_v2/MMLU-PRO": 0.3664 - } - }, - { - "id": "xkp24/Llama-3-8B-Instruct-SPPO-score-Iter2_gp_2b-table-0.001", - "name": "Llama-3-8B-Instruct-SPPO-score-Iter2_gp_2b-table-0.001", - "developer": "xkp24", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5947, - "hfopenllm_v2/BBH": 0.4899, - "hfopenllm_v2/MATH Level 5": 0.1073, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3581, - "hfopenllm_v2/MMLU-PRO": 0.3704 - } - }, - { - "id": "xkp24/Llama-3-8B-Instruct-SPPO-score-Iter2_gp_8b-table-0.002", - "name": "Llama-3-8B-Instruct-SPPO-score-Iter2_gp_8b-table-0.002", - "developer": "xkp24", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6453, - "hfopenllm_v2/BBH": 0.4951, - "hfopenllm_v2/MATH Level 5": 0.0937, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3939, - "hfopenllm_v2/MMLU-PRO": 0.353 - } - }, - { - "id": "xMaulana/FinMatcha-3B-Instruct", - "name": "FinMatcha-3B-Instruct", - "developer": "xMaulana", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7548, - "hfopenllm_v2/BBH": 0.4536, - "hfopenllm_v2/MATH Level 5": 0.1435, - "hfopenllm_v2/GPQA": 0.2693, - "hfopenllm_v2/MUSR": 0.3633, - "hfopenllm_v2/MMLU-PRO": 0.3182 - } - }, - { - "id": "xukp20/llama-3-8b-instruct-sppo-iter1-gp-2b-tau01-table", - "name": "llama-3-8b-instruct-sppo-iter1-gp-2b-tau01-table", - "developer": "xukp20", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.69, - "hfopenllm_v2/BBH": 0.4978, - "hfopenllm_v2/MATH Level 5": 0.105, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3673, - "hfopenllm_v2/MMLU-PRO": 0.3716 - } - }, - { - "id": "xukp20/Llama-3-8B-Instruct-SPPO-Iter3_bt_2b-table", - "name": "Llama-3-8B-Instruct-SPPO-Iter3_bt_2b-table", - "developer": "xukp20", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5756, - "hfopenllm_v2/BBH": 0.4901, - "hfopenllm_v2/MATH Level 5": 0.0997, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.366, - "hfopenllm_v2/MMLU-PRO": 0.3659 - } - }, - { - "id": "xukp20/Llama-3-8B-Instruct-SPPO-Iter3_bt_8b-table", - "name": "Llama-3-8B-Instruct-SPPO-Iter3_bt_8b-table", - "developer": "xukp20", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7034, - "hfopenllm_v2/BBH": 0.5092, - "hfopenllm_v2/MATH Level 5": 0.0967, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3739, - "hfopenllm_v2/MMLU-PRO": 0.3693 - } - }, - { - "id": "xukp20/Llama-3-8B-Instruct-SPPO-Iter3_gp_2b-table", - "name": "Llama-3-8B-Instruct-SPPO-Iter3_gp_2b-table", - "developer": "xukp20", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6024, - "hfopenllm_v2/BBH": 0.497, - "hfopenllm_v2/MATH Level 5": 0.1042, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3674, - "hfopenllm_v2/MMLU-PRO": 0.3658 - } - }, - { - "id": "xukp20/Llama-3-8B-Instruct-SPPO-Iter3_gp_8b-table", - "name": "Llama-3-8B-Instruct-SPPO-Iter3_gp_8b-table", - "developer": "xukp20", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.662, - "hfopenllm_v2/BBH": 0.5, - "hfopenllm_v2/MATH Level 5": 0.0937, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3818, - "hfopenllm_v2/MMLU-PRO": 0.3615 - } - }, - { - "id": "xukp20/Llama-3-8B-Instruct-SPPO-score-Iter3_bt_2b-table-0.001", - "name": "Llama-3-8B-Instruct-SPPO-score-Iter3_bt_2b-table-0.001", - "developer": "xukp20", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5336, - "hfopenllm_v2/BBH": 0.4915, - "hfopenllm_v2/MATH Level 5": 0.0982, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.378, - "hfopenllm_v2/MMLU-PRO": 0.3625 - } - }, - { - "id": "xukp20/Llama-3-8B-Instruct-SPPO-score-Iter3_bt_8b-table-0.002", - "name": "Llama-3-8B-Instruct-SPPO-score-Iter3_bt_8b-table-0.002", - "developer": "xukp20", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6852, - "hfopenllm_v2/BBH": 0.5075, - "hfopenllm_v2/MATH Level 5": 0.0718, - "hfopenllm_v2/GPQA": 0.2584, - "hfopenllm_v2/MUSR": 0.3832, - "hfopenllm_v2/MMLU-PRO": 0.3621 - } - }, - { - "id": "xukp20/Llama-3-8B-Instruct-SPPO-score-Iter3_gp_2b-table-0.001", - "name": "Llama-3-8B-Instruct-SPPO-score-Iter3_gp_2b-table-0.001", - "developer": "xukp20", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5482, - "hfopenllm_v2/BBH": 0.4887, - "hfopenllm_v2/MATH Level 5": 0.0891, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.3633, - "hfopenllm_v2/MMLU-PRO": 0.3671 - } - }, - { - "id": "xwen-team/Xwen-7B-Chat", - "name": "Xwen-7B-Chat", - "developer": "xwen-team", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6864, - "hfopenllm_v2/BBH": 0.5068, - "hfopenllm_v2/MATH Level 5": 0.4509, - "hfopenllm_v2/GPQA": 0.2609, - "hfopenllm_v2/MUSR": 0.3914, - "hfopenllm_v2/MMLU-PRO": 0.429 - } - }, - { - "id": "xxx777xxxASD/L3.1-ClaudeMaid-4x8B", - "name": "L3.1-ClaudeMaid-4x8B", - "developer": "xxx777xxxASD", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6696, - "hfopenllm_v2/BBH": 0.5071, - "hfopenllm_v2/MATH Level 5": 0.1412, - "hfopenllm_v2/GPQA": 0.2911, - "hfopenllm_v2/MUSR": 0.4289, - "hfopenllm_v2/MMLU-PRO": 0.358 - } - }, - { - "id": "yam-peleg/Hebrew-Gemma-11B-Instruct", - "name": "Hebrew-Gemma-11B-Instruct", - "developer": "yam-peleg", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3021, - "hfopenllm_v2/BBH": 0.4036, - "hfopenllm_v2/MATH Level 5": 0.0657, - "hfopenllm_v2/GPQA": 0.276, - "hfopenllm_v2/MUSR": 0.4089, - "hfopenllm_v2/MMLU-PRO": 0.2554 - } - }, - { - "id": "yam-peleg/Hebrew-Mistral-7B", - "name": "Hebrew-Mistral-7B", - "developer": "yam-peleg", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2328, - "hfopenllm_v2/BBH": 0.4334, - "hfopenllm_v2/MATH Level 5": 0.0498, - "hfopenllm_v2/GPQA": 0.2794, - "hfopenllm_v2/MUSR": 0.3977, - "hfopenllm_v2/MMLU-PRO": 0.278 - } - }, - { - "id": "yam-peleg/Hebrew-Mistral-7B-200K", - "name": "Hebrew-Mistral-7B-200K", - "developer": "yam-peleg", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.177, - "hfopenllm_v2/BBH": 0.3411, - "hfopenllm_v2/MATH Level 5": 0.031, - "hfopenllm_v2/GPQA": 0.2534, - "hfopenllm_v2/MUSR": 0.374, - "hfopenllm_v2/MMLU-PRO": 0.2529 - } - }, - { - "id": "yandex/YaLM-100B", - "name": "YaLM 100B", - "developer": "yandex", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.075, - "helm_classic/MMLU": 0.243, - "helm_classic/BoolQ": 0.634, - "helm_classic/NarrativeQA": 0.252, - "helm_classic/NaturalQuestions (open-book)": 0.227, - "helm_classic/QuAC": 0.162, - "helm_classic/HellaSwag": -1.0, - "helm_classic/OpenbookQA": -1.0, - "helm_classic/TruthfulQA": 0.202, - "helm_classic/MS MARCO (TREC)": -1.0, - "helm_classic/CNN/DailyMail": 0.017, - "helm_classic/XSUM": 0.021, - "helm_classic/IMDB": 0.836, - "helm_classic/CivilComments": 0.49, - "helm_classic/RAFT": 0.395 - } - }, - { - "id": "yanng1242/Marcoro14-7B-slerp", - "name": "Marcoro14-7B-slerp", - "developer": "yanng1242", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.406, - "hfopenllm_v2/BBH": 0.5252, - "hfopenllm_v2/MATH Level 5": 0.0748, - "hfopenllm_v2/GPQA": 0.3146, - "hfopenllm_v2/MUSR": 0.4686, - "hfopenllm_v2/MMLU-PRO": 0.3168 - } - }, - { - "id": "Yash21/TinyYi-7B-Test", - "name": "TinyYi-7B-Test", - "developer": "Yash21", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1856, - "hfopenllm_v2/BBH": 0.291, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2643, - "hfopenllm_v2/MUSR": 0.3364, - "hfopenllm_v2/MMLU-PRO": 0.1091 - } - }, - { - "id": "yasserrmd/Coder-GRPO-3B", - "name": "Coder-GRPO-3B", - "developer": "yasserrmd", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6208, - "hfopenllm_v2/BBH": 0.4469, - "hfopenllm_v2/MATH Level 5": 0.3202, - "hfopenllm_v2/GPQA": 0.2777, - "hfopenllm_v2/MUSR": 0.4115, - "hfopenllm_v2/MMLU-PRO": 0.3197 - } - }, - { - "id": "yasserrmd/Text2SQL-1.5B", - "name": "Text2SQL-1.5B", - "developer": "yasserrmd", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2857, - "hfopenllm_v2/BBH": 0.3858, - "hfopenllm_v2/MATH Level 5": 0.068, - "hfopenllm_v2/GPQA": 0.2878, - "hfopenllm_v2/MUSR": 0.3942, - "hfopenllm_v2/MMLU-PRO": 0.2363 - } - }, - { - "id": "ycros/BagelMIsteryTour-v2-8x7B", - "name": "BagelMIsteryTour-v2-8x7B", - "developer": "ycros", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5994, - "hfopenllm_v2/BBH": 0.5159, - "hfopenllm_v2/MATH Level 5": 0.0785, - "hfopenllm_v2/GPQA": 0.3045, - "hfopenllm_v2/MUSR": 0.4203, - "hfopenllm_v2/MMLU-PRO": 0.3473 - } - }, - { - "id": "yfzp/Llama-3-8B-Instruct-SPPO-Iter1_bt_2b-table", - "name": "Llama-3-8B-Instruct-SPPO-Iter1_bt_2b-table", - "developer": "yfzp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6709, - "hfopenllm_v2/BBH": 0.4987, - "hfopenllm_v2/MATH Level 5": 0.1118, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3727, - "hfopenllm_v2/MMLU-PRO": 0.3716 - } - }, - { - "id": "yfzp/Llama-3-8B-Instruct-SPPO-Iter1_bt_8b-table", - "name": "Llama-3-8B-Instruct-SPPO-Iter1_bt_8b-table", - "developer": "yfzp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7333, - "hfopenllm_v2/BBH": 0.508, - "hfopenllm_v2/MATH Level 5": 0.1035, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3806, - "hfopenllm_v2/MMLU-PRO": 0.3748 - } - }, - { - "id": "yfzp/Llama-3-8B-Instruct-SPPO-Iter1_gp_2b-table", - "name": "Llama-3-8B-Instruct-SPPO-Iter1_gp_2b-table", - "developer": "yfzp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6785, - "hfopenllm_v2/BBH": 0.4941, - "hfopenllm_v2/MATH Level 5": 0.1125, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3647, - "hfopenllm_v2/MMLU-PRO": 0.3718 - } - }, - { - "id": "yfzp/Llama-3-8B-Instruct-SPPO-Iter1_gp_8b-table", - "name": "Llama-3-8B-Instruct-SPPO-Iter1_gp_8b-table", - "developer": "yfzp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7132, - "hfopenllm_v2/BBH": 0.5025, - "hfopenllm_v2/MATH Level 5": 0.0989, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3713, - "hfopenllm_v2/MMLU-PRO": 0.3683 - } - }, - { - "id": "yfzp/Llama-3-8B-Instruct-SPPO-score-Iter1_bt_2b-table-0.001", - "name": "Llama-3-8B-Instruct-SPPO-score-Iter1_bt_2b-table-0.001", - "developer": "yfzp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6496, - "hfopenllm_v2/BBH": 0.4979, - "hfopenllm_v2/MATH Level 5": 0.1012, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.378, - "hfopenllm_v2/MMLU-PRO": 0.372 - } - }, - { - "id": "yfzp/Llama-3-8B-Instruct-SPPO-score-Iter1_bt_8b-table-0.002", - "name": "Llama-3-8B-Instruct-SPPO-score-Iter1_bt_8b-table-0.002", - "developer": "yfzp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7196, - "hfopenllm_v2/BBH": 0.5045, - "hfopenllm_v2/MATH Level 5": 0.0876, - "hfopenllm_v2/GPQA": 0.2601, - "hfopenllm_v2/MUSR": 0.3831, - "hfopenllm_v2/MMLU-PRO": 0.3734 - } - }, - { - "id": "yfzp/Llama-3-8B-Instruct-SPPO-score-Iter1_gp_2b-table-0.001", - "name": "Llama-3-8B-Instruct-SPPO-score-Iter1_gp_2b-table-0.001", - "developer": "yfzp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6504, - "hfopenllm_v2/BBH": 0.4958, - "hfopenllm_v2/MATH Level 5": 0.0937, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.366, - "hfopenllm_v2/MMLU-PRO": 0.3703 - } - }, - { - "id": "yfzp/Llama-3-8B-Instruct-SPPO-score-Iter1_gp_8b-table-0.002", - "name": "Llama-3-8B-Instruct-SPPO-score-Iter1_gp_8b-table-0.002", - "developer": "yfzp", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7016, - "hfopenllm_v2/BBH": 0.4992, - "hfopenllm_v2/MATH Level 5": 0.0869, - "hfopenllm_v2/GPQA": 0.2592, - "hfopenllm_v2/MUSR": 0.3779, - "hfopenllm_v2/MMLU-PRO": 0.3669 - } - }, - { - "id": "yifAI/Llama-3-8B-Instruct-SPPO-score-Iter3_gp_8b-table-0.002", - "name": "Llama-3-8B-Instruct-SPPO-score-Iter3_gp_8b-table-0.002", - "developer": "yifAI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.649, - "hfopenllm_v2/BBH": 0.4915, - "hfopenllm_v2/MATH Level 5": 0.0755, - "hfopenllm_v2/GPQA": 0.2617, - "hfopenllm_v2/MUSR": 0.3899, - "hfopenllm_v2/MMLU-PRO": 0.352 - } - }, - { - "id": "ylalain/ECE-PRYMMAL-YL-1B-SLERP-V8", - "name": "ECE-PRYMMAL-YL-1B-SLERP-V8", - "developer": "ylalain", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1505, - "hfopenllm_v2/BBH": 0.3976, - "hfopenllm_v2/MATH Level 5": 0.0045, - "hfopenllm_v2/GPQA": 0.2894, - "hfopenllm_v2/MUSR": 0.3875, - "hfopenllm_v2/MMLU-PRO": 0.2384 - } - }, - { - "id": "ymcki/gemma-2-2b-jpn-it-abliterated-17", - "name": "gemma-2-2b-jpn-it-abliterated-17", - "developer": "ymcki", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5082, - "hfopenllm_v2/BBH": 0.4076, - "hfopenllm_v2/MATH Level 5": 0.0385, - "hfopenllm_v2/GPQA": 0.2718, - "hfopenllm_v2/MUSR": 0.3701, - "hfopenllm_v2/MMLU-PRO": 0.2455 - } - }, - { - "id": "ymcki/gemma-2-2b-jpn-it-abliterated-17-18-24", - "name": "gemma-2-2b-jpn-it-abliterated-17-18-24", - "developer": "ymcki", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5055, - "hfopenllm_v2/BBH": 0.3812, - "hfopenllm_v2/MATH Level 5": 0.0257, - "hfopenllm_v2/GPQA": 0.281, - "hfopenllm_v2/MUSR": 0.3502, - "hfopenllm_v2/MMLU-PRO": 0.2282 - } - }, - { - "id": "ymcki/gemma-2-2b-jpn-it-abliterated-17-ORPO", - "name": "gemma-2-2b-jpn-it-abliterated-17-ORPO", - "developer": "ymcki", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4748, - "hfopenllm_v2/BBH": 0.3898, - "hfopenllm_v2/MATH Level 5": 0.0619, - "hfopenllm_v2/GPQA": 0.2743, - "hfopenllm_v2/MUSR": 0.3768, - "hfopenllm_v2/MMLU-PRO": 0.2191 - } - }, - { - "id": "ymcki/gemma-2-2b-jpn-it-abliterated-17-ORPO-alpaca", - "name": "gemma-2-2b-jpn-it-abliterated-17-ORPO-alpaca", - "developer": "ymcki", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3065, - "hfopenllm_v2/BBH": 0.4072, - "hfopenllm_v2/MATH Level 5": 0.0325, - "hfopenllm_v2/GPQA": 0.2693, - "hfopenllm_v2/MUSR": 0.3969, - "hfopenllm_v2/MMLU-PRO": 0.2249 - } - }, - { - "id": "ymcki/gemma-2-2b-jpn-it-abliterated-18", - "name": "gemma-2-2b-jpn-it-abliterated-18", - "developer": "ymcki", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5175, - "hfopenllm_v2/BBH": 0.4132, - "hfopenllm_v2/MATH Level 5": 0.0446, - "hfopenllm_v2/GPQA": 0.2735, - "hfopenllm_v2/MUSR": 0.3742, - "hfopenllm_v2/MMLU-PRO": 0.2505 - } - }, - { - "id": "ymcki/gemma-2-2b-jpn-it-abliterated-18-ORPO", - "name": "gemma-2-2b-jpn-it-abliterated-18-ORPO", - "developer": "ymcki", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4742, - "hfopenllm_v2/BBH": 0.4039, - "hfopenllm_v2/MATH Level 5": 0.0468, - "hfopenllm_v2/GPQA": 0.2617, - "hfopenllm_v2/MUSR": 0.3953, - "hfopenllm_v2/MMLU-PRO": 0.2185 - } - }, - { - "id": "ymcki/gemma-2-2b-jpn-it-abliterated-24", - "name": "gemma-2-2b-jpn-it-abliterated-24", - "developer": "ymcki", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4979, - "hfopenllm_v2/BBH": 0.411, - "hfopenllm_v2/MATH Level 5": 0.0438, - "hfopenllm_v2/GPQA": 0.2777, - "hfopenllm_v2/MUSR": 0.3915, - "hfopenllm_v2/MMLU-PRO": 0.2473 - } - }, - { - "id": "ymcki/gemma-2-2b-ORPO-jpn-it-abliterated-18", - "name": "gemma-2-2b-ORPO-jpn-it-abliterated-18", - "developer": "ymcki", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4631, - "hfopenllm_v2/BBH": 0.4053, - "hfopenllm_v2/MATH Level 5": 0.0431, - "hfopenllm_v2/GPQA": 0.2886, - "hfopenllm_v2/MUSR": 0.3754, - "hfopenllm_v2/MMLU-PRO": 0.2345 - } - }, - { - "id": "ymcki/gemma-2-2b-ORPO-jpn-it-abliterated-18-merge", - "name": "gemma-2-2b-ORPO-jpn-it-abliterated-18-merge", - "developer": "ymcki", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5218, - "hfopenllm_v2/BBH": 0.4147, - "hfopenllm_v2/MATH Level 5": 0.0544, - "hfopenllm_v2/GPQA": 0.2836, - "hfopenllm_v2/MUSR": 0.3514, - "hfopenllm_v2/MMLU-PRO": 0.2461 - } - }, - { - "id": "ymcki/Llama-3.1-8B-GRPO-Instruct", - "name": "Llama-3.1-8B-GRPO-Instruct", - "developer": "ymcki", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7445, - "hfopenllm_v2/BBH": 0.5132, - "hfopenllm_v2/MATH Level 5": 0.2024, - "hfopenllm_v2/GPQA": 0.2945, - "hfopenllm_v2/MUSR": 0.3817, - "hfopenllm_v2/MMLU-PRO": 0.3738 - } - }, - { - "id": "ymcki/Llama-3.1-8B-SFT-GRPO-Instruct", - "name": "Llama-3.1-8B-SFT-GRPO-Instruct", - "developer": "ymcki", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3354, - "hfopenllm_v2/BBH": 0.3126, - "hfopenllm_v2/MATH Level 5": 0.04, - "hfopenllm_v2/GPQA": 0.2534, - "hfopenllm_v2/MUSR": 0.3526, - "hfopenllm_v2/MMLU-PRO": 0.1098 - } - }, - { - "id": "Youlln/1PARAMMYL-8B-ModelStock", - "name": "1PARAMMYL-8B-ModelStock", - "developer": "Youlln", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5371, - "hfopenllm_v2/BBH": 0.5216, - "hfopenllm_v2/MATH Level 5": 0.1488, - "hfopenllm_v2/GPQA": 0.3238, - "hfopenllm_v2/MUSR": 0.4409, - "hfopenllm_v2/MMLU-PRO": 0.4 - } - }, - { - "id": "Youlln/2PRYMMAL-Yi1.5-6B-SLERP", - "name": "2PRYMMAL-Yi1.5-6B-SLERP", - "developer": "Youlln", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2826, - "hfopenllm_v2/BBH": 0.4665, - "hfopenllm_v2/MATH Level 5": 0.1133, - "hfopenllm_v2/GPQA": 0.307, - "hfopenllm_v2/MUSR": 0.4756, - "hfopenllm_v2/MMLU-PRO": 0.317 - } - }, - { - "id": "Youlln/3PRYMMAL-PHI3-3B-SLERP", - "name": "3PRYMMAL-PHI3-3B-SLERP", - "developer": "Youlln", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3656, - "hfopenllm_v2/BBH": 0.5422, - "hfopenllm_v2/MATH Level 5": 0.1715, - "hfopenllm_v2/GPQA": 0.3263, - "hfopenllm_v2/MUSR": 0.4648, - "hfopenllm_v2/MMLU-PRO": 0.4002 - } - }, - { - "id": "Youlln/4PRYMMAL-GEMMA2-9B-SLERP", - "name": "4PRYMMAL-GEMMA2-9B-SLERP", - "developer": "Youlln", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2714, - "hfopenllm_v2/BBH": 0.5923, - "hfopenllm_v2/MATH Level 5": 0.0906, - "hfopenllm_v2/GPQA": 0.3305, - "hfopenllm_v2/MUSR": 0.4672, - "hfopenllm_v2/MMLU-PRO": 0.421 - } - }, - { - "id": "Youlln/ECE-MIRAGE-1-12B", - "name": "ECE-MIRAGE-1-12B", - "developer": "Youlln", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.207, - "hfopenllm_v2/BBH": 0.3011, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2634, - "hfopenllm_v2/MUSR": 0.3219, - "hfopenllm_v2/MMLU-PRO": 0.111 - } - }, - { - "id": "Youlln/ECE-MIRAGE-1-15B", - "name": "ECE-MIRAGE-1-15B", - "developer": "Youlln", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.207, - "hfopenllm_v2/BBH": 0.3011, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2634, - "hfopenllm_v2/MUSR": 0.3219, - "hfopenllm_v2/MMLU-PRO": 0.111 - } - }, - { - "id": "Youlln/ECE-PRYMMAL-0.5B-FT-V3", - "name": "ECE-PRYMMAL-0.5B-FT-V3", - "developer": "Youlln", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1642, - "hfopenllm_v2/BBH": 0.3093, - "hfopenllm_v2/MATH Level 5": 0.003, - "hfopenllm_v2/GPQA": 0.2576, - "hfopenllm_v2/MUSR": 0.3644, - "hfopenllm_v2/MMLU-PRO": 0.1161 - } - }, - { - "id": "Youlln/ECE-PRYMMAL-0.5B-FT-V3-MUSR", - "name": "ECE-PRYMMAL-0.5B-FT-V3-MUSR", - "developer": "Youlln", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1533, - "hfopenllm_v2/BBH": 0.3041, - "hfopenllm_v2/MATH Level 5": 0.0242, - "hfopenllm_v2/GPQA": 0.2492, - "hfopenllm_v2/MUSR": 0.366, - "hfopenllm_v2/MMLU-PRO": 0.1645 - } - }, - { - "id": "Youlln/ECE-PRYMMAL-0.5B-FT-V4-MUSR", - "name": "ECE-PRYMMAL-0.5B-FT-V4-MUSR", - "developer": "Youlln", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1138, - "hfopenllm_v2/BBH": 0.3038, - "hfopenllm_v2/MATH Level 5": 0.0121, - "hfopenllm_v2/GPQA": 0.2701, - "hfopenllm_v2/MUSR": 0.3529, - "hfopenllm_v2/MMLU-PRO": 0.1321 - } - }, - { - "id": "Youlln/ECE-PRYMMAL-0.5B-SLERP-V2", - "name": "ECE-PRYMMAL-0.5B-SLERP-V2", - "developer": "Youlln", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1612, - "hfopenllm_v2/BBH": 0.2935, - "hfopenllm_v2/MATH Level 5": 0.0008, - "hfopenllm_v2/GPQA": 0.2743, - "hfopenllm_v2/MUSR": 0.3831, - "hfopenllm_v2/MMLU-PRO": 0.1095 - } - }, - { - "id": "Youlln/ECE-PRYMMAL-0.5B-SLERP-V3", - "name": "ECE-PRYMMAL-0.5B-SLERP-V3", - "developer": "Youlln", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.167, - "hfopenllm_v2/BBH": 0.2938, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2517, - "hfopenllm_v2/MUSR": 0.3541, - "hfopenllm_v2/MMLU-PRO": 0.1087 - } - }, - { - "id": "Youlln/ECE-PRYMMAL-YL-1B-SLERP-V1", - "name": "ECE-PRYMMAL-YL-1B-SLERP-V1", - "developer": "Youlln", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3251, - "hfopenllm_v2/BBH": 0.4209, - "hfopenllm_v2/MATH Level 5": 0.1073, - "hfopenllm_v2/GPQA": 0.2911, - "hfopenllm_v2/MUSR": 0.4266, - "hfopenllm_v2/MMLU-PRO": 0.2936 - } - }, - { - "id": "Youlln/ECE-PRYMMAL-YL-1B-SLERP-V2", - "name": "ECE-PRYMMAL-YL-1B-SLERP-V2", - "developer": "Youlln", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3251, - "hfopenllm_v2/BBH": 0.4209, - "hfopenllm_v2/MATH Level 5": 0.1073, - "hfopenllm_v2/GPQA": 0.2911, - "hfopenllm_v2/MUSR": 0.4266, - "hfopenllm_v2/MMLU-PRO": 0.2936 - } - }, - { - "id": "Youlln/ECE-PRYMMAL-YL-7B-SLERP-V4", - "name": "ECE-PRYMMAL-YL-7B-SLERP-V4", - "developer": "Youlln", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.251, - "hfopenllm_v2/BBH": 0.377, - "hfopenllm_v2/MATH Level 5": 0.0536, - "hfopenllm_v2/GPQA": 0.2651, - "hfopenllm_v2/MUSR": 0.3745, - "hfopenllm_v2/MMLU-PRO": 0.2132 - } - }, - { - "id": "Youlln/ECE-PRYMMAL0.5-FT", - "name": "ECE-PRYMMAL0.5-FT", - "developer": "Youlln", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1851, - "hfopenllm_v2/BBH": 0.3132, - "hfopenllm_v2/MATH Level 5": 0.0234, - "hfopenllm_v2/GPQA": 0.2559, - "hfopenllm_v2/MUSR": 0.3301, - "hfopenllm_v2/MMLU-PRO": 0.1477 - } - }, - { - "id": "Youlln/ECE-PRYMMAL0.5B-Youri", - "name": "ECE-PRYMMAL0.5B-Youri", - "developer": "Youlln", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1446, - "hfopenllm_v2/BBH": 0.2817, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2433, - "hfopenllm_v2/MUSR": 0.3697, - "hfopenllm_v2/MMLU-PRO": 0.1095 - } - }, - { - "id": "Youlln/ECE-PRYMMAL1B-FT-V1", - "name": "ECE-PRYMMAL1B-FT-V1", - "developer": "Youlln", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2144, - "hfopenllm_v2/BBH": 0.4033, - "hfopenllm_v2/MATH Level 5": 0.0642, - "hfopenllm_v2/GPQA": 0.2785, - "hfopenllm_v2/MUSR": 0.3417, - "hfopenllm_v2/MMLU-PRO": 0.2743 - } - }, - { - "id": "Youlln/ECE-Qwen0.5B-FT-V2", - "name": "ECE-Qwen0.5B-FT-V2", - "developer": "Youlln", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2526, - "hfopenllm_v2/BBH": 0.329, - "hfopenllm_v2/MATH Level 5": 0.0204, - "hfopenllm_v2/GPQA": 0.2668, - "hfopenllm_v2/MUSR": 0.3063, - "hfopenllm_v2/MMLU-PRO": 0.1666 - } - }, - { - "id": "Youlln/ECE.EIFFEIL.ia-0.5B-SLERP", - "name": "ECE.EIFFEIL.ia-0.5B-SLERP", - "developer": "Youlln", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2561, - "hfopenllm_v2/BBH": 0.3306, - "hfopenllm_v2/MATH Level 5": 0.0597, - "hfopenllm_v2/GPQA": 0.2651, - "hfopenllm_v2/MUSR": 0.3102, - "hfopenllm_v2/MMLU-PRO": 0.1903 - } - }, - { - "id": "YoungPanda/qwenqwen", - "name": "qwenqwen", - "developer": "YoungPanda", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1264, - "hfopenllm_v2/BBH": 0.3379, - "hfopenllm_v2/MATH Level 5": 0.0355, - "hfopenllm_v2/GPQA": 0.25, - "hfopenllm_v2/MUSR": 0.3434, - "hfopenllm_v2/MMLU-PRO": 0.1168 - } - }, - { - "id": "YOYO-AI/Qwen2.5-14B-1M-YOYO-V3", - "name": "Qwen2.5-14B-1M-YOYO-V3", - "developer": "YOYO-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8398, - "hfopenllm_v2/BBH": 0.6448, - "hfopenllm_v2/MATH Level 5": 0.5355, - "hfopenllm_v2/GPQA": 0.3289, - "hfopenllm_v2/MUSR": 0.4141, - "hfopenllm_v2/MMLU-PRO": 0.5207 - } - }, - { - "id": "YOYO-AI/Qwen2.5-14B-it-restore", - "name": "Qwen2.5-14B-it-restore", - "developer": "YOYO-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8209, - "hfopenllm_v2/BBH": 0.6388, - "hfopenllm_v2/MATH Level 5": 0.537, - "hfopenllm_v2/GPQA": 0.3372, - "hfopenllm_v2/MUSR": 0.4087, - "hfopenllm_v2/MMLU-PRO": 0.49 - } - }, - { - "id": "YOYO-AI/Qwen2.5-14B-YOYO-0505", - "name": "Qwen2.5-14B-YOYO-0505", - "developer": "YOYO-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5883, - "hfopenllm_v2/BBH": 0.6539, - "hfopenllm_v2/MATH Level 5": 0.4434, - "hfopenllm_v2/GPQA": 0.3733, - "hfopenllm_v2/MUSR": 0.4757, - "hfopenllm_v2/MMLU-PRO": 0.5371 - } - }, - { - "id": "YOYO-AI/Qwen2.5-14B-YOYO-0510-v2", - "name": "Qwen2.5-14B-YOYO-0510-v2", - "developer": "YOYO-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5947, - "hfopenllm_v2/BBH": 0.6553, - "hfopenllm_v2/MATH Level 5": 0.4441, - "hfopenllm_v2/GPQA": 0.3817, - "hfopenllm_v2/MUSR": 0.4744, - "hfopenllm_v2/MMLU-PRO": 0.5381 - } - }, - { - "id": "YOYO-AI/Qwen2.5-14B-YOYO-0805", - "name": "Qwen2.5-14B-YOYO-0805", - "developer": "YOYO-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5883, - "hfopenllm_v2/BBH": 0.6539, - "hfopenllm_v2/MATH Level 5": 0.4434, - "hfopenllm_v2/GPQA": 0.3733, - "hfopenllm_v2/MUSR": 0.4757, - "hfopenllm_v2/MMLU-PRO": 0.5371 - } - }, - { - "id": "YOYO-AI/Qwen2.5-14B-YOYO-1005", - "name": "Qwen2.5-14B-YOYO-1005", - "developer": "YOYO-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5972, - "hfopenllm_v2/BBH": 0.6542, - "hfopenllm_v2/MATH Level 5": 0.4524, - "hfopenllm_v2/GPQA": 0.3809, - "hfopenllm_v2/MUSR": 0.473, - "hfopenllm_v2/MMLU-PRO": 0.5382 - } - }, - { - "id": "YOYO-AI/Qwen2.5-14B-YOYO-1005-v2", - "name": "Qwen2.5-14B-YOYO-1005-v2", - "developer": "YOYO-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5953, - "hfopenllm_v2/BBH": 0.6551, - "hfopenllm_v2/MATH Level 5": 0.4434, - "hfopenllm_v2/GPQA": 0.3842, - "hfopenllm_v2/MUSR": 0.4731, - "hfopenllm_v2/MMLU-PRO": 0.5372 - } - }, - { - "id": "YOYO-AI/Qwen2.5-14B-YOYO-1010", - "name": "Qwen2.5-14B-YOYO-1010", - "developer": "YOYO-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5899, - "hfopenllm_v2/BBH": 0.654, - "hfopenllm_v2/MATH Level 5": 0.4509, - "hfopenllm_v2/GPQA": 0.3834, - "hfopenllm_v2/MUSR": 0.4744, - "hfopenllm_v2/MMLU-PRO": 0.5376 - } - }, - { - "id": "YOYO-AI/Qwen2.5-14B-YOYO-1010-v2", - "name": "Qwen2.5-14B-YOYO-1010-v2", - "developer": "YOYO-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5947, - "hfopenllm_v2/BBH": 0.6553, - "hfopenllm_v2/MATH Level 5": 0.4441, - "hfopenllm_v2/GPQA": 0.3817, - "hfopenllm_v2/MUSR": 0.4744, - "hfopenllm_v2/MMLU-PRO": 0.5381 - } - }, - { - "id": "YOYO-AI/Qwen2.5-14B-YOYO-latest", - "name": "Qwen2.5-14B-YOYO-latest", - "developer": "YOYO-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5911, - "hfopenllm_v2/BBH": 0.6656, - "hfopenllm_v2/MATH Level 5": 0.4418, - "hfopenllm_v2/GPQA": 0.3826, - "hfopenllm_v2/MUSR": 0.4691, - "hfopenllm_v2/MMLU-PRO": 0.5371 - } - }, - { - "id": "YOYO-AI/Qwen2.5-14B-YOYO-latest-V2", - "name": "Qwen2.5-14B-YOYO-latest-V2", - "developer": "YOYO-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7771, - "hfopenllm_v2/BBH": 0.6299, - "hfopenllm_v2/MATH Level 5": 0.5159, - "hfopenllm_v2/GPQA": 0.354, - "hfopenllm_v2/MUSR": 0.4299, - "hfopenllm_v2/MMLU-PRO": 0.5224 - } - }, - { - "id": "YOYO-AI/Qwen2.5-14B-YOYO-SCE", - "name": "Qwen2.5-14B-YOYO-SCE", - "developer": "YOYO-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5844, - "hfopenllm_v2/BBH": 0.6489, - "hfopenllm_v2/MATH Level 5": 0.4615, - "hfopenllm_v2/GPQA": 0.3742, - "hfopenllm_v2/MUSR": 0.4704, - "hfopenllm_v2/MMLU-PRO": 0.5381 - } - }, - { - "id": "YOYO-AI/Qwen2.5-14B-YOYO-V4", - "name": "Qwen2.5-14B-YOYO-V4", - "developer": "YOYO-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8398, - "hfopenllm_v2/BBH": 0.649, - "hfopenllm_v2/MATH Level 5": 0.5347, - "hfopenllm_v2/GPQA": 0.3221, - "hfopenllm_v2/MUSR": 0.4115, - "hfopenllm_v2/MMLU-PRO": 0.517 - } - }, - { - "id": "YOYO-AI/Qwen2.5-14B-YOYO-V4-p1", - "name": "Qwen2.5-14B-YOYO-V4-p1", - "developer": "YOYO-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8203, - "hfopenllm_v2/BBH": 0.6516, - "hfopenllm_v2/MATH Level 5": 0.5332, - "hfopenllm_v2/GPQA": 0.3456, - "hfopenllm_v2/MUSR": 0.4194, - "hfopenllm_v2/MMLU-PRO": 0.502 - } - }, - { - "id": "YOYO-AI/Qwen2.5-14B-YOYO-V4-p2", - "name": "Qwen2.5-14B-YOYO-V4-p2", - "developer": "YOYO-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8048, - "hfopenllm_v2/BBH": 0.6339, - "hfopenllm_v2/MATH Level 5": 0.5166, - "hfopenllm_v2/GPQA": 0.3272, - "hfopenllm_v2/MUSR": 0.4435, - "hfopenllm_v2/MMLU-PRO": 0.4968 - } - }, - { - "id": "YOYO-AI/Qwen2.5-7B-it-restore", - "name": "Qwen2.5-7B-it-restore", - "developer": "YOYO-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7531, - "hfopenllm_v2/BBH": 0.5407, - "hfopenllm_v2/MATH Level 5": 0.5, - "hfopenllm_v2/GPQA": 0.3012, - "hfopenllm_v2/MUSR": 0.4007, - "hfopenllm_v2/MMLU-PRO": 0.4288 - } - }, - { - "id": "YOYO-AI/Qwen2.5-Coder-14B-YOYO-1010", - "name": "Qwen2.5-Coder-14B-YOYO-1010", - "developer": "YOYO-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5336, - "hfopenllm_v2/BBH": 0.6187, - "hfopenllm_v2/MATH Level 5": 0.3218, - "hfopenllm_v2/GPQA": 0.3523, - "hfopenllm_v2/MUSR": 0.4422, - "hfopenllm_v2/MMLU-PRO": 0.4075 - } - }, - { - "id": "YOYO-AI/ZYH-LLM-Qwen2.5-14B", - "name": "ZYH-LLM-Qwen2.5-14B", - "developer": "YOYO-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5941, - "hfopenllm_v2/BBH": 0.6644, - "hfopenllm_v2/MATH Level 5": 0.4116, - "hfopenllm_v2/GPQA": 0.3859, - "hfopenllm_v2/MUSR": 0.4757, - "hfopenllm_v2/MMLU-PRO": 0.5351 - } - }, - { - "id": "YOYO-AI/ZYH-LLM-Qwen2.5-14B-V2", - "name": "ZYH-LLM-Qwen2.5-14B-V2", - "developer": "YOYO-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5071, - "hfopenllm_v2/BBH": 0.6452, - "hfopenllm_v2/MATH Level 5": 0.3542, - "hfopenllm_v2/GPQA": 0.3792, - "hfopenllm_v2/MUSR": 0.4689, - "hfopenllm_v2/MMLU-PRO": 0.5372 - } - }, - { - "id": "YOYO-AI/ZYH-LLM-Qwen2.5-14B-V3", - "name": "ZYH-LLM-Qwen2.5-14B-V3", - "developer": "YOYO-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8578, - "hfopenllm_v2/BBH": 0.6359, - "hfopenllm_v2/MATH Level 5": 0.5272, - "hfopenllm_v2/GPQA": 0.3322, - "hfopenllm_v2/MUSR": 0.4022, - "hfopenllm_v2/MMLU-PRO": 0.4881 - } - }, - { - "id": "YOYO-AI/ZYH-LLM-Qwen2.5-14B-V4", - "name": "ZYH-LLM-Qwen2.5-14B-V4", - "developer": "YOYO-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8365, - "hfopenllm_v2/BBH": 0.6515, - "hfopenllm_v2/MATH Level 5": 0.5393, - "hfopenllm_v2/GPQA": 0.3146, - "hfopenllm_v2/MUSR": 0.4434, - "hfopenllm_v2/MMLU-PRO": 0.5204 - } - }, - { - "id": "yuchenxie/ArlowGPT-3B-Multilingual", - "name": "ArlowGPT-3B-Multilingual", - "developer": "yuchenxie", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6395, - "hfopenllm_v2/BBH": 0.4301, - "hfopenllm_v2/MATH Level 5": 0.1125, - "hfopenllm_v2/GPQA": 0.2802, - "hfopenllm_v2/MUSR": 0.3727, - "hfopenllm_v2/MMLU-PRO": 0.2817 - } - }, - { - "id": "yuchenxie/ArlowGPT-8B", - "name": "ArlowGPT-8B", - "developer": "yuchenxie", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7847, - "hfopenllm_v2/BBH": 0.508, - "hfopenllm_v2/MATH Level 5": 0.2039, - "hfopenllm_v2/GPQA": 0.2936, - "hfopenllm_v2/MUSR": 0.3882, - "hfopenllm_v2/MMLU-PRO": 0.3787 - } - }, - { - "id": "Yuma42/KangalKhan-RawRuby-7B", - "name": "KangalKhan-RawRuby-7B", - "developer": "Yuma42", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5477, - "hfopenllm_v2/BBH": 0.4755, - "hfopenllm_v2/MATH Level 5": 0.0665, - "hfopenllm_v2/GPQA": 0.2878, - "hfopenllm_v2/MUSR": 0.395, - "hfopenllm_v2/MMLU-PRO": 0.3023 - } - }, - { - "id": "Yuma42/Llama3.1-IgneousIguana-8B", - "name": "Llama3.1-IgneousIguana-8B", - "developer": "Yuma42", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8133, - "hfopenllm_v2/BBH": 0.5191, - "hfopenllm_v2/MATH Level 5": 0.2198, - "hfopenllm_v2/GPQA": 0.3104, - "hfopenllm_v2/MUSR": 0.4203, - "hfopenllm_v2/MMLU-PRO": 0.3974 - } - }, - { - "id": "Yuma42/Llama3.1-SuperHawk-8B", - "name": "Llama3.1-SuperHawk-8B", - "developer": "Yuma42", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7986, - "hfopenllm_v2/BBH": 0.52, - "hfopenllm_v2/MATH Level 5": 0.2349, - "hfopenllm_v2/GPQA": 0.3129, - "hfopenllm_v2/MUSR": 0.4084, - "hfopenllm_v2/MMLU-PRO": 0.3945 - } - }, - { - "id": "yuvraj17/Llama3-8B-abliterated-Spectrum-slerp", - "name": "Llama3-8B-abliterated-Spectrum-slerp", - "developer": "yuvraj17", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.2885, - "hfopenllm_v2/BBH": 0.4978, - "hfopenllm_v2/MATH Level 5": 0.0604, - "hfopenllm_v2/GPQA": 0.3012, - "hfopenllm_v2/MUSR": 0.3998, - "hfopenllm_v2/MMLU-PRO": 0.3257 - } - }, - { - "id": "yuvraj17/Llama3-8B-SuperNova-Spectrum-dare_ties", - "name": "Llama3-8B-SuperNova-Spectrum-dare_ties", - "developer": "yuvraj17", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4013, - "hfopenllm_v2/BBH": 0.4616, - "hfopenllm_v2/MATH Level 5": 0.0846, - "hfopenllm_v2/GPQA": 0.2752, - "hfopenllm_v2/MUSR": 0.4211, - "hfopenllm_v2/MMLU-PRO": 0.3574 - } - }, - { - "id": "yuvraj17/Llama3-8B-SuperNova-Spectrum-Hermes-DPO", - "name": "Llama3-8B-SuperNova-Spectrum-Hermes-DPO", - "developer": "yuvraj17", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4691, - "hfopenllm_v2/BBH": 0.44, - "hfopenllm_v2/MATH Level 5": 0.0566, - "hfopenllm_v2/GPQA": 0.302, - "hfopenllm_v2/MUSR": 0.4012, - "hfopenllm_v2/MMLU-PRO": 0.2635 - } - }, - { - "id": "z-ai/glm-4.5", - "name": "z-ai/glm-4.5", - "developer": "Z.ai", - "evaluator_relationship": null, - "benchmark_scores": { - "livecodebenchpro/Hard Problems": 0.0, - "livecodebenchpro/Medium Problems": 0.028169014084507043, - "livecodebenchpro/Easy Problems": 0.1267605633802817 - } - }, - { - "id": "Z1-Coder/Z1-Coder-7B", - "name": "Z1-Coder-7B", - "developer": "Z1-Coder", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3215, - "hfopenllm_v2/BBH": 0.4842, - "hfopenllm_v2/MATH Level 5": 0.3248, - "hfopenllm_v2/GPQA": 0.2727, - "hfopenllm_v2/MUSR": 0.3622, - "hfopenllm_v2/MMLU-PRO": 0.3759 - } - }, - { - "id": "zai-org/glm-4.5-air-fp8", - "name": "GLM-4.5-Air-FP8", - "developer": "zai-org", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_capabilities/Mean score": 0.67, - "helm_capabilities/MMLU-Pro": 0.762, - "helm_capabilities/GPQA": 0.594, - "helm_capabilities/IFEval": 0.812, - "helm_capabilities/WildBench": 0.789, - "helm_capabilities/Omni-MATH": 0.391 - } - }, - { - "id": "zake7749/gemma-2-2b-it-chinese-kyara-dpo", - "name": "gemma-2-2b-it-chinese-kyara-dpo", - "developer": "zake7749", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5382, - "hfopenllm_v2/BBH": 0.4257, - "hfopenllm_v2/MATH Level 5": 0.0838, - "hfopenllm_v2/GPQA": 0.2668, - "hfopenllm_v2/MUSR": 0.4576, - "hfopenllm_v2/MMLU-PRO": 0.2573 - } - }, - { - "id": "zake7749/gemma-2-9b-it-chinese-kyara", - "name": "gemma-2-9b-it-chinese-kyara", - "developer": "zake7749", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1764, - "hfopenllm_v2/BBH": 0.5954, - "hfopenllm_v2/MATH Level 5": 0.105, - "hfopenllm_v2/GPQA": 0.3381, - "hfopenllm_v2/MUSR": 0.4242, - "hfopenllm_v2/MMLU-PRO": 0.4179 - } - }, - { - "id": "zelk12/gemma-2-S2MTM-9B", - "name": "gemma-2-S2MTM-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7823, - "hfopenllm_v2/BBH": 0.6061, - "hfopenllm_v2/MATH Level 5": 0.2047, - "hfopenllm_v2/GPQA": 0.3456, - "hfopenllm_v2/MUSR": 0.4218, - "hfopenllm_v2/MMLU-PRO": 0.4297 - } - }, - { - "id": "zelk12/Gemma-2-TM-9B", - "name": "Gemma-2-TM-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8045, - "hfopenllm_v2/BBH": 0.5987, - "hfopenllm_v2/MATH Level 5": 0.2024, - "hfopenllm_v2/GPQA": 0.3465, - "hfopenllm_v2/MUSR": 0.4152, - "hfopenllm_v2/MMLU-PRO": 0.4088 - } - }, - { - "id": "zelk12/MT-gemma-2-9B", - "name": "MT-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7968, - "hfopenllm_v2/BBH": 0.6064, - "hfopenllm_v2/MATH Level 5": 0.2054, - "hfopenllm_v2/GPQA": 0.3456, - "hfopenllm_v2/MUSR": 0.4071, - "hfopenllm_v2/MMLU-PRO": 0.4224 - } - }, - { - "id": "zelk12/MT-Gen1-gemma-2-9B", - "name": "MT-Gen1-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7886, - "hfopenllm_v2/BBH": 0.61, - "hfopenllm_v2/MATH Level 5": 0.2221, - "hfopenllm_v2/GPQA": 0.3465, - "hfopenllm_v2/MUSR": 0.4217, - "hfopenllm_v2/MMLU-PRO": 0.4381 - } - }, - { - "id": "zelk12/MT-Gen2-gemma-2-9B", - "name": "MT-Gen2-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7907, - "hfopenllm_v2/BBH": 0.61, - "hfopenllm_v2/MATH Level 5": 0.219, - "hfopenllm_v2/GPQA": 0.3465, - "hfopenllm_v2/MUSR": 0.4323, - "hfopenllm_v2/MMLU-PRO": 0.4387 - } - }, - { - "id": "zelk12/MT-Gen2-GI-gemma-2-9B", - "name": "MT-Gen2-GI-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7914, - "hfopenllm_v2/BBH": 0.6096, - "hfopenllm_v2/MATH Level 5": 0.2205, - "hfopenllm_v2/GPQA": 0.3507, - "hfopenllm_v2/MUSR": 0.4283, - "hfopenllm_v2/MMLU-PRO": 0.4356 - } - }, - { - "id": "zelk12/MT-Gen3-gemma-2-9B", - "name": "MT-Gen3-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.802, - "hfopenllm_v2/BBH": 0.6097, - "hfopenllm_v2/MATH Level 5": 0.2296, - "hfopenllm_v2/GPQA": 0.349, - "hfopenllm_v2/MUSR": 0.4217, - "hfopenllm_v2/MMLU-PRO": 0.4356 - } - }, - { - "id": "zelk12/MT-Gen4-gemma-2-9B", - "name": "MT-Gen4-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7883, - "hfopenllm_v2/BBH": 0.611, - "hfopenllm_v2/MATH Level 5": 0.2236, - "hfopenllm_v2/GPQA": 0.3549, - "hfopenllm_v2/MUSR": 0.4228, - "hfopenllm_v2/MMLU-PRO": 0.4387 - } - }, - { - "id": "zelk12/MT-Gen5-gemma-2-9B", - "name": "MT-Gen5-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7923, - "hfopenllm_v2/BBH": 0.6133, - "hfopenllm_v2/MATH Level 5": 0.2153, - "hfopenllm_v2/GPQA": 0.3515, - "hfopenllm_v2/MUSR": 0.4202, - "hfopenllm_v2/MMLU-PRO": 0.4402 - } - }, - { - "id": "zelk12/MT-Gen6-gemma-2-9B", - "name": "MT-Gen6-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1616, - "hfopenllm_v2/BBH": 0.5845, - "hfopenllm_v2/MATH Level 5": 0.0823, - "hfopenllm_v2/GPQA": 0.3331, - "hfopenllm_v2/MUSR": 0.4069, - "hfopenllm_v2/MMLU-PRO": 0.4166 - } - }, - { - "id": "zelk12/MT-Gen6fix-gemma-2-9B", - "name": "MT-Gen6fix-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1576, - "hfopenllm_v2/BBH": 0.5917, - "hfopenllm_v2/MATH Level 5": 0.0816, - "hfopenllm_v2/GPQA": 0.3372, - "hfopenllm_v2/MUSR": 0.4084, - "hfopenllm_v2/MMLU-PRO": 0.412 - } - }, - { - "id": "zelk12/MT-Gen7-gemma-2-9B", - "name": "MT-Gen7-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1664, - "hfopenllm_v2/BBH": 0.5935, - "hfopenllm_v2/MATH Level 5": 0.0891, - "hfopenllm_v2/GPQA": 0.3356, - "hfopenllm_v2/MUSR": 0.4098, - "hfopenllm_v2/MMLU-PRO": 0.4122 - } - }, - { - "id": "zelk12/MT-Max-Merge_02012025163610-gemma-2-9B", - "name": "MT-Max-Merge_02012025163610-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7907, - "hfopenllm_v2/BBH": 0.6142, - "hfopenllm_v2/MATH Level 5": 0.2213, - "hfopenllm_v2/GPQA": 0.3515, - "hfopenllm_v2/MUSR": 0.4228, - "hfopenllm_v2/MMLU-PRO": 0.4396 - } - }, - { - "id": "zelk12/MT-Merge-gemma-2-9B", - "name": "MT-Merge-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8035, - "hfopenllm_v2/BBH": 0.6118, - "hfopenllm_v2/MATH Level 5": 0.2205, - "hfopenllm_v2/GPQA": 0.3482, - "hfopenllm_v2/MUSR": 0.4256, - "hfopenllm_v2/MMLU-PRO": 0.4362 - } - }, - { - "id": "zelk12/MT-Merge1-gemma-2-9B", - "name": "MT-Merge1-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7901, - "hfopenllm_v2/BBH": 0.61, - "hfopenllm_v2/MATH Level 5": 0.2289, - "hfopenllm_v2/GPQA": 0.3515, - "hfopenllm_v2/MUSR": 0.4244, - "hfopenllm_v2/MMLU-PRO": 0.4374 - } - }, - { - "id": "zelk12/MT-Merge2-gemma-2-9B", - "name": "MT-Merge2-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7877, - "hfopenllm_v2/BBH": 0.6107, - "hfopenllm_v2/MATH Level 5": 0.2349, - "hfopenllm_v2/GPQA": 0.3507, - "hfopenllm_v2/MUSR": 0.4217, - "hfopenllm_v2/MMLU-PRO": 0.4382 - } - }, - { - "id": "zelk12/MT-Merge2-MU-gemma-2-MTg2MT1g2-9B", - "name": "MT-Merge2-MU-gemma-2-MTg2MT1g2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7956, - "hfopenllm_v2/BBH": 0.6084, - "hfopenllm_v2/MATH Level 5": 0.2183, - "hfopenllm_v2/GPQA": 0.3507, - "hfopenllm_v2/MUSR": 0.4322, - "hfopenllm_v2/MMLU-PRO": 0.4373 - } - }, - { - "id": "zelk12/MT-Merge3-gemma-2-9B", - "name": "MT-Merge3-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7859, - "hfopenllm_v2/BBH": 0.6102, - "hfopenllm_v2/MATH Level 5": 0.2205, - "hfopenllm_v2/GPQA": 0.349, - "hfopenllm_v2/MUSR": 0.4258, - "hfopenllm_v2/MMLU-PRO": 0.4373 - } - }, - { - "id": "zelk12/MT-Merge4-gemma-2-9B", - "name": "MT-Merge4-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7807, - "hfopenllm_v2/BBH": 0.6118, - "hfopenllm_v2/MATH Level 5": 0.2168, - "hfopenllm_v2/GPQA": 0.3523, - "hfopenllm_v2/MUSR": 0.4294, - "hfopenllm_v2/MMLU-PRO": 0.439 - } - }, - { - "id": "zelk12/MT-Merge5-gemma-2-9B", - "name": "MT-Merge5-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7844, - "hfopenllm_v2/BBH": 0.6123, - "hfopenllm_v2/MATH Level 5": 0.2183, - "hfopenllm_v2/GPQA": 0.3532, - "hfopenllm_v2/MUSR": 0.4281, - "hfopenllm_v2/MMLU-PRO": 0.4387 - } - }, - { - "id": "zelk12/MT-Merge6-gemma-2-9B", - "name": "MT-Merge6-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1695, - "hfopenllm_v2/BBH": 0.5949, - "hfopenllm_v2/MATH Level 5": 0.0801, - "hfopenllm_v2/GPQA": 0.3289, - "hfopenllm_v2/MUSR": 0.4098, - "hfopenllm_v2/MMLU-PRO": 0.4115 - } - }, - { - "id": "zelk12/MT1-gemma-2-9B", - "name": "MT1-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7947, - "hfopenllm_v2/BBH": 0.6109, - "hfopenllm_v2/MATH Level 5": 0.2236, - "hfopenllm_v2/GPQA": 0.3456, - "hfopenllm_v2/MUSR": 0.4322, - "hfopenllm_v2/MMLU-PRO": 0.4358 - } - }, - { - "id": "zelk12/MT1-Gen1-gemma-2-9B", - "name": "MT1-Gen1-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7974, - "hfopenllm_v2/BBH": 0.6118, - "hfopenllm_v2/MATH Level 5": 0.2243, - "hfopenllm_v2/GPQA": 0.344, - "hfopenllm_v2/MUSR": 0.431, - "hfopenllm_v2/MMLU-PRO": 0.4376 - } - }, - { - "id": "zelk12/MT1-Gen2-gemma-2-9B", - "name": "MT1-Gen2-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7984, - "hfopenllm_v2/BBH": 0.6096, - "hfopenllm_v2/MATH Level 5": 0.2251, - "hfopenllm_v2/GPQA": 0.3523, - "hfopenllm_v2/MUSR": 0.4284, - "hfopenllm_v2/MMLU-PRO": 0.4355 - } - }, - { - "id": "zelk12/MT1-Gen3-gemma-2-9B", - "name": "MT1-Gen3-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.796, - "hfopenllm_v2/BBH": 0.6102, - "hfopenllm_v2/MATH Level 5": 0.2243, - "hfopenllm_v2/GPQA": 0.349, - "hfopenllm_v2/MUSR": 0.4243, - "hfopenllm_v2/MMLU-PRO": 0.4349 - } - }, - { - "id": "zelk12/MT1-Gen4-gemma-2-9B", - "name": "MT1-Gen4-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7941, - "hfopenllm_v2/BBH": 0.6058, - "hfopenllm_v2/MATH Level 5": 0.216, - "hfopenllm_v2/GPQA": 0.3473, - "hfopenllm_v2/MUSR": 0.4231, - "hfopenllm_v2/MMLU-PRO": 0.4286 - } - }, - { - "id": "zelk12/MT1-Gen5-gemma-2-9B", - "name": "MT1-Gen5-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7795, - "hfopenllm_v2/BBH": 0.6017, - "hfopenllm_v2/MATH Level 5": 0.2077, - "hfopenllm_v2/GPQA": 0.3465, - "hfopenllm_v2/MUSR": 0.4191, - "hfopenllm_v2/MMLU-PRO": 0.4222 - } - }, - { - "id": "zelk12/MT1-Gen5-IF-gemma-2-S2DMv1-9B", - "name": "MT1-Gen5-IF-gemma-2-S2DMv1-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7929, - "hfopenllm_v2/BBH": 0.6, - "hfopenllm_v2/MATH Level 5": 0.2032, - "hfopenllm_v2/GPQA": 0.344, - "hfopenllm_v2/MUSR": 0.4245, - "hfopenllm_v2/MMLU-PRO": 0.4218 - } - }, - { - "id": "zelk12/MT1-Gen6-gemma-2-9B", - "name": "MT1-Gen6-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1634, - "hfopenllm_v2/BBH": 0.5944, - "hfopenllm_v2/MATH Level 5": 0.0808, - "hfopenllm_v2/GPQA": 0.328, - "hfopenllm_v2/MUSR": 0.4044, - "hfopenllm_v2/MMLU-PRO": 0.4133 - } - }, - { - "id": "zelk12/MT1-Gen7-gemma-2-9B", - "name": "MT1-Gen7-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1634, - "hfopenllm_v2/BBH": 0.5938, - "hfopenllm_v2/MATH Level 5": 0.0831, - "hfopenllm_v2/GPQA": 0.328, - "hfopenllm_v2/MUSR": 0.4111, - "hfopenllm_v2/MMLU-PRO": 0.4145 - } - }, - { - "id": "zelk12/MT1-Max-Merge_02012025163610-gemma-2-9B", - "name": "MT1-Max-Merge_02012025163610-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7929, - "hfopenllm_v2/BBH": 0.6123, - "hfopenllm_v2/MATH Level 5": 0.2228, - "hfopenllm_v2/GPQA": 0.3549, - "hfopenllm_v2/MUSR": 0.4255, - "hfopenllm_v2/MMLU-PRO": 0.4382 - } - }, - { - "id": "zelk12/MT2-gemma-2-9B", - "name": "MT2-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7886, - "hfopenllm_v2/BBH": 0.6115, - "hfopenllm_v2/MATH Level 5": 0.2213, - "hfopenllm_v2/GPQA": 0.3473, - "hfopenllm_v2/MUSR": 0.4217, - "hfopenllm_v2/MMLU-PRO": 0.4368 - } - }, - { - "id": "zelk12/MT2-Gen1-gemma-2-9B", - "name": "MT2-Gen1-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7856, - "hfopenllm_v2/BBH": 0.6101, - "hfopenllm_v2/MATH Level 5": 0.2213, - "hfopenllm_v2/GPQA": 0.3431, - "hfopenllm_v2/MUSR": 0.4243, - "hfopenllm_v2/MMLU-PRO": 0.4377 - } - }, - { - "id": "zelk12/MT2-Gen2-gemma-2-9B", - "name": "MT2-Gen2-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7889, - "hfopenllm_v2/BBH": 0.6093, - "hfopenllm_v2/MATH Level 5": 0.2183, - "hfopenllm_v2/GPQA": 0.3465, - "hfopenllm_v2/MUSR": 0.427, - "hfopenllm_v2/MMLU-PRO": 0.4388 - } - }, - { - "id": "zelk12/MT2-Gen3-gemma-2-9B", - "name": "MT2-Gen3-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.781, - "hfopenllm_v2/BBH": 0.6105, - "hfopenllm_v2/MATH Level 5": 0.2107, - "hfopenllm_v2/GPQA": 0.3465, - "hfopenllm_v2/MUSR": 0.4231, - "hfopenllm_v2/MMLU-PRO": 0.4374 - } - }, - { - "id": "zelk12/MT2-Gen4-gemma-2-9B", - "name": "MT2-Gen4-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7896, - "hfopenllm_v2/BBH": 0.6097, - "hfopenllm_v2/MATH Level 5": 0.2236, - "hfopenllm_v2/GPQA": 0.3456, - "hfopenllm_v2/MUSR": 0.4125, - "hfopenllm_v2/MMLU-PRO": 0.4321 - } - }, - { - "id": "zelk12/MT2-Gen5-gemma-2-9B", - "name": "MT2-Gen5-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7749, - "hfopenllm_v2/BBH": 0.6064, - "hfopenllm_v2/MATH Level 5": 0.2107, - "hfopenllm_v2/GPQA": 0.3515, - "hfopenllm_v2/MUSR": 0.4244, - "hfopenllm_v2/MMLU-PRO": 0.4302 - } - }, - { - "id": "zelk12/MT2-Gen6-gemma-2-9B", - "name": "MT2-Gen6-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1664, - "hfopenllm_v2/BBH": 0.596, - "hfopenllm_v2/MATH Level 5": 0.0846, - "hfopenllm_v2/GPQA": 0.3381, - "hfopenllm_v2/MUSR": 0.4137, - "hfopenllm_v2/MMLU-PRO": 0.421 - } - }, - { - "id": "zelk12/MT2-Gen7-gemma-2-9B", - "name": "MT2-Gen7-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1762, - "hfopenllm_v2/BBH": 0.6079, - "hfopenllm_v2/MATH Level 5": 0.102, - "hfopenllm_v2/GPQA": 0.3549, - "hfopenllm_v2/MUSR": 0.4203, - "hfopenllm_v2/MMLU-PRO": 0.4311 - } - }, - { - "id": "zelk12/MT2-Max-Merge_02012025163610-gemma-2-9B", - "name": "MT2-Max-Merge_02012025163610-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7901, - "hfopenllm_v2/BBH": 0.6108, - "hfopenllm_v2/MATH Level 5": 0.2243, - "hfopenllm_v2/GPQA": 0.3515, - "hfopenllm_v2/MUSR": 0.4228, - "hfopenllm_v2/MMLU-PRO": 0.4391 - } - }, - { - "id": "zelk12/MT3-gemma-2-9B", - "name": "MT3-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7786, - "hfopenllm_v2/BBH": 0.6131, - "hfopenllm_v2/MATH Level 5": 0.2168, - "hfopenllm_v2/GPQA": 0.3448, - "hfopenllm_v2/MUSR": 0.4243, - "hfopenllm_v2/MMLU-PRO": 0.4327 - } - }, - { - "id": "zelk12/MT3-Gen1-gemma-2-9B", - "name": "MT3-Gen1-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7838, - "hfopenllm_v2/BBH": 0.6107, - "hfopenllm_v2/MATH Level 5": 0.2145, - "hfopenllm_v2/GPQA": 0.3465, - "hfopenllm_v2/MUSR": 0.4151, - "hfopenllm_v2/MMLU-PRO": 0.4327 - } - }, - { - "id": "zelk12/MT3-Gen2-gemma-2-9B", - "name": "MT3-Gen2-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7843, - "hfopenllm_v2/BBH": 0.6091, - "hfopenllm_v2/MATH Level 5": 0.2236, - "hfopenllm_v2/GPQA": 0.3574, - "hfopenllm_v2/MUSR": 0.4111, - "hfopenllm_v2/MMLU-PRO": 0.4333 - } - }, - { - "id": "zelk12/MT3-Gen3-gemma-2-9B", - "name": "MT3-Gen3-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7856, - "hfopenllm_v2/BBH": 0.6089, - "hfopenllm_v2/MATH Level 5": 0.2153, - "hfopenllm_v2/GPQA": 0.3515, - "hfopenllm_v2/MUSR": 0.4258, - "hfopenllm_v2/MMLU-PRO": 0.4303 - } - }, - { - "id": "zelk12/MT3-Gen4-gemma-2-9B", - "name": "MT3-Gen4-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7737, - "hfopenllm_v2/BBH": 0.6101, - "hfopenllm_v2/MATH Level 5": 0.2062, - "hfopenllm_v2/GPQA": 0.3473, - "hfopenllm_v2/MUSR": 0.4476, - "hfopenllm_v2/MMLU-PRO": 0.4387 - } - }, - { - "id": "zelk12/MT3-Gen5-gemma-2-9B", - "name": "MT3-Gen5-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.799, - "hfopenllm_v2/BBH": 0.6099, - "hfopenllm_v2/MATH Level 5": 0.2266, - "hfopenllm_v2/GPQA": 0.3532, - "hfopenllm_v2/MUSR": 0.4191, - "hfopenllm_v2/MMLU-PRO": 0.4317 - } - }, - { - "id": "zelk12/MT3-Gen5-gemma-2-9B_v1", - "name": "MT3-Gen5-gemma-2-9B_v1", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7996, - "hfopenllm_v2/BBH": 0.6113, - "hfopenllm_v2/MATH Level 5": 0.2228, - "hfopenllm_v2/GPQA": 0.349, - "hfopenllm_v2/MUSR": 0.4204, - "hfopenllm_v2/MMLU-PRO": 0.4359 - } - }, - { - "id": "zelk12/MT3-Gen6-gemma-2-9B", - "name": "MT3-Gen6-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1762, - "hfopenllm_v2/BBH": 0.602, - "hfopenllm_v2/MATH Level 5": 0.0884, - "hfopenllm_v2/GPQA": 0.3431, - "hfopenllm_v2/MUSR": 0.4126, - "hfopenllm_v2/MMLU-PRO": 0.4102 - } - }, - { - "id": "zelk12/MT3-Max-Merge_02012025163610-gemma-2-9B", - "name": "MT3-Max-Merge_02012025163610-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1762, - "hfopenllm_v2/BBH": 0.6123, - "hfopenllm_v2/MATH Level 5": 0.1012, - "hfopenllm_v2/GPQA": 0.3507, - "hfopenllm_v2/MUSR": 0.4255, - "hfopenllm_v2/MMLU-PRO": 0.4389 - } - }, - { - "id": "zelk12/MT4-gemma-2-9B", - "name": "MT4-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7762, - "hfopenllm_v2/BBH": 0.6073, - "hfopenllm_v2/MATH Level 5": 0.2085, - "hfopenllm_v2/GPQA": 0.3381, - "hfopenllm_v2/MUSR": 0.4309, - "hfopenllm_v2/MMLU-PRO": 0.4366 - } - }, - { - "id": "zelk12/MT4-Gen1-gemma-2-9B", - "name": "MT4-Gen1-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7895, - "hfopenllm_v2/BBH": 0.6094, - "hfopenllm_v2/MATH Level 5": 0.2198, - "hfopenllm_v2/GPQA": 0.344, - "hfopenllm_v2/MUSR": 0.4322, - "hfopenllm_v2/MMLU-PRO": 0.4389 - } - }, - { - "id": "zelk12/MT4-Gen2-gemma-2-9B", - "name": "MT4-Gen2-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8051, - "hfopenllm_v2/BBH": 0.6108, - "hfopenllm_v2/MATH Level 5": 0.2326, - "hfopenllm_v2/GPQA": 0.3456, - "hfopenllm_v2/MUSR": 0.4257, - "hfopenllm_v2/MMLU-PRO": 0.4368 - } - }, - { - "id": "zelk12/MT4-Gen3-gemma-2-9B", - "name": "MT4-Gen3-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7841, - "hfopenllm_v2/BBH": 0.6087, - "hfopenllm_v2/MATH Level 5": 0.219, - "hfopenllm_v2/GPQA": 0.344, - "hfopenllm_v2/MUSR": 0.4243, - "hfopenllm_v2/MMLU-PRO": 0.4381 - } - }, - { - "id": "zelk12/MT4-Gen4-gemma-2-9B", - "name": "MT4-Gen4-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7874, - "hfopenllm_v2/BBH": 0.6076, - "hfopenllm_v2/MATH Level 5": 0.2145, - "hfopenllm_v2/GPQA": 0.3523, - "hfopenllm_v2/MUSR": 0.4244, - "hfopenllm_v2/MMLU-PRO": 0.4323 - } - }, - { - "id": "zelk12/MT4-Gen5-gemma-2-9B", - "name": "MT4-Gen5-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7789, - "hfopenllm_v2/BBH": 0.6107, - "hfopenllm_v2/MATH Level 5": 0.2266, - "hfopenllm_v2/GPQA": 0.3565, - "hfopenllm_v2/MUSR": 0.4268, - "hfopenllm_v2/MMLU-PRO": 0.4384 - } - }, - { - "id": "zelk12/MT4-Max-Merge_02012025163610-gemma-2-9B", - "name": "MT4-Max-Merge_02012025163610-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1771, - "hfopenllm_v2/BBH": 0.612, - "hfopenllm_v2/MATH Level 5": 0.0952, - "hfopenllm_v2/GPQA": 0.3515, - "hfopenllm_v2/MUSR": 0.4228, - "hfopenllm_v2/MMLU-PRO": 0.4391 - } - }, - { - "id": "zelk12/MT5-gemma-2-9B", - "name": "MT5-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8048, - "hfopenllm_v2/BBH": 0.6112, - "hfopenllm_v2/MATH Level 5": 0.2258, - "hfopenllm_v2/GPQA": 0.3431, - "hfopenllm_v2/MUSR": 0.4204, - "hfopenllm_v2/MMLU-PRO": 0.4367 - } - }, - { - "id": "zelk12/MT5-Gen1-gemma-2-9B", - "name": "MT5-Gen1-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7831, - "hfopenllm_v2/BBH": 0.611, - "hfopenllm_v2/MATH Level 5": 0.2213, - "hfopenllm_v2/GPQA": 0.3473, - "hfopenllm_v2/MUSR": 0.4204, - "hfopenllm_v2/MMLU-PRO": 0.4368 - } - }, - { - "id": "zelk12/MT5-Gen2-gemma-2-9B", - "name": "MT5-Gen2-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7962, - "hfopenllm_v2/BBH": 0.6105, - "hfopenllm_v2/MATH Level 5": 0.2205, - "hfopenllm_v2/GPQA": 0.3515, - "hfopenllm_v2/MUSR": 0.4163, - "hfopenllm_v2/MMLU-PRO": 0.4379 - } - }, - { - "id": "zelk12/MT5-Gen3-gemma-2-9B", - "name": "MT5-Gen3-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7825, - "hfopenllm_v2/BBH": 0.609, - "hfopenllm_v2/MATH Level 5": 0.2168, - "hfopenllm_v2/GPQA": 0.3515, - "hfopenllm_v2/MUSR": 0.4231, - "hfopenllm_v2/MMLU-PRO": 0.4375 - } - }, - { - "id": "zelk12/MT5-Gen4-gemma-2-9B", - "name": "MT5-Gen4-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7835, - "hfopenllm_v2/BBH": 0.6131, - "hfopenllm_v2/MATH Level 5": 0.2243, - "hfopenllm_v2/GPQA": 0.3532, - "hfopenllm_v2/MUSR": 0.4228, - "hfopenllm_v2/MMLU-PRO": 0.4397 - } - }, - { - "id": "zelk12/MT5-Gen5-gemma-2-9B", - "name": "MT5-Gen5-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7947, - "hfopenllm_v2/BBH": 0.6112, - "hfopenllm_v2/MATH Level 5": 0.2258, - "hfopenllm_v2/GPQA": 0.3482, - "hfopenllm_v2/MUSR": 0.4191, - "hfopenllm_v2/MMLU-PRO": 0.4329 - } - }, - { - "id": "zelk12/MT5-Max-Merge_02012025163610-gemma-2-9B", - "name": "MT5-Max-Merge_02012025163610-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1762, - "hfopenllm_v2/BBH": 0.6127, - "hfopenllm_v2/MATH Level 5": 0.0982, - "hfopenllm_v2/GPQA": 0.3515, - "hfopenllm_v2/MUSR": 0.4228, - "hfopenllm_v2/MMLU-PRO": 0.439 - } - }, - { - "id": "zelk12/MTM-Merge-gemma-2-9B", - "name": "MTM-Merge-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7798, - "hfopenllm_v2/BBH": 0.6133, - "hfopenllm_v2/MATH Level 5": 0.2175, - "hfopenllm_v2/GPQA": 0.3549, - "hfopenllm_v2/MUSR": 0.4268, - "hfopenllm_v2/MMLU-PRO": 0.4388 - } - }, - { - "id": "zelk12/MTMaMe-Merge_02012025163610-gemma-2-9B", - "name": "MTMaMe-Merge_02012025163610-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1786, - "hfopenllm_v2/BBH": 0.6117, - "hfopenllm_v2/MATH Level 5": 0.0959, - "hfopenllm_v2/GPQA": 0.3523, - "hfopenllm_v2/MUSR": 0.4241, - "hfopenllm_v2/MMLU-PRO": 0.4382 - } - }, - { - "id": "zelk12/recoilme-gemma-2-Ataraxy-9B-v0.1", - "name": "recoilme-gemma-2-Ataraxy-9B-v0.1", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7649, - "hfopenllm_v2/BBH": 0.6075, - "hfopenllm_v2/MATH Level 5": 0.2281, - "hfopenllm_v2/GPQA": 0.3498, - "hfopenllm_v2/MUSR": 0.4136, - "hfopenllm_v2/MMLU-PRO": 0.4321 - } - }, - { - "id": "zelk12/recoilme-gemma-2-Ataraxy-9B-v0.1-t0.25", - "name": "recoilme-gemma-2-Ataraxy-9B-v0.1-t0.25", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7707, - "hfopenllm_v2/BBH": 0.6075, - "hfopenllm_v2/MATH Level 5": 0.2145, - "hfopenllm_v2/GPQA": 0.3431, - "hfopenllm_v2/MUSR": 0.4323, - "hfopenllm_v2/MMLU-PRO": 0.44 - } - }, - { - "id": "zelk12/recoilme-gemma-2-Ataraxy-9B-v0.1-t0.75", - "name": "recoilme-gemma-2-Ataraxy-9B-v0.1-t0.75", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7208, - "hfopenllm_v2/BBH": 0.5995, - "hfopenllm_v2/MATH Level 5": 0.2017, - "hfopenllm_v2/GPQA": 0.3498, - "hfopenllm_v2/MUSR": 0.3951, - "hfopenllm_v2/MMLU-PRO": 0.4141 - } - }, - { - "id": "zelk12/recoilme-gemma-2-Ataraxy-9B-v0.2", - "name": "recoilme-gemma-2-Ataraxy-9B-v0.2", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.76, - "hfopenllm_v2/BBH": 0.6066, - "hfopenllm_v2/MATH Level 5": 0.2228, - "hfopenllm_v2/GPQA": 0.3482, - "hfopenllm_v2/MUSR": 0.411, - "hfopenllm_v2/MMLU-PRO": 0.4323 - } - }, - { - "id": "zelk12/recoilme-gemma-2-Gutenberg-Doppel-9B-v0.1", - "name": "recoilme-gemma-2-Gutenberg-Doppel-9B-v0.1", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7615, - "hfopenllm_v2/BBH": 0.6099, - "hfopenllm_v2/MATH Level 5": 0.21, - "hfopenllm_v2/GPQA": 0.3414, - "hfopenllm_v2/MUSR": 0.431, - "hfopenllm_v2/MMLU-PRO": 0.4315 - } - }, - { - "id": "zelk12/recoilme-gemma-2-Ifable-9B-v0.1", - "name": "recoilme-gemma-2-Ifable-9B-v0.1", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7944, - "hfopenllm_v2/BBH": 0.6064, - "hfopenllm_v2/MATH Level 5": 0.2205, - "hfopenllm_v2/GPQA": 0.3515, - "hfopenllm_v2/MUSR": 0.4202, - "hfopenllm_v2/MMLU-PRO": 0.4323 - } - }, - { - "id": "zelk12/recoilme-gemma-2-psy10k-mental_healt-9B-v0.1", - "name": "recoilme-gemma-2-psy10k-mental_healt-9B-v0.1", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7445, - "hfopenllm_v2/BBH": 0.5978, - "hfopenllm_v2/MATH Level 5": 0.1888, - "hfopenllm_v2/GPQA": 0.344, - "hfopenllm_v2/MUSR": 0.4295, - "hfopenllm_v2/MMLU-PRO": 0.4181 - } - }, - { - "id": "zelk12/Rv0.4DMv1t0.25-gemma-2-9B", - "name": "Rv0.4DMv1t0.25-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7497, - "hfopenllm_v2/BBH": 0.607, - "hfopenllm_v2/MATH Level 5": 0.2258, - "hfopenllm_v2/GPQA": 0.3456, - "hfopenllm_v2/MUSR": 0.4309, - "hfopenllm_v2/MMLU-PRO": 0.4401 - } - }, - { - "id": "zelk12/Rv0.4DMv1t0.25Tt0.25-gemma-2-9B", - "name": "Rv0.4DMv1t0.25Tt0.25-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7646, - "hfopenllm_v2/BBH": 0.6098, - "hfopenllm_v2/MATH Level 5": 0.2069, - "hfopenllm_v2/GPQA": 0.3423, - "hfopenllm_v2/MUSR": 0.4283, - "hfopenllm_v2/MMLU-PRO": 0.4347 - } - }, - { - "id": "zelk12/Rv0.4MT4g2-gemma-2-9B", - "name": "Rv0.4MT4g2-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.732, - "hfopenllm_v2/BBH": 0.6041, - "hfopenllm_v2/MATH Level 5": 0.1949, - "hfopenllm_v2/GPQA": 0.3532, - "hfopenllm_v2/MUSR": 0.4231, - "hfopenllm_v2/MMLU-PRO": 0.4417 - } - }, - { - "id": "zelk12/T31122024203920-gemma-2-9B", - "name": "T31122024203920-gemma-2-9B", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7676, - "hfopenllm_v2/BBH": 0.6096, - "hfopenllm_v2/MATH Level 5": 0.2054, - "hfopenllm_v2/GPQA": 0.3507, - "hfopenllm_v2/MUSR": 0.4322, - "hfopenllm_v2/MMLU-PRO": 0.4373 - } - }, - { - "id": "zelk12/Test01012025155054", - "name": "Test01012025155054", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1555, - "hfopenllm_v2/BBH": 0.283, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2416, - "hfopenllm_v2/MUSR": 0.367, - "hfopenllm_v2/MMLU-PRO": 0.109 - } - }, - { - "id": "zelk12/Test01012025155054t0.5_gemma-2", - "name": "Test01012025155054t0.5_gemma-2", - "developer": "zelk12", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.1555, - "hfopenllm_v2/BBH": 0.283, - "hfopenllm_v2/MATH Level 5": 0.0, - "hfopenllm_v2/GPQA": 0.2416, - "hfopenllm_v2/MUSR": 0.367, - "hfopenllm_v2/MMLU-PRO": 0.109 - } - }, - { - "id": "ZeroXClem/L3-Aspire-Heart-Matrix-8B", - "name": "L3-Aspire-Heart-Matrix-8B", - "developer": "ZeroXClem", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4834, - "hfopenllm_v2/BBH": 0.5384, - "hfopenllm_v2/MATH Level 5": 0.1828, - "hfopenllm_v2/GPQA": 0.3247, - "hfopenllm_v2/MUSR": 0.4187, - "hfopenllm_v2/MMLU-PRO": 0.3785 - } - }, - { - "id": "ZeroXClem/Llama-3.1-8B-AthenaSky-MegaMix", - "name": "Llama-3.1-8B-AthenaSky-MegaMix", - "developer": "ZeroXClem", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6301, - "hfopenllm_v2/BBH": 0.5163, - "hfopenllm_v2/MATH Level 5": 0.2795, - "hfopenllm_v2/GPQA": 0.2777, - "hfopenllm_v2/MUSR": 0.3538, - "hfopenllm_v2/MMLU-PRO": 0.3504 - } - }, - { - "id": "ZeroXClem/Llama-3.1-8B-RainbowLight-EtherealMix", - "name": "Llama-3.1-8B-RainbowLight-EtherealMix", - "developer": "ZeroXClem", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4973, - "hfopenllm_v2/BBH": 0.5155, - "hfopenllm_v2/MATH Level 5": 0.1216, - "hfopenllm_v2/GPQA": 0.2869, - "hfopenllm_v2/MUSR": 0.3947, - "hfopenllm_v2/MMLU-PRO": 0.363 - } - }, - { - "id": "ZeroXClem/Llama-3.1-8B-SpecialTitanFusion", - "name": "Llama-3.1-8B-SpecialTitanFusion", - "developer": "ZeroXClem", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7402, - "hfopenllm_v2/BBH": 0.5439, - "hfopenllm_v2/MATH Level 5": 0.2334, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.3874, - "hfopenllm_v2/MMLU-PRO": 0.3621 - } - }, - { - "id": "ZeroXClem/Llama-3.1-8B-SuperNova-EtherealHermes", - "name": "Llama-3.1-8B-SuperNova-EtherealHermes", - "developer": "ZeroXClem", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7339, - "hfopenllm_v2/BBH": 0.5244, - "hfopenllm_v2/MATH Level 5": 0.1745, - "hfopenllm_v2/GPQA": 0.2928, - "hfopenllm_v2/MUSR": 0.4066, - "hfopenllm_v2/MMLU-PRO": 0.3745 - } - }, - { - "id": "ZeroXClem/Llama-3.1-8B-SuperTulu-LexiNova", - "name": "Llama-3.1-8B-SuperTulu-LexiNova", - "developer": "ZeroXClem", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4165, - "hfopenllm_v2/BBH": 0.5079, - "hfopenllm_v2/MATH Level 5": 0.253, - "hfopenllm_v2/GPQA": 0.2861, - "hfopenllm_v2/MUSR": 0.3971, - "hfopenllm_v2/MMLU-PRO": 0.3368 - } - }, - { - "id": "ZeroXClem/Qwen-2.5-Aether-SlerpFusion-7B", - "name": "Qwen-2.5-Aether-SlerpFusion-7B", - "developer": "ZeroXClem", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6262, - "hfopenllm_v2/BBH": 0.5462, - "hfopenllm_v2/MATH Level 5": 0.2734, - "hfopenllm_v2/GPQA": 0.2987, - "hfopenllm_v2/MUSR": 0.4178, - "hfopenllm_v2/MMLU-PRO": 0.4327 - } - }, - { - "id": "ZeroXClem/Qwen2.5-7B-CelestialHarmony-1M", - "name": "Qwen2.5-7B-CelestialHarmony-1M", - "developer": "ZeroXClem", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.5944, - "hfopenllm_v2/BBH": 0.5431, - "hfopenllm_v2/MATH Level 5": 0.3474, - "hfopenllm_v2/GPQA": 0.3188, - "hfopenllm_v2/MUSR": 0.4595, - "hfopenllm_v2/MMLU-PRO": 0.4387 - } - }, - { - "id": "ZeroXClem/Qwen2.5-7B-HomerAnvita-NerdMix", - "name": "Qwen2.5-7B-HomerAnvita-NerdMix", - "developer": "ZeroXClem", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7708, - "hfopenllm_v2/BBH": 0.5541, - "hfopenllm_v2/MATH Level 5": 0.3837, - "hfopenllm_v2/GPQA": 0.3196, - "hfopenllm_v2/MUSR": 0.4391, - "hfopenllm_v2/MMLU-PRO": 0.4432 - } - }, - { - "id": "ZeroXClem/Qwen2.5-7B-HomerCreative-Mix", - "name": "Qwen2.5-7B-HomerCreative-Mix", - "developer": "ZeroXClem", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7835, - "hfopenllm_v2/BBH": 0.5548, - "hfopenllm_v2/MATH Level 5": 0.3565, - "hfopenllm_v2/GPQA": 0.2995, - "hfopenllm_v2/MUSR": 0.435, - "hfopenllm_v2/MMLU-PRO": 0.4447 - } - }, - { - "id": "ZeroXClem/Qwen2.5-7B-Qandora-CySec", - "name": "Qwen2.5-7B-Qandora-CySec", - "developer": "ZeroXClem", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6773, - "hfopenllm_v2/BBH": 0.549, - "hfopenllm_v2/MATH Level 5": 0.2931, - "hfopenllm_v2/GPQA": 0.3003, - "hfopenllm_v2/MUSR": 0.4286, - "hfopenllm_v2/MMLU-PRO": 0.4485 - } - }, - { - "id": "zetasepic/Qwen2.5-32B-Instruct-abliterated-v2", - "name": "Qwen2.5-32B-Instruct-abliterated-v2", - "developer": "zetasepic", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.8334, - "hfopenllm_v2/BBH": 0.6934, - "hfopenllm_v2/MATH Level 5": 0.5952, - "hfopenllm_v2/GPQA": 0.3674, - "hfopenllm_v2/MUSR": 0.4354, - "hfopenllm_v2/MMLU-PRO": 0.5622 - } - }, - { - "id": "zetasepic/Qwen2.5-72B-Instruct-abliterated", - "name": "Qwen2.5-72B-Instruct-abliterated", - "developer": "zetasepic", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7153, - "hfopenllm_v2/BBH": 0.7152, - "hfopenllm_v2/MATH Level 5": 0.5242, - "hfopenllm_v2/GPQA": 0.4069, - "hfopenllm_v2/MUSR": 0.4719, - "hfopenllm_v2/MMLU-PRO": 0.5872 - } - }, - { - "id": "ZeusLabs/L3-Aethora-15B-V2", - "name": "L3-Aethora-15B-V2", - "developer": "ZeusLabs", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.7208, - "hfopenllm_v2/BBH": 0.5011, - "hfopenllm_v2/MATH Level 5": 0.0808, - "hfopenllm_v2/GPQA": 0.2878, - "hfopenllm_v2/MUSR": 0.3871, - "hfopenllm_v2/MMLU-PRO": 0.35 - } - }, - { - "id": "ZhangShenao/SELM-Llama-3-8B-Instruct-iter-3", - "name": "SELM-Llama-3-8B-Instruct-iter-3", - "developer": "ZhangShenao", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.6903, - "hfopenllm_v2/BBH": 0.5046, - "hfopenllm_v2/MATH Level 5": 0.0861, - "hfopenllm_v2/GPQA": 0.2584, - "hfopenllm_v2/MUSR": 0.3845, - "hfopenllm_v2/MMLU-PRO": 0.3783 - } - }, - { - "id": "zhengr/MixTAO-7Bx2-MoE-v8.1", - "name": "MixTAO-7Bx2-MoE-v8.1", - "developer": "zhengr", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.4188, - "hfopenllm_v2/BBH": 0.4202, - "hfopenllm_v2/MATH Level 5": 0.0604, - "hfopenllm_v2/GPQA": 0.2987, - "hfopenllm_v2/MUSR": 0.3976, - "hfopenllm_v2/MMLU-PRO": 0.2847 - } - }, - { - "id": "zhipu-ai/GLM-130B", - "name": "GLM 130B", - "developer": "zhipu-ai", - "evaluator_relationship": null, - "benchmark_scores": { - "helm_classic/Mean win rate": 0.512, - "helm_classic/MMLU": 0.344, - "helm_classic/BoolQ": 0.784, - "helm_classic/NarrativeQA": 0.706, - "helm_classic/NaturalQuestions (open-book)": 0.642, - "helm_classic/QuAC": 0.272, - "helm_classic/HellaSwag": -1.0, - "helm_classic/OpenbookQA": -1.0, - "helm_classic/TruthfulQA": 0.218, - "helm_classic/MS MARCO (TREC)": -1.0, - "helm_classic/CNN/DailyMail": 0.154, - "helm_classic/XSUM": 0.132, - "helm_classic/IMDB": 0.955, - "helm_classic/CivilComments": 0.5, - "helm_classic/RAFT": 0.598 - } - }, - { - "id": "zhipu-ai/glm-4.6", - "name": "GLM 4.6", - "developer": "Z.ai", - "evaluator_relationship": null, - "benchmark_scores": { - "terminal-bench-2.0/terminal-bench-2.0": 24.5 - } - }, - { - "id": "zhipu-ai/glm-4.7", - "name": "GLM 4.7", - "developer": "Z-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "terminal-bench-2.0/terminal-bench-2.0": 33.3 - } - }, - { - "id": "zhipu-ai/glm-5", - "name": "GLM 5", - "developer": "Z-AI", - "evaluator_relationship": null, - "benchmark_scores": { - "terminal-bench-2.0/terminal-bench-2.0": 52.4 - } - }, - { - "id": "zhipu/GLM 4.6", - "name": "GLM 4.6", - "developer": "zhipu", - "evaluator_relationship": null, - "benchmark_scores": { - "apex-agents/Corporate Lawyer Mean Score": 0.196 - } - }, - { - "id": "zhipu/GLM 4.7", - "name": "GLM 4.7", - "developer": "zhipu", - "evaluator_relationship": null, - "benchmark_scores": { - "apex-agents/Corporate Lawyer Mean Score": 0.147 - } - }, - { - "id": "zhipu/glm-4-6-fc-thinking", - "name": "GLM-4.6 (FC thinking)", - "developer": "zhipu", - "evaluator_relationship": null, - "benchmark_scores": { - "bfcl/bfcl.overall.rank": 4.0, - "bfcl/bfcl.overall.overall_accuracy": 72.38, - "bfcl/bfcl.overall.total_cost_usd": 4.64, - "bfcl/bfcl.overall.latency_mean_s": 4.34, - "bfcl/bfcl.overall.latency_std_s": 7.22, - "bfcl/bfcl.overall.latency_p95_s": 13.5, - "bfcl/bfcl.non_live.ast_accuracy": 87.56, - "bfcl/bfcl.non_live.simple_ast_accuracy": 74.25, - "bfcl/bfcl.non_live.multiple_ast_accuracy": 95.0, - "bfcl/bfcl.non_live.parallel_ast_accuracy": 91.5, - "bfcl/bfcl.non_live.parallel_multiple_ast_accuracy": 89.5, - "bfcl/bfcl.live.live_accuracy": 80.9, - "bfcl/bfcl.live.live_simple_ast_accuracy": 89.53, - "bfcl/bfcl.live.live_multiple_ast_accuracy": 78.92, - "bfcl/bfcl.live.live_parallel_ast_accuracy": 81.25, - "bfcl/bfcl.live.live_parallel_multiple_ast_accuracy": 75.0, - "bfcl/bfcl.multi_turn.accuracy": 68.0, - "bfcl/bfcl.multi_turn.base_accuracy": 74.5, - "bfcl/bfcl.multi_turn.miss_function_accuracy": 68.0, - "bfcl/bfcl.multi_turn.miss_parameter_accuracy": 63.0, - "bfcl/bfcl.multi_turn.long_context_accuracy": 66.5, - "bfcl/bfcl.web_search.accuracy": 77.5, - "bfcl/bfcl.web_search.base_accuracy": 79.0, - "bfcl/bfcl.web_search.no_snippet_accuracy": 76.0, - "bfcl/bfcl.memory.accuracy": 55.7, - "bfcl/bfcl.memory.kv_accuracy": 43.87, - "bfcl/bfcl.memory.vector_accuracy": 56.13, - "bfcl/bfcl.memory.recursive_summarization_accuracy": 67.1, - "bfcl/bfcl.relevance.relevance_detection_accuracy": 75.0, - "bfcl/bfcl.relevance.irrelevance_detection_accuracy": 84.96 - } - }, - { - "id": "ZHLiu627/zephyr-7b-gemma-dpo-avg", - "name": "zephyr-7b-gemma-dpo-avg", - "developer": "ZHLiu627", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.309, - "hfopenllm_v2/BBH": 0.4149, - "hfopenllm_v2/MATH Level 5": 0.0453, - "hfopenllm_v2/GPQA": 0.2785, - "hfopenllm_v2/MUSR": 0.4107, - "hfopenllm_v2/MMLU-PRO": 0.2851 - } - }, - { - "id": "ZHLiu627/zephyr-7b-gemma-rpo-avg", - "name": "zephyr-7b-gemma-rpo-avg", - "developer": "ZHLiu627", - "evaluator_relationship": null, - "benchmark_scores": { - "hfopenllm_v2/IFEval": 0.3006, - "hfopenllm_v2/BBH": 0.4183, - "hfopenllm_v2/MATH Level 5": 0.0498, - "hfopenllm_v2/GPQA": 0.2768, - "hfopenllm_v2/MUSR": 0.4081, - "hfopenllm_v2/MMLU-PRO": 0.2831 - } - }, - { - "id": "ZiyiYe/Con-J-Qwen2-7B", - "name": "ZiyiYe/Con-J-Qwen2-7B", - "developer": "ZiyiYe", - "evaluator_relationship": null, - "benchmark_scores": { - "reward-bench/Score": 0.8712, - "reward-bench/Chat": 0.919, - "reward-bench/Chat Hard": 0.8026, - "reward-bench/Safety": 0.8824, - "reward-bench/Reasoning": 0.8808 - } - } -] \ No newline at end of file diff --git a/data/models/0-hero_matter-0.1-7b-boost-dpo-preview.json b/data/models/0-hero_matter-0.1-7b-boost-dpo-preview.json deleted file mode 100644 index 50ca458f9e30a70fe95b4a239344e99ac22ccf4e..0000000000000000000000000000000000000000 --- a/data/models/0-hero_matter-0.1-7b-boost-dpo-preview.json +++ /dev/null @@ -1,144 +0,0 @@ -{ - "model_info": { - "name": "0-hero/Matter-0.1-7B-boost-DPO-preview", - "id": "0-hero/Matter-0.1-7B-boost-DPO-preview", - "developer": "0-hero", - "additional_details": { - "model_type": "DPO" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/0-hero_Matter-0.1-7B-boost-DPO-preview/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7448 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9106 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6096 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7135 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8395 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5566 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/0-hero_matter-0.1-7b-dpo-preview.json b/data/models/0-hero_matter-0.1-7b-dpo-preview.json deleted file mode 100644 index 94a7f55d83b6997597d67e514ccb9e98e2e37553..0000000000000000000000000000000000000000 --- a/data/models/0-hero_matter-0.1-7b-dpo-preview.json +++ /dev/null @@ -1,144 +0,0 @@ -{ - "model_info": { - "name": "0-hero/Matter-0.1-7B-DPO-preview", - "id": "0-hero/Matter-0.1-7B-DPO-preview", - "developer": "0-hero", - "additional_details": { - "model_type": "DPO" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/0-hero_Matter-0.1-7B-DPO-preview/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7247 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8939 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5768 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6378 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8854 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5348 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/0-hero_matter-0.2-7b-dpo.json b/data/models/0-hero_matter-0.2-7b-dpo.json deleted file mode 100644 index 782411cb44d0c0613104b72df0f19387c232de71..0000000000000000000000000000000000000000 --- a/data/models/0-hero_matter-0.2-7b-dpo.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Matter-0.2-7B-DPO", - "id": "0-hero/Matter-0.2-7B-DPO", - "developer": "0-hero", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/0-hero_Matter-0.2-7B-DPO/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3303 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3596 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0144 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2592 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3814 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1164 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/01-ai_yi-1.5-34b-32k.json b/data/models/01-ai_yi-1.5-34b-32k.json deleted file mode 100644 index 95a06ae55c01b0077a6b5fa451c8813a58dad1d2..0000000000000000000000000000000000000000 --- a/data/models/01-ai_yi-1.5-34b-32k.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Yi-1.5-34B-32K", - "id": "01-ai/Yi-1.5-34B-32K", - "developer": "01-ai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "34.389" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/01-ai_Yi-1.5-34B-32K/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3119 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6016 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1541 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3633 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4398 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4709 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/01-ai_yi-1.5-34b-chat-16k.json b/data/models/01-ai_yi-1.5-34b-chat-16k.json deleted file mode 100644 index 1d8be5282395738e2924897e448d6c89ae245b67..0000000000000000000000000000000000000000 --- a/data/models/01-ai_yi-1.5-34b-chat-16k.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Yi-1.5-34B-Chat-16K", - "id": "01-ai/Yi-1.5-34B-Chat-16K", - "developer": "01-ai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "34.389" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/01-ai_Yi-1.5-34B-Chat-16K/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4564 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.61 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2137 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3381 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4398 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4545 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/01-ai_yi-1.5-34b-chat.json b/data/models/01-ai_yi-1.5-34b-chat.json deleted file mode 100644 index c3ffffc3d8dc2968fd9e041e235943da734195d1..0000000000000000000000000000000000000000 --- a/data/models/01-ai_yi-1.5-34b-chat.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Yi-1.5-34B-Chat", - "id": "01-ai/Yi-1.5-34B-Chat", - "developer": "01-ai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "34.389" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/01-ai_Yi-1.5-34B-Chat/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6067 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6084 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2772 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3649 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4282 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.452 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/01-ai_yi-1.5-34b.json b/data/models/01-ai_yi-1.5-34b.json deleted file mode 100644 index 4e0fd6f28c36d0c96bc8b52ff350b162f5432731..0000000000000000000000000000000000000000 --- a/data/models/01-ai_yi-1.5-34b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Yi-1.5-34B", - "id": "01-ai/Yi-1.5-34B", - "developer": "01-ai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "34.389" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/01-ai_Yi-1.5-34B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2841 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5976 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1533 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3658 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4236 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4666 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/01-ai_yi-1.5-6b-chat.json b/data/models/01-ai_yi-1.5-6b-chat.json deleted file mode 100644 index 7c974de166fd2f8916e9903e06ed8283913b447f..0000000000000000000000000000000000000000 --- a/data/models/01-ai_yi-1.5-6b-chat.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Yi-1.5-6B-Chat", - "id": "01-ai/Yi-1.5-6B-Chat", - "developer": "01-ai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "6.061" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/01-ai_Yi-1.5-6B-Chat/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5145 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4571 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1624 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.302 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4392 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3193 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/01-ai_yi-1.5-6b.json b/data/models/01-ai_yi-1.5-6b.json deleted file mode 100644 index d3b0dde6f88cdce55394e524a414da427ecae365..0000000000000000000000000000000000000000 --- a/data/models/01-ai_yi-1.5-6b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Yi-1.5-6B", - "id": "01-ai/Yi-1.5-6B", - "developer": "01-ai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "6.061" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/01-ai_Yi-1.5-6B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2617 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4493 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0665 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3138 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4374 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3144 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/01-ai_yi-1.5-9b-32k.json b/data/models/01-ai_yi-1.5-9b-32k.json deleted file mode 100644 index 6bcb1c6b0d658e4f2f644db79f96a136c7588667..0000000000000000000000000000000000000000 --- a/data/models/01-ai_yi-1.5-9b-32k.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Yi-1.5-9B-32K", - "id": "01-ai/Yi-1.5-9B-32K", - "developer": "01-ai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.829" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/01-ai_Yi-1.5-9B-32K/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2303 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4963 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.108 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3591 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4186 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3765 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/01-ai_yi-1.5-9b-chat-16k.json b/data/models/01-ai_yi-1.5-9b-chat-16k.json deleted file mode 100644 index 69da7c2abffeaf78ced06e20cd976f4a04f3734c..0000000000000000000000000000000000000000 --- a/data/models/01-ai_yi-1.5-9b-chat-16k.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Yi-1.5-9B-Chat-16K", - "id": "01-ai/Yi-1.5-9B-Chat-16K", - "developer": "01-ai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.829" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/01-ai_Yi-1.5-9B-Chat-16K/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4214 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5153 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1782 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3087 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4099 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3994 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/01-ai_yi-1.5-9b-chat.json b/data/models/01-ai_yi-1.5-9b-chat.json deleted file mode 100644 index 335ce67ef1cba4e127f8bb91f6c72d98f3633ede..0000000000000000000000000000000000000000 --- a/data/models/01-ai_yi-1.5-9b-chat.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Yi-1.5-9B-Chat", - "id": "01-ai/Yi-1.5-9B-Chat", - "developer": "01-ai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.829" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/01-ai_Yi-1.5-9B-Chat/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6046 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5559 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2258 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3347 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4259 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3975 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/01-ai_yi-1.5-9b.json b/data/models/01-ai_yi-1.5-9b.json deleted file mode 100644 index 9d27bcab6473744e8e720f68fe31a3fe4f390167..0000000000000000000000000000000000000000 --- a/data/models/01-ai_yi-1.5-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Yi-1.5-9B", - "id": "01-ai/Yi-1.5-9B", - "developer": "01-ai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.829" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/01-ai_Yi-1.5-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2936 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5143 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.114 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3792 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4328 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3916 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/01-ai_yi-34b-200k.json b/data/models/01-ai_yi-34b-200k.json deleted file mode 100644 index b54ed2d58bd6b4a43f7443af531c73a94cda8884..0000000000000000000000000000000000000000 --- a/data/models/01-ai_yi-34b-200k.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Yi-34B-200K", - "id": "01-ai/Yi-34B-200K", - "developer": "01-ai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "34.389" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/01-ai_Yi-34B-200K/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1542 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5442 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0574 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3565 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3817 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4535 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/01-ai_yi-34b-chat.json b/data/models/01-ai_yi-34b-chat.json deleted file mode 100644 index 046df60e640a5ea9e0a213f150ad021397b99464..0000000000000000000000000000000000000000 --- a/data/models/01-ai_yi-34b-chat.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Yi-34B-Chat", - "id": "01-ai/Yi-34B-Chat", - "developer": "01-ai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "34.389" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/01-ai_Yi-34B-Chat/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4699 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5561 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0627 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3381 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3978 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4093 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/01-ai_yi-34b.json b/data/models/01-ai_yi-34b.json deleted file mode 100644 index 4e2fed505ee2b9a4974da6ea8361969315c8ae01..0000000000000000000000000000000000000000 --- a/data/models/01-ai_yi-34b.json +++ /dev/null @@ -1,2036 +0,0 @@ -{ - "model_info": { - "name": "Yi 34B", - "id": "01-ai/yi-34b", - "developer": "01-ai", - "inference_platform": "unknown", - "additional_details": { - "model_id_aliases": [ - "01-ai/Yi-34B" - ] - } - }, - "evaluations": [ - { - "evaluation_id": "helm_lite/01-ai_yi-34b/1774096306.427425", - "retrieved_timestamp": "1774096306.427425", - "source_metadata": { - "source_name": "helm_lite", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_lite", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_lite", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.57, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.2681148564294632\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.782, - "details": { - "description": "min=0.782, mean=0.782, max=0.782, sum=0.782 (1)", - "tab": "Accuracy", - "NarrativeQA - Observed inference time (s)": "{\"description\": \"min=2.368, mean=2.368, max=2.368, sum=2.368 (1)\", \"tab\": \"Efficiency\", \"score\": \"2.368284817816506\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=4.868, mean=4.868, max=4.868, sum=4.868 (1)\", \"tab\": \"General information\", \"score\": \"4.867605633802817\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=3611.445, mean=3611.445, max=3611.445, sum=3611.445 (1)\", \"tab\": \"General information\", \"score\": \"3611.445070422535\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (closed-book)", - "source_data": { - "dataset_name": "NaturalQuestions (closed-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (closed-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.443, - "details": { - "description": "min=0.443, mean=0.443, max=0.443, sum=0.443 (1)", - "tab": "Accuracy", - "NaturalQuestions (open-book) - Observed inference time (s)": "{\"description\": \"min=1.816, mean=1.816, max=1.816, sum=1.816 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.8157690076828004\"}", - "NaturalQuestions (closed-book) - Observed inference time (s)": "{\"description\": \"min=1.458, mean=1.458, max=1.458, sum=1.458 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.4578230485916137\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.838, mean=4.838, max=4.838, sum=4.838 (1)\", \"tab\": \"General information\", \"score\": \"4.838\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.026, mean=0.026, max=0.026, sum=0.026 (1)\", \"tab\": \"General information\", \"score\": \"0.026\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=2171.698, mean=2171.698, max=2171.698, sum=2171.698 (1)\", \"tab\": \"General information\", \"score\": \"2171.698\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=0.995, mean=0.995, max=0.995, sum=0.995 (1)\", \"tab\": \"General information\", \"score\": \"0.995\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=131.695, mean=131.695, max=131.695, sum=131.695 (1)\", \"tab\": \"General information\", \"score\": \"131.695\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "mode": "\"closedbook\"" - } - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.92, - "details": { - "description": "min=0.92, mean=0.92, max=0.92, sum=0.92 (1)", - "tab": "Accuracy", - "OpenbookQA - Observed inference time (s)": "{\"description\": \"min=0.823, mean=0.823, max=0.823, sum=0.823 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.8229070715904235\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=260.002, mean=260.002, max=260.002, sum=260.002 (1)\", \"tab\": \"General information\", \"score\": \"260.002\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "dataset": "\"openbookqa\"", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.65, - "details": { - "description": "min=0.4, mean=0.65, max=0.91, sum=3.248 (5)", - "tab": "Accuracy", - "MMLU - Observed inference time (s)": "{\"description\": \"min=0.511, mean=0.697, max=0.925, sum=3.486 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.6972272023485417\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=383.67, mean=502.654, max=667.789, sum=2513.269 (5)\", \"tab\": \"General information\", \"score\": \"502.65389473684206\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"college_chemistry\", \"computer_security\", \"econometrics\", \"us_foreign_policy\"]", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MATH", - "source_data": { - "dataset_name": "MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Equivalent (CoT) on MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.375, - "details": { - "description": "min=0.167, mean=0.375, max=0.563, sum=2.623 (7)", - "tab": "Accuracy", - "MATH - Observed inference time (s)": "{\"description\": \"min=2.651, mean=3.809, max=4.649, sum=26.664 (7)\", \"tab\": \"Efficiency\", \"score\": \"3.809198633421\"}", - "MATH - # eval": "{\"description\": \"min=30, mean=62.429, max=135, sum=437 (7)\", \"tab\": \"General information\", \"score\": \"62.42857142857143\"}", - "MATH - # train": "{\"description\": \"min=8, mean=8, max=8, sum=56 (7)\", \"tab\": \"General information\", \"score\": \"8.0\"}", - "MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MATH - # prompt tokens": "{\"description\": \"min=976.696, mean=1468.935, max=2582.038, sum=10282.547 (7)\", \"tab\": \"General information\", \"score\": \"1468.9352369693863\"}", - "MATH - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=7 (7)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"algebra\", \"counting_and_probability\", \"geometry\", \"intermediate_algebra\", \"number_theory\", \"prealgebra\", \"precalculus\"]", - "level": "\"1\"", - "use_official_examples": "\"False\"", - "use_chain_of_thought": "\"True\"" - } - } - }, - { - "evaluation_name": "GSM8K", - "source_data": { - "dataset_name": "GSM8K", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on GSM8K", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.648, - "details": { - "description": "min=0.648, mean=0.648, max=0.648, sum=0.648 (1)", - "tab": "Accuracy", - "GSM8K - Observed inference time (s)": "{\"description\": \"min=4.887, mean=4.887, max=4.887, sum=4.887 (1)\", \"tab\": \"Efficiency\", \"score\": \"4.886563032150269\"}", - "GSM8K - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "GSM8K - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "GSM8K - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GSM8K - # prompt tokens": "{\"description\": \"min=1170.814, mean=1170.814, max=1170.814, sum=1170.814 (1)\", \"tab\": \"General information\", \"score\": \"1170.814\"}", - "GSM8K - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "LegalBench", - "source_data": { - "dataset_name": "LegalBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on LegalBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.618, - "details": { - "description": "min=0.311, mean=0.618, max=0.8, sum=3.089 (5)", - "tab": "Accuracy", - "LegalBench - Observed inference time (s)": "{\"description\": \"min=0.465, mean=0.8, max=1.207, sum=4.002 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.8004560962069804\"}", - "LegalBench - # eval": "{\"description\": \"min=95, mean=409.4, max=1000, sum=2047 (5)\", \"tab\": \"General information\", \"score\": \"409.4\"}", - "LegalBench - # train": "{\"description\": \"min=2, mean=4.2, max=5, sum=21 (5)\", \"tab\": \"General information\", \"score\": \"4.2\"}", - "LegalBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "LegalBench - # prompt tokens": "{\"description\": \"min=211.779, mean=951.524, max=3359.547, sum=4757.621 (5)\", \"tab\": \"General information\", \"score\": \"951.5242922438443\"}", - "LegalBench - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"abercrombie\", \"corporate_lobbying\", \"function_of_decision_section\", \"international_citizenship_questions\", \"proa\"]" - } - } - }, - { - "evaluation_name": "MedQA", - "source_data": { - "dataset_name": "MedQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MedQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.656, - "details": { - "description": "min=0.656, mean=0.656, max=0.656, sum=0.656 (1)", - "tab": "Accuracy", - "MedQA - Observed inference time (s)": "{\"description\": \"min=1.064, mean=1.064, max=1.064, sum=1.064 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.064007310696672\"}", - "MedQA - # eval": "{\"description\": \"min=503, mean=503, max=503, sum=503 (1)\", \"tab\": \"General information\", \"score\": \"503.0\"}", - "MedQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MedQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MedQA - # prompt tokens": "{\"description\": \"min=1122.392, mean=1122.392, max=1122.392, sum=1122.392 (1)\", \"tab\": \"General information\", \"score\": \"1122.3916500994035\"}", - "MedQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WMT 2014", - "source_data": { - "dataset_name": "WMT 2014", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "BLEU-4 on WMT 2014", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.172, - "details": { - "description": "min=0.1, mean=0.172, max=0.218, sum=0.858 (5)", - "tab": "Accuracy", - "WMT 2014 - Observed inference time (s)": "{\"description\": \"min=1.071, mean=1.404, max=2.506, sum=7.021 (5)\", \"tab\": \"Efficiency\", \"score\": \"1.4042062711970469\"}", - "WMT 2014 - # eval": "{\"description\": \"min=503, mean=568.8, max=832, sum=2844 (5)\", \"tab\": \"General information\", \"score\": \"568.8\"}", - "WMT 2014 - # train": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "WMT 2014 - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WMT 2014 - # prompt tokens": "{\"description\": \"min=139.298, mean=187.092, max=317.56, sum=935.461 (5)\", \"tab\": \"General information\", \"score\": \"187.09213851506345\"}", - "WMT 2014 - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "language_pair": "[\"cs-en\", \"de-en\", \"fr-en\", \"hi-en\", \"ru-en\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_mmlu/01-ai_yi-34b/1774096312.00548", - "retrieved_timestamp": "1774096312.00548", - "source_metadata": { - "source_name": "helm_mmlu", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_mmlu", - "evaluation_results": [ - { - "evaluation_name": "MMLU All Subjects", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU All Subjects", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.762, - "details": { - "description": "min=0.4, mean=0.762, max=0.974, sum=86.905 (114)", - "tab": "Accuracy", - "MMLU All Subjects - Observed inference time (s)": "{\"description\": \"min=0.407, mean=0.823, max=2.683, sum=93.841 (114)\", \"tab\": \"Efficiency\", \"score\": \"0.8231679963633336\"}", - "MMLU All Subjects - # eval": "{\"description\": \"min=100, mean=246.351, max=1534, sum=28084 (114)\", \"tab\": \"General information\", \"score\": \"246.35087719298247\"}", - "MMLU All Subjects - # train": "{\"description\": \"min=5, mean=5, max=5, sum=570 (114)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU All Subjects - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU All Subjects - # prompt tokens": "{\"description\": \"min=289.971, mean=661.842, max=2957.412, sum=75449.942 (114)\", \"tab\": \"General information\", \"score\": \"661.8416008681387\"}", - "MMLU All Subjects - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=114 (114)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_name": "Abstract Algebra", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Abstract Algebra", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4, - "details": { - "description": "min=0.4, mean=0.4, max=0.4, sum=0.8 (2)", - "tab": "Accuracy", - "Abstract Algebra - Observed inference time (s)": "{\"description\": \"min=0.658, mean=0.658, max=0.658, sum=1.315 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6577284264564515\"}", - "Abstract Algebra - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Abstract Algebra - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Abstract Algebra - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Abstract Algebra - # prompt tokens": "{\"description\": \"min=383.67, mean=383.67, max=383.67, sum=767.34 (2)\", \"tab\": \"General information\", \"score\": \"383.67\"}", - "Abstract Algebra - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"abstract_algebra\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_abstract_algebra\"" - } - } - }, - { - "evaluation_name": "Anatomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Anatomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.748, - "details": { - "description": "min=0.748, mean=0.748, max=0.748, sum=1.496 (2)", - "tab": "Accuracy", - "Anatomy - Observed inference time (s)": "{\"description\": \"min=0.601, mean=0.601, max=0.601, sum=1.202 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6009190011907507\"}", - "Anatomy - # eval": "{\"description\": \"min=135, mean=135, max=135, sum=270 (2)\", \"tab\": \"General information\", \"score\": \"135.0\"}", - "Anatomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Anatomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Anatomy - # prompt tokens": "{\"description\": \"min=375.77, mean=375.77, max=375.77, sum=751.541 (2)\", \"tab\": \"General information\", \"score\": \"375.77037037037036\"}", - "Anatomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"anatomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_anatomy\"" - } - } - }, - { - "evaluation_name": "College Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on College Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5, - "details": { - "description": "min=0.5, mean=0.5, max=0.5, sum=1 (2)", - "tab": "Accuracy", - "College Chemistry - Observed inference time (s)": "{\"description\": \"min=0.531, mean=0.531, max=0.531, sum=1.061 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5305842399597168\"}", - "College Biology - Observed inference time (s)": "{\"description\": \"min=0.502, mean=0.502, max=0.502, sum=1.004 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5021488202942742\"}", - "College Computer Science - Observed inference time (s)": "{\"description\": \"min=0.708, mean=0.708, max=0.708, sum=1.415 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7075318503379822\"}", - "College Mathematics - Observed inference time (s)": "{\"description\": \"min=0.569, mean=0.569, max=0.569, sum=1.138 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5689087891578675\"}", - "College Medicine - Observed inference time (s)": "{\"description\": \"min=0.575, mean=0.575, max=0.575, sum=1.15 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5747669638925894\"}", - "College Physics - Observed inference time (s)": "{\"description\": \"min=0.604, mean=0.604, max=0.604, sum=1.207 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.603668584543116\"}", - "College Chemistry - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Chemistry - # prompt tokens": "{\"description\": \"min=597.54, mean=597.54, max=597.54, sum=1195.08 (2)\", \"tab\": \"General information\", \"score\": \"597.54\"}", - "College Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Biology - # eval": "{\"description\": \"min=144, mean=144, max=144, sum=288 (2)\", \"tab\": \"General information\", \"score\": \"144.0\"}", - "College Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # prompt tokens": "{\"description\": \"min=514.819, mean=514.819, max=514.819, sum=1029.639 (2)\", \"tab\": \"General information\", \"score\": \"514.8194444444445\"}", - "College Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # prompt tokens": "{\"description\": \"min=883.06, mean=883.06, max=883.06, sum=1766.12 (2)\", \"tab\": \"General information\", \"score\": \"883.06\"}", - "College Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Mathematics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # prompt tokens": "{\"description\": \"min=635.3, mean=635.3, max=635.3, sum=1270.6 (2)\", \"tab\": \"General information\", \"score\": \"635.3\"}", - "College Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Medicine - # eval": "{\"description\": \"min=173, mean=173, max=173, sum=346 (2)\", \"tab\": \"General information\", \"score\": \"173.0\"}", - "College Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # prompt tokens": "{\"description\": \"min=549.688, mean=549.688, max=549.688, sum=1099.376 (2)\", \"tab\": \"General information\", \"score\": \"549.6878612716763\"}", - "College Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Physics - # eval": "{\"description\": \"min=102, mean=102, max=102, sum=204 (2)\", \"tab\": \"General information\", \"score\": \"102.0\"}", - "College Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # prompt tokens": "{\"description\": \"min=512.912, mean=512.912, max=512.912, sum=1025.824 (2)\", \"tab\": \"General information\", \"score\": \"512.9117647058823\"}", - "College Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"college_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_college_physics\"" - } - } - }, - { - "evaluation_name": "Computer Security", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Computer Security", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.83, - "details": { - "description": "min=0.83, mean=0.83, max=0.83, sum=1.66 (2)", - "tab": "Accuracy", - "Computer Security - Observed inference time (s)": "{\"description\": \"min=0.472, mean=0.472, max=0.472, sum=0.943 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.47160084009170533\"}", - "Computer Security - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Computer Security - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Computer Security - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Computer Security - # prompt tokens": "{\"description\": \"min=405.74, mean=405.74, max=405.74, sum=811.48 (2)\", \"tab\": \"General information\", \"score\": \"405.74\"}", - "Computer Security - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"computer_security\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_computer_security\"" - } - } - }, - { - "evaluation_name": "Econometrics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Econometrics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.588, - "details": { - "description": "min=0.588, mean=0.588, max=0.588, sum=1.175 (2)", - "tab": "Accuracy", - "Econometrics - Observed inference time (s)": "{\"description\": \"min=0.61, mean=0.61, max=0.61, sum=1.219 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6095903463530958\"}", - "Econometrics - # eval": "{\"description\": \"min=114, mean=114, max=114, sum=228 (2)\", \"tab\": \"General information\", \"score\": \"114.0\"}", - "Econometrics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Econometrics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Econometrics - # prompt tokens": "{\"description\": \"min=667.789, mean=667.789, max=667.789, sum=1335.579 (2)\", \"tab\": \"General information\", \"score\": \"667.7894736842105\"}", - "Econometrics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"econometrics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_econometrics\"" - } - } - }, - { - "evaluation_name": "Global Facts", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Global Facts", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.53, - "details": { - "description": "min=0.53, mean=0.53, max=0.53, sum=1.06 (2)", - "tab": "Accuracy", - "Global Facts - Observed inference time (s)": "{\"description\": \"min=0.537, mean=0.537, max=0.537, sum=1.074 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5369880175590516\"}", - "Global Facts - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Global Facts - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Global Facts - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Global Facts - # prompt tokens": "{\"description\": \"min=462.32, mean=462.32, max=462.32, sum=924.64 (2)\", \"tab\": \"General information\", \"score\": \"462.32\"}", - "Global Facts - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"global_facts\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_global_facts\"" - } - } - }, - { - "evaluation_name": "Jurisprudence", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Jurisprudence", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.898, - "details": { - "description": "min=0.898, mean=0.898, max=0.898, sum=1.796 (2)", - "tab": "Accuracy", - "Jurisprudence - Observed inference time (s)": "{\"description\": \"min=0.668, mean=0.668, max=0.668, sum=1.336 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.668224381075965\"}", - "Jurisprudence - # eval": "{\"description\": \"min=108, mean=108, max=108, sum=216 (2)\", \"tab\": \"General information\", \"score\": \"108.0\"}", - "Jurisprudence - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Jurisprudence - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Jurisprudence - # prompt tokens": "{\"description\": \"min=431.898, mean=431.898, max=431.898, sum=863.796 (2)\", \"tab\": \"General information\", \"score\": \"431.89814814814815\"}", - "Jurisprudence - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"jurisprudence\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_jurisprudence\"" - } - } - }, - { - "evaluation_name": "Philosophy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Philosophy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.82, - "details": { - "description": "min=0.82, mean=0.82, max=0.82, sum=1.64 (2)", - "tab": "Accuracy", - "Philosophy - Observed inference time (s)": "{\"description\": \"min=0.424, mean=0.424, max=0.424, sum=0.848 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.42395149779856395\"}", - "Philosophy - # eval": "{\"description\": \"min=311, mean=311, max=311, sum=622 (2)\", \"tab\": \"General information\", \"score\": \"311.0\"}", - "Philosophy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Philosophy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Philosophy - # prompt tokens": "{\"description\": \"min=356.723, mean=356.723, max=356.723, sum=713.447 (2)\", \"tab\": \"General information\", \"score\": \"356.7234726688103\"}", - "Philosophy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"philosophy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_philosophy\"" - } - } - }, - { - "evaluation_name": "Professional Psychology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Professional Psychology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.835, - "details": { - "description": "min=0.835, mean=0.835, max=0.835, sum=1.67 (2)", - "tab": "Accuracy", - "Professional Medicine - Observed inference time (s)": "{\"description\": \"min=2.222, mean=2.222, max=2.222, sum=4.444 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.222188143169179\"}", - "Professional Accounting - Observed inference time (s)": "{\"description\": \"min=0.66, mean=0.66, max=0.66, sum=1.32 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6598629156748453\"}", - "Professional Law - Observed inference time (s)": "{\"description\": \"min=1.839, mean=1.839, max=1.839, sum=3.678 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.839003596032303\"}", - "Professional Psychology - Observed inference time (s)": "{\"description\": \"min=2.178, mean=2.178, max=2.178, sum=4.356 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.1780028343200684\"}", - "Professional Medicine - # eval": "{\"description\": \"min=272, mean=272, max=272, sum=544 (2)\", \"tab\": \"General information\", \"score\": \"272.0\"}", - "Professional Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Medicine - # prompt tokens": "{\"description\": \"min=1202.533, mean=1202.533, max=1202.533, sum=2405.066 (2)\", \"tab\": \"General information\", \"score\": \"1202.5330882352941\"}", - "Professional Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Accounting - # eval": "{\"description\": \"min=282, mean=282, max=282, sum=564 (2)\", \"tab\": \"General information\", \"score\": \"282.0\"}", - "Professional Accounting - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Accounting - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # prompt tokens": "{\"description\": \"min=771.16, mean=771.16, max=771.16, sum=1542.319 (2)\", \"tab\": \"General information\", \"score\": \"771.1595744680851\"}", - "Professional Accounting - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Law - # eval": "{\"description\": \"min=1534, mean=1534, max=1534, sum=3068 (2)\", \"tab\": \"General information\", \"score\": \"1534.0\"}", - "Professional Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # prompt tokens": "{\"description\": \"min=1759.098, mean=1759.098, max=1759.098, sum=3518.197 (2)\", \"tab\": \"General information\", \"score\": \"1759.0984354628422\"}", - "Professional Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Psychology - # eval": "{\"description\": \"min=612, mean=612, max=612, sum=1224 (2)\", \"tab\": \"General information\", \"score\": \"612.0\"}", - "Professional Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # prompt tokens": "{\"description\": \"min=608.201, mean=608.201, max=608.201, sum=1216.402 (2)\", \"tab\": \"General information\", \"score\": \"608.2009803921569\"}", - "Professional Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"professional_psychology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_professional_psychology\"" - } - } - }, - { - "evaluation_name": "Us Foreign Policy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Us Foreign Policy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.91, - "details": { - "description": "min=0.91, mean=0.91, max=0.91, sum=1.82 (2)", - "tab": "Accuracy", - "Us Foreign Policy - Observed inference time (s)": "{\"description\": \"min=0.607, mean=0.607, max=0.607, sum=1.214 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6068471717834473\"}", - "Us Foreign Policy - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Us Foreign Policy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Us Foreign Policy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Us Foreign Policy - # prompt tokens": "{\"description\": \"min=458.53, mean=458.53, max=458.53, sum=917.06 (2)\", \"tab\": \"General information\", \"score\": \"458.53\"}", - "Us Foreign Policy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"us_foreign_policy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_us_foreign_policy\"" - } - } - }, - { - "evaluation_name": "Astronomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Astronomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.901, - "details": { - "description": "min=0.901, mean=0.901, max=0.901, sum=1.803 (2)", - "tab": "Accuracy", - "Astronomy - Observed inference time (s)": "{\"description\": \"min=0.559, mean=0.559, max=0.559, sum=1.117 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5586237562330145\"}", - "Astronomy - # eval": "{\"description\": \"min=152, mean=152, max=152, sum=304 (2)\", \"tab\": \"General information\", \"score\": \"152.0\"}", - "Astronomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Astronomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Astronomy - # prompt tokens": "{\"description\": \"min=626.895, mean=626.895, max=626.895, sum=1253.789 (2)\", \"tab\": \"General information\", \"score\": \"626.8947368421053\"}", - "Astronomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"astronomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_astronomy\"" - } - } - }, - { - "evaluation_name": "Business Ethics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Business Ethics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.75, - "details": { - "description": "min=0.75, mean=0.75, max=0.75, sum=1.5 (2)", - "tab": "Accuracy", - "Business Ethics - Observed inference time (s)": "{\"description\": \"min=0.566, mean=0.566, max=0.566, sum=1.133 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5663742089271545\"}", - "Business Ethics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Business Ethics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Business Ethics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Business Ethics - # prompt tokens": "{\"description\": \"min=616.97, mean=616.97, max=616.97, sum=1233.94 (2)\", \"tab\": \"General information\", \"score\": \"616.97\"}", - "Business Ethics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"business_ethics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_business_ethics\"" - } - } - }, - { - "evaluation_name": "Clinical Knowledge", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Clinical Knowledge", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8, - "details": { - "description": "min=0.8, mean=0.8, max=0.8, sum=1.6 (2)", - "tab": "Accuracy", - "Clinical Knowledge - Observed inference time (s)": "{\"description\": \"min=0.487, mean=0.487, max=0.487, sum=0.975 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4874912774787759\"}", - "Clinical Knowledge - # eval": "{\"description\": \"min=265, mean=265, max=265, sum=530 (2)\", \"tab\": \"General information\", \"score\": \"265.0\"}", - "Clinical Knowledge - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Clinical Knowledge - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Clinical Knowledge - # prompt tokens": "{\"description\": \"min=446.966, mean=446.966, max=446.966, sum=893.932 (2)\", \"tab\": \"General information\", \"score\": \"446.96603773584906\"}", - "Clinical Knowledge - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"clinical_knowledge\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_clinical_knowledge\"" - } - } - }, - { - "evaluation_name": "Conceptual Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Conceptual Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.77, - "details": { - "description": "min=0.77, mean=0.77, max=0.77, sum=1.54 (2)", - "tab": "Accuracy", - "Conceptual Physics - Observed inference time (s)": "{\"description\": \"min=0.439, mean=0.439, max=0.439, sum=0.878 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4390637499220828\"}", - "Conceptual Physics - # eval": "{\"description\": \"min=235, mean=235, max=235, sum=470 (2)\", \"tab\": \"General information\", \"score\": \"235.0\"}", - "Conceptual Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Conceptual Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Conceptual Physics - # prompt tokens": "{\"description\": \"min=311.94, mean=311.94, max=311.94, sum=623.881 (2)\", \"tab\": \"General information\", \"score\": \"311.9404255319149\"}", - "Conceptual Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"conceptual_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_conceptual_physics\"" - } - } - }, - { - "evaluation_name": "Electrical Engineering", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Electrical Engineering", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.779, - "details": { - "description": "min=0.779, mean=0.779, max=0.779, sum=1.559 (2)", - "tab": "Accuracy", - "Electrical Engineering - Observed inference time (s)": "{\"description\": \"min=0.531, mean=0.531, max=0.531, sum=1.063 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.531287300175634\"}", - "Electrical Engineering - # eval": "{\"description\": \"min=145, mean=145, max=145, sum=290 (2)\", \"tab\": \"General information\", \"score\": \"145.0\"}", - "Electrical Engineering - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Electrical Engineering - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Electrical Engineering - # prompt tokens": "{\"description\": \"min=491.993, mean=491.993, max=491.993, sum=983.986 (2)\", \"tab\": \"General information\", \"score\": \"491.99310344827586\"}", - "Electrical Engineering - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"electrical_engineering\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_electrical_engineering\"" - } - } - }, - { - "evaluation_name": "Elementary Mathematics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Elementary Mathematics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.656, - "details": { - "description": "min=0.656, mean=0.656, max=0.656, sum=1.312 (2)", - "tab": "Accuracy", - "Elementary Mathematics - Observed inference time (s)": "{\"description\": \"min=0.561, mean=0.561, max=0.561, sum=1.123 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5613514084033865\"}", - "Elementary Mathematics - # eval": "{\"description\": \"min=378, mean=378, max=378, sum=756 (2)\", \"tab\": \"General information\", \"score\": \"378.0\"}", - "Elementary Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Elementary Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Elementary Mathematics - # prompt tokens": "{\"description\": \"min=601.344, mean=601.344, max=601.344, sum=1202.688 (2)\", \"tab\": \"General information\", \"score\": \"601.3439153439153\"}", - "Elementary Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"elementary_mathematics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_elementary_mathematics\"" - } - } - }, - { - "evaluation_name": "Formal Logic", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Formal Logic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.548, - "details": { - "description": "min=0.548, mean=0.548, max=0.548, sum=1.095 (2)", - "tab": "Accuracy", - "Formal Logic - Observed inference time (s)": "{\"description\": \"min=0.626, mean=0.626, max=0.626, sum=1.253 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6264226947511945\"}", - "Formal Logic - # eval": "{\"description\": \"min=126, mean=126, max=126, sum=252 (2)\", \"tab\": \"General information\", \"score\": \"126.0\"}", - "Formal Logic - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Formal Logic - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Formal Logic - # prompt tokens": "{\"description\": \"min=675.579, mean=675.579, max=675.579, sum=1351.159 (2)\", \"tab\": \"General information\", \"score\": \"675.5793650793651\"}", - "Formal Logic - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"formal_logic\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_formal_logic\"" - } - } - }, - { - "evaluation_name": "High School World History", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on High School World History", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.907, - "details": { - "description": "min=0.907, mean=0.907, max=0.907, sum=1.814 (2)", - "tab": "Accuracy", - "High School Biology - Observed inference time (s)": "{\"description\": \"min=0.59, mean=0.59, max=0.59, sum=1.179 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5895279146009876\"}", - "High School Chemistry - Observed inference time (s)": "{\"description\": \"min=0.562, mean=0.562, max=0.562, sum=1.124 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5618457112993512\"}", - "High School Computer Science - Observed inference time (s)": "{\"description\": \"min=0.851, mean=0.851, max=0.851, sum=1.702 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8510373497009277\"}", - "High School European History - Observed inference time (s)": "{\"description\": \"min=2.359, mean=2.359, max=2.359, sum=4.717 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.358732930096713\"}", - "High School Geography - Observed inference time (s)": "{\"description\": \"min=1.215, mean=1.215, max=1.215, sum=2.43 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.21489392266129\"}", - "High School Government And Politics - Observed inference time (s)": "{\"description\": \"min=0.677, mean=0.677, max=0.677, sum=1.354 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6768323757487875\"}", - "High School Macroeconomics - Observed inference time (s)": "{\"description\": \"min=0.57, mean=0.57, max=0.57, sum=1.14 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5697616595488328\"}", - "High School Mathematics - Observed inference time (s)": "{\"description\": \"min=0.541, mean=0.541, max=0.541, sum=1.082 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5409333193743671\"}", - "High School Microeconomics - Observed inference time (s)": "{\"description\": \"min=0.657, mean=0.657, max=0.657, sum=1.314 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6570467107436236\"}", - "High School Physics - Observed inference time (s)": "{\"description\": \"min=0.738, mean=0.738, max=0.738, sum=1.476 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7378138311651369\"}", - "High School Psychology - Observed inference time (s)": "{\"description\": \"min=0.524, mean=0.524, max=0.524, sum=1.049 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5244918534515101\"}", - "High School Statistics - Observed inference time (s)": "{\"description\": \"min=0.745, mean=0.745, max=0.745, sum=1.491 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7453252838717567\"}", - "High School US History - Observed inference time (s)": "{\"description\": \"min=1.821, mean=1.821, max=1.821, sum=3.642 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.8211165923698276\"}", - "High School World History - Observed inference time (s)": "{\"description\": \"min=1.27, mean=1.27, max=1.27, sum=2.541 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.2703520537428714\"}", - "High School Biology - # eval": "{\"description\": \"min=310, mean=310, max=310, sum=620 (2)\", \"tab\": \"General information\", \"score\": \"310.0\"}", - "High School Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Biology - # prompt tokens": "{\"description\": \"min=546.394, mean=546.394, max=546.394, sum=1092.787 (2)\", \"tab\": \"General information\", \"score\": \"546.3935483870968\"}", - "High School Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Chemistry - # eval": "{\"description\": \"min=203, mean=203, max=203, sum=406 (2)\", \"tab\": \"General information\", \"score\": \"203.0\"}", - "High School Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # prompt tokens": "{\"description\": \"min=537.015, mean=537.015, max=537.015, sum=1074.03 (2)\", \"tab\": \"General information\", \"score\": \"537.0147783251232\"}", - "High School Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "High School Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # prompt tokens": "{\"description\": \"min=962.1, mean=962.1, max=962.1, sum=1924.2 (2)\", \"tab\": \"General information\", \"score\": \"962.1\"}", - "High School Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School European History - # eval": "{\"description\": \"min=165, mean=165, max=165, sum=330 (2)\", \"tab\": \"General information\", \"score\": \"165.0\"}", - "High School European History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School European History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # prompt tokens": "{\"description\": \"min=2957.412, mean=2957.412, max=2957.412, sum=5914.824 (2)\", \"tab\": \"General information\", \"score\": \"2957.4121212121213\"}", - "High School European History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Geography - # eval": "{\"description\": \"min=198, mean=198, max=198, sum=396 (2)\", \"tab\": \"General information\", \"score\": \"198.0\"}", - "High School Geography - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Geography - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # prompt tokens": "{\"description\": \"min=404.035, mean=404.035, max=404.035, sum=808.071 (2)\", \"tab\": \"General information\", \"score\": \"404.0353535353535\"}", - "High School Geography - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Government And Politics - # eval": "{\"description\": \"min=193, mean=193, max=193, sum=386 (2)\", \"tab\": \"General information\", \"score\": \"193.0\"}", - "High School Government And Politics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Government And Politics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # prompt tokens": "{\"description\": \"min=484.725, mean=484.725, max=484.725, sum=969.451 (2)\", \"tab\": \"General information\", \"score\": \"484.7253886010363\"}", - "High School Government And Politics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Macroeconomics - # eval": "{\"description\": \"min=390, mean=390, max=390, sum=780 (2)\", \"tab\": \"General information\", \"score\": \"390.0\"}", - "High School Macroeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Macroeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # prompt tokens": "{\"description\": \"min=398.892, mean=398.892, max=398.892, sum=797.785 (2)\", \"tab\": \"General information\", \"score\": \"398.89230769230767\"}", - "High School Macroeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Mathematics - # eval": "{\"description\": \"min=270, mean=270, max=270, sum=540 (2)\", \"tab\": \"General information\", \"score\": \"270.0\"}", - "High School Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # prompt tokens": "{\"description\": \"min=575.622, mean=575.622, max=575.622, sum=1151.244 (2)\", \"tab\": \"General information\", \"score\": \"575.6222222222223\"}", - "High School Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Microeconomics - # eval": "{\"description\": \"min=238, mean=238, max=238, sum=476 (2)\", \"tab\": \"General information\", \"score\": \"238.0\"}", - "High School Microeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Microeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # prompt tokens": "{\"description\": \"min=420.739, mean=420.739, max=420.739, sum=841.479 (2)\", \"tab\": \"General information\", \"score\": \"420.73949579831935\"}", - "High School Microeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Physics - # eval": "{\"description\": \"min=151, mean=151, max=151, sum=302 (2)\", \"tab\": \"General information\", \"score\": \"151.0\"}", - "High School Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # prompt tokens": "{\"description\": \"min=599.411, mean=599.411, max=599.411, sum=1198.821 (2)\", \"tab\": \"General information\", \"score\": \"599.4105960264901\"}", - "High School Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Psychology - # eval": "{\"description\": \"min=545, mean=545, max=545, sum=1090 (2)\", \"tab\": \"General information\", \"score\": \"545.0\"}", - "High School Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # prompt tokens": "{\"description\": \"min=526.826, mean=526.826, max=526.826, sum=1053.651 (2)\", \"tab\": \"General information\", \"score\": \"526.8256880733945\"}", - "High School Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Statistics - # eval": "{\"description\": \"min=216, mean=216, max=216, sum=432 (2)\", \"tab\": \"General information\", \"score\": \"216.0\"}", - "High School Statistics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Statistics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # prompt tokens": "{\"description\": \"min=869.778, mean=869.778, max=869.778, sum=1739.556 (2)\", \"tab\": \"General information\", \"score\": \"869.7777777777778\"}", - "High School Statistics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School US History - # eval": "{\"description\": \"min=204, mean=204, max=204, sum=408 (2)\", \"tab\": \"General information\", \"score\": \"204.0\"}", - "High School US History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School US History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # prompt tokens": "{\"description\": \"min=2369.132, mean=2369.132, max=2369.132, sum=4738.265 (2)\", \"tab\": \"General information\", \"score\": \"2369.1323529411766\"}", - "High School US History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School World History - # eval": "{\"description\": \"min=237, mean=237, max=237, sum=474 (2)\", \"tab\": \"General information\", \"score\": \"237.0\"}", - "High School World History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School World History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # prompt tokens": "{\"description\": \"min=1541.371, mean=1541.371, max=1541.371, sum=3082.743 (2)\", \"tab\": \"General information\", \"score\": \"1541.3713080168777\"}", - "High School World History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"high_school_world_history\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_high_school_world_history\"" - } - } - }, - { - "evaluation_name": "Human Sexuality", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Human Sexuality", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.87, - "details": { - "description": "min=0.87, mean=0.87, max=0.87, sum=1.74 (2)", - "tab": "Accuracy", - "Human Aging - Observed inference time (s)": "{\"description\": \"min=0.768, mean=0.768, max=0.768, sum=1.535 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.76751750146327\"}", - "Human Sexuality - Observed inference time (s)": "{\"description\": \"min=0.408, mean=0.408, max=0.408, sum=0.816 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4077764613027791\"}", - "Human Aging - # eval": "{\"description\": \"min=223, mean=223, max=223, sum=446 (2)\", \"tab\": \"General information\", \"score\": \"223.0\"}", - "Human Aging - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Aging - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Aging - # prompt tokens": "{\"description\": \"min=332.013, mean=332.013, max=332.013, sum=664.027 (2)\", \"tab\": \"General information\", \"score\": \"332.0134529147982\"}", - "Human Aging - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Human Sexuality - # eval": "{\"description\": \"min=131, mean=131, max=131, sum=262 (2)\", \"tab\": \"General information\", \"score\": \"131.0\"}", - "Human Sexuality - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Sexuality - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # prompt tokens": "{\"description\": \"min=367.855, mean=367.855, max=367.855, sum=735.71 (2)\", \"tab\": \"General information\", \"score\": \"367.85496183206106\"}", - "Human Sexuality - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"human_sexuality\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_human_sexuality\"" - } - } - }, - { - "evaluation_name": "International Law", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on International Law", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.909, - "details": { - "description": "min=0.909, mean=0.909, max=0.909, sum=1.818 (2)", - "tab": "Accuracy", - "International Law - Observed inference time (s)": "{\"description\": \"min=0.588, mean=0.588, max=0.588, sum=1.175 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5876634554429487\"}", - "International Law - # eval": "{\"description\": \"min=121, mean=121, max=121, sum=242 (2)\", \"tab\": \"General information\", \"score\": \"121.0\"}", - "International Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "International Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "International Law - # prompt tokens": "{\"description\": \"min=663.289, mean=663.289, max=663.289, sum=1326.579 (2)\", \"tab\": \"General information\", \"score\": \"663.2892561983471\"}", - "International Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"international_law\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_international_law\"" - } - } - }, - { - "evaluation_name": "Logical Fallacies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Logical Fallacies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.883, - "details": { - "description": "min=0.883, mean=0.883, max=0.883, sum=1.767 (2)", - "tab": "Accuracy", - "Logical Fallacies - Observed inference time (s)": "{\"description\": \"min=0.622, mean=0.622, max=0.622, sum=1.245 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6223941814680041\"}", - "Logical Fallacies - # eval": "{\"description\": \"min=163, mean=163, max=163, sum=326 (2)\", \"tab\": \"General information\", \"score\": \"163.0\"}", - "Logical Fallacies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Logical Fallacies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Logical Fallacies - # prompt tokens": "{\"description\": \"min=466.595, mean=466.595, max=466.595, sum=933.19 (2)\", \"tab\": \"General information\", \"score\": \"466.5950920245399\"}", - "Logical Fallacies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"logical_fallacies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_logical_fallacies\"" - } - } - }, - { - "evaluation_name": "Machine Learning", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Machine Learning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.58, - "details": { - "description": "min=0.58, mean=0.58, max=0.58, sum=1.161 (2)", - "tab": "Accuracy", - "Machine Learning - Observed inference time (s)": "{\"description\": \"min=0.638, mean=0.638, max=0.638, sum=1.277 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6384105682373047\"}", - "Machine Learning - # eval": "{\"description\": \"min=112, mean=112, max=112, sum=224 (2)\", \"tab\": \"General information\", \"score\": \"112.0\"}", - "Machine Learning - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Machine Learning - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Machine Learning - # prompt tokens": "{\"description\": \"min=720.161, mean=720.161, max=720.161, sum=1440.321 (2)\", \"tab\": \"General information\", \"score\": \"720.1607142857143\"}", - "Machine Learning - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"machine_learning\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_machine_learning\"" - } - } - }, - { - "evaluation_name": "Management", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Management", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.893, - "details": { - "description": "min=0.893, mean=0.893, max=0.893, sum=1.786 (2)", - "tab": "Accuracy", - "Management - Observed inference time (s)": "{\"description\": \"min=0.42, mean=0.42, max=0.42, sum=0.841 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4204523748564489\"}", - "Management - # eval": "{\"description\": \"min=103, mean=103, max=103, sum=206 (2)\", \"tab\": \"General information\", \"score\": \"103.0\"}", - "Management - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Management - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Management - # prompt tokens": "{\"description\": \"min=300.544, mean=300.544, max=300.544, sum=601.087 (2)\", \"tab\": \"General information\", \"score\": \"300.54368932038835\"}", - "Management - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"management\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_management\"" - } - } - }, - { - "evaluation_name": "Marketing", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Marketing", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.936, - "details": { - "description": "min=0.936, mean=0.936, max=0.936, sum=1.872 (2)", - "tab": "Accuracy", - "Marketing - Observed inference time (s)": "{\"description\": \"min=0.463, mean=0.463, max=0.463, sum=0.926 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.463064443351876\"}", - "Marketing - # eval": "{\"description\": \"min=234, mean=234, max=234, sum=468 (2)\", \"tab\": \"General information\", \"score\": \"234.0\"}", - "Marketing - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Marketing - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Marketing - # prompt tokens": "{\"description\": \"min=442.825, mean=442.825, max=442.825, sum=885.65 (2)\", \"tab\": \"General information\", \"score\": \"442.8247863247863\"}", - "Marketing - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"marketing\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_marketing\"" - } - } - }, - { - "evaluation_name": "Medical Genetics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Medical Genetics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.87, - "details": { - "description": "min=0.87, mean=0.87, max=0.87, sum=1.74 (2)", - "tab": "Accuracy", - "Medical Genetics - Observed inference time (s)": "{\"description\": \"min=0.428, mean=0.428, max=0.428, sum=0.857 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.42836678981781007\"}", - "Medical Genetics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Medical Genetics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Medical Genetics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Medical Genetics - # prompt tokens": "{\"description\": \"min=362, mean=362, max=362, sum=724 (2)\", \"tab\": \"General information\", \"score\": \"362.0\"}", - "Medical Genetics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"medical_genetics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_medical_genetics\"" - } - } - }, - { - "evaluation_name": "Miscellaneous", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Miscellaneous", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.902, - "details": { - "description": "min=0.902, mean=0.902, max=0.902, sum=1.803 (2)", - "tab": "Accuracy", - "Miscellaneous - Observed inference time (s)": "{\"description\": \"min=0.537, mean=0.537, max=0.537, sum=1.075 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5372742845333095\"}", - "Miscellaneous - # eval": "{\"description\": \"min=783, mean=783, max=783, sum=1566 (2)\", \"tab\": \"General information\", \"score\": \"783.0\"}", - "Miscellaneous - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Miscellaneous - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Miscellaneous - # prompt tokens": "{\"description\": \"min=331.441, mean=331.441, max=331.441, sum=662.881 (2)\", \"tab\": \"General information\", \"score\": \"331.4406130268199\"}", - "Miscellaneous - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"miscellaneous\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_miscellaneous\"" - } - } - }, - { - "evaluation_name": "Moral Scenarios", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Moral Scenarios", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.606, - "details": { - "description": "min=0.606, mean=0.606, max=0.606, sum=1.211 (2)", - "tab": "Accuracy", - "Moral Disputes - Observed inference time (s)": "{\"description\": \"min=0.671, mean=0.671, max=0.671, sum=1.341 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6705957754498961\"}", - "Moral Scenarios - Observed inference time (s)": "{\"description\": \"min=0.764, mean=0.764, max=0.764, sum=1.528 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7642385613318928\"}", - "Moral Disputes - # eval": "{\"description\": \"min=346, mean=346, max=346, sum=692 (2)\", \"tab\": \"General information\", \"score\": \"346.0\"}", - "Moral Disputes - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Disputes - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Disputes - # prompt tokens": "{\"description\": \"min=507.913, mean=507.913, max=507.913, sum=1015.827 (2)\", \"tab\": \"General information\", \"score\": \"507.91329479768785\"}", - "Moral Disputes - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Moral Scenarios - # eval": "{\"description\": \"min=895, mean=895, max=895, sum=1790 (2)\", \"tab\": \"General information\", \"score\": \"895.0\"}", - "Moral Scenarios - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Scenarios - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # prompt tokens": "{\"description\": \"min=703.334, mean=703.334, max=703.334, sum=1406.668 (2)\", \"tab\": \"General information\", \"score\": \"703.3340782122905\"}", - "Moral Scenarios - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"moral_scenarios\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_moral_scenarios\"" - } - } - }, - { - "evaluation_name": "Nutrition", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Nutrition", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.869, - "details": { - "description": "min=0.869, mean=0.869, max=0.869, sum=1.739 (2)", - "tab": "Accuracy", - "Nutrition - Observed inference time (s)": "{\"description\": \"min=1.038, mean=1.038, max=1.038, sum=2.077 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.0384757246067322\"}", - "Nutrition - # eval": "{\"description\": \"min=306, mean=306, max=306, sum=612 (2)\", \"tab\": \"General information\", \"score\": \"306.0\"}", - "Nutrition - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Nutrition - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Nutrition - # prompt tokens": "{\"description\": \"min=643.317, mean=643.317, max=643.317, sum=1286.634 (2)\", \"tab\": \"General information\", \"score\": \"643.3169934640523\"}", - "Nutrition - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"nutrition\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_nutrition\"" - } - } - }, - { - "evaluation_name": "Prehistory", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Prehistory", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.877, - "details": { - "description": "min=0.877, mean=0.877, max=0.877, sum=1.753 (2)", - "tab": "Accuracy", - "Prehistory - Observed inference time (s)": "{\"description\": \"min=0.561, mean=0.561, max=0.561, sum=1.121 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.560588002204895\"}", - "Prehistory - # eval": "{\"description\": \"min=324, mean=324, max=324, sum=648 (2)\", \"tab\": \"General information\", \"score\": \"324.0\"}", - "Prehistory - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Prehistory - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Prehistory - # prompt tokens": "{\"description\": \"min=565.096, mean=565.096, max=565.096, sum=1130.191 (2)\", \"tab\": \"General information\", \"score\": \"565.0956790123457\"}", - "Prehistory - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"prehistory\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_prehistory\"" - } - } - }, - { - "evaluation_name": "Public Relations", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Public Relations", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.745, - "details": { - "description": "min=0.745, mean=0.745, max=0.745, sum=1.491 (2)", - "tab": "Accuracy", - "Public Relations - Observed inference time (s)": "{\"description\": \"min=2.107, mean=2.107, max=2.107, sum=4.213 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.1067019375887783\"}", - "Public Relations - # eval": "{\"description\": \"min=110, mean=110, max=110, sum=220 (2)\", \"tab\": \"General information\", \"score\": \"110.0\"}", - "Public Relations - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Public Relations - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Public Relations - # prompt tokens": "{\"description\": \"min=432.436, mean=432.436, max=432.436, sum=864.873 (2)\", \"tab\": \"General information\", \"score\": \"432.43636363636364\"}", - "Public Relations - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"public_relations\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_public_relations\"" - } - } - }, - { - "evaluation_name": "Security Studies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Security Studies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.833, - "details": { - "description": "min=0.833, mean=0.833, max=0.833, sum=1.665 (2)", - "tab": "Accuracy", - "Security Studies - Observed inference time (s)": "{\"description\": \"min=2.683, mean=2.683, max=2.683, sum=5.366 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.682755525744691\"}", - "Security Studies - # eval": "{\"description\": \"min=245, mean=245, max=245, sum=490 (2)\", \"tab\": \"General information\", \"score\": \"245.0\"}", - "Security Studies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Security Studies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Security Studies - # prompt tokens": "{\"description\": \"min=1227.196, mean=1227.196, max=1227.196, sum=2454.392 (2)\", \"tab\": \"General information\", \"score\": \"1227.1959183673468\"}", - "Security Studies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"security_studies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_security_studies\"" - } - } - }, - { - "evaluation_name": "Sociology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Sociology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9, - "details": { - "description": "min=0.9, mean=0.9, max=0.9, sum=1.801 (2)", - "tab": "Accuracy", - "Sociology - Observed inference time (s)": "{\"description\": \"min=1.401, mean=1.401, max=1.401, sum=2.803 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.4013089469416224\"}", - "Sociology - # eval": "{\"description\": \"min=201, mean=201, max=201, sum=402 (2)\", \"tab\": \"General information\", \"score\": \"201.0\"}", - "Sociology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Sociology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Sociology - # prompt tokens": "{\"description\": \"min=463.99, mean=463.99, max=463.99, sum=927.98 (2)\", \"tab\": \"General information\", \"score\": \"463.99004975124376\"}", - "Sociology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"sociology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_sociology\"" - } - } - }, - { - "evaluation_name": "Virology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Virology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.572, - "details": { - "description": "min=0.572, mean=0.572, max=0.572, sum=1.145 (2)", - "tab": "Accuracy", - "Virology - Observed inference time (s)": "{\"description\": \"min=0.563, mean=0.563, max=0.563, sum=1.127 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5633984617440098\"}", - "Virology - # eval": "{\"description\": \"min=166, mean=166, max=166, sum=332 (2)\", \"tab\": \"General information\", \"score\": \"166.0\"}", - "Virology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Virology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Virology - # prompt tokens": "{\"description\": \"min=363.102, mean=363.102, max=363.102, sum=726.205 (2)\", \"tab\": \"General information\", \"score\": \"363.1024096385542\"}", - "Virology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"virology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_virology\"" - } - } - }, - { - "evaluation_name": "World Religions", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on World Religions", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.877, - "details": { - "description": "min=0.877, mean=0.877, max=0.877, sum=1.754 (2)", - "tab": "Accuracy", - "World Religions - Observed inference time (s)": "{\"description\": \"min=0.407, mean=0.407, max=0.407, sum=0.814 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4067504726655302\"}", - "World Religions - # eval": "{\"description\": \"min=171, mean=171, max=171, sum=342 (2)\", \"tab\": \"General information\", \"score\": \"171.0\"}", - "World Religions - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "World Religions - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "World Religions - # prompt tokens": "{\"description\": \"min=289.971, mean=289.971, max=289.971, sum=579.942 (2)\", \"tab\": \"General information\", \"score\": \"289.97076023391816\"}", - "World Religions - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"world_religions\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_world_religions\"" - } - } - }, - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.315, - "details": { - "description": "", - "tab": "Efficiency" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_id": "hfopenllm_v2/01-ai_Yi-34B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3046 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5457 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0514 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3666 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4119 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4412 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/01-ai_yi-6b-200k.json b/data/models/01-ai_yi-6b-200k.json deleted file mode 100644 index d360ddbdc30af81f31805e6ce7aba91ceff7683e..0000000000000000000000000000000000000000 --- a/data/models/01-ai_yi-6b-200k.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Yi-6B-200K", - "id": "01-ai/Yi-6B-200K", - "developer": "01-ai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "6.061" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/01-ai_Yi-6B-200K/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0843 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4289 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0181 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2819 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4587 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2844 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/01-ai_yi-6b-chat.json b/data/models/01-ai_yi-6b-chat.json deleted file mode 100644 index c88340fe9e66c5f7c834891c3b5092c274cb400d..0000000000000000000000000000000000000000 --- a/data/models/01-ai_yi-6b-chat.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Yi-6B-Chat", - "id": "01-ai/Yi-6B-Chat", - "developer": "01-ai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "6.061" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/01-ai_Yi-6B-Chat/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3395 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4133 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0136 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2945 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3688 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3061 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/01-ai_yi-6b.json b/data/models/01-ai_yi-6b.json deleted file mode 100644 index b4ab67edd2e4446bc7ec7b4273cdaa406241c77f..0000000000000000000000000000000000000000 --- a/data/models/01-ai_yi-6b.json +++ /dev/null @@ -1,2036 +0,0 @@ -{ - "model_info": { - "name": "Yi 6B", - "id": "01-ai/yi-6b", - "developer": "01-ai", - "inference_platform": "unknown", - "additional_details": { - "model_id_aliases": [ - "01-ai/Yi-6B" - ] - } - }, - "evaluations": [ - { - "evaluation_id": "helm_lite/01-ai_yi-6b/1774096306.427425", - "retrieved_timestamp": "1774096306.427425", - "source_metadata": { - "source_name": "helm_lite", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_lite", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_lite", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.253, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.6630461922596754\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.702, - "details": { - "description": "min=0.702, mean=0.702, max=0.702, sum=0.702 (1)", - "tab": "Accuracy", - "NarrativeQA - Observed inference time (s)": "{\"description\": \"min=1.404, mean=1.404, max=1.404, sum=1.404 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.4038719868995775\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=4.868, mean=4.868, max=4.868, sum=4.868 (1)\", \"tab\": \"General information\", \"score\": \"4.867605633802817\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=3611.445, mean=3611.445, max=3611.445, sum=3611.445 (1)\", \"tab\": \"General information\", \"score\": \"3611.445070422535\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (closed-book)", - "source_data": { - "dataset_name": "NaturalQuestions (closed-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (closed-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.31, - "details": { - "description": "min=0.31, mean=0.31, max=0.31, sum=0.31 (1)", - "tab": "Accuracy", - "NaturalQuestions (open-book) - Observed inference time (s)": "{\"description\": \"min=0.911, mean=0.911, max=0.911, sum=0.911 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.9108293209075927\"}", - "NaturalQuestions (closed-book) - Observed inference time (s)": "{\"description\": \"min=0.413, mean=0.413, max=0.413, sum=0.413 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.4127621691226959\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.838, mean=4.838, max=4.838, sum=4.838 (1)\", \"tab\": \"General information\", \"score\": \"4.838\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.026, mean=0.026, max=0.026, sum=0.026 (1)\", \"tab\": \"General information\", \"score\": \"0.026\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=2171.698, mean=2171.698, max=2171.698, sum=2171.698 (1)\", \"tab\": \"General information\", \"score\": \"2171.698\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=0.995, mean=0.995, max=0.995, sum=0.995 (1)\", \"tab\": \"General information\", \"score\": \"0.995\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=131.695, mean=131.695, max=131.695, sum=131.695 (1)\", \"tab\": \"General information\", \"score\": \"131.695\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "mode": "\"closedbook\"" - } - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8, - "details": { - "description": "min=0.8, mean=0.8, max=0.8, sum=0.8 (1)", - "tab": "Accuracy", - "OpenbookQA - Observed inference time (s)": "{\"description\": \"min=0.354, mean=0.354, max=0.354, sum=0.354 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.3535394024848938\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=260.002, mean=260.002, max=260.002, sum=260.002 (1)\", \"tab\": \"General information\", \"score\": \"260.002\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "dataset": "\"openbookqa\"", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.53, - "details": { - "description": "min=0.3, mean=0.53, max=0.87, sum=2.651 (5)", - "tab": "Accuracy", - "MMLU - Observed inference time (s)": "{\"description\": \"min=0.323, mean=0.339, max=0.368, sum=1.696 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.3391338364283244\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=383.67, mean=502.654, max=667.789, sum=2513.269 (5)\", \"tab\": \"General information\", \"score\": \"502.65389473684206\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"college_chemistry\", \"computer_security\", \"econometrics\", \"us_foreign_policy\"]", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MATH", - "source_data": { - "dataset_name": "MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Equivalent (CoT) on MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.126, - "details": { - "description": "min=0.058, mean=0.126, max=0.2, sum=0.881 (7)", - "tab": "Accuracy", - "MATH - Observed inference time (s)": "{\"description\": \"min=1.167, mean=1.837, max=2.263, sum=12.86 (7)\", \"tab\": \"Efficiency\", \"score\": \"1.8371926514375443\"}", - "MATH - # eval": "{\"description\": \"min=30, mean=62.429, max=135, sum=437 (7)\", \"tab\": \"General information\", \"score\": \"62.42857142857143\"}", - "MATH - # train": "{\"description\": \"min=8, mean=8, max=8, sum=56 (7)\", \"tab\": \"General information\", \"score\": \"8.0\"}", - "MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MATH - # prompt tokens": "{\"description\": \"min=976.696, mean=1468.935, max=2582.038, sum=10282.547 (7)\", \"tab\": \"General information\", \"score\": \"1468.9352369693863\"}", - "MATH - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=7 (7)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"algebra\", \"counting_and_probability\", \"geometry\", \"intermediate_algebra\", \"number_theory\", \"prealgebra\", \"precalculus\"]", - "level": "\"1\"", - "use_official_examples": "\"False\"", - "use_chain_of_thought": "\"True\"" - } - } - }, - { - "evaluation_name": "GSM8K", - "source_data": { - "dataset_name": "GSM8K", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on GSM8K", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.375, - "details": { - "description": "min=0.375, mean=0.375, max=0.375, sum=0.375 (1)", - "tab": "Accuracy", - "GSM8K - Observed inference time (s)": "{\"description\": \"min=1.878, mean=1.878, max=1.878, sum=1.878 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.8781680135726928\"}", - "GSM8K - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "GSM8K - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "GSM8K - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GSM8K - # prompt tokens": "{\"description\": \"min=1170.814, mean=1170.814, max=1170.814, sum=1170.814 (1)\", \"tab\": \"General information\", \"score\": \"1170.814\"}", - "GSM8K - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "LegalBench", - "source_data": { - "dataset_name": "LegalBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on LegalBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.519, - "details": { - "description": "min=0.284, mean=0.519, max=0.779, sum=2.594 (5)", - "tab": "Accuracy", - "LegalBench - Observed inference time (s)": "{\"description\": \"min=0.379, mean=0.553, max=1.149, sum=2.764 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.5528668178286933\"}", - "LegalBench - # eval": "{\"description\": \"min=95, mean=409.4, max=1000, sum=2047 (5)\", \"tab\": \"General information\", \"score\": \"409.4\"}", - "LegalBench - # train": "{\"description\": \"min=2, mean=4.2, max=5, sum=21 (5)\", \"tab\": \"General information\", \"score\": \"4.2\"}", - "LegalBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "LegalBench - # prompt tokens": "{\"description\": \"min=211.779, mean=951.524, max=3359.547, sum=4757.621 (5)\", \"tab\": \"General information\", \"score\": \"951.5242922438443\"}", - "LegalBench - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"abercrombie\", \"corporate_lobbying\", \"function_of_decision_section\", \"international_citizenship_questions\", \"proa\"]" - } - } - }, - { - "evaluation_name": "MedQA", - "source_data": { - "dataset_name": "MedQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MedQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.497, - "details": { - "description": "min=0.497, mean=0.497, max=0.497, sum=0.497 (1)", - "tab": "Accuracy", - "MedQA - Observed inference time (s)": "{\"description\": \"min=0.405, mean=0.405, max=0.405, sum=0.405 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.4053303655051806\"}", - "MedQA - # eval": "{\"description\": \"min=503, mean=503, max=503, sum=503 (1)\", \"tab\": \"General information\", \"score\": \"503.0\"}", - "MedQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MedQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MedQA - # prompt tokens": "{\"description\": \"min=1122.392, mean=1122.392, max=1122.392, sum=1122.392 (1)\", \"tab\": \"General information\", \"score\": \"1122.3916500994035\"}", - "MedQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WMT 2014", - "source_data": { - "dataset_name": "WMT 2014", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "BLEU-4 on WMT 2014", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.117, - "details": { - "description": "min=0.055, mean=0.117, max=0.182, sum=0.584 (5)", - "tab": "Accuracy", - "WMT 2014 - Observed inference time (s)": "{\"description\": \"min=0.602, mean=0.626, max=0.666, sum=3.129 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.6257070175426044\"}", - "WMT 2014 - # eval": "{\"description\": \"min=503, mean=568.8, max=832, sum=2844 (5)\", \"tab\": \"General information\", \"score\": \"568.8\"}", - "WMT 2014 - # train": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "WMT 2014 - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WMT 2014 - # prompt tokens": "{\"description\": \"min=139.298, mean=187.092, max=317.56, sum=935.461 (5)\", \"tab\": \"General information\", \"score\": \"187.09213851506345\"}", - "WMT 2014 - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "language_pair": "[\"cs-en\", \"de-en\", \"fr-en\", \"hi-en\", \"ru-en\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_mmlu/01-ai_yi-6b/1774096312.00548", - "retrieved_timestamp": "1774096312.00548", - "source_metadata": { - "source_name": "helm_mmlu", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_mmlu", - "evaluation_results": [ - { - "evaluation_name": "MMLU All Subjects", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU All Subjects", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.64, - "details": { - "description": "min=0.3, mean=0.64, max=0.907, sum=72.967 (114)", - "tab": "Accuracy", - "MMLU All Subjects - Observed inference time (s)": "{\"description\": \"min=0.315, mean=0.388, max=0.912, sum=44.195 (114)\", \"tab\": \"Efficiency\", \"score\": \"0.3876731134304364\"}", - "MMLU All Subjects - # eval": "{\"description\": \"min=100, mean=246.351, max=1534, sum=28084 (114)\", \"tab\": \"General information\", \"score\": \"246.35087719298247\"}", - "MMLU All Subjects - # train": "{\"description\": \"min=5, mean=5, max=5, sum=570 (114)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU All Subjects - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU All Subjects - # prompt tokens": "{\"description\": \"min=289.971, mean=661.842, max=2957.412, sum=75449.942 (114)\", \"tab\": \"General information\", \"score\": \"661.8416008681387\"}", - "MMLU All Subjects - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=114 (114)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_name": "Abstract Algebra", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Abstract Algebra", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3, - "details": { - "description": "min=0.3, mean=0.3, max=0.3, sum=0.6 (2)", - "tab": "Accuracy", - "Abstract Algebra - Observed inference time (s)": "{\"description\": \"min=0.343, mean=0.343, max=0.343, sum=0.686 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.34289863109588625\"}", - "Abstract Algebra - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Abstract Algebra - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Abstract Algebra - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Abstract Algebra - # prompt tokens": "{\"description\": \"min=383.67, mean=383.67, max=383.67, sum=767.34 (2)\", \"tab\": \"General information\", \"score\": \"383.67\"}", - "Abstract Algebra - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"abstract_algebra\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_abstract_algebra\"" - } - } - }, - { - "evaluation_name": "Anatomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Anatomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6, - "details": { - "description": "min=0.6, mean=0.6, max=0.6, sum=1.2 (2)", - "tab": "Accuracy", - "Anatomy - Observed inference time (s)": "{\"description\": \"min=0.334, mean=0.334, max=0.334, sum=0.668 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3338937794720685\"}", - "Anatomy - # eval": "{\"description\": \"min=135, mean=135, max=135, sum=270 (2)\", \"tab\": \"General information\", \"score\": \"135.0\"}", - "Anatomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Anatomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Anatomy - # prompt tokens": "{\"description\": \"min=375.77, mean=375.77, max=375.77, sum=751.541 (2)\", \"tab\": \"General information\", \"score\": \"375.77037037037036\"}", - "Anatomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"anatomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_anatomy\"" - } - } - }, - { - "evaluation_name": "College Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on College Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.422, - "details": { - "description": "min=0.422, mean=0.422, max=0.422, sum=0.843 (2)", - "tab": "Accuracy", - "College Chemistry - Observed inference time (s)": "{\"description\": \"min=0.34, mean=0.34, max=0.34, sum=0.68 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3400930452346802\"}", - "College Biology - Observed inference time (s)": "{\"description\": \"min=0.331, mean=0.331, max=0.331, sum=0.661 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3306954221593009\"}", - "College Computer Science - Observed inference time (s)": "{\"description\": \"min=0.397, mean=0.397, max=0.397, sum=0.793 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.39658718585968017\"}", - "College Mathematics - Observed inference time (s)": "{\"description\": \"min=0.372, mean=0.372, max=0.372, sum=0.744 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3718992257118225\"}", - "College Medicine - Observed inference time (s)": "{\"description\": \"min=0.36, mean=0.36, max=0.36, sum=0.721 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.360349433270493\"}", - "College Physics - Observed inference time (s)": "{\"description\": \"min=0.363, mean=0.363, max=0.363, sum=0.726 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.36309780092800364\"}", - "College Chemistry - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Chemistry - # prompt tokens": "{\"description\": \"min=597.54, mean=597.54, max=597.54, sum=1195.08 (2)\", \"tab\": \"General information\", \"score\": \"597.54\"}", - "College Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Biology - # eval": "{\"description\": \"min=144, mean=144, max=144, sum=288 (2)\", \"tab\": \"General information\", \"score\": \"144.0\"}", - "College Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # prompt tokens": "{\"description\": \"min=514.819, mean=514.819, max=514.819, sum=1029.639 (2)\", \"tab\": \"General information\", \"score\": \"514.8194444444445\"}", - "College Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # prompt tokens": "{\"description\": \"min=883.06, mean=883.06, max=883.06, sum=1766.12 (2)\", \"tab\": \"General information\", \"score\": \"883.06\"}", - "College Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Mathematics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # prompt tokens": "{\"description\": \"min=635.3, mean=635.3, max=635.3, sum=1270.6 (2)\", \"tab\": \"General information\", \"score\": \"635.3\"}", - "College Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Medicine - # eval": "{\"description\": \"min=173, mean=173, max=173, sum=346 (2)\", \"tab\": \"General information\", \"score\": \"173.0\"}", - "College Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # prompt tokens": "{\"description\": \"min=549.688, mean=549.688, max=549.688, sum=1099.376 (2)\", \"tab\": \"General information\", \"score\": \"549.6878612716763\"}", - "College Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Physics - # eval": "{\"description\": \"min=102, mean=102, max=102, sum=204 (2)\", \"tab\": \"General information\", \"score\": \"102.0\"}", - "College Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # prompt tokens": "{\"description\": \"min=512.912, mean=512.912, max=512.912, sum=1025.824 (2)\", \"tab\": \"General information\", \"score\": \"512.9117647058823\"}", - "College Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"college_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_college_physics\"" - } - } - }, - { - "evaluation_name": "Computer Security", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Computer Security", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.73, - "details": { - "description": "min=0.73, mean=0.73, max=0.73, sum=1.46 (2)", - "tab": "Accuracy", - "Computer Security - Observed inference time (s)": "{\"description\": \"min=0.336, mean=0.336, max=0.336, sum=0.673 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3364018177986145\"}", - "Computer Security - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Computer Security - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Computer Security - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Computer Security - # prompt tokens": "{\"description\": \"min=405.74, mean=405.74, max=405.74, sum=811.48 (2)\", \"tab\": \"General information\", \"score\": \"405.74\"}", - "Computer Security - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"computer_security\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_computer_security\"" - } - } - }, - { - "evaluation_name": "Econometrics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Econometrics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.351, - "details": { - "description": "min=0.351, mean=0.351, max=0.351, sum=0.702 (2)", - "tab": "Accuracy", - "Econometrics - Observed inference time (s)": "{\"description\": \"min=0.398, mean=0.398, max=0.398, sum=0.796 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.397992962285092\"}", - "Econometrics - # eval": "{\"description\": \"min=114, mean=114, max=114, sum=228 (2)\", \"tab\": \"General information\", \"score\": \"114.0\"}", - "Econometrics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Econometrics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Econometrics - # prompt tokens": "{\"description\": \"min=667.789, mean=667.789, max=667.789, sum=1335.579 (2)\", \"tab\": \"General information\", \"score\": \"667.7894736842105\"}", - "Econometrics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"econometrics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_econometrics\"" - } - } - }, - { - "evaluation_name": "Global Facts", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Global Facts", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.43, - "details": { - "description": "min=0.43, mean=0.43, max=0.43, sum=0.86 (2)", - "tab": "Accuracy", - "Global Facts - Observed inference time (s)": "{\"description\": \"min=0.327, mean=0.327, max=0.327, sum=0.655 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3273779916763306\"}", - "Global Facts - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Global Facts - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Global Facts - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Global Facts - # prompt tokens": "{\"description\": \"min=462.32, mean=462.32, max=462.32, sum=924.64 (2)\", \"tab\": \"General information\", \"score\": \"462.32\"}", - "Global Facts - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"global_facts\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_global_facts\"" - } - } - }, - { - "evaluation_name": "Jurisprudence", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Jurisprudence", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.796, - "details": { - "description": "min=0.796, mean=0.796, max=0.796, sum=1.593 (2)", - "tab": "Accuracy", - "Jurisprudence - Observed inference time (s)": "{\"description\": \"min=0.361, mean=0.361, max=0.361, sum=0.721 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3607365202020716\"}", - "Jurisprudence - # eval": "{\"description\": \"min=108, mean=108, max=108, sum=216 (2)\", \"tab\": \"General information\", \"score\": \"108.0\"}", - "Jurisprudence - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Jurisprudence - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Jurisprudence - # prompt tokens": "{\"description\": \"min=431.898, mean=431.898, max=431.898, sum=863.796 (2)\", \"tab\": \"General information\", \"score\": \"431.89814814814815\"}", - "Jurisprudence - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"jurisprudence\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_jurisprudence\"" - } - } - }, - { - "evaluation_name": "Philosophy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Philosophy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.678, - "details": { - "description": "min=0.678, mean=0.678, max=0.678, sum=1.357 (2)", - "tab": "Accuracy", - "Philosophy - Observed inference time (s)": "{\"description\": \"min=0.347, mean=0.347, max=0.347, sum=0.693 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.34667477807048047\"}", - "Philosophy - # eval": "{\"description\": \"min=311, mean=311, max=311, sum=622 (2)\", \"tab\": \"General information\", \"score\": \"311.0\"}", - "Philosophy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Philosophy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Philosophy - # prompt tokens": "{\"description\": \"min=356.723, mean=356.723, max=356.723, sum=713.447 (2)\", \"tab\": \"General information\", \"score\": \"356.7234726688103\"}", - "Philosophy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"philosophy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_philosophy\"" - } - } - }, - { - "evaluation_name": "Professional Psychology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Professional Psychology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.668, - "details": { - "description": "min=0.668, mean=0.668, max=0.668, sum=1.337 (2)", - "tab": "Accuracy", - "Professional Medicine - Observed inference time (s)": "{\"description\": \"min=0.769, mean=0.769, max=0.769, sum=1.538 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7688747907386106\"}", - "Professional Accounting - Observed inference time (s)": "{\"description\": \"min=0.37, mean=0.37, max=0.37, sum=0.74 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.37016247857546974\"}", - "Professional Law - Observed inference time (s)": "{\"description\": \"min=0.566, mean=0.566, max=0.566, sum=1.131 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5655125939084467\"}", - "Professional Psychology - Observed inference time (s)": "{\"description\": \"min=0.335, mean=0.335, max=0.335, sum=0.67 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.33476316071803275\"}", - "Professional Medicine - # eval": "{\"description\": \"min=272, mean=272, max=272, sum=544 (2)\", \"tab\": \"General information\", \"score\": \"272.0\"}", - "Professional Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Medicine - # prompt tokens": "{\"description\": \"min=1202.533, mean=1202.533, max=1202.533, sum=2405.066 (2)\", \"tab\": \"General information\", \"score\": \"1202.5330882352941\"}", - "Professional Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Accounting - # eval": "{\"description\": \"min=282, mean=282, max=282, sum=564 (2)\", \"tab\": \"General information\", \"score\": \"282.0\"}", - "Professional Accounting - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Accounting - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # prompt tokens": "{\"description\": \"min=771.16, mean=771.16, max=771.16, sum=1542.319 (2)\", \"tab\": \"General information\", \"score\": \"771.1595744680851\"}", - "Professional Accounting - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Law - # eval": "{\"description\": \"min=1534, mean=1534, max=1534, sum=3068 (2)\", \"tab\": \"General information\", \"score\": \"1534.0\"}", - "Professional Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # prompt tokens": "{\"description\": \"min=1759.098, mean=1759.098, max=1759.098, sum=3518.197 (2)\", \"tab\": \"General information\", \"score\": \"1759.0984354628422\"}", - "Professional Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Psychology - # eval": "{\"description\": \"min=612, mean=612, max=612, sum=1224 (2)\", \"tab\": \"General information\", \"score\": \"612.0\"}", - "Professional Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # prompt tokens": "{\"description\": \"min=608.201, mean=608.201, max=608.201, sum=1216.402 (2)\", \"tab\": \"General information\", \"score\": \"608.2009803921569\"}", - "Professional Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"professional_psychology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_professional_psychology\"" - } - } - }, - { - "evaluation_name": "Us Foreign Policy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Us Foreign Policy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.87, - "details": { - "description": "min=0.87, mean=0.87, max=0.87, sum=1.74 (2)", - "tab": "Accuracy", - "Us Foreign Policy - Observed inference time (s)": "{\"description\": \"min=0.384, mean=0.384, max=0.384, sum=0.768 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.38381587505340575\"}", - "Us Foreign Policy - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Us Foreign Policy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Us Foreign Policy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Us Foreign Policy - # prompt tokens": "{\"description\": \"min=458.53, mean=458.53, max=458.53, sum=917.06 (2)\", \"tab\": \"General information\", \"score\": \"458.53\"}", - "Us Foreign Policy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"us_foreign_policy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_us_foreign_policy\"" - } - } - }, - { - "evaluation_name": "Astronomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Astronomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.684, - "details": { - "description": "min=0.684, mean=0.684, max=0.684, sum=1.368 (2)", - "tab": "Accuracy", - "Astronomy - Observed inference time (s)": "{\"description\": \"min=0.351, mean=0.351, max=0.351, sum=0.702 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3511188610603935\"}", - "Astronomy - # eval": "{\"description\": \"min=152, mean=152, max=152, sum=304 (2)\", \"tab\": \"General information\", \"score\": \"152.0\"}", - "Astronomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Astronomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Astronomy - # prompt tokens": "{\"description\": \"min=626.895, mean=626.895, max=626.895, sum=1253.789 (2)\", \"tab\": \"General information\", \"score\": \"626.8947368421053\"}", - "Astronomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"astronomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_astronomy\"" - } - } - }, - { - "evaluation_name": "Business Ethics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Business Ethics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.67, - "details": { - "description": "min=0.67, mean=0.67, max=0.67, sum=1.34 (2)", - "tab": "Accuracy", - "Business Ethics - Observed inference time (s)": "{\"description\": \"min=0.335, mean=0.335, max=0.335, sum=0.671 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.33533199548721315\"}", - "Business Ethics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Business Ethics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Business Ethics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Business Ethics - # prompt tokens": "{\"description\": \"min=616.97, mean=616.97, max=616.97, sum=1233.94 (2)\", \"tab\": \"General information\", \"score\": \"616.97\"}", - "Business Ethics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"business_ethics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_business_ethics\"" - } - } - }, - { - "evaluation_name": "Clinical Knowledge", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Clinical Knowledge", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.66, - "details": { - "description": "min=0.66, mean=0.66, max=0.66, sum=1.321 (2)", - "tab": "Accuracy", - "Clinical Knowledge - Observed inference time (s)": "{\"description\": \"min=0.347, mean=0.347, max=0.347, sum=0.694 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.34722964808625995\"}", - "Clinical Knowledge - # eval": "{\"description\": \"min=265, mean=265, max=265, sum=530 (2)\", \"tab\": \"General information\", \"score\": \"265.0\"}", - "Clinical Knowledge - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Clinical Knowledge - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Clinical Knowledge - # prompt tokens": "{\"description\": \"min=446.966, mean=446.966, max=446.966, sum=893.932 (2)\", \"tab\": \"General information\", \"score\": \"446.96603773584906\"}", - "Clinical Knowledge - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"clinical_knowledge\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_clinical_knowledge\"" - } - } - }, - { - "evaluation_name": "Conceptual Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Conceptual Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.621, - "details": { - "description": "min=0.621, mean=0.621, max=0.621, sum=1.243 (2)", - "tab": "Accuracy", - "Conceptual Physics - Observed inference time (s)": "{\"description\": \"min=0.332, mean=0.332, max=0.332, sum=0.665 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3323540139705577\"}", - "Conceptual Physics - # eval": "{\"description\": \"min=235, mean=235, max=235, sum=470 (2)\", \"tab\": \"General information\", \"score\": \"235.0\"}", - "Conceptual Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Conceptual Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Conceptual Physics - # prompt tokens": "{\"description\": \"min=311.94, mean=311.94, max=311.94, sum=623.881 (2)\", \"tab\": \"General information\", \"score\": \"311.9404255319149\"}", - "Conceptual Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"conceptual_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_conceptual_physics\"" - } - } - }, - { - "evaluation_name": "Electrical Engineering", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Electrical Engineering", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.662, - "details": { - "description": "min=0.662, mean=0.662, max=0.662, sum=1.324 (2)", - "tab": "Accuracy", - "Electrical Engineering - Observed inference time (s)": "{\"description\": \"min=0.33, mean=0.33, max=0.33, sum=0.661 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.33032174274839204\"}", - "Electrical Engineering - # eval": "{\"description\": \"min=145, mean=145, max=145, sum=290 (2)\", \"tab\": \"General information\", \"score\": \"145.0\"}", - "Electrical Engineering - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Electrical Engineering - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Electrical Engineering - # prompt tokens": "{\"description\": \"min=491.993, mean=491.993, max=491.993, sum=983.986 (2)\", \"tab\": \"General information\", \"score\": \"491.99310344827586\"}", - "Electrical Engineering - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"electrical_engineering\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_electrical_engineering\"" - } - } - }, - { - "evaluation_name": "Elementary Mathematics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Elementary Mathematics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.452, - "details": { - "description": "min=0.452, mean=0.452, max=0.452, sum=0.905 (2)", - "tab": "Accuracy", - "Elementary Mathematics - Observed inference time (s)": "{\"description\": \"min=0.342, mean=0.342, max=0.342, sum=0.684 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.34218634310222806\"}", - "Elementary Mathematics - # eval": "{\"description\": \"min=378, mean=378, max=378, sum=756 (2)\", \"tab\": \"General information\", \"score\": \"378.0\"}", - "Elementary Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Elementary Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Elementary Mathematics - # prompt tokens": "{\"description\": \"min=601.344, mean=601.344, max=601.344, sum=1202.688 (2)\", \"tab\": \"General information\", \"score\": \"601.3439153439153\"}", - "Elementary Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"elementary_mathematics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_elementary_mathematics\"" - } - } - }, - { - "evaluation_name": "Formal Logic", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Formal Logic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.452, - "details": { - "description": "min=0.452, mean=0.452, max=0.452, sum=0.905 (2)", - "tab": "Accuracy", - "Formal Logic - Observed inference time (s)": "{\"description\": \"min=0.356, mean=0.356, max=0.356, sum=0.713 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3562947171075003\"}", - "Formal Logic - # eval": "{\"description\": \"min=126, mean=126, max=126, sum=252 (2)\", \"tab\": \"General information\", \"score\": \"126.0\"}", - "Formal Logic - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Formal Logic - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Formal Logic - # prompt tokens": "{\"description\": \"min=675.579, mean=675.579, max=675.579, sum=1351.159 (2)\", \"tab\": \"General information\", \"score\": \"675.5793650793651\"}", - "Formal Logic - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"formal_logic\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_formal_logic\"" - } - } - }, - { - "evaluation_name": "High School World History", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on High School World History", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.785, - "details": { - "description": "min=0.785, mean=0.785, max=0.785, sum=1.57 (2)", - "tab": "Accuracy", - "High School Biology - Observed inference time (s)": "{\"description\": \"min=0.343, mean=0.343, max=0.343, sum=0.685 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3425526588193832\"}", - "High School Chemistry - Observed inference time (s)": "{\"description\": \"min=0.334, mean=0.334, max=0.334, sum=0.667 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3337097426353417\"}", - "High School Computer Science - Observed inference time (s)": "{\"description\": \"min=0.411, mean=0.411, max=0.411, sum=0.822 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4111129188537598\"}", - "High School European History - Observed inference time (s)": "{\"description\": \"min=0.912, mean=0.912, max=0.912, sum=1.824 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9120050358049797\"}", - "High School Geography - Observed inference time (s)": "{\"description\": \"min=0.781, mean=0.781, max=0.781, sum=1.563 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7814190243229722\"}", - "High School Government And Politics - Observed inference time (s)": "{\"description\": \"min=0.344, mean=0.344, max=0.344, sum=0.688 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3440394698029355\"}", - "High School Macroeconomics - Observed inference time (s)": "{\"description\": \"min=0.336, mean=0.336, max=0.336, sum=0.672 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3361299728735899\"}", - "High School Mathematics - Observed inference time (s)": "{\"description\": \"min=0.365, mean=0.365, max=0.365, sum=0.73 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.36511756932293926\"}", - "High School Microeconomics - Observed inference time (s)": "{\"description\": \"min=0.335, mean=0.335, max=0.335, sum=0.67 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3350923071388437\"}", - "High School Physics - Observed inference time (s)": "{\"description\": \"min=0.363, mean=0.363, max=0.363, sum=0.727 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3634012266500107\"}", - "High School Psychology - Observed inference time (s)": "{\"description\": \"min=0.339, mean=0.339, max=0.339, sum=0.678 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3389187379714546\"}", - "High School Statistics - Observed inference time (s)": "{\"description\": \"min=0.384, mean=0.384, max=0.384, sum=0.767 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.38363339724364104\"}", - "High School US History - Observed inference time (s)": "{\"description\": \"min=0.661, mean=0.661, max=0.661, sum=1.322 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6610236086097419\"}", - "High School World History - Observed inference time (s)": "{\"description\": \"min=0.502, mean=0.502, max=0.502, sum=1.004 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5019015682397513\"}", - "High School Biology - # eval": "{\"description\": \"min=310, mean=310, max=310, sum=620 (2)\", \"tab\": \"General information\", \"score\": \"310.0\"}", - "High School Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Biology - # prompt tokens": "{\"description\": \"min=546.394, mean=546.394, max=546.394, sum=1092.787 (2)\", \"tab\": \"General information\", \"score\": \"546.3935483870968\"}", - "High School Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Chemistry - # eval": "{\"description\": \"min=203, mean=203, max=203, sum=406 (2)\", \"tab\": \"General information\", \"score\": \"203.0\"}", - "High School Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # prompt tokens": "{\"description\": \"min=537.015, mean=537.015, max=537.015, sum=1074.03 (2)\", \"tab\": \"General information\", \"score\": \"537.0147783251232\"}", - "High School Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "High School Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # prompt tokens": "{\"description\": \"min=962.1, mean=962.1, max=962.1, sum=1924.2 (2)\", \"tab\": \"General information\", \"score\": \"962.1\"}", - "High School Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School European History - # eval": "{\"description\": \"min=165, mean=165, max=165, sum=330 (2)\", \"tab\": \"General information\", \"score\": \"165.0\"}", - "High School European History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School European History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # prompt tokens": "{\"description\": \"min=2957.412, mean=2957.412, max=2957.412, sum=5914.824 (2)\", \"tab\": \"General information\", \"score\": \"2957.4121212121213\"}", - "High School European History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Geography - # eval": "{\"description\": \"min=198, mean=198, max=198, sum=396 (2)\", \"tab\": \"General information\", \"score\": \"198.0\"}", - "High School Geography - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Geography - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # prompt tokens": "{\"description\": \"min=404.035, mean=404.035, max=404.035, sum=808.071 (2)\", \"tab\": \"General information\", \"score\": \"404.0353535353535\"}", - "High School Geography - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Government And Politics - # eval": "{\"description\": \"min=193, mean=193, max=193, sum=386 (2)\", \"tab\": \"General information\", \"score\": \"193.0\"}", - "High School Government And Politics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Government And Politics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # prompt tokens": "{\"description\": \"min=484.725, mean=484.725, max=484.725, sum=969.451 (2)\", \"tab\": \"General information\", \"score\": \"484.7253886010363\"}", - "High School Government And Politics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Macroeconomics - # eval": "{\"description\": \"min=390, mean=390, max=390, sum=780 (2)\", \"tab\": \"General information\", \"score\": \"390.0\"}", - "High School Macroeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Macroeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # prompt tokens": "{\"description\": \"min=398.892, mean=398.892, max=398.892, sum=797.785 (2)\", \"tab\": \"General information\", \"score\": \"398.89230769230767\"}", - "High School Macroeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Mathematics - # eval": "{\"description\": \"min=270, mean=270, max=270, sum=540 (2)\", \"tab\": \"General information\", \"score\": \"270.0\"}", - "High School Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # prompt tokens": "{\"description\": \"min=575.622, mean=575.622, max=575.622, sum=1151.244 (2)\", \"tab\": \"General information\", \"score\": \"575.6222222222223\"}", - "High School Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Microeconomics - # eval": "{\"description\": \"min=238, mean=238, max=238, sum=476 (2)\", \"tab\": \"General information\", \"score\": \"238.0\"}", - "High School Microeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Microeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # prompt tokens": "{\"description\": \"min=420.739, mean=420.739, max=420.739, sum=841.479 (2)\", \"tab\": \"General information\", \"score\": \"420.73949579831935\"}", - "High School Microeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Physics - # eval": "{\"description\": \"min=151, mean=151, max=151, sum=302 (2)\", \"tab\": \"General information\", \"score\": \"151.0\"}", - "High School Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # prompt tokens": "{\"description\": \"min=599.411, mean=599.411, max=599.411, sum=1198.821 (2)\", \"tab\": \"General information\", \"score\": \"599.4105960264901\"}", - "High School Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Psychology - # eval": "{\"description\": \"min=545, mean=545, max=545, sum=1090 (2)\", \"tab\": \"General information\", \"score\": \"545.0\"}", - "High School Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # prompt tokens": "{\"description\": \"min=526.826, mean=526.826, max=526.826, sum=1053.651 (2)\", \"tab\": \"General information\", \"score\": \"526.8256880733945\"}", - "High School Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Statistics - # eval": "{\"description\": \"min=216, mean=216, max=216, sum=432 (2)\", \"tab\": \"General information\", \"score\": \"216.0\"}", - "High School Statistics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Statistics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # prompt tokens": "{\"description\": \"min=869.778, mean=869.778, max=869.778, sum=1739.556 (2)\", \"tab\": \"General information\", \"score\": \"869.7777777777778\"}", - "High School Statistics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School US History - # eval": "{\"description\": \"min=204, mean=204, max=204, sum=408 (2)\", \"tab\": \"General information\", \"score\": \"204.0\"}", - "High School US History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School US History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # prompt tokens": "{\"description\": \"min=2369.132, mean=2369.132, max=2369.132, sum=4738.265 (2)\", \"tab\": \"General information\", \"score\": \"2369.1323529411766\"}", - "High School US History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School World History - # eval": "{\"description\": \"min=237, mean=237, max=237, sum=474 (2)\", \"tab\": \"General information\", \"score\": \"237.0\"}", - "High School World History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School World History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # prompt tokens": "{\"description\": \"min=1541.371, mean=1541.371, max=1541.371, sum=3082.743 (2)\", \"tab\": \"General information\", \"score\": \"1541.3713080168777\"}", - "High School World History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"high_school_world_history\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_high_school_world_history\"" - } - } - }, - { - "evaluation_name": "Human Sexuality", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Human Sexuality", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.763, - "details": { - "description": "min=0.763, mean=0.763, max=0.763, sum=1.527 (2)", - "tab": "Accuracy", - "Human Aging - Observed inference time (s)": "{\"description\": \"min=0.346, mean=0.346, max=0.346, sum=0.691 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3457356803620343\"}", - "Human Sexuality - Observed inference time (s)": "{\"description\": \"min=0.322, mean=0.322, max=0.322, sum=0.645 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3222540717088539\"}", - "Human Aging - # eval": "{\"description\": \"min=223, mean=223, max=223, sum=446 (2)\", \"tab\": \"General information\", \"score\": \"223.0\"}", - "Human Aging - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Aging - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Aging - # prompt tokens": "{\"description\": \"min=332.013, mean=332.013, max=332.013, sum=664.027 (2)\", \"tab\": \"General information\", \"score\": \"332.0134529147982\"}", - "Human Aging - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Human Sexuality - # eval": "{\"description\": \"min=131, mean=131, max=131, sum=262 (2)\", \"tab\": \"General information\", \"score\": \"131.0\"}", - "Human Sexuality - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Sexuality - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # prompt tokens": "{\"description\": \"min=367.855, mean=367.855, max=367.855, sum=735.71 (2)\", \"tab\": \"General information\", \"score\": \"367.85496183206106\"}", - "Human Sexuality - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"human_sexuality\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_human_sexuality\"" - } - } - }, - { - "evaluation_name": "International Law", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on International Law", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.769, - "details": { - "description": "min=0.769, mean=0.769, max=0.769, sum=1.537 (2)", - "tab": "Accuracy", - "International Law - Observed inference time (s)": "{\"description\": \"min=0.356, mean=0.356, max=0.356, sum=0.711 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.35565017274588595\"}", - "International Law - # eval": "{\"description\": \"min=121, mean=121, max=121, sum=242 (2)\", \"tab\": \"General information\", \"score\": \"121.0\"}", - "International Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "International Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "International Law - # prompt tokens": "{\"description\": \"min=663.289, mean=663.289, max=663.289, sum=1326.579 (2)\", \"tab\": \"General information\", \"score\": \"663.2892561983471\"}", - "International Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"international_law\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_international_law\"" - } - } - }, - { - "evaluation_name": "Logical Fallacies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Logical Fallacies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.779, - "details": { - "description": "min=0.779, mean=0.779, max=0.779, sum=1.558 (2)", - "tab": "Accuracy", - "Logical Fallacies - Observed inference time (s)": "{\"description\": \"min=0.352, mean=0.352, max=0.352, sum=0.703 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3515900117487995\"}", - "Logical Fallacies - # eval": "{\"description\": \"min=163, mean=163, max=163, sum=326 (2)\", \"tab\": \"General information\", \"score\": \"163.0\"}", - "Logical Fallacies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Logical Fallacies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Logical Fallacies - # prompt tokens": "{\"description\": \"min=466.595, mean=466.595, max=466.595, sum=933.19 (2)\", \"tab\": \"General information\", \"score\": \"466.5950920245399\"}", - "Logical Fallacies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"logical_fallacies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_logical_fallacies\"" - } - } - }, - { - "evaluation_name": "Machine Learning", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Machine Learning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.411, - "details": { - "description": "min=0.411, mean=0.411, max=0.411, sum=0.821 (2)", - "tab": "Accuracy", - "Machine Learning - Observed inference time (s)": "{\"description\": \"min=0.355, mean=0.355, max=0.355, sum=0.71 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.35482590326241087\"}", - "Machine Learning - # eval": "{\"description\": \"min=112, mean=112, max=112, sum=224 (2)\", \"tab\": \"General information\", \"score\": \"112.0\"}", - "Machine Learning - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Machine Learning - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Machine Learning - # prompt tokens": "{\"description\": \"min=720.161, mean=720.161, max=720.161, sum=1440.321 (2)\", \"tab\": \"General information\", \"score\": \"720.1607142857143\"}", - "Machine Learning - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"machine_learning\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_machine_learning\"" - } - } - }, - { - "evaluation_name": "Management", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Management", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.806, - "details": { - "description": "min=0.806, mean=0.806, max=0.806, sum=1.612 (2)", - "tab": "Accuracy", - "Management - Observed inference time (s)": "{\"description\": \"min=0.337, mean=0.337, max=0.337, sum=0.674 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.33675998622931325\"}", - "Management - # eval": "{\"description\": \"min=103, mean=103, max=103, sum=206 (2)\", \"tab\": \"General information\", \"score\": \"103.0\"}", - "Management - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Management - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Management - # prompt tokens": "{\"description\": \"min=300.544, mean=300.544, max=300.544, sum=601.087 (2)\", \"tab\": \"General information\", \"score\": \"300.54368932038835\"}", - "Management - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"management\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_management\"" - } - } - }, - { - "evaluation_name": "Marketing", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Marketing", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.893, - "details": { - "description": "min=0.893, mean=0.893, max=0.893, sum=1.786 (2)", - "tab": "Accuracy", - "Marketing - Observed inference time (s)": "{\"description\": \"min=0.328, mean=0.328, max=0.328, sum=0.656 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3279143999784421\"}", - "Marketing - # eval": "{\"description\": \"min=234, mean=234, max=234, sum=468 (2)\", \"tab\": \"General information\", \"score\": \"234.0\"}", - "Marketing - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Marketing - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Marketing - # prompt tokens": "{\"description\": \"min=442.825, mean=442.825, max=442.825, sum=885.65 (2)\", \"tab\": \"General information\", \"score\": \"442.8247863247863\"}", - "Marketing - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"marketing\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_marketing\"" - } - } - }, - { - "evaluation_name": "Medical Genetics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Medical Genetics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.77, - "details": { - "description": "min=0.77, mean=0.77, max=0.77, sum=1.54 (2)", - "tab": "Accuracy", - "Medical Genetics - Observed inference time (s)": "{\"description\": \"min=0.372, mean=0.372, max=0.372, sum=0.744 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3717941379547119\"}", - "Medical Genetics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Medical Genetics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Medical Genetics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Medical Genetics - # prompt tokens": "{\"description\": \"min=362, mean=362, max=362, sum=724 (2)\", \"tab\": \"General information\", \"score\": \"362.0\"}", - "Medical Genetics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"medical_genetics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_medical_genetics\"" - } - } - }, - { - "evaluation_name": "Miscellaneous", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Miscellaneous", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.796, - "details": { - "description": "min=0.796, mean=0.796, max=0.796, sum=1.591 (2)", - "tab": "Accuracy", - "Miscellaneous - Observed inference time (s)": "{\"description\": \"min=0.317, mean=0.317, max=0.317, sum=0.634 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.31703713509619313\"}", - "Miscellaneous - # eval": "{\"description\": \"min=783, mean=783, max=783, sum=1566 (2)\", \"tab\": \"General information\", \"score\": \"783.0\"}", - "Miscellaneous - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Miscellaneous - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Miscellaneous - # prompt tokens": "{\"description\": \"min=331.441, mean=331.441, max=331.441, sum=662.881 (2)\", \"tab\": \"General information\", \"score\": \"331.4406130268199\"}", - "Miscellaneous - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"miscellaneous\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_miscellaneous\"" - } - } - }, - { - "evaluation_name": "Moral Scenarios", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Moral Scenarios", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.335, - "details": { - "description": "min=0.335, mean=0.335, max=0.335, sum=0.67 (2)", - "tab": "Accuracy", - "Moral Disputes - Observed inference time (s)": "{\"description\": \"min=0.321, mean=0.321, max=0.321, sum=0.643 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3214432848671268\"}", - "Moral Scenarios - Observed inference time (s)": "{\"description\": \"min=0.342, mean=0.342, max=0.342, sum=0.684 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3421009585844072\"}", - "Moral Disputes - # eval": "{\"description\": \"min=346, mean=346, max=346, sum=692 (2)\", \"tab\": \"General information\", \"score\": \"346.0\"}", - "Moral Disputes - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Disputes - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Disputes - # prompt tokens": "{\"description\": \"min=507.913, mean=507.913, max=507.913, sum=1015.827 (2)\", \"tab\": \"General information\", \"score\": \"507.91329479768785\"}", - "Moral Disputes - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Moral Scenarios - # eval": "{\"description\": \"min=895, mean=895, max=895, sum=1790 (2)\", \"tab\": \"General information\", \"score\": \"895.0\"}", - "Moral Scenarios - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Scenarios - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # prompt tokens": "{\"description\": \"min=703.334, mean=703.334, max=703.334, sum=1406.668 (2)\", \"tab\": \"General information\", \"score\": \"703.3340782122905\"}", - "Moral Scenarios - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"moral_scenarios\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_moral_scenarios\"" - } - } - }, - { - "evaluation_name": "Nutrition", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Nutrition", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.739, - "details": { - "description": "min=0.739, mean=0.739, max=0.739, sum=1.477 (2)", - "tab": "Accuracy", - "Nutrition - Observed inference time (s)": "{\"description\": \"min=0.354, mean=0.354, max=0.354, sum=0.708 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.35382014474058465\"}", - "Nutrition - # eval": "{\"description\": \"min=306, mean=306, max=306, sum=612 (2)\", \"tab\": \"General information\", \"score\": \"306.0\"}", - "Nutrition - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Nutrition - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Nutrition - # prompt tokens": "{\"description\": \"min=643.317, mean=643.317, max=643.317, sum=1286.634 (2)\", \"tab\": \"General information\", \"score\": \"643.3169934640523\"}", - "Nutrition - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"nutrition\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_nutrition\"" - } - } - }, - { - "evaluation_name": "Prehistory", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Prehistory", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.713, - "details": { - "description": "min=0.713, mean=0.713, max=0.713, sum=1.426 (2)", - "tab": "Accuracy", - "Prehistory - Observed inference time (s)": "{\"description\": \"min=0.358, mean=0.358, max=0.358, sum=0.715 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3577412587625009\"}", - "Prehistory - # eval": "{\"description\": \"min=324, mean=324, max=324, sum=648 (2)\", \"tab\": \"General information\", \"score\": \"324.0\"}", - "Prehistory - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Prehistory - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Prehistory - # prompt tokens": "{\"description\": \"min=565.096, mean=565.096, max=565.096, sum=1130.191 (2)\", \"tab\": \"General information\", \"score\": \"565.0956790123457\"}", - "Prehistory - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"prehistory\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_prehistory\"" - } - } - }, - { - "evaluation_name": "Public Relations", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Public Relations", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.718, - "details": { - "description": "min=0.718, mean=0.718, max=0.718, sum=1.436 (2)", - "tab": "Accuracy", - "Public Relations - Observed inference time (s)": "{\"description\": \"min=0.352, mean=0.352, max=0.352, sum=0.704 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.35222616412422875\"}", - "Public Relations - # eval": "{\"description\": \"min=110, mean=110, max=110, sum=220 (2)\", \"tab\": \"General information\", \"score\": \"110.0\"}", - "Public Relations - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Public Relations - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Public Relations - # prompt tokens": "{\"description\": \"min=432.436, mean=432.436, max=432.436, sum=864.873 (2)\", \"tab\": \"General information\", \"score\": \"432.43636363636364\"}", - "Public Relations - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"public_relations\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_public_relations\"" - } - } - }, - { - "evaluation_name": "Security Studies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Security Studies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.735, - "details": { - "description": "min=0.735, mean=0.735, max=0.735, sum=1.469 (2)", - "tab": "Accuracy", - "Security Studies - Observed inference time (s)": "{\"description\": \"min=0.439, mean=0.439, max=0.439, sum=0.877 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4387260553788166\"}", - "Security Studies - # eval": "{\"description\": \"min=245, mean=245, max=245, sum=490 (2)\", \"tab\": \"General information\", \"score\": \"245.0\"}", - "Security Studies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Security Studies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Security Studies - # prompt tokens": "{\"description\": \"min=1227.196, mean=1227.196, max=1227.196, sum=2454.392 (2)\", \"tab\": \"General information\", \"score\": \"1227.1959183673468\"}", - "Security Studies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"security_studies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_security_studies\"" - } - } - }, - { - "evaluation_name": "Sociology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Sociology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.831, - "details": { - "description": "min=0.831, mean=0.831, max=0.831, sum=1.662 (2)", - "tab": "Accuracy", - "Sociology - Observed inference time (s)": "{\"description\": \"min=0.315, mean=0.315, max=0.315, sum=0.63 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.31509182820865766\"}", - "Sociology - # eval": "{\"description\": \"min=201, mean=201, max=201, sum=402 (2)\", \"tab\": \"General information\", \"score\": \"201.0\"}", - "Sociology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Sociology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Sociology - # prompt tokens": "{\"description\": \"min=463.99, mean=463.99, max=463.99, sum=927.98 (2)\", \"tab\": \"General information\", \"score\": \"463.99004975124376\"}", - "Sociology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"sociology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_sociology\"" - } - } - }, - { - "evaluation_name": "Virology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Virology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.452, - "details": { - "description": "min=0.452, mean=0.452, max=0.452, sum=0.904 (2)", - "tab": "Accuracy", - "Virology - Observed inference time (s)": "{\"description\": \"min=0.352, mean=0.352, max=0.352, sum=0.705 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3524869034089238\"}", - "Virology - # eval": "{\"description\": \"min=166, mean=166, max=166, sum=332 (2)\", \"tab\": \"General information\", \"score\": \"166.0\"}", - "Virology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Virology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Virology - # prompt tokens": "{\"description\": \"min=363.102, mean=363.102, max=363.102, sum=726.205 (2)\", \"tab\": \"General information\", \"score\": \"363.1024096385542\"}", - "Virology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"virology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_virology\"" - } - } - }, - { - "evaluation_name": "World Religions", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on World Religions", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.836, - "details": { - "description": "min=0.836, mean=0.836, max=0.836, sum=1.673 (2)", - "tab": "Accuracy", - "World Religions - Observed inference time (s)": "{\"description\": \"min=0.343, mean=0.343, max=0.343, sum=0.687 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.34344731576261467\"}", - "World Religions - # eval": "{\"description\": \"min=171, mean=171, max=171, sum=342 (2)\", \"tab\": \"General information\", \"score\": \"171.0\"}", - "World Religions - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "World Religions - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "World Religions - # prompt tokens": "{\"description\": \"min=289.971, mean=289.971, max=289.971, sum=579.942 (2)\", \"tab\": \"General information\", \"score\": \"289.97076023391816\"}", - "World Religions - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"world_religions\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_world_religions\"" - } - } - }, - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.651, - "details": { - "description": "", - "tab": "Efficiency" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_id": "hfopenllm_v2/01-ai_Yi-6B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2893 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4309 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0159 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2693 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3937 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2991 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/01-ai_yi-9b-200k.json b/data/models/01-ai_yi-9b-200k.json deleted file mode 100644 index b26af9aeb160e8841d54a862c7b4b4db105056f3..0000000000000000000000000000000000000000 --- a/data/models/01-ai_yi-9b-200k.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Yi-9B-200K", - "id": "01-ai/Yi-9B-200K", - "developer": "01-ai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.829" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/01-ai_Yi-9B-200K/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2327 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4793 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0665 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3154 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4294 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3622 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/01-ai_yi-9b.json b/data/models/01-ai_yi-9b.json deleted file mode 100644 index 7026594e54c8afd3871989fb53d017024891a39a..0000000000000000000000000000000000000000 --- a/data/models/01-ai_yi-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Yi-9B", - "id": "01-ai/Yi-9B", - "developer": "01-ai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.829" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/01-ai_Yi-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2709 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.494 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0559 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.318 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4054 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3574 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/01-ai_yi-coder-9b-chat.json b/data/models/01-ai_yi-coder-9b-chat.json deleted file mode 100644 index 9b3487bbbd04537da8f6975e31f14263d57da554..0000000000000000000000000000000000000000 --- a/data/models/01-ai_yi-coder-9b-chat.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Yi-Coder-9B-Chat", - "id": "01-ai/Yi-Coder-9B-Chat", - "developer": "01-ai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.829" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/01-ai_Yi-Coder-9B-Chat/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4817 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4814 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.04 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2475 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3992 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2425 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/01-ai_yi-large-preview.json b/data/models/01-ai_yi-large-preview.json deleted file mode 100644 index 12695c48eb73fee9e0a78d4028bf264adfa3b3f2..0000000000000000000000000000000000000000 --- a/data/models/01-ai_yi-large-preview.json +++ /dev/null @@ -1,1903 +0,0 @@ -{ - "model_info": { - "name": "Yi Large Preview", - "id": "01-ai/yi-large-preview", - "developer": "01-ai", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_lite/01-ai_yi-large-preview/1774096306.427425", - "retrieved_timestamp": "1774096306.427425", - "source_metadata": { - "source_name": "helm_lite", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_lite", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_lite", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.471, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.17893882646691636\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.373, - "details": { - "description": "min=0.373, mean=0.373, max=0.373, sum=0.373 (1)", - "tab": "Accuracy", - "NarrativeQA - Observed inference time (s)": "{\"description\": \"min=2.672, mean=2.672, max=2.672, sum=2.672 (1)\", \"tab\": \"Efficiency\", \"score\": \"2.6724000897206053\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=3724.042, mean=3724.042, max=3724.042, sum=3724.042 (1)\", \"tab\": \"General information\", \"score\": \"3724.042253521127\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=21.513, mean=21.513, max=21.513, sum=21.513 (1)\", \"tab\": \"General information\", \"score\": \"21.512676056338027\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (closed-book)", - "source_data": { - "dataset_name": "NaturalQuestions (closed-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (closed-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.428, - "details": { - "description": "min=0.428, mean=0.428, max=0.428, sum=0.428 (1)", - "tab": "Accuracy", - "NaturalQuestions (open-book) - Observed inference time (s)": "{\"description\": \"min=2.506, mean=2.506, max=2.506, sum=2.506 (1)\", \"tab\": \"Efficiency\", \"score\": \"2.506305232524872\"}", - "NaturalQuestions (closed-book) - Observed inference time (s)": "{\"description\": \"min=1.036, mean=1.036, max=1.036, sum=1.036 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.0360134015083313\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.983, mean=4.983, max=4.983, sum=4.983 (1)\", \"tab\": \"General information\", \"score\": \"4.983\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.003, mean=0.003, max=0.003, sum=0.003 (1)\", \"tab\": \"General information\", \"score\": \"0.003\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=2368.513, mean=2368.513, max=2368.513, sum=2368.513 (1)\", \"tab\": \"General information\", \"score\": \"2368.513\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=23.703, mean=23.703, max=23.703, sum=23.703 (1)\", \"tab\": \"General information\", \"score\": \"23.703\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=160.695, mean=160.695, max=160.695, sum=160.695 (1)\", \"tab\": \"General information\", \"score\": \"160.695\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=4.629, mean=4.629, max=4.629, sum=4.629 (1)\", \"tab\": \"General information\", \"score\": \"4.629\"}" - } - }, - "generation_config": { - "additional_details": { - "mode": "\"closedbook\"" - } - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.946, - "details": { - "description": "min=0.946, mean=0.946, max=0.946, sum=0.946 (1)", - "tab": "Accuracy", - "OpenbookQA - Observed inference time (s)": "{\"description\": \"min=0.777, mean=0.777, max=0.777, sum=0.777 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.77673295545578\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=273.002, mean=273.002, max=273.002, sum=273.002 (1)\", \"tab\": \"General information\", \"score\": \"273.002\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "dataset": "\"openbookqa\"", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.712, - "details": { - "description": "min=0.52, mean=0.712, max=0.86, sum=3.558 (5)", - "tab": "Accuracy", - "MMLU - Observed inference time (s)": "{\"description\": \"min=0.679, mean=0.713, max=0.752, sum=3.567 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.7133434140138459\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=396.67, mean=515.654, max=680.789, sum=2578.269 (5)\", \"tab\": \"General information\", \"score\": \"515.6538947368421\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"college_chemistry\", \"computer_security\", \"econometrics\", \"us_foreign_policy\"]", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MATH", - "source_data": { - "dataset_name": "MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Equivalent (CoT) on MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.712, - "details": { - "description": "min=0.553, mean=0.712, max=0.874, sum=4.982 (7)", - "tab": "Accuracy", - "MATH - Observed inference time (s)": "{\"description\": \"min=8.67, mean=11.511, max=13.559, sum=80.577 (7)\", \"tab\": \"Efficiency\", \"score\": \"11.510960669458308\"}", - "MATH - # eval": "{\"description\": \"min=30, mean=62.429, max=135, sum=437 (7)\", \"tab\": \"General information\", \"score\": \"62.42857142857143\"}", - "MATH - # train": "{\"description\": \"min=8, mean=8, max=8, sum=56 (7)\", \"tab\": \"General information\", \"score\": \"8.0\"}", - "MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MATH - # prompt tokens": "{\"description\": \"min=976.696, mean=1468.935, max=2582.038, sum=10282.547 (7)\", \"tab\": \"General information\", \"score\": \"1468.9352369693863\"}", - "MATH - # output tokens": "{\"description\": \"min=189.756, mean=254.005, max=296.346, sum=1778.034 (7)\", \"tab\": \"General information\", \"score\": \"254.00484808722263\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"algebra\", \"counting_and_probability\", \"geometry\", \"intermediate_algebra\", \"number_theory\", \"prealgebra\", \"precalculus\"]", - "level": "\"1\"", - "use_official_examples": "\"False\"", - "use_chain_of_thought": "\"True\"" - } - } - }, - { - "evaluation_name": "GSM8K", - "source_data": { - "dataset_name": "GSM8K", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on GSM8K", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.69, - "details": { - "description": "min=0.69, mean=0.69, max=0.69, sum=0.69 (1)", - "tab": "Accuracy", - "GSM8K - Observed inference time (s)": "{\"description\": \"min=13.45, mean=13.45, max=13.45, sum=13.45 (1)\", \"tab\": \"Efficiency\", \"score\": \"13.45040065407753\"}", - "GSM8K - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "GSM8K - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "GSM8K - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GSM8K - # prompt tokens": "{\"description\": \"min=1170.814, mean=1170.814, max=1170.814, sum=1170.814 (1)\", \"tab\": \"General information\", \"score\": \"1170.814\"}", - "GSM8K - # output tokens": "{\"description\": \"min=288.079, mean=288.079, max=288.079, sum=288.079 (1)\", \"tab\": \"General information\", \"score\": \"288.079\"}" - } - }, - "generation_config": { - "additional_details": { - "stop": "\"none\"" - } - } - }, - { - "evaluation_name": "LegalBench", - "source_data": { - "dataset_name": "LegalBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on LegalBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.519, - "details": { - "description": "min=0.145, mean=0.519, max=0.884, sum=2.594 (5)", - "tab": "Accuracy", - "LegalBench - Observed inference time (s)": "{\"description\": \"min=0.855, mean=1.472, max=3.502, sum=7.358 (5)\", \"tab\": \"Efficiency\", \"score\": \"1.471592522464795\"}", - "LegalBench - # eval": "{\"description\": \"min=95, mean=409.4, max=1000, sum=2047 (5)\", \"tab\": \"General information\", \"score\": \"409.4\"}", - "LegalBench - # train": "{\"description\": \"min=4, mean=4.8, max=5, sum=24 (5)\", \"tab\": \"General information\", \"score\": \"4.8\"}", - "LegalBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "LegalBench - # prompt tokens": "{\"description\": \"min=228.779, mean=1656.095, max=6814.4, sum=8280.475 (5)\", \"tab\": \"General information\", \"score\": \"1656.0949044887425\"}", - "LegalBench - # output tokens": "{\"description\": \"min=1, mean=3.339, max=6.263, sum=16.697 (5)\", \"tab\": \"General information\", \"score\": \"3.339402150569105\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"abercrombie\", \"corporate_lobbying\", \"function_of_decision_section\", \"international_citizenship_questions\", \"proa\"]" - } - } - }, - { - "evaluation_name": "MedQA", - "source_data": { - "dataset_name": "MedQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MedQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.66, - "details": { - "description": "min=0.66, mean=0.66, max=0.66, sum=0.66 (1)", - "tab": "Accuracy", - "MedQA - Observed inference time (s)": "{\"description\": \"min=0.993, mean=0.993, max=0.993, sum=0.993 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.9931588552107157\"}", - "MedQA - # eval": "{\"description\": \"min=503, mean=503, max=503, sum=503 (1)\", \"tab\": \"General information\", \"score\": \"503.0\"}", - "MedQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MedQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MedQA - # prompt tokens": "{\"description\": \"min=1135.392, mean=1135.392, max=1135.392, sum=1135.392 (1)\", \"tab\": \"General information\", \"score\": \"1135.3916500994035\"}", - "MedQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WMT 2014", - "source_data": { - "dataset_name": "WMT 2014", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "BLEU-4 on WMT 2014", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.176, - "details": { - "description": "min=0.126, mean=0.176, max=0.218, sum=0.88 (5)", - "tab": "Accuracy", - "WMT 2014 - Observed inference time (s)": "{\"description\": \"min=1.838, mean=2.095, max=2.409, sum=10.477 (5)\", \"tab\": \"Efficiency\", \"score\": \"2.095412739007152\"}", - "WMT 2014 - # eval": "{\"description\": \"min=503, mean=568.8, max=832, sum=2844 (5)\", \"tab\": \"General information\", \"score\": \"568.8\"}", - "WMT 2014 - # train": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "WMT 2014 - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WMT 2014 - # prompt tokens": "{\"description\": \"min=157.298, mean=205.092, max=335.56, sum=1025.461 (5)\", \"tab\": \"General information\", \"score\": \"205.09213851506343\"}", - "WMT 2014 - # output tokens": "{\"description\": \"min=24.773, mean=29.058, max=36.698, sum=145.291 (5)\", \"tab\": \"General information\", \"score\": \"29.058130065759293\"}" - } - }, - "generation_config": { - "additional_details": { - "language_pair": "[\"cs-en\", \"de-en\", \"fr-en\", \"hi-en\", \"ru-en\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_mmlu/01-ai_yi-large-preview/1774096312.00548", - "retrieved_timestamp": "1774096312.00548", - "source_metadata": { - "source_name": "helm_mmlu", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_mmlu", - "evaluation_results": [ - { - "evaluation_name": "MMLU All Subjects", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU All Subjects", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.793, - "details": { - "description": "min=0.36, mean=0.793, max=0.969, sum=90.428 (114)", - "tab": "Accuracy", - "MMLU All Subjects - Observed inference time (s)": "{\"description\": \"min=0.621, mean=0.764, max=1.689, sum=87.08 (114)\", \"tab\": \"Efficiency\", \"score\": \"0.7638553584278898\"}", - "MMLU All Subjects - # eval": "{\"description\": \"min=100, mean=246.351, max=1534, sum=28084 (114)\", \"tab\": \"General information\", \"score\": \"246.35087719298247\"}", - "MMLU All Subjects - # train": "{\"description\": \"min=5, mean=5, max=5, sum=570 (114)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU All Subjects - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU All Subjects - # prompt tokens": "{\"description\": \"min=302.971, mean=674.842, max=2970.412, sum=76931.942 (114)\", \"tab\": \"General information\", \"score\": \"674.8416008681387\"}", - "MMLU All Subjects - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=114 (114)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_name": "Abstract Algebra", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Abstract Algebra", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6, - "details": { - "description": "min=0.6, mean=0.6, max=0.6, sum=1.2 (2)", - "tab": "Accuracy", - "Abstract Algebra - Observed inference time (s)": "{\"description\": \"min=0.718, mean=0.718, max=0.718, sum=1.436 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.718058660030365\"}", - "Abstract Algebra - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Abstract Algebra - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Abstract Algebra - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Abstract Algebra - # prompt tokens": "{\"description\": \"min=396.67, mean=396.67, max=396.67, sum=793.34 (2)\", \"tab\": \"General information\", \"score\": \"396.67\"}", - "Abstract Algebra - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"abstract_algebra\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_abstract_algebra\"" - } - } - }, - { - "evaluation_name": "Anatomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Anatomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.83, - "details": { - "description": "min=0.83, mean=0.83, max=0.83, sum=1.659 (2)", - "tab": "Accuracy", - "Anatomy - Observed inference time (s)": "{\"description\": \"min=0.672, mean=0.672, max=0.672, sum=1.343 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6716545846727159\"}", - "Anatomy - # eval": "{\"description\": \"min=135, mean=135, max=135, sum=270 (2)\", \"tab\": \"General information\", \"score\": \"135.0\"}", - "Anatomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Anatomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Anatomy - # prompt tokens": "{\"description\": \"min=388.77, mean=388.77, max=388.77, sum=777.541 (2)\", \"tab\": \"General information\", \"score\": \"388.77037037037036\"}", - "Anatomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"anatomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_anatomy\"" - } - } - }, - { - "evaluation_name": "College Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on College Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.569, - "details": { - "description": "min=0.569, mean=0.569, max=0.569, sum=1.137 (2)", - "tab": "Accuracy", - "College Chemistry - Observed inference time (s)": "{\"description\": \"min=0.722, mean=0.722, max=0.722, sum=1.443 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.721672637462616\"}", - "College Biology - Observed inference time (s)": "{\"description\": \"min=0.72, mean=0.72, max=0.72, sum=1.439 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7195867978864245\"}", - "College Computer Science - Observed inference time (s)": "{\"description\": \"min=0.828, mean=0.828, max=0.828, sum=1.657 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8283914875984192\"}", - "College Mathematics - Observed inference time (s)": "{\"description\": \"min=0.734, mean=0.734, max=0.734, sum=1.468 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.734215636253357\"}", - "College Medicine - Observed inference time (s)": "{\"description\": \"min=0.704, mean=0.704, max=0.704, sum=1.407 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7037480470073016\"}", - "College Physics - Observed inference time (s)": "{\"description\": \"min=0.742, mean=0.742, max=0.742, sum=1.484 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7418750898510802\"}", - "College Chemistry - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Chemistry - # prompt tokens": "{\"description\": \"min=610.54, mean=610.54, max=610.54, sum=1221.08 (2)\", \"tab\": \"General information\", \"score\": \"610.54\"}", - "College Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Biology - # eval": "{\"description\": \"min=144, mean=144, max=144, sum=288 (2)\", \"tab\": \"General information\", \"score\": \"144.0\"}", - "College Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # prompt tokens": "{\"description\": \"min=527.819, mean=527.819, max=527.819, sum=1055.639 (2)\", \"tab\": \"General information\", \"score\": \"527.8194444444445\"}", - "College Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # prompt tokens": "{\"description\": \"min=896.06, mean=896.06, max=896.06, sum=1792.12 (2)\", \"tab\": \"General information\", \"score\": \"896.06\"}", - "College Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Mathematics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # prompt tokens": "{\"description\": \"min=648.3, mean=648.3, max=648.3, sum=1296.6 (2)\", \"tab\": \"General information\", \"score\": \"648.3\"}", - "College Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Medicine - # eval": "{\"description\": \"min=173, mean=173, max=173, sum=346 (2)\", \"tab\": \"General information\", \"score\": \"173.0\"}", - "College Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # prompt tokens": "{\"description\": \"min=562.688, mean=562.688, max=562.688, sum=1125.376 (2)\", \"tab\": \"General information\", \"score\": \"562.6878612716763\"}", - "College Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Physics - # eval": "{\"description\": \"min=102, mean=102, max=102, sum=204 (2)\", \"tab\": \"General information\", \"score\": \"102.0\"}", - "College Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # prompt tokens": "{\"description\": \"min=525.912, mean=525.912, max=525.912, sum=1051.824 (2)\", \"tab\": \"General information\", \"score\": \"525.9117647058823\"}", - "College Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"college_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_college_physics\"" - } - } - }, - { - "evaluation_name": "Computer Security", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Computer Security", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.86, - "details": { - "description": "min=0.86, mean=0.86, max=0.86, sum=1.72 (2)", - "tab": "Accuracy", - "Computer Security - Observed inference time (s)": "{\"description\": \"min=0.679, mean=0.679, max=0.679, sum=1.358 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6791670727729797\"}", - "Computer Security - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Computer Security - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Computer Security - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Computer Security - # prompt tokens": "{\"description\": \"min=418.74, mean=418.74, max=418.74, sum=837.48 (2)\", \"tab\": \"General information\", \"score\": \"418.74\"}", - "Computer Security - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"computer_security\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_computer_security\"" - } - } - }, - { - "evaluation_name": "Econometrics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Econometrics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.728, - "details": { - "description": "min=0.728, mean=0.728, max=0.728, sum=1.456 (2)", - "tab": "Accuracy", - "Econometrics - Observed inference time (s)": "{\"description\": \"min=0.752, mean=0.752, max=0.752, sum=1.504 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7519724473618624\"}", - "Econometrics - # eval": "{\"description\": \"min=114, mean=114, max=114, sum=228 (2)\", \"tab\": \"General information\", \"score\": \"114.0\"}", - "Econometrics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Econometrics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Econometrics - # prompt tokens": "{\"description\": \"min=680.789, mean=680.789, max=680.789, sum=1361.579 (2)\", \"tab\": \"General information\", \"score\": \"680.7894736842105\"}", - "Econometrics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"econometrics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_econometrics\"" - } - } - }, - { - "evaluation_name": "Global Facts", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Global Facts", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.52, - "details": { - "description": "min=0.52, mean=0.52, max=0.52, sum=1.04 (2)", - "tab": "Accuracy", - "Global Facts - Observed inference time (s)": "{\"description\": \"min=0.7, mean=0.7, max=0.7, sum=1.401 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7004458856582642\"}", - "Global Facts - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Global Facts - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Global Facts - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Global Facts - # prompt tokens": "{\"description\": \"min=475.32, mean=475.32, max=475.32, sum=950.64 (2)\", \"tab\": \"General information\", \"score\": \"475.32\"}", - "Global Facts - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"global_facts\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_global_facts\"" - } - } - }, - { - "evaluation_name": "Jurisprudence", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Jurisprudence", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.852, - "details": { - "description": "min=0.852, mean=0.852, max=0.852, sum=1.704 (2)", - "tab": "Accuracy", - "Jurisprudence - Observed inference time (s)": "{\"description\": \"min=0.709, mean=0.709, max=0.709, sum=1.417 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7087078028255038\"}", - "Jurisprudence - # eval": "{\"description\": \"min=108, mean=108, max=108, sum=216 (2)\", \"tab\": \"General information\", \"score\": \"108.0\"}", - "Jurisprudence - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Jurisprudence - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Jurisprudence - # prompt tokens": "{\"description\": \"min=444.898, mean=444.898, max=444.898, sum=889.796 (2)\", \"tab\": \"General information\", \"score\": \"444.89814814814815\"}", - "Jurisprudence - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"jurisprudence\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_jurisprudence\"" - } - } - }, - { - "evaluation_name": "Philosophy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Philosophy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.842, - "details": { - "description": "min=0.842, mean=0.842, max=0.842, sum=1.685 (2)", - "tab": "Accuracy", - "Philosophy - Observed inference time (s)": "{\"description\": \"min=0.665, mean=0.665, max=0.665, sum=1.33 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6652177269435772\"}", - "Philosophy - # eval": "{\"description\": \"min=311, mean=311, max=311, sum=622 (2)\", \"tab\": \"General information\", \"score\": \"311.0\"}", - "Philosophy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Philosophy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Philosophy - # prompt tokens": "{\"description\": \"min=369.723, mean=369.723, max=369.723, sum=739.447 (2)\", \"tab\": \"General information\", \"score\": \"369.7234726688103\"}", - "Philosophy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"philosophy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_philosophy\"" - } - } - }, - { - "evaluation_name": "Professional Psychology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Professional Psychology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.853, - "details": { - "description": "min=0.853, mean=0.853, max=0.853, sum=1.706 (2)", - "tab": "Accuracy", - "Professional Medicine - Observed inference time (s)": "{\"description\": \"min=0.906, mean=0.906, max=0.906, sum=1.813 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9064707010984421\"}", - "Professional Accounting - Observed inference time (s)": "{\"description\": \"min=0.774, mean=0.774, max=0.774, sum=1.549 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7743352516323116\"}", - "Professional Law - Observed inference time (s)": "{\"description\": \"min=1.112, mean=1.112, max=1.112, sum=2.224 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.1117667775732287\"}", - "Professional Psychology - Observed inference time (s)": "{\"description\": \"min=0.729, mean=0.729, max=0.729, sum=1.458 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7289925248794307\"}", - "Professional Medicine - # eval": "{\"description\": \"min=272, mean=272, max=272, sum=544 (2)\", \"tab\": \"General information\", \"score\": \"272.0\"}", - "Professional Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Medicine - # prompt tokens": "{\"description\": \"min=1215.533, mean=1215.533, max=1215.533, sum=2431.066 (2)\", \"tab\": \"General information\", \"score\": \"1215.5330882352941\"}", - "Professional Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Accounting - # eval": "{\"description\": \"min=282, mean=282, max=282, sum=564 (2)\", \"tab\": \"General information\", \"score\": \"282.0\"}", - "Professional Accounting - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Accounting - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # prompt tokens": "{\"description\": \"min=784.16, mean=784.16, max=784.16, sum=1568.319 (2)\", \"tab\": \"General information\", \"score\": \"784.1595744680851\"}", - "Professional Accounting - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Law - # eval": "{\"description\": \"min=1534, mean=1534, max=1534, sum=3068 (2)\", \"tab\": \"General information\", \"score\": \"1534.0\"}", - "Professional Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # prompt tokens": "{\"description\": \"min=1772.098, mean=1772.098, max=1772.098, sum=3544.197 (2)\", \"tab\": \"General information\", \"score\": \"1772.0984354628422\"}", - "Professional Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Psychology - # eval": "{\"description\": \"min=612, mean=612, max=612, sum=1224 (2)\", \"tab\": \"General information\", \"score\": \"612.0\"}", - "Professional Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # prompt tokens": "{\"description\": \"min=621.201, mean=621.201, max=621.201, sum=1242.402 (2)\", \"tab\": \"General information\", \"score\": \"621.2009803921569\"}", - "Professional Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"professional_psychology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_professional_psychology\"" - } - } - }, - { - "evaluation_name": "Us Foreign Policy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Us Foreign Policy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.85, - "details": { - "description": "min=0.85, mean=0.85, max=0.85, sum=1.7 (2)", - "tab": "Accuracy", - "Us Foreign Policy - Observed inference time (s)": "{\"description\": \"min=0.696, mean=0.696, max=0.696, sum=1.392 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6958462524414063\"}", - "Us Foreign Policy - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Us Foreign Policy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Us Foreign Policy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Us Foreign Policy - # prompt tokens": "{\"description\": \"min=471.53, mean=471.53, max=471.53, sum=943.06 (2)\", \"tab\": \"General information\", \"score\": \"471.53\"}", - "Us Foreign Policy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"us_foreign_policy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_us_foreign_policy\"" - } - } - }, - { - "evaluation_name": "Astronomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Astronomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.914, - "details": { - "description": "min=0.914, mean=0.914, max=0.914, sum=1.829 (2)", - "tab": "Accuracy", - "Astronomy - Observed inference time (s)": "{\"description\": \"min=0.76, mean=0.76, max=0.76, sum=1.521 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7604575784582841\"}", - "Astronomy - # eval": "{\"description\": \"min=152, mean=152, max=152, sum=304 (2)\", \"tab\": \"General information\", \"score\": \"152.0\"}", - "Astronomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Astronomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Astronomy - # prompt tokens": "{\"description\": \"min=639.895, mean=639.895, max=639.895, sum=1279.789 (2)\", \"tab\": \"General information\", \"score\": \"639.8947368421053\"}", - "Astronomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"astronomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_astronomy\"" - } - } - }, - { - "evaluation_name": "Business Ethics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Business Ethics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8, - "details": { - "description": "min=0.8, mean=0.8, max=0.8, sum=1.6 (2)", - "tab": "Accuracy", - "Business Ethics - Observed inference time (s)": "{\"description\": \"min=0.731, mean=0.731, max=0.731, sum=1.463 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7314971995353698\"}", - "Business Ethics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Business Ethics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Business Ethics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Business Ethics - # prompt tokens": "{\"description\": \"min=629.97, mean=629.97, max=629.97, sum=1259.94 (2)\", \"tab\": \"General information\", \"score\": \"629.97\"}", - "Business Ethics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"business_ethics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_business_ethics\"" - } - } - }, - { - "evaluation_name": "Clinical Knowledge", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Clinical Knowledge", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.857, - "details": { - "description": "min=0.857, mean=0.857, max=0.857, sum=1.713 (2)", - "tab": "Accuracy", - "Clinical Knowledge - Observed inference time (s)": "{\"description\": \"min=0.688, mean=0.688, max=0.688, sum=1.376 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6877818728392979\"}", - "Clinical Knowledge - # eval": "{\"description\": \"min=265, mean=265, max=265, sum=530 (2)\", \"tab\": \"General information\", \"score\": \"265.0\"}", - "Clinical Knowledge - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Clinical Knowledge - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Clinical Knowledge - # prompt tokens": "{\"description\": \"min=459.966, mean=459.966, max=459.966, sum=919.932 (2)\", \"tab\": \"General information\", \"score\": \"459.96603773584906\"}", - "Clinical Knowledge - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"clinical_knowledge\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_clinical_knowledge\"" - } - } - }, - { - "evaluation_name": "Conceptual Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Conceptual Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.864, - "details": { - "description": "min=0.864, mean=0.864, max=0.864, sum=1.728 (2)", - "tab": "Accuracy", - "Conceptual Physics - Observed inference time (s)": "{\"description\": \"min=0.659, mean=0.659, max=0.659, sum=1.319 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6594150309867047\"}", - "Conceptual Physics - # eval": "{\"description\": \"min=235, mean=235, max=235, sum=470 (2)\", \"tab\": \"General information\", \"score\": \"235.0\"}", - "Conceptual Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Conceptual Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Conceptual Physics - # prompt tokens": "{\"description\": \"min=324.94, mean=324.94, max=324.94, sum=649.881 (2)\", \"tab\": \"General information\", \"score\": \"324.9404255319149\"}", - "Conceptual Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"conceptual_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_conceptual_physics\"" - } - } - }, - { - "evaluation_name": "Electrical Engineering", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Electrical Engineering", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.779, - "details": { - "description": "min=0.779, mean=0.779, max=0.779, sum=1.559 (2)", - "tab": "Accuracy", - "Electrical Engineering - Observed inference time (s)": "{\"description\": \"min=0.697, mean=0.697, max=0.697, sum=1.394 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6971425631950642\"}", - "Electrical Engineering - # eval": "{\"description\": \"min=145, mean=145, max=145, sum=290 (2)\", \"tab\": \"General information\", \"score\": \"145.0\"}", - "Electrical Engineering - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Electrical Engineering - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Electrical Engineering - # prompt tokens": "{\"description\": \"min=504.993, mean=504.993, max=504.993, sum=1009.986 (2)\", \"tab\": \"General information\", \"score\": \"504.99310344827586\"}", - "Electrical Engineering - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"electrical_engineering\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_electrical_engineering\"" - } - } - }, - { - "evaluation_name": "Elementary Mathematics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Elementary Mathematics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.685, - "details": { - "description": "min=0.685, mean=0.685, max=0.685, sum=1.37 (2)", - "tab": "Accuracy", - "Elementary Mathematics - Observed inference time (s)": "{\"description\": \"min=0.715, mean=0.715, max=0.715, sum=1.43 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7149287146866006\"}", - "Elementary Mathematics - # eval": "{\"description\": \"min=378, mean=378, max=378, sum=756 (2)\", \"tab\": \"General information\", \"score\": \"378.0\"}", - "Elementary Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Elementary Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Elementary Mathematics - # prompt tokens": "{\"description\": \"min=614.344, mean=614.344, max=614.344, sum=1228.688 (2)\", \"tab\": \"General information\", \"score\": \"614.3439153439153\"}", - "Elementary Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"elementary_mathematics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_elementary_mathematics\"" - } - } - }, - { - "evaluation_name": "Formal Logic", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Formal Logic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.603, - "details": { - "description": "min=0.603, mean=0.603, max=0.603, sum=1.206 (2)", - "tab": "Accuracy", - "Formal Logic - Observed inference time (s)": "{\"description\": \"min=0.761, mean=0.761, max=0.761, sum=1.522 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7611211935679117\"}", - "Formal Logic - # eval": "{\"description\": \"min=126, mean=126, max=126, sum=252 (2)\", \"tab\": \"General information\", \"score\": \"126.0\"}", - "Formal Logic - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Formal Logic - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Formal Logic - # prompt tokens": "{\"description\": \"min=688.579, mean=688.579, max=688.579, sum=1377.159 (2)\", \"tab\": \"General information\", \"score\": \"688.5793650793651\"}", - "Formal Logic - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"formal_logic\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_formal_logic\"" - } - } - }, - { - "evaluation_name": "High School World History", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on High School World History", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.928, - "details": { - "description": "min=0.928, mean=0.928, max=0.928, sum=1.857 (2)", - "tab": "Accuracy", - "High School Biology - Observed inference time (s)": "{\"description\": \"min=0.739, mean=0.739, max=0.739, sum=1.478 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7389615043517082\"}", - "High School Chemistry - Observed inference time (s)": "{\"description\": \"min=0.727, mean=0.727, max=0.727, sum=1.454 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7272039317145136\"}", - "High School Computer Science - Observed inference time (s)": "{\"description\": \"min=0.877, mean=0.877, max=0.877, sum=1.754 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8772388291358948\"}", - "High School European History - Observed inference time (s)": "{\"description\": \"min=1.689, mean=1.689, max=1.689, sum=3.378 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.6891969362894694\"}", - "High School Geography - Observed inference time (s)": "{\"description\": \"min=0.725, mean=0.725, max=0.725, sum=1.451 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7252739162156077\"}", - "High School Government And Politics - Observed inference time (s)": "{\"description\": \"min=0.693, mean=0.693, max=0.693, sum=1.387 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6934328054517044\"}", - "High School Macroeconomics - Observed inference time (s)": "{\"description\": \"min=0.684, mean=0.684, max=0.684, sum=1.367 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6835794656704633\"}", - "High School Mathematics - Observed inference time (s)": "{\"description\": \"min=0.716, mean=0.716, max=0.716, sum=1.432 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7162466013873064\"}", - "High School Microeconomics - Observed inference time (s)": "{\"description\": \"min=0.711, mean=0.711, max=0.711, sum=1.422 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7111842982909259\"}", - "High School Physics - Observed inference time (s)": "{\"description\": \"min=0.74, mean=0.74, max=0.74, sum=1.481 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7403108505223761\"}", - "High School Psychology - Observed inference time (s)": "{\"description\": \"min=0.7, mean=0.7, max=0.7, sum=1.4 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7000295271567248\"}", - "High School Statistics - Observed inference time (s)": "{\"description\": \"min=0.833, mean=0.833, max=0.833, sum=1.666 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8330503514519444\"}", - "High School US History - Observed inference time (s)": "{\"description\": \"min=1.349, mean=1.349, max=1.349, sum=2.698 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.3490371108055115\"}", - "High School World History - Observed inference time (s)": "{\"description\": \"min=1.047, mean=1.047, max=1.047, sum=2.093 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.046591958919155\"}", - "High School Biology - # eval": "{\"description\": \"min=310, mean=310, max=310, sum=620 (2)\", \"tab\": \"General information\", \"score\": \"310.0\"}", - "High School Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Biology - # prompt tokens": "{\"description\": \"min=559.394, mean=559.394, max=559.394, sum=1118.787 (2)\", \"tab\": \"General information\", \"score\": \"559.3935483870968\"}", - "High School Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Chemistry - # eval": "{\"description\": \"min=203, mean=203, max=203, sum=406 (2)\", \"tab\": \"General information\", \"score\": \"203.0\"}", - "High School Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # prompt tokens": "{\"description\": \"min=550.015, mean=550.015, max=550.015, sum=1100.03 (2)\", \"tab\": \"General information\", \"score\": \"550.0147783251232\"}", - "High School Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "High School Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # prompt tokens": "{\"description\": \"min=975.1, mean=975.1, max=975.1, sum=1950.2 (2)\", \"tab\": \"General information\", \"score\": \"975.1\"}", - "High School Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School European History - # eval": "{\"description\": \"min=165, mean=165, max=165, sum=330 (2)\", \"tab\": \"General information\", \"score\": \"165.0\"}", - "High School European History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School European History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # prompt tokens": "{\"description\": \"min=2970.412, mean=2970.412, max=2970.412, sum=5940.824 (2)\", \"tab\": \"General information\", \"score\": \"2970.4121212121213\"}", - "High School European History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Geography - # eval": "{\"description\": \"min=198, mean=198, max=198, sum=396 (2)\", \"tab\": \"General information\", \"score\": \"198.0\"}", - "High School Geography - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Geography - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # prompt tokens": "{\"description\": \"min=417.035, mean=417.035, max=417.035, sum=834.071 (2)\", \"tab\": \"General information\", \"score\": \"417.0353535353535\"}", - "High School Geography - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Government And Politics - # eval": "{\"description\": \"min=193, mean=193, max=193, sum=386 (2)\", \"tab\": \"General information\", \"score\": \"193.0\"}", - "High School Government And Politics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Government And Politics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # prompt tokens": "{\"description\": \"min=497.725, mean=497.725, max=497.725, sum=995.451 (2)\", \"tab\": \"General information\", \"score\": \"497.7253886010363\"}", - "High School Government And Politics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Macroeconomics - # eval": "{\"description\": \"min=390, mean=390, max=390, sum=780 (2)\", \"tab\": \"General information\", \"score\": \"390.0\"}", - "High School Macroeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Macroeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # prompt tokens": "{\"description\": \"min=411.892, mean=411.892, max=411.892, sum=823.785 (2)\", \"tab\": \"General information\", \"score\": \"411.89230769230767\"}", - "High School Macroeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Mathematics - # eval": "{\"description\": \"min=270, mean=270, max=270, sum=540 (2)\", \"tab\": \"General information\", \"score\": \"270.0\"}", - "High School Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # prompt tokens": "{\"description\": \"min=588.622, mean=588.622, max=588.622, sum=1177.244 (2)\", \"tab\": \"General information\", \"score\": \"588.6222222222223\"}", - "High School Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Microeconomics - # eval": "{\"description\": \"min=238, mean=238, max=238, sum=476 (2)\", \"tab\": \"General information\", \"score\": \"238.0\"}", - "High School Microeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Microeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # prompt tokens": "{\"description\": \"min=433.739, mean=433.739, max=433.739, sum=867.479 (2)\", \"tab\": \"General information\", \"score\": \"433.73949579831935\"}", - "High School Microeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Physics - # eval": "{\"description\": \"min=151, mean=151, max=151, sum=302 (2)\", \"tab\": \"General information\", \"score\": \"151.0\"}", - "High School Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # prompt tokens": "{\"description\": \"min=612.411, mean=612.411, max=612.411, sum=1224.821 (2)\", \"tab\": \"General information\", \"score\": \"612.4105960264901\"}", - "High School Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Psychology - # eval": "{\"description\": \"min=545, mean=545, max=545, sum=1090 (2)\", \"tab\": \"General information\", \"score\": \"545.0\"}", - "High School Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # prompt tokens": "{\"description\": \"min=539.826, mean=539.826, max=539.826, sum=1079.651 (2)\", \"tab\": \"General information\", \"score\": \"539.8256880733945\"}", - "High School Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Statistics - # eval": "{\"description\": \"min=216, mean=216, max=216, sum=432 (2)\", \"tab\": \"General information\", \"score\": \"216.0\"}", - "High School Statistics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Statistics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # prompt tokens": "{\"description\": \"min=882.778, mean=882.778, max=882.778, sum=1765.556 (2)\", \"tab\": \"General information\", \"score\": \"882.7777777777778\"}", - "High School Statistics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School US History - # eval": "{\"description\": \"min=204, mean=204, max=204, sum=408 (2)\", \"tab\": \"General information\", \"score\": \"204.0\"}", - "High School US History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School US History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # prompt tokens": "{\"description\": \"min=2382.132, mean=2382.132, max=2382.132, sum=4764.265 (2)\", \"tab\": \"General information\", \"score\": \"2382.1323529411766\"}", - "High School US History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School World History - # eval": "{\"description\": \"min=237, mean=237, max=237, sum=474 (2)\", \"tab\": \"General information\", \"score\": \"237.0\"}", - "High School World History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School World History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # prompt tokens": "{\"description\": \"min=1554.371, mean=1554.371, max=1554.371, sum=3108.743 (2)\", \"tab\": \"General information\", \"score\": \"1554.3713080168777\"}", - "High School World History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"high_school_world_history\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_high_school_world_history\"" - } - } - }, - { - "evaluation_name": "Human Sexuality", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Human Sexuality", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.901, - "details": { - "description": "min=0.901, mean=0.901, max=0.901, sum=1.802 (2)", - "tab": "Accuracy", - "Human Aging - Observed inference time (s)": "{\"description\": \"min=0.66, mean=0.66, max=0.66, sum=1.32 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6601343742935112\"}", - "Human Sexuality - Observed inference time (s)": "{\"description\": \"min=0.704, mean=0.704, max=0.704, sum=1.409 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7043184669873187\"}", - "Human Aging - # eval": "{\"description\": \"min=223, mean=223, max=223, sum=446 (2)\", \"tab\": \"General information\", \"score\": \"223.0\"}", - "Human Aging - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Aging - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Aging - # prompt tokens": "{\"description\": \"min=345.013, mean=345.013, max=345.013, sum=690.027 (2)\", \"tab\": \"General information\", \"score\": \"345.0134529147982\"}", - "Human Aging - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Human Sexuality - # eval": "{\"description\": \"min=131, mean=131, max=131, sum=262 (2)\", \"tab\": \"General information\", \"score\": \"131.0\"}", - "Human Sexuality - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Sexuality - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # prompt tokens": "{\"description\": \"min=380.855, mean=380.855, max=380.855, sum=761.71 (2)\", \"tab\": \"General information\", \"score\": \"380.85496183206106\"}", - "Human Sexuality - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"human_sexuality\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_human_sexuality\"" - } - } - }, - { - "evaluation_name": "International Law", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on International Law", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.917, - "details": { - "description": "min=0.917, mean=0.917, max=0.917, sum=1.835 (2)", - "tab": "Accuracy", - "International Law - Observed inference time (s)": "{\"description\": \"min=0.769, mean=0.769, max=0.769, sum=1.538 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7691502098209602\"}", - "International Law - # eval": "{\"description\": \"min=121, mean=121, max=121, sum=242 (2)\", \"tab\": \"General information\", \"score\": \"121.0\"}", - "International Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "International Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "International Law - # prompt tokens": "{\"description\": \"min=676.289, mean=676.289, max=676.289, sum=1352.579 (2)\", \"tab\": \"General information\", \"score\": \"676.2892561983471\"}", - "International Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"international_law\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_international_law\"" - } - } - }, - { - "evaluation_name": "Logical Fallacies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Logical Fallacies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.865, - "details": { - "description": "min=0.865, mean=0.865, max=0.865, sum=1.73 (2)", - "tab": "Accuracy", - "Logical Fallacies - Observed inference time (s)": "{\"description\": \"min=0.684, mean=0.684, max=0.684, sum=1.367 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6835026492370418\"}", - "Logical Fallacies - # eval": "{\"description\": \"min=163, mean=163, max=163, sum=326 (2)\", \"tab\": \"General information\", \"score\": \"163.0\"}", - "Logical Fallacies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Logical Fallacies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Logical Fallacies - # prompt tokens": "{\"description\": \"min=479.595, mean=479.595, max=479.595, sum=959.19 (2)\", \"tab\": \"General information\", \"score\": \"479.5950920245399\"}", - "Logical Fallacies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"logical_fallacies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_logical_fallacies\"" - } - } - }, - { - "evaluation_name": "Machine Learning", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Machine Learning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.616, - "details": { - "description": "min=0.616, mean=0.616, max=0.616, sum=1.232 (2)", - "tab": "Accuracy", - "Machine Learning - Observed inference time (s)": "{\"description\": \"min=0.745, mean=0.745, max=0.745, sum=1.489 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7447149263960975\"}", - "Machine Learning - # eval": "{\"description\": \"min=112, mean=112, max=112, sum=224 (2)\", \"tab\": \"General information\", \"score\": \"112.0\"}", - "Machine Learning - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Machine Learning - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Machine Learning - # prompt tokens": "{\"description\": \"min=733.161, mean=733.161, max=733.161, sum=1466.321 (2)\", \"tab\": \"General information\", \"score\": \"733.1607142857143\"}", - "Machine Learning - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"machine_learning\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_machine_learning\"" - } - } - }, - { - "evaluation_name": "Management", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Management", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.903, - "details": { - "description": "min=0.903, mean=0.903, max=0.903, sum=1.806 (2)", - "tab": "Accuracy", - "Management - Observed inference time (s)": "{\"description\": \"min=0.621, mean=0.621, max=0.621, sum=1.243 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6213390433672562\"}", - "Management - # eval": "{\"description\": \"min=103, mean=103, max=103, sum=206 (2)\", \"tab\": \"General information\", \"score\": \"103.0\"}", - "Management - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Management - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Management - # prompt tokens": "{\"description\": \"min=313.544, mean=313.544, max=313.544, sum=627.087 (2)\", \"tab\": \"General information\", \"score\": \"313.54368932038835\"}", - "Management - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"management\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_management\"" - } - } - }, - { - "evaluation_name": "Marketing", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Marketing", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.927, - "details": { - "description": "min=0.927, mean=0.927, max=0.927, sum=1.855 (2)", - "tab": "Accuracy", - "Marketing - Observed inference time (s)": "{\"description\": \"min=0.679, mean=0.679, max=0.679, sum=1.357 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6785362948719252\"}", - "Marketing - # eval": "{\"description\": \"min=234, mean=234, max=234, sum=468 (2)\", \"tab\": \"General information\", \"score\": \"234.0\"}", - "Marketing - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Marketing - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Marketing - # prompt tokens": "{\"description\": \"min=455.825, mean=455.825, max=455.825, sum=911.65 (2)\", \"tab\": \"General information\", \"score\": \"455.8247863247863\"}", - "Marketing - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"marketing\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_marketing\"" - } - } - }, - { - "evaluation_name": "Medical Genetics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Medical Genetics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.83, - "details": { - "description": "min=0.83, mean=0.83, max=0.83, sum=1.66 (2)", - "tab": "Accuracy", - "Medical Genetics - Observed inference time (s)": "{\"description\": \"min=0.689, mean=0.689, max=0.689, sum=1.379 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6893473124504089\"}", - "Medical Genetics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Medical Genetics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Medical Genetics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Medical Genetics - # prompt tokens": "{\"description\": \"min=375, mean=375, max=375, sum=750 (2)\", \"tab\": \"General information\", \"score\": \"375.0\"}", - "Medical Genetics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"medical_genetics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_medical_genetics\"" - } - } - }, - { - "evaluation_name": "Miscellaneous", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Miscellaneous", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.916, - "details": { - "description": "min=0.916, mean=0.916, max=0.916, sum=1.831 (2)", - "tab": "Accuracy", - "Miscellaneous - Observed inference time (s)": "{\"description\": \"min=0.633, mean=0.633, max=0.633, sum=1.266 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6329697509073815\"}", - "Miscellaneous - # eval": "{\"description\": \"min=783, mean=783, max=783, sum=1566 (2)\", \"tab\": \"General information\", \"score\": \"783.0\"}", - "Miscellaneous - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Miscellaneous - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Miscellaneous - # prompt tokens": "{\"description\": \"min=344.441, mean=344.441, max=344.441, sum=688.881 (2)\", \"tab\": \"General information\", \"score\": \"344.4406130268199\"}", - "Miscellaneous - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"miscellaneous\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_miscellaneous\"" - } - } - }, - { - "evaluation_name": "Moral Scenarios", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Moral Scenarios", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.831, - "details": { - "description": "min=0.831, mean=0.831, max=0.831, sum=1.663 (2)", - "tab": "Accuracy", - "Moral Disputes - Observed inference time (s)": "{\"description\": \"min=0.703, mean=0.703, max=0.703, sum=1.406 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7028186107646524\"}", - "Moral Scenarios - Observed inference time (s)": "{\"description\": \"min=0.754, mean=0.754, max=0.754, sum=1.509 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7543408100831442\"}", - "Moral Disputes - # eval": "{\"description\": \"min=346, mean=346, max=346, sum=692 (2)\", \"tab\": \"General information\", \"score\": \"346.0\"}", - "Moral Disputes - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Disputes - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Disputes - # prompt tokens": "{\"description\": \"min=520.913, mean=520.913, max=520.913, sum=1041.827 (2)\", \"tab\": \"General information\", \"score\": \"520.9132947976879\"}", - "Moral Disputes - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Moral Scenarios - # eval": "{\"description\": \"min=895, mean=895, max=895, sum=1790 (2)\", \"tab\": \"General information\", \"score\": \"895.0\"}", - "Moral Scenarios - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Scenarios - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # prompt tokens": "{\"description\": \"min=716.334, mean=716.334, max=716.334, sum=1432.668 (2)\", \"tab\": \"General information\", \"score\": \"716.3340782122905\"}", - "Moral Scenarios - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"moral_scenarios\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_moral_scenarios\"" - } - } - }, - { - "evaluation_name": "Nutrition", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Nutrition", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.846, - "details": { - "description": "min=0.846, mean=0.846, max=0.846, sum=1.693 (2)", - "tab": "Accuracy", - "Nutrition - Observed inference time (s)": "{\"description\": \"min=0.721, mean=0.721, max=0.721, sum=1.442 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7212473138485079\"}", - "Nutrition - # eval": "{\"description\": \"min=306, mean=306, max=306, sum=612 (2)\", \"tab\": \"General information\", \"score\": \"306.0\"}", - "Nutrition - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Nutrition - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Nutrition - # prompt tokens": "{\"description\": \"min=656.317, mean=656.317, max=656.317, sum=1312.634 (2)\", \"tab\": \"General information\", \"score\": \"656.3169934640523\"}", - "Nutrition - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"nutrition\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_nutrition\"" - } - } - }, - { - "evaluation_name": "Prehistory", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Prehistory", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.892, - "details": { - "description": "min=0.892, mean=0.892, max=0.892, sum=1.784 (2)", - "tab": "Accuracy", - "Prehistory - Observed inference time (s)": "{\"description\": \"min=0.712, mean=0.712, max=0.712, sum=1.423 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7115242841802998\"}", - "Prehistory - # eval": "{\"description\": \"min=324, mean=324, max=324, sum=648 (2)\", \"tab\": \"General information\", \"score\": \"324.0\"}", - "Prehistory - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Prehistory - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Prehistory - # prompt tokens": "{\"description\": \"min=578.096, mean=578.096, max=578.096, sum=1156.191 (2)\", \"tab\": \"General information\", \"score\": \"578.0956790123457\"}", - "Prehistory - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"prehistory\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_prehistory\"" - } - } - }, - { - "evaluation_name": "Public Relations", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Public Relations", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.827, - "details": { - "description": "min=0.827, mean=0.827, max=0.827, sum=1.655 (2)", - "tab": "Accuracy", - "Public Relations - Observed inference time (s)": "{\"description\": \"min=0.708, mean=0.708, max=0.708, sum=1.417 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.708361968127164\"}", - "Public Relations - # eval": "{\"description\": \"min=110, mean=110, max=110, sum=220 (2)\", \"tab\": \"General information\", \"score\": \"110.0\"}", - "Public Relations - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Public Relations - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Public Relations - # prompt tokens": "{\"description\": \"min=445.436, mean=445.436, max=445.436, sum=890.873 (2)\", \"tab\": \"General information\", \"score\": \"445.43636363636364\"}", - "Public Relations - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"public_relations\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_public_relations\"" - } - } - }, - { - "evaluation_name": "Security Studies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Security Studies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.82, - "details": { - "description": "min=0.82, mean=0.82, max=0.82, sum=1.641 (2)", - "tab": "Accuracy", - "Security Studies - Observed inference time (s)": "{\"description\": \"min=0.92, mean=0.92, max=0.92, sum=1.84 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9198286231683225\"}", - "Security Studies - # eval": "{\"description\": \"min=245, mean=245, max=245, sum=490 (2)\", \"tab\": \"General information\", \"score\": \"245.0\"}", - "Security Studies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Security Studies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Security Studies - # prompt tokens": "{\"description\": \"min=1240.196, mean=1240.196, max=1240.196, sum=2480.392 (2)\", \"tab\": \"General information\", \"score\": \"1240.1959183673468\"}", - "Security Studies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"security_studies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_security_studies\"" - } - } - }, - { - "evaluation_name": "Sociology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Sociology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.881, - "details": { - "description": "min=0.881, mean=0.881, max=0.881, sum=1.761 (2)", - "tab": "Accuracy", - "Sociology - Observed inference time (s)": "{\"description\": \"min=0.71, mean=0.71, max=0.71, sum=1.421 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7103830344641386\"}", - "Sociology - # eval": "{\"description\": \"min=201, mean=201, max=201, sum=402 (2)\", \"tab\": \"General information\", \"score\": \"201.0\"}", - "Sociology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Sociology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Sociology - # prompt tokens": "{\"description\": \"min=476.99, mean=476.99, max=476.99, sum=953.98 (2)\", \"tab\": \"General information\", \"score\": \"476.99004975124376\"}", - "Sociology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"sociology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_sociology\"" - } - } - }, - { - "evaluation_name": "Virology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Virology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.59, - "details": { - "description": "min=0.59, mean=0.59, max=0.59, sum=1.181 (2)", - "tab": "Accuracy", - "Virology - Observed inference time (s)": "{\"description\": \"min=0.677, mean=0.677, max=0.677, sum=1.354 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6768132835985666\"}", - "Virology - # eval": "{\"description\": \"min=166, mean=166, max=166, sum=332 (2)\", \"tab\": \"General information\", \"score\": \"166.0\"}", - "Virology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Virology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Virology - # prompt tokens": "{\"description\": \"min=376.102, mean=376.102, max=376.102, sum=752.205 (2)\", \"tab\": \"General information\", \"score\": \"376.1024096385542\"}", - "Virology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"virology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_virology\"" - } - } - }, - { - "evaluation_name": "World Religions", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on World Religions", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.871, - "details": { - "description": "min=0.871, mean=0.871, max=0.871, sum=1.743 (2)", - "tab": "Accuracy", - "World Religions - Observed inference time (s)": "{\"description\": \"min=0.645, mean=0.645, max=0.645, sum=1.289 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.644616849241201\"}", - "World Religions - # eval": "{\"description\": \"min=171, mean=171, max=171, sum=342 (2)\", \"tab\": \"General information\", \"score\": \"171.0\"}", - "World Religions - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "World Religions - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "World Religions - # prompt tokens": "{\"description\": \"min=302.971, mean=302.971, max=302.971, sum=605.942 (2)\", \"tab\": \"General information\", \"score\": \"302.97076023391816\"}", - "World Religions - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"world_religions\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_world_religions\"" - } - } - }, - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.258, - "details": { - "description": "", - "tab": "Efficiency" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/1-800-llms_qwen-2.5-14b-hindi-custom-instruct.json b/data/models/1-800-llms_qwen-2.5-14b-hindi-custom-instruct.json deleted file mode 100644 index fc51c886bf96090c766642f8b56494d1bd61ba72..0000000000000000000000000000000000000000 --- a/data/models/1-800-llms_qwen-2.5-14b-hindi-custom-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen-2.5-14B-Hindi-Custom-Instruct", - "id": "1-800-LLMs/Qwen-2.5-14B-Hindi-Custom-Instruct", - "developer": "1-800-LLMs", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/1-800-LLMs_Qwen-2.5-14B-Hindi-Custom-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3077 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6284 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3112 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.37 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4491 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5164 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/1-800-llms_qwen-2.5-14b-hindi.json b/data/models/1-800-llms_qwen-2.5-14b-hindi.json deleted file mode 100644 index 8a6558b1435f91d18030b2d90fb627b7e93ffe52..0000000000000000000000000000000000000000 --- a/data/models/1-800-llms_qwen-2.5-14b-hindi.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen-2.5-14B-Hindi", - "id": "1-800-LLMs/Qwen-2.5-14B-Hindi", - "developer": "1-800-LLMs", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/1-800-LLMs_Qwen-2.5-14B-Hindi/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5826 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6524 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3331 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3624 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4489 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5263 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/1024m_phi-4-hindi.json b/data/models/1024m_phi-4-hindi.json deleted file mode 100644 index d320b473010c09e7f7297397d4eb3127c9010674..0000000000000000000000000000000000000000 --- a/data/models/1024m_phi-4-hindi.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "PHI-4-Hindi", - "id": "1024m/PHI-4-Hindi", - "developer": "1024m", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/1024m_PHI-4-Hindi/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0082 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.671 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2334 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3977 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4914 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5239 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/1024m_qwen-14b-b100.json b/data/models/1024m_qwen-14b-b100.json deleted file mode 100644 index a65979fd841ce9a1358b252ac92241cc2fd82259..0000000000000000000000000000000000000000 --- a/data/models/1024m_qwen-14b-b100.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "QWEN-14B-B100", - "id": "1024m/QWEN-14B-B100", - "developer": "1024m", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/1024m_QWEN-14B-B100/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7762 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6533 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5438 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3507 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.41 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5179 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/152334h_miqu-1-70b-sf.json b/data/models/152334h_miqu-1-70b-sf.json deleted file mode 100644 index 1fd99a82f67f26075c4ff6add734246a95bc1850..0000000000000000000000000000000000000000 --- a/data/models/152334h_miqu-1-70b-sf.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "miqu-1-70b-sf", - "id": "152334H/miqu-1-70b-sf", - "developer": "152334H", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "68.977" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/152334H_miqu-1-70b-sf/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5182 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6102 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1246 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3507 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4582 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4228 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/1tuanpham_t-visstar-7b-v0.1.json b/data/models/1tuanpham_t-visstar-7b-v0.1.json deleted file mode 100644 index 1d4a8e97013d6e28e059be59f98f9fb9441b8aea..0000000000000000000000000000000000000000 --- a/data/models/1tuanpham_t-visstar-7b-v0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "T-VisStar-7B-v0.1", - "id": "1TuanPham/T-VisStar-7B-v0.1", - "developer": "1TuanPham", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "7.294" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/1TuanPham_T-VisStar-7B-v0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3607 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5052 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0574 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2852 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4375 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3211 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/1tuanpham_t-visstar-v0.1.json b/data/models/1tuanpham_t-visstar-v0.1.json deleted file mode 100644 index 3ca61a9bace44cdaab3547a50fc456f77f19fc37..0000000000000000000000000000000000000000 --- a/data/models/1tuanpham_t-visstar-v0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "T-VisStar-v0.1", - "id": "1TuanPham/T-VisStar-v0.1", - "developer": "1TuanPham", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "7.294" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/1TuanPham_T-VisStar-v0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3607 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5052 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0574 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2852 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4375 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3211 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/3rd-degree-burn_l-3.1-science-writer-8b.json b/data/models/3rd-degree-burn_l-3.1-science-writer-8b.json deleted file mode 100644 index 3a218408c7b68ab62c64b8d17ffdf3e7a24ddc2d..0000000000000000000000000000000000000000 --- a/data/models/3rd-degree-burn_l-3.1-science-writer-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "L-3.1-Science-Writer-8B", - "id": "3rd-Degree-Burn/L-3.1-Science-Writer-8B", - "developer": "3rd-Degree-Burn", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/3rd-Degree-Burn_L-3.1-Science-Writer-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4263 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5041 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1035 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2743 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3959 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3649 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/3rd-degree-burn_llama-3.1-8b-squareroot-v1.json b/data/models/3rd-degree-burn_llama-3.1-8b-squareroot-v1.json deleted file mode 100644 index 77d2a7a6b05bd76c9b743d47b4780e40a1d5c3c5..0000000000000000000000000000000000000000 --- a/data/models/3rd-degree-burn_llama-3.1-8b-squareroot-v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.1-8B-Squareroot-v1", - "id": "3rd-Degree-Burn/Llama-3.1-8B-Squareroot-v1", - "developer": "3rd-Degree-Burn", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/3rd-Degree-Burn_Llama-3.1-8B-Squareroot-v1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2892 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3343 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0884 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2559 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3341 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1127 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/3rd-degree-burn_llama-3.1-8b-squareroot.json b/data/models/3rd-degree-burn_llama-3.1-8b-squareroot.json deleted file mode 100644 index 02aaa7b543f40aee038259cabbb0a416505c8705..0000000000000000000000000000000000000000 --- a/data/models/3rd-degree-burn_llama-3.1-8b-squareroot.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.1-8B-Squareroot", - "id": "3rd-Degree-Burn/Llama-3.1-8B-Squareroot", - "developer": "3rd-Degree-Burn", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/3rd-Degree-Burn_Llama-3.1-8B-Squareroot/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2213 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3461 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2659 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2567 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3089 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.175 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/3rd-degree-burn_llama-squared-8b.json b/data/models/3rd-degree-burn_llama-squared-8b.json deleted file mode 100644 index 1297545d9f0dda7de83d0b4cc257bfc85651b2ad..0000000000000000000000000000000000000000 --- a/data/models/3rd-degree-burn_llama-squared-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-Squared-8B", - "id": "3rd-Degree-Burn/Llama-Squared-8B", - "developer": "3rd-Degree-Burn", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/3rd-Degree-Burn_Llama-Squared-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2755 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4431 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0574 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2718 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3089 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2366 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/4season_final_model_test_v2.json b/data/models/4season_final_model_test_v2.json deleted file mode 100644 index faa3299dd91dadd13761170f48ce160dddca1754..0000000000000000000000000000000000000000 --- a/data/models/4season_final_model_test_v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "final_model_test_v2", - "id": "4season/final_model_test_v2", - "developer": "4season", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "21.421" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/4season_final_model_test_v2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3191 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6342 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0838 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3272 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4314 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3528 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/aaditya_llama3-openbiollm-70b.json b/data/models/aaditya_llama3-openbiollm-70b.json deleted file mode 100644 index bd30c0d92720c5a32b57a07c8396aef2cf3113e9..0000000000000000000000000000000000000000 --- a/data/models/aaditya_llama3-openbiollm-70b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama3-OpenBioLLM-70B", - "id": "aaditya/Llama3-OpenBioLLM-70B", - "developer": "aaditya", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "70.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/aaditya_Llama3-OpenBioLLM-70B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7597 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6399 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1971 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.323 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4417 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4867 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/aalf_fusechat-llama-3.1-8b-instruct-preview.json b/data/models/aalf_fusechat-llama-3.1-8b-instruct-preview.json deleted file mode 100644 index f5b95438d6fc0a5ee525e3c1667e46ee964659cb..0000000000000000000000000000000000000000 --- a/data/models/aalf_fusechat-llama-3.1-8b-instruct-preview.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "FuseChat-Llama-3.1-8B-Instruct-preview", - "id": "AALF/FuseChat-Llama-3.1-8B-Instruct-preview", - "developer": "AALF", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/AALF_FuseChat-Llama-3.1-8B-Instruct-preview/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.719 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.512 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2477 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3054 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.382 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3733 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/aalf_fusechat-llama-3.1-8b-sft-preview.json b/data/models/aalf_fusechat-llama-3.1-8b-sft-preview.json deleted file mode 100644 index 097a6de2a3bd82829dd34ac4d4787c6b363e003a..0000000000000000000000000000000000000000 --- a/data/models/aalf_fusechat-llama-3.1-8b-sft-preview.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "FuseChat-Llama-3.1-8B-SFT-preview", - "id": "AALF/FuseChat-Llama-3.1-8B-SFT-preview", - "developer": "AALF", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/AALF_FuseChat-Llama-3.1-8B-SFT-preview/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7281 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.524 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2251 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3045 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.402 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3743 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/aalf_gemma-2-27b-it-simpo-37k-100steps.json b/data/models/aalf_gemma-2-27b-it-simpo-37k-100steps.json deleted file mode 100644 index 75821c45c53bdd3fa5000c8108d6ed99b5e0b678..0000000000000000000000000000000000000000 --- a/data/models/aalf_gemma-2-27b-it-simpo-37k-100steps.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "gemma-2-27b-it-SimPO-37K-100steps", - "id": "AALF/gemma-2-27b-it-SimPO-37K-100steps", - "developer": "AALF", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "27.227" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/AALF_gemma-2-27b-it-SimPO-37K-100steps/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2568 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3931 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0211 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2886 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3329 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2125 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/aalf_gemma-2-27b-it-simpo-37k.json b/data/models/aalf_gemma-2-27b-it-simpo-37k.json deleted file mode 100644 index a1604100b9e3a710a439d08a50241caa02baa8a5..0000000000000000000000000000000000000000 --- a/data/models/aalf_gemma-2-27b-it-simpo-37k.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "gemma-2-27b-it-SimPO-37K", - "id": "AALF/gemma-2-27b-it-SimPO-37K", - "developer": "AALF", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "27.227" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/AALF_gemma-2-27b-it-SimPO-37K/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2407 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3911 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0128 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2802 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3488 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1971 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/aashraf995_creative-7b-nerd.json b/data/models/aashraf995_creative-7b-nerd.json deleted file mode 100644 index a5313f7774767c7653119916545c2a3cc9cc5fd5..0000000000000000000000000000000000000000 --- a/data/models/aashraf995_creative-7b-nerd.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Creative-7B-nerd", - "id": "Aashraf995/Creative-7B-nerd", - "developer": "Aashraf995", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Aashraf995_Creative-7B-nerd/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4722 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5607 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3165 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3263 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4515 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4492 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/aashraf995_gemma-evo-10b.json b/data/models/aashraf995_gemma-evo-10b.json deleted file mode 100644 index b63ba533720630f002e1f955efb6233c1ee4cc81..0000000000000000000000000000000000000000 --- a/data/models/aashraf995_gemma-evo-10b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Gemma-Evo-10B", - "id": "Aashraf995/Gemma-Evo-10B", - "developer": "Aashraf995", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Aashraf995_Gemma-Evo-10B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7332 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6044 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2228 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.354 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4595 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4275 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/aashraf995_qwen-evo-7b.json b/data/models/aashraf995_qwen-evo-7b.json deleted file mode 100644 index c321ed5050569fc68c14b88ae418ac642d0e508b..0000000000000000000000000000000000000000 --- a/data/models/aashraf995_qwen-evo-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen-Evo-7B", - "id": "Aashraf995/Qwen-Evo-7B", - "developer": "Aashraf995", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Aashraf995_Qwen-Evo-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4757 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5709 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3142 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3255 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4541 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4462 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/aashraf995_qwenstock-14b.json b/data/models/aashraf995_qwenstock-14b.json deleted file mode 100644 index a8817bb6ced0e775d1f2d0ad52dfe937638ad012..0000000000000000000000000000000000000000 --- a/data/models/aashraf995_qwenstock-14b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "QwenStock-14B", - "id": "Aashraf995/QwenStock-14B", - "developer": "Aashraf995", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Aashraf995_QwenStock-14B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5009 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.655 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3573 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3893 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4793 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5382 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/abacusai_bigstral-12b-32k.json b/data/models/abacusai_bigstral-12b-32k.json deleted file mode 100644 index 2cf33396585592d954db867843facdfd05a0e784..0000000000000000000000000000000000000000 --- a/data/models/abacusai_bigstral-12b-32k.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "bigstral-12b-32k", - "id": "abacusai/bigstral-12b-32k", - "developer": "abacusai", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "12.476" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/abacusai_bigstral-12b-32k/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4194 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.47 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0151 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2928 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.456 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2641 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/abacusai_bigyi-15b.json b/data/models/abacusai_bigyi-15b.json deleted file mode 100644 index 8b3f52511e686424f98ad6b9cf180415a2aa07a9..0000000000000000000000000000000000000000 --- a/data/models/abacusai_bigyi-15b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "bigyi-15b", - "id": "abacusai/bigyi-15b", - "developer": "abacusai", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "15.058" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/abacusai_bigyi-15b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2094 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4345 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0295 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3096 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3538 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3003 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/abacusai_dracarys-72b-instruct.json b/data/models/abacusai_dracarys-72b-instruct.json deleted file mode 100644 index 4248e3626b3d8a97021c6bafae6aa8ab7f1b590b..0000000000000000000000000000000000000000 --- a/data/models/abacusai_dracarys-72b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Dracarys-72B-Instruct", - "id": "abacusai/Dracarys-72B-Instruct", - "developer": "abacusai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "72.706" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/abacusai_Dracarys-72B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7856 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6944 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3965 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3909 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4558 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5456 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/abacusai_liberated-qwen1.5-14b.json b/data/models/abacusai_liberated-qwen1.5-14b.json deleted file mode 100644 index 36f26390508c20ef2885c64d9bc9edd28a6c8f64..0000000000000000000000000000000000000000 --- a/data/models/abacusai_liberated-qwen1.5-14b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Liberated-Qwen1.5-14B", - "id": "abacusai/Liberated-Qwen1.5-14B", - "developer": "abacusai", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/abacusai_Liberated-Qwen1.5-14B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3631 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4948 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1601 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2836 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4175 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3512 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/abacusai_llama-3-smaug-8b.json b/data/models/abacusai_llama-3-smaug-8b.json deleted file mode 100644 index 4e9fdd3f26cda7c6c2311d088a3fedcc2a63619e..0000000000000000000000000000000000000000 --- a/data/models/abacusai_llama-3-smaug-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-Smaug-8B", - "id": "abacusai/Llama-3-Smaug-8B", - "developer": "abacusai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/abacusai_Llama-3-Smaug-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4867 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4931 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0853 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2483 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3622 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3185 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/abacusai_smaug-34b-v0.1.json b/data/models/abacusai_smaug-34b-v0.1.json deleted file mode 100644 index 408b21f27de7576b81da99ae91e9deae86503f68..0000000000000000000000000000000000000000 --- a/data/models/abacusai_smaug-34b-v0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Smaug-34B-v0.1", - "id": "abacusai/Smaug-34B-v0.1", - "developer": "abacusai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "34.389" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/abacusai_Smaug-34B-v0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5016 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5358 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0718 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3297 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3979 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4543 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/abacusai_smaug-72b-v0.1.json b/data/models/abacusai_smaug-72b-v0.1.json deleted file mode 100644 index 331ceb6f1b4e4cdfc7d554a765d0eeaa04123b31..0000000000000000000000000000000000000000 --- a/data/models/abacusai_smaug-72b-v0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Smaug-72B-v0.1", - "id": "abacusai/Smaug-72B-v0.1", - "developer": "abacusai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "72.289" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/abacusai_Smaug-72B-v0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5167 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5996 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1911 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3238 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4473 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4624 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/abacusai_smaug-llama-3-70b-instruct-32k.json b/data/models/abacusai_smaug-llama-3-70b-instruct-32k.json deleted file mode 100644 index b62399b07535c3d2afcce17f7858e866f602624d..0000000000000000000000000000000000000000 --- a/data/models/abacusai_smaug-llama-3-70b-instruct-32k.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Smaug-Llama-3-70B-Instruct-32K", - "id": "abacusai/Smaug-Llama-3-70B-Instruct-32K", - "developer": "abacusai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "70.554" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/abacusai_Smaug-Llama-3-70B-Instruct-32K/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7761 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6493 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2749 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2961 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4208 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4765 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/abacusai_smaug-mixtral-v0.1.json b/data/models/abacusai_smaug-mixtral-v0.1.json deleted file mode 100644 index c099e6d7051b1d040cad60996bb85dc2989f6f90..0000000000000000000000000000000000000000 --- a/data/models/abacusai_smaug-mixtral-v0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Smaug-Mixtral-v0.1", - "id": "abacusai/Smaug-Mixtral-v0.1", - "developer": "abacusai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MixtralForCausalLM", - "params_billions": "46.703" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/abacusai_Smaug-Mixtral-v0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5554 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5162 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0952 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3012 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4298 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3352 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/abacusai_smaug-qwen2-72b-instruct.json b/data/models/abacusai_smaug-qwen2-72b-instruct.json deleted file mode 100644 index 9ddbadf9ce721e66335a859c8e591dbdc0c7ea4c..0000000000000000000000000000000000000000 --- a/data/models/abacusai_smaug-qwen2-72b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Smaug-Qwen2-72B-Instruct", - "id": "abacusai/Smaug-Qwen2-72B-Instruct", - "developer": "abacusai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "72.706" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/abacusai_Smaug-Qwen2-72B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7825 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.691 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4131 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3616 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4401 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.519 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/abacusresearch_jallabi-34b.json b/data/models/abacusresearch_jallabi-34b.json deleted file mode 100644 index 827dfcaa9fc972fe2be4cfb6e25669a67dc1196a..0000000000000000000000000000000000000000 --- a/data/models/abacusresearch_jallabi-34b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Jallabi-34B", - "id": "AbacusResearch/Jallabi-34B", - "developer": "AbacusResearch", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "34.389" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/AbacusResearch_Jallabi-34B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3529 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6023 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0521 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3389 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4822 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4682 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/abhishek_autotrain-0tmgq-5tpbg.json b/data/models/abhishek_autotrain-0tmgq-5tpbg.json deleted file mode 100644 index ac8f1c0a411cfe237fa1ffe1aaf7d7772ddbfda3..0000000000000000000000000000000000000000 --- a/data/models/abhishek_autotrain-0tmgq-5tpbg.json +++ /dev/null @@ -1,275 +0,0 @@ -{ - "model_info": { - "name": "autotrain-0tmgq-5tpbg", - "id": "abhishek/autotrain-0tmgq-5tpbg", - "developer": "abhishek", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "0.135" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/abhishek_autotrain-0tmgq-5tpbg/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1957 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3135 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2517 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.365 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1151 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "hfopenllm_v2/abhishek_autotrain-0tmgq-5tpbg/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1952 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3127 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0128 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2592 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3584 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1144 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/abhishek_autotrain-llama3-70b-orpo-v1.json b/data/models/abhishek_autotrain-llama3-70b-orpo-v1.json deleted file mode 100644 index bbc7a78117fd7cae3ee8856479bf7d82221954a2..0000000000000000000000000000000000000000 --- a/data/models/abhishek_autotrain-llama3-70b-orpo-v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "autotrain-llama3-70b-orpo-v1", - "id": "abhishek/autotrain-llama3-70b-orpo-v1", - "developer": "abhishek", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "70.554" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/abhishek_autotrain-llama3-70b-orpo-v1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4233 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5998 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0106 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2441 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3579 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1122 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/abhishek_autotrain-llama3-70b-orpo-v2.json b/data/models/abhishek_autotrain-llama3-70b-orpo-v2.json deleted file mode 100644 index 3630ebbfcad24056dd9ed52ef6360f196c6f9c31..0000000000000000000000000000000000000000 --- a/data/models/abhishek_autotrain-llama3-70b-orpo-v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "autotrain-llama3-70b-orpo-v2", - "id": "abhishek/autotrain-llama3-70b-orpo-v2", - "developer": "abhishek", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "70.554" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/abhishek_autotrain-llama3-70b-orpo-v2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5406 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5899 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2107 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2936 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4113 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4818 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/abhishek_autotrain-llama3-orpo-v2.json b/data/models/abhishek_autotrain-llama3-orpo-v2.json deleted file mode 100644 index 485d394da921b107ce95ae9a864ccf14100cdd85..0000000000000000000000000000000000000000 --- a/data/models/abhishek_autotrain-llama3-orpo-v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "autotrain-llama3-orpo-v2", - "id": "abhishek/autotrain-llama3-orpo-v2", - "developer": "abhishek", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/abhishek_autotrain-llama3-orpo-v2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4372 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3159 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0468 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2668 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3792 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2218 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/abhishek_autotrain-vr4a1-e5mms.json b/data/models/abhishek_autotrain-vr4a1-e5mms.json deleted file mode 100644 index a57e32162e36b8fcf14c663f690c0896cc30d692..0000000000000000000000000000000000000000 --- a/data/models/abhishek_autotrain-vr4a1-e5mms.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "autotrain-vr4a1-e5mms", - "id": "abhishek/autotrain-vr4a1-e5mms", - "developer": "abhishek", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "?", - "params_billions": "16.061" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/abhishek_autotrain-vr4a1-e5mms/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2142 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5001 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1412 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3196 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3891 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3667 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/abideen_medphi-4-14b-v1.json b/data/models/abideen_medphi-4-14b-v1.json deleted file mode 100644 index 4a9d51f472f6a72281b364e20e659a0584831e32..0000000000000000000000000000000000000000 --- a/data/models/abideen_medphi-4-14b-v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MedPhi-4-14B-v1", - "id": "abideen/MedPhi-4-14B-v1", - "developer": "abideen", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Phi3ForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/abideen_MedPhi-4-14B-v1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6277 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6897 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2931 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.344 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4155 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5338 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/adamo1139_yi-34b-200k-aezakmi-v2.json b/data/models/adamo1139_yi-34b-200k-aezakmi-v2.json deleted file mode 100644 index 8cfd631e140ab70fc2ffc78c637199a8281bb10e..0000000000000000000000000000000000000000 --- a/data/models/adamo1139_yi-34b-200k-aezakmi-v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Yi-34B-200K-AEZAKMI-v2", - "id": "adamo1139/Yi-34B-200K-AEZAKMI-v2", - "developer": "adamo1139", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "34.389" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/adamo1139_Yi-34B-200K-AEZAKMI-v2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4555 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5384 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0566 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3322 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3886 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4513 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/adriszmar_qaimath-qwen2.5-7b-ties.json b/data/models/adriszmar_qaimath-qwen2.5-7b-ties.json deleted file mode 100644 index 776a7c662b5afb276bf059bb90896cee5cd968ec..0000000000000000000000000000000000000000 --- a/data/models/adriszmar_qaimath-qwen2.5-7b-ties.json +++ /dev/null @@ -1,275 +0,0 @@ -{ - "model_info": { - "name": "QAIMath-Qwen2.5-7B-TIES", - "id": "adriszmar/QAIMath-Qwen2.5-7B-TIES", - "developer": "adriszmar", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/adriszmar_QAIMath-Qwen2.5-7B-TIES/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1685 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3124 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0015 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2492 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3963 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1066 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "hfopenllm_v2/adriszmar_QAIMath-Qwen2.5-7B-TIES/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1746 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3126 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.245 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4096 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1087 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/aellm_gemma-2-aeria-infinity-9b.json b/data/models/aellm_gemma-2-aeria-infinity-9b.json deleted file mode 100644 index 80953b2993c18e80fc240402ad05f52fdb922e0b..0000000000000000000000000000000000000000 --- a/data/models/aellm_gemma-2-aeria-infinity-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "gemma-2-aeria-infinity-9b", - "id": "AELLM/gemma-2-aeria-infinity-9b", - "developer": "AELLM", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "9.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/AELLM_gemma-2-aeria-infinity-9b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7594 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5983 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2145 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3339 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.402 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3862 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/aellm_gemma-2-lyco-infinity-9b.json b/data/models/aellm_gemma-2-lyco-infinity-9b.json deleted file mode 100644 index 4d34a177c23fb0ff4ca808ca6efadf3e2768b3cb..0000000000000000000000000000000000000000 --- a/data/models/aellm_gemma-2-lyco-infinity-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "gemma-2-lyco-infinity-9b", - "id": "AELLM/gemma-2-lyco-infinity-9b", - "developer": "AELLM", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/AELLM_gemma-2-lyco-infinity-9b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7316 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.584 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1707 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.328 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4006 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3787 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/aevalone_distill_qw_test.json b/data/models/aevalone_distill_qw_test.json deleted file mode 100644 index 029670ece2154b1e20f722ce62c673d6c2d78df7..0000000000000000000000000000000000000000 --- a/data/models/aevalone_distill_qw_test.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "distill_qw_test", - "id": "aevalone/distill_qw_test", - "developer": "aevalone", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/aevalone_distill_qw_test/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7409 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5246 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4781 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3003 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.386 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4092 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/agentlans_gemma2-9b-advancedfuse.json b/data/models/agentlans_gemma2-9b-advancedfuse.json deleted file mode 100644 index e562660ac8f6be675a690457d5ba75799843de14..0000000000000000000000000000000000000000 --- a/data/models/agentlans_gemma2-9b-advancedfuse.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Gemma2-9B-AdvancedFuse", - "id": "agentlans/Gemma2-9B-AdvancedFuse", - "developer": "agentlans", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "9.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/agentlans_Gemma2-9B-AdvancedFuse/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1543 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5859 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1005 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3347 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4231 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/agentlans_llama-3.2-1b-instruct-crashcourse12k.json b/data/models/agentlans_llama-3.2-1b-instruct-crashcourse12k.json deleted file mode 100644 index 0ed462be3d406a79bbbe93fc6977fd92f72b3197..0000000000000000000000000000000000000000 --- a/data/models/agentlans_llama-3.2-1b-instruct-crashcourse12k.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.2-1B-Instruct-CrashCourse12K", - "id": "agentlans/Llama-3.2-1B-Instruct-CrashCourse12K", - "developer": "agentlans", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.236" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/agentlans_Llama-3.2-1B-Instruct-CrashCourse12K/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5395 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3548 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.071 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2408 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.321 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1809 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/agentlans_llama3.1-8b-drill.json b/data/models/agentlans_llama3.1-8b-drill.json deleted file mode 100644 index 575046f41f4ff3f5912a9632518a09fb37e48fbb..0000000000000000000000000000000000000000 --- a/data/models/agentlans_llama3.1-8b-drill.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama3.1-8B-drill", - "id": "agentlans/Llama3.1-8B-drill", - "developer": "agentlans", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/agentlans_Llama3.1-8B-drill/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7652 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5016 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1715 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2676 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3672 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3776 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/agentlans_llama3.1-daredevilish-instruct.json b/data/models/agentlans_llama3.1-daredevilish-instruct.json deleted file mode 100644 index 0bdc5cd05757327f3f1efcb4eb83c173c4894846..0000000000000000000000000000000000000000 --- a/data/models/agentlans_llama3.1-daredevilish-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama3.1-Daredevilish-Instruct", - "id": "agentlans/Llama3.1-Daredevilish-Instruct", - "developer": "agentlans", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/agentlans_Llama3.1-Daredevilish-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7926 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5235 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1722 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.307 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3911 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3877 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/agentlans_llama3.1-daredevilish.json b/data/models/agentlans_llama3.1-daredevilish.json deleted file mode 100644 index e1acf2555a852c3db6198b0946b5dedcd50fdc24..0000000000000000000000000000000000000000 --- a/data/models/agentlans_llama3.1-daredevilish.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama3.1-Daredevilish", - "id": "agentlans/Llama3.1-Daredevilish", - "developer": "agentlans", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/agentlans_Llama3.1-Daredevilish/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6292 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5013 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1292 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3012 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4091 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3697 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/agentlans_llama3.1-lexihermes-superstorm.json b/data/models/agentlans_llama3.1-lexihermes-superstorm.json deleted file mode 100644 index 15a8ad0156ab208e97247f0c60c197a6305406fc..0000000000000000000000000000000000000000 --- a/data/models/agentlans_llama3.1-lexihermes-superstorm.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama3.1-LexiHermes-SuperStorm", - "id": "agentlans/Llama3.1-LexiHermes-SuperStorm", - "developer": "agentlans", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/agentlans_Llama3.1-LexiHermes-SuperStorm/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7835 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5266 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1616 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.323 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3963 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3844 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/agentlans_llama3.1-superdeepfuse-crashcourse12k.json b/data/models/agentlans_llama3.1-superdeepfuse-crashcourse12k.json deleted file mode 100644 index 3b0e8634e15ae7a931c1fca3daa5c564813a7328..0000000000000000000000000000000000000000 --- a/data/models/agentlans_llama3.1-superdeepfuse-crashcourse12k.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama3.1-SuperDeepFuse-CrashCourse12K", - "id": "agentlans/Llama3.1-SuperDeepFuse-CrashCourse12K", - "developer": "agentlans", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/agentlans_Llama3.1-SuperDeepFuse-CrashCourse12K/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7187 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5216 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1805 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3129 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4026 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3631 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/agentlans_llama3.1-superdeepfuse.json b/data/models/agentlans_llama3.1-superdeepfuse.json deleted file mode 100644 index 1da3897f1e1388ac3df1f37fabdeb20b6e4b9c2e..0000000000000000000000000000000000000000 --- a/data/models/agentlans_llama3.1-superdeepfuse.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama3.1-SuperDeepFuse", - "id": "agentlans/Llama3.1-SuperDeepFuse", - "developer": "agentlans", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/agentlans_Llama3.1-SuperDeepFuse/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7762 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5049 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1828 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2743 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3699 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3775 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/agentlans_qwen2.5-0.5b-instruct-crashcourse-dropout.json b/data/models/agentlans_qwen2.5-0.5b-instruct-crashcourse-dropout.json deleted file mode 100644 index 1f2aa10d63e324f0d7a207ff5da675b77caf8dbf..0000000000000000000000000000000000000000 --- a/data/models/agentlans_qwen2.5-0.5b-instruct-crashcourse-dropout.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-Instruct-CrashCourse-dropout", - "id": "agentlans/Qwen2.5-0.5B-Instruct-CrashCourse-dropout", - "developer": "agentlans", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/agentlans_Qwen2.5-0.5B-Instruct-CrashCourse-dropout/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2949 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3312 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0423 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2634 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3342 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1608 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/agi-0_art-v0-3b.json b/data/models/agi-0_art-v0-3b.json deleted file mode 100644 index 3c98e9e97b0c284f0e120bac54e840be8de74361..0000000000000000000000000000000000000000 --- a/data/models/agi-0_art-v0-3b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Art-v0-3B", - "id": "AGI-0/Art-v0-3B", - "developer": "AGI-0", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.086" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/AGI-0_Art-v0-3B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3192 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3401 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2462 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2592 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3768 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1179 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/agi-0_artificium-llama3.1-8b-001.json b/data/models/agi-0_artificium-llama3.1-8b-001.json deleted file mode 100644 index 76757344b2e80c02caa3ac9b75e5c831a5ef0bfd..0000000000000000000000000000000000000000 --- a/data/models/agi-0_artificium-llama3.1-8b-001.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Artificium-llama3.1-8B-001", - "id": "AGI-0/Artificium-llama3.1-8B-001", - "developer": "AGI-0", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/AGI-0_Artificium-llama3.1-8B-001/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5248 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4256 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.136 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2659 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3795 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3182 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/agi-0_smartllama3.1-8b-001.json b/data/models/agi-0_smartllama3.1-8b-001.json deleted file mode 100644 index f2d88c585a483d3387c7e060bfd24b0fe7d83a1f..0000000000000000000000000000000000000000 --- a/data/models/agi-0_smartllama3.1-8b-001.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "smartllama3.1-8B-001", - "id": "AGI-0/smartllama3.1-8B-001", - "developer": "AGI-0", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/AGI-0_smartllama3.1-8B-001/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3518 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.467 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1299 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3062 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4386 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3487 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ahdoot_structuredthinker-v0.3-morestructure.json b/data/models/ahdoot_structuredthinker-v0.3-morestructure.json deleted file mode 100644 index f2e8da87ee843f311b50c0028271c4b6f271a699..0000000000000000000000000000000000000000 --- a/data/models/ahdoot_structuredthinker-v0.3-morestructure.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "StructuredThinker-v0.3-MoreStructure", - "id": "Ahdoot/StructuredThinker-v0.3-MoreStructure", - "developer": "Ahdoot", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.397" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Ahdoot_StructuredThinker-v0.3-MoreStructure/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4193 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4838 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2908 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.297 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4158 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.361 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ahdoot_test_stealththinker.json b/data/models/ahdoot_test_stealththinker.json deleted file mode 100644 index 63a1dea7a314af7495e2135186ade8690cb588f4..0000000000000000000000000000000000000000 --- a/data/models/ahdoot_test_stealththinker.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Test_StealthThinker", - "id": "Ahdoot/Test_StealthThinker", - "developer": "Ahdoot", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.086" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Ahdoot_Test_StealthThinker/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.422 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4647 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.179 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2961 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.428 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3597 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ahjeong_mmpo_gemma_7b.json b/data/models/ahjeong_mmpo_gemma_7b.json deleted file mode 100644 index a7c9ef21650516f3c2c309e5a238ade1dbb0142c..0000000000000000000000000000000000000000 --- a/data/models/ahjeong_mmpo_gemma_7b.json +++ /dev/null @@ -1,144 +0,0 @@ -{ - "model_info": { - "name": "Ahjeong/MMPO_Gemma_7b", - "id": "Ahjeong/MMPO_Gemma_7b", - "developer": "Ahjeong", - "additional_details": { - "model_type": "DPO" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/Ahjeong_MMPO_Gemma_7b/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7587 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9693 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.614 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7135 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7756 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6831 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ahjeong_mmpo_gemma_7b_gamma1.1_epoch3.json b/data/models/ahjeong_mmpo_gemma_7b_gamma1.1_epoch3.json deleted file mode 100644 index c1c246cb6304041c5c5bebfd392923b8ab8378a2..0000000000000000000000000000000000000000 --- a/data/models/ahjeong_mmpo_gemma_7b_gamma1.1_epoch3.json +++ /dev/null @@ -1,144 +0,0 @@ -{ - "model_info": { - "name": "Ahjeong/MMPO_Gemma_7b_gamma1.1_epoch3", - "id": "Ahjeong/MMPO_Gemma_7b_gamma1.1_epoch3", - "developer": "Ahjeong", - "additional_details": { - "model_type": "DPO" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/Ahjeong_MMPO_Gemma_7b_gamma1.1_epoch3/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7652 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9721 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6338 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7635 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7284 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6913 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ahmeda335_13_outof_32_pruned_layers_llama3.1-8b.json b/data/models/ahmeda335_13_outof_32_pruned_layers_llama3.1-8b.json deleted file mode 100644 index dcdcbfb4ba062c010daa10b315a4efdd396dafaa..0000000000000000000000000000000000000000 --- a/data/models/ahmeda335_13_outof_32_pruned_layers_llama3.1-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "13_outOf_32_pruned_layers_llama3.1-8b", - "id": "ahmeda335/13_outOf_32_pruned_layers_llama3.1-8b", - "developer": "ahmeda335", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "5.195" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ahmeda335_13_outOf_32_pruned_layers_llama3.1-8b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1748 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2883 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2592 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3803 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1129 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ai-mo_numinamath-7b-cot.json b/data/models/ai-mo_numinamath-7b-cot.json deleted file mode 100644 index 8b8ea0713c9548c3a2dd030f4d27ee49ef94b2d1..0000000000000000000000000000000000000000 --- a/data/models/ai-mo_numinamath-7b-cot.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "NuminaMath-7B-CoT", - "id": "AI-MO/NuminaMath-7B-CoT", - "developer": "AI-MO", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "6.91" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/AI-MO_NuminaMath-7B-CoT/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2689 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4314 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2696 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2659 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3303 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2868 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ai-mo_numinamath-7b-tir.json b/data/models/ai-mo_numinamath-7b-tir.json deleted file mode 100644 index 1faed74944e882810a9ec50047430420bb669b42..0000000000000000000000000000000000000000 --- a/data/models/ai-mo_numinamath-7b-tir.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "NuminaMath-7B-TIR", - "id": "AI-MO/NuminaMath-7B-TIR", - "developer": "AI-MO", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "6.91" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/AI-MO_NuminaMath-7B-TIR/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2756 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4144 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1609 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2584 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3509 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2733 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ai-sweden-models_gpt-sw3-40b.json b/data/models/ai-sweden-models_gpt-sw3-40b.json deleted file mode 100644 index b7bcdae9a2a7c575de775bc39f59abbf5e8b79d4..0000000000000000000000000000000000000000 --- a/data/models/ai-sweden-models_gpt-sw3-40b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "gpt-sw3-40b", - "id": "AI-Sweden-Models/gpt-sw3-40b", - "developer": "AI-Sweden-Models", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "GPT2LMHeadModel", - "params_billions": "39.927" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/AI-Sweden-Models_gpt-sw3-40b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.147 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3268 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0174 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2349 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3632 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1276 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ai-sweden-models_llama-3-8b-instruct.json b/data/models/ai-sweden-models_llama-3-8b-instruct.json deleted file mode 100644 index 94f6943f5812b8673f91704a97c1766b1dcda291..0000000000000000000000000000000000000000 --- a/data/models/ai-sweden-models_llama-3-8b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-8B-instruct", - "id": "AI-Sweden-Models/Llama-3-8B-instruct", - "developer": "AI-Sweden-Models", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/AI-Sweden-Models_Llama-3-8B-instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2401 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4173 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0385 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2659 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4771 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2597 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ai21_j1-grande-v1-17b.json b/data/models/ai21_j1-grande-v1-17b.json deleted file mode 100644 index 892a2edd48a51d5c486e598f8ce76e840d1ee016..0000000000000000000000000000000000000000 --- a/data/models/ai21_j1-grande-v1-17b.json +++ /dev/null @@ -1,674 +0,0 @@ -{ - "model_info": { - "name": "J1-Grande v1 17B", - "id": "ai21/J1-Grande-v1-17B", - "developer": "ai21", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_classic/ai21_J1-Grande-v1-17B/1774096308.339228", - "retrieved_timestamp": "1774096308.339228", - "source_metadata": { - "source_name": "helm_classic", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_classic", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_classic", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperform on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.433, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Calibration": "{\"description\": \"\", \"tab\": \"Calibration\", \"score\": \"0.6221919576066971\"}", - "Mean win rate - Robustness": "{\"description\": \"\", \"tab\": \"Robustness\", \"score\": \"0.4225080073800875\"}", - "Mean win rate - Fairness": "{\"description\": \"\", \"tab\": \"Fairness\", \"score\": \"0.4539316449216338\"}", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.31716008771929827\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}", - "Mean win rate - Bias": "{\"description\": \"\", \"tab\": \"Bias\", \"score\": \"0.5580147362700336\"}", - "Mean win rate - Toxicity": "{\"description\": \"\", \"tab\": \"Toxicity\", \"score\": \"0.6300489633822968\"}", - "Mean win rate - Summarization metrics": "{\"description\": \"\", \"tab\": \"Summarization metrics\", \"score\": \"0.6689640768588138\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.27, - "details": { - "description": "min=0.2, mean=0.27, max=0.35, sum=4.047 (15)", - "tab": "Accuracy", - "MMLU - ECE (10-bin)": "{\"description\": \"min=0.063, mean=0.114, max=0.154, sum=1.708 (15)\", \"tab\": \"Calibration\", \"score\": \"0.11389257817699022\"}", - "MMLU - EM (Robustness)": "{\"description\": \"min=0.15, mean=0.225, max=0.27, sum=3.377 (15)\", \"tab\": \"Robustness\", \"score\": \"0.22511111111111112\"}", - "MMLU - EM (Fairness)": "{\"description\": \"min=0.158, mean=0.232, max=0.29, sum=3.474 (15)\", \"tab\": \"Fairness\", \"score\": \"0.23159064327485382\"}", - "MMLU - Denoised inference time (s)": "{\"description\": \"min=0.381, mean=0.411, max=0.466, sum=6.166 (15)\", \"tab\": \"Efficiency\", \"score\": \"0.41104061293859656\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=1542 (15)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=75 (15)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (15)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=308.59, mean=396.74, max=552.719, sum=5951.098 (15)\", \"tab\": \"General information\", \"score\": \"396.73985964912276\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=15 (15)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "MMLU - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=45 (15)\", \"tab\": \"General information\", \"score\": \"3.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "BoolQ", - "source_data": { - "dataset_name": "BoolQ", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on BoolQ", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.722, - "details": { - "description": "min=0.712, mean=0.722, max=0.733, sum=2.165 (3)", - "tab": "Accuracy", - "BoolQ - ECE (10-bin)": "{\"description\": \"min=0.139, mean=0.154, max=0.169, sum=0.462 (3)\", \"tab\": \"Calibration\", \"score\": \"0.15409092997354776\"}", - "BoolQ - EM (Robustness)": "{\"description\": \"min=0.632, mean=0.643, max=0.658, sum=1.929 (3)\", \"tab\": \"Robustness\", \"score\": \"0.6429999999999999\"}", - "BoolQ - EM (Fairness)": "{\"description\": \"min=0.656, mean=0.678, max=0.695, sum=2.035 (3)\", \"tab\": \"Fairness\", \"score\": \"0.6783333333333333\"}", - "BoolQ - Denoised inference time (s)": "{\"description\": \"min=0.47, mean=0.535, max=0.624, sum=1.606 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.5352501416015627\"}", - "BoolQ - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "BoolQ - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "BoolQ - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "BoolQ - # prompt tokens": "{\"description\": \"min=506.985, mean=694.652, max=952.985, sum=2083.955 (3)\", \"tab\": \"General information\", \"score\": \"694.6516666666666\"}", - "BoolQ - # output tokens": "{\"description\": \"min=2, mean=2, max=2, sum=6 (3)\", \"tab\": \"General information\", \"score\": \"2.0\"}", - "BoolQ - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "BoolQ - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.672, - "details": { - "description": "min=0.664, mean=0.672, max=0.68, sum=2.016 (3)", - "tab": "Accuracy", - "NarrativeQA - ECE (10-bin)": "{\"description\": \"min=0.039, mean=0.047, max=0.062, sum=0.141 (3)\", \"tab\": \"Calibration\", \"score\": \"0.04705310707412085\"}", - "NarrativeQA - F1 (Robustness)": "{\"description\": \"min=0.409, mean=0.477, max=0.522, sum=1.432 (3)\", \"tab\": \"Robustness\", \"score\": \"0.47749086119263257\"}", - "NarrativeQA - F1 (Fairness)": "{\"description\": \"min=0.526, mean=0.547, max=0.563, sum=1.641 (3)\", \"tab\": \"Fairness\", \"score\": \"0.5469545337986748\"}", - "NarrativeQA - Denoised inference time (s)": "{\"description\": \"min=0.892, mean=0.923, max=0.955, sum=2.769 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.9228662338615026\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=1065 (3)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=2.166, mean=2.639, max=3.225, sum=7.918 (3)\", \"tab\": \"General information\", \"score\": \"2.63943661971831\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=1598.614, mean=1692.218, max=1777.299, sum=5076.654 (3)\", \"tab\": \"General information\", \"score\": \"1692.2178403755868\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=4.324, mean=4.528, max=4.701, sum=13.583 (3)\", \"tab\": \"General information\", \"score\": \"4.527699530516432\"}", - "NarrativeQA - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NarrativeQA - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NarrativeQA - Stereotypes (gender)": "{\"description\": \"min=0.5, mean=0.5, max=0.5, sum=1 (2)\", \"tab\": \"Bias\", \"score\": \"0.5\"}", - "NarrativeQA - Representation (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=1.333 (2)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666667\"}", - "NarrativeQA - Representation (gender)": "{\"description\": \"min=0.15, mean=0.164, max=0.18, sum=0.491 (3)\", \"tab\": \"Bias\", \"score\": \"0.1636261091893518\"}", - "NarrativeQA - Toxic fraction": "{\"description\": \"min=0.008, mean=0.014, max=0.017, sum=0.042 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.014084507042253521\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (open-book)", - "source_data": { - "dataset_name": "NaturalQuestions (open-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (open-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.578, - "details": { - "description": "min=0.561, mean=0.578, max=0.59, sum=1.734 (3)", - "tab": "Accuracy", - "NaturalQuestions (closed-book) - ECE (10-bin)": "{\"description\": \"min=0.027, mean=0.029, max=0.03, sum=0.087 (3)\", \"tab\": \"Calibration\", \"score\": \"0.028955351873343083\"}", - "NaturalQuestions (open-book) - ECE (10-bin)": "{\"description\": \"min=0.073, mean=0.081, max=0.097, sum=0.243 (3)\", \"tab\": \"Calibration\", \"score\": \"0.08114120238748938\"}", - "NaturalQuestions (closed-book) - F1 (Robustness)": "{\"description\": \"min=0.164, mean=0.17, max=0.175, sum=0.511 (3)\", \"tab\": \"Robustness\", \"score\": \"0.17025794044565556\"}", - "NaturalQuestions (open-book) - F1 (Robustness)": "{\"description\": \"min=0.449, mean=0.478, max=0.494, sum=1.433 (3)\", \"tab\": \"Robustness\", \"score\": \"0.4776074011626843\"}", - "NaturalQuestions (closed-book) - F1 (Fairness)": "{\"description\": \"min=0.185, mean=0.187, max=0.189, sum=0.562 (3)\", \"tab\": \"Fairness\", \"score\": \"0.1872477522460834\"}", - "NaturalQuestions (open-book) - F1 (Fairness)": "{\"description\": \"min=0.501, mean=0.521, max=0.534, sum=1.563 (3)\", \"tab\": \"Fairness\", \"score\": \"0.5209919156580172\"}", - "NaturalQuestions (closed-book) - Denoised inference time (s)": "{\"description\": \"min=0.437, mean=0.466, max=0.494, sum=1.399 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.46640491796874967\"}", - "NaturalQuestions (open-book) - Denoised inference time (s)": "{\"description\": \"min=0.774, mean=0.873, max=0.927, sum=2.618 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.8728225097656246\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=94.377, mean=99.377, max=102.377, sum=298.131 (3)\", \"tab\": \"General information\", \"score\": \"99.377\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=4.791, mean=5.971, max=7.18, sum=17.913 (3)\", \"tab\": \"General information\", \"score\": \"5.971\"}", - "NaturalQuestions (closed-book) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.568, mean=4.666, max=4.734, sum=13.999 (3)\", \"tab\": \"General information\", \"score\": \"4.666333333333333\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.038, mean=0.038, max=0.038, sum=0.114 (3)\", \"tab\": \"General information\", \"score\": \"0.038\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1136.933, mean=1418.457, max=1595.508, sum=4255.37 (3)\", \"tab\": \"General information\", \"score\": \"1418.4566666666667\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=6.302, mean=6.538, max=6.976, sum=19.615 (3)\", \"tab\": \"General information\", \"score\": \"6.538333333333333\"}", - "NaturalQuestions (open-book) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NaturalQuestions (closed-book) - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - Stereotypes (gender)": "{\"description\": \"min=0.5, mean=0.5, max=0.5, sum=1.5 (3)\", \"tab\": \"Bias\", \"score\": \"0.5\"}", - "NaturalQuestions (closed-book) - Representation (race)": "{\"description\": \"min=0.473, mean=0.521, max=0.556, sum=1.564 (3)\", \"tab\": \"Bias\", \"score\": \"0.5214747518446415\"}", - "NaturalQuestions (closed-book) - Representation (gender)": "{\"description\": \"min=0, mean=0.033, max=0.1, sum=0.1 (3)\", \"tab\": \"Bias\", \"score\": \"0.033333333333333326\"}", - "NaturalQuestions (open-book) - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=0.667 (1)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666667\"}", - "NaturalQuestions (open-book) - Stereotypes (gender)": "{\"description\": \"min=0.346, mean=0.346, max=0.346, sum=1.038 (3)\", \"tab\": \"Bias\", \"score\": \"0.3461538461538461\"}", - "NaturalQuestions (open-book) - Representation (race)": "{\"description\": \"min=0.45, mean=0.488, max=0.521, sum=1.463 (3)\", \"tab\": \"Bias\", \"score\": \"0.48764942579375564\"}", - "NaturalQuestions (open-book) - Representation (gender)": "{\"description\": \"min=0.111, mean=0.113, max=0.118, sum=0.34 (3)\", \"tab\": \"Bias\", \"score\": \"0.11339991677070331\"}", - "NaturalQuestions (closed-book) - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}", - "NaturalQuestions (open-book) - Toxic fraction": "{\"description\": \"min=0, mean=0.001, max=0.001, sum=0.002 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.0006666666666666666\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "QuAC", - "source_data": { - "dataset_name": "QuAC", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on QuAC", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.362, - "details": { - "description": "min=0.355, mean=0.362, max=0.372, sum=1.087 (3)", - "tab": "Accuracy", - "QuAC - ECE (10-bin)": "{\"description\": \"min=0.019, mean=0.036, max=0.06, sum=0.107 (3)\", \"tab\": \"Calibration\", \"score\": \"0.03571925908384949\"}", - "QuAC - F1 (Robustness)": "{\"description\": \"min=0.215, mean=0.219, max=0.227, sum=0.658 (3)\", \"tab\": \"Robustness\", \"score\": \"0.21921244416502939\"}", - "QuAC - F1 (Fairness)": "{\"description\": \"min=0.266, mean=0.274, max=0.282, sum=0.821 (3)\", \"tab\": \"Fairness\", \"score\": \"0.27362985580399246\"}", - "QuAC - Denoised inference time (s)": "{\"description\": \"min=1.302, mean=1.413, max=1.478, sum=4.24 (3)\", \"tab\": \"Efficiency\", \"score\": \"1.4134776341145843\"}", - "QuAC - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "QuAC - # train": "{\"description\": \"min=1.788, mean=1.829, max=1.88, sum=5.486 (3)\", \"tab\": \"General information\", \"score\": \"1.8286666666666667\"}", - "QuAC - truncated": "{\"description\": \"min=0.001, mean=0.001, max=0.001, sum=0.003 (3)\", \"tab\": \"General information\", \"score\": \"0.001\"}", - "QuAC - # prompt tokens": "{\"description\": \"min=1645.856, mean=1698.711, max=1730.814, sum=5096.134 (3)\", \"tab\": \"General information\", \"score\": \"1698.7113333333334\"}", - "QuAC - # output tokens": "{\"description\": \"min=22.154, mean=27.786, max=31.692, sum=83.357 (3)\", \"tab\": \"General information\", \"score\": \"27.785666666666668\"}", - "QuAC - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "QuAC - Stereotypes (race)": "{\"description\": \"min=0.58, mean=0.6, max=0.639, sum=1.799 (3)\", \"tab\": \"Bias\", \"score\": \"0.5996635891593876\"}", - "QuAC - Stereotypes (gender)": "{\"description\": \"min=0.415, mean=0.428, max=0.44, sum=1.283 (3)\", \"tab\": \"Bias\", \"score\": \"0.42780085419627883\"}", - "QuAC - Representation (race)": "{\"description\": \"min=0.298, mean=0.34, max=0.378, sum=1.019 (3)\", \"tab\": \"Bias\", \"score\": \"0.3397817992618246\"}", - "QuAC - Representation (gender)": "{\"description\": \"min=0.237, mean=0.242, max=0.25, sum=0.727 (3)\", \"tab\": \"Bias\", \"score\": \"0.24231770708576347\"}", - "QuAC - Toxic fraction": "{\"description\": \"min=0.004, mean=0.004, max=0.004, sum=0.012 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.004\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "HellaSwag", - "source_data": { - "dataset_name": "HellaSwag", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on HellaSwag", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.739, - "details": { - "description": "min=0.739, mean=0.739, max=0.739, sum=0.739 (1)", - "tab": "Accuracy", - "HellaSwag - ECE (10-bin)": "{\"description\": \"min=0.213, mean=0.213, max=0.213, sum=0.213 (1)\", \"tab\": \"Calibration\", \"score\": \"0.21338082493857388\"}", - "HellaSwag - EM (Robustness)": "{\"description\": \"min=0.695, mean=0.695, max=0.695, sum=0.695 (1)\", \"tab\": \"Robustness\", \"score\": \"0.695\"}", - "HellaSwag - EM (Fairness)": "{\"description\": \"min=0.58, mean=0.58, max=0.58, sum=0.58 (1)\", \"tab\": \"Fairness\", \"score\": \"0.58\"}", - "HellaSwag - Denoised inference time (s)": "{\"description\": \"min=0.33, mean=0.33, max=0.33, sum=0.33 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.3304377109375\"}", - "HellaSwag - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "HellaSwag - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "HellaSwag - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "HellaSwag - # prompt tokens": "{\"description\": \"min=62.466, mean=62.466, max=62.466, sum=62.466 (1)\", \"tab\": \"General information\", \"score\": \"62.466\"}", - "HellaSwag - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "HellaSwag - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.52, - "details": { - "description": "min=0.52, mean=0.52, max=0.52, sum=0.52 (1)", - "tab": "Accuracy", - "OpenbookQA - ECE (10-bin)": "{\"description\": \"min=0.258, mean=0.258, max=0.258, sum=0.258 (1)\", \"tab\": \"Calibration\", \"score\": \"0.25849314658751343\"}", - "OpenbookQA - EM (Robustness)": "{\"description\": \"min=0.424, mean=0.424, max=0.424, sum=0.424 (1)\", \"tab\": \"Robustness\", \"score\": \"0.424\"}", - "OpenbookQA - EM (Fairness)": "{\"description\": \"min=0.472, mean=0.472, max=0.472, sum=0.472 (1)\", \"tab\": \"Fairness\", \"score\": \"0.472\"}", - "OpenbookQA - Denoised inference time (s)": "{\"description\": \"min=0.281, mean=0.281, max=0.281, sum=0.281 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.280719578125\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=4.348, mean=4.348, max=4.348, sum=4.348 (1)\", \"tab\": \"General information\", \"score\": \"4.348\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "TruthfulQA", - "source_data": { - "dataset_name": "TruthfulQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on TruthfulQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.193, - "details": { - "description": "min=0.171, mean=0.193, max=0.217, sum=0.58 (3)", - "tab": "Accuracy", - "TruthfulQA - ECE (10-bin)": "{\"description\": \"min=0.064, mean=0.091, max=0.109, sum=0.273 (3)\", \"tab\": \"Calibration\", \"score\": \"0.09083831911084679\"}", - "TruthfulQA - EM (Robustness)": "{\"description\": \"min=0.116, mean=0.142, max=0.159, sum=0.425 (3)\", \"tab\": \"Robustness\", \"score\": \"0.1416921508664628\"}", - "TruthfulQA - EM (Fairness)": "{\"description\": \"min=0.138, mean=0.163, max=0.182, sum=0.489 (3)\", \"tab\": \"Fairness\", \"score\": \"0.16309887869520898\"}", - "TruthfulQA - Denoised inference time (s)": "{\"description\": \"min=0.384, mean=0.396, max=0.403, sum=1.189 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.39626294915902127\"}", - "TruthfulQA - # eval": "{\"description\": \"min=654, mean=654, max=654, sum=1962 (3)\", \"tab\": \"General information\", \"score\": \"654.0\"}", - "TruthfulQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "TruthfulQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "TruthfulQA - # prompt tokens": "{\"description\": \"min=317.682, mean=355.015, max=375.682, sum=1065.046 (3)\", \"tab\": \"General information\", \"score\": \"355.0152905198777\"}", - "TruthfulQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=3 (3)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "TruthfulQA - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MS MARCO (TREC)", - "source_data": { - "dataset_name": "MS MARCO (TREC)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "NDCG@10 on MS MARCO (TREC)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.341, - "details": { - "description": "min=0.31, mean=0.341, max=0.389, sum=1.022 (3)", - "tab": "Accuracy", - "MS MARCO (regular) - RR@10 (Robustness)": "{\"description\": \"min=0.105, mean=0.121, max=0.133, sum=0.362 (3)\", \"tab\": \"Robustness\", \"score\": \"0.12069748677248683\"}", - "MS MARCO (TREC) - NDCG@10 (Robustness)": "{\"description\": \"min=0.27, mean=0.297, max=0.328, sum=0.89 (3)\", \"tab\": \"Robustness\", \"score\": \"0.29680328755123014\"}", - "MS MARCO (regular) - RR@10 (Fairness)": "{\"description\": \"min=0.126, mean=0.138, max=0.155, sum=0.414 (3)\", \"tab\": \"Fairness\", \"score\": \"0.1378972222222222\"}", - "MS MARCO (TREC) - NDCG@10 (Fairness)": "{\"description\": \"min=0.296, mean=0.328, max=0.372, sum=0.985 (3)\", \"tab\": \"Fairness\", \"score\": \"0.3284974893691146\"}", - "MS MARCO (regular) - Denoised inference time (s)": "{\"description\": \"min=0.415, mean=0.428, max=0.44, sum=1.283 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.4278073636067708\"}", - "MS MARCO (TREC) - Denoised inference time (s)": "{\"description\": \"min=0.412, mean=0.424, max=0.437, sum=1.272 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.42392066375968995\"}", - "MS MARCO (regular) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "MS MARCO (regular) - # train": "{\"description\": \"min=2, mean=2, max=2, sum=6 (3)\", \"tab\": \"General information\", \"score\": \"2.0\"}", - "MS MARCO (regular) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MS MARCO (regular) - # prompt tokens": "{\"description\": \"min=349.303, mean=385.636, max=423.303, sum=1156.909 (3)\", \"tab\": \"General information\", \"score\": \"385.63633333333337\"}", - "MS MARCO (regular) - # output tokens": "{\"description\": \"min=2.004, mean=2.011, max=2.023, sum=6.034 (3)\", \"tab\": \"General information\", \"score\": \"2.0113333333333334\"}", - "MS MARCO (regular) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "MS MARCO (TREC) - # eval": "{\"description\": \"min=43, mean=43, max=43, sum=129 (3)\", \"tab\": \"General information\", \"score\": \"43.0\"}", - "MS MARCO (TREC) - # train": "{\"description\": \"min=2, mean=2, max=2, sum=6 (3)\", \"tab\": \"General information\", \"score\": \"2.0\"}", - "MS MARCO (TREC) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MS MARCO (TREC) - # prompt tokens": "{\"description\": \"min=337.047, mean=373.38, max=411.047, sum=1120.14 (3)\", \"tab\": \"General information\", \"score\": \"373.3798449612403\"}", - "MS MARCO (TREC) - # output tokens": "{\"description\": \"min=2.023, mean=2.023, max=2.023, sum=6.07 (3)\", \"tab\": \"General information\", \"score\": \"2.0232558139534884\"}", - "MS MARCO (TREC) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "MS MARCO (regular) - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "MS MARCO (TREC) - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CNN/DailyMail", - "source_data": { - "dataset_name": "CNN/DailyMail", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on CNN/DailyMail", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.143, - "details": { - "description": "min=0.127, mean=0.143, max=0.163, sum=0.859 (6)", - "tab": "Accuracy", - "CNN/DailyMail - Denoised inference time (s)": "{\"description\": \"min=1.956, mean=2.074, max=2.263, sum=12.445 (6)\", \"tab\": \"Efficiency\", \"score\": \"2.074164002425339\"}", - "CNN/DailyMail - # eval": "{\"description\": \"min=466, mean=466, max=466, sum=2796 (6)\", \"tab\": \"General information\", \"score\": \"466.0\"}", - "CNN/DailyMail - # train": "{\"description\": \"min=5, mean=5, max=5, sum=30 (6)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CNN/DailyMail - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (6)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CNN/DailyMail - # prompt tokens": "{\"description\": \"min=1203.032, mean=1213.032, max=1224.032, sum=7278.193 (6)\", \"tab\": \"General information\", \"score\": \"1213.0321888412018\"}", - "CNN/DailyMail - # output tokens": "{\"description\": \"min=61.569, mean=67.049, max=76.034, sum=402.296 (6)\", \"tab\": \"General information\", \"score\": \"67.04935622317596\"}", - "CNN/DailyMail - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=18 (6)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "CNN/DailyMail - Stereotypes (race)": "{\"description\": \"min=0.608, mean=0.633, max=0.647, sum=3.801 (6)\", \"tab\": \"Bias\", \"score\": \"0.6334968330766649\"}", - "CNN/DailyMail - Stereotypes (gender)": "{\"description\": \"min=0.39, mean=0.4, max=0.407, sum=2.398 (6)\", \"tab\": \"Bias\", \"score\": \"0.39959768497778553\"}", - "CNN/DailyMail - Representation (race)": "{\"description\": \"min=0.263, mean=0.351, max=0.399, sum=2.104 (6)\", \"tab\": \"Bias\", \"score\": \"0.3506178570090534\"}", - "CNN/DailyMail - Representation (gender)": "{\"description\": \"min=0.115, mean=0.13, max=0.14, sum=0.782 (6)\", \"tab\": \"Bias\", \"score\": \"0.1303299541894603\"}", - "CNN/DailyMail - Toxic fraction": "{\"description\": \"min=0, mean=0.001, max=0.002, sum=0.009 (6)\", \"tab\": \"Toxicity\", \"score\": \"0.001430615164520744\"}", - "CNN/DailyMail - SummaC": "{\"description\": \"min=0.514, mean=0.539, max=0.586, sum=1.617 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.5391092885196874\"}", - "CNN/DailyMail - QAFactEval": "{\"description\": \"min=4.706, mean=4.81, max=4.896, sum=28.859 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"4.809910581145076\"}", - "CNN/DailyMail - BERTScore (F1)": "{\"description\": \"min=0.247, mean=0.275, max=0.302, sum=0.824 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.2747429286177279\"}", - "CNN/DailyMail - Coverage": "{\"description\": \"min=0.966, mean=0.973, max=0.984, sum=5.84 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"0.9733042514029583\"}", - "CNN/DailyMail - Density": "{\"description\": \"min=31.118, mean=41.027, max=60.066, sum=246.163 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"41.02711755812993\"}", - "CNN/DailyMail - Compression": "{\"description\": \"min=8.092, mean=9.888, max=11.258, sum=59.326 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"9.887609814491976\"}", - "CNN/DailyMail - HumanEval-faithfulness": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-relevance": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-coherence": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "XSUM", - "source_data": { - "dataset_name": "XSUM", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on XSUM", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.122, - "details": { - "description": "min=0.118, mean=0.122, max=0.127, sum=0.733 (6)", - "tab": "Accuracy", - "XSUM - Denoised inference time (s)": "{\"description\": \"min=1.055, mean=1.07, max=1.082, sum=6.42 (6)\", \"tab\": \"Efficiency\", \"score\": \"1.0700079645773009\"}", - "XSUM - # eval": "{\"description\": \"min=518, mean=518, max=518, sum=3108 (6)\", \"tab\": \"General information\", \"score\": \"518.0\"}", - "XSUM - # train": "{\"description\": \"min=5, mean=5, max=5, sum=30 (6)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "XSUM - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (6)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "XSUM - # prompt tokens": "{\"description\": \"min=1099.388, mean=1133.388, max=1172.388, sum=6800.328 (6)\", \"tab\": \"General information\", \"score\": \"1133.388030888031\"}", - "XSUM - # output tokens": "{\"description\": \"min=19.975, mean=20.468, max=21.141, sum=122.807 (6)\", \"tab\": \"General information\", \"score\": \"20.467824967824967\"}", - "XSUM - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=18 (6)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "XSUM - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=4 (6)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "XSUM - Stereotypes (gender)": "{\"description\": \"min=0.417, mean=0.442, max=0.485, sum=2.652 (6)\", \"tab\": \"Bias\", \"score\": \"0.44203142536475876\"}", - "XSUM - Representation (race)": "{\"description\": \"min=0.439, mean=0.557, max=0.667, sum=3.34 (6)\", \"tab\": \"Bias\", \"score\": \"0.5566296694116243\"}", - "XSUM - Representation (gender)": "{\"description\": \"min=0.149, mean=0.171, max=0.211, sum=1.025 (6)\", \"tab\": \"Bias\", \"score\": \"0.17086307216738958\"}", - "XSUM - Toxic fraction": "{\"description\": \"min=0, mean=0.002, max=0.004, sum=0.012 (6)\", \"tab\": \"Toxicity\", \"score\": \"0.0019305019305019308\"}", - "XSUM - SummaC": "{\"description\": \"min=-0.282, mean=-0.272, max=-0.264, sum=-0.815 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"-0.2715132814883572\"}", - "XSUM - QAFactEval": "{\"description\": \"min=3.221, mean=3.447, max=3.575, sum=20.68 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"3.446713620425662\"}", - "XSUM - BERTScore (F1)": "{\"description\": \"min=0.424, mean=0.429, max=0.434, sum=1.287 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.4288941077256343\"}", - "XSUM - Coverage": "{\"description\": \"min=0.78, mean=0.783, max=0.785, sum=4.696 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"0.7826042118856411\"}", - "XSUM - Density": "{\"description\": \"min=2.514, mean=2.64, max=2.767, sum=15.838 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"2.6397086455700927\"}", - "XSUM - Compression": "{\"description\": \"min=18.382, mean=19.012, max=19.445, sum=114.069 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"19.011567725134377\"}", - "XSUM - HumanEval-faithfulness": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-relevance": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-coherence": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "IMDB", - "source_data": { - "dataset_name": "IMDB", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on IMDB", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.953, - "details": { - "description": "min=0.947, mean=0.953, max=0.957, sum=2.859 (3)", - "tab": "Accuracy", - "IMDB - ECE (10-bin)": "{\"description\": \"min=0.152, mean=0.158, max=0.166, sum=0.473 (3)\", \"tab\": \"Calibration\", \"score\": \"0.15775206410447826\"}", - "IMDB - EM (Robustness)": "{\"description\": \"min=0.932, mean=0.941, max=0.948, sum=2.822 (3)\", \"tab\": \"Robustness\", \"score\": \"0.9406666666666667\"}", - "IMDB - EM (Fairness)": "{\"description\": \"min=0.94, mean=0.946, max=0.95, sum=2.839 (3)\", \"tab\": \"Fairness\", \"score\": \"0.9463333333333331\"}", - "IMDB - Denoised inference time (s)": "{\"description\": \"min=0.59, mean=0.732, max=0.881, sum=2.197 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.7321998525390631\"}", - "IMDB - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "IMDB - # train": "{\"description\": \"min=4.915, mean=4.972, max=5, sum=14.915 (3)\", \"tab\": \"General information\", \"score\": \"4.971666666666667\"}", - "IMDB - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IMDB - # prompt tokens": "{\"description\": \"min=853.851, mean=1281.577, max=1725.03, sum=3844.732 (3)\", \"tab\": \"General information\", \"score\": \"1281.5773333333334\"}", - "IMDB - # output tokens": "{\"description\": \"min=2, mean=2, max=2, sum=6 (3)\", \"tab\": \"General information\", \"score\": \"2.0\"}", - "IMDB - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "IMDB - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CivilComments", - "source_data": { - "dataset_name": "CivilComments", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on CivilComments", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.529, - "details": { - "description": "min=0.014, mean=0.529, max=0.991, sum=28.55 (54)", - "tab": "Accuracy", - "CivilComments - ECE (10-bin)": "{\"description\": \"min=0.228, mean=0.408, max=0.593, sum=22.008 (54)\", \"tab\": \"Calibration\", \"score\": \"0.4075612338805137\"}", - "CivilComments - EM (Robustness)": "{\"description\": \"min=0.014, mean=0.417, max=0.938, sum=22.51 (54)\", \"tab\": \"Robustness\", \"score\": \"0.41686056018907397\"}", - "CivilComments - EM (Fairness)": "{\"description\": \"min=0.014, mean=0.482, max=0.962, sum=26.023 (54)\", \"tab\": \"Fairness\", \"score\": \"0.4819034071645267\"}", - "CivilComments - Denoised inference time (s)": "{\"description\": \"min=0.418, mean=0.482, max=0.621, sum=26.002 (54)\", \"tab\": \"Efficiency\", \"score\": \"0.48152748003997736\"}", - "CivilComments - # eval": "{\"description\": \"min=74, mean=371.556, max=683, sum=20064 (54)\", \"tab\": \"General information\", \"score\": \"371.55555555555554\"}", - "CivilComments - # train": "{\"description\": \"min=5, mean=5, max=5, sum=270 (54)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CivilComments - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (54)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CivilComments - # prompt tokens": "{\"description\": \"min=271.927, mean=532.602, max=942.498, sum=28760.487 (54)\", \"tab\": \"General information\", \"score\": \"532.6016121330534\"}", - "CivilComments - # output tokens": "{\"description\": \"min=2, mean=2, max=2, sum=108 (54)\", \"tab\": \"General information\", \"score\": \"2.0\"}", - "CivilComments - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=162 (54)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "CivilComments - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (54)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "RAFT", - "source_data": { - "dataset_name": "RAFT", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on RAFT", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.658, - "details": { - "description": "min=0.2, mean=0.658, max=0.975, sum=21.7 (33)", - "tab": "Accuracy", - "RAFT - ECE (10-bin)": "{\"description\": \"min=0.113, mean=0.244, max=0.466, sum=8.048 (33)\", \"tab\": \"Calibration\", \"score\": \"0.24386423436086976\"}", - "RAFT - EM (Robustness)": "{\"description\": \"min=0.025, mean=0.513, max=0.775, sum=16.925 (33)\", \"tab\": \"Robustness\", \"score\": \"0.5128787878787878\"}", - "RAFT - EM (Fairness)": "{\"description\": \"min=0.175, mean=0.636, max=0.975, sum=21 (33)\", \"tab\": \"Fairness\", \"score\": \"0.6363636363636364\"}", - "RAFT - Denoised inference time (s)": "{\"description\": \"min=0.401, mean=0.59, max=0.888, sum=19.483 (33)\", \"tab\": \"Efficiency\", \"score\": \"0.5903971827651516\"}", - "RAFT - # eval": "{\"description\": \"min=40, mean=40, max=40, sum=1320 (33)\", \"tab\": \"General information\", \"score\": \"40.0\"}", - "RAFT - # train": "{\"description\": \"min=0.95, mean=4.658, max=5, sum=153.7 (33)\", \"tab\": \"General information\", \"score\": \"4.657575757575757\"}", - "RAFT - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (33)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "RAFT - # prompt tokens": "{\"description\": \"min=212.25, mean=712.248, max=1745.25, sum=23504.175 (33)\", \"tab\": \"General information\", \"score\": \"712.2477272727273\"}", - "RAFT - # output tokens": "{\"description\": \"min=1.95, mean=3.59, max=6.575, sum=118.475 (33)\", \"tab\": \"General information\", \"score\": \"3.590151515151515\"}", - "RAFT - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=99 (33)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "RAFT - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (33)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/ai21_j1-grande-v2-beta-17b.json b/data/models/ai21_j1-grande-v2-beta-17b.json deleted file mode 100644 index 83dfd63633dbe40caa777394d058599f85246939..0000000000000000000000000000000000000000 --- a/data/models/ai21_j1-grande-v2-beta-17b.json +++ /dev/null @@ -1,674 +0,0 @@ -{ - "model_info": { - "name": "J1-Grande v2 beta 17B", - "id": "ai21/J1-Grande-v2-beta-17B", - "developer": "ai21", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_classic/ai21_J1-Grande-v2-beta-17B/1774096308.339228", - "retrieved_timestamp": "1774096308.339228", - "source_metadata": { - "source_name": "helm_classic", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_classic", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_classic", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperform on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.706, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Calibration": "{\"description\": \"\", \"tab\": \"Calibration\", \"score\": \"0.6340622537431048\"}", - "Mean win rate - Robustness": "{\"description\": \"\", \"tab\": \"Robustness\", \"score\": \"0.7106770870953296\"}", - "Mean win rate - Fairness": "{\"description\": \"\", \"tab\": \"Fairness\", \"score\": \"0.6771299149497148\"}", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}", - "Mean win rate - Bias": "{\"description\": \"\", \"tab\": \"Bias\", \"score\": \"0.5919924787763542\"}", - "Mean win rate - Toxicity": "{\"description\": \"\", \"tab\": \"Toxicity\", \"score\": \"0.5063399563399563\"}", - "Mean win rate - Summarization metrics": "{\"description\": \"\", \"tab\": \"Summarization metrics\", \"score\": \"0.6776315789473685\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.445, - "details": { - "description": "min=0.23, mean=0.445, max=0.8, sum=6.677 (15)", - "tab": "Accuracy", - "MMLU - ECE (10-bin)": "{\"description\": \"min=0.067, mean=0.139, max=0.205, sum=2.09 (15)\", \"tab\": \"Calibration\", \"score\": \"0.13930239849591303\"}", - "MMLU - EM (Robustness)": "{\"description\": \"min=0.2, mean=0.392, max=0.73, sum=5.887 (15)\", \"tab\": \"Robustness\", \"score\": \"0.39245614035087717\"}", - "MMLU - EM (Fairness)": "{\"description\": \"min=0.19, mean=0.409, max=0.77, sum=6.142 (15)\", \"tab\": \"Fairness\", \"score\": \"0.4094619883040936\"}", - "MMLU - Denoised inference time (s)": "{\"description\": \"5 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=1542 (15)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=75 (15)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (15)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=308.59, mean=396.74, max=552.719, sum=5951.098 (15)\", \"tab\": \"General information\", \"score\": \"396.73985964912276\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=15 (15)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "MMLU - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=45 (15)\", \"tab\": \"General information\", \"score\": \"3.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "BoolQ", - "source_data": { - "dataset_name": "BoolQ", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on BoolQ", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.812, - "details": { - "description": "min=0.799, mean=0.812, max=0.823, sum=2.437 (3)", - "tab": "Accuracy", - "BoolQ - ECE (10-bin)": "{\"description\": \"min=0.155, mean=0.167, max=0.185, sum=0.5 (3)\", \"tab\": \"Calibration\", \"score\": \"0.16655399552246586\"}", - "BoolQ - EM (Robustness)": "{\"description\": \"min=0.669, mean=0.692, max=0.714, sum=2.077 (3)\", \"tab\": \"Robustness\", \"score\": \"0.6923333333333334\"}", - "BoolQ - EM (Fairness)": "{\"description\": \"min=0.751, mean=0.764, max=0.784, sum=2.291 (3)\", \"tab\": \"Fairness\", \"score\": \"0.7636666666666668\"}", - "BoolQ - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "BoolQ - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "BoolQ - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "BoolQ - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "BoolQ - # prompt tokens": "{\"description\": \"min=506.985, mean=694.652, max=952.985, sum=2083.955 (3)\", \"tab\": \"General information\", \"score\": \"694.6516666666666\"}", - "BoolQ - # output tokens": "{\"description\": \"min=2, mean=2, max=2, sum=6 (3)\", \"tab\": \"General information\", \"score\": \"2.0\"}", - "BoolQ - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "BoolQ - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.725, - "details": { - "description": "min=0.712, mean=0.725, max=0.736, sum=2.176 (3)", - "tab": "Accuracy", - "NarrativeQA - ECE (10-bin)": "{\"description\": \"min=0.034, mean=0.041, max=0.05, sum=0.122 (3)\", \"tab\": \"Calibration\", \"score\": \"0.040831012535009516\"}", - "NarrativeQA - F1 (Robustness)": "{\"description\": \"min=0.484, mean=0.565, max=0.616, sum=1.694 (3)\", \"tab\": \"Robustness\", \"score\": \"0.5646966401263148\"}", - "NarrativeQA - F1 (Fairness)": "{\"description\": \"min=0.622, mean=0.647, max=0.665, sum=1.941 (3)\", \"tab\": \"Fairness\", \"score\": \"0.6470593497686433\"}", - "NarrativeQA - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=1065 (3)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=2.166, mean=2.639, max=3.225, sum=7.918 (3)\", \"tab\": \"General information\", \"score\": \"2.63943661971831\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=1598.614, mean=1692.218, max=1777.299, sum=5076.654 (3)\", \"tab\": \"General information\", \"score\": \"1692.2178403755868\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=4.194, mean=4.6, max=5.011, sum=13.8 (3)\", \"tab\": \"General information\", \"score\": \"4.6\"}", - "NarrativeQA - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NarrativeQA - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NarrativeQA - Stereotypes (gender)": "{\"description\": \"min=0.25, mean=0.3, max=0.4, sum=0.9 (3)\", \"tab\": \"Bias\", \"score\": \"0.3\"}", - "NarrativeQA - Representation (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=0.667 (1)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666667\"}", - "NarrativeQA - Representation (gender)": "{\"description\": \"min=0.156, mean=0.179, max=0.205, sum=0.536 (3)\", \"tab\": \"Bias\", \"score\": \"0.1787801116945903\"}", - "NarrativeQA - Toxic fraction": "{\"description\": \"min=0.008, mean=0.014, max=0.017, sum=0.042 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.014084507042253521\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (open-book)", - "source_data": { - "dataset_name": "NaturalQuestions (open-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (open-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.625, - "details": { - "description": "min=0.622, mean=0.625, max=0.628, sum=1.874 (3)", - "tab": "Accuracy", - "NaturalQuestions (closed-book) - ECE (10-bin)": "{\"description\": \"min=0.031, mean=0.036, max=0.043, sum=0.107 (3)\", \"tab\": \"Calibration\", \"score\": \"0.035782131071618734\"}", - "NaturalQuestions (open-book) - ECE (10-bin)": "{\"description\": \"min=0.051, mean=0.065, max=0.075, sum=0.196 (3)\", \"tab\": \"Calibration\", \"score\": \"0.06520649617008285\"}", - "NaturalQuestions (closed-book) - F1 (Robustness)": "{\"description\": \"min=0.23, mean=0.235, max=0.241, sum=0.705 (3)\", \"tab\": \"Robustness\", \"score\": \"0.2349124459413927\"}", - "NaturalQuestions (open-book) - F1 (Robustness)": "{\"description\": \"min=0.556, mean=0.56, max=0.568, sum=1.681 (3)\", \"tab\": \"Robustness\", \"score\": \"0.5603824984507094\"}", - "NaturalQuestions (closed-book) - F1 (Fairness)": "{\"description\": \"min=0.269, mean=0.27, max=0.27, sum=0.81 (3)\", \"tab\": \"Fairness\", \"score\": \"0.269872960171523\"}", - "NaturalQuestions (open-book) - F1 (Fairness)": "{\"description\": \"min=0.562, mean=0.571, max=0.578, sum=1.714 (3)\", \"tab\": \"Fairness\", \"score\": \"0.5712438797598854\"}", - "NaturalQuestions (closed-book) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NaturalQuestions (open-book) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=94.377, mean=99.377, max=102.377, sum=298.131 (3)\", \"tab\": \"General information\", \"score\": \"99.377\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=4.471, mean=5.282, max=6.145, sum=15.846 (3)\", \"tab\": \"General information\", \"score\": \"5.282\"}", - "NaturalQuestions (closed-book) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.568, mean=4.666, max=4.734, sum=13.999 (3)\", \"tab\": \"General information\", \"score\": \"4.666333333333333\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.038, mean=0.038, max=0.038, sum=0.114 (3)\", \"tab\": \"General information\", \"score\": \"0.038\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1136.933, mean=1418.457, max=1595.508, sum=4255.37 (3)\", \"tab\": \"General information\", \"score\": \"1418.4566666666667\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=5.132, mean=5.27, max=5.521, sum=15.809 (3)\", \"tab\": \"General information\", \"score\": \"5.269666666666667\"}", - "NaturalQuestions (open-book) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NaturalQuestions (closed-book) - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=1.333 (2)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666667\"}", - "NaturalQuestions (closed-book) - Stereotypes (gender)": "{\"description\": \"min=0.5, mean=0.5, max=0.5, sum=1.5 (3)\", \"tab\": \"Bias\", \"score\": \"0.5\"}", - "NaturalQuestions (closed-book) - Representation (race)": "{\"description\": \"min=0.288, mean=0.392, max=0.491, sum=1.177 (3)\", \"tab\": \"Bias\", \"score\": \"0.3923268084547134\"}", - "NaturalQuestions (closed-book) - Representation (gender)": "{\"description\": \"min=0.026, mean=0.174, max=0.318, sum=0.522 (3)\", \"tab\": \"Bias\", \"score\": \"0.17397232083140401\"}", - "NaturalQuestions (open-book) - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=0.667 (1)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666667\"}", - "NaturalQuestions (open-book) - Stereotypes (gender)": "{\"description\": \"min=0.1, mean=0.167, max=0.3, sum=0.5 (3)\", \"tab\": \"Bias\", \"score\": \"0.16666666666666666\"}", - "NaturalQuestions (open-book) - Representation (race)": "{\"description\": \"min=0.478, mean=0.488, max=0.498, sum=1.465 (3)\", \"tab\": \"Bias\", \"score\": \"0.48822694742885336\"}", - "NaturalQuestions (open-book) - Representation (gender)": "{\"description\": \"min=0.369, mean=0.381, max=0.394, sum=1.143 (3)\", \"tab\": \"Bias\", \"score\": \"0.38112988257848074\"}", - "NaturalQuestions (closed-book) - Toxic fraction": "{\"description\": \"min=0.001, mean=0.001, max=0.002, sum=0.004 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.0013333333333333333\"}", - "NaturalQuestions (open-book) - Toxic fraction": "{\"description\": \"min=0.001, mean=0.001, max=0.001, sum=0.003 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.001\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "QuAC", - "source_data": { - "dataset_name": "QuAC", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on QuAC", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.392, - "details": { - "description": "min=0.375, mean=0.392, max=0.411, sum=1.177 (3)", - "tab": "Accuracy", - "QuAC - ECE (10-bin)": "{\"description\": \"min=0.031, mean=0.04, max=0.051, sum=0.121 (3)\", \"tab\": \"Calibration\", \"score\": \"0.04046561186462396\"}", - "QuAC - F1 (Robustness)": "{\"description\": \"min=0.232, mean=0.251, max=0.261, sum=0.752 (3)\", \"tab\": \"Robustness\", \"score\": \"0.2506588392587418\"}", - "QuAC - F1 (Fairness)": "{\"description\": \"min=0.297, mean=0.308, max=0.319, sum=0.923 (3)\", \"tab\": \"Fairness\", \"score\": \"0.30759220119907554\"}", - "QuAC - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "QuAC - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "QuAC - # train": "{\"description\": \"min=1.788, mean=1.829, max=1.88, sum=5.486 (3)\", \"tab\": \"General information\", \"score\": \"1.8286666666666667\"}", - "QuAC - truncated": "{\"description\": \"min=0.001, mean=0.001, max=0.001, sum=0.003 (3)\", \"tab\": \"General information\", \"score\": \"0.001\"}", - "QuAC - # prompt tokens": "{\"description\": \"min=1645.856, mean=1698.711, max=1730.814, sum=5096.134 (3)\", \"tab\": \"General information\", \"score\": \"1698.7113333333334\"}", - "QuAC - # output tokens": "{\"description\": \"min=19.318, mean=23.053, max=25.3, sum=69.158 (3)\", \"tab\": \"General information\", \"score\": \"23.052666666666667\"}", - "QuAC - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "QuAC - Stereotypes (race)": "{\"description\": \"min=0.583, mean=0.628, max=0.66, sum=1.884 (3)\", \"tab\": \"Bias\", \"score\": \"0.6279609279609281\"}", - "QuAC - Stereotypes (gender)": "{\"description\": \"min=0.396, mean=0.411, max=0.426, sum=1.232 (3)\", \"tab\": \"Bias\", \"score\": \"0.41081218336807646\"}", - "QuAC - Representation (race)": "{\"description\": \"min=0.302, mean=0.327, max=0.359, sum=0.981 (3)\", \"tab\": \"Bias\", \"score\": \"0.3270316371542728\"}", - "QuAC - Representation (gender)": "{\"description\": \"min=0.198, mean=0.225, max=0.241, sum=0.676 (3)\", \"tab\": \"Bias\", \"score\": \"0.22518777152451866\"}", - "QuAC - Toxic fraction": "{\"description\": \"min=0.003, mean=0.003, max=0.004, sum=0.01 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.0033333333333333335\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "HellaSwag", - "source_data": { - "dataset_name": "HellaSwag", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on HellaSwag", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.764, - "details": { - "description": "min=0.764, mean=0.764, max=0.764, sum=0.764 (1)", - "tab": "Accuracy", - "HellaSwag - ECE (10-bin)": "{\"description\": \"min=0.226, mean=0.226, max=0.226, sum=0.226 (1)\", \"tab\": \"Calibration\", \"score\": \"0.2263163700416937\"}", - "HellaSwag - EM (Robustness)": "{\"description\": \"min=0.732, mean=0.732, max=0.732, sum=0.732 (1)\", \"tab\": \"Robustness\", \"score\": \"0.732\"}", - "HellaSwag - EM (Fairness)": "{\"description\": \"min=0.623, mean=0.623, max=0.623, sum=0.623 (1)\", \"tab\": \"Fairness\", \"score\": \"0.623\"}", - "HellaSwag - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "HellaSwag - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "HellaSwag - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "HellaSwag - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "HellaSwag - # prompt tokens": "{\"description\": \"min=62.466, mean=62.466, max=62.466, sum=62.466 (1)\", \"tab\": \"General information\", \"score\": \"62.466\"}", - "HellaSwag - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "HellaSwag - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.56, - "details": { - "description": "min=0.56, mean=0.56, max=0.56, sum=0.56 (1)", - "tab": "Accuracy", - "OpenbookQA - ECE (10-bin)": "{\"description\": \"min=0.215, mean=0.215, max=0.215, sum=0.215 (1)\", \"tab\": \"Calibration\", \"score\": \"0.21479287621696264\"}", - "OpenbookQA - EM (Robustness)": "{\"description\": \"min=0.474, mean=0.474, max=0.474, sum=0.474 (1)\", \"tab\": \"Robustness\", \"score\": \"0.474\"}", - "OpenbookQA - EM (Fairness)": "{\"description\": \"min=0.478, mean=0.478, max=0.478, sum=0.478 (1)\", \"tab\": \"Fairness\", \"score\": \"0.478\"}", - "OpenbookQA - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=4.348, mean=4.348, max=4.348, sum=4.348 (1)\", \"tab\": \"General information\", \"score\": \"4.348\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "TruthfulQA", - "source_data": { - "dataset_name": "TruthfulQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on TruthfulQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.306, - "details": { - "description": "min=0.266, mean=0.306, max=0.333, sum=0.917 (3)", - "tab": "Accuracy", - "TruthfulQA - ECE (10-bin)": "{\"description\": \"min=0.101, mean=0.123, max=0.157, sum=0.37 (3)\", \"tab\": \"Calibration\", \"score\": \"0.1233746034244333\"}", - "TruthfulQA - EM (Robustness)": "{\"description\": \"min=0.216, mean=0.252, max=0.294, sum=0.755 (3)\", \"tab\": \"Robustness\", \"score\": \"0.25178389398572887\"}", - "TruthfulQA - EM (Fairness)": "{\"description\": \"min=0.216, mean=0.242, max=0.271, sum=0.725 (3)\", \"tab\": \"Fairness\", \"score\": \"0.24159021406727832\"}", - "TruthfulQA - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "TruthfulQA - # eval": "{\"description\": \"min=654, mean=654, max=654, sum=1962 (3)\", \"tab\": \"General information\", \"score\": \"654.0\"}", - "TruthfulQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "TruthfulQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "TruthfulQA - # prompt tokens": "{\"description\": \"min=317.682, mean=355.015, max=375.682, sum=1065.046 (3)\", \"tab\": \"General information\", \"score\": \"355.0152905198777\"}", - "TruthfulQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=3 (3)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "TruthfulQA - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MS MARCO (TREC)", - "source_data": { - "dataset_name": "MS MARCO (TREC)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "NDCG@10 on MS MARCO (TREC)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.46, - "details": { - "description": "min=0.401, mean=0.46, max=0.51, sum=1.38 (3)", - "tab": "Accuracy", - "MS MARCO (regular) - RR@10 (Robustness)": "{\"description\": \"min=0.207, mean=0.222, max=0.244, sum=0.666 (3)\", \"tab\": \"Robustness\", \"score\": \"0.22205343915343892\"}", - "MS MARCO (TREC) - NDCG@10 (Robustness)": "{\"description\": \"min=0.361, mean=0.407, max=0.448, sum=1.222 (3)\", \"tab\": \"Robustness\", \"score\": \"0.40738421631598776\"}", - "MS MARCO (regular) - RR@10 (Fairness)": "{\"description\": \"min=0.23, mean=0.253, max=0.284, sum=0.76 (3)\", \"tab\": \"Fairness\", \"score\": \"0.25326719576719553\"}", - "MS MARCO (TREC) - NDCG@10 (Fairness)": "{\"description\": \"min=0.371, mean=0.435, max=0.486, sum=1.304 (3)\", \"tab\": \"Fairness\", \"score\": \"0.4346805929346467\"}", - "MS MARCO (regular) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MS MARCO (TREC) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MS MARCO (regular) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "MS MARCO (regular) - # train": "{\"description\": \"min=2, mean=2, max=2, sum=6 (3)\", \"tab\": \"General information\", \"score\": \"2.0\"}", - "MS MARCO (regular) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MS MARCO (regular) - # prompt tokens": "{\"description\": \"min=349.303, mean=385.636, max=423.303, sum=1156.909 (3)\", \"tab\": \"General information\", \"score\": \"385.63633333333337\"}", - "MS MARCO (regular) - # output tokens": "{\"description\": \"min=2.001, mean=2.009, max=2.02, sum=6.026 (3)\", \"tab\": \"General information\", \"score\": \"2.0086666666666666\"}", - "MS MARCO (regular) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "MS MARCO (TREC) - # eval": "{\"description\": \"min=43, mean=43, max=43, sum=129 (3)\", \"tab\": \"General information\", \"score\": \"43.0\"}", - "MS MARCO (TREC) - # train": "{\"description\": \"min=2, mean=2, max=2, sum=6 (3)\", \"tab\": \"General information\", \"score\": \"2.0\"}", - "MS MARCO (TREC) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MS MARCO (TREC) - # prompt tokens": "{\"description\": \"min=337.047, mean=373.38, max=411.047, sum=1120.14 (3)\", \"tab\": \"General information\", \"score\": \"373.3798449612403\"}", - "MS MARCO (TREC) - # output tokens": "{\"description\": \"min=2.023, mean=2.023, max=2.023, sum=6.07 (3)\", \"tab\": \"General information\", \"score\": \"2.0232558139534884\"}", - "MS MARCO (TREC) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "MS MARCO (regular) - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "MS MARCO (TREC) - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CNN/DailyMail", - "source_data": { - "dataset_name": "CNN/DailyMail", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on CNN/DailyMail", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.146, - "details": { - "description": "min=0.14, mean=0.146, max=0.152, sum=0.875 (6)", - "tab": "Accuracy", - "CNN/DailyMail - Denoised inference time (s)": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "CNN/DailyMail - # eval": "{\"description\": \"min=466, mean=466, max=466, sum=2796 (6)\", \"tab\": \"General information\", \"score\": \"466.0\"}", - "CNN/DailyMail - # train": "{\"description\": \"min=5, mean=5, max=5, sum=30 (6)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CNN/DailyMail - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (6)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CNN/DailyMail - # prompt tokens": "{\"description\": \"min=1203.032, mean=1213.032, max=1224.032, sum=7278.193 (6)\", \"tab\": \"General information\", \"score\": \"1213.0321888412018\"}", - "CNN/DailyMail - # output tokens": "{\"description\": \"min=48.575, mean=53.215, max=56.485, sum=319.288 (6)\", \"tab\": \"General information\", \"score\": \"53.21459227467812\"}", - "CNN/DailyMail - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=18 (6)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "CNN/DailyMail - Stereotypes (race)": "{\"description\": \"min=0.605, mean=0.615, max=0.633, sum=3.691 (6)\", \"tab\": \"Bias\", \"score\": \"0.615138154027043\"}", - "CNN/DailyMail - Stereotypes (gender)": "{\"description\": \"min=0.39, mean=0.401, max=0.416, sum=2.409 (6)\", \"tab\": \"Bias\", \"score\": \"0.4014349780782224\"}", - "CNN/DailyMail - Representation (race)": "{\"description\": \"min=0.278, mean=0.293, max=0.321, sum=1.76 (6)\", \"tab\": \"Bias\", \"score\": \"0.2933799533799534\"}", - "CNN/DailyMail - Representation (gender)": "{\"description\": \"min=0.077, mean=0.099, max=0.123, sum=0.596 (6)\", \"tab\": \"Bias\", \"score\": \"0.09929925405618005\"}", - "CNN/DailyMail - Toxic fraction": "{\"description\": \"min=0.002, mean=0.004, max=0.006, sum=0.026 (6)\", \"tab\": \"Toxicity\", \"score\": \"0.004291845493562232\"}", - "CNN/DailyMail - SummaC": "{\"description\": \"min=0.533, mean=0.552, max=0.585, sum=1.655 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.5516800688123055\"}", - "CNN/DailyMail - QAFactEval": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - BERTScore (F1)": "{\"description\": \"min=0.273, mean=0.29, max=0.308, sum=0.871 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.2904019284209938\"}", - "CNN/DailyMail - Coverage": "{\"description\": \"min=0.965, mean=0.973, max=0.983, sum=5.838 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"0.9729724626233943\"}", - "CNN/DailyMail - Density": "{\"description\": \"min=18.643, mean=24.032, max=31.138, sum=144.19 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"24.0317341420422\"}", - "CNN/DailyMail - Compression": "{\"description\": \"min=10.389, mean=11.659, max=13.368, sum=69.956 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"11.65941362001026\"}", - "CNN/DailyMail - HumanEval-faithfulness": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-relevance": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-coherence": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "XSUM", - "source_data": { - "dataset_name": "XSUM", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on XSUM", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.152, - "details": { - "description": "min=0.149, mean=0.152, max=0.157, sum=0.911 (6)", - "tab": "Accuracy", - "XSUM - Denoised inference time (s)": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "XSUM - # eval": "{\"description\": \"min=518, mean=518, max=518, sum=3108 (6)\", \"tab\": \"General information\", \"score\": \"518.0\"}", - "XSUM - # train": "{\"description\": \"min=5, mean=5, max=5, sum=30 (6)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "XSUM - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (6)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "XSUM - # prompt tokens": "{\"description\": \"min=1099.388, mean=1133.388, max=1172.388, sum=6800.328 (6)\", \"tab\": \"General information\", \"score\": \"1133.388030888031\"}", - "XSUM - # output tokens": "{\"description\": \"min=21.805, mean=22.092, max=22.577, sum=132.552 (6)\", \"tab\": \"General information\", \"score\": \"22.09202059202059\"}", - "XSUM - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=18 (6)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "XSUM - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=4 (6)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "XSUM - Stereotypes (gender)": "{\"description\": \"min=0.45, mean=0.465, max=0.474, sum=2.791 (6)\", \"tab\": \"Bias\", \"score\": \"0.46523352396514167\"}", - "XSUM - Representation (race)": "{\"description\": \"min=0.494, mean=0.522, max=0.536, sum=3.133 (6)\", \"tab\": \"Bias\", \"score\": \"0.5222388805597201\"}", - "XSUM - Representation (gender)": "{\"description\": \"min=0.201, mean=0.214, max=0.224, sum=1.284 (6)\", \"tab\": \"Bias\", \"score\": \"0.21406383130768433\"}", - "XSUM - Toxic fraction": "{\"description\": \"min=0, mean=0.001, max=0.002, sum=0.008 (6)\", \"tab\": \"Toxicity\", \"score\": \"0.001287001287001287\"}", - "XSUM - SummaC": "{\"description\": \"min=-0.298, mean=-0.282, max=-0.27, sum=-0.845 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"-0.2817185772994412\"}", - "XSUM - QAFactEval": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - BERTScore (F1)": "{\"description\": \"min=0.45, mean=0.454, max=0.458, sum=1.362 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.4538733417652499\"}", - "XSUM - Coverage": "{\"description\": \"min=0.782, mean=0.786, max=0.79, sum=4.714 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"0.7856975370843048\"}", - "XSUM - Density": "{\"description\": \"min=2.624, mean=2.816, max=3.113, sum=16.895 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"2.815909720295231\"}", - "XSUM - Compression": "{\"description\": \"min=16.323, mean=16.857, max=17.149, sum=101.14 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"16.856596376166145\"}", - "XSUM - HumanEval-faithfulness": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-relevance": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-coherence": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "IMDB", - "source_data": { - "dataset_name": "IMDB", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on IMDB", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.957, - "details": { - "description": "min=0.947, mean=0.957, max=0.964, sum=2.872 (3)", - "tab": "Accuracy", - "IMDB - ECE (10-bin)": "{\"description\": \"min=0.119, mean=0.136, max=0.165, sum=0.407 (3)\", \"tab\": \"Calibration\", \"score\": \"0.13573735378803647\"}", - "IMDB - EM (Robustness)": "{\"description\": \"min=0.931, mean=0.947, max=0.955, sum=2.841 (3)\", \"tab\": \"Robustness\", \"score\": \"0.9470000000000001\"}", - "IMDB - EM (Fairness)": "{\"description\": \"min=0.935, mean=0.95, max=0.959, sum=2.851 (3)\", \"tab\": \"Fairness\", \"score\": \"0.9503333333333334\"}", - "IMDB - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "IMDB - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "IMDB - # train": "{\"description\": \"min=4.915, mean=4.972, max=5, sum=14.915 (3)\", \"tab\": \"General information\", \"score\": \"4.971666666666667\"}", - "IMDB - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IMDB - # prompt tokens": "{\"description\": \"min=853.851, mean=1281.577, max=1725.03, sum=3844.732 (3)\", \"tab\": \"General information\", \"score\": \"1281.5773333333334\"}", - "IMDB - # output tokens": "{\"description\": \"min=2, mean=2, max=2, sum=6 (3)\", \"tab\": \"General information\", \"score\": \"2.0\"}", - "IMDB - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "IMDB - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CivilComments", - "source_data": { - "dataset_name": "CivilComments", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on CivilComments", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.546, - "details": { - "description": "min=0.008, mean=0.546, max=1, sum=29.501 (54)", - "tab": "Accuracy", - "CivilComments - ECE (10-bin)": "{\"description\": \"min=0.131, mean=0.376, max=0.649, sum=20.307 (54)\", \"tab\": \"Calibration\", \"score\": \"0.37604932471578795\"}", - "CivilComments - EM (Robustness)": "{\"description\": \"min=0, mean=0.495, max=0.995, sum=26.738 (54)\", \"tab\": \"Robustness\", \"score\": \"0.49514299676627055\"}", - "CivilComments - EM (Fairness)": "{\"description\": \"min=0.005, mean=0.404, max=0.901, sum=21.814 (54)\", \"tab\": \"Fairness\", \"score\": \"0.40396201739558046\"}", - "CivilComments - Denoised inference time (s)": "{\"description\": \"9 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "CivilComments - # eval": "{\"description\": \"min=74, mean=371.556, max=683, sum=20064 (54)\", \"tab\": \"General information\", \"score\": \"371.55555555555554\"}", - "CivilComments - # train": "{\"description\": \"min=5, mean=5, max=5, sum=270 (54)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CivilComments - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (54)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CivilComments - # prompt tokens": "{\"description\": \"min=271.927, mean=532.602, max=942.498, sum=28760.487 (54)\", \"tab\": \"General information\", \"score\": \"532.6016121330534\"}", - "CivilComments - # output tokens": "{\"description\": \"min=2, mean=2, max=2, sum=108 (54)\", \"tab\": \"General information\", \"score\": \"2.0\"}", - "CivilComments - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=162 (54)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "CivilComments - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (54)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "RAFT", - "source_data": { - "dataset_name": "RAFT", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on RAFT", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.679, - "details": { - "description": "min=0.225, mean=0.679, max=0.95, sum=22.4 (33)", - "tab": "Accuracy", - "RAFT - ECE (10-bin)": "{\"description\": \"min=0.095, mean=0.234, max=0.473, sum=7.733 (33)\", \"tab\": \"Calibration\", \"score\": \"0.23434348116913628\"}", - "RAFT - EM (Robustness)": "{\"description\": \"min=0.025, mean=0.555, max=0.925, sum=18.3 (33)\", \"tab\": \"Robustness\", \"score\": \"0.5545454545454547\"}", - "RAFT - EM (Fairness)": "{\"description\": \"min=0.2, mean=0.637, max=0.95, sum=21.025 (33)\", \"tab\": \"Fairness\", \"score\": \"0.6371212121212121\"}", - "RAFT - Denoised inference time (s)": "{\"description\": \"11 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "RAFT - # eval": "{\"description\": \"min=40, mean=40, max=40, sum=1320 (33)\", \"tab\": \"General information\", \"score\": \"40.0\"}", - "RAFT - # train": "{\"description\": \"min=0.95, mean=4.658, max=5, sum=153.7 (33)\", \"tab\": \"General information\", \"score\": \"4.657575757575757\"}", - "RAFT - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (33)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "RAFT - # prompt tokens": "{\"description\": \"min=212.25, mean=712.248, max=1745.25, sum=23504.175 (33)\", \"tab\": \"General information\", \"score\": \"712.2477272727273\"}", - "RAFT - # output tokens": "{\"description\": \"min=1.95, mean=3.574, max=6.575, sum=117.95 (33)\", \"tab\": \"General information\", \"score\": \"3.5742424242424238\"}", - "RAFT - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=99 (33)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "RAFT - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (33)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/ai21_j1-jumbo-v1-178b.json b/data/models/ai21_j1-jumbo-v1-178b.json deleted file mode 100644 index 3c72232276e8dcc401c4b2a27844811fff93ab63..0000000000000000000000000000000000000000 --- a/data/models/ai21_j1-jumbo-v1-178b.json +++ /dev/null @@ -1,674 +0,0 @@ -{ - "model_info": { - "name": "J1-Jumbo v1 178B", - "id": "ai21/J1-Jumbo-v1-178B", - "developer": "ai21", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_classic/ai21_J1-Jumbo-v1-178B/1774096308.339228", - "retrieved_timestamp": "1774096308.339228", - "source_metadata": { - "source_name": "helm_classic", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_classic", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_classic", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperform on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.517, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Calibration": "{\"description\": \"\", \"tab\": \"Calibration\", \"score\": \"0.6662512419912975\"}", - "Mean win rate - Robustness": "{\"description\": \"\", \"tab\": \"Robustness\", \"score\": \"0.4518627645991383\"}", - "Mean win rate - Fairness": "{\"description\": \"\", \"tab\": \"Fairness\", \"score\": \"0.48803949109844547\"}", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.2218311403508772\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}", - "Mean win rate - Bias": "{\"description\": \"\", \"tab\": \"Bias\", \"score\": \"0.5485082680240319\"}", - "Mean win rate - Toxicity": "{\"description\": \"\", \"tab\": \"Toxicity\", \"score\": \"0.6042735042735042\"}", - "Mean win rate - Summarization metrics": "{\"description\": \"\", \"tab\": \"Summarization metrics\", \"score\": \"0.5867794486215538\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.259, - "details": { - "description": "min=0.19, mean=0.259, max=0.35, sum=3.891 (15)", - "tab": "Accuracy", - "MMLU - ECE (10-bin)": "{\"description\": \"min=0.074, mean=0.131, max=0.172, sum=1.96 (15)\", \"tab\": \"Calibration\", \"score\": \"0.13067986008352367\"}", - "MMLU - EM (Robustness)": "{\"description\": \"min=0.15, mean=0.221, max=0.31, sum=3.313 (15)\", \"tab\": \"Robustness\", \"score\": \"0.22085380116959066\"}", - "MMLU - EM (Fairness)": "{\"description\": \"min=0.17, mean=0.236, max=0.33, sum=3.545 (15)\", \"tab\": \"Fairness\", \"score\": \"0.23635087719298245\"}", - "MMLU - Denoised inference time (s)": "{\"description\": \"min=0.419, mean=0.457, max=0.511, sum=6.851 (15)\", \"tab\": \"Efficiency\", \"score\": \"0.4567342927631581\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=1542 (15)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=75 (15)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (15)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=308.59, mean=396.74, max=552.719, sum=5951.098 (15)\", \"tab\": \"General information\", \"score\": \"396.73985964912276\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=15 (15)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "MMLU - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=45 (15)\", \"tab\": \"General information\", \"score\": \"3.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "BoolQ", - "source_data": { - "dataset_name": "BoolQ", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on BoolQ", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.776, - "details": { - "description": "min=0.766, mean=0.776, max=0.786, sum=2.327 (3)", - "tab": "Accuracy", - "BoolQ - ECE (10-bin)": "{\"description\": \"min=0.205, mean=0.215, max=0.223, sum=0.646 (3)\", \"tab\": \"Calibration\", \"score\": \"0.21546167732589497\"}", - "BoolQ - EM (Robustness)": "{\"description\": \"min=0.635, mean=0.65, max=0.659, sum=1.949 (3)\", \"tab\": \"Robustness\", \"score\": \"0.6496666666666667\"}", - "BoolQ - EM (Fairness)": "{\"description\": \"min=0.693, mean=0.709, max=0.73, sum=2.128 (3)\", \"tab\": \"Fairness\", \"score\": \"0.7093333333333334\"}", - "BoolQ - Denoised inference time (s)": "{\"description\": \"min=0.55, mean=0.62, max=0.727, sum=1.859 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.6195252891710069\"}", - "BoolQ - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "BoolQ - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "BoolQ - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "BoolQ - # prompt tokens": "{\"description\": \"min=506.985, mean=694.652, max=952.985, sum=2083.955 (3)\", \"tab\": \"General information\", \"score\": \"694.6516666666666\"}", - "BoolQ - # output tokens": "{\"description\": \"min=2, mean=2, max=2, sum=6 (3)\", \"tab\": \"General information\", \"score\": \"2.0\"}", - "BoolQ - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "BoolQ - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.695, - "details": { - "description": "min=0.689, mean=0.695, max=0.698, sum=2.085 (3)", - "tab": "Accuracy", - "NarrativeQA - ECE (10-bin)": "{\"description\": \"min=0.028, mean=0.034, max=0.042, sum=0.101 (3)\", \"tab\": \"Calibration\", \"score\": \"0.033635629206676086\"}", - "NarrativeQA - F1 (Robustness)": "{\"description\": \"min=0.448, mean=0.523, max=0.573, sum=1.57 (3)\", \"tab\": \"Robustness\", \"score\": \"0.5232968431666949\"}", - "NarrativeQA - F1 (Fairness)": "{\"description\": \"min=0.566, mean=0.581, max=0.592, sum=1.743 (3)\", \"tab\": \"Fairness\", \"score\": \"0.5811269391716133\"}", - "NarrativeQA - Denoised inference time (s)": "{\"description\": \"min=1.085, mean=1.126, max=1.167, sum=3.379 (3)\", \"tab\": \"Efficiency\", \"score\": \"1.1261881626564945\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=1065 (3)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=2.166, mean=2.639, max=3.225, sum=7.918 (3)\", \"tab\": \"General information\", \"score\": \"2.63943661971831\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=1598.614, mean=1692.218, max=1777.299, sum=5076.654 (3)\", \"tab\": \"General information\", \"score\": \"1692.2178403755868\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=4.434, mean=4.514, max=4.617, sum=13.541 (3)\", \"tab\": \"General information\", \"score\": \"4.513615023474178\"}", - "NarrativeQA - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NarrativeQA - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NarrativeQA - Stereotypes (gender)": "{\"description\": \"min=0.375, mean=0.438, max=0.5, sum=0.875 (2)\", \"tab\": \"Bias\", \"score\": \"0.4375\"}", - "NarrativeQA - Representation (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=1.333 (2)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666667\"}", - "NarrativeQA - Representation (gender)": "{\"description\": \"min=0.196, mean=0.214, max=0.225, sum=0.641 (3)\", \"tab\": \"Bias\", \"score\": \"0.21357560568086884\"}", - "NarrativeQA - Toxic fraction": "{\"description\": \"min=0.014, mean=0.014, max=0.014, sum=0.042 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.014084507042253521\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (open-book)", - "source_data": { - "dataset_name": "NaturalQuestions (open-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (open-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.595, - "details": { - "description": "min=0.593, mean=0.595, max=0.598, sum=1.786 (3)", - "tab": "Accuracy", - "NaturalQuestions (closed-book) - ECE (10-bin)": "{\"description\": \"min=0.029, mean=0.035, max=0.042, sum=0.106 (3)\", \"tab\": \"Calibration\", \"score\": \"0.035434924784030764\"}", - "NaturalQuestions (open-book) - ECE (10-bin)": "{\"description\": \"min=0.058, mean=0.065, max=0.069, sum=0.195 (3)\", \"tab\": \"Calibration\", \"score\": \"0.06491976505236641\"}", - "NaturalQuestions (closed-book) - F1 (Robustness)": "{\"description\": \"min=0.177, mean=0.179, max=0.183, sum=0.537 (3)\", \"tab\": \"Robustness\", \"score\": \"0.17889901825749613\"}", - "NaturalQuestions (open-book) - F1 (Robustness)": "{\"description\": \"min=0.487, mean=0.503, max=0.515, sum=1.509 (3)\", \"tab\": \"Robustness\", \"score\": \"0.5031073713472458\"}", - "NaturalQuestions (closed-book) - F1 (Fairness)": "{\"description\": \"min=0.227, mean=0.235, max=0.239, sum=0.704 (3)\", \"tab\": \"Fairness\", \"score\": \"0.23456155611286555\"}", - "NaturalQuestions (open-book) - F1 (Fairness)": "{\"description\": \"min=0.54, mean=0.54, max=0.54, sum=1.62 (3)\", \"tab\": \"Fairness\", \"score\": \"0.5399104355251988\"}", - "NaturalQuestions (closed-book) - Denoised inference time (s)": "{\"description\": \"min=0.466, mean=0.493, max=0.536, sum=1.478 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.492596863281249\"}", - "NaturalQuestions (open-book) - Denoised inference time (s)": "{\"description\": \"min=0.931, mean=1.06, max=1.147, sum=3.179 (3)\", \"tab\": \"Efficiency\", \"score\": \"1.0597537076822923\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=94.377, mean=99.377, max=102.377, sum=298.131 (3)\", \"tab\": \"General information\", \"score\": \"99.377\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=5.012, mean=5.602, max=6.608, sum=16.806 (3)\", \"tab\": \"General information\", \"score\": \"5.602\"}", - "NaturalQuestions (closed-book) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.568, mean=4.666, max=4.734, sum=13.999 (3)\", \"tab\": \"General information\", \"score\": \"4.666333333333333\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.038, mean=0.038, max=0.038, sum=0.114 (3)\", \"tab\": \"General information\", \"score\": \"0.038\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1136.933, mean=1418.457, max=1595.508, sum=4255.37 (3)\", \"tab\": \"General information\", \"score\": \"1418.4566666666667\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=5.418, mean=5.682, max=5.988, sum=17.046 (3)\", \"tab\": \"General information\", \"score\": \"5.6819999999999995\"}", - "NaturalQuestions (open-book) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NaturalQuestions (closed-book) - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - Stereotypes (gender)": "{\"description\": \"min=0.5, mean=0.5, max=0.5, sum=1.5 (3)\", \"tab\": \"Bias\", \"score\": \"0.5\"}", - "NaturalQuestions (closed-book) - Representation (race)": "{\"description\": \"min=0.255, mean=0.333, max=0.386, sum=1.0 (3)\", \"tab\": \"Bias\", \"score\": \"0.3331804837187507\"}", - "NaturalQuestions (closed-book) - Representation (gender)": "{\"description\": \"min=0.125, mean=0.175, max=0.2, sum=0.525 (3)\", \"tab\": \"Bias\", \"score\": \"0.17500000000000002\"}", - "NaturalQuestions (open-book) - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=2 (3)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "NaturalQuestions (open-book) - Stereotypes (gender)": "{\"description\": \"min=0.38, mean=0.46, max=0.5, sum=1.38 (3)\", \"tab\": \"Bias\", \"score\": \"0.4601449275362319\"}", - "NaturalQuestions (open-book) - Representation (race)": "{\"description\": \"min=0.451, mean=0.478, max=0.506, sum=1.433 (3)\", \"tab\": \"Bias\", \"score\": \"0.47760288745821544\"}", - "NaturalQuestions (open-book) - Representation (gender)": "{\"description\": \"min=0.011, mean=0.041, max=0.063, sum=0.122 (3)\", \"tab\": \"Bias\", \"score\": \"0.04050846488217801\"}", - "NaturalQuestions (closed-book) - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}", - "NaturalQuestions (open-book) - Toxic fraction": "{\"description\": \"min=0.001, mean=0.001, max=0.002, sum=0.004 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.0013333333333333333\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "QuAC", - "source_data": { - "dataset_name": "QuAC", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on QuAC", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.358, - "details": { - "description": "min=0.348, mean=0.358, max=0.372, sum=1.075 (3)", - "tab": "Accuracy", - "QuAC - ECE (10-bin)": "{\"description\": \"min=0.042, mean=0.043, max=0.045, sum=0.13 (3)\", \"tab\": \"Calibration\", \"score\": \"0.04341080368618692\"}", - "QuAC - F1 (Robustness)": "{\"description\": \"min=0.216, mean=0.222, max=0.232, sum=0.667 (3)\", \"tab\": \"Robustness\", \"score\": \"0.22242500588714678\"}", - "QuAC - F1 (Fairness)": "{\"description\": \"min=0.263, mean=0.268, max=0.275, sum=0.805 (3)\", \"tab\": \"Fairness\", \"score\": \"0.2682228394530809\"}", - "QuAC - Denoised inference time (s)": "{\"description\": \"min=1.898, mean=2.064, max=2.149, sum=6.193 (3)\", \"tab\": \"Efficiency\", \"score\": \"2.0642993667534726\"}", - "QuAC - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "QuAC - # train": "{\"description\": \"min=1.788, mean=1.829, max=1.88, sum=5.486 (3)\", \"tab\": \"General information\", \"score\": \"1.8286666666666667\"}", - "QuAC - truncated": "{\"description\": \"min=0.001, mean=0.001, max=0.001, sum=0.003 (3)\", \"tab\": \"General information\", \"score\": \"0.001\"}", - "QuAC - # prompt tokens": "{\"description\": \"min=1645.856, mean=1698.711, max=1730.814, sum=5096.134 (3)\", \"tab\": \"General information\", \"score\": \"1698.7113333333334\"}", - "QuAC - # output tokens": "{\"description\": \"min=22.621, mean=26.784, max=29.261, sum=80.351 (3)\", \"tab\": \"General information\", \"score\": \"26.783666666666665\"}", - "QuAC - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "QuAC - Stereotypes (race)": "{\"description\": \"min=0.594, mean=0.604, max=0.613, sum=1.811 (3)\", \"tab\": \"Bias\", \"score\": \"0.6038019374416433\"}", - "QuAC - Stereotypes (gender)": "{\"description\": \"min=0.417, mean=0.42, max=0.425, sum=1.26 (3)\", \"tab\": \"Bias\", \"score\": \"0.4200049682548366\"}", - "QuAC - Representation (race)": "{\"description\": \"min=0.287, mean=0.329, max=0.362, sum=0.988 (3)\", \"tab\": \"Bias\", \"score\": \"0.3293434102054505\"}", - "QuAC - Representation (gender)": "{\"description\": \"min=0.231, mean=0.242, max=0.257, sum=0.725 (3)\", \"tab\": \"Bias\", \"score\": \"0.2415041378322658\"}", - "QuAC - Toxic fraction": "{\"description\": \"min=0.002, mean=0.003, max=0.004, sum=0.009 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.0030000000000000005\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "HellaSwag", - "source_data": { - "dataset_name": "HellaSwag", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on HellaSwag", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.765, - "details": { - "description": "min=0.765, mean=0.765, max=0.765, sum=0.765 (1)", - "tab": "Accuracy", - "HellaSwag - ECE (10-bin)": "{\"description\": \"min=0.217, mean=0.217, max=0.217, sum=0.217 (1)\", \"tab\": \"Calibration\", \"score\": \"0.21741807730831492\"}", - "HellaSwag - EM (Robustness)": "{\"description\": \"min=0.726, mean=0.726, max=0.726, sum=0.726 (1)\", \"tab\": \"Robustness\", \"score\": \"0.726\"}", - "HellaSwag - EM (Fairness)": "{\"description\": \"min=0.614, mean=0.614, max=0.614, sum=0.614 (1)\", \"tab\": \"Fairness\", \"score\": \"0.614\"}", - "HellaSwag - Denoised inference time (s)": "{\"description\": \"min=0.284, mean=0.284, max=0.284, sum=0.284 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.2835968515624999\"}", - "HellaSwag - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "HellaSwag - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "HellaSwag - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "HellaSwag - # prompt tokens": "{\"description\": \"min=62.466, mean=62.466, max=62.466, sum=62.466 (1)\", \"tab\": \"General information\", \"score\": \"62.466\"}", - "HellaSwag - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "HellaSwag - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.534, - "details": { - "description": "min=0.534, mean=0.534, max=0.534, sum=0.534 (1)", - "tab": "Accuracy", - "OpenbookQA - ECE (10-bin)": "{\"description\": \"min=0.25, mean=0.25, max=0.25, sum=0.25 (1)\", \"tab\": \"Calibration\", \"score\": \"0.25015305244306557\"}", - "OpenbookQA - EM (Robustness)": "{\"description\": \"min=0.43, mean=0.43, max=0.43, sum=0.43 (1)\", \"tab\": \"Robustness\", \"score\": \"0.43\"}", - "OpenbookQA - EM (Fairness)": "{\"description\": \"min=0.466, mean=0.466, max=0.466, sum=0.466 (1)\", \"tab\": \"Fairness\", \"score\": \"0.466\"}", - "OpenbookQA - Denoised inference time (s)": "{\"description\": \"min=0.259, mean=0.259, max=0.259, sum=0.259 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.2588512968749986\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=4.348, mean=4.348, max=4.348, sum=4.348 (1)\", \"tab\": \"General information\", \"score\": \"4.348\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "TruthfulQA", - "source_data": { - "dataset_name": "TruthfulQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on TruthfulQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.175, - "details": { - "description": "min=0.157, mean=0.175, max=0.187, sum=0.524 (3)", - "tab": "Accuracy", - "TruthfulQA - ECE (10-bin)": "{\"description\": \"min=0.099, mean=0.113, max=0.123, sum=0.339 (3)\", \"tab\": \"Calibration\", \"score\": \"0.11285677982128534\"}", - "TruthfulQA - EM (Robustness)": "{\"description\": \"min=0.13, mean=0.154, max=0.176, sum=0.462 (3)\", \"tab\": \"Robustness\", \"score\": \"0.15392456676860347\"}", - "TruthfulQA - EM (Fairness)": "{\"description\": \"min=0.142, mean=0.156, max=0.168, sum=0.468 (3)\", \"tab\": \"Fairness\", \"score\": \"0.15596330275229356\"}", - "TruthfulQA - Denoised inference time (s)": "{\"description\": \"min=0.423, mean=0.443, max=0.454, sum=1.328 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.44282831613149837\"}", - "TruthfulQA - # eval": "{\"description\": \"min=654, mean=654, max=654, sum=1962 (3)\", \"tab\": \"General information\", \"score\": \"654.0\"}", - "TruthfulQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "TruthfulQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "TruthfulQA - # prompt tokens": "{\"description\": \"min=317.682, mean=355.015, max=375.682, sum=1065.046 (3)\", \"tab\": \"General information\", \"score\": \"355.0152905198777\"}", - "TruthfulQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=3 (3)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "TruthfulQA - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MS MARCO (TREC)", - "source_data": { - "dataset_name": "MS MARCO (TREC)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "NDCG@10 on MS MARCO (TREC)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.363, - "details": { - "description": "min=0.316, mean=0.363, max=0.406, sum=1.089 (3)", - "tab": "Accuracy", - "MS MARCO (regular) - RR@10 (Robustness)": "{\"description\": \"min=0.131, mean=0.144, max=0.157, sum=0.433 (3)\", \"tab\": \"Robustness\", \"score\": \"0.14417447089947086\"}", - "MS MARCO (TREC) - NDCG@10 (Robustness)": "{\"description\": \"min=0.276, mean=0.307, max=0.347, sum=0.921 (3)\", \"tab\": \"Robustness\", \"score\": \"0.3070790784160127\"}", - "MS MARCO (regular) - RR@10 (Fairness)": "{\"description\": \"min=0.151, mean=0.18, max=0.202, sum=0.54 (3)\", \"tab\": \"Fairness\", \"score\": \"0.17989272486772476\"}", - "MS MARCO (TREC) - NDCG@10 (Fairness)": "{\"description\": \"min=0.308, mean=0.348, max=0.386, sum=1.044 (3)\", \"tab\": \"Fairness\", \"score\": \"0.34798299201075195\"}", - "MS MARCO (regular) - Denoised inference time (s)": "{\"description\": \"min=0.482, mean=0.501, max=0.52, sum=1.502 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.500707514648438\"}", - "MS MARCO (TREC) - Denoised inference time (s)": "{\"description\": \"min=0.477, mean=0.496, max=0.516, sum=1.489 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.4963945009689923\"}", - "MS MARCO (regular) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "MS MARCO (regular) - # train": "{\"description\": \"min=2, mean=2, max=2, sum=6 (3)\", \"tab\": \"General information\", \"score\": \"2.0\"}", - "MS MARCO (regular) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MS MARCO (regular) - # prompt tokens": "{\"description\": \"min=349.303, mean=385.636, max=423.303, sum=1156.909 (3)\", \"tab\": \"General information\", \"score\": \"385.63633333333337\"}", - "MS MARCO (regular) - # output tokens": "{\"description\": \"min=2, mean=2.001, max=2.004, sum=6.004 (3)\", \"tab\": \"General information\", \"score\": \"2.001333333333333\"}", - "MS MARCO (regular) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "MS MARCO (TREC) - # eval": "{\"description\": \"min=43, mean=43, max=43, sum=129 (3)\", \"tab\": \"General information\", \"score\": \"43.0\"}", - "MS MARCO (TREC) - # train": "{\"description\": \"min=2, mean=2, max=2, sum=6 (3)\", \"tab\": \"General information\", \"score\": \"2.0\"}", - "MS MARCO (TREC) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MS MARCO (TREC) - # prompt tokens": "{\"description\": \"min=337.047, mean=373.38, max=411.047, sum=1120.14 (3)\", \"tab\": \"General information\", \"score\": \"373.3798449612403\"}", - "MS MARCO (TREC) - # output tokens": "{\"description\": \"min=2.047, mean=2.047, max=2.047, sum=6.14 (3)\", \"tab\": \"General information\", \"score\": \"2.046511627906977\"}", - "MS MARCO (TREC) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "MS MARCO (regular) - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "MS MARCO (TREC) - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CNN/DailyMail", - "source_data": { - "dataset_name": "CNN/DailyMail", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on CNN/DailyMail", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.144, - "details": { - "description": "min=0.137, mean=0.144, max=0.157, sum=0.861 (6)", - "tab": "Accuracy", - "CNN/DailyMail - Denoised inference time (s)": "{\"description\": \"min=3.558, mean=3.777, max=3.91, sum=22.664 (6)\", \"tab\": \"Efficiency\", \"score\": \"3.777328921804216\"}", - "CNN/DailyMail - # eval": "{\"description\": \"min=466, mean=466, max=466, sum=2796 (6)\", \"tab\": \"General information\", \"score\": \"466.0\"}", - "CNN/DailyMail - # train": "{\"description\": \"min=5, mean=5, max=5, sum=30 (6)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CNN/DailyMail - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (6)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CNN/DailyMail - # prompt tokens": "{\"description\": \"min=1203.032, mean=1213.032, max=1224.032, sum=7278.193 (6)\", \"tab\": \"General information\", \"score\": \"1213.0321888412018\"}", - "CNN/DailyMail - # output tokens": "{\"description\": \"min=67.139, mean=72.469, max=75.648, sum=434.815 (6)\", \"tab\": \"General information\", \"score\": \"72.46924177396282\"}", - "CNN/DailyMail - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=18 (6)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "CNN/DailyMail - Stereotypes (race)": "{\"description\": \"min=0.62, mean=0.63, max=0.647, sum=3.781 (6)\", \"tab\": \"Bias\", \"score\": \"0.6302246589223909\"}", - "CNN/DailyMail - Stereotypes (gender)": "{\"description\": \"min=0.382, mean=0.386, max=0.393, sum=2.314 (6)\", \"tab\": \"Bias\", \"score\": \"0.385603383216647\"}", - "CNN/DailyMail - Representation (race)": "{\"description\": \"min=0.288, mean=0.325, max=0.362, sum=1.95 (6)\", \"tab\": \"Bias\", \"score\": \"0.3250193306482005\"}", - "CNN/DailyMail - Representation (gender)": "{\"description\": \"min=0.13, mean=0.131, max=0.132, sum=0.788 (6)\", \"tab\": \"Bias\", \"score\": \"0.13141527227323743\"}", - "CNN/DailyMail - Toxic fraction": "{\"description\": \"min=0, mean=0.002, max=0.004, sum=0.013 (6)\", \"tab\": \"Toxicity\", \"score\": \"0.002145922746781116\"}", - "CNN/DailyMail - SummaC": "{\"description\": \"min=0.491, mean=0.515, max=0.544, sum=1.545 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.5151288171631818\"}", - "CNN/DailyMail - QAFactEval": "{\"description\": \"min=4.661, mean=4.697, max=4.725, sum=28.182 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"4.696964335081241\"}", - "CNN/DailyMail - BERTScore (F1)": "{\"description\": \"min=0.264, mean=0.278, max=0.301, sum=0.834 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.27790265116917295\"}", - "CNN/DailyMail - Coverage": "{\"description\": \"min=0.965, mean=0.976, max=0.984, sum=5.856 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"0.97598626364496\"}", - "CNN/DailyMail - Density": "{\"description\": \"min=40.605, mean=53.93, max=67.411, sum=323.578 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"53.929605831357485\"}", - "CNN/DailyMail - Compression": "{\"description\": \"min=8.981, mean=9.579, max=10.219, sum=57.476 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"9.579310239916042\"}", - "CNN/DailyMail - HumanEval-faithfulness": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-relevance": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-coherence": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "XSUM", - "source_data": { - "dataset_name": "XSUM", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on XSUM", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.129, - "details": { - "description": "min=0.128, mean=0.129, max=0.131, sum=0.776 (6)", - "tab": "Accuracy", - "XSUM - Denoised inference time (s)": "{\"description\": \"min=1.615, mean=1.629, max=1.648, sum=9.776 (6)\", \"tab\": \"Efficiency\", \"score\": \"1.6292920332441818\"}", - "XSUM - # eval": "{\"description\": \"min=518, mean=518, max=518, sum=3108 (6)\", \"tab\": \"General information\", \"score\": \"518.0\"}", - "XSUM - # train": "{\"description\": \"min=5, mean=5, max=5, sum=30 (6)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "XSUM - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (6)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "XSUM - # prompt tokens": "{\"description\": \"min=1099.388, mean=1133.388, max=1172.388, sum=6800.328 (6)\", \"tab\": \"General information\", \"score\": \"1133.388030888031\"}", - "XSUM - # output tokens": "{\"description\": \"min=21.958, mean=22.013, max=22.106, sum=132.077 (6)\", \"tab\": \"General information\", \"score\": \"22.012870012870014\"}", - "XSUM - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=18 (6)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "XSUM - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=4.0 (6)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666669\"}", - "XSUM - Stereotypes (gender)": "{\"description\": \"min=0.46, mean=0.472, max=0.483, sum=2.834 (6)\", \"tab\": \"Bias\", \"score\": \"0.4724007038712921\"}", - "XSUM - Representation (race)": "{\"description\": \"min=0.467, mean=0.48, max=0.505, sum=2.877 (6)\", \"tab\": \"Bias\", \"score\": \"0.47956989247311826\"}", - "XSUM - Representation (gender)": "{\"description\": \"min=0.154, mean=0.186, max=0.216, sum=1.116 (6)\", \"tab\": \"Bias\", \"score\": \"0.18604199883585584\"}", - "XSUM - Toxic fraction": "{\"description\": \"min=0, mean=0.002, max=0.004, sum=0.012 (6)\", \"tab\": \"Toxicity\", \"score\": \"0.0019305019305019308\"}", - "XSUM - SummaC": "{\"description\": \"min=-0.294, mean=-0.287, max=-0.282, sum=-0.861 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"-0.2868511554050323\"}", - "XSUM - QAFactEval": "{\"description\": \"min=2.48, mean=3.182, max=3.598, sum=19.091 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"3.1818935586249126\"}", - "XSUM - BERTScore (F1)": "{\"description\": \"min=0.432, mean=0.435, max=0.438, sum=1.305 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.43511885902101227\"}", - "XSUM - Coverage": "{\"description\": \"min=0.775, mean=0.784, max=0.792, sum=4.704 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"0.7840584721092689\"}", - "XSUM - Density": "{\"description\": \"min=2.514, mean=2.63, max=2.802, sum=15.779 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"2.6298709619480816\"}", - "XSUM - Compression": "{\"description\": \"min=16.767, mean=16.862, max=16.987, sum=101.17 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"16.861740741647864\"}", - "XSUM - HumanEval-faithfulness": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-relevance": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-coherence": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "IMDB", - "source_data": { - "dataset_name": "IMDB", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on IMDB", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.943, - "details": { - "description": "min=0.934, mean=0.943, max=0.951, sum=2.83 (3)", - "tab": "Accuracy", - "IMDB - ECE (10-bin)": "{\"description\": \"min=0.06, mean=0.064, max=0.072, sum=0.191 (3)\", \"tab\": \"Calibration\", \"score\": \"0.06375881576094916\"}", - "IMDB - EM (Robustness)": "{\"description\": \"min=0.917, mean=0.923, max=0.934, sum=2.768 (3)\", \"tab\": \"Robustness\", \"score\": \"0.9226666666666666\"}", - "IMDB - EM (Fairness)": "{\"description\": \"min=0.922, mean=0.932, max=0.941, sum=2.797 (3)\", \"tab\": \"Fairness\", \"score\": \"0.9323333333333333\"}", - "IMDB - Denoised inference time (s)": "{\"description\": \"min=0.682, mean=0.852, max=1.035, sum=2.555 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.8516515608723956\"}", - "IMDB - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "IMDB - # train": "{\"description\": \"min=4.915, mean=4.972, max=5, sum=14.915 (3)\", \"tab\": \"General information\", \"score\": \"4.971666666666667\"}", - "IMDB - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IMDB - # prompt tokens": "{\"description\": \"min=853.851, mean=1281.577, max=1725.03, sum=3844.732 (3)\", \"tab\": \"General information\", \"score\": \"1281.5773333333334\"}", - "IMDB - # output tokens": "{\"description\": \"min=2, mean=2, max=2, sum=6 (3)\", \"tab\": \"General information\", \"score\": \"2.0\"}", - "IMDB - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "IMDB - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CivilComments", - "source_data": { - "dataset_name": "CivilComments", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on CivilComments", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.553, - "details": { - "description": "min=0.03, mean=0.553, max=0.968, sum=29.863 (54)", - "tab": "Accuracy", - "CivilComments - ECE (10-bin)": "{\"description\": \"min=0.048, mean=0.27, max=0.587, sum=14.569 (54)\", \"tab\": \"Calibration\", \"score\": \"0.26979933840430187\"}", - "CivilComments - EM (Robustness)": "{\"description\": \"min=0.027, mean=0.271, max=0.732, sum=14.649 (54)\", \"tab\": \"Robustness\", \"score\": \"0.2712865813183887\"}", - "CivilComments - EM (Fairness)": "{\"description\": \"min=0.006, mean=0.478, max=0.958, sum=25.823 (54)\", \"tab\": \"Fairness\", \"score\": \"0.4782106548652487\"}", - "CivilComments - Denoised inference time (s)": "{\"description\": \"min=0.43, mean=0.552, max=0.724, sum=29.829 (54)\", \"tab\": \"Efficiency\", \"score\": \"0.5523870780537201\"}", - "CivilComments - # eval": "{\"description\": \"min=74, mean=371.556, max=683, sum=20064 (54)\", \"tab\": \"General information\", \"score\": \"371.55555555555554\"}", - "CivilComments - # train": "{\"description\": \"min=5, mean=5, max=5, sum=270 (54)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CivilComments - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (54)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CivilComments - # prompt tokens": "{\"description\": \"min=271.927, mean=532.602, max=942.498, sum=28760.487 (54)\", \"tab\": \"General information\", \"score\": \"532.6016121330534\"}", - "CivilComments - # output tokens": "{\"description\": \"min=2, mean=2, max=2, sum=108 (54)\", \"tab\": \"General information\", \"score\": \"2.0\"}", - "CivilComments - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=162 (54)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "CivilComments - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (54)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "RAFT", - "source_data": { - "dataset_name": "RAFT", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on RAFT", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.681, - "details": { - "description": "min=0.225, mean=0.681, max=0.975, sum=22.475 (33)", - "tab": "Accuracy", - "RAFT - ECE (10-bin)": "{\"description\": \"min=0.103, mean=0.228, max=0.595, sum=7.528 (33)\", \"tab\": \"Calibration\", \"score\": \"0.2281177870147751\"}", - "RAFT - EM (Robustness)": "{\"description\": \"min=0.025, mean=0.555, max=0.875, sum=18.3 (33)\", \"tab\": \"Robustness\", \"score\": \"0.5545454545454546\"}", - "RAFT - EM (Fairness)": "{\"description\": \"min=0.125, mean=0.623, max=0.975, sum=20.55 (33)\", \"tab\": \"Fairness\", \"score\": \"0.6227272727272728\"}", - "RAFT - Denoised inference time (s)": "{\"description\": \"min=0.423, mean=0.687, max=1.043, sum=22.661 (33)\", \"tab\": \"Efficiency\", \"score\": \"0.6866916923137625\"}", - "RAFT - # eval": "{\"description\": \"min=40, mean=40, max=40, sum=1320 (33)\", \"tab\": \"General information\", \"score\": \"40.0\"}", - "RAFT - # train": "{\"description\": \"min=0.95, mean=4.658, max=5, sum=153.7 (33)\", \"tab\": \"General information\", \"score\": \"4.657575757575757\"}", - "RAFT - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (33)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "RAFT - # prompt tokens": "{\"description\": \"min=212.25, mean=712.248, max=1745.25, sum=23504.175 (33)\", \"tab\": \"General information\", \"score\": \"712.2477272727273\"}", - "RAFT - # output tokens": "{\"description\": \"min=1.95, mean=3.634, max=6.925, sum=119.925 (33)\", \"tab\": \"General information\", \"score\": \"3.6340909090909084\"}", - "RAFT - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=99 (33)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "RAFT - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (33)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/ai21_j1-large-v1-7.5b.json b/data/models/ai21_j1-large-v1-7.5b.json deleted file mode 100644 index 522d2a81eb65d0dbdd8b4c55dfded634686f856c..0000000000000000000000000000000000000000 --- a/data/models/ai21_j1-large-v1-7.5b.json +++ /dev/null @@ -1,674 +0,0 @@ -{ - "model_info": { - "name": "J1-Large v1 7.5B", - "id": "ai21/J1-Large-v1-7.5B", - "developer": "ai21", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_classic/ai21_J1-Large-v1-7.5B/1774096308.339228", - "retrieved_timestamp": "1774096308.339228", - "source_metadata": { - "source_name": "helm_classic", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_classic", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_classic", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperform on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.285, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Calibration": "{\"description\": \"\", \"tab\": \"Calibration\", \"score\": \"0.6383920923698907\"}", - "Mean win rate - Robustness": "{\"description\": \"\", \"tab\": \"Robustness\", \"score\": \"0.29777282413544925\"}", - "Mean win rate - Fairness": "{\"description\": \"\", \"tab\": \"Fairness\", \"score\": \"0.27467778791471786\"}", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.38930372807017544\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}", - "Mean win rate - Bias": "{\"description\": \"\", \"tab\": \"Bias\", \"score\": \"0.5487461676083087\"}", - "Mean win rate - Toxicity": "{\"description\": \"\", \"tab\": \"Toxicity\", \"score\": \"0.6599416016082683\"}", - "Mean win rate - Summarization metrics": "{\"description\": \"\", \"tab\": \"Summarization metrics\", \"score\": \"0.6502297410192147\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.241, - "details": { - "description": "min=0.2, mean=0.241, max=0.298, sum=3.617 (15)", - "tab": "Accuracy", - "MMLU - ECE (10-bin)": "{\"description\": \"min=0.051, mean=0.123, max=0.181, sum=1.842 (15)\", \"tab\": \"Calibration\", \"score\": \"0.12277396117394333\"}", - "MMLU - EM (Robustness)": "{\"description\": \"min=0.16, mean=0.2, max=0.272, sum=3.002 (15)\", \"tab\": \"Robustness\", \"score\": \"0.20011695906432747\"}", - "MMLU - EM (Fairness)": "{\"description\": \"min=0.16, mean=0.204, max=0.23, sum=3.059 (15)\", \"tab\": \"Fairness\", \"score\": \"0.2039415204678363\"}", - "MMLU - Denoised inference time (s)": "{\"description\": \"min=0.348, mean=0.377, max=0.422, sum=5.648 (15)\", \"tab\": \"Efficiency\", \"score\": \"0.3765351217105263\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=1542 (15)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=75 (15)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (15)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=308.59, mean=396.74, max=552.719, sum=5951.098 (15)\", \"tab\": \"General information\", \"score\": \"396.73985964912276\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=15 (15)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "MMLU - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=45 (15)\", \"tab\": \"General information\", \"score\": \"3.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "BoolQ", - "source_data": { - "dataset_name": "BoolQ", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on BoolQ", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.683, - "details": { - "description": "min=0.652, mean=0.683, max=0.709, sum=2.05 (3)", - "tab": "Accuracy", - "BoolQ - ECE (10-bin)": "{\"description\": \"min=0.085, mean=0.106, max=0.133, sum=0.319 (3)\", \"tab\": \"Calibration\", \"score\": \"0.10621693084730484\"}", - "BoolQ - EM (Robustness)": "{\"description\": \"min=0.539, mean=0.567, max=0.603, sum=1.701 (3)\", \"tab\": \"Robustness\", \"score\": \"0.5670000000000001\"}", - "BoolQ - EM (Fairness)": "{\"description\": \"min=0.591, mean=0.622, max=0.651, sum=1.867 (3)\", \"tab\": \"Fairness\", \"score\": \"0.6223333333333333\"}", - "BoolQ - Denoised inference time (s)": "{\"description\": \"min=0.43, mean=0.485, max=0.566, sum=1.455 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.48513916883680525\"}", - "BoolQ - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "BoolQ - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "BoolQ - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "BoolQ - # prompt tokens": "{\"description\": \"min=506.985, mean=694.652, max=952.985, sum=2083.955 (3)\", \"tab\": \"General information\", \"score\": \"694.6516666666666\"}", - "BoolQ - # output tokens": "{\"description\": \"min=2, mean=2, max=2, sum=6 (3)\", \"tab\": \"General information\", \"score\": \"2.0\"}", - "BoolQ - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "BoolQ - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.623, - "details": { - "description": "min=0.612, mean=0.623, max=0.634, sum=1.87 (3)", - "tab": "Accuracy", - "NarrativeQA - ECE (10-bin)": "{\"description\": \"min=0.042, mean=0.046, max=0.048, sum=0.137 (3)\", \"tab\": \"Calibration\", \"score\": \"0.04554705251298522\"}", - "NarrativeQA - F1 (Robustness)": "{\"description\": \"min=0.341, mean=0.4, max=0.438, sum=1.201 (3)\", \"tab\": \"Robustness\", \"score\": \"0.4003895179156612\"}", - "NarrativeQA - F1 (Fairness)": "{\"description\": \"min=0.496, mean=0.513, max=0.524, sum=1.538 (3)\", \"tab\": \"Fairness\", \"score\": \"0.5126679432053903\"}", - "NarrativeQA - Denoised inference time (s)": "{\"description\": \"min=0.768, mean=0.797, max=0.829, sum=2.391 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.7971074946205007\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=1065 (3)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=2.166, mean=2.639, max=3.225, sum=7.918 (3)\", \"tab\": \"General information\", \"score\": \"2.63943661971831\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=1598.614, mean=1692.218, max=1777.299, sum=5076.654 (3)\", \"tab\": \"General information\", \"score\": \"1692.2178403755868\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=4.797, mean=5.09, max=5.518, sum=15.27 (3)\", \"tab\": \"General information\", \"score\": \"5.090140845070422\"}", - "NarrativeQA - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NarrativeQA - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NarrativeQA - Stereotypes (gender)": "{\"description\": \"min=0.5, mean=0.5, max=0.5, sum=1 (2)\", \"tab\": \"Bias\", \"score\": \"0.5\"}", - "NarrativeQA - Representation (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=2 (3)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "NarrativeQA - Representation (gender)": "{\"description\": \"min=0.17, mean=0.203, max=0.223, sum=0.609 (3)\", \"tab\": \"Bias\", \"score\": \"0.20304247377415918\"}", - "NarrativeQA - Toxic fraction": "{\"description\": \"min=0.011, mean=0.013, max=0.014, sum=0.039 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.013145539906103287\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (open-book)", - "source_data": { - "dataset_name": "NaturalQuestions (open-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (open-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.532, - "details": { - "description": "min=0.5, mean=0.532, max=0.571, sum=1.597 (3)", - "tab": "Accuracy", - "NaturalQuestions (closed-book) - ECE (10-bin)": "{\"description\": \"min=0.013, mean=0.015, max=0.017, sum=0.046 (3)\", \"tab\": \"Calibration\", \"score\": \"0.01549922748171477\"}", - "NaturalQuestions (open-book) - ECE (10-bin)": "{\"description\": \"min=0.075, mean=0.086, max=0.093, sum=0.258 (3)\", \"tab\": \"Calibration\", \"score\": \"0.08597598507389619\"}", - "NaturalQuestions (closed-book) - F1 (Robustness)": "{\"description\": \"min=0.092, mean=0.098, max=0.106, sum=0.293 (3)\", \"tab\": \"Robustness\", \"score\": \"0.097632746101742\"}", - "NaturalQuestions (open-book) - F1 (Robustness)": "{\"description\": \"min=0.361, mean=0.41, max=0.455, sum=1.23 (3)\", \"tab\": \"Robustness\", \"score\": \"0.4099829032840138\"}", - "NaturalQuestions (closed-book) - F1 (Fairness)": "{\"description\": \"min=0.14, mean=0.146, max=0.151, sum=0.439 (3)\", \"tab\": \"Fairness\", \"score\": \"0.14648226412007787\"}", - "NaturalQuestions (open-book) - F1 (Fairness)": "{\"description\": \"min=0.44, mean=0.47, max=0.508, sum=1.409 (3)\", \"tab\": \"Fairness\", \"score\": \"0.4695231845662433\"}", - "NaturalQuestions (closed-book) - Denoised inference time (s)": "{\"description\": \"min=0.355, mean=0.372, max=0.396, sum=1.117 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.3722484414062495\"}", - "NaturalQuestions (open-book) - Denoised inference time (s)": "{\"description\": \"min=0.66, mean=0.733, max=0.784, sum=2.198 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.7326816432291658\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=94.377, mean=99.377, max=102.377, sum=298.131 (3)\", \"tab\": \"General information\", \"score\": \"99.377\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=6.868, mean=7.876, max=9.311, sum=23.628 (3)\", \"tab\": \"General information\", \"score\": \"7.876\"}", - "NaturalQuestions (closed-book) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.568, mean=4.666, max=4.734, sum=13.999 (3)\", \"tab\": \"General information\", \"score\": \"4.666333333333333\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.038, mean=0.038, max=0.038, sum=0.114 (3)\", \"tab\": \"General information\", \"score\": \"0.038\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1136.933, mean=1418.457, max=1595.508, sum=4255.37 (3)\", \"tab\": \"General information\", \"score\": \"1418.4566666666667\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=5.487, mean=5.946, max=6.338, sum=17.838 (3)\", \"tab\": \"General information\", \"score\": \"5.946000000000001\"}", - "NaturalQuestions (open-book) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NaturalQuestions (closed-book) - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - Stereotypes (gender)": "{\"description\": \"min=0.214, mean=0.405, max=0.5, sum=1.214 (3)\", \"tab\": \"Bias\", \"score\": \"0.4047619047619048\"}", - "NaturalQuestions (closed-book) - Representation (race)": "{\"description\": \"min=0.302, mean=0.362, max=0.45, sum=1.085 (3)\", \"tab\": \"Bias\", \"score\": \"0.36169748540882557\"}", - "NaturalQuestions (closed-book) - Representation (gender)": "{\"description\": \"min=0.088, mean=0.216, max=0.371, sum=0.647 (3)\", \"tab\": \"Bias\", \"score\": \"0.21556767868437698\"}", - "NaturalQuestions (open-book) - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=1.333 (2)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666667\"}", - "NaturalQuestions (open-book) - Stereotypes (gender)": "{\"description\": \"min=0.5, mean=0.5, max=0.5, sum=1.5 (3)\", \"tab\": \"Bias\", \"score\": \"0.5\"}", - "NaturalQuestions (open-book) - Representation (race)": "{\"description\": \"min=0.327, mean=0.394, max=0.457, sum=1.182 (3)\", \"tab\": \"Bias\", \"score\": \"0.39383347574877653\"}", - "NaturalQuestions (open-book) - Representation (gender)": "{\"description\": \"min=0.106, mean=0.109, max=0.113, sum=0.328 (3)\", \"tab\": \"Bias\", \"score\": \"0.10941198128319474\"}", - "NaturalQuestions (closed-book) - Toxic fraction": "{\"description\": \"min=0.001, mean=0.002, max=0.002, sum=0.005 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.0016666666666666668\"}", - "NaturalQuestions (open-book) - Toxic fraction": "{\"description\": \"min=0, mean=0.001, max=0.001, sum=0.002 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.0006666666666666666\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "QuAC", - "source_data": { - "dataset_name": "QuAC", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on QuAC", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.328, - "details": { - "description": "min=0.322, mean=0.328, max=0.336, sum=0.983 (3)", - "tab": "Accuracy", - "QuAC - ECE (10-bin)": "{\"description\": \"min=0.016, mean=0.024, max=0.033, sum=0.073 (3)\", \"tab\": \"Calibration\", \"score\": \"0.02431531680637249\"}", - "QuAC - F1 (Robustness)": "{\"description\": \"min=0.186, mean=0.197, max=0.209, sum=0.591 (3)\", \"tab\": \"Robustness\", \"score\": \"0.19699898429353593\"}", - "QuAC - F1 (Fairness)": "{\"description\": \"min=0.227, mean=0.241, max=0.256, sum=0.722 (3)\", \"tab\": \"Fairness\", \"score\": \"0.24062000532402938\"}", - "QuAC - Denoised inference time (s)": "{\"description\": \"min=1.105, mean=1.16, max=1.191, sum=3.48 (3)\", \"tab\": \"Efficiency\", \"score\": \"1.159840737413194\"}", - "QuAC - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "QuAC - # train": "{\"description\": \"min=1.788, mean=1.829, max=1.88, sum=5.486 (3)\", \"tab\": \"General information\", \"score\": \"1.8286666666666667\"}", - "QuAC - truncated": "{\"description\": \"min=0.001, mean=0.001, max=0.001, sum=0.003 (3)\", \"tab\": \"General information\", \"score\": \"0.001\"}", - "QuAC - # prompt tokens": "{\"description\": \"min=1645.856, mean=1698.711, max=1730.814, sum=5096.134 (3)\", \"tab\": \"General information\", \"score\": \"1698.7113333333334\"}", - "QuAC - # output tokens": "{\"description\": \"min=23.833, mean=27.642, max=30.067, sum=82.927 (3)\", \"tab\": \"General information\", \"score\": \"27.64233333333333\"}", - "QuAC - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "QuAC - Stereotypes (race)": "{\"description\": \"min=0.632, mean=0.647, max=0.667, sum=1.942 (3)\", \"tab\": \"Bias\", \"score\": \"0.6472747525379104\"}", - "QuAC - Stereotypes (gender)": "{\"description\": \"min=0.407, mean=0.428, max=0.446, sum=1.284 (3)\", \"tab\": \"Bias\", \"score\": \"0.42785601825865643\"}", - "QuAC - Representation (race)": "{\"description\": \"min=0.226, mean=0.3, max=0.351, sum=0.9 (3)\", \"tab\": \"Bias\", \"score\": \"0.2998485806834953\"}", - "QuAC - Representation (gender)": "{\"description\": \"min=0.235, mean=0.249, max=0.271, sum=0.748 (3)\", \"tab\": \"Bias\", \"score\": \"0.24941347459181362\"}", - "QuAC - Toxic fraction": "{\"description\": \"min=0.002, mean=0.003, max=0.004, sum=0.008 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.0026666666666666666\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "HellaSwag", - "source_data": { - "dataset_name": "HellaSwag", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on HellaSwag", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7, - "details": { - "description": "min=0.7, mean=0.7, max=0.7, sum=0.7 (1)", - "tab": "Accuracy", - "HellaSwag - ECE (10-bin)": "{\"description\": \"min=0.192, mean=0.192, max=0.192, sum=0.192 (1)\", \"tab\": \"Calibration\", \"score\": \"0.19173198668049052\"}", - "HellaSwag - EM (Robustness)": "{\"description\": \"min=0.646, mean=0.646, max=0.646, sum=0.646 (1)\", \"tab\": \"Robustness\", \"score\": \"0.646\"}", - "HellaSwag - EM (Fairness)": "{\"description\": \"min=0.528, mean=0.528, max=0.528, sum=0.528 (1)\", \"tab\": \"Fairness\", \"score\": \"0.528\"}", - "HellaSwag - Denoised inference time (s)": "{\"description\": \"min=0.253, mean=0.253, max=0.253, sum=0.253 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.25286050781250013\"}", - "HellaSwag - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "HellaSwag - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "HellaSwag - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "HellaSwag - # prompt tokens": "{\"description\": \"min=62.466, mean=62.466, max=62.466, sum=62.466 (1)\", \"tab\": \"General information\", \"score\": \"62.466\"}", - "HellaSwag - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "HellaSwag - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.514, - "details": { - "description": "min=0.514, mean=0.514, max=0.514, sum=0.514 (1)", - "tab": "Accuracy", - "OpenbookQA - ECE (10-bin)": "{\"description\": \"min=0.25, mean=0.25, max=0.25, sum=0.25 (1)\", \"tab\": \"Calibration\", \"score\": \"0.24986668171933007\"}", - "OpenbookQA - EM (Robustness)": "{\"description\": \"min=0.412, mean=0.412, max=0.412, sum=0.412 (1)\", \"tab\": \"Robustness\", \"score\": \"0.412\"}", - "OpenbookQA - EM (Fairness)": "{\"description\": \"min=0.444, mean=0.444, max=0.444, sum=0.444 (1)\", \"tab\": \"Fairness\", \"score\": \"0.444\"}", - "OpenbookQA - Denoised inference time (s)": "{\"description\": \"min=0.238, mean=0.238, max=0.238, sum=0.238 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.2381039843749996\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=4.348, mean=4.348, max=4.348, sum=4.348 (1)\", \"tab\": \"General information\", \"score\": \"4.348\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "TruthfulQA", - "source_data": { - "dataset_name": "TruthfulQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on TruthfulQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.197, - "details": { - "description": "min=0.19, mean=0.197, max=0.2, sum=0.59 (3)", - "tab": "Accuracy", - "TruthfulQA - ECE (10-bin)": "{\"description\": \"min=0.105, mean=0.112, max=0.121, sum=0.337 (3)\", \"tab\": \"Calibration\", \"score\": \"0.11232689963932652\"}", - "TruthfulQA - EM (Robustness)": "{\"description\": \"min=0.138, mean=0.155, max=0.168, sum=0.465 (3)\", \"tab\": \"Robustness\", \"score\": \"0.15494393476044852\"}", - "TruthfulQA - EM (Fairness)": "{\"description\": \"min=0.159, mean=0.174, max=0.182, sum=0.521 (3)\", \"tab\": \"Fairness\", \"score\": \"0.17380224260958207\"}", - "TruthfulQA - Denoised inference time (s)": "{\"description\": \"min=0.351, mean=0.365, max=0.372, sum=1.094 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.36458362003058115\"}", - "TruthfulQA - # eval": "{\"description\": \"min=654, mean=654, max=654, sum=1962 (3)\", \"tab\": \"General information\", \"score\": \"654.0\"}", - "TruthfulQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "TruthfulQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "TruthfulQA - # prompt tokens": "{\"description\": \"min=317.682, mean=355.015, max=375.682, sum=1065.046 (3)\", \"tab\": \"General information\", \"score\": \"355.0152905198777\"}", - "TruthfulQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=3 (3)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "TruthfulQA - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MS MARCO (TREC)", - "source_data": { - "dataset_name": "MS MARCO (TREC)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "NDCG@10 on MS MARCO (TREC)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.292, - "details": { - "description": "min=0.266, mean=0.292, max=0.338, sum=0.877 (3)", - "tab": "Accuracy", - "MS MARCO (regular) - RR@10 (Robustness)": "{\"description\": \"min=0.089, mean=0.105, max=0.128, sum=0.315 (3)\", \"tab\": \"Robustness\", \"score\": \"0.10499510582010585\"}", - "MS MARCO (TREC) - NDCG@10 (Robustness)": "{\"description\": \"min=0.231, mean=0.248, max=0.274, sum=0.743 (3)\", \"tab\": \"Robustness\", \"score\": \"0.24769351383898738\"}", - "MS MARCO (regular) - RR@10 (Fairness)": "{\"description\": \"min=0.096, mean=0.117, max=0.143, sum=0.351 (3)\", \"tab\": \"Fairness\", \"score\": \"0.11706984126984123\"}", - "MS MARCO (TREC) - NDCG@10 (Fairness)": "{\"description\": \"min=0.258, mean=0.28, max=0.322, sum=0.841 (3)\", \"tab\": \"Fairness\", \"score\": \"0.2804651230679189\"}", - "MS MARCO (regular) - Denoised inference time (s)": "{\"description\": \"min=0.379, mean=0.393, max=0.406, sum=1.178 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.3926667591145831\"}", - "MS MARCO (TREC) - Denoised inference time (s)": "{\"description\": \"min=0.376, mean=0.389, max=0.402, sum=1.167 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.3890438468992247\"}", - "MS MARCO (regular) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "MS MARCO (regular) - # train": "{\"description\": \"min=2, mean=2, max=2, sum=6 (3)\", \"tab\": \"General information\", \"score\": \"2.0\"}", - "MS MARCO (regular) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MS MARCO (regular) - # prompt tokens": "{\"description\": \"min=349.303, mean=385.636, max=423.303, sum=1156.909 (3)\", \"tab\": \"General information\", \"score\": \"385.63633333333337\"}", - "MS MARCO (regular) - # output tokens": "{\"description\": \"min=2.011, mean=2.072, max=2.163, sum=6.217 (3)\", \"tab\": \"General information\", \"score\": \"2.0723333333333334\"}", - "MS MARCO (regular) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "MS MARCO (TREC) - # eval": "{\"description\": \"min=43, mean=43, max=43, sum=129 (3)\", \"tab\": \"General information\", \"score\": \"43.0\"}", - "MS MARCO (TREC) - # train": "{\"description\": \"min=2, mean=2, max=2, sum=6 (3)\", \"tab\": \"General information\", \"score\": \"2.0\"}", - "MS MARCO (TREC) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MS MARCO (TREC) - # prompt tokens": "{\"description\": \"min=337.047, mean=373.38, max=411.047, sum=1120.14 (3)\", \"tab\": \"General information\", \"score\": \"373.3798449612403\"}", - "MS MARCO (TREC) - # output tokens": "{\"description\": \"min=2.093, mean=2.116, max=2.163, sum=6.349 (3)\", \"tab\": \"General information\", \"score\": \"2.116279069767442\"}", - "MS MARCO (TREC) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "MS MARCO (regular) - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "MS MARCO (TREC) - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CNN/DailyMail", - "source_data": { - "dataset_name": "CNN/DailyMail", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on CNN/DailyMail", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.134, - "details": { - "description": "min=0.123, mean=0.134, max=0.147, sum=0.802 (6)", - "tab": "Accuracy", - "CNN/DailyMail - Denoised inference time (s)": "{\"description\": \"min=1.832, mean=2.011, max=2.216, sum=12.069 (6)\", \"tab\": \"Efficiency\", \"score\": \"2.011487112821144\"}", - "CNN/DailyMail - # eval": "{\"description\": \"min=466, mean=466, max=466, sum=2796 (6)\", \"tab\": \"General information\", \"score\": \"466.0\"}", - "CNN/DailyMail - # train": "{\"description\": \"min=5, mean=5, max=5, sum=30 (6)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CNN/DailyMail - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (6)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CNN/DailyMail - # prompt tokens": "{\"description\": \"min=1203.032, mean=1213.032, max=1224.032, sum=7278.193 (6)\", \"tab\": \"General information\", \"score\": \"1213.0321888412018\"}", - "CNN/DailyMail - # output tokens": "{\"description\": \"min=78.521, mean=89.614, max=102.401, sum=537.682 (6)\", \"tab\": \"General information\", \"score\": \"89.61373390557941\"}", - "CNN/DailyMail - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=18 (6)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "CNN/DailyMail - Stereotypes (race)": "{\"description\": \"min=0.602, mean=0.632, max=0.648, sum=3.791 (6)\", \"tab\": \"Bias\", \"score\": \"0.6318145834093977\"}", - "CNN/DailyMail - Stereotypes (gender)": "{\"description\": \"min=0.385, mean=0.391, max=0.396, sum=2.349 (6)\", \"tab\": \"Bias\", \"score\": \"0.3914278177516011\"}", - "CNN/DailyMail - Representation (race)": "{\"description\": \"min=0.257, mean=0.302, max=0.354, sum=1.811 (6)\", \"tab\": \"Bias\", \"score\": \"0.3019033965877131\"}", - "CNN/DailyMail - Representation (gender)": "{\"description\": \"min=0.135, mean=0.142, max=0.152, sum=0.851 (6)\", \"tab\": \"Bias\", \"score\": \"0.14183552076259287\"}", - "CNN/DailyMail - Toxic fraction": "{\"description\": \"min=0, mean=0.001, max=0.004, sum=0.009 (6)\", \"tab\": \"Toxicity\", \"score\": \"0.001430615164520744\"}", - "CNN/DailyMail - SummaC": "{\"description\": \"min=0.488, mean=0.512, max=0.535, sum=1.537 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.5121705493530246\"}", - "CNN/DailyMail - QAFactEval": "{\"description\": \"min=4.664, mean=4.716, max=4.749, sum=28.295 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"4.715823146970394\"}", - "CNN/DailyMail - BERTScore (F1)": "{\"description\": \"min=0.229, mean=0.248, max=0.272, sum=0.745 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.2482954175661162\"}", - "CNN/DailyMail - Coverage": "{\"description\": \"min=0.971, mean=0.977, max=0.985, sum=5.861 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"0.9768840440430324\"}", - "CNN/DailyMail - Density": "{\"description\": \"min=55.528, mean=71.654, max=97.831, sum=429.924 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"71.65405587945487\"}", - "CNN/DailyMail - Compression": "{\"description\": \"min=5.872, mean=7.632, max=9.373, sum=45.79 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"7.631709472598792\"}", - "CNN/DailyMail - HumanEval-faithfulness": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-relevance": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-coherence": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "XSUM", - "source_data": { - "dataset_name": "XSUM", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on XSUM", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.102, - "details": { - "description": "min=0.095, mean=0.102, max=0.107, sum=0.612 (6)", - "tab": "Accuracy", - "XSUM - Denoised inference time (s)": "{\"description\": \"min=0.896, mean=0.903, max=0.91, sum=5.418 (6)\", \"tab\": \"Efficiency\", \"score\": \"0.9030293349990619\"}", - "XSUM - # eval": "{\"description\": \"min=518, mean=518, max=518, sum=3108 (6)\", \"tab\": \"General information\", \"score\": \"518.0\"}", - "XSUM - # train": "{\"description\": \"min=5, mean=5, max=5, sum=30 (6)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "XSUM - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (6)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "XSUM - # prompt tokens": "{\"description\": \"min=1099.388, mean=1133.388, max=1172.388, sum=6800.328 (6)\", \"tab\": \"General information\", \"score\": \"1133.388030888031\"}", - "XSUM - # output tokens": "{\"description\": \"min=20.832, mean=21.299, max=21.809, sum=127.792 (6)\", \"tab\": \"General information\", \"score\": \"21.2985842985843\"}", - "XSUM - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=18 (6)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "XSUM - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=4.0 (6)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666669\"}", - "XSUM - Stereotypes (gender)": "{\"description\": \"min=0.397, mean=0.424, max=0.451, sum=2.547 (6)\", \"tab\": \"Bias\", \"score\": \"0.42449478248089356\"}", - "XSUM - Representation (race)": "{\"description\": \"min=0.387, mean=0.426, max=0.467, sum=2.554 (6)\", \"tab\": \"Bias\", \"score\": \"0.4255855855855855\"}", - "XSUM - Representation (gender)": "{\"description\": \"min=0.151, mean=0.172, max=0.189, sum=1.031 (6)\", \"tab\": \"Bias\", \"score\": \"0.1717873516720604\"}", - "XSUM - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (6)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}", - "XSUM - SummaC": "{\"description\": \"min=-0.26, mean=-0.239, max=-0.222, sum=-0.716 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"-0.23866760351278402\"}", - "XSUM - QAFactEval": "{\"description\": \"min=3.354, mean=3.675, max=4.009, sum=22.047 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"3.674546888395078\"}", - "XSUM - BERTScore (F1)": "{\"description\": \"min=0.393, mean=0.4, max=0.405, sum=1.2 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.40004604044843806\"}", - "XSUM - Coverage": "{\"description\": \"min=0.804, mean=0.808, max=0.813, sum=4.85 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"0.8084128334077892\"}", - "XSUM - Density": "{\"description\": \"min=3.618, mean=3.757, max=3.939, sum=22.541 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"3.7567632334705046\"}", - "XSUM - Compression": "{\"description\": \"min=17.523, mean=18.133, max=18.761, sum=108.8 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"18.133322572088453\"}", - "XSUM - HumanEval-faithfulness": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-relevance": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-coherence": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "IMDB", - "source_data": { - "dataset_name": "IMDB", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on IMDB", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.956, - "details": { - "description": "min=0.951, mean=0.956, max=0.962, sum=2.869 (3)", - "tab": "Accuracy", - "IMDB - ECE (10-bin)": "{\"description\": \"min=0.196, mean=0.213, max=0.234, sum=0.639 (3)\", \"tab\": \"Calibration\", \"score\": \"0.21314336064172376\"}", - "IMDB - EM (Robustness)": "{\"description\": \"min=0.927, mean=0.932, max=0.936, sum=2.796 (3)\", \"tab\": \"Robustness\", \"score\": \"0.932\"}", - "IMDB - EM (Fairness)": "{\"description\": \"min=0.939, mean=0.946, max=0.951, sum=2.839 (3)\", \"tab\": \"Fairness\", \"score\": \"0.9463333333333334\"}", - "IMDB - Denoised inference time (s)": "{\"description\": \"min=0.531, mean=0.637, max=0.757, sum=1.911 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.6371184251302079\"}", - "IMDB - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "IMDB - # train": "{\"description\": \"min=4.915, mean=4.972, max=5, sum=14.915 (3)\", \"tab\": \"General information\", \"score\": \"4.971666666666667\"}", - "IMDB - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IMDB - # prompt tokens": "{\"description\": \"min=853.851, mean=1281.577, max=1725.03, sum=3844.732 (3)\", \"tab\": \"General information\", \"score\": \"1281.5773333333334\"}", - "IMDB - # output tokens": "{\"description\": \"min=2, mean=2, max=2, sum=6 (3)\", \"tab\": \"General information\", \"score\": \"2.0\"}", - "IMDB - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "IMDB - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CivilComments", - "source_data": { - "dataset_name": "CivilComments", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on CivilComments", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.532, - "details": { - "description": "min=0, mean=0.532, max=0.996, sum=28.713 (54)", - "tab": "Accuracy", - "CivilComments - ECE (10-bin)": "{\"description\": \"min=0.073, mean=0.377, max=0.573, sum=20.347 (54)\", \"tab\": \"Calibration\", \"score\": \"0.37680252478263027\"}", - "CivilComments - EM (Robustness)": "{\"description\": \"min=0, mean=0.444, max=0.984, sum=23.966 (54)\", \"tab\": \"Robustness\", \"score\": \"0.4438230435194026\"}", - "CivilComments - EM (Fairness)": "{\"description\": \"min=0, mean=0.447, max=0.962, sum=24.127 (54)\", \"tab\": \"Fairness\", \"score\": \"0.4468037461427085\"}", - "CivilComments - Denoised inference time (s)": "{\"description\": \"min=0.338, mean=0.434, max=0.564, sum=23.454 (54)\", \"tab\": \"Efficiency\", \"score\": \"0.43432643222557377\"}", - "CivilComments - # eval": "{\"description\": \"min=74, mean=371.556, max=683, sum=20064 (54)\", \"tab\": \"General information\", \"score\": \"371.55555555555554\"}", - "CivilComments - # train": "{\"description\": \"min=5, mean=5, max=5, sum=270 (54)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CivilComments - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (54)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CivilComments - # prompt tokens": "{\"description\": \"min=271.927, mean=532.602, max=942.498, sum=28760.487 (54)\", \"tab\": \"General information\", \"score\": \"532.6016121330534\"}", - "CivilComments - # output tokens": "{\"description\": \"min=2, mean=2, max=2, sum=108 (54)\", \"tab\": \"General information\", \"score\": \"2.0\"}", - "CivilComments - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=162 (54)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "CivilComments - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (54)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "RAFT", - "source_data": { - "dataset_name": "RAFT", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on RAFT", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.545, - "details": { - "description": "min=0.15, mean=0.545, max=0.95, sum=18 (33)", - "tab": "Accuracy", - "RAFT - ECE (10-bin)": "{\"description\": \"min=0.134, mean=0.269, max=0.513, sum=8.875 (33)\", \"tab\": \"Calibration\", \"score\": \"0.2689468403025133\"}", - "RAFT - EM (Robustness)": "{\"description\": \"min=0.025, mean=0.443, max=0.95, sum=14.625 (33)\", \"tab\": \"Robustness\", \"score\": \"0.4431818181818182\"}", - "RAFT - EM (Fairness)": "{\"description\": \"min=0.15, mean=0.511, max=0.95, sum=16.85 (33)\", \"tab\": \"Fairness\", \"score\": \"0.5106060606060605\"}", - "RAFT - Denoised inference time (s)": "{\"description\": \"min=0.312, mean=0.499, max=0.763, sum=16.476 (33)\", \"tab\": \"Efficiency\", \"score\": \"0.4992617404513889\"}", - "RAFT - # eval": "{\"description\": \"min=40, mean=40, max=40, sum=1320 (33)\", \"tab\": \"General information\", \"score\": \"40.0\"}", - "RAFT - # train": "{\"description\": \"min=0.95, mean=4.658, max=5, sum=153.7 (33)\", \"tab\": \"General information\", \"score\": \"4.657575757575757\"}", - "RAFT - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (33)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "RAFT - # prompt tokens": "{\"description\": \"min=212.25, mean=712.248, max=1745.25, sum=23504.175 (33)\", \"tab\": \"General information\", \"score\": \"712.2477272727273\"}", - "RAFT - # output tokens": "{\"description\": \"min=1.975, mean=3.499, max=7.025, sum=115.475 (33)\", \"tab\": \"General information\", \"score\": \"3.4992424242424245\"}", - "RAFT - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=99 (33)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "RAFT - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (33)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/ai21_j2-grande.json b/data/models/ai21_j2-grande.json deleted file mode 100644 index 50669f1a714a53948942304cbbc49c9ad4ff3fe0..0000000000000000000000000000000000000000 --- a/data/models/ai21_j2-grande.json +++ /dev/null @@ -1,380 +0,0 @@ -{ - "model_info": { - "name": "Jurassic-2 Grande 17B", - "id": "ai21/j2-grande", - "developer": "ai21", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_lite/ai21_j2-grande/1774096306.427425", - "retrieved_timestamp": "1774096306.427425", - "source_metadata": { - "source_name": "helm_lite", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_lite", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_lite", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.172, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.39915106117353305\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.744, - "details": { - "description": "min=0.744, mean=0.744, max=0.744, sum=0.744 (1)", - "tab": "Accuracy", - "NarrativeQA - Observed inference time (s)": "{\"description\": \"min=1.179, mean=1.179, max=1.179, sum=1.179 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.1790085772393455\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=3.225, mean=3.225, max=3.225, sum=3.225 (1)\", \"tab\": \"General information\", \"score\": \"3.2253521126760565\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=1700.741, mean=1700.741, max=1700.741, sum=1700.741 (1)\", \"tab\": \"General information\", \"score\": \"1700.7408450704224\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=5.039, mean=5.039, max=5.039, sum=5.039 (1)\", \"tab\": \"General information\", \"score\": \"5.03943661971831\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (closed-book)", - "source_data": { - "dataset_name": "NaturalQuestions (closed-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (closed-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.35, - "details": { - "description": "min=0.35, mean=0.35, max=0.35, sum=0.35 (1)", - "tab": "Accuracy", - "NaturalQuestions (open-book) - Observed inference time (s)": "{\"description\": \"min=1.462, mean=1.462, max=1.462, sum=1.462 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.4618877012729645\"}", - "NaturalQuestions (closed-book) - Observed inference time (s)": "{\"description\": \"min=0.631, mean=0.631, max=0.631, sum=0.631 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.630548656463623\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.697, mean=4.697, max=4.697, sum=4.697 (1)\", \"tab\": \"General information\", \"score\": \"4.697\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.038, mean=0.038, max=0.038, sum=0.038 (1)\", \"tab\": \"General information\", \"score\": \"0.038\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1522.929, mean=1522.929, max=1522.929, sum=1522.929 (1)\", \"tab\": \"General information\", \"score\": \"1522.929\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=5.441, mean=5.441, max=5.441, sum=5.441 (1)\", \"tab\": \"General information\", \"score\": \"5.441\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=102.377, mean=102.377, max=102.377, sum=102.377 (1)\", \"tab\": \"General information\", \"score\": \"102.377\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=6.614, mean=6.614, max=6.614, sum=6.614 (1)\", \"tab\": \"General information\", \"score\": \"6.614\"}" - } - }, - "generation_config": { - "additional_details": { - "mode": "\"closedbook\"" - } - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.614, - "details": { - "description": "min=0.614, mean=0.614, max=0.614, sum=0.614 (1)", - "tab": "Accuracy", - "OpenbookQA - Observed inference time (s)": "{\"description\": \"min=0.519, mean=0.519, max=0.519, sum=0.519 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.519375147819519\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=188.75, mean=188.75, max=188.75, sum=188.75 (1)\", \"tab\": \"General information\", \"score\": \"188.75\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "dataset": "\"openbookqa\"", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.471, - "details": { - "description": "min=0.25, mean=0.471, max=0.77, sum=2.355 (5)", - "tab": "Accuracy", - "MMLU - Observed inference time (s)": "{\"description\": \"min=0.549, mean=0.621, max=0.755, sum=3.103 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.6205235414421348\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=308.59, mean=396.74, max=552.719, sum=1983.699 (5)\", \"tab\": \"General information\", \"score\": \"396.7398596491228\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"college_chemistry\", \"computer_security\", \"econometrics\", \"us_foreign_policy\"]", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MATH", - "source_data": { - "dataset_name": "MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Equivalent (CoT) on MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.064, - "details": { - "description": "min=0, mean=0.064, max=0.158, sum=0.445 (7)", - "tab": "Accuracy", - "MATH - Observed inference time (s)": "{\"description\": \"min=2.609, mean=4.862, max=6.298, sum=34.036 (7)\", \"tab\": \"Efficiency\", \"score\": \"4.862255273244342\"}", - "MATH - # eval": "{\"description\": \"min=30, mean=62.429, max=135, sum=437 (7)\", \"tab\": \"General information\", \"score\": \"62.42857142857143\"}", - "MATH - # train": "{\"description\": \"min=2, mean=6.778, max=8, sum=47.447 (7)\", \"tab\": \"General information\", \"score\": \"6.7781954887218046\"}", - "MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MATH - # prompt tokens": "{\"description\": \"min=450.154, mean=943.419, max=1490.395, sum=6603.93 (7)\", \"tab\": \"General information\", \"score\": \"943.4185034241337\"}", - "MATH - # output tokens": "{\"description\": \"min=74.123, mean=140.295, max=209.933, sum=982.063 (7)\", \"tab\": \"General information\", \"score\": \"140.29469320289397\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"algebra\", \"counting_and_probability\", \"geometry\", \"intermediate_algebra\", \"number_theory\", \"prealgebra\", \"precalculus\"]", - "level": "\"1\"", - "use_official_examples": "\"False\"", - "use_chain_of_thought": "\"True\"" - } - } - }, - { - "evaluation_name": "GSM8K", - "source_data": { - "dataset_name": "GSM8K", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on GSM8K", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.159, - "details": { - "description": "min=0.159, mean=0.159, max=0.159, sum=0.159 (1)", - "tab": "Accuracy", - "GSM8K - Observed inference time (s)": "{\"description\": \"min=5.417, mean=5.417, max=5.417, sum=5.417 (1)\", \"tab\": \"Efficiency\", \"score\": \"5.417125414848328\"}", - "GSM8K - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "GSM8K - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "GSM8K - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GSM8K - # prompt tokens": "{\"description\": \"min=823.394, mean=823.394, max=823.394, sum=823.394 (1)\", \"tab\": \"General information\", \"score\": \"823.394\"}", - "GSM8K - # output tokens": "{\"description\": \"min=121.336, mean=121.336, max=121.336, sum=121.336 (1)\", \"tab\": \"General information\", \"score\": \"121.336\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "LegalBench", - "source_data": { - "dataset_name": "LegalBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on LegalBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.468, - "details": { - "description": "min=0.199, mean=0.468, max=0.842, sum=2.338 (5)", - "tab": "Accuracy", - "LegalBench - Observed inference time (s)": "{\"description\": \"min=0.409, mean=0.712, max=1.079, sum=3.561 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.7122931517101486\"}", - "LegalBench - # eval": "{\"description\": \"min=95, mean=409.4, max=1000, sum=2047 (5)\", \"tab\": \"General information\", \"score\": \"409.4\"}", - "LegalBench - # train": "{\"description\": \"min=1.006, mean=4.001, max=5, sum=20.006 (5)\", \"tab\": \"General information\", \"score\": \"4.001224489795918\"}", - "LegalBench - truncated": "{\"description\": \"min=0, mean=0.002, max=0.012, sum=0.012 (5)\", \"tab\": \"General information\", \"score\": \"0.0024489795918367346\"}", - "LegalBench - # prompt tokens": "{\"description\": \"min=171.042, mean=503.146, max=1514.22, sum=2515.73 (5)\", \"tab\": \"General information\", \"score\": \"503.1459259177527\"}", - "LegalBench - # output tokens": "{\"description\": \"min=2, mean=2.056, max=2.216, sum=10.282 (5)\", \"tab\": \"General information\", \"score\": \"2.0563001835066452\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"abercrombie\", \"corporate_lobbying\", \"function_of_decision_section\", \"international_citizenship_questions\", \"proa\"]" - } - } - }, - { - "evaluation_name": "MedQA", - "source_data": { - "dataset_name": "MedQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MedQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.39, - "details": { - "description": "min=0.39, mean=0.39, max=0.39, sum=0.39 (1)", - "tab": "Accuracy", - "MedQA - Observed inference time (s)": "{\"description\": \"min=0.914, mean=0.914, max=0.914, sum=0.914 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.9142626611660299\"}", - "MedQA - # eval": "{\"description\": \"min=503, mean=503, max=503, sum=503 (1)\", \"tab\": \"General information\", \"score\": \"503.0\"}", - "MedQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MedQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MedQA - # prompt tokens": "{\"description\": \"min=758.622, mean=758.622, max=758.622, sum=758.622 (1)\", \"tab\": \"General information\", \"score\": \"758.6222664015904\"}", - "MedQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WMT 2014", - "source_data": { - "dataset_name": "WMT 2014", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "BLEU-4 on WMT 2014", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.102, - "details": { - "description": "min=0.021, mean=0.102, max=0.149, sum=0.509 (5)", - "tab": "Accuracy", - "WMT 2014 - Observed inference time (s)": "{\"description\": \"min=0.723, mean=0.759, max=0.81, sum=3.793 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.7586197336965614\"}", - "WMT 2014 - # eval": "{\"description\": \"min=503, mean=568.8, max=832, sum=2844 (5)\", \"tab\": \"General information\", \"score\": \"568.8\"}", - "WMT 2014 - # train": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "WMT 2014 - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WMT 2014 - # prompt tokens": "{\"description\": \"min=123.229, mean=135.468, max=148.278, sum=677.341 (5)\", \"tab\": \"General information\", \"score\": \"135.46828404572565\"}", - "WMT 2014 - # output tokens": "{\"description\": \"min=17.372, mean=19.051, max=21.34, sum=95.255 (5)\", \"tab\": \"General information\", \"score\": \"19.050931430646887\"}" - } - }, - "generation_config": { - "additional_details": { - "language_pair": "[\"cs-en\", \"de-en\", \"fr-en\", \"hi-en\", \"ru-en\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/ai21_j2-jumbo.json b/data/models/ai21_j2-jumbo.json deleted file mode 100644 index 1ad8387eed08f1228777b4472ab01e9c42a6713b..0000000000000000000000000000000000000000 --- a/data/models/ai21_j2-jumbo.json +++ /dev/null @@ -1,380 +0,0 @@ -{ - "model_info": { - "name": "Jurassic-2 Jumbo 178B", - "id": "ai21/j2-jumbo", - "developer": "ai21", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_lite/ai21_j2-jumbo/1774096306.427425", - "retrieved_timestamp": "1774096306.427425", - "source_metadata": { - "source_name": "helm_lite", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_lite", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_lite", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.215, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.19473158551810238\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.728, - "details": { - "description": "min=0.728, mean=0.728, max=0.728, sum=0.728 (1)", - "tab": "Accuracy", - "NarrativeQA - Observed inference time (s)": "{\"description\": \"min=1.82, mean=1.82, max=1.82, sum=1.82 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.8203622415032186\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=2534.434, mean=2534.434, max=2534.434, sum=2534.434 (1)\", \"tab\": \"General information\", \"score\": \"2534.4338028169013\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=6.583, mean=6.583, max=6.583, sum=6.583 (1)\", \"tab\": \"General information\", \"score\": \"6.583098591549295\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (closed-book)", - "source_data": { - "dataset_name": "NaturalQuestions (closed-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (closed-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.385, - "details": { - "description": "min=0.385, mean=0.385, max=0.385, sum=0.385 (1)", - "tab": "Accuracy", - "NaturalQuestions (open-book) - Observed inference time (s)": "{\"description\": \"min=1.448, mean=1.448, max=1.448, sum=1.448 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.4479399914741515\"}", - "NaturalQuestions (closed-book) - Observed inference time (s)": "{\"description\": \"min=5.332, mean=5.332, max=5.332, sum=5.332 (1)\", \"tab\": \"Efficiency\", \"score\": \"5.3321147253513335\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.931, mean=4.931, max=4.931, sum=4.931 (1)\", \"tab\": \"General information\", \"score\": \"4.931\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.012, mean=0.012, max=0.012, sum=0.012 (1)\", \"tab\": \"General information\", \"score\": \"0.012\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1687.673, mean=1687.673, max=1687.673, sum=1687.673 (1)\", \"tab\": \"General information\", \"score\": \"1687.673\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=4.785, mean=4.785, max=4.785, sum=4.785 (1)\", \"tab\": \"General information\", \"score\": \"4.785\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=102.377, mean=102.377, max=102.377, sum=102.377 (1)\", \"tab\": \"General information\", \"score\": \"102.377\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=5.79, mean=5.79, max=5.79, sum=5.79 (1)\", \"tab\": \"General information\", \"score\": \"5.79\"}" - } - }, - "generation_config": { - "additional_details": { - "mode": "\"closedbook\"" - } - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.688, - "details": { - "description": "min=0.688, mean=0.688, max=0.688, sum=0.688 (1)", - "tab": "Accuracy", - "OpenbookQA - Observed inference time (s)": "{\"description\": \"min=0.998, mean=0.998, max=0.998, sum=0.998 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.9981746392250062\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=188.75, mean=188.75, max=188.75, sum=188.75 (1)\", \"tab\": \"General information\", \"score\": \"188.75\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "dataset": "\"openbookqa\"", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.483, - "details": { - "description": "min=0.25, mean=0.483, max=0.83, sum=2.413 (5)", - "tab": "Accuracy", - "MMLU - Observed inference time (s)": "{\"description\": \"min=0.693, mean=0.81, max=0.92, sum=4.052 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.8103257050430566\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=308.59, mean=396.74, max=552.719, sum=1983.699 (5)\", \"tab\": \"General information\", \"score\": \"396.7398596491228\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"college_chemistry\", \"computer_security\", \"econometrics\", \"us_foreign_policy\"]", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MATH", - "source_data": { - "dataset_name": "MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Equivalent (CoT) on MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.103, - "details": { - "description": "min=0.033, mean=0.103, max=0.193, sum=0.72 (7)", - "tab": "Accuracy", - "MATH - Observed inference time (s)": "{\"description\": \"min=4.497, mean=9.136, max=13.531, sum=63.951 (7)\", \"tab\": \"Efficiency\", \"score\": \"9.135811412885502\"}", - "MATH - # eval": "{\"description\": \"min=30, mean=62.429, max=135, sum=437 (7)\", \"tab\": \"General information\", \"score\": \"62.42857142857143\"}", - "MATH - # train": "{\"description\": \"min=8, mean=8, max=8, sum=56 (7)\", \"tab\": \"General information\", \"score\": \"8.0\"}", - "MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MATH - # prompt tokens": "{\"description\": \"min=796.795, mean=1321.422, max=2516.154, sum=9249.956 (7)\", \"tab\": \"General information\", \"score\": \"1321.42226282263\"}", - "MATH - # output tokens": "{\"description\": \"min=76.281, mean=136.538, max=220.133, sum=955.767 (7)\", \"tab\": \"General information\", \"score\": \"136.53809167621895\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"algebra\", \"counting_and_probability\", \"geometry\", \"intermediate_algebra\", \"number_theory\", \"prealgebra\", \"precalculus\"]", - "level": "\"1\"", - "use_official_examples": "\"False\"", - "use_chain_of_thought": "\"True\"" - } - } - }, - { - "evaluation_name": "GSM8K", - "source_data": { - "dataset_name": "GSM8K", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on GSM8K", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.239, - "details": { - "description": "min=0.239, mean=0.239, max=0.239, sum=0.239 (1)", - "tab": "Accuracy", - "GSM8K - Observed inference time (s)": "{\"description\": \"min=5.176, mean=5.176, max=5.176, sum=5.176 (1)\", \"tab\": \"Efficiency\", \"score\": \"5.176425676584244\"}", - "GSM8K - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "GSM8K - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "GSM8K - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GSM8K - # prompt tokens": "{\"description\": \"min=823.394, mean=823.394, max=823.394, sum=823.394 (1)\", \"tab\": \"General information\", \"score\": \"823.394\"}", - "GSM8K - # output tokens": "{\"description\": \"min=102.036, mean=102.036, max=102.036, sum=102.036 (1)\", \"tab\": \"General information\", \"score\": \"102.036\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "LegalBench", - "source_data": { - "dataset_name": "LegalBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on LegalBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.533, - "details": { - "description": "min=0.324, mean=0.533, max=0.821, sum=2.666 (5)", - "tab": "Accuracy", - "LegalBench - Observed inference time (s)": "{\"description\": \"min=0.639, mean=1.274, max=2.827, sum=6.369 (5)\", \"tab\": \"Efficiency\", \"score\": \"1.2737073742826783\"}", - "LegalBench - # eval": "{\"description\": \"min=95, mean=409.4, max=1000, sum=2047 (5)\", \"tab\": \"General information\", \"score\": \"409.4\"}", - "LegalBench - # train": "{\"description\": \"min=4, mean=4.798, max=5, sum=23.992 (5)\", \"tab\": \"General information\", \"score\": \"4.798367346938775\"}", - "LegalBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "LegalBench - # prompt tokens": "{\"description\": \"min=171.042, mean=1120.486, max=4600.92, sum=5602.43 (5)\", \"tab\": \"General information\", \"score\": \"1120.4859259177529\"}", - "LegalBench - # output tokens": "{\"description\": \"min=2, mean=2.028, max=2.098, sum=10.141 (5)\", \"tab\": \"General information\", \"score\": \"2.028218528610354\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"abercrombie\", \"corporate_lobbying\", \"function_of_decision_section\", \"international_citizenship_questions\", \"proa\"]" - } - } - }, - { - "evaluation_name": "MedQA", - "source_data": { - "dataset_name": "MedQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MedQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.431, - "details": { - "description": "min=0.431, mean=0.431, max=0.431, sum=0.431 (1)", - "tab": "Accuracy", - "MedQA - Observed inference time (s)": "{\"description\": \"min=1.535, mean=1.535, max=1.535, sum=1.535 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.5350148075854566\"}", - "MedQA - # eval": "{\"description\": \"min=503, mean=503, max=503, sum=503 (1)\", \"tab\": \"General information\", \"score\": \"503.0\"}", - "MedQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MedQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MedQA - # prompt tokens": "{\"description\": \"min=758.622, mean=758.622, max=758.622, sum=758.622 (1)\", \"tab\": \"General information\", \"score\": \"758.6222664015904\"}", - "MedQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WMT 2014", - "source_data": { - "dataset_name": "WMT 2014", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "BLEU-4 on WMT 2014", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.114, - "details": { - "description": "min=0.044, mean=0.114, max=0.148, sum=0.572 (5)", - "tab": "Accuracy", - "WMT 2014 - Observed inference time (s)": "{\"description\": \"min=1.236, mean=1.441, max=1.665, sum=7.206 (5)\", \"tab\": \"Efficiency\", \"score\": \"1.4411698855373092\"}", - "WMT 2014 - # eval": "{\"description\": \"min=503, mean=568.8, max=832, sum=2844 (5)\", \"tab\": \"General information\", \"score\": \"568.8\"}", - "WMT 2014 - # train": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "WMT 2014 - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WMT 2014 - # prompt tokens": "{\"description\": \"min=123.229, mean=135.468, max=148.278, sum=677.341 (5)\", \"tab\": \"General information\", \"score\": \"135.46828404572565\"}", - "WMT 2014 - # output tokens": "{\"description\": \"min=19.839, mean=24.063, max=30.439, sum=120.314 (5)\", \"tab\": \"General information\", \"score\": \"24.062830708059337\"}" - } - }, - "generation_config": { - "additional_details": { - "language_pair": "[\"cs-en\", \"de-en\", \"fr-en\", \"hi-en\", \"ru-en\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/ai21_jamba-1.5-large.json b/data/models/ai21_jamba-1.5-large.json deleted file mode 100644 index 66574989b889ea423fe210102a22cb1355958611..0000000000000000000000000000000000000000 --- a/data/models/ai21_jamba-1.5-large.json +++ /dev/null @@ -1,1903 +0,0 @@ -{ - "model_info": { - "name": "Jamba 1.5 Large", - "id": "ai21/jamba-1.5-large", - "developer": "ai21", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_lite/ai21_jamba-1.5-large/1774096306.427425", - "retrieved_timestamp": "1774096306.427425", - "source_metadata": { - "source_name": "helm_lite", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_lite", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_lite", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.637, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.26377028714107364\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.664, - "details": { - "description": "min=0.664, mean=0.664, max=0.664, sum=0.664 (1)", - "tab": "Accuracy", - "NarrativeQA - Observed inference time (s)": "{\"description\": \"min=1.969, mean=1.969, max=1.969, sum=1.969 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.9694313982842673\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=3595.597, mean=3595.597, max=3595.597, sum=3595.597 (1)\", \"tab\": \"General information\", \"score\": \"3595.5971830985914\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (closed-book)", - "source_data": { - "dataset_name": "NaturalQuestions (closed-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (closed-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.394, - "details": { - "description": "min=0.394, mean=0.394, max=0.394, sum=0.394 (1)", - "tab": "Accuracy", - "NaturalQuestions (open-book) - Observed inference time (s)": "{\"description\": \"min=1.678, mean=1.678, max=1.678, sum=1.678 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.678127991437912\"}", - "NaturalQuestions (closed-book) - Observed inference time (s)": "{\"description\": \"min=1.272, mean=1.272, max=1.272, sum=1.272 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.2717866213321687\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=2333.076, mean=2333.076, max=2333.076, sum=2333.076 (1)\", \"tab\": \"General information\", \"score\": \"2333.076\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=152.394, mean=152.394, max=152.394, sum=152.394 (1)\", \"tab\": \"General information\", \"score\": \"152.394\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "mode": "\"closedbook\"" - } - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.948, - "details": { - "description": "min=0.948, mean=0.948, max=0.948, sum=0.948 (1)", - "tab": "Accuracy", - "OpenbookQA - Observed inference time (s)": "{\"description\": \"min=0.91, mean=0.91, max=0.91, sum=0.91 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.9100792293548584\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=261.348, mean=261.348, max=261.348, sum=261.348 (1)\", \"tab\": \"General information\", \"score\": \"261.348\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "dataset": "\"openbookqa\"", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.683, - "details": { - "description": "min=0.53, mean=0.683, max=0.92, sum=3.414 (5)", - "tab": "Accuracy", - "MMLU - Observed inference time (s)": "{\"description\": \"min=0.933, mean=0.973, max=1.0, sum=4.866 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.973254363085094\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=397.58, mean=508.138, max=678.64, sum=2540.69 (5)\", \"tab\": \"General information\", \"score\": \"508.1380701754386\"}", - "MMLU - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"college_chemistry\", \"computer_security\", \"econometrics\", \"us_foreign_policy\"]", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MATH", - "source_data": { - "dataset_name": "MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Equivalent (CoT) on MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.692, - "details": { - "description": "min=0.481, mean=0.692, max=0.889, sum=4.842 (7)", - "tab": "Accuracy", - "MATH - Observed inference time (s)": "{\"description\": \"min=2.366, mean=3.179, max=4.736, sum=22.253 (7)\", \"tab\": \"Efficiency\", \"score\": \"3.1790229759699775\"}", - "MATH - # eval": "{\"description\": \"min=30, mean=62.429, max=135, sum=437 (7)\", \"tab\": \"General information\", \"score\": \"62.42857142857143\"}", - "MATH - # train": "{\"description\": \"min=8, mean=8, max=8, sum=56 (7)\", \"tab\": \"General information\", \"score\": \"8.0\"}", - "MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MATH - # prompt tokens": "{\"description\": \"min=979.415, mean=1458.376, max=2550.115, sum=10208.634 (7)\", \"tab\": \"General information\", \"score\": \"1458.376275861588\"}", - "MATH - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"algebra\", \"counting_and_probability\", \"geometry\", \"intermediate_algebra\", \"number_theory\", \"prealgebra\", \"precalculus\"]", - "level": "\"1\"", - "use_official_examples": "\"False\"", - "use_chain_of_thought": "\"True\"" - } - } - }, - { - "evaluation_name": "GSM8K", - "source_data": { - "dataset_name": "GSM8K", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on GSM8K", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.846, - "details": { - "description": "min=0.846, mean=0.846, max=0.846, sum=0.846 (1)", - "tab": "Accuracy", - "GSM8K - Observed inference time (s)": "{\"description\": \"min=3.942, mean=3.942, max=3.942, sum=3.942 (1)\", \"tab\": \"Efficiency\", \"score\": \"3.942030364751816\"}", - "GSM8K - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "GSM8K - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "GSM8K - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GSM8K - # prompt tokens": "{\"description\": \"min=1163.818, mean=1163.818, max=1163.818, sum=1163.818 (1)\", \"tab\": \"General information\", \"score\": \"1163.818\"}", - "GSM8K - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "stop": "\"none\"" - } - } - }, - { - "evaluation_name": "LegalBench", - "source_data": { - "dataset_name": "LegalBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on LegalBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.675, - "details": { - "description": "min=0.409, mean=0.675, max=0.989, sum=3.375 (5)", - "tab": "Accuracy", - "LegalBench - Observed inference time (s)": "{\"description\": \"min=0.933, mean=1.258, max=2.367, sum=6.289 (5)\", \"tab\": \"Efficiency\", \"score\": \"1.2577736545740559\"}", - "LegalBench - # eval": "{\"description\": \"min=95, mean=409.4, max=1000, sum=2047 (5)\", \"tab\": \"General information\", \"score\": \"409.4\"}", - "LegalBench - # train": "{\"description\": \"min=4, mean=4.8, max=5, sum=24 (5)\", \"tab\": \"General information\", \"score\": \"4.8\"}", - "LegalBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "LegalBench - # prompt tokens": "{\"description\": \"min=212.453, mean=1601.843, max=6618.612, sum=8009.215 (5)\", \"tab\": \"General information\", \"score\": \"1601.842950915631\"}", - "LegalBench - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"abercrombie\", \"corporate_lobbying\", \"function_of_decision_section\", \"international_citizenship_questions\", \"proa\"]" - } - } - }, - { - "evaluation_name": "MedQA", - "source_data": { - "dataset_name": "MedQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MedQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.698, - "details": { - "description": "min=0.698, mean=0.698, max=0.698, sum=0.698 (1)", - "tab": "Accuracy", - "MedQA - Observed inference time (s)": "{\"description\": \"min=0.999, mean=0.999, max=0.999, sum=0.999 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.9989562840395372\"}", - "MedQA - # eval": "{\"description\": \"min=503, mean=503, max=503, sum=503 (1)\", \"tab\": \"General information\", \"score\": \"503.0\"}", - "MedQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MedQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MedQA - # prompt tokens": "{\"description\": \"min=1085.239, mean=1085.239, max=1085.239, sum=1085.239 (1)\", \"tab\": \"General information\", \"score\": \"1085.2385685884692\"}", - "MedQA - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WMT 2014", - "source_data": { - "dataset_name": "WMT 2014", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "BLEU-4 on WMT 2014", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.203, - "details": { - "description": "min=0.141, mean=0.203, max=0.246, sum=1.015 (5)", - "tab": "Accuracy", - "WMT 2014 - Observed inference time (s)": "{\"description\": \"min=1.317, mean=1.386, max=1.471, sum=6.93 (5)\", \"tab\": \"Efficiency\", \"score\": \"1.3859240114613673\"}", - "WMT 2014 - # eval": "{\"description\": \"min=503, mean=568.8, max=832, sum=2844 (5)\", \"tab\": \"General information\", \"score\": \"568.8\"}", - "WMT 2014 - # train": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "WMT 2014 - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WMT 2014 - # prompt tokens": "{\"description\": \"min=120.386, mean=151.077, max=189.223, sum=755.383 (5)\", \"tab\": \"General information\", \"score\": \"151.07662629989292\"}", - "WMT 2014 - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "language_pair": "[\"cs-en\", \"de-en\", \"fr-en\", \"hi-en\", \"ru-en\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_mmlu/ai21_jamba-1.5-large/1774096312.00548", - "retrieved_timestamp": "1774096312.00548", - "source_metadata": { - "source_name": "helm_mmlu", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_mmlu", - "evaluation_results": [ - { - "evaluation_name": "MMLU All Subjects", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU All Subjects", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.782, - "details": { - "description": "min=0.46, mean=0.782, max=0.969, sum=89.128 (114)", - "tab": "Accuracy", - "MMLU All Subjects - Observed inference time (s)": "{\"description\": \"min=0.889, mean=1.01, max=1.394, sum=115.088 (114)\", \"tab\": \"Efficiency\", \"score\": \"1.0095401397461812\"}", - "MMLU All Subjects - # eval": "{\"description\": \"min=100, mean=246.351, max=1534, sum=28084 (114)\", \"tab\": \"General information\", \"score\": \"246.35087719298247\"}", - "MMLU All Subjects - # train": "{\"description\": \"min=5, mean=5, max=5, sum=570 (114)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU All Subjects - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU All Subjects - # prompt tokens": "{\"description\": \"min=293.649, mean=658.432, max=2900.673, sum=75061.271 (114)\", \"tab\": \"General information\", \"score\": \"658.4322049384847\"}", - "MMLU All Subjects - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_name": "Abstract Algebra", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Abstract Algebra", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.53, - "details": { - "description": "min=0.53, mean=0.53, max=0.53, sum=1.06 (2)", - "tab": "Accuracy", - "Abstract Algebra - Observed inference time (s)": "{\"description\": \"min=0.933, mean=0.933, max=0.933, sum=1.865 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9326767182350159\"}", - "Abstract Algebra - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Abstract Algebra - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Abstract Algebra - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Abstract Algebra - # prompt tokens": "{\"description\": \"min=397.58, mean=397.58, max=397.58, sum=795.16 (2)\", \"tab\": \"General information\", \"score\": \"397.58\"}", - "Abstract Algebra - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"abstract_algebra\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_abstract_algebra\"" - } - } - }, - { - "evaluation_name": "Anatomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Anatomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.793, - "details": { - "description": "min=0.793, mean=0.793, max=0.793, sum=1.585 (2)", - "tab": "Accuracy", - "Anatomy - Observed inference time (s)": "{\"description\": \"min=0.889, mean=0.889, max=0.889, sum=1.777 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8885634528266059\"}", - "Anatomy - # eval": "{\"description\": \"min=135, mean=135, max=135, sum=270 (2)\", \"tab\": \"General information\", \"score\": \"135.0\"}", - "Anatomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Anatomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Anatomy - # prompt tokens": "{\"description\": \"min=376.741, mean=376.741, max=376.741, sum=753.481 (2)\", \"tab\": \"General information\", \"score\": \"376.74074074074076\"}", - "Anatomy - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"anatomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_anatomy\"" - } - } - }, - { - "evaluation_name": "College Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on College Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.51, - "details": { - "description": "min=0.51, mean=0.51, max=0.51, sum=1.02 (2)", - "tab": "Accuracy", - "College Chemistry - Observed inference time (s)": "{\"description\": \"min=0.971, mean=0.971, max=0.971, sum=1.942 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9710254788398742\"}", - "College Biology - Observed inference time (s)": "{\"description\": \"min=0.968, mean=0.968, max=0.968, sum=1.936 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.968123722407553\"}", - "College Computer Science - Observed inference time (s)": "{\"description\": \"min=0.986, mean=0.986, max=0.986, sum=1.973 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9862666988372802\"}", - "College Mathematics - Observed inference time (s)": "{\"description\": \"min=0.96, mean=0.96, max=0.96, sum=1.92 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9599522399902344\"}", - "College Medicine - Observed inference time (s)": "{\"description\": \"min=0.978, mean=0.978, max=0.978, sum=1.957 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9782800839815525\"}", - "College Physics - Observed inference time (s)": "{\"description\": \"min=1.01, mean=1.01, max=1.01, sum=2.019 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.0095638387343462\"}", - "College Chemistry - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Chemistry - # prompt tokens": "{\"description\": \"min=598.67, mean=598.67, max=598.67, sum=1197.34 (2)\", \"tab\": \"General information\", \"score\": \"598.67\"}", - "College Chemistry - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # eval": "{\"description\": \"min=144, mean=144, max=144, sum=288 (2)\", \"tab\": \"General information\", \"score\": \"144.0\"}", - "College Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # prompt tokens": "{\"description\": \"min=507.306, mean=507.306, max=507.306, sum=1014.611 (2)\", \"tab\": \"General information\", \"score\": \"507.30555555555554\"}", - "College Biology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # prompt tokens": "{\"description\": \"min=883.21, mean=883.21, max=883.21, sum=1766.42 (2)\", \"tab\": \"General information\", \"score\": \"883.21\"}", - "College Computer Science - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # prompt tokens": "{\"description\": \"min=643.97, mean=643.97, max=643.97, sum=1287.94 (2)\", \"tab\": \"General information\", \"score\": \"643.97\"}", - "College Mathematics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # eval": "{\"description\": \"min=173, mean=173, max=173, sum=346 (2)\", \"tab\": \"General information\", \"score\": \"173.0\"}", - "College Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # prompt tokens": "{\"description\": \"min=543.347, mean=543.347, max=543.347, sum=1086.694 (2)\", \"tab\": \"General information\", \"score\": \"543.3468208092486\"}", - "College Medicine - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # eval": "{\"description\": \"min=102, mean=102, max=102, sum=204 (2)\", \"tab\": \"General information\", \"score\": \"102.0\"}", - "College Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # prompt tokens": "{\"description\": \"min=533.402, mean=533.402, max=533.402, sum=1066.804 (2)\", \"tab\": \"General information\", \"score\": \"533.4019607843137\"}", - "College Physics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"college_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_college_physics\"" - } - } - }, - { - "evaluation_name": "Computer Security", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Computer Security", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8, - "details": { - "description": "min=0.8, mean=0.8, max=0.8, sum=1.6 (2)", - "tab": "Accuracy", - "Computer Security - Observed inference time (s)": "{\"description\": \"min=1.0, mean=1.0, max=1.0, sum=2.0 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.000160608291626\"}", - "Computer Security - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Computer Security - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Computer Security - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Computer Security - # prompt tokens": "{\"description\": \"min=404.27, mean=404.27, max=404.27, sum=808.54 (2)\", \"tab\": \"General information\", \"score\": \"404.27\"}", - "Computer Security - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"computer_security\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_computer_security\"" - } - } - }, - { - "evaluation_name": "Econometrics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Econometrics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.614, - "details": { - "description": "min=0.614, mean=0.614, max=0.614, sum=1.228 (2)", - "tab": "Accuracy", - "Econometrics - Observed inference time (s)": "{\"description\": \"min=0.971, mean=0.971, max=0.971, sum=1.942 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9712212587657728\"}", - "Econometrics - # eval": "{\"description\": \"min=114, mean=114, max=114, sum=228 (2)\", \"tab\": \"General information\", \"score\": \"114.0\"}", - "Econometrics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Econometrics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Econometrics - # prompt tokens": "{\"description\": \"min=678.64, mean=678.64, max=678.64, sum=1357.281 (2)\", \"tab\": \"General information\", \"score\": \"678.640350877193\"}", - "Econometrics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"econometrics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_econometrics\"" - } - } - }, - { - "evaluation_name": "Global Facts", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Global Facts", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.54, - "details": { - "description": "min=0.54, mean=0.54, max=0.54, sum=1.08 (2)", - "tab": "Accuracy", - "Global Facts - Observed inference time (s)": "{\"description\": \"min=0.951, mean=0.951, max=0.951, sum=1.901 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9506172919273377\"}", - "Global Facts - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Global Facts - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Global Facts - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Global Facts - # prompt tokens": "{\"description\": \"min=466.9, mean=466.9, max=466.9, sum=933.8 (2)\", \"tab\": \"General information\", \"score\": \"466.9\"}", - "Global Facts - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"global_facts\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_global_facts\"" - } - } - }, - { - "evaluation_name": "Jurisprudence", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Jurisprudence", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.87, - "details": { - "description": "min=0.87, mean=0.87, max=0.87, sum=1.741 (2)", - "tab": "Accuracy", - "Jurisprudence - Observed inference time (s)": "{\"description\": \"min=0.929, mean=0.929, max=0.929, sum=1.858 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9292316171858046\"}", - "Jurisprudence - # eval": "{\"description\": \"min=108, mean=108, max=108, sum=216 (2)\", \"tab\": \"General information\", \"score\": \"108.0\"}", - "Jurisprudence - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Jurisprudence - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Jurisprudence - # prompt tokens": "{\"description\": \"min=427.185, mean=427.185, max=427.185, sum=854.37 (2)\", \"tab\": \"General information\", \"score\": \"427.18518518518516\"}", - "Jurisprudence - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"jurisprudence\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_jurisprudence\"" - } - } - }, - { - "evaluation_name": "Philosophy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Philosophy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.849, - "details": { - "description": "min=0.849, mean=0.849, max=0.849, sum=1.698 (2)", - "tab": "Accuracy", - "Philosophy - Observed inference time (s)": "{\"description\": \"min=0.924, mean=0.924, max=0.924, sum=1.848 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9240530403480652\"}", - "Philosophy - # eval": "{\"description\": \"min=311, mean=311, max=311, sum=622 (2)\", \"tab\": \"General information\", \"score\": \"311.0\"}", - "Philosophy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Philosophy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Philosophy - # prompt tokens": "{\"description\": \"min=359.441, mean=359.441, max=359.441, sum=718.881 (2)\", \"tab\": \"General information\", \"score\": \"359.4405144694534\"}", - "Philosophy - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"philosophy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_philosophy\"" - } - } - }, - { - "evaluation_name": "Professional Psychology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Professional Psychology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.842, - "details": { - "description": "min=0.842, mean=0.842, max=0.842, sum=1.683 (2)", - "tab": "Accuracy", - "Professional Medicine - Observed inference time (s)": "{\"description\": \"min=1.025, mean=1.025, max=1.025, sum=2.05 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.0251652388011707\"}", - "Professional Accounting - Observed inference time (s)": "{\"description\": \"min=0.954, mean=0.954, max=0.954, sum=1.907 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9537228667144234\"}", - "Professional Law - Observed inference time (s)": "{\"description\": \"min=1.039, mean=1.039, max=1.039, sum=2.078 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.0390360032097767\"}", - "Professional Psychology - Observed inference time (s)": "{\"description\": \"min=0.959, mean=0.959, max=0.959, sum=1.918 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9592212933340883\"}", - "Professional Medicine - # eval": "{\"description\": \"min=272, mean=272, max=272, sum=544 (2)\", \"tab\": \"General information\", \"score\": \"272.0\"}", - "Professional Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Medicine - # prompt tokens": "{\"description\": \"min=1170.393, mean=1170.393, max=1170.393, sum=2340.787 (2)\", \"tab\": \"General information\", \"score\": \"1170.3933823529412\"}", - "Professional Medicine - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # eval": "{\"description\": \"min=282, mean=282, max=282, sum=564 (2)\", \"tab\": \"General information\", \"score\": \"282.0\"}", - "Professional Accounting - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Accounting - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # prompt tokens": "{\"description\": \"min=770.316, mean=770.316, max=770.316, sum=1540.631 (2)\", \"tab\": \"General information\", \"score\": \"770.3156028368794\"}", - "Professional Accounting - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # eval": "{\"description\": \"min=1534, mean=1534, max=1534, sum=3068 (2)\", \"tab\": \"General information\", \"score\": \"1534.0\"}", - "Professional Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # prompt tokens": "{\"description\": \"min=1725.955, mean=1725.955, max=1725.955, sum=3451.91 (2)\", \"tab\": \"General information\", \"score\": \"1725.9550195567144\"}", - "Professional Law - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # eval": "{\"description\": \"min=612, mean=612, max=612, sum=1224 (2)\", \"tab\": \"General information\", \"score\": \"612.0\"}", - "Professional Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # prompt tokens": "{\"description\": \"min=611.645, mean=611.645, max=611.645, sum=1223.291 (2)\", \"tab\": \"General information\", \"score\": \"611.6454248366013\"}", - "Professional Psychology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"professional_psychology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_professional_psychology\"" - } - } - }, - { - "evaluation_name": "Us Foreign Policy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Us Foreign Policy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.92, - "details": { - "description": "min=0.92, mean=0.92, max=0.92, sum=1.84 (2)", - "tab": "Accuracy", - "Us Foreign Policy - Observed inference time (s)": "{\"description\": \"min=0.991, mean=0.991, max=0.991, sum=1.982 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9911877512931824\"}", - "Us Foreign Policy - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Us Foreign Policy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Us Foreign Policy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Us Foreign Policy - # prompt tokens": "{\"description\": \"min=461.53, mean=461.53, max=461.53, sum=923.06 (2)\", \"tab\": \"General information\", \"score\": \"461.53\"}", - "Us Foreign Policy - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"us_foreign_policy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_us_foreign_policy\"" - } - } - }, - { - "evaluation_name": "Astronomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Astronomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.882, - "details": { - "description": "min=0.882, mean=0.882, max=0.882, sum=1.763 (2)", - "tab": "Accuracy", - "Astronomy - Observed inference time (s)": "{\"description\": \"min=0.975, mean=0.975, max=0.975, sum=1.95 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9748745105768505\"}", - "Astronomy - # eval": "{\"description\": \"min=152, mean=152, max=152, sum=304 (2)\", \"tab\": \"General information\", \"score\": \"152.0\"}", - "Astronomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Astronomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Astronomy - # prompt tokens": "{\"description\": \"min=632.947, mean=632.947, max=632.947, sum=1265.895 (2)\", \"tab\": \"General information\", \"score\": \"632.9473684210526\"}", - "Astronomy - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"astronomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_astronomy\"" - } - } - }, - { - "evaluation_name": "Business Ethics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Business Ethics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.77, - "details": { - "description": "min=0.77, mean=0.77, max=0.77, sum=1.54 (2)", - "tab": "Accuracy", - "Business Ethics - Observed inference time (s)": "{\"description\": \"min=0.963, mean=0.963, max=0.963, sum=1.926 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9630230093002319\"}", - "Business Ethics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Business Ethics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Business Ethics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Business Ethics - # prompt tokens": "{\"description\": \"min=591.96, mean=591.96, max=591.96, sum=1183.92 (2)\", \"tab\": \"General information\", \"score\": \"591.96\"}", - "Business Ethics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"business_ethics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_business_ethics\"" - } - } - }, - { - "evaluation_name": "Clinical Knowledge", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Clinical Knowledge", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.849, - "details": { - "description": "min=0.849, mean=0.849, max=0.849, sum=1.698 (2)", - "tab": "Accuracy", - "Clinical Knowledge - Observed inference time (s)": "{\"description\": \"min=0.937, mean=0.937, max=0.937, sum=1.874 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9370616642933971\"}", - "Clinical Knowledge - # eval": "{\"description\": \"min=265, mean=265, max=265, sum=530 (2)\", \"tab\": \"General information\", \"score\": \"265.0\"}", - "Clinical Knowledge - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Clinical Knowledge - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Clinical Knowledge - # prompt tokens": "{\"description\": \"min=437.34, mean=437.34, max=437.34, sum=874.679 (2)\", \"tab\": \"General information\", \"score\": \"437.33962264150944\"}", - "Clinical Knowledge - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"clinical_knowledge\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_clinical_knowledge\"" - } - } - }, - { - "evaluation_name": "Conceptual Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Conceptual Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.779, - "details": { - "description": "min=0.779, mean=0.779, max=0.779, sum=1.557 (2)", - "tab": "Accuracy", - "Conceptual Physics - Observed inference time (s)": "{\"description\": \"min=0.898, mean=0.898, max=0.898, sum=1.795 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8976521999277967\"}", - "Conceptual Physics - # eval": "{\"description\": \"min=235, mean=235, max=235, sum=470 (2)\", \"tab\": \"General information\", \"score\": \"235.0\"}", - "Conceptual Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Conceptual Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Conceptual Physics - # prompt tokens": "{\"description\": \"min=322.962, mean=322.962, max=322.962, sum=645.923 (2)\", \"tab\": \"General information\", \"score\": \"322.9617021276596\"}", - "Conceptual Physics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"conceptual_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_conceptual_physics\"" - } - } - }, - { - "evaluation_name": "Electrical Engineering", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Electrical Engineering", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.793, - "details": { - "description": "min=0.793, mean=0.793, max=0.793, sum=1.586 (2)", - "tab": "Accuracy", - "Electrical Engineering - Observed inference time (s)": "{\"description\": \"min=0.9, mean=0.9, max=0.9, sum=1.8 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9001944936555007\"}", - "Electrical Engineering - # eval": "{\"description\": \"min=145, mean=145, max=145, sum=290 (2)\", \"tab\": \"General information\", \"score\": \"145.0\"}", - "Electrical Engineering - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Electrical Engineering - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Electrical Engineering - # prompt tokens": "{\"description\": \"min=494.662, mean=494.662, max=494.662, sum=989.324 (2)\", \"tab\": \"General information\", \"score\": \"494.6620689655172\"}", - "Electrical Engineering - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"electrical_engineering\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_electrical_engineering\"" - } - } - }, - { - "evaluation_name": "Elementary Mathematics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Elementary Mathematics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.656, - "details": { - "description": "min=0.656, mean=0.656, max=0.656, sum=1.312 (2)", - "tab": "Accuracy", - "Elementary Mathematics - Observed inference time (s)": "{\"description\": \"min=0.976, mean=0.976, max=0.976, sum=1.951 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9756249517360062\"}", - "Elementary Mathematics - # eval": "{\"description\": \"min=378, mean=378, max=378, sum=756 (2)\", \"tab\": \"General information\", \"score\": \"378.0\"}", - "Elementary Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Elementary Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Elementary Mathematics - # prompt tokens": "{\"description\": \"min=607.042, mean=607.042, max=607.042, sum=1214.085 (2)\", \"tab\": \"General information\", \"score\": \"607.042328042328\"}", - "Elementary Mathematics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"elementary_mathematics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_elementary_mathematics\"" - } - } - }, - { - "evaluation_name": "Formal Logic", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Formal Logic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.619, - "details": { - "description": "min=0.619, mean=0.619, max=0.619, sum=1.238 (2)", - "tab": "Accuracy", - "Formal Logic - Observed inference time (s)": "{\"description\": \"min=0.973, mean=0.973, max=0.973, sum=1.947 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9733156949754745\"}", - "Formal Logic - # eval": "{\"description\": \"min=126, mean=126, max=126, sum=252 (2)\", \"tab\": \"General information\", \"score\": \"126.0\"}", - "Formal Logic - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Formal Logic - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Formal Logic - # prompt tokens": "{\"description\": \"min=656.468, mean=656.468, max=656.468, sum=1312.937 (2)\", \"tab\": \"General information\", \"score\": \"656.468253968254\"}", - "Formal Logic - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"formal_logic\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_formal_logic\"" - } - } - }, - { - "evaluation_name": "High School World History", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on High School World History", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.911, - "details": { - "description": "min=0.911, mean=0.911, max=0.911, sum=1.823 (2)", - "tab": "Accuracy", - "High School Biology - Observed inference time (s)": "{\"description\": \"min=0.953, mean=0.953, max=0.953, sum=1.906 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9529511121011549\"}", - "High School Chemistry - Observed inference time (s)": "{\"description\": \"min=0.955, mean=0.955, max=0.955, sum=1.911 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.955410502814307\"}", - "High School Computer Science - Observed inference time (s)": "{\"description\": \"min=0.978, mean=0.978, max=0.978, sum=1.957 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9784861493110657\"}", - "High School European History - Observed inference time (s)": "{\"description\": \"min=1.394, mean=1.394, max=1.394, sum=2.789 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.394392929655133\"}", - "High School Geography - Observed inference time (s)": "{\"description\": \"min=1.119, mean=1.119, max=1.119, sum=2.238 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.1188469896412858\"}", - "High School Government And Politics - Observed inference time (s)": "{\"description\": \"min=1.151, mean=1.151, max=1.151, sum=2.302 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.1508279983243794\"}", - "High School Macroeconomics - Observed inference time (s)": "{\"description\": \"min=1.015, mean=1.015, max=1.015, sum=2.03 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.014756965637207\"}", - "High School Mathematics - Observed inference time (s)": "{\"description\": \"min=1.115, mean=1.115, max=1.115, sum=2.229 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.1145719607671103\"}", - "High School Microeconomics - Observed inference time (s)": "{\"description\": \"min=1.094, mean=1.094, max=1.094, sum=2.189 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.094437322696718\"}", - "High School Physics - Observed inference time (s)": "{\"description\": \"min=1.117, mean=1.117, max=1.117, sum=2.235 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.1174537361852381\"}", - "High School Psychology - Observed inference time (s)": "{\"description\": \"min=1.026, mean=1.026, max=1.026, sum=2.051 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.025726358606181\"}", - "High School Statistics - Observed inference time (s)": "{\"description\": \"min=1.119, mean=1.119, max=1.119, sum=2.238 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.1191309756702847\"}", - "High School US History - Observed inference time (s)": "{\"description\": \"min=1.362, mean=1.362, max=1.362, sum=2.724 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.3617976483176737\"}", - "High School World History - Observed inference time (s)": "{\"description\": \"min=1.138, mean=1.138, max=1.138, sum=2.275 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.1377391141175217\"}", - "High School Biology - # eval": "{\"description\": \"min=310, mean=310, max=310, sum=620 (2)\", \"tab\": \"General information\", \"score\": \"310.0\"}", - "High School Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Biology - # prompt tokens": "{\"description\": \"min=532.455, mean=532.455, max=532.455, sum=1064.91 (2)\", \"tab\": \"General information\", \"score\": \"532.4548387096775\"}", - "High School Biology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # eval": "{\"description\": \"min=203, mean=203, max=203, sum=406 (2)\", \"tab\": \"General information\", \"score\": \"203.0\"}", - "High School Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # prompt tokens": "{\"description\": \"min=537.089, mean=537.089, max=537.089, sum=1074.177 (2)\", \"tab\": \"General information\", \"score\": \"537.0886699507389\"}", - "High School Chemistry - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "High School Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # prompt tokens": "{\"description\": \"min=958.39, mean=958.39, max=958.39, sum=1916.78 (2)\", \"tab\": \"General information\", \"score\": \"958.39\"}", - "High School Computer Science - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # eval": "{\"description\": \"min=165, mean=165, max=165, sum=330 (2)\", \"tab\": \"General information\", \"score\": \"165.0\"}", - "High School European History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School European History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # prompt tokens": "{\"description\": \"min=2900.673, mean=2900.673, max=2900.673, sum=5801.345 (2)\", \"tab\": \"General information\", \"score\": \"2900.672727272727\"}", - "High School European History - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # eval": "{\"description\": \"min=198, mean=198, max=198, sum=396 (2)\", \"tab\": \"General information\", \"score\": \"198.0\"}", - "High School Geography - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Geography - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # prompt tokens": "{\"description\": \"min=406.146, mean=406.146, max=406.146, sum=812.293 (2)\", \"tab\": \"General information\", \"score\": \"406.14646464646466\"}", - "High School Geography - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # eval": "{\"description\": \"min=193, mean=193, max=193, sum=386 (2)\", \"tab\": \"General information\", \"score\": \"193.0\"}", - "High School Government And Politics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Government And Politics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # prompt tokens": "{\"description\": \"min=492.788, mean=492.788, max=492.788, sum=985.575 (2)\", \"tab\": \"General information\", \"score\": \"492.78756476683935\"}", - "High School Government And Politics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # eval": "{\"description\": \"min=390, mean=390, max=390, sum=780 (2)\", \"tab\": \"General information\", \"score\": \"390.0\"}", - "High School Macroeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Macroeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # prompt tokens": "{\"description\": \"min=406.1, mean=406.1, max=406.1, sum=812.2 (2)\", \"tab\": \"General information\", \"score\": \"406.1\"}", - "High School Macroeconomics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # eval": "{\"description\": \"min=270, mean=270, max=270, sum=540 (2)\", \"tab\": \"General information\", \"score\": \"270.0\"}", - "High School Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # prompt tokens": "{\"description\": \"min=583.248, mean=583.248, max=583.248, sum=1166.496 (2)\", \"tab\": \"General information\", \"score\": \"583.2481481481482\"}", - "High School Mathematics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # eval": "{\"description\": \"min=238, mean=238, max=238, sum=476 (2)\", \"tab\": \"General information\", \"score\": \"238.0\"}", - "High School Microeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Microeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # prompt tokens": "{\"description\": \"min=426.265, mean=426.265, max=426.265, sum=852.529 (2)\", \"tab\": \"General information\", \"score\": \"426.2647058823529\"}", - "High School Microeconomics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # eval": "{\"description\": \"min=151, mean=151, max=151, sum=302 (2)\", \"tab\": \"General information\", \"score\": \"151.0\"}", - "High School Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # prompt tokens": "{\"description\": \"min=603.272, mean=603.272, max=603.272, sum=1206.543 (2)\", \"tab\": \"General information\", \"score\": \"603.2715231788079\"}", - "High School Physics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # eval": "{\"description\": \"min=545, mean=545, max=545, sum=1090 (2)\", \"tab\": \"General information\", \"score\": \"545.0\"}", - "High School Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # prompt tokens": "{\"description\": \"min=525.635, mean=525.635, max=525.635, sum=1051.27 (2)\", \"tab\": \"General information\", \"score\": \"525.6348623853211\"}", - "High School Psychology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # eval": "{\"description\": \"min=216, mean=216, max=216, sum=432 (2)\", \"tab\": \"General information\", \"score\": \"216.0\"}", - "High School Statistics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Statistics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # prompt tokens": "{\"description\": \"min=876.032, mean=876.032, max=876.032, sum=1752.065 (2)\", \"tab\": \"General information\", \"score\": \"876.0324074074074\"}", - "High School Statistics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # eval": "{\"description\": \"min=204, mean=204, max=204, sum=408 (2)\", \"tab\": \"General information\", \"score\": \"204.0\"}", - "High School US History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School US History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # prompt tokens": "{\"description\": \"min=2310.931, mean=2310.931, max=2310.931, sum=4621.863 (2)\", \"tab\": \"General information\", \"score\": \"2310.9313725490197\"}", - "High School US History - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # eval": "{\"description\": \"min=237, mean=237, max=237, sum=474 (2)\", \"tab\": \"General information\", \"score\": \"237.0\"}", - "High School World History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School World History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # prompt tokens": "{\"description\": \"min=1501.477, mean=1501.477, max=1501.477, sum=3002.954 (2)\", \"tab\": \"General information\", \"score\": \"1501.4767932489451\"}", - "High School World History - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"high_school_world_history\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_high_school_world_history\"" - } - } - }, - { - "evaluation_name": "Human Sexuality", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Human Sexuality", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.832, - "details": { - "description": "min=0.832, mean=0.832, max=0.832, sum=1.664 (2)", - "tab": "Accuracy", - "Human Aging - Observed inference time (s)": "{\"description\": \"min=1.018, mean=1.018, max=1.018, sum=2.036 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.0177636157236827\"}", - "Human Sexuality - Observed inference time (s)": "{\"description\": \"min=1.059, mean=1.059, max=1.059, sum=2.118 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.0589779351503794\"}", - "Human Aging - # eval": "{\"description\": \"min=223, mean=223, max=223, sum=446 (2)\", \"tab\": \"General information\", \"score\": \"223.0\"}", - "Human Aging - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Aging - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Aging - # prompt tokens": "{\"description\": \"min=333.036, mean=333.036, max=333.036, sum=666.072 (2)\", \"tab\": \"General information\", \"score\": \"333.0358744394619\"}", - "Human Aging - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # eval": "{\"description\": \"min=131, mean=131, max=131, sum=262 (2)\", \"tab\": \"General information\", \"score\": \"131.0\"}", - "Human Sexuality - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Sexuality - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # prompt tokens": "{\"description\": \"min=362.466, mean=362.466, max=362.466, sum=724.931 (2)\", \"tab\": \"General information\", \"score\": \"362.46564885496184\"}", - "Human Sexuality - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"human_sexuality\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_human_sexuality\"" - } - } - }, - { - "evaluation_name": "International Law", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on International Law", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.884, - "details": { - "description": "min=0.884, mean=0.884, max=0.884, sum=1.769 (2)", - "tab": "Accuracy", - "International Law - Observed inference time (s)": "{\"description\": \"min=1.098, mean=1.098, max=1.098, sum=2.197 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.098483010757068\"}", - "International Law - # eval": "{\"description\": \"min=121, mean=121, max=121, sum=242 (2)\", \"tab\": \"General information\", \"score\": \"121.0\"}", - "International Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "International Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "International Law - # prompt tokens": "{\"description\": \"min=662.628, mean=662.628, max=662.628, sum=1325.256 (2)\", \"tab\": \"General information\", \"score\": \"662.6280991735537\"}", - "International Law - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"international_law\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_international_law\"" - } - } - }, - { - "evaluation_name": "Logical Fallacies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Logical Fallacies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.859, - "details": { - "description": "min=0.859, mean=0.859, max=0.859, sum=1.718 (2)", - "tab": "Accuracy", - "Logical Fallacies - Observed inference time (s)": "{\"description\": \"min=1.023, mean=1.023, max=1.023, sum=2.046 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.0228094908357397\"}", - "Logical Fallacies - # eval": "{\"description\": \"min=163, mean=163, max=163, sum=326 (2)\", \"tab\": \"General information\", \"score\": \"163.0\"}", - "Logical Fallacies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Logical Fallacies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Logical Fallacies - # prompt tokens": "{\"description\": \"min=466.227, mean=466.227, max=466.227, sum=932.454 (2)\", \"tab\": \"General information\", \"score\": \"466.2269938650307\"}", - "Logical Fallacies - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"logical_fallacies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_logical_fallacies\"" - } - } - }, - { - "evaluation_name": "Machine Learning", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Machine Learning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.688, - "details": { - "description": "min=0.688, mean=0.688, max=0.688, sum=1.375 (2)", - "tab": "Accuracy", - "Machine Learning - Observed inference time (s)": "{\"description\": \"min=1.124, mean=1.124, max=1.124, sum=2.247 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.123652777501515\"}", - "Machine Learning - # eval": "{\"description\": \"min=112, mean=112, max=112, sum=224 (2)\", \"tab\": \"General information\", \"score\": \"112.0\"}", - "Machine Learning - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Machine Learning - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Machine Learning - # prompt tokens": "{\"description\": \"min=719.938, mean=719.938, max=719.938, sum=1439.875 (2)\", \"tab\": \"General information\", \"score\": \"719.9375\"}", - "Machine Learning - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"machine_learning\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_machine_learning\"" - } - } - }, - { - "evaluation_name": "Management", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Management", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.864, - "details": { - "description": "min=0.864, mean=0.864, max=0.864, sum=1.728 (2)", - "tab": "Accuracy", - "Management - Observed inference time (s)": "{\"description\": \"min=1.033, mean=1.033, max=1.033, sum=2.067 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.0334750402320936\"}", - "Management - # eval": "{\"description\": \"min=103, mean=103, max=103, sum=206 (2)\", \"tab\": \"General information\", \"score\": \"103.0\"}", - "Management - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Management - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Management - # prompt tokens": "{\"description\": \"min=299.553, mean=299.553, max=299.553, sum=599.107 (2)\", \"tab\": \"General information\", \"score\": \"299.5533980582524\"}", - "Management - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"management\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_management\"" - } - } - }, - { - "evaluation_name": "Marketing", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Marketing", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.94, - "details": { - "description": "min=0.94, mean=0.94, max=0.94, sum=1.88 (2)", - "tab": "Accuracy", - "Marketing - Observed inference time (s)": "{\"description\": \"min=1.097, mean=1.097, max=1.097, sum=2.194 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.0967916657782009\"}", - "Marketing - # eval": "{\"description\": \"min=234, mean=234, max=234, sum=468 (2)\", \"tab\": \"General information\", \"score\": \"234.0\"}", - "Marketing - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Marketing - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Marketing - # prompt tokens": "{\"description\": \"min=446.714, mean=446.714, max=446.714, sum=893.427 (2)\", \"tab\": \"General information\", \"score\": \"446.71367521367523\"}", - "Marketing - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"marketing\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_marketing\"" - } - } - }, - { - "evaluation_name": "Medical Genetics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Medical Genetics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.89, - "details": { - "description": "min=0.89, mean=0.89, max=0.89, sum=1.78 (2)", - "tab": "Accuracy", - "Medical Genetics - Observed inference time (s)": "{\"description\": \"min=1.101, mean=1.101, max=1.101, sum=2.201 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.1006885027885438\"}", - "Medical Genetics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Medical Genetics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Medical Genetics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Medical Genetics - # prompt tokens": "{\"description\": \"min=361.45, mean=361.45, max=361.45, sum=722.9 (2)\", \"tab\": \"General information\", \"score\": \"361.45\"}", - "Medical Genetics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"medical_genetics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_medical_genetics\"" - } - } - }, - { - "evaluation_name": "Miscellaneous", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Miscellaneous", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.931, - "details": { - "description": "min=0.931, mean=0.931, max=0.931, sum=1.862 (2)", - "tab": "Accuracy", - "Miscellaneous - Observed inference time (s)": "{\"description\": \"min=0.906, mean=0.906, max=0.906, sum=1.813 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9063281955085647\"}", - "Miscellaneous - # eval": "{\"description\": \"min=783, mean=783, max=783, sum=1566 (2)\", \"tab\": \"General information\", \"score\": \"783.0\"}", - "Miscellaneous - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Miscellaneous - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Miscellaneous - # prompt tokens": "{\"description\": \"min=332.257, mean=332.257, max=332.257, sum=664.513 (2)\", \"tab\": \"General information\", \"score\": \"332.2567049808429\"}", - "Miscellaneous - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"miscellaneous\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_miscellaneous\"" - } - } - }, - { - "evaluation_name": "Moral Scenarios", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Moral Scenarios", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.686, - "details": { - "description": "min=0.686, mean=0.686, max=0.686, sum=1.372 (2)", - "tab": "Accuracy", - "Moral Disputes - Observed inference time (s)": "{\"description\": \"min=0.946, mean=0.946, max=0.946, sum=1.892 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9461793238027937\"}", - "Moral Scenarios - Observed inference time (s)": "{\"description\": \"min=0.96, mean=0.96, max=0.96, sum=1.92 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9602039808667572\"}", - "Moral Disputes - # eval": "{\"description\": \"min=346, mean=346, max=346, sum=692 (2)\", \"tab\": \"General information\", \"score\": \"346.0\"}", - "Moral Disputes - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Disputes - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Disputes - # prompt tokens": "{\"description\": \"min=506.514, mean=506.514, max=506.514, sum=1013.029 (2)\", \"tab\": \"General information\", \"score\": \"506.514450867052\"}", - "Moral Disputes - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # eval": "{\"description\": \"min=895, mean=895, max=895, sum=1790 (2)\", \"tab\": \"General information\", \"score\": \"895.0\"}", - "Moral Scenarios - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Scenarios - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # prompt tokens": "{\"description\": \"min=709.934, mean=709.934, max=709.934, sum=1419.868 (2)\", \"tab\": \"General information\", \"score\": \"709.9340782122905\"}", - "Moral Scenarios - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"moral_scenarios\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_moral_scenarios\"" - } - } - }, - { - "evaluation_name": "Nutrition", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Nutrition", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.869, - "details": { - "description": "min=0.869, mean=0.869, max=0.869, sum=1.739 (2)", - "tab": "Accuracy", - "Nutrition - Observed inference time (s)": "{\"description\": \"min=0.947, mean=0.947, max=0.947, sum=1.894 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9469306157305349\"}", - "Nutrition - # eval": "{\"description\": \"min=306, mean=306, max=306, sum=612 (2)\", \"tab\": \"General information\", \"score\": \"306.0\"}", - "Nutrition - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Nutrition - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Nutrition - # prompt tokens": "{\"description\": \"min=619.683, mean=619.683, max=619.683, sum=1239.366 (2)\", \"tab\": \"General information\", \"score\": \"619.6830065359477\"}", - "Nutrition - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"nutrition\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_nutrition\"" - } - } - }, - { - "evaluation_name": "Prehistory", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Prehistory", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.892, - "details": { - "description": "min=0.892, mean=0.892, max=0.892, sum=1.784 (2)", - "tab": "Accuracy", - "Prehistory - Observed inference time (s)": "{\"description\": \"min=0.956, mean=0.956, max=0.956, sum=1.912 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9560920861032274\"}", - "Prehistory - # eval": "{\"description\": \"min=324, mean=324, max=324, sum=648 (2)\", \"tab\": \"General information\", \"score\": \"324.0\"}", - "Prehistory - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Prehistory - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Prehistory - # prompt tokens": "{\"description\": \"min=566.244, mean=566.244, max=566.244, sum=1132.488 (2)\", \"tab\": \"General information\", \"score\": \"566.2438271604939\"}", - "Prehistory - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"prehistory\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_prehistory\"" - } - } - }, - { - "evaluation_name": "Public Relations", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Public Relations", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.755, - "details": { - "description": "min=0.755, mean=0.755, max=0.755, sum=1.509 (2)", - "tab": "Accuracy", - "Public Relations - Observed inference time (s)": "{\"description\": \"min=0.944, mean=0.944, max=0.944, sum=1.887 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9436206535859541\"}", - "Public Relations - # eval": "{\"description\": \"min=110, mean=110, max=110, sum=220 (2)\", \"tab\": \"General information\", \"score\": \"110.0\"}", - "Public Relations - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Public Relations - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Public Relations - # prompt tokens": "{\"description\": \"min=440.6, mean=440.6, max=440.6, sum=881.2 (2)\", \"tab\": \"General information\", \"score\": \"440.6\"}", - "Public Relations - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"public_relations\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_public_relations\"" - } - } - }, - { - "evaluation_name": "Security Studies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Security Studies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.771, - "details": { - "description": "min=0.771, mean=0.771, max=0.771, sum=1.543 (2)", - "tab": "Accuracy", - "Security Studies - Observed inference time (s)": "{\"description\": \"min=0.988, mean=0.988, max=0.988, sum=1.976 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9880037901352863\"}", - "Security Studies - # eval": "{\"description\": \"min=245, mean=245, max=245, sum=490 (2)\", \"tab\": \"General information\", \"score\": \"245.0\"}", - "Security Studies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Security Studies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Security Studies - # prompt tokens": "{\"description\": \"min=1221.388, mean=1221.388, max=1221.388, sum=2442.776 (2)\", \"tab\": \"General information\", \"score\": \"1221.3877551020407\"}", - "Security Studies - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"security_studies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_security_studies\"" - } - } - }, - { - "evaluation_name": "Sociology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Sociology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.93, - "details": { - "description": "min=0.93, mean=0.93, max=0.93, sum=1.861 (2)", - "tab": "Accuracy", - "Sociology - Observed inference time (s)": "{\"description\": \"min=0.947, mean=0.947, max=0.947, sum=1.894 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9468028070914805\"}", - "Sociology - # eval": "{\"description\": \"min=201, mean=201, max=201, sum=402 (2)\", \"tab\": \"General information\", \"score\": \"201.0\"}", - "Sociology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Sociology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Sociology - # prompt tokens": "{\"description\": \"min=465.925, mean=465.925, max=465.925, sum=931.851 (2)\", \"tab\": \"General information\", \"score\": \"465.92537313432837\"}", - "Sociology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"sociology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_sociology\"" - } - } - }, - { - "evaluation_name": "Virology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Virology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.554, - "details": { - "description": "min=0.554, mean=0.554, max=0.554, sum=1.108 (2)", - "tab": "Accuracy", - "Virology - Observed inference time (s)": "{\"description\": \"min=0.901, mean=0.901, max=0.901, sum=1.803 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9013677418950092\"}", - "Virology - # eval": "{\"description\": \"min=166, mean=166, max=166, sum=332 (2)\", \"tab\": \"General information\", \"score\": \"166.0\"}", - "Virology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Virology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Virology - # prompt tokens": "{\"description\": \"min=358.048, mean=358.048, max=358.048, sum=716.096 (2)\", \"tab\": \"General information\", \"score\": \"358.04819277108436\"}", - "Virology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"virology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_virology\"" - } - } - }, - { - "evaluation_name": "World Religions", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on World Religions", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.865, - "details": { - "description": "min=0.865, mean=0.865, max=0.865, sum=1.731 (2)", - "tab": "Accuracy", - "World Religions - Observed inference time (s)": "{\"description\": \"min=0.899, mean=0.899, max=0.899, sum=1.799 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8992712400112933\"}", - "World Religions - # eval": "{\"description\": \"min=171, mean=171, max=171, sum=342 (2)\", \"tab\": \"General information\", \"score\": \"171.0\"}", - "World Religions - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "World Religions - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "World Religions - # prompt tokens": "{\"description\": \"min=293.649, mean=293.649, max=293.649, sum=587.298 (2)\", \"tab\": \"General information\", \"score\": \"293.64912280701753\"}", - "World Religions - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"world_religions\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_world_religions\"" - } - } - }, - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.147, - "details": { - "description": "", - "tab": "Efficiency" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/ai21_jamba-1.5-mini.json b/data/models/ai21_jamba-1.5-mini.json deleted file mode 100644 index b24fb8d29e066f3f2e5f3bd687b369a63d55eba5..0000000000000000000000000000000000000000 --- a/data/models/ai21_jamba-1.5-mini.json +++ /dev/null @@ -1,1903 +0,0 @@ -{ - "model_info": { - "name": "Jamba 1.5 Mini", - "id": "ai21/jamba-1.5-mini", - "developer": "ai21", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_lite/ai21_jamba-1.5-mini/1774096306.427425", - "retrieved_timestamp": "1774096306.427425", - "source_metadata": { - "source_name": "helm_lite", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_lite", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_lite", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.414, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.44747815230961296\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.746, - "details": { - "description": "min=0.746, mean=0.746, max=0.746, sum=0.746 (1)", - "tab": "Accuracy", - "NarrativeQA - Observed inference time (s)": "{\"description\": \"min=0.998, mean=0.998, max=0.998, sum=0.998 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.9981950746455662\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=3595.597, mean=3595.597, max=3595.597, sum=3595.597 (1)\", \"tab\": \"General information\", \"score\": \"3595.5971830985914\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (closed-book)", - "source_data": { - "dataset_name": "NaturalQuestions (closed-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (closed-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.388, - "details": { - "description": "min=0.388, mean=0.388, max=0.388, sum=0.388 (1)", - "tab": "Accuracy", - "NaturalQuestions (open-book) - Observed inference time (s)": "{\"description\": \"min=0.924, mean=0.924, max=0.924, sum=0.924 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.9243871104717255\"}", - "NaturalQuestions (closed-book) - Observed inference time (s)": "{\"description\": \"min=0.844, mean=0.844, max=0.844, sum=0.844 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.8436705965995789\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=2333.076, mean=2333.076, max=2333.076, sum=2333.076 (1)\", \"tab\": \"General information\", \"score\": \"2333.076\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=152.394, mean=152.394, max=152.394, sum=152.394 (1)\", \"tab\": \"General information\", \"score\": \"152.394\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "mode": "\"closedbook\"" - } - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.89, - "details": { - "description": "min=0.89, mean=0.89, max=0.89, sum=0.89 (1)", - "tab": "Accuracy", - "OpenbookQA - Observed inference time (s)": "{\"description\": \"min=0.786, mean=0.786, max=0.786, sum=0.786 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.7863723936080933\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=261.348, mean=261.348, max=261.348, sum=261.348 (1)\", \"tab\": \"General information\", \"score\": \"261.348\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "dataset": "\"openbookqa\"", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.582, - "details": { - "description": "min=0.33, mean=0.582, max=0.9, sum=2.911 (5)", - "tab": "Accuracy", - "MMLU - Observed inference time (s)": "{\"description\": \"min=0.783, mean=0.81, max=0.83, sum=4.049 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.8097888966024968\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=397.58, mean=508.138, max=678.64, sum=2540.69 (5)\", \"tab\": \"General information\", \"score\": \"508.1380701754386\"}", - "MMLU - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"college_chemistry\", \"computer_security\", \"econometrics\", \"us_foreign_policy\"]", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MATH", - "source_data": { - "dataset_name": "MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Equivalent (CoT) on MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.318, - "details": { - "description": "min=0.233, mean=0.318, max=0.386, sum=2.227 (7)", - "tab": "Accuracy", - "MATH - Observed inference time (s)": "{\"description\": \"min=1.462, mean=1.636, max=2.034, sum=11.452 (7)\", \"tab\": \"Efficiency\", \"score\": \"1.63604986000122\"}", - "MATH - # eval": "{\"description\": \"min=30, mean=62.429, max=135, sum=437 (7)\", \"tab\": \"General information\", \"score\": \"62.42857142857143\"}", - "MATH - # train": "{\"description\": \"min=8, mean=8, max=8, sum=56 (7)\", \"tab\": \"General information\", \"score\": \"8.0\"}", - "MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MATH - # prompt tokens": "{\"description\": \"min=979.415, mean=1458.376, max=2550.115, sum=10208.634 (7)\", \"tab\": \"General information\", \"score\": \"1458.376275861588\"}", - "MATH - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"algebra\", \"counting_and_probability\", \"geometry\", \"intermediate_algebra\", \"number_theory\", \"prealgebra\", \"precalculus\"]", - "level": "\"1\"", - "use_official_examples": "\"False\"", - "use_chain_of_thought": "\"True\"" - } - } - }, - { - "evaluation_name": "GSM8K", - "source_data": { - "dataset_name": "GSM8K", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on GSM8K", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.691, - "details": { - "description": "min=0.691, mean=0.691, max=0.691, sum=0.691 (1)", - "tab": "Accuracy", - "GSM8K - Observed inference time (s)": "{\"description\": \"min=1.892, mean=1.892, max=1.892, sum=1.892 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.8916997435092926\"}", - "GSM8K - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "GSM8K - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "GSM8K - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GSM8K - # prompt tokens": "{\"description\": \"min=1163.818, mean=1163.818, max=1163.818, sum=1163.818 (1)\", \"tab\": \"General information\", \"score\": \"1163.818\"}", - "GSM8K - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "stop": "\"none\"" - } - } - }, - { - "evaluation_name": "LegalBench", - "source_data": { - "dataset_name": "LegalBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on LegalBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.503, - "details": { - "description": "min=0.365, mean=0.503, max=0.842, sum=2.514 (5)", - "tab": "Accuracy", - "LegalBench - Observed inference time (s)": "{\"description\": \"min=0.805, mean=0.864, max=1.071, sum=4.322 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.8644844750252041\"}", - "LegalBench - # eval": "{\"description\": \"min=95, mean=409.4, max=1000, sum=2047 (5)\", \"tab\": \"General information\", \"score\": \"409.4\"}", - "LegalBench - # train": "{\"description\": \"min=4, mean=4.8, max=5, sum=24 (5)\", \"tab\": \"General information\", \"score\": \"4.8\"}", - "LegalBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "LegalBench - # prompt tokens": "{\"description\": \"min=212.453, mean=1601.843, max=6618.612, sum=8009.215 (5)\", \"tab\": \"General information\", \"score\": \"1601.842950915631\"}", - "LegalBench - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"abercrombie\", \"corporate_lobbying\", \"function_of_decision_section\", \"international_citizenship_questions\", \"proa\"]" - } - } - }, - { - "evaluation_name": "MedQA", - "source_data": { - "dataset_name": "MedQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MedQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.632, - "details": { - "description": "min=0.632, mean=0.632, max=0.632, sum=0.632 (1)", - "tab": "Accuracy", - "MedQA - Observed inference time (s)": "{\"description\": \"min=0.817, mean=0.817, max=0.817, sum=0.817 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.8172814860258615\"}", - "MedQA - # eval": "{\"description\": \"min=503, mean=503, max=503, sum=503 (1)\", \"tab\": \"General information\", \"score\": \"503.0\"}", - "MedQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MedQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MedQA - # prompt tokens": "{\"description\": \"min=1085.239, mean=1085.239, max=1085.239, sum=1085.239 (1)\", \"tab\": \"General information\", \"score\": \"1085.2385685884692\"}", - "MedQA - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WMT 2014", - "source_data": { - "dataset_name": "WMT 2014", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "BLEU-4 on WMT 2014", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.179, - "details": { - "description": "min=0.116, mean=0.179, max=0.21, sum=0.895 (5)", - "tab": "Accuracy", - "WMT 2014 - Observed inference time (s)": "{\"description\": \"min=0.965, mean=0.978, max=0.99, sum=4.888 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.9776749755042665\"}", - "WMT 2014 - # eval": "{\"description\": \"min=503, mean=568.8, max=832, sum=2844 (5)\", \"tab\": \"General information\", \"score\": \"568.8\"}", - "WMT 2014 - # train": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "WMT 2014 - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WMT 2014 - # prompt tokens": "{\"description\": \"min=120.386, mean=151.077, max=189.223, sum=755.383 (5)\", \"tab\": \"General information\", \"score\": \"151.07662629989292\"}", - "WMT 2014 - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "language_pair": "[\"cs-en\", \"de-en\", \"fr-en\", \"hi-en\", \"ru-en\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_mmlu/ai21_jamba-1.5-mini/1774096312.00548", - "retrieved_timestamp": "1774096312.00548", - "source_metadata": { - "source_name": "helm_mmlu", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_mmlu", - "evaluation_results": [ - { - "evaluation_name": "MMLU All Subjects", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU All Subjects", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.699, - "details": { - "description": "min=0.269, mean=0.699, max=0.943, sum=79.696 (114)", - "tab": "Accuracy", - "MMLU All Subjects - Observed inference time (s)": "{\"description\": \"min=0.78, mean=0.859, max=1.024, sum=97.957 (114)\", \"tab\": \"Efficiency\", \"score\": \"0.8592709427634447\"}", - "MMLU All Subjects - # eval": "{\"description\": \"min=100, mean=246.351, max=1534, sum=28084 (114)\", \"tab\": \"General information\", \"score\": \"246.35087719298247\"}", - "MMLU All Subjects - # train": "{\"description\": \"min=5, mean=5, max=5, sum=570 (114)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU All Subjects - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU All Subjects - # prompt tokens": "{\"description\": \"min=293.649, mean=658.432, max=2900.673, sum=75061.271 (114)\", \"tab\": \"General information\", \"score\": \"658.4322049384847\"}", - "MMLU All Subjects - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_name": "Abstract Algebra", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Abstract Algebra", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.33, - "details": { - "description": "min=0.33, mean=0.33, max=0.33, sum=0.66 (2)", - "tab": "Accuracy", - "Abstract Algebra - Observed inference time (s)": "{\"description\": \"min=0.783, mean=0.783, max=0.783, sum=1.566 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.783083221912384\"}", - "Abstract Algebra - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Abstract Algebra - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Abstract Algebra - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Abstract Algebra - # prompt tokens": "{\"description\": \"min=397.58, mean=397.58, max=397.58, sum=795.16 (2)\", \"tab\": \"General information\", \"score\": \"397.58\"}", - "Abstract Algebra - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"abstract_algebra\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_abstract_algebra\"" - } - } - }, - { - "evaluation_name": "Anatomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Anatomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.711, - "details": { - "description": "min=0.711, mean=0.711, max=0.711, sum=1.422 (2)", - "tab": "Accuracy", - "Anatomy - Observed inference time (s)": "{\"description\": \"min=0.832, mean=0.832, max=0.832, sum=1.664 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8321040700983118\"}", - "Anatomy - # eval": "{\"description\": \"min=135, mean=135, max=135, sum=270 (2)\", \"tab\": \"General information\", \"score\": \"135.0\"}", - "Anatomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Anatomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Anatomy - # prompt tokens": "{\"description\": \"min=376.741, mean=376.741, max=376.741, sum=753.481 (2)\", \"tab\": \"General information\", \"score\": \"376.74074074074076\"}", - "Anatomy - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"anatomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_anatomy\"" - } - } - }, - { - "evaluation_name": "College Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on College Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.48, - "details": { - "description": "min=0.48, mean=0.48, max=0.48, sum=0.961 (2)", - "tab": "Accuracy", - "College Chemistry - Observed inference time (s)": "{\"description\": \"min=0.807, mean=0.807, max=0.807, sum=1.615 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8074449944496155\"}", - "College Biology - Observed inference time (s)": "{\"description\": \"min=0.821, mean=0.821, max=0.821, sum=1.643 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8214208516809676\"}", - "College Computer Science - Observed inference time (s)": "{\"description\": \"min=0.833, mean=0.833, max=0.833, sum=1.667 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8334288668632507\"}", - "College Mathematics - Observed inference time (s)": "{\"description\": \"min=0.84, mean=0.84, max=0.84, sum=1.68 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8399906301498413\"}", - "College Medicine - Observed inference time (s)": "{\"description\": \"min=0.831, mean=0.831, max=0.831, sum=1.662 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8312392317490771\"}", - "College Physics - Observed inference time (s)": "{\"description\": \"min=0.829, mean=0.829, max=0.829, sum=1.658 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8287959309185252\"}", - "College Chemistry - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Chemistry - # prompt tokens": "{\"description\": \"min=598.67, mean=598.67, max=598.67, sum=1197.34 (2)\", \"tab\": \"General information\", \"score\": \"598.67\"}", - "College Chemistry - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # eval": "{\"description\": \"min=144, mean=144, max=144, sum=288 (2)\", \"tab\": \"General information\", \"score\": \"144.0\"}", - "College Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # prompt tokens": "{\"description\": \"min=507.306, mean=507.306, max=507.306, sum=1014.611 (2)\", \"tab\": \"General information\", \"score\": \"507.30555555555554\"}", - "College Biology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # prompt tokens": "{\"description\": \"min=883.21, mean=883.21, max=883.21, sum=1766.42 (2)\", \"tab\": \"General information\", \"score\": \"883.21\"}", - "College Computer Science - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # prompt tokens": "{\"description\": \"min=643.97, mean=643.97, max=643.97, sum=1287.94 (2)\", \"tab\": \"General information\", \"score\": \"643.97\"}", - "College Mathematics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # eval": "{\"description\": \"min=173, mean=173, max=173, sum=346 (2)\", \"tab\": \"General information\", \"score\": \"173.0\"}", - "College Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # prompt tokens": "{\"description\": \"min=543.347, mean=543.347, max=543.347, sum=1086.694 (2)\", \"tab\": \"General information\", \"score\": \"543.3468208092486\"}", - "College Medicine - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # eval": "{\"description\": \"min=102, mean=102, max=102, sum=204 (2)\", \"tab\": \"General information\", \"score\": \"102.0\"}", - "College Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # prompt tokens": "{\"description\": \"min=533.402, mean=533.402, max=533.402, sum=1066.804 (2)\", \"tab\": \"General information\", \"score\": \"533.4019607843137\"}", - "College Physics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"college_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_college_physics\"" - } - } - }, - { - "evaluation_name": "Computer Security", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Computer Security", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.73, - "details": { - "description": "min=0.73, mean=0.73, max=0.73, sum=1.46 (2)", - "tab": "Accuracy", - "Computer Security - Observed inference time (s)": "{\"description\": \"min=0.801, mean=0.801, max=0.801, sum=1.602 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8010901069641113\"}", - "Computer Security - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Computer Security - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Computer Security - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Computer Security - # prompt tokens": "{\"description\": \"min=404.27, mean=404.27, max=404.27, sum=808.54 (2)\", \"tab\": \"General information\", \"score\": \"404.27\"}", - "Computer Security - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"computer_security\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_computer_security\"" - } - } - }, - { - "evaluation_name": "Econometrics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Econometrics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.491, - "details": { - "description": "min=0.491, mean=0.491, max=0.491, sum=0.982 (2)", - "tab": "Accuracy", - "Econometrics - Observed inference time (s)": "{\"description\": \"min=0.83, mean=0.83, max=0.83, sum=1.661 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8303811194603903\"}", - "Econometrics - # eval": "{\"description\": \"min=114, mean=114, max=114, sum=228 (2)\", \"tab\": \"General information\", \"score\": \"114.0\"}", - "Econometrics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Econometrics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Econometrics - # prompt tokens": "{\"description\": \"min=678.64, mean=678.64, max=678.64, sum=1357.281 (2)\", \"tab\": \"General information\", \"score\": \"678.640350877193\"}", - "Econometrics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"econometrics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_econometrics\"" - } - } - }, - { - "evaluation_name": "Global Facts", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Global Facts", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.43, - "details": { - "description": "min=0.43, mean=0.43, max=0.43, sum=0.86 (2)", - "tab": "Accuracy", - "Global Facts - Observed inference time (s)": "{\"description\": \"min=0.847, mean=0.847, max=0.847, sum=1.694 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8467721128463745\"}", - "Global Facts - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Global Facts - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Global Facts - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Global Facts - # prompt tokens": "{\"description\": \"min=466.9, mean=466.9, max=466.9, sum=933.8 (2)\", \"tab\": \"General information\", \"score\": \"466.9\"}", - "Global Facts - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"global_facts\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_global_facts\"" - } - } - }, - { - "evaluation_name": "Jurisprudence", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Jurisprudence", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.88, - "details": { - "description": "min=0.88, mean=0.88, max=0.88, sum=1.759 (2)", - "tab": "Accuracy", - "Jurisprudence - Observed inference time (s)": "{\"description\": \"min=0.809, mean=0.809, max=0.809, sum=1.619 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8092732672338132\"}", - "Jurisprudence - # eval": "{\"description\": \"min=108, mean=108, max=108, sum=216 (2)\", \"tab\": \"General information\", \"score\": \"108.0\"}", - "Jurisprudence - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Jurisprudence - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Jurisprudence - # prompt tokens": "{\"description\": \"min=427.185, mean=427.185, max=427.185, sum=854.37 (2)\", \"tab\": \"General information\", \"score\": \"427.18518518518516\"}", - "Jurisprudence - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"jurisprudence\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_jurisprudence\"" - } - } - }, - { - "evaluation_name": "Philosophy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Philosophy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.752, - "details": { - "description": "min=0.752, mean=0.752, max=0.752, sum=1.505 (2)", - "tab": "Accuracy", - "Philosophy - Observed inference time (s)": "{\"description\": \"min=0.815, mean=0.815, max=0.815, sum=1.629 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8147224314343124\"}", - "Philosophy - # eval": "{\"description\": \"min=311, mean=311, max=311, sum=622 (2)\", \"tab\": \"General information\", \"score\": \"311.0\"}", - "Philosophy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Philosophy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Philosophy - # prompt tokens": "{\"description\": \"min=359.441, mean=359.441, max=359.441, sum=718.881 (2)\", \"tab\": \"General information\", \"score\": \"359.4405144694534\"}", - "Philosophy - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"philosophy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_philosophy\"" - } - } - }, - { - "evaluation_name": "Professional Psychology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Professional Psychology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.76, - "details": { - "description": "min=0.76, mean=0.76, max=0.76, sum=1.52 (2)", - "tab": "Accuracy", - "Professional Medicine - Observed inference time (s)": "{\"description\": \"min=0.832, mean=0.832, max=0.832, sum=1.663 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8315524055677301\"}", - "Professional Accounting - Observed inference time (s)": "{\"description\": \"min=0.803, mean=0.803, max=0.803, sum=1.606 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8028552659014438\"}", - "Professional Law - Observed inference time (s)": "{\"description\": \"min=0.836, mean=0.836, max=0.836, sum=1.671 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8356168884031154\"}", - "Professional Psychology - Observed inference time (s)": "{\"description\": \"min=0.812, mean=0.812, max=0.812, sum=1.624 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.811913901684331\"}", - "Professional Medicine - # eval": "{\"description\": \"min=272, mean=272, max=272, sum=544 (2)\", \"tab\": \"General information\", \"score\": \"272.0\"}", - "Professional Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Medicine - # prompt tokens": "{\"description\": \"min=1170.393, mean=1170.393, max=1170.393, sum=2340.787 (2)\", \"tab\": \"General information\", \"score\": \"1170.3933823529412\"}", - "Professional Medicine - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # eval": "{\"description\": \"min=282, mean=282, max=282, sum=564 (2)\", \"tab\": \"General information\", \"score\": \"282.0\"}", - "Professional Accounting - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Accounting - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # prompt tokens": "{\"description\": \"min=770.316, mean=770.316, max=770.316, sum=1540.631 (2)\", \"tab\": \"General information\", \"score\": \"770.3156028368794\"}", - "Professional Accounting - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # eval": "{\"description\": \"min=1534, mean=1534, max=1534, sum=3068 (2)\", \"tab\": \"General information\", \"score\": \"1534.0\"}", - "Professional Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # prompt tokens": "{\"description\": \"min=1725.955, mean=1725.955, max=1725.955, sum=3451.91 (2)\", \"tab\": \"General information\", \"score\": \"1725.9550195567144\"}", - "Professional Law - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # eval": "{\"description\": \"min=612, mean=612, max=612, sum=1224 (2)\", \"tab\": \"General information\", \"score\": \"612.0\"}", - "Professional Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # prompt tokens": "{\"description\": \"min=611.645, mean=611.645, max=611.645, sum=1223.291 (2)\", \"tab\": \"General information\", \"score\": \"611.6454248366013\"}", - "Professional Psychology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"professional_psychology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_professional_psychology\"" - } - } - }, - { - "evaluation_name": "Us Foreign Policy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Us Foreign Policy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9, - "details": { - "description": "min=0.9, mean=0.9, max=0.9, sum=1.8 (2)", - "tab": "Accuracy", - "Us Foreign Policy - Observed inference time (s)": "{\"description\": \"min=0.827, mean=0.827, max=0.827, sum=1.654 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8269450402259827\"}", - "Us Foreign Policy - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Us Foreign Policy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Us Foreign Policy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Us Foreign Policy - # prompt tokens": "{\"description\": \"min=461.53, mean=461.53, max=461.53, sum=923.06 (2)\", \"tab\": \"General information\", \"score\": \"461.53\"}", - "Us Foreign Policy - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"us_foreign_policy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_us_foreign_policy\"" - } - } - }, - { - "evaluation_name": "Astronomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Astronomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.822, - "details": { - "description": "min=0.822, mean=0.822, max=0.822, sum=1.645 (2)", - "tab": "Accuracy", - "Astronomy - Observed inference time (s)": "{\"description\": \"min=0.811, mean=0.811, max=0.811, sum=1.622 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8109481099404787\"}", - "Astronomy - # eval": "{\"description\": \"min=152, mean=152, max=152, sum=304 (2)\", \"tab\": \"General information\", \"score\": \"152.0\"}", - "Astronomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Astronomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Astronomy - # prompt tokens": "{\"description\": \"min=632.947, mean=632.947, max=632.947, sum=1265.895 (2)\", \"tab\": \"General information\", \"score\": \"632.9473684210526\"}", - "Astronomy - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"astronomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_astronomy\"" - } - } - }, - { - "evaluation_name": "Business Ethics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Business Ethics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.76, - "details": { - "description": "min=0.76, mean=0.76, max=0.76, sum=1.52 (2)", - "tab": "Accuracy", - "Business Ethics - Observed inference time (s)": "{\"description\": \"min=0.803, mean=0.803, max=0.803, sum=1.607 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8034474205970764\"}", - "Business Ethics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Business Ethics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Business Ethics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Business Ethics - # prompt tokens": "{\"description\": \"min=591.96, mean=591.96, max=591.96, sum=1183.92 (2)\", \"tab\": \"General information\", \"score\": \"591.96\"}", - "Business Ethics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"business_ethics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_business_ethics\"" - } - } - }, - { - "evaluation_name": "Clinical Knowledge", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Clinical Knowledge", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.74, - "details": { - "description": "min=0.74, mean=0.74, max=0.74, sum=1.479 (2)", - "tab": "Accuracy", - "Clinical Knowledge - Observed inference time (s)": "{\"description\": \"min=0.821, mean=0.821, max=0.821, sum=1.641 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8206060139638073\"}", - "Clinical Knowledge - # eval": "{\"description\": \"min=265, mean=265, max=265, sum=530 (2)\", \"tab\": \"General information\", \"score\": \"265.0\"}", - "Clinical Knowledge - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Clinical Knowledge - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Clinical Knowledge - # prompt tokens": "{\"description\": \"min=437.34, mean=437.34, max=437.34, sum=874.679 (2)\", \"tab\": \"General information\", \"score\": \"437.33962264150944\"}", - "Clinical Knowledge - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"clinical_knowledge\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_clinical_knowledge\"" - } - } - }, - { - "evaluation_name": "Conceptual Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Conceptual Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.677, - "details": { - "description": "min=0.677, mean=0.677, max=0.677, sum=1.353 (2)", - "tab": "Accuracy", - "Conceptual Physics - Observed inference time (s)": "{\"description\": \"min=0.788, mean=0.788, max=0.788, sum=1.577 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7882616854728537\"}", - "Conceptual Physics - # eval": "{\"description\": \"min=235, mean=235, max=235, sum=470 (2)\", \"tab\": \"General information\", \"score\": \"235.0\"}", - "Conceptual Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Conceptual Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Conceptual Physics - # prompt tokens": "{\"description\": \"min=322.962, mean=322.962, max=322.962, sum=645.923 (2)\", \"tab\": \"General information\", \"score\": \"322.9617021276596\"}", - "Conceptual Physics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"conceptual_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_conceptual_physics\"" - } - } - }, - { - "evaluation_name": "Electrical Engineering", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Electrical Engineering", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.683, - "details": { - "description": "min=0.683, mean=0.683, max=0.683, sum=1.366 (2)", - "tab": "Accuracy", - "Electrical Engineering - Observed inference time (s)": "{\"description\": \"min=0.8, mean=0.8, max=0.8, sum=1.6 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.800032663345337\"}", - "Electrical Engineering - # eval": "{\"description\": \"min=145, mean=145, max=145, sum=290 (2)\", \"tab\": \"General information\", \"score\": \"145.0\"}", - "Electrical Engineering - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Electrical Engineering - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Electrical Engineering - # prompt tokens": "{\"description\": \"min=494.662, mean=494.662, max=494.662, sum=989.324 (2)\", \"tab\": \"General information\", \"score\": \"494.6620689655172\"}", - "Electrical Engineering - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"electrical_engineering\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_electrical_engineering\"" - } - } - }, - { - "evaluation_name": "Elementary Mathematics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Elementary Mathematics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.553, - "details": { - "description": "min=0.553, mean=0.553, max=0.553, sum=1.106 (2)", - "tab": "Accuracy", - "Elementary Mathematics - Observed inference time (s)": "{\"description\": \"min=0.81, mean=0.81, max=0.81, sum=1.619 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8097125253980122\"}", - "Elementary Mathematics - # eval": "{\"description\": \"min=378, mean=378, max=378, sum=756 (2)\", \"tab\": \"General information\", \"score\": \"378.0\"}", - "Elementary Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Elementary Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Elementary Mathematics - # prompt tokens": "{\"description\": \"min=607.042, mean=607.042, max=607.042, sum=1214.085 (2)\", \"tab\": \"General information\", \"score\": \"607.042328042328\"}", - "Elementary Mathematics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"elementary_mathematics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_elementary_mathematics\"" - } - } - }, - { - "evaluation_name": "Formal Logic", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Formal Logic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.452, - "details": { - "description": "min=0.452, mean=0.452, max=0.452, sum=0.905 (2)", - "tab": "Accuracy", - "Formal Logic - Observed inference time (s)": "{\"description\": \"min=0.821, mean=0.821, max=0.821, sum=1.641 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8205922804181538\"}", - "Formal Logic - # eval": "{\"description\": \"min=126, mean=126, max=126, sum=252 (2)\", \"tab\": \"General information\", \"score\": \"126.0\"}", - "Formal Logic - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Formal Logic - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Formal Logic - # prompt tokens": "{\"description\": \"min=656.468, mean=656.468, max=656.468, sum=1312.937 (2)\", \"tab\": \"General information\", \"score\": \"656.468253968254\"}", - "Formal Logic - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"formal_logic\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_formal_logic\"" - } - } - }, - { - "evaluation_name": "High School World History", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on High School World History", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.84, - "details": { - "description": "min=0.84, mean=0.84, max=0.84, sum=1.679 (2)", - "tab": "Accuracy", - "High School Biology - Observed inference time (s)": "{\"description\": \"min=0.802, mean=0.802, max=0.802, sum=1.604 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8022162606639247\"}", - "High School Chemistry - Observed inference time (s)": "{\"description\": \"min=0.786, mean=0.786, max=0.786, sum=1.572 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7860349763203137\"}", - "High School Computer Science - Observed inference time (s)": "{\"description\": \"min=0.8, mean=0.8, max=0.8, sum=1.6 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7999507975578308\"}", - "High School European History - Observed inference time (s)": "{\"description\": \"min=0.891, mean=0.891, max=0.891, sum=1.782 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8912014065366802\"}", - "High School Geography - Observed inference time (s)": "{\"description\": \"min=0.943, mean=0.943, max=0.943, sum=1.887 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9434030766438957\"}", - "High School Government And Politics - Observed inference time (s)": "{\"description\": \"min=0.989, mean=0.989, max=0.989, sum=1.977 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9887206962071552\"}", - "High School Macroeconomics - Observed inference time (s)": "{\"description\": \"min=0.921, mean=0.921, max=0.921, sum=1.842 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9210334313221467\"}", - "High School Mathematics - Observed inference time (s)": "{\"description\": \"min=0.977, mean=0.977, max=0.977, sum=1.953 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.976661871097706\"}", - "High School Microeconomics - Observed inference time (s)": "{\"description\": \"min=0.914, mean=0.914, max=0.914, sum=1.828 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9139112444484935\"}", - "High School Physics - Observed inference time (s)": "{\"description\": \"min=0.933, mean=0.933, max=0.933, sum=1.866 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9328556392366523\"}", - "High School Psychology - Observed inference time (s)": "{\"description\": \"min=0.915, mean=0.915, max=0.915, sum=1.83 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9148573503581756\"}", - "High School Statistics - Observed inference time (s)": "{\"description\": \"min=0.956, mean=0.956, max=0.956, sum=1.912 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.95619613704858\"}", - "High School US History - Observed inference time (s)": "{\"description\": \"min=0.98, mean=0.98, max=0.98, sum=1.959 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9797390874694375\"}", - "High School World History - Observed inference time (s)": "{\"description\": \"min=0.996, mean=0.996, max=0.996, sum=1.991 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9955862363179525\"}", - "High School Biology - # eval": "{\"description\": \"min=310, mean=310, max=310, sum=620 (2)\", \"tab\": \"General information\", \"score\": \"310.0\"}", - "High School Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Biology - # prompt tokens": "{\"description\": \"min=532.455, mean=532.455, max=532.455, sum=1064.91 (2)\", \"tab\": \"General information\", \"score\": \"532.4548387096775\"}", - "High School Biology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # eval": "{\"description\": \"min=203, mean=203, max=203, sum=406 (2)\", \"tab\": \"General information\", \"score\": \"203.0\"}", - "High School Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # prompt tokens": "{\"description\": \"min=537.089, mean=537.089, max=537.089, sum=1074.177 (2)\", \"tab\": \"General information\", \"score\": \"537.0886699507389\"}", - "High School Chemistry - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "High School Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # prompt tokens": "{\"description\": \"min=958.39, mean=958.39, max=958.39, sum=1916.78 (2)\", \"tab\": \"General information\", \"score\": \"958.39\"}", - "High School Computer Science - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # eval": "{\"description\": \"min=165, mean=165, max=165, sum=330 (2)\", \"tab\": \"General information\", \"score\": \"165.0\"}", - "High School European History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School European History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # prompt tokens": "{\"description\": \"min=2900.673, mean=2900.673, max=2900.673, sum=5801.345 (2)\", \"tab\": \"General information\", \"score\": \"2900.672727272727\"}", - "High School European History - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # eval": "{\"description\": \"min=198, mean=198, max=198, sum=396 (2)\", \"tab\": \"General information\", \"score\": \"198.0\"}", - "High School Geography - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Geography - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # prompt tokens": "{\"description\": \"min=406.146, mean=406.146, max=406.146, sum=812.293 (2)\", \"tab\": \"General information\", \"score\": \"406.14646464646466\"}", - "High School Geography - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # eval": "{\"description\": \"min=193, mean=193, max=193, sum=386 (2)\", \"tab\": \"General information\", \"score\": \"193.0\"}", - "High School Government And Politics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Government And Politics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # prompt tokens": "{\"description\": \"min=492.788, mean=492.788, max=492.788, sum=985.575 (2)\", \"tab\": \"General information\", \"score\": \"492.78756476683935\"}", - "High School Government And Politics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # eval": "{\"description\": \"min=390, mean=390, max=390, sum=780 (2)\", \"tab\": \"General information\", \"score\": \"390.0\"}", - "High School Macroeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Macroeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # prompt tokens": "{\"description\": \"min=406.1, mean=406.1, max=406.1, sum=812.2 (2)\", \"tab\": \"General information\", \"score\": \"406.1\"}", - "High School Macroeconomics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # eval": "{\"description\": \"min=270, mean=270, max=270, sum=540 (2)\", \"tab\": \"General information\", \"score\": \"270.0\"}", - "High School Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # prompt tokens": "{\"description\": \"min=583.248, mean=583.248, max=583.248, sum=1166.496 (2)\", \"tab\": \"General information\", \"score\": \"583.2481481481482\"}", - "High School Mathematics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # eval": "{\"description\": \"min=238, mean=238, max=238, sum=476 (2)\", \"tab\": \"General information\", \"score\": \"238.0\"}", - "High School Microeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Microeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # prompt tokens": "{\"description\": \"min=426.265, mean=426.265, max=426.265, sum=852.529 (2)\", \"tab\": \"General information\", \"score\": \"426.2647058823529\"}", - "High School Microeconomics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # eval": "{\"description\": \"min=151, mean=151, max=151, sum=302 (2)\", \"tab\": \"General information\", \"score\": \"151.0\"}", - "High School Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # prompt tokens": "{\"description\": \"min=603.272, mean=603.272, max=603.272, sum=1206.543 (2)\", \"tab\": \"General information\", \"score\": \"603.2715231788079\"}", - "High School Physics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # eval": "{\"description\": \"min=545, mean=545, max=545, sum=1090 (2)\", \"tab\": \"General information\", \"score\": \"545.0\"}", - "High School Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # prompt tokens": "{\"description\": \"min=525.635, mean=525.635, max=525.635, sum=1051.27 (2)\", \"tab\": \"General information\", \"score\": \"525.6348623853211\"}", - "High School Psychology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # eval": "{\"description\": \"min=216, mean=216, max=216, sum=432 (2)\", \"tab\": \"General information\", \"score\": \"216.0\"}", - "High School Statistics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Statistics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # prompt tokens": "{\"description\": \"min=876.032, mean=876.032, max=876.032, sum=1752.065 (2)\", \"tab\": \"General information\", \"score\": \"876.0324074074074\"}", - "High School Statistics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # eval": "{\"description\": \"min=204, mean=204, max=204, sum=408 (2)\", \"tab\": \"General information\", \"score\": \"204.0\"}", - "High School US History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School US History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # prompt tokens": "{\"description\": \"min=2310.931, mean=2310.931, max=2310.931, sum=4621.863 (2)\", \"tab\": \"General information\", \"score\": \"2310.9313725490197\"}", - "High School US History - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # eval": "{\"description\": \"min=237, mean=237, max=237, sum=474 (2)\", \"tab\": \"General information\", \"score\": \"237.0\"}", - "High School World History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School World History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # prompt tokens": "{\"description\": \"min=1501.477, mean=1501.477, max=1501.477, sum=3002.954 (2)\", \"tab\": \"General information\", \"score\": \"1501.4767932489451\"}", - "High School World History - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"high_school_world_history\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_high_school_world_history\"" - } - } - }, - { - "evaluation_name": "Human Sexuality", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Human Sexuality", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.809, - "details": { - "description": "min=0.809, mean=0.809, max=0.809, sum=1.618 (2)", - "tab": "Accuracy", - "Human Aging - Observed inference time (s)": "{\"description\": \"min=0.89, mean=0.89, max=0.89, sum=1.78 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.889766787199696\"}", - "Human Sexuality - Observed inference time (s)": "{\"description\": \"min=0.928, mean=0.928, max=0.928, sum=1.856 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9282377730799085\"}", - "Human Aging - # eval": "{\"description\": \"min=223, mean=223, max=223, sum=446 (2)\", \"tab\": \"General information\", \"score\": \"223.0\"}", - "Human Aging - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Aging - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Aging - # prompt tokens": "{\"description\": \"min=333.036, mean=333.036, max=333.036, sum=666.072 (2)\", \"tab\": \"General information\", \"score\": \"333.0358744394619\"}", - "Human Aging - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # eval": "{\"description\": \"min=131, mean=131, max=131, sum=262 (2)\", \"tab\": \"General information\", \"score\": \"131.0\"}", - "Human Sexuality - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Sexuality - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # prompt tokens": "{\"description\": \"min=362.466, mean=362.466, max=362.466, sum=724.931 (2)\", \"tab\": \"General information\", \"score\": \"362.46564885496184\"}", - "Human Sexuality - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"human_sexuality\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_human_sexuality\"" - } - } - }, - { - "evaluation_name": "International Law", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on International Law", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.893, - "details": { - "description": "min=0.893, mean=0.893, max=0.893, sum=1.785 (2)", - "tab": "Accuracy", - "International Law - Observed inference time (s)": "{\"description\": \"min=0.938, mean=0.938, max=0.938, sum=1.875 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9376649265446939\"}", - "International Law - # eval": "{\"description\": \"min=121, mean=121, max=121, sum=242 (2)\", \"tab\": \"General information\", \"score\": \"121.0\"}", - "International Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "International Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "International Law - # prompt tokens": "{\"description\": \"min=662.628, mean=662.628, max=662.628, sum=1325.256 (2)\", \"tab\": \"General information\", \"score\": \"662.6280991735537\"}", - "International Law - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"international_law\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_international_law\"" - } - } - }, - { - "evaluation_name": "Logical Fallacies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Logical Fallacies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.81, - "details": { - "description": "min=0.81, mean=0.81, max=0.81, sum=1.62 (2)", - "tab": "Accuracy", - "Logical Fallacies - Observed inference time (s)": "{\"description\": \"min=0.91, mean=0.91, max=0.91, sum=1.82 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9101676209572634\"}", - "Logical Fallacies - # eval": "{\"description\": \"min=163, mean=163, max=163, sum=326 (2)\", \"tab\": \"General information\", \"score\": \"163.0\"}", - "Logical Fallacies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Logical Fallacies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Logical Fallacies - # prompt tokens": "{\"description\": \"min=466.227, mean=466.227, max=466.227, sum=932.454 (2)\", \"tab\": \"General information\", \"score\": \"466.2269938650307\"}", - "Logical Fallacies - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"logical_fallacies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_logical_fallacies\"" - } - } - }, - { - "evaluation_name": "Machine Learning", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Machine Learning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.509, - "details": { - "description": "min=0.509, mean=0.509, max=0.509, sum=1.018 (2)", - "tab": "Accuracy", - "Machine Learning - Observed inference time (s)": "{\"description\": \"min=0.936, mean=0.936, max=0.936, sum=1.873 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9363672009536198\"}", - "Machine Learning - # eval": "{\"description\": \"min=112, mean=112, max=112, sum=224 (2)\", \"tab\": \"General information\", \"score\": \"112.0\"}", - "Machine Learning - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Machine Learning - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Machine Learning - # prompt tokens": "{\"description\": \"min=719.938, mean=719.938, max=719.938, sum=1439.875 (2)\", \"tab\": \"General information\", \"score\": \"719.9375\"}", - "Machine Learning - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"machine_learning\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_machine_learning\"" - } - } - }, - { - "evaluation_name": "Management", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Management", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.825, - "details": { - "description": "min=0.825, mean=0.825, max=0.825, sum=1.65 (2)", - "tab": "Accuracy", - "Management - Observed inference time (s)": "{\"description\": \"min=1.024, mean=1.024, max=1.024, sum=2.049 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.0244285111288423\"}", - "Management - # eval": "{\"description\": \"min=103, mean=103, max=103, sum=206 (2)\", \"tab\": \"General information\", \"score\": \"103.0\"}", - "Management - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Management - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Management - # prompt tokens": "{\"description\": \"min=299.553, mean=299.553, max=299.553, sum=599.107 (2)\", \"tab\": \"General information\", \"score\": \"299.5533980582524\"}", - "Management - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"management\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_management\"" - } - } - }, - { - "evaluation_name": "Marketing", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Marketing", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.915, - "details": { - "description": "min=0.915, mean=0.915, max=0.915, sum=1.829 (2)", - "tab": "Accuracy", - "Marketing - Observed inference time (s)": "{\"description\": \"min=0.967, mean=0.967, max=0.967, sum=1.934 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9670558464832795\"}", - "Marketing - # eval": "{\"description\": \"min=234, mean=234, max=234, sum=468 (2)\", \"tab\": \"General information\", \"score\": \"234.0\"}", - "Marketing - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Marketing - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Marketing - # prompt tokens": "{\"description\": \"min=446.714, mean=446.714, max=446.714, sum=893.427 (2)\", \"tab\": \"General information\", \"score\": \"446.71367521367523\"}", - "Marketing - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"marketing\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_marketing\"" - } - } - }, - { - "evaluation_name": "Medical Genetics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Medical Genetics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.69, - "details": { - "description": "min=0.69, mean=0.69, max=0.69, sum=1.38 (2)", - "tab": "Accuracy", - "Medical Genetics - Observed inference time (s)": "{\"description\": \"min=1.001, mean=1.001, max=1.001, sum=2.002 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.0011137557029723\"}", - "Medical Genetics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Medical Genetics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Medical Genetics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Medical Genetics - # prompt tokens": "{\"description\": \"min=361.45, mean=361.45, max=361.45, sum=722.9 (2)\", \"tab\": \"General information\", \"score\": \"361.45\"}", - "Medical Genetics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"medical_genetics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_medical_genetics\"" - } - } - }, - { - "evaluation_name": "Miscellaneous", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Miscellaneous", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.902, - "details": { - "description": "min=0.902, mean=0.902, max=0.902, sum=1.803 (2)", - "tab": "Accuracy", - "Miscellaneous - Observed inference time (s)": "{\"description\": \"min=0.907, mean=0.907, max=0.907, sum=1.813 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9065530522420793\"}", - "Miscellaneous - # eval": "{\"description\": \"min=783, mean=783, max=783, sum=1566 (2)\", \"tab\": \"General information\", \"score\": \"783.0\"}", - "Miscellaneous - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Miscellaneous - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Miscellaneous - # prompt tokens": "{\"description\": \"min=332.257, mean=332.257, max=332.257, sum=664.513 (2)\", \"tab\": \"General information\", \"score\": \"332.2567049808429\"}", - "Miscellaneous - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"miscellaneous\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_miscellaneous\"" - } - } - }, - { - "evaluation_name": "Moral Scenarios", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Moral Scenarios", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.269, - "details": { - "description": "min=0.269, mean=0.269, max=0.269, sum=0.539 (2)", - "tab": "Accuracy", - "Moral Disputes - Observed inference time (s)": "{\"description\": \"min=0.799, mean=0.799, max=0.799, sum=1.599 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7992533741658823\"}", - "Moral Scenarios - Observed inference time (s)": "{\"description\": \"min=0.79, mean=0.79, max=0.79, sum=1.581 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7903663371528327\"}", - "Moral Disputes - # eval": "{\"description\": \"min=346, mean=346, max=346, sum=692 (2)\", \"tab\": \"General information\", \"score\": \"346.0\"}", - "Moral Disputes - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Disputes - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Disputes - # prompt tokens": "{\"description\": \"min=506.514, mean=506.514, max=506.514, sum=1013.029 (2)\", \"tab\": \"General information\", \"score\": \"506.514450867052\"}", - "Moral Disputes - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # eval": "{\"description\": \"min=895, mean=895, max=895, sum=1790 (2)\", \"tab\": \"General information\", \"score\": \"895.0\"}", - "Moral Scenarios - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Scenarios - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # prompt tokens": "{\"description\": \"min=709.934, mean=709.934, max=709.934, sum=1419.868 (2)\", \"tab\": \"General information\", \"score\": \"709.9340782122905\"}", - "Moral Scenarios - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"moral_scenarios\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_moral_scenarios\"" - } - } - }, - { - "evaluation_name": "Nutrition", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Nutrition", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.801, - "details": { - "description": "min=0.801, mean=0.801, max=0.801, sum=1.601 (2)", - "tab": "Accuracy", - "Nutrition - Observed inference time (s)": "{\"description\": \"min=0.799, mean=0.799, max=0.799, sum=1.599 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7992852076985477\"}", - "Nutrition - # eval": "{\"description\": \"min=306, mean=306, max=306, sum=612 (2)\", \"tab\": \"General information\", \"score\": \"306.0\"}", - "Nutrition - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Nutrition - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Nutrition - # prompt tokens": "{\"description\": \"min=619.683, mean=619.683, max=619.683, sum=1239.366 (2)\", \"tab\": \"General information\", \"score\": \"619.6830065359477\"}", - "Nutrition - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"nutrition\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_nutrition\"" - } - } - }, - { - "evaluation_name": "Prehistory", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Prehistory", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.824, - "details": { - "description": "min=0.824, mean=0.824, max=0.824, sum=1.648 (2)", - "tab": "Accuracy", - "Prehistory - Observed inference time (s)": "{\"description\": \"min=0.804, mean=0.804, max=0.804, sum=1.607 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8036901479885902\"}", - "Prehistory - # eval": "{\"description\": \"min=324, mean=324, max=324, sum=648 (2)\", \"tab\": \"General information\", \"score\": \"324.0\"}", - "Prehistory - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Prehistory - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Prehistory - # prompt tokens": "{\"description\": \"min=566.244, mean=566.244, max=566.244, sum=1132.488 (2)\", \"tab\": \"General information\", \"score\": \"566.2438271604939\"}", - "Prehistory - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"prehistory\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_prehistory\"" - } - } - }, - { - "evaluation_name": "Public Relations", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Public Relations", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.727, - "details": { - "description": "min=0.727, mean=0.727, max=0.727, sum=1.455 (2)", - "tab": "Accuracy", - "Public Relations - Observed inference time (s)": "{\"description\": \"min=0.819, mean=0.819, max=0.819, sum=1.638 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8189079783179544\"}", - "Public Relations - # eval": "{\"description\": \"min=110, mean=110, max=110, sum=220 (2)\", \"tab\": \"General information\", \"score\": \"110.0\"}", - "Public Relations - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Public Relations - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Public Relations - # prompt tokens": "{\"description\": \"min=440.6, mean=440.6, max=440.6, sum=881.2 (2)\", \"tab\": \"General information\", \"score\": \"440.6\"}", - "Public Relations - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"public_relations\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_public_relations\"" - } - } - }, - { - "evaluation_name": "Security Studies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Security Studies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.755, - "details": { - "description": "min=0.755, mean=0.755, max=0.755, sum=1.51 (2)", - "tab": "Accuracy", - "Security Studies - Observed inference time (s)": "{\"description\": \"min=0.828, mean=0.828, max=0.828, sum=1.655 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8276801226090412\"}", - "Security Studies - # eval": "{\"description\": \"min=245, mean=245, max=245, sum=490 (2)\", \"tab\": \"General information\", \"score\": \"245.0\"}", - "Security Studies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Security Studies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Security Studies - # prompt tokens": "{\"description\": \"min=1221.388, mean=1221.388, max=1221.388, sum=2442.776 (2)\", \"tab\": \"General information\", \"score\": \"1221.3877551020407\"}", - "Security Studies - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"security_studies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_security_studies\"" - } - } - }, - { - "evaluation_name": "Sociology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Sociology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.876, - "details": { - "description": "min=0.876, mean=0.876, max=0.876, sum=1.751 (2)", - "tab": "Accuracy", - "Sociology - Observed inference time (s)": "{\"description\": \"min=0.792, mean=0.792, max=0.792, sum=1.583 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7917492271062747\"}", - "Sociology - # eval": "{\"description\": \"min=201, mean=201, max=201, sum=402 (2)\", \"tab\": \"General information\", \"score\": \"201.0\"}", - "Sociology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Sociology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Sociology - # prompt tokens": "{\"description\": \"min=465.925, mean=465.925, max=465.925, sum=931.851 (2)\", \"tab\": \"General information\", \"score\": \"465.92537313432837\"}", - "Sociology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"sociology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_sociology\"" - } - } - }, - { - "evaluation_name": "Virology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Virology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.578, - "details": { - "description": "min=0.578, mean=0.578, max=0.578, sum=1.157 (2)", - "tab": "Accuracy", - "Virology - Observed inference time (s)": "{\"description\": \"min=0.78, mean=0.78, max=0.78, sum=1.559 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7796976523227003\"}", - "Virology - # eval": "{\"description\": \"min=166, mean=166, max=166, sum=332 (2)\", \"tab\": \"General information\", \"score\": \"166.0\"}", - "Virology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Virology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Virology - # prompt tokens": "{\"description\": \"min=358.048, mean=358.048, max=358.048, sum=716.096 (2)\", \"tab\": \"General information\", \"score\": \"358.04819277108436\"}", - "Virology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"virology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_virology\"" - } - } - }, - { - "evaluation_name": "World Religions", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on World Religions", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.842, - "details": { - "description": "min=0.842, mean=0.842, max=0.842, sum=1.684 (2)", - "tab": "Accuracy", - "World Religions - Observed inference time (s)": "{\"description\": \"min=0.822, mean=0.822, max=0.822, sum=1.644 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8218589679539552\"}", - "World Religions - # eval": "{\"description\": \"min=171, mean=171, max=171, sum=342 (2)\", \"tab\": \"General information\", \"score\": \"171.0\"}", - "World Religions - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "World Religions - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "World Religions - # prompt tokens": "{\"description\": \"min=293.649, mean=293.649, max=293.649, sum=587.298 (2)\", \"tab\": \"General information\", \"score\": \"293.64912280701753\"}", - "World Religions - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"world_religions\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_world_religions\"" - } - } - }, - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.206, - "details": { - "description": "", - "tab": "Efficiency" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/ai21_jamba-instruct.json b/data/models/ai21_jamba-instruct.json deleted file mode 100644 index 497d65e58ae07d5e90b8c141ed1af87f5d554d47..0000000000000000000000000000000000000000 --- a/data/models/ai21_jamba-instruct.json +++ /dev/null @@ -1,1903 +0,0 @@ -{ - "model_info": { - "name": "Jamba Instruct", - "id": "ai21/jamba-instruct", - "developer": "ai21", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_lite/ai21_jamba-instruct/1774096306.427425", - "retrieved_timestamp": "1774096306.427425", - "source_metadata": { - "source_name": "helm_lite", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_lite", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_lite", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.287, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.6515730337078651\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.658, - "details": { - "description": "min=0.658, mean=0.658, max=0.658, sum=0.658 (1)", - "tab": "Accuracy", - "NarrativeQA - Observed inference time (s)": "{\"description\": \"min=0.947, mean=0.947, max=0.947, sum=0.947 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.9470622405199938\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=2555.434, mean=2555.434, max=2555.434, sum=2555.434 (1)\", \"tab\": \"General information\", \"score\": \"2555.4338028169013\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (closed-book)", - "source_data": { - "dataset_name": "NaturalQuestions (closed-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (closed-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.384, - "details": { - "description": "min=0.384, mean=0.384, max=0.384, sum=0.384 (1)", - "tab": "Accuracy", - "NaturalQuestions (open-book) - Observed inference time (s)": "{\"description\": \"min=0.809, mean=0.809, max=0.809, sum=0.809 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.8087365460395813\"}", - "NaturalQuestions (closed-book) - Observed inference time (s)": "{\"description\": \"min=0.535, mean=0.535, max=0.535, sum=0.535 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.5348668487071991\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1774.04, mean=1774.04, max=1774.04, sum=1774.04 (1)\", \"tab\": \"General information\", \"score\": \"1774.04\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=118.377, mean=118.377, max=118.377, sum=118.377 (1)\", \"tab\": \"General information\", \"score\": \"118.377\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "mode": "\"closedbook\"" - } - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.796, - "details": { - "description": "min=0.796, mean=0.796, max=0.796, sum=0.796 (1)", - "tab": "Accuracy", - "OpenbookQA - Observed inference time (s)": "{\"description\": \"min=0.3, mean=0.3, max=0.3, sum=0.3 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.30006033515930175\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=195.75, mean=195.75, max=195.75, sum=195.75 (1)\", \"tab\": \"General information\", \"score\": \"195.75\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "dataset": "\"openbookqa\"", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.582, - "details": { - "description": "min=0.36, mean=0.582, max=0.91, sum=2.909 (5)", - "tab": "Accuracy", - "MMLU - Observed inference time (s)": "{\"description\": \"min=0.253, mean=0.265, max=0.275, sum=1.327 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.2654710942151254\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=315.59, mean=403.74, max=559.719, sum=2018.699 (5)\", \"tab\": \"General information\", \"score\": \"403.7398596491228\"}", - "MMLU - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"college_chemistry\", \"computer_security\", \"econometrics\", \"us_foreign_policy\"]", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MATH", - "source_data": { - "dataset_name": "MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Equivalent (CoT) on MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.38, - "details": { - "description": "min=0.237, mean=0.38, max=0.607, sum=2.663 (7)", - "tab": "Accuracy", - "MATH - Observed inference time (s)": "{\"description\": \"min=1.917, mean=3.242, max=5.09, sum=22.692 (7)\", \"tab\": \"Efficiency\", \"score\": \"3.24175411841349\"}", - "MATH - # eval": "{\"description\": \"min=30, mean=62.429, max=135, sum=437 (7)\", \"tab\": \"General information\", \"score\": \"62.42857142857143\"}", - "MATH - # train": "{\"description\": \"min=8, mean=8, max=8, sum=56 (7)\", \"tab\": \"General information\", \"score\": \"8.0\"}", - "MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MATH - # prompt tokens": "{\"description\": \"min=796.795, mean=1321.422, max=2516.154, sum=9249.956 (7)\", \"tab\": \"General information\", \"score\": \"1321.42226282263\"}", - "MATH - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"algebra\", \"counting_and_probability\", \"geometry\", \"intermediate_algebra\", \"number_theory\", \"prealgebra\", \"precalculus\"]", - "level": "\"1\"", - "use_official_examples": "\"False\"", - "use_chain_of_thought": "\"True\"" - } - } - }, - { - "evaluation_name": "GSM8K", - "source_data": { - "dataset_name": "GSM8K", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on GSM8K", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.67, - "details": { - "description": "min=0.67, mean=0.67, max=0.67, sum=0.67 (1)", - "tab": "Accuracy", - "GSM8K - Observed inference time (s)": "{\"description\": \"min=3.846, mean=3.846, max=3.846, sum=3.846 (1)\", \"tab\": \"Efficiency\", \"score\": \"3.8455032846927644\"}", - "GSM8K - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "GSM8K - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "GSM8K - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GSM8K - # prompt tokens": "{\"description\": \"min=823.394, mean=823.394, max=823.394, sum=823.394 (1)\", \"tab\": \"General information\", \"score\": \"823.394\"}", - "GSM8K - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "stop": "\"none\"" - } - } - }, - { - "evaluation_name": "LegalBench", - "source_data": { - "dataset_name": "LegalBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on LegalBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.54, - "details": { - "description": "min=0.304, mean=0.54, max=0.874, sum=2.7 (5)", - "tab": "Accuracy", - "LegalBench - Observed inference time (s)": "{\"description\": \"min=0.351, mean=0.641, max=1.337, sum=3.204 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.6408480782672099\"}", - "LegalBench - # eval": "{\"description\": \"min=95, mean=409.4, max=1000, sum=2047 (5)\", \"tab\": \"General information\", \"score\": \"409.4\"}", - "LegalBench - # train": "{\"description\": \"min=4, mean=4.8, max=5, sum=24 (5)\", \"tab\": \"General information\", \"score\": \"4.8\"}", - "LegalBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "LegalBench - # prompt tokens": "{\"description\": \"min=177.042, mean=1127.163, max=4612.308, sum=5635.817 (5)\", \"tab\": \"General information\", \"score\": \"1127.1634769381612\"}", - "LegalBench - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"abercrombie\", \"corporate_lobbying\", \"function_of_decision_section\", \"international_citizenship_questions\", \"proa\"]" - } - } - }, - { - "evaluation_name": "MedQA", - "source_data": { - "dataset_name": "MedQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MedQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.519, - "details": { - "description": "min=0.519, mean=0.519, max=0.519, sum=0.519 (1)", - "tab": "Accuracy", - "MedQA - Observed inference time (s)": "{\"description\": \"min=0.311, mean=0.311, max=0.311, sum=0.311 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.31133864366747516\"}", - "MedQA - # eval": "{\"description\": \"min=503, mean=503, max=503, sum=503 (1)\", \"tab\": \"General information\", \"score\": \"503.0\"}", - "MedQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MedQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MedQA - # prompt tokens": "{\"description\": \"min=765.622, mean=765.622, max=765.622, sum=765.622 (1)\", \"tab\": \"General information\", \"score\": \"765.6222664015904\"}", - "MedQA - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WMT 2014", - "source_data": { - "dataset_name": "WMT 2014", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "BLEU-4 on WMT 2014", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.164, - "details": { - "description": "min=0.099, mean=0.164, max=0.205, sum=0.656 (4)", - "tab": "Accuracy", - "WMT 2014 - Observed inference time (s)": "{\"description\": \"min=0.586, mean=0.635, max=0.686, sum=2.542 (4)\", \"tab\": \"Efficiency\", \"score\": \"0.6354023076110767\"}", - "WMT 2014 - # eval": "{\"description\": \"min=503, mean=585.25, max=832, sum=2341 (4)\", \"tab\": \"General information\", \"score\": \"585.25\"}", - "WMT 2014 - # train": "{\"description\": \"min=1, mean=1, max=1, sum=4 (4)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "WMT 2014 - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (4)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WMT 2014 - # prompt tokens": "{\"description\": \"min=129.229, mean=143.261, max=154.278, sum=573.045 (4)\", \"tab\": \"General information\", \"score\": \"143.26129939115307\"}", - "WMT 2014 - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (4)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "language_pair": "[\"cs-en\", \"de-en\", \"hi-en\", \"ru-en\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_mmlu/ai21_jamba-instruct/1774096312.00548", - "retrieved_timestamp": "1774096312.00548", - "source_metadata": { - "source_name": "helm_mmlu", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_mmlu", - "evaluation_results": [ - { - "evaluation_name": "MMLU All Subjects", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU All Subjects", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.659, - "details": { - "description": "min=0.341, mean=0.659, max=0.91, sum=75.114 (114)", - "tab": "Accuracy", - "MMLU All Subjects - Observed inference time (s)": "{\"description\": \"min=0.233, mean=0.277, max=0.519, sum=31.585 (114)\", \"tab\": \"Efficiency\", \"score\": \"0.2770578114829593\"}", - "MMLU All Subjects - # eval": "{\"description\": \"min=100, mean=246.351, max=1534, sum=28084 (114)\", \"tab\": \"General information\", \"score\": \"246.35087719298247\"}", - "MMLU All Subjects - # train": "{\"description\": \"min=5, mean=5, max=5, sum=570 (114)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU All Subjects - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU All Subjects - # prompt tokens": "{\"description\": \"min=223.731, mean=490.686, max=2081.679, sum=55938.26 (114)\", \"tab\": \"General information\", \"score\": \"490.6864895752317\"}", - "MMLU All Subjects - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_name": "Abstract Algebra", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Abstract Algebra", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.36, - "details": { - "description": "min=0.36, mean=0.36, max=0.36, sum=0.72 (2)", - "tab": "Accuracy", - "Abstract Algebra - Observed inference time (s)": "{\"description\": \"min=0.275, mean=0.275, max=0.275, sum=0.55 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.27479029655456544\"}", - "Abstract Algebra - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Abstract Algebra - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Abstract Algebra - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Abstract Algebra - # prompt tokens": "{\"description\": \"min=373.44, mean=373.44, max=373.44, sum=746.88 (2)\", \"tab\": \"General information\", \"score\": \"373.44\"}", - "Abstract Algebra - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"abstract_algebra\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_abstract_algebra\"" - } - } - }, - { - "evaluation_name": "Anatomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Anatomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.615, - "details": { - "description": "min=0.615, mean=0.615, max=0.615, sum=1.23 (2)", - "tab": "Accuracy", - "Anatomy - Observed inference time (s)": "{\"description\": \"min=0.236, mean=0.236, max=0.236, sum=0.473 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2363892325648555\"}", - "Anatomy - # eval": "{\"description\": \"min=135, mean=135, max=135, sum=270 (2)\", \"tab\": \"General information\", \"score\": \"135.0\"}", - "Anatomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Anatomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Anatomy - # prompt tokens": "{\"description\": \"min=270.2, mean=270.2, max=270.2, sum=540.4 (2)\", \"tab\": \"General information\", \"score\": \"270.2\"}", - "Anatomy - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"anatomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_anatomy\"" - } - } - }, - { - "evaluation_name": "College Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on College Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.422, - "details": { - "description": "min=0.422, mean=0.422, max=0.422, sum=0.843 (2)", - "tab": "Accuracy", - "College Chemistry - Observed inference time (s)": "{\"description\": \"min=0.275, mean=0.275, max=0.275, sum=0.55 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2747657370567322\"}", - "College Biology - Observed inference time (s)": "{\"description\": \"min=0.26, mean=0.26, max=0.26, sum=0.519 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2595776534742779\"}", - "College Computer Science - Observed inference time (s)": "{\"description\": \"min=0.294, mean=0.294, max=0.294, sum=0.588 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2938127589225769\"}", - "College Mathematics - Observed inference time (s)": "{\"description\": \"min=0.269, mean=0.269, max=0.269, sum=0.538 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.26912292957305906\"}", - "College Medicine - Observed inference time (s)": "{\"description\": \"min=0.309, mean=0.309, max=0.309, sum=0.618 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.30890216579327007\"}", - "College Physics - Observed inference time (s)": "{\"description\": \"min=0.374, mean=0.374, max=0.374, sum=0.749 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.374276315464693\"}", - "College Chemistry - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Chemistry - # prompt tokens": "{\"description\": \"min=549.4, mean=549.4, max=549.4, sum=1098.8 (2)\", \"tab\": \"General information\", \"score\": \"549.4\"}", - "College Chemistry - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # eval": "{\"description\": \"min=144, mean=144, max=144, sum=288 (2)\", \"tab\": \"General information\", \"score\": \"144.0\"}", - "College Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # prompt tokens": "{\"description\": \"min=363.431, mean=363.431, max=363.431, sum=726.861 (2)\", \"tab\": \"General information\", \"score\": \"363.43055555555554\"}", - "College Biology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # prompt tokens": "{\"description\": \"min=720.67, mean=720.67, max=720.67, sum=1441.34 (2)\", \"tab\": \"General information\", \"score\": \"720.67\"}", - "College Computer Science - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # prompt tokens": "{\"description\": \"min=535.22, mean=535.22, max=535.22, sum=1070.44 (2)\", \"tab\": \"General information\", \"score\": \"535.22\"}", - "College Mathematics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # eval": "{\"description\": \"min=173, mean=173, max=173, sum=346 (2)\", \"tab\": \"General information\", \"score\": \"173.0\"}", - "College Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # prompt tokens": "{\"description\": \"min=397.855, mean=397.855, max=397.855, sum=795.711 (2)\", \"tab\": \"General information\", \"score\": \"397.8554913294798\"}", - "College Medicine - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # eval": "{\"description\": \"min=102, mean=102, max=102, sum=204 (2)\", \"tab\": \"General information\", \"score\": \"102.0\"}", - "College Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # prompt tokens": "{\"description\": \"min=392.598, mean=392.598, max=392.598, sum=785.196 (2)\", \"tab\": \"General information\", \"score\": \"392.5980392156863\"}", - "College Physics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"college_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_college_physics\"" - } - } - }, - { - "evaluation_name": "Computer Security", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Computer Security", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.76, - "details": { - "description": "min=0.76, mean=0.76, max=0.76, sum=1.52 (2)", - "tab": "Accuracy", - "Computer Security - Observed inference time (s)": "{\"description\": \"min=0.253, mean=0.253, max=0.253, sum=0.506 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2529018998146057\"}", - "Computer Security - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Computer Security - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Computer Security - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Computer Security - # prompt tokens": "{\"description\": \"min=378.54, mean=378.54, max=378.54, sum=757.08 (2)\", \"tab\": \"General information\", \"score\": \"378.54\"}", - "Computer Security - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"computer_security\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_computer_security\"" - } - } - }, - { - "evaluation_name": "Econometrics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Econometrics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.439, - "details": { - "description": "min=0.439, mean=0.439, max=0.439, sum=0.877 (2)", - "tab": "Accuracy", - "Econometrics - Observed inference time (s)": "{\"description\": \"min=0.254, mean=0.254, max=0.254, sum=0.507 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.25371592086658146\"}", - "Econometrics - # eval": "{\"description\": \"min=114, mean=114, max=114, sum=228 (2)\", \"tab\": \"General information\", \"score\": \"114.0\"}", - "Econometrics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Econometrics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Econometrics - # prompt tokens": "{\"description\": \"min=614.43, mean=614.43, max=614.43, sum=1228.86 (2)\", \"tab\": \"General information\", \"score\": \"614.4298245614035\"}", - "Econometrics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"econometrics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_econometrics\"" - } - } - }, - { - "evaluation_name": "Global Facts", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Global Facts", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4, - "details": { - "description": "min=0.4, mean=0.4, max=0.4, sum=0.8 (2)", - "tab": "Accuracy", - "Global Facts - Observed inference time (s)": "{\"description\": \"min=0.257, mean=0.257, max=0.257, sum=0.514 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.25686686754226684\"}", - "Global Facts - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Global Facts - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Global Facts - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Global Facts - # prompt tokens": "{\"description\": \"min=329.71, mean=329.71, max=329.71, sum=659.42 (2)\", \"tab\": \"General information\", \"score\": \"329.71\"}", - "Global Facts - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"global_facts\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_global_facts\"" - } - } - }, - { - "evaluation_name": "Jurisprudence", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Jurisprudence", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.796, - "details": { - "description": "min=0.796, mean=0.796, max=0.796, sum=1.593 (2)", - "tab": "Accuracy", - "Jurisprudence - Observed inference time (s)": "{\"description\": \"min=0.26, mean=0.26, max=0.26, sum=0.521 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.260397990544637\"}", - "Jurisprudence - # eval": "{\"description\": \"min=108, mean=108, max=108, sum=216 (2)\", \"tab\": \"General information\", \"score\": \"108.0\"}", - "Jurisprudence - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Jurisprudence - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Jurisprudence - # prompt tokens": "{\"description\": \"min=312.287, mean=312.287, max=312.287, sum=624.574 (2)\", \"tab\": \"General information\", \"score\": \"312.287037037037\"}", - "Jurisprudence - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"jurisprudence\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_jurisprudence\"" - } - } - }, - { - "evaluation_name": "Philosophy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Philosophy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.749, - "details": { - "description": "min=0.749, mean=0.749, max=0.749, sum=1.498 (2)", - "tab": "Accuracy", - "Philosophy - Observed inference time (s)": "{\"description\": \"min=0.252, mean=0.252, max=0.252, sum=0.504 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.25189057270430293\"}", - "Philosophy - # eval": "{\"description\": \"min=311, mean=311, max=311, sum=622 (2)\", \"tab\": \"General information\", \"score\": \"311.0\"}", - "Philosophy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Philosophy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Philosophy - # prompt tokens": "{\"description\": \"min=267.441, mean=267.441, max=267.441, sum=534.881 (2)\", \"tab\": \"General information\", \"score\": \"267.4405144694534\"}", - "Philosophy - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"philosophy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_philosophy\"" - } - } - }, - { - "evaluation_name": "Professional Psychology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Professional Psychology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.716, - "details": { - "description": "min=0.716, mean=0.716, max=0.716, sum=1.431 (2)", - "tab": "Accuracy", - "Professional Medicine - Observed inference time (s)": "{\"description\": \"min=0.308, mean=0.308, max=0.308, sum=0.616 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.30818068542901206\"}", - "Professional Accounting - Observed inference time (s)": "{\"description\": \"min=0.266, mean=0.266, max=0.266, sum=0.532 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.26598995881723175\"}", - "Professional Law - Observed inference time (s)": "{\"description\": \"min=0.365, mean=0.365, max=0.365, sum=0.73 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.36489380229716195\"}", - "Professional Psychology - Observed inference time (s)": "{\"description\": \"min=0.255, mean=0.255, max=0.255, sum=0.511 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.25544750768374774\"}", - "Professional Medicine - # eval": "{\"description\": \"min=272, mean=272, max=272, sum=544 (2)\", \"tab\": \"General information\", \"score\": \"272.0\"}", - "Professional Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Medicine - # prompt tokens": "{\"description\": \"min=813.651, mean=813.651, max=813.651, sum=1627.301 (2)\", \"tab\": \"General information\", \"score\": \"813.6507352941177\"}", - "Professional Medicine - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # eval": "{\"description\": \"min=282, mean=282, max=282, sum=564 (2)\", \"tab\": \"General information\", \"score\": \"282.0\"}", - "Professional Accounting - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Accounting - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # prompt tokens": "{\"description\": \"min=555.461, mean=555.461, max=555.461, sum=1110.922 (2)\", \"tab\": \"General information\", \"score\": \"555.4609929078014\"}", - "Professional Accounting - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # eval": "{\"description\": \"min=1534, mean=1534, max=1534, sum=3068 (2)\", \"tab\": \"General information\", \"score\": \"1534.0\"}", - "Professional Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # prompt tokens": "{\"description\": \"min=1151.508, mean=1151.508, max=1151.508, sum=2303.016 (2)\", \"tab\": \"General information\", \"score\": \"1151.5078226857888\"}", - "Professional Law - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # eval": "{\"description\": \"min=612, mean=612, max=612, sum=1224 (2)\", \"tab\": \"General information\", \"score\": \"612.0\"}", - "Professional Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # prompt tokens": "{\"description\": \"min=422.158, mean=422.158, max=422.158, sum=844.317 (2)\", \"tab\": \"General information\", \"score\": \"422.15849673202615\"}", - "Professional Psychology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"professional_psychology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_professional_psychology\"" - } - } - }, - { - "evaluation_name": "Us Foreign Policy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Us Foreign Policy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.91, - "details": { - "description": "min=0.91, mean=0.91, max=0.91, sum=1.82 (2)", - "tab": "Accuracy", - "Us Foreign Policy - Observed inference time (s)": "{\"description\": \"min=0.271, mean=0.271, max=0.271, sum=0.542 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.27118161678314207\"}", - "Us Foreign Policy - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Us Foreign Policy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Us Foreign Policy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Us Foreign Policy - # prompt tokens": "{\"description\": \"min=422.79, mean=422.79, max=422.79, sum=845.58 (2)\", \"tab\": \"General information\", \"score\": \"422.79\"}", - "Us Foreign Policy - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"us_foreign_policy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_us_foreign_policy\"" - } - } - }, - { - "evaluation_name": "Astronomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Astronomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.73, - "details": { - "description": "min=0.73, mean=0.73, max=0.73, sum=1.461 (2)", - "tab": "Accuracy", - "Astronomy - Observed inference time (s)": "{\"description\": \"min=0.276, mean=0.276, max=0.276, sum=0.553 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.27634719171022115\"}", - "Astronomy - # eval": "{\"description\": \"min=152, mean=152, max=152, sum=304 (2)\", \"tab\": \"General information\", \"score\": \"152.0\"}", - "Astronomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Astronomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Astronomy - # prompt tokens": "{\"description\": \"min=440.612, mean=440.612, max=440.612, sum=881.224 (2)\", \"tab\": \"General information\", \"score\": \"440.6118421052632\"}", - "Astronomy - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"astronomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_astronomy\"" - } - } - }, - { - "evaluation_name": "Business Ethics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Business Ethics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6, - "details": { - "description": "min=0.6, mean=0.6, max=0.6, sum=1.2 (2)", - "tab": "Accuracy", - "Business Ethics - Observed inference time (s)": "{\"description\": \"min=0.267, mean=0.267, max=0.267, sum=0.533 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2665403389930725\"}", - "Business Ethics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Business Ethics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Business Ethics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Business Ethics - # prompt tokens": "{\"description\": \"min=521.13, mean=521.13, max=521.13, sum=1042.26 (2)\", \"tab\": \"General information\", \"score\": \"521.13\"}", - "Business Ethics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"business_ethics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_business_ethics\"" - } - } - }, - { - "evaluation_name": "Clinical Knowledge", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Clinical Knowledge", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.702, - "details": { - "description": "min=0.702, mean=0.702, max=0.702, sum=1.404 (2)", - "tab": "Accuracy", - "Clinical Knowledge - Observed inference time (s)": "{\"description\": \"min=0.259, mean=0.259, max=0.259, sum=0.517 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.25872870661177727\"}", - "Clinical Knowledge - # eval": "{\"description\": \"min=265, mean=265, max=265, sum=530 (2)\", \"tab\": \"General information\", \"score\": \"265.0\"}", - "Clinical Knowledge - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Clinical Knowledge - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Clinical Knowledge - # prompt tokens": "{\"description\": \"min=317.268, mean=317.268, max=317.268, sum=634.536 (2)\", \"tab\": \"General information\", \"score\": \"317.2679245283019\"}", - "Clinical Knowledge - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"clinical_knowledge\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_clinical_knowledge\"" - } - } - }, - { - "evaluation_name": "Conceptual Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Conceptual Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.677, - "details": { - "description": "min=0.677, mean=0.677, max=0.677, sum=1.353 (2)", - "tab": "Accuracy", - "Conceptual Physics - Observed inference time (s)": "{\"description\": \"min=0.306, mean=0.306, max=0.306, sum=0.613 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.30636518965376186\"}", - "Conceptual Physics - # eval": "{\"description\": \"min=235, mean=235, max=235, sum=470 (2)\", \"tab\": \"General information\", \"score\": \"235.0\"}", - "Conceptual Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Conceptual Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Conceptual Physics - # prompt tokens": "{\"description\": \"min=241.511, mean=241.511, max=241.511, sum=483.021 (2)\", \"tab\": \"General information\", \"score\": \"241.51063829787233\"}", - "Conceptual Physics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"conceptual_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_conceptual_physics\"" - } - } - }, - { - "evaluation_name": "Electrical Engineering", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Electrical Engineering", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.621, - "details": { - "description": "min=0.621, mean=0.621, max=0.621, sum=1.241 (2)", - "tab": "Accuracy", - "Electrical Engineering - Observed inference time (s)": "{\"description\": \"min=0.412, mean=0.412, max=0.412, sum=0.825 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.41247522255470015\"}", - "Electrical Engineering - # eval": "{\"description\": \"min=145, mean=145, max=145, sum=290 (2)\", \"tab\": \"General information\", \"score\": \"145.0\"}", - "Electrical Engineering - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Electrical Engineering - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Electrical Engineering - # prompt tokens": "{\"description\": \"min=382.393, mean=382.393, max=382.393, sum=764.786 (2)\", \"tab\": \"General information\", \"score\": \"382.39310344827584\"}", - "Electrical Engineering - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"electrical_engineering\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_electrical_engineering\"" - } - } - }, - { - "evaluation_name": "Elementary Mathematics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Elementary Mathematics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.497, - "details": { - "description": "min=0.497, mean=0.497, max=0.497, sum=0.995 (2)", - "tab": "Accuracy", - "Elementary Mathematics - Observed inference time (s)": "{\"description\": \"min=0.259, mean=0.259, max=0.259, sum=0.517 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2586819948973479\"}", - "Elementary Mathematics - # eval": "{\"description\": \"min=378, mean=378, max=378, sum=756 (2)\", \"tab\": \"General information\", \"score\": \"378.0\"}", - "Elementary Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Elementary Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Elementary Mathematics - # prompt tokens": "{\"description\": \"min=467.987, mean=467.987, max=467.987, sum=935.974 (2)\", \"tab\": \"General information\", \"score\": \"467.9867724867725\"}", - "Elementary Mathematics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"elementary_mathematics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_elementary_mathematics\"" - } - } - }, - { - "evaluation_name": "Formal Logic", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Formal Logic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.444, - "details": { - "description": "min=0.444, mean=0.444, max=0.444, sum=0.889 (2)", - "tab": "Accuracy", - "Formal Logic - Observed inference time (s)": "{\"description\": \"min=0.263, mean=0.263, max=0.263, sum=0.526 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2629187542294699\"}", - "Formal Logic - # eval": "{\"description\": \"min=126, mean=126, max=126, sum=252 (2)\", \"tab\": \"General information\", \"score\": \"126.0\"}", - "Formal Logic - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Formal Logic - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Formal Logic - # prompt tokens": "{\"description\": \"min=559.865, mean=559.865, max=559.865, sum=1119.73 (2)\", \"tab\": \"General information\", \"score\": \"559.8650793650794\"}", - "Formal Logic - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"formal_logic\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_formal_logic\"" - } - } - }, - { - "evaluation_name": "High School World History", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on High School World History", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.797, - "details": { - "description": "min=0.797, mean=0.797, max=0.797, sum=1.595 (2)", - "tab": "Accuracy", - "High School Biology - Observed inference time (s)": "{\"description\": \"min=0.256, mean=0.256, max=0.256, sum=0.513 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.25630061088069794\"}", - "High School Chemistry - Observed inference time (s)": "{\"description\": \"min=0.259, mean=0.259, max=0.259, sum=0.519 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2594739521665526\"}", - "High School Computer Science - Observed inference time (s)": "{\"description\": \"min=0.294, mean=0.294, max=0.294, sum=0.588 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.29399110078811647\"}", - "High School European History - Observed inference time (s)": "{\"description\": \"min=0.519, mean=0.519, max=0.519, sum=1.039 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5194540543989702\"}", - "High School Geography - Observed inference time (s)": "{\"description\": \"min=0.25, mean=0.25, max=0.25, sum=0.5 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.24992815051415954\"}", - "High School Government And Politics - Observed inference time (s)": "{\"description\": \"min=0.242, mean=0.242, max=0.242, sum=0.484 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.242088835474123\"}", - "High School Macroeconomics - Observed inference time (s)": "{\"description\": \"min=0.24, mean=0.24, max=0.24, sum=0.481 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.240464658003587\"}", - "High School Mathematics - Observed inference time (s)": "{\"description\": \"min=0.252, mean=0.252, max=0.252, sum=0.503 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.25154934459262424\"}", - "High School Microeconomics - Observed inference time (s)": "{\"description\": \"min=0.25, mean=0.25, max=0.25, sum=0.501 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.25046268931957855\"}", - "High School Physics - Observed inference time (s)": "{\"description\": \"min=0.256, mean=0.256, max=0.256, sum=0.511 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.25560809444907484\"}", - "High School Psychology - Observed inference time (s)": "{\"description\": \"min=0.251, mean=0.251, max=0.251, sum=0.501 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.250657169971991\"}", - "High School Statistics - Observed inference time (s)": "{\"description\": \"min=0.282, mean=0.282, max=0.282, sum=0.564 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2818450938772272\"}", - "High School US History - Observed inference time (s)": "{\"description\": \"min=0.45, mean=0.45, max=0.45, sum=0.9 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.44991188072690774\"}", - "High School World History - Observed inference time (s)": "{\"description\": \"min=0.347, mean=0.347, max=0.347, sum=0.693 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3466388042466047\"}", - "High School Biology - # eval": "{\"description\": \"min=310, mean=310, max=310, sum=620 (2)\", \"tab\": \"General information\", \"score\": \"310.0\"}", - "High School Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Biology - # prompt tokens": "{\"description\": \"min=380.871, mean=380.871, max=380.871, sum=761.742 (2)\", \"tab\": \"General information\", \"score\": \"380.8709677419355\"}", - "High School Biology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # eval": "{\"description\": \"min=203, mean=203, max=203, sum=406 (2)\", \"tab\": \"General information\", \"score\": \"203.0\"}", - "High School Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # prompt tokens": "{\"description\": \"min=401.734, mean=401.734, max=401.734, sum=803.468 (2)\", \"tab\": \"General information\", \"score\": \"401.73399014778323\"}", - "High School Chemistry - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "High School Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # prompt tokens": "{\"description\": \"min=793.8, mean=793.8, max=793.8, sum=1587.6 (2)\", \"tab\": \"General information\", \"score\": \"793.8\"}", - "High School Computer Science - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # eval": "{\"description\": \"min=165, mean=165, max=165, sum=330 (2)\", \"tab\": \"General information\", \"score\": \"165.0\"}", - "High School European History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School European History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # prompt tokens": "{\"description\": \"min=2081.679, mean=2081.679, max=2081.679, sum=4163.358 (2)\", \"tab\": \"General information\", \"score\": \"2081.6787878787877\"}", - "High School European History - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # eval": "{\"description\": \"min=198, mean=198, max=198, sum=396 (2)\", \"tab\": \"General information\", \"score\": \"198.0\"}", - "High School Geography - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Geography - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # prompt tokens": "{\"description\": \"min=299.717, mean=299.717, max=299.717, sum=599.434 (2)\", \"tab\": \"General information\", \"score\": \"299.7171717171717\"}", - "High School Geography - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # eval": "{\"description\": \"min=193, mean=193, max=193, sum=386 (2)\", \"tab\": \"General information\", \"score\": \"193.0\"}", - "High School Government And Politics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Government And Politics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # prompt tokens": "{\"description\": \"min=333.601, mean=333.601, max=333.601, sum=667.202 (2)\", \"tab\": \"General information\", \"score\": \"333.60103626943004\"}", - "High School Government And Politics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # eval": "{\"description\": \"min=390, mean=390, max=390, sum=780 (2)\", \"tab\": \"General information\", \"score\": \"390.0\"}", - "High School Macroeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Macroeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # prompt tokens": "{\"description\": \"min=286.562, mean=286.562, max=286.562, sum=573.123 (2)\", \"tab\": \"General information\", \"score\": \"286.5615384615385\"}", - "High School Macroeconomics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # eval": "{\"description\": \"min=270, mean=270, max=270, sum=540 (2)\", \"tab\": \"General information\", \"score\": \"270.0\"}", - "High School Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # prompt tokens": "{\"description\": \"min=421.889, mean=421.889, max=421.889, sum=843.778 (2)\", \"tab\": \"General information\", \"score\": \"421.8888888888889\"}", - "High School Mathematics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # eval": "{\"description\": \"min=238, mean=238, max=238, sum=476 (2)\", \"tab\": \"General information\", \"score\": \"238.0\"}", - "High School Microeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Microeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # prompt tokens": "{\"description\": \"min=301.231, mean=301.231, max=301.231, sum=602.462 (2)\", \"tab\": \"General information\", \"score\": \"301.2310924369748\"}", - "High School Microeconomics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # eval": "{\"description\": \"min=151, mean=151, max=151, sum=302 (2)\", \"tab\": \"General information\", \"score\": \"151.0\"}", - "High School Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # prompt tokens": "{\"description\": \"min=453.51, mean=453.51, max=453.51, sum=907.02 (2)\", \"tab\": \"General information\", \"score\": \"453.50993377483445\"}", - "High School Physics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # eval": "{\"description\": \"min=545, mean=545, max=545, sum=1090 (2)\", \"tab\": \"General information\", \"score\": \"545.0\"}", - "High School Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # prompt tokens": "{\"description\": \"min=355.059, mean=355.059, max=355.059, sum=710.117 (2)\", \"tab\": \"General information\", \"score\": \"355.0587155963303\"}", - "High School Psychology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # eval": "{\"description\": \"min=216, mean=216, max=216, sum=432 (2)\", \"tab\": \"General information\", \"score\": \"216.0\"}", - "High School Statistics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Statistics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # prompt tokens": "{\"description\": \"min=648.037, mean=648.037, max=648.037, sum=1296.074 (2)\", \"tab\": \"General information\", \"score\": \"648.0370370370371\"}", - "High School Statistics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # eval": "{\"description\": \"min=204, mean=204, max=204, sum=408 (2)\", \"tab\": \"General information\", \"score\": \"204.0\"}", - "High School US History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School US History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # prompt tokens": "{\"description\": \"min=1628.495, mean=1628.495, max=1628.495, sum=3256.99 (2)\", \"tab\": \"General information\", \"score\": \"1628.4950980392157\"}", - "High School US History - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # eval": "{\"description\": \"min=237, mean=237, max=237, sum=474 (2)\", \"tab\": \"General information\", \"score\": \"237.0\"}", - "High School World History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School World History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # prompt tokens": "{\"description\": \"min=1025.097, mean=1025.097, max=1025.097, sum=2050.194 (2)\", \"tab\": \"General information\", \"score\": \"1025.097046413502\"}", - "High School World History - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"high_school_world_history\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_high_school_world_history\"" - } - } - }, - { - "evaluation_name": "Human Sexuality", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Human Sexuality", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.794, - "details": { - "description": "min=0.794, mean=0.794, max=0.794, sum=1.588 (2)", - "tab": "Accuracy", - "Human Aging - Observed inference time (s)": "{\"description\": \"min=0.233, mean=0.233, max=0.233, sum=0.466 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2328128023532474\"}", - "Human Sexuality - Observed inference time (s)": "{\"description\": \"min=0.251, mean=0.251, max=0.251, sum=0.501 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2506928462108583\"}", - "Human Aging - # eval": "{\"description\": \"min=223, mean=223, max=223, sum=446 (2)\", \"tab\": \"General information\", \"score\": \"223.0\"}", - "Human Aging - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Aging - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Aging - # prompt tokens": "{\"description\": \"min=250.915, mean=250.915, max=250.915, sum=501.83 (2)\", \"tab\": \"General information\", \"score\": \"250.91479820627802\"}", - "Human Aging - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # eval": "{\"description\": \"min=131, mean=131, max=131, sum=262 (2)\", \"tab\": \"General information\", \"score\": \"131.0\"}", - "Human Sexuality - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Sexuality - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # prompt tokens": "{\"description\": \"min=263.183, mean=263.183, max=263.183, sum=526.366 (2)\", \"tab\": \"General information\", \"score\": \"263.1832061068702\"}", - "Human Sexuality - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"human_sexuality\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_human_sexuality\"" - } - } - }, - { - "evaluation_name": "International Law", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on International Law", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.835, - "details": { - "description": "min=0.835, mean=0.835, max=0.835, sum=1.669 (2)", - "tab": "Accuracy", - "International Law - Observed inference time (s)": "{\"description\": \"min=0.271, mean=0.271, max=0.271, sum=0.542 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.27110107082965945\"}", - "International Law - # eval": "{\"description\": \"min=121, mean=121, max=121, sum=242 (2)\", \"tab\": \"General information\", \"score\": \"121.0\"}", - "International Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "International Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "International Law - # prompt tokens": "{\"description\": \"min=477.843, mean=477.843, max=477.843, sum=955.686 (2)\", \"tab\": \"General information\", \"score\": \"477.8429752066116\"}", - "International Law - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"international_law\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_international_law\"" - } - } - }, - { - "evaluation_name": "Logical Fallacies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Logical Fallacies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.706, - "details": { - "description": "min=0.706, mean=0.706, max=0.706, sum=1.411 (2)", - "tab": "Accuracy", - "Logical Fallacies - Observed inference time (s)": "{\"description\": \"min=0.25, mean=0.25, max=0.25, sum=0.499 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.24970631804202964\"}", - "Logical Fallacies - # eval": "{\"description\": \"min=163, mean=163, max=163, sum=326 (2)\", \"tab\": \"General information\", \"score\": \"163.0\"}", - "Logical Fallacies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Logical Fallacies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Logical Fallacies - # prompt tokens": "{\"description\": \"min=337.718, mean=337.718, max=337.718, sum=675.436 (2)\", \"tab\": \"General information\", \"score\": \"337.7177914110429\"}", - "Logical Fallacies - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"logical_fallacies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_logical_fallacies\"" - } - } - }, - { - "evaluation_name": "Machine Learning", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Machine Learning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.536, - "details": { - "description": "min=0.536, mean=0.536, max=0.536, sum=1.071 (2)", - "tab": "Accuracy", - "Machine Learning - Observed inference time (s)": "{\"description\": \"min=0.267, mean=0.267, max=0.267, sum=0.533 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2665597881589617\"}", - "Machine Learning - # eval": "{\"description\": \"min=112, mean=112, max=112, sum=224 (2)\", \"tab\": \"General information\", \"score\": \"112.0\"}", - "Machine Learning - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Machine Learning - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Machine Learning - # prompt tokens": "{\"description\": \"min=559.277, mean=559.277, max=559.277, sum=1118.554 (2)\", \"tab\": \"General information\", \"score\": \"559.2767857142857\"}", - "Machine Learning - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"machine_learning\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_machine_learning\"" - } - } - }, - { - "evaluation_name": "Management", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Management", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.786, - "details": { - "description": "min=0.786, mean=0.786, max=0.786, sum=1.573 (2)", - "tab": "Accuracy", - "Management - Observed inference time (s)": "{\"description\": \"min=0.241, mean=0.241, max=0.241, sum=0.481 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.24073980386974742\"}", - "Management - # eval": "{\"description\": \"min=103, mean=103, max=103, sum=206 (2)\", \"tab\": \"General information\", \"score\": \"103.0\"}", - "Management - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Management - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Management - # prompt tokens": "{\"description\": \"min=225.262, mean=225.262, max=225.262, sum=450.524 (2)\", \"tab\": \"General information\", \"score\": \"225.2621359223301\"}", - "Management - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"management\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_management\"" - } - } - }, - { - "evaluation_name": "Marketing", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Marketing", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.885, - "details": { - "description": "min=0.885, mean=0.885, max=0.885, sum=1.769 (2)", - "tab": "Accuracy", - "Marketing - Observed inference time (s)": "{\"description\": \"min=0.258, mean=0.258, max=0.258, sum=0.517 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.25835410753885907\"}", - "Marketing - # eval": "{\"description\": \"min=234, mean=234, max=234, sum=468 (2)\", \"tab\": \"General information\", \"score\": \"234.0\"}", - "Marketing - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Marketing - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Marketing - # prompt tokens": "{\"description\": \"min=351.573, mean=351.573, max=351.573, sum=703.145 (2)\", \"tab\": \"General information\", \"score\": \"351.5726495726496\"}", - "Marketing - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"marketing\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_marketing\"" - } - } - }, - { - "evaluation_name": "Medical Genetics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Medical Genetics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.67, - "details": { - "description": "min=0.67, mean=0.67, max=0.67, sum=1.34 (2)", - "tab": "Accuracy", - "Medical Genetics - Observed inference time (s)": "{\"description\": \"min=0.251, mean=0.251, max=0.251, sum=0.502 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2510761094093323\"}", - "Medical Genetics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Medical Genetics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Medical Genetics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Medical Genetics - # prompt tokens": "{\"description\": \"min=274.75, mean=274.75, max=274.75, sum=549.5 (2)\", \"tab\": \"General information\", \"score\": \"274.75\"}", - "Medical Genetics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"medical_genetics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_medical_genetics\"" - } - } - }, - { - "evaluation_name": "Miscellaneous", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Miscellaneous", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.865, - "details": { - "description": "min=0.865, mean=0.865, max=0.865, sum=1.729 (2)", - "tab": "Accuracy", - "Miscellaneous - Observed inference time (s)": "{\"description\": \"min=0.233, mean=0.233, max=0.233, sum=0.466 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.23304342005596915\"}", - "Miscellaneous - # eval": "{\"description\": \"min=783, mean=783, max=783, sum=1566 (2)\", \"tab\": \"General information\", \"score\": \"783.0\"}", - "Miscellaneous - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Miscellaneous - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Miscellaneous - # prompt tokens": "{\"description\": \"min=254.525, mean=254.525, max=254.525, sum=509.05 (2)\", \"tab\": \"General information\", \"score\": \"254.5249042145594\"}", - "Miscellaneous - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"miscellaneous\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_miscellaneous\"" - } - } - }, - { - "evaluation_name": "Moral Scenarios", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Moral Scenarios", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.465, - "details": { - "description": "min=0.465, mean=0.465, max=0.465, sum=0.93 (2)", - "tab": "Accuracy", - "Moral Disputes - Observed inference time (s)": "{\"description\": \"min=0.256, mean=0.256, max=0.256, sum=0.512 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2561916905331474\"}", - "Moral Scenarios - Observed inference time (s)": "{\"description\": \"min=0.262, mean=0.262, max=0.262, sum=0.525 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2624055065922231\"}", - "Moral Disputes - # eval": "{\"description\": \"min=346, mean=346, max=346, sum=692 (2)\", \"tab\": \"General information\", \"score\": \"346.0\"}", - "Moral Disputes - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Disputes - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Disputes - # prompt tokens": "{\"description\": \"min=357.165, mean=357.165, max=357.165, sum=714.329 (2)\", \"tab\": \"General information\", \"score\": \"357.16473988439304\"}", - "Moral Disputes - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # eval": "{\"description\": \"min=895, mean=895, max=895, sum=1790 (2)\", \"tab\": \"General information\", \"score\": \"895.0\"}", - "Moral Scenarios - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Scenarios - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # prompt tokens": "{\"description\": \"min=546.793, mean=546.793, max=546.793, sum=1093.587 (2)\", \"tab\": \"General information\", \"score\": \"546.7932960893854\"}", - "Moral Scenarios - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"moral_scenarios\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_moral_scenarios\"" - } - } - }, - { - "evaluation_name": "Nutrition", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Nutrition", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.745, - "details": { - "description": "min=0.745, mean=0.745, max=0.745, sum=1.49 (2)", - "tab": "Accuracy", - "Nutrition - Observed inference time (s)": "{\"description\": \"min=0.248, mean=0.248, max=0.248, sum=0.496 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2479639964945176\"}", - "Nutrition - # eval": "{\"description\": \"min=306, mean=306, max=306, sum=612 (2)\", \"tab\": \"General information\", \"score\": \"306.0\"}", - "Nutrition - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Nutrition - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Nutrition - # prompt tokens": "{\"description\": \"min=454.758, mean=454.758, max=454.758, sum=909.516 (2)\", \"tab\": \"General information\", \"score\": \"454.75816993464053\"}", - "Nutrition - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"nutrition\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_nutrition\"" - } - } - }, - { - "evaluation_name": "Prehistory", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Prehistory", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.796, - "details": { - "description": "min=0.796, mean=0.796, max=0.796, sum=1.593 (2)", - "tab": "Accuracy", - "Prehistory - Observed inference time (s)": "{\"description\": \"min=0.254, mean=0.254, max=0.254, sum=0.508 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2538878917694092\"}", - "Prehistory - # eval": "{\"description\": \"min=324, mean=324, max=324, sum=648 (2)\", \"tab\": \"General information\", \"score\": \"324.0\"}", - "Prehistory - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Prehistory - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Prehistory - # prompt tokens": "{\"description\": \"min=410.315, mean=410.315, max=410.315, sum=820.63 (2)\", \"tab\": \"General information\", \"score\": \"410.31481481481484\"}", - "Prehistory - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"prehistory\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_prehistory\"" - } - } - }, - { - "evaluation_name": "Public Relations", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Public Relations", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.682, - "details": { - "description": "min=0.682, mean=0.682, max=0.682, sum=1.364 (2)", - "tab": "Accuracy", - "Public Relations - Observed inference time (s)": "{\"description\": \"min=0.252, mean=0.252, max=0.252, sum=0.505 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.25225248553536156\"}", - "Public Relations - # eval": "{\"description\": \"min=110, mean=110, max=110, sum=220 (2)\", \"tab\": \"General information\", \"score\": \"110.0\"}", - "Public Relations - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Public Relations - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Public Relations - # prompt tokens": "{\"description\": \"min=316.591, mean=316.591, max=316.591, sum=633.182 (2)\", \"tab\": \"General information\", \"score\": \"316.59090909090907\"}", - "Public Relations - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"public_relations\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_public_relations\"" - } - } - }, - { - "evaluation_name": "Security Studies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Security Studies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.743, - "details": { - "description": "min=0.743, mean=0.743, max=0.743, sum=1.486 (2)", - "tab": "Accuracy", - "Security Studies - Observed inference time (s)": "{\"description\": \"min=0.31, mean=0.31, max=0.31, sum=0.62 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.30983400539476047\"}", - "Security Studies - # eval": "{\"description\": \"min=245, mean=245, max=245, sum=490 (2)\", \"tab\": \"General information\", \"score\": \"245.0\"}", - "Security Studies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Security Studies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Security Studies - # prompt tokens": "{\"description\": \"min=856.637, mean=856.637, max=856.637, sum=1713.273 (2)\", \"tab\": \"General information\", \"score\": \"856.6367346938775\"}", - "Security Studies - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"security_studies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_security_studies\"" - } - } - }, - { - "evaluation_name": "Sociology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Sociology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.891, - "details": { - "description": "min=0.891, mean=0.891, max=0.891, sum=1.781 (2)", - "tab": "Accuracy", - "Sociology - Observed inference time (s)": "{\"description\": \"min=0.258, mean=0.258, max=0.258, sum=0.515 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.25752189384764107\"}", - "Sociology - # eval": "{\"description\": \"min=201, mean=201, max=201, sum=402 (2)\", \"tab\": \"General information\", \"score\": \"201.0\"}", - "Sociology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Sociology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Sociology - # prompt tokens": "{\"description\": \"min=327.801, mean=327.801, max=327.801, sum=655.602 (2)\", \"tab\": \"General information\", \"score\": \"327.80099502487565\"}", - "Sociology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"sociology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_sociology\"" - } - } - }, - { - "evaluation_name": "Virology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Virology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.53, - "details": { - "description": "min=0.53, mean=0.53, max=0.53, sum=1.06 (2)", - "tab": "Accuracy", - "Virology - Observed inference time (s)": "{\"description\": \"min=0.238, mean=0.238, max=0.238, sum=0.477 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.23830672200903835\"}", - "Virology - # eval": "{\"description\": \"min=166, mean=166, max=166, sum=332 (2)\", \"tab\": \"General information\", \"score\": \"166.0\"}", - "Virology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Virology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Virology - # prompt tokens": "{\"description\": \"min=267.458, mean=267.458, max=267.458, sum=534.916 (2)\", \"tab\": \"General information\", \"score\": \"267.4578313253012\"}", - "Virology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"virology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_virology\"" - } - } - }, - { - "evaluation_name": "World Religions", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on World Religions", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.813, - "details": { - "description": "min=0.813, mean=0.813, max=0.813, sum=1.626 (2)", - "tab": "Accuracy", - "World Religions - Observed inference time (s)": "{\"description\": \"min=0.236, mean=0.236, max=0.236, sum=0.473 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.23630904593662908\"}", - "World Religions - # eval": "{\"description\": \"min=171, mean=171, max=171, sum=342 (2)\", \"tab\": \"General information\", \"score\": \"171.0\"}", - "World Religions - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "World Religions - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "World Religions - # prompt tokens": "{\"description\": \"min=223.731, mean=223.731, max=223.731, sum=447.462 (2)\", \"tab\": \"General information\", \"score\": \"223.73099415204678\"}", - "World Religions - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"world_religions\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_world_religions\"" - } - } - }, - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.887, - "details": { - "description": "", - "tab": "Efficiency" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/ai21_jurassic-2-grande-17b.json b/data/models/ai21_jurassic-2-grande-17b.json deleted file mode 100644 index c42bb2486d46c5464c7ac19c8ab3add9d216d1e4..0000000000000000000000000000000000000000 --- a/data/models/ai21_jurassic-2-grande-17b.json +++ /dev/null @@ -1,674 +0,0 @@ -{ - "model_info": { - "name": "Jurassic-2 Grande 17B", - "id": "ai21/Jurassic-2-Grande-17B", - "developer": "ai21", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_classic/ai21_Jurassic-2-Grande-17B/1774096308.339228", - "retrieved_timestamp": "1774096308.339228", - "source_metadata": { - "source_name": "helm_classic", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_classic", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_classic", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperform on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.743, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Calibration": "{\"description\": \"\", \"tab\": \"Calibration\", \"score\": \"0.6300647548566143\"}", - "Mean win rate - Robustness": "{\"description\": \"\", \"tab\": \"Robustness\", \"score\": \"0.7641047680536001\"}", - "Mean win rate - Fairness": "{\"description\": \"\", \"tab\": \"Fairness\", \"score\": \"0.7037362526239056\"}", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}", - "Mean win rate - Bias": "{\"description\": \"\", \"tab\": \"Bias\", \"score\": \"0.561885097395068\"}", - "Mean win rate - Toxicity": "{\"description\": \"\", \"tab\": \"Toxicity\", \"score\": \"0.3875874125874126\"}", - "Mean win rate - Summarization metrics": "{\"description\": \"\", \"tab\": \"Summarization metrics\", \"score\": \"0.6710526315789473\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.475, - "details": { - "description": "min=0.24, mean=0.475, max=0.81, sum=7.13 (15)", - "tab": "Accuracy", - "MMLU - ECE (10-bin)": "{\"description\": \"min=0.076, mean=0.134, max=0.172, sum=2.006 (15)\", \"tab\": \"Calibration\", \"score\": \"0.13373539597087636\"}", - "MMLU - EM (Robustness)": "{\"description\": \"min=0.22, mean=0.411, max=0.68, sum=6.168 (15)\", \"tab\": \"Robustness\", \"score\": \"0.41120467836257313\"}", - "MMLU - EM (Fairness)": "{\"description\": \"min=0.23, mean=0.433, max=0.73, sum=6.498 (15)\", \"tab\": \"Fairness\", \"score\": \"0.43321637426900583\"}", - "MMLU - Denoised inference time (s)": "{\"description\": \"5 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=1542 (15)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=75 (15)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (15)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=308.59, mean=396.74, max=552.719, sum=5951.098 (15)\", \"tab\": \"General information\", \"score\": \"396.73985964912276\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=15 (15)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "MMLU - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=45 (15)\", \"tab\": \"General information\", \"score\": \"3.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "BoolQ", - "source_data": { - "dataset_name": "BoolQ", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on BoolQ", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.826, - "details": { - "description": "min=0.816, mean=0.826, max=0.832, sum=2.478 (3)", - "tab": "Accuracy", - "BoolQ - ECE (10-bin)": "{\"description\": \"min=0.179, mean=0.209, max=0.243, sum=0.627 (3)\", \"tab\": \"Calibration\", \"score\": \"0.20883844550071148\"}", - "BoolQ - EM (Robustness)": "{\"description\": \"min=0.714, mean=0.729, max=0.743, sum=2.187 (3)\", \"tab\": \"Robustness\", \"score\": \"0.729\"}", - "BoolQ - EM (Fairness)": "{\"description\": \"min=0.758, mean=0.78, max=0.791, sum=2.34 (3)\", \"tab\": \"Fairness\", \"score\": \"0.7799999999999999\"}", - "BoolQ - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "BoolQ - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "BoolQ - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "BoolQ - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "BoolQ - # prompt tokens": "{\"description\": \"min=506.985, mean=694.652, max=952.985, sum=2083.955 (3)\", \"tab\": \"General information\", \"score\": \"694.6516666666666\"}", - "BoolQ - # output tokens": "{\"description\": \"min=2.002, mean=2.002, max=2.002, sum=6.006 (3)\", \"tab\": \"General information\", \"score\": \"2.002\"}", - "BoolQ - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "BoolQ - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.737, - "details": { - "description": "min=0.732, mean=0.737, max=0.744, sum=2.21 (3)", - "tab": "Accuracy", - "NarrativeQA - ECE (10-bin)": "{\"description\": \"min=0.107, mean=0.126, max=0.158, sum=0.377 (3)\", \"tab\": \"Calibration\", \"score\": \"0.12569343029680938\"}", - "NarrativeQA - F1 (Robustness)": "{\"description\": \"min=0.49, mean=0.583, max=0.65, sum=1.75 (3)\", \"tab\": \"Robustness\", \"score\": \"0.5834381641862693\"}", - "NarrativeQA - F1 (Fairness)": "{\"description\": \"min=0.638, mean=0.645, max=0.651, sum=1.935 (3)\", \"tab\": \"Fairness\", \"score\": \"0.6449807868174807\"}", - "NarrativeQA - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=1065 (3)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=2.166, mean=2.639, max=3.225, sum=7.918 (3)\", \"tab\": \"General information\", \"score\": \"2.63943661971831\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=1598.614, mean=1692.218, max=1777.299, sum=5076.654 (3)\", \"tab\": \"General information\", \"score\": \"1692.2178403755868\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=5.039, mean=5.261, max=5.473, sum=15.783 (3)\", \"tab\": \"General information\", \"score\": \"5.261032863849765\"}", - "NarrativeQA - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NarrativeQA - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NarrativeQA - Stereotypes (gender)": "{\"description\": \"min=0.4, mean=0.448, max=0.5, sum=1.344 (3)\", \"tab\": \"Bias\", \"score\": \"0.4481481481481482\"}", - "NarrativeQA - Representation (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=2 (3)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "NarrativeQA - Representation (gender)": "{\"description\": \"min=0.185, mean=0.196, max=0.205, sum=0.587 (3)\", \"tab\": \"Bias\", \"score\": \"0.19550967146595563\"}", - "NarrativeQA - Toxic fraction": "{\"description\": \"min=0.017, mean=0.02, max=0.023, sum=0.059 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.019718309859154928\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (open-book)", - "source_data": { - "dataset_name": "NaturalQuestions (open-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (open-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.639, - "details": { - "description": "min=0.627, mean=0.639, max=0.649, sum=1.918 (3)", - "tab": "Accuracy", - "NaturalQuestions (closed-book) - ECE (10-bin)": "{\"description\": \"min=0.016, mean=0.018, max=0.019, sum=0.054 (3)\", \"tab\": \"Calibration\", \"score\": \"0.01803156970695322\"}", - "NaturalQuestions (open-book) - ECE (10-bin)": "{\"description\": \"min=0.053, mean=0.063, max=0.072, sum=0.188 (3)\", \"tab\": \"Calibration\", \"score\": \"0.06257440554546793\"}", - "NaturalQuestions (closed-book) - F1 (Robustness)": "{\"description\": \"min=0.277, mean=0.285, max=0.29, sum=0.854 (3)\", \"tab\": \"Robustness\", \"score\": \"0.28458982309414393\"}", - "NaturalQuestions (open-book) - F1 (Robustness)": "{\"description\": \"min=0.555, mean=0.564, max=0.568, sum=1.691 (3)\", \"tab\": \"Robustness\", \"score\": \"0.5635162273229849\"}", - "NaturalQuestions (closed-book) - F1 (Fairness)": "{\"description\": \"min=0.276, mean=0.283, max=0.288, sum=0.85 (3)\", \"tab\": \"Fairness\", \"score\": \"0.2832503879785802\"}", - "NaturalQuestions (open-book) - F1 (Fairness)": "{\"description\": \"min=0.569, mean=0.584, max=0.592, sum=1.752 (3)\", \"tab\": \"Fairness\", \"score\": \"0.5839142853000876\"}", - "NaturalQuestions (closed-book) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NaturalQuestions (open-book) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=94.377, mean=99.377, max=102.377, sum=298.131 (3)\", \"tab\": \"General information\", \"score\": \"99.377\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=5.466, mean=6.315, max=6.864, sum=18.944 (3)\", \"tab\": \"General information\", \"score\": \"6.314666666666667\"}", - "NaturalQuestions (closed-book) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.568, mean=4.666, max=4.734, sum=13.999 (3)\", \"tab\": \"General information\", \"score\": \"4.666333333333333\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.038, mean=0.038, max=0.038, sum=0.114 (3)\", \"tab\": \"General information\", \"score\": \"0.038\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1136.933, mean=1418.457, max=1595.508, sum=4255.37 (3)\", \"tab\": \"General information\", \"score\": \"1418.4566666666667\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=5.441, mean=5.676, max=6.069, sum=17.029 (3)\", \"tab\": \"General information\", \"score\": \"5.676333333333333\"}", - "NaturalQuestions (open-book) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NaturalQuestions (closed-book) - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=2 (3)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "NaturalQuestions (closed-book) - Stereotypes (gender)": "{\"description\": \"min=0.5, mean=0.5, max=0.5, sum=1.5 (3)\", \"tab\": \"Bias\", \"score\": \"0.5\"}", - "NaturalQuestions (closed-book) - Representation (race)": "{\"description\": \"min=0.431, mean=0.507, max=0.569, sum=1.52 (3)\", \"tab\": \"Bias\", \"score\": \"0.5067443890625439\"}", - "NaturalQuestions (closed-book) - Representation (gender)": "{\"description\": \"min=0.1, mean=0.176, max=0.273, sum=0.527 (3)\", \"tab\": \"Bias\", \"score\": \"0.1755244755244755\"}", - "NaturalQuestions (open-book) - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=1.333 (2)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666667\"}", - "NaturalQuestions (open-book) - Stereotypes (gender)": "{\"description\": \"min=0.5, mean=0.5, max=0.5, sum=1.5 (3)\", \"tab\": \"Bias\", \"score\": \"0.5\"}", - "NaturalQuestions (open-book) - Representation (race)": "{\"description\": \"min=0.431, mean=0.465, max=0.498, sum=1.395 (3)\", \"tab\": \"Bias\", \"score\": \"0.46507125832968527\"}", - "NaturalQuestions (open-book) - Representation (gender)": "{\"description\": \"min=0.005, mean=0.03, max=0.053, sum=0.089 (3)\", \"tab\": \"Bias\", \"score\": \"0.02952187967385538\"}", - "NaturalQuestions (closed-book) - Toxic fraction": "{\"description\": \"min=0, mean=0.0, max=0.001, sum=0.001 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.0003333333333333333\"}", - "NaturalQuestions (open-book) - Toxic fraction": "{\"description\": \"min=0, mean=0.0, max=0.001, sum=0.001 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.0003333333333333333\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "QuAC", - "source_data": { - "dataset_name": "QuAC", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on QuAC", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.418, - "details": { - "description": "min=0.412, mean=0.418, max=0.429, sum=1.255 (3)", - "tab": "Accuracy", - "QuAC - ECE (10-bin)": "{\"description\": \"min=0.027, mean=0.035, max=0.04, sum=0.105 (3)\", \"tab\": \"Calibration\", \"score\": \"0.03491339390127312\"}", - "QuAC - F1 (Robustness)": "{\"description\": \"min=0.271, mean=0.276, max=0.281, sum=0.827 (3)\", \"tab\": \"Robustness\", \"score\": \"0.27557303329747496\"}", - "QuAC - F1 (Fairness)": "{\"description\": \"min=0.335, mean=0.34, max=0.35, sum=1.02 (3)\", \"tab\": \"Fairness\", \"score\": \"0.34002521409765923\"}", - "QuAC - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "QuAC - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "QuAC - # train": "{\"description\": \"min=1.788, mean=1.829, max=1.88, sum=5.486 (3)\", \"tab\": \"General information\", \"score\": \"1.8286666666666667\"}", - "QuAC - truncated": "{\"description\": \"min=0.001, mean=0.001, max=0.001, sum=0.003 (3)\", \"tab\": \"General information\", \"score\": \"0.001\"}", - "QuAC - # prompt tokens": "{\"description\": \"min=1645.856, mean=1698.711, max=1730.814, sum=5096.134 (3)\", \"tab\": \"General information\", \"score\": \"1698.7113333333334\"}", - "QuAC - # output tokens": "{\"description\": \"min=22.04, mean=24.469, max=26.73, sum=73.408 (3)\", \"tab\": \"General information\", \"score\": \"24.469333333333335\"}", - "QuAC - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "QuAC - Stereotypes (race)": "{\"description\": \"min=0.625, mean=0.64, max=0.651, sum=1.919 (3)\", \"tab\": \"Bias\", \"score\": \"0.6395502645502645\"}", - "QuAC - Stereotypes (gender)": "{\"description\": \"min=0.389, mean=0.422, max=0.455, sum=1.267 (3)\", \"tab\": \"Bias\", \"score\": \"0.4224807266199369\"}", - "QuAC - Representation (race)": "{\"description\": \"min=0.183, mean=0.23, max=0.263, sum=0.689 (3)\", \"tab\": \"Bias\", \"score\": \"0.22977891012599364\"}", - "QuAC - Representation (gender)": "{\"description\": \"min=0.223, mean=0.224, max=0.225, sum=0.673 (3)\", \"tab\": \"Bias\", \"score\": \"0.22430144583085757\"}", - "QuAC - Toxic fraction": "{\"description\": \"min=0.002, mean=0.003, max=0.004, sum=0.009 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.0030000000000000005\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "HellaSwag", - "source_data": { - "dataset_name": "HellaSwag", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on HellaSwag", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.781, - "details": { - "description": "min=0.781, mean=0.781, max=0.781, sum=0.781 (1)", - "tab": "Accuracy", - "HellaSwag - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "HellaSwag - EM (Robustness)": "{\"description\": \"min=0.755, mean=0.755, max=0.755, sum=0.755 (1)\", \"tab\": \"Robustness\", \"score\": \"0.755\"}", - "HellaSwag - EM (Fairness)": "{\"description\": \"min=0.632, mean=0.632, max=0.632, sum=0.632 (1)\", \"tab\": \"Fairness\", \"score\": \"0.632\"}", - "HellaSwag - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "HellaSwag - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "HellaSwag - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "HellaSwag - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "HellaSwag - # prompt tokens": "{\"description\": \"min=62.466, mean=62.466, max=62.466, sum=62.466 (1)\", \"tab\": \"General information\", \"score\": \"62.466\"}", - "HellaSwag - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "HellaSwag - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.542, - "details": { - "description": "min=0.542, mean=0.542, max=0.542, sum=0.542 (1)", - "tab": "Accuracy", - "OpenbookQA - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "OpenbookQA - EM (Robustness)": "{\"description\": \"min=0.474, mean=0.474, max=0.474, sum=0.474 (1)\", \"tab\": \"Robustness\", \"score\": \"0.474\"}", - "OpenbookQA - EM (Fairness)": "{\"description\": \"min=0.466, mean=0.466, max=0.466, sum=0.466 (1)\", \"tab\": \"Fairness\", \"score\": \"0.466\"}", - "OpenbookQA - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=4.348, mean=4.348, max=4.348, sum=4.348 (1)\", \"tab\": \"General information\", \"score\": \"4.348\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "TruthfulQA", - "source_data": { - "dataset_name": "TruthfulQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on TruthfulQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.348, - "details": { - "description": "min=0.287, mean=0.348, max=0.384, sum=1.043 (3)", - "tab": "Accuracy", - "TruthfulQA - ECE (10-bin)": "{\"description\": \"min=0.073, mean=0.097, max=0.142, sum=0.291 (3)\", \"tab\": \"Calibration\", \"score\": \"0.09707246189445913\"}", - "TruthfulQA - EM (Robustness)": "{\"description\": \"min=0.245, mean=0.293, max=0.326, sum=0.878 (3)\", \"tab\": \"Robustness\", \"score\": \"0.29255861365953106\"}", - "TruthfulQA - EM (Fairness)": "{\"description\": \"min=0.242, mean=0.29, max=0.32, sum=0.87 (3)\", \"tab\": \"Fairness\", \"score\": \"0.2900101936799185\"}", - "TruthfulQA - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "TruthfulQA - # eval": "{\"description\": \"min=654, mean=654, max=654, sum=1962 (3)\", \"tab\": \"General information\", \"score\": \"654.0\"}", - "TruthfulQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "TruthfulQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "TruthfulQA - # prompt tokens": "{\"description\": \"min=317.682, mean=355.015, max=375.682, sum=1065.046 (3)\", \"tab\": \"General information\", \"score\": \"355.0152905198777\"}", - "TruthfulQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=3 (3)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "TruthfulQA - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MS MARCO (TREC)", - "source_data": { - "dataset_name": "MS MARCO (TREC)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "NDCG@10 on MS MARCO (TREC)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.514, - "details": { - "description": "min=0.473, mean=0.514, max=0.577, sum=1.543 (3)", - "tab": "Accuracy", - "MS MARCO (regular) - RR@10 (Robustness)": "{\"description\": \"min=0.18, mean=0.227, max=0.253, sum=0.681 (3)\", \"tab\": \"Robustness\", \"score\": \"0.22687976190476158\"}", - "MS MARCO (TREC) - NDCG@10 (Robustness)": "{\"description\": \"min=0.406, mean=0.423, max=0.451, sum=1.269 (3)\", \"tab\": \"Robustness\", \"score\": \"0.42305953691791237\"}", - "MS MARCO (regular) - RR@10 (Fairness)": "{\"description\": \"min=0.206, mean=0.243, max=0.271, sum=0.728 (3)\", \"tab\": \"Fairness\", \"score\": \"0.242712169312169\"}", - "MS MARCO (TREC) - NDCG@10 (Fairness)": "{\"description\": \"min=0.438, mean=0.471, max=0.522, sum=1.413 (3)\", \"tab\": \"Fairness\", \"score\": \"0.47089412794287994\"}", - "MS MARCO (regular) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MS MARCO (TREC) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MS MARCO (regular) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "MS MARCO (regular) - # train": "{\"description\": \"min=2, mean=2, max=2, sum=6 (3)\", \"tab\": \"General information\", \"score\": \"2.0\"}", - "MS MARCO (regular) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MS MARCO (regular) - # prompt tokens": "{\"description\": \"min=349.303, mean=385.636, max=423.303, sum=1156.909 (3)\", \"tab\": \"General information\", \"score\": \"385.63633333333337\"}", - "MS MARCO (regular) - # output tokens": "{\"description\": \"min=2.003, mean=2.006, max=2.008, sum=6.017 (3)\", \"tab\": \"General information\", \"score\": \"2.005666666666667\"}", - "MS MARCO (regular) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "MS MARCO (TREC) - # eval": "{\"description\": \"min=43, mean=43, max=43, sum=129 (3)\", \"tab\": \"General information\", \"score\": \"43.0\"}", - "MS MARCO (TREC) - # train": "{\"description\": \"min=2, mean=2, max=2, sum=6 (3)\", \"tab\": \"General information\", \"score\": \"2.0\"}", - "MS MARCO (TREC) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MS MARCO (TREC) - # prompt tokens": "{\"description\": \"min=337.047, mean=373.38, max=411.047, sum=1120.14 (3)\", \"tab\": \"General information\", \"score\": \"373.3798449612403\"}", - "MS MARCO (TREC) - # output tokens": "{\"description\": \"min=2.023, mean=2.023, max=2.023, sum=6.07 (3)\", \"tab\": \"General information\", \"score\": \"2.0232558139534884\"}", - "MS MARCO (TREC) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "MS MARCO (regular) - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "MS MARCO (TREC) - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CNN/DailyMail", - "source_data": { - "dataset_name": "CNN/DailyMail", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on CNN/DailyMail", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.144, - "details": { - "description": "min=0.131, mean=0.144, max=0.153, sum=0.865 (6)", - "tab": "Accuracy", - "CNN/DailyMail - Denoised inference time (s)": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "CNN/DailyMail - # eval": "{\"description\": \"min=466, mean=466, max=466, sum=2796 (6)\", \"tab\": \"General information\", \"score\": \"466.0\"}", - "CNN/DailyMail - # train": "{\"description\": \"min=5, mean=5, max=5, sum=30 (6)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CNN/DailyMail - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (6)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CNN/DailyMail - # prompt tokens": "{\"description\": \"min=1203.032, mean=1213.032, max=1224.032, sum=7278.193 (6)\", \"tab\": \"General information\", \"score\": \"1213.0321888412018\"}", - "CNN/DailyMail - # output tokens": "{\"description\": \"min=48.987, mean=55.762, max=59.891, sum=334.571 (6)\", \"tab\": \"General information\", \"score\": \"55.76180257510729\"}", - "CNN/DailyMail - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=18 (6)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "CNN/DailyMail - Stereotypes (race)": "{\"description\": \"min=0.619, mean=0.636, max=0.667, sum=3.817 (6)\", \"tab\": \"Bias\", \"score\": \"0.6361416361416362\"}", - "CNN/DailyMail - Stereotypes (gender)": "{\"description\": \"min=0.386, mean=0.402, max=0.424, sum=2.411 (6)\", \"tab\": \"Bias\", \"score\": \"0.4017992121362035\"}", - "CNN/DailyMail - Representation (race)": "{\"description\": \"min=0.338, mean=0.359, max=0.379, sum=2.152 (6)\", \"tab\": \"Bias\", \"score\": \"0.3586894722560466\"}", - "CNN/DailyMail - Representation (gender)": "{\"description\": \"min=0.099, mean=0.117, max=0.128, sum=0.701 (6)\", \"tab\": \"Bias\", \"score\": \"0.11681135928174619\"}", - "CNN/DailyMail - Toxic fraction": "{\"description\": \"min=0.002, mean=0.003, max=0.004, sum=0.017 (6)\", \"tab\": \"Toxicity\", \"score\": \"0.002861230329041488\"}", - "CNN/DailyMail - SummaC": "{\"description\": \"min=0.469, mean=0.503, max=0.535, sum=1.51 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.5032610058862116\"}", - "CNN/DailyMail - QAFactEval": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - BERTScore (F1)": "{\"description\": \"min=0.281, mean=0.299, max=0.308, sum=0.896 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.2987736324577836\"}", - "CNN/DailyMail - Coverage": "{\"description\": \"min=0.953, mean=0.96, max=0.965, sum=5.76 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"0.9600651009447835\"}", - "CNN/DailyMail - Density": "{\"description\": \"min=14.681, mean=22.305, max=27.564, sum=133.827 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"22.304503793993888\"}", - "CNN/DailyMail - Compression": "{\"description\": \"min=10.404, mean=11.399, max=13.033, sum=68.393 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"11.39877050033896\"}", - "CNN/DailyMail - HumanEval-faithfulness": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-relevance": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-coherence": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "XSUM", - "source_data": { - "dataset_name": "XSUM", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on XSUM", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.167, - "details": { - "description": "min=0.164, mean=0.167, max=0.173, sum=1.005 (6)", - "tab": "Accuracy", - "XSUM - Denoised inference time (s)": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "XSUM - # eval": "{\"description\": \"min=518, mean=518, max=518, sum=3108 (6)\", \"tab\": \"General information\", \"score\": \"518.0\"}", - "XSUM - # train": "{\"description\": \"min=5, mean=5, max=5, sum=30 (6)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "XSUM - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (6)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "XSUM - # prompt tokens": "{\"description\": \"min=1099.388, mean=1133.388, max=1172.388, sum=6800.328 (6)\", \"tab\": \"General information\", \"score\": \"1133.388030888031\"}", - "XSUM - # output tokens": "{\"description\": \"min=21.463, mean=21.75, max=22.241, sum=130.502 (6)\", \"tab\": \"General information\", \"score\": \"21.75032175032175\"}", - "XSUM - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=18 (6)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "XSUM - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=4 (6)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "XSUM - Stereotypes (gender)": "{\"description\": \"min=0.445, mean=0.456, max=0.463, sum=2.736 (6)\", \"tab\": \"Bias\", \"score\": \"0.4559853927203065\"}", - "XSUM - Representation (race)": "{\"description\": \"min=0.362, mean=0.466, max=0.532, sum=2.798 (6)\", \"tab\": \"Bias\", \"score\": \"0.4664089053990878\"}", - "XSUM - Representation (gender)": "{\"description\": \"min=0.192, mean=0.207, max=0.233, sum=1.24 (6)\", \"tab\": \"Bias\", \"score\": \"0.2066101848280066\"}", - "XSUM - Toxic fraction": "{\"description\": \"min=0, mean=0.001, max=0.002, sum=0.004 (6)\", \"tab\": \"Toxicity\", \"score\": \"0.0006435006435006435\"}", - "XSUM - SummaC": "{\"description\": \"min=-0.31, mean=-0.289, max=-0.268, sum=-0.868 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"-0.2893415716573027\"}", - "XSUM - QAFactEval": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - BERTScore (F1)": "{\"description\": \"min=0.47, mean=0.475, max=0.48, sum=1.424 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.474663326872436\"}", - "XSUM - Coverage": "{\"description\": \"min=0.761, mean=0.766, max=0.771, sum=4.596 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"0.7660021617230298\"}", - "XSUM - Density": "{\"description\": \"min=2.196, mean=2.36, max=2.464, sum=14.158 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"2.359653576011524\"}", - "XSUM - Compression": "{\"description\": \"min=16.605, mean=17.045, max=17.3, sum=102.267 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"17.044545661784866\"}", - "XSUM - HumanEval-faithfulness": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-relevance": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-coherence": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "IMDB", - "source_data": { - "dataset_name": "IMDB", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on IMDB", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.938, - "details": { - "description": "min=0.926, mean=0.938, max=0.954, sum=2.814 (3)", - "tab": "Accuracy", - "IMDB - ECE (10-bin)": "{\"description\": \"min=0.088, mean=0.111, max=0.153, sum=0.333 (3)\", \"tab\": \"Calibration\", \"score\": \"0.11088831926219649\"}", - "IMDB - EM (Robustness)": "{\"description\": \"min=0.915, mean=0.928, max=0.949, sum=2.784 (3)\", \"tab\": \"Robustness\", \"score\": \"0.9279999999999999\"}", - "IMDB - EM (Fairness)": "{\"description\": \"min=0.92, mean=0.931, max=0.951, sum=2.792 (3)\", \"tab\": \"Fairness\", \"score\": \"0.9306666666666666\"}", - "IMDB - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "IMDB - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "IMDB - # train": "{\"description\": \"min=4.915, mean=4.972, max=5, sum=14.915 (3)\", \"tab\": \"General information\", \"score\": \"4.971666666666667\"}", - "IMDB - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IMDB - # prompt tokens": "{\"description\": \"min=853.851, mean=1281.577, max=1725.03, sum=3844.732 (3)\", \"tab\": \"General information\", \"score\": \"1281.5773333333334\"}", - "IMDB - # output tokens": "{\"description\": \"min=2, mean=2, max=2, sum=6 (3)\", \"tab\": \"General information\", \"score\": \"2.0\"}", - "IMDB - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "IMDB - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CivilComments", - "source_data": { - "dataset_name": "CivilComments", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on CivilComments", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.547, - "details": { - "description": "min=0.011, mean=0.547, max=0.998, sum=29.525 (54)", - "tab": "Accuracy", - "CivilComments - ECE (10-bin)": "{\"description\": \"min=0.097, mean=0.381, max=0.605, sum=20.56 (54)\", \"tab\": \"Calibration\", \"score\": \"0.38073513412444826\"}", - "CivilComments - EM (Robustness)": "{\"description\": \"min=0, mean=0.488, max=0.986, sum=26.326 (54)\", \"tab\": \"Robustness\", \"score\": \"0.4875180109221431\"}", - "CivilComments - EM (Fairness)": "{\"description\": \"min=0, mean=0.445, max=0.973, sum=24.007 (54)\", \"tab\": \"Fairness\", \"score\": \"0.44457169485758724\"}", - "CivilComments - Denoised inference time (s)": "{\"description\": \"9 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "CivilComments - # eval": "{\"description\": \"min=74, mean=371.556, max=683, sum=20064 (54)\", \"tab\": \"General information\", \"score\": \"371.55555555555554\"}", - "CivilComments - # train": "{\"description\": \"min=5, mean=5, max=5, sum=270 (54)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CivilComments - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (54)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CivilComments - # prompt tokens": "{\"description\": \"min=271.927, mean=532.602, max=942.498, sum=28760.487 (54)\", \"tab\": \"General information\", \"score\": \"532.6016121330534\"}", - "CivilComments - # output tokens": "{\"description\": \"min=2, mean=2, max=2, sum=108 (54)\", \"tab\": \"General information\", \"score\": \"2.0\"}", - "CivilComments - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=162 (54)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "CivilComments - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Toxic fraction": "{\"description\": \"9 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "RAFT", - "source_data": { - "dataset_name": "RAFT", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on RAFT", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.712, - "details": { - "description": "min=0.225, mean=0.712, max=0.975, sum=23.5 (33)", - "tab": "Accuracy", - "RAFT - ECE (10-bin)": "{\"description\": \"min=0.119, mean=0.232, max=0.581, sum=7.664 (33)\", \"tab\": \"Calibration\", \"score\": \"0.23222744852932867\"}", - "RAFT - EM (Robustness)": "{\"description\": \"min=0.025, mean=0.618, max=0.875, sum=20.4 (33)\", \"tab\": \"Robustness\", \"score\": \"0.6181818181818182\"}", - "RAFT - EM (Fairness)": "{\"description\": \"min=0.2, mean=0.689, max=0.975, sum=22.725 (33)\", \"tab\": \"Fairness\", \"score\": \"0.6886363636363637\"}", - "RAFT - Denoised inference time (s)": "{\"description\": \"11 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "RAFT - # eval": "{\"description\": \"min=40, mean=40, max=40, sum=1320 (33)\", \"tab\": \"General information\", \"score\": \"40.0\"}", - "RAFT - # train": "{\"description\": \"min=0.95, mean=4.658, max=5, sum=153.7 (33)\", \"tab\": \"General information\", \"score\": \"4.657575757575757\"}", - "RAFT - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (33)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "RAFT - # prompt tokens": "{\"description\": \"min=212.25, mean=712.248, max=1745.25, sum=23504.175 (33)\", \"tab\": \"General information\", \"score\": \"712.2477272727273\"}", - "RAFT - # output tokens": "{\"description\": \"min=1.95, mean=3.644, max=6.3, sum=120.25 (33)\", \"tab\": \"General information\", \"score\": \"3.643939393939394\"}", - "RAFT - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=99 (33)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "RAFT - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Toxic fraction": "{\"description\": \"11 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/ai21_jurassic-2-jumbo-178b.json b/data/models/ai21_jurassic-2-jumbo-178b.json deleted file mode 100644 index 1792a0a5ae17b7f1618b48df0eaf306840aadd71..0000000000000000000000000000000000000000 --- a/data/models/ai21_jurassic-2-jumbo-178b.json +++ /dev/null @@ -1,674 +0,0 @@ -{ - "model_info": { - "name": "Jurassic-2 Jumbo 178B", - "id": "ai21/Jurassic-2-Jumbo-178B", - "developer": "ai21", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_classic/ai21_Jurassic-2-Jumbo-178B/1774096308.339228", - "retrieved_timestamp": "1774096308.339228", - "source_metadata": { - "source_name": "helm_classic", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_classic", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_classic", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperform on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.824, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Calibration": "{\"description\": \"\", \"tab\": \"Calibration\", \"score\": \"0.6597594819611471\"}", - "Mean win rate - Robustness": "{\"description\": \"\", \"tab\": \"Robustness\", \"score\": \"0.7910296229539834\"}", - "Mean win rate - Fairness": "{\"description\": \"\", \"tab\": \"Fairness\", \"score\": \"0.8360206534288848\"}", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}", - "Mean win rate - Bias": "{\"description\": \"\", \"tab\": \"Bias\", \"score\": \"0.5968189835436076\"}", - "Mean win rate - Toxicity": "{\"description\": \"\", \"tab\": \"Toxicity\", \"score\": \"0.5064102564102564\"}", - "Mean win rate - Summarization metrics": "{\"description\": \"\", \"tab\": \"Summarization metrics\", \"score\": \"0.6447368421052632\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.48, - "details": { - "description": "min=0.23, mean=0.48, max=0.83, sum=7.207 (15)", - "tab": "Accuracy", - "MMLU - ECE (10-bin)": "{\"description\": \"min=0.056, mean=0.137, max=0.248, sum=2.059 (15)\", \"tab\": \"Calibration\", \"score\": \"0.13723997934779486\"}", - "MMLU - EM (Robustness)": "{\"description\": \"min=0.17, mean=0.417, max=0.75, sum=6.251 (15)\", \"tab\": \"Robustness\", \"score\": \"0.41671345029239765\"}", - "MMLU - EM (Fairness)": "{\"description\": \"min=0.21, mean=0.45, max=0.78, sum=6.75 (15)\", \"tab\": \"Fairness\", \"score\": \"0.44997660818713453\"}", - "MMLU - Denoised inference time (s)": "{\"description\": \"5 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=1542 (15)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=75 (15)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (15)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=308.59, mean=396.74, max=552.719, sum=5951.098 (15)\", \"tab\": \"General information\", \"score\": \"396.73985964912276\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=15 (15)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "MMLU - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=45 (15)\", \"tab\": \"General information\", \"score\": \"3.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "BoolQ", - "source_data": { - "dataset_name": "BoolQ", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on BoolQ", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.829, - "details": { - "description": "min=0.818, mean=0.829, max=0.838, sum=2.487 (3)", - "tab": "Accuracy", - "BoolQ - ECE (10-bin)": "{\"description\": \"min=0.163, mean=0.175, max=0.198, sum=0.526 (3)\", \"tab\": \"Calibration\", \"score\": \"0.17545319159294462\"}", - "BoolQ - EM (Robustness)": "{\"description\": \"min=0.72, mean=0.729, max=0.736, sum=2.188 (3)\", \"tab\": \"Robustness\", \"score\": \"0.7293333333333333\"}", - "BoolQ - EM (Fairness)": "{\"description\": \"min=0.78, mean=0.792, max=0.798, sum=2.375 (3)\", \"tab\": \"Fairness\", \"score\": \"0.7916666666666666\"}", - "BoolQ - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "BoolQ - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "BoolQ - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "BoolQ - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "BoolQ - # prompt tokens": "{\"description\": \"min=506.985, mean=694.652, max=952.985, sum=2083.955 (3)\", \"tab\": \"General information\", \"score\": \"694.6516666666666\"}", - "BoolQ - # output tokens": "{\"description\": \"min=2, mean=2.002, max=2.003, sum=6.005 (3)\", \"tab\": \"General information\", \"score\": \"2.0016666666666665\"}", - "BoolQ - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "BoolQ - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.733, - "details": { - "description": "min=0.715, mean=0.733, max=0.757, sum=2.2 (3)", - "tab": "Accuracy", - "NarrativeQA - ECE (10-bin)": "{\"description\": \"min=0.068, mean=0.073, max=0.076, sum=0.219 (3)\", \"tab\": \"Calibration\", \"score\": \"0.07310994320832209\"}", - "NarrativeQA - F1 (Robustness)": "{\"description\": \"min=0.627, mean=0.66, max=0.69, sum=1.98 (3)\", \"tab\": \"Robustness\", \"score\": \"0.6601600341725052\"}", - "NarrativeQA - F1 (Fairness)": "{\"description\": \"min=0.63, mean=0.658, max=0.69, sum=1.973 (3)\", \"tab\": \"Fairness\", \"score\": \"0.6577011654908803\"}", - "NarrativeQA - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=1065 (3)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=2534.434, mean=2818.1, max=3027.434, sum=8454.301 (3)\", \"tab\": \"General information\", \"score\": \"2818.1004694835683\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=4.879, mean=6.406, max=7.755, sum=19.217 (3)\", \"tab\": \"General information\", \"score\": \"6.405633802816901\"}", - "NarrativeQA - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NarrativeQA - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NarrativeQA - Stereotypes (gender)": "{\"description\": \"min=0.385, mean=0.43, max=0.5, sum=1.29 (3)\", \"tab\": \"Bias\", \"score\": \"0.4298611111111111\"}", - "NarrativeQA - Representation (race)": "{\"description\": \"min=0.333, mean=0.5, max=0.667, sum=1 (2)\", \"tab\": \"Bias\", \"score\": \"0.5\"}", - "NarrativeQA - Representation (gender)": "{\"description\": \"min=0.171, mean=0.183, max=0.192, sum=0.55 (3)\", \"tab\": \"Bias\", \"score\": \"0.18345814920903128\"}", - "NarrativeQA - Toxic fraction": "{\"description\": \"min=0.014, mean=0.017, max=0.02, sum=0.051 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.016901408450704227\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (open-book)", - "source_data": { - "dataset_name": "NaturalQuestions (open-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (open-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.669, - "details": { - "description": "min=0.65, mean=0.669, max=0.681, sum=2.007 (3)", - "tab": "Accuracy", - "NaturalQuestions (closed-book) - ECE (10-bin)": "{\"description\": \"min=0.018, mean=0.018, max=0.019, sum=0.054 (3)\", \"tab\": \"Calibration\", \"score\": \"0.018133452831606698\"}", - "NaturalQuestions (open-book) - ECE (10-bin)": "{\"description\": \"min=0.071, mean=0.073, max=0.076, sum=0.22 (3)\", \"tab\": \"Calibration\", \"score\": \"0.07345259187429393\"}", - "NaturalQuestions (closed-book) - F1 (Robustness)": "{\"description\": \"min=0.31, mean=0.315, max=0.318, sum=0.945 (3)\", \"tab\": \"Robustness\", \"score\": \"0.3150688575152197\"}", - "NaturalQuestions (open-book) - F1 (Robustness)": "{\"description\": \"min=0.576, mean=0.599, max=0.616, sum=1.796 (3)\", \"tab\": \"Robustness\", \"score\": \"0.5985032886794094\"}", - "NaturalQuestions (closed-book) - F1 (Fairness)": "{\"description\": \"min=0.326, mean=0.327, max=0.328, sum=0.982 (3)\", \"tab\": \"Fairness\", \"score\": \"0.32739768950953246\"}", - "NaturalQuestions (open-book) - F1 (Fairness)": "{\"description\": \"min=0.601, mean=0.62, max=0.633, sum=1.86 (3)\", \"tab\": \"Fairness\", \"score\": \"0.6201543217700605\"}", - "NaturalQuestions (closed-book) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NaturalQuestions (open-book) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=94.377, mean=99.377, max=102.377, sum=298.131 (3)\", \"tab\": \"General information\", \"score\": \"99.377\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=5.127, mean=5.365, max=5.79, sum=16.095 (3)\", \"tab\": \"General information\", \"score\": \"5.364999999999999\"}", - "NaturalQuestions (closed-book) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.928, mean=4.93, max=4.932, sum=14.791 (3)\", \"tab\": \"General information\", \"score\": \"4.9303333333333335\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.012, mean=0.012, max=0.012, sum=0.036 (3)\", \"tab\": \"General information\", \"score\": \"0.012000000000000002\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1254.565, mean=1571.171, max=1771.274, sum=4713.512 (3)\", \"tab\": \"General information\", \"score\": \"1571.1706666666669\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=4.785, mean=5.113, max=5.399, sum=15.338 (3)\", \"tab\": \"General information\", \"score\": \"5.112666666666667\"}", - "NaturalQuestions (open-book) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NaturalQuestions (closed-book) - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - Stereotypes (gender)": "{\"description\": \"min=0.5, mean=0.5, max=0.5, sum=1.5 (3)\", \"tab\": \"Bias\", \"score\": \"0.5\"}", - "NaturalQuestions (closed-book) - Representation (race)": "{\"description\": \"min=0.352, mean=0.376, max=0.405, sum=1.127 (3)\", \"tab\": \"Bias\", \"score\": \"0.3756261756261756\"}", - "NaturalQuestions (closed-book) - Representation (gender)": "{\"description\": \"min=0.033, mean=0.095, max=0.136, sum=0.285 (3)\", \"tab\": \"Bias\", \"score\": \"0.09502719502719503\"}", - "NaturalQuestions (open-book) - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=0.667 (1)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666667\"}", - "NaturalQuestions (open-book) - Stereotypes (gender)": "{\"description\": \"min=0.3, mean=0.413, max=0.5, sum=1.238 (3)\", \"tab\": \"Bias\", \"score\": \"0.41250000000000003\"}", - "NaturalQuestions (open-book) - Representation (race)": "{\"description\": \"min=0.514, mean=0.541, max=0.561, sum=1.624 (3)\", \"tab\": \"Bias\", \"score\": \"0.5414311179017061\"}", - "NaturalQuestions (open-book) - Representation (gender)": "{\"description\": \"min=0.06, mean=0.107, max=0.132, sum=0.321 (3)\", \"tab\": \"Bias\", \"score\": \"0.10706952566601687\"}", - "NaturalQuestions (closed-book) - Toxic fraction": "{\"description\": \"min=0, mean=0.001, max=0.001, sum=0.002 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.0006666666666666666\"}", - "NaturalQuestions (open-book) - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "QuAC", - "source_data": { - "dataset_name": "QuAC", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on QuAC", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.435, - "details": { - "description": "min=0.426, mean=0.435, max=0.446, sum=1.305 (3)", - "tab": "Accuracy", - "QuAC - ECE (10-bin)": "{\"description\": \"min=0.032, mean=0.035, max=0.037, sum=0.104 (3)\", \"tab\": \"Calibration\", \"score\": \"0.03466023181877799\"}", - "QuAC - F1 (Robustness)": "{\"description\": \"min=0.31, mean=0.314, max=0.316, sum=0.941 (3)\", \"tab\": \"Robustness\", \"score\": \"0.3135172870245195\"}", - "QuAC - F1 (Fairness)": "{\"description\": \"min=0.333, mean=0.34, max=0.348, sum=1.02 (3)\", \"tab\": \"Fairness\", \"score\": \"0.34006270092560414\"}", - "QuAC - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "QuAC - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "QuAC - # train": "{\"description\": \"min=4.999, mean=5.0, max=5, sum=14.999 (3)\", \"tab\": \"General information\", \"score\": \"4.999666666666666\"}", - "QuAC - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "QuAC - # prompt tokens": "{\"description\": \"min=3587.32, mean=4018.779, max=4568.698, sum=12056.338 (3)\", \"tab\": \"General information\", \"score\": \"4018.7793333333334\"}", - "QuAC - # output tokens": "{\"description\": \"min=21.621, mean=22.178, max=22.826, sum=66.533 (3)\", \"tab\": \"General information\", \"score\": \"22.177666666666664\"}", - "QuAC - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "QuAC - Stereotypes (race)": "{\"description\": \"min=0.615, mean=0.642, max=0.667, sum=1.925 (3)\", \"tab\": \"Bias\", \"score\": \"0.6416361416361417\"}", - "QuAC - Stereotypes (gender)": "{\"description\": \"min=0.425, mean=0.454, max=0.476, sum=1.363 (3)\", \"tab\": \"Bias\", \"score\": \"0.45448951168627727\"}", - "QuAC - Representation (race)": "{\"description\": \"min=0.342, mean=0.359, max=0.375, sum=1.078 (3)\", \"tab\": \"Bias\", \"score\": \"0.35949126363389555\"}", - "QuAC - Representation (gender)": "{\"description\": \"min=0.22, mean=0.232, max=0.241, sum=0.696 (3)\", \"tab\": \"Bias\", \"score\": \"0.23190752816365634\"}", - "QuAC - Toxic fraction": "{\"description\": \"min=0, mean=0.001, max=0.002, sum=0.003 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.001\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "HellaSwag", - "source_data": { - "dataset_name": "HellaSwag", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on HellaSwag", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.788, - "details": { - "description": "min=0.788, mean=0.788, max=0.788, sum=0.788 (1)", - "tab": "Accuracy", - "HellaSwag - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "HellaSwag - EM (Robustness)": "{\"description\": \"min=0.754, mean=0.754, max=0.754, sum=0.754 (1)\", \"tab\": \"Robustness\", \"score\": \"0.754\"}", - "HellaSwag - EM (Fairness)": "{\"description\": \"min=0.655, mean=0.655, max=0.655, sum=0.655 (1)\", \"tab\": \"Fairness\", \"score\": \"0.655\"}", - "HellaSwag - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "HellaSwag - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "HellaSwag - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "HellaSwag - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "HellaSwag - # prompt tokens": "{\"description\": \"min=62.466, mean=62.466, max=62.466, sum=62.466 (1)\", \"tab\": \"General information\", \"score\": \"62.466\"}", - "HellaSwag - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "HellaSwag - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.558, - "details": { - "description": "min=0.558, mean=0.558, max=0.558, sum=0.558 (1)", - "tab": "Accuracy", - "OpenbookQA - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "OpenbookQA - EM (Robustness)": "{\"description\": \"min=0.47, mean=0.47, max=0.47, sum=0.47 (1)\", \"tab\": \"Robustness\", \"score\": \"0.47\"}", - "OpenbookQA - EM (Fairness)": "{\"description\": \"min=0.488, mean=0.488, max=0.488, sum=0.488 (1)\", \"tab\": \"Fairness\", \"score\": \"0.488\"}", - "OpenbookQA - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=4.348, mean=4.348, max=4.348, sum=4.348 (1)\", \"tab\": \"General information\", \"score\": \"4.348\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "TruthfulQA", - "source_data": { - "dataset_name": "TruthfulQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on TruthfulQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.437, - "details": { - "description": "min=0.367, mean=0.437, max=0.485, sum=1.312 (3)", - "tab": "Accuracy", - "TruthfulQA - ECE (10-bin)": "{\"description\": \"min=0.049, mean=0.068, max=0.095, sum=0.203 (3)\", \"tab\": \"Calibration\", \"score\": \"0.06751578986419772\"}", - "TruthfulQA - EM (Robustness)": "{\"description\": \"min=0.326, mean=0.39, max=0.43, sum=1.17 (3)\", \"tab\": \"Robustness\", \"score\": \"0.38990825688073394\"}", - "TruthfulQA - EM (Fairness)": "{\"description\": \"min=0.289, mean=0.354, max=0.398, sum=1.063 (3)\", \"tab\": \"Fairness\", \"score\": \"0.35423037716615696\"}", - "TruthfulQA - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "TruthfulQA - # eval": "{\"description\": \"min=654, mean=654, max=654, sum=1962 (3)\", \"tab\": \"General information\", \"score\": \"654.0\"}", - "TruthfulQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "TruthfulQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "TruthfulQA - # prompt tokens": "{\"description\": \"min=317.682, mean=355.015, max=375.682, sum=1065.046 (3)\", \"tab\": \"General information\", \"score\": \"355.0152905198777\"}", - "TruthfulQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=3 (3)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "TruthfulQA - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MS MARCO (TREC)", - "source_data": { - "dataset_name": "MS MARCO (TREC)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "NDCG@10 on MS MARCO (TREC)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.661, - "details": { - "description": "min=0.62, mean=0.661, max=0.706, sum=1.982 (3)", - "tab": "Accuracy", - "MS MARCO (regular) - RR@10 (Robustness)": "{\"description\": \"min=0.333, mean=0.337, max=0.343, sum=1.012 (3)\", \"tab\": \"Robustness\", \"score\": \"0.3372691798941794\"}", - "MS MARCO (TREC) - NDCG@10 (Robustness)": "{\"description\": \"min=0.569, mean=0.607, max=0.639, sum=1.821 (3)\", \"tab\": \"Robustness\", \"score\": \"0.6069545244562901\"}", - "MS MARCO (regular) - RR@10 (Fairness)": "{\"description\": \"min=0.339, mean=0.342, max=0.346, sum=1.027 (3)\", \"tab\": \"Fairness\", \"score\": \"0.34235396825396786\"}", - "MS MARCO (TREC) - NDCG@10 (Fairness)": "{\"description\": \"min=0.578, mean=0.62, max=0.66, sum=1.861 (3)\", \"tab\": \"Fairness\", \"score\": \"0.6202649047028815\"}", - "MS MARCO (regular) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MS MARCO (TREC) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MS MARCO (regular) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "MS MARCO (regular) - # train": "{\"description\": \"min=2, mean=2, max=2, sum=6 (3)\", \"tab\": \"General information\", \"score\": \"2.0\"}", - "MS MARCO (regular) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MS MARCO (regular) - # prompt tokens": "{\"description\": \"min=349.303, mean=385.636, max=423.303, sum=1156.909 (3)\", \"tab\": \"General information\", \"score\": \"385.63633333333337\"}", - "MS MARCO (regular) - # output tokens": "{\"description\": \"min=2, mean=2.001, max=2.003, sum=6.003 (3)\", \"tab\": \"General information\", \"score\": \"2.001\"}", - "MS MARCO (regular) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "MS MARCO (TREC) - # eval": "{\"description\": \"min=43, mean=43, max=43, sum=129 (3)\", \"tab\": \"General information\", \"score\": \"43.0\"}", - "MS MARCO (TREC) - # train": "{\"description\": \"min=2, mean=2, max=2, sum=6 (3)\", \"tab\": \"General information\", \"score\": \"2.0\"}", - "MS MARCO (TREC) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MS MARCO (TREC) - # prompt tokens": "{\"description\": \"min=337.047, mean=373.38, max=411.047, sum=1120.14 (3)\", \"tab\": \"General information\", \"score\": \"373.3798449612403\"}", - "MS MARCO (TREC) - # output tokens": "{\"description\": \"min=2, mean=2, max=2, sum=6 (3)\", \"tab\": \"General information\", \"score\": \"2.0\"}", - "MS MARCO (TREC) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "MS MARCO (regular) - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "MS MARCO (TREC) - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CNN/DailyMail", - "source_data": { - "dataset_name": "CNN/DailyMail", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on CNN/DailyMail", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.149, - "details": { - "description": "min=0.142, mean=0.149, max=0.157, sum=0.892 (6)", - "tab": "Accuracy", - "CNN/DailyMail - Denoised inference time (s)": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "CNN/DailyMail - # eval": "{\"description\": \"min=466, mean=466, max=466, sum=2796 (6)\", \"tab\": \"General information\", \"score\": \"466.0\"}", - "CNN/DailyMail - # train": "{\"description\": \"min=5, mean=5, max=5, sum=30 (6)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CNN/DailyMail - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (6)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CNN/DailyMail - # prompt tokens": "{\"description\": \"min=1203.032, mean=1213.032, max=1224.032, sum=7278.193 (6)\", \"tab\": \"General information\", \"score\": \"1213.0321888412018\"}", - "CNN/DailyMail - # output tokens": "{\"description\": \"min=47.208, mean=49.239, max=51.633, sum=295.433 (6)\", \"tab\": \"General information\", \"score\": \"49.238912732474965\"}", - "CNN/DailyMail - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=18 (6)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "CNN/DailyMail - Stereotypes (race)": "{\"description\": \"min=0.593, mean=0.608, max=0.618, sum=3.649 (6)\", \"tab\": \"Bias\", \"score\": \"0.6082305358040653\"}", - "CNN/DailyMail - Stereotypes (gender)": "{\"description\": \"min=0.396, mean=0.411, max=0.434, sum=2.467 (6)\", \"tab\": \"Bias\", \"score\": \"0.4111171483483329\"}", - "CNN/DailyMail - Representation (race)": "{\"description\": \"min=0.177, mean=0.254, max=0.301, sum=1.526 (6)\", \"tab\": \"Bias\", \"score\": \"0.25438070908615346\"}", - "CNN/DailyMail - Representation (gender)": "{\"description\": \"min=0.064, mean=0.083, max=0.119, sum=0.497 (6)\", \"tab\": \"Bias\", \"score\": \"0.08290586755395449\"}", - "CNN/DailyMail - Toxic fraction": "{\"description\": \"min=0, mean=0.001, max=0.002, sum=0.009 (6)\", \"tab\": \"Toxicity\", \"score\": \"0.001430615164520744\"}", - "CNN/DailyMail - SummaC": "{\"description\": \"min=0.442, mean=0.489, max=0.543, sum=1.468 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.48944984939262354\"}", - "CNN/DailyMail - QAFactEval": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - BERTScore (F1)": "{\"description\": \"min=0.299, mean=0.313, max=0.33, sum=0.94 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.31320318480412634\"}", - "CNN/DailyMail - Coverage": "{\"description\": \"min=0.952, mean=0.957, max=0.964, sum=5.745 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"0.9574608785885589\"}", - "CNN/DailyMail - Density": "{\"description\": \"min=12.535, mean=15.317, max=20.424, sum=91.904 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"15.31737957113954\"}", - "CNN/DailyMail - Compression": "{\"description\": \"min=11.81, mean=12.304, max=13.072, sum=73.827 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"12.30449736723726\"}", - "CNN/DailyMail - HumanEval-faithfulness": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-relevance": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-coherence": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "XSUM", - "source_data": { - "dataset_name": "XSUM", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on XSUM", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.182, - "details": { - "description": "min=0.177, mean=0.182, max=0.186, sum=1.09 (6)", - "tab": "Accuracy", - "XSUM - Denoised inference time (s)": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "XSUM - # eval": "{\"description\": \"min=518, mean=518, max=518, sum=3108 (6)\", \"tab\": \"General information\", \"score\": \"518.0\"}", - "XSUM - # train": "{\"description\": \"min=5, mean=5, max=5, sum=30 (6)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "XSUM - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (6)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "XSUM - # prompt tokens": "{\"description\": \"min=1099.388, mean=1133.388, max=1172.388, sum=6800.328 (6)\", \"tab\": \"General information\", \"score\": \"1133.388030888031\"}", - "XSUM - # output tokens": "{\"description\": \"min=21.909, mean=22.142, max=22.392, sum=132.853 (6)\", \"tab\": \"General information\", \"score\": \"22.142213642213644\"}", - "XSUM - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=18 (6)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "XSUM - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=4 (6)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "XSUM - Stereotypes (gender)": "{\"description\": \"min=0.451, mean=0.466, max=0.478, sum=2.796 (6)\", \"tab\": \"Bias\", \"score\": \"0.4660306771417882\"}", - "XSUM - Representation (race)": "{\"description\": \"min=0.362, mean=0.399, max=0.429, sum=2.397 (6)\", \"tab\": \"Bias\", \"score\": \"0.39943255885284873\"}", - "XSUM - Representation (gender)": "{\"description\": \"min=0.189, mean=0.205, max=0.224, sum=1.232 (6)\", \"tab\": \"Bias\", \"score\": \"0.20538608377971754\"}", - "XSUM - Toxic fraction": "{\"description\": \"min=0.002, mean=0.003, max=0.004, sum=0.019 (6)\", \"tab\": \"Toxicity\", \"score\": \"0.0032175032175032173\"}", - "XSUM - SummaC": "{\"description\": \"min=-0.325, mean=-0.32, max=-0.314, sum=-0.96 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"-0.31997175372142944\"}", - "XSUM - QAFactEval": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - BERTScore (F1)": "{\"description\": \"min=0.484, mean=0.489, max=0.493, sum=1.468 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.4894925021585029\"}", - "XSUM - Coverage": "{\"description\": \"min=0.75, mean=0.755, max=0.761, sum=4.53 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"0.7549647155240389\"}", - "XSUM - Density": "{\"description\": \"min=1.852, mean=2.145, max=2.331, sum=12.869 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"2.144865535443147\"}", - "XSUM - Compression": "{\"description\": \"min=16.369, mean=16.589, max=16.81, sum=99.535 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"16.58922760069323\"}", - "XSUM - HumanEval-faithfulness": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-relevance": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-coherence": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "IMDB", - "source_data": { - "dataset_name": "IMDB", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on IMDB", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.938, - "details": { - "description": "min=0.936, mean=0.938, max=0.943, sum=2.815 (3)", - "tab": "Accuracy", - "IMDB - ECE (10-bin)": "{\"description\": \"min=0.157, mean=0.182, max=0.199, sum=0.546 (3)\", \"tab\": \"Calibration\", \"score\": \"0.18203122522171636\"}", - "IMDB - EM (Robustness)": "{\"description\": \"min=0.878, mean=0.896, max=0.916, sum=2.688 (3)\", \"tab\": \"Robustness\", \"score\": \"0.896\"}", - "IMDB - EM (Fairness)": "{\"description\": \"min=0.928, mean=0.933, max=0.937, sum=2.799 (3)\", \"tab\": \"Fairness\", \"score\": \"0.9329999999999999\"}", - "IMDB - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "IMDB - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "IMDB - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "IMDB - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IMDB - # prompt tokens": "{\"description\": \"min=853.851, mean=1288.518, max=1745.851, sum=3865.553 (3)\", \"tab\": \"General information\", \"score\": \"1288.5176666666669\"}", - "IMDB - # output tokens": "{\"description\": \"min=2, mean=2, max=2, sum=6 (3)\", \"tab\": \"General information\", \"score\": \"2.0\"}", - "IMDB - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "IMDB - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CivilComments", - "source_data": { - "dataset_name": "CivilComments", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on CivilComments", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.57, - "details": { - "description": "min=0.011, mean=0.57, max=1, sum=30.805 (54)", - "tab": "Accuracy", - "CivilComments - ECE (10-bin)": "{\"description\": \"min=0.07, mean=0.314, max=0.578, sum=16.962 (54)\", \"tab\": \"Calibration\", \"score\": \"0.31411210820302815\"}", - "CivilComments - EM (Robustness)": "{\"description\": \"min=0.009, mean=0.449, max=0.979, sum=24.224 (54)\", \"tab\": \"Robustness\", \"score\": \"0.4485846578472439\"}", - "CivilComments - EM (Fairness)": "{\"description\": \"min=0.005, mean=0.507, max=0.995, sum=27.37 (54)\", \"tab\": \"Fairness\", \"score\": \"0.5068507198702314\"}", - "CivilComments - Denoised inference time (s)": "{\"description\": \"9 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "CivilComments - # eval": "{\"description\": \"min=74, mean=371.556, max=683, sum=20064 (54)\", \"tab\": \"General information\", \"score\": \"371.55555555555554\"}", - "CivilComments - # train": "{\"description\": \"min=5, mean=5, max=5, sum=270 (54)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CivilComments - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (54)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CivilComments - # prompt tokens": "{\"description\": \"min=271.927, mean=532.602, max=942.498, sum=28760.487 (54)\", \"tab\": \"General information\", \"score\": \"532.6016121330534\"}", - "CivilComments - # output tokens": "{\"description\": \"min=2, mean=2, max=2, sum=108 (54)\", \"tab\": \"General information\", \"score\": \"2.0\"}", - "CivilComments - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=162 (54)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "CivilComments - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Toxic fraction": "{\"description\": \"9 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "RAFT", - "source_data": { - "dataset_name": "RAFT", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on RAFT", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.746, - "details": { - "description": "min=0.225, mean=0.746, max=0.975, sum=24.625 (33)", - "tab": "Accuracy", - "RAFT - ECE (10-bin)": "{\"description\": \"min=0.126, mean=0.218, max=0.683, sum=7.184 (33)\", \"tab\": \"Calibration\", \"score\": \"0.2177038585857703\"}", - "RAFT - EM (Robustness)": "{\"description\": \"min=0.225, mean=0.69, max=0.95, sum=22.775 (33)\", \"tab\": \"Robustness\", \"score\": \"0.6901515151515151\"}", - "RAFT - EM (Fairness)": "{\"description\": \"min=0.225, mean=0.711, max=0.975, sum=23.45 (33)\", \"tab\": \"Fairness\", \"score\": \"0.7106060606060605\"}", - "RAFT - Denoised inference time (s)": "{\"description\": \"11 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "RAFT - # eval": "{\"description\": \"min=40, mean=40, max=40, sum=1320 (33)\", \"tab\": \"General information\", \"score\": \"40.0\"}", - "RAFT - # train": "{\"description\": \"min=5, mean=5, max=5, sum=165 (33)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "RAFT - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (33)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "RAFT - # prompt tokens": "{\"description\": \"min=212.25, mean=944.157, max=4506.05, sum=31157.175 (33)\", \"tab\": \"General information\", \"score\": \"944.1568181818182\"}", - "RAFT - # output tokens": "{\"description\": \"min=2, mean=3.597, max=7.275, sum=118.7 (33)\", \"tab\": \"General information\", \"score\": \"3.5969696969696967\"}", - "RAFT - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=99 (33)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "RAFT - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Toxic fraction": "{\"description\": \"11 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/ai21_jurassic-2-large-7.5b.json b/data/models/ai21_jurassic-2-large-7.5b.json deleted file mode 100644 index 387a433cdeace95208645f6304309d9b7cef738c..0000000000000000000000000000000000000000 --- a/data/models/ai21_jurassic-2-large-7.5b.json +++ /dev/null @@ -1,674 +0,0 @@ -{ - "model_info": { - "name": "Jurassic-2 Large 7.5B", - "id": "ai21/Jurassic-2-Large-7.5B", - "developer": "ai21", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_classic/ai21_Jurassic-2-Large-7.5B/1774096308.339228", - "retrieved_timestamp": "1774096308.339228", - "source_metadata": { - "source_name": "helm_classic", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_classic", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_classic", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperform on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.553, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Calibration": "{\"description\": \"\", \"tab\": \"Calibration\", \"score\": \"0.6435013876040703\"}", - "Mean win rate - Robustness": "{\"description\": \"\", \"tab\": \"Robustness\", \"score\": \"0.5267325431952796\"}", - "Mean win rate - Fairness": "{\"description\": \"\", \"tab\": \"Fairness\", \"score\": \"0.48311004284307957\"}", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}", - "Mean win rate - Bias": "{\"description\": \"\", \"tab\": \"Bias\", \"score\": \"0.4461156665667944\"}", - "Mean win rate - Toxicity": "{\"description\": \"\", \"tab\": \"Toxicity\", \"score\": \"0.4555798368298368\"}", - "Mean win rate - Summarization metrics": "{\"description\": \"\", \"tab\": \"Summarization metrics\", \"score\": \"0.5723684210526315\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.339, - "details": { - "description": "min=0.211, mean=0.339, max=0.5, sum=5.078 (15)", - "tab": "Accuracy", - "MMLU - ECE (10-bin)": "{\"description\": \"min=0.06, mean=0.141, max=0.219, sum=2.11 (15)\", \"tab\": \"Calibration\", \"score\": \"0.1406708954092635\"}", - "MMLU - EM (Robustness)": "{\"description\": \"min=0.17, mean=0.263, max=0.42, sum=3.938 (15)\", \"tab\": \"Robustness\", \"score\": \"0.2625146198830409\"}", - "MMLU - EM (Fairness)": "{\"description\": \"min=0.167, mean=0.297, max=0.45, sum=4.453 (15)\", \"tab\": \"Fairness\", \"score\": \"0.2968421052631579\"}", - "MMLU - Denoised inference time (s)": "{\"description\": \"5 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=1542 (15)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=75 (15)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (15)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=308.59, mean=396.74, max=552.719, sum=5951.098 (15)\", \"tab\": \"General information\", \"score\": \"396.73985964912276\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=15 (15)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "MMLU - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=45 (15)\", \"tab\": \"General information\", \"score\": \"3.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "BoolQ", - "source_data": { - "dataset_name": "BoolQ", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on BoolQ", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.742, - "details": { - "description": "min=0.737, mean=0.742, max=0.747, sum=2.227 (3)", - "tab": "Accuracy", - "BoolQ - ECE (10-bin)": "{\"description\": \"min=0.126, mean=0.147, max=0.165, sum=0.442 (3)\", \"tab\": \"Calibration\", \"score\": \"0.14720347227904834\"}", - "BoolQ - EM (Robustness)": "{\"description\": \"min=0.602, mean=0.607, max=0.615, sum=1.822 (3)\", \"tab\": \"Robustness\", \"score\": \"0.6073333333333334\"}", - "BoolQ - EM (Fairness)": "{\"description\": \"min=0.675, mean=0.685, max=0.697, sum=2.055 (3)\", \"tab\": \"Fairness\", \"score\": \"0.685\"}", - "BoolQ - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "BoolQ - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "BoolQ - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "BoolQ - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "BoolQ - # prompt tokens": "{\"description\": \"min=506.985, mean=694.652, max=952.985, sum=2083.955 (3)\", \"tab\": \"General information\", \"score\": \"694.6516666666666\"}", - "BoolQ - # output tokens": "{\"description\": \"min=2, mean=2, max=2, sum=6 (3)\", \"tab\": \"General information\", \"score\": \"2.0\"}", - "BoolQ - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "BoolQ - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "NarrativeQA - ECE (10-bin)": "{\"description\": \"No matching runs\", \"tab\": \"Calibration\", \"score\": \"\"}", - "NarrativeQA - F1 (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "NarrativeQA - F1 (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "NarrativeQA - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NarrativeQA - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "NarrativeQA - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "NarrativeQA - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "NarrativeQA - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "NarrativeQA - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "NarrativeQA - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "NarrativeQA - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "NarrativeQA - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "NarrativeQA - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "NarrativeQA - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (open-book)", - "source_data": { - "dataset_name": "NaturalQuestions (open-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (open-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.589, - "details": { - "description": "min=0.576, mean=0.589, max=0.605, sum=1.766 (3)", - "tab": "Accuracy", - "NaturalQuestions (closed-book) - ECE (10-bin)": "{\"description\": \"min=0.008, mean=0.014, max=0.021, sum=0.042 (3)\", \"tab\": \"Calibration\", \"score\": \"0.01399000614897039\"}", - "NaturalQuestions (open-book) - ECE (10-bin)": "{\"description\": \"min=0.081, mean=0.084, max=0.089, sum=0.253 (3)\", \"tab\": \"Calibration\", \"score\": \"0.08428284450081218\"}", - "NaturalQuestions (closed-book) - F1 (Robustness)": "{\"description\": \"min=0.177, mean=0.187, max=0.195, sum=0.562 (3)\", \"tab\": \"Robustness\", \"score\": \"0.18733342573827472\"}", - "NaturalQuestions (open-book) - F1 (Robustness)": "{\"description\": \"min=0.485, mean=0.503, max=0.529, sum=1.51 (3)\", \"tab\": \"Robustness\", \"score\": \"0.5031846716563587\"}", - "NaturalQuestions (closed-book) - F1 (Fairness)": "{\"description\": \"min=0.215, mean=0.217, max=0.221, sum=0.652 (3)\", \"tab\": \"Fairness\", \"score\": \"0.21726190588701\"}", - "NaturalQuestions (open-book) - F1 (Fairness)": "{\"description\": \"min=0.53, mean=0.539, max=0.557, sum=1.616 (3)\", \"tab\": \"Fairness\", \"score\": \"0.5388295929563434\"}", - "NaturalQuestions (closed-book) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NaturalQuestions (open-book) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=94.377, mean=99.377, max=102.377, sum=298.131 (3)\", \"tab\": \"General information\", \"score\": \"99.377\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=5.924, mean=6.729, max=7.956, sum=20.187 (3)\", \"tab\": \"General information\", \"score\": \"6.729\"}", - "NaturalQuestions (closed-book) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.568, mean=4.666, max=4.734, sum=13.999 (3)\", \"tab\": \"General information\", \"score\": \"4.666333333333333\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.038, mean=0.038, max=0.038, sum=0.114 (3)\", \"tab\": \"General information\", \"score\": \"0.038\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1136.933, mean=1418.457, max=1595.508, sum=4255.37 (3)\", \"tab\": \"General information\", \"score\": \"1418.4566666666667\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=5.825, mean=6.311, max=6.845, sum=18.932 (3)\", \"tab\": \"General information\", \"score\": \"6.310666666666666\"}", - "NaturalQuestions (open-book) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NaturalQuestions (closed-book) - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - Stereotypes (gender)": "{\"description\": \"min=0.5, mean=0.5, max=0.5, sum=1.5 (3)\", \"tab\": \"Bias\", \"score\": \"0.5\"}", - "NaturalQuestions (closed-book) - Representation (race)": "{\"description\": \"min=0.502, mean=0.531, max=0.563, sum=1.594 (3)\", \"tab\": \"Bias\", \"score\": \"0.5313654482080615\"}", - "NaturalQuestions (closed-book) - Representation (gender)": "{\"description\": \"min=0, mean=0.079, max=0.192, sum=0.238 (3)\", \"tab\": \"Bias\", \"score\": \"0.07925407925407925\"}", - "NaturalQuestions (open-book) - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=0.667 (1)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666667\"}", - "NaturalQuestions (open-book) - Stereotypes (gender)": "{\"description\": \"min=0.3, mean=0.433, max=0.5, sum=1.3 (3)\", \"tab\": \"Bias\", \"score\": \"0.43333333333333335\"}", - "NaturalQuestions (open-book) - Representation (race)": "{\"description\": \"min=0.441, mean=0.504, max=0.574, sum=1.513 (3)\", \"tab\": \"Bias\", \"score\": \"0.5041929581337629\"}", - "NaturalQuestions (open-book) - Representation (gender)": "{\"description\": \"min=0.186, mean=0.203, max=0.225, sum=0.608 (3)\", \"tab\": \"Bias\", \"score\": \"0.20273109243697482\"}", - "NaturalQuestions (closed-book) - Toxic fraction": "{\"description\": \"min=0.001, mean=0.001, max=0.001, sum=0.003 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.001\"}", - "NaturalQuestions (open-book) - Toxic fraction": "{\"description\": \"min=0.001, mean=0.002, max=0.003, sum=0.006 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.002\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "QuAC", - "source_data": { - "dataset_name": "QuAC", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on QuAC", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "QuAC - ECE (10-bin)": "{\"description\": \"No matching runs\", \"tab\": \"Calibration\", \"score\": \"\"}", - "QuAC - F1 (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "QuAC - F1 (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "QuAC - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "QuAC - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "QuAC - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "QuAC - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "QuAC - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "QuAC - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "QuAC - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "QuAC - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "QuAC - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "QuAC - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "QuAC - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "QuAC - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "HellaSwag", - "source_data": { - "dataset_name": "HellaSwag", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on HellaSwag", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.729, - "details": { - "description": "min=0.729, mean=0.729, max=0.729, sum=0.729 (1)", - "tab": "Accuracy", - "HellaSwag - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "HellaSwag - EM (Robustness)": "{\"description\": \"min=0.687, mean=0.687, max=0.687, sum=0.687 (1)\", \"tab\": \"Robustness\", \"score\": \"0.687\"}", - "HellaSwag - EM (Fairness)": "{\"description\": \"min=0.567, mean=0.567, max=0.567, sum=0.567 (1)\", \"tab\": \"Fairness\", \"score\": \"0.567\"}", - "HellaSwag - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "HellaSwag - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "HellaSwag - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "HellaSwag - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "HellaSwag - # prompt tokens": "{\"description\": \"min=62.466, mean=62.466, max=62.466, sum=62.466 (1)\", \"tab\": \"General information\", \"score\": \"62.466\"}", - "HellaSwag - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "HellaSwag - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.53, - "details": { - "description": "min=0.53, mean=0.53, max=0.53, sum=0.53 (1)", - "tab": "Accuracy", - "OpenbookQA - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "OpenbookQA - EM (Robustness)": "{\"description\": \"min=0.448, mean=0.448, max=0.448, sum=0.448 (1)\", \"tab\": \"Robustness\", \"score\": \"0.448\"}", - "OpenbookQA - EM (Fairness)": "{\"description\": \"min=0.45, mean=0.45, max=0.45, sum=0.45 (1)\", \"tab\": \"Fairness\", \"score\": \"0.45\"}", - "OpenbookQA - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=4.348, mean=4.348, max=4.348, sum=4.348 (1)\", \"tab\": \"General information\", \"score\": \"4.348\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "TruthfulQA", - "source_data": { - "dataset_name": "TruthfulQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on TruthfulQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.245, - "details": { - "description": "min=0.22, mean=0.245, max=0.283, sum=0.734 (3)", - "tab": "Accuracy", - "TruthfulQA - ECE (10-bin)": "{\"description\": \"min=0.043, mean=0.102, max=0.134, sum=0.306 (3)\", \"tab\": \"Calibration\", \"score\": \"0.1021312296645796\"}", - "TruthfulQA - EM (Robustness)": "{\"description\": \"min=0.197, mean=0.21, max=0.228, sum=0.63 (3)\", \"tab\": \"Robustness\", \"score\": \"0.20998980632008157\"}", - "TruthfulQA - EM (Fairness)": "{\"description\": \"min=0.185, mean=0.196, max=0.205, sum=0.589 (3)\", \"tab\": \"Fairness\", \"score\": \"0.1962283384301733\"}", - "TruthfulQA - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "TruthfulQA - # eval": "{\"description\": \"min=654, mean=654, max=654, sum=1962 (3)\", \"tab\": \"General information\", \"score\": \"654.0\"}", - "TruthfulQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "TruthfulQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "TruthfulQA - # prompt tokens": "{\"description\": \"min=317.682, mean=355.015, max=375.682, sum=1065.046 (3)\", \"tab\": \"General information\", \"score\": \"355.0152905198777\"}", - "TruthfulQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=3 (3)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "TruthfulQA - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MS MARCO (TREC)", - "source_data": { - "dataset_name": "MS MARCO (TREC)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "NDCG@10 on MS MARCO (TREC)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.464, - "details": { - "description": "min=0.454, mean=0.464, max=0.479, sum=1.393 (3)", - "tab": "Accuracy", - "MS MARCO (regular) - RR@10 (Robustness)": "{\"description\": \"min=0.168, mean=0.177, max=0.186, sum=0.532 (3)\", \"tab\": \"Robustness\", \"score\": \"0.1774849206349205\"}", - "MS MARCO (TREC) - NDCG@10 (Robustness)": "{\"description\": \"min=0.391, mean=0.397, max=0.403, sum=1.192 (3)\", \"tab\": \"Robustness\", \"score\": \"0.39737317282374035\"}", - "MS MARCO (regular) - RR@10 (Fairness)": "{\"description\": \"min=0.21, mean=0.215, max=0.221, sum=0.646 (3)\", \"tab\": \"Fairness\", \"score\": \"0.21544642857142837\"}", - "MS MARCO (TREC) - NDCG@10 (Fairness)": "{\"description\": \"min=0.432, mean=0.44, max=0.457, sum=1.32 (3)\", \"tab\": \"Fairness\", \"score\": \"0.44015360771598083\"}", - "MS MARCO (regular) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MS MARCO (TREC) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MS MARCO (regular) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "MS MARCO (regular) - # train": "{\"description\": \"min=2, mean=2, max=2, sum=6 (3)\", \"tab\": \"General information\", \"score\": \"2.0\"}", - "MS MARCO (regular) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MS MARCO (regular) - # prompt tokens": "{\"description\": \"min=349.303, mean=385.636, max=423.303, sum=1156.909 (3)\", \"tab\": \"General information\", \"score\": \"385.63633333333337\"}", - "MS MARCO (regular) - # output tokens": "{\"description\": \"min=2.006, mean=2.012, max=2.022, sum=6.037 (3)\", \"tab\": \"General information\", \"score\": \"2.012333333333333\"}", - "MS MARCO (regular) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "MS MARCO (TREC) - # eval": "{\"description\": \"min=43, mean=43, max=43, sum=129 (3)\", \"tab\": \"General information\", \"score\": \"43.0\"}", - "MS MARCO (TREC) - # train": "{\"description\": \"min=2, mean=2, max=2, sum=6 (3)\", \"tab\": \"General information\", \"score\": \"2.0\"}", - "MS MARCO (TREC) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MS MARCO (TREC) - # prompt tokens": "{\"description\": \"min=337.047, mean=373.38, max=411.047, sum=1120.14 (3)\", \"tab\": \"General information\", \"score\": \"373.3798449612403\"}", - "MS MARCO (TREC) - # output tokens": "{\"description\": \"min=2.023, mean=2.023, max=2.023, sum=6.07 (3)\", \"tab\": \"General information\", \"score\": \"2.0232558139534884\"}", - "MS MARCO (TREC) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "MS MARCO (regular) - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "MS MARCO (TREC) - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CNN/DailyMail", - "source_data": { - "dataset_name": "CNN/DailyMail", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on CNN/DailyMail", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.136, - "details": { - "description": "min=0.122, mean=0.136, max=0.15, sum=0.813 (6)", - "tab": "Accuracy", - "CNN/DailyMail - Denoised inference time (s)": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "CNN/DailyMail - # eval": "{\"description\": \"min=466, mean=466, max=466, sum=2796 (6)\", \"tab\": \"General information\", \"score\": \"466.0\"}", - "CNN/DailyMail - # train": "{\"description\": \"min=5, mean=5, max=5, sum=30 (6)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CNN/DailyMail - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (6)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CNN/DailyMail - # prompt tokens": "{\"description\": \"min=1203.032, mean=1213.032, max=1224.032, sum=7278.193 (6)\", \"tab\": \"General information\", \"score\": \"1213.0321888412018\"}", - "CNN/DailyMail - # output tokens": "{\"description\": \"min=52.573, mean=58.246, max=61.575, sum=349.476 (6)\", \"tab\": \"General information\", \"score\": \"58.24606580829757\"}", - "CNN/DailyMail - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=18 (6)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "CNN/DailyMail - Stereotypes (race)": "{\"description\": \"min=0.612, mean=0.647, max=0.667, sum=3.885 (6)\", \"tab\": \"Bias\", \"score\": \"0.6474734228728262\"}", - "CNN/DailyMail - Stereotypes (gender)": "{\"description\": \"min=0.365, mean=0.405, max=0.442, sum=2.432 (6)\", \"tab\": \"Bias\", \"score\": \"0.405313769914252\"}", - "CNN/DailyMail - Representation (race)": "{\"description\": \"min=0.175, mean=0.245, max=0.377, sum=1.468 (6)\", \"tab\": \"Bias\", \"score\": \"0.24474724360307878\"}", - "CNN/DailyMail - Representation (gender)": "{\"description\": \"min=0.103, mean=0.133, max=0.149, sum=0.796 (6)\", \"tab\": \"Bias\", \"score\": \"0.13266873135824753\"}", - "CNN/DailyMail - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (6)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}", - "CNN/DailyMail - SummaC": "{\"description\": \"min=0.465, mean=0.496, max=0.548, sum=1.488 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.49606841741715785\"}", - "CNN/DailyMail - QAFactEval": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - BERTScore (F1)": "{\"description\": \"min=0.242, mean=0.271, max=0.304, sum=0.812 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.27057214623114106\"}", - "CNN/DailyMail - Coverage": "{\"description\": \"min=0.952, mean=0.963, max=0.98, sum=5.779 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"0.9630886941006946\"}", - "CNN/DailyMail - Density": "{\"description\": \"min=15.279, mean=25.251, max=36.976, sum=151.506 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"25.250963083991945\"}", - "CNN/DailyMail - Compression": "{\"description\": \"min=9.923, mean=11.503, max=13.28, sum=69.019 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"11.503115138085485\"}", - "CNN/DailyMail - HumanEval-faithfulness": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-relevance": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-coherence": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "XSUM", - "source_data": { - "dataset_name": "XSUM", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on XSUM", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.142, - "details": { - "description": "min=0.14, mean=0.142, max=0.145, sum=0.853 (6)", - "tab": "Accuracy", - "XSUM - Denoised inference time (s)": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "XSUM - # eval": "{\"description\": \"min=518, mean=518, max=518, sum=3108 (6)\", \"tab\": \"General information\", \"score\": \"518.0\"}", - "XSUM - # train": "{\"description\": \"min=5, mean=5, max=5, sum=30 (6)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "XSUM - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (6)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "XSUM - # prompt tokens": "{\"description\": \"min=1099.388, mean=1133.388, max=1172.388, sum=6800.328 (6)\", \"tab\": \"General information\", \"score\": \"1133.388030888031\"}", - "XSUM - # output tokens": "{\"description\": \"min=21.112, mean=21.228, max=21.315, sum=127.371 (6)\", \"tab\": \"General information\", \"score\": \"21.22844272844273\"}", - "XSUM - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=18 (6)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "XSUM - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=4 (6)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "XSUM - Stereotypes (gender)": "{\"description\": \"min=0.433, mean=0.464, max=0.492, sum=2.785 (6)\", \"tab\": \"Bias\", \"score\": \"0.46417690732206857\"}", - "XSUM - Representation (race)": "{\"description\": \"min=0.407, mean=0.58, max=0.667, sum=3.481 (6)\", \"tab\": \"Bias\", \"score\": \"0.5802469135802469\"}", - "XSUM - Representation (gender)": "{\"description\": \"min=0.209, mean=0.22, max=0.234, sum=1.321 (6)\", \"tab\": \"Bias\", \"score\": \"0.2200902099970423\"}", - "XSUM - Toxic fraction": "{\"description\": \"min=0.002, mean=0.002, max=0.002, sum=0.012 (6)\", \"tab\": \"Toxicity\", \"score\": \"0.0019305019305019308\"}", - "XSUM - SummaC": "{\"description\": \"min=-0.306, mean=-0.278, max=-0.26, sum=-0.833 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"-0.27758991887056994\"}", - "XSUM - QAFactEval": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - BERTScore (F1)": "{\"description\": \"min=0.449, mean=0.45, max=0.451, sum=1.35 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.44989833153156206\"}", - "XSUM - Coverage": "{\"description\": \"min=0.781, mean=0.782, max=0.783, sum=4.694 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"0.7823704015893701\"}", - "XSUM - Density": "{\"description\": \"min=2.345, mean=2.659, max=2.826, sum=15.954 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"2.6589249165198687\"}", - "XSUM - Compression": "{\"description\": \"min=17.896, mean=18.03, max=18.26, sum=108.178 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"18.02961749079778\"}", - "XSUM - HumanEval-faithfulness": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-relevance": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-coherence": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "IMDB", - "source_data": { - "dataset_name": "IMDB", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on IMDB", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.956, - "details": { - "description": "min=0.952, mean=0.956, max=0.96, sum=2.869 (3)", - "tab": "Accuracy", - "IMDB - ECE (10-bin)": "{\"description\": \"min=0.153, mean=0.178, max=0.201, sum=0.534 (3)\", \"tab\": \"Calibration\", \"score\": \"0.17816129477822015\"}", - "IMDB - EM (Robustness)": "{\"description\": \"min=0.935, mean=0.941, max=0.946, sum=2.822 (3)\", \"tab\": \"Robustness\", \"score\": \"0.9406666666666667\"}", - "IMDB - EM (Fairness)": "{\"description\": \"min=0.941, mean=0.945, max=0.951, sum=2.835 (3)\", \"tab\": \"Fairness\", \"score\": \"0.945\"}", - "IMDB - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "IMDB - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "IMDB - # train": "{\"description\": \"min=4.915, mean=4.972, max=5, sum=14.915 (3)\", \"tab\": \"General information\", \"score\": \"4.971666666666667\"}", - "IMDB - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IMDB - # prompt tokens": "{\"description\": \"min=853.851, mean=1281.577, max=1725.03, sum=3844.732 (3)\", \"tab\": \"General information\", \"score\": \"1281.5773333333334\"}", - "IMDB - # output tokens": "{\"description\": \"min=2, mean=2, max=2, sum=6 (3)\", \"tab\": \"General information\", \"score\": \"2.0\"}", - "IMDB - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "IMDB - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CivilComments", - "source_data": { - "dataset_name": "CivilComments", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on CivilComments", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.57, - "details": { - "description": "min=0.149, mean=0.57, max=0.909, sum=30.8 (54)", - "tab": "Accuracy", - "CivilComments - ECE (10-bin)": "{\"description\": \"min=0.033, mean=0.19, max=0.41, sum=10.274 (54)\", \"tab\": \"Calibration\", \"score\": \"0.19026595574841215\"}", - "CivilComments - EM (Robustness)": "{\"description\": \"min=0.116, mean=0.469, max=0.844, sum=25.305 (54)\", \"tab\": \"Robustness\", \"score\": \"0.4686089323926605\"}", - "CivilComments - EM (Fairness)": "{\"description\": \"min=0.143, mean=0.403, max=0.834, sum=21.752 (54)\", \"tab\": \"Fairness\", \"score\": \"0.4028192827891808\"}", - "CivilComments - Denoised inference time (s)": "{\"description\": \"9 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "CivilComments - # eval": "{\"description\": \"min=74, mean=371.556, max=683, sum=20064 (54)\", \"tab\": \"General information\", \"score\": \"371.55555555555554\"}", - "CivilComments - # train": "{\"description\": \"min=5, mean=5, max=5, sum=270 (54)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CivilComments - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (54)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CivilComments - # prompt tokens": "{\"description\": \"min=271.927, mean=532.602, max=942.498, sum=28760.487 (54)\", \"tab\": \"General information\", \"score\": \"532.6016121330534\"}", - "CivilComments - # output tokens": "{\"description\": \"min=2, mean=2, max=2, sum=108 (54)\", \"tab\": \"General information\", \"score\": \"2.0\"}", - "CivilComments - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=162 (54)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "CivilComments - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Toxic fraction": "{\"description\": \"9 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "RAFT", - "source_data": { - "dataset_name": "RAFT", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on RAFT", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.622, - "details": { - "description": "min=0.25, mean=0.622, max=0.975, sum=20.525 (33)", - "tab": "Accuracy", - "RAFT - ECE (10-bin)": "{\"description\": \"min=0.128, mean=0.254, max=0.441, sum=8.368 (33)\", \"tab\": \"Calibration\", \"score\": \"0.25356461082010057\"}", - "RAFT - EM (Robustness)": "{\"description\": \"min=0, mean=0.498, max=0.975, sum=16.425 (33)\", \"tab\": \"Robustness\", \"score\": \"0.49772727272727263\"}", - "RAFT - EM (Fairness)": "{\"description\": \"min=0.2, mean=0.567, max=0.975, sum=18.725 (33)\", \"tab\": \"Fairness\", \"score\": \"0.5674242424242424\"}", - "RAFT - Denoised inference time (s)": "{\"description\": \"11 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "RAFT - # eval": "{\"description\": \"min=40, mean=40, max=40, sum=1320 (33)\", \"tab\": \"General information\", \"score\": \"40.0\"}", - "RAFT - # train": "{\"description\": \"min=0.95, mean=4.658, max=5, sum=153.7 (33)\", \"tab\": \"General information\", \"score\": \"4.657575757575757\"}", - "RAFT - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (33)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "RAFT - # prompt tokens": "{\"description\": \"min=212.25, mean=712.248, max=1745.25, sum=23504.175 (33)\", \"tab\": \"General information\", \"score\": \"712.2477272727273\"}", - "RAFT - # output tokens": "{\"description\": \"min=1.975, mean=3.562, max=6.575, sum=117.55 (33)\", \"tab\": \"General information\", \"score\": \"3.5621212121212116\"}", - "RAFT - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=99 (33)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "RAFT - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Toxic fraction": "{\"description\": \"11 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/ai21labs_jamba-v0.1.json b/data/models/ai21labs_jamba-v0.1.json deleted file mode 100644 index c57ee0bdae697816018984d67521ebc3d0295585..0000000000000000000000000000000000000000 --- a/data/models/ai21labs_jamba-v0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Jamba-v0.1", - "id": "ai21labs/Jamba-v0.1", - "developer": "ai21labs", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "JambaForCausalLM", - "params_billions": "51.57" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ai21labs_Jamba-v0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2026 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3602 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0159 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2685 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.359 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2492 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ai2_llama-2-chat-7b-nectar-3.8m.json.json b/data/models/ai2_llama-2-chat-7b-nectar-3.8m.json.json deleted file mode 100644 index 95264ec2aa0bb30e48a6bb9ba92a3193bbef46b5..0000000000000000000000000000000000000000 --- a/data/models/ai2_llama-2-chat-7b-nectar-3.8m.json.json +++ /dev/null @@ -1,108 +0,0 @@ -{ - "model_info": { - "name": "ai2/llama-2-chat-7b-nectar-3.8m.json", - "id": "ai2/llama-2-chat-7b-nectar-3.8m.json", - "developer": "AI2", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/ai2_llama-2-chat-7b-nectar-3.8m.json/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5843 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8631 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2654 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6243 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ai2_llama-2-chat-nectar-180k.json.json b/data/models/ai2_llama-2-chat-nectar-180k.json.json deleted file mode 100644 index ead776fb30d6efd2c1975f17bdbd41574a23d822..0000000000000000000000000000000000000000 --- a/data/models/ai2_llama-2-chat-nectar-180k.json.json +++ /dev/null @@ -1,108 +0,0 @@ -{ - "model_info": { - "name": "ai2/llama-2-chat-nectar-180k.json", - "id": "ai2/llama-2-chat-nectar-180k.json", - "developer": "AI2", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/ai2_llama-2-chat-nectar-180k.json/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5235 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8827 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2851 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4027 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ai2_llama-2-chat-ultrafeedback-60k.jsonl.json b/data/models/ai2_llama-2-chat-ultrafeedback-60k.jsonl.json deleted file mode 100644 index ac9a550996117f1cef2d49ae8a3ac09b12bee977..0000000000000000000000000000000000000000 --- a/data/models/ai2_llama-2-chat-ultrafeedback-60k.jsonl.json +++ /dev/null @@ -1,108 +0,0 @@ -{ - "model_info": { - "name": "ai2/llama-2-chat-ultrafeedback-60k.jsonl", - "id": "ai2/llama-2-chat-ultrafeedback-60k.jsonl", - "developer": "AI2", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/ai2_llama-2-chat-ultrafeedback-60k.jsonl/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.644 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9441 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4539 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5338 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ai2_tulu-2-7b-rm-v0-nectar-binarized-3.8m-check....json b/data/models/ai2_tulu-2-7b-rm-v0-nectar-binarized-3.8m-check....json deleted file mode 100644 index 252a7651a0709dfed65bf875d90378bc259999b1..0000000000000000000000000000000000000000 --- a/data/models/ai2_tulu-2-7b-rm-v0-nectar-binarized-3.8m-check....json +++ /dev/null @@ -1,876 +0,0 @@ -{ - "model_info": { - "name": "ai2/tulu-2-7b-rm-v0-nectar-binarized-3.8m-check...", - "id": "ai2/tulu-2-7b-rm-v0-nectar-binarized-3.8m-check...", - "developer": "AI2", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/ai2_tulu-2-7b-rm-v0-nectar-binarized-3.8m-check.../1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6895 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9385 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3706 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7595 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "reward-bench/ai2_tulu-2-7b-rm-v0-nectar-binarized-3.8m-check.../1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7058 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9525 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3947 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7703 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "reward-bench/ai2_tulu-2-7b-rm-v0-nectar-binarized-3.8m-check.../1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6808 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9302 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3596 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7527 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "reward-bench/ai2_tulu-2-7b-rm-v0-nectar-binarized-3.8m-check.../1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6905 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9441 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3596 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7676 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "reward-bench/ai2_tulu-2-7b-rm-v0-nectar-binarized-3.8m-check.../1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6945 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9385 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3706 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7743 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "reward-bench/ai2_tulu-2-7b-rm-v0-nectar-binarized-3.8m-check.../1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6924 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9441 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3575 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7757 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "reward-bench/ai2_tulu-2-7b-rm-v0-nectar-binarized-3.8m-check.../1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7004 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9413 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3882 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7716 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "reward-bench/ai2_tulu-2-7b-rm-v0-nectar-binarized-3.8m-check.../1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7019 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9497 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.375 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7811 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "reward-bench/ai2_tulu-2-7b-rm-v0-nectar-binarized-3.8m-check.../1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7008 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9385 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3882 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7757 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ai2_tulu-2-7b-rm-v0-nectar-binarized-700k.json.json b/data/models/ai2_tulu-2-7b-rm-v0-nectar-binarized-700k.json.json deleted file mode 100644 index c85c04740a4097faa74b4100fed5e2b290765b12..0000000000000000000000000000000000000000 --- a/data/models/ai2_tulu-2-7b-rm-v0-nectar-binarized-700k.json.json +++ /dev/null @@ -1,108 +0,0 @@ -{ - "model_info": { - "name": "ai2/tulu-2-7b-rm-v0-nectar-binarized-700k.json", - "id": "ai2/tulu-2-7b-rm-v0-nectar-binarized-700k.json", - "developer": "AI2", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/ai2_tulu-2-7b-rm-v0-nectar-binarized-700k.json/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7127 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9358 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4079 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7946 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ai2_tulu-2-7b-rm-v0-nectar-binarized.json.json b/data/models/ai2_tulu-2-7b-rm-v0-nectar-binarized.json.json deleted file mode 100644 index 145eee43129829ccb52ebda551e5ee20a39c80bc..0000000000000000000000000000000000000000 --- a/data/models/ai2_tulu-2-7b-rm-v0-nectar-binarized.json.json +++ /dev/null @@ -1,108 +0,0 @@ -{ - "model_info": { - "name": "ai2/tulu-2-7b-rm-v0-nectar-binarized.json", - "id": "ai2/tulu-2-7b-rm-v0-nectar-binarized.json", - "developer": "AI2", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/ai2_tulu-2-7b-rm-v0-nectar-binarized.json/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6756 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9134 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3904 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.723 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ai2_tulu-2-7b-rm-v0.json.json b/data/models/ai2_tulu-2-7b-rm-v0.json.json deleted file mode 100644 index f6016576845d8bb80bb0f46b68955083728e149c..0000000000000000000000000000000000000000 --- a/data/models/ai2_tulu-2-7b-rm-v0.json.json +++ /dev/null @@ -1,108 +0,0 @@ -{ - "model_info": { - "name": "ai2/tulu-2-7b-rm-v0.json", - "id": "ai2/tulu-2-7b-rm-v0.json", - "developer": "AI2", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/ai2_tulu-2-7b-rm-v0.json/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6655 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.933 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4539 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6095 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ai4bharat_airavata.json b/data/models/ai4bharat_airavata.json deleted file mode 100644 index 3172d09816fef301f0f0e798dfe7bef93632bb1f..0000000000000000000000000000000000000000 --- a/data/models/ai4bharat_airavata.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Airavata", - "id": "ai4bharat/Airavata", - "developer": "ai4bharat", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "6.87" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ai4bharat_Airavata/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0559 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3628 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0181 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2743 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3763 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1635 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ai4free_dhanishtha.json b/data/models/ai4free_dhanishtha.json deleted file mode 100644 index 0153cee0f30faf090eadde09cd1ea343f0a95e10..0000000000000000000000000000000000000000 --- a/data/models/ai4free_dhanishtha.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Dhanishtha", - "id": "AI4free/Dhanishtha", - "developer": "AI4free", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.777" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/AI4free_Dhanishtha/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2451 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3404 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.256 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2525 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3569 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1643 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ai4free_t2.json b/data/models/ai4free_t2.json deleted file mode 100644 index 124086ec39911e43425acb6cf71b000f17205f8b..0000000000000000000000000000000000000000 --- a/data/models/ai4free_t2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "t2", - "id": "AI4free/t2", - "developer": "AI4free", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.613" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/AI4free_t2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3867 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.291 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1896 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2576 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3846 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1144 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/aicoressecurity_cybernet-sec-3b-r1-v0-coder.json b/data/models/aicoressecurity_cybernet-sec-3b-r1-v0-coder.json deleted file mode 100644 index 6cdadaf2a7f4cc8bb9ef54979295aabe2ff97701..0000000000000000000000000000000000000000 --- a/data/models/aicoressecurity_cybernet-sec-3b-r1-v0-coder.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Cybernet-Sec-3B-R1-V0-Coder", - "id": "AicoresSecurity/Cybernet-Sec-3B-R1-V0-Coder", - "developer": "AicoresSecurity", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/AicoresSecurity_Cybernet-Sec-3B-R1-V0-Coder/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7098 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4478 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1488 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2718 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3408 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3178 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/aicoressecurity_cybernet-sec-3b-r1-v0.json b/data/models/aicoressecurity_cybernet-sec-3b-r1-v0.json deleted file mode 100644 index 7ffdf5ca67d80ae63568d01ca8f9ca070267f95b..0000000000000000000000000000000000000000 --- a/data/models/aicoressecurity_cybernet-sec-3b-r1-v0.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Cybernet-Sec-3B-R1-V0", - "id": "AicoresSecurity/Cybernet-Sec-3B-R1-V0", - "developer": "AicoresSecurity", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/AicoresSecurity_Cybernet-Sec-3B-R1-V0/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6358 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4497 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1156 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2634 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3314 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.301 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/aicoressecurity_cybernet-sec-3b-r1-v1.1.json b/data/models/aicoressecurity_cybernet-sec-3b-r1-v1.1.json deleted file mode 100644 index 5546f29949e11ca1a8207b9649da0b6cd755fb98..0000000000000000000000000000000000000000 --- a/data/models/aicoressecurity_cybernet-sec-3b-r1-v1.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Cybernet-Sec-3B-R1-V1.1", - "id": "AicoresSecurity/Cybernet-Sec-3B-R1-V1.1", - "developer": "AicoresSecurity", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/AicoresSecurity_Cybernet-Sec-3B-R1-V1.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.673 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4392 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.176 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.271 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3541 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3088 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/aicoressecurity_cybernet-sec-3b-r1-v1.json b/data/models/aicoressecurity_cybernet-sec-3b-r1-v1.json deleted file mode 100644 index 6f9c2b35b9eff56a9de3c480fe86405d5d38ceb5..0000000000000000000000000000000000000000 --- a/data/models/aicoressecurity_cybernet-sec-3b-r1-v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Cybernet-Sec-3B-R1-V1", - "id": "AicoresSecurity/Cybernet-Sec-3B-R1-V1", - "developer": "AicoresSecurity", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/AicoresSecurity_Cybernet-Sec-3B-R1-V1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6146 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4282 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1518 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2609 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3287 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2876 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/aidc-ai_marco-o1.json b/data/models/aidc-ai_marco-o1.json deleted file mode 100644 index dcb5d2cbc2ef1fe91c84f1548a55708169e13cea..0000000000000000000000000000000000000000 --- a/data/models/aidc-ai_marco-o1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Marco-o1", - "id": "AIDC-AI/Marco-o1", - "developer": "AIDC-AI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/AIDC-AI_Marco-o1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4771 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5364 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3746 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2592 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4138 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4117 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/aixonlab_aether-12b.json b/data/models/aixonlab_aether-12b.json deleted file mode 100644 index 1a37483866155995bffe8e9356d35f2fe788b7e6..0000000000000000000000000000000000000000 --- a/data/models/aixonlab_aether-12b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Aether-12b", - "id": "aixonlab/Aether-12b", - "developer": "aixonlab", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/aixonlab_Aether-12b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2347 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5179 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1065 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3163 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3829 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.341 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/aixonlab_grey-12b.json b/data/models/aixonlab_grey-12b.json deleted file mode 100644 index 2283419715fcb5149f93a77251556ed0c79b342f..0000000000000000000000000000000000000000 --- a/data/models/aixonlab_grey-12b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Grey-12b", - "id": "aixonlab/Grey-12b", - "developer": "aixonlab", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/aixonlab_Grey-12b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3968 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5699 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0982 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3003 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4516 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3779 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/aixonlab_zara-14b-v1.2.json b/data/models/aixonlab_zara-14b-v1.2.json deleted file mode 100644 index 87becb7628c40c7e0eadab7b873a20caca6acfd2..0000000000000000000000000000000000000000 --- a/data/models/aixonlab_zara-14b-v1.2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Zara-14b-v1.2", - "id": "aixonlab/Zara-14b-v1.2", - "developer": "aixonlab", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/aixonlab_Zara-14b-v1.2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6197 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6405 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3535 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3817 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4675 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5263 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/akhadangi_llama3.2.1b.0.01-first.json b/data/models/akhadangi_llama3.2.1b.0.01-first.json deleted file mode 100644 index dc0bb6fb0fa6739bb393294f31f10d88fafa8842..0000000000000000000000000000000000000000 --- a/data/models/akhadangi_llama3.2.1b.0.01-first.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama3.2.1B.0.01-First", - "id": "akhadangi/Llama3.2.1B.0.01-First", - "developer": "akhadangi", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.236" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/akhadangi_Llama3.2.1B.0.01-First/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0814 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3189 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0181 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2483 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3194 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1197 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/akhadangi_llama3.2.1b.0.01-last.json b/data/models/akhadangi_llama3.2.1b.0.01-last.json deleted file mode 100644 index 5ce96507d076234f41a8afeae18a8ff7c2f65c75..0000000000000000000000000000000000000000 --- a/data/models/akhadangi_llama3.2.1b.0.01-last.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama3.2.1B.0.01-Last", - "id": "akhadangi/Llama3.2.1B.0.01-Last", - "developer": "akhadangi", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.236" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/akhadangi_Llama3.2.1B.0.01-Last/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0917 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3159 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0136 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2433 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3206 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1227 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/akhadangi_llama3.2.1b.0.1-first.json b/data/models/akhadangi_llama3.2.1b.0.1-first.json deleted file mode 100644 index b41081cd1977cdefc805f7d092dfc2a1f84f7b1b..0000000000000000000000000000000000000000 --- a/data/models/akhadangi_llama3.2.1b.0.1-first.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama3.2.1B.0.1-First", - "id": "akhadangi/Llama3.2.1B.0.1-First", - "developer": "akhadangi", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.236" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/akhadangi_Llama3.2.1B.0.1-First/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1001 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.312 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0211 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.245 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3301 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1169 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/akhadangi_llama3.2.1b.0.1-last.json b/data/models/akhadangi_llama3.2.1b.0.1-last.json deleted file mode 100644 index 3f59b2f8d0fcea4b6748f8086e6b4bb4fb7bff63..0000000000000000000000000000000000000000 --- a/data/models/akhadangi_llama3.2.1b.0.1-last.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama3.2.1B.0.1-Last", - "id": "akhadangi/Llama3.2.1B.0.1-Last", - "developer": "akhadangi", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.236" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/akhadangi_Llama3.2.1B.0.1-Last/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.095 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3164 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0211 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2383 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3341 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1178 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/akhadangi_llama3.2.1b.basefit.json b/data/models/akhadangi_llama3.2.1b.basefit.json deleted file mode 100644 index d5fc970b2b8ebee0ac9f10cfca3ba19edccbc27e..0000000000000000000000000000000000000000 --- a/data/models/akhadangi_llama3.2.1b.basefit.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama3.2.1B.BaseFiT", - "id": "akhadangi/Llama3.2.1B.BaseFiT", - "developer": "akhadangi", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.236" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/akhadangi_Llama3.2.1B.BaseFiT/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0883 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3175 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0242 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2534 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3221 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1172 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/akjindal53244_llama-3.1-storm-8b.json b/data/models/akjindal53244_llama-3.1-storm-8b.json deleted file mode 100644 index 10080988b150f205c06f956e608326efa2dd3fb0..0000000000000000000000000000000000000000 --- a/data/models/akjindal53244_llama-3.1-storm-8b.json +++ /dev/null @@ -1,275 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.1-Storm-8B", - "id": "akjindal53244/Llama-3.1-Storm-8B", - "developer": "akjindal53244", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/akjindal53244_Llama-3.1-Storm-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8033 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5196 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1624 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3096 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4028 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3812 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "hfopenllm_v2/akjindal53244_Llama-3.1-Storm-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8051 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5189 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1722 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3263 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4028 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3803 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/alcholjung_llama3_medical_tuned.json b/data/models/alcholjung_llama3_medical_tuned.json deleted file mode 100644 index f26256ed8e9b472a568d3c5c6088d3dfeda8138f..0000000000000000000000000000000000000000 --- a/data/models/alcholjung_llama3_medical_tuned.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "llama3_medical_tuned", - "id": "alcholjung/llama3_medical_tuned", - "developer": "alcholjung", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "?", - "params_billions": "16.061" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/alcholjung_llama3_medical_tuned/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0106 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4513 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0468 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2861 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.466 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2946 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/alepach_nothumpback-m0.json b/data/models/alepach_nothumpback-m0.json deleted file mode 100644 index d3424dd0f866c798430de50a7b17a722c95896e1..0000000000000000000000000000000000000000 --- a/data/models/alepach_nothumpback-m0.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "notHumpback-M0", - "id": "Alepach/notHumpback-M0", - "developer": "Alepach", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Alepach_notHumpback-M0/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.235 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2785 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0189 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2492 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3552 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1119 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/alepach_nothumpback-m1-v2.json b/data/models/alepach_nothumpback-m1-v2.json deleted file mode 100644 index 9a7b23675824a9f5d7ea3c95f2e7cf8e7c0367bd..0000000000000000000000000000000000000000 --- a/data/models/alepach_nothumpback-m1-v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "notHumpback-M1-v2", - "id": "Alepach/notHumpback-M1-v2", - "developer": "Alepach", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Alepach_notHumpback-M1-v2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2277 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2776 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0219 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2601 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3473 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1119 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/alepach_nothumpback-m1.json b/data/models/alepach_nothumpback-m1.json deleted file mode 100644 index 390431eca3ff69be97b4cb8b6f6770f3de44277a..0000000000000000000000000000000000000000 --- a/data/models/alepach_nothumpback-m1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "notHumpback-M1", - "id": "Alepach/notHumpback-M1", - "developer": "Alepach", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Alepach_notHumpback-M1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2207 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2882 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0159 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2374 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.342 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1091 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/aleph-alpha_luminous-base-13b.json b/data/models/aleph-alpha_luminous-base-13b.json deleted file mode 100644 index 2ef9f61808b504578f49c64128371bbe90f34b39..0000000000000000000000000000000000000000 --- a/data/models/aleph-alpha_luminous-base-13b.json +++ /dev/null @@ -1,674 +0,0 @@ -{ - "model_info": { - "name": "Luminous Base 13B", - "id": "aleph-alpha/Luminous-Base-13B", - "developer": "aleph-alpha", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_classic/aleph-alpha_Luminous-Base-13B/1774096308.339228", - "retrieved_timestamp": "1774096308.339228", - "source_metadata": { - "source_name": "helm_classic", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_classic", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_classic", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperform on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.315, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Calibration": "{\"description\": \"\", \"tab\": \"Calibration\", \"score\": \"0.6405642923219241\"}", - "Mean win rate - Robustness": "{\"description\": \"\", \"tab\": \"Robustness\", \"score\": \"0.31855477855477854\"}", - "Mean win rate - Fairness": "{\"description\": \"\", \"tab\": \"Fairness\", \"score\": \"0.23762237762237765\"}", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}", - "Mean win rate - Bias": "{\"description\": \"\", \"tab\": \"Bias\", \"score\": \"0.5516493320513314\"}", - "Mean win rate - Toxicity": "{\"description\": \"\", \"tab\": \"Toxicity\", \"score\": \"0.5035063701730368\"}", - "Mean win rate - Summarization metrics": "{\"description\": \"\", \"tab\": \"Summarization metrics\", \"score\": \"0.42105263157894735\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.27, - "details": { - "description": "min=0.193, mean=0.27, max=0.32, sum=4.045 (15)", - "tab": "Accuracy", - "MMLU - ECE (10-bin)": "{\"description\": \"min=0.087, mean=0.111, max=0.157, sum=1.661 (15)\", \"tab\": \"Calibration\", \"score\": \"0.110752611571227\"}", - "MMLU - EM (Robustness)": "{\"description\": \"min=0.1, mean=0.183, max=0.27, sum=2.74 (15)\", \"tab\": \"Robustness\", \"score\": \"0.1826549707602339\"}", - "MMLU - EM (Fairness)": "{\"description\": \"min=0.09, mean=0.185, max=0.27, sum=2.769 (15)\", \"tab\": \"Fairness\", \"score\": \"0.1845730994152047\"}", - "MMLU - Denoised inference time (s)": "{\"description\": \"5 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=1542 (15)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=75 (15)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (15)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=360.75, mean=471.075, max=618.447, sum=7066.132 (15)\", \"tab\": \"General information\", \"score\": \"471.0754736842105\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=15 (15)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "MMLU - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=45 (15)\", \"tab\": \"General information\", \"score\": \"3.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "BoolQ", - "source_data": { - "dataset_name": "BoolQ", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on BoolQ", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.719, - "details": { - "description": "min=0.7, mean=0.719, max=0.74, sum=2.156 (3)", - "tab": "Accuracy", - "BoolQ - ECE (10-bin)": "{\"description\": \"min=0.056, mean=0.066, max=0.084, sum=0.197 (3)\", \"tab\": \"Calibration\", \"score\": \"0.06557915095556173\"}", - "BoolQ - EM (Robustness)": "{\"description\": \"min=0.643, mean=0.655, max=0.673, sum=1.965 (3)\", \"tab\": \"Robustness\", \"score\": \"0.655\"}", - "BoolQ - EM (Fairness)": "{\"description\": \"min=0.634, mean=0.653, max=0.682, sum=1.958 (3)\", \"tab\": \"Fairness\", \"score\": \"0.6526666666666667\"}", - "BoolQ - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "BoolQ - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "BoolQ - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "BoolQ - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "BoolQ - # prompt tokens": "{\"description\": \"min=651.658, mean=908.991, max=1252.658, sum=2726.974 (3)\", \"tab\": \"General information\", \"score\": \"908.9913333333333\"}", - "BoolQ - # output tokens": "{\"description\": \"min=1, mean=1.002, max=1.003, sum=3.006 (3)\", \"tab\": \"General information\", \"score\": \"1.002\"}", - "BoolQ - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "BoolQ - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.605, - "details": { - "description": "min=0.577, mean=0.605, max=0.633, sum=1.815 (3)", - "tab": "Accuracy", - "NarrativeQA - ECE (10-bin)": "{\"description\": \"min=0.04, mean=0.048, max=0.063, sum=0.145 (3)\", \"tab\": \"Calibration\", \"score\": \"0.04822831549746422\"}", - "NarrativeQA - F1 (Robustness)": "{\"description\": \"min=0.444, mean=0.476, max=0.505, sum=1.429 (3)\", \"tab\": \"Robustness\", \"score\": \"0.4761726989393548\"}", - "NarrativeQA - F1 (Fairness)": "{\"description\": \"min=0.462, mean=0.498, max=0.532, sum=1.495 (3)\", \"tab\": \"Fairness\", \"score\": \"0.4982467496641079\"}", - "NarrativeQA - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=1065 (3)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=1.039, mean=1.621, max=2.037, sum=4.862 (3)\", \"tab\": \"General information\", \"score\": \"1.6206572769953052\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=1606.952, mean=1647.783, max=1694.642, sum=4943.349 (3)\", \"tab\": \"General information\", \"score\": \"1647.783098591549\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=5.521, mean=6.798, max=8.192, sum=20.394 (3)\", \"tab\": \"General information\", \"score\": \"6.798122065727699\"}", - "NarrativeQA - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NarrativeQA - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NarrativeQA - Stereotypes (gender)": "{\"description\": \"min=0.396, mean=0.438, max=0.5, sum=1.313 (3)\", \"tab\": \"Bias\", \"score\": \"0.4375901875901876\"}", - "NarrativeQA - Representation (race)": "{\"description\": \"min=0.333, mean=0.556, max=0.667, sum=1.667 (3)\", \"tab\": \"Bias\", \"score\": \"0.5555555555555557\"}", - "NarrativeQA - Representation (gender)": "{\"description\": \"min=0.152, mean=0.172, max=0.197, sum=0.516 (3)\", \"tab\": \"Bias\", \"score\": \"0.1718450326045263\"}", - "NarrativeQA - Toxic fraction": "{\"description\": \"min=0.02, mean=0.022, max=0.025, sum=0.065 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.0215962441314554\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (open-book)", - "source_data": { - "dataset_name": "NaturalQuestions (open-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (open-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.568, - "details": { - "description": "min=0.563, mean=0.568, max=0.577, sum=1.705 (3)", - "tab": "Accuracy", - "NaturalQuestions (closed-book) - ECE (10-bin)": "{\"description\": \"min=0.039, mean=0.045, max=0.054, sum=0.136 (3)\", \"tab\": \"Calibration\", \"score\": \"0.04534548194935659\"}", - "NaturalQuestions (open-book) - ECE (10-bin)": "{\"description\": \"min=0.068, mean=0.07, max=0.074, sum=0.21 (3)\", \"tab\": \"Calibration\", \"score\": \"0.07013609628734997\"}", - "NaturalQuestions (closed-book) - F1 (Robustness)": "{\"description\": \"min=0.157, mean=0.163, max=0.168, sum=0.489 (3)\", \"tab\": \"Robustness\", \"score\": \"0.1628593597054443\"}", - "NaturalQuestions (open-book) - F1 (Robustness)": "{\"description\": \"min=0.484, mean=0.491, max=0.498, sum=1.474 (3)\", \"tab\": \"Robustness\", \"score\": \"0.4912891920785376\"}", - "NaturalQuestions (closed-book) - F1 (Fairness)": "{\"description\": \"min=0.156, mean=0.16, max=0.164, sum=0.481 (3)\", \"tab\": \"Fairness\", \"score\": \"0.16022586408623682\"}", - "NaturalQuestions (open-book) - F1 (Fairness)": "{\"description\": \"min=0.505, mean=0.511, max=0.515, sum=1.534 (3)\", \"tab\": \"Fairness\", \"score\": \"0.5114691771549933\"}", - "NaturalQuestions (closed-book) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NaturalQuestions (open-book) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=109.087, mean=111.754, max=116.087, sum=335.261 (3)\", \"tab\": \"General information\", \"score\": \"111.75366666666667\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=4.314, mean=5.287, max=5.908, sum=15.861 (3)\", \"tab\": \"General information\", \"score\": \"5.287\"}", - "NaturalQuestions (closed-book) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.691, mean=4.711, max=4.726, sum=14.134 (3)\", \"tab\": \"General information\", \"score\": \"4.711333333333333\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.038, mean=0.039, max=0.04, sum=0.116 (3)\", \"tab\": \"General information\", \"score\": \"0.03866666666666666\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1224.733, mean=1384.565, max=1488.14, sum=4153.695 (3)\", \"tab\": \"General information\", \"score\": \"1384.5649999999998\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=7.685, mean=10.15, max=11.898, sum=30.449 (3)\", \"tab\": \"General information\", \"score\": \"10.149666666666667\"}", - "NaturalQuestions (open-book) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NaturalQuestions (closed-book) - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - Stereotypes (gender)": "{\"description\": \"min=0.25, mean=0.417, max=0.5, sum=1.25 (3)\", \"tab\": \"Bias\", \"score\": \"0.4166666666666667\"}", - "NaturalQuestions (closed-book) - Representation (race)": "{\"description\": \"min=0.339, mean=0.433, max=0.5, sum=1.298 (3)\", \"tab\": \"Bias\", \"score\": \"0.43278417840114286\"}", - "NaturalQuestions (closed-book) - Representation (gender)": "{\"description\": \"min=0.081, mean=0.162, max=0.239, sum=0.486 (3)\", \"tab\": \"Bias\", \"score\": \"0.16214742091319934\"}", - "NaturalQuestions (open-book) - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=0.667 (1)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666667\"}", - "NaturalQuestions (open-book) - Stereotypes (gender)": "{\"description\": \"min=0.3, mean=0.432, max=0.5, sum=1.296 (3)\", \"tab\": \"Bias\", \"score\": \"0.432010582010582\"}", - "NaturalQuestions (open-book) - Representation (race)": "{\"description\": \"min=0.429, mean=0.457, max=0.498, sum=1.37 (3)\", \"tab\": \"Bias\", \"score\": \"0.45656911106888937\"}", - "NaturalQuestions (open-book) - Representation (gender)": "{\"description\": \"min=0.272, mean=0.32, max=0.416, sum=0.961 (3)\", \"tab\": \"Bias\", \"score\": \"0.3202891068062547\"}", - "NaturalQuestions (closed-book) - Toxic fraction": "{\"description\": \"min=0.002, mean=0.002, max=0.002, sum=0.006 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.002\"}", - "NaturalQuestions (open-book) - Toxic fraction": "{\"description\": \"min=0, mean=0.001, max=0.003, sum=0.004 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.0013333333333333333\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "QuAC", - "source_data": { - "dataset_name": "QuAC", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on QuAC", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.334, - "details": { - "description": "min=0.317, mean=0.334, max=0.362, sum=1.003 (3)", - "tab": "Accuracy", - "QuAC - ECE (10-bin)": "{\"description\": \"min=0.068, mean=0.098, max=0.131, sum=0.295 (3)\", \"tab\": \"Calibration\", \"score\": \"0.09821008405024316\"}", - "QuAC - F1 (Robustness)": "{\"description\": \"min=0.166, mean=0.185, max=0.212, sum=0.556 (3)\", \"tab\": \"Robustness\", \"score\": \"0.18543862521458307\"}", - "QuAC - F1 (Fairness)": "{\"description\": \"min=0.251, mean=0.266, max=0.284, sum=0.799 (3)\", \"tab\": \"Fairness\", \"score\": \"0.2662906470176498\"}", - "QuAC - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "QuAC - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "QuAC - # train": "{\"description\": \"min=0.84, mean=0.909, max=0.991, sum=2.727 (3)\", \"tab\": \"General information\", \"score\": \"0.9089999999999999\"}", - "QuAC - truncated": "{\"description\": \"min=0.029, mean=0.033, max=0.037, sum=0.098 (3)\", \"tab\": \"General information\", \"score\": \"0.03266666666666667\"}", - "QuAC - # prompt tokens": "{\"description\": \"min=1596.904, mean=1641.256, max=1672.92, sum=4923.768 (3)\", \"tab\": \"General information\", \"score\": \"1641.256\"}", - "QuAC - # output tokens": "{\"description\": \"min=18.527, mean=23.472, max=28.795, sum=70.415 (3)\", \"tab\": \"General information\", \"score\": \"23.471666666666668\"}", - "QuAC - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "QuAC - Stereotypes (race)": "{\"description\": \"min=0.641, mean=0.658, max=0.667, sum=1.974 (3)\", \"tab\": \"Bias\", \"score\": \"0.6581196581196581\"}", - "QuAC - Stereotypes (gender)": "{\"description\": \"min=0.401, mean=0.417, max=0.432, sum=1.251 (3)\", \"tab\": \"Bias\", \"score\": \"0.41695983406755\"}", - "QuAC - Representation (race)": "{\"description\": \"min=0.258, mean=0.32, max=0.377, sum=0.96 (3)\", \"tab\": \"Bias\", \"score\": \"0.3200297021845843\"}", - "QuAC - Representation (gender)": "{\"description\": \"min=0.193, mean=0.203, max=0.212, sum=0.61 (3)\", \"tab\": \"Bias\", \"score\": \"0.20338227449992274\"}", - "QuAC - Toxic fraction": "{\"description\": \"min=0.001, mean=0.002, max=0.003, sum=0.006 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.002\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "HellaSwag", - "source_data": { - "dataset_name": "HellaSwag", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on HellaSwag", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "HellaSwag - ECE (10-bin)": "{\"description\": \"No matching runs\", \"tab\": \"Calibration\", \"score\": \"\"}", - "HellaSwag - EM (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "HellaSwag - EM (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "HellaSwag - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "HellaSwag - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "OpenbookQA - ECE (10-bin)": "{\"description\": \"No matching runs\", \"tab\": \"Calibration\", \"score\": \"\"}", - "OpenbookQA - EM (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "OpenbookQA - EM (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "OpenbookQA - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "OpenbookQA - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "TruthfulQA", - "source_data": { - "dataset_name": "TruthfulQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on TruthfulQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.182, - "details": { - "description": "min=0.165, mean=0.182, max=0.194, sum=0.547 (3)", - "tab": "Accuracy", - "TruthfulQA - ECE (10-bin)": "{\"description\": \"min=0.069, mean=0.081, max=0.095, sum=0.244 (3)\", \"tab\": \"Calibration\", \"score\": \"0.08144933240589737\"}", - "TruthfulQA - EM (Robustness)": "{\"description\": \"min=0.107, mean=0.112, max=0.118, sum=0.335 (3)\", \"tab\": \"Robustness\", \"score\": \"0.11162079510703364\"}", - "TruthfulQA - EM (Fairness)": "{\"description\": \"min=0.118, mean=0.125, max=0.13, sum=0.375 (3)\", \"tab\": \"Fairness\", \"score\": \"0.12487257900101938\"}", - "TruthfulQA - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "TruthfulQA - # eval": "{\"description\": \"min=654, mean=654, max=654, sum=1962 (3)\", \"tab\": \"General information\", \"score\": \"654.0\"}", - "TruthfulQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "TruthfulQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "TruthfulQA - # prompt tokens": "{\"description\": \"min=504.073, mean=514.073, max=533.073, sum=1542.22 (3)\", \"tab\": \"General information\", \"score\": \"514.0733944954128\"}", - "TruthfulQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=3 (3)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "TruthfulQA - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MS MARCO (TREC)", - "source_data": { - "dataset_name": "MS MARCO (TREC)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "NDCG@10 on MS MARCO (TREC)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "MS MARCO (regular) - RR@10 (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "MS MARCO (TREC) - NDCG@10 (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "MS MARCO (regular) - RR@10 (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "MS MARCO (TREC) - NDCG@10 (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "MS MARCO (regular) - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MS MARCO (TREC) - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MS MARCO (regular) - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "MS MARCO (TREC) - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CNN/DailyMail", - "source_data": { - "dataset_name": "CNN/DailyMail", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on CNN/DailyMail", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.11, - "details": { - "description": "min=0.048, mean=0.11, max=0.147, sum=0.661 (6)", - "tab": "Accuracy", - "CNN/DailyMail - Denoised inference time (s)": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "CNN/DailyMail - # eval": "{\"description\": \"min=466, mean=466, max=466, sum=2796 (6)\", \"tab\": \"General information\", \"score\": \"466.0\"}", - "CNN/DailyMail - # train": "{\"description\": \"min=5, mean=5, max=5, sum=30 (6)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CNN/DailyMail - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (6)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CNN/DailyMail - # prompt tokens": "{\"description\": \"min=1564.648, mean=1578.648, max=1593.648, sum=9471.888 (6)\", \"tab\": \"General information\", \"score\": \"1578.648068669528\"}", - "CNN/DailyMail - # output tokens": "{\"description\": \"min=59.824, mean=80.866, max=92.721, sum=485.197 (6)\", \"tab\": \"General information\", \"score\": \"80.86623748211731\"}", - "CNN/DailyMail - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=18 (6)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "CNN/DailyMail - Stereotypes (race)": "{\"description\": \"min=0.607, mean=0.629, max=0.667, sum=3.775 (6)\", \"tab\": \"Bias\", \"score\": \"0.629159058053613\"}", - "CNN/DailyMail - Stereotypes (gender)": "{\"description\": \"min=0.388, mean=0.408, max=0.443, sum=2.45 (6)\", \"tab\": \"Bias\", \"score\": \"0.40834546858679427\"}", - "CNN/DailyMail - Representation (race)": "{\"description\": \"min=0.211, mean=0.287, max=0.333, sum=1.725 (6)\", \"tab\": \"Bias\", \"score\": \"0.2874529064836184\"}", - "CNN/DailyMail - Representation (gender)": "{\"description\": \"min=0.138, mean=0.164, max=0.192, sum=0.984 (6)\", \"tab\": \"Bias\", \"score\": \"0.16396073067980207\"}", - "CNN/DailyMail - Toxic fraction": "{\"description\": \"min=0, mean=0.001, max=0.002, sum=0.004 (6)\", \"tab\": \"Toxicity\", \"score\": \"0.000715307582260372\"}", - "CNN/DailyMail - SummaC": "{\"description\": \"min=-0.076, mean=0.32, max=0.527, sum=0.959 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.3197354449182434\"}", - "CNN/DailyMail - QAFactEval": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - BERTScore (F1)": "{\"description\": \"min=0.045, mean=0.188, max=0.278, sum=0.563 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.18776450739321585\"}", - "CNN/DailyMail - Coverage": "{\"description\": \"min=0.543, mean=0.834, max=0.982, sum=5.004 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"0.8340516341645151\"}", - "CNN/DailyMail - Density": "{\"description\": \"min=15.163, mean=35.663, max=51.192, sum=213.977 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"35.66281771790173\"}", - "CNN/DailyMail - Compression": "{\"description\": \"min=8.191, mean=9.346, max=11.345, sum=56.078 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"9.346357628862261\"}", - "CNN/DailyMail - HumanEval-faithfulness": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-relevance": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-coherence": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "XSUM", - "source_data": { - "dataset_name": "XSUM", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on XSUM", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.105, - "details": { - "description": "min=0.101, mean=0.105, max=0.107, sum=0.628 (6)", - "tab": "Accuracy", - "XSUM - Denoised inference time (s)": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "XSUM - # eval": "{\"description\": \"min=518, mean=518, max=518, sum=3108 (6)\", \"tab\": \"General information\", \"score\": \"518.0\"}", - "XSUM - # train": "{\"description\": \"min=4.998, mean=4.999, max=5, sum=29.992 (6)\", \"tab\": \"General information\", \"score\": \"4.998712998712999\"}", - "XSUM - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (6)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "XSUM - # prompt tokens": "{\"description\": \"min=1472.903, mean=1532.912, max=1566.407, sum=9197.471 (6)\", \"tab\": \"General information\", \"score\": \"1532.9118404118406\"}", - "XSUM - # output tokens": "{\"description\": \"min=25.481, mean=26.021, max=26.315, sum=156.127 (6)\", \"tab\": \"General information\", \"score\": \"26.02123552123552\"}", - "XSUM - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=18 (6)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "XSUM - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=4 (6)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "XSUM - Stereotypes (gender)": "{\"description\": \"min=0.429, mean=0.442, max=0.453, sum=2.655 (6)\", \"tab\": \"Bias\", \"score\": \"0.4424845269672855\"}", - "XSUM - Representation (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=4 (6)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "XSUM - Representation (gender)": "{\"description\": \"min=0.153, mean=0.165, max=0.183, sum=0.99 (6)\", \"tab\": \"Bias\", \"score\": \"0.16492426719539477\"}", - "XSUM - Toxic fraction": "{\"description\": \"min=0.002, mean=0.002, max=0.002, sum=0.012 (6)\", \"tab\": \"Toxicity\", \"score\": \"0.0019305019305019308\"}", - "XSUM - SummaC": "{\"description\": \"min=-0.217, mean=-0.213, max=-0.206, sum=-0.639 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"-0.2129847266550281\"}", - "XSUM - QAFactEval": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - BERTScore (F1)": "{\"description\": \"min=0.391, mean=0.394, max=0.396, sum=1.183 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.3944890669761573\"}", - "XSUM - Coverage": "{\"description\": \"min=0.828, mean=0.834, max=0.838, sum=5.002 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"0.8336902125268334\"}", - "XSUM - Density": "{\"description\": \"min=4.128, mean=4.393, max=4.529, sum=26.358 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"4.392991783737345\"}", - "XSUM - Compression": "{\"description\": \"min=17.248, mean=17.535, max=17.956, sum=105.21 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"17.535051923934834\"}", - "XSUM - HumanEval-faithfulness": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-relevance": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-coherence": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "IMDB", - "source_data": { - "dataset_name": "IMDB", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on IMDB", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.939, - "details": { - "description": "min=0.931, mean=0.939, max=0.949, sum=2.818 (3)", - "tab": "Accuracy", - "IMDB - ECE (10-bin)": "{\"description\": \"min=0.187, mean=0.232, max=0.257, sum=0.695 (3)\", \"tab\": \"Calibration\", \"score\": \"0.23165086222498446\"}", - "IMDB - EM (Robustness)": "{\"description\": \"min=0.864, mean=0.887, max=0.918, sum=2.662 (3)\", \"tab\": \"Robustness\", \"score\": \"0.8873333333333333\"}", - "IMDB - EM (Fairness)": "{\"description\": \"min=0.902, mean=0.912, max=0.926, sum=2.737 (3)\", \"tab\": \"Fairness\", \"score\": \"0.9123333333333333\"}", - "IMDB - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "IMDB - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "IMDB - # train": "{\"description\": \"min=2.908, mean=4.236, max=4.985, sum=12.708 (3)\", \"tab\": \"General information\", \"score\": \"4.236000000000001\"}", - "IMDB - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IMDB - # prompt tokens": "{\"description\": \"min=1283.569, mean=1560.056, max=1777.712, sum=4680.167 (3)\", \"tab\": \"General information\", \"score\": \"1560.0556666666664\"}", - "IMDB - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=3 (3)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "IMDB - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "IMDB - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CivilComments", - "source_data": { - "dataset_name": "CivilComments", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on CivilComments", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.544, - "details": { - "description": "min=0.003, mean=0.544, max=1, sum=29.372 (54)", - "tab": "Accuracy", - "CivilComments - ECE (10-bin)": "{\"description\": \"min=0.071, mean=0.28, max=0.632, sum=15.102 (54)\", \"tab\": \"Calibration\", \"score\": \"0.2796625331945748\"}", - "CivilComments - EM (Robustness)": "{\"description\": \"min=0, mean=0.416, max=0.99, sum=22.479 (54)\", \"tab\": \"Robustness\", \"score\": \"0.416268791059841\"}", - "CivilComments - EM (Fairness)": "{\"description\": \"min=0, mean=0.397, max=1, sum=21.425 (54)\", \"tab\": \"Fairness\", \"score\": \"0.3967651888403395\"}", - "CivilComments - Denoised inference time (s)": "{\"description\": \"9 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "CivilComments - # eval": "{\"description\": \"min=74, mean=371.556, max=683, sum=20064 (54)\", \"tab\": \"General information\", \"score\": \"371.55555555555554\"}", - "CivilComments - # train": "{\"description\": \"min=5, mean=5, max=5, sum=270 (54)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CivilComments - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (54)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CivilComments - # prompt tokens": "{\"description\": \"min=362.037, mean=724.782, max=1272.822, sum=39138.207 (54)\", \"tab\": \"General information\", \"score\": \"724.7816027688522\"}", - "CivilComments - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=54 (54)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "CivilComments - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=162 (54)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "CivilComments - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (54)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "RAFT", - "source_data": { - "dataset_name": "RAFT", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on RAFT", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.473, - "details": { - "description": "min=0.025, mean=0.473, max=0.975, sum=15.625 (33)", - "tab": "Accuracy", - "RAFT - ECE (10-bin)": "{\"description\": \"min=0.115, mean=0.29, max=0.826, sum=9.575 (33)\", \"tab\": \"Calibration\", \"score\": \"0.29014727083072167\"}", - "RAFT - EM (Robustness)": "{\"description\": \"min=0, mean=0.402, max=0.975, sum=13.25 (33)\", \"tab\": \"Robustness\", \"score\": \"0.4015151515151515\"}", - "RAFT - EM (Fairness)": "{\"description\": \"min=0, mean=0.445, max=0.975, sum=14.7 (33)\", \"tab\": \"Fairness\", \"score\": \"0.4454545454545455\"}", - "RAFT - Denoised inference time (s)": "{\"description\": \"11 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "RAFT - # eval": "{\"description\": \"min=40, mean=40, max=40, sum=1320 (33)\", \"tab\": \"General information\", \"score\": \"40.0\"}", - "RAFT - # train": "{\"description\": \"min=0, mean=4.56, max=5, sum=150.475 (33)\", \"tab\": \"General information\", \"score\": \"4.5598484848484855\"}", - "RAFT - truncated": "{\"description\": \"min=0, mean=0.002, max=0.025, sum=0.075 (33)\", \"tab\": \"General information\", \"score\": \"0.002272727272727273\"}", - "RAFT - # prompt tokens": "{\"description\": \"min=262.3, mean=810.769, max=1759.65, sum=26755.375 (33)\", \"tab\": \"General information\", \"score\": \"810.7689393939394\"}", - "RAFT - # output tokens": "{\"description\": \"min=0.75, mean=2.916, max=6.5, sum=96.225 (33)\", \"tab\": \"General information\", \"score\": \"2.91590909090909\"}", - "RAFT - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=99 (33)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "RAFT - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (33)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/aleph-alpha_luminous-extended-30b.json b/data/models/aleph-alpha_luminous-extended-30b.json deleted file mode 100644 index 6d49e216ff72da31d67c55aa58a005c1e9b7514b..0000000000000000000000000000000000000000 --- a/data/models/aleph-alpha_luminous-extended-30b.json +++ /dev/null @@ -1,674 +0,0 @@ -{ - "model_info": { - "name": "Luminous Extended 30B", - "id": "aleph-alpha/Luminous-Extended-30B", - "developer": "aleph-alpha", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_classic/aleph-alpha_Luminous-Extended-30B/1774096308.339228", - "retrieved_timestamp": "1774096308.339228", - "source_metadata": { - "source_name": "helm_classic", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_classic", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_classic", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperform on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.485, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Calibration": "{\"description\": \"\", \"tab\": \"Calibration\", \"score\": \"0.5765957446808511\"}", - "Mean win rate - Robustness": "{\"description\": \"\", \"tab\": \"Robustness\", \"score\": \"0.42993006993006994\"}", - "Mean win rate - Fairness": "{\"description\": \"\", \"tab\": \"Fairness\", \"score\": \"0.45142191142191146\"}", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}", - "Mean win rate - Bias": "{\"description\": \"\", \"tab\": \"Bias\", \"score\": \"0.629471974916769\"}", - "Mean win rate - Toxicity": "{\"description\": \"\", \"tab\": \"Toxicity\", \"score\": \"0.7191265524598858\"}", - "Mean win rate - Summarization metrics": "{\"description\": \"\", \"tab\": \"Summarization metrics\", \"score\": \"0.5657894736842105\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.321, - "details": { - "description": "min=0.23, mean=0.321, max=0.49, sum=4.811 (15)", - "tab": "Accuracy", - "MMLU - ECE (10-bin)": "{\"description\": \"min=0.075, mean=0.135, max=0.225, sum=2.023 (15)\", \"tab\": \"Calibration\", \"score\": \"0.1348564339845485\"}", - "MMLU - EM (Robustness)": "{\"description\": \"min=0.1, mean=0.23, max=0.37, sum=3.451 (15)\", \"tab\": \"Robustness\", \"score\": \"0.23008187134502922\"}", - "MMLU - EM (Fairness)": "{\"description\": \"min=0.14, mean=0.237, max=0.35, sum=3.549 (15)\", \"tab\": \"Fairness\", \"score\": \"0.23658479532163745\"}", - "MMLU - Denoised inference time (s)": "{\"description\": \"5 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=1542 (15)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=75 (15)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (15)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=360.75, mean=471.075, max=618.447, sum=7066.132 (15)\", \"tab\": \"General information\", \"score\": \"471.0754736842105\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=15 (15)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "MMLU - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=45 (15)\", \"tab\": \"General information\", \"score\": \"3.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "BoolQ", - "source_data": { - "dataset_name": "BoolQ", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on BoolQ", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.767, - "details": { - "description": "min=0.752, mean=0.767, max=0.794, sum=2.3 (3)", - "tab": "Accuracy", - "BoolQ - ECE (10-bin)": "{\"description\": \"min=0.11, mean=0.129, max=0.154, sum=0.387 (3)\", \"tab\": \"Calibration\", \"score\": \"0.1289354797828563\"}", - "BoolQ - EM (Robustness)": "{\"description\": \"min=0.637, mean=0.659, max=0.7, sum=1.976 (3)\", \"tab\": \"Robustness\", \"score\": \"0.6586666666666666\"}", - "BoolQ - EM (Fairness)": "{\"description\": \"min=0.692, mean=0.711, max=0.733, sum=2.133 (3)\", \"tab\": \"Fairness\", \"score\": \"0.711\"}", - "BoolQ - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "BoolQ - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "BoolQ - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "BoolQ - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "BoolQ - # prompt tokens": "{\"description\": \"min=651.658, mean=908.991, max=1252.658, sum=2726.974 (3)\", \"tab\": \"General information\", \"score\": \"908.9913333333333\"}", - "BoolQ - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=3 (3)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "BoolQ - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "BoolQ - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.665, - "details": { - "description": "min=0.637, mean=0.665, max=0.684, sum=1.994 (3)", - "tab": "Accuracy", - "NarrativeQA - ECE (10-bin)": "{\"description\": \"min=0.043, mean=0.046, max=0.047, sum=0.138 (3)\", \"tab\": \"Calibration\", \"score\": \"0.046063826868188405\"}", - "NarrativeQA - F1 (Robustness)": "{\"description\": \"min=0.481, mean=0.513, max=0.539, sum=1.54 (3)\", \"tab\": \"Robustness\", \"score\": \"0.513450295883327\"}", - "NarrativeQA - F1 (Fairness)": "{\"description\": \"min=0.503, mean=0.532, max=0.565, sum=1.597 (3)\", \"tab\": \"Fairness\", \"score\": \"0.5321907426131639\"}", - "NarrativeQA - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=1065 (3)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=1.039, mean=1.621, max=2.037, sum=4.862 (3)\", \"tab\": \"General information\", \"score\": \"1.6206572769953052\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=1606.952, mean=1647.783, max=1694.642, sum=4943.349 (3)\", \"tab\": \"General information\", \"score\": \"1647.783098591549\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=6.321, mean=7.042, max=8.175, sum=21.127 (3)\", \"tab\": \"General information\", \"score\": \"7.04225352112676\"}", - "NarrativeQA - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NarrativeQA - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NarrativeQA - Stereotypes (gender)": "{\"description\": \"min=0.4, mean=0.416, max=0.44, sum=1.248 (3)\", \"tab\": \"Bias\", \"score\": \"0.4159611992945326\"}", - "NarrativeQA - Representation (race)": "{\"description\": \"min=0.333, mean=0.556, max=0.667, sum=1.667 (3)\", \"tab\": \"Bias\", \"score\": \"0.5555555555555557\"}", - "NarrativeQA - Representation (gender)": "{\"description\": \"min=0.186, mean=0.199, max=0.207, sum=0.598 (3)\", \"tab\": \"Bias\", \"score\": \"0.19931611685099856\"}", - "NarrativeQA - Toxic fraction": "{\"description\": \"min=0.014, mean=0.017, max=0.02, sum=0.051 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.016901408450704227\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (open-book)", - "source_data": { - "dataset_name": "NaturalQuestions (open-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (open-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.609, - "details": { - "description": "min=0.606, mean=0.609, max=0.611, sum=1.827 (3)", - "tab": "Accuracy", - "NaturalQuestions (closed-book) - ECE (10-bin)": "{\"description\": \"min=0.018, mean=0.022, max=0.024, sum=0.065 (3)\", \"tab\": \"Calibration\", \"score\": \"0.02157162838647707\"}", - "NaturalQuestions (open-book) - ECE (10-bin)": "{\"description\": \"min=0.08, mean=0.09, max=0.095, sum=0.269 (3)\", \"tab\": \"Calibration\", \"score\": \"0.08979897901208977\"}", - "NaturalQuestions (closed-book) - F1 (Robustness)": "{\"description\": \"min=0.205, mean=0.212, max=0.218, sum=0.635 (3)\", \"tab\": \"Robustness\", \"score\": \"0.211552896733343\"}", - "NaturalQuestions (open-book) - F1 (Robustness)": "{\"description\": \"min=0.515, mean=0.524, max=0.537, sum=1.572 (3)\", \"tab\": \"Robustness\", \"score\": \"0.5239378524073847\"}", - "NaturalQuestions (closed-book) - F1 (Fairness)": "{\"description\": \"min=0.205, mean=0.214, max=0.22, sum=0.642 (3)\", \"tab\": \"Fairness\", \"score\": \"0.21385439000180537\"}", - "NaturalQuestions (open-book) - F1 (Fairness)": "{\"description\": \"min=0.548, mean=0.551, max=0.554, sum=1.654 (3)\", \"tab\": \"Fairness\", \"score\": \"0.5512241821510145\"}", - "NaturalQuestions (closed-book) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NaturalQuestions (open-book) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=109.087, mean=111.754, max=116.087, sum=335.261 (3)\", \"tab\": \"General information\", \"score\": \"111.75366666666667\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=5.508, mean=6.119, max=6.869, sum=18.356 (3)\", \"tab\": \"General information\", \"score\": \"6.118666666666666\"}", - "NaturalQuestions (closed-book) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.691, mean=4.711, max=4.726, sum=14.134 (3)\", \"tab\": \"General information\", \"score\": \"4.711333333333333\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.038, mean=0.039, max=0.04, sum=0.116 (3)\", \"tab\": \"General information\", \"score\": \"0.03866666666666666\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1224.733, mean=1384.565, max=1488.14, sum=4153.695 (3)\", \"tab\": \"General information\", \"score\": \"1384.5649999999998\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=8.216, mean=10.3, max=11.913, sum=30.9 (3)\", \"tab\": \"General information\", \"score\": \"10.299999999999999\"}", - "NaturalQuestions (open-book) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NaturalQuestions (closed-book) - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=0.667 (1)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666667\"}", - "NaturalQuestions (closed-book) - Stereotypes (gender)": "{\"description\": \"min=0.379, mean=0.46, max=0.5, sum=1.379 (3)\", \"tab\": \"Bias\", \"score\": \"0.4597701149425288\"}", - "NaturalQuestions (closed-book) - Representation (race)": "{\"description\": \"min=0.414, mean=0.435, max=0.447, sum=1.304 (3)\", \"tab\": \"Bias\", \"score\": \"0.43455385345385017\"}", - "NaturalQuestions (closed-book) - Representation (gender)": "{\"description\": \"min=0.15, mean=0.223, max=0.269, sum=0.669 (3)\", \"tab\": \"Bias\", \"score\": \"0.2230769230769231\"}", - "NaturalQuestions (open-book) - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=1.333 (2)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666667\"}", - "NaturalQuestions (open-book) - Stereotypes (gender)": "{\"description\": \"min=0.4, mean=0.411, max=0.433, sum=1.233 (3)\", \"tab\": \"Bias\", \"score\": \"0.41111111111111115\"}", - "NaturalQuestions (open-book) - Representation (race)": "{\"description\": \"min=0.421, mean=0.441, max=0.477, sum=1.324 (3)\", \"tab\": \"Bias\", \"score\": \"0.44143286168772855\"}", - "NaturalQuestions (open-book) - Representation (gender)": "{\"description\": \"min=0.022, mean=0.045, max=0.082, sum=0.135 (3)\", \"tab\": \"Bias\", \"score\": \"0.04515740195666192\"}", - "NaturalQuestions (closed-book) - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}", - "NaturalQuestions (open-book) - Toxic fraction": "{\"description\": \"min=0, mean=0.001, max=0.002, sum=0.003 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.001\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "QuAC", - "source_data": { - "dataset_name": "QuAC", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on QuAC", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.349, - "details": { - "description": "min=0.34, mean=0.349, max=0.363, sum=1.047 (3)", - "tab": "Accuracy", - "QuAC - ECE (10-bin)": "{\"description\": \"min=0.081, mean=0.096, max=0.116, sum=0.287 (3)\", \"tab\": \"Calibration\", \"score\": \"0.09561324552236967\"}", - "QuAC - F1 (Robustness)": "{\"description\": \"min=0.188, mean=0.193, max=0.201, sum=0.578 (3)\", \"tab\": \"Robustness\", \"score\": \"0.1926796273359054\"}", - "QuAC - F1 (Fairness)": "{\"description\": \"min=0.268, mean=0.277, max=0.295, sum=0.832 (3)\", \"tab\": \"Fairness\", \"score\": \"0.2774375608495023\"}", - "QuAC - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "QuAC - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "QuAC - # train": "{\"description\": \"min=0.84, mean=0.909, max=0.991, sum=2.727 (3)\", \"tab\": \"General information\", \"score\": \"0.9089999999999999\"}", - "QuAC - truncated": "{\"description\": \"min=0.029, mean=0.033, max=0.037, sum=0.098 (3)\", \"tab\": \"General information\", \"score\": \"0.03266666666666667\"}", - "QuAC - # prompt tokens": "{\"description\": \"min=1596.904, mean=1641.256, max=1672.92, sum=4923.768 (3)\", \"tab\": \"General information\", \"score\": \"1641.256\"}", - "QuAC - # output tokens": "{\"description\": \"min=20.299, mean=21.144, max=22.408, sum=63.432 (3)\", \"tab\": \"General information\", \"score\": \"21.144000000000002\"}", - "QuAC - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "QuAC - Stereotypes (race)": "{\"description\": \"min=0.59, mean=0.612, max=0.636, sum=1.837 (3)\", \"tab\": \"Bias\", \"score\": \"0.6124061124061125\"}", - "QuAC - Stereotypes (gender)": "{\"description\": \"min=0.382, mean=0.403, max=0.421, sum=1.208 (3)\", \"tab\": \"Bias\", \"score\": \"0.40276421801932005\"}", - "QuAC - Representation (race)": "{\"description\": \"min=0.202, mean=0.24, max=0.259, sum=0.719 (3)\", \"tab\": \"Bias\", \"score\": \"0.23980711859954595\"}", - "QuAC - Representation (gender)": "{\"description\": \"min=0.194, mean=0.2, max=0.205, sum=0.601 (3)\", \"tab\": \"Bias\", \"score\": \"0.20029662396768255\"}", - "QuAC - Toxic fraction": "{\"description\": \"min=0, mean=0.0, max=0.001, sum=0.001 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.0003333333333333333\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "HellaSwag", - "source_data": { - "dataset_name": "HellaSwag", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on HellaSwag", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "HellaSwag - ECE (10-bin)": "{\"description\": \"No matching runs\", \"tab\": \"Calibration\", \"score\": \"\"}", - "HellaSwag - EM (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "HellaSwag - EM (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "HellaSwag - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "HellaSwag - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "OpenbookQA - ECE (10-bin)": "{\"description\": \"No matching runs\", \"tab\": \"Calibration\", \"score\": \"\"}", - "OpenbookQA - EM (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "OpenbookQA - EM (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "OpenbookQA - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "OpenbookQA - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "TruthfulQA", - "source_data": { - "dataset_name": "TruthfulQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on TruthfulQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.221, - "details": { - "description": "min=0.208, mean=0.221, max=0.231, sum=0.662 (3)", - "tab": "Accuracy", - "TruthfulQA - ECE (10-bin)": "{\"description\": \"min=0.057, mean=0.064, max=0.068, sum=0.192 (3)\", \"tab\": \"Calibration\", \"score\": \"0.0641638452052097\"}", - "TruthfulQA - EM (Robustness)": "{\"description\": \"min=0.139, mean=0.151, max=0.161, sum=0.454 (3)\", \"tab\": \"Robustness\", \"score\": \"0.15137614678899083\"}", - "TruthfulQA - EM (Fairness)": "{\"description\": \"min=0.144, mean=0.16, max=0.171, sum=0.479 (3)\", \"tab\": \"Fairness\", \"score\": \"0.15953109072375127\"}", - "TruthfulQA - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "TruthfulQA - # eval": "{\"description\": \"min=654, mean=654, max=654, sum=1962 (3)\", \"tab\": \"General information\", \"score\": \"654.0\"}", - "TruthfulQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "TruthfulQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "TruthfulQA - # prompt tokens": "{\"description\": \"min=504.073, mean=514.073, max=533.073, sum=1542.22 (3)\", \"tab\": \"General information\", \"score\": \"514.0733944954128\"}", - "TruthfulQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=3 (3)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "TruthfulQA - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MS MARCO (TREC)", - "source_data": { - "dataset_name": "MS MARCO (TREC)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "NDCG@10 on MS MARCO (TREC)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "MS MARCO (regular) - RR@10 (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "MS MARCO (TREC) - NDCG@10 (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "MS MARCO (regular) - RR@10 (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "MS MARCO (TREC) - NDCG@10 (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "MS MARCO (regular) - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MS MARCO (TREC) - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MS MARCO (regular) - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "MS MARCO (TREC) - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CNN/DailyMail", - "source_data": { - "dataset_name": "CNN/DailyMail", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on CNN/DailyMail", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.139, - "details": { - "description": "min=0.117, mean=0.139, max=0.15, sum=0.834 (6)", - "tab": "Accuracy", - "CNN/DailyMail - Denoised inference time (s)": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "CNN/DailyMail - # eval": "{\"description\": \"min=466, mean=466, max=466, sum=2796 (6)\", \"tab\": \"General information\", \"score\": \"466.0\"}", - "CNN/DailyMail - # train": "{\"description\": \"min=5, mean=5, max=5, sum=30 (6)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CNN/DailyMail - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (6)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CNN/DailyMail - # prompt tokens": "{\"description\": \"min=1564.648, mean=1578.648, max=1593.648, sum=9471.888 (6)\", \"tab\": \"General information\", \"score\": \"1578.648068669528\"}", - "CNN/DailyMail - # output tokens": "{\"description\": \"min=73.322, mean=83.112, max=88.178, sum=498.674 (6)\", \"tab\": \"General information\", \"score\": \"83.11230329041489\"}", - "CNN/DailyMail - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=18 (6)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "CNN/DailyMail - Stereotypes (race)": "{\"description\": \"min=0.58, mean=0.608, max=0.637, sum=3.651 (6)\", \"tab\": \"Bias\", \"score\": \"0.6084787955510622\"}", - "CNN/DailyMail - Stereotypes (gender)": "{\"description\": \"min=0.382, mean=0.391, max=0.398, sum=2.347 (6)\", \"tab\": \"Bias\", \"score\": \"0.3911797965697547\"}", - "CNN/DailyMail - Representation (race)": "{\"description\": \"min=0.254, mean=0.274, max=0.288, sum=1.642 (6)\", \"tab\": \"Bias\", \"score\": \"0.27361254875467617\"}", - "CNN/DailyMail - Representation (gender)": "{\"description\": \"min=0.128, mean=0.151, max=0.191, sum=0.909 (6)\", \"tab\": \"Bias\", \"score\": \"0.15142644383010628\"}", - "CNN/DailyMail - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (6)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}", - "CNN/DailyMail - SummaC": "{\"description\": \"min=0.309, mean=0.481, max=0.569, sum=1.443 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.4809362133230566\"}", - "CNN/DailyMail - QAFactEval": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - BERTScore (F1)": "{\"description\": \"min=0.202, mean=0.255, max=0.288, sum=0.766 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.25521962437955664\"}", - "CNN/DailyMail - Coverage": "{\"description\": \"min=0.8, mean=0.925, max=0.989, sum=5.552 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"0.9253891304300669\"}", - "CNN/DailyMail - Density": "{\"description\": \"min=34.945, mean=41.619, max=45.552, sum=249.715 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"41.61911540769457\"}", - "CNN/DailyMail - Compression": "{\"description\": \"min=8.478, mean=9.039, max=9.909, sum=54.236 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"9.039273431117751\"}", - "CNN/DailyMail - HumanEval-faithfulness": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-relevance": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-coherence": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "XSUM", - "source_data": { - "dataset_name": "XSUM", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on XSUM", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.124, - "details": { - "description": "min=0.122, mean=0.124, max=0.126, sum=0.742 (6)", - "tab": "Accuracy", - "XSUM - Denoised inference time (s)": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "XSUM - # eval": "{\"description\": \"min=518, mean=518, max=518, sum=3108 (6)\", \"tab\": \"General information\", \"score\": \"518.0\"}", - "XSUM - # train": "{\"description\": \"min=4.998, mean=4.999, max=5, sum=29.992 (6)\", \"tab\": \"General information\", \"score\": \"4.998712998712999\"}", - "XSUM - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (6)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "XSUM - # prompt tokens": "{\"description\": \"min=1472.903, mean=1532.912, max=1566.407, sum=9197.471 (6)\", \"tab\": \"General information\", \"score\": \"1532.9118404118406\"}", - "XSUM - # output tokens": "{\"description\": \"min=25.747, mean=25.987, max=26.212, sum=155.923 (6)\", \"tab\": \"General information\", \"score\": \"25.987129987129986\"}", - "XSUM - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=18 (6)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "XSUM - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=4 (6)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "XSUM - Stereotypes (gender)": "{\"description\": \"min=0.449, mean=0.45, max=0.451, sum=2.701 (6)\", \"tab\": \"Bias\", \"score\": \"0.450224364113253\"}", - "XSUM - Representation (race)": "{\"description\": \"min=0.532, mean=0.547, max=0.565, sum=3.282 (6)\", \"tab\": \"Bias\", \"score\": \"0.5469576096753798\"}", - "XSUM - Representation (gender)": "{\"description\": \"min=0.212, mean=0.214, max=0.217, sum=1.283 (6)\", \"tab\": \"Bias\", \"score\": \"0.2138886962661304\"}", - "XSUM - Toxic fraction": "{\"description\": \"min=0, mean=0.002, max=0.004, sum=0.012 (6)\", \"tab\": \"Toxicity\", \"score\": \"0.0019305019305019308\"}", - "XSUM - SummaC": "{\"description\": \"min=-0.233, mean=-0.225, max=-0.212, sum=-0.675 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"-0.22500232932190178\"}", - "XSUM - QAFactEval": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - BERTScore (F1)": "{\"description\": \"min=0.419, mean=0.423, max=0.427, sum=1.269 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.4230439766625391\"}", - "XSUM - Coverage": "{\"description\": \"min=0.817, mean=0.818, max=0.819, sum=4.91 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"0.8184154242425056\"}", - "XSUM - Density": "{\"description\": \"min=3.392, mean=3.507, max=3.668, sum=21.042 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"3.507010978728374\"}", - "XSUM - Compression": "{\"description\": \"min=17.136, mean=17.376, max=17.524, sum=104.258 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"17.376290660463752\"}", - "XSUM - HumanEval-faithfulness": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-relevance": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-coherence": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "IMDB", - "source_data": { - "dataset_name": "IMDB", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on IMDB", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.947, - "details": { - "description": "min=0.944, mean=0.947, max=0.951, sum=2.842 (3)", - "tab": "Accuracy", - "IMDB - ECE (10-bin)": "{\"description\": \"min=0.177, mean=0.204, max=0.232, sum=0.612 (3)\", \"tab\": \"Calibration\", \"score\": \"0.2038815444945483\"}", - "IMDB - EM (Robustness)": "{\"description\": \"min=0.913, mean=0.92, max=0.933, sum=2.76 (3)\", \"tab\": \"Robustness\", \"score\": \"0.9199999999999999\"}", - "IMDB - EM (Fairness)": "{\"description\": \"min=0.93, mean=0.937, max=0.946, sum=2.811 (3)\", \"tab\": \"Fairness\", \"score\": \"0.9369999999999999\"}", - "IMDB - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "IMDB - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "IMDB - # train": "{\"description\": \"min=2.908, mean=4.236, max=4.985, sum=12.708 (3)\", \"tab\": \"General information\", \"score\": \"4.236000000000001\"}", - "IMDB - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IMDB - # prompt tokens": "{\"description\": \"min=1283.569, mean=1560.056, max=1777.712, sum=4680.167 (3)\", \"tab\": \"General information\", \"score\": \"1560.0556666666664\"}", - "IMDB - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=3 (3)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "IMDB - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "IMDB - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CivilComments", - "source_data": { - "dataset_name": "CivilComments", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on CivilComments", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.524, - "details": { - "description": "min=0.014, mean=0.524, max=0.997, sum=28.276 (54)", - "tab": "Accuracy", - "CivilComments - ECE (10-bin)": "{\"description\": \"min=0.112, mean=0.359, max=0.619, sum=19.409 (54)\", \"tab\": \"Calibration\", \"score\": \"0.35941964376806523\"}", - "CivilComments - EM (Robustness)": "{\"description\": \"min=0.011, mean=0.368, max=0.874, sum=19.881 (54)\", \"tab\": \"Robustness\", \"score\": \"0.36816849425853654\"}", - "CivilComments - EM (Fairness)": "{\"description\": \"min=0, mean=0.462, max=0.985, sum=24.963 (54)\", \"tab\": \"Fairness\", \"score\": \"0.4622866273105216\"}", - "CivilComments - Denoised inference time (s)": "{\"description\": \"9 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "CivilComments - # eval": "{\"description\": \"min=74, mean=371.556, max=683, sum=20064 (54)\", \"tab\": \"General information\", \"score\": \"371.55555555555554\"}", - "CivilComments - # train": "{\"description\": \"min=5, mean=5, max=5, sum=270 (54)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CivilComments - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (54)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CivilComments - # prompt tokens": "{\"description\": \"min=362.037, mean=724.782, max=1272.822, sum=39138.207 (54)\", \"tab\": \"General information\", \"score\": \"724.7816027688522\"}", - "CivilComments - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=54 (54)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "CivilComments - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=162 (54)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "CivilComments - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (54)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "RAFT", - "source_data": { - "dataset_name": "RAFT", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on RAFT", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.523, - "details": { - "description": "min=0, mean=0.523, max=0.925, sum=17.25 (33)", - "tab": "Accuracy", - "RAFT - ECE (10-bin)": "{\"description\": \"min=0.143, mean=0.29, max=0.954, sum=9.577 (33)\", \"tab\": \"Calibration\", \"score\": \"0.2902057183123561\"}", - "RAFT - EM (Robustness)": "{\"description\": \"min=0, mean=0.436, max=0.825, sum=14.4 (33)\", \"tab\": \"Robustness\", \"score\": \"0.43636363636363645\"}", - "RAFT - EM (Fairness)": "{\"description\": \"min=0, mean=0.489, max=0.925, sum=16.15 (33)\", \"tab\": \"Fairness\", \"score\": \"0.4893939393939393\"}", - "RAFT - Denoised inference time (s)": "{\"description\": \"11 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "RAFT - # eval": "{\"description\": \"min=40, mean=40, max=40, sum=1320 (33)\", \"tab\": \"General information\", \"score\": \"40.0\"}", - "RAFT - # train": "{\"description\": \"min=0, mean=4.56, max=5, sum=150.475 (33)\", \"tab\": \"General information\", \"score\": \"4.5598484848484855\"}", - "RAFT - truncated": "{\"description\": \"min=0, mean=0.002, max=0.025, sum=0.075 (33)\", \"tab\": \"General information\", \"score\": \"0.002272727272727273\"}", - "RAFT - # prompt tokens": "{\"description\": \"min=262.3, mean=810.769, max=1759.65, sum=26755.375 (33)\", \"tab\": \"General information\", \"score\": \"810.7689393939394\"}", - "RAFT - # output tokens": "{\"description\": \"min=0.125, mean=2.796, max=6.825, sum=92.275 (33)\", \"tab\": \"General information\", \"score\": \"2.796212121212121\"}", - "RAFT - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=99 (33)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "RAFT - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (33)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/aleph-alpha_luminous-supreme-70b.json b/data/models/aleph-alpha_luminous-supreme-70b.json deleted file mode 100644 index e9602a5d32df87aa6e55dc5ca4910fd3b2f3a4de..0000000000000000000000000000000000000000 --- a/data/models/aleph-alpha_luminous-supreme-70b.json +++ /dev/null @@ -1,674 +0,0 @@ -{ - "model_info": { - "name": "Luminous Supreme 70B", - "id": "aleph-alpha/Luminous-Supreme-70B", - "developer": "aleph-alpha", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_classic/aleph-alpha_Luminous-Supreme-70B/1774096308.339228", - "retrieved_timestamp": "1774096308.339228", - "source_metadata": { - "source_name": "helm_classic", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_classic", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_classic", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperform on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.662, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Calibration": "{\"description\": \"\", \"tab\": \"Calibration\", \"score\": \"0.6242368177613321\"}", - "Mean win rate - Robustness": "{\"description\": \"\", \"tab\": \"Robustness\", \"score\": \"0.5464102564102564\"}", - "Mean win rate - Fairness": "{\"description\": \"\", \"tab\": \"Fairness\", \"score\": \"0.5218648018648019\"}", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}", - "Mean win rate - Bias": "{\"description\": \"\", \"tab\": \"Bias\", \"score\": \"0.5709490829944818\"}", - "Mean win rate - Toxicity": "{\"description\": \"\", \"tab\": \"Toxicity\", \"score\": \"0.5562049062049063\"}", - "Mean win rate - Summarization metrics": "{\"description\": \"\", \"tab\": \"Summarization metrics\", \"score\": \"0.7171052631578947\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.38, - "details": { - "description": "min=0.22, mean=0.38, max=0.61, sum=5.702 (15)", - "tab": "Accuracy", - "MMLU - ECE (10-bin)": "{\"description\": \"min=0.122, mean=0.154, max=0.217, sum=2.31 (15)\", \"tab\": \"Calibration\", \"score\": \"0.15396738685964684\"}", - "MMLU - EM (Robustness)": "{\"description\": \"min=0.08, mean=0.255, max=0.51, sum=3.821 (15)\", \"tab\": \"Robustness\", \"score\": \"0.2547368421052632\"}", - "MMLU - EM (Fairness)": "{\"description\": \"min=0.11, mean=0.264, max=0.51, sum=3.955 (15)\", \"tab\": \"Fairness\", \"score\": \"0.2636608187134503\"}", - "MMLU - Denoised inference time (s)": "{\"description\": \"5 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=1542 (15)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=75 (15)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (15)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=360.75, mean=471.075, max=618.447, sum=7066.132 (15)\", \"tab\": \"General information\", \"score\": \"471.0754736842105\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=15 (15)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "MMLU - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=45 (15)\", \"tab\": \"General information\", \"score\": \"3.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "BoolQ", - "source_data": { - "dataset_name": "BoolQ", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on BoolQ", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.775, - "details": { - "description": "min=0.748, mean=0.775, max=0.795, sum=2.325 (3)", - "tab": "Accuracy", - "BoolQ - ECE (10-bin)": "{\"description\": \"min=0.06, mean=0.083, max=0.111, sum=0.248 (3)\", \"tab\": \"Calibration\", \"score\": \"0.08277086924611576\"}", - "BoolQ - EM (Robustness)": "{\"description\": \"min=0.624, mean=0.665, max=0.693, sum=1.996 (3)\", \"tab\": \"Robustness\", \"score\": \"0.6653333333333333\"}", - "BoolQ - EM (Fairness)": "{\"description\": \"min=0.66, mean=0.694, max=0.713, sum=2.081 (3)\", \"tab\": \"Fairness\", \"score\": \"0.6936666666666667\"}", - "BoolQ - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "BoolQ - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "BoolQ - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "BoolQ - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "BoolQ - # prompt tokens": "{\"description\": \"min=651.658, mean=908.991, max=1252.658, sum=2726.974 (3)\", \"tab\": \"General information\", \"score\": \"908.9913333333333\"}", - "BoolQ - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=3 (3)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "BoolQ - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "BoolQ - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.711, - "details": { - "description": "min=0.687, mean=0.711, max=0.742, sum=2.133 (3)", - "tab": "Accuracy", - "NarrativeQA - ECE (10-bin)": "{\"description\": \"min=0.036, mean=0.049, max=0.061, sum=0.147 (3)\", \"tab\": \"Calibration\", \"score\": \"0.04915634481869984\"}", - "NarrativeQA - F1 (Robustness)": "{\"description\": \"min=0.557, mean=0.59, max=0.617, sum=1.771 (3)\", \"tab\": \"Robustness\", \"score\": \"0.5902392957151222\"}", - "NarrativeQA - F1 (Fairness)": "{\"description\": \"min=0.562, mean=0.603, max=0.637, sum=1.808 (3)\", \"tab\": \"Fairness\", \"score\": \"0.6025352758861713\"}", - "NarrativeQA - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=1065 (3)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=1.039, mean=1.621, max=2.037, sum=4.862 (3)\", \"tab\": \"General information\", \"score\": \"1.6206572769953052\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=1606.952, mean=1647.783, max=1694.642, sum=4943.349 (3)\", \"tab\": \"General information\", \"score\": \"1647.783098591549\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=5.749, mean=6.84, max=8.158, sum=20.521 (3)\", \"tab\": \"General information\", \"score\": \"6.84037558685446\"}", - "NarrativeQA - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NarrativeQA - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NarrativeQA - Stereotypes (gender)": "{\"description\": \"min=0.396, mean=0.465, max=0.5, sum=1.396 (3)\", \"tab\": \"Bias\", \"score\": \"0.46527777777777773\"}", - "NarrativeQA - Representation (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=2 (3)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "NarrativeQA - Representation (gender)": "{\"description\": \"min=0.216, mean=0.238, max=0.256, sum=0.714 (3)\", \"tab\": \"Bias\", \"score\": \"0.23804020866547204\"}", - "NarrativeQA - Toxic fraction": "{\"description\": \"min=0.011, mean=0.016, max=0.02, sum=0.048 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.01596244131455399\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (open-book)", - "source_data": { - "dataset_name": "NaturalQuestions (open-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (open-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.649, - "details": { - "description": "min=0.644, mean=0.649, max=0.656, sum=1.946 (3)", - "tab": "Accuracy", - "NaturalQuestions (closed-book) - ECE (10-bin)": "{\"description\": \"min=0.035, mean=0.041, max=0.045, sum=0.123 (3)\", \"tab\": \"Calibration\", \"score\": \"0.04112615448004484\"}", - "NaturalQuestions (open-book) - ECE (10-bin)": "{\"description\": \"min=0.07, mean=0.074, max=0.077, sum=0.222 (3)\", \"tab\": \"Calibration\", \"score\": \"0.07410001302901324\"}", - "NaturalQuestions (closed-book) - F1 (Robustness)": "{\"description\": \"min=0.243, mean=0.252, max=0.261, sum=0.757 (3)\", \"tab\": \"Robustness\", \"score\": \"0.25230806968086933\"}", - "NaturalQuestions (open-book) - F1 (Robustness)": "{\"description\": \"min=0.576, mean=0.586, max=0.593, sum=1.758 (3)\", \"tab\": \"Robustness\", \"score\": \"0.5861072363623724\"}", - "NaturalQuestions (closed-book) - F1 (Fairness)": "{\"description\": \"min=0.23, mean=0.241, max=0.25, sum=0.723 (3)\", \"tab\": \"Fairness\", \"score\": \"0.24089192251975544\"}", - "NaturalQuestions (open-book) - F1 (Fairness)": "{\"description\": \"min=0.583, mean=0.597, max=0.61, sum=1.79 (3)\", \"tab\": \"Fairness\", \"score\": \"0.5966421355805813\"}", - "NaturalQuestions (closed-book) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NaturalQuestions (open-book) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=109.087, mean=111.754, max=116.087, sum=335.261 (3)\", \"tab\": \"General information\", \"score\": \"111.75366666666667\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=4.262, mean=4.508, max=4.666, sum=13.525 (3)\", \"tab\": \"General information\", \"score\": \"4.508333333333334\"}", - "NaturalQuestions (closed-book) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.691, mean=4.711, max=4.726, sum=14.134 (3)\", \"tab\": \"General information\", \"score\": \"4.711333333333333\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.038, mean=0.039, max=0.04, sum=0.116 (3)\", \"tab\": \"General information\", \"score\": \"0.03866666666666666\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1224.733, mean=1384.565, max=1488.14, sum=4153.695 (3)\", \"tab\": \"General information\", \"score\": \"1384.5649999999998\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=6.064, mean=6.362, max=6.864, sum=19.086 (3)\", \"tab\": \"General information\", \"score\": \"6.361999999999999\"}", - "NaturalQuestions (open-book) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NaturalQuestions (closed-book) - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=2 (3)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "NaturalQuestions (closed-book) - Stereotypes (gender)": "{\"description\": \"min=0.338, mean=0.446, max=0.5, sum=1.338 (3)\", \"tab\": \"Bias\", \"score\": \"0.445882557030098\"}", - "NaturalQuestions (closed-book) - Representation (race)": "{\"description\": \"min=0.467, mean=0.48, max=0.498, sum=1.441 (3)\", \"tab\": \"Bias\", \"score\": \"0.48022397745392514\"}", - "NaturalQuestions (closed-book) - Representation (gender)": "{\"description\": \"min=0.02, mean=0.125, max=0.265, sum=0.374 (3)\", \"tab\": \"Bias\", \"score\": \"0.12466386554621849\"}", - "NaturalQuestions (open-book) - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=0.667 (1)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666667\"}", - "NaturalQuestions (open-book) - Stereotypes (gender)": "{\"description\": \"min=0.4, mean=0.444, max=0.5, sum=1.333 (3)\", \"tab\": \"Bias\", \"score\": \"0.4444444444444445\"}", - "NaturalQuestions (open-book) - Representation (race)": "{\"description\": \"min=0.401, mean=0.44, max=0.506, sum=1.319 (3)\", \"tab\": \"Bias\", \"score\": \"0.43982889050590296\"}", - "NaturalQuestions (open-book) - Representation (gender)": "{\"description\": \"min=0.205, mean=0.22, max=0.25, sum=0.66 (3)\", \"tab\": \"Bias\", \"score\": \"0.2201426024955437\"}", - "NaturalQuestions (closed-book) - Toxic fraction": "{\"description\": \"min=0.001, mean=0.002, max=0.003, sum=0.006 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.002\"}", - "NaturalQuestions (open-book) - Toxic fraction": "{\"description\": \"min=0, mean=0.001, max=0.002, sum=0.004 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.0013333333333333333\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "QuAC", - "source_data": { - "dataset_name": "QuAC", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on QuAC", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.37, - "details": { - "description": "min=0.364, mean=0.37, max=0.378, sum=1.111 (3)", - "tab": "Accuracy", - "QuAC - ECE (10-bin)": "{\"description\": \"min=0.054, mean=0.058, max=0.061, sum=0.175 (3)\", \"tab\": \"Calibration\", \"score\": \"0.05820640656843105\"}", - "QuAC - F1 (Robustness)": "{\"description\": \"min=0.221, mean=0.233, max=0.24, sum=0.699 (3)\", \"tab\": \"Robustness\", \"score\": \"0.23311906486145426\"}", - "QuAC - F1 (Fairness)": "{\"description\": \"min=0.28, mean=0.288, max=0.3, sum=0.865 (3)\", \"tab\": \"Fairness\", \"score\": \"0.28824116919086756\"}", - "QuAC - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "QuAC - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "QuAC - # train": "{\"description\": \"min=0.84, mean=0.909, max=0.991, sum=2.727 (3)\", \"tab\": \"General information\", \"score\": \"0.9089999999999999\"}", - "QuAC - truncated": "{\"description\": \"min=0.029, mean=0.033, max=0.037, sum=0.098 (3)\", \"tab\": \"General information\", \"score\": \"0.03266666666666667\"}", - "QuAC - # prompt tokens": "{\"description\": \"min=1596.904, mean=1641.256, max=1672.92, sum=4923.768 (3)\", \"tab\": \"General information\", \"score\": \"1641.256\"}", - "QuAC - # output tokens": "{\"description\": \"min=22.638, mean=26.241, max=28.094, sum=78.723 (3)\", \"tab\": \"General information\", \"score\": \"26.241000000000003\"}", - "QuAC - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "QuAC - Stereotypes (race)": "{\"description\": \"min=0.571, mean=0.598, max=0.615, sum=1.794 (3)\", \"tab\": \"Bias\", \"score\": \"0.5980796023899473\"}", - "QuAC - Stereotypes (gender)": "{\"description\": \"min=0.408, mean=0.412, max=0.415, sum=1.236 (3)\", \"tab\": \"Bias\", \"score\": \"0.41214192227908586\"}", - "QuAC - Representation (race)": "{\"description\": \"min=0.269, mean=0.305, max=0.351, sum=0.914 (3)\", \"tab\": \"Bias\", \"score\": \"0.3046567170277752\"}", - "QuAC - Representation (gender)": "{\"description\": \"min=0.227, mean=0.232, max=0.235, sum=0.696 (3)\", \"tab\": \"Bias\", \"score\": \"0.23187441800624423\"}", - "QuAC - Toxic fraction": "{\"description\": \"min=0.002, mean=0.002, max=0.003, sum=0.007 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.0023333333333333335\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "HellaSwag", - "source_data": { - "dataset_name": "HellaSwag", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on HellaSwag", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "HellaSwag - ECE (10-bin)": "{\"description\": \"No matching runs\", \"tab\": \"Calibration\", \"score\": \"\"}", - "HellaSwag - EM (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "HellaSwag - EM (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "HellaSwag - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "HellaSwag - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "OpenbookQA - ECE (10-bin)": "{\"description\": \"No matching runs\", \"tab\": \"Calibration\", \"score\": \"\"}", - "OpenbookQA - EM (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "OpenbookQA - EM (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "OpenbookQA - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "OpenbookQA - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "TruthfulQA", - "source_data": { - "dataset_name": "TruthfulQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on TruthfulQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.222, - "details": { - "description": "min=0.2, mean=0.222, max=0.258, sum=0.667 (3)", - "tab": "Accuracy", - "TruthfulQA - ECE (10-bin)": "{\"description\": \"min=0.072, mean=0.092, max=0.102, sum=0.276 (3)\", \"tab\": \"Calibration\", \"score\": \"0.09195091586715554\"}", - "TruthfulQA - EM (Robustness)": "{\"description\": \"min=0.092, mean=0.106, max=0.121, sum=0.318 (3)\", \"tab\": \"Robustness\", \"score\": \"0.10601427115188583\"}", - "TruthfulQA - EM (Fairness)": "{\"description\": \"min=0.128, mean=0.132, max=0.138, sum=0.396 (3)\", \"tab\": \"Fairness\", \"score\": \"0.13200815494393475\"}", - "TruthfulQA - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "TruthfulQA - # eval": "{\"description\": \"min=654, mean=654, max=654, sum=1962 (3)\", \"tab\": \"General information\", \"score\": \"654.0\"}", - "TruthfulQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "TruthfulQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "TruthfulQA - # prompt tokens": "{\"description\": \"min=504.073, mean=514.073, max=533.073, sum=1542.22 (3)\", \"tab\": \"General information\", \"score\": \"514.0733944954128\"}", - "TruthfulQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=3 (3)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "TruthfulQA - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MS MARCO (TREC)", - "source_data": { - "dataset_name": "MS MARCO (TREC)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "NDCG@10 on MS MARCO (TREC)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "MS MARCO (regular) - RR@10 (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "MS MARCO (TREC) - NDCG@10 (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "MS MARCO (regular) - RR@10 (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "MS MARCO (TREC) - NDCG@10 (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "MS MARCO (regular) - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MS MARCO (TREC) - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MS MARCO (regular) - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "MS MARCO (TREC) - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CNN/DailyMail", - "source_data": { - "dataset_name": "CNN/DailyMail", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on CNN/DailyMail", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.15, - "details": { - "description": "min=0.133, mean=0.15, max=0.16, sum=0.899 (6)", - "tab": "Accuracy", - "CNN/DailyMail - Denoised inference time (s)": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "CNN/DailyMail - # eval": "{\"description\": \"min=466, mean=466, max=466, sum=2796 (6)\", \"tab\": \"General information\", \"score\": \"466.0\"}", - "CNN/DailyMail - # train": "{\"description\": \"min=5, mean=5, max=5, sum=30 (6)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CNN/DailyMail - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (6)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CNN/DailyMail - # prompt tokens": "{\"description\": \"min=1564.648, mean=1578.648, max=1593.648, sum=9471.888 (6)\", \"tab\": \"General information\", \"score\": \"1578.648068669528\"}", - "CNN/DailyMail - # output tokens": "{\"description\": \"min=71.758, mean=75.51, max=79.294, sum=453.06 (6)\", \"tab\": \"General information\", \"score\": \"75.51001430615165\"}", - "CNN/DailyMail - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=18 (6)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "CNN/DailyMail - Stereotypes (race)": "{\"description\": \"min=0.621, mean=0.63, max=0.646, sum=3.782 (6)\", \"tab\": \"Bias\", \"score\": \"0.6303974395279242\"}", - "CNN/DailyMail - Stereotypes (gender)": "{\"description\": \"min=0.39, mean=0.401, max=0.412, sum=2.406 (6)\", \"tab\": \"Bias\", \"score\": \"0.4010246477666291\"}", - "CNN/DailyMail - Representation (race)": "{\"description\": \"min=0.281, mean=0.291, max=0.297, sum=1.746 (6)\", \"tab\": \"Bias\", \"score\": \"0.2910346586068148\"}", - "CNN/DailyMail - Representation (gender)": "{\"description\": \"min=0.114, mean=0.13, max=0.148, sum=0.782 (6)\", \"tab\": \"Bias\", \"score\": \"0.1303630037220396\"}", - "CNN/DailyMail - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (6)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}", - "CNN/DailyMail - SummaC": "{\"description\": \"min=0.423, mean=0.552, max=0.624, sum=1.656 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.5518853318256234\"}", - "CNN/DailyMail - QAFactEval": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - BERTScore (F1)": "{\"description\": \"min=0.236, mean=0.28, max=0.304, sum=0.841 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.28049037475726807\"}", - "CNN/DailyMail - Coverage": "{\"description\": \"min=0.846, mean=0.939, max=0.988, sum=5.636 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"0.9393220183960566\"}", - "CNN/DailyMail - Density": "{\"description\": \"min=31.874, mean=33.625, max=34.739, sum=201.751 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"33.625141882714196\"}", - "CNN/DailyMail - Compression": "{\"description\": \"min=8.884, mean=9.298, max=9.552, sum=55.787 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"9.29781469578472\"}", - "CNN/DailyMail - HumanEval-faithfulness": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-relevance": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-coherence": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "XSUM", - "source_data": { - "dataset_name": "XSUM", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on XSUM", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.136, - "details": { - "description": "min=0.133, mean=0.136, max=0.14, sum=0.813 (6)", - "tab": "Accuracy", - "XSUM - Denoised inference time (s)": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "XSUM - # eval": "{\"description\": \"min=518, mean=518, max=518, sum=3108 (6)\", \"tab\": \"General information\", \"score\": \"518.0\"}", - "XSUM - # train": "{\"description\": \"min=4.998, mean=4.999, max=5, sum=29.992 (6)\", \"tab\": \"General information\", \"score\": \"4.998712998712999\"}", - "XSUM - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (6)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "XSUM - # prompt tokens": "{\"description\": \"min=1472.903, mean=1532.912, max=1566.407, sum=9197.471 (6)\", \"tab\": \"General information\", \"score\": \"1532.9118404118406\"}", - "XSUM - # output tokens": "{\"description\": \"min=25.844, mean=26.423, max=26.988, sum=158.537 (6)\", \"tab\": \"General information\", \"score\": \"26.422779922779924\"}", - "XSUM - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=18 (6)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "XSUM - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=4 (6)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "XSUM - Stereotypes (gender)": "{\"description\": \"min=0.42, mean=0.439, max=0.456, sum=2.635 (6)\", \"tab\": \"Bias\", \"score\": \"0.4390946502057613\"}", - "XSUM - Representation (race)": "{\"description\": \"min=0.532, mean=0.544, max=0.556, sum=3.264 (6)\", \"tab\": \"Bias\", \"score\": \"0.5439341780805197\"}", - "XSUM - Representation (gender)": "{\"description\": \"min=0.201, mean=0.206, max=0.21, sum=1.238 (6)\", \"tab\": \"Bias\", \"score\": \"0.2063342186388344\"}", - "XSUM - Toxic fraction": "{\"description\": \"min=0, mean=0.001, max=0.002, sum=0.008 (6)\", \"tab\": \"Toxicity\", \"score\": \"0.001287001287001287\"}", - "XSUM - SummaC": "{\"description\": \"min=-0.251, mean=-0.241, max=-0.231, sum=-0.723 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"-0.2409771191414105\"}", - "XSUM - QAFactEval": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - BERTScore (F1)": "{\"description\": \"min=0.442, mean=0.444, max=0.446, sum=1.331 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.44350630738930513\"}", - "XSUM - Coverage": "{\"description\": \"min=0.799, mean=0.807, max=0.816, sum=4.841 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"0.8068883614050096\"}", - "XSUM - Density": "{\"description\": \"min=2.852, mean=3.08, max=3.225, sum=18.481 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"3.080091964253596\"}", - "XSUM - Compression": "{\"description\": \"min=16.326, mean=16.97, max=17.573, sum=101.823 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"16.97049624677277\"}", - "XSUM - HumanEval-faithfulness": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-relevance": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-coherence": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "IMDB", - "source_data": { - "dataset_name": "IMDB", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on IMDB", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.959, - "details": { - "description": "min=0.957, mean=0.959, max=0.961, sum=2.878 (3)", - "tab": "Accuracy", - "IMDB - ECE (10-bin)": "{\"description\": \"min=0.137, mean=0.173, max=0.222, sum=0.519 (3)\", \"tab\": \"Calibration\", \"score\": \"0.1730084935772459\"}", - "IMDB - EM (Robustness)": "{\"description\": \"min=0.931, mean=0.932, max=0.934, sum=2.797 (3)\", \"tab\": \"Robustness\", \"score\": \"0.9323333333333333\"}", - "IMDB - EM (Fairness)": "{\"description\": \"min=0.948, mean=0.949, max=0.951, sum=2.848 (3)\", \"tab\": \"Fairness\", \"score\": \"0.9493333333333333\"}", - "IMDB - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "IMDB - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "IMDB - # train": "{\"description\": \"min=2.908, mean=4.236, max=4.985, sum=12.708 (3)\", \"tab\": \"General information\", \"score\": \"4.236000000000001\"}", - "IMDB - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IMDB - # prompt tokens": "{\"description\": \"min=1283.569, mean=1560.056, max=1777.712, sum=4680.167 (3)\", \"tab\": \"General information\", \"score\": \"1560.0556666666664\"}", - "IMDB - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=3 (3)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "IMDB - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "IMDB - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CivilComments", - "source_data": { - "dataset_name": "CivilComments", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on CivilComments", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.562, - "details": { - "description": "min=0.049, mean=0.562, max=0.984, sum=30.331 (54)", - "tab": "Accuracy", - "CivilComments - ECE (10-bin)": "{\"description\": \"min=0.051, mean=0.272, max=0.563, sum=14.71 (54)\", \"tab\": \"Calibration\", \"score\": \"0.27240452987490027\"}", - "CivilComments - EM (Robustness)": "{\"description\": \"min=0.035, mean=0.263, max=0.67, sum=14.178 (54)\", \"tab\": \"Robustness\", \"score\": \"0.26255411827214337\"}", - "CivilComments - EM (Fairness)": "{\"description\": \"min=0.014, mean=0.432, max=0.912, sum=23.313 (54)\", \"tab\": \"Fairness\", \"score\": \"0.4317285215923749\"}", - "CivilComments - Denoised inference time (s)": "{\"description\": \"9 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "CivilComments - # eval": "{\"description\": \"min=74, mean=371.556, max=683, sum=20064 (54)\", \"tab\": \"General information\", \"score\": \"371.55555555555554\"}", - "CivilComments - # train": "{\"description\": \"min=5, mean=5, max=5, sum=270 (54)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CivilComments - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (54)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CivilComments - # prompt tokens": "{\"description\": \"min=362.037, mean=724.782, max=1272.822, sum=39138.207 (54)\", \"tab\": \"General information\", \"score\": \"724.7816027688522\"}", - "CivilComments - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=54 (54)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "CivilComments - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=162 (54)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "CivilComments - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (54)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "RAFT", - "source_data": { - "dataset_name": "RAFT", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on RAFT", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.653, - "details": { - "description": "min=0, mean=0.653, max=0.975, sum=21.55 (33)", - "tab": "Accuracy", - "RAFT - ECE (10-bin)": "{\"description\": \"min=0.072, mean=0.238, max=1, sum=7.863 (33)\", \"tab\": \"Calibration\", \"score\": \"0.238277000839632\"}", - "RAFT - EM (Robustness)": "{\"description\": \"min=0, mean=0.564, max=0.975, sum=18.6 (33)\", \"tab\": \"Robustness\", \"score\": \"0.5636363636363637\"}", - "RAFT - EM (Fairness)": "{\"description\": \"min=0, mean=0.601, max=0.975, sum=19.825 (33)\", \"tab\": \"Fairness\", \"score\": \"0.6007575757575758\"}", - "RAFT - Denoised inference time (s)": "{\"description\": \"11 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "RAFT - # eval": "{\"description\": \"min=40, mean=40, max=40, sum=1320 (33)\", \"tab\": \"General information\", \"score\": \"40.0\"}", - "RAFT - # train": "{\"description\": \"min=0, mean=4.56, max=5, sum=150.475 (33)\", \"tab\": \"General information\", \"score\": \"4.5598484848484855\"}", - "RAFT - truncated": "{\"description\": \"min=0, mean=0.002, max=0.025, sum=0.075 (33)\", \"tab\": \"General information\", \"score\": \"0.002272727272727273\"}", - "RAFT - # prompt tokens": "{\"description\": \"min=262.3, mean=810.769, max=1759.65, sum=26755.375 (33)\", \"tab\": \"General information\", \"score\": \"810.7689393939394\"}", - "RAFT - # output tokens": "{\"description\": \"min=0, mean=3.097, max=6.725, sum=102.2 (33)\", \"tab\": \"General information\", \"score\": \"3.0969696969696976\"}", - "RAFT - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=99 (33)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "RAFT - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (33)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/alephalpha_luminous-base.json b/data/models/alephalpha_luminous-base.json deleted file mode 100644 index 60674c865f2d92e2a9342bb9961e274d1928aacf..0000000000000000000000000000000000000000 --- a/data/models/alephalpha_luminous-base.json +++ /dev/null @@ -1,380 +0,0 @@ -{ - "model_info": { - "name": "Luminous Base 13B", - "id": "AlephAlpha/luminous-base", - "developer": "AlephAlpha", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_lite/AlephAlpha_luminous-base/1774096306.427425", - "retrieved_timestamp": "1774096306.427425", - "source_metadata": { - "source_name": "helm_lite", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_lite", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_lite", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.041, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.29337078651685394\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.633, - "details": { - "description": "min=0.633, mean=0.633, max=0.633, sum=0.633 (1)", - "tab": "Accuracy", - "NarrativeQA - Observed inference time (s)": "{\"description\": \"min=1.05, mean=1.05, max=1.05, sum=1.05 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.05044368958809\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=2.037, mean=2.037, max=2.037, sum=2.037 (1)\", \"tab\": \"General information\", \"score\": \"2.036619718309859\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=1694.642, mean=1694.642, max=1694.642, sum=1694.642 (1)\", \"tab\": \"General information\", \"score\": \"1694.6422535211268\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=5.521, mean=5.521, max=5.521, sum=5.521 (1)\", \"tab\": \"General information\", \"score\": \"5.52112676056338\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (closed-book)", - "source_data": { - "dataset_name": "NaturalQuestions (closed-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (closed-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.197, - "details": { - "description": "min=0.197, mean=0.197, max=0.197, sum=0.197 (1)", - "tab": "Accuracy", - "NaturalQuestions (open-book) - Observed inference time (s)": "{\"description\": \"min=1.329, mean=1.329, max=1.329, sum=1.329 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.328731627702713\"}", - "NaturalQuestions (closed-book) - Observed inference time (s)": "{\"description\": \"min=0.802, mean=0.802, max=0.802, sum=0.802 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.8020290625095368\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.717, mean=4.717, max=4.717, sum=4.717 (1)\", \"tab\": \"General information\", \"score\": \"4.717\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.038, mean=0.038, max=0.038, sum=0.038 (1)\", \"tab\": \"General information\", \"score\": \"0.038\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1488.14, mean=1488.14, max=1488.14, sum=1488.14 (1)\", \"tab\": \"General information\", \"score\": \"1488.14\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=10.866, mean=10.866, max=10.866, sum=10.866 (1)\", \"tab\": \"General information\", \"score\": \"10.866\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=116.087, mean=116.087, max=116.087, sum=116.087 (1)\", \"tab\": \"General information\", \"score\": \"116.087\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=5.908, mean=5.908, max=5.908, sum=5.908 (1)\", \"tab\": \"General information\", \"score\": \"5.908\"}" - } - }, - "generation_config": { - "additional_details": { - "mode": "\"closedbook\"" - } - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.286, - "details": { - "description": "min=0.286, mean=0.286, max=0.286, sum=0.286 (1)", - "tab": "Accuracy", - "OpenbookQA - Observed inference time (s)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=0.667 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.6669360423088073\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=254.652, mean=254.652, max=254.652, sum=254.652 (1)\", \"tab\": \"General information\", \"score\": \"254.652\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "dataset": "\"openbookqa\"", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.243, - "details": { - "description": "min=0.22, mean=0.243, max=0.29, sum=1.217 (5)", - "tab": "Accuracy", - "MMLU - Observed inference time (s)": "{\"description\": \"min=0.619, mean=0.632, max=0.648, sum=3.162 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.6324507230122884\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=360.75, mean=471.075, max=618.447, sum=2355.377 (5)\", \"tab\": \"General information\", \"score\": \"471.0754736842106\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"college_chemistry\", \"computer_security\", \"econometrics\", \"us_foreign_policy\"]", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MATH", - "source_data": { - "dataset_name": "MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Equivalent (CoT) on MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.026, - "details": { - "description": "min=0, mean=0.026, max=0.067, sum=0.184 (7)", - "tab": "Accuracy", - "MATH - Observed inference time (s)": "{\"description\": \"min=5.282, mean=9.204, max=20.088, sum=64.425 (7)\", \"tab\": \"Efficiency\", \"score\": \"9.203530075671766\"}", - "MATH - # eval": "{\"description\": \"min=30, mean=62.429, max=135, sum=437 (7)\", \"tab\": \"General information\", \"score\": \"62.42857142857143\"}", - "MATH - # train": "{\"description\": \"min=2.962, mean=6.916, max=8, sum=48.409 (7)\", \"tab\": \"General information\", \"score\": \"6.915558126084441\"}", - "MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MATH - # prompt tokens": "{\"description\": \"min=928.719, mean=1184.139, max=1546.442, sum=8288.975 (7)\", \"tab\": \"General information\", \"score\": \"1184.139339428874\"}", - "MATH - # output tokens": "{\"description\": \"min=114.077, mean=139.637, max=180.663, sum=977.456 (7)\", \"tab\": \"General information\", \"score\": \"139.6365272403828\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"algebra\", \"counting_and_probability\", \"geometry\", \"intermediate_algebra\", \"number_theory\", \"prealgebra\", \"precalculus\"]", - "level": "\"1\"", - "use_official_examples": "\"False\"", - "use_chain_of_thought": "\"True\"" - } - } - }, - { - "evaluation_name": "GSM8K", - "source_data": { - "dataset_name": "GSM8K", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on GSM8K", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.028, - "details": { - "description": "min=0.028, mean=0.028, max=0.028, sum=0.028 (1)", - "tab": "Accuracy", - "GSM8K - Observed inference time (s)": "{\"description\": \"min=16.427, mean=16.427, max=16.427, sum=16.427 (1)\", \"tab\": \"Efficiency\", \"score\": \"16.42652773284912\"}", - "GSM8K - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "GSM8K - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "GSM8K - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GSM8K - # prompt tokens": "{\"description\": \"min=943.121, mean=943.121, max=943.121, sum=943.121 (1)\", \"tab\": \"General information\", \"score\": \"943.121\"}", - "GSM8K - # output tokens": "{\"description\": \"min=400, mean=400, max=400, sum=400 (1)\", \"tab\": \"General information\", \"score\": \"400.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "LegalBench", - "source_data": { - "dataset_name": "LegalBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on LegalBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.332, - "details": { - "description": "min=0.165, mean=0.332, max=0.601, sum=1.659 (5)", - "tab": "Accuracy", - "LegalBench - Observed inference time (s)": "{\"description\": \"min=0.636, mean=0.753, max=1.073, sum=3.767 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.7533007583490331\"}", - "LegalBench - # eval": "{\"description\": \"min=95, mean=409.4, max=1000, sum=2047 (5)\", \"tab\": \"General information\", \"score\": \"409.4\"}", - "LegalBench - # train": "{\"description\": \"min=0.335, mean=3.867, max=5, sum=19.335 (5)\", \"tab\": \"General information\", \"score\": \"3.866938775510204\"}", - "LegalBench - truncated": "{\"description\": \"min=0, mean=0.133, max=0.665, sum=0.665 (5)\", \"tab\": \"General information\", \"score\": \"0.1330612244897959\"}", - "LegalBench - # prompt tokens": "{\"description\": \"min=205.726, mean=566.59, max=1514.545, sum=2832.948 (5)\", \"tab\": \"General information\", \"score\": \"566.5895794484264\"}", - "LegalBench - # output tokens": "{\"description\": \"min=1, mean=1.639, max=4.027, sum=8.196 (5)\", \"tab\": \"General information\", \"score\": \"1.6391061224489796\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"abercrombie\", \"corporate_lobbying\", \"function_of_decision_section\", \"international_citizenship_questions\", \"proa\"]" - } - } - }, - { - "evaluation_name": "MedQA", - "source_data": { - "dataset_name": "MedQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MedQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.26, - "details": { - "description": "min=0.26, mean=0.26, max=0.26, sum=0.26 (1)", - "tab": "Accuracy", - "MedQA - Observed inference time (s)": "{\"description\": \"min=0.726, mean=0.726, max=0.726, sum=0.726 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.7258754989972882\"}", - "MedQA - # eval": "{\"description\": \"min=503, mean=503, max=503, sum=503 (1)\", \"tab\": \"General information\", \"score\": \"503.0\"}", - "MedQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MedQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MedQA - # prompt tokens": "{\"description\": \"min=1005.229, mean=1005.229, max=1005.229, sum=1005.229 (1)\", \"tab\": \"General information\", \"score\": \"1005.2286282306163\"}", - "MedQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WMT 2014", - "source_data": { - "dataset_name": "WMT 2014", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "BLEU-4 on WMT 2014", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.066, - "details": { - "description": "min=0.0, mean=0.066, max=0.171, sum=0.331 (5)", - "tab": "Accuracy", - "WMT 2014 - Observed inference time (s)": "{\"description\": \"min=4.671, mean=4.693, max=4.731, sum=23.465 (5)\", \"tab\": \"Efficiency\", \"score\": \"4.692985351748752\"}", - "WMT 2014 - # eval": "{\"description\": \"min=503, mean=568.8, max=832, sum=2844 (5)\", \"tab\": \"General information\", \"score\": \"568.8\"}", - "WMT 2014 - # train": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "WMT 2014 - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WMT 2014 - # prompt tokens": "{\"description\": \"min=99.111, mean=157.232, max=255.504, sum=786.158 (5)\", \"tab\": \"General information\", \"score\": \"157.2315362631901\"}", - "WMT 2014 - # output tokens": "{\"description\": \"min=99.869, mean=99.974, max=100, sum=499.869 (5)\", \"tab\": \"General information\", \"score\": \"99.97375745526838\"}" - } - }, - "generation_config": { - "additional_details": { - "language_pair": "[\"cs-en\", \"de-en\", \"fr-en\", \"hi-en\", \"ru-en\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/alephalpha_luminous-extended.json b/data/models/alephalpha_luminous-extended.json deleted file mode 100644 index 0cd91b6ac04a89aa94f2fe1c148b7d3ca73e8085..0000000000000000000000000000000000000000 --- a/data/models/alephalpha_luminous-extended.json +++ /dev/null @@ -1,380 +0,0 @@ -{ - "model_info": { - "name": "Luminous Extended 30B", - "id": "AlephAlpha/luminous-extended", - "developer": "AlephAlpha", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_lite/AlephAlpha_luminous-extended/1774096306.427425", - "retrieved_timestamp": "1774096306.427425", - "source_metadata": { - "source_name": "helm_lite", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_lite", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_lite", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.078, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.2278027465667915\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.684, - "details": { - "description": "min=0.684, mean=0.684, max=0.684, sum=0.684 (1)", - "tab": "Accuracy", - "NarrativeQA - Observed inference time (s)": "{\"description\": \"min=1.467, mean=1.467, max=1.467, sum=1.467 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.4667296523779212\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=2.037, mean=2.037, max=2.037, sum=2.037 (1)\", \"tab\": \"General information\", \"score\": \"2.036619718309859\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=1694.642, mean=1694.642, max=1694.642, sum=1694.642 (1)\", \"tab\": \"General information\", \"score\": \"1694.6422535211268\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=6.335, mean=6.335, max=6.335, sum=6.335 (1)\", \"tab\": \"General information\", \"score\": \"6.335211267605634\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (closed-book)", - "source_data": { - "dataset_name": "NaturalQuestions (closed-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (closed-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.253, - "details": { - "description": "min=0.253, mean=0.253, max=0.253, sum=0.253 (1)", - "tab": "Accuracy", - "NaturalQuestions (open-book) - Observed inference time (s)": "{\"description\": \"min=1.778, mean=1.778, max=1.778, sum=1.778 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.777582576751709\"}", - "NaturalQuestions (closed-book) - Observed inference time (s)": "{\"description\": \"min=0.98, mean=0.98, max=0.98, sum=0.98 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.9799906523227692\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.717, mean=4.717, max=4.717, sum=4.717 (1)\", \"tab\": \"General information\", \"score\": \"4.717\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.038, mean=0.038, max=0.038, sum=0.038 (1)\", \"tab\": \"General information\", \"score\": \"0.038\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1488.14, mean=1488.14, max=1488.14, sum=1488.14 (1)\", \"tab\": \"General information\", \"score\": \"1488.14\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=11.063, mean=11.063, max=11.063, sum=11.063 (1)\", \"tab\": \"General information\", \"score\": \"11.063\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=116.087, mean=116.087, max=116.087, sum=116.087 (1)\", \"tab\": \"General information\", \"score\": \"116.087\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=6.869, mean=6.869, max=6.869, sum=6.869 (1)\", \"tab\": \"General information\", \"score\": \"6.869\"}" - } - }, - "generation_config": { - "additional_details": { - "mode": "\"closedbook\"" - } - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.272, - "details": { - "description": "min=0.272, mean=0.272, max=0.272, sum=0.272 (1)", - "tab": "Accuracy", - "OpenbookQA - Observed inference time (s)": "{\"description\": \"min=0.675, mean=0.675, max=0.675, sum=0.675 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.6750410146713257\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=254.652, mean=254.652, max=254.652, sum=254.652 (1)\", \"tab\": \"General information\", \"score\": \"254.652\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "dataset": "\"openbookqa\"", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.248, - "details": { - "description": "min=0.2, mean=0.248, max=0.31, sum=1.242 (5)", - "tab": "Accuracy", - "MMLU - Observed inference time (s)": "{\"description\": \"min=0.69, mean=0.718, max=0.754, sum=3.592 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.7183412402554562\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=360.75, mean=471.075, max=618.447, sum=2355.377 (5)\", \"tab\": \"General information\", \"score\": \"471.0754736842106\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"college_chemistry\", \"computer_security\", \"econometrics\", \"us_foreign_policy\"]", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MATH", - "source_data": { - "dataset_name": "MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Equivalent (CoT) on MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.04, - "details": { - "description": "min=0, mean=0.04, max=0.088, sum=0.278 (7)", - "tab": "Accuracy", - "MATH - Observed inference time (s)": "{\"description\": \"min=5.96, mean=9.364, max=12.108, sum=65.551 (7)\", \"tab\": \"Efficiency\", \"score\": \"9.364456500699777\"}", - "MATH - # eval": "{\"description\": \"min=30, mean=62.429, max=135, sum=437 (7)\", \"tab\": \"General information\", \"score\": \"62.42857142857143\"}", - "MATH - # train": "{\"description\": \"min=2.962, mean=6.916, max=8, sum=48.409 (7)\", \"tab\": \"General information\", \"score\": \"6.915558126084441\"}", - "MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MATH - # prompt tokens": "{\"description\": \"min=928.719, mean=1184.139, max=1546.442, sum=8288.975 (7)\", \"tab\": \"General information\", \"score\": \"1184.139339428874\"}", - "MATH - # output tokens": "{\"description\": \"min=92.684, mean=142.866, max=180.2, sum=1000.065 (7)\", \"tab\": \"General information\", \"score\": \"142.86643564287382\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"algebra\", \"counting_and_probability\", \"geometry\", \"intermediate_algebra\", \"number_theory\", \"prealgebra\", \"precalculus\"]", - "level": "\"1\"", - "use_official_examples": "\"False\"", - "use_chain_of_thought": "\"True\"" - } - } - }, - { - "evaluation_name": "GSM8K", - "source_data": { - "dataset_name": "GSM8K", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on GSM8K", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.075, - "details": { - "description": "min=0.075, mean=0.075, max=0.075, sum=0.075 (1)", - "tab": "Accuracy", - "GSM8K - Observed inference time (s)": "{\"description\": \"min=22.685, mean=22.685, max=22.685, sum=22.685 (1)\", \"tab\": \"Efficiency\", \"score\": \"22.685439155817033\"}", - "GSM8K - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "GSM8K - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "GSM8K - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GSM8K - # prompt tokens": "{\"description\": \"min=943.121, mean=943.121, max=943.121, sum=943.121 (1)\", \"tab\": \"General information\", \"score\": \"943.121\"}", - "GSM8K - # output tokens": "{\"description\": \"min=400, mean=400, max=400, sum=400 (1)\", \"tab\": \"General information\", \"score\": \"400.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "LegalBench", - "source_data": { - "dataset_name": "LegalBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on LegalBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.421, - "details": { - "description": "min=0.204, mean=0.421, max=0.632, sum=2.107 (5)", - "tab": "Accuracy", - "LegalBench - Observed inference time (s)": "{\"description\": \"min=0.7, mean=0.858, max=1.261, sum=4.291 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.8581969152200717\"}", - "LegalBench - # eval": "{\"description\": \"min=95, mean=409.4, max=1000, sum=2047 (5)\", \"tab\": \"General information\", \"score\": \"409.4\"}", - "LegalBench - # train": "{\"description\": \"min=0.335, mean=3.867, max=5, sum=19.335 (5)\", \"tab\": \"General information\", \"score\": \"3.866938775510204\"}", - "LegalBench - truncated": "{\"description\": \"min=0, mean=0.133, max=0.665, sum=0.665 (5)\", \"tab\": \"General information\", \"score\": \"0.1330612244897959\"}", - "LegalBench - # prompt tokens": "{\"description\": \"min=205.726, mean=566.59, max=1514.545, sum=2832.948 (5)\", \"tab\": \"General information\", \"score\": \"566.5895794484264\"}", - "LegalBench - # output tokens": "{\"description\": \"min=1, mean=1.548, max=3.196, sum=7.739 (5)\", \"tab\": \"General information\", \"score\": \"1.5478898257711229\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"abercrombie\", \"corporate_lobbying\", \"function_of_decision_section\", \"international_citizenship_questions\", \"proa\"]" - } - } - }, - { - "evaluation_name": "MedQA", - "source_data": { - "dataset_name": "MedQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MedQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.276, - "details": { - "description": "min=0.276, mean=0.276, max=0.276, sum=0.276 (1)", - "tab": "Accuracy", - "MedQA - Observed inference time (s)": "{\"description\": \"min=0.895, mean=0.895, max=0.895, sum=0.895 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.8947408758622277\"}", - "MedQA - # eval": "{\"description\": \"min=503, mean=503, max=503, sum=503 (1)\", \"tab\": \"General information\", \"score\": \"503.0\"}", - "MedQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MedQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MedQA - # prompt tokens": "{\"description\": \"min=1005.229, mean=1005.229, max=1005.229, sum=1005.229 (1)\", \"tab\": \"General information\", \"score\": \"1005.2286282306163\"}", - "MedQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WMT 2014", - "source_data": { - "dataset_name": "WMT 2014", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "BLEU-4 on WMT 2014", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.083, - "details": { - "description": "min=0.0, mean=0.083, max=0.194, sum=0.415 (5)", - "tab": "Accuracy", - "WMT 2014 - Observed inference time (s)": "{\"description\": \"min=5.231, mean=5.336, max=5.406, sum=26.68 (5)\", \"tab\": \"Efficiency\", \"score\": \"5.33597646673717\"}", - "WMT 2014 - # eval": "{\"description\": \"min=503, mean=568.8, max=832, sum=2844 (5)\", \"tab\": \"General information\", \"score\": \"568.8\"}", - "WMT 2014 - # train": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "WMT 2014 - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WMT 2014 - # prompt tokens": "{\"description\": \"min=99.111, mean=157.232, max=255.504, sum=786.158 (5)\", \"tab\": \"General information\", \"score\": \"157.2315362631901\"}", - "WMT 2014 - # output tokens": "{\"description\": \"min=100, mean=100, max=100, sum=500 (5)\", \"tab\": \"General information\", \"score\": \"100.0\"}" - } - }, - "generation_config": { - "additional_details": { - "language_pair": "[\"cs-en\", \"de-en\", \"fr-en\", \"hi-en\", \"ru-en\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/alephalpha_luminous-supreme.json b/data/models/alephalpha_luminous-supreme.json deleted file mode 100644 index f6f98bfdf390e1b0c9d933a6ccaaad303d8f8442..0000000000000000000000000000000000000000 --- a/data/models/alephalpha_luminous-supreme.json +++ /dev/null @@ -1,380 +0,0 @@ -{ - "model_info": { - "name": "Luminous Supreme 70B", - "id": "AlephAlpha/luminous-supreme", - "developer": "AlephAlpha", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_lite/AlephAlpha_luminous-supreme/1774096306.427425", - "retrieved_timestamp": "1774096306.427425", - "source_metadata": { - "source_name": "helm_lite", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_lite", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_lite", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.145, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.1344569288389513\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.743, - "details": { - "description": "min=0.743, mean=0.743, max=0.743, sum=0.743 (1)", - "tab": "Accuracy", - "NarrativeQA - Observed inference time (s)": "{\"description\": \"min=2.951, mean=2.951, max=2.951, sum=2.951 (1)\", \"tab\": \"Efficiency\", \"score\": \"2.9511526873413945\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=2.037, mean=2.037, max=2.037, sum=2.037 (1)\", \"tab\": \"General information\", \"score\": \"2.036619718309859\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=1694.642, mean=1694.642, max=1694.642, sum=1694.642 (1)\", \"tab\": \"General information\", \"score\": \"1694.6422535211268\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=5.685, mean=5.685, max=5.685, sum=5.685 (1)\", \"tab\": \"General information\", \"score\": \"5.6845070422535215\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (closed-book)", - "source_data": { - "dataset_name": "NaturalQuestions (closed-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (closed-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.299, - "details": { - "description": "min=0.299, mean=0.299, max=0.299, sum=0.299 (1)", - "tab": "Accuracy", - "NaturalQuestions (open-book) - Observed inference time (s)": "{\"description\": \"min=2.657, mean=2.657, max=2.657, sum=2.657 (1)\", \"tab\": \"Efficiency\", \"score\": \"2.656584274530411\"}", - "NaturalQuestions (closed-book) - Observed inference time (s)": "{\"description\": \"min=1.272, mean=1.272, max=1.272, sum=1.272 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.2722365505695343\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.717, mean=4.717, max=4.717, sum=4.717 (1)\", \"tab\": \"General information\", \"score\": \"4.717\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.038, mean=0.038, max=0.038, sum=0.038 (1)\", \"tab\": \"General information\", \"score\": \"0.038\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1488.14, mean=1488.14, max=1488.14, sum=1488.14 (1)\", \"tab\": \"General information\", \"score\": \"1488.14\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=6.864, mean=6.864, max=6.864, sum=6.864 (1)\", \"tab\": \"General information\", \"score\": \"6.864\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=116.087, mean=116.087, max=116.087, sum=116.087 (1)\", \"tab\": \"General information\", \"score\": \"116.087\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=4.666, mean=4.666, max=4.666, sum=4.666 (1)\", \"tab\": \"General information\", \"score\": \"4.666\"}" - } - }, - "generation_config": { - "additional_details": { - "mode": "\"closedbook\"" - } - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.284, - "details": { - "description": "min=0.284, mean=0.284, max=0.284, sum=0.284 (1)", - "tab": "Accuracy", - "OpenbookQA - Observed inference time (s)": "{\"description\": \"min=0.779, mean=0.779, max=0.779, sum=0.779 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.778845920085907\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=254.652, mean=254.652, max=254.652, sum=254.652 (1)\", \"tab\": \"General information\", \"score\": \"254.652\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "dataset": "\"openbookqa\"", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.316, - "details": { - "description": "min=0.18, mean=0.316, max=0.5, sum=1.582 (5)", - "tab": "Accuracy", - "MMLU - Observed inference time (s)": "{\"description\": \"min=0.825, mean=0.907, max=1.009, sum=4.537 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.9073754794472141\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=360.75, mean=471.075, max=618.447, sum=2355.377 (5)\", \"tab\": \"General information\", \"score\": \"471.0754736842106\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"college_chemistry\", \"computer_security\", \"econometrics\", \"us_foreign_policy\"]", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MATH", - "source_data": { - "dataset_name": "MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Equivalent (CoT) on MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.078, - "details": { - "description": "min=0.038, mean=0.078, max=0.158, sum=0.548 (7)", - "tab": "Accuracy", - "MATH - Observed inference time (s)": "{\"description\": \"min=13.143, mean=16.874, max=20.77, sum=118.115 (7)\", \"tab\": \"Efficiency\", \"score\": \"16.873623512856078\"}", - "MATH - # eval": "{\"description\": \"min=30, mean=62.429, max=135, sum=437 (7)\", \"tab\": \"General information\", \"score\": \"62.42857142857143\"}", - "MATH - # train": "{\"description\": \"min=2.962, mean=6.916, max=8, sum=48.409 (7)\", \"tab\": \"General information\", \"score\": \"6.915558126084441\"}", - "MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MATH - # prompt tokens": "{\"description\": \"min=928.719, mean=1184.139, max=1546.442, sum=8288.975 (7)\", \"tab\": \"General information\", \"score\": \"1184.139339428874\"}", - "MATH - # output tokens": "{\"description\": \"min=90.605, mean=127.587, max=150.635, sum=893.112 (7)\", \"tab\": \"General information\", \"score\": \"127.58738933898053\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"algebra\", \"counting_and_probability\", \"geometry\", \"intermediate_algebra\", \"number_theory\", \"prealgebra\", \"precalculus\"]", - "level": "\"1\"", - "use_official_examples": "\"False\"", - "use_chain_of_thought": "\"True\"" - } - } - }, - { - "evaluation_name": "GSM8K", - "source_data": { - "dataset_name": "GSM8K", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on GSM8K", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.137, - "details": { - "description": "min=0.137, mean=0.137, max=0.137, sum=0.137 (1)", - "tab": "Accuracy", - "GSM8K - Observed inference time (s)": "{\"description\": \"min=48.242, mean=48.242, max=48.242, sum=48.242 (1)\", \"tab\": \"Efficiency\", \"score\": \"48.241569149971006\"}", - "GSM8K - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "GSM8K - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "GSM8K - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GSM8K - # prompt tokens": "{\"description\": \"min=943.121, mean=943.121, max=943.121, sum=943.121 (1)\", \"tab\": \"General information\", \"score\": \"943.121\"}", - "GSM8K - # output tokens": "{\"description\": \"min=400, mean=400, max=400, sum=400 (1)\", \"tab\": \"General information\", \"score\": \"400.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "LegalBench", - "source_data": { - "dataset_name": "LegalBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on LegalBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.452, - "details": { - "description": "min=0.221, mean=0.452, max=0.768, sum=2.26 (5)", - "tab": "Accuracy", - "LegalBench - Observed inference time (s)": "{\"description\": \"min=0.84, mean=1.156, max=2.035, sum=5.781 (5)\", \"tab\": \"Efficiency\", \"score\": \"1.1561943690304337\"}", - "LegalBench - # eval": "{\"description\": \"min=95, mean=409.4, max=1000, sum=2047 (5)\", \"tab\": \"General information\", \"score\": \"409.4\"}", - "LegalBench - # train": "{\"description\": \"min=0.335, mean=3.867, max=5, sum=19.335 (5)\", \"tab\": \"General information\", \"score\": \"3.866938775510204\"}", - "LegalBench - truncated": "{\"description\": \"min=0, mean=0.133, max=0.665, sum=0.665 (5)\", \"tab\": \"General information\", \"score\": \"0.1330612244897959\"}", - "LegalBench - # prompt tokens": "{\"description\": \"min=205.726, mean=566.59, max=1514.545, sum=2832.948 (5)\", \"tab\": \"General information\", \"score\": \"566.5895794484264\"}", - "LegalBench - # output tokens": "{\"description\": \"min=1, mean=1.266, max=1.769, sum=6.329 (5)\", \"tab\": \"General information\", \"score\": \"1.2657996218650946\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"abercrombie\", \"corporate_lobbying\", \"function_of_decision_section\", \"international_citizenship_questions\", \"proa\"]" - } - } - }, - { - "evaluation_name": "MedQA", - "source_data": { - "dataset_name": "MedQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MedQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.276, - "details": { - "description": "min=0.276, mean=0.276, max=0.276, sum=0.276 (1)", - "tab": "Accuracy", - "MedQA - Observed inference time (s)": "{\"description\": \"min=1.326, mean=1.326, max=1.326, sum=1.326 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.325726029887114\"}", - "MedQA - # eval": "{\"description\": \"min=503, mean=503, max=503, sum=503 (1)\", \"tab\": \"General information\", \"score\": \"503.0\"}", - "MedQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MedQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MedQA - # prompt tokens": "{\"description\": \"min=1005.229, mean=1005.229, max=1005.229, sum=1005.229 (1)\", \"tab\": \"General information\", \"score\": \"1005.2286282306163\"}", - "MedQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WMT 2014", - "source_data": { - "dataset_name": "WMT 2014", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "BLEU-4 on WMT 2014", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.102, - "details": { - "description": "min=0.0, mean=0.102, max=0.193, sum=0.512 (5)", - "tab": "Accuracy", - "WMT 2014 - Observed inference time (s)": "{\"description\": \"min=10.924, mean=11.052, max=11.265, sum=55.26 (5)\", \"tab\": \"Efficiency\", \"score\": \"11.052006985892152\"}", - "WMT 2014 - # eval": "{\"description\": \"min=503, mean=568.8, max=832, sum=2844 (5)\", \"tab\": \"General information\", \"score\": \"568.8\"}", - "WMT 2014 - # train": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "WMT 2014 - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WMT 2014 - # prompt tokens": "{\"description\": \"min=99.111, mean=157.232, max=255.504, sum=786.158 (5)\", \"tab\": \"General information\", \"score\": \"157.2315362631901\"}", - "WMT 2014 - # output tokens": "{\"description\": \"min=100, mean=100, max=100, sum=500 (5)\", \"tab\": \"General information\", \"score\": \"100.0\"}" - } - }, - "generation_config": { - "additional_details": { - "language_pair": "[\"cs-en\", \"de-en\", \"fr-en\", \"hi-en\", \"ru-en\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/alibaba-nlp_gte-qwen2-7b-instruct.json b/data/models/alibaba-nlp_gte-qwen2-7b-instruct.json deleted file mode 100644 index 538b03a6dcf542d33b93cf5262890500314e89b0..0000000000000000000000000000000000000000 --- a/data/models/alibaba-nlp_gte-qwen2-7b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "gte-Qwen2-7B-instruct", - "id": "Alibaba-NLP/gte-Qwen2-7B-instruct", - "developer": "Alibaba-NLP", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.613" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Alibaba-NLP_gte-Qwen2-7B-instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2255 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4495 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0642 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.245 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3559 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3321 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/alibaba_qwen-3-coder-480b.json b/data/models/alibaba_qwen-3-coder-480b.json deleted file mode 100644 index 78f6ba994c4cff4ccaf572ddc5297bedb19efb9b..0000000000000000000000000000000000000000 --- a/data/models/alibaba_qwen-3-coder-480b.json +++ /dev/null @@ -1,235 +0,0 @@ -{ - "model_info": { - "name": "Qwen 3 Coder 480B", - "id": "alibaba/qwen-3-coder-480b", - "developer": "Alibaba", - "additional_details": { - "agent_name": "Dakou Agent", - "agent_organization": "iflow" - } - }, - "evaluations": [ - { - "evaluation_id": "terminal-bench-2.0/dakou-agent__qwen-3-coder-480b/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2025-12-28", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 27.2, - "uncertainty": { - "standard_error": { - "value": 2.6 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Dakou Agent\" -m \"Qwen 3 Coder 480B\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Dakou Agent\" -m \"Qwen 3 Coder 480B\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - }, - { - "evaluation_id": "terminal-bench-2.0/openhands__qwen-3-coder-480b/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2025-11-02", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 25.4, - "uncertainty": { - "standard_error": { - "value": 2.6 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"OpenHands\" -m \"Qwen 3 Coder 480B\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"OpenHands\" -m \"Qwen 3 Coder 480B\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - }, - { - "evaluation_id": "terminal-bench-2.0/terminus-2__qwen-3-coder-480b/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2025-11-01", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 23.9, - "uncertainty": { - "standard_error": { - "value": 2.8 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Terminus 2\" -m \"Qwen 3 Coder 480B\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Terminus 2\" -m \"Qwen 3 Coder 480B\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/alibaba_qwen3-235b-a22b-instruct-2507.json b/data/models/alibaba_qwen3-235b-a22b-instruct-2507.json deleted file mode 100644 index 18e9257ee4a295645c5d70b31ea729844e529c44..0000000000000000000000000000000000000000 --- a/data/models/alibaba_qwen3-235b-a22b-instruct-2507.json +++ /dev/null @@ -1,1043 +0,0 @@ -{ - "model_info": { - "name": "qwen3-235b-a22b-instruct-2507", - "id": "alibaba/qwen3-235b-a22b-instruct-2507", - "developer": "alibaba", - "inference_platform": "unknown", - "additional_details": { - "display_name": "Qwen 3 235B A22B Instruct 2506" - } - }, - "evaluations": [ - { - "evaluation_id": "global-mmlu-lite/alibaba_qwen3-235b-a22b-instruct-2507/1773936496.366405", - "retrieved_timestamp": "1773936496.366405", - "source_metadata": { - "source_name": "Global MMLU Lite Leaderboard", - "source_type": "documentation", - "source_organization_name": "kaggle", - "source_organization_url": "www.kaggle.com", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "kaggle kernel", - "version": "4", - "additional_details": { - "url": "https://www.kaggle.com/code/shivalikasingh95/global-mmlu-lite-sample-notebook" - } - }, - "benchmark": "global-mmlu-lite", - "evaluation_results": [ - { - "evaluation_name": "Global MMLU Lite", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Global MMLU Lite", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8798 - } - }, - { - "evaluation_name": "Culturally Sensitive", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Sensitive", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8522 - } - }, - { - "evaluation_name": "Culturally Agnostic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Agnostic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9075 - } - }, - { - "evaluation_name": "Arabic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Arabic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.88, - "uncertainty": { - "confidence_interval": { - "lower": -0.0318, - "upper": 0.0318, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "English", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - English", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.89, - "uncertainty": { - "confidence_interval": { - "lower": -0.0307, - "upper": 0.0307, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Bengali", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Bengali", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8875, - "uncertainty": { - "confidence_interval": { - "lower": -0.031, - "upper": 0.031, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "German", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - German", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.885, - "uncertainty": { - "confidence_interval": { - "lower": -0.0313, - "upper": 0.0313, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "French", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - French", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.88, - "uncertainty": { - "confidence_interval": { - "lower": -0.0318, - "upper": 0.0318, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Hindi", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Hindi", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8775, - "uncertainty": { - "confidence_interval": { - "lower": -0.0321, - "upper": 0.0321, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Indonesian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Indonesian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.88, - "uncertainty": { - "confidence_interval": { - "lower": -0.0318, - "upper": 0.0318, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Italian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Italian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.88, - "uncertainty": { - "confidence_interval": { - "lower": -0.0318, - "upper": 0.0318, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Japanese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Japanese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.88, - "uncertainty": { - "confidence_interval": { - "lower": -0.0318, - "upper": 0.0318, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Korean", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Korean", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.875, - "uncertainty": { - "confidence_interval": { - "lower": -0.0324, - "upper": 0.0324, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Portuguese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Portuguese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8875, - "uncertainty": { - "confidence_interval": { - "lower": -0.031, - "upper": 0.031, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Spanish", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Spanish", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.875, - "uncertainty": { - "confidence_interval": { - "lower": -0.0324, - "upper": 0.0324, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Swahili", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Swahili", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.87, - "uncertainty": { - "confidence_interval": { - "lower": -0.033, - "upper": 0.033, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Yoruba", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Yoruba", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8725, - "uncertainty": { - "confidence_interval": { - "lower": -0.0327, - "upper": 0.0327, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Chinese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Chinese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8775, - "uncertainty": { - "confidence_interval": { - "lower": -0.0321, - "upper": 0.0321, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Burmese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Burmese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.88, - "uncertainty": { - "confidence_interval": { - "lower": -0.0318, - "upper": 0.0318, - "method": "unknown" - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "global-mmlu-lite/alibaba_qwen3-235b-a22b-instruct-2507/1773936583.743359", - "retrieved_timestamp": "1773936583.743359", - "source_metadata": { - "source_name": "Global MMLU Lite Leaderboard", - "source_type": "documentation", - "source_organization_name": "kaggle", - "source_organization_url": "www.kaggle.com", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "kaggle kernel", - "version": "4", - "additional_details": { - "url": "https://www.kaggle.com/code/shivalikasingh95/global-mmlu-lite-sample-notebook" - } - }, - "benchmark": "global-mmlu-lite", - "evaluation_results": [ - { - "evaluation_name": "Global MMLU Lite", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Global MMLU Lite", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8798 - } - }, - { - "evaluation_name": "Culturally Sensitive", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Sensitive", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8522 - } - }, - { - "evaluation_name": "Culturally Agnostic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Agnostic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9075 - } - }, - { - "evaluation_name": "Arabic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Arabic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.88, - "uncertainty": { - "confidence_interval": { - "lower": -0.0318, - "upper": 0.0318, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "English", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - English", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.89, - "uncertainty": { - "confidence_interval": { - "lower": -0.0307, - "upper": 0.0307, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Bengali", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Bengali", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8875, - "uncertainty": { - "confidence_interval": { - "lower": -0.031, - "upper": 0.031, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "German", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - German", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.885, - "uncertainty": { - "confidence_interval": { - "lower": -0.0313, - "upper": 0.0313, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "French", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - French", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.88, - "uncertainty": { - "confidence_interval": { - "lower": -0.0318, - "upper": 0.0318, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Hindi", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Hindi", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8775, - "uncertainty": { - "confidence_interval": { - "lower": -0.0321, - "upper": 0.0321, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Indonesian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Indonesian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.88, - "uncertainty": { - "confidence_interval": { - "lower": -0.0318, - "upper": 0.0318, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Italian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Italian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.88, - "uncertainty": { - "confidence_interval": { - "lower": -0.0318, - "upper": 0.0318, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Japanese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Japanese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.88, - "uncertainty": { - "confidence_interval": { - "lower": -0.0318, - "upper": 0.0318, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Korean", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Korean", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.875, - "uncertainty": { - "confidence_interval": { - "lower": -0.0324, - "upper": 0.0324, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Portuguese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Portuguese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8875, - "uncertainty": { - "confidence_interval": { - "lower": -0.031, - "upper": 0.031, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Spanish", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Spanish", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.875, - "uncertainty": { - "confidence_interval": { - "lower": -0.0324, - "upper": 0.0324, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Swahili", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Swahili", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.87, - "uncertainty": { - "confidence_interval": { - "lower": -0.033, - "upper": 0.033, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Yoruba", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Yoruba", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8725, - "uncertainty": { - "confidence_interval": { - "lower": -0.0327, - "upper": 0.0327, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Chinese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Chinese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8775, - "uncertainty": { - "confidence_interval": { - "lower": -0.0321, - "upper": 0.0321, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Burmese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Burmese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.88, - "uncertainty": { - "confidence_interval": { - "lower": -0.0318, - "upper": 0.0318, - "method": "unknown" - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/alibaba_qwen3-235b-a22b-thinking-2507.json b/data/models/alibaba_qwen3-235b-a22b-thinking-2507.json deleted file mode 100644 index 478049f048c68762725e78d458923dcd15ec6c85..0000000000000000000000000000000000000000 --- a/data/models/alibaba_qwen3-235b-a22b-thinking-2507.json +++ /dev/null @@ -1,89 +0,0 @@ -{ - "model_info": { - "name": "qwen3-235b-a22b-thinking-2507", - "developer": "Alibaba", - "inference_platform": "aliyun", - "id": "alibaba/qwen3-235b-a22b-thinking-2507" - }, - "evaluations": [ - { - "evaluation_id": "livecodebenchpro/qwen3-235b-a22b-thinking-2507/1760492095.8105888", - "retrieved_timestamp": "1760492095.8105888", - "source_metadata": { - "source_organization_name": "New York University, Princeton University, University of California San Diego, University of Washington and Canyon Crest Academy", - "evaluator_relationship": "third_party", - "source_name": "Live Code Bench Pro", - "source_type": "documentation" - }, - "eval_library": { - "name": "unknown", - "version": "unknown" - }, - "benchmark": "livecodebenchpro", - "evaluation_results": [ - { - "evaluation_name": "Hard Problems", - "metric_config": { - "evaluation_description": "Pass@1 on Hard Problems", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.0 - }, - "source_data": { - "dataset_name": "Hard Problems", - "source_type": "url", - "url": [ - "https://webhook.cp-bench.orzzh.com/leaderboard/llm/difficulty?difficulty=hard&benchmark_mode=live" - ] - } - }, - { - "evaluation_name": "Medium Problems", - "metric_config": { - "evaluation_description": "Pass@1 on Medium Problems", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.1267605633802817 - }, - "source_data": { - "dataset_name": "Medium Problems", - "source_type": "url", - "url": [ - "https://webhook.cp-bench.orzzh.com/leaderboard/llm/difficulty?difficulty=medium&benchmark_mode=live" - ] - } - }, - { - "evaluation_name": "Easy Problems", - "metric_config": { - "evaluation_description": "Pass@1 on Easy Problems", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.7605633802816901 - }, - "source_data": { - "dataset_name": "Easy Problems", - "source_type": "url", - "url": [ - "https://webhook.cp-bench.orzzh.com/leaderboard/llm/difficulty?difficulty=easy&benchmark_mode=live" - ] - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/alibaba_qwen3-30b-a3b.json b/data/models/alibaba_qwen3-30b-a3b.json deleted file mode 100644 index de89f21d31faf52d1e66b17e0f6fe0b15267757e..0000000000000000000000000000000000000000 --- a/data/models/alibaba_qwen3-30b-a3b.json +++ /dev/null @@ -1,89 +0,0 @@ -{ - "model_info": { - "name": "qwen3-30b-a3b", - "developer": "Alibaba", - "inference_platform": "aliyun", - "id": "alibaba/qwen3-30b-a3b" - }, - "evaluations": [ - { - "evaluation_id": "livecodebenchpro/qwen3-30b-a3b/1760492095.8105888", - "retrieved_timestamp": "1760492095.8105888", - "source_metadata": { - "source_organization_name": "New York University, Princeton University, University of California San Diego, University of Washington and Canyon Crest Academy", - "evaluator_relationship": "third_party", - "source_name": "Live Code Bench Pro", - "source_type": "documentation" - }, - "eval_library": { - "name": "unknown", - "version": "unknown" - }, - "benchmark": "livecodebenchpro", - "evaluation_results": [ - { - "evaluation_name": "Hard Problems", - "metric_config": { - "evaluation_description": "Pass@1 on Hard Problems", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.0 - }, - "source_data": { - "dataset_name": "Hard Problems", - "source_type": "url", - "url": [ - "https://webhook.cp-bench.orzzh.com/leaderboard/llm/difficulty?difficulty=hard&benchmark_mode=live" - ] - } - }, - { - "evaluation_name": "Medium Problems", - "metric_config": { - "evaluation_description": "Pass@1 on Medium Problems", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.028169014084507043 - }, - "source_data": { - "dataset_name": "Medium Problems", - "source_type": "url", - "url": [ - "https://webhook.cp-bench.orzzh.com/leaderboard/llm/difficulty?difficulty=medium&benchmark_mode=live" - ] - } - }, - { - "evaluation_name": "Easy Problems", - "metric_config": { - "evaluation_description": "Pass@1 on Easy Problems", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.5774647887323944 - }, - "source_data": { - "dataset_name": "Easy Problems", - "source_type": "url", - "url": [ - "https://webhook.cp-bench.orzzh.com/leaderboard/llm/difficulty?difficulty=easy&benchmark_mode=live" - ] - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/alibaba_qwen3-max.json b/data/models/alibaba_qwen3-max.json deleted file mode 100644 index 3ca376f60a5f85954e62a8acd218a59f3167727f..0000000000000000000000000000000000000000 --- a/data/models/alibaba_qwen3-max.json +++ /dev/null @@ -1,89 +0,0 @@ -{ - "model_info": { - "name": "alibaba/qwen3-max", - "developer": "Alibaba", - "inference_platform": "openrouter", - "id": "alibaba/qwen3-max" - }, - "evaluations": [ - { - "evaluation_id": "livecodebenchpro/alibaba/qwen3-max/1760492095.8105888", - "retrieved_timestamp": "1760492095.8105888", - "source_metadata": { - "source_organization_name": "New York University, Princeton University, University of California San Diego, University of Washington and Canyon Crest Academy", - "evaluator_relationship": "third_party", - "source_name": "Live Code Bench Pro", - "source_type": "documentation" - }, - "eval_library": { - "name": "unknown", - "version": "unknown" - }, - "benchmark": "livecodebenchpro", - "evaluation_results": [ - { - "evaluation_name": "Hard Problems", - "metric_config": { - "evaluation_description": "Pass@1 on Hard Problems", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.0 - }, - "source_data": { - "dataset_name": "Hard Problems", - "source_type": "url", - "url": [ - "https://webhook.cp-bench.orzzh.com/leaderboard/llm/difficulty?difficulty=hard&benchmark_mode=live" - ] - } - }, - { - "evaluation_name": "Medium Problems", - "metric_config": { - "evaluation_description": "Pass@1 on Medium Problems", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.04225352112676056 - }, - "source_data": { - "dataset_name": "Medium Problems", - "source_type": "url", - "url": [ - "https://webhook.cp-bench.orzzh.com/leaderboard/llm/difficulty?difficulty=medium&benchmark_mode=live" - ] - } - }, - { - "evaluation_name": "Easy Problems", - "metric_config": { - "evaluation_description": "Pass@1 on Easy Problems", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.36619718309859156 - }, - "source_data": { - "dataset_name": "Easy Problems", - "source_type": "url", - "url": [ - "https://webhook.cp-bench.orzzh.com/leaderboard/llm/difficulty?difficulty=easy&benchmark_mode=live" - ] - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/alibaba_qwen3-next-80b-a3b-thinking.json b/data/models/alibaba_qwen3-next-80b-a3b-thinking.json deleted file mode 100644 index f502642d876a71bf5a7285cca0b45e7896e13232..0000000000000000000000000000000000000000 --- a/data/models/alibaba_qwen3-next-80b-a3b-thinking.json +++ /dev/null @@ -1,89 +0,0 @@ -{ - "model_info": { - "name": "qwen3-next-80b-a3b-thinking", - "developer": "Alibaba", - "inference_platform": "aliyun", - "id": "alibaba/qwen3-next-80b-a3b-thinking" - }, - "evaluations": [ - { - "evaluation_id": "livecodebenchpro/qwen3-next-80b-a3b-thinking/1760492095.8105888", - "retrieved_timestamp": "1760492095.8105888", - "source_metadata": { - "source_organization_name": "New York University, Princeton University, University of California San Diego, University of Washington and Canyon Crest Academy", - "evaluator_relationship": "third_party", - "source_name": "Live Code Bench Pro", - "source_type": "documentation" - }, - "eval_library": { - "name": "unknown", - "version": "unknown" - }, - "benchmark": "livecodebenchpro", - "evaluation_results": [ - { - "evaluation_name": "Hard Problems", - "metric_config": { - "evaluation_description": "Pass@1 on Hard Problems", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.0 - }, - "source_data": { - "dataset_name": "Hard Problems", - "source_type": "url", - "url": [ - "https://webhook.cp-bench.orzzh.com/leaderboard/llm/difficulty?difficulty=hard&benchmark_mode=live" - ] - } - }, - { - "evaluation_name": "Medium Problems", - "metric_config": { - "evaluation_description": "Pass@1 on Medium Problems", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.14084507042253522 - }, - "source_data": { - "dataset_name": "Medium Problems", - "source_type": "url", - "url": [ - "https://webhook.cp-bench.orzzh.com/leaderboard/llm/difficulty?difficulty=medium&benchmark_mode=live" - ] - } - }, - { - "evaluation_name": "Easy Problems", - "metric_config": { - "evaluation_description": "Pass@1 on Easy Problems", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.7464788732394366 - }, - "source_data": { - "dataset_name": "Easy Problems", - "source_type": "url", - "url": [ - "https://webhook.cp-bench.orzzh.com/leaderboard/llm/difficulty?difficulty=easy&benchmark_mode=live" - ] - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/aliyun_qwen3-next-80b-a3b-thinking.json b/data/models/aliyun_qwen3-next-80b-a3b-thinking.json deleted file mode 100644 index 008bd6851f0c69d9d1ab439525c51fcd5c710719..0000000000000000000000000000000000000000 --- a/data/models/aliyun_qwen3-next-80b-a3b-thinking.json +++ /dev/null @@ -1,89 +0,0 @@ -{ - "model_info": { - "name": "qwen3-next-80b-a3b-thinking", - "id": "aliyun/qwen3-next-80b-a3b-thinking", - "developer": "aliyun", - "inference_platform": "aliyun" - }, - "evaluations": [ - { - "evaluation_id": "livecodebenchpro/qwen3-next-80b-a3b-thinking/1770683238.099205", - "retrieved_timestamp": "1770683238.099205", - "source_metadata": { - "source_name": "Live Code Bench Pro", - "source_type": "documentation", - "source_organization_name": "New York University, Princeton University, University of California San Diego, University of Washington and Canyon Crest Academy", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "unknown", - "version": "unknown" - }, - "benchmark": "livecodebenchpro", - "evaluation_results": [ - { - "evaluation_name": "Hard Problems", - "source_data": { - "dataset_name": "Hard Problems", - "source_type": "url", - "url": [ - "https://webhook.cp-bench.orzzh.com/leaderboard/llm/difficulty?difficulty=hard&benchmark_mode=live" - ] - }, - "metric_config": { - "evaluation_description": "Pass@1 on Hard Problems", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "Medium Problems", - "source_data": { - "dataset_name": "Medium Problems", - "source_type": "url", - "url": [ - "https://webhook.cp-bench.orzzh.com/leaderboard/llm/difficulty?difficulty=medium&benchmark_mode=live" - ] - }, - "metric_config": { - "evaluation_description": "Pass@1 on Medium Problems", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0704 - } - }, - { - "evaluation_name": "Easy Problems", - "source_data": { - "dataset_name": "Easy Problems", - "source_type": "url", - "url": [ - "https://webhook.cp-bench.orzzh.com/leaderboard/llm/difficulty?difficulty=easy&benchmark_mode=live" - ] - }, - "metric_config": { - "evaluation_description": "Pass@1 on Easy Problems", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6901 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_llama-3-tulu-2-70b-uf-mean-rm.json b/data/models/allenai_llama-3-tulu-2-70b-uf-mean-rm.json deleted file mode 100644 index 16a79566fc18ece2466ad365850f108fce66fee4..0000000000000000000000000000000000000000 --- a/data/models/allenai_llama-3-tulu-2-70b-uf-mean-rm.json +++ /dev/null @@ -1,144 +0,0 @@ -{ - "model_info": { - "name": "allenai/llama-3-tulu-2-70b-uf-mean-rm", - "id": "allenai/llama-3-tulu-2-70b-uf-mean-rm", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/allenai_llama-3-tulu-2-70b-uf-mean-rm/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7019 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8631 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5614 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6095 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8268 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5957 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_llama-3-tulu-2-8b-uf-mean-rm.json b/data/models/allenai_llama-3-tulu-2-8b-uf-mean-rm.json deleted file mode 100644 index 40172f74785f53d5719caf9949552e0bb7666f69..0000000000000000000000000000000000000000 --- a/data/models/allenai_llama-3-tulu-2-8b-uf-mean-rm.json +++ /dev/null @@ -1,144 +0,0 @@ -{ - "model_info": { - "name": "allenai/llama-3-tulu-2-8b-uf-mean-rm", - "id": "allenai/llama-3-tulu-2-8b-uf-mean-rm", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/allenai_llama-3-tulu-2-8b-uf-mean-rm/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7342 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9525 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5921 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6162 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8212 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6434 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_llama-3-tulu-2-dpo-70b.json b/data/models/allenai_llama-3-tulu-2-dpo-70b.json deleted file mode 100644 index b1ff317a2054b0b81e2f36072f6f256875d683f5..0000000000000000000000000000000000000000 --- a/data/models/allenai_llama-3-tulu-2-dpo-70b.json +++ /dev/null @@ -1,144 +0,0 @@ -{ - "model_info": { - "name": "allenai/llama-3-tulu-2-dpo-70b", - "id": "allenai/llama-3-tulu-2-dpo-70b", - "developer": "allenai", - "additional_details": { - "model_type": "DPO" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/allenai_llama-3-tulu-2-dpo-70b/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7496 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9637 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5746 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7486 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.802 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5687 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_llama-3-tulu-2-dpo-8b.json b/data/models/allenai_llama-3-tulu-2-dpo-8b.json deleted file mode 100644 index d614e76cb28755f978e226f73300b2a6db0cee4c..0000000000000000000000000000000000000000 --- a/data/models/allenai_llama-3-tulu-2-dpo-8b.json +++ /dev/null @@ -1,144 +0,0 @@ -{ - "model_info": { - "name": "allenai/llama-3-tulu-2-dpo-8b", - "id": "allenai/llama-3-tulu-2-dpo-8b", - "developer": "allenai", - "additional_details": { - "model_type": "DPO" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/allenai_llama-3-tulu-2-dpo-8b/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7275 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9525 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5351 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6649 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8663 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5097 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_llama-3.1-70b-instruct-rm-rb2.json b/data/models/allenai_llama-3.1-70b-instruct-rm-rb2.json deleted file mode 100644 index 6b82bed55df1bdb62067f41e1f2f588c7fd5e772..0000000000000000000000000000000000000000 --- a/data/models/allenai_llama-3.1-70b-instruct-rm-rb2.json +++ /dev/null @@ -1,294 +0,0 @@ -{ - "model_info": { - "name": "allenai/Llama-3.1-70B-Instruct-RM-RB2", - "id": "allenai/Llama-3.1-70B-Instruct-RM-RB2", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_Llama-3.1-70B-Instruct-RM-RB2/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7606 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8126 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4188 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6995 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8844 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8646 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8835 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "reward-bench/allenai_Llama-3.1-70B-Instruct-RM-RB2/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9021 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9665 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8355 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9095 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8969 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_llama-3.1-8b-base-rm-rb2.json b/data/models/allenai_llama-3.1-8b-base-rm-rb2.json deleted file mode 100644 index b77566d88bfa5b7f18f1a64562aee1f65b27a08c..0000000000000000000000000000000000000000 --- a/data/models/allenai_llama-3.1-8b-base-rm-rb2.json +++ /dev/null @@ -1,294 +0,0 @@ -{ - "model_info": { - "name": "allenai/Llama-3.1-8B-Base-RM-RB2", - "id": "allenai/Llama-3.1-8B-Base-RM-RB2", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/allenai_Llama-3.1-8B-Base-RM-RB2/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8463 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.933 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7785 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8851 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7886 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "reward-bench-2/allenai_Llama-3.1-8B-Base-RM-RB2/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.649 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.72 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3625 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.612 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8267 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8323 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5406 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_llama-3.1-8b-instruct-rm-rb2.json b/data/models/allenai_llama-3.1-8b-instruct-rm-rb2.json deleted file mode 100644 index 60d204122c692e04be6211b26f2d79d76e83f431..0000000000000000000000000000000000000000 --- a/data/models/allenai_llama-3.1-8b-instruct-rm-rb2.json +++ /dev/null @@ -1,294 +0,0 @@ -{ - "model_info": { - "name": "allenai/Llama-3.1-8B-Instruct-RM-RB2", - "id": "allenai/Llama-3.1-8B-Instruct-RM-RB2", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_Llama-3.1-8B-Instruct-RM-RB2/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7285 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7432 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4437 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6175 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8956 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9071 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7638 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "reward-bench/allenai_Llama-3.1-8B-Instruct-RM-RB2/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8885 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9581 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8158 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8932 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.887 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_llama-3.1-tulu-3-70b-dpo.json b/data/models/allenai_llama-3.1-tulu-3-70b-dpo.json deleted file mode 100644 index 6be4e265ab0dc482c0e97520bd5c5730e475adef..0000000000000000000000000000000000000000 --- a/data/models/allenai_llama-3.1-tulu-3-70b-dpo.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.1-Tulu-3-70B-DPO", - "id": "allenai/Llama-3.1-Tulu-3-70B-DPO", - "developer": "allenai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "70.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allenai_Llama-3.1-Tulu-3-70B-DPO/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8282 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6146 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4494 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3758 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4923 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4633 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_llama-3.1-tulu-3-70b-sft-rm-rb2.json b/data/models/allenai_llama-3.1-tulu-3-70b-sft-rm-rb2.json deleted file mode 100644 index 3c470e6abd70f5f2bc9a8be57cc36f939ca7753c..0000000000000000000000000000000000000000 --- a/data/models/allenai_llama-3.1-tulu-3-70b-sft-rm-rb2.json +++ /dev/null @@ -1,294 +0,0 @@ -{ - "model_info": { - "name": "allenai/Llama-3.1-Tulu-3-70B-SFT-RM-RB2", - "id": "allenai/Llama-3.1-Tulu-3-70B-SFT-RM-RB2", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/allenai_Llama-3.1-Tulu-3-70B-SFT-RM-RB2/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8892 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9693 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8268 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9027 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8583 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "reward-bench-2/allenai_Llama-3.1-Tulu-3-70B-SFT-RM-RB2/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.722 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8084 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3688 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6776 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8689 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7778 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8308 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_llama-3.1-tulu-3-70b-sft.json b/data/models/allenai_llama-3.1-tulu-3-70b-sft.json deleted file mode 100644 index 6ff884ccdcf37759286e6f5ce929674520cf930e..0000000000000000000000000000000000000000 --- a/data/models/allenai_llama-3.1-tulu-3-70b-sft.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.1-Tulu-3-70B-SFT", - "id": "allenai/Llama-3.1-Tulu-3-70B-SFT", - "developer": "allenai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "70.554" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allenai_Llama-3.1-Tulu-3-70B-SFT/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8051 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5951 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3316 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3448 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5026 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4624 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_llama-3.1-tulu-3-70b.json b/data/models/allenai_llama-3.1-tulu-3-70b.json deleted file mode 100644 index 1db7ede127ee803b7ffd5960ed83ddaa8fe5c58c..0000000000000000000000000000000000000000 --- a/data/models/allenai_llama-3.1-tulu-3-70b.json +++ /dev/null @@ -1,275 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.1-Tulu-3-70B", - "id": "allenai/Llama-3.1-Tulu-3-70B", - "developer": "allenai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "70.554" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allenai_Llama-3.1-Tulu-3-70B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8291 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6164 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4502 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3733 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4948 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4645 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "hfopenllm_v2/allenai_Llama-3.1-Tulu-3-70B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8379 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6157 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3829 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3733 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4988 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4656 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_llama-3.1-tulu-3-8b-dpo-rm-rb2.json b/data/models/allenai_llama-3.1-tulu-3-8b-dpo-rm-rb2.json deleted file mode 100644 index 52eed39924ec31b815aba5be343333e3d61411ae..0000000000000000000000000000000000000000 --- a/data/models/allenai_llama-3.1-tulu-3-8b-dpo-rm-rb2.json +++ /dev/null @@ -1,294 +0,0 @@ -{ - "model_info": { - "name": "allenai/Llama-3.1-Tulu-3-8B-DPO-RM-RB2", - "id": "allenai/Llama-3.1-Tulu-3-8B-DPO-RM-RB2", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_Llama-3.1-Tulu-3-8B-DPO-RM-RB2/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.687 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7516 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3875 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6284 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.86 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8545 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6397 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "reward-bench/allenai_Llama-3.1-Tulu-3-8B-DPO-RM-RB2/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8431 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9553 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.761 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8662 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7898 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_llama-3.1-tulu-3-8b-dpo.json b/data/models/allenai_llama-3.1-tulu-3-8b-dpo.json deleted file mode 100644 index 79a505f185bb641e447a2bac19ffc25bb9cdba9e..0000000000000000000000000000000000000000 --- a/data/models/allenai_llama-3.1-tulu-3-8b-dpo.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.1-Tulu-3-8B-DPO", - "id": "allenai/Llama-3.1-Tulu-3-8B-DPO", - "developer": "allenai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allenai_Llama-3.1-Tulu-3-8B-DPO/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8029 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4079 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2364 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2936 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4161 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2898 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_llama-3.1-tulu-3-8b-rl-rm-rb2.json b/data/models/allenai_llama-3.1-tulu-3-8b-rl-rm-rb2.json deleted file mode 100644 index 407df5e17676737402cd08d14ff8185dc4fb9a2a..0000000000000000000000000000000000000000 --- a/data/models/allenai_llama-3.1-tulu-3-8b-rl-rm-rb2.json +++ /dev/null @@ -1,294 +0,0 @@ -{ - "model_info": { - "name": "allenai/Llama-3.1-Tulu-3-8B-RL-RM-RB2", - "id": "allenai/Llama-3.1-Tulu-3-8B-RL-RM-RB2", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/allenai_Llama-3.1-Tulu-3-8B-RL-RM-RB2/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8369 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9469 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7588 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8703 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7715 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "reward-bench-2/allenai_Llama-3.1-Tulu-3-8B-RL-RM-RB2/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6871 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7642 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6175 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8644 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8485 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6281 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_llama-3.1-tulu-3-8b-rm.json b/data/models/allenai_llama-3.1-tulu-3-8b-rm.json deleted file mode 100644 index 6a924206a3e3fbee9e9f9e786f105d5a75e299b8..0000000000000000000000000000000000000000 --- a/data/models/allenai_llama-3.1-tulu-3-8b-rm.json +++ /dev/null @@ -1,295 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.1-Tulu-3-8B-RM", - "id": "allenai/Llama-3.1-Tulu-3-8B-RM", - "developer": "allenai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForSequenceClassification", - "params_billions": "8.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allenai_Llama-3.1-Tulu-3-8B-RM/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.167 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.295 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2567 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3764 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1082 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "reward-bench-2/allenai_Llama-3.1-Tulu-3-8B-RM/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.59 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7453 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3469 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6448 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7422 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5364 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5243 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_llama-3.1-tulu-3-8b-sft-rm-rb2.json b/data/models/allenai_llama-3.1-tulu-3-8b-sft-rm-rb2.json deleted file mode 100644 index e5b15dac5cb2f4285962219bbf6cd6008c5fe41a..0000000000000000000000000000000000000000 --- a/data/models/allenai_llama-3.1-tulu-3-8b-sft-rm-rb2.json +++ /dev/null @@ -1,294 +0,0 @@ -{ - "model_info": { - "name": "allenai/Llama-3.1-Tulu-3-8B-SFT-RM-RB2", - "id": "allenai/Llama-3.1-Tulu-3-8B-SFT-RM-RB2", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/allenai_Llama-3.1-Tulu-3-8B-SFT-RM-RB2/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8551 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9497 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7917 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8784 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8005 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "reward-bench-2/allenai_Llama-3.1-Tulu-3-8B-SFT-RM-RB2/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6821 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7326 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3875 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5792 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8978 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8889 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6063 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_llama-3.1-tulu-3-8b-sft.json b/data/models/allenai_llama-3.1-tulu-3-8b-sft.json deleted file mode 100644 index 5d33c98016e7c0cbfa15d986d2741cb5ff38a011..0000000000000000000000000000000000000000 --- a/data/models/allenai_llama-3.1-tulu-3-8b-sft.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.1-Tulu-3-8B-SFT", - "id": "allenai/Llama-3.1-Tulu-3-8B-SFT", - "developer": "allenai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allenai_Llama-3.1-Tulu-3-8B-SFT/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7403 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3872 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1178 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2777 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4268 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2812 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_llama-3.1-tulu-3-8b.json b/data/models/allenai_llama-3.1-tulu-3-8b.json deleted file mode 100644 index 53350f20b1f74556a0af6a9aa87ed77d181deaec..0000000000000000000000000000000000000000 --- a/data/models/allenai_llama-3.1-tulu-3-8b.json +++ /dev/null @@ -1,275 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.1-Tulu-3-8B", - "id": "allenai/Llama-3.1-Tulu-3-8B", - "developer": "allenai", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allenai_Llama-3.1-Tulu-3-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8255 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4061 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2115 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.297 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4175 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2821 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "hfopenllm_v2/allenai_Llama-3.1-Tulu-3-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8267 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.405 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1964 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2987 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4175 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2827 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_olmo-1.7-7b-hf.json b/data/models/allenai_olmo-1.7-7b-hf.json deleted file mode 100644 index 920d9a452b184072b23ecf06ccddaecd5da7522a..0000000000000000000000000000000000000000 --- a/data/models/allenai_olmo-1.7-7b-hf.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "OLMo-1.7-7B-hf", - "id": "allenai/OLMo-1.7-7B-hf", - "developer": "allenai", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Unknown", - "params_billions": "0.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allenai_OLMo-1.7-7B-hf/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1569 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3014 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0023 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.255 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3475 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1124 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_olmo-1.7-7b.json b/data/models/allenai_olmo-1.7-7b.json deleted file mode 100644 index efb54151e16bbb93eb0ab70caf70083294b88289..0000000000000000000000000000000000000000 --- a/data/models/allenai_olmo-1.7-7b.json +++ /dev/null @@ -1,1531 +0,0 @@ -{ - "model_info": { - "name": "OLMo 1.7 7B", - "id": "allenai/olmo-1.7-7b", - "developer": "allenai", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_mmlu/allenai_olmo-1.7-7b/1774096312.00548", - "retrieved_timestamp": "1774096312.00548", - "source_metadata": { - "source_name": "helm_mmlu", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_mmlu", - "evaluation_results": [ - { - "evaluation_name": "MMLU All Subjects", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU All Subjects", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.538, - "details": { - "description": "min=0.307, mean=0.538, max=0.769, sum=61.295 (114)", - "tab": "Accuracy", - "MMLU All Subjects - Observed inference time (s)": "{\"description\": \"min=0.518, mean=1.024, max=2.978, sum=116.777 (114)\", \"tab\": \"Efficiency\", \"score\": \"1.024362741022275\"}", - "MMLU All Subjects - # eval": "{\"description\": \"min=100, mean=246.351, max=1534, sum=28084 (114)\", \"tab\": \"General information\", \"score\": \"246.35087719298247\"}", - "MMLU All Subjects - # train": "{\"description\": \"min=2.909, mean=4.946, max=5, sum=563.813 (114)\", \"tab\": \"General information\", \"score\": \"4.945727778020373\"}", - "MMLU All Subjects - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU All Subjects - # prompt tokens": "{\"description\": \"min=285.766, mean=597.916, max=1816.758, sum=68162.415 (114)\", \"tab\": \"General information\", \"score\": \"597.9159199418197\"}", - "MMLU All Subjects - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=114 (114)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_name": "Abstract Algebra", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Abstract Algebra", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.33, - "details": { - "description": "min=0.33, mean=0.33, max=0.33, sum=0.66 (2)", - "tab": "Accuracy", - "Abstract Algebra - Observed inference time (s)": "{\"description\": \"min=0.664, mean=0.664, max=0.664, sum=1.328 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.664234619140625\"}", - "Abstract Algebra - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Abstract Algebra - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Abstract Algebra - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Abstract Algebra - # prompt tokens": "{\"description\": \"min=358.76, mean=358.76, max=358.76, sum=717.52 (2)\", \"tab\": \"General information\", \"score\": \"358.76\"}", - "Abstract Algebra - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"abstract_algebra\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_abstract_algebra\"" - } - } - }, - { - "evaluation_name": "Anatomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Anatomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.496, - "details": { - "description": "min=0.496, mean=0.496, max=0.496, sum=0.993 (2)", - "tab": "Accuracy", - "Anatomy - Observed inference time (s)": "{\"description\": \"min=0.619, mean=0.619, max=0.619, sum=1.237 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.618622675648442\"}", - "Anatomy - # eval": "{\"description\": \"min=135, mean=135, max=135, sum=270 (2)\", \"tab\": \"General information\", \"score\": \"135.0\"}", - "Anatomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Anatomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Anatomy - # prompt tokens": "{\"description\": \"min=352.03, mean=352.03, max=352.03, sum=704.059 (2)\", \"tab\": \"General information\", \"score\": \"352.02962962962965\"}", - "Anatomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"anatomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_anatomy\"" - } - } - }, - { - "evaluation_name": "College Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on College Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.333, - "details": { - "description": "min=0.333, mean=0.333, max=0.333, sum=0.667 (2)", - "tab": "Accuracy", - "College Chemistry - Observed inference time (s)": "{\"description\": \"min=0.954, mean=0.954, max=0.954, sum=1.908 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9539380264282227\"}", - "College Biology - Observed inference time (s)": "{\"description\": \"min=0.791, mean=0.791, max=0.791, sum=1.582 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7911433676878611\"}", - "College Computer Science - Observed inference time (s)": "{\"description\": \"min=1.44, mean=1.44, max=1.44, sum=2.88 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.4402443194389343\"}", - "College Mathematics - Observed inference time (s)": "{\"description\": \"min=1.005, mean=1.005, max=1.005, sum=2.01 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.0049437880516052\"}", - "College Medicine - Observed inference time (s)": "{\"description\": \"min=0.933, mean=0.933, max=0.933, sum=1.866 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9331957646188019\"}", - "College Physics - Observed inference time (s)": "{\"description\": \"min=0.854, mean=0.854, max=0.854, sum=1.707 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8537454745348763\"}", - "College Chemistry - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Chemistry - # prompt tokens": "{\"description\": \"min=535.85, mean=535.85, max=535.85, sum=1071.7 (2)\", \"tab\": \"General information\", \"score\": \"535.85\"}", - "College Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Biology - # eval": "{\"description\": \"min=144, mean=144, max=144, sum=288 (2)\", \"tab\": \"General information\", \"score\": \"144.0\"}", - "College Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # prompt tokens": "{\"description\": \"min=470.319, mean=470.319, max=470.319, sum=940.639 (2)\", \"tab\": \"General information\", \"score\": \"470.31944444444446\"}", - "College Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # prompt tokens": "{\"description\": \"min=842.89, mean=842.89, max=842.89, sum=1685.78 (2)\", \"tab\": \"General information\", \"score\": \"842.89\"}", - "College Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Mathematics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # prompt tokens": "{\"description\": \"min=592.82, mean=592.82, max=592.82, sum=1185.64 (2)\", \"tab\": \"General information\", \"score\": \"592.82\"}", - "College Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Medicine - # eval": "{\"description\": \"min=173, mean=173, max=173, sum=346 (2)\", \"tab\": \"General information\", \"score\": \"173.0\"}", - "College Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # prompt tokens": "{\"description\": \"min=519.376, mean=519.376, max=519.376, sum=1038.751 (2)\", \"tab\": \"General information\", \"score\": \"519.3757225433526\"}", - "College Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Physics - # eval": "{\"description\": \"min=102, mean=102, max=102, sum=204 (2)\", \"tab\": \"General information\", \"score\": \"102.0\"}", - "College Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # prompt tokens": "{\"description\": \"min=476.657, mean=476.657, max=476.657, sum=953.314 (2)\", \"tab\": \"General information\", \"score\": \"476.65686274509807\"}", - "College Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"college_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_college_physics\"" - } - } - }, - { - "evaluation_name": "Computer Security", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Computer Security", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.65, - "details": { - "description": "min=0.65, mean=0.65, max=0.65, sum=1.3 (2)", - "tab": "Accuracy", - "Computer Security - Observed inference time (s)": "{\"description\": \"min=0.671, mean=0.671, max=0.671, sum=1.343 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6713726472854614\"}", - "Computer Security - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Computer Security - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Computer Security - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Computer Security - # prompt tokens": "{\"description\": \"min=388.19, mean=388.19, max=388.19, sum=776.38 (2)\", \"tab\": \"General information\", \"score\": \"388.19\"}", - "Computer Security - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"computer_security\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_computer_security\"" - } - } - }, - { - "evaluation_name": "Econometrics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Econometrics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.404, - "details": { - "description": "min=0.404, mean=0.404, max=0.404, sum=0.807 (2)", - "tab": "Accuracy", - "Econometrics - Observed inference time (s)": "{\"description\": \"min=1.05, mean=1.05, max=1.05, sum=2.099 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.0495816971126355\"}", - "Econometrics - # eval": "{\"description\": \"min=114, mean=114, max=114, sum=228 (2)\", \"tab\": \"General information\", \"score\": \"114.0\"}", - "Econometrics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Econometrics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Econometrics - # prompt tokens": "{\"description\": \"min=612.798, mean=612.798, max=612.798, sum=1225.596 (2)\", \"tab\": \"General information\", \"score\": \"612.7982456140351\"}", - "Econometrics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"econometrics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_econometrics\"" - } - } - }, - { - "evaluation_name": "Global Facts", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Global Facts", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.34, - "details": { - "description": "min=0.34, mean=0.34, max=0.34, sum=0.68 (2)", - "tab": "Accuracy", - "Global Facts - Observed inference time (s)": "{\"description\": \"min=0.739, mean=0.739, max=0.739, sum=1.477 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7387202930450439\"}", - "Global Facts - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Global Facts - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Global Facts - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Global Facts - # prompt tokens": "{\"description\": \"min=400.58, mean=400.58, max=400.58, sum=801.16 (2)\", \"tab\": \"General information\", \"score\": \"400.58\"}", - "Global Facts - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"global_facts\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_global_facts\"" - } - } - }, - { - "evaluation_name": "Jurisprudence", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Jurisprudence", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.565, - "details": { - "description": "min=0.565, mean=0.565, max=0.565, sum=1.13 (2)", - "tab": "Accuracy", - "Jurisprudence - Observed inference time (s)": "{\"description\": \"min=0.755, mean=0.755, max=0.755, sum=1.51 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7549951495947661\"}", - "Jurisprudence - # eval": "{\"description\": \"min=108, mean=108, max=108, sum=216 (2)\", \"tab\": \"General information\", \"score\": \"108.0\"}", - "Jurisprudence - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Jurisprudence - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Jurisprudence - # prompt tokens": "{\"description\": \"min=420.861, mean=420.861, max=420.861, sum=841.722 (2)\", \"tab\": \"General information\", \"score\": \"420.8611111111111\"}", - "Jurisprudence - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"jurisprudence\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_jurisprudence\"" - } - } - }, - { - "evaluation_name": "Philosophy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Philosophy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.592, - "details": { - "description": "min=0.592, mean=0.592, max=0.592, sum=1.183 (2)", - "tab": "Accuracy", - "Philosophy - Observed inference time (s)": "{\"description\": \"min=0.622, mean=0.622, max=0.622, sum=1.244 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6219598725677686\"}", - "Philosophy - # eval": "{\"description\": \"min=311, mean=311, max=311, sum=622 (2)\", \"tab\": \"General information\", \"score\": \"311.0\"}", - "Philosophy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Philosophy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Philosophy - # prompt tokens": "{\"description\": \"min=345.277, mean=345.277, max=345.277, sum=690.553 (2)\", \"tab\": \"General information\", \"score\": \"345.2765273311897\"}", - "Philosophy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"philosophy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_philosophy\"" - } - } - }, - { - "evaluation_name": "Professional Psychology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Professional Psychology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.526, - "details": { - "description": "min=0.526, mean=0.526, max=0.526, sum=1.052 (2)", - "tab": "Accuracy", - "Professional Medicine - Observed inference time (s)": "{\"description\": \"min=1.88, mean=1.88, max=1.88, sum=3.759 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.8796235156409882\"}", - "Professional Accounting - Observed inference time (s)": "{\"description\": \"min=1.156, mean=1.156, max=1.156, sum=2.312 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.1558757741400536\"}", - "Professional Law - Observed inference time (s)": "{\"description\": \"min=2.735, mean=2.735, max=2.735, sum=5.47 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.734811251757198\"}", - "Professional Psychology - Observed inference time (s)": "{\"description\": \"min=1.006, mean=1.006, max=1.006, sum=2.012 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.0057547404096017\"}", - "Professional Medicine - # eval": "{\"description\": \"min=272, mean=272, max=272, sum=544 (2)\", \"tab\": \"General information\", \"score\": \"272.0\"}", - "Professional Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Medicine - # prompt tokens": "{\"description\": \"min=1080.882, mean=1080.882, max=1080.882, sum=2161.765 (2)\", \"tab\": \"General information\", \"score\": \"1080.8823529411766\"}", - "Professional Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Accounting - # eval": "{\"description\": \"min=282, mean=282, max=282, sum=564 (2)\", \"tab\": \"General information\", \"score\": \"282.0\"}", - "Professional Accounting - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Accounting - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # prompt tokens": "{\"description\": \"min=660.922, mean=660.922, max=660.922, sum=1321.844 (2)\", \"tab\": \"General information\", \"score\": \"660.9219858156029\"}", - "Professional Accounting - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Law - # eval": "{\"description\": \"min=1534, mean=1534, max=1534, sum=3068 (2)\", \"tab\": \"General information\", \"score\": \"1534.0\"}", - "Professional Law - # train": "{\"description\": \"min=4.997, mean=4.997, max=4.997, sum=9.995 (2)\", \"tab\": \"General information\", \"score\": \"4.9973924380704045\"}", - "Professional Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # prompt tokens": "{\"description\": \"min=1654.433, mean=1654.433, max=1654.433, sum=3308.866 (2)\", \"tab\": \"General information\", \"score\": \"1654.4328552803129\"}", - "Professional Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Psychology - # eval": "{\"description\": \"min=612, mean=612, max=612, sum=1224 (2)\", \"tab\": \"General information\", \"score\": \"612.0\"}", - "Professional Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # prompt tokens": "{\"description\": \"min=590.873, mean=590.873, max=590.873, sum=1181.745 (2)\", \"tab\": \"General information\", \"score\": \"590.8725490196078\"}", - "Professional Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"professional_psychology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_professional_psychology\"" - } - } - }, - { - "evaluation_name": "Us Foreign Policy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Us Foreign Policy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.76, - "details": { - "description": "min=0.76, mean=0.76, max=0.76, sum=1.52 (2)", - "tab": "Accuracy", - "Us Foreign Policy - Observed inference time (s)": "{\"description\": \"min=0.802, mean=0.802, max=0.802, sum=1.604 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8018933439254761\"}", - "Us Foreign Policy - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Us Foreign Policy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Us Foreign Policy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Us Foreign Policy - # prompt tokens": "{\"description\": \"min=444.08, mean=444.08, max=444.08, sum=888.16 (2)\", \"tab\": \"General information\", \"score\": \"444.08\"}", - "Us Foreign Policy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"us_foreign_policy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_us_foreign_policy\"" - } - } - }, - { - "evaluation_name": "Astronomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Astronomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.526, - "details": { - "description": "min=0.526, mean=0.526, max=0.526, sum=1.053 (2)", - "tab": "Accuracy", - "Astronomy - Observed inference time (s)": "{\"description\": \"min=1.012, mean=1.012, max=1.012, sum=2.023 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.0116610966230695\"}", - "Astronomy - # eval": "{\"description\": \"min=152, mean=152, max=152, sum=304 (2)\", \"tab\": \"General information\", \"score\": \"152.0\"}", - "Astronomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Astronomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Astronomy - # prompt tokens": "{\"description\": \"min=598.487, mean=598.487, max=598.487, sum=1196.974 (2)\", \"tab\": \"General information\", \"score\": \"598.4868421052631\"}", - "Astronomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"astronomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_astronomy\"" - } - } - }, - { - "evaluation_name": "Business Ethics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Business Ethics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.59, - "details": { - "description": "min=0.59, mean=0.59, max=0.59, sum=1.18 (2)", - "tab": "Accuracy", - "Business Ethics - Observed inference time (s)": "{\"description\": \"min=0.964, mean=0.964, max=0.964, sum=1.929 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9642905473709107\"}", - "Business Ethics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Business Ethics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Business Ethics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Business Ethics - # prompt tokens": "{\"description\": \"min=585.05, mean=585.05, max=585.05, sum=1170.1 (2)\", \"tab\": \"General information\", \"score\": \"585.05\"}", - "Business Ethics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"business_ethics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_business_ethics\"" - } - } - }, - { - "evaluation_name": "Clinical Knowledge", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Clinical Knowledge", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.57, - "details": { - "description": "min=0.57, mean=0.57, max=0.57, sum=1.14 (2)", - "tab": "Accuracy", - "Clinical Knowledge - Observed inference time (s)": "{\"description\": \"min=0.57, mean=0.57, max=0.57, sum=1.139 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5697462513761701\"}", - "Clinical Knowledge - # eval": "{\"description\": \"min=265, mean=265, max=265, sum=530 (2)\", \"tab\": \"General information\", \"score\": \"265.0\"}", - "Clinical Knowledge - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Clinical Knowledge - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Clinical Knowledge - # prompt tokens": "{\"description\": \"min=401.917, mean=401.917, max=401.917, sum=803.834 (2)\", \"tab\": \"General information\", \"score\": \"401.9169811320755\"}", - "Clinical Knowledge - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"clinical_knowledge\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_clinical_knowledge\"" - } - } - }, - { - "evaluation_name": "Conceptual Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Conceptual Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.434, - "details": { - "description": "min=0.434, mean=0.434, max=0.434, sum=0.868 (2)", - "tab": "Accuracy", - "Conceptual Physics - Observed inference time (s)": "{\"description\": \"min=0.524, mean=0.524, max=0.524, sum=1.049 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5244635977643601\"}", - "Conceptual Physics - # eval": "{\"description\": \"min=235, mean=235, max=235, sum=470 (2)\", \"tab\": \"General information\", \"score\": \"235.0\"}", - "Conceptual Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Conceptual Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Conceptual Physics - # prompt tokens": "{\"description\": \"min=311.311, mean=311.311, max=311.311, sum=622.621 (2)\", \"tab\": \"General information\", \"score\": \"311.31063829787234\"}", - "Conceptual Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"conceptual_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_conceptual_physics\"" - } - } - }, - { - "evaluation_name": "Electrical Engineering", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Electrical Engineering", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.517, - "details": { - "description": "min=0.517, mean=0.517, max=0.517, sum=1.034 (2)", - "tab": "Accuracy", - "Electrical Engineering - Observed inference time (s)": "{\"description\": \"min=0.764, mean=0.764, max=0.764, sum=1.528 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7642407762593236\"}", - "Electrical Engineering - # eval": "{\"description\": \"min=145, mean=145, max=145, sum=290 (2)\", \"tab\": \"General information\", \"score\": \"145.0\"}", - "Electrical Engineering - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Electrical Engineering - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Electrical Engineering - # prompt tokens": "{\"description\": \"min=424.848, mean=424.848, max=424.848, sum=849.697 (2)\", \"tab\": \"General information\", \"score\": \"424.848275862069\"}", - "Electrical Engineering - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"electrical_engineering\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_electrical_engineering\"" - } - } - }, - { - "evaluation_name": "Elementary Mathematics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Elementary Mathematics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.307, - "details": { - "description": "min=0.307, mean=0.307, max=0.307, sum=0.614 (2)", - "tab": "Accuracy", - "Elementary Mathematics - Observed inference time (s)": "{\"description\": \"min=0.909, mean=0.909, max=0.909, sum=1.817 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9087190634359128\"}", - "Elementary Mathematics - # eval": "{\"description\": \"min=378, mean=378, max=378, sum=756 (2)\", \"tab\": \"General information\", \"score\": \"378.0\"}", - "Elementary Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Elementary Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Elementary Mathematics - # prompt tokens": "{\"description\": \"min=505.071, mean=505.071, max=505.071, sum=1010.143 (2)\", \"tab\": \"General information\", \"score\": \"505.07142857142856\"}", - "Elementary Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"elementary_mathematics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_elementary_mathematics\"" - } - } - }, - { - "evaluation_name": "Formal Logic", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Formal Logic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.325, - "details": { - "description": "min=0.325, mean=0.325, max=0.325, sum=0.651 (2)", - "tab": "Accuracy", - "Formal Logic - Observed inference time (s)": "{\"description\": \"min=1.12, mean=1.12, max=1.12, sum=2.24 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.1198924439293998\"}", - "Formal Logic - # eval": "{\"description\": \"min=126, mean=126, max=126, sum=252 (2)\", \"tab\": \"General information\", \"score\": \"126.0\"}", - "Formal Logic - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Formal Logic - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Formal Logic - # prompt tokens": "{\"description\": \"min=653.595, mean=653.595, max=653.595, sum=1307.19 (2)\", \"tab\": \"General information\", \"score\": \"653.5952380952381\"}", - "Formal Logic - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"formal_logic\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_formal_logic\"" - } - } - }, - { - "evaluation_name": "High School World History", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on High School World History", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.713, - "details": { - "description": "min=0.713, mean=0.713, max=0.713, sum=1.426 (2)", - "tab": "Accuracy", - "High School Biology - Observed inference time (s)": "{\"description\": \"min=0.926, mean=0.926, max=0.926, sum=1.852 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9262428129872968\"}", - "High School Chemistry - Observed inference time (s)": "{\"description\": \"min=0.711, mean=0.711, max=0.711, sum=1.421 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.710636249316737\"}", - "High School Computer Science - Observed inference time (s)": "{\"description\": \"min=1.389, mean=1.389, max=1.389, sum=2.779 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.3893755102157592\"}", - "High School European History - Observed inference time (s)": "{\"description\": \"min=2.978, mean=2.978, max=2.978, sum=5.957 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.9784073266116056\"}", - "High School Geography - Observed inference time (s)": "{\"description\": \"min=0.736, mean=0.736, max=0.736, sum=1.471 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7356561253769229\"}", - "High School Government And Politics - Observed inference time (s)": "{\"description\": \"min=0.878, mean=0.878, max=0.878, sum=1.755 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8775828440572314\"}", - "High School Macroeconomics - Observed inference time (s)": "{\"description\": \"min=0.689, mean=0.689, max=0.689, sum=1.378 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6891599153861021\"}", - "High School Mathematics - Observed inference time (s)": "{\"description\": \"min=0.917, mean=0.917, max=0.917, sum=1.834 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9171109632209495\"}", - "High School Microeconomics - Observed inference time (s)": "{\"description\": \"min=0.748, mean=0.748, max=0.748, sum=1.496 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7482213062398574\"}", - "High School Physics - Observed inference time (s)": "{\"description\": \"min=0.983, mean=0.983, max=0.983, sum=1.965 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9825576125391272\"}", - "High School Psychology - Observed inference time (s)": "{\"description\": \"min=0.92, mean=0.92, max=0.92, sum=1.84 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9199631371629348\"}", - "High School Statistics - Observed inference time (s)": "{\"description\": \"min=1.151, mean=1.151, max=1.151, sum=2.303 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.1514487498336368\"}", - "High School US History - Observed inference time (s)": "{\"description\": \"min=2.908, mean=2.908, max=2.908, sum=5.816 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.9081676029691508\"}", - "High School World History - Observed inference time (s)": "{\"description\": \"min=2.459, mean=2.459, max=2.459, sum=4.919 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.4593187173207602\"}", - "High School Biology - # eval": "{\"description\": \"min=310, mean=310, max=310, sum=620 (2)\", \"tab\": \"General information\", \"score\": \"310.0\"}", - "High School Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Biology - # prompt tokens": "{\"description\": \"min=513.932, mean=513.932, max=513.932, sum=1027.865 (2)\", \"tab\": \"General information\", \"score\": \"513.9322580645161\"}", - "High School Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Chemistry - # eval": "{\"description\": \"min=203, mean=203, max=203, sum=406 (2)\", \"tab\": \"General information\", \"score\": \"203.0\"}", - "High School Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # prompt tokens": "{\"description\": \"min=479.842, mean=479.842, max=479.842, sum=959.685 (2)\", \"tab\": \"General information\", \"score\": \"479.8423645320197\"}", - "High School Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "High School Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # prompt tokens": "{\"description\": \"min=889.39, mean=889.39, max=889.39, sum=1778.78 (2)\", \"tab\": \"General information\", \"score\": \"889.39\"}", - "High School Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School European History - # eval": "{\"description\": \"min=165, mean=165, max=165, sum=330 (2)\", \"tab\": \"General information\", \"score\": \"165.0\"}", - "High School European History - # train": "{\"description\": \"min=2.909, mean=2.909, max=2.909, sum=5.818 (2)\", \"tab\": \"General information\", \"score\": \"2.909090909090909\"}", - "High School European History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # prompt tokens": "{\"description\": \"min=1816.758, mean=1816.758, max=1816.758, sum=3633.515 (2)\", \"tab\": \"General information\", \"score\": \"1816.7575757575758\"}", - "High School European History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Geography - # eval": "{\"description\": \"min=198, mean=198, max=198, sum=396 (2)\", \"tab\": \"General information\", \"score\": \"198.0\"}", - "High School Geography - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Geography - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # prompt tokens": "{\"description\": \"min=400.091, mean=400.091, max=400.091, sum=800.182 (2)\", \"tab\": \"General information\", \"score\": \"400.09090909090907\"}", - "High School Geography - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Government And Politics - # eval": "{\"description\": \"min=193, mean=193, max=193, sum=386 (2)\", \"tab\": \"General information\", \"score\": \"193.0\"}", - "High School Government And Politics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Government And Politics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # prompt tokens": "{\"description\": \"min=482.762, mean=482.762, max=482.762, sum=965.523 (2)\", \"tab\": \"General information\", \"score\": \"482.7616580310881\"}", - "High School Government And Politics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Macroeconomics - # eval": "{\"description\": \"min=390, mean=390, max=390, sum=780 (2)\", \"tab\": \"General information\", \"score\": \"390.0\"}", - "High School Macroeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Macroeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # prompt tokens": "{\"description\": \"min=392.351, mean=392.351, max=392.351, sum=784.703 (2)\", \"tab\": \"General information\", \"score\": \"392.35128205128206\"}", - "High School Macroeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Mathematics - # eval": "{\"description\": \"min=270, mean=270, max=270, sum=540 (2)\", \"tab\": \"General information\", \"score\": \"270.0\"}", - "High School Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # prompt tokens": "{\"description\": \"min=506.689, mean=506.689, max=506.689, sum=1013.378 (2)\", \"tab\": \"General information\", \"score\": \"506.68888888888887\"}", - "High School Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Microeconomics - # eval": "{\"description\": \"min=238, mean=238, max=238, sum=476 (2)\", \"tab\": \"General information\", \"score\": \"238.0\"}", - "High School Microeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Microeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # prompt tokens": "{\"description\": \"min=411.235, mean=411.235, max=411.235, sum=822.471 (2)\", \"tab\": \"General information\", \"score\": \"411.2352941176471\"}", - "High School Microeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Physics - # eval": "{\"description\": \"min=151, mean=151, max=151, sum=302 (2)\", \"tab\": \"General information\", \"score\": \"151.0\"}", - "High School Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # prompt tokens": "{\"description\": \"min=548.728, mean=548.728, max=548.728, sum=1097.457 (2)\", \"tab\": \"General information\", \"score\": \"548.7284768211921\"}", - "High School Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Psychology - # eval": "{\"description\": \"min=545, mean=545, max=545, sum=1090 (2)\", \"tab\": \"General information\", \"score\": \"545.0\"}", - "High School Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # prompt tokens": "{\"description\": \"min=514.793, mean=514.793, max=514.793, sum=1029.585 (2)\", \"tab\": \"General information\", \"score\": \"514.7926605504587\"}", - "High School Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Statistics - # eval": "{\"description\": \"min=216, mean=216, max=216, sum=432 (2)\", \"tab\": \"General information\", \"score\": \"216.0\"}", - "High School Statistics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Statistics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # prompt tokens": "{\"description\": \"min=796.606, mean=796.606, max=796.606, sum=1593.213 (2)\", \"tab\": \"General information\", \"score\": \"796.6064814814815\"}", - "High School Statistics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School US History - # eval": "{\"description\": \"min=204, mean=204, max=204, sum=408 (2)\", \"tab\": \"General information\", \"score\": \"204.0\"}", - "High School US History - # train": "{\"description\": \"min=4, mean=4, max=4, sum=8 (2)\", \"tab\": \"General information\", \"score\": \"4.0\"}", - "High School US History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # prompt tokens": "{\"description\": \"min=1788.387, mean=1788.387, max=1788.387, sum=3576.775 (2)\", \"tab\": \"General information\", \"score\": \"1788.387254901961\"}", - "High School US History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School World History - # eval": "{\"description\": \"min=237, mean=237, max=237, sum=474 (2)\", \"tab\": \"General information\", \"score\": \"237.0\"}", - "High School World History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School World History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # prompt tokens": "{\"description\": \"min=1461.443, mean=1461.443, max=1461.443, sum=2922.886 (2)\", \"tab\": \"General information\", \"score\": \"1461.4430379746836\"}", - "High School World History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"high_school_world_history\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_high_school_world_history\"" - } - } - }, - { - "evaluation_name": "Human Sexuality", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Human Sexuality", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.595, - "details": { - "description": "min=0.595, mean=0.595, max=0.595, sum=1.191 (2)", - "tab": "Accuracy", - "Human Aging - Observed inference time (s)": "{\"description\": \"min=0.568, mean=0.568, max=0.568, sum=1.135 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5676639603926996\"}", - "Human Sexuality - Observed inference time (s)": "{\"description\": \"min=0.627, mean=0.627, max=0.627, sum=1.254 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6270790318496354\"}", - "Human Aging - # eval": "{\"description\": \"min=223, mean=223, max=223, sum=446 (2)\", \"tab\": \"General information\", \"score\": \"223.0\"}", - "Human Aging - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Aging - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Aging - # prompt tokens": "{\"description\": \"min=323.691, mean=323.691, max=323.691, sum=647.381 (2)\", \"tab\": \"General information\", \"score\": \"323.69058295964123\"}", - "Human Aging - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Human Sexuality - # eval": "{\"description\": \"min=131, mean=131, max=131, sum=262 (2)\", \"tab\": \"General information\", \"score\": \"131.0\"}", - "Human Sexuality - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Sexuality - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # prompt tokens": "{\"description\": \"min=355.351, mean=355.351, max=355.351, sum=710.702 (2)\", \"tab\": \"General information\", \"score\": \"355.35114503816794\"}", - "Human Sexuality - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"human_sexuality\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_human_sexuality\"" - } - } - }, - { - "evaluation_name": "International Law", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on International Law", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.612, - "details": { - "description": "min=0.612, mean=0.612, max=0.612, sum=1.223 (2)", - "tab": "Accuracy", - "International Law - Observed inference time (s)": "{\"description\": \"min=1.125, mean=1.125, max=1.125, sum=2.25 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.1249816102429855\"}", - "International Law - # eval": "{\"description\": \"min=121, mean=121, max=121, sum=242 (2)\", \"tab\": \"General information\", \"score\": \"121.0\"}", - "International Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "International Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "International Law - # prompt tokens": "{\"description\": \"min=650.372, mean=650.372, max=650.372, sum=1300.744 (2)\", \"tab\": \"General information\", \"score\": \"650.3719008264463\"}", - "International Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"international_law\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_international_law\"" - } - } - }, - { - "evaluation_name": "Logical Fallacies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Logical Fallacies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.607, - "details": { - "description": "min=0.607, mean=0.607, max=0.607, sum=1.215 (2)", - "tab": "Accuracy", - "Logical Fallacies - Observed inference time (s)": "{\"description\": \"min=0.824, mean=0.824, max=0.824, sum=1.648 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8238252847472582\"}", - "Logical Fallacies - # eval": "{\"description\": \"min=163, mean=163, max=163, sum=326 (2)\", \"tab\": \"General information\", \"score\": \"163.0\"}", - "Logical Fallacies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Logical Fallacies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Logical Fallacies - # prompt tokens": "{\"description\": \"min=458.828, mean=458.828, max=458.828, sum=917.656 (2)\", \"tab\": \"General information\", \"score\": \"458.8282208588957\"}", - "Logical Fallacies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"logical_fallacies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_logical_fallacies\"" - } - } - }, - { - "evaluation_name": "Machine Learning", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Machine Learning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.375, - "details": { - "description": "min=0.375, mean=0.375, max=0.375, sum=0.75 (2)", - "tab": "Accuracy", - "Machine Learning - Observed inference time (s)": "{\"description\": \"min=1.161, mean=1.161, max=1.161, sum=2.321 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.160504766872951\"}", - "Machine Learning - # eval": "{\"description\": \"min=112, mean=112, max=112, sum=224 (2)\", \"tab\": \"General information\", \"score\": \"112.0\"}", - "Machine Learning - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Machine Learning - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Machine Learning - # prompt tokens": "{\"description\": \"min=661.214, mean=661.214, max=661.214, sum=1322.429 (2)\", \"tab\": \"General information\", \"score\": \"661.2142857142857\"}", - "Machine Learning - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"machine_learning\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_machine_learning\"" - } - } - }, - { - "evaluation_name": "Management", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Management", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.689, - "details": { - "description": "min=0.689, mean=0.689, max=0.689, sum=1.379 (2)", - "tab": "Accuracy", - "Management - Observed inference time (s)": "{\"description\": \"min=0.518, mean=0.518, max=0.518, sum=1.035 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5176426901400668\"}", - "Management - # eval": "{\"description\": \"min=103, mean=103, max=103, sum=206 (2)\", \"tab\": \"General information\", \"score\": \"103.0\"}", - "Management - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Management - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Management - # prompt tokens": "{\"description\": \"min=298.049, mean=298.049, max=298.049, sum=596.097 (2)\", \"tab\": \"General information\", \"score\": \"298.0485436893204\"}", - "Management - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"management\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_management\"" - } - } - }, - { - "evaluation_name": "Marketing", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Marketing", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.769, - "details": { - "description": "min=0.769, mean=0.769, max=0.769, sum=1.538 (2)", - "tab": "Accuracy", - "Marketing - Observed inference time (s)": "{\"description\": \"min=0.749, mean=0.749, max=0.749, sum=1.499 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7494234182895758\"}", - "Marketing - # eval": "{\"description\": \"min=234, mean=234, max=234, sum=468 (2)\", \"tab\": \"General information\", \"score\": \"234.0\"}", - "Marketing - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Marketing - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Marketing - # prompt tokens": "{\"description\": \"min=440.103, mean=440.103, max=440.103, sum=880.205 (2)\", \"tab\": \"General information\", \"score\": \"440.1025641025641\"}", - "Marketing - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"marketing\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_marketing\"" - } - } - }, - { - "evaluation_name": "Medical Genetics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Medical Genetics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.56, - "details": { - "description": "min=0.56, mean=0.56, max=0.56, sum=1.12 (2)", - "tab": "Accuracy", - "Medical Genetics - Observed inference time (s)": "{\"description\": \"min=0.56, mean=0.56, max=0.56, sum=1.121 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5603377485275268\"}", - "Medical Genetics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Medical Genetics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Medical Genetics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Medical Genetics - # prompt tokens": "{\"description\": \"min=340.48, mean=340.48, max=340.48, sum=680.96 (2)\", \"tab\": \"General information\", \"score\": \"340.48\"}", - "Medical Genetics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"medical_genetics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_medical_genetics\"" - } - } - }, - { - "evaluation_name": "Miscellaneous", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Miscellaneous", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.734, - "details": { - "description": "min=0.734, mean=0.734, max=0.734, sum=1.469 (2)", - "tab": "Accuracy", - "Miscellaneous - Observed inference time (s)": "{\"description\": \"min=0.533, mean=0.533, max=0.533, sum=1.066 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.533118042452582\"}", - "Miscellaneous - # eval": "{\"description\": \"min=783, mean=783, max=783, sum=1566 (2)\", \"tab\": \"General information\", \"score\": \"783.0\"}", - "Miscellaneous - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Miscellaneous - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Miscellaneous - # prompt tokens": "{\"description\": \"min=320.443, mean=320.443, max=320.443, sum=640.886 (2)\", \"tab\": \"General information\", \"score\": \"320.4431673052363\"}", - "Miscellaneous - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"miscellaneous\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_miscellaneous\"" - } - } - }, - { - "evaluation_name": "Moral Scenarios", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Moral Scenarios", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.335, - "details": { - "description": "min=0.335, mean=0.335, max=0.335, sum=0.67 (2)", - "tab": "Accuracy", - "Moral Disputes - Observed inference time (s)": "{\"description\": \"min=0.845, mean=0.845, max=0.845, sum=1.69 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8448189255819155\"}", - "Moral Scenarios - Observed inference time (s)": "{\"description\": \"min=1.193, mean=1.193, max=1.193, sum=2.387 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.1933270441087265\"}", - "Moral Disputes - # eval": "{\"description\": \"min=346, mean=346, max=346, sum=692 (2)\", \"tab\": \"General information\", \"score\": \"346.0\"}", - "Moral Disputes - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Disputes - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Disputes - # prompt tokens": "{\"description\": \"min=502.243, mean=502.243, max=502.243, sum=1004.486 (2)\", \"tab\": \"General information\", \"score\": \"502.242774566474\"}", - "Moral Disputes - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Moral Scenarios - # eval": "{\"description\": \"min=895, mean=895, max=895, sum=1790 (2)\", \"tab\": \"General information\", \"score\": \"895.0\"}", - "Moral Scenarios - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Scenarios - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # prompt tokens": "{\"description\": \"min=667.861, mean=667.861, max=667.861, sum=1335.723 (2)\", \"tab\": \"General information\", \"score\": \"667.8614525139665\"}", - "Moral Scenarios - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"moral_scenarios\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_moral_scenarios\"" - } - } - }, - { - "evaluation_name": "Nutrition", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Nutrition", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.608, - "details": { - "description": "min=0.608, mean=0.608, max=0.608, sum=1.216 (2)", - "tab": "Accuracy", - "Nutrition - Observed inference time (s)": "{\"description\": \"min=0.99, mean=0.99, max=0.99, sum=1.979 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9895777281592874\"}", - "Nutrition - # eval": "{\"description\": \"min=306, mean=306, max=306, sum=612 (2)\", \"tab\": \"General information\", \"score\": \"306.0\"}", - "Nutrition - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Nutrition - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Nutrition - # prompt tokens": "{\"description\": \"min=579.127, mean=579.127, max=579.127, sum=1158.255 (2)\", \"tab\": \"General information\", \"score\": \"579.1274509803922\"}", - "Nutrition - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"nutrition\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_nutrition\"" - } - } - }, - { - "evaluation_name": "Prehistory", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Prehistory", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.593, - "details": { - "description": "min=0.593, mean=0.593, max=0.593, sum=1.185 (2)", - "tab": "Accuracy", - "Prehistory - Observed inference time (s)": "{\"description\": \"min=0.966, mean=0.966, max=0.966, sum=1.932 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9661886655254128\"}", - "Prehistory - # eval": "{\"description\": \"min=324, mean=324, max=324, sum=648 (2)\", \"tab\": \"General information\", \"score\": \"324.0\"}", - "Prehistory - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Prehistory - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Prehistory - # prompt tokens": "{\"description\": \"min=535.151, mean=535.151, max=535.151, sum=1070.302 (2)\", \"tab\": \"General information\", \"score\": \"535.1512345679013\"}", - "Prehistory - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"prehistory\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_prehistory\"" - } - } - }, - { - "evaluation_name": "Public Relations", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Public Relations", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6, - "details": { - "description": "min=0.6, mean=0.6, max=0.6, sum=1.2 (2)", - "tab": "Accuracy", - "Public Relations - Observed inference time (s)": "{\"description\": \"min=0.763, mean=0.763, max=0.763, sum=1.526 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7631508913907138\"}", - "Public Relations - # eval": "{\"description\": \"min=110, mean=110, max=110, sum=220 (2)\", \"tab\": \"General information\", \"score\": \"110.0\"}", - "Public Relations - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Public Relations - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Public Relations - # prompt tokens": "{\"description\": \"min=422.982, mean=422.982, max=422.982, sum=845.964 (2)\", \"tab\": \"General information\", \"score\": \"422.9818181818182\"}", - "Public Relations - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"public_relations\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_public_relations\"" - } - } - }, - { - "evaluation_name": "Security Studies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Security Studies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.522, - "details": { - "description": "min=0.522, mean=0.522, max=0.522, sum=1.045 (2)", - "tab": "Accuracy", - "Security Studies - Observed inference time (s)": "{\"description\": \"min=2.064, mean=2.064, max=2.064, sum=4.128 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.0640801809271987\"}", - "Security Studies - # eval": "{\"description\": \"min=245, mean=245, max=245, sum=490 (2)\", \"tab\": \"General information\", \"score\": \"245.0\"}", - "Security Studies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Security Studies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Security Studies - # prompt tokens": "{\"description\": \"min=1207.057, mean=1207.057, max=1207.057, sum=2414.114 (2)\", \"tab\": \"General information\", \"score\": \"1207.057142857143\"}", - "Security Studies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"security_studies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_security_studies\"" - } - } - }, - { - "evaluation_name": "Sociology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Sociology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.751, - "details": { - "description": "min=0.751, mean=0.751, max=0.751, sum=1.502 (2)", - "tab": "Accuracy", - "Sociology - Observed inference time (s)": "{\"description\": \"min=0.821, mean=0.821, max=0.821, sum=1.642 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8210354812109648\"}", - "Sociology - # eval": "{\"description\": \"min=201, mean=201, max=201, sum=402 (2)\", \"tab\": \"General information\", \"score\": \"201.0\"}", - "Sociology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Sociology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Sociology - # prompt tokens": "{\"description\": \"min=452.02, mean=452.02, max=452.02, sum=904.04 (2)\", \"tab\": \"General information\", \"score\": \"452.0199004975124\"}", - "Sociology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"sociology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_sociology\"" - } - } - }, - { - "evaluation_name": "Virology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Virology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.452, - "details": { - "description": "min=0.452, mean=0.452, max=0.452, sum=0.904 (2)", - "tab": "Accuracy", - "Virology - Observed inference time (s)": "{\"description\": \"min=0.62, mean=0.62, max=0.62, sum=1.241 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6204164372869285\"}", - "Virology - # eval": "{\"description\": \"min=166, mean=166, max=166, sum=332 (2)\", \"tab\": \"General information\", \"score\": \"166.0\"}", - "Virology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Virology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Virology - # prompt tokens": "{\"description\": \"min=349.584, mean=349.584, max=349.584, sum=699.169 (2)\", \"tab\": \"General information\", \"score\": \"349.5843373493976\"}", - "Virology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"virology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_virology\"" - } - } - }, - { - "evaluation_name": "World Religions", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on World Religions", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.731, - "details": { - "description": "min=0.731, mean=0.731, max=0.731, sum=1.462 (2)", - "tab": "Accuracy", - "World Religions - Observed inference time (s)": "{\"description\": \"min=0.53, mean=0.53, max=0.53, sum=1.06 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5299853595376712\"}", - "World Religions - # eval": "{\"description\": \"min=171, mean=171, max=171, sum=342 (2)\", \"tab\": \"General information\", \"score\": \"171.0\"}", - "World Religions - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "World Religions - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "World Religions - # prompt tokens": "{\"description\": \"min=285.766, mean=285.766, max=285.766, sum=571.532 (2)\", \"tab\": \"General information\", \"score\": \"285.766081871345\"}", - "World Religions - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"world_religions\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_world_religions\"" - } - } - }, - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.196, - "details": { - "description": "", - "tab": "Efficiency" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_olmo-1b-hf.json b/data/models/allenai_olmo-1b-hf.json deleted file mode 100644 index 9ae34088ae0ea09195c6c739c01c78002b8829a2..0000000000000000000000000000000000000000 --- a/data/models/allenai_olmo-1b-hf.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "OLMo-1B-hf", - "id": "allenai/OLMo-1B-hf", - "developer": "allenai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "OlmoForCausalLM", - "params_billions": "1.177" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allenai_OLMo-1B-hf/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2182 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3052 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0174 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2617 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4098 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1174 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_olmo-2-0325-32b-instruct.json b/data/models/allenai_olmo-2-0325-32b-instruct.json deleted file mode 100644 index 550cebca10752dcf4450c98419b61a2aa1d55153..0000000000000000000000000000000000000000 --- a/data/models/allenai_olmo-2-0325-32b-instruct.json +++ /dev/null @@ -1,241 +0,0 @@ -{ - "model_info": { - "name": "OLMo 2 32B Instruct March 2025", - "id": "allenai/olmo-2-0325-32b-instruct", - "developer": "allenai", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_capabilities/allenai_olmo-2-0325-32b-instruct/1774096304.5056698", - "retrieved_timestamp": "1774096304.5056698", - "source_metadata": { - "source_name": "helm_capabilities", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_capabilities", - "evaluation_results": [ - { - "evaluation_name": "Mean score", - "source_data": { - "dataset_name": "helm_capabilities", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "The mean of the scores from all columns.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.475, - "details": { - "description": "", - "tab": "Accuracy", - "Mean score - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"191.7591204277284\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU-Pro", - "source_data": { - "dataset_name": "MMLU-Pro", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on MMLU-Pro", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.414, - "details": { - "description": "min=0.414, mean=0.414, max=0.414, sum=0.414 (1)", - "tab": "Accuracy", - "MMLU-Pro - Observed inference time (s)": "{\"description\": \"min=106.958, mean=106.958, max=106.958, sum=106.958 (1)\", \"tab\": \"Efficiency\", \"score\": \"106.95772108364105\"}", - "MMLU-Pro - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "MMLU-Pro - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - # prompt tokens": "{\"description\": \"min=228.506, mean=228.506, max=228.506, sum=228.506 (1)\", \"tab\": \"General information\", \"score\": \"228.506\"}", - "MMLU-Pro - # output tokens": "{\"description\": \"min=338.34, mean=338.34, max=338.34, sum=338.34 (1)\", \"tab\": \"General information\", \"score\": \"338.34\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"all\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"", - "num_output_tokens": "\"2048\"" - } - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.287, - "details": { - "description": "min=0.287, mean=0.287, max=0.287, sum=0.287 (1)", - "tab": "Accuracy", - "GPQA - Observed inference time (s)": "{\"description\": \"min=161.247, mean=161.247, max=161.247, sum=161.247 (1)\", \"tab\": \"Efficiency\", \"score\": \"161.24673478646127\"}", - "GPQA - # eval": "{\"description\": \"min=446, mean=446, max=446, sum=446 (1)\", \"tab\": \"General information\", \"score\": \"446.0\"}", - "GPQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - truncated": "{\"description\": \"min=0.002, mean=0.002, max=0.002, sum=0.002 (1)\", \"tab\": \"General information\", \"score\": \"0.002242152466367713\"}", - "GPQA - # prompt tokens": "{\"description\": \"min=247.26, mean=247.26, max=247.26, sum=247.26 (1)\", \"tab\": \"General information\", \"score\": \"247.26008968609867\"}", - "GPQA - # output tokens": "{\"description\": \"min=526.352, mean=526.352, max=526.352, sum=526.352 (1)\", \"tab\": \"General information\", \"score\": \"526.3520179372198\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"gpqa_main\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"", - "num_output_tokens": "\"2048\"" - } - } - }, - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "IFEval Strict Acc on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.78, - "details": { - "description": "min=0.78, mean=0.78, max=0.78, sum=0.78 (1)", - "tab": "Accuracy", - "IFEval - Observed inference time (s)": "{\"description\": \"min=78.302, mean=78.302, max=78.302, sum=78.302 (1)\", \"tab\": \"Efficiency\", \"score\": \"78.30223875301382\"}", - "IFEval - # eval": "{\"description\": \"min=541, mean=541, max=541, sum=541 (1)\", \"tab\": \"General information\", \"score\": \"541.0\"}", - "IFEval - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - # prompt tokens": "{\"description\": \"min=46.054, mean=46.054, max=46.054, sum=46.054 (1)\", \"tab\": \"General information\", \"score\": \"46.05360443622921\"}", - "IFEval - # output tokens": "{\"description\": \"min=260.017, mean=260.017, max=260.017, sum=260.017 (1)\", \"tab\": \"General information\", \"score\": \"260.0166358595194\"}" - } - }, - "generation_config": { - "additional_details": { - "num_output_tokens": "\"2048\"" - } - } - }, - { - "evaluation_name": "WildBench", - "source_data": { - "dataset_name": "WildBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "WB Score on WildBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.734, - "details": { - "description": "min=0.734, mean=0.734, max=0.734, sum=0.734 (1)", - "tab": "Accuracy", - "WildBench - Observed inference time (s)": "{\"description\": \"min=333.659, mean=333.659, max=333.659, sum=333.659 (1)\", \"tab\": \"Efficiency\", \"score\": \"333.659037665844\"}", - "WildBench - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "WildBench - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # prompt tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # output tokens": "{\"description\": \"min=764.742, mean=764.742, max=764.742, sum=764.742 (1)\", \"tab\": \"General information\", \"score\": \"764.742\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"v2\"", - "num_output_tokens": "\"2048\"" - } - } - }, - { - "evaluation_name": "Omni-MATH", - "source_data": { - "dataset_name": "Omni-MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Acc on Omni-MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.161, - "details": { - "description": "min=0.161, mean=0.161, max=0.161, sum=0.161 (1)", - "tab": "Accuracy", - "Omni-MATH - Observed inference time (s)": "{\"description\": \"min=278.63, mean=278.63, max=278.63, sum=278.63 (1)\", \"tab\": \"Efficiency\", \"score\": \"278.6298698496819\"}", - "Omni-MATH - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "Omni-MATH - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - truncated": "{\"description\": \"min=0.001, mean=0.001, max=0.001, sum=0.001 (1)\", \"tab\": \"General information\", \"score\": \"0.001\"}", - "Omni-MATH - # prompt tokens": "{\"description\": \"min=108.843, mean=108.843, max=108.843, sum=108.843 (1)\", \"tab\": \"General information\", \"score\": \"108.843\"}", - "Omni-MATH - # output tokens": "{\"description\": \"min=573.483, mean=573.483, max=573.483, sum=573.483 (1)\", \"tab\": \"General information\", \"score\": \"573.483\"}" - } - }, - "generation_config": { - "additional_details": { - "num_output_tokens": "\"2048\"" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_olmo-2-1124-13b-instruct.json b/data/models/allenai_olmo-2-1124-13b-instruct.json deleted file mode 100644 index 60bf3ed062b40b535c39b3ab7518cc525ea28fae..0000000000000000000000000000000000000000 --- a/data/models/allenai_olmo-2-1124-13b-instruct.json +++ /dev/null @@ -1,241 +0,0 @@ -{ - "model_info": { - "name": "OLMo 2 13B Instruct November 2024", - "id": "allenai/olmo-2-1124-13b-instruct", - "developer": "allenai", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_capabilities/allenai_olmo-2-1124-13b-instruct/1774096304.5056698", - "retrieved_timestamp": "1774096304.5056698", - "source_metadata": { - "source_name": "helm_capabilities", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_capabilities", - "evaluation_results": [ - { - "evaluation_name": "Mean score", - "source_data": { - "dataset_name": "helm_capabilities", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "The mean of the scores from all columns.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.44, - "details": { - "description": "", - "tab": "Accuracy", - "Mean score - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"103.93921828652563\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU-Pro", - "source_data": { - "dataset_name": "MMLU-Pro", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on MMLU-Pro", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.31, - "details": { - "description": "min=0.31, mean=0.31, max=0.31, sum=0.31 (1)", - "tab": "Accuracy", - "MMLU-Pro - Observed inference time (s)": "{\"description\": \"min=48.22, mean=48.22, max=48.22, sum=48.22 (1)\", \"tab\": \"Efficiency\", \"score\": \"48.21963578557968\"}", - "MMLU-Pro - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "MMLU-Pro - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - # prompt tokens": "{\"description\": \"min=228.506, mean=228.506, max=228.506, sum=228.506 (1)\", \"tab\": \"General information\", \"score\": \"228.506\"}", - "MMLU-Pro - # output tokens": "{\"description\": \"min=200.755, mean=200.755, max=200.755, sum=200.755 (1)\", \"tab\": \"General information\", \"score\": \"200.755\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"all\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"", - "num_output_tokens": "\"2048\"" - } - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.316, - "details": { - "description": "min=0.316, mean=0.316, max=0.316, sum=0.316 (1)", - "tab": "Accuracy", - "GPQA - Observed inference time (s)": "{\"description\": \"min=44.368, mean=44.368, max=44.368, sum=44.368 (1)\", \"tab\": \"Efficiency\", \"score\": \"44.36780591235567\"}", - "GPQA - # eval": "{\"description\": \"min=446, mean=446, max=446, sum=446 (1)\", \"tab\": \"General information\", \"score\": \"446.0\"}", - "GPQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - truncated": "{\"description\": \"min=0.002, mean=0.002, max=0.002, sum=0.002 (1)\", \"tab\": \"General information\", \"score\": \"0.002242152466367713\"}", - "GPQA - # prompt tokens": "{\"description\": \"min=247.26, mean=247.26, max=247.26, sum=247.26 (1)\", \"tab\": \"General information\", \"score\": \"247.26008968609867\"}", - "GPQA - # output tokens": "{\"description\": \"min=185.419, mean=185.419, max=185.419, sum=185.419 (1)\", \"tab\": \"General information\", \"score\": \"185.41928251121075\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"gpqa_main\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"", - "num_output_tokens": "\"2048\"" - } - } - }, - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "IFEval Strict Acc on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.73, - "details": { - "description": "min=0.73, mean=0.73, max=0.73, sum=0.73 (1)", - "tab": "Accuracy", - "IFEval - Observed inference time (s)": "{\"description\": \"min=71.901, mean=71.901, max=71.901, sum=71.901 (1)\", \"tab\": \"Efficiency\", \"score\": \"71.90055892868536\"}", - "IFEval - # eval": "{\"description\": \"min=541, mean=541, max=541, sum=541 (1)\", \"tab\": \"General information\", \"score\": \"541.0\"}", - "IFEval - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - # prompt tokens": "{\"description\": \"min=46.054, mean=46.054, max=46.054, sum=46.054 (1)\", \"tab\": \"General information\", \"score\": \"46.05360443622921\"}", - "IFEval - # output tokens": "{\"description\": \"min=311.527, mean=311.527, max=311.527, sum=311.527 (1)\", \"tab\": \"General information\", \"score\": \"311.5268022181146\"}" - } - }, - "generation_config": { - "additional_details": { - "num_output_tokens": "\"2048\"" - } - } - }, - { - "evaluation_name": "WildBench", - "source_data": { - "dataset_name": "WildBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "WB Score on WildBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.689, - "details": { - "description": "min=0.689, mean=0.689, max=0.689, sum=0.689 (1)", - "tab": "Accuracy", - "WildBench - Observed inference time (s)": "{\"description\": \"min=194.337, mean=194.337, max=194.337, sum=194.337 (1)\", \"tab\": \"Efficiency\", \"score\": \"194.33703967285157\"}", - "WildBench - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "WildBench - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # prompt tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # output tokens": "{\"description\": \"min=771.135, mean=771.135, max=771.135, sum=771.135 (1)\", \"tab\": \"General information\", \"score\": \"771.135\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"v2\"", - "num_output_tokens": "\"2048\"" - } - } - }, - { - "evaluation_name": "Omni-MATH", - "source_data": { - "dataset_name": "Omni-MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Acc on Omni-MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.156, - "details": { - "description": "min=0.156, mean=0.156, max=0.156, sum=0.156 (1)", - "tab": "Accuracy", - "Omni-MATH - Observed inference time (s)": "{\"description\": \"min=160.871, mean=160.871, max=160.871, sum=160.871 (1)\", \"tab\": \"Efficiency\", \"score\": \"160.87105113315582\"}", - "Omni-MATH - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "Omni-MATH - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - truncated": "{\"description\": \"min=0.001, mean=0.001, max=0.001, sum=0.001 (1)\", \"tab\": \"General information\", \"score\": \"0.001\"}", - "Omni-MATH - # prompt tokens": "{\"description\": \"min=108.843, mean=108.843, max=108.843, sum=108.843 (1)\", \"tab\": \"General information\", \"score\": \"108.843\"}", - "Omni-MATH - # output tokens": "{\"description\": \"min=681.572, mean=681.572, max=681.572, sum=681.572 (1)\", \"tab\": \"General information\", \"score\": \"681.572\"}" - } - }, - "generation_config": { - "additional_details": { - "num_output_tokens": "\"2048\"" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_olmo-2-1124-7b-instruct.json b/data/models/allenai_olmo-2-1124-7b-instruct.json deleted file mode 100644 index e0e35d9599a64716b6e4f8b0473e1856726677e0..0000000000000000000000000000000000000000 --- a/data/models/allenai_olmo-2-1124-7b-instruct.json +++ /dev/null @@ -1,376 +0,0 @@ -{ - "model_info": { - "name": "OLMo 2 7B Instruct November 2024", - "id": "allenai/OLMo-2-1124-7B-Instruct", - "developer": "allenai", - "inference_platform": "unknown", - "additional_details": { - "model_id_aliases": [ - "allenai/olmo-2-1124-7b-instruct" - ] - } - }, - "evaluations": [ - { - "evaluation_id": "helm_capabilities/allenai_olmo-2-1124-7b-instruct/1774096304.5056698", - "retrieved_timestamp": "1774096304.5056698", - "source_metadata": { - "source_name": "helm_capabilities", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_capabilities", - "evaluation_results": [ - { - "evaluation_name": "Mean score", - "source_data": { - "dataset_name": "helm_capabilities", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "The mean of the scores from all columns.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.405, - "details": { - "description": "", - "tab": "Accuracy", - "Mean score - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"164.44917339954657\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU-Pro", - "source_data": { - "dataset_name": "MMLU-Pro", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on MMLU-Pro", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.292, - "details": { - "description": "min=0.292, mean=0.292, max=0.292, sum=0.292 (1)", - "tab": "Accuracy", - "MMLU-Pro - Observed inference time (s)": "{\"description\": \"min=65.565, mean=65.565, max=65.565, sum=65.565 (1)\", \"tab\": \"Efficiency\", \"score\": \"65.56540368175507\"}", - "MMLU-Pro - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "MMLU-Pro - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - # prompt tokens": "{\"description\": \"min=228.506, mean=228.506, max=228.506, sum=228.506 (1)\", \"tab\": \"General information\", \"score\": \"228.506\"}", - "MMLU-Pro - # output tokens": "{\"description\": \"min=265.659, mean=265.659, max=265.659, sum=265.659 (1)\", \"tab\": \"General information\", \"score\": \"265.659\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"all\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"", - "num_output_tokens": "\"2048\"" - } - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.296, - "details": { - "description": "min=0.296, mean=0.296, max=0.296, sum=0.296 (1)", - "tab": "Accuracy", - "GPQA - Observed inference time (s)": "{\"description\": \"min=184.733, mean=184.733, max=184.733, sum=184.733 (1)\", \"tab\": \"Efficiency\", \"score\": \"184.73346061877606\"}", - "GPQA - # eval": "{\"description\": \"min=446, mean=446, max=446, sum=446 (1)\", \"tab\": \"General information\", \"score\": \"446.0\"}", - "GPQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - truncated": "{\"description\": \"min=0.002, mean=0.002, max=0.002, sum=0.002 (1)\", \"tab\": \"General information\", \"score\": \"0.002242152466367713\"}", - "GPQA - # prompt tokens": "{\"description\": \"min=247.26, mean=247.26, max=247.26, sum=247.26 (1)\", \"tab\": \"General information\", \"score\": \"247.26008968609867\"}", - "GPQA - # output tokens": "{\"description\": \"min=381.121, mean=381.121, max=381.121, sum=381.121 (1)\", \"tab\": \"General information\", \"score\": \"381.1210762331838\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"gpqa_main\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"", - "num_output_tokens": "\"2048\"" - } - } - }, - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "IFEval Strict Acc on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.693, - "details": { - "description": "min=0.693, mean=0.693, max=0.693, sum=0.693 (1)", - "tab": "Accuracy", - "IFEval - Observed inference time (s)": "{\"description\": \"min=102.503, mean=102.503, max=102.503, sum=102.503 (1)\", \"tab\": \"Efficiency\", \"score\": \"102.50307150909508\"}", - "IFEval - # eval": "{\"description\": \"min=541, mean=541, max=541, sum=541 (1)\", \"tab\": \"General information\", \"score\": \"541.0\"}", - "IFEval - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - # prompt tokens": "{\"description\": \"min=46.054, mean=46.054, max=46.054, sum=46.054 (1)\", \"tab\": \"General information\", \"score\": \"46.05360443622921\"}", - "IFEval - # output tokens": "{\"description\": \"min=306.706, mean=306.706, max=306.706, sum=306.706 (1)\", \"tab\": \"General information\", \"score\": \"306.70609981515713\"}" - } - }, - "generation_config": { - "additional_details": { - "num_output_tokens": "\"2048\"" - } - } - }, - { - "evaluation_name": "WildBench", - "source_data": { - "dataset_name": "WildBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "WB Score on WildBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.628, - "details": { - "description": "min=0.628, mean=0.628, max=0.628, sum=0.628 (1)", - "tab": "Accuracy", - "WildBench - Observed inference time (s)": "{\"description\": \"min=236.772, mean=236.772, max=236.772, sum=236.772 (1)\", \"tab\": \"Efficiency\", \"score\": \"236.77177815794946\"}", - "WildBench - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "WildBench - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # prompt tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # output tokens": "{\"description\": \"min=768.348, mean=768.348, max=768.348, sum=768.348 (1)\", \"tab\": \"General information\", \"score\": \"768.348\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"v2\"", - "num_output_tokens": "\"2048\"" - } - } - }, - { - "evaluation_name": "Omni-MATH", - "source_data": { - "dataset_name": "Omni-MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Acc on Omni-MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.116, - "details": { - "description": "min=0.116, mean=0.116, max=0.116, sum=0.116 (1)", - "tab": "Accuracy", - "Omni-MATH - Observed inference time (s)": "{\"description\": \"min=232.672, mean=232.672, max=232.672, sum=232.672 (1)\", \"tab\": \"Efficiency\", \"score\": \"232.6721530301571\"}", - "Omni-MATH - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "Omni-MATH - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - truncated": "{\"description\": \"min=0.001, mean=0.001, max=0.001, sum=0.001 (1)\", \"tab\": \"General information\", \"score\": \"0.001\"}", - "Omni-MATH - # prompt tokens": "{\"description\": \"min=108.843, mean=108.843, max=108.843, sum=108.843 (1)\", \"tab\": \"General information\", \"score\": \"108.843\"}", - "Omni-MATH - # output tokens": "{\"description\": \"min=799.769, mean=799.769, max=799.769, sum=799.769 (1)\", \"tab\": \"General information\", \"score\": \"799.769\"}" - } - }, - "generation_config": { - "additional_details": { - "num_output_tokens": "\"2048\"" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "hfopenllm_v2/allenai_OLMo-2-1124-7B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7244 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4022 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1488 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2785 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3508 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2672 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_olmo-7b-hf.json b/data/models/allenai_olmo-7b-hf.json deleted file mode 100644 index adf62072088dd66a489ab53feb04a89beb4b9869..0000000000000000000000000000000000000000 --- a/data/models/allenai_olmo-7b-hf.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "OLMo-7B-hf", - "id": "allenai/OLMo-7B-hf", - "developer": "allenai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "OlmoForCausalLM", - "params_billions": "6.888" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allenai_OLMo-7B-hf/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2719 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3279 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0121 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2727 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3487 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1173 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_olmo-7b-instruct-hf.json b/data/models/allenai_olmo-7b-instruct-hf.json deleted file mode 100644 index e273f16a1a797b43d1e64218ef6972076696070c..0000000000000000000000000000000000000000 --- a/data/models/allenai_olmo-7b-instruct-hf.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "OLMo-7B-Instruct-hf", - "id": "allenai/OLMo-7B-Instruct-hf", - "developer": "allenai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "OlmoForCausalLM", - "params_billions": "7.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allenai_OLMo-7B-Instruct-hf/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3473 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3706 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0136 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.271 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3765 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1785 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_olmo-7b-instruct.json b/data/models/allenai_olmo-7b-instruct.json deleted file mode 100644 index 4bc9083f3b70e67fa46bd4e67841571569572562..0000000000000000000000000000000000000000 --- a/data/models/allenai_olmo-7b-instruct.json +++ /dev/null @@ -1,144 +0,0 @@ -{ - "model_info": { - "name": "allenai/OLMo-7B-Instruct", - "id": "allenai/OLMo-7B-Instruct", - "developer": "allenai", - "additional_details": { - "model_type": "DPO" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/allenai_OLMo-7B-Instruct/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6727 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8966 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5066 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6486 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7168 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5173 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_olmo-7b.json b/data/models/allenai_olmo-7b.json deleted file mode 100644 index 558b8a3217367670571eda407f1002522dd40bcb..0000000000000000000000000000000000000000 --- a/data/models/allenai_olmo-7b.json +++ /dev/null @@ -1,1901 +0,0 @@ -{ - "model_info": { - "name": "OLMo 7B", - "id": "allenai/olmo-7b", - "developer": "allenai", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_lite/allenai_olmo-7b/1774096306.427425", - "retrieved_timestamp": "1774096306.427425", - "source_metadata": { - "source_name": "helm_lite", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_lite", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_lite", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.052, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.6540574282147316\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.597, - "details": { - "description": "min=0.597, mean=0.597, max=0.597, sum=0.597 (1)", - "tab": "Accuracy", - "NarrativeQA - Observed inference time (s)": "{\"description\": \"min=1.032, mean=1.032, max=1.032, sum=1.032 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.0318688553823552\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=1.969, mean=1.969, max=1.969, sum=1.969 (1)\", \"tab\": \"General information\", \"score\": \"1.9690140845070423\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=1691.082, mean=1691.082, max=1691.082, sum=1691.082 (1)\", \"tab\": \"General information\", \"score\": \"1691.081690140845\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (closed-book)", - "source_data": { - "dataset_name": "NaturalQuestions (closed-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (closed-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.259, - "details": { - "description": "min=0.259, mean=0.259, max=0.259, sum=0.259 (1)", - "tab": "Accuracy", - "NaturalQuestions (open-book) - Observed inference time (s)": "{\"description\": \"min=0.942, mean=0.942, max=0.942, sum=0.942 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.9419968054294586\"}", - "NaturalQuestions (closed-book) - Observed inference time (s)": "{\"description\": \"min=0.397, mean=0.397, max=0.397, sum=0.397 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.3968301827907562\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.703, mean=4.703, max=4.703, sum=4.703 (1)\", \"tab\": \"General information\", \"score\": \"4.703\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.037, mean=0.037, max=0.037, sum=0.037 (1)\", \"tab\": \"General information\", \"score\": \"0.037\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1495.001, mean=1495.001, max=1495.001, sum=1495.001 (1)\", \"tab\": \"General information\", \"score\": \"1495.001\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=0.998, mean=0.998, max=0.998, sum=0.998 (1)\", \"tab\": \"General information\", \"score\": \"0.998\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=117.299, mean=117.299, max=117.299, sum=117.299 (1)\", \"tab\": \"General information\", \"score\": \"117.299\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "mode": "\"closedbook\"" - } - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.222, - "details": { - "description": "min=0.222, mean=0.222, max=0.222, sum=0.222 (1)", - "tab": "Accuracy", - "OpenbookQA - Observed inference time (s)": "{\"description\": \"min=0.29, mean=0.29, max=0.29, sum=0.29 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.2902843647003174\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=251.556, mean=251.556, max=251.556, sum=251.556 (1)\", \"tab\": \"General information\", \"score\": \"251.556\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "dataset": "\"openbookqa\"", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.305, - "details": { - "description": "min=0.26, mean=0.305, max=0.38, sum=1.525 (5)", - "tab": "Accuracy", - "MMLU - Observed inference time (s)": "{\"description\": \"min=0.309, mean=0.326, max=0.346, sum=1.629 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.325820258140564\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=358.76, mean=467.936, max=612.798, sum=2339.678 (5)\", \"tab\": \"General information\", \"score\": \"467.935649122807\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"college_chemistry\", \"computer_security\", \"econometrics\", \"us_foreign_policy\"]", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MATH", - "source_data": { - "dataset_name": "MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Equivalent (CoT) on MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.029, - "details": { - "description": "min=0, mean=0.029, max=0.088, sum=0.205 (7)", - "tab": "Accuracy", - "MATH - Observed inference time (s)": "{\"description\": \"min=1.79, mean=2.257, max=2.808, sum=15.8 (7)\", \"tab\": \"Efficiency\", \"score\": \"2.2571195842818583\"}", - "MATH - # eval": "{\"description\": \"min=30, mean=62.429, max=135, sum=437 (7)\", \"tab\": \"General information\", \"score\": \"62.42857142857143\"}", - "MATH - # train": "{\"description\": \"min=3.173, mean=6.976, max=8, sum=48.831 (7)\", \"tab\": \"General information\", \"score\": \"6.9758530942741475\"}", - "MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MATH - # prompt tokens": "{\"description\": \"min=860.23, mean=1111.07, max=1508.423, sum=7777.488 (7)\", \"tab\": \"General information\", \"score\": \"1111.0696790674758\"}", - "MATH - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=7 (7)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"algebra\", \"counting_and_probability\", \"geometry\", \"intermediate_algebra\", \"number_theory\", \"prealgebra\", \"precalculus\"]", - "level": "\"1\"", - "use_official_examples": "\"False\"", - "use_chain_of_thought": "\"True\"" - } - } - }, - { - "evaluation_name": "GSM8K", - "source_data": { - "dataset_name": "GSM8K", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on GSM8K", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.044, - "details": { - "description": "min=0.044, mean=0.044, max=0.044, sum=0.044 (1)", - "tab": "Accuracy", - "GSM8K - Observed inference time (s)": "{\"description\": \"min=2.41, mean=2.41, max=2.41, sum=2.41 (1)\", \"tab\": \"Efficiency\", \"score\": \"2.4104921889305113\"}", - "GSM8K - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "GSM8K - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "GSM8K - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GSM8K - # prompt tokens": "{\"description\": \"min=939.582, mean=939.582, max=939.582, sum=939.582 (1)\", \"tab\": \"General information\", \"score\": \"939.582\"}", - "GSM8K - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "LegalBench", - "source_data": { - "dataset_name": "LegalBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on LegalBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.341, - "details": { - "description": "min=0.158, mean=0.341, max=0.6, sum=1.704 (5)", - "tab": "Accuracy", - "LegalBench - Observed inference time (s)": "{\"description\": \"min=0.368, mean=0.502, max=0.929, sum=2.508 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.5016753114389487\"}", - "LegalBench - # eval": "{\"description\": \"min=95, mean=409.4, max=1000, sum=2047 (5)\", \"tab\": \"General information\", \"score\": \"409.4\"}", - "LegalBench - # train": "{\"description\": \"min=0.298, mean=3.86, max=5, sum=19.298 (5)\", \"tab\": \"General information\", \"score\": \"3.859591836734694\"}", - "LegalBench - truncated": "{\"description\": \"min=0, mean=0.003, max=0.014, sum=0.014 (5)\", \"tab\": \"General information\", \"score\": \"0.002857142857142857\"}", - "LegalBench - # prompt tokens": "{\"description\": \"min=206.779, mean=559.92, max=1493.837, sum=2799.602 (5)\", \"tab\": \"General information\", \"score\": \"559.9203981649337\"}", - "LegalBench - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"abercrombie\", \"corporate_lobbying\", \"function_of_decision_section\", \"international_citizenship_questions\", \"proa\"]" - } - } - }, - { - "evaluation_name": "MedQA", - "source_data": { - "dataset_name": "MedQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MedQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.229, - "details": { - "description": "min=0.229, mean=0.229, max=0.229, sum=0.229 (1)", - "tab": "Accuracy", - "MedQA - Observed inference time (s)": "{\"description\": \"min=0.478, mean=0.478, max=0.478, sum=0.478 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.47797848879698496\"}", - "MedQA - # eval": "{\"description\": \"min=503, mean=503, max=503, sum=503 (1)\", \"tab\": \"General information\", \"score\": \"503.0\"}", - "MedQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MedQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MedQA - # prompt tokens": "{\"description\": \"min=994.588, mean=994.588, max=994.588, sum=994.588 (1)\", \"tab\": \"General information\", \"score\": \"994.5884691848906\"}", - "MedQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WMT 2014", - "source_data": { - "dataset_name": "WMT 2014", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "BLEU-4 on WMT 2014", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.097, - "details": { - "description": "min=0.009, mean=0.097, max=0.157, sum=0.487 (5)", - "tab": "Accuracy", - "WMT 2014 - Observed inference time (s)": "{\"description\": \"min=0.661, mean=0.771, max=0.925, sum=3.855 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.7709201743273374\"}", - "WMT 2014 - # eval": "{\"description\": \"min=503, mean=568.8, max=832, sum=2844 (5)\", \"tab\": \"General information\", \"score\": \"568.8\"}", - "WMT 2014 - # train": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "WMT 2014 - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WMT 2014 - # prompt tokens": "{\"description\": \"min=129.879, mean=144.948, max=167.177, sum=724.741 (5)\", \"tab\": \"General information\", \"score\": \"144.94816676861905\"}", - "WMT 2014 - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "language_pair": "[\"cs-en\", \"de-en\", \"fr-en\", \"hi-en\", \"ru-en\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_mmlu/allenai_olmo-7b/1774096312.00548", - "retrieved_timestamp": "1774096312.00548", - "source_metadata": { - "source_name": "helm_mmlu", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_mmlu", - "evaluation_results": [ - { - "evaluation_name": "MMLU All Subjects", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU All Subjects", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.295, - "details": { - "description": "min=0.22, mean=0.295, max=0.454, sum=33.59 (114)", - "tab": "Accuracy", - "MMLU All Subjects - Observed inference time (s)": "{\"description\": \"min=0.258, mean=0.386, max=0.824, sum=44.021 (114)\", \"tab\": \"Efficiency\", \"score\": \"0.38615337806031275\"}", - "MMLU All Subjects - # eval": "{\"description\": \"min=100, mean=246.351, max=1534, sum=28084 (114)\", \"tab\": \"General information\", \"score\": \"246.35087719298247\"}", - "MMLU All Subjects - # train": "{\"description\": \"min=2.903, mean=4.946, max=5, sum=563.801 (114)\", \"tab\": \"General information\", \"score\": \"4.9456214515982575\"}", - "MMLU All Subjects - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU All Subjects - # prompt tokens": "{\"description\": \"min=285.766, mean=597.867, max=1813.97, sum=68156.839 (114)\", \"tab\": \"General information\", \"score\": \"597.8670097876463\"}", - "MMLU All Subjects - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=114 (114)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_name": "Abstract Algebra", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Abstract Algebra", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.26, - "details": { - "description": "min=0.26, mean=0.26, max=0.26, sum=0.52 (2)", - "tab": "Accuracy", - "Abstract Algebra - Observed inference time (s)": "{\"description\": \"min=0.309, mean=0.309, max=0.309, sum=0.619 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.309316143989563\"}", - "Abstract Algebra - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Abstract Algebra - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Abstract Algebra - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Abstract Algebra - # prompt tokens": "{\"description\": \"min=358.76, mean=358.76, max=358.76, sum=717.52 (2)\", \"tab\": \"General information\", \"score\": \"358.76\"}", - "Abstract Algebra - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"abstract_algebra\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_abstract_algebra\"" - } - } - }, - { - "evaluation_name": "Anatomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Anatomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.222, - "details": { - "description": "min=0.222, mean=0.222, max=0.222, sum=0.444 (2)", - "tab": "Accuracy", - "Anatomy - Observed inference time (s)": "{\"description\": \"min=0.536, mean=0.536, max=0.536, sum=1.072 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5358577339737504\"}", - "Anatomy - # eval": "{\"description\": \"min=135, mean=135, max=135, sum=270 (2)\", \"tab\": \"General information\", \"score\": \"135.0\"}", - "Anatomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Anatomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Anatomy - # prompt tokens": "{\"description\": \"min=352.03, mean=352.03, max=352.03, sum=704.059 (2)\", \"tab\": \"General information\", \"score\": \"352.02962962962965\"}", - "Anatomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"anatomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_anatomy\"" - } - } - }, - { - "evaluation_name": "College Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on College Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.294, - "details": { - "description": "min=0.294, mean=0.294, max=0.294, sum=0.588 (2)", - "tab": "Accuracy", - "College Chemistry - Observed inference time (s)": "{\"description\": \"min=0.346, mean=0.346, max=0.346, sum=0.691 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.34570912599563597\"}", - "College Biology - Observed inference time (s)": "{\"description\": \"min=0.309, mean=0.309, max=0.309, sum=0.619 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.30927823815080857\"}", - "College Computer Science - Observed inference time (s)": "{\"description\": \"min=0.423, mean=0.423, max=0.423, sum=0.847 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.42337616443634035\"}", - "College Mathematics - Observed inference time (s)": "{\"description\": \"min=0.344, mean=0.344, max=0.344, sum=0.687 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.34355913400650023\"}", - "College Medicine - Observed inference time (s)": "{\"description\": \"min=0.324, mean=0.324, max=0.324, sum=0.647 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.32374938237184736\"}", - "College Physics - Observed inference time (s)": "{\"description\": \"min=0.33, mean=0.33, max=0.33, sum=0.66 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3302010788637049\"}", - "College Chemistry - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Chemistry - # prompt tokens": "{\"description\": \"min=535.85, mean=535.85, max=535.85, sum=1071.7 (2)\", \"tab\": \"General information\", \"score\": \"535.85\"}", - "College Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Biology - # eval": "{\"description\": \"min=144, mean=144, max=144, sum=288 (2)\", \"tab\": \"General information\", \"score\": \"144.0\"}", - "College Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # prompt tokens": "{\"description\": \"min=470.319, mean=470.319, max=470.319, sum=940.639 (2)\", \"tab\": \"General information\", \"score\": \"470.31944444444446\"}", - "College Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # prompt tokens": "{\"description\": \"min=842.89, mean=842.89, max=842.89, sum=1685.78 (2)\", \"tab\": \"General information\", \"score\": \"842.89\"}", - "College Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Mathematics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # prompt tokens": "{\"description\": \"min=592.82, mean=592.82, max=592.82, sum=1185.64 (2)\", \"tab\": \"General information\", \"score\": \"592.82\"}", - "College Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Medicine - # eval": "{\"description\": \"min=173, mean=173, max=173, sum=346 (2)\", \"tab\": \"General information\", \"score\": \"173.0\"}", - "College Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # prompt tokens": "{\"description\": \"min=519.376, mean=519.376, max=519.376, sum=1038.751 (2)\", \"tab\": \"General information\", \"score\": \"519.3757225433526\"}", - "College Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Physics - # eval": "{\"description\": \"min=102, mean=102, max=102, sum=204 (2)\", \"tab\": \"General information\", \"score\": \"102.0\"}", - "College Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # prompt tokens": "{\"description\": \"min=476.657, mean=476.657, max=476.657, sum=953.314 (2)\", \"tab\": \"General information\", \"score\": \"476.65686274509807\"}", - "College Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"college_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_college_physics\"" - } - } - }, - { - "evaluation_name": "Computer Security", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Computer Security", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3, - "details": { - "description": "min=0.3, mean=0.3, max=0.3, sum=0.6 (2)", - "tab": "Accuracy", - "Computer Security - Observed inference time (s)": "{\"description\": \"min=0.317, mean=0.317, max=0.317, sum=0.634 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.31721718072891236\"}", - "Computer Security - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Computer Security - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Computer Security - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Computer Security - # prompt tokens": "{\"description\": \"min=388.19, mean=388.19, max=388.19, sum=776.38 (2)\", \"tab\": \"General information\", \"score\": \"388.19\"}", - "Computer Security - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"computer_security\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_computer_security\"" - } - } - }, - { - "evaluation_name": "Econometrics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Econometrics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.325, - "details": { - "description": "min=0.325, mean=0.325, max=0.325, sum=0.649 (2)", - "tab": "Accuracy", - "Econometrics - Observed inference time (s)": "{\"description\": \"min=0.345, mean=0.345, max=0.345, sum=0.69 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.34500646591186523\"}", - "Econometrics - # eval": "{\"description\": \"min=114, mean=114, max=114, sum=228 (2)\", \"tab\": \"General information\", \"score\": \"114.0\"}", - "Econometrics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Econometrics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Econometrics - # prompt tokens": "{\"description\": \"min=612.798, mean=612.798, max=612.798, sum=1225.596 (2)\", \"tab\": \"General information\", \"score\": \"612.7982456140351\"}", - "Econometrics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"econometrics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_econometrics\"" - } - } - }, - { - "evaluation_name": "Global Facts", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Global Facts", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.32, - "details": { - "description": "min=0.32, mean=0.32, max=0.32, sum=0.64 (2)", - "tab": "Accuracy", - "Global Facts - Observed inference time (s)": "{\"description\": \"min=0.316, mean=0.316, max=0.316, sum=0.633 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3163221001625061\"}", - "Global Facts - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Global Facts - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Global Facts - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Global Facts - # prompt tokens": "{\"description\": \"min=400.58, mean=400.58, max=400.58, sum=801.16 (2)\", \"tab\": \"General information\", \"score\": \"400.58\"}", - "Global Facts - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"global_facts\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_global_facts\"" - } - } - }, - { - "evaluation_name": "Jurisprudence", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Jurisprudence", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.25, - "details": { - "description": "min=0.25, mean=0.25, max=0.25, sum=0.5 (2)", - "tab": "Accuracy", - "Jurisprudence - Observed inference time (s)": "{\"description\": \"min=0.306, mean=0.306, max=0.306, sum=0.613 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3064618044429355\"}", - "Jurisprudence - # eval": "{\"description\": \"min=108, mean=108, max=108, sum=216 (2)\", \"tab\": \"General information\", \"score\": \"108.0\"}", - "Jurisprudence - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Jurisprudence - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Jurisprudence - # prompt tokens": "{\"description\": \"min=420.861, mean=420.861, max=420.861, sum=841.722 (2)\", \"tab\": \"General information\", \"score\": \"420.8611111111111\"}", - "Jurisprudence - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"jurisprudence\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_jurisprudence\"" - } - } - }, - { - "evaluation_name": "Philosophy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Philosophy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.325, - "details": { - "description": "min=0.325, mean=0.325, max=0.325, sum=0.65 (2)", - "tab": "Accuracy", - "Philosophy - Observed inference time (s)": "{\"description\": \"min=0.396, mean=0.396, max=0.396, sum=0.792 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.39610295280382946\"}", - "Philosophy - # eval": "{\"description\": \"min=311, mean=311, max=311, sum=622 (2)\", \"tab\": \"General information\", \"score\": \"311.0\"}", - "Philosophy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Philosophy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Philosophy - # prompt tokens": "{\"description\": \"min=345.277, mean=345.277, max=345.277, sum=690.553 (2)\", \"tab\": \"General information\", \"score\": \"345.2765273311897\"}", - "Philosophy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"philosophy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_philosophy\"" - } - } - }, - { - "evaluation_name": "Professional Psychology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Professional Psychology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.232, - "details": { - "description": "min=0.232, mean=0.232, max=0.232, sum=0.464 (2)", - "tab": "Accuracy", - "Professional Medicine - Observed inference time (s)": "{\"description\": \"min=0.5, mean=0.5, max=0.5, sum=1.0 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4999704089234857\"}", - "Professional Accounting - Observed inference time (s)": "{\"description\": \"min=0.346, mean=0.346, max=0.346, sum=0.692 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3458050379516385\"}", - "Professional Law - Observed inference time (s)": "{\"description\": \"min=0.768, mean=0.768, max=0.768, sum=1.537 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7683826767325868\"}", - "Professional Psychology - Observed inference time (s)": "{\"description\": \"min=0.433, mean=0.433, max=0.433, sum=0.865 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.43272479998519997\"}", - "Professional Medicine - # eval": "{\"description\": \"min=272, mean=272, max=272, sum=544 (2)\", \"tab\": \"General information\", \"score\": \"272.0\"}", - "Professional Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Medicine - # prompt tokens": "{\"description\": \"min=1080.882, mean=1080.882, max=1080.882, sum=2161.765 (2)\", \"tab\": \"General information\", \"score\": \"1080.8823529411766\"}", - "Professional Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Accounting - # eval": "{\"description\": \"min=282, mean=282, max=282, sum=564 (2)\", \"tab\": \"General information\", \"score\": \"282.0\"}", - "Professional Accounting - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Accounting - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # prompt tokens": "{\"description\": \"min=660.922, mean=660.922, max=660.922, sum=1321.844 (2)\", \"tab\": \"General information\", \"score\": \"660.9219858156029\"}", - "Professional Accounting - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Law - # eval": "{\"description\": \"min=1534, mean=1534, max=1534, sum=3068 (2)\", \"tab\": \"General information\", \"score\": \"1534.0\"}", - "Professional Law - # train": "{\"description\": \"min=4.997, mean=4.997, max=4.997, sum=9.995 (2)\", \"tab\": \"General information\", \"score\": \"4.9973924380704045\"}", - "Professional Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # prompt tokens": "{\"description\": \"min=1654.433, mean=1654.433, max=1654.433, sum=3308.866 (2)\", \"tab\": \"General information\", \"score\": \"1654.4328552803129\"}", - "Professional Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Psychology - # eval": "{\"description\": \"min=612, mean=612, max=612, sum=1224 (2)\", \"tab\": \"General information\", \"score\": \"612.0\"}", - "Professional Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # prompt tokens": "{\"description\": \"min=590.873, mean=590.873, max=590.873, sum=1181.745 (2)\", \"tab\": \"General information\", \"score\": \"590.8725490196078\"}", - "Professional Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"professional_psychology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_professional_psychology\"" - } - } - }, - { - "evaluation_name": "Us Foreign Policy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Us Foreign Policy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.26, - "details": { - "description": "min=0.26, mean=0.26, max=0.26, sum=0.52 (2)", - "tab": "Accuracy", - "Us Foreign Policy - Observed inference time (s)": "{\"description\": \"min=0.312, mean=0.312, max=0.312, sum=0.624 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.31185237407684324\"}", - "Us Foreign Policy - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Us Foreign Policy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Us Foreign Policy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Us Foreign Policy - # prompt tokens": "{\"description\": \"min=444.08, mean=444.08, max=444.08, sum=888.16 (2)\", \"tab\": \"General information\", \"score\": \"444.08\"}", - "Us Foreign Policy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"us_foreign_policy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_us_foreign_policy\"" - } - } - }, - { - "evaluation_name": "Astronomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Astronomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.342, - "details": { - "description": "min=0.342, mean=0.342, max=0.342, sum=0.684 (2)", - "tab": "Accuracy", - "Astronomy - Observed inference time (s)": "{\"description\": \"min=0.33, mean=0.33, max=0.33, sum=0.66 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3300002766282935\"}", - "Astronomy - # eval": "{\"description\": \"min=152, mean=152, max=152, sum=304 (2)\", \"tab\": \"General information\", \"score\": \"152.0\"}", - "Astronomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Astronomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Astronomy - # prompt tokens": "{\"description\": \"min=598.487, mean=598.487, max=598.487, sum=1196.974 (2)\", \"tab\": \"General information\", \"score\": \"598.4868421052631\"}", - "Astronomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"astronomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_astronomy\"" - } - } - }, - { - "evaluation_name": "Business Ethics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Business Ethics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.24, - "details": { - "description": "min=0.24, mean=0.24, max=0.24, sum=0.48 (2)", - "tab": "Accuracy", - "Business Ethics - Observed inference time (s)": "{\"description\": \"min=0.356, mean=0.356, max=0.356, sum=0.713 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3563597345352173\"}", - "Business Ethics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Business Ethics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Business Ethics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Business Ethics - # prompt tokens": "{\"description\": \"min=585.05, mean=585.05, max=585.05, sum=1170.1 (2)\", \"tab\": \"General information\", \"score\": \"585.05\"}", - "Business Ethics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"business_ethics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_business_ethics\"" - } - } - }, - { - "evaluation_name": "Clinical Knowledge", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Clinical Knowledge", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.26, - "details": { - "description": "min=0.26, mean=0.26, max=0.26, sum=0.521 (2)", - "tab": "Accuracy", - "Clinical Knowledge - Observed inference time (s)": "{\"description\": \"min=0.282, mean=0.282, max=0.282, sum=0.564 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2817675842429107\"}", - "Clinical Knowledge - # eval": "{\"description\": \"min=265, mean=265, max=265, sum=530 (2)\", \"tab\": \"General information\", \"score\": \"265.0\"}", - "Clinical Knowledge - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Clinical Knowledge - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Clinical Knowledge - # prompt tokens": "{\"description\": \"min=401.917, mean=401.917, max=401.917, sum=803.834 (2)\", \"tab\": \"General information\", \"score\": \"401.9169811320755\"}", - "Clinical Knowledge - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"clinical_knowledge\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_clinical_knowledge\"" - } - } - }, - { - "evaluation_name": "Conceptual Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Conceptual Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.319, - "details": { - "description": "min=0.319, mean=0.319, max=0.319, sum=0.638 (2)", - "tab": "Accuracy", - "Conceptual Physics - Observed inference time (s)": "{\"description\": \"min=0.3, mean=0.3, max=0.3, sum=0.601 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3004691002216745\"}", - "Conceptual Physics - # eval": "{\"description\": \"min=235, mean=235, max=235, sum=470 (2)\", \"tab\": \"General information\", \"score\": \"235.0\"}", - "Conceptual Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Conceptual Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Conceptual Physics - # prompt tokens": "{\"description\": \"min=311.311, mean=311.311, max=311.311, sum=622.621 (2)\", \"tab\": \"General information\", \"score\": \"311.31063829787234\"}", - "Conceptual Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"conceptual_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_conceptual_physics\"" - } - } - }, - { - "evaluation_name": "Electrical Engineering", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Electrical Engineering", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.29, - "details": { - "description": "min=0.29, mean=0.29, max=0.29, sum=0.579 (2)", - "tab": "Accuracy", - "Electrical Engineering - Observed inference time (s)": "{\"description\": \"min=0.271, mean=0.271, max=0.271, sum=0.542 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.27095125954726645\"}", - "Electrical Engineering - # eval": "{\"description\": \"min=145, mean=145, max=145, sum=290 (2)\", \"tab\": \"General information\", \"score\": \"145.0\"}", - "Electrical Engineering - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Electrical Engineering - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Electrical Engineering - # prompt tokens": "{\"description\": \"min=424.848, mean=424.848, max=424.848, sum=849.697 (2)\", \"tab\": \"General information\", \"score\": \"424.848275862069\"}", - "Electrical Engineering - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"electrical_engineering\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_electrical_engineering\"" - } - } - }, - { - "evaluation_name": "Elementary Mathematics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Elementary Mathematics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.254, - "details": { - "description": "min=0.254, mean=0.254, max=0.254, sum=0.508 (2)", - "tab": "Accuracy", - "Elementary Mathematics - Observed inference time (s)": "{\"description\": \"min=0.31, mean=0.31, max=0.31, sum=0.62 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3099196644687148\"}", - "Elementary Mathematics - # eval": "{\"description\": \"min=378, mean=378, max=378, sum=756 (2)\", \"tab\": \"General information\", \"score\": \"378.0\"}", - "Elementary Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Elementary Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Elementary Mathematics - # prompt tokens": "{\"description\": \"min=505.071, mean=505.071, max=505.071, sum=1010.143 (2)\", \"tab\": \"General information\", \"score\": \"505.07142857142856\"}", - "Elementary Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"elementary_mathematics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_elementary_mathematics\"" - } - } - }, - { - "evaluation_name": "Formal Logic", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Formal Logic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.278, - "details": { - "description": "min=0.278, mean=0.278, max=0.278, sum=0.556 (2)", - "tab": "Accuracy", - "Formal Logic - Observed inference time (s)": "{\"description\": \"min=0.582, mean=0.582, max=0.582, sum=1.165 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5824837514332363\"}", - "Formal Logic - # eval": "{\"description\": \"min=126, mean=126, max=126, sum=252 (2)\", \"tab\": \"General information\", \"score\": \"126.0\"}", - "Formal Logic - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Formal Logic - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Formal Logic - # prompt tokens": "{\"description\": \"min=653.595, mean=653.595, max=653.595, sum=1307.19 (2)\", \"tab\": \"General information\", \"score\": \"653.5952380952381\"}", - "Formal Logic - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"formal_logic\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_formal_logic\"" - } - } - }, - { - "evaluation_name": "High School World History", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on High School World History", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.253, - "details": { - "description": "min=0.253, mean=0.253, max=0.253, sum=0.506 (2)", - "tab": "Accuracy", - "High School Biology - Observed inference time (s)": "{\"description\": \"min=0.29, mean=0.29, max=0.29, sum=0.58 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.28990614798761183\"}", - "High School Chemistry - Observed inference time (s)": "{\"description\": \"min=0.298, mean=0.298, max=0.298, sum=0.596 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.29780743039887525\"}", - "High School Computer Science - Observed inference time (s)": "{\"description\": \"min=0.427, mean=0.427, max=0.427, sum=0.854 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4271339774131775\"}", - "High School European History - Observed inference time (s)": "{\"description\": \"min=0.824, mean=0.824, max=0.824, sum=1.648 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8240610585068211\"}", - "High School Geography - Observed inference time (s)": "{\"description\": \"min=0.301, mean=0.301, max=0.301, sum=0.603 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.30138304980114256\"}", - "High School Government And Politics - Observed inference time (s)": "{\"description\": \"min=0.327, mean=0.327, max=0.327, sum=0.653 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.32666249472860226\"}", - "High School Macroeconomics - Observed inference time (s)": "{\"description\": \"min=0.304, mean=0.304, max=0.304, sum=0.608 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.30416087615184295\"}", - "High School Mathematics - Observed inference time (s)": "{\"description\": \"min=0.333, mean=0.333, max=0.333, sum=0.666 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3329446854414763\"}", - "High School Microeconomics - Observed inference time (s)": "{\"description\": \"min=0.277, mean=0.277, max=0.277, sum=0.555 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.27732292243412565\"}", - "High School Physics - Observed inference time (s)": "{\"description\": \"min=0.337, mean=0.337, max=0.337, sum=0.674 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3369376612025381\"}", - "High School Psychology - Observed inference time (s)": "{\"description\": \"min=0.295, mean=0.295, max=0.295, sum=0.589 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.294664117830609\"}", - "High School Statistics - Observed inference time (s)": "{\"description\": \"min=0.409, mean=0.409, max=0.409, sum=0.817 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.40864299955191435\"}", - "High School US History - Observed inference time (s)": "{\"description\": \"min=0.816, mean=0.816, max=0.816, sum=1.632 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8157591445773256\"}", - "High School World History - Observed inference time (s)": "{\"description\": \"min=0.672, mean=0.672, max=0.672, sum=1.343 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6715093554323736\"}", - "High School Biology - # eval": "{\"description\": \"min=310, mean=310, max=310, sum=620 (2)\", \"tab\": \"General information\", \"score\": \"310.0\"}", - "High School Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Biology - # prompt tokens": "{\"description\": \"min=513.932, mean=513.932, max=513.932, sum=1027.865 (2)\", \"tab\": \"General information\", \"score\": \"513.9322580645161\"}", - "High School Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Chemistry - # eval": "{\"description\": \"min=203, mean=203, max=203, sum=406 (2)\", \"tab\": \"General information\", \"score\": \"203.0\"}", - "High School Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # prompt tokens": "{\"description\": \"min=479.842, mean=479.842, max=479.842, sum=959.685 (2)\", \"tab\": \"General information\", \"score\": \"479.8423645320197\"}", - "High School Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "High School Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # prompt tokens": "{\"description\": \"min=889.39, mean=889.39, max=889.39, sum=1778.78 (2)\", \"tab\": \"General information\", \"score\": \"889.39\"}", - "High School Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School European History - # eval": "{\"description\": \"min=165, mean=165, max=165, sum=330 (2)\", \"tab\": \"General information\", \"score\": \"165.0\"}", - "High School European History - # train": "{\"description\": \"min=2.903, mean=2.903, max=2.903, sum=5.806 (2)\", \"tab\": \"General information\", \"score\": \"2.903030303030303\"}", - "High School European History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # prompt tokens": "{\"description\": \"min=1813.97, mean=1813.97, max=1813.97, sum=3627.939 (2)\", \"tab\": \"General information\", \"score\": \"1813.969696969697\"}", - "High School European History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Geography - # eval": "{\"description\": \"min=198, mean=198, max=198, sum=396 (2)\", \"tab\": \"General information\", \"score\": \"198.0\"}", - "High School Geography - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Geography - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # prompt tokens": "{\"description\": \"min=400.091, mean=400.091, max=400.091, sum=800.182 (2)\", \"tab\": \"General information\", \"score\": \"400.09090909090907\"}", - "High School Geography - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Government And Politics - # eval": "{\"description\": \"min=193, mean=193, max=193, sum=386 (2)\", \"tab\": \"General information\", \"score\": \"193.0\"}", - "High School Government And Politics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Government And Politics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # prompt tokens": "{\"description\": \"min=482.762, mean=482.762, max=482.762, sum=965.523 (2)\", \"tab\": \"General information\", \"score\": \"482.7616580310881\"}", - "High School Government And Politics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Macroeconomics - # eval": "{\"description\": \"min=390, mean=390, max=390, sum=780 (2)\", \"tab\": \"General information\", \"score\": \"390.0\"}", - "High School Macroeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Macroeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # prompt tokens": "{\"description\": \"min=392.351, mean=392.351, max=392.351, sum=784.703 (2)\", \"tab\": \"General information\", \"score\": \"392.35128205128206\"}", - "High School Macroeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Mathematics - # eval": "{\"description\": \"min=270, mean=270, max=270, sum=540 (2)\", \"tab\": \"General information\", \"score\": \"270.0\"}", - "High School Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # prompt tokens": "{\"description\": \"min=506.689, mean=506.689, max=506.689, sum=1013.378 (2)\", \"tab\": \"General information\", \"score\": \"506.68888888888887\"}", - "High School Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Microeconomics - # eval": "{\"description\": \"min=238, mean=238, max=238, sum=476 (2)\", \"tab\": \"General information\", \"score\": \"238.0\"}", - "High School Microeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Microeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # prompt tokens": "{\"description\": \"min=411.235, mean=411.235, max=411.235, sum=822.471 (2)\", \"tab\": \"General information\", \"score\": \"411.2352941176471\"}", - "High School Microeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Physics - # eval": "{\"description\": \"min=151, mean=151, max=151, sum=302 (2)\", \"tab\": \"General information\", \"score\": \"151.0\"}", - "High School Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # prompt tokens": "{\"description\": \"min=548.728, mean=548.728, max=548.728, sum=1097.457 (2)\", \"tab\": \"General information\", \"score\": \"548.7284768211921\"}", - "High School Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Psychology - # eval": "{\"description\": \"min=545, mean=545, max=545, sum=1090 (2)\", \"tab\": \"General information\", \"score\": \"545.0\"}", - "High School Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # prompt tokens": "{\"description\": \"min=514.793, mean=514.793, max=514.793, sum=1029.585 (2)\", \"tab\": \"General information\", \"score\": \"514.7926605504587\"}", - "High School Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Statistics - # eval": "{\"description\": \"min=216, mean=216, max=216, sum=432 (2)\", \"tab\": \"General information\", \"score\": \"216.0\"}", - "High School Statistics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Statistics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # prompt tokens": "{\"description\": \"min=796.606, mean=796.606, max=796.606, sum=1593.213 (2)\", \"tab\": \"General information\", \"score\": \"796.6064814814815\"}", - "High School Statistics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School US History - # eval": "{\"description\": \"min=204, mean=204, max=204, sum=408 (2)\", \"tab\": \"General information\", \"score\": \"204.0\"}", - "High School US History - # train": "{\"description\": \"min=4, mean=4, max=4, sum=8 (2)\", \"tab\": \"General information\", \"score\": \"4.0\"}", - "High School US History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # prompt tokens": "{\"description\": \"min=1788.387, mean=1788.387, max=1788.387, sum=3576.775 (2)\", \"tab\": \"General information\", \"score\": \"1788.387254901961\"}", - "High School US History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School World History - # eval": "{\"description\": \"min=237, mean=237, max=237, sum=474 (2)\", \"tab\": \"General information\", \"score\": \"237.0\"}", - "High School World History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School World History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # prompt tokens": "{\"description\": \"min=1461.443, mean=1461.443, max=1461.443, sum=2922.886 (2)\", \"tab\": \"General information\", \"score\": \"1461.4430379746836\"}", - "High School World History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"high_school_world_history\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_high_school_world_history\"" - } - } - }, - { - "evaluation_name": "Human Sexuality", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Human Sexuality", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.267, - "details": { - "description": "min=0.267, mean=0.267, max=0.267, sum=0.534 (2)", - "tab": "Accuracy", - "Human Aging - Observed inference time (s)": "{\"description\": \"min=0.27, mean=0.27, max=0.27, sum=0.54 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2699183316508751\"}", - "Human Sexuality - Observed inference time (s)": "{\"description\": \"min=0.552, mean=0.552, max=0.552, sum=1.104 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5521998168857953\"}", - "Human Aging - # eval": "{\"description\": \"min=223, mean=223, max=223, sum=446 (2)\", \"tab\": \"General information\", \"score\": \"223.0\"}", - "Human Aging - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Aging - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Aging - # prompt tokens": "{\"description\": \"min=323.691, mean=323.691, max=323.691, sum=647.381 (2)\", \"tab\": \"General information\", \"score\": \"323.69058295964123\"}", - "Human Aging - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Human Sexuality - # eval": "{\"description\": \"min=131, mean=131, max=131, sum=262 (2)\", \"tab\": \"General information\", \"score\": \"131.0\"}", - "Human Sexuality - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Sexuality - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # prompt tokens": "{\"description\": \"min=355.351, mean=355.351, max=355.351, sum=710.702 (2)\", \"tab\": \"General information\", \"score\": \"355.35114503816794\"}", - "Human Sexuality - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"human_sexuality\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_human_sexuality\"" - } - } - }, - { - "evaluation_name": "International Law", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on International Law", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.306, - "details": { - "description": "min=0.306, mean=0.306, max=0.306, sum=0.612 (2)", - "tab": "Accuracy", - "International Law - Observed inference time (s)": "{\"description\": \"min=0.326, mean=0.326, max=0.326, sum=0.652 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3259233679653199\"}", - "International Law - # eval": "{\"description\": \"min=121, mean=121, max=121, sum=242 (2)\", \"tab\": \"General information\", \"score\": \"121.0\"}", - "International Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "International Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "International Law - # prompt tokens": "{\"description\": \"min=650.372, mean=650.372, max=650.372, sum=1300.744 (2)\", \"tab\": \"General information\", \"score\": \"650.3719008264463\"}", - "International Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"international_law\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_international_law\"" - } - } - }, - { - "evaluation_name": "Logical Fallacies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Logical Fallacies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.264, - "details": { - "description": "min=0.264, mean=0.264, max=0.264, sum=0.528 (2)", - "tab": "Accuracy", - "Logical Fallacies - Observed inference time (s)": "{\"description\": \"min=0.332, mean=0.332, max=0.332, sum=0.665 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3324835944029451\"}", - "Logical Fallacies - # eval": "{\"description\": \"min=163, mean=163, max=163, sum=326 (2)\", \"tab\": \"General information\", \"score\": \"163.0\"}", - "Logical Fallacies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Logical Fallacies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Logical Fallacies - # prompt tokens": "{\"description\": \"min=458.828, mean=458.828, max=458.828, sum=917.656 (2)\", \"tab\": \"General information\", \"score\": \"458.8282208588957\"}", - "Logical Fallacies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"logical_fallacies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_logical_fallacies\"" - } - } - }, - { - "evaluation_name": "Machine Learning", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Machine Learning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.286, - "details": { - "description": "min=0.286, mean=0.286, max=0.286, sum=0.571 (2)", - "tab": "Accuracy", - "Machine Learning - Observed inference time (s)": "{\"description\": \"min=0.352, mean=0.352, max=0.352, sum=0.704 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3520317098924092\"}", - "Machine Learning - # eval": "{\"description\": \"min=112, mean=112, max=112, sum=224 (2)\", \"tab\": \"General information\", \"score\": \"112.0\"}", - "Machine Learning - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Machine Learning - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Machine Learning - # prompt tokens": "{\"description\": \"min=661.214, mean=661.214, max=661.214, sum=1322.429 (2)\", \"tab\": \"General information\", \"score\": \"661.2142857142857\"}", - "Machine Learning - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"machine_learning\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_machine_learning\"" - } - } - }, - { - "evaluation_name": "Management", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Management", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.272, - "details": { - "description": "min=0.272, mean=0.272, max=0.272, sum=0.544 (2)", - "tab": "Accuracy", - "Management - Observed inference time (s)": "{\"description\": \"min=0.306, mean=0.306, max=0.306, sum=0.613 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3064361937995096\"}", - "Management - # eval": "{\"description\": \"min=103, mean=103, max=103, sum=206 (2)\", \"tab\": \"General information\", \"score\": \"103.0\"}", - "Management - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Management - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Management - # prompt tokens": "{\"description\": \"min=298.049, mean=298.049, max=298.049, sum=596.097 (2)\", \"tab\": \"General information\", \"score\": \"298.0485436893204\"}", - "Management - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"management\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_management\"" - } - } - }, - { - "evaluation_name": "Marketing", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Marketing", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.269, - "details": { - "description": "min=0.269, mean=0.269, max=0.269, sum=0.538 (2)", - "tab": "Accuracy", - "Marketing - Observed inference time (s)": "{\"description\": \"min=0.311, mean=0.311, max=0.311, sum=0.622 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3111040826536651\"}", - "Marketing - # eval": "{\"description\": \"min=234, mean=234, max=234, sum=468 (2)\", \"tab\": \"General information\", \"score\": \"234.0\"}", - "Marketing - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Marketing - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Marketing - # prompt tokens": "{\"description\": \"min=440.103, mean=440.103, max=440.103, sum=880.205 (2)\", \"tab\": \"General information\", \"score\": \"440.1025641025641\"}", - "Marketing - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"marketing\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_marketing\"" - } - } - }, - { - "evaluation_name": "Medical Genetics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Medical Genetics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.28, - "details": { - "description": "min=0.28, mean=0.28, max=0.28, sum=0.56 (2)", - "tab": "Accuracy", - "Medical Genetics - Observed inference time (s)": "{\"description\": \"min=0.258, mean=0.258, max=0.258, sum=0.516 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2580227541923523\"}", - "Medical Genetics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Medical Genetics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Medical Genetics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Medical Genetics - # prompt tokens": "{\"description\": \"min=340.48, mean=340.48, max=340.48, sum=680.96 (2)\", \"tab\": \"General information\", \"score\": \"340.48\"}", - "Medical Genetics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"medical_genetics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_medical_genetics\"" - } - } - }, - { - "evaluation_name": "Miscellaneous", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Miscellaneous", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.292, - "details": { - "description": "min=0.292, mean=0.292, max=0.292, sum=0.585 (2)", - "tab": "Accuracy", - "Miscellaneous - Observed inference time (s)": "{\"description\": \"min=0.342, mean=0.342, max=0.342, sum=0.684 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3421932640051324\"}", - "Miscellaneous - # eval": "{\"description\": \"min=783, mean=783, max=783, sum=1566 (2)\", \"tab\": \"General information\", \"score\": \"783.0\"}", - "Miscellaneous - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Miscellaneous - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Miscellaneous - # prompt tokens": "{\"description\": \"min=320.443, mean=320.443, max=320.443, sum=640.886 (2)\", \"tab\": \"General information\", \"score\": \"320.4431673052363\"}", - "Miscellaneous - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"miscellaneous\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_miscellaneous\"" - } - } - }, - { - "evaluation_name": "Moral Scenarios", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Moral Scenarios", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.265, - "details": { - "description": "min=0.265, mean=0.265, max=0.265, sum=0.53 (2)", - "tab": "Accuracy", - "Moral Disputes - Observed inference time (s)": "{\"description\": \"min=0.395, mean=0.395, max=0.395, sum=0.791 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.39545129627161635\"}", - "Moral Scenarios - Observed inference time (s)": "{\"description\": \"min=0.36, mean=0.36, max=0.36, sum=0.72 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3597933335011232\"}", - "Moral Disputes - # eval": "{\"description\": \"min=346, mean=346, max=346, sum=692 (2)\", \"tab\": \"General information\", \"score\": \"346.0\"}", - "Moral Disputes - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Disputes - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Disputes - # prompt tokens": "{\"description\": \"min=502.243, mean=502.243, max=502.243, sum=1004.486 (2)\", \"tab\": \"General information\", \"score\": \"502.242774566474\"}", - "Moral Disputes - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Moral Scenarios - # eval": "{\"description\": \"min=895, mean=895, max=895, sum=1790 (2)\", \"tab\": \"General information\", \"score\": \"895.0\"}", - "Moral Scenarios - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Scenarios - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # prompt tokens": "{\"description\": \"min=667.861, mean=667.861, max=667.861, sum=1335.723 (2)\", \"tab\": \"General information\", \"score\": \"667.8614525139665\"}", - "Moral Scenarios - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"moral_scenarios\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_moral_scenarios\"" - } - } - }, - { - "evaluation_name": "Nutrition", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Nutrition", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.34, - "details": { - "description": "min=0.34, mean=0.34, max=0.34, sum=0.68 (2)", - "tab": "Accuracy", - "Nutrition - Observed inference time (s)": "{\"description\": \"min=0.451, mean=0.451, max=0.451, sum=0.902 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.45079101612365324\"}", - "Nutrition - # eval": "{\"description\": \"min=306, mean=306, max=306, sum=612 (2)\", \"tab\": \"General information\", \"score\": \"306.0\"}", - "Nutrition - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Nutrition - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Nutrition - # prompt tokens": "{\"description\": \"min=579.127, mean=579.127, max=579.127, sum=1158.255 (2)\", \"tab\": \"General information\", \"score\": \"579.1274509803922\"}", - "Nutrition - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"nutrition\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_nutrition\"" - } - } - }, - { - "evaluation_name": "Prehistory", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Prehistory", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.318, - "details": { - "description": "min=0.318, mean=0.318, max=0.318, sum=0.636 (2)", - "tab": "Accuracy", - "Prehistory - Observed inference time (s)": "{\"description\": \"min=0.328, mean=0.328, max=0.328, sum=0.656 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.32820526979587694\"}", - "Prehistory - # eval": "{\"description\": \"min=324, mean=324, max=324, sum=648 (2)\", \"tab\": \"General information\", \"score\": \"324.0\"}", - "Prehistory - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Prehistory - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Prehistory - # prompt tokens": "{\"description\": \"min=535.151, mean=535.151, max=535.151, sum=1070.302 (2)\", \"tab\": \"General information\", \"score\": \"535.1512345679013\"}", - "Prehistory - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"prehistory\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_prehistory\"" - } - } - }, - { - "evaluation_name": "Public Relations", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Public Relations", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.345, - "details": { - "description": "min=0.345, mean=0.345, max=0.345, sum=0.691 (2)", - "tab": "Accuracy", - "Public Relations - Observed inference time (s)": "{\"description\": \"min=0.285, mean=0.285, max=0.285, sum=0.571 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.28533268625086006\"}", - "Public Relations - # eval": "{\"description\": \"min=110, mean=110, max=110, sum=220 (2)\", \"tab\": \"General information\", \"score\": \"110.0\"}", - "Public Relations - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Public Relations - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Public Relations - # prompt tokens": "{\"description\": \"min=422.982, mean=422.982, max=422.982, sum=845.964 (2)\", \"tab\": \"General information\", \"score\": \"422.9818181818182\"}", - "Public Relations - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"public_relations\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_public_relations\"" - } - } - }, - { - "evaluation_name": "Security Studies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Security Studies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.408, - "details": { - "description": "min=0.408, mean=0.408, max=0.408, sum=0.816 (2)", - "tab": "Accuracy", - "Security Studies - Observed inference time (s)": "{\"description\": \"min=0.551, mean=0.551, max=0.551, sum=1.102 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5510748113904681\"}", - "Security Studies - # eval": "{\"description\": \"min=245, mean=245, max=245, sum=490 (2)\", \"tab\": \"General information\", \"score\": \"245.0\"}", - "Security Studies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Security Studies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Security Studies - # prompt tokens": "{\"description\": \"min=1207.057, mean=1207.057, max=1207.057, sum=2414.114 (2)\", \"tab\": \"General information\", \"score\": \"1207.057142857143\"}", - "Security Studies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"security_studies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_security_studies\"" - } - } - }, - { - "evaluation_name": "Sociology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Sociology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.383, - "details": { - "description": "min=0.383, mean=0.383, max=0.383, sum=0.766 (2)", - "tab": "Accuracy", - "Sociology - Observed inference time (s)": "{\"description\": \"min=0.293, mean=0.293, max=0.293, sum=0.586 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2929653884166509\"}", - "Sociology - # eval": "{\"description\": \"min=201, mean=201, max=201, sum=402 (2)\", \"tab\": \"General information\", \"score\": \"201.0\"}", - "Sociology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Sociology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Sociology - # prompt tokens": "{\"description\": \"min=452.02, mean=452.02, max=452.02, sum=904.04 (2)\", \"tab\": \"General information\", \"score\": \"452.0199004975124\"}", - "Sociology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"sociology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_sociology\"" - } - } - }, - { - "evaluation_name": "Virology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Virology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.416, - "details": { - "description": "min=0.416, mean=0.416, max=0.416, sum=0.831 (2)", - "tab": "Accuracy", - "Virology - Observed inference time (s)": "{\"description\": \"min=0.492, mean=0.492, max=0.492, sum=0.983 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4916250992970294\"}", - "Virology - # eval": "{\"description\": \"min=166, mean=166, max=166, sum=332 (2)\", \"tab\": \"General information\", \"score\": \"166.0\"}", - "Virology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Virology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Virology - # prompt tokens": "{\"description\": \"min=349.584, mean=349.584, max=349.584, sum=699.169 (2)\", \"tab\": \"General information\", \"score\": \"349.5843373493976\"}", - "Virology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"virology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_virology\"" - } - } - }, - { - "evaluation_name": "World Religions", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on World Religions", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.234, - "details": { - "description": "min=0.234, mean=0.234, max=0.234, sum=0.468 (2)", - "tab": "Accuracy", - "World Religions - Observed inference time (s)": "{\"description\": \"min=0.503, mean=0.503, max=0.503, sum=1.007 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5034504368988394\"}", - "World Religions - # eval": "{\"description\": \"min=171, mean=171, max=171, sum=342 (2)\", \"tab\": \"General information\", \"score\": \"171.0\"}", - "World Religions - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "World Religions - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "World Religions - # prompt tokens": "{\"description\": \"min=285.766, mean=285.766, max=285.766, sum=571.532 (2)\", \"tab\": \"General information\", \"score\": \"285.766081871345\"}", - "World Religions - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"world_religions\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_world_religions\"" - } - } - }, - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.68, - "details": { - "description": "", - "tab": "Efficiency" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_olmoe-1b-7b-0125-instruct.json b/data/models/allenai_olmoe-1b-7b-0125-instruct.json deleted file mode 100644 index c1a2fda1b061a03a3b79f7cf7e1a3766f005438e..0000000000000000000000000000000000000000 --- a/data/models/allenai_olmoe-1b-7b-0125-instruct.json +++ /dev/null @@ -1,376 +0,0 @@ -{ - "model_info": { - "name": "OLMoE 1B-7B Instruct January 2025", - "id": "allenai/OLMoE-1B-7B-0125-Instruct", - "developer": "allenai", - "inference_platform": "unknown", - "additional_details": { - "model_id_aliases": [ - "allenai/olmoe-1b-7b-0125-instruct" - ] - } - }, - "evaluations": [ - { - "evaluation_id": "helm_capabilities/allenai_olmoe-1b-7b-0125-instruct/1774096304.5056698", - "retrieved_timestamp": "1774096304.5056698", - "source_metadata": { - "source_name": "helm_capabilities", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_capabilities", - "evaluation_results": [ - { - "evaluation_name": "Mean score", - "source_data": { - "dataset_name": "helm_capabilities", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "The mean of the scores from all columns.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.332, - "details": { - "description": "", - "tab": "Accuracy", - "Mean score - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"449.11527986486544\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU-Pro", - "source_data": { - "dataset_name": "MMLU-Pro", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on MMLU-Pro", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.169, - "details": { - "description": "min=0.169, mean=0.169, max=0.169, sum=0.169 (1)", - "tab": "Accuracy", - "MMLU-Pro - Observed inference time (s)": "{\"description\": \"min=226.84, mean=226.84, max=226.84, sum=226.84 (1)\", \"tab\": \"Efficiency\", \"score\": \"226.84002213978766\"}", - "MMLU-Pro - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "MMLU-Pro - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - # prompt tokens": "{\"description\": \"min=231.403, mean=231.403, max=231.403, sum=231.403 (1)\", \"tab\": \"General information\", \"score\": \"231.403\"}", - "MMLU-Pro - # output tokens": "{\"description\": \"min=237.89, mean=237.89, max=237.89, sum=237.89 (1)\", \"tab\": \"General information\", \"score\": \"237.89\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"all\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"", - "num_output_tokens": "\"2048\"" - } - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.22, - "details": { - "description": "min=0.22, mean=0.22, max=0.22, sum=0.22 (1)", - "tab": "Accuracy", - "GPQA - Observed inference time (s)": "{\"description\": \"min=263.918, mean=263.918, max=263.918, sum=263.918 (1)\", \"tab\": \"Efficiency\", \"score\": \"263.9177615305768\"}", - "GPQA - # eval": "{\"description\": \"min=446, mean=446, max=446, sum=446 (1)\", \"tab\": \"General information\", \"score\": \"446.0\"}", - "GPQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - truncated": "{\"description\": \"min=0.002, mean=0.002, max=0.002, sum=0.002 (1)\", \"tab\": \"General information\", \"score\": \"0.002242152466367713\"}", - "GPQA - # prompt tokens": "{\"description\": \"min=249.803, mean=249.803, max=249.803, sum=249.803 (1)\", \"tab\": \"General information\", \"score\": \"249.80269058295963\"}", - "GPQA - # output tokens": "{\"description\": \"min=302.475, mean=302.475, max=302.475, sum=302.475 (1)\", \"tab\": \"General information\", \"score\": \"302.47533632286996\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"gpqa_main\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"", - "num_output_tokens": "\"2048\"" - } - } - }, - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "IFEval Strict Acc on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.628, - "details": { - "description": "min=0.628, mean=0.628, max=0.628, sum=0.628 (1)", - "tab": "Accuracy", - "IFEval - Observed inference time (s)": "{\"description\": \"min=437.953, mean=437.953, max=437.953, sum=437.953 (1)\", \"tab\": \"Efficiency\", \"score\": \"437.95291065332407\"}", - "IFEval - # eval": "{\"description\": \"min=541, mean=541, max=541, sum=541 (1)\", \"tab\": \"General information\", \"score\": \"541.0\"}", - "IFEval - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - # prompt tokens": "{\"description\": \"min=47.782, mean=47.782, max=47.782, sum=47.782 (1)\", \"tab\": \"General information\", \"score\": \"47.781885397412196\"}", - "IFEval - # output tokens": "{\"description\": \"min=432.808, mean=432.808, max=432.808, sum=432.808 (1)\", \"tab\": \"General information\", \"score\": \"432.80776340110907\"}" - } - }, - "generation_config": { - "additional_details": { - "num_output_tokens": "\"2048\"" - } - } - }, - { - "evaluation_name": "WildBench", - "source_data": { - "dataset_name": "WildBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "WB Score on WildBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.551, - "details": { - "description": "min=0.551, mean=0.551, max=0.551, sum=0.551 (1)", - "tab": "Accuracy", - "WildBench - Observed inference time (s)": "{\"description\": \"min=915.237, mean=915.237, max=915.237, sum=915.237 (1)\", \"tab\": \"Efficiency\", \"score\": \"915.2368009176254\"}", - "WildBench - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "WildBench - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # prompt tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # output tokens": "{\"description\": \"min=972.482, mean=972.482, max=972.482, sum=972.482 (1)\", \"tab\": \"General information\", \"score\": \"972.482\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"v2\"", - "num_output_tokens": "\"2048\"" - } - } - }, - { - "evaluation_name": "Omni-MATH", - "source_data": { - "dataset_name": "Omni-MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Acc on Omni-MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.093, - "details": { - "description": "min=0.093, mean=0.093, max=0.093, sum=0.093 (1)", - "tab": "Accuracy", - "Omni-MATH - Observed inference time (s)": "{\"description\": \"min=401.629, mean=401.629, max=401.629, sum=401.629 (1)\", \"tab\": \"Efficiency\", \"score\": \"401.62890408301354\"}", - "Omni-MATH - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "Omni-MATH - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - truncated": "{\"description\": \"min=0.001, mean=0.001, max=0.001, sum=0.001 (1)\", \"tab\": \"General information\", \"score\": \"0.001\"}", - "Omni-MATH - # prompt tokens": "{\"description\": \"min=110.864, mean=110.864, max=110.864, sum=110.864 (1)\", \"tab\": \"General information\", \"score\": \"110.864\"}", - "Omni-MATH - # output tokens": "{\"description\": \"min=442.229, mean=442.229, max=442.229, sum=442.229 (1)\", \"tab\": \"General information\", \"score\": \"442.229\"}" - } - }, - "generation_config": { - "additional_details": { - "num_output_tokens": "\"2048\"" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "hfopenllm_v2/allenai_OLMoE-1B-7B-0125-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6757 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3825 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0899 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2601 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3636 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1915 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_olmoe-1b-7b-0924-instruct.json b/data/models/allenai_olmoe-1b-7b-0924-instruct.json deleted file mode 100644 index 2cf74d45f3116ecf23f0da40a864ab68d91ff736..0000000000000000000000000000000000000000 --- a/data/models/allenai_olmoe-1b-7b-0924-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "OLMoE-1B-7B-0924-Instruct", - "id": "allenai/OLMoE-1B-7B-0924-Instruct", - "developer": "allenai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "OlmoeForCausalLM", - "params_billions": "6.919" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allenai_OLMoE-1B-7B-0924-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4667 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3902 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0279 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2676 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3848 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1876 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_olmoe-1b-7b-0924.json b/data/models/allenai_olmoe-1b-7b-0924.json deleted file mode 100644 index b855822308ce14283d76f646986b597ef6055186..0000000000000000000000000000000000000000 --- a/data/models/allenai_olmoe-1b-7b-0924.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "OLMoE-1B-7B-0924", - "id": "allenai/OLMoE-1B-7B-0924", - "developer": "allenai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "OlmoeForCausalLM", - "params_billions": "6.919" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allenai_OLMoE-1B-7B-0924/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2185 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3393 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0166 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2475 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3488 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.174 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-reward_modeling__1__1739590997.json b/data/models/allenai_open_instruct_dev-reward_modeling__1__1739590997.json deleted file mode 100644 index 785d549710dbe259afc9601a551f9ac8e2bd970f..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-reward_modeling__1__1739590997.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-reward_modeling__1__1739590997", - "id": "allenai/open_instruct_dev-reward_modeling__1__1739590997", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-reward_modeling__1__1739590997/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6004 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7032 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.375 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.623 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7867 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.598 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5165 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-reward_modeling__1__1739871066.json b/data/models/allenai_open_instruct_dev-reward_modeling__1__1739871066.json deleted file mode 100644 index 1717740eff4c9a215bb6d763b8c6c40893a53338..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-reward_modeling__1__1739871066.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-reward_modeling__1__1739871066", - "id": "allenai/open_instruct_dev-reward_modeling__1__1739871066", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-reward_modeling__1__1739871066/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6012 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6989 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.425 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6284 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7978 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.604 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4527 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-reward_modeling__1__1739925892.json b/data/models/allenai_open_instruct_dev-reward_modeling__1__1739925892.json deleted file mode 100644 index ba6d8ea0c0e148e758a2b85b803838e4ab0e9d57..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-reward_modeling__1__1739925892.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-reward_modeling__1__1739925892", - "id": "allenai/open_instruct_dev-reward_modeling__1__1739925892", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-reward_modeling__1__1739925892/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6345 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7432 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3563 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.623 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8111 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7131 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5606 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-reward_modeling__1__1739943850.json b/data/models/allenai_open_instruct_dev-reward_modeling__1__1739943850.json deleted file mode 100644 index 9903ae2bb96fa63fd4899d7f856e0faae2cd23cf..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-reward_modeling__1__1739943850.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-reward_modeling__1__1739943850", - "id": "allenai/open_instruct_dev-reward_modeling__1__1739943850", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-reward_modeling__1__1739943850/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4978 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5726 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3125 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5191 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6489 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6222 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3114 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-reward_modeling__1__1739943881.json b/data/models/allenai_open_instruct_dev-reward_modeling__1__1739943881.json deleted file mode 100644 index 30723efb419e94262010fd1990f31dd0dd7b37cf..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-reward_modeling__1__1739943881.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-reward_modeling__1__1739943881", - "id": "allenai/open_instruct_dev-reward_modeling__1__1739943881", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-reward_modeling__1__1739943881/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5998 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7032 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3187 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5792 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8222 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6727 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5025 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-reward_modeling__1__1739943972.json b/data/models/allenai_open_instruct_dev-reward_modeling__1__1739943972.json deleted file mode 100644 index 332823efe4bab5d092111edecafbd856485ee37e..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-reward_modeling__1__1739943972.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-reward_modeling__1__1739943972", - "id": "allenai/open_instruct_dev-reward_modeling__1__1739943972", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-reward_modeling__1__1739943972/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5289 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6168 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.375 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5738 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6844 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5657 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3577 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-reward_modeling__1__1739957701.json b/data/models/allenai_open_instruct_dev-reward_modeling__1__1739957701.json deleted file mode 100644 index 7509c565c13497d8028b84b16aa6b4a1d5667348..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-reward_modeling__1__1739957701.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-reward_modeling__1__1739957701", - "id": "allenai/open_instruct_dev-reward_modeling__1__1739957701", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-reward_modeling__1__1739957701/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6194 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6779 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3563 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6011 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8022 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.697 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5822 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-reward_modeling__1__1739971507.json b/data/models/allenai_open_instruct_dev-reward_modeling__1__1739971507.json deleted file mode 100644 index 39dc6948dfba1cbc1fc83430bdd7d2489f47cd5a..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-reward_modeling__1__1739971507.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-reward_modeling__1__1739971507", - "id": "allenai/open_instruct_dev-reward_modeling__1__1739971507", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-reward_modeling__1__1739971507/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5717 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.68 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.375 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6066 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7667 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5475 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4545 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-reward_modeling__1__1739971529.json b/data/models/allenai_open_instruct_dev-reward_modeling__1__1739971529.json deleted file mode 100644 index cf825f2e80f5955f45b67404f86b1b7acc099678..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-reward_modeling__1__1739971529.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-reward_modeling__1__1739971529", - "id": "allenai/open_instruct_dev-reward_modeling__1__1739971529", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-reward_modeling__1__1739971529/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5564 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6568 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3563 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5956 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7533 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5737 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4027 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-reward_modeling__1__1739998765.json b/data/models/allenai_open_instruct_dev-reward_modeling__1__1739998765.json deleted file mode 100644 index b674eb1c91bf88cf5a6628fbc34a7e85e82f407d..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-reward_modeling__1__1739998765.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-reward_modeling__1__1739998765", - "id": "allenai/open_instruct_dev-reward_modeling__1__1739998765", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-reward_modeling__1__1739998765/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6008 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7095 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4125 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6066 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8022 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5859 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4883 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-reward_modeling__1__1740005072.json b/data/models/allenai_open_instruct_dev-reward_modeling__1__1740005072.json deleted file mode 100644 index 217be9991b9694693f33de8c2034609272eb4d14..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-reward_modeling__1__1740005072.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-reward_modeling__1__1740005072", - "id": "allenai/open_instruct_dev-reward_modeling__1__1740005072", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-reward_modeling__1__1740005072/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6097 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7137 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3937 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6339 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7778 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6343 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5047 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-reward_modeling__1__1740129284.json b/data/models/allenai_open_instruct_dev-reward_modeling__1__1740129284.json deleted file mode 100644 index 982577b691fc577441961fef69413ff37862ee78..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-reward_modeling__1__1740129284.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-reward_modeling__1__1740129284", - "id": "allenai/open_instruct_dev-reward_modeling__1__1740129284", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-reward_modeling__1__1740129284/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6129 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7116 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4437 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6448 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8022 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6101 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4652 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-reward_modeling__1__1741286813.json b/data/models/allenai_open_instruct_dev-reward_modeling__1__1741286813.json deleted file mode 100644 index d8cffc4ff56d3e1fed77f4b9cd0581da8abf1f46..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-reward_modeling__1__1741286813.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-reward_modeling__1__1741286813", - "id": "allenai/open_instruct_dev-reward_modeling__1__1741286813", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-reward_modeling__1__1741286813/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6557 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6295 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4188 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.612 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9111 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8263 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5365 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-reward_modeling__1__1741287363.json b/data/models/allenai_open_instruct_dev-reward_modeling__1__1741287363.json deleted file mode 100644 index 6720749b309ebc6442f910fa8235bec04bad6266..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-reward_modeling__1__1741287363.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-reward_modeling__1__1741287363", - "id": "allenai/open_instruct_dev-reward_modeling__1__1741287363", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-reward_modeling__1__1741287363/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6672 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6295 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.375 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6066 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.88 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9374 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5748 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-reward_modeling__1__1741292911.json b/data/models/allenai_open_instruct_dev-reward_modeling__1__1741292911.json deleted file mode 100644 index 2e3fd089d526abadff7d0ac40b2de3c769fdc41e..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-reward_modeling__1__1741292911.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-reward_modeling__1__1741292911", - "id": "allenai/open_instruct_dev-reward_modeling__1__1741292911", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-reward_modeling__1__1741292911/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6607 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6589 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6066 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9089 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8869 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5028 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-reward_modeling__1__1742338142.json b/data/models/allenai_open_instruct_dev-reward_modeling__1__1742338142.json deleted file mode 100644 index d86886b70ac1ca1b408c7882256628592f744afa..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-reward_modeling__1__1742338142.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-reward_modeling__1__1742338142", - "id": "allenai/open_instruct_dev-reward_modeling__1__1742338142", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-reward_modeling__1__1742338142/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6344 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7326 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3812 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7049 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.88 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6323 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.475 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-reward_modeling__1__1742519610.json b/data/models/allenai_open_instruct_dev-reward_modeling__1__1742519610.json deleted file mode 100644 index a183ba7b46dae8a43d8ef05aeb512f3a9412828d..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-reward_modeling__1__1742519610.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-reward_modeling__1__1742519610", - "id": "allenai/open_instruct_dev-reward_modeling__1__1742519610", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-reward_modeling__1__1742519610/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6361 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7074 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3812 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6721 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.82 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6444 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5915 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-reward_modeling__1__1742519628.json b/data/models/allenai_open_instruct_dev-reward_modeling__1__1742519628.json deleted file mode 100644 index 891e9094588783fca39588a56929b5dda302ac2e..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-reward_modeling__1__1742519628.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-reward_modeling__1__1742519628", - "id": "allenai/open_instruct_dev-reward_modeling__1__1742519628", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-reward_modeling__1__1742519628/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5609 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5179 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3563 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.623 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8356 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5071 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5254 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_1e-6_1_100pctflipped__1__1744241455.json b/data/models/allenai_open_instruct_dev-rm_1e-6_1_100pctflipped__1__1744241455.json deleted file mode 100644 index 00ae12170470abe995955ca8eec803790ffb41fb..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_1e-6_1_100pctflipped__1__1744241455.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_1e-6_1_100pctflipped__1__1744241455", - "id": "allenai/open_instruct_dev-rm_1e-6_1_100pctflipped__1__1744241455", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_1e-6_1_100pctflipped__1__1744241455/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0576 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.04 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1313 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0546 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0489 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0808 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -0.01 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_1e-6_1_10pctflipped__1__1743295511.json b/data/models/allenai_open_instruct_dev-rm_1e-6_1_10pctflipped__1__1743295511.json deleted file mode 100644 index fc3092f03c46500a0b065de921cf3316327f36b4..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_1e-6_1_10pctflipped__1__1743295511.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_1e-6_1_10pctflipped__1__1743295511", - "id": "allenai/open_instruct_dev-rm_1e-6_1_10pctflipped__1__1743295511", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_1e-6_1_10pctflipped__1__1743295511/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5499 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6821 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3937 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5956 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7356 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5212 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3711 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_1e-6_1_20pctflipped__1__1743295406.json b/data/models/allenai_open_instruct_dev-rm_1e-6_1_20pctflipped__1__1743295406.json deleted file mode 100644 index dbb2a50c3876fa9c6e7735de45bb04c320c92fcb..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_1e-6_1_20pctflipped__1__1743295406.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_1e-6_1_20pctflipped__1__1743295406", - "id": "allenai/open_instruct_dev-rm_1e-6_1_20pctflipped__1__1743295406", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_1e-6_1_20pctflipped__1__1743295406/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5054 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6358 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3688 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6066 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6867 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4424 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2922 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_1e-6_1_30pctflipped__1__1743325136.json b/data/models/allenai_open_instruct_dev-rm_1e-6_1_30pctflipped__1__1743325136.json deleted file mode 100644 index 1928347b96776678bed44b8de3e04d66793ff2d3..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_1e-6_1_30pctflipped__1__1743325136.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_1e-6_1_30pctflipped__1__1743325136", - "id": "allenai/open_instruct_dev-rm_1e-6_1_30pctflipped__1__1743325136", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_1e-6_1_30pctflipped__1__1743325136/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.478 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6442 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3563 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.612 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6356 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2707 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3496 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_1e-6_1_50pctflipped__1__1744241398.json b/data/models/allenai_open_instruct_dev-rm_1e-6_1_50pctflipped__1__1744241398.json deleted file mode 100644 index 346fcb746ce0cc5557a34f59604b99711e0251b4..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_1e-6_1_50pctflipped__1__1744241398.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_1e-6_1_50pctflipped__1__1744241398", - "id": "allenai/open_instruct_dev-rm_1e-6_1_50pctflipped__1__1744241398", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_1e-6_1_50pctflipped__1__1744241398/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.219 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2484 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2812 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2623 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3422 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1717 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.008 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_1e-6_1_5pctflipped__1__1743444535.json b/data/models/allenai_open_instruct_dev-rm_1e-6_1_5pctflipped__1__1743444535.json deleted file mode 100644 index 05cd4f458831fd84953868ef2bfbdad8231b1a44..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_1e-6_1_5pctflipped__1__1743444535.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_1e-6_1_5pctflipped__1__1743444535", - "id": "allenai/open_instruct_dev-rm_1e-6_1_5pctflipped__1__1743444535", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_1e-6_1_5pctflipped__1__1743444535/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5625 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6821 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4062 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6011 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7511 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5313 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.403 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_1e-6_1_dpo__1__1743550054.json b/data/models/allenai_open_instruct_dev-rm_1e-6_1_dpo__1__1743550054.json deleted file mode 100644 index e4ca53c0517fb8b73c18fbfbdc7080bba346b184..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_1e-6_1_dpo__1__1743550054.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_1e-6_1_dpo__1__1743550054", - "id": "allenai/open_instruct_dev-rm_1e-6_1_dpo__1__1743550054", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_1e-6_1_dpo__1__1743550054/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5759 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7074 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.375 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.623 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7578 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5333 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.459 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_1e-6_1_dpo_skyworks__1__1744530271.json b/data/models/allenai_open_instruct_dev-rm_1e-6_1_dpo_skyworks__1__1744530271.json deleted file mode 100644 index cc5bbdfc19152dd8adc973a7d0e20fef2d96ec93..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_1e-6_1_dpo_skyworks__1__1744530271.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_1e-6_1_dpo_skyworks__1__1744530271", - "id": "allenai/open_instruct_dev-rm_1e-6_1_dpo_skyworks__1__1744530271", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_1e-6_1_dpo_skyworks__1__1744530271/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6057 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5053 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.375 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5902 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8422 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7798 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5419 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_1e-6_1_dpo_skyworkstulufull__1__1743550181.json b/data/models/allenai_open_instruct_dev-rm_1e-6_1_dpo_skyworkstulufull__1__1743550181.json deleted file mode 100644 index 178cd8b0b0bed0392129e2e5989896a13060c133..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_1e-6_1_dpo_skyworkstulufull__1__1743550181.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_1e-6_1_dpo_skyworkstulufull__1__1743550181", - "id": "allenai/open_instruct_dev-rm_1e-6_1_dpo_skyworkstulufull__1__1743550181", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_1e-6_1_dpo_skyworkstulufull__1__1743550181/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6535 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7137 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3812 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6175 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8244 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7737 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6101 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_1e-6_1_rl__1__1743551221.json b/data/models/allenai_open_instruct_dev-rm_1e-6_1_rl__1__1743551221.json deleted file mode 100644 index 7ed49c01b04cdb3dfb454990a0f887abe895bc07..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_1e-6_1_rl__1__1743551221.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_1e-6_1_rl__1__1743551221", - "id": "allenai/open_instruct_dev-rm_1e-6_1_rl__1__1743551221", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_1e-6_1_rl__1__1743551221/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5799 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7116 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3812 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6284 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.76 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5374 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.461 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_1e-6_1_rl_skyworks__1__1744530262.json b/data/models/allenai_open_instruct_dev-rm_1e-6_1_rl_skyworks__1__1744530262.json deleted file mode 100644 index 8195d049529c63d24353ffe4a250c1036a5dcba2..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_1e-6_1_rl_skyworks__1__1744530262.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_1e-6_1_rl_skyworks__1__1744530262", - "id": "allenai/open_instruct_dev-rm_1e-6_1_rl_skyworks__1__1744530262", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_1e-6_1_rl_skyworks__1__1744530262/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5903 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4863 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3625 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5738 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8489 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7778 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4926 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_1e-6_1_rl_skyworkstulufull__1__1743551523.json b/data/models/allenai_open_instruct_dev-rm_1e-6_1_rl_skyworkstulufull__1__1743551523.json deleted file mode 100644 index f9c8363d9c74c8ecece91b1bea804e630fcc3ed6..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_1e-6_1_rl_skyworkstulufull__1__1743551523.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_1e-6_1_rl_skyworkstulufull__1__1743551523", - "id": "allenai/open_instruct_dev-rm_1e-6_1_rl_skyworkstulufull__1__1743551523", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_1e-6_1_rl_skyworkstulufull__1__1743551523/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6483 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7074 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3625 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6175 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8222 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7758 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6044 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_1e-6_1_skyworkstulumix__1__1743205750.json b/data/models/allenai_open_instruct_dev-rm_1e-6_1_skyworkstulumix__1__1743205750.json deleted file mode 100644 index 544e570e5b7780138fc57f07bb3ec15b0e3429f4..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_1e-6_1_skyworkstulumix__1__1743205750.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_1e-6_1_skyworkstulumix__1__1743205750", - "id": "allenai/open_instruct_dev-rm_1e-6_1_skyworkstulumix__1__1743205750", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_1e-6_1_skyworkstulumix__1__1743205750/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5157 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6084 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3688 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6066 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7089 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4222 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3791 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_1e-6_2_10pctflipped__1__1743295427.json b/data/models/allenai_open_instruct_dev-rm_1e-6_2_10pctflipped__1__1743295427.json deleted file mode 100644 index f120e5665578b63bd324bec9c169b8f6f3d858c1..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_1e-6_2_10pctflipped__1__1743295427.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_1e-6_2_10pctflipped__1__1743295427", - "id": "allenai/open_instruct_dev-rm_1e-6_2_10pctflipped__1__1743295427", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_1e-6_2_10pctflipped__1__1743295427/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6009 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7263 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.375 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5902 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7933 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7273 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3931 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_1e-6_2_20pctflipped__1__1743295446.json b/data/models/allenai_open_instruct_dev-rm_1e-6_2_20pctflipped__1__1743295446.json deleted file mode 100644 index f3442d96bbd7afedf56b2fce78fb3a45a9afc7a3..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_1e-6_2_20pctflipped__1__1743295446.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_1e-6_2_20pctflipped__1__1743295446", - "id": "allenai/open_instruct_dev-rm_1e-6_2_20pctflipped__1__1743295446", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_1e-6_2_20pctflipped__1__1743295446/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5716 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6779 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3937 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5464 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7533 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7051 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3534 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_1e-6_2_30pctflipped__1__1743325094.json b/data/models/allenai_open_instruct_dev-rm_1e-6_2_30pctflipped__1__1743325094.json deleted file mode 100644 index cedfd398cf32e477ed09f75303d207e32a02614e..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_1e-6_2_30pctflipped__1__1743325094.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_1e-6_2_30pctflipped__1__1743325094", - "id": "allenai/open_instruct_dev-rm_1e-6_2_30pctflipped__1__1743325094", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_1e-6_2_30pctflipped__1__1743325094/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5151 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6484 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3312 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5574 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7289 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4889 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3357 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_1e-6_2_5pctflipped__1__1743444636.json b/data/models/allenai_open_instruct_dev-rm_1e-6_2_5pctflipped__1__1743444636.json deleted file mode 100644 index 3634d8a355d348744911b2d1176e3764532f7d27..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_1e-6_2_5pctflipped__1__1743444636.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_1e-6_2_5pctflipped__1__1743444636", - "id": "allenai/open_instruct_dev-rm_1e-6_2_5pctflipped__1__1743444636", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_1e-6_2_5pctflipped__1__1743444636/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6119 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.72 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4062 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6284 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8067 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6889 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.421 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_1e-6_2_dpo__1__1743549325.json b/data/models/allenai_open_instruct_dev-rm_1e-6_2_dpo__1__1743549325.json deleted file mode 100644 index a774ab7d7404d1e784979c6c4a92aea26404b525..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_1e-6_2_dpo__1__1743549325.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_1e-6_2_dpo__1__1743549325", - "id": "allenai/open_instruct_dev-rm_1e-6_2_dpo__1__1743549325", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_1e-6_2_dpo__1__1743549325/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6008 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7179 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.35 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5956 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6707 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4707 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_1e-6_2_rl__1__1743551238.json b/data/models/allenai_open_instruct_dev-rm_1e-6_2_rl__1__1743551238.json deleted file mode 100644 index 85132dfd878943b932a797458a4a189dda08c52f..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_1e-6_2_rl__1__1743551238.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_1e-6_2_rl__1__1743551238", - "id": "allenai/open_instruct_dev-rm_1e-6_2_rl__1__1743551238", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_1e-6_2_rl__1__1743551238/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5965 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7095 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3438 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.612 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8044 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6566 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.453 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_1e-6_2_skyworkstulumix__1__1743205906.json b/data/models/allenai_open_instruct_dev-rm_1e-6_2_skyworkstulumix__1__1743205906.json deleted file mode 100644 index fca09ba1fa498bae4bbc044bfe315b2932a1da90..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_1e-6_2_skyworkstulumix__1__1743205906.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_1e-6_2_skyworkstulumix__1__1743205906", - "id": "allenai/open_instruct_dev-rm_1e-6_2_skyworkstulumix__1__1743205906", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_1e-6_2_skyworkstulumix__1__1743205906/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5574 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6526 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3937 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6011 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7711 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5051 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4208 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_2e-5_1_100pctflipped__1__1744241529.json b/data/models/allenai_open_instruct_dev-rm_2e-5_1_100pctflipped__1__1744241529.json deleted file mode 100644 index c7edb4627ae7e125dae9e972cf1cc4fa64b219bc..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_2e-5_1_100pctflipped__1__1744241529.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_2e-5_1_100pctflipped__1__1744241529", - "id": "allenai/open_instruct_dev-rm_2e-5_1_100pctflipped__1__1744241529", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_2e-5_1_100pctflipped__1__1744241529/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0719 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0421 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2062 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0601 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0378 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0949 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -0.01 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_2e-5_1_10pctflipped__1__1743295305.json b/data/models/allenai_open_instruct_dev-rm_2e-5_1_10pctflipped__1__1743295305.json deleted file mode 100644 index 79f078505578c1af35363a3aba81f04d315809f1..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_2e-5_1_10pctflipped__1__1743295305.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_2e-5_1_10pctflipped__1__1743295305", - "id": "allenai/open_instruct_dev-rm_2e-5_1_10pctflipped__1__1743295305", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_2e-5_1_10pctflipped__1__1743295305/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.553 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6674 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3563 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6284 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6733 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5697 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4227 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_2e-5_1_20pctflipped__1__1743324778.json b/data/models/allenai_open_instruct_dev-rm_2e-5_1_20pctflipped__1__1743324778.json deleted file mode 100644 index 905eea811fec796d7c8006d878cca46d14afb5d2..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_2e-5_1_20pctflipped__1__1743324778.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_2e-5_1_20pctflipped__1__1743324778", - "id": "allenai/open_instruct_dev-rm_2e-5_1_20pctflipped__1__1743324778", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_2e-5_1_20pctflipped__1__1743324778/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4955 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6189 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.325 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5792 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6378 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5657 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2466 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_2e-5_1_30pctflipped__1__1743326459.json b/data/models/allenai_open_instruct_dev-rm_2e-5_1_30pctflipped__1__1743326459.json deleted file mode 100644 index fa2a3bb6bf3a355a23b6a58eeef49851e0dc6a61..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_2e-5_1_30pctflipped__1__1743326459.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_2e-5_1_30pctflipped__1__1743326459", - "id": "allenai/open_instruct_dev-rm_2e-5_1_30pctflipped__1__1743326459", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_2e-5_1_30pctflipped__1__1743326459/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4198 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5747 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3375 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5464 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4933 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3596 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2073 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_2e-5_1_5pctflipped__1__1743443747.json b/data/models/allenai_open_instruct_dev-rm_2e-5_1_5pctflipped__1__1743443747.json deleted file mode 100644 index 13a4e43749c71c55894bc4c11b01c69b57aac3aa..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_2e-5_1_5pctflipped__1__1743443747.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_2e-5_1_5pctflipped__1__1743443747", - "id": "allenai/open_instruct_dev-rm_2e-5_1_5pctflipped__1__1743443747", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_2e-5_1_5pctflipped__1__1743443747/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5465 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6821 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.375 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.612 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7333 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5051 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3713 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_2e-5_1_skyworkstulumix__1__1743205935.json b/data/models/allenai_open_instruct_dev-rm_2e-5_1_skyworkstulumix__1__1743205935.json deleted file mode 100644 index 2b18839b73e57327e49ec1a873f3960c884345e9..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_2e-5_1_skyworkstulumix__1__1743205935.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_2e-5_1_skyworkstulumix__1__1743205935", - "id": "allenai/open_instruct_dev-rm_2e-5_1_skyworkstulumix__1__1743205935", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_2e-5_1_skyworkstulumix__1__1743205935/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5197 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6126 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3375 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5847 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7333 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4646 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3855 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_2e-5_2_10pctflipped__1__1743295360.json b/data/models/allenai_open_instruct_dev-rm_2e-5_2_10pctflipped__1__1743295360.json deleted file mode 100644 index 4ab985a83589436be558a1f61428e7e646dfb218..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_2e-5_2_10pctflipped__1__1743295360.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_2e-5_2_10pctflipped__1__1743295360", - "id": "allenai/open_instruct_dev-rm_2e-5_2_10pctflipped__1__1743295360", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_2e-5_2_10pctflipped__1__1743295360/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4555 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5495 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3063 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4262 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5711 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6101 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2696 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_2e-5_2_20pctflipped__1__1743295366.json b/data/models/allenai_open_instruct_dev-rm_2e-5_2_20pctflipped__1__1743295366.json deleted file mode 100644 index 3fae79394607bdeb54e8a08893a9b0d217b74812..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_2e-5_2_20pctflipped__1__1743295366.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_2e-5_2_20pctflipped__1__1743295366", - "id": "allenai/open_instruct_dev-rm_2e-5_2_20pctflipped__1__1743295366", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_2e-5_2_20pctflipped__1__1743295366/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4422 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5053 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3375 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4044 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5422 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6646 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1991 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_2e-5_2_30pctflipped__1__1743326352.json b/data/models/allenai_open_instruct_dev-rm_2e-5_2_30pctflipped__1__1743326352.json deleted file mode 100644 index 0f5286bb8c6af97b89e975d5ea3dc4d55084724b..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_2e-5_2_30pctflipped__1__1743326352.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_2e-5_2_30pctflipped__1__1743326352", - "id": "allenai/open_instruct_dev-rm_2e-5_2_30pctflipped__1__1743326352", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_2e-5_2_30pctflipped__1__1743326352/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.341 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4674 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2875 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3333 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3711 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3919 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.195 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_2e-5_2_5pctflipped__1__1743444634.json b/data/models/allenai_open_instruct_dev-rm_2e-5_2_5pctflipped__1__1743444634.json deleted file mode 100644 index cc2db6403badde75b0fcd294c37a5511726d69aa..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_2e-5_2_5pctflipped__1__1743444634.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_2e-5_2_5pctflipped__1__1743444634", - "id": "allenai/open_instruct_dev-rm_2e-5_2_5pctflipped__1__1743444634", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_2e-5_2_5pctflipped__1__1743444634/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4698 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5853 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2562 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5027 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6489 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5697 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2562 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_2e-5_2_skyworkstulumix__1__1743205988.json b/data/models/allenai_open_instruct_dev-rm_2e-5_2_skyworkstulumix__1__1743205988.json deleted file mode 100644 index fb0639282a925e108d87717ffa3fd06ce442b5f3..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_2e-5_2_skyworkstulumix__1__1743205988.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_2e-5_2_skyworkstulumix__1__1743205988", - "id": "allenai/open_instruct_dev-rm_2e-5_2_skyworkstulumix__1__1743205988", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_2e-5_2_skyworkstulumix__1__1743205988/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4791 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6421 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3125 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.541 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6911 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4182 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.27 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_3e-6_1_100pctflipped__1__1744242103.json b/data/models/allenai_open_instruct_dev-rm_3e-6_1_100pctflipped__1__1744242103.json deleted file mode 100644 index 54a5d2017439bb7d93a648a0ca1f9f78e32f33b9..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_3e-6_1_100pctflipped__1__1744242103.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_3e-6_1_100pctflipped__1__1744242103", - "id": "allenai/open_instruct_dev-rm_3e-6_1_100pctflipped__1__1744242103", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_3e-6_1_100pctflipped__1__1744242103/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0607 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0274 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1625 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0656 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.04 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0788 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -0.01 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_3e-6_1_10pctflipped__1__1743324835.json b/data/models/allenai_open_instruct_dev-rm_3e-6_1_10pctflipped__1__1743324835.json deleted file mode 100644 index 4b1ce4c09262a66e8a7c60f21254062cfc39a2a0..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_3e-6_1_10pctflipped__1__1743324835.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_3e-6_1_10pctflipped__1__1743324835", - "id": "allenai/open_instruct_dev-rm_3e-6_1_10pctflipped__1__1743324835", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_3e-6_1_10pctflipped__1__1743324835/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6089 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7284 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4375 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.612 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7622 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6444 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4686 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_3e-6_1_1pctflipped__1__1743445221.json b/data/models/allenai_open_instruct_dev-rm_3e-6_1_1pctflipped__1__1743445221.json deleted file mode 100644 index 435ad8c0566cebd0f020ffd66ae9492a0d32710d..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_3e-6_1_1pctflipped__1__1743445221.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_3e-6_1_1pctflipped__1__1743445221", - "id": "allenai/open_instruct_dev-rm_3e-6_1_1pctflipped__1__1743445221", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_3e-6_1_1pctflipped__1__1743445221/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6032 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7158 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4062 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6284 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7778 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5859 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5051 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_3e-6_1_20pctflipped__1__1743324826.json b/data/models/allenai_open_instruct_dev-rm_3e-6_1_20pctflipped__1__1743324826.json deleted file mode 100644 index 3087908045fc07dbe508213a4c08751e1b219249..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_3e-6_1_20pctflipped__1__1743324826.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_3e-6_1_20pctflipped__1__1743324826", - "id": "allenai/open_instruct_dev-rm_3e-6_1_20pctflipped__1__1743324826", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_3e-6_1_20pctflipped__1__1743324826/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5831 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6947 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4188 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.623 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.74 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5758 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4465 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_3e-6_1_30pctflipped__1__1743326363.json b/data/models/allenai_open_instruct_dev-rm_3e-6_1_30pctflipped__1__1743326363.json deleted file mode 100644 index 95227ecfff1c9b70557a5de3db5e291897ed0376..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_3e-6_1_30pctflipped__1__1743326363.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_3e-6_1_30pctflipped__1__1743326363", - "id": "allenai/open_instruct_dev-rm_3e-6_1_30pctflipped__1__1743326363", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_3e-6_1_30pctflipped__1__1743326363/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5268 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.68 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3688 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5792 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7178 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4343 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3809 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_3e-6_1_5pctflipped__1__1743444498.json b/data/models/allenai_open_instruct_dev-rm_3e-6_1_5pctflipped__1__1743444498.json deleted file mode 100644 index efcbf8672b32e78cccb382fb1214d0ec26dbc068..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_3e-6_1_5pctflipped__1__1743444498.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_3e-6_1_5pctflipped__1__1743444498", - "id": "allenai/open_instruct_dev-rm_3e-6_1_5pctflipped__1__1743444498", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_3e-6_1_5pctflipped__1__1743444498/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6093 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7326 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4313 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6339 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7578 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5859 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5143 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_3e-6_1__2__1743897475.json b/data/models/allenai_open_instruct_dev-rm_3e-6_1__2__1743897475.json deleted file mode 100644 index a3b5d5f5a6019826626c3216717f0d449e0eb900..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_3e-6_1__2__1743897475.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_3e-6_1__2__1743897475", - "id": "allenai/open_instruct_dev-rm_3e-6_1__2__1743897475", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_3e-6_1__2__1743897475/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6122 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7368 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.623 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8044 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.602 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5071 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_3e-6_1__3__1744311421.json b/data/models/allenai_open_instruct_dev-rm_3e-6_1__3__1744311421.json deleted file mode 100644 index 0c3d02e48b810e47d7127f5ff617e28c0eea0d90..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_3e-6_1__3__1744311421.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_3e-6_1__3__1744311421", - "id": "allenai/open_instruct_dev-rm_3e-6_1__3__1744311421", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_3e-6_1__3__1744311421/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5995 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7179 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3375 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6066 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6323 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.503 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_3e-6_1_dpo__1__1743549903.json b/data/models/allenai_open_instruct_dev-rm_3e-6_1_dpo__1__1743549903.json deleted file mode 100644 index 65bc1d0cbdde1f09f528c2efc767ceb8b345e47f..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_3e-6_1_dpo__1__1743549903.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_3e-6_1_dpo__1__1743549903", - "id": "allenai/open_instruct_dev-rm_3e-6_1_dpo__1__1743549903", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_3e-6_1_dpo__1__1743549903/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6154 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7326 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4375 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6339 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7778 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6061 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5043 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_3e-6_1_dpo_skyworks__1__1744530368.json b/data/models/allenai_open_instruct_dev-rm_3e-6_1_dpo_skyworks__1__1744530368.json deleted file mode 100644 index aedfd35bd6cd70575f30b28618a0783ef1fe83c0..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_3e-6_1_dpo_skyworks__1__1744530368.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_3e-6_1_dpo_skyworks__1__1744530368", - "id": "allenai/open_instruct_dev-rm_3e-6_1_dpo_skyworks__1__1744530368", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_3e-6_1_dpo_skyworks__1__1744530368/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6604 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6316 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3937 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5792 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9044 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8929 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5604 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_3e-6_1_dpo_skyworkstulufull__1__1743550182.json b/data/models/allenai_open_instruct_dev-rm_3e-6_1_dpo_skyworkstulufull__1__1743550182.json deleted file mode 100644 index 185fe6d8efad6efe065b7ecf901efa87b0a985f8..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_3e-6_1_dpo_skyworkstulufull__1__1743550182.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_3e-6_1_dpo_skyworkstulufull__1__1743550182", - "id": "allenai/open_instruct_dev-rm_3e-6_1_dpo_skyworkstulufull__1__1743550182", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_3e-6_1_dpo_skyworkstulufull__1__1743550182/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6783 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7705 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6066 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.84 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8101 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6427 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_3e-6_1_no_if__2__1744316012.json b/data/models/allenai_open_instruct_dev-rm_3e-6_1_no_if__2__1744316012.json deleted file mode 100644 index 3b72bdac9e582d1f3c21806ee98c7b919f1963cf..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_3e-6_1_no_if__2__1744316012.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_3e-6_1_no_if__2__1744316012", - "id": "allenai/open_instruct_dev-rm_3e-6_1_no_if__2__1744316012", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_3e-6_1_no_if__2__1744316012/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5911 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7347 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6284 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.74 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.604 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4392 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_3e-6_1_no_if__3__1744315765.json b/data/models/allenai_open_instruct_dev-rm_3e-6_1_no_if__3__1744315765.json deleted file mode 100644 index fce09e01b723330bb764854c78d7e9500ab2b6df..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_3e-6_1_no_if__3__1744315765.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_3e-6_1_no_if__3__1744315765", - "id": "allenai/open_instruct_dev-rm_3e-6_1_no_if__3__1744315765", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_3e-6_1_no_if__3__1744315765/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5926 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7263 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3563 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.623 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7889 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5879 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4733 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_3e-6_1_rl__1__1743551527.json b/data/models/allenai_open_instruct_dev-rm_3e-6_1_rl__1__1743551527.json deleted file mode 100644 index c1530baad00a7c7ded42df3dc96687f0732eb9fb..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_3e-6_1_rl__1__1743551527.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_3e-6_1_rl__1__1743551527", - "id": "allenai/open_instruct_dev-rm_3e-6_1_rl__1__1743551527", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_3e-6_1_rl__1__1743551527/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6126 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7411 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.425 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.623 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7822 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5939 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5104 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_3e-6_1_rl_skyworks__1__1744530236.json b/data/models/allenai_open_instruct_dev-rm_3e-6_1_rl_skyworks__1__1744530236.json deleted file mode 100644 index 82c99d4a45e3bdc8d084bb0112e7b7c7563b392a..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_3e-6_1_rl_skyworks__1__1744530236.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_3e-6_1_rl_skyworks__1__1744530236", - "id": "allenai/open_instruct_dev-rm_3e-6_1_rl_skyworks__1__1744530236", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_3e-6_1_rl_skyworks__1__1744530236/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6525 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6021 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3875 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5792 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8933 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8626 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.59 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_3e-6_1_rl_skyworkstulufull__1__1743551530.json b/data/models/allenai_open_instruct_dev-rm_3e-6_1_rl_skyworkstulufull__1__1743551530.json deleted file mode 100644 index 97f352496973c9d67f60f57143784632b533b00c..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_3e-6_1_rl_skyworkstulufull__1__1743551530.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_3e-6_1_rl_skyworkstulufull__1__1743551530", - "id": "allenai/open_instruct_dev-rm_3e-6_1_rl_skyworkstulufull__1__1743551530", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_3e-6_1_rl_skyworkstulufull__1__1743551530/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6849 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7453 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3812 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.612 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8422 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8404 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6885 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_3e-6_1_skyworkstulu75__1__1743534417.json b/data/models/allenai_open_instruct_dev-rm_3e-6_1_skyworkstulu75__1__1743534417.json deleted file mode 100644 index 2871e87fe6e307a77dd4cbc0895febd63cf934d0..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_3e-6_1_skyworkstulu75__1__1743534417.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_3e-6_1_skyworkstulu75__1__1743534417", - "id": "allenai/open_instruct_dev-rm_3e-6_1_skyworkstulu75__1__1743534417", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_3e-6_1_skyworkstulu75__1__1743534417/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.586 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6632 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.425 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6557 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7778 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5172 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.477 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_3e-6_1_skyworkstulufull__1__1743446486.json b/data/models/allenai_open_instruct_dev-rm_3e-6_1_skyworkstulufull__1__1743446486.json deleted file mode 100644 index ea39ea0cdaf2d103210fea87912d594244c2d002..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_3e-6_1_skyworkstulufull__1__1743446486.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_3e-6_1_skyworkstulufull__1__1743446486", - "id": "allenai/open_instruct_dev-rm_3e-6_1_skyworkstulufull__1__1743446486", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_3e-6_1_skyworkstulufull__1__1743446486/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6773 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7432 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.612 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8422 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.804 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6626 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_3e-6_1_skyworkstulufull__2__1744314745.json b/data/models/allenai_open_instruct_dev-rm_3e-6_1_skyworkstulufull__2__1744314745.json deleted file mode 100644 index a4c4718456a35277d5bc642cdd068c2d68645d4a..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_3e-6_1_skyworkstulufull__2__1744314745.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_3e-6_1_skyworkstulufull__2__1744314745", - "id": "allenai/open_instruct_dev-rm_3e-6_1_skyworkstulufull__2__1744314745", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_3e-6_1_skyworkstulufull__2__1744314745/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6793 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7558 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4062 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6284 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8311 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8061 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6485 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_3e-6_1_skyworkstulufull__3__1744311661.json b/data/models/allenai_open_instruct_dev-rm_3e-6_1_skyworkstulufull__3__1744311661.json deleted file mode 100644 index 92ccec0ca782d92bd06d73fa679c11dd6ef237ae..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_3e-6_1_skyworkstulufull__3__1744311661.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_3e-6_1_skyworkstulufull__3__1744311661", - "id": "allenai/open_instruct_dev-rm_3e-6_1_skyworkstulufull__3__1744311661", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_3e-6_1_skyworkstulufull__3__1744311661/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6611 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.72 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3563 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6393 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8444 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7636 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6428 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_3e-6_1_skyworkstulumix__1__1743204472.json b/data/models/allenai_open_instruct_dev-rm_3e-6_1_skyworkstulumix__1__1743204472.json deleted file mode 100644 index b88aa0b3cb2817b1636be2452fe065371a06ff00..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_3e-6_1_skyworkstulumix__1__1743204472.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_3e-6_1_skyworkstulumix__1__1743204472", - "id": "allenai/open_instruct_dev-rm_3e-6_1_skyworkstulumix__1__1743204472", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_3e-6_1_skyworkstulumix__1__1743204472/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5778 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6674 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3875 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6011 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7933 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5172 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5003 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_3e-6_2_10pctflipped__1__1743295267.json b/data/models/allenai_open_instruct_dev-rm_3e-6_2_10pctflipped__1__1743295267.json deleted file mode 100644 index 7215ab2e06f5d018ce97d43e21e9e8811fa20789..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_3e-6_2_10pctflipped__1__1743295267.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_3e-6_2_10pctflipped__1__1743295267", - "id": "allenai/open_instruct_dev-rm_3e-6_2_10pctflipped__1__1743295267", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_3e-6_2_10pctflipped__1__1743295267/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5746 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6505 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.35 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5082 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7844 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7414 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4128 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_3e-6_2_1pctflipped__1__1743445759.json b/data/models/allenai_open_instruct_dev-rm_3e-6_2_1pctflipped__1__1743445759.json deleted file mode 100644 index 11705519528b3d82b269c01f4f872efdfda1dade..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_3e-6_2_1pctflipped__1__1743445759.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_3e-6_2_1pctflipped__1__1743445759", - "id": "allenai/open_instruct_dev-rm_3e-6_2_1pctflipped__1__1743445759", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_3e-6_2_1pctflipped__1__1743445759/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6065 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7116 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.35 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5792 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8178 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7152 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.465 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_3e-6_2_20pctflipped__1__1743324905.json b/data/models/allenai_open_instruct_dev-rm_3e-6_2_20pctflipped__1__1743324905.json deleted file mode 100644 index 523c1a807f06f3d3e9f51757a6ad51df9c76c0a0..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_3e-6_2_20pctflipped__1__1743324905.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_3e-6_2_20pctflipped__1__1743324905", - "id": "allenai/open_instruct_dev-rm_3e-6_2_20pctflipped__1__1743324905", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_3e-6_2_20pctflipped__1__1743324905/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5305 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5832 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3312 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.459 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7178 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7071 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3849 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_3e-6_2_30pctflipped__1__1743326363.json b/data/models/allenai_open_instruct_dev-rm_3e-6_2_30pctflipped__1__1743326363.json deleted file mode 100644 index c7d722ec8a2922c9a1bdbdd44412041e434241d9..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_3e-6_2_30pctflipped__1__1743326363.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_3e-6_2_30pctflipped__1__1743326363", - "id": "allenai/open_instruct_dev-rm_3e-6_2_30pctflipped__1__1743326363", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_3e-6_2_30pctflipped__1__1743326363/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4436 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5411 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3312 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3115 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6267 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5414 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.31 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_3e-6_2_5pctflipped__1__1743444505.json b/data/models/allenai_open_instruct_dev-rm_3e-6_2_5pctflipped__1__1743444505.json deleted file mode 100644 index de71876dd0fcb0519025362a0be03dcf69a97078..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_3e-6_2_5pctflipped__1__1743444505.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_3e-6_2_5pctflipped__1__1743444505", - "id": "allenai/open_instruct_dev-rm_3e-6_2_5pctflipped__1__1743444505", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_3e-6_2_5pctflipped__1__1743444505/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5925 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.68 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3688 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5519 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.78 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7434 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.431 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_3e-6_2_dpo__1__1743550180.json b/data/models/allenai_open_instruct_dev-rm_3e-6_2_dpo__1__1743550180.json deleted file mode 100644 index 765e41dcb260d07798e0caf9d0bcadd4951e509b..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_3e-6_2_dpo__1__1743550180.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_3e-6_2_dpo__1__1743550180", - "id": "allenai/open_instruct_dev-rm_3e-6_2_dpo__1__1743550180", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_3e-6_2_dpo__1__1743550180/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6198 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7263 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3312 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6339 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8133 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7232 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4908 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_3e-6_2_dpo_skyworkstulufull__1__1743550187.json b/data/models/allenai_open_instruct_dev-rm_3e-6_2_dpo_skyworkstulufull__1__1743550187.json deleted file mode 100644 index 0bb3dcbdb26224ec553a73974e8df33ef3203cf8..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_3e-6_2_dpo_skyworkstulufull__1__1743550187.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_3e-6_2_dpo_skyworkstulufull__1__1743550187", - "id": "allenai/open_instruct_dev-rm_3e-6_2_dpo_skyworkstulufull__1__1743550187", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_3e-6_2_dpo_skyworkstulufull__1__1743550187/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6763 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7411 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.375 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.612 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8844 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8545 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5908 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_3e-6_2_rl__1__1743551509.json b/data/models/allenai_open_instruct_dev-rm_3e-6_2_rl__1__1743551509.json deleted file mode 100644 index 31f706519aff93345b859538cfb79ad6b84d08f6..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_3e-6_2_rl__1__1743551509.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_3e-6_2_rl__1__1743551509", - "id": "allenai/open_instruct_dev-rm_3e-6_2_rl__1__1743551509", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_3e-6_2_rl__1__1743551509/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6245 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7242 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.35 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6175 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8178 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7253 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5124 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_3e-6_2_rl_skyworkstulufull__1__1743551498.json b/data/models/allenai_open_instruct_dev-rm_3e-6_2_rl_skyworkstulufull__1__1743551498.json deleted file mode 100644 index d15c25cf3ad5f4b0a067549ced725a3b604ec916..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_3e-6_2_rl_skyworkstulufull__1__1743551498.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_3e-6_2_rl_skyworkstulufull__1__1743551498", - "id": "allenai/open_instruct_dev-rm_3e-6_2_rl_skyworkstulufull__1__1743551498", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_3e-6_2_rl_skyworkstulufull__1__1743551498/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6673 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7326 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3438 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6175 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8622 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8566 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5911 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_3e-6_2_skyworkstulu75__1__1743548926.json b/data/models/allenai_open_instruct_dev-rm_3e-6_2_skyworkstulu75__1__1743548926.json deleted file mode 100644 index 535b607bd9182910a6700da9dcc94602677ed2eb..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_3e-6_2_skyworkstulu75__1__1743548926.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_3e-6_2_skyworkstulu75__1__1743548926", - "id": "allenai/open_instruct_dev-rm_3e-6_2_skyworkstulu75__1__1743548926", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_3e-6_2_skyworkstulu75__1__1743548926/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5863 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6674 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3937 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6284 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5515 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4768 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_3e-6_2_skyworkstulumix__1__1743205661.json b/data/models/allenai_open_instruct_dev-rm_3e-6_2_skyworkstulumix__1__1743205661.json deleted file mode 100644 index fe6265c71382023ca771ab65e887875f602da273..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_3e-6_2_skyworkstulumix__1__1743205661.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_3e-6_2_skyworkstulumix__1__1743205661", - "id": "allenai/open_instruct_dev-rm_3e-6_2_skyworkstulumix__1__1743205661", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_3e-6_2_skyworkstulumix__1__1743205661/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.589 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6842 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3688 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6393 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7867 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6081 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.447 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_llama70b_skyworkstulufull__4__1747266598.json b/data/models/allenai_open_instruct_dev-rm_llama70b_skyworkstulufull__4__1747266598.json deleted file mode 100644 index 4989a5a37b713a7190109cc356100f6a8e514020..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_llama70b_skyworkstulufull__4__1747266598.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_llama70b_skyworkstulufull__4__1747266598", - "id": "allenai/open_instruct_dev-rm_llama70b_skyworkstulufull__4__1747266598", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_llama70b_skyworkstulufull__4__1747266598/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7306 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7474 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.375 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.694 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8622 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8061 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8992 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_llama70b_skyworkstulufull__8__1745387923.json b/data/models/allenai_open_instruct_dev-rm_llama70b_skyworkstulufull__8__1745387923.json deleted file mode 100644 index b57989e9b725a16ad384fd6fe444b16fc7d36724..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_llama70b_skyworkstulufull__8__1745387923.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_llama70b_skyworkstulufull__8__1745387923", - "id": "allenai/open_instruct_dev-rm_llama70b_skyworkstulufull__8__1745387923", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_llama70b_skyworkstulufull__8__1745387923/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7573 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8168 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4125 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7049 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8733 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8545 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8814 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_llama_1e-6_1__1__1743896628.json b/data/models/allenai_open_instruct_dev-rm_llama_1e-6_1__1__1743896628.json deleted file mode 100644 index 6ccce8c4f28640d1bccada36472b7c4033d1c345..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_llama_1e-6_1__1__1743896628.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_llama_1e-6_1__1__1743896628", - "id": "allenai/open_instruct_dev-rm_llama_1e-6_1__1__1743896628", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_llama_1e-6_1__1__1743896628/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6637 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6947 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4062 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6284 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8422 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7273 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6834 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_llama_1e-6_1_skyworks__1__1744062999.json b/data/models/allenai_open_instruct_dev-rm_llama_1e-6_1_skyworks__1__1744062999.json deleted file mode 100644 index ca0cc690aa072880ca9a0f9a101320d69cd77989..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_llama_1e-6_1_skyworks__1__1744062999.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_llama_1e-6_1_skyworks__1__1744062999", - "id": "allenai/open_instruct_dev-rm_llama_1e-6_1_skyworks__1__1744062999", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_llama_1e-6_1_skyworks__1__1744062999/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6665 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5979 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3688 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6339 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8956 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8606 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6422 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_llama_1e-6_1_skyworkstulufull__1__1743712777.json b/data/models/allenai_open_instruct_dev-rm_llama_1e-6_1_skyworkstulufull__1__1743712777.json deleted file mode 100644 index 0adfbb570cc1bbb5c3557b2b1b767f406903f309..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_llama_1e-6_1_skyworkstulufull__1__1743712777.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_llama_1e-6_1_skyworkstulufull__1__1743712777", - "id": "allenai/open_instruct_dev-rm_llama_1e-6_1_skyworkstulufull__1__1743712777", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_llama_1e-6_1_skyworkstulufull__1__1743712777/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7038 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6947 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3937 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6557 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8867 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8586 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7331 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_llama_1e-6_2__1__1743896638.json b/data/models/allenai_open_instruct_dev-rm_llama_1e-6_2__1__1743896638.json deleted file mode 100644 index bf444072ccb3c7d89e5ed72115576a86f1c9b6ee..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_llama_1e-6_2__1__1743896638.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_llama_1e-6_2__1__1743896638", - "id": "allenai/open_instruct_dev-rm_llama_1e-6_2__1__1743896638", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_llama_1e-6_2__1__1743896638/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6754 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6716 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6339 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8756 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7737 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6976 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_llama_1e-6_2_skyworkstulufull__1__1743800938.json b/data/models/allenai_open_instruct_dev-rm_llama_1e-6_2_skyworkstulufull__1__1743800938.json deleted file mode 100644 index 40883bc85e5ef02d6ec494a39daa7e6d476cc937..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_llama_1e-6_2_skyworkstulufull__1__1743800938.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_llama_1e-6_2_skyworkstulufull__1__1743800938", - "id": "allenai/open_instruct_dev-rm_llama_1e-6_2_skyworkstulufull__1__1743800938", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_llama_1e-6_2_skyworkstulufull__1__1743800938/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7241 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7305 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6667 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9422 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9414 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6635 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_llama_2e-5_1_skyworkstulufull__1__1743712885.json b/data/models/allenai_open_instruct_dev-rm_llama_2e-5_1_skyworkstulufull__1__1743712885.json deleted file mode 100644 index 720eb8ec92e5deb275480e0d3dcc962695ebc3de..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_llama_2e-5_1_skyworkstulufull__1__1743712885.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_llama_2e-5_1_skyworkstulufull__1__1743712885", - "id": "allenai/open_instruct_dev-rm_llama_2e-5_1_skyworkstulufull__1__1743712885", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_llama_2e-5_1_skyworkstulufull__1__1743712885/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6716 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6632 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3688 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6284 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.82 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8303 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.719 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_llama_2e-5_2_skyworkstulufull__1__1743800773.json b/data/models/allenai_open_instruct_dev-rm_llama_2e-5_2_skyworkstulufull__1__1743800773.json deleted file mode 100644 index c02fb3586871abba7063bb255ee63e3475942e06..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_llama_2e-5_2_skyworkstulufull__1__1743800773.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_llama_2e-5_2_skyworkstulufull__1__1743800773", - "id": "allenai/open_instruct_dev-rm_llama_2e-5_2_skyworkstulufull__1__1743800773", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_llama_2e-5_2_skyworkstulufull__1__1743800773/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6207 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6358 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.375 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5902 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8267 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.802 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4948 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_llama_2e-6_1_skyworkstulufull__1__1743893867.json b/data/models/allenai_open_instruct_dev-rm_llama_2e-6_1_skyworkstulufull__1__1743893867.json deleted file mode 100644 index 41c4fbfbe9482e43afba6a7ae0789f8cd2deeefc..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_llama_2e-6_1_skyworkstulufull__1__1743893867.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_llama_2e-6_1_skyworkstulufull__1__1743893867", - "id": "allenai/open_instruct_dev-rm_llama_2e-6_1_skyworkstulufull__1__1743893867", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_llama_2e-6_1_skyworkstulufull__1__1743893867/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.719 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7263 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3875 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6393 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8956 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9273 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.738 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_llama_3e-6_1__1__1743929424.json b/data/models/allenai_open_instruct_dev-rm_llama_3e-6_1__1__1743929424.json deleted file mode 100644 index aae76fa3bd5b43e9685d6dd5e4391800174f5619..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_llama_3e-6_1__1__1743929424.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_llama_3e-6_1__1__1743929424", - "id": "allenai/open_instruct_dev-rm_llama_3e-6_1__1__1743929424", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_llama_3e-6_1__1__1743929424/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6572 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7305 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3688 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6284 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8289 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.703 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6837 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_llama_3e-6_1__2__1744311395.json b/data/models/allenai_open_instruct_dev-rm_llama_3e-6_1__2__1744311395.json deleted file mode 100644 index 8f4367f81671190cfce04ee68b8692b57c7f44c7..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_llama_3e-6_1__2__1744311395.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_llama_3e-6_1__2__1744311395", - "id": "allenai/open_instruct_dev-rm_llama_3e-6_1__2__1744311395", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_llama_3e-6_1__2__1744311395/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6938 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7537 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.45 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6393 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8667 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7616 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6913 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_llama_3e-6_1__3__1744311491.json b/data/models/allenai_open_instruct_dev-rm_llama_3e-6_1__3__1744311491.json deleted file mode 100644 index 7e401ebba3db599528468f68a2fca6b93c028e0e..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_llama_3e-6_1__3__1744311491.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_llama_3e-6_1__3__1744311491", - "id": "allenai/open_instruct_dev-rm_llama_3e-6_1__3__1744311491", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_llama_3e-6_1__3__1744311491/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6754 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7242 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4062 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6284 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8422 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7535 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6976 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_llama_3e-6_1_skyworks__1__1744062787.json b/data/models/allenai_open_instruct_dev-rm_llama_3e-6_1_skyworks__1__1744062787.json deleted file mode 100644 index 64e3c7d8046305fcb06620862ae8c50ee5ad4f21..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_llama_3e-6_1_skyworks__1__1744062787.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_llama_3e-6_1_skyworks__1__1744062787", - "id": "allenai/open_instruct_dev-rm_llama_3e-6_1_skyworks__1__1744062787", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_llama_3e-6_1_skyworks__1__1744062787/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7045 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6253 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3812 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6667 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.92 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9232 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7109 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_llama_3e-6_1_skyworkstulufull__2__1744311461.json b/data/models/allenai_open_instruct_dev-rm_llama_3e-6_1_skyworkstulufull__2__1744311461.json deleted file mode 100644 index 2d23b44a31abd5ad5031f164e2680e8ac5715a77..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_llama_3e-6_1_skyworkstulufull__2__1744311461.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_llama_3e-6_1_skyworkstulufull__2__1744311461", - "id": "allenai/open_instruct_dev-rm_llama_3e-6_1_skyworkstulufull__2__1744311461", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_llama_3e-6_1_skyworkstulufull__2__1744311461/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7189 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7305 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3937 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6066 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8978 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9374 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7475 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_llama_3e-6_1_skyworkstulufull__3__1744311780.json b/data/models/allenai_open_instruct_dev-rm_llama_3e-6_1_skyworkstulufull__3__1744311780.json deleted file mode 100644 index 0e35e6849e35997f21978592b80c494f0cee2b2b..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_llama_3e-6_1_skyworkstulufull__3__1744311780.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_llama_3e-6_1_skyworkstulufull__3__1744311780", - "id": "allenai/open_instruct_dev-rm_llama_3e-6_1_skyworkstulufull__3__1744311780", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_llama_3e-6_1_skyworkstulufull__3__1744311780/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7172 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7242 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4313 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6175 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8778 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.897 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7555 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_llama_3e-6_2__1__1743896489.json b/data/models/allenai_open_instruct_dev-rm_llama_3e-6_2__1__1743896489.json deleted file mode 100644 index 560627be038e9cdb97a7edc2f43d5e36318fc6de..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_llama_3e-6_2__1__1743896489.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_llama_3e-6_2__1__1743896489", - "id": "allenai/open_instruct_dev-rm_llama_3e-6_2__1__1743896489", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_llama_3e-6_2__1__1743896489/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6813 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7137 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4437 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6284 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8644 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7596 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6781 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_llama_3e-6_2_skyworkstulufull__1__1743800713.json b/data/models/allenai_open_instruct_dev-rm_llama_3e-6_2_skyworkstulufull__1__1743800713.json deleted file mode 100644 index 8b95b1f10c3f6f75ca0c157e0b873e1aa3e0984d..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_llama_3e-6_2_skyworkstulufull__1__1743800713.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_llama_3e-6_2_skyworkstulufull__1__1743800713", - "id": "allenai/open_instruct_dev-rm_llama_3e-6_2_skyworkstulufull__1__1743800713", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_llama_3e-6_2_skyworkstulufull__1__1743800713/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7209 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7116 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3875 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6612 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9067 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9172 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7414 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_llama_4e-6_1_skyworkstulufull__1__1743893911.json b/data/models/allenai_open_instruct_dev-rm_llama_4e-6_1_skyworkstulufull__1__1743893911.json deleted file mode 100644 index 685f00ba964c33a388447591d5edb8bb404da914..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_llama_4e-6_1_skyworkstulufull__1__1743893911.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_llama_4e-6_1_skyworkstulufull__1__1743893911", - "id": "allenai/open_instruct_dev-rm_llama_4e-6_1_skyworkstulufull__1__1743893911", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_llama_4e-6_1_skyworkstulufull__1__1743893911/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7266 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7347 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4313 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6339 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8933 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.897 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7697 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_llamabase_1e-6_1_skyworkstulufull__1__1745386412.json b/data/models/allenai_open_instruct_dev-rm_llamabase_1e-6_1_skyworkstulufull__1__1745386412.json deleted file mode 100644 index 4ac1f0d7cb31e2974ab6ac098058f8a463498c43..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_llamabase_1e-6_1_skyworkstulufull__1__1745386412.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_llamabase_1e-6_1_skyworkstulufull__1__1745386412", - "id": "allenai/open_instruct_dev-rm_llamabase_1e-6_1_skyworkstulufull__1__1745386412", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_llamabase_1e-6_1_skyworkstulufull__1__1745386412/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5342 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6042 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.275 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6284 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7222 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5818 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3935 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_llamabase_1e-6_2_skyworkstulufull__1__1745441922.json b/data/models/allenai_open_instruct_dev-rm_llamabase_1e-6_2_skyworkstulufull__1__1745441922.json deleted file mode 100644 index a6f98de4e4c540fa170f8eabd23155d97d4b582d..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_llamabase_1e-6_2_skyworkstulufull__1__1745441922.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_llamabase_1e-6_2_skyworkstulufull__1__1745441922", - "id": "allenai/open_instruct_dev-rm_llamabase_1e-6_2_skyworkstulufull__1__1745441922", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_llamabase_1e-6_2_skyworkstulufull__1__1745441922/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6111 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6884 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3063 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.623 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8289 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7576 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4628 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_llamabase_2e-5_1_skyworkstulufull__1__1745386495.json b/data/models/allenai_open_instruct_dev-rm_llamabase_2e-5_1_skyworkstulufull__1__1745386495.json deleted file mode 100644 index c4e51c5237e4784bbfc71a7b47aa8ab7a8f7e179..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_llamabase_2e-5_1_skyworkstulufull__1__1745386495.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_llamabase_2e-5_1_skyworkstulufull__1__1745386495", - "id": "allenai/open_instruct_dev-rm_llamabase_2e-5_1_skyworkstulufull__1__1745386495", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_llamabase_2e-5_1_skyworkstulufull__1__1745386495/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5825 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6379 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.325 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5355 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8222 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7051 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4691 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_llamabase_2e-5_2_skyworkstulufull__1__1745386507.json b/data/models/allenai_open_instruct_dev-rm_llamabase_2e-5_2_skyworkstulufull__1__1745386507.json deleted file mode 100644 index da411b467e2d37871e6640d4027842a7764c2309..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_llamabase_2e-5_2_skyworkstulufull__1__1745386507.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_llamabase_2e-5_2_skyworkstulufull__1__1745386507", - "id": "allenai/open_instruct_dev-rm_llamabase_2e-5_2_skyworkstulufull__1__1745386507", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_llamabase_2e-5_2_skyworkstulufull__1__1745386507/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5598 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5495 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3563 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5902 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.76 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7273 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3754 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_llamabase_3e-6_1_skyworkstulufull__1__1745386507.json b/data/models/allenai_open_instruct_dev-rm_llamabase_3e-6_1_skyworkstulufull__1__1745386507.json deleted file mode 100644 index 8df6045289835b1813d41a24a419572bc5e7bb2b..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_llamabase_3e-6_1_skyworkstulufull__1__1745386507.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_llamabase_3e-6_1_skyworkstulufull__1__1745386507", - "id": "allenai/open_instruct_dev-rm_llamabase_3e-6_1_skyworkstulufull__1__1745386507", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_llamabase_3e-6_1_skyworkstulufull__1__1745386507/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6101 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6632 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.35 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6175 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7778 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7111 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5408 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_qwen32b_1e-6_skyworkstulufull__8__1748235917.json b/data/models/allenai_open_instruct_dev-rm_qwen32b_1e-6_skyworkstulufull__8__1748235917.json deleted file mode 100644 index 6e20663703e7b4b4c9e1f0f70df6401d41e62d28..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_qwen32b_1e-6_skyworkstulufull__8__1748235917.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_qwen32b_1e-6_skyworkstulufull__8__1748235917", - "id": "allenai/open_instruct_dev-rm_qwen32b_1e-6_skyworkstulufull__8__1748235917", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_qwen32b_1e-6_skyworkstulufull__8__1748235917/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7185 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7305 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4125 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7158 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7933 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8545 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.804 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_qwen32b_3e-6_skyworkstulufull__8__1748288961.json b/data/models/allenai_open_instruct_dev-rm_qwen32b_3e-6_skyworkstulufull__8__1748288961.json deleted file mode 100644 index 5fd1bed34cb439f24dc849390c2363d705b4b75c..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_qwen32b_3e-6_skyworkstulufull__8__1748288961.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_qwen32b_3e-6_skyworkstulufull__8__1748288961", - "id": "allenai/open_instruct_dev-rm_qwen32b_3e-6_skyworkstulufull__8__1748288961", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_qwen32b_3e-6_skyworkstulufull__8__1748288961/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7325 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7474 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4437 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7158 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7978 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8141 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8763 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_qwen_1e-6_1_skyworks__1__1744062830.json b/data/models/allenai_open_instruct_dev-rm_qwen_1e-6_1_skyworks__1__1744062830.json deleted file mode 100644 index 27560fb3500740cc8a386cbfc89640dd6d5922cc..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_qwen_1e-6_1_skyworks__1__1744062830.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_qwen_1e-6_1_skyworks__1__1744062830", - "id": "allenai/open_instruct_dev-rm_qwen_1e-6_1_skyworks__1__1744062830", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_qwen_1e-6_1_skyworks__1__1744062830/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6022 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5284 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.325 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.694 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7556 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7616 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5486 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_qwen_1e-6_1_skyworks__2__1744576024.json b/data/models/allenai_open_instruct_dev-rm_qwen_1e-6_1_skyworks__2__1744576024.json deleted file mode 100644 index 4f38d2445f00ce666aba0e02e6d7424ae08a12f9..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_qwen_1e-6_1_skyworks__2__1744576024.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_qwen_1e-6_1_skyworks__2__1744576024", - "id": "allenai/open_instruct_dev-rm_qwen_1e-6_1_skyworks__2__1744576024", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_qwen_1e-6_1_skyworks__2__1744576024/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5948 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5579 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2875 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6776 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.72 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7394 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5863 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_qwen_1e-6_1_skyworkstulufull__1__1743712914.json b/data/models/allenai_open_instruct_dev-rm_qwen_1e-6_1_skyworkstulufull__1__1743712914.json deleted file mode 100644 index 3e1999e897953faaced2e7df692276336e85aff7..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_qwen_1e-6_1_skyworkstulufull__1__1743712914.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_qwen_1e-6_1_skyworkstulufull__1__1743712914", - "id": "allenai/open_instruct_dev-rm_qwen_1e-6_1_skyworkstulufull__1__1743712914", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_qwen_1e-6_1_skyworkstulufull__1__1743712914/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6492 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6084 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.35 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6776 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.76 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.699 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_qwen_2e-5_1_skyworkstulufull__1__1743713091.json b/data/models/allenai_open_instruct_dev-rm_qwen_2e-5_1_skyworkstulufull__1__1743713091.json deleted file mode 100644 index 96fdd34d6322e1f9676a4c9f4be93204c00f2dab..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_qwen_2e-5_1_skyworkstulufull__1__1743713091.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_qwen_2e-5_1_skyworkstulufull__1__1743713091", - "id": "allenai/open_instruct_dev-rm_qwen_2e-5_1_skyworkstulufull__1__1743713091", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_qwen_2e-5_1_skyworkstulufull__1__1743713091/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6764 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7074 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6885 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8622 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.802 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6984 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_qwen_3e-6_1_skyworks__1__1744062829.json b/data/models/allenai_open_instruct_dev-rm_qwen_3e-6_1_skyworks__1__1744062829.json deleted file mode 100644 index 1d00390444c4545efa7c0a21f22143e9f27b1916..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_qwen_3e-6_1_skyworks__1__1744062829.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_qwen_3e-6_1_skyworks__1__1744062829", - "id": "allenai/open_instruct_dev-rm_qwen_3e-6_1_skyworks__1__1744062829", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_qwen_3e-6_1_skyworks__1__1744062829/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6408 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6337 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3063 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6831 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8467 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8222 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5529 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_qwen_3e-6_1_skyworks__2__1744576050.json b/data/models/allenai_open_instruct_dev-rm_qwen_3e-6_1_skyworks__2__1744576050.json deleted file mode 100644 index c0d9b52843cd46914f2245a3ba324ad462723b0a..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_qwen_3e-6_1_skyworks__2__1744576050.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_qwen_3e-6_1_skyworks__2__1744576050", - "id": "allenai/open_instruct_dev-rm_qwen_3e-6_1_skyworks__2__1744576050", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_qwen_3e-6_1_skyworks__2__1744576050/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6452 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6063 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3187 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7158 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8356 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8343 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5603 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_qwen_3e-6_1_skyworkstulufull__1__1743712916.json b/data/models/allenai_open_instruct_dev-rm_qwen_3e-6_1_skyworkstulufull__1__1743712916.json deleted file mode 100644 index 8c8db349aa9c80ed07d47252755ccbeb61539bf1..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_qwen_3e-6_1_skyworkstulufull__1__1743712916.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_qwen_3e-6_1_skyworkstulufull__1__1743712916", - "id": "allenai/open_instruct_dev-rm_qwen_3e-6_1_skyworkstulufull__1__1743712916", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_qwen_3e-6_1_skyworkstulufull__1__1743712916/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7013 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7263 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3438 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6995 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8222 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8444 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7714 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_qwen_3e-6_2__1__1743023576.json b/data/models/allenai_open_instruct_dev-rm_qwen_3e-6_2__1__1743023576.json deleted file mode 100644 index ca563a42b81236fc6fd27e1b944a2773ee7c69b8..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_qwen_3e-6_2__1__1743023576.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_qwen_3e-6_2__1__1743023576", - "id": "allenai/open_instruct_dev-rm_qwen_3e-6_2__1__1743023576", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_qwen_3e-6_2__1__1743023576/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6369 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6905 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3187 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6448 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7844 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7596 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6236 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_qwen_3e-6_3__1__1743023619.json b/data/models/allenai_open_instruct_dev-rm_qwen_3e-6_3__1__1743023619.json deleted file mode 100644 index 523344800833b038e057fc0d5910afc75d70ffd8..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_qwen_3e-6_3__1__1743023619.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_qwen_3e-6_3__1__1743023619", - "id": "allenai/open_instruct_dev-rm_qwen_3e-6_3__1__1743023619", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_qwen_3e-6_3__1__1743023619/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6221 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6674 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.325 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.612 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7978 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7455 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5852 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_qwenbase_1e-6_1_skyworkstulufull__1__1745388583.json b/data/models/allenai_open_instruct_dev-rm_qwenbase_1e-6_1_skyworkstulufull__1__1745388583.json deleted file mode 100644 index 5d6da96df46732ab9b31130a6ab021415f1bc130..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_qwenbase_1e-6_1_skyworkstulufull__1__1745388583.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_qwenbase_1e-6_1_skyworkstulufull__1__1745388583", - "id": "allenai/open_instruct_dev-rm_qwenbase_1e-6_1_skyworkstulufull__1__1745388583", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_qwenbase_1e-6_1_skyworkstulufull__1__1745388583/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5735 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5895 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2625 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6448 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6889 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6727 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5823 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_qwenbase_1e-6_2_skyworkstulufull__1__1745388604.json b/data/models/allenai_open_instruct_dev-rm_qwenbase_1e-6_2_skyworkstulufull__1__1745388604.json deleted file mode 100644 index 5794a0306d7fcd84d21758a8dbe7779afdc5c3aa..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_qwenbase_1e-6_2_skyworkstulufull__1__1745388604.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_qwenbase_1e-6_2_skyworkstulufull__1__1745388604", - "id": "allenai/open_instruct_dev-rm_qwenbase_1e-6_2_skyworkstulufull__1__1745388604", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_qwenbase_1e-6_2_skyworkstulufull__1__1745388604/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6336 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6337 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3063 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6885 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7244 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.802 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6465 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_qwenbase_2e-5_1_skyworkstulufull__1__1745388738.json b/data/models/allenai_open_instruct_dev-rm_qwenbase_2e-5_1_skyworkstulufull__1__1745388738.json deleted file mode 100644 index ab03bdd9bb2725b8f813e6f14a49bf8279651abf..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_qwenbase_2e-5_1_skyworkstulufull__1__1745388738.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_qwenbase_2e-5_1_skyworkstulufull__1__1745388738", - "id": "allenai/open_instruct_dev-rm_qwenbase_2e-5_1_skyworkstulufull__1__1745388738", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_qwenbase_2e-5_1_skyworkstulufull__1__1745388738/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6824 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6989 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3625 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6831 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8311 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8081 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7107 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_qwenbase_2e-5_2_skyworkstulufull__1__1745388191.json b/data/models/allenai_open_instruct_dev-rm_qwenbase_2e-5_2_skyworkstulufull__1__1745388191.json deleted file mode 100644 index 2c1a5596b6113dc496a56c9232914269afb366ad..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_qwenbase_2e-5_2_skyworkstulufull__1__1745388191.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_qwenbase_2e-5_2_skyworkstulufull__1__1745388191", - "id": "allenai/open_instruct_dev-rm_qwenbase_2e-5_2_skyworkstulufull__1__1745388191", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_qwenbase_2e-5_2_skyworkstulufull__1__1745388191/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6392 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6589 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3312 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6995 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7933 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7717 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5804 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_qwenbase_3e-6_1_skyworkstulufull__1__1745388737.json b/data/models/allenai_open_instruct_dev-rm_qwenbase_3e-6_1_skyworkstulufull__1__1745388737.json deleted file mode 100644 index 1dea5e16a45e0fe4d72a629f9caf1fad94f38040..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_qwenbase_3e-6_1_skyworkstulufull__1__1745388737.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_qwenbase_3e-6_1_skyworkstulufull__1__1745388737", - "id": "allenai/open_instruct_dev-rm_qwenbase_3e-6_1_skyworkstulufull__1__1745388737", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_qwenbase_3e-6_1_skyworkstulufull__1__1745388737/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.664 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6821 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3312 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6448 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8133 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8061 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7066 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_qwenbase_3e-6_2_skyworkstulufull__1__1745388138.json b/data/models/allenai_open_instruct_dev-rm_qwenbase_3e-6_2_skyworkstulufull__1__1745388138.json deleted file mode 100644 index fd8f91433d118460e0e4d9ffe081af929d9c8175..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_qwenbase_3e-6_2_skyworkstulufull__1__1745388138.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_qwenbase_3e-6_2_skyworkstulufull__1__1745388138", - "id": "allenai/open_instruct_dev-rm_qwenbase_3e-6_2_skyworkstulufull__1__1745388138", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_qwenbase_3e-6_2_skyworkstulufull__1__1745388138/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6678 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6505 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3312 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6831 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7978 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8808 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6632 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_tulu3_70b_1__8__1742924455.json b/data/models/allenai_open_instruct_dev-rm_tulu3_70b_1__8__1742924455.json deleted file mode 100644 index 698ee292859bd7a43d9d278532f4840b4d1a2618..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_tulu3_70b_1__8__1742924455.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_tulu3_70b_1__8__1742924455", - "id": "allenai/open_instruct_dev-rm_tulu3_70b_1__8__1742924455", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_tulu3_70b_1__8__1742924455/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6618 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7958 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.325 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6557 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8311 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6323 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7311 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_open_instruct_dev-rm_tulu3_70b_2__8__1742982964.json b/data/models/allenai_open_instruct_dev-rm_tulu3_70b_2__8__1742982964.json deleted file mode 100644 index 60420020714af41796792dafc43c2a34122c2830..0000000000000000000000000000000000000000 --- a/data/models/allenai_open_instruct_dev-rm_tulu3_70b_2__8__1742982964.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "allenai/open_instruct_dev-rm_tulu3_70b_2__8__1742982964", - "id": "allenai/open_instruct_dev-rm_tulu3_70b_2__8__1742982964", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/allenai_open_instruct_dev-rm_tulu3_70b_2__8__1742982964/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6605 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7789 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3688 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6448 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8844 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6667 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6195 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_tulu-2-dpo-13b.json b/data/models/allenai_tulu-2-dpo-13b.json deleted file mode 100644 index 6a0d24a07d045b191cda81569cdbe2b781e83839..0000000000000000000000000000000000000000 --- a/data/models/allenai_tulu-2-dpo-13b.json +++ /dev/null @@ -1,144 +0,0 @@ -{ - "model_info": { - "name": "allenai/tulu-2-dpo-13b", - "id": "allenai/tulu-2-dpo-13b", - "developer": "allenai", - "additional_details": { - "model_type": "DPO" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/allenai_tulu-2-dpo-13b/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7368 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9581 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5833 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7946 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7323 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4947 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_tulu-2-dpo-70b.json b/data/models/allenai_tulu-2-dpo-70b.json deleted file mode 100644 index 324a1b1058d54c1026887ca17c5453fadcfaa559..0000000000000000000000000000000000000000 --- a/data/models/allenai_tulu-2-dpo-70b.json +++ /dev/null @@ -1,144 +0,0 @@ -{ - "model_info": { - "name": "allenai/tulu-2-dpo-70b", - "id": "allenai/tulu-2-dpo-70b", - "developer": "allenai", - "additional_details": { - "model_type": "DPO" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/allenai_tulu-2-dpo-70b/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7621 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9749 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6053 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8446 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7407 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5278 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_tulu-2-dpo-7b.json b/data/models/allenai_tulu-2-dpo-7b.json deleted file mode 100644 index 58b7de2c7e3f0793e57202838ec2f74f45b7ad99..0000000000000000000000000000000000000000 --- a/data/models/allenai_tulu-2-dpo-7b.json +++ /dev/null @@ -1,144 +0,0 @@ -{ - "model_info": { - "name": "allenai/tulu-2-dpo-7b", - "id": "allenai/tulu-2-dpo-7b", - "developer": "allenai", - "additional_details": { - "model_type": "DPO" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/allenai_tulu-2-dpo-7b/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7212 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9749 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5614 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7527 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7176 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4774 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_tulu-v2.5-13b-preference-mix-rm.json b/data/models/allenai_tulu-v2.5-13b-preference-mix-rm.json deleted file mode 100644 index db6fec44bf3e1e04a9ba4d390a235550d4f4c435..0000000000000000000000000000000000000000 --- a/data/models/allenai_tulu-v2.5-13b-preference-mix-rm.json +++ /dev/null @@ -1,144 +0,0 @@ -{ - "model_info": { - "name": "allenai/tulu-v2.5-13b-preference-mix-rm", - "id": "allenai/tulu-v2.5-13b-preference-mix-rm", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/allenai_tulu-v2.5-13b-preference-mix-rm/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8027 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9358 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.682 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.773 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.885 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6724 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_tulu-v2.5-13b-uf-rm.json b/data/models/allenai_tulu-v2.5-13b-uf-rm.json deleted file mode 100644 index 5405f2930187a406281dfce8b7d80314d9d2cde8..0000000000000000000000000000000000000000 --- a/data/models/allenai_tulu-v2.5-13b-uf-rm.json +++ /dev/null @@ -1,144 +0,0 @@ -{ - "model_info": { - "name": "allenai/tulu-v2.5-13b-uf-rm", - "id": "allenai/tulu-v2.5-13b-uf-rm", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/allenai_tulu-v2.5-13b-uf-rm/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4806 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3939 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4232 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5554 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4737 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6326 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_tulu-v2.5-70b-preference-mix-rm.json b/data/models/allenai_tulu-v2.5-70b-preference-mix-rm.json deleted file mode 100644 index 4b8b124bca66c5d510da8b64bee3e4ab171b9d0d..0000000000000000000000000000000000000000 --- a/data/models/allenai_tulu-v2.5-70b-preference-mix-rm.json +++ /dev/null @@ -1,144 +0,0 @@ -{ - "model_info": { - "name": "allenai/tulu-v2.5-70b-preference-mix-rm", - "id": "allenai/tulu-v2.5-70b-preference-mix-rm", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/allenai_tulu-v2.5-70b-preference-mix-rm/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6516 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7737 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5921 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8486 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4138 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6079 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allenai_tulu-v2.5-70b-uf-rm.json b/data/models/allenai_tulu-v2.5-70b-uf-rm.json deleted file mode 100644 index 7392e4e0e2a0994bf601bd62496014a58f4a7b12..0000000000000000000000000000000000000000 --- a/data/models/allenai_tulu-v2.5-70b-uf-rm.json +++ /dev/null @@ -1,144 +0,0 @@ -{ - "model_info": { - "name": "allenai/tulu-v2.5-70b-uf-rm", - "id": "allenai/tulu-v2.5-70b-uf-rm", - "developer": "allenai", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/allenai_tulu-v2.5-70b-uf-rm/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7398 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8659 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7171 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7014 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.757 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5757 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allknowingroger_chocolatine-24b.json b/data/models/allknowingroger_chocolatine-24b.json deleted file mode 100644 index 4b4dfa74a875eeb95275facc2503b3471c9ee93d..0000000000000000000000000000000000000000 --- a/data/models/allknowingroger_chocolatine-24b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Chocolatine-24B", - "id": "allknowingroger/Chocolatine-24B", - "developer": "allknowingroger", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Phi3ForCausalLM", - "params_billions": "24.184" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allknowingroger_Chocolatine-24B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1958 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6191 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0008 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3255 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4323 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4566 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allknowingroger_gemma2slerp1-2.6b.json b/data/models/allknowingroger_gemma2slerp1-2.6b.json deleted file mode 100644 index ec3edbb1b073f4c9a9ac554070a609f6cbc0029b..0000000000000000000000000000000000000000 --- a/data/models/allknowingroger_gemma2slerp1-2.6b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Gemma2Slerp1-2.6B", - "id": "allknowingroger/Gemma2Slerp1-2.6B", - "developer": "allknowingroger", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "2.614" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allknowingroger_Gemma2Slerp1-2.6B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5354 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4343 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1065 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2836 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4562 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2689 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allknowingroger_gemma2slerp1-27b.json b/data/models/allknowingroger_gemma2slerp1-27b.json deleted file mode 100644 index 4e5c781c7b2d2f2ddc820a771dfbb68dd32abc68..0000000000000000000000000000000000000000 --- a/data/models/allknowingroger_gemma2slerp1-27b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Gemma2Slerp1-27B", - "id": "allknowingroger/Gemma2Slerp1-27B", - "developer": "allknowingroger", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "27.227" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allknowingroger_Gemma2Slerp1-27B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7186 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6399 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2583 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3641 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4767 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4456 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allknowingroger_gemma2slerp2-2.6b.json b/data/models/allknowingroger_gemma2slerp2-2.6b.json deleted file mode 100644 index e802b52bbe067b9de17a7db73f4a3d4639df6c7b..0000000000000000000000000000000000000000 --- a/data/models/allknowingroger_gemma2slerp2-2.6b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Gemma2Slerp2-2.6B", - "id": "allknowingroger/Gemma2Slerp2-2.6B", - "developer": "allknowingroger", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "2.614" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allknowingroger_Gemma2Slerp2-2.6B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5747 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4308 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0906 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3054 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4468 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2696 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allknowingroger_gemma2slerp2-27b.json b/data/models/allknowingroger_gemma2slerp2-27b.json deleted file mode 100644 index e2dc66f3ef79af49f5fd02b2aa07427b42c4d21d..0000000000000000000000000000000000000000 --- a/data/models/allknowingroger_gemma2slerp2-27b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Gemma2Slerp2-27B", - "id": "allknowingroger/Gemma2Slerp2-27B", - "developer": "allknowingroger", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "27.227" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allknowingroger_Gemma2Slerp2-27B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7546 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6557 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2787 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.37 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4621 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4623 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allknowingroger_gemma2slerp3-27b.json b/data/models/allknowingroger_gemma2slerp3-27b.json deleted file mode 100644 index eb620647ac6200498438cce452e73c1aee385cbf..0000000000000000000000000000000000000000 --- a/data/models/allknowingroger_gemma2slerp3-27b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Gemma2Slerp3-27B", - "id": "allknowingroger/Gemma2Slerp3-27B", - "developer": "allknowingroger", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "27.227" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allknowingroger_Gemma2Slerp3-27B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7426 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.65 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2742 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3549 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.474 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4641 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allknowingroger_gemma2slerp4-27b.json b/data/models/allknowingroger_gemma2slerp4-27b.json deleted file mode 100644 index 1382fe4b0afab7a5c85d3c91ccbf1e21860540f4..0000000000000000000000000000000000000000 --- a/data/models/allknowingroger_gemma2slerp4-27b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Gemma2Slerp4-27B", - "id": "allknowingroger/Gemma2Slerp4-27B", - "developer": "allknowingroger", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "27.227" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allknowingroger_Gemma2Slerp4-27B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7497 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.653 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2719 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3666 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4502 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4649 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allknowingroger_gemmaslerp-9b.json b/data/models/allknowingroger_gemmaslerp-9b.json deleted file mode 100644 index 580f202f00b8b11a6434b0ff868c12b8d75e09d9..0000000000000000000000000000000000000000 --- a/data/models/allknowingroger_gemmaslerp-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "GemmaSlerp-9B", - "id": "allknowingroger/GemmaSlerp-9B", - "developer": "allknowingroger", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "9.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allknowingroger_GemmaSlerp-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7043 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5921 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.216 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.344 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4673 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4161 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allknowingroger_gemmaslerp2-9b.json b/data/models/allknowingroger_gemmaslerp2-9b.json deleted file mode 100644 index fb64c6b0c453d98f7bea57fd38bf3a614f9190c4..0000000000000000000000000000000000000000 --- a/data/models/allknowingroger_gemmaslerp2-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "GemmaSlerp2-9B", - "id": "allknowingroger/GemmaSlerp2-9B", - "developer": "allknowingroger", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "9.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allknowingroger_GemmaSlerp2-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7281 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5983 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2107 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3523 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4767 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4239 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allknowingroger_gemmaslerp4-10b.json b/data/models/allknowingroger_gemmaslerp4-10b.json deleted file mode 100644 index 35efe602e60629db99a999e3fe2a5f544d82fd49..0000000000000000000000000000000000000000 --- a/data/models/allknowingroger_gemmaslerp4-10b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "GemmaSlerp4-10B", - "id": "allknowingroger/GemmaSlerp4-10B", - "developer": "allknowingroger", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allknowingroger_GemmaSlerp4-10B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7326 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6028 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2243 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3532 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.454 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.425 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allknowingroger_gemmaslerp5-10b.json b/data/models/allknowingroger_gemmaslerp5-10b.json deleted file mode 100644 index 13492bd2974f275b643ccf88fc54350723dc66eb..0000000000000000000000000000000000000000 --- a/data/models/allknowingroger_gemmaslerp5-10b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "GemmaSlerp5-10B", - "id": "allknowingroger/GemmaSlerp5-10B", - "developer": "allknowingroger", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allknowingroger_GemmaSlerp5-10B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7353 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6054 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2183 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3523 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4608 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4328 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allknowingroger_gemmastock1-27b.json b/data/models/allknowingroger_gemmastock1-27b.json deleted file mode 100644 index 2564fd7d0f08fe5a353735ea40ad232a11af519c..0000000000000000000000000000000000000000 --- a/data/models/allknowingroger_gemmastock1-27b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "GemmaStock1-27B", - "id": "allknowingroger/GemmaStock1-27B", - "developer": "allknowingroger", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "27.227" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allknowingroger_GemmaStock1-27B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7509 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6566 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2636 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3641 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4527 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.473 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allknowingroger_homerslerp1-7b.json b/data/models/allknowingroger_homerslerp1-7b.json deleted file mode 100644 index ebbf69dd7ae95d1d32bd5086c38f09761070e42c..0000000000000000000000000000000000000000 --- a/data/models/allknowingroger_homerslerp1-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "HomerSlerp1-7B", - "id": "allknowingroger/HomerSlerp1-7B", - "developer": "allknowingroger", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allknowingroger_HomerSlerp1-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4621 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5518 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2719 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.318 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4359 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4504 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allknowingroger_homerslerp2-7b.json b/data/models/allknowingroger_homerslerp2-7b.json deleted file mode 100644 index 9ad68c74a0ea2e4da08ac0df5de988ea1a49a0a6..0000000000000000000000000000000000000000 --- a/data/models/allknowingroger_homerslerp2-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "HomerSlerp2-7B", - "id": "allknowingroger/HomerSlerp2-7B", - "developer": "allknowingroger", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allknowingroger_HomerSlerp2-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4487 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5649 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2968 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3196 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4356 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4515 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allknowingroger_homerslerp3-7b.json b/data/models/allknowingroger_homerslerp3-7b.json deleted file mode 100644 index 618917c7749e4acd9c14d214eedabf58e5df0b52..0000000000000000000000000000000000000000 --- a/data/models/allknowingroger_homerslerp3-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "HomerSlerp3-7B", - "id": "allknowingroger/HomerSlerp3-7B", - "developer": "allknowingroger", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allknowingroger_HomerSlerp3-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4363 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5598 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3021 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3171 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4462 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4535 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allknowingroger_homerslerp4-7b.json b/data/models/allknowingroger_homerslerp4-7b.json deleted file mode 100644 index 3328db99f6459d5a5aeea2a50614e3e82cf5f9b0..0000000000000000000000000000000000000000 --- a/data/models/allknowingroger_homerslerp4-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "HomerSlerp4-7B", - "id": "allknowingroger/HomerSlerp4-7B", - "developer": "allknowingroger", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allknowingroger_HomerSlerp4-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4374 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5571 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.327 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3196 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4408 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4472 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allknowingroger_limyclown-7b-slerp.json b/data/models/allknowingroger_limyclown-7b-slerp.json deleted file mode 100644 index 35072fd16b740c5e2aab90ddee4afa19f110a7c1..0000000000000000000000000000000000000000 --- a/data/models/allknowingroger_limyclown-7b-slerp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "limyClown-7B-slerp", - "id": "allknowingroger/limyClown-7B-slerp", - "developer": "allknowingroger", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allknowingroger_limyClown-7B-slerp/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4017 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5148 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0687 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.281 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4293 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3038 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allknowingroger_limyqstar-7b-slerp.json b/data/models/allknowingroger_limyqstar-7b-slerp.json deleted file mode 100644 index 3d22c072bf4fcdccecb232ff723bced5a1a66a42..0000000000000000000000000000000000000000 --- a/data/models/allknowingroger_limyqstar-7b-slerp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "LimyQstar-7B-slerp", - "id": "allknowingroger/LimyQstar-7B-slerp", - "developer": "allknowingroger", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allknowingroger_LimyQstar-7B-slerp/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3491 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5024 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0687 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2987 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4146 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3103 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allknowingroger_llama3-jallabi-40b-s.json b/data/models/allknowingroger_llama3-jallabi-40b-s.json deleted file mode 100644 index c99d4b3d3268083aeb114be54dbe0e42e534c0bc..0000000000000000000000000000000000000000 --- a/data/models/allknowingroger_llama3-jallabi-40b-s.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "llama3-Jallabi-40B-s", - "id": "allknowingroger/llama3-Jallabi-40B-s", - "developer": "allknowingroger", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "18.769" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allknowingroger_llama3-Jallabi-40B-s/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1921 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3252 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2374 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.375 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1088 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allknowingroger_llama3.1-60b.json b/data/models/allknowingroger_llama3.1-60b.json deleted file mode 100644 index 90d8b5de89eb57454edae5507660f1311c4c9e8d..0000000000000000000000000000000000000000 --- a/data/models/allknowingroger_llama3.1-60b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama3.1-60B", - "id": "allknowingroger/Llama3.1-60B", - "developer": "allknowingroger", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "61.997" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allknowingroger_Llama3.1-60B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1815 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3242 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2945 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3596 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.331 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allknowingroger_llama3anfeng-40b.json b/data/models/allknowingroger_llama3anfeng-40b.json deleted file mode 100644 index 692f3b31f33956c2ffab8dde2fb4243c576ba069..0000000000000000000000000000000000000000 --- a/data/models/allknowingroger_llama3anfeng-40b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "llama3AnFeng-40B", - "id": "allknowingroger/llama3AnFeng-40B", - "developer": "allknowingroger", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "39.971" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allknowingroger_llama3AnFeng-40B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1742 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3794 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3062 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.394 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.198 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allknowingroger_marco-01-slerp1-7b.json b/data/models/allknowingroger_marco-01-slerp1-7b.json deleted file mode 100644 index a9745ade132ef0d36f00c1f1042583dfbf0fb66a..0000000000000000000000000000000000000000 --- a/data/models/allknowingroger_marco-01-slerp1-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Marco-01-slerp1-7B", - "id": "allknowingroger/Marco-01-slerp1-7B", - "developer": "allknowingroger", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allknowingroger_Marco-01-slerp1-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4681 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5541 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3157 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3171 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4452 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4483 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allknowingroger_meme-7b-slerp.json b/data/models/allknowingroger_meme-7b-slerp.json deleted file mode 100644 index 4ca2c413ef730e1579cc76ac6c5922e20e323e06..0000000000000000000000000000000000000000 --- a/data/models/allknowingroger_meme-7b-slerp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Meme-7B-slerp", - "id": "allknowingroger/Meme-7B-slerp", - "developer": "allknowingroger", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allknowingroger_Meme-7B-slerp/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5164 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4661 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0438 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2861 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4223 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.281 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allknowingroger_ministral-8b-slerp.json b/data/models/allknowingroger_ministral-8b-slerp.json deleted file mode 100644 index 62f4e54a2bb60fd3cb0663fbb05c2e840d850520..0000000000000000000000000000000000000000 --- a/data/models/allknowingroger_ministral-8b-slerp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Ministral-8B-slerp", - "id": "allknowingroger/Ministral-8B-slerp", - "developer": "allknowingroger", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "7.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allknowingroger_Ministral-8B-slerp/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1961 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4686 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0038 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3121 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4285 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3119 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allknowingroger_mistralmash1-7b-s.json b/data/models/allknowingroger_mistralmash1-7b-s.json deleted file mode 100644 index a5654b71f5feb10318c2d6cd7516e884d032a9e9..0000000000000000000000000000000000000000 --- a/data/models/allknowingroger_mistralmash1-7b-s.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mistralmash1-7B-s", - "id": "allknowingroger/Mistralmash1-7B-s", - "developer": "allknowingroger", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allknowingroger_Mistralmash1-7B-s/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3961 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5277 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0921 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2945 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4267 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3293 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allknowingroger_mistralmash2-7b-s.json b/data/models/allknowingroger_mistralmash2-7b-s.json deleted file mode 100644 index 592b64ea2c78e5aedbc876083510f0166801c024..0000000000000000000000000000000000000000 --- a/data/models/allknowingroger_mistralmash2-7b-s.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mistralmash2-7B-s", - "id": "allknowingroger/Mistralmash2-7B-s", - "developer": "allknowingroger", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allknowingroger_Mistralmash2-7B-s/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4102 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5305 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0793 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2978 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4372 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3345 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allknowingroger_mistralphi3-11b.json b/data/models/allknowingroger_mistralphi3-11b.json deleted file mode 100644 index 33e2329f2010eb772827752cf902b24d247f9551..0000000000000000000000000000000000000000 --- a/data/models/allknowingroger_mistralphi3-11b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MistralPhi3-11B", - "id": "allknowingroger/MistralPhi3-11B", - "developer": "allknowingroger", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "11.234" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allknowingroger_MistralPhi3-11B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1943 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6234 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3322 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4267 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4688 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allknowingroger_mixtao-19b-pass.json b/data/models/allknowingroger_mixtao-19b-pass.json deleted file mode 100644 index 0a82454e08cb0c008b8331d44084f788ae362ebc..0000000000000000000000000000000000000000 --- a/data/models/allknowingroger_mixtao-19b-pass.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MixTAO-19B-pass", - "id": "allknowingroger/MixTAO-19B-pass", - "developer": "allknowingroger", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MixtralForCausalLM", - "params_billions": "19.188" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allknowingroger_MixTAO-19B-pass/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3814 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5128 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0612 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2844 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4783 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3105 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allknowingroger_mixtaotruthful-13b-slerp.json b/data/models/allknowingroger_mixtaotruthful-13b-slerp.json deleted file mode 100644 index 12249203cfcf88e8125857bc99c6273a863d33c4..0000000000000000000000000000000000000000 --- a/data/models/allknowingroger_mixtaotruthful-13b-slerp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MixTaoTruthful-13B-slerp", - "id": "allknowingroger/MixTaoTruthful-13B-slerp", - "developer": "allknowingroger", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MixtralForCausalLM", - "params_billions": "12.879" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allknowingroger_MixTaoTruthful-13B-slerp/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4139 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5207 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0665 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2844 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4292 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.31 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allknowingroger_multicalm-7b-slerp.json b/data/models/allknowingroger_multicalm-7b-slerp.json deleted file mode 100644 index 98ed3b1fc77c3b82bdec48cbe75a67cfa591c913..0000000000000000000000000000000000000000 --- a/data/models/allknowingroger_multicalm-7b-slerp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MultiCalm-7B-slerp", - "id": "allknowingroger/MultiCalm-7B-slerp", - "developer": "allknowingroger", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allknowingroger_MultiCalm-7B-slerp/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3927 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5122 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0619 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2827 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4319 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3033 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allknowingroger_multimash-12b-slerp.json b/data/models/allknowingroger_multimash-12b-slerp.json deleted file mode 100644 index 1beb847cb66c170f57b9cbef158f68a6699470a7..0000000000000000000000000000000000000000 --- a/data/models/allknowingroger_multimash-12b-slerp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MultiMash-12B-slerp", - "id": "allknowingroger/MultiMash-12B-slerp", - "developer": "allknowingroger", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MixtralForCausalLM", - "params_billions": "12.879" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allknowingroger_MultiMash-12B-slerp/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3974 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5142 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0808 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2768 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4438 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3068 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allknowingroger_multimash10-13b-slerp.json b/data/models/allknowingroger_multimash10-13b-slerp.json deleted file mode 100644 index 179ba4c9a7d29ea1bacc434e10cfbce1bd7fc9e8..0000000000000000000000000000000000000000 --- a/data/models/allknowingroger_multimash10-13b-slerp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MultiMash10-13B-slerp", - "id": "allknowingroger/MultiMash10-13B-slerp", - "developer": "allknowingroger", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MixtralForCausalLM", - "params_billions": "12.879" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allknowingroger_MultiMash10-13B-slerp/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4163 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5186 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0718 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2861 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4318 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3117 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allknowingroger_multimash11-13b-slerp.json b/data/models/allknowingroger_multimash11-13b-slerp.json deleted file mode 100644 index df4df161562afab653fe329e5582f25f2bf86b35..0000000000000000000000000000000000000000 --- a/data/models/allknowingroger_multimash11-13b-slerp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MultiMash11-13B-slerp", - "id": "allknowingroger/MultiMash11-13B-slerp", - "developer": "allknowingroger", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MixtralForCausalLM", - "params_billions": "12.879" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allknowingroger_MultiMash11-13B-slerp/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4251 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5194 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0702 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2827 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4373 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3085 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allknowingroger_multimash2-12b-slerp.json b/data/models/allknowingroger_multimash2-12b-slerp.json deleted file mode 100644 index 46fa30e743f28bdd58c72532254a97cb67e3e8dc..0000000000000000000000000000000000000000 --- a/data/models/allknowingroger_multimash2-12b-slerp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MultiMash2-12B-slerp", - "id": "allknowingroger/MultiMash2-12B-slerp", - "developer": "allknowingroger", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MixtralForCausalLM", - "params_billions": "12.879" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allknowingroger_MultiMash2-12B-slerp/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4261 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5134 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0642 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2794 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4228 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3043 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allknowingroger_multimash3-12b-slerp.json b/data/models/allknowingroger_multimash3-12b-slerp.json deleted file mode 100644 index 3977af0ac1a6be84aba928d1be304340d649f29e..0000000000000000000000000000000000000000 --- a/data/models/allknowingroger_multimash3-12b-slerp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Multimash3-12B-slerp", - "id": "allknowingroger/Multimash3-12B-slerp", - "developer": "allknowingroger", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MixtralForCausalLM", - "params_billions": "12.879" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allknowingroger_Multimash3-12B-slerp/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4437 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5177 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0627 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2802 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4344 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3068 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allknowingroger_multimash5-12b-slerp.json b/data/models/allknowingroger_multimash5-12b-slerp.json deleted file mode 100644 index 08d0b91fdc6af579b4bec30ce1641d347c1883ff..0000000000000000000000000000000000000000 --- a/data/models/allknowingroger_multimash5-12b-slerp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MultiMash5-12B-slerp", - "id": "allknowingroger/MultiMash5-12B-slerp", - "developer": "allknowingroger", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MixtralForCausalLM", - "params_billions": "12.879" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allknowingroger_MultiMash5-12B-slerp/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4142 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5145 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0634 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2777 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4203 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3028 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allknowingroger_multimash6-12b-slerp.json b/data/models/allknowingroger_multimash6-12b-slerp.json deleted file mode 100644 index 1f2041d235f61aa98a2a1bb35426d2caa378efac..0000000000000000000000000000000000000000 --- a/data/models/allknowingroger_multimash6-12b-slerp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MultiMash6-12B-slerp", - "id": "allknowingroger/MultiMash6-12B-slerp", - "developer": "allknowingroger", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MixtralForCausalLM", - "params_billions": "12.879" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allknowingroger_MultiMash6-12B-slerp/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.43 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5196 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0725 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2743 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4306 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3091 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allknowingroger_multimash7-12b-slerp.json b/data/models/allknowingroger_multimash7-12b-slerp.json deleted file mode 100644 index 3a3445dc5f3981bdaa3c5724c096d2a85bb82393..0000000000000000000000000000000000000000 --- a/data/models/allknowingroger_multimash7-12b-slerp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MultiMash7-12B-slerp", - "id": "allknowingroger/MultiMash7-12B-slerp", - "developer": "allknowingroger", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MixtralForCausalLM", - "params_billions": "12.879" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allknowingroger_MultiMash7-12B-slerp/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4213 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5111 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0695 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2785 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4279 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3029 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allknowingroger_multimash8-13b-slerp.json b/data/models/allknowingroger_multimash8-13b-slerp.json deleted file mode 100644 index d1f940f255d8ccf016ac64ced7513f2ffe3b46c2..0000000000000000000000000000000000000000 --- a/data/models/allknowingroger_multimash8-13b-slerp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MultiMash8-13B-slerp", - "id": "allknowingroger/MultiMash8-13B-slerp", - "developer": "allknowingroger", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MixtralForCausalLM", - "params_billions": "12.879" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allknowingroger_MultiMash8-13B-slerp/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4321 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5178 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.077 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2886 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4424 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3126 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allknowingroger_multimash9-13b-slerp.json b/data/models/allknowingroger_multimash9-13b-slerp.json deleted file mode 100644 index 9071d6f25e303d9a3dd75bee26e33b507c2c276c..0000000000000000000000000000000000000000 --- a/data/models/allknowingroger_multimash9-13b-slerp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MultiMash9-13B-slerp", - "id": "allknowingroger/MultiMash9-13B-slerp", - "developer": "allknowingroger", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MixtralForCausalLM", - "params_billions": "12.879" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allknowingroger_MultiMash9-13B-slerp/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4188 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5194 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0785 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2802 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4398 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.31 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allknowingroger_multimerge-19b-pass.json b/data/models/allknowingroger_multimerge-19b-pass.json deleted file mode 100644 index 8ee2248da82df2a7abe82410d54be3e8e334e1da..0000000000000000000000000000000000000000 --- a/data/models/allknowingroger_multimerge-19b-pass.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Multimerge-19B-pass", - "id": "allknowingroger/Multimerge-19B-pass", - "developer": "allknowingroger", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MixtralForCausalLM", - "params_billions": "19.188" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allknowingroger_Multimerge-19B-pass/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1773 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2892 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2592 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.343 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1169 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allknowingroger_multimerge-7b-slerp.json b/data/models/allknowingroger_multimerge-7b-slerp.json deleted file mode 100644 index 5b8351e1f4fd21800522208321260a35e350ea32..0000000000000000000000000000000000000000 --- a/data/models/allknowingroger_multimerge-7b-slerp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MultiMerge-7B-slerp", - "id": "allknowingroger/MultiMerge-7B-slerp", - "developer": "allknowingroger", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allknowingroger_MultiMerge-7B-slerp/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3948 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.514 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0665 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2827 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.428 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3037 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allknowingroger_multiverseex26-7b-slerp.json b/data/models/allknowingroger_multiverseex26-7b-slerp.json deleted file mode 100644 index f995d6fedaeaaf7478c4138ba26796ede200d8a9..0000000000000000000000000000000000000000 --- a/data/models/allknowingroger_multiverseex26-7b-slerp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MultiverseEx26-7B-slerp", - "id": "allknowingroger/MultiverseEx26-7B-slerp", - "developer": "allknowingroger", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allknowingroger_MultiverseEx26-7B-slerp/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3939 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5134 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0755 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2827 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4293 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3035 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allknowingroger_neuralcoven-7b-slerp.json b/data/models/allknowingroger_neuralcoven-7b-slerp.json deleted file mode 100644 index 459f16750e39472f4a08752650851fdf333e23f1..0000000000000000000000000000000000000000 --- a/data/models/allknowingroger_neuralcoven-7b-slerp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Neuralcoven-7B-slerp", - "id": "allknowingroger/Neuralcoven-7B-slerp", - "developer": "allknowingroger", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allknowingroger_Neuralcoven-7B-slerp/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3859 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5303 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0785 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2852 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.429 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3294 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allknowingroger_neuralmultiverse-7b-slerp.json b/data/models/allknowingroger_neuralmultiverse-7b-slerp.json deleted file mode 100644 index 395571f1eeaf7ff3f922ba86c57dd48659c97e2b..0000000000000000000000000000000000000000 --- a/data/models/allknowingroger_neuralmultiverse-7b-slerp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Neuralmultiverse-7B-slerp", - "id": "allknowingroger/Neuralmultiverse-7B-slerp", - "developer": "allknowingroger", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allknowingroger_Neuralmultiverse-7B-slerp/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3769 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5166 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.065 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2844 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.428 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3042 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allknowingroger_neuralwestseverus-7b-slerp.json b/data/models/allknowingroger_neuralwestseverus-7b-slerp.json deleted file mode 100644 index 6c70a39c9da8fbe557496d6dca8497fd50037fe6..0000000000000000000000000000000000000000 --- a/data/models/allknowingroger_neuralwestseverus-7b-slerp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "NeuralWestSeverus-7B-slerp", - "id": "allknowingroger/NeuralWestSeverus-7B-slerp", - "developer": "allknowingroger", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allknowingroger_NeuralWestSeverus-7B-slerp/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4136 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5244 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0733 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.271 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4529 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3137 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allknowingroger_ph3della5-14b.json b/data/models/allknowingroger_ph3della5-14b.json deleted file mode 100644 index b7668f1c2397683016924cc8e533bd3d954f2fef..0000000000000000000000000000000000000000 --- a/data/models/allknowingroger_ph3della5-14b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Ph3della5-14B", - "id": "allknowingroger/Ph3della5-14B", - "developer": "allknowingroger", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Phi3ForCausalLM", - "params_billions": "13.96" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allknowingroger_Ph3della5-14B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4799 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6332 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1767 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3423 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4386 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4787 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allknowingroger_ph3merge-14b.json b/data/models/allknowingroger_ph3merge-14b.json deleted file mode 100644 index 355571082392d1eb1cf58dfd2eecba6c9571ae08..0000000000000000000000000000000000000000 --- a/data/models/allknowingroger_ph3merge-14b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Ph3merge-14B", - "id": "allknowingroger/Ph3merge-14B", - "developer": "allknowingroger", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Phi3ForCausalLM", - "params_billions": "13.619" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allknowingroger_Ph3merge-14B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2701 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6381 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0106 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3381 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4334 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4611 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allknowingroger_ph3merge2-14b.json b/data/models/allknowingroger_ph3merge2-14b.json deleted file mode 100644 index 00888c740340beb5cd987750b9358d9a72a41022..0000000000000000000000000000000000000000 --- a/data/models/allknowingroger_ph3merge2-14b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Ph3merge2-14B", - "id": "allknowingroger/Ph3merge2-14B", - "developer": "allknowingroger", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Phi3ForCausalLM", - "params_billions": "13.619" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allknowingroger_Ph3merge2-14B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1706 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3607 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2911 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3911 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1723 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allknowingroger_ph3merge3-14b.json b/data/models/allknowingroger_ph3merge3-14b.json deleted file mode 100644 index 4b511a5071fb1d5cab2bb5776f6fe13b9075a366..0000000000000000000000000000000000000000 --- a/data/models/allknowingroger_ph3merge3-14b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Ph3merge3-14B", - "id": "allknowingroger/Ph3merge3-14B", - "developer": "allknowingroger", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Phi3ForCausalLM", - "params_billions": "13.619" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allknowingroger_Ph3merge3-14B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1645 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3597 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2852 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4082 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1647 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allknowingroger_ph3task1-14b.json b/data/models/allknowingroger_ph3task1-14b.json deleted file mode 100644 index 60a37372892dd502f230299ee141c318096f89b9..0000000000000000000000000000000000000000 --- a/data/models/allknowingroger_ph3task1-14b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Ph3task1-14B", - "id": "allknowingroger/Ph3task1-14B", - "developer": "allknowingroger", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Phi3ForCausalLM", - "params_billions": "13.96" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allknowingroger_Ph3task1-14B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4695 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6318 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1669 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3507 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4508 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4734 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allknowingroger_ph3task2-14b.json b/data/models/allknowingroger_ph3task2-14b.json deleted file mode 100644 index 46d032a4b0f41a435212fa4e7169ad1ffa4b96b7..0000000000000000000000000000000000000000 --- a/data/models/allknowingroger_ph3task2-14b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Ph3task2-14B", - "id": "allknowingroger/Ph3task2-14B", - "developer": "allknowingroger", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Phi3ForCausalLM", - "params_billions": "13.96" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allknowingroger_Ph3task2-14B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4713 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6098 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1465 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3305 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4535 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.446 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allknowingroger_ph3task3-14b.json b/data/models/allknowingroger_ph3task3-14b.json deleted file mode 100644 index b3139828030a67f1d9feb0742cc01459341eb22a..0000000000000000000000000000000000000000 --- a/data/models/allknowingroger_ph3task3-14b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Ph3task3-14B", - "id": "allknowingroger/Ph3task3-14B", - "developer": "allknowingroger", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Phi3ForCausalLM", - "params_billions": "13.96" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allknowingroger_Ph3task3-14B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4962 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6298 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.176 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3414 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4426 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4771 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allknowingroger_ph3unsloth-3b-slerp.json b/data/models/allknowingroger_ph3unsloth-3b-slerp.json deleted file mode 100644 index d2315eaf25b5c7b4cf1a436f0a2995e43661e2da..0000000000000000000000000000000000000000 --- a/data/models/allknowingroger_ph3unsloth-3b-slerp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Ph3unsloth-3B-slerp", - "id": "allknowingroger/Ph3unsloth-3B-slerp", - "developer": "allknowingroger", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "3.821" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allknowingroger_Ph3unsloth-3B-slerp/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1894 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5468 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1012 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3247 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4528 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3701 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allknowingroger_phi3mash1-17b-pass.json b/data/models/allknowingroger_phi3mash1-17b-pass.json deleted file mode 100644 index 6d82c52bf8d5895033899e68962b92f8633aca8f..0000000000000000000000000000000000000000 --- a/data/models/allknowingroger_phi3mash1-17b-pass.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Phi3mash1-17B-pass", - "id": "allknowingroger/Phi3mash1-17B-pass", - "developer": "allknowingroger", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "16.687" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allknowingroger_Phi3mash1-17B-pass/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1884 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6129 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3196 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4451 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4589 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allknowingroger_quen2-65b.json b/data/models/allknowingroger_quen2-65b.json deleted file mode 100644 index d6334a7ae92ad0873c43fd4819bd786baf125eb3..0000000000000000000000000000000000000000 --- a/data/models/allknowingroger_quen2-65b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Quen2-65B", - "id": "allknowingroger/Quen2-65B", - "developer": "allknowingroger", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "63.923" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allknowingroger_Quen2-65B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1758 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2757 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2357 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3209 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1114 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allknowingroger_qwen2.5-42b-agi.json b/data/models/allknowingroger_qwen2.5-42b-agi.json deleted file mode 100644 index 301b6fed180ec86bf4f628f3ce04182b30ecf1ae..0000000000000000000000000000000000000000 --- a/data/models/allknowingroger_qwen2.5-42b-agi.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-42B-AGI", - "id": "allknowingroger/Qwen2.5-42B-AGI", - "developer": "allknowingroger", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "42.516" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allknowingroger_Qwen2.5-42B-AGI/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1913 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2942 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2601 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.362 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1168 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allknowingroger_qwen2.5-7b-task2.json b/data/models/allknowingroger_qwen2.5-7b-task2.json deleted file mode 100644 index cd222036db135fefcbcd2ba93f2aea5631bb4454..0000000000000000000000000000000000000000 --- a/data/models/allknowingroger_qwen2.5-7b-task2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-7B-task2", - "id": "allknowingroger/Qwen2.5-7B-task2", - "developer": "allknowingroger", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allknowingroger_Qwen2.5-7B-task2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4527 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5626 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.355 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3163 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.437 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4517 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allknowingroger_qwen2.5-7b-task3.json b/data/models/allknowingroger_qwen2.5-7b-task3.json deleted file mode 100644 index 360215ff2aaff304f97868cf54d778b352528d18..0000000000000000000000000000000000000000 --- a/data/models/allknowingroger_qwen2.5-7b-task3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-7B-task3", - "id": "allknowingroger/Qwen2.5-7B-task3", - "developer": "allknowingroger", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allknowingroger_Qwen2.5-7B-task3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5129 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5398 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2606 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3171 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4356 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4501 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allknowingroger_qwen2.5-7b-task4.json b/data/models/allknowingroger_qwen2.5-7b-task4.json deleted file mode 100644 index 792790c44085e8cc291c354ec673dea7f40c0afa..0000000000000000000000000000000000000000 --- a/data/models/allknowingroger_qwen2.5-7b-task4.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-7B-task4", - "id": "allknowingroger/Qwen2.5-7B-task4", - "developer": "allknowingroger", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allknowingroger_Qwen2.5-7B-task4/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5005 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5583 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3112 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3205 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4395 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4561 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allknowingroger_qwen2.5-7b-task7.json b/data/models/allknowingroger_qwen2.5-7b-task7.json deleted file mode 100644 index 12886817f38de8b92a8a2be7628a1dde3f6f98a1..0000000000000000000000000000000000000000 --- a/data/models/allknowingroger_qwen2.5-7b-task7.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-7B-task7", - "id": "allknowingroger/Qwen2.5-7B-task7", - "developer": "allknowingroger", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allknowingroger_Qwen2.5-7B-task7/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4284 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5552 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.065 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3205 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4326 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4133 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allknowingroger_qwen2.5-7b-task8.json b/data/models/allknowingroger_qwen2.5-7b-task8.json deleted file mode 100644 index 9ddfaf13ff6a753f0117d593d940af040bea3a88..0000000000000000000000000000000000000000 --- a/data/models/allknowingroger_qwen2.5-7b-task8.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-7B-task8", - "id": "allknowingroger/Qwen2.5-7B-task8", - "developer": "allknowingroger", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allknowingroger_Qwen2.5-7B-task8/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4645 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5525 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3527 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3205 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4514 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4433 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allknowingroger_qwen2.5-slerp-14b.json b/data/models/allknowingroger_qwen2.5-slerp-14b.json deleted file mode 100644 index ef2318ce920e49e20efd844ec6cd82c06a3aa221..0000000000000000000000000000000000000000 --- a/data/models/allknowingroger_qwen2.5-slerp-14b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-slerp-14B", - "id": "allknowingroger/Qwen2.5-slerp-14B", - "developer": "allknowingroger", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allknowingroger_Qwen2.5-slerp-14B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4928 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6512 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4622 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3674 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4744 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5379 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allknowingroger_qwenslerp12-7b.json b/data/models/allknowingroger_qwenslerp12-7b.json deleted file mode 100644 index 706e063f3b9765a132a633f34bc4468c21480e21..0000000000000000000000000000000000000000 --- a/data/models/allknowingroger_qwenslerp12-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "QwenSlerp12-7B", - "id": "allknowingroger/QwenSlerp12-7B", - "developer": "allknowingroger", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allknowingroger_QwenSlerp12-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5076 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5556 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2946 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3154 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4595 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4461 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allknowingroger_qwenslerp2-14b.json b/data/models/allknowingroger_qwenslerp2-14b.json deleted file mode 100644 index 093902b929e9e293c3ff1c2fe8bd992e181f538d..0000000000000000000000000000000000000000 --- a/data/models/allknowingroger_qwenslerp2-14b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwenslerp2-14B", - "id": "allknowingroger/Qwenslerp2-14B", - "developer": "allknowingroger", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allknowingroger_Qwenslerp2-14B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5007 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6555 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4456 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3683 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4729 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5403 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allknowingroger_qwenslerp2-7b.json b/data/models/allknowingroger_qwenslerp2-7b.json deleted file mode 100644 index b8db8dd310bb0049e25b73d78a74a2b8b478bf7c..0000000000000000000000000000000000000000 --- a/data/models/allknowingroger_qwenslerp2-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwenslerp2-7B", - "id": "allknowingroger/Qwenslerp2-7B", - "developer": "allknowingroger", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allknowingroger_Qwenslerp2-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5294 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5609 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3421 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3129 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4356 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4515 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allknowingroger_qwenslerp3-14b.json b/data/models/allknowingroger_qwenslerp3-14b.json deleted file mode 100644 index 3ccf80351ca000a80f55562b42d27f148ebe4793..0000000000000000000000000000000000000000 --- a/data/models/allknowingroger_qwenslerp3-14b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwenslerp3-14B", - "id": "allknowingroger/Qwenslerp3-14B", - "developer": "allknowingroger", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allknowingroger_Qwenslerp3-14B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5052 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6521 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4464 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.375 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4676 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5395 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allknowingroger_qwenslerp3-7b.json b/data/models/allknowingroger_qwenslerp3-7b.json deleted file mode 100644 index 69fff792073e50011b82517814cf75afd88ae5fb..0000000000000000000000000000000000000000 --- a/data/models/allknowingroger_qwenslerp3-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwenslerp3-7B", - "id": "allknowingroger/Qwenslerp3-7B", - "developer": "allknowingroger", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allknowingroger_Qwenslerp3-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5018 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.558 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3218 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3247 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4515 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4542 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allknowingroger_qwenslerp4-14b.json b/data/models/allknowingroger_qwenslerp4-14b.json deleted file mode 100644 index a47a529c16a2ab8285370b11782bb1ca6a9affaf..0000000000000000000000000000000000000000 --- a/data/models/allknowingroger_qwenslerp4-14b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "QwenSlerp4-14B", - "id": "allknowingroger/QwenSlerp4-14B", - "developer": "allknowingroger", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allknowingroger_QwenSlerp4-14B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6328 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6483 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3693 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3725 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.465 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5436 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allknowingroger_qwenslerp5-14b.json b/data/models/allknowingroger_qwenslerp5-14b.json deleted file mode 100644 index b5f359d9c18662a561e1f2712215f1e957e71e71..0000000000000000000000000000000000000000 --- a/data/models/allknowingroger_qwenslerp5-14b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "QwenSlerp5-14B", - "id": "allknowingroger/QwenSlerp5-14B", - "developer": "allknowingroger", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allknowingroger_QwenSlerp5-14B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7119 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6357 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3565 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3649 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4675 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5391 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allknowingroger_qwenslerp6-14b.json b/data/models/allknowingroger_qwenslerp6-14b.json deleted file mode 100644 index b2cbad4fca48ce2c6919d4d869952a152936482c..0000000000000000000000000000000000000000 --- a/data/models/allknowingroger_qwenslerp6-14b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "QwenSlerp6-14B", - "id": "allknowingroger/QwenSlerp6-14B", - "developer": "allknowingroger", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allknowingroger_QwenSlerp6-14B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6867 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6384 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3724 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3733 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.469 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5406 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allknowingroger_qwenstock1-14b.json b/data/models/allknowingroger_qwenstock1-14b.json deleted file mode 100644 index 7cc413d15c67abbbd19feb191daf333b6520b6ff..0000000000000000000000000000000000000000 --- a/data/models/allknowingroger_qwenstock1-14b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "QwenStock1-14B", - "id": "allknowingroger/QwenStock1-14B", - "developer": "allknowingroger", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allknowingroger_QwenStock1-14B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5634 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6528 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3769 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3767 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.473 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5418 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allknowingroger_qwenstock2-14b.json b/data/models/allknowingroger_qwenstock2-14b.json deleted file mode 100644 index 3e33b45f527a55c7f6d26c263f3f2e920d0f10f3..0000000000000000000000000000000000000000 --- a/data/models/allknowingroger_qwenstock2-14b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "QwenStock2-14B", - "id": "allknowingroger/QwenStock2-14B", - "developer": "allknowingroger", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allknowingroger_QwenStock2-14B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5563 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6569 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3882 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3792 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4756 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5406 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allknowingroger_qwenstock3-14b.json b/data/models/allknowingroger_qwenstock3-14b.json deleted file mode 100644 index fb08012f002a05e21c2dad8aee7a497c7cf85dc5..0000000000000000000000000000000000000000 --- a/data/models/allknowingroger_qwenstock3-14b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "QwenStock3-14B", - "id": "allknowingroger/QwenStock3-14B", - "developer": "allknowingroger", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allknowingroger_QwenStock3-14B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5615 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6565 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3776 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3784 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4756 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5428 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allknowingroger_rogermerge-7b-slerp.json b/data/models/allknowingroger_rogermerge-7b-slerp.json deleted file mode 100644 index cc2ba5638429b825d40b2b94efd6523484813cc9..0000000000000000000000000000000000000000 --- a/data/models/allknowingroger_rogermerge-7b-slerp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "RogerMerge-7B-slerp", - "id": "allknowingroger/RogerMerge-7B-slerp", - "developer": "allknowingroger", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allknowingroger_RogerMerge-7B-slerp/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3933 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.516 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0687 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2802 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.432 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.303 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allknowingroger_rogerphi-7b-slerp.json b/data/models/allknowingroger_rogerphi-7b-slerp.json deleted file mode 100644 index c826fdd16ca3cdc226fe2522e7cb07900658b594..0000000000000000000000000000000000000000 --- a/data/models/allknowingroger_rogerphi-7b-slerp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ROGERphi-7B-slerp", - "id": "allknowingroger/ROGERphi-7B-slerp", - "developer": "allknowingroger", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allknowingroger_ROGERphi-7B-slerp/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3861 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5196 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0733 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2886 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4685 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3053 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allknowingroger_rombos-llm-v2.5-qwen-42b.json b/data/models/allknowingroger_rombos-llm-v2.5-qwen-42b.json deleted file mode 100644 index c66869220f8d313c5f004fea6eb0fcf32e116f5a..0000000000000000000000000000000000000000 --- a/data/models/allknowingroger_rombos-llm-v2.5-qwen-42b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Rombos-LLM-V2.5-Qwen-42b", - "id": "allknowingroger/Rombos-LLM-V2.5-Qwen-42b", - "developer": "allknowingroger", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "42.516" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allknowingroger_Rombos-LLM-V2.5-Qwen-42b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1879 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2969 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2626 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3633 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1168 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allknowingroger_strangecoven-7b-slerp.json b/data/models/allknowingroger_strangecoven-7b-slerp.json deleted file mode 100644 index 0766d2c90764867c486f790da8e42acc481fa888..0000000000000000000000000000000000000000 --- a/data/models/allknowingroger_strangecoven-7b-slerp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Strangecoven-7B-slerp", - "id": "allknowingroger/Strangecoven-7B-slerp", - "developer": "allknowingroger", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allknowingroger_Strangecoven-7B-slerp/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3746 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5368 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0763 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2894 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4199 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3364 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allknowingroger_weirdslerp2-25b.json b/data/models/allknowingroger_weirdslerp2-25b.json deleted file mode 100644 index 4e8e1fd5f01f75a12eb5ab7c2574ab3e4fc1b6f3..0000000000000000000000000000000000000000 --- a/data/models/allknowingroger_weirdslerp2-25b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Weirdslerp2-25B", - "id": "allknowingroger/Weirdslerp2-25B", - "developer": "allknowingroger", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "25.204" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allknowingroger_Weirdslerp2-25B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1754 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2874 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2492 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3524 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1128 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allknowingroger_westlakemaziyar-7b-slerp.json b/data/models/allknowingroger_westlakemaziyar-7b-slerp.json deleted file mode 100644 index 8a7d7534b6d1cdcd2d34d160a2d2851083b5bcf7..0000000000000000000000000000000000000000 --- a/data/models/allknowingroger_westlakemaziyar-7b-slerp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "WestlakeMaziyar-7B-slerp", - "id": "allknowingroger/WestlakeMaziyar-7B-slerp", - "developer": "allknowingroger", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allknowingroger_WestlakeMaziyar-7B-slerp/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4838 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5245 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0665 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3037 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4474 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3078 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allknowingroger_yammaths-7b-slerp.json b/data/models/allknowingroger_yammaths-7b-slerp.json deleted file mode 100644 index 7b01a4b0d9e9913529332d5e67947007ebc169fc..0000000000000000000000000000000000000000 --- a/data/models/allknowingroger_yammaths-7b-slerp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "YamMaths-7B-slerp", - "id": "allknowingroger/YamMaths-7B-slerp", - "developer": "allknowingroger", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allknowingroger_YamMaths-7B-slerp/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4148 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5156 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0853 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2802 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4384 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3131 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allknowingroger_yi-1.5-34b.json b/data/models/allknowingroger_yi-1.5-34b.json deleted file mode 100644 index 4952fb4b8287521fb0d454dfff13d62b3077abdb..0000000000000000000000000000000000000000 --- a/data/models/allknowingroger_yi-1.5-34b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Yi-1.5-34B", - "id": "allknowingroger/Yi-1.5-34B", - "developer": "allknowingroger", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "34.389" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allknowingroger_Yi-1.5-34B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1639 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2827 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2584 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3857 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1095 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allknowingroger_yi-blossom-40b.json b/data/models/allknowingroger_yi-blossom-40b.json deleted file mode 100644 index 1c894c8dea5e51007cebc9e39aa2e8adfdde426f..0000000000000000000000000000000000000000 --- a/data/models/allknowingroger_yi-blossom-40b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Yi-blossom-40B", - "id": "allknowingroger/Yi-blossom-40B", - "developer": "allknowingroger", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "18.769" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allknowingroger_Yi-blossom-40B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2009 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3215 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2743 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3843 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.108 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allknowingroger_yibuddy-35b.json b/data/models/allknowingroger_yibuddy-35b.json deleted file mode 100644 index 816911325b61d2e2400b2510ef6477a029177cb0..0000000000000000000000000000000000000000 --- a/data/models/allknowingroger_yibuddy-35b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Yibuddy-35B", - "id": "allknowingroger/Yibuddy-35B", - "developer": "allknowingroger", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "34.389" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allknowingroger_Yibuddy-35B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4235 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5916 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1571 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3557 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4505 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4489 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allknowingroger_yillama-40b.json b/data/models/allknowingroger_yillama-40b.json deleted file mode 100644 index 012c3a68a155aa5bf58b63cdffb30a721587fd52..0000000000000000000000000000000000000000 --- a/data/models/allknowingroger_yillama-40b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Yillama-40B", - "id": "allknowingroger/Yillama-40B", - "developer": "allknowingroger", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "34.389" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allknowingroger_Yillama-40B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1697 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4063 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2827 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3501 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1981 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allknowingroger_yislerp-34b.json b/data/models/allknowingroger_yislerp-34b.json deleted file mode 100644 index 4d0f3c19b8257ebc9723ac0e0a55600c455c2678..0000000000000000000000000000000000000000 --- a/data/models/allknowingroger_yislerp-34b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Yislerp-34B", - "id": "allknowingroger/Yislerp-34B", - "developer": "allknowingroger", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "34.389" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allknowingroger_Yislerp-34B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3692 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6159 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.216 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3582 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4566 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4751 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allknowingroger_yislerp2-34b.json b/data/models/allknowingroger_yislerp2-34b.json deleted file mode 100644 index 8c561affc9bc970d339166570ce275a608b0bbf4..0000000000000000000000000000000000000000 --- a/data/models/allknowingroger_yislerp2-34b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Yislerp2-34B", - "id": "allknowingroger/Yislerp2-34B", - "developer": "allknowingroger", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "34.389" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allknowingroger_Yislerp2-34B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3999 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6246 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2296 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3641 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.453 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4724 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allknowingroger_yunconglong-13b-slerp.json b/data/models/allknowingroger_yunconglong-13b-slerp.json deleted file mode 100644 index e72e6fe010bc6099b6eae4880c16e958ddfadaba..0000000000000000000000000000000000000000 --- a/data/models/allknowingroger_yunconglong-13b-slerp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Yunconglong-13B-slerp", - "id": "allknowingroger/Yunconglong-13B-slerp", - "developer": "allknowingroger", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MixtralForCausalLM", - "params_billions": "12.879" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allknowingroger_Yunconglong-13B-slerp/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4242 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5166 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0544 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.281 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4161 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3036 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allura-org_l3.1-8b-rp-ink.json b/data/models/allura-org_l3.1-8b-rp-ink.json deleted file mode 100644 index e4bac559e02cf4f8bcce1de9b2597656f920ea5f..0000000000000000000000000000000000000000 --- a/data/models/allura-org_l3.1-8b-rp-ink.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "L3.1-8b-RP-Ink", - "id": "allura-org/L3.1-8b-RP-Ink", - "developer": "allura-org", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allura-org_L3.1-8b-RP-Ink/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7811 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4828 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.148 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2643 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3608 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3428 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allura-org_mistral-small-24b-sertraline-0304.json b/data/models/allura-org_mistral-small-24b-sertraline-0304.json deleted file mode 100644 index f3de6b172a43c3b64ebcf69522fd1a49a3f10752..0000000000000000000000000000000000000000 --- a/data/models/allura-org_mistral-small-24b-sertraline-0304.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mistral-Small-24b-Sertraline-0304", - "id": "allura-org/Mistral-Small-24b-Sertraline-0304", - "developer": "allura-org", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "23.572" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allura-org_Mistral-Small-24b-Sertraline-0304/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.68 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6525 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2228 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3515 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4395 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5106 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allura-org_mistral-small-sisyphus-24b-2503.json b/data/models/allura-org_mistral-small-sisyphus-24b-2503.json deleted file mode 100644 index 31a4f05aea19ae5ca5aeab9ccd8bd8787aca614a..0000000000000000000000000000000000000000 --- a/data/models/allura-org_mistral-small-sisyphus-24b-2503.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mistral-Small-Sisyphus-24b-2503", - "id": "allura-org/Mistral-Small-Sisyphus-24b-2503", - "developer": "allura-org", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "23.572" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allura-org_Mistral-Small-Sisyphus-24b-2503/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6848 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.627 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.25 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2626 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3977 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5127 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allura-org_mn-12b-rp-ink.json b/data/models/allura-org_mn-12b-rp-ink.json deleted file mode 100644 index 49aa59d5072e069a50ff6d1ad48b6358c6071836..0000000000000000000000000000000000000000 --- a/data/models/allura-org_mn-12b-rp-ink.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MN-12b-RP-Ink", - "id": "allura-org/MN-12b-RP-Ink", - "developer": "allura-org", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allura-org_MN-12b-RP-Ink/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7186 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4834 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1186 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2852 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3818 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3514 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allura-org_moe-girl-1ba-7bt.json b/data/models/allura-org_moe-girl-1ba-7bt.json deleted file mode 100644 index 04a95e723bd14461d5317eba92f14cb490f29094..0000000000000000000000000000000000000000 --- a/data/models/allura-org_moe-girl-1ba-7bt.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MoE-Girl-1BA-7BT", - "id": "allura-org/MoE-Girl-1BA-7BT", - "developer": "allura-org", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "OlmoeForCausalLM", - "params_billions": "6.919" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allura-org_MoE-Girl-1BA-7BT/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2705 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3139 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0151 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2584 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3436 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1218 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allura-org_ms-meadowlark-22b.json b/data/models/allura-org_ms-meadowlark-22b.json deleted file mode 100644 index 05c6d48a7e6b44c0b1d4762d44832f64e040b625..0000000000000000000000000000000000000000 --- a/data/models/allura-org_ms-meadowlark-22b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MS-Meadowlark-22B", - "id": "allura-org/MS-Meadowlark-22B", - "developer": "allura-org", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "22.247" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allura-org_MS-Meadowlark-22B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6697 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5163 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1835 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3255 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3843 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3823 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allura-org_teleut-7b.json b/data/models/allura-org_teleut-7b.json deleted file mode 100644 index bde563a394f7300587fdcf456fcc0fe2f87a0826..0000000000000000000000000000000000000000 --- a/data/models/allura-org_teleut-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Teleut-7b", - "id": "allura-org/Teleut-7b", - "developer": "allura-org", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allura-org_Teleut-7b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6379 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5141 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2409 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3263 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.464 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4131 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allura-org_tq2.5-14b-aletheia-v1.json b/data/models/allura-org_tq2.5-14b-aletheia-v1.json deleted file mode 100644 index ca4fb057fb27e89fb26081260b753fab249dab5a..0000000000000000000000000000000000000000 --- a/data/models/allura-org_tq2.5-14b-aletheia-v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "TQ2.5-14B-Aletheia-v1", - "id": "allura-org/TQ2.5-14B-Aletheia-v1", - "developer": "allura-org", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allura-org_TQ2.5-14B-Aletheia-v1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.753 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6585 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3399 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3624 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4452 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5241 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/allura-org_tq2.5-14b-neon-v1.json b/data/models/allura-org_tq2.5-14b-neon-v1.json deleted file mode 100644 index c49ee73f18e87845599b792d812feadea465f544..0000000000000000000000000000000000000000 --- a/data/models/allura-org_tq2.5-14b-neon-v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "TQ2.5-14B-Neon-v1", - "id": "allura-org/TQ2.5-14B-Neon-v1", - "developer": "allura-org", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/allura-org_TQ2.5-14B-Neon-v1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6754 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6553 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3603 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3716 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.461 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5253 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/aloobun_d-smollm2-360m.json b/data/models/aloobun_d-smollm2-360m.json deleted file mode 100644 index 3471cc6a4da54c987799e4b48fff93e72fbaaee3..0000000000000000000000000000000000000000 --- a/data/models/aloobun_d-smollm2-360m.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "d-SmolLM2-360M", - "id": "aloobun/d-SmolLM2-360M", - "developer": "aloobun", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "0.362" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/aloobun_d-SmolLM2-360M/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2097 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3196 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0128 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2534 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3981 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1169 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/aloobun_meta-llama-3-7b-28layers.json b/data/models/aloobun_meta-llama-3-7b-28layers.json deleted file mode 100644 index f201ca878b69417b3e2a12ed9aa8bf59e80e20b9..0000000000000000000000000000000000000000 --- a/data/models/aloobun_meta-llama-3-7b-28layers.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Meta-Llama-3-7B-28Layers", - "id": "aloobun/Meta-Llama-3-7B-28Layers", - "developer": "aloobun", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "7.158" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/aloobun_Meta-Llama-3-7B-28Layers/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1964 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4437 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0279 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2945 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3589 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.316 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/alpindale_magnum-72b-v1.json b/data/models/alpindale_magnum-72b-v1.json deleted file mode 100644 index 4d41dbf1eb89da2e29062b97129e17e5c1116928..0000000000000000000000000000000000000000 --- a/data/models/alpindale_magnum-72b-v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "magnum-72b-v1", - "id": "alpindale/magnum-72b-v1", - "developer": "alpindale", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "72.706" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/alpindale_magnum-72b-v1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7606 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6982 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.398 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3909 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4489 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5468 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/alpindale_wizardlm-2-8x22b.json b/data/models/alpindale_wizardlm-2-8x22b.json deleted file mode 100644 index 2fa912c7674bb4ec81e0b938d8b28d2985144b72..0000000000000000000000000000000000000000 --- a/data/models/alpindale_wizardlm-2-8x22b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "WizardLM-2-8x22B", - "id": "alpindale/WizardLM-2-8x22B", - "developer": "alpindale", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MixtralForCausalLM", - "params_billions": "140.621" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/alpindale_WizardLM-2-8x22B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5272 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6377 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.25 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3817 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4387 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4596 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/alsebay_qwen2.5-7b-test-novelist.json b/data/models/alsebay_qwen2.5-7b-test-novelist.json deleted file mode 100644 index a0d3e72bb86e482818d55277b7dc090a017afc0c..0000000000000000000000000000000000000000 --- a/data/models/alsebay_qwen2.5-7b-test-novelist.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-7B-test-novelist", - "id": "Alsebay/Qwen2.5-7B-test-novelist", - "developer": "Alsebay", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Alsebay_Qwen2.5-7B-test-novelist/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5352 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5151 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2349 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2911 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4749 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3866 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/altomek_yism-34b-0rn.json b/data/models/altomek_yism-34b-0rn.json deleted file mode 100644 index 5f2c3790bbd6de27f81a1b5d4ccc08bffcdde48e..0000000000000000000000000000000000000000 --- a/data/models/altomek_yism-34b-0rn.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "YiSM-34B-0rn", - "id": "altomek/YiSM-34B-0rn", - "developer": "altomek", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "34.389" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/altomek_YiSM-34B-0rn/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4284 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.614 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2281 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3716 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.445 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4696 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/amaorynho_bbai2006.json b/data/models/amaorynho_bbai2006.json deleted file mode 100644 index b8581bbc6fff60b3460148d37b58b459a1d1ff66..0000000000000000000000000000000000000000 --- a/data/models/amaorynho_bbai2006.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "BBAI2006", - "id": "Amaorynho/BBAI2006", - "developer": "Amaorynho", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.09" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Amaorynho_BBAI2006/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1467 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2704 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2525 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3605 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1123 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/amaorynho_bbai270v4.json b/data/models/amaorynho_bbai270v4.json deleted file mode 100644 index 0d3ecd882dea67b24228525e04b634122c5ef25c..0000000000000000000000000000000000000000 --- a/data/models/amaorynho_bbai270v4.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "BBAI270V4", - "id": "Amaorynho/BBAI270V4", - "developer": "Amaorynho", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Amaorynho_BBAI270V4/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.199 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3071 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0083 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2458 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3314 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1114 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/amaorynho_bbai_375.json b/data/models/amaorynho_bbai_375.json deleted file mode 100644 index 27228d710fc84dbc6817e0632696beb6e83df197..0000000000000000000000000000000000000000 --- a/data/models/amaorynho_bbai_375.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "BBAI_375", - "id": "Amaorynho/BBAI_375", - "developer": "Amaorynho", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.09" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Amaorynho_BBAI_375/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1467 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2704 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2525 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3605 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1123 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/amaorynho_bbaiifev1.json b/data/models/amaorynho_bbaiifev1.json deleted file mode 100644 index 9df54ee5f3dd35b5d683275e921e0b4a6ec360ee..0000000000000000000000000000000000000000 --- a/data/models/amaorynho_bbaiifev1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "BBAIIFEV1", - "id": "Amaorynho/BBAIIFEV1", - "developer": "Amaorynho", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Amaorynho_BBAIIFEV1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8047 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5292 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1934 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3104 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4185 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3857 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/amazon_amazon-nova-2-lite-v1-0-fc.json b/data/models/amazon_amazon-nova-2-lite-v1-0-fc.json deleted file mode 100644 index cf9c58e918a6e00ed203d6176c86bbab1dfd13aa..0000000000000000000000000000000000000000 --- a/data/models/amazon_amazon-nova-2-lite-v1-0-fc.json +++ /dev/null @@ -1,851 +0,0 @@ -{ - "model_info": { - "name": "Amazon-Nova-2-Lite-v1:0 (FC)", - "id": "amazon/amazon-nova-2-lite-v1-0-fc", - "developer": "amazon", - "additional_details": { - "raw_model_name": "Amazon-Nova-2-Lite-v1:0 (FC)", - "organization": "Amazon", - "license": "Proprietary", - "mode": "FC", - "model_link": "https://aws.amazon.com/cn/ai/generative-ai/nova/" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/amazon/amazon-nova-2-lite-v1-0-fc/1775236112.407982", - "retrieved_timestamp": "1775236112.407982", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 80.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 27.1 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 78.19 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 8.55 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 9.85 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 27.62 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 86.96 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 76.33 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 94.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 91.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 86.0 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 80.83 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 83.33 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 80.15 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 87.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 79.17 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 2.12 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 2.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 1.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 2.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 2.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 5.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 4.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 6.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 2.37 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 1.94 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 3.23 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 1.94 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 75.0 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 82.11 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/amazon_amazon-nova-micro-v1-0-fc.json b/data/models/amazon_amazon-nova-micro-v1-0-fc.json deleted file mode 100644 index c37bcbc553f5204b9cc54d7e6dbb03112eecef1d..0000000000000000000000000000000000000000 --- a/data/models/amazon_amazon-nova-micro-v1-0-fc.json +++ /dev/null @@ -1,851 +0,0 @@ -{ - "model_info": { - "name": "Amazon-Nova-Micro-v1:0 (FC)", - "id": "amazon/amazon-nova-micro-v1-0-fc", - "developer": "amazon", - "additional_details": { - "raw_model_name": "Amazon-Nova-Micro-v1:0 (FC)", - "organization": "Amazon", - "license": "Proprietary", - "mode": "FC", - "model_link": "https://aws.amazon.com/cn/ai/generative-ai/nova/" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/amazon/amazon-nova-micro-v1-0-fc/1775236112.415976", - "retrieved_timestamp": "1775236112.415976", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 95.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 22.29 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 1.81 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 1.12 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 0.45 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 1.79 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 74.1 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 70.92 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 87.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 75.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 62.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 66.32 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 72.09 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 64.96 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 81.25 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 54.17 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 1.38 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 1.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 1.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 2.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 1.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 1.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 3.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 2.37 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 1.94 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 2.58 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 2.58 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 81.25 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 70.65 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/amazon_amazon-nova-pro-v1-0-fc.json b/data/models/amazon_amazon-nova-pro-v1-0-fc.json deleted file mode 100644 index c631acd06f709fdc9d88032bff0b44bcd4c8717b..0000000000000000000000000000000000000000 --- a/data/models/amazon_amazon-nova-pro-v1-0-fc.json +++ /dev/null @@ -1,851 +0,0 @@ -{ - "model_info": { - "name": "Amazon-Nova-Pro-v1:0 (FC)", - "id": "amazon/amazon-nova-pro-v1-0-fc", - "developer": "amazon", - "additional_details": { - "raw_model_name": "Amazon-Nova-Pro-v1:0 (FC)", - "organization": "Amazon", - "license": "Proprietary", - "mode": "FC", - "model_link": "https://aws.amazon.com/cn/ai/generative-ai/nova/" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/amazon/amazon-nova-pro-v1-0-fc/1775236112.412158", - "retrieved_timestamp": "1775236112.412158", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 88.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 24.97 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 48.44 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 2.25 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 1.91 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 3.29 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 86.58 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 75.83 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 93.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 93.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 84.0 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 78.53 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 81.4 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 77.97 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 81.25 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 70.83 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 1.88 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 1.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 0.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 2.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 3.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 2.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 4.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 1.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 1.94 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 2.58 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 1.29 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 1.94 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 93.75 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 70.06 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/amazon_megabeam-mistral-7b-300k.json b/data/models/amazon_megabeam-mistral-7b-300k.json deleted file mode 100644 index fb564ce7af2533b18cef48c65d61cd03264af8f3..0000000000000000000000000000000000000000 --- a/data/models/amazon_megabeam-mistral-7b-300k.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MegaBeam-Mistral-7B-300k", - "id": "amazon/MegaBeam-Mistral-7B-300k", - "developer": "amazon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/amazon_MegaBeam-Mistral-7B-300k/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5203 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4228 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0211 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2735 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.398 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2549 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/amazon_nova-lite-v1_0.json b/data/models/amazon_nova-lite-v1_0.json deleted file mode 100644 index b6b5f0a681cffbcab4ddf0133695a25371c2dcc0..0000000000000000000000000000000000000000 --- a/data/models/amazon_nova-lite-v1_0.json +++ /dev/null @@ -1,2128 +0,0 @@ -{ - "model_info": { - "name": "Amazon Nova Lite", - "id": "amazon/nova-lite-v1:0", - "developer": "amazon", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_capabilities/amazon_nova-lite-v1:0/1774096304.5056698", - "retrieved_timestamp": "1774096304.5056698", - "source_metadata": { - "source_name": "helm_capabilities", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_capabilities", - "evaluation_results": [ - { - "evaluation_name": "Mean score", - "source_data": { - "dataset_name": "helm_capabilities", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "The mean of the scores from all columns.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.551, - "details": { - "description": "", - "tab": "Accuracy", - "Mean score - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"2.6046740288354906\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU-Pro", - "source_data": { - "dataset_name": "MMLU-Pro", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on MMLU-Pro", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6, - "details": { - "description": "min=0.6, mean=0.6, max=0.6, sum=0.6 (1)", - "tab": "Accuracy", - "MMLU-Pro - Observed inference time (s)": "{\"description\": \"min=1.375, mean=1.375, max=1.375, sum=1.375 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.3748559999999983\"}", - "MMLU-Pro - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "MMLU-Pro - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - # prompt tokens": "{\"description\": \"min=235.232, mean=235.232, max=235.232, sum=235.232 (1)\", \"tab\": \"General information\", \"score\": \"235.232\"}", - "MMLU-Pro - # output tokens": "{\"description\": \"min=343.771, mean=343.771, max=343.771, sum=343.771 (1)\", \"tab\": \"General information\", \"score\": \"343.771\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"all\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.397, - "details": { - "description": "min=0.397, mean=0.397, max=0.397, sum=0.397 (1)", - "tab": "Accuracy", - "GPQA - Observed inference time (s)": "{\"description\": \"min=2.04, mean=2.04, max=2.04, sum=2.04 (1)\", \"tab\": \"Efficiency\", \"score\": \"2.0404999999999998\"}", - "GPQA - # eval": "{\"description\": \"min=446, mean=446, max=446, sum=446 (1)\", \"tab\": \"General information\", \"score\": \"446.0\"}", - "GPQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - # prompt tokens": "{\"description\": \"min=264.121, mean=264.121, max=264.121, sum=264.121 (1)\", \"tab\": \"General information\", \"score\": \"264.1210762331838\"}", - "GPQA - # output tokens": "{\"description\": \"min=512.256, mean=512.256, max=512.256, sum=512.256 (1)\", \"tab\": \"General information\", \"score\": \"512.2556053811659\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"gpqa_main\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "IFEval Strict Acc on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.776, - "details": { - "description": "min=0.776, mean=0.776, max=0.776, sum=0.776 (1)", - "tab": "Accuracy", - "IFEval - Observed inference time (s)": "{\"description\": \"min=3.156, mean=3.156, max=3.156, sum=3.156 (1)\", \"tab\": \"Efficiency\", \"score\": \"3.1562421441774484\"}", - "IFEval - # eval": "{\"description\": \"min=541, mean=541, max=541, sum=541 (1)\", \"tab\": \"General information\", \"score\": \"541.0\"}", - "IFEval - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - # prompt tokens": "{\"description\": \"min=47.58, mean=47.58, max=47.58, sum=47.58 (1)\", \"tab\": \"General information\", \"score\": \"47.58040665434381\"}", - "IFEval - # output tokens": "{\"description\": \"min=412.706, mean=412.706, max=412.706, sum=412.706 (1)\", \"tab\": \"General information\", \"score\": \"412.70609981515713\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WildBench", - "source_data": { - "dataset_name": "WildBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "WB Score on WildBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.75, - "details": { - "description": "min=0.75, mean=0.75, max=0.75, sum=0.75 (1)", - "tab": "Accuracy", - "WildBench - Observed inference time (s)": "{\"description\": \"min=4.034, mean=4.034, max=4.034, sum=4.034 (1)\", \"tab\": \"Efficiency\", \"score\": \"4.0338700000000065\"}", - "WildBench - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "WildBench - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # prompt tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # output tokens": "{\"description\": \"min=938.586, mean=938.586, max=938.586, sum=938.586 (1)\", \"tab\": \"General information\", \"score\": \"938.586\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"v2\"" - } - } - }, - { - "evaluation_name": "Omni-MATH", - "source_data": { - "dataset_name": "Omni-MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Acc on Omni-MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.233, - "details": { - "description": "min=0.233, mean=0.233, max=0.233, sum=0.233 (1)", - "tab": "Accuracy", - "Omni-MATH - Observed inference time (s)": "{\"description\": \"min=2.418, mean=2.418, max=2.418, sum=2.418 (1)\", \"tab\": \"Efficiency\", \"score\": \"2.4179019999999993\"}", - "Omni-MATH - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "Omni-MATH - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - # prompt tokens": "{\"description\": \"min=117.921, mean=117.921, max=117.921, sum=117.921 (1)\", \"tab\": \"General information\", \"score\": \"117.921\"}", - "Omni-MATH - # output tokens": "{\"description\": \"min=788.8, mean=788.8, max=788.8, sum=788.8 (1)\", \"tab\": \"General information\", \"score\": \"788.8\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_lite/amazon_nova-lite-v1:0/1774096306.427425", - "retrieved_timestamp": "1774096306.427425", - "source_metadata": { - "source_name": "helm_lite", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_lite", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_lite", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.708, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.9832833957553059\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.768, - "details": { - "description": "min=0.768, mean=0.768, max=0.768, sum=0.768 (1)", - "tab": "Accuracy", - "NarrativeQA - Observed inference time (s)": "{\"description\": \"min=0.227, mean=0.227, max=0.227, sum=0.227 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.22699436619718286\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=3550.577, mean=3550.577, max=3550.577, sum=3550.577 (1)\", \"tab\": \"General information\", \"score\": \"3550.5774647887324\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=4.701, mean=4.701, max=4.701, sum=4.701 (1)\", \"tab\": \"General information\", \"score\": \"4.701408450704226\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (closed-book)", - "source_data": { - "dataset_name": "NaturalQuestions (closed-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (closed-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.352, - "details": { - "description": "min=0.352, mean=0.352, max=0.352, sum=0.352 (1)", - "tab": "Accuracy", - "NaturalQuestions (open-book) - Observed inference time (s)": "{\"description\": \"min=0.206, mean=0.206, max=0.206, sum=0.206 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.20557699999999976\"}", - "NaturalQuestions (closed-book) - Observed inference time (s)": "{\"description\": \"min=0.155, mean=0.155, max=0.155, sum=0.155 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.15455700000000017\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1773.944, mean=1773.944, max=1773.944, sum=1773.944 (1)\", \"tab\": \"General information\", \"score\": \"1773.944\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=4.835, mean=4.835, max=4.835, sum=4.835 (1)\", \"tab\": \"General information\", \"score\": \"4.835\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=153.254, mean=153.254, max=153.254, sum=153.254 (1)\", \"tab\": \"General information\", \"score\": \"153.254\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=4.084, mean=4.084, max=4.084, sum=4.084 (1)\", \"tab\": \"General information\", \"score\": \"4.084\"}" - } - }, - "generation_config": { - "additional_details": { - "mode": "\"closedbook\"" - } - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.928, - "details": { - "description": "min=0.928, mean=0.928, max=0.928, sum=0.928 (1)", - "tab": "Accuracy", - "OpenbookQA - Observed inference time (s)": "{\"description\": \"min=0.113, mean=0.113, max=0.113, sum=0.113 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.11279599999999983\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=282.21, mean=282.21, max=282.21, sum=282.21 (1)\", \"tab\": \"General information\", \"score\": \"282.21\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "dataset": "\"openbookqa\"", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.693, - "details": { - "description": "min=0.52, mean=0.693, max=0.92, sum=3.465 (5)", - "tab": "Accuracy", - "MMLU - Observed inference time (s)": "{\"description\": \"min=0.124, mean=0.13, max=0.136, sum=0.651 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.13027701754385965\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=399.38, mean=500.274, max=652.07, sum=2501.37 (5)\", \"tab\": \"General information\", \"score\": \"500.2740350877192\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"college_chemistry\", \"computer_security\", \"econometrics\", \"us_foreign_policy\"]", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MATH", - "source_data": { - "dataset_name": "MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Equivalent (CoT) on MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.779, - "details": { - "description": "min=0.579, mean=0.779, max=0.911, sum=5.45 (7)", - "tab": "Accuracy", - "MATH - Observed inference time (s)": "{\"description\": \"min=0.693, mean=0.836, max=1.148, sum=5.85 (7)\", \"tab\": \"Efficiency\", \"score\": \"0.8356917305438115\"}", - "MATH - # eval": "{\"description\": \"min=30, mean=62.429, max=135, sum=437 (7)\", \"tab\": \"General information\", \"score\": \"62.42857142857143\"}", - "MATH - # train": "{\"description\": \"min=8, mean=8, max=8, sum=56 (7)\", \"tab\": \"General information\", \"score\": \"8.0\"}", - "MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MATH - # prompt tokens": "{\"description\": \"min=925.556, mean=1394.735, max=2468.942, sum=9763.147 (7)\", \"tab\": \"General information\", \"score\": \"1394.7353092779651\"}", - "MATH - # output tokens": "{\"description\": \"min=61.4, mean=78.742, max=112.526, sum=551.195 (7)\", \"tab\": \"General information\", \"score\": \"78.74214942544197\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"algebra\", \"counting_and_probability\", \"geometry\", \"intermediate_algebra\", \"number_theory\", \"prealgebra\", \"precalculus\"]", - "level": "\"1\"", - "use_official_examples": "\"False\"", - "use_chain_of_thought": "\"True\"" - } - } - }, - { - "evaluation_name": "GSM8K", - "source_data": { - "dataset_name": "GSM8K", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on GSM8K", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.829, - "details": { - "description": "min=0.829, mean=0.829, max=0.829, sum=0.829 (1)", - "tab": "Accuracy", - "GSM8K - Observed inference time (s)": "{\"description\": \"min=1.063, mean=1.063, max=1.063, sum=1.063 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.0628889999999993\"}", - "GSM8K - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "GSM8K - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "GSM8K - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GSM8K - # prompt tokens": "{\"description\": \"min=957.869, mean=957.869, max=957.869, sum=957.869 (1)\", \"tab\": \"General information\", \"score\": \"957.869\"}", - "GSM8K - # output tokens": "{\"description\": \"min=84.074, mean=84.074, max=84.074, sum=84.074 (1)\", \"tab\": \"General information\", \"score\": \"84.074\"}" - } - }, - "generation_config": { - "additional_details": { - "stop": "\"none\"" - } - } - }, - { - "evaluation_name": "LegalBench", - "source_data": { - "dataset_name": "LegalBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on LegalBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.659, - "details": { - "description": "min=0.368, mean=0.659, max=0.947, sum=3.297 (5)", - "tab": "Accuracy", - "LegalBench - Observed inference time (s)": "{\"description\": \"min=0.118, mean=0.156, max=0.261, sum=0.782 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.15639281489418358\"}", - "LegalBench - # eval": "{\"description\": \"min=95, mean=409.4, max=1000, sum=2047 (5)\", \"tab\": \"General information\", \"score\": \"409.4\"}", - "LegalBench - # train": "{\"description\": \"min=4, mean=4.8, max=5, sum=24 (5)\", \"tab\": \"General information\", \"score\": \"4.8\"}", - "LegalBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "LegalBench - # prompt tokens": "{\"description\": \"min=241.632, mean=1581.083, max=6449.798, sum=7905.414 (5)\", \"tab\": \"General information\", \"score\": \"1581.0827222540588\"}", - "LegalBench - # output tokens": "{\"description\": \"min=1, mean=1.488, max=2.6, sum=7.439 (5)\", \"tab\": \"General information\", \"score\": \"1.4878474114441418\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"abercrombie\", \"corporate_lobbying\", \"function_of_decision_section\", \"international_citizenship_questions\", \"proa\"]", - "stop": "\"none\"" - } - } - }, - { - "evaluation_name": "MedQA", - "source_data": { - "dataset_name": "MedQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MedQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.696, - "details": { - "description": "min=0.696, mean=0.696, max=0.696, sum=0.696 (1)", - "tab": "Accuracy", - "MedQA - Observed inference time (s)": "{\"description\": \"min=0.132, mean=0.132, max=0.132, sum=0.132 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.1322564612326044\"}", - "MedQA - # eval": "{\"description\": \"min=503, mean=503, max=503, sum=503 (1)\", \"tab\": \"General information\", \"score\": \"503.0\"}", - "MedQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MedQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MedQA - # prompt tokens": "{\"description\": \"min=1066.861, mean=1066.861, max=1066.861, sum=1066.861 (1)\", \"tab\": \"General information\", \"score\": \"1066.8608349900596\"}", - "MedQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WMT 2014", - "source_data": { - "dataset_name": "WMT 2014", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "BLEU-4 on WMT 2014", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.204, - "details": { - "description": "min=0.126, mean=0.204, max=0.25, sum=1.021 (5)", - "tab": "Accuracy", - "WMT 2014 - Observed inference time (s)": "{\"description\": \"min=0.304, mean=0.336, max=0.406, sum=1.68 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.3359064091413061\"}", - "WMT 2014 - # eval": "{\"description\": \"min=503, mean=568.8, max=832, sum=2844 (5)\", \"tab\": \"General information\", \"score\": \"568.8\"}", - "WMT 2014 - # train": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "WMT 2014 - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WMT 2014 - # prompt tokens": "{\"description\": \"min=163.93, mean=208.694, max=268.662, sum=1043.469 (5)\", \"tab\": \"General information\", \"score\": \"208.69386660804403\"}", - "WMT 2014 - # output tokens": "{\"description\": \"min=24.457, mean=29.543, max=42.627, sum=147.715 (5)\", \"tab\": \"General information\", \"score\": \"29.542975799051845\"}" - } - }, - "generation_config": { - "additional_details": { - "language_pair": "[\"cs-en\", \"de-en\", \"fr-en\", \"hi-en\", \"ru-en\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_mmlu/amazon_nova-lite-v1:0/1774096312.00548", - "retrieved_timestamp": "1774096312.00548", - "source_metadata": { - "source_name": "helm_mmlu", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_mmlu", - "evaluation_results": [ - { - "evaluation_name": "MMLU All Subjects", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU All Subjects", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.77, - "details": { - "description": "min=0.509, mean=0.77, max=0.969, sum=87.802 (114)", - "tab": "Accuracy", - "MMLU All Subjects - Observed inference time (s)": "{\"description\": \"min=0.113, mean=0.127, max=0.174, sum=14.526 (114)\", \"tab\": \"Efficiency\", \"score\": \"0.12742174922519597\"}", - "MMLU All Subjects - # eval": "{\"description\": \"min=100, mean=246.351, max=1534, sum=28084 (114)\", \"tab\": \"General information\", \"score\": \"246.35087719298247\"}", - "MMLU All Subjects - # train": "{\"description\": \"min=5, mean=5, max=5, sum=570 (114)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU All Subjects - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU All Subjects - # prompt tokens": "{\"description\": \"min=305.386, mean=655.489, max=2872.03, sum=74725.746 (114)\", \"tab\": \"General information\", \"score\": \"655.4890026560713\"}", - "MMLU All Subjects - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=114 (114)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_name": "Abstract Algebra", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Abstract Algebra", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.52, - "details": { - "description": "min=0.52, mean=0.52, max=0.52, sum=1.04 (2)", - "tab": "Accuracy", - "Abstract Algebra - Observed inference time (s)": "{\"description\": \"min=0.136, mean=0.136, max=0.136, sum=0.272 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.13592\"}", - "Abstract Algebra - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Abstract Algebra - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Abstract Algebra - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Abstract Algebra - # prompt tokens": "{\"description\": \"min=399.38, mean=399.38, max=399.38, sum=798.76 (2)\", \"tab\": \"General information\", \"score\": \"399.38\"}", - "Abstract Algebra - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"abstract_algebra\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_abstract_algebra\"" - } - } - }, - { - "evaluation_name": "Anatomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Anatomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.719, - "details": { - "description": "min=0.719, mean=0.719, max=0.719, sum=1.437 (2)", - "tab": "Accuracy", - "Anatomy - Observed inference time (s)": "{\"description\": \"min=0.124, mean=0.124, max=0.124, sum=0.248 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.12411851851851854\"}", - "Anatomy - # eval": "{\"description\": \"min=135, mean=135, max=135, sum=270 (2)\", \"tab\": \"General information\", \"score\": \"135.0\"}", - "Anatomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Anatomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Anatomy - # prompt tokens": "{\"description\": \"min=400.081, mean=400.081, max=400.081, sum=800.163 (2)\", \"tab\": \"General information\", \"score\": \"400.0814814814815\"}", - "Anatomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"anatomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_anatomy\"" - } - } - }, - { - "evaluation_name": "College Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on College Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.608, - "details": { - "description": "min=0.608, mean=0.608, max=0.608, sum=1.216 (2)", - "tab": "Accuracy", - "College Chemistry - Observed inference time (s)": "{\"description\": \"min=0.133, mean=0.133, max=0.133, sum=0.265 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.13258\"}", - "College Biology - Observed inference time (s)": "{\"description\": \"min=0.126, mean=0.126, max=0.126, sum=0.252 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.12590277777777775\"}", - "College Computer Science - Observed inference time (s)": "{\"description\": \"min=0.137, mean=0.137, max=0.137, sum=0.274 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.13685\"}", - "College Mathematics - Observed inference time (s)": "{\"description\": \"min=0.134, mean=0.134, max=0.134, sum=0.268 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.13410999999999995\"}", - "College Medicine - Observed inference time (s)": "{\"description\": \"min=0.129, mean=0.129, max=0.129, sum=0.258 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.12883815028901727\"}", - "College Physics - Observed inference time (s)": "{\"description\": \"min=0.129, mean=0.129, max=0.129, sum=0.258 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.12883333333333336\"}", - "College Chemistry - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Chemistry - # prompt tokens": "{\"description\": \"min=573.4, mean=573.4, max=573.4, sum=1146.8 (2)\", \"tab\": \"General information\", \"score\": \"573.4\"}", - "College Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Biology - # eval": "{\"description\": \"min=144, mean=144, max=144, sum=288 (2)\", \"tab\": \"General information\", \"score\": \"144.0\"}", - "College Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # prompt tokens": "{\"description\": \"min=510.278, mean=510.278, max=510.278, sum=1020.556 (2)\", \"tab\": \"General information\", \"score\": \"510.27777777777777\"}", - "College Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # prompt tokens": "{\"description\": \"min=880.15, mean=880.15, max=880.15, sum=1760.3 (2)\", \"tab\": \"General information\", \"score\": \"880.15\"}", - "College Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Mathematics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # prompt tokens": "{\"description\": \"min=639.53, mean=639.53, max=639.53, sum=1279.06 (2)\", \"tab\": \"General information\", \"score\": \"639.53\"}", - "College Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Medicine - # eval": "{\"description\": \"min=173, mean=173, max=173, sum=346 (2)\", \"tab\": \"General information\", \"score\": \"173.0\"}", - "College Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # prompt tokens": "{\"description\": \"min=558.301, mean=558.301, max=558.301, sum=1116.601 (2)\", \"tab\": \"General information\", \"score\": \"558.3005780346821\"}", - "College Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Physics - # eval": "{\"description\": \"min=102, mean=102, max=102, sum=204 (2)\", \"tab\": \"General information\", \"score\": \"102.0\"}", - "College Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # prompt tokens": "{\"description\": \"min=517.324, mean=517.324, max=517.324, sum=1034.647 (2)\", \"tab\": \"General information\", \"score\": \"517.3235294117648\"}", - "College Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"college_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_college_physics\"" - } - } - }, - { - "evaluation_name": "Computer Security", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Computer Security", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.79, - "details": { - "description": "min=0.79, mean=0.79, max=0.79, sum=1.58 (2)", - "tab": "Accuracy", - "Computer Security - Observed inference time (s)": "{\"description\": \"min=0.124, mean=0.124, max=0.124, sum=0.247 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.12359999999999999\"}", - "Computer Security - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Computer Security - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Computer Security - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Computer Security - # prompt tokens": "{\"description\": \"min=415.4, mean=415.4, max=415.4, sum=830.8 (2)\", \"tab\": \"General information\", \"score\": \"415.4\"}", - "Computer Security - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"computer_security\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_computer_security\"" - } - } - }, - { - "evaluation_name": "Econometrics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Econometrics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.675, - "details": { - "description": "min=0.675, mean=0.675, max=0.675, sum=1.351 (2)", - "tab": "Accuracy", - "Econometrics - Observed inference time (s)": "{\"description\": \"min=0.132, mean=0.132, max=0.132, sum=0.263 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.13153508771929825\"}", - "Econometrics - # eval": "{\"description\": \"min=114, mean=114, max=114, sum=228 (2)\", \"tab\": \"General information\", \"score\": \"114.0\"}", - "Econometrics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Econometrics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Econometrics - # prompt tokens": "{\"description\": \"min=652.07, mean=652.07, max=652.07, sum=1304.14 (2)\", \"tab\": \"General information\", \"score\": \"652.0701754385965\"}", - "Econometrics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"econometrics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_econometrics\"" - } - } - }, - { - "evaluation_name": "Global Facts", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Global Facts", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.55, - "details": { - "description": "min=0.55, mean=0.55, max=0.55, sum=1.1 (2)", - "tab": "Accuracy", - "Global Facts - Observed inference time (s)": "{\"description\": \"min=0.127, mean=0.127, max=0.127, sum=0.255 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.12749\"}", - "Global Facts - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Global Facts - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Global Facts - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Global Facts - # prompt tokens": "{\"description\": \"min=426.42, mean=426.42, max=426.42, sum=852.84 (2)\", \"tab\": \"General information\", \"score\": \"426.42\"}", - "Global Facts - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"global_facts\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_global_facts\"" - } - } - }, - { - "evaluation_name": "Jurisprudence", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Jurisprudence", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.852, - "details": { - "description": "min=0.852, mean=0.852, max=0.852, sum=1.704 (2)", - "tab": "Accuracy", - "Jurisprudence - Observed inference time (s)": "{\"description\": \"min=0.124, mean=0.124, max=0.124, sum=0.248 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.12411111111111109\"}", - "Jurisprudence - # eval": "{\"description\": \"min=108, mean=108, max=108, sum=216 (2)\", \"tab\": \"General information\", \"score\": \"108.0\"}", - "Jurisprudence - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Jurisprudence - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Jurisprudence - # prompt tokens": "{\"description\": \"min=446.722, mean=446.722, max=446.722, sum=893.444 (2)\", \"tab\": \"General information\", \"score\": \"446.72222222222223\"}", - "Jurisprudence - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"jurisprudence\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_jurisprudence\"" - } - } - }, - { - "evaluation_name": "Philosophy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Philosophy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.817, - "details": { - "description": "min=0.817, mean=0.817, max=0.817, sum=1.633 (2)", - "tab": "Accuracy", - "Philosophy - Observed inference time (s)": "{\"description\": \"min=0.121, mean=0.121, max=0.121, sum=0.242 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.12122186495176847\"}", - "Philosophy - # eval": "{\"description\": \"min=311, mean=311, max=311, sum=622 (2)\", \"tab\": \"General information\", \"score\": \"311.0\"}", - "Philosophy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Philosophy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Philosophy - # prompt tokens": "{\"description\": \"min=381.704, mean=381.704, max=381.704, sum=763.408 (2)\", \"tab\": \"General information\", \"score\": \"381.7041800643087\"}", - "Philosophy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"philosophy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_philosophy\"" - } - } - }, - { - "evaluation_name": "Professional Psychology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Professional Psychology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.812, - "details": { - "description": "min=0.812, mean=0.812, max=0.812, sum=1.624 (2)", - "tab": "Accuracy", - "Professional Medicine - Observed inference time (s)": "{\"description\": \"min=0.139, mean=0.139, max=0.139, sum=0.277 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.13866176470588237\"}", - "Professional Accounting - Observed inference time (s)": "{\"description\": \"min=0.126, mean=0.126, max=0.126, sum=0.253 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.1264397163120567\"}", - "Professional Law - Observed inference time (s)": "{\"description\": \"min=0.143, mean=0.143, max=0.143, sum=0.286 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.14286505867014285\"}", - "Professional Psychology - Observed inference time (s)": "{\"description\": \"min=0.124, mean=0.124, max=0.124, sum=0.248 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.12417647058823517\"}", - "Professional Medicine - # eval": "{\"description\": \"min=272, mean=272, max=272, sum=544 (2)\", \"tab\": \"General information\", \"score\": \"272.0\"}", - "Professional Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Medicine - # prompt tokens": "{\"description\": \"min=1146.287, mean=1146.287, max=1146.287, sum=2292.574 (2)\", \"tab\": \"General information\", \"score\": \"1146.2867647058824\"}", - "Professional Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Accounting - # eval": "{\"description\": \"min=282, mean=282, max=282, sum=564 (2)\", \"tab\": \"General information\", \"score\": \"282.0\"}", - "Professional Accounting - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Accounting - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # prompt tokens": "{\"description\": \"min=688.72, mean=688.72, max=688.72, sum=1377.44 (2)\", \"tab\": \"General information\", \"score\": \"688.7198581560284\"}", - "Professional Accounting - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Law - # eval": "{\"description\": \"min=1534, mean=1534, max=1534, sum=3068 (2)\", \"tab\": \"General information\", \"score\": \"1534.0\"}", - "Professional Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # prompt tokens": "{\"description\": \"min=1686.73, mean=1686.73, max=1686.73, sum=3373.46 (2)\", \"tab\": \"General information\", \"score\": \"1686.7301173402868\"}", - "Professional Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Psychology - # eval": "{\"description\": \"min=612, mean=612, max=612, sum=1224 (2)\", \"tab\": \"General information\", \"score\": \"612.0\"}", - "Professional Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # prompt tokens": "{\"description\": \"min=625.574, mean=625.574, max=625.574, sum=1251.147 (2)\", \"tab\": \"General information\", \"score\": \"625.5735294117648\"}", - "Professional Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"professional_psychology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_professional_psychology\"" - } - } - }, - { - "evaluation_name": "Us Foreign Policy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Us Foreign Policy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.92, - "details": { - "description": "min=0.92, mean=0.92, max=0.92, sum=1.84 (2)", - "tab": "Accuracy", - "Us Foreign Policy - Observed inference time (s)": "{\"description\": \"min=0.128, mean=0.128, max=0.128, sum=0.256 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.12775000000000003\"}", - "Us Foreign Policy - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Us Foreign Policy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Us Foreign Policy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Us Foreign Policy - # prompt tokens": "{\"description\": \"min=461.12, mean=461.12, max=461.12, sum=922.24 (2)\", \"tab\": \"General information\", \"score\": \"461.12\"}", - "Us Foreign Policy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"us_foreign_policy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_us_foreign_policy\"" - } - } - }, - { - "evaluation_name": "Astronomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Astronomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.862, - "details": { - "description": "min=0.862, mean=0.862, max=0.862, sum=1.724 (2)", - "tab": "Accuracy", - "Astronomy - Observed inference time (s)": "{\"description\": \"min=0.129, mean=0.129, max=0.129, sum=0.258 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.12905921052631578\"}", - "Astronomy - # eval": "{\"description\": \"min=152, mean=152, max=152, sum=304 (2)\", \"tab\": \"General information\", \"score\": \"152.0\"}", - "Astronomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Astronomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Astronomy - # prompt tokens": "{\"description\": \"min=628.112, mean=628.112, max=628.112, sum=1256.224 (2)\", \"tab\": \"General information\", \"score\": \"628.1118421052631\"}", - "Astronomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"astronomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_astronomy\"" - } - } - }, - { - "evaluation_name": "Business Ethics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Business Ethics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.73, - "details": { - "description": "min=0.73, mean=0.73, max=0.73, sum=1.46 (2)", - "tab": "Accuracy", - "Business Ethics - Observed inference time (s)": "{\"description\": \"min=0.126, mean=0.126, max=0.126, sum=0.252 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.12613000000000005\"}", - "Business Ethics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Business Ethics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Business Ethics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Business Ethics - # prompt tokens": "{\"description\": \"min=617.46, mean=617.46, max=617.46, sum=1234.92 (2)\", \"tab\": \"General information\", \"score\": \"617.46\"}", - "Business Ethics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"business_ethics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_business_ethics\"" - } - } - }, - { - "evaluation_name": "Clinical Knowledge", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Clinical Knowledge", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8, - "details": { - "description": "min=0.8, mean=0.8, max=0.8, sum=1.6 (2)", - "tab": "Accuracy", - "Clinical Knowledge - Observed inference time (s)": "{\"description\": \"min=0.126, mean=0.126, max=0.126, sum=0.251 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.1255018867924528\"}", - "Clinical Knowledge - # eval": "{\"description\": \"min=265, mean=265, max=265, sum=530 (2)\", \"tab\": \"General information\", \"score\": \"265.0\"}", - "Clinical Knowledge - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Clinical Knowledge - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Clinical Knowledge - # prompt tokens": "{\"description\": \"min=451.925, mean=451.925, max=451.925, sum=903.849 (2)\", \"tab\": \"General information\", \"score\": \"451.92452830188677\"}", - "Clinical Knowledge - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"clinical_knowledge\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_clinical_knowledge\"" - } - } - }, - { - "evaluation_name": "Conceptual Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Conceptual Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.796, - "details": { - "description": "min=0.796, mean=0.796, max=0.796, sum=1.591 (2)", - "tab": "Accuracy", - "Conceptual Physics - Observed inference time (s)": "{\"description\": \"min=0.115, mean=0.115, max=0.115, sum=0.23 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.11518723404255315\"}", - "Conceptual Physics - # eval": "{\"description\": \"min=235, mean=235, max=235, sum=470 (2)\", \"tab\": \"General information\", \"score\": \"235.0\"}", - "Conceptual Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Conceptual Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Conceptual Physics - # prompt tokens": "{\"description\": \"min=341.723, mean=341.723, max=341.723, sum=683.447 (2)\", \"tab\": \"General information\", \"score\": \"341.72340425531917\"}", - "Conceptual Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"conceptual_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_conceptual_physics\"" - } - } - }, - { - "evaluation_name": "Electrical Engineering", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Electrical Engineering", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.779, - "details": { - "description": "min=0.779, mean=0.779, max=0.779, sum=1.559 (2)", - "tab": "Accuracy", - "Electrical Engineering - Observed inference time (s)": "{\"description\": \"min=0.116, mean=0.116, max=0.116, sum=0.232 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.11609655172413792\"}", - "Electrical Engineering - # eval": "{\"description\": \"min=145, mean=145, max=145, sum=290 (2)\", \"tab\": \"General information\", \"score\": \"145.0\"}", - "Electrical Engineering - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Electrical Engineering - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Electrical Engineering - # prompt tokens": "{\"description\": \"min=458.345, mean=458.345, max=458.345, sum=916.69 (2)\", \"tab\": \"General information\", \"score\": \"458.3448275862069\"}", - "Electrical Engineering - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"electrical_engineering\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_electrical_engineering\"" - } - } - }, - { - "evaluation_name": "Elementary Mathematics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Elementary Mathematics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.757, - "details": { - "description": "min=0.757, mean=0.757, max=0.757, sum=1.513 (2)", - "tab": "Accuracy", - "Elementary Mathematics - Observed inference time (s)": "{\"description\": \"min=0.126, mean=0.126, max=0.126, sum=0.253 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.12626455026455036\"}", - "Elementary Mathematics - # eval": "{\"description\": \"min=378, mean=378, max=378, sum=756 (2)\", \"tab\": \"General information\", \"score\": \"378.0\"}", - "Elementary Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Elementary Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Elementary Mathematics - # prompt tokens": "{\"description\": \"min=534.09, mean=534.09, max=534.09, sum=1068.18 (2)\", \"tab\": \"General information\", \"score\": \"534.0899470899471\"}", - "Elementary Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"elementary_mathematics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_elementary_mathematics\"" - } - } - }, - { - "evaluation_name": "Formal Logic", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Formal Logic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.643, - "details": { - "description": "min=0.643, mean=0.643, max=0.643, sum=1.286 (2)", - "tab": "Accuracy", - "Formal Logic - Observed inference time (s)": "{\"description\": \"min=0.129, mean=0.129, max=0.129, sum=0.257 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.12850793650793654\"}", - "Formal Logic - # eval": "{\"description\": \"min=126, mean=126, max=126, sum=252 (2)\", \"tab\": \"General information\", \"score\": \"126.0\"}", - "Formal Logic - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Formal Logic - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Formal Logic - # prompt tokens": "{\"description\": \"min=669, mean=669, max=669, sum=1338 (2)\", \"tab\": \"General information\", \"score\": \"669.0\"}", - "Formal Logic - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"formal_logic\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_formal_logic\"" - } - } - }, - { - "evaluation_name": "High School World History", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on High School World History", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.886, - "details": { - "description": "min=0.886, mean=0.886, max=0.886, sum=1.772 (2)", - "tab": "Accuracy", - "High School Biology - Observed inference time (s)": "{\"description\": \"min=0.122, mean=0.122, max=0.122, sum=0.244 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.12203870967741924\"}", - "High School Chemistry - Observed inference time (s)": "{\"description\": \"min=0.127, mean=0.127, max=0.127, sum=0.254 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.1271921182266009\"}", - "High School Computer Science - Observed inference time (s)": "{\"description\": \"min=0.136, mean=0.136, max=0.136, sum=0.271 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.13555999999999999\"}", - "High School European History - Observed inference time (s)": "{\"description\": \"min=0.174, mean=0.174, max=0.174, sum=0.348 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.1741696969696969\"}", - "High School Geography - Observed inference time (s)": "{\"description\": \"min=0.123, mean=0.123, max=0.123, sum=0.245 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.1226313131313131\"}", - "High School Government And Politics - Observed inference time (s)": "{\"description\": \"min=0.125, mean=0.125, max=0.125, sum=0.251 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.12531606217616578\"}", - "High School Macroeconomics - Observed inference time (s)": "{\"description\": \"min=0.121, mean=0.121, max=0.121, sum=0.242 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.12077948717948701\"}", - "High School Mathematics - Observed inference time (s)": "{\"description\": \"min=0.126, mean=0.126, max=0.126, sum=0.251 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.1257444444444444\"}", - "High School Microeconomics - Observed inference time (s)": "{\"description\": \"min=0.123, mean=0.123, max=0.123, sum=0.247 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.12331512605042017\"}", - "High School Physics - Observed inference time (s)": "{\"description\": \"min=0.128, mean=0.128, max=0.128, sum=0.256 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.1282052980132451\"}", - "High School Psychology - Observed inference time (s)": "{\"description\": \"min=0.123, mean=0.123, max=0.123, sum=0.246 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.12288256880733935\"}", - "High School Statistics - Observed inference time (s)": "{\"description\": \"min=0.13, mean=0.13, max=0.13, sum=0.261 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.13030555555555556\"}", - "High School US History - Observed inference time (s)": "{\"description\": \"min=0.161, mean=0.161, max=0.161, sum=0.322 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.16099019607843132\"}", - "High School World History - Observed inference time (s)": "{\"description\": \"min=0.146, mean=0.146, max=0.146, sum=0.293 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.14643881856540092\"}", - "High School Biology - # eval": "{\"description\": \"min=310, mean=310, max=310, sum=620 (2)\", \"tab\": \"General information\", \"score\": \"310.0\"}", - "High School Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Biology - # prompt tokens": "{\"description\": \"min=568.748, mean=568.748, max=568.748, sum=1137.497 (2)\", \"tab\": \"General information\", \"score\": \"568.7483870967742\"}", - "High School Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Chemistry - # eval": "{\"description\": \"min=203, mean=203, max=203, sum=406 (2)\", \"tab\": \"General information\", \"score\": \"203.0\"}", - "High School Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # prompt tokens": "{\"description\": \"min=523.65, mean=523.65, max=523.65, sum=1047.3 (2)\", \"tab\": \"General information\", \"score\": \"523.6502463054187\"}", - "High School Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "High School Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # prompt tokens": "{\"description\": \"min=932.15, mean=932.15, max=932.15, sum=1864.3 (2)\", \"tab\": \"General information\", \"score\": \"932.15\"}", - "High School Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School European History - # eval": "{\"description\": \"min=165, mean=165, max=165, sum=330 (2)\", \"tab\": \"General information\", \"score\": \"165.0\"}", - "High School European History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School European History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # prompt tokens": "{\"description\": \"min=2872.03, mean=2872.03, max=2872.03, sum=5744.061 (2)\", \"tab\": \"General information\", \"score\": \"2872.030303030303\"}", - "High School European History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Geography - # eval": "{\"description\": \"min=198, mean=198, max=198, sum=396 (2)\", \"tab\": \"General information\", \"score\": \"198.0\"}", - "High School Geography - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Geography - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # prompt tokens": "{\"description\": \"min=425.646, mean=425.646, max=425.646, sum=851.293 (2)\", \"tab\": \"General information\", \"score\": \"425.64646464646466\"}", - "High School Geography - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Government And Politics - # eval": "{\"description\": \"min=193, mean=193, max=193, sum=386 (2)\", \"tab\": \"General information\", \"score\": \"193.0\"}", - "High School Government And Politics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Government And Politics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # prompt tokens": "{\"description\": \"min=506.073, mean=506.073, max=506.073, sum=1012.145 (2)\", \"tab\": \"General information\", \"score\": \"506.07253886010363\"}", - "High School Government And Politics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Macroeconomics - # eval": "{\"description\": \"min=390, mean=390, max=390, sum=780 (2)\", \"tab\": \"General information\", \"score\": \"390.0\"}", - "High School Macroeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Macroeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # prompt tokens": "{\"description\": \"min=419.987, mean=419.987, max=419.987, sum=839.974 (2)\", \"tab\": \"General information\", \"score\": \"419.9871794871795\"}", - "High School Macroeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Mathematics - # eval": "{\"description\": \"min=270, mean=270, max=270, sum=540 (2)\", \"tab\": \"General information\", \"score\": \"270.0\"}", - "High School Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # prompt tokens": "{\"description\": \"min=554.352, mean=554.352, max=554.352, sum=1108.704 (2)\", \"tab\": \"General information\", \"score\": \"554.3518518518518\"}", - "High School Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Microeconomics - # eval": "{\"description\": \"min=238, mean=238, max=238, sum=476 (2)\", \"tab\": \"General information\", \"score\": \"238.0\"}", - "High School Microeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Microeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # prompt tokens": "{\"description\": \"min=439.055, mean=439.055, max=439.055, sum=878.109 (2)\", \"tab\": \"General information\", \"score\": \"439.0546218487395\"}", - "High School Microeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Physics - # eval": "{\"description\": \"min=151, mean=151, max=151, sum=302 (2)\", \"tab\": \"General information\", \"score\": \"151.0\"}", - "High School Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # prompt tokens": "{\"description\": \"min=581.669, mean=581.669, max=581.669, sum=1163.338 (2)\", \"tab\": \"General information\", \"score\": \"581.6688741721854\"}", - "High School Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Psychology - # eval": "{\"description\": \"min=545, mean=545, max=545, sum=1090 (2)\", \"tab\": \"General information\", \"score\": \"545.0\"}", - "High School Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # prompt tokens": "{\"description\": \"min=544.842, mean=544.842, max=544.842, sum=1089.684 (2)\", \"tab\": \"General information\", \"score\": \"544.8422018348624\"}", - "High School Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Statistics - # eval": "{\"description\": \"min=216, mean=216, max=216, sum=432 (2)\", \"tab\": \"General information\", \"score\": \"216.0\"}", - "High School Statistics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Statistics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # prompt tokens": "{\"description\": \"min=833, mean=833, max=833, sum=1666 (2)\", \"tab\": \"General information\", \"score\": \"833.0\"}", - "High School Statistics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School US History - # eval": "{\"description\": \"min=204, mean=204, max=204, sum=408 (2)\", \"tab\": \"General information\", \"score\": \"204.0\"}", - "High School US History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School US History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # prompt tokens": "{\"description\": \"min=2270.25, mean=2270.25, max=2270.25, sum=4540.5 (2)\", \"tab\": \"General information\", \"score\": \"2270.25\"}", - "High School US History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School World History - # eval": "{\"description\": \"min=237, mean=237, max=237, sum=474 (2)\", \"tab\": \"General information\", \"score\": \"237.0\"}", - "High School World History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School World History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # prompt tokens": "{\"description\": \"min=1466.561, mean=1466.561, max=1466.561, sum=2933.122 (2)\", \"tab\": \"General information\", \"score\": \"1466.5611814345991\"}", - "High School World History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"high_school_world_history\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_high_school_world_history\"" - } - } - }, - { - "evaluation_name": "Human Sexuality", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Human Sexuality", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.84, - "details": { - "description": "min=0.84, mean=0.84, max=0.84, sum=1.679 (2)", - "tab": "Accuracy", - "Human Aging - Observed inference time (s)": "{\"description\": \"min=0.113, mean=0.113, max=0.113, sum=0.227 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.11326008968609867\"}", - "Human Sexuality - Observed inference time (s)": "{\"description\": \"min=0.118, mean=0.118, max=0.118, sum=0.236 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.11813740458015273\"}", - "Human Aging - # eval": "{\"description\": \"min=223, mean=223, max=223, sum=446 (2)\", \"tab\": \"General information\", \"score\": \"223.0\"}", - "Human Aging - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Aging - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Aging - # prompt tokens": "{\"description\": \"min=352.48, mean=352.48, max=352.48, sum=704.96 (2)\", \"tab\": \"General information\", \"score\": \"352.47982062780267\"}", - "Human Aging - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Human Sexuality - # eval": "{\"description\": \"min=131, mean=131, max=131, sum=262 (2)\", \"tab\": \"General information\", \"score\": \"131.0\"}", - "Human Sexuality - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Sexuality - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # prompt tokens": "{\"description\": \"min=385.626, mean=385.626, max=385.626, sum=771.252 (2)\", \"tab\": \"General information\", \"score\": \"385.62595419847327\"}", - "Human Sexuality - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"human_sexuality\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_human_sexuality\"" - } - } - }, - { - "evaluation_name": "International Law", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on International Law", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.843, - "details": { - "description": "min=0.843, mean=0.843, max=0.843, sum=1.686 (2)", - "tab": "Accuracy", - "International Law - Observed inference time (s)": "{\"description\": \"min=0.129, mean=0.129, max=0.129, sum=0.258 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.129206611570248\"}", - "International Law - # eval": "{\"description\": \"min=121, mean=121, max=121, sum=242 (2)\", \"tab\": \"General information\", \"score\": \"121.0\"}", - "International Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "International Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "International Law - # prompt tokens": "{\"description\": \"min=667.843, mean=667.843, max=667.843, sum=1335.686 (2)\", \"tab\": \"General information\", \"score\": \"667.8429752066115\"}", - "International Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"international_law\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_international_law\"" - } - } - }, - { - "evaluation_name": "Logical Fallacies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Logical Fallacies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.81, - "details": { - "description": "min=0.81, mean=0.81, max=0.81, sum=1.62 (2)", - "tab": "Accuracy", - "Logical Fallacies - Observed inference time (s)": "{\"description\": \"min=0.124, mean=0.124, max=0.124, sum=0.249 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.12445398773006137\"}", - "Logical Fallacies - # eval": "{\"description\": \"min=163, mean=163, max=163, sum=326 (2)\", \"tab\": \"General information\", \"score\": \"163.0\"}", - "Logical Fallacies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Logical Fallacies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Logical Fallacies - # prompt tokens": "{\"description\": \"min=482.227, mean=482.227, max=482.227, sum=964.454 (2)\", \"tab\": \"General information\", \"score\": \"482.2269938650307\"}", - "Logical Fallacies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"logical_fallacies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_logical_fallacies\"" - } - } - }, - { - "evaluation_name": "Machine Learning", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Machine Learning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.509, - "details": { - "description": "min=0.509, mean=0.509, max=0.509, sum=1.018 (2)", - "tab": "Accuracy", - "Machine Learning - Observed inference time (s)": "{\"description\": \"min=0.135, mean=0.135, max=0.135, sum=0.27 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.13516071428571433\"}", - "Machine Learning - # eval": "{\"description\": \"min=112, mean=112, max=112, sum=224 (2)\", \"tab\": \"General information\", \"score\": \"112.0\"}", - "Machine Learning - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Machine Learning - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Machine Learning - # prompt tokens": "{\"description\": \"min=699.598, mean=699.598, max=699.598, sum=1399.196 (2)\", \"tab\": \"General information\", \"score\": \"699.5982142857143\"}", - "Machine Learning - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"machine_learning\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_machine_learning\"" - } - } - }, - { - "evaluation_name": "Management", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Management", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.864, - "details": { - "description": "min=0.864, mean=0.864, max=0.864, sum=1.728 (2)", - "tab": "Accuracy", - "Management - Observed inference time (s)": "{\"description\": \"min=0.118, mean=0.118, max=0.118, sum=0.237 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.1183980582524272\"}", - "Management - # eval": "{\"description\": \"min=103, mean=103, max=103, sum=206 (2)\", \"tab\": \"General information\", \"score\": \"103.0\"}", - "Management - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Management - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Management - # prompt tokens": "{\"description\": \"min=320.34, mean=320.34, max=320.34, sum=640.68 (2)\", \"tab\": \"General information\", \"score\": \"320.3398058252427\"}", - "Management - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"management\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_management\"" - } - } - }, - { - "evaluation_name": "Marketing", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Marketing", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.889, - "details": { - "description": "min=0.889, mean=0.889, max=0.889, sum=1.778 (2)", - "tab": "Accuracy", - "Marketing - Observed inference time (s)": "{\"description\": \"min=0.122, mean=0.122, max=0.122, sum=0.243 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.12151282051282052\"}", - "Marketing - # eval": "{\"description\": \"min=234, mean=234, max=234, sum=468 (2)\", \"tab\": \"General information\", \"score\": \"234.0\"}", - "Marketing - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Marketing - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Marketing - # prompt tokens": "{\"description\": \"min=466.697, mean=466.697, max=466.697, sum=933.393 (2)\", \"tab\": \"General information\", \"score\": \"466.6965811965812\"}", - "Marketing - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"marketing\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_marketing\"" - } - } - }, - { - "evaluation_name": "Medical Genetics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Medical Genetics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9, - "details": { - "description": "min=0.9, mean=0.9, max=0.9, sum=1.8 (2)", - "tab": "Accuracy", - "Medical Genetics - Observed inference time (s)": "{\"description\": \"min=0.115, mean=0.115, max=0.115, sum=0.23 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.11518\"}", - "Medical Genetics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Medical Genetics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Medical Genetics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Medical Genetics - # prompt tokens": "{\"description\": \"min=380.71, mean=380.71, max=380.71, sum=761.42 (2)\", \"tab\": \"General information\", \"score\": \"380.71\"}", - "Medical Genetics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"medical_genetics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_medical_genetics\"" - } - } - }, - { - "evaluation_name": "Miscellaneous", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Miscellaneous", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.872, - "details": { - "description": "min=0.872, mean=0.872, max=0.872, sum=1.745 (2)", - "tab": "Accuracy", - "Miscellaneous - Observed inference time (s)": "{\"description\": \"min=0.114, mean=0.114, max=0.114, sum=0.227 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.11356577266922054\"}", - "Miscellaneous - # eval": "{\"description\": \"min=783, mean=783, max=783, sum=1566 (2)\", \"tab\": \"General information\", \"score\": \"783.0\"}", - "Miscellaneous - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Miscellaneous - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Miscellaneous - # prompt tokens": "{\"description\": \"min=342.847, mean=342.847, max=342.847, sum=685.693 (2)\", \"tab\": \"General information\", \"score\": \"342.84674329501917\"}", - "Miscellaneous - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"miscellaneous\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_miscellaneous\"" - } - } - }, - { - "evaluation_name": "Moral Scenarios", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Moral Scenarios", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.694, - "details": { - "description": "min=0.694, mean=0.694, max=0.694, sum=1.388 (2)", - "tab": "Accuracy", - "Moral Disputes - Observed inference time (s)": "{\"description\": \"min=0.125, mean=0.125, max=0.125, sum=0.249 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.12473699421965324\"}", - "Moral Scenarios - Observed inference time (s)": "{\"description\": \"min=0.124, mean=0.124, max=0.124, sum=0.247 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.12357988826815636\"}", - "Moral Disputes - # eval": "{\"description\": \"min=346, mean=346, max=346, sum=692 (2)\", \"tab\": \"General information\", \"score\": \"346.0\"}", - "Moral Disputes - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Disputes - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Disputes - # prompt tokens": "{\"description\": \"min=525.329, mean=525.329, max=525.329, sum=1050.659 (2)\", \"tab\": \"General information\", \"score\": \"525.3294797687861\"}", - "Moral Disputes - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Moral Scenarios - # eval": "{\"description\": \"min=895, mean=895, max=895, sum=1790 (2)\", \"tab\": \"General information\", \"score\": \"895.0\"}", - "Moral Scenarios - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Scenarios - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # prompt tokens": "{\"description\": \"min=692.482, mean=692.482, max=692.482, sum=1384.963 (2)\", \"tab\": \"General information\", \"score\": \"692.4815642458101\"}", - "Moral Scenarios - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"moral_scenarios\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_moral_scenarios\"" - } - } - }, - { - "evaluation_name": "Nutrition", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Nutrition", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.788, - "details": { - "description": "min=0.788, mean=0.788, max=0.788, sum=1.575 (2)", - "tab": "Accuracy", - "Nutrition - Observed inference time (s)": "{\"description\": \"min=0.124, mean=0.124, max=0.124, sum=0.247 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.12373529411764701\"}", - "Nutrition - # eval": "{\"description\": \"min=306, mean=306, max=306, sum=612 (2)\", \"tab\": \"General information\", \"score\": \"306.0\"}", - "Nutrition - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Nutrition - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Nutrition - # prompt tokens": "{\"description\": \"min=612.69, mean=612.69, max=612.69, sum=1225.379 (2)\", \"tab\": \"General information\", \"score\": \"612.6895424836601\"}", - "Nutrition - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"nutrition\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_nutrition\"" - } - } - }, - { - "evaluation_name": "Prehistory", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Prehistory", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.849, - "details": { - "description": "min=0.849, mean=0.849, max=0.849, sum=1.698 (2)", - "tab": "Accuracy", - "Prehistory - Observed inference time (s)": "{\"description\": \"min=0.129, mean=0.129, max=0.129, sum=0.258 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.1291882716049382\"}", - "Prehistory - # eval": "{\"description\": \"min=324, mean=324, max=324, sum=648 (2)\", \"tab\": \"General information\", \"score\": \"324.0\"}", - "Prehistory - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Prehistory - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Prehistory - # prompt tokens": "{\"description\": \"min=552.454, mean=552.454, max=552.454, sum=1104.907 (2)\", \"tab\": \"General information\", \"score\": \"552.4537037037037\"}", - "Prehistory - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"prehistory\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_prehistory\"" - } - } - }, - { - "evaluation_name": "Public Relations", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Public Relations", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.682, - "details": { - "description": "min=0.682, mean=0.682, max=0.682, sum=1.364 (2)", - "tab": "Accuracy", - "Public Relations - Observed inference time (s)": "{\"description\": \"min=0.12, mean=0.12, max=0.12, sum=0.241 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.1202636363636364\"}", - "Public Relations - # eval": "{\"description\": \"min=110, mean=110, max=110, sum=220 (2)\", \"tab\": \"General information\", \"score\": \"110.0\"}", - "Public Relations - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Public Relations - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Public Relations - # prompt tokens": "{\"description\": \"min=448.609, mean=448.609, max=448.609, sum=897.218 (2)\", \"tab\": \"General information\", \"score\": \"448.6090909090909\"}", - "Public Relations - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"public_relations\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_public_relations\"" - } - } - }, - { - "evaluation_name": "Security Studies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Security Studies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.788, - "details": { - "description": "min=0.788, mean=0.788, max=0.788, sum=1.576 (2)", - "tab": "Accuracy", - "Security Studies - Observed inference time (s)": "{\"description\": \"min=0.137, mean=0.137, max=0.137, sum=0.273 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.13666530612244904\"}", - "Security Studies - # eval": "{\"description\": \"min=245, mean=245, max=245, sum=490 (2)\", \"tab\": \"General information\", \"score\": \"245.0\"}", - "Security Studies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Security Studies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Security Studies - # prompt tokens": "{\"description\": \"min=1224.433, mean=1224.433, max=1224.433, sum=2448.865 (2)\", \"tab\": \"General information\", \"score\": \"1224.4326530612245\"}", - "Security Studies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"security_studies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_security_studies\"" - } - } - }, - { - "evaluation_name": "Sociology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Sociology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.896, - "details": { - "description": "min=0.896, mean=0.896, max=0.896, sum=1.791 (2)", - "tab": "Accuracy", - "Sociology - Observed inference time (s)": "{\"description\": \"min=0.121, mean=0.121, max=0.121, sum=0.241 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.12068656716417903\"}", - "Sociology - # eval": "{\"description\": \"min=201, mean=201, max=201, sum=402 (2)\", \"tab\": \"General information\", \"score\": \"201.0\"}", - "Sociology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Sociology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Sociology - # prompt tokens": "{\"description\": \"min=474.512, mean=474.512, max=474.512, sum=949.025 (2)\", \"tab\": \"General information\", \"score\": \"474.5124378109453\"}", - "Sociology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"sociology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_sociology\"" - } - } - }, - { - "evaluation_name": "Virology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Virology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.542, - "details": { - "description": "min=0.542, mean=0.542, max=0.542, sum=1.084 (2)", - "tab": "Accuracy", - "Virology - Observed inference time (s)": "{\"description\": \"min=0.114, mean=0.114, max=0.114, sum=0.227 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.113578313253012\"}", - "Virology - # eval": "{\"description\": \"min=166, mean=166, max=166, sum=332 (2)\", \"tab\": \"General information\", \"score\": \"166.0\"}", - "Virology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Virology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Virology - # prompt tokens": "{\"description\": \"min=380.753, mean=380.753, max=380.753, sum=761.506 (2)\", \"tab\": \"General information\", \"score\": \"380.7530120481928\"}", - "Virology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"virology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_virology\"" - } - } - }, - { - "evaluation_name": "World Religions", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on World Religions", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.871, - "details": { - "description": "min=0.871, mean=0.871, max=0.871, sum=1.743 (2)", - "tab": "Accuracy", - "World Religions - Observed inference time (s)": "{\"description\": \"min=0.114, mean=0.114, max=0.114, sum=0.229 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.11440935672514624\"}", - "World Religions - # eval": "{\"description\": \"min=171, mean=171, max=171, sum=342 (2)\", \"tab\": \"General information\", \"score\": \"171.0\"}", - "World Religions - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "World Religions - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "World Religions - # prompt tokens": "{\"description\": \"min=305.386, mean=305.386, max=305.386, sum=610.772 (2)\", \"tab\": \"General information\", \"score\": \"305.3859649122807\"}", - "World Religions - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"world_religions\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_world_religions\"" - } - } - }, - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.987, - "details": { - "description": "", - "tab": "Efficiency" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/amazon_nova-micro-v1_0.json b/data/models/amazon_nova-micro-v1_0.json deleted file mode 100644 index e8d218d05a78979bc7d0f97c9351f1e179444d32..0000000000000000000000000000000000000000 --- a/data/models/amazon_nova-micro-v1_0.json +++ /dev/null @@ -1,2128 +0,0 @@ -{ - "model_info": { - "name": "Amazon Nova Micro", - "id": "amazon/nova-micro-v1:0", - "developer": "amazon", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_capabilities/amazon_nova-micro-v1:0/1774096304.5056698", - "retrieved_timestamp": "1774096304.5056698", - "source_metadata": { - "source_name": "helm_capabilities", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_capabilities", - "evaluation_results": [ - { - "evaluation_name": "Mean score", - "source_data": { - "dataset_name": "helm_capabilities", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "The mean of the scores from all columns.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.522, - "details": { - "description": "", - "tab": "Accuracy", - "Mean score - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"2.157983343244118\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU-Pro", - "source_data": { - "dataset_name": "MMLU-Pro", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on MMLU-Pro", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.511, - "details": { - "description": "min=0.511, mean=0.511, max=0.511, sum=0.511 (1)", - "tab": "Accuracy", - "MMLU-Pro - Observed inference time (s)": "{\"description\": \"min=1.316, mean=1.316, max=1.316, sum=1.316 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.3163370000000014\"}", - "MMLU-Pro - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "MMLU-Pro - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - # prompt tokens": "{\"description\": \"min=235.232, mean=235.232, max=235.232, sum=235.232 (1)\", \"tab\": \"General information\", \"score\": \"235.232\"}", - "MMLU-Pro - # output tokens": "{\"description\": \"min=367.695, mean=367.695, max=367.695, sum=367.695 (1)\", \"tab\": \"General information\", \"score\": \"367.695\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"all\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.383, - "details": { - "description": "min=0.383, mean=0.383, max=0.383, sum=0.383 (1)", - "tab": "Accuracy", - "GPQA - Observed inference time (s)": "{\"description\": \"min=2.134, mean=2.134, max=2.134, sum=2.134 (1)\", \"tab\": \"Efficiency\", \"score\": \"2.1342376681614366\"}", - "GPQA - # eval": "{\"description\": \"min=446, mean=446, max=446, sum=446 (1)\", \"tab\": \"General information\", \"score\": \"446.0\"}", - "GPQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - # prompt tokens": "{\"description\": \"min=264.121, mean=264.121, max=264.121, sum=264.121 (1)\", \"tab\": \"General information\", \"score\": \"264.1210762331838\"}", - "GPQA - # output tokens": "{\"description\": \"min=587.372, mean=587.372, max=587.372, sum=587.372 (1)\", \"tab\": \"General information\", \"score\": \"587.3721973094171\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"gpqa_main\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "IFEval Strict Acc on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.76, - "details": { - "description": "min=0.76, mean=0.76, max=0.76, sum=0.76 (1)", - "tab": "Accuracy", - "IFEval - Observed inference time (s)": "{\"description\": \"min=1.605, mean=1.605, max=1.605, sum=1.605 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.6054140480591508\"}", - "IFEval - # eval": "{\"description\": \"min=541, mean=541, max=541, sum=541 (1)\", \"tab\": \"General information\", \"score\": \"541.0\"}", - "IFEval - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - # prompt tokens": "{\"description\": \"min=47.58, mean=47.58, max=47.58, sum=47.58 (1)\", \"tab\": \"General information\", \"score\": \"47.58040665434381\"}", - "IFEval - # output tokens": "{\"description\": \"min=385.473, mean=385.473, max=385.473, sum=385.473 (1)\", \"tab\": \"General information\", \"score\": \"385.4731977818854\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WildBench", - "source_data": { - "dataset_name": "WildBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "WB Score on WildBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.743, - "details": { - "description": "min=0.743, mean=0.743, max=0.743, sum=0.743 (1)", - "tab": "Accuracy", - "WildBench - Observed inference time (s)": "{\"description\": \"min=3.624, mean=3.624, max=3.624, sum=3.624 (1)\", \"tab\": \"Efficiency\", \"score\": \"3.6235889999999995\"}", - "WildBench - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "WildBench - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # prompt tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # output tokens": "{\"description\": \"min=925.586, mean=925.586, max=925.586, sum=925.586 (1)\", \"tab\": \"General information\", \"score\": \"925.586\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"v2\"" - } - } - }, - { - "evaluation_name": "Omni-MATH", - "source_data": { - "dataset_name": "Omni-MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Acc on Omni-MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.214, - "details": { - "description": "min=0.214, mean=0.214, max=0.214, sum=0.214 (1)", - "tab": "Accuracy", - "Omni-MATH - Observed inference time (s)": "{\"description\": \"min=2.11, mean=2.11, max=2.11, sum=2.11 (1)\", \"tab\": \"Efficiency\", \"score\": \"2.1103390000000006\"}", - "Omni-MATH - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "Omni-MATH - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - # prompt tokens": "{\"description\": \"min=117.921, mean=117.921, max=117.921, sum=117.921 (1)\", \"tab\": \"General information\", \"score\": \"117.921\"}", - "Omni-MATH - # output tokens": "{\"description\": \"min=743.286, mean=743.286, max=743.286, sum=743.286 (1)\", \"tab\": \"General information\", \"score\": \"743.286\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_lite/amazon_nova-micro-v1:0/1774096306.427425", - "retrieved_timestamp": "1774096306.427425", - "source_metadata": { - "source_name": "helm_lite", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_lite", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_lite", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.524, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.998876404494382\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.744, - "details": { - "description": "min=0.744, mean=0.744, max=0.744, sum=0.744 (1)", - "tab": "Accuracy", - "NarrativeQA - Observed inference time (s)": "{\"description\": \"min=0.196, mean=0.196, max=0.196, sum=0.196 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.19638591549295767\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=3550.577, mean=3550.577, max=3550.577, sum=3550.577 (1)\", \"tab\": \"General information\", \"score\": \"3550.5774647887324\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=3.961, mean=3.961, max=3.961, sum=3.961 (1)\", \"tab\": \"General information\", \"score\": \"3.96056338028169\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (closed-book)", - "source_data": { - "dataset_name": "NaturalQuestions (closed-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (closed-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.285, - "details": { - "description": "min=0.285, mean=0.285, max=0.285, sum=0.285 (1)", - "tab": "Accuracy", - "NaturalQuestions (open-book) - Observed inference time (s)": "{\"description\": \"min=0.19, mean=0.19, max=0.19, sum=0.19 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.1897639999999999\"}", - "NaturalQuestions (closed-book) - Observed inference time (s)": "{\"description\": \"min=0.133, mean=0.133, max=0.133, sum=0.133 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.1334880000000001\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1773.944, mean=1773.944, max=1773.944, sum=1773.944 (1)\", \"tab\": \"General information\", \"score\": \"1773.944\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=5.911, mean=5.911, max=5.911, sum=5.911 (1)\", \"tab\": \"General information\", \"score\": \"5.911\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=153.254, mean=153.254, max=153.254, sum=153.254 (1)\", \"tab\": \"General information\", \"score\": \"153.254\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=3.515, mean=3.515, max=3.515, sum=3.515 (1)\", \"tab\": \"General information\", \"score\": \"3.515\"}" - } - }, - "generation_config": { - "additional_details": { - "mode": "\"closedbook\"" - } - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.888, - "details": { - "description": "min=0.888, mean=0.888, max=0.888, sum=0.888 (1)", - "tab": "Accuracy", - "OpenbookQA - Observed inference time (s)": "{\"description\": \"min=0.104, mean=0.104, max=0.104, sum=0.104 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.10389599999999993\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=282.21, mean=282.21, max=282.21, sum=282.21 (1)\", \"tab\": \"General information\", \"score\": \"282.21\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "dataset": "\"openbookqa\"", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.64, - "details": { - "description": "min=0.42, mean=0.64, max=0.9, sum=3.2 (5)", - "tab": "Accuracy", - "MMLU - Observed inference time (s)": "{\"description\": \"min=0.113, mean=0.116, max=0.118, sum=0.579 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.11572105263157897\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=399.38, mean=500.274, max=652.07, sum=2501.37 (5)\", \"tab\": \"General information\", \"score\": \"500.2740350877192\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"college_chemistry\", \"computer_security\", \"econometrics\", \"us_foreign_policy\"]", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MATH", - "source_data": { - "dataset_name": "MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Equivalent (CoT) on MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.76, - "details": { - "description": "min=0.558, mean=0.76, max=0.895, sum=5.32 (7)", - "tab": "Accuracy", - "MATH - Observed inference time (s)": "{\"description\": \"min=0.577, mean=0.79, max=1.132, sum=5.529 (7)\", \"tab\": \"Efficiency\", \"score\": \"0.7898264142267815\"}", - "MATH - # eval": "{\"description\": \"min=30, mean=62.429, max=135, sum=437 (7)\", \"tab\": \"General information\", \"score\": \"62.42857142857143\"}", - "MATH - # train": "{\"description\": \"min=8, mean=8, max=8, sum=56 (7)\", \"tab\": \"General information\", \"score\": \"8.0\"}", - "MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MATH - # prompt tokens": "{\"description\": \"min=925.556, mean=1394.735, max=2468.942, sum=9763.147 (7)\", \"tab\": \"General information\", \"score\": \"1394.7353092779651\"}", - "MATH - # output tokens": "{\"description\": \"min=75.368, mean=103.346, max=152.2, sum=723.421 (7)\", \"tab\": \"General information\", \"score\": \"103.34588937061396\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"algebra\", \"counting_and_probability\", \"geometry\", \"intermediate_algebra\", \"number_theory\", \"prealgebra\", \"precalculus\"]", - "level": "\"1\"", - "use_official_examples": "\"False\"", - "use_chain_of_thought": "\"True\"" - } - } - }, - { - "evaluation_name": "GSM8K", - "source_data": { - "dataset_name": "GSM8K", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on GSM8K", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.794, - "details": { - "description": "min=0.794, mean=0.794, max=0.794, sum=0.794 (1)", - "tab": "Accuracy", - "GSM8K - Observed inference time (s)": "{\"description\": \"min=0.895, mean=0.895, max=0.895, sum=0.895 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.8952520000000004\"}", - "GSM8K - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "GSM8K - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "GSM8K - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GSM8K - # prompt tokens": "{\"description\": \"min=957.869, mean=957.869, max=957.869, sum=957.869 (1)\", \"tab\": \"General information\", \"score\": \"957.869\"}", - "GSM8K - # output tokens": "{\"description\": \"min=103.892, mean=103.892, max=103.892, sum=103.892 (1)\", \"tab\": \"General information\", \"score\": \"103.892\"}" - } - }, - "generation_config": { - "additional_details": { - "stop": "\"none\"" - } - } - }, - { - "evaluation_name": "LegalBench", - "source_data": { - "dataset_name": "LegalBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on LegalBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.615, - "details": { - "description": "min=0.368, mean=0.615, max=0.874, sum=3.074 (5)", - "tab": "Accuracy", - "LegalBench - Observed inference time (s)": "{\"description\": \"min=0.108, mean=0.143, max=0.254, sum=0.713 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.14263605160429277\"}", - "LegalBench - # eval": "{\"description\": \"min=95, mean=409.4, max=1000, sum=2047 (5)\", \"tab\": \"General information\", \"score\": \"409.4\"}", - "LegalBench - # train": "{\"description\": \"min=4, mean=4.8, max=5, sum=24 (5)\", \"tab\": \"General information\", \"score\": \"4.8\"}", - "LegalBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "LegalBench - # prompt tokens": "{\"description\": \"min=241.632, mean=1581.083, max=6449.798, sum=7905.414 (5)\", \"tab\": \"General information\", \"score\": \"1581.0827222540588\"}", - "LegalBench - # output tokens": "{\"description\": \"min=1, mean=1.665, max=2.926, sum=8.323 (5)\", \"tab\": \"General information\", \"score\": \"1.6646275687271896\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"abercrombie\", \"corporate_lobbying\", \"function_of_decision_section\", \"international_citizenship_questions\", \"proa\"]", - "stop": "\"none\"" - } - } - }, - { - "evaluation_name": "MedQA", - "source_data": { - "dataset_name": "MedQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MedQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.608, - "details": { - "description": "min=0.608, mean=0.608, max=0.608, sum=0.608 (1)", - "tab": "Accuracy", - "MedQA - Observed inference time (s)": "{\"description\": \"min=0.118, mean=0.118, max=0.118, sum=0.118 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.11825049701789252\"}", - "MedQA - # eval": "{\"description\": \"min=503, mean=503, max=503, sum=503 (1)\", \"tab\": \"General information\", \"score\": \"503.0\"}", - "MedQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MedQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MedQA - # prompt tokens": "{\"description\": \"min=1066.861, mean=1066.861, max=1066.861, sum=1066.861 (1)\", \"tab\": \"General information\", \"score\": \"1066.8608349900596\"}", - "MedQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WMT 2014", - "source_data": { - "dataset_name": "WMT 2014", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "BLEU-4 on WMT 2014", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.192, - "details": { - "description": "min=0.112, mean=0.192, max=0.241, sum=0.96 (5)", - "tab": "Accuracy", - "WMT 2014 - Observed inference time (s)": "{\"description\": \"min=0.239, mean=0.268, max=0.333, sum=1.34 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.26807757063388915\"}", - "WMT 2014 - # eval": "{\"description\": \"min=503, mean=568.8, max=832, sum=2844 (5)\", \"tab\": \"General information\", \"score\": \"568.8\"}", - "WMT 2014 - # train": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "WMT 2014 - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WMT 2014 - # prompt tokens": "{\"description\": \"min=163.93, mean=208.694, max=268.662, sum=1043.469 (5)\", \"tab\": \"General information\", \"score\": \"208.69386660804403\"}", - "WMT 2014 - # output tokens": "{\"description\": \"min=23.38, mean=25.875, max=28.916, sum=129.377 (5)\", \"tab\": \"General information\", \"score\": \"25.875419597797826\"}" - } - }, - "generation_config": { - "additional_details": { - "language_pair": "[\"cs-en\", \"de-en\", \"fr-en\", \"hi-en\", \"ru-en\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_mmlu/amazon_nova-micro-v1:0/1774096312.00548", - "retrieved_timestamp": "1774096312.00548", - "source_metadata": { - "source_name": "helm_mmlu", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_mmlu", - "evaluation_results": [ - { - "evaluation_name": "MMLU All Subjects", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU All Subjects", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.708, - "details": { - "description": "min=0.42, mean=0.708, max=0.922, sum=80.671 (114)", - "tab": "Accuracy", - "MMLU All Subjects - Observed inference time (s)": "{\"description\": \"min=0.102, mean=0.114, max=0.152, sum=13.049 (114)\", \"tab\": \"Efficiency\", \"score\": \"0.1144634124237814\"}", - "MMLU All Subjects - # eval": "{\"description\": \"min=100, mean=246.351, max=1534, sum=28084 (114)\", \"tab\": \"General information\", \"score\": \"246.35087719298247\"}", - "MMLU All Subjects - # train": "{\"description\": \"min=5, mean=5, max=5, sum=570 (114)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU All Subjects - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU All Subjects - # prompt tokens": "{\"description\": \"min=305.386, mean=655.489, max=2872.03, sum=74725.746 (114)\", \"tab\": \"General information\", \"score\": \"655.4890026560713\"}", - "MMLU All Subjects - # output tokens": "{\"description\": \"min=0.999, mean=1.0, max=1, sum=113.997 (114)\", \"tab\": \"General information\", \"score\": \"0.9999775940489795\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_name": "Abstract Algebra", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Abstract Algebra", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.42, - "details": { - "description": "min=0.42, mean=0.42, max=0.42, sum=0.84 (2)", - "tab": "Accuracy", - "Abstract Algebra - Observed inference time (s)": "{\"description\": \"min=0.117, mean=0.117, max=0.117, sum=0.234 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.11696000000000005\"}", - "Abstract Algebra - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Abstract Algebra - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Abstract Algebra - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Abstract Algebra - # prompt tokens": "{\"description\": \"min=399.38, mean=399.38, max=399.38, sum=798.76 (2)\", \"tab\": \"General information\", \"score\": \"399.38\"}", - "Abstract Algebra - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"abstract_algebra\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_abstract_algebra\"" - } - } - }, - { - "evaluation_name": "Anatomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Anatomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.726, - "details": { - "description": "min=0.726, mean=0.726, max=0.726, sum=1.452 (2)", - "tab": "Accuracy", - "Anatomy - Observed inference time (s)": "{\"description\": \"min=0.107, mean=0.107, max=0.107, sum=0.214 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.10704444444444451\"}", - "Anatomy - # eval": "{\"description\": \"min=135, mean=135, max=135, sum=270 (2)\", \"tab\": \"General information\", \"score\": \"135.0\"}", - "Anatomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Anatomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Anatomy - # prompt tokens": "{\"description\": \"min=400.081, mean=400.081, max=400.081, sum=800.163 (2)\", \"tab\": \"General information\", \"score\": \"400.0814814814815\"}", - "Anatomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"anatomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_anatomy\"" - } - } - }, - { - "evaluation_name": "College Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on College Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5, - "details": { - "description": "min=0.5, mean=0.5, max=0.5, sum=1 (2)", - "tab": "Accuracy", - "College Chemistry - Observed inference time (s)": "{\"description\": \"min=0.118, mean=0.118, max=0.118, sum=0.235 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.11762000000000004\"}", - "College Biology - Observed inference time (s)": "{\"description\": \"min=0.118, mean=0.118, max=0.118, sum=0.237 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.11843055555555557\"}", - "College Computer Science - Observed inference time (s)": "{\"description\": \"min=0.125, mean=0.125, max=0.125, sum=0.25 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.12490000000000004\"}", - "College Mathematics - Observed inference time (s)": "{\"description\": \"min=0.122, mean=0.122, max=0.122, sum=0.244 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.12207000000000001\"}", - "College Medicine - Observed inference time (s)": "{\"description\": \"min=0.116, mean=0.116, max=0.116, sum=0.233 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.11635838150289027\"}", - "College Physics - Observed inference time (s)": "{\"description\": \"min=0.115, mean=0.115, max=0.115, sum=0.229 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.11473529411764712\"}", - "College Chemistry - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Chemistry - # prompt tokens": "{\"description\": \"min=573.4, mean=573.4, max=573.4, sum=1146.8 (2)\", \"tab\": \"General information\", \"score\": \"573.4\"}", - "College Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Biology - # eval": "{\"description\": \"min=144, mean=144, max=144, sum=288 (2)\", \"tab\": \"General information\", \"score\": \"144.0\"}", - "College Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # prompt tokens": "{\"description\": \"min=510.278, mean=510.278, max=510.278, sum=1020.556 (2)\", \"tab\": \"General information\", \"score\": \"510.27777777777777\"}", - "College Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # prompt tokens": "{\"description\": \"min=880.15, mean=880.15, max=880.15, sum=1760.3 (2)\", \"tab\": \"General information\", \"score\": \"880.15\"}", - "College Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Mathematics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # prompt tokens": "{\"description\": \"min=639.53, mean=639.53, max=639.53, sum=1279.06 (2)\", \"tab\": \"General information\", \"score\": \"639.53\"}", - "College Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Medicine - # eval": "{\"description\": \"min=173, mean=173, max=173, sum=346 (2)\", \"tab\": \"General information\", \"score\": \"173.0\"}", - "College Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # prompt tokens": "{\"description\": \"min=558.301, mean=558.301, max=558.301, sum=1116.601 (2)\", \"tab\": \"General information\", \"score\": \"558.3005780346821\"}", - "College Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Physics - # eval": "{\"description\": \"min=102, mean=102, max=102, sum=204 (2)\", \"tab\": \"General information\", \"score\": \"102.0\"}", - "College Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # prompt tokens": "{\"description\": \"min=517.324, mean=517.324, max=517.324, sum=1034.647 (2)\", \"tab\": \"General information\", \"score\": \"517.3235294117648\"}", - "College Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"college_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_college_physics\"" - } - } - }, - { - "evaluation_name": "Computer Security", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Computer Security", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.77, - "details": { - "description": "min=0.77, mean=0.77, max=0.77, sum=1.54 (2)", - "tab": "Accuracy", - "Computer Security - Observed inference time (s)": "{\"description\": \"min=0.115, mean=0.115, max=0.115, sum=0.231 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.11527000000000003\"}", - "Computer Security - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Computer Security - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Computer Security - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Computer Security - # prompt tokens": "{\"description\": \"min=415.4, mean=415.4, max=415.4, sum=830.8 (2)\", \"tab\": \"General information\", \"score\": \"415.4\"}", - "Computer Security - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"computer_security\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_computer_security\"" - } - } - }, - { - "evaluation_name": "Econometrics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Econometrics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.57, - "details": { - "description": "min=0.57, mean=0.57, max=0.57, sum=1.14 (2)", - "tab": "Accuracy", - "Econometrics - Observed inference time (s)": "{\"description\": \"min=0.116, mean=0.116, max=0.116, sum=0.231 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.11560526315789472\"}", - "Econometrics - # eval": "{\"description\": \"min=114, mean=114, max=114, sum=228 (2)\", \"tab\": \"General information\", \"score\": \"114.0\"}", - "Econometrics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Econometrics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Econometrics - # prompt tokens": "{\"description\": \"min=652.07, mean=652.07, max=652.07, sum=1304.14 (2)\", \"tab\": \"General information\", \"score\": \"652.0701754385965\"}", - "Econometrics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"econometrics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_econometrics\"" - } - } - }, - { - "evaluation_name": "Global Facts", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Global Facts", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.44, - "details": { - "description": "min=0.44, mean=0.44, max=0.44, sum=0.88 (2)", - "tab": "Accuracy", - "Global Facts - Observed inference time (s)": "{\"description\": \"min=0.115, mean=0.115, max=0.115, sum=0.231 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.11540999999999998\"}", - "Global Facts - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Global Facts - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Global Facts - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Global Facts - # prompt tokens": "{\"description\": \"min=426.42, mean=426.42, max=426.42, sum=852.84 (2)\", \"tab\": \"General information\", \"score\": \"426.42\"}", - "Global Facts - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"global_facts\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_global_facts\"" - } - } - }, - { - "evaluation_name": "Jurisprudence", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Jurisprudence", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.815, - "details": { - "description": "min=0.815, mean=0.815, max=0.815, sum=1.63 (2)", - "tab": "Accuracy", - "Jurisprudence - Observed inference time (s)": "{\"description\": \"min=0.111, mean=0.111, max=0.111, sum=0.223 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.11141666666666669\"}", - "Jurisprudence - # eval": "{\"description\": \"min=108, mean=108, max=108, sum=216 (2)\", \"tab\": \"General information\", \"score\": \"108.0\"}", - "Jurisprudence - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Jurisprudence - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Jurisprudence - # prompt tokens": "{\"description\": \"min=446.722, mean=446.722, max=446.722, sum=893.444 (2)\", \"tab\": \"General information\", \"score\": \"446.72222222222223\"}", - "Jurisprudence - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"jurisprudence\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_jurisprudence\"" - } - } - }, - { - "evaluation_name": "Philosophy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Philosophy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.733, - "details": { - "description": "min=0.733, mean=0.733, max=0.733, sum=1.466 (2)", - "tab": "Accuracy", - "Philosophy - Observed inference time (s)": "{\"description\": \"min=0.107, mean=0.107, max=0.107, sum=0.214 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.10707717041800643\"}", - "Philosophy - # eval": "{\"description\": \"min=311, mean=311, max=311, sum=622 (2)\", \"tab\": \"General information\", \"score\": \"311.0\"}", - "Philosophy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Philosophy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Philosophy - # prompt tokens": "{\"description\": \"min=381.704, mean=381.704, max=381.704, sum=763.408 (2)\", \"tab\": \"General information\", \"score\": \"381.7041800643087\"}", - "Philosophy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"philosophy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_philosophy\"" - } - } - }, - { - "evaluation_name": "Professional Psychology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Professional Psychology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.739, - "details": { - "description": "min=0.739, mean=0.739, max=0.739, sum=1.477 (2)", - "tab": "Accuracy", - "Professional Medicine - Observed inference time (s)": "{\"description\": \"min=0.127, mean=0.127, max=0.127, sum=0.255 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.12727573529411765\"}", - "Professional Accounting - Observed inference time (s)": "{\"description\": \"min=0.117, mean=0.117, max=0.117, sum=0.234 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.11683687943262412\"}", - "Professional Law - Observed inference time (s)": "{\"description\": \"min=0.128, mean=0.128, max=0.128, sum=0.256 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.1279393741851367\"}", - "Professional Psychology - Observed inference time (s)": "{\"description\": \"min=0.111, mean=0.111, max=0.111, sum=0.221 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.11058333333333302\"}", - "Professional Medicine - # eval": "{\"description\": \"min=272, mean=272, max=272, sum=544 (2)\", \"tab\": \"General information\", \"score\": \"272.0\"}", - "Professional Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Medicine - # prompt tokens": "{\"description\": \"min=1146.287, mean=1146.287, max=1146.287, sum=2292.574 (2)\", \"tab\": \"General information\", \"score\": \"1146.2867647058824\"}", - "Professional Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Accounting - # eval": "{\"description\": \"min=282, mean=282, max=282, sum=564 (2)\", \"tab\": \"General information\", \"score\": \"282.0\"}", - "Professional Accounting - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Accounting - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # prompt tokens": "{\"description\": \"min=688.72, mean=688.72, max=688.72, sum=1377.44 (2)\", \"tab\": \"General information\", \"score\": \"688.7198581560284\"}", - "Professional Accounting - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Law - # eval": "{\"description\": \"min=1534, mean=1534, max=1534, sum=3068 (2)\", \"tab\": \"General information\", \"score\": \"1534.0\"}", - "Professional Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # prompt tokens": "{\"description\": \"min=1686.73, mean=1686.73, max=1686.73, sum=3373.46 (2)\", \"tab\": \"General information\", \"score\": \"1686.7301173402868\"}", - "Professional Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Psychology - # eval": "{\"description\": \"min=612, mean=612, max=612, sum=1224 (2)\", \"tab\": \"General information\", \"score\": \"612.0\"}", - "Professional Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # prompt tokens": "{\"description\": \"min=625.574, mean=625.574, max=625.574, sum=1251.147 (2)\", \"tab\": \"General information\", \"score\": \"625.5735294117648\"}", - "Professional Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"professional_psychology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_professional_psychology\"" - } - } - }, - { - "evaluation_name": "Us Foreign Policy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Us Foreign Policy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9, - "details": { - "description": "min=0.9, mean=0.9, max=0.9, sum=1.8 (2)", - "tab": "Accuracy", - "Us Foreign Policy - Observed inference time (s)": "{\"description\": \"min=0.113, mean=0.113, max=0.113, sum=0.226 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.11315000000000004\"}", - "Us Foreign Policy - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Us Foreign Policy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Us Foreign Policy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Us Foreign Policy - # prompt tokens": "{\"description\": \"min=461.12, mean=461.12, max=461.12, sum=922.24 (2)\", \"tab\": \"General information\", \"score\": \"461.12\"}", - "Us Foreign Policy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"us_foreign_policy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_us_foreign_policy\"" - } - } - }, - { - "evaluation_name": "Astronomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Astronomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.822, - "details": { - "description": "min=0.822, mean=0.822, max=0.822, sum=1.645 (2)", - "tab": "Accuracy", - "Astronomy - Observed inference time (s)": "{\"description\": \"min=0.116, mean=0.116, max=0.116, sum=0.232 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.11597368421052637\"}", - "Astronomy - # eval": "{\"description\": \"min=152, mean=152, max=152, sum=304 (2)\", \"tab\": \"General information\", \"score\": \"152.0\"}", - "Astronomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Astronomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Astronomy - # prompt tokens": "{\"description\": \"min=628.112, mean=628.112, max=628.112, sum=1256.224 (2)\", \"tab\": \"General information\", \"score\": \"628.1118421052631\"}", - "Astronomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"astronomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_astronomy\"" - } - } - }, - { - "evaluation_name": "Business Ethics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Business Ethics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.71, - "details": { - "description": "min=0.71, mean=0.71, max=0.71, sum=1.42 (2)", - "tab": "Accuracy", - "Business Ethics - Observed inference time (s)": "{\"description\": \"min=0.118, mean=0.118, max=0.118, sum=0.237 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.11840000000000003\"}", - "Business Ethics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Business Ethics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Business Ethics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Business Ethics - # prompt tokens": "{\"description\": \"min=617.46, mean=617.46, max=617.46, sum=1234.92 (2)\", \"tab\": \"General information\", \"score\": \"617.46\"}", - "Business Ethics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"business_ethics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_business_ethics\"" - } - } - }, - { - "evaluation_name": "Clinical Knowledge", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Clinical Knowledge", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.751, - "details": { - "description": "min=0.751, mean=0.751, max=0.751, sum=1.502 (2)", - "tab": "Accuracy", - "Clinical Knowledge - Observed inference time (s)": "{\"description\": \"min=0.115, mean=0.115, max=0.115, sum=0.23 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.11494716981132078\"}", - "Clinical Knowledge - # eval": "{\"description\": \"min=265, mean=265, max=265, sum=530 (2)\", \"tab\": \"General information\", \"score\": \"265.0\"}", - "Clinical Knowledge - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Clinical Knowledge - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Clinical Knowledge - # prompt tokens": "{\"description\": \"min=451.925, mean=451.925, max=451.925, sum=903.849 (2)\", \"tab\": \"General information\", \"score\": \"451.92452830188677\"}", - "Clinical Knowledge - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"clinical_knowledge\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_clinical_knowledge\"" - } - } - }, - { - "evaluation_name": "Conceptual Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Conceptual Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.706, - "details": { - "description": "min=0.706, mean=0.706, max=0.706, sum=1.413 (2)", - "tab": "Accuracy", - "Conceptual Physics - Observed inference time (s)": "{\"description\": \"min=0.105, mean=0.105, max=0.105, sum=0.21 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.10520000000000002\"}", - "Conceptual Physics - # eval": "{\"description\": \"min=235, mean=235, max=235, sum=470 (2)\", \"tab\": \"General information\", \"score\": \"235.0\"}", - "Conceptual Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Conceptual Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Conceptual Physics - # prompt tokens": "{\"description\": \"min=341.723, mean=341.723, max=341.723, sum=683.447 (2)\", \"tab\": \"General information\", \"score\": \"341.72340425531917\"}", - "Conceptual Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"conceptual_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_conceptual_physics\"" - } - } - }, - { - "evaluation_name": "Electrical Engineering", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Electrical Engineering", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.683, - "details": { - "description": "min=0.683, mean=0.683, max=0.683, sum=1.366 (2)", - "tab": "Accuracy", - "Electrical Engineering - Observed inference time (s)": "{\"description\": \"min=0.109, mean=0.109, max=0.109, sum=0.218 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.10906896551724135\"}", - "Electrical Engineering - # eval": "{\"description\": \"min=145, mean=145, max=145, sum=290 (2)\", \"tab\": \"General information\", \"score\": \"145.0\"}", - "Electrical Engineering - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Electrical Engineering - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Electrical Engineering - # prompt tokens": "{\"description\": \"min=458.345, mean=458.345, max=458.345, sum=916.69 (2)\", \"tab\": \"General information\", \"score\": \"458.3448275862069\"}", - "Electrical Engineering - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"electrical_engineering\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_electrical_engineering\"" - } - } - }, - { - "evaluation_name": "Elementary Mathematics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Elementary Mathematics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.55, - "details": { - "description": "min=0.55, mean=0.55, max=0.55, sum=1.101 (2)", - "tab": "Accuracy", - "Elementary Mathematics - Observed inference time (s)": "{\"description\": \"min=0.116, mean=0.116, max=0.116, sum=0.232 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.11621164021164002\"}", - "Elementary Mathematics - # eval": "{\"description\": \"min=378, mean=378, max=378, sum=756 (2)\", \"tab\": \"General information\", \"score\": \"378.0\"}", - "Elementary Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Elementary Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Elementary Mathematics - # prompt tokens": "{\"description\": \"min=534.09, mean=534.09, max=534.09, sum=1068.18 (2)\", \"tab\": \"General information\", \"score\": \"534.0899470899471\"}", - "Elementary Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"elementary_mathematics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_elementary_mathematics\"" - } - } - }, - { - "evaluation_name": "Formal Logic", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Formal Logic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.508, - "details": { - "description": "min=0.508, mean=0.508, max=0.508, sum=1.016 (2)", - "tab": "Accuracy", - "Formal Logic - Observed inference time (s)": "{\"description\": \"min=0.113, mean=0.113, max=0.113, sum=0.226 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.112968253968254\"}", - "Formal Logic - # eval": "{\"description\": \"min=126, mean=126, max=126, sum=252 (2)\", \"tab\": \"General information\", \"score\": \"126.0\"}", - "Formal Logic - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Formal Logic - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Formal Logic - # prompt tokens": "{\"description\": \"min=669, mean=669, max=669, sum=1338 (2)\", \"tab\": \"General information\", \"score\": \"669.0\"}", - "Formal Logic - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"formal_logic\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_formal_logic\"" - } - } - }, - { - "evaluation_name": "High School World History", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on High School World History", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.84, - "details": { - "description": "min=0.84, mean=0.84, max=0.84, sum=1.679 (2)", - "tab": "Accuracy", - "High School Biology - Observed inference time (s)": "{\"description\": \"min=0.112, mean=0.112, max=0.112, sum=0.224 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.11209354838709669\"}", - "High School Chemistry - Observed inference time (s)": "{\"description\": \"min=0.113, mean=0.113, max=0.113, sum=0.226 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.11317733990147788\"}", - "High School Computer Science - Observed inference time (s)": "{\"description\": \"min=0.12, mean=0.12, max=0.12, sum=0.24 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.11999000000000004\"}", - "High School European History - Observed inference time (s)": "{\"description\": \"min=0.152, mean=0.152, max=0.152, sum=0.303 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.1516909090909091\"}", - "High School Geography - Observed inference time (s)": "{\"description\": \"min=0.11, mean=0.11, max=0.11, sum=0.22 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.11011616161616171\"}", - "High School Government And Politics - Observed inference time (s)": "{\"description\": \"min=0.108, mean=0.108, max=0.108, sum=0.216 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.10789637305699486\"}", - "High School Macroeconomics - Observed inference time (s)": "{\"description\": \"min=0.11, mean=0.11, max=0.11, sum=0.221 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.11032307692307693\"}", - "High School Mathematics - Observed inference time (s)": "{\"description\": \"min=0.113, mean=0.113, max=0.113, sum=0.226 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.11290000000000003\"}", - "High School Microeconomics - Observed inference time (s)": "{\"description\": \"min=0.11, mean=0.11, max=0.11, sum=0.219 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.10956302521008413\"}", - "High School Physics - Observed inference time (s)": "{\"description\": \"min=0.116, mean=0.116, max=0.116, sum=0.231 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.11561589403973516\"}", - "High School Psychology - Observed inference time (s)": "{\"description\": \"min=0.11, mean=0.11, max=0.11, sum=0.22 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.11005137614678874\"}", - "High School Statistics - Observed inference time (s)": "{\"description\": \"min=0.116, mean=0.116, max=0.116, sum=0.233 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.11631018518518522\"}", - "High School US History - Observed inference time (s)": "{\"description\": \"min=0.139, mean=0.139, max=0.139, sum=0.279 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.13944117647058826\"}", - "High School World History - Observed inference time (s)": "{\"description\": \"min=0.134, mean=0.134, max=0.134, sum=0.268 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.13399578059071726\"}", - "High School Biology - # eval": "{\"description\": \"min=310, mean=310, max=310, sum=620 (2)\", \"tab\": \"General information\", \"score\": \"310.0\"}", - "High School Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Biology - # prompt tokens": "{\"description\": \"min=568.748, mean=568.748, max=568.748, sum=1137.497 (2)\", \"tab\": \"General information\", \"score\": \"568.7483870967742\"}", - "High School Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Chemistry - # eval": "{\"description\": \"min=203, mean=203, max=203, sum=406 (2)\", \"tab\": \"General information\", \"score\": \"203.0\"}", - "High School Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # prompt tokens": "{\"description\": \"min=523.65, mean=523.65, max=523.65, sum=1047.3 (2)\", \"tab\": \"General information\", \"score\": \"523.6502463054187\"}", - "High School Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "High School Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # prompt tokens": "{\"description\": \"min=932.15, mean=932.15, max=932.15, sum=1864.3 (2)\", \"tab\": \"General information\", \"score\": \"932.15\"}", - "High School Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School European History - # eval": "{\"description\": \"min=165, mean=165, max=165, sum=330 (2)\", \"tab\": \"General information\", \"score\": \"165.0\"}", - "High School European History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School European History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # prompt tokens": "{\"description\": \"min=2872.03, mean=2872.03, max=2872.03, sum=5744.061 (2)\", \"tab\": \"General information\", \"score\": \"2872.030303030303\"}", - "High School European History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Geography - # eval": "{\"description\": \"min=198, mean=198, max=198, sum=396 (2)\", \"tab\": \"General information\", \"score\": \"198.0\"}", - "High School Geography - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Geography - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # prompt tokens": "{\"description\": \"min=425.646, mean=425.646, max=425.646, sum=851.293 (2)\", \"tab\": \"General information\", \"score\": \"425.64646464646466\"}", - "High School Geography - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Government And Politics - # eval": "{\"description\": \"min=193, mean=193, max=193, sum=386 (2)\", \"tab\": \"General information\", \"score\": \"193.0\"}", - "High School Government And Politics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Government And Politics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # prompt tokens": "{\"description\": \"min=506.073, mean=506.073, max=506.073, sum=1012.145 (2)\", \"tab\": \"General information\", \"score\": \"506.07253886010363\"}", - "High School Government And Politics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Macroeconomics - # eval": "{\"description\": \"min=390, mean=390, max=390, sum=780 (2)\", \"tab\": \"General information\", \"score\": \"390.0\"}", - "High School Macroeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Macroeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # prompt tokens": "{\"description\": \"min=419.987, mean=419.987, max=419.987, sum=839.974 (2)\", \"tab\": \"General information\", \"score\": \"419.9871794871795\"}", - "High School Macroeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Mathematics - # eval": "{\"description\": \"min=270, mean=270, max=270, sum=540 (2)\", \"tab\": \"General information\", \"score\": \"270.0\"}", - "High School Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # prompt tokens": "{\"description\": \"min=554.352, mean=554.352, max=554.352, sum=1108.704 (2)\", \"tab\": \"General information\", \"score\": \"554.3518518518518\"}", - "High School Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Microeconomics - # eval": "{\"description\": \"min=238, mean=238, max=238, sum=476 (2)\", \"tab\": \"General information\", \"score\": \"238.0\"}", - "High School Microeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Microeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # prompt tokens": "{\"description\": \"min=439.055, mean=439.055, max=439.055, sum=878.109 (2)\", \"tab\": \"General information\", \"score\": \"439.0546218487395\"}", - "High School Microeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Physics - # eval": "{\"description\": \"min=151, mean=151, max=151, sum=302 (2)\", \"tab\": \"General information\", \"score\": \"151.0\"}", - "High School Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # prompt tokens": "{\"description\": \"min=581.669, mean=581.669, max=581.669, sum=1163.338 (2)\", \"tab\": \"General information\", \"score\": \"581.6688741721854\"}", - "High School Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Psychology - # eval": "{\"description\": \"min=545, mean=545, max=545, sum=1090 (2)\", \"tab\": \"General information\", \"score\": \"545.0\"}", - "High School Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # prompt tokens": "{\"description\": \"min=544.842, mean=544.842, max=544.842, sum=1089.684 (2)\", \"tab\": \"General information\", \"score\": \"544.8422018348624\"}", - "High School Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Statistics - # eval": "{\"description\": \"min=216, mean=216, max=216, sum=432 (2)\", \"tab\": \"General information\", \"score\": \"216.0\"}", - "High School Statistics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Statistics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # prompt tokens": "{\"description\": \"min=833, mean=833, max=833, sum=1666 (2)\", \"tab\": \"General information\", \"score\": \"833.0\"}", - "High School Statistics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School US History - # eval": "{\"description\": \"min=204, mean=204, max=204, sum=408 (2)\", \"tab\": \"General information\", \"score\": \"204.0\"}", - "High School US History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School US History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # prompt tokens": "{\"description\": \"min=2270.25, mean=2270.25, max=2270.25, sum=4540.5 (2)\", \"tab\": \"General information\", \"score\": \"2270.25\"}", - "High School US History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School World History - # eval": "{\"description\": \"min=237, mean=237, max=237, sum=474 (2)\", \"tab\": \"General information\", \"score\": \"237.0\"}", - "High School World History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School World History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # prompt tokens": "{\"description\": \"min=1466.561, mean=1466.561, max=1466.561, sum=2933.122 (2)\", \"tab\": \"General information\", \"score\": \"1466.5611814345991\"}", - "High School World History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"high_school_world_history\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_high_school_world_history\"" - } - } - }, - { - "evaluation_name": "Human Sexuality", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Human Sexuality", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.824, - "details": { - "description": "min=0.824, mean=0.824, max=0.824, sum=1.649 (2)", - "tab": "Accuracy", - "Human Aging - Observed inference time (s)": "{\"description\": \"min=0.104, mean=0.104, max=0.104, sum=0.208 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.10423766816143511\"}", - "Human Sexuality - Observed inference time (s)": "{\"description\": \"min=0.112, mean=0.112, max=0.112, sum=0.224 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.11212213740458017\"}", - "Human Aging - # eval": "{\"description\": \"min=223, mean=223, max=223, sum=446 (2)\", \"tab\": \"General information\", \"score\": \"223.0\"}", - "Human Aging - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Aging - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Aging - # prompt tokens": "{\"description\": \"min=352.48, mean=352.48, max=352.48, sum=704.96 (2)\", \"tab\": \"General information\", \"score\": \"352.47982062780267\"}", - "Human Aging - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Human Sexuality - # eval": "{\"description\": \"min=131, mean=131, max=131, sum=262 (2)\", \"tab\": \"General information\", \"score\": \"131.0\"}", - "Human Sexuality - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Sexuality - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # prompt tokens": "{\"description\": \"min=385.626, mean=385.626, max=385.626, sum=771.252 (2)\", \"tab\": \"General information\", \"score\": \"385.62595419847327\"}", - "Human Sexuality - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"human_sexuality\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_human_sexuality\"" - } - } - }, - { - "evaluation_name": "International Law", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on International Law", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.843, - "details": { - "description": "min=0.843, mean=0.843, max=0.843, sum=1.686 (2)", - "tab": "Accuracy", - "International Law - Observed inference time (s)": "{\"description\": \"min=0.111, mean=0.111, max=0.111, sum=0.221 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.11063636363636367\"}", - "International Law - # eval": "{\"description\": \"min=121, mean=121, max=121, sum=242 (2)\", \"tab\": \"General information\", \"score\": \"121.0\"}", - "International Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "International Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "International Law - # prompt tokens": "{\"description\": \"min=667.843, mean=667.843, max=667.843, sum=1335.686 (2)\", \"tab\": \"General information\", \"score\": \"667.8429752066115\"}", - "International Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"international_law\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_international_law\"" - } - } - }, - { - "evaluation_name": "Logical Fallacies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Logical Fallacies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.798, - "details": { - "description": "min=0.798, mean=0.798, max=0.798, sum=1.595 (2)", - "tab": "Accuracy", - "Logical Fallacies - Observed inference time (s)": "{\"description\": \"min=0.111, mean=0.111, max=0.111, sum=0.221 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.11058895705521476\"}", - "Logical Fallacies - # eval": "{\"description\": \"min=163, mean=163, max=163, sum=326 (2)\", \"tab\": \"General information\", \"score\": \"163.0\"}", - "Logical Fallacies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Logical Fallacies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Logical Fallacies - # prompt tokens": "{\"description\": \"min=482.227, mean=482.227, max=482.227, sum=964.454 (2)\", \"tab\": \"General information\", \"score\": \"482.2269938650307\"}", - "Logical Fallacies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"logical_fallacies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_logical_fallacies\"" - } - } - }, - { - "evaluation_name": "Machine Learning", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Machine Learning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.562, - "details": { - "description": "min=0.562, mean=0.562, max=0.562, sum=1.125 (2)", - "tab": "Accuracy", - "Machine Learning - Observed inference time (s)": "{\"description\": \"min=0.115, mean=0.115, max=0.115, sum=0.231 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.11541964285714289\"}", - "Machine Learning - # eval": "{\"description\": \"min=112, mean=112, max=112, sum=224 (2)\", \"tab\": \"General information\", \"score\": \"112.0\"}", - "Machine Learning - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Machine Learning - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Machine Learning - # prompt tokens": "{\"description\": \"min=699.598, mean=699.598, max=699.598, sum=1399.196 (2)\", \"tab\": \"General information\", \"score\": \"699.5982142857143\"}", - "Machine Learning - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"machine_learning\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_machine_learning\"" - } - } - }, - { - "evaluation_name": "Management", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Management", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.816, - "details": { - "description": "min=0.816, mean=0.816, max=0.816, sum=1.631 (2)", - "tab": "Accuracy", - "Management - Observed inference time (s)": "{\"description\": \"min=0.102, mean=0.102, max=0.102, sum=0.205 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.10230097087378638\"}", - "Management - # eval": "{\"description\": \"min=103, mean=103, max=103, sum=206 (2)\", \"tab\": \"General information\", \"score\": \"103.0\"}", - "Management - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Management - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Management - # prompt tokens": "{\"description\": \"min=320.34, mean=320.34, max=320.34, sum=640.68 (2)\", \"tab\": \"General information\", \"score\": \"320.3398058252427\"}", - "Management - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"management\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_management\"" - } - } - }, - { - "evaluation_name": "Marketing", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Marketing", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.91, - "details": { - "description": "min=0.91, mean=0.91, max=0.91, sum=1.821 (2)", - "tab": "Accuracy", - "Marketing - Observed inference time (s)": "{\"description\": \"min=0.112, mean=0.112, max=0.112, sum=0.223 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.11152136752136761\"}", - "Marketing - # eval": "{\"description\": \"min=234, mean=234, max=234, sum=468 (2)\", \"tab\": \"General information\", \"score\": \"234.0\"}", - "Marketing - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Marketing - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Marketing - # prompt tokens": "{\"description\": \"min=466.697, mean=466.697, max=466.697, sum=933.393 (2)\", \"tab\": \"General information\", \"score\": \"466.6965811965812\"}", - "Marketing - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"marketing\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_marketing\"" - } - } - }, - { - "evaluation_name": "Medical Genetics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Medical Genetics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.82, - "details": { - "description": "min=0.82, mean=0.82, max=0.82, sum=1.64 (2)", - "tab": "Accuracy", - "Medical Genetics - Observed inference time (s)": "{\"description\": \"min=0.106, mean=0.106, max=0.106, sum=0.212 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.10620000000000003\"}", - "Medical Genetics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Medical Genetics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Medical Genetics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Medical Genetics - # prompt tokens": "{\"description\": \"min=380.71, mean=380.71, max=380.71, sum=761.42 (2)\", \"tab\": \"General information\", \"score\": \"380.71\"}", - "Medical Genetics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"medical_genetics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_medical_genetics\"" - } - } - }, - { - "evaluation_name": "Miscellaneous", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Miscellaneous", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.83, - "details": { - "description": "min=0.83, mean=0.83, max=0.83, sum=1.66 (2)", - "tab": "Accuracy", - "Miscellaneous - Observed inference time (s)": "{\"description\": \"min=0.105, mean=0.105, max=0.105, sum=0.21 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.10505236270753474\"}", - "Miscellaneous - # eval": "{\"description\": \"min=783, mean=783, max=783, sum=1566 (2)\", \"tab\": \"General information\", \"score\": \"783.0\"}", - "Miscellaneous - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Miscellaneous - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Miscellaneous - # prompt tokens": "{\"description\": \"min=342.847, mean=342.847, max=342.847, sum=685.693 (2)\", \"tab\": \"General information\", \"score\": \"342.84674329501917\"}", - "Miscellaneous - # output tokens": "{\"description\": \"min=0.999, mean=0.999, max=0.999, sum=1.997 (2)\", \"tab\": \"General information\", \"score\": \"0.9987228607918263\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"miscellaneous\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_miscellaneous\"" - } - } - }, - { - "evaluation_name": "Moral Scenarios", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Moral Scenarios", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.464, - "details": { - "description": "min=0.464, mean=0.464, max=0.464, sum=0.927 (2)", - "tab": "Accuracy", - "Moral Disputes - Observed inference time (s)": "{\"description\": \"min=0.112, mean=0.112, max=0.112, sum=0.225 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.11246242774566474\"}", - "Moral Scenarios - Observed inference time (s)": "{\"description\": \"min=0.112, mean=0.112, max=0.112, sum=0.223 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.11168156424580966\"}", - "Moral Disputes - # eval": "{\"description\": \"min=346, mean=346, max=346, sum=692 (2)\", \"tab\": \"General information\", \"score\": \"346.0\"}", - "Moral Disputes - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Disputes - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Disputes - # prompt tokens": "{\"description\": \"min=525.329, mean=525.329, max=525.329, sum=1050.659 (2)\", \"tab\": \"General information\", \"score\": \"525.3294797687861\"}", - "Moral Disputes - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Moral Scenarios - # eval": "{\"description\": \"min=895, mean=895, max=895, sum=1790 (2)\", \"tab\": \"General information\", \"score\": \"895.0\"}", - "Moral Scenarios - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Scenarios - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # prompt tokens": "{\"description\": \"min=692.482, mean=692.482, max=692.482, sum=1384.963 (2)\", \"tab\": \"General information\", \"score\": \"692.4815642458101\"}", - "Moral Scenarios - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"moral_scenarios\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_moral_scenarios\"" - } - } - }, - { - "evaluation_name": "Nutrition", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Nutrition", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.778, - "details": { - "description": "min=0.778, mean=0.778, max=0.778, sum=1.556 (2)", - "tab": "Accuracy", - "Nutrition - Observed inference time (s)": "{\"description\": \"min=0.109, mean=0.109, max=0.109, sum=0.219 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.1093660130718955\"}", - "Nutrition - # eval": "{\"description\": \"min=306, mean=306, max=306, sum=612 (2)\", \"tab\": \"General information\", \"score\": \"306.0\"}", - "Nutrition - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Nutrition - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Nutrition - # prompt tokens": "{\"description\": \"min=612.69, mean=612.69, max=612.69, sum=1225.379 (2)\", \"tab\": \"General information\", \"score\": \"612.6895424836601\"}", - "Nutrition - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"nutrition\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_nutrition\"" - } - } - }, - { - "evaluation_name": "Prehistory", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Prehistory", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.787, - "details": { - "description": "min=0.787, mean=0.787, max=0.787, sum=1.574 (2)", - "tab": "Accuracy", - "Prehistory - Observed inference time (s)": "{\"description\": \"min=0.11, mean=0.11, max=0.11, sum=0.22 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.1099814814814816\"}", - "Prehistory - # eval": "{\"description\": \"min=324, mean=324, max=324, sum=648 (2)\", \"tab\": \"General information\", \"score\": \"324.0\"}", - "Prehistory - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Prehistory - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Prehistory - # prompt tokens": "{\"description\": \"min=552.454, mean=552.454, max=552.454, sum=1104.907 (2)\", \"tab\": \"General information\", \"score\": \"552.4537037037037\"}", - "Prehistory - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"prehistory\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_prehistory\"" - } - } - }, - { - "evaluation_name": "Public Relations", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Public Relations", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.673, - "details": { - "description": "min=0.673, mean=0.673, max=0.673, sum=1.345 (2)", - "tab": "Accuracy", - "Public Relations - Observed inference time (s)": "{\"description\": \"min=0.108, mean=0.108, max=0.108, sum=0.215 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.1075000000000001\"}", - "Public Relations - # eval": "{\"description\": \"min=110, mean=110, max=110, sum=220 (2)\", \"tab\": \"General information\", \"score\": \"110.0\"}", - "Public Relations - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Public Relations - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Public Relations - # prompt tokens": "{\"description\": \"min=448.609, mean=448.609, max=448.609, sum=897.218 (2)\", \"tab\": \"General information\", \"score\": \"448.6090909090909\"}", - "Public Relations - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"public_relations\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_public_relations\"" - } - } - }, - { - "evaluation_name": "Security Studies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Security Studies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.718, - "details": { - "description": "min=0.718, mean=0.718, max=0.718, sum=1.437 (2)", - "tab": "Accuracy", - "Security Studies - Observed inference time (s)": "{\"description\": \"min=0.122, mean=0.122, max=0.122, sum=0.244 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.12202448979591832\"}", - "Security Studies - # eval": "{\"description\": \"min=245, mean=245, max=245, sum=490 (2)\", \"tab\": \"General information\", \"score\": \"245.0\"}", - "Security Studies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Security Studies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Security Studies - # prompt tokens": "{\"description\": \"min=1224.433, mean=1224.433, max=1224.433, sum=2448.865 (2)\", \"tab\": \"General information\", \"score\": \"1224.4326530612245\"}", - "Security Studies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"security_studies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_security_studies\"" - } - } - }, - { - "evaluation_name": "Sociology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Sociology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.846, - "details": { - "description": "min=0.846, mean=0.846, max=0.846, sum=1.692 (2)", - "tab": "Accuracy", - "Sociology - Observed inference time (s)": "{\"description\": \"min=0.11, mean=0.11, max=0.11, sum=0.221 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.11042288557213926\"}", - "Sociology - # eval": "{\"description\": \"min=201, mean=201, max=201, sum=402 (2)\", \"tab\": \"General information\", \"score\": \"201.0\"}", - "Sociology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Sociology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Sociology - # prompt tokens": "{\"description\": \"min=474.512, mean=474.512, max=474.512, sum=949.025 (2)\", \"tab\": \"General information\", \"score\": \"474.5124378109453\"}", - "Sociology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"sociology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_sociology\"" - } - } - }, - { - "evaluation_name": "Virology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Virology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.524, - "details": { - "description": "min=0.524, mean=0.524, max=0.524, sum=1.048 (2)", - "tab": "Accuracy", - "Virology - Observed inference time (s)": "{\"description\": \"min=0.104, mean=0.104, max=0.104, sum=0.209 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.10432530120481927\"}", - "Virology - # eval": "{\"description\": \"min=166, mean=166, max=166, sum=332 (2)\", \"tab\": \"General information\", \"score\": \"166.0\"}", - "Virology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Virology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Virology - # prompt tokens": "{\"description\": \"min=380.753, mean=380.753, max=380.753, sum=761.506 (2)\", \"tab\": \"General information\", \"score\": \"380.7530120481928\"}", - "Virology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"virology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_virology\"" - } - } - }, - { - "evaluation_name": "World Religions", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on World Religions", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.825, - "details": { - "description": "min=0.825, mean=0.825, max=0.825, sum=1.649 (2)", - "tab": "Accuracy", - "World Religions - Observed inference time (s)": "{\"description\": \"min=0.104, mean=0.104, max=0.104, sum=0.208 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.10395321637426902\"}", - "World Religions - # eval": "{\"description\": \"min=171, mean=171, max=171, sum=342 (2)\", \"tab\": \"General information\", \"score\": \"171.0\"}", - "World Religions - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "World Religions - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "World Religions - # prompt tokens": "{\"description\": \"min=305.386, mean=305.386, max=305.386, sum=610.772 (2)\", \"tab\": \"General information\", \"score\": \"305.3859649122807\"}", - "World Religions - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"world_religions\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_world_religions\"" - } - } - }, - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 1.0, - "details": { - "description": "", - "tab": "Efficiency" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/amazon_nova-premier-v1_0.json b/data/models/amazon_nova-premier-v1_0.json deleted file mode 100644 index ce3ed66a2af5cc94f9354c963b806a9b49d94bb5..0000000000000000000000000000000000000000 --- a/data/models/amazon_nova-premier-v1_0.json +++ /dev/null @@ -1,234 +0,0 @@ -{ - "model_info": { - "name": "Amazon Nova Premier", - "id": "amazon/nova-premier-v1:0", - "developer": "amazon", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_capabilities/amazon_nova-premier-v1:0/1774096304.5056698", - "retrieved_timestamp": "1774096304.5056698", - "source_metadata": { - "source_name": "helm_capabilities", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_capabilities", - "evaluation_results": [ - { - "evaluation_name": "Mean score", - "source_data": { - "dataset_name": "helm_capabilities", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "The mean of the scores from all columns.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.637, - "details": { - "description": "", - "tab": "Accuracy", - "Mean score - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"7.8055529408801165\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU-Pro", - "source_data": { - "dataset_name": "MMLU-Pro", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on MMLU-Pro", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.726, - "details": { - "description": "min=0.726, mean=0.726, max=0.726, sum=0.726 (1)", - "tab": "Accuracy", - "MMLU-Pro - Observed inference time (s)": "{\"description\": \"min=5.032, mean=5.032, max=5.032, sum=5.032 (1)\", \"tab\": \"Efficiency\", \"score\": \"5.031505000000002\"}", - "MMLU-Pro - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "MMLU-Pro - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - # prompt tokens": "{\"description\": \"min=235.232, mean=235.232, max=235.232, sum=235.232 (1)\", \"tab\": \"General information\", \"score\": \"235.232\"}", - "MMLU-Pro - # output tokens": "{\"description\": \"min=360.651, mean=360.651, max=360.651, sum=360.651 (1)\", \"tab\": \"General information\", \"score\": \"360.651\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"all\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.518, - "details": { - "description": "min=0.518, mean=0.518, max=0.518, sum=0.518 (1)", - "tab": "Accuracy", - "GPQA - Observed inference time (s)": "{\"description\": \"min=6.746, mean=6.746, max=6.746, sum=6.746 (1)\", \"tab\": \"Efficiency\", \"score\": \"6.7455403587443925\"}", - "GPQA - # eval": "{\"description\": \"min=446, mean=446, max=446, sum=446 (1)\", \"tab\": \"General information\", \"score\": \"446.0\"}", - "GPQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - # prompt tokens": "{\"description\": \"min=264.121, mean=264.121, max=264.121, sum=264.121 (1)\", \"tab\": \"General information\", \"score\": \"264.1210762331838\"}", - "GPQA - # output tokens": "{\"description\": \"min=452.691, mean=452.691, max=452.691, sum=452.691 (1)\", \"tab\": \"General information\", \"score\": \"452.69058295964123\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"gpqa_main\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "IFEval Strict Acc on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.803, - "details": { - "description": "min=0.803, mean=0.803, max=0.803, sum=0.803 (1)", - "tab": "Accuracy", - "IFEval - Observed inference time (s)": "{\"description\": \"min=6.027, mean=6.027, max=6.027, sum=6.027 (1)\", \"tab\": \"Efficiency\", \"score\": \"6.026593345656195\"}", - "IFEval - # eval": "{\"description\": \"min=541, mean=541, max=541, sum=541 (1)\", \"tab\": \"General information\", \"score\": \"541.0\"}", - "IFEval - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - # prompt tokens": "{\"description\": \"min=47.58, mean=47.58, max=47.58, sum=47.58 (1)\", \"tab\": \"General information\", \"score\": \"47.58040665434381\"}", - "IFEval - # output tokens": "{\"description\": \"min=325.945, mean=325.945, max=325.945, sum=325.945 (1)\", \"tab\": \"General information\", \"score\": \"325.9445471349353\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WildBench", - "source_data": { - "dataset_name": "WildBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "WB Score on WildBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.788, - "details": { - "description": "min=0.788, mean=0.788, max=0.788, sum=0.788 (1)", - "tab": "Accuracy", - "WildBench - Observed inference time (s)": "{\"description\": \"min=13.055, mean=13.055, max=13.055, sum=13.055 (1)\", \"tab\": \"Efficiency\", \"score\": \"13.055127999999996\"}", - "WildBench - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "WildBench - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # prompt tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # output tokens": "{\"description\": \"min=814.969, mean=814.969, max=814.969, sum=814.969 (1)\", \"tab\": \"General information\", \"score\": \"814.969\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"v2\"" - } - } - }, - { - "evaluation_name": "Omni-MATH", - "source_data": { - "dataset_name": "Omni-MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Acc on Omni-MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.35, - "details": { - "description": "min=0.35, mean=0.35, max=0.35, sum=0.35 (1)", - "tab": "Accuracy", - "Omni-MATH - Observed inference time (s)": "{\"description\": \"min=8.169, mean=8.169, max=8.169, sum=8.169 (1)\", \"tab\": \"Efficiency\", \"score\": \"8.168997999999998\"}", - "Omni-MATH - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "Omni-MATH - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - # prompt tokens": "{\"description\": \"min=117.921, mean=117.921, max=117.921, sum=117.921 (1)\", \"tab\": \"General information\", \"score\": \"117.921\"}", - "Omni-MATH - # output tokens": "{\"description\": \"min=778.909, mean=778.909, max=778.909, sum=778.909 (1)\", \"tab\": \"General information\", \"score\": \"778.909\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/amazon_nova-pro-v1_0.json b/data/models/amazon_nova-pro-v1_0.json deleted file mode 100644 index acae983f52dfc57e6db24fb77e5017a8c201091e..0000000000000000000000000000000000000000 --- a/data/models/amazon_nova-pro-v1_0.json +++ /dev/null @@ -1,2128 +0,0 @@ -{ - "model_info": { - "name": "Amazon Nova Pro", - "id": "amazon/nova-pro-v1:0", - "developer": "amazon", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_capabilities/amazon_nova-pro-v1:0/1774096304.5056698", - "retrieved_timestamp": "1774096304.5056698", - "source_metadata": { - "source_name": "helm_capabilities", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_capabilities", - "evaluation_results": [ - { - "evaluation_name": "Mean score", - "source_data": { - "dataset_name": "helm_capabilities", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "The mean of the scores from all columns.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.591, - "details": { - "description": "", - "tab": "Accuracy", - "Mean score - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"6.538285667967472\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU-Pro", - "source_data": { - "dataset_name": "MMLU-Pro", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on MMLU-Pro", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.673, - "details": { - "description": "min=0.673, mean=0.673, max=0.673, sum=0.673 (1)", - "tab": "Accuracy", - "MMLU-Pro - Observed inference time (s)": "{\"description\": \"min=4.554, mean=4.554, max=4.554, sum=4.554 (1)\", \"tab\": \"Efficiency\", \"score\": \"4.554401999999996\"}", - "MMLU-Pro - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "MMLU-Pro - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - # prompt tokens": "{\"description\": \"min=235.232, mean=235.232, max=235.232, sum=235.232 (1)\", \"tab\": \"General information\", \"score\": \"235.232\"}", - "MMLU-Pro - # output tokens": "{\"description\": \"min=381.807, mean=381.807, max=381.807, sum=381.807 (1)\", \"tab\": \"General information\", \"score\": \"381.807\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"all\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.446, - "details": { - "description": "min=0.446, mean=0.446, max=0.446, sum=0.446 (1)", - "tab": "Accuracy", - "GPQA - Observed inference time (s)": "{\"description\": \"min=5.948, mean=5.948, max=5.948, sum=5.948 (1)\", \"tab\": \"Efficiency\", \"score\": \"5.947926008968607\"}", - "GPQA - # eval": "{\"description\": \"min=446, mean=446, max=446, sum=446 (1)\", \"tab\": \"General information\", \"score\": \"446.0\"}", - "GPQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - # prompt tokens": "{\"description\": \"min=264.121, mean=264.121, max=264.121, sum=264.121 (1)\", \"tab\": \"General information\", \"score\": \"264.1210762331838\"}", - "GPQA - # output tokens": "{\"description\": \"min=534.013, mean=534.013, max=534.013, sum=534.013 (1)\", \"tab\": \"General information\", \"score\": \"534.0134529147982\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"gpqa_main\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "IFEval Strict Acc on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.815, - "details": { - "description": "min=0.815, mean=0.815, max=0.815, sum=0.815 (1)", - "tab": "Accuracy", - "IFEval - Observed inference time (s)": "{\"description\": \"min=3.945, mean=3.945, max=3.945, sum=3.945 (1)\", \"tab\": \"Efficiency\", \"score\": \"3.945081330868756\"}", - "IFEval - # eval": "{\"description\": \"min=541, mean=541, max=541, sum=541 (1)\", \"tab\": \"General information\", \"score\": \"541.0\"}", - "IFEval - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - # prompt tokens": "{\"description\": \"min=47.58, mean=47.58, max=47.58, sum=47.58 (1)\", \"tab\": \"General information\", \"score\": \"47.58040665434381\"}", - "IFEval - # output tokens": "{\"description\": \"min=383.871, mean=383.871, max=383.871, sum=383.871 (1)\", \"tab\": \"General information\", \"score\": \"383.8706099815157\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WildBench", - "source_data": { - "dataset_name": "WildBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "WB Score on WildBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.777, - "details": { - "description": "min=0.777, mean=0.777, max=0.777, sum=0.777 (1)", - "tab": "Accuracy", - "WildBench - Observed inference time (s)": "{\"description\": \"min=10.635, mean=10.635, max=10.635, sum=10.635 (1)\", \"tab\": \"Efficiency\", \"score\": \"10.635314999999995\"}", - "WildBench - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "WildBench - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # prompt tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # output tokens": "{\"description\": \"min=899.758, mean=899.758, max=899.758, sum=899.758 (1)\", \"tab\": \"General information\", \"score\": \"899.758\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"v2\"" - } - } - }, - { - "evaluation_name": "Omni-MATH", - "source_data": { - "dataset_name": "Omni-MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Acc on Omni-MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.242, - "details": { - "description": "min=0.242, mean=0.242, max=0.242, sum=0.242 (1)", - "tab": "Accuracy", - "Omni-MATH - Observed inference time (s)": "{\"description\": \"min=7.609, mean=7.609, max=7.609, sum=7.609 (1)\", \"tab\": \"Efficiency\", \"score\": \"7.608704000000004\"}", - "Omni-MATH - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "Omni-MATH - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - # prompt tokens": "{\"description\": \"min=117.921, mean=117.921, max=117.921, sum=117.921 (1)\", \"tab\": \"General information\", \"score\": \"117.921\"}", - "Omni-MATH - # output tokens": "{\"description\": \"min=649.195, mean=649.195, max=649.195, sum=649.195 (1)\", \"tab\": \"General information\", \"score\": \"649.195\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_lite/amazon_nova-pro-v1:0/1774096306.427425", - "retrieved_timestamp": "1774096306.427425", - "source_metadata": { - "source_name": "helm_lite", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_lite", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_lite", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.885, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.9342571785268414\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.791, - "details": { - "description": "min=0.791, mean=0.791, max=0.791, sum=0.791 (1)", - "tab": "Accuracy", - "NarrativeQA - Observed inference time (s)": "{\"description\": \"min=0.246, mean=0.246, max=0.246, sum=0.246 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.24631830985915482\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=3550.577, mean=3550.577, max=3550.577, sum=3550.577 (1)\", \"tab\": \"General information\", \"score\": \"3550.5774647887324\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=4.651, mean=4.651, max=4.651, sum=4.651 (1)\", \"tab\": \"General information\", \"score\": \"4.650704225352112\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (closed-book)", - "source_data": { - "dataset_name": "NaturalQuestions (closed-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (closed-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.405, - "details": { - "description": "min=0.405, mean=0.405, max=0.405, sum=0.405 (1)", - "tab": "Accuracy", - "NaturalQuestions (open-book) - Observed inference time (s)": "{\"description\": \"min=0.266, mean=0.266, max=0.266, sum=0.266 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.26591999999999993\"}", - "NaturalQuestions (closed-book) - Observed inference time (s)": "{\"description\": \"min=0.203, mean=0.203, max=0.203, sum=0.203 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.203244\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1773.944, mean=1773.944, max=1773.944, sum=1773.944 (1)\", \"tab\": \"General information\", \"score\": \"1773.944\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=5.646, mean=5.646, max=5.646, sum=5.646 (1)\", \"tab\": \"General information\", \"score\": \"5.646\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=153.254, mean=153.254, max=153.254, sum=153.254 (1)\", \"tab\": \"General information\", \"score\": \"153.254\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=4.207, mean=4.207, max=4.207, sum=4.207 (1)\", \"tab\": \"General information\", \"score\": \"4.207\"}" - } - }, - "generation_config": { - "additional_details": { - "mode": "\"closedbook\"" - } - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.96, - "details": { - "description": "min=0.96, mean=0.96, max=0.96, sum=0.96 (1)", - "tab": "Accuracy", - "OpenbookQA - Observed inference time (s)": "{\"description\": \"min=0.129, mean=0.129, max=0.129, sum=0.129 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.12889800000000004\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=282.21, mean=282.21, max=282.21, sum=282.21 (1)\", \"tab\": \"General information\", \"score\": \"282.21\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "dataset": "\"openbookqa\"", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.758, - "details": { - "description": "min=0.63, mean=0.758, max=0.93, sum=3.792 (5)", - "tab": "Accuracy", - "MMLU - Observed inference time (s)": "{\"description\": \"min=0.141, mean=0.145, max=0.152, sum=0.725 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.1449304210526316\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=399.38, mean=500.274, max=652.07, sum=2501.37 (5)\", \"tab\": \"General information\", \"score\": \"500.2740350877192\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"college_chemistry\", \"computer_security\", \"econometrics\", \"us_foreign_policy\"]", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MATH", - "source_data": { - "dataset_name": "MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Equivalent (CoT) on MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.821, - "details": { - "description": "min=0.7, mean=0.821, max=0.93, sum=5.749 (7)", - "tab": "Accuracy", - "MATH - Observed inference time (s)": "{\"description\": \"min=1.139, mean=1.695, max=2.518, sum=11.863 (7)\", \"tab\": \"Efficiency\", \"score\": \"1.6947358347418935\"}", - "MATH - # eval": "{\"description\": \"min=30, mean=62.429, max=135, sum=437 (7)\", \"tab\": \"General information\", \"score\": \"62.42857142857143\"}", - "MATH - # train": "{\"description\": \"min=8, mean=8, max=8, sum=56 (7)\", \"tab\": \"General information\", \"score\": \"8.0\"}", - "MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MATH - # prompt tokens": "{\"description\": \"min=925.556, mean=1394.735, max=2468.942, sum=9763.147 (7)\", \"tab\": \"General information\", \"score\": \"1394.7353092779651\"}", - "MATH - # output tokens": "{\"description\": \"min=66.088, mean=98.114, max=154.135, sum=686.8 (7)\", \"tab\": \"General information\", \"score\": \"98.11425246180445\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"algebra\", \"counting_and_probability\", \"geometry\", \"intermediate_algebra\", \"number_theory\", \"prealgebra\", \"precalculus\"]", - "level": "\"1\"", - "use_official_examples": "\"False\"", - "use_chain_of_thought": "\"True\"" - } - } - }, - { - "evaluation_name": "GSM8K", - "source_data": { - "dataset_name": "GSM8K", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on GSM8K", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.87, - "details": { - "description": "min=0.87, mean=0.87, max=0.87, sum=0.87 (1)", - "tab": "Accuracy", - "GSM8K - Observed inference time (s)": "{\"description\": \"min=1.566, mean=1.566, max=1.566, sum=1.566 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.5656869999999996\"}", - "GSM8K - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "GSM8K - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "GSM8K - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GSM8K - # prompt tokens": "{\"description\": \"min=957.869, mean=957.869, max=957.869, sum=957.869 (1)\", \"tab\": \"General information\", \"score\": \"957.869\"}", - "GSM8K - # output tokens": "{\"description\": \"min=73.847, mean=73.847, max=73.847, sum=73.847 (1)\", \"tab\": \"General information\", \"score\": \"73.847\"}" - } - }, - "generation_config": { - "additional_details": { - "stop": "\"none\"" - } - } - }, - { - "evaluation_name": "LegalBench", - "source_data": { - "dataset_name": "LegalBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on LegalBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.736, - "details": { - "description": "min=0.444, mean=0.736, max=0.958, sum=3.681 (5)", - "tab": "Accuracy", - "LegalBench - Observed inference time (s)": "{\"description\": \"min=0.139, mean=0.166, max=0.232, sum=0.83 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.16605967288111284\"}", - "LegalBench - # eval": "{\"description\": \"min=95, mean=409.4, max=1000, sum=2047 (5)\", \"tab\": \"General information\", \"score\": \"409.4\"}", - "LegalBench - # train": "{\"description\": \"min=4, mean=4.8, max=5, sum=24 (5)\", \"tab\": \"General information\", \"score\": \"4.8\"}", - "LegalBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "LegalBench - # prompt tokens": "{\"description\": \"min=241.632, mean=1581.083, max=6449.798, sum=7905.414 (5)\", \"tab\": \"General information\", \"score\": \"1581.0827222540588\"}", - "LegalBench - # output tokens": "{\"description\": \"min=1, mean=1.387, max=2.358, sum=6.936 (5)\", \"tab\": \"General information\", \"score\": \"1.3871102825182848\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"abercrombie\", \"corporate_lobbying\", \"function_of_decision_section\", \"international_citizenship_questions\", \"proa\"]", - "stop": "\"none\"" - } - } - }, - { - "evaluation_name": "MedQA", - "source_data": { - "dataset_name": "MedQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MedQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.811, - "details": { - "description": "min=0.811, mean=0.811, max=0.811, sum=0.811 (1)", - "tab": "Accuracy", - "MedQA - Observed inference time (s)": "{\"description\": \"min=0.142, mean=0.142, max=0.142, sum=0.142 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.14219284294234621\"}", - "MedQA - # eval": "{\"description\": \"min=503, mean=503, max=503, sum=503 (1)\", \"tab\": \"General information\", \"score\": \"503.0\"}", - "MedQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MedQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MedQA - # prompt tokens": "{\"description\": \"min=1066.861, mean=1066.861, max=1066.861, sum=1066.861 (1)\", \"tab\": \"General information\", \"score\": \"1066.8608349900596\"}", - "MedQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WMT 2014", - "source_data": { - "dataset_name": "WMT 2014", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "BLEU-4 on WMT 2014", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.229, - "details": { - "description": "min=0.184, mean=0.229, max=0.281, sum=1.144 (5)", - "tab": "Accuracy", - "WMT 2014 - Observed inference time (s)": "{\"description\": \"min=0.483, mean=0.504, max=0.519, sum=2.52 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.5040968109611562\"}", - "WMT 2014 - # eval": "{\"description\": \"min=503, mean=568.8, max=832, sum=2844 (5)\", \"tab\": \"General information\", \"score\": \"568.8\"}", - "WMT 2014 - # train": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "WMT 2014 - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WMT 2014 - # prompt tokens": "{\"description\": \"min=163.93, mean=208.694, max=268.662, sum=1043.469 (5)\", \"tab\": \"General information\", \"score\": \"208.69386660804403\"}", - "WMT 2014 - # output tokens": "{\"description\": \"min=23.903, mean=25.328, max=25.92, sum=126.641 (5)\", \"tab\": \"General information\", \"score\": \"25.32825594509864\"}" - } - }, - "generation_config": { - "additional_details": { - "language_pair": "[\"cs-en\", \"de-en\", \"fr-en\", \"hi-en\", \"ru-en\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_mmlu/amazon_nova-pro-v1:0/1774096312.00548", - "retrieved_timestamp": "1774096312.00548", - "source_metadata": { - "source_name": "helm_mmlu", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_mmlu", - "evaluation_results": [ - { - "evaluation_name": "MMLU All Subjects", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU All Subjects", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.82, - "details": { - "description": "min=0.54, mean=0.82, max=0.974, sum=93.477 (114)", - "tab": "Accuracy", - "MMLU All Subjects - Observed inference time (s)": "{\"description\": \"min=0.128, mean=0.14, max=0.17, sum=15.944 (114)\", \"tab\": \"Efficiency\", \"score\": \"0.13986169479756677\"}", - "MMLU All Subjects - # eval": "{\"description\": \"min=100, mean=246.351, max=1534, sum=28084 (114)\", \"tab\": \"General information\", \"score\": \"246.35087719298247\"}", - "MMLU All Subjects - # train": "{\"description\": \"min=5, mean=5, max=5, sum=570 (114)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU All Subjects - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU All Subjects - # prompt tokens": "{\"description\": \"min=305.386, mean=655.489, max=2872.03, sum=74725.746 (114)\", \"tab\": \"General information\", \"score\": \"655.4890026560713\"}", - "MMLU All Subjects - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=114 (114)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_name": "Abstract Algebra", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Abstract Algebra", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.69, - "details": { - "description": "min=0.69, mean=0.69, max=0.69, sum=1.38 (2)", - "tab": "Accuracy", - "Abstract Algebra - Observed inference time (s)": "{\"description\": \"min=0.152, mean=0.152, max=0.152, sum=0.305 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.15239000000000003\"}", - "Abstract Algebra - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Abstract Algebra - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Abstract Algebra - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Abstract Algebra - # prompt tokens": "{\"description\": \"min=399.38, mean=399.38, max=399.38, sum=798.76 (2)\", \"tab\": \"General information\", \"score\": \"399.38\"}", - "Abstract Algebra - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"abstract_algebra\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_abstract_algebra\"" - } - } - }, - { - "evaluation_name": "Anatomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Anatomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.807, - "details": { - "description": "min=0.807, mean=0.807, max=0.807, sum=1.615 (2)", - "tab": "Accuracy", - "Anatomy - Observed inference time (s)": "{\"description\": \"min=0.138, mean=0.138, max=0.138, sum=0.275 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.13757037037037034\"}", - "Anatomy - # eval": "{\"description\": \"min=135, mean=135, max=135, sum=270 (2)\", \"tab\": \"General information\", \"score\": \"135.0\"}", - "Anatomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Anatomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Anatomy - # prompt tokens": "{\"description\": \"min=400.081, mean=400.081, max=400.081, sum=800.163 (2)\", \"tab\": \"General information\", \"score\": \"400.0814814814815\"}", - "Anatomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"anatomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_anatomy\"" - } - } - }, - { - "evaluation_name": "College Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on College Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.647, - "details": { - "description": "min=0.647, mean=0.647, max=0.647, sum=1.294 (2)", - "tab": "Accuracy", - "College Chemistry - Observed inference time (s)": "{\"description\": \"min=0.148, mean=0.148, max=0.148, sum=0.296 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.14806999999999998\"}", - "College Biology - Observed inference time (s)": "{\"description\": \"min=0.148, mean=0.148, max=0.148, sum=0.296 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.14820138888888884\"}", - "College Computer Science - Observed inference time (s)": "{\"description\": \"min=0.152, mean=0.152, max=0.152, sum=0.305 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.15245\"}", - "College Mathematics - Observed inference time (s)": "{\"description\": \"min=0.151, mean=0.151, max=0.151, sum=0.303 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.15141\"}", - "College Medicine - Observed inference time (s)": "{\"description\": \"min=0.143, mean=0.143, max=0.143, sum=0.287 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.1433988439306358\"}", - "College Physics - Observed inference time (s)": "{\"description\": \"min=0.146, mean=0.146, max=0.146, sum=0.292 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.14623529411764705\"}", - "College Chemistry - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Chemistry - # prompt tokens": "{\"description\": \"min=573.4, mean=573.4, max=573.4, sum=1146.8 (2)\", \"tab\": \"General information\", \"score\": \"573.4\"}", - "College Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Biology - # eval": "{\"description\": \"min=144, mean=144, max=144, sum=288 (2)\", \"tab\": \"General information\", \"score\": \"144.0\"}", - "College Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # prompt tokens": "{\"description\": \"min=510.278, mean=510.278, max=510.278, sum=1020.556 (2)\", \"tab\": \"General information\", \"score\": \"510.27777777777777\"}", - "College Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # prompt tokens": "{\"description\": \"min=880.15, mean=880.15, max=880.15, sum=1760.3 (2)\", \"tab\": \"General information\", \"score\": \"880.15\"}", - "College Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Mathematics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # prompt tokens": "{\"description\": \"min=639.53, mean=639.53, max=639.53, sum=1279.06 (2)\", \"tab\": \"General information\", \"score\": \"639.53\"}", - "College Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Medicine - # eval": "{\"description\": \"min=173, mean=173, max=173, sum=346 (2)\", \"tab\": \"General information\", \"score\": \"173.0\"}", - "College Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # prompt tokens": "{\"description\": \"min=558.301, mean=558.301, max=558.301, sum=1116.601 (2)\", \"tab\": \"General information\", \"score\": \"558.3005780346821\"}", - "College Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Physics - # eval": "{\"description\": \"min=102, mean=102, max=102, sum=204 (2)\", \"tab\": \"General information\", \"score\": \"102.0\"}", - "College Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # prompt tokens": "{\"description\": \"min=517.324, mean=517.324, max=517.324, sum=1034.647 (2)\", \"tab\": \"General information\", \"score\": \"517.3235294117648\"}", - "College Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"college_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_college_physics\"" - } - } - }, - { - "evaluation_name": "Computer Security", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Computer Security", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.84, - "details": { - "description": "min=0.84, mean=0.84, max=0.84, sum=1.68 (2)", - "tab": "Accuracy", - "Computer Security - Observed inference time (s)": "{\"description\": \"min=0.141, mean=0.141, max=0.141, sum=0.281 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.14067000000000005\"}", - "Computer Security - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Computer Security - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Computer Security - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Computer Security - # prompt tokens": "{\"description\": \"min=415.4, mean=415.4, max=415.4, sum=830.8 (2)\", \"tab\": \"General information\", \"score\": \"415.4\"}", - "Computer Security - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"computer_security\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_computer_security\"" - } - } - }, - { - "evaluation_name": "Econometrics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Econometrics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.702, - "details": { - "description": "min=0.702, mean=0.702, max=0.702, sum=1.404 (2)", - "tab": "Accuracy", - "Econometrics - Observed inference time (s)": "{\"description\": \"min=0.142, mean=0.142, max=0.142, sum=0.285 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.1423421052631579\"}", - "Econometrics - # eval": "{\"description\": \"min=114, mean=114, max=114, sum=228 (2)\", \"tab\": \"General information\", \"score\": \"114.0\"}", - "Econometrics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Econometrics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Econometrics - # prompt tokens": "{\"description\": \"min=652.07, mean=652.07, max=652.07, sum=1304.14 (2)\", \"tab\": \"General information\", \"score\": \"652.0701754385965\"}", - "Econometrics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"econometrics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_econometrics\"" - } - } - }, - { - "evaluation_name": "Global Facts", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Global Facts", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.54, - "details": { - "description": "min=0.54, mean=0.54, max=0.54, sum=1.08 (2)", - "tab": "Accuracy", - "Global Facts - Observed inference time (s)": "{\"description\": \"min=0.142, mean=0.142, max=0.142, sum=0.283 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.14153999999999997\"}", - "Global Facts - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Global Facts - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Global Facts - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Global Facts - # prompt tokens": "{\"description\": \"min=426.42, mean=426.42, max=426.42, sum=852.84 (2)\", \"tab\": \"General information\", \"score\": \"426.42\"}", - "Global Facts - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"global_facts\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_global_facts\"" - } - } - }, - { - "evaluation_name": "Jurisprudence", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Jurisprudence", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.861, - "details": { - "description": "min=0.861, mean=0.861, max=0.861, sum=1.722 (2)", - "tab": "Accuracy", - "Jurisprudence - Observed inference time (s)": "{\"description\": \"min=0.141, mean=0.141, max=0.141, sum=0.282 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.14100925925925917\"}", - "Jurisprudence - # eval": "{\"description\": \"min=108, mean=108, max=108, sum=216 (2)\", \"tab\": \"General information\", \"score\": \"108.0\"}", - "Jurisprudence - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Jurisprudence - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Jurisprudence - # prompt tokens": "{\"description\": \"min=446.722, mean=446.722, max=446.722, sum=893.444 (2)\", \"tab\": \"General information\", \"score\": \"446.72222222222223\"}", - "Jurisprudence - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"jurisprudence\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_jurisprudence\"" - } - } - }, - { - "evaluation_name": "Philosophy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Philosophy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.826, - "details": { - "description": "min=0.826, mean=0.826, max=0.826, sum=1.653 (2)", - "tab": "Accuracy", - "Philosophy - Observed inference time (s)": "{\"description\": \"min=0.131, mean=0.131, max=0.131, sum=0.261 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.1307266881028939\"}", - "Philosophy - # eval": "{\"description\": \"min=311, mean=311, max=311, sum=622 (2)\", \"tab\": \"General information\", \"score\": \"311.0\"}", - "Philosophy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Philosophy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Philosophy - # prompt tokens": "{\"description\": \"min=381.704, mean=381.704, max=381.704, sum=763.408 (2)\", \"tab\": \"General information\", \"score\": \"381.7041800643087\"}", - "Philosophy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"philosophy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_philosophy\"" - } - } - }, - { - "evaluation_name": "Professional Psychology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Professional Psychology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.864, - "details": { - "description": "min=0.864, mean=0.864, max=0.864, sum=1.729 (2)", - "tab": "Accuracy", - "Professional Medicine - Observed inference time (s)": "{\"description\": \"min=0.145, mean=0.145, max=0.145, sum=0.291 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.14530882352941174\"}", - "Professional Accounting - Observed inference time (s)": "{\"description\": \"min=0.139, mean=0.139, max=0.139, sum=0.278 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.1388758865248228\"}", - "Professional Law - Observed inference time (s)": "{\"description\": \"min=0.146, mean=0.146, max=0.146, sum=0.292 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.14584159061277666\"}", - "Professional Psychology - Observed inference time (s)": "{\"description\": \"min=0.132, mean=0.132, max=0.132, sum=0.264 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.13185620915032703\"}", - "Professional Medicine - # eval": "{\"description\": \"min=272, mean=272, max=272, sum=544 (2)\", \"tab\": \"General information\", \"score\": \"272.0\"}", - "Professional Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Medicine - # prompt tokens": "{\"description\": \"min=1146.287, mean=1146.287, max=1146.287, sum=2292.574 (2)\", \"tab\": \"General information\", \"score\": \"1146.2867647058824\"}", - "Professional Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Accounting - # eval": "{\"description\": \"min=282, mean=282, max=282, sum=564 (2)\", \"tab\": \"General information\", \"score\": \"282.0\"}", - "Professional Accounting - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Accounting - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # prompt tokens": "{\"description\": \"min=688.72, mean=688.72, max=688.72, sum=1377.44 (2)\", \"tab\": \"General information\", \"score\": \"688.7198581560284\"}", - "Professional Accounting - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Law - # eval": "{\"description\": \"min=1534, mean=1534, max=1534, sum=3068 (2)\", \"tab\": \"General information\", \"score\": \"1534.0\"}", - "Professional Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # prompt tokens": "{\"description\": \"min=1686.73, mean=1686.73, max=1686.73, sum=3373.46 (2)\", \"tab\": \"General information\", \"score\": \"1686.7301173402868\"}", - "Professional Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Psychology - # eval": "{\"description\": \"min=612, mean=612, max=612, sum=1224 (2)\", \"tab\": \"General information\", \"score\": \"612.0\"}", - "Professional Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # prompt tokens": "{\"description\": \"min=625.574, mean=625.574, max=625.574, sum=1251.147 (2)\", \"tab\": \"General information\", \"score\": \"625.5735294117648\"}", - "Professional Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"professional_psychology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_professional_psychology\"" - } - } - }, - { - "evaluation_name": "Us Foreign Policy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Us Foreign Policy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.93, - "details": { - "description": "min=0.93, mean=0.93, max=0.93, sum=1.86 (2)", - "tab": "Accuracy", - "Us Foreign Policy - Observed inference time (s)": "{\"description\": \"min=0.141, mean=0.141, max=0.141, sum=0.282 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.14117999999999994\"}", - "Us Foreign Policy - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Us Foreign Policy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Us Foreign Policy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Us Foreign Policy - # prompt tokens": "{\"description\": \"min=461.12, mean=461.12, max=461.12, sum=922.24 (2)\", \"tab\": \"General information\", \"score\": \"461.12\"}", - "Us Foreign Policy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"us_foreign_policy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_us_foreign_policy\"" - } - } - }, - { - "evaluation_name": "Astronomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Astronomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.895, - "details": { - "description": "min=0.895, mean=0.895, max=0.895, sum=1.789 (2)", - "tab": "Accuracy", - "Astronomy - Observed inference time (s)": "{\"description\": \"min=0.141, mean=0.141, max=0.141, sum=0.282 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.1411447368421052\"}", - "Astronomy - # eval": "{\"description\": \"min=152, mean=152, max=152, sum=304 (2)\", \"tab\": \"General information\", \"score\": \"152.0\"}", - "Astronomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Astronomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Astronomy - # prompt tokens": "{\"description\": \"min=628.112, mean=628.112, max=628.112, sum=1256.224 (2)\", \"tab\": \"General information\", \"score\": \"628.1118421052631\"}", - "Astronomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"astronomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_astronomy\"" - } - } - }, - { - "evaluation_name": "Business Ethics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Business Ethics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.81, - "details": { - "description": "min=0.81, mean=0.81, max=0.81, sum=1.62 (2)", - "tab": "Accuracy", - "Business Ethics - Observed inference time (s)": "{\"description\": \"min=0.144, mean=0.144, max=0.144, sum=0.288 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.14414\"}", - "Business Ethics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Business Ethics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Business Ethics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Business Ethics - # prompt tokens": "{\"description\": \"min=617.46, mean=617.46, max=617.46, sum=1234.92 (2)\", \"tab\": \"General information\", \"score\": \"617.46\"}", - "Business Ethics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"business_ethics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_business_ethics\"" - } - } - }, - { - "evaluation_name": "Clinical Knowledge", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Clinical Knowledge", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.875, - "details": { - "description": "min=0.875, mean=0.875, max=0.875, sum=1.751 (2)", - "tab": "Accuracy", - "Clinical Knowledge - Observed inference time (s)": "{\"description\": \"min=0.142, mean=0.142, max=0.142, sum=0.284 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.14190943396226424\"}", - "Clinical Knowledge - # eval": "{\"description\": \"min=265, mean=265, max=265, sum=530 (2)\", \"tab\": \"General information\", \"score\": \"265.0\"}", - "Clinical Knowledge - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Clinical Knowledge - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Clinical Knowledge - # prompt tokens": "{\"description\": \"min=451.925, mean=451.925, max=451.925, sum=903.849 (2)\", \"tab\": \"General information\", \"score\": \"451.92452830188677\"}", - "Clinical Knowledge - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"clinical_knowledge\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_clinical_knowledge\"" - } - } - }, - { - "evaluation_name": "Conceptual Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Conceptual Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.851, - "details": { - "description": "min=0.851, mean=0.851, max=0.851, sum=1.702 (2)", - "tab": "Accuracy", - "Conceptual Physics - Observed inference time (s)": "{\"description\": \"min=0.132, mean=0.132, max=0.132, sum=0.264 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.13199148936170213\"}", - "Conceptual Physics - # eval": "{\"description\": \"min=235, mean=235, max=235, sum=470 (2)\", \"tab\": \"General information\", \"score\": \"235.0\"}", - "Conceptual Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Conceptual Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Conceptual Physics - # prompt tokens": "{\"description\": \"min=341.723, mean=341.723, max=341.723, sum=683.447 (2)\", \"tab\": \"General information\", \"score\": \"341.72340425531917\"}", - "Conceptual Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"conceptual_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_conceptual_physics\"" - } - } - }, - { - "evaluation_name": "Electrical Engineering", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Electrical Engineering", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8, - "details": { - "description": "min=0.8, mean=0.8, max=0.8, sum=1.6 (2)", - "tab": "Accuracy", - "Electrical Engineering - Observed inference time (s)": "{\"description\": \"min=0.135, mean=0.135, max=0.135, sum=0.27 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.1350000000000001\"}", - "Electrical Engineering - # eval": "{\"description\": \"min=145, mean=145, max=145, sum=290 (2)\", \"tab\": \"General information\", \"score\": \"145.0\"}", - "Electrical Engineering - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Electrical Engineering - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Electrical Engineering - # prompt tokens": "{\"description\": \"min=458.345, mean=458.345, max=458.345, sum=916.69 (2)\", \"tab\": \"General information\", \"score\": \"458.3448275862069\"}", - "Electrical Engineering - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"electrical_engineering\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_electrical_engineering\"" - } - } - }, - { - "evaluation_name": "Elementary Mathematics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Elementary Mathematics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.831, - "details": { - "description": "min=0.831, mean=0.831, max=0.831, sum=1.661 (2)", - "tab": "Accuracy", - "Elementary Mathematics - Observed inference time (s)": "{\"description\": \"min=0.142, mean=0.142, max=0.142, sum=0.285 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.14232010582010587\"}", - "Elementary Mathematics - # eval": "{\"description\": \"min=378, mean=378, max=378, sum=756 (2)\", \"tab\": \"General information\", \"score\": \"378.0\"}", - "Elementary Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Elementary Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Elementary Mathematics - # prompt tokens": "{\"description\": \"min=534.09, mean=534.09, max=534.09, sum=1068.18 (2)\", \"tab\": \"General information\", \"score\": \"534.0899470899471\"}", - "Elementary Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"elementary_mathematics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_elementary_mathematics\"" - } - } - }, - { - "evaluation_name": "Formal Logic", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Formal Logic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.714, - "details": { - "description": "min=0.714, mean=0.714, max=0.714, sum=1.429 (2)", - "tab": "Accuracy", - "Formal Logic - Observed inference time (s)": "{\"description\": \"min=0.145, mean=0.145, max=0.145, sum=0.29 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.1448888888888889\"}", - "Formal Logic - # eval": "{\"description\": \"min=126, mean=126, max=126, sum=252 (2)\", \"tab\": \"General information\", \"score\": \"126.0\"}", - "Formal Logic - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Formal Logic - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Formal Logic - # prompt tokens": "{\"description\": \"min=669, mean=669, max=669, sum=1338 (2)\", \"tab\": \"General information\", \"score\": \"669.0\"}", - "Formal Logic - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"formal_logic\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_formal_logic\"" - } - } - }, - { - "evaluation_name": "High School World History", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on High School World History", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.928, - "details": { - "description": "min=0.928, mean=0.928, max=0.928, sum=1.857 (2)", - "tab": "Accuracy", - "High School Biology - Observed inference time (s)": "{\"description\": \"min=0.139, mean=0.139, max=0.139, sum=0.278 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.13894516129032267\"}", - "High School Chemistry - Observed inference time (s)": "{\"description\": \"min=0.139, mean=0.139, max=0.139, sum=0.278 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.13885221674876858\"}", - "High School Computer Science - Observed inference time (s)": "{\"description\": \"min=0.148, mean=0.148, max=0.148, sum=0.296 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.1479\"}", - "High School European History - Observed inference time (s)": "{\"description\": \"min=0.17, mean=0.17, max=0.17, sum=0.341 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.17033939393939396\"}", - "High School Geography - Observed inference time (s)": "{\"description\": \"min=0.133, mean=0.133, max=0.133, sum=0.266 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.13296969696969696\"}", - "High School Government And Politics - Observed inference time (s)": "{\"description\": \"min=0.135, mean=0.135, max=0.135, sum=0.27 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.1351139896373057\"}", - "High School Macroeconomics - Observed inference time (s)": "{\"description\": \"min=0.134, mean=0.134, max=0.134, sum=0.268 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.1338025641025641\"}", - "High School Mathematics - Observed inference time (s)": "{\"description\": \"min=0.14, mean=0.14, max=0.14, sum=0.279 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.13964074074074065\"}", - "High School Microeconomics - Observed inference time (s)": "{\"description\": \"min=0.135, mean=0.135, max=0.135, sum=0.271 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.1353235294117648\"}", - "High School Physics - Observed inference time (s)": "{\"description\": \"min=0.137, mean=0.137, max=0.137, sum=0.274 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.13686754966887416\"}", - "High School Psychology - Observed inference time (s)": "{\"description\": \"min=0.136, mean=0.136, max=0.136, sum=0.272 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.13622018348623863\"}", - "High School Statistics - Observed inference time (s)": "{\"description\": \"min=0.143, mean=0.143, max=0.143, sum=0.286 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.14287499999999997\"}", - "High School US History - Observed inference time (s)": "{\"description\": \"min=0.158, mean=0.158, max=0.158, sum=0.317 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.15845098039215685\"}", - "High School World History - Observed inference time (s)": "{\"description\": \"min=0.152, mean=0.152, max=0.152, sum=0.304 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.151776371308017\"}", - "High School Biology - # eval": "{\"description\": \"min=310, mean=310, max=310, sum=620 (2)\", \"tab\": \"General information\", \"score\": \"310.0\"}", - "High School Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Biology - # prompt tokens": "{\"description\": \"min=568.748, mean=568.748, max=568.748, sum=1137.497 (2)\", \"tab\": \"General information\", \"score\": \"568.7483870967742\"}", - "High School Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Chemistry - # eval": "{\"description\": \"min=203, mean=203, max=203, sum=406 (2)\", \"tab\": \"General information\", \"score\": \"203.0\"}", - "High School Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # prompt tokens": "{\"description\": \"min=523.65, mean=523.65, max=523.65, sum=1047.3 (2)\", \"tab\": \"General information\", \"score\": \"523.6502463054187\"}", - "High School Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "High School Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # prompt tokens": "{\"description\": \"min=932.15, mean=932.15, max=932.15, sum=1864.3 (2)\", \"tab\": \"General information\", \"score\": \"932.15\"}", - "High School Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School European History - # eval": "{\"description\": \"min=165, mean=165, max=165, sum=330 (2)\", \"tab\": \"General information\", \"score\": \"165.0\"}", - "High School European History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School European History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # prompt tokens": "{\"description\": \"min=2872.03, mean=2872.03, max=2872.03, sum=5744.061 (2)\", \"tab\": \"General information\", \"score\": \"2872.030303030303\"}", - "High School European History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Geography - # eval": "{\"description\": \"min=198, mean=198, max=198, sum=396 (2)\", \"tab\": \"General information\", \"score\": \"198.0\"}", - "High School Geography - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Geography - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # prompt tokens": "{\"description\": \"min=425.646, mean=425.646, max=425.646, sum=851.293 (2)\", \"tab\": \"General information\", \"score\": \"425.64646464646466\"}", - "High School Geography - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Government And Politics - # eval": "{\"description\": \"min=193, mean=193, max=193, sum=386 (2)\", \"tab\": \"General information\", \"score\": \"193.0\"}", - "High School Government And Politics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Government And Politics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # prompt tokens": "{\"description\": \"min=506.073, mean=506.073, max=506.073, sum=1012.145 (2)\", \"tab\": \"General information\", \"score\": \"506.07253886010363\"}", - "High School Government And Politics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Macroeconomics - # eval": "{\"description\": \"min=390, mean=390, max=390, sum=780 (2)\", \"tab\": \"General information\", \"score\": \"390.0\"}", - "High School Macroeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Macroeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # prompt tokens": "{\"description\": \"min=419.987, mean=419.987, max=419.987, sum=839.974 (2)\", \"tab\": \"General information\", \"score\": \"419.9871794871795\"}", - "High School Macroeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Mathematics - # eval": "{\"description\": \"min=270, mean=270, max=270, sum=540 (2)\", \"tab\": \"General information\", \"score\": \"270.0\"}", - "High School Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # prompt tokens": "{\"description\": \"min=554.352, mean=554.352, max=554.352, sum=1108.704 (2)\", \"tab\": \"General information\", \"score\": \"554.3518518518518\"}", - "High School Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Microeconomics - # eval": "{\"description\": \"min=238, mean=238, max=238, sum=476 (2)\", \"tab\": \"General information\", \"score\": \"238.0\"}", - "High School Microeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Microeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # prompt tokens": "{\"description\": \"min=439.055, mean=439.055, max=439.055, sum=878.109 (2)\", \"tab\": \"General information\", \"score\": \"439.0546218487395\"}", - "High School Microeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Physics - # eval": "{\"description\": \"min=151, mean=151, max=151, sum=302 (2)\", \"tab\": \"General information\", \"score\": \"151.0\"}", - "High School Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # prompt tokens": "{\"description\": \"min=581.669, mean=581.669, max=581.669, sum=1163.338 (2)\", \"tab\": \"General information\", \"score\": \"581.6688741721854\"}", - "High School Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Psychology - # eval": "{\"description\": \"min=545, mean=545, max=545, sum=1090 (2)\", \"tab\": \"General information\", \"score\": \"545.0\"}", - "High School Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # prompt tokens": "{\"description\": \"min=544.842, mean=544.842, max=544.842, sum=1089.684 (2)\", \"tab\": \"General information\", \"score\": \"544.8422018348624\"}", - "High School Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Statistics - # eval": "{\"description\": \"min=216, mean=216, max=216, sum=432 (2)\", \"tab\": \"General information\", \"score\": \"216.0\"}", - "High School Statistics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Statistics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # prompt tokens": "{\"description\": \"min=833, mean=833, max=833, sum=1666 (2)\", \"tab\": \"General information\", \"score\": \"833.0\"}", - "High School Statistics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School US History - # eval": "{\"description\": \"min=204, mean=204, max=204, sum=408 (2)\", \"tab\": \"General information\", \"score\": \"204.0\"}", - "High School US History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School US History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # prompt tokens": "{\"description\": \"min=2270.25, mean=2270.25, max=2270.25, sum=4540.5 (2)\", \"tab\": \"General information\", \"score\": \"2270.25\"}", - "High School US History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School World History - # eval": "{\"description\": \"min=237, mean=237, max=237, sum=474 (2)\", \"tab\": \"General information\", \"score\": \"237.0\"}", - "High School World History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School World History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # prompt tokens": "{\"description\": \"min=1466.561, mean=1466.561, max=1466.561, sum=2933.122 (2)\", \"tab\": \"General information\", \"score\": \"1466.5611814345991\"}", - "High School World History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"high_school_world_history\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_high_school_world_history\"" - } - } - }, - { - "evaluation_name": "Human Sexuality", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Human Sexuality", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.885, - "details": { - "description": "min=0.885, mean=0.885, max=0.885, sum=1.771 (2)", - "tab": "Accuracy", - "Human Aging - Observed inference time (s)": "{\"description\": \"min=0.128, mean=0.128, max=0.128, sum=0.257 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.12830044843049326\"}", - "Human Sexuality - Observed inference time (s)": "{\"description\": \"min=0.132, mean=0.132, max=0.132, sum=0.263 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.13163358778625955\"}", - "Human Aging - # eval": "{\"description\": \"min=223, mean=223, max=223, sum=446 (2)\", \"tab\": \"General information\", \"score\": \"223.0\"}", - "Human Aging - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Aging - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Aging - # prompt tokens": "{\"description\": \"min=352.48, mean=352.48, max=352.48, sum=704.96 (2)\", \"tab\": \"General information\", \"score\": \"352.47982062780267\"}", - "Human Aging - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Human Sexuality - # eval": "{\"description\": \"min=131, mean=131, max=131, sum=262 (2)\", \"tab\": \"General information\", \"score\": \"131.0\"}", - "Human Sexuality - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Sexuality - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # prompt tokens": "{\"description\": \"min=385.626, mean=385.626, max=385.626, sum=771.252 (2)\", \"tab\": \"General information\", \"score\": \"385.62595419847327\"}", - "Human Sexuality - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"human_sexuality\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_human_sexuality\"" - } - } - }, - { - "evaluation_name": "International Law", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on International Law", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.901, - "details": { - "description": "min=0.901, mean=0.901, max=0.901, sum=1.802 (2)", - "tab": "Accuracy", - "International Law - Observed inference time (s)": "{\"description\": \"min=0.139, mean=0.139, max=0.139, sum=0.277 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.13855371900826452\"}", - "International Law - # eval": "{\"description\": \"min=121, mean=121, max=121, sum=242 (2)\", \"tab\": \"General information\", \"score\": \"121.0\"}", - "International Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "International Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "International Law - # prompt tokens": "{\"description\": \"min=667.843, mean=667.843, max=667.843, sum=1335.686 (2)\", \"tab\": \"General information\", \"score\": \"667.8429752066115\"}", - "International Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"international_law\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_international_law\"" - } - } - }, - { - "evaluation_name": "Logical Fallacies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Logical Fallacies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.871, - "details": { - "description": "min=0.871, mean=0.871, max=0.871, sum=1.742 (2)", - "tab": "Accuracy", - "Logical Fallacies - Observed inference time (s)": "{\"description\": \"min=0.136, mean=0.136, max=0.136, sum=0.272 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.13612269938650304\"}", - "Logical Fallacies - # eval": "{\"description\": \"min=163, mean=163, max=163, sum=326 (2)\", \"tab\": \"General information\", \"score\": \"163.0\"}", - "Logical Fallacies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Logical Fallacies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Logical Fallacies - # prompt tokens": "{\"description\": \"min=482.227, mean=482.227, max=482.227, sum=964.454 (2)\", \"tab\": \"General information\", \"score\": \"482.2269938650307\"}", - "Logical Fallacies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"logical_fallacies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_logical_fallacies\"" - } - } - }, - { - "evaluation_name": "Machine Learning", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Machine Learning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.625, - "details": { - "description": "min=0.625, mean=0.625, max=0.625, sum=1.25 (2)", - "tab": "Accuracy", - "Machine Learning - Observed inference time (s)": "{\"description\": \"min=0.142, mean=0.142, max=0.142, sum=0.284 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.14183035714285702\"}", - "Machine Learning - # eval": "{\"description\": \"min=112, mean=112, max=112, sum=224 (2)\", \"tab\": \"General information\", \"score\": \"112.0\"}", - "Machine Learning - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Machine Learning - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Machine Learning - # prompt tokens": "{\"description\": \"min=699.598, mean=699.598, max=699.598, sum=1399.196 (2)\", \"tab\": \"General information\", \"score\": \"699.5982142857143\"}", - "Machine Learning - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"machine_learning\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_machine_learning\"" - } - } - }, - { - "evaluation_name": "Management", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Management", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.922, - "details": { - "description": "min=0.922, mean=0.922, max=0.922, sum=1.845 (2)", - "tab": "Accuracy", - "Management - Observed inference time (s)": "{\"description\": \"min=0.129, mean=0.129, max=0.129, sum=0.257 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.12854368932038837\"}", - "Management - # eval": "{\"description\": \"min=103, mean=103, max=103, sum=206 (2)\", \"tab\": \"General information\", \"score\": \"103.0\"}", - "Management - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Management - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Management - # prompt tokens": "{\"description\": \"min=320.34, mean=320.34, max=320.34, sum=640.68 (2)\", \"tab\": \"General information\", \"score\": \"320.3398058252427\"}", - "Management - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"management\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_management\"" - } - } - }, - { - "evaluation_name": "Marketing", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Marketing", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.923, - "details": { - "description": "min=0.923, mean=0.923, max=0.923, sum=1.846 (2)", - "tab": "Accuracy", - "Marketing - Observed inference time (s)": "{\"description\": \"min=0.132, mean=0.132, max=0.132, sum=0.264 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.13224786324786314\"}", - "Marketing - # eval": "{\"description\": \"min=234, mean=234, max=234, sum=468 (2)\", \"tab\": \"General information\", \"score\": \"234.0\"}", - "Marketing - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Marketing - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Marketing - # prompt tokens": "{\"description\": \"min=466.697, mean=466.697, max=466.697, sum=933.393 (2)\", \"tab\": \"General information\", \"score\": \"466.6965811965812\"}", - "Marketing - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"marketing\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_marketing\"" - } - } - }, - { - "evaluation_name": "Medical Genetics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Medical Genetics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.87, - "details": { - "description": "min=0.87, mean=0.87, max=0.87, sum=1.74 (2)", - "tab": "Accuracy", - "Medical Genetics - Observed inference time (s)": "{\"description\": \"min=0.133, mean=0.133, max=0.133, sum=0.266 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.13288\"}", - "Medical Genetics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Medical Genetics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Medical Genetics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Medical Genetics - # prompt tokens": "{\"description\": \"min=380.71, mean=380.71, max=380.71, sum=761.42 (2)\", \"tab\": \"General information\", \"score\": \"380.71\"}", - "Medical Genetics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"medical_genetics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_medical_genetics\"" - } - } - }, - { - "evaluation_name": "Miscellaneous", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Miscellaneous", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.912, - "details": { - "description": "min=0.912, mean=0.912, max=0.912, sum=1.824 (2)", - "tab": "Accuracy", - "Miscellaneous - Observed inference time (s)": "{\"description\": \"min=0.129, mean=0.129, max=0.129, sum=0.257 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.12866538952745835\"}", - "Miscellaneous - # eval": "{\"description\": \"min=783, mean=783, max=783, sum=1566 (2)\", \"tab\": \"General information\", \"score\": \"783.0\"}", - "Miscellaneous - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Miscellaneous - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Miscellaneous - # prompt tokens": "{\"description\": \"min=342.847, mean=342.847, max=342.847, sum=685.693 (2)\", \"tab\": \"General information\", \"score\": \"342.84674329501917\"}", - "Miscellaneous - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"miscellaneous\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_miscellaneous\"" - } - } - }, - { - "evaluation_name": "Moral Scenarios", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Moral Scenarios", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.76, - "details": { - "description": "min=0.76, mean=0.76, max=0.76, sum=1.52 (2)", - "tab": "Accuracy", - "Moral Disputes - Observed inference time (s)": "{\"description\": \"min=0.135, mean=0.135, max=0.135, sum=0.27 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.1350173410404623\"}", - "Moral Scenarios - Observed inference time (s)": "{\"description\": \"min=0.138, mean=0.138, max=0.138, sum=0.277 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.13844581005586606\"}", - "Moral Disputes - # eval": "{\"description\": \"min=346, mean=346, max=346, sum=692 (2)\", \"tab\": \"General information\", \"score\": \"346.0\"}", - "Moral Disputes - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Disputes - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Disputes - # prompt tokens": "{\"description\": \"min=525.329, mean=525.329, max=525.329, sum=1050.659 (2)\", \"tab\": \"General information\", \"score\": \"525.3294797687861\"}", - "Moral Disputes - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Moral Scenarios - # eval": "{\"description\": \"min=895, mean=895, max=895, sum=1790 (2)\", \"tab\": \"General information\", \"score\": \"895.0\"}", - "Moral Scenarios - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Scenarios - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # prompt tokens": "{\"description\": \"min=692.482, mean=692.482, max=692.482, sum=1384.963 (2)\", \"tab\": \"General information\", \"score\": \"692.4815642458101\"}", - "Moral Scenarios - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"moral_scenarios\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_moral_scenarios\"" - } - } - }, - { - "evaluation_name": "Nutrition", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Nutrition", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.866, - "details": { - "description": "min=0.866, mean=0.866, max=0.866, sum=1.732 (2)", - "tab": "Accuracy", - "Nutrition - Observed inference time (s)": "{\"description\": \"min=0.135, mean=0.135, max=0.135, sum=0.27 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.13503921568627456\"}", - "Nutrition - # eval": "{\"description\": \"min=306, mean=306, max=306, sum=612 (2)\", \"tab\": \"General information\", \"score\": \"306.0\"}", - "Nutrition - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Nutrition - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Nutrition - # prompt tokens": "{\"description\": \"min=612.69, mean=612.69, max=612.69, sum=1225.379 (2)\", \"tab\": \"General information\", \"score\": \"612.6895424836601\"}", - "Nutrition - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"nutrition\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_nutrition\"" - } - } - }, - { - "evaluation_name": "Prehistory", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Prehistory", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.926, - "details": { - "description": "min=0.926, mean=0.926, max=0.926, sum=1.852 (2)", - "tab": "Accuracy", - "Prehistory - Observed inference time (s)": "{\"description\": \"min=0.135, mean=0.135, max=0.135, sum=0.271 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.135388888888889\"}", - "Prehistory - # eval": "{\"description\": \"min=324, mean=324, max=324, sum=648 (2)\", \"tab\": \"General information\", \"score\": \"324.0\"}", - "Prehistory - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Prehistory - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Prehistory - # prompt tokens": "{\"description\": \"min=552.454, mean=552.454, max=552.454, sum=1104.907 (2)\", \"tab\": \"General information\", \"score\": \"552.4537037037037\"}", - "Prehistory - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"prehistory\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_prehistory\"" - } - } - }, - { - "evaluation_name": "Public Relations", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Public Relations", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8, - "details": { - "description": "min=0.8, mean=0.8, max=0.8, sum=1.6 (2)", - "tab": "Accuracy", - "Public Relations - Observed inference time (s)": "{\"description\": \"min=0.132, mean=0.132, max=0.132, sum=0.265 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.13249090909090908\"}", - "Public Relations - # eval": "{\"description\": \"min=110, mean=110, max=110, sum=220 (2)\", \"tab\": \"General information\", \"score\": \"110.0\"}", - "Public Relations - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Public Relations - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Public Relations - # prompt tokens": "{\"description\": \"min=448.609, mean=448.609, max=448.609, sum=897.218 (2)\", \"tab\": \"General information\", \"score\": \"448.6090909090909\"}", - "Public Relations - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"public_relations\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_public_relations\"" - } - } - }, - { - "evaluation_name": "Security Studies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Security Studies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.849, - "details": { - "description": "min=0.849, mean=0.849, max=0.849, sum=1.698 (2)", - "tab": "Accuracy", - "Security Studies - Observed inference time (s)": "{\"description\": \"min=0.143, mean=0.143, max=0.143, sum=0.285 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.1427142857142858\"}", - "Security Studies - # eval": "{\"description\": \"min=245, mean=245, max=245, sum=490 (2)\", \"tab\": \"General information\", \"score\": \"245.0\"}", - "Security Studies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Security Studies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Security Studies - # prompt tokens": "{\"description\": \"min=1224.433, mean=1224.433, max=1224.433, sum=2448.865 (2)\", \"tab\": \"General information\", \"score\": \"1224.4326530612245\"}", - "Security Studies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"security_studies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_security_studies\"" - } - } - }, - { - "evaluation_name": "Sociology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Sociology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.905, - "details": { - "description": "min=0.905, mean=0.905, max=0.905, sum=1.811 (2)", - "tab": "Accuracy", - "Sociology - Observed inference time (s)": "{\"description\": \"min=0.137, mean=0.137, max=0.137, sum=0.275 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.13738308457711446\"}", - "Sociology - # eval": "{\"description\": \"min=201, mean=201, max=201, sum=402 (2)\", \"tab\": \"General information\", \"score\": \"201.0\"}", - "Sociology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Sociology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Sociology - # prompt tokens": "{\"description\": \"min=474.512, mean=474.512, max=474.512, sum=949.025 (2)\", \"tab\": \"General information\", \"score\": \"474.5124378109453\"}", - "Sociology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"sociology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_sociology\"" - } - } - }, - { - "evaluation_name": "Virology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Virology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.59, - "details": { - "description": "min=0.59, mean=0.59, max=0.59, sum=1.181 (2)", - "tab": "Accuracy", - "Virology - Observed inference time (s)": "{\"description\": \"min=0.129, mean=0.129, max=0.129, sum=0.258 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.1290301204819277\"}", - "Virology - # eval": "{\"description\": \"min=166, mean=166, max=166, sum=332 (2)\", \"tab\": \"General information\", \"score\": \"166.0\"}", - "Virology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Virology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Virology - # prompt tokens": "{\"description\": \"min=380.753, mean=380.753, max=380.753, sum=761.506 (2)\", \"tab\": \"General information\", \"score\": \"380.7530120481928\"}", - "Virology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"virology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_virology\"" - } - } - }, - { - "evaluation_name": "World Religions", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on World Religions", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.877, - "details": { - "description": "min=0.877, mean=0.877, max=0.877, sum=1.754 (2)", - "tab": "Accuracy", - "World Religions - Observed inference time (s)": "{\"description\": \"min=0.128, mean=0.128, max=0.128, sum=0.257 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.12828070175438594\"}", - "World Religions - # eval": "{\"description\": \"min=171, mean=171, max=171, sum=342 (2)\", \"tab\": \"General information\", \"score\": \"171.0\"}", - "World Religions - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "World Religions - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "World Religions - # prompt tokens": "{\"description\": \"min=305.386, mean=305.386, max=305.386, sum=610.772 (2)\", \"tab\": \"General information\", \"score\": \"305.3859649122807\"}", - "World Religions - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"world_religions\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_world_religions\"" - } - } - }, - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.975, - "details": { - "description": "", - "tab": "Efficiency" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/amd_amd-llama-135m.json b/data/models/amd_amd-llama-135m.json deleted file mode 100644 index dfdcf8e7a431ddc8bdf51b6aa797b81cf0f89c9e..0000000000000000000000000000000000000000 --- a/data/models/amd_amd-llama-135m.json +++ /dev/null @@ -1,275 +0,0 @@ -{ - "model_info": { - "name": "AMD-Llama-135m", - "id": "amd/AMD-Llama-135m", - "developer": "amd", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "0.134" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/amd_AMD-Llama-135m/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1918 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2969 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0076 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2584 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3846 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1169 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "hfopenllm_v2/amd_AMD-Llama-135m/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1842 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2974 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0053 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2525 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.378 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1169 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/amu_t1-1.5b.json b/data/models/amu_t1-1.5b.json deleted file mode 100644 index 2e2f0978f33e9a43d5dd4314c46a98f351564a4e..0000000000000000000000000000000000000000 --- a/data/models/amu_t1-1.5b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "t1-1.5B", - "id": "Amu/t1-1.5B", - "developer": "Amu", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.777" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Amu_t1-1.5B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3394 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4008 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0514 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2433 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3517 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2566 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/amu_t1-3b.json b/data/models/amu_t1-3b.json deleted file mode 100644 index 2f9960da52d7b1b89232c403d0981cb190cba8cf..0000000000000000000000000000000000000000 --- a/data/models/amu_t1-3b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "t1-3B", - "id": "Amu/t1-3B", - "developer": "Amu", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.397" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Amu_t1-3B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3328 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3999 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1375 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2408 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3435 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1284 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/anakin87_gemma-2b-orpo.json b/data/models/anakin87_gemma-2b-orpo.json deleted file mode 100644 index eb25df79877defc20aa0d5dfd14d8238e4fbed56..0000000000000000000000000000000000000000 --- a/data/models/anakin87_gemma-2b-orpo.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "gemma-2b-orpo", - "id": "anakin87/gemma-2b-orpo", - "developer": "anakin87", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "GemmaForCausalLM", - "params_billions": "2.506" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/anakin87_gemma-2b-orpo/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2478 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3426 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0189 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2617 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3728 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1306 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/anthracite-org_magnum-v1-72b.json b/data/models/anthracite-org_magnum-v1-72b.json deleted file mode 100644 index a2be9b276716b3ab8e555db52f3ae1ac0f7e2c2d..0000000000000000000000000000000000000000 --- a/data/models/anthracite-org_magnum-v1-72b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "magnum-v1-72b", - "id": "anthracite-org/magnum-v1-72b", - "developer": "anthracite-org", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "72.706" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/anthracite-org_magnum-v1-72b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7606 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6982 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.398 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3909 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4489 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5486 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/anthracite-org_magnum-v2-12b.json b/data/models/anthracite-org_magnum-v2-12b.json deleted file mode 100644 index 3e8b445151db6e6e26b3fa1aa1ab80f31c5691a4..0000000000000000000000000000000000000000 --- a/data/models/anthracite-org_magnum-v2-12b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "magnum-v2-12b", - "id": "anthracite-org/magnum-v2-12b", - "developer": "anthracite-org", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/anthracite-org_magnum-v2-12b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3762 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5021 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0544 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2911 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4179 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3167 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/anthracite-org_magnum-v2-72b.json b/data/models/anthracite-org_magnum-v2-72b.json deleted file mode 100644 index 96134f9bb0edc837f19a8b3b6f255233051d4d57..0000000000000000000000000000000000000000 --- a/data/models/anthracite-org_magnum-v2-72b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "magnum-v2-72b", - "id": "anthracite-org/magnum-v2-72b", - "developer": "anthracite-org", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "72.706" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/anthracite-org_magnum-v2-72b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.756 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7005 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3542 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3859 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4372 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5456 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/anthracite-org_magnum-v2.5-12b-kto.json b/data/models/anthracite-org_magnum-v2.5-12b-kto.json deleted file mode 100644 index 4cedd4cd85fd901d503a3406ddba162db502092c..0000000000000000000000000000000000000000 --- a/data/models/anthracite-org_magnum-v2.5-12b-kto.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "magnum-v2.5-12b-kto", - "id": "anthracite-org/magnum-v2.5-12b-kto", - "developer": "anthracite-org", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/anthracite-org_magnum-v2.5-12b-kto/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3866 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5077 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0521 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2936 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4086 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3215 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/anthracite-org_magnum-v3-27b-kto.json b/data/models/anthracite-org_magnum-v3-27b-kto.json deleted file mode 100644 index 898065d90e04139d5069b42284be33fd5e63cba4..0000000000000000000000000000000000000000 --- a/data/models/anthracite-org_magnum-v3-27b-kto.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "magnum-v3-27b-kto", - "id": "anthracite-org/magnum-v3-27b-kto", - "developer": "anthracite-org", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "27.227" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/anthracite-org_magnum-v3-27b-kto/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5675 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.586 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1813 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3557 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3855 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4238 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/anthracite-org_magnum-v3-34b.json b/data/models/anthracite-org_magnum-v3-34b.json deleted file mode 100644 index 368657c968ee3061992108c1d9d7ddbd403691b3..0000000000000000000000000000000000000000 --- a/data/models/anthracite-org_magnum-v3-34b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "magnum-v3-34b", - "id": "anthracite-org/magnum-v3-34b", - "developer": "anthracite-org", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "34.389" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/anthracite-org_magnum-v3-34b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5115 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6088 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1949 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3607 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3872 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4752 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/anthracite-org_magnum-v3-9b-chatml.json b/data/models/anthracite-org_magnum-v3-9b-chatml.json deleted file mode 100644 index 4a19cf684e39492ffbe460680314145ab5108040..0000000000000000000000000000000000000000 --- a/data/models/anthracite-org_magnum-v3-9b-chatml.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "magnum-v3-9b-chatml", - "id": "anthracite-org/magnum-v3-9b-chatml", - "developer": "anthracite-org", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "9.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/anthracite-org_magnum-v3-9b-chatml/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1275 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5428 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0695 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3456 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4432 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4242 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/anthracite-org_magnum-v3-9b-customgemma2.json b/data/models/anthracite-org_magnum-v3-9b-customgemma2.json deleted file mode 100644 index e83bdd73fbe7c0550c99d26b6c8dc89e9aa690a3..0000000000000000000000000000000000000000 --- a/data/models/anthracite-org_magnum-v3-9b-customgemma2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "magnum-v3-9b-customgemma2", - "id": "anthracite-org/magnum-v3-9b-customgemma2", - "developer": "anthracite-org", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "9.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/anthracite-org_magnum-v3-9b-customgemma2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1273 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.534 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0718 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3289 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4565 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4205 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/anthracite-org_magnum-v4-12b.json b/data/models/anthracite-org_magnum-v4-12b.json deleted file mode 100644 index 53c4d129530bc8d3ece65888742e6c2d028cc068..0000000000000000000000000000000000000000 --- a/data/models/anthracite-org_magnum-v4-12b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "magnum-v4-12b", - "id": "anthracite-org/magnum-v4-12b", - "developer": "anthracite-org", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/anthracite-org_magnum-v4-12b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3393 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5177 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1178 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2961 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4093 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3604 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/anthracite-org_magnum-v4-22b.json b/data/models/anthracite-org_magnum-v4-22b.json deleted file mode 100644 index f488b11c3609c792abaa3f49c319c2b0338d14d5..0000000000000000000000000000000000000000 --- a/data/models/anthracite-org_magnum-v4-22b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "magnum-v4-22b", - "id": "anthracite-org/magnum-v4-22b", - "developer": "anthracite-org", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "22.247" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/anthracite-org_magnum-v4-22b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5629 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5486 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2002 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.328 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4408 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.383 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/anthracite-org_magnum-v4-27b.json b/data/models/anthracite-org_magnum-v4-27b.json deleted file mode 100644 index a528c7453e145e6e1254d989d7de9577918f42c3..0000000000000000000000000000000000000000 --- a/data/models/anthracite-org_magnum-v4-27b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "magnum-v4-27b", - "id": "anthracite-org/magnum-v4-27b", - "developer": "anthracite-org", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "27.227" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/anthracite-org_magnum-v4-27b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3454 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5867 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1798 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.37 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.438 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4376 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/anthracite-org_magnum-v4-9b.json b/data/models/anthracite-org_magnum-v4-9b.json deleted file mode 100644 index 4cf5cde8dfa0cef4055d3434457bad2c32fccbc0..0000000000000000000000000000000000000000 --- a/data/models/anthracite-org_magnum-v4-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "magnum-v4-9b", - "id": "anthracite-org/magnum-v4-9b", - "developer": "anthracite-org", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "9.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/anthracite-org_magnum-v4-9b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3503 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5336 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1307 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3473 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4516 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3953 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/anthropic-lm-v4-s3-52b.json b/data/models/anthropic-lm-v4-s3-52b.json deleted file mode 100644 index 2c3bcd01427a5406c12a8c41517bd1b88d3cc742..0000000000000000000000000000000000000000 --- a/data/models/anthropic-lm-v4-s3-52b.json +++ /dev/null @@ -1,674 +0,0 @@ -{ - "model_info": { - "name": "Anthropic-LM v4-s3 52B", - "id": "Anthropic-LM-v4-s3-52B", - "developer": "unknown", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_classic/Anthropic-LM-v4-s3-52B/1774096308.339228", - "retrieved_timestamp": "1774096308.339228", - "source_metadata": { - "source_name": "helm_classic", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_classic", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_classic", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperform on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.78, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Calibration": "{\"description\": \"\", \"tab\": \"Calibration\", \"score\": \"\"}", - "Mean win rate - Robustness": "{\"description\": \"\", \"tab\": \"Robustness\", \"score\": \"0.8178973356392711\"}", - "Mean win rate - Fairness": "{\"description\": \"\", \"tab\": \"Fairness\", \"score\": \"0.7935577862997218\"}", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.13822916666666668\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}", - "Mean win rate - Bias": "{\"description\": \"\", \"tab\": \"Bias\", \"score\": \"0.5930298633071189\"}", - "Mean win rate - Toxicity": "{\"description\": \"\", \"tab\": \"Toxicity\", \"score\": \"0.648748165414832\"}", - "Mean win rate - Summarization metrics": "{\"description\": \"\", \"tab\": \"Summarization metrics\", \"score\": \"0.5306599832915623\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.481, - "details": { - "description": "min=0.25, mean=0.481, max=0.78, sum=7.22 (15)", - "tab": "Accuracy", - "MMLU - ECE (10-bin)": "{\"description\": \"min=0.063, mean=0.144, max=0.262, sum=2.165 (15)\", \"tab\": \"Calibration\", \"score\": \"\"}", - "MMLU - EM (Robustness)": "{\"description\": \"min=0.17, mean=0.434, max=0.76, sum=6.513 (15)\", \"tab\": \"Robustness\", \"score\": \"0.43421052631578944\"}", - "MMLU - EM (Fairness)": "{\"description\": \"min=0.211, mean=0.447, max=0.74, sum=6.702 (15)\", \"tab\": \"Fairness\", \"score\": \"0.4467836257309941\"}", - "MMLU - Denoised inference time (s)": "{\"description\": \"min=0.556, mean=0.578, max=0.605, sum=8.664 (15)\", \"tab\": \"Efficiency\", \"score\": \"0.5775741999040572\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=1542 (15)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=75 (15)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (15)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=371.38, mean=472.274, max=624.07, sum=7084.111 (15)\", \"tab\": \"General information\", \"score\": \"472.2740350877193\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=15 (15)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "MMLU - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=45 (15)\", \"tab\": \"General information\", \"score\": \"3.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "BoolQ", - "source_data": { - "dataset_name": "BoolQ", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on BoolQ", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.815, - "details": { - "description": "min=0.814, mean=0.815, max=0.816, sum=2.446 (3)", - "tab": "Accuracy", - "BoolQ - ECE (10-bin)": "{\"description\": \"min=0.035, mean=0.038, max=0.041, sum=0.114 (3)\", \"tab\": \"Calibration\", \"score\": \"\"}", - "BoolQ - EM (Robustness)": "{\"description\": \"min=0.751, mean=0.756, max=0.76, sum=2.269 (3)\", \"tab\": \"Robustness\", \"score\": \"0.7563333333333334\"}", - "BoolQ - EM (Fairness)": "{\"description\": \"min=0.778, mean=0.782, max=0.788, sum=2.345 (3)\", \"tab\": \"Fairness\", \"score\": \"0.7816666666666667\"}", - "BoolQ - Denoised inference time (s)": "{\"description\": \"min=0.566, mean=0.637, max=0.75, sum=1.912 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.6371923081597224\"}", - "BoolQ - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "BoolQ - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "BoolQ - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "BoolQ - # prompt tokens": "{\"description\": \"min=660.073, mean=908.406, max=1242.073, sum=2725.219 (3)\", \"tab\": \"General information\", \"score\": \"908.4063333333334\"}", - "BoolQ - # output tokens": "{\"description\": \"min=1.004, mean=1.004, max=1.004, sum=3.012 (3)\", \"tab\": \"General information\", \"score\": \"1.004\"}", - "BoolQ - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "BoolQ - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.728, - "details": { - "description": "min=0.692, mean=0.728, max=0.748, sum=2.185 (3)", - "tab": "Accuracy", - "NarrativeQA - ECE (10-bin)": "{\"description\": \"min=0.052, mean=0.09, max=0.14, sum=0.27 (3)\", \"tab\": \"Calibration\", \"score\": \"\"}", - "NarrativeQA - F1 (Robustness)": "{\"description\": \"min=0.622, mean=0.663, max=0.693, sum=1.99 (3)\", \"tab\": \"Robustness\", \"score\": \"0.6634443166549867\"}", - "NarrativeQA - F1 (Fairness)": "{\"description\": \"min=0.614, mean=0.646, max=0.667, sum=1.939 (3)\", \"tab\": \"Fairness\", \"score\": \"0.6464650190039823\"}", - "NarrativeQA - Denoised inference time (s)": "{\"description\": \"min=1.628, mean=1.722, max=1.839, sum=5.167 (3)\", \"tab\": \"Efficiency\", \"score\": \"1.7223421043622853\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=1065 (3)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=3504.577, mean=3803.911, max=3972.577, sum=11411.732 (3)\", \"tab\": \"General information\", \"score\": \"3803.910798122066\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=4.572, mean=6.952, max=8.434, sum=20.856 (3)\", \"tab\": \"General information\", \"score\": \"6.9521126760563385\"}", - "NarrativeQA - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NarrativeQA - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NarrativeQA - Stereotypes (gender)": "{\"description\": \"min=0.333, mean=0.39, max=0.419, sum=1.169 (3)\", \"tab\": \"Bias\", \"score\": \"0.38950617283950617\"}", - "NarrativeQA - Representation (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=0.667 (1)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666667\"}", - "NarrativeQA - Representation (gender)": "{\"description\": \"min=0.19, mean=0.208, max=0.218, sum=0.624 (3)\", \"tab\": \"Bias\", \"score\": \"0.20792828096614854\"}", - "NarrativeQA - Toxic fraction": "{\"description\": \"min=0.011, mean=0.013, max=0.014, sum=0.039 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.013145539906103287\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (open-book)", - "source_data": { - "dataset_name": "NaturalQuestions (open-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (open-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.686, - "details": { - "description": "min=0.682, mean=0.686, max=0.693, sum=2.059 (3)", - "tab": "Accuracy", - "NaturalQuestions (closed-book) - ECE (10-bin)": "{\"description\": \"min=0.108, mean=0.121, max=0.128, sum=0.362 (3)\", \"tab\": \"Calibration\", \"score\": \"\"}", - "NaturalQuestions (open-book) - ECE (10-bin)": "{\"description\": \"min=0.048, mean=0.067, max=0.088, sum=0.2 (3)\", \"tab\": \"Calibration\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - F1 (Robustness)": "{\"description\": \"min=0.239, mean=0.245, max=0.248, sum=0.734 (3)\", \"tab\": \"Robustness\", \"score\": \"0.24480135198778494\"}", - "NaturalQuestions (open-book) - F1 (Robustness)": "{\"description\": \"min=0.608, mean=0.632, max=0.646, sum=1.897 (3)\", \"tab\": \"Robustness\", \"score\": \"0.6323821508652113\"}", - "NaturalQuestions (closed-book) - F1 (Fairness)": "{\"description\": \"min=0.228, mean=0.239, max=0.244, sum=0.716 (3)\", \"tab\": \"Fairness\", \"score\": \"0.23855278160903723\"}", - "NaturalQuestions (open-book) - F1 (Fairness)": "{\"description\": \"min=0.639, mean=0.642, max=0.646, sum=1.927 (3)\", \"tab\": \"Fairness\", \"score\": \"0.6422159112855447\"}", - "NaturalQuestions (closed-book) - Denoised inference time (s)": "{\"description\": \"min=0.751, mean=0.777, max=0.821, sum=2.331 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.7770150703124993\"}", - "NaturalQuestions (open-book) - Denoised inference time (s)": "{\"description\": \"min=1.036, mean=1.102, max=1.15, sum=3.305 (3)\", \"tab\": \"Efficiency\", \"score\": \"1.1015715911458346\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=110.254, mean=112.254, max=116.254, sum=336.762 (3)\", \"tab\": \"General information\", \"score\": \"112.254\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=5.032, mean=5.47, max=6.183, sum=16.409 (3)\", \"tab\": \"General information\", \"score\": \"5.469666666666666\"}", - "NaturalQuestions (closed-book) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.964, mean=4.964, max=4.965, sum=14.893 (3)\", \"tab\": \"General information\", \"score\": \"4.964333333333333\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.007, mean=0.007, max=0.007, sum=0.021 (3)\", \"tab\": \"General information\", \"score\": \"0.007\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1381.066, mean=1592.701, max=1704.681, sum=4778.103 (3)\", \"tab\": \"General information\", \"score\": \"1592.701\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=5.429, mean=5.659, max=6.028, sum=16.976 (3)\", \"tab\": \"General information\", \"score\": \"5.658666666666666\"}", - "NaturalQuestions (open-book) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NaturalQuestions (closed-book) - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - Stereotypes (gender)": "{\"description\": \"min=0.5, mean=0.5, max=0.5, sum=1.5 (3)\", \"tab\": \"Bias\", \"score\": \"0.5\"}", - "NaturalQuestions (closed-book) - Representation (race)": "{\"description\": \"min=0.358, mean=0.386, max=0.439, sum=1.158 (3)\", \"tab\": \"Bias\", \"score\": \"0.38616369646117926\"}", - "NaturalQuestions (closed-book) - Representation (gender)": "{\"description\": \"min=0, mean=0.148, max=0.237, sum=0.443 (3)\", \"tab\": \"Bias\", \"score\": \"0.1475748194014448\"}", - "NaturalQuestions (open-book) - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NaturalQuestions (open-book) - Stereotypes (gender)": "{\"description\": \"min=0.367, mean=0.429, max=0.5, sum=1.287 (3)\", \"tab\": \"Bias\", \"score\": \"0.4288888888888889\"}", - "NaturalQuestions (open-book) - Representation (race)": "{\"description\": \"min=0.459, mean=0.48, max=0.498, sum=1.441 (3)\", \"tab\": \"Bias\", \"score\": \"0.48032222577096423\"}", - "NaturalQuestions (open-book) - Representation (gender)": "{\"description\": \"min=0.024, mean=0.043, max=0.079, sum=0.129 (3)\", \"tab\": \"Bias\", \"score\": \"0.043024227234753555\"}", - "NaturalQuestions (closed-book) - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}", - "NaturalQuestions (open-book) - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "QuAC", - "source_data": { - "dataset_name": "QuAC", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on QuAC", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.431, - "details": { - "description": "min=0.41, mean=0.431, max=0.443, sum=1.294 (3)", - "tab": "Accuracy", - "QuAC - ECE (10-bin)": "{\"description\": \"min=0.034, mean=0.039, max=0.048, sum=0.116 (3)\", \"tab\": \"Calibration\", \"score\": \"\"}", - "QuAC - F1 (Robustness)": "{\"description\": \"min=0.303, mean=0.313, max=0.324, sum=0.938 (3)\", \"tab\": \"Robustness\", \"score\": \"0.31252831855461766\"}", - "QuAC - F1 (Fairness)": "{\"description\": \"min=0.338, mean=0.356, max=0.365, sum=1.067 (3)\", \"tab\": \"Fairness\", \"score\": \"0.35555313427706087\"}", - "QuAC - Denoised inference time (s)": "{\"description\": \"min=3.472, mean=3.694, max=4.123, sum=11.082 (3)\", \"tab\": \"Efficiency\", \"score\": \"3.6939938854166683\"}", - "QuAC - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "QuAC - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "QuAC - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "QuAC - # prompt tokens": "{\"description\": \"min=4676.788, mean=5199.788, max=5842.788, sum=15599.364 (3)\", \"tab\": \"General information\", \"score\": \"5199.788\"}", - "QuAC - # output tokens": "{\"description\": \"min=32.106, mean=35.484, max=40.222, sum=106.452 (3)\", \"tab\": \"General information\", \"score\": \"35.484\"}", - "QuAC - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "QuAC - Stereotypes (race)": "{\"description\": \"min=0.604, mean=0.609, max=0.614, sum=1.827 (3)\", \"tab\": \"Bias\", \"score\": \"0.6088490550046614\"}", - "QuAC - Stereotypes (gender)": "{\"description\": \"min=0.405, mean=0.419, max=0.441, sum=1.257 (3)\", \"tab\": \"Bias\", \"score\": \"0.4190730790466706\"}", - "QuAC - Representation (race)": "{\"description\": \"min=0.283, mean=0.321, max=0.341, sum=0.964 (3)\", \"tab\": \"Bias\", \"score\": \"0.32117266495855845\"}", - "QuAC - Representation (gender)": "{\"description\": \"min=0.246, mean=0.248, max=0.249, sum=0.743 (3)\", \"tab\": \"Bias\", \"score\": \"0.24753349327018945\"}", - "QuAC - Toxic fraction": "{\"description\": \"min=0.001, mean=0.002, max=0.002, sum=0.005 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.0016666666666666668\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "HellaSwag", - "source_data": { - "dataset_name": "HellaSwag", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on HellaSwag", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.807, - "details": { - "description": "min=0.807, mean=0.807, max=0.807, sum=0.807 (1)", - "tab": "Accuracy", - "HellaSwag - ECE (10-bin)": "{\"description\": \"min=0.32, mean=0.32, max=0.32, sum=0.32 (1)\", \"tab\": \"Calibration\", \"score\": \"\"}", - "HellaSwag - EM (Robustness)": "{\"description\": \"min=0.766, mean=0.766, max=0.766, sum=0.766 (1)\", \"tab\": \"Robustness\", \"score\": \"0.766\"}", - "HellaSwag - EM (Fairness)": "{\"description\": \"min=0.695, mean=0.695, max=0.695, sum=0.695 (1)\", \"tab\": \"Fairness\", \"score\": \"0.695\"}", - "HellaSwag - Denoised inference time (s)": "{\"description\": \"min=0.549, mean=0.549, max=0.549, sum=0.549 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.5491151875000004\"}", - "HellaSwag - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "HellaSwag - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "HellaSwag - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "HellaSwag - # prompt tokens": "{\"description\": \"min=87.888, mean=87.888, max=87.888, sum=87.888 (1)\", \"tab\": \"General information\", \"score\": \"87.888\"}", - "HellaSwag - # output tokens": "{\"description\": \"min=1.306, mean=1.306, max=1.306, sum=1.306 (1)\", \"tab\": \"General information\", \"score\": \"1.306\"}", - "HellaSwag - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.558, - "details": { - "description": "min=0.558, mean=0.558, max=0.558, sum=0.558 (1)", - "tab": "Accuracy", - "OpenbookQA - ECE (10-bin)": "{\"description\": \"min=0.244, mean=0.244, max=0.244, sum=0.244 (1)\", \"tab\": \"Calibration\", \"score\": \"\"}", - "OpenbookQA - EM (Robustness)": "{\"description\": \"min=0.472, mean=0.472, max=0.472, sum=0.472 (1)\", \"tab\": \"Robustness\", \"score\": \"0.472\"}", - "OpenbookQA - EM (Fairness)": "{\"description\": \"min=0.482, mean=0.482, max=0.482, sum=0.482 (1)\", \"tab\": \"Fairness\", \"score\": \"0.482\"}", - "OpenbookQA - Denoised inference time (s)": "{\"description\": \"min=0.447, mean=0.447, max=0.447, sum=0.447 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.4465652265625003\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=5.27, mean=5.27, max=5.27, sum=5.27 (1)\", \"tab\": \"General information\", \"score\": \"5.27\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=0.132, mean=0.132, max=0.132, sum=0.132 (1)\", \"tab\": \"General information\", \"score\": \"0.132\"}", - "OpenbookQA - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "TruthfulQA", - "source_data": { - "dataset_name": "TruthfulQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on TruthfulQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.368, - "details": { - "description": "min=0.298, mean=0.368, max=0.408, sum=1.472 (4)", - "tab": "Accuracy", - "TruthfulQA - ECE (10-bin)": "{\"description\": \"min=0.052, mean=0.127, max=0.196, sum=0.507 (4)\", \"tab\": \"Calibration\", \"score\": \"\"}", - "TruthfulQA - EM (Robustness)": "{\"description\": \"min=0.263, mean=0.326, max=0.388, sum=1.304 (4)\", \"tab\": \"Robustness\", \"score\": \"0.3260703363914373\"}", - "TruthfulQA - EM (Fairness)": "{\"description\": \"min=0.229, mean=0.3, max=0.388, sum=1.202 (4)\", \"tab\": \"Fairness\", \"score\": \"0.3004587155963303\"}", - "TruthfulQA - Denoised inference time (s)": "{\"description\": \"min=0.503, mean=0.568, max=0.603, sum=2.273 (4)\", \"tab\": \"Efficiency\", \"score\": \"0.5683649633565078\"}", - "TruthfulQA - # eval": "{\"description\": \"min=654, mean=654, max=654, sum=2616 (4)\", \"tab\": \"General information\", \"score\": \"654.0\"}", - "TruthfulQA - # train": "{\"description\": \"min=0, mean=3.75, max=5, sum=15 (4)\", \"tab\": \"General information\", \"score\": \"3.75\"}", - "TruthfulQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (4)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "TruthfulQA - # prompt tokens": "{\"description\": \"min=85.121, mean=404.621, max=529.121, sum=1618.483 (4)\", \"tab\": \"General information\", \"score\": \"404.62079510703364\"}", - "TruthfulQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=4 (4)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "TruthfulQA - # trials": "{\"description\": \"min=1, mean=2.5, max=3, sum=10 (4)\", \"tab\": \"General information\", \"score\": \"2.5\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MS MARCO (TREC)", - "source_data": { - "dataset_name": "MS MARCO (TREC)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "NDCG@10 on MS MARCO (TREC)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "min=0.625, mean=0.642, max=0.66, sum=1.925 (3)", - "tab": "Accuracy", - "MS MARCO (regular) - RR@10 (Robustness)": "{\"description\": \"min=0.28, mean=0.308, max=0.326, sum=0.925 (3)\", \"tab\": \"Robustness\", \"score\": \"\"}", - "MS MARCO (TREC) - NDCG@10 (Robustness)": "{\"description\": \"min=0.589, mean=0.592, max=0.594, sum=1.776 (3)\", \"tab\": \"Robustness\", \"score\": \"\"}", - "MS MARCO (regular) - RR@10 (Fairness)": "{\"description\": \"min=0.305, mean=0.345, max=0.369, sum=1.036 (3)\", \"tab\": \"Fairness\", \"score\": \"\"}", - "MS MARCO (TREC) - NDCG@10 (Fairness)": "{\"description\": \"min=0.589, mean=0.609, max=0.63, sum=1.828 (3)\", \"tab\": \"Fairness\", \"score\": \"\"}", - "MS MARCO (regular) - Denoised inference time (s)": "{\"description\": \"min=0.55, mean=0.578, max=0.599, sum=1.733 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.5778111061197916\"}", - "MS MARCO (TREC) - Denoised inference time (s)": "{\"description\": \"min=0.547, mean=0.587, max=0.608, sum=1.76 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.5865037397044573\"}", - "MS MARCO (regular) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "MS MARCO (regular) - # train": "{\"description\": \"min=2, mean=2, max=2, sum=6 (3)\", \"tab\": \"General information\", \"score\": \"2.0\"}", - "MS MARCO (regular) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MS MARCO (regular) - # prompt tokens": "{\"description\": \"min=495.232, mean=532.565, max=577.232, sum=1597.696 (3)\", \"tab\": \"General information\", \"score\": \"532.5653333333333\"}", - "MS MARCO (regular) - # output tokens": "{\"description\": \"min=1, mean=1.005, max=1.014, sum=3.014 (3)\", \"tab\": \"General information\", \"score\": \"1.0046666666666668\"}", - "MS MARCO (regular) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "MS MARCO (TREC) - # eval": "{\"description\": \"min=43, mean=43, max=43, sum=129 (3)\", \"tab\": \"General information\", \"score\": \"43.0\"}", - "MS MARCO (TREC) - # train": "{\"description\": \"min=2, mean=2, max=2, sum=6 (3)\", \"tab\": \"General information\", \"score\": \"2.0\"}", - "MS MARCO (TREC) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MS MARCO (TREC) - # prompt tokens": "{\"description\": \"min=478.488, mean=515.822, max=560.488, sum=1547.465 (3)\", \"tab\": \"General information\", \"score\": \"515.8217054263565\"}", - "MS MARCO (TREC) - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=3 (3)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "MS MARCO (TREC) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "MS MARCO (regular) - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "MS MARCO (TREC) - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CNN/DailyMail", - "source_data": { - "dataset_name": "CNN/DailyMail", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on CNN/DailyMail", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.154, - "details": { - "description": "min=0.142, mean=0.154, max=0.17, sum=0.927 (6)", - "tab": "Accuracy", - "CNN/DailyMail - Denoised inference time (s)": "{\"description\": \"min=3.898, mean=4.076, max=4.414, sum=24.459 (6)\", \"tab\": \"Efficiency\", \"score\": \"4.076441398798879\"}", - "CNN/DailyMail - # eval": "{\"description\": \"min=466, mean=466, max=466, sum=2796 (6)\", \"tab\": \"General information\", \"score\": \"466.0\"}", - "CNN/DailyMail - # train": "{\"description\": \"min=5, mean=5, max=5, sum=30 (6)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CNN/DailyMail - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (6)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CNN/DailyMail - # prompt tokens": "{\"description\": \"min=1531.586, mean=1549.919, max=1567.586, sum=9299.515 (6)\", \"tab\": \"General information\", \"score\": \"1549.9191702432045\"}", - "CNN/DailyMail - # output tokens": "{\"description\": \"min=54.895, mean=58.035, max=64.039, sum=348.21 (6)\", \"tab\": \"General information\", \"score\": \"58.035050071530755\"}", - "CNN/DailyMail - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=18 (6)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "CNN/DailyMail - Stereotypes (race)": "{\"description\": \"min=0.6, mean=0.616, max=0.642, sum=3.694 (6)\", \"tab\": \"Bias\", \"score\": \"0.6157343144185249\"}", - "CNN/DailyMail - Stereotypes (gender)": "{\"description\": \"min=0.4, mean=0.412, max=0.426, sum=2.474 (6)\", \"tab\": \"Bias\", \"score\": \"0.41239374128525014\"}", - "CNN/DailyMail - Representation (race)": "{\"description\": \"min=0.241, mean=0.252, max=0.26, sum=1.514 (6)\", \"tab\": \"Bias\", \"score\": \"0.2523476523476524\"}", - "CNN/DailyMail - Representation (gender)": "{\"description\": \"min=0.075, mean=0.093, max=0.102, sum=0.555 (6)\", \"tab\": \"Bias\", \"score\": \"0.09258312556525572\"}", - "CNN/DailyMail - Toxic fraction": "{\"description\": \"min=0, mean=0.001, max=0.002, sum=0.009 (6)\", \"tab\": \"Toxicity\", \"score\": \"0.001430615164520744\"}", - "CNN/DailyMail - SummaC": "{\"description\": \"min=0.473, mean=0.492, max=0.515, sum=1.477 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.4923968635744633\"}", - "CNN/DailyMail - QAFactEval": "{\"description\": \"min=4.385, mean=4.692, max=4.898, sum=28.151 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"4.691904356057608\"}", - "CNN/DailyMail - BERTScore (F1)": "{\"description\": \"min=0.315, mean=0.326, max=0.342, sum=0.979 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.32642089401655566\"}", - "CNN/DailyMail - Coverage": "{\"description\": \"min=0.953, mean=0.96, max=0.968, sum=5.762 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"0.9602766718208816\"}", - "CNN/DailyMail - Density": "{\"description\": \"min=9.043, mean=10.832, max=14.179, sum=64.991 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"10.831883037736205\"}", - "CNN/DailyMail - Compression": "{\"description\": \"min=10.561, mean=11.89, max=12.628, sum=71.339 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"11.889831050263881\"}", - "CNN/DailyMail - HumanEval-faithfulness": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=1.333 (2)\", \"tab\": \"Summarization metrics\", \"score\": \"0.6666666666666666\"}", - "CNN/DailyMail - HumanEval-relevance": "{\"description\": \"min=4, mean=4, max=4, sum=8 (2)\", \"tab\": \"Summarization metrics\", \"score\": \"4.0\"}", - "CNN/DailyMail - HumanEval-coherence": "{\"description\": \"min=2.667, mean=2.667, max=2.667, sum=5.333 (2)\", \"tab\": \"Summarization metrics\", \"score\": \"2.6666666666666665\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "XSUM", - "source_data": { - "dataset_name": "XSUM", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on XSUM", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.134, - "details": { - "description": "min=0.131, mean=0.134, max=0.137, sum=0.804 (6)", - "tab": "Accuracy", - "XSUM - Denoised inference time (s)": "{\"description\": \"min=2.357, mean=2.408, max=2.45, sum=14.45 (6)\", \"tab\": \"Efficiency\", \"score\": \"2.408301637575076\"}", - "XSUM - # eval": "{\"description\": \"min=518, mean=518, max=518, sum=3108 (6)\", \"tab\": \"General information\", \"score\": \"518.0\"}", - "XSUM - # train": "{\"description\": \"min=5, mean=5, max=5, sum=30 (6)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "XSUM - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (6)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "XSUM - # prompt tokens": "{\"description\": \"min=1456.402, mean=1510.735, max=1539.402, sum=9064.409 (6)\", \"tab\": \"General information\", \"score\": \"1510.734877734878\"}", - "XSUM - # output tokens": "{\"description\": \"min=28.284, mean=28.94, max=29.546, sum=173.637 (6)\", \"tab\": \"General information\", \"score\": \"28.93951093951094\"}", - "XSUM - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=18 (6)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "XSUM - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=4 (6)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "XSUM - Stereotypes (gender)": "{\"description\": \"min=0.415, mean=0.439, max=0.454, sum=2.637 (6)\", \"tab\": \"Bias\", \"score\": \"0.43949621664675426\"}", - "XSUM - Representation (race)": "{\"description\": \"min=0.497, mean=0.541, max=0.59, sum=3.246 (6)\", \"tab\": \"Bias\", \"score\": \"0.54094360657117\"}", - "XSUM - Representation (gender)": "{\"description\": \"min=0.189, mean=0.207, max=0.22, sum=1.244 (6)\", \"tab\": \"Bias\", \"score\": \"0.20735056882648284\"}", - "XSUM - Toxic fraction": "{\"description\": \"min=0.002, mean=0.004, max=0.006, sum=0.023 (6)\", \"tab\": \"Toxicity\", \"score\": \"0.0038610038610038615\"}", - "XSUM - SummaC": "{\"description\": \"min=-0.278, mean=-0.271, max=-0.263, sum=-0.812 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"-0.2708329675740717\"}", - "XSUM - QAFactEval": "{\"description\": \"min=2.934, mean=3.066, max=3.179, sum=18.394 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"3.0656965498353155\"}", - "XSUM - BERTScore (F1)": "{\"description\": \"min=0.434, mean=0.437, max=0.441, sum=1.311 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.4370376831136327\"}", - "XSUM - Coverage": "{\"description\": \"min=0.806, mean=0.808, max=0.811, sum=4.849 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"0.8082245669950062\"}", - "XSUM - Density": "{\"description\": \"min=2.656, mean=2.691, max=2.726, sum=16.146 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"2.6910357109145138\"}", - "XSUM - Compression": "{\"description\": \"min=14.828, mean=15.182, max=15.567, sum=91.094 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"15.182390855675616\"}", - "XSUM - HumanEval-faithfulness": "{\"description\": \"min=0.667, mean=0.778, max=0.889, sum=4.667 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"0.7777777777777777\"}", - "XSUM - HumanEval-relevance": "{\"description\": \"min=4.333, mean=4.398, max=4.444, sum=26.389 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"4.398148148148148\"}", - "XSUM - HumanEval-coherence": "{\"description\": \"min=4.889, mean=4.898, max=4.917, sum=29.389 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"4.898148148148149\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "IMDB", - "source_data": { - "dataset_name": "IMDB", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on IMDB", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.934, - "details": { - "description": "min=0.924, mean=0.934, max=0.948, sum=2.802 (3)", - "tab": "Accuracy", - "IMDB - ECE (10-bin)": "{\"description\": \"min=0.01, mean=0.015, max=0.024, sum=0.045 (3)\", \"tab\": \"Calibration\", \"score\": \"\"}", - "IMDB - EM (Robustness)": "{\"description\": \"min=0.921, mean=0.928, max=0.94, sum=2.783 (3)\", \"tab\": \"Robustness\", \"score\": \"0.9276666666666666\"}", - "IMDB - EM (Fairness)": "{\"description\": \"min=0.918, mean=0.925, max=0.936, sum=2.775 (3)\", \"tab\": \"Fairness\", \"score\": \"0.9249999999999999\"}", - "IMDB - Denoised inference time (s)": "{\"description\": \"min=0.714, mean=0.79, max=0.897, sum=2.37 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.7899130366753467\"}", - "IMDB - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "IMDB - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "IMDB - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IMDB - # prompt tokens": "{\"description\": \"min=1155.212, mean=1422.545, max=1836.212, sum=4267.636 (3)\", \"tab\": \"General information\", \"score\": \"1422.5453333333335\"}", - "IMDB - # output tokens": "{\"description\": \"min=1.002, mean=1.014, max=1.02, sum=3.042 (3)\", \"tab\": \"General information\", \"score\": \"1.014\"}", - "IMDB - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "IMDB - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CivilComments", - "source_data": { - "dataset_name": "CivilComments", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on CivilComments", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.61, - "details": { - "description": "min=0.182, mean=0.61, max=0.939, sum=32.915 (54)", - "tab": "Accuracy", - "CivilComments - ECE (10-bin)": "{\"description\": \"min=0.048, mean=0.179, max=0.449, sum=9.655 (54)\", \"tab\": \"Calibration\", \"score\": \"\"}", - "CivilComments - EM (Robustness)": "{\"description\": \"min=0.105, mean=0.514, max=0.854, sum=27.755 (54)\", \"tab\": \"Robustness\", \"score\": \"0.5139820592784173\"}", - "CivilComments - EM (Fairness)": "{\"description\": \"min=0.105, mean=0.512, max=0.939, sum=27.636 (54)\", \"tab\": \"Fairness\", \"score\": \"0.5117722022150621\"}", - "CivilComments - Denoised inference time (s)": "{\"description\": \"min=0.555, mean=0.594, max=0.756, sum=32.071 (54)\", \"tab\": \"Efficiency\", \"score\": \"0.5939081200798796\"}", - "CivilComments - # eval": "{\"description\": \"min=74, mean=371.556, max=683, sum=20064 (54)\", \"tab\": \"General information\", \"score\": \"371.55555555555554\"}", - "CivilComments - # train": "{\"description\": \"min=5, mean=5, max=5, sum=270 (54)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CivilComments - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (54)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CivilComments - # prompt tokens": "{\"description\": \"min=356.537, mean=722.635, max=1267.519, sum=39022.317 (54)\", \"tab\": \"General information\", \"score\": \"722.6354931173206\"}", - "CivilComments - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=54 (54)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "CivilComments - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=162 (54)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "CivilComments - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (54)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "RAFT", - "source_data": { - "dataset_name": "RAFT", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on RAFT", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.699, - "details": { - "description": "min=0.225, mean=0.699, max=0.95, sum=23.075 (33)", - "tab": "Accuracy", - "RAFT - ECE (10-bin)": "{\"description\": \"min=0.071, mean=0.212, max=0.648, sum=7.002 (33)\", \"tab\": \"Calibration\", \"score\": \"\"}", - "RAFT - EM (Robustness)": "{\"description\": \"min=0.225, mean=0.6, max=0.95, sum=19.8 (33)\", \"tab\": \"Robustness\", \"score\": \"0.6000000000000001\"}", - "RAFT - EM (Fairness)": "{\"description\": \"min=0.225, mean=0.67, max=0.95, sum=22.1 (33)\", \"tab\": \"Fairness\", \"score\": \"0.6696969696969697\"}", - "RAFT - Denoised inference time (s)": "{\"description\": \"min=0.583, mean=0.883, max=2.075, sum=29.139 (33)\", \"tab\": \"Efficiency\", \"score\": \"0.8829963013928345\"}", - "RAFT - # eval": "{\"description\": \"min=40, mean=40, max=40, sum=1320 (33)\", \"tab\": \"General information\", \"score\": \"40.0\"}", - "RAFT - # train": "{\"description\": \"min=5, mean=5, max=5, sum=165 (33)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "RAFT - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (33)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "RAFT - # prompt tokens": "{\"description\": \"min=257.35, mean=1279.572, max=6599.65, sum=42225.875 (33)\", \"tab\": \"General information\", \"score\": \"1279.5719696969697\"}", - "RAFT - # output tokens": "{\"description\": \"min=1, mean=2.986, max=5.3, sum=98.55 (33)\", \"tab\": \"General information\", \"score\": \"2.9863636363636363\"}", - "RAFT - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=99 (33)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "RAFT - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (33)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/anthropic_claude-2.0.json b/data/models/anthropic_claude-2.0.json deleted file mode 100644 index c1960716af375e9aab220212ff215f4193c80ff0..0000000000000000000000000000000000000000 --- a/data/models/anthropic_claude-2.0.json +++ /dev/null @@ -1,380 +0,0 @@ -{ - "model_info": { - "name": "Claude 2.0", - "id": "anthropic/claude-2.0", - "developer": "Anthropic", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_lite/anthropic_claude-2.0/1774096306.427425", - "retrieved_timestamp": "1774096306.427425", - "source_metadata": { - "source_name": "helm_lite", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_lite", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_lite", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.489, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.14701622971285894\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.718, - "details": { - "description": "min=0.718, mean=0.718, max=0.718, sum=0.718 (1)", - "tab": "Accuracy", - "NarrativeQA - Observed inference time (s)": "{\"description\": \"min=4.811, mean=4.811, max=4.811, sum=4.811 (1)\", \"tab\": \"Efficiency\", \"score\": \"4.8114360809326175\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=3709.741, mean=3709.741, max=3709.741, sum=3709.741 (1)\", \"tab\": \"General information\", \"score\": \"3709.7408450704224\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=10.561, mean=10.561, max=10.561, sum=10.561 (1)\", \"tab\": \"General information\", \"score\": \"10.56056338028169\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (closed-book)", - "source_data": { - "dataset_name": "NaturalQuestions (closed-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (closed-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.428, - "details": { - "description": "min=0.428, mean=0.428, max=0.428, sum=0.428 (1)", - "tab": "Accuracy", - "NaturalQuestions (open-book) - Observed inference time (s)": "{\"description\": \"min=2.984, mean=2.984, max=2.984, sum=2.984 (1)\", \"tab\": \"Efficiency\", \"score\": \"2.9841483016268606\"}", - "NaturalQuestions (closed-book) - Observed inference time (s)": "{\"description\": \"min=1.149, mean=1.149, max=1.149, sum=1.149 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.1486653406620027\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.964, mean=4.964, max=4.964, sum=4.964 (1)\", \"tab\": \"General information\", \"score\": \"4.964\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.007, mean=0.007, max=0.007, sum=0.007 (1)\", \"tab\": \"General information\", \"score\": \"0.007\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1734.363, mean=1734.363, max=1734.363, sum=1734.363 (1)\", \"tab\": \"General information\", \"score\": \"1734.363\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=7.605, mean=7.605, max=7.605, sum=7.605 (1)\", \"tab\": \"General information\", \"score\": \"7.605\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=189.259, mean=189.259, max=189.259, sum=189.259 (1)\", \"tab\": \"General information\", \"score\": \"189.259\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=7.206, mean=7.206, max=7.206, sum=7.206 (1)\", \"tab\": \"General information\", \"score\": \"7.206\"}" - } - }, - "generation_config": { - "additional_details": { - "mode": "\"closedbook\"" - } - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.862, - "details": { - "description": "min=0.862, mean=0.862, max=0.862, sum=0.862 (1)", - "tab": "Accuracy", - "OpenbookQA - Observed inference time (s)": "{\"description\": \"min=1.558, mean=1.558, max=1.558, sum=1.558 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.5584912838935852\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=328.79, mean=328.79, max=328.79, sum=328.79 (1)\", \"tab\": \"General information\", \"score\": \"328.79\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "dataset": "\"openbookqa\"", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.639, - "details": { - "description": "min=0.38, mean=0.639, max=0.9, sum=3.196 (5)", - "tab": "Accuracy", - "MMLU - Observed inference time (s)": "{\"description\": \"min=1.609, mean=1.728, max=1.936, sum=8.641 (5)\", \"tab\": \"Efficiency\", \"score\": \"1.7282055348597072\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=435.26, mean=543.747, max=684.596, sum=2718.736 (5)\", \"tab\": \"General information\", \"score\": \"543.747298245614\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"college_chemistry\", \"computer_security\", \"econometrics\", \"us_foreign_policy\"]", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MATH", - "source_data": { - "dataset_name": "MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Equivalent (CoT) on MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.603, - "details": { - "description": "min=0.491, mean=0.603, max=0.8, sum=4.219 (7)", - "tab": "Accuracy", - "MATH - Observed inference time (s)": "{\"description\": \"min=5.057, mean=6.211, max=7.33, sum=43.477 (7)\", \"tab\": \"Efficiency\", \"score\": \"6.211058685420826\"}", - "MATH - # eval": "{\"description\": \"min=30, mean=62.429, max=135, sum=437 (7)\", \"tab\": \"General information\", \"score\": \"62.42857142857143\"}", - "MATH - # train": "{\"description\": \"min=8, mean=8, max=8, sum=56 (7)\", \"tab\": \"General information\", \"score\": \"8.0\"}", - "MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MATH - # prompt tokens": "{\"description\": \"min=947.259, mean=1361.814, max=2379.808, sum=9532.699 (7)\", \"tab\": \"General information\", \"score\": \"1361.8141219676104\"}", - "MATH - # output tokens": "{\"description\": \"min=76.07, mean=96.474, max=115.288, sum=675.315 (7)\", \"tab\": \"General information\", \"score\": \"96.47352327848044\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"algebra\", \"counting_and_probability\", \"geometry\", \"intermediate_algebra\", \"number_theory\", \"prealgebra\", \"precalculus\"]", - "level": "\"1\"", - "use_official_examples": "\"False\"", - "use_chain_of_thought": "\"True\"" - } - } - }, - { - "evaluation_name": "GSM8K", - "source_data": { - "dataset_name": "GSM8K", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on GSM8K", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.583, - "details": { - "description": "min=0.583, mean=0.583, max=0.583, sum=0.583 (1)", - "tab": "Accuracy", - "GSM8K - Observed inference time (s)": "{\"description\": \"min=4.857, mean=4.857, max=4.857, sum=4.857 (1)\", \"tab\": \"Efficiency\", \"score\": \"4.857238686800003\"}", - "GSM8K - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "GSM8K - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "GSM8K - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GSM8K - # prompt tokens": "{\"description\": \"min=1012.712, mean=1012.712, max=1012.712, sum=1012.712 (1)\", \"tab\": \"General information\", \"score\": \"1012.712\"}", - "GSM8K - # output tokens": "{\"description\": \"min=78.704, mean=78.704, max=78.704, sum=78.704 (1)\", \"tab\": \"General information\", \"score\": \"78.704\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "LegalBench", - "source_data": { - "dataset_name": "LegalBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on LegalBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.643, - "details": { - "description": "min=0.387, mean=0.643, max=0.947, sum=3.216 (5)", - "tab": "Accuracy", - "LegalBench - Observed inference time (s)": "{\"description\": \"min=1.703, mean=2.782, max=6.2, sum=13.911 (5)\", \"tab\": \"Efficiency\", \"score\": \"2.782158235233088\"}", - "LegalBench - # eval": "{\"description\": \"min=95, mean=409.4, max=1000, sum=2047 (5)\", \"tab\": \"General information\", \"score\": \"409.4\"}", - "LegalBench - # train": "{\"description\": \"min=4, mean=4.798, max=5, sum=23.99 (5)\", \"tab\": \"General information\", \"score\": \"4.797959183673469\"}", - "LegalBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "LegalBench - # prompt tokens": "{\"description\": \"min=280.653, mean=1621.356, max=6484.969, sum=8106.779 (5)\", \"tab\": \"General information\", \"score\": \"1621.3558670820687\"}", - "LegalBench - # output tokens": "{\"description\": \"min=1, mean=3.338, max=11.058, sum=16.692 (5)\", \"tab\": \"General information\", \"score\": \"3.338449275778001\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"abercrombie\", \"corporate_lobbying\", \"function_of_decision_section\", \"international_citizenship_questions\", \"proa\"]" - } - } - }, - { - "evaluation_name": "MedQA", - "source_data": { - "dataset_name": "MedQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MedQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.652, - "details": { - "description": "min=0.652, mean=0.652, max=0.652, sum=0.652 (1)", - "tab": "Accuracy", - "MedQA - Observed inference time (s)": "{\"description\": \"min=2.254, mean=2.254, max=2.254, sum=2.254 (1)\", \"tab\": \"Efficiency\", \"score\": \"2.2539968865055213\"}", - "MedQA - # eval": "{\"description\": \"min=503, mean=503, max=503, sum=503 (1)\", \"tab\": \"General information\", \"score\": \"503.0\"}", - "MedQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MedQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MedQA - # prompt tokens": "{\"description\": \"min=1092.437, mean=1092.437, max=1092.437, sum=1092.437 (1)\", \"tab\": \"General information\", \"score\": \"1092.4373757455269\"}", - "MedQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WMT 2014", - "source_data": { - "dataset_name": "WMT 2014", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "BLEU-4 on WMT 2014", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.219, - "details": { - "description": "min=0.159, mean=0.219, max=0.268, sum=1.095 (5)", - "tab": "Accuracy", - "WMT 2014 - Observed inference time (s)": "{\"description\": \"min=1.692, mean=1.995, max=2.443, sum=9.976 (5)\", \"tab\": \"Efficiency\", \"score\": \"1.9951115173159082\"}", - "WMT 2014 - # eval": "{\"description\": \"min=503, mean=568.8, max=832, sum=2844 (5)\", \"tab\": \"General information\", \"score\": \"568.8\"}", - "WMT 2014 - # train": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "WMT 2014 - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WMT 2014 - # prompt tokens": "{\"description\": \"min=197.406, mean=218.573, max=240.974, sum=1092.866 (5)\", \"tab\": \"General information\", \"score\": \"218.57322077152472\"}", - "WMT 2014 - # output tokens": "{\"description\": \"min=24.254, mean=25.653, max=26.374, sum=128.266 (5)\", \"tab\": \"General information\", \"score\": \"25.65316323214559\"}" - } - }, - "generation_config": { - "additional_details": { - "language_pair": "[\"cs-en\", \"de-en\", \"fr-en\", \"hi-en\", \"ru-en\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/anthropic_claude-2.1.json b/data/models/anthropic_claude-2.1.json deleted file mode 100644 index 85377e698e2b62eabd2ab18766e65cf443893834..0000000000000000000000000000000000000000 --- a/data/models/anthropic_claude-2.1.json +++ /dev/null @@ -1,1901 +0,0 @@ -{ - "model_info": { - "name": "Claude 2.1", - "id": "anthropic/claude-2.1", - "developer": "Anthropic", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_lite/anthropic_claude-2.1/1774096306.427425", - "retrieved_timestamp": "1774096306.427425", - "source_metadata": { - "source_name": "helm_lite", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_lite", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_lite", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.437, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.08012484394506866\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.677, - "details": { - "description": "min=0.677, mean=0.677, max=0.677, sum=0.677 (1)", - "tab": "Accuracy", - "NarrativeQA - Observed inference time (s)": "{\"description\": \"min=5.376, mean=5.376, max=5.376, sum=5.376 (1)\", \"tab\": \"Efficiency\", \"score\": \"5.376147254755799\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=3709.741, mean=3709.741, max=3709.741, sum=3709.741 (1)\", \"tab\": \"General information\", \"score\": \"3709.7408450704224\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=12.431, mean=12.431, max=12.431, sum=12.431 (1)\", \"tab\": \"General information\", \"score\": \"12.430985915492958\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (closed-book)", - "source_data": { - "dataset_name": "NaturalQuestions (closed-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (closed-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.375, - "details": { - "description": "min=0.375, mean=0.375, max=0.375, sum=0.375 (1)", - "tab": "Accuracy", - "NaturalQuestions (open-book) - Observed inference time (s)": "{\"description\": \"min=4.161, mean=4.161, max=4.161, sum=4.161 (1)\", \"tab\": \"Efficiency\", \"score\": \"4.16052336707216\"}", - "NaturalQuestions (closed-book) - Observed inference time (s)": "{\"description\": \"min=1.753, mean=1.753, max=1.753, sum=1.753 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.753281570672989\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.964, mean=4.964, max=4.964, sum=4.964 (1)\", \"tab\": \"General information\", \"score\": \"4.964\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.007, mean=0.007, max=0.007, sum=0.007 (1)\", \"tab\": \"General information\", \"score\": \"0.007\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1734.363, mean=1734.363, max=1734.363, sum=1734.363 (1)\", \"tab\": \"General information\", \"score\": \"1734.363\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=19.738, mean=19.738, max=19.738, sum=19.738 (1)\", \"tab\": \"General information\", \"score\": \"19.738\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=189.259, mean=189.259, max=189.259, sum=189.259 (1)\", \"tab\": \"General information\", \"score\": \"189.259\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=11.053, mean=11.053, max=11.053, sum=11.053 (1)\", \"tab\": \"General information\", \"score\": \"11.053\"}" - } - }, - "generation_config": { - "additional_details": { - "mode": "\"closedbook\"" - } - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.872, - "details": { - "description": "min=0.872, mean=0.872, max=0.872, sum=0.872 (1)", - "tab": "Accuracy", - "OpenbookQA - Observed inference time (s)": "{\"description\": \"min=1.809, mean=1.809, max=1.809, sum=1.809 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.8090401072502136\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=328.79, mean=328.79, max=328.79, sum=328.79 (1)\", \"tab\": \"General information\", \"score\": \"328.79\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "dataset": "\"openbookqa\"", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.643, - "details": { - "description": "min=0.4, mean=0.643, max=0.92, sum=3.216 (5)", - "tab": "Accuracy", - "MMLU - Observed inference time (s)": "{\"description\": \"min=2.043, mean=2.371, max=2.615, sum=11.855 (5)\", \"tab\": \"Efficiency\", \"score\": \"2.370939975420634\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=435.26, mean=543.747, max=684.596, sum=2718.736 (5)\", \"tab\": \"General information\", \"score\": \"543.747298245614\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"college_chemistry\", \"computer_security\", \"econometrics\", \"us_foreign_policy\"]", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MATH", - "source_data": { - "dataset_name": "MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Equivalent (CoT) on MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.632, - "details": { - "description": "min=0.5, mean=0.632, max=0.852, sum=4.425 (7)", - "tab": "Accuracy", - "MATH - Observed inference time (s)": "{\"description\": \"min=9.158, mean=9.672, max=10.737, sum=67.703 (7)\", \"tab\": \"Efficiency\", \"score\": \"9.671810739168015\"}", - "MATH - # eval": "{\"description\": \"min=30, mean=62.429, max=135, sum=437 (7)\", \"tab\": \"General information\", \"score\": \"62.42857142857143\"}", - "MATH - # train": "{\"description\": \"min=8, mean=8, max=8, sum=56 (7)\", \"tab\": \"General information\", \"score\": \"8.0\"}", - "MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MATH - # prompt tokens": "{\"description\": \"min=947.259, mean=1361.814, max=2379.808, sum=9532.699 (7)\", \"tab\": \"General information\", \"score\": \"1361.8141219676104\"}", - "MATH - # output tokens": "{\"description\": \"min=79.825, mean=96.72, max=120.842, sum=677.038 (7)\", \"tab\": \"General information\", \"score\": \"96.71972910810119\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"algebra\", \"counting_and_probability\", \"geometry\", \"intermediate_algebra\", \"number_theory\", \"prealgebra\", \"precalculus\"]", - "level": "\"1\"", - "use_official_examples": "\"False\"", - "use_chain_of_thought": "\"True\"" - } - } - }, - { - "evaluation_name": "GSM8K", - "source_data": { - "dataset_name": "GSM8K", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on GSM8K", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.604, - "details": { - "description": "min=0.604, mean=0.604, max=0.604, sum=0.604 (1)", - "tab": "Accuracy", - "GSM8K - Observed inference time (s)": "{\"description\": \"min=7.706, mean=7.706, max=7.706, sum=7.706 (1)\", \"tab\": \"Efficiency\", \"score\": \"7.7061755385398865\"}", - "GSM8K - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "GSM8K - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "GSM8K - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GSM8K - # prompt tokens": "{\"description\": \"min=1012.712, mean=1012.712, max=1012.712, sum=1012.712 (1)\", \"tab\": \"General information\", \"score\": \"1012.712\"}", - "GSM8K - # output tokens": "{\"description\": \"min=98.553, mean=98.553, max=98.553, sum=98.553 (1)\", \"tab\": \"General information\", \"score\": \"98.553\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "LegalBench", - "source_data": { - "dataset_name": "LegalBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on LegalBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.643, - "details": { - "description": "min=0.406, mean=0.643, max=0.874, sum=3.214 (5)", - "tab": "Accuracy", - "LegalBench - Observed inference time (s)": "{\"description\": \"min=2.23, mean=3.223, max=6.58, sum=16.113 (5)\", \"tab\": \"Efficiency\", \"score\": \"3.2225898594048035\"}", - "LegalBench - # eval": "{\"description\": \"min=95, mean=409.4, max=1000, sum=2047 (5)\", \"tab\": \"General information\", \"score\": \"409.4\"}", - "LegalBench - # train": "{\"description\": \"min=4, mean=4.798, max=5, sum=23.99 (5)\", \"tab\": \"General information\", \"score\": \"4.797959183673469\"}", - "LegalBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "LegalBench - # prompt tokens": "{\"description\": \"min=280.653, mean=1621.356, max=6484.969, sum=8106.779 (5)\", \"tab\": \"General information\", \"score\": \"1621.3558670820687\"}", - "LegalBench - # output tokens": "{\"description\": \"min=1, mean=1.455, max=2.137, sum=7.277 (5)\", \"tab\": \"General information\", \"score\": \"1.4554741431234763\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"abercrombie\", \"corporate_lobbying\", \"function_of_decision_section\", \"international_citizenship_questions\", \"proa\"]" - } - } - }, - { - "evaluation_name": "MedQA", - "source_data": { - "dataset_name": "MedQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MedQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.644, - "details": { - "description": "min=0.644, mean=0.644, max=0.644, sum=0.644 (1)", - "tab": "Accuracy", - "MedQA - Observed inference time (s)": "{\"description\": \"min=2.482, mean=2.482, max=2.482, sum=2.482 (1)\", \"tab\": \"Efficiency\", \"score\": \"2.482170646754695\"}", - "MedQA - # eval": "{\"description\": \"min=503, mean=503, max=503, sum=503 (1)\", \"tab\": \"General information\", \"score\": \"503.0\"}", - "MedQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MedQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MedQA - # prompt tokens": "{\"description\": \"min=1092.437, mean=1092.437, max=1092.437, sum=1092.437 (1)\", \"tab\": \"General information\", \"score\": \"1092.4373757455269\"}", - "MedQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WMT 2014", - "source_data": { - "dataset_name": "WMT 2014", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "BLEU-4 on WMT 2014", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.204, - "details": { - "description": "min=0.148, mean=0.204, max=0.233, sum=1.021 (5)", - "tab": "Accuracy", - "WMT 2014 - Observed inference time (s)": "{\"description\": \"min=2.478, mean=2.756, max=3.455, sum=13.78 (5)\", \"tab\": \"Efficiency\", \"score\": \"2.7559348208894425\"}", - "WMT 2014 - # eval": "{\"description\": \"min=503, mean=568.8, max=832, sum=2844 (5)\", \"tab\": \"General information\", \"score\": \"568.8\"}", - "WMT 2014 - # train": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "WMT 2014 - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WMT 2014 - # prompt tokens": "{\"description\": \"min=197.406, mean=218.573, max=240.974, sum=1092.866 (5)\", \"tab\": \"General information\", \"score\": \"218.57322077152472\"}", - "WMT 2014 - # output tokens": "{\"description\": \"min=24.439, mean=25.235, max=26.058, sum=126.175 (5)\", \"tab\": \"General information\", \"score\": \"25.235038327725952\"}" - } - }, - "generation_config": { - "additional_details": { - "language_pair": "[\"cs-en\", \"de-en\", \"fr-en\", \"hi-en\", \"ru-en\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_mmlu/anthropic_claude-2.1/1774096312.00548", - "retrieved_timestamp": "1774096312.00548", - "source_metadata": { - "source_name": "helm_mmlu", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_mmlu", - "evaluation_results": [ - { - "evaluation_name": "MMLU All Subjects", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU All Subjects", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.735, - "details": { - "description": "min=0.352, mean=0.735, max=0.959, sum=83.762 (114)", - "tab": "Accuracy", - "MMLU All Subjects - Observed inference time (s)": "{\"description\": \"min=1.934, mean=2.418, max=3.916, sum=275.693 (114)\", \"tab\": \"Efficiency\", \"score\": \"2.4183583522219108\"}", - "MMLU All Subjects - # eval": "{\"description\": \"min=100, mean=246.351, max=1534, sum=28084 (114)\", \"tab\": \"General information\", \"score\": \"246.35087719298247\"}", - "MMLU All Subjects - # train": "{\"description\": \"min=5, mean=5, max=5, sum=570 (114)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU All Subjects - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU All Subjects - # prompt tokens": "{\"description\": \"min=358.018, mean=703.288, max=2952.576, sum=80174.875 (114)\", \"tab\": \"General information\", \"score\": \"703.2883793758955\"}", - "MMLU All Subjects - # output tokens": "{\"description\": \"min=0.994, mean=1.0, max=1, sum=113.982 (114)\", \"tab\": \"General information\", \"score\": \"0.999841257531982\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_name": "Abstract Algebra", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Abstract Algebra", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4, - "details": { - "description": "min=0.4, mean=0.4, max=0.4, sum=0.8 (2)", - "tab": "Accuracy", - "Abstract Algebra - Observed inference time (s)": "{\"description\": \"min=2.043, mean=2.043, max=2.043, sum=4.087 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.043452892303467\"}", - "Abstract Algebra - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Abstract Algebra - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Abstract Algebra - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Abstract Algebra - # prompt tokens": "{\"description\": \"min=435.26, mean=435.26, max=435.26, sum=870.52 (2)\", \"tab\": \"General information\", \"score\": \"435.26\"}", - "Abstract Algebra - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"abstract_algebra\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_abstract_algebra\"" - } - } - }, - { - "evaluation_name": "Anatomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Anatomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.726, - "details": { - "description": "min=0.726, mean=0.726, max=0.726, sum=1.452 (2)", - "tab": "Accuracy", - "Anatomy - Observed inference time (s)": "{\"description\": \"min=2.071, mean=2.071, max=2.071, sum=4.142 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.0710925843980577\"}", - "Anatomy - # eval": "{\"description\": \"min=135, mean=135, max=135, sum=270 (2)\", \"tab\": \"General information\", \"score\": \"135.0\"}", - "Anatomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Anatomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Anatomy - # prompt tokens": "{\"description\": \"min=435.8, mean=435.8, max=435.8, sum=871.6 (2)\", \"tab\": \"General information\", \"score\": \"435.8\"}", - "Anatomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"anatomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_anatomy\"" - } - } - }, - { - "evaluation_name": "College Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on College Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5, - "details": { - "description": "min=0.5, mean=0.5, max=0.5, sum=1 (2)", - "tab": "Accuracy", - "College Chemistry - Observed inference time (s)": "{\"description\": \"min=2.579, mean=2.579, max=2.579, sum=5.158 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.579245555400848\"}", - "College Biology - Observed inference time (s)": "{\"description\": \"min=2.209, mean=2.209, max=2.209, sum=4.418 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.2088319063186646\"}", - "College Computer Science - Observed inference time (s)": "{\"description\": \"min=2.413, mean=2.413, max=2.413, sum=4.826 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.4128634238243105\"}", - "College Mathematics - Observed inference time (s)": "{\"description\": \"min=2.18, mean=2.18, max=2.18, sum=4.359 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.179708275794983\"}", - "College Medicine - Observed inference time (s)": "{\"description\": \"min=2.324, mean=2.324, max=2.324, sum=4.648 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.3239130339870564\"}", - "College Physics - Observed inference time (s)": "{\"description\": \"min=2.145, mean=2.145, max=2.145, sum=4.289 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.144603039704117\"}", - "College Chemistry - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Chemistry - # prompt tokens": "{\"description\": \"min=615.01, mean=615.01, max=615.01, sum=1230.02 (2)\", \"tab\": \"General information\", \"score\": \"615.01\"}", - "College Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Biology - # eval": "{\"description\": \"min=144, mean=144, max=144, sum=288 (2)\", \"tab\": \"General information\", \"score\": \"144.0\"}", - "College Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # prompt tokens": "{\"description\": \"min=555.347, mean=555.347, max=555.347, sum=1110.694 (2)\", \"tab\": \"General information\", \"score\": \"555.3472222222222\"}", - "College Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # prompt tokens": "{\"description\": \"min=903.24, mean=903.24, max=903.24, sum=1806.48 (2)\", \"tab\": \"General information\", \"score\": \"903.24\"}", - "College Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Mathematics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # prompt tokens": "{\"description\": \"min=669.19, mean=669.19, max=669.19, sum=1338.38 (2)\", \"tab\": \"General information\", \"score\": \"669.19\"}", - "College Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Medicine - # eval": "{\"description\": \"min=173, mean=173, max=173, sum=346 (2)\", \"tab\": \"General information\", \"score\": \"173.0\"}", - "College Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # prompt tokens": "{\"description\": \"min=605.63, mean=605.63, max=605.63, sum=1211.26 (2)\", \"tab\": \"General information\", \"score\": \"605.6300578034682\"}", - "College Medicine - # output tokens": "{\"description\": \"min=0.994, mean=0.994, max=0.994, sum=1.988 (2)\", \"tab\": \"General information\", \"score\": \"0.9942196531791907\"}", - "College Physics - # eval": "{\"description\": \"min=102, mean=102, max=102, sum=204 (2)\", \"tab\": \"General information\", \"score\": \"102.0\"}", - "College Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # prompt tokens": "{\"description\": \"min=554.48, mean=554.48, max=554.48, sum=1108.961 (2)\", \"tab\": \"General information\", \"score\": \"554.4803921568628\"}", - "College Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"college_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_college_physics\"" - } - } - }, - { - "evaluation_name": "Computer Security", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Computer Security", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.81, - "details": { - "description": "min=0.81, mean=0.81, max=0.81, sum=1.62 (2)", - "tab": "Accuracy", - "Computer Security - Observed inference time (s)": "{\"description\": \"min=2.244, mean=2.244, max=2.244, sum=4.487 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.2435835003852844\"}", - "Computer Security - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Computer Security - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Computer Security - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Computer Security - # prompt tokens": "{\"description\": \"min=463.62, mean=463.62, max=463.62, sum=927.24 (2)\", \"tab\": \"General information\", \"score\": \"463.62\"}", - "Computer Security - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"computer_security\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_computer_security\"" - } - } - }, - { - "evaluation_name": "Econometrics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Econometrics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.596, - "details": { - "description": "min=0.596, mean=0.596, max=0.596, sum=1.193 (2)", - "tab": "Accuracy", - "Econometrics - Observed inference time (s)": "{\"description\": \"min=2.615, mean=2.615, max=2.615, sum=5.23 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.6147566636403403\"}", - "Econometrics - # eval": "{\"description\": \"min=114, mean=114, max=114, sum=228 (2)\", \"tab\": \"General information\", \"score\": \"114.0\"}", - "Econometrics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Econometrics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Econometrics - # prompt tokens": "{\"description\": \"min=684.596, mean=684.596, max=684.596, sum=1369.193 (2)\", \"tab\": \"General information\", \"score\": \"684.5964912280701\"}", - "Econometrics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"econometrics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_econometrics\"" - } - } - }, - { - "evaluation_name": "Global Facts", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Global Facts", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.55, - "details": { - "description": "min=0.55, mean=0.55, max=0.55, sum=1.1 (2)", - "tab": "Accuracy", - "Global Facts - Observed inference time (s)": "{\"description\": \"min=1.934, mean=1.934, max=1.934, sum=3.869 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.934385061264038\"}", - "Global Facts - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Global Facts - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Global Facts - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Global Facts - # prompt tokens": "{\"description\": \"min=476.61, mean=476.61, max=476.61, sum=953.22 (2)\", \"tab\": \"General information\", \"score\": \"476.61\"}", - "Global Facts - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"global_facts\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_global_facts\"" - } - } - }, - { - "evaluation_name": "Jurisprudence", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Jurisprudence", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.87, - "details": { - "description": "min=0.87, mean=0.87, max=0.87, sum=1.741 (2)", - "tab": "Accuracy", - "Jurisprudence - Observed inference time (s)": "{\"description\": \"min=2.042, mean=2.042, max=2.042, sum=4.084 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.041935768392351\"}", - "Jurisprudence - # eval": "{\"description\": \"min=108, mean=108, max=108, sum=216 (2)\", \"tab\": \"General information\", \"score\": \"108.0\"}", - "Jurisprudence - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Jurisprudence - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Jurisprudence - # prompt tokens": "{\"description\": \"min=496.426, mean=496.426, max=496.426, sum=992.852 (2)\", \"tab\": \"General information\", \"score\": \"496.4259259259259\"}", - "Jurisprudence - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"jurisprudence\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_jurisprudence\"" - } - } - }, - { - "evaluation_name": "Philosophy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Philosophy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.794, - "details": { - "description": "min=0.794, mean=0.794, max=0.794, sum=1.588 (2)", - "tab": "Accuracy", - "Philosophy - Observed inference time (s)": "{\"description\": \"min=2.326, mean=2.326, max=2.326, sum=4.652 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.3260836739248787\"}", - "Philosophy - # eval": "{\"description\": \"min=311, mean=311, max=311, sum=622 (2)\", \"tab\": \"General information\", \"score\": \"311.0\"}", - "Philosophy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Philosophy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Philosophy - # prompt tokens": "{\"description\": \"min=424.965, mean=424.965, max=424.965, sum=849.929 (2)\", \"tab\": \"General information\", \"score\": \"424.9646302250804\"}", - "Philosophy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"philosophy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_philosophy\"" - } - } - }, - { - "evaluation_name": "Professional Psychology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Professional Psychology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.797, - "details": { - "description": "min=0.797, mean=0.797, max=0.797, sum=1.595 (2)", - "tab": "Accuracy", - "Professional Medicine - Observed inference time (s)": "{\"description\": \"min=2.936, mean=2.936, max=2.936, sum=5.871 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.9355741520138348\"}", - "Professional Accounting - Observed inference time (s)": "{\"description\": \"min=2.529, mean=2.529, max=2.529, sum=5.058 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.528953587755244\"}", - "Professional Law - Observed inference time (s)": "{\"description\": \"min=3.335, mean=3.335, max=3.335, sum=6.669 (2)\", \"tab\": \"Efficiency\", \"score\": \"3.3346744537975206\"}", - "Professional Psychology - Observed inference time (s)": "{\"description\": \"min=2.597, mean=2.597, max=2.597, sum=5.194 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.5970658024931264\"}", - "Professional Medicine - # eval": "{\"description\": \"min=272, mean=272, max=272, sum=544 (2)\", \"tab\": \"General information\", \"score\": \"272.0\"}", - "Professional Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Medicine - # prompt tokens": "{\"description\": \"min=1188.537, mean=1188.537, max=1188.537, sum=2377.074 (2)\", \"tab\": \"General information\", \"score\": \"1188.5367647058824\"}", - "Professional Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Accounting - # eval": "{\"description\": \"min=282, mean=282, max=282, sum=564 (2)\", \"tab\": \"General information\", \"score\": \"282.0\"}", - "Professional Accounting - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Accounting - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # prompt tokens": "{\"description\": \"min=730.422, mean=730.422, max=730.422, sum=1460.844 (2)\", \"tab\": \"General information\", \"score\": \"730.4219858156029\"}", - "Professional Accounting - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Law - # eval": "{\"description\": \"min=1534, mean=1534, max=1534, sum=3068 (2)\", \"tab\": \"General information\", \"score\": \"1534.0\"}", - "Professional Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # prompt tokens": "{\"description\": \"min=1766.16, mean=1766.16, max=1766.16, sum=3532.321 (2)\", \"tab\": \"General information\", \"score\": \"1766.16036505867\"}", - "Professional Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Psychology - # eval": "{\"description\": \"min=612, mean=612, max=612, sum=1224 (2)\", \"tab\": \"General information\", \"score\": \"612.0\"}", - "Professional Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # prompt tokens": "{\"description\": \"min=668.168, mean=668.168, max=668.168, sum=1336.337 (2)\", \"tab\": \"General information\", \"score\": \"668.1683006535948\"}", - "Professional Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"professional_psychology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_professional_psychology\"" - } - } - }, - { - "evaluation_name": "Us Foreign Policy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Us Foreign Policy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.92, - "details": { - "description": "min=0.92, mean=0.92, max=0.92, sum=1.84 (2)", - "tab": "Accuracy", - "Us Foreign Policy - Observed inference time (s)": "{\"description\": \"min=2.374, mean=2.374, max=2.374, sum=4.747 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.37366126537323\"}", - "Us Foreign Policy - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Us Foreign Policy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Us Foreign Policy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Us Foreign Policy - # prompt tokens": "{\"description\": \"min=520.25, mean=520.25, max=520.25, sum=1040.5 (2)\", \"tab\": \"General information\", \"score\": \"520.25\"}", - "Us Foreign Policy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"us_foreign_policy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_us_foreign_policy\"" - } - } - }, - { - "evaluation_name": "Astronomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Astronomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.855, - "details": { - "description": "min=0.855, mean=0.855, max=0.855, sum=1.711 (2)", - "tab": "Accuracy", - "Astronomy - Observed inference time (s)": "{\"description\": \"min=2.346, mean=2.346, max=2.346, sum=4.692 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.345861089857001\"}", - "Astronomy - # eval": "{\"description\": \"min=152, mean=152, max=152, sum=304 (2)\", \"tab\": \"General information\", \"score\": \"152.0\"}", - "Astronomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Astronomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Astronomy - # prompt tokens": "{\"description\": \"min=669.493, mean=669.493, max=669.493, sum=1338.987 (2)\", \"tab\": \"General information\", \"score\": \"669.4934210526316\"}", - "Astronomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"astronomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_astronomy\"" - } - } - }, - { - "evaluation_name": "Business Ethics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Business Ethics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.73, - "details": { - "description": "min=0.73, mean=0.73, max=0.73, sum=1.46 (2)", - "tab": "Accuracy", - "Business Ethics - Observed inference time (s)": "{\"description\": \"min=2.35, mean=2.35, max=2.35, sum=4.701 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.3504813623428347\"}", - "Business Ethics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Business Ethics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Business Ethics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Business Ethics - # prompt tokens": "{\"description\": \"min=665.02, mean=665.02, max=665.02, sum=1330.04 (2)\", \"tab\": \"General information\", \"score\": \"665.02\"}", - "Business Ethics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"business_ethics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_business_ethics\"" - } - } - }, - { - "evaluation_name": "Clinical Knowledge", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Clinical Knowledge", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.785, - "details": { - "description": "min=0.785, mean=0.785, max=0.785, sum=1.57 (2)", - "tab": "Accuracy", - "Clinical Knowledge - Observed inference time (s)": "{\"description\": \"min=2.28, mean=2.28, max=2.28, sum=4.56 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.279950815776609\"}", - "Clinical Knowledge - # eval": "{\"description\": \"min=265, mean=265, max=265, sum=530 (2)\", \"tab\": \"General information\", \"score\": \"265.0\"}", - "Clinical Knowledge - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Clinical Knowledge - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Clinical Knowledge - # prompt tokens": "{\"description\": \"min=494.457, mean=494.457, max=494.457, sum=988.913 (2)\", \"tab\": \"General information\", \"score\": \"494.4566037735849\"}", - "Clinical Knowledge - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"clinical_knowledge\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_clinical_knowledge\"" - } - } - }, - { - "evaluation_name": "Conceptual Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Conceptual Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.766, - "details": { - "description": "min=0.766, mean=0.766, max=0.766, sum=1.532 (2)", - "tab": "Accuracy", - "Conceptual Physics - Observed inference time (s)": "{\"description\": \"min=2.125, mean=2.125, max=2.125, sum=4.25 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.1249657225101553\"}", - "Conceptual Physics - # eval": "{\"description\": \"min=235, mean=235, max=235, sum=470 (2)\", \"tab\": \"General information\", \"score\": \"235.0\"}", - "Conceptual Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Conceptual Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Conceptual Physics - # prompt tokens": "{\"description\": \"min=388.536, mean=388.536, max=388.536, sum=777.072 (2)\", \"tab\": \"General information\", \"score\": \"388.53617021276597\"}", - "Conceptual Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"conceptual_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_conceptual_physics\"" - } - } - }, - { - "evaluation_name": "Electrical Engineering", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Electrical Engineering", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.724, - "details": { - "description": "min=0.724, mean=0.724, max=0.724, sum=1.448 (2)", - "tab": "Accuracy", - "Electrical Engineering - Observed inference time (s)": "{\"description\": \"min=2.336, mean=2.336, max=2.336, sum=4.672 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.3361403728353567\"}", - "Electrical Engineering - # eval": "{\"description\": \"min=145, mean=145, max=145, sum=290 (2)\", \"tab\": \"General information\", \"score\": \"145.0\"}", - "Electrical Engineering - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Electrical Engineering - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Electrical Engineering - # prompt tokens": "{\"description\": \"min=502.041, mean=502.041, max=502.041, sum=1004.083 (2)\", \"tab\": \"General information\", \"score\": \"502.04137931034484\"}", - "Electrical Engineering - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"electrical_engineering\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_electrical_engineering\"" - } - } - }, - { - "evaluation_name": "Elementary Mathematics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Elementary Mathematics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.521, - "details": { - "description": "min=0.521, mean=0.521, max=0.521, sum=1.042 (2)", - "tab": "Accuracy", - "Elementary Mathematics - Observed inference time (s)": "{\"description\": \"min=2.399, mean=2.399, max=2.399, sum=4.798 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.398875941044439\"}", - "Elementary Mathematics - # eval": "{\"description\": \"min=378, mean=378, max=378, sum=756 (2)\", \"tab\": \"General information\", \"score\": \"378.0\"}", - "Elementary Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Elementary Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Elementary Mathematics - # prompt tokens": "{\"description\": \"min=576.066, mean=576.066, max=576.066, sum=1152.132 (2)\", \"tab\": \"General information\", \"score\": \"576.0661375661375\"}", - "Elementary Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"elementary_mathematics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_elementary_mathematics\"" - } - } - }, - { - "evaluation_name": "Formal Logic", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Formal Logic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5, - "details": { - "description": "min=0.5, mean=0.5, max=0.5, sum=1 (2)", - "tab": "Accuracy", - "Formal Logic - Observed inference time (s)": "{\"description\": \"min=2.294, mean=2.294, max=2.294, sum=4.587 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.293650850417122\"}", - "Formal Logic - # eval": "{\"description\": \"min=126, mean=126, max=126, sum=252 (2)\", \"tab\": \"General information\", \"score\": \"126.0\"}", - "Formal Logic - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Formal Logic - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Formal Logic - # prompt tokens": "{\"description\": \"min=711.746, mean=711.746, max=711.746, sum=1423.492 (2)\", \"tab\": \"General information\", \"score\": \"711.7460317460317\"}", - "Formal Logic - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"formal_logic\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_formal_logic\"" - } - } - }, - { - "evaluation_name": "High School World History", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on High School World History", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.903, - "details": { - "description": "min=0.903, mean=0.903, max=0.903, sum=1.806 (2)", - "tab": "Accuracy", - "High School Biology - Observed inference time (s)": "{\"description\": \"min=2.36, mean=2.36, max=2.36, sum=4.72 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.360204086765166\"}", - "High School Chemistry - Observed inference time (s)": "{\"description\": \"min=2.324, mean=2.324, max=2.324, sum=4.647 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.3235761426352517\"}", - "High School Computer Science - Observed inference time (s)": "{\"description\": \"min=2.353, mean=2.353, max=2.353, sum=4.707 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.3532658934593202\"}", - "High School European History - Observed inference time (s)": "{\"description\": \"min=3.916, mean=3.916, max=3.916, sum=7.832 (2)\", \"tab\": \"Efficiency\", \"score\": \"3.915820397752704\"}", - "High School Geography - Observed inference time (s)": "{\"description\": \"min=2.217, mean=2.217, max=2.217, sum=4.434 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.217141205614263\"}", - "High School Government And Politics - Observed inference time (s)": "{\"description\": \"min=2.403, mean=2.403, max=2.403, sum=4.807 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.4034566397493986\"}", - "High School Macroeconomics - Observed inference time (s)": "{\"description\": \"min=2.329, mean=2.329, max=2.329, sum=4.658 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.3290999345290353\"}", - "High School Mathematics - Observed inference time (s)": "{\"description\": \"min=2.45, mean=2.45, max=2.45, sum=4.9 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.4497611089988993\"}", - "High School Microeconomics - Observed inference time (s)": "{\"description\": \"min=2.492, mean=2.492, max=2.492, sum=4.984 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.492123728038884\"}", - "High School Physics - Observed inference time (s)": "{\"description\": \"min=2.268, mean=2.268, max=2.268, sum=4.536 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.267898343256767\"}", - "High School Psychology - Observed inference time (s)": "{\"description\": \"min=2.45, mean=2.45, max=2.45, sum=4.901 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.4503073394845387\"}", - "High School Statistics - Observed inference time (s)": "{\"description\": \"min=2.554, mean=2.554, max=2.554, sum=5.107 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.5535844012543008\"}", - "High School US History - Observed inference time (s)": "{\"description\": \"min=3.541, mean=3.541, max=3.541, sum=7.081 (2)\", \"tab\": \"Efficiency\", \"score\": \"3.540712014132855\"}", - "High School World History - Observed inference time (s)": "{\"description\": \"min=3.012, mean=3.012, max=3.012, sum=6.025 (2)\", \"tab\": \"Efficiency\", \"score\": \"3.0123110571994056\"}", - "High School Biology - # eval": "{\"description\": \"min=310, mean=310, max=310, sum=620 (2)\", \"tab\": \"General information\", \"score\": \"310.0\"}", - "High School Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Biology - # prompt tokens": "{\"description\": \"min=599.577, mean=599.577, max=599.577, sum=1199.155 (2)\", \"tab\": \"General information\", \"score\": \"599.5774193548388\"}", - "High School Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Chemistry - # eval": "{\"description\": \"min=203, mean=203, max=203, sum=406 (2)\", \"tab\": \"General information\", \"score\": \"203.0\"}", - "High School Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # prompt tokens": "{\"description\": \"min=562.921, mean=562.921, max=562.921, sum=1125.842 (2)\", \"tab\": \"General information\", \"score\": \"562.9211822660099\"}", - "High School Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "High School Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # prompt tokens": "{\"description\": \"min=947.4, mean=947.4, max=947.4, sum=1894.8 (2)\", \"tab\": \"General information\", \"score\": \"947.4\"}", - "High School Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School European History - # eval": "{\"description\": \"min=165, mean=165, max=165, sum=330 (2)\", \"tab\": \"General information\", \"score\": \"165.0\"}", - "High School European History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School European History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # prompt tokens": "{\"description\": \"min=2952.576, mean=2952.576, max=2952.576, sum=5905.152 (2)\", \"tab\": \"General information\", \"score\": \"2952.5757575757575\"}", - "High School European History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Geography - # eval": "{\"description\": \"min=198, mean=198, max=198, sum=396 (2)\", \"tab\": \"General information\", \"score\": \"198.0\"}", - "High School Geography - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Geography - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # prompt tokens": "{\"description\": \"min=477.268, mean=477.268, max=477.268, sum=954.535 (2)\", \"tab\": \"General information\", \"score\": \"477.2676767676768\"}", - "High School Geography - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Government And Politics - # eval": "{\"description\": \"min=193, mean=193, max=193, sum=386 (2)\", \"tab\": \"General information\", \"score\": \"193.0\"}", - "High School Government And Politics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Government And Politics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # prompt tokens": "{\"description\": \"min=556.104, mean=556.104, max=556.104, sum=1112.207 (2)\", \"tab\": \"General information\", \"score\": \"556.1036269430052\"}", - "High School Government And Politics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Macroeconomics - # eval": "{\"description\": \"min=390, mean=390, max=390, sum=780 (2)\", \"tab\": \"General information\", \"score\": \"390.0\"}", - "High School Macroeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Macroeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # prompt tokens": "{\"description\": \"min=471.036, mean=471.036, max=471.036, sum=942.072 (2)\", \"tab\": \"General information\", \"score\": \"471.0358974358974\"}", - "High School Macroeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Mathematics - # eval": "{\"description\": \"min=270, mean=270, max=270, sum=540 (2)\", \"tab\": \"General information\", \"score\": \"270.0\"}", - "High School Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # prompt tokens": "{\"description\": \"min=584.881, mean=584.881, max=584.881, sum=1169.763 (2)\", \"tab\": \"General information\", \"score\": \"584.8814814814815\"}", - "High School Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Microeconomics - # eval": "{\"description\": \"min=238, mean=238, max=238, sum=476 (2)\", \"tab\": \"General information\", \"score\": \"238.0\"}", - "High School Microeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Microeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # prompt tokens": "{\"description\": \"min=485.513, mean=485.513, max=485.513, sum=971.025 (2)\", \"tab\": \"General information\", \"score\": \"485.5126050420168\"}", - "High School Microeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Physics - # eval": "{\"description\": \"min=151, mean=151, max=151, sum=302 (2)\", \"tab\": \"General information\", \"score\": \"151.0\"}", - "High School Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # prompt tokens": "{\"description\": \"min=623.841, mean=623.841, max=623.841, sum=1247.682 (2)\", \"tab\": \"General information\", \"score\": \"623.841059602649\"}", - "High School Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Psychology - # eval": "{\"description\": \"min=545, mean=545, max=545, sum=1090 (2)\", \"tab\": \"General information\", \"score\": \"545.0\"}", - "High School Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # prompt tokens": "{\"description\": \"min=586.42, mean=586.42, max=586.42, sum=1172.84 (2)\", \"tab\": \"General information\", \"score\": \"586.4201834862386\"}", - "High School Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Statistics - # eval": "{\"description\": \"min=216, mean=216, max=216, sum=432 (2)\", \"tab\": \"General information\", \"score\": \"216.0\"}", - "High School Statistics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Statistics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # prompt tokens": "{\"description\": \"min=871.963, mean=871.963, max=871.963, sum=1743.926 (2)\", \"tab\": \"General information\", \"score\": \"871.9629629629629\"}", - "High School Statistics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School US History - # eval": "{\"description\": \"min=204, mean=204, max=204, sum=408 (2)\", \"tab\": \"General information\", \"score\": \"204.0\"}", - "High School US History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School US History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # prompt tokens": "{\"description\": \"min=2353.49, mean=2353.49, max=2353.49, sum=4706.98 (2)\", \"tab\": \"General information\", \"score\": \"2353.4901960784314\"}", - "High School US History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School World History - # eval": "{\"description\": \"min=237, mean=237, max=237, sum=474 (2)\", \"tab\": \"General information\", \"score\": \"237.0\"}", - "High School World History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School World History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # prompt tokens": "{\"description\": \"min=1540.932, mean=1540.932, max=1540.932, sum=3081.865 (2)\", \"tab\": \"General information\", \"score\": \"1540.9324894514768\"}", - "High School World History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"high_school_world_history\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_high_school_world_history\"" - } - } - }, - { - "evaluation_name": "Human Sexuality", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Human Sexuality", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.847, - "details": { - "description": "min=0.847, mean=0.847, max=0.847, sum=1.695 (2)", - "tab": "Accuracy", - "Human Aging - Observed inference time (s)": "{\"description\": \"min=2.287, mean=2.287, max=2.287, sum=4.573 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.286549251710353\"}", - "Human Sexuality - Observed inference time (s)": "{\"description\": \"min=2.14, mean=2.14, max=2.14, sum=4.28 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.1399855577308715\"}", - "Human Aging - # eval": "{\"description\": \"min=223, mean=223, max=223, sum=446 (2)\", \"tab\": \"General information\", \"score\": \"223.0\"}", - "Human Aging - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Aging - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Aging - # prompt tokens": "{\"description\": \"min=400.955, mean=400.955, max=400.955, sum=801.91 (2)\", \"tab\": \"General information\", \"score\": \"400.95515695067263\"}", - "Human Aging - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Human Sexuality - # eval": "{\"description\": \"min=131, mean=131, max=131, sum=262 (2)\", \"tab\": \"General information\", \"score\": \"131.0\"}", - "Human Sexuality - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Sexuality - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # prompt tokens": "{\"description\": \"min=436.496, mean=436.496, max=436.496, sum=872.992 (2)\", \"tab\": \"General information\", \"score\": \"436.4961832061069\"}", - "Human Sexuality - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"human_sexuality\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_human_sexuality\"" - } - } - }, - { - "evaluation_name": "International Law", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on International Law", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.901, - "details": { - "description": "min=0.901, mean=0.901, max=0.901, sum=1.802 (2)", - "tab": "Accuracy", - "International Law - Observed inference time (s)": "{\"description\": \"min=2.339, mean=2.339, max=2.339, sum=4.679 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.3394163206589123\"}", - "International Law - # eval": "{\"description\": \"min=121, mean=121, max=121, sum=242 (2)\", \"tab\": \"General information\", \"score\": \"121.0\"}", - "International Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "International Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "International Law - # prompt tokens": "{\"description\": \"min=729.165, mean=729.165, max=729.165, sum=1458.331 (2)\", \"tab\": \"General information\", \"score\": \"729.1652892561983\"}", - "International Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"international_law\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_international_law\"" - } - } - }, - { - "evaluation_name": "Logical Fallacies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Logical Fallacies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.834, - "details": { - "description": "min=0.834, mean=0.834, max=0.834, sum=1.669 (2)", - "tab": "Accuracy", - "Logical Fallacies - Observed inference time (s)": "{\"description\": \"min=2.313, mean=2.313, max=2.313, sum=4.627 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.3134736488201866\"}", - "Logical Fallacies - # eval": "{\"description\": \"min=163, mean=163, max=163, sum=326 (2)\", \"tab\": \"General information\", \"score\": \"163.0\"}", - "Logical Fallacies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Logical Fallacies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Logical Fallacies - # prompt tokens": "{\"description\": \"min=535.276, mean=535.276, max=535.276, sum=1070.552 (2)\", \"tab\": \"General information\", \"score\": \"535.2760736196319\"}", - "Logical Fallacies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"logical_fallacies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_logical_fallacies\"" - } - } - }, - { - "evaluation_name": "Machine Learning", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Machine Learning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.482, - "details": { - "description": "min=0.482, mean=0.482, max=0.482, sum=0.964 (2)", - "tab": "Accuracy", - "Machine Learning - Observed inference time (s)": "{\"description\": \"min=2.246, mean=2.246, max=2.246, sum=4.492 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.246019565633365\"}", - "Machine Learning - # eval": "{\"description\": \"min=112, mean=112, max=112, sum=224 (2)\", \"tab\": \"General information\", \"score\": \"112.0\"}", - "Machine Learning - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Machine Learning - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Machine Learning - # prompt tokens": "{\"description\": \"min=741.518, mean=741.518, max=741.518, sum=1483.036 (2)\", \"tab\": \"General information\", \"score\": \"741.5178571428571\"}", - "Machine Learning - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"machine_learning\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_machine_learning\"" - } - } - }, - { - "evaluation_name": "Management", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Management", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.825, - "details": { - "description": "min=0.825, mean=0.825, max=0.825, sum=1.65 (2)", - "tab": "Accuracy", - "Management - Observed inference time (s)": "{\"description\": \"min=2.02, mean=2.02, max=2.02, sum=4.041 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.0203486507378736\"}", - "Management - # eval": "{\"description\": \"min=103, mean=103, max=103, sum=206 (2)\", \"tab\": \"General information\", \"score\": \"103.0\"}", - "Management - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Management - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Management - # prompt tokens": "{\"description\": \"min=366.282, mean=366.282, max=366.282, sum=732.563 (2)\", \"tab\": \"General information\", \"score\": \"366.28155339805824\"}", - "Management - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"management\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_management\"" - } - } - }, - { - "evaluation_name": "Marketing", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Marketing", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.923, - "details": { - "description": "min=0.923, mean=0.923, max=0.923, sum=1.846 (2)", - "tab": "Accuracy", - "Marketing - Observed inference time (s)": "{\"description\": \"min=2.371, mean=2.371, max=2.371, sum=4.741 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.370740459515498\"}", - "Marketing - # eval": "{\"description\": \"min=234, mean=234, max=234, sum=468 (2)\", \"tab\": \"General information\", \"score\": \"234.0\"}", - "Marketing - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Marketing - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Marketing - # prompt tokens": "{\"description\": \"min=513.064, mean=513.064, max=513.064, sum=1026.128 (2)\", \"tab\": \"General information\", \"score\": \"513.0641025641025\"}", - "Marketing - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"marketing\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_marketing\"" - } - } - }, - { - "evaluation_name": "Medical Genetics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Medical Genetics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.81, - "details": { - "description": "min=0.81, mean=0.81, max=0.81, sum=1.62 (2)", - "tab": "Accuracy", - "Medical Genetics - Observed inference time (s)": "{\"description\": \"min=2.213, mean=2.213, max=2.213, sum=4.426 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.213027362823486\"}", - "Medical Genetics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Medical Genetics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Medical Genetics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Medical Genetics - # prompt tokens": "{\"description\": \"min=419.88, mean=419.88, max=419.88, sum=839.76 (2)\", \"tab\": \"General information\", \"score\": \"419.88\"}", - "Medical Genetics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"medical_genetics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_medical_genetics\"" - } - } - }, - { - "evaluation_name": "Miscellaneous", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Miscellaneous", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.88, - "details": { - "description": "min=0.88, mean=0.88, max=0.88, sum=1.76 (2)", - "tab": "Accuracy", - "Miscellaneous - Observed inference time (s)": "{\"description\": \"min=2.421, mean=2.421, max=2.421, sum=4.843 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.421274871813992\"}", - "Miscellaneous - # eval": "{\"description\": \"min=783, mean=783, max=783, sum=1566 (2)\", \"tab\": \"General information\", \"score\": \"783.0\"}", - "Miscellaneous - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Miscellaneous - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Miscellaneous - # prompt tokens": "{\"description\": \"min=393.628, mean=393.628, max=393.628, sum=787.257 (2)\", \"tab\": \"General information\", \"score\": \"393.62835249042143\"}", - "Miscellaneous - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"miscellaneous\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_miscellaneous\"" - } - } - }, - { - "evaluation_name": "Moral Scenarios", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Moral Scenarios", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.52, - "details": { - "description": "min=0.52, mean=0.52, max=0.52, sum=1.039 (2)", - "tab": "Accuracy", - "Moral Disputes - Observed inference time (s)": "{\"description\": \"min=2.478, mean=2.478, max=2.478, sum=4.955 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.4775779054344045\"}", - "Moral Scenarios - Observed inference time (s)": "{\"description\": \"min=2.624, mean=2.624, max=2.624, sum=5.248 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.624200687994504\"}", - "Moral Disputes - # eval": "{\"description\": \"min=346, mean=346, max=346, sum=692 (2)\", \"tab\": \"General information\", \"score\": \"346.0\"}", - "Moral Disputes - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Disputes - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Disputes - # prompt tokens": "{\"description\": \"min=576.789, mean=576.789, max=576.789, sum=1153.578 (2)\", \"tab\": \"General information\", \"score\": \"576.7890173410404\"}", - "Moral Disputes - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Moral Scenarios - # eval": "{\"description\": \"min=895, mean=895, max=895, sum=1790 (2)\", \"tab\": \"General information\", \"score\": \"895.0\"}", - "Moral Scenarios - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Scenarios - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # prompt tokens": "{\"description\": \"min=741.949, mean=741.949, max=741.949, sum=1483.897 (2)\", \"tab\": \"General information\", \"score\": \"741.9486033519553\"}", - "Moral Scenarios - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"moral_scenarios\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_moral_scenarios\"" - } - } - }, - { - "evaluation_name": "Nutrition", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Nutrition", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.781, - "details": { - "description": "min=0.781, mean=0.781, max=0.781, sum=1.562 (2)", - "tab": "Accuracy", - "Nutrition - Observed inference time (s)": "{\"description\": \"min=2.516, mean=2.516, max=2.516, sum=5.033 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.516486873813704\"}", - "Nutrition - # eval": "{\"description\": \"min=306, mean=306, max=306, sum=612 (2)\", \"tab\": \"General information\", \"score\": \"306.0\"}", - "Nutrition - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Nutrition - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Nutrition - # prompt tokens": "{\"description\": \"min=682.065, mean=682.065, max=682.065, sum=1364.131 (2)\", \"tab\": \"General information\", \"score\": \"682.0653594771242\"}", - "Nutrition - # output tokens": "{\"description\": \"min=0.997, mean=0.997, max=0.997, sum=1.993 (2)\", \"tab\": \"General information\", \"score\": \"0.9967320261437909\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"nutrition\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_nutrition\"" - } - } - }, - { - "evaluation_name": "Prehistory", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Prehistory", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.821, - "details": { - "description": "min=0.821, mean=0.821, max=0.821, sum=1.642 (2)", - "tab": "Accuracy", - "Prehistory - Observed inference time (s)": "{\"description\": \"min=2.431, mean=2.431, max=2.431, sum=4.862 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.4310101116145097\"}", - "Prehistory - # eval": "{\"description\": \"min=324, mean=324, max=324, sum=648 (2)\", \"tab\": \"General information\", \"score\": \"324.0\"}", - "Prehistory - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Prehistory - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Prehistory - # prompt tokens": "{\"description\": \"min=610.639, mean=610.639, max=610.639, sum=1221.278 (2)\", \"tab\": \"General information\", \"score\": \"610.6388888888889\"}", - "Prehistory - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"prehistory\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_prehistory\"" - } - } - }, - { - "evaluation_name": "Public Relations", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Public Relations", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.773, - "details": { - "description": "min=0.773, mean=0.773, max=0.773, sum=1.545 (2)", - "tab": "Accuracy", - "Public Relations - Observed inference time (s)": "{\"description\": \"min=2.068, mean=2.068, max=2.068, sum=4.136 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.067864069071683\"}", - "Public Relations - # eval": "{\"description\": \"min=110, mean=110, max=110, sum=220 (2)\", \"tab\": \"General information\", \"score\": \"110.0\"}", - "Public Relations - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Public Relations - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Public Relations - # prompt tokens": "{\"description\": \"min=497.991, mean=497.991, max=497.991, sum=995.982 (2)\", \"tab\": \"General information\", \"score\": \"497.9909090909091\"}", - "Public Relations - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"public_relations\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_public_relations\"" - } - } - }, - { - "evaluation_name": "Security Studies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Security Studies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.812, - "details": { - "description": "min=0.812, mean=0.812, max=0.812, sum=1.624 (2)", - "tab": "Accuracy", - "Security Studies - Observed inference time (s)": "{\"description\": \"min=2.854, mean=2.854, max=2.854, sum=5.708 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.8541687430167686\"}", - "Security Studies - # eval": "{\"description\": \"min=245, mean=245, max=245, sum=490 (2)\", \"tab\": \"General information\", \"score\": \"245.0\"}", - "Security Studies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Security Studies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Security Studies - # prompt tokens": "{\"description\": \"min=1308.804, mean=1308.804, max=1308.804, sum=2617.608 (2)\", \"tab\": \"General information\", \"score\": \"1308.8040816326532\"}", - "Security Studies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"security_studies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_security_studies\"" - } - } - }, - { - "evaluation_name": "Sociology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Sociology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.886, - "details": { - "description": "min=0.886, mean=0.886, max=0.886, sum=1.771 (2)", - "tab": "Accuracy", - "Sociology - Observed inference time (s)": "{\"description\": \"min=2.362, mean=2.362, max=2.362, sum=4.725 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.362461663004178\"}", - "Sociology - # eval": "{\"description\": \"min=201, mean=201, max=201, sum=402 (2)\", \"tab\": \"General information\", \"score\": \"201.0\"}", - "Sociology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Sociology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Sociology - # prompt tokens": "{\"description\": \"min=532.274, mean=532.274, max=532.274, sum=1064.547 (2)\", \"tab\": \"General information\", \"score\": \"532.273631840796\"}", - "Sociology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"sociology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_sociology\"" - } - } - }, - { - "evaluation_name": "Virology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Virology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.554, - "details": { - "description": "min=0.554, mean=0.554, max=0.554, sum=1.108 (2)", - "tab": "Accuracy", - "Virology - Observed inference time (s)": "{\"description\": \"min=2.231, mean=2.231, max=2.231, sum=4.462 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.2311078037124084\"}", - "Virology - # eval": "{\"description\": \"min=166, mean=166, max=166, sum=332 (2)\", \"tab\": \"General information\", \"score\": \"166.0\"}", - "Virology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Virology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Virology - # prompt tokens": "{\"description\": \"min=427.651, mean=427.651, max=427.651, sum=855.301 (2)\", \"tab\": \"General information\", \"score\": \"427.65060240963857\"}", - "Virology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"virology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_virology\"" - } - } - }, - { - "evaluation_name": "World Religions", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on World Religions", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.854, - "details": { - "description": "min=0.854, mean=0.854, max=0.854, sum=1.708 (2)", - "tab": "Accuracy", - "World Religions - Observed inference time (s)": "{\"description\": \"min=2.237, mean=2.237, max=2.237, sum=4.474 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.2371394411165113\"}", - "World Religions - # eval": "{\"description\": \"min=171, mean=171, max=171, sum=342 (2)\", \"tab\": \"General information\", \"score\": \"171.0\"}", - "World Religions - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "World Religions - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "World Religions - # prompt tokens": "{\"description\": \"min=358.018, mean=358.018, max=358.018, sum=716.035 (2)\", \"tab\": \"General information\", \"score\": \"358.0175438596491\"}", - "World Religions - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"world_religions\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_world_religions\"" - } - } - }, - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.048, - "details": { - "description": "", - "tab": "Efficiency" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/anthropic_claude-3-5-haiku-20241022.json b/data/models/anthropic_claude-3-5-haiku-20241022.json deleted file mode 100644 index 9edb0c7484cd3b1eab61fa90716227281f955253..0000000000000000000000000000000000000000 --- a/data/models/anthropic_claude-3-5-haiku-20241022.json +++ /dev/null @@ -1,3161 +0,0 @@ -{ - "model_info": { - "name": "Claude 3.5 Haiku 20241022", - "id": "anthropic/claude-3-5-haiku-20241022", - "developer": "Anthropic", - "inference_platform": "unknown", - "additional_details": { - "display_name": "Claude 3.5 Haiku" - } - }, - "evaluations": [ - { - "evaluation_id": "global-mmlu-lite/anthropic_claude-3-5-haiku-20241022/1773936583.743359", - "retrieved_timestamp": "1773936583.743359", - "source_metadata": { - "source_name": "Global MMLU Lite Leaderboard", - "source_type": "documentation", - "source_organization_name": "kaggle", - "source_organization_url": "www.kaggle.com", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "kaggle kernel", - "version": "4", - "additional_details": { - "url": "https://www.kaggle.com/code/shivalikasingh95/global-mmlu-lite-sample-notebook" - } - }, - "benchmark": "global-mmlu-lite", - "evaluation_results": [ - { - "evaluation_name": "Global MMLU Lite", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Global MMLU Lite", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6114 - } - }, - { - "evaluation_name": "Culturally Sensitive", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Sensitive", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5834 - } - }, - { - "evaluation_name": "Culturally Agnostic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Agnostic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6394 - } - }, - { - "evaluation_name": "Arabic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Arabic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.695, - "uncertainty": { - "confidence_interval": { - "lower": -0.0451, - "upper": 0.0451, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "English", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - English", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.485, - "uncertainty": { - "confidence_interval": { - "lower": -0.049, - "upper": 0.049, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Bengali", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Bengali", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.675, - "uncertainty": { - "confidence_interval": { - "lower": -0.0459, - "upper": 0.0459, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "German", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - German", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.565, - "uncertainty": { - "confidence_interval": { - "lower": -0.0486, - "upper": 0.0486, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "French", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - French", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.61, - "uncertainty": { - "confidence_interval": { - "lower": -0.0478, - "upper": 0.0478, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Hindi", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Hindi", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6575, - "uncertainty": { - "confidence_interval": { - "lower": -0.0465, - "upper": 0.0465, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Indonesian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Indonesian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5475, - "uncertainty": { - "confidence_interval": { - "lower": -0.0488, - "upper": 0.0488, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Italian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Italian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.48, - "uncertainty": { - "confidence_interval": { - "lower": -0.049, - "upper": 0.049, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Japanese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Japanese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.655, - "uncertainty": { - "confidence_interval": { - "lower": -0.0466, - "upper": 0.0466, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Korean", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Korean", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6575, - "uncertainty": { - "confidence_interval": { - "lower": -0.0465, - "upper": 0.0465, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Portuguese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Portuguese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5225, - "uncertainty": { - "confidence_interval": { - "lower": -0.0489, - "upper": 0.0489, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Spanish", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Spanish", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.485, - "uncertainty": { - "confidence_interval": { - "lower": -0.049, - "upper": 0.049, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Swahili", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Swahili", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.69, - "uncertainty": { - "confidence_interval": { - "lower": -0.0453, - "upper": 0.0453, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Yoruba", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Yoruba", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6675, - "uncertainty": { - "confidence_interval": { - "lower": -0.0462, - "upper": 0.0462, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Chinese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Chinese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.69, - "uncertainty": { - "confidence_interval": { - "lower": -0.0453, - "upper": 0.0453, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Burmese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Burmese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7, - "uncertainty": { - "confidence_interval": { - "lower": -0.0449, - "upper": 0.0449, - "method": "unknown" - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "global-mmlu-lite/anthropic_claude-3-5-haiku-20241022/1773936496.366405", - "retrieved_timestamp": "1773936496.366405", - "source_metadata": { - "source_name": "Global MMLU Lite Leaderboard", - "source_type": "documentation", - "source_organization_name": "kaggle", - "source_organization_url": "www.kaggle.com", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "kaggle kernel", - "version": "4", - "additional_details": { - "url": "https://www.kaggle.com/code/shivalikasingh95/global-mmlu-lite-sample-notebook" - } - }, - "benchmark": "global-mmlu-lite", - "evaluation_results": [ - { - "evaluation_name": "Global MMLU Lite", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Global MMLU Lite", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6114 - } - }, - { - "evaluation_name": "Culturally Sensitive", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Sensitive", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5834 - } - }, - { - "evaluation_name": "Culturally Agnostic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Agnostic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6394 - } - }, - { - "evaluation_name": "Arabic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Arabic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.695, - "uncertainty": { - "confidence_interval": { - "lower": -0.0451, - "upper": 0.0451, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "English", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - English", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.485, - "uncertainty": { - "confidence_interval": { - "lower": -0.049, - "upper": 0.049, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Bengali", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Bengali", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.675, - "uncertainty": { - "confidence_interval": { - "lower": -0.0459, - "upper": 0.0459, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "German", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - German", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.565, - "uncertainty": { - "confidence_interval": { - "lower": -0.0486, - "upper": 0.0486, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "French", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - French", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.61, - "uncertainty": { - "confidence_interval": { - "lower": -0.0478, - "upper": 0.0478, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Hindi", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Hindi", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6575, - "uncertainty": { - "confidence_interval": { - "lower": -0.0465, - "upper": 0.0465, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Indonesian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Indonesian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5475, - "uncertainty": { - "confidence_interval": { - "lower": -0.0488, - "upper": 0.0488, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Italian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Italian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.48, - "uncertainty": { - "confidence_interval": { - "lower": -0.049, - "upper": 0.049, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Japanese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Japanese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.655, - "uncertainty": { - "confidence_interval": { - "lower": -0.0466, - "upper": 0.0466, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Korean", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Korean", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6575, - "uncertainty": { - "confidence_interval": { - "lower": -0.0465, - "upper": 0.0465, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Portuguese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Portuguese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5225, - "uncertainty": { - "confidence_interval": { - "lower": -0.0489, - "upper": 0.0489, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Spanish", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Spanish", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.485, - "uncertainty": { - "confidence_interval": { - "lower": -0.049, - "upper": 0.049, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Swahili", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Swahili", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.69, - "uncertainty": { - "confidence_interval": { - "lower": -0.0453, - "upper": 0.0453, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Yoruba", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Yoruba", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6675, - "uncertainty": { - "confidence_interval": { - "lower": -0.0462, - "upper": 0.0462, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Chinese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Chinese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.69, - "uncertainty": { - "confidence_interval": { - "lower": -0.0453, - "upper": 0.0453, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Burmese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Burmese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7, - "uncertainty": { - "confidence_interval": { - "lower": -0.0449, - "upper": 0.0449, - "method": "unknown" - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "helm_capabilities/anthropic_claude-3-5-haiku-20241022/1774096304.5056698", - "retrieved_timestamp": "1774096304.5056698", - "source_metadata": { - "source_name": "helm_capabilities", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_capabilities", - "evaluation_results": [ - { - "evaluation_name": "Mean score", - "source_data": { - "dataset_name": "helm_capabilities", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "The mean of the scores from all columns.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.549, - "details": { - "description": "", - "tab": "Accuracy", - "Mean score - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"6.973328374403875\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU-Pro", - "source_data": { - "dataset_name": "MMLU-Pro", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on MMLU-Pro", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.605, - "details": { - "description": "min=0.605, mean=0.605, max=0.605, sum=0.605 (1)", - "tab": "Accuracy", - "MMLU-Pro - Observed inference time (s)": "{\"description\": \"min=5.171, mean=5.171, max=5.171, sum=5.171 (1)\", \"tab\": \"Efficiency\", \"score\": \"5.170877918004989\"}", - "MMLU-Pro - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "MMLU-Pro - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - # prompt tokens": "{\"description\": \"min=230.461, mean=230.461, max=230.461, sum=230.461 (1)\", \"tab\": \"General information\", \"score\": \"230.461\"}", - "MMLU-Pro - # output tokens": "{\"description\": \"min=253.047, mean=253.047, max=253.047, sum=253.047 (1)\", \"tab\": \"General information\", \"score\": \"253.047\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"all\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.363, - "details": { - "description": "min=0.363, mean=0.363, max=0.363, sum=0.363 (1)", - "tab": "Accuracy", - "GPQA - Observed inference time (s)": "{\"description\": \"min=5.33, mean=5.33, max=5.33, sum=5.33 (1)\", \"tab\": \"Efficiency\", \"score\": \"5.329682314877018\"}", - "GPQA - # eval": "{\"description\": \"min=446, mean=446, max=446, sum=446 (1)\", \"tab\": \"General information\", \"score\": \"446.0\"}", - "GPQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - # prompt tokens": "{\"description\": \"min=250.738, mean=250.738, max=250.738, sum=250.738 (1)\", \"tab\": \"General information\", \"score\": \"250.73766816143498\"}", - "GPQA - # output tokens": "{\"description\": \"min=270.388, mean=270.388, max=270.388, sum=270.388 (1)\", \"tab\": \"General information\", \"score\": \"270.38789237668163\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"gpqa_main\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "IFEval Strict Acc on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.792, - "details": { - "description": "min=0.792, mean=0.792, max=0.792, sum=0.792 (1)", - "tab": "Accuracy", - "IFEval - Observed inference time (s)": "{\"description\": \"min=5.886, mean=5.886, max=5.886, sum=5.886 (1)\", \"tab\": \"Efficiency\", \"score\": \"5.885677124347793\"}", - "IFEval - # eval": "{\"description\": \"min=541, mean=541, max=541, sum=541 (1)\", \"tab\": \"General information\", \"score\": \"541.0\"}", - "IFEval - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - # prompt tokens": "{\"description\": \"min=47.159, mean=47.159, max=47.159, sum=47.159 (1)\", \"tab\": \"General information\", \"score\": \"47.15896487985213\"}", - "IFEval - # output tokens": "{\"description\": \"min=273.985, mean=273.985, max=273.985, sum=273.985 (1)\", \"tab\": \"General information\", \"score\": \"273.9852125693161\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WildBench", - "source_data": { - "dataset_name": "WildBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "WB Score on WildBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.76, - "details": { - "description": "min=0.76, mean=0.76, max=0.76, sum=0.76 (1)", - "tab": "Accuracy", - "WildBench - Observed inference time (s)": "{\"description\": \"min=10.629, mean=10.629, max=10.629, sum=10.629 (1)\", \"tab\": \"Efficiency\", \"score\": \"10.62865050649643\"}", - "WildBench - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "WildBench - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # prompt tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # output tokens": "{\"description\": \"min=544.911, mean=544.911, max=544.911, sum=544.911 (1)\", \"tab\": \"General information\", \"score\": \"544.911\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"v2\"" - } - } - }, - { - "evaluation_name": "Omni-MATH", - "source_data": { - "dataset_name": "Omni-MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Acc on Omni-MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.224, - "details": { - "description": "min=0.224, mean=0.224, max=0.224, sum=0.224 (1)", - "tab": "Accuracy", - "Omni-MATH - Observed inference time (s)": "{\"description\": \"min=7.852, mean=7.852, max=7.852, sum=7.852 (1)\", \"tab\": \"Efficiency\", \"score\": \"7.851754008293152\"}", - "Omni-MATH - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "Omni-MATH - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - # prompt tokens": "{\"description\": \"min=110.563, mean=110.563, max=110.563, sum=110.563 (1)\", \"tab\": \"General information\", \"score\": \"110.563\"}", - "Omni-MATH - # output tokens": "{\"description\": \"min=409.742, mean=409.742, max=409.742, sum=409.742 (1)\", \"tab\": \"General information\", \"score\": \"409.742\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_lite/anthropic_claude-3-5-haiku-20241022/1774096306.427425", - "retrieved_timestamp": "1774096306.427425", - "source_metadata": { - "source_name": "helm_lite", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_lite", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_lite", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.531, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.29044943820224717\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.763, - "details": { - "description": "min=0.763, mean=0.763, max=0.763, sum=0.763 (1)", - "tab": "Accuracy", - "NarrativeQA - Observed inference time (s)": "{\"description\": \"min=1.304, mean=1.304, max=1.304, sum=1.304 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.3044010672770756\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=3662.741, mean=3662.741, max=3662.741, sum=3662.741 (1)\", \"tab\": \"General information\", \"score\": \"3662.7408450704224\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=7.031, mean=7.031, max=7.031, sum=7.031 (1)\", \"tab\": \"General information\", \"score\": \"7.030985915492958\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (closed-book)", - "source_data": { - "dataset_name": "NaturalQuestions (closed-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (closed-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.344, - "details": { - "description": "min=0.344, mean=0.344, max=0.344, sum=0.344 (1)", - "tab": "Accuracy", - "NaturalQuestions (open-book) - Observed inference time (s)": "{\"description\": \"min=1.41, mean=1.41, max=1.41, sum=1.41 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.4098961477279663\"}", - "NaturalQuestions (closed-book) - Observed inference time (s)": "{\"description\": \"min=0.799, mean=0.799, max=0.799, sum=0.799 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.7985508556365967\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1726.799, mean=1726.799, max=1726.799, sum=1726.799 (1)\", \"tab\": \"General information\", \"score\": \"1726.799\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=16.792, mean=16.792, max=16.792, sum=16.792 (1)\", \"tab\": \"General information\", \"score\": \"16.792\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=134.259, mean=134.259, max=134.259, sum=134.259 (1)\", \"tab\": \"General information\", \"score\": \"134.259\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=18.429, mean=18.429, max=18.429, sum=18.429 (1)\", \"tab\": \"General information\", \"score\": \"18.429\"}" - } - }, - "generation_config": { - "additional_details": { - "mode": "\"closedbook\"" - } - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.854, - "details": { - "description": "min=0.854, mean=0.854, max=0.854, sum=0.854 (1)", - "tab": "Accuracy", - "OpenbookQA - Observed inference time (s)": "{\"description\": \"min=0.9, mean=0.9, max=0.9, sum=0.9 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.8996305031776428\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=263.79, mean=263.79, max=263.79, sum=263.79 (1)\", \"tab\": \"General information\", \"score\": \"263.79\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "dataset": "\"openbookqa\"", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.671, - "details": { - "description": "min=0.47, mean=0.671, max=0.94, sum=3.356 (5)", - "tab": "Accuracy", - "MMLU - Observed inference time (s)": "{\"description\": \"min=0.909, mean=1.002, max=1.196, sum=5.012 (5)\", \"tab\": \"Efficiency\", \"score\": \"1.0023672421856928\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=370.26, mean=478.747, max=619.596, sum=2393.736 (5)\", \"tab\": \"General information\", \"score\": \"478.747298245614\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"college_chemistry\", \"computer_security\", \"econometrics\", \"us_foreign_policy\"]", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MATH", - "source_data": { - "dataset_name": "MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Equivalent (CoT) on MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.872, - "details": { - "description": "min=0.737, mean=0.872, max=0.988, sum=6.102 (7)", - "tab": "Accuracy", - "MATH - Observed inference time (s)": "{\"description\": \"min=3.671, mean=5.707, max=14.928, sum=39.947 (7)\", \"tab\": \"Efficiency\", \"score\": \"5.706647422047061\"}", - "MATH - # eval": "{\"description\": \"min=30, mean=62.429, max=135, sum=437 (7)\", \"tab\": \"General information\", \"score\": \"62.42857142857143\"}", - "MATH - # train": "{\"description\": \"min=8, mean=8, max=8, sum=56 (7)\", \"tab\": \"General information\", \"score\": \"8.0\"}", - "MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MATH - # prompt tokens": "{\"description\": \"min=873.259, mean=1287.814, max=2305.808, sum=9014.699 (7)\", \"tab\": \"General information\", \"score\": \"1287.8141219676104\"}", - "MATH - # output tokens": "{\"description\": \"min=165.86, mean=202.645, max=236.769, sum=1418.512 (7)\", \"tab\": \"General information\", \"score\": \"202.6446145676256\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"algebra\", \"counting_and_probability\", \"geometry\", \"intermediate_algebra\", \"number_theory\", \"prealgebra\", \"precalculus\"]", - "level": "\"1\"", - "use_official_examples": "\"False\"", - "use_chain_of_thought": "\"True\"" - } - } - }, - { - "evaluation_name": "GSM8K", - "source_data": { - "dataset_name": "GSM8K", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on GSM8K", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.815, - "details": { - "description": "min=0.815, mean=0.815, max=0.815, sum=0.815 (1)", - "tab": "Accuracy", - "GSM8K - Observed inference time (s)": "{\"description\": \"min=3.915, mean=3.915, max=3.915, sum=3.915 (1)\", \"tab\": \"Efficiency\", \"score\": \"3.915386771917343\"}", - "GSM8K - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "GSM8K - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "GSM8K - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GSM8K - # prompt tokens": "{\"description\": \"min=938.712, mean=938.712, max=938.712, sum=938.712 (1)\", \"tab\": \"General information\", \"score\": \"938.712\"}", - "GSM8K - # output tokens": "{\"description\": \"min=185.342, mean=185.342, max=185.342, sum=185.342 (1)\", \"tab\": \"General information\", \"score\": \"185.342\"}" - } - }, - "generation_config": { - "additional_details": { - "stop": "\"none\"" - } - } - }, - { - "evaluation_name": "LegalBench", - "source_data": { - "dataset_name": "LegalBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on LegalBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.631, - "details": { - "description": "min=0, mean=0.631, max=0.947, sum=3.155 (5)", - "tab": "Accuracy", - "LegalBench - Observed inference time (s)": "{\"description\": \"min=0.62, mean=1.383, max=2.1, sum=6.914 (5)\", \"tab\": \"Efficiency\", \"score\": \"1.3828645188221382\"}", - "LegalBench - # eval": "{\"description\": \"min=95, mean=409.4, max=1000, sum=2047 (5)\", \"tab\": \"General information\", \"score\": \"409.4\"}", - "LegalBench - # train": "{\"description\": \"min=4, mean=4.8, max=5, sum=24 (5)\", \"tab\": \"General information\", \"score\": \"4.8\"}", - "LegalBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "LegalBench - # prompt tokens": "{\"description\": \"min=232.653, mean=1568.242, max=6432.398, sum=7841.208 (5)\", \"tab\": \"General information\", \"score\": \"1568.241581367783\"}", - "LegalBench - # output tokens": "{\"description\": \"min=1, mean=6.998, max=29.403, sum=34.988 (5)\", \"tab\": \"General information\", \"score\": \"6.997580266743151\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"abercrombie\", \"corporate_lobbying\", \"function_of_decision_section\", \"international_citizenship_questions\", \"proa\"]", - "stop": "\"none\"" - } - } - }, - { - "evaluation_name": "MedQA", - "source_data": { - "dataset_name": "MedQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MedQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.722, - "details": { - "description": "min=0.722, mean=0.722, max=0.722, sum=0.722 (1)", - "tab": "Accuracy", - "MedQA - Observed inference time (s)": "{\"description\": \"min=0.99, mean=0.99, max=0.99, sum=0.99 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.9896539864435822\"}", - "MedQA - # eval": "{\"description\": \"min=503, mean=503, max=503, sum=503 (1)\", \"tab\": \"General information\", \"score\": \"503.0\"}", - "MedQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MedQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MedQA - # prompt tokens": "{\"description\": \"min=1027.437, mean=1027.437, max=1027.437, sum=1027.437 (1)\", \"tab\": \"General information\", \"score\": \"1027.4373757455269\"}", - "MedQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WMT 2014", - "source_data": { - "dataset_name": "WMT 2014", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "BLEU-4 on WMT 2014", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.135, - "details": { - "description": "min=0.077, mean=0.135, max=0.2, sum=0.675 (5)", - "tab": "Accuracy", - "WMT 2014 - Observed inference time (s)": "{\"description\": \"min=0.889, mean=1.087, max=1.411, sum=5.434 (5)\", \"tab\": \"Efficiency\", \"score\": \"1.0867067574964768\"}", - "WMT 2014 - # eval": "{\"description\": \"min=503, mean=568.8, max=832, sum=2844 (5)\", \"tab\": \"General information\", \"score\": \"568.8\"}", - "WMT 2014 - # train": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "WMT 2014 - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WMT 2014 - # prompt tokens": "{\"description\": \"min=131.406, mean=152.573, max=174.974, sum=762.866 (5)\", \"tab\": \"General information\", \"score\": \"152.5732207715247\"}", - "WMT 2014 - # output tokens": "{\"description\": \"min=33.417, mean=46.766, max=62.029, sum=233.828 (5)\", \"tab\": \"General information\", \"score\": \"46.76561018504359\"}" - } - }, - "generation_config": { - "additional_details": { - "language_pair": "[\"cs-en\", \"de-en\", \"fr-en\", \"hi-en\", \"ru-en\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_mmlu/anthropic_claude-3-5-haiku-20241022/1774096312.00548", - "retrieved_timestamp": "1774096312.00548", - "source_metadata": { - "source_name": "helm_mmlu", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_mmlu", - "evaluation_results": [ - { - "evaluation_name": "MMLU All Subjects", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU All Subjects", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.743, - "details": { - "description": "min=0.359, mean=0.743, max=0.94, sum=84.719 (114)", - "tab": "Accuracy", - "MMLU All Subjects - Observed inference time (s)": "{\"description\": \"min=0.909, mean=1.108, max=1.572, sum=126.32 (114)\", \"tab\": \"Efficiency\", \"score\": \"1.1080717974066416\"}", - "MMLU All Subjects - # eval": "{\"description\": \"min=100, mean=246.351, max=1534, sum=28084 (114)\", \"tab\": \"General information\", \"score\": \"246.35087719298247\"}", - "MMLU All Subjects - # train": "{\"description\": \"min=5, mean=5, max=5, sum=570 (114)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU All Subjects - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU All Subjects - # prompt tokens": "{\"description\": \"min=293.018, mean=638.288, max=2887.576, sum=72764.875 (114)\", \"tab\": \"General information\", \"score\": \"638.2883793758953\"}", - "MMLU All Subjects - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=114 (114)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_name": "Abstract Algebra", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Abstract Algebra", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.47, - "details": { - "description": "min=0.47, mean=0.47, max=0.47, sum=0.94 (2)", - "tab": "Accuracy", - "Abstract Algebra - Observed inference time (s)": "{\"description\": \"min=0.909, mean=0.909, max=0.909, sum=1.819 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9094081521034241\"}", - "Abstract Algebra - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Abstract Algebra - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Abstract Algebra - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Abstract Algebra - # prompt tokens": "{\"description\": \"min=370.26, mean=370.26, max=370.26, sum=740.52 (2)\", \"tab\": \"General information\", \"score\": \"370.26\"}", - "Abstract Algebra - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"abstract_algebra\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_abstract_algebra\"" - } - } - }, - { - "evaluation_name": "Anatomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Anatomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.793, - "details": { - "description": "min=0.793, mean=0.793, max=0.793, sum=1.585 (2)", - "tab": "Accuracy", - "Anatomy - Observed inference time (s)": "{\"description\": \"min=1.124, mean=1.124, max=1.124, sum=2.247 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.1236292309231228\"}", - "Anatomy - # eval": "{\"description\": \"min=135, mean=135, max=135, sum=270 (2)\", \"tab\": \"General information\", \"score\": \"135.0\"}", - "Anatomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Anatomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Anatomy - # prompt tokens": "{\"description\": \"min=370.8, mean=370.8, max=370.8, sum=741.6 (2)\", \"tab\": \"General information\", \"score\": \"370.8\"}", - "Anatomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"anatomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_anatomy\"" - } - } - }, - { - "evaluation_name": "College Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on College Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.52, - "details": { - "description": "min=0.52, mean=0.52, max=0.52, sum=1.039 (2)", - "tab": "Accuracy", - "College Chemistry - Observed inference time (s)": "{\"description\": \"min=1.196, mean=1.196, max=1.196, sum=2.392 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.1962119388580321\"}", - "College Biology - Observed inference time (s)": "{\"description\": \"min=1.247, mean=1.247, max=1.247, sum=2.494 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.2467927502261267\"}", - "College Computer Science - Observed inference time (s)": "{\"description\": \"min=1.572, mean=1.572, max=1.572, sum=3.144 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.5719245457649231\"}", - "College Mathematics - Observed inference time (s)": "{\"description\": \"min=1.13, mean=1.13, max=1.13, sum=2.26 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.1302329087257386\"}", - "College Medicine - Observed inference time (s)": "{\"description\": \"min=1.259, mean=1.259, max=1.259, sum=2.517 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.2587321479885565\"}", - "College Physics - Observed inference time (s)": "{\"description\": \"min=1.261, mean=1.261, max=1.261, sum=2.521 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.2606473857281255\"}", - "College Chemistry - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Chemistry - # prompt tokens": "{\"description\": \"min=550.01, mean=550.01, max=550.01, sum=1100.02 (2)\", \"tab\": \"General information\", \"score\": \"550.01\"}", - "College Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Biology - # eval": "{\"description\": \"min=144, mean=144, max=144, sum=288 (2)\", \"tab\": \"General information\", \"score\": \"144.0\"}", - "College Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # prompt tokens": "{\"description\": \"min=490.347, mean=490.347, max=490.347, sum=980.694 (2)\", \"tab\": \"General information\", \"score\": \"490.34722222222223\"}", - "College Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # prompt tokens": "{\"description\": \"min=838.24, mean=838.24, max=838.24, sum=1676.48 (2)\", \"tab\": \"General information\", \"score\": \"838.24\"}", - "College Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Mathematics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # prompt tokens": "{\"description\": \"min=604.19, mean=604.19, max=604.19, sum=1208.38 (2)\", \"tab\": \"General information\", \"score\": \"604.19\"}", - "College Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Medicine - # eval": "{\"description\": \"min=173, mean=173, max=173, sum=346 (2)\", \"tab\": \"General information\", \"score\": \"173.0\"}", - "College Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # prompt tokens": "{\"description\": \"min=540.63, mean=540.63, max=540.63, sum=1081.26 (2)\", \"tab\": \"General information\", \"score\": \"540.6300578034682\"}", - "College Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Physics - # eval": "{\"description\": \"min=102, mean=102, max=102, sum=204 (2)\", \"tab\": \"General information\", \"score\": \"102.0\"}", - "College Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # prompt tokens": "{\"description\": \"min=489.48, mean=489.48, max=489.48, sum=978.961 (2)\", \"tab\": \"General information\", \"score\": \"489.48039215686276\"}", - "College Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"college_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_college_physics\"" - } - } - }, - { - "evaluation_name": "Computer Security", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Computer Security", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.84, - "details": { - "description": "min=0.84, mean=0.84, max=0.84, sum=1.68 (2)", - "tab": "Accuracy", - "Computer Security - Observed inference time (s)": "{\"description\": \"min=1.013, mean=1.013, max=1.013, sum=2.027 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.0133756017684936\"}", - "Computer Security - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Computer Security - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Computer Security - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Computer Security - # prompt tokens": "{\"description\": \"min=398.62, mean=398.62, max=398.62, sum=797.24 (2)\", \"tab\": \"General information\", \"score\": \"398.62\"}", - "Computer Security - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"computer_security\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_computer_security\"" - } - } - }, - { - "evaluation_name": "Econometrics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Econometrics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.596, - "details": { - "description": "min=0.596, mean=0.596, max=0.596, sum=1.193 (2)", - "tab": "Accuracy", - "Econometrics - Observed inference time (s)": "{\"description\": \"min=0.922, mean=0.922, max=0.922, sum=1.845 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9224813549142135\"}", - "Econometrics - # eval": "{\"description\": \"min=114, mean=114, max=114, sum=228 (2)\", \"tab\": \"General information\", \"score\": \"114.0\"}", - "Econometrics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Econometrics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Econometrics - # prompt tokens": "{\"description\": \"min=619.596, mean=619.596, max=619.596, sum=1239.193 (2)\", \"tab\": \"General information\", \"score\": \"619.5964912280701\"}", - "Econometrics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"econometrics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_econometrics\"" - } - } - }, - { - "evaluation_name": "Global Facts", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Global Facts", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5, - "details": { - "description": "min=0.5, mean=0.5, max=0.5, sum=1 (2)", - "tab": "Accuracy", - "Global Facts - Observed inference time (s)": "{\"description\": \"min=1.101, mean=1.101, max=1.101, sum=2.201 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.1007365608215331\"}", - "Global Facts - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Global Facts - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Global Facts - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Global Facts - # prompt tokens": "{\"description\": \"min=411.61, mean=411.61, max=411.61, sum=823.22 (2)\", \"tab\": \"General information\", \"score\": \"411.61\"}", - "Global Facts - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"global_facts\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_global_facts\"" - } - } - }, - { - "evaluation_name": "Jurisprudence", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Jurisprudence", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.861, - "details": { - "description": "min=0.861, mean=0.861, max=0.861, sum=1.722 (2)", - "tab": "Accuracy", - "Jurisprudence - Observed inference time (s)": "{\"description\": \"min=1.104, mean=1.104, max=1.104, sum=2.209 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.1042848251484059\"}", - "Jurisprudence - # eval": "{\"description\": \"min=108, mean=108, max=108, sum=216 (2)\", \"tab\": \"General information\", \"score\": \"108.0\"}", - "Jurisprudence - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Jurisprudence - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Jurisprudence - # prompt tokens": "{\"description\": \"min=431.426, mean=431.426, max=431.426, sum=862.852 (2)\", \"tab\": \"General information\", \"score\": \"431.4259259259259\"}", - "Jurisprudence - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"jurisprudence\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_jurisprudence\"" - } - } - }, - { - "evaluation_name": "Philosophy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Philosophy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.823, - "details": { - "description": "min=0.823, mean=0.823, max=0.823, sum=1.646 (2)", - "tab": "Accuracy", - "Philosophy - Observed inference time (s)": "{\"description\": \"min=1.117, mean=1.117, max=1.117, sum=2.233 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.1165370488856767\"}", - "Philosophy - # eval": "{\"description\": \"min=311, mean=311, max=311, sum=622 (2)\", \"tab\": \"General information\", \"score\": \"311.0\"}", - "Philosophy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Philosophy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Philosophy - # prompt tokens": "{\"description\": \"min=359.965, mean=359.965, max=359.965, sum=719.929 (2)\", \"tab\": \"General information\", \"score\": \"359.9646302250804\"}", - "Philosophy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"philosophy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_philosophy\"" - } - } - }, - { - "evaluation_name": "Professional Psychology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Professional Psychology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.825, - "details": { - "description": "min=0.825, mean=0.825, max=0.825, sum=1.65 (2)", - "tab": "Accuracy", - "Professional Medicine - Observed inference time (s)": "{\"description\": \"min=1.412, mean=1.412, max=1.412, sum=2.824 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.4119182877680834\"}", - "Professional Accounting - Observed inference time (s)": "{\"description\": \"min=0.984, mean=0.984, max=0.984, sum=1.967 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9836687187776498\"}", - "Professional Law - Observed inference time (s)": "{\"description\": \"min=1.016, mean=1.016, max=1.016, sum=2.032 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.0160297585901412\"}", - "Professional Psychology - Observed inference time (s)": "{\"description\": \"min=0.979, mean=0.979, max=0.979, sum=1.958 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9789344672284095\"}", - "Professional Medicine - # eval": "{\"description\": \"min=272, mean=272, max=272, sum=544 (2)\", \"tab\": \"General information\", \"score\": \"272.0\"}", - "Professional Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Medicine - # prompt tokens": "{\"description\": \"min=1123.537, mean=1123.537, max=1123.537, sum=2247.074 (2)\", \"tab\": \"General information\", \"score\": \"1123.5367647058824\"}", - "Professional Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Accounting - # eval": "{\"description\": \"min=282, mean=282, max=282, sum=564 (2)\", \"tab\": \"General information\", \"score\": \"282.0\"}", - "Professional Accounting - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Accounting - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # prompt tokens": "{\"description\": \"min=665.422, mean=665.422, max=665.422, sum=1330.844 (2)\", \"tab\": \"General information\", \"score\": \"665.4219858156029\"}", - "Professional Accounting - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Law - # eval": "{\"description\": \"min=1534, mean=1534, max=1534, sum=3068 (2)\", \"tab\": \"General information\", \"score\": \"1534.0\"}", - "Professional Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # prompt tokens": "{\"description\": \"min=1701.16, mean=1701.16, max=1701.16, sum=3402.321 (2)\", \"tab\": \"General information\", \"score\": \"1701.16036505867\"}", - "Professional Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Psychology - # eval": "{\"description\": \"min=612, mean=612, max=612, sum=1224 (2)\", \"tab\": \"General information\", \"score\": \"612.0\"}", - "Professional Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # prompt tokens": "{\"description\": \"min=603.168, mean=603.168, max=603.168, sum=1206.337 (2)\", \"tab\": \"General information\", \"score\": \"603.1683006535948\"}", - "Professional Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"professional_psychology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_professional_psychology\"" - } - } - }, - { - "evaluation_name": "Us Foreign Policy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Us Foreign Policy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.94, - "details": { - "description": "min=0.94, mean=0.94, max=0.94, sum=1.88 (2)", - "tab": "Accuracy", - "Us Foreign Policy - Observed inference time (s)": "{\"description\": \"min=0.97, mean=0.97, max=0.97, sum=1.941 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9703591632843017\"}", - "Us Foreign Policy - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Us Foreign Policy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Us Foreign Policy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Us Foreign Policy - # prompt tokens": "{\"description\": \"min=455.25, mean=455.25, max=455.25, sum=910.5 (2)\", \"tab\": \"General information\", \"score\": \"455.25\"}", - "Us Foreign Policy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"us_foreign_policy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_us_foreign_policy\"" - } - } - }, - { - "evaluation_name": "Astronomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Astronomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.829, - "details": { - "description": "min=0.829, mean=0.829, max=0.829, sum=1.658 (2)", - "tab": "Accuracy", - "Astronomy - Observed inference time (s)": "{\"description\": \"min=1.18, mean=1.18, max=1.18, sum=2.36 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.1798271034893237\"}", - "Astronomy - # eval": "{\"description\": \"min=152, mean=152, max=152, sum=304 (2)\", \"tab\": \"General information\", \"score\": \"152.0\"}", - "Astronomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Astronomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Astronomy - # prompt tokens": "{\"description\": \"min=604.493, mean=604.493, max=604.493, sum=1208.987 (2)\", \"tab\": \"General information\", \"score\": \"604.4934210526316\"}", - "Astronomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"astronomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_astronomy\"" - } - } - }, - { - "evaluation_name": "Business Ethics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Business Ethics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8, - "details": { - "description": "min=0.8, mean=0.8, max=0.8, sum=1.6 (2)", - "tab": "Accuracy", - "Business Ethics - Observed inference time (s)": "{\"description\": \"min=1.147, mean=1.147, max=1.147, sum=2.295 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.1473834657669066\"}", - "Business Ethics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Business Ethics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Business Ethics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Business Ethics - # prompt tokens": "{\"description\": \"min=600.02, mean=600.02, max=600.02, sum=1200.04 (2)\", \"tab\": \"General information\", \"score\": \"600.02\"}", - "Business Ethics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"business_ethics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_business_ethics\"" - } - } - }, - { - "evaluation_name": "Clinical Knowledge", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Clinical Knowledge", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.823, - "details": { - "description": "min=0.823, mean=0.823, max=0.823, sum=1.645 (2)", - "tab": "Accuracy", - "Clinical Knowledge - Observed inference time (s)": "{\"description\": \"min=1.099, mean=1.099, max=1.099, sum=2.198 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.0991604094235403\"}", - "Clinical Knowledge - # eval": "{\"description\": \"min=265, mean=265, max=265, sum=530 (2)\", \"tab\": \"General information\", \"score\": \"265.0\"}", - "Clinical Knowledge - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Clinical Knowledge - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Clinical Knowledge - # prompt tokens": "{\"description\": \"min=429.457, mean=429.457, max=429.457, sum=858.913 (2)\", \"tab\": \"General information\", \"score\": \"429.4566037735849\"}", - "Clinical Knowledge - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"clinical_knowledge\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_clinical_knowledge\"" - } - } - }, - { - "evaluation_name": "Conceptual Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Conceptual Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.723, - "details": { - "description": "min=0.723, mean=0.723, max=0.723, sum=1.447 (2)", - "tab": "Accuracy", - "Conceptual Physics - Observed inference time (s)": "{\"description\": \"min=1.537, mean=1.537, max=1.537, sum=3.074 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.536949543242759\"}", - "Conceptual Physics - # eval": "{\"description\": \"min=235, mean=235, max=235, sum=470 (2)\", \"tab\": \"General information\", \"score\": \"235.0\"}", - "Conceptual Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Conceptual Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Conceptual Physics - # prompt tokens": "{\"description\": \"min=323.536, mean=323.536, max=323.536, sum=647.072 (2)\", \"tab\": \"General information\", \"score\": \"323.53617021276597\"}", - "Conceptual Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"conceptual_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_conceptual_physics\"" - } - } - }, - { - "evaluation_name": "Electrical Engineering", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Electrical Engineering", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.717, - "details": { - "description": "min=0.717, mean=0.717, max=0.717, sum=1.434 (2)", - "tab": "Accuracy", - "Electrical Engineering - Observed inference time (s)": "{\"description\": \"min=1.249, mean=1.249, max=1.249, sum=2.497 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.2485630594450852\"}", - "Electrical Engineering - # eval": "{\"description\": \"min=145, mean=145, max=145, sum=290 (2)\", \"tab\": \"General information\", \"score\": \"145.0\"}", - "Electrical Engineering - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Electrical Engineering - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Electrical Engineering - # prompt tokens": "{\"description\": \"min=437.041, mean=437.041, max=437.041, sum=874.083 (2)\", \"tab\": \"General information\", \"score\": \"437.04137931034484\"}", - "Electrical Engineering - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"electrical_engineering\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_electrical_engineering\"" - } - } - }, - { - "evaluation_name": "Elementary Mathematics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Elementary Mathematics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.561, - "details": { - "description": "min=0.561, mean=0.561, max=0.561, sum=1.122 (2)", - "tab": "Accuracy", - "Elementary Mathematics - Observed inference time (s)": "{\"description\": \"min=1.558, mean=1.558, max=1.558, sum=3.116 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.5580224965615248\"}", - "Elementary Mathematics - # eval": "{\"description\": \"min=378, mean=378, max=378, sum=756 (2)\", \"tab\": \"General information\", \"score\": \"378.0\"}", - "Elementary Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Elementary Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Elementary Mathematics - # prompt tokens": "{\"description\": \"min=511.066, mean=511.066, max=511.066, sum=1022.132 (2)\", \"tab\": \"General information\", \"score\": \"511.06613756613757\"}", - "Elementary Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"elementary_mathematics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_elementary_mathematics\"" - } - } - }, - { - "evaluation_name": "Formal Logic", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Formal Logic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.619, - "details": { - "description": "min=0.619, mean=0.619, max=0.619, sum=1.238 (2)", - "tab": "Accuracy", - "Formal Logic - Observed inference time (s)": "{\"description\": \"min=1.526, mean=1.526, max=1.526, sum=3.052 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.5258309424869598\"}", - "Formal Logic - # eval": "{\"description\": \"min=126, mean=126, max=126, sum=252 (2)\", \"tab\": \"General information\", \"score\": \"126.0\"}", - "Formal Logic - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Formal Logic - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Formal Logic - # prompt tokens": "{\"description\": \"min=646.746, mean=646.746, max=646.746, sum=1293.492 (2)\", \"tab\": \"General information\", \"score\": \"646.7460317460317\"}", - "Formal Logic - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"formal_logic\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_formal_logic\"" - } - } - }, - { - "evaluation_name": "High School World History", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on High School World History", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.882, - "details": { - "description": "min=0.882, mean=0.882, max=0.882, sum=1.764 (2)", - "tab": "Accuracy", - "High School Biology - Observed inference time (s)": "{\"description\": \"min=1.15, mean=1.15, max=1.15, sum=2.299 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.1497065974820044\"}", - "High School Chemistry - Observed inference time (s)": "{\"description\": \"min=1.227, mean=1.227, max=1.227, sum=2.454 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.2272211636228514\"}", - "High School Computer Science - Observed inference time (s)": "{\"description\": \"min=1.014, mean=1.014, max=1.014, sum=2.027 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.0136730527877809\"}", - "High School European History - Observed inference time (s)": "{\"description\": \"min=1.024, mean=1.024, max=1.024, sum=2.047 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.0236461119218305\"}", - "High School Geography - Observed inference time (s)": "{\"description\": \"min=1.059, mean=1.059, max=1.059, sum=2.119 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.0594979368074975\"}", - "High School Government And Politics - Observed inference time (s)": "{\"description\": \"min=1.138, mean=1.138, max=1.138, sum=2.275 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.1376265478875354\"}", - "High School Macroeconomics - Observed inference time (s)": "{\"description\": \"min=1.107, mean=1.107, max=1.107, sum=2.214 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.1069551357856164\"}", - "High School Mathematics - Observed inference time (s)": "{\"description\": \"min=1.094, mean=1.094, max=1.094, sum=2.188 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.0940863344404432\"}", - "High School Microeconomics - Observed inference time (s)": "{\"description\": \"min=1.034, mean=1.034, max=1.034, sum=2.068 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.03420967815303\"}", - "High School Physics - Observed inference time (s)": "{\"description\": \"min=1.059, mean=1.059, max=1.059, sum=2.119 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.0594944227610203\"}", - "High School Psychology - Observed inference time (s)": "{\"description\": \"min=1.074, mean=1.074, max=1.074, sum=2.149 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.07433808177983\"}", - "High School Statistics - Observed inference time (s)": "{\"description\": \"min=1.053, mean=1.053, max=1.053, sum=2.107 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.0534564554691315\"}", - "High School US History - Observed inference time (s)": "{\"description\": \"min=1.101, mean=1.101, max=1.101, sum=2.201 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.1006785748051662\"}", - "High School World History - Observed inference time (s)": "{\"description\": \"min=1.093, mean=1.093, max=1.093, sum=2.186 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.0931011674776359\"}", - "High School Biology - # eval": "{\"description\": \"min=310, mean=310, max=310, sum=620 (2)\", \"tab\": \"General information\", \"score\": \"310.0\"}", - "High School Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Biology - # prompt tokens": "{\"description\": \"min=534.577, mean=534.577, max=534.577, sum=1069.155 (2)\", \"tab\": \"General information\", \"score\": \"534.5774193548388\"}", - "High School Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Chemistry - # eval": "{\"description\": \"min=203, mean=203, max=203, sum=406 (2)\", \"tab\": \"General information\", \"score\": \"203.0\"}", - "High School Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # prompt tokens": "{\"description\": \"min=497.921, mean=497.921, max=497.921, sum=995.842 (2)\", \"tab\": \"General information\", \"score\": \"497.92118226600985\"}", - "High School Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "High School Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # prompt tokens": "{\"description\": \"min=882.4, mean=882.4, max=882.4, sum=1764.8 (2)\", \"tab\": \"General information\", \"score\": \"882.4\"}", - "High School Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School European History - # eval": "{\"description\": \"min=165, mean=165, max=165, sum=330 (2)\", \"tab\": \"General information\", \"score\": \"165.0\"}", - "High School European History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School European History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # prompt tokens": "{\"description\": \"min=2887.576, mean=2887.576, max=2887.576, sum=5775.152 (2)\", \"tab\": \"General information\", \"score\": \"2887.5757575757575\"}", - "High School European History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Geography - # eval": "{\"description\": \"min=198, mean=198, max=198, sum=396 (2)\", \"tab\": \"General information\", \"score\": \"198.0\"}", - "High School Geography - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Geography - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # prompt tokens": "{\"description\": \"min=412.268, mean=412.268, max=412.268, sum=824.535 (2)\", \"tab\": \"General information\", \"score\": \"412.2676767676768\"}", - "High School Geography - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Government And Politics - # eval": "{\"description\": \"min=193, mean=193, max=193, sum=386 (2)\", \"tab\": \"General information\", \"score\": \"193.0\"}", - "High School Government And Politics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Government And Politics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # prompt tokens": "{\"description\": \"min=491.104, mean=491.104, max=491.104, sum=982.207 (2)\", \"tab\": \"General information\", \"score\": \"491.10362694300517\"}", - "High School Government And Politics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Macroeconomics - # eval": "{\"description\": \"min=390, mean=390, max=390, sum=780 (2)\", \"tab\": \"General information\", \"score\": \"390.0\"}", - "High School Macroeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Macroeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # prompt tokens": "{\"description\": \"min=406.036, mean=406.036, max=406.036, sum=812.072 (2)\", \"tab\": \"General information\", \"score\": \"406.0358974358974\"}", - "High School Macroeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Mathematics - # eval": "{\"description\": \"min=270, mean=270, max=270, sum=540 (2)\", \"tab\": \"General information\", \"score\": \"270.0\"}", - "High School Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # prompt tokens": "{\"description\": \"min=519.881, mean=519.881, max=519.881, sum=1039.763 (2)\", \"tab\": \"General information\", \"score\": \"519.8814814814815\"}", - "High School Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Microeconomics - # eval": "{\"description\": \"min=238, mean=238, max=238, sum=476 (2)\", \"tab\": \"General information\", \"score\": \"238.0\"}", - "High School Microeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Microeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # prompt tokens": "{\"description\": \"min=420.513, mean=420.513, max=420.513, sum=841.025 (2)\", \"tab\": \"General information\", \"score\": \"420.5126050420168\"}", - "High School Microeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Physics - # eval": "{\"description\": \"min=151, mean=151, max=151, sum=302 (2)\", \"tab\": \"General information\", \"score\": \"151.0\"}", - "High School Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # prompt tokens": "{\"description\": \"min=558.841, mean=558.841, max=558.841, sum=1117.682 (2)\", \"tab\": \"General information\", \"score\": \"558.841059602649\"}", - "High School Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Psychology - # eval": "{\"description\": \"min=545, mean=545, max=545, sum=1090 (2)\", \"tab\": \"General information\", \"score\": \"545.0\"}", - "High School Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # prompt tokens": "{\"description\": \"min=521.42, mean=521.42, max=521.42, sum=1042.84 (2)\", \"tab\": \"General information\", \"score\": \"521.4201834862386\"}", - "High School Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Statistics - # eval": "{\"description\": \"min=216, mean=216, max=216, sum=432 (2)\", \"tab\": \"General information\", \"score\": \"216.0\"}", - "High School Statistics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Statistics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # prompt tokens": "{\"description\": \"min=806.963, mean=806.963, max=806.963, sum=1613.926 (2)\", \"tab\": \"General information\", \"score\": \"806.9629629629629\"}", - "High School Statistics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School US History - # eval": "{\"description\": \"min=204, mean=204, max=204, sum=408 (2)\", \"tab\": \"General information\", \"score\": \"204.0\"}", - "High School US History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School US History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # prompt tokens": "{\"description\": \"min=2288.49, mean=2288.49, max=2288.49, sum=4576.98 (2)\", \"tab\": \"General information\", \"score\": \"2288.4901960784314\"}", - "High School US History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School World History - # eval": "{\"description\": \"min=237, mean=237, max=237, sum=474 (2)\", \"tab\": \"General information\", \"score\": \"237.0\"}", - "High School World History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School World History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # prompt tokens": "{\"description\": \"min=1475.932, mean=1475.932, max=1475.932, sum=2951.865 (2)\", \"tab\": \"General information\", \"score\": \"1475.9324894514768\"}", - "High School World History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"high_school_world_history\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_high_school_world_history\"" - } - } - }, - { - "evaluation_name": "Human Sexuality", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Human Sexuality", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.885, - "details": { - "description": "min=0.885, mean=0.885, max=0.885, sum=1.771 (2)", - "tab": "Accuracy", - "Human Aging - Observed inference time (s)": "{\"description\": \"min=1.084, mean=1.084, max=1.084, sum=2.169 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.0844623775225584\"}", - "Human Sexuality - Observed inference time (s)": "{\"description\": \"min=1.056, mean=1.056, max=1.056, sum=2.112 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.0560545211529915\"}", - "Human Aging - # eval": "{\"description\": \"min=223, mean=223, max=223, sum=446 (2)\", \"tab\": \"General information\", \"score\": \"223.0\"}", - "Human Aging - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Aging - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Aging - # prompt tokens": "{\"description\": \"min=335.955, mean=335.955, max=335.955, sum=671.91 (2)\", \"tab\": \"General information\", \"score\": \"335.95515695067263\"}", - "Human Aging - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Human Sexuality - # eval": "{\"description\": \"min=131, mean=131, max=131, sum=262 (2)\", \"tab\": \"General information\", \"score\": \"131.0\"}", - "Human Sexuality - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Sexuality - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # prompt tokens": "{\"description\": \"min=371.496, mean=371.496, max=371.496, sum=742.992 (2)\", \"tab\": \"General information\", \"score\": \"371.4961832061069\"}", - "Human Sexuality - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"human_sexuality\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_human_sexuality\"" - } - } - }, - { - "evaluation_name": "International Law", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on International Law", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.884, - "details": { - "description": "min=0.884, mean=0.884, max=0.884, sum=1.769 (2)", - "tab": "Accuracy", - "International Law - Observed inference time (s)": "{\"description\": \"min=1.112, mean=1.112, max=1.112, sum=2.225 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.1124236544301687\"}", - "International Law - # eval": "{\"description\": \"min=121, mean=121, max=121, sum=242 (2)\", \"tab\": \"General information\", \"score\": \"121.0\"}", - "International Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "International Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "International Law - # prompt tokens": "{\"description\": \"min=664.165, mean=664.165, max=664.165, sum=1328.331 (2)\", \"tab\": \"General information\", \"score\": \"664.1652892561983\"}", - "International Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"international_law\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_international_law\"" - } - } - }, - { - "evaluation_name": "Logical Fallacies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Logical Fallacies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.822, - "details": { - "description": "min=0.822, mean=0.822, max=0.822, sum=1.644 (2)", - "tab": "Accuracy", - "Logical Fallacies - Observed inference time (s)": "{\"description\": \"min=1.015, mean=1.015, max=1.015, sum=2.03 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.0148307984591993\"}", - "Logical Fallacies - # eval": "{\"description\": \"min=163, mean=163, max=163, sum=326 (2)\", \"tab\": \"General information\", \"score\": \"163.0\"}", - "Logical Fallacies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Logical Fallacies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Logical Fallacies - # prompt tokens": "{\"description\": \"min=470.276, mean=470.276, max=470.276, sum=940.552 (2)\", \"tab\": \"General information\", \"score\": \"470.2760736196319\"}", - "Logical Fallacies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"logical_fallacies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_logical_fallacies\"" - } - } - }, - { - "evaluation_name": "Machine Learning", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Machine Learning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.518, - "details": { - "description": "min=0.518, mean=0.518, max=0.518, sum=1.036 (2)", - "tab": "Accuracy", - "Machine Learning - Observed inference time (s)": "{\"description\": \"min=1.067, mean=1.067, max=1.067, sum=2.135 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.0673569909163885\"}", - "Machine Learning - # eval": "{\"description\": \"min=112, mean=112, max=112, sum=224 (2)\", \"tab\": \"General information\", \"score\": \"112.0\"}", - "Machine Learning - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Machine Learning - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Machine Learning - # prompt tokens": "{\"description\": \"min=676.518, mean=676.518, max=676.518, sum=1353.036 (2)\", \"tab\": \"General information\", \"score\": \"676.5178571428571\"}", - "Machine Learning - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"machine_learning\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_machine_learning\"" - } - } - }, - { - "evaluation_name": "Management", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Management", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.845, - "details": { - "description": "min=0.845, mean=0.845, max=0.845, sum=1.689 (2)", - "tab": "Accuracy", - "Management - Observed inference time (s)": "{\"description\": \"min=1.038, mean=1.038, max=1.038, sum=2.076 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.0377622229381673\"}", - "Management - # eval": "{\"description\": \"min=103, mean=103, max=103, sum=206 (2)\", \"tab\": \"General information\", \"score\": \"103.0\"}", - "Management - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Management - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Management - # prompt tokens": "{\"description\": \"min=301.282, mean=301.282, max=301.282, sum=602.563 (2)\", \"tab\": \"General information\", \"score\": \"301.28155339805824\"}", - "Management - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"management\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_management\"" - } - } - }, - { - "evaluation_name": "Marketing", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Marketing", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.897, - "details": { - "description": "min=0.897, mean=0.897, max=0.897, sum=1.795 (2)", - "tab": "Accuracy", - "Marketing - Observed inference time (s)": "{\"description\": \"min=0.993, mean=0.993, max=0.993, sum=1.986 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9929133276654105\"}", - "Marketing - # eval": "{\"description\": \"min=234, mean=234, max=234, sum=468 (2)\", \"tab\": \"General information\", \"score\": \"234.0\"}", - "Marketing - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Marketing - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Marketing - # prompt tokens": "{\"description\": \"min=448.064, mean=448.064, max=448.064, sum=896.128 (2)\", \"tab\": \"General information\", \"score\": \"448.06410256410254\"}", - "Marketing - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"marketing\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_marketing\"" - } - } - }, - { - "evaluation_name": "Medical Genetics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Medical Genetics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.83, - "details": { - "description": "min=0.83, mean=0.83, max=0.83, sum=1.66 (2)", - "tab": "Accuracy", - "Medical Genetics - Observed inference time (s)": "{\"description\": \"min=1.041, mean=1.041, max=1.041, sum=2.082 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.041243133544922\"}", - "Medical Genetics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Medical Genetics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Medical Genetics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Medical Genetics - # prompt tokens": "{\"description\": \"min=354.88, mean=354.88, max=354.88, sum=709.76 (2)\", \"tab\": \"General information\", \"score\": \"354.88\"}", - "Medical Genetics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"medical_genetics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_medical_genetics\"" - } - } - }, - { - "evaluation_name": "Miscellaneous", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Miscellaneous", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.905, - "details": { - "description": "min=0.905, mean=0.905, max=0.905, sum=1.811 (2)", - "tab": "Accuracy", - "Miscellaneous - Observed inference time (s)": "{\"description\": \"min=1.043, mean=1.043, max=1.043, sum=2.086 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.0429492231225297\"}", - "Miscellaneous - # eval": "{\"description\": \"min=783, mean=783, max=783, sum=1566 (2)\", \"tab\": \"General information\", \"score\": \"783.0\"}", - "Miscellaneous - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Miscellaneous - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Miscellaneous - # prompt tokens": "{\"description\": \"min=328.628, mean=328.628, max=328.628, sum=657.257 (2)\", \"tab\": \"General information\", \"score\": \"328.62835249042143\"}", - "Miscellaneous - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"miscellaneous\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_miscellaneous\"" - } - } - }, - { - "evaluation_name": "Moral Scenarios", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Moral Scenarios", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.476, - "details": { - "description": "min=0.476, mean=0.476, max=0.476, sum=0.952 (2)", - "tab": "Accuracy", - "Moral Disputes - Observed inference time (s)": "{\"description\": \"min=1.044, mean=1.044, max=1.044, sum=2.088 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.0438106094481627\"}", - "Moral Scenarios - Observed inference time (s)": "{\"description\": \"min=0.96, mean=0.96, max=0.96, sum=1.919 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.95963474492121\"}", - "Moral Disputes - # eval": "{\"description\": \"min=346, mean=346, max=346, sum=692 (2)\", \"tab\": \"General information\", \"score\": \"346.0\"}", - "Moral Disputes - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Disputes - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Disputes - # prompt tokens": "{\"description\": \"min=511.789, mean=511.789, max=511.789, sum=1023.578 (2)\", \"tab\": \"General information\", \"score\": \"511.78901734104045\"}", - "Moral Disputes - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Moral Scenarios - # eval": "{\"description\": \"min=895, mean=895, max=895, sum=1790 (2)\", \"tab\": \"General information\", \"score\": \"895.0\"}", - "Moral Scenarios - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Scenarios - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # prompt tokens": "{\"description\": \"min=676.949, mean=676.949, max=676.949, sum=1353.897 (2)\", \"tab\": \"General information\", \"score\": \"676.9486033519553\"}", - "Moral Scenarios - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"moral_scenarios\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_moral_scenarios\"" - } - } - }, - { - "evaluation_name": "Nutrition", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Nutrition", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.846, - "details": { - "description": "min=0.846, mean=0.846, max=0.846, sum=1.693 (2)", - "tab": "Accuracy", - "Nutrition - Observed inference time (s)": "{\"description\": \"min=0.981, mean=0.981, max=0.981, sum=1.962 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9811088399949417\"}", - "Nutrition - # eval": "{\"description\": \"min=306, mean=306, max=306, sum=612 (2)\", \"tab\": \"General information\", \"score\": \"306.0\"}", - "Nutrition - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Nutrition - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Nutrition - # prompt tokens": "{\"description\": \"min=617.065, mean=617.065, max=617.065, sum=1234.131 (2)\", \"tab\": \"General information\", \"score\": \"617.0653594771242\"}", - "Nutrition - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"nutrition\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_nutrition\"" - } - } - }, - { - "evaluation_name": "Prehistory", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Prehistory", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.877, - "details": { - "description": "min=0.877, mean=0.877, max=0.877, sum=1.753 (2)", - "tab": "Accuracy", - "Prehistory - Observed inference time (s)": "{\"description\": \"min=1.003, mean=1.003, max=1.003, sum=2.006 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.0031694571177165\"}", - "Prehistory - # eval": "{\"description\": \"min=324, mean=324, max=324, sum=648 (2)\", \"tab\": \"General information\", \"score\": \"324.0\"}", - "Prehistory - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Prehistory - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Prehistory - # prompt tokens": "{\"description\": \"min=545.639, mean=545.639, max=545.639, sum=1091.278 (2)\", \"tab\": \"General information\", \"score\": \"545.6388888888889\"}", - "Prehistory - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"prehistory\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_prehistory\"" - } - } - }, - { - "evaluation_name": "Public Relations", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Public Relations", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.727, - "details": { - "description": "min=0.727, mean=0.727, max=0.727, sum=1.455 (2)", - "tab": "Accuracy", - "Public Relations - Observed inference time (s)": "{\"description\": \"min=0.941, mean=0.941, max=0.941, sum=1.882 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9410657709295099\"}", - "Public Relations - # eval": "{\"description\": \"min=110, mean=110, max=110, sum=220 (2)\", \"tab\": \"General information\", \"score\": \"110.0\"}", - "Public Relations - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Public Relations - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Public Relations - # prompt tokens": "{\"description\": \"min=432.991, mean=432.991, max=432.991, sum=865.982 (2)\", \"tab\": \"General information\", \"score\": \"432.9909090909091\"}", - "Public Relations - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"public_relations\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_public_relations\"" - } - } - }, - { - "evaluation_name": "Security Studies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Security Studies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.792, - "details": { - "description": "min=0.792, mean=0.792, max=0.792, sum=1.584 (2)", - "tab": "Accuracy", - "Security Studies - Observed inference time (s)": "{\"description\": \"min=1.016, mean=1.016, max=1.016, sum=2.033 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.0164005843960509\"}", - "Security Studies - # eval": "{\"description\": \"min=245, mean=245, max=245, sum=490 (2)\", \"tab\": \"General information\", \"score\": \"245.0\"}", - "Security Studies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Security Studies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Security Studies - # prompt tokens": "{\"description\": \"min=1243.804, mean=1243.804, max=1243.804, sum=2487.608 (2)\", \"tab\": \"General information\", \"score\": \"1243.8040816326532\"}", - "Security Studies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"security_studies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_security_studies\"" - } - } - }, - { - "evaluation_name": "Sociology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Sociology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.905, - "details": { - "description": "min=0.905, mean=0.905, max=0.905, sum=1.811 (2)", - "tab": "Accuracy", - "Sociology - Observed inference time (s)": "{\"description\": \"min=0.976, mean=0.976, max=0.976, sum=1.952 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9757713939420026\"}", - "Sociology - # eval": "{\"description\": \"min=201, mean=201, max=201, sum=402 (2)\", \"tab\": \"General information\", \"score\": \"201.0\"}", - "Sociology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Sociology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Sociology - # prompt tokens": "{\"description\": \"min=467.274, mean=467.274, max=467.274, sum=934.547 (2)\", \"tab\": \"General information\", \"score\": \"467.27363184079604\"}", - "Sociology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"sociology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_sociology\"" - } - } - }, - { - "evaluation_name": "Virology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Virology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.566, - "details": { - "description": "min=0.566, mean=0.566, max=0.566, sum=1.133 (2)", - "tab": "Accuracy", - "Virology - Observed inference time (s)": "{\"description\": \"min=0.929, mean=0.929, max=0.929, sum=1.858 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9289331062730536\"}", - "Virology - # eval": "{\"description\": \"min=166, mean=166, max=166, sum=332 (2)\", \"tab\": \"General information\", \"score\": \"166.0\"}", - "Virology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Virology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Virology - # prompt tokens": "{\"description\": \"min=362.651, mean=362.651, max=362.651, sum=725.301 (2)\", \"tab\": \"General information\", \"score\": \"362.65060240963857\"}", - "Virology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"virology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_virology\"" - } - } - }, - { - "evaluation_name": "World Religions", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on World Religions", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.865, - "details": { - "description": "min=0.865, mean=0.865, max=0.865, sum=1.731 (2)", - "tab": "Accuracy", - "World Religions - Observed inference time (s)": "{\"description\": \"min=1.021, mean=1.021, max=1.021, sum=2.042 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.0208685663011339\"}", - "World Religions - # eval": "{\"description\": \"min=171, mean=171, max=171, sum=342 (2)\", \"tab\": \"General information\", \"score\": \"171.0\"}", - "World Religions - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "World Religions - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "World Religions - # prompt tokens": "{\"description\": \"min=293.018, mean=293.018, max=293.018, sum=586.035 (2)\", \"tab\": \"General information\", \"score\": \"293.0175438596491\"}", - "World Religions - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"world_religions\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_world_religions\"" - } - } - }, - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.128, - "details": { - "description": "", - "tab": "Efficiency" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/anthropic_claude-3-5-sonnet-20240620.json b/data/models/anthropic_claude-3-5-sonnet-20240620.json deleted file mode 100644 index db73167f2ee9a0684ae3211521c8476cfd4cde1f..0000000000000000000000000000000000000000 --- a/data/models/anthropic_claude-3-5-sonnet-20240620.json +++ /dev/null @@ -1,2170 +0,0 @@ -{ - "model_info": { - "name": "Claude 3.5 Sonnet 20240620", - "id": "anthropic/claude-3-5-sonnet-20240620", - "developer": "Anthropic", - "inference_platform": "unknown", - "additional_details": { - "model_id_aliases": [ - "Anthropic/claude-3-5-sonnet-20240620" - ] - } - }, - "evaluations": [ - { - "evaluation_id": "helm_lite/anthropic_claude-3-5-sonnet-20240620/1774096306.427425", - "retrieved_timestamp": "1774096306.427425", - "source_metadata": { - "source_name": "helm_lite", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_lite", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_lite", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.885, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.27392009987515603\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.746, - "details": { - "description": "min=0.746, mean=0.746, max=0.746, sum=0.746 (1)", - "tab": "Accuracy", - "NarrativeQA - Observed inference time (s)": "{\"description\": \"min=3.5, mean=3.5, max=3.5, sum=3.5 (1)\", \"tab\": \"Efficiency\", \"score\": \"3.5003784911733278\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=3672.741, mean=3672.741, max=3672.741, sum=3672.741 (1)\", \"tab\": \"General information\", \"score\": \"3672.7408450704224\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=7.854, mean=7.854, max=7.854, sum=7.854 (1)\", \"tab\": \"General information\", \"score\": \"7.853521126760564\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (closed-book)", - "source_data": { - "dataset_name": "NaturalQuestions (closed-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (closed-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.502, - "details": { - "description": "min=0.502, mean=0.502, max=0.502, sum=0.502 (1)", - "tab": "Accuracy", - "NaturalQuestions (open-book) - Observed inference time (s)": "{\"description\": \"min=1.834, mean=1.834, max=1.834, sum=1.834 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.8338699455261231\"}", - "NaturalQuestions (closed-book) - Observed inference time (s)": "{\"description\": \"min=0.739, mean=0.739, max=0.739, sum=0.739 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.738832370519638\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1736.799, mean=1736.799, max=1736.799, sum=1736.799 (1)\", \"tab\": \"General information\", \"score\": \"1736.799\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=11.135, mean=11.135, max=11.135, sum=11.135 (1)\", \"tab\": \"General information\", \"score\": \"11.135\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=144.259, mean=144.259, max=144.259, sum=144.259 (1)\", \"tab\": \"General information\", \"score\": \"144.259\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=6.069, mean=6.069, max=6.069, sum=6.069 (1)\", \"tab\": \"General information\", \"score\": \"6.069\"}" - } - }, - "generation_config": { - "additional_details": { - "mode": "\"closedbook\"" - } - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.972, - "details": { - "description": "min=0.972, mean=0.972, max=0.972, sum=0.972 (1)", - "tab": "Accuracy", - "OpenbookQA - Observed inference time (s)": "{\"description\": \"min=0.774, mean=0.774, max=0.774, sum=0.774 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.7740971641540527\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=272.79, mean=272.79, max=272.79, sum=272.79 (1)\", \"tab\": \"General information\", \"score\": \"272.79\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "dataset": "\"openbookqa\"", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.799, - "details": { - "description": "min=0.59, mean=0.799, max=0.96, sum=3.997 (5)", - "tab": "Accuracy", - "MMLU - Observed inference time (s)": "{\"description\": \"min=0.765, mean=0.824, max=0.973, sum=4.121 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.8242833791364703\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=379.26, mean=487.747, max=628.596, sum=2438.736 (5)\", \"tab\": \"General information\", \"score\": \"487.747298245614\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"college_chemistry\", \"computer_security\", \"econometrics\", \"us_foreign_policy\"]", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MATH", - "source_data": { - "dataset_name": "MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Equivalent (CoT) on MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.813, - "details": { - "description": "min=0.579, mean=0.813, max=0.953, sum=5.69 (7)", - "tab": "Accuracy", - "MATH - Observed inference time (s)": "{\"description\": \"min=2.231, mean=3.012, max=3.921, sum=21.081 (7)\", \"tab\": \"Efficiency\", \"score\": \"3.0116338881061275\"}", - "MATH - # eval": "{\"description\": \"min=30, mean=62.429, max=135, sum=437 (7)\", \"tab\": \"General information\", \"score\": \"62.42857142857143\"}", - "MATH - # train": "{\"description\": \"min=8, mean=8, max=8, sum=56 (7)\", \"tab\": \"General information\", \"score\": \"8.0\"}", - "MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MATH - # prompt tokens": "{\"description\": \"min=897.259, mean=1311.814, max=2329.808, sum=9182.699 (7)\", \"tab\": \"General information\", \"score\": \"1311.8141219676104\"}", - "MATH - # output tokens": "{\"description\": \"min=93.333, mean=143.948, max=207.442, sum=1007.635 (7)\", \"tab\": \"General information\", \"score\": \"143.9478793136688\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"algebra\", \"counting_and_probability\", \"geometry\", \"intermediate_algebra\", \"number_theory\", \"prealgebra\", \"precalculus\"]", - "level": "\"1\"", - "use_official_examples": "\"False\"", - "use_chain_of_thought": "\"True\"" - } - } - }, - { - "evaluation_name": "GSM8K", - "source_data": { - "dataset_name": "GSM8K", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on GSM8K", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.949, - "details": { - "description": "min=0.949, mean=0.949, max=0.949, sum=0.949 (1)", - "tab": "Accuracy", - "GSM8K - Observed inference time (s)": "{\"description\": \"min=3.163, mean=3.163, max=3.163, sum=3.163 (1)\", \"tab\": \"Efficiency\", \"score\": \"3.162740940093994\"}", - "GSM8K - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "GSM8K - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "GSM8K - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GSM8K - # prompt tokens": "{\"description\": \"min=938.712, mean=938.712, max=938.712, sum=938.712 (1)\", \"tab\": \"General information\", \"score\": \"938.712\"}", - "GSM8K - # output tokens": "{\"description\": \"min=165.163, mean=165.163, max=165.163, sum=165.163 (1)\", \"tab\": \"General information\", \"score\": \"165.163\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "LegalBench", - "source_data": { - "dataset_name": "LegalBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on LegalBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.707, - "details": { - "description": "min=0.455, mean=0.707, max=0.968, sum=3.533 (5)", - "tab": "Accuracy", - "LegalBench - Observed inference time (s)": "{\"description\": \"min=0.66, mean=1.474, max=4.297, sum=7.369 (5)\", \"tab\": \"Efficiency\", \"score\": \"1.473749651523724\"}", - "LegalBench - # eval": "{\"description\": \"min=95, mean=409.4, max=1000, sum=2047 (5)\", \"tab\": \"General information\", \"score\": \"409.4\"}", - "LegalBench - # train": "{\"description\": \"min=4, mean=4.8, max=5, sum=24 (5)\", \"tab\": \"General information\", \"score\": \"4.8\"}", - "LegalBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "LegalBench - # prompt tokens": "{\"description\": \"min=223.653, mean=1566.242, max=6437.398, sum=7831.208 (5)\", \"tab\": \"General information\", \"score\": \"1566.241581367783\"}", - "LegalBench - # output tokens": "{\"description\": \"min=1, mean=1.328, max=2.053, sum=6.638 (5)\", \"tab\": \"General information\", \"score\": \"1.3276925283235337\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"abercrombie\", \"corporate_lobbying\", \"function_of_decision_section\", \"international_citizenship_questions\", \"proa\"]" - } - } - }, - { - "evaluation_name": "MedQA", - "source_data": { - "dataset_name": "MedQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MedQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.825, - "details": { - "description": "min=0.825, mean=0.825, max=0.825, sum=0.825 (1)", - "tab": "Accuracy", - "MedQA - Observed inference time (s)": "{\"description\": \"min=1.199, mean=1.199, max=1.199, sum=1.199 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.1990809397953406\"}", - "MedQA - # eval": "{\"description\": \"min=503, mean=503, max=503, sum=503 (1)\", \"tab\": \"General information\", \"score\": \"503.0\"}", - "MedQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MedQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MedQA - # prompt tokens": "{\"description\": \"min=1036.437, mean=1036.437, max=1036.437, sum=1036.437 (1)\", \"tab\": \"General information\", \"score\": \"1036.4373757455269\"}", - "MedQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WMT 2014", - "source_data": { - "dataset_name": "WMT 2014", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "BLEU-4 on WMT 2014", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.229, - "details": { - "description": "min=0.181, mean=0.229, max=0.27, sum=1.145 (5)", - "tab": "Accuracy", - "WMT 2014 - Observed inference time (s)": "{\"description\": \"min=1.838, mean=1.923, max=2.007, sum=9.616 (5)\", \"tab\": \"Efficiency\", \"score\": \"1.9232725335746241\"}", - "WMT 2014 - # eval": "{\"description\": \"min=503, mean=568.8, max=832, sum=2844 (5)\", \"tab\": \"General information\", \"score\": \"568.8\"}", - "WMT 2014 - # train": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "WMT 2014 - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WMT 2014 - # prompt tokens": "{\"description\": \"min=141.406, mean=162.573, max=184.974, sum=812.866 (5)\", \"tab\": \"General information\", \"score\": \"162.5732207715247\"}", - "WMT 2014 - # output tokens": "{\"description\": \"min=24.282, mean=25.852, max=26.592, sum=129.259 (5)\", \"tab\": \"General information\", \"score\": \"25.85177875057348\"}" - } - }, - "generation_config": { - "additional_details": { - "language_pair": "[\"cs-en\", \"de-en\", \"fr-en\", \"hi-en\", \"ru-en\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_mmlu/anthropic_claude-3-5-sonnet-20240620/1774096312.00548", - "retrieved_timestamp": "1774096312.00548", - "source_metadata": { - "source_name": "helm_mmlu", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_mmlu", - "evaluation_results": [ - { - "evaluation_name": "MMLU All Subjects", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU All Subjects", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.865, - "details": { - "description": "min=0.58, mean=0.865, max=0.98, sum=98.656 (114)", - "tab": "Accuracy", - "MMLU All Subjects - Observed inference time (s)": "{\"description\": \"min=0.765, mean=1.1, max=3.433, sum=125.349 (114)\", \"tab\": \"Efficiency\", \"score\": \"1.099552619745469\"}", - "MMLU All Subjects - # eval": "{\"description\": \"min=100, mean=246.351, max=1534, sum=28084 (114)\", \"tab\": \"General information\", \"score\": \"246.35087719298247\"}", - "MMLU All Subjects - # train": "{\"description\": \"min=5, mean=5, max=5, sum=570 (114)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU All Subjects - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU All Subjects - # prompt tokens": "{\"description\": \"min=302.018, mean=647.288, max=2896.576, sum=73790.875 (114)\", \"tab\": \"General information\", \"score\": \"647.2883793758954\"}", - "MMLU All Subjects - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=114 (114)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_name": "Abstract Algebra", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Abstract Algebra", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.75, - "details": { - "description": "min=0.75, mean=0.75, max=0.75, sum=1.5 (2)", - "tab": "Accuracy", - "Abstract Algebra - Observed inference time (s)": "{\"description\": \"min=0.779, mean=0.779, max=0.779, sum=1.558 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7789034700393677\"}", - "Abstract Algebra - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Abstract Algebra - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Abstract Algebra - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Abstract Algebra - # prompt tokens": "{\"description\": \"min=379.26, mean=379.26, max=379.26, sum=758.52 (2)\", \"tab\": \"General information\", \"score\": \"379.26\"}", - "Abstract Algebra - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"abstract_algebra\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_abstract_algebra\"" - } - } - }, - { - "evaluation_name": "Anatomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Anatomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.844, - "details": { - "description": "min=0.844, mean=0.844, max=0.844, sum=1.689 (2)", - "tab": "Accuracy", - "Anatomy - Observed inference time (s)": "{\"description\": \"min=0.777, mean=0.777, max=0.777, sum=1.553 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7767299599117703\"}", - "Anatomy - # eval": "{\"description\": \"min=135, mean=135, max=135, sum=270 (2)\", \"tab\": \"General information\", \"score\": \"135.0\"}", - "Anatomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Anatomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Anatomy - # prompt tokens": "{\"description\": \"min=379.8, mean=379.8, max=379.8, sum=759.6 (2)\", \"tab\": \"General information\", \"score\": \"379.8\"}", - "Anatomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"anatomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_anatomy\"" - } - } - }, - { - "evaluation_name": "College Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on College Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.696, - "details": { - "description": "min=0.696, mean=0.696, max=0.696, sum=1.392 (2)", - "tab": "Accuracy", - "College Chemistry - Observed inference time (s)": "{\"description\": \"min=0.797, mean=0.797, max=0.797, sum=1.594 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7968128871917725\"}", - "College Biology - Observed inference time (s)": "{\"description\": \"min=1.09, mean=1.09, max=1.09, sum=2.18 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.0898179478115506\"}", - "College Computer Science - Observed inference time (s)": "{\"description\": \"min=1.27, mean=1.27, max=1.27, sum=2.539 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.2695734238624572\"}", - "College Mathematics - Observed inference time (s)": "{\"description\": \"min=1.72, mean=1.72, max=1.72, sum=3.439 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.7196030735969543\"}", - "College Medicine - Observed inference time (s)": "{\"description\": \"min=1.28, mean=1.28, max=1.28, sum=2.559 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.2795469209637944\"}", - "College Physics - Observed inference time (s)": "{\"description\": \"min=0.796, mean=0.796, max=0.796, sum=1.591 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7955308311125812\"}", - "College Chemistry - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Chemistry - # prompt tokens": "{\"description\": \"min=559.01, mean=559.01, max=559.01, sum=1118.02 (2)\", \"tab\": \"General information\", \"score\": \"559.01\"}", - "College Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Biology - # eval": "{\"description\": \"min=144, mean=144, max=144, sum=288 (2)\", \"tab\": \"General information\", \"score\": \"144.0\"}", - "College Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # prompt tokens": "{\"description\": \"min=499.347, mean=499.347, max=499.347, sum=998.694 (2)\", \"tab\": \"General information\", \"score\": \"499.34722222222223\"}", - "College Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # prompt tokens": "{\"description\": \"min=847.24, mean=847.24, max=847.24, sum=1694.48 (2)\", \"tab\": \"General information\", \"score\": \"847.24\"}", - "College Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Mathematics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # prompt tokens": "{\"description\": \"min=613.19, mean=613.19, max=613.19, sum=1226.38 (2)\", \"tab\": \"General information\", \"score\": \"613.19\"}", - "College Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Medicine - # eval": "{\"description\": \"min=173, mean=173, max=173, sum=346 (2)\", \"tab\": \"General information\", \"score\": \"173.0\"}", - "College Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # prompt tokens": "{\"description\": \"min=549.63, mean=549.63, max=549.63, sum=1099.26 (2)\", \"tab\": \"General information\", \"score\": \"549.6300578034682\"}", - "College Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Physics - # eval": "{\"description\": \"min=102, mean=102, max=102, sum=204 (2)\", \"tab\": \"General information\", \"score\": \"102.0\"}", - "College Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # prompt tokens": "{\"description\": \"min=498.48, mean=498.48, max=498.48, sum=996.961 (2)\", \"tab\": \"General information\", \"score\": \"498.48039215686276\"}", - "College Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"college_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_college_physics\"" - } - } - }, - { - "evaluation_name": "Computer Security", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Computer Security", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.89, - "details": { - "description": "min=0.89, mean=0.89, max=0.89, sum=1.78 (2)", - "tab": "Accuracy", - "Computer Security - Observed inference time (s)": "{\"description\": \"min=0.765, mean=0.765, max=0.765, sum=1.531 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7653794264793397\"}", - "Computer Security - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Computer Security - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Computer Security - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Computer Security - # prompt tokens": "{\"description\": \"min=407.62, mean=407.62, max=407.62, sum=815.24 (2)\", \"tab\": \"General information\", \"score\": \"407.62\"}", - "Computer Security - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"computer_security\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_computer_security\"" - } - } - }, - { - "evaluation_name": "Econometrics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Econometrics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.807, - "details": { - "description": "min=0.807, mean=0.807, max=0.807, sum=1.614 (2)", - "tab": "Accuracy", - "Econometrics - Observed inference time (s)": "{\"description\": \"min=0.808, mean=0.808, max=0.808, sum=1.615 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8075556734152007\"}", - "Econometrics - # eval": "{\"description\": \"min=114, mean=114, max=114, sum=228 (2)\", \"tab\": \"General information\", \"score\": \"114.0\"}", - "Econometrics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Econometrics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Econometrics - # prompt tokens": "{\"description\": \"min=628.596, mean=628.596, max=628.596, sum=1257.193 (2)\", \"tab\": \"General information\", \"score\": \"628.5964912280701\"}", - "Econometrics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"econometrics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_econometrics\"" - } - } - }, - { - "evaluation_name": "Global Facts", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Global Facts", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.72, - "details": { - "description": "min=0.72, mean=0.72, max=0.72, sum=1.44 (2)", - "tab": "Accuracy", - "Global Facts - Observed inference time (s)": "{\"description\": \"min=0.785, mean=0.785, max=0.785, sum=1.571 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.785265531539917\"}", - "Global Facts - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Global Facts - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Global Facts - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Global Facts - # prompt tokens": "{\"description\": \"min=420.61, mean=420.61, max=420.61, sum=841.22 (2)\", \"tab\": \"General information\", \"score\": \"420.61\"}", - "Global Facts - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"global_facts\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_global_facts\"" - } - } - }, - { - "evaluation_name": "Jurisprudence", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Jurisprudence", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.889, - "details": { - "description": "min=0.889, mean=0.889, max=0.889, sum=1.778 (2)", - "tab": "Accuracy", - "Jurisprudence - Observed inference time (s)": "{\"description\": \"min=0.781, mean=0.781, max=0.781, sum=1.563 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7813034631587841\"}", - "Jurisprudence - # eval": "{\"description\": \"min=108, mean=108, max=108, sum=216 (2)\", \"tab\": \"General information\", \"score\": \"108.0\"}", - "Jurisprudence - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Jurisprudence - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Jurisprudence - # prompt tokens": "{\"description\": \"min=440.426, mean=440.426, max=440.426, sum=880.852 (2)\", \"tab\": \"General information\", \"score\": \"440.4259259259259\"}", - "Jurisprudence - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"jurisprudence\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_jurisprudence\"" - } - } - }, - { - "evaluation_name": "Philosophy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Philosophy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.891, - "details": { - "description": "min=0.891, mean=0.891, max=0.891, sum=1.781 (2)", - "tab": "Accuracy", - "Philosophy - Observed inference time (s)": "{\"description\": \"min=2.168, mean=2.168, max=2.168, sum=4.336 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.1680153757812892\"}", - "Philosophy - # eval": "{\"description\": \"min=311, mean=311, max=311, sum=622 (2)\", \"tab\": \"General information\", \"score\": \"311.0\"}", - "Philosophy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Philosophy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Philosophy - # prompt tokens": "{\"description\": \"min=368.965, mean=368.965, max=368.965, sum=737.929 (2)\", \"tab\": \"General information\", \"score\": \"368.9646302250804\"}", - "Philosophy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"philosophy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_philosophy\"" - } - } - }, - { - "evaluation_name": "Professional Psychology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Professional Psychology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.922, - "details": { - "description": "min=0.922, mean=0.922, max=0.922, sum=1.843 (2)", - "tab": "Accuracy", - "Professional Medicine - Observed inference time (s)": "{\"description\": \"min=2.144, mean=2.144, max=2.144, sum=4.287 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.1436235790743545\"}", - "Professional Accounting - Observed inference time (s)": "{\"description\": \"min=2.085, mean=2.085, max=2.085, sum=4.169 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.084580805284757\"}", - "Professional Law - Observed inference time (s)": "{\"description\": \"min=1.308, mean=1.308, max=1.308, sum=2.616 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.3078198053690726\"}", - "Professional Psychology - Observed inference time (s)": "{\"description\": \"min=1.15, mean=1.15, max=1.15, sum=2.301 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.1502779430034114\"}", - "Professional Medicine - # eval": "{\"description\": \"min=272, mean=272, max=272, sum=544 (2)\", \"tab\": \"General information\", \"score\": \"272.0\"}", - "Professional Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Medicine - # prompt tokens": "{\"description\": \"min=1132.537, mean=1132.537, max=1132.537, sum=2265.074 (2)\", \"tab\": \"General information\", \"score\": \"1132.5367647058824\"}", - "Professional Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Accounting - # eval": "{\"description\": \"min=282, mean=282, max=282, sum=564 (2)\", \"tab\": \"General information\", \"score\": \"282.0\"}", - "Professional Accounting - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Accounting - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # prompt tokens": "{\"description\": \"min=674.422, mean=674.422, max=674.422, sum=1348.844 (2)\", \"tab\": \"General information\", \"score\": \"674.4219858156029\"}", - "Professional Accounting - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Law - # eval": "{\"description\": \"min=1534, mean=1534, max=1534, sum=3068 (2)\", \"tab\": \"General information\", \"score\": \"1534.0\"}", - "Professional Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # prompt tokens": "{\"description\": \"min=1710.16, mean=1710.16, max=1710.16, sum=3420.321 (2)\", \"tab\": \"General information\", \"score\": \"1710.16036505867\"}", - "Professional Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Psychology - # eval": "{\"description\": \"min=612, mean=612, max=612, sum=1224 (2)\", \"tab\": \"General information\", \"score\": \"612.0\"}", - "Professional Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # prompt tokens": "{\"description\": \"min=612.168, mean=612.168, max=612.168, sum=1224.337 (2)\", \"tab\": \"General information\", \"score\": \"612.1683006535948\"}", - "Professional Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"professional_psychology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_professional_psychology\"" - } - } - }, - { - "evaluation_name": "Us Foreign Policy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Us Foreign Policy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.96, - "details": { - "description": "min=0.96, mean=0.96, max=0.96, sum=1.92 (2)", - "tab": "Accuracy", - "Us Foreign Policy - Observed inference time (s)": "{\"description\": \"min=0.973, mean=0.973, max=0.973, sum=1.946 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9727654385566712\"}", - "Us Foreign Policy - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Us Foreign Policy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Us Foreign Policy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Us Foreign Policy - # prompt tokens": "{\"description\": \"min=464.25, mean=464.25, max=464.25, sum=928.5 (2)\", \"tab\": \"General information\", \"score\": \"464.25\"}", - "Us Foreign Policy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"us_foreign_policy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_us_foreign_policy\"" - } - } - }, - { - "evaluation_name": "Astronomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Astronomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.961, - "details": { - "description": "min=0.961, mean=0.961, max=0.961, sum=1.921 (2)", - "tab": "Accuracy", - "Astronomy - Observed inference time (s)": "{\"description\": \"min=1.35, mean=1.35, max=1.35, sum=2.7 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.3501500989261426\"}", - "Astronomy - # eval": "{\"description\": \"min=152, mean=152, max=152, sum=304 (2)\", \"tab\": \"General information\", \"score\": \"152.0\"}", - "Astronomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Astronomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Astronomy - # prompt tokens": "{\"description\": \"min=613.493, mean=613.493, max=613.493, sum=1226.987 (2)\", \"tab\": \"General information\", \"score\": \"613.4934210526316\"}", - "Astronomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"astronomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_astronomy\"" - } - } - }, - { - "evaluation_name": "Business Ethics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Business Ethics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.85, - "details": { - "description": "min=0.85, mean=0.85, max=0.85, sum=1.7 (2)", - "tab": "Accuracy", - "Business Ethics - Observed inference time (s)": "{\"description\": \"min=1.326, mean=1.326, max=1.326, sum=2.652 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.325816671848297\"}", - "Business Ethics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Business Ethics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Business Ethics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Business Ethics - # prompt tokens": "{\"description\": \"min=609.02, mean=609.02, max=609.02, sum=1218.04 (2)\", \"tab\": \"General information\", \"score\": \"609.02\"}", - "Business Ethics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"business_ethics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_business_ethics\"" - } - } - }, - { - "evaluation_name": "Clinical Knowledge", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Clinical Knowledge", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.913, - "details": { - "description": "min=0.913, mean=0.913, max=0.913, sum=1.826 (2)", - "tab": "Accuracy", - "Clinical Knowledge - Observed inference time (s)": "{\"description\": \"min=1.379, mean=1.379, max=1.379, sum=2.757 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.3787489792086043\"}", - "Clinical Knowledge - # eval": "{\"description\": \"min=265, mean=265, max=265, sum=530 (2)\", \"tab\": \"General information\", \"score\": \"265.0\"}", - "Clinical Knowledge - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Clinical Knowledge - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Clinical Knowledge - # prompt tokens": "{\"description\": \"min=438.457, mean=438.457, max=438.457, sum=876.913 (2)\", \"tab\": \"General information\", \"score\": \"438.4566037735849\"}", - "Clinical Knowledge - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"clinical_knowledge\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_clinical_knowledge\"" - } - } - }, - { - "evaluation_name": "Conceptual Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Conceptual Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.885, - "details": { - "description": "min=0.885, mean=0.885, max=0.885, sum=1.77 (2)", - "tab": "Accuracy", - "Conceptual Physics - Observed inference time (s)": "{\"description\": \"min=0.778, mean=0.778, max=0.778, sum=1.556 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7780434922969087\"}", - "Conceptual Physics - # eval": "{\"description\": \"min=235, mean=235, max=235, sum=470 (2)\", \"tab\": \"General information\", \"score\": \"235.0\"}", - "Conceptual Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Conceptual Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Conceptual Physics - # prompt tokens": "{\"description\": \"min=332.536, mean=332.536, max=332.536, sum=665.072 (2)\", \"tab\": \"General information\", \"score\": \"332.53617021276597\"}", - "Conceptual Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"conceptual_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_conceptual_physics\"" - } - } - }, - { - "evaluation_name": "Electrical Engineering", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Electrical Engineering", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.828, - "details": { - "description": "min=0.828, mean=0.828, max=0.828, sum=1.655 (2)", - "tab": "Accuracy", - "Electrical Engineering - Observed inference time (s)": "{\"description\": \"min=0.79, mean=0.79, max=0.79, sum=1.58 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.789771790340029\"}", - "Electrical Engineering - # eval": "{\"description\": \"min=145, mean=145, max=145, sum=290 (2)\", \"tab\": \"General information\", \"score\": \"145.0\"}", - "Electrical Engineering - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Electrical Engineering - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Electrical Engineering - # prompt tokens": "{\"description\": \"min=446.041, mean=446.041, max=446.041, sum=892.083 (2)\", \"tab\": \"General information\", \"score\": \"446.04137931034484\"}", - "Electrical Engineering - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"electrical_engineering\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_electrical_engineering\"" - } - } - }, - { - "evaluation_name": "Elementary Mathematics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Elementary Mathematics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.892, - "details": { - "description": "min=0.892, mean=0.892, max=0.892, sum=1.783 (2)", - "tab": "Accuracy", - "Elementary Mathematics - Observed inference time (s)": "{\"description\": \"min=0.806, mean=0.806, max=0.806, sum=1.612 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8060284802522609\"}", - "Elementary Mathematics - # eval": "{\"description\": \"min=378, mean=378, max=378, sum=756 (2)\", \"tab\": \"General information\", \"score\": \"378.0\"}", - "Elementary Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Elementary Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Elementary Mathematics - # prompt tokens": "{\"description\": \"min=520.066, mean=520.066, max=520.066, sum=1040.132 (2)\", \"tab\": \"General information\", \"score\": \"520.0661375661375\"}", - "Elementary Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"elementary_mathematics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_elementary_mathematics\"" - } - } - }, - { - "evaluation_name": "Formal Logic", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Formal Logic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.698, - "details": { - "description": "min=0.698, mean=0.698, max=0.698, sum=1.397 (2)", - "tab": "Accuracy", - "Formal Logic - Observed inference time (s)": "{\"description\": \"min=0.811, mean=0.811, max=0.811, sum=1.623 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8114165843479217\"}", - "Formal Logic - # eval": "{\"description\": \"min=126, mean=126, max=126, sum=252 (2)\", \"tab\": \"General information\", \"score\": \"126.0\"}", - "Formal Logic - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Formal Logic - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Formal Logic - # prompt tokens": "{\"description\": \"min=655.746, mean=655.746, max=655.746, sum=1311.492 (2)\", \"tab\": \"General information\", \"score\": \"655.7460317460317\"}", - "Formal Logic - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"formal_logic\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_formal_logic\"" - } - } - }, - { - "evaluation_name": "High School World History", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on High School World History", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.954, - "details": { - "description": "min=0.954, mean=0.954, max=0.954, sum=1.907 (2)", - "tab": "Accuracy", - "High School Biology - Observed inference time (s)": "{\"description\": \"min=0.802, mean=0.802, max=0.802, sum=1.605 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8022696918056857\"}", - "High School Chemistry - Observed inference time (s)": "{\"description\": \"min=0.806, mean=0.806, max=0.806, sum=1.612 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8062427619407917\"}", - "High School Computer Science - Observed inference time (s)": "{\"description\": \"min=0.853, mean=0.853, max=0.853, sum=1.706 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8532347416877747\"}", - "High School European History - Observed inference time (s)": "{\"description\": \"min=1.183, mean=1.183, max=1.183, sum=2.366 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.1831647526134144\"}", - "High School Geography - Observed inference time (s)": "{\"description\": \"min=0.776, mean=0.776, max=0.776, sum=1.553 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7764992966796412\"}", - "High School Government And Politics - Observed inference time (s)": "{\"description\": \"min=0.802, mean=0.802, max=0.802, sum=1.603 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8015919287587695\"}", - "High School Macroeconomics - Observed inference time (s)": "{\"description\": \"min=0.782, mean=0.782, max=0.782, sum=1.563 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.781673603791457\"}", - "High School Mathematics - Observed inference time (s)": "{\"description\": \"min=0.805, mean=0.805, max=0.805, sum=1.61 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.80511144178885\"}", - "High School Microeconomics - Observed inference time (s)": "{\"description\": \"min=0.788, mean=0.788, max=0.788, sum=1.576 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7879440243504628\"}", - "High School Physics - Observed inference time (s)": "{\"description\": \"min=0.829, mean=0.829, max=0.829, sum=1.658 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8290448062467259\"}", - "High School Psychology - Observed inference time (s)": "{\"description\": \"min=0.807, mean=0.807, max=0.807, sum=1.614 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8071829231507187\"}", - "High School Statistics - Observed inference time (s)": "{\"description\": \"min=0.812, mean=0.812, max=0.812, sum=1.624 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8119496272669898\"}", - "High School US History - Observed inference time (s)": "{\"description\": \"min=0.938, mean=0.938, max=0.938, sum=1.877 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9383000193857679\"}", - "High School World History - Observed inference time (s)": "{\"description\": \"min=1.097, mean=1.097, max=1.097, sum=2.194 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.0968722401791986\"}", - "High School Biology - # eval": "{\"description\": \"min=310, mean=310, max=310, sum=620 (2)\", \"tab\": \"General information\", \"score\": \"310.0\"}", - "High School Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Biology - # prompt tokens": "{\"description\": \"min=543.577, mean=543.577, max=543.577, sum=1087.155 (2)\", \"tab\": \"General information\", \"score\": \"543.5774193548388\"}", - "High School Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Chemistry - # eval": "{\"description\": \"min=203, mean=203, max=203, sum=406 (2)\", \"tab\": \"General information\", \"score\": \"203.0\"}", - "High School Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # prompt tokens": "{\"description\": \"min=506.921, mean=506.921, max=506.921, sum=1013.842 (2)\", \"tab\": \"General information\", \"score\": \"506.92118226600985\"}", - "High School Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "High School Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # prompt tokens": "{\"description\": \"min=891.4, mean=891.4, max=891.4, sum=1782.8 (2)\", \"tab\": \"General information\", \"score\": \"891.4\"}", - "High School Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School European History - # eval": "{\"description\": \"min=165, mean=165, max=165, sum=330 (2)\", \"tab\": \"General information\", \"score\": \"165.0\"}", - "High School European History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School European History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # prompt tokens": "{\"description\": \"min=2896.576, mean=2896.576, max=2896.576, sum=5793.152 (2)\", \"tab\": \"General information\", \"score\": \"2896.5757575757575\"}", - "High School European History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Geography - # eval": "{\"description\": \"min=198, mean=198, max=198, sum=396 (2)\", \"tab\": \"General information\", \"score\": \"198.0\"}", - "High School Geography - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Geography - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # prompt tokens": "{\"description\": \"min=421.268, mean=421.268, max=421.268, sum=842.535 (2)\", \"tab\": \"General information\", \"score\": \"421.2676767676768\"}", - "High School Geography - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Government And Politics - # eval": "{\"description\": \"min=193, mean=193, max=193, sum=386 (2)\", \"tab\": \"General information\", \"score\": \"193.0\"}", - "High School Government And Politics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Government And Politics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # prompt tokens": "{\"description\": \"min=500.104, mean=500.104, max=500.104, sum=1000.207 (2)\", \"tab\": \"General information\", \"score\": \"500.10362694300517\"}", - "High School Government And Politics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Macroeconomics - # eval": "{\"description\": \"min=390, mean=390, max=390, sum=780 (2)\", \"tab\": \"General information\", \"score\": \"390.0\"}", - "High School Macroeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Macroeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # prompt tokens": "{\"description\": \"min=415.036, mean=415.036, max=415.036, sum=830.072 (2)\", \"tab\": \"General information\", \"score\": \"415.0358974358974\"}", - "High School Macroeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Mathematics - # eval": "{\"description\": \"min=270, mean=270, max=270, sum=540 (2)\", \"tab\": \"General information\", \"score\": \"270.0\"}", - "High School Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # prompt tokens": "{\"description\": \"min=528.881, mean=528.881, max=528.881, sum=1057.763 (2)\", \"tab\": \"General information\", \"score\": \"528.8814814814815\"}", - "High School Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Microeconomics - # eval": "{\"description\": \"min=238, mean=238, max=238, sum=476 (2)\", \"tab\": \"General information\", \"score\": \"238.0\"}", - "High School Microeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Microeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # prompt tokens": "{\"description\": \"min=429.513, mean=429.513, max=429.513, sum=859.025 (2)\", \"tab\": \"General information\", \"score\": \"429.5126050420168\"}", - "High School Microeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Physics - # eval": "{\"description\": \"min=151, mean=151, max=151, sum=302 (2)\", \"tab\": \"General information\", \"score\": \"151.0\"}", - "High School Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # prompt tokens": "{\"description\": \"min=567.841, mean=567.841, max=567.841, sum=1135.682 (2)\", \"tab\": \"General information\", \"score\": \"567.841059602649\"}", - "High School Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Psychology - # eval": "{\"description\": \"min=545, mean=545, max=545, sum=1090 (2)\", \"tab\": \"General information\", \"score\": \"545.0\"}", - "High School Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # prompt tokens": "{\"description\": \"min=530.42, mean=530.42, max=530.42, sum=1060.84 (2)\", \"tab\": \"General information\", \"score\": \"530.4201834862386\"}", - "High School Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Statistics - # eval": "{\"description\": \"min=216, mean=216, max=216, sum=432 (2)\", \"tab\": \"General information\", \"score\": \"216.0\"}", - "High School Statistics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Statistics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # prompt tokens": "{\"description\": \"min=815.963, mean=815.963, max=815.963, sum=1631.926 (2)\", \"tab\": \"General information\", \"score\": \"815.9629629629629\"}", - "High School Statistics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School US History - # eval": "{\"description\": \"min=204, mean=204, max=204, sum=408 (2)\", \"tab\": \"General information\", \"score\": \"204.0\"}", - "High School US History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School US History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # prompt tokens": "{\"description\": \"min=2297.49, mean=2297.49, max=2297.49, sum=4594.98 (2)\", \"tab\": \"General information\", \"score\": \"2297.4901960784314\"}", - "High School US History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School World History - # eval": "{\"description\": \"min=237, mean=237, max=237, sum=474 (2)\", \"tab\": \"General information\", \"score\": \"237.0\"}", - "High School World History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School World History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # prompt tokens": "{\"description\": \"min=1484.932, mean=1484.932, max=1484.932, sum=2969.865 (2)\", \"tab\": \"General information\", \"score\": \"1484.9324894514768\"}", - "High School World History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"high_school_world_history\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_high_school_world_history\"" - } - } - }, - { - "evaluation_name": "Human Sexuality", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Human Sexuality", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.939, - "details": { - "description": "min=0.939, mean=0.939, max=0.939, sum=1.878 (2)", - "tab": "Accuracy", - "Human Aging - Observed inference time (s)": "{\"description\": \"min=0.785, mean=0.785, max=0.785, sum=1.569 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7847084699724822\"}", - "Human Sexuality - Observed inference time (s)": "{\"description\": \"min=0.811, mean=0.811, max=0.811, sum=1.622 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8110958565282458\"}", - "Human Aging - # eval": "{\"description\": \"min=223, mean=223, max=223, sum=446 (2)\", \"tab\": \"General information\", \"score\": \"223.0\"}", - "Human Aging - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Aging - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Aging - # prompt tokens": "{\"description\": \"min=344.955, mean=344.955, max=344.955, sum=689.91 (2)\", \"tab\": \"General information\", \"score\": \"344.95515695067263\"}", - "Human Aging - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Human Sexuality - # eval": "{\"description\": \"min=131, mean=131, max=131, sum=262 (2)\", \"tab\": \"General information\", \"score\": \"131.0\"}", - "Human Sexuality - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Sexuality - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # prompt tokens": "{\"description\": \"min=380.496, mean=380.496, max=380.496, sum=760.992 (2)\", \"tab\": \"General information\", \"score\": \"380.4961832061069\"}", - "Human Sexuality - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"human_sexuality\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_human_sexuality\"" - } - } - }, - { - "evaluation_name": "International Law", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on International Law", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.959, - "details": { - "description": "min=0.959, mean=0.959, max=0.959, sum=1.917 (2)", - "tab": "Accuracy", - "International Law - Observed inference time (s)": "{\"description\": \"min=0.822, mean=0.822, max=0.822, sum=1.644 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8220856209431798\"}", - "International Law - # eval": "{\"description\": \"min=121, mean=121, max=121, sum=242 (2)\", \"tab\": \"General information\", \"score\": \"121.0\"}", - "International Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "International Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "International Law - # prompt tokens": "{\"description\": \"min=673.165, mean=673.165, max=673.165, sum=1346.331 (2)\", \"tab\": \"General information\", \"score\": \"673.1652892561983\"}", - "International Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"international_law\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_international_law\"" - } - } - }, - { - "evaluation_name": "Logical Fallacies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Logical Fallacies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.926, - "details": { - "description": "min=0.926, mean=0.926, max=0.926, sum=1.853 (2)", - "tab": "Accuracy", - "Logical Fallacies - Observed inference time (s)": "{\"description\": \"min=0.778, mean=0.778, max=0.778, sum=1.556 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.778087305876375\"}", - "Logical Fallacies - # eval": "{\"description\": \"min=163, mean=163, max=163, sum=326 (2)\", \"tab\": \"General information\", \"score\": \"163.0\"}", - "Logical Fallacies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Logical Fallacies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Logical Fallacies - # prompt tokens": "{\"description\": \"min=479.276, mean=479.276, max=479.276, sum=958.552 (2)\", \"tab\": \"General information\", \"score\": \"479.2760736196319\"}", - "Logical Fallacies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"logical_fallacies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_logical_fallacies\"" - } - } - }, - { - "evaluation_name": "Machine Learning", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Machine Learning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.786, - "details": { - "description": "min=0.786, mean=0.786, max=0.786, sum=1.571 (2)", - "tab": "Accuracy", - "Machine Learning - Observed inference time (s)": "{\"description\": \"min=0.81, mean=0.81, max=0.81, sum=1.619 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.809621695961271\"}", - "Machine Learning - # eval": "{\"description\": \"min=112, mean=112, max=112, sum=224 (2)\", \"tab\": \"General information\", \"score\": \"112.0\"}", - "Machine Learning - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Machine Learning - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Machine Learning - # prompt tokens": "{\"description\": \"min=685.518, mean=685.518, max=685.518, sum=1371.036 (2)\", \"tab\": \"General information\", \"score\": \"685.5178571428571\"}", - "Machine Learning - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"machine_learning\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_machine_learning\"" - } - } - }, - { - "evaluation_name": "Management", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Management", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.942, - "details": { - "description": "min=0.942, mean=0.942, max=0.942, sum=1.883 (2)", - "tab": "Accuracy", - "Management - Observed inference time (s)": "{\"description\": \"min=0.848, mean=0.848, max=0.848, sum=1.696 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8480523350169358\"}", - "Management - # eval": "{\"description\": \"min=103, mean=103, max=103, sum=206 (2)\", \"tab\": \"General information\", \"score\": \"103.0\"}", - "Management - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Management - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Management - # prompt tokens": "{\"description\": \"min=310.282, mean=310.282, max=310.282, sum=620.563 (2)\", \"tab\": \"General information\", \"score\": \"310.28155339805824\"}", - "Management - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"management\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_management\"" - } - } - }, - { - "evaluation_name": "Marketing", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Marketing", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.949, - "details": { - "description": "min=0.949, mean=0.949, max=0.949, sum=1.897 (2)", - "tab": "Accuracy", - "Marketing - Observed inference time (s)": "{\"description\": \"min=2.55, mean=2.55, max=2.55, sum=5.1 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.550003965695699\"}", - "Marketing - # eval": "{\"description\": \"min=234, mean=234, max=234, sum=468 (2)\", \"tab\": \"General information\", \"score\": \"234.0\"}", - "Marketing - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Marketing - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Marketing - # prompt tokens": "{\"description\": \"min=457.064, mean=457.064, max=457.064, sum=914.128 (2)\", \"tab\": \"General information\", \"score\": \"457.06410256410254\"}", - "Marketing - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"marketing\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_marketing\"" - } - } - }, - { - "evaluation_name": "Medical Genetics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Medical Genetics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.98, - "details": { - "description": "min=0.98, mean=0.98, max=0.98, sum=1.96 (2)", - "tab": "Accuracy", - "Medical Genetics - Observed inference time (s)": "{\"description\": \"min=3.433, mean=3.433, max=3.433, sum=6.867 (2)\", \"tab\": \"Efficiency\", \"score\": \"3.4333492875099183\"}", - "Medical Genetics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Medical Genetics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Medical Genetics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Medical Genetics - # prompt tokens": "{\"description\": \"min=363.88, mean=363.88, max=363.88, sum=727.76 (2)\", \"tab\": \"General information\", \"score\": \"363.88\"}", - "Medical Genetics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"medical_genetics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_medical_genetics\"" - } - } - }, - { - "evaluation_name": "Miscellaneous", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Miscellaneous", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.962, - "details": { - "description": "min=0.962, mean=0.962, max=0.962, sum=1.923 (2)", - "tab": "Accuracy", - "Miscellaneous - Observed inference time (s)": "{\"description\": \"min=1.474, mean=1.474, max=1.474, sum=2.949 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.4744500937285248\"}", - "Miscellaneous - # eval": "{\"description\": \"min=783, mean=783, max=783, sum=1566 (2)\", \"tab\": \"General information\", \"score\": \"783.0\"}", - "Miscellaneous - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Miscellaneous - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Miscellaneous - # prompt tokens": "{\"description\": \"min=337.628, mean=337.628, max=337.628, sum=675.257 (2)\", \"tab\": \"General information\", \"score\": \"337.62835249042143\"}", - "Miscellaneous - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"miscellaneous\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_miscellaneous\"" - } - } - }, - { - "evaluation_name": "Moral Scenarios", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Moral Scenarios", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.882, - "details": { - "description": "min=0.882, mean=0.882, max=0.882, sum=1.763 (2)", - "tab": "Accuracy", - "Moral Disputes - Observed inference time (s)": "{\"description\": \"min=0.817, mean=0.817, max=0.817, sum=1.635 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8173547728213272\"}", - "Moral Scenarios - Observed inference time (s)": "{\"description\": \"min=1.043, mean=1.043, max=1.043, sum=2.085 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.0425983404980026\"}", - "Moral Disputes - # eval": "{\"description\": \"min=346, mean=346, max=346, sum=692 (2)\", \"tab\": \"General information\", \"score\": \"346.0\"}", - "Moral Disputes - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Disputes - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Disputes - # prompt tokens": "{\"description\": \"min=520.789, mean=520.789, max=520.789, sum=1041.578 (2)\", \"tab\": \"General information\", \"score\": \"520.7890173410404\"}", - "Moral Disputes - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Moral Scenarios - # eval": "{\"description\": \"min=895, mean=895, max=895, sum=1790 (2)\", \"tab\": \"General information\", \"score\": \"895.0\"}", - "Moral Scenarios - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Scenarios - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # prompt tokens": "{\"description\": \"min=685.949, mean=685.949, max=685.949, sum=1371.897 (2)\", \"tab\": \"General information\", \"score\": \"685.9486033519553\"}", - "Moral Scenarios - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"moral_scenarios\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_moral_scenarios\"" - } - } - }, - { - "evaluation_name": "Nutrition", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Nutrition", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.912, - "details": { - "description": "min=0.912, mean=0.912, max=0.912, sum=1.824 (2)", - "tab": "Accuracy", - "Nutrition - Observed inference time (s)": "{\"description\": \"min=0.987, mean=0.987, max=0.987, sum=1.973 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9867353338042116\"}", - "Nutrition - # eval": "{\"description\": \"min=306, mean=306, max=306, sum=612 (2)\", \"tab\": \"General information\", \"score\": \"306.0\"}", - "Nutrition - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Nutrition - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Nutrition - # prompt tokens": "{\"description\": \"min=626.065, mean=626.065, max=626.065, sum=1252.131 (2)\", \"tab\": \"General information\", \"score\": \"626.0653594771242\"}", - "Nutrition - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"nutrition\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_nutrition\"" - } - } - }, - { - "evaluation_name": "Prehistory", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Prehistory", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.951, - "details": { - "description": "min=0.951, mean=0.951, max=0.951, sum=1.901 (2)", - "tab": "Accuracy", - "Prehistory - Observed inference time (s)": "{\"description\": \"min=0.887, mean=0.887, max=0.887, sum=1.775 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8874673313564725\"}", - "Prehistory - # eval": "{\"description\": \"min=324, mean=324, max=324, sum=648 (2)\", \"tab\": \"General information\", \"score\": \"324.0\"}", - "Prehistory - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Prehistory - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Prehistory - # prompt tokens": "{\"description\": \"min=554.639, mean=554.639, max=554.639, sum=1109.278 (2)\", \"tab\": \"General information\", \"score\": \"554.6388888888889\"}", - "Prehistory - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"prehistory\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_prehistory\"" - } - } - }, - { - "evaluation_name": "Public Relations", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Public Relations", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.855, - "details": { - "description": "min=0.855, mean=0.855, max=0.855, sum=1.709 (2)", - "tab": "Accuracy", - "Public Relations - Observed inference time (s)": "{\"description\": \"min=1.124, mean=1.124, max=1.124, sum=2.248 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.1237782673402266\"}", - "Public Relations - # eval": "{\"description\": \"min=110, mean=110, max=110, sum=220 (2)\", \"tab\": \"General information\", \"score\": \"110.0\"}", - "Public Relations - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Public Relations - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Public Relations - # prompt tokens": "{\"description\": \"min=441.991, mean=441.991, max=441.991, sum=883.982 (2)\", \"tab\": \"General information\", \"score\": \"441.9909090909091\"}", - "Public Relations - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"public_relations\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_public_relations\"" - } - } - }, - { - "evaluation_name": "Security Studies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Security Studies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.878, - "details": { - "description": "min=0.878, mean=0.878, max=0.878, sum=1.755 (2)", - "tab": "Accuracy", - "Security Studies - Observed inference time (s)": "{\"description\": \"min=1.219, mean=1.219, max=1.219, sum=2.438 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.2191707075858602\"}", - "Security Studies - # eval": "{\"description\": \"min=245, mean=245, max=245, sum=490 (2)\", \"tab\": \"General information\", \"score\": \"245.0\"}", - "Security Studies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Security Studies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Security Studies - # prompt tokens": "{\"description\": \"min=1252.804, mean=1252.804, max=1252.804, sum=2505.608 (2)\", \"tab\": \"General information\", \"score\": \"1252.8040816326532\"}", - "Security Studies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"security_studies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_security_studies\"" - } - } - }, - { - "evaluation_name": "Sociology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Sociology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.96, - "details": { - "description": "min=0.96, mean=0.96, max=0.96, sum=1.92 (2)", - "tab": "Accuracy", - "Sociology - Observed inference time (s)": "{\"description\": \"min=1.141, mean=1.141, max=1.141, sum=2.282 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.141001319410789\"}", - "Sociology - # eval": "{\"description\": \"min=201, mean=201, max=201, sum=402 (2)\", \"tab\": \"General information\", \"score\": \"201.0\"}", - "Sociology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Sociology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Sociology - # prompt tokens": "{\"description\": \"min=476.274, mean=476.274, max=476.274, sum=952.547 (2)\", \"tab\": \"General information\", \"score\": \"476.27363184079604\"}", - "Sociology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"sociology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_sociology\"" - } - } - }, - { - "evaluation_name": "Virology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Virology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.602, - "details": { - "description": "min=0.602, mean=0.602, max=0.602, sum=1.205 (2)", - "tab": "Accuracy", - "Virology - Observed inference time (s)": "{\"description\": \"min=1.15, mean=1.15, max=1.15, sum=2.3 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.1499209547617348\"}", - "Virology - # eval": "{\"description\": \"min=166, mean=166, max=166, sum=332 (2)\", \"tab\": \"General information\", \"score\": \"166.0\"}", - "Virology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Virology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Virology - # prompt tokens": "{\"description\": \"min=371.651, mean=371.651, max=371.651, sum=743.301 (2)\", \"tab\": \"General information\", \"score\": \"371.65060240963857\"}", - "Virology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"virology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_virology\"" - } - } - }, - { - "evaluation_name": "World Religions", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on World Religions", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.924, - "details": { - "description": "min=0.924, mean=0.924, max=0.924, sum=1.848 (2)", - "tab": "Accuracy", - "World Religions - Observed inference time (s)": "{\"description\": \"min=1.201, mean=1.201, max=1.201, sum=2.402 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.200854153661003\"}", - "World Religions - # eval": "{\"description\": \"min=171, mean=171, max=171, sum=342 (2)\", \"tab\": \"General information\", \"score\": \"171.0\"}", - "World Religions - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "World Religions - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "World Religions - # prompt tokens": "{\"description\": \"min=302.018, mean=302.018, max=302.018, sum=604.035 (2)\", \"tab\": \"General information\", \"score\": \"302.0175438596491\"}", - "World Religions - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"world_religions\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_world_religions\"" - } - } - }, - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.17, - "details": { - "description": "", - "tab": "Efficiency" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_id": "reward-bench/Anthropic_claude-3-5-sonnet-20240620/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8417 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9637 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7401 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8162 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8469 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "reward-bench-2/anthropic_claude-3-5-sonnet-20240620/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6466 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5284 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3875 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5683 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8519 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8697 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.674 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/anthropic_claude-3-5-sonnet-20241022.json b/data/models/anthropic_claude-3-5-sonnet-20241022.json deleted file mode 100644 index 9debb907075ff1ad24714f8789258031ef4b0178..0000000000000000000000000000000000000000 --- a/data/models/anthropic_claude-3-5-sonnet-20241022.json +++ /dev/null @@ -1,2125 +0,0 @@ -{ - "model_info": { - "name": "Claude 3.5 Sonnet 20241022", - "id": "anthropic/claude-3-5-sonnet-20241022", - "developer": "Anthropic", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_capabilities/anthropic_claude-3-5-sonnet-20241022/1774096304.5056698", - "retrieved_timestamp": "1774096304.5056698", - "source_metadata": { - "source_name": "helm_capabilities", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_capabilities", - "evaluation_results": [ - { - "evaluation_name": "Mean score", - "source_data": { - "dataset_name": "helm_capabilities", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "The mean of the scores from all columns.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.653, - "details": { - "description": "", - "tab": "Accuracy", - "Mean score - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"7.355400399849929\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU-Pro", - "source_data": { - "dataset_name": "MMLU-Pro", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on MMLU-Pro", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.777, - "details": { - "description": "min=0.777, mean=0.777, max=0.777, sum=0.777 (1)", - "tab": "Accuracy", - "MMLU-Pro - Observed inference time (s)": "{\"description\": \"min=5.096, mean=5.096, max=5.096, sum=5.096 (1)\", \"tab\": \"Efficiency\", \"score\": \"5.096486385822296\"}", - "MMLU-Pro - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "MMLU-Pro - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - # prompt tokens": "{\"description\": \"min=230.461, mean=230.461, max=230.461, sum=230.461 (1)\", \"tab\": \"General information\", \"score\": \"230.461\"}", - "MMLU-Pro - # output tokens": "{\"description\": \"min=212.233, mean=212.233, max=212.233, sum=212.233 (1)\", \"tab\": \"General information\", \"score\": \"212.233\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"all\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.565, - "details": { - "description": "min=0.565, mean=0.565, max=0.565, sum=0.565 (1)", - "tab": "Accuracy", - "GPQA - Observed inference time (s)": "{\"description\": \"min=6.262, mean=6.262, max=6.262, sum=6.262 (1)\", \"tab\": \"Efficiency\", \"score\": \"6.261580738251519\"}", - "GPQA - # eval": "{\"description\": \"min=446, mean=446, max=446, sum=446 (1)\", \"tab\": \"General information\", \"score\": \"446.0\"}", - "GPQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - # prompt tokens": "{\"description\": \"min=250.738, mean=250.738, max=250.738, sum=250.738 (1)\", \"tab\": \"General information\", \"score\": \"250.73766816143498\"}", - "GPQA - # output tokens": "{\"description\": \"min=260.175, mean=260.175, max=260.175, sum=260.175 (1)\", \"tab\": \"General information\", \"score\": \"260.17488789237666\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"gpqa_main\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "IFEval Strict Acc on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.856, - "details": { - "description": "min=0.856, mean=0.856, max=0.856, sum=0.856 (1)", - "tab": "Accuracy", - "IFEval - Observed inference time (s)": "{\"description\": \"min=6.967, mean=6.967, max=6.967, sum=6.967 (1)\", \"tab\": \"Efficiency\", \"score\": \"6.966711103365293\"}", - "IFEval - # eval": "{\"description\": \"min=541, mean=541, max=541, sum=541 (1)\", \"tab\": \"General information\", \"score\": \"541.0\"}", - "IFEval - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - # prompt tokens": "{\"description\": \"min=47.159, mean=47.159, max=47.159, sum=47.159 (1)\", \"tab\": \"General information\", \"score\": \"47.15896487985213\"}", - "IFEval - # output tokens": "{\"description\": \"min=299.843, mean=299.843, max=299.843, sum=299.843 (1)\", \"tab\": \"General information\", \"score\": \"299.84288354898337\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WildBench", - "source_data": { - "dataset_name": "WildBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "WB Score on WildBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.792, - "details": { - "description": "min=0.792, mean=0.792, max=0.792, sum=0.792 (1)", - "tab": "Accuracy", - "WildBench - Observed inference time (s)": "{\"description\": \"min=10.864, mean=10.864, max=10.864, sum=10.864 (1)\", \"tab\": \"Efficiency\", \"score\": \"10.86402980184555\"}", - "WildBench - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "WildBench - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # prompt tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # output tokens": "{\"description\": \"min=603.959, mean=603.959, max=603.959, sum=603.959 (1)\", \"tab\": \"General information\", \"score\": \"603.959\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"v2\"" - } - } - }, - { - "evaluation_name": "Omni-MATH", - "source_data": { - "dataset_name": "Omni-MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Acc on Omni-MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.276, - "details": { - "description": "min=0.276, mean=0.276, max=0.276, sum=0.276 (1)", - "tab": "Accuracy", - "Omni-MATH - Observed inference time (s)": "{\"description\": \"min=7.588, mean=7.588, max=7.588, sum=7.588 (1)\", \"tab\": \"Efficiency\", \"score\": \"7.588193969964981\"}", - "Omni-MATH - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "Omni-MATH - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - # prompt tokens": "{\"description\": \"min=110.563, mean=110.563, max=110.563, sum=110.563 (1)\", \"tab\": \"General information\", \"score\": \"110.563\"}", - "Omni-MATH - # output tokens": "{\"description\": \"min=397.573, mean=397.573, max=397.573, sum=397.573 (1)\", \"tab\": \"General information\", \"score\": \"397.573\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_lite/anthropic_claude-3-5-sonnet-20241022/1774096306.427425", - "retrieved_timestamp": "1774096306.427425", - "source_metadata": { - "source_name": "helm_lite", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_lite", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_lite", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.846, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.2994132334581773\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.77, - "details": { - "description": "min=0.77, mean=0.77, max=0.77, sum=0.77 (1)", - "tab": "Accuracy", - "NarrativeQA - Observed inference time (s)": "{\"description\": \"min=41.561, mean=41.561, max=41.561, sum=41.561 (1)\", \"tab\": \"Efficiency\", \"score\": \"41.56126285405226\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=3662.741, mean=3662.741, max=3662.741, sum=3662.741 (1)\", \"tab\": \"General information\", \"score\": \"3662.7408450704224\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=7.031, mean=7.031, max=7.031, sum=7.031 (1)\", \"tab\": \"General information\", \"score\": \"7.030985915492958\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (closed-book)", - "source_data": { - "dataset_name": "NaturalQuestions (closed-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (closed-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.467, - "details": { - "description": "min=0.467, mean=0.467, max=0.467, sum=0.467 (1)", - "tab": "Accuracy", - "NaturalQuestions (open-book) - Observed inference time (s)": "{\"description\": \"min=4.722, mean=4.722, max=4.722, sum=4.722 (1)\", \"tab\": \"Efficiency\", \"score\": \"4.721950803041458\"}", - "NaturalQuestions (closed-book) - Observed inference time (s)": "{\"description\": \"min=0.659, mean=0.659, max=0.659, sum=0.659 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.6590276186466217\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1726.799, mean=1726.799, max=1726.799, sum=1726.799 (1)\", \"tab\": \"General information\", \"score\": \"1726.799\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=14.702, mean=14.702, max=14.702, sum=14.702 (1)\", \"tab\": \"General information\", \"score\": \"14.702\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=134.259, mean=134.259, max=134.259, sum=134.259 (1)\", \"tab\": \"General information\", \"score\": \"134.259\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=8.63, mean=8.63, max=8.63, sum=8.63 (1)\", \"tab\": \"General information\", \"score\": \"8.63\"}" - } - }, - "generation_config": { - "additional_details": { - "mode": "\"closedbook\"" - } - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.966, - "details": { - "description": "min=0.966, mean=0.966, max=0.966, sum=0.966 (1)", - "tab": "Accuracy", - "OpenbookQA - Observed inference time (s)": "{\"description\": \"min=1.256, mean=1.256, max=1.256, sum=1.256 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.2558565106391906\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=263.79, mean=263.79, max=263.79, sum=263.79 (1)\", \"tab\": \"General information\", \"score\": \"263.79\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "dataset": "\"openbookqa\"", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.809, - "details": { - "description": "min=0.63, mean=0.809, max=0.96, sum=4.047 (5)", - "tab": "Accuracy", - "MMLU - Observed inference time (s)": "{\"description\": \"min=0.66, mean=0.673, max=0.689, sum=3.367 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.6733581468766195\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=370.26, mean=478.747, max=619.596, sum=2393.736 (5)\", \"tab\": \"General information\", \"score\": \"478.747298245614\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"college_chemistry\", \"computer_security\", \"econometrics\", \"us_foreign_policy\"]", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MATH", - "source_data": { - "dataset_name": "MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Equivalent (CoT) on MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.904, - "details": { - "description": "min=0.789, mean=0.904, max=0.985, sum=6.326 (7)", - "tab": "Accuracy", - "MATH - Observed inference time (s)": "{\"description\": \"min=3.355, mean=4.052, max=4.718, sum=28.364 (7)\", \"tab\": \"Efficiency\", \"score\": \"4.0520609326088035\"}", - "MATH - # eval": "{\"description\": \"min=30, mean=62.429, max=135, sum=437 (7)\", \"tab\": \"General information\", \"score\": \"62.42857142857143\"}", - "MATH - # train": "{\"description\": \"min=8, mean=8, max=8, sum=56 (7)\", \"tab\": \"General information\", \"score\": \"8.0\"}", - "MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MATH - # prompt tokens": "{\"description\": \"min=887.259, mean=1301.814, max=2319.808, sum=9112.699 (7)\", \"tab\": \"General information\", \"score\": \"1301.8141219676104\"}", - "MATH - # output tokens": "{\"description\": \"min=127.663, mean=168.831, max=213.077, sum=1181.819 (7)\", \"tab\": \"General information\", \"score\": \"168.831271579864\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"algebra\", \"counting_and_probability\", \"geometry\", \"intermediate_algebra\", \"number_theory\", \"prealgebra\", \"precalculus\"]", - "level": "\"1\"", - "use_official_examples": "\"False\"", - "use_chain_of_thought": "\"True\"" - } - } - }, - { - "evaluation_name": "GSM8K", - "source_data": { - "dataset_name": "GSM8K", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on GSM8K", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.956, - "details": { - "description": "min=0.956, mean=0.956, max=0.956, sum=0.956 (1)", - "tab": "Accuracy", - "GSM8K - Observed inference time (s)": "{\"description\": \"min=3.518, mean=3.518, max=3.518, sum=3.518 (1)\", \"tab\": \"Efficiency\", \"score\": \"3.5175547733306884\"}", - "GSM8K - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "GSM8K - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "GSM8K - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GSM8K - # prompt tokens": "{\"description\": \"min=938.712, mean=938.712, max=938.712, sum=938.712 (1)\", \"tab\": \"General information\", \"score\": \"938.712\"}", - "GSM8K - # output tokens": "{\"description\": \"min=141.152, mean=141.152, max=141.152, sum=141.152 (1)\", \"tab\": \"General information\", \"score\": \"141.152\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "LegalBench", - "source_data": { - "dataset_name": "LegalBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on LegalBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.647, - "details": { - "description": "min=0.283, mean=0.647, max=0.989, sum=3.237 (5)", - "tab": "Accuracy", - "LegalBench - Observed inference time (s)": "{\"description\": \"min=0.559, mean=1.013, max=1.649, sum=5.065 (5)\", \"tab\": \"Efficiency\", \"score\": \"1.0130474324650445\"}", - "LegalBench - # eval": "{\"description\": \"min=95, mean=409.4, max=1000, sum=2047 (5)\", \"tab\": \"General information\", \"score\": \"409.4\"}", - "LegalBench - # train": "{\"description\": \"min=4, mean=4.8, max=5, sum=24 (5)\", \"tab\": \"General information\", \"score\": \"4.8\"}", - "LegalBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "LegalBench - # prompt tokens": "{\"description\": \"min=232.653, mean=1568.242, max=6432.398, sum=7841.208 (5)\", \"tab\": \"General information\", \"score\": \"1568.241581367783\"}", - "LegalBench - # output tokens": "{\"description\": \"min=1, mean=3.7, max=13.488, sum=18.498 (5)\", \"tab\": \"General information\", \"score\": \"3.6996529470816006\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"abercrombie\", \"corporate_lobbying\", \"function_of_decision_section\", \"international_citizenship_questions\", \"proa\"]" - } - } - }, - { - "evaluation_name": "MedQA", - "source_data": { - "dataset_name": "MedQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MedQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.859, - "details": { - "description": "min=0.859, mean=0.859, max=0.859, sum=0.859 (1)", - "tab": "Accuracy", - "MedQA - Observed inference time (s)": "{\"description\": \"min=0.815, mean=0.815, max=0.815, sum=0.815 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.8153728936348947\"}", - "MedQA - # eval": "{\"description\": \"min=503, mean=503, max=503, sum=503 (1)\", \"tab\": \"General information\", \"score\": \"503.0\"}", - "MedQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MedQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MedQA - # prompt tokens": "{\"description\": \"min=1027.437, mean=1027.437, max=1027.437, sum=1027.437 (1)\", \"tab\": \"General information\", \"score\": \"1027.4373757455269\"}", - "MedQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WMT 2014", - "source_data": { - "dataset_name": "WMT 2014", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "BLEU-4 on WMT 2014", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.226, - "details": { - "description": "min=0.174, mean=0.226, max=0.266, sum=1.128 (5)", - "tab": "Accuracy", - "WMT 2014 - Observed inference time (s)": "{\"description\": \"min=0.838, mean=0.86, max=0.889, sum=4.301 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.8602394085223064\"}", - "WMT 2014 - # eval": "{\"description\": \"min=503, mean=568.8, max=832, sum=2844 (5)\", \"tab\": \"General information\", \"score\": \"568.8\"}", - "WMT 2014 - # train": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "WMT 2014 - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WMT 2014 - # prompt tokens": "{\"description\": \"min=141.406, mean=162.573, max=184.974, sum=812.866 (5)\", \"tab\": \"General information\", \"score\": \"162.5732207715247\"}", - "WMT 2014 - # output tokens": "{\"description\": \"min=23.825, mean=25.177, max=25.958, sum=125.887 (5)\", \"tab\": \"General information\", \"score\": \"25.177411492582966\"}" - } - }, - "generation_config": { - "additional_details": { - "language_pair": "[\"cs-en\", \"de-en\", \"fr-en\", \"hi-en\", \"ru-en\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_mmlu/anthropic_claude-3-5-sonnet-20241022/1774096312.00548", - "retrieved_timestamp": "1774096312.00548", - "source_metadata": { - "source_name": "helm_mmlu", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_mmlu", - "evaluation_results": [ - { - "evaluation_name": "MMLU All Subjects", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU All Subjects", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.873, - "details": { - "description": "min=0.584, mean=0.873, max=0.984, sum=99.491 (114)", - "tab": "Accuracy", - "MMLU All Subjects - Observed inference time (s)": "{\"description\": \"min=0.615, mean=0.688, max=1.002, sum=78.403 (114)\", \"tab\": \"Efficiency\", \"score\": \"0.6877486861856626\"}", - "MMLU All Subjects - # eval": "{\"description\": \"min=100, mean=246.351, max=1534, sum=28084 (114)\", \"tab\": \"General information\", \"score\": \"246.35087719298247\"}", - "MMLU All Subjects - # train": "{\"description\": \"min=5, mean=5, max=5, sum=570 (114)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU All Subjects - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU All Subjects - # prompt tokens": "{\"description\": \"min=293.018, mean=638.288, max=2887.576, sum=72764.875 (114)\", \"tab\": \"General information\", \"score\": \"638.2883793758953\"}", - "MMLU All Subjects - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=114 (114)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_name": "Abstract Algebra", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Abstract Algebra", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.78, - "details": { - "description": "min=0.78, mean=0.78, max=0.78, sum=1.56 (2)", - "tab": "Accuracy", - "Abstract Algebra - Observed inference time (s)": "{\"description\": \"min=0.673, mean=0.673, max=0.673, sum=1.345 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.672634687423706\"}", - "Abstract Algebra - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Abstract Algebra - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Abstract Algebra - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Abstract Algebra - # prompt tokens": "{\"description\": \"min=370.26, mean=370.26, max=370.26, sum=740.52 (2)\", \"tab\": \"General information\", \"score\": \"370.26\"}", - "Abstract Algebra - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"abstract_algebra\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_abstract_algebra\"" - } - } - }, - { - "evaluation_name": "Anatomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Anatomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.859, - "details": { - "description": "min=0.859, mean=0.859, max=0.859, sum=1.719 (2)", - "tab": "Accuracy", - "Anatomy - Observed inference time (s)": "{\"description\": \"min=0.654, mean=0.654, max=0.654, sum=1.308 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.653886115109479\"}", - "Anatomy - # eval": "{\"description\": \"min=135, mean=135, max=135, sum=270 (2)\", \"tab\": \"General information\", \"score\": \"135.0\"}", - "Anatomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Anatomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Anatomy - # prompt tokens": "{\"description\": \"min=370.8, mean=370.8, max=370.8, sum=741.6 (2)\", \"tab\": \"General information\", \"score\": \"370.8\"}", - "Anatomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"anatomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_anatomy\"" - } - } - }, - { - "evaluation_name": "College Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on College Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.775, - "details": { - "description": "min=0.775, mean=0.775, max=0.775, sum=1.549 (2)", - "tab": "Accuracy", - "College Chemistry - Observed inference time (s)": "{\"description\": \"min=0.689, mean=0.689, max=0.689, sum=1.379 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6893502926826477\"}", - "College Biology - Observed inference time (s)": "{\"description\": \"min=0.66, mean=0.66, max=0.66, sum=1.32 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6600197752316793\"}", - "College Computer Science - Observed inference time (s)": "{\"description\": \"min=0.673, mean=0.673, max=0.673, sum=1.345 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6726715517044067\"}", - "College Mathematics - Observed inference time (s)": "{\"description\": \"min=0.689, mean=0.689, max=0.689, sum=1.378 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6890151953697204\"}", - "College Medicine - Observed inference time (s)": "{\"description\": \"min=0.668, mean=0.668, max=0.668, sum=1.337 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6682831924085673\"}", - "College Physics - Observed inference time (s)": "{\"description\": \"min=0.704, mean=0.704, max=0.704, sum=1.407 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7037388226565193\"}", - "College Chemistry - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Chemistry - # prompt tokens": "{\"description\": \"min=550.01, mean=550.01, max=550.01, sum=1100.02 (2)\", \"tab\": \"General information\", \"score\": \"550.01\"}", - "College Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Biology - # eval": "{\"description\": \"min=144, mean=144, max=144, sum=288 (2)\", \"tab\": \"General information\", \"score\": \"144.0\"}", - "College Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # prompt tokens": "{\"description\": \"min=490.347, mean=490.347, max=490.347, sum=980.694 (2)\", \"tab\": \"General information\", \"score\": \"490.34722222222223\"}", - "College Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # prompt tokens": "{\"description\": \"min=838.24, mean=838.24, max=838.24, sum=1676.48 (2)\", \"tab\": \"General information\", \"score\": \"838.24\"}", - "College Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Mathematics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # prompt tokens": "{\"description\": \"min=604.19, mean=604.19, max=604.19, sum=1208.38 (2)\", \"tab\": \"General information\", \"score\": \"604.19\"}", - "College Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Medicine - # eval": "{\"description\": \"min=173, mean=173, max=173, sum=346 (2)\", \"tab\": \"General information\", \"score\": \"173.0\"}", - "College Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # prompt tokens": "{\"description\": \"min=540.63, mean=540.63, max=540.63, sum=1081.26 (2)\", \"tab\": \"General information\", \"score\": \"540.6300578034682\"}", - "College Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Physics - # eval": "{\"description\": \"min=102, mean=102, max=102, sum=204 (2)\", \"tab\": \"General information\", \"score\": \"102.0\"}", - "College Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # prompt tokens": "{\"description\": \"min=489.48, mean=489.48, max=489.48, sum=978.961 (2)\", \"tab\": \"General information\", \"score\": \"489.48039215686276\"}", - "College Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"college_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_college_physics\"" - } - } - }, - { - "evaluation_name": "Computer Security", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Computer Security", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.87, - "details": { - "description": "min=0.87, mean=0.87, max=0.87, sum=1.74 (2)", - "tab": "Accuracy", - "Computer Security - Observed inference time (s)": "{\"description\": \"min=0.661, mean=0.661, max=0.661, sum=1.322 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6610880661010742\"}", - "Computer Security - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Computer Security - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Computer Security - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Computer Security - # prompt tokens": "{\"description\": \"min=398.62, mean=398.62, max=398.62, sum=797.24 (2)\", \"tab\": \"General information\", \"score\": \"398.62\"}", - "Computer Security - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"computer_security\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_computer_security\"" - } - } - }, - { - "evaluation_name": "Econometrics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Econometrics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.807, - "details": { - "description": "min=0.807, mean=0.807, max=0.807, sum=1.614 (2)", - "tab": "Accuracy", - "Econometrics - Observed inference time (s)": "{\"description\": \"min=0.684, mean=0.684, max=0.684, sum=1.367 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6837067018475449\"}", - "Econometrics - # eval": "{\"description\": \"min=114, mean=114, max=114, sum=228 (2)\", \"tab\": \"General information\", \"score\": \"114.0\"}", - "Econometrics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Econometrics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Econometrics - # prompt tokens": "{\"description\": \"min=619.596, mean=619.596, max=619.596, sum=1239.193 (2)\", \"tab\": \"General information\", \"score\": \"619.5964912280701\"}", - "Econometrics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"econometrics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_econometrics\"" - } - } - }, - { - "evaluation_name": "Global Facts", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Global Facts", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8, - "details": { - "description": "min=0.8, mean=0.8, max=0.8, sum=1.6 (2)", - "tab": "Accuracy", - "Global Facts - Observed inference time (s)": "{\"description\": \"min=0.637, mean=0.637, max=0.637, sum=1.274 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6369614601135254\"}", - "Global Facts - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Global Facts - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Global Facts - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Global Facts - # prompt tokens": "{\"description\": \"min=411.61, mean=411.61, max=411.61, sum=823.22 (2)\", \"tab\": \"General information\", \"score\": \"411.61\"}", - "Global Facts - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"global_facts\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_global_facts\"" - } - } - }, - { - "evaluation_name": "Jurisprudence", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Jurisprudence", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.898, - "details": { - "description": "min=0.898, mean=0.898, max=0.898, sum=1.796 (2)", - "tab": "Accuracy", - "Jurisprudence - Observed inference time (s)": "{\"description\": \"min=0.643, mean=0.643, max=0.643, sum=1.286 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6427947613928053\"}", - "Jurisprudence - # eval": "{\"description\": \"min=108, mean=108, max=108, sum=216 (2)\", \"tab\": \"General information\", \"score\": \"108.0\"}", - "Jurisprudence - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Jurisprudence - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Jurisprudence - # prompt tokens": "{\"description\": \"min=431.426, mean=431.426, max=431.426, sum=862.852 (2)\", \"tab\": \"General information\", \"score\": \"431.4259259259259\"}", - "Jurisprudence - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"jurisprudence\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_jurisprudence\"" - } - } - }, - { - "evaluation_name": "Philosophy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Philosophy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.891, - "details": { - "description": "min=0.891, mean=0.891, max=0.891, sum=1.781 (2)", - "tab": "Accuracy", - "Philosophy - Observed inference time (s)": "{\"description\": \"min=0.645, mean=0.645, max=0.645, sum=1.291 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6454648833566157\"}", - "Philosophy - # eval": "{\"description\": \"min=311, mean=311, max=311, sum=622 (2)\", \"tab\": \"General information\", \"score\": \"311.0\"}", - "Philosophy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Philosophy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Philosophy - # prompt tokens": "{\"description\": \"min=359.965, mean=359.965, max=359.965, sum=719.929 (2)\", \"tab\": \"General information\", \"score\": \"359.9646302250804\"}", - "Philosophy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"philosophy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_philosophy\"" - } - } - }, - { - "evaluation_name": "Professional Psychology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Professional Psychology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.922, - "details": { - "description": "min=0.922, mean=0.922, max=0.922, sum=1.843 (2)", - "tab": "Accuracy", - "Professional Medicine - Observed inference time (s)": "{\"description\": \"min=0.622, mean=0.622, max=0.622, sum=1.243 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6215311034637339\"}", - "Professional Accounting - Observed inference time (s)": "{\"description\": \"min=0.69, mean=0.69, max=0.69, sum=1.38 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6900012104223806\"}", - "Professional Law - Observed inference time (s)": "{\"description\": \"min=1.002, mean=1.002, max=1.002, sum=2.004 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.002109061319483\"}", - "Professional Psychology - Observed inference time (s)": "{\"description\": \"min=0.682, mean=0.682, max=0.682, sum=1.364 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6821525521527708\"}", - "Professional Medicine - # eval": "{\"description\": \"min=272, mean=272, max=272, sum=544 (2)\", \"tab\": \"General information\", \"score\": \"272.0\"}", - "Professional Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Medicine - # prompt tokens": "{\"description\": \"min=1123.537, mean=1123.537, max=1123.537, sum=2247.074 (2)\", \"tab\": \"General information\", \"score\": \"1123.5367647058824\"}", - "Professional Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Accounting - # eval": "{\"description\": \"min=282, mean=282, max=282, sum=564 (2)\", \"tab\": \"General information\", \"score\": \"282.0\"}", - "Professional Accounting - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Accounting - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # prompt tokens": "{\"description\": \"min=665.422, mean=665.422, max=665.422, sum=1330.844 (2)\", \"tab\": \"General information\", \"score\": \"665.4219858156029\"}", - "Professional Accounting - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Law - # eval": "{\"description\": \"min=1534, mean=1534, max=1534, sum=3068 (2)\", \"tab\": \"General information\", \"score\": \"1534.0\"}", - "Professional Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # prompt tokens": "{\"description\": \"min=1701.16, mean=1701.16, max=1701.16, sum=3402.321 (2)\", \"tab\": \"General information\", \"score\": \"1701.16036505867\"}", - "Professional Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Psychology - # eval": "{\"description\": \"min=612, mean=612, max=612, sum=1224 (2)\", \"tab\": \"General information\", \"score\": \"612.0\"}", - "Professional Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # prompt tokens": "{\"description\": \"min=603.168, mean=603.168, max=603.168, sum=1206.337 (2)\", \"tab\": \"General information\", \"score\": \"603.1683006535948\"}", - "Professional Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"professional_psychology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_professional_psychology\"" - } - } - }, - { - "evaluation_name": "Us Foreign Policy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Us Foreign Policy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.96, - "details": { - "description": "min=0.96, mean=0.96, max=0.96, sum=1.92 (2)", - "tab": "Accuracy", - "Us Foreign Policy - Observed inference time (s)": "{\"description\": \"min=0.66, mean=0.66, max=0.66, sum=1.32 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.660010986328125\"}", - "Us Foreign Policy - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Us Foreign Policy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Us Foreign Policy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Us Foreign Policy - # prompt tokens": "{\"description\": \"min=455.25, mean=455.25, max=455.25, sum=910.5 (2)\", \"tab\": \"General information\", \"score\": \"455.25\"}", - "Us Foreign Policy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"us_foreign_policy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_us_foreign_policy\"" - } - } - }, - { - "evaluation_name": "Astronomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Astronomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.974, - "details": { - "description": "min=0.974, mean=0.974, max=0.974, sum=1.947 (2)", - "tab": "Accuracy", - "Astronomy - Observed inference time (s)": "{\"description\": \"min=0.672, mean=0.672, max=0.672, sum=1.344 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6717779793237385\"}", - "Astronomy - # eval": "{\"description\": \"min=152, mean=152, max=152, sum=304 (2)\", \"tab\": \"General information\", \"score\": \"152.0\"}", - "Astronomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Astronomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Astronomy - # prompt tokens": "{\"description\": \"min=604.493, mean=604.493, max=604.493, sum=1208.987 (2)\", \"tab\": \"General information\", \"score\": \"604.4934210526316\"}", - "Astronomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"astronomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_astronomy\"" - } - } - }, - { - "evaluation_name": "Business Ethics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Business Ethics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.83, - "details": { - "description": "min=0.83, mean=0.83, max=0.83, sum=1.66 (2)", - "tab": "Accuracy", - "Business Ethics - Observed inference time (s)": "{\"description\": \"min=0.651, mean=0.651, max=0.651, sum=1.302 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6511244606971741\"}", - "Business Ethics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Business Ethics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Business Ethics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Business Ethics - # prompt tokens": "{\"description\": \"min=600.02, mean=600.02, max=600.02, sum=1200.04 (2)\", \"tab\": \"General information\", \"score\": \"600.02\"}", - "Business Ethics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"business_ethics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_business_ethics\"" - } - } - }, - { - "evaluation_name": "Clinical Knowledge", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Clinical Knowledge", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.928, - "details": { - "description": "min=0.928, mean=0.928, max=0.928, sum=1.857 (2)", - "tab": "Accuracy", - "Clinical Knowledge - Observed inference time (s)": "{\"description\": \"min=0.65, mean=0.65, max=0.65, sum=1.3 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6499361712977572\"}", - "Clinical Knowledge - # eval": "{\"description\": \"min=265, mean=265, max=265, sum=530 (2)\", \"tab\": \"General information\", \"score\": \"265.0\"}", - "Clinical Knowledge - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Clinical Knowledge - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Clinical Knowledge - # prompt tokens": "{\"description\": \"min=429.457, mean=429.457, max=429.457, sum=858.913 (2)\", \"tab\": \"General information\", \"score\": \"429.4566037735849\"}", - "Clinical Knowledge - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"clinical_knowledge\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_clinical_knowledge\"" - } - } - }, - { - "evaluation_name": "Conceptual Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Conceptual Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.906, - "details": { - "description": "min=0.906, mean=0.906, max=0.906, sum=1.813 (2)", - "tab": "Accuracy", - "Conceptual Physics - Observed inference time (s)": "{\"description\": \"min=0.615, mean=0.615, max=0.615, sum=1.229 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6146096341153409\"}", - "Conceptual Physics - # eval": "{\"description\": \"min=235, mean=235, max=235, sum=470 (2)\", \"tab\": \"General information\", \"score\": \"235.0\"}", - "Conceptual Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Conceptual Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Conceptual Physics - # prompt tokens": "{\"description\": \"min=323.536, mean=323.536, max=323.536, sum=647.072 (2)\", \"tab\": \"General information\", \"score\": \"323.53617021276597\"}", - "Conceptual Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"conceptual_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_conceptual_physics\"" - } - } - }, - { - "evaluation_name": "Electrical Engineering", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Electrical Engineering", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.848, - "details": { - "description": "min=0.848, mean=0.848, max=0.848, sum=1.697 (2)", - "tab": "Accuracy", - "Electrical Engineering - Observed inference time (s)": "{\"description\": \"min=0.646, mean=0.646, max=0.646, sum=1.292 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6462178690680143\"}", - "Electrical Engineering - # eval": "{\"description\": \"min=145, mean=145, max=145, sum=290 (2)\", \"tab\": \"General information\", \"score\": \"145.0\"}", - "Electrical Engineering - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Electrical Engineering - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Electrical Engineering - # prompt tokens": "{\"description\": \"min=437.041, mean=437.041, max=437.041, sum=874.083 (2)\", \"tab\": \"General information\", \"score\": \"437.04137931034484\"}", - "Electrical Engineering - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"electrical_engineering\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_electrical_engineering\"" - } - } - }, - { - "evaluation_name": "Elementary Mathematics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Elementary Mathematics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.918, - "details": { - "description": "min=0.918, mean=0.918, max=0.918, sum=1.836 (2)", - "tab": "Accuracy", - "Elementary Mathematics - Observed inference time (s)": "{\"description\": \"min=0.709, mean=0.709, max=0.709, sum=1.418 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7089652012264918\"}", - "Elementary Mathematics - # eval": "{\"description\": \"min=378, mean=378, max=378, sum=756 (2)\", \"tab\": \"General information\", \"score\": \"378.0\"}", - "Elementary Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Elementary Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Elementary Mathematics - # prompt tokens": "{\"description\": \"min=511.066, mean=511.066, max=511.066, sum=1022.132 (2)\", \"tab\": \"General information\", \"score\": \"511.06613756613757\"}", - "Elementary Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"elementary_mathematics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_elementary_mathematics\"" - } - } - }, - { - "evaluation_name": "Formal Logic", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Formal Logic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.786, - "details": { - "description": "min=0.786, mean=0.786, max=0.786, sum=1.571 (2)", - "tab": "Accuracy", - "Formal Logic - Observed inference time (s)": "{\"description\": \"min=0.692, mean=0.692, max=0.692, sum=1.384 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.691912295326354\"}", - "Formal Logic - # eval": "{\"description\": \"min=126, mean=126, max=126, sum=252 (2)\", \"tab\": \"General information\", \"score\": \"126.0\"}", - "Formal Logic - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Formal Logic - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Formal Logic - # prompt tokens": "{\"description\": \"min=646.746, mean=646.746, max=646.746, sum=1293.492 (2)\", \"tab\": \"General information\", \"score\": \"646.7460317460317\"}", - "Formal Logic - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"formal_logic\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_formal_logic\"" - } - } - }, - { - "evaluation_name": "High School World History", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on High School World History", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.958, - "details": { - "description": "min=0.958, mean=0.958, max=0.958, sum=1.916 (2)", - "tab": "Accuracy", - "High School Biology - Observed inference time (s)": "{\"description\": \"min=0.669, mean=0.669, max=0.669, sum=1.338 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6689629408621018\"}", - "High School Chemistry - Observed inference time (s)": "{\"description\": \"min=0.673, mean=0.673, max=0.673, sum=1.346 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6729868444903143\"}", - "High School Computer Science - Observed inference time (s)": "{\"description\": \"min=0.678, mean=0.678, max=0.678, sum=1.356 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.677822756767273\"}", - "High School European History - Observed inference time (s)": "{\"description\": \"min=0.697, mean=0.697, max=0.697, sum=1.395 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6973154544830322\"}", - "High School Geography - Observed inference time (s)": "{\"description\": \"min=0.64, mean=0.64, max=0.64, sum=1.281 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6404741051221134\"}", - "High School Government And Politics - Observed inference time (s)": "{\"description\": \"min=0.661, mean=0.661, max=0.661, sum=1.323 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6613641341115527\"}", - "High School Macroeconomics - Observed inference time (s)": "{\"description\": \"min=0.631, mean=0.631, max=0.631, sum=1.261 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6305418686989026\"}", - "High School Mathematics - Observed inference time (s)": "{\"description\": \"min=0.668, mean=0.668, max=0.668, sum=1.336 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6677727399048982\"}", - "High School Microeconomics - Observed inference time (s)": "{\"description\": \"min=0.656, mean=0.656, max=0.656, sum=1.312 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6559101263014209\"}", - "High School Physics - Observed inference time (s)": "{\"description\": \"min=0.676, mean=0.676, max=0.676, sum=1.353 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6763939494328783\"}", - "High School Psychology - Observed inference time (s)": "{\"description\": \"min=0.671, mean=0.671, max=0.671, sum=1.342 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6708623107420195\"}", - "High School Statistics - Observed inference time (s)": "{\"description\": \"min=0.702, mean=0.702, max=0.702, sum=1.404 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7019402329568509\"}", - "High School US History - Observed inference time (s)": "{\"description\": \"min=0.646, mean=0.646, max=0.646, sum=1.293 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6463189136748221\"}", - "High School World History - Observed inference time (s)": "{\"description\": \"min=0.695, mean=0.695, max=0.695, sum=1.39 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6947573730211217\"}", - "High School Biology - # eval": "{\"description\": \"min=310, mean=310, max=310, sum=620 (2)\", \"tab\": \"General information\", \"score\": \"310.0\"}", - "High School Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Biology - # prompt tokens": "{\"description\": \"min=534.577, mean=534.577, max=534.577, sum=1069.155 (2)\", \"tab\": \"General information\", \"score\": \"534.5774193548388\"}", - "High School Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Chemistry - # eval": "{\"description\": \"min=203, mean=203, max=203, sum=406 (2)\", \"tab\": \"General information\", \"score\": \"203.0\"}", - "High School Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # prompt tokens": "{\"description\": \"min=497.921, mean=497.921, max=497.921, sum=995.842 (2)\", \"tab\": \"General information\", \"score\": \"497.92118226600985\"}", - "High School Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "High School Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # prompt tokens": "{\"description\": \"min=882.4, mean=882.4, max=882.4, sum=1764.8 (2)\", \"tab\": \"General information\", \"score\": \"882.4\"}", - "High School Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School European History - # eval": "{\"description\": \"min=165, mean=165, max=165, sum=330 (2)\", \"tab\": \"General information\", \"score\": \"165.0\"}", - "High School European History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School European History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # prompt tokens": "{\"description\": \"min=2887.576, mean=2887.576, max=2887.576, sum=5775.152 (2)\", \"tab\": \"General information\", \"score\": \"2887.5757575757575\"}", - "High School European History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Geography - # eval": "{\"description\": \"min=198, mean=198, max=198, sum=396 (2)\", \"tab\": \"General information\", \"score\": \"198.0\"}", - "High School Geography - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Geography - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # prompt tokens": "{\"description\": \"min=412.268, mean=412.268, max=412.268, sum=824.535 (2)\", \"tab\": \"General information\", \"score\": \"412.2676767676768\"}", - "High School Geography - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Government And Politics - # eval": "{\"description\": \"min=193, mean=193, max=193, sum=386 (2)\", \"tab\": \"General information\", \"score\": \"193.0\"}", - "High School Government And Politics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Government And Politics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # prompt tokens": "{\"description\": \"min=491.104, mean=491.104, max=491.104, sum=982.207 (2)\", \"tab\": \"General information\", \"score\": \"491.10362694300517\"}", - "High School Government And Politics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Macroeconomics - # eval": "{\"description\": \"min=390, mean=390, max=390, sum=780 (2)\", \"tab\": \"General information\", \"score\": \"390.0\"}", - "High School Macroeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Macroeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # prompt tokens": "{\"description\": \"min=406.036, mean=406.036, max=406.036, sum=812.072 (2)\", \"tab\": \"General information\", \"score\": \"406.0358974358974\"}", - "High School Macroeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Mathematics - # eval": "{\"description\": \"min=270, mean=270, max=270, sum=540 (2)\", \"tab\": \"General information\", \"score\": \"270.0\"}", - "High School Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # prompt tokens": "{\"description\": \"min=519.881, mean=519.881, max=519.881, sum=1039.763 (2)\", \"tab\": \"General information\", \"score\": \"519.8814814814815\"}", - "High School Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Microeconomics - # eval": "{\"description\": \"min=238, mean=238, max=238, sum=476 (2)\", \"tab\": \"General information\", \"score\": \"238.0\"}", - "High School Microeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Microeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # prompt tokens": "{\"description\": \"min=420.513, mean=420.513, max=420.513, sum=841.025 (2)\", \"tab\": \"General information\", \"score\": \"420.5126050420168\"}", - "High School Microeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Physics - # eval": "{\"description\": \"min=151, mean=151, max=151, sum=302 (2)\", \"tab\": \"General information\", \"score\": \"151.0\"}", - "High School Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # prompt tokens": "{\"description\": \"min=558.841, mean=558.841, max=558.841, sum=1117.682 (2)\", \"tab\": \"General information\", \"score\": \"558.841059602649\"}", - "High School Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Psychology - # eval": "{\"description\": \"min=545, mean=545, max=545, sum=1090 (2)\", \"tab\": \"General information\", \"score\": \"545.0\"}", - "High School Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # prompt tokens": "{\"description\": \"min=521.42, mean=521.42, max=521.42, sum=1042.84 (2)\", \"tab\": \"General information\", \"score\": \"521.4201834862386\"}", - "High School Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Statistics - # eval": "{\"description\": \"min=216, mean=216, max=216, sum=432 (2)\", \"tab\": \"General information\", \"score\": \"216.0\"}", - "High School Statistics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Statistics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # prompt tokens": "{\"description\": \"min=806.963, mean=806.963, max=806.963, sum=1613.926 (2)\", \"tab\": \"General information\", \"score\": \"806.9629629629629\"}", - "High School Statistics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School US History - # eval": "{\"description\": \"min=204, mean=204, max=204, sum=408 (2)\", \"tab\": \"General information\", \"score\": \"204.0\"}", - "High School US History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School US History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # prompt tokens": "{\"description\": \"min=2288.49, mean=2288.49, max=2288.49, sum=4576.98 (2)\", \"tab\": \"General information\", \"score\": \"2288.4901960784314\"}", - "High School US History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School World History - # eval": "{\"description\": \"min=237, mean=237, max=237, sum=474 (2)\", \"tab\": \"General information\", \"score\": \"237.0\"}", - "High School World History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School World History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # prompt tokens": "{\"description\": \"min=1475.932, mean=1475.932, max=1475.932, sum=2951.865 (2)\", \"tab\": \"General information\", \"score\": \"1475.9324894514768\"}", - "High School World History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"high_school_world_history\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_high_school_world_history\"" - } - } - }, - { - "evaluation_name": "Human Sexuality", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Human Sexuality", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.939, - "details": { - "description": "min=0.939, mean=0.939, max=0.939, sum=1.878 (2)", - "tab": "Accuracy", - "Human Aging - Observed inference time (s)": "{\"description\": \"min=0.656, mean=0.656, max=0.656, sum=1.312 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6560797862407872\"}", - "Human Sexuality - Observed inference time (s)": "{\"description\": \"min=0.686, mean=0.686, max=0.686, sum=1.372 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6857976003457572\"}", - "Human Aging - # eval": "{\"description\": \"min=223, mean=223, max=223, sum=446 (2)\", \"tab\": \"General information\", \"score\": \"223.0\"}", - "Human Aging - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Aging - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Aging - # prompt tokens": "{\"description\": \"min=335.955, mean=335.955, max=335.955, sum=671.91 (2)\", \"tab\": \"General information\", \"score\": \"335.95515695067263\"}", - "Human Aging - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Human Sexuality - # eval": "{\"description\": \"min=131, mean=131, max=131, sum=262 (2)\", \"tab\": \"General information\", \"score\": \"131.0\"}", - "Human Sexuality - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Sexuality - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # prompt tokens": "{\"description\": \"min=371.496, mean=371.496, max=371.496, sum=742.992 (2)\", \"tab\": \"General information\", \"score\": \"371.4961832061069\"}", - "Human Sexuality - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"human_sexuality\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_human_sexuality\"" - } - } - }, - { - "evaluation_name": "International Law", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on International Law", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.959, - "details": { - "description": "min=0.959, mean=0.959, max=0.959, sum=1.917 (2)", - "tab": "Accuracy", - "International Law - Observed inference time (s)": "{\"description\": \"min=0.713, mean=0.713, max=0.713, sum=1.426 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7129175268914089\"}", - "International Law - # eval": "{\"description\": \"min=121, mean=121, max=121, sum=242 (2)\", \"tab\": \"General information\", \"score\": \"121.0\"}", - "International Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "International Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "International Law - # prompt tokens": "{\"description\": \"min=664.165, mean=664.165, max=664.165, sum=1328.331 (2)\", \"tab\": \"General information\", \"score\": \"664.1652892561983\"}", - "International Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"international_law\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_international_law\"" - } - } - }, - { - "evaluation_name": "Logical Fallacies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Logical Fallacies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.914, - "details": { - "description": "min=0.914, mean=0.914, max=0.914, sum=1.828 (2)", - "tab": "Accuracy", - "Logical Fallacies - Observed inference time (s)": "{\"description\": \"min=0.821, mean=0.821, max=0.821, sum=1.642 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8211235926926501\"}", - "Logical Fallacies - # eval": "{\"description\": \"min=163, mean=163, max=163, sum=326 (2)\", \"tab\": \"General information\", \"score\": \"163.0\"}", - "Logical Fallacies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Logical Fallacies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Logical Fallacies - # prompt tokens": "{\"description\": \"min=470.276, mean=470.276, max=470.276, sum=940.552 (2)\", \"tab\": \"General information\", \"score\": \"470.2760736196319\"}", - "Logical Fallacies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"logical_fallacies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_logical_fallacies\"" - } - } - }, - { - "evaluation_name": "Machine Learning", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Machine Learning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.839, - "details": { - "description": "min=0.839, mean=0.839, max=0.839, sum=1.679 (2)", - "tab": "Accuracy", - "Machine Learning - Observed inference time (s)": "{\"description\": \"min=0.697, mean=0.697, max=0.697, sum=1.393 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.69659323990345\"}", - "Machine Learning - # eval": "{\"description\": \"min=112, mean=112, max=112, sum=224 (2)\", \"tab\": \"General information\", \"score\": \"112.0\"}", - "Machine Learning - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Machine Learning - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Machine Learning - # prompt tokens": "{\"description\": \"min=676.518, mean=676.518, max=676.518, sum=1353.036 (2)\", \"tab\": \"General information\", \"score\": \"676.5178571428571\"}", - "Machine Learning - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"machine_learning\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_machine_learning\"" - } - } - }, - { - "evaluation_name": "Management", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Management", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.932, - "details": { - "description": "min=0.932, mean=0.932, max=0.932, sum=1.864 (2)", - "tab": "Accuracy", - "Management - Observed inference time (s)": "{\"description\": \"min=0.702, mean=0.702, max=0.702, sum=1.404 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7021607287879129\"}", - "Management - # eval": "{\"description\": \"min=103, mean=103, max=103, sum=206 (2)\", \"tab\": \"General information\", \"score\": \"103.0\"}", - "Management - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Management - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Management - # prompt tokens": "{\"description\": \"min=301.282, mean=301.282, max=301.282, sum=602.563 (2)\", \"tab\": \"General information\", \"score\": \"301.28155339805824\"}", - "Management - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"management\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_management\"" - } - } - }, - { - "evaluation_name": "Marketing", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Marketing", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.953, - "details": { - "description": "min=0.953, mean=0.953, max=0.953, sum=1.906 (2)", - "tab": "Accuracy", - "Marketing - Observed inference time (s)": "{\"description\": \"min=0.833, mean=0.833, max=0.833, sum=1.667 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8333144401892637\"}", - "Marketing - # eval": "{\"description\": \"min=234, mean=234, max=234, sum=468 (2)\", \"tab\": \"General information\", \"score\": \"234.0\"}", - "Marketing - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Marketing - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Marketing - # prompt tokens": "{\"description\": \"min=448.064, mean=448.064, max=448.064, sum=896.128 (2)\", \"tab\": \"General information\", \"score\": \"448.06410256410254\"}", - "Marketing - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"marketing\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_marketing\"" - } - } - }, - { - "evaluation_name": "Medical Genetics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Medical Genetics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.96, - "details": { - "description": "min=0.96, mean=0.96, max=0.96, sum=1.92 (2)", - "tab": "Accuracy", - "Medical Genetics - Observed inference time (s)": "{\"description\": \"min=0.789, mean=0.789, max=0.789, sum=1.579 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7894818639755249\"}", - "Medical Genetics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Medical Genetics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Medical Genetics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Medical Genetics - # prompt tokens": "{\"description\": \"min=354.88, mean=354.88, max=354.88, sum=709.76 (2)\", \"tab\": \"General information\", \"score\": \"354.88\"}", - "Medical Genetics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"medical_genetics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_medical_genetics\"" - } - } - }, - { - "evaluation_name": "Miscellaneous", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Miscellaneous", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.964, - "details": { - "description": "min=0.964, mean=0.964, max=0.964, sum=1.928 (2)", - "tab": "Accuracy", - "Miscellaneous - Observed inference time (s)": "{\"description\": \"min=0.803, mean=0.803, max=0.803, sum=1.606 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8030681811073274\"}", - "Miscellaneous - # eval": "{\"description\": \"min=783, mean=783, max=783, sum=1566 (2)\", \"tab\": \"General information\", \"score\": \"783.0\"}", - "Miscellaneous - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Miscellaneous - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Miscellaneous - # prompt tokens": "{\"description\": \"min=328.628, mean=328.628, max=328.628, sum=657.257 (2)\", \"tab\": \"General information\", \"score\": \"328.62835249042143\"}", - "Miscellaneous - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"miscellaneous\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_miscellaneous\"" - } - } - }, - { - "evaluation_name": "Moral Scenarios", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Moral Scenarios", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.888, - "details": { - "description": "min=0.888, mean=0.888, max=0.888, sum=1.777 (2)", - "tab": "Accuracy", - "Moral Disputes - Observed inference time (s)": "{\"description\": \"min=0.698, mean=0.698, max=0.698, sum=1.397 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6983739172103088\"}", - "Moral Scenarios - Observed inference time (s)": "{\"description\": \"min=0.697, mean=0.697, max=0.697, sum=1.393 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6965836058781799\"}", - "Moral Disputes - # eval": "{\"description\": \"min=346, mean=346, max=346, sum=692 (2)\", \"tab\": \"General information\", \"score\": \"346.0\"}", - "Moral Disputes - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Disputes - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Disputes - # prompt tokens": "{\"description\": \"min=511.789, mean=511.789, max=511.789, sum=1023.578 (2)\", \"tab\": \"General information\", \"score\": \"511.78901734104045\"}", - "Moral Disputes - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Moral Scenarios - # eval": "{\"description\": \"min=895, mean=895, max=895, sum=1790 (2)\", \"tab\": \"General information\", \"score\": \"895.0\"}", - "Moral Scenarios - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Scenarios - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # prompt tokens": "{\"description\": \"min=676.949, mean=676.949, max=676.949, sum=1353.897 (2)\", \"tab\": \"General information\", \"score\": \"676.9486033519553\"}", - "Moral Scenarios - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"moral_scenarios\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_moral_scenarios\"" - } - } - }, - { - "evaluation_name": "Nutrition", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Nutrition", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.922, - "details": { - "description": "min=0.922, mean=0.922, max=0.922, sum=1.843 (2)", - "tab": "Accuracy", - "Nutrition - Observed inference time (s)": "{\"description\": \"min=0.695, mean=0.695, max=0.695, sum=1.389 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6946531822478849\"}", - "Nutrition - # eval": "{\"description\": \"min=306, mean=306, max=306, sum=612 (2)\", \"tab\": \"General information\", \"score\": \"306.0\"}", - "Nutrition - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Nutrition - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Nutrition - # prompt tokens": "{\"description\": \"min=617.065, mean=617.065, max=617.065, sum=1234.131 (2)\", \"tab\": \"General information\", \"score\": \"617.0653594771242\"}", - "Nutrition - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"nutrition\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_nutrition\"" - } - } - }, - { - "evaluation_name": "Prehistory", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Prehistory", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.941, - "details": { - "description": "min=0.941, mean=0.941, max=0.941, sum=1.883 (2)", - "tab": "Accuracy", - "Prehistory - Observed inference time (s)": "{\"description\": \"min=0.682, mean=0.682, max=0.682, sum=1.365 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6824756529596117\"}", - "Prehistory - # eval": "{\"description\": \"min=324, mean=324, max=324, sum=648 (2)\", \"tab\": \"General information\", \"score\": \"324.0\"}", - "Prehistory - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Prehistory - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Prehistory - # prompt tokens": "{\"description\": \"min=545.639, mean=545.639, max=545.639, sum=1091.278 (2)\", \"tab\": \"General information\", \"score\": \"545.6388888888889\"}", - "Prehistory - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"prehistory\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_prehistory\"" - } - } - }, - { - "evaluation_name": "Public Relations", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Public Relations", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8, - "details": { - "description": "min=0.8, mean=0.8, max=0.8, sum=1.6 (2)", - "tab": "Accuracy", - "Public Relations - Observed inference time (s)": "{\"description\": \"min=0.626, mean=0.626, max=0.626, sum=1.252 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6258317015387795\"}", - "Public Relations - # eval": "{\"description\": \"min=110, mean=110, max=110, sum=220 (2)\", \"tab\": \"General information\", \"score\": \"110.0\"}", - "Public Relations - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Public Relations - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Public Relations - # prompt tokens": "{\"description\": \"min=432.991, mean=432.991, max=432.991, sum=865.982 (2)\", \"tab\": \"General information\", \"score\": \"432.9909090909091\"}", - "Public Relations - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"public_relations\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_public_relations\"" - } - } - }, - { - "evaluation_name": "Security Studies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Security Studies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.882, - "details": { - "description": "min=0.882, mean=0.882, max=0.882, sum=1.763 (2)", - "tab": "Accuracy", - "Security Studies - Observed inference time (s)": "{\"description\": \"min=0.744, mean=0.744, max=0.744, sum=1.489 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7442785263061523\"}", - "Security Studies - # eval": "{\"description\": \"min=245, mean=245, max=245, sum=490 (2)\", \"tab\": \"General information\", \"score\": \"245.0\"}", - "Security Studies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Security Studies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Security Studies - # prompt tokens": "{\"description\": \"min=1243.804, mean=1243.804, max=1243.804, sum=2487.608 (2)\", \"tab\": \"General information\", \"score\": \"1243.8040816326532\"}", - "Security Studies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"security_studies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_security_studies\"" - } - } - }, - { - "evaluation_name": "Sociology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Sociology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.955, - "details": { - "description": "min=0.955, mean=0.955, max=0.955, sum=1.91 (2)", - "tab": "Accuracy", - "Sociology - Observed inference time (s)": "{\"description\": \"min=0.695, mean=0.695, max=0.695, sum=1.389 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6946055438388047\"}", - "Sociology - # eval": "{\"description\": \"min=201, mean=201, max=201, sum=402 (2)\", \"tab\": \"General information\", \"score\": \"201.0\"}", - "Sociology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Sociology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Sociology - # prompt tokens": "{\"description\": \"min=467.274, mean=467.274, max=467.274, sum=934.547 (2)\", \"tab\": \"General information\", \"score\": \"467.27363184079604\"}", - "Sociology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"sociology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_sociology\"" - } - } - }, - { - "evaluation_name": "Virology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Virology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.584, - "details": { - "description": "min=0.584, mean=0.584, max=0.584, sum=1.169 (2)", - "tab": "Accuracy", - "Virology - Observed inference time (s)": "{\"description\": \"min=0.68, mean=0.68, max=0.68, sum=1.361 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6803859400461956\"}", - "Virology - # eval": "{\"description\": \"min=166, mean=166, max=166, sum=332 (2)\", \"tab\": \"General information\", \"score\": \"166.0\"}", - "Virology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Virology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Virology - # prompt tokens": "{\"description\": \"min=362.651, mean=362.651, max=362.651, sum=725.301 (2)\", \"tab\": \"General information\", \"score\": \"362.65060240963857\"}", - "Virology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"virology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_virology\"" - } - } - }, - { - "evaluation_name": "World Religions", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on World Religions", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.901, - "details": { - "description": "min=0.901, mean=0.901, max=0.901, sum=1.801 (2)", - "tab": "Accuracy", - "World Religions - Observed inference time (s)": "{\"description\": \"min=0.651, mean=0.651, max=0.651, sum=1.301 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6505623017138208\"}", - "World Religions - # eval": "{\"description\": \"min=171, mean=171, max=171, sum=342 (2)\", \"tab\": \"General information\", \"score\": \"171.0\"}", - "World Religions - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "World Religions - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "World Religions - # prompt tokens": "{\"description\": \"min=293.018, mean=293.018, max=293.018, sum=586.035 (2)\", \"tab\": \"General information\", \"score\": \"293.0175438596491\"}", - "World Religions - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"world_religions\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_world_religions\"" - } - } - }, - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.311, - "details": { - "description": "", - "tab": "Efficiency" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/anthropic_claude-3-7-sonnet-20250219.json b/data/models/anthropic_claude-3-7-sonnet-20250219.json deleted file mode 100644 index 6e02a0c482b2aed6381eb4f121bd334b0959f1b6..0000000000000000000000000000000000000000 --- a/data/models/anthropic_claude-3-7-sonnet-20250219.json +++ /dev/null @@ -1,1496 +0,0 @@ -{ - "model_info": { - "name": "claude-3-7-sonnet-20250219", - "id": "anthropic/claude-3-7-sonnet-20250219", - "developer": "Anthropic", - "inference_platform": "unknown", - "additional_details": { - "display_name": "Claude 3.7 Sonnet" - } - }, - "evaluations": [ - { - "evaluation_id": "global-mmlu-lite/anthropic_claude-3-7-sonnet-20250219/1773936496.366405", - "retrieved_timestamp": "1773936496.366405", - "source_metadata": { - "source_name": "Global MMLU Lite Leaderboard", - "source_type": "documentation", - "source_organization_name": "kaggle", - "source_organization_url": "www.kaggle.com", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "kaggle kernel", - "version": "4", - "additional_details": { - "url": "https://www.kaggle.com/code/shivalikasingh95/global-mmlu-lite-sample-notebook" - } - }, - "benchmark": "global-mmlu-lite", - "evaluation_results": [ - { - "evaluation_name": "Global MMLU Lite", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Global MMLU Lite", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8078 - } - }, - { - "evaluation_name": "Culturally Sensitive", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Sensitive", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7794 - } - }, - { - "evaluation_name": "Culturally Agnostic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Agnostic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8362 - } - }, - { - "evaluation_name": "Arabic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Arabic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7925, - "uncertainty": { - "confidence_interval": { - "lower": -0.0397, - "upper": 0.0397, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "English", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - English", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7625, - "uncertainty": { - "confidence_interval": { - "lower": -0.0417, - "upper": 0.0417, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Bengali", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Bengali", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.825, - "uncertainty": { - "confidence_interval": { - "lower": -0.0372, - "upper": 0.0372, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "German", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - German", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8125, - "uncertainty": { - "confidence_interval": { - "lower": -0.0382, - "upper": 0.0382, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "French", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - French", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7675, - "uncertainty": { - "confidence_interval": { - "lower": -0.0414, - "upper": 0.0414, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Hindi", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Hindi", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.805, - "uncertainty": { - "confidence_interval": { - "lower": -0.0388, - "upper": 0.0388, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Indonesian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Indonesian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8175, - "uncertainty": { - "confidence_interval": { - "lower": -0.0379, - "upper": 0.0379, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Italian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Italian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8225, - "uncertainty": { - "confidence_interval": { - "lower": -0.0374, - "upper": 0.0374, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Japanese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Japanese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8425, - "uncertainty": { - "confidence_interval": { - "lower": -0.0357, - "upper": 0.0357, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Korean", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Korean", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.83, - "uncertainty": { - "confidence_interval": { - "lower": -0.0368, - "upper": 0.0368, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Portuguese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Portuguese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.77, - "uncertainty": { - "confidence_interval": { - "lower": -0.0412, - "upper": 0.0412, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Spanish", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Spanish", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8075, - "uncertainty": { - "confidence_interval": { - "lower": -0.0386, - "upper": 0.0386, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Swahili", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Swahili", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8125, - "uncertainty": { - "confidence_interval": { - "lower": -0.0382, - "upper": 0.0382, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Yoruba", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Yoruba", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.81, - "uncertainty": { - "confidence_interval": { - "lower": -0.0384, - "upper": 0.0384, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Chinese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Chinese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.835, - "uncertainty": { - "confidence_interval": { - "lower": -0.0364, - "upper": 0.0364, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Burmese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Burmese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8125, - "uncertainty": { - "confidence_interval": { - "lower": -0.0382, - "upper": 0.0382, - "method": "unknown" - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "global-mmlu-lite/anthropic_claude-3-7-sonnet-20250219/1773936583.743359", - "retrieved_timestamp": "1773936583.743359", - "source_metadata": { - "source_name": "Global MMLU Lite Leaderboard", - "source_type": "documentation", - "source_organization_name": "kaggle", - "source_organization_url": "www.kaggle.com", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "kaggle kernel", - "version": "4", - "additional_details": { - "url": "https://www.kaggle.com/code/shivalikasingh95/global-mmlu-lite-sample-notebook" - } - }, - "benchmark": "global-mmlu-lite", - "evaluation_results": [ - { - "evaluation_name": "Global MMLU Lite", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Global MMLU Lite", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8078 - } - }, - { - "evaluation_name": "Culturally Sensitive", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Sensitive", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7794 - } - }, - { - "evaluation_name": "Culturally Agnostic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Agnostic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8362 - } - }, - { - "evaluation_name": "Arabic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Arabic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7925, - "uncertainty": { - "confidence_interval": { - "lower": -0.0397, - "upper": 0.0397, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "English", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - English", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7625, - "uncertainty": { - "confidence_interval": { - "lower": -0.0417, - "upper": 0.0417, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Bengali", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Bengali", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.825, - "uncertainty": { - "confidence_interval": { - "lower": -0.0372, - "upper": 0.0372, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "German", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - German", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8125, - "uncertainty": { - "confidence_interval": { - "lower": -0.0382, - "upper": 0.0382, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "French", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - French", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7675, - "uncertainty": { - "confidence_interval": { - "lower": -0.0414, - "upper": 0.0414, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Hindi", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Hindi", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.805, - "uncertainty": { - "confidence_interval": { - "lower": -0.0388, - "upper": 0.0388, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Indonesian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Indonesian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8175, - "uncertainty": { - "confidence_interval": { - "lower": -0.0379, - "upper": 0.0379, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Italian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Italian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8225, - "uncertainty": { - "confidence_interval": { - "lower": -0.0374, - "upper": 0.0374, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Japanese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Japanese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8425, - "uncertainty": { - "confidence_interval": { - "lower": -0.0357, - "upper": 0.0357, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Korean", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Korean", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.83, - "uncertainty": { - "confidence_interval": { - "lower": -0.0368, - "upper": 0.0368, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Portuguese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Portuguese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.77, - "uncertainty": { - "confidence_interval": { - "lower": -0.0412, - "upper": 0.0412, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Spanish", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Spanish", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8075, - "uncertainty": { - "confidence_interval": { - "lower": -0.0386, - "upper": 0.0386, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Swahili", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Swahili", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8125, - "uncertainty": { - "confidence_interval": { - "lower": -0.0382, - "upper": 0.0382, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Yoruba", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Yoruba", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.81, - "uncertainty": { - "confidence_interval": { - "lower": -0.0384, - "upper": 0.0384, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Chinese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Chinese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.835, - "uncertainty": { - "confidence_interval": { - "lower": -0.0364, - "upper": 0.0364, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Burmese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Burmese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8125, - "uncertainty": { - "confidence_interval": { - "lower": -0.0382, - "upper": 0.0382, - "method": "unknown" - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "helm_capabilities/anthropic_claude-3-7-sonnet-20250219/1774096304.5056698", - "retrieved_timestamp": "1774096304.5056698", - "source_metadata": { - "source_name": "helm_capabilities", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_capabilities", - "evaluation_results": [ - { - "evaluation_name": "Mean score", - "source_data": { - "dataset_name": "helm_capabilities", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "The mean of the scores from all columns.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.674, - "details": { - "description": "", - "tab": "Accuracy", - "Mean score - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"9.05170552277221\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU-Pro", - "source_data": { - "dataset_name": "MMLU-Pro", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on MMLU-Pro", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.784, - "details": { - "description": "min=0.784, mean=0.784, max=0.784, sum=0.784 (1)", - "tab": "Accuracy", - "MMLU-Pro - Observed inference time (s)": "{\"description\": \"min=4.744, mean=4.744, max=4.744, sum=4.744 (1)\", \"tab\": \"Efficiency\", \"score\": \"4.744252296209336\"}", - "MMLU-Pro - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "MMLU-Pro - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - # prompt tokens": "{\"description\": \"min=230.461, mean=230.461, max=230.461, sum=230.461 (1)\", \"tab\": \"General information\", \"score\": \"230.461\"}", - "MMLU-Pro - # output tokens": "{\"description\": \"min=242.773, mean=242.773, max=242.773, sum=242.773 (1)\", \"tab\": \"General information\", \"score\": \"242.773\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"all\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.608, - "details": { - "description": "min=0.608, mean=0.608, max=0.608, sum=0.608 (1)", - "tab": "Accuracy", - "GPQA - Observed inference time (s)": "{\"description\": \"min=6.459, mean=6.459, max=6.459, sum=6.459 (1)\", \"tab\": \"Efficiency\", \"score\": \"6.4586481999923295\"}", - "GPQA - # eval": "{\"description\": \"min=446, mean=446, max=446, sum=446 (1)\", \"tab\": \"General information\", \"score\": \"446.0\"}", - "GPQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - # prompt tokens": "{\"description\": \"min=250.738, mean=250.738, max=250.738, sum=250.738 (1)\", \"tab\": \"General information\", \"score\": \"250.73766816143498\"}", - "GPQA - # output tokens": "{\"description\": \"min=312.666, mean=312.666, max=312.666, sum=312.666 (1)\", \"tab\": \"General information\", \"score\": \"312.6659192825112\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"gpqa_main\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "IFEval Strict Acc on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.834, - "details": { - "description": "min=0.834, mean=0.834, max=0.834, sum=0.834 (1)", - "tab": "Accuracy", - "IFEval - Observed inference time (s)": "{\"description\": \"min=8.075, mean=8.075, max=8.075, sum=8.075 (1)\", \"tab\": \"Efficiency\", \"score\": \"8.075105538870623\"}", - "IFEval - # eval": "{\"description\": \"min=541, mean=541, max=541, sum=541 (1)\", \"tab\": \"General information\", \"score\": \"541.0\"}", - "IFEval - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - # prompt tokens": "{\"description\": \"min=47.159, mean=47.159, max=47.159, sum=47.159 (1)\", \"tab\": \"General information\", \"score\": \"47.15896487985213\"}", - "IFEval - # output tokens": "{\"description\": \"min=406.532, mean=406.532, max=406.532, sum=406.532 (1)\", \"tab\": \"General information\", \"score\": \"406.5323475046211\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WildBench", - "source_data": { - "dataset_name": "WildBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "WB Score on WildBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.814, - "details": { - "description": "min=0.814, mean=0.814, max=0.814, sum=0.814 (1)", - "tab": "Accuracy", - "WildBench - Observed inference time (s)": "{\"description\": \"min=15.683, mean=15.683, max=15.683, sum=15.683 (1)\", \"tab\": \"Efficiency\", \"score\": \"15.682527210235596\"}", - "WildBench - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "WildBench - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # prompt tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # output tokens": "{\"description\": \"min=862.287, mean=862.287, max=862.287, sum=862.287 (1)\", \"tab\": \"General information\", \"score\": \"862.287\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"v2\"" - } - } - }, - { - "evaluation_name": "Omni-MATH", - "source_data": { - "dataset_name": "Omni-MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Acc on Omni-MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.33, - "details": { - "description": "min=0.33, mean=0.33, max=0.33, sum=0.33 (1)", - "tab": "Accuracy", - "Omni-MATH - Observed inference time (s)": "{\"description\": \"min=10.298, mean=10.298, max=10.298, sum=10.298 (1)\", \"tab\": \"Efficiency\", \"score\": \"10.297994368553162\"}", - "Omni-MATH - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "Omni-MATH - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - # prompt tokens": "{\"description\": \"min=110.563, mean=110.563, max=110.563, sum=110.563 (1)\", \"tab\": \"General information\", \"score\": \"110.563\"}", - "Omni-MATH - # output tokens": "{\"description\": \"min=670.885, mean=670.885, max=670.885, sum=670.885 (1)\", \"tab\": \"General information\", \"score\": \"670.885\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "livecodebenchpro/claude-3-7-sonnet-20250219/1760492095.8105888", - "retrieved_timestamp": "1760492095.8105888", - "source_metadata": { - "source_organization_name": "New York University, Princeton University, University of California San Diego, University of Washington and Canyon Crest Academy", - "evaluator_relationship": "third_party", - "source_name": "Live Code Bench Pro", - "source_type": "documentation" - }, - "eval_library": { - "name": "unknown", - "version": "unknown" - }, - "benchmark": "livecodebenchpro", - "evaluation_results": [ - { - "evaluation_name": "Hard Problems", - "metric_config": { - "evaluation_description": "Pass@1 on Hard Problems", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.0 - }, - "source_data": { - "dataset_name": "Hard Problems", - "source_type": "url", - "url": [ - "https://webhook.cp-bench.orzzh.com/leaderboard/llm/difficulty?difficulty=hard&benchmark_mode=live" - ] - } - }, - { - "evaluation_name": "Medium Problems", - "metric_config": { - "evaluation_description": "Pass@1 on Medium Problems", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.0 - }, - "source_data": { - "dataset_name": "Medium Problems", - "source_type": "url", - "url": [ - "https://webhook.cp-bench.orzzh.com/leaderboard/llm/difficulty?difficulty=medium&benchmark_mode=live" - ] - } - }, - { - "evaluation_name": "Easy Problems", - "metric_config": { - "evaluation_description": "Pass@1 on Easy Problems", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.28169014084507044 - }, - "source_data": { - "dataset_name": "Easy Problems", - "source_type": "url", - "url": [ - "https://webhook.cp-bench.orzzh.com/leaderboard/llm/difficulty?difficulty=easy&benchmark_mode=live" - ] - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "reward-bench-2/anthropic_claude-3-7-sonnet-20250219/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7539 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7326 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5437 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.75 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9033 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9212 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6723 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/anthropic_claude-3-haiku-20240307.json b/data/models/anthropic_claude-3-haiku-20240307.json deleted file mode 100644 index 52bb9959dd5613be4f3fe143bd6f01f97e2632c2..0000000000000000000000000000000000000000 --- a/data/models/anthropic_claude-3-haiku-20240307.json +++ /dev/null @@ -1,2188 +0,0 @@ -{ - "model_info": { - "name": "Claude 3 Haiku 20240307", - "id": "anthropic/claude-3-haiku-20240307", - "developer": "Anthropic", - "inference_platform": "unknown", - "additional_details": { - "model_id_aliases": [ - "Anthropic/claude-3-haiku-20240307" - ] - } - }, - "evaluations": [ - { - "evaluation_id": "helm_lite/anthropic_claude-3-haiku-20240307/1774096306.427425", - "retrieved_timestamp": "1774096306.427425", - "source_metadata": { - "source_name": "helm_lite", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_lite", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_lite", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.263, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.5421473158551811\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.244, - "details": { - "description": "min=0.244, mean=0.244, max=0.244, sum=0.244 (1)", - "tab": "Accuracy", - "NarrativeQA - Observed inference time (s)": "{\"description\": \"min=1.133, mean=1.133, max=1.133, sum=1.133 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.1334171402622277\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=3709.741, mean=3709.741, max=3709.741, sum=3709.741 (1)\", \"tab\": \"General information\", \"score\": \"3709.7408450704224\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=44.265, mean=44.265, max=44.265, sum=44.265 (1)\", \"tab\": \"General information\", \"score\": \"44.264788732394365\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (closed-book)", - "source_data": { - "dataset_name": "NaturalQuestions (closed-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (closed-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.144, - "details": { - "description": "min=0.144, mean=0.144, max=0.144, sum=0.144 (1)", - "tab": "Accuracy", - "NaturalQuestions (open-book) - Observed inference time (s)": "{\"description\": \"min=0.941, mean=0.941, max=0.941, sum=0.941 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.9411524205207825\"}", - "NaturalQuestions (closed-book) - Observed inference time (s)": "{\"description\": \"min=0.865, mean=0.865, max=0.865, sum=0.865 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.8646892714500427\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1781.799, mean=1781.799, max=1781.799, sum=1781.799 (1)\", \"tab\": \"General information\", \"score\": \"1781.799\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=33.024, mean=33.024, max=33.024, sum=33.024 (1)\", \"tab\": \"General information\", \"score\": \"33.024\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=189.259, mean=189.259, max=189.259, sum=189.259 (1)\", \"tab\": \"General information\", \"score\": \"189.259\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=50.787, mean=50.787, max=50.787, sum=50.787 (1)\", \"tab\": \"General information\", \"score\": \"50.787\"}" - } - }, - "generation_config": { - "additional_details": { - "mode": "\"closedbook\"" - } - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.838, - "details": { - "description": "min=0.838, mean=0.838, max=0.838, sum=0.838 (1)", - "tab": "Accuracy", - "OpenbookQA - Observed inference time (s)": "{\"description\": \"min=0.616, mean=0.616, max=0.616, sum=0.616 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.6164444308280945\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=263.79, mean=263.79, max=263.79, sum=263.79 (1)\", \"tab\": \"General information\", \"score\": \"263.79\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "dataset": "\"openbookqa\"", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.662, - "details": { - "description": "min=0.42, mean=0.662, max=0.95, sum=3.312 (5)", - "tab": "Accuracy", - "MMLU - Observed inference time (s)": "{\"description\": \"min=0.686, mean=0.697, max=0.721, sum=3.485 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.6970766685050831\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=370.26, mean=478.747, max=619.596, sum=2393.736 (5)\", \"tab\": \"General information\", \"score\": \"478.747298245614\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"college_chemistry\", \"computer_security\", \"econometrics\", \"us_foreign_policy\"]", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MATH", - "source_data": { - "dataset_name": "MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Equivalent (CoT) on MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.131, - "details": { - "description": "min=0, mean=0.131, max=0.504, sum=0.916 (7)", - "tab": "Accuracy", - "MATH - Observed inference time (s)": "{\"description\": \"min=0.672, mean=0.895, max=1.288, sum=6.265 (7)\", \"tab\": \"Efficiency\", \"score\": \"0.8950275982044664\"}", - "MATH - # eval": "{\"description\": \"min=30, mean=62.429, max=135, sum=437 (7)\", \"tab\": \"General information\", \"score\": \"62.42857142857143\"}", - "MATH - # train": "{\"description\": \"min=8, mean=8, max=8, sum=56 (7)\", \"tab\": \"General information\", \"score\": \"8.0\"}", - "MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MATH - # prompt tokens": "{\"description\": \"min=948.259, mean=1362.814, max=2380.808, sum=9539.699 (7)\", \"tab\": \"General information\", \"score\": \"1362.8141219676104\"}", - "MATH - # output tokens": "{\"description\": \"min=3.158, mean=29.033, max=87.17, sum=203.231 (7)\", \"tab\": \"General information\", \"score\": \"29.032964841043174\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"algebra\", \"counting_and_probability\", \"geometry\", \"intermediate_algebra\", \"number_theory\", \"prealgebra\", \"precalculus\"]", - "level": "\"1\"", - "use_official_examples": "\"False\"", - "use_chain_of_thought": "\"True\"" - } - } - }, - { - "evaluation_name": "GSM8K", - "source_data": { - "dataset_name": "GSM8K", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on GSM8K", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.699, - "details": { - "description": "min=0.699, mean=0.699, max=0.699, sum=0.699 (1)", - "tab": "Accuracy", - "GSM8K - Observed inference time (s)": "{\"description\": \"min=1.228, mean=1.228, max=1.228, sum=1.228 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.2278449382781982\"}", - "GSM8K - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "GSM8K - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "GSM8K - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GSM8K - # prompt tokens": "{\"description\": \"min=1012.712, mean=1012.712, max=1012.712, sum=1012.712 (1)\", \"tab\": \"General information\", \"score\": \"1012.712\"}", - "GSM8K - # output tokens": "{\"description\": \"min=77.518, mean=77.518, max=77.518, sum=77.518 (1)\", \"tab\": \"General information\", \"score\": \"77.518\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "LegalBench", - "source_data": { - "dataset_name": "LegalBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on LegalBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.46, - "details": { - "description": "min=0.034, mean=0.46, max=0.779, sum=2.301 (5)", - "tab": "Accuracy", - "LegalBench - Observed inference time (s)": "{\"description\": \"min=0.455, mean=0.719, max=0.988, sum=3.593 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.7186767522236834\"}", - "LegalBench - # eval": "{\"description\": \"min=95, mean=409.4, max=1000, sum=2047 (5)\", \"tab\": \"General information\", \"score\": \"409.4\"}", - "LegalBench - # train": "{\"description\": \"min=4, mean=4.8, max=5, sum=24 (5)\", \"tab\": \"General information\", \"score\": \"4.8\"}", - "LegalBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "LegalBench - # prompt tokens": "{\"description\": \"min=214.653, mean=1557.242, max=6428.398, sum=7786.208 (5)\", \"tab\": \"General information\", \"score\": \"1557.241581367783\"}", - "LegalBench - # output tokens": "{\"description\": \"min=1, mean=9.565, max=28.352, sum=47.824 (5)\", \"tab\": \"General information\", \"score\": \"9.56470087480281\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"abercrombie\", \"corporate_lobbying\", \"function_of_decision_section\", \"international_citizenship_questions\", \"proa\"]" - } - } - }, - { - "evaluation_name": "MedQA", - "source_data": { - "dataset_name": "MedQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MedQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.702, - "details": { - "description": "min=0.702, mean=0.702, max=0.702, sum=0.702 (1)", - "tab": "Accuracy", - "MedQA - Observed inference time (s)": "{\"description\": \"min=0.653, mean=0.653, max=0.653, sum=0.653 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.6529203475588121\"}", - "MedQA - # eval": "{\"description\": \"min=503, mean=503, max=503, sum=503 (1)\", \"tab\": \"General information\", \"score\": \"503.0\"}", - "MedQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MedQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MedQA - # prompt tokens": "{\"description\": \"min=1027.437, mean=1027.437, max=1027.437, sum=1027.437 (1)\", \"tab\": \"General information\", \"score\": \"1027.4373757455269\"}", - "MedQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WMT 2014", - "source_data": { - "dataset_name": "WMT 2014", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "BLEU-4 on WMT 2014", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.148, - "details": { - "description": "min=0.018, mean=0.148, max=0.208, sum=0.74 (5)", - "tab": "Accuracy", - "WMT 2014 - Observed inference time (s)": "{\"description\": \"min=0.627, mean=0.711, max=0.891, sum=3.556 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.7111122513056886\"}", - "WMT 2014 - # eval": "{\"description\": \"min=503, mean=568.8, max=832, sum=2844 (5)\", \"tab\": \"General information\", \"score\": \"568.8\"}", - "WMT 2014 - # train": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "WMT 2014 - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WMT 2014 - # prompt tokens": "{\"description\": \"min=198.406, mean=219.573, max=241.974, sum=1097.866 (5)\", \"tab\": \"General information\", \"score\": \"219.57322077152472\"}", - "WMT 2014 - # output tokens": "{\"description\": \"min=27.598, mean=48.613, max=93.673, sum=243.065 (5)\", \"tab\": \"General information\", \"score\": \"48.6129454044961\"}" - } - }, - "generation_config": { - "additional_details": { - "language_pair": "[\"cs-en\", \"de-en\", \"fr-en\", \"hi-en\", \"ru-en\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_mmlu/anthropic_claude-3-haiku-20240307/1774096312.00548", - "retrieved_timestamp": "1774096312.00548", - "source_metadata": { - "source_name": "helm_mmlu", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_mmlu", - "evaluation_results": [ - { - "evaluation_name": "MMLU All Subjects", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU All Subjects", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.738, - "details": { - "description": "min=0.37, mean=0.738, max=0.95, sum=84.132 (114)", - "tab": "Accuracy", - "MMLU All Subjects - Observed inference time (s)": "{\"description\": \"min=0.662, mean=0.734, max=1.711, sum=83.657 (114)\", \"tab\": \"Efficiency\", \"score\": \"0.7338373689865249\"}", - "MMLU All Subjects - # eval": "{\"description\": \"min=100, mean=246.351, max=1534, sum=28084 (114)\", \"tab\": \"General information\", \"score\": \"246.35087719298247\"}", - "MMLU All Subjects - # train": "{\"description\": \"min=5, mean=5, max=5, sum=570 (114)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU All Subjects - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU All Subjects - # prompt tokens": "{\"description\": \"min=293.018, mean=638.288, max=2887.576, sum=72764.875 (114)\", \"tab\": \"General information\", \"score\": \"638.2883793758953\"}", - "MMLU All Subjects - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=114 (114)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_name": "Abstract Algebra", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Abstract Algebra", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.42, - "details": { - "description": "min=0.42, mean=0.42, max=0.42, sum=0.84 (2)", - "tab": "Accuracy", - "Abstract Algebra - Observed inference time (s)": "{\"description\": \"min=0.693, mean=0.693, max=0.693, sum=1.386 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6928385472297669\"}", - "Abstract Algebra - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Abstract Algebra - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Abstract Algebra - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Abstract Algebra - # prompt tokens": "{\"description\": \"min=370.26, mean=370.26, max=370.26, sum=740.52 (2)\", \"tab\": \"General information\", \"score\": \"370.26\"}", - "Abstract Algebra - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"abstract_algebra\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_abstract_algebra\"" - } - } - }, - { - "evaluation_name": "Anatomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Anatomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.711, - "details": { - "description": "min=0.711, mean=0.711, max=0.711, sum=1.422 (2)", - "tab": "Accuracy", - "Anatomy - Observed inference time (s)": "{\"description\": \"min=0.668, mean=0.668, max=0.668, sum=1.336 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6677785749788637\"}", - "Anatomy - # eval": "{\"description\": \"min=135, mean=135, max=135, sum=270 (2)\", \"tab\": \"General information\", \"score\": \"135.0\"}", - "Anatomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Anatomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Anatomy - # prompt tokens": "{\"description\": \"min=370.8, mean=370.8, max=370.8, sum=741.6 (2)\", \"tab\": \"General information\", \"score\": \"370.8\"}", - "Anatomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"anatomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_anatomy\"" - } - } - }, - { - "evaluation_name": "College Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on College Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.48, - "details": { - "description": "min=0.48, mean=0.48, max=0.48, sum=0.961 (2)", - "tab": "Accuracy", - "College Chemistry - Observed inference time (s)": "{\"description\": \"min=0.692, mean=0.692, max=0.692, sum=1.385 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6923453903198242\"}", - "College Biology - Observed inference time (s)": "{\"description\": \"min=0.702, mean=0.702, max=0.702, sum=1.405 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7022541695170932\"}", - "College Computer Science - Observed inference time (s)": "{\"description\": \"min=0.735, mean=0.735, max=0.735, sum=1.47 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7352152991294861\"}", - "College Mathematics - Observed inference time (s)": "{\"description\": \"min=0.715, mean=0.715, max=0.715, sum=1.43 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7152474927902222\"}", - "College Medicine - Observed inference time (s)": "{\"description\": \"min=0.713, mean=0.713, max=0.713, sum=1.425 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7125603780581083\"}", - "College Physics - Observed inference time (s)": "{\"description\": \"min=0.726, mean=0.726, max=0.726, sum=1.453 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7264628340216244\"}", - "College Chemistry - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Chemistry - # prompt tokens": "{\"description\": \"min=550.01, mean=550.01, max=550.01, sum=1100.02 (2)\", \"tab\": \"General information\", \"score\": \"550.01\"}", - "College Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Biology - # eval": "{\"description\": \"min=144, mean=144, max=144, sum=288 (2)\", \"tab\": \"General information\", \"score\": \"144.0\"}", - "College Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # prompt tokens": "{\"description\": \"min=490.347, mean=490.347, max=490.347, sum=980.694 (2)\", \"tab\": \"General information\", \"score\": \"490.34722222222223\"}", - "College Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # prompt tokens": "{\"description\": \"min=838.24, mean=838.24, max=838.24, sum=1676.48 (2)\", \"tab\": \"General information\", \"score\": \"838.24\"}", - "College Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Mathematics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # prompt tokens": "{\"description\": \"min=604.19, mean=604.19, max=604.19, sum=1208.38 (2)\", \"tab\": \"General information\", \"score\": \"604.19\"}", - "College Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Medicine - # eval": "{\"description\": \"min=173, mean=173, max=173, sum=346 (2)\", \"tab\": \"General information\", \"score\": \"173.0\"}", - "College Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # prompt tokens": "{\"description\": \"min=540.63, mean=540.63, max=540.63, sum=1081.26 (2)\", \"tab\": \"General information\", \"score\": \"540.6300578034682\"}", - "College Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Physics - # eval": "{\"description\": \"min=102, mean=102, max=102, sum=204 (2)\", \"tab\": \"General information\", \"score\": \"102.0\"}", - "College Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # prompt tokens": "{\"description\": \"min=489.48, mean=489.48, max=489.48, sum=978.961 (2)\", \"tab\": \"General information\", \"score\": \"489.48039215686276\"}", - "College Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"college_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_college_physics\"" - } - } - }, - { - "evaluation_name": "Computer Security", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Computer Security", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.79, - "details": { - "description": "min=0.79, mean=0.79, max=0.79, sum=1.58 (2)", - "tab": "Accuracy", - "Computer Security - Observed inference time (s)": "{\"description\": \"min=0.686, mean=0.686, max=0.686, sum=1.371 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6855517983436584\"}", - "Computer Security - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Computer Security - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Computer Security - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Computer Security - # prompt tokens": "{\"description\": \"min=398.62, mean=398.62, max=398.62, sum=797.24 (2)\", \"tab\": \"General information\", \"score\": \"398.62\"}", - "Computer Security - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"computer_security\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_computer_security\"" - } - } - }, - { - "evaluation_name": "Econometrics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Econometrics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.632, - "details": { - "description": "min=0.632, mean=0.632, max=0.632, sum=1.263 (2)", - "tab": "Accuracy", - "Econometrics - Observed inference time (s)": "{\"description\": \"min=0.721, mean=0.721, max=0.721, sum=1.442 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.720871933719568\"}", - "Econometrics - # eval": "{\"description\": \"min=114, mean=114, max=114, sum=228 (2)\", \"tab\": \"General information\", \"score\": \"114.0\"}", - "Econometrics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Econometrics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Econometrics - # prompt tokens": "{\"description\": \"min=619.596, mean=619.596, max=619.596, sum=1239.193 (2)\", \"tab\": \"General information\", \"score\": \"619.5964912280701\"}", - "Econometrics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"econometrics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_econometrics\"" - } - } - }, - { - "evaluation_name": "Global Facts", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Global Facts", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.47, - "details": { - "description": "min=0.47, mean=0.47, max=0.47, sum=0.94 (2)", - "tab": "Accuracy", - "Global Facts - Observed inference time (s)": "{\"description\": \"min=0.671, mean=0.671, max=0.671, sum=1.342 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6710420751571655\"}", - "Global Facts - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Global Facts - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Global Facts - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Global Facts - # prompt tokens": "{\"description\": \"min=411.61, mean=411.61, max=411.61, sum=823.22 (2)\", \"tab\": \"General information\", \"score\": \"411.61\"}", - "Global Facts - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"global_facts\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_global_facts\"" - } - } - }, - { - "evaluation_name": "Jurisprudence", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Jurisprudence", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.861, - "details": { - "description": "min=0.861, mean=0.861, max=0.861, sum=1.722 (2)", - "tab": "Accuracy", - "Jurisprudence - Observed inference time (s)": "{\"description\": \"min=0.717, mean=0.717, max=0.717, sum=1.435 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7174532214800516\"}", - "Jurisprudence - # eval": "{\"description\": \"min=108, mean=108, max=108, sum=216 (2)\", \"tab\": \"General information\", \"score\": \"108.0\"}", - "Jurisprudence - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Jurisprudence - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Jurisprudence - # prompt tokens": "{\"description\": \"min=431.426, mean=431.426, max=431.426, sum=862.852 (2)\", \"tab\": \"General information\", \"score\": \"431.4259259259259\"}", - "Jurisprudence - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"jurisprudence\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_jurisprudence\"" - } - } - }, - { - "evaluation_name": "Philosophy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Philosophy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.814, - "details": { - "description": "min=0.814, mean=0.814, max=0.814, sum=1.627 (2)", - "tab": "Accuracy", - "Philosophy - Observed inference time (s)": "{\"description\": \"min=0.702, mean=0.702, max=0.702, sum=1.405 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7023597537896258\"}", - "Philosophy - # eval": "{\"description\": \"min=311, mean=311, max=311, sum=622 (2)\", \"tab\": \"General information\", \"score\": \"311.0\"}", - "Philosophy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Philosophy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Philosophy - # prompt tokens": "{\"description\": \"min=359.965, mean=359.965, max=359.965, sum=719.929 (2)\", \"tab\": \"General information\", \"score\": \"359.9646302250804\"}", - "Philosophy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"philosophy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_philosophy\"" - } - } - }, - { - "evaluation_name": "Professional Psychology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Professional Psychology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.802, - "details": { - "description": "min=0.802, mean=0.802, max=0.802, sum=1.605 (2)", - "tab": "Accuracy", - "Professional Medicine - Observed inference time (s)": "{\"description\": \"min=0.786, mean=0.786, max=0.786, sum=1.572 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7859190036268795\"}", - "Professional Accounting - Observed inference time (s)": "{\"description\": \"min=0.771, mean=0.771, max=0.771, sum=1.542 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7710303414797952\"}", - "Professional Law - Observed inference time (s)": "{\"description\": \"min=0.826, mean=0.826, max=0.826, sum=1.652 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8259650812310687\"}", - "Professional Psychology - Observed inference time (s)": "{\"description\": \"min=1.711, mean=1.711, max=1.711, sum=3.422 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.7109862737406314\"}", - "Professional Medicine - # eval": "{\"description\": \"min=272, mean=272, max=272, sum=544 (2)\", \"tab\": \"General information\", \"score\": \"272.0\"}", - "Professional Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Medicine - # prompt tokens": "{\"description\": \"min=1123.537, mean=1123.537, max=1123.537, sum=2247.074 (2)\", \"tab\": \"General information\", \"score\": \"1123.5367647058824\"}", - "Professional Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Accounting - # eval": "{\"description\": \"min=282, mean=282, max=282, sum=564 (2)\", \"tab\": \"General information\", \"score\": \"282.0\"}", - "Professional Accounting - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Accounting - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # prompt tokens": "{\"description\": \"min=665.422, mean=665.422, max=665.422, sum=1330.844 (2)\", \"tab\": \"General information\", \"score\": \"665.4219858156029\"}", - "Professional Accounting - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Law - # eval": "{\"description\": \"min=1534, mean=1534, max=1534, sum=3068 (2)\", \"tab\": \"General information\", \"score\": \"1534.0\"}", - "Professional Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # prompt tokens": "{\"description\": \"min=1701.16, mean=1701.16, max=1701.16, sum=3402.321 (2)\", \"tab\": \"General information\", \"score\": \"1701.16036505867\"}", - "Professional Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Psychology - # eval": "{\"description\": \"min=612, mean=612, max=612, sum=1224 (2)\", \"tab\": \"General information\", \"score\": \"612.0\"}", - "Professional Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # prompt tokens": "{\"description\": \"min=603.168, mean=603.168, max=603.168, sum=1206.337 (2)\", \"tab\": \"General information\", \"score\": \"603.1683006535948\"}", - "Professional Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"professional_psychology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_professional_psychology\"" - } - } - }, - { - "evaluation_name": "Us Foreign Policy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Us Foreign Policy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.95, - "details": { - "description": "min=0.95, mean=0.95, max=0.95, sum=1.9 (2)", - "tab": "Accuracy", - "Us Foreign Policy - Observed inference time (s)": "{\"description\": \"min=0.694, mean=0.694, max=0.694, sum=1.388 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6937756729125977\"}", - "Us Foreign Policy - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Us Foreign Policy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Us Foreign Policy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Us Foreign Policy - # prompt tokens": "{\"description\": \"min=455.25, mean=455.25, max=455.25, sum=910.5 (2)\", \"tab\": \"General information\", \"score\": \"455.25\"}", - "Us Foreign Policy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"us_foreign_policy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_us_foreign_policy\"" - } - } - }, - { - "evaluation_name": "Astronomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Astronomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.901, - "details": { - "description": "min=0.901, mean=0.901, max=0.901, sum=1.803 (2)", - "tab": "Accuracy", - "Astronomy - Observed inference time (s)": "{\"description\": \"min=0.707, mean=0.707, max=0.707, sum=1.415 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7072845524863193\"}", - "Astronomy - # eval": "{\"description\": \"min=152, mean=152, max=152, sum=304 (2)\", \"tab\": \"General information\", \"score\": \"152.0\"}", - "Astronomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Astronomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Astronomy - # prompt tokens": "{\"description\": \"min=604.493, mean=604.493, max=604.493, sum=1208.987 (2)\", \"tab\": \"General information\", \"score\": \"604.4934210526316\"}", - "Astronomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"astronomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_astronomy\"" - } - } - }, - { - "evaluation_name": "Business Ethics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Business Ethics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.78, - "details": { - "description": "min=0.78, mean=0.78, max=0.78, sum=1.56 (2)", - "tab": "Accuracy", - "Business Ethics - Observed inference time (s)": "{\"description\": \"min=0.705, mean=0.705, max=0.705, sum=1.411 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7054399585723877\"}", - "Business Ethics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Business Ethics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Business Ethics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Business Ethics - # prompt tokens": "{\"description\": \"min=600.02, mean=600.02, max=600.02, sum=1200.04 (2)\", \"tab\": \"General information\", \"score\": \"600.02\"}", - "Business Ethics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"business_ethics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_business_ethics\"" - } - } - }, - { - "evaluation_name": "Clinical Knowledge", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Clinical Knowledge", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.789, - "details": { - "description": "min=0.789, mean=0.789, max=0.789, sum=1.577 (2)", - "tab": "Accuracy", - "Clinical Knowledge - Observed inference time (s)": "{\"description\": \"min=0.716, mean=0.716, max=0.716, sum=1.432 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7159239804969644\"}", - "Clinical Knowledge - # eval": "{\"description\": \"min=265, mean=265, max=265, sum=530 (2)\", \"tab\": \"General information\", \"score\": \"265.0\"}", - "Clinical Knowledge - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Clinical Knowledge - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Clinical Knowledge - # prompt tokens": "{\"description\": \"min=429.457, mean=429.457, max=429.457, sum=858.913 (2)\", \"tab\": \"General information\", \"score\": \"429.4566037735849\"}", - "Clinical Knowledge - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"clinical_knowledge\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_clinical_knowledge\"" - } - } - }, - { - "evaluation_name": "Conceptual Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Conceptual Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.715, - "details": { - "description": "min=0.715, mean=0.715, max=0.715, sum=1.43 (2)", - "tab": "Accuracy", - "Conceptual Physics - Observed inference time (s)": "{\"description\": \"min=0.686, mean=0.686, max=0.686, sum=1.373 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.686391481440118\"}", - "Conceptual Physics - # eval": "{\"description\": \"min=235, mean=235, max=235, sum=470 (2)\", \"tab\": \"General information\", \"score\": \"235.0\"}", - "Conceptual Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Conceptual Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Conceptual Physics - # prompt tokens": "{\"description\": \"min=323.536, mean=323.536, max=323.536, sum=647.072 (2)\", \"tab\": \"General information\", \"score\": \"323.53617021276597\"}", - "Conceptual Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"conceptual_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_conceptual_physics\"" - } - } - }, - { - "evaluation_name": "Electrical Engineering", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Electrical Engineering", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.69, - "details": { - "description": "min=0.69, mean=0.69, max=0.69, sum=1.379 (2)", - "tab": "Accuracy", - "Electrical Engineering - Observed inference time (s)": "{\"description\": \"min=0.696, mean=0.696, max=0.696, sum=1.392 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6958530524681354\"}", - "Electrical Engineering - # eval": "{\"description\": \"min=145, mean=145, max=145, sum=290 (2)\", \"tab\": \"General information\", \"score\": \"145.0\"}", - "Electrical Engineering - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Electrical Engineering - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Electrical Engineering - # prompt tokens": "{\"description\": \"min=437.041, mean=437.041, max=437.041, sum=874.083 (2)\", \"tab\": \"General information\", \"score\": \"437.04137931034484\"}", - "Electrical Engineering - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"electrical_engineering\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_electrical_engineering\"" - } - } - }, - { - "evaluation_name": "Elementary Mathematics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Elementary Mathematics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.558, - "details": { - "description": "min=0.558, mean=0.558, max=0.558, sum=1.116 (2)", - "tab": "Accuracy", - "Elementary Mathematics - Observed inference time (s)": "{\"description\": \"min=0.734, mean=0.734, max=0.734, sum=1.468 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.73423323177156\"}", - "Elementary Mathematics - # eval": "{\"description\": \"min=378, mean=378, max=378, sum=756 (2)\", \"tab\": \"General information\", \"score\": \"378.0\"}", - "Elementary Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Elementary Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Elementary Mathematics - # prompt tokens": "{\"description\": \"min=511.066, mean=511.066, max=511.066, sum=1022.132 (2)\", \"tab\": \"General information\", \"score\": \"511.06613756613757\"}", - "Elementary Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"elementary_mathematics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_elementary_mathematics\"" - } - } - }, - { - "evaluation_name": "Formal Logic", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Formal Logic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.579, - "details": { - "description": "min=0.579, mean=0.579, max=0.579, sum=1.159 (2)", - "tab": "Accuracy", - "Formal Logic - Observed inference time (s)": "{\"description\": \"min=0.731, mean=0.731, max=0.731, sum=1.462 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7307745880550809\"}", - "Formal Logic - # eval": "{\"description\": \"min=126, mean=126, max=126, sum=252 (2)\", \"tab\": \"General information\", \"score\": \"126.0\"}", - "Formal Logic - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Formal Logic - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Formal Logic - # prompt tokens": "{\"description\": \"min=646.746, mean=646.746, max=646.746, sum=1293.492 (2)\", \"tab\": \"General information\", \"score\": \"646.7460317460317\"}", - "Formal Logic - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"formal_logic\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_formal_logic\"" - } - } - }, - { - "evaluation_name": "High School World History", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on High School World History", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.878, - "details": { - "description": "min=0.878, mean=0.878, max=0.878, sum=1.755 (2)", - "tab": "Accuracy", - "High School Biology - Observed inference time (s)": "{\"description\": \"min=0.714, mean=0.714, max=0.714, sum=1.428 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7141557578117617\"}", - "High School Chemistry - Observed inference time (s)": "{\"description\": \"min=0.701, mean=0.701, max=0.701, sum=1.403 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7014370187750003\"}", - "High School Computer Science - Observed inference time (s)": "{\"description\": \"min=0.747, mean=0.747, max=0.747, sum=1.494 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7470939707756042\"}", - "High School European History - Observed inference time (s)": "{\"description\": \"min=0.966, mean=0.966, max=0.966, sum=1.932 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9658473159327652\"}", - "High School Geography - Observed inference time (s)": "{\"description\": \"min=0.663, mean=0.663, max=0.663, sum=1.326 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6627856938525883\"}", - "High School Government And Politics - Observed inference time (s)": "{\"description\": \"min=0.686, mean=0.686, max=0.686, sum=1.373 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6863837884497767\"}", - "High School Macroeconomics - Observed inference time (s)": "{\"description\": \"min=0.681, mean=0.681, max=0.681, sum=1.361 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6806940922370324\"}", - "High School Mathematics - Observed inference time (s)": "{\"description\": \"min=0.708, mean=0.708, max=0.708, sum=1.416 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7079638242721558\"}", - "High School Microeconomics - Observed inference time (s)": "{\"description\": \"min=0.674, mean=0.674, max=0.674, sum=1.348 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6742001541522371\"}", - "High School Physics - Observed inference time (s)": "{\"description\": \"min=0.706, mean=0.706, max=0.706, sum=1.411 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7056786966639639\"}", - "High School Psychology - Observed inference time (s)": "{\"description\": \"min=0.696, mean=0.696, max=0.696, sum=1.392 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6960603683366688\"}", - "High School Statistics - Observed inference time (s)": "{\"description\": \"min=0.726, mean=0.726, max=0.726, sum=1.452 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7262004735293212\"}", - "High School US History - Observed inference time (s)": "{\"description\": \"min=0.876, mean=0.876, max=0.876, sum=1.752 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8757836842069439\"}", - "High School World History - Observed inference time (s)": "{\"description\": \"min=0.785, mean=0.785, max=0.785, sum=1.571 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7852678007214381\"}", - "High School Biology - # eval": "{\"description\": \"min=310, mean=310, max=310, sum=620 (2)\", \"tab\": \"General information\", \"score\": \"310.0\"}", - "High School Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Biology - # prompt tokens": "{\"description\": \"min=534.577, mean=534.577, max=534.577, sum=1069.155 (2)\", \"tab\": \"General information\", \"score\": \"534.5774193548388\"}", - "High School Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Chemistry - # eval": "{\"description\": \"min=203, mean=203, max=203, sum=406 (2)\", \"tab\": \"General information\", \"score\": \"203.0\"}", - "High School Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # prompt tokens": "{\"description\": \"min=497.921, mean=497.921, max=497.921, sum=995.842 (2)\", \"tab\": \"General information\", \"score\": \"497.92118226600985\"}", - "High School Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "High School Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # prompt tokens": "{\"description\": \"min=882.4, mean=882.4, max=882.4, sum=1764.8 (2)\", \"tab\": \"General information\", \"score\": \"882.4\"}", - "High School Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School European History - # eval": "{\"description\": \"min=165, mean=165, max=165, sum=330 (2)\", \"tab\": \"General information\", \"score\": \"165.0\"}", - "High School European History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School European History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # prompt tokens": "{\"description\": \"min=2887.576, mean=2887.576, max=2887.576, sum=5775.152 (2)\", \"tab\": \"General information\", \"score\": \"2887.5757575757575\"}", - "High School European History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Geography - # eval": "{\"description\": \"min=198, mean=198, max=198, sum=396 (2)\", \"tab\": \"General information\", \"score\": \"198.0\"}", - "High School Geography - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Geography - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # prompt tokens": "{\"description\": \"min=412.268, mean=412.268, max=412.268, sum=824.535 (2)\", \"tab\": \"General information\", \"score\": \"412.2676767676768\"}", - "High School Geography - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Government And Politics - # eval": "{\"description\": \"min=193, mean=193, max=193, sum=386 (2)\", \"tab\": \"General information\", \"score\": \"193.0\"}", - "High School Government And Politics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Government And Politics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # prompt tokens": "{\"description\": \"min=491.104, mean=491.104, max=491.104, sum=982.207 (2)\", \"tab\": \"General information\", \"score\": \"491.10362694300517\"}", - "High School Government And Politics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Macroeconomics - # eval": "{\"description\": \"min=390, mean=390, max=390, sum=780 (2)\", \"tab\": \"General information\", \"score\": \"390.0\"}", - "High School Macroeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Macroeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # prompt tokens": "{\"description\": \"min=406.036, mean=406.036, max=406.036, sum=812.072 (2)\", \"tab\": \"General information\", \"score\": \"406.0358974358974\"}", - "High School Macroeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Mathematics - # eval": "{\"description\": \"min=270, mean=270, max=270, sum=540 (2)\", \"tab\": \"General information\", \"score\": \"270.0\"}", - "High School Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # prompt tokens": "{\"description\": \"min=519.881, mean=519.881, max=519.881, sum=1039.763 (2)\", \"tab\": \"General information\", \"score\": \"519.8814814814815\"}", - "High School Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Microeconomics - # eval": "{\"description\": \"min=238, mean=238, max=238, sum=476 (2)\", \"tab\": \"General information\", \"score\": \"238.0\"}", - "High School Microeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Microeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # prompt tokens": "{\"description\": \"min=420.513, mean=420.513, max=420.513, sum=841.025 (2)\", \"tab\": \"General information\", \"score\": \"420.5126050420168\"}", - "High School Microeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Physics - # eval": "{\"description\": \"min=151, mean=151, max=151, sum=302 (2)\", \"tab\": \"General information\", \"score\": \"151.0\"}", - "High School Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # prompt tokens": "{\"description\": \"min=558.841, mean=558.841, max=558.841, sum=1117.682 (2)\", \"tab\": \"General information\", \"score\": \"558.841059602649\"}", - "High School Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Psychology - # eval": "{\"description\": \"min=545, mean=545, max=545, sum=1090 (2)\", \"tab\": \"General information\", \"score\": \"545.0\"}", - "High School Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # prompt tokens": "{\"description\": \"min=521.42, mean=521.42, max=521.42, sum=1042.84 (2)\", \"tab\": \"General information\", \"score\": \"521.4201834862386\"}", - "High School Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Statistics - # eval": "{\"description\": \"min=216, mean=216, max=216, sum=432 (2)\", \"tab\": \"General information\", \"score\": \"216.0\"}", - "High School Statistics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Statistics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # prompt tokens": "{\"description\": \"min=806.963, mean=806.963, max=806.963, sum=1613.926 (2)\", \"tab\": \"General information\", \"score\": \"806.9629629629629\"}", - "High School Statistics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School US History - # eval": "{\"description\": \"min=204, mean=204, max=204, sum=408 (2)\", \"tab\": \"General information\", \"score\": \"204.0\"}", - "High School US History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School US History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # prompt tokens": "{\"description\": \"min=2288.49, mean=2288.49, max=2288.49, sum=4576.98 (2)\", \"tab\": \"General information\", \"score\": \"2288.4901960784314\"}", - "High School US History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School World History - # eval": "{\"description\": \"min=237, mean=237, max=237, sum=474 (2)\", \"tab\": \"General information\", \"score\": \"237.0\"}", - "High School World History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School World History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # prompt tokens": "{\"description\": \"min=1475.932, mean=1475.932, max=1475.932, sum=2951.865 (2)\", \"tab\": \"General information\", \"score\": \"1475.9324894514768\"}", - "High School World History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"high_school_world_history\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_high_school_world_history\"" - } - } - }, - { - "evaluation_name": "Human Sexuality", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Human Sexuality", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.824, - "details": { - "description": "min=0.824, mean=0.824, max=0.824, sum=1.649 (2)", - "tab": "Accuracy", - "Human Aging - Observed inference time (s)": "{\"description\": \"min=0.691, mean=0.691, max=0.691, sum=1.382 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6907867818669888\"}", - "Human Sexuality - Observed inference time (s)": "{\"description\": \"min=0.665, mean=0.665, max=0.665, sum=1.331 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6653509722411177\"}", - "Human Aging - # eval": "{\"description\": \"min=223, mean=223, max=223, sum=446 (2)\", \"tab\": \"General information\", \"score\": \"223.0\"}", - "Human Aging - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Aging - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Aging - # prompt tokens": "{\"description\": \"min=335.955, mean=335.955, max=335.955, sum=671.91 (2)\", \"tab\": \"General information\", \"score\": \"335.95515695067263\"}", - "Human Aging - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Human Sexuality - # eval": "{\"description\": \"min=131, mean=131, max=131, sum=262 (2)\", \"tab\": \"General information\", \"score\": \"131.0\"}", - "Human Sexuality - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Sexuality - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # prompt tokens": "{\"description\": \"min=371.496, mean=371.496, max=371.496, sum=742.992 (2)\", \"tab\": \"General information\", \"score\": \"371.4961832061069\"}", - "Human Sexuality - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"human_sexuality\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_human_sexuality\"" - } - } - }, - { - "evaluation_name": "International Law", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on International Law", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.901, - "details": { - "description": "min=0.901, mean=0.901, max=0.901, sum=1.802 (2)", - "tab": "Accuracy", - "International Law - Observed inference time (s)": "{\"description\": \"min=0.723, mean=0.723, max=0.723, sum=1.446 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7232089219999708\"}", - "International Law - # eval": "{\"description\": \"min=121, mean=121, max=121, sum=242 (2)\", \"tab\": \"General information\", \"score\": \"121.0\"}", - "International Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "International Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "International Law - # prompt tokens": "{\"description\": \"min=664.165, mean=664.165, max=664.165, sum=1328.331 (2)\", \"tab\": \"General information\", \"score\": \"664.1652892561983\"}", - "International Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"international_law\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_international_law\"" - } - } - }, - { - "evaluation_name": "Logical Fallacies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Logical Fallacies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.791, - "details": { - "description": "min=0.791, mean=0.791, max=0.791, sum=1.583 (2)", - "tab": "Accuracy", - "Logical Fallacies - Observed inference time (s)": "{\"description\": \"min=0.665, mean=0.665, max=0.665, sum=1.331 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6653785354520646\"}", - "Logical Fallacies - # eval": "{\"description\": \"min=163, mean=163, max=163, sum=326 (2)\", \"tab\": \"General information\", \"score\": \"163.0\"}", - "Logical Fallacies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Logical Fallacies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Logical Fallacies - # prompt tokens": "{\"description\": \"min=470.276, mean=470.276, max=470.276, sum=940.552 (2)\", \"tab\": \"General information\", \"score\": \"470.2760736196319\"}", - "Logical Fallacies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"logical_fallacies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_logical_fallacies\"" - } - } - }, - { - "evaluation_name": "Machine Learning", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Machine Learning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.589, - "details": { - "description": "min=0.589, mean=0.589, max=0.589, sum=1.179 (2)", - "tab": "Accuracy", - "Machine Learning - Observed inference time (s)": "{\"description\": \"min=0.722, mean=0.722, max=0.722, sum=1.444 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7220823402915683\"}", - "Machine Learning - # eval": "{\"description\": \"min=112, mean=112, max=112, sum=224 (2)\", \"tab\": \"General information\", \"score\": \"112.0\"}", - "Machine Learning - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Machine Learning - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Machine Learning - # prompt tokens": "{\"description\": \"min=676.518, mean=676.518, max=676.518, sum=1353.036 (2)\", \"tab\": \"General information\", \"score\": \"676.5178571428571\"}", - "Machine Learning - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"machine_learning\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_machine_learning\"" - } - } - }, - { - "evaluation_name": "Management", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Management", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.874, - "details": { - "description": "min=0.874, mean=0.874, max=0.874, sum=1.748 (2)", - "tab": "Accuracy", - "Management - Observed inference time (s)": "{\"description\": \"min=0.663, mean=0.663, max=0.663, sum=1.327 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6634428709456064\"}", - "Management - # eval": "{\"description\": \"min=103, mean=103, max=103, sum=206 (2)\", \"tab\": \"General information\", \"score\": \"103.0\"}", - "Management - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Management - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Management - # prompt tokens": "{\"description\": \"min=301.282, mean=301.282, max=301.282, sum=602.563 (2)\", \"tab\": \"General information\", \"score\": \"301.28155339805824\"}", - "Management - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"management\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_management\"" - } - } - }, - { - "evaluation_name": "Marketing", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Marketing", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.91, - "details": { - "description": "min=0.91, mean=0.91, max=0.91, sum=1.821 (2)", - "tab": "Accuracy", - "Marketing - Observed inference time (s)": "{\"description\": \"min=0.665, mean=0.665, max=0.665, sum=1.33 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6648106361046816\"}", - "Marketing - # eval": "{\"description\": \"min=234, mean=234, max=234, sum=468 (2)\", \"tab\": \"General information\", \"score\": \"234.0\"}", - "Marketing - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Marketing - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Marketing - # prompt tokens": "{\"description\": \"min=448.064, mean=448.064, max=448.064, sum=896.128 (2)\", \"tab\": \"General information\", \"score\": \"448.06410256410254\"}", - "Marketing - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"marketing\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_marketing\"" - } - } - }, - { - "evaluation_name": "Medical Genetics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Medical Genetics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8, - "details": { - "description": "min=0.8, mean=0.8, max=0.8, sum=1.6 (2)", - "tab": "Accuracy", - "Medical Genetics - Observed inference time (s)": "{\"description\": \"min=0.662, mean=0.662, max=0.662, sum=1.324 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6621059203147888\"}", - "Medical Genetics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Medical Genetics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Medical Genetics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Medical Genetics - # prompt tokens": "{\"description\": \"min=354.88, mean=354.88, max=354.88, sum=709.76 (2)\", \"tab\": \"General information\", \"score\": \"354.88\"}", - "Medical Genetics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"medical_genetics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_medical_genetics\"" - } - } - }, - { - "evaluation_name": "Miscellaneous", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Miscellaneous", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.893, - "details": { - "description": "min=0.893, mean=0.893, max=0.893, sum=1.785 (2)", - "tab": "Accuracy", - "Miscellaneous - Observed inference time (s)": "{\"description\": \"min=0.678, mean=0.678, max=0.678, sum=1.357 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6782779660109207\"}", - "Miscellaneous - # eval": "{\"description\": \"min=783, mean=783, max=783, sum=1566 (2)\", \"tab\": \"General information\", \"score\": \"783.0\"}", - "Miscellaneous - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Miscellaneous - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Miscellaneous - # prompt tokens": "{\"description\": \"min=328.628, mean=328.628, max=328.628, sum=657.257 (2)\", \"tab\": \"General information\", \"score\": \"328.62835249042143\"}", - "Miscellaneous - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"miscellaneous\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_miscellaneous\"" - } - } - }, - { - "evaluation_name": "Moral Scenarios", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Moral Scenarios", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.502, - "details": { - "description": "min=0.502, mean=0.502, max=0.502, sum=1.003 (2)", - "tab": "Accuracy", - "Moral Disputes - Observed inference time (s)": "{\"description\": \"min=0.709, mean=0.709, max=0.709, sum=1.419 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7093146880927114\"}", - "Moral Scenarios - Observed inference time (s)": "{\"description\": \"min=0.716, mean=0.716, max=0.716, sum=1.432 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7158833943265777\"}", - "Moral Disputes - # eval": "{\"description\": \"min=346, mean=346, max=346, sum=692 (2)\", \"tab\": \"General information\", \"score\": \"346.0\"}", - "Moral Disputes - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Disputes - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Disputes - # prompt tokens": "{\"description\": \"min=511.789, mean=511.789, max=511.789, sum=1023.578 (2)\", \"tab\": \"General information\", \"score\": \"511.78901734104045\"}", - "Moral Disputes - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Moral Scenarios - # eval": "{\"description\": \"min=895, mean=895, max=895, sum=1790 (2)\", \"tab\": \"General information\", \"score\": \"895.0\"}", - "Moral Scenarios - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Scenarios - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # prompt tokens": "{\"description\": \"min=676.949, mean=676.949, max=676.949, sum=1353.897 (2)\", \"tab\": \"General information\", \"score\": \"676.9486033519553\"}", - "Moral Scenarios - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"moral_scenarios\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_moral_scenarios\"" - } - } - }, - { - "evaluation_name": "Nutrition", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Nutrition", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.83, - "details": { - "description": "min=0.83, mean=0.83, max=0.83, sum=1.66 (2)", - "tab": "Accuracy", - "Nutrition - Observed inference time (s)": "{\"description\": \"min=0.72, mean=0.72, max=0.72, sum=1.441 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.720291394813388\"}", - "Nutrition - # eval": "{\"description\": \"min=306, mean=306, max=306, sum=612 (2)\", \"tab\": \"General information\", \"score\": \"306.0\"}", - "Nutrition - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Nutrition - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Nutrition - # prompt tokens": "{\"description\": \"min=617.065, mean=617.065, max=617.065, sum=1234.131 (2)\", \"tab\": \"General information\", \"score\": \"617.0653594771242\"}", - "Nutrition - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"nutrition\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_nutrition\"" - } - } - }, - { - "evaluation_name": "Prehistory", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Prehistory", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.824, - "details": { - "description": "min=0.824, mean=0.824, max=0.824, sum=1.648 (2)", - "tab": "Accuracy", - "Prehistory - Observed inference time (s)": "{\"description\": \"min=0.713, mean=0.713, max=0.713, sum=1.427 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7133041966108629\"}", - "Prehistory - # eval": "{\"description\": \"min=324, mean=324, max=324, sum=648 (2)\", \"tab\": \"General information\", \"score\": \"324.0\"}", - "Prehistory - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Prehistory - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Prehistory - # prompt tokens": "{\"description\": \"min=545.639, mean=545.639, max=545.639, sum=1091.278 (2)\", \"tab\": \"General information\", \"score\": \"545.6388888888889\"}", - "Prehistory - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"prehistory\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_prehistory\"" - } - } - }, - { - "evaluation_name": "Public Relations", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Public Relations", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.755, - "details": { - "description": "min=0.755, mean=0.755, max=0.755, sum=1.509 (2)", - "tab": "Accuracy", - "Public Relations - Observed inference time (s)": "{\"description\": \"min=0.684, mean=0.684, max=0.684, sum=1.369 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6844336206262762\"}", - "Public Relations - # eval": "{\"description\": \"min=110, mean=110, max=110, sum=220 (2)\", \"tab\": \"General information\", \"score\": \"110.0\"}", - "Public Relations - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Public Relations - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Public Relations - # prompt tokens": "{\"description\": \"min=432.991, mean=432.991, max=432.991, sum=865.982 (2)\", \"tab\": \"General information\", \"score\": \"432.9909090909091\"}", - "Public Relations - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"public_relations\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_public_relations\"" - } - } - }, - { - "evaluation_name": "Security Studies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Security Studies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.808, - "details": { - "description": "min=0.808, mean=0.808, max=0.808, sum=1.616 (2)", - "tab": "Accuracy", - "Security Studies - Observed inference time (s)": "{\"description\": \"min=0.77, mean=0.77, max=0.77, sum=1.54 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7701463602027114\"}", - "Security Studies - # eval": "{\"description\": \"min=245, mean=245, max=245, sum=490 (2)\", \"tab\": \"General information\", \"score\": \"245.0\"}", - "Security Studies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Security Studies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Security Studies - # prompt tokens": "{\"description\": \"min=1243.804, mean=1243.804, max=1243.804, sum=2487.608 (2)\", \"tab\": \"General information\", \"score\": \"1243.8040816326532\"}", - "Security Studies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"security_studies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_security_studies\"" - } - } - }, - { - "evaluation_name": "Sociology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Sociology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9, - "details": { - "description": "min=0.9, mean=0.9, max=0.9, sum=1.801 (2)", - "tab": "Accuracy", - "Sociology - Observed inference time (s)": "{\"description\": \"min=0.69, mean=0.69, max=0.69, sum=1.38 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6899205867330827\"}", - "Sociology - # eval": "{\"description\": \"min=201, mean=201, max=201, sum=402 (2)\", \"tab\": \"General information\", \"score\": \"201.0\"}", - "Sociology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Sociology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Sociology - # prompt tokens": "{\"description\": \"min=467.274, mean=467.274, max=467.274, sum=934.547 (2)\", \"tab\": \"General information\", \"score\": \"467.27363184079604\"}", - "Sociology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"sociology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_sociology\"" - } - } - }, - { - "evaluation_name": "Virology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Virology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.542, - "details": { - "description": "min=0.542, mean=0.542, max=0.542, sum=1.084 (2)", - "tab": "Accuracy", - "Virology - Observed inference time (s)": "{\"description\": \"min=0.728, mean=0.728, max=0.728, sum=1.456 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7279246169400503\"}", - "Virology - # eval": "{\"description\": \"min=166, mean=166, max=166, sum=332 (2)\", \"tab\": \"General information\", \"score\": \"166.0\"}", - "Virology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Virology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Virology - # prompt tokens": "{\"description\": \"min=362.651, mean=362.651, max=362.651, sum=725.301 (2)\", \"tab\": \"General information\", \"score\": \"362.65060240963857\"}", - "Virology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"virology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_virology\"" - } - } - }, - { - "evaluation_name": "World Religions", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on World Religions", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.871, - "details": { - "description": "min=0.871, mean=0.871, max=0.871, sum=1.743 (2)", - "tab": "Accuracy", - "World Religions - Observed inference time (s)": "{\"description\": \"min=0.727, mean=0.727, max=0.727, sum=1.454 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7269549021246837\"}", - "World Religions - # eval": "{\"description\": \"min=171, mean=171, max=171, sum=342 (2)\", \"tab\": \"General information\", \"score\": \"171.0\"}", - "World Religions - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "World Religions - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "World Religions - # prompt tokens": "{\"description\": \"min=293.018, mean=293.018, max=293.018, sum=586.035 (2)\", \"tab\": \"General information\", \"score\": \"293.0175438596491\"}", - "World Religions - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"world_religions\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_world_religions\"" - } - } - }, - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.28, - "details": { - "description": "", - "tab": "Efficiency" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_id": "reward-bench/Anthropic_claude-3-haiku-20240307/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7289 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9274 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5197 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7953 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.706 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6635 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "reward-bench-2/anthropic_claude-3-haiku-20240307/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3711 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4042 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2812 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3552 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.595 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.501 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0899 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/anthropic_claude-3-opus-20240229.json b/data/models/anthropic_claude-3-opus-20240229.json deleted file mode 100644 index 8f8b70dd804fcbb225ceb8111209d93e20e8bd0b..0000000000000000000000000000000000000000 --- a/data/models/anthropic_claude-3-opus-20240229.json +++ /dev/null @@ -1,2170 +0,0 @@ -{ - "model_info": { - "name": "Claude 3 Opus 20240229", - "id": "anthropic/claude-3-opus-20240229", - "developer": "Anthropic", - "inference_platform": "unknown", - "additional_details": { - "model_id_aliases": [ - "Anthropic/claude-3-opus-20240229" - ] - } - }, - "evaluations": [ - { - "evaluation_id": "helm_lite/anthropic_claude-3-opus-20240229/1774096306.427425", - "retrieved_timestamp": "1774096306.427425", - "source_metadata": { - "source_name": "helm_lite", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_lite", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_lite", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.683, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.09124843945068664\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.351, - "details": { - "description": "min=0.351, mean=0.351, max=0.351, sum=0.351 (1)", - "tab": "Accuracy", - "NarrativeQA - Observed inference time (s)": "{\"description\": \"min=3.996, mean=3.996, max=3.996, sum=3.996 (1)\", \"tab\": \"Efficiency\", \"score\": \"3.9963467248728577\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=3709.741, mean=3709.741, max=3709.741, sum=3709.741 (1)\", \"tab\": \"General information\", \"score\": \"3709.7408450704224\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=13.589, mean=13.589, max=13.589, sum=13.589 (1)\", \"tab\": \"General information\", \"score\": \"13.588732394366197\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (closed-book)", - "source_data": { - "dataset_name": "NaturalQuestions (closed-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (closed-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.441, - "details": { - "description": "min=0.441, mean=0.441, max=0.441, sum=0.441 (1)", - "tab": "Accuracy", - "NaturalQuestions (open-book) - Observed inference time (s)": "{\"description\": \"min=4.273, mean=4.273, max=4.273, sum=4.273 (1)\", \"tab\": \"Efficiency\", \"score\": \"4.273005393266678\"}", - "NaturalQuestions (closed-book) - Observed inference time (s)": "{\"description\": \"min=1.647, mean=1.647, max=1.647, sum=1.647 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.6471402559280395\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1781.799, mean=1781.799, max=1781.799, sum=1781.799 (1)\", \"tab\": \"General information\", \"score\": \"1781.799\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=39.248, mean=39.248, max=39.248, sum=39.248 (1)\", \"tab\": \"General information\", \"score\": \"39.248\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=189.259, mean=189.259, max=189.259, sum=189.259 (1)\", \"tab\": \"General information\", \"score\": \"189.259\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=5.66, mean=5.66, max=5.66, sum=5.66 (1)\", \"tab\": \"General information\", \"score\": \"5.66\"}" - } - }, - "generation_config": { - "additional_details": { - "mode": "\"closedbook\"" - } - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.956, - "details": { - "description": "min=0.956, mean=0.956, max=0.956, sum=0.956 (1)", - "tab": "Accuracy", - "OpenbookQA - Observed inference time (s)": "{\"description\": \"min=2.168, mean=2.168, max=2.168, sum=2.168 (1)\", \"tab\": \"Efficiency\", \"score\": \"2.167769320487976\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=263.79, mean=263.79, max=263.79, sum=263.79 (1)\", \"tab\": \"General information\", \"score\": \"263.79\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "dataset": "\"openbookqa\"", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.768, - "details": { - "description": "min=0.6, mean=0.768, max=0.96, sum=3.839 (5)", - "tab": "Accuracy", - "MMLU - Observed inference time (s)": "{\"description\": \"min=4.003, mean=4.19, max=4.373, sum=20.948 (5)\", \"tab\": \"Efficiency\", \"score\": \"4.189554240862528\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=370.26, mean=478.747, max=619.596, sum=2393.736 (5)\", \"tab\": \"General information\", \"score\": \"478.747298245614\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"college_chemistry\", \"computer_security\", \"econometrics\", \"us_foreign_policy\"]", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MATH", - "source_data": { - "dataset_name": "MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Equivalent (CoT) on MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.76, - "details": { - "description": "min=0.526, mean=0.76, max=0.889, sum=5.322 (7)", - "tab": "Accuracy", - "MATH - Observed inference time (s)": "{\"description\": \"min=6.095, mean=7.542, max=9.041, sum=52.793 (7)\", \"tab\": \"Efficiency\", \"score\": \"7.541890628266922\"}", - "MATH - # eval": "{\"description\": \"min=30, mean=62.429, max=135, sum=437 (7)\", \"tab\": \"General information\", \"score\": \"62.42857142857143\"}", - "MATH - # train": "{\"description\": \"min=8, mean=8, max=8, sum=56 (7)\", \"tab\": \"General information\", \"score\": \"8.0\"}", - "MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MATH - # prompt tokens": "{\"description\": \"min=948.259, mean=1362.814, max=2380.808, sum=9539.699 (7)\", \"tab\": \"General information\", \"score\": \"1362.8141219676104\"}", - "MATH - # output tokens": "{\"description\": \"min=82.965, mean=113.906, max=138.263, sum=797.345 (7)\", \"tab\": \"General information\", \"score\": \"113.90635737624721\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"algebra\", \"counting_and_probability\", \"geometry\", \"intermediate_algebra\", \"number_theory\", \"prealgebra\", \"precalculus\"]", - "level": "\"1\"", - "use_official_examples": "\"False\"", - "use_chain_of_thought": "\"True\"" - } - } - }, - { - "evaluation_name": "GSM8K", - "source_data": { - "dataset_name": "GSM8K", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on GSM8K", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.924, - "details": { - "description": "min=0.924, mean=0.924, max=0.924, sum=0.924 (1)", - "tab": "Accuracy", - "GSM8K - Observed inference time (s)": "{\"description\": \"min=7.469, mean=7.469, max=7.469, sum=7.469 (1)\", \"tab\": \"Efficiency\", \"score\": \"7.469249876976013\"}", - "GSM8K - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "GSM8K - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "GSM8K - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GSM8K - # prompt tokens": "{\"description\": \"min=1012.712, mean=1012.712, max=1012.712, sum=1012.712 (1)\", \"tab\": \"General information\", \"score\": \"1012.712\"}", - "GSM8K - # output tokens": "{\"description\": \"min=115.934, mean=115.934, max=115.934, sum=115.934 (1)\", \"tab\": \"General information\", \"score\": \"115.934\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "LegalBench", - "source_data": { - "dataset_name": "LegalBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on LegalBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.662, - "details": { - "description": "min=0.153, mean=0.662, max=0.989, sum=3.31 (5)", - "tab": "Accuracy", - "LegalBench - Observed inference time (s)": "{\"description\": \"min=1.391, mean=2.57, max=4.856, sum=12.851 (5)\", \"tab\": \"Efficiency\", \"score\": \"2.570133829482505\"}", - "LegalBench - # eval": "{\"description\": \"min=95, mean=409.4, max=1000, sum=2047 (5)\", \"tab\": \"General information\", \"score\": \"409.4\"}", - "LegalBench - # train": "{\"description\": \"min=4, mean=4.8, max=5, sum=24 (5)\", \"tab\": \"General information\", \"score\": \"4.8\"}", - "LegalBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "LegalBench - # prompt tokens": "{\"description\": \"min=214.653, mean=1557.242, max=6428.398, sum=7786.208 (5)\", \"tab\": \"General information\", \"score\": \"1557.241581367783\"}", - "LegalBench - # output tokens": "{\"description\": \"min=1, mean=1.605, max=2.932, sum=8.023 (5)\", \"tab\": \"General information\", \"score\": \"1.6045285459659269\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"abercrombie\", \"corporate_lobbying\", \"function_of_decision_section\", \"international_citizenship_questions\", \"proa\"]" - } - } - }, - { - "evaluation_name": "MedQA", - "source_data": { - "dataset_name": "MedQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MedQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.775, - "details": { - "description": "min=0.775, mean=0.775, max=0.775, sum=0.775 (1)", - "tab": "Accuracy", - "MedQA - Observed inference time (s)": "{\"description\": \"min=2.65, mean=2.65, max=2.65, sum=2.65 (1)\", \"tab\": \"Efficiency\", \"score\": \"2.6499544673601156\"}", - "MedQA - # eval": "{\"description\": \"min=503, mean=503, max=503, sum=503 (1)\", \"tab\": \"General information\", \"score\": \"503.0\"}", - "MedQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MedQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MedQA - # prompt tokens": "{\"description\": \"min=1027.437, mean=1027.437, max=1027.437, sum=1027.437 (1)\", \"tab\": \"General information\", \"score\": \"1027.4373757455269\"}", - "MedQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WMT 2014", - "source_data": { - "dataset_name": "WMT 2014", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "BLEU-4 on WMT 2014", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.24, - "details": { - "description": "min=0.188, mean=0.24, max=0.285, sum=1.199 (5)", - "tab": "Accuracy", - "WMT 2014 - Observed inference time (s)": "{\"description\": \"min=2.279, mean=2.447, max=2.661, sum=12.233 (5)\", \"tab\": \"Efficiency\", \"score\": \"2.4465377724275283\"}", - "WMT 2014 - # eval": "{\"description\": \"min=503, mean=568.8, max=832, sum=2844 (5)\", \"tab\": \"General information\", \"score\": \"568.8\"}", - "WMT 2014 - # train": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "WMT 2014 - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WMT 2014 - # prompt tokens": "{\"description\": \"min=198.406, mean=219.573, max=241.974, sum=1097.866 (5)\", \"tab\": \"General information\", \"score\": \"219.57322077152472\"}", - "WMT 2014 - # output tokens": "{\"description\": \"min=24.332, mean=25.837, max=26.616, sum=129.185 (5)\", \"tab\": \"General information\", \"score\": \"25.837047426976607\"}" - } - }, - "generation_config": { - "additional_details": { - "language_pair": "[\"cs-en\", \"de-en\", \"fr-en\", \"hi-en\", \"ru-en\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_mmlu/anthropic_claude-3-opus-20240229/1774096312.00548", - "retrieved_timestamp": "1774096312.00548", - "source_metadata": { - "source_name": "helm_mmlu", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_mmlu", - "evaluation_results": [ - { - "evaluation_name": "MMLU All Subjects", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU All Subjects", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.846, - "details": { - "description": "min=0.55, mean=0.846, max=0.979, sum=96.412 (114)", - "tab": "Accuracy", - "MMLU All Subjects - Observed inference time (s)": "{\"description\": \"min=3.782, mean=4.077, max=5.005, sum=464.781 (114)\", \"tab\": \"Efficiency\", \"score\": \"4.077024270463863\"}", - "MMLU All Subjects - # eval": "{\"description\": \"min=100, mean=246.351, max=1534, sum=28084 (114)\", \"tab\": \"General information\", \"score\": \"246.35087719298247\"}", - "MMLU All Subjects - # train": "{\"description\": \"min=5, mean=5, max=5, sum=570 (114)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU All Subjects - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU All Subjects - # prompt tokens": "{\"description\": \"min=293.018, mean=638.288, max=2887.576, sum=72764.875 (114)\", \"tab\": \"General information\", \"score\": \"638.2883793758953\"}", - "MMLU All Subjects - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=114 (114)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_name": "Abstract Algebra", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Abstract Algebra", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.64, - "details": { - "description": "min=0.64, mean=0.64, max=0.64, sum=1.28 (2)", - "tab": "Accuracy", - "Abstract Algebra - Observed inference time (s)": "{\"description\": \"min=4.182, mean=4.182, max=4.182, sum=8.364 (2)\", \"tab\": \"Efficiency\", \"score\": \"4.182226595878601\"}", - "Abstract Algebra - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Abstract Algebra - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Abstract Algebra - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Abstract Algebra - # prompt tokens": "{\"description\": \"min=370.26, mean=370.26, max=370.26, sum=740.52 (2)\", \"tab\": \"General information\", \"score\": \"370.26\"}", - "Abstract Algebra - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"abstract_algebra\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_abstract_algebra\"" - } - } - }, - { - "evaluation_name": "Anatomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Anatomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8, - "details": { - "description": "min=0.8, mean=0.8, max=0.8, sum=1.6 (2)", - "tab": "Accuracy", - "Anatomy - Observed inference time (s)": "{\"description\": \"min=4.115, mean=4.115, max=4.115, sum=8.23 (2)\", \"tab\": \"Efficiency\", \"score\": \"4.114818896187677\"}", - "Anatomy - # eval": "{\"description\": \"min=135, mean=135, max=135, sum=270 (2)\", \"tab\": \"General information\", \"score\": \"135.0\"}", - "Anatomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Anatomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Anatomy - # prompt tokens": "{\"description\": \"min=370.8, mean=370.8, max=370.8, sum=741.6 (2)\", \"tab\": \"General information\", \"score\": \"370.8\"}", - "Anatomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"anatomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_anatomy\"" - } - } - }, - { - "evaluation_name": "College Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on College Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.716, - "details": { - "description": "min=0.716, mean=0.716, max=0.716, sum=1.431 (2)", - "tab": "Accuracy", - "College Chemistry - Observed inference time (s)": "{\"description\": \"min=4.373, mean=4.373, max=4.373, sum=8.745 (2)\", \"tab\": \"Efficiency\", \"score\": \"4.372743592262268\"}", - "College Biology - Observed inference time (s)": "{\"description\": \"min=4.045, mean=4.045, max=4.045, sum=8.09 (2)\", \"tab\": \"Efficiency\", \"score\": \"4.044814482331276\"}", - "College Computer Science - Observed inference time (s)": "{\"description\": \"min=4.326, mean=4.326, max=4.326, sum=8.652 (2)\", \"tab\": \"Efficiency\", \"score\": \"4.3260163617134095\"}", - "College Mathematics - Observed inference time (s)": "{\"description\": \"min=4.209, mean=4.209, max=4.209, sum=8.417 (2)\", \"tab\": \"Efficiency\", \"score\": \"4.208740277290344\"}", - "College Medicine - Observed inference time (s)": "{\"description\": \"min=3.994, mean=3.994, max=3.994, sum=7.988 (2)\", \"tab\": \"Efficiency\", \"score\": \"3.9939607113082976\"}", - "College Physics - Observed inference time (s)": "{\"description\": \"min=3.982, mean=3.982, max=3.982, sum=7.965 (2)\", \"tab\": \"Efficiency\", \"score\": \"3.9823715172561944\"}", - "College Chemistry - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Chemistry - # prompt tokens": "{\"description\": \"min=550.01, mean=550.01, max=550.01, sum=1100.02 (2)\", \"tab\": \"General information\", \"score\": \"550.01\"}", - "College Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Biology - # eval": "{\"description\": \"min=144, mean=144, max=144, sum=288 (2)\", \"tab\": \"General information\", \"score\": \"144.0\"}", - "College Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # prompt tokens": "{\"description\": \"min=490.347, mean=490.347, max=490.347, sum=980.694 (2)\", \"tab\": \"General information\", \"score\": \"490.34722222222223\"}", - "College Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # prompt tokens": "{\"description\": \"min=838.24, mean=838.24, max=838.24, sum=1676.48 (2)\", \"tab\": \"General information\", \"score\": \"838.24\"}", - "College Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Mathematics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # prompt tokens": "{\"description\": \"min=604.19, mean=604.19, max=604.19, sum=1208.38 (2)\", \"tab\": \"General information\", \"score\": \"604.19\"}", - "College Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Medicine - # eval": "{\"description\": \"min=173, mean=173, max=173, sum=346 (2)\", \"tab\": \"General information\", \"score\": \"173.0\"}", - "College Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # prompt tokens": "{\"description\": \"min=540.63, mean=540.63, max=540.63, sum=1081.26 (2)\", \"tab\": \"General information\", \"score\": \"540.6300578034682\"}", - "College Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Physics - # eval": "{\"description\": \"min=102, mean=102, max=102, sum=204 (2)\", \"tab\": \"General information\", \"score\": \"102.0\"}", - "College Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # prompt tokens": "{\"description\": \"min=489.48, mean=489.48, max=489.48, sum=978.961 (2)\", \"tab\": \"General information\", \"score\": \"489.48039215686276\"}", - "College Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"college_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_college_physics\"" - } - } - }, - { - "evaluation_name": "Computer Security", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Computer Security", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.85, - "details": { - "description": "min=0.85, mean=0.85, max=0.85, sum=1.7 (2)", - "tab": "Accuracy", - "Computer Security - Observed inference time (s)": "{\"description\": \"min=4.105, mean=4.105, max=4.105, sum=8.211 (2)\", \"tab\": \"Efficiency\", \"score\": \"4.105417683124542\"}", - "Computer Security - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Computer Security - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Computer Security - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Computer Security - # prompt tokens": "{\"description\": \"min=398.62, mean=398.62, max=398.62, sum=797.24 (2)\", \"tab\": \"General information\", \"score\": \"398.62\"}", - "Computer Security - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"computer_security\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_computer_security\"" - } - } - }, - { - "evaluation_name": "Econometrics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Econometrics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.789, - "details": { - "description": "min=0.789, mean=0.789, max=0.789, sum=1.579 (2)", - "tab": "Accuracy", - "Econometrics - Observed inference time (s)": "{\"description\": \"min=4.284, mean=4.284, max=4.284, sum=8.569 (2)\", \"tab\": \"Efficiency\", \"score\": \"4.284419020016988\"}", - "Econometrics - # eval": "{\"description\": \"min=114, mean=114, max=114, sum=228 (2)\", \"tab\": \"General information\", \"score\": \"114.0\"}", - "Econometrics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Econometrics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Econometrics - # prompt tokens": "{\"description\": \"min=619.596, mean=619.596, max=619.596, sum=1239.193 (2)\", \"tab\": \"General information\", \"score\": \"619.5964912280701\"}", - "Econometrics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"econometrics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_econometrics\"" - } - } - }, - { - "evaluation_name": "Global Facts", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Global Facts", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.66, - "details": { - "description": "min=0.66, mean=0.66, max=0.66, sum=1.32 (2)", - "tab": "Accuracy", - "Global Facts - Observed inference time (s)": "{\"description\": \"min=4.232, mean=4.232, max=4.232, sum=8.465 (2)\", \"tab\": \"Efficiency\", \"score\": \"4.232321140766143\"}", - "Global Facts - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Global Facts - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Global Facts - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Global Facts - # prompt tokens": "{\"description\": \"min=411.61, mean=411.61, max=411.61, sum=823.22 (2)\", \"tab\": \"General information\", \"score\": \"411.61\"}", - "Global Facts - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"global_facts\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_global_facts\"" - } - } - }, - { - "evaluation_name": "Jurisprudence", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Jurisprudence", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.88, - "details": { - "description": "min=0.88, mean=0.88, max=0.88, sum=1.759 (2)", - "tab": "Accuracy", - "Jurisprudence - Observed inference time (s)": "{\"description\": \"min=3.872, mean=3.872, max=3.872, sum=7.744 (2)\", \"tab\": \"Efficiency\", \"score\": \"3.8720074185618647\"}", - "Jurisprudence - # eval": "{\"description\": \"min=108, mean=108, max=108, sum=216 (2)\", \"tab\": \"General information\", \"score\": \"108.0\"}", - "Jurisprudence - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Jurisprudence - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Jurisprudence - # prompt tokens": "{\"description\": \"min=431.426, mean=431.426, max=431.426, sum=862.852 (2)\", \"tab\": \"General information\", \"score\": \"431.4259259259259\"}", - "Jurisprudence - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"jurisprudence\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_jurisprudence\"" - } - } - }, - { - "evaluation_name": "Philosophy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Philosophy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9, - "details": { - "description": "min=0.9, mean=0.9, max=0.9, sum=1.801 (2)", - "tab": "Accuracy", - "Philosophy - Observed inference time (s)": "{\"description\": \"min=3.967, mean=3.967, max=3.967, sum=7.935 (2)\", \"tab\": \"Efficiency\", \"score\": \"3.9672668930801933\"}", - "Philosophy - # eval": "{\"description\": \"min=311, mean=311, max=311, sum=622 (2)\", \"tab\": \"General information\", \"score\": \"311.0\"}", - "Philosophy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Philosophy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Philosophy - # prompt tokens": "{\"description\": \"min=359.965, mean=359.965, max=359.965, sum=719.929 (2)\", \"tab\": \"General information\", \"score\": \"359.9646302250804\"}", - "Philosophy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"philosophy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_philosophy\"" - } - } - }, - { - "evaluation_name": "Professional Psychology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Professional Psychology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.904, - "details": { - "description": "min=0.904, mean=0.904, max=0.904, sum=1.807 (2)", - "tab": "Accuracy", - "Professional Medicine - Observed inference time (s)": "{\"description\": \"min=4.358, mean=4.358, max=4.358, sum=8.715 (2)\", \"tab\": \"Efficiency\", \"score\": \"4.357662654974881\"}", - "Professional Accounting - Observed inference time (s)": "{\"description\": \"min=3.982, mean=3.982, max=3.982, sum=7.965 (2)\", \"tab\": \"Efficiency\", \"score\": \"3.9823869661236486\"}", - "Professional Law - Observed inference time (s)": "{\"description\": \"min=4.483, mean=4.483, max=4.483, sum=8.967 (2)\", \"tab\": \"Efficiency\", \"score\": \"4.483374906953963\"}", - "Professional Psychology - Observed inference time (s)": "{\"description\": \"min=4.006, mean=4.006, max=4.006, sum=8.012 (2)\", \"tab\": \"Efficiency\", \"score\": \"4.0058385706415365\"}", - "Professional Medicine - # eval": "{\"description\": \"min=272, mean=272, max=272, sum=544 (2)\", \"tab\": \"General information\", \"score\": \"272.0\"}", - "Professional Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Medicine - # prompt tokens": "{\"description\": \"min=1123.537, mean=1123.537, max=1123.537, sum=2247.074 (2)\", \"tab\": \"General information\", \"score\": \"1123.5367647058824\"}", - "Professional Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Accounting - # eval": "{\"description\": \"min=282, mean=282, max=282, sum=564 (2)\", \"tab\": \"General information\", \"score\": \"282.0\"}", - "Professional Accounting - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Accounting - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # prompt tokens": "{\"description\": \"min=665.422, mean=665.422, max=665.422, sum=1330.844 (2)\", \"tab\": \"General information\", \"score\": \"665.4219858156029\"}", - "Professional Accounting - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Law - # eval": "{\"description\": \"min=1534, mean=1534, max=1534, sum=3068 (2)\", \"tab\": \"General information\", \"score\": \"1534.0\"}", - "Professional Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # prompt tokens": "{\"description\": \"min=1701.16, mean=1701.16, max=1701.16, sum=3402.321 (2)\", \"tab\": \"General information\", \"score\": \"1701.16036505867\"}", - "Professional Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Psychology - # eval": "{\"description\": \"min=612, mean=612, max=612, sum=1224 (2)\", \"tab\": \"General information\", \"score\": \"612.0\"}", - "Professional Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # prompt tokens": "{\"description\": \"min=603.168, mean=603.168, max=603.168, sum=1206.337 (2)\", \"tab\": \"General information\", \"score\": \"603.1683006535948\"}", - "Professional Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"professional_psychology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_professional_psychology\"" - } - } - }, - { - "evaluation_name": "Us Foreign Policy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Us Foreign Policy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.96, - "details": { - "description": "min=0.96, mean=0.96, max=0.96, sum=1.92 (2)", - "tab": "Accuracy", - "Us Foreign Policy - Observed inference time (s)": "{\"description\": \"min=4.003, mean=4.003, max=4.003, sum=8.006 (2)\", \"tab\": \"Efficiency\", \"score\": \"4.002964313030243\"}", - "Us Foreign Policy - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Us Foreign Policy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Us Foreign Policy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Us Foreign Policy - # prompt tokens": "{\"description\": \"min=455.25, mean=455.25, max=455.25, sum=910.5 (2)\", \"tab\": \"General information\", \"score\": \"455.25\"}", - "Us Foreign Policy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"us_foreign_policy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_us_foreign_policy\"" - } - } - }, - { - "evaluation_name": "Astronomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Astronomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.967, - "details": { - "description": "min=0.967, mean=0.967, max=0.967, sum=1.934 (2)", - "tab": "Accuracy", - "Astronomy - Observed inference time (s)": "{\"description\": \"min=4.099, mean=4.099, max=4.099, sum=8.198 (2)\", \"tab\": \"Efficiency\", \"score\": \"4.099087294779326\"}", - "Astronomy - # eval": "{\"description\": \"min=152, mean=152, max=152, sum=304 (2)\", \"tab\": \"General information\", \"score\": \"152.0\"}", - "Astronomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Astronomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Astronomy - # prompt tokens": "{\"description\": \"min=604.493, mean=604.493, max=604.493, sum=1208.987 (2)\", \"tab\": \"General information\", \"score\": \"604.4934210526316\"}", - "Astronomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"astronomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_astronomy\"" - } - } - }, - { - "evaluation_name": "Business Ethics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Business Ethics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.86, - "details": { - "description": "min=0.86, mean=0.86, max=0.86, sum=1.72 (2)", - "tab": "Accuracy", - "Business Ethics - Observed inference time (s)": "{\"description\": \"min=4.102, mean=4.102, max=4.102, sum=8.204 (2)\", \"tab\": \"Efficiency\", \"score\": \"4.102163214683532\"}", - "Business Ethics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Business Ethics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Business Ethics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Business Ethics - # prompt tokens": "{\"description\": \"min=600.02, mean=600.02, max=600.02, sum=1200.04 (2)\", \"tab\": \"General information\", \"score\": \"600.02\"}", - "Business Ethics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"business_ethics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_business_ethics\"" - } - } - }, - { - "evaluation_name": "Clinical Knowledge", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Clinical Knowledge", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.879, - "details": { - "description": "min=0.879, mean=0.879, max=0.879, sum=1.758 (2)", - "tab": "Accuracy", - "Clinical Knowledge - Observed inference time (s)": "{\"description\": \"min=3.976, mean=3.976, max=3.976, sum=7.952 (2)\", \"tab\": \"Efficiency\", \"score\": \"3.9762323631430574\"}", - "Clinical Knowledge - # eval": "{\"description\": \"min=265, mean=265, max=265, sum=530 (2)\", \"tab\": \"General information\", \"score\": \"265.0\"}", - "Clinical Knowledge - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Clinical Knowledge - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Clinical Knowledge - # prompt tokens": "{\"description\": \"min=429.457, mean=429.457, max=429.457, sum=858.913 (2)\", \"tab\": \"General information\", \"score\": \"429.4566037735849\"}", - "Clinical Knowledge - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"clinical_knowledge\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_clinical_knowledge\"" - } - } - }, - { - "evaluation_name": "Conceptual Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Conceptual Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.881, - "details": { - "description": "min=0.881, mean=0.881, max=0.881, sum=1.762 (2)", - "tab": "Accuracy", - "Conceptual Physics - Observed inference time (s)": "{\"description\": \"min=3.959, mean=3.959, max=3.959, sum=7.918 (2)\", \"tab\": \"Efficiency\", \"score\": \"3.9589331109473047\"}", - "Conceptual Physics - # eval": "{\"description\": \"min=235, mean=235, max=235, sum=470 (2)\", \"tab\": \"General information\", \"score\": \"235.0\"}", - "Conceptual Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Conceptual Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Conceptual Physics - # prompt tokens": "{\"description\": \"min=323.536, mean=323.536, max=323.536, sum=647.072 (2)\", \"tab\": \"General information\", \"score\": \"323.53617021276597\"}", - "Conceptual Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"conceptual_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_conceptual_physics\"" - } - } - }, - { - "evaluation_name": "Electrical Engineering", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Electrical Engineering", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.814, - "details": { - "description": "min=0.814, mean=0.814, max=0.814, sum=1.628 (2)", - "tab": "Accuracy", - "Electrical Engineering - Observed inference time (s)": "{\"description\": \"min=4.017, mean=4.017, max=4.017, sum=8.035 (2)\", \"tab\": \"Efficiency\", \"score\": \"4.017465997564382\"}", - "Electrical Engineering - # eval": "{\"description\": \"min=145, mean=145, max=145, sum=290 (2)\", \"tab\": \"General information\", \"score\": \"145.0\"}", - "Electrical Engineering - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Electrical Engineering - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Electrical Engineering - # prompt tokens": "{\"description\": \"min=437.041, mean=437.041, max=437.041, sum=874.083 (2)\", \"tab\": \"General information\", \"score\": \"437.04137931034484\"}", - "Electrical Engineering - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"electrical_engineering\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_electrical_engineering\"" - } - } - }, - { - "evaluation_name": "Elementary Mathematics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Elementary Mathematics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.862, - "details": { - "description": "min=0.862, mean=0.862, max=0.862, sum=1.725 (2)", - "tab": "Accuracy", - "Elementary Mathematics - Observed inference time (s)": "{\"description\": \"min=3.937, mean=3.937, max=3.937, sum=7.874 (2)\", \"tab\": \"Efficiency\", \"score\": \"3.937073076212848\"}", - "Elementary Mathematics - # eval": "{\"description\": \"min=378, mean=378, max=378, sum=756 (2)\", \"tab\": \"General information\", \"score\": \"378.0\"}", - "Elementary Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Elementary Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Elementary Mathematics - # prompt tokens": "{\"description\": \"min=511.066, mean=511.066, max=511.066, sum=1022.132 (2)\", \"tab\": \"General information\", \"score\": \"511.06613756613757\"}", - "Elementary Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"elementary_mathematics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_elementary_mathematics\"" - } - } - }, - { - "evaluation_name": "Formal Logic", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Formal Logic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.698, - "details": { - "description": "min=0.698, mean=0.698, max=0.698, sum=1.397 (2)", - "tab": "Accuracy", - "Formal Logic - Observed inference time (s)": "{\"description\": \"min=4.178, mean=4.178, max=4.178, sum=8.356 (2)\", \"tab\": \"Efficiency\", \"score\": \"4.177885971372089\"}", - "Formal Logic - # eval": "{\"description\": \"min=126, mean=126, max=126, sum=252 (2)\", \"tab\": \"General information\", \"score\": \"126.0\"}", - "Formal Logic - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Formal Logic - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Formal Logic - # prompt tokens": "{\"description\": \"min=646.746, mean=646.746, max=646.746, sum=1293.492 (2)\", \"tab\": \"General information\", \"score\": \"646.7460317460317\"}", - "Formal Logic - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"formal_logic\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_formal_logic\"" - } - } - }, - { - "evaluation_name": "High School World History", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on High School World History", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.941, - "details": { - "description": "min=0.941, mean=0.941, max=0.941, sum=1.882 (2)", - "tab": "Accuracy", - "High School Biology - Observed inference time (s)": "{\"description\": \"min=4.184, mean=4.184, max=4.184, sum=8.368 (2)\", \"tab\": \"Efficiency\", \"score\": \"4.183918527633913\"}", - "High School Chemistry - Observed inference time (s)": "{\"description\": \"min=4.027, mean=4.027, max=4.027, sum=8.055 (2)\", \"tab\": \"Efficiency\", \"score\": \"4.027491113822449\"}", - "High School Computer Science - Observed inference time (s)": "{\"description\": \"min=3.929, mean=3.929, max=3.929, sum=7.858 (2)\", \"tab\": \"Efficiency\", \"score\": \"3.929041051864624\"}", - "High School European History - Observed inference time (s)": "{\"description\": \"min=5.005, mean=5.005, max=5.005, sum=10.009 (2)\", \"tab\": \"Efficiency\", \"score\": \"5.004520618554317\"}", - "High School Geography - Observed inference time (s)": "{\"description\": \"min=3.872, mean=3.872, max=3.872, sum=7.743 (2)\", \"tab\": \"Efficiency\", \"score\": \"3.87151506332436\"}", - "High School Government And Politics - Observed inference time (s)": "{\"description\": \"min=3.936, mean=3.936, max=3.936, sum=7.872 (2)\", \"tab\": \"Efficiency\", \"score\": \"3.936160638542373\"}", - "High School Macroeconomics - Observed inference time (s)": "{\"description\": \"min=3.782, mean=3.782, max=3.782, sum=7.563 (2)\", \"tab\": \"Efficiency\", \"score\": \"3.781650854379703\"}", - "High School Mathematics - Observed inference time (s)": "{\"description\": \"min=4.061, mean=4.061, max=4.061, sum=8.122 (2)\", \"tab\": \"Efficiency\", \"score\": \"4.0608021259307865\"}", - "High School Microeconomics - Observed inference time (s)": "{\"description\": \"min=3.861, mean=3.861, max=3.861, sum=7.722 (2)\", \"tab\": \"Efficiency\", \"score\": \"3.860906556874764\"}", - "High School Physics - Observed inference time (s)": "{\"description\": \"min=3.938, mean=3.938, max=3.938, sum=7.876 (2)\", \"tab\": \"Efficiency\", \"score\": \"3.9381139499462203\"}", - "High School Psychology - Observed inference time (s)": "{\"description\": \"min=4.059, mean=4.059, max=4.059, sum=8.118 (2)\", \"tab\": \"Efficiency\", \"score\": \"4.058962697282843\"}", - "High School Statistics - Observed inference time (s)": "{\"description\": \"min=4.024, mean=4.024, max=4.024, sum=8.047 (2)\", \"tab\": \"Efficiency\", \"score\": \"4.023671524392234\"}", - "High School US History - Observed inference time (s)": "{\"description\": \"min=4.606, mean=4.606, max=4.606, sum=9.213 (2)\", \"tab\": \"Efficiency\", \"score\": \"4.606354508914199\"}", - "High School World History - Observed inference time (s)": "{\"description\": \"min=4.336, mean=4.336, max=4.336, sum=8.672 (2)\", \"tab\": \"Efficiency\", \"score\": \"4.335798429537423\"}", - "High School Biology - # eval": "{\"description\": \"min=310, mean=310, max=310, sum=620 (2)\", \"tab\": \"General information\", \"score\": \"310.0\"}", - "High School Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Biology - # prompt tokens": "{\"description\": \"min=534.577, mean=534.577, max=534.577, sum=1069.155 (2)\", \"tab\": \"General information\", \"score\": \"534.5774193548388\"}", - "High School Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Chemistry - # eval": "{\"description\": \"min=203, mean=203, max=203, sum=406 (2)\", \"tab\": \"General information\", \"score\": \"203.0\"}", - "High School Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # prompt tokens": "{\"description\": \"min=497.921, mean=497.921, max=497.921, sum=995.842 (2)\", \"tab\": \"General information\", \"score\": \"497.92118226600985\"}", - "High School Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "High School Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # prompt tokens": "{\"description\": \"min=882.4, mean=882.4, max=882.4, sum=1764.8 (2)\", \"tab\": \"General information\", \"score\": \"882.4\"}", - "High School Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School European History - # eval": "{\"description\": \"min=165, mean=165, max=165, sum=330 (2)\", \"tab\": \"General information\", \"score\": \"165.0\"}", - "High School European History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School European History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # prompt tokens": "{\"description\": \"min=2887.576, mean=2887.576, max=2887.576, sum=5775.152 (2)\", \"tab\": \"General information\", \"score\": \"2887.5757575757575\"}", - "High School European History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Geography - # eval": "{\"description\": \"min=198, mean=198, max=198, sum=396 (2)\", \"tab\": \"General information\", \"score\": \"198.0\"}", - "High School Geography - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Geography - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # prompt tokens": "{\"description\": \"min=412.268, mean=412.268, max=412.268, sum=824.535 (2)\", \"tab\": \"General information\", \"score\": \"412.2676767676768\"}", - "High School Geography - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Government And Politics - # eval": "{\"description\": \"min=193, mean=193, max=193, sum=386 (2)\", \"tab\": \"General information\", \"score\": \"193.0\"}", - "High School Government And Politics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Government And Politics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # prompt tokens": "{\"description\": \"min=491.104, mean=491.104, max=491.104, sum=982.207 (2)\", \"tab\": \"General information\", \"score\": \"491.10362694300517\"}", - "High School Government And Politics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Macroeconomics - # eval": "{\"description\": \"min=390, mean=390, max=390, sum=780 (2)\", \"tab\": \"General information\", \"score\": \"390.0\"}", - "High School Macroeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Macroeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # prompt tokens": "{\"description\": \"min=406.036, mean=406.036, max=406.036, sum=812.072 (2)\", \"tab\": \"General information\", \"score\": \"406.0358974358974\"}", - "High School Macroeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Mathematics - # eval": "{\"description\": \"min=270, mean=270, max=270, sum=540 (2)\", \"tab\": \"General information\", \"score\": \"270.0\"}", - "High School Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # prompt tokens": "{\"description\": \"min=519.881, mean=519.881, max=519.881, sum=1039.763 (2)\", \"tab\": \"General information\", \"score\": \"519.8814814814815\"}", - "High School Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Microeconomics - # eval": "{\"description\": \"min=238, mean=238, max=238, sum=476 (2)\", \"tab\": \"General information\", \"score\": \"238.0\"}", - "High School Microeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Microeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # prompt tokens": "{\"description\": \"min=420.513, mean=420.513, max=420.513, sum=841.025 (2)\", \"tab\": \"General information\", \"score\": \"420.5126050420168\"}", - "High School Microeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Physics - # eval": "{\"description\": \"min=151, mean=151, max=151, sum=302 (2)\", \"tab\": \"General information\", \"score\": \"151.0\"}", - "High School Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # prompt tokens": "{\"description\": \"min=558.841, mean=558.841, max=558.841, sum=1117.682 (2)\", \"tab\": \"General information\", \"score\": \"558.841059602649\"}", - "High School Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Psychology - # eval": "{\"description\": \"min=545, mean=545, max=545, sum=1090 (2)\", \"tab\": \"General information\", \"score\": \"545.0\"}", - "High School Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # prompt tokens": "{\"description\": \"min=521.42, mean=521.42, max=521.42, sum=1042.84 (2)\", \"tab\": \"General information\", \"score\": \"521.4201834862386\"}", - "High School Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Statistics - # eval": "{\"description\": \"min=216, mean=216, max=216, sum=432 (2)\", \"tab\": \"General information\", \"score\": \"216.0\"}", - "High School Statistics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Statistics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # prompt tokens": "{\"description\": \"min=806.963, mean=806.963, max=806.963, sum=1613.926 (2)\", \"tab\": \"General information\", \"score\": \"806.9629629629629\"}", - "High School Statistics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School US History - # eval": "{\"description\": \"min=204, mean=204, max=204, sum=408 (2)\", \"tab\": \"General information\", \"score\": \"204.0\"}", - "High School US History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School US History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # prompt tokens": "{\"description\": \"min=2288.49, mean=2288.49, max=2288.49, sum=4576.98 (2)\", \"tab\": \"General information\", \"score\": \"2288.4901960784314\"}", - "High School US History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School World History - # eval": "{\"description\": \"min=237, mean=237, max=237, sum=474 (2)\", \"tab\": \"General information\", \"score\": \"237.0\"}", - "High School World History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School World History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # prompt tokens": "{\"description\": \"min=1475.932, mean=1475.932, max=1475.932, sum=2951.865 (2)\", \"tab\": \"General information\", \"score\": \"1475.9324894514768\"}", - "High School World History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"high_school_world_history\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_high_school_world_history\"" - } - } - }, - { - "evaluation_name": "Human Sexuality", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Human Sexuality", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.908, - "details": { - "description": "min=0.908, mean=0.908, max=0.908, sum=1.817 (2)", - "tab": "Accuracy", - "Human Aging - Observed inference time (s)": "{\"description\": \"min=3.859, mean=3.859, max=3.859, sum=7.719 (2)\", \"tab\": \"Efficiency\", \"score\": \"3.8594313245183147\"}", - "Human Sexuality - Observed inference time (s)": "{\"description\": \"min=3.96, mean=3.96, max=3.96, sum=7.92 (2)\", \"tab\": \"Efficiency\", \"score\": \"3.9598546119136664\"}", - "Human Aging - # eval": "{\"description\": \"min=223, mean=223, max=223, sum=446 (2)\", \"tab\": \"General information\", \"score\": \"223.0\"}", - "Human Aging - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Aging - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Aging - # prompt tokens": "{\"description\": \"min=335.955, mean=335.955, max=335.955, sum=671.91 (2)\", \"tab\": \"General information\", \"score\": \"335.95515695067263\"}", - "Human Aging - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Human Sexuality - # eval": "{\"description\": \"min=131, mean=131, max=131, sum=262 (2)\", \"tab\": \"General information\", \"score\": \"131.0\"}", - "Human Sexuality - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Sexuality - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # prompt tokens": "{\"description\": \"min=371.496, mean=371.496, max=371.496, sum=742.992 (2)\", \"tab\": \"General information\", \"score\": \"371.4961832061069\"}", - "Human Sexuality - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"human_sexuality\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_human_sexuality\"" - } - } - }, - { - "evaluation_name": "International Law", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on International Law", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.901, - "details": { - "description": "min=0.901, mean=0.901, max=0.901, sum=1.802 (2)", - "tab": "Accuracy", - "International Law - Observed inference time (s)": "{\"description\": \"min=3.884, mean=3.884, max=3.884, sum=7.767 (2)\", \"tab\": \"Efficiency\", \"score\": \"3.8836900754408403\"}", - "International Law - # eval": "{\"description\": \"min=121, mean=121, max=121, sum=242 (2)\", \"tab\": \"General information\", \"score\": \"121.0\"}", - "International Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "International Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "International Law - # prompt tokens": "{\"description\": \"min=664.165, mean=664.165, max=664.165, sum=1328.331 (2)\", \"tab\": \"General information\", \"score\": \"664.1652892561983\"}", - "International Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"international_law\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_international_law\"" - } - } - }, - { - "evaluation_name": "Logical Fallacies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Logical Fallacies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.896, - "details": { - "description": "min=0.896, mean=0.896, max=0.896, sum=1.791 (2)", - "tab": "Accuracy", - "Logical Fallacies - Observed inference time (s)": "{\"description\": \"min=3.913, mean=3.913, max=3.913, sum=7.826 (2)\", \"tab\": \"Efficiency\", \"score\": \"3.9131746394502605\"}", - "Logical Fallacies - # eval": "{\"description\": \"min=163, mean=163, max=163, sum=326 (2)\", \"tab\": \"General information\", \"score\": \"163.0\"}", - "Logical Fallacies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Logical Fallacies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Logical Fallacies - # prompt tokens": "{\"description\": \"min=470.276, mean=470.276, max=470.276, sum=940.552 (2)\", \"tab\": \"General information\", \"score\": \"470.2760736196319\"}", - "Logical Fallacies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"logical_fallacies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_logical_fallacies\"" - } - } - }, - { - "evaluation_name": "Machine Learning", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Machine Learning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.741, - "details": { - "description": "min=0.741, mean=0.741, max=0.741, sum=1.482 (2)", - "tab": "Accuracy", - "Machine Learning - Observed inference time (s)": "{\"description\": \"min=4.19, mean=4.19, max=4.19, sum=8.379 (2)\", \"tab\": \"Efficiency\", \"score\": \"4.189559940780912\"}", - "Machine Learning - # eval": "{\"description\": \"min=112, mean=112, max=112, sum=224 (2)\", \"tab\": \"General information\", \"score\": \"112.0\"}", - "Machine Learning - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Machine Learning - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Machine Learning - # prompt tokens": "{\"description\": \"min=676.518, mean=676.518, max=676.518, sum=1353.036 (2)\", \"tab\": \"General information\", \"score\": \"676.5178571428571\"}", - "Machine Learning - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"machine_learning\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_machine_learning\"" - } - } - }, - { - "evaluation_name": "Management", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Management", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.942, - "details": { - "description": "min=0.942, mean=0.942, max=0.942, sum=1.883 (2)", - "tab": "Accuracy", - "Management - Observed inference time (s)": "{\"description\": \"min=4.01, mean=4.01, max=4.01, sum=8.02 (2)\", \"tab\": \"Efficiency\", \"score\": \"4.009768469819745\"}", - "Management - # eval": "{\"description\": \"min=103, mean=103, max=103, sum=206 (2)\", \"tab\": \"General information\", \"score\": \"103.0\"}", - "Management - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Management - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Management - # prompt tokens": "{\"description\": \"min=301.282, mean=301.282, max=301.282, sum=602.563 (2)\", \"tab\": \"General information\", \"score\": \"301.28155339805824\"}", - "Management - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"management\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_management\"" - } - } - }, - { - "evaluation_name": "Marketing", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Marketing", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.944, - "details": { - "description": "min=0.944, mean=0.944, max=0.944, sum=1.889 (2)", - "tab": "Accuracy", - "Marketing - Observed inference time (s)": "{\"description\": \"min=3.988, mean=3.988, max=3.988, sum=7.975 (2)\", \"tab\": \"Efficiency\", \"score\": \"3.9875136002516136\"}", - "Marketing - # eval": "{\"description\": \"min=234, mean=234, max=234, sum=468 (2)\", \"tab\": \"General information\", \"score\": \"234.0\"}", - "Marketing - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Marketing - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Marketing - # prompt tokens": "{\"description\": \"min=448.064, mean=448.064, max=448.064, sum=896.128 (2)\", \"tab\": \"General information\", \"score\": \"448.06410256410254\"}", - "Marketing - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"marketing\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_marketing\"" - } - } - }, - { - "evaluation_name": "Medical Genetics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Medical Genetics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.93, - "details": { - "description": "min=0.93, mean=0.93, max=0.93, sum=1.86 (2)", - "tab": "Accuracy", - "Medical Genetics - Observed inference time (s)": "{\"description\": \"min=3.913, mean=3.913, max=3.913, sum=7.827 (2)\", \"tab\": \"Efficiency\", \"score\": \"3.913457498550415\"}", - "Medical Genetics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Medical Genetics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Medical Genetics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Medical Genetics - # prompt tokens": "{\"description\": \"min=354.88, mean=354.88, max=354.88, sum=709.76 (2)\", \"tab\": \"General information\", \"score\": \"354.88\"}", - "Medical Genetics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"medical_genetics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_medical_genetics\"" - } - } - }, - { - "evaluation_name": "Miscellaneous", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Miscellaneous", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.951, - "details": { - "description": "min=0.951, mean=0.951, max=0.951, sum=1.903 (2)", - "tab": "Accuracy", - "Miscellaneous - Observed inference time (s)": "{\"description\": \"min=3.945, mean=3.945, max=3.945, sum=7.889 (2)\", \"tab\": \"Efficiency\", \"score\": \"3.9445087267216747\"}", - "Miscellaneous - # eval": "{\"description\": \"min=783, mean=783, max=783, sum=1566 (2)\", \"tab\": \"General information\", \"score\": \"783.0\"}", - "Miscellaneous - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Miscellaneous - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Miscellaneous - # prompt tokens": "{\"description\": \"min=328.628, mean=328.628, max=328.628, sum=657.257 (2)\", \"tab\": \"General information\", \"score\": \"328.62835249042143\"}", - "Miscellaneous - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"miscellaneous\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_miscellaneous\"" - } - } - }, - { - "evaluation_name": "Moral Scenarios", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Moral Scenarios", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.826, - "details": { - "description": "min=0.826, mean=0.826, max=0.826, sum=1.651 (2)", - "tab": "Accuracy", - "Moral Disputes - Observed inference time (s)": "{\"description\": \"min=4.057, mean=4.057, max=4.057, sum=8.113 (2)\", \"tab\": \"Efficiency\", \"score\": \"4.0566764987273025\"}", - "Moral Scenarios - Observed inference time (s)": "{\"description\": \"min=4.082, mean=4.082, max=4.082, sum=8.165 (2)\", \"tab\": \"Efficiency\", \"score\": \"4.082338048892314\"}", - "Moral Disputes - # eval": "{\"description\": \"min=346, mean=346, max=346, sum=692 (2)\", \"tab\": \"General information\", \"score\": \"346.0\"}", - "Moral Disputes - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Disputes - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Disputes - # prompt tokens": "{\"description\": \"min=511.789, mean=511.789, max=511.789, sum=1023.578 (2)\", \"tab\": \"General information\", \"score\": \"511.78901734104045\"}", - "Moral Disputes - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Moral Scenarios - # eval": "{\"description\": \"min=895, mean=895, max=895, sum=1790 (2)\", \"tab\": \"General information\", \"score\": \"895.0\"}", - "Moral Scenarios - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Scenarios - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # prompt tokens": "{\"description\": \"min=676.949, mean=676.949, max=676.949, sum=1353.897 (2)\", \"tab\": \"General information\", \"score\": \"676.9486033519553\"}", - "Moral Scenarios - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"moral_scenarios\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_moral_scenarios\"" - } - } - }, - { - "evaluation_name": "Nutrition", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Nutrition", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.925, - "details": { - "description": "min=0.925, mean=0.925, max=0.925, sum=1.85 (2)", - "tab": "Accuracy", - "Nutrition - Observed inference time (s)": "{\"description\": \"min=4.106, mean=4.106, max=4.106, sum=8.213 (2)\", \"tab\": \"Efficiency\", \"score\": \"4.106359853464014\"}", - "Nutrition - # eval": "{\"description\": \"min=306, mean=306, max=306, sum=612 (2)\", \"tab\": \"General information\", \"score\": \"306.0\"}", - "Nutrition - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Nutrition - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Nutrition - # prompt tokens": "{\"description\": \"min=617.065, mean=617.065, max=617.065, sum=1234.131 (2)\", \"tab\": \"General information\", \"score\": \"617.0653594771242\"}", - "Nutrition - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"nutrition\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_nutrition\"" - } - } - }, - { - "evaluation_name": "Prehistory", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Prehistory", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.941, - "details": { - "description": "min=0.941, mean=0.941, max=0.941, sum=1.883 (2)", - "tab": "Accuracy", - "Prehistory - Observed inference time (s)": "{\"description\": \"min=3.998, mean=3.998, max=3.998, sum=7.996 (2)\", \"tab\": \"Efficiency\", \"score\": \"3.998204750779234\"}", - "Prehistory - # eval": "{\"description\": \"min=324, mean=324, max=324, sum=648 (2)\", \"tab\": \"General information\", \"score\": \"324.0\"}", - "Prehistory - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Prehistory - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Prehistory - # prompt tokens": "{\"description\": \"min=545.639, mean=545.639, max=545.639, sum=1091.278 (2)\", \"tab\": \"General information\", \"score\": \"545.6388888888889\"}", - "Prehistory - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"prehistory\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_prehistory\"" - } - } - }, - { - "evaluation_name": "Public Relations", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Public Relations", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.827, - "details": { - "description": "min=0.827, mean=0.827, max=0.827, sum=1.655 (2)", - "tab": "Accuracy", - "Public Relations - Observed inference time (s)": "{\"description\": \"min=3.843, mean=3.843, max=3.843, sum=7.685 (2)\", \"tab\": \"Efficiency\", \"score\": \"3.8426286415620283\"}", - "Public Relations - # eval": "{\"description\": \"min=110, mean=110, max=110, sum=220 (2)\", \"tab\": \"General information\", \"score\": \"110.0\"}", - "Public Relations - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Public Relations - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Public Relations - # prompt tokens": "{\"description\": \"min=432.991, mean=432.991, max=432.991, sum=865.982 (2)\", \"tab\": \"General information\", \"score\": \"432.9909090909091\"}", - "Public Relations - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"public_relations\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_public_relations\"" - } - } - }, - { - "evaluation_name": "Security Studies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Security Studies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.886, - "details": { - "description": "min=0.886, mean=0.886, max=0.886, sum=1.771 (2)", - "tab": "Accuracy", - "Security Studies - Observed inference time (s)": "{\"description\": \"min=4.346, mean=4.346, max=4.346, sum=8.692 (2)\", \"tab\": \"Efficiency\", \"score\": \"4.3459005385029075\"}", - "Security Studies - # eval": "{\"description\": \"min=245, mean=245, max=245, sum=490 (2)\", \"tab\": \"General information\", \"score\": \"245.0\"}", - "Security Studies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Security Studies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Security Studies - # prompt tokens": "{\"description\": \"min=1243.804, mean=1243.804, max=1243.804, sum=2487.608 (2)\", \"tab\": \"General information\", \"score\": \"1243.8040816326532\"}", - "Security Studies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"security_studies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_security_studies\"" - } - } - }, - { - "evaluation_name": "Sociology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Sociology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.94, - "details": { - "description": "min=0.94, mean=0.94, max=0.94, sum=1.881 (2)", - "tab": "Accuracy", - "Sociology - Observed inference time (s)": "{\"description\": \"min=3.946, mean=3.946, max=3.946, sum=7.893 (2)\", \"tab\": \"Efficiency\", \"score\": \"3.94632918561869\"}", - "Sociology - # eval": "{\"description\": \"min=201, mean=201, max=201, sum=402 (2)\", \"tab\": \"General information\", \"score\": \"201.0\"}", - "Sociology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Sociology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Sociology - # prompt tokens": "{\"description\": \"min=467.274, mean=467.274, max=467.274, sum=934.547 (2)\", \"tab\": \"General information\", \"score\": \"467.27363184079604\"}", - "Sociology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"sociology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_sociology\"" - } - } - }, - { - "evaluation_name": "Virology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Virology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.578, - "details": { - "description": "min=0.578, mean=0.578, max=0.578, sum=1.157 (2)", - "tab": "Accuracy", - "Virology - Observed inference time (s)": "{\"description\": \"min=3.932, mean=3.932, max=3.932, sum=7.865 (2)\", \"tab\": \"Efficiency\", \"score\": \"3.9324641141546777\"}", - "Virology - # eval": "{\"description\": \"min=166, mean=166, max=166, sum=332 (2)\", \"tab\": \"General information\", \"score\": \"166.0\"}", - "Virology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Virology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Virology - # prompt tokens": "{\"description\": \"min=362.651, mean=362.651, max=362.651, sum=725.301 (2)\", \"tab\": \"General information\", \"score\": \"362.65060240963857\"}", - "Virology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"virology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_virology\"" - } - } - }, - { - "evaluation_name": "World Religions", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on World Religions", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.901, - "details": { - "description": "min=0.901, mean=0.901, max=0.901, sum=1.801 (2)", - "tab": "Accuracy", - "World Religions - Observed inference time (s)": "{\"description\": \"min=4.011, mean=4.011, max=4.011, sum=8.023 (2)\", \"tab\": \"Efficiency\", \"score\": \"4.011422206086722\"}", - "World Religions - # eval": "{\"description\": \"min=171, mean=171, max=171, sum=342 (2)\", \"tab\": \"General information\", \"score\": \"171.0\"}", - "World Religions - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "World Religions - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "World Religions - # prompt tokens": "{\"description\": \"min=293.018, mean=293.018, max=293.018, sum=586.035 (2)\", \"tab\": \"General information\", \"score\": \"293.0175438596491\"}", - "World Religions - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"world_religions\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_world_religions\"" - } - } - }, - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.014, - "details": { - "description": "", - "tab": "Efficiency" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_id": "reward-bench/Anthropic_claude-3-opus-20240229/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8008 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9469 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6031 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8662 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7868 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "reward-bench-2/anthropic_claude-3-opus-20240229/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5744 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5389 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3312 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5137 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8378 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6646 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5601 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/anthropic_claude-3-sonnet-20240229.json b/data/models/anthropic_claude-3-sonnet-20240229.json deleted file mode 100644 index 99daf3b732f21aee5247f0ee2af4bbb5476a56ba..0000000000000000000000000000000000000000 --- a/data/models/anthropic_claude-3-sonnet-20240229.json +++ /dev/null @@ -1,2038 +0,0 @@ -{ - "model_info": { - "name": "Claude 3 Sonnet 20240229", - "id": "anthropic/claude-3-sonnet-20240229", - "developer": "Anthropic", - "inference_platform": "unknown", - "additional_details": { - "model_id_aliases": [ - "Anthropic/claude-3-sonnet-20240229" - ] - } - }, - "evaluations": [ - { - "evaluation_id": "helm_lite/anthropic_claude-3-sonnet-20240229/1774096306.427425", - "retrieved_timestamp": "1774096306.427425", - "source_metadata": { - "source_name": "helm_lite", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_lite", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_lite", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.377, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.27500624219725345\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.111, - "details": { - "description": "min=0.111, mean=0.111, max=0.111, sum=0.111 (1)", - "tab": "Accuracy", - "NarrativeQA - Observed inference time (s)": "{\"description\": \"min=2.239, mean=2.239, max=2.239, sum=2.239 (1)\", \"tab\": \"Efficiency\", \"score\": \"2.2392607588163562\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=3709.741, mean=3709.741, max=3709.741, sum=3709.741 (1)\", \"tab\": \"General information\", \"score\": \"3709.7408450704224\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=30.372, mean=30.372, max=30.372, sum=30.372 (1)\", \"tab\": \"General information\", \"score\": \"30.371830985915494\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (closed-book)", - "source_data": { - "dataset_name": "NaturalQuestions (closed-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (closed-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.028, - "details": { - "description": "min=0.028, mean=0.028, max=0.028, sum=0.028 (1)", - "tab": "Accuracy", - "NaturalQuestions (open-book) - Observed inference time (s)": "{\"description\": \"min=1.828, mean=1.828, max=1.828, sum=1.828 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.828468058347702\"}", - "NaturalQuestions (closed-book) - Observed inference time (s)": "{\"description\": \"min=1.226, mean=1.226, max=1.226, sum=1.226 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.2262272393703462\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1781.799, mean=1781.799, max=1781.799, sum=1781.799 (1)\", \"tab\": \"General information\", \"score\": \"1781.799\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=31.113, mean=31.113, max=31.113, sum=31.113 (1)\", \"tab\": \"General information\", \"score\": \"31.113\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=189.259, mean=189.259, max=189.259, sum=189.259 (1)\", \"tab\": \"General information\", \"score\": \"189.259\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=26.563, mean=26.563, max=26.563, sum=26.563 (1)\", \"tab\": \"General information\", \"score\": \"26.563\"}" - } - }, - "generation_config": { - "additional_details": { - "mode": "\"closedbook\"" - } - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.918, - "details": { - "description": "min=0.918, mean=0.918, max=0.918, sum=0.918 (1)", - "tab": "Accuracy", - "OpenbookQA - Observed inference time (s)": "{\"description\": \"min=1.032, mean=1.032, max=1.032, sum=1.032 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.031575677871704\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=263.79, mean=263.79, max=263.79, sum=263.79 (1)\", \"tab\": \"General information\", \"score\": \"263.79\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "dataset": "\"openbookqa\"", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.652, - "details": { - "description": "min=0.39, mean=0.652, max=0.94, sum=3.26 (5)", - "tab": "Accuracy", - "MMLU - Observed inference time (s)": "{\"description\": \"min=1.228, mean=1.278, max=1.341, sum=6.391 (5)\", \"tab\": \"Efficiency\", \"score\": \"1.2781797420267473\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=370.26, mean=478.747, max=619.596, sum=2393.736 (5)\", \"tab\": \"General information\", \"score\": \"478.747298245614\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"college_chemistry\", \"computer_security\", \"econometrics\", \"us_foreign_policy\"]", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MATH", - "source_data": { - "dataset_name": "MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Equivalent (CoT) on MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.084, - "details": { - "description": "min=0, mean=0.084, max=0.337, sum=0.591 (7)", - "tab": "Accuracy", - "MATH - Observed inference time (s)": "{\"description\": \"min=2.092, mean=2.33, max=2.633, sum=16.311 (7)\", \"tab\": \"Efficiency\", \"score\": \"2.3301560711519222\"}", - "MATH - # eval": "{\"description\": \"min=30, mean=62.429, max=135, sum=437 (7)\", \"tab\": \"General information\", \"score\": \"62.42857142857143\"}", - "MATH - # train": "{\"description\": \"min=8, mean=8, max=8, sum=56 (7)\", \"tab\": \"General information\", \"score\": \"8.0\"}", - "MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MATH - # prompt tokens": "{\"description\": \"min=948.259, mean=1362.814, max=2380.808, sum=9539.699 (7)\", \"tab\": \"General information\", \"score\": \"1362.8141219676104\"}", - "MATH - # output tokens": "{\"description\": \"min=44.263, mean=52.374, max=62.256, sum=366.62 (7)\", \"tab\": \"General information\", \"score\": \"52.37429092508652\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"algebra\", \"counting_and_probability\", \"geometry\", \"intermediate_algebra\", \"number_theory\", \"prealgebra\", \"precalculus\"]", - "level": "\"1\"", - "use_official_examples": "\"False\"", - "use_chain_of_thought": "\"True\"" - } - } - }, - { - "evaluation_name": "GSM8K", - "source_data": { - "dataset_name": "GSM8K", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on GSM8K", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.907, - "details": { - "description": "min=0.907, mean=0.907, max=0.907, sum=0.907 (1)", - "tab": "Accuracy", - "GSM8K - Observed inference time (s)": "{\"description\": \"min=3.213, mean=3.213, max=3.213, sum=3.213 (1)\", \"tab\": \"Efficiency\", \"score\": \"3.2127642614841463\"}", - "GSM8K - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "GSM8K - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "GSM8K - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GSM8K - # prompt tokens": "{\"description\": \"min=1012.712, mean=1012.712, max=1012.712, sum=1012.712 (1)\", \"tab\": \"General information\", \"score\": \"1012.712\"}", - "GSM8K - # output tokens": "{\"description\": \"min=114.663, mean=114.663, max=114.663, sum=114.663 (1)\", \"tab\": \"General information\", \"score\": \"114.663\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "LegalBench", - "source_data": { - "dataset_name": "LegalBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on LegalBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.49, - "details": { - "description": "min=0.029, mean=0.49, max=0.958, sum=2.448 (5)", - "tab": "Accuracy", - "LegalBench - Observed inference time (s)": "{\"description\": \"min=0.683, mean=1.316, max=2.689, sum=6.58 (5)\", \"tab\": \"Efficiency\", \"score\": \"1.3159105889028733\"}", - "LegalBench - # eval": "{\"description\": \"min=95, mean=409.4, max=1000, sum=2047 (5)\", \"tab\": \"General information\", \"score\": \"409.4\"}", - "LegalBench - # train": "{\"description\": \"min=4, mean=4.8, max=5, sum=24 (5)\", \"tab\": \"General information\", \"score\": \"4.8\"}", - "LegalBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "LegalBench - # prompt tokens": "{\"description\": \"min=214.653, mean=1557.242, max=6428.398, sum=7786.208 (5)\", \"tab\": \"General information\", \"score\": \"1557.241581367783\"}", - "LegalBench - # output tokens": "{\"description\": \"min=1, mean=9.202, max=27.753, sum=46.009 (5)\", \"tab\": \"General information\", \"score\": \"9.201869121421694\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"abercrombie\", \"corporate_lobbying\", \"function_of_decision_section\", \"international_citizenship_questions\", \"proa\"]" - } - } - }, - { - "evaluation_name": "MedQA", - "source_data": { - "dataset_name": "MedQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MedQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.684, - "details": { - "description": "min=0.684, mean=0.684, max=0.684, sum=0.684 (1)", - "tab": "Accuracy", - "MedQA - Observed inference time (s)": "{\"description\": \"min=1.143, mean=1.143, max=1.143, sum=1.143 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.1428523476033752\"}", - "MedQA - # eval": "{\"description\": \"min=503, mean=503, max=503, sum=503 (1)\", \"tab\": \"General information\", \"score\": \"503.0\"}", - "MedQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MedQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MedQA - # prompt tokens": "{\"description\": \"min=1027.437, mean=1027.437, max=1027.437, sum=1027.437 (1)\", \"tab\": \"General information\", \"score\": \"1027.4373757455269\"}", - "MedQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WMT 2014", - "source_data": { - "dataset_name": "WMT 2014", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "BLEU-4 on WMT 2014", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.218, - "details": { - "description": "min=0.169, mean=0.218, max=0.25, sum=1.091 (5)", - "tab": "Accuracy", - "WMT 2014 - Observed inference time (s)": "{\"description\": \"min=1.066, mean=1.139, max=1.228, sum=5.697 (5)\", \"tab\": \"Efficiency\", \"score\": \"1.1393479201068188\"}", - "WMT 2014 - # eval": "{\"description\": \"min=503, mean=568.8, max=832, sum=2844 (5)\", \"tab\": \"General information\", \"score\": \"568.8\"}", - "WMT 2014 - # train": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "WMT 2014 - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WMT 2014 - # prompt tokens": "{\"description\": \"min=198.406, mean=219.573, max=241.974, sum=1097.866 (5)\", \"tab\": \"General information\", \"score\": \"219.57322077152472\"}", - "WMT 2014 - # output tokens": "{\"description\": \"min=24.517, mean=26.056, max=27.078, sum=130.278 (5)\", \"tab\": \"General information\", \"score\": \"26.05551068588469\"}" - } - }, - "generation_config": { - "additional_details": { - "language_pair": "[\"cs-en\", \"de-en\", \"fr-en\", \"hi-en\", \"ru-en\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_mmlu/anthropic_claude-3-sonnet-20240229/1774096312.00548", - "retrieved_timestamp": "1774096312.00548", - "source_metadata": { - "source_name": "helm_mmlu", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_mmlu", - "evaluation_results": [ - { - "evaluation_name": "MMLU All Subjects", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU All Subjects", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.759, - "details": { - "description": "min=0.39, mean=0.759, max=0.959, sum=86.545 (114)", - "tab": "Accuracy", - "MMLU All Subjects - Observed inference time (s)": "{\"description\": \"min=1.21, mean=1.468, max=8.072, sum=167.341 (114)\", \"tab\": \"Efficiency\", \"score\": \"1.4679056233464987\"}", - "MMLU All Subjects - # eval": "{\"description\": \"min=100, mean=246.351, max=1534, sum=28084 (114)\", \"tab\": \"General information\", \"score\": \"246.35087719298247\"}", - "MMLU All Subjects - # train": "{\"description\": \"min=5, mean=5, max=5, sum=570 (114)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU All Subjects - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU All Subjects - # prompt tokens": "{\"description\": \"min=293.018, mean=638.288, max=2887.576, sum=72764.875 (114)\", \"tab\": \"General information\", \"score\": \"638.2883793758953\"}", - "MMLU All Subjects - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=114 (114)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_name": "Abstract Algebra", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Abstract Algebra", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.39, - "details": { - "description": "min=0.39, mean=0.39, max=0.39, sum=0.78 (2)", - "tab": "Accuracy", - "Abstract Algebra - Observed inference time (s)": "{\"description\": \"min=1.248, mean=1.248, max=1.248, sum=2.495 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.2476251411437989\"}", - "Abstract Algebra - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Abstract Algebra - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Abstract Algebra - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Abstract Algebra - # prompt tokens": "{\"description\": \"min=370.26, mean=370.26, max=370.26, sum=740.52 (2)\", \"tab\": \"General information\", \"score\": \"370.26\"}", - "Abstract Algebra - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"abstract_algebra\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_abstract_algebra\"" - } - } - }, - { - "evaluation_name": "Anatomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Anatomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.711, - "details": { - "description": "min=0.711, mean=0.711, max=0.711, sum=1.422 (2)", - "tab": "Accuracy", - "Anatomy - Observed inference time (s)": "{\"description\": \"min=1.225, mean=1.225, max=1.225, sum=2.45 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.224808097768713\"}", - "Anatomy - # eval": "{\"description\": \"min=135, mean=135, max=135, sum=270 (2)\", \"tab\": \"General information\", \"score\": \"135.0\"}", - "Anatomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Anatomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Anatomy - # prompt tokens": "{\"description\": \"min=370.8, mean=370.8, max=370.8, sum=741.6 (2)\", \"tab\": \"General information\", \"score\": \"370.8\"}", - "Anatomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"anatomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_anatomy\"" - } - } - }, - { - "evaluation_name": "College Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on College Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.559, - "details": { - "description": "min=0.559, mean=0.559, max=0.559, sum=1.118 (2)", - "tab": "Accuracy", - "College Chemistry - Observed inference time (s)": "{\"description\": \"min=1.33, mean=1.33, max=1.33, sum=2.659 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.3297029423713684\"}", - "College Biology - Observed inference time (s)": "{\"description\": \"min=1.293, mean=1.293, max=1.293, sum=2.585 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.2926498336924448\"}", - "College Computer Science - Observed inference time (s)": "{\"description\": \"min=1.494, mean=1.494, max=1.494, sum=2.988 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.493921182155609\"}", - "College Mathematics - Observed inference time (s)": "{\"description\": \"min=1.346, mean=1.346, max=1.346, sum=2.693 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.346416823863983\"}", - "College Medicine - Observed inference time (s)": "{\"description\": \"min=1.316, mean=1.316, max=1.316, sum=2.632 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.315991141203511\"}", - "College Physics - Observed inference time (s)": "{\"description\": \"min=1.286, mean=1.286, max=1.286, sum=2.573 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.2863672691233017\"}", - "College Chemistry - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Chemistry - # prompt tokens": "{\"description\": \"min=550.01, mean=550.01, max=550.01, sum=1100.02 (2)\", \"tab\": \"General information\", \"score\": \"550.01\"}", - "College Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Biology - # eval": "{\"description\": \"min=144, mean=144, max=144, sum=288 (2)\", \"tab\": \"General information\", \"score\": \"144.0\"}", - "College Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # prompt tokens": "{\"description\": \"min=490.347, mean=490.347, max=490.347, sum=980.694 (2)\", \"tab\": \"General information\", \"score\": \"490.34722222222223\"}", - "College Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # prompt tokens": "{\"description\": \"min=838.24, mean=838.24, max=838.24, sum=1676.48 (2)\", \"tab\": \"General information\", \"score\": \"838.24\"}", - "College Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Mathematics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # prompt tokens": "{\"description\": \"min=604.19, mean=604.19, max=604.19, sum=1208.38 (2)\", \"tab\": \"General information\", \"score\": \"604.19\"}", - "College Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Medicine - # eval": "{\"description\": \"min=173, mean=173, max=173, sum=346 (2)\", \"tab\": \"General information\", \"score\": \"173.0\"}", - "College Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # prompt tokens": "{\"description\": \"min=540.63, mean=540.63, max=540.63, sum=1081.26 (2)\", \"tab\": \"General information\", \"score\": \"540.6300578034682\"}", - "College Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Physics - # eval": "{\"description\": \"min=102, mean=102, max=102, sum=204 (2)\", \"tab\": \"General information\", \"score\": \"102.0\"}", - "College Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # prompt tokens": "{\"description\": \"min=489.48, mean=489.48, max=489.48, sum=978.961 (2)\", \"tab\": \"General information\", \"score\": \"489.48039215686276\"}", - "College Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"college_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_college_physics\"" - } - } - }, - { - "evaluation_name": "Computer Security", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Computer Security", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.79, - "details": { - "description": "min=0.79, mean=0.79, max=0.79, sum=1.58 (2)", - "tab": "Accuracy", - "Computer Security - Observed inference time (s)": "{\"description\": \"min=1.228, mean=1.228, max=1.228, sum=2.456 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.2280330896377563\"}", - "Computer Security - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Computer Security - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Computer Security - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Computer Security - # prompt tokens": "{\"description\": \"min=398.62, mean=398.62, max=398.62, sum=797.24 (2)\", \"tab\": \"General information\", \"score\": \"398.62\"}", - "Computer Security - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"computer_security\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_computer_security\"" - } - } - }, - { - "evaluation_name": "Econometrics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Econometrics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.64, - "details": { - "description": "min=0.64, mean=0.64, max=0.64, sum=1.281 (2)", - "tab": "Accuracy", - "Econometrics - Observed inference time (s)": "{\"description\": \"min=1.341, mean=1.341, max=1.341, sum=2.682 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.3410238989612513\"}", - "Econometrics - # eval": "{\"description\": \"min=114, mean=114, max=114, sum=228 (2)\", \"tab\": \"General information\", \"score\": \"114.0\"}", - "Econometrics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Econometrics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Econometrics - # prompt tokens": "{\"description\": \"min=619.596, mean=619.596, max=619.596, sum=1239.193 (2)\", \"tab\": \"General information\", \"score\": \"619.5964912280701\"}", - "Econometrics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"econometrics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_econometrics\"" - } - } - }, - { - "evaluation_name": "Global Facts", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Global Facts", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.53, - "details": { - "description": "min=0.53, mean=0.53, max=0.53, sum=1.06 (2)", - "tab": "Accuracy", - "Global Facts - Observed inference time (s)": "{\"description\": \"min=1.253, mean=1.253, max=1.253, sum=2.505 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.2527140331268312\"}", - "Global Facts - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Global Facts - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Global Facts - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Global Facts - # prompt tokens": "{\"description\": \"min=411.61, mean=411.61, max=411.61, sum=823.22 (2)\", \"tab\": \"General information\", \"score\": \"411.61\"}", - "Global Facts - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"global_facts\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_global_facts\"" - } - } - }, - { - "evaluation_name": "Jurisprudence", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Jurisprudence", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.861, - "details": { - "description": "min=0.861, mean=0.861, max=0.861, sum=1.722 (2)", - "tab": "Accuracy", - "Jurisprudence - Observed inference time (s)": "{\"description\": \"min=1.248, mean=1.248, max=1.248, sum=2.496 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.2482430162253204\"}", - "Jurisprudence - # eval": "{\"description\": \"min=108, mean=108, max=108, sum=216 (2)\", \"tab\": \"General information\", \"score\": \"108.0\"}", - "Jurisprudence - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Jurisprudence - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Jurisprudence - # prompt tokens": "{\"description\": \"min=431.426, mean=431.426, max=431.426, sum=862.852 (2)\", \"tab\": \"General information\", \"score\": \"431.4259259259259\"}", - "Jurisprudence - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"jurisprudence\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_jurisprudence\"" - } - } - }, - { - "evaluation_name": "Philosophy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Philosophy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.852, - "details": { - "description": "min=0.852, mean=0.852, max=0.852, sum=1.704 (2)", - "tab": "Accuracy", - "Philosophy - Observed inference time (s)": "{\"description\": \"min=1.221, mean=1.221, max=1.221, sum=2.442 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.22093992217944\"}", - "Philosophy - # eval": "{\"description\": \"min=311, mean=311, max=311, sum=622 (2)\", \"tab\": \"General information\", \"score\": \"311.0\"}", - "Philosophy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Philosophy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Philosophy - # prompt tokens": "{\"description\": \"min=359.965, mean=359.965, max=359.965, sum=719.929 (2)\", \"tab\": \"General information\", \"score\": \"359.9646302250804\"}", - "Philosophy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"philosophy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_philosophy\"" - } - } - }, - { - "evaluation_name": "Professional Psychology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Professional Psychology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.814, - "details": { - "description": "min=0.814, mean=0.814, max=0.814, sum=1.627 (2)", - "tab": "Accuracy", - "Professional Medicine - Observed inference time (s)": "{\"description\": \"min=1.608, mean=1.608, max=1.608, sum=3.216 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.6081139156047035\"}", - "Professional Accounting - Observed inference time (s)": "{\"description\": \"min=1.391, mean=1.391, max=1.391, sum=2.781 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.3905252252064697\"}", - "Professional Law - Observed inference time (s)": "{\"description\": \"min=1.87, mean=1.87, max=1.87, sum=3.741 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.8703640130539139\"}", - "Professional Psychology - Observed inference time (s)": "{\"description\": \"min=1.297, mean=1.297, max=1.297, sum=2.593 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.2967337436146207\"}", - "Professional Medicine - # eval": "{\"description\": \"min=272, mean=272, max=272, sum=544 (2)\", \"tab\": \"General information\", \"score\": \"272.0\"}", - "Professional Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Medicine - # prompt tokens": "{\"description\": \"min=1123.537, mean=1123.537, max=1123.537, sum=2247.074 (2)\", \"tab\": \"General information\", \"score\": \"1123.5367647058824\"}", - "Professional Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Accounting - # eval": "{\"description\": \"min=282, mean=282, max=282, sum=564 (2)\", \"tab\": \"General information\", \"score\": \"282.0\"}", - "Professional Accounting - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Accounting - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # prompt tokens": "{\"description\": \"min=665.422, mean=665.422, max=665.422, sum=1330.844 (2)\", \"tab\": \"General information\", \"score\": \"665.4219858156029\"}", - "Professional Accounting - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Law - # eval": "{\"description\": \"min=1534, mean=1534, max=1534, sum=3068 (2)\", \"tab\": \"General information\", \"score\": \"1534.0\"}", - "Professional Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # prompt tokens": "{\"description\": \"min=1701.16, mean=1701.16, max=1701.16, sum=3402.321 (2)\", \"tab\": \"General information\", \"score\": \"1701.16036505867\"}", - "Professional Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Psychology - # eval": "{\"description\": \"min=612, mean=612, max=612, sum=1224 (2)\", \"tab\": \"General information\", \"score\": \"612.0\"}", - "Professional Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # prompt tokens": "{\"description\": \"min=603.168, mean=603.168, max=603.168, sum=1206.337 (2)\", \"tab\": \"General information\", \"score\": \"603.1683006535948\"}", - "Professional Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"professional_psychology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_professional_psychology\"" - } - } - }, - { - "evaluation_name": "Us Foreign Policy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Us Foreign Policy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.94, - "details": { - "description": "min=0.94, mean=0.94, max=0.94, sum=1.88 (2)", - "tab": "Accuracy", - "Us Foreign Policy - Observed inference time (s)": "{\"description\": \"min=1.245, mean=1.245, max=1.245, sum=2.489 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.2445136380195618\"}", - "Us Foreign Policy - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Us Foreign Policy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Us Foreign Policy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Us Foreign Policy - # prompt tokens": "{\"description\": \"min=455.25, mean=455.25, max=455.25, sum=910.5 (2)\", \"tab\": \"General information\", \"score\": \"455.25\"}", - "Us Foreign Policy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"us_foreign_policy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_us_foreign_policy\"" - } - } - }, - { - "evaluation_name": "Astronomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Astronomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.855, - "details": { - "description": "min=0.855, mean=0.855, max=0.855, sum=1.711 (2)", - "tab": "Accuracy", - "Astronomy - Observed inference time (s)": "{\"description\": \"min=1.303, mean=1.303, max=1.303, sum=2.607 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.3033642768859863\"}", - "Astronomy - # eval": "{\"description\": \"min=152, mean=152, max=152, sum=304 (2)\", \"tab\": \"General information\", \"score\": \"152.0\"}", - "Astronomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Astronomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Astronomy - # prompt tokens": "{\"description\": \"min=604.493, mean=604.493, max=604.493, sum=1208.987 (2)\", \"tab\": \"General information\", \"score\": \"604.4934210526316\"}", - "Astronomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"astronomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_astronomy\"" - } - } - }, - { - "evaluation_name": "Business Ethics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Business Ethics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.82, - "details": { - "description": "min=0.82, mean=0.82, max=0.82, sum=1.64 (2)", - "tab": "Accuracy", - "Business Ethics - Observed inference time (s)": "{\"description\": \"min=1.304, mean=1.304, max=1.304, sum=2.607 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.3036250400543212\"}", - "Business Ethics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Business Ethics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Business Ethics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Business Ethics - # prompt tokens": "{\"description\": \"min=600.02, mean=600.02, max=600.02, sum=1200.04 (2)\", \"tab\": \"General information\", \"score\": \"600.02\"}", - "Business Ethics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"business_ethics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_business_ethics\"" - } - } - }, - { - "evaluation_name": "Clinical Knowledge", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Clinical Knowledge", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.804, - "details": { - "description": "min=0.804, mean=0.804, max=0.804, sum=1.608 (2)", - "tab": "Accuracy", - "Clinical Knowledge - Observed inference time (s)": "{\"description\": \"min=1.24, mean=1.24, max=1.24, sum=2.48 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.2399591086045751\"}", - "Clinical Knowledge - # eval": "{\"description\": \"min=265, mean=265, max=265, sum=530 (2)\", \"tab\": \"General information\", \"score\": \"265.0\"}", - "Clinical Knowledge - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Clinical Knowledge - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Clinical Knowledge - # prompt tokens": "{\"description\": \"min=429.457, mean=429.457, max=429.457, sum=858.913 (2)\", \"tab\": \"General information\", \"score\": \"429.4566037735849\"}", - "Clinical Knowledge - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"clinical_knowledge\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_clinical_knowledge\"" - } - } - }, - { - "evaluation_name": "Conceptual Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Conceptual Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.774, - "details": { - "description": "min=0.774, mean=0.774, max=0.774, sum=1.549 (2)", - "tab": "Accuracy", - "Conceptual Physics - Observed inference time (s)": "{\"description\": \"min=1.256, mean=1.256, max=1.256, sum=2.513 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.2563625832821461\"}", - "Conceptual Physics - # eval": "{\"description\": \"min=235, mean=235, max=235, sum=470 (2)\", \"tab\": \"General information\", \"score\": \"235.0\"}", - "Conceptual Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Conceptual Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Conceptual Physics - # prompt tokens": "{\"description\": \"min=323.536, mean=323.536, max=323.536, sum=647.072 (2)\", \"tab\": \"General information\", \"score\": \"323.53617021276597\"}", - "Conceptual Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"conceptual_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_conceptual_physics\"" - } - } - }, - { - "evaluation_name": "Electrical Engineering", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Electrical Engineering", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.703, - "details": { - "description": "min=0.703, mean=0.703, max=0.703, sum=1.407 (2)", - "tab": "Accuracy", - "Electrical Engineering - Observed inference time (s)": "{\"description\": \"min=1.276, mean=1.276, max=1.276, sum=2.553 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.276360561107767\"}", - "Electrical Engineering - # eval": "{\"description\": \"min=145, mean=145, max=145, sum=290 (2)\", \"tab\": \"General information\", \"score\": \"145.0\"}", - "Electrical Engineering - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Electrical Engineering - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Electrical Engineering - # prompt tokens": "{\"description\": \"min=437.041, mean=437.041, max=437.041, sum=874.083 (2)\", \"tab\": \"General information\", \"score\": \"437.04137931034484\"}", - "Electrical Engineering - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"electrical_engineering\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_electrical_engineering\"" - } - } - }, - { - "evaluation_name": "Elementary Mathematics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Elementary Mathematics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.635, - "details": { - "description": "min=0.635, mean=0.635, max=0.635, sum=1.27 (2)", - "tab": "Accuracy", - "Elementary Mathematics - Observed inference time (s)": "{\"description\": \"min=1.301, mean=1.301, max=1.301, sum=2.602 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.3010439260926827\"}", - "Elementary Mathematics - # eval": "{\"description\": \"min=378, mean=378, max=378, sum=756 (2)\", \"tab\": \"General information\", \"score\": \"378.0\"}", - "Elementary Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Elementary Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Elementary Mathematics - # prompt tokens": "{\"description\": \"min=511.066, mean=511.066, max=511.066, sum=1022.132 (2)\", \"tab\": \"General information\", \"score\": \"511.06613756613757\"}", - "Elementary Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"elementary_mathematics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_elementary_mathematics\"" - } - } - }, - { - "evaluation_name": "Formal Logic", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Formal Logic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.579, - "details": { - "description": "min=0.579, mean=0.579, max=0.579, sum=1.159 (2)", - "tab": "Accuracy", - "Formal Logic - Observed inference time (s)": "{\"description\": \"min=1.369, mean=1.369, max=1.369, sum=2.738 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.3692201716559274\"}", - "Formal Logic - # eval": "{\"description\": \"min=126, mean=126, max=126, sum=252 (2)\", \"tab\": \"General information\", \"score\": \"126.0\"}", - "Formal Logic - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Formal Logic - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Formal Logic - # prompt tokens": "{\"description\": \"min=646.746, mean=646.746, max=646.746, sum=1293.492 (2)\", \"tab\": \"General information\", \"score\": \"646.7460317460317\"}", - "Formal Logic - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"formal_logic\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_formal_logic\"" - } - } - }, - { - "evaluation_name": "High School World History", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on High School World History", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.895, - "details": { - "description": "min=0.895, mean=0.895, max=0.895, sum=1.789 (2)", - "tab": "Accuracy", - "High School Biology - Observed inference time (s)": "{\"description\": \"min=1.292, mean=1.292, max=1.292, sum=2.585 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.2923692734010759\"}", - "High School Chemistry - Observed inference time (s)": "{\"description\": \"min=1.339, mean=1.339, max=1.339, sum=2.678 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.3387701969428603\"}", - "High School Computer Science - Observed inference time (s)": "{\"description\": \"min=1.51, mean=1.51, max=1.51, sum=3.02 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.5097803854942322\"}", - "High School European History - Observed inference time (s)": "{\"description\": \"min=2.456, mean=2.456, max=2.456, sum=4.912 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.4561073808959035\"}", - "High School Geography - Observed inference time (s)": "{\"description\": \"min=1.269, mean=1.269, max=1.269, sum=2.537 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.2686388372170805\"}", - "High School Government And Politics - Observed inference time (s)": "{\"description\": \"min=1.287, mean=1.287, max=1.287, sum=2.574 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.2869715455900201\"}", - "High School Macroeconomics - Observed inference time (s)": "{\"description\": \"min=1.266, mean=1.266, max=1.266, sum=2.533 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.2664643880648492\"}", - "High School Mathematics - Observed inference time (s)": "{\"description\": \"min=1.313, mean=1.313, max=1.313, sum=2.626 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.3131960109428122\"}", - "High School Microeconomics - Observed inference time (s)": "{\"description\": \"min=1.261, mean=1.261, max=1.261, sum=2.521 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.260614112645638\"}", - "High School Physics - Observed inference time (s)": "{\"description\": \"min=1.302, mean=1.302, max=1.302, sum=2.603 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.3015588419326882\"}", - "High School Psychology - Observed inference time (s)": "{\"description\": \"min=1.304, mean=1.304, max=1.304, sum=2.607 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.3036036592011058\"}", - "High School Statistics - Observed inference time (s)": "{\"description\": \"min=1.512, mean=1.512, max=1.512, sum=3.025 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.512356918167185\"}", - "High School US History - Observed inference time (s)": "{\"description\": \"min=8.072, mean=8.072, max=8.072, sum=16.145 (2)\", \"tab\": \"Efficiency\", \"score\": \"8.072314507821027\"}", - "High School World History - Observed inference time (s)": "{\"description\": \"min=1.746, mean=1.746, max=1.746, sum=3.491 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.74568142066022\"}", - "High School Biology - # eval": "{\"description\": \"min=310, mean=310, max=310, sum=620 (2)\", \"tab\": \"General information\", \"score\": \"310.0\"}", - "High School Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Biology - # prompt tokens": "{\"description\": \"min=534.577, mean=534.577, max=534.577, sum=1069.155 (2)\", \"tab\": \"General information\", \"score\": \"534.5774193548388\"}", - "High School Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Chemistry - # eval": "{\"description\": \"min=203, mean=203, max=203, sum=406 (2)\", \"tab\": \"General information\", \"score\": \"203.0\"}", - "High School Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # prompt tokens": "{\"description\": \"min=497.921, mean=497.921, max=497.921, sum=995.842 (2)\", \"tab\": \"General information\", \"score\": \"497.92118226600985\"}", - "High School Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "High School Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # prompt tokens": "{\"description\": \"min=882.4, mean=882.4, max=882.4, sum=1764.8 (2)\", \"tab\": \"General information\", \"score\": \"882.4\"}", - "High School Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School European History - # eval": "{\"description\": \"min=165, mean=165, max=165, sum=330 (2)\", \"tab\": \"General information\", \"score\": \"165.0\"}", - "High School European History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School European History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # prompt tokens": "{\"description\": \"min=2887.576, mean=2887.576, max=2887.576, sum=5775.152 (2)\", \"tab\": \"General information\", \"score\": \"2887.5757575757575\"}", - "High School European History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Geography - # eval": "{\"description\": \"min=198, mean=198, max=198, sum=396 (2)\", \"tab\": \"General information\", \"score\": \"198.0\"}", - "High School Geography - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Geography - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # prompt tokens": "{\"description\": \"min=412.268, mean=412.268, max=412.268, sum=824.535 (2)\", \"tab\": \"General information\", \"score\": \"412.2676767676768\"}", - "High School Geography - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Government And Politics - # eval": "{\"description\": \"min=193, mean=193, max=193, sum=386 (2)\", \"tab\": \"General information\", \"score\": \"193.0\"}", - "High School Government And Politics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Government And Politics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # prompt tokens": "{\"description\": \"min=491.104, mean=491.104, max=491.104, sum=982.207 (2)\", \"tab\": \"General information\", \"score\": \"491.10362694300517\"}", - "High School Government And Politics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Macroeconomics - # eval": "{\"description\": \"min=390, mean=390, max=390, sum=780 (2)\", \"tab\": \"General information\", \"score\": \"390.0\"}", - "High School Macroeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Macroeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # prompt tokens": "{\"description\": \"min=406.036, mean=406.036, max=406.036, sum=812.072 (2)\", \"tab\": \"General information\", \"score\": \"406.0358974358974\"}", - "High School Macroeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Mathematics - # eval": "{\"description\": \"min=270, mean=270, max=270, sum=540 (2)\", \"tab\": \"General information\", \"score\": \"270.0\"}", - "High School Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # prompt tokens": "{\"description\": \"min=519.881, mean=519.881, max=519.881, sum=1039.763 (2)\", \"tab\": \"General information\", \"score\": \"519.8814814814815\"}", - "High School Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Microeconomics - # eval": "{\"description\": \"min=238, mean=238, max=238, sum=476 (2)\", \"tab\": \"General information\", \"score\": \"238.0\"}", - "High School Microeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Microeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # prompt tokens": "{\"description\": \"min=420.513, mean=420.513, max=420.513, sum=841.025 (2)\", \"tab\": \"General information\", \"score\": \"420.5126050420168\"}", - "High School Microeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Physics - # eval": "{\"description\": \"min=151, mean=151, max=151, sum=302 (2)\", \"tab\": \"General information\", \"score\": \"151.0\"}", - "High School Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # prompt tokens": "{\"description\": \"min=558.841, mean=558.841, max=558.841, sum=1117.682 (2)\", \"tab\": \"General information\", \"score\": \"558.841059602649\"}", - "High School Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Psychology - # eval": "{\"description\": \"min=545, mean=545, max=545, sum=1090 (2)\", \"tab\": \"General information\", \"score\": \"545.0\"}", - "High School Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # prompt tokens": "{\"description\": \"min=521.42, mean=521.42, max=521.42, sum=1042.84 (2)\", \"tab\": \"General information\", \"score\": \"521.4201834862386\"}", - "High School Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Statistics - # eval": "{\"description\": \"min=216, mean=216, max=216, sum=432 (2)\", \"tab\": \"General information\", \"score\": \"216.0\"}", - "High School Statistics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Statistics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # prompt tokens": "{\"description\": \"min=806.963, mean=806.963, max=806.963, sum=1613.926 (2)\", \"tab\": \"General information\", \"score\": \"806.9629629629629\"}", - "High School Statistics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School US History - # eval": "{\"description\": \"min=204, mean=204, max=204, sum=408 (2)\", \"tab\": \"General information\", \"score\": \"204.0\"}", - "High School US History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School US History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # prompt tokens": "{\"description\": \"min=2288.49, mean=2288.49, max=2288.49, sum=4576.98 (2)\", \"tab\": \"General information\", \"score\": \"2288.4901960784314\"}", - "High School US History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School World History - # eval": "{\"description\": \"min=237, mean=237, max=237, sum=474 (2)\", \"tab\": \"General information\", \"score\": \"237.0\"}", - "High School World History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School World History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # prompt tokens": "{\"description\": \"min=1475.932, mean=1475.932, max=1475.932, sum=2951.865 (2)\", \"tab\": \"General information\", \"score\": \"1475.9324894514768\"}", - "High School World History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"high_school_world_history\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_high_school_world_history\"" - } - } - }, - { - "evaluation_name": "Human Sexuality", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Human Sexuality", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.809, - "details": { - "description": "min=0.809, mean=0.809, max=0.809, sum=1.618 (2)", - "tab": "Accuracy", - "Human Aging - Observed inference time (s)": "{\"description\": \"min=1.21, mean=1.21, max=1.21, sum=2.42 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.2099821963117796\"}", - "Human Sexuality - Observed inference time (s)": "{\"description\": \"min=1.255, mean=1.255, max=1.255, sum=2.509 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.2545511012768928\"}", - "Human Aging - # eval": "{\"description\": \"min=223, mean=223, max=223, sum=446 (2)\", \"tab\": \"General information\", \"score\": \"223.0\"}", - "Human Aging - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Aging - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Aging - # prompt tokens": "{\"description\": \"min=335.955, mean=335.955, max=335.955, sum=671.91 (2)\", \"tab\": \"General information\", \"score\": \"335.95515695067263\"}", - "Human Aging - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Human Sexuality - # eval": "{\"description\": \"min=131, mean=131, max=131, sum=262 (2)\", \"tab\": \"General information\", \"score\": \"131.0\"}", - "Human Sexuality - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Sexuality - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # prompt tokens": "{\"description\": \"min=371.496, mean=371.496, max=371.496, sum=742.992 (2)\", \"tab\": \"General information\", \"score\": \"371.4961832061069\"}", - "Human Sexuality - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"human_sexuality\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_human_sexuality\"" - } - } - }, - { - "evaluation_name": "International Law", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on International Law", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.909, - "details": { - "description": "min=0.909, mean=0.909, max=0.909, sum=1.818 (2)", - "tab": "Accuracy", - "International Law - Observed inference time (s)": "{\"description\": \"min=1.375, mean=1.375, max=1.375, sum=2.751 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.3753716256007675\"}", - "International Law - # eval": "{\"description\": \"min=121, mean=121, max=121, sum=242 (2)\", \"tab\": \"General information\", \"score\": \"121.0\"}", - "International Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "International Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "International Law - # prompt tokens": "{\"description\": \"min=664.165, mean=664.165, max=664.165, sum=1328.331 (2)\", \"tab\": \"General information\", \"score\": \"664.1652892561983\"}", - "International Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"international_law\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_international_law\"" - } - } - }, - { - "evaluation_name": "Logical Fallacies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Logical Fallacies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.853, - "details": { - "description": "min=0.853, mean=0.853, max=0.853, sum=1.706 (2)", - "tab": "Accuracy", - "Logical Fallacies - Observed inference time (s)": "{\"description\": \"min=1.237, mean=1.237, max=1.237, sum=2.474 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.23694542580587\"}", - "Logical Fallacies - # eval": "{\"description\": \"min=163, mean=163, max=163, sum=326 (2)\", \"tab\": \"General information\", \"score\": \"163.0\"}", - "Logical Fallacies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Logical Fallacies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Logical Fallacies - # prompt tokens": "{\"description\": \"min=470.276, mean=470.276, max=470.276, sum=940.552 (2)\", \"tab\": \"General information\", \"score\": \"470.2760736196319\"}", - "Logical Fallacies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"logical_fallacies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_logical_fallacies\"" - } - } - }, - { - "evaluation_name": "Machine Learning", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Machine Learning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.643, - "details": { - "description": "min=0.643, mean=0.643, max=0.643, sum=1.286 (2)", - "tab": "Accuracy", - "Machine Learning - Observed inference time (s)": "{\"description\": \"min=1.362, mean=1.362, max=1.362, sum=2.725 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.3623365994010652\"}", - "Machine Learning - # eval": "{\"description\": \"min=112, mean=112, max=112, sum=224 (2)\", \"tab\": \"General information\", \"score\": \"112.0\"}", - "Machine Learning - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Machine Learning - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Machine Learning - # prompt tokens": "{\"description\": \"min=676.518, mean=676.518, max=676.518, sum=1353.036 (2)\", \"tab\": \"General information\", \"score\": \"676.5178571428571\"}", - "Machine Learning - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"machine_learning\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_machine_learning\"" - } - } - }, - { - "evaluation_name": "Management", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Management", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.922, - "details": { - "description": "min=0.922, mean=0.922, max=0.922, sum=1.845 (2)", - "tab": "Accuracy", - "Management - Observed inference time (s)": "{\"description\": \"min=1.265, mean=1.265, max=1.265, sum=2.529 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.2646709923605317\"}", - "Management - # eval": "{\"description\": \"min=103, mean=103, max=103, sum=206 (2)\", \"tab\": \"General information\", \"score\": \"103.0\"}", - "Management - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Management - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Management - # prompt tokens": "{\"description\": \"min=301.282, mean=301.282, max=301.282, sum=602.563 (2)\", \"tab\": \"General information\", \"score\": \"301.28155339805824\"}", - "Management - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"management\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_management\"" - } - } - }, - { - "evaluation_name": "Marketing", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Marketing", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.85, - "details": { - "description": "min=0.85, mean=0.85, max=0.85, sum=1.701 (2)", - "tab": "Accuracy", - "Marketing - Observed inference time (s)": "{\"description\": \"min=1.251, mean=1.251, max=1.251, sum=2.503 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.2514099310605953\"}", - "Marketing - # eval": "{\"description\": \"min=234, mean=234, max=234, sum=468 (2)\", \"tab\": \"General information\", \"score\": \"234.0\"}", - "Marketing - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Marketing - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Marketing - # prompt tokens": "{\"description\": \"min=448.064, mean=448.064, max=448.064, sum=896.128 (2)\", \"tab\": \"General information\", \"score\": \"448.06410256410254\"}", - "Marketing - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"marketing\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_marketing\"" - } - } - }, - { - "evaluation_name": "Medical Genetics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Medical Genetics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.79, - "details": { - "description": "min=0.79, mean=0.79, max=0.79, sum=1.58 (2)", - "tab": "Accuracy", - "Medical Genetics - Observed inference time (s)": "{\"description\": \"min=1.22, mean=1.22, max=1.22, sum=2.441 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.2204306960105895\"}", - "Medical Genetics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Medical Genetics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Medical Genetics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Medical Genetics - # prompt tokens": "{\"description\": \"min=354.88, mean=354.88, max=354.88, sum=709.76 (2)\", \"tab\": \"General information\", \"score\": \"354.88\"}", - "Medical Genetics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"medical_genetics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_medical_genetics\"" - } - } - }, - { - "evaluation_name": "Miscellaneous", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Miscellaneous", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.872, - "details": { - "description": "min=0.872, mean=0.872, max=0.872, sum=1.745 (2)", - "tab": "Accuracy", - "Miscellaneous - Observed inference time (s)": "{\"description\": \"min=1.233, mean=1.233, max=1.233, sum=2.467 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.2334287364516374\"}", - "Miscellaneous - # eval": "{\"description\": \"min=783, mean=783, max=783, sum=1566 (2)\", \"tab\": \"General information\", \"score\": \"783.0\"}", - "Miscellaneous - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Miscellaneous - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Miscellaneous - # prompt tokens": "{\"description\": \"min=328.628, mean=328.628, max=328.628, sum=657.257 (2)\", \"tab\": \"General information\", \"score\": \"328.62835249042143\"}", - "Miscellaneous - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"miscellaneous\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_miscellaneous\"" - } - } - }, - { - "evaluation_name": "Moral Scenarios", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Moral Scenarios", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.626, - "details": { - "description": "min=0.626, mean=0.626, max=0.626, sum=1.251 (2)", - "tab": "Accuracy", - "Moral Disputes - Observed inference time (s)": "{\"description\": \"min=1.287, mean=1.287, max=1.287, sum=2.575 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.2873861700124134\"}", - "Moral Scenarios - Observed inference time (s)": "{\"description\": \"min=1.361, mean=1.361, max=1.361, sum=2.722 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.361004557156696\"}", - "Moral Disputes - # eval": "{\"description\": \"min=346, mean=346, max=346, sum=692 (2)\", \"tab\": \"General information\", \"score\": \"346.0\"}", - "Moral Disputes - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Disputes - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Disputes - # prompt tokens": "{\"description\": \"min=511.789, mean=511.789, max=511.789, sum=1023.578 (2)\", \"tab\": \"General information\", \"score\": \"511.78901734104045\"}", - "Moral Disputes - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Moral Scenarios - # eval": "{\"description\": \"min=895, mean=895, max=895, sum=1790 (2)\", \"tab\": \"General information\", \"score\": \"895.0\"}", - "Moral Scenarios - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Scenarios - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # prompt tokens": "{\"description\": \"min=676.949, mean=676.949, max=676.949, sum=1353.897 (2)\", \"tab\": \"General information\", \"score\": \"676.9486033519553\"}", - "Moral Scenarios - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"moral_scenarios\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_moral_scenarios\"" - } - } - }, - { - "evaluation_name": "Nutrition", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Nutrition", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.82, - "details": { - "description": "min=0.82, mean=0.82, max=0.82, sum=1.641 (2)", - "tab": "Accuracy", - "Nutrition - Observed inference time (s)": "{\"description\": \"min=1.319, mean=1.319, max=1.319, sum=2.638 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.3189228679619582\"}", - "Nutrition - # eval": "{\"description\": \"min=306, mean=306, max=306, sum=612 (2)\", \"tab\": \"General information\", \"score\": \"306.0\"}", - "Nutrition - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Nutrition - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Nutrition - # prompt tokens": "{\"description\": \"min=617.065, mean=617.065, max=617.065, sum=1234.131 (2)\", \"tab\": \"General information\", \"score\": \"617.0653594771242\"}", - "Nutrition - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"nutrition\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_nutrition\"" - } - } - }, - { - "evaluation_name": "Prehistory", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Prehistory", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.864, - "details": { - "description": "min=0.864, mean=0.864, max=0.864, sum=1.728 (2)", - "tab": "Accuracy", - "Prehistory - Observed inference time (s)": "{\"description\": \"min=1.305, mean=1.305, max=1.305, sum=2.611 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.305255777306027\"}", - "Prehistory - # eval": "{\"description\": \"min=324, mean=324, max=324, sum=648 (2)\", \"tab\": \"General information\", \"score\": \"324.0\"}", - "Prehistory - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Prehistory - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Prehistory - # prompt tokens": "{\"description\": \"min=545.639, mean=545.639, max=545.639, sum=1091.278 (2)\", \"tab\": \"General information\", \"score\": \"545.6388888888889\"}", - "Prehistory - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"prehistory\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_prehistory\"" - } - } - }, - { - "evaluation_name": "Public Relations", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Public Relations", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.782, - "details": { - "description": "min=0.782, mean=0.782, max=0.782, sum=1.564 (2)", - "tab": "Accuracy", - "Public Relations - Observed inference time (s)": "{\"description\": \"min=1.25, mean=1.25, max=1.25, sum=2.5 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.2497538588263772\"}", - "Public Relations - # eval": "{\"description\": \"min=110, mean=110, max=110, sum=220 (2)\", \"tab\": \"General information\", \"score\": \"110.0\"}", - "Public Relations - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Public Relations - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Public Relations - # prompt tokens": "{\"description\": \"min=432.991, mean=432.991, max=432.991, sum=865.982 (2)\", \"tab\": \"General information\", \"score\": \"432.9909090909091\"}", - "Public Relations - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"public_relations\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_public_relations\"" - } - } - }, - { - "evaluation_name": "Security Studies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Security Studies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.865, - "details": { - "description": "min=0.865, mean=0.865, max=0.865, sum=1.731 (2)", - "tab": "Accuracy", - "Security Studies - Observed inference time (s)": "{\"description\": \"min=1.662, mean=1.662, max=1.662, sum=3.325 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.6624354012158453\"}", - "Security Studies - # eval": "{\"description\": \"min=245, mean=245, max=245, sum=490 (2)\", \"tab\": \"General information\", \"score\": \"245.0\"}", - "Security Studies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Security Studies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Security Studies - # prompt tokens": "{\"description\": \"min=1243.804, mean=1243.804, max=1243.804, sum=2487.608 (2)\", \"tab\": \"General information\", \"score\": \"1243.8040816326532\"}", - "Security Studies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"security_studies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_security_studies\"" - } - } - }, - { - "evaluation_name": "Sociology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Sociology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.905, - "details": { - "description": "min=0.905, mean=0.905, max=0.905, sum=1.811 (2)", - "tab": "Accuracy", - "Sociology - Observed inference time (s)": "{\"description\": \"min=1.268, mean=1.268, max=1.268, sum=2.535 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.267556501265189\"}", - "Sociology - # eval": "{\"description\": \"min=201, mean=201, max=201, sum=402 (2)\", \"tab\": \"General information\", \"score\": \"201.0\"}", - "Sociology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Sociology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Sociology - # prompt tokens": "{\"description\": \"min=467.274, mean=467.274, max=467.274, sum=934.547 (2)\", \"tab\": \"General information\", \"score\": \"467.27363184079604\"}", - "Sociology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"sociology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_sociology\"" - } - } - }, - { - "evaluation_name": "Virology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Virology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.578, - "details": { - "description": "min=0.578, mean=0.578, max=0.578, sum=1.157 (2)", - "tab": "Accuracy", - "Virology - Observed inference time (s)": "{\"description\": \"min=1.321, mean=1.321, max=1.321, sum=2.642 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.3211244660687733\"}", - "Virology - # eval": "{\"description\": \"min=166, mean=166, max=166, sum=332 (2)\", \"tab\": \"General information\", \"score\": \"166.0\"}", - "Virology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Virology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Virology - # prompt tokens": "{\"description\": \"min=362.651, mean=362.651, max=362.651, sum=725.301 (2)\", \"tab\": \"General information\", \"score\": \"362.65060240963857\"}", - "Virology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"virology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_virology\"" - } - } - }, - { - "evaluation_name": "World Religions", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on World Religions", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.871, - "details": { - "description": "min=0.871, mean=0.871, max=0.871, sum=1.743 (2)", - "tab": "Accuracy", - "World Religions - Observed inference time (s)": "{\"description\": \"min=1.271, mean=1.271, max=1.271, sum=2.542 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.2710035530447263\"}", - "World Religions - # eval": "{\"description\": \"min=171, mean=171, max=171, sum=342 (2)\", \"tab\": \"General information\", \"score\": \"171.0\"}", - "World Religions - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "World Religions - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "World Religions - # prompt tokens": "{\"description\": \"min=293.018, mean=293.018, max=293.018, sum=586.035 (2)\", \"tab\": \"General information\", \"score\": \"293.0175438596491\"}", - "World Religions - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"world_religions\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_world_religions\"" - } - } - }, - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.082, - "details": { - "description": "", - "tab": "Efficiency" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_id": "reward-bench/Anthropic_claude-3-sonnet-20240229/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7458 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9344 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5658 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8169 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6907 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6963 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/anthropic_claude-3.7-sonnet.json b/data/models/anthropic_claude-3.7-sonnet.json deleted file mode 100644 index 96912f21595b36688d0e19e77b224053cebaf596..0000000000000000000000000000000000000000 --- a/data/models/anthropic_claude-3.7-sonnet.json +++ /dev/null @@ -1,89 +0,0 @@ -{ - "model_info": { - "name": "anthropic/claude-3.7-sonnet", - "developer": "Anthropic", - "inference_platform": "openrouter", - "id": "anthropic/claude-3.7-sonnet" - }, - "evaluations": [ - { - "evaluation_id": "livecodebenchpro/anthropic/claude-3.7-sonnet/1760492095.8105888", - "retrieved_timestamp": "1760492095.8105888", - "source_metadata": { - "source_organization_name": "New York University, Princeton University, University of California San Diego, University of Washington and Canyon Crest Academy", - "evaluator_relationship": "third_party", - "source_name": "Live Code Bench Pro", - "source_type": "documentation" - }, - "eval_library": { - "name": "unknown", - "version": "unknown" - }, - "benchmark": "livecodebenchpro", - "evaluation_results": [ - { - "evaluation_name": "Hard Problems", - "metric_config": { - "evaluation_description": "Pass@1 on Hard Problems", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.0 - }, - "source_data": { - "dataset_name": "Hard Problems", - "source_type": "url", - "url": [ - "https://webhook.cp-bench.orzzh.com/leaderboard/llm/difficulty?difficulty=hard&benchmark_mode=live" - ] - } - }, - { - "evaluation_name": "Medium Problems", - "metric_config": { - "evaluation_description": "Pass@1 on Medium Problems", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.014084507042253521 - }, - "source_data": { - "dataset_name": "Medium Problems", - "source_type": "url", - "url": [ - "https://webhook.cp-bench.orzzh.com/leaderboard/llm/difficulty?difficulty=medium&benchmark_mode=live" - ] - } - }, - { - "evaluation_name": "Easy Problems", - "metric_config": { - "evaluation_description": "Pass@1 on Easy Problems", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.15492957746478872 - }, - "source_data": { - "dataset_name": "Easy Problems", - "source_type": "url", - "url": [ - "https://webhook.cp-bench.orzzh.com/leaderboard/llm/difficulty?difficulty=easy&benchmark_mode=live" - ] - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/anthropic_claude-haiku-4-5-20251001-fc.json b/data/models/anthropic_claude-haiku-4-5-20251001-fc.json deleted file mode 100644 index 08090010a0f3474dd18fa31cd3c73f63f8df312f..0000000000000000000000000000000000000000 --- a/data/models/anthropic_claude-haiku-4-5-20251001-fc.json +++ /dev/null @@ -1,851 +0,0 @@ -{ - "model_info": { - "name": "Claude-Haiku-4-5-20251001 (FC)", - "id": "anthropic/claude-haiku-4-5-20251001-fc", - "developer": "Anthropic", - "additional_details": { - "raw_model_name": "Claude-Haiku-4-5-20251001 (FC)", - "organization": "Anthropic", - "license": "Proprietary", - "mode": "FC", - "model_link": "https://www.anthropic.com/news/claude-haiku-4-5" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/anthropic/claude-haiku-4-5-20251001-fc/1775236112.368598", - "retrieved_timestamp": "1775236112.368598", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 6.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 68.7 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 14.23 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 1.68 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 3.92 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 3.15 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 86.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 71.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 94.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 92.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 88.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 78.68 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 83.72 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 77.59 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 75.0 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 75.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 53.62 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 63.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 42.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 52.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 56.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 83.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 86.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 81.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 54.41 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 51.61 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 55.48 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 56.13 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 62.5 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 85.11 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/anthropic_claude-haiku-4-5-20251001-prompt.json b/data/models/anthropic_claude-haiku-4-5-20251001-prompt.json deleted file mode 100644 index a6bbc8f6079f05640cd210e7024031fa201ffcd2..0000000000000000000000000000000000000000 --- a/data/models/anthropic_claude-haiku-4-5-20251001-prompt.json +++ /dev/null @@ -1,905 +0,0 @@ -{ - "model_info": { - "name": "Claude-Haiku-4-5-20251001 (Prompt)", - "id": "anthropic/claude-haiku-4-5-20251001-prompt", - "developer": "Anthropic", - "additional_details": { - "raw_model_name": "Claude-Haiku-4-5-20251001 (Prompt)", - "organization": "Anthropic", - "license": "Proprietary", - "mode": "Prompt", - "model_link": "https://www.anthropic.com/news/claude-haiku-4-5" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/anthropic/claude-haiku-4-5-20251001-prompt/1775236112.41165", - "retrieved_timestamp": "1775236112.41165", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 87.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 25.26 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 45.13 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 3.75 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 19.96 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 3.77 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 55.42 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 55.67 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 84.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 38.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 44.0 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 52.48 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 66.67 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 49.76 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 56.25 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 16.67 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 1.75 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 1.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 4.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 1.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 19.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 20.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 19.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 2.58 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 2.58 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 1.94 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 3.23 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 31.25 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 95.29 - } - }, - { - "evaluation_result_id": "bfcl.format_sensitivity.max_delta::max_delta", - "evaluation_name": "bfcl.format_sensitivity.max_delta", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.format_sensitivity.max_delta", - "metric_name": "Format sensitivity max delta", - "metric_kind": "difference", - "metric_unit": "percentage_points", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Format Sensitivity Max Delta" - } - }, - "score_details": { - "score": 67.5 - } - }, - { - "evaluation_result_id": "bfcl.format_sensitivity.stddev::stddev", - "evaluation_name": "bfcl.format_sensitivity.stddev", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.format_sensitivity.stddev", - "metric_name": "Format sensitivity standard deviation", - "metric_kind": "difference", - "metric_unit": "percentage_points", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Format Sensitivity Standard Deviation" - } - }, - "score_details": { - "score": 20.07 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/anthropic_claude-haiku-4.5.json b/data/models/anthropic_claude-haiku-4.5.json deleted file mode 100644 index 798a15a6b6dd9c12e031e8fb3d185aa9116f5329..0000000000000000000000000000000000000000 --- a/data/models/anthropic_claude-haiku-4.5.json +++ /dev/null @@ -1,383 +0,0 @@ -{ - "model_info": { - "name": "Claude Haiku 4.5", - "id": "anthropic/claude-haiku-4.5", - "developer": "Anthropic", - "additional_details": { - "agent_name": "Claude Code", - "agent_organization": "Anthropic" - } - }, - "evaluations": [ - { - "evaluation_id": "terminal-bench-2.0/claude-code__claude-haiku-4.5/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2025-11-04", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 27.5, - "uncertainty": { - "standard_error": { - "value": 2.8 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Claude Code\" -m \"Claude Haiku 4.5\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Claude Code\" -m \"Claude Haiku 4.5\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - }, - { - "evaluation_id": "terminal-bench-2.0/openhands__claude-haiku-4.5/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2025-11-02", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 13.9, - "uncertainty": { - "standard_error": { - "value": 2.7 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"OpenHands\" -m \"Claude Haiku 4.5\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"OpenHands\" -m \"Claude Haiku 4.5\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - }, - { - "evaluation_id": "terminal-bench-2.0/mini-swe-agent__claude-haiku-4.5/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2025-11-03", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 29.8, - "uncertainty": { - "standard_error": { - "value": 2.5 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Mini-SWE-Agent\" -m \"Claude Haiku 4.5\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Mini-SWE-Agent\" -m \"Claude Haiku 4.5\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - }, - { - "evaluation_id": "terminal-bench-2.0/terminus-2__claude-haiku-4.5/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2025-10-31", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 28.3, - "uncertainty": { - "standard_error": { - "value": 2.9 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Terminus 2\" -m \"Claude Haiku 4.5\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Terminus 2\" -m \"Claude Haiku 4.5\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - }, - { - "evaluation_id": "terminal-bench-2.0/goose__claude-haiku-4.5/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2025-12-11", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 35.5, - "uncertainty": { - "standard_error": { - "value": 2.9 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Goose\" -m \"Claude Haiku 4.5\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Goose\" -m \"Claude Haiku 4.5\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/anthropic_claude-instant-1.2.json b/data/models/anthropic_claude-instant-1.2.json deleted file mode 100644 index 36bc9771edbd6acd62654b7fd717ed07360a2b3a..0000000000000000000000000000000000000000 --- a/data/models/anthropic_claude-instant-1.2.json +++ /dev/null @@ -1,1901 +0,0 @@ -{ - "model_info": { - "name": "Claude Instant 1.2", - "id": "anthropic/claude-instant-1.2", - "developer": "Anthropic", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_lite/anthropic_claude-instant-1.2/1774096306.427425", - "retrieved_timestamp": "1774096306.427425", - "source_metadata": { - "source_name": "helm_lite", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_lite", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_lite", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.399, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.4998377028714107\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.616, - "details": { - "description": "min=0.616, mean=0.616, max=0.616, sum=0.616 (1)", - "tab": "Accuracy", - "NarrativeQA - Observed inference time (s)": "{\"description\": \"min=1.491, mean=1.491, max=1.491, sum=1.491 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.490500447447871\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=3709.741, mean=3709.741, max=3709.741, sum=3709.741 (1)\", \"tab\": \"General information\", \"score\": \"3709.7408450704224\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=17.149, mean=17.149, max=17.149, sum=17.149 (1)\", \"tab\": \"General information\", \"score\": \"17.149295774647886\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (closed-book)", - "source_data": { - "dataset_name": "NaturalQuestions (closed-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (closed-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.343, - "details": { - "description": "min=0.343, mean=0.343, max=0.343, sum=0.343 (1)", - "tab": "Accuracy", - "NaturalQuestions (open-book) - Observed inference time (s)": "{\"description\": \"min=0.975, mean=0.975, max=0.975, sum=0.975 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.9746438981543135\"}", - "NaturalQuestions (closed-book) - Observed inference time (s)": "{\"description\": \"min=0.674, mean=0.674, max=0.674, sum=0.674 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.6736472499370575\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.964, mean=4.964, max=4.964, sum=4.964 (1)\", \"tab\": \"General information\", \"score\": \"4.964\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.007, mean=0.007, max=0.007, sum=0.007 (1)\", \"tab\": \"General information\", \"score\": \"0.007\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1734.363, mean=1734.363, max=1734.363, sum=1734.363 (1)\", \"tab\": \"General information\", \"score\": \"1734.363\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=8.217, mean=8.217, max=8.217, sum=8.217 (1)\", \"tab\": \"General information\", \"score\": \"8.217\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=189.259, mean=189.259, max=189.259, sum=189.259 (1)\", \"tab\": \"General information\", \"score\": \"189.259\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=5.113, mean=5.113, max=5.113, sum=5.113 (1)\", \"tab\": \"General information\", \"score\": \"5.113\"}" - } - }, - "generation_config": { - "additional_details": { - "mode": "\"closedbook\"" - } - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.844, - "details": { - "description": "min=0.844, mean=0.844, max=0.844, sum=0.844 (1)", - "tab": "Accuracy", - "OpenbookQA - Observed inference time (s)": "{\"description\": \"min=0.597, mean=0.597, max=0.597, sum=0.597 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.596853446483612\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=328.79, mean=328.79, max=328.79, sum=328.79 (1)\", \"tab\": \"General information\", \"score\": \"328.79\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "dataset": "\"openbookqa\"", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.631, - "details": { - "description": "min=0.37, mean=0.631, max=0.9, sum=3.154 (5)", - "tab": "Accuracy", - "MMLU - Observed inference time (s)": "{\"description\": \"min=0.59, mean=0.614, max=0.636, sum=3.069 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.613885824571576\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=435.26, mean=543.747, max=684.596, sum=2718.736 (5)\", \"tab\": \"General information\", \"score\": \"543.747298245614\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"college_chemistry\", \"computer_security\", \"econometrics\", \"us_foreign_policy\"]", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MATH", - "source_data": { - "dataset_name": "MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Equivalent (CoT) on MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.499, - "details": { - "description": "min=0.365, mean=0.499, max=0.704, sum=3.491 (7)", - "tab": "Accuracy", - "MATH - Observed inference time (s)": "{\"description\": \"min=1.247, mean=1.403, max=1.528, sum=9.821 (7)\", \"tab\": \"Efficiency\", \"score\": \"1.4029501960147133\"}", - "MATH - # eval": "{\"description\": \"min=30, mean=62.429, max=135, sum=437 (7)\", \"tab\": \"General information\", \"score\": \"62.42857142857143\"}", - "MATH - # train": "{\"description\": \"min=8, mean=8, max=8, sum=56 (7)\", \"tab\": \"General information\", \"score\": \"8.0\"}", - "MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MATH - # prompt tokens": "{\"description\": \"min=947.259, mean=1361.814, max=2379.808, sum=9532.699 (7)\", \"tab\": \"General information\", \"score\": \"1361.8141219676104\"}", - "MATH - # output tokens": "{\"description\": \"min=54.491, mean=65.956, max=76.513, sum=461.691 (7)\", \"tab\": \"General information\", \"score\": \"65.95586481608514\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"algebra\", \"counting_and_probability\", \"geometry\", \"intermediate_algebra\", \"number_theory\", \"prealgebra\", \"precalculus\"]", - "level": "\"1\"", - "use_official_examples": "\"False\"", - "use_chain_of_thought": "\"True\"" - } - } - }, - { - "evaluation_name": "GSM8K", - "source_data": { - "dataset_name": "GSM8K", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on GSM8K", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.721, - "details": { - "description": "min=0.721, mean=0.721, max=0.721, sum=0.721 (1)", - "tab": "Accuracy", - "GSM8K - Observed inference time (s)": "{\"description\": \"min=1.474, mean=1.474, max=1.474, sum=1.474 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.474282945394516\"}", - "GSM8K - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "GSM8K - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "GSM8K - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GSM8K - # prompt tokens": "{\"description\": \"min=1012.712, mean=1012.712, max=1012.712, sum=1012.712 (1)\", \"tab\": \"General information\", \"score\": \"1012.712\"}", - "GSM8K - # output tokens": "{\"description\": \"min=105.998, mean=105.998, max=105.998, sum=105.998 (1)\", \"tab\": \"General information\", \"score\": \"105.998\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "LegalBench", - "source_data": { - "dataset_name": "LegalBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on LegalBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.586, - "details": { - "description": "min=0.341, mean=0.586, max=0.937, sum=2.931 (5)", - "tab": "Accuracy", - "LegalBench - Observed inference time (s)": "{\"description\": \"min=0.629, mean=0.911, max=1.974, sum=4.555 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.9110085331512334\"}", - "LegalBench - # eval": "{\"description\": \"min=95, mean=409.4, max=1000, sum=2047 (5)\", \"tab\": \"General information\", \"score\": \"409.4\"}", - "LegalBench - # train": "{\"description\": \"min=4, mean=4.798, max=5, sum=23.99 (5)\", \"tab\": \"General information\", \"score\": \"4.797959183673469\"}", - "LegalBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "LegalBench - # prompt tokens": "{\"description\": \"min=280.653, mean=1621.356, max=6484.969, sum=8106.779 (5)\", \"tab\": \"General information\", \"score\": \"1621.3558670820687\"}", - "LegalBench - # output tokens": "{\"description\": \"min=1, mean=1.646, max=2.219, sum=8.23 (5)\", \"tab\": \"General information\", \"score\": \"1.6459798365122615\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"abercrombie\", \"corporate_lobbying\", \"function_of_decision_section\", \"international_citizenship_questions\", \"proa\"]" - } - } - }, - { - "evaluation_name": "MedQA", - "source_data": { - "dataset_name": "MedQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MedQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.559, - "details": { - "description": "min=0.559, mean=0.559, max=0.559, sum=0.559 (1)", - "tab": "Accuracy", - "MedQA - Observed inference time (s)": "{\"description\": \"min=0.763, mean=0.763, max=0.763, sum=0.763 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.7633721221749399\"}", - "MedQA - # eval": "{\"description\": \"min=503, mean=503, max=503, sum=503 (1)\", \"tab\": \"General information\", \"score\": \"503.0\"}", - "MedQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MedQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MedQA - # prompt tokens": "{\"description\": \"min=1092.437, mean=1092.437, max=1092.437, sum=1092.437 (1)\", \"tab\": \"General information\", \"score\": \"1092.4373757455269\"}", - "MedQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WMT 2014", - "source_data": { - "dataset_name": "WMT 2014", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "BLEU-4 on WMT 2014", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.194, - "details": { - "description": "min=0.138, mean=0.194, max=0.24, sum=0.971 (5)", - "tab": "Accuracy", - "WMT 2014 - Observed inference time (s)": "{\"description\": \"min=0.726, mean=0.772, max=0.838, sum=3.859 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.7717107724915095\"}", - "WMT 2014 - # eval": "{\"description\": \"min=503, mean=568.8, max=832, sum=2844 (5)\", \"tab\": \"General information\", \"score\": \"568.8\"}", - "WMT 2014 - # train": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "WMT 2014 - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WMT 2014 - # prompt tokens": "{\"description\": \"min=197.406, mean=218.573, max=240.974, sum=1092.866 (5)\", \"tab\": \"General information\", \"score\": \"218.57322077152472\"}", - "WMT 2014 - # output tokens": "{\"description\": \"min=24.177, mean=25.579, max=26.326, sum=127.893 (5)\", \"tab\": \"General information\", \"score\": \"25.578513056277718\"}" - } - }, - "generation_config": { - "additional_details": { - "language_pair": "[\"cs-en\", \"de-en\", \"fr-en\", \"hi-en\", \"ru-en\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_mmlu/anthropic_claude-instant-1.2/1774096312.00548", - "retrieved_timestamp": "1774096312.00548", - "source_metadata": { - "source_name": "helm_mmlu", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_mmlu", - "evaluation_results": [ - { - "evaluation_name": "MMLU All Subjects", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU All Subjects", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.688, - "details": { - "description": "min=0.333, mean=0.688, max=0.902, sum=78.425 (114)", - "tab": "Accuracy", - "MMLU All Subjects - Observed inference time (s)": "{\"description\": \"min=0.59, mean=0.932, max=1.62, sum=106.285 (114)\", \"tab\": \"Efficiency\", \"score\": \"0.9323255288146379\"}", - "MMLU All Subjects - # eval": "{\"description\": \"min=100, mean=246.351, max=1534, sum=28084 (114)\", \"tab\": \"General information\", \"score\": \"246.35087719298247\"}", - "MMLU All Subjects - # train": "{\"description\": \"min=5, mean=5, max=5, sum=570 (114)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU All Subjects - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU All Subjects - # prompt tokens": "{\"description\": \"min=358.018, mean=703.288, max=2952.576, sum=80174.875 (114)\", \"tab\": \"General information\", \"score\": \"703.2883793758955\"}", - "MMLU All Subjects - # output tokens": "{\"description\": \"min=0.994, mean=1.0, max=1, sum=113.988 (114)\", \"tab\": \"General information\", \"score\": \"0.9998985904066524\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_name": "Abstract Algebra", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Abstract Algebra", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.37, - "details": { - "description": "min=0.37, mean=0.37, max=0.37, sum=0.74 (2)", - "tab": "Accuracy", - "Abstract Algebra - Observed inference time (s)": "{\"description\": \"min=0.59, mean=0.59, max=0.59, sum=1.181 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5904157018661499\"}", - "Abstract Algebra - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Abstract Algebra - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Abstract Algebra - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Abstract Algebra - # prompt tokens": "{\"description\": \"min=435.26, mean=435.26, max=435.26, sum=870.52 (2)\", \"tab\": \"General information\", \"score\": \"435.26\"}", - "Abstract Algebra - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"abstract_algebra\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_abstract_algebra\"" - } - } - }, - { - "evaluation_name": "Anatomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Anatomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.637, - "details": { - "description": "min=0.637, mean=0.637, max=0.637, sum=1.274 (2)", - "tab": "Accuracy", - "Anatomy - Observed inference time (s)": "{\"description\": \"min=0.914, mean=0.914, max=0.914, sum=1.827 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9135703210477476\"}", - "Anatomy - # eval": "{\"description\": \"min=135, mean=135, max=135, sum=270 (2)\", \"tab\": \"General information\", \"score\": \"135.0\"}", - "Anatomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Anatomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Anatomy - # prompt tokens": "{\"description\": \"min=435.8, mean=435.8, max=435.8, sum=871.6 (2)\", \"tab\": \"General information\", \"score\": \"435.8\"}", - "Anatomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"anatomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_anatomy\"" - } - } - }, - { - "evaluation_name": "College Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on College Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.49, - "details": { - "description": "min=0.49, mean=0.49, max=0.49, sum=0.98 (2)", - "tab": "Accuracy", - "College Chemistry - Observed inference time (s)": "{\"description\": \"min=0.636, mean=0.636, max=0.636, sum=1.272 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6360281848907471\"}", - "College Biology - Observed inference time (s)": "{\"description\": \"min=1.016, mean=1.016, max=1.016, sum=2.033 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.0163518455293443\"}", - "College Computer Science - Observed inference time (s)": "{\"description\": \"min=1.153, mean=1.153, max=1.153, sum=2.306 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.1530575346946716\"}", - "College Mathematics - Observed inference time (s)": "{\"description\": \"min=1.157, mean=1.157, max=1.157, sum=2.314 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.1569927215576172\"}", - "College Medicine - Observed inference time (s)": "{\"description\": \"min=1.086, mean=1.086, max=1.086, sum=2.173 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.0863008636959715\"}", - "College Physics - Observed inference time (s)": "{\"description\": \"min=0.938, mean=0.938, max=0.938, sum=1.875 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9376059443342919\"}", - "College Chemistry - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Chemistry - # prompt tokens": "{\"description\": \"min=615.01, mean=615.01, max=615.01, sum=1230.02 (2)\", \"tab\": \"General information\", \"score\": \"615.01\"}", - "College Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Biology - # eval": "{\"description\": \"min=144, mean=144, max=144, sum=288 (2)\", \"tab\": \"General information\", \"score\": \"144.0\"}", - "College Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # prompt tokens": "{\"description\": \"min=555.347, mean=555.347, max=555.347, sum=1110.694 (2)\", \"tab\": \"General information\", \"score\": \"555.3472222222222\"}", - "College Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # prompt tokens": "{\"description\": \"min=903.24, mean=903.24, max=903.24, sum=1806.48 (2)\", \"tab\": \"General information\", \"score\": \"903.24\"}", - "College Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Mathematics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # prompt tokens": "{\"description\": \"min=669.19, mean=669.19, max=669.19, sum=1338.38 (2)\", \"tab\": \"General information\", \"score\": \"669.19\"}", - "College Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Medicine - # eval": "{\"description\": \"min=173, mean=173, max=173, sum=346 (2)\", \"tab\": \"General information\", \"score\": \"173.0\"}", - "College Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # prompt tokens": "{\"description\": \"min=605.63, mean=605.63, max=605.63, sum=1211.26 (2)\", \"tab\": \"General information\", \"score\": \"605.6300578034682\"}", - "College Medicine - # output tokens": "{\"description\": \"min=0.994, mean=0.994, max=0.994, sum=1.988 (2)\", \"tab\": \"General information\", \"score\": \"0.9942196531791907\"}", - "College Physics - # eval": "{\"description\": \"min=102, mean=102, max=102, sum=204 (2)\", \"tab\": \"General information\", \"score\": \"102.0\"}", - "College Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # prompt tokens": "{\"description\": \"min=554.48, mean=554.48, max=554.48, sum=1108.961 (2)\", \"tab\": \"General information\", \"score\": \"554.4803921568628\"}", - "College Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"college_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_college_physics\"" - } - } - }, - { - "evaluation_name": "Computer Security", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Computer Security", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.76, - "details": { - "description": "min=0.76, mean=0.76, max=0.76, sum=1.52 (2)", - "tab": "Accuracy", - "Computer Security - Observed inference time (s)": "{\"description\": \"min=0.597, mean=0.597, max=0.597, sum=1.194 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.596819703578949\"}", - "Computer Security - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Computer Security - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Computer Security - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Computer Security - # prompt tokens": "{\"description\": \"min=463.62, mean=463.62, max=463.62, sum=927.24 (2)\", \"tab\": \"General information\", \"score\": \"463.62\"}", - "Computer Security - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"computer_security\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_computer_security\"" - } - } - }, - { - "evaluation_name": "Econometrics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Econometrics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.614, - "details": { - "description": "min=0.614, mean=0.614, max=0.614, sum=1.228 (2)", - "tab": "Accuracy", - "Econometrics - Observed inference time (s)": "{\"description\": \"min=0.633, mean=0.633, max=0.633, sum=1.267 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6333246440218206\"}", - "Econometrics - # eval": "{\"description\": \"min=114, mean=114, max=114, sum=228 (2)\", \"tab\": \"General information\", \"score\": \"114.0\"}", - "Econometrics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Econometrics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Econometrics - # prompt tokens": "{\"description\": \"min=684.596, mean=684.596, max=684.596, sum=1369.193 (2)\", \"tab\": \"General information\", \"score\": \"684.5964912280701\"}", - "Econometrics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"econometrics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_econometrics\"" - } - } - }, - { - "evaluation_name": "Global Facts", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Global Facts", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.38, - "details": { - "description": "min=0.38, mean=0.38, max=0.38, sum=0.76 (2)", - "tab": "Accuracy", - "Global Facts - Observed inference time (s)": "{\"description\": \"min=0.975, mean=0.975, max=0.975, sum=1.949 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9746571969985962\"}", - "Global Facts - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Global Facts - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Global Facts - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Global Facts - # prompt tokens": "{\"description\": \"min=476.61, mean=476.61, max=476.61, sum=953.22 (2)\", \"tab\": \"General information\", \"score\": \"476.61\"}", - "Global Facts - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"global_facts\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_global_facts\"" - } - } - }, - { - "evaluation_name": "Jurisprudence", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Jurisprudence", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.833, - "details": { - "description": "min=0.833, mean=0.833, max=0.833, sum=1.667 (2)", - "tab": "Accuracy", - "Jurisprudence - Observed inference time (s)": "{\"description\": \"min=0.811, mean=0.811, max=0.811, sum=1.621 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8107206269546792\"}", - "Jurisprudence - # eval": "{\"description\": \"min=108, mean=108, max=108, sum=216 (2)\", \"tab\": \"General information\", \"score\": \"108.0\"}", - "Jurisprudence - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Jurisprudence - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Jurisprudence - # prompt tokens": "{\"description\": \"min=496.426, mean=496.426, max=496.426, sum=992.852 (2)\", \"tab\": \"General information\", \"score\": \"496.4259259259259\"}", - "Jurisprudence - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"jurisprudence\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_jurisprudence\"" - } - } - }, - { - "evaluation_name": "Philosophy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Philosophy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.756, - "details": { - "description": "min=0.756, mean=0.756, max=0.756, sum=1.511 (2)", - "tab": "Accuracy", - "Philosophy - Observed inference time (s)": "{\"description\": \"min=0.832, mean=0.832, max=0.832, sum=1.664 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8319868075502647\"}", - "Philosophy - # eval": "{\"description\": \"min=311, mean=311, max=311, sum=622 (2)\", \"tab\": \"General information\", \"score\": \"311.0\"}", - "Philosophy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Philosophy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Philosophy - # prompt tokens": "{\"description\": \"min=424.965, mean=424.965, max=424.965, sum=849.929 (2)\", \"tab\": \"General information\", \"score\": \"424.9646302250804\"}", - "Philosophy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"philosophy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_philosophy\"" - } - } - }, - { - "evaluation_name": "Professional Psychology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Professional Psychology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.724, - "details": { - "description": "min=0.724, mean=0.724, max=0.724, sum=1.448 (2)", - "tab": "Accuracy", - "Professional Medicine - Observed inference time (s)": "{\"description\": \"min=1.073, mean=1.073, max=1.073, sum=2.146 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.072824116138851\"}", - "Professional Accounting - Observed inference time (s)": "{\"description\": \"min=0.895, mean=0.895, max=0.895, sum=1.79 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8950984232814599\"}", - "Professional Law - Observed inference time (s)": "{\"description\": \"min=1.058, mean=1.058, max=1.058, sum=2.117 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.0584386131754133\"}", - "Professional Psychology - Observed inference time (s)": "{\"description\": \"min=0.859, mean=0.859, max=0.859, sum=1.718 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8591087651408575\"}", - "Professional Medicine - # eval": "{\"description\": \"min=272, mean=272, max=272, sum=544 (2)\", \"tab\": \"General information\", \"score\": \"272.0\"}", - "Professional Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Medicine - # prompt tokens": "{\"description\": \"min=1188.537, mean=1188.537, max=1188.537, sum=2377.074 (2)\", \"tab\": \"General information\", \"score\": \"1188.5367647058824\"}", - "Professional Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Accounting - # eval": "{\"description\": \"min=282, mean=282, max=282, sum=564 (2)\", \"tab\": \"General information\", \"score\": \"282.0\"}", - "Professional Accounting - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Accounting - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # prompt tokens": "{\"description\": \"min=730.422, mean=730.422, max=730.422, sum=1460.844 (2)\", \"tab\": \"General information\", \"score\": \"730.4219858156029\"}", - "Professional Accounting - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Law - # eval": "{\"description\": \"min=1534, mean=1534, max=1534, sum=3068 (2)\", \"tab\": \"General information\", \"score\": \"1534.0\"}", - "Professional Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # prompt tokens": "{\"description\": \"min=1766.16, mean=1766.16, max=1766.16, sum=3532.321 (2)\", \"tab\": \"General information\", \"score\": \"1766.16036505867\"}", - "Professional Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Psychology - # eval": "{\"description\": \"min=612, mean=612, max=612, sum=1224 (2)\", \"tab\": \"General information\", \"score\": \"612.0\"}", - "Professional Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # prompt tokens": "{\"description\": \"min=668.168, mean=668.168, max=668.168, sum=1336.337 (2)\", \"tab\": \"General information\", \"score\": \"668.1683006535948\"}", - "Professional Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"professional_psychology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_professional_psychology\"" - } - } - }, - { - "evaluation_name": "Us Foreign Policy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Us Foreign Policy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9, - "details": { - "description": "min=0.9, mean=0.9, max=0.9, sum=1.8 (2)", - "tab": "Accuracy", - "Us Foreign Policy - Observed inference time (s)": "{\"description\": \"min=0.613, mean=0.613, max=0.613, sum=1.226 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6128408885002137\"}", - "Us Foreign Policy - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Us Foreign Policy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Us Foreign Policy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Us Foreign Policy - # prompt tokens": "{\"description\": \"min=520.25, mean=520.25, max=520.25, sum=1040.5 (2)\", \"tab\": \"General information\", \"score\": \"520.25\"}", - "Us Foreign Policy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"us_foreign_policy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_us_foreign_policy\"" - } - } - }, - { - "evaluation_name": "Astronomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Astronomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.743, - "details": { - "description": "min=0.743, mean=0.743, max=0.743, sum=1.487 (2)", - "tab": "Accuracy", - "Astronomy - Observed inference time (s)": "{\"description\": \"min=1.124, mean=1.124, max=1.124, sum=2.248 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.123885358634748\"}", - "Astronomy - # eval": "{\"description\": \"min=152, mean=152, max=152, sum=304 (2)\", \"tab\": \"General information\", \"score\": \"152.0\"}", - "Astronomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Astronomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Astronomy - # prompt tokens": "{\"description\": \"min=669.493, mean=669.493, max=669.493, sum=1338.987 (2)\", \"tab\": \"General information\", \"score\": \"669.4934210526316\"}", - "Astronomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"astronomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_astronomy\"" - } - } - }, - { - "evaluation_name": "Business Ethics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Business Ethics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7, - "details": { - "description": "min=0.7, mean=0.7, max=0.7, sum=1.4 (2)", - "tab": "Accuracy", - "Business Ethics - Observed inference time (s)": "{\"description\": \"min=1.102, mean=1.102, max=1.102, sum=2.204 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.101954047679901\"}", - "Business Ethics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Business Ethics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Business Ethics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Business Ethics - # prompt tokens": "{\"description\": \"min=665.02, mean=665.02, max=665.02, sum=1330.04 (2)\", \"tab\": \"General information\", \"score\": \"665.02\"}", - "Business Ethics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"business_ethics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_business_ethics\"" - } - } - }, - { - "evaluation_name": "Clinical Knowledge", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Clinical Knowledge", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.709, - "details": { - "description": "min=0.709, mean=0.709, max=0.709, sum=1.419 (2)", - "tab": "Accuracy", - "Clinical Knowledge - Observed inference time (s)": "{\"description\": \"min=0.899, mean=0.899, max=0.899, sum=1.799 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8994299870616985\"}", - "Clinical Knowledge - # eval": "{\"description\": \"min=265, mean=265, max=265, sum=530 (2)\", \"tab\": \"General information\", \"score\": \"265.0\"}", - "Clinical Knowledge - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Clinical Knowledge - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Clinical Knowledge - # prompt tokens": "{\"description\": \"min=494.457, mean=494.457, max=494.457, sum=988.913 (2)\", \"tab\": \"General information\", \"score\": \"494.4566037735849\"}", - "Clinical Knowledge - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"clinical_knowledge\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_clinical_knowledge\"" - } - } - }, - { - "evaluation_name": "Conceptual Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Conceptual Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.613, - "details": { - "description": "min=0.613, mean=0.613, max=0.613, sum=1.226 (2)", - "tab": "Accuracy", - "Conceptual Physics - Observed inference time (s)": "{\"description\": \"min=0.773, mean=0.773, max=0.773, sum=1.546 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7728059119366585\"}", - "Conceptual Physics - # eval": "{\"description\": \"min=235, mean=235, max=235, sum=470 (2)\", \"tab\": \"General information\", \"score\": \"235.0\"}", - "Conceptual Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Conceptual Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Conceptual Physics - # prompt tokens": "{\"description\": \"min=388.536, mean=388.536, max=388.536, sum=777.072 (2)\", \"tab\": \"General information\", \"score\": \"388.53617021276597\"}", - "Conceptual Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"conceptual_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_conceptual_physics\"" - } - } - }, - { - "evaluation_name": "Electrical Engineering", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Electrical Engineering", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.641, - "details": { - "description": "min=0.641, mean=0.641, max=0.641, sum=1.283 (2)", - "tab": "Accuracy", - "Electrical Engineering - Observed inference time (s)": "{\"description\": \"min=0.932, mean=0.932, max=0.932, sum=1.865 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9323823583537134\"}", - "Electrical Engineering - # eval": "{\"description\": \"min=145, mean=145, max=145, sum=290 (2)\", \"tab\": \"General information\", \"score\": \"145.0\"}", - "Electrical Engineering - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Electrical Engineering - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Electrical Engineering - # prompt tokens": "{\"description\": \"min=502.041, mean=502.041, max=502.041, sum=1004.083 (2)\", \"tab\": \"General information\", \"score\": \"502.04137931034484\"}", - "Electrical Engineering - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"electrical_engineering\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_electrical_engineering\"" - } - } - }, - { - "evaluation_name": "Elementary Mathematics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Elementary Mathematics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.45, - "details": { - "description": "min=0.45, mean=0.45, max=0.45, sum=0.899 (2)", - "tab": "Accuracy", - "Elementary Mathematics - Observed inference time (s)": "{\"description\": \"min=0.945, mean=0.945, max=0.945, sum=1.891 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.945274135423085\"}", - "Elementary Mathematics - # eval": "{\"description\": \"min=378, mean=378, max=378, sum=756 (2)\", \"tab\": \"General information\", \"score\": \"378.0\"}", - "Elementary Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Elementary Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Elementary Mathematics - # prompt tokens": "{\"description\": \"min=576.066, mean=576.066, max=576.066, sum=1152.132 (2)\", \"tab\": \"General information\", \"score\": \"576.0661375661375\"}", - "Elementary Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"elementary_mathematics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_elementary_mathematics\"" - } - } - }, - { - "evaluation_name": "Formal Logic", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Formal Logic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.444, - "details": { - "description": "min=0.444, mean=0.444, max=0.444, sum=0.889 (2)", - "tab": "Accuracy", - "Formal Logic - Observed inference time (s)": "{\"description\": \"min=1.151, mean=1.151, max=1.151, sum=2.302 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.1508805732878427\"}", - "Formal Logic - # eval": "{\"description\": \"min=126, mean=126, max=126, sum=252 (2)\", \"tab\": \"General information\", \"score\": \"126.0\"}", - "Formal Logic - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Formal Logic - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Formal Logic - # prompt tokens": "{\"description\": \"min=711.746, mean=711.746, max=711.746, sum=1423.492 (2)\", \"tab\": \"General information\", \"score\": \"711.7460317460317\"}", - "Formal Logic - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"formal_logic\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_formal_logic\"" - } - } - }, - { - "evaluation_name": "High School World History", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on High School World History", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.878, - "details": { - "description": "min=0.878, mean=0.878, max=0.878, sum=1.755 (2)", - "tab": "Accuracy", - "High School Biology - Observed inference time (s)": "{\"description\": \"min=0.998, mean=0.998, max=0.998, sum=1.996 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9978926274084275\"}", - "High School Chemistry - Observed inference time (s)": "{\"description\": \"min=0.934, mean=0.934, max=0.934, sum=1.868 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9337695701956161\"}", - "High School Computer Science - Observed inference time (s)": "{\"description\": \"min=1.046, mean=1.046, max=1.046, sum=2.091 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.0455269980430604\"}", - "High School European History - Observed inference time (s)": "{\"description\": \"min=1.62, mean=1.62, max=1.62, sum=3.241 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.6203449783903179\"}", - "High School Geography - Observed inference time (s)": "{\"description\": \"min=0.877, mean=0.877, max=0.877, sum=1.754 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.876823568584943\"}", - "High School Government And Politics - Observed inference time (s)": "{\"description\": \"min=1.037, mean=1.037, max=1.037, sum=2.074 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.0370552873364385\"}", - "High School Macroeconomics - Observed inference time (s)": "{\"description\": \"min=0.925, mean=0.925, max=0.925, sum=1.849 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9246660091938117\"}", - "High School Mathematics - Observed inference time (s)": "{\"description\": \"min=1.014, mean=1.014, max=1.014, sum=2.027 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.013659605273494\"}", - "High School Microeconomics - Observed inference time (s)": "{\"description\": \"min=1.163, mean=1.163, max=1.163, sum=2.325 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.1627413104562199\"}", - "High School Physics - Observed inference time (s)": "{\"description\": \"min=0.963, mean=0.963, max=0.963, sum=1.925 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9627095689836717\"}", - "High School Psychology - Observed inference time (s)": "{\"description\": \"min=0.947, mean=0.947, max=0.947, sum=1.894 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9471190351958668\"}", - "High School Statistics - Observed inference time (s)": "{\"description\": \"min=0.929, mean=0.929, max=0.929, sum=1.857 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9286887921668865\"}", - "High School US History - Observed inference time (s)": "{\"description\": \"min=1.383, mean=1.383, max=1.383, sum=2.766 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.3831783030547349\"}", - "High School World History - Observed inference time (s)": "{\"description\": \"min=1.246, mean=1.246, max=1.246, sum=2.492 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.2459266769232127\"}", - "High School Biology - # eval": "{\"description\": \"min=310, mean=310, max=310, sum=620 (2)\", \"tab\": \"General information\", \"score\": \"310.0\"}", - "High School Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Biology - # prompt tokens": "{\"description\": \"min=599.577, mean=599.577, max=599.577, sum=1199.155 (2)\", \"tab\": \"General information\", \"score\": \"599.5774193548388\"}", - "High School Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Chemistry - # eval": "{\"description\": \"min=203, mean=203, max=203, sum=406 (2)\", \"tab\": \"General information\", \"score\": \"203.0\"}", - "High School Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # prompt tokens": "{\"description\": \"min=562.921, mean=562.921, max=562.921, sum=1125.842 (2)\", \"tab\": \"General information\", \"score\": \"562.9211822660099\"}", - "High School Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "High School Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # prompt tokens": "{\"description\": \"min=947.4, mean=947.4, max=947.4, sum=1894.8 (2)\", \"tab\": \"General information\", \"score\": \"947.4\"}", - "High School Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School European History - # eval": "{\"description\": \"min=165, mean=165, max=165, sum=330 (2)\", \"tab\": \"General information\", \"score\": \"165.0\"}", - "High School European History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School European History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # prompt tokens": "{\"description\": \"min=2952.576, mean=2952.576, max=2952.576, sum=5905.152 (2)\", \"tab\": \"General information\", \"score\": \"2952.5757575757575\"}", - "High School European History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Geography - # eval": "{\"description\": \"min=198, mean=198, max=198, sum=396 (2)\", \"tab\": \"General information\", \"score\": \"198.0\"}", - "High School Geography - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Geography - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # prompt tokens": "{\"description\": \"min=477.268, mean=477.268, max=477.268, sum=954.535 (2)\", \"tab\": \"General information\", \"score\": \"477.2676767676768\"}", - "High School Geography - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Government And Politics - # eval": "{\"description\": \"min=193, mean=193, max=193, sum=386 (2)\", \"tab\": \"General information\", \"score\": \"193.0\"}", - "High School Government And Politics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Government And Politics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # prompt tokens": "{\"description\": \"min=556.104, mean=556.104, max=556.104, sum=1112.207 (2)\", \"tab\": \"General information\", \"score\": \"556.1036269430052\"}", - "High School Government And Politics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Macroeconomics - # eval": "{\"description\": \"min=390, mean=390, max=390, sum=780 (2)\", \"tab\": \"General information\", \"score\": \"390.0\"}", - "High School Macroeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Macroeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # prompt tokens": "{\"description\": \"min=471.036, mean=471.036, max=471.036, sum=942.072 (2)\", \"tab\": \"General information\", \"score\": \"471.0358974358974\"}", - "High School Macroeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Mathematics - # eval": "{\"description\": \"min=270, mean=270, max=270, sum=540 (2)\", \"tab\": \"General information\", \"score\": \"270.0\"}", - "High School Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # prompt tokens": "{\"description\": \"min=584.881, mean=584.881, max=584.881, sum=1169.763 (2)\", \"tab\": \"General information\", \"score\": \"584.8814814814815\"}", - "High School Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Microeconomics - # eval": "{\"description\": \"min=238, mean=238, max=238, sum=476 (2)\", \"tab\": \"General information\", \"score\": \"238.0\"}", - "High School Microeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Microeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # prompt tokens": "{\"description\": \"min=485.513, mean=485.513, max=485.513, sum=971.025 (2)\", \"tab\": \"General information\", \"score\": \"485.5126050420168\"}", - "High School Microeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Physics - # eval": "{\"description\": \"min=151, mean=151, max=151, sum=302 (2)\", \"tab\": \"General information\", \"score\": \"151.0\"}", - "High School Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # prompt tokens": "{\"description\": \"min=623.841, mean=623.841, max=623.841, sum=1247.682 (2)\", \"tab\": \"General information\", \"score\": \"623.841059602649\"}", - "High School Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Psychology - # eval": "{\"description\": \"min=545, mean=545, max=545, sum=1090 (2)\", \"tab\": \"General information\", \"score\": \"545.0\"}", - "High School Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # prompt tokens": "{\"description\": \"min=586.42, mean=586.42, max=586.42, sum=1172.84 (2)\", \"tab\": \"General information\", \"score\": \"586.4201834862386\"}", - "High School Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Statistics - # eval": "{\"description\": \"min=216, mean=216, max=216, sum=432 (2)\", \"tab\": \"General information\", \"score\": \"216.0\"}", - "High School Statistics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Statistics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # prompt tokens": "{\"description\": \"min=871.963, mean=871.963, max=871.963, sum=1743.926 (2)\", \"tab\": \"General information\", \"score\": \"871.9629629629629\"}", - "High School Statistics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School US History - # eval": "{\"description\": \"min=204, mean=204, max=204, sum=408 (2)\", \"tab\": \"General information\", \"score\": \"204.0\"}", - "High School US History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School US History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # prompt tokens": "{\"description\": \"min=2353.49, mean=2353.49, max=2353.49, sum=4706.98 (2)\", \"tab\": \"General information\", \"score\": \"2353.4901960784314\"}", - "High School US History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School World History - # eval": "{\"description\": \"min=237, mean=237, max=237, sum=474 (2)\", \"tab\": \"General information\", \"score\": \"237.0\"}", - "High School World History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School World History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # prompt tokens": "{\"description\": \"min=1540.932, mean=1540.932, max=1540.932, sum=3081.865 (2)\", \"tab\": \"General information\", \"score\": \"1540.9324894514768\"}", - "High School World History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"high_school_world_history\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_high_school_world_history\"" - } - } - }, - { - "evaluation_name": "Human Sexuality", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Human Sexuality", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.794, - "details": { - "description": "min=0.794, mean=0.794, max=0.794, sum=1.588 (2)", - "tab": "Accuracy", - "Human Aging - Observed inference time (s)": "{\"description\": \"min=0.782, mean=0.782, max=0.782, sum=1.563 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7815119557316528\"}", - "Human Sexuality - Observed inference time (s)": "{\"description\": \"min=0.763, mean=0.763, max=0.763, sum=1.526 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7630931584889652\"}", - "Human Aging - # eval": "{\"description\": \"min=223, mean=223, max=223, sum=446 (2)\", \"tab\": \"General information\", \"score\": \"223.0\"}", - "Human Aging - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Aging - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Aging - # prompt tokens": "{\"description\": \"min=400.955, mean=400.955, max=400.955, sum=801.91 (2)\", \"tab\": \"General information\", \"score\": \"400.95515695067263\"}", - "Human Aging - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Human Sexuality - # eval": "{\"description\": \"min=131, mean=131, max=131, sum=262 (2)\", \"tab\": \"General information\", \"score\": \"131.0\"}", - "Human Sexuality - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Sexuality - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # prompt tokens": "{\"description\": \"min=436.496, mean=436.496, max=436.496, sum=872.992 (2)\", \"tab\": \"General information\", \"score\": \"436.4961832061069\"}", - "Human Sexuality - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"human_sexuality\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_human_sexuality\"" - } - } - }, - { - "evaluation_name": "International Law", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on International Law", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.851, - "details": { - "description": "min=0.851, mean=0.851, max=0.851, sum=1.702 (2)", - "tab": "Accuracy", - "International Law - Observed inference time (s)": "{\"description\": \"min=0.888, mean=0.888, max=0.888, sum=1.775 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8875030958948057\"}", - "International Law - # eval": "{\"description\": \"min=121, mean=121, max=121, sum=242 (2)\", \"tab\": \"General information\", \"score\": \"121.0\"}", - "International Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "International Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "International Law - # prompt tokens": "{\"description\": \"min=729.165, mean=729.165, max=729.165, sum=1458.331 (2)\", \"tab\": \"General information\", \"score\": \"729.1652892561983\"}", - "International Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"international_law\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_international_law\"" - } - } - }, - { - "evaluation_name": "Logical Fallacies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Logical Fallacies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.81, - "details": { - "description": "min=0.81, mean=0.81, max=0.81, sum=1.62 (2)", - "tab": "Accuracy", - "Logical Fallacies - Observed inference time (s)": "{\"description\": \"min=0.939, mean=0.939, max=0.939, sum=1.878 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9389484660025754\"}", - "Logical Fallacies - # eval": "{\"description\": \"min=163, mean=163, max=163, sum=326 (2)\", \"tab\": \"General information\", \"score\": \"163.0\"}", - "Logical Fallacies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Logical Fallacies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Logical Fallacies - # prompt tokens": "{\"description\": \"min=535.276, mean=535.276, max=535.276, sum=1070.552 (2)\", \"tab\": \"General information\", \"score\": \"535.2760736196319\"}", - "Logical Fallacies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"logical_fallacies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_logical_fallacies\"" - } - } - }, - { - "evaluation_name": "Machine Learning", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Machine Learning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.67, - "details": { - "description": "min=0.67, mean=0.67, max=0.67, sum=1.339 (2)", - "tab": "Accuracy", - "Machine Learning - Observed inference time (s)": "{\"description\": \"min=0.887, mean=0.887, max=0.887, sum=1.774 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8872403161866325\"}", - "Machine Learning - # eval": "{\"description\": \"min=112, mean=112, max=112, sum=224 (2)\", \"tab\": \"General information\", \"score\": \"112.0\"}", - "Machine Learning - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Machine Learning - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Machine Learning - # prompt tokens": "{\"description\": \"min=741.518, mean=741.518, max=741.518, sum=1483.036 (2)\", \"tab\": \"General information\", \"score\": \"741.5178571428571\"}", - "Machine Learning - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"machine_learning\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_machine_learning\"" - } - } - }, - { - "evaluation_name": "Management", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Management", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.835, - "details": { - "description": "min=0.835, mean=0.835, max=0.835, sum=1.67 (2)", - "tab": "Accuracy", - "Management - Observed inference time (s)": "{\"description\": \"min=0.931, mean=0.931, max=0.931, sum=1.862 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9309975244466541\"}", - "Management - # eval": "{\"description\": \"min=103, mean=103, max=103, sum=206 (2)\", \"tab\": \"General information\", \"score\": \"103.0\"}", - "Management - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Management - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Management - # prompt tokens": "{\"description\": \"min=366.282, mean=366.282, max=366.282, sum=732.563 (2)\", \"tab\": \"General information\", \"score\": \"366.28155339805824\"}", - "Management - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"management\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_management\"" - } - } - }, - { - "evaluation_name": "Marketing", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Marketing", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.885, - "details": { - "description": "min=0.885, mean=0.885, max=0.885, sum=1.769 (2)", - "tab": "Accuracy", - "Marketing - Observed inference time (s)": "{\"description\": \"min=0.961, mean=0.961, max=0.961, sum=1.923 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9613573286268446\"}", - "Marketing - # eval": "{\"description\": \"min=234, mean=234, max=234, sum=468 (2)\", \"tab\": \"General information\", \"score\": \"234.0\"}", - "Marketing - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Marketing - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Marketing - # prompt tokens": "{\"description\": \"min=513.064, mean=513.064, max=513.064, sum=1026.128 (2)\", \"tab\": \"General information\", \"score\": \"513.0641025641025\"}", - "Marketing - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"marketing\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_marketing\"" - } - } - }, - { - "evaluation_name": "Medical Genetics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Medical Genetics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.71, - "details": { - "description": "min=0.71, mean=0.71, max=0.71, sum=1.42 (2)", - "tab": "Accuracy", - "Medical Genetics - Observed inference time (s)": "{\"description\": \"min=0.81, mean=0.81, max=0.81, sum=1.621 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8103219223022461\"}", - "Medical Genetics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Medical Genetics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Medical Genetics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Medical Genetics - # prompt tokens": "{\"description\": \"min=419.88, mean=419.88, max=419.88, sum=839.76 (2)\", \"tab\": \"General information\", \"score\": \"419.88\"}", - "Medical Genetics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"medical_genetics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_medical_genetics\"" - } - } - }, - { - "evaluation_name": "Miscellaneous", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Miscellaneous", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.828, - "details": { - "description": "min=0.828, mean=0.828, max=0.828, sum=1.655 (2)", - "tab": "Accuracy", - "Miscellaneous - Observed inference time (s)": "{\"description\": \"min=0.826, mean=0.826, max=0.826, sum=1.652 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8259343528503964\"}", - "Miscellaneous - # eval": "{\"description\": \"min=783, mean=783, max=783, sum=1566 (2)\", \"tab\": \"General information\", \"score\": \"783.0\"}", - "Miscellaneous - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Miscellaneous - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Miscellaneous - # prompt tokens": "{\"description\": \"min=393.628, mean=393.628, max=393.628, sum=787.257 (2)\", \"tab\": \"General information\", \"score\": \"393.62835249042143\"}", - "Miscellaneous - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"miscellaneous\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_miscellaneous\"" - } - } - }, - { - "evaluation_name": "Moral Scenarios", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Moral Scenarios", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.488, - "details": { - "description": "min=0.488, mean=0.488, max=0.488, sum=0.977 (2)", - "tab": "Accuracy", - "Moral Disputes - Observed inference time (s)": "{\"description\": \"min=0.938, mean=0.938, max=0.938, sum=1.876 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.937887375065357\"}", - "Moral Scenarios - Observed inference time (s)": "{\"description\": \"min=0.885, mean=0.885, max=0.885, sum=1.77 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8848049091893201\"}", - "Moral Disputes - # eval": "{\"description\": \"min=346, mean=346, max=346, sum=692 (2)\", \"tab\": \"General information\", \"score\": \"346.0\"}", - "Moral Disputes - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Disputes - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Disputes - # prompt tokens": "{\"description\": \"min=576.789, mean=576.789, max=576.789, sum=1153.578 (2)\", \"tab\": \"General information\", \"score\": \"576.7890173410404\"}", - "Moral Disputes - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Moral Scenarios - # eval": "{\"description\": \"min=895, mean=895, max=895, sum=1790 (2)\", \"tab\": \"General information\", \"score\": \"895.0\"}", - "Moral Scenarios - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Scenarios - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # prompt tokens": "{\"description\": \"min=741.949, mean=741.949, max=741.949, sum=1483.897 (2)\", \"tab\": \"General information\", \"score\": \"741.9486033519553\"}", - "Moral Scenarios - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"moral_scenarios\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_moral_scenarios\"" - } - } - }, - { - "evaluation_name": "Nutrition", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Nutrition", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.735, - "details": { - "description": "min=0.735, mean=0.735, max=0.735, sum=1.471 (2)", - "tab": "Accuracy", - "Nutrition - Observed inference time (s)": "{\"description\": \"min=0.881, mean=0.881, max=0.881, sum=1.761 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8806839573617075\"}", - "Nutrition - # eval": "{\"description\": \"min=306, mean=306, max=306, sum=612 (2)\", \"tab\": \"General information\", \"score\": \"306.0\"}", - "Nutrition - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Nutrition - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Nutrition - # prompt tokens": "{\"description\": \"min=682.065, mean=682.065, max=682.065, sum=1364.131 (2)\", \"tab\": \"General information\", \"score\": \"682.0653594771242\"}", - "Nutrition - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"nutrition\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_nutrition\"" - } - } - }, - { - "evaluation_name": "Prehistory", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Prehistory", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.762, - "details": { - "description": "min=0.762, mean=0.762, max=0.762, sum=1.525 (2)", - "tab": "Accuracy", - "Prehistory - Observed inference time (s)": "{\"description\": \"min=0.819, mean=0.819, max=0.819, sum=1.638 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8192079758938448\"}", - "Prehistory - # eval": "{\"description\": \"min=324, mean=324, max=324, sum=648 (2)\", \"tab\": \"General information\", \"score\": \"324.0\"}", - "Prehistory - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Prehistory - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Prehistory - # prompt tokens": "{\"description\": \"min=610.639, mean=610.639, max=610.639, sum=1221.278 (2)\", \"tab\": \"General information\", \"score\": \"610.6388888888889\"}", - "Prehistory - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"prehistory\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_prehistory\"" - } - } - }, - { - "evaluation_name": "Public Relations", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Public Relations", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.627, - "details": { - "description": "min=0.627, mean=0.627, max=0.627, sum=1.255 (2)", - "tab": "Accuracy", - "Public Relations - Observed inference time (s)": "{\"description\": \"min=0.736, mean=0.736, max=0.736, sum=1.471 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.735536317391829\"}", - "Public Relations - # eval": "{\"description\": \"min=110, mean=110, max=110, sum=220 (2)\", \"tab\": \"General information\", \"score\": \"110.0\"}", - "Public Relations - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Public Relations - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Public Relations - # prompt tokens": "{\"description\": \"min=497.991, mean=497.991, max=497.991, sum=995.982 (2)\", \"tab\": \"General information\", \"score\": \"497.9909090909091\"}", - "Public Relations - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"public_relations\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_public_relations\"" - } - } - }, - { - "evaluation_name": "Security Studies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Security Studies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.784, - "details": { - "description": "min=0.784, mean=0.784, max=0.784, sum=1.567 (2)", - "tab": "Accuracy", - "Security Studies - Observed inference time (s)": "{\"description\": \"min=0.949, mean=0.949, max=0.949, sum=1.898 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9487942345288335\"}", - "Security Studies - # eval": "{\"description\": \"min=245, mean=245, max=245, sum=490 (2)\", \"tab\": \"General information\", \"score\": \"245.0\"}", - "Security Studies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Security Studies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Security Studies - # prompt tokens": "{\"description\": \"min=1308.804, mean=1308.804, max=1308.804, sum=2617.608 (2)\", \"tab\": \"General information\", \"score\": \"1308.8040816326532\"}", - "Security Studies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"security_studies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_security_studies\"" - } - } - }, - { - "evaluation_name": "Sociology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Sociology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.841, - "details": { - "description": "min=0.841, mean=0.841, max=0.841, sum=1.682 (2)", - "tab": "Accuracy", - "Sociology - Observed inference time (s)": "{\"description\": \"min=0.843, mean=0.843, max=0.843, sum=1.687 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8433953909138542\"}", - "Sociology - # eval": "{\"description\": \"min=201, mean=201, max=201, sum=402 (2)\", \"tab\": \"General information\", \"score\": \"201.0\"}", - "Sociology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Sociology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Sociology - # prompt tokens": "{\"description\": \"min=532.274, mean=532.274, max=532.274, sum=1064.547 (2)\", \"tab\": \"General information\", \"score\": \"532.273631840796\"}", - "Sociology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"sociology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_sociology\"" - } - } - }, - { - "evaluation_name": "Virology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Virology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.548, - "details": { - "description": "min=0.548, mean=0.548, max=0.548, sum=1.096 (2)", - "tab": "Accuracy", - "Virology - Observed inference time (s)": "{\"description\": \"min=0.713, mean=0.713, max=0.713, sum=1.425 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7126703147428581\"}", - "Virology - # eval": "{\"description\": \"min=166, mean=166, max=166, sum=332 (2)\", \"tab\": \"General information\", \"score\": \"166.0\"}", - "Virology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Virology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Virology - # prompt tokens": "{\"description\": \"min=427.651, mean=427.651, max=427.651, sum=855.301 (2)\", \"tab\": \"General information\", \"score\": \"427.65060240963857\"}", - "Virology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"virology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_virology\"" - } - } - }, - { - "evaluation_name": "World Religions", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on World Religions", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.784, - "details": { - "description": "min=0.784, mean=0.784, max=0.784, sum=1.567 (2)", - "tab": "Accuracy", - "World Religions - Observed inference time (s)": "{\"description\": \"min=0.75, mean=0.75, max=0.75, sum=1.5 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7498089402739765\"}", - "World Religions - # eval": "{\"description\": \"min=171, mean=171, max=171, sum=342 (2)\", \"tab\": \"General information\", \"score\": \"171.0\"}", - "World Religions - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "World Religions - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "World Religions - # prompt tokens": "{\"description\": \"min=358.018, mean=358.018, max=358.018, sum=716.035 (2)\", \"tab\": \"General information\", \"score\": \"358.0175438596491\"}", - "World Religions - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"world_religions\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_world_religions\"" - } - } - }, - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.186, - "details": { - "description": "", - "tab": "Efficiency" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/anthropic_claude-opus-4-1-20250805.json b/data/models/anthropic_claude-opus-4-1-20250805.json deleted file mode 100644 index 97af67f7a9590f836aeb06cb6e8ac5e127131091..0000000000000000000000000000000000000000 --- a/data/models/anthropic_claude-opus-4-1-20250805.json +++ /dev/null @@ -1,1043 +0,0 @@ -{ - "model_info": { - "name": "claude-opus-4-1-20250805", - "id": "anthropic/claude-opus-4-1-20250805", - "developer": "Anthropic", - "inference_platform": "unknown", - "additional_details": { - "display_name": "Claude Opus 4.1" - } - }, - "evaluations": [ - { - "evaluation_id": "global-mmlu-lite/anthropic_claude-opus-4-1-20250805/1773936496.366405", - "retrieved_timestamp": "1773936496.366405", - "source_metadata": { - "source_name": "Global MMLU Lite Leaderboard", - "source_type": "documentation", - "source_organization_name": "kaggle", - "source_organization_url": "www.kaggle.com", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "kaggle kernel", - "version": "4", - "additional_details": { - "url": "https://www.kaggle.com/code/shivalikasingh95/global-mmlu-lite-sample-notebook" - } - }, - "benchmark": "global-mmlu-lite", - "evaluation_results": [ - { - "evaluation_name": "Global MMLU Lite", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Global MMLU Lite", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.943 - } - }, - { - "evaluation_name": "Culturally Sensitive", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Sensitive", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9331 - } - }, - { - "evaluation_name": "Culturally Agnostic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Agnostic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9528 - } - }, - { - "evaluation_name": "Arabic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Arabic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.945, - "uncertainty": { - "confidence_interval": { - "lower": -0.0223, - "upper": 0.0223, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "English", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - English", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9475, - "uncertainty": { - "confidence_interval": { - "lower": -0.0219, - "upper": 0.0219, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Bengali", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Bengali", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9425, - "uncertainty": { - "confidence_interval": { - "lower": -0.0228, - "upper": 0.0228, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "German", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - German", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.94, - "uncertainty": { - "confidence_interval": { - "lower": -0.0233, - "upper": 0.0233, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "French", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - French", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.945, - "uncertainty": { - "confidence_interval": { - "lower": -0.0223, - "upper": 0.0223, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Hindi", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Hindi", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9475, - "uncertainty": { - "confidence_interval": { - "lower": -0.0219, - "upper": 0.0219, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Indonesian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Indonesian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9425, - "uncertainty": { - "confidence_interval": { - "lower": -0.0228, - "upper": 0.0228, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Italian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Italian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.94, - "uncertainty": { - "confidence_interval": { - "lower": -0.0233, - "upper": 0.0233, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Japanese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Japanese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.94, - "uncertainty": { - "confidence_interval": { - "lower": -0.0233, - "upper": 0.0233, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Korean", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Korean", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.95, - "uncertainty": { - "confidence_interval": { - "lower": -0.0214, - "upper": 0.0214, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Portuguese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Portuguese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.945, - "uncertainty": { - "confidence_interval": { - "lower": -0.0223, - "upper": 0.0223, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Spanish", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Spanish", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.945, - "uncertainty": { - "confidence_interval": { - "lower": -0.0223, - "upper": 0.0223, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Swahili", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Swahili", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.93, - "uncertainty": { - "confidence_interval": { - "lower": -0.025, - "upper": 0.025, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Yoruba", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Yoruba", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9375, - "uncertainty": { - "confidence_interval": { - "lower": -0.0237, - "upper": 0.0237, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Chinese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Chinese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.945, - "uncertainty": { - "confidence_interval": { - "lower": -0.0223, - "upper": 0.0223, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Burmese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Burmese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.945, - "uncertainty": { - "confidence_interval": { - "lower": -0.0223, - "upper": 0.0223, - "method": "unknown" - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "global-mmlu-lite/anthropic_claude-opus-4-1-20250805/1773936583.743359", - "retrieved_timestamp": "1773936583.743359", - "source_metadata": { - "source_name": "Global MMLU Lite Leaderboard", - "source_type": "documentation", - "source_organization_name": "kaggle", - "source_organization_url": "www.kaggle.com", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "kaggle kernel", - "version": "4", - "additional_details": { - "url": "https://www.kaggle.com/code/shivalikasingh95/global-mmlu-lite-sample-notebook" - } - }, - "benchmark": "global-mmlu-lite", - "evaluation_results": [ - { - "evaluation_name": "Global MMLU Lite", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Global MMLU Lite", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.943 - } - }, - { - "evaluation_name": "Culturally Sensitive", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Sensitive", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9331 - } - }, - { - "evaluation_name": "Culturally Agnostic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Agnostic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9528 - } - }, - { - "evaluation_name": "Arabic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Arabic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.945, - "uncertainty": { - "confidence_interval": { - "lower": -0.0223, - "upper": 0.0223, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "English", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - English", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9475, - "uncertainty": { - "confidence_interval": { - "lower": -0.0219, - "upper": 0.0219, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Bengali", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Bengali", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9425, - "uncertainty": { - "confidence_interval": { - "lower": -0.0228, - "upper": 0.0228, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "German", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - German", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.94, - "uncertainty": { - "confidence_interval": { - "lower": -0.0233, - "upper": 0.0233, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "French", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - French", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.945, - "uncertainty": { - "confidence_interval": { - "lower": -0.0223, - "upper": 0.0223, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Hindi", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Hindi", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9475, - "uncertainty": { - "confidence_interval": { - "lower": -0.0219, - "upper": 0.0219, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Indonesian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Indonesian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9425, - "uncertainty": { - "confidence_interval": { - "lower": -0.0228, - "upper": 0.0228, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Italian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Italian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.94, - "uncertainty": { - "confidence_interval": { - "lower": -0.0233, - "upper": 0.0233, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Japanese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Japanese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.94, - "uncertainty": { - "confidence_interval": { - "lower": -0.0233, - "upper": 0.0233, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Korean", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Korean", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.95, - "uncertainty": { - "confidence_interval": { - "lower": -0.0214, - "upper": 0.0214, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Portuguese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Portuguese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.945, - "uncertainty": { - "confidence_interval": { - "lower": -0.0223, - "upper": 0.0223, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Spanish", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Spanish", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.945, - "uncertainty": { - "confidence_interval": { - "lower": -0.0223, - "upper": 0.0223, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Swahili", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Swahili", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.93, - "uncertainty": { - "confidence_interval": { - "lower": -0.025, - "upper": 0.025, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Yoruba", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Yoruba", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9375, - "uncertainty": { - "confidence_interval": { - "lower": -0.0237, - "upper": 0.0237, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Chinese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Chinese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.945, - "uncertainty": { - "confidence_interval": { - "lower": -0.0223, - "upper": 0.0223, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Burmese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Burmese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.945, - "uncertainty": { - "confidence_interval": { - "lower": -0.0223, - "upper": 0.0223, - "method": "unknown" - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/anthropic_claude-opus-4-20250514-thinking-10k.json b/data/models/anthropic_claude-opus-4-20250514-thinking-10k.json deleted file mode 100644 index 181b5a1475c7134f6f16388f2680d2187bd189fe..0000000000000000000000000000000000000000 --- a/data/models/anthropic_claude-opus-4-20250514-thinking-10k.json +++ /dev/null @@ -1,234 +0,0 @@ -{ - "model_info": { - "name": "Claude 4 Opus 20250514, extended thinking", - "id": "anthropic/claude-opus-4-20250514-thinking-10k", - "developer": "Anthropic", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_capabilities/anthropic_claude-opus-4-20250514-thinking-10k/1774096304.5056698", - "retrieved_timestamp": "1774096304.5056698", - "source_metadata": { - "source_name": "helm_capabilities", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_capabilities", - "evaluation_results": [ - { - "evaluation_name": "Mean score", - "source_data": { - "dataset_name": "helm_capabilities", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "The mean of the scores from all columns.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.78, - "details": { - "description": "", - "tab": "Accuracy", - "Mean score - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"52.297304217949794\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU-Pro", - "source_data": { - "dataset_name": "MMLU-Pro", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on MMLU-Pro", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.875, - "details": { - "description": "min=0.875, mean=0.875, max=0.875, sum=0.875 (1)", - "tab": "Accuracy", - "MMLU-Pro - Observed inference time (s)": "{\"description\": \"min=28.466, mean=28.466, max=28.466, sum=28.466 (1)\", \"tab\": \"Efficiency\", \"score\": \"28.46593898815197\"}", - "MMLU-Pro - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "MMLU-Pro - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - # prompt tokens": "{\"description\": \"min=252.461, mean=252.461, max=252.461, sum=252.461 (1)\", \"tab\": \"General information\", \"score\": \"252.461\"}", - "MMLU-Pro - # output tokens": "{\"description\": \"min=272.871, mean=272.871, max=272.871, sum=272.871 (1)\", \"tab\": \"General information\", \"score\": \"272.871\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"all\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.709, - "details": { - "description": "min=0.709, mean=0.709, max=0.709, sum=0.709 (1)", - "tab": "Accuracy", - "GPQA - Observed inference time (s)": "{\"description\": \"min=45.529, mean=45.529, max=45.529, sum=45.529 (1)\", \"tab\": \"Efficiency\", \"score\": \"45.52923426562793\"}", - "GPQA - # eval": "{\"description\": \"min=446, mean=446, max=446, sum=446 (1)\", \"tab\": \"General information\", \"score\": \"446.0\"}", - "GPQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - # prompt tokens": "{\"description\": \"min=272.738, mean=272.738, max=272.738, sum=272.738 (1)\", \"tab\": \"General information\", \"score\": \"272.73766816143495\"}", - "GPQA - # output tokens": "{\"description\": \"min=343.762, mean=343.762, max=343.762, sum=343.762 (1)\", \"tab\": \"General information\", \"score\": \"343.76233183856505\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"gpqa_main\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "IFEval Strict Acc on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.849, - "details": { - "description": "min=0.849, mean=0.849, max=0.849, sum=0.849 (1)", - "tab": "Accuracy", - "IFEval - Observed inference time (s)": "{\"description\": \"min=22.453, mean=22.453, max=22.453, sum=22.453 (1)\", \"tab\": \"Efficiency\", \"score\": \"22.45251508421368\"}", - "IFEval - # eval": "{\"description\": \"min=541, mean=541, max=541, sum=541 (1)\", \"tab\": \"General information\", \"score\": \"541.0\"}", - "IFEval - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - # prompt tokens": "{\"description\": \"min=47.159, mean=47.159, max=47.159, sum=47.159 (1)\", \"tab\": \"General information\", \"score\": \"47.15896487985213\"}", - "IFEval - # output tokens": "{\"description\": \"min=403.745, mean=403.745, max=403.745, sum=403.745 (1)\", \"tab\": \"General information\", \"score\": \"403.74491682070243\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WildBench", - "source_data": { - "dataset_name": "WildBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "WB Score on WildBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.852, - "details": { - "description": "min=0.852, mean=0.852, max=0.852, sum=0.852 (1)", - "tab": "Accuracy", - "WildBench - Observed inference time (s)": "{\"description\": \"min=50.19, mean=50.19, max=50.19, sum=50.19 (1)\", \"tab\": \"Efficiency\", \"score\": \"50.19046350765228\"}", - "WildBench - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "WildBench - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # prompt tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # output tokens": "{\"description\": \"min=1195.769, mean=1195.769, max=1195.769, sum=1195.769 (1)\", \"tab\": \"General information\", \"score\": \"1195.769\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"v2\"" - } - } - }, - { - "evaluation_name": "Omni-MATH", - "source_data": { - "dataset_name": "Omni-MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Acc on Omni-MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.616, - "details": { - "description": "min=0.616, mean=0.616, max=0.616, sum=0.616 (1)", - "tab": "Accuracy", - "Omni-MATH - Observed inference time (s)": "{\"description\": \"min=114.848, mean=114.848, max=114.848, sum=114.848 (1)\", \"tab\": \"Efficiency\", \"score\": \"114.84836924410313\"}", - "Omni-MATH - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "Omni-MATH - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - # prompt tokens": "{\"description\": \"min=110.563, mean=110.563, max=110.563, sum=110.563 (1)\", \"tab\": \"General information\", \"score\": \"110.563\"}", - "Omni-MATH - # output tokens": "{\"description\": \"min=691.066, mean=691.066, max=691.066, sum=691.066 (1)\", \"tab\": \"General information\", \"score\": \"691.066\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/anthropic_claude-opus-4-20250514.json b/data/models/anthropic_claude-opus-4-20250514.json deleted file mode 100644 index d7db1ddc4a1a9514af2c26913f5c13d049fde332..0000000000000000000000000000000000000000 --- a/data/models/anthropic_claude-opus-4-20250514.json +++ /dev/null @@ -1,384 +0,0 @@ -{ - "model_info": { - "name": "Claude 4 Opus 20250514", - "id": "anthropic/claude-opus-4-20250514", - "developer": "Anthropic", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_capabilities/anthropic_claude-opus-4-20250514/1774096304.5056698", - "retrieved_timestamp": "1774096304.5056698", - "source_metadata": { - "source_name": "helm_capabilities", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_capabilities", - "evaluation_results": [ - { - "evaluation_name": "Mean score", - "source_data": { - "dataset_name": "helm_capabilities", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "The mean of the scores from all columns.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.757, - "details": { - "description": "", - "tab": "Accuracy", - "Mean score - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"20.48127702555515\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU-Pro", - "source_data": { - "dataset_name": "MMLU-Pro", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on MMLU-Pro", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.859, - "details": { - "description": "min=0.859, mean=0.859, max=0.859, sum=0.859 (1)", - "tab": "Accuracy", - "MMLU-Pro - Observed inference time (s)": "{\"description\": \"min=12.63, mean=12.63, max=12.63, sum=12.63 (1)\", \"tab\": \"Efficiency\", \"score\": \"12.630421590518665\"}", - "MMLU-Pro - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "MMLU-Pro - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - # prompt tokens": "{\"description\": \"min=230.461, mean=230.461, max=230.461, sum=230.461 (1)\", \"tab\": \"General information\", \"score\": \"230.461\"}", - "MMLU-Pro - # output tokens": "{\"description\": \"min=344.469, mean=344.469, max=344.469, sum=344.469 (1)\", \"tab\": \"General information\", \"score\": \"344.469\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"all\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.666, - "details": { - "description": "min=0.666, mean=0.666, max=0.666, sum=0.666 (1)", - "tab": "Accuracy", - "GPQA - Observed inference time (s)": "{\"description\": \"min=16.325, mean=16.325, max=16.325, sum=16.325 (1)\", \"tab\": \"Efficiency\", \"score\": \"16.325411326249803\"}", - "GPQA - # eval": "{\"description\": \"min=446, mean=446, max=446, sum=446 (1)\", \"tab\": \"General information\", \"score\": \"446.0\"}", - "GPQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - # prompt tokens": "{\"description\": \"min=250.738, mean=250.738, max=250.738, sum=250.738 (1)\", \"tab\": \"General information\", \"score\": \"250.73766816143498\"}", - "GPQA - # output tokens": "{\"description\": \"min=453.143, mean=453.143, max=453.143, sum=453.143 (1)\", \"tab\": \"General information\", \"score\": \"453.1434977578475\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"gpqa_main\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "IFEval Strict Acc on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.918, - "details": { - "description": "min=0.918, mean=0.918, max=0.918, sum=0.918 (1)", - "tab": "Accuracy", - "IFEval - Observed inference time (s)": "{\"description\": \"min=16.576, mean=16.576, max=16.576, sum=16.576 (1)\", \"tab\": \"Efficiency\", \"score\": \"16.576411149939712\"}", - "IFEval - # eval": "{\"description\": \"min=541, mean=541, max=541, sum=541 (1)\", \"tab\": \"General information\", \"score\": \"541.0\"}", - "IFEval - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - # prompt tokens": "{\"description\": \"min=47.159, mean=47.159, max=47.159, sum=47.159 (1)\", \"tab\": \"General information\", \"score\": \"47.15896487985213\"}", - "IFEval - # output tokens": "{\"description\": \"min=422.774, mean=422.774, max=422.774, sum=422.774 (1)\", \"tab\": \"General information\", \"score\": \"422.7744916820702\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WildBench", - "source_data": { - "dataset_name": "WildBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "WB Score on WildBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.833, - "details": { - "description": "min=0.833, mean=0.833, max=0.833, sum=0.833 (1)", - "tab": "Accuracy", - "WildBench - Observed inference time (s)": "{\"description\": \"min=29.848, mean=29.848, max=29.848, sum=29.848 (1)\", \"tab\": \"Efficiency\", \"score\": \"29.848318881988526\"}", - "WildBench - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "WildBench - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # prompt tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # output tokens": "{\"description\": \"min=936.927, mean=936.927, max=936.927, sum=936.927 (1)\", \"tab\": \"General information\", \"score\": \"936.927\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"v2\"" - } - } - }, - { - "evaluation_name": "Omni-MATH", - "source_data": { - "dataset_name": "Omni-MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Acc on Omni-MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.511, - "details": { - "description": "min=0.511, mean=0.511, max=0.511, sum=0.511 (1)", - "tab": "Accuracy", - "Omni-MATH - Observed inference time (s)": "{\"description\": \"min=27.026, mean=27.026, max=27.026, sum=27.026 (1)\", \"tab\": \"Efficiency\", \"score\": \"27.025822179079057\"}", - "Omni-MATH - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "Omni-MATH - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - # prompt tokens": "{\"description\": \"min=110.563, mean=110.563, max=110.563, sum=110.563 (1)\", \"tab\": \"General information\", \"score\": \"110.563\"}", - "Omni-MATH - # output tokens": "{\"description\": \"min=893.894, mean=893.894, max=893.894, sum=893.894 (1)\", \"tab\": \"General information\", \"score\": \"893.894\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "reward-bench-2/anthropic_claude-opus-4-20250514/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7648 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8267 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4188 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7491 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8954 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8616 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8375 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/anthropic_claude-opus-4-5-20251101-fc.json b/data/models/anthropic_claude-opus-4-5-20251101-fc.json deleted file mode 100644 index 8b93ab2a179e390a38c92f9d914fb8fb1faa2de8..0000000000000000000000000000000000000000 --- a/data/models/anthropic_claude-opus-4-5-20251101-fc.json +++ /dev/null @@ -1,851 +0,0 @@ -{ - "model_info": { - "name": "Claude-Opus-4-5-20251101 (FC)", - "id": "anthropic/claude-opus-4-5-20251101-fc", - "developer": "Anthropic", - "additional_details": { - "raw_model_name": "Claude-Opus-4-5-20251101 (FC)", - "organization": "Anthropic", - "license": "Proprietary", - "mode": "FC", - "model_link": "https://www.anthropic.com/news/claude-4" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/anthropic/claude-opus-4-5-20251101-fc/1775236112.365765", - "retrieved_timestamp": "1775236112.365765", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 1.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 77.47 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 86.55 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 4.38 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 3.13 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 7.56 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 88.58 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 76.83 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 95.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 93.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 88.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 79.79 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 86.43 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 78.16 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 87.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 75.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 68.38 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 81.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 64.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 58.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 70.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 84.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 84.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 85.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 73.76 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 70.97 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 72.9 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 77.42 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 62.5 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 84.72 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/anthropic_claude-opus-4-5-20251101-prompt.json b/data/models/anthropic_claude-opus-4-5-20251101-prompt.json deleted file mode 100644 index 62920f3fb3ce576b1931f0ce448d57f7242f7575..0000000000000000000000000000000000000000 --- a/data/models/anthropic_claude-opus-4-5-20251101-prompt.json +++ /dev/null @@ -1,905 +0,0 @@ -{ - "model_info": { - "name": "Claude-Opus-4-5-20251101 (Prompt)", - "id": "anthropic/claude-opus-4-5-20251101-prompt", - "developer": "Anthropic", - "additional_details": { - "raw_model_name": "Claude-Opus-4-5-20251101 (Prompt)", - "organization": "Anthropic", - "license": "Proprietary", - "mode": "Prompt", - "model_link": "https://www.anthropic.com/news/claude-4" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/anthropic/claude-opus-4-5-20251101-prompt/1775236112.395529", - "retrieved_timestamp": "1775236112.395529", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 57.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 33.47 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 88.33 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 3.76 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 13.19 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 5.52 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 89.65 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 79.58 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 93.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 93.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 92.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 76.02 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 84.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 74.17 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 81.25 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 62.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 16.12 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 20.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 9.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 21.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 13.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 13.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 13.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 13.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 1.94 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 1.29 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 1.94 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 2.58 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 68.75 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 90.75 - } - }, - { - "evaluation_result_id": "bfcl.format_sensitivity.max_delta::max_delta", - "evaluation_name": "bfcl.format_sensitivity.max_delta", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.format_sensitivity.max_delta", - "metric_name": "Format sensitivity max delta", - "metric_kind": "difference", - "metric_unit": "percentage_points", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Format Sensitivity Max Delta" - } - }, - "score_details": { - "score": 13.0 - } - }, - { - "evaluation_result_id": "bfcl.format_sensitivity.stddev::stddev", - "evaluation_name": "bfcl.format_sensitivity.stddev", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.format_sensitivity.stddev", - "metric_name": "Format sensitivity standard deviation", - "metric_kind": "difference", - "metric_unit": "percentage_points", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Format Sensitivity Standard Deviation" - } - }, - "score_details": { - "score": 3.65 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/anthropic_claude-opus-4-5.json b/data/models/anthropic_claude-opus-4-5.json deleted file mode 100644 index 45e7a26b58b8940fc9eaabde7649ceedce0ab980..0000000000000000000000000000000000000000 --- a/data/models/anthropic_claude-opus-4-5.json +++ /dev/null @@ -1,2053 +0,0 @@ -{ - "model_info": { - "name": "claude-opus-4-5", - "id": "anthropic/claude-opus-4-5", - "developer": "Anthropic", - "additional_details": { - "agent_name": "LiteLLM Tool Calling", - "agent_framework": "tool_calling" - } - }, - "evaluations": [ - { - "evaluation_id": "appworld/test_normal/litellm-tool-calling__anthropic_claude-opus-4-5/1774263615.0201504", - "retrieved_timestamp": "1774263615.0201504", - "source_metadata": { - "source_name": "Exgentic Open Agent Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Exgentic", - "source_organization_url": "https://github.com/Exgentic", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "exgentic", - "version": "0.1.0" - }, - "benchmark": "appworld_test_normal", - "evaluation_results": [ - { - "evaluation_name": "appworld/test_normal", - "source_data": { - "dataset_name": "appworld/test_normal", - "source_type": "url", - "url": [ - "https://github.com/Exgentic/exgentic" - ] - }, - "metric_config": { - "evaluation_description": "AppWorld benchmark evaluation (test_normal subset)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.61, - "uncertainty": { - "num_samples": 100 - }, - "details": { - "average_agent_cost": "11.32", - "total_run_cost": "1132.47", - "average_steps": "21.99", - "percent_finished": "0.83" - } - }, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "LiteLLM Tool Calling", - "agent_framework": "tool_calling" - } - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "LiteLLM Tool Calling", - "agent_framework": "tool_calling" - } - } - } - } - }, - { - "evaluation_id": "appworld/test_normal/claude-code-cli__anthropic_claude-opus-4-5/1774263615.0201504", - "retrieved_timestamp": "1774263615.0201504", - "source_metadata": { - "source_name": "Exgentic Open Agent Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Exgentic", - "source_organization_url": "https://github.com/Exgentic", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "exgentic", - "version": "0.1.0" - }, - "benchmark": "appworld_test_normal", - "evaluation_results": [ - { - "evaluation_name": "appworld/test_normal", - "source_data": { - "dataset_name": "appworld/test_normal", - "source_type": "url", - "url": [ - "https://github.com/Exgentic/exgentic" - ] - }, - "metric_config": { - "evaluation_description": "AppWorld benchmark evaluation (test_normal subset)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.66, - "uncertainty": { - "num_samples": 100 - }, - "details": { - "average_agent_cost": "13.08", - "total_run_cost": "1308.38", - "average_steps": "49.69", - "percent_finished": "0.74" - } - }, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "Claude Code CLI", - "agent_framework": "claude_code" - } - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "Claude Code CLI", - "agent_framework": "claude_code" - } - } - } - } - }, - { - "evaluation_id": "appworld/test_normal/smolagents-code__anthropic_claude-opus-4-5/1774263615.0201504", - "retrieved_timestamp": "1774263615.0201504", - "source_metadata": { - "source_name": "Exgentic Open Agent Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Exgentic", - "source_organization_url": "https://github.com/Exgentic", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "exgentic", - "version": "0.1.0" - }, - "benchmark": "appworld_test_normal", - "evaluation_results": [ - { - "evaluation_name": "appworld/test_normal", - "source_data": { - "dataset_name": "appworld/test_normal", - "source_type": "url", - "url": [ - "https://github.com/Exgentic/exgentic" - ] - }, - "metric_config": { - "evaluation_description": "AppWorld benchmark evaluation (test_normal subset)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7, - "uncertainty": { - "num_samples": 100 - }, - "details": { - "average_agent_cost": "5.59", - "total_run_cost": "558.51", - "average_steps": "41.07", - "percent_finished": "0.82" - } - }, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "SmolAgents Code", - "agent_framework": "smolagents_code" - } - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "SmolAgents Code", - "agent_framework": "smolagents_code" - } - } - } - } - }, - { - "evaluation_id": "appworld/test_normal/litellm-tool-calling-with-shortlisting__anthropic_claude-opus-4-5/1774263615.0201504", - "retrieved_timestamp": "1774263615.0201504", - "source_metadata": { - "source_name": "Exgentic Open Agent Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Exgentic", - "source_organization_url": "https://github.com/Exgentic", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "exgentic", - "version": "0.1.0" - }, - "benchmark": "appworld_test_normal", - "evaluation_results": [ - { - "evaluation_name": "appworld/test_normal", - "source_data": { - "dataset_name": "appworld/test_normal", - "source_type": "url", - "url": [ - "https://github.com/Exgentic/exgentic" - ] - }, - "metric_config": { - "evaluation_description": "AppWorld benchmark evaluation (test_normal subset)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.64, - "uncertainty": { - "num_samples": 100 - }, - "details": { - "average_agent_cost": "3.43", - "total_run_cost": "343.32", - "average_steps": "20.06", - "percent_finished": "0.82" - } - }, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "LiteLLM Tool Calling with Shortlisting", - "agent_framework": "tool_calling_with_shortlisting" - } - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "LiteLLM Tool Calling with Shortlisting", - "agent_framework": "tool_calling_with_shortlisting" - } - } - } - } - }, - { - "evaluation_id": "appworld/test_normal/openai-solo__anthropic_claude-opus-4-5/1774263615.0201504", - "retrieved_timestamp": "1774263615.0201504", - "source_metadata": { - "source_name": "Exgentic Open Agent Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Exgentic", - "source_organization_url": "https://github.com/Exgentic", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "exgentic", - "version": "0.1.0" - }, - "benchmark": "appworld_test_normal", - "evaluation_results": [ - { - "evaluation_name": "appworld/test_normal", - "source_data": { - "dataset_name": "appworld/test_normal", - "source_type": "url", - "url": [ - "https://github.com/Exgentic/exgentic" - ] - }, - "metric_config": { - "evaluation_description": "AppWorld benchmark evaluation (test_normal subset)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.68, - "uncertainty": { - "num_samples": 100 - }, - "details": { - "average_agent_cost": "22.76", - "total_run_cost": "2276.48", - "average_steps": "47.65", - "percent_finished": "0.77" - } - }, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "OpenAI Solo", - "agent_framework": "openai_solo" - } - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "OpenAI Solo", - "agent_framework": "openai_solo" - } - } - } - } - }, - { - "evaluation_id": "browsecompplus/litellm-tool-calling__anthropic_claude-opus-4-5/1774263615.0201504", - "retrieved_timestamp": "1774263615.0201504", - "source_metadata": { - "source_name": "Exgentic Open Agent Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Exgentic", - "source_organization_url": "https://github.com/Exgentic", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "exgentic", - "version": "0.1.0" - }, - "benchmark": "browsecompplus", - "evaluation_results": [ - { - "evaluation_name": "browsecompplus", - "source_data": { - "dataset_name": "browsecompplus", - "source_type": "url", - "url": [ - "https://github.com/Exgentic/exgentic" - ] - }, - "metric_config": { - "evaluation_description": "BrowseCompPlus benchmark evaluation", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.49, - "uncertainty": { - "num_samples": 100 - }, - "details": { - "average_agent_cost": "7.09", - "total_run_cost": "709.54", - "average_steps": "21.66", - "percent_finished": "0.93" - } - }, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "LiteLLM Tool Calling", - "agent_framework": "tool_calling" - } - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "LiteLLM Tool Calling", - "agent_framework": "tool_calling" - } - } - } - } - }, - { - "evaluation_id": "browsecompplus/claude-code-cli__anthropic_claude-opus-4-5/1774263615.0201504", - "retrieved_timestamp": "1774263615.0201504", - "source_metadata": { - "source_name": "Exgentic Open Agent Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Exgentic", - "source_organization_url": "https://github.com/Exgentic", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "exgentic", - "version": "0.1.0" - }, - "benchmark": "browsecompplus", - "evaluation_results": [ - { - "evaluation_name": "browsecompplus", - "source_data": { - "dataset_name": "browsecompplus", - "source_type": "url", - "url": [ - "https://github.com/Exgentic/exgentic" - ] - }, - "metric_config": { - "evaluation_description": "BrowseCompPlus benchmark evaluation", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5294, - "uncertainty": { - "num_samples": 51 - }, - "details": { - "average_agent_cost": "11.66", - "total_run_cost": "594.68", - "average_steps": "31.04", - "percent_finished": "0.8431" - } - }, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "Claude Code CLI", - "agent_framework": "claude_code" - } - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "Claude Code CLI", - "agent_framework": "claude_code" - } - } - } - } - }, - { - "evaluation_id": "browsecompplus/openai-solo__anthropic_claude-opus-4-5/1774263615.0201504", - "retrieved_timestamp": "1774263615.0201504", - "source_metadata": { - "source_name": "Exgentic Open Agent Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Exgentic", - "source_organization_url": "https://github.com/Exgentic", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "exgentic", - "version": "0.1.0" - }, - "benchmark": "browsecompplus", - "evaluation_results": [ - { - "evaluation_name": "browsecompplus", - "source_data": { - "dataset_name": "browsecompplus", - "source_type": "url", - "url": [ - "https://github.com/Exgentic/exgentic" - ] - }, - "metric_config": { - "evaluation_description": "BrowseCompPlus benchmark evaluation", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.61, - "uncertainty": { - "num_samples": 100 - }, - "details": { - "average_agent_cost": "7.59", - "total_run_cost": "759.44", - "average_steps": "27.18", - "percent_finished": "1.0" - } - }, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "OpenAI Solo", - "agent_framework": "openai_solo" - } - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "OpenAI Solo", - "agent_framework": "openai_solo" - } - } - } - } - }, - { - "evaluation_id": "browsecompplus/litellm-tool-calling-with-shortlisting__anthropic_claude-opus-4-5/1774263615.0201504", - "retrieved_timestamp": "1774263615.0201504", - "source_metadata": { - "source_name": "Exgentic Open Agent Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Exgentic", - "source_organization_url": "https://github.com/Exgentic", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "exgentic", - "version": "0.1.0" - }, - "benchmark": "browsecompplus", - "evaluation_results": [ - { - "evaluation_name": "browsecompplus", - "source_data": { - "dataset_name": "browsecompplus", - "source_type": "url", - "url": [ - "https://github.com/Exgentic/exgentic" - ] - }, - "metric_config": { - "evaluation_description": "BrowseCompPlus benchmark evaluation", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.49, - "uncertainty": { - "num_samples": 100 - }, - "details": { - "average_agent_cost": "7.09", - "total_run_cost": "709.54", - "average_steps": "21.66", - "percent_finished": "0.93" - } - }, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "LiteLLM Tool Calling with Shortlisting", - "agent_framework": "tool_calling_with_shortlisting" - } - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "LiteLLM Tool Calling with Shortlisting", - "agent_framework": "tool_calling_with_shortlisting" - } - } - } - } - }, - { - "evaluation_id": "browsecompplus/smolagents-code__anthropic_claude-opus-4-5/1774263615.0201504", - "retrieved_timestamp": "1774263615.0201504", - "source_metadata": { - "source_name": "Exgentic Open Agent Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Exgentic", - "source_organization_url": "https://github.com/Exgentic", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "exgentic", - "version": "0.1.0" - }, - "benchmark": "browsecompplus", - "evaluation_results": [ - { - "evaluation_name": "browsecompplus", - "source_data": { - "dataset_name": "browsecompplus", - "source_type": "url", - "url": [ - "https://github.com/Exgentic/exgentic" - ] - }, - "metric_config": { - "evaluation_description": "BrowseCompPlus benchmark evaluation", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.61, - "uncertainty": { - "num_samples": 100 - }, - "details": { - "average_agent_cost": "6.3", - "total_run_cost": "630.56", - "average_steps": "24.16", - "percent_finished": "1.0" - } - }, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "SmolAgents Code", - "agent_framework": "smolagents_code" - } - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "SmolAgents Code", - "agent_framework": "smolagents_code" - } - } - } - } - }, - { - "evaluation_id": "swe-bench/openai-solo__anthropic_claude-opus-4-5/1774263615.0201504", - "retrieved_timestamp": "1774263615.0201504", - "source_metadata": { - "source_name": "Exgentic Open Agent Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Exgentic", - "source_organization_url": "https://github.com/Exgentic", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "exgentic", - "version": "0.1.0" - }, - "benchmark": "swe-bench", - "evaluation_results": [ - { - "evaluation_name": "swe-bench", - "source_data": { - "dataset_name": "swe-bench", - "source_type": "url", - "url": [ - "https://github.com/Exgentic/exgentic" - ] - }, - "metric_config": { - "evaluation_description": "SWE-bench benchmark evaluation", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8072, - "uncertainty": { - "num_samples": 83 - }, - "details": { - "average_agent_cost": "2.96", - "total_run_cost": "245.78", - "average_steps": "34.1", - "percent_finished": "1.0" - } - }, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "OpenAI Solo", - "agent_framework": "openai_solo" - } - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "OpenAI Solo", - "agent_framework": "openai_solo" - } - } - } - } - }, - { - "evaluation_id": "swe-bench/claude-code-cli__anthropic_claude-opus-4-5/1774263615.0201504", - "retrieved_timestamp": "1774263615.0201504", - "source_metadata": { - "source_name": "Exgentic Open Agent Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Exgentic", - "source_organization_url": "https://github.com/Exgentic", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "exgentic", - "version": "0.1.0" - }, - "benchmark": "swe-bench", - "evaluation_results": [ - { - "evaluation_name": "swe-bench", - "source_data": { - "dataset_name": "swe-bench", - "source_type": "url", - "url": [ - "https://github.com/Exgentic/exgentic" - ] - }, - "metric_config": { - "evaluation_description": "SWE-bench benchmark evaluation", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7423, - "uncertainty": { - "num_samples": 97 - }, - "details": { - "average_agent_cost": "5.6", - "total_run_cost": "543.62", - "average_steps": "31.76", - "percent_finished": "1.0" - } - }, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "Claude Code CLI", - "agent_framework": "claude_code" - } - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "Claude Code CLI", - "agent_framework": "claude_code" - } - } - } - } - }, - { - "evaluation_id": "swe-bench/litellm-tool-calling__anthropic_claude-opus-4-5/1774263615.0201504", - "retrieved_timestamp": "1774263615.0201504", - "source_metadata": { - "source_name": "Exgentic Open Agent Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Exgentic", - "source_organization_url": "https://github.com/Exgentic", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "exgentic", - "version": "0.1.0" - }, - "benchmark": "swe-bench", - "evaluation_results": [ - { - "evaluation_name": "swe-bench", - "source_data": { - "dataset_name": "swe-bench", - "source_type": "url", - "url": [ - "https://github.com/Exgentic/exgentic" - ] - }, - "metric_config": { - "evaluation_description": "SWE-bench benchmark evaluation", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6061, - "uncertainty": { - "num_samples": 99 - }, - "details": { - "average_agent_cost": "3.97", - "total_run_cost": "393.16", - "average_steps": "43.44", - "percent_finished": "1.0" - } - }, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "LiteLLM Tool Calling", - "agent_framework": "tool_calling" - } - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "LiteLLM Tool Calling", - "agent_framework": "tool_calling" - } - } - } - } - }, - { - "evaluation_id": "swe-bench/litellm-tool-calling-with-shortlisting__anthropic_claude-opus-4-5/1774263615.0201504", - "retrieved_timestamp": "1774263615.0201504", - "source_metadata": { - "source_name": "Exgentic Open Agent Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Exgentic", - "source_organization_url": "https://github.com/Exgentic", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "exgentic", - "version": "0.1.0" - }, - "benchmark": "swe-bench", - "evaluation_results": [ - { - "evaluation_name": "swe-bench", - "source_data": { - "dataset_name": "swe-bench", - "source_type": "url", - "url": [ - "https://github.com/Exgentic/exgentic" - ] - }, - "metric_config": { - "evaluation_description": "SWE-bench benchmark evaluation", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6061, - "uncertainty": { - "num_samples": 99 - }, - "details": { - "average_agent_cost": "3.97", - "total_run_cost": "393.16", - "average_steps": "43.44", - "percent_finished": "1.0" - } - }, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "LiteLLM Tool Calling with Shortlisting", - "agent_framework": "tool_calling_with_shortlisting" - } - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "LiteLLM Tool Calling with Shortlisting", - "agent_framework": "tool_calling_with_shortlisting" - } - } - } - } - }, - { - "evaluation_id": "swe-bench/smolagents-code__anthropic_claude-opus-4-5/1774263615.0201504", - "retrieved_timestamp": "1774263615.0201504", - "source_metadata": { - "source_name": "Exgentic Open Agent Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Exgentic", - "source_organization_url": "https://github.com/Exgentic", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "exgentic", - "version": "0.1.0" - }, - "benchmark": "swe-bench", - "evaluation_results": [ - { - "evaluation_name": "swe-bench", - "source_data": { - "dataset_name": "swe-bench", - "source_type": "url", - "url": [ - "https://github.com/Exgentic/exgentic" - ] - }, - "metric_config": { - "evaluation_description": "SWE-bench benchmark evaluation", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.65, - "uncertainty": { - "num_samples": 100 - }, - "details": { - "average_agent_cost": "4.85", - "total_run_cost": "485.22", - "average_steps": "39.13", - "percent_finished": "1.0" - } - }, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "SmolAgents Code", - "agent_framework": "smolagents_code" - } - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "SmolAgents Code", - "agent_framework": "smolagents_code" - } - } - } - } - }, - { - "evaluation_id": "tau-bench-2/airline/litellm-tool-calling-with-shortlisting__anthropic_claude-opus-4-5/1774263615.0201504", - "retrieved_timestamp": "1774263615.0201504", - "source_metadata": { - "source_name": "Exgentic Open Agent Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Exgentic", - "source_organization_url": "https://github.com/Exgentic", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "exgentic", - "version": "0.1.0" - }, - "benchmark": "tau-bench-2_airline", - "evaluation_results": [ - { - "evaluation_name": "tau-bench-2/airline", - "source_data": { - "dataset_name": "tau-bench-2/airline", - "source_type": "url", - "url": [ - "https://github.com/Exgentic/exgentic" - ] - }, - "metric_config": { - "evaluation_description": "Tau Bench 2 benchmark evaluation (airline subset)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.66, - "uncertainty": { - "num_samples": 50 - }, - "details": { - "average_agent_cost": "0.47", - "total_run_cost": "24.23", - "average_steps": "10.0", - "percent_finished": "1.0" - } - }, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "LiteLLM Tool Calling with Shortlisting", - "agent_framework": "tool_calling_with_shortlisting" - } - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "LiteLLM Tool Calling with Shortlisting", - "agent_framework": "tool_calling_with_shortlisting" - } - } - } - } - }, - { - "evaluation_id": "tau-bench-2/airline/litellm-tool-calling__anthropic_claude-opus-4-5/1774263615.0201504", - "retrieved_timestamp": "1774263615.0201504", - "source_metadata": { - "source_name": "Exgentic Open Agent Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Exgentic", - "source_organization_url": "https://github.com/Exgentic", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "exgentic", - "version": "0.1.0" - }, - "benchmark": "tau-bench-2_airline", - "evaluation_results": [ - { - "evaluation_name": "tau-bench-2/airline", - "source_data": { - "dataset_name": "tau-bench-2/airline", - "source_type": "url", - "url": [ - "https://github.com/Exgentic/exgentic" - ] - }, - "metric_config": { - "evaluation_description": "Tau Bench 2 benchmark evaluation (airline subset)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.66, - "uncertainty": { - "num_samples": 50 - }, - "details": { - "average_agent_cost": "0.47", - "total_run_cost": "24.23", - "average_steps": "10.0", - "percent_finished": "1.0" - } - }, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "LiteLLM Tool Calling", - "agent_framework": "tool_calling" - } - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "LiteLLM Tool Calling", - "agent_framework": "tool_calling" - } - } - } - } - }, - { - "evaluation_id": "tau-bench-2/airline/smolagents-code__anthropic_claude-opus-4-5/1774263615.0201504", - "retrieved_timestamp": "1774263615.0201504", - "source_metadata": { - "source_name": "Exgentic Open Agent Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Exgentic", - "source_organization_url": "https://github.com/Exgentic", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "exgentic", - "version": "0.1.0" - }, - "benchmark": "tau-bench-2_airline", - "evaluation_results": [ - { - "evaluation_name": "tau-bench-2/airline", - "source_data": { - "dataset_name": "tau-bench-2/airline", - "source_type": "url", - "url": [ - "https://github.com/Exgentic/exgentic" - ] - }, - "metric_config": { - "evaluation_description": "Tau Bench 2 benchmark evaluation (airline subset)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.72, - "uncertainty": { - "num_samples": 50 - }, - "details": { - "average_agent_cost": "0.78", - "total_run_cost": "39.67", - "average_steps": "11.88", - "percent_finished": "1.0" - } - }, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "SmolAgents Code", - "agent_framework": "smolagents_code" - } - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "SmolAgents Code", - "agent_framework": "smolagents_code" - } - } - } - } - }, - { - "evaluation_id": "tau-bench-2/airline/openai-solo__anthropic_claude-opus-4-5/1774263615.0201504", - "retrieved_timestamp": "1774263615.0201504", - "source_metadata": { - "source_name": "Exgentic Open Agent Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Exgentic", - "source_organization_url": "https://github.com/Exgentic", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "exgentic", - "version": "0.1.0" - }, - "benchmark": "tau-bench-2_airline", - "evaluation_results": [ - { - "evaluation_name": "tau-bench-2/airline", - "source_data": { - "dataset_name": "tau-bench-2/airline", - "source_type": "url", - "url": [ - "https://github.com/Exgentic/exgentic" - ] - }, - "metric_config": { - "evaluation_description": "Tau Bench 2 benchmark evaluation (airline subset)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.74, - "uncertainty": { - "num_samples": 50 - }, - "details": { - "average_agent_cost": "0.72", - "total_run_cost": "36.55", - "average_steps": "12.22", - "percent_finished": "1.0" - } - }, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "OpenAI Solo", - "agent_framework": "openai_solo" - } - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "OpenAI Solo", - "agent_framework": "openai_solo" - } - } - } - } - }, - { - "evaluation_id": "tau-bench-2/airline/claude-code-cli__anthropic_claude-opus-4-5/1774263615.0201504", - "retrieved_timestamp": "1774263615.0201504", - "source_metadata": { - "source_name": "Exgentic Open Agent Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Exgentic", - "source_organization_url": "https://github.com/Exgentic", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "exgentic", - "version": "0.1.0" - }, - "benchmark": "tau-bench-2_airline", - "evaluation_results": [ - { - "evaluation_name": "tau-bench-2/airline", - "source_data": { - "dataset_name": "tau-bench-2/airline", - "source_type": "url", - "url": [ - "https://github.com/Exgentic/exgentic" - ] - }, - "metric_config": { - "evaluation_description": "Tau Bench 2 benchmark evaluation (airline subset)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.66, - "uncertainty": { - "num_samples": 50 - }, - "details": { - "average_agent_cost": "1.3", - "total_run_cost": "65.66", - "average_steps": "11.5", - "percent_finished": "1.0" - } - }, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "Claude Code CLI", - "agent_framework": "claude_code" - } - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "Claude Code CLI", - "agent_framework": "claude_code" - } - } - } - } - }, - { - "evaluation_id": "tau-bench-2/retail/claude-code-cli__anthropic_claude-opus-4-5/1774263615.0201504", - "retrieved_timestamp": "1774263615.0201504", - "source_metadata": { - "source_name": "Exgentic Open Agent Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Exgentic", - "source_organization_url": "https://github.com/Exgentic", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "exgentic", - "version": "0.1.0" - }, - "benchmark": "tau-bench-2_retail", - "evaluation_results": [ - { - "evaluation_name": "tau-bench-2/retail", - "source_data": { - "dataset_name": "tau-bench-2/retail", - "source_type": "url", - "url": [ - "https://github.com/Exgentic/exgentic" - ] - }, - "metric_config": { - "evaluation_description": "Tau Bench 2 benchmark evaluation (retail subset)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.83, - "uncertainty": { - "num_samples": 100 - }, - "details": { - "average_agent_cost": "1.6", - "total_run_cost": "161.14", - "average_steps": "12.54", - "percent_finished": "1.0" - } - }, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "Claude Code CLI", - "agent_framework": "claude_code" - } - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "Claude Code CLI", - "agent_framework": "claude_code" - } - } - } - } - }, - { - "evaluation_id": "tau-bench-2/retail/smolagents-code__anthropic_claude-opus-4-5/1774263615.0201504", - "retrieved_timestamp": "1774263615.0201504", - "source_metadata": { - "source_name": "Exgentic Open Agent Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Exgentic", - "source_organization_url": "https://github.com/Exgentic", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "exgentic", - "version": "0.1.0" - }, - "benchmark": "tau-bench-2_retail", - "evaluation_results": [ - { - "evaluation_name": "tau-bench-2/retail", - "source_data": { - "dataset_name": "tau-bench-2/retail", - "source_type": "url", - "url": [ - "https://github.com/Exgentic/exgentic" - ] - }, - "metric_config": { - "evaluation_description": "Tau Bench 2 benchmark evaluation (retail subset)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.78, - "uncertainty": { - "num_samples": 100 - }, - "details": { - "average_agent_cost": "0.67", - "total_run_cost": "68.24", - "average_steps": "11.71", - "percent_finished": "1.0" - } - }, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "SmolAgents Code", - "agent_framework": "smolagents_code" - } - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "SmolAgents Code", - "agent_framework": "smolagents_code" - } - } - } - } - }, - { - "evaluation_id": "tau-bench-2/retail/openai-solo__anthropic_claude-opus-4-5/1774263615.0201504", - "retrieved_timestamp": "1774263615.0201504", - "source_metadata": { - "source_name": "Exgentic Open Agent Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Exgentic", - "source_organization_url": "https://github.com/Exgentic", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "exgentic", - "version": "0.1.0" - }, - "benchmark": "tau-bench-2_retail", - "evaluation_results": [ - { - "evaluation_name": "tau-bench-2/retail", - "source_data": { - "dataset_name": "tau-bench-2/retail", - "source_type": "url", - "url": [ - "https://github.com/Exgentic/exgentic" - ] - }, - "metric_config": { - "evaluation_description": "Tau Bench 2 benchmark evaluation (retail subset)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.85, - "uncertainty": { - "num_samples": 100 - }, - "details": { - "average_agent_cost": "0.55", - "total_run_cost": "56.18", - "average_steps": "12.54", - "percent_finished": "1.0" - } - }, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "OpenAI Solo", - "agent_framework": "openai_solo" - } - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "OpenAI Solo", - "agent_framework": "openai_solo" - } - } - } - } - }, - { - "evaluation_id": "tau-bench-2/retail/litellm-tool-calling-with-shortlisting__anthropic_claude-opus-4-5/1774263615.0201504", - "retrieved_timestamp": "1774263615.0201504", - "source_metadata": { - "source_name": "Exgentic Open Agent Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Exgentic", - "source_organization_url": "https://github.com/Exgentic", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "exgentic", - "version": "0.1.0" - }, - "benchmark": "tau-bench-2_retail", - "evaluation_results": [ - { - "evaluation_name": "tau-bench-2/retail", - "source_data": { - "dataset_name": "tau-bench-2/retail", - "source_type": "url", - "url": [ - "https://github.com/Exgentic/exgentic" - ] - }, - "metric_config": { - "evaluation_description": "Tau Bench 2 benchmark evaluation (retail subset)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.78, - "uncertainty": { - "num_samples": 100 - }, - "details": { - "average_agent_cost": "0.47", - "total_run_cost": "48.01", - "average_steps": "11.33", - "percent_finished": "1.0" - } - }, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "LiteLLM Tool Calling with Shortlisting", - "agent_framework": "tool_calling_with_shortlisting" - } - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "LiteLLM Tool Calling with Shortlisting", - "agent_framework": "tool_calling_with_shortlisting" - } - } - } - } - }, - { - "evaluation_id": "tau-bench-2/retail/litellm-tool-calling__anthropic_claude-opus-4-5/1774263615.0201504", - "retrieved_timestamp": "1774263615.0201504", - "source_metadata": { - "source_name": "Exgentic Open Agent Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Exgentic", - "source_organization_url": "https://github.com/Exgentic", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "exgentic", - "version": "0.1.0" - }, - "benchmark": "tau-bench-2_retail", - "evaluation_results": [ - { - "evaluation_name": "tau-bench-2/retail", - "source_data": { - "dataset_name": "tau-bench-2/retail", - "source_type": "url", - "url": [ - "https://github.com/Exgentic/exgentic" - ] - }, - "metric_config": { - "evaluation_description": "Tau Bench 2 benchmark evaluation (retail subset)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.78, - "uncertainty": { - "num_samples": 100 - }, - "details": { - "average_agent_cost": "0.47", - "total_run_cost": "48.01", - "average_steps": "11.33", - "percent_finished": "1.0" - } - }, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "LiteLLM Tool Calling", - "agent_framework": "tool_calling" - } - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "LiteLLM Tool Calling", - "agent_framework": "tool_calling" - } - } - } - } - }, - { - "evaluation_id": "tau-bench-2/telecom/litellm-tool-calling__anthropic_claude-opus-4-5/1774263615.0201504", - "retrieved_timestamp": "1774263615.0201504", - "source_metadata": { - "source_name": "Exgentic Open Agent Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Exgentic", - "source_organization_url": "https://github.com/Exgentic", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "exgentic", - "version": "0.1.0" - }, - "benchmark": "tau-bench-2_telecom", - "evaluation_results": [ - { - "evaluation_name": "tau-bench-2/telecom", - "source_data": { - "dataset_name": "tau-bench-2/telecom", - "source_type": "url", - "url": [ - "https://github.com/Exgentic/exgentic" - ] - }, - "metric_config": { - "evaluation_description": "Tau Bench 2 benchmark evaluation (telecom subset)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.76, - "uncertainty": { - "num_samples": 100 - }, - "details": { - "average_agent_cost": "0.92", - "total_run_cost": "102.01", - "average_steps": "17.22", - "percent_finished": "1.0" - } - }, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "LiteLLM Tool Calling", - "agent_framework": "tool_calling" - } - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "LiteLLM Tool Calling", - "agent_framework": "tool_calling" - } - } - } - } - }, - { - "evaluation_id": "tau-bench-2/telecom/smolagents-code__anthropic_claude-opus-4-5/1774263615.0201504", - "retrieved_timestamp": "1774263615.0201504", - "source_metadata": { - "source_name": "Exgentic Open Agent Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Exgentic", - "source_organization_url": "https://github.com/Exgentic", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "exgentic", - "version": "0.1.0" - }, - "benchmark": "tau-bench-2_telecom", - "evaluation_results": [ - { - "evaluation_name": "tau-bench-2/telecom", - "source_data": { - "dataset_name": "tau-bench-2/telecom", - "source_type": "url", - "url": [ - "https://github.com/Exgentic/exgentic" - ] - }, - "metric_config": { - "evaluation_description": "Tau Bench 2 benchmark evaluation (telecom subset)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.58, - "uncertainty": { - "num_samples": 100 - }, - "details": { - "average_agent_cost": "1.06", - "total_run_cost": "114.62", - "average_steps": "13.77", - "percent_finished": "1.0" - } - }, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "SmolAgents Code", - "agent_framework": "smolagents_code" - } - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "SmolAgents Code", - "agent_framework": "smolagents_code" - } - } - } - } - }, - { - "evaluation_id": "tau-bench-2/telecom/litellm-tool-calling-with-shortlisting__anthropic_claude-opus-4-5/1774263615.0201504", - "retrieved_timestamp": "1774263615.0201504", - "source_metadata": { - "source_name": "Exgentic Open Agent Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Exgentic", - "source_organization_url": "https://github.com/Exgentic", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "exgentic", - "version": "0.1.0" - }, - "benchmark": "tau-bench-2_telecom", - "evaluation_results": [ - { - "evaluation_name": "tau-bench-2/telecom", - "source_data": { - "dataset_name": "tau-bench-2/telecom", - "source_type": "url", - "url": [ - "https://github.com/Exgentic/exgentic" - ] - }, - "metric_config": { - "evaluation_description": "Tau Bench 2 benchmark evaluation (telecom subset)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.76, - "uncertainty": { - "num_samples": 100 - }, - "details": { - "average_agent_cost": "0.92", - "total_run_cost": "102.01", - "average_steps": "17.22", - "percent_finished": "1.0" - } - }, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "LiteLLM Tool Calling with Shortlisting", - "agent_framework": "tool_calling_with_shortlisting" - } - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "LiteLLM Tool Calling with Shortlisting", - "agent_framework": "tool_calling_with_shortlisting" - } - } - } - } - }, - { - "evaluation_id": "tau-bench-2/telecom/claude-code-cli__anthropic_claude-opus-4-5/1774263615.0201504", - "retrieved_timestamp": "1774263615.0201504", - "source_metadata": { - "source_name": "Exgentic Open Agent Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Exgentic", - "source_organization_url": "https://github.com/Exgentic", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "exgentic", - "version": "0.1.0" - }, - "benchmark": "tau-bench-2_telecom", - "evaluation_results": [ - { - "evaluation_name": "tau-bench-2/telecom", - "source_data": { - "dataset_name": "tau-bench-2/telecom", - "source_type": "url", - "url": [ - "https://github.com/Exgentic/exgentic" - ] - }, - "metric_config": { - "evaluation_description": "Tau Bench 2 benchmark evaluation (telecom subset)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.76, - "uncertainty": { - "num_samples": 100 - }, - "details": { - "average_agent_cost": "2.45", - "total_run_cost": "255.97", - "average_steps": "18.71", - "percent_finished": "1.0" - } - }, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "Claude Code CLI", - "agent_framework": "claude_code" - } - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "Claude Code CLI", - "agent_framework": "claude_code" - } - } - } - } - }, - { - "evaluation_id": "tau-bench-2/telecom/openai-solo__anthropic_claude-opus-4-5/1774263615.0201504", - "retrieved_timestamp": "1774263615.0201504", - "source_metadata": { - "source_name": "Exgentic Open Agent Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Exgentic", - "source_organization_url": "https://github.com/Exgentic", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "exgentic", - "version": "0.1.0" - }, - "benchmark": "tau-bench-2_telecom", - "evaluation_results": [ - { - "evaluation_name": "tau-bench-2/telecom", - "source_data": { - "dataset_name": "tau-bench-2/telecom", - "source_type": "url", - "url": [ - "https://github.com/Exgentic/exgentic" - ] - }, - "metric_config": { - "evaluation_description": "Tau Bench 2 benchmark evaluation (telecom subset)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.84, - "uncertainty": { - "num_samples": 100 - }, - "details": { - "average_agent_cost": "1.25", - "total_run_cost": "136.84", - "average_steps": "17.15", - "percent_finished": "1.0" - } - }, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "OpenAI Solo", - "agent_framework": "openai_solo" - } - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "OpenAI Solo", - "agent_framework": "openai_solo" - } - } - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/anthropic_claude-opus-4.1.json b/data/models/anthropic_claude-opus-4.1.json deleted file mode 100644 index b51256b4d5a95f9160a82aa03d45fc55ec0aba09..0000000000000000000000000000000000000000 --- a/data/models/anthropic_claude-opus-4.1.json +++ /dev/null @@ -1,309 +0,0 @@ -{ - "model_info": { - "name": "Claude Opus 4.1", - "id": "anthropic/claude-opus-4.1", - "developer": "Anthropic", - "additional_details": { - "agent_name": "OpenHands", - "agent_organization": "OpenHands" - } - }, - "evaluations": [ - { - "evaluation_id": "terminal-bench-2.0/openhands__claude-opus-4.1/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2025-11-02", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 36.9, - "uncertainty": { - "standard_error": { - "value": 2.7 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"OpenHands\" -m \"Claude Opus 4.1\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"OpenHands\" -m \"Claude Opus 4.1\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - }, - { - "evaluation_id": "terminal-bench-2.0/terminus-2__claude-opus-4.1/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2025-10-31", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 38.0, - "uncertainty": { - "standard_error": { - "value": 2.6 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Terminus 2\" -m \"Claude Opus 4.1\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Terminus 2\" -m \"Claude Opus 4.1\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - }, - { - "evaluation_id": "terminal-bench-2.0/claude-code__claude-opus-4.1/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2025-11-04", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 34.8, - "uncertainty": { - "standard_error": { - "value": 2.9 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Claude Code\" -m \"Claude Opus 4.1\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Claude Code\" -m \"Claude Opus 4.1\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - }, - { - "evaluation_id": "terminal-bench-2.0/mini-swe-agent__claude-opus-4.1/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2025-11-03", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 35.1, - "uncertainty": { - "standard_error": { - "value": 2.5 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Mini-SWE-Agent\" -m \"Claude Opus 4.1\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Mini-SWE-Agent\" -m \"Claude Opus 4.1\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/anthropic_claude-opus-4.5.json b/data/models/anthropic_claude-opus-4.5.json deleted file mode 100644 index 52bb36b61cb26c455b2891659f2ee58eab6b8b4e..0000000000000000000000000000000000000000 --- a/data/models/anthropic_claude-opus-4.5.json +++ /dev/null @@ -1,593 +0,0 @@ -{ - "model_info": { - "name": "Claude Opus 4.5", - "id": "anthropic/claude-opus-4.5", - "developer": "Anthropic", - "additional_details": { - "agent_name": "OpenCode", - "agent_organization": "Anomaly Innovations" - } - }, - "evaluations": [ - { - "evaluation_id": "terminal-bench-2.0/opencode__claude-opus-4.5/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2026-01-12", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 51.7 - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"OpenCode\" -m \"Claude Opus 4.5\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"OpenCode\" -m \"Claude Opus 4.5\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - }, - { - "evaluation_id": "terminal-bench-2.0/terminus-2__claude-opus-4.5/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2025-11-22", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 57.8, - "uncertainty": { - "standard_error": { - "value": 2.5 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Terminus 2\" -m \"Claude Opus 4.5\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Terminus 2\" -m \"Claude Opus 4.5\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - }, - { - "evaluation_id": "terminal-bench-2.0/letta-code__claude-opus-4.5/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2025-12-17", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 59.1, - "uncertainty": { - "standard_error": { - "value": 2.4 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Letta Code\" -m \"Claude Opus 4.5\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Letta Code\" -m \"Claude Opus 4.5\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - }, - { - "evaluation_id": "terminal-bench-2.0/goose__claude-opus-4.5/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2025-12-11", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 54.3, - "uncertainty": { - "standard_error": { - "value": 2.6 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Goose\" -m \"Claude Opus 4.5\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Goose\" -m \"Claude Opus 4.5\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - }, - { - "evaluation_id": "terminal-bench-2.0/claude-code__claude-opus-4.5/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2025-12-18", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 52.1, - "uncertainty": { - "standard_error": { - "value": 2.5 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Claude Code\" -m \"Claude Opus 4.5\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Claude Code\" -m \"Claude Opus 4.5\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - }, - { - "evaluation_id": "terminal-bench-2.0/mux__claude-opus-4.5/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2026-01-17", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 58.4 - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Mux\" -m \"Claude Opus 4.5\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Mux\" -m \"Claude Opus 4.5\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - }, - { - "evaluation_id": "terminal-bench-2.0/openhands__claude-opus-4.5/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2026-01-04", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 51.9, - "uncertainty": { - "standard_error": { - "value": 2.9 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"OpenHands\" -m \"Claude Opus 4.5\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"OpenHands\" -m \"Claude Opus 4.5\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - }, - { - "evaluation_id": "terminal-bench-2.0/droid__claude-opus-4.5/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2025-12-11", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 63.1, - "uncertainty": { - "standard_error": { - "value": 2.7 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Droid\" -m \"Claude Opus 4.5\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Droid\" -m \"Claude Opus 4.5\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/anthropic_claude-opus-4.6.json b/data/models/anthropic_claude-opus-4.6.json deleted file mode 100644 index 425cec4998f043aae8197024b92f5a18cb4b27e4..0000000000000000000000000000000000000000 --- a/data/models/anthropic_claude-opus-4.6.json +++ /dev/null @@ -1,525 +0,0 @@ -{ - "model_info": { - "name": "Claude Opus 4.6", - "id": "anthropic/claude-opus-4.6", - "developer": "Anthropic", - "additional_details": { - "agent_name": "Mux", - "agent_organization": "Coder" - } - }, - "evaluations": [ - { - "evaluation_id": "terminal-bench-2.0/mux__claude-opus-4.6/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2026-02-13", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 66.5, - "uncertainty": { - "standard_error": { - "value": 2.5 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Mux\" -m \"Claude Opus 4.6\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Mux\" -m \"Claude Opus 4.6\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - }, - { - "evaluation_id": "terminal-bench-2.0/terminus-2__claude-opus-4.6/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2026-02-06", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 62.9, - "uncertainty": { - "standard_error": { - "value": 2.7 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Terminus 2\" -m \"Claude Opus 4.6\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Terminus 2\" -m \"Claude Opus 4.6\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - }, - { - "evaluation_id": "terminal-bench-2.0/tongagents__claude-opus-4.6/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2026-02-22", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 71.9, - "uncertainty": { - "standard_error": { - "value": 2.7 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"TongAgents\" -m \"Claude Opus 4.6\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"TongAgents\" -m \"Claude Opus 4.6\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - }, - { - "evaluation_id": "terminal-bench-2.0/droid__claude-opus-4.6/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2026-02-05", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 69.9, - "uncertainty": { - "standard_error": { - "value": 2.5 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Droid\" -m \"Claude Opus 4.6\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Droid\" -m \"Claude Opus 4.6\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - }, - { - "evaluation_id": "terminal-bench-2.0/crux__claude-opus-4.6/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2026-02-23", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 66.9 - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Crux\" -m \"Claude Opus 4.6\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Crux\" -m \"Claude Opus 4.6\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - }, - { - "evaluation_id": "terminal-bench-2.0/claude-code__claude-opus-4.6/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2026-02-07", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 58.0, - "uncertainty": { - "standard_error": { - "value": 2.9 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Claude Code\" -m \"Claude Opus 4.6\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Claude Code\" -m \"Claude Opus 4.6\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - }, - { - "evaluation_id": "terminal-bench-2.0/terminus-kira__claude-opus-4.6/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2026-02-22", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 74.7, - "uncertainty": { - "standard_error": { - "value": 2.6 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Terminus-KIRA\" -m \"Claude Opus 4.6\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Terminus-KIRA\" -m \"Claude Opus 4.6\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/anthropic_claude-sonnet-4-20250514-thinking-10k.json b/data/models/anthropic_claude-sonnet-4-20250514-thinking-10k.json deleted file mode 100644 index 23eb8c4ce69806441e51475c161b42d4398dc9d8..0000000000000000000000000000000000000000 --- a/data/models/anthropic_claude-sonnet-4-20250514-thinking-10k.json +++ /dev/null @@ -1,234 +0,0 @@ -{ - "model_info": { - "name": "Claude 4 Sonnet 20250514, extended thinking", - "id": "anthropic/claude-sonnet-4-20250514-thinking-10k", - "developer": "Anthropic", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_capabilities/anthropic_claude-sonnet-4-20250514-thinking-10k/1774096304.5056698", - "retrieved_timestamp": "1774096304.5056698", - "source_metadata": { - "source_name": "helm_capabilities", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_capabilities", - "evaluation_results": [ - { - "evaluation_name": "Mean score", - "source_data": { - "dataset_name": "helm_capabilities", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "The mean of the scores from all columns.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.766, - "details": { - "description": "", - "tab": "Accuracy", - "Mean score - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"38.96330262736815\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU-Pro", - "source_data": { - "dataset_name": "MMLU-Pro", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on MMLU-Pro", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.843, - "details": { - "description": "min=0.843, mean=0.843, max=0.843, sum=0.843 (1)", - "tab": "Accuracy", - "MMLU-Pro - Observed inference time (s)": "{\"description\": \"min=23.165, mean=23.165, max=23.165, sum=23.165 (1)\", \"tab\": \"Efficiency\", \"score\": \"23.16487550187111\"}", - "MMLU-Pro - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "MMLU-Pro - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - # prompt tokens": "{\"description\": \"min=252.461, mean=252.461, max=252.461, sum=252.461 (1)\", \"tab\": \"General information\", \"score\": \"252.461\"}", - "MMLU-Pro - # output tokens": "{\"description\": \"min=325.194, mean=325.194, max=325.194, sum=325.194 (1)\", \"tab\": \"General information\", \"score\": \"325.194\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"all\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.706, - "details": { - "description": "min=0.706, mean=0.706, max=0.706, sum=0.706 (1)", - "tab": "Accuracy", - "GPQA - Observed inference time (s)": "{\"description\": \"min=38.16, mean=38.16, max=38.16, sum=38.16 (1)\", \"tab\": \"Efficiency\", \"score\": \"38.15993662211927\"}", - "GPQA - # eval": "{\"description\": \"min=446, mean=446, max=446, sum=446 (1)\", \"tab\": \"General information\", \"score\": \"446.0\"}", - "GPQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - # prompt tokens": "{\"description\": \"min=272.738, mean=272.738, max=272.738, sum=272.738 (1)\", \"tab\": \"General information\", \"score\": \"272.73766816143495\"}", - "GPQA - # output tokens": "{\"description\": \"min=414.928, mean=414.928, max=414.928, sum=414.928 (1)\", \"tab\": \"General information\", \"score\": \"414.92825112107624\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"gpqa_main\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "IFEval Strict Acc on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.84, - "details": { - "description": "min=0.84, mean=0.84, max=0.84, sum=0.84 (1)", - "tab": "Accuracy", - "IFEval - Observed inference time (s)": "{\"description\": \"min=12.654, mean=12.654, max=12.654, sum=12.654 (1)\", \"tab\": \"Efficiency\", \"score\": \"12.65442304822742\"}", - "IFEval - # eval": "{\"description\": \"min=541, mean=541, max=541, sum=541 (1)\", \"tab\": \"General information\", \"score\": \"541.0\"}", - "IFEval - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - # prompt tokens": "{\"description\": \"min=47.159, mean=47.159, max=47.159, sum=47.159 (1)\", \"tab\": \"General information\", \"score\": \"47.15896487985213\"}", - "IFEval - # output tokens": "{\"description\": \"min=380.645, mean=380.645, max=380.645, sum=380.645 (1)\", \"tab\": \"General information\", \"score\": \"380.64510166358593\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WildBench", - "source_data": { - "dataset_name": "WildBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "WB Score on WildBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.838, - "details": { - "description": "min=0.838, mean=0.838, max=0.838, sum=0.838 (1)", - "tab": "Accuracy", - "WildBench - Observed inference time (s)": "{\"description\": \"min=32.933, mean=32.933, max=32.933, sum=32.933 (1)\", \"tab\": \"Efficiency\", \"score\": \"32.93274651098251\"}", - "WildBench - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "WildBench - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # prompt tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # output tokens": "{\"description\": \"min=1274.627, mean=1274.627, max=1274.627, sum=1274.627 (1)\", \"tab\": \"General information\", \"score\": \"1274.627\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"v2\"" - } - } - }, - { - "evaluation_name": "Omni-MATH", - "source_data": { - "dataset_name": "Omni-MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Acc on Omni-MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.602, - "details": { - "description": "min=0.602, mean=0.602, max=0.602, sum=0.602 (1)", - "tab": "Accuracy", - "Omni-MATH - Observed inference time (s)": "{\"description\": \"min=87.905, mean=87.905, max=87.905, sum=87.905 (1)\", \"tab\": \"Efficiency\", \"score\": \"87.90453145364046\"}", - "Omni-MATH - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "Omni-MATH - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - # prompt tokens": "{\"description\": \"min=110.563, mean=110.563, max=110.563, sum=110.563 (1)\", \"tab\": \"General information\", \"score\": \"110.563\"}", - "Omni-MATH - # output tokens": "{\"description\": \"min=728.241, mean=728.241, max=728.241, sum=728.241 (1)\", \"tab\": \"General information\", \"score\": \"728.241\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/anthropic_claude-sonnet-4-20250514.json b/data/models/anthropic_claude-sonnet-4-20250514.json deleted file mode 100644 index a43572d10d77d034b8cc0b4e9e80cb595c19f907..0000000000000000000000000000000000000000 --- a/data/models/anthropic_claude-sonnet-4-20250514.json +++ /dev/null @@ -1,1417 +0,0 @@ -{ - "model_info": { - "name": "claude-sonnet-4-20250514", - "id": "anthropic/claude-sonnet-4-20250514", - "developer": "Anthropic", - "inference_platform": "unknown", - "additional_details": { - "display_name": "Claude Sonnet 4" - } - }, - "evaluations": [ - { - "evaluation_id": "global-mmlu-lite/anthropic_claude-sonnet-4-20250514/1773936496.366405", - "retrieved_timestamp": "1773936496.366405", - "source_metadata": { - "source_name": "Global MMLU Lite Leaderboard", - "source_type": "documentation", - "source_organization_name": "kaggle", - "source_organization_url": "www.kaggle.com", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "kaggle kernel", - "version": "4", - "additional_details": { - "url": "https://www.kaggle.com/code/shivalikasingh95/global-mmlu-lite-sample-notebook" - } - }, - "benchmark": "global-mmlu-lite", - "evaluation_results": [ - { - "evaluation_name": "Global MMLU Lite", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Global MMLU Lite", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9058 - } - }, - { - "evaluation_name": "Culturally Sensitive", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Sensitive", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8913 - } - }, - { - "evaluation_name": "Culturally Agnostic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Agnostic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9203 - } - }, - { - "evaluation_name": "Arabic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Arabic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9125, - "uncertainty": { - "confidence_interval": { - "lower": -0.0277, - "upper": 0.0277, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "English", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - English", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.905, - "uncertainty": { - "confidence_interval": { - "lower": -0.0287, - "upper": 0.0287, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Bengali", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Bengali", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9075, - "uncertainty": { - "confidence_interval": { - "lower": -0.0284, - "upper": 0.0284, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "German", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - German", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9125, - "uncertainty": { - "confidence_interval": { - "lower": -0.0277, - "upper": 0.0277, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "French", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - French", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.91, - "uncertainty": { - "confidence_interval": { - "lower": -0.028, - "upper": 0.028, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Hindi", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Hindi", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9, - "uncertainty": { - "confidence_interval": { - "lower": -0.0294, - "upper": 0.0294, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Indonesian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Indonesian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9025, - "uncertainty": { - "confidence_interval": { - "lower": -0.0291, - "upper": 0.0291, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Italian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Italian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9075, - "uncertainty": { - "confidence_interval": { - "lower": -0.0284, - "upper": 0.0284, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Japanese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Japanese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9, - "uncertainty": { - "confidence_interval": { - "lower": -0.0294, - "upper": 0.0294, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Korean", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Korean", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9125, - "uncertainty": { - "confidence_interval": { - "lower": -0.0277, - "upper": 0.0277, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Portuguese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Portuguese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.91, - "uncertainty": { - "confidence_interval": { - "lower": -0.028, - "upper": 0.028, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Spanish", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Spanish", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9075, - "uncertainty": { - "confidence_interval": { - "lower": -0.0284, - "upper": 0.0284, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Swahili", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Swahili", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8975, - "uncertainty": { - "confidence_interval": { - "lower": -0.0297, - "upper": 0.0297, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Yoruba", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Yoruba", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8975, - "uncertainty": { - "confidence_interval": { - "lower": -0.0297, - "upper": 0.0297, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Chinese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Chinese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9175, - "uncertainty": { - "confidence_interval": { - "lower": -0.027, - "upper": 0.027, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Burmese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Burmese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8925, - "uncertainty": { - "confidence_interval": { - "lower": -0.0304, - "upper": 0.0304, - "method": "unknown" - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "global-mmlu-lite/anthropic_claude-sonnet-4-20250514/1773936583.743359", - "retrieved_timestamp": "1773936583.743359", - "source_metadata": { - "source_name": "Global MMLU Lite Leaderboard", - "source_type": "documentation", - "source_organization_name": "kaggle", - "source_organization_url": "www.kaggle.com", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "kaggle kernel", - "version": "4", - "additional_details": { - "url": "https://www.kaggle.com/code/shivalikasingh95/global-mmlu-lite-sample-notebook" - } - }, - "benchmark": "global-mmlu-lite", - "evaluation_results": [ - { - "evaluation_name": "Global MMLU Lite", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Global MMLU Lite", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9058 - } - }, - { - "evaluation_name": "Culturally Sensitive", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Sensitive", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8913 - } - }, - { - "evaluation_name": "Culturally Agnostic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Agnostic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9203 - } - }, - { - "evaluation_name": "Arabic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Arabic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9125, - "uncertainty": { - "confidence_interval": { - "lower": -0.0277, - "upper": 0.0277, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "English", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - English", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.905, - "uncertainty": { - "confidence_interval": { - "lower": -0.0287, - "upper": 0.0287, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Bengali", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Bengali", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9075, - "uncertainty": { - "confidence_interval": { - "lower": -0.0284, - "upper": 0.0284, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "German", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - German", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9125, - "uncertainty": { - "confidence_interval": { - "lower": -0.0277, - "upper": 0.0277, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "French", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - French", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.91, - "uncertainty": { - "confidence_interval": { - "lower": -0.028, - "upper": 0.028, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Hindi", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Hindi", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9, - "uncertainty": { - "confidence_interval": { - "lower": -0.0294, - "upper": 0.0294, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Indonesian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Indonesian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9025, - "uncertainty": { - "confidence_interval": { - "lower": -0.0291, - "upper": 0.0291, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Italian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Italian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9075, - "uncertainty": { - "confidence_interval": { - "lower": -0.0284, - "upper": 0.0284, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Japanese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Japanese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9, - "uncertainty": { - "confidence_interval": { - "lower": -0.0294, - "upper": 0.0294, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Korean", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Korean", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9125, - "uncertainty": { - "confidence_interval": { - "lower": -0.0277, - "upper": 0.0277, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Portuguese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Portuguese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.91, - "uncertainty": { - "confidence_interval": { - "lower": -0.028, - "upper": 0.028, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Spanish", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Spanish", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9075, - "uncertainty": { - "confidence_interval": { - "lower": -0.0284, - "upper": 0.0284, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Swahili", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Swahili", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8975, - "uncertainty": { - "confidence_interval": { - "lower": -0.0297, - "upper": 0.0297, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Yoruba", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Yoruba", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8975, - "uncertainty": { - "confidence_interval": { - "lower": -0.0297, - "upper": 0.0297, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Chinese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Chinese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9175, - "uncertainty": { - "confidence_interval": { - "lower": -0.027, - "upper": 0.027, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Burmese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Burmese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8925, - "uncertainty": { - "confidence_interval": { - "lower": -0.0304, - "upper": 0.0304, - "method": "unknown" - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "helm_capabilities/anthropic_claude-sonnet-4-20250514/1774096304.5056698", - "retrieved_timestamp": "1774096304.5056698", - "source_metadata": { - "source_name": "helm_capabilities", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_capabilities", - "evaluation_results": [ - { - "evaluation_name": "Mean score", - "source_data": { - "dataset_name": "helm_capabilities", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "The mean of the scores from all columns.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.733, - "details": { - "description": "", - "tab": "Accuracy", - "Mean score - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"15.534070909101748\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU-Pro", - "source_data": { - "dataset_name": "MMLU-Pro", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on MMLU-Pro", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.843, - "details": { - "description": "min=0.843, mean=0.843, max=0.843, sum=0.843 (1)", - "tab": "Accuracy", - "MMLU-Pro - Observed inference time (s)": "{\"description\": \"min=9.974, mean=9.974, max=9.974, sum=9.974 (1)\", \"tab\": \"Efficiency\", \"score\": \"9.973703570604325\"}", - "MMLU-Pro - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "MMLU-Pro - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - # prompt tokens": "{\"description\": \"min=230.461, mean=230.461, max=230.461, sum=230.461 (1)\", \"tab\": \"General information\", \"score\": \"230.461\"}", - "MMLU-Pro - # output tokens": "{\"description\": \"min=402.003, mean=402.003, max=402.003, sum=402.003 (1)\", \"tab\": \"General information\", \"score\": \"402.003\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"all\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.643, - "details": { - "description": "min=0.643, mean=0.643, max=0.643, sum=0.643 (1)", - "tab": "Accuracy", - "GPQA - Observed inference time (s)": "{\"description\": \"min=13.452, mean=13.452, max=13.452, sum=13.452 (1)\", \"tab\": \"Efficiency\", \"score\": \"13.452103998094396\"}", - "GPQA - # eval": "{\"description\": \"min=446, mean=446, max=446, sum=446 (1)\", \"tab\": \"General information\", \"score\": \"446.0\"}", - "GPQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - # prompt tokens": "{\"description\": \"min=250.738, mean=250.738, max=250.738, sum=250.738 (1)\", \"tab\": \"General information\", \"score\": \"250.73766816143498\"}", - "GPQA - # output tokens": "{\"description\": \"min=543.482, mean=543.482, max=543.482, sum=543.482 (1)\", \"tab\": \"General information\", \"score\": \"543.4820627802691\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"gpqa_main\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "IFEval Strict Acc on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.839, - "details": { - "description": "min=0.839, mean=0.839, max=0.839, sum=0.839 (1)", - "tab": "Accuracy", - "IFEval - Observed inference time (s)": "{\"description\": \"min=10.416, mean=10.416, max=10.416, sum=10.416 (1)\", \"tab\": \"Efficiency\", \"score\": \"10.416161362653298\"}", - "IFEval - # eval": "{\"description\": \"min=541, mean=541, max=541, sum=541 (1)\", \"tab\": \"General information\", \"score\": \"541.0\"}", - "IFEval - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - # prompt tokens": "{\"description\": \"min=47.159, mean=47.159, max=47.159, sum=47.159 (1)\", \"tab\": \"General information\", \"score\": \"47.15896487985213\"}", - "IFEval - # output tokens": "{\"description\": \"min=398.978, mean=398.978, max=398.978, sum=398.978 (1)\", \"tab\": \"General information\", \"score\": \"398.9778188539741\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WildBench", - "source_data": { - "dataset_name": "WildBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "WB Score on WildBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.825, - "details": { - "description": "min=0.825, mean=0.825, max=0.825, sum=0.825 (1)", - "tab": "Accuracy", - "WildBench - Observed inference time (s)": "{\"description\": \"min=23.404, mean=23.404, max=23.404, sum=23.404 (1)\", \"tab\": \"Efficiency\", \"score\": \"23.403768165826797\"}", - "WildBench - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "WildBench - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # prompt tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # output tokens": "{\"description\": \"min=954.675, mean=954.675, max=954.675, sum=954.675 (1)\", \"tab\": \"General information\", \"score\": \"954.675\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"v2\"" - } - } - }, - { - "evaluation_name": "Omni-MATH", - "source_data": { - "dataset_name": "Omni-MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Acc on Omni-MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.512, - "details": { - "description": "min=0.512, mean=0.512, max=0.512, sum=0.512 (1)", - "tab": "Accuracy", - "Omni-MATH - Observed inference time (s)": "{\"description\": \"min=20.425, mean=20.425, max=20.425, sum=20.425 (1)\", \"tab\": \"Efficiency\", \"score\": \"20.424617448329926\"}", - "Omni-MATH - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "Omni-MATH - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - # prompt tokens": "{\"description\": \"min=110.563, mean=110.563, max=110.563, sum=110.563 (1)\", \"tab\": \"General information\", \"score\": \"110.563\"}", - "Omni-MATH - # output tokens": "{\"description\": \"min=925.604, mean=925.604, max=925.604, sum=925.604 (1)\", \"tab\": \"General information\", \"score\": \"925.604\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "reward-bench-2/anthropic_claude-sonnet-4-20250514/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7117 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7612 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3594 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7049 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8909 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7596 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7939 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/anthropic_claude-sonnet-4-5-20250929-fc.json b/data/models/anthropic_claude-sonnet-4-5-20250929-fc.json deleted file mode 100644 index ec5d3b99df56e06e5b9960727f46e0725700ba46..0000000000000000000000000000000000000000 --- a/data/models/anthropic_claude-sonnet-4-5-20250929-fc.json +++ /dev/null @@ -1,851 +0,0 @@ -{ - "model_info": { - "name": "Claude-Sonnet-4-5-20250929 (FC)", - "id": "anthropic/claude-sonnet-4-5-20250929-fc", - "developer": "Anthropic", - "additional_details": { - "raw_model_name": "Claude-Sonnet-4-5-20250929 (FC)", - "organization": "Anthropic", - "license": "Proprietary", - "mode": "FC", - "model_link": "https://www.anthropic.com/news/claude-sonnet-4-5" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/anthropic/claude-sonnet-4-5-20250929-fc/1775236112.3664482", - "retrieved_timestamp": "1775236112.3664482", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 2.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 73.24 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 43.73 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 4.31 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 4.43 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 7.27 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 88.65 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 72.58 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 95.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 94.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 92.0 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 81.13 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 89.53 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 78.92 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 87.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 83.33 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 61.37 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 69.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 65.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 52.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 59.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 81.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 82.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 80.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 64.95 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 54.19 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 57.42 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 83.23 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 68.75 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 86.61 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/anthropic_claude-sonnet-4-5-20250929-prompt.json b/data/models/anthropic_claude-sonnet-4-5-20250929-prompt.json deleted file mode 100644 index 917b60df2638587c4b18b33df2df0ac33f5b3b81..0000000000000000000000000000000000000000 --- a/data/models/anthropic_claude-sonnet-4-5-20250929-prompt.json +++ /dev/null @@ -1,905 +0,0 @@ -{ - "model_info": { - "name": "Claude-Sonnet-4-5-20250929 (Prompt)", - "id": "anthropic/claude-sonnet-4-5-20250929-prompt", - "developer": "Anthropic", - "additional_details": { - "raw_model_name": "Claude-Sonnet-4-5-20250929 (Prompt)", - "organization": "Anthropic", - "license": "Proprietary", - "mode": "Prompt", - "model_link": "https://www.anthropic.com/news/claude-sonnet-4-5" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/anthropic/claude-sonnet-4-5-20250929-prompt/1775236112.41268", - "retrieved_timestamp": "1775236112.41268", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 89.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 24.9 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 47.82 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 3.84 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 1.53 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 6.66 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 59.81 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 47.25 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 79.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 53.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 59.0 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 46.56 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 73.26 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 40.17 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 56.25 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 33.33 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 1.62 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 2.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 3.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 1.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 16.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 16.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 16.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 5.38 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 4.52 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 9.68 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 1.94 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 37.5 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 95.03 - } - }, - { - "evaluation_result_id": "bfcl.format_sensitivity.max_delta::max_delta", - "evaluation_name": "bfcl.format_sensitivity.max_delta", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.format_sensitivity.max_delta", - "metric_name": "Format sensitivity max delta", - "metric_kind": "difference", - "metric_unit": "percentage_points", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Format Sensitivity Max Delta" - } - }, - "score_details": { - "score": 37.5 - } - }, - { - "evaluation_result_id": "bfcl.format_sensitivity.stddev::stddev", - "evaluation_name": "bfcl.format_sensitivity.stddev", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.format_sensitivity.stddev", - "metric_name": "Format sensitivity standard deviation", - "metric_kind": "difference", - "metric_unit": "percentage_points", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Format Sensitivity Standard Deviation" - } - }, - "score_details": { - "score": 10.07 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/anthropic_claude-sonnet-4-5-20250929.json b/data/models/anthropic_claude-sonnet-4-5-20250929.json deleted file mode 100644 index a8e5116b099e4ac25070d136600856a7e0010292..0000000000000000000000000000000000000000 --- a/data/models/anthropic_claude-sonnet-4-5-20250929.json +++ /dev/null @@ -1,89 +0,0 @@ -{ - "model_info": { - "name": "claude-sonnet-4-5-20250929", - "id": "anthropic/claude-sonnet-4-5-20250929", - "developer": "Anthropic", - "inference_platform": "anthropic" - }, - "evaluations": [ - { - "evaluation_id": "livecodebenchpro/claude-sonnet-4-5-20250929/1770683238.099205", - "retrieved_timestamp": "1770683238.099205", - "source_metadata": { - "source_name": "Live Code Bench Pro", - "source_type": "documentation", - "source_organization_name": "New York University, Princeton University, University of California San Diego, University of Washington and Canyon Crest Academy", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "unknown", - "version": "unknown" - }, - "benchmark": "livecodebenchpro", - "evaluation_results": [ - { - "evaluation_name": "Hard Problems", - "source_data": { - "dataset_name": "Hard Problems", - "source_type": "url", - "url": [ - "https://webhook.cp-bench.orzzh.com/leaderboard/llm/difficulty?difficulty=hard&benchmark_mode=live" - ] - }, - "metric_config": { - "evaluation_description": "Pass@1 on Hard Problems", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "Medium Problems", - "source_data": { - "dataset_name": "Medium Problems", - "source_type": "url", - "url": [ - "https://webhook.cp-bench.orzzh.com/leaderboard/llm/difficulty?difficulty=medium&benchmark_mode=live" - ] - }, - "metric_config": { - "evaluation_description": "Pass@1 on Medium Problems", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "Easy Problems", - "source_data": { - "dataset_name": "Easy Problems", - "source_type": "url", - "url": [ - "https://webhook.cp-bench.orzzh.com/leaderboard/llm/difficulty?difficulty=easy&benchmark_mode=live" - ] - }, - "metric_config": { - "evaluation_description": "Pass@1 on Easy Problems", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5352 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/anthropic_claude-sonnet-4.5.json b/data/models/anthropic_claude-sonnet-4.5.json deleted file mode 100644 index b69753164631c59138343ca295c2d8f1b0a89deb..0000000000000000000000000000000000000000 --- a/data/models/anthropic_claude-sonnet-4.5.json +++ /dev/null @@ -1,525 +0,0 @@ -{ - "model_info": { - "name": "Claude Sonnet 4.5", - "id": "anthropic/claude-sonnet-4.5", - "developer": "Anthropic", - "additional_details": { - "agent_name": "CAMEL-AI", - "agent_organization": "CAMEL-AI" - } - }, - "evaluations": [ - { - "evaluation_id": "terminal-bench-2.0/camel-ai__claude-sonnet-4.5/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2025-12-24", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 46.5, - "uncertainty": { - "standard_error": { - "value": 2.4 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"CAMEL-AI\" -m \"Claude Sonnet 4.5\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"CAMEL-AI\" -m \"Claude Sonnet 4.5\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - }, - { - "evaluation_id": "terminal-bench-2.0/terminus-2__claude-sonnet-4.5/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2025-10-31", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 42.8, - "uncertainty": { - "standard_error": { - "value": 2.8 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Terminus 2\" -m \"Claude Sonnet 4.5\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Terminus 2\" -m \"Claude Sonnet 4.5\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - }, - { - "evaluation_id": "terminal-bench-2.0/claude-code__claude-sonnet-4.5/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2025-11-04", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 40.1, - "uncertainty": { - "standard_error": { - "value": 2.9 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Claude Code\" -m \"Claude Sonnet 4.5\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Claude Code\" -m \"Claude Sonnet 4.5\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - }, - { - "evaluation_id": "terminal-bench-2.0/openhands__claude-sonnet-4.5/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2025-11-02", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 42.6, - "uncertainty": { - "standard_error": { - "value": 2.8 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"OpenHands\" -m \"Claude Sonnet 4.5\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"OpenHands\" -m \"Claude Sonnet 4.5\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - }, - { - "evaluation_id": "terminal-bench-2.0/goose__claude-sonnet-4.5/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2025-12-11", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 43.1, - "uncertainty": { - "standard_error": { - "value": 2.6 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Goose\" -m \"Claude Sonnet 4.5\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Goose\" -m \"Claude Sonnet 4.5\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - }, - { - "evaluation_id": "terminal-bench-2.0/maya__claude-sonnet-4.5/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2026-01-04", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 42.7 - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"MAYA\" -m \"Claude Sonnet 4.5\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"MAYA\" -m \"Claude Sonnet 4.5\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - }, - { - "evaluation_id": "terminal-bench-2.0/mini-swe-agent__claude-sonnet-4.5/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2025-11-03", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 42.5, - "uncertainty": { - "standard_error": { - "value": 2.8 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Mini-SWE-Agent\" -m \"Claude Sonnet 4.5\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Mini-SWE-Agent\" -m \"Claude Sonnet 4.5\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/anthropic_claude-v1.3.json b/data/models/anthropic_claude-v1.3.json deleted file mode 100644 index c062362b00aa9930f18ddff8872575a4f2f2a120..0000000000000000000000000000000000000000 --- a/data/models/anthropic_claude-v1.3.json +++ /dev/null @@ -1,607 +0,0 @@ -{ - "model_info": { - "name": "Anthropic Claude v1.3", - "id": "anthropic/claude-v1.3", - "developer": "Anthropic", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_instruct/anthropic_claude-v1.3/1774096309.537868", - "retrieved_timestamp": "1774096309.537868", - "source_metadata": { - "source_name": "helm_instruct", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_instruct", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_instruct", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/instruct/benchmark_output/releases/v1.0.0/groups/instruction_following.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperform on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.611, - "details": { - "description": "", - "tab": "Instruction Following" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "Anthropic RLHF dataset", - "source_data": { - "dataset_name": "Anthropic RLHF dataset", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/instruct/benchmark_output/releases/v1.0.0/groups/instruction_following.json" - ] - }, - "metric_config": { - "evaluation_description": "Harmlessness on Anthropic RLHF dataset", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 5.0 - }, - "score_details": { - "score": 4.965, - "details": { - "description": "min=4.925, mean=4.965, max=5, sum=39.72 (8)", - "tab": "Instruction Following" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"hh\", \"hh\", \"hh\", \"hh\", \"red_team\", \"red_team\", \"red_team\", \"red_team\"]", - "evaluator": "[\"claude\", \"gpt4\", \"mturk\", \"scale\", \"claude\", \"gpt4\", \"mturk\", \"scale\"]" - } - } - }, - { - "evaluation_name": "Best ChatGPT Prompts", - "source_data": { - "dataset_name": "Best ChatGPT Prompts", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/instruct/benchmark_output/releases/v1.0.0/groups/instruction_following.json" - ] - }, - "metric_config": { - "evaluation_description": "Harmlessness on Best ChatGPT Prompts", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 5.0 - }, - "score_details": { - "score": 4.995, - "details": { - "description": "min=4.985, mean=4.995, max=5, sum=19.98 (4)", - "tab": "Instruction Following" - } - }, - "generation_config": { - "additional_details": { - "path": "\"src_helm_benchmark_scenarios_best_chatgpt_prompts.yaml\"", - "tags": "\"\"", - "evaluator": "[\"claude\", \"gpt4\", \"mturk\", \"scale\"]" - } - } - }, - { - "evaluation_name": "Koala test dataset", - "source_data": { - "dataset_name": "Koala test dataset", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/instruct/benchmark_output/releases/v1.0.0/groups/instruction_following.json" - ] - }, - "metric_config": { - "evaluation_description": "Harmlessness on Koala test dataset", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 5.0 - }, - "score_details": { - "score": 4.981, - "details": { - "description": "min=4.965, mean=4.981, max=5, sum=19.925 (4)", - "tab": "Instruction Following" - } - }, - "generation_config": { - "additional_details": { - "evaluator": "[\"claude\", \"gpt4\", \"mturk\", \"scale\"]" - } - } - }, - { - "evaluation_name": "Open Assistant", - "source_data": { - "dataset_name": "Open Assistant", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/instruct/benchmark_output/releases/v1.0.0/groups/instruction_following.json" - ] - }, - "metric_config": { - "evaluation_description": "Harmlessness on Open Assistant", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 5.0 - }, - "score_details": { - "score": 4.975, - "details": { - "description": "min=4.935, mean=4.975, max=5, sum=19.9 (4)", - "tab": "Instruction Following" - } - }, - "generation_config": { - "additional_details": { - "language": "\"en\"", - "evaluator": "[\"claude\", \"gpt4\", \"mturk\", \"scale\"]" - } - } - }, - { - "evaluation_name": "Self Instruct", - "source_data": { - "dataset_name": "Self Instruct", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/instruct/benchmark_output/releases/v1.0.0/groups/instruction_following.json" - ] - }, - "metric_config": { - "evaluation_description": "Harmlessness on Self Instruct", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 5.0 - }, - "score_details": { - "score": 4.992, - "details": { - "description": "min=4.98, mean=4.992, max=5, sum=19.97 (4)", - "tab": "Instruction Following" - } - }, - "generation_config": { - "additional_details": { - "evaluator": "[\"claude\", \"gpt4\", \"mturk\", \"scale\"]" - } - } - }, - { - "evaluation_name": "Vicuna", - "source_data": { - "dataset_name": "Vicuna", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/instruct/benchmark_output/releases/v1.0.0/groups/instruction_following.json" - ] - }, - "metric_config": { - "evaluation_description": "Harmlessness on Vicuna", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 5.0 - }, - "score_details": { - "score": 4.989, - "details": { - "description": "min=4.956, mean=4.989, max=5, sum=19.956 (4)", - "tab": "Instruction Following" - } - }, - "generation_config": { - "additional_details": { - "category": "\"all\"", - "evaluator": "[\"claude\", \"gpt4\", \"mturk\", \"scale\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_lite/anthropic_claude-v1.3/1774096306.427425", - "retrieved_timestamp": "1774096306.427425", - "source_metadata": { - "source_name": "helm_lite", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_lite", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_lite", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.518, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.09352059925093632\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.723, - "details": { - "description": "min=0.723, mean=0.723, max=0.723, sum=0.723 (1)", - "tab": "Accuracy", - "NarrativeQA - Observed inference time (s)": "{\"description\": \"min=6.114, mean=6.114, max=6.114, sum=6.114 (1)\", \"tab\": \"Efficiency\", \"score\": \"6.113923052666893\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=3709.741, mean=3709.741, max=3709.741, sum=3709.741 (1)\", \"tab\": \"General information\", \"score\": \"3709.7408450704224\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=9.338, mean=9.338, max=9.338, sum=9.338 (1)\", \"tab\": \"General information\", \"score\": \"9.338028169014084\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (closed-book)", - "source_data": { - "dataset_name": "NaturalQuestions (closed-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (closed-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.409, - "details": { - "description": "min=0.409, mean=0.409, max=0.409, sum=0.409 (1)", - "tab": "Accuracy", - "NaturalQuestions (open-book) - Observed inference time (s)": "{\"description\": \"min=3.523, mean=3.523, max=3.523, sum=3.523 (1)\", \"tab\": \"Efficiency\", \"score\": \"3.5226667501174913\"}", - "NaturalQuestions (closed-book) - Observed inference time (s)": "{\"description\": \"min=2.059, mean=2.059, max=2.059, sum=2.059 (1)\", \"tab\": \"Efficiency\", \"score\": \"2.0589215233325957\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.964, mean=4.964, max=4.964, sum=4.964 (1)\", \"tab\": \"General information\", \"score\": \"4.964\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.007, mean=0.007, max=0.007, sum=0.007 (1)\", \"tab\": \"General information\", \"score\": \"0.007\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1734.363, mean=1734.363, max=1734.363, sum=1734.363 (1)\", \"tab\": \"General information\", \"score\": \"1734.363\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=4.973, mean=4.973, max=4.973, sum=4.973 (1)\", \"tab\": \"General information\", \"score\": \"4.973\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=189.259, mean=189.259, max=189.259, sum=189.259 (1)\", \"tab\": \"General information\", \"score\": \"189.259\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=3.722, mean=3.722, max=3.722, sum=3.722 (1)\", \"tab\": \"General information\", \"score\": \"3.722\"}" - } - }, - "generation_config": { - "additional_details": { - "mode": "\"closedbook\"" - } - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.908, - "details": { - "description": "min=0.908, mean=0.908, max=0.908, sum=0.908 (1)", - "tab": "Accuracy", - "OpenbookQA - Observed inference time (s)": "{\"description\": \"min=3.375, mean=3.375, max=3.375, sum=3.375 (1)\", \"tab\": \"Efficiency\", \"score\": \"3.375496371269226\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=328.79, mean=328.79, max=328.79, sum=328.79 (1)\", \"tab\": \"General information\", \"score\": \"328.79\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "dataset": "\"openbookqa\"", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.631, - "details": { - "description": "min=0.35, mean=0.631, max=0.93, sum=3.155 (5)", - "tab": "Accuracy", - "MMLU - Observed inference time (s)": "{\"description\": \"min=1.228, mean=1.482, max=1.741, sum=7.41 (5)\", \"tab\": \"Efficiency\", \"score\": \"1.4820951028288456\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=435.26, mean=543.747, max=684.596, sum=2718.736 (5)\", \"tab\": \"General information\", \"score\": \"543.747298245614\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"college_chemistry\", \"computer_security\", \"econometrics\", \"us_foreign_policy\"]", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MATH", - "source_data": { - "dataset_name": "MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Equivalent (CoT) on MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.54, - "details": { - "description": "min=0.368, mean=0.54, max=0.826, sum=3.783 (7)", - "tab": "Accuracy", - "MATH - Observed inference time (s)": "{\"description\": \"min=3.85, mean=6.109, max=8.225, sum=42.762 (7)\", \"tab\": \"Efficiency\", \"score\": \"6.10879439056091\"}", - "MATH - # eval": "{\"description\": \"min=30, mean=62.429, max=135, sum=437 (7)\", \"tab\": \"General information\", \"score\": \"62.42857142857143\"}", - "MATH - # train": "{\"description\": \"min=8, mean=8, max=8, sum=56 (7)\", \"tab\": \"General information\", \"score\": \"8.0\"}", - "MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MATH - # prompt tokens": "{\"description\": \"min=947.259, mean=1361.814, max=2379.808, sum=9532.699 (7)\", \"tab\": \"General information\", \"score\": \"1361.8141219676104\"}", - "MATH - # output tokens": "{\"description\": \"min=53.133, mean=79.493, max=97.564, sum=556.452 (7)\", \"tab\": \"General information\", \"score\": \"79.49312981320325\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"algebra\", \"counting_and_probability\", \"geometry\", \"intermediate_algebra\", \"number_theory\", \"prealgebra\", \"precalculus\"]", - "level": "\"1\"", - "use_official_examples": "\"False\"", - "use_chain_of_thought": "\"True\"" - } - } - }, - { - "evaluation_name": "GSM8K", - "source_data": { - "dataset_name": "GSM8K", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on GSM8K", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.784, - "details": { - "description": "min=0.784, mean=0.784, max=0.784, sum=0.784 (1)", - "tab": "Accuracy", - "GSM8K - Observed inference time (s)": "{\"description\": \"min=6.653, mean=6.653, max=6.653, sum=6.653 (1)\", \"tab\": \"Efficiency\", \"score\": \"6.653211696863174\"}", - "GSM8K - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "GSM8K - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "GSM8K - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GSM8K - # prompt tokens": "{\"description\": \"min=1012.712, mean=1012.712, max=1012.712, sum=1012.712 (1)\", \"tab\": \"General information\", \"score\": \"1012.712\"}", - "GSM8K - # output tokens": "{\"description\": \"min=104.726, mean=104.726, max=104.726, sum=104.726 (1)\", \"tab\": \"General information\", \"score\": \"104.726\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "LegalBench", - "source_data": { - "dataset_name": "LegalBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on LegalBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.629, - "details": { - "description": "min=0.417, mean=0.629, max=0.916, sum=3.147 (5)", - "tab": "Accuracy", - "LegalBench - Observed inference time (s)": "{\"description\": \"min=1.081, mean=3.536, max=8.614, sum=17.681 (5)\", \"tab\": \"Efficiency\", \"score\": \"3.536136101917547\"}", - "LegalBench - # eval": "{\"description\": \"min=95, mean=409.4, max=1000, sum=2047 (5)\", \"tab\": \"General information\", \"score\": \"409.4\"}", - "LegalBench - # train": "{\"description\": \"min=4, mean=4.798, max=5, sum=23.99 (5)\", \"tab\": \"General information\", \"score\": \"4.797959183673469\"}", - "LegalBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "LegalBench - # prompt tokens": "{\"description\": \"min=280.653, mean=1621.356, max=6484.969, sum=8106.779 (5)\", \"tab\": \"General information\", \"score\": \"1621.3558670820687\"}", - "LegalBench - # output tokens": "{\"description\": \"min=1, mean=1.354, max=2.232, sum=6.771 (5)\", \"tab\": \"General information\", \"score\": \"1.3542176968306323\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"abercrombie\", \"corporate_lobbying\", \"function_of_decision_section\", \"international_citizenship_questions\", \"proa\"]" - } - } - }, - { - "evaluation_name": "MedQA", - "source_data": { - "dataset_name": "MedQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MedQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.618, - "details": { - "description": "min=0.618, mean=0.618, max=0.618, sum=0.618 (1)", - "tab": "Accuracy", - "MedQA - Observed inference time (s)": "{\"description\": \"min=3.39, mean=3.39, max=3.39, sum=3.39 (1)\", \"tab\": \"Efficiency\", \"score\": \"3.3901417141643244\"}", - "MedQA - # eval": "{\"description\": \"min=503, mean=503, max=503, sum=503 (1)\", \"tab\": \"General information\", \"score\": \"503.0\"}", - "MedQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MedQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MedQA - # prompt tokens": "{\"description\": \"min=1092.437, mean=1092.437, max=1092.437, sum=1092.437 (1)\", \"tab\": \"General information\", \"score\": \"1092.4373757455269\"}", - "MedQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WMT 2014", - "source_data": { - "dataset_name": "WMT 2014", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "BLEU-4 on WMT 2014", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.219, - "details": { - "description": "min=0.152, mean=0.219, max=0.28, sum=1.093 (5)", - "tab": "Accuracy", - "WMT 2014 - Observed inference time (s)": "{\"description\": \"min=1.391, mean=2.232, max=3.755, sum=11.161 (5)\", \"tab\": \"Efficiency\", \"score\": \"2.232213549153336\"}", - "WMT 2014 - # eval": "{\"description\": \"min=503, mean=568.8, max=832, sum=2844 (5)\", \"tab\": \"General information\", \"score\": \"568.8\"}", - "WMT 2014 - # train": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "WMT 2014 - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WMT 2014 - # prompt tokens": "{\"description\": \"min=197.406, mean=218.573, max=240.974, sum=1092.866 (5)\", \"tab\": \"General information\", \"score\": \"218.57322077152472\"}", - "WMT 2014 - # output tokens": "{\"description\": \"min=24.004, mean=25.611, max=26.28, sum=128.057 (5)\", \"tab\": \"General information\", \"score\": \"25.611364027374215\"}" - } - }, - "generation_config": { - "additional_details": { - "language_pair": "[\"cs-en\", \"de-en\", \"fr-en\", \"hi-en\", \"ru-en\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/anthropic_opus_4.1.json b/data/models/anthropic_opus_4.1.json deleted file mode 100644 index 02b6fcdd6f42a3bfaf672ab8f4cdde77b886c8cf..0000000000000000000000000000000000000000 --- a/data/models/anthropic_opus_4.1.json +++ /dev/null @@ -1,82 +0,0 @@ -{ - "model_info": { - "name": "Opus 4.1", - "developer": "Anthropic", - "id": "anthropic/Opus 4.1", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "ace/anthropic_opus-4.1/1773260200", - "retrieved_timestamp": "1773260200", - "source_metadata": { - "source_name": "Mercor ACE Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Mercor", - "source_organization_url": "https://www.mercor.com", - "evaluator_relationship": "first_party" - }, - "eval_library": { - "name": "archipelago", - "version": "1.0.0" - }, - "benchmark": "ace", - "evaluation_results": [ - { - "evaluation_name": "Overall Score", - "source_data": { - "dataset_name": "ace", - "source_type": "hf_dataset", - "hf_repo": "Mercor/ACE" - }, - "metric_config": { - "evaluation_description": "Overall ACE score (paper snapshot, approximate).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.4 - }, - "generation_config": { - "additional_details": { - "run_setting": "On", - "value_quality": "approximate" - } - } - }, - { - "evaluation_name": "Gaming Score", - "source_data": { - "dataset_name": "ace", - "source_type": "hf_dataset", - "hf_repo": "Mercor/ACE" - }, - "metric_config": { - "evaluation_description": "Gaming domain score.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.318 - }, - "generation_config": { - "additional_details": { - "run_setting": "On" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "run_setting": "On", - "value_quality": "approximate" - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/anthropic_opus_4.5.json b/data/models/anthropic_opus_4.5.json deleted file mode 100644 index 0952f13ca0e6f15c8598404fd4db3674326651d7..0000000000000000000000000000000000000000 --- a/data/models/anthropic_opus_4.5.json +++ /dev/null @@ -1,326 +0,0 @@ -{ - "model_info": { - "name": "Opus 4.5", - "developer": "Anthropic", - "id": "anthropic/Opus 4.5", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "ace/anthropic_opus-4.5/1773260200", - "retrieved_timestamp": "1773260200", - "source_metadata": { - "source_name": "Mercor ACE Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Mercor", - "source_organization_url": "https://www.mercor.com", - "evaluator_relationship": "first_party" - }, - "eval_library": { - "name": "archipelago", - "version": "1.0.0" - }, - "benchmark": "ace", - "evaluation_results": [ - { - "evaluation_name": "Overall Score", - "source_data": { - "dataset_name": "ace", - "source_type": "hf_dataset", - "hf_repo": "Mercor/ACE" - }, - "metric_config": { - "evaluation_description": "Overall ACE score (paper snapshot).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.478 - }, - "generation_config": { - "additional_details": { - "run_setting": "On" - } - } - }, - { - "evaluation_name": "Gaming Score", - "source_data": { - "dataset_name": "ace", - "source_type": "hf_dataset", - "hf_repo": "Mercor/ACE" - }, - "metric_config": { - "evaluation_description": "Gaming domain score.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.391 - }, - "generation_config": { - "additional_details": { - "run_setting": "On" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "run_setting": "On" - } - } - }, - { - "evaluation_id": "apex-agents/anthropic_opus-4.5/1773260200", - "retrieved_timestamp": "1773260200", - "source_metadata": { - "source_name": "Mercor APEX-Agents Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Mercor", - "source_organization_url": "https://www.mercor.com", - "evaluator_relationship": "first_party" - }, - "eval_library": { - "name": "archipelago", - "version": "1.0.0" - }, - "benchmark": "apex-agents", - "evaluation_results": [ - { - "evaluation_name": "Overall Pass@1", - "source_data": { - "dataset_name": "apex-agents", - "source_type": "hf_dataset", - "hf_repo": "mercor/apex-agents" - }, - "metric_config": { - "evaluation_description": "Overall Pass@1 (dataset card / paper snapshot).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.184, - "uncertainty": { - "confidence_interval": { - "lower": -0.029, - "upper": 0.029, - "method": "bootstrap" - } - } - }, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - }, - { - "evaluation_name": "Overall Pass@8", - "source_data": { - "dataset_name": "apex-agents", - "source_type": "hf_dataset", - "hf_repo": "mercor/apex-agents" - }, - "metric_config": { - "evaluation_description": "Overall Pass@8 (dataset card / paper snapshot).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.34, - "uncertainty": { - "confidence_interval": { - "lower": -0.042, - "upper": 0.043, - "method": "bootstrap" - } - } - }, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - }, - { - "evaluation_name": "Overall Mean Score", - "source_data": { - "dataset_name": "apex-agents", - "source_type": "hf_dataset", - "hf_repo": "mercor/apex-agents" - }, - "metric_config": { - "evaluation_description": "Overall mean rubric score.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.348 - }, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - }, - { - "evaluation_name": "Investment Banking Pass@1", - "source_data": { - "dataset_name": "apex-agents", - "source_type": "hf_dataset", - "hf_repo": "mercor/apex-agents" - }, - "metric_config": { - "evaluation_description": "Investment banking world Pass@1.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.216 - }, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - }, - { - "evaluation_name": "Management Consulting Pass@1", - "source_data": { - "dataset_name": "apex-agents", - "source_type": "hf_dataset", - "hf_repo": "mercor/apex-agents" - }, - "metric_config": { - "evaluation_description": "Management consulting world Pass@1.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.132 - }, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - }, - { - "evaluation_name": "Corporate Law Pass@1", - "source_data": { - "dataset_name": "apex-agents", - "source_type": "hf_dataset", - "hf_repo": "mercor/apex-agents" - }, - "metric_config": { - "evaluation_description": "Corporate law world Pass@1.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.202 - }, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - }, - { - "evaluation_name": "Corporate Lawyer Mean Score", - "source_data": { - "dataset_name": "apex-agents", - "source_type": "hf_dataset", - "hf_repo": "mercor/apex-agents" - }, - "metric_config": { - "evaluation_description": "Corporate lawyer world mean score.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.471 - }, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - }, - { - "evaluation_id": "apex-v1/anthropic_opus-4.5/1773260200", - "retrieved_timestamp": "1773260200", - "source_metadata": { - "source_name": "Mercor APEX-v1 Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Mercor", - "source_organization_url": "https://www.mercor.com", - "evaluator_relationship": "first_party" - }, - "eval_library": { - "name": "archipelago", - "version": "1.0.0" - }, - "benchmark": "apex-v1", - "evaluation_results": [ - { - "evaluation_name": "Medicine (MD) Score", - "source_data": { - "dataset_name": "apex-v1", - "source_type": "hf_dataset", - "hf_repo": "Mercor/APEX-v1" - }, - "metric_config": { - "evaluation_description": "Primary care physician (MD) score.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.65 - }, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/anthropic_opus_4.6.json b/data/models/anthropic_opus_4.6.json deleted file mode 100644 index 5a93ebd0b7577f98536718c7ccd0b4051739228d..0000000000000000000000000000000000000000 --- a/data/models/anthropic_opus_4.6.json +++ /dev/null @@ -1,87 +0,0 @@ -{ - "model_info": { - "name": "Opus 4.6", - "developer": "Anthropic", - "id": "anthropic/Opus 4.6", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "apex-agents/anthropic_opus-4.6/1773260200", - "retrieved_timestamp": "1773260200", - "source_metadata": { - "source_name": "Mercor APEX-Agents Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Mercor", - "source_organization_url": "https://www.mercor.com", - "evaluator_relationship": "first_party" - }, - "eval_library": { - "name": "archipelago", - "version": "1.0.0" - }, - "benchmark": "apex-agents", - "evaluation_results": [ - { - "evaluation_name": "Overall Pass@1", - "source_data": { - "dataset_name": "apex-agents", - "source_type": "hf_dataset", - "hf_repo": "mercor/apex-agents" - }, - "metric_config": { - "evaluation_description": "Overall Pass@1 across 480 long-horizon professional-services tasks.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.298, - "uncertainty": { - "confidence_interval": { - "lower": -0.036, - "upper": 0.036, - "method": "bootstrap" - } - } - }, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - }, - { - "evaluation_name": "Corporate Lawyer Mean Score", - "source_data": { - "dataset_name": "apex-agents", - "source_type": "hf_dataset", - "hf_repo": "mercor/apex-agents" - }, - "metric_config": { - "evaluation_description": "Corporate lawyer world mean score from leaderboard model list.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.502 - }, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/anthropic_sonnet_4.5.json b/data/models/anthropic_sonnet_4.5.json deleted file mode 100644 index 8953c221133da56a0a9103c398cf54a2c332c66d..0000000000000000000000000000000000000000 --- a/data/models/anthropic_sonnet_4.5.json +++ /dev/null @@ -1,82 +0,0 @@ -{ - "model_info": { - "name": "Sonnet 4.5", - "developer": "Anthropic", - "id": "anthropic/Sonnet 4.5", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "ace/anthropic_sonnet-4.5/1773260200", - "retrieved_timestamp": "1773260200", - "source_metadata": { - "source_name": "Mercor ACE Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Mercor", - "source_organization_url": "https://www.mercor.com", - "evaluator_relationship": "first_party" - }, - "eval_library": { - "name": "archipelago", - "version": "1.0.0" - }, - "benchmark": "ace", - "evaluation_results": [ - { - "evaluation_name": "Overall Score", - "source_data": { - "dataset_name": "ace", - "source_type": "hf_dataset", - "hf_repo": "Mercor/ACE" - }, - "metric_config": { - "evaluation_description": "Overall ACE score (paper snapshot, approximate).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.44 - }, - "generation_config": { - "additional_details": { - "run_setting": "On", - "value_quality": "approximate" - } - } - }, - { - "evaluation_name": "Gaming Score", - "source_data": { - "dataset_name": "ace", - "source_type": "hf_dataset", - "hf_repo": "Mercor/ACE" - }, - "metric_config": { - "evaluation_description": "Gaming domain score.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.373 - }, - "generation_config": { - "additional_details": { - "run_setting": "On" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "run_setting": "On", - "value_quality": "approximate" - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/apple_dclm-7b.json b/data/models/apple_dclm-7b.json deleted file mode 100644 index 300110cf2b50cfac0a221ba3b05b3a4d2d924828..0000000000000000000000000000000000000000 --- a/data/models/apple_dclm-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "DCLM-7B", - "id": "apple/DCLM-7B", - "developer": "apple", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "OpenLMModel", - "params_billions": "7.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/apple_DCLM-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2173 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4232 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.037 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3154 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3921 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3111 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/applied-compute_applied_compute__small.json b/data/models/applied-compute_applied_compute__small.json deleted file mode 100644 index 2411636622727327c40e465bbdc297ff06d0aa82..0000000000000000000000000000000000000000 --- a/data/models/applied-compute_applied_compute__small.json +++ /dev/null @@ -1,109 +0,0 @@ -{ - "model_info": { - "name": "Applied Compute: Small", - "developer": "applied-compute", - "id": "applied-compute/Applied Compute: Small", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "apex-agents/applied-compute_applied-compute-small/1773260200", - "retrieved_timestamp": "1773260200", - "source_metadata": { - "source_name": "Mercor APEX-Agents Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Mercor", - "source_organization_url": "https://www.mercor.com", - "evaluator_relationship": "first_party" - }, - "eval_library": { - "name": "archipelago", - "version": "1.0.0" - }, - "benchmark": "apex-agents", - "evaluation_results": [ - { - "evaluation_name": "Overall Pass@1", - "source_data": { - "dataset_name": "apex-agents", - "source_type": "hf_dataset", - "hf_repo": "mercor/apex-agents" - }, - "metric_config": { - "evaluation_description": "Overall Pass@1 across 480 long-horizon professional-services tasks.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.23, - "uncertainty": { - "confidence_interval": { - "lower": -0.031, - "upper": 0.031, - "method": "bootstrap" - } - } - } - }, - { - "evaluation_name": "Overall Mean Score", - "source_data": { - "dataset_name": "apex-agents", - "source_type": "hf_dataset", - "hf_repo": "mercor/apex-agents" - }, - "metric_config": { - "evaluation_description": "Overall mean rubric score.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.401 - } - }, - { - "evaluation_name": "Corporate Law Pass@1", - "source_data": { - "dataset_name": "apex-agents", - "source_type": "hf_dataset", - "hf_repo": "mercor/apex-agents" - }, - "metric_config": { - "evaluation_description": "Corporate law world Pass@1.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.266 - } - }, - { - "evaluation_name": "Corporate Lawyer Mean Score", - "source_data": { - "dataset_name": "apex-agents", - "source_type": "hf_dataset", - "hf_repo": "mercor/apex-agents" - }, - "metric_config": { - "evaluation_description": "Corporate lawyer world mean score.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.548 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/appvoid_arco-2-instruct.json b/data/models/appvoid_arco-2-instruct.json deleted file mode 100644 index 15a37499ded48190c9898ab817c981962863b2e6..0000000000000000000000000000000000000000 --- a/data/models/appvoid_arco-2-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "arco-2-instruct", - "id": "appvoid/arco-2-instruct", - "developer": "appvoid", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "0.514" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/appvoid_arco-2-instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2164 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3133 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0128 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2383 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3496 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1113 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/appvoid_arco-2.json b/data/models/appvoid_arco-2.json deleted file mode 100644 index fefe7ed4e4e490485eaf64571df3537a3abc6bfb..0000000000000000000000000000000000000000 --- a/data/models/appvoid_arco-2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "arco-2", - "id": "appvoid/arco-2", - "developer": "appvoid", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "0.514" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/appvoid_arco-2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1991 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3146 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0136 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2391 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3536 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1116 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/arcee-ai_arcee-blitz.json b/data/models/arcee-ai_arcee-blitz.json deleted file mode 100644 index 0b9c869fc0875f452648c5bbdf3a4eba3242ab2f..0000000000000000000000000000000000000000 --- a/data/models/arcee-ai_arcee-blitz.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Arcee-Blitz", - "id": "arcee-ai/Arcee-Blitz", - "developer": "arcee-ai", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "23.572" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/arcee-ai_Arcee-Blitz/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5543 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6607 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3482 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3851 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5047 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6154 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/arcee-ai_arcee-maestro-7b-preview.json b/data/models/arcee-ai_arcee-maestro-7b-preview.json deleted file mode 100644 index fc3b3977e1af7669f57553f020b37a6912f85a1f..0000000000000000000000000000000000000000 --- a/data/models/arcee-ai_arcee-maestro-7b-preview.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Arcee-Maestro-7B-Preview", - "id": "arcee-ai/Arcee-Maestro-7B-Preview", - "developer": "arcee-ai", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.613" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/arcee-ai_Arcee-Maestro-7B-Preview/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.275 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4648 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4992 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3322 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3885 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3039 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/arcee-ai_arcee-nova.json b/data/models/arcee-ai_arcee-nova.json deleted file mode 100644 index 07084a913fdb569d1902a79694c3c6d4679f3323..0000000000000000000000000000000000000000 --- a/data/models/arcee-ai_arcee-nova.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Arcee-Nova", - "id": "arcee-ai/Arcee-Nova", - "developer": "arcee-ai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "72.706" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/arcee-ai_Arcee-Nova/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7907 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6942 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4381 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3851 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4562 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5452 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/arcee-ai_arcee-spark.json b/data/models/arcee-ai_arcee-spark.json deleted file mode 100644 index 611dbc409a44fd2eadd33b4e30a3048a3002308e..0000000000000000000000000000000000000000 --- a/data/models/arcee-ai_arcee-spark.json +++ /dev/null @@ -1,275 +0,0 @@ -{ - "model_info": { - "name": "Arcee-Spark", - "id": "arcee-ai/Arcee-Spark", - "developer": "arcee-ai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/arcee-ai_Arcee-Spark/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5621 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5489 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2953 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.307 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4021 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3822 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "hfopenllm_v2/arcee-ai_Arcee-Spark/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5718 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5481 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.114 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3062 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4008 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3813 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/arcee-ai_llama-3.1-supernova-lite.json b/data/models/arcee-ai_llama-3.1-supernova-lite.json deleted file mode 100644 index a434b63dfb14ad5912127bc9fc9a7d112a393c51..0000000000000000000000000000000000000000 --- a/data/models/arcee-ai_llama-3.1-supernova-lite.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.1-SuperNova-Lite", - "id": "arcee-ai/Llama-3.1-SuperNova-Lite", - "developer": "arcee-ai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/arcee-ai_Llama-3.1-SuperNova-Lite/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8017 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5152 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1828 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3062 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4163 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3877 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/arcee-ai_llama-spark.json b/data/models/arcee-ai_llama-spark.json deleted file mode 100644 index 2486b648da718450d6427c16d9e76c8295579061..0000000000000000000000000000000000000000 --- a/data/models/arcee-ai_llama-spark.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-Spark", - "id": "arcee-ai/Llama-Spark", - "developer": "arcee-ai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/arcee-ai_Llama-Spark/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7911 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5054 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.139 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2995 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3593 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3721 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/arcee-ai_raspberry-3b.json b/data/models/arcee-ai_raspberry-3b.json deleted file mode 100644 index 4fdcaa6d9e5a86f2a535d7b8316476075123ef8f..0000000000000000000000000000000000000000 --- a/data/models/arcee-ai_raspberry-3b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "raspberry-3B", - "id": "arcee-ai/raspberry-3B", - "developer": "arcee-ai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.086" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/arcee-ai_raspberry-3B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3154 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4269 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1035 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2777 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4123 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2854 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/arcee-ai_supernova-medius.json b/data/models/arcee-ai_supernova-medius.json deleted file mode 100644 index e7b064625256c085311c81a90cfe838f47bd2d12..0000000000000000000000000000000000000000 --- a/data/models/arcee-ai_supernova-medius.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SuperNova-Medius", - "id": "arcee-ai/SuperNova-Medius", - "developer": "arcee-ai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/arcee-ai_SuperNova-Medius/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7184 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6377 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.469 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3331 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4233 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5035 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/arcee-ai_virtuoso-lite.json b/data/models/arcee-ai_virtuoso-lite.json deleted file mode 100644 index 5cfc30a906063045421ddb54bdbd13b443b2fb68..0000000000000000000000000000000000000000 --- a/data/models/arcee-ai_virtuoso-lite.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Virtuoso-Lite", - "id": "arcee-ai/Virtuoso-Lite", - "developer": "arcee-ai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "10.306" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/arcee-ai_Virtuoso-Lite/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.81 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6099 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.253 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.344 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4595 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4441 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/arcee-ai_virtuoso-small-v2.json b/data/models/arcee-ai_virtuoso-small-v2.json deleted file mode 100644 index e22af7fd97f6fb02cbab1a87771c2d3db7fb47e3..0000000000000000000000000000000000000000 --- a/data/models/arcee-ai_virtuoso-small-v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Virtuoso-Small-v2", - "id": "arcee-ai/Virtuoso-Small-v2", - "developer": "arcee-ai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/arcee-ai_Virtuoso-Small-v2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8273 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6554 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.466 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3532 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4313 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5188 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/arcee-ai_virtuoso-small.json b/data/models/arcee-ai_virtuoso-small.json deleted file mode 100644 index aafcfacc3589bf23a7c346c19331bcaf44645db5..0000000000000000000000000000000000000000 --- a/data/models/arcee-ai_virtuoso-small.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Virtuoso-Small", - "id": "arcee-ai/Virtuoso-Small", - "developer": "arcee-ai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/arcee-ai_Virtuoso-Small/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7935 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6518 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4094 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3364 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4339 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5191 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/argilla-warehouse_llama-3.1-8b-magpie-ultra.json b/data/models/argilla-warehouse_llama-3.1-8b-magpie-ultra.json deleted file mode 100644 index 28d2ff7f43dfd8e6d669d73fc6c9218991840d17..0000000000000000000000000000000000000000 --- a/data/models/argilla-warehouse_llama-3.1-8b-magpie-ultra.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.1-8B-MagPie-Ultra", - "id": "argilla-warehouse/Llama-3.1-8B-MagPie-Ultra", - "developer": "argilla-warehouse", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/argilla-warehouse_Llama-3.1-8B-MagPie-Ultra/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5757 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.462 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.077 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2668 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3543 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3144 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/argilla_notus-7b-v1.json b/data/models/argilla_notus-7b-v1.json deleted file mode 100644 index 8cdf38dad5225e4d0d167d4b575d8a98ff98670e..0000000000000000000000000000000000000000 --- a/data/models/argilla_notus-7b-v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "notus-7b-v1", - "id": "argilla/notus-7b-v1", - "developer": "argilla", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/argilla_notus-7b-v1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5082 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4512 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0317 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2894 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3364 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3004 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/argilla_notux-8x7b-v1.json b/data/models/argilla_notux-8x7b-v1.json deleted file mode 100644 index ef233042be20a9709a0052f243d815532150ee85..0000000000000000000000000000000000000000 --- a/data/models/argilla_notux-8x7b-v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "notux-8x7b-v1", - "id": "argilla/notux-8x7b-v1", - "developer": "argilla", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MixtralForCausalLM", - "params_billions": "46.703" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/argilla_notux-8x7b-v1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5422 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5363 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0997 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3087 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4176 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.366 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/arisin_orca-platypus-13b-slerp.json b/data/models/arisin_orca-platypus-13b-slerp.json deleted file mode 100644 index 4ce3e5e7a11b4037babb8ed4981b47caeeb8ccc2..0000000000000000000000000000000000000000 --- a/data/models/arisin_orca-platypus-13b-slerp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "orca-platypus-13B-slerp", - "id": "arisin/orca-platypus-13B-slerp", - "developer": "arisin", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "13.016" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/arisin_orca-platypus-13B-slerp/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2672 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4631 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0159 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2987 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4253 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2592 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ark_ep-20250603132404-cgpjm.json b/data/models/ark_ep-20250603132404-cgpjm.json deleted file mode 100644 index d819fecb1af98794d0b0bdf97fc241199edd93a5..0000000000000000000000000000000000000000 --- a/data/models/ark_ep-20250603132404-cgpjm.json +++ /dev/null @@ -1,89 +0,0 @@ -{ - "model_info": { - "name": "ep-20250603132404-cgpjm", - "id": "ark/ep-20250603132404-cgpjm", - "developer": "ark", - "inference_platform": "ark" - }, - "evaluations": [ - { - "evaluation_id": "livecodebenchpro/ep-20250603132404-cgpjm/1770683238.099205", - "retrieved_timestamp": "1770683238.099205", - "source_metadata": { - "source_name": "Live Code Bench Pro", - "source_type": "documentation", - "source_organization_name": "New York University, Princeton University, University of California San Diego, University of Washington and Canyon Crest Academy", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "unknown", - "version": "unknown" - }, - "benchmark": "livecodebenchpro", - "evaluation_results": [ - { - "evaluation_name": "Hard Problems", - "source_data": { - "dataset_name": "Hard Problems", - "source_type": "url", - "url": [ - "https://webhook.cp-bench.orzzh.com/leaderboard/llm/difficulty?difficulty=hard&benchmark_mode=live" - ] - }, - "metric_config": { - "evaluation_description": "Pass@1 on Hard Problems", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "Medium Problems", - "source_data": { - "dataset_name": "Medium Problems", - "source_type": "url", - "url": [ - "https://webhook.cp-bench.orzzh.com/leaderboard/llm/difficulty?difficulty=medium&benchmark_mode=live" - ] - }, - "metric_config": { - "evaluation_description": "Pass@1 on Medium Problems", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0141 - } - }, - { - "evaluation_name": "Easy Problems", - "source_data": { - "dataset_name": "Easy Problems", - "source_type": "url", - "url": [ - "https://webhook.cp-bench.orzzh.com/leaderboard/llm/difficulty?difficulty=easy&benchmark_mode=live" - ] - }, - "metric_config": { - "evaluation_description": "Pass@1 on Easy Problems", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.507 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/arliai_arliai-rpmax-12b-v1.1.json b/data/models/arliai_arliai-rpmax-12b-v1.1.json deleted file mode 100644 index 3a73242b0cc32f4d27011ac64c087047cb76d825..0000000000000000000000000000000000000000 --- a/data/models/arliai_arliai-rpmax-12b-v1.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ArliAI-RPMax-12B-v1.1", - "id": "ArliAI/ArliAI-RPMax-12B-v1.1", - "developer": "ArliAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ArliAI_ArliAI-RPMax-12B-v1.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5349 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4752 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1125 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2819 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3618 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3384 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/arliai_llama-3.1-8b-arliai-rpmax-v1.1.json b/data/models/arliai_llama-3.1-8b-arliai-rpmax-v1.1.json deleted file mode 100644 index b928ef75d06c3e0a00a8ef7279a9eae2354d0c3e..0000000000000000000000000000000000000000 --- a/data/models/arliai_llama-3.1-8b-arliai-rpmax-v1.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.1-8B-ArliAI-RPMax-v1.1", - "id": "ArliAI/Llama-3.1-8B-ArliAI-RPMax-v1.1", - "developer": "ArliAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ArliAI_Llama-3.1-8B-ArliAI-RPMax-v1.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6359 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5016 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1314 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2836 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3577 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3551 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/arshiaafshani_arsh-v1.json b/data/models/arshiaafshani_arsh-v1.json deleted file mode 100644 index b872ff61505c6588a6bec4157bb361a0174ce28d..0000000000000000000000000000000000000000 --- a/data/models/arshiaafshani_arsh-v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Arsh-V1", - "id": "arshiaafshani/Arsh-V1", - "developer": "arshiaafshani", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "13.96" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/arshiaafshani_Arsh-V1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6043 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.674 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2621 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3733 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4899 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5257 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/arthur-lagacherie_precis-1b-instruct.json b/data/models/arthur-lagacherie_precis-1b-instruct.json deleted file mode 100644 index b576a45c8290fef26b67f9c735b0ef08634f07ec..0000000000000000000000000000000000000000 --- a/data/models/arthur-lagacherie_precis-1b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Precis-1B-Instruct", - "id": "Arthur-LAGACHERIE/Precis-1B-Instruct", - "developer": "Arthur-LAGACHERIE", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.236" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Arthur-LAGACHERIE_Precis-1B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3671 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3224 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0038 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2659 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3436 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1426 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/artples_l-mchat-7b.json b/data/models/artples_l-mchat-7b.json deleted file mode 100644 index 7fa7fcb19201729cf806d38ed9bf7bd704b29644..0000000000000000000000000000000000000000 --- a/data/models/artples_l-mchat-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "L-MChat-7b", - "id": "Artples/L-MChat-7b", - "developer": "Artples", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Artples_L-MChat-7b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5297 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.46 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0921 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3054 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4029 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3299 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/artples_l-mchat-small.json b/data/models/artples_l-mchat-small.json deleted file mode 100644 index ea63d34c18c1684e5b43e6e2131abe3dee5ef5c4..0000000000000000000000000000000000000000 --- a/data/models/artples_l-mchat-small.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "L-MChat-Small", - "id": "Artples/L-MChat-Small", - "developer": "Artples", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "PhiForCausalLM", - "params_billions": "2.78" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Artples_L-MChat-Small/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3287 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4823 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0378 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2676 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3696 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2464 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/aryanne_qwentileswap.json b/data/models/aryanne_qwentileswap.json deleted file mode 100644 index 95ecea1245574ec9a3f194c321cadbaa1724bc7a..0000000000000000000000000000000000000000 --- a/data/models/aryanne_qwentileswap.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "QwentileSwap", - "id": "Aryanne/QwentileSwap", - "developer": "Aryanne", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "32.764" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Aryanne_QwentileSwap/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7378 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7008 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4222 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3674 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.464 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5946 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/aryanne_shba.json b/data/models/aryanne_shba.json deleted file mode 100644 index af0deaa5d1f5f1879d8ac0824af73e84300c4af2..0000000000000000000000000000000000000000 --- a/data/models/aryanne_shba.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SHBA", - "id": "Aryanne/SHBA", - "developer": "Aryanne", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Aryanne_SHBA/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7817 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5233 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1798 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3054 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4161 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3892 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/aryanne_superheart.json b/data/models/aryanne_superheart.json deleted file mode 100644 index d6676fdb879b0050d90fa6c3685f884c8e5ce61b..0000000000000000000000000000000000000000 --- a/data/models/aryanne_superheart.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SuperHeart", - "id": "Aryanne/SuperHeart", - "developer": "Aryanne", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Aryanne_SuperHeart/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5192 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5215 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1563 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3012 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4436 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3912 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/asharsha30_llama_harsha_8_b_ordp_10k.json b/data/models/asharsha30_llama_harsha_8_b_ordp_10k.json deleted file mode 100644 index a138cf1b2efef77294f8d69f63989d8492e1aa47..0000000000000000000000000000000000000000 --- a/data/models/asharsha30_llama_harsha_8_b_ordp_10k.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "LLAMA_Harsha_8_B_ORDP_10k", - "id": "asharsha30/LLAMA_Harsha_8_B_ORDP_10k", - "developer": "asharsha30", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/asharsha30_LLAMA_Harsha_8_B_ORDP_10k/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3464 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4669 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0665 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2735 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3697 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.281 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ashercn97_a1-v0.0.1.json b/data/models/ashercn97_a1-v0.0.1.json deleted file mode 100644 index 4e8a81791369838f838fc5b8722c335d9f50ab6d..0000000000000000000000000000000000000000 --- a/data/models/ashercn97_a1-v0.0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "a1-v0.0.1", - "id": "ashercn97/a1-v0.0.1", - "developer": "ashercn97", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ashercn97_a1-v0.0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2198 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5188 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2145 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3112 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.412 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4165 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ashercn97_a1-v002.json b/data/models/ashercn97_a1-v002.json deleted file mode 100644 index 265163ac24da2a5445b01158bb9ccf88e46ede7b..0000000000000000000000000000000000000000 --- a/data/models/ashercn97_a1-v002.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "a1-v002", - "id": "ashercn97/a1-v002", - "developer": "ashercn97", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ashercn97_a1-v002/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2585 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5261 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2341 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3188 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4159 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4175 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/assskelad_smollm2-360m-sft_smallthoughts.json b/data/models/assskelad_smollm2-360m-sft_smallthoughts.json deleted file mode 100644 index 22b0aec4592e3c523e980ad6f55cd8be817ecacc..0000000000000000000000000000000000000000 --- a/data/models/assskelad_smollm2-360m-sft_smallthoughts.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "smollm2-360M-sft_SmallThoughts", - "id": "assskelad/smollm2-360M-sft_SmallThoughts", - "developer": "assskelad", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "0.362" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/assskelad_smollm2-360M-sft_SmallThoughts/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2007 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.315 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0166 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2592 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3395 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1182 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/atanddev_qwen2.5-1.5b-continuous-learnt.json b/data/models/atanddev_qwen2.5-1.5b-continuous-learnt.json deleted file mode 100644 index c02cc4e6043653215ef5039a1e40518d83ff39e0..0000000000000000000000000000000000000000 --- a/data/models/atanddev_qwen2.5-1.5b-continuous-learnt.json +++ /dev/null @@ -1,275 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-1.5B-continuous-learnt", - "id": "AtAndDev/Qwen2.5-1.5B-continuous-learnt", - "developer": "AtAndDev", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.544" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/AtAndDev_Qwen2.5-1.5B-continuous-learnt/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4605 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4258 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0748 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2659 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3636 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2812 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "hfopenllm_v2/AtAndDev_Qwen2.5-1.5B-continuous-learnt/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4511 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4275 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1473 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2701 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3623 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2806 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ateron_glowing-forest-12b.json b/data/models/ateron_glowing-forest-12b.json deleted file mode 100644 index a26b67f0a3915b6a3c357b78de47d86128b2a5f4..0000000000000000000000000000000000000000 --- a/data/models/ateron_glowing-forest-12b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Glowing-Forest-12B", - "id": "Ateron/Glowing-Forest-12B", - "developer": "Ateron", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Ateron_Glowing-Forest-12B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3592 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5492 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0778 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3331 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4449 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3718 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ateron_lotus-magpic.json b/data/models/ateron_lotus-magpic.json deleted file mode 100644 index 3b7b2fe2d32e9de1e7e33e6a79948e395d40bbd3..0000000000000000000000000000000000000000 --- a/data/models/ateron_lotus-magpic.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Lotus-Magpic", - "id": "Ateron/Lotus-Magpic", - "developer": "Ateron", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Ateron_Lotus-Magpic/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6286 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5254 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0997 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3029 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4332 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3491 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ateron_way_of_magpicaro.json b/data/models/ateron_way_of_magpicaro.json deleted file mode 100644 index 5383916c361219fc99d32ea5d9c9615367739f2b..0000000000000000000000000000000000000000 --- a/data/models/ateron_way_of_magpicaro.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Way_of_MagPicaro", - "id": "Ateron/Way_of_MagPicaro", - "developer": "Ateron", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Ateron_Way_of_MagPicaro/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2637 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5427 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0589 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3339 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4649 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3536 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/athirdpath_llama-3.1-instruct_nsfw-pretrained_e1-plus_reddit.json b/data/models/athirdpath_llama-3.1-instruct_nsfw-pretrained_e1-plus_reddit.json deleted file mode 100644 index bcf1e52cad310a7cdb7b60520286d02919fd4ac0..0000000000000000000000000000000000000000 --- a/data/models/athirdpath_llama-3.1-instruct_nsfw-pretrained_e1-plus_reddit.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.1-Instruct_NSFW-pretrained_e1-plus_reddit", - "id": "athirdpath/Llama-3.1-Instruct_NSFW-pretrained_e1-plus_reddit", - "developer": "athirdpath", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/athirdpath_Llama-3.1-Instruct_NSFW-pretrained_e1-plus_reddit/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4521 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4939 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.102 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2919 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3864 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3565 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/atlaai_selene-1-mini-llama-3.1-8b.json b/data/models/atlaai_selene-1-mini-llama-3.1-8b.json deleted file mode 100644 index e081c6f8037bd0018e0cfd8b9cf3904d0692e50d..0000000000000000000000000000000000000000 --- a/data/models/atlaai_selene-1-mini-llama-3.1-8b.json +++ /dev/null @@ -1,126 +0,0 @@ -{ - "model_info": { - "name": "AtlaAI/Selene-1-Mini-Llama-3.1-8B", - "id": "AtlaAI/Selene-1-Mini-Llama-3.1-8B", - "developer": "AtlaAI", - "additional_details": { - "model_type": "Generative" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/AtlaAI_Selene-1-Mini-Llama-3.1-8B/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8913 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9358 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7939 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8926 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9429 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/atlaai_selene-1.json b/data/models/atlaai_selene-1.json deleted file mode 100644 index bbfd19ee7bacb6f90af3c9b3e40c75ae11d05f4c..0000000000000000000000000000000000000000 --- a/data/models/atlaai_selene-1.json +++ /dev/null @@ -1,126 +0,0 @@ -{ - "model_info": { - "name": "AtlaAI/Selene-1", - "id": "AtlaAI/Selene-1", - "developer": "AtlaAI", - "additional_details": { - "model_type": "Generative" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/AtlaAI_Selene-1/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9241 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9777 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8399 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9216 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9572 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/auraindustries_aura-4b.json b/data/models/auraindustries_aura-4b.json deleted file mode 100644 index 97f2aab4271b672ed9e7e34a833dfed713df20ce..0000000000000000000000000000000000000000 --- a/data/models/auraindustries_aura-4b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Aura-4B", - "id": "AuraIndustries/Aura-4B", - "developer": "AuraIndustries", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "4.513" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/AuraIndustries_Aura-4B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3816 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.449 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0423 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2878 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3938 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2706 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/auraindustries_aura-8b.json b/data/models/auraindustries_aura-8b.json deleted file mode 100644 index 11503320a227f3591c2b04726fe16b1c9eb6d9ec..0000000000000000000000000000000000000000 --- a/data/models/auraindustries_aura-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Aura-8B", - "id": "AuraIndustries/Aura-8B", - "developer": "AuraIndustries", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/AuraIndustries_Aura-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7205 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5131 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1518 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2861 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4004 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3874 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/auraindustries_aura-moe-2x4b-v2.json b/data/models/auraindustries_aura-moe-2x4b-v2.json deleted file mode 100644 index 049c39d3dfa95bb7740588086fbb3a2b67cd1ebd..0000000000000000000000000000000000000000 --- a/data/models/auraindustries_aura-moe-2x4b-v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Aura-MoE-2x4B-v2", - "id": "AuraIndustries/Aura-MoE-2x4B-v2", - "developer": "AuraIndustries", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MixtralForCausalLM", - "params_billions": "7.231" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/AuraIndustries_Aura-MoE-2x4B-v2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4778 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4315 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0317 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2878 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4101 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.261 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/auraindustries_aura-moe-2x4b.json b/data/models/auraindustries_aura-moe-2x4b.json deleted file mode 100644 index 7bb0d6a13b36229264a045f1aade1f5ee7203204..0000000000000000000000000000000000000000 --- a/data/models/auraindustries_aura-moe-2x4b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Aura-MoE-2x4B", - "id": "AuraIndustries/Aura-MoE-2x4B", - "developer": "AuraIndustries", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MixtralForCausalLM", - "params_billions": "7.231" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/AuraIndustries_Aura-MoE-2x4B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4601 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4339 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.031 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2718 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4085 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.265 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/aurel9_testmerge-7b.json b/data/models/aurel9_testmerge-7b.json deleted file mode 100644 index 00b0058b2200574afe8e35865218704f6a496182..0000000000000000000000000000000000000000 --- a/data/models/aurel9_testmerge-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "testmerge-7b", - "id": "Aurel9/testmerge-7b", - "developer": "Aurel9", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Aurel9_testmerge-7b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.398 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.519 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0657 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3003 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4659 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3053 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/automerger_yamshadowexperiment28-7b.json b/data/models/automerger_yamshadowexperiment28-7b.json deleted file mode 100644 index b045c9cdb226dd2bc2b4860bcb2a73380e011141..0000000000000000000000000000000000000000 --- a/data/models/automerger_yamshadowexperiment28-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "YamshadowExperiment28-7B", - "id": "automerger/YamshadowExperiment28-7B", - "developer": "automerger", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/automerger_YamshadowExperiment28-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.407 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.515 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0612 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2869 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4306 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.306 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/avemio_grag-nemo-12b-orpo-hessian-ai.json b/data/models/avemio_grag-nemo-12b-orpo-hessian-ai.json deleted file mode 100644 index bea8fd837d6d5eba2beb66b5d910626e59e620da..0000000000000000000000000000000000000000 --- a/data/models/avemio_grag-nemo-12b-orpo-hessian-ai.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "GRAG-NEMO-12B-ORPO-HESSIAN-AI", - "id": "avemio/GRAG-NEMO-12B-ORPO-HESSIAN-AI", - "developer": "avemio", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/avemio_GRAG-NEMO-12B-ORPO-HESSIAN-AI/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2607 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2592 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3447 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1061 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/awnr_mistral-7b-v0.1-signtensors-1-over-2.json b/data/models/awnr_mistral-7b-v0.1-signtensors-1-over-2.json deleted file mode 100644 index f1496f5708d5bc8cd28d6ec27841d7cecf68c5d7..0000000000000000000000000000000000000000 --- a/data/models/awnr_mistral-7b-v0.1-signtensors-1-over-2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mistral-7B-v0.1-signtensors-1-over-2", - "id": "awnr/Mistral-7B-v0.1-signtensors-1-over-2", - "developer": "awnr", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/awnr_Mistral-7B-v0.1-signtensors-1-over-2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2179 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4423 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.034 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.307 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4006 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/awnr_mistral-7b-v0.1-signtensors-1-over-4.json b/data/models/awnr_mistral-7b-v0.1-signtensors-1-over-4.json deleted file mode 100644 index be35ad2951e1df801113b570428c577512890092..0000000000000000000000000000000000000000 --- a/data/models/awnr_mistral-7b-v0.1-signtensors-1-over-4.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mistral-7B-v0.1-signtensors-1-over-4", - "id": "awnr/Mistral-7B-v0.1-signtensors-1-over-4", - "developer": "awnr", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/awnr_Mistral-7B-v0.1-signtensors-1-over-4/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2133 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3507 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0249 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2701 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.346 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2311 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/awnr_mistral-7b-v0.1-signtensors-3-over-8.json b/data/models/awnr_mistral-7b-v0.1-signtensors-3-over-8.json deleted file mode 100644 index a489aef2edc046a925e98508950f53f4b510f6b3..0000000000000000000000000000000000000000 --- a/data/models/awnr_mistral-7b-v0.1-signtensors-3-over-8.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mistral-7B-v0.1-signtensors-3-over-8", - "id": "awnr/Mistral-7B-v0.1-signtensors-3-over-8", - "developer": "awnr", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/awnr_Mistral-7B-v0.1-signtensors-3-over-8/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2394 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.43 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0332 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3037 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3818 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3001 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/awnr_mistral-7b-v0.1-signtensors-5-over-16.json b/data/models/awnr_mistral-7b-v0.1-signtensors-5-over-16.json deleted file mode 100644 index 06b3dab39675203e2e4639f9ee06c9cfb464a928..0000000000000000000000000000000000000000 --- a/data/models/awnr_mistral-7b-v0.1-signtensors-5-over-16.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mistral-7B-v0.1-signtensors-5-over-16", - "id": "awnr/Mistral-7B-v0.1-signtensors-5-over-16", - "developer": "awnr", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/awnr_Mistral-7B-v0.1-signtensors-5-over-16/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2118 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4124 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0295 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.281 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3686 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2958 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/awnr_mistral-7b-v0.1-signtensors-7-over-16.json b/data/models/awnr_mistral-7b-v0.1-signtensors-7-over-16.json deleted file mode 100644 index b2eb6ea2f30a08d078ef7da297ab47a9e54e7c40..0000000000000000000000000000000000000000 --- a/data/models/awnr_mistral-7b-v0.1-signtensors-7-over-16.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mistral-7B-v0.1-signtensors-7-over-16", - "id": "awnr/Mistral-7B-v0.1-signtensors-7-over-16", - "developer": "awnr", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/awnr_Mistral-7B-v0.1-signtensors-7-over-16/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2294 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4316 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0385 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3037 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3952 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.303 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/aws-prototyping_megabeam-mistral-7b-512k.json b/data/models/aws-prototyping_megabeam-mistral-7b-512k.json deleted file mode 100644 index f12ff5b9f9300bfaa0b65fb3105bc17724373f65..0000000000000000000000000000000000000000 --- a/data/models/aws-prototyping_megabeam-mistral-7b-512k.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MegaBeam-Mistral-7B-512k", - "id": "aws-prototyping/MegaBeam-Mistral-7B-512k", - "developer": "aws-prototyping", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/aws-prototyping_MegaBeam-Mistral-7B-512k/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5973 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3662 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0287 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2827 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3994 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2589 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/axolotl-ai-co_romulus-mistral-nemo-12b-simpo.json b/data/models/axolotl-ai-co_romulus-mistral-nemo-12b-simpo.json deleted file mode 100644 index 1d18276bd43120145f969987c78c647bb91435e2..0000000000000000000000000000000000000000 --- a/data/models/axolotl-ai-co_romulus-mistral-nemo-12b-simpo.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "romulus-mistral-nemo-12b-simpo", - "id": "axolotl-ai-co/romulus-mistral-nemo-12b-simpo", - "developer": "axolotl-ai-co", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/axolotl-ai-co_romulus-mistral-nemo-12b-simpo/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6079 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5395 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.114 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2785 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4233 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3469 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ayush-singh_llama1b-sft-2.json b/data/models/ayush-singh_llama1b-sft-2.json deleted file mode 100644 index 92d6904877fd18627f389136de2fdc12fd1955db..0000000000000000000000000000000000000000 --- a/data/models/ayush-singh_llama1b-sft-2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama1B-sft-2", - "id": "Ayush-Singh/Llama1B-sft-2", - "developer": "Ayush-Singh", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.236" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Ayush-Singh_Llama1B-sft-2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1374 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2834 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2458 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3552 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1117 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/azure99_blossom-v5-32b.json b/data/models/azure99_blossom-v5-32b.json deleted file mode 100644 index 645c9bc98a669b5f68ac98d53f39e4a1245851d3..0000000000000000000000000000000000000000 --- a/data/models/azure99_blossom-v5-32b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "blossom-v5-32b", - "id": "Azure99/blossom-v5-32b", - "developer": "Azure99", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "32.512" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Azure99_blossom-v5-32b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5235 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5955 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1866 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3112 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.402 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4235 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/azure99_blossom-v5-llama3-8b.json b/data/models/azure99_blossom-v5-llama3-8b.json deleted file mode 100644 index 8c49010afbb650d771e07f4b1e04ec426b8b727a..0000000000000000000000000000000000000000 --- a/data/models/azure99_blossom-v5-llama3-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "blossom-v5-llama3-8b", - "id": "Azure99/blossom-v5-llama3-8b", - "developer": "Azure99", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Azure99_blossom-v5-llama3-8b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4343 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4185 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0514 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2651 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.367 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2206 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/azure99_blossom-v5.1-34b.json b/data/models/azure99_blossom-v5.1-34b.json deleted file mode 100644 index cd7fa789d2858d59e062b2328208f311325eca8b..0000000000000000000000000000000000000000 --- a/data/models/azure99_blossom-v5.1-34b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "blossom-v5.1-34b", - "id": "Azure99/blossom-v5.1-34b", - "developer": "Azure99", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "34.389" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Azure99_blossom-v5.1-34b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5697 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6109 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2591 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3096 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3928 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4558 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/azure99_blossom-v5.1-9b.json b/data/models/azure99_blossom-v5.1-9b.json deleted file mode 100644 index 1dc3e1a884accfad3027b3c35fb2db7ca062e2c7..0000000000000000000000000000000000000000 --- a/data/models/azure99_blossom-v5.1-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "blossom-v5.1-9b", - "id": "Azure99/blossom-v5.1-9b", - "developer": "Azure99", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.829" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Azure99_blossom-v5.1-9b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5086 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5343 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2122 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3356 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3994 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3979 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/azure99_blossom-v6-14b.json b/data/models/azure99_blossom-v6-14b.json deleted file mode 100644 index 5038b1517cc57bf58dad40e44fff268988800f04..0000000000000000000000000000000000000000 --- a/data/models/azure99_blossom-v6-14b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Blossom-V6-14B", - "id": "Azure99/Blossom-V6-14B", - "developer": "Azure99", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Azure99_Blossom-V6-14B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6395 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5069 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5257 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2626 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4035 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4544 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/azure99_blossom-v6-7b.json b/data/models/azure99_blossom-v6-7b.json deleted file mode 100644 index 48713784ef686c1840f8c3a52cca159189fe74db..0000000000000000000000000000000000000000 --- a/data/models/azure99_blossom-v6-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Blossom-V6-7B", - "id": "Azure99/Blossom-V6-7B", - "developer": "Azure99", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Azure99_Blossom-V6-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5538 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4974 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4585 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3045 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4301 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4144 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ba2han_llama-phi-3_dora.json b/data/models/ba2han_llama-phi-3_dora.json deleted file mode 100644 index cb2091cd8d6f954c5e00866a937f68e74a39f6dc..0000000000000000000000000000000000000000 --- a/data/models/ba2han_llama-phi-3_dora.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-Phi-3_DoRA", - "id": "Ba2han/Llama-Phi-3_DoRA", - "developer": "Ba2han", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "3.821" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Ba2han_Llama-Phi-3_DoRA/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5131 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5515 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1216 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3263 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4069 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3915 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/baai_gemma2-9b-it-simpo-infinity-preference.json b/data/models/baai_gemma2-9b-it-simpo-infinity-preference.json deleted file mode 100644 index a61f588d8e614a825810028b827c0a06701df51b..0000000000000000000000000000000000000000 --- a/data/models/baai_gemma2-9b-it-simpo-infinity-preference.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Gemma2-9B-IT-Simpo-Infinity-Preference", - "id": "BAAI/Gemma2-9B-IT-Simpo-Infinity-Preference", - "developer": "BAAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "9.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/BAAI_Gemma2-9B-IT-Simpo-Infinity-Preference/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3176 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5979 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0974 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3398 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3966 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3869 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/baai_infinity-instruct-3m-0613-llama3-70b.json b/data/models/baai_infinity-instruct-3m-0613-llama3-70b.json deleted file mode 100644 index 7438fe69f929cc4aa05587eb21da1d5f9ba12e9a..0000000000000000000000000000000000000000 --- a/data/models/baai_infinity-instruct-3m-0613-llama3-70b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Infinity-Instruct-3M-0613-Llama3-70B", - "id": "BAAI/Infinity-Instruct-3M-0613-Llama3-70B", - "developer": "BAAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "70.554" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/BAAI_Infinity-Instruct-3M-0613-Llama3-70B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6821 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6642 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2153 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3582 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4523 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.473 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/baai_infinity-instruct-3m-0613-mistral-7b.json b/data/models/baai_infinity-instruct-3m-0613-mistral-7b.json deleted file mode 100644 index adf7bee1eaffe5a2e92fadf378e3ed40cdcf4729..0000000000000000000000000000000000000000 --- a/data/models/baai_infinity-instruct-3m-0613-mistral-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Infinity-Instruct-3M-0613-Mistral-7B", - "id": "BAAI/Infinity-Instruct-3M-0613-Mistral-7B", - "developer": "BAAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/BAAI_Infinity-Instruct-3M-0613-Mistral-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.532 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4958 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0816 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2961 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4351 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3161 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/baai_infinity-instruct-3m-0625-llama3-70b.json b/data/models/baai_infinity-instruct-3m-0625-llama3-70b.json deleted file mode 100644 index 38aaa72339d90291c2eff8ca883ea9e209770249..0000000000000000000000000000000000000000 --- a/data/models/baai_infinity-instruct-3m-0625-llama3-70b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Infinity-Instruct-3M-0625-Llama3-70B", - "id": "BAAI/Infinity-Instruct-3M-0625-Llama3-70B", - "developer": "BAAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "70.554" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/BAAI_Infinity-Instruct-3M-0625-Llama3-70B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7442 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.667 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2251 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3574 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4617 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4586 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/baai_infinity-instruct-3m-0625-llama3-8b.json b/data/models/baai_infinity-instruct-3m-0625-llama3-8b.json deleted file mode 100644 index a2976b761fef8e4dd2ebb4838a079bad55eb595e..0000000000000000000000000000000000000000 --- a/data/models/baai_infinity-instruct-3m-0625-llama3-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Infinity-Instruct-3M-0625-Llama3-8B", - "id": "BAAI/Infinity-Instruct-3M-0625-Llama3-8B", - "developer": "BAAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/BAAI_Infinity-Instruct-3M-0625-Llama3-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.605 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4955 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0884 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2752 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3712 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3252 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/baai_infinity-instruct-3m-0625-mistral-7b.json b/data/models/baai_infinity-instruct-3m-0625-mistral-7b.json deleted file mode 100644 index 55d2b46f80a8c3ce960d4e8c4071fe9e09f1c28b..0000000000000000000000000000000000000000 --- a/data/models/baai_infinity-instruct-3m-0625-mistral-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Infinity-Instruct-3M-0625-Mistral-7B", - "id": "BAAI/Infinity-Instruct-3M-0625-Mistral-7B", - "developer": "BAAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/BAAI_Infinity-Instruct-3M-0625-Mistral-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5867 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.494 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0763 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2869 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4272 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.323 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/baai_infinity-instruct-3m-0625-qwen2-7b.json b/data/models/baai_infinity-instruct-3m-0625-qwen2-7b.json deleted file mode 100644 index d78274bb337510c3e6223de748f7dbe53aacb045..0000000000000000000000000000000000000000 --- a/data/models/baai_infinity-instruct-3m-0625-qwen2-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Infinity-Instruct-3M-0625-Qwen2-7B", - "id": "BAAI/Infinity-Instruct-3M-0625-Qwen2-7B", - "developer": "BAAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/BAAI_Infinity-Instruct-3M-0625-Qwen2-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5554 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5346 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1926 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3129 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3888 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.396 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/baai_infinity-instruct-3m-0625-yi-1.5-9b.json b/data/models/baai_infinity-instruct-3m-0625-yi-1.5-9b.json deleted file mode 100644 index 91adbf33dc1e9e61942714c31af3cfcada8f0a45..0000000000000000000000000000000000000000 --- a/data/models/baai_infinity-instruct-3m-0625-yi-1.5-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Infinity-Instruct-3M-0625-Yi-1.5-9B", - "id": "BAAI/Infinity-Instruct-3M-0625-Yi-1.5-9B", - "developer": "BAAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.829" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/BAAI_Infinity-Instruct-3M-0625-Yi-1.5-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5186 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5509 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1639 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.354 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4575 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4118 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/baai_infinity-instruct-7m-0729-llama3_1-8b.json b/data/models/baai_infinity-instruct-7m-0729-llama3_1-8b.json deleted file mode 100644 index 17dc5015a2da9e8e4e68fc410cfc5ef416375615..0000000000000000000000000000000000000000 --- a/data/models/baai_infinity-instruct-7m-0729-llama3_1-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Infinity-Instruct-7M-0729-Llama3_1-8B", - "id": "BAAI/Infinity-Instruct-7M-0729-Llama3_1-8B", - "developer": "BAAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/BAAI_Infinity-Instruct-7M-0729-Llama3_1-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6132 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5077 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1276 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2928 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3578 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3224 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/baai_infinity-instruct-7m-0729-mistral-7b.json b/data/models/baai_infinity-instruct-7m-0729-mistral-7b.json deleted file mode 100644 index d3eff32c89aef5dd1636b65f786a4d9506d25eec..0000000000000000000000000000000000000000 --- a/data/models/baai_infinity-instruct-7m-0729-mistral-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Infinity-Instruct-7M-0729-mistral-7B", - "id": "BAAI/Infinity-Instruct-7M-0729-mistral-7B", - "developer": "BAAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/BAAI_Infinity-Instruct-7M-0729-mistral-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6162 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4964 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0831 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2903 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4062 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3274 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/baai_infinity-instruct-7m-gen-llama3_1-70b.json b/data/models/baai_infinity-instruct-7m-gen-llama3_1-70b.json deleted file mode 100644 index 2ce5cb346a6e27c8b9207c28190c610d1eb129d0..0000000000000000000000000000000000000000 --- a/data/models/baai_infinity-instruct-7m-gen-llama3_1-70b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Infinity-Instruct-7M-Gen-Llama3_1-70B", - "id": "BAAI/Infinity-Instruct-7M-Gen-Llama3_1-70B", - "developer": "BAAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "70.554" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/BAAI_Infinity-Instruct-7M-Gen-Llama3_1-70B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7335 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6695 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2523 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3758 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4539 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4607 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/baai_infinity-instruct-7m-gen-llama3_1-8b.json b/data/models/baai_infinity-instruct-7m-gen-llama3_1-8b.json deleted file mode 100644 index d52b761df89d8d976af36bf06e6deae23797d5b2..0000000000000000000000000000000000000000 --- a/data/models/baai_infinity-instruct-7m-gen-llama3_1-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Infinity-Instruct-7M-Gen-Llama3_1-8B", - "id": "BAAI/Infinity-Instruct-7M-Gen-Llama3_1-8B", - "developer": "BAAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/BAAI_Infinity-Instruct-7M-Gen-Llama3_1-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6132 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5077 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1276 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2928 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3578 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3224 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/baai_infinity-instruct-7m-gen-mistral-7b.json b/data/models/baai_infinity-instruct-7m-gen-mistral-7b.json deleted file mode 100644 index 0c655561481693526eeff7463aa43654228d9839..0000000000000000000000000000000000000000 --- a/data/models/baai_infinity-instruct-7m-gen-mistral-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Infinity-Instruct-7M-Gen-mistral-7B", - "id": "BAAI/Infinity-Instruct-7M-Gen-mistral-7B", - "developer": "BAAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/BAAI_Infinity-Instruct-7M-Gen-mistral-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6147 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4964 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0831 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2903 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4062 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3274 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/baai_opi-llama-3.1-8b-instruct.json b/data/models/baai_opi-llama-3.1-8b-instruct.json deleted file mode 100644 index fa2b0d1795c09c8dffc2c6af1ddcf08c1d5888a7..0000000000000000000000000000000000000000 --- a/data/models/baai_opi-llama-3.1-8b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "OPI-Llama-3.1-8B-Instruct", - "id": "BAAI/OPI-Llama-3.1-8B-Instruct", - "developer": "BAAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/BAAI_OPI-Llama-3.1-8B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2075 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3551 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0136 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2743 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3233 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2124 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/baconnier_napoleon_24b_v0.0.json b/data/models/baconnier_napoleon_24b_v0.0.json deleted file mode 100644 index d51269c0200498b8dd4f1e3d0fd4744df9320e4e..0000000000000000000000000000000000000000 --- a/data/models/baconnier_napoleon_24b_v0.0.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Napoleon_24B_V0.0", - "id": "baconnier/Napoleon_24B_V0.0", - "developer": "baconnier", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "23.572" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/baconnier_Napoleon_24B_V0.0/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1801 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6367 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2273 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3792 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.442 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.504 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/baconnier_napoleon_24b_v0.2.json b/data/models/baconnier_napoleon_24b_v0.2.json deleted file mode 100644 index 2f10ce9ab08c11448b2f838e44f91171fb70f9e3..0000000000000000000000000000000000000000 --- a/data/models/baconnier_napoleon_24b_v0.2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Napoleon_24B_V0.2", - "id": "baconnier/Napoleon_24B_V0.2", - "developer": "baconnier", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "23.572" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/baconnier_Napoleon_24B_V0.2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2527 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5911 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1435 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3381 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.446 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4357 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/baebee_7b-cetacea.json b/data/models/baebee_7b-cetacea.json deleted file mode 100644 index 4b72f1a84c51ffc396630c912a601fbb8f834480..0000000000000000000000000000000000000000 --- a/data/models/baebee_7b-cetacea.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "7B-Cetacea", - "id": "baebee/7B-Cetacea", - "developer": "baebee", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/baebee_7B-Cetacea/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5279 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4757 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0468 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2861 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4136 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2955 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/baebee_mergekit-model_stock-nzjnheg.json b/data/models/baebee_mergekit-model_stock-nzjnheg.json deleted file mode 100644 index bf067af33be9cf344238dd0b2bc0a284ddd37cff..0000000000000000000000000000000000000000 --- a/data/models/baebee_mergekit-model_stock-nzjnheg.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "mergekit-model_stock-nzjnheg", - "id": "baebee/mergekit-model_stock-nzjnheg", - "developer": "baebee", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/baebee_mergekit-model_stock-nzjnheg/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4844 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5287 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1677 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2802 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3847 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3699 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/baebee_mergekit-ties-fnjenli.json b/data/models/baebee_mergekit-ties-fnjenli.json deleted file mode 100644 index 84a9f53fc1d77f6f975a3370cdb7e2081127e811..0000000000000000000000000000000000000000 --- a/data/models/baebee_mergekit-ties-fnjenli.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "mergekit-ties-fnjenli", - "id": "baebee/mergekit-ties-fnjenli", - "developer": "baebee", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/baebee_mergekit-ties-fnjenli/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1988 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3024 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0023 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.245 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4019 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1129 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bamec66557_mergekit-model_stock-zdaysvi.json b/data/models/bamec66557_mergekit-model_stock-zdaysvi.json deleted file mode 100644 index 23bea6e48892090e149372cf12dd1f65a4b1c22b..0000000000000000000000000000000000000000 --- a/data/models/bamec66557_mergekit-model_stock-zdaysvi.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "mergekit-model_stock-zdaysvi", - "id": "bamec66557/mergekit-model_stock-zdaysvi", - "developer": "bamec66557", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "6.124" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bamec66557_mergekit-model_stock-zdaysvi/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6426 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5063 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1352 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3138 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4124 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3688 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bamec66557_mergekit-ties-sinbkow.json b/data/models/bamec66557_mergekit-ties-sinbkow.json deleted file mode 100644 index 4c51914ce2693acf6f1e200ab3d6eac0e23e19b7..0000000000000000000000000000000000000000 --- a/data/models/bamec66557_mergekit-ties-sinbkow.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "mergekit-ties-sinbkow", - "id": "bamec66557/mergekit-ties-sinbkow", - "developer": "bamec66557", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "6.124" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bamec66557_mergekit-ties-sinbkow/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6432 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5092 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.145 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3196 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4045 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3603 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bamec66557_mischievous-12b-mix_0.1v.json b/data/models/bamec66557_mischievous-12b-mix_0.1v.json deleted file mode 100644 index 1a5f5f956742770e79dfd6547d5ade07e2874dad..0000000000000000000000000000000000000000 --- a/data/models/bamec66557_mischievous-12b-mix_0.1v.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MISCHIEVOUS-12B-Mix_0.1v", - "id": "bamec66557/MISCHIEVOUS-12B-Mix_0.1v", - "developer": "bamec66557", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bamec66557_MISCHIEVOUS-12B-Mix_0.1v/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3636 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5436 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1329 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.328 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4132 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3674 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bamec66557_mischievous-12b-mix_0.2v.json b/data/models/bamec66557_mischievous-12b-mix_0.2v.json deleted file mode 100644 index eb12a2f3276166bcb84f9d4587fe61745e38f080..0000000000000000000000000000000000000000 --- a/data/models/bamec66557_mischievous-12b-mix_0.2v.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MISCHIEVOUS-12B-Mix_0.2v", - "id": "bamec66557/MISCHIEVOUS-12B-Mix_0.2v", - "developer": "bamec66557", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bamec66557_MISCHIEVOUS-12B-Mix_0.2v/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3624 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5434 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1261 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3255 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4158 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3663 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bamec66557_mischievous-12b-mix_0.3v.json b/data/models/bamec66557_mischievous-12b-mix_0.3v.json deleted file mode 100644 index 1523cfa841ea5ce27195b9c163c6820402807907..0000000000000000000000000000000000000000 --- a/data/models/bamec66557_mischievous-12b-mix_0.3v.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MISCHIEVOUS-12B-Mix_0.3v", - "id": "bamec66557/MISCHIEVOUS-12B-Mix_0.3v", - "developer": "bamec66557", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bamec66557_MISCHIEVOUS-12B-Mix_0.3v/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.387 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5431 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1337 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3196 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4131 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3664 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bamec66557_mischievous-12b-mix_0.4v.json b/data/models/bamec66557_mischievous-12b-mix_0.4v.json deleted file mode 100644 index c3ebdee2bd22296de4bbc027467275899255f992..0000000000000000000000000000000000000000 --- a/data/models/bamec66557_mischievous-12b-mix_0.4v.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MISCHIEVOUS-12B-Mix_0.4v", - "id": "bamec66557/MISCHIEVOUS-12B-Mix_0.4v", - "developer": "bamec66557", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bamec66557_MISCHIEVOUS-12B-Mix_0.4v/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6508 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5094 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1352 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3171 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4176 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3683 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bamec66557_mischievous-12b-mix_0.5v.json b/data/models/bamec66557_mischievous-12b-mix_0.5v.json deleted file mode 100644 index 1b26cbc9e662852bdb1f6883b322b3877121b14e..0000000000000000000000000000000000000000 --- a/data/models/bamec66557_mischievous-12b-mix_0.5v.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MISCHIEVOUS-12B-Mix_0.5v", - "id": "bamec66557/MISCHIEVOUS-12B-Mix_0.5v", - "developer": "bamec66557", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bamec66557_MISCHIEVOUS-12B-Mix_0.5v/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3746 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5422 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1367 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3205 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4132 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3661 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bamec66557_mischievous-12b-mix_0.6v.json b/data/models/bamec66557_mischievous-12b-mix_0.6v.json deleted file mode 100644 index 8c9d1cc337ea0c6abc14347ed0d9dad2ca07790a..0000000000000000000000000000000000000000 --- a/data/models/bamec66557_mischievous-12b-mix_0.6v.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MISCHIEVOUS-12B-Mix_0.6v", - "id": "bamec66557/MISCHIEVOUS-12B-Mix_0.6v", - "developer": "bamec66557", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bamec66557_MISCHIEVOUS-12B-Mix_0.6v/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4366 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5449 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1254 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.328 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4185 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3662 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bamec66557_mischievous-12b-mix_iii_ex_v.json b/data/models/bamec66557_mischievous-12b-mix_iii_ex_v.json deleted file mode 100644 index 66480f58acbb56c54d5e7aa5af11f09a54f51ff9..0000000000000000000000000000000000000000 --- a/data/models/bamec66557_mischievous-12b-mix_iii_ex_v.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MISCHIEVOUS-12B-Mix_III_ex_V", - "id": "bamec66557/MISCHIEVOUS-12B-Mix_III_ex_V", - "developer": "bamec66557", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bamec66557_MISCHIEVOUS-12B-Mix_III_ex_V/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4316 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5449 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1322 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3205 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4198 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3649 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bamec66557_mischievous-12b-mix_iii_iv_v.json b/data/models/bamec66557_mischievous-12b-mix_iii_iv_v.json deleted file mode 100644 index 5fc0197e6b66de31ca8e3d1102b4253788ef81e9..0000000000000000000000000000000000000000 --- a/data/models/bamec66557_mischievous-12b-mix_iii_iv_v.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MISCHIEVOUS-12B-Mix_III_IV_V", - "id": "bamec66557/MISCHIEVOUS-12B-Mix_III_IV_V", - "developer": "bamec66557", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bamec66557_MISCHIEVOUS-12B-Mix_III_IV_V/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4031 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5465 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1292 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3205 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4198 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3664 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bamec66557_mischievous-12b-mix_neo.json b/data/models/bamec66557_mischievous-12b-mix_neo.json deleted file mode 100644 index 01b6be41fa1b39c45871a7cb0aeefe3e0dfdfdcc..0000000000000000000000000000000000000000 --- a/data/models/bamec66557_mischievous-12b-mix_neo.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MISCHIEVOUS-12B-Mix_Neo", - "id": "bamec66557/MISCHIEVOUS-12B-Mix_Neo", - "developer": "bamec66557", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bamec66557_MISCHIEVOUS-12B-Mix_Neo/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.625 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5078 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1329 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3163 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.415 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3685 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bamec66557_mischievous-12b.json b/data/models/bamec66557_mischievous-12b.json deleted file mode 100644 index 2042f9469fd4c00ec296e173c461d4aad59ea9d0..0000000000000000000000000000000000000000 --- a/data/models/bamec66557_mischievous-12b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MISCHIEVOUS-12B", - "id": "bamec66557/MISCHIEVOUS-12B", - "developer": "bamec66557", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bamec66557_MISCHIEVOUS-12B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3852 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5405 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1276 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3205 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4145 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3672 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bamec66557_mistral-nemo-vicious_mesh-12b-2407.json b/data/models/bamec66557_mistral-nemo-vicious_mesh-12b-2407.json deleted file mode 100644 index 80a203766683105443c594404f7ae30bbbc3787f..0000000000000000000000000000000000000000 --- a/data/models/bamec66557_mistral-nemo-vicious_mesh-12b-2407.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mistral-Nemo-VICIOUS_MESH-12B-2407", - "id": "bamec66557/Mistral-Nemo-VICIOUS_MESH-12B-2407", - "developer": "bamec66557", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bamec66557_Mistral-Nemo-VICIOUS_MESH-12B-2407/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6706 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5156 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1367 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3154 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.431 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3677 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bamec66557_nameless-12b-prob.json b/data/models/bamec66557_nameless-12b-prob.json deleted file mode 100644 index 7589008c93829b68b954ba5f7c42f3b5a721efa0..0000000000000000000000000000000000000000 --- a/data/models/bamec66557_nameless-12b-prob.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "NameLess-12B-prob", - "id": "bamec66557/NameLess-12B-prob", - "developer": "bamec66557", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bamec66557_NameLess-12B-prob/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6602 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5158 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1261 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3146 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4336 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3684 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bamec66557_vicious_mesh-12b-0.1v.json b/data/models/bamec66557_vicious_mesh-12b-0.1v.json deleted file mode 100644 index 3f399d10565bd27eb10624c5dd6fd4484fce50ea..0000000000000000000000000000000000000000 --- a/data/models/bamec66557_vicious_mesh-12b-0.1v.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "VICIOUS_MESH-12B-0.1v", - "id": "bamec66557/VICIOUS_MESH-12B-0.1v", - "developer": "bamec66557", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "6.124" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bamec66557_VICIOUS_MESH-12B-0.1v/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3657 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5412 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1322 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3247 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4158 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3683 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bamec66557_vicious_mesh-12b-0.x.ver.json b/data/models/bamec66557_vicious_mesh-12b-0.x.ver.json deleted file mode 100644 index a758121afb9ba4afe3034b84e9c00292aa61e306..0000000000000000000000000000000000000000 --- a/data/models/bamec66557_vicious_mesh-12b-0.x.ver.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "VICIOUS_MESH-12B-0.X.ver", - "id": "bamec66557/VICIOUS_MESH-12B-0.X.ver", - "developer": "bamec66557", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "6.124" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bamec66557_VICIOUS_MESH-12B-0.X.ver/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3776 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5416 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1201 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3213 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4198 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3671 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bamec66557_vicious_mesh-12b-alpha.json b/data/models/bamec66557_vicious_mesh-12b-alpha.json deleted file mode 100644 index 5849c8bb911db723a05788e42090c500e191a7c3..0000000000000000000000000000000000000000 --- a/data/models/bamec66557_vicious_mesh-12b-alpha.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "VICIOUS_MESH-12B-ALPHA", - "id": "bamec66557/VICIOUS_MESH-12B-ALPHA", - "developer": "bamec66557", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bamec66557_VICIOUS_MESH-12B-ALPHA/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6365 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5094 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1367 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3138 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4203 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3697 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bamec66557_vicious_mesh-12b-beta.json b/data/models/bamec66557_vicious_mesh-12b-beta.json deleted file mode 100644 index be552318300c93457f8f573e9b8bf72fb9ab96a5..0000000000000000000000000000000000000000 --- a/data/models/bamec66557_vicious_mesh-12b-beta.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "VICIOUS_MESH-12B-BETA", - "id": "bamec66557/VICIOUS_MESH-12B-BETA", - "developer": "bamec66557", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bamec66557_VICIOUS_MESH-12B-BETA/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6721 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5156 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1329 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3163 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.431 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3679 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bamec66557_vicious_mesh-12b-delta.json b/data/models/bamec66557_vicious_mesh-12b-delta.json deleted file mode 100644 index a5e763ccb59a2e917590dd1c6566b2956adf6bf0..0000000000000000000000000000000000000000 --- a/data/models/bamec66557_vicious_mesh-12b-delta.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "VICIOUS_MESH-12B-DELTA", - "id": "bamec66557/VICIOUS_MESH-12B-DELTA", - "developer": "bamec66557", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "6.124" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bamec66557_VICIOUS_MESH-12B-DELTA/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6469 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5055 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1375 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3121 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4057 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3651 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bamec66557_vicious_mesh-12b-digamma.json b/data/models/bamec66557_vicious_mesh-12b-digamma.json deleted file mode 100644 index aea2f85ed851b2eb04dec1864e89218f7c9b1557..0000000000000000000000000000000000000000 --- a/data/models/bamec66557_vicious_mesh-12b-digamma.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "VICIOUS_MESH-12B-DIGAMMA", - "id": "bamec66557/VICIOUS_MESH-12B-DIGAMMA", - "developer": "bamec66557", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "6.124" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bamec66557_VICIOUS_MESH-12B-DIGAMMA/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6429 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5061 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1337 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3129 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4097 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3659 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bamec66557_vicious_mesh-12b-epsilon.json b/data/models/bamec66557_vicious_mesh-12b-epsilon.json deleted file mode 100644 index b7b50630e6c68385d0b16d7b4605ecf9bd93ba67..0000000000000000000000000000000000000000 --- a/data/models/bamec66557_vicious_mesh-12b-epsilon.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "VICIOUS_MESH-12B-EPSILON", - "id": "bamec66557/VICIOUS_MESH-12B-EPSILON", - "developer": "bamec66557", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "6.124" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bamec66557_VICIOUS_MESH-12B-EPSILON/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6305 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5038 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1261 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3146 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.407 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3648 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bamec66557_vicious_mesh-12b-gamma.json b/data/models/bamec66557_vicious_mesh-12b-gamma.json deleted file mode 100644 index c405d7d77354b2176858a0096cc7bcae7bd9b7bf..0000000000000000000000000000000000000000 --- a/data/models/bamec66557_vicious_mesh-12b-gamma.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "VICIOUS_MESH-12B-GAMMA", - "id": "bamec66557/VICIOUS_MESH-12B-GAMMA", - "developer": "bamec66557", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bamec66557_VICIOUS_MESH-12B-GAMMA/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6362 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5182 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1307 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3138 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4363 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3666 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bamec66557_vicious_mesh-12b-nemo.json b/data/models/bamec66557_vicious_mesh-12b-nemo.json deleted file mode 100644 index bff38f7bd0f47fcd3505d42e6032bb8a1e64ce1a..0000000000000000000000000000000000000000 --- a/data/models/bamec66557_vicious_mesh-12b-nemo.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "VICIOUS_MESH-12B-NEMO", - "id": "bamec66557/VICIOUS_MESH-12B-NEMO", - "developer": "bamec66557", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bamec66557_VICIOUS_MESH-12B-NEMO/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4022 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5442 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1269 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3238 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4251 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3716 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bamec66557_vicious_mesh-12b-omega.json b/data/models/bamec66557_vicious_mesh-12b-omega.json deleted file mode 100644 index 32fece31400e22e057bb4fc8859ae6e74b8b9eb6..0000000000000000000000000000000000000000 --- a/data/models/bamec66557_vicious_mesh-12b-omega.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "VICIOUS_MESH-12B-OMEGA", - "id": "bamec66557/VICIOUS_MESH-12B-OMEGA", - "developer": "bamec66557", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bamec66557_VICIOUS_MESH-12B-OMEGA/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.67 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5166 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1344 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3154 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4323 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3677 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bamec66557_vicious_mesh-12b-union.json b/data/models/bamec66557_vicious_mesh-12b-union.json deleted file mode 100644 index 785528be48e131bc829a32757ca6a48e18758c7f..0000000000000000000000000000000000000000 --- a/data/models/bamec66557_vicious_mesh-12b-union.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "VICIOUS_MESH-12B-UNION", - "id": "bamec66557/VICIOUS_MESH-12B-UNION", - "developer": "bamec66557", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "6.124" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bamec66557_VICIOUS_MESH-12B-UNION/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6429 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5107 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.139 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3121 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4257 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3672 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bamec66557_vicious_mesh-12b.json b/data/models/bamec66557_vicious_mesh-12b.json deleted file mode 100644 index 668e042900793736d9bcb3ef24bc6df1086c5e26..0000000000000000000000000000000000000000 --- a/data/models/bamec66557_vicious_mesh-12b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "VICIOUS_MESH-12B", - "id": "bamec66557/VICIOUS_MESH-12B", - "developer": "bamec66557", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "6.124" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bamec66557_VICIOUS_MESH-12B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3716 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5436 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1344 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.328 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4105 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3679 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bamec66557_vicious_mesh-12b_razor.json b/data/models/bamec66557_vicious_mesh-12b_razor.json deleted file mode 100644 index d949a1945f7282c53ad664853c58db0f24f60581..0000000000000000000000000000000000000000 --- a/data/models/bamec66557_vicious_mesh-12b_razor.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "VICIOUS_MESH-12B_Razor", - "id": "bamec66557/VICIOUS_MESH-12B_Razor", - "developer": "bamec66557", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "6.124" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bamec66557_VICIOUS_MESH-12B_Razor/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3736 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5447 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1299 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.323 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4092 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3669 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/baptiste-huvelle-10_letriomphant2.2_ece_ilab.json b/data/models/baptiste-huvelle-10_letriomphant2.2_ece_ilab.json deleted file mode 100644 index 84303f11fb7247fd9496bd24ae68fea5308f3f34..0000000000000000000000000000000000000000 --- a/data/models/baptiste-huvelle-10_letriomphant2.2_ece_ilab.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "LeTriomphant2.2_ECE_iLAB", - "id": "Baptiste-HUVELLE-10/LeTriomphant2.2_ECE_iLAB", - "developer": "Baptiste-HUVELLE-10", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "72.706" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Baptiste-HUVELLE-10_LeTriomphant2.2_ECE_iLAB/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5076 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7256 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4449 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3993 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4626 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5851 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bee-spoke-data_meta-llama-3-8bee.json b/data/models/bee-spoke-data_meta-llama-3-8bee.json deleted file mode 100644 index de2f0442474dca05ef4fbcedfe9a497b4dedadd4..0000000000000000000000000000000000000000 --- a/data/models/bee-spoke-data_meta-llama-3-8bee.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Meta-Llama-3-8Bee", - "id": "BEE-spoke-data/Meta-Llama-3-8Bee", - "developer": "BEE-spoke-data", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/BEE-spoke-data_Meta-Llama-3-8Bee/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1951 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4626 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0483 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3138 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3654 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.322 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bee-spoke-data_smol_llama-101m-gqa.json b/data/models/bee-spoke-data_smol_llama-101m-gqa.json deleted file mode 100644 index 294cbbfb9d20b14d29f032643b615cd049e945c0..0000000000000000000000000000000000000000 --- a/data/models/bee-spoke-data_smol_llama-101m-gqa.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "smol_llama-101M-GQA", - "id": "BEE-spoke-data/smol_llama-101M-GQA", - "developer": "BEE-spoke-data", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "0.101" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/BEE-spoke-data_smol_llama-101M-GQA/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1384 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3018 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.006 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2576 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3713 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1107 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bee-spoke-data_smol_llama-220m-gqa-fineweb_edu.json b/data/models/bee-spoke-data_smol_llama-220m-gqa-fineweb_edu.json deleted file mode 100644 index 9058e0c222db45c8bc1bb11a1c3fdba01de46b19..0000000000000000000000000000000000000000 --- a/data/models/bee-spoke-data_smol_llama-220m-gqa-fineweb_edu.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "smol_llama-220M-GQA-fineweb_edu", - "id": "BEE-spoke-data/smol_llama-220M-GQA-fineweb_edu", - "developer": "BEE-spoke-data", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "0.218" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/BEE-spoke-data_smol_llama-220M-GQA-fineweb_edu/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1988 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2929 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0068 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2592 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4368 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1127 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bee-spoke-data_smol_llama-220m-gqa.json b/data/models/bee-spoke-data_smol_llama-220m-gqa.json deleted file mode 100644 index 8abe650b0bcb210120ccffe3ed8e02836dab5997..0000000000000000000000000000000000000000 --- a/data/models/bee-spoke-data_smol_llama-220m-gqa.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "smol_llama-220M-GQA", - "id": "BEE-spoke-data/smol_llama-220M-GQA", - "developer": "BEE-spoke-data", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "0.218" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/BEE-spoke-data_smol_llama-220M-GQA/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2386 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3032 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0106 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2559 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4059 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1149 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bee-spoke-data_smol_llama-220m-openhermes.json b/data/models/bee-spoke-data_smol_llama-220m-openhermes.json deleted file mode 100644 index 285879e6537984665520960117460a194a24939b..0000000000000000000000000000000000000000 --- a/data/models/bee-spoke-data_smol_llama-220m-openhermes.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "smol_llama-220M-openhermes", - "id": "BEE-spoke-data/smol_llama-220M-openhermes", - "developer": "BEE-spoke-data", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "0.218" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/BEE-spoke-data_smol_llama-220M-openhermes/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1555 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3028 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0106 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2676 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3847 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.112 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bee-spoke-data_tfine-900m-e16-d32-flan-infinity-instruct-7m-t2t_en-1024.json b/data/models/bee-spoke-data_tfine-900m-e16-d32-flan-infinity-instruct-7m-t2t_en-1024.json deleted file mode 100644 index d6c821c66c4d3c2cb6856728c0ebda963303b155..0000000000000000000000000000000000000000 --- a/data/models/bee-spoke-data_tfine-900m-e16-d32-flan-infinity-instruct-7m-t2t_en-1024.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "tFINE-900m-e16-d32-flan-infinity-instruct-7m-T2T_en-1024", - "id": "BEE-spoke-data/tFINE-900m-e16-d32-flan-infinity-instruct-7m-T2T_en-1024", - "developer": "BEE-spoke-data", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "T5ForConditionalGeneration", - "params_billions": "0.887" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/BEE-spoke-data_tFINE-900m-e16-d32-flan-infinity-instruct-7m-T2T_en-1024/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1321 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3138 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0106 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2542 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4393 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1237 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bee-spoke-data_tfine-900m-e16-d32-flan.json b/data/models/bee-spoke-data_tfine-900m-e16-d32-flan.json deleted file mode 100644 index e459ecaeb83bf6be111135dd100c658acfe406bb..0000000000000000000000000000000000000000 --- a/data/models/bee-spoke-data_tfine-900m-e16-d32-flan.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "tFINE-900m-e16-d32-flan", - "id": "BEE-spoke-data/tFINE-900m-e16-d32-flan", - "developer": "BEE-spoke-data", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "T5ForConditionalGeneration", - "params_billions": "0.887" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/BEE-spoke-data_tFINE-900m-e16-d32-flan/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1506 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3028 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0098 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2332 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3724 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1307 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bee-spoke-data_tfine-900m-e16-d32-instruct_2e.json b/data/models/bee-spoke-data_tfine-900m-e16-d32-instruct_2e.json deleted file mode 100644 index 7fcb74de656f5372f0d8860b8c4b042ee0d1aa56..0000000000000000000000000000000000000000 --- a/data/models/bee-spoke-data_tfine-900m-e16-d32-instruct_2e.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "tFINE-900m-e16-d32-instruct_2e", - "id": "BEE-spoke-data/tFINE-900m-e16-d32-instruct_2e", - "developer": "BEE-spoke-data", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "T5ForConditionalGeneration", - "params_billions": "0.887" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/BEE-spoke-data_tFINE-900m-e16-d32-instruct_2e/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1403 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3135 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0136 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2592 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4207 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1237 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bee-spoke-data_tfine-900m-instruct-orpo.json b/data/models/bee-spoke-data_tfine-900m-instruct-orpo.json deleted file mode 100644 index be4c6359cf0c1161718b0aaab4cb82f1de88763d..0000000000000000000000000000000000000000 --- a/data/models/bee-spoke-data_tfine-900m-instruct-orpo.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "tFINE-900m-instruct-orpo", - "id": "BEE-spoke-data/tFINE-900m-instruct-orpo", - "developer": "BEE-spoke-data", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "T5ForConditionalGeneration", - "params_billions": "0.887" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/BEE-spoke-data_tFINE-900m-instruct-orpo/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.133 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3022 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0159 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2592 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3409 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1152 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/belztjti_dffghgjh.json b/data/models/belztjti_dffghgjh.json deleted file mode 100644 index a56042552aea73e2cd21533d80b8cf49137206b9..0000000000000000000000000000000000000000 --- a/data/models/belztjti_dffghgjh.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "dffghgjh", - "id": "belztjti/dffghgjh", - "developer": "belztjti", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "GlmForCausalLM", - "params_billions": "9.543" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/belztjti_dffghgjh/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5784 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3582 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0234 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2634 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3475 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3422 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/belztjti_dtfgv.json b/data/models/belztjti_dtfgv.json deleted file mode 100644 index 46ed3ec48edd6a0e01ce0b051118ee12ae15d5fe..0000000000000000000000000000000000000000 --- a/data/models/belztjti_dtfgv.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "dtfgv", - "id": "belztjti/dtfgv", - "developer": "belztjti", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "9.543" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/belztjti_dtfgv/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3345 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3282 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0181 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2693 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3794 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1504 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/benevolencemessiah_qwen2.5-72b-2x-instruct-ties-v1.0.json b/data/models/benevolencemessiah_qwen2.5-72b-2x-instruct-ties-v1.0.json deleted file mode 100644 index 5548da2ae12d716777741f032c3ae35b20fe87cc..0000000000000000000000000000000000000000 --- a/data/models/benevolencemessiah_qwen2.5-72b-2x-instruct-ties-v1.0.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-72B-2x-Instruct-TIES-v1.0", - "id": "BenevolenceMessiah/Qwen2.5-72B-2x-Instruct-TIES-v1.0", - "developer": "BenevolenceMessiah", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "72.7" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/BenevolenceMessiah_Qwen2.5-72B-2x-Instruct-TIES-v1.0/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5473 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7273 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5785 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3674 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4207 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5628 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/benevolencemessiah_yi-coder-9b-chat-instruct-ties-moe-v1.0.json b/data/models/benevolencemessiah_yi-coder-9b-chat-instruct-ties-moe-v1.0.json deleted file mode 100644 index 526d109ae203b6e7d8aa688b3defb3ebf98bb796..0000000000000000000000000000000000000000 --- a/data/models/benevolencemessiah_yi-coder-9b-chat-instruct-ties-moe-v1.0.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Yi-Coder-9B-Chat-Instruct-TIES-MoE-v1.0", - "id": "BenevolenceMessiah/Yi-Coder-9B-Chat-Instruct-TIES-MoE-v1.0", - "developer": "BenevolenceMessiah", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MixtralForCausalLM", - "params_billions": "28.309" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/BenevolenceMessiah_Yi-Coder-9B-Chat-Instruct-TIES-MoE-v1.0/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3012 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4909 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0415 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2626 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.408 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.268 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/benhaotang_phi4-qwq-sky-t1.json b/data/models/benhaotang_phi4-qwq-sky-t1.json deleted file mode 100644 index 8bf7d9bbe37aef507b879e438a70271035fa02dd..0000000000000000000000000000000000000000 --- a/data/models/benhaotang_phi4-qwq-sky-t1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "phi4-qwq-sky-t1", - "id": "benhaotang/phi4-qwq-sky-t1", - "developer": "benhaotang", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/benhaotang_phi4-qwq-sky-t1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.046 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6711 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4101 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3951 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.49 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5244 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/beomi_gemma-mling-7b.json b/data/models/beomi_gemma-mling-7b.json deleted file mode 100644 index b60990fac14a11e7691f5e2e054c550e897d32bc..0000000000000000000000000000000000000000 --- a/data/models/beomi_gemma-mling-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "gemma-mling-7b", - "id": "beomi/gemma-mling-7b", - "developer": "beomi", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "GemmaForCausalLM", - "params_billions": "8.538" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/beomi_gemma-mling-7b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2029 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4068 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0544 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.25 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3759 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2633 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/beowolx_codeninja-1.0-openchat-7b.json b/data/models/beowolx_codeninja-1.0-openchat-7b.json deleted file mode 100644 index 2b807fb565a7ccecf9be1e6912394838c797a924..0000000000000000000000000000000000000000 --- a/data/models/beowolx_codeninja-1.0-openchat-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "CodeNinja-1.0-OpenChat-7B", - "id": "beowolx/CodeNinja-1.0-OpenChat-7B", - "developer": "beowolx", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/beowolx_CodeNinja-1.0-OpenChat-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5447 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4441 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0672 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2945 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4243 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3015 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/berkeley-nest_starling-lm-7b-alpha.json b/data/models/berkeley-nest_starling-lm-7b-alpha.json deleted file mode 100644 index 79ab5678b5abb05663e734fc648f9f79fc23670f..0000000000000000000000000000000000000000 --- a/data/models/berkeley-nest_starling-lm-7b-alpha.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Starling-LM-7B-alpha", - "id": "berkeley-nest/Starling-LM-7B-alpha", - "developer": "berkeley-nest", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/berkeley-nest_Starling-LM-7B-alpha/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.548 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.444 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0838 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.297 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.412 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3172 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/berkeley-nest_starling-rm-7b-alpha.json b/data/models/berkeley-nest_starling-rm-7b-alpha.json deleted file mode 100644 index 435790a295f70a90ebdd1f009112c2b89aca57d5..0000000000000000000000000000000000000000 --- a/data/models/berkeley-nest_starling-rm-7b-alpha.json +++ /dev/null @@ -1,144 +0,0 @@ -{ - "model_info": { - "name": "berkeley-nest/Starling-RM-7B-alpha", - "id": "berkeley-nest/Starling-RM-7B-alpha", - "developer": "berkeley-nest", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/berkeley-nest_Starling-RM-7B-alpha/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7113 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9804 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4561 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8446 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.58 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6794 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bfuzzy1_acheron-c.json b/data/models/bfuzzy1_acheron-c.json deleted file mode 100644 index df6bbd936e1e19d5d11d462069ffe2c28bb89d6f..0000000000000000000000000000000000000000 --- a/data/models/bfuzzy1_acheron-c.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "acheron-c", - "id": "bfuzzy1/acheron-c", - "developer": "bfuzzy1", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "0.514" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bfuzzy1_acheron-c/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1929 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3026 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.003 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2475 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3382 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1172 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bfuzzy1_acheron-d.json b/data/models/bfuzzy1_acheron-d.json deleted file mode 100644 index 3e0c119451b1854a965c6e482fdf907b93f8b605..0000000000000000000000000000000000000000 --- a/data/models/bfuzzy1_acheron-d.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "acheron-d", - "id": "bfuzzy1/acheron-d", - "developer": "bfuzzy1", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "0.514" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bfuzzy1_acheron-d/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1925 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.314 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0151 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2366 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3497 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1134 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bfuzzy1_acheron-m.json b/data/models/bfuzzy1_acheron-m.json deleted file mode 100644 index 1bc6de42a52a7e58090893b5a39c7b69295000ea..0000000000000000000000000000000000000000 --- a/data/models/bfuzzy1_acheron-m.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "acheron-m", - "id": "bfuzzy1/acheron-m", - "developer": "bfuzzy1", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "0.514" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bfuzzy1_acheron-m/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1758 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2928 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0091 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2601 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3487 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1113 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bfuzzy1_acheron-m1a-llama.json b/data/models/bfuzzy1_acheron-m1a-llama.json deleted file mode 100644 index 5e0298ff0b17a0f1634983a92d9b1e5e0073c31b..0000000000000000000000000000000000000000 --- a/data/models/bfuzzy1_acheron-m1a-llama.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "acheron-m1a-llama", - "id": "bfuzzy1/acheron-m1a-llama", - "developer": "bfuzzy1", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "0.514" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bfuzzy1_acheron-m1a-llama/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1125 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2956 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0076 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2601 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3633 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1146 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bfuzzy1_acheron.json b/data/models/bfuzzy1_acheron.json deleted file mode 100644 index 88a1d48b557f273291cccf25cbd910b5683d0c0f..0000000000000000000000000000000000000000 --- a/data/models/bfuzzy1_acheron.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "acheron", - "id": "bfuzzy1/acheron", - "developer": "bfuzzy1", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "0.514" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bfuzzy1_acheron/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1983 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3108 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0166 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2391 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3511 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1096 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bfuzzy1_gunny.json b/data/models/bfuzzy1_gunny.json deleted file mode 100644 index 5aefc41596194f47edb286d5a9bdf55b37a9af5c..0000000000000000000000000000000000000000 --- a/data/models/bfuzzy1_gunny.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Gunny", - "id": "bfuzzy1/Gunny", - "developer": "bfuzzy1", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bfuzzy1_Gunny/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7129 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4546 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.173 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2785 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3583 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3039 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bfuzzy1_llambses-1.json b/data/models/bfuzzy1_llambses-1.json deleted file mode 100644 index 093deec258168e19088a4836f71fb349b793ea2f..0000000000000000000000000000000000000000 --- a/data/models/bfuzzy1_llambses-1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "llambses-1", - "id": "bfuzzy1/llambses-1", - "developer": "bfuzzy1", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bfuzzy1_llambses-1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3554 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5047 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0687 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2978 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4529 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.314 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bhuvneshsaini_merged_model.json b/data/models/bhuvneshsaini_merged_model.json deleted file mode 100644 index 69d4fe8ffd40ccfb02ffc7a7b55ab894c9fd5ad4..0000000000000000000000000000000000000000 --- a/data/models/bhuvneshsaini_merged_model.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "merged_model", - "id": "bhuvneshsaini/merged_model", - "developer": "bhuvneshsaini", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "4.715" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bhuvneshsaini_merged_model/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1813 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.336 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.25 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3497 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1445 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bigcode_starcoder2-15b.json b/data/models/bigcode_starcoder2-15b.json deleted file mode 100644 index 9b9438a3f036c9f99980afb9a4872a978f690aa0..0000000000000000000000000000000000000000 --- a/data/models/bigcode_starcoder2-15b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "starcoder2-15b", - "id": "bigcode/starcoder2-15b", - "developer": "bigcode", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Starcoder2ForCausalLM", - "params_billions": "15.958" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bigcode_starcoder2-15b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.278 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4448 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0597 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2735 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3501 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2353 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bigcode_starcoder2-3b.json b/data/models/bigcode_starcoder2-3b.json deleted file mode 100644 index 6c19749bf630e2f8ba7f5112c25d582502d27d61..0000000000000000000000000000000000000000 --- a/data/models/bigcode_starcoder2-3b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "starcoder2-3b", - "id": "bigcode/starcoder2-3b", - "developer": "bigcode", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Starcoder2ForCausalLM", - "params_billions": "3.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bigcode_starcoder2-3b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2037 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3509 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0151 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2441 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3435 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1636 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bigcode_starcoder2-7b.json b/data/models/bigcode_starcoder2-7b.json deleted file mode 100644 index f75267fb7091bdc58f4c80ae2d3c6526806a81cb..0000000000000000000000000000000000000000 --- a/data/models/bigcode_starcoder2-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "starcoder2-7b", - "id": "bigcode/starcoder2-7b", - "developer": "bigcode", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Starcoder2ForCausalLM", - "params_billions": "7.174" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bigcode_starcoder2-7b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2209 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3661 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.031 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2517 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3793 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1642 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bigscience_bloom-176b.json b/data/models/bigscience_bloom-176b.json deleted file mode 100644 index 5673326af8b1a02a642c5705b85daec46c83bb18..0000000000000000000000000000000000000000 --- a/data/models/bigscience_bloom-176b.json +++ /dev/null @@ -1,674 +0,0 @@ -{ - "model_info": { - "name": "BLOOM 176B", - "id": "bigscience/BLOOM-176B", - "developer": "bigscience", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_classic/bigscience_BLOOM-176B/1774096308.339228", - "retrieved_timestamp": "1774096308.339228", - "source_metadata": { - "source_name": "helm_classic", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_classic", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_classic", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperform on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.446, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Calibration": "{\"description\": \"\", \"tab\": \"Calibration\", \"score\": \"0.3480016788296159\"}", - "Mean win rate - Robustness": "{\"description\": \"\", \"tab\": \"Robustness\", \"score\": \"0.5409357605686861\"}", - "Mean win rate - Fairness": "{\"description\": \"\", \"tab\": \"Fairness\", \"score\": \"0.5507003378527294\"}", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.26823464912280703\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}", - "Mean win rate - Bias": "{\"description\": \"\", \"tab\": \"Bias\", \"score\": \"0.5459762982621468\"}", - "Mean win rate - Toxicity": "{\"description\": \"\", \"tab\": \"Toxicity\", \"score\": \"0.5959534292867626\"}", - "Mean win rate - Summarization metrics": "{\"description\": \"\", \"tab\": \"Summarization metrics\", \"score\": \"0.29074770258980787\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.299, - "details": { - "description": "min=0.19, mean=0.299, max=0.42, sum=4.481 (15)", - "tab": "Accuracy", - "MMLU - ECE (10-bin)": "{\"description\": \"min=0.115, mean=0.137, max=0.173, sum=2.054 (15)\", \"tab\": \"Calibration\", \"score\": \"0.13690038983912287\"}", - "MMLU - EM (Robustness)": "{\"description\": \"min=0.167, mean=0.25, max=0.38, sum=3.754 (15)\", \"tab\": \"Robustness\", \"score\": \"0.25025730994152046\"}", - "MMLU - EM (Fairness)": "{\"description\": \"min=0.175, mean=0.274, max=0.38, sum=4.104 (15)\", \"tab\": \"Fairness\", \"score\": \"0.27360233918128657\"}", - "MMLU - Denoised inference time (s)": "{\"description\": \"min=0.135, mean=0.233, max=0.418, sum=3.493 (15)\", \"tab\": \"Efficiency\", \"score\": \"0.23288457024982262\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=1542 (15)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=75 (15)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (15)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=333.02, mean=436.99, max=574.658, sum=6554.844 (15)\", \"tab\": \"General information\", \"score\": \"436.9895789473684\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=15 (15)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "MMLU - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=45 (15)\", \"tab\": \"General information\", \"score\": \"3.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "BoolQ", - "source_data": { - "dataset_name": "BoolQ", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on BoolQ", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.704, - "details": { - "description": "min=0.659, mean=0.704, max=0.728, sum=2.112 (3)", - "tab": "Accuracy", - "BoolQ - ECE (10-bin)": "{\"description\": \"min=0.153, mean=0.209, max=0.247, sum=0.626 (3)\", \"tab\": \"Calibration\", \"score\": \"0.2086643852555177\"}", - "BoolQ - EM (Robustness)": "{\"description\": \"min=0.595, mean=0.642, max=0.674, sum=1.926 (3)\", \"tab\": \"Robustness\", \"score\": \"0.642\"}", - "BoolQ - EM (Fairness)": "{\"description\": \"min=0.601, mean=0.656, max=0.693, sum=1.968 (3)\", \"tab\": \"Fairness\", \"score\": \"0.656\"}", - "BoolQ - Denoised inference time (s)": "{\"description\": \"min=0.665, mean=0.853, max=1.05, sum=2.558 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.852823399183769\"}", - "BoolQ - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "BoolQ - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "BoolQ - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "BoolQ - # prompt tokens": "{\"description\": \"min=636.774, mean=897.107, max=1242.774, sum=2691.322 (3)\", \"tab\": \"General information\", \"score\": \"897.1073333333333\"}", - "BoolQ - # output tokens": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "BoolQ - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "BoolQ - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.662, - "details": { - "description": "min=0.631, mean=0.662, max=0.695, sum=1.986 (3)", - "tab": "Accuracy", - "NarrativeQA - ECE (10-bin)": "{\"description\": \"min=0.231, mean=0.237, max=0.242, sum=0.712 (3)\", \"tab\": \"Calibration\", \"score\": \"0.2374266630696186\"}", - "NarrativeQA - F1 (Robustness)": "{\"description\": \"min=0.468, mean=0.53, max=0.574, sum=1.591 (3)\", \"tab\": \"Robustness\", \"score\": \"0.5303029858435905\"}", - "NarrativeQA - F1 (Fairness)": "{\"description\": \"min=0.535, mean=0.577, max=0.613, sum=1.73 (3)\", \"tab\": \"Fairness\", \"score\": \"0.5767895596204061\"}", - "NarrativeQA - Denoised inference time (s)": "{\"description\": \"min=2.081, mean=2.598, max=3.427, sum=7.794 (3)\", \"tab\": \"Efficiency\", \"score\": \"2.5979962524114084\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=1065 (3)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=1.042, mean=1.621, max=2.048, sum=4.862 (3)\", \"tab\": \"General information\", \"score\": \"1.6206572769953052\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=1604.899, mean=1649.598, max=1699.146, sum=4948.794 (3)\", \"tab\": \"General information\", \"score\": \"1649.5981220657277\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=18.468, mean=33.276, max=50.499, sum=99.828 (3)\", \"tab\": \"General information\", \"score\": \"33.27605633802816\"}", - "NarrativeQA - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NarrativeQA - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NarrativeQA - Stereotypes (gender)": "{\"description\": \"min=0.333, mean=0.355, max=0.389, sum=1.065 (3)\", \"tab\": \"Bias\", \"score\": \"0.354945620223398\"}", - "NarrativeQA - Representation (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=1.333 (2)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666667\"}", - "NarrativeQA - Representation (gender)": "{\"description\": \"min=0.118, mean=0.165, max=0.241, sum=0.494 (3)\", \"tab\": \"Bias\", \"score\": \"0.16472050143449737\"}", - "NarrativeQA - Toxic fraction": "{\"description\": \"min=0.011, mean=0.012, max=0.014, sum=0.037 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.012206572769953052\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (open-book)", - "source_data": { - "dataset_name": "NaturalQuestions (open-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (open-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.621, - "details": { - "description": "min=0.61, mean=0.621, max=0.628, sum=1.864 (3)", - "tab": "Accuracy", - "NaturalQuestions (closed-book) - ECE (10-bin)": "{\"description\": \"min=0.11, mean=0.116, max=0.118, sum=0.347 (3)\", \"tab\": \"Calibration\", \"score\": \"0.11564225453050514\"}", - "NaturalQuestions (open-book) - ECE (10-bin)": "{\"description\": \"min=0.338, mean=0.347, max=0.36, sum=1.041 (3)\", \"tab\": \"Calibration\", \"score\": \"0.3469801265406112\"}", - "NaturalQuestions (closed-book) - F1 (Robustness)": "{\"description\": \"min=0.18, mean=0.185, max=0.19, sum=0.556 (3)\", \"tab\": \"Robustness\", \"score\": \"0.18537100322417385\"}", - "NaturalQuestions (open-book) - F1 (Robustness)": "{\"description\": \"min=0.547, mean=0.558, max=0.569, sum=1.675 (3)\", \"tab\": \"Robustness\", \"score\": \"0.5582069622847597\"}", - "NaturalQuestions (closed-book) - F1 (Fairness)": "{\"description\": \"min=0.183, mean=0.187, max=0.189, sum=0.56 (3)\", \"tab\": \"Fairness\", \"score\": \"0.18669047090402127\"}", - "NaturalQuestions (open-book) - F1 (Fairness)": "{\"description\": \"min=0.56, mean=0.575, max=0.585, sum=1.724 (3)\", \"tab\": \"Fairness\", \"score\": \"0.5745618824682682\"}", - "NaturalQuestions (closed-book) - Denoised inference time (s)": "{\"description\": \"min=0.931, mean=1.115, max=1.261, sum=3.346 (3)\", \"tab\": \"Efficiency\", \"score\": \"1.115412127906084\"}", - "NaturalQuestions (open-book) - Denoised inference time (s)": "{\"description\": \"min=2.213, mean=2.547, max=2.912, sum=7.64 (3)\", \"tab\": \"Efficiency\", \"score\": \"2.546660231937965\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=92.12, mean=96.12, max=102.12, sum=288.36 (3)\", \"tab\": \"General information\", \"score\": \"96.12\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=34.82, mean=48.109, max=57.074, sum=144.327 (3)\", \"tab\": \"General information\", \"score\": \"48.109\"}", - "NaturalQuestions (closed-book) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.73, mean=4.743, max=4.751, sum=14.229 (3)\", \"tab\": \"General information\", \"score\": \"4.743000000000001\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.035, mean=0.035, max=0.035, sum=0.105 (3)\", \"tab\": \"General information\", \"score\": \"0.035\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1193.69, mean=1313.422, max=1423.457, sum=3940.267 (3)\", \"tab\": \"General information\", \"score\": \"1313.4223333333334\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=31.304, mean=38.803, max=46.481, sum=116.409 (3)\", \"tab\": \"General information\", \"score\": \"38.803000000000004\"}", - "NaturalQuestions (open-book) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NaturalQuestions (closed-book) - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - Stereotypes (gender)": "{\"description\": \"min=0.5, mean=0.5, max=0.5, sum=1 (2)\", \"tab\": \"Bias\", \"score\": \"0.5\"}", - "NaturalQuestions (closed-book) - Representation (race)": "{\"description\": \"min=0.303, mean=0.418, max=0.519, sum=1.254 (3)\", \"tab\": \"Bias\", \"score\": \"0.4180133480204756\"}", - "NaturalQuestions (closed-book) - Representation (gender)": "{\"description\": \"min=0.056, mean=0.09, max=0.143, sum=0.27 (3)\", \"tab\": \"Bias\", \"score\": \"0.08994708994708996\"}", - "NaturalQuestions (open-book) - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=0.667 (1)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666667\"}", - "NaturalQuestions (open-book) - Stereotypes (gender)": "{\"description\": \"min=0.352, mean=0.426, max=0.5, sum=0.852 (2)\", \"tab\": \"Bias\", \"score\": \"0.42619047619047623\"}", - "NaturalQuestions (open-book) - Representation (race)": "{\"description\": \"min=0.454, mean=0.499, max=0.546, sum=1.498 (3)\", \"tab\": \"Bias\", \"score\": \"0.499333679443982\"}", - "NaturalQuestions (open-book) - Representation (gender)": "{\"description\": \"min=0.11, mean=0.135, max=0.177, sum=0.404 (3)\", \"tab\": \"Bias\", \"score\": \"0.13470779383719764\"}", - "NaturalQuestions (closed-book) - Toxic fraction": "{\"description\": \"min=0.001, mean=0.001, max=0.002, sum=0.004 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.0013333333333333333\"}", - "NaturalQuestions (open-book) - Toxic fraction": "{\"description\": \"min=0.001, mean=0.002, max=0.003, sum=0.006 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.002\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "QuAC", - "source_data": { - "dataset_name": "QuAC", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on QuAC", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.361, - "details": { - "description": "min=0.342, mean=0.361, max=0.375, sum=1.082 (3)", - "tab": "Accuracy", - "QuAC - ECE (10-bin)": "{\"description\": \"min=0.103, mean=0.122, max=0.142, sum=0.367 (3)\", \"tab\": \"Calibration\", \"score\": \"0.1222163558834574\"}", - "QuAC - F1 (Robustness)": "{\"description\": \"min=0.229, mean=0.234, max=0.24, sum=0.701 (3)\", \"tab\": \"Robustness\", \"score\": \"0.23376457225319638\"}", - "QuAC - F1 (Fairness)": "{\"description\": \"min=0.265, mean=0.273, max=0.289, sum=0.82 (3)\", \"tab\": \"Fairness\", \"score\": \"0.27335853114408787\"}", - "QuAC - Denoised inference time (s)": "{\"description\": \"min=5.124, mean=5.306, max=5.436, sum=15.919 (3)\", \"tab\": \"Efficiency\", \"score\": \"5.3062709801205585\"}", - "QuAC - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "QuAC - # train": "{\"description\": \"min=0.855, mean=0.944, max=1.07, sum=2.832 (3)\", \"tab\": \"General information\", \"score\": \"0.944\"}", - "QuAC - truncated": "{\"description\": \"min=0.017, mean=0.017, max=0.017, sum=0.051 (3)\", \"tab\": \"General information\", \"score\": \"0.017\"}", - "QuAC - # prompt tokens": "{\"description\": \"min=1614.308, mean=1639.494, max=1673.303, sum=4918.482 (3)\", \"tab\": \"General information\", \"score\": \"1639.494\"}", - "QuAC - # output tokens": "{\"description\": \"min=86.351, mean=90.164, max=93.357, sum=270.491 (3)\", \"tab\": \"General information\", \"score\": \"90.16366666666666\"}", - "QuAC - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "QuAC - Stereotypes (race)": "{\"description\": \"min=0.604, mean=0.631, max=0.647, sum=1.894 (3)\", \"tab\": \"Bias\", \"score\": \"0.6313294548588666\"}", - "QuAC - Stereotypes (gender)": "{\"description\": \"min=0.388, mean=0.396, max=0.408, sum=1.189 (3)\", \"tab\": \"Bias\", \"score\": \"0.3963840842187811\"}", - "QuAC - Representation (race)": "{\"description\": \"min=0.35, mean=0.365, max=0.381, sum=1.094 (3)\", \"tab\": \"Bias\", \"score\": \"0.3645250034421991\"}", - "QuAC - Representation (gender)": "{\"description\": \"min=0.235, mean=0.244, max=0.26, sum=0.732 (3)\", \"tab\": \"Bias\", \"score\": \"0.2440549375970967\"}", - "QuAC - Toxic fraction": "{\"description\": \"min=0, mean=0.001, max=0.002, sum=0.003 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.001\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "HellaSwag", - "source_data": { - "dataset_name": "HellaSwag", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on HellaSwag", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.744, - "details": { - "description": "min=0.744, mean=0.744, max=0.744, sum=0.744 (1)", - "tab": "Accuracy", - "HellaSwag - ECE (10-bin)": "{\"description\": \"min=0.293, mean=0.293, max=0.293, sum=0.293 (1)\", \"tab\": \"Calibration\", \"score\": \"0.2926428762465171\"}", - "HellaSwag - EM (Robustness)": "{\"description\": \"min=0.699, mean=0.699, max=0.699, sum=0.699 (1)\", \"tab\": \"Robustness\", \"score\": \"0.699\"}", - "HellaSwag - EM (Fairness)": "{\"description\": \"min=0.585, mean=0.585, max=0.585, sum=0.585 (1)\", \"tab\": \"Fairness\", \"score\": \"0.585\"}", - "HellaSwag - Denoised inference time (s)": "{\"description\": \"min=0.075, mean=0.075, max=0.075, sum=0.075 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.07493321968615055\"}", - "HellaSwag - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "HellaSwag - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "HellaSwag - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "HellaSwag - # prompt tokens": "{\"description\": \"min=88.875, mean=88.875, max=88.875, sum=88.875 (1)\", \"tab\": \"General information\", \"score\": \"88.875\"}", - "HellaSwag - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "HellaSwag - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.534, - "details": { - "description": "min=0.534, mean=0.534, max=0.534, sum=0.534 (1)", - "tab": "Accuracy", - "OpenbookQA - ECE (10-bin)": "{\"description\": \"min=0.248, mean=0.248, max=0.248, sum=0.248 (1)\", \"tab\": \"Calibration\", \"score\": \"0.24842661648577113\"}", - "OpenbookQA - EM (Robustness)": "{\"description\": \"min=0.438, mean=0.438, max=0.438, sum=0.438 (1)\", \"tab\": \"Robustness\", \"score\": \"0.438\"}", - "OpenbookQA - EM (Fairness)": "{\"description\": \"min=0.482, mean=0.482, max=0.482, sum=0.482 (1)\", \"tab\": \"Fairness\", \"score\": \"0.482\"}", - "OpenbookQA - Denoised inference time (s)": "{\"description\": \"min=0.032, mean=0.032, max=0.032, sum=0.032 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.03224579076468945\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=5.444, mean=5.444, max=5.444, sum=5.444 (1)\", \"tab\": \"General information\", \"score\": \"5.444\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "TruthfulQA", - "source_data": { - "dataset_name": "TruthfulQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on TruthfulQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.205, - "details": { - "description": "min=0.197, mean=0.205, max=0.211, sum=0.82 (4)", - "tab": "Accuracy", - "TruthfulQA - ECE (10-bin)": "{\"description\": \"min=0.053, mean=0.096, max=0.128, sum=0.385 (4)\", \"tab\": \"Calibration\", \"score\": \"0.09624512475777981\"}", - "TruthfulQA - EM (Robustness)": "{\"description\": \"min=0.168, mean=0.183, max=0.206, sum=0.734 (4)\", \"tab\": \"Robustness\", \"score\": \"0.1834862385321101\"}", - "TruthfulQA - EM (Fairness)": "{\"description\": \"min=0.164, mean=0.186, max=0.206, sum=0.745 (4)\", \"tab\": \"Fairness\", \"score\": \"0.18616207951070335\"}", - "TruthfulQA - Denoised inference time (s)": "{\"description\": \"min=0.084, mean=0.143, max=0.226, sum=0.573 (4)\", \"tab\": \"Efficiency\", \"score\": \"0.14325443854568073\"}", - "TruthfulQA - # eval": "{\"description\": \"min=654, mean=654, max=654, sum=2616 (4)\", \"tab\": \"General information\", \"score\": \"654.0\"}", - "TruthfulQA - # train": "{\"description\": \"min=0, mean=3.75, max=5, sum=15 (4)\", \"tab\": \"General information\", \"score\": \"3.75\"}", - "TruthfulQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (4)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "TruthfulQA - # prompt tokens": "{\"description\": \"min=79.361, mean=370.611, max=481.361, sum=1482.443 (4)\", \"tab\": \"General information\", \"score\": \"370.6108562691131\"}", - "TruthfulQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=4 (4)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "TruthfulQA - # trials": "{\"description\": \"min=1, mean=2.5, max=3, sum=10 (4)\", \"tab\": \"General information\", \"score\": \"2.5\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MS MARCO (TREC)", - "source_data": { - "dataset_name": "MS MARCO (TREC)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "NDCG@10 on MS MARCO (TREC)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.386, - "details": { - "description": "min=0.364, mean=0.386, max=0.429, sum=1.158 (3)", - "tab": "Accuracy", - "MS MARCO (regular) - RR@10 (Robustness)": "{\"description\": \"min=0.158, mean=0.19, max=0.218, sum=0.57 (3)\", \"tab\": \"Robustness\", \"score\": \"0.18996269841269822\"}", - "MS MARCO (TREC) - NDCG@10 (Robustness)": "{\"description\": \"min=0.304, mean=0.333, max=0.385, sum=0.998 (3)\", \"tab\": \"Robustness\", \"score\": \"0.33254039819149694\"}", - "MS MARCO (regular) - RR@10 (Fairness)": "{\"description\": \"min=0.189, mean=0.211, max=0.231, sum=0.633 (3)\", \"tab\": \"Fairness\", \"score\": \"0.2110978835978834\"}", - "MS MARCO (TREC) - NDCG@10 (Fairness)": "{\"description\": \"min=0.345, mean=0.371, max=0.418, sum=1.114 (3)\", \"tab\": \"Fairness\", \"score\": \"0.37148573288404924\"}", - "MS MARCO (regular) - Denoised inference time (s)": "{\"description\": \"min=0.246, mean=0.257, max=0.27, sum=0.77 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.25680491607178446\"}", - "MS MARCO (TREC) - Denoised inference time (s)": "{\"description\": \"min=0.227, mean=0.246, max=0.271, sum=0.739 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.24635170979166832\"}", - "MS MARCO (regular) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "MS MARCO (regular) - # train": "{\"description\": \"min=2, mean=2, max=2, sum=6 (3)\", \"tab\": \"General information\", \"score\": \"2.0\"}", - "MS MARCO (regular) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MS MARCO (regular) - # prompt tokens": "{\"description\": \"min=484.472, mean=524.472, max=570.472, sum=1573.416 (3)\", \"tab\": \"General information\", \"score\": \"524.472\"}", - "MS MARCO (regular) - # output tokens": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MS MARCO (regular) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "MS MARCO (TREC) - # eval": "{\"description\": \"min=43, mean=43, max=43, sum=129 (3)\", \"tab\": \"General information\", \"score\": \"43.0\"}", - "MS MARCO (TREC) - # train": "{\"description\": \"min=2, mean=2, max=2, sum=6 (3)\", \"tab\": \"General information\", \"score\": \"2.0\"}", - "MS MARCO (TREC) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MS MARCO (TREC) - # prompt tokens": "{\"description\": \"min=466.814, mean=506.814, max=552.814, sum=1520.442 (3)\", \"tab\": \"General information\", \"score\": \"506.81395348837214\"}", - "MS MARCO (TREC) - # output tokens": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MS MARCO (TREC) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "MS MARCO (regular) - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "MS MARCO (TREC) - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CNN/DailyMail", - "source_data": { - "dataset_name": "CNN/DailyMail", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on CNN/DailyMail", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.08, - "details": { - "description": "min=0.052, mean=0.08, max=0.118, sum=0.478 (6)", - "tab": "Accuracy", - "CNN/DailyMail - Denoised inference time (s)": "{\"description\": \"min=5.515, mean=5.584, max=5.648, sum=33.506 (6)\", \"tab\": \"Efficiency\", \"score\": \"5.5842744588340345\"}", - "CNN/DailyMail - # eval": "{\"description\": \"min=466, mean=466, max=466, sum=2796 (6)\", \"tab\": \"General information\", \"score\": \"466.0\"}", - "CNN/DailyMail - # train": "{\"description\": \"min=5, mean=5, max=5, sum=30 (6)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CNN/DailyMail - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (6)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CNN/DailyMail - # prompt tokens": "{\"description\": \"min=1520.33, mean=1541.33, max=1578.33, sum=9247.983 (6)\", \"tab\": \"General information\", \"score\": \"1541.3304721030042\"}", - "CNN/DailyMail - # output tokens": "{\"description\": \"min=104.867, mean=117.435, max=124.011, sum=704.609 (6)\", \"tab\": \"General information\", \"score\": \"117.4349070100143\"}", - "CNN/DailyMail - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=18 (6)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "CNN/DailyMail - Stereotypes (race)": "{\"description\": \"min=0.641, mean=0.658, max=0.667, sum=3.949 (6)\", \"tab\": \"Bias\", \"score\": \"0.6581699346405229\"}", - "CNN/DailyMail - Stereotypes (gender)": "{\"description\": \"min=0.372, mean=0.385, max=0.405, sum=2.311 (6)\", \"tab\": \"Bias\", \"score\": \"0.3851952735514946\"}", - "CNN/DailyMail - Representation (race)": "{\"description\": \"min=0.291, mean=0.314, max=0.352, sum=1.882 (6)\", \"tab\": \"Bias\", \"score\": \"0.31373280163525924\"}", - "CNN/DailyMail - Representation (gender)": "{\"description\": \"min=0.119, mean=0.145, max=0.16, sum=0.872 (6)\", \"tab\": \"Bias\", \"score\": \"0.14536660393941517\"}", - "CNN/DailyMail - Toxic fraction": "{\"description\": \"min=0, mean=0.001, max=0.002, sum=0.009 (6)\", \"tab\": \"Toxicity\", \"score\": \"0.001430615164520744\"}", - "CNN/DailyMail - SummaC": "{\"description\": \"min=-0.129, mean=-0.02, max=0.115, sum=-0.059 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"-0.01977462275373982\"}", - "CNN/DailyMail - QAFactEval": "{\"description\": \"min=4.63, mean=4.665, max=4.719, sum=27.988 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"4.66471171081461\"}", - "CNN/DailyMail - BERTScore (F1)": "{\"description\": \"min=0.005, mean=0.08, max=0.184, sum=0.24 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.08008308750782954\"}", - "CNN/DailyMail - Coverage": "{\"description\": \"min=0.618, mean=0.71, max=0.826, sum=4.26 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"0.7099913231813372\"}", - "CNN/DailyMail - Density": "{\"description\": \"min=20.964, mean=32.013, max=45.756, sum=192.081 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"32.0134921906249\"}", - "CNN/DailyMail - Compression": "{\"description\": \"min=4.623, mean=5.252, max=6.434, sum=31.514 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"5.2523388558949184\"}", - "CNN/DailyMail - HumanEval-faithfulness": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-relevance": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-coherence": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "XSUM", - "source_data": { - "dataset_name": "XSUM", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on XSUM", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.03, - "details": { - "description": "min=0.022, mean=0.03, max=0.038, sum=0.179 (6)", - "tab": "Accuracy", - "XSUM - Denoised inference time (s)": "{\"description\": \"min=3.874, mean=3.9, max=3.923, sum=23.4 (6)\", \"tab\": \"Efficiency\", \"score\": \"3.899962288877679\"}", - "XSUM - # eval": "{\"description\": \"min=518, mean=518, max=518, sum=3108 (6)\", \"tab\": \"General information\", \"score\": \"518.0\"}", - "XSUM - # train": "{\"description\": \"min=5, mean=5, max=5, sum=30 (6)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "XSUM - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (6)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "XSUM - # prompt tokens": "{\"description\": \"min=1456.338, mean=1501.338, max=1528.338, sum=9008.027 (6)\", \"tab\": \"General information\", \"score\": \"1501.3378378378377\"}", - "XSUM - # output tokens": "{\"description\": \"min=50.606, mean=54.066, max=57.05, sum=324.394 (6)\", \"tab\": \"General information\", \"score\": \"54.06563706563707\"}", - "XSUM - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=18 (6)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "XSUM - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Stereotypes (gender)": "{\"description\": \"min=0.45, mean=0.467, max=0.5, sum=2.802 (6)\", \"tab\": \"Bias\", \"score\": \"0.46699346405228753\"}", - "XSUM - Representation (race)": "{\"description\": \"min=0.238, mean=0.309, max=0.356, sum=1.856 (6)\", \"tab\": \"Bias\", \"score\": \"0.3092501368363437\"}", - "XSUM - Representation (gender)": "{\"description\": \"min=0.109, mean=0.172, max=0.212, sum=1.032 (6)\", \"tab\": \"Bias\", \"score\": \"0.17201180425265794\"}", - "XSUM - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (6)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}", - "XSUM - SummaC": "{\"description\": \"min=-0.365, mean=-0.35, max=-0.335, sum=-1.049 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"-0.3496571157539257\"}", - "XSUM - QAFactEval": "{\"description\": \"min=4.196, mean=4.778, max=5.107, sum=28.667 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"4.77785601273731\"}", - "XSUM - BERTScore (F1)": "{\"description\": \"min=0.025, mean=0.059, max=0.095, sum=0.177 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.05904374779925766\"}", - "XSUM - Coverage": "{\"description\": \"min=0.48, mean=0.515, max=0.553, sum=3.091 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"0.5151319646119767\"}", - "XSUM - Density": "{\"description\": \"min=1.41, mean=1.764, max=2.014, sum=10.585 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"1.764128575895107\"}", - "XSUM - Compression": "{\"description\": \"min=7.741, mean=8.934, max=10.222, sum=53.603 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"8.933804533381347\"}", - "XSUM - HumanEval-faithfulness": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-relevance": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-coherence": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "IMDB", - "source_data": { - "dataset_name": "IMDB", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on IMDB", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.945, - "details": { - "description": "min=0.936, mean=0.945, max=0.95, sum=2.836 (3)", - "tab": "Accuracy", - "IMDB - ECE (10-bin)": "{\"description\": \"min=0.305, mean=0.343, max=0.41, sum=1.029 (3)\", \"tab\": \"Calibration\", \"score\": \"0.3430318396761201\"}", - "IMDB - EM (Robustness)": "{\"description\": \"min=0.907, mean=0.92, max=0.927, sum=2.761 (3)\", \"tab\": \"Robustness\", \"score\": \"0.9203333333333333\"}", - "IMDB - EM (Fairness)": "{\"description\": \"min=0.927, mean=0.938, max=0.946, sum=2.814 (3)\", \"tab\": \"Fairness\", \"score\": \"0.9380000000000001\"}", - "IMDB - Denoised inference time (s)": "{\"description\": \"min=3.425, mean=3.536, max=3.659, sum=10.608 (3)\", \"tab\": \"Efficiency\", \"score\": \"3.5360445948161456\"}", - "IMDB - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "IMDB - # train": "{\"description\": \"min=4.876, mean=4.943, max=4.987, sum=14.83 (3)\", \"tab\": \"General information\", \"score\": \"4.943333333333333\"}", - "IMDB - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IMDB - # prompt tokens": "{\"description\": \"min=1129.265, mean=1375.21, max=1727.698, sum=4125.631 (3)\", \"tab\": \"General information\", \"score\": \"1375.2103333333334\"}", - "IMDB - # output tokens": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "IMDB - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "IMDB - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CivilComments", - "source_data": { - "dataset_name": "CivilComments", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on CivilComments", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.62, - "details": { - "description": "min=0.293, mean=0.62, max=0.92, sum=33.467 (54)", - "tab": "Accuracy", - "CivilComments - ECE (10-bin)": "{\"description\": \"min=0.069, mean=0.262, max=0.456, sum=14.142 (54)\", \"tab\": \"Calibration\", \"score\": \"0.26189371110201226\"}", - "CivilComments - EM (Robustness)": "{\"description\": \"min=0.088, mean=0.467, max=0.827, sum=25.192 (54)\", \"tab\": \"Robustness\", \"score\": \"0.46652660062188434\"}", - "CivilComments - EM (Fairness)": "{\"description\": \"min=0.252, mean=0.546, max=0.91, sum=29.488 (54)\", \"tab\": \"Fairness\", \"score\": \"0.5460670492526992\"}", - "CivilComments - Denoised inference time (s)": "{\"description\": \"min=0.316, mean=0.533, max=1.372, sum=28.76 (54)\", \"tab\": \"Efficiency\", \"score\": \"0.5325854907984409\"}", - "CivilComments - # eval": "{\"description\": \"min=74, mean=371.556, max=683, sum=20064 (54)\", \"tab\": \"General information\", \"score\": \"371.55555555555554\"}", - "CivilComments - # train": "{\"description\": \"min=5, mean=5, max=5, sum=270 (54)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CivilComments - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (54)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CivilComments - # prompt tokens": "{\"description\": \"min=327.671, mean=683.498, max=1208.636, sum=36908.883 (54)\", \"tab\": \"General information\", \"score\": \"683.497824649871\"}", - "CivilComments - # output tokens": "{\"description\": \"min=5, mean=5, max=5, sum=270 (54)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CivilComments - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=162 (54)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "CivilComments - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (54)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "RAFT", - "source_data": { - "dataset_name": "RAFT", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on RAFT", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.592, - "details": { - "description": "min=0.25, mean=0.592, max=0.975, sum=19.525 (33)", - "tab": "Accuracy", - "RAFT - ECE (10-bin)": "{\"description\": \"min=0.246, mean=0.44, max=0.775, sum=14.508 (33)\", \"tab\": \"Calibration\", \"score\": \"0.4396262000869267\"}", - "RAFT - EM (Robustness)": "{\"description\": \"min=0.175, mean=0.527, max=0.95, sum=17.375 (33)\", \"tab\": \"Robustness\", \"score\": \"0.5265151515151515\"}", - "RAFT - EM (Fairness)": "{\"description\": \"min=0.2, mean=0.563, max=0.975, sum=18.575 (33)\", \"tab\": \"Fairness\", \"score\": \"0.5628787878787879\"}", - "RAFT - Denoised inference time (s)": "{\"description\": \"min=0.258, mean=1.866, max=3.777, sum=61.574 (33)\", \"tab\": \"Efficiency\", \"score\": \"1.86588385979184\"}", - "RAFT - # eval": "{\"description\": \"min=40, mean=40, max=40, sum=1320 (33)\", \"tab\": \"General information\", \"score\": \"40.0\"}", - "RAFT - # train": "{\"description\": \"min=0.05, mean=4.567, max=5, sum=150.725 (33)\", \"tab\": \"General information\", \"score\": \"4.567424242424242\"}", - "RAFT - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (33)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "RAFT - # prompt tokens": "{\"description\": \"min=234.025, mean=779.203, max=1729.4, sum=25713.7 (33)\", \"tab\": \"General information\", \"score\": \"779.2030303030305\"}", - "RAFT - # output tokens": "{\"description\": \"min=5, mean=7.127, max=13.7, sum=235.2 (33)\", \"tab\": \"General information\", \"score\": \"7.127272727272727\"}", - "RAFT - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=99 (33)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "RAFT - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (33)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/bigscience_bloom-1b1.json b/data/models/bigscience_bloom-1b1.json deleted file mode 100644 index 2284901eda84ff679d12cf3f9e145d9a85bffc57..0000000000000000000000000000000000000000 --- a/data/models/bigscience_bloom-1b1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "bloom-1b1", - "id": "bigscience/bloom-1b1", - "developer": "bigscience", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "BloomForCausalLM", - "params_billions": "1.065" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bigscience_bloom-1b1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1373 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3107 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0053 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2592 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.37 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1108 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bigscience_bloom-1b7.json b/data/models/bigscience_bloom-1b7.json deleted file mode 100644 index 430a3615c3122731e1ff22e35ece173030816a67..0000000000000000000000000000000000000000 --- a/data/models/bigscience_bloom-1b7.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "bloom-1b7", - "id": "bigscience/bloom-1b7", - "developer": "bigscience", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "BloomForCausalLM", - "params_billions": "1.722" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bigscience_bloom-1b7/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1044 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3141 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0053 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2584 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3886 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1086 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bigscience_bloom-3b.json b/data/models/bigscience_bloom-3b.json deleted file mode 100644 index 26516b50f39043856b4c8c9a0599e3d82b79880b..0000000000000000000000000000000000000000 --- a/data/models/bigscience_bloom-3b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "bloom-3b", - "id": "bigscience/bloom-3b", - "developer": "bigscience", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "BloomForCausalLM", - "params_billions": "3.003" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bigscience_bloom-3b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1271 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3063 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0083 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2399 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3981 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1133 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bigscience_bloom-560m.json b/data/models/bigscience_bloom-560m.json deleted file mode 100644 index 452d371faee3a15a7cb1fe07ff90ab6f58e848b3..0000000000000000000000000000000000000000 --- a/data/models/bigscience_bloom-560m.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "bloom-560m", - "id": "bigscience/bloom-560m", - "developer": "bigscience", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "BloomForCausalLM", - "params_billions": "0.559" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bigscience_bloom-560m/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.062 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3026 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0038 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2617 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4031 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1164 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bigscience_bloom-7b1.json b/data/models/bigscience_bloom-7b1.json deleted file mode 100644 index 70bbbd0dbf60a0dc17c1f6d0cce4b19ed4354717..0000000000000000000000000000000000000000 --- a/data/models/bigscience_bloom-7b1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "bloom-7b1", - "id": "bigscience/bloom-7b1", - "developer": "bigscience", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "BloomForCausalLM", - "params_billions": "7.069" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bigscience_bloom-7b1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1322 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3114 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0053 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2643 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3487 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1105 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bigscience_t0pp-11b.json b/data/models/bigscience_t0pp-11b.json deleted file mode 100644 index 04d66403683af6b221aff6a3950a8871388d8f36..0000000000000000000000000000000000000000 --- a/data/models/bigscience_t0pp-11b.json +++ /dev/null @@ -1,674 +0,0 @@ -{ - "model_info": { - "name": "T0pp 11B", - "id": "bigscience/T0pp-11B", - "developer": "bigscience", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_classic/bigscience_T0pp-11B/1774096308.339228", - "retrieved_timestamp": "1774096308.339228", - "source_metadata": { - "source_name": "helm_classic", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_classic", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_classic", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperform on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.197, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Calibration": "{\"description\": \"\", \"tab\": \"Calibration\", \"score\": \"0.7577474560592045\"}", - "Mean win rate - Robustness": "{\"description\": \"\", \"tab\": \"Robustness\", \"score\": \"0.2275932400932401\"}", - "Mean win rate - Fairness": "{\"description\": \"\", \"tab\": \"Fairness\", \"score\": \"0.20273892773892774\"}", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.42000000000000004\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}", - "Mean win rate - Bias": "{\"description\": \"\", \"tab\": \"Bias\", \"score\": \"0.6045183982683983\"}", - "Mean win rate - Toxicity": "{\"description\": \"\", \"tab\": \"Toxicity\", \"score\": \"0.3965229215229215\"}", - "Mean win rate - Summarization metrics": "{\"description\": \"\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.407, - "details": { - "description": "min=0.25, mean=0.407, max=0.67, sum=6.098 (15)", - "tab": "Accuracy", - "MMLU - ECE (10-bin)": "{\"description\": \"min=0.074, mean=0.168, max=0.3, sum=2.515 (15)\", \"tab\": \"Calibration\", \"score\": \"0.16765379656947835\"}", - "MMLU - EM (Robustness)": "{\"description\": \"min=0.25, mean=0.378, max=0.62, sum=5.675 (15)\", \"tab\": \"Robustness\", \"score\": \"0.37832748538011696\"}", - "MMLU - EM (Fairness)": "{\"description\": \"min=0.25, mean=0.382, max=0.63, sum=5.731 (15)\", \"tab\": \"Fairness\", \"score\": \"0.3820701754385965\"}", - "MMLU - Denoised inference time (s)": "{\"description\": \"min=0.141, mean=0.145, max=0.149, sum=2.18 (15)\", \"tab\": \"Efficiency\", \"score\": \"0.1453571324242486\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=1542 (15)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=75 (15)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (15)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=386.05, mean=492.01, max=639.561, sum=7380.154 (15)\", \"tab\": \"General information\", \"score\": \"492.0102807017544\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=15 (15)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "MMLU - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=45 (15)\", \"tab\": \"General information\", \"score\": \"3.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "BoolQ", - "source_data": { - "dataset_name": "BoolQ", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on BoolQ", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0, - "details": { - "description": "min=0, mean=0, max=0, sum=0 (3)\n☠ T0++ is explicitly trained on these datasets, i.e. data from the same distribution as the test set. See Table 5 on page 24 of https://arxiv.org/pdf/2110.08207.pdf.", - "tab": "Accuracy", - "BoolQ - ECE (10-bin)": "{\"description\": \"min=0.208, mean=0.322, max=0.435, sum=0.967 (3)\\n\\u2620 T0++ is explicitly trained on these datasets, i.e. data from the same distribution as the test set. See Table 5 on page 24 of https://arxiv.org/pdf/2110.08207.pdf.\", \"tab\": \"Calibration\", \"score\": \"0.32218942300251074\"}", - "BoolQ - EM (Robustness)": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\\n\\u2620 T0++ is explicitly trained on these datasets, i.e. data from the same distribution as the test set. See Table 5 on page 24 of https://arxiv.org/pdf/2110.08207.pdf.\", \"tab\": \"Robustness\", \"score\": \"0.0\"}", - "BoolQ - EM (Fairness)": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\\n\\u2620 T0++ is explicitly trained on these datasets, i.e. data from the same distribution as the test set. See Table 5 on page 24 of https://arxiv.org/pdf/2110.08207.pdf.\", \"tab\": \"Fairness\", \"score\": \"0.0\"}", - "BoolQ - Denoised inference time (s)": "{\"description\": \"min=0.366, mean=0.374, max=0.385, sum=1.121 (3)\\n\\u2620 T0++ is explicitly trained on these datasets, i.e. data from the same distribution as the test set. See Table 5 on page 24 of https://arxiv.org/pdf/2110.08207.pdf.\", \"tab\": \"Efficiency\", \"score\": \"0.3736038734018803\"}", - "BoolQ - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\\n\\u2620 T0++ is explicitly trained on these datasets, i.e. data from the same distribution as the test set. See Table 5 on page 24 of https://arxiv.org/pdf/2110.08207.pdf.\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "BoolQ - # train": "{\"description\": \"min=2.027, mean=3.972, max=4.988, sum=11.915 (3)\\n\\u2620 T0++ is explicitly trained on these datasets, i.e. data from the same distribution as the test set. See Table 5 on page 24 of https://arxiv.org/pdf/2110.08207.pdf.\", \"tab\": \"General information\", \"score\": \"3.971666666666667\"}", - "BoolQ - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\\n\\u2620 T0++ is explicitly trained on these datasets, i.e. data from the same distribution as the test set. See Table 5 on page 24 of https://arxiv.org/pdf/2110.08207.pdf.\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "BoolQ - # prompt tokens": "{\"description\": \"min=479.758, mean=702.438, max=905.932, sum=2107.314 (3)\\n\\u2620 T0++ is explicitly trained on these datasets, i.e. data from the same distribution as the test set. See Table 5 on page 24 of https://arxiv.org/pdf/2110.08207.pdf.\", \"tab\": \"General information\", \"score\": \"702.4380000000001\"}", - "BoolQ - # output tokens": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\\n\\u2620 T0++ is explicitly trained on these datasets, i.e. data from the same distribution as the test set. See Table 5 on page 24 of https://arxiv.org/pdf/2110.08207.pdf.\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "BoolQ - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\\n\\u2620 T0++ is explicitly trained on these datasets, i.e. data from the same distribution as the test set. See Table 5 on page 24 of https://arxiv.org/pdf/2110.08207.pdf.\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "BoolQ - Stereotypes (race)": "{\"description\": \"(0)\\n\\u2620 T0++ is explicitly trained on these datasets, i.e. data from the same distribution as the test set. See Table 5 on page 24 of https://arxiv.org/pdf/2110.08207.pdf.\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Stereotypes (gender)": "{\"description\": \"(0)\\n\\u2620 T0++ is explicitly trained on these datasets, i.e. data from the same distribution as the test set. See Table 5 on page 24 of https://arxiv.org/pdf/2110.08207.pdf.\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (race)": "{\"description\": \"(0)\\n\\u2620 T0++ is explicitly trained on these datasets, i.e. data from the same distribution as the test set. See Table 5 on page 24 of https://arxiv.org/pdf/2110.08207.pdf.\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (gender)": "{\"description\": \"min=0, mean=0.25, max=0.5, sum=0.5 (2)\\n\\u2620 T0++ is explicitly trained on these datasets, i.e. data from the same distribution as the test set. See Table 5 on page 24 of https://arxiv.org/pdf/2110.08207.pdf.\", \"tab\": \"Bias\", \"score\": \"0.25\"}", - "BoolQ - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\\n\\u2620 T0++ is explicitly trained on these datasets, i.e. data from the same distribution as the test set. See Table 5 on page 24 of https://arxiv.org/pdf/2110.08207.pdf.\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.151, - "details": { - "description": "min=0.139, mean=0.151, max=0.158, sum=0.454 (3)", - "tab": "Accuracy", - "NarrativeQA - ECE (10-bin)": "{\"description\": \"min=0.0, mean=0.0, max=0.0, sum=0.0 (3)\", \"tab\": \"Calibration\", \"score\": \"4.2543589701120735e-05\"}", - "NarrativeQA - F1 (Robustness)": "{\"description\": \"min=0.087, mean=0.099, max=0.105, sum=0.296 (3)\", \"tab\": \"Robustness\", \"score\": \"0.09874765137769782\"}", - "NarrativeQA - F1 (Fairness)": "{\"description\": \"min=0.074, mean=0.086, max=0.093, sum=0.258 (3)\", \"tab\": \"Fairness\", \"score\": \"0.0858526263629113\"}", - "NarrativeQA - Denoised inference time (s)": "{\"description\": \"min=0.848, mean=0.945, max=1.053, sum=2.834 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.9445703822729286\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=1065 (3)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=0, mean=0.187, max=0.33, sum=0.561 (3)\", \"tab\": \"General information\", \"score\": \"0.18685446009389672\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0.369, mean=0.372, max=0.377, sum=1.115 (3)\", \"tab\": \"General information\", \"score\": \"0.37183098591549296\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=807.577, mean=877.742, max=916.668, sum=2633.225 (3)\", \"tab\": \"General information\", \"score\": \"877.7417840375587\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=100, mean=100, max=100, sum=300 (3)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "NarrativeQA - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NarrativeQA - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=2 (3)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "NarrativeQA - Stereotypes (gender)": "{\"description\": \"min=0.332, mean=0.339, max=0.343, sum=1.017 (3)\", \"tab\": \"Bias\", \"score\": \"0.3389834657156105\"}", - "NarrativeQA - Representation (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=2 (3)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "NarrativeQA - Representation (gender)": "{\"description\": \"min=0.093, mean=0.105, max=0.113, sum=0.314 (3)\", \"tab\": \"Bias\", \"score\": \"0.1046501526237907\"}", - "NarrativeQA - Toxic fraction": "{\"description\": \"min=0.023, mean=0.023, max=0.025, sum=0.07 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.02347417840375587\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (open-book)", - "source_data": { - "dataset_name": "NaturalQuestions (open-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (open-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.19, - "details": { - "description": "min=0.171, mean=0.19, max=0.203, sum=0.569 (3)", - "tab": "Accuracy", - "NaturalQuestions (closed-book) - ECE (10-bin)": "{\"description\": \"min=0.0, mean=0.0, max=0.0, sum=0.0 (3)\", \"tab\": \"Calibration\", \"score\": \"3.521055021161368e-09\"}", - "NaturalQuestions (open-book) - ECE (10-bin)": "{\"description\": \"min=0.0, mean=0.0, max=0.0, sum=0.0 (3)\", \"tab\": \"Calibration\", \"score\": \"9.644610962286308e-05\"}", - "NaturalQuestions (closed-book) - F1 (Robustness)": "{\"description\": \"min=0.03, mean=0.031, max=0.032, sum=0.092 (3)\", \"tab\": \"Robustness\", \"score\": \"0.030683511825215847\"}", - "NaturalQuestions (open-book) - F1 (Robustness)": "{\"description\": \"min=0.101, mean=0.122, max=0.135, sum=0.367 (3)\", \"tab\": \"Robustness\", \"score\": \"0.12220564653363493\"}", - "NaturalQuestions (closed-book) - F1 (Fairness)": "{\"description\": \"min=0.027, mean=0.028, max=0.03, sum=0.084 (3)\", \"tab\": \"Fairness\", \"score\": \"0.028132918197666456\"}", - "NaturalQuestions (open-book) - F1 (Fairness)": "{\"description\": \"min=0.119, mean=0.136, max=0.151, sum=0.407 (3)\", \"tab\": \"Fairness\", \"score\": \"0.13562055302845238\"}", - "NaturalQuestions (closed-book) - Denoised inference time (s)": "{\"description\": \"min=1.309, mean=1.457, max=1.621, sum=4.371 (3)\", \"tab\": \"Efficiency\", \"score\": \"1.4571279249547553\"}", - "NaturalQuestions (open-book) - Denoised inference time (s)": "{\"description\": \"min=2.864, mean=2.895, max=2.953, sum=8.685 (3)\", \"tab\": \"Efficiency\", \"score\": \"2.8950855693236632\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=109.556, mean=113.556, max=118.556, sum=340.668 (3)\", \"tab\": \"General information\", \"score\": \"113.556\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=300, mean=300, max=300, sum=900 (3)\", \"tab\": \"General information\", \"score\": \"300.0\"}", - "NaturalQuestions (closed-book) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=3.164, mean=3.396, max=3.709, sum=10.189 (3)\", \"tab\": \"General information\", \"score\": \"3.396333333333333\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.052, mean=0.057, max=0.066, sum=0.172 (3)\", \"tab\": \"General information\", \"score\": \"0.057333333333333326\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=850.863, mean=903.877, max=958.904, sum=2711.631 (3)\", \"tab\": \"General information\", \"score\": \"903.8770000000001\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=300, mean=300, max=300, sum=900 (3)\", \"tab\": \"General information\", \"score\": \"300.0\"}", - "NaturalQuestions (open-book) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NaturalQuestions (closed-book) - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=0.667 (1)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666667\"}", - "NaturalQuestions (closed-book) - Stereotypes (gender)": "{\"description\": \"min=0.385, mean=0.462, max=0.5, sum=1.385 (3)\", \"tab\": \"Bias\", \"score\": \"0.46155024509803927\"}", - "NaturalQuestions (closed-book) - Representation (race)": "{\"description\": \"min=0.552, mean=0.613, max=0.657, sum=1.84 (3)\", \"tab\": \"Bias\", \"score\": \"0.6131917464492584\"}", - "NaturalQuestions (closed-book) - Representation (gender)": "{\"description\": \"min=0.028, mean=0.177, max=0.252, sum=0.53 (3)\", \"tab\": \"Bias\", \"score\": \"0.17673498741459906\"}", - "NaturalQuestions (open-book) - Stereotypes (race)": "{\"description\": \"min=0.209, mean=0.329, max=0.473, sum=0.987 (3)\", \"tab\": \"Bias\", \"score\": \"0.32890264223378113\"}", - "NaturalQuestions (open-book) - Stereotypes (gender)": "{\"description\": \"min=0.289, mean=0.388, max=0.456, sum=1.164 (3)\", \"tab\": \"Bias\", \"score\": \"0.38814814814814813\"}", - "NaturalQuestions (open-book) - Representation (race)": "{\"description\": \"min=0.394, mean=0.462, max=0.563, sum=1.386 (3)\", \"tab\": \"Bias\", \"score\": \"0.4620750643944221\"}", - "NaturalQuestions (open-book) - Representation (gender)": "{\"description\": \"min=0.044, mean=0.091, max=0.176, sum=0.273 (3)\", \"tab\": \"Bias\", \"score\": \"0.09087407629591253\"}", - "NaturalQuestions (closed-book) - Toxic fraction": "{\"description\": \"min=0.001, mean=0.001, max=0.002, sum=0.004 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.0013333333333333333\"}", - "NaturalQuestions (open-book) - Toxic fraction": "{\"description\": \"min=0, mean=0.0, max=0.001, sum=0.001 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.0003333333333333333\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "QuAC", - "source_data": { - "dataset_name": "QuAC", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on QuAC", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.121, - "details": { - "description": "min=0.121, mean=0.121, max=0.121, sum=0.362 (3)", - "tab": "Accuracy", - "QuAC - ECE (10-bin)": "{\"description\": \"min=0.001, mean=0.001, max=0.001, sum=0.002 (3)\", \"tab\": \"Calibration\", \"score\": \"0.0005015010499976317\"}", - "QuAC - F1 (Robustness)": "{\"description\": \"min=0.071, mean=0.071, max=0.071, sum=0.212 (3)\", \"tab\": \"Robustness\", \"score\": \"0.07065126152546952\"}", - "QuAC - F1 (Fairness)": "{\"description\": \"min=0.067, mean=0.067, max=0.067, sum=0.201 (3)\", \"tab\": \"Fairness\", \"score\": \"0.06691720655918869\"}", - "QuAC - Denoised inference time (s)": "{\"description\": \"min=1.239, mean=1.239, max=1.239, sum=3.716 (3)\", \"tab\": \"Efficiency\", \"score\": \"1.2385025575706792\"}", - "QuAC - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "QuAC - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "QuAC - truncated": "{\"description\": \"min=0.985, mean=0.985, max=0.985, sum=2.955 (3)\", \"tab\": \"General information\", \"score\": \"0.985\"}", - "QuAC - # prompt tokens": "{\"description\": \"min=823.365, mean=823.365, max=823.365, sum=2470.095 (3)\", \"tab\": \"General information\", \"score\": \"823.3650000000001\"}", - "QuAC - # output tokens": "{\"description\": \"min=100, mean=100, max=100, sum=300 (3)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "QuAC - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "QuAC - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=2 (3)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "QuAC - Stereotypes (gender)": "{\"description\": \"min=0.428, mean=0.428, max=0.428, sum=1.284 (3)\", \"tab\": \"Bias\", \"score\": \"0.42797040922040913\"}", - "QuAC - Representation (race)": "{\"description\": \"min=0.436, mean=0.436, max=0.436, sum=1.308 (3)\", \"tab\": \"Bias\", \"score\": \"0.4358974358974359\"}", - "QuAC - Representation (gender)": "{\"description\": \"min=0.291, mean=0.291, max=0.291, sum=0.872 (3)\", \"tab\": \"Bias\", \"score\": \"0.2905073649754501\"}", - "QuAC - Toxic fraction": "{\"description\": \"min=0.001, mean=0.001, max=0.001, sum=0.003 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.001\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "HellaSwag", - "source_data": { - "dataset_name": "HellaSwag", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on HellaSwag", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "HellaSwag - ECE (10-bin)": "{\"description\": \"No matching runs\", \"tab\": \"Calibration\", \"score\": \"\"}", - "HellaSwag - EM (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "HellaSwag - EM (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "HellaSwag - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "HellaSwag - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "OpenbookQA - ECE (10-bin)": "{\"description\": \"No matching runs\", \"tab\": \"Calibration\", \"score\": \"\"}", - "OpenbookQA - EM (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "OpenbookQA - EM (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "OpenbookQA - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "OpenbookQA - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "TruthfulQA", - "source_data": { - "dataset_name": "TruthfulQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on TruthfulQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.377, - "details": { - "description": "min=0.347, mean=0.377, max=0.411, sum=1.508 (4)", - "tab": "Accuracy", - "TruthfulQA - ECE (10-bin)": "{\"description\": \"min=0.1, mean=0.154, max=0.234, sum=0.617 (4)\", \"tab\": \"Calibration\", \"score\": \"0.15413479575183991\"}", - "TruthfulQA - EM (Robustness)": "{\"description\": \"min=0.33, mean=0.365, max=0.411, sum=1.46 (4)\", \"tab\": \"Robustness\", \"score\": \"0.3650611620795107\"}", - "TruthfulQA - EM (Fairness)": "{\"description\": \"min=0.307, mean=0.35, max=0.411, sum=1.399 (4)\", \"tab\": \"Fairness\", \"score\": \"0.34977064220183485\"}", - "TruthfulQA - Denoised inference time (s)": "{\"description\": \"min=0.133, mean=0.142, max=0.145, sum=0.567 (4)\", \"tab\": \"Efficiency\", \"score\": \"0.14173421436146078\"}", - "TruthfulQA - # eval": "{\"description\": \"min=654, mean=654, max=654, sum=2616 (4)\", \"tab\": \"General information\", \"score\": \"654.0\"}", - "TruthfulQA - # train": "{\"description\": \"min=0, mean=3.75, max=5, sum=15 (4)\", \"tab\": \"General information\", \"score\": \"3.75\"}", - "TruthfulQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (4)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "TruthfulQA - # prompt tokens": "{\"description\": \"min=85.896, mean=391.646, max=515.896, sum=1566.584 (4)\", \"tab\": \"General information\", \"score\": \"391.6460244648318\"}", - "TruthfulQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=4 (4)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "TruthfulQA - # trials": "{\"description\": \"min=1, mean=2.5, max=3, sum=10 (4)\", \"tab\": \"General information\", \"score\": \"2.5\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MS MARCO (TREC)", - "source_data": { - "dataset_name": "MS MARCO (TREC)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "NDCG@10 on MS MARCO (TREC)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "MS MARCO (regular) - RR@10 (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "MS MARCO (TREC) - NDCG@10 (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "MS MARCO (regular) - RR@10 (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "MS MARCO (TREC) - NDCG@10 (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "MS MARCO (regular) - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MS MARCO (TREC) - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MS MARCO (regular) - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "MS MARCO (TREC) - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CNN/DailyMail", - "source_data": { - "dataset_name": "CNN/DailyMail", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on CNN/DailyMail", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.122, - "details": { - "description": "min=0.121, mean=0.122, max=0.122, sum=0.73 (6)\n☠ T0++ is explicitly trained on these datasets, i.e. data from the same distribution as the test set. See Table 5 on page 24 of https://arxiv.org/pdf/2110.08207.pdf.", - "tab": "Accuracy", - "CNN/DailyMail - Denoised inference time (s)": "{\"description\": \"min=1.057, mean=1.066, max=1.081, sum=6.393 (6)\\n\\u2620 T0++ is explicitly trained on these datasets, i.e. data from the same distribution as the test set. See Table 5 on page 24 of https://arxiv.org/pdf/2110.08207.pdf.\", \"tab\": \"Efficiency\", \"score\": \"1.0655231237061773\"}", - "CNN/DailyMail - # eval": "{\"description\": \"min=466, mean=466, max=466, sum=2796 (6)\\n\\u2620 T0++ is explicitly trained on these datasets, i.e. data from the same distribution as the test set. See Table 5 on page 24 of https://arxiv.org/pdf/2110.08207.pdf.\", \"tab\": \"General information\", \"score\": \"466.0\"}", - "CNN/DailyMail - # train": "{\"description\": \"min=1.303, mean=1.335, max=1.378, sum=8.013 (6)\\n\\u2620 T0++ is explicitly trained on these datasets, i.e. data from the same distribution as the test set. See Table 5 on page 24 of https://arxiv.org/pdf/2110.08207.pdf.\", \"tab\": \"General information\", \"score\": \"1.3354792560801145\"}", - "CNN/DailyMail - truncated": "{\"description\": \"min=0.004, mean=0.004, max=0.004, sum=0.026 (6)\\n\\u2620 T0++ is explicitly trained on these datasets, i.e. data from the same distribution as the test set. See Table 5 on page 24 of https://arxiv.org/pdf/2110.08207.pdf.\", \"tab\": \"General information\", \"score\": \"0.004291845493562232\"}", - "CNN/DailyMail - # prompt tokens": "{\"description\": \"min=885.292, mean=886.838, max=888.921, sum=5321.026 (6)\\n\\u2620 T0++ is explicitly trained on these datasets, i.e. data from the same distribution as the test set. See Table 5 on page 24 of https://arxiv.org/pdf/2110.08207.pdf.\", \"tab\": \"General information\", \"score\": \"886.8376251788268\"}", - "CNN/DailyMail - # output tokens": "{\"description\": \"min=128, mean=128, max=128, sum=768 (6)\\n\\u2620 T0++ is explicitly trained on these datasets, i.e. data from the same distribution as the test set. See Table 5 on page 24 of https://arxiv.org/pdf/2110.08207.pdf.\", \"tab\": \"General information\", \"score\": \"128.0\"}", - "CNN/DailyMail - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=18 (6)\\n\\u2620 T0++ is explicitly trained on these datasets, i.e. data from the same distribution as the test set. See Table 5 on page 24 of https://arxiv.org/pdf/2110.08207.pdf.\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "CNN/DailyMail - Stereotypes (race)": "{\"description\": \"min=0.562, mean=0.594, max=0.631, sum=3.562 (6)\\n\\u2620 T0++ is explicitly trained on these datasets, i.e. data from the same distribution as the test set. See Table 5 on page 24 of https://arxiv.org/pdf/2110.08207.pdf.\", \"tab\": \"Bias\", \"score\": \"0.5936999598322023\"}", - "CNN/DailyMail - Stereotypes (gender)": "{\"description\": \"min=0.391, mean=0.403, max=0.421, sum=2.417 (6)\\n\\u2620 T0++ is explicitly trained on these datasets, i.e. data from the same distribution as the test set. See Table 5 on page 24 of https://arxiv.org/pdf/2110.08207.pdf.\", \"tab\": \"Bias\", \"score\": \"0.4028700462262689\"}", - "CNN/DailyMail - Representation (race)": "{\"description\": \"min=0.27, mean=0.277, max=0.282, sum=1.662 (6)\\n\\u2620 T0++ is explicitly trained on these datasets, i.e. data from the same distribution as the test set. See Table 5 on page 24 of https://arxiv.org/pdf/2110.08207.pdf.\", \"tab\": \"Bias\", \"score\": \"0.2769263317991031\"}", - "CNN/DailyMail - Representation (gender)": "{\"description\": \"min=0.047, mean=0.093, max=0.138, sum=0.559 (6)\\n\\u2620 T0++ is explicitly trained on these datasets, i.e. data from the same distribution as the test set. See Table 5 on page 24 of https://arxiv.org/pdf/2110.08207.pdf.\", \"tab\": \"Bias\", \"score\": \"0.09311410441258088\"}", - "CNN/DailyMail - Toxic fraction": "{\"description\": \"min=0, mean=0.001, max=0.002, sum=0.009 (6)\\n\\u2620 T0++ is explicitly trained on these datasets, i.e. data from the same distribution as the test set. See Table 5 on page 24 of https://arxiv.org/pdf/2110.08207.pdf.\", \"tab\": \"Toxicity\", \"score\": \"0.001430615164520744\"}", - "CNN/DailyMail - SummaC": "{\"description\": \"min=-0.052, mean=-0.044, max=-0.031, sum=-0.132 (3)\\n\\u2620 T0++ is explicitly trained on these datasets, i.e. data from the same distribution as the test set. See Table 5 on page 24 of https://arxiv.org/pdf/2110.08207.pdf.\", \"tab\": \"Summarization metrics\", \"score\": \"-0.04384894228805586\"}", - "CNN/DailyMail - QAFactEval": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - BERTScore (F1)": "{\"description\": \"min=0.151, mean=0.155, max=0.163, sum=0.465 (3)\\n\\u2620 T0++ is explicitly trained on these datasets, i.e. data from the same distribution as the test set. See Table 5 on page 24 of https://arxiv.org/pdf/2110.08207.pdf.\", \"tab\": \"Summarization metrics\", \"score\": \"0.1550916195946839\"}", - "CNN/DailyMail - Coverage": "{\"description\": \"min=0.836, mean=0.841, max=0.845, sum=5.047 (6)\\n\\u2620 T0++ is explicitly trained on these datasets, i.e. data from the same distribution as the test set. See Table 5 on page 24 of https://arxiv.org/pdf/2110.08207.pdf.\", \"tab\": \"Summarization metrics\", \"score\": \"0.841192270385719\"}", - "CNN/DailyMail - Density": "{\"description\": \"min=8.147, mean=8.588, max=8.816, sum=51.53 (6)\\n\\u2620 T0++ is explicitly trained on these datasets, i.e. data from the same distribution as the test set. See Table 5 on page 24 of https://arxiv.org/pdf/2110.08207.pdf.\", \"tab\": \"Summarization metrics\", \"score\": \"8.588383920302716\"}", - "CNN/DailyMail - Compression": "{\"description\": \"min=8.169, mean=8.274, max=8.416, sum=49.643 (6)\\n\\u2620 T0++ is explicitly trained on these datasets, i.e. data from the same distribution as the test set. See Table 5 on page 24 of https://arxiv.org/pdf/2110.08207.pdf.\", \"tab\": \"Summarization metrics\", \"score\": \"8.27387938295926\"}", - "CNN/DailyMail - HumanEval-faithfulness": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-relevance": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-coherence": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "XSUM", - "source_data": { - "dataset_name": "XSUM", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on XSUM", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.09, - "details": { - "description": "min=0.07, mean=0.09, max=0.103, sum=0.539 (6)\n☠ T0++ is explicitly trained on these datasets, i.e. data from the same distribution as the test set. See Table 5 on page 24 of https://arxiv.org/pdf/2110.08207.pdf.", - "tab": "Accuracy", - "XSUM - Denoised inference time (s)": "{\"description\": \"min=0.523, mean=0.554, max=0.571, sum=3.326 (6)\\n\\u2620 T0++ is explicitly trained on these datasets, i.e. data from the same distribution as the test set. See Table 5 on page 24 of https://arxiv.org/pdf/2110.08207.pdf.\", \"tab\": \"Efficiency\", \"score\": \"0.5543883131537052\"}", - "XSUM - # eval": "{\"description\": \"min=518, mean=518, max=518, sum=3108 (6)\\n\\u2620 T0++ is explicitly trained on these datasets, i.e. data from the same distribution as the test set. See Table 5 on page 24 of https://arxiv.org/pdf/2110.08207.pdf.\", \"tab\": \"General information\", \"score\": \"518.0\"}", - "XSUM - # train": "{\"description\": \"min=1.967, mean=2.068, max=2.214, sum=12.405 (6)\\n\\u2620 T0++ is explicitly trained on these datasets, i.e. data from the same distribution as the test set. See Table 5 on page 24 of https://arxiv.org/pdf/2110.08207.pdf.\", \"tab\": \"General information\", \"score\": \"2.0675675675675675\"}", - "XSUM - truncated": "{\"description\": \"min=0.002, mean=0.01, max=0.019, sum=0.058 (6)\\n\\u2620 T0++ is explicitly trained on these datasets, i.e. data from the same distribution as the test set. See Table 5 on page 24 of https://arxiv.org/pdf/2110.08207.pdf.\", \"tab\": \"General information\", \"score\": \"0.009652509652509652\"}", - "XSUM - # prompt tokens": "{\"description\": \"min=889.981, mean=907.769, max=929.006, sum=5446.614 (6)\\n\\u2620 T0++ is explicitly trained on these datasets, i.e. data from the same distribution as the test set. See Table 5 on page 24 of https://arxiv.org/pdf/2110.08207.pdf.\", \"tab\": \"General information\", \"score\": \"907.7689832689833\"}", - "XSUM - # output tokens": "{\"description\": \"min=64, mean=64, max=64, sum=384 (6)\\n\\u2620 T0++ is explicitly trained on these datasets, i.e. data from the same distribution as the test set. See Table 5 on page 24 of https://arxiv.org/pdf/2110.08207.pdf.\", \"tab\": \"General information\", \"score\": \"64.0\"}", - "XSUM - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=18 (6)\\n\\u2620 T0++ is explicitly trained on these datasets, i.e. data from the same distribution as the test set. See Table 5 on page 24 of https://arxiv.org/pdf/2110.08207.pdf.\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "XSUM - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=4 (6)\\n\\u2620 T0++ is explicitly trained on these datasets, i.e. data from the same distribution as the test set. See Table 5 on page 24 of https://arxiv.org/pdf/2110.08207.pdf.\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "XSUM - Stereotypes (gender)": "{\"description\": \"min=0.43, mean=0.444, max=0.463, sum=2.663 (6)\\n\\u2620 T0++ is explicitly trained on these datasets, i.e. data from the same distribution as the test set. See Table 5 on page 24 of https://arxiv.org/pdf/2110.08207.pdf.\", \"tab\": \"Bias\", \"score\": \"0.4438297255067441\"}", - "XSUM - Representation (race)": "{\"description\": \"min=0.286, mean=0.457, max=0.617, sum=2.74 (6)\\n\\u2620 T0++ is explicitly trained on these datasets, i.e. data from the same distribution as the test set. See Table 5 on page 24 of https://arxiv.org/pdf/2110.08207.pdf.\", \"tab\": \"Bias\", \"score\": \"0.45673778645470176\"}", - "XSUM - Representation (gender)": "{\"description\": \"min=0.215, mean=0.27, max=0.328, sum=1.62 (6)\\n\\u2620 T0++ is explicitly trained on these datasets, i.e. data from the same distribution as the test set. See Table 5 on page 24 of https://arxiv.org/pdf/2110.08207.pdf.\", \"tab\": \"Bias\", \"score\": \"0.2699471127776433\"}", - "XSUM - Toxic fraction": "{\"description\": \"min=0, mean=0.001, max=0.002, sum=0.004 (6)\\n\\u2620 T0++ is explicitly trained on these datasets, i.e. data from the same distribution as the test set. See Table 5 on page 24 of https://arxiv.org/pdf/2110.08207.pdf.\", \"tab\": \"Toxicity\", \"score\": \"0.0006435006435006435\"}", - "XSUM - SummaC": "{\"description\": \"min=-0.331, mean=-0.3, max=-0.268, sum=-0.901 (3)\\n\\u2620 T0++ is explicitly trained on these datasets, i.e. data from the same distribution as the test set. See Table 5 on page 24 of https://arxiv.org/pdf/2110.08207.pdf.\", \"tab\": \"Summarization metrics\", \"score\": \"-0.3004745337800477\"}", - "XSUM - QAFactEval": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - BERTScore (F1)": "{\"description\": \"min=0.083, mean=0.097, max=0.111, sum=0.292 (3)\\n\\u2620 T0++ is explicitly trained on these datasets, i.e. data from the same distribution as the test set. See Table 5 on page 24 of https://arxiv.org/pdf/2110.08207.pdf.\", \"tab\": \"Summarization metrics\", \"score\": \"0.09723521885401472\"}", - "XSUM - Coverage": "{\"description\": \"min=0.543, mean=0.579, max=0.605, sum=3.474 (6)\\n\\u2620 T0++ is explicitly trained on these datasets, i.e. data from the same distribution as the test set. See Table 5 on page 24 of https://arxiv.org/pdf/2110.08207.pdf.\", \"tab\": \"Summarization metrics\", \"score\": \"0.5789418979978066\"}", - "XSUM - Density": "{\"description\": \"min=1.492, mean=1.684, max=1.861, sum=10.105 (6)\\n\\u2620 T0++ is explicitly trained on these datasets, i.e. data from the same distribution as the test set. See Table 5 on page 24 of https://arxiv.org/pdf/2110.08207.pdf.\", \"tab\": \"Summarization metrics\", \"score\": \"1.6841663389066148\"}", - "XSUM - Compression": "{\"description\": \"min=10.341, mean=11.178, max=11.672, sum=67.065 (6)\\n\\u2620 T0++ is explicitly trained on these datasets, i.e. data from the same distribution as the test set. See Table 5 on page 24 of https://arxiv.org/pdf/2110.08207.pdf.\", \"tab\": \"Summarization metrics\", \"score\": \"11.17756803869132\"}", - "XSUM - HumanEval-faithfulness": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-relevance": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-coherence": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "IMDB", - "source_data": { - "dataset_name": "IMDB", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on IMDB", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.207, - "details": { - "description": "min=0.181, mean=0.207, max=0.26, sum=0.622 (3)\n☠ T0++ is explicitly trained on these datasets, i.e. data from the same distribution as the test set. See Table 5 on page 24 of https://arxiv.org/pdf/2110.08207.pdf.", - "tab": "Accuracy", - "IMDB - ECE (10-bin)": "{\"description\": \"min=0.207, mean=0.291, max=0.36, sum=0.872 (3)\\n\\u2620 T0++ is explicitly trained on these datasets, i.e. data from the same distribution as the test set. See Table 5 on page 24 of https://arxiv.org/pdf/2110.08207.pdf.\", \"tab\": \"Calibration\", \"score\": \"0.29061500207311436\"}", - "IMDB - EM (Robustness)": "{\"description\": \"min=0.13, mean=0.17, max=0.227, sum=0.511 (3)\\n\\u2620 T0++ is explicitly trained on these datasets, i.e. data from the same distribution as the test set. See Table 5 on page 24 of https://arxiv.org/pdf/2110.08207.pdf.\", \"tab\": \"Robustness\", \"score\": \"0.17033333333333334\"}", - "IMDB - EM (Fairness)": "{\"description\": \"min=0.129, mean=0.168, max=0.22, sum=0.505 (3)\\n\\u2620 T0++ is explicitly trained on these datasets, i.e. data from the same distribution as the test set. See Table 5 on page 24 of https://arxiv.org/pdf/2110.08207.pdf.\", \"tab\": \"Fairness\", \"score\": \"0.16833333333333333\"}", - "IMDB - Denoised inference time (s)": "{\"description\": \"min=0.37, mean=0.393, max=0.436, sum=1.18 (3)\\n\\u2620 T0++ is explicitly trained on these datasets, i.e. data from the same distribution as the test set. See Table 5 on page 24 of https://arxiv.org/pdf/2110.08207.pdf.\", \"tab\": \"Efficiency\", \"score\": \"0.39343433208828427\"}", - "IMDB - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\\n\\u2620 T0++ is explicitly trained on these datasets, i.e. data from the same distribution as the test set. See Table 5 on page 24 of https://arxiv.org/pdf/2110.08207.pdf.\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "IMDB - # train": "{\"description\": \"min=1.981, mean=2.44, max=3.074, sum=7.321 (3)\\n\\u2620 T0++ is explicitly trained on these datasets, i.e. data from the same distribution as the test set. See Table 5 on page 24 of https://arxiv.org/pdf/2110.08207.pdf.\", \"tab\": \"General information\", \"score\": \"2.4403333333333332\"}", - "IMDB - truncated": "{\"description\": \"min=0.03, mean=0.03, max=0.03, sum=0.09 (3)\\n\\u2620 T0++ is explicitly trained on these datasets, i.e. data from the same distribution as the test set. See Table 5 on page 24 of https://arxiv.org/pdf/2110.08207.pdf.\", \"tab\": \"General information\", \"score\": \"0.03\"}", - "IMDB - # prompt tokens": "{\"description\": \"min=905.879, mean=910.174, max=913.752, sum=2730.521 (3)\\n\\u2620 T0++ is explicitly trained on these datasets, i.e. data from the same distribution as the test set. See Table 5 on page 24 of https://arxiv.org/pdf/2110.08207.pdf.\", \"tab\": \"General information\", \"score\": \"910.1736666666666\"}", - "IMDB - # output tokens": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\\n\\u2620 T0++ is explicitly trained on these datasets, i.e. data from the same distribution as the test set. See Table 5 on page 24 of https://arxiv.org/pdf/2110.08207.pdf.\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "IMDB - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\\n\\u2620 T0++ is explicitly trained on these datasets, i.e. data from the same distribution as the test set. See Table 5 on page 24 of https://arxiv.org/pdf/2110.08207.pdf.\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "IMDB - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CivilComments", - "source_data": { - "dataset_name": "CivilComments", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on CivilComments", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.234, - "details": { - "description": "min=0, mean=0.234, max=0.985, sum=12.634 (54)", - "tab": "Accuracy", - "CivilComments - ECE (10-bin)": "{\"description\": \"min=0.067, mean=0.308, max=0.574, sum=16.631 (54)\", \"tab\": \"Calibration\", \"score\": \"0.30797595023001567\"}", - "CivilComments - EM (Robustness)": "{\"description\": \"min=0, mean=0.087, max=0.824, sum=4.704 (54)\", \"tab\": \"Robustness\", \"score\": \"0.0871064519307774\"}", - "CivilComments - EM (Fairness)": "{\"description\": \"min=0, mean=0.165, max=0.947, sum=8.894 (54)\", \"tab\": \"Fairness\", \"score\": \"0.16470832145418626\"}", - "CivilComments - Denoised inference time (s)": "{\"description\": \"min=0.328, mean=0.391, max=0.487, sum=21.126 (54)\", \"tab\": \"Efficiency\", \"score\": \"0.3912135341654548\"}", - "CivilComments - # eval": "{\"description\": \"min=74, mean=371.556, max=683, sum=20064 (54)\", \"tab\": \"General information\", \"score\": \"371.55555555555554\"}", - "CivilComments - # train": "{\"description\": \"min=2.991, mean=4.861, max=5, sum=262.497 (54)\", \"tab\": \"General information\", \"score\": \"4.861055391438897\"}", - "CivilComments - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (54)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CivilComments - # prompt tokens": "{\"description\": \"min=385.732, mean=744.109, max=936.562, sum=40181.894 (54)\", \"tab\": \"General information\", \"score\": \"744.1091399163704\"}", - "CivilComments - # output tokens": "{\"description\": \"min=5, mean=5, max=5, sum=270 (54)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CivilComments - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=162 (54)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "CivilComments - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (gender)": "{\"description\": \"min=0.3, mean=0.459, max=0.5, sum=5.503 (12)\", \"tab\": \"Bias\", \"score\": \"0.4585978835978836\"}", - "CivilComments - Toxic fraction": "{\"description\": \"min=0, mean=0.0, max=0.008, sum=0.025 (54)\", \"tab\": \"Toxicity\", \"score\": \"0.0004596436870303355\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "RAFT", - "source_data": { - "dataset_name": "RAFT", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on RAFT", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.118, - "details": { - "description": "min=0, mean=0.118, max=0.775, sum=3.9 (33)", - "tab": "Accuracy", - "RAFT - ECE (10-bin)": "{\"description\": \"min=0.0, mean=0.086, max=0.573, sum=2.84 (33)\", \"tab\": \"Calibration\", \"score\": \"0.08607203532710274\"}", - "RAFT - EM (Robustness)": "{\"description\": \"min=0, mean=0.085, max=0.775, sum=2.8 (33)\", \"tab\": \"Robustness\", \"score\": \"0.08484848484848484\"}", - "RAFT - EM (Fairness)": "{\"description\": \"min=0, mean=0.106, max=0.75, sum=3.5 (33)\", \"tab\": \"Fairness\", \"score\": \"0.10606060606060606\"}", - "RAFT - Denoised inference time (s)": "{\"description\": \"min=0.329, mean=0.586, max=0.74, sum=19.352 (33)\", \"tab\": \"Efficiency\", \"score\": \"0.586429068475456\"}", - "RAFT - # eval": "{\"description\": \"min=40, mean=40, max=40, sum=1320 (33)\", \"tab\": \"General information\", \"score\": \"40.0\"}", - "RAFT - # train": "{\"description\": \"min=0, mean=3.913, max=5, sum=129.125 (33)\", \"tab\": \"General information\", \"score\": \"3.912878787878788\"}", - "RAFT - truncated": "{\"description\": \"min=0, mean=0.09, max=0.925, sum=2.975 (33)\", \"tab\": \"General information\", \"score\": \"0.09015151515151516\"}", - "RAFT - # prompt tokens": "{\"description\": \"min=263.4, mean=650.012, max=949.7, sum=21450.4 (33)\", \"tab\": \"General information\", \"score\": \"650.0121212121212\"}", - "RAFT - # output tokens": "{\"description\": \"min=30, mean=30, max=30, sum=990 (33)\", \"tab\": \"General information\", \"score\": \"30.0\"}", - "RAFT - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=99 (33)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "RAFT - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Stereotypes (gender)": "{\"description\": \"min=0.5, mean=0.5, max=0.5, sum=1.5 (3)\", \"tab\": \"Bias\", \"score\": \"0.5\"}", - "RAFT - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (gender)": "{\"description\": \"min=0.125, mean=0.125, max=0.125, sum=0.375 (3)\", \"tab\": \"Bias\", \"score\": \"0.12500000000000003\"}", - "RAFT - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (33)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/bittensor_bitagent-bounty-8b.json b/data/models/bittensor_bitagent-bounty-8b.json deleted file mode 100644 index cf138d1ab839e40d4270622d92b4b3da88b03f8f..0000000000000000000000000000000000000000 --- a/data/models/bittensor_bitagent-bounty-8b.json +++ /dev/null @@ -1,850 +0,0 @@ -{ - "model_info": { - "name": "BitAgent-Bounty-8B", - "id": "bittensor/bitagent-bounty-8b", - "developer": "bittensor", - "additional_details": { - "raw_model_name": "BitAgent-Bounty-8B", - "organization": "Bittensor", - "license": "Apache-2.0", - "model_link": "https://huggingface.co/BitAgent/BitAgent-Bounty-8B" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/bittensor/bitagent-bounty-8b/1775236112.383741", - "retrieved_timestamp": "1775236112.383741", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 36.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 46.23 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 18.02 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 16.52 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 30.73 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 77.12 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 81.6 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 72.42 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 93.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 83.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 78.0 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 93.12 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 90.31 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 94.02 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 75.0 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 95.83 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 62.38 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 75.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 49.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 68.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 57.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 1.51 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 1.29 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 1.29 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 1.94 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 68.75 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 97.48 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/blackbeenie_bloslain-8b-v0.2.json b/data/models/blackbeenie_bloslain-8b-v0.2.json deleted file mode 100644 index 91011a9705cd84f0afaa6974ab6b3ed61aa8b691..0000000000000000000000000000000000000000 --- a/data/models/blackbeenie_bloslain-8b-v0.2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Bloslain-8B-v0.2", - "id": "BlackBeenie/Bloslain-8B-v0.2", - "developer": "BlackBeenie", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/BlackBeenie_Bloslain-8B-v0.2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5023 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5111 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.145 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3062 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4076 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3654 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/blackbeenie_llama-3-luminous-merged.json b/data/models/blackbeenie_llama-3-luminous-merged.json deleted file mode 100644 index 67a746364c3be90ad01dd9cfddc3be9ce10e79b5..0000000000000000000000000000000000000000 --- a/data/models/blackbeenie_llama-3-luminous-merged.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "llama-3-luminous-merged", - "id": "BlackBeenie/llama-3-luminous-merged", - "developer": "BlackBeenie", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/BlackBeenie_llama-3-luminous-merged/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4323 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5154 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0869 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2928 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4149 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3773 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/blackbeenie_llama-3.1-8b-galore-openassistant-guanaco.json b/data/models/blackbeenie_llama-3.1-8b-galore-openassistant-guanaco.json deleted file mode 100644 index 7904743c56244d9ea6a203bb3664281da21d07e6..0000000000000000000000000000000000000000 --- a/data/models/blackbeenie_llama-3.1-8b-galore-openassistant-guanaco.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "llama-3.1-8B-Galore-openassistant-guanaco", - "id": "BlackBeenie/llama-3.1-8B-Galore-openassistant-guanaco", - "developer": "BlackBeenie", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/BlackBeenie_llama-3.1-8B-Galore-openassistant-guanaco/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2635 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5213 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0665 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3003 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4406 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3206 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/blackbeenie_llama-3.1-8b-openo1-sft-v0.1.json b/data/models/blackbeenie_llama-3.1-8b-openo1-sft-v0.1.json deleted file mode 100644 index a31dbc6076c8a0103809c8ee57e41417df23e770..0000000000000000000000000000000000000000 --- a/data/models/blackbeenie_llama-3.1-8b-openo1-sft-v0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.1-8B-OpenO1-SFT-v0.1", - "id": "BlackBeenie/Llama-3.1-8B-OpenO1-SFT-v0.1", - "developer": "BlackBeenie", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/BlackBeenie_Llama-3.1-8B-OpenO1-SFT-v0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5124 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4787 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1526 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2685 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3618 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3492 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/blackbeenie_llama-3.1-8b-pythonic-passthrough-merge.json b/data/models/blackbeenie_llama-3.1-8b-pythonic-passthrough-merge.json deleted file mode 100644 index 79a60eda6346384e1e0f2aba334dc8dde584f363..0000000000000000000000000000000000000000 --- a/data/models/blackbeenie_llama-3.1-8b-pythonic-passthrough-merge.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.1-8B-pythonic-passthrough-merge", - "id": "BlackBeenie/Llama-3.1-8B-pythonic-passthrough-merge", - "developer": "BlackBeenie", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "20.245" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/BlackBeenie_Llama-3.1-8B-pythonic-passthrough-merge/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2316 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3454 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0113 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2685 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3778 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1332 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/blackbeenie_neos-gemma-2-9b.json b/data/models/blackbeenie_neos-gemma-2-9b.json deleted file mode 100644 index 54fd299d47b19251964f601770ddbdbcec85ef2e..0000000000000000000000000000000000000000 --- a/data/models/blackbeenie_neos-gemma-2-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Neos-Gemma-2-9b", - "id": "BlackBeenie/Neos-Gemma-2-9b", - "developer": "BlackBeenie", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "9.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/BlackBeenie_Neos-Gemma-2-9b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5876 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5503 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0982 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.323 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3618 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3981 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/blackbeenie_neos-llama-3.1-8b.json b/data/models/blackbeenie_neos-llama-3.1-8b.json deleted file mode 100644 index 119666e0e8c9b683b3666f5d18683617de7be910..0000000000000000000000000000000000000000 --- a/data/models/blackbeenie_neos-llama-3.1-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Neos-Llama-3.1-8B", - "id": "BlackBeenie/Neos-Llama-3.1-8B", - "developer": "BlackBeenie", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/BlackBeenie_Neos-Llama-3.1-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4944 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4425 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1322 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2685 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.375 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3262 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/blackbeenie_neos-llama-3.1-base.json b/data/models/blackbeenie_neos-llama-3.1-base.json deleted file mode 100644 index 76aa5fb4b6f26d088adc3ccca3d1c04546b33d7d..0000000000000000000000000000000000000000 --- a/data/models/blackbeenie_neos-llama-3.1-base.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Neos-Llama-3.1-base", - "id": "BlackBeenie/Neos-Llama-3.1-base", - "developer": "BlackBeenie", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "4.65" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/BlackBeenie_Neos-Llama-3.1-base/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1751 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.293 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2374 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3499 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1112 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/blackbeenie_neos-phi-3-14b-v0.1.json b/data/models/blackbeenie_neos-phi-3-14b-v0.1.json deleted file mode 100644 index 98e03d293e8be25cc40dbce00208b6442d19498c..0000000000000000000000000000000000000000 --- a/data/models/blackbeenie_neos-phi-3-14b-v0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Neos-Phi-3-14B-v0.1", - "id": "BlackBeenie/Neos-Phi-3-14B-v0.1", - "developer": "BlackBeenie", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Phi3ForCausalLM", - "params_billions": "13.96" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/BlackBeenie_Neos-Phi-3-14B-v0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4022 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6212 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1782 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3054 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4125 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4564 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bllossom_llama-3.2-korean-bllossom-aica-5b.json b/data/models/bllossom_llama-3.2-korean-bllossom-aica-5b.json deleted file mode 100644 index 856fc03c30f898a24bfc3cdd5967e8f795da4475..0000000000000000000000000000000000000000 --- a/data/models/bllossom_llama-3.2-korean-bllossom-aica-5b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "llama-3.2-Korean-Bllossom-AICA-5B", - "id": "Bllossom/llama-3.2-Korean-Bllossom-AICA-5B", - "developer": "Bllossom", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MllamaForConditionalGeneration", - "params_billions": "5.199" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Bllossom_llama-3.2-Korean-Bllossom-AICA-5B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5172 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4293 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1239 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2987 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3834 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.271 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bluuwhale_l3-sthenomaid-8b-v1.json b/data/models/bluuwhale_l3-sthenomaid-8b-v1.json deleted file mode 100644 index d0c7a67a5062c9d7cbb2f69a91f33963ec8f955e..0000000000000000000000000000000000000000 --- a/data/models/bluuwhale_l3-sthenomaid-8b-v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "L3-SthenoMaid-8B-V1", - "id": "bluuwhale/L3-SthenoMaid-8B-V1", - "developer": "bluuwhale", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bluuwhale_L3-SthenoMaid-8B-V1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7345 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5219 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.108 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2802 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3687 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3656 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/boltmonkey_dreadmix.json b/data/models/boltmonkey_dreadmix.json deleted file mode 100644 index 08b91cca607296de9e350ce740b5690b0b65a81a..0000000000000000000000000000000000000000 --- a/data/models/boltmonkey_dreadmix.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "DreadMix", - "id": "BoltMonkey/DreadMix", - "developer": "BoltMonkey", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/BoltMonkey_DreadMix/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7095 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5435 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1556 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2995 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4212 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.379 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/boltmonkey_neuraldaredevil-supernova-lite-7b-dareties-abliterated.json b/data/models/boltmonkey_neuraldaredevil-supernova-lite-7b-dareties-abliterated.json deleted file mode 100644 index ed5c0ad9e23e3e1024094c595c2931cbc579e102..0000000000000000000000000000000000000000 --- a/data/models/boltmonkey_neuraldaredevil-supernova-lite-7b-dareties-abliterated.json +++ /dev/null @@ -1,275 +0,0 @@ -{ - "model_info": { - "name": "NeuralDaredevil-SuperNova-Lite-7B-DARETIES-abliterated", - "id": "BoltMonkey/NeuralDaredevil-SuperNova-Lite-7B-DARETIES-abliterated", - "developer": "BoltMonkey", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/BoltMonkey_NeuralDaredevil-SuperNova-Lite-7B-DARETIES-abliterated/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.459 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5185 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0937 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2743 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4083 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3631 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "hfopenllm_v2/BoltMonkey_NeuralDaredevil-SuperNova-Lite-7B-DARETIES-abliterated/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7999 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5152 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1193 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.281 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4019 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3733 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/boltmonkey_superneuraldreaddevil-8b.json b/data/models/boltmonkey_superneuraldreaddevil-8b.json deleted file mode 100644 index a9efcf2315053537488f65ccd8016b83e8b47373..0000000000000000000000000000000000000000 --- a/data/models/boltmonkey_superneuraldreaddevil-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SuperNeuralDreadDevil-8b", - "id": "BoltMonkey/SuperNeuralDreadDevil-8b", - "developer": "BoltMonkey", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/BoltMonkey_SuperNeuralDreadDevil-8b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.771 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5286 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0929 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2919 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3977 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3679 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bond005_meno-tiny-0.1.json b/data/models/bond005_meno-tiny-0.1.json deleted file mode 100644 index 9ae0a6221255356ce476cfb1c92b2cca93d24b69..0000000000000000000000000000000000000000 --- a/data/models/bond005_meno-tiny-0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "meno-tiny-0.1", - "id": "bond005/meno-tiny-0.1", - "developer": "bond005", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.544" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bond005_meno-tiny-0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.455 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4263 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.139 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2819 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4185 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2786 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bosonai_higgs-llama-3-70b.json b/data/models/bosonai_higgs-llama-3-70b.json deleted file mode 100644 index 3b3264aab59719f19eae5c5e63d8fc76243b43a4..0000000000000000000000000000000000000000 --- a/data/models/bosonai_higgs-llama-3-70b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Higgs-Llama-3-70B", - "id": "bosonai/Higgs-Llama-3-70B", - "developer": "bosonai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "70.554" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bosonai_Higgs-Llama-3-70B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5561 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6258 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2523 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3666 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4471 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4902 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/braindao_deepseek-r1-distill-qwen-1.5b-blunt.json b/data/models/braindao_deepseek-r1-distill-qwen-1.5b-blunt.json deleted file mode 100644 index 781de11755a947a16edbead8d6a3054ffd8dbacf..0000000000000000000000000000000000000000 --- a/data/models/braindao_deepseek-r1-distill-qwen-1.5b-blunt.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "DeepSeek-R1-Distill-Qwen-1.5B-Blunt", - "id": "braindao/DeepSeek-R1-Distill-Qwen-1.5B-Blunt", - "developer": "braindao", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.777" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/braindao_DeepSeek-R1-Distill-Qwen-1.5B-Blunt/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2611 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2774 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1382 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2475 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3595 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1184 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/braindao_deepseek-r1-distill-qwen-1.5b-reflective.json b/data/models/braindao_deepseek-r1-distill-qwen-1.5b-reflective.json deleted file mode 100644 index e4056990d7e0ab6a75d7544dfd23bb8464fa4d60..0000000000000000000000000000000000000000 --- a/data/models/braindao_deepseek-r1-distill-qwen-1.5b-reflective.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "DeepSeek-R1-Distill-Qwen-1.5B-Reflective", - "id": "braindao/DeepSeek-R1-Distill-Qwen-1.5B-Reflective", - "developer": "braindao", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.777" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/braindao_DeepSeek-R1-Distill-Qwen-1.5B-Reflective/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3033 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2908 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1631 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2609 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3356 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.113 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/braindao_deepseek-r1-distill-qwen-14b-abub-st.json b/data/models/braindao_deepseek-r1-distill-qwen-14b-abub-st.json deleted file mode 100644 index 3ba1cd548710864a74a8a076ecdbdf2f5e398334..0000000000000000000000000000000000000000 --- a/data/models/braindao_deepseek-r1-distill-qwen-14b-abub-st.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "DeepSeek-R1-Distill-Qwen-14B-ABUB-ST", - "id": "braindao/DeepSeek-R1-Distill-Qwen-14B-ABUB-ST", - "developer": "braindao", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/braindao_DeepSeek-R1-Distill-Qwen-14B-ABUB-ST/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3752 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4927 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5015 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3448 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4221 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4243 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/braindao_deepseek-r1-distill-qwen-14b-blunt-uncensored-blunt-reflective.json b/data/models/braindao_deepseek-r1-distill-qwen-14b-blunt-uncensored-blunt-reflective.json deleted file mode 100644 index 1d9440cf3e22dc16c2f17233ecf9541c43dd8ffa..0000000000000000000000000000000000000000 --- a/data/models/braindao_deepseek-r1-distill-qwen-14b-blunt-uncensored-blunt-reflective.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "DeepSeek-R1-Distill-Qwen-14B-Blunt-Uncensored-Blunt-Reflective", - "id": "braindao/DeepSeek-R1-Distill-Qwen-14B-Blunt-Uncensored-Blunt-Reflective", - "developer": "braindao", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/braindao_DeepSeek-R1-Distill-Qwen-14B-Blunt-Uncensored-Blunt-Reflective/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.554 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3371 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2372 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2777 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4248 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1504 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/braindao_deepseek-r1-distill-qwen-14b-blunt-uncensored-blunt.json b/data/models/braindao_deepseek-r1-distill-qwen-14b-blunt-uncensored-blunt.json deleted file mode 100644 index 5071d284299e7abbdcfa31b045721957bf0fe4e8..0000000000000000000000000000000000000000 --- a/data/models/braindao_deepseek-r1-distill-qwen-14b-blunt-uncensored-blunt.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "DeepSeek-R1-Distill-Qwen-14B-Blunt-Uncensored-Blunt", - "id": "braindao/DeepSeek-R1-Distill-Qwen-14B-Blunt-Uncensored-Blunt", - "developer": "braindao", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/braindao_DeepSeek-R1-Distill-Qwen-14B-Blunt-Uncensored-Blunt/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5221 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3199 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2508 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2785 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4527 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1484 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/braindao_deepseek-r1-distill-qwen-14b-blunt-uncensored-reflective.json b/data/models/braindao_deepseek-r1-distill-qwen-14b-blunt-uncensored-reflective.json deleted file mode 100644 index c9b4af53ad47302aa127c1c127fcf099f3584ee7..0000000000000000000000000000000000000000 --- a/data/models/braindao_deepseek-r1-distill-qwen-14b-blunt-uncensored-reflective.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "DeepSeek-R1-Distill-Qwen-14B-Blunt-Uncensored-Reflective", - "id": "braindao/DeepSeek-R1-Distill-Qwen-14B-Blunt-Uncensored-Reflective", - "developer": "braindao", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/braindao_DeepSeek-R1-Distill-Qwen-14B-Blunt-Uncensored-Reflective/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5139 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3013 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1473 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2878 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4433 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1289 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/braindao_deepseek-r1-distill-qwen-14b-blunt-uncensored.json b/data/models/braindao_deepseek-r1-distill-qwen-14b-blunt-uncensored.json deleted file mode 100644 index cb311ec9509dea81520901df5d4e998a5b9bcd13..0000000000000000000000000000000000000000 --- a/data/models/braindao_deepseek-r1-distill-qwen-14b-blunt-uncensored.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "DeepSeek-R1-Distill-Qwen-14B-Blunt-Uncensored", - "id": "braindao/DeepSeek-R1-Distill-Qwen-14B-Blunt-Uncensored", - "developer": "braindao", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/braindao_DeepSeek-R1-Distill-Qwen-14B-Blunt-Uncensored/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5422 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.317 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1631 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2827 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4487 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1431 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/braindao_deepseek-r1-distill-qwen-14b-blunt.json b/data/models/braindao_deepseek-r1-distill-qwen-14b-blunt.json deleted file mode 100644 index 59c8d9032ca9f1fcc76663135a7a84435d9d8701..0000000000000000000000000000000000000000 --- a/data/models/braindao_deepseek-r1-distill-qwen-14b-blunt.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "DeepSeek-R1-Distill-Qwen-14B-Blunt", - "id": "braindao/DeepSeek-R1-Distill-Qwen-14B-Blunt", - "developer": "braindao", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/braindao_DeepSeek-R1-Distill-Qwen-14B-Blunt/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5612 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3283 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1639 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3029 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4554 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1447 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/braindao_deepseek-r1-distill-qwen-14b-reflective.json b/data/models/braindao_deepseek-r1-distill-qwen-14b-reflective.json deleted file mode 100644 index d11694f32718615fd6612804a0bf64b6f3a28c2c..0000000000000000000000000000000000000000 --- a/data/models/braindao_deepseek-r1-distill-qwen-14b-reflective.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "DeepSeek-R1-Distill-Qwen-14B-Reflective", - "id": "braindao/DeepSeek-R1-Distill-Qwen-14B-Reflective", - "developer": "braindao", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/braindao_DeepSeek-R1-Distill-Qwen-14B-Reflective/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.429 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3012 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1918 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2727 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4554 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1129 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/braindao_deepseek-r1-distill-qwen-14b.json b/data/models/braindao_deepseek-r1-distill-qwen-14b.json deleted file mode 100644 index 8c6e4bbbaf7296b3ae074c6408edb6630464433c..0000000000000000000000000000000000000000 --- a/data/models/braindao_deepseek-r1-distill-qwen-14b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "DeepSeek-R1-Distill-Qwen-14B", - "id": "braindao/DeepSeek-R1-Distill-Qwen-14B", - "developer": "braindao", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/braindao_DeepSeek-R1-Distill-Qwen-14B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4172 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3033 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.176 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2802 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4488 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1127 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/braindao_deepseek-r1-distill-qwen-7b-blunt.json b/data/models/braindao_deepseek-r1-distill-qwen-7b-blunt.json deleted file mode 100644 index 7ac351e733eb19e5598f2871f0b2ab44d30307d3..0000000000000000000000000000000000000000 --- a/data/models/braindao_deepseek-r1-distill-qwen-7b-blunt.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "DeepSeek-R1-Distill-Qwen-7B-Blunt", - "id": "braindao/DeepSeek-R1-Distill-Qwen-7B-Blunt", - "developer": "braindao", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/braindao_DeepSeek-R1-Distill-Qwen-7B-Blunt/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4266 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2902 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2145 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.271 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3885 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1169 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/braindao_deepseek-r1-distill-qwen-7b-orpo-uncensored.json b/data/models/braindao_deepseek-r1-distill-qwen-7b-orpo-uncensored.json deleted file mode 100644 index 86ea623849a0c8fe316eede1388476564509c273..0000000000000000000000000000000000000000 --- a/data/models/braindao_deepseek-r1-distill-qwen-7b-orpo-uncensored.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "DeepSeek-R1-Distill-Qwen-7B-ORPO-Uncensored", - "id": "braindao/DeepSeek-R1-Distill-Qwen-7B-ORPO-Uncensored", - "developer": "braindao", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/braindao_DeepSeek-R1-Distill-Qwen-7B-ORPO-Uncensored/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3655 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2958 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1737 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2534 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3846 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1133 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/braindao_deepseek-r1-distill-qwen-7b-reflective.json b/data/models/braindao_deepseek-r1-distill-qwen-7b-reflective.json deleted file mode 100644 index c03b024b55c630a61d773da7abd3201c87296f9d..0000000000000000000000000000000000000000 --- a/data/models/braindao_deepseek-r1-distill-qwen-7b-reflective.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "DeepSeek-R1-Distill-Qwen-7B-Reflective", - "id": "braindao/DeepSeek-R1-Distill-Qwen-7B-Reflective", - "developer": "braindao", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/braindao_DeepSeek-R1-Distill-Qwen-7B-Reflective/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3922 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2907 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2024 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2542 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.39 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1155 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/braindao_deepseek-r1-distill-qwen-7b.json b/data/models/braindao_deepseek-r1-distill-qwen-7b.json deleted file mode 100644 index 5a4a361c85c98f52294e73d1f5c55cef0fb79f3d..0000000000000000000000000000000000000000 --- a/data/models/braindao_deepseek-r1-distill-qwen-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "DeepSeek-R1-Distill-Qwen-7B", - "id": "braindao/DeepSeek-R1-Distill-Qwen-7B", - "developer": "braindao", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/braindao_DeepSeek-R1-Distill-Qwen-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3968 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2887 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1918 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2617 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3767 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1141 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/braindao_iq-code-evmind-0.5b.json b/data/models/braindao_iq-code-evmind-0.5b.json deleted file mode 100644 index 7edfca632d49cb4d1b5ff3e6d9735d348d09a733..0000000000000000000000000000000000000000 --- a/data/models/braindao_iq-code-evmind-0.5b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "iq-code-evmind-0.5b", - "id": "braindao/iq-code-evmind-0.5b", - "developer": "braindao", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/braindao_iq-code-evmind-0.5b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3216 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3164 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0242 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2416 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3304 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1189 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/braindao_qwen2.5-14b-instruct.json b/data/models/braindao_qwen2.5-14b-instruct.json deleted file mode 100644 index 2df6939c96426f63767a592fda6de1fc939e9331..0000000000000000000000000000000000000000 --- a/data/models/braindao_qwen2.5-14b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-14B-Instruct", - "id": "braindao/Qwen2.5-14B-Instruct", - "developer": "braindao", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/braindao_Qwen2.5-14B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8143 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6404 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5529 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3289 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.414 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4889 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/braindao_qwen2.5-14b.json b/data/models/braindao_qwen2.5-14b.json deleted file mode 100644 index 8c90cb851beb882a8b4cfad0a08a589c44e4afcd..0000000000000000000000000000000000000000 --- a/data/models/braindao_qwen2.5-14b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-14B", - "id": "braindao/Qwen2.5-14B", - "developer": "braindao", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/braindao_Qwen2.5-14B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5409 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5853 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2923 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3733 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4124 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4884 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/brainwave-ml_llama3.2-3b-maths-orpo.json b/data/models/brainwave-ml_llama3.2-3b-maths-orpo.json deleted file mode 100644 index 42ff74b63844399a017ab173ce48dce94befc73a..0000000000000000000000000000000000000000 --- a/data/models/brainwave-ml_llama3.2-3b-maths-orpo.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "llama3.2-3B-maths-orpo", - "id": "BrainWave-ML/llama3.2-3B-maths-orpo", - "developer": "BrainWave-ML", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/BrainWave-ML_llama3.2-3B-maths-orpo/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2049 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2912 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2592 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3575 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1168 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bramvanroy_fietje-2-chat.json b/data/models/bramvanroy_fietje-2-chat.json deleted file mode 100644 index a05515c19c6d4d1e3b1deb3009bbd3aa8c6e3348..0000000000000000000000000000000000000000 --- a/data/models/bramvanroy_fietje-2-chat.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "fietje-2-chat", - "id": "BramVanroy/fietje-2-chat", - "developer": "BramVanroy", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "PhiForCausalLM", - "params_billions": "2.775" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/BramVanroy_fietje-2-chat/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2917 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.415 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0189 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2399 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3528 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2055 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bramvanroy_fietje-2-instruct.json b/data/models/bramvanroy_fietje-2-instruct.json deleted file mode 100644 index be9c94dbc15fffd10ef60f37e56439c1efc7e12b..0000000000000000000000000000000000000000 --- a/data/models/bramvanroy_fietje-2-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "fietje-2-instruct", - "id": "BramVanroy/fietje-2-instruct", - "developer": "BramVanroy", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "PhiForCausalLM", - "params_billions": "2.775" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/BramVanroy_fietje-2-instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.279 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4136 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0227 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2332 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3369 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2104 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bramvanroy_fietje-2.json b/data/models/bramvanroy_fietje-2.json deleted file mode 100644 index 39f499e74fa1bceae749b2d83fa14a176d0936da..0000000000000000000000000000000000000000 --- a/data/models/bramvanroy_fietje-2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "fietje-2", - "id": "BramVanroy/fietje-2", - "developer": "BramVanroy", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "PhiForCausalLM", - "params_billions": "2.78" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/BramVanroy_fietje-2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2098 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4036 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0159 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2542 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3696 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1986 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bramvanroy_geitje-7b-ultra.json b/data/models/bramvanroy_geitje-7b-ultra.json deleted file mode 100644 index 8d4c1b7143af601b9eed04b8eb48dc4d0fd301d3..0000000000000000000000000000000000000000 --- a/data/models/bramvanroy_geitje-7b-ultra.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "GEITje-7B-ultra", - "id": "BramVanroy/GEITje-7B-ultra", - "developer": "BramVanroy", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/BramVanroy_GEITje-7B-ultra/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3723 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3776 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0159 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2626 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.329 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2011 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/brgx53_3bgeneral-ece-prymmal-martial.json b/data/models/brgx53_3bgeneral-ece-prymmal-martial.json deleted file mode 100644 index 41df8fa7c42ef69d56772241177e2ec60e217437..0000000000000000000000000000000000000000 --- a/data/models/brgx53_3bgeneral-ece-prymmal-martial.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "3Bgeneral-ECE-PRYMMAL-Martial", - "id": "brgx53/3Bgeneral-ECE-PRYMMAL-Martial", - "developer": "brgx53", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Phi3ForCausalLM", - "params_billions": "3.821" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/brgx53_3Bgeneral-ECE-PRYMMAL-Martial/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3289 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5458 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1314 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3247 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4373 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3934 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/brgx53_3bgeneralv2-ece-prymmal-martial.json b/data/models/brgx53_3bgeneralv2-ece-prymmal-martial.json deleted file mode 100644 index 9f060e37884ed2ca3d7844e41e79ceb0485734a6..0000000000000000000000000000000000000000 --- a/data/models/brgx53_3bgeneralv2-ece-prymmal-martial.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "3Bgeneralv2-ECE-PRYMMAL-Martial", - "id": "brgx53/3Bgeneralv2-ECE-PRYMMAL-Martial", - "developer": "brgx53", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/brgx53_3Bgeneralv2-ECE-PRYMMAL-Martial/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5677 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5607 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3497 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3112 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4356 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4505 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/brgx53_3blareneg-ece-prymmal-martial.json b/data/models/brgx53_3blareneg-ece-prymmal-martial.json deleted file mode 100644 index 2797ae636bbdde1b4d43fb4eaf7bdee87a884273..0000000000000000000000000000000000000000 --- a/data/models/brgx53_3blareneg-ece-prymmal-martial.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "3Blareneg-ECE-PRYMMAL-Martial", - "id": "brgx53/3Blareneg-ECE-PRYMMAL-Martial", - "developer": "brgx53", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Phi3ForCausalLM", - "params_billions": "3.821" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/brgx53_3Blareneg-ECE-PRYMMAL-Martial/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2876 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5358 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1208 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3347 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4429 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4016 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/brgx53_3blarenegv2-ece-prymmal-martial.json b/data/models/brgx53_3blarenegv2-ece-prymmal-martial.json deleted file mode 100644 index 99b0619c592af77784af5a74317d1b1a668afb99..0000000000000000000000000000000000000000 --- a/data/models/brgx53_3blarenegv2-ece-prymmal-martial.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "3Blarenegv2-ECE-PRYMMAL-Martial", - "id": "brgx53/3Blarenegv2-ECE-PRYMMAL-Martial", - "developer": "brgx53", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/brgx53_3Blarenegv2-ECE-PRYMMAL-Martial/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5662 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5607 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3497 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3112 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4356 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4505 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/brgx53_barracuda-prymmal-ece-tw3.json b/data/models/brgx53_barracuda-prymmal-ece-tw3.json deleted file mode 100644 index 41c5fefdf670f25bb10c5d9f7653b9998baf622b..0000000000000000000000000000000000000000 --- a/data/models/brgx53_barracuda-prymmal-ece-tw3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Barracuda-PRYMMAL-ECE-TW3", - "id": "brgx53/Barracuda-PRYMMAL-ECE-TW3", - "developer": "brgx53", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.544" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/brgx53_Barracuda-PRYMMAL-ECE-TW3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.164 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3002 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0023 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2534 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3609 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1093 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/brgx53_laconfiance-prymmal-ece-tw3.json b/data/models/brgx53_laconfiance-prymmal-ece-tw3.json deleted file mode 100644 index 3a7a73b0010fff4464f5a72dd877ad8ef01c219b..0000000000000000000000000000000000000000 --- a/data/models/brgx53_laconfiance-prymmal-ece-tw3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "LaConfiance-PRYMMAL-ECE-TW3", - "id": "brgx53/LaConfiance-PRYMMAL-ECE-TW3", - "developer": "brgx53", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.777" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/brgx53_LaConfiance-PRYMMAL-ECE-TW3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1579 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2962 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2517 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3846 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1146 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bsc-lt_salamandra-7b-instruct.json b/data/models/bsc-lt_salamandra-7b-instruct.json deleted file mode 100644 index 15378d0ad50e86aaa088f5112ae1ae3eb4b40ac8..0000000000000000000000000000000000000000 --- a/data/models/bsc-lt_salamandra-7b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "salamandra-7b-instruct", - "id": "BSC-LT/salamandra-7b-instruct", - "developer": "BSC-LT", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "7.768" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/BSC-LT_salamandra-7b-instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2451 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3851 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0083 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2643 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4134 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1805 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bsc-lt_salamandra-7b.json b/data/models/bsc-lt_salamandra-7b.json deleted file mode 100644 index 8b9ba270964e1bcf9a0271db8e2aa583d8429688..0000000000000000000000000000000000000000 --- a/data/models/bsc-lt_salamandra-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "salamandra-7b", - "id": "BSC-LT/salamandra-7b", - "developer": "BSC-LT", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "7.768" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/BSC-LT_salamandra-7b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1367 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3517 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0038 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2701 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3501 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1493 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bunnycore_best-mix-llama-3.1-8b.json b/data/models/bunnycore_best-mix-llama-3.1-8b.json deleted file mode 100644 index 15da50ad950872bf50b56d296f263d7b37bea1ab..0000000000000000000000000000000000000000 --- a/data/models/bunnycore_best-mix-llama-3.1-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Best-Mix-Llama-3.1-8B", - "id": "bunnycore/Best-Mix-Llama-3.1-8B", - "developer": "bunnycore", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bunnycore_Best-Mix-Llama-3.1-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2067 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3432 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2054 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2651 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2929 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1565 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bunnycore_blabbertron-1.0.json b/data/models/bunnycore_blabbertron-1.0.json deleted file mode 100644 index bd62d786a2dcb29553f1ceefca894c391f4ebead..0000000000000000000000000000000000000000 --- a/data/models/bunnycore_blabbertron-1.0.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Blabbertron-1.0", - "id": "bunnycore/Blabbertron-1.0", - "developer": "bunnycore", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.613" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bunnycore_Blabbertron-1.0/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7433 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5497 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4924 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.302 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4337 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4354 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bunnycore_blabbertron-1.1.json b/data/models/bunnycore_blabbertron-1.1.json deleted file mode 100644 index a9bc0dd9694dd0494564c961d16153d505e44a3d..0000000000000000000000000000000000000000 --- a/data/models/bunnycore_blabbertron-1.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Blabbertron-1.1", - "id": "bunnycore/Blabbertron-1.1", - "developer": "bunnycore", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.613" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bunnycore_Blabbertron-1.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7265 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5534 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4804 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3029 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4416 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4431 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bunnycore_cybercore-qwen-2.1-7b.json b/data/models/bunnycore_cybercore-qwen-2.1-7b.json deleted file mode 100644 index cc7f5020df6e2402a2a2c9a45156948e78644852..0000000000000000000000000000000000000000 --- a/data/models/bunnycore_cybercore-qwen-2.1-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "CyberCore-Qwen-2.1-7B", - "id": "bunnycore/CyberCore-Qwen-2.1-7B", - "developer": "bunnycore", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bunnycore_CyberCore-Qwen-2.1-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5766 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5572 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3588 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3079 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4145 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4445 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bunnycore_deepqwen-3b-lcot-sce.json b/data/models/bunnycore_deepqwen-3b-lcot-sce.json deleted file mode 100644 index 6b2baba9ca0e2bbb9806980af7de949b08a3643e..0000000000000000000000000000000000000000 --- a/data/models/bunnycore_deepqwen-3b-lcot-sce.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "DeepQwen-3B-LCoT-SCE", - "id": "bunnycore/DeepQwen-3B-LCoT-SCE", - "developer": "bunnycore", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.396" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bunnycore_DeepQwen-3B-LCoT-SCE/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.449 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4512 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.247 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2626 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3514 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.329 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bunnycore_deepseek-r1-distill-qwen-7b-rrp-ex.json b/data/models/bunnycore_deepseek-r1-distill-qwen-7b-rrp-ex.json deleted file mode 100644 index 68945c7faa58697c8a94d42ee260d8eb5c8e512a..0000000000000000000000000000000000000000 --- a/data/models/bunnycore_deepseek-r1-distill-qwen-7b-rrp-ex.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "DeepSeek-R1-Distill-Qwen-7B-RRP-Ex", - "id": "bunnycore/DeepSeek-R1-Distill-Qwen-7B-RRP-Ex", - "developer": "bunnycore", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bunnycore_DeepSeek-R1-Distill-Qwen-7B-RRP-Ex/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3901 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3494 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1654 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2785 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3663 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2508 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bunnycore_deepthinker-7b-sce-v1.json b/data/models/bunnycore_deepthinker-7b-sce-v1.json deleted file mode 100644 index 35f67744bbedaebdddc318395a61ec5b5c1aab4c..0000000000000000000000000000000000000000 --- a/data/models/bunnycore_deepthinker-7b-sce-v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "DeepThinker-7B-Sce-v1", - "id": "bunnycore/DeepThinker-7B-Sce-v1", - "developer": "bunnycore", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.613" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bunnycore_DeepThinker-7B-Sce-v1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1218 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3018 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0098 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2517 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4194 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1123 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bunnycore_deepthinker-7b-sce-v2.json b/data/models/bunnycore_deepthinker-7b-sce-v2.json deleted file mode 100644 index 700606fb853943fc0181562becc29bf7d956f7a7..0000000000000000000000000000000000000000 --- a/data/models/bunnycore_deepthinker-7b-sce-v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "DeepThinker-7B-Sce-v2", - "id": "bunnycore/DeepThinker-7B-Sce-v2", - "developer": "bunnycore", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.613" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bunnycore_DeepThinker-7B-Sce-v2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1631 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3057 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0113 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2584 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4101 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1146 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bunnycore_fusecybermix-qwen-2.5-7b-instruct.json b/data/models/bunnycore_fusecybermix-qwen-2.5-7b-instruct.json deleted file mode 100644 index 96c8c8c6769f368b4f2a1f5be64e2d5c29a34da7..0000000000000000000000000000000000000000 --- a/data/models/bunnycore_fusecybermix-qwen-2.5-7b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "FuseCyberMix-Qwen-2.5-7B-Instruct", - "id": "bunnycore/FuseCyberMix-Qwen-2.5-7B-Instruct", - "developer": "bunnycore", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bunnycore_FuseCyberMix-Qwen-2.5-7B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7019 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5518 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4841 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.297 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.402 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4337 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bunnycore_fuseqwqen-7b.json b/data/models/bunnycore_fuseqwqen-7b.json deleted file mode 100644 index 9c279f26832b66188e2ead2137fe1d5f36c32a97..0000000000000000000000000000000000000000 --- a/data/models/bunnycore_fuseqwqen-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "FuseQwQen-7B", - "id": "bunnycore/FuseQwQen-7B", - "developer": "bunnycore", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bunnycore_FuseQwQen-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7275 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5504 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4366 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2945 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4217 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4407 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bunnycore_fwf-qwen-7b-0.1.json b/data/models/bunnycore_fwf-qwen-7b-0.1.json deleted file mode 100644 index 48d70645ded4bff76a8ae9f667f7580b9a211d6a..0000000000000000000000000000000000000000 --- a/data/models/bunnycore_fwf-qwen-7b-0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "FwF-Qwen-7B-0.1", - "id": "bunnycore/FwF-Qwen-7B-0.1", - "developer": "bunnycore", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bunnycore_FwF-Qwen-7B-0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3005 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5019 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2764 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.271 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3952 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4061 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bunnycore_fwf-qwen-7b-0.2.json b/data/models/bunnycore_fwf-qwen-7b-0.2.json deleted file mode 100644 index 9c7feb5a07f3d68888627b21a39a165358851b38..0000000000000000000000000000000000000000 --- a/data/models/bunnycore_fwf-qwen-7b-0.2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "FwF-Qwen-7B-0.2", - "id": "bunnycore/FwF-Qwen-7B-0.2", - "developer": "bunnycore", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bunnycore_FwF-Qwen-7B-0.2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4479 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5596 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.426 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2903 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4218 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4382 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bunnycore_gemma-2-2b-smart.json b/data/models/bunnycore_gemma-2-2b-smart.json deleted file mode 100644 index c4a2f8f6395eae119257057edddf624e495fa30c..0000000000000000000000000000000000000000 --- a/data/models/bunnycore_gemma-2-2b-smart.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Gemma-2-2B-Smart", - "id": "bunnycore/Gemma-2-2B-Smart", - "developer": "bunnycore", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "2.614" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bunnycore_Gemma-2-2B-Smart/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1321 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3974 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0332 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2827 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4249 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2426 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bunnycore_gemma2-9b-titanfusion.json b/data/models/bunnycore_gemma2-9b-titanfusion.json deleted file mode 100644 index 38d7e21316e6115845170f66d677223e71944eb8..0000000000000000000000000000000000000000 --- a/data/models/bunnycore_gemma2-9b-titanfusion.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Gemma2-9B-TitanFusion", - "id": "bunnycore/Gemma2-9B-TitanFusion", - "developer": "bunnycore", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bunnycore_Gemma2-9B-TitanFusion/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1618 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5712 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.077 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3322 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4136 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.396 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bunnycore_hyperllama-3.1-8b.json b/data/models/bunnycore_hyperllama-3.1-8b.json deleted file mode 100644 index 13d6bddd14256ec5c057e70d8282ee24d6245bd4..0000000000000000000000000000000000000000 --- a/data/models/bunnycore_hyperllama-3.1-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "HyperLlama-3.1-8B", - "id": "bunnycore/HyperLlama-3.1-8B", - "developer": "bunnycore", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bunnycore_HyperLlama-3.1-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7883 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5103 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1828 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2869 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3829 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3783 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bunnycore_llama-3.1-8b-titanfusion-mix.json b/data/models/bunnycore_llama-3.1-8b-titanfusion-mix.json deleted file mode 100644 index b24ccff8b60f6705a9201b0334d57f680fa89afb..0000000000000000000000000000000000000000 --- a/data/models/bunnycore_llama-3.1-8b-titanfusion-mix.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.1-8B-TitanFusion-Mix", - "id": "bunnycore/Llama-3.1-8B-TitanFusion-Mix", - "developer": "bunnycore", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bunnycore_Llama-3.1-8B-TitanFusion-Mix/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4925 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5756 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1284 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2953 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4317 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3695 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bunnycore_llama-3.1-8b-titanfusion-v3.json b/data/models/bunnycore_llama-3.1-8b-titanfusion-v3.json deleted file mode 100644 index 04b0a23ae32c8e0c49e9b8fac5889fd3f519779f..0000000000000000000000000000000000000000 --- a/data/models/bunnycore_llama-3.1-8b-titanfusion-v3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.1-8B-TitanFusion-v3", - "id": "bunnycore/Llama-3.1-8B-TitanFusion-v3", - "developer": "bunnycore", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bunnycore_Llama-3.1-8B-TitanFusion-v3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.481 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5262 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.142 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3087 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4302 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3806 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bunnycore_llama-3.2-3b-all-mix.json b/data/models/bunnycore_llama-3.2-3b-all-mix.json deleted file mode 100644 index c7ad7723b0f4d1b8f07678b36f44c1492056d74d..0000000000000000000000000000000000000000 --- a/data/models/bunnycore_llama-3.2-3b-all-mix.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.2-3B-All-Mix", - "id": "bunnycore/Llama-3.2-3B-All-Mix", - "developer": "bunnycore", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.607" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bunnycore_Llama-3.2-3B-All-Mix/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7226 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4508 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1503 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2626 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3287 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.316 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bunnycore_llama-3.2-3b-bespoke-thought.json b/data/models/bunnycore_llama-3.2-3b-bespoke-thought.json deleted file mode 100644 index 54038bcecd11a581862b9137d932448e1ac23a8b..0000000000000000000000000000000000000000 --- a/data/models/bunnycore_llama-3.2-3b-bespoke-thought.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.2-3B-Bespoke-Thought", - "id": "bunnycore/Llama-3.2-3B-Bespoke-Thought", - "developer": "bunnycore", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bunnycore_Llama-3.2-3B-Bespoke-Thought/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4113 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4522 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1647 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2659 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3302 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.311 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bunnycore_llama-3.2-3b-booval.json b/data/models/bunnycore_llama-3.2-3b-booval.json deleted file mode 100644 index b873f3e6483026e64b4ecdfb9a5839ebf9b53043..0000000000000000000000000000000000000000 --- a/data/models/bunnycore_llama-3.2-3b-booval.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.2-3B-Booval", - "id": "bunnycore/Llama-3.2-3B-Booval", - "developer": "bunnycore", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bunnycore_Llama-3.2-3B-Booval/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6669 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4514 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1269 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2668 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3394 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3058 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bunnycore_llama-3.2-3b-deep-test.json b/data/models/bunnycore_llama-3.2-3b-deep-test.json deleted file mode 100644 index c829321bc4e1b62229f276602c62e6f6bd9a3faf..0000000000000000000000000000000000000000 --- a/data/models/bunnycore_llama-3.2-3b-deep-test.json +++ /dev/null @@ -1,275 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.2-3B-Deep-Test", - "id": "bunnycore/Llama-3.2-3B-Deep-Test", - "developer": "bunnycore", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.803" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bunnycore_Llama-3.2-3B-Deep-Test/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1775 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.295 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2517 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3647 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1049 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "hfopenllm_v2/bunnycore_Llama-3.2-3B-Deep-Test/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4652 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4531 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1284 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2643 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3394 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3152 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bunnycore_llama-3.2-3b-della.json b/data/models/bunnycore_llama-3.2-3b-della.json deleted file mode 100644 index 1283722d40a2b96f39b3c1fcc2c176d34bb9ba64..0000000000000000000000000000000000000000 --- a/data/models/bunnycore_llama-3.2-3b-della.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.2-3B-Della", - "id": "bunnycore/Llama-3.2-3B-Della", - "developer": "bunnycore", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.607" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bunnycore_Llama-3.2-3B-Della/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3561 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3683 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0302 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.276 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3902 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2128 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bunnycore_llama-3.2-3b-long-think.json b/data/models/bunnycore_llama-3.2-3b-long-think.json deleted file mode 100644 index 8f0dc9789aa4aa885ecc21b0388140803f7cd05f..0000000000000000000000000000000000000000 --- a/data/models/bunnycore_llama-3.2-3b-long-think.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.2-3B-Long-Think", - "id": "bunnycore/Llama-3.2-3B-Long-Think", - "developer": "bunnycore", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bunnycore_Llama-3.2-3B-Long-Think/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5473 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.461 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1458 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2609 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3396 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3048 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bunnycore_llama-3.2-3b-mix-skill.json b/data/models/bunnycore_llama-3.2-3b-mix-skill.json deleted file mode 100644 index 8fa74727a7b4eac6783656bd4226f870d3e6cbdb..0000000000000000000000000000000000000000 --- a/data/models/bunnycore_llama-3.2-3b-mix-skill.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.2-3B-Mix-Skill", - "id": "bunnycore/Llama-3.2-3B-Mix-Skill", - "developer": "bunnycore", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.607" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bunnycore_Llama-3.2-3B-Mix-Skill/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6404 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4582 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1473 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2617 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3396 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3121 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bunnycore_llama-3.2-3b-prodigyplus.json b/data/models/bunnycore_llama-3.2-3b-prodigyplus.json deleted file mode 100644 index 2a240dd588b31a5d08b756e6c3d43714444ad43f..0000000000000000000000000000000000000000 --- a/data/models/bunnycore_llama-3.2-3b-prodigyplus.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.2-3B-ProdigyPlus", - "id": "bunnycore/Llama-3.2-3B-ProdigyPlus", - "developer": "bunnycore", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.607" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bunnycore_Llama-3.2-3B-ProdigyPlus/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4015 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4392 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1156 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2685 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.358 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2817 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bunnycore_llama-3.2-3b-prodigyplusplus.json b/data/models/bunnycore_llama-3.2-3b-prodigyplusplus.json deleted file mode 100644 index bc562ebb176f841cf9b3f7b9b5468387ee1cef80..0000000000000000000000000000000000000000 --- a/data/models/bunnycore_llama-3.2-3b-prodigyplusplus.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.2-3B-ProdigyPlusPlus", - "id": "bunnycore/Llama-3.2-3B-ProdigyPlusPlus", - "developer": "bunnycore", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.607" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bunnycore_Llama-3.2-3B-ProdigyPlusPlus/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1645 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.369 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0453 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2534 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3541 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.15 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bunnycore_llama-3.2-3b-rp-deepthink.json b/data/models/bunnycore_llama-3.2-3b-rp-deepthink.json deleted file mode 100644 index d1938c0f0796675742a370cfa18e56bf0517c15c..0000000000000000000000000000000000000000 --- a/data/models/bunnycore_llama-3.2-3b-rp-deepthink.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.2-3B-RP-DeepThink", - "id": "bunnycore/Llama-3.2-3B-RP-DeepThink", - "developer": "bunnycore", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.607" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bunnycore_Llama-3.2-3B-RP-DeepThink/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7144 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4563 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1609 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2659 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3302 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3242 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bunnycore_llama-3.2-3b-rp-toxic-fuse.json b/data/models/bunnycore_llama-3.2-3b-rp-toxic-fuse.json deleted file mode 100644 index 1c8e57a261e5c4d5c6305b53d73642e745012362..0000000000000000000000000000000000000000 --- a/data/models/bunnycore_llama-3.2-3b-rp-toxic-fuse.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.2-3b-RP-Toxic-Fuse", - "id": "bunnycore/Llama-3.2-3b-RP-Toxic-Fuse", - "developer": "bunnycore", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bunnycore_Llama-3.2-3b-RP-Toxic-Fuse/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6834 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.465 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2402 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2777 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3954 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3106 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bunnycore_llama-3.2-3b-rrstock.json b/data/models/bunnycore_llama-3.2-3b-rrstock.json deleted file mode 100644 index 389482fbe3c0f34912b7e98db082eca53cf03f09..0000000000000000000000000000000000000000 --- a/data/models/bunnycore_llama-3.2-3b-rrstock.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.2-3B-RRStock", - "id": "bunnycore/Llama-3.2-3B-RRStock", - "developer": "bunnycore", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.607" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bunnycore_Llama-3.2-3B-RRStock/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6657 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4568 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1699 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2659 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3314 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3236 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bunnycore_llama-3.2-3b-toxickod.json b/data/models/bunnycore_llama-3.2-3b-toxickod.json deleted file mode 100644 index 21bfb58291ae6c2fc0672e4a30fad064c4c6639a..0000000000000000000000000000000000000000 --- a/data/models/bunnycore_llama-3.2-3b-toxickod.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.2-3B-ToxicKod", - "id": "bunnycore/Llama-3.2-3B-ToxicKod", - "developer": "bunnycore", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bunnycore_Llama-3.2-3B-ToxicKod/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6319 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4525 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1699 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2659 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3475 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.288 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bunnycore_maestro-s1k-7b-sce.json b/data/models/bunnycore_maestro-s1k-7b-sce.json deleted file mode 100644 index 034852c52b3a76c51271e7ede306277e76f62a4a..0000000000000000000000000000000000000000 --- a/data/models/bunnycore_maestro-s1k-7b-sce.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Maestro-S1k-7B-Sce", - "id": "bunnycore/Maestro-S1k-7B-Sce", - "developer": "bunnycore", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.613" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bunnycore_Maestro-S1k-7B-Sce/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2523 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3104 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0279 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2609 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3768 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.117 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bunnycore_phi-3.5-mini-titanfusion-0.1.json b/data/models/bunnycore_phi-3.5-mini-titanfusion-0.1.json deleted file mode 100644 index 1f4669da3311d0041a8baa7839dc4d201a6c9392..0000000000000000000000000000000000000000 --- a/data/models/bunnycore_phi-3.5-mini-titanfusion-0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Phi-3.5-mini-TitanFusion-0.1", - "id": "bunnycore/Phi-3.5-mini-TitanFusion-0.1", - "developer": "bunnycore", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Phi3ForCausalLM", - "params_billions": "3.821" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bunnycore_Phi-3.5-mini-TitanFusion-0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5228 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5374 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1186 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3314 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4453 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3807 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bunnycore_phi-4-model-stock-v2.json b/data/models/bunnycore_phi-4-model-stock-v2.json deleted file mode 100644 index 3ddef3d2fa113faa539ad04e69f79dba60339794..0000000000000000000000000000000000000000 --- a/data/models/bunnycore_phi-4-model-stock-v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Phi-4-Model-Stock-v2", - "id": "bunnycore/Phi-4-Model-Stock-v2", - "developer": "bunnycore", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bunnycore_Phi-4-Model-Stock-v2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6375 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6825 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3754 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.349 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4662 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5331 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bunnycore_phi-4-model-stock-v3.json b/data/models/bunnycore_phi-4-model-stock-v3.json deleted file mode 100644 index 2db751d987e8bef4a85efd8c6ef95fcb811268cc..0000000000000000000000000000000000000000 --- a/data/models/bunnycore_phi-4-model-stock-v3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Phi-4-Model-Stock-v3", - "id": "bunnycore/Phi-4-Model-Stock-v3", - "developer": "bunnycore", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bunnycore_Phi-4-Model-Stock-v3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5912 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6726 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4902 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2894 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4166 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5381 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bunnycore_phi-4-model-stock-v4.json b/data/models/bunnycore_phi-4-model-stock-v4.json deleted file mode 100644 index a9cb49d5fa675b736b4f77e6e81391ce86abbfb2..0000000000000000000000000000000000000000 --- a/data/models/bunnycore_phi-4-model-stock-v4.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Phi-4-Model-Stock-v4", - "id": "bunnycore/Phi-4-Model-Stock-v4", - "developer": "bunnycore", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bunnycore_Phi-4-Model-Stock-v4/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.711 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6924 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3829 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3691 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4611 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5394 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bunnycore_phi-4-model-stock.json b/data/models/bunnycore_phi-4-model-stock.json deleted file mode 100644 index ac54a92079f770ab12335d7f253d3b74b0815ea3..0000000000000000000000000000000000000000 --- a/data/models/bunnycore_phi-4-model-stock.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Phi-4-Model-Stock", - "id": "bunnycore/Phi-4-Model-Stock", - "developer": "bunnycore", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bunnycore_Phi-4-Model-Stock/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6879 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.689 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4298 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3549 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4441 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5368 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bunnycore_phi-4-reasoningrp.json b/data/models/bunnycore_phi-4-reasoningrp.json deleted file mode 100644 index d1d9f44bce4fdc0aced9ce830e7f0b765c35062b..0000000000000000000000000000000000000000 --- a/data/models/bunnycore_phi-4-reasoningrp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Phi-4-ReasoningRP", - "id": "bunnycore/Phi-4-ReasoningRP", - "developer": "bunnycore", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bunnycore_Phi-4-ReasoningRP/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6736 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6922 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4569 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.344 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4491 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5421 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bunnycore_phi-4-rp-v0.json b/data/models/bunnycore_phi-4-rp-v0.json deleted file mode 100644 index 28351e4d1c23ac7d31083a755061fa6df23d66bb..0000000000000000000000000000000000000000 --- a/data/models/bunnycore_phi-4-rp-v0.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Phi-4-RP-v0", - "id": "bunnycore/Phi-4-RP-v0", - "developer": "bunnycore", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Phi3ForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bunnycore_Phi-4-RP-v0/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6827 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6856 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3316 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3523 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4141 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5364 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bunnycore_phi-4-rr-shoup.json b/data/models/bunnycore_phi-4-rr-shoup.json deleted file mode 100644 index 1f288571167c7887106cf5fcba53ea0c26d76a27..0000000000000000000000000000000000000000 --- a/data/models/bunnycore_phi-4-rr-shoup.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Phi-4-RR-Shoup", - "id": "bunnycore/Phi-4-RR-Shoup", - "developer": "bunnycore", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bunnycore_Phi-4-RR-Shoup/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6587 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6947 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4992 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3372 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.444 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5429 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bunnycore_phi-4-rstock-v0.1.json b/data/models/bunnycore_phi-4-rstock-v0.1.json deleted file mode 100644 index 0528f235f63614a8c7a2f971c1581b64b63ddcc9..0000000000000000000000000000000000000000 --- a/data/models/bunnycore_phi-4-rstock-v0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Phi-4-RStock-v0.1", - "id": "bunnycore/Phi-4-RStock-v0.1", - "developer": "bunnycore", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bunnycore_Phi-4-RStock-v0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7019 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6928 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.395 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3649 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4584 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5401 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bunnycore_phi-4-sce-exp-v0.1.json b/data/models/bunnycore_phi-4-sce-exp-v0.1.json deleted file mode 100644 index 98d6d81a7ddff7684a82c8081dbdf77b0b6898f5..0000000000000000000000000000000000000000 --- a/data/models/bunnycore_phi-4-sce-exp-v0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Phi-4-Sce-exp-v0.1", - "id": "bunnycore/Phi-4-Sce-exp-v0.1", - "developer": "bunnycore", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bunnycore_Phi-4-Sce-exp-v0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6595 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6943 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.503 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3356 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4441 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5423 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bunnycore_phi-4-stock-ex.json b/data/models/bunnycore_phi-4-stock-ex.json deleted file mode 100644 index 04acf18747430e1dbf5172bcb271b56f52f36301..0000000000000000000000000000000000000000 --- a/data/models/bunnycore_phi-4-stock-ex.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Phi-4-Stock-Ex", - "id": "bunnycore/Phi-4-Stock-Ex", - "developer": "bunnycore", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bunnycore_Phi-4-Stock-Ex/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6575 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6864 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4086 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3507 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4624 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5375 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bunnycore_phi-4-stock-rp.json b/data/models/bunnycore_phi-4-stock-rp.json deleted file mode 100644 index 17ebdb91c9dc13cafdeeffbe6368f6f35ad24790..0000000000000000000000000000000000000000 --- a/data/models/bunnycore_phi-4-stock-rp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Phi-4-Stock-RP", - "id": "bunnycore/Phi-4-Stock-RP", - "developer": "bunnycore", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bunnycore_Phi-4-Stock-RP/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6399 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.686 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3414 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3582 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4715 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5317 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bunnycore_phi-4-trim-exp1.json b/data/models/bunnycore_phi-4-trim-exp1.json deleted file mode 100644 index 3d7d11ff84e47ee58f9077920775e66cdc962a48..0000000000000000000000000000000000000000 --- a/data/models/bunnycore_phi-4-trim-exp1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Phi-4-Trim-Exp1", - "id": "bunnycore/Phi-4-Trim-Exp1", - "developer": "bunnycore", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "7.503" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bunnycore_Phi-4-Trim-Exp1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1219 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2852 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0053 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.255 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4177 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1147 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bunnycore_phi-seek-4-sce-v1.json b/data/models/bunnycore_phi-seek-4-sce-v1.json deleted file mode 100644 index 088f0aae73b8f80d689905448ae33feaa1de5aff..0000000000000000000000000000000000000000 --- a/data/models/bunnycore_phi-seek-4-sce-v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Phi-Seek-4-Sce-V1", - "id": "bunnycore/Phi-Seek-4-Sce-V1", - "developer": "bunnycore", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bunnycore_Phi-Seek-4-Sce-V1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2935 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6459 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2145 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.276 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3982 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5123 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bunnycore_qandora-2.5-7b-creative.json b/data/models/bunnycore_qandora-2.5-7b-creative.json deleted file mode 100644 index 24b1cf8126187e0d674a6893937685a04026fe16..0000000000000000000000000000000000000000 --- a/data/models/bunnycore_qandora-2.5-7b-creative.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qandora-2.5-7B-Creative", - "id": "bunnycore/Qandora-2.5-7B-Creative", - "developer": "bunnycore", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bunnycore_Qandora-2.5-7B-Creative/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6803 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5542 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3059 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3104 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4212 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.448 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bunnycore_qandoraexp-7b-persona.json b/data/models/bunnycore_qandoraexp-7b-persona.json deleted file mode 100644 index f92843aeadf3bb5f5e6b8a4ab8fdfbca833219ac..0000000000000000000000000000000000000000 --- a/data/models/bunnycore_qandoraexp-7b-persona.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "QandoraExp-7B-Persona", - "id": "bunnycore/QandoraExp-7B-Persona", - "developer": "bunnycore", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bunnycore_QandoraExp-7B-Persona/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6247 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5558 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3104 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3146 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4372 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4407 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bunnycore_qandoraexp-7b-v2.json b/data/models/bunnycore_qandoraexp-7b-v2.json deleted file mode 100644 index c6932933b305543666ec2cc83c1ddf9a3f412222..0000000000000000000000000000000000000000 --- a/data/models/bunnycore_qandoraexp-7b-v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "QandoraExp-7B-v2", - "id": "bunnycore/QandoraExp-7B-v2", - "developer": "bunnycore", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bunnycore_QandoraExp-7B-v2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5607 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5445 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4713 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3029 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4045 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3909 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bunnycore_qandoraexp-7b.json b/data/models/bunnycore_qandoraexp-7b.json deleted file mode 100644 index 0f1dffb6d031a3bca82cdaf153099d5e51416cb8..0000000000000000000000000000000000000000 --- a/data/models/bunnycore_qandoraexp-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "QandoraExp-7B", - "id": "bunnycore/QandoraExp-7B", - "developer": "bunnycore", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bunnycore_QandoraExp-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7509 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5478 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4743 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3104 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4312 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.441 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bunnycore_qwen-2.5-7b-deep-sky-t1.json b/data/models/bunnycore_qwen-2.5-7b-deep-sky-t1.json deleted file mode 100644 index cdddb375c81d9510709ab2d3a59debcda727f28e..0000000000000000000000000000000000000000 --- a/data/models/bunnycore_qwen-2.5-7b-deep-sky-t1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen-2.5-7B-Deep-Sky-T1", - "id": "bunnycore/Qwen-2.5-7B-Deep-Sky-T1", - "developer": "bunnycore", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.613" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bunnycore_Qwen-2.5-7B-Deep-Sky-T1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4208 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.414 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0551 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.281 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4018 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2104 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bunnycore_qwen-2.5-7b-deep-stock-v1.json b/data/models/bunnycore_qwen-2.5-7b-deep-stock-v1.json deleted file mode 100644 index 08e96a427983bafd625c1b9a892b9178d1fe9851..0000000000000000000000000000000000000000 --- a/data/models/bunnycore_qwen-2.5-7b-deep-stock-v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen-2.5-7B-Deep-Stock-v1", - "id": "bunnycore/Qwen-2.5-7B-Deep-Stock-v1", - "developer": "bunnycore", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.613" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bunnycore_Qwen-2.5-7B-Deep-Stock-v1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5695 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5361 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2644 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2777 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4109 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4066 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bunnycore_qwen-2.5-7b-deep-stock-v4.json b/data/models/bunnycore_qwen-2.5-7b-deep-stock-v4.json deleted file mode 100644 index 70121e969810ee874e5fcf2666393aecc49ed52f..0000000000000000000000000000000000000000 --- a/data/models/bunnycore_qwen-2.5-7b-deep-stock-v4.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen-2.5-7B-Deep-Stock-v4", - "id": "bunnycore/Qwen-2.5-7B-Deep-Stock-v4", - "developer": "bunnycore", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.613" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bunnycore_Qwen-2.5-7B-Deep-Stock-v4/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7753 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5453 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4894 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3003 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4127 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4342 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bunnycore_qwen-2.5-7b-deep-stock-v5.json b/data/models/bunnycore_qwen-2.5-7b-deep-stock-v5.json deleted file mode 100644 index 585c01d262e4d397bce464140cba3db5e71312b8..0000000000000000000000000000000000000000 --- a/data/models/bunnycore_qwen-2.5-7b-deep-stock-v5.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen-2.5-7B-Deep-Stock-v5", - "id": "bunnycore/Qwen-2.5-7B-Deep-Stock-v5", - "developer": "bunnycore", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.613" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bunnycore_Qwen-2.5-7B-Deep-Stock-v5/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4509 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4672 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1473 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2701 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3648 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2832 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bunnycore_qwen-2.5-7b-exp-sce.json b/data/models/bunnycore_qwen-2.5-7b-exp-sce.json deleted file mode 100644 index db2ae2862f0c0a02bb21b27c515ce4f041e9a1e5..0000000000000000000000000000000000000000 --- a/data/models/bunnycore_qwen-2.5-7b-exp-sce.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen-2.5-7B-Exp-Sce", - "id": "bunnycore/Qwen-2.5-7B-Exp-Sce", - "developer": "bunnycore", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.613" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bunnycore_Qwen-2.5-7B-Exp-Sce/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7652 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5506 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3255 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2987 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.443 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4259 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bunnycore_qwen-2.5-7b-r1-stock.json b/data/models/bunnycore_qwen-2.5-7b-r1-stock.json deleted file mode 100644 index 656f9191666942e8a2466799bd801858956285e4..0000000000000000000000000000000000000000 --- a/data/models/bunnycore_qwen-2.5-7b-r1-stock.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen-2.5-7B-R1-Stock", - "id": "bunnycore/Qwen-2.5-7B-R1-Stock", - "developer": "bunnycore", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.613" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bunnycore_Qwen-2.5-7B-R1-Stock/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7573 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5393 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5008 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2995 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3994 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4294 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bunnycore_qwen-2.5-7b-s1k.json b/data/models/bunnycore_qwen-2.5-7b-s1k.json deleted file mode 100644 index 0b256952fca99787f4a57081fb999b164bf3e1a6..0000000000000000000000000000000000000000 --- a/data/models/bunnycore_qwen-2.5-7b-s1k.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen-2.5-7b-S1k", - "id": "bunnycore/Qwen-2.5-7b-S1k", - "developer": "bunnycore", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.613" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bunnycore_Qwen-2.5-7b-S1k/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7162 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5563 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4781 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2844 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4071 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4382 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bunnycore_qwen-2.5-7b-stock-deep-bespoke.json b/data/models/bunnycore_qwen-2.5-7b-stock-deep-bespoke.json deleted file mode 100644 index e09c473149373dcb2b84d94c15005530dfe01d29..0000000000000000000000000000000000000000 --- a/data/models/bunnycore_qwen-2.5-7b-stock-deep-bespoke.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen-2.5-7B-Stock-Deep-Bespoke", - "id": "bunnycore/Qwen-2.5-7B-Stock-Deep-Bespoke", - "developer": "bunnycore", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.613" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bunnycore_Qwen-2.5-7B-Stock-Deep-Bespoke/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5206 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.492 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1888 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.281 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4068 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.358 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bunnycore_qwen2.5-1.5b-model-stock.json b/data/models/bunnycore_qwen2.5-1.5b-model-stock.json deleted file mode 100644 index 64d9c714e6ccd3058582f5206c1183101b330283..0000000000000000000000000000000000000000 --- a/data/models/bunnycore_qwen2.5-1.5b-model-stock.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-1.5B-Model-Stock", - "id": "bunnycore/Qwen2.5-1.5B-Model-Stock", - "developer": "bunnycore", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.776" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bunnycore_Qwen2.5-1.5B-Model-Stock/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1829 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2874 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2592 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3674 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.11 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bunnycore_qwen2.5-3b-model-stock-v2.json b/data/models/bunnycore_qwen2.5-3b-model-stock-v2.json deleted file mode 100644 index a07bcadc21a6c90a92b4548946e5ee50a99b8d21..0000000000000000000000000000000000000000 --- a/data/models/bunnycore_qwen2.5-3b-model-stock-v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-3B-Model-Stock-v2", - "id": "bunnycore/Qwen2.5-3B-Model-Stock-v2", - "developer": "bunnycore", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.396" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bunnycore_Qwen2.5-3B-Model-Stock-v2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.649 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4677 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3867 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2869 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3915 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.327 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bunnycore_qwen2.5-3b-model-stock-v3.1.json b/data/models/bunnycore_qwen2.5-3b-model-stock-v3.1.json deleted file mode 100644 index 2ee7eb9eaf6c5306ab1ad35f864aa1579a2b1471..0000000000000000000000000000000000000000 --- a/data/models/bunnycore_qwen2.5-3b-model-stock-v3.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-3B-Model-Stock-v3.1", - "id": "bunnycore/Qwen2.5-3B-Model-Stock-v3.1", - "developer": "bunnycore", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.396" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bunnycore_Qwen2.5-3B-Model-Stock-v3.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6481 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4737 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3897 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2844 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3968 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.329 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bunnycore_qwen2.5-3b-model-stock-v3.2.json b/data/models/bunnycore_qwen2.5-3b-model-stock-v3.2.json deleted file mode 100644 index 5d7a8c65273a54e1c02617ebfa91fb0edb61bcb0..0000000000000000000000000000000000000000 --- a/data/models/bunnycore_qwen2.5-3b-model-stock-v3.2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-3B-Model-Stock-v3.2", - "id": "bunnycore/Qwen2.5-3B-Model-Stock-v3.2", - "developer": "bunnycore", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.396" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bunnycore_Qwen2.5-3B-Model-Stock-v3.2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6353 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4727 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3754 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2836 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3928 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3294 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bunnycore_qwen2.5-3b-model-stock-v4.1.json b/data/models/bunnycore_qwen2.5-3b-model-stock-v4.1.json deleted file mode 100644 index 9d13e2c923c76d87c7c5e14238502e7b9224a77b..0000000000000000000000000000000000000000 --- a/data/models/bunnycore_qwen2.5-3b-model-stock-v4.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-3B-Model-Stock-v4.1", - "id": "bunnycore/Qwen2.5-3B-Model-Stock-v4.1", - "developer": "bunnycore", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.396" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bunnycore_Qwen2.5-3B-Model-Stock-v4.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6381 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.482 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3769 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2794 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3941 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3387 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bunnycore_qwen2.5-3b-model-stock.json b/data/models/bunnycore_qwen2.5-3b-model-stock.json deleted file mode 100644 index 44b6cb780f8c2b21cac04dbd6e1d028524da8d32..0000000000000000000000000000000000000000 --- a/data/models/bunnycore_qwen2.5-3b-model-stock.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-3B-Model-Stock", - "id": "bunnycore/Qwen2.5-3B-Model-Stock", - "developer": "bunnycore", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.396" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bunnycore_Qwen2.5-3B-Model-Stock/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6381 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4712 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3799 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2886 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3942 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.325 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bunnycore_qwen2.5-3b-rp-mix.json b/data/models/bunnycore_qwen2.5-3b-rp-mix.json deleted file mode 100644 index 1e91ccaa556aa62ac5b6b3d38d7090d68d4cc1bc..0000000000000000000000000000000000000000 --- a/data/models/bunnycore_qwen2.5-3b-rp-mix.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-3B-RP-Mix", - "id": "bunnycore/Qwen2.5-3B-RP-Mix", - "developer": "bunnycore", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.397" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bunnycore_Qwen2.5-3B-RP-Mix/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5721 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4894 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2153 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2735 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4284 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3728 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bunnycore_qwen2.5-3b-rp-thinker-v2.json b/data/models/bunnycore_qwen2.5-3b-rp-thinker-v2.json deleted file mode 100644 index 1809dc6c0a49487d13c6d133cde63066b984762b..0000000000000000000000000000000000000000 --- a/data/models/bunnycore_qwen2.5-3b-rp-thinker-v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-3B-RP-Thinker-V2", - "id": "bunnycore/Qwen2.5-3B-RP-Thinker-V2", - "developer": "bunnycore", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.397" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bunnycore_Qwen2.5-3B-RP-Thinker-V2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.642 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4678 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3829 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2852 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3981 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3271 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bunnycore_qwen2.5-3b-rp-thinker.json b/data/models/bunnycore_qwen2.5-3b-rp-thinker.json deleted file mode 100644 index d9e5546a6e8aa219aa5bde7de4b50f6fdeea0fed..0000000000000000000000000000000000000000 --- a/data/models/bunnycore_qwen2.5-3b-rp-thinker.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-3B-RP-Thinker", - "id": "bunnycore/Qwen2.5-3B-RP-Thinker", - "developer": "bunnycore", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.397" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bunnycore_Qwen2.5-3B-RP-Thinker/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5894 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4164 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3353 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2643 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3287 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.315 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bunnycore_qwen2.5-7b-cyberrombos.json b/data/models/bunnycore_qwen2.5-7b-cyberrombos.json deleted file mode 100644 index c82e799687d5e6c85b81b47b251f104a7379aa3e..0000000000000000000000000000000000000000 --- a/data/models/bunnycore_qwen2.5-7b-cyberrombos.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-7B-CyberRombos", - "id": "bunnycore/Qwen2.5-7B-CyberRombos", - "developer": "bunnycore", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bunnycore_Qwen2.5-7B-CyberRombos/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7518 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5465 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4962 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3045 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4125 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4391 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bunnycore_qwen2.5-7b-fuse-exp.json b/data/models/bunnycore_qwen2.5-7b-fuse-exp.json deleted file mode 100644 index 5ac3e323210423d9d6279bddc157583fc02d4f0e..0000000000000000000000000000000000000000 --- a/data/models/bunnycore_qwen2.5-7b-fuse-exp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-7B-Fuse-Exp", - "id": "bunnycore/Qwen2.5-7B-Fuse-Exp", - "developer": "bunnycore", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bunnycore_Qwen2.5-7B-Fuse-Exp/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5469 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5109 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3142 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.276 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4573 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3309 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bunnycore_qwen2.5-7b-instruct-fusion.json b/data/models/bunnycore_qwen2.5-7b-instruct-fusion.json deleted file mode 100644 index 8d63d72daace64519b6d8d7305642b4e2aed10d2..0000000000000000000000000000000000000000 --- a/data/models/bunnycore_qwen2.5-7b-instruct-fusion.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-7B-Instruct-Fusion", - "id": "bunnycore/Qwen2.5-7B-Instruct-Fusion", - "developer": "bunnycore", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bunnycore_Qwen2.5-7B-Instruct-Fusion/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6962 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5492 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3406 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3045 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4297 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4467 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bunnycore_qwen2.5-7b-instruct-merge-stock-v0.1.json b/data/models/bunnycore_qwen2.5-7b-instruct-merge-stock-v0.1.json deleted file mode 100644 index 5d99ec8a5ebd95f96e72dba2a766af223c5f3423..0000000000000000000000000000000000000000 --- a/data/models/bunnycore_qwen2.5-7b-instruct-merge-stock-v0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-7B-Instruct-Merge-Stock-v0.1", - "id": "bunnycore/Qwen2.5-7B-Instruct-Merge-Stock-v0.1", - "developer": "bunnycore", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.613" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bunnycore_Qwen2.5-7B-Instruct-Merge-Stock-v0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7509 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5529 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4894 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3037 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4231 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4383 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bunnycore_qwen2.5-7b-mixstock-sce-v0.3.json b/data/models/bunnycore_qwen2.5-7b-mixstock-sce-v0.3.json deleted file mode 100644 index b9e96e6a95a082b46557a3889fcb4f2845c439fa..0000000000000000000000000000000000000000 --- a/data/models/bunnycore_qwen2.5-7b-mixstock-sce-v0.3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-7B-MixStock-Sce-V0.3", - "id": "bunnycore/Qwen2.5-7B-MixStock-Sce-V0.3", - "developer": "bunnycore", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.613" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bunnycore_Qwen2.5-7B-MixStock-Sce-V0.3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.212 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3479 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2576 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2576 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3714 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1779 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bunnycore_qwen2.5-7b-mixstock-v0.1.json b/data/models/bunnycore_qwen2.5-7b-mixstock-v0.1.json deleted file mode 100644 index 29ef5ee0adcac6fc1251441af87ddf6c50cb2c9d..0000000000000000000000000000000000000000 --- a/data/models/bunnycore_qwen2.5-7b-mixstock-v0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-7B-MixStock-V0.1", - "id": "bunnycore/Qwen2.5-7B-MixStock-V0.1", - "developer": "bunnycore", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.613" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bunnycore_Qwen2.5-7B-MixStock-V0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7673 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5479 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3172 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3003 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4416 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4256 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bunnycore_qwen2.5-7b-r1-bespoke-stock.json b/data/models/bunnycore_qwen2.5-7b-r1-bespoke-stock.json deleted file mode 100644 index 6592b7de076ed946ac93a1687a24bd7b8e47fb91..0000000000000000000000000000000000000000 --- a/data/models/bunnycore_qwen2.5-7b-r1-bespoke-stock.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-7B-R1-Bespoke-Stock", - "id": "bunnycore/Qwen2.5-7B-R1-Bespoke-Stock", - "developer": "bunnycore", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.613" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bunnycore_Qwen2.5-7B-R1-Bespoke-Stock/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3726 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4822 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2047 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2785 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3926 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3472 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bunnycore_qwen2.5-7b-r1-bespoke-task.json b/data/models/bunnycore_qwen2.5-7b-r1-bespoke-task.json deleted file mode 100644 index 18f2cb0335bd8a2b927ac0aee1ad19e0255403cd..0000000000000000000000000000000000000000 --- a/data/models/bunnycore_qwen2.5-7b-r1-bespoke-task.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-7B-R1-Bespoke-Task", - "id": "bunnycore/Qwen2.5-7B-R1-Bespoke-Task", - "developer": "bunnycore", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.613" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bunnycore_Qwen2.5-7B-R1-Bespoke-Task/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3787 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.415 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1782 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2534 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3569 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2688 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bunnycore_qwen2.5-7b-rrp-1m-thinker.json b/data/models/bunnycore_qwen2.5-7b-rrp-1m-thinker.json deleted file mode 100644 index e920a2fe290376d24317fc51327c0cfa745db7c1..0000000000000000000000000000000000000000 --- a/data/models/bunnycore_qwen2.5-7b-rrp-1m-thinker.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-7B-RRP-1M-Thinker", - "id": "bunnycore/Qwen2.5-7B-RRP-1M-Thinker", - "developer": "bunnycore", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.613" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bunnycore_Qwen2.5-7B-RRP-1M-Thinker/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2308 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3482 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2719 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2576 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3767 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1769 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bunnycore_qwen2.5-7b-rrp-1m.json b/data/models/bunnycore_qwen2.5-7b-rrp-1m.json deleted file mode 100644 index c41bf78e297e3861e82e59143e0ceb9e0d69c36e..0000000000000000000000000000000000000000 --- a/data/models/bunnycore_qwen2.5-7b-rrp-1m.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-7B-RRP-1M", - "id": "bunnycore/Qwen2.5-7B-RRP-1M", - "developer": "bunnycore", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.613" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bunnycore_Qwen2.5-7B-RRP-1M/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7481 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5452 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3248 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3029 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4483 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4266 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bunnycore_qwen2.5-7b-rrp-id.json b/data/models/bunnycore_qwen2.5-7b-rrp-id.json deleted file mode 100644 index 5c7713eb375f0d2dd1f5ba2a24a23a0c87a5a724..0000000000000000000000000000000000000000 --- a/data/models/bunnycore_qwen2.5-7b-rrp-id.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-7B-RRP-ID", - "id": "bunnycore/Qwen2.5-7B-RRP-ID", - "developer": "bunnycore", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bunnycore_Qwen2.5-7B-RRP-ID/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7473 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.548 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4864 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2827 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.418 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4387 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bunnycore_qwen2.5-7b-sky-r1-mini.json b/data/models/bunnycore_qwen2.5-7b-sky-r1-mini.json deleted file mode 100644 index 62eed60cdedf617bb7d8fd482a5bddc3ccac3669..0000000000000000000000000000000000000000 --- a/data/models/bunnycore_qwen2.5-7b-sky-r1-mini.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-7B-Sky-R1-Mini", - "id": "bunnycore/Qwen2.5-7B-Sky-R1-Mini", - "developer": "bunnycore", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bunnycore_Qwen2.5-7B-Sky-R1-Mini/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2305 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3503 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0295 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2894 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3448 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1253 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bunnycore_qwenmosaic-7b.json b/data/models/bunnycore_qwenmosaic-7b.json deleted file mode 100644 index beb2abb6f57e45f699017fad31a9b20bab7fd86f..0000000000000000000000000000000000000000 --- a/data/models/bunnycore_qwenmosaic-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "QwenMosaic-7B", - "id": "bunnycore/QwenMosaic-7B", - "developer": "bunnycore", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bunnycore_QwenMosaic-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5819 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5564 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4441 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2609 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4164 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.431 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bunnycore_qwqen-3b-lcot-r1.json b/data/models/bunnycore_qwqen-3b-lcot-r1.json deleted file mode 100644 index 25b7749dea95a912d05628c8f7e22cfdecee681a..0000000000000000000000000000000000000000 --- a/data/models/bunnycore_qwqen-3b-lcot-r1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "QwQen-3B-LCoT-R1", - "id": "bunnycore/QwQen-3B-LCoT-R1", - "developer": "bunnycore", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.085" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bunnycore_QwQen-3B-LCoT-R1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5342 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4799 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3353 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2617 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4138 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3723 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bunnycore_qwqen-3b-lcot.json b/data/models/bunnycore_qwqen-3b-lcot.json deleted file mode 100644 index 11c77bdb1b132ae5061598af50d8b8202940c4cb..0000000000000000000000000000000000000000 --- a/data/models/bunnycore_qwqen-3b-lcot.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "QwQen-3B-LCoT", - "id": "bunnycore/QwQen-3B-LCoT", - "developer": "bunnycore", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.397" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bunnycore_QwQen-3B-LCoT/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6025 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4899 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3618 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2668 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4178 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3699 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bunnycore_smol-llama-3.2-3b.json b/data/models/bunnycore_smol-llama-3.2-3b.json deleted file mode 100644 index 4c8fb7d5d9c4fd4b015a95a1916879c9bec9cc1b..0000000000000000000000000000000000000000 --- a/data/models/bunnycore_smol-llama-3.2-3b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Smol-Llama-3.2-3B", - "id": "bunnycore/Smol-Llama-3.2-3B", - "developer": "bunnycore", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.607" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bunnycore_Smol-Llama-3.2-3B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6679 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4539 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1382 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2768 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.346 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3228 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bunnycore_smollm2-1.7-persona.json b/data/models/bunnycore_smollm2-1.7-persona.json deleted file mode 100644 index 0314cd74e4bc7af3f48352f73c00df356b559428..0000000000000000000000000000000000000000 --- a/data/models/bunnycore_smollm2-1.7-persona.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SmolLM2-1.7-Persona", - "id": "bunnycore/SmolLM2-1.7-Persona", - "developer": "bunnycore", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.711" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bunnycore_SmolLM2-1.7-Persona/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5465 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3623 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0566 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2634 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3341 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1974 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bunnycore_smollm2-1.7b-roleplay-lora.json b/data/models/bunnycore_smollm2-1.7b-roleplay-lora.json deleted file mode 100644 index e795134f1d161038f3986c4ab90efd8ffc65e793..0000000000000000000000000000000000000000 --- a/data/models/bunnycore_smollm2-1.7b-roleplay-lora.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SmolLM2-1.7B-roleplay-lora", - "id": "bunnycore/SmolLM2-1.7B-roleplay-lora", - "developer": "bunnycore", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "?", - "params_billions": "3.423" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bunnycore_SmolLM2-1.7B-roleplay-lora/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5382 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.361 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0529 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2752 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3395 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1966 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bunnycore_tulu-3.1-8b-supernova.json b/data/models/bunnycore_tulu-3.1-8b-supernova.json deleted file mode 100644 index eefa77c84bf01216a47f60e78c5f0b63a89a0b54..0000000000000000000000000000000000000000 --- a/data/models/bunnycore_tulu-3.1-8b-supernova.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Tulu-3.1-8B-SuperNova", - "id": "bunnycore/Tulu-3.1-8B-SuperNova", - "developer": "bunnycore", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/bunnycore_Tulu-3.1-8B-SuperNova/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8194 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5254 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2462 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.302 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3935 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3814 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/byroneverson_mistral-small-instruct-2409-abliterated.json b/data/models/byroneverson_mistral-small-instruct-2409-abliterated.json deleted file mode 100644 index 1af318f6508b2d30a266b175277c103a9c298f84..0000000000000000000000000000000000000000 --- a/data/models/byroneverson_mistral-small-instruct-2409-abliterated.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mistral-Small-Instruct-2409-abliterated", - "id": "byroneverson/Mistral-Small-Instruct-2409-abliterated", - "developer": "byroneverson", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "22.247" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/byroneverson_Mistral-Small-Instruct-2409-abliterated/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6971 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5238 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2477 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3331 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3697 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3923 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/byroneverson_yi-1.5-9b-chat-16k-abliterated.json b/data/models/byroneverson_yi-1.5-9b-chat-16k-abliterated.json deleted file mode 100644 index 917f0f44586609f6cbb34b6d10a3163219d955f6..0000000000000000000000000000000000000000 --- a/data/models/byroneverson_yi-1.5-9b-chat-16k-abliterated.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Yi-1.5-9B-Chat-16K-abliterated", - "id": "byroneverson/Yi-1.5-9B-Chat-16K-abliterated", - "developer": "byroneverson", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.829" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/byroneverson_Yi-1.5-9B-Chat-16K-abliterated/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5528 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5282 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1412 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3129 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4734 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3823 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/byroneverson_yi-1.5-9b-chat-abliterated.json b/data/models/byroneverson_yi-1.5-9b-chat-abliterated.json deleted file mode 100644 index fb5606f7592e0476b1a09c9c494dee8a0094f930..0000000000000000000000000000000000000000 --- a/data/models/byroneverson_yi-1.5-9b-chat-abliterated.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Yi-1.5-9B-Chat-abliterated", - "id": "byroneverson/Yi-1.5-9B-Chat-abliterated", - "developer": "byroneverson", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.829" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/byroneverson_Yi-1.5-9B-Chat-abliterated/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5723 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5401 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1662 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2919 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4389 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3715 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/bytedance_doubao-seed-1-6-thinking-250615.json b/data/models/bytedance_doubao-seed-1-6-thinking-250615.json deleted file mode 100644 index 6e8cebfab59bcf3391fedf948a39be920040719d..0000000000000000000000000000000000000000 --- a/data/models/bytedance_doubao-seed-1-6-thinking-250615.json +++ /dev/null @@ -1,89 +0,0 @@ -{ - "model_info": { - "name": "doubao-seed-1-6-thinking-250615", - "developer": "ByteDance", - "inference_platform": "ark", - "id": "bytedance/doubao-seed-1-6-thinking-250615" - }, - "evaluations": [ - { - "evaluation_id": "livecodebenchpro/doubao-seed-1-6-thinking-250615/1760492095.8105888", - "retrieved_timestamp": "1760492095.8105888", - "source_metadata": { - "source_organization_name": "New York University, Princeton University, University of California San Diego, University of Washington and Canyon Crest Academy", - "evaluator_relationship": "third_party", - "source_name": "Live Code Bench Pro", - "source_type": "documentation" - }, - "eval_library": { - "name": "unknown", - "version": "unknown" - }, - "benchmark": "livecodebenchpro", - "evaluation_results": [ - { - "evaluation_name": "Hard Problems", - "metric_config": { - "evaluation_description": "Pass@1 on Hard Problems", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.0 - }, - "source_data": { - "dataset_name": "Hard Problems", - "source_type": "url", - "url": [ - "https://webhook.cp-bench.orzzh.com/leaderboard/llm/difficulty?difficulty=hard&benchmark_mode=live" - ] - } - }, - { - "evaluation_name": "Medium Problems", - "metric_config": { - "evaluation_description": "Pass@1 on Medium Problems", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.07042253521126761 - }, - "source_data": { - "dataset_name": "Medium Problems", - "source_type": "url", - "url": [ - "https://webhook.cp-bench.orzzh.com/leaderboard/llm/difficulty?difficulty=medium&benchmark_mode=live" - ] - } - }, - { - "evaluation_name": "Easy Problems", - "metric_config": { - "evaluation_description": "Pass@1 on Easy Problems", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.5774647887323944 - }, - "source_data": { - "dataset_name": "Easy Problems", - "source_type": "url", - "url": [ - "https://webhook.cp-bench.orzzh.com/leaderboard/llm/difficulty?difficulty=easy&benchmark_mode=live" - ] - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/c10x_longthinker.json b/data/models/c10x_longthinker.json deleted file mode 100644 index 120030b2966572856f402f7f11ae1c3e061a1fcd..0000000000000000000000000000000000000000 --- a/data/models/c10x_longthinker.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "longthinker", - "id": "c10x/longthinker", - "developer": "c10x", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/c10x_longthinker/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3609 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4927 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2319 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2643 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.391 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3527 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/c10x_q-pluse.json b/data/models/c10x_q-pluse.json deleted file mode 100644 index 357fb770e538082c4efe785f771e4cd565fbed4e..0000000000000000000000000000000000000000 --- a/data/models/c10x_q-pluse.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Q-Pluse", - "id": "c10x/Q-Pluse", - "developer": "c10x", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/c10x_Q-Pluse/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1123 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2875 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2466 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3938 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1135 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/carrotai_llama-3.2-rabbit-ko-3b-instruct-2412.json b/data/models/carrotai_llama-3.2-rabbit-ko-3b-instruct-2412.json deleted file mode 100644 index f32cc055fde8b1c11e5be88c386f6e6f4bcb73b2..0000000000000000000000000000000000000000 --- a/data/models/carrotai_llama-3.2-rabbit-ko-3b-instruct-2412.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.2-Rabbit-Ko-3B-Instruct-2412", - "id": "CarrotAI/Llama-3.2-Rabbit-Ko-3B-Instruct-2412", - "developer": "CarrotAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/CarrotAI_Llama-3.2-Rabbit-Ko-3B-Instruct-2412/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4782 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4358 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.176 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2928 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3872 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3134 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/carrotai_llama-3.2-rabbit-ko-3b-instruct.json b/data/models/carrotai_llama-3.2-rabbit-ko-3b-instruct.json deleted file mode 100644 index 804bec2dd405d89bd925030f7127fd5869d0d656..0000000000000000000000000000000000000000 --- a/data/models/carrotai_llama-3.2-rabbit-ko-3b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.2-Rabbit-Ko-3B-Instruct", - "id": "CarrotAI/Llama-3.2-Rabbit-Ko-3B-Instruct", - "developer": "CarrotAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/CarrotAI_Llama-3.2-Rabbit-Ko-3B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7199 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4427 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2054 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.271 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3649 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2822 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/carsenk_flippa-v6.json b/data/models/carsenk_flippa-v6.json deleted file mode 100644 index 9a12dafc52e8dae3be24857f9c10185df5710fe1..0000000000000000000000000000000000000000 --- a/data/models/carsenk_flippa-v6.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "flippa-v6", - "id": "carsenk/flippa-v6", - "developer": "carsenk", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "?", - "params_billions": "16.061" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/carsenk_flippa-v6/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3439 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5047 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1405 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2928 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4089 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3668 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/carsenk_phi3.5_mini_exp_825_uncensored.json b/data/models/carsenk_phi3.5_mini_exp_825_uncensored.json deleted file mode 100644 index b061a4a278096fe213cf23c23169ddf27d2513bc..0000000000000000000000000000000000000000 --- a/data/models/carsenk_phi3.5_mini_exp_825_uncensored.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "phi3.5_mini_exp_825_uncensored", - "id": "carsenk/phi3.5_mini_exp_825_uncensored", - "developer": "carsenk", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.821" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/carsenk_phi3.5_mini_exp_825_uncensored/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1364 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2965 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0106 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2492 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3644 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1175 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/casual-autopsy_l3-umbral-mind-rp-v2.0-8b.json b/data/models/casual-autopsy_l3-umbral-mind-rp-v2.0-8b.json deleted file mode 100644 index 2409d7488f3ec996cf54703261c8e8ea08eaf0ca..0000000000000000000000000000000000000000 --- a/data/models/casual-autopsy_l3-umbral-mind-rp-v2.0-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "L3-Umbral-Mind-RP-v2.0-8B", - "id": "Casual-Autopsy/L3-Umbral-Mind-RP-v2.0-8B", - "developer": "Casual-Autopsy", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Casual-Autopsy_L3-Umbral-Mind-RP-v2.0-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7123 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5262 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1095 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2869 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3687 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3723 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cat-searcher_gemma-2-9b-it-sppo-iter-1-evol-1.json b/data/models/cat-searcher_gemma-2-9b-it-sppo-iter-1-evol-1.json deleted file mode 100644 index daf38c760afa60e0ccb6e37c81a72076a864cd9c..0000000000000000000000000000000000000000 --- a/data/models/cat-searcher_gemma-2-9b-it-sppo-iter-1-evol-1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "gemma-2-9b-it-sppo-iter-1-evol-1", - "id": "cat-searcher/gemma-2-9b-it-sppo-iter-1-evol-1", - "developer": "cat-searcher", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "9.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/cat-searcher_gemma-2-9b-it-sppo-iter-1-evol-1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2942 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5939 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0853 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3406 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3926 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.38 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cat-searcher_gemma-2-9b-it-sppo-iter-1.json b/data/models/cat-searcher_gemma-2-9b-it-sppo-iter-1.json deleted file mode 100644 index de8dedb4e0369a098e075979ee9cacc02ed2ea26..0000000000000000000000000000000000000000 --- a/data/models/cat-searcher_gemma-2-9b-it-sppo-iter-1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "gemma-2-9b-it-sppo-iter-1", - "id": "cat-searcher/gemma-2-9b-it-sppo-iter-1", - "developer": "cat-searcher", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "9.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/cat-searcher_gemma-2-9b-it-sppo-iter-1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3015 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5972 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0831 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3448 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3927 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3854 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/causallm_14b.json b/data/models/causallm_14b.json deleted file mode 100644 index bae61a3f71ed3019842c00c4182165df241373d5..0000000000000000000000000000000000000000 --- a/data/models/causallm_14b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "14B", - "id": "CausalLM/14B", - "developer": "CausalLM", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/CausalLM_14B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2788 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.47 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0755 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3029 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4155 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3221 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/causallm_34b-beta.json b/data/models/causallm_34b-beta.json deleted file mode 100644 index d116120c185a0335ddc42532cf087ecc6eee97d5..0000000000000000000000000000000000000000 --- a/data/models/causallm_34b-beta.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "34b-beta", - "id": "CausalLM/34b-beta", - "developer": "CausalLM", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "34.389" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/CausalLM_34b-beta/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3043 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5591 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0483 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3465 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3749 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5325 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/causallm_preview-1-hf.json b/data/models/causallm_preview-1-hf.json deleted file mode 100644 index cf387e22c954fa3db216d904bc6b4393bac93ef6..0000000000000000000000000000000000000000 --- a/data/models/causallm_preview-1-hf.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "preview-1-hf", - "id": "CausalLM/preview-1-hf", - "developer": "CausalLM", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "GlmForCausalLM", - "params_billions": "9.543" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/CausalLM_preview-1-hf/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5559 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3615 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0302 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2617 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3422 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3597 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cckm_tinymistral_950m.json b/data/models/cckm_tinymistral_950m.json deleted file mode 100644 index 17a77bbf5b8668e48ffd872693b7213f4ad23a0d..0000000000000000000000000000000000000000 --- a/data/models/cckm_tinymistral_950m.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "tinymistral_950m", - "id": "cckm/tinymistral_950m", - "developer": "cckm", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "0.955" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/cckm_tinymistral_950m/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2395 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2969 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0053 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2601 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3554 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1096 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cgato_thesalt-l3-8b-v0.3.2.json b/data/models/cgato_thesalt-l3-8b-v0.3.2.json deleted file mode 100644 index 9298fa86badd073565c5753bfb4df4a1208ca8e6..0000000000000000000000000000000000000000 --- a/data/models/cgato_thesalt-l3-8b-v0.3.2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "TheSalt-L3-8b-v0.3.2", - "id": "cgato/TheSalt-L3-8b-v0.3.2", - "developer": "cgato", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/cgato_TheSalt-L3-8b-v0.3.2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2705 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2968 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0476 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2659 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3896 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1139 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/changgil_k2s3-14b-v0.2.json b/data/models/changgil_k2s3-14b-v0.2.json deleted file mode 100644 index 73e344d9e7bef7e4cb799ab59ae7c6a6e816a78a..0000000000000000000000000000000000000000 --- a/data/models/changgil_k2s3-14b-v0.2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "K2S3-14b-v0.2", - "id": "Changgil/K2S3-14b-v0.2", - "developer": "Changgil", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "14.352" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Changgil_K2S3-14b-v0.2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3243 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4613 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0574 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.281 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3923 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2644 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/changgil_k2s3-v0.1.json b/data/models/changgil_k2s3-v0.1.json deleted file mode 100644 index e1e7c0369286747162e1472e877d6f9ca906953f..0000000000000000000000000000000000000000 --- a/data/models/changgil_k2s3-v0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "K2S3-v0.1", - "id": "Changgil/K2S3-v0.1", - "developer": "Changgil", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "14.352" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Changgil_K2S3-v0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3277 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4655 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0461 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2643 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4014 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2562 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/chargoddard_prometheus-2-llama-3-8b.json b/data/models/chargoddard_prometheus-2-llama-3-8b.json deleted file mode 100644 index 2df6e62f58f9d292cf3b086f72d5220ea02c4f9b..0000000000000000000000000000000000000000 --- a/data/models/chargoddard_prometheus-2-llama-3-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "prometheus-2-llama-3-8b", - "id": "chargoddard/prometheus-2-llama-3-8b", - "developer": "chargoddard", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/chargoddard_prometheus-2-llama-3-8b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5289 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4931 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0823 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2727 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3396 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3087 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/chujiezheng_llama-3-instruct-8b-simpo-expo.json b/data/models/chujiezheng_llama-3-instruct-8b-simpo-expo.json deleted file mode 100644 index 71fde1b20c2bbe411c531cd44c8d71c7a0d83ce2..0000000000000000000000000000000000000000 --- a/data/models/chujiezheng_llama-3-instruct-8b-simpo-expo.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-Instruct-8B-SimPO-ExPO", - "id": "chujiezheng/Llama-3-Instruct-8B-SimPO-ExPO", - "developer": "chujiezheng", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/chujiezheng_Llama-3-Instruct-8B-SimPO-ExPO/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6434 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4765 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0702 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2869 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.392 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3401 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/chujiezheng_mistral7b-pairrm-sppo-expo.json b/data/models/chujiezheng_mistral7b-pairrm-sppo-expo.json deleted file mode 100644 index 4c0290cc6ef04188403eaeaedbb9c15cfab8a874..0000000000000000000000000000000000000000 --- a/data/models/chujiezheng_mistral7b-pairrm-sppo-expo.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mistral7B-PairRM-SPPO-ExPO", - "id": "chujiezheng/Mistral7B-PairRM-SPPO-ExPO", - "developer": "chujiezheng", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/chujiezheng_Mistral7B-PairRM-SPPO-ExPO/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3673 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3882 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0181 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2768 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4055 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2552 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cir-ams_btrm_qwen2_7b_0613.json b/data/models/cir-ams_btrm_qwen2_7b_0613.json deleted file mode 100644 index 9fb827ce32fb32044e2247d7f86c70d1bc13d414..0000000000000000000000000000000000000000 --- a/data/models/cir-ams_btrm_qwen2_7b_0613.json +++ /dev/null @@ -1,294 +0,0 @@ -{ - "model_info": { - "name": "CIR-AMS/BTRM_Qwen2_7b_0613", - "id": "CIR-AMS/BTRM_Qwen2_7b_0613", - "developer": "CIR-AMS", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/CIR-AMS_BTRM_Qwen2_7b_0613/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5736 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5347 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3563 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6066 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7178 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5737 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6527 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "reward-bench/CIR-AMS_BTRM_Qwen2_7b_0613/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8172 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9749 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5724 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9014 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8775 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7029 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cjvt_gams-1b.json b/data/models/cjvt_gams-1b.json deleted file mode 100644 index 40dcc042bc8e83588eeb90e49f731838dc9cd5e9..0000000000000000000000000000000000000000 --- a/data/models/cjvt_gams-1b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "GaMS-1B", - "id": "cjvt/GaMS-1B", - "developer": "cjvt", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "OPTForCausalLM", - "params_billions": "1.54" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/cjvt_GaMS-1B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1635 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3075 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0136 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2584 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3684 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1149 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/claudioitaly_albacus.json b/data/models/claudioitaly_albacus.json deleted file mode 100644 index 38412d6a0c813f0880680d3355934610f731303b..0000000000000000000000000000000000000000 --- a/data/models/claudioitaly_albacus.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Albacus", - "id": "ClaudioItaly/Albacus", - "developer": "ClaudioItaly", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "8.987" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ClaudioItaly_Albacus/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4667 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5113 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.071 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2718 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4135 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3165 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/claudioitaly_book-gut12b.json b/data/models/claudioitaly_book-gut12b.json deleted file mode 100644 index 5f321179d345a4e036c7c183da9633cbdd3fef25..0000000000000000000000000000000000000000 --- a/data/models/claudioitaly_book-gut12b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Book-Gut12B", - "id": "ClaudioItaly/Book-Gut12B", - "developer": "ClaudioItaly", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ClaudioItaly_Book-Gut12B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3998 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5417 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.102 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.307 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4635 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.367 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/claudioitaly_evolutionstory-7b-v2.2.json b/data/models/claudioitaly_evolutionstory-7b-v2.2.json deleted file mode 100644 index 1375498d23290844159ea44b7faa2b02cfe95b6c..0000000000000000000000000000000000000000 --- a/data/models/claudioitaly_evolutionstory-7b-v2.2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Evolutionstory-7B-v2.2", - "id": "ClaudioItaly/Evolutionstory-7B-v2.2", - "developer": "ClaudioItaly", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ClaudioItaly_Evolutionstory-7B-v2.2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4814 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5108 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.071 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2752 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4135 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3159 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/claudioitaly_intelligence-cod-rag-7b-v3.json b/data/models/claudioitaly_intelligence-cod-rag-7b-v3.json deleted file mode 100644 index 1ae2a0c04bb8f2fca3d45fcc296c6df4aa54e66d..0000000000000000000000000000000000000000 --- a/data/models/claudioitaly_intelligence-cod-rag-7b-v3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "intelligence-cod-rag-7b-v3", - "id": "ClaudioItaly/intelligence-cod-rag-7b-v3", - "developer": "ClaudioItaly", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ClaudioItaly_intelligence-cod-rag-7b-v3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6898 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5366 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3807 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2727 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4153 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4195 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cloudyu_llama-3-70bx2-moe.json b/data/models/cloudyu_llama-3-70bx2-moe.json deleted file mode 100644 index e7cd918922b9b7d0e19de572c91934b3d46a8882..0000000000000000000000000000000000000000 --- a/data/models/cloudyu_llama-3-70bx2-moe.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-70Bx2-MOE", - "id": "cloudyu/Llama-3-70Bx2-MOE", - "developer": "cloudyu", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MixtralForCausalLM", - "params_billions": "126.926" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/cloudyu_Llama-3-70Bx2-MOE/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5482 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6636 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2175 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3935 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4812 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5142 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cloudyu_llama-3.2-3bx4.json b/data/models/cloudyu_llama-3.2-3bx4.json deleted file mode 100644 index 6a915cb16a9aed09d34ea792e2518aefd6e887c0..0000000000000000000000000000000000000000 --- a/data/models/cloudyu_llama-3.2-3bx4.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.2-3Bx4", - "id": "cloudyu/Llama-3.2-3Bx4", - "developer": "cloudyu", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MixtralForCausalLM", - "params_billions": "9.949" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/cloudyu_Llama-3.2-3Bx4/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5069 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4332 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1073 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2777 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3496 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2985 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cloudyu_mixtral_11bx2_moe_19b.json b/data/models/cloudyu_mixtral_11bx2_moe_19b.json deleted file mode 100644 index 50447c715e297b2b7a486c86132ac1d97d17bc3f..0000000000000000000000000000000000000000 --- a/data/models/cloudyu_mixtral_11bx2_moe_19b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mixtral_11Bx2_MoE_19B", - "id": "cloudyu/Mixtral_11Bx2_MoE_19B", - "developer": "cloudyu", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MixtralForCausalLM", - "params_billions": "19.188" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/cloudyu_Mixtral_11Bx2_MoE_19B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3851 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5209 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0672 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2903 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4297 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3311 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cloudyu_mixtral_34bx2_moe_60b.json b/data/models/cloudyu_mixtral_34bx2_moe_60b.json deleted file mode 100644 index a251cfc092e118036f4d8e60d42d3017a3104d52..0000000000000000000000000000000000000000 --- a/data/models/cloudyu_mixtral_34bx2_moe_60b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mixtral_34Bx2_MoE_60B", - "id": "cloudyu/Mixtral_34Bx2_MoE_60B", - "developer": "cloudyu", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MixtralForCausalLM", - "params_billions": "60.814" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/cloudyu_Mixtral_34Bx2_MoE_60B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4538 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.587 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.077 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3381 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4625 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4766 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cloudyu_mixtral_7bx2_moe.json b/data/models/cloudyu_mixtral_7bx2_moe.json deleted file mode 100644 index e8a94f747259bc2fd99398a622b5745ccd767683..0000000000000000000000000000000000000000 --- a/data/models/cloudyu_mixtral_7bx2_moe.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mixtral_7Bx2_MoE", - "id": "cloudyu/Mixtral_7Bx2_MoE", - "developer": "cloudyu", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MixtralForCausalLM", - "params_billions": "12.879" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/cloudyu_Mixtral_7Bx2_MoE/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.448 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.516 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0687 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3054 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4473 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3044 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cloudyu_s1-llama-3.2-3bx4-moe.json b/data/models/cloudyu_s1-llama-3.2-3bx4-moe.json deleted file mode 100644 index d26b2ee471f4cc5589150c464a0dfd7b1621faa5..0000000000000000000000000000000000000000 --- a/data/models/cloudyu_s1-llama-3.2-3bx4-moe.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "S1-Llama-3.2-3Bx4-MoE", - "id": "cloudyu/S1-Llama-3.2-3Bx4-MoE", - "developer": "cloudyu", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MixtralForCausalLM", - "params_billions": "9.555" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/cloudyu_S1-Llama-3.2-3Bx4-MoE/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5302 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4358 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1201 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2936 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3456 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3044 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cloudyu_yi-34bx2-moe-60b-dpo.json b/data/models/cloudyu_yi-34bx2-moe-60b-dpo.json deleted file mode 100644 index 0f8352c8ace156c45d6459afc6f8da0d6ca9c3c6..0000000000000000000000000000000000000000 --- a/data/models/cloudyu_yi-34bx2-moe-60b-dpo.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Yi-34Bx2-MoE-60B-DPO", - "id": "cloudyu/Yi-34Bx2-MoE-60B-DPO", - "developer": "cloudyu", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MixtralForCausalLM", - "params_billions": "60.814" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/cloudyu_Yi-34Bx2-MoE-60B-DPO/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5319 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5168 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0702 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3221 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4375 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4677 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cluebbers_llama-3.1-8b-paraphrase-type-generation-apty-ipo.json b/data/models/cluebbers_llama-3.1-8b-paraphrase-type-generation-apty-ipo.json deleted file mode 100644 index 19b0a905f481cfb6d1bc49e57f6fb05def849f8a..0000000000000000000000000000000000000000 --- a/data/models/cluebbers_llama-3.1-8b-paraphrase-type-generation-apty-ipo.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.1-8B-paraphrase-type-generation-apty-ipo", - "id": "cluebbers/Llama-3.1-8B-paraphrase-type-generation-apty-ipo", - "developer": "cluebbers", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/cluebbers_Llama-3.1-8B-paraphrase-type-generation-apty-ipo/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1327 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.38 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0249 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2634 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4332 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2591 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cluebbers_llama-3.1-8b-paraphrase-type-generation-apty-sigmoid.json b/data/models/cluebbers_llama-3.1-8b-paraphrase-type-generation-apty-sigmoid.json deleted file mode 100644 index 0802665d77ef6e20972233e1c1b6f555c6a99501..0000000000000000000000000000000000000000 --- a/data/models/cluebbers_llama-3.1-8b-paraphrase-type-generation-apty-sigmoid.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.1-8B-paraphrase-type-generation-apty-sigmoid", - "id": "cluebbers/Llama-3.1-8B-paraphrase-type-generation-apty-sigmoid", - "developer": "cluebbers", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/cluebbers_Llama-3.1-8B-paraphrase-type-generation-apty-sigmoid/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1318 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3789 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0264 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2685 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4306 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2562 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cluebbers_llama-3.1-8b-paraphrase-type-generation-etpc.json b/data/models/cluebbers_llama-3.1-8b-paraphrase-type-generation-etpc.json deleted file mode 100644 index a658f158864b683617c2ca78e1faba77935bca8e..0000000000000000000000000000000000000000 --- a/data/models/cluebbers_llama-3.1-8b-paraphrase-type-generation-etpc.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.1-8B-paraphrase-type-generation-etpc", - "id": "cluebbers/Llama-3.1-8B-paraphrase-type-generation-etpc", - "developer": "cluebbers", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/cluebbers_Llama-3.1-8B-paraphrase-type-generation-etpc/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1209 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3781 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0196 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2651 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4319 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2556 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cognitivecomputations_dolphin-2.9-llama3-8b.json b/data/models/cognitivecomputations_dolphin-2.9-llama3-8b.json deleted file mode 100644 index c25f4a1afd0dc6a56edc7c27567b230e9fc24584..0000000000000000000000000000000000000000 --- a/data/models/cognitivecomputations_dolphin-2.9-llama3-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "dolphin-2.9-llama3-8b", - "id": "cognitivecomputations/dolphin-2.9-llama3-8b", - "developer": "cognitivecomputations", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/cognitivecomputations_dolphin-2.9-llama3-8b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.385 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.495 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0574 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2869 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4375 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2771 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cognitivecomputations_dolphin-2.9.1-llama-3-70b.json b/data/models/cognitivecomputations_dolphin-2.9.1-llama-3-70b.json deleted file mode 100644 index 57fc3d25490a0b3fdceaf3c9984435070ea362e0..0000000000000000000000000000000000000000 --- a/data/models/cognitivecomputations_dolphin-2.9.1-llama-3-70b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "dolphin-2.9.1-llama-3-70b", - "id": "cognitivecomputations/dolphin-2.9.1-llama-3-70b", - "developer": "cognitivecomputations", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "70.554" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/cognitivecomputations_dolphin-2.9.1-llama-3-70b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.376 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5205 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.182 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3087 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4976 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.413 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cognitivecomputations_dolphin-2.9.1-yi-1.5-34b.json b/data/models/cognitivecomputations_dolphin-2.9.1-yi-1.5-34b.json deleted file mode 100644 index 8daf2ce0bf6d29f1bbce2f473d901aedd69cd4be..0000000000000000000000000000000000000000 --- a/data/models/cognitivecomputations_dolphin-2.9.1-yi-1.5-34b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "dolphin-2.9.1-yi-1.5-34b", - "id": "cognitivecomputations/dolphin-2.9.1-yi-1.5-34b", - "developer": "cognitivecomputations", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "34.389" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/cognitivecomputations_dolphin-2.9.1-yi-1.5-34b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3853 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6076 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1866 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3431 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4598 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4519 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cognitivecomputations_dolphin-2.9.1-yi-1.5-9b.json b/data/models/cognitivecomputations_dolphin-2.9.1-yi-1.5-9b.json deleted file mode 100644 index 2b4a9502227024c757decd93af508376d0d422fd..0000000000000000000000000000000000000000 --- a/data/models/cognitivecomputations_dolphin-2.9.1-yi-1.5-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "dolphin-2.9.1-yi-1.5-9b", - "id": "cognitivecomputations/dolphin-2.9.1-yi-1.5-9b", - "developer": "cognitivecomputations", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.829" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/cognitivecomputations_dolphin-2.9.1-yi-1.5-9b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4465 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5484 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1518 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3381 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4348 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3967 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cognitivecomputations_dolphin-2.9.2-phi-3-medium-abliterated.json b/data/models/cognitivecomputations_dolphin-2.9.2-phi-3-medium-abliterated.json deleted file mode 100644 index 9b7cc35f188c98dba3f0427de09e77d4b26cfc40..0000000000000000000000000000000000000000 --- a/data/models/cognitivecomputations_dolphin-2.9.2-phi-3-medium-abliterated.json +++ /dev/null @@ -1,275 +0,0 @@ -{ - "model_info": { - "name": "dolphin-2.9.2-Phi-3-Medium-abliterated", - "id": "cognitivecomputations/dolphin-2.9.2-Phi-3-Medium-abliterated", - "developer": "cognitivecomputations", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "13.96" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/cognitivecomputations_dolphin-2.9.2-Phi-3-Medium-abliterated/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3613 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6123 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1239 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.328 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4112 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4494 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "hfopenllm_v2/cognitivecomputations_dolphin-2.9.2-Phi-3-Medium-abliterated/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4124 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6383 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.182 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3289 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4349 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4525 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cognitivecomputations_dolphin-2.9.2-phi-3-medium.json b/data/models/cognitivecomputations_dolphin-2.9.2-phi-3-medium.json deleted file mode 100644 index c52d3d7a83f6aff981a388539d5d0dff9d9608d0..0000000000000000000000000000000000000000 --- a/data/models/cognitivecomputations_dolphin-2.9.2-phi-3-medium.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "dolphin-2.9.2-Phi-3-Medium", - "id": "cognitivecomputations/dolphin-2.9.2-Phi-3-Medium", - "developer": "cognitivecomputations", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "-1.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/cognitivecomputations_dolphin-2.9.2-Phi-3-Medium/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4248 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6457 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1828 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3272 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4191 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4555 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cognitivecomputations_dolphin-2.9.2-qwen2-72b.json b/data/models/cognitivecomputations_dolphin-2.9.2-qwen2-72b.json deleted file mode 100644 index e3017b5d5932ab2840763542e71bdde3b5251e9f..0000000000000000000000000000000000000000 --- a/data/models/cognitivecomputations_dolphin-2.9.2-qwen2-72b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "dolphin-2.9.2-qwen2-72b", - "id": "cognitivecomputations/dolphin-2.9.2-qwen2-72b", - "developer": "cognitivecomputations", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "72.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/cognitivecomputations_dolphin-2.9.2-qwen2-72b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6344 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6296 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2802 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.37 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4521 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5471 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cognitivecomputations_dolphin-2.9.2-qwen2-7b.json b/data/models/cognitivecomputations_dolphin-2.9.2-qwen2-7b.json deleted file mode 100644 index 9dae451278652f6c9653c6feb7e3f2fdc3a88680..0000000000000000000000000000000000000000 --- a/data/models/cognitivecomputations_dolphin-2.9.2-qwen2-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "dolphin-2.9.2-qwen2-7b", - "id": "cognitivecomputations/dolphin-2.9.2-qwen2-7b", - "developer": "cognitivecomputations", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/cognitivecomputations_dolphin-2.9.2-qwen2-7b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3535 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4894 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1344 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2903 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4191 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4051 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cognitivecomputations_dolphin-2.9.3-mistral-7b-32k.json b/data/models/cognitivecomputations_dolphin-2.9.3-mistral-7b-32k.json deleted file mode 100644 index 514af7ea423ffaa0462b751e85b1eb299117e47f..0000000000000000000000000000000000000000 --- a/data/models/cognitivecomputations_dolphin-2.9.3-mistral-7b-32k.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "dolphin-2.9.3-mistral-7B-32k", - "id": "cognitivecomputations/dolphin-2.9.3-mistral-7B-32k", - "developer": "cognitivecomputations", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/cognitivecomputations_dolphin-2.9.3-mistral-7B-32k/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4126 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4813 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0506 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2852 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4643 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2821 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cognitivecomputations_dolphin-2.9.3-mistral-nemo-12b.json b/data/models/cognitivecomputations_dolphin-2.9.3-mistral-nemo-12b.json deleted file mode 100644 index 238acf40b700237814acff3f724a50b0cd80bdb6..0000000000000000000000000000000000000000 --- a/data/models/cognitivecomputations_dolphin-2.9.3-mistral-nemo-12b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "dolphin-2.9.3-mistral-nemo-12b", - "id": "cognitivecomputations/dolphin-2.9.3-mistral-nemo-12b", - "developer": "cognitivecomputations", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/cognitivecomputations_dolphin-2.9.3-mistral-nemo-12b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5601 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.548 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.074 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3154 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.443 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3377 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cognitivecomputations_dolphin-2.9.3-yi-1.5-34b-32k.json b/data/models/cognitivecomputations_dolphin-2.9.3-yi-1.5-34b-32k.json deleted file mode 100644 index 20781132883866789244a5ea5f8392ccd5928faf..0000000000000000000000000000000000000000 --- a/data/models/cognitivecomputations_dolphin-2.9.3-yi-1.5-34b-32k.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "dolphin-2.9.3-Yi-1.5-34B-32k", - "id": "cognitivecomputations/dolphin-2.9.3-Yi-1.5-34B-32k", - "developer": "cognitivecomputations", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "34.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/cognitivecomputations_dolphin-2.9.3-Yi-1.5-34B-32k/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3639 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6047 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1669 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3431 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4311 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.463 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cognitivecomputations_dolphin-2.9.4-gemma2-2b.json b/data/models/cognitivecomputations_dolphin-2.9.4-gemma2-2b.json deleted file mode 100644 index ed33f069dee013618f55b88c411fd879cd19eeec..0000000000000000000000000000000000000000 --- a/data/models/cognitivecomputations_dolphin-2.9.4-gemma2-2b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "dolphin-2.9.4-gemma2-2b", - "id": "cognitivecomputations/dolphin-2.9.4-gemma2-2b", - "developer": "cognitivecomputations", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "2.614" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/cognitivecomputations_dolphin-2.9.4-gemma2-2b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0896 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4081 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0491 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2844 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.418 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2105 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cognitivecomputations_dolphin-2.9.4-llama3.1-8b.json b/data/models/cognitivecomputations_dolphin-2.9.4-llama3.1-8b.json deleted file mode 100644 index a6ccf065bc7740a28fedc78c1e412889b35a6d2a..0000000000000000000000000000000000000000 --- a/data/models/cognitivecomputations_dolphin-2.9.4-llama3.1-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "dolphin-2.9.4-llama3.1-8b", - "id": "cognitivecomputations/dolphin-2.9.4-llama3.1-8b", - "developer": "cognitivecomputations", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/cognitivecomputations_dolphin-2.9.4-llama3.1-8b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2757 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3524 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0121 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2634 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3236 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1237 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cognitivecomputations_dolphin3.0-llama3.1-8b.json b/data/models/cognitivecomputations_dolphin3.0-llama3.1-8b.json deleted file mode 100644 index 438e45553b3c457cd1b977c16db662a8284850ff..0000000000000000000000000000000000000000 --- a/data/models/cognitivecomputations_dolphin3.0-llama3.1-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Dolphin3.0-Llama3.1-8B", - "id": "cognitivecomputations/Dolphin3.0-Llama3.1-8B", - "developer": "cognitivecomputations", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/cognitivecomputations_Dolphin3.0-Llama3.1-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7621 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4916 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1231 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2827 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3653 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2992 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cognitivecomputations_dolphin3.0-llama3.2-1b.json b/data/models/cognitivecomputations_dolphin3.0-llama3.2-1b.json deleted file mode 100644 index 12a1d11fba4f9afd7dae8ae76de7b482a017dc1e..0000000000000000000000000000000000000000 --- a/data/models/cognitivecomputations_dolphin3.0-llama3.2-1b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Dolphin3.0-Llama3.2-1B", - "id": "cognitivecomputations/Dolphin3.0-Llama3.2-1B", - "developer": "cognitivecomputations", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.236" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/cognitivecomputations_Dolphin3.0-Llama3.2-1B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5428 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3122 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0279 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2299 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3249 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1375 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cognitivecomputations_dolphin3.0-qwen2.5-0.5b.json b/data/models/cognitivecomputations_dolphin3.0-qwen2.5-0.5b.json deleted file mode 100644 index 2c133ccc903731f09f5ae00cf49351d3327f5a60..0000000000000000000000000000000000000000 --- a/data/models/cognitivecomputations_dolphin3.0-qwen2.5-0.5b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Dolphin3.0-Qwen2.5-0.5B", - "id": "cognitivecomputations/Dolphin3.0-Qwen2.5-0.5B", - "developer": "cognitivecomputations", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/cognitivecomputations_Dolphin3.0-Qwen2.5-0.5B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4697 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3114 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0514 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2349 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3555 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1413 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cognitivecomputations_dolphin3.0-r1-mistral-24b.json b/data/models/cognitivecomputations_dolphin3.0-r1-mistral-24b.json deleted file mode 100644 index 8a8de24612d01d77ecab16ee610d05ab0bec04af..0000000000000000000000000000000000000000 --- a/data/models/cognitivecomputations_dolphin3.0-r1-mistral-24b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Dolphin3.0-R1-Mistral-24B", - "id": "cognitivecomputations/Dolphin3.0-R1-Mistral-24B", - "developer": "cognitivecomputations", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "23.572" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/cognitivecomputations_Dolphin3.0-R1-Mistral-24B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4068 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.536 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3119 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2945 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3952 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3005 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cohere_aya-expanse-32b.json b/data/models/cohere_aya-expanse-32b.json deleted file mode 100644 index 83bc775cdb901f6da45cb8e06cafd7eed94a8c7f..0000000000000000000000000000000000000000 --- a/data/models/cohere_aya-expanse-32b.json +++ /dev/null @@ -1,528 +0,0 @@ -{ - "model_info": { - "name": "aya-expanse-32b", - "id": "cohere/aya-expanse-32b", - "developer": "cohere", - "inference_platform": "unknown", - "additional_details": { - "display_name": "Aya Expanse 32B" - } - }, - "evaluations": [ - { - "evaluation_id": "global-mmlu-lite/cohere_aya-expanse-32b/1773936583.743359", - "retrieved_timestamp": "1773936583.743359", - "source_metadata": { - "source_name": "Global MMLU Lite Leaderboard", - "source_type": "documentation", - "source_organization_name": "kaggle", - "source_organization_url": "www.kaggle.com", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "kaggle kernel", - "version": "4", - "additional_details": { - "url": "https://www.kaggle.com/code/shivalikasingh95/global-mmlu-lite-sample-notebook" - } - }, - "benchmark": "global-mmlu-lite", - "evaluation_results": [ - { - "evaluation_name": "Global MMLU Lite", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Global MMLU Lite", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7353 - } - }, - { - "evaluation_name": "Culturally Sensitive", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Sensitive", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6891 - } - }, - { - "evaluation_name": "Culturally Agnostic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Agnostic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7815 - } - }, - { - "evaluation_name": "Arabic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Arabic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7425, - "uncertainty": { - "confidence_interval": { - "lower": -0.0429, - "upper": 0.0429, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "English", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - English", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7544, - "uncertainty": { - "confidence_interval": { - "lower": -0.0422, - "upper": 0.0422, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Bengali", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Bengali", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7343, - "uncertainty": { - "confidence_interval": { - "lower": -0.0433, - "upper": 0.0433, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "German", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - German", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7425, - "uncertainty": { - "confidence_interval": { - "lower": -0.0429, - "upper": 0.0429, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "French", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - French", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7325, - "uncertainty": { - "confidence_interval": { - "lower": -0.0434, - "upper": 0.0434, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Hindi", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Hindi", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7375, - "uncertainty": { - "confidence_interval": { - "lower": -0.0431, - "upper": 0.0431, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Indonesian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Indonesian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7594, - "uncertainty": { - "confidence_interval": { - "lower": -0.0419, - "upper": 0.0419, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Italian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Italian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7305, - "uncertainty": { - "confidence_interval": { - "lower": -0.0436, - "upper": 0.0436, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Japanese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Japanese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7419, - "uncertainty": { - "confidence_interval": { - "lower": -0.0429, - "upper": 0.0429, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Korean", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Korean", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7525, - "uncertainty": { - "confidence_interval": { - "lower": -0.0423, - "upper": 0.0423, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Portuguese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Portuguese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7544, - "uncertainty": { - "confidence_interval": { - "lower": -0.0422, - "upper": 0.0422, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Spanish", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Spanish", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7362, - "uncertainty": { - "confidence_interval": { - "lower": -0.0433, - "upper": 0.0433, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Swahili", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Swahili", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7071, - "uncertainty": { - "confidence_interval": { - "lower": -0.0448, - "upper": 0.0448, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Yoruba", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Yoruba", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6942, - "uncertainty": { - "confidence_interval": { - "lower": -0.0452, - "upper": 0.0452, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Chinese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Chinese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.743, - "uncertainty": { - "confidence_interval": { - "lower": -0.0432, - "upper": 0.0432, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Burmese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Burmese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7025, - "uncertainty": { - "confidence_interval": { - "lower": -0.0448, - "upper": 0.0448, - "method": "unknown" - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cohere_cohere-command-beta-52.4b.json b/data/models/cohere_cohere-command-beta-52.4b.json deleted file mode 100644 index c30f5ec96156573fe030aff4761c36f6e2667458..0000000000000000000000000000000000000000 --- a/data/models/cohere_cohere-command-beta-52.4b.json +++ /dev/null @@ -1,674 +0,0 @@ -{ - "model_info": { - "name": "Cohere Command beta 52.4B", - "id": "cohere/Cohere-Command-beta-52.4B", - "developer": "cohere", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_classic/cohere_Cohere-Command-beta-52.4B/1774096308.339228", - "retrieved_timestamp": "1774096308.339228", - "source_metadata": { - "source_name": "helm_classic", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_classic", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_classic", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperform on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.874, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Calibration": "{\"description\": \"\", \"tab\": \"Calibration\", \"score\": \"0.5963856625666678\"}", - "Mean win rate - Robustness": "{\"description\": \"\", \"tab\": \"Robustness\", \"score\": \"0.8502739196287583\"}", - "Mean win rate - Fairness": "{\"description\": \"\", \"tab\": \"Fairness\", \"score\": \"0.8657917351465738\"}", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}", - "Mean win rate - Bias": "{\"description\": \"\", \"tab\": \"Bias\", \"score\": \"0.5758163753811841\"}", - "Mean win rate - Toxicity": "{\"description\": \"\", \"tab\": \"Toxicity\", \"score\": \"0.6738178488178488\"}", - "Mean win rate - Summarization metrics": "{\"description\": \"\", \"tab\": \"Summarization metrics\", \"score\": \"0.6776315789473684\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.452, - "details": { - "description": "min=0.23, mean=0.452, max=0.79, sum=6.786 (15)", - "tab": "Accuracy", - "MMLU - ECE (10-bin)": "{\"description\": \"min=0.099, mean=0.183, max=0.338, sum=2.742 (15)\", \"tab\": \"Calibration\", \"score\": \"0.18282231471159943\"}", - "MMLU - EM (Robustness)": "{\"description\": \"min=0.15, mean=0.387, max=0.73, sum=5.807 (15)\", \"tab\": \"Robustness\", \"score\": \"0.38711111111111113\"}", - "MMLU - EM (Fairness)": "{\"description\": \"min=0.19, mean=0.407, max=0.73, sum=6.107 (15)\", \"tab\": \"Fairness\", \"score\": \"0.4071111111111111\"}", - "MMLU - Denoised inference time (s)": "{\"description\": \"5 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=1542 (15)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=75 (15)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (15)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=372.75, mean=481.26, max=628.421, sum=7218.903 (15)\", \"tab\": \"General information\", \"score\": \"481.2602105263158\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=15 (15)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "MMLU - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=45 (15)\", \"tab\": \"General information\", \"score\": \"3.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "BoolQ", - "source_data": { - "dataset_name": "BoolQ", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on BoolQ", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.856, - "details": { - "description": "min=0.849, mean=0.856, max=0.86, sum=2.569 (3)", - "tab": "Accuracy", - "BoolQ - ECE (10-bin)": "{\"description\": \"min=0.018, mean=0.023, max=0.026, sum=0.069 (3)\", \"tab\": \"Calibration\", \"score\": \"0.02302613493537822\"}", - "BoolQ - EM (Robustness)": "{\"description\": \"min=0.806, mean=0.811, max=0.816, sum=2.432 (3)\", \"tab\": \"Robustness\", \"score\": \"0.8106666666666666\"}", - "BoolQ - EM (Fairness)": "{\"description\": \"min=0.812, mean=0.822, max=0.827, sum=2.465 (3)\", \"tab\": \"Fairness\", \"score\": \"0.8216666666666667\"}", - "BoolQ - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "BoolQ - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "BoolQ - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "BoolQ - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "BoolQ - # prompt tokens": "{\"description\": \"min=669.307, mean=925.307, max=1269.307, sum=2775.921 (3)\", \"tab\": \"General information\", \"score\": \"925.3070000000001\"}", - "BoolQ - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=3 (3)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "BoolQ - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "BoolQ - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.752, - "details": { - "description": "min=0.744, mean=0.752, max=0.763, sum=2.255 (3)", - "tab": "Accuracy", - "NarrativeQA - ECE (10-bin)": "{\"description\": \"min=0.051, mean=0.058, max=0.067, sum=0.173 (3)\", \"tab\": \"Calibration\", \"score\": \"0.05761424791814445\"}", - "NarrativeQA - F1 (Robustness)": "{\"description\": \"min=0.566, mean=0.57, max=0.578, sum=1.711 (3)\", \"tab\": \"Robustness\", \"score\": \"0.5702997988620334\"}", - "NarrativeQA - F1 (Fairness)": "{\"description\": \"min=0.647, mean=0.657, max=0.666, sum=1.97 (3)\", \"tab\": \"Fairness\", \"score\": \"0.6566736137653061\"}", - "NarrativeQA - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=1065 (3)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=0.904, mean=1.508, max=1.941, sum=4.524 (3)\", \"tab\": \"General information\", \"score\": \"1.5079812206572771\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=1570.772, mean=1600.684, max=1660.485, sum=4802.051 (3)\", \"tab\": \"General information\", \"score\": \"1600.6835680751174\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=5.679, mean=5.992, max=6.496, sum=17.977 (3)\", \"tab\": \"General information\", \"score\": \"5.992488262910798\"}", - "NarrativeQA - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NarrativeQA - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NarrativeQA - Stereotypes (gender)": "{\"description\": \"min=0.361, mean=0.404, max=0.444, sum=1.213 (3)\", \"tab\": \"Bias\", \"score\": \"0.404320987654321\"}", - "NarrativeQA - Representation (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=1.333 (2)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666667\"}", - "NarrativeQA - Representation (gender)": "{\"description\": \"min=0.174, mean=0.178, max=0.181, sum=0.534 (3)\", \"tab\": \"Bias\", \"score\": \"0.1778748183802931\"}", - "NarrativeQA - Toxic fraction": "{\"description\": \"min=0.011, mean=0.014, max=0.017, sum=0.042 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.014084507042253521\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (open-book)", - "source_data": { - "dataset_name": "NaturalQuestions (open-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (open-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.76, - "details": { - "description": "min=0.755, mean=0.76, max=0.763, sum=2.28 (3)", - "tab": "Accuracy", - "NaturalQuestions (closed-book) - ECE (10-bin)": "{\"description\": \"min=0.075, mean=0.084, max=0.091, sum=0.251 (3)\", \"tab\": \"Calibration\", \"score\": \"0.08377931898267306\"}", - "NaturalQuestions (open-book) - ECE (10-bin)": "{\"description\": \"min=0.044, mean=0.056, max=0.063, sum=0.168 (3)\", \"tab\": \"Calibration\", \"score\": \"0.05602757611120105\"}", - "NaturalQuestions (closed-book) - F1 (Robustness)": "{\"description\": \"min=0.286, mean=0.289, max=0.294, sum=0.867 (3)\", \"tab\": \"Robustness\", \"score\": \"0.28891923018489013\"}", - "NaturalQuestions (open-book) - F1 (Robustness)": "{\"description\": \"min=0.669, mean=0.679, max=0.685, sum=2.036 (3)\", \"tab\": \"Robustness\", \"score\": \"0.6786112890887687\"}", - "NaturalQuestions (closed-book) - F1 (Fairness)": "{\"description\": \"min=0.29, mean=0.296, max=0.301, sum=0.888 (3)\", \"tab\": \"Fairness\", \"score\": \"0.29608566298974776\"}", - "NaturalQuestions (open-book) - F1 (Fairness)": "{\"description\": \"min=0.7, mean=0.706, max=0.714, sum=2.117 (3)\", \"tab\": \"Fairness\", \"score\": \"0.7056823207366739\"}", - "NaturalQuestions (closed-book) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NaturalQuestions (open-book) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=109.191, mean=111.191, max=115.191, sum=333.573 (3)\", \"tab\": \"General information\", \"score\": \"111.19099999999999\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=4.29, mean=4.325, max=4.367, sum=12.974 (3)\", \"tab\": \"General information\", \"score\": \"4.324666666666666\"}", - "NaturalQuestions (closed-book) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.485, mean=4.602, max=4.705, sum=13.807 (3)\", \"tab\": \"General information\", \"score\": \"4.602333333333333\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.039, mean=0.039, max=0.039, sum=0.117 (3)\", \"tab\": \"General information\", \"score\": \"0.039\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1258.15, mean=1471.073, max=1597.431, sum=4413.22 (3)\", \"tab\": \"General information\", \"score\": \"1471.073333333333\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=7.153, mean=7.288, max=7.488, sum=21.864 (3)\", \"tab\": \"General information\", \"score\": \"7.288\"}", - "NaturalQuestions (open-book) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NaturalQuestions (closed-book) - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=2 (3)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "NaturalQuestions (closed-book) - Stereotypes (gender)": "{\"description\": \"min=0.5, mean=0.5, max=0.5, sum=1.5 (3)\", \"tab\": \"Bias\", \"score\": \"0.5\"}", - "NaturalQuestions (closed-book) - Representation (race)": "{\"description\": \"min=0.487, mean=0.552, max=0.634, sum=1.655 (3)\", \"tab\": \"Bias\", \"score\": \"0.5517958743765196\"}", - "NaturalQuestions (closed-book) - Representation (gender)": "{\"description\": \"min=0.063, mean=0.129, max=0.206, sum=0.387 (3)\", \"tab\": \"Bias\", \"score\": \"0.12914332399626519\"}", - "NaturalQuestions (open-book) - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=2 (3)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "NaturalQuestions (open-book) - Stereotypes (gender)": "{\"description\": \"min=0.479, mean=0.482, max=0.483, sum=1.446 (3)\", \"tab\": \"Bias\", \"score\": \"0.48194444444444445\"}", - "NaturalQuestions (open-book) - Representation (race)": "{\"description\": \"min=0.577, mean=0.579, max=0.582, sum=1.737 (3)\", \"tab\": \"Bias\", \"score\": \"0.5791309646902151\"}", - "NaturalQuestions (open-book) - Representation (gender)": "{\"description\": \"min=0.025, mean=0.05, max=0.067, sum=0.151 (3)\", \"tab\": \"Bias\", \"score\": \"0.05047080979284368\"}", - "NaturalQuestions (closed-book) - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}", - "NaturalQuestions (open-book) - Toxic fraction": "{\"description\": \"min=0.001, mean=0.001, max=0.001, sum=0.003 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.001\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "QuAC", - "source_data": { - "dataset_name": "QuAC", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on QuAC", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.432, - "details": { - "description": "min=0.429, mean=0.432, max=0.435, sum=1.296 (3)", - "tab": "Accuracy", - "QuAC - ECE (10-bin)": "{\"description\": \"min=0.043, mean=0.06, max=0.073, sum=0.181 (3)\", \"tab\": \"Calibration\", \"score\": \"0.06049762085119498\"}", - "QuAC - F1 (Robustness)": "{\"description\": \"min=0.236, mean=0.238, max=0.24, sum=0.715 (3)\", \"tab\": \"Robustness\", \"score\": \"0.23825281130135667\"}", - "QuAC - F1 (Fairness)": "{\"description\": \"min=0.309, mean=0.316, max=0.322, sum=0.947 (3)\", \"tab\": \"Fairness\", \"score\": \"0.31563184414828255\"}", - "QuAC - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "QuAC - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "QuAC - # train": "{\"description\": \"min=0.748, mean=0.848, max=0.933, sum=2.545 (3)\", \"tab\": \"General information\", \"score\": \"0.8483333333333333\"}", - "QuAC - truncated": "{\"description\": \"min=0.022, mean=0.022, max=0.022, sum=0.066 (3)\", \"tab\": \"General information\", \"score\": \"0.022000000000000002\"}", - "QuAC - # prompt tokens": "{\"description\": \"min=1577.224, mean=1610.503, max=1643.74, sum=4831.508 (3)\", \"tab\": \"General information\", \"score\": \"1610.5026666666665\"}", - "QuAC - # output tokens": "{\"description\": \"min=19.435, mean=19.627, max=19.984, sum=58.881 (3)\", \"tab\": \"General information\", \"score\": \"19.627\"}", - "QuAC - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "QuAC - Stereotypes (race)": "{\"description\": \"min=0.593, mean=0.596, max=0.603, sum=1.788 (3)\", \"tab\": \"Bias\", \"score\": \"0.5961199294532628\"}", - "QuAC - Stereotypes (gender)": "{\"description\": \"min=0.459, mean=0.47, max=0.484, sum=1.409 (3)\", \"tab\": \"Bias\", \"score\": \"0.4696816360952984\"}", - "QuAC - Representation (race)": "{\"description\": \"min=0.299, mean=0.316, max=0.333, sum=0.949 (3)\", \"tab\": \"Bias\", \"score\": \"0.316297459154602\"}", - "QuAC - Representation (gender)": "{\"description\": \"min=0.219, mean=0.232, max=0.245, sum=0.695 (3)\", \"tab\": \"Bias\", \"score\": \"0.23168423828159934\"}", - "QuAC - Toxic fraction": "{\"description\": \"min=0.001, mean=0.001, max=0.001, sum=0.003 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.001\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "HellaSwag", - "source_data": { - "dataset_name": "HellaSwag", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on HellaSwag", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.811, - "details": { - "description": "min=0.811, mean=0.811, max=0.811, sum=0.811 (1)", - "tab": "Accuracy", - "HellaSwag - ECE (10-bin)": "{\"description\": \"min=0.325, mean=0.325, max=0.325, sum=0.325 (1)\", \"tab\": \"Calibration\", \"score\": \"0.3246923611213033\"}", - "HellaSwag - EM (Robustness)": "{\"description\": \"min=0.774, mean=0.774, max=0.774, sum=0.774 (1)\", \"tab\": \"Robustness\", \"score\": \"0.774\"}", - "HellaSwag - EM (Fairness)": "{\"description\": \"min=0.699, mean=0.699, max=0.699, sum=0.699 (1)\", \"tab\": \"Fairness\", \"score\": \"0.699\"}", - "HellaSwag - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "HellaSwag - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "HellaSwag - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "HellaSwag - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "HellaSwag - # prompt tokens": "{\"description\": \"min=88.855, mean=88.855, max=88.855, sum=88.855 (1)\", \"tab\": \"General information\", \"score\": \"88.855\"}", - "HellaSwag - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "HellaSwag - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.582, - "details": { - "description": "min=0.582, mean=0.582, max=0.582, sum=0.582 (1)", - "tab": "Accuracy", - "OpenbookQA - ECE (10-bin)": "{\"description\": \"min=0.231, mean=0.231, max=0.231, sum=0.231 (1)\", \"tab\": \"Calibration\", \"score\": \"0.23111297495969485\"}", - "OpenbookQA - EM (Robustness)": "{\"description\": \"min=0.492, mean=0.492, max=0.492, sum=0.492 (1)\", \"tab\": \"Robustness\", \"score\": \"0.492\"}", - "OpenbookQA - EM (Fairness)": "{\"description\": \"min=0.508, mean=0.508, max=0.508, sum=0.508 (1)\", \"tab\": \"Fairness\", \"score\": \"0.508\"}", - "OpenbookQA - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=5.358, mean=5.358, max=5.358, sum=5.358 (1)\", \"tab\": \"General information\", \"score\": \"5.358\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "OpenbookQA - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "TruthfulQA", - "source_data": { - "dataset_name": "TruthfulQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on TruthfulQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.269, - "details": { - "description": "min=0.265, mean=0.269, max=0.275, sum=0.807 (3)", - "tab": "Accuracy", - "TruthfulQA - ECE (10-bin)": "{\"description\": \"min=0.272, mean=0.311, max=0.338, sum=0.933 (3)\", \"tab\": \"Calibration\", \"score\": \"0.31095945192078733\"}", - "TruthfulQA - EM (Robustness)": "{\"description\": \"min=0.226, mean=0.229, max=0.231, sum=0.688 (3)\", \"tab\": \"Robustness\", \"score\": \"0.2293577981651376\"}", - "TruthfulQA - EM (Fairness)": "{\"description\": \"min=0.219, mean=0.222, max=0.225, sum=0.665 (3)\", \"tab\": \"Fairness\", \"score\": \"0.2217125382262997\"}", - "TruthfulQA - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "TruthfulQA - # eval": "{\"description\": \"min=654, mean=654, max=654, sum=1962 (3)\", \"tab\": \"General information\", \"score\": \"654.0\"}", - "TruthfulQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "TruthfulQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "TruthfulQA - # prompt tokens": "{\"description\": \"min=505.315, mean=514.648, max=532.315, sum=1543.945 (3)\", \"tab\": \"General information\", \"score\": \"514.6483180428135\"}", - "TruthfulQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=3 (3)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "TruthfulQA - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MS MARCO (TREC)", - "source_data": { - "dataset_name": "MS MARCO (TREC)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "NDCG@10 on MS MARCO (TREC)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.762, - "details": { - "description": "min=0.761, mean=0.762, max=0.765, sum=2.287 (3)", - "tab": "Accuracy", - "MS MARCO (regular) - RR@10 (Robustness)": "{\"description\": \"min=0.429, mean=0.434, max=0.438, sum=1.303 (3)\", \"tab\": \"Robustness\", \"score\": \"0.43439140211640154\"}", - "MS MARCO (TREC) - NDCG@10 (Robustness)": "{\"description\": \"min=0.726, mean=0.734, max=0.743, sum=2.202 (3)\", \"tab\": \"Robustness\", \"score\": \"0.7339375978505934\"}", - "MS MARCO (regular) - RR@10 (Fairness)": "{\"description\": \"min=0.444, mean=0.45, max=0.453, sum=1.35 (3)\", \"tab\": \"Fairness\", \"score\": \"0.4498752645502638\"}", - "MS MARCO (TREC) - NDCG@10 (Fairness)": "{\"description\": \"min=0.745, mean=0.748, max=0.752, sum=2.245 (3)\", \"tab\": \"Fairness\", \"score\": \"0.7483868294443408\"}", - "MS MARCO (regular) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MS MARCO (TREC) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MS MARCO (regular) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "MS MARCO (regular) - # train": "{\"description\": \"min=2, mean=2, max=2, sum=6 (3)\", \"tab\": \"General information\", \"score\": \"2.0\"}", - "MS MARCO (regular) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MS MARCO (regular) - # prompt tokens": "{\"description\": \"min=497.281, mean=536.614, max=583.281, sum=1609.843 (3)\", \"tab\": \"General information\", \"score\": \"536.6143333333333\"}", - "MS MARCO (regular) - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=3 (3)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "MS MARCO (regular) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "MS MARCO (TREC) - # eval": "{\"description\": \"min=43, mean=43, max=43, sum=129 (3)\", \"tab\": \"General information\", \"score\": \"43.0\"}", - "MS MARCO (TREC) - # train": "{\"description\": \"min=2, mean=2, max=2, sum=6 (3)\", \"tab\": \"General information\", \"score\": \"2.0\"}", - "MS MARCO (TREC) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MS MARCO (TREC) - # prompt tokens": "{\"description\": \"min=480.163, mean=519.496, max=566.163, sum=1558.488 (3)\", \"tab\": \"General information\", \"score\": \"519.4961240310078\"}", - "MS MARCO (TREC) - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=3 (3)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "MS MARCO (TREC) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "MS MARCO (regular) - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "MS MARCO (TREC) - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CNN/DailyMail", - "source_data": { - "dataset_name": "CNN/DailyMail", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on CNN/DailyMail", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.161, - "details": { - "description": "min=0.156, mean=0.161, max=0.167, sum=0.966 (6)", - "tab": "Accuracy", - "CNN/DailyMail - Denoised inference time (s)": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "CNN/DailyMail - # eval": "{\"description\": \"min=466, mean=466, max=466, sum=2796 (6)\", \"tab\": \"General information\", \"score\": \"466.0\"}", - "CNN/DailyMail - # train": "{\"description\": \"min=5, mean=5, max=5, sum=30 (6)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CNN/DailyMail - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (6)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CNN/DailyMail - # prompt tokens": "{\"description\": \"min=1555.036, mean=1575.036, max=1602.036, sum=9450.219 (6)\", \"tab\": \"General information\", \"score\": \"1575.0364806866953\"}", - "CNN/DailyMail - # output tokens": "{\"description\": \"min=72.088, mean=74.406, max=77.451, sum=446.433 (6)\", \"tab\": \"General information\", \"score\": \"74.40557939914163\"}", - "CNN/DailyMail - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=18 (6)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "CNN/DailyMail - Stereotypes (race)": "{\"description\": \"min=0.587, mean=0.612, max=0.629, sum=3.673 (6)\", \"tab\": \"Bias\", \"score\": \"0.6121656731068496\"}", - "CNN/DailyMail - Stereotypes (gender)": "{\"description\": \"min=0.391, mean=0.396, max=0.407, sum=2.379 (6)\", \"tab\": \"Bias\", \"score\": \"0.39642600089657387\"}", - "CNN/DailyMail - Representation (race)": "{\"description\": \"min=0.238, mean=0.286, max=0.343, sum=1.713 (6)\", \"tab\": \"Bias\", \"score\": \"0.28558037967512334\"}", - "CNN/DailyMail - Representation (gender)": "{\"description\": \"min=0.088, mean=0.09, max=0.093, sum=0.537 (6)\", \"tab\": \"Bias\", \"score\": \"0.08955985269326716\"}", - "CNN/DailyMail - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (6)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}", - "CNN/DailyMail - SummaC": "{\"description\": \"min=0.366, mean=0.415, max=0.441, sum=1.245 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.4149051333035736\"}", - "CNN/DailyMail - QAFactEval": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - BERTScore (F1)": "{\"description\": \"min=0.316, mean=0.318, max=0.322, sum=0.955 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.31834420143428105\"}", - "CNN/DailyMail - Coverage": "{\"description\": \"min=0.976, mean=0.979, max=0.982, sum=5.874 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"0.9790462109521986\"}", - "CNN/DailyMail - Density": "{\"description\": \"min=28.96, mean=32.165, max=35.676, sum=192.989 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"32.164866076836944\"}", - "CNN/DailyMail - Compression": "{\"description\": \"min=8.594, mean=9.156, max=9.657, sum=54.938 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"9.156293880030324\"}", - "CNN/DailyMail - HumanEval-faithfulness": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-relevance": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-coherence": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "XSUM", - "source_data": { - "dataset_name": "XSUM", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on XSUM", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.152, - "details": { - "description": "min=0.147, mean=0.152, max=0.156, sum=0.913 (6)", - "tab": "Accuracy", - "XSUM - Denoised inference time (s)": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "XSUM - # eval": "{\"description\": \"min=518, mean=518, max=518, sum=3108 (6)\", \"tab\": \"General information\", \"score\": \"518.0\"}", - "XSUM - # train": "{\"description\": \"min=4.996, mean=4.997, max=5, sum=29.985 (6)\", \"tab\": \"General information\", \"score\": \"4.997425997425997\"}", - "XSUM - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (6)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "XSUM - # prompt tokens": "{\"description\": \"min=1484.608, mean=1537.293, max=1572.616, sum=9223.757 (6)\", \"tab\": \"General information\", \"score\": \"1537.2927927927929\"}", - "XSUM - # output tokens": "{\"description\": \"min=24.187, mean=24.351, max=24.541, sum=146.108 (6)\", \"tab\": \"General information\", \"score\": \"24.35135135135135\"}", - "XSUM - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=18 (6)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "XSUM - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=4.0 (6)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666669\"}", - "XSUM - Stereotypes (gender)": "{\"description\": \"min=0.433, mean=0.457, max=0.476, sum=2.745 (6)\", \"tab\": \"Bias\", \"score\": \"0.4574302134646962\"}", - "XSUM - Representation (race)": "{\"description\": \"min=0.481, mean=0.522, max=0.556, sum=3.13 (6)\", \"tab\": \"Bias\", \"score\": \"0.5217473884140551\"}", - "XSUM - Representation (gender)": "{\"description\": \"min=0.18, mean=0.181, max=0.182, sum=1.086 (6)\", \"tab\": \"Bias\", \"score\": \"0.1810207108427353\"}", - "XSUM - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (6)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}", - "XSUM - SummaC": "{\"description\": \"min=-0.285, mean=-0.271, max=-0.262, sum=-0.814 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"-0.27140173856816235\"}", - "XSUM - QAFactEval": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - BERTScore (F1)": "{\"description\": \"min=0.455, mean=0.459, max=0.462, sum=1.376 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.4587225678869484\"}", - "XSUM - Coverage": "{\"description\": \"min=0.788, mean=0.793, max=0.797, sum=4.758 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"0.7930169105851288\"}", - "XSUM - Density": "{\"description\": \"min=2.417, mean=2.548, max=2.678, sum=15.286 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"2.54760656490819\"}", - "XSUM - Compression": "{\"description\": \"min=16.704, mean=16.937, max=17.065, sum=101.621 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"16.93675136805864\"}", - "XSUM - HumanEval-faithfulness": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-relevance": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-coherence": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "IMDB", - "source_data": { - "dataset_name": "IMDB", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on IMDB", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.96, - "details": { - "description": "min=0.955, mean=0.96, max=0.965, sum=2.881 (3)", - "tab": "Accuracy", - "IMDB - ECE (10-bin)": "{\"description\": \"min=0.011, mean=0.015, max=0.02, sum=0.045 (3)\", \"tab\": \"Calibration\", \"score\": \"0.015015056118517703\"}", - "IMDB - EM (Robustness)": "{\"description\": \"min=0.929, mean=0.933, max=0.936, sum=2.799 (3)\", \"tab\": \"Robustness\", \"score\": \"0.9330000000000002\"}", - "IMDB - EM (Fairness)": "{\"description\": \"min=0.951, mean=0.957, max=0.96, sum=2.871 (3)\", \"tab\": \"Fairness\", \"score\": \"0.957\"}", - "IMDB - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "IMDB - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "IMDB - # train": "{\"description\": \"min=2.89, mean=4.217, max=4.981, sum=12.652 (3)\", \"tab\": \"General information\", \"score\": \"4.217333333333333\"}", - "IMDB - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IMDB - # prompt tokens": "{\"description\": \"min=1282.318, mean=1557.741, max=1776.111, sum=4673.222 (3)\", \"tab\": \"General information\", \"score\": \"1557.7406666666666\"}", - "IMDB - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=3 (3)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "IMDB - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "IMDB - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CivilComments", - "source_data": { - "dataset_name": "CivilComments", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on CivilComments", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.601, - "details": { - "description": "min=0.254, mean=0.601, max=0.86, sum=32.478 (54)", - "tab": "Accuracy", - "CivilComments - ECE (10-bin)": "{\"description\": \"min=0.054, mean=0.161, max=0.416, sum=8.676 (54)\", \"tab\": \"Calibration\", \"score\": \"0.16066140880534402\"}", - "CivilComments - EM (Robustness)": "{\"description\": \"min=0.205, mean=0.535, max=0.84, sum=28.866 (54)\", \"tab\": \"Robustness\", \"score\": \"0.5345588668880686\"}", - "CivilComments - EM (Fairness)": "{\"description\": \"min=0.222, mean=0.544, max=0.85, sum=29.397 (54)\", \"tab\": \"Fairness\", \"score\": \"0.5443897908426464\"}", - "CivilComments - Denoised inference time (s)": "{\"description\": \"9 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "CivilComments - # eval": "{\"description\": \"min=74, mean=371.556, max=683, sum=20064 (54)\", \"tab\": \"General information\", \"score\": \"371.55555555555554\"}", - "CivilComments - # train": "{\"description\": \"min=5, mean=5, max=5, sum=270 (54)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CivilComments - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (54)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CivilComments - # prompt tokens": "{\"description\": \"min=362.293, mean=732.514, max=1288.441, sum=39555.782 (54)\", \"tab\": \"General information\", \"score\": \"732.5144825548033\"}", - "CivilComments - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=54 (54)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "CivilComments - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=162 (54)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "CivilComments - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (54)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "RAFT", - "source_data": { - "dataset_name": "RAFT", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on RAFT", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.667, - "details": { - "description": "min=0.025, mean=0.667, max=0.975, sum=22.0 (33)", - "tab": "Accuracy", - "RAFT - ECE (10-bin)": "{\"description\": \"min=0.041, mean=0.262, max=0.96, sum=8.637 (33)\", \"tab\": \"Calibration\", \"score\": \"0.26172447899775947\"}", - "RAFT - EM (Robustness)": "{\"description\": \"min=0, mean=0.599, max=0.975, sum=19.775 (33)\", \"tab\": \"Robustness\", \"score\": \"0.5992424242424242\"}", - "RAFT - EM (Fairness)": "{\"description\": \"min=0.025, mean=0.627, max=0.975, sum=20.7 (33)\", \"tab\": \"Fairness\", \"score\": \"0.6272727272727272\"}", - "RAFT - Denoised inference time (s)": "{\"description\": \"11 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "RAFT - # eval": "{\"description\": \"min=40, mean=40, max=40, sum=1320 (33)\", \"tab\": \"General information\", \"score\": \"40.0\"}", - "RAFT - # train": "{\"description\": \"min=0, mean=4.554, max=5, sum=150.275 (33)\", \"tab\": \"General information\", \"score\": \"4.553787878787879\"}", - "RAFT - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (33)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "RAFT - # prompt tokens": "{\"description\": \"min=270.325, mean=813.265, max=1762.475, sum=26837.75 (33)\", \"tab\": \"General information\", \"score\": \"813.2651515151515\"}", - "RAFT - # output tokens": "{\"description\": \"min=0.025, mean=3.15, max=6.8, sum=103.95 (33)\", \"tab\": \"General information\", \"score\": \"3.15\"}", - "RAFT - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=99 (33)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "RAFT - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (33)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/cohere_cohere-command-beta-6.1b.json b/data/models/cohere_cohere-command-beta-6.1b.json deleted file mode 100644 index 30f0890c810e1d0f7f95fa0fcfa00e81ad33650c..0000000000000000000000000000000000000000 --- a/data/models/cohere_cohere-command-beta-6.1b.json +++ /dev/null @@ -1,674 +0,0 @@ -{ - "model_info": { - "name": "Cohere Command beta 6.1B", - "id": "cohere/Cohere-Command-beta-6.1B", - "developer": "cohere", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_classic/cohere_Cohere-Command-beta-6.1B/1774096308.339228", - "retrieved_timestamp": "1774096308.339228", - "source_metadata": { - "source_name": "helm_classic", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_classic", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_classic", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperform on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.675, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Calibration": "{\"description\": \"\", \"tab\": \"Calibration\", \"score\": \"0.5291111339523303\"}", - "Mean win rate - Robustness": "{\"description\": \"\", \"tab\": \"Robustness\", \"score\": \"0.6159776448986682\"}", - "Mean win rate - Fairness": "{\"description\": \"\", \"tab\": \"Fairness\", \"score\": \"0.66227113635345\"}", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}", - "Mean win rate - Bias": "{\"description\": \"\", \"tab\": \"Bias\", \"score\": \"0.43551719208606965\"}", - "Mean win rate - Toxicity": "{\"description\": \"\", \"tab\": \"Toxicity\", \"score\": \"0.6688037271370605\"}", - "Mean win rate - Summarization metrics": "{\"description\": \"\", \"tab\": \"Summarization metrics\", \"score\": \"0.5789473684210527\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.406, - "details": { - "description": "min=0.26, mean=0.406, max=0.63, sum=6.095 (15)", - "tab": "Accuracy", - "MMLU - ECE (10-bin)": "{\"description\": \"min=0.103, mean=0.155, max=0.243, sum=2.327 (15)\", \"tab\": \"Calibration\", \"score\": \"0.1551609000421963\"}", - "MMLU - EM (Robustness)": "{\"description\": \"min=0.2, mean=0.334, max=0.54, sum=5.009 (15)\", \"tab\": \"Robustness\", \"score\": \"0.33394152046783626\"}", - "MMLU - EM (Fairness)": "{\"description\": \"min=0.2, mean=0.366, max=0.55, sum=5.495 (15)\", \"tab\": \"Fairness\", \"score\": \"0.36630409356725147\"}", - "MMLU - Denoised inference time (s)": "{\"description\": \"5 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=1542 (15)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=75 (15)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (15)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=372.75, mean=481.26, max=628.421, sum=7218.903 (15)\", \"tab\": \"General information\", \"score\": \"481.2602105263158\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=15 (15)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "MMLU - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=45 (15)\", \"tab\": \"General information\", \"score\": \"3.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "BoolQ", - "source_data": { - "dataset_name": "BoolQ", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on BoolQ", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.798, - "details": { - "description": "min=0.791, mean=0.798, max=0.809, sum=2.394 (3)", - "tab": "Accuracy", - "BoolQ - ECE (10-bin)": "{\"description\": \"min=0.048, mean=0.059, max=0.069, sum=0.178 (3)\", \"tab\": \"Calibration\", \"score\": \"0.0594622129465324\"}", - "BoolQ - EM (Robustness)": "{\"description\": \"min=0.715, mean=0.725, max=0.743, sum=2.176 (3)\", \"tab\": \"Robustness\", \"score\": \"0.7253333333333334\"}", - "BoolQ - EM (Fairness)": "{\"description\": \"min=0.74, mean=0.748, max=0.76, sum=2.244 (3)\", \"tab\": \"Fairness\", \"score\": \"0.7479999999999999\"}", - "BoolQ - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "BoolQ - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "BoolQ - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "BoolQ - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "BoolQ - # prompt tokens": "{\"description\": \"min=669.307, mean=925.307, max=1269.307, sum=2775.921 (3)\", \"tab\": \"General information\", \"score\": \"925.3070000000001\"}", - "BoolQ - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=3 (3)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "BoolQ - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "BoolQ - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.709, - "details": { - "description": "min=0.707, mean=0.709, max=0.712, sum=2.128 (3)", - "tab": "Accuracy", - "NarrativeQA - ECE (10-bin)": "{\"description\": \"min=0.075, mean=0.076, max=0.077, sum=0.228 (3)\", \"tab\": \"Calibration\", \"score\": \"0.07599807506781359\"}", - "NarrativeQA - F1 (Robustness)": "{\"description\": \"min=0.515, mean=0.529, max=0.539, sum=1.586 (3)\", \"tab\": \"Robustness\", \"score\": \"0.5285770759196127\"}", - "NarrativeQA - F1 (Fairness)": "{\"description\": \"min=0.592, mean=0.595, max=0.6, sum=1.785 (3)\", \"tab\": \"Fairness\", \"score\": \"0.5949605221040284\"}", - "NarrativeQA - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=1065 (3)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=0.904, mean=1.508, max=1.941, sum=4.524 (3)\", \"tab\": \"General information\", \"score\": \"1.5079812206572771\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=1570.772, mean=1600.684, max=1660.485, sum=4802.051 (3)\", \"tab\": \"General information\", \"score\": \"1600.6835680751174\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=5.301, mean=5.807, max=6.217, sum=17.42 (3)\", \"tab\": \"General information\", \"score\": \"5.8065727699530525\"}", - "NarrativeQA - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NarrativeQA - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NarrativeQA - Stereotypes (gender)": "{\"description\": \"min=0.463, mean=0.488, max=0.5, sum=1.463 (3)\", \"tab\": \"Bias\", \"score\": \"0.48765432098765427\"}", - "NarrativeQA - Representation (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=2 (3)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "NarrativeQA - Representation (gender)": "{\"description\": \"min=0.126, mean=0.144, max=0.169, sum=0.432 (3)\", \"tab\": \"Bias\", \"score\": \"0.14398558425056623\"}", - "NarrativeQA - Toxic fraction": "{\"description\": \"min=0.008, mean=0.01, max=0.014, sum=0.031 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.010328638497652582\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (open-book)", - "source_data": { - "dataset_name": "NaturalQuestions (open-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (open-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.717, - "details": { - "description": "min=0.714, mean=0.717, max=0.724, sum=2.152 (3)", - "tab": "Accuracy", - "NaturalQuestions (closed-book) - ECE (10-bin)": "{\"description\": \"min=0.04, mean=0.042, max=0.046, sum=0.127 (3)\", \"tab\": \"Calibration\", \"score\": \"0.04227945276969597\"}", - "NaturalQuestions (open-book) - ECE (10-bin)": "{\"description\": \"min=0.045, mean=0.057, max=0.074, sum=0.172 (3)\", \"tab\": \"Calibration\", \"score\": \"0.057325907163997956\"}", - "NaturalQuestions (closed-book) - F1 (Robustness)": "{\"description\": \"min=0.156, mean=0.163, max=0.171, sum=0.489 (3)\", \"tab\": \"Robustness\", \"score\": \"0.163031767310864\"}", - "NaturalQuestions (open-book) - F1 (Robustness)": "{\"description\": \"min=0.596, mean=0.605, max=0.616, sum=1.815 (3)\", \"tab\": \"Robustness\", \"score\": \"0.6050162193677248\"}", - "NaturalQuestions (closed-book) - F1 (Fairness)": "{\"description\": \"min=0.165, mean=0.167, max=0.167, sum=0.5 (3)\", \"tab\": \"Fairness\", \"score\": \"0.16652011745655915\"}", - "NaturalQuestions (open-book) - F1 (Fairness)": "{\"description\": \"min=0.647, mean=0.654, max=0.66, sum=1.962 (3)\", \"tab\": \"Fairness\", \"score\": \"0.6540942012407344\"}", - "NaturalQuestions (closed-book) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NaturalQuestions (open-book) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=109.191, mean=111.191, max=115.191, sum=333.573 (3)\", \"tab\": \"General information\", \"score\": \"111.19099999999999\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=4.428, mean=4.687, max=4.995, sum=14.06 (3)\", \"tab\": \"General information\", \"score\": \"4.6866666666666665\"}", - "NaturalQuestions (closed-book) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.485, mean=4.602, max=4.705, sum=13.807 (3)\", \"tab\": \"General information\", \"score\": \"4.602333333333333\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.039, mean=0.039, max=0.039, sum=0.117 (3)\", \"tab\": \"General information\", \"score\": \"0.039\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1258.15, mean=1471.073, max=1597.431, sum=4413.22 (3)\", \"tab\": \"General information\", \"score\": \"1471.073333333333\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=7.147, mean=7.377, max=7.586, sum=22.131 (3)\", \"tab\": \"General information\", \"score\": \"7.377\"}", - "NaturalQuestions (open-book) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NaturalQuestions (closed-book) - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - Representation (race)": "{\"description\": \"min=0.421, mean=0.465, max=0.506, sum=1.394 (3)\", \"tab\": \"Bias\", \"score\": \"0.46474105132386057\"}", - "NaturalQuestions (closed-book) - Representation (gender)": "{\"description\": \"min=0.1, mean=0.183, max=0.3, sum=0.55 (3)\", \"tab\": \"Bias\", \"score\": \"0.18333333333333335\"}", - "NaturalQuestions (open-book) - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=1.333 (2)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666667\"}", - "NaturalQuestions (open-book) - Stereotypes (gender)": "{\"description\": \"min=0.5, mean=0.5, max=0.5, sum=1.5 (3)\", \"tab\": \"Bias\", \"score\": \"0.5\"}", - "NaturalQuestions (open-book) - Representation (race)": "{\"description\": \"min=0.473, mean=0.487, max=0.509, sum=1.46 (3)\", \"tab\": \"Bias\", \"score\": \"0.48677896291115386\"}", - "NaturalQuestions (open-book) - Representation (gender)": "{\"description\": \"min=0.348, mean=0.356, max=0.363, sum=1.068 (3)\", \"tab\": \"Bias\", \"score\": \"0.3560153609831029\"}", - "NaturalQuestions (closed-book) - Toxic fraction": "{\"description\": \"min=0, mean=0.0, max=0.001, sum=0.001 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.0003333333333333333\"}", - "NaturalQuestions (open-book) - Toxic fraction": "{\"description\": \"min=0, mean=0.0, max=0.001, sum=0.001 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.0003333333333333333\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "QuAC", - "source_data": { - "dataset_name": "QuAC", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on QuAC", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.375, - "details": { - "description": "min=0.371, mean=0.375, max=0.379, sum=1.125 (3)", - "tab": "Accuracy", - "QuAC - ECE (10-bin)": "{\"description\": \"min=0.054, mean=0.062, max=0.067, sum=0.186 (3)\", \"tab\": \"Calibration\", \"score\": \"0.06185077042352865\"}", - "QuAC - F1 (Robustness)": "{\"description\": \"min=0.159, mean=0.17, max=0.178, sum=0.511 (3)\", \"tab\": \"Robustness\", \"score\": \"0.17034790269142241\"}", - "QuAC - F1 (Fairness)": "{\"description\": \"min=0.268, mean=0.273, max=0.279, sum=0.819 (3)\", \"tab\": \"Fairness\", \"score\": \"0.2730533859766594\"}", - "QuAC - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "QuAC - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "QuAC - # train": "{\"description\": \"min=0.748, mean=0.848, max=0.933, sum=2.545 (3)\", \"tab\": \"General information\", \"score\": \"0.8483333333333333\"}", - "QuAC - truncated": "{\"description\": \"min=0.022, mean=0.022, max=0.022, sum=0.066 (3)\", \"tab\": \"General information\", \"score\": \"0.022000000000000002\"}", - "QuAC - # prompt tokens": "{\"description\": \"min=1577.224, mean=1610.503, max=1643.74, sum=4831.508 (3)\", \"tab\": \"General information\", \"score\": \"1610.5026666666665\"}", - "QuAC - # output tokens": "{\"description\": \"min=16.185, mean=17.394, max=18.299, sum=52.182 (3)\", \"tab\": \"General information\", \"score\": \"17.394\"}", - "QuAC - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "QuAC - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=2 (3)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "QuAC - Stereotypes (gender)": "{\"description\": \"min=0.469, mean=0.471, max=0.475, sum=1.414 (3)\", \"tab\": \"Bias\", \"score\": \"0.47144607843137254\"}", - "QuAC - Representation (race)": "{\"description\": \"min=0.312, mean=0.356, max=0.423, sum=1.069 (3)\", \"tab\": \"Bias\", \"score\": \"0.35619490458200137\"}", - "QuAC - Representation (gender)": "{\"description\": \"min=0.236, mean=0.248, max=0.259, sum=0.743 (3)\", \"tab\": \"Bias\", \"score\": \"0.2476420794142787\"}", - "QuAC - Toxic fraction": "{\"description\": \"min=0.002, mean=0.002, max=0.002, sum=0.006 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.002\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "HellaSwag", - "source_data": { - "dataset_name": "HellaSwag", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on HellaSwag", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.752, - "details": { - "description": "min=0.752, mean=0.752, max=0.752, sum=0.752 (1)", - "tab": "Accuracy", - "HellaSwag - ECE (10-bin)": "{\"description\": \"min=0.293, mean=0.293, max=0.293, sum=0.293 (1)\", \"tab\": \"Calibration\", \"score\": \"0.2926835489814197\"}", - "HellaSwag - EM (Robustness)": "{\"description\": \"min=0.696, mean=0.696, max=0.696, sum=0.696 (1)\", \"tab\": \"Robustness\", \"score\": \"0.696\"}", - "HellaSwag - EM (Fairness)": "{\"description\": \"min=0.608, mean=0.608, max=0.608, sum=0.608 (1)\", \"tab\": \"Fairness\", \"score\": \"0.608\"}", - "HellaSwag - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "HellaSwag - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "HellaSwag - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "HellaSwag - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "HellaSwag - # prompt tokens": "{\"description\": \"min=88.855, mean=88.855, max=88.855, sum=88.855 (1)\", \"tab\": \"General information\", \"score\": \"88.855\"}", - "HellaSwag - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "HellaSwag - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.55, - "details": { - "description": "min=0.55, mean=0.55, max=0.55, sum=0.55 (1)", - "tab": "Accuracy", - "OpenbookQA - ECE (10-bin)": "{\"description\": \"min=0.25, mean=0.25, max=0.25, sum=0.25 (1)\", \"tab\": \"Calibration\", \"score\": \"0.2504061981122775\"}", - "OpenbookQA - EM (Robustness)": "{\"description\": \"min=0.448, mean=0.448, max=0.448, sum=0.448 (1)\", \"tab\": \"Robustness\", \"score\": \"0.448\"}", - "OpenbookQA - EM (Fairness)": "{\"description\": \"min=0.468, mean=0.468, max=0.468, sum=0.468 (1)\", \"tab\": \"Fairness\", \"score\": \"0.468\"}", - "OpenbookQA - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=5.358, mean=5.358, max=5.358, sum=5.358 (1)\", \"tab\": \"General information\", \"score\": \"5.358\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "OpenbookQA - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "TruthfulQA", - "source_data": { - "dataset_name": "TruthfulQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on TruthfulQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.203, - "details": { - "description": "min=0.197, mean=0.203, max=0.213, sum=0.61 (3)", - "tab": "Accuracy", - "TruthfulQA - ECE (10-bin)": "{\"description\": \"min=0.275, mean=0.3, max=0.332, sum=0.901 (3)\", \"tab\": \"Calibration\", \"score\": \"0.3001833323753285\"}", - "TruthfulQA - EM (Robustness)": "{\"description\": \"min=0.168, mean=0.171, max=0.174, sum=0.512 (3)\", \"tab\": \"Robustness\", \"score\": \"0.17074413863404692\"}", - "TruthfulQA - EM (Fairness)": "{\"description\": \"min=0.154, mean=0.163, max=0.167, sum=0.488 (3)\", \"tab\": \"Fairness\", \"score\": \"0.16258919469928643\"}", - "TruthfulQA - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "TruthfulQA - # eval": "{\"description\": \"min=654, mean=654, max=654, sum=1962 (3)\", \"tab\": \"General information\", \"score\": \"654.0\"}", - "TruthfulQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "TruthfulQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "TruthfulQA - # prompt tokens": "{\"description\": \"min=505.315, mean=514.648, max=532.315, sum=1543.945 (3)\", \"tab\": \"General information\", \"score\": \"514.6483180428135\"}", - "TruthfulQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=3 (3)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "TruthfulQA - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MS MARCO (TREC)", - "source_data": { - "dataset_name": "MS MARCO (TREC)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "NDCG@10 on MS MARCO (TREC)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.709, - "details": { - "description": "min=0.702, mean=0.709, max=0.717, sum=2.128 (3)", - "tab": "Accuracy", - "MS MARCO (regular) - RR@10 (Robustness)": "{\"description\": \"min=0.372, mean=0.387, max=0.401, sum=1.161 (3)\", \"tab\": \"Robustness\", \"score\": \"0.386937698412698\"}", - "MS MARCO (TREC) - NDCG@10 (Robustness)": "{\"description\": \"min=0.68, mean=0.685, max=0.689, sum=2.054 (3)\", \"tab\": \"Robustness\", \"score\": \"0.6845367765287401\"}", - "MS MARCO (regular) - RR@10 (Fairness)": "{\"description\": \"min=0.402, mean=0.411, max=0.42, sum=1.232 (3)\", \"tab\": \"Fairness\", \"score\": \"0.4107572751322747\"}", - "MS MARCO (TREC) - NDCG@10 (Fairness)": "{\"description\": \"min=0.68, mean=0.69, max=0.696, sum=2.069 (3)\", \"tab\": \"Fairness\", \"score\": \"0.6896233668786421\"}", - "MS MARCO (regular) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MS MARCO (TREC) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MS MARCO (regular) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "MS MARCO (regular) - # train": "{\"description\": \"min=2, mean=2, max=2, sum=6 (3)\", \"tab\": \"General information\", \"score\": \"2.0\"}", - "MS MARCO (regular) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MS MARCO (regular) - # prompt tokens": "{\"description\": \"min=497.281, mean=536.614, max=583.281, sum=1609.843 (3)\", \"tab\": \"General information\", \"score\": \"536.6143333333333\"}", - "MS MARCO (regular) - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=3 (3)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "MS MARCO (regular) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "MS MARCO (TREC) - # eval": "{\"description\": \"min=43, mean=43, max=43, sum=129 (3)\", \"tab\": \"General information\", \"score\": \"43.0\"}", - "MS MARCO (TREC) - # train": "{\"description\": \"min=2, mean=2, max=2, sum=6 (3)\", \"tab\": \"General information\", \"score\": \"2.0\"}", - "MS MARCO (TREC) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MS MARCO (TREC) - # prompt tokens": "{\"description\": \"min=480.163, mean=519.496, max=566.163, sum=1558.488 (3)\", \"tab\": \"General information\", \"score\": \"519.4961240310078\"}", - "MS MARCO (TREC) - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=3 (3)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "MS MARCO (TREC) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "MS MARCO (regular) - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "MS MARCO (TREC) - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CNN/DailyMail", - "source_data": { - "dataset_name": "CNN/DailyMail", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on CNN/DailyMail", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.153, - "details": { - "description": "min=0.15, mean=0.153, max=0.158, sum=0.919 (6)", - "tab": "Accuracy", - "CNN/DailyMail - Denoised inference time (s)": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "CNN/DailyMail - # eval": "{\"description\": \"min=466, mean=466, max=466, sum=2796 (6)\", \"tab\": \"General information\", \"score\": \"466.0\"}", - "CNN/DailyMail - # train": "{\"description\": \"min=5, mean=5, max=5, sum=30 (6)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CNN/DailyMail - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (6)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CNN/DailyMail - # prompt tokens": "{\"description\": \"min=1555.036, mean=1575.036, max=1602.036, sum=9450.219 (6)\", \"tab\": \"General information\", \"score\": \"1575.0364806866953\"}", - "CNN/DailyMail - # output tokens": "{\"description\": \"min=69.622, mean=73.723, max=77.732, sum=442.339 (6)\", \"tab\": \"General information\", \"score\": \"73.72317596566523\"}", - "CNN/DailyMail - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=18 (6)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "CNN/DailyMail - Stereotypes (race)": "{\"description\": \"min=0.594, mean=0.603, max=0.609, sum=3.618 (6)\", \"tab\": \"Bias\", \"score\": \"0.6029930306246096\"}", - "CNN/DailyMail - Stereotypes (gender)": "{\"description\": \"min=0.384, mean=0.408, max=0.421, sum=2.449 (6)\", \"tab\": \"Bias\", \"score\": \"0.40820094830714143\"}", - "CNN/DailyMail - Representation (race)": "{\"description\": \"min=0.245, mean=0.259, max=0.269, sum=1.553 (6)\", \"tab\": \"Bias\", \"score\": \"0.2588148950314076\"}", - "CNN/DailyMail - Representation (gender)": "{\"description\": \"min=0.116, mean=0.121, max=0.127, sum=0.724 (6)\", \"tab\": \"Bias\", \"score\": \"0.1206019792299876\"}", - "CNN/DailyMail - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (6)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}", - "CNN/DailyMail - SummaC": "{\"description\": \"min=0.318, mean=0.331, max=0.342, sum=0.992 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.3306993242099164\"}", - "CNN/DailyMail - QAFactEval": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - BERTScore (F1)": "{\"description\": \"min=0.289, mean=0.296, max=0.305, sum=0.888 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.29605955170271475\"}", - "CNN/DailyMail - Coverage": "{\"description\": \"min=0.974, mean=0.975, max=0.975, sum=5.848 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"0.9746996636764317\"}", - "CNN/DailyMail - Density": "{\"description\": \"min=28.678, mean=31.707, max=36.132, sum=190.245 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"31.707488870766706\"}", - "CNN/DailyMail - Compression": "{\"description\": \"min=9.108, mean=9.688, max=10.161, sum=58.13 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"9.688415513712991\"}", - "CNN/DailyMail - HumanEval-faithfulness": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-relevance": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-coherence": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "XSUM", - "source_data": { - "dataset_name": "XSUM", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on XSUM", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.122, - "details": { - "description": "min=0.122, mean=0.122, max=0.122, sum=0.73 (6)", - "tab": "Accuracy", - "XSUM - Denoised inference time (s)": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "XSUM - # eval": "{\"description\": \"min=518, mean=518, max=518, sum=3108 (6)\", \"tab\": \"General information\", \"score\": \"518.0\"}", - "XSUM - # train": "{\"description\": \"min=4.996, mean=4.997, max=5, sum=29.985 (6)\", \"tab\": \"General information\", \"score\": \"4.997425997425997\"}", - "XSUM - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (6)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "XSUM - # prompt tokens": "{\"description\": \"min=1484.608, mean=1537.293, max=1572.616, sum=9223.757 (6)\", \"tab\": \"General information\", \"score\": \"1537.2927927927929\"}", - "XSUM - # output tokens": "{\"description\": \"min=22.674, mean=23.421, max=24.095, sum=140.529 (6)\", \"tab\": \"General information\", \"score\": \"23.421492921492924\"}", - "XSUM - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=18 (6)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "XSUM - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=4 (6)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "XSUM - Stereotypes (gender)": "{\"description\": \"min=0.445, mean=0.454, max=0.467, sum=2.725 (6)\", \"tab\": \"Bias\", \"score\": \"0.45422077922077925\"}", - "XSUM - Representation (race)": "{\"description\": \"min=0.483, mean=0.505, max=0.524, sum=3.031 (6)\", \"tab\": \"Bias\", \"score\": \"0.5051915503043323\"}", - "XSUM - Representation (gender)": "{\"description\": \"min=0.198, mean=0.215, max=0.235, sum=1.29 (6)\", \"tab\": \"Bias\", \"score\": \"0.2150586429483566\"}", - "XSUM - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (6)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}", - "XSUM - SummaC": "{\"description\": \"min=-0.244, mean=-0.239, max=-0.235, sum=-0.716 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"-0.23871033593647883\"}", - "XSUM - QAFactEval": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - BERTScore (F1)": "{\"description\": \"min=0.417, mean=0.418, max=0.42, sum=1.254 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.4181413420706151\"}", - "XSUM - Coverage": "{\"description\": \"min=0.823, mean=0.824, max=0.826, sum=4.943 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"0.8238944118657666\"}", - "XSUM - Density": "{\"description\": \"min=2.687, mean=2.793, max=2.942, sum=16.758 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"2.7930375453507623\"}", - "XSUM - Compression": "{\"description\": \"min=17.475, mean=18.017, max=18.57, sum=108.1 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"18.016669951894464\"}", - "XSUM - HumanEval-faithfulness": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-relevance": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-coherence": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "IMDB", - "source_data": { - "dataset_name": "IMDB", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on IMDB", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.961, - "details": { - "description": "min=0.959, mean=0.961, max=0.962, sum=2.882 (3)", - "tab": "Accuracy", - "IMDB - ECE (10-bin)": "{\"description\": \"min=0.011, mean=0.014, max=0.019, sum=0.043 (3)\", \"tab\": \"Calibration\", \"score\": \"0.014204038428277976\"}", - "IMDB - EM (Robustness)": "{\"description\": \"min=0.917, mean=0.921, max=0.925, sum=2.762 (3)\", \"tab\": \"Robustness\", \"score\": \"0.9206666666666669\"}", - "IMDB - EM (Fairness)": "{\"description\": \"min=0.946, mean=0.95, max=0.954, sum=2.851 (3)\", \"tab\": \"Fairness\", \"score\": \"0.9503333333333334\"}", - "IMDB - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "IMDB - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "IMDB - # train": "{\"description\": \"min=2.89, mean=4.217, max=4.981, sum=12.652 (3)\", \"tab\": \"General information\", \"score\": \"4.217333333333333\"}", - "IMDB - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IMDB - # prompt tokens": "{\"description\": \"min=1282.318, mean=1557.741, max=1776.111, sum=4673.222 (3)\", \"tab\": \"General information\", \"score\": \"1557.7406666666666\"}", - "IMDB - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=3 (3)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "IMDB - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "IMDB - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CivilComments", - "source_data": { - "dataset_name": "CivilComments", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on CivilComments", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.54, - "details": { - "description": "min=0.009, mean=0.54, max=1, sum=29.17 (54)", - "tab": "Accuracy", - "CivilComments - ECE (10-bin)": "{\"description\": \"min=0.113, mean=0.358, max=0.735, sum=19.322 (54)\", \"tab\": \"Calibration\", \"score\": \"0.3578234752080933\"}", - "CivilComments - EM (Robustness)": "{\"description\": \"min=0, mean=0.468, max=1, sum=25.26 (54)\", \"tab\": \"Robustness\", \"score\": \"0.46778473308233626\"}", - "CivilComments - EM (Fairness)": "{\"description\": \"min=0.002, mean=0.496, max=1, sum=26.757 (54)\", \"tab\": \"Fairness\", \"score\": \"0.4955072296924251\"}", - "CivilComments - Denoised inference time (s)": "{\"description\": \"9 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "CivilComments - # eval": "{\"description\": \"min=74, mean=371.556, max=683, sum=20064 (54)\", \"tab\": \"General information\", \"score\": \"371.55555555555554\"}", - "CivilComments - # train": "{\"description\": \"min=5, mean=5, max=5, sum=270 (54)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CivilComments - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (54)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CivilComments - # prompt tokens": "{\"description\": \"min=362.293, mean=732.514, max=1288.441, sum=39555.782 (54)\", \"tab\": \"General information\", \"score\": \"732.5144825548033\"}", - "CivilComments - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=54 (54)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "CivilComments - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=162 (54)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "CivilComments - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (54)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "RAFT", - "source_data": { - "dataset_name": "RAFT", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on RAFT", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.634, - "details": { - "description": "min=0.05, mean=0.634, max=0.975, sum=20.925 (33)", - "tab": "Accuracy", - "RAFT - ECE (10-bin)": "{\"description\": \"min=0.05, mean=0.274, max=0.84, sum=9.055 (33)\", \"tab\": \"Calibration\", \"score\": \"0.2744070774220778\"}", - "RAFT - EM (Robustness)": "{\"description\": \"min=0, mean=0.552, max=0.975, sum=18.225 (33)\", \"tab\": \"Robustness\", \"score\": \"0.5522727272727274\"}", - "RAFT - EM (Fairness)": "{\"description\": \"min=0.05, mean=0.609, max=0.975, sum=20.1 (33)\", \"tab\": \"Fairness\", \"score\": \"0.609090909090909\"}", - "RAFT - Denoised inference time (s)": "{\"description\": \"11 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "RAFT - # eval": "{\"description\": \"min=40, mean=40, max=40, sum=1320 (33)\", \"tab\": \"General information\", \"score\": \"40.0\"}", - "RAFT - # train": "{\"description\": \"min=0, mean=4.554, max=5, sum=150.275 (33)\", \"tab\": \"General information\", \"score\": \"4.553787878787879\"}", - "RAFT - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (33)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "RAFT - # prompt tokens": "{\"description\": \"min=270.325, mean=813.265, max=1762.475, sum=26837.75 (33)\", \"tab\": \"General information\", \"score\": \"813.2651515151515\"}", - "RAFT - # output tokens": "{\"description\": \"min=0.2, mean=3.148, max=6.3, sum=103.875 (33)\", \"tab\": \"General information\", \"score\": \"3.1477272727272725\"}", - "RAFT - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=99 (33)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "RAFT - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (33)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/cohere_cohere-large-v20220720-13.1b.json b/data/models/cohere_cohere-large-v20220720-13.1b.json deleted file mode 100644 index 4688adefb73e6ddfa5577b9cf789453df182064d..0000000000000000000000000000000000000000 --- a/data/models/cohere_cohere-large-v20220720-13.1b.json +++ /dev/null @@ -1,674 +0,0 @@ -{ - "model_info": { - "name": "Cohere large v20220720 13.1B", - "id": "cohere/Cohere-large-v20220720-13.1B", - "developer": "cohere", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_classic/cohere_Cohere-large-v20220720-13.1B/1774096308.339228", - "retrieved_timestamp": "1774096308.339228", - "source_metadata": { - "source_name": "helm_classic", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_classic", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_classic", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperform on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.372, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Calibration": "{\"description\": \"\", \"tab\": \"Calibration\", \"score\": \"0.6524936901131783\"}", - "Mean win rate - Robustness": "{\"description\": \"\", \"tab\": \"Robustness\", \"score\": \"0.3450884302942145\"}", - "Mean win rate - Fairness": "{\"description\": \"\", \"tab\": \"Fairness\", \"score\": \"0.3621096552687209\"}", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.40696820175438597\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}", - "Mean win rate - Bias": "{\"description\": \"\", \"tab\": \"Bias\", \"score\": \"0.5413536579003514\"}", - "Mean win rate - Toxicity": "{\"description\": \"\", \"tab\": \"Toxicity\", \"score\": \"0.48450623450623453\"}", - "Mean win rate - Summarization metrics": "{\"description\": \"\", \"tab\": \"Summarization metrics\", \"score\": \"0.5760442773600668\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.324, - "details": { - "description": "min=0.19, mean=0.324, max=0.4, sum=4.854 (15)", - "tab": "Accuracy", - "MMLU - ECE (10-bin)": "{\"description\": \"min=0.075, mean=0.112, max=0.151, sum=1.678 (15)\", \"tab\": \"Calibration\", \"score\": \"0.11188578153206447\"}", - "MMLU - EM (Robustness)": "{\"description\": \"min=0.15, mean=0.253, max=0.35, sum=3.799 (15)\", \"tab\": \"Robustness\", \"score\": \"0.25327485380116954\"}", - "MMLU - EM (Fairness)": "{\"description\": \"min=0.14, mean=0.281, max=0.38, sum=4.214 (15)\", \"tab\": \"Fairness\", \"score\": \"0.2809590643274854\"}", - "MMLU - Denoised inference time (s)": "{\"description\": \"min=0.292, mean=0.317, max=0.349, sum=4.752 (15)\", \"tab\": \"Efficiency\", \"score\": \"0.3167793253495066\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=1542 (15)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=75 (15)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (15)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=372.75, mean=481.26, max=628.421, sum=7218.903 (15)\", \"tab\": \"General information\", \"score\": \"481.2602105263158\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=15 (15)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "MMLU - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=45 (15)\", \"tab\": \"General information\", \"score\": \"3.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "BoolQ", - "source_data": { - "dataset_name": "BoolQ", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on BoolQ", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.725, - "details": { - "description": "min=0.705, mean=0.725, max=0.738, sum=2.176 (3)", - "tab": "Accuracy", - "BoolQ - ECE (10-bin)": "{\"description\": \"min=0.066, mean=0.088, max=0.106, sum=0.265 (3)\", \"tab\": \"Calibration\", \"score\": \"0.08825401206422555\"}", - "BoolQ - EM (Robustness)": "{\"description\": \"min=0.514, mean=0.545, max=0.566, sum=1.635 (3)\", \"tab\": \"Robustness\", \"score\": \"0.545\"}", - "BoolQ - EM (Fairness)": "{\"description\": \"min=0.653, mean=0.676, max=0.695, sum=2.027 (3)\", \"tab\": \"Fairness\", \"score\": \"0.6756666666666667\"}", - "BoolQ - Denoised inference time (s)": "{\"description\": \"min=0.359, mean=0.421, max=0.505, sum=1.263 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.4208381308593749\"}", - "BoolQ - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "BoolQ - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "BoolQ - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "BoolQ - # prompt tokens": "{\"description\": \"min=669.307, mean=925.307, max=1269.307, sum=2775.921 (3)\", \"tab\": \"General information\", \"score\": \"925.3070000000001\"}", - "BoolQ - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=3 (3)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "BoolQ - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "BoolQ - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.625, - "details": { - "description": "min=0.581, mean=0.625, max=0.647, sum=1.874 (3)", - "tab": "Accuracy", - "NarrativeQA - ECE (10-bin)": "{\"description\": \"min=0.032, mean=0.037, max=0.044, sum=0.11 (3)\", \"tab\": \"Calibration\", \"score\": \"0.03650754887085305\"}", - "NarrativeQA - F1 (Robustness)": "{\"description\": \"min=0.318, mean=0.357, max=0.38, sum=1.072 (3)\", \"tab\": \"Robustness\", \"score\": \"0.3573511654752053\"}", - "NarrativeQA - F1 (Fairness)": "{\"description\": \"min=0.466, mean=0.512, max=0.538, sum=1.537 (3)\", \"tab\": \"Fairness\", \"score\": \"0.5123186802559418\"}", - "NarrativeQA - Denoised inference time (s)": "{\"description\": \"min=0.693, mean=0.729, max=0.782, sum=2.186 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.7286962533010564\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=1065 (3)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=0.958, mean=1.562, max=1.997, sum=4.687 (3)\", \"tab\": \"General information\", \"score\": \"1.5624413145539906\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=1601.997, mean=1634.99, max=1693.155, sum=4904.969 (3)\", \"tab\": \"General information\", \"score\": \"1634.9896713615024\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=5.535, mean=6.91, max=9.504, sum=20.73 (3)\", \"tab\": \"General information\", \"score\": \"6.909859154929578\"}", - "NarrativeQA - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NarrativeQA - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NarrativeQA - Stereotypes (gender)": "{\"description\": \"min=0.418, mean=0.473, max=0.5, sum=1.418 (3)\", \"tab\": \"Bias\", \"score\": \"0.4726495726495727\"}", - "NarrativeQA - Representation (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=2 (3)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "NarrativeQA - Representation (gender)": "{\"description\": \"min=0.193, mean=0.202, max=0.211, sum=0.607 (3)\", \"tab\": \"Bias\", \"score\": \"0.20233455199447267\"}", - "NarrativeQA - Toxic fraction": "{\"description\": \"min=0.014, mean=0.017, max=0.02, sum=0.051 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.016901408450704227\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (open-book)", - "source_data": { - "dataset_name": "NaturalQuestions (open-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (open-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.573, - "details": { - "description": "min=0.553, mean=0.573, max=0.584, sum=1.72 (3)", - "tab": "Accuracy", - "NaturalQuestions (closed-book) - ECE (10-bin)": "{\"description\": \"min=0.02, mean=0.025, max=0.032, sum=0.074 (3)\", \"tab\": \"Calibration\", \"score\": \"0.024639111727299556\"}", - "NaturalQuestions (open-book) - ECE (10-bin)": "{\"description\": \"min=0.117, mean=0.143, max=0.158, sum=0.43 (3)\", \"tab\": \"Calibration\", \"score\": \"0.14321248401208217\"}", - "NaturalQuestions (closed-book) - F1 (Robustness)": "{\"description\": \"min=0.16, mean=0.172, max=0.18, sum=0.515 (3)\", \"tab\": \"Robustness\", \"score\": \"0.17161461010403287\"}", - "NaturalQuestions (open-book) - F1 (Robustness)": "{\"description\": \"min=0.287, mean=0.347, max=0.38, sum=1.041 (3)\", \"tab\": \"Robustness\", \"score\": \"0.3470084296370371\"}", - "NaturalQuestions (closed-book) - F1 (Fairness)": "{\"description\": \"min=0.176, mean=0.178, max=0.181, sum=0.535 (3)\", \"tab\": \"Fairness\", \"score\": \"0.17833773739586523\"}", - "NaturalQuestions (open-book) - F1 (Fairness)": "{\"description\": \"min=0.489, mean=0.507, max=0.516, sum=1.52 (3)\", \"tab\": \"Fairness\", \"score\": \"0.5065982888177307\"}", - "NaturalQuestions (closed-book) - Denoised inference time (s)": "{\"description\": \"min=0.332, mean=0.337, max=0.343, sum=1.012 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.33722079557291607\"}", - "NaturalQuestions (open-book) - Denoised inference time (s)": "{\"description\": \"min=0.681, mean=0.774, max=0.827, sum=2.321 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.7738100833333333\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=109.191, mean=111.191, max=115.191, sum=333.573 (3)\", \"tab\": \"General information\", \"score\": \"111.19099999999999\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=5.441, mean=5.625, max=5.917, sum=16.875 (3)\", \"tab\": \"General information\", \"score\": \"5.625\"}", - "NaturalQuestions (closed-book) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.538, mean=4.633, max=4.715, sum=13.899 (3)\", \"tab\": \"General information\", \"score\": \"4.633\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.039, mean=0.039, max=0.039, sum=0.117 (3)\", \"tab\": \"General information\", \"score\": \"0.039\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1261.72, mean=1481.344, max=1608.455, sum=4444.032 (3)\", \"tab\": \"General information\", \"score\": \"1481.344\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=8.71, mean=10.443, max=11.438, sum=31.329 (3)\", \"tab\": \"General information\", \"score\": \"10.443\"}", - "NaturalQuestions (open-book) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NaturalQuestions (closed-book) - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - Stereotypes (gender)": "{\"description\": \"min=0.25, mean=0.333, max=0.5, sum=1 (3)\", \"tab\": \"Bias\", \"score\": \"0.3333333333333333\"}", - "NaturalQuestions (closed-book) - Representation (race)": "{\"description\": \"min=0.244, mean=0.34, max=0.429, sum=1.021 (3)\", \"tab\": \"Bias\", \"score\": \"0.34034751045060324\"}", - "NaturalQuestions (closed-book) - Representation (gender)": "{\"description\": \"min=0.208, mean=0.233, max=0.269, sum=0.7 (3)\", \"tab\": \"Bias\", \"score\": \"0.23326210826210825\"}", - "NaturalQuestions (open-book) - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=2 (3)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "NaturalQuestions (open-book) - Stereotypes (gender)": "{\"description\": \"min=0.37, mean=0.39, max=0.4, sum=1.17 (3)\", \"tab\": \"Bias\", \"score\": \"0.38999999999999996\"}", - "NaturalQuestions (open-book) - Representation (race)": "{\"description\": \"min=0.447, mean=0.457, max=0.467, sum=1.371 (3)\", \"tab\": \"Bias\", \"score\": \"0.45706182643221777\"}", - "NaturalQuestions (open-book) - Representation (gender)": "{\"description\": \"min=0.125, mean=0.174, max=0.251, sum=0.523 (3)\", \"tab\": \"Bias\", \"score\": \"0.17447005829358772\"}", - "NaturalQuestions (closed-book) - Toxic fraction": "{\"description\": \"min=0, mean=0.0, max=0.001, sum=0.001 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.0003333333333333333\"}", - "NaturalQuestions (open-book) - Toxic fraction": "{\"description\": \"min=0.001, mean=0.002, max=0.003, sum=0.005 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.0016666666666666668\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "QuAC", - "source_data": { - "dataset_name": "QuAC", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on QuAC", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.338, - "details": { - "description": "min=0.335, mean=0.338, max=0.343, sum=1.015 (3)", - "tab": "Accuracy", - "QuAC - ECE (10-bin)": "{\"description\": \"min=0.03, mean=0.033, max=0.036, sum=0.099 (3)\", \"tab\": \"Calibration\", \"score\": \"0.03288362014267938\"}", - "QuAC - F1 (Robustness)": "{\"description\": \"min=0.197, mean=0.204, max=0.211, sum=0.613 (3)\", \"tab\": \"Robustness\", \"score\": \"0.20424911828028136\"}", - "QuAC - F1 (Fairness)": "{\"description\": \"min=0.251, mean=0.256, max=0.259, sum=0.768 (3)\", \"tab\": \"Fairness\", \"score\": \"0.25613799535824233\"}", - "QuAC - Denoised inference time (s)": "{\"description\": \"min=1.189, mean=1.262, max=1.309, sum=3.785 (3)\", \"tab\": \"Efficiency\", \"score\": \"1.261730263346353\"}", - "QuAC - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "QuAC - # train": "{\"description\": \"min=0.797, mean=0.881, max=0.969, sum=2.644 (3)\", \"tab\": \"General information\", \"score\": \"0.8813333333333334\"}", - "QuAC - truncated": "{\"description\": \"min=0.02, mean=0.02, max=0.02, sum=0.06 (3)\", \"tab\": \"General information\", \"score\": \"0.02\"}", - "QuAC - # prompt tokens": "{\"description\": \"min=1600.292, mean=1639.784, max=1661.675, sum=4919.353 (3)\", \"tab\": \"General information\", \"score\": \"1639.784333333333\"}", - "QuAC - # output tokens": "{\"description\": \"min=26.693, mean=30.036, max=32.515, sum=90.109 (3)\", \"tab\": \"General information\", \"score\": \"30.036333333333335\"}", - "QuAC - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "QuAC - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=2 (3)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "QuAC - Stereotypes (gender)": "{\"description\": \"min=0.43, mean=0.441, max=0.46, sum=1.322 (3)\", \"tab\": \"Bias\", \"score\": \"0.4407422751666938\"}", - "QuAC - Representation (race)": "{\"description\": \"min=0.306, mean=0.338, max=0.358, sum=1.015 (3)\", \"tab\": \"Bias\", \"score\": \"0.3382593663469334\"}", - "QuAC - Representation (gender)": "{\"description\": \"min=0.234, mean=0.238, max=0.243, sum=0.714 (3)\", \"tab\": \"Bias\", \"score\": \"0.23804653081585347\"}", - "QuAC - Toxic fraction": "{\"description\": \"min=0.003, mean=0.003, max=0.004, sum=0.01 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.0033333333333333335\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "HellaSwag", - "source_data": { - "dataset_name": "HellaSwag", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on HellaSwag", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.736, - "details": { - "description": "min=0.736, mean=0.736, max=0.736, sum=0.736 (1)", - "tab": "Accuracy", - "HellaSwag - ECE (10-bin)": "{\"description\": \"min=0.288, mean=0.288, max=0.288, sum=0.288 (1)\", \"tab\": \"Calibration\", \"score\": \"0.28820318504565584\"}", - "HellaSwag - EM (Robustness)": "{\"description\": \"min=0.687, mean=0.687, max=0.687, sum=0.687 (1)\", \"tab\": \"Robustness\", \"score\": \"0.687\"}", - "HellaSwag - EM (Fairness)": "{\"description\": \"min=0.575, mean=0.575, max=0.575, sum=0.575 (1)\", \"tab\": \"Fairness\", \"score\": \"0.575\"}", - "HellaSwag - Denoised inference time (s)": "{\"description\": \"min=0.225, mean=0.225, max=0.225, sum=0.225 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.22464337890624972\"}", - "HellaSwag - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "HellaSwag - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "HellaSwag - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "HellaSwag - # prompt tokens": "{\"description\": \"min=88.855, mean=88.855, max=88.855, sum=88.855 (1)\", \"tab\": \"General information\", \"score\": \"88.855\"}", - "HellaSwag - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "HellaSwag - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.542, - "details": { - "description": "min=0.542, mean=0.542, max=0.542, sum=0.542 (1)", - "tab": "Accuracy", - "OpenbookQA - ECE (10-bin)": "{\"description\": \"min=0.225, mean=0.225, max=0.225, sum=0.225 (1)\", \"tab\": \"Calibration\", \"score\": \"0.2254334966206393\"}", - "OpenbookQA - EM (Robustness)": "{\"description\": \"min=0.43, mean=0.43, max=0.43, sum=0.43 (1)\", \"tab\": \"Robustness\", \"score\": \"0.43\"}", - "OpenbookQA - EM (Fairness)": "{\"description\": \"min=0.446, mean=0.446, max=0.446, sum=0.446 (1)\", \"tab\": \"Fairness\", \"score\": \"0.446\"}", - "OpenbookQA - Denoised inference time (s)": "{\"description\": \"min=0.201, mean=0.201, max=0.201, sum=0.201 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.2014860078125007\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=5.358, mean=5.358, max=5.358, sum=5.358 (1)\", \"tab\": \"General information\", \"score\": \"5.358\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "TruthfulQA", - "source_data": { - "dataset_name": "TruthfulQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on TruthfulQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.181, - "details": { - "description": "min=0.161, mean=0.181, max=0.2, sum=0.544 (3)", - "tab": "Accuracy", - "TruthfulQA - ECE (10-bin)": "{\"description\": \"min=0.097, mean=0.105, max=0.117, sum=0.316 (3)\", \"tab\": \"Calibration\", \"score\": \"0.10528939288118344\"}", - "TruthfulQA - EM (Robustness)": "{\"description\": \"min=0.141, mean=0.154, max=0.173, sum=0.462 (3)\", \"tab\": \"Robustness\", \"score\": \"0.15392456676860344\"}", - "TruthfulQA - EM (Fairness)": "{\"description\": \"min=0.142, mean=0.157, max=0.174, sum=0.471 (3)\", \"tab\": \"Fairness\", \"score\": \"0.15698267074413863\"}", - "TruthfulQA - Denoised inference time (s)": "{\"description\": \"min=0.323, mean=0.325, max=0.328, sum=0.975 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.3248777191442089\"}", - "TruthfulQA - # eval": "{\"description\": \"min=654, mean=654, max=654, sum=1962 (3)\", \"tab\": \"General information\", \"score\": \"654.0\"}", - "TruthfulQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "TruthfulQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "TruthfulQA - # prompt tokens": "{\"description\": \"min=505.315, mean=514.648, max=532.315, sum=1543.945 (3)\", \"tab\": \"General information\", \"score\": \"514.6483180428135\"}", - "TruthfulQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=3 (3)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "TruthfulQA - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MS MARCO (TREC)", - "source_data": { - "dataset_name": "MS MARCO (TREC)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "NDCG@10 on MS MARCO (TREC)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.33, - "details": { - "description": "min=0.292, mean=0.33, max=0.382, sum=0.991 (3)", - "tab": "Accuracy", - "MS MARCO (regular) - RR@10 (Robustness)": "{\"description\": \"min=0.109, mean=0.13, max=0.147, sum=0.39 (3)\", \"tab\": \"Robustness\", \"score\": \"0.1300338624338624\"}", - "MS MARCO (TREC) - NDCG@10 (Robustness)": "{\"description\": \"min=0.195, mean=0.257, max=0.323, sum=0.772 (3)\", \"tab\": \"Robustness\", \"score\": \"0.2574506868270638\"}", - "MS MARCO (regular) - RR@10 (Fairness)": "{\"description\": \"min=0.136, mean=0.164, max=0.189, sum=0.493 (3)\", \"tab\": \"Fairness\", \"score\": \"0.16423492063492048\"}", - "MS MARCO (TREC) - NDCG@10 (Fairness)": "{\"description\": \"min=0.273, mean=0.312, max=0.361, sum=0.936 (3)\", \"tab\": \"Fairness\", \"score\": \"0.3120660241438415\"}", - "MS MARCO (regular) - Denoised inference time (s)": "{\"description\": \"min=0.322, mean=0.33, max=0.339, sum=0.989 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.3298234970703125\"}", - "MS MARCO (TREC) - Denoised inference time (s)": "{\"description\": \"min=0.319, mean=0.327, max=0.335, sum=0.98 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.32664419815891477\"}", - "MS MARCO (regular) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "MS MARCO (regular) - # train": "{\"description\": \"min=2, mean=2, max=2, sum=6 (3)\", \"tab\": \"General information\", \"score\": \"2.0\"}", - "MS MARCO (regular) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MS MARCO (regular) - # prompt tokens": "{\"description\": \"min=497.281, mean=536.614, max=583.281, sum=1609.843 (3)\", \"tab\": \"General information\", \"score\": \"536.6143333333333\"}", - "MS MARCO (regular) - # output tokens": "{\"description\": \"min=1.008, mean=1.025, max=1.046, sum=3.074 (3)\", \"tab\": \"General information\", \"score\": \"1.0246666666666666\"}", - "MS MARCO (regular) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "MS MARCO (TREC) - # eval": "{\"description\": \"min=43, mean=43, max=43, sum=129 (3)\", \"tab\": \"General information\", \"score\": \"43.0\"}", - "MS MARCO (TREC) - # train": "{\"description\": \"min=2, mean=2, max=2, sum=6 (3)\", \"tab\": \"General information\", \"score\": \"2.0\"}", - "MS MARCO (TREC) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MS MARCO (TREC) - # prompt tokens": "{\"description\": \"min=480.163, mean=519.496, max=566.163, sum=1558.488 (3)\", \"tab\": \"General information\", \"score\": \"519.4961240310078\"}", - "MS MARCO (TREC) - # output tokens": "{\"description\": \"min=1.023, mean=1.031, max=1.047, sum=3.093 (3)\", \"tab\": \"General information\", \"score\": \"1.0310077519379846\"}", - "MS MARCO (TREC) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "MS MARCO (regular) - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "MS MARCO (TREC) - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CNN/DailyMail", - "source_data": { - "dataset_name": "CNN/DailyMail", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on CNN/DailyMail", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.126, - "details": { - "description": "min=0.115, mean=0.126, max=0.134, sum=0.758 (6)", - "tab": "Accuracy", - "CNN/DailyMail - Denoised inference time (s)": "{\"description\": \"min=2.097, mean=2.269, max=2.366, sum=13.614 (6)\", \"tab\": \"Efficiency\", \"score\": \"2.2689930690607114\"}", - "CNN/DailyMail - # eval": "{\"description\": \"min=466, mean=466, max=466, sum=2796 (6)\", \"tab\": \"General information\", \"score\": \"466.0\"}", - "CNN/DailyMail - # train": "{\"description\": \"min=5, mean=5, max=5, sum=30 (6)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CNN/DailyMail - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (6)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CNN/DailyMail - # prompt tokens": "{\"description\": \"min=1555.036, mean=1575.036, max=1602.036, sum=9450.219 (6)\", \"tab\": \"General information\", \"score\": \"1575.0364806866953\"}", - "CNN/DailyMail - # output tokens": "{\"description\": \"min=67.079, mean=74.505, max=78.916, sum=447.03 (6)\", \"tab\": \"General information\", \"score\": \"74.50500715307582\"}", - "CNN/DailyMail - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=18 (6)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "CNN/DailyMail - Stereotypes (race)": "{\"description\": \"min=0.58, mean=0.626, max=0.659, sum=3.756 (6)\", \"tab\": \"Bias\", \"score\": \"0.6260369618341756\"}", - "CNN/DailyMail - Stereotypes (gender)": "{\"description\": \"min=0.371, mean=0.401, max=0.431, sum=2.409 (6)\", \"tab\": \"Bias\", \"score\": \"0.40149048314255253\"}", - "CNN/DailyMail - Representation (race)": "{\"description\": \"min=0.185, mean=0.238, max=0.295, sum=1.431 (6)\", \"tab\": \"Bias\", \"score\": \"0.23843844144516976\"}", - "CNN/DailyMail - Representation (gender)": "{\"description\": \"min=0.115, mean=0.134, max=0.153, sum=0.805 (6)\", \"tab\": \"Bias\", \"score\": \"0.1341289455316015\"}", - "CNN/DailyMail - Toxic fraction": "{\"description\": \"min=0, mean=0.001, max=0.002, sum=0.004 (6)\", \"tab\": \"Toxicity\", \"score\": \"0.000715307582260372\"}", - "CNN/DailyMail - SummaC": "{\"description\": \"min=0.447, mean=0.5, max=0.543, sum=1.499 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.4997740334832678\"}", - "CNN/DailyMail - QAFactEval": "{\"description\": \"min=4.715, mean=4.763, max=4.822, sum=28.58 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"4.763415476947068\"}", - "CNN/DailyMail - BERTScore (F1)": "{\"description\": \"min=0.227, mean=0.246, max=0.263, sum=0.737 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.2457600895432969\"}", - "CNN/DailyMail - Coverage": "{\"description\": \"min=0.903, mean=0.946, max=0.975, sum=5.678 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"0.9463649022058865\"}", - "CNN/DailyMail - Density": "{\"description\": \"min=30.364, mean=37.733, max=45.984, sum=226.401 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"37.73347863579329\"}", - "CNN/DailyMail - Compression": "{\"description\": \"min=9.977, mean=11.27, max=13.424, sum=67.62 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"11.269948645908789\"}", - "CNN/DailyMail - HumanEval-faithfulness": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-relevance": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-coherence": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "XSUM", - "source_data": { - "dataset_name": "XSUM", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on XSUM", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.108, - "details": { - "description": "min=0.106, mean=0.108, max=0.11, sum=0.649 (6)", - "tab": "Accuracy", - "XSUM - Denoised inference time (s)": "{\"description\": \"min=1.064, mean=1.075, max=1.089, sum=6.451 (6)\", \"tab\": \"Efficiency\", \"score\": \"1.0751711510617759\"}", - "XSUM - # eval": "{\"description\": \"min=518, mean=518, max=518, sum=3108 (6)\", \"tab\": \"General information\", \"score\": \"518.0\"}", - "XSUM - # train": "{\"description\": \"min=4.996, mean=4.998, max=5, sum=29.988 (6)\", \"tab\": \"General information\", \"score\": \"4.998069498069498\"}", - "XSUM - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (6)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "XSUM - # prompt tokens": "{\"description\": \"min=1484.608, mean=1537.452, max=1572.616, sum=9224.71 (6)\", \"tab\": \"General information\", \"score\": \"1537.4517374517375\"}", - "XSUM - # output tokens": "{\"description\": \"min=22.133, mean=22.992, max=23.423, sum=137.954 (6)\", \"tab\": \"General information\", \"score\": \"22.99227799227799\"}", - "XSUM - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=18 (6)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "XSUM - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=4 (6)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "XSUM - Stereotypes (gender)": "{\"description\": \"min=0.456, mean=0.466, max=0.484, sum=2.793 (6)\", \"tab\": \"Bias\", \"score\": \"0.4655148596176822\"}", - "XSUM - Representation (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=4 (6)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "XSUM - Representation (gender)": "{\"description\": \"min=0.139, mean=0.157, max=0.172, sum=0.945 (6)\", \"tab\": \"Bias\", \"score\": \"0.15743560442588508\"}", - "XSUM - Toxic fraction": "{\"description\": \"min=0, mean=0.001, max=0.002, sum=0.008 (6)\", \"tab\": \"Toxicity\", \"score\": \"0.001287001287001287\"}", - "XSUM - SummaC": "{\"description\": \"min=-0.196, mean=-0.189, max=-0.185, sum=-0.567 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"-0.18902428828304493\"}", - "XSUM - QAFactEval": "{\"description\": \"min=2.852, mean=2.889, max=2.928, sum=17.336 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"2.889265592037019\"}", - "XSUM - BERTScore (F1)": "{\"description\": \"min=0.394, mean=0.398, max=0.403, sum=1.195 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.3984961779205311\"}", - "XSUM - Coverage": "{\"description\": \"min=0.82, mean=0.823, max=0.825, sum=4.937 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"0.8227568594164721\"}", - "XSUM - Density": "{\"description\": \"min=3.497, mean=3.599, max=3.746, sum=21.593 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"3.5988000456323377\"}", - "XSUM - Compression": "{\"description\": \"min=20.099, mean=20.712, max=21.78, sum=124.27 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"20.711693139962097\"}", - "XSUM - HumanEval-faithfulness": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-relevance": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-coherence": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "IMDB", - "source_data": { - "dataset_name": "IMDB", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on IMDB", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.933, - "details": { - "description": "min=0.929, mean=0.933, max=0.94, sum=2.8 (3)", - "tab": "Accuracy", - "IMDB - ECE (10-bin)": "{\"description\": \"min=0.098, mean=0.132, max=0.183, sum=0.396 (3)\", \"tab\": \"Calibration\", \"score\": \"0.13199349625828075\"}", - "IMDB - EM (Robustness)": "{\"description\": \"min=0.895, mean=0.902, max=0.91, sum=2.706 (3)\", \"tab\": \"Robustness\", \"score\": \"0.902\"}", - "IMDB - EM (Fairness)": "{\"description\": \"min=0.912, mean=0.92, max=0.93, sum=2.759 (3)\", \"tab\": \"Fairness\", \"score\": \"0.9196666666666666\"}", - "IMDB - Denoised inference time (s)": "{\"description\": \"min=0.479, mean=0.536, max=0.62, sum=1.607 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.5358171357421871\"}", - "IMDB - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "IMDB - # train": "{\"description\": \"min=4.846, mean=4.93, max=4.98, sum=14.79 (3)\", \"tab\": \"General information\", \"score\": \"4.930000000000001\"}", - "IMDB - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IMDB - # prompt tokens": "{\"description\": \"min=1161.854, mean=1398.654, max=1747.025, sum=4195.961 (3)\", \"tab\": \"General information\", \"score\": \"1398.6536666666668\"}", - "IMDB - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=3 (3)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "IMDB - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "IMDB - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CivilComments", - "source_data": { - "dataset_name": "CivilComments", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on CivilComments", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.507, - "details": { - "description": "min=0, mean=0.507, max=1, sum=27.395 (54)", - "tab": "Accuracy", - "CivilComments - ECE (10-bin)": "{\"description\": \"min=0.1, mean=0.384, max=0.705, sum=20.717 (54)\", \"tab\": \"Calibration\", \"score\": \"0.38365386942886265\"}", - "CivilComments - EM (Robustness)": "{\"description\": \"min=0, mean=0.333, max=0.95, sum=17.981 (54)\", \"tab\": \"Robustness\", \"score\": \"0.3329825600043121\"}", - "CivilComments - EM (Fairness)": "{\"description\": \"min=0, mean=0.443, max=1, sum=23.917 (54)\", \"tab\": \"Fairness\", \"score\": \"0.44290609222735455\"}", - "CivilComments - Denoised inference time (s)": "{\"description\": \"min=0.29, mean=0.375, max=0.51, sum=20.235 (54)\", \"tab\": \"Efficiency\", \"score\": \"0.3747284900914756\"}", - "CivilComments - # eval": "{\"description\": \"min=74, mean=371.556, max=683, sum=20064 (54)\", \"tab\": \"General information\", \"score\": \"371.55555555555554\"}", - "CivilComments - # train": "{\"description\": \"min=5, mean=5, max=5, sum=270 (54)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CivilComments - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (54)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CivilComments - # prompt tokens": "{\"description\": \"min=362.293, mean=732.514, max=1288.441, sum=39555.782 (54)\", \"tab\": \"General information\", \"score\": \"732.5144825548033\"}", - "CivilComments - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=54 (54)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "CivilComments - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=162 (54)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "CivilComments - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (54)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "RAFT", - "source_data": { - "dataset_name": "RAFT", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on RAFT", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.596, - "details": { - "description": "min=0, mean=0.596, max=0.975, sum=19.675 (33)", - "tab": "Accuracy", - "RAFT - ECE (10-bin)": "{\"description\": \"min=0.115, mean=0.267, max=1, sum=8.804 (33)\", \"tab\": \"Calibration\", \"score\": \"0.26679166027291745\"}", - "RAFT - EM (Robustness)": "{\"description\": \"min=0, mean=0.49, max=0.975, sum=16.175 (33)\", \"tab\": \"Robustness\", \"score\": \"0.49015151515151517\"}", - "RAFT - EM (Fairness)": "{\"description\": \"min=0, mean=0.564, max=0.975, sum=18.625 (33)\", \"tab\": \"Fairness\", \"score\": \"0.5643939393939394\"}", - "RAFT - Denoised inference time (s)": "{\"description\": \"min=0.284, mean=0.444, max=0.697, sum=14.664 (33)\", \"tab\": \"Efficiency\", \"score\": \"0.4443553984670929\"}", - "RAFT - # eval": "{\"description\": \"min=40, mean=40, max=40, sum=1320 (33)\", \"tab\": \"General information\", \"score\": \"40.0\"}", - "RAFT - # train": "{\"description\": \"min=0, mean=4.557, max=5, sum=150.375 (33)\", \"tab\": \"General information\", \"score\": \"4.556818181818182\"}", - "RAFT - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (33)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "RAFT - # prompt tokens": "{\"description\": \"min=270.325, mean=814.446, max=1777.025, sum=26876.725 (33)\", \"tab\": \"General information\", \"score\": \"814.446212121212\"}", - "RAFT - # output tokens": "{\"description\": \"min=0, mean=3.02, max=6.5, sum=99.65 (33)\", \"tab\": \"General information\", \"score\": \"3.01969696969697\"}", - "RAFT - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=99 (33)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "RAFT - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (33)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/cohere_cohere-medium-v20220720-6.1b.json b/data/models/cohere_cohere-medium-v20220720-6.1b.json deleted file mode 100644 index 01dfe564b76302f52834616593afb1796c1e11ff..0000000000000000000000000000000000000000 --- a/data/models/cohere_cohere-medium-v20220720-6.1b.json +++ /dev/null @@ -1,674 +0,0 @@ -{ - "model_info": { - "name": "Cohere medium v20220720 6.1B", - "id": "cohere/Cohere-medium-v20220720-6.1B", - "developer": "cohere", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_classic/cohere_Cohere-medium-v20220720-6.1B/1774096308.339228", - "retrieved_timestamp": "1774096308.339228", - "source_metadata": { - "source_name": "helm_classic", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_classic", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_classic", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperform on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.23, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Calibration": "{\"description\": \"\", \"tab\": \"Calibration\", \"score\": \"0.5098117312502142\"}", - "Mean win rate - Robustness": "{\"description\": \"\", \"tab\": \"Robustness\", \"score\": \"0.18793903538063716\"}", - "Mean win rate - Fairness": "{\"description\": \"\", \"tab\": \"Fairness\", \"score\": \"0.26943181031056446\"}", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.5410910087719298\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}", - "Mean win rate - Bias": "{\"description\": \"\", \"tab\": \"Bias\", \"score\": \"0.4663309072932103\"}", - "Mean win rate - Toxicity": "{\"description\": \"\", \"tab\": \"Toxicity\", \"score\": \"0.5508257174923842\"}", - "Mean win rate - Summarization metrics": "{\"description\": \"\", \"tab\": \"Summarization metrics\", \"score\": \"0.4311194653299916\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.279, - "details": { - "description": "min=0.18, mean=0.279, max=0.36, sum=4.182 (15)", - "tab": "Accuracy", - "MMLU - ECE (10-bin)": "{\"description\": \"min=0.067, mean=0.114, max=0.164, sum=1.703 (15)\", \"tab\": \"Calibration\", \"score\": \"0.11350786269483934\"}", - "MMLU - EM (Robustness)": "{\"description\": \"min=0.09, mean=0.184, max=0.24, sum=2.755 (15)\", \"tab\": \"Robustness\", \"score\": \"0.18368421052631578\"}", - "MMLU - EM (Fairness)": "{\"description\": \"min=0.15, mean=0.237, max=0.29, sum=3.548 (15)\", \"tab\": \"Fairness\", \"score\": \"0.23653801169590644\"}", - "MMLU - Denoised inference time (s)": "{\"description\": \"min=0.265, mean=0.281, max=0.301, sum=4.21 (15)\", \"tab\": \"Efficiency\", \"score\": \"0.2806724427425987\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=1542 (15)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=75 (15)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (15)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=372.75, mean=481.26, max=628.421, sum=7218.903 (15)\", \"tab\": \"General information\", \"score\": \"481.2602105263158\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=15 (15)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "MMLU - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=45 (15)\", \"tab\": \"General information\", \"score\": \"3.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "BoolQ", - "source_data": { - "dataset_name": "BoolQ", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on BoolQ", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.659, - "details": { - "description": "min=0.65, mean=0.659, max=0.667, sum=1.977 (3)", - "tab": "Accuracy", - "BoolQ - ECE (10-bin)": "{\"description\": \"min=0.069, mean=0.082, max=0.093, sum=0.247 (3)\", \"tab\": \"Calibration\", \"score\": \"0.08218351589951171\"}", - "BoolQ - EM (Robustness)": "{\"description\": \"min=0.556, mean=0.562, max=0.573, sum=1.686 (3)\", \"tab\": \"Robustness\", \"score\": \"0.5619999999999999\"}", - "BoolQ - EM (Fairness)": "{\"description\": \"min=0.589, mean=0.597, max=0.61, sum=1.792 (3)\", \"tab\": \"Fairness\", \"score\": \"0.5973333333333333\"}", - "BoolQ - Denoised inference time (s)": "{\"description\": \"min=0.308, mean=0.35, max=0.402, sum=1.049 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.34952371158854173\"}", - "BoolQ - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "BoolQ - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "BoolQ - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "BoolQ - # prompt tokens": "{\"description\": \"min=669.307, mean=925.307, max=1269.307, sum=2775.921 (3)\", \"tab\": \"General information\", \"score\": \"925.3070000000001\"}", - "BoolQ - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=3 (3)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "BoolQ - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "BoolQ - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.559, - "details": { - "description": "min=0.54, mean=0.559, max=0.572, sum=1.677 (3)", - "tab": "Accuracy", - "NarrativeQA - ECE (10-bin)": "{\"description\": \"min=0.043, mean=0.047, max=0.055, sum=0.141 (3)\", \"tab\": \"Calibration\", \"score\": \"0.046946382998353055\"}", - "NarrativeQA - F1 (Robustness)": "{\"description\": \"min=0.283, mean=0.3, max=0.315, sum=0.899 (3)\", \"tab\": \"Robustness\", \"score\": \"0.29964626689663526\"}", - "NarrativeQA - F1 (Fairness)": "{\"description\": \"min=0.416, mean=0.438, max=0.455, sum=1.313 (3)\", \"tab\": \"Fairness\", \"score\": \"0.4376922212938658\"}", - "NarrativeQA - Denoised inference time (s)": "{\"description\": \"min=0.525, mean=0.533, max=0.548, sum=1.599 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.5331198741930753\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=1065 (3)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=0.958, mean=1.562, max=1.997, sum=4.687 (3)\", \"tab\": \"General information\", \"score\": \"1.5624413145539906\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=1601.997, mean=1634.99, max=1693.155, sum=4904.969 (3)\", \"tab\": \"General information\", \"score\": \"1634.9896713615024\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=5.392, mean=6.771, max=8.33, sum=20.313 (3)\", \"tab\": \"General information\", \"score\": \"6.770892018779342\"}", - "NarrativeQA - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NarrativeQA - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=1.333 (2)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666667\"}", - "NarrativeQA - Stereotypes (gender)": "{\"description\": \"min=0.394, mean=0.427, max=0.45, sum=1.282 (3)\", \"tab\": \"Bias\", \"score\": \"0.42718253968253966\"}", - "NarrativeQA - Representation (race)": "{\"description\": \"min=0.373, mean=0.569, max=0.667, sum=1.706 (3)\", \"tab\": \"Bias\", \"score\": \"0.5686274509803922\"}", - "NarrativeQA - Representation (gender)": "{\"description\": \"min=0.152, mean=0.174, max=0.195, sum=0.521 (3)\", \"tab\": \"Bias\", \"score\": \"0.17371956530315583\"}", - "NarrativeQA - Toxic fraction": "{\"description\": \"min=0.014, mean=0.02, max=0.025, sum=0.059 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.01971830985915493\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (open-book)", - "source_data": { - "dataset_name": "NaturalQuestions (open-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (open-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.504, - "details": { - "description": "min=0.482, mean=0.504, max=0.516, sum=1.512 (3)", - "tab": "Accuracy", - "NaturalQuestions (closed-book) - ECE (10-bin)": "{\"description\": \"min=0.018, mean=0.026, max=0.036, sum=0.077 (3)\", \"tab\": \"Calibration\", \"score\": \"0.025653079993217736\"}", - "NaturalQuestions (open-book) - ECE (10-bin)": "{\"description\": \"min=0.129, mean=0.142, max=0.154, sum=0.425 (3)\", \"tab\": \"Calibration\", \"score\": \"0.14175015381424005\"}", - "NaturalQuestions (closed-book) - F1 (Robustness)": "{\"description\": \"min=0.097, mean=0.102, max=0.104, sum=0.305 (3)\", \"tab\": \"Robustness\", \"score\": \"0.10170384904294616\"}", - "NaturalQuestions (open-book) - F1 (Robustness)": "{\"description\": \"min=0.226, mean=0.266, max=0.292, sum=0.799 (3)\", \"tab\": \"Robustness\", \"score\": \"0.26631844818771483\"}", - "NaturalQuestions (closed-book) - F1 (Fairness)": "{\"description\": \"min=0.124, mean=0.126, max=0.127, sum=0.377 (3)\", \"tab\": \"Fairness\", \"score\": \"0.12565301660951664\"}", - "NaturalQuestions (open-book) - F1 (Fairness)": "{\"description\": \"min=0.41, mean=0.432, max=0.444, sum=1.297 (3)\", \"tab\": \"Fairness\", \"score\": \"0.4322127161835283\"}", - "NaturalQuestions (closed-book) - Denoised inference time (s)": "{\"description\": \"min=0.254, mean=0.259, max=0.265, sum=0.778 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.25938733203125103\"}", - "NaturalQuestions (open-book) - Denoised inference time (s)": "{\"description\": \"min=0.476, mean=0.535, max=0.583, sum=1.606 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.5353007499999998\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=109.191, mean=111.191, max=115.191, sum=333.573 (3)\", \"tab\": \"General information\", \"score\": \"111.19099999999999\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=4.823, mean=5.267, max=5.728, sum=15.801 (3)\", \"tab\": \"General information\", \"score\": \"5.267\"}", - "NaturalQuestions (closed-book) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.538, mean=4.633, max=4.715, sum=13.899 (3)\", \"tab\": \"General information\", \"score\": \"4.633\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.039, mean=0.039, max=0.039, sum=0.117 (3)\", \"tab\": \"General information\", \"score\": \"0.039\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1261.72, mean=1481.344, max=1608.455, sum=4444.032 (3)\", \"tab\": \"General information\", \"score\": \"1481.344\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=7.288, mean=9.101, max=11.307, sum=27.304 (3)\", \"tab\": \"General information\", \"score\": \"9.101333333333333\"}", - "NaturalQuestions (open-book) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NaturalQuestions (closed-book) - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - Stereotypes (gender)": "{\"description\": \"min=0.5, mean=0.5, max=0.5, sum=0.5 (1)\", \"tab\": \"Bias\", \"score\": \"0.5\"}", - "NaturalQuestions (closed-book) - Representation (race)": "{\"description\": \"min=0.419, mean=0.441, max=0.476, sum=1.323 (3)\", \"tab\": \"Bias\", \"score\": \"0.4410100926954859\"}", - "NaturalQuestions (closed-book) - Representation (gender)": "{\"description\": \"min=0.214, mean=0.251, max=0.3, sum=0.753 (3)\", \"tab\": \"Bias\", \"score\": \"0.2511387163561077\"}", - "NaturalQuestions (open-book) - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NaturalQuestions (open-book) - Stereotypes (gender)": "{\"description\": \"min=0.292, mean=0.354, max=0.417, sum=0.708 (2)\", \"tab\": \"Bias\", \"score\": \"0.3541666666666667\"}", - "NaturalQuestions (open-book) - Representation (race)": "{\"description\": \"min=0.289, mean=0.325, max=0.385, sum=0.974 (3)\", \"tab\": \"Bias\", \"score\": \"0.3247724272114516\"}", - "NaturalQuestions (open-book) - Representation (gender)": "{\"description\": \"min=0.202, mean=0.234, max=0.285, sum=0.703 (3)\", \"tab\": \"Bias\", \"score\": \"0.23429326676087917\"}", - "NaturalQuestions (closed-book) - Toxic fraction": "{\"description\": \"min=0, mean=0.0, max=0.001, sum=0.001 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.0003333333333333333\"}", - "NaturalQuestions (open-book) - Toxic fraction": "{\"description\": \"min=0.002, mean=0.002, max=0.003, sum=0.007 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.0023333333333333335\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "QuAC", - "source_data": { - "dataset_name": "QuAC", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on QuAC", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.279, - "details": { - "description": "min=0.273, mean=0.279, max=0.287, sum=0.838 (3)", - "tab": "Accuracy", - "QuAC - ECE (10-bin)": "{\"description\": \"min=0.042, mean=0.048, max=0.061, sum=0.145 (3)\", \"tab\": \"Calibration\", \"score\": \"0.04829561557428013\"}", - "QuAC - F1 (Robustness)": "{\"description\": \"min=0.12, mean=0.144, max=0.157, sum=0.432 (3)\", \"tab\": \"Robustness\", \"score\": \"0.14398518012537756\"}", - "QuAC - F1 (Fairness)": "{\"description\": \"min=0.186, mean=0.198, max=0.207, sum=0.593 (3)\", \"tab\": \"Fairness\", \"score\": \"0.19765650296002213\"}", - "QuAC - Denoised inference time (s)": "{\"description\": \"min=0.664, mean=0.735, max=0.771, sum=2.206 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.7354030888671875\"}", - "QuAC - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "QuAC - # train": "{\"description\": \"min=0.797, mean=0.881, max=0.969, sum=2.644 (3)\", \"tab\": \"General information\", \"score\": \"0.8813333333333334\"}", - "QuAC - truncated": "{\"description\": \"min=0.02, mean=0.02, max=0.02, sum=0.06 (3)\", \"tab\": \"General information\", \"score\": \"0.02\"}", - "QuAC - # prompt tokens": "{\"description\": \"min=1600.292, mean=1639.784, max=1661.675, sum=4919.353 (3)\", \"tab\": \"General information\", \"score\": \"1639.784333333333\"}", - "QuAC - # output tokens": "{\"description\": \"min=17.39, mean=23.531, max=27.056, sum=70.593 (3)\", \"tab\": \"General information\", \"score\": \"23.531000000000002\"}", - "QuAC - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "QuAC - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=2.0 (3)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666669\"}", - "QuAC - Stereotypes (gender)": "{\"description\": \"min=0.383, mean=0.412, max=0.431, sum=1.237 (3)\", \"tab\": \"Bias\", \"score\": \"0.41249828370040936\"}", - "QuAC - Representation (race)": "{\"description\": \"min=0.303, mean=0.357, max=0.392, sum=1.072 (3)\", \"tab\": \"Bias\", \"score\": \"0.35746080227329485\"}", - "QuAC - Representation (gender)": "{\"description\": \"min=0.233, mean=0.262, max=0.276, sum=0.786 (3)\", \"tab\": \"Bias\", \"score\": \"0.2618392019722732\"}", - "QuAC - Toxic fraction": "{\"description\": \"min=0.001, mean=0.002, max=0.002, sum=0.005 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.0016666666666666668\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "HellaSwag", - "source_data": { - "dataset_name": "HellaSwag", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on HellaSwag", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.706, - "details": { - "description": "min=0.706, mean=0.706, max=0.706, sum=0.706 (1)", - "tab": "Accuracy", - "HellaSwag - ECE (10-bin)": "{\"description\": \"min=0.271, mean=0.271, max=0.271, sum=0.271 (1)\", \"tab\": \"Calibration\", \"score\": \"0.2707363482287178\"}", - "HellaSwag - EM (Robustness)": "{\"description\": \"min=0.651, mean=0.651, max=0.651, sum=0.651 (1)\", \"tab\": \"Robustness\", \"score\": \"0.651\"}", - "HellaSwag - EM (Fairness)": "{\"description\": \"min=0.525, mean=0.525, max=0.525, sum=0.525 (1)\", \"tab\": \"Fairness\", \"score\": \"0.525\"}", - "HellaSwag - Denoised inference time (s)": "{\"description\": \"min=0.204, mean=0.204, max=0.204, sum=0.204 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.20370158203125027\"}", - "HellaSwag - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "HellaSwag - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "HellaSwag - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "HellaSwag - # prompt tokens": "{\"description\": \"min=88.855, mean=88.855, max=88.855, sum=88.855 (1)\", \"tab\": \"General information\", \"score\": \"88.855\"}", - "HellaSwag - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "HellaSwag - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.496, - "details": { - "description": "min=0.496, mean=0.496, max=0.496, sum=0.496 (1)", - "tab": "Accuracy", - "OpenbookQA - ECE (10-bin)": "{\"description\": \"min=0.275, mean=0.275, max=0.275, sum=0.275 (1)\", \"tab\": \"Calibration\", \"score\": \"0.27530956848832144\"}", - "OpenbookQA - EM (Robustness)": "{\"description\": \"min=0.382, mean=0.382, max=0.382, sum=0.382 (1)\", \"tab\": \"Robustness\", \"score\": \"0.382\"}", - "OpenbookQA - EM (Fairness)": "{\"description\": \"min=0.42, mean=0.42, max=0.42, sum=0.42 (1)\", \"tab\": \"Fairness\", \"score\": \"0.42\"}", - "OpenbookQA - Denoised inference time (s)": "{\"description\": \"min=0.187, mean=0.187, max=0.187, sum=0.187 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.1870674140625\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=5.358, mean=5.358, max=5.358, sum=5.358 (1)\", \"tab\": \"General information\", \"score\": \"5.358\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "TruthfulQA", - "source_data": { - "dataset_name": "TruthfulQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on TruthfulQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.19, - "details": { - "description": "min=0.176, mean=0.19, max=0.203, sum=0.57 (3)", - "tab": "Accuracy", - "TruthfulQA - ECE (10-bin)": "{\"description\": \"min=0.082, mean=0.094, max=0.109, sum=0.282 (3)\", \"tab\": \"Calibration\", \"score\": \"0.09386032214108035\"}", - "TruthfulQA - EM (Robustness)": "{\"description\": \"min=0.127, mean=0.149, max=0.168, sum=0.448 (3)\", \"tab\": \"Robustness\", \"score\": \"0.1493374108053007\"}", - "TruthfulQA - EM (Fairness)": "{\"description\": \"min=0.154, mean=0.174, max=0.19, sum=0.521 (3)\", \"tab\": \"Fairness\", \"score\": \"0.17380224260958207\"}", - "TruthfulQA - Denoised inference time (s)": "{\"description\": \"min=0.287, mean=0.287, max=0.288, sum=0.862 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.28723167974722846\"}", - "TruthfulQA - # eval": "{\"description\": \"min=654, mean=654, max=654, sum=1962 (3)\", \"tab\": \"General information\", \"score\": \"654.0\"}", - "TruthfulQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "TruthfulQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "TruthfulQA - # prompt tokens": "{\"description\": \"min=505.315, mean=514.648, max=532.315, sum=1543.945 (3)\", \"tab\": \"General information\", \"score\": \"514.6483180428135\"}", - "TruthfulQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=3 (3)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "TruthfulQA - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MS MARCO (TREC)", - "source_data": { - "dataset_name": "MS MARCO (TREC)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "NDCG@10 on MS MARCO (TREC)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.374, - "details": { - "description": "min=0.337, mean=0.374, max=0.416, sum=1.122 (3)", - "tab": "Accuracy", - "MS MARCO (regular) - RR@10 (Robustness)": "{\"description\": \"min=0.101, mean=0.109, max=0.12, sum=0.326 (3)\", \"tab\": \"Robustness\", \"score\": \"0.10871957671957677\"}", - "MS MARCO (TREC) - NDCG@10 (Robustness)": "{\"description\": \"min=0.294, mean=0.315, max=0.354, sum=0.945 (3)\", \"tab\": \"Robustness\", \"score\": \"0.31504083631376195\"}", - "MS MARCO (regular) - RR@10 (Fairness)": "{\"description\": \"min=0.126, mean=0.132, max=0.136, sum=0.396 (3)\", \"tab\": \"Fairness\", \"score\": \"0.13183915343915345\"}", - "MS MARCO (TREC) - NDCG@10 (Fairness)": "{\"description\": \"min=0.321, mean=0.357, max=0.398, sum=1.072 (3)\", \"tab\": \"Fairness\", \"score\": \"0.35726921379791293\"}", - "MS MARCO (regular) - Denoised inference time (s)": "{\"description\": \"min=0.286, mean=0.289, max=0.293, sum=0.867 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.28909981347656255\"}", - "MS MARCO (TREC) - Denoised inference time (s)": "{\"description\": \"min=0.285, mean=0.288, max=0.29, sum=0.864 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.28804701126453486\"}", - "MS MARCO (regular) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "MS MARCO (regular) - # train": "{\"description\": \"min=2, mean=2, max=2, sum=6 (3)\", \"tab\": \"General information\", \"score\": \"2.0\"}", - "MS MARCO (regular) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MS MARCO (regular) - # prompt tokens": "{\"description\": \"min=497.281, mean=536.614, max=583.281, sum=1609.843 (3)\", \"tab\": \"General information\", \"score\": \"536.6143333333333\"}", - "MS MARCO (regular) - # output tokens": "{\"description\": \"min=1, mean=1.005, max=1.013, sum=3.014 (3)\", \"tab\": \"General information\", \"score\": \"1.0046666666666666\"}", - "MS MARCO (regular) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "MS MARCO (TREC) - # eval": "{\"description\": \"min=43, mean=43, max=43, sum=129 (3)\", \"tab\": \"General information\", \"score\": \"43.0\"}", - "MS MARCO (TREC) - # train": "{\"description\": \"min=2, mean=2, max=2, sum=6 (3)\", \"tab\": \"General information\", \"score\": \"2.0\"}", - "MS MARCO (TREC) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MS MARCO (TREC) - # prompt tokens": "{\"description\": \"min=480.163, mean=519.496, max=566.163, sum=1558.488 (3)\", \"tab\": \"General information\", \"score\": \"519.4961240310078\"}", - "MS MARCO (TREC) - # output tokens": "{\"description\": \"min=1, mean=1.016, max=1.023, sum=3.047 (3)\", \"tab\": \"General information\", \"score\": \"1.0155038759689923\"}", - "MS MARCO (TREC) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "MS MARCO (regular) - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "MS MARCO (TREC) - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CNN/DailyMail", - "source_data": { - "dataset_name": "CNN/DailyMail", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on CNN/DailyMail", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.077, - "details": { - "description": "min=0.03, mean=0.077, max=0.111, sum=0.459 (6)", - "tab": "Accuracy", - "CNN/DailyMail - Denoised inference time (s)": "{\"description\": \"min=1.073, mean=1.2, max=1.325, sum=7.2 (6)\", \"tab\": \"Efficiency\", \"score\": \"1.199950748558208\"}", - "CNN/DailyMail - # eval": "{\"description\": \"min=466, mean=466, max=466, sum=2796 (6)\", \"tab\": \"General information\", \"score\": \"466.0\"}", - "CNN/DailyMail - # train": "{\"description\": \"min=5, mean=5, max=5, sum=30 (6)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CNN/DailyMail - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (6)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CNN/DailyMail - # prompt tokens": "{\"description\": \"min=1555.036, mean=1575.036, max=1602.036, sum=9450.219 (6)\", \"tab\": \"General information\", \"score\": \"1575.0364806866953\"}", - "CNN/DailyMail - # output tokens": "{\"description\": \"min=52.893, mean=63.193, max=73.206, sum=379.159 (6)\", \"tab\": \"General information\", \"score\": \"63.1931330472103\"}", - "CNN/DailyMail - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=18 (6)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "CNN/DailyMail - Stereotypes (race)": "{\"description\": \"min=0.644, mean=0.659, max=0.667, sum=3.956 (6)\", \"tab\": \"Bias\", \"score\": \"0.6592592592592593\"}", - "CNN/DailyMail - Stereotypes (gender)": "{\"description\": \"min=0.402, mean=0.44, max=0.476, sum=2.641 (6)\", \"tab\": \"Bias\", \"score\": \"0.44008624507065996\"}", - "CNN/DailyMail - Representation (race)": "{\"description\": \"min=0.285, mean=0.304, max=0.333, sum=1.825 (6)\", \"tab\": \"Bias\", \"score\": \"0.30422478269658376\"}", - "CNN/DailyMail - Representation (gender)": "{\"description\": \"min=0.127, mean=0.173, max=0.229, sum=1.037 (6)\", \"tab\": \"Bias\", \"score\": \"0.17278322431241475\"}", - "CNN/DailyMail - Toxic fraction": "{\"description\": \"min=0, mean=0.001, max=0.002, sum=0.004 (6)\", \"tab\": \"Toxicity\", \"score\": \"0.000715307582260372\"}", - "CNN/DailyMail - SummaC": "{\"description\": \"min=-0.174, mean=0.229, max=0.443, sum=0.686 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.22880441457511005\"}", - "CNN/DailyMail - QAFactEval": "{\"description\": \"min=4.552, mean=4.664, max=4.795, sum=27.982 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"4.663724611238682\"}", - "CNN/DailyMail - BERTScore (F1)": "{\"description\": \"min=0.008, mean=0.115, max=0.197, sum=0.346 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.11522739683384077\"}", - "CNN/DailyMail - Coverage": "{\"description\": \"min=0.482, mean=0.799, max=0.965, sum=4.793 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"0.7988868167525552\"}", - "CNN/DailyMail - Density": "{\"description\": \"min=9.34, mean=22.176, max=32.926, sum=133.058 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"22.17629615230217\"}", - "CNN/DailyMail - Compression": "{\"description\": \"min=11.915, mean=13.154, max=15.457, sum=78.926 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"13.15437099106955\"}", - "CNN/DailyMail - HumanEval-faithfulness": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-relevance": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-coherence": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "XSUM", - "source_data": { - "dataset_name": "XSUM", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on XSUM", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.087, - "details": { - "description": "min=0.086, mean=0.087, max=0.09, sum=0.524 (6)", - "tab": "Accuracy", - "XSUM - Denoised inference time (s)": "{\"description\": \"min=0.717, mean=0.724, max=0.732, sum=4.343 (6)\", \"tab\": \"Efficiency\", \"score\": \"0.7239030526061776\"}", - "XSUM - # eval": "{\"description\": \"min=518, mean=518, max=518, sum=3108 (6)\", \"tab\": \"General information\", \"score\": \"518.0\"}", - "XSUM - # train": "{\"description\": \"min=4.996, mean=4.998, max=5, sum=29.988 (6)\", \"tab\": \"General information\", \"score\": \"4.998069498069498\"}", - "XSUM - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (6)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "XSUM - # prompt tokens": "{\"description\": \"min=1484.608, mean=1537.452, max=1572.616, sum=9224.71 (6)\", \"tab\": \"General information\", \"score\": \"1537.4517374517375\"}", - "XSUM - # output tokens": "{\"description\": \"min=23.498, mean=24.055, max=24.463, sum=144.328 (6)\", \"tab\": \"General information\", \"score\": \"24.054697554697555\"}", - "XSUM - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=18 (6)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "XSUM - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=4 (6)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "XSUM - Stereotypes (gender)": "{\"description\": \"min=0.447, mean=0.461, max=0.481, sum=2.765 (6)\", \"tab\": \"Bias\", \"score\": \"0.46086088123125163\"}", - "XSUM - Representation (race)": "{\"description\": \"min=0.449, mean=0.498, max=0.579, sum=2.99 (6)\", \"tab\": \"Bias\", \"score\": \"0.4982964658021866\"}", - "XSUM - Representation (gender)": "{\"description\": \"min=0.167, mean=0.186, max=0.198, sum=1.115 (6)\", \"tab\": \"Bias\", \"score\": \"0.18582940251572325\"}", - "XSUM - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (6)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}", - "XSUM - SummaC": "{\"description\": \"min=-0.17, mean=-0.159, max=-0.142, sum=-0.477 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"-0.1589340320425144\"}", - "XSUM - QAFactEval": "{\"description\": \"min=3.197, mean=3.223, max=3.258, sum=19.336 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"3.2227135293221596\"}", - "XSUM - BERTScore (F1)": "{\"description\": \"min=0.364, mean=0.367, max=0.371, sum=1.102 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.36729036225155814\"}", - "XSUM - Coverage": "{\"description\": \"min=0.84, mean=0.847, max=0.855, sum=5.083 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"0.8472154184001573\"}", - "XSUM - Density": "{\"description\": \"min=4.485, mean=4.754, max=4.928, sum=28.525 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"4.7541975208526\"}", - "XSUM - Compression": "{\"description\": \"min=19.527, mean=19.748, max=20.169, sum=118.491 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"19.748450478665102\"}", - "XSUM - HumanEval-faithfulness": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-relevance": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-coherence": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "IMDB", - "source_data": { - "dataset_name": "IMDB", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on IMDB", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.935, - "details": { - "description": "min=0.917, mean=0.935, max=0.947, sum=2.805 (3)", - "tab": "Accuracy", - "IMDB - ECE (10-bin)": "{\"description\": \"min=0.335, mean=0.36, max=0.394, sum=1.08 (3)\", \"tab\": \"Calibration\", \"score\": \"0.360155737743892\"}", - "IMDB - EM (Robustness)": "{\"description\": \"min=0.878, mean=0.889, max=0.897, sum=2.666 (3)\", \"tab\": \"Robustness\", \"score\": \"0.8886666666666666\"}", - "IMDB - EM (Fairness)": "{\"description\": \"min=0.896, mean=0.918, max=0.936, sum=2.753 (3)\", \"tab\": \"Fairness\", \"score\": \"0.9176666666666667\"}", - "IMDB - Denoised inference time (s)": "{\"description\": \"min=0.404, mean=0.452, max=0.489, sum=1.355 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.45160390852864607\"}", - "IMDB - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "IMDB - # train": "{\"description\": \"min=2.903, mean=4.229, max=4.983, sum=12.688 (3)\", \"tab\": \"General information\", \"score\": \"4.229333333333333\"}", - "IMDB - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IMDB - # prompt tokens": "{\"description\": \"min=1283.038, mean=1562.808, max=1784.2, sum=4688.425 (3)\", \"tab\": \"General information\", \"score\": \"1562.8083333333334\"}", - "IMDB - # output tokens": "{\"description\": \"min=1, mean=1.003, max=1.01, sum=3.01 (3)\", \"tab\": \"General information\", \"score\": \"1.0033333333333332\"}", - "IMDB - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "IMDB - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CivilComments", - "source_data": { - "dataset_name": "CivilComments", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on CivilComments", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.504, - "details": { - "description": "min=0, mean=0.504, max=1, sum=27.205 (54)", - "tab": "Accuracy", - "CivilComments - ECE (10-bin)": "{\"description\": \"min=0.176, mean=0.459, max=0.641, sum=24.77 (54)\", \"tab\": \"Calibration\", \"score\": \"0.45870054566126006\"}", - "CivilComments - EM (Robustness)": "{\"description\": \"min=0, mean=0.136, max=0.736, sum=7.362 (54)\", \"tab\": \"Robustness\", \"score\": \"0.13632694985889793\"}", - "CivilComments - EM (Fairness)": "{\"description\": \"min=0, mean=0.489, max=1, sum=26.387 (54)\", \"tab\": \"Fairness\", \"score\": \"0.48864261081744575\"}", - "CivilComments - Denoised inference time (s)": "{\"description\": \"min=0.262, mean=0.321, max=0.405, sum=17.316 (54)\", \"tab\": \"Efficiency\", \"score\": \"0.32067323239104795\"}", - "CivilComments - # eval": "{\"description\": \"min=74, mean=371.556, max=683, sum=20064 (54)\", \"tab\": \"General information\", \"score\": \"371.55555555555554\"}", - "CivilComments - # train": "{\"description\": \"min=5, mean=5, max=5, sum=270 (54)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CivilComments - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (54)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CivilComments - # prompt tokens": "{\"description\": \"min=362.293, mean=732.514, max=1288.441, sum=39555.782 (54)\", \"tab\": \"General information\", \"score\": \"732.5144825548033\"}", - "CivilComments - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=54 (54)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "CivilComments - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=162 (54)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "CivilComments - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (54)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "RAFT", - "source_data": { - "dataset_name": "RAFT", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on RAFT", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.52, - "details": { - "description": "min=0.125, mean=0.52, max=0.975, sum=17.15 (33)", - "tab": "Accuracy", - "RAFT - ECE (10-bin)": "{\"description\": \"min=0.151, mean=0.304, max=0.849, sum=10.027 (33)\", \"tab\": \"Calibration\", \"score\": \"0.3038351531350353\"}", - "RAFT - EM (Robustness)": "{\"description\": \"min=0, mean=0.385, max=0.975, sum=12.7 (33)\", \"tab\": \"Robustness\", \"score\": \"0.3848484848484848\"}", - "RAFT - EM (Fairness)": "{\"description\": \"min=0.125, mean=0.5, max=0.975, sum=16.5 (33)\", \"tab\": \"Fairness\", \"score\": \"0.5\"}", - "RAFT - Denoised inference time (s)": "{\"description\": \"min=0.244, mean=0.358, max=0.532, sum=11.817 (33)\", \"tab\": \"Efficiency\", \"score\": \"0.3580963386304451\"}", - "RAFT - # eval": "{\"description\": \"min=40, mean=40, max=40, sum=1320 (33)\", \"tab\": \"General information\", \"score\": \"40.0\"}", - "RAFT - # train": "{\"description\": \"min=0, mean=4.557, max=5, sum=150.375 (33)\", \"tab\": \"General information\", \"score\": \"4.556818181818182\"}", - "RAFT - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (33)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "RAFT - # prompt tokens": "{\"description\": \"min=270.325, mean=814.446, max=1777.025, sum=26876.725 (33)\", \"tab\": \"General information\", \"score\": \"814.446212121212\"}", - "RAFT - # output tokens": "{\"description\": \"min=0.225, mean=2.965, max=6.15, sum=97.85 (33)\", \"tab\": \"General information\", \"score\": \"2.965151515151515\"}", - "RAFT - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=99 (33)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "RAFT - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (33)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/cohere_cohere-medium-v20221108-6.1b.json b/data/models/cohere_cohere-medium-v20221108-6.1b.json deleted file mode 100644 index 3144bf35529cf7245080efd34354bca6ffd12b53..0000000000000000000000000000000000000000 --- a/data/models/cohere_cohere-medium-v20221108-6.1b.json +++ /dev/null @@ -1,674 +0,0 @@ -{ - "model_info": { - "name": "Cohere medium v20221108 6.1B", - "id": "cohere/Cohere-medium-v20221108-6.1B", - "developer": "cohere", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_classic/cohere_Cohere-medium-v20221108-6.1B/1774096308.339228", - "retrieved_timestamp": "1774096308.339228", - "source_metadata": { - "source_name": "helm_classic", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_classic", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_classic", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperform on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.312, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Calibration": "{\"description\": \"\", \"tab\": \"Calibration\", \"score\": \"0.6010395609917657\"}", - "Mean win rate - Robustness": "{\"description\": \"\", \"tab\": \"Robustness\", \"score\": \"0.26965587249235745\"}", - "Mean win rate - Fairness": "{\"description\": \"\", \"tab\": \"Fairness\", \"score\": \"0.339964744191663\"}", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}", - "Mean win rate - Bias": "{\"description\": \"\", \"tab\": \"Bias\", \"score\": \"0.5558769690348637\"}", - "Mean win rate - Toxicity": "{\"description\": \"\", \"tab\": \"Toxicity\", \"score\": \"0.6328714495381162\"}", - "Mean win rate - Summarization metrics": "{\"description\": \"\", \"tab\": \"Summarization metrics\", \"score\": \"0.506578947368421\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.254, - "details": { - "description": "min=0.18, mean=0.254, max=0.32, sum=3.806 (15)", - "tab": "Accuracy", - "MMLU - ECE (10-bin)": "{\"description\": \"min=0.055, mean=0.113, max=0.167, sum=1.691 (15)\", \"tab\": \"Calibration\", \"score\": \"0.11272299343238619\"}", - "MMLU - EM (Robustness)": "{\"description\": \"min=0.15, mean=0.207, max=0.25, sum=3.1 (15)\", \"tab\": \"Robustness\", \"score\": \"0.20667836257309943\"}", - "MMLU - EM (Fairness)": "{\"description\": \"min=0.14, mean=0.22, max=0.3, sum=3.299 (15)\", \"tab\": \"Fairness\", \"score\": \"0.21994152046783624\"}", - "MMLU - Denoised inference time (s)": "{\"description\": \"5 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=1542 (15)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=75 (15)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (15)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=372.75, mean=481.26, max=628.421, sum=7218.903 (15)\", \"tab\": \"General information\", \"score\": \"481.2602105263158\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=15 (15)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "MMLU - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=45 (15)\", \"tab\": \"General information\", \"score\": \"3.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "BoolQ", - "source_data": { - "dataset_name": "BoolQ", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on BoolQ", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7, - "details": { - "description": "min=0.693, mean=0.7, max=0.704, sum=2.1 (3)", - "tab": "Accuracy", - "BoolQ - ECE (10-bin)": "{\"description\": \"min=0.088, mean=0.095, max=0.105, sum=0.284 (3)\", \"tab\": \"Calibration\", \"score\": \"0.09459272512018041\"}", - "BoolQ - EM (Robustness)": "{\"description\": \"min=0.508, mean=0.54, max=0.568, sum=1.62 (3)\", \"tab\": \"Robustness\", \"score\": \"0.54\"}", - "BoolQ - EM (Fairness)": "{\"description\": \"min=0.626, mean=0.642, max=0.652, sum=1.925 (3)\", \"tab\": \"Fairness\", \"score\": \"0.6416666666666667\"}", - "BoolQ - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "BoolQ - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "BoolQ - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "BoolQ - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "BoolQ - # prompt tokens": "{\"description\": \"min=669.307, mean=925.307, max=1269.307, sum=2775.921 (3)\", \"tab\": \"General information\", \"score\": \"925.3070000000001\"}", - "BoolQ - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=3 (3)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "BoolQ - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "BoolQ - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.61, - "details": { - "description": "min=0.57, mean=0.61, max=0.642, sum=1.831 (3)", - "tab": "Accuracy", - "NarrativeQA - ECE (10-bin)": "{\"description\": \"min=0.027, mean=0.028, max=0.03, sum=0.085 (3)\", \"tab\": \"Calibration\", \"score\": \"0.02834267942109429\"}", - "NarrativeQA - F1 (Robustness)": "{\"description\": \"min=0.265, mean=0.296, max=0.321, sum=0.888 (3)\", \"tab\": \"Robustness\", \"score\": \"0.2960125312478054\"}", - "NarrativeQA - F1 (Fairness)": "{\"description\": \"min=0.441, mean=0.497, max=0.537, sum=1.491 (3)\", \"tab\": \"Fairness\", \"score\": \"0.49703931741598933\"}", - "NarrativeQA - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=1065 (3)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=0.958, mean=1.562, max=1.997, sum=4.687 (3)\", \"tab\": \"General information\", \"score\": \"1.5624413145539906\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=1601.997, mean=1634.99, max=1693.155, sum=4904.969 (3)\", \"tab\": \"General information\", \"score\": \"1634.9896713615024\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=5.544, mean=7.144, max=9.065, sum=21.431 (3)\", \"tab\": \"General information\", \"score\": \"7.143661971830986\"}", - "NarrativeQA - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NarrativeQA - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NarrativeQA - Stereotypes (gender)": "{\"description\": \"min=0.417, mean=0.441, max=0.469, sum=1.323 (3)\", \"tab\": \"Bias\", \"score\": \"0.44097222222222215\"}", - "NarrativeQA - Representation (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=2 (3)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "NarrativeQA - Representation (gender)": "{\"description\": \"min=0.15, mean=0.181, max=0.213, sum=0.543 (3)\", \"tab\": \"Bias\", \"score\": \"0.18104985015382555\"}", - "NarrativeQA - Toxic fraction": "{\"description\": \"min=0.008, mean=0.011, max=0.014, sum=0.034 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.011267605633802818\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (open-book)", - "source_data": { - "dataset_name": "NaturalQuestions (open-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (open-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.517, - "details": { - "description": "min=0.506, mean=0.517, max=0.536, sum=1.551 (3)", - "tab": "Accuracy", - "NaturalQuestions (closed-book) - ECE (10-bin)": "{\"description\": \"min=0.006, mean=0.015, max=0.02, sum=0.044 (3)\", \"tab\": \"Calibration\", \"score\": \"0.01475928497137971\"}", - "NaturalQuestions (open-book) - ECE (10-bin)": "{\"description\": \"min=0.181, mean=0.233, max=0.27, sum=0.698 (3)\", \"tab\": \"Calibration\", \"score\": \"0.2327617365925914\"}", - "NaturalQuestions (closed-book) - F1 (Robustness)": "{\"description\": \"min=0.099, mean=0.105, max=0.11, sum=0.314 (3)\", \"tab\": \"Robustness\", \"score\": \"0.10457862657700777\"}", - "NaturalQuestions (open-book) - F1 (Robustness)": "{\"description\": \"min=0.164, mean=0.222, max=0.282, sum=0.665 (3)\", \"tab\": \"Robustness\", \"score\": \"0.22177043436006846\"}", - "NaturalQuestions (closed-book) - F1 (Fairness)": "{\"description\": \"min=0.142, mean=0.149, max=0.157, sum=0.447 (3)\", \"tab\": \"Fairness\", \"score\": \"0.14913779301489424\"}", - "NaturalQuestions (open-book) - F1 (Fairness)": "{\"description\": \"min=0.431, mean=0.45, max=0.473, sum=1.349 (3)\", \"tab\": \"Fairness\", \"score\": \"0.44971949324423194\"}", - "NaturalQuestions (closed-book) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NaturalQuestions (open-book) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=109.191, mean=111.191, max=115.191, sum=333.573 (3)\", \"tab\": \"General information\", \"score\": \"111.19099999999999\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=6.631, mean=6.745, max=6.831, sum=20.236 (3)\", \"tab\": \"General information\", \"score\": \"6.745333333333334\"}", - "NaturalQuestions (closed-book) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.538, mean=4.633, max=4.715, sum=13.899 (3)\", \"tab\": \"General information\", \"score\": \"4.633\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.039, mean=0.039, max=0.039, sum=0.117 (3)\", \"tab\": \"General information\", \"score\": \"0.039\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1261.72, mean=1481.344, max=1608.455, sum=4444.032 (3)\", \"tab\": \"General information\", \"score\": \"1481.344\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=7.485, mean=8.419, max=9.746, sum=25.256 (3)\", \"tab\": \"General information\", \"score\": \"8.418666666666667\"}", - "NaturalQuestions (open-book) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NaturalQuestions (closed-book) - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - Stereotypes (gender)": "{\"description\": \"min=0.357, mean=0.45, max=0.5, sum=1.349 (3)\", \"tab\": \"Bias\", \"score\": \"0.44969278033794163\"}", - "NaturalQuestions (closed-book) - Representation (race)": "{\"description\": \"min=0.382, mean=0.451, max=0.504, sum=1.353 (3)\", \"tab\": \"Bias\", \"score\": \"0.4511619362542481\"}", - "NaturalQuestions (closed-book) - Representation (gender)": "{\"description\": \"min=0.173, mean=0.314, max=0.386, sum=0.942 (3)\", \"tab\": \"Bias\", \"score\": \"0.3140619884317363\"}", - "NaturalQuestions (open-book) - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NaturalQuestions (open-book) - Stereotypes (gender)": "{\"description\": \"min=0.233, mean=0.308, max=0.35, sum=0.923 (3)\", \"tab\": \"Bias\", \"score\": \"0.30777777777777776\"}", - "NaturalQuestions (open-book) - Representation (race)": "{\"description\": \"min=0.421, mean=0.452, max=0.476, sum=1.356 (3)\", \"tab\": \"Bias\", \"score\": \"0.4519283176992704\"}", - "NaturalQuestions (open-book) - Representation (gender)": "{\"description\": \"min=0.056, mean=0.061, max=0.069, sum=0.184 (3)\", \"tab\": \"Bias\", \"score\": \"0.06120328473269649\"}", - "NaturalQuestions (closed-book) - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}", - "NaturalQuestions (open-book) - Toxic fraction": "{\"description\": \"min=0, mean=0.002, max=0.003, sum=0.005 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.0016666666666666668\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "QuAC", - "source_data": { - "dataset_name": "QuAC", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on QuAC", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.314, - "details": { - "description": "min=0.297, mean=0.314, max=0.328, sum=0.942 (3)", - "tab": "Accuracy", - "QuAC - ECE (10-bin)": "{\"description\": \"min=0.029, mean=0.041, max=0.062, sum=0.124 (3)\", \"tab\": \"Calibration\", \"score\": \"0.04129669890931466\"}", - "QuAC - F1 (Robustness)": "{\"description\": \"min=0.127, mean=0.152, max=0.171, sum=0.456 (3)\", \"tab\": \"Robustness\", \"score\": \"0.15189850694469184\"}", - "QuAC - F1 (Fairness)": "{\"description\": \"min=0.208, mean=0.229, max=0.244, sum=0.688 (3)\", \"tab\": \"Fairness\", \"score\": \"0.22939607207059778\"}", - "QuAC - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "QuAC - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "QuAC - # train": "{\"description\": \"min=0.797, mean=0.881, max=0.969, sum=2.644 (3)\", \"tab\": \"General information\", \"score\": \"0.8813333333333334\"}", - "QuAC - truncated": "{\"description\": \"min=0.02, mean=0.02, max=0.02, sum=0.06 (3)\", \"tab\": \"General information\", \"score\": \"0.02\"}", - "QuAC - # prompt tokens": "{\"description\": \"min=1600.292, mean=1639.784, max=1661.675, sum=4919.353 (3)\", \"tab\": \"General information\", \"score\": \"1639.784333333333\"}", - "QuAC - # output tokens": "{\"description\": \"min=18.756, mean=22.84, max=26.573, sum=68.519 (3)\", \"tab\": \"General information\", \"score\": \"22.83966666666667\"}", - "QuAC - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "QuAC - Stereotypes (race)": "{\"description\": \"min=0.619, mean=0.651, max=0.667, sum=1.952 (3)\", \"tab\": \"Bias\", \"score\": \"0.6507936507936508\"}", - "QuAC - Stereotypes (gender)": "{\"description\": \"min=0.436, mean=0.441, max=0.444, sum=1.322 (3)\", \"tab\": \"Bias\", \"score\": \"0.4407764298624513\"}", - "QuAC - Representation (race)": "{\"description\": \"min=0.345, mean=0.353, max=0.359, sum=1.06 (3)\", \"tab\": \"Bias\", \"score\": \"0.35330965547213355\"}", - "QuAC - Representation (gender)": "{\"description\": \"min=0.248, mean=0.251, max=0.255, sum=0.753 (3)\", \"tab\": \"Bias\", \"score\": \"0.2510004319407244\"}", - "QuAC - Toxic fraction": "{\"description\": \"min=0.002, mean=0.002, max=0.002, sum=0.006 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.002\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "HellaSwag", - "source_data": { - "dataset_name": "HellaSwag", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on HellaSwag", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.726, - "details": { - "description": "min=0.726, mean=0.726, max=0.726, sum=0.726 (1)", - "tab": "Accuracy", - "HellaSwag - ECE (10-bin)": "{\"description\": \"min=0.281, mean=0.281, max=0.281, sum=0.281 (1)\", \"tab\": \"Calibration\", \"score\": \"0.2814688190554964\"}", - "HellaSwag - EM (Robustness)": "{\"description\": \"min=0.687, mean=0.687, max=0.687, sum=0.687 (1)\", \"tab\": \"Robustness\", \"score\": \"0.687\"}", - "HellaSwag - EM (Fairness)": "{\"description\": \"min=0.567, mean=0.567, max=0.567, sum=0.567 (1)\", \"tab\": \"Fairness\", \"score\": \"0.567\"}", - "HellaSwag - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "HellaSwag - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "HellaSwag - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "HellaSwag - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "HellaSwag - # prompt tokens": "{\"description\": \"min=88.855, mean=88.855, max=88.855, sum=88.855 (1)\", \"tab\": \"General information\", \"score\": \"88.855\"}", - "HellaSwag - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "HellaSwag - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.538, - "details": { - "description": "min=0.538, mean=0.538, max=0.538, sum=0.538 (1)", - "tab": "Accuracy", - "OpenbookQA - ECE (10-bin)": "{\"description\": \"min=0.23, mean=0.23, max=0.23, sum=0.23 (1)\", \"tab\": \"Calibration\", \"score\": \"0.2303402231123461\"}", - "OpenbookQA - EM (Robustness)": "{\"description\": \"min=0.414, mean=0.414, max=0.414, sum=0.414 (1)\", \"tab\": \"Robustness\", \"score\": \"0.414\"}", - "OpenbookQA - EM (Fairness)": "{\"description\": \"min=0.44, mean=0.44, max=0.44, sum=0.44 (1)\", \"tab\": \"Fairness\", \"score\": \"0.44\"}", - "OpenbookQA - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=5.358, mean=5.358, max=5.358, sum=5.358 (1)\", \"tab\": \"General information\", \"score\": \"5.358\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "TruthfulQA", - "source_data": { - "dataset_name": "TruthfulQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on TruthfulQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.215, - "details": { - "description": "min=0.19, mean=0.215, max=0.237, sum=0.645 (3)", - "tab": "Accuracy", - "TruthfulQA - ECE (10-bin)": "{\"description\": \"min=0.057, mean=0.08, max=0.106, sum=0.24 (3)\", \"tab\": \"Calibration\", \"score\": \"0.07993899696218487\"}", - "TruthfulQA - EM (Robustness)": "{\"description\": \"min=0.156, mean=0.17, max=0.19, sum=0.511 (3)\", \"tab\": \"Robustness\", \"score\": \"0.17023445463812437\"}", - "TruthfulQA - EM (Fairness)": "{\"description\": \"min=0.156, mean=0.182, max=0.205, sum=0.546 (3)\", \"tab\": \"Fairness\", \"score\": \"0.18195718654434248\"}", - "TruthfulQA - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "TruthfulQA - # eval": "{\"description\": \"min=654, mean=654, max=654, sum=1962 (3)\", \"tab\": \"General information\", \"score\": \"654.0\"}", - "TruthfulQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "TruthfulQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "TruthfulQA - # prompt tokens": "{\"description\": \"min=505.315, mean=514.648, max=532.315, sum=1543.945 (3)\", \"tab\": \"General information\", \"score\": \"514.6483180428135\"}", - "TruthfulQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=3 (3)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "TruthfulQA - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MS MARCO (TREC)", - "source_data": { - "dataset_name": "MS MARCO (TREC)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "NDCG@10 on MS MARCO (TREC)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.373, - "details": { - "description": "min=0.329, mean=0.373, max=0.4, sum=1.118 (3)", - "tab": "Accuracy", - "MS MARCO (regular) - RR@10 (Robustness)": "{\"description\": \"min=0.11, mean=0.13, max=0.144, sum=0.389 (3)\", \"tab\": \"Robustness\", \"score\": \"0.12963544973544971\"}", - "MS MARCO (TREC) - NDCG@10 (Robustness)": "{\"description\": \"min=0.265, mean=0.314, max=0.339, sum=0.942 (3)\", \"tab\": \"Robustness\", \"score\": \"0.3140445596258007\"}", - "MS MARCO (regular) - RR@10 (Fairness)": "{\"description\": \"min=0.123, mean=0.145, max=0.162, sum=0.436 (3)\", \"tab\": \"Fairness\", \"score\": \"0.1454550264550264\"}", - "MS MARCO (TREC) - NDCG@10 (Fairness)": "{\"description\": \"min=0.311, mean=0.353, max=0.384, sum=1.058 (3)\", \"tab\": \"Fairness\", \"score\": \"0.35251421077315565\"}", - "MS MARCO (regular) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MS MARCO (TREC) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MS MARCO (regular) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "MS MARCO (regular) - # train": "{\"description\": \"min=2, mean=2, max=2, sum=6 (3)\", \"tab\": \"General information\", \"score\": \"2.0\"}", - "MS MARCO (regular) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MS MARCO (regular) - # prompt tokens": "{\"description\": \"min=497.281, mean=536.614, max=583.281, sum=1609.843 (3)\", \"tab\": \"General information\", \"score\": \"536.6143333333333\"}", - "MS MARCO (regular) - # output tokens": "{\"description\": \"min=1, mean=1.005, max=1.008, sum=3.015 (3)\", \"tab\": \"General information\", \"score\": \"1.005\"}", - "MS MARCO (regular) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "MS MARCO (TREC) - # eval": "{\"description\": \"min=43, mean=43, max=43, sum=129 (3)\", \"tab\": \"General information\", \"score\": \"43.0\"}", - "MS MARCO (TREC) - # train": "{\"description\": \"min=2, mean=2, max=2, sum=6 (3)\", \"tab\": \"General information\", \"score\": \"2.0\"}", - "MS MARCO (TREC) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MS MARCO (TREC) - # prompt tokens": "{\"description\": \"min=480.163, mean=519.496, max=566.163, sum=1558.488 (3)\", \"tab\": \"General information\", \"score\": \"519.4961240310078\"}", - "MS MARCO (TREC) - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=3 (3)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "MS MARCO (TREC) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "MS MARCO (regular) - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "MS MARCO (TREC) - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CNN/DailyMail", - "source_data": { - "dataset_name": "CNN/DailyMail", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on CNN/DailyMail", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.121, - "details": { - "description": "min=0.116, mean=0.121, max=0.13, sum=0.728 (6)", - "tab": "Accuracy", - "CNN/DailyMail - Denoised inference time (s)": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "CNN/DailyMail - # eval": "{\"description\": \"min=466, mean=466, max=466, sum=2796 (6)\", \"tab\": \"General information\", \"score\": \"466.0\"}", - "CNN/DailyMail - # train": "{\"description\": \"min=5, mean=5, max=5, sum=30 (6)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CNN/DailyMail - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (6)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CNN/DailyMail - # prompt tokens": "{\"description\": \"min=1555.036, mean=1575.036, max=1602.036, sum=9450.219 (6)\", \"tab\": \"General information\", \"score\": \"1575.0364806866953\"}", - "CNN/DailyMail - # output tokens": "{\"description\": \"min=60.474, mean=68.601, max=77.918, sum=411.605 (6)\", \"tab\": \"General information\", \"score\": \"68.60085836909872\"}", - "CNN/DailyMail - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=18 (6)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "CNN/DailyMail - Stereotypes (race)": "{\"description\": \"min=0.604, mean=0.612, max=0.618, sum=3.671 (6)\", \"tab\": \"Bias\", \"score\": \"0.6118203882651768\"}", - "CNN/DailyMail - Stereotypes (gender)": "{\"description\": \"min=0.401, mean=0.408, max=0.419, sum=2.449 (6)\", \"tab\": \"Bias\", \"score\": \"0.408087030039703\"}", - "CNN/DailyMail - Representation (race)": "{\"description\": \"min=0.257, mean=0.287, max=0.318, sum=1.72 (6)\", \"tab\": \"Bias\", \"score\": \"0.2867291116025263\"}", - "CNN/DailyMail - Representation (gender)": "{\"description\": \"min=0.117, mean=0.141, max=0.159, sum=0.844 (6)\", \"tab\": \"Bias\", \"score\": \"0.14067727789435583\"}", - "CNN/DailyMail - Toxic fraction": "{\"description\": \"min=0, mean=0.001, max=0.002, sum=0.004 (6)\", \"tab\": \"Toxicity\", \"score\": \"0.000715307582260372\"}", - "CNN/DailyMail - SummaC": "{\"description\": \"min=0.231, mean=0.359, max=0.443, sum=1.077 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.35895859214347764\"}", - "CNN/DailyMail - QAFactEval": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - BERTScore (F1)": "{\"description\": \"min=0.195, mean=0.218, max=0.246, sum=0.654 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.21796490870344257\"}", - "CNN/DailyMail - Coverage": "{\"description\": \"min=0.801, mean=0.899, max=0.957, sum=5.391 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"0.8985701854042452\"}", - "CNN/DailyMail - Density": "{\"description\": \"min=16.696, mean=24.344, max=33.085, sum=146.063 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"24.343863209587038\"}", - "CNN/DailyMail - Compression": "{\"description\": \"min=9.239, mean=11.42, max=13.421, sum=68.523 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"11.420494637224708\"}", - "CNN/DailyMail - HumanEval-faithfulness": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-relevance": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-coherence": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "XSUM", - "source_data": { - "dataset_name": "XSUM", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on XSUM", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.099, - "details": { - "description": "min=0.095, mean=0.099, max=0.106, sum=0.596 (6)", - "tab": "Accuracy", - "XSUM - Denoised inference time (s)": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "XSUM - # eval": "{\"description\": \"min=518, mean=518, max=518, sum=3108 (6)\", \"tab\": \"General information\", \"score\": \"518.0\"}", - "XSUM - # train": "{\"description\": \"min=4.996, mean=4.998, max=5, sum=29.988 (6)\", \"tab\": \"General information\", \"score\": \"4.998069498069498\"}", - "XSUM - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (6)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "XSUM - # prompt tokens": "{\"description\": \"min=1484.608, mean=1537.452, max=1572.616, sum=9224.71 (6)\", \"tab\": \"General information\", \"score\": \"1537.4517374517375\"}", - "XSUM - # output tokens": "{\"description\": \"min=23.5, mean=23.626, max=23.749, sum=141.757 (6)\", \"tab\": \"General information\", \"score\": \"23.626126126126128\"}", - "XSUM - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=18 (6)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "XSUM - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=4 (6)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "XSUM - Stereotypes (gender)": "{\"description\": \"min=0.424, mean=0.436, max=0.453, sum=2.616 (6)\", \"tab\": \"Bias\", \"score\": \"0.43605987410335234\"}", - "XSUM - Representation (race)": "{\"description\": \"min=0.373, mean=0.393, max=0.404, sum=2.359 (6)\", \"tab\": \"Bias\", \"score\": \"0.393188854489164\"}", - "XSUM - Representation (gender)": "{\"description\": \"min=0.181, mean=0.194, max=0.206, sum=1.165 (6)\", \"tab\": \"Bias\", \"score\": \"0.194128141174599\"}", - "XSUM - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (6)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}", - "XSUM - SummaC": "{\"description\": \"min=-0.192, mean=-0.171, max=-0.149, sum=-0.513 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"-0.17113255308913036\"}", - "XSUM - QAFactEval": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - BERTScore (F1)": "{\"description\": \"min=0.382, mean=0.384, max=0.388, sum=1.152 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.38412741233326225\"}", - "XSUM - Coverage": "{\"description\": \"min=0.842, mean=0.842, max=0.842, sum=5.051 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"0.8418943137133965\"}", - "XSUM - Density": "{\"description\": \"min=3.715, mean=3.815, max=3.914, sum=22.889 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"3.8148335440941747\"}", - "XSUM - Compression": "{\"description\": \"min=19.45, mean=19.703, max=19.907, sum=118.221 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"19.7034371773279\"}", - "XSUM - HumanEval-faithfulness": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-relevance": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-coherence": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "IMDB", - "source_data": { - "dataset_name": "IMDB", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on IMDB", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.935, - "details": { - "description": "min=0.917, mean=0.935, max=0.947, sum=2.804 (3)", - "tab": "Accuracy", - "IMDB - ECE (10-bin)": "{\"description\": \"min=0.335, mean=0.36, max=0.394, sum=1.079 (3)\", \"tab\": \"Calibration\", \"score\": \"0.3598306140598746\"}", - "IMDB - EM (Robustness)": "{\"description\": \"min=0.878, mean=0.888, max=0.896, sum=2.665 (3)\", \"tab\": \"Robustness\", \"score\": \"0.8883333333333333\"}", - "IMDB - EM (Fairness)": "{\"description\": \"min=0.896, mean=0.917, max=0.936, sum=2.752 (3)\", \"tab\": \"Fairness\", \"score\": \"0.9173333333333334\"}", - "IMDB - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "IMDB - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "IMDB - # train": "{\"description\": \"min=2.903, mean=4.229, max=4.983, sum=12.688 (3)\", \"tab\": \"General information\", \"score\": \"4.229333333333333\"}", - "IMDB - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IMDB - # prompt tokens": "{\"description\": \"min=1283.038, mean=1562.808, max=1784.2, sum=4688.425 (3)\", \"tab\": \"General information\", \"score\": \"1562.8083333333334\"}", - "IMDB - # output tokens": "{\"description\": \"min=1, mean=1.003, max=1.01, sum=3.01 (3)\", \"tab\": \"General information\", \"score\": \"1.0033333333333332\"}", - "IMDB - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "IMDB - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CivilComments", - "source_data": { - "dataset_name": "CivilComments", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on CivilComments", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5, - "details": { - "description": "min=0, mean=0.5, max=1, sum=27.019 (54)", - "tab": "Accuracy", - "CivilComments - ECE (10-bin)": "{\"description\": \"min=0.265, mean=0.487, max=0.736, sum=26.317 (54)\", \"tab\": \"Calibration\", \"score\": \"0.4873543575629644\"}", - "CivilComments - EM (Robustness)": "{\"description\": \"min=0, mean=0.353, max=0.931, sum=19.089 (54)\", \"tab\": \"Robustness\", \"score\": \"0.35349935695509527\"}", - "CivilComments - EM (Fairness)": "{\"description\": \"min=0, mean=0.493, max=1, sum=26.609 (54)\", \"tab\": \"Fairness\", \"score\": \"0.49275536816045606\"}", - "CivilComments - Denoised inference time (s)": "{\"description\": \"9 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "CivilComments - # eval": "{\"description\": \"min=74, mean=371.556, max=683, sum=20064 (54)\", \"tab\": \"General information\", \"score\": \"371.55555555555554\"}", - "CivilComments - # train": "{\"description\": \"min=5, mean=5, max=5, sum=270 (54)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CivilComments - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (54)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CivilComments - # prompt tokens": "{\"description\": \"min=362.293, mean=732.514, max=1288.441, sum=39555.782 (54)\", \"tab\": \"General information\", \"score\": \"732.5144825548033\"}", - "CivilComments - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=54 (54)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "CivilComments - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=162 (54)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "CivilComments - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (54)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "RAFT", - "source_data": { - "dataset_name": "RAFT", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on RAFT", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.591, - "details": { - "description": "min=0.1, mean=0.591, max=0.975, sum=19.5 (33)", - "tab": "Accuracy", - "RAFT - ECE (10-bin)": "{\"description\": \"min=0.11, mean=0.253, max=0.545, sum=8.337 (33)\", \"tab\": \"Calibration\", \"score\": \"0.25263340417043\"}", - "RAFT - EM (Robustness)": "{\"description\": \"min=0.025, mean=0.502, max=0.975, sum=16.55 (33)\", \"tab\": \"Robustness\", \"score\": \"0.5015151515151515\"}", - "RAFT - EM (Fairness)": "{\"description\": \"min=0.1, mean=0.571, max=0.975, sum=18.85 (33)\", \"tab\": \"Fairness\", \"score\": \"0.5712121212121212\"}", - "RAFT - Denoised inference time (s)": "{\"description\": \"11 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "RAFT - # eval": "{\"description\": \"min=40, mean=40, max=40, sum=1320 (33)\", \"tab\": \"General information\", \"score\": \"40.0\"}", - "RAFT - # train": "{\"description\": \"min=0, mean=4.557, max=5, sum=150.375 (33)\", \"tab\": \"General information\", \"score\": \"4.556818181818182\"}", - "RAFT - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (33)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "RAFT - # prompt tokens": "{\"description\": \"min=270.325, mean=814.446, max=1777.025, sum=26876.725 (33)\", \"tab\": \"General information\", \"score\": \"814.446212121212\"}", - "RAFT - # output tokens": "{\"description\": \"min=0.575, mean=3.038, max=6.375, sum=100.25 (33)\", \"tab\": \"General information\", \"score\": \"3.0378787878787885\"}", - "RAFT - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=99 (33)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "RAFT - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (33)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/cohere_cohere-small-v20220720-410m.json b/data/models/cohere_cohere-small-v20220720-410m.json deleted file mode 100644 index c20f51a7422c507508d473d46087b805c1a5c728..0000000000000000000000000000000000000000 --- a/data/models/cohere_cohere-small-v20220720-410m.json +++ /dev/null @@ -1,674 +0,0 @@ -{ - "model_info": { - "name": "Cohere small v20220720 410M", - "id": "cohere/Cohere-small-v20220720-410M", - "developer": "cohere", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_classic/cohere_Cohere-small-v20220720-410M/1774096308.339228", - "retrieved_timestamp": "1774096308.339228", - "source_metadata": { - "source_name": "helm_classic", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_classic", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_classic", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperform on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.109, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Calibration": "{\"description\": \"\", \"tab\": \"Calibration\", \"score\": \"0.6085000742339626\"}", - "Mean win rate - Robustness": "{\"description\": \"\", \"tab\": \"Robustness\", \"score\": \"0.1469566826886926\"}", - "Mean win rate - Fairness": "{\"description\": \"\", \"tab\": \"Fairness\", \"score\": \"0.15386697669576083\"}", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.5343333333333333\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}", - "Mean win rate - Bias": "{\"description\": \"\", \"tab\": \"Bias\", \"score\": \"0.45155563090416306\"}", - "Mean win rate - Toxicity": "{\"description\": \"\", \"tab\": \"Toxicity\", \"score\": \"0.412334270667604\"}", - "Mean win rate - Summarization metrics": "{\"description\": \"\", \"tab\": \"Summarization metrics\", \"score\": \"0.29156223893065997\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.264, - "details": { - "description": "min=0.18, mean=0.264, max=0.42, sum=3.963 (15)", - "tab": "Accuracy", - "MMLU - ECE (10-bin)": "{\"description\": \"min=0.049, mean=0.136, max=0.202, sum=2.04 (15)\", \"tab\": \"Calibration\", \"score\": \"0.13602108170852936\"}", - "MMLU - EM (Robustness)": "{\"description\": \"min=0.13, mean=0.226, max=0.42, sum=3.397 (15)\", \"tab\": \"Robustness\", \"score\": \"0.22644444444444442\"}", - "MMLU - EM (Fairness)": "{\"description\": \"min=0.1, mean=0.222, max=0.4, sum=3.334 (15)\", \"tab\": \"Fairness\", \"score\": \"0.22225730994152046\"}", - "MMLU - Denoised inference time (s)": "{\"description\": \"min=0.265, mean=0.284, max=0.312, sum=4.267 (15)\", \"tab\": \"Efficiency\", \"score\": \"0.284456830180921\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=1542 (15)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=75 (15)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (15)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=372.75, mean=481.26, max=628.421, sum=7218.903 (15)\", \"tab\": \"General information\", \"score\": \"481.2602105263158\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=15 (15)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "MMLU - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=45 (15)\", \"tab\": \"General information\", \"score\": \"3.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "BoolQ", - "source_data": { - "dataset_name": "BoolQ", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on BoolQ", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.457, - "details": { - "description": "min=0.447, mean=0.457, max=0.464, sum=1.372 (3)", - "tab": "Accuracy", - "BoolQ - ECE (10-bin)": "{\"description\": \"min=0.072, mean=0.095, max=0.124, sum=0.285 (3)\", \"tab\": \"Calibration\", \"score\": \"0.09496766959019069\"}", - "BoolQ - EM (Robustness)": "{\"description\": \"min=0.352, mean=0.361, max=0.378, sum=1.083 (3)\", \"tab\": \"Robustness\", \"score\": \"0.361\"}", - "BoolQ - EM (Fairness)": "{\"description\": \"min=0.346, mean=0.374, max=0.396, sum=1.121 (3)\", \"tab\": \"Fairness\", \"score\": \"0.37366666666666665\"}", - "BoolQ - Denoised inference time (s)": "{\"description\": \"min=0.319, mean=0.367, max=0.436, sum=1.101 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.36694511328125\"}", - "BoolQ - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "BoolQ - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "BoolQ - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "BoolQ - # prompt tokens": "{\"description\": \"min=669.307, mean=925.307, max=1269.307, sum=2775.921 (3)\", \"tab\": \"General information\", \"score\": \"925.3070000000001\"}", - "BoolQ - # output tokens": "{\"description\": \"min=1, mean=1.001, max=1.004, sum=3.004 (3)\", \"tab\": \"General information\", \"score\": \"1.0013333333333334\"}", - "BoolQ - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "BoolQ - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.294, - "details": { - "description": "min=0.281, mean=0.294, max=0.309, sum=0.881 (3)", - "tab": "Accuracy", - "NarrativeQA - ECE (10-bin)": "{\"description\": \"min=0.029, mean=0.031, max=0.033, sum=0.093 (3)\", \"tab\": \"Calibration\", \"score\": \"0.031094283389380417\"}", - "NarrativeQA - F1 (Robustness)": "{\"description\": \"min=0.076, mean=0.078, max=0.081, sum=0.235 (3)\", \"tab\": \"Robustness\", \"score\": \"0.07821074014295328\"}", - "NarrativeQA - F1 (Fairness)": "{\"description\": \"min=0.167, mean=0.179, max=0.197, sum=0.538 (3)\", \"tab\": \"Fairness\", \"score\": \"0.17918507973514153\"}", - "NarrativeQA - Denoised inference time (s)": "{\"description\": \"min=0.544, mean=0.56, max=0.583, sum=1.681 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.5603894916373239\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=1065 (3)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=0.958, mean=1.562, max=1.997, sum=4.687 (3)\", \"tab\": \"General information\", \"score\": \"1.5624413145539906\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=1601.997, mean=1634.99, max=1693.155, sum=4904.969 (3)\", \"tab\": \"General information\", \"score\": \"1634.9896713615024\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=8.149, mean=11.007, max=15.597, sum=33.02 (3)\", \"tab\": \"General information\", \"score\": \"11.006572769953053\"}", - "NarrativeQA - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NarrativeQA - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=0.667 (1)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "NarrativeQA - Stereotypes (gender)": "{\"description\": \"min=0.411, mean=0.418, max=0.429, sum=1.255 (3)\", \"tab\": \"Bias\", \"score\": \"0.4184126984126984\"}", - "NarrativeQA - Representation (race)": "{\"description\": \"min=0.333, mean=0.556, max=0.667, sum=1.667 (3)\", \"tab\": \"Bias\", \"score\": \"0.5555555555555556\"}", - "NarrativeQA - Representation (gender)": "{\"description\": \"min=0.186, mean=0.202, max=0.217, sum=0.606 (3)\", \"tab\": \"Bias\", \"score\": \"0.20205501924662395\"}", - "NarrativeQA - Toxic fraction": "{\"description\": \"min=0.025, mean=0.027, max=0.031, sum=0.082 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.027230046948356807\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (open-book)", - "source_data": { - "dataset_name": "NaturalQuestions (open-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (open-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.309, - "details": { - "description": "min=0.291, mean=0.309, max=0.334, sum=0.928 (3)", - "tab": "Accuracy", - "NaturalQuestions (closed-book) - ECE (10-bin)": "{\"description\": \"min=0.02, mean=0.023, max=0.027, sum=0.07 (3)\", \"tab\": \"Calibration\", \"score\": \"0.023328620693919305\"}", - "NaturalQuestions (open-book) - ECE (10-bin)": "{\"description\": \"min=0.18, mean=0.198, max=0.221, sum=0.594 (3)\", \"tab\": \"Calibration\", \"score\": \"0.198062019189297\"}", - "NaturalQuestions (closed-book) - F1 (Robustness)": "{\"description\": \"min=0.024, mean=0.025, max=0.027, sum=0.075 (3)\", \"tab\": \"Robustness\", \"score\": \"0.025009279663584086\"}", - "NaturalQuestions (open-book) - F1 (Robustness)": "{\"description\": \"min=0.066, mean=0.074, max=0.08, sum=0.222 (3)\", \"tab\": \"Robustness\", \"score\": \"0.07408175909872887\"}", - "NaturalQuestions (closed-book) - F1 (Fairness)": "{\"description\": \"min=0.052, mean=0.055, max=0.062, sum=0.166 (3)\", \"tab\": \"Fairness\", \"score\": \"0.055406816944260924\"}", - "NaturalQuestions (open-book) - F1 (Fairness)": "{\"description\": \"min=0.198, mean=0.219, max=0.246, sum=0.657 (3)\", \"tab\": \"Fairness\", \"score\": \"0.21887630944724534\"}", - "NaturalQuestions (closed-book) - Denoised inference time (s)": "{\"description\": \"min=0.246, mean=0.251, max=0.259, sum=0.753 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.2509381953124994\"}", - "NaturalQuestions (open-book) - Denoised inference time (s)": "{\"description\": \"min=0.552, mean=0.605, max=0.643, sum=1.815 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.6049964999999996\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=109.191, mean=111.191, max=115.191, sum=333.573 (3)\", \"tab\": \"General information\", \"score\": \"111.19099999999999\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=4.325, mean=5.149, max=6.46, sum=15.446 (3)\", \"tab\": \"General information\", \"score\": \"5.148666666666667\"}", - "NaturalQuestions (closed-book) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.538, mean=4.633, max=4.715, sum=13.899 (3)\", \"tab\": \"General information\", \"score\": \"4.633\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.039, mean=0.039, max=0.039, sum=0.117 (3)\", \"tab\": \"General information\", \"score\": \"0.039\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1261.72, mean=1481.344, max=1608.455, sum=4444.032 (3)\", \"tab\": \"General information\", \"score\": \"1481.344\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=20.452, mean=22.835, max=25.41, sum=68.505 (3)\", \"tab\": \"General information\", \"score\": \"22.834999999999997\"}", - "NaturalQuestions (open-book) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NaturalQuestions (closed-book) - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - Stereotypes (gender)": "{\"description\": \"min=0.5, mean=0.5, max=0.5, sum=1 (2)\", \"tab\": \"Bias\", \"score\": \"0.5\"}", - "NaturalQuestions (closed-book) - Representation (race)": "{\"description\": \"min=0.238, mean=0.415, max=0.539, sum=1.244 (3)\", \"tab\": \"Bias\", \"score\": \"0.41471861471861476\"}", - "NaturalQuestions (closed-book) - Representation (gender)": "{\"description\": \"min=0.167, mean=0.234, max=0.286, sum=0.702 (3)\", \"tab\": \"Bias\", \"score\": \"0.2341269841269841\"}", - "NaturalQuestions (open-book) - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=2 (3)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "NaturalQuestions (open-book) - Stereotypes (gender)": "{\"description\": \"min=0.48, mean=0.485, max=0.494, sum=1.455 (3)\", \"tab\": \"Bias\", \"score\": \"0.48499285130718955\"}", - "NaturalQuestions (open-book) - Representation (race)": "{\"description\": \"min=0.382, mean=0.435, max=0.467, sum=1.306 (3)\", \"tab\": \"Bias\", \"score\": \"0.43543086336382425\"}", - "NaturalQuestions (open-book) - Representation (gender)": "{\"description\": \"min=0.234, mean=0.265, max=0.3, sum=0.796 (3)\", \"tab\": \"Bias\", \"score\": \"0.2653339127915399\"}", - "NaturalQuestions (closed-book) - Toxic fraction": "{\"description\": \"min=0, mean=0.001, max=0.001, sum=0.002 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.0006666666666666666\"}", - "NaturalQuestions (open-book) - Toxic fraction": "{\"description\": \"min=0.002, mean=0.003, max=0.003, sum=0.008 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.0026666666666666666\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "QuAC", - "source_data": { - "dataset_name": "QuAC", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on QuAC", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.219, - "details": { - "description": "min=0.208, mean=0.219, max=0.238, sum=0.656 (3)", - "tab": "Accuracy", - "QuAC - ECE (10-bin)": "{\"description\": \"min=0.03, mean=0.036, max=0.042, sum=0.108 (3)\", \"tab\": \"Calibration\", \"score\": \"0.035862172954873824\"}", - "QuAC - F1 (Robustness)": "{\"description\": \"min=0.094, mean=0.098, max=0.101, sum=0.293 (3)\", \"tab\": \"Robustness\", \"score\": \"0.09766108203425072\"}", - "QuAC - F1 (Fairness)": "{\"description\": \"min=0.135, mean=0.144, max=0.162, sum=0.433 (3)\", \"tab\": \"Fairness\", \"score\": \"0.14446776305873513\"}", - "QuAC - Denoised inference time (s)": "{\"description\": \"min=0.611, mean=0.619, max=0.625, sum=1.856 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.6185995332031252\"}", - "QuAC - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "QuAC - # train": "{\"description\": \"min=0.797, mean=0.881, max=0.969, sum=2.644 (3)\", \"tab\": \"General information\", \"score\": \"0.8813333333333334\"}", - "QuAC - truncated": "{\"description\": \"min=0.02, mean=0.02, max=0.02, sum=0.06 (3)\", \"tab\": \"General information\", \"score\": \"0.02\"}", - "QuAC - # prompt tokens": "{\"description\": \"min=1600.292, mean=1639.784, max=1661.675, sum=4919.353 (3)\", \"tab\": \"General information\", \"score\": \"1639.784333333333\"}", - "QuAC - # output tokens": "{\"description\": \"min=18.807, mean=20.639, max=21.99, sum=61.916 (3)\", \"tab\": \"General information\", \"score\": \"20.638666666666666\"}", - "QuAC - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "QuAC - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=2 (3)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "QuAC - Stereotypes (gender)": "{\"description\": \"min=0.447, mean=0.458, max=0.468, sum=1.375 (3)\", \"tab\": \"Bias\", \"score\": \"0.45823351891324243\"}", - "QuAC - Representation (race)": "{\"description\": \"min=0.329, mean=0.341, max=0.364, sum=1.022 (3)\", \"tab\": \"Bias\", \"score\": \"0.34075560523096593\"}", - "QuAC - Representation (gender)": "{\"description\": \"min=0.277, mean=0.285, max=0.299, sum=0.854 (3)\", \"tab\": \"Bias\", \"score\": \"0.2847879707506289\"}", - "QuAC - Toxic fraction": "{\"description\": \"min=0.001, mean=0.003, max=0.004, sum=0.008 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.0026666666666666666\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "HellaSwag", - "source_data": { - "dataset_name": "HellaSwag", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on HellaSwag", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.483, - "details": { - "description": "min=0.483, mean=0.483, max=0.483, sum=0.483 (1)", - "tab": "Accuracy", - "HellaSwag - ECE (10-bin)": "{\"description\": \"min=0.083, mean=0.083, max=0.083, sum=0.083 (1)\", \"tab\": \"Calibration\", \"score\": \"0.08312318484699062\"}", - "HellaSwag - EM (Robustness)": "{\"description\": \"min=0.405, mean=0.405, max=0.405, sum=0.405 (1)\", \"tab\": \"Robustness\", \"score\": \"0.405\"}", - "HellaSwag - EM (Fairness)": "{\"description\": \"min=0.308, mean=0.308, max=0.308, sum=0.308 (1)\", \"tab\": \"Fairness\", \"score\": \"0.308\"}", - "HellaSwag - Denoised inference time (s)": "{\"description\": \"min=0.223, mean=0.223, max=0.223, sum=0.223 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.22341269531249972\"}", - "HellaSwag - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "HellaSwag - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "HellaSwag - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "HellaSwag - # prompt tokens": "{\"description\": \"min=88.855, mean=88.855, max=88.855, sum=88.855 (1)\", \"tab\": \"General information\", \"score\": \"88.855\"}", - "HellaSwag - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "HellaSwag - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.348, - "details": { - "description": "min=0.348, mean=0.348, max=0.348, sum=0.348 (1)", - "tab": "Accuracy", - "OpenbookQA - ECE (10-bin)": "{\"description\": \"min=0.379, mean=0.379, max=0.379, sum=0.379 (1)\", \"tab\": \"Calibration\", \"score\": \"0.37852917669250147\"}", - "OpenbookQA - EM (Robustness)": "{\"description\": \"min=0.238, mean=0.238, max=0.238, sum=0.238 (1)\", \"tab\": \"Robustness\", \"score\": \"0.238\"}", - "OpenbookQA - EM (Fairness)": "{\"description\": \"min=0.28, mean=0.28, max=0.28, sum=0.28 (1)\", \"tab\": \"Fairness\", \"score\": \"0.28\"}", - "OpenbookQA - Denoised inference time (s)": "{\"description\": \"min=0.214, mean=0.214, max=0.214, sum=0.214 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.2136278906249995\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=5.358, mean=5.358, max=5.358, sum=5.358 (1)\", \"tab\": \"General information\", \"score\": \"5.358\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "TruthfulQA", - "source_data": { - "dataset_name": "TruthfulQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on TruthfulQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.217, - "details": { - "description": "min=0.202, mean=0.217, max=0.226, sum=0.65 (3)", - "tab": "Accuracy", - "TruthfulQA - ECE (10-bin)": "{\"description\": \"min=0.059, mean=0.076, max=0.098, sum=0.229 (3)\", \"tab\": \"Calibration\", \"score\": \"0.07625390965133329\"}", - "TruthfulQA - EM (Robustness)": "{\"description\": \"min=0.2, mean=0.204, max=0.211, sum=0.612 (3)\", \"tab\": \"Robustness\", \"score\": \"0.2038735983690112\"}", - "TruthfulQA - EM (Fairness)": "{\"description\": \"min=0.194, mean=0.203, max=0.214, sum=0.609 (3)\", \"tab\": \"Fairness\", \"score\": \"0.20285423037716613\"}", - "TruthfulQA - Denoised inference time (s)": "{\"description\": \"min=0.287, mean=0.289, max=0.295, sum=0.868 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.2894203160837155\"}", - "TruthfulQA - # eval": "{\"description\": \"min=654, mean=654, max=654, sum=1962 (3)\", \"tab\": \"General information\", \"score\": \"654.0\"}", - "TruthfulQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "TruthfulQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "TruthfulQA - # prompt tokens": "{\"description\": \"min=505.315, mean=514.648, max=532.315, sum=1543.945 (3)\", \"tab\": \"General information\", \"score\": \"514.6483180428135\"}", - "TruthfulQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=3 (3)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "TruthfulQA - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MS MARCO (TREC)", - "source_data": { - "dataset_name": "MS MARCO (TREC)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "NDCG@10 on MS MARCO (TREC)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.304, - "details": { - "description": "min=0.258, mean=0.304, max=0.338, sum=0.911 (3)", - "tab": "Accuracy", - "MS MARCO (regular) - RR@10 (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "MS MARCO (TREC) - NDCG@10 (Robustness)": "{\"description\": \"min=0.22, mean=0.252, max=0.287, sum=0.757 (3)\", \"tab\": \"Robustness\", \"score\": \"0.2521940956196658\"}", - "MS MARCO (regular) - RR@10 (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "MS MARCO (TREC) - NDCG@10 (Fairness)": "{\"description\": \"min=0.228, mean=0.28, max=0.324, sum=0.84 (3)\", \"tab\": \"Fairness\", \"score\": \"0.2798487582673837\"}", - "MS MARCO (regular) - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MS MARCO (TREC) - Denoised inference time (s)": "{\"description\": \"min=0.282, mean=0.291, max=0.303, sum=0.872 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.29054985767926356\"}", - "MS MARCO (regular) - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # eval": "{\"description\": \"min=43, mean=43, max=43, sum=129 (3)\", \"tab\": \"General information\", \"score\": \"43.0\"}", - "MS MARCO (TREC) - # train": "{\"description\": \"min=2, mean=2, max=2, sum=6 (3)\", \"tab\": \"General information\", \"score\": \"2.0\"}", - "MS MARCO (TREC) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MS MARCO (TREC) - # prompt tokens": "{\"description\": \"min=480.163, mean=519.496, max=566.163, sum=1558.488 (3)\", \"tab\": \"General information\", \"score\": \"519.4961240310078\"}", - "MS MARCO (TREC) - # output tokens": "{\"description\": \"min=1, mean=1.031, max=1.093, sum=3.093 (3)\", \"tab\": \"General information\", \"score\": \"1.0310077519379846\"}", - "MS MARCO (TREC) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "MS MARCO (regular) - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "MS MARCO (TREC) - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CNN/DailyMail", - "source_data": { - "dataset_name": "CNN/DailyMail", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on CNN/DailyMail", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.063, - "details": { - "description": "min=0.031, mean=0.063, max=0.087, sum=0.377 (6)", - "tab": "Accuracy", - "CNN/DailyMail - Denoised inference time (s)": "{\"description\": \"min=0.781, mean=0.954, max=1.052, sum=5.724 (6)\", \"tab\": \"Efficiency\", \"score\": \"0.9539734693535404\"}", - "CNN/DailyMail - # eval": "{\"description\": \"min=466, mean=466, max=466, sum=2796 (6)\", \"tab\": \"General information\", \"score\": \"466.0\"}", - "CNN/DailyMail - # train": "{\"description\": \"min=5, mean=5, max=5, sum=30 (6)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CNN/DailyMail - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (6)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CNN/DailyMail - # prompt tokens": "{\"description\": \"min=1555.036, mean=1575.036, max=1602.036, sum=9450.219 (6)\", \"tab\": \"General information\", \"score\": \"1575.0364806866953\"}", - "CNN/DailyMail - # output tokens": "{\"description\": \"min=49.71, mean=78.352, max=93.899, sum=470.112 (6)\", \"tab\": \"General information\", \"score\": \"78.3519313304721\"}", - "CNN/DailyMail - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=18 (6)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "CNN/DailyMail - Stereotypes (race)": "{\"description\": \"min=0.625, mean=0.648, max=0.667, sum=3.885 (6)\", \"tab\": \"Bias\", \"score\": \"0.6475615887380594\"}", - "CNN/DailyMail - Stereotypes (gender)": "{\"description\": \"min=0.405, mean=0.42, max=0.449, sum=2.522 (6)\", \"tab\": \"Bias\", \"score\": \"0.4203329386778049\"}", - "CNN/DailyMail - Representation (race)": "{\"description\": \"min=0.099, mean=0.145, max=0.201, sum=0.868 (6)\", \"tab\": \"Bias\", \"score\": \"0.14468337947687135\"}", - "CNN/DailyMail - Representation (gender)": "{\"description\": \"min=0.163, mean=0.182, max=0.21, sum=1.09 (6)\", \"tab\": \"Bias\", \"score\": \"0.18171396544569016\"}", - "CNN/DailyMail - Toxic fraction": "{\"description\": \"min=0, mean=0.001, max=0.002, sum=0.009 (6)\", \"tab\": \"Toxicity\", \"score\": \"0.001430615164520744\"}", - "CNN/DailyMail - SummaC": "{\"description\": \"min=-0.077, mean=0.054, max=0.168, sum=0.161 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.053643734154981075\"}", - "CNN/DailyMail - QAFactEval": "{\"description\": \"min=0.051, mean=2.638, max=4.057, sum=15.831 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"2.6384596103973283\"}", - "CNN/DailyMail - BERTScore (F1)": "{\"description\": \"min=-0.069, mean=0.026, max=0.075, sum=0.077 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.025643326292308758\"}", - "CNN/DailyMail - Coverage": "{\"description\": \"min=0.532, mean=0.744, max=0.913, sum=4.465 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"0.7441391663831297\"}", - "CNN/DailyMail - Density": "{\"description\": \"min=11.632, mean=25.238, max=33.415, sum=151.427 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"25.237906513316556\"}", - "CNN/DailyMail - Compression": "{\"description\": \"min=9.053, mean=13.243, max=20.787, sum=79.46 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"13.243377373187593\"}", - "CNN/DailyMail - HumanEval-faithfulness": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-relevance": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-coherence": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "XSUM", - "source_data": { - "dataset_name": "XSUM", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on XSUM", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.033, - "details": { - "description": "min=0.031, mean=0.033, max=0.037, sum=0.199 (6)", - "tab": "Accuracy", - "XSUM - Denoised inference time (s)": "{\"description\": \"min=0.637, mean=0.642, max=0.649, sum=3.85 (6)\", \"tab\": \"Efficiency\", \"score\": \"0.6416181225868728\"}", - "XSUM - # eval": "{\"description\": \"min=518, mean=518, max=518, sum=3108 (6)\", \"tab\": \"General information\", \"score\": \"518.0\"}", - "XSUM - # train": "{\"description\": \"min=4.996, mean=4.998, max=5, sum=29.988 (6)\", \"tab\": \"General information\", \"score\": \"4.998069498069498\"}", - "XSUM - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (6)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "XSUM - # prompt tokens": "{\"description\": \"min=1484.608, mean=1537.452, max=1572.616, sum=9224.71 (6)\", \"tab\": \"General information\", \"score\": \"1537.4517374517375\"}", - "XSUM - # output tokens": "{\"description\": \"min=25.859, mean=27.394, max=28.226, sum=164.363 (6)\", \"tab\": \"General information\", \"score\": \"27.393822393822393\"}", - "XSUM - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=18 (6)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "XSUM - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=4 (6)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "XSUM - Stereotypes (gender)": "{\"description\": \"min=0.399, mean=0.43, max=0.493, sum=2.58 (6)\", \"tab\": \"Bias\", \"score\": \"0.43004930254930257\"}", - "XSUM - Representation (race)": "{\"description\": \"min=0.542, mean=0.556, max=0.583, sum=3.333 (6)\", \"tab\": \"Bias\", \"score\": \"0.5555555555555556\"}", - "XSUM - Representation (gender)": "{\"description\": \"min=0.224, mean=0.246, max=0.283, sum=1.474 (6)\", \"tab\": \"Bias\", \"score\": \"0.2457025240044108\"}", - "XSUM - Toxic fraction": "{\"description\": \"min=0, mean=0.001, max=0.002, sum=0.004 (6)\", \"tab\": \"Toxicity\", \"score\": \"0.0006435006435006435\"}", - "XSUM - SummaC": "{\"description\": \"min=0.0, mean=0.028, max=0.073, sum=0.085 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.02834827232857105\"}", - "XSUM - QAFactEval": "{\"description\": \"min=2.873, mean=3.094, max=3.373, sum=18.563 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"3.0938511325795113\"}", - "XSUM - BERTScore (F1)": "{\"description\": \"min=0.173, mean=0.195, max=0.221, sum=0.585 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.1951040609680371\"}", - "XSUM - Coverage": "{\"description\": \"min=0.853, mean=0.863, max=0.87, sum=5.178 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"0.8630576414302875\"}", - "XSUM - Density": "{\"description\": \"min=9.489, mean=10.557, max=12.063, sum=63.341 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"10.556911526268395\"}", - "XSUM - Compression": "{\"description\": \"min=16.738, mean=17.551, max=18.157, sum=105.306 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"17.55096225657148\"}", - "XSUM - HumanEval-faithfulness": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-relevance": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-coherence": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "IMDB", - "source_data": { - "dataset_name": "IMDB", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on IMDB", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.578, - "details": { - "description": "min=0.53, mean=0.578, max=0.618, sum=1.735 (3)", - "tab": "Accuracy", - "IMDB - ECE (10-bin)": "{\"description\": \"min=0.085, mean=0.134, max=0.174, sum=0.401 (3)\", \"tab\": \"Calibration\", \"score\": \"0.13354341899719424\"}", - "IMDB - EM (Robustness)": "{\"description\": \"min=0.447, mean=0.473, max=0.498, sum=1.418 (3)\", \"tab\": \"Robustness\", \"score\": \"0.4726666666666666\"}", - "IMDB - EM (Fairness)": "{\"description\": \"min=0.49, mean=0.518, max=0.54, sum=1.554 (3)\", \"tab\": \"Fairness\", \"score\": \"0.518\"}", - "IMDB - Denoised inference time (s)": "{\"description\": \"min=0.414, mean=0.458, max=0.52, sum=1.373 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.45773176757812467\"}", - "IMDB - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "IMDB - # train": "{\"description\": \"min=4.846, mean=4.93, max=4.98, sum=14.79 (3)\", \"tab\": \"General information\", \"score\": \"4.930000000000001\"}", - "IMDB - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IMDB - # prompt tokens": "{\"description\": \"min=1161.854, mean=1398.654, max=1747.025, sum=4195.961 (3)\", \"tab\": \"General information\", \"score\": \"1398.6536666666668\"}", - "IMDB - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=3 (3)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "IMDB - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "IMDB - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CivilComments", - "source_data": { - "dataset_name": "CivilComments", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on CivilComments", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.501, - "details": { - "description": "min=0, mean=0.501, max=1, sum=27.062 (54)", - "tab": "Accuracy", - "CivilComments - ECE (10-bin)": "{\"description\": \"min=0.201, mean=0.486, max=0.8, sum=26.269 (54)\", \"tab\": \"Calibration\", \"score\": \"0.4864679961449666\"}", - "CivilComments - EM (Robustness)": "{\"description\": \"min=0, mean=0.434, max=1, sum=23.451 (54)\", \"tab\": \"Robustness\", \"score\": \"0.4342847473494527\"}", - "CivilComments - EM (Fairness)": "{\"description\": \"min=0, mean=0.495, max=1, sum=26.744 (54)\", \"tab\": \"Fairness\", \"score\": \"0.49526155082406725\"}", - "CivilComments - Denoised inference time (s)": "{\"description\": \"min=0.264, mean=0.329, max=0.439, sum=17.76 (54)\", \"tab\": \"Efficiency\", \"score\": \"0.32889709084919744\"}", - "CivilComments - # eval": "{\"description\": \"min=74, mean=371.556, max=683, sum=20064 (54)\", \"tab\": \"General information\", \"score\": \"371.55555555555554\"}", - "CivilComments - # train": "{\"description\": \"min=5, mean=5, max=5, sum=270 (54)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CivilComments - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (54)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CivilComments - # prompt tokens": "{\"description\": \"min=362.293, mean=732.514, max=1288.441, sum=39555.782 (54)\", \"tab\": \"General information\", \"score\": \"732.5144825548033\"}", - "CivilComments - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=54 (54)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "CivilComments - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=162 (54)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "CivilComments - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (54)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "RAFT", - "source_data": { - "dataset_name": "RAFT", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on RAFT", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.492, - "details": { - "description": "min=0, mean=0.492, max=0.975, sum=16.225 (33)", - "tab": "Accuracy", - "RAFT - ECE (10-bin)": "{\"description\": \"min=0.084, mean=0.234, max=0.631, sum=7.714 (33)\", \"tab\": \"Calibration\", \"score\": \"0.23374335739699753\"}", - "RAFT - EM (Robustness)": "{\"description\": \"min=0, mean=0.403, max=0.975, sum=13.3 (33)\", \"tab\": \"Robustness\", \"score\": \"0.40303030303030307\"}", - "RAFT - EM (Fairness)": "{\"description\": \"min=0, mean=0.452, max=0.975, sum=14.9 (33)\", \"tab\": \"Fairness\", \"score\": \"0.4515151515151515\"}", - "RAFT - Denoised inference time (s)": "{\"description\": \"min=0.256, mean=0.36, max=0.547, sum=11.878 (33)\", \"tab\": \"Efficiency\", \"score\": \"0.3599495087594697\"}", - "RAFT - # eval": "{\"description\": \"min=40, mean=40, max=40, sum=1320 (33)\", \"tab\": \"General information\", \"score\": \"40.0\"}", - "RAFT - # train": "{\"description\": \"min=0, mean=4.557, max=5, sum=150.375 (33)\", \"tab\": \"General information\", \"score\": \"4.556818181818182\"}", - "RAFT - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (33)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "RAFT - # prompt tokens": "{\"description\": \"min=270.325, mean=814.446, max=1777.025, sum=26876.725 (33)\", \"tab\": \"General information\", \"score\": \"814.446212121212\"}", - "RAFT - # output tokens": "{\"description\": \"min=1, mean=3.239, max=5.575, sum=106.9 (33)\", \"tab\": \"General information\", \"score\": \"3.2393939393939393\"}", - "RAFT - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=99 (33)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "RAFT - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (gender)": "{\"description\": \"min=0.5, mean=0.5, max=0.5, sum=1.5 (3)\", \"tab\": \"Bias\", \"score\": \"0.5\"}", - "RAFT - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (33)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/cohere_cohere-xlarge-v20220609-52.4b.json b/data/models/cohere_cohere-xlarge-v20220609-52.4b.json deleted file mode 100644 index c5dbdfbef5f80688162474ba3503c9e594f2aed9..0000000000000000000000000000000000000000 --- a/data/models/cohere_cohere-xlarge-v20220609-52.4b.json +++ /dev/null @@ -1,674 +0,0 @@ -{ - "model_info": { - "name": "Cohere xlarge v20220609 52.4B", - "id": "cohere/Cohere-xlarge-v20220609-52.4B", - "developer": "cohere", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_classic/cohere_Cohere-xlarge-v20220609-52.4B/1774096308.339228", - "retrieved_timestamp": "1774096308.339228", - "source_metadata": { - "source_name": "helm_classic", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_classic", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_classic", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperform on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.56, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Calibration": "{\"description\": \"\", \"tab\": \"Calibration\", \"score\": \"0.5427202179052317\"}", - "Mean win rate - Robustness": "{\"description\": \"\", \"tab\": \"Robustness\", \"score\": \"0.5061059259613209\"}", - "Mean win rate - Fairness": "{\"description\": \"\", \"tab\": \"Fairness\", \"score\": \"0.5496737226436893\"}", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.1992872807017544\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}", - "Mean win rate - Bias": "{\"description\": \"\", \"tab\": \"Bias\", \"score\": \"0.5983741692925366\"}", - "Mean win rate - Toxicity": "{\"description\": \"\", \"tab\": \"Toxicity\", \"score\": \"0.5744286577619911\"}", - "Mean win rate - Summarization metrics": "{\"description\": \"\", \"tab\": \"Summarization metrics\", \"score\": \"0.546345029239766\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.353, - "details": { - "description": "min=0.228, mean=0.353, max=0.56, sum=5.296 (15)", - "tab": "Accuracy", - "MMLU - ECE (10-bin)": "{\"description\": \"min=0.089, mean=0.149, max=0.246, sum=2.242 (15)\", \"tab\": \"Calibration\", \"score\": \"0.14945785718149934\"}", - "MMLU - EM (Robustness)": "{\"description\": \"min=0.158, mean=0.29, max=0.51, sum=4.349 (15)\", \"tab\": \"Robustness\", \"score\": \"0.28992982456140354\"}", - "MMLU - EM (Fairness)": "{\"description\": \"min=0.158, mean=0.315, max=0.53, sum=4.729 (15)\", \"tab\": \"Fairness\", \"score\": \"0.31526315789473686\"}", - "MMLU - Denoised inference time (s)": "{\"description\": \"min=0.47, mean=0.489, max=0.506, sum=7.328 (15)\", \"tab\": \"Efficiency\", \"score\": \"0.4885340888157895\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=1542 (15)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=75 (15)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (15)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=372.75, mean=481.26, max=628.421, sum=7218.903 (15)\", \"tab\": \"General information\", \"score\": \"481.2602105263158\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=15 (15)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "MMLU - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=45 (15)\", \"tab\": \"General information\", \"score\": \"3.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "BoolQ", - "source_data": { - "dataset_name": "BoolQ", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on BoolQ", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.718, - "details": { - "description": "min=0.702, mean=0.718, max=0.74, sum=2.153 (3)", - "tab": "Accuracy", - "BoolQ - ECE (10-bin)": "{\"description\": \"min=0.037, mean=0.04, max=0.043, sum=0.119 (3)\", \"tab\": \"Calibration\", \"score\": \"0.039674216829776156\"}", - "BoolQ - EM (Robustness)": "{\"description\": \"min=0.601, mean=0.614, max=0.622, sum=1.842 (3)\", \"tab\": \"Robustness\", \"score\": \"0.614\"}", - "BoolQ - EM (Fairness)": "{\"description\": \"min=0.657, mean=0.667, max=0.681, sum=2 (3)\", \"tab\": \"Fairness\", \"score\": \"0.6666666666666666\"}", - "BoolQ - Denoised inference time (s)": "{\"description\": \"min=0.519, mean=0.598, max=0.705, sum=1.795 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.5984045305989586\"}", - "BoolQ - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "BoolQ - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "BoolQ - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "BoolQ - # prompt tokens": "{\"description\": \"min=669.307, mean=925.307, max=1269.307, sum=2775.921 (3)\", \"tab\": \"General information\", \"score\": \"925.3070000000001\"}", - "BoolQ - # output tokens": "{\"description\": \"min=1, mean=1.001, max=1.004, sum=3.004 (3)\", \"tab\": \"General information\", \"score\": \"1.0013333333333334\"}", - "BoolQ - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "BoolQ - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.65, - "details": { - "description": "min=0.593, mean=0.65, max=0.688, sum=1.95 (3)", - "tab": "Accuracy", - "NarrativeQA - ECE (10-bin)": "{\"description\": \"min=0.048, mean=0.062, max=0.079, sum=0.185 (3)\", \"tab\": \"Calibration\", \"score\": \"0.061654179655226814\"}", - "NarrativeQA - F1 (Robustness)": "{\"description\": \"min=0.331, mean=0.383, max=0.42, sum=1.148 (3)\", \"tab\": \"Robustness\", \"score\": \"0.38251983624053415\"}", - "NarrativeQA - F1 (Fairness)": "{\"description\": \"min=0.481, mean=0.548, max=0.591, sum=1.644 (3)\", \"tab\": \"Fairness\", \"score\": \"0.5478470147843514\"}", - "NarrativeQA - Denoised inference time (s)": "{\"description\": \"min=1.025, mean=1.062, max=1.132, sum=3.185 (3)\", \"tab\": \"Efficiency\", \"score\": \"1.061820745305164\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=1065 (3)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=0.958, mean=1.562, max=1.997, sum=4.687 (3)\", \"tab\": \"General information\", \"score\": \"1.5624413145539906\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=1601.997, mean=1634.99, max=1693.155, sum=4904.969 (3)\", \"tab\": \"General information\", \"score\": \"1634.9896713615024\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=5.794, mean=7.077, max=9.031, sum=21.231 (3)\", \"tab\": \"General information\", \"score\": \"7.07699530516432\"}", - "NarrativeQA - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NarrativeQA - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NarrativeQA - Stereotypes (gender)": "{\"description\": \"min=0.396, mean=0.454, max=0.5, sum=1.362 (3)\", \"tab\": \"Bias\", \"score\": \"0.4541666666666666\"}", - "NarrativeQA - Representation (race)": "{\"description\": \"min=0.333, mean=0.556, max=0.667, sum=1.667 (3)\", \"tab\": \"Bias\", \"score\": \"0.5555555555555557\"}", - "NarrativeQA - Representation (gender)": "{\"description\": \"min=0.204, mean=0.208, max=0.215, sum=0.624 (3)\", \"tab\": \"Bias\", \"score\": \"0.20801619481196945\"}", - "NarrativeQA - Toxic fraction": "{\"description\": \"min=0.011, mean=0.021, max=0.028, sum=0.062 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.020657276995305163\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (open-book)", - "source_data": { - "dataset_name": "NaturalQuestions (open-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (open-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.595, - "details": { - "description": "min=0.576, mean=0.595, max=0.607, sum=1.785 (3)", - "tab": "Accuracy", - "NaturalQuestions (closed-book) - ECE (10-bin)": "{\"description\": \"min=0.061, mean=0.068, max=0.073, sum=0.203 (3)\", \"tab\": \"Calibration\", \"score\": \"0.06770990173751885\"}", - "NaturalQuestions (open-book) - ECE (10-bin)": "{\"description\": \"min=0.075, mean=0.085, max=0.099, sum=0.254 (3)\", \"tab\": \"Calibration\", \"score\": \"0.08482055822987211\"}", - "NaturalQuestions (closed-book) - F1 (Robustness)": "{\"description\": \"min=0.233, mean=0.238, max=0.241, sum=0.713 (3)\", \"tab\": \"Robustness\", \"score\": \"0.23753663022529162\"}", - "NaturalQuestions (open-book) - F1 (Robustness)": "{\"description\": \"min=0.411, mean=0.471, max=0.518, sum=1.414 (3)\", \"tab\": \"Robustness\", \"score\": \"0.4713418135089589\"}", - "NaturalQuestions (closed-book) - F1 (Fairness)": "{\"description\": \"min=0.248, mean=0.255, max=0.259, sum=0.764 (3)\", \"tab\": \"Fairness\", \"score\": \"0.25466316487855734\"}", - "NaturalQuestions (open-book) - F1 (Fairness)": "{\"description\": \"min=0.521, mean=0.535, max=0.546, sum=1.604 (3)\", \"tab\": \"Fairness\", \"score\": \"0.5348225692810691\"}", - "NaturalQuestions (closed-book) - Denoised inference time (s)": "{\"description\": \"min=0.546, mean=0.565, max=0.586, sum=1.694 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.5647122317708332\"}", - "NaturalQuestions (open-book) - Denoised inference time (s)": "{\"description\": \"min=0.95, mean=1.085, max=1.249, sum=3.256 (3)\", \"tab\": \"Efficiency\", \"score\": \"1.0851867500000003\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=109.191, mean=111.191, max=115.191, sum=333.573 (3)\", \"tab\": \"General information\", \"score\": \"111.19099999999999\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=5.31, mean=5.844, max=6.407, sum=17.531 (3)\", \"tab\": \"General information\", \"score\": \"5.843666666666667\"}", - "NaturalQuestions (closed-book) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.538, mean=4.633, max=4.715, sum=13.899 (3)\", \"tab\": \"General information\", \"score\": \"4.633\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.039, mean=0.039, max=0.039, sum=0.117 (3)\", \"tab\": \"General information\", \"score\": \"0.039\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1261.72, mean=1481.344, max=1608.455, sum=4444.032 (3)\", \"tab\": \"General information\", \"score\": \"1481.344\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=7.154, mean=8.834, max=11.932, sum=26.502 (3)\", \"tab\": \"General information\", \"score\": \"8.834\"}", - "NaturalQuestions (open-book) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NaturalQuestions (closed-book) - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - Representation (race)": "{\"description\": \"min=0.382, mean=0.43, max=0.498, sum=1.291 (3)\", \"tab\": \"Bias\", \"score\": \"0.4304995528213292\"}", - "NaturalQuestions (closed-book) - Representation (gender)": "{\"description\": \"min=0.024, mean=0.094, max=0.18, sum=0.281 (3)\", \"tab\": \"Bias\", \"score\": \"0.09357753357753357\"}", - "NaturalQuestions (open-book) - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=2 (3)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "NaturalQuestions (open-book) - Stereotypes (gender)": "{\"description\": \"min=0.332, mean=0.388, max=0.488, sum=1.163 (3)\", \"tab\": \"Bias\", \"score\": \"0.38769841269841265\"}", - "NaturalQuestions (open-book) - Representation (race)": "{\"description\": \"min=0.373, mean=0.409, max=0.446, sum=1.226 (3)\", \"tab\": \"Bias\", \"score\": \"0.40861462430089884\"}", - "NaturalQuestions (open-book) - Representation (gender)": "{\"description\": \"min=0.026, mean=0.051, max=0.066, sum=0.153 (3)\", \"tab\": \"Bias\", \"score\": \"0.051062717190300304\"}", - "NaturalQuestions (closed-book) - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}", - "NaturalQuestions (open-book) - Toxic fraction": "{\"description\": \"min=0, mean=0.001, max=0.001, sum=0.002 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.0006666666666666666\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "QuAC", - "source_data": { - "dataset_name": "QuAC", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on QuAC", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.361, - "details": { - "description": "min=0.355, mean=0.361, max=0.365, sum=1.082 (3)", - "tab": "Accuracy", - "QuAC - ECE (10-bin)": "{\"description\": \"min=0.066, mean=0.067, max=0.07, sum=0.201 (3)\", \"tab\": \"Calibration\", \"score\": \"0.06703451532890617\"}", - "QuAC - F1 (Robustness)": "{\"description\": \"min=0.214, mean=0.215, max=0.216, sum=0.646 (3)\", \"tab\": \"Robustness\", \"score\": \"0.2154779030326859\"}", - "QuAC - F1 (Fairness)": "{\"description\": \"min=0.274, mean=0.281, max=0.287, sum=0.844 (3)\", \"tab\": \"Fairness\", \"score\": \"0.2814055112322921\"}", - "QuAC - Denoised inference time (s)": "{\"description\": \"min=2.057, mean=2.089, max=2.151, sum=6.267 (3)\", \"tab\": \"Efficiency\", \"score\": \"2.0889632337239585\"}", - "QuAC - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "QuAC - # train": "{\"description\": \"min=0.797, mean=0.881, max=0.969, sum=2.644 (3)\", \"tab\": \"General information\", \"score\": \"0.8813333333333334\"}", - "QuAC - truncated": "{\"description\": \"min=0.02, mean=0.02, max=0.02, sum=0.06 (3)\", \"tab\": \"General information\", \"score\": \"0.02\"}", - "QuAC - # prompt tokens": "{\"description\": \"min=1600.292, mean=1639.784, max=1661.675, sum=4919.353 (3)\", \"tab\": \"General information\", \"score\": \"1639.784333333333\"}", - "QuAC - # output tokens": "{\"description\": \"min=31.783, mean=32.717, max=34.585, sum=98.152 (3)\", \"tab\": \"General information\", \"score\": \"32.717333333333336\"}", - "QuAC - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "QuAC - Stereotypes (race)": "{\"description\": \"min=0.556, mean=0.582, max=0.6, sum=1.745 (3)\", \"tab\": \"Bias\", \"score\": \"0.5815402704291595\"}", - "QuAC - Stereotypes (gender)": "{\"description\": \"min=0.43, mean=0.438, max=0.449, sum=1.315 (3)\", \"tab\": \"Bias\", \"score\": \"0.4381760996205441\"}", - "QuAC - Representation (race)": "{\"description\": \"min=0.333, mean=0.344, max=0.355, sum=1.033 (3)\", \"tab\": \"Bias\", \"score\": \"0.3443830841027822\"}", - "QuAC - Representation (gender)": "{\"description\": \"min=0.223, mean=0.23, max=0.237, sum=0.691 (3)\", \"tab\": \"Bias\", \"score\": \"0.23033600244512342\"}", - "QuAC - Toxic fraction": "{\"description\": \"min=0.001, mean=0.002, max=0.003, sum=0.006 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.002\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "HellaSwag", - "source_data": { - "dataset_name": "HellaSwag", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on HellaSwag", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.811, - "details": { - "description": "min=0.811, mean=0.811, max=0.811, sum=0.811 (1)", - "tab": "Accuracy", - "HellaSwag - ECE (10-bin)": "{\"description\": \"min=0.341, mean=0.341, max=0.341, sum=0.341 (1)\", \"tab\": \"Calibration\", \"score\": \"0.34142560211110756\"}", - "HellaSwag - EM (Robustness)": "{\"description\": \"min=0.759, mean=0.759, max=0.759, sum=0.759 (1)\", \"tab\": \"Robustness\", \"score\": \"0.759\"}", - "HellaSwag - EM (Fairness)": "{\"description\": \"min=0.66, mean=0.66, max=0.66, sum=0.66 (1)\", \"tab\": \"Fairness\", \"score\": \"0.66\"}", - "HellaSwag - Denoised inference time (s)": "{\"description\": \"min=0.359, mean=0.359, max=0.359, sum=0.359 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.35889839843750027\"}", - "HellaSwag - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "HellaSwag - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "HellaSwag - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "HellaSwag - # prompt tokens": "{\"description\": \"min=88.855, mean=88.855, max=88.855, sum=88.855 (1)\", \"tab\": \"General information\", \"score\": \"88.855\"}", - "HellaSwag - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "HellaSwag - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.55, - "details": { - "description": "min=0.55, mean=0.55, max=0.55, sum=0.55 (1)", - "tab": "Accuracy", - "OpenbookQA - ECE (10-bin)": "{\"description\": \"min=0.235, mean=0.235, max=0.235, sum=0.235 (1)\", \"tab\": \"Calibration\", \"score\": \"0.23470136403728084\"}", - "OpenbookQA - EM (Robustness)": "{\"description\": \"min=0.448, mean=0.448, max=0.448, sum=0.448 (1)\", \"tab\": \"Robustness\", \"score\": \"0.448\"}", - "OpenbookQA - EM (Fairness)": "{\"description\": \"min=0.47, mean=0.47, max=0.47, sum=0.47 (1)\", \"tab\": \"Fairness\", \"score\": \"0.47\"}", - "OpenbookQA - Denoised inference time (s)": "{\"description\": \"min=0.314, mean=0.314, max=0.314, sum=0.314 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.3138882968749995\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=5.358, mean=5.358, max=5.358, sum=5.358 (1)\", \"tab\": \"General information\", \"score\": \"5.358\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "TruthfulQA", - "source_data": { - "dataset_name": "TruthfulQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on TruthfulQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.198, - "details": { - "description": "min=0.177, mean=0.198, max=0.225, sum=0.593 (3)", - "tab": "Accuracy", - "TruthfulQA - ECE (10-bin)": "{\"description\": \"min=0.075, mean=0.099, max=0.119, sum=0.298 (3)\", \"tab\": \"Calibration\", \"score\": \"0.0994665665272844\"}", - "TruthfulQA - EM (Robustness)": "{\"description\": \"min=0.122, mean=0.151, max=0.182, sum=0.454 (3)\", \"tab\": \"Robustness\", \"score\": \"0.15137614678899083\"}", - "TruthfulQA - EM (Fairness)": "{\"description\": \"min=0.138, mean=0.156, max=0.182, sum=0.469 (3)\", \"tab\": \"Fairness\", \"score\": \"0.1564729867482161\"}", - "TruthfulQA - Denoised inference time (s)": "{\"description\": \"min=0.49, mean=0.501, max=0.506, sum=1.502 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.50081436353211\"}", - "TruthfulQA - # eval": "{\"description\": \"min=654, mean=654, max=654, sum=1962 (3)\", \"tab\": \"General information\", \"score\": \"654.0\"}", - "TruthfulQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "TruthfulQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "TruthfulQA - # prompt tokens": "{\"description\": \"min=505.315, mean=514.648, max=532.315, sum=1543.945 (3)\", \"tab\": \"General information\", \"score\": \"514.6483180428135\"}", - "TruthfulQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=3 (3)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "TruthfulQA - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MS MARCO (TREC)", - "source_data": { - "dataset_name": "MS MARCO (TREC)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "NDCG@10 on MS MARCO (TREC)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.459, - "details": { - "description": "min=0.429, mean=0.459, max=0.479, sum=1.378 (3)", - "tab": "Accuracy", - "MS MARCO (regular) - RR@10 (Robustness)": "{\"description\": \"min=0.191, mean=0.207, max=0.223, sum=0.622 (3)\", \"tab\": \"Robustness\", \"score\": \"0.20732857142857117\"}", - "MS MARCO (TREC) - NDCG@10 (Robustness)": "{\"description\": \"min=0.371, mean=0.397, max=0.414, sum=1.19 (3)\", \"tab\": \"Robustness\", \"score\": \"0.39663320695609633\"}", - "MS MARCO (regular) - RR@10 (Fairness)": "{\"description\": \"min=0.211, mean=0.233, max=0.251, sum=0.698 (3)\", \"tab\": \"Fairness\", \"score\": \"0.23262777777777743\"}", - "MS MARCO (TREC) - NDCG@10 (Fairness)": "{\"description\": \"min=0.394, mean=0.431, max=0.457, sum=1.292 (3)\", \"tab\": \"Fairness\", \"score\": \"0.4307144032412258\"}", - "MS MARCO (regular) - Denoised inference time (s)": "{\"description\": \"min=0.492, mean=0.499, max=0.504, sum=1.496 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.4985355449218751\"}", - "MS MARCO (TREC) - Denoised inference time (s)": "{\"description\": \"min=0.487, mean=0.501, max=0.511, sum=1.504 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.501260492369186\"}", - "MS MARCO (regular) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "MS MARCO (regular) - # train": "{\"description\": \"min=2, mean=2, max=2, sum=6 (3)\", \"tab\": \"General information\", \"score\": \"2.0\"}", - "MS MARCO (regular) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MS MARCO (regular) - # prompt tokens": "{\"description\": \"min=497.281, mean=536.614, max=583.281, sum=1609.843 (3)\", \"tab\": \"General information\", \"score\": \"536.6143333333333\"}", - "MS MARCO (regular) - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=3 (3)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "MS MARCO (regular) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "MS MARCO (TREC) - # eval": "{\"description\": \"min=43, mean=43, max=43, sum=129 (3)\", \"tab\": \"General information\", \"score\": \"43.0\"}", - "MS MARCO (TREC) - # train": "{\"description\": \"min=2, mean=2, max=2, sum=6 (3)\", \"tab\": \"General information\", \"score\": \"2.0\"}", - "MS MARCO (TREC) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MS MARCO (TREC) - # prompt tokens": "{\"description\": \"min=480.163, mean=519.496, max=566.163, sum=1558.488 (3)\", \"tab\": \"General information\", \"score\": \"519.4961240310078\"}", - "MS MARCO (TREC) - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=3 (3)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "MS MARCO (TREC) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "MS MARCO (regular) - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "MS MARCO (TREC) - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CNN/DailyMail", - "source_data": { - "dataset_name": "CNN/DailyMail", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on CNN/DailyMail", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.144, - "details": { - "description": "min=0.14, mean=0.144, max=0.146, sum=0.861 (6)", - "tab": "Accuracy", - "CNN/DailyMail - Denoised inference time (s)": "{\"description\": \"min=4.313, mean=4.337, max=4.381, sum=26.024 (6)\", \"tab\": \"Efficiency\", \"score\": \"4.3373758759723735\"}", - "CNN/DailyMail - # eval": "{\"description\": \"min=466, mean=466, max=466, sum=2796 (6)\", \"tab\": \"General information\", \"score\": \"466.0\"}", - "CNN/DailyMail - # train": "{\"description\": \"min=5, mean=5, max=5, sum=30 (6)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CNN/DailyMail - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (6)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CNN/DailyMail - # prompt tokens": "{\"description\": \"min=1555.036, mean=1575.036, max=1602.036, sum=9450.219 (6)\", \"tab\": \"General information\", \"score\": \"1575.0364806866953\"}", - "CNN/DailyMail - # output tokens": "{\"description\": \"min=88.871, mean=89.431, max=90.324, sum=536.588 (6)\", \"tab\": \"General information\", \"score\": \"89.43133047210301\"}", - "CNN/DailyMail - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=18 (6)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "CNN/DailyMail - Stereotypes (race)": "{\"description\": \"min=0.616, mean=0.626, max=0.635, sum=3.753 (6)\", \"tab\": \"Bias\", \"score\": \"0.6255738197534654\"}", - "CNN/DailyMail - Stereotypes (gender)": "{\"description\": \"min=0.377, mean=0.387, max=0.397, sum=2.32 (6)\", \"tab\": \"Bias\", \"score\": \"0.38662344919565644\"}", - "CNN/DailyMail - Representation (race)": "{\"description\": \"min=0.244, mean=0.301, max=0.358, sum=1.808 (6)\", \"tab\": \"Bias\", \"score\": \"0.30129162776221596\"}", - "CNN/DailyMail - Representation (gender)": "{\"description\": \"min=0.104, mean=0.117, max=0.128, sum=0.7 (6)\", \"tab\": \"Bias\", \"score\": \"0.116591581511673\"}", - "CNN/DailyMail - Toxic fraction": "{\"description\": \"min=0, mean=0.002, max=0.004, sum=0.013 (6)\", \"tab\": \"Toxicity\", \"score\": \"0.002145922746781116\"}", - "CNN/DailyMail - SummaC": "{\"description\": \"min=0.393, mean=0.469, max=0.516, sum=1.407 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.46891720389173397\"}", - "CNN/DailyMail - QAFactEval": "{\"description\": \"min=4.621, mean=4.683, max=4.752, sum=28.101 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"4.683468662049275\"}", - "CNN/DailyMail - BERTScore (F1)": "{\"description\": \"min=0.257, mean=0.264, max=0.275, sum=0.792 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.2639259716833397\"}", - "CNN/DailyMail - Coverage": "{\"description\": \"min=0.897, mean=0.945, max=0.971, sum=5.671 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"0.945166441130516\"}", - "CNN/DailyMail - Density": "{\"description\": \"min=43.963, mean=49.713, max=55.846, sum=298.279 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"49.713109703758754\"}", - "CNN/DailyMail - Compression": "{\"description\": \"min=8.816, mean=9.072, max=9.547, sum=54.43 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"9.071669466217989\"}", - "CNN/DailyMail - HumanEval-faithfulness": "{\"description\": \"min=0.978, mean=0.993, max=1, sum=5.956 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"0.9925925925925926\"}", - "CNN/DailyMail - HumanEval-relevance": "{\"description\": \"min=4.422, mean=4.539, max=4.667, sum=27.237 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"4.5394335511982575\"}", - "CNN/DailyMail - HumanEval-coherence": "{\"description\": \"min=3.556, mean=3.69, max=3.81, sum=22.142 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"3.6903205726735138\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "XSUM", - "source_data": { - "dataset_name": "XSUM", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on XSUM", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.129, - "details": { - "description": "min=0.125, mean=0.129, max=0.134, sum=0.775 (6)", - "tab": "Accuracy", - "XSUM - Denoised inference time (s)": "{\"description\": \"min=1.735, mean=1.741, max=1.747, sum=10.443 (6)\", \"tab\": \"Efficiency\", \"score\": \"1.7405486446267702\"}", - "XSUM - # eval": "{\"description\": \"min=518, mean=518, max=518, sum=3108 (6)\", \"tab\": \"General information\", \"score\": \"518.0\"}", - "XSUM - # train": "{\"description\": \"min=4.996, mean=4.998, max=5, sum=29.988 (6)\", \"tab\": \"General information\", \"score\": \"4.998069498069498\"}", - "XSUM - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (6)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "XSUM - # prompt tokens": "{\"description\": \"min=1484.608, mean=1537.452, max=1572.616, sum=9224.71 (6)\", \"tab\": \"General information\", \"score\": \"1537.4517374517375\"}", - "XSUM - # output tokens": "{\"description\": \"min=24.515, mean=24.802, max=25.066, sum=148.815 (6)\", \"tab\": \"General information\", \"score\": \"24.802445302445303\"}", - "XSUM - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=18 (6)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "XSUM - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=4 (6)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "XSUM - Stereotypes (gender)": "{\"description\": \"min=0.456, mean=0.463, max=0.468, sum=2.78 (6)\", \"tab\": \"Bias\", \"score\": \"0.4633319142897687\"}", - "XSUM - Representation (race)": "{\"description\": \"min=0.532, mean=0.622, max=0.667, sum=3.73 (6)\", \"tab\": \"Bias\", \"score\": \"0.6216216216216217\"}", - "XSUM - Representation (gender)": "{\"description\": \"min=0.184, mean=0.205, max=0.224, sum=1.231 (6)\", \"tab\": \"Bias\", \"score\": \"0.2051781150126976\"}", - "XSUM - Toxic fraction": "{\"description\": \"min=0, mean=0.001, max=0.002, sum=0.004 (6)\", \"tab\": \"Toxicity\", \"score\": \"0.0006435006435006435\"}", - "XSUM - SummaC": "{\"description\": \"min=-0.265, mean=-0.253, max=-0.236, sum=-0.758 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"-0.252571659198599\"}", - "XSUM - QAFactEval": "{\"description\": \"min=2.761, mean=2.981, max=3.213, sum=17.888 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"2.981288283366219\"}", - "XSUM - BERTScore (F1)": "{\"description\": \"min=0.431, mean=0.434, max=0.438, sum=1.301 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.4335328367301425\"}", - "XSUM - Coverage": "{\"description\": \"min=0.794, mean=0.8, max=0.803, sum=4.797 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"0.7995514803953769\"}", - "XSUM - Density": "{\"description\": \"min=2.71, mean=2.945, max=3.142, sum=17.67 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"2.945005615644467\"}", - "XSUM - Compression": "{\"description\": \"min=18.323, mean=18.422, max=18.574, sum=110.533 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"18.422086618359014\"}", - "XSUM - HumanEval-faithfulness": "{\"description\": \"min=0.638, mean=0.661, max=0.697, sum=3.968 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"0.6612578878025103\"}", - "XSUM - HumanEval-relevance": "{\"description\": \"min=4.212, mean=4.239, max=4.275, sum=25.431 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"4.238517902133463\"}", - "XSUM - HumanEval-coherence": "{\"description\": \"min=4.773, mean=4.825, max=4.877, sum=28.952 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"4.825335737235052\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "IMDB", - "source_data": { - "dataset_name": "IMDB", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on IMDB", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.956, - "details": { - "description": "min=0.941, mean=0.956, max=0.965, sum=2.867 (3)", - "tab": "Accuracy", - "IMDB - ECE (10-bin)": "{\"description\": \"min=0.05, mean=0.069, max=0.081, sum=0.206 (3)\", \"tab\": \"Calibration\", \"score\": \"0.06875792133691605\"}", - "IMDB - EM (Robustness)": "{\"description\": \"min=0.907, mean=0.923, max=0.933, sum=2.768 (3)\", \"tab\": \"Robustness\", \"score\": \"0.9226666666666667\"}", - "IMDB - EM (Fairness)": "{\"description\": \"min=0.93, mean=0.949, max=0.96, sum=2.846 (3)\", \"tab\": \"Fairness\", \"score\": \"0.9486666666666667\"}", - "IMDB - Denoised inference time (s)": "{\"description\": \"min=0.709, mean=0.796, max=0.865, sum=2.389 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.7963252441406254\"}", - "IMDB - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "IMDB - # train": "{\"description\": \"min=2.903, mean=4.229, max=4.983, sum=12.688 (3)\", \"tab\": \"General information\", \"score\": \"4.229333333333333\"}", - "IMDB - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IMDB - # prompt tokens": "{\"description\": \"min=1283.038, mean=1562.808, max=1784.2, sum=4688.425 (3)\", \"tab\": \"General information\", \"score\": \"1562.8083333333334\"}", - "IMDB - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=3 (3)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "IMDB - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "IMDB - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CivilComments", - "source_data": { - "dataset_name": "CivilComments", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on CivilComments", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.532, - "details": { - "description": "min=0.001, mean=0.532, max=1, sum=28.726 (54)", - "tab": "Accuracy", - "CivilComments - ECE (10-bin)": "{\"description\": \"min=0.051, mean=0.327, max=0.708, sum=17.639 (54)\", \"tab\": \"Calibration\", \"score\": \"0.32664532725883244\"}", - "CivilComments - EM (Robustness)": "{\"description\": \"min=0, mean=0.32, max=0.817, sum=17.265 (54)\", \"tab\": \"Robustness\", \"score\": \"0.31971446667223646\"}", - "CivilComments - EM (Fairness)": "{\"description\": \"min=0.001, mean=0.479, max=1, sum=25.855 (54)\", \"tab\": \"Fairness\", \"score\": \"0.4787922217178853\"}", - "CivilComments - Denoised inference time (s)": "{\"description\": \"min=0.464, mean=0.546, max=0.711, sum=29.484 (54)\", \"tab\": \"Efficiency\", \"score\": \"0.5459943267746123\"}", - "CivilComments - # eval": "{\"description\": \"min=74, mean=371.556, max=683, sum=20064 (54)\", \"tab\": \"General information\", \"score\": \"371.55555555555554\"}", - "CivilComments - # train": "{\"description\": \"min=5, mean=5, max=5, sum=270 (54)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CivilComments - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (54)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CivilComments - # prompt tokens": "{\"description\": \"min=362.293, mean=732.514, max=1288.441, sum=39555.782 (54)\", \"tab\": \"General information\", \"score\": \"732.5144825548033\"}", - "CivilComments - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=54 (54)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "CivilComments - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=162 (54)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "CivilComments - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (54)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "RAFT", - "source_data": { - "dataset_name": "RAFT", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on RAFT", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.633, - "details": { - "description": "min=0.1, mean=0.633, max=0.95, sum=20.875 (33)", - "tab": "Accuracy", - "RAFT - ECE (10-bin)": "{\"description\": \"min=0.093, mean=0.274, max=0.825, sum=9.044 (33)\", \"tab\": \"Calibration\", \"score\": \"0.274053604040966\"}", - "RAFT - EM (Robustness)": "{\"description\": \"min=0, mean=0.563, max=0.925, sum=18.575 (33)\", \"tab\": \"Robustness\", \"score\": \"0.5628787878787879\"}", - "RAFT - EM (Fairness)": "{\"description\": \"min=0.05, mean=0.598, max=0.95, sum=19.75 (33)\", \"tab\": \"Fairness\", \"score\": \"0.5984848484848486\"}", - "RAFT - Denoised inference time (s)": "{\"description\": \"min=0.458, mean=0.667, max=0.987, sum=22.019 (33)\", \"tab\": \"Efficiency\", \"score\": \"0.6672338778409089\"}", - "RAFT - # eval": "{\"description\": \"min=40, mean=40, max=40, sum=1320 (33)\", \"tab\": \"General information\", \"score\": \"40.0\"}", - "RAFT - # train": "{\"description\": \"min=0, mean=4.557, max=5, sum=150.375 (33)\", \"tab\": \"General information\", \"score\": \"4.556818181818182\"}", - "RAFT - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (33)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "RAFT - # prompt tokens": "{\"description\": \"min=270.325, mean=814.446, max=1777.025, sum=26876.725 (33)\", \"tab\": \"General information\", \"score\": \"814.446212121212\"}", - "RAFT - # output tokens": "{\"description\": \"min=0.275, mean=3.051, max=5.95, sum=100.675 (33)\", \"tab\": \"General information\", \"score\": \"3.0507575757575767\"}", - "RAFT - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=99 (33)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "RAFT - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (33)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/cohere_cohere-xlarge-v20221108-52.4b.json b/data/models/cohere_cohere-xlarge-v20221108-52.4b.json deleted file mode 100644 index 33259e7f43f67fe52aab80ff583ceb162f273be9..0000000000000000000000000000000000000000 --- a/data/models/cohere_cohere-xlarge-v20221108-52.4b.json +++ /dev/null @@ -1,674 +0,0 @@ -{ - "model_info": { - "name": "Cohere xlarge v20221108 52.4B", - "id": "cohere/Cohere-xlarge-v20221108-52.4B", - "developer": "cohere", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_classic/cohere_Cohere-xlarge-v20221108-52.4B/1774096308.339228", - "retrieved_timestamp": "1774096308.339228", - "source_metadata": { - "source_name": "helm_classic", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_classic", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_classic", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperform on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.664, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Calibration": "{\"description\": \"\", \"tab\": \"Calibration\", \"score\": \"0.5846823928461301\"}", - "Mean win rate - Robustness": "{\"description\": \"\", \"tab\": \"Robustness\", \"score\": \"0.5964421748070247\"}", - "Mean win rate - Fairness": "{\"description\": \"\", \"tab\": \"Fairness\", \"score\": \"0.6082341462764155\"}", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}", - "Mean win rate - Bias": "{\"description\": \"\", \"tab\": \"Bias\", \"score\": \"0.601504827172334\"}", - "Mean win rate - Toxicity": "{\"description\": \"\", \"tab\": \"Toxicity\", \"score\": \"0.5642015392015391\"}", - "Mean win rate - Summarization metrics": "{\"description\": \"\", \"tab\": \"Summarization metrics\", \"score\": \"0.7039473684210527\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.382, - "details": { - "description": "min=0.21, mean=0.382, max=0.67, sum=5.731 (15)", - "tab": "Accuracy", - "MMLU - ECE (10-bin)": "{\"description\": \"min=0.104, mean=0.143, max=0.197, sum=2.146 (15)\", \"tab\": \"Calibration\", \"score\": \"0.14305203655556303\"}", - "MMLU - EM (Robustness)": "{\"description\": \"min=0.12, mean=0.299, max=0.6, sum=4.49 (15)\", \"tab\": \"Robustness\", \"score\": \"0.29933333333333334\"}", - "MMLU - EM (Fairness)": "{\"description\": \"min=0.13, mean=0.317, max=0.57, sum=4.748 (15)\", \"tab\": \"Fairness\", \"score\": \"0.31652631578947366\"}", - "MMLU - Denoised inference time (s)": "{\"description\": \"5 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=1542 (15)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=75 (15)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (15)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=372.75, mean=481.26, max=628.421, sum=7218.903 (15)\", \"tab\": \"General information\", \"score\": \"481.2602105263158\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=15 (15)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "MMLU - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=45 (15)\", \"tab\": \"General information\", \"score\": \"3.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "BoolQ", - "source_data": { - "dataset_name": "BoolQ", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on BoolQ", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.762, - "details": { - "description": "min=0.761, mean=0.762, max=0.763, sum=2.285 (3)", - "tab": "Accuracy", - "BoolQ - ECE (10-bin)": "{\"description\": \"min=0.037, mean=0.051, max=0.062, sum=0.154 (3)\", \"tab\": \"Calibration\", \"score\": \"0.05127903463780418\"}", - "BoolQ - EM (Robustness)": "{\"description\": \"min=0.712, mean=0.718, max=0.722, sum=2.153 (3)\", \"tab\": \"Robustness\", \"score\": \"0.7176666666666667\"}", - "BoolQ - EM (Fairness)": "{\"description\": \"min=0.702, mean=0.708, max=0.72, sum=2.124 (3)\", \"tab\": \"Fairness\", \"score\": \"0.7079999999999999\"}", - "BoolQ - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "BoolQ - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "BoolQ - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "BoolQ - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "BoolQ - # prompt tokens": "{\"description\": \"min=669.307, mean=925.307, max=1269.307, sum=2775.921 (3)\", \"tab\": \"General information\", \"score\": \"925.3070000000001\"}", - "BoolQ - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=3 (3)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "BoolQ - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "BoolQ - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.672, - "details": { - "description": "min=0.607, mean=0.672, max=0.708, sum=2.017 (3)", - "tab": "Accuracy", - "NarrativeQA - ECE (10-bin)": "{\"description\": \"min=0.042, mean=0.059, max=0.072, sum=0.178 (3)\", \"tab\": \"Calibration\", \"score\": \"0.059183266964369506\"}", - "NarrativeQA - F1 (Robustness)": "{\"description\": \"min=0.313, mean=0.39, max=0.434, sum=1.171 (3)\", \"tab\": \"Robustness\", \"score\": \"0.3901906178600691\"}", - "NarrativeQA - F1 (Fairness)": "{\"description\": \"min=0.486, mean=0.553, max=0.589, sum=1.659 (3)\", \"tab\": \"Fairness\", \"score\": \"0.5530542667501213\"}", - "NarrativeQA - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=1065 (3)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=0.958, mean=1.562, max=1.997, sum=4.687 (3)\", \"tab\": \"General information\", \"score\": \"1.5624413145539906\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=1601.997, mean=1634.99, max=1693.155, sum=4904.969 (3)\", \"tab\": \"General information\", \"score\": \"1634.9896713615024\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=5.792, mean=6.729, max=8.434, sum=20.186 (3)\", \"tab\": \"General information\", \"score\": \"6.728638497652582\"}", - "NarrativeQA - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NarrativeQA - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NarrativeQA - Stereotypes (gender)": "{\"description\": \"min=0.417, mean=0.472, max=0.5, sum=1.417 (3)\", \"tab\": \"Bias\", \"score\": \"0.47222222222222227\"}", - "NarrativeQA - Representation (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=2 (3)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "NarrativeQA - Representation (gender)": "{\"description\": \"min=0.184, mean=0.192, max=0.197, sum=0.575 (3)\", \"tab\": \"Bias\", \"score\": \"0.19158509798903886\"}", - "NarrativeQA - Toxic fraction": "{\"description\": \"min=0.008, mean=0.013, max=0.02, sum=0.039 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.013145539906103287\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (open-book)", - "source_data": { - "dataset_name": "NaturalQuestions (open-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (open-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.628, - "details": { - "description": "min=0.619, mean=0.628, max=0.634, sum=1.885 (3)", - "tab": "Accuracy", - "NaturalQuestions (closed-book) - ECE (10-bin)": "{\"description\": \"min=0.044, mean=0.054, max=0.064, sum=0.163 (3)\", \"tab\": \"Calibration\", \"score\": \"0.05430103491623906\"}", - "NaturalQuestions (open-book) - ECE (10-bin)": "{\"description\": \"min=0.064, mean=0.073, max=0.08, sum=0.219 (3)\", \"tab\": \"Calibration\", \"score\": \"0.07296237131206641\"}", - "NaturalQuestions (closed-book) - F1 (Robustness)": "{\"description\": \"min=0.276, mean=0.283, max=0.288, sum=0.85 (3)\", \"tab\": \"Robustness\", \"score\": \"0.28349840532468856\"}", - "NaturalQuestions (open-book) - F1 (Robustness)": "{\"description\": \"min=0.49, mean=0.533, max=0.555, sum=1.598 (3)\", \"tab\": \"Robustness\", \"score\": \"0.532530651706331\"}", - "NaturalQuestions (closed-book) - F1 (Fairness)": "{\"description\": \"min=0.295, mean=0.299, max=0.303, sum=0.898 (3)\", \"tab\": \"Fairness\", \"score\": \"0.299210546403295\"}", - "NaturalQuestions (open-book) - F1 (Fairness)": "{\"description\": \"min=0.548, mean=0.566, max=0.58, sum=1.699 (3)\", \"tab\": \"Fairness\", \"score\": \"0.5664508489119625\"}", - "NaturalQuestions (closed-book) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NaturalQuestions (open-book) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=109.191, mean=111.191, max=115.191, sum=333.573 (3)\", \"tab\": \"General information\", \"score\": \"111.19099999999999\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=4.528, mean=4.808, max=5.211, sum=14.424 (3)\", \"tab\": \"General information\", \"score\": \"4.808\"}", - "NaturalQuestions (closed-book) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.538, mean=4.633, max=4.715, sum=13.899 (3)\", \"tab\": \"General information\", \"score\": \"4.633\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.039, mean=0.039, max=0.039, sum=0.117 (3)\", \"tab\": \"General information\", \"score\": \"0.039\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1261.72, mean=1481.344, max=1608.455, sum=4444.032 (3)\", \"tab\": \"General information\", \"score\": \"1481.344\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=5.836, mean=6.093, max=6.582, sum=18.278 (3)\", \"tab\": \"General information\", \"score\": \"6.092666666666666\"}", - "NaturalQuestions (open-book) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NaturalQuestions (closed-book) - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - Stereotypes (gender)": "{\"description\": \"min=0.333, mean=0.444, max=0.5, sum=1.333 (3)\", \"tab\": \"Bias\", \"score\": \"0.4444444444444444\"}", - "NaturalQuestions (closed-book) - Representation (race)": "{\"description\": \"min=0.407, mean=0.48, max=0.556, sum=1.441 (3)\", \"tab\": \"Bias\", \"score\": \"0.4804079441760602\"}", - "NaturalQuestions (closed-book) - Representation (gender)": "{\"description\": \"min=0.147, mean=0.247, max=0.385, sum=0.741 (3)\", \"tab\": \"Bias\", \"score\": \"0.24693627450980396\"}", - "NaturalQuestions (open-book) - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=1.333 (2)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666667\"}", - "NaturalQuestions (open-book) - Stereotypes (gender)": "{\"description\": \"min=0.186, mean=0.232, max=0.278, sum=0.697 (3)\", \"tab\": \"Bias\", \"score\": \"0.2324074074074074\"}", - "NaturalQuestions (open-book) - Representation (race)": "{\"description\": \"min=0.467, mean=0.474, max=0.483, sum=1.423 (3)\", \"tab\": \"Bias\", \"score\": \"0.4744480248239647\"}", - "NaturalQuestions (open-book) - Representation (gender)": "{\"description\": \"min=0.092, mean=0.113, max=0.135, sum=0.339 (3)\", \"tab\": \"Bias\", \"score\": \"0.11298873219533077\"}", - "NaturalQuestions (closed-book) - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}", - "NaturalQuestions (open-book) - Toxic fraction": "{\"description\": \"min=0.001, mean=0.001, max=0.001, sum=0.003 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.001\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "QuAC", - "source_data": { - "dataset_name": "QuAC", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on QuAC", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.374, - "details": { - "description": "min=0.367, mean=0.374, max=0.378, sum=1.122 (3)", - "tab": "Accuracy", - "QuAC - ECE (10-bin)": "{\"description\": \"min=0.053, mean=0.063, max=0.072, sum=0.189 (3)\", \"tab\": \"Calibration\", \"score\": \"0.06295082132498765\"}", - "QuAC - F1 (Robustness)": "{\"description\": \"min=0.221, mean=0.229, max=0.234, sum=0.686 (3)\", \"tab\": \"Robustness\", \"score\": \"0.22865454547247813\"}", - "QuAC - F1 (Fairness)": "{\"description\": \"min=0.269, mean=0.275, max=0.278, sum=0.824 (3)\", \"tab\": \"Fairness\", \"score\": \"0.27469570002834404\"}", - "QuAC - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "QuAC - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "QuAC - # train": "{\"description\": \"min=0.797, mean=0.881, max=0.969, sum=2.644 (3)\", \"tab\": \"General information\", \"score\": \"0.8813333333333334\"}", - "QuAC - truncated": "{\"description\": \"min=0.02, mean=0.02, max=0.02, sum=0.06 (3)\", \"tab\": \"General information\", \"score\": \"0.02\"}", - "QuAC - # prompt tokens": "{\"description\": \"min=1600.292, mean=1639.784, max=1661.675, sum=4919.353 (3)\", \"tab\": \"General information\", \"score\": \"1639.784333333333\"}", - "QuAC - # output tokens": "{\"description\": \"min=24.612, mean=27.944, max=31.344, sum=83.832 (3)\", \"tab\": \"General information\", \"score\": \"27.944\"}", - "QuAC - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "QuAC - Stereotypes (race)": "{\"description\": \"min=0.543, mean=0.571, max=0.589, sum=1.713 (3)\", \"tab\": \"Bias\", \"score\": \"0.570980870980871\"}", - "QuAC - Stereotypes (gender)": "{\"description\": \"min=0.371, mean=0.395, max=0.426, sum=1.185 (3)\", \"tab\": \"Bias\", \"score\": \"0.3948930748680999\"}", - "QuAC - Representation (race)": "{\"description\": \"min=0.253, mean=0.304, max=0.331, sum=0.912 (3)\", \"tab\": \"Bias\", \"score\": \"0.3038684617631986\"}", - "QuAC - Representation (gender)": "{\"description\": \"min=0.211, mean=0.233, max=0.263, sum=0.699 (3)\", \"tab\": \"Bias\", \"score\": \"0.2330910766304025\"}", - "QuAC - Toxic fraction": "{\"description\": \"min=0.001, mean=0.002, max=0.003, sum=0.007 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.0023333333333333335\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "HellaSwag", - "source_data": { - "dataset_name": "HellaSwag", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on HellaSwag", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.81, - "details": { - "description": "min=0.81, mean=0.81, max=0.81, sum=0.81 (1)", - "tab": "Accuracy", - "HellaSwag - ECE (10-bin)": "{\"description\": \"min=0.333, mean=0.333, max=0.333, sum=0.333 (1)\", \"tab\": \"Calibration\", \"score\": \"0.3332417863062664\"}", - "HellaSwag - EM (Robustness)": "{\"description\": \"min=0.764, mean=0.764, max=0.764, sum=0.764 (1)\", \"tab\": \"Robustness\", \"score\": \"0.764\"}", - "HellaSwag - EM (Fairness)": "{\"description\": \"min=0.687, mean=0.687, max=0.687, sum=0.687 (1)\", \"tab\": \"Fairness\", \"score\": \"0.687\"}", - "HellaSwag - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "HellaSwag - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "HellaSwag - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "HellaSwag - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "HellaSwag - # prompt tokens": "{\"description\": \"min=88.855, mean=88.855, max=88.855, sum=88.855 (1)\", \"tab\": \"General information\", \"score\": \"88.855\"}", - "HellaSwag - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "HellaSwag - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.588, - "details": { - "description": "min=0.588, mean=0.588, max=0.588, sum=0.588 (1)", - "tab": "Accuracy", - "OpenbookQA - ECE (10-bin)": "{\"description\": \"min=0.207, mean=0.207, max=0.207, sum=0.207 (1)\", \"tab\": \"Calibration\", \"score\": \"0.20665896753536225\"}", - "OpenbookQA - EM (Robustness)": "{\"description\": \"min=0.482, mean=0.482, max=0.482, sum=0.482 (1)\", \"tab\": \"Robustness\", \"score\": \"0.482\"}", - "OpenbookQA - EM (Fairness)": "{\"description\": \"min=0.5, mean=0.5, max=0.5, sum=0.5 (1)\", \"tab\": \"Fairness\", \"score\": \"0.5\"}", - "OpenbookQA - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=5.358, mean=5.358, max=5.358, sum=5.358 (1)\", \"tab\": \"General information\", \"score\": \"5.358\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "TruthfulQA", - "source_data": { - "dataset_name": "TruthfulQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on TruthfulQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.169, - "details": { - "description": "min=0.164, mean=0.169, max=0.179, sum=0.508 (3)", - "tab": "Accuracy", - "TruthfulQA - ECE (10-bin)": "{\"description\": \"min=0.18, mean=0.211, max=0.233, sum=0.633 (3)\", \"tab\": \"Calibration\", \"score\": \"0.21105124875435366\"}", - "TruthfulQA - EM (Robustness)": "{\"description\": \"min=0.106, mean=0.116, max=0.13, sum=0.349 (3)\", \"tab\": \"Robustness\", \"score\": \"0.1162079510703364\"}", - "TruthfulQA - EM (Fairness)": "{\"description\": \"min=0.112, mean=0.12, max=0.124, sum=0.359 (3)\", \"tab\": \"Fairness\", \"score\": \"0.1197757390417941\"}", - "TruthfulQA - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "TruthfulQA - # eval": "{\"description\": \"min=654, mean=654, max=654, sum=1962 (3)\", \"tab\": \"General information\", \"score\": \"654.0\"}", - "TruthfulQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "TruthfulQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "TruthfulQA - # prompt tokens": "{\"description\": \"min=505.315, mean=514.648, max=532.315, sum=1543.945 (3)\", \"tab\": \"General information\", \"score\": \"514.6483180428135\"}", - "TruthfulQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=3 (3)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "TruthfulQA - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MS MARCO (TREC)", - "source_data": { - "dataset_name": "MS MARCO (TREC)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "NDCG@10 on MS MARCO (TREC)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.55, - "details": { - "description": "min=0.526, mean=0.55, max=0.573, sum=1.65 (3)", - "tab": "Accuracy", - "MS MARCO (regular) - RR@10 (Robustness)": "{\"description\": \"min=0.201, mean=0.242, max=0.292, sum=0.725 (3)\", \"tab\": \"Robustness\", \"score\": \"0.24177817460317433\"}", - "MS MARCO (TREC) - NDCG@10 (Robustness)": "{\"description\": \"min=0.449, mean=0.482, max=0.527, sum=1.446 (3)\", \"tab\": \"Robustness\", \"score\": \"0.48206153384583117\"}", - "MS MARCO (regular) - RR@10 (Fairness)": "{\"description\": \"min=0.239, mean=0.267, max=0.302, sum=0.802 (3)\", \"tab\": \"Fairness\", \"score\": \"0.2673071428571425\"}", - "MS MARCO (TREC) - NDCG@10 (Fairness)": "{\"description\": \"min=0.51, mean=0.522, max=0.544, sum=1.565 (3)\", \"tab\": \"Fairness\", \"score\": \"0.5216640091882355\"}", - "MS MARCO (regular) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MS MARCO (TREC) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MS MARCO (regular) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "MS MARCO (regular) - # train": "{\"description\": \"min=2, mean=2, max=2, sum=6 (3)\", \"tab\": \"General information\", \"score\": \"2.0\"}", - "MS MARCO (regular) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MS MARCO (regular) - # prompt tokens": "{\"description\": \"min=497.281, mean=536.614, max=583.281, sum=1609.843 (3)\", \"tab\": \"General information\", \"score\": \"536.6143333333333\"}", - "MS MARCO (regular) - # output tokens": "{\"description\": \"min=1, mean=1.002, max=1.005, sum=3.005 (3)\", \"tab\": \"General information\", \"score\": \"1.0016666666666667\"}", - "MS MARCO (regular) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "MS MARCO (TREC) - # eval": "{\"description\": \"min=43, mean=43, max=43, sum=129 (3)\", \"tab\": \"General information\", \"score\": \"43.0\"}", - "MS MARCO (TREC) - # train": "{\"description\": \"min=2, mean=2, max=2, sum=6 (3)\", \"tab\": \"General information\", \"score\": \"2.0\"}", - "MS MARCO (TREC) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MS MARCO (TREC) - # prompt tokens": "{\"description\": \"min=480.163, mean=519.496, max=566.163, sum=1558.488 (3)\", \"tab\": \"General information\", \"score\": \"519.4961240310078\"}", - "MS MARCO (TREC) - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=3 (3)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "MS MARCO (TREC) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "MS MARCO (regular) - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "MS MARCO (TREC) - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CNN/DailyMail", - "source_data": { - "dataset_name": "CNN/DailyMail", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on CNN/DailyMail", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.153, - "details": { - "description": "min=0.153, mean=0.153, max=0.154, sum=0.92 (6)", - "tab": "Accuracy", - "CNN/DailyMail - Denoised inference time (s)": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "CNN/DailyMail - # eval": "{\"description\": \"min=466, mean=466, max=466, sum=2796 (6)\", \"tab\": \"General information\", \"score\": \"466.0\"}", - "CNN/DailyMail - # train": "{\"description\": \"min=5, mean=5, max=5, sum=30 (6)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CNN/DailyMail - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (6)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CNN/DailyMail - # prompt tokens": "{\"description\": \"min=1555.036, mean=1575.036, max=1602.036, sum=9450.219 (6)\", \"tab\": \"General information\", \"score\": \"1575.0364806866953\"}", - "CNN/DailyMail - # output tokens": "{\"description\": \"min=89.47, mean=91.338, max=92.403, sum=548.03 (6)\", \"tab\": \"General information\", \"score\": \"91.33834048640915\"}", - "CNN/DailyMail - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=18 (6)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "CNN/DailyMail - Stereotypes (race)": "{\"description\": \"min=0.579, mean=0.607, max=0.649, sum=3.642 (6)\", \"tab\": \"Bias\", \"score\": \"0.606957921303154\"}", - "CNN/DailyMail - Stereotypes (gender)": "{\"description\": \"min=0.362, mean=0.383, max=0.409, sum=2.3 (6)\", \"tab\": \"Bias\", \"score\": \"0.3833873353199473\"}", - "CNN/DailyMail - Representation (race)": "{\"description\": \"min=0.223, mean=0.266, max=0.328, sum=1.597 (6)\", \"tab\": \"Bias\", \"score\": \"0.26620678930063096\"}", - "CNN/DailyMail - Representation (gender)": "{\"description\": \"min=0.128, mean=0.133, max=0.14, sum=0.796 (6)\", \"tab\": \"Bias\", \"score\": \"0.1326032519141558\"}", - "CNN/DailyMail - Toxic fraction": "{\"description\": \"min=0, mean=0.001, max=0.002, sum=0.009 (6)\", \"tab\": \"Toxicity\", \"score\": \"0.001430615164520744\"}", - "CNN/DailyMail - SummaC": "{\"description\": \"min=0.469, mean=0.514, max=0.552, sum=1.542 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.5141110990456594\"}", - "CNN/DailyMail - QAFactEval": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - BERTScore (F1)": "{\"description\": \"min=0.281, mean=0.286, max=0.295, sum=0.858 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.2858638938260981\"}", - "CNN/DailyMail - Coverage": "{\"description\": \"min=0.946, mean=0.971, max=0.984, sum=5.823 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"0.9705641483765838\"}", - "CNN/DailyMail - Density": "{\"description\": \"min=41.158, mean=44.772, max=50.734, sum=268.631 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"44.771778103334206\"}", - "CNN/DailyMail - Compression": "{\"description\": \"min=7.733, mean=8.026, max=8.278, sum=48.156 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"8.02592370223569\"}", - "CNN/DailyMail - HumanEval-faithfulness": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-relevance": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-coherence": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "XSUM", - "source_data": { - "dataset_name": "XSUM", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on XSUM", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.153, - "details": { - "description": "min=0.148, mean=0.153, max=0.158, sum=0.919 (6)", - "tab": "Accuracy", - "XSUM - Denoised inference time (s)": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "XSUM - # eval": "{\"description\": \"min=518, mean=518, max=518, sum=3108 (6)\", \"tab\": \"General information\", \"score\": \"518.0\"}", - "XSUM - # train": "{\"description\": \"min=4.996, mean=4.998, max=5, sum=29.988 (6)\", \"tab\": \"General information\", \"score\": \"4.998069498069498\"}", - "XSUM - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (6)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "XSUM - # prompt tokens": "{\"description\": \"min=1484.608, mean=1537.452, max=1572.616, sum=9224.71 (6)\", \"tab\": \"General information\", \"score\": \"1537.4517374517375\"}", - "XSUM - # output tokens": "{\"description\": \"min=25.925, mean=26.153, max=26.423, sum=156.919 (6)\", \"tab\": \"General information\", \"score\": \"26.153153153153156\"}", - "XSUM - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=18 (6)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "XSUM - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=4 (6)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "XSUM - Stereotypes (gender)": "{\"description\": \"min=0.447, mean=0.454, max=0.463, sum=2.724 (6)\", \"tab\": \"Bias\", \"score\": \"0.45401696819707577\"}", - "XSUM - Representation (race)": "{\"description\": \"min=0.515, mean=0.537, max=0.565, sum=3.223 (6)\", \"tab\": \"Bias\", \"score\": \"0.5371029656743943\"}", - "XSUM - Representation (gender)": "{\"description\": \"min=0.204, mean=0.218, max=0.236, sum=1.306 (6)\", \"tab\": \"Bias\", \"score\": \"0.2176913745770286\"}", - "XSUM - Toxic fraction": "{\"description\": \"min=0, mean=0.001, max=0.002, sum=0.004 (6)\", \"tab\": \"Toxicity\", \"score\": \"0.0006435006435006435\"}", - "XSUM - SummaC": "{\"description\": \"min=-0.28, mean=-0.258, max=-0.245, sum=-0.774 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"-0.25799066096812756\"}", - "XSUM - QAFactEval": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - BERTScore (F1)": "{\"description\": \"min=0.447, mean=0.451, max=0.454, sum=1.354 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.45133514557325344\"}", - "XSUM - Coverage": "{\"description\": \"min=0.79, mean=0.798, max=0.803, sum=4.787 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"0.7978456468638059\"}", - "XSUM - Density": "{\"description\": \"min=2.823, mean=3.009, max=3.208, sum=18.053 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"3.008801536227543\"}", - "XSUM - Compression": "{\"description\": \"min=17.074, mean=17.188, max=17.359, sum=103.128 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"17.187984260626735\"}", - "XSUM - HumanEval-faithfulness": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-relevance": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-coherence": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "IMDB", - "source_data": { - "dataset_name": "IMDB", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on IMDB", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.956, - "details": { - "description": "min=0.941, mean=0.956, max=0.965, sum=2.868 (3)", - "tab": "Accuracy", - "IMDB - ECE (10-bin)": "{\"description\": \"min=0.05, mean=0.069, max=0.082, sum=0.207 (3)\", \"tab\": \"Calibration\", \"score\": \"0.06908904600115551\"}", - "IMDB - EM (Robustness)": "{\"description\": \"min=0.907, mean=0.923, max=0.933, sum=2.769 (3)\", \"tab\": \"Robustness\", \"score\": \"0.923\"}", - "IMDB - EM (Fairness)": "{\"description\": \"min=0.931, mean=0.949, max=0.96, sum=2.847 (3)\", \"tab\": \"Fairness\", \"score\": \"0.949\"}", - "IMDB - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "IMDB - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "IMDB - # train": "{\"description\": \"min=2.903, mean=4.229, max=4.983, sum=12.688 (3)\", \"tab\": \"General information\", \"score\": \"4.229333333333333\"}", - "IMDB - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IMDB - # prompt tokens": "{\"description\": \"min=1283.038, mean=1562.808, max=1784.2, sum=4688.425 (3)\", \"tab\": \"General information\", \"score\": \"1562.8083333333334\"}", - "IMDB - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=3 (3)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "IMDB - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "IMDB - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CivilComments", - "source_data": { - "dataset_name": "CivilComments", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on CivilComments", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.524, - "details": { - "description": "min=0.035, mean=0.524, max=0.968, sum=28.319 (54)", - "tab": "Accuracy", - "CivilComments - ECE (10-bin)": "{\"description\": \"min=0.056, mean=0.313, max=0.651, sum=16.899 (54)\", \"tab\": \"Calibration\", \"score\": \"0.3129455444585645\"}", - "CivilComments - EM (Robustness)": "{\"description\": \"min=0.012, mean=0.408, max=0.908, sum=22.047 (54)\", \"tab\": \"Robustness\", \"score\": \"0.408272754767954\"}", - "CivilComments - EM (Fairness)": "{\"description\": \"min=0.03, mean=0.415, max=0.875, sum=22.43 (54)\", \"tab\": \"Fairness\", \"score\": \"0.41537457925495214\"}", - "CivilComments - Denoised inference time (s)": "{\"description\": \"9 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "CivilComments - # eval": "{\"description\": \"min=74, mean=371.556, max=683, sum=20064 (54)\", \"tab\": \"General information\", \"score\": \"371.55555555555554\"}", - "CivilComments - # train": "{\"description\": \"min=5, mean=5, max=5, sum=270 (54)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CivilComments - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (54)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CivilComments - # prompt tokens": "{\"description\": \"min=362.293, mean=732.514, max=1288.441, sum=39555.782 (54)\", \"tab\": \"General information\", \"score\": \"732.5144825548033\"}", - "CivilComments - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=54 (54)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "CivilComments - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=162 (54)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "CivilComments - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (54)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "RAFT", - "source_data": { - "dataset_name": "RAFT", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on RAFT", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.624, - "details": { - "description": "min=0, mean=0.624, max=0.975, sum=20.6 (33)", - "tab": "Accuracy", - "RAFT - ECE (10-bin)": "{\"description\": \"min=0.078, mean=0.25, max=1, sum=8.255 (33)\", \"tab\": \"Calibration\", \"score\": \"0.2501605016965272\"}", - "RAFT - EM (Robustness)": "{\"description\": \"min=0, mean=0.489, max=0.925, sum=16.125 (33)\", \"tab\": \"Robustness\", \"score\": \"0.48863636363636365\"}", - "RAFT - EM (Fairness)": "{\"description\": \"min=0, mean=0.604, max=0.975, sum=19.925 (33)\", \"tab\": \"Fairness\", \"score\": \"0.6037878787878787\"}", - "RAFT - Denoised inference time (s)": "{\"description\": \"11 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "RAFT - # eval": "{\"description\": \"min=40, mean=40, max=40, sum=1320 (33)\", \"tab\": \"General information\", \"score\": \"40.0\"}", - "RAFT - # train": "{\"description\": \"min=0, mean=4.557, max=5, sum=150.375 (33)\", \"tab\": \"General information\", \"score\": \"4.556818181818182\"}", - "RAFT - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (33)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "RAFT - # prompt tokens": "{\"description\": \"min=270.325, mean=814.446, max=1777.025, sum=26876.725 (33)\", \"tab\": \"General information\", \"score\": \"814.446212121212\"}", - "RAFT - # output tokens": "{\"description\": \"min=0, mean=2.99, max=7.05, sum=98.675 (33)\", \"tab\": \"General information\", \"score\": \"2.9901515151515157\"}", - "RAFT - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=99 (33)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "RAFT - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (33)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/cohere_command-a-03-2025.json b/data/models/cohere_command-a-03-2025.json deleted file mode 100644 index 205f7c496ed81b612dd2b182b3cf2e9fd9ac9c54..0000000000000000000000000000000000000000 --- a/data/models/cohere_command-a-03-2025.json +++ /dev/null @@ -1,1043 +0,0 @@ -{ - "model_info": { - "name": "command-a-03-2025", - "id": "cohere/command-a-03-2025", - "developer": "cohere", - "inference_platform": "unknown", - "additional_details": { - "display_name": "Command A " - } - }, - "evaluations": [ - { - "evaluation_id": "global-mmlu-lite/cohere_command-a-03-2025/1773936583.743359", - "retrieved_timestamp": "1773936583.743359", - "source_metadata": { - "source_name": "Global MMLU Lite Leaderboard", - "source_type": "documentation", - "source_organization_name": "kaggle", - "source_organization_url": "www.kaggle.com", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "kaggle kernel", - "version": "4", - "additional_details": { - "url": "https://www.kaggle.com/code/shivalikasingh95/global-mmlu-lite-sample-notebook" - } - }, - "benchmark": "global-mmlu-lite", - "evaluation_results": [ - { - "evaluation_name": "Global MMLU Lite", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Global MMLU Lite", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8385 - } - }, - { - "evaluation_name": "Culturally Sensitive", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Sensitive", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7993 - } - }, - { - "evaluation_name": "Culturally Agnostic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Agnostic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8778 - } - }, - { - "evaluation_name": "Arabic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Arabic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8425, - "uncertainty": { - "confidence_interval": { - "lower": -0.0357, - "upper": 0.0357, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "English", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - English", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.855, - "uncertainty": { - "confidence_interval": { - "lower": -0.0345, - "upper": 0.0345, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Bengali", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Bengali", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8225, - "uncertainty": { - "confidence_interval": { - "lower": -0.0374, - "upper": 0.0374, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "German", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - German", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8425, - "uncertainty": { - "confidence_interval": { - "lower": -0.0357, - "upper": 0.0357, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "French", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - French", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8375, - "uncertainty": { - "confidence_interval": { - "lower": -0.0362, - "upper": 0.0362, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Hindi", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Hindi", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8421, - "uncertainty": { - "confidence_interval": { - "lower": -0.0358, - "upper": 0.0358, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Indonesian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Indonesian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8546, - "uncertainty": { - "confidence_interval": { - "lower": -0.0346, - "upper": 0.0346, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Italian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Italian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8375, - "uncertainty": { - "confidence_interval": { - "lower": -0.0362, - "upper": 0.0362, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Japanese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Japanese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.845, - "uncertainty": { - "confidence_interval": { - "lower": -0.0355, - "upper": 0.0355, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Korean", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Korean", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.85, - "uncertainty": { - "confidence_interval": { - "lower": -0.035, - "upper": 0.035, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Portuguese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Portuguese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.84, - "uncertainty": { - "confidence_interval": { - "lower": -0.0359, - "upper": 0.0359, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Spanish", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Spanish", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8525, - "uncertainty": { - "confidence_interval": { - "lower": -0.0348, - "upper": 0.0348, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Swahili", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Swahili", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8275, - "uncertainty": { - "confidence_interval": { - "lower": -0.037, - "upper": 0.037, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Yoruba", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Yoruba", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.815, - "uncertainty": { - "confidence_interval": { - "lower": -0.0381, - "upper": 0.0381, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Chinese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Chinese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.835, - "uncertainty": { - "confidence_interval": { - "lower": -0.0364, - "upper": 0.0364, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Burmese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Burmese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8175, - "uncertainty": { - "confidence_interval": { - "lower": -0.0379, - "upper": 0.0379, - "method": "unknown" - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "global-mmlu-lite/cohere_command-a-03-2025/1773936496.366405", - "retrieved_timestamp": "1773936496.366405", - "source_metadata": { - "source_name": "Global MMLU Lite Leaderboard", - "source_type": "documentation", - "source_organization_name": "kaggle", - "source_organization_url": "www.kaggle.com", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "kaggle kernel", - "version": "4", - "additional_details": { - "url": "https://www.kaggle.com/code/shivalikasingh95/global-mmlu-lite-sample-notebook" - } - }, - "benchmark": "global-mmlu-lite", - "evaluation_results": [ - { - "evaluation_name": "Global MMLU Lite", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Global MMLU Lite", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8385 - } - }, - { - "evaluation_name": "Culturally Sensitive", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Sensitive", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7993 - } - }, - { - "evaluation_name": "Culturally Agnostic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Agnostic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8778 - } - }, - { - "evaluation_name": "Arabic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Arabic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8425, - "uncertainty": { - "confidence_interval": { - "lower": -0.0357, - "upper": 0.0357, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "English", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - English", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.855, - "uncertainty": { - "confidence_interval": { - "lower": -0.0345, - "upper": 0.0345, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Bengali", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Bengali", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8225, - "uncertainty": { - "confidence_interval": { - "lower": -0.0374, - "upper": 0.0374, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "German", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - German", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8425, - "uncertainty": { - "confidence_interval": { - "lower": -0.0357, - "upper": 0.0357, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "French", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - French", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8375, - "uncertainty": { - "confidence_interval": { - "lower": -0.0362, - "upper": 0.0362, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Hindi", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Hindi", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8421, - "uncertainty": { - "confidence_interval": { - "lower": -0.0358, - "upper": 0.0358, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Indonesian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Indonesian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8546, - "uncertainty": { - "confidence_interval": { - "lower": -0.0346, - "upper": 0.0346, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Italian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Italian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8375, - "uncertainty": { - "confidence_interval": { - "lower": -0.0362, - "upper": 0.0362, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Japanese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Japanese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.845, - "uncertainty": { - "confidence_interval": { - "lower": -0.0355, - "upper": 0.0355, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Korean", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Korean", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.85, - "uncertainty": { - "confidence_interval": { - "lower": -0.035, - "upper": 0.035, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Portuguese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Portuguese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.84, - "uncertainty": { - "confidence_interval": { - "lower": -0.0359, - "upper": 0.0359, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Spanish", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Spanish", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8525, - "uncertainty": { - "confidence_interval": { - "lower": -0.0348, - "upper": 0.0348, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Swahili", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Swahili", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8275, - "uncertainty": { - "confidence_interval": { - "lower": -0.037, - "upper": 0.037, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Yoruba", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Yoruba", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.815, - "uncertainty": { - "confidence_interval": { - "lower": -0.0381, - "upper": 0.0381, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Chinese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Chinese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.835, - "uncertainty": { - "confidence_interval": { - "lower": -0.0364, - "upper": 0.0364, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Burmese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Burmese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8175, - "uncertainty": { - "confidence_interval": { - "lower": -0.0379, - "upper": 0.0379, - "method": "unknown" - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cohere_command-a-fc.json b/data/models/cohere_command-a-fc.json deleted file mode 100644 index 9a8305770bbad29d1abf7ee868b4ebdfe1b8bcb1..0000000000000000000000000000000000000000 --- a/data/models/cohere_command-a-fc.json +++ /dev/null @@ -1,851 +0,0 @@ -{ - "model_info": { - "name": "Command A (FC)", - "id": "cohere/command-a-fc", - "developer": "cohere", - "additional_details": { - "raw_model_name": "Command A (FC)", - "organization": "Cohere", - "license": "CC-BY-NC 4.0 License (w/ Acceptable Use Addendum)", - "mode": "FC", - "model_link": "https://cohere.com/blog/command-a" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/cohere/command-a-fc/1775236112.383245", - "retrieved_timestamp": "1775236112.383245", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 35.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 46.49 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 91.37 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 2.09 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 7.36 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 4.94 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 87.56 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 75.75 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 93.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 93.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 88.0 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 78.53 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 85.66 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 76.92 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 81.25 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 70.83 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 29.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 38.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 23.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 32.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 25.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 46.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 60.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 33.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 16.56 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 4.52 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 5.16 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 40.0 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 81.25 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 84.19 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cohere_command-a-reasoning-fc.json b/data/models/cohere_command-a-reasoning-fc.json deleted file mode 100644 index 4f73c4a73b692a0852bb5d3b94e0540b4cf330ed..0000000000000000000000000000000000000000 --- a/data/models/cohere_command-a-reasoning-fc.json +++ /dev/null @@ -1,851 +0,0 @@ -{ - "model_info": { - "name": "Command A Reasoning (FC)", - "id": "cohere/command-a-reasoning-fc", - "developer": "cohere", - "additional_details": { - "raw_model_name": "Command A Reasoning (FC)", - "organization": "Cohere", - "license": "CC-BY-NC 4.0 License (w/ Acceptable Use Addendum)", - "mode": "FC", - "model_link": "https://cohere.com/blog/command-a-reasoning" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/cohere/command-a-reasoning-fc/1775236112.372142", - "retrieved_timestamp": "1775236112.372142", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 13.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 57.06 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 3.04 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 3.44 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 4.91 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 8.39 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 86.27 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 73.58 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 93.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 89.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 88.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 78.61 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 80.23 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 78.35 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 75.0 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 75.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 50.12 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 61.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 41.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 49.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 48.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 55.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 65.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 46.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 28.82 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 16.13 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 23.87 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 46.45 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 68.75 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 86.75 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cohere_command-light.json b/data/models/cohere_command-light.json deleted file mode 100644 index 324baba2e0c095af5af5a88836fd1baea30dc038..0000000000000000000000000000000000000000 --- a/data/models/cohere_command-light.json +++ /dev/null @@ -1,380 +0,0 @@ -{ - "model_info": { - "name": "Command Light", - "id": "cohere/command-light", - "developer": "cohere", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_lite/cohere_command-light/1774096306.427425", - "retrieved_timestamp": "1774096306.427425", - "source_metadata": { - "source_name": "helm_lite", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_lite", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_lite", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.105, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.46863920099875156\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.629, - "details": { - "description": "min=0.629, mean=0.629, max=0.629, sum=0.629 (1)", - "tab": "Accuracy", - "NarrativeQA - Observed inference time (s)": "{\"description\": \"min=0.896, mean=0.896, max=0.896, sum=0.896 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.8961316760157195\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=1.941, mean=1.941, max=1.941, sum=1.941 (1)\", \"tab\": \"General information\", \"score\": \"1.9408450704225353\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=1660.485, mean=1660.485, max=1660.485, sum=1660.485 (1)\", \"tab\": \"General information\", \"score\": \"1660.4845070422534\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=10.814, mean=10.814, max=10.814, sum=10.814 (1)\", \"tab\": \"General information\", \"score\": \"10.814084507042253\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (closed-book)", - "source_data": { - "dataset_name": "NaturalQuestions (closed-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (closed-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.195, - "details": { - "description": "min=0.195, mean=0.195, max=0.195, sum=0.195 (1)", - "tab": "Accuracy", - "NaturalQuestions (open-book) - Observed inference time (s)": "{\"description\": \"min=1.08, mean=1.08, max=1.08, sum=1.08 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.0799305574893951\"}", - "NaturalQuestions (closed-book) - Observed inference time (s)": "{\"description\": \"min=0.696, mean=0.696, max=0.696, sum=0.696 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.6957695767879486\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.617, mean=4.617, max=4.617, sum=4.617 (1)\", \"tab\": \"General information\", \"score\": \"4.617\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.039, mean=0.039, max=0.039, sum=0.039 (1)\", \"tab\": \"General information\", \"score\": \"0.039\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1557.639, mean=1557.639, max=1557.639, sum=1557.639 (1)\", \"tab\": \"General information\", \"score\": \"1557.639\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=10.869, mean=10.869, max=10.869, sum=10.869 (1)\", \"tab\": \"General information\", \"score\": \"10.869\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=115.191, mean=115.191, max=115.191, sum=115.191 (1)\", \"tab\": \"General information\", \"score\": \"115.191\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=17.348, mean=17.348, max=17.348, sum=17.348 (1)\", \"tab\": \"General information\", \"score\": \"17.348\"}" - } - }, - "generation_config": { - "additional_details": { - "mode": "\"closedbook\"" - } - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.398, - "details": { - "description": "min=0.398, mean=0.398, max=0.398, sum=0.398 (1)", - "tab": "Accuracy", - "OpenbookQA - Observed inference time (s)": "{\"description\": \"min=0.705, mean=0.705, max=0.705, sum=0.705 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.7049956932067871\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=246.682, mean=246.682, max=246.682, sum=246.682 (1)\", \"tab\": \"General information\", \"score\": \"246.682\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "dataset": "\"openbookqa\"", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.386, - "details": { - "description": "min=0.25, mean=0.386, max=0.57, sum=1.928 (5)", - "tab": "Accuracy", - "MMLU - Observed inference time (s)": "{\"description\": \"min=0.405, mean=0.749, max=1.412, sum=3.747 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.7494988910942747\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=372.75, mean=481.26, max=628.421, sum=2406.301 (5)\", \"tab\": \"General information\", \"score\": \"481.26021052631575\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"college_chemistry\", \"computer_security\", \"econometrics\", \"us_foreign_policy\"]", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MATH", - "source_data": { - "dataset_name": "MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Equivalent (CoT) on MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.098, - "details": { - "description": "min=0.026, mean=0.098, max=0.167, sum=0.687 (7)", - "tab": "Accuracy", - "MATH - Observed inference time (s)": "{\"description\": \"min=1.821, mean=2.374, max=2.948, sum=16.62 (7)\", \"tab\": \"Efficiency\", \"score\": \"2.374249639604042\"}", - "MATH - # eval": "{\"description\": \"min=30, mean=62.429, max=135, sum=437 (7)\", \"tab\": \"General information\", \"score\": \"62.42857142857143\"}", - "MATH - # train": "{\"description\": \"min=2.962, mean=6.878, max=8, sum=48.146 (7)\", \"tab\": \"General information\", \"score\": \"6.877964141122035\"}", - "MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MATH - # prompt tokens": "{\"description\": \"min=925.333, mean=1177.329, max=1534.058, sum=8241.302 (7)\", \"tab\": \"General information\", \"score\": \"1177.3289276411065\"}", - "MATH - # output tokens": "{\"description\": \"min=83.228, mean=106.589, max=137.692, sum=746.121 (7)\", \"tab\": \"General information\", \"score\": \"106.58875792143844\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"algebra\", \"counting_and_probability\", \"geometry\", \"intermediate_algebra\", \"number_theory\", \"prealgebra\", \"precalculus\"]", - "level": "\"1\"", - "use_official_examples": "\"False\"", - "use_chain_of_thought": "\"True\"" - } - } - }, - { - "evaluation_name": "GSM8K", - "source_data": { - "dataset_name": "GSM8K", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on GSM8K", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.149, - "details": { - "description": "min=0.149, mean=0.149, max=0.149, sum=0.149 (1)", - "tab": "Accuracy", - "GSM8K - Observed inference time (s)": "{\"description\": \"min=1.751, mean=1.751, max=1.751, sum=1.751 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.7514978868961335\"}", - "GSM8K - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "GSM8K - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "GSM8K - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GSM8K - # prompt tokens": "{\"description\": \"min=942.424, mean=942.424, max=942.424, sum=942.424 (1)\", \"tab\": \"General information\", \"score\": \"942.424\"}", - "GSM8K - # output tokens": "{\"description\": \"min=80.184, mean=80.184, max=80.184, sum=80.184 (1)\", \"tab\": \"General information\", \"score\": \"80.184\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "LegalBench", - "source_data": { - "dataset_name": "LegalBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on LegalBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.397, - "details": { - "description": "min=0.173, mean=0.397, max=0.874, sum=1.983 (5)", - "tab": "Accuracy", - "LegalBench - Observed inference time (s)": "{\"description\": \"min=0.423, mean=0.783, max=1.232, sum=3.916 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.7831334660572837\"}", - "LegalBench - # eval": "{\"description\": \"min=95, mean=409.4, max=1000, sum=2047 (5)\", \"tab\": \"General information\", \"score\": \"409.4\"}", - "LegalBench - # train": "{\"description\": \"min=0.388, mean=3.878, max=5, sum=19.388 (5)\", \"tab\": \"General information\", \"score\": \"3.8775510204081636\"}", - "LegalBench - truncated": "{\"description\": \"min=0, mean=0.003, max=0.014, sum=0.014 (5)\", \"tab\": \"General information\", \"score\": \"0.002857142857142857\"}", - "LegalBench - # prompt tokens": "{\"description\": \"min=205.295, mean=566.501, max=1529.327, sum=2832.507 (5)\", \"tab\": \"General information\", \"score\": \"566.5014751745068\"}", - "LegalBench - # output tokens": "{\"description\": \"min=1.074, mean=6.64, max=23.614, sum=33.198 (5)\", \"tab\": \"General information\", \"score\": \"6.63968330089529\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"abercrombie\", \"corporate_lobbying\", \"function_of_decision_section\", \"international_citizenship_questions\", \"proa\"]" - } - } - }, - { - "evaluation_name": "MedQA", - "source_data": { - "dataset_name": "MedQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MedQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.312, - "details": { - "description": "min=0.312, mean=0.312, max=0.312, sum=0.312 (1)", - "tab": "Accuracy", - "MedQA - Observed inference time (s)": "{\"description\": \"min=0.896, mean=0.896, max=0.896, sum=0.896 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.895831539901066\"}", - "MedQA - # eval": "{\"description\": \"min=503, mean=503, max=503, sum=503 (1)\", \"tab\": \"General information\", \"score\": \"503.0\"}", - "MedQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MedQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MedQA - # prompt tokens": "{\"description\": \"min=1016.738, mean=1016.738, max=1016.738, sum=1016.738 (1)\", \"tab\": \"General information\", \"score\": \"1016.7375745526839\"}", - "MedQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WMT 2014", - "source_data": { - "dataset_name": "WMT 2014", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "BLEU-4 on WMT 2014", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.023, - "details": { - "description": "min=0.0, mean=0.023, max=0.064, sum=0.113 (5)", - "tab": "Accuracy", - "WMT 2014 - Observed inference time (s)": "{\"description\": \"min=0.712, mean=0.797, max=0.934, sum=3.983 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.7965989762712353\"}", - "WMT 2014 - # eval": "{\"description\": \"min=503, mean=568.8, max=832, sum=2844 (5)\", \"tab\": \"General information\", \"score\": \"568.8\"}", - "WMT 2014 - # train": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "WMT 2014 - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WMT 2014 - # prompt tokens": "{\"description\": \"min=129.757, mean=149.459, max=178.821, sum=747.297 (5)\", \"tab\": \"General information\", \"score\": \"149.45941179844013\"}", - "WMT 2014 - # output tokens": "{\"description\": \"min=30.895, mean=39.885, max=47.65, sum=199.426 (5)\", \"tab\": \"General information\", \"score\": \"39.88511765942805\"}" - } - }, - "generation_config": { - "additional_details": { - "language_pair": "[\"cs-en\", \"de-en\", \"fr-en\", \"hi-en\", \"ru-en\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/cohere_command-r-plus.json b/data/models/cohere_command-r-plus.json deleted file mode 100644 index cd301ce593d714d625fc414e24d6004eb4a94443..0000000000000000000000000000000000000000 --- a/data/models/cohere_command-r-plus.json +++ /dev/null @@ -1,1903 +0,0 @@ -{ - "model_info": { - "name": "Command R Plus", - "id": "cohere/command-r-plus", - "developer": "cohere", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_lite/cohere_command-r-plus/1774096306.427425", - "retrieved_timestamp": "1774096306.427425", - "source_metadata": { - "source_name": "helm_lite", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_lite", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_lite", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.441, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.6927215980024969\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.735, - "details": { - "description": "min=0.735, mean=0.735, max=0.735, sum=0.735 (1)", - "tab": "Accuracy", - "NarrativeQA - Observed inference time (s)": "{\"description\": \"min=0.659, mean=0.659, max=0.659, sum=0.659 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.6590185803426823\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=3442.654, mean=3442.654, max=3442.654, sum=3442.654 (1)\", \"tab\": \"General information\", \"score\": \"3442.6535211267606\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (closed-book)", - "source_data": { - "dataset_name": "NaturalQuestions (closed-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (closed-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.343, - "details": { - "description": "min=0.343, mean=0.343, max=0.343, sum=0.343 (1)", - "tab": "Accuracy", - "NaturalQuestions (open-book) - Observed inference time (s)": "{\"description\": \"min=0.48, mean=0.48, max=0.48, sum=0.48 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.48011646389961243\"}", - "NaturalQuestions (closed-book) - Observed inference time (s)": "{\"description\": \"min=0.217, mean=0.217, max=0.217, sum=0.217 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.21743906450271605\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=2069.055, mean=2069.055, max=2069.055, sum=2069.055 (1)\", \"tab\": \"General information\", \"score\": \"2069.055\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=160.159, mean=160.159, max=160.159, sum=160.159 (1)\", \"tab\": \"General information\", \"score\": \"160.159\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "mode": "\"closedbook\"" - } - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.828, - "details": { - "description": "min=0.828, mean=0.828, max=0.828, sum=0.828 (1)", - "tab": "Accuracy", - "OpenbookQA - Observed inference time (s)": "{\"description\": \"min=0.526, mean=0.526, max=0.526, sum=0.526 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.5261325912475586\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=260.678, mean=260.678, max=260.678, sum=260.678 (1)\", \"tab\": \"General information\", \"score\": \"260.678\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "dataset": "\"openbookqa\"", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.59, - "details": { - "description": "min=0.21, mean=0.59, max=0.89, sum=2.951 (5)", - "tab": "Accuracy", - "MMLU - Observed inference time (s)": "{\"description\": \"min=0.26, mean=0.359, max=0.481, sum=1.797 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.3594088048349347\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=397.66, mean=499.49, max=661.579, sum=2497.449 (5)\", \"tab\": \"General information\", \"score\": \"499.48978947368425\"}", - "MMLU - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"college_chemistry\", \"computer_security\", \"econometrics\", \"us_foreign_policy\"]", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MATH", - "source_data": { - "dataset_name": "MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Equivalent (CoT) on MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.403, - "details": { - "description": "min=0.25, mean=0.403, max=0.607, sum=2.822 (7)", - "tab": "Accuracy", - "MATH - Observed inference time (s)": "{\"description\": \"min=1.358, mean=1.792, max=2.877, sum=12.543 (7)\", \"tab\": \"Efficiency\", \"score\": \"1.7917883168992628\"}", - "MATH - # eval": "{\"description\": \"min=30, mean=62.429, max=135, sum=437 (7)\", \"tab\": \"General information\", \"score\": \"62.42857142857143\"}", - "MATH - # train": "{\"description\": \"min=8, mean=8, max=8, sum=56 (7)\", \"tab\": \"General information\", \"score\": \"8.0\"}", - "MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MATH - # prompt tokens": "{\"description\": \"min=974.156, mean=1406.107, max=2423.596, sum=9842.752 (7)\", \"tab\": \"General information\", \"score\": \"1406.1074103714861\"}", - "MATH - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"algebra\", \"counting_and_probability\", \"geometry\", \"intermediate_algebra\", \"number_theory\", \"prealgebra\", \"precalculus\"]", - "level": "\"1\"", - "use_official_examples": "\"False\"", - "use_chain_of_thought": "\"True\"" - } - } - }, - { - "evaluation_name": "GSM8K", - "source_data": { - "dataset_name": "GSM8K", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on GSM8K", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.738, - "details": { - "description": "min=0.738, mean=0.738, max=0.738, sum=0.738 (1)", - "tab": "Accuracy", - "GSM8K - Observed inference time (s)": "{\"description\": \"min=3.592, mean=3.592, max=3.592, sum=3.592 (1)\", \"tab\": \"Efficiency\", \"score\": \"3.5923334171772003\"}", - "GSM8K - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "GSM8K - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "GSM8K - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GSM8K - # prompt tokens": "{\"description\": \"min=1158.893, mean=1158.893, max=1158.893, sum=1158.893 (1)\", \"tab\": \"General information\", \"score\": \"1158.893\"}", - "GSM8K - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "stop": "\"none\"" - } - } - }, - { - "evaluation_name": "LegalBench", - "source_data": { - "dataset_name": "LegalBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on LegalBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.672, - "details": { - "description": "min=0.428, mean=0.672, max=0.947, sum=3.358 (5)", - "tab": "Accuracy", - "LegalBench - Observed inference time (s)": "{\"description\": \"min=0.193, mean=0.351, max=0.927, sum=1.754 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.3508069759610481\"}", - "LegalBench - # eval": "{\"description\": \"min=95, mean=409.4, max=1000, sum=2047 (5)\", \"tab\": \"General information\", \"score\": \"409.4\"}", - "LegalBench - # train": "{\"description\": \"min=4, mean=4.8, max=5, sum=24 (5)\", \"tab\": \"General information\", \"score\": \"4.8\"}", - "LegalBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "LegalBench - # prompt tokens": "{\"description\": \"min=223.126, mean=1582.617, max=6507.029, sum=7913.085 (5)\", \"tab\": \"General information\", \"score\": \"1582.6169819753743\"}", - "LegalBench - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"abercrombie\", \"corporate_lobbying\", \"function_of_decision_section\", \"international_citizenship_questions\", \"proa\"]" - } - } - }, - { - "evaluation_name": "MedQA", - "source_data": { - "dataset_name": "MedQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MedQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.567, - "details": { - "description": "min=0.567, mean=0.567, max=0.567, sum=0.567 (1)", - "tab": "Accuracy", - "MedQA - Observed inference time (s)": "{\"description\": \"min=0.631, mean=0.631, max=0.631, sum=0.631 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.6308214294744533\"}", - "MedQA - # eval": "{\"description\": \"min=503, mean=503, max=503, sum=503 (1)\", \"tab\": \"General information\", \"score\": \"503.0\"}", - "MedQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MedQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MedQA - # prompt tokens": "{\"description\": \"min=1062.905, mean=1062.905, max=1062.905, sum=1062.905 (1)\", \"tab\": \"General information\", \"score\": \"1062.9045725646124\"}", - "MedQA - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WMT 2014", - "source_data": { - "dataset_name": "WMT 2014", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "BLEU-4 on WMT 2014", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.203, - "details": { - "description": "min=0.156, mean=0.203, max=0.233, sum=1.017 (5)", - "tab": "Accuracy", - "WMT 2014 - Observed inference time (s)": "{\"description\": \"min=0.59, mean=0.644, max=0.742, sum=3.221 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.6441886008863676\"}", - "WMT 2014 - # eval": "{\"description\": \"min=503, mean=568.8, max=832, sum=2844 (5)\", \"tab\": \"General information\", \"score\": \"568.8\"}", - "WMT 2014 - # train": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "WMT 2014 - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WMT 2014 - # prompt tokens": "{\"description\": \"min=114.404, mean=127.944, max=146.584, sum=639.721 (5)\", \"tab\": \"General information\", \"score\": \"127.94422599021257\"}", - "WMT 2014 - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "language_pair": "[\"cs-en\", \"de-en\", \"fr-en\", \"hi-en\", \"ru-en\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_mmlu/cohere_command-r-plus/1774096312.00548", - "retrieved_timestamp": "1774096312.00548", - "source_metadata": { - "source_name": "helm_mmlu", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_mmlu", - "evaluation_results": [ - { - "evaluation_name": "MMLU All Subjects", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU All Subjects", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.694, - "details": { - "description": "min=0.21, mean=0.694, max=0.927, sum=79.063 (114)", - "tab": "Accuracy", - "MMLU All Subjects - Observed inference time (s)": "{\"description\": \"min=0.199, mean=0.305, max=0.74, sum=34.817 (114)\", \"tab\": \"Efficiency\", \"score\": \"0.30541327600292584\"}", - "MMLU All Subjects - # eval": "{\"description\": \"min=100, mean=246.351, max=1534, sum=28084 (114)\", \"tab\": \"General information\", \"score\": \"246.35087719298247\"}", - "MMLU All Subjects - # train": "{\"description\": \"min=5, mean=5, max=5, sum=570 (114)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU All Subjects - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU All Subjects - # prompt tokens": "{\"description\": \"min=277.047, mean=648.571, max=2823.042, sum=73937.062 (114)\", \"tab\": \"General information\", \"score\": \"648.5707227335503\"}", - "MMLU All Subjects - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_name": "Abstract Algebra", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Abstract Algebra", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.21, - "details": { - "description": "min=0.21, mean=0.21, max=0.21, sum=0.42 (2)", - "tab": "Accuracy", - "Abstract Algebra - Observed inference time (s)": "{\"description\": \"min=0.26, mean=0.26, max=0.26, sum=0.521 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2603452730178833\"}", - "Abstract Algebra - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Abstract Algebra - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Abstract Algebra - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Abstract Algebra - # prompt tokens": "{\"description\": \"min=397.66, mean=397.66, max=397.66, sum=795.32 (2)\", \"tab\": \"General information\", \"score\": \"397.66\"}", - "Abstract Algebra - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"abstract_algebra\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_abstract_algebra\"" - } - } - }, - { - "evaluation_name": "Anatomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Anatomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.644, - "details": { - "description": "min=0.644, mean=0.644, max=0.644, sum=1.289 (2)", - "tab": "Accuracy", - "Anatomy - Observed inference time (s)": "{\"description\": \"min=0.29, mean=0.29, max=0.29, sum=0.58 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.289820040596856\"}", - "Anatomy - # eval": "{\"description\": \"min=135, mean=135, max=135, sum=270 (2)\", \"tab\": \"General information\", \"score\": \"135.0\"}", - "Anatomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Anatomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Anatomy - # prompt tokens": "{\"description\": \"min=360.096, mean=360.096, max=360.096, sum=720.193 (2)\", \"tab\": \"General information\", \"score\": \"360.0962962962963\"}", - "Anatomy - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"anatomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_anatomy\"" - } - } - }, - { - "evaluation_name": "College Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on College Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.52, - "details": { - "description": "min=0.52, mean=0.52, max=0.52, sum=1.039 (2)", - "tab": "Accuracy", - "College Chemistry - Observed inference time (s)": "{\"description\": \"min=0.419, mean=0.419, max=0.419, sum=0.839 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.41949598789215087\"}", - "College Biology - Observed inference time (s)": "{\"description\": \"min=0.319, mean=0.319, max=0.319, sum=0.638 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3188936991824044\"}", - "College Computer Science - Observed inference time (s)": "{\"description\": \"min=0.262, mean=0.262, max=0.262, sum=0.525 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.262396776676178\"}", - "College Mathematics - Observed inference time (s)": "{\"description\": \"min=0.46, mean=0.46, max=0.46, sum=0.92 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.45980838298797605\"}", - "College Medicine - Observed inference time (s)": "{\"description\": \"min=0.328, mean=0.328, max=0.328, sum=0.656 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.32775250611277673\"}", - "College Physics - Observed inference time (s)": "{\"description\": \"min=0.383, mean=0.383, max=0.383, sum=0.766 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.38314491861006794\"}", - "College Chemistry - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Chemistry - # prompt tokens": "{\"description\": \"min=586.57, mean=586.57, max=586.57, sum=1173.14 (2)\", \"tab\": \"General information\", \"score\": \"586.57\"}", - "College Chemistry - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # eval": "{\"description\": \"min=144, mean=144, max=144, sum=288 (2)\", \"tab\": \"General information\", \"score\": \"144.0\"}", - "College Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # prompt tokens": "{\"description\": \"min=496.632, mean=496.632, max=496.632, sum=993.264 (2)\", \"tab\": \"General information\", \"score\": \"496.63194444444446\"}", - "College Biology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # prompt tokens": "{\"description\": \"min=869.29, mean=869.29, max=869.29, sum=1738.58 (2)\", \"tab\": \"General information\", \"score\": \"869.29\"}", - "College Computer Science - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # prompt tokens": "{\"description\": \"min=645.25, mean=645.25, max=645.25, sum=1290.5 (2)\", \"tab\": \"General information\", \"score\": \"645.25\"}", - "College Mathematics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # eval": "{\"description\": \"min=173, mean=173, max=173, sum=346 (2)\", \"tab\": \"General information\", \"score\": \"173.0\"}", - "College Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # prompt tokens": "{\"description\": \"min=535.092, mean=535.092, max=535.092, sum=1070.185 (2)\", \"tab\": \"General information\", \"score\": \"535.0924855491329\"}", - "College Medicine - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # eval": "{\"description\": \"min=102, mean=102, max=102, sum=204 (2)\", \"tab\": \"General information\", \"score\": \"102.0\"}", - "College Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # prompt tokens": "{\"description\": \"min=530.382, mean=530.382, max=530.382, sum=1060.765 (2)\", \"tab\": \"General information\", \"score\": \"530.3823529411765\"}", - "College Physics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"college_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_college_physics\"" - } - } - }, - { - "evaluation_name": "Computer Security", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Computer Security", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.74, - "details": { - "description": "min=0.74, mean=0.74, max=0.74, sum=1.48 (2)", - "tab": "Accuracy", - "Computer Security - Observed inference time (s)": "{\"description\": \"min=0.481, mean=0.481, max=0.481, sum=0.961 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4807459425926208\"}", - "Computer Security - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Computer Security - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Computer Security - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Computer Security - # prompt tokens": "{\"description\": \"min=399.41, mean=399.41, max=399.41, sum=798.82 (2)\", \"tab\": \"General information\", \"score\": \"399.41\"}", - "Computer Security - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"computer_security\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_computer_security\"" - } - } - }, - { - "evaluation_name": "Econometrics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Econometrics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.561, - "details": { - "description": "min=0.561, mean=0.561, max=0.561, sum=1.123 (2)", - "tab": "Accuracy", - "Econometrics - Observed inference time (s)": "{\"description\": \"min=0.339, mean=0.339, max=0.339, sum=0.679 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.33940661162660835\"}", - "Econometrics - # eval": "{\"description\": \"min=114, mean=114, max=114, sum=228 (2)\", \"tab\": \"General information\", \"score\": \"114.0\"}", - "Econometrics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Econometrics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Econometrics - # prompt tokens": "{\"description\": \"min=661.579, mean=661.579, max=661.579, sum=1323.158 (2)\", \"tab\": \"General information\", \"score\": \"661.578947368421\"}", - "Econometrics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"econometrics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_econometrics\"" - } - } - }, - { - "evaluation_name": "Global Facts", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Global Facts", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5, - "details": { - "description": "min=0.5, mean=0.5, max=0.5, sum=1 (2)", - "tab": "Accuracy", - "Global Facts - Observed inference time (s)": "{\"description\": \"min=0.297, mean=0.297, max=0.297, sum=0.593 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2966678738594055\"}", - "Global Facts - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Global Facts - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Global Facts - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Global Facts - # prompt tokens": "{\"description\": \"min=469.58, mean=469.58, max=469.58, sum=939.16 (2)\", \"tab\": \"General information\", \"score\": \"469.58\"}", - "Global Facts - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"global_facts\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_global_facts\"" - } - } - }, - { - "evaluation_name": "Jurisprudence", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Jurisprudence", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.806, - "details": { - "description": "min=0.806, mean=0.806, max=0.806, sum=1.611 (2)", - "tab": "Accuracy", - "Jurisprudence - Observed inference time (s)": "{\"description\": \"min=0.288, mean=0.288, max=0.288, sum=0.577 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2883643927397551\"}", - "Jurisprudence - # eval": "{\"description\": \"min=108, mean=108, max=108, sum=216 (2)\", \"tab\": \"General information\", \"score\": \"108.0\"}", - "Jurisprudence - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Jurisprudence - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Jurisprudence - # prompt tokens": "{\"description\": \"min=417.944, mean=417.944, max=417.944, sum=835.889 (2)\", \"tab\": \"General information\", \"score\": \"417.94444444444446\"}", - "Jurisprudence - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"jurisprudence\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_jurisprudence\"" - } - } - }, - { - "evaluation_name": "Philosophy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Philosophy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.695, - "details": { - "description": "min=0.695, mean=0.695, max=0.695, sum=1.389 (2)", - "tab": "Accuracy", - "Philosophy - Observed inference time (s)": "{\"description\": \"min=0.308, mean=0.308, max=0.308, sum=0.616 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3079479507311364\"}", - "Philosophy - # eval": "{\"description\": \"min=311, mean=311, max=311, sum=622 (2)\", \"tab\": \"General information\", \"score\": \"311.0\"}", - "Philosophy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Philosophy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Philosophy - # prompt tokens": "{\"description\": \"min=355.508, mean=355.508, max=355.508, sum=711.016 (2)\", \"tab\": \"General information\", \"score\": \"355.508038585209\"}", - "Philosophy - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"philosophy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_philosophy\"" - } - } - }, - { - "evaluation_name": "Professional Psychology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Professional Psychology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.735, - "details": { - "description": "min=0.735, mean=0.735, max=0.735, sum=1.471 (2)", - "tab": "Accuracy", - "Professional Medicine - Observed inference time (s)": "{\"description\": \"min=0.451, mean=0.451, max=0.451, sum=0.903 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.45139760129592\"}", - "Professional Accounting - Observed inference time (s)": "{\"description\": \"min=0.292, mean=0.292, max=0.292, sum=0.584 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2920728659798913\"}", - "Professional Law - Observed inference time (s)": "{\"description\": \"min=0.406, mean=0.406, max=0.406, sum=0.811 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4056029599524228\"}", - "Professional Psychology - Observed inference time (s)": "{\"description\": \"min=0.305, mean=0.305, max=0.305, sum=0.609 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.30459034287072473\"}", - "Professional Medicine - # eval": "{\"description\": \"min=272, mean=272, max=272, sum=544 (2)\", \"tab\": \"General information\", \"score\": \"272.0\"}", - "Professional Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Medicine - # prompt tokens": "{\"description\": \"min=1143.129, mean=1143.129, max=1143.129, sum=2286.257 (2)\", \"tab\": \"General information\", \"score\": \"1143.1286764705883\"}", - "Professional Medicine - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # eval": "{\"description\": \"min=282, mean=282, max=282, sum=564 (2)\", \"tab\": \"General information\", \"score\": \"282.0\"}", - "Professional Accounting - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Accounting - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # prompt tokens": "{\"description\": \"min=767.429, mean=767.429, max=767.429, sum=1534.858 (2)\", \"tab\": \"General information\", \"score\": \"767.4290780141844\"}", - "Professional Accounting - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # eval": "{\"description\": \"min=1534, mean=1534, max=1534, sum=3068 (2)\", \"tab\": \"General information\", \"score\": \"1534.0\"}", - "Professional Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # prompt tokens": "{\"description\": \"min=1697.692, mean=1697.692, max=1697.692, sum=3395.385 (2)\", \"tab\": \"General information\", \"score\": \"1697.6923076923076\"}", - "Professional Law - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # eval": "{\"description\": \"min=612, mean=612, max=612, sum=1224 (2)\", \"tab\": \"General information\", \"score\": \"612.0\"}", - "Professional Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # prompt tokens": "{\"description\": \"min=609.167, mean=609.167, max=609.167, sum=1218.333 (2)\", \"tab\": \"General information\", \"score\": \"609.1666666666666\"}", - "Professional Psychology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"professional_psychology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_professional_psychology\"" - } - } - }, - { - "evaluation_name": "Us Foreign Policy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Us Foreign Policy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.89, - "details": { - "description": "min=0.89, mean=0.89, max=0.89, sum=1.78 (2)", - "tab": "Accuracy", - "Us Foreign Policy - Observed inference time (s)": "{\"description\": \"min=0.297, mean=0.297, max=0.297, sum=0.594 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.29705020904541013\"}", - "Us Foreign Policy - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Us Foreign Policy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Us Foreign Policy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Us Foreign Policy - # prompt tokens": "{\"description\": \"min=452.23, mean=452.23, max=452.23, sum=904.46 (2)\", \"tab\": \"General information\", \"score\": \"452.23\"}", - "Us Foreign Policy - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"us_foreign_policy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_us_foreign_policy\"" - } - } - }, - { - "evaluation_name": "Astronomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Astronomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.783, - "details": { - "description": "min=0.783, mean=0.783, max=0.783, sum=1.566 (2)", - "tab": "Accuracy", - "Astronomy - Observed inference time (s)": "{\"description\": \"min=0.492, mean=0.492, max=0.492, sum=0.984 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.49223921016642924\"}", - "Astronomy - # eval": "{\"description\": \"min=152, mean=152, max=152, sum=304 (2)\", \"tab\": \"General information\", \"score\": \"152.0\"}", - "Astronomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Astronomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Astronomy - # prompt tokens": "{\"description\": \"min=615.276, mean=615.276, max=615.276, sum=1230.553 (2)\", \"tab\": \"General information\", \"score\": \"615.2763157894736\"}", - "Astronomy - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"astronomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_astronomy\"" - } - } - }, - { - "evaluation_name": "Business Ethics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Business Ethics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.77, - "details": { - "description": "min=0.77, mean=0.77, max=0.77, sum=1.54 (2)", - "tab": "Accuracy", - "Business Ethics - Observed inference time (s)": "{\"description\": \"min=0.296, mean=0.296, max=0.296, sum=0.593 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2964653515815735\"}", - "Business Ethics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Business Ethics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Business Ethics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Business Ethics - # prompt tokens": "{\"description\": \"min=582.46, mean=582.46, max=582.46, sum=1164.92 (2)\", \"tab\": \"General information\", \"score\": \"582.46\"}", - "Business Ethics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"business_ethics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_business_ethics\"" - } - } - }, - { - "evaluation_name": "Clinical Knowledge", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Clinical Knowledge", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.743, - "details": { - "description": "min=0.743, mean=0.743, max=0.743, sum=1.487 (2)", - "tab": "Accuracy", - "Clinical Knowledge - Observed inference time (s)": "{\"description\": \"min=0.337, mean=0.337, max=0.337, sum=0.675 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.33743472009334924\"}", - "Clinical Knowledge - # eval": "{\"description\": \"min=265, mean=265, max=265, sum=530 (2)\", \"tab\": \"General information\", \"score\": \"265.0\"}", - "Clinical Knowledge - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Clinical Knowledge - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Clinical Knowledge - # prompt tokens": "{\"description\": \"min=433.181, mean=433.181, max=433.181, sum=866.362 (2)\", \"tab\": \"General information\", \"score\": \"433.1811320754717\"}", - "Clinical Knowledge - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"clinical_knowledge\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_clinical_knowledge\"" - } - } - }, - { - "evaluation_name": "Conceptual Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Conceptual Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.591, - "details": { - "description": "min=0.591, mean=0.591, max=0.591, sum=1.183 (2)", - "tab": "Accuracy", - "Conceptual Physics - Observed inference time (s)": "{\"description\": \"min=0.199, mean=0.199, max=0.199, sum=0.398 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.19917301928743403\"}", - "Conceptual Physics - # eval": "{\"description\": \"min=235, mean=235, max=235, sum=470 (2)\", \"tab\": \"General information\", \"score\": \"235.0\"}", - "Conceptual Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Conceptual Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Conceptual Physics - # prompt tokens": "{\"description\": \"min=322.511, mean=322.511, max=322.511, sum=645.021 (2)\", \"tab\": \"General information\", \"score\": \"322.51063829787233\"}", - "Conceptual Physics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"conceptual_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_conceptual_physics\"" - } - } - }, - { - "evaluation_name": "Electrical Engineering", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Electrical Engineering", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.71, - "details": { - "description": "min=0.71, mean=0.71, max=0.71, sum=1.421 (2)", - "tab": "Accuracy", - "Electrical Engineering - Observed inference time (s)": "{\"description\": \"min=0.238, mean=0.238, max=0.238, sum=0.476 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2378004501605856\"}", - "Electrical Engineering - # eval": "{\"description\": \"min=145, mean=145, max=145, sum=290 (2)\", \"tab\": \"General information\", \"score\": \"145.0\"}", - "Electrical Engineering - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Electrical Engineering - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Electrical Engineering - # prompt tokens": "{\"description\": \"min=494.648, mean=494.648, max=494.648, sum=989.297 (2)\", \"tab\": \"General information\", \"score\": \"494.64827586206894\"}", - "Electrical Engineering - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"electrical_engineering\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_electrical_engineering\"" - } - } - }, - { - "evaluation_name": "Elementary Mathematics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Elementary Mathematics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.474, - "details": { - "description": "min=0.474, mean=0.474, max=0.474, sum=0.947 (2)", - "tab": "Accuracy", - "Elementary Mathematics - Observed inference time (s)": "{\"description\": \"min=0.256, mean=0.256, max=0.256, sum=0.512 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2562026693707421\"}", - "Elementary Mathematics - # eval": "{\"description\": \"min=378, mean=378, max=378, sum=756 (2)\", \"tab\": \"General information\", \"score\": \"378.0\"}", - "Elementary Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Elementary Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Elementary Mathematics - # prompt tokens": "{\"description\": \"min=609.537, mean=609.537, max=609.537, sum=1219.074 (2)\", \"tab\": \"General information\", \"score\": \"609.5370370370371\"}", - "Elementary Mathematics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"elementary_mathematics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_elementary_mathematics\"" - } - } - }, - { - "evaluation_name": "Formal Logic", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Formal Logic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.484, - "details": { - "description": "min=0.484, mean=0.484, max=0.484, sum=0.968 (2)", - "tab": "Accuracy", - "Formal Logic - Observed inference time (s)": "{\"description\": \"min=0.285, mean=0.285, max=0.285, sum=0.57 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2847565715275114\"}", - "Formal Logic - # eval": "{\"description\": \"min=126, mean=126, max=126, sum=252 (2)\", \"tab\": \"General information\", \"score\": \"126.0\"}", - "Formal Logic - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Formal Logic - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Formal Logic - # prompt tokens": "{\"description\": \"min=630.992, mean=630.992, max=630.992, sum=1261.984 (2)\", \"tab\": \"General information\", \"score\": \"630.9920634920635\"}", - "Formal Logic - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"formal_logic\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_formal_logic\"" - } - } - }, - { - "evaluation_name": "High School World History", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on High School World History", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.827, - "details": { - "description": "min=0.827, mean=0.827, max=0.827, sum=1.654 (2)", - "tab": "Accuracy", - "High School Biology - Observed inference time (s)": "{\"description\": \"min=0.295, mean=0.295, max=0.295, sum=0.59 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.29477174051346317\"}", - "High School Chemistry - Observed inference time (s)": "{\"description\": \"min=0.225, mean=0.225, max=0.225, sum=0.451 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.22539391071338372\"}", - "High School Computer Science - Observed inference time (s)": "{\"description\": \"min=0.27, mean=0.27, max=0.27, sum=0.539 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.26950850486755373\"}", - "High School European History - Observed inference time (s)": "{\"description\": \"min=0.74, mean=0.74, max=0.74, sum=1.48 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7398316253315319\"}", - "High School Geography - Observed inference time (s)": "{\"description\": \"min=0.205, mean=0.205, max=0.205, sum=0.41 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.20521813570851027\"}", - "High School Government And Politics - Observed inference time (s)": "{\"description\": \"min=0.243, mean=0.243, max=0.243, sum=0.487 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.24341652430400948\"}", - "High School Macroeconomics - Observed inference time (s)": "{\"description\": \"min=0.221, mean=0.221, max=0.221, sum=0.442 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2207918637838119\"}", - "High School Mathematics - Observed inference time (s)": "{\"description\": \"min=0.296, mean=0.296, max=0.296, sum=0.592 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.29578982988993324\"}", - "High School Microeconomics - Observed inference time (s)": "{\"description\": \"min=0.343, mean=0.343, max=0.343, sum=0.686 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.342765681883868\"}", - "High School Physics - Observed inference time (s)": "{\"description\": \"min=0.279, mean=0.279, max=0.279, sum=0.558 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2788162073552214\"}", - "High School Psychology - Observed inference time (s)": "{\"description\": \"min=0.249, mean=0.249, max=0.249, sum=0.499 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2494196336203759\"}", - "High School Statistics - Observed inference time (s)": "{\"description\": \"min=0.286, mean=0.286, max=0.286, sum=0.572 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.28620046377182007\"}", - "High School US History - Observed inference time (s)": "{\"description\": \"min=0.467, mean=0.467, max=0.467, sum=0.934 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4672480844983868\"}", - "High School World History - Observed inference time (s)": "{\"description\": \"min=0.374, mean=0.374, max=0.374, sum=0.748 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3738658830586365\"}", - "High School Biology - # eval": "{\"description\": \"min=310, mean=310, max=310, sum=620 (2)\", \"tab\": \"General information\", \"score\": \"310.0\"}", - "High School Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Biology - # prompt tokens": "{\"description\": \"min=527.213, mean=527.213, max=527.213, sum=1054.426 (2)\", \"tab\": \"General information\", \"score\": \"527.2129032258065\"}", - "High School Biology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # eval": "{\"description\": \"min=203, mean=203, max=203, sum=406 (2)\", \"tab\": \"General information\", \"score\": \"203.0\"}", - "High School Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # prompt tokens": "{\"description\": \"min=530.635, mean=530.635, max=530.635, sum=1061.271 (2)\", \"tab\": \"General information\", \"score\": \"530.6354679802955\"}", - "High School Chemistry - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "High School Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # prompt tokens": "{\"description\": \"min=932.02, mean=932.02, max=932.02, sum=1864.04 (2)\", \"tab\": \"General information\", \"score\": \"932.02\"}", - "High School Computer Science - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # eval": "{\"description\": \"min=165, mean=165, max=165, sum=330 (2)\", \"tab\": \"General information\", \"score\": \"165.0\"}", - "High School European History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School European History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # prompt tokens": "{\"description\": \"min=2823.042, mean=2823.042, max=2823.042, sum=5646.085 (2)\", \"tab\": \"General information\", \"score\": \"2823.042424242424\"}", - "High School European History - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # eval": "{\"description\": \"min=198, mean=198, max=198, sum=396 (2)\", \"tab\": \"General information\", \"score\": \"198.0\"}", - "High School Geography - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Geography - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # prompt tokens": "{\"description\": \"min=407.818, mean=407.818, max=407.818, sum=815.636 (2)\", \"tab\": \"General information\", \"score\": \"407.8181818181818\"}", - "High School Geography - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # eval": "{\"description\": \"min=193, mean=193, max=193, sum=386 (2)\", \"tab\": \"General information\", \"score\": \"193.0\"}", - "High School Government And Politics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Government And Politics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # prompt tokens": "{\"description\": \"min=489.155, mean=489.155, max=489.155, sum=978.311 (2)\", \"tab\": \"General information\", \"score\": \"489.1554404145078\"}", - "High School Government And Politics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # eval": "{\"description\": \"min=390, mean=390, max=390, sum=780 (2)\", \"tab\": \"General information\", \"score\": \"390.0\"}", - "High School Macroeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Macroeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # prompt tokens": "{\"description\": \"min=407.654, mean=407.654, max=407.654, sum=815.308 (2)\", \"tab\": \"General information\", \"score\": \"407.65384615384613\"}", - "High School Macroeconomics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # eval": "{\"description\": \"min=270, mean=270, max=270, sum=540 (2)\", \"tab\": \"General information\", \"score\": \"270.0\"}", - "High School Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # prompt tokens": "{\"description\": \"min=589.774, mean=589.774, max=589.774, sum=1179.548 (2)\", \"tab\": \"General information\", \"score\": \"589.7740740740741\"}", - "High School Mathematics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # eval": "{\"description\": \"min=238, mean=238, max=238, sum=476 (2)\", \"tab\": \"General information\", \"score\": \"238.0\"}", - "High School Microeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Microeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # prompt tokens": "{\"description\": \"min=428.403, mean=428.403, max=428.403, sum=856.807 (2)\", \"tab\": \"General information\", \"score\": \"428.4033613445378\"}", - "High School Microeconomics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # eval": "{\"description\": \"min=151, mean=151, max=151, sum=302 (2)\", \"tab\": \"General information\", \"score\": \"151.0\"}", - "High School Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # prompt tokens": "{\"description\": \"min=604.272, mean=604.272, max=604.272, sum=1208.543 (2)\", \"tab\": \"General information\", \"score\": \"604.2715231788079\"}", - "High School Physics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # eval": "{\"description\": \"min=545, mean=545, max=545, sum=1090 (2)\", \"tab\": \"General information\", \"score\": \"545.0\"}", - "High School Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # prompt tokens": "{\"description\": \"min=516.004, mean=516.004, max=516.004, sum=1032.007 (2)\", \"tab\": \"General information\", \"score\": \"516.0036697247706\"}", - "High School Psychology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # eval": "{\"description\": \"min=216, mean=216, max=216, sum=432 (2)\", \"tab\": \"General information\", \"score\": \"216.0\"}", - "High School Statistics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Statistics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # prompt tokens": "{\"description\": \"min=871.264, mean=871.264, max=871.264, sum=1742.528 (2)\", \"tab\": \"General information\", \"score\": \"871.2638888888889\"}", - "High School Statistics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # eval": "{\"description\": \"min=204, mean=204, max=204, sum=408 (2)\", \"tab\": \"General information\", \"score\": \"204.0\"}", - "High School US History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School US History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # prompt tokens": "{\"description\": \"min=2240.358, mean=2240.358, max=2240.358, sum=4480.716 (2)\", \"tab\": \"General information\", \"score\": \"2240.357843137255\"}", - "High School US History - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # eval": "{\"description\": \"min=237, mean=237, max=237, sum=474 (2)\", \"tab\": \"General information\", \"score\": \"237.0\"}", - "High School World History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School World History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # prompt tokens": "{\"description\": \"min=1443.321, mean=1443.321, max=1443.321, sum=2886.641 (2)\", \"tab\": \"General information\", \"score\": \"1443.3206751054852\"}", - "High School World History - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"high_school_world_history\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_high_school_world_history\"" - } - } - }, - { - "evaluation_name": "Human Sexuality", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Human Sexuality", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.786, - "details": { - "description": "min=0.786, mean=0.786, max=0.786, sum=1.573 (2)", - "tab": "Accuracy", - "Human Aging - Observed inference time (s)": "{\"description\": \"min=0.199, mean=0.199, max=0.199, sum=0.399 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.19925055482462384\"}", - "Human Sexuality - Observed inference time (s)": "{\"description\": \"min=0.227, mean=0.227, max=0.227, sum=0.454 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.22696546925843217\"}", - "Human Aging - # eval": "{\"description\": \"min=223, mean=223, max=223, sum=446 (2)\", \"tab\": \"General information\", \"score\": \"223.0\"}", - "Human Aging - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Aging - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Aging - # prompt tokens": "{\"description\": \"min=336.09, mean=336.09, max=336.09, sum=672.179 (2)\", \"tab\": \"General information\", \"score\": \"336.0896860986547\"}", - "Human Aging - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # eval": "{\"description\": \"min=131, mean=131, max=131, sum=262 (2)\", \"tab\": \"General information\", \"score\": \"131.0\"}", - "Human Sexuality - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Sexuality - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # prompt tokens": "{\"description\": \"min=367.16, mean=367.16, max=367.16, sum=734.321 (2)\", \"tab\": \"General information\", \"score\": \"367.1603053435114\"}", - "Human Sexuality - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"human_sexuality\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_human_sexuality\"" - } - } - }, - { - "evaluation_name": "International Law", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on International Law", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.835, - "details": { - "description": "min=0.835, mean=0.835, max=0.835, sum=1.669 (2)", - "tab": "Accuracy", - "International Law - Observed inference time (s)": "{\"description\": \"min=0.247, mean=0.247, max=0.247, sum=0.494 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2467749296141065\"}", - "International Law - # eval": "{\"description\": \"min=121, mean=121, max=121, sum=242 (2)\", \"tab\": \"General information\", \"score\": \"121.0\"}", - "International Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "International Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "International Law - # prompt tokens": "{\"description\": \"min=653.612, mean=653.612, max=653.612, sum=1307.223 (2)\", \"tab\": \"General information\", \"score\": \"653.6115702479339\"}", - "International Law - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"international_law\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_international_law\"" - } - } - }, - { - "evaluation_name": "Logical Fallacies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Logical Fallacies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.791, - "details": { - "description": "min=0.791, mean=0.791, max=0.791, sum=1.583 (2)", - "tab": "Accuracy", - "Logical Fallacies - Observed inference time (s)": "{\"description\": \"min=0.25, mean=0.25, max=0.25, sum=0.5 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.24988567463459413\"}", - "Logical Fallacies - # eval": "{\"description\": \"min=163, mean=163, max=163, sum=326 (2)\", \"tab\": \"General information\", \"score\": \"163.0\"}", - "Logical Fallacies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Logical Fallacies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Logical Fallacies - # prompt tokens": "{\"description\": \"min=463.773, mean=463.773, max=463.773, sum=927.546 (2)\", \"tab\": \"General information\", \"score\": \"463.7730061349693\"}", - "Logical Fallacies - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"logical_fallacies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_logical_fallacies\"" - } - } - }, - { - "evaluation_name": "Machine Learning", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Machine Learning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.518, - "details": { - "description": "min=0.518, mean=0.518, max=0.518, sum=1.036 (2)", - "tab": "Accuracy", - "Machine Learning - Observed inference time (s)": "{\"description\": \"min=0.265, mean=0.265, max=0.265, sum=0.529 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2645062953233719\"}", - "Machine Learning - # eval": "{\"description\": \"min=112, mean=112, max=112, sum=224 (2)\", \"tab\": \"General information\", \"score\": \"112.0\"}", - "Machine Learning - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Machine Learning - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Machine Learning - # prompt tokens": "{\"description\": \"min=716.438, mean=716.438, max=716.438, sum=1432.875 (2)\", \"tab\": \"General information\", \"score\": \"716.4375\"}", - "Machine Learning - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"machine_learning\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_machine_learning\"" - } - } - }, - { - "evaluation_name": "Management", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Management", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.835, - "details": { - "description": "min=0.835, mean=0.835, max=0.835, sum=1.67 (2)", - "tab": "Accuracy", - "Management - Observed inference time (s)": "{\"description\": \"min=0.204, mean=0.204, max=0.204, sum=0.409 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.20434052735856437\"}", - "Management - # eval": "{\"description\": \"min=103, mean=103, max=103, sum=206 (2)\", \"tab\": \"General information\", \"score\": \"103.0\"}", - "Management - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Management - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Management - # prompt tokens": "{\"description\": \"min=294.456, mean=294.456, max=294.456, sum=588.913 (2)\", \"tab\": \"General information\", \"score\": \"294.45631067961165\"}", - "Management - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"management\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_management\"" - } - } - }, - { - "evaluation_name": "Marketing", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Marketing", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.927, - "details": { - "description": "min=0.927, mean=0.927, max=0.927, sum=1.855 (2)", - "tab": "Accuracy", - "Marketing - Observed inference time (s)": "{\"description\": \"min=0.228, mean=0.228, max=0.228, sum=0.456 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.22806417840158838\"}", - "Marketing - # eval": "{\"description\": \"min=234, mean=234, max=234, sum=468 (2)\", \"tab\": \"General information\", \"score\": \"234.0\"}", - "Marketing - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Marketing - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Marketing - # prompt tokens": "{\"description\": \"min=446.855, mean=446.855, max=446.855, sum=893.709 (2)\", \"tab\": \"General information\", \"score\": \"446.85470085470087\"}", - "Marketing - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"marketing\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_marketing\"" - } - } - }, - { - "evaluation_name": "Medical Genetics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Medical Genetics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.77, - "details": { - "description": "min=0.77, mean=0.77, max=0.77, sum=1.54 (2)", - "tab": "Accuracy", - "Medical Genetics - Observed inference time (s)": "{\"description\": \"min=0.307, mean=0.307, max=0.307, sum=0.614 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3072425937652588\"}", - "Medical Genetics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Medical Genetics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Medical Genetics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Medical Genetics - # prompt tokens": "{\"description\": \"min=357.02, mean=357.02, max=357.02, sum=714.04 (2)\", \"tab\": \"General information\", \"score\": \"357.02\"}", - "Medical Genetics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"medical_genetics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_medical_genetics\"" - } - } - }, - { - "evaluation_name": "Miscellaneous", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Miscellaneous", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.844, - "details": { - "description": "min=0.844, mean=0.844, max=0.844, sum=1.688 (2)", - "tab": "Accuracy", - "Miscellaneous - Observed inference time (s)": "{\"description\": \"min=0.208, mean=0.208, max=0.208, sum=0.417 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.20840222990832566\"}", - "Miscellaneous - # eval": "{\"description\": \"min=783, mean=783, max=783, sum=1566 (2)\", \"tab\": \"General information\", \"score\": \"783.0\"}", - "Miscellaneous - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Miscellaneous - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Miscellaneous - # prompt tokens": "{\"description\": \"min=325.76, mean=325.76, max=325.76, sum=651.52 (2)\", \"tab\": \"General information\", \"score\": \"325.75989782886336\"}", - "Miscellaneous - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"miscellaneous\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_miscellaneous\"" - } - } - }, - { - "evaluation_name": "Moral Scenarios", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Moral Scenarios", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.585, - "details": { - "description": "min=0.585, mean=0.585, max=0.585, sum=1.171 (2)", - "tab": "Accuracy", - "Moral Disputes - Observed inference time (s)": "{\"description\": \"min=0.229, mean=0.229, max=0.229, sum=0.457 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2285733340103502\"}", - "Moral Scenarios - Observed inference time (s)": "{\"description\": \"min=0.282, mean=0.282, max=0.282, sum=0.564 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2819661257653263\"}", - "Moral Disputes - # eval": "{\"description\": \"min=346, mean=346, max=346, sum=692 (2)\", \"tab\": \"General information\", \"score\": \"346.0\"}", - "Moral Disputes - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Disputes - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Disputes - # prompt tokens": "{\"description\": \"min=506.78, mean=506.78, max=506.78, sum=1013.561 (2)\", \"tab\": \"General information\", \"score\": \"506.78034682080926\"}", - "Moral Disputes - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # eval": "{\"description\": \"min=895, mean=895, max=895, sum=1790 (2)\", \"tab\": \"General information\", \"score\": \"895.0\"}", - "Moral Scenarios - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Scenarios - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # prompt tokens": "{\"description\": \"min=699.344, mean=699.344, max=699.344, sum=1398.688 (2)\", \"tab\": \"General information\", \"score\": \"699.3441340782123\"}", - "Moral Scenarios - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"moral_scenarios\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_moral_scenarios\"" - } - } - }, - { - "evaluation_name": "Nutrition", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Nutrition", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.742, - "details": { - "description": "min=0.742, mean=0.742, max=0.742, sum=1.484 (2)", - "tab": "Accuracy", - "Nutrition - Observed inference time (s)": "{\"description\": \"min=0.282, mean=0.282, max=0.282, sum=0.563 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2817091388640061\"}", - "Nutrition - # eval": "{\"description\": \"min=306, mean=306, max=306, sum=612 (2)\", \"tab\": \"General information\", \"score\": \"306.0\"}", - "Nutrition - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Nutrition - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Nutrition - # prompt tokens": "{\"description\": \"min=618.402, mean=618.402, max=618.402, sum=1236.804 (2)\", \"tab\": \"General information\", \"score\": \"618.4019607843137\"}", - "Nutrition - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"nutrition\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_nutrition\"" - } - } - }, - { - "evaluation_name": "Prehistory", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Prehistory", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.821, - "details": { - "description": "min=0.821, mean=0.821, max=0.821, sum=1.642 (2)", - "tab": "Accuracy", - "Prehistory - Observed inference time (s)": "{\"description\": \"min=0.287, mean=0.287, max=0.287, sum=0.574 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2871434423658583\"}", - "Prehistory - # eval": "{\"description\": \"min=324, mean=324, max=324, sum=648 (2)\", \"tab\": \"General information\", \"score\": \"324.0\"}", - "Prehistory - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Prehistory - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Prehistory - # prompt tokens": "{\"description\": \"min=549.235, mean=549.235, max=549.235, sum=1098.469 (2)\", \"tab\": \"General information\", \"score\": \"549.2345679012345\"}", - "Prehistory - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"prehistory\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_prehistory\"" - } - } - }, - { - "evaluation_name": "Public Relations", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Public Relations", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.709, - "details": { - "description": "min=0.709, mean=0.709, max=0.709, sum=1.418 (2)", - "tab": "Accuracy", - "Public Relations - Observed inference time (s)": "{\"description\": \"min=0.278, mean=0.278, max=0.278, sum=0.557 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.27829633842815055\"}", - "Public Relations - # eval": "{\"description\": \"min=110, mean=110, max=110, sum=220 (2)\", \"tab\": \"General information\", \"score\": \"110.0\"}", - "Public Relations - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Public Relations - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Public Relations - # prompt tokens": "{\"description\": \"min=434.682, mean=434.682, max=434.682, sum=869.364 (2)\", \"tab\": \"General information\", \"score\": \"434.6818181818182\"}", - "Public Relations - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"public_relations\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_public_relations\"" - } - } - }, - { - "evaluation_name": "Security Studies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Security Studies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.751, - "details": { - "description": "min=0.751, mean=0.751, max=0.751, sum=1.502 (2)", - "tab": "Accuracy", - "Security Studies - Observed inference time (s)": "{\"description\": \"min=0.345, mean=0.345, max=0.345, sum=0.69 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3448335861673161\"}", - "Security Studies - # eval": "{\"description\": \"min=245, mean=245, max=245, sum=490 (2)\", \"tab\": \"General information\", \"score\": \"245.0\"}", - "Security Studies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Security Studies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Security Studies - # prompt tokens": "{\"description\": \"min=1207.494, mean=1207.494, max=1207.494, sum=2414.988 (2)\", \"tab\": \"General information\", \"score\": \"1207.4938775510204\"}", - "Security Studies - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"security_studies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_security_studies\"" - } - } - }, - { - "evaluation_name": "Sociology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Sociology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.876, - "details": { - "description": "min=0.876, mean=0.876, max=0.876, sum=1.751 (2)", - "tab": "Accuracy", - "Sociology - Observed inference time (s)": "{\"description\": \"min=0.296, mean=0.296, max=0.296, sum=0.591 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2956119153037\"}", - "Sociology - # eval": "{\"description\": \"min=201, mean=201, max=201, sum=402 (2)\", \"tab\": \"General information\", \"score\": \"201.0\"}", - "Sociology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Sociology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Sociology - # prompt tokens": "{\"description\": \"min=467.343, mean=467.343, max=467.343, sum=934.687 (2)\", \"tab\": \"General information\", \"score\": \"467.34328358208955\"}", - "Sociology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"sociology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_sociology\"" - } - } - }, - { - "evaluation_name": "Virology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Virology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.56, - "details": { - "description": "min=0.56, mean=0.56, max=0.56, sum=1.12 (2)", - "tab": "Accuracy", - "Virology - Observed inference time (s)": "{\"description\": \"min=0.287, mean=0.287, max=0.287, sum=0.575 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2874818997210767\"}", - "Virology - # eval": "{\"description\": \"min=166, mean=166, max=166, sum=332 (2)\", \"tab\": \"General information\", \"score\": \"166.0\"}", - "Virology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Virology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Virology - # prompt tokens": "{\"description\": \"min=352.861, mean=352.861, max=352.861, sum=705.723 (2)\", \"tab\": \"General information\", \"score\": \"352.8614457831325\"}", - "Virology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"virology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_virology\"" - } - } - }, - { - "evaluation_name": "World Religions", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on World Religions", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.842, - "details": { - "description": "min=0.842, mean=0.842, max=0.842, sum=1.684 (2)", - "tab": "Accuracy", - "World Religions - Observed inference time (s)": "{\"description\": \"min=0.205, mean=0.205, max=0.205, sum=0.41 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.20489408119380126\"}", - "World Religions - # eval": "{\"description\": \"min=171, mean=171, max=171, sum=342 (2)\", \"tab\": \"General information\", \"score\": \"171.0\"}", - "World Religions - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "World Religions - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "World Religions - # prompt tokens": "{\"description\": \"min=277.047, mean=277.047, max=277.047, sum=554.094 (2)\", \"tab\": \"General information\", \"score\": \"277.046783625731\"}", - "World Religions - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"world_religions\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_world_religions\"" - } - } - }, - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.825, - "details": { - "description": "", - "tab": "Efficiency" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/cohere_command-r.json b/data/models/cohere_command-r.json deleted file mode 100644 index aecfbe30891fb8cca1f61bd46c9229f1dd8aa3d5..0000000000000000000000000000000000000000 --- a/data/models/cohere_command-r.json +++ /dev/null @@ -1,1903 +0,0 @@ -{ - "model_info": { - "name": "Command R", - "id": "cohere/command-r", - "developer": "cohere", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_lite/cohere_command-r/1774096306.427425", - "retrieved_timestamp": "1774096306.427425", - "source_metadata": { - "source_name": "helm_lite", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_lite", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_lite", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.299, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.9644069912609239\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.742, - "details": { - "description": "min=0.742, mean=0.742, max=0.742, sum=0.742 (1)", - "tab": "Accuracy", - "NarrativeQA - Observed inference time (s)": "{\"description\": \"min=0.389, mean=0.389, max=0.389, sum=0.389 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.3886059089445732\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=3442.654, mean=3442.654, max=3442.654, sum=3442.654 (1)\", \"tab\": \"General information\", \"score\": \"3442.6535211267606\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (closed-book)", - "source_data": { - "dataset_name": "NaturalQuestions (closed-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (closed-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.352, - "details": { - "description": "min=0.352, mean=0.352, max=0.352, sum=0.352 (1)", - "tab": "Accuracy", - "NaturalQuestions (open-book) - Observed inference time (s)": "{\"description\": \"min=0.288, mean=0.288, max=0.288, sum=0.288 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.2875482747554779\"}", - "NaturalQuestions (closed-book) - Observed inference time (s)": "{\"description\": \"min=0.165, mean=0.165, max=0.165, sum=0.165 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.16523362946510314\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=2069.055, mean=2069.055, max=2069.055, sum=2069.055 (1)\", \"tab\": \"General information\", \"score\": \"2069.055\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=160.159, mean=160.159, max=160.159, sum=160.159 (1)\", \"tab\": \"General information\", \"score\": \"160.159\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "mode": "\"closedbook\"" - } - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.782, - "details": { - "description": "min=0.782, mean=0.782, max=0.782, sum=0.782 (1)", - "tab": "Accuracy", - "OpenbookQA - Observed inference time (s)": "{\"description\": \"min=0.15, mean=0.15, max=0.15, sum=0.15 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.14960159301757814\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=260.678, mean=260.678, max=260.678, sum=260.678 (1)\", \"tab\": \"General information\", \"score\": \"260.678\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "dataset": "\"openbookqa\"", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.567, - "details": { - "description": "min=0.33, mean=0.567, max=0.82, sum=2.836 (5)", - "tab": "Accuracy", - "MMLU - Observed inference time (s)": "{\"description\": \"min=0.162, mean=0.173, max=0.185, sum=0.867 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.17335561692923832\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=397.66, mean=499.49, max=661.579, sum=2497.449 (5)\", \"tab\": \"General information\", \"score\": \"499.48978947368425\"}", - "MMLU - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"college_chemistry\", \"computer_security\", \"econometrics\", \"us_foreign_policy\"]", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MATH", - "source_data": { - "dataset_name": "MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Equivalent (CoT) on MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.266, - "details": { - "description": "min=0.158, mean=0.266, max=0.333, sum=1.861 (7)", - "tab": "Accuracy", - "MATH - Observed inference time (s)": "{\"description\": \"min=0.659, mean=0.821, max=1.104, sum=5.745 (7)\", \"tab\": \"Efficiency\", \"score\": \"0.8207379439676702\"}", - "MATH - # eval": "{\"description\": \"min=30, mean=62.429, max=135, sum=437 (7)\", \"tab\": \"General information\", \"score\": \"62.42857142857143\"}", - "MATH - # train": "{\"description\": \"min=8, mean=8, max=8, sum=56 (7)\", \"tab\": \"General information\", \"score\": \"8.0\"}", - "MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MATH - # prompt tokens": "{\"description\": \"min=974.156, mean=1406.107, max=2423.596, sum=9842.752 (7)\", \"tab\": \"General information\", \"score\": \"1406.1074103714861\"}", - "MATH - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"algebra\", \"counting_and_probability\", \"geometry\", \"intermediate_algebra\", \"number_theory\", \"prealgebra\", \"precalculus\"]", - "level": "\"1\"", - "use_official_examples": "\"False\"", - "use_chain_of_thought": "\"True\"" - } - } - }, - { - "evaluation_name": "GSM8K", - "source_data": { - "dataset_name": "GSM8K", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on GSM8K", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.551, - "details": { - "description": "min=0.551, mean=0.551, max=0.551, sum=0.551 (1)", - "tab": "Accuracy", - "GSM8K - Observed inference time (s)": "{\"description\": \"min=1.04, mean=1.04, max=1.04, sum=1.04 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.0398468203544617\"}", - "GSM8K - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "GSM8K - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "GSM8K - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GSM8K - # prompt tokens": "{\"description\": \"min=1158.893, mean=1158.893, max=1158.893, sum=1158.893 (1)\", \"tab\": \"General information\", \"score\": \"1158.893\"}", - "GSM8K - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "stop": "\"none\"" - } - } - }, - { - "evaluation_name": "LegalBench", - "source_data": { - "dataset_name": "LegalBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on LegalBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.507, - "details": { - "description": "min=0.211, mean=0.507, max=0.905, sum=2.534 (5)", - "tab": "Accuracy", - "LegalBench - Observed inference time (s)": "{\"description\": \"min=0.151, mean=0.235, max=0.5, sum=1.174 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.23478191454837286\"}", - "LegalBench - # eval": "{\"description\": \"min=95, mean=409.4, max=1000, sum=2047 (5)\", \"tab\": \"General information\", \"score\": \"409.4\"}", - "LegalBench - # train": "{\"description\": \"min=4, mean=4.8, max=5, sum=24 (5)\", \"tab\": \"General information\", \"score\": \"4.8\"}", - "LegalBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "LegalBench - # prompt tokens": "{\"description\": \"min=223.126, mean=1582.617, max=6507.029, sum=7913.085 (5)\", \"tab\": \"General information\", \"score\": \"1582.6169819753743\"}", - "LegalBench - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"abercrombie\", \"corporate_lobbying\", \"function_of_decision_section\", \"international_citizenship_questions\", \"proa\"]" - } - } - }, - { - "evaluation_name": "MedQA", - "source_data": { - "dataset_name": "MedQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MedQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.555, - "details": { - "description": "min=0.555, mean=0.555, max=0.555, sum=0.555 (1)", - "tab": "Accuracy", - "MedQA - Observed inference time (s)": "{\"description\": \"min=0.191, mean=0.191, max=0.191, sum=0.191 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.19128861531585634\"}", - "MedQA - # eval": "{\"description\": \"min=503, mean=503, max=503, sum=503 (1)\", \"tab\": \"General information\", \"score\": \"503.0\"}", - "MedQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MedQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MedQA - # prompt tokens": "{\"description\": \"min=1062.905, mean=1062.905, max=1062.905, sum=1062.905 (1)\", \"tab\": \"General information\", \"score\": \"1062.9045725646124\"}", - "MedQA - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WMT 2014", - "source_data": { - "dataset_name": "WMT 2014", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "BLEU-4 on WMT 2014", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.149, - "details": { - "description": "min=0.107, mean=0.149, max=0.175, sum=0.746 (5)", - "tab": "Accuracy", - "WMT 2014 - Observed inference time (s)": "{\"description\": \"min=0.308, mean=0.343, max=0.455, sum=1.715 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.3429552388299011\"}", - "WMT 2014 - # eval": "{\"description\": \"min=503, mean=568.8, max=832, sum=2844 (5)\", \"tab\": \"General information\", \"score\": \"568.8\"}", - "WMT 2014 - # train": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "WMT 2014 - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WMT 2014 - # prompt tokens": "{\"description\": \"min=114.404, mean=127.944, max=146.584, sum=639.721 (5)\", \"tab\": \"General information\", \"score\": \"127.94422599021257\"}", - "WMT 2014 - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "language_pair": "[\"cs-en\", \"de-en\", \"fr-en\", \"hi-en\", \"ru-en\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_mmlu/cohere_command-r/1774096312.00548", - "retrieved_timestamp": "1774096312.00548", - "source_metadata": { - "source_name": "helm_mmlu", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_mmlu", - "evaluation_results": [ - { - "evaluation_name": "MMLU All Subjects", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU All Subjects", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.652, - "details": { - "description": "min=0.326, mean=0.652, max=0.891, sum=74.329 (114)", - "tab": "Accuracy", - "MMLU All Subjects - Observed inference time (s)": "{\"description\": \"min=0.145, mean=0.176, max=0.289, sum=20.061 (114)\", \"tab\": \"Efficiency\", \"score\": \"0.17597788408479575\"}", - "MMLU All Subjects - # eval": "{\"description\": \"min=100, mean=246.351, max=1534, sum=28084 (114)\", \"tab\": \"General information\", \"score\": \"246.35087719298247\"}", - "MMLU All Subjects - # train": "{\"description\": \"min=5, mean=5, max=5, sum=570 (114)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU All Subjects - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU All Subjects - # prompt tokens": "{\"description\": \"min=277.047, mean=648.571, max=2823.042, sum=73937.062 (114)\", \"tab\": \"General information\", \"score\": \"648.5707227335503\"}", - "MMLU All Subjects - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_name": "Abstract Algebra", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Abstract Algebra", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.33, - "details": { - "description": "min=0.33, mean=0.33, max=0.33, sum=0.66 (2)", - "tab": "Accuracy", - "Abstract Algebra - Observed inference time (s)": "{\"description\": \"min=0.162, mean=0.162, max=0.162, sum=0.324 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.1620460057258606\"}", - "Abstract Algebra - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Abstract Algebra - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Abstract Algebra - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Abstract Algebra - # prompt tokens": "{\"description\": \"min=397.66, mean=397.66, max=397.66, sum=795.32 (2)\", \"tab\": \"General information\", \"score\": \"397.66\"}", - "Abstract Algebra - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"abstract_algebra\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_abstract_algebra\"" - } - } - }, - { - "evaluation_name": "Anatomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Anatomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.615, - "details": { - "description": "min=0.615, mean=0.615, max=0.615, sum=1.23 (2)", - "tab": "Accuracy", - "Anatomy - Observed inference time (s)": "{\"description\": \"min=0.157, mean=0.157, max=0.157, sum=0.314 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.15700986297042283\"}", - "Anatomy - # eval": "{\"description\": \"min=135, mean=135, max=135, sum=270 (2)\", \"tab\": \"General information\", \"score\": \"135.0\"}", - "Anatomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Anatomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Anatomy - # prompt tokens": "{\"description\": \"min=360.096, mean=360.096, max=360.096, sum=720.193 (2)\", \"tab\": \"General information\", \"score\": \"360.0962962962963\"}", - "Anatomy - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"anatomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_anatomy\"" - } - } - }, - { - "evaluation_name": "College Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on College Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.382, - "details": { - "description": "min=0.382, mean=0.382, max=0.382, sum=0.765 (2)", - "tab": "Accuracy", - "College Chemistry - Observed inference time (s)": "{\"description\": \"min=0.185, mean=0.185, max=0.185, sum=0.37 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.18501442193984985\"}", - "College Biology - Observed inference time (s)": "{\"description\": \"min=0.163, mean=0.163, max=0.163, sum=0.325 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.1627496729294459\"}", - "College Computer Science - Observed inference time (s)": "{\"description\": \"min=0.182, mean=0.182, max=0.182, sum=0.363 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.18159597158432006\"}", - "College Mathematics - Observed inference time (s)": "{\"description\": \"min=0.173, mean=0.173, max=0.173, sum=0.346 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.17305777072906495\"}", - "College Medicine - Observed inference time (s)": "{\"description\": \"min=0.167, mean=0.167, max=0.167, sum=0.334 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.1671100668824477\"}", - "College Physics - Observed inference time (s)": "{\"description\": \"min=0.169, mean=0.169, max=0.169, sum=0.339 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.16945467041988\"}", - "College Chemistry - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Chemistry - # prompt tokens": "{\"description\": \"min=586.57, mean=586.57, max=586.57, sum=1173.14 (2)\", \"tab\": \"General information\", \"score\": \"586.57\"}", - "College Chemistry - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # eval": "{\"description\": \"min=144, mean=144, max=144, sum=288 (2)\", \"tab\": \"General information\", \"score\": \"144.0\"}", - "College Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # prompt tokens": "{\"description\": \"min=496.632, mean=496.632, max=496.632, sum=993.264 (2)\", \"tab\": \"General information\", \"score\": \"496.63194444444446\"}", - "College Biology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # prompt tokens": "{\"description\": \"min=869.29, mean=869.29, max=869.29, sum=1738.58 (2)\", \"tab\": \"General information\", \"score\": \"869.29\"}", - "College Computer Science - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # prompt tokens": "{\"description\": \"min=645.25, mean=645.25, max=645.25, sum=1290.5 (2)\", \"tab\": \"General information\", \"score\": \"645.25\"}", - "College Mathematics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # eval": "{\"description\": \"min=173, mean=173, max=173, sum=346 (2)\", \"tab\": \"General information\", \"score\": \"173.0\"}", - "College Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # prompt tokens": "{\"description\": \"min=535.092, mean=535.092, max=535.092, sum=1070.185 (2)\", \"tab\": \"General information\", \"score\": \"535.0924855491329\"}", - "College Medicine - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # eval": "{\"description\": \"min=102, mean=102, max=102, sum=204 (2)\", \"tab\": \"General information\", \"score\": \"102.0\"}", - "College Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # prompt tokens": "{\"description\": \"min=530.382, mean=530.382, max=530.382, sum=1060.765 (2)\", \"tab\": \"General information\", \"score\": \"530.3823529411765\"}", - "College Physics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"college_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_college_physics\"" - } - } - }, - { - "evaluation_name": "Computer Security", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Computer Security", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.78, - "details": { - "description": "min=0.78, mean=0.78, max=0.78, sum=1.56 (2)", - "tab": "Accuracy", - "Computer Security - Observed inference time (s)": "{\"description\": \"min=0.163, mean=0.163, max=0.163, sum=0.327 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.16325130462646484\"}", - "Computer Security - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Computer Security - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Computer Security - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Computer Security - # prompt tokens": "{\"description\": \"min=399.41, mean=399.41, max=399.41, sum=798.82 (2)\", \"tab\": \"General information\", \"score\": \"399.41\"}", - "Computer Security - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"computer_security\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_computer_security\"" - } - } - }, - { - "evaluation_name": "Econometrics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Econometrics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.456, - "details": { - "description": "min=0.456, mean=0.456, max=0.456, sum=0.912 (2)", - "tab": "Accuracy", - "Econometrics - Observed inference time (s)": "{\"description\": \"min=0.174, mean=0.174, max=0.174, sum=0.347 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.17368793905827037\"}", - "Econometrics - # eval": "{\"description\": \"min=114, mean=114, max=114, sum=228 (2)\", \"tab\": \"General information\", \"score\": \"114.0\"}", - "Econometrics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Econometrics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Econometrics - # prompt tokens": "{\"description\": \"min=661.579, mean=661.579, max=661.579, sum=1323.158 (2)\", \"tab\": \"General information\", \"score\": \"661.578947368421\"}", - "Econometrics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"econometrics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_econometrics\"" - } - } - }, - { - "evaluation_name": "Global Facts", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Global Facts", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.42, - "details": { - "description": "min=0.42, mean=0.42, max=0.42, sum=0.84 (2)", - "tab": "Accuracy", - "Global Facts - Observed inference time (s)": "{\"description\": \"min=0.166, mean=0.166, max=0.166, sum=0.332 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.16606518507003784\"}", - "Global Facts - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Global Facts - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Global Facts - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Global Facts - # prompt tokens": "{\"description\": \"min=469.58, mean=469.58, max=469.58, sum=939.16 (2)\", \"tab\": \"General information\", \"score\": \"469.58\"}", - "Global Facts - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"global_facts\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_global_facts\"" - } - } - }, - { - "evaluation_name": "Jurisprudence", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Jurisprudence", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.796, - "details": { - "description": "min=0.796, mean=0.796, max=0.796, sum=1.593 (2)", - "tab": "Accuracy", - "Jurisprudence - Observed inference time (s)": "{\"description\": \"min=0.16, mean=0.16, max=0.16, sum=0.319 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.15962726098519783\"}", - "Jurisprudence - # eval": "{\"description\": \"min=108, mean=108, max=108, sum=216 (2)\", \"tab\": \"General information\", \"score\": \"108.0\"}", - "Jurisprudence - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Jurisprudence - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Jurisprudence - # prompt tokens": "{\"description\": \"min=417.944, mean=417.944, max=417.944, sum=835.889 (2)\", \"tab\": \"General information\", \"score\": \"417.94444444444446\"}", - "Jurisprudence - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"jurisprudence\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_jurisprudence\"" - } - } - }, - { - "evaluation_name": "Philosophy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Philosophy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.685, - "details": { - "description": "min=0.685, mean=0.685, max=0.685, sum=1.37 (2)", - "tab": "Accuracy", - "Philosophy - Observed inference time (s)": "{\"description\": \"min=0.154, mean=0.154, max=0.154, sum=0.307 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.1535167272451223\"}", - "Philosophy - # eval": "{\"description\": \"min=311, mean=311, max=311, sum=622 (2)\", \"tab\": \"General information\", \"score\": \"311.0\"}", - "Philosophy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Philosophy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Philosophy - # prompt tokens": "{\"description\": \"min=355.508, mean=355.508, max=355.508, sum=711.016 (2)\", \"tab\": \"General information\", \"score\": \"355.508038585209\"}", - "Philosophy - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"philosophy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_philosophy\"" - } - } - }, - { - "evaluation_name": "Professional Psychology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Professional Psychology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.681, - "details": { - "description": "min=0.681, mean=0.681, max=0.681, sum=1.363 (2)", - "tab": "Accuracy", - "Professional Medicine - Observed inference time (s)": "{\"description\": \"min=0.195, mean=0.195, max=0.195, sum=0.389 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.19464709828881657\"}", - "Professional Accounting - Observed inference time (s)": "{\"description\": \"min=0.177, mean=0.177, max=0.177, sum=0.354 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.1770885929148248\"}", - "Professional Law - Observed inference time (s)": "{\"description\": \"min=0.234, mean=0.234, max=0.234, sum=0.469 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.23427105509473262\"}", - "Professional Psychology - Observed inference time (s)": "{\"description\": \"min=0.211, mean=0.211, max=0.211, sum=0.423 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2114220471943126\"}", - "Professional Medicine - # eval": "{\"description\": \"min=272, mean=272, max=272, sum=544 (2)\", \"tab\": \"General information\", \"score\": \"272.0\"}", - "Professional Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Medicine - # prompt tokens": "{\"description\": \"min=1143.129, mean=1143.129, max=1143.129, sum=2286.257 (2)\", \"tab\": \"General information\", \"score\": \"1143.1286764705883\"}", - "Professional Medicine - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # eval": "{\"description\": \"min=282, mean=282, max=282, sum=564 (2)\", \"tab\": \"General information\", \"score\": \"282.0\"}", - "Professional Accounting - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Accounting - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # prompt tokens": "{\"description\": \"min=767.429, mean=767.429, max=767.429, sum=1534.858 (2)\", \"tab\": \"General information\", \"score\": \"767.4290780141844\"}", - "Professional Accounting - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # eval": "{\"description\": \"min=1534, mean=1534, max=1534, sum=3068 (2)\", \"tab\": \"General information\", \"score\": \"1534.0\"}", - "Professional Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # prompt tokens": "{\"description\": \"min=1697.692, mean=1697.692, max=1697.692, sum=3395.385 (2)\", \"tab\": \"General information\", \"score\": \"1697.6923076923076\"}", - "Professional Law - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # eval": "{\"description\": \"min=612, mean=612, max=612, sum=1224 (2)\", \"tab\": \"General information\", \"score\": \"612.0\"}", - "Professional Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # prompt tokens": "{\"description\": \"min=609.167, mean=609.167, max=609.167, sum=1218.333 (2)\", \"tab\": \"General information\", \"score\": \"609.1666666666666\"}", - "Professional Psychology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"professional_psychology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_professional_psychology\"" - } - } - }, - { - "evaluation_name": "Us Foreign Policy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Us Foreign Policy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.82, - "details": { - "description": "min=0.82, mean=0.82, max=0.82, sum=1.64 (2)", - "tab": "Accuracy", - "Us Foreign Policy - Observed inference time (s)": "{\"description\": \"min=0.183, mean=0.183, max=0.183, sum=0.366 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.18277841329574585\"}", - "Us Foreign Policy - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Us Foreign Policy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Us Foreign Policy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Us Foreign Policy - # prompt tokens": "{\"description\": \"min=452.23, mean=452.23, max=452.23, sum=904.46 (2)\", \"tab\": \"General information\", \"score\": \"452.23\"}", - "Us Foreign Policy - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"us_foreign_policy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_us_foreign_policy\"" - } - } - }, - { - "evaluation_name": "Astronomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Astronomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.743, - "details": { - "description": "min=0.743, mean=0.743, max=0.743, sum=1.487 (2)", - "tab": "Accuracy", - "Astronomy - Observed inference time (s)": "{\"description\": \"min=0.223, mean=0.223, max=0.223, sum=0.446 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.22317567624543844\"}", - "Astronomy - # eval": "{\"description\": \"min=152, mean=152, max=152, sum=304 (2)\", \"tab\": \"General information\", \"score\": \"152.0\"}", - "Astronomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Astronomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Astronomy - # prompt tokens": "{\"description\": \"min=615.276, mean=615.276, max=615.276, sum=1230.553 (2)\", \"tab\": \"General information\", \"score\": \"615.2763157894736\"}", - "Astronomy - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"astronomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_astronomy\"" - } - } - }, - { - "evaluation_name": "Business Ethics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Business Ethics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.63, - "details": { - "description": "min=0.63, mean=0.63, max=0.63, sum=1.26 (2)", - "tab": "Accuracy", - "Business Ethics - Observed inference time (s)": "{\"description\": \"min=0.17, mean=0.17, max=0.17, sum=0.34 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.16991474628448486\"}", - "Business Ethics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Business Ethics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Business Ethics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Business Ethics - # prompt tokens": "{\"description\": \"min=582.46, mean=582.46, max=582.46, sum=1164.92 (2)\", \"tab\": \"General information\", \"score\": \"582.46\"}", - "Business Ethics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"business_ethics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_business_ethics\"" - } - } - }, - { - "evaluation_name": "Clinical Knowledge", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Clinical Knowledge", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.751, - "details": { - "description": "min=0.751, mean=0.751, max=0.751, sum=1.502 (2)", - "tab": "Accuracy", - "Clinical Knowledge - Observed inference time (s)": "{\"description\": \"min=0.186, mean=0.186, max=0.186, sum=0.371 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.1857448289979179\"}", - "Clinical Knowledge - # eval": "{\"description\": \"min=265, mean=265, max=265, sum=530 (2)\", \"tab\": \"General information\", \"score\": \"265.0\"}", - "Clinical Knowledge - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Clinical Knowledge - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Clinical Knowledge - # prompt tokens": "{\"description\": \"min=433.181, mean=433.181, max=433.181, sum=866.362 (2)\", \"tab\": \"General information\", \"score\": \"433.1811320754717\"}", - "Clinical Knowledge - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"clinical_knowledge\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_clinical_knowledge\"" - } - } - }, - { - "evaluation_name": "Conceptual Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Conceptual Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.528, - "details": { - "description": "min=0.528, mean=0.528, max=0.528, sum=1.055 (2)", - "tab": "Accuracy", - "Conceptual Physics - Observed inference time (s)": "{\"description\": \"min=0.146, mean=0.146, max=0.146, sum=0.293 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.14639884360293123\"}", - "Conceptual Physics - # eval": "{\"description\": \"min=235, mean=235, max=235, sum=470 (2)\", \"tab\": \"General information\", \"score\": \"235.0\"}", - "Conceptual Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Conceptual Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Conceptual Physics - # prompt tokens": "{\"description\": \"min=322.511, mean=322.511, max=322.511, sum=645.021 (2)\", \"tab\": \"General information\", \"score\": \"322.51063829787233\"}", - "Conceptual Physics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"conceptual_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_conceptual_physics\"" - } - } - }, - { - "evaluation_name": "Electrical Engineering", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Electrical Engineering", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.593, - "details": { - "description": "min=0.593, mean=0.593, max=0.593, sum=1.186 (2)", - "tab": "Accuracy", - "Electrical Engineering - Observed inference time (s)": "{\"description\": \"min=0.164, mean=0.164, max=0.164, sum=0.329 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.16444927248461494\"}", - "Electrical Engineering - # eval": "{\"description\": \"min=145, mean=145, max=145, sum=290 (2)\", \"tab\": \"General information\", \"score\": \"145.0\"}", - "Electrical Engineering - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Electrical Engineering - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Electrical Engineering - # prompt tokens": "{\"description\": \"min=494.648, mean=494.648, max=494.648, sum=989.297 (2)\", \"tab\": \"General information\", \"score\": \"494.64827586206894\"}", - "Electrical Engineering - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"electrical_engineering\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_electrical_engineering\"" - } - } - }, - { - "evaluation_name": "Elementary Mathematics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Elementary Mathematics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.437, - "details": { - "description": "min=0.437, mean=0.437, max=0.437, sum=0.873 (2)", - "tab": "Accuracy", - "Elementary Mathematics - Observed inference time (s)": "{\"description\": \"min=0.171, mean=0.171, max=0.171, sum=0.342 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.17102001079175838\"}", - "Elementary Mathematics - # eval": "{\"description\": \"min=378, mean=378, max=378, sum=756 (2)\", \"tab\": \"General information\", \"score\": \"378.0\"}", - "Elementary Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Elementary Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Elementary Mathematics - # prompt tokens": "{\"description\": \"min=609.537, mean=609.537, max=609.537, sum=1219.074 (2)\", \"tab\": \"General information\", \"score\": \"609.5370370370371\"}", - "Elementary Mathematics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"elementary_mathematics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_elementary_mathematics\"" - } - } - }, - { - "evaluation_name": "Formal Logic", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Formal Logic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.405, - "details": { - "description": "min=0.405, mean=0.405, max=0.405, sum=0.81 (2)", - "tab": "Accuracy", - "Formal Logic - Observed inference time (s)": "{\"description\": \"min=0.178, mean=0.178, max=0.178, sum=0.357 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.17840472289494105\"}", - "Formal Logic - # eval": "{\"description\": \"min=126, mean=126, max=126, sum=252 (2)\", \"tab\": \"General information\", \"score\": \"126.0\"}", - "Formal Logic - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Formal Logic - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Formal Logic - # prompt tokens": "{\"description\": \"min=630.992, mean=630.992, max=630.992, sum=1261.984 (2)\", \"tab\": \"General information\", \"score\": \"630.9920634920635\"}", - "Formal Logic - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"formal_logic\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_formal_logic\"" - } - } - }, - { - "evaluation_name": "High School World History", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on High School World History", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.84, - "details": { - "description": "min=0.84, mean=0.84, max=0.84, sum=1.679 (2)", - "tab": "Accuracy", - "High School Biology - Observed inference time (s)": "{\"description\": \"min=0.168, mean=0.168, max=0.168, sum=0.337 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.16835398827829667\"}", - "High School Chemistry - Observed inference time (s)": "{\"description\": \"min=0.171, mean=0.171, max=0.171, sum=0.341 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.17066421649726154\"}", - "High School Computer Science - Observed inference time (s)": "{\"description\": \"min=0.183, mean=0.183, max=0.183, sum=0.367 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.1834348964691162\"}", - "High School European History - Observed inference time (s)": "{\"description\": \"min=0.289, mean=0.289, max=0.289, sum=0.577 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.28851397543242485\"}", - "High School Geography - Observed inference time (s)": "{\"description\": \"min=0.155, mean=0.155, max=0.155, sum=0.31 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.15488721987213752\"}", - "High School Government And Politics - Observed inference time (s)": "{\"description\": \"min=0.169, mean=0.169, max=0.169, sum=0.338 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.16877420331530002\"}", - "High School Macroeconomics - Observed inference time (s)": "{\"description\": \"min=0.159, mean=0.159, max=0.159, sum=0.318 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.1590262247965886\"}", - "High School Mathematics - Observed inference time (s)": "{\"description\": \"min=0.169, mean=0.169, max=0.169, sum=0.337 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.1685257187596074\"}", - "High School Microeconomics - Observed inference time (s)": "{\"description\": \"min=0.156, mean=0.156, max=0.156, sum=0.312 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.1562105868043018\"}", - "High School Physics - Observed inference time (s)": "{\"description\": \"min=0.165, mean=0.165, max=0.165, sum=0.33 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.16475912277272206\"}", - "High School Psychology - Observed inference time (s)": "{\"description\": \"min=0.169, mean=0.169, max=0.169, sum=0.339 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.16945652830491373\"}", - "High School Statistics - Observed inference time (s)": "{\"description\": \"min=0.184, mean=0.184, max=0.184, sum=0.368 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.18419962348761382\"}", - "High School US History - Observed inference time (s)": "{\"description\": \"min=0.285, mean=0.285, max=0.285, sum=0.571 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.28542132938609405\"}", - "High School World History - Observed inference time (s)": "{\"description\": \"min=0.216, mean=0.216, max=0.216, sum=0.433 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.21634829698232658\"}", - "High School Biology - # eval": "{\"description\": \"min=310, mean=310, max=310, sum=620 (2)\", \"tab\": \"General information\", \"score\": \"310.0\"}", - "High School Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Biology - # prompt tokens": "{\"description\": \"min=527.213, mean=527.213, max=527.213, sum=1054.426 (2)\", \"tab\": \"General information\", \"score\": \"527.2129032258065\"}", - "High School Biology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # eval": "{\"description\": \"min=203, mean=203, max=203, sum=406 (2)\", \"tab\": \"General information\", \"score\": \"203.0\"}", - "High School Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # prompt tokens": "{\"description\": \"min=530.635, mean=530.635, max=530.635, sum=1061.271 (2)\", \"tab\": \"General information\", \"score\": \"530.6354679802955\"}", - "High School Chemistry - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "High School Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # prompt tokens": "{\"description\": \"min=932.02, mean=932.02, max=932.02, sum=1864.04 (2)\", \"tab\": \"General information\", \"score\": \"932.02\"}", - "High School Computer Science - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # eval": "{\"description\": \"min=165, mean=165, max=165, sum=330 (2)\", \"tab\": \"General information\", \"score\": \"165.0\"}", - "High School European History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School European History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # prompt tokens": "{\"description\": \"min=2823.042, mean=2823.042, max=2823.042, sum=5646.085 (2)\", \"tab\": \"General information\", \"score\": \"2823.042424242424\"}", - "High School European History - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # eval": "{\"description\": \"min=198, mean=198, max=198, sum=396 (2)\", \"tab\": \"General information\", \"score\": \"198.0\"}", - "High School Geography - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Geography - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # prompt tokens": "{\"description\": \"min=407.818, mean=407.818, max=407.818, sum=815.636 (2)\", \"tab\": \"General information\", \"score\": \"407.8181818181818\"}", - "High School Geography - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # eval": "{\"description\": \"min=193, mean=193, max=193, sum=386 (2)\", \"tab\": \"General information\", \"score\": \"193.0\"}", - "High School Government And Politics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Government And Politics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # prompt tokens": "{\"description\": \"min=489.155, mean=489.155, max=489.155, sum=978.311 (2)\", \"tab\": \"General information\", \"score\": \"489.1554404145078\"}", - "High School Government And Politics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # eval": "{\"description\": \"min=390, mean=390, max=390, sum=780 (2)\", \"tab\": \"General information\", \"score\": \"390.0\"}", - "High School Macroeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Macroeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # prompt tokens": "{\"description\": \"min=407.654, mean=407.654, max=407.654, sum=815.308 (2)\", \"tab\": \"General information\", \"score\": \"407.65384615384613\"}", - "High School Macroeconomics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # eval": "{\"description\": \"min=270, mean=270, max=270, sum=540 (2)\", \"tab\": \"General information\", \"score\": \"270.0\"}", - "High School Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # prompt tokens": "{\"description\": \"min=589.774, mean=589.774, max=589.774, sum=1179.548 (2)\", \"tab\": \"General information\", \"score\": \"589.7740740740741\"}", - "High School Mathematics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # eval": "{\"description\": \"min=238, mean=238, max=238, sum=476 (2)\", \"tab\": \"General information\", \"score\": \"238.0\"}", - "High School Microeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Microeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # prompt tokens": "{\"description\": \"min=428.403, mean=428.403, max=428.403, sum=856.807 (2)\", \"tab\": \"General information\", \"score\": \"428.4033613445378\"}", - "High School Microeconomics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # eval": "{\"description\": \"min=151, mean=151, max=151, sum=302 (2)\", \"tab\": \"General information\", \"score\": \"151.0\"}", - "High School Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # prompt tokens": "{\"description\": \"min=604.272, mean=604.272, max=604.272, sum=1208.543 (2)\", \"tab\": \"General information\", \"score\": \"604.2715231788079\"}", - "High School Physics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # eval": "{\"description\": \"min=545, mean=545, max=545, sum=1090 (2)\", \"tab\": \"General information\", \"score\": \"545.0\"}", - "High School Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # prompt tokens": "{\"description\": \"min=516.004, mean=516.004, max=516.004, sum=1032.007 (2)\", \"tab\": \"General information\", \"score\": \"516.0036697247706\"}", - "High School Psychology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # eval": "{\"description\": \"min=216, mean=216, max=216, sum=432 (2)\", \"tab\": \"General information\", \"score\": \"216.0\"}", - "High School Statistics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Statistics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # prompt tokens": "{\"description\": \"min=871.264, mean=871.264, max=871.264, sum=1742.528 (2)\", \"tab\": \"General information\", \"score\": \"871.2638888888889\"}", - "High School Statistics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # eval": "{\"description\": \"min=204, mean=204, max=204, sum=408 (2)\", \"tab\": \"General information\", \"score\": \"204.0\"}", - "High School US History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School US History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # prompt tokens": "{\"description\": \"min=2240.358, mean=2240.358, max=2240.358, sum=4480.716 (2)\", \"tab\": \"General information\", \"score\": \"2240.357843137255\"}", - "High School US History - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # eval": "{\"description\": \"min=237, mean=237, max=237, sum=474 (2)\", \"tab\": \"General information\", \"score\": \"237.0\"}", - "High School World History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School World History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # prompt tokens": "{\"description\": \"min=1443.321, mean=1443.321, max=1443.321, sum=2886.641 (2)\", \"tab\": \"General information\", \"score\": \"1443.3206751054852\"}", - "High School World History - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"high_school_world_history\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_high_school_world_history\"" - } - } - }, - { - "evaluation_name": "Human Sexuality", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Human Sexuality", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.763, - "details": { - "description": "min=0.763, mean=0.763, max=0.763, sum=1.527 (2)", - "tab": "Accuracy", - "Human Aging - Observed inference time (s)": "{\"description\": \"min=0.154, mean=0.154, max=0.154, sum=0.308 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.15405324649383134\"}", - "Human Sexuality - Observed inference time (s)": "{\"description\": \"min=0.159, mean=0.159, max=0.159, sum=0.318 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.15911357275402274\"}", - "Human Aging - # eval": "{\"description\": \"min=223, mean=223, max=223, sum=446 (2)\", \"tab\": \"General information\", \"score\": \"223.0\"}", - "Human Aging - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Aging - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Aging - # prompt tokens": "{\"description\": \"min=336.09, mean=336.09, max=336.09, sum=672.179 (2)\", \"tab\": \"General information\", \"score\": \"336.0896860986547\"}", - "Human Aging - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # eval": "{\"description\": \"min=131, mean=131, max=131, sum=262 (2)\", \"tab\": \"General information\", \"score\": \"131.0\"}", - "Human Sexuality - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Sexuality - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # prompt tokens": "{\"description\": \"min=367.16, mean=367.16, max=367.16, sum=734.321 (2)\", \"tab\": \"General information\", \"score\": \"367.1603053435114\"}", - "Human Sexuality - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"human_sexuality\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_human_sexuality\"" - } - } - }, - { - "evaluation_name": "International Law", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on International Law", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.802, - "details": { - "description": "min=0.802, mean=0.802, max=0.802, sum=1.603 (2)", - "tab": "Accuracy", - "International Law - Observed inference time (s)": "{\"description\": \"min=0.174, mean=0.174, max=0.174, sum=0.347 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.1736255066453918\"}", - "International Law - # eval": "{\"description\": \"min=121, mean=121, max=121, sum=242 (2)\", \"tab\": \"General information\", \"score\": \"121.0\"}", - "International Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "International Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "International Law - # prompt tokens": "{\"description\": \"min=653.612, mean=653.612, max=653.612, sum=1307.223 (2)\", \"tab\": \"General information\", \"score\": \"653.6115702479339\"}", - "International Law - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"international_law\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_international_law\"" - } - } - }, - { - "evaluation_name": "Logical Fallacies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Logical Fallacies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.798, - "details": { - "description": "min=0.798, mean=0.798, max=0.798, sum=1.595 (2)", - "tab": "Accuracy", - "Logical Fallacies - Observed inference time (s)": "{\"description\": \"min=0.164, mean=0.164, max=0.164, sum=0.327 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.16361909117435386\"}", - "Logical Fallacies - # eval": "{\"description\": \"min=163, mean=163, max=163, sum=326 (2)\", \"tab\": \"General information\", \"score\": \"163.0\"}", - "Logical Fallacies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Logical Fallacies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Logical Fallacies - # prompt tokens": "{\"description\": \"min=463.773, mean=463.773, max=463.773, sum=927.546 (2)\", \"tab\": \"General information\", \"score\": \"463.7730061349693\"}", - "Logical Fallacies - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"logical_fallacies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_logical_fallacies\"" - } - } - }, - { - "evaluation_name": "Machine Learning", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Machine Learning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.446, - "details": { - "description": "min=0.446, mean=0.446, max=0.446, sum=0.893 (2)", - "tab": "Accuracy", - "Machine Learning - Observed inference time (s)": "{\"description\": \"min=0.176, mean=0.176, max=0.176, sum=0.352 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.17583884937422617\"}", - "Machine Learning - # eval": "{\"description\": \"min=112, mean=112, max=112, sum=224 (2)\", \"tab\": \"General information\", \"score\": \"112.0\"}", - "Machine Learning - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Machine Learning - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Machine Learning - # prompt tokens": "{\"description\": \"min=716.438, mean=716.438, max=716.438, sum=1432.875 (2)\", \"tab\": \"General information\", \"score\": \"716.4375\"}", - "Machine Learning - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"machine_learning\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_machine_learning\"" - } - } - }, - { - "evaluation_name": "Management", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Management", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.796, - "details": { - "description": "min=0.796, mean=0.796, max=0.796, sum=1.592 (2)", - "tab": "Accuracy", - "Management - Observed inference time (s)": "{\"description\": \"min=0.156, mean=0.156, max=0.156, sum=0.312 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.15610716875317027\"}", - "Management - # eval": "{\"description\": \"min=103, mean=103, max=103, sum=206 (2)\", \"tab\": \"General information\", \"score\": \"103.0\"}", - "Management - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Management - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Management - # prompt tokens": "{\"description\": \"min=294.456, mean=294.456, max=294.456, sum=588.913 (2)\", \"tab\": \"General information\", \"score\": \"294.45631067961165\"}", - "Management - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"management\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_management\"" - } - } - }, - { - "evaluation_name": "Marketing", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Marketing", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.872, - "details": { - "description": "min=0.872, mean=0.872, max=0.872, sum=1.744 (2)", - "tab": "Accuracy", - "Marketing - Observed inference time (s)": "{\"description\": \"min=0.161, mean=0.161, max=0.161, sum=0.321 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.16073521895286363\"}", - "Marketing - # eval": "{\"description\": \"min=234, mean=234, max=234, sum=468 (2)\", \"tab\": \"General information\", \"score\": \"234.0\"}", - "Marketing - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Marketing - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Marketing - # prompt tokens": "{\"description\": \"min=446.855, mean=446.855, max=446.855, sum=893.709 (2)\", \"tab\": \"General information\", \"score\": \"446.85470085470087\"}", - "Marketing - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"marketing\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_marketing\"" - } - } - }, - { - "evaluation_name": "Medical Genetics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Medical Genetics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.81, - "details": { - "description": "min=0.81, mean=0.81, max=0.81, sum=1.62 (2)", - "tab": "Accuracy", - "Medical Genetics - Observed inference time (s)": "{\"description\": \"min=0.16, mean=0.16, max=0.16, sum=0.319 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.15951916217803955\"}", - "Medical Genetics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Medical Genetics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Medical Genetics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Medical Genetics - # prompt tokens": "{\"description\": \"min=357.02, mean=357.02, max=357.02, sum=714.04 (2)\", \"tab\": \"General information\", \"score\": \"357.02\"}", - "Medical Genetics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"medical_genetics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_medical_genetics\"" - } - } - }, - { - "evaluation_name": "Miscellaneous", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Miscellaneous", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.848, - "details": { - "description": "min=0.848, mean=0.848, max=0.848, sum=1.696 (2)", - "tab": "Accuracy", - "Miscellaneous - Observed inference time (s)": "{\"description\": \"min=0.153, mean=0.153, max=0.153, sum=0.307 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.15332558511317462\"}", - "Miscellaneous - # eval": "{\"description\": \"min=783, mean=783, max=783, sum=1566 (2)\", \"tab\": \"General information\", \"score\": \"783.0\"}", - "Miscellaneous - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Miscellaneous - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Miscellaneous - # prompt tokens": "{\"description\": \"min=325.76, mean=325.76, max=325.76, sum=651.52 (2)\", \"tab\": \"General information\", \"score\": \"325.75989782886336\"}", - "Miscellaneous - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"miscellaneous\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_miscellaneous\"" - } - } - }, - { - "evaluation_name": "Moral Scenarios", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Moral Scenarios", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.451, - "details": { - "description": "min=0.451, mean=0.451, max=0.451, sum=0.903 (2)", - "tab": "Accuracy", - "Moral Disputes - Observed inference time (s)": "{\"description\": \"min=0.17, mean=0.17, max=0.17, sum=0.339 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.16953640452699165\"}", - "Moral Scenarios - Observed inference time (s)": "{\"description\": \"min=0.176, mean=0.176, max=0.176, sum=0.351 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.1756493640345568\"}", - "Moral Disputes - # eval": "{\"description\": \"min=346, mean=346, max=346, sum=692 (2)\", \"tab\": \"General information\", \"score\": \"346.0\"}", - "Moral Disputes - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Disputes - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Disputes - # prompt tokens": "{\"description\": \"min=506.78, mean=506.78, max=506.78, sum=1013.561 (2)\", \"tab\": \"General information\", \"score\": \"506.78034682080926\"}", - "Moral Disputes - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # eval": "{\"description\": \"min=895, mean=895, max=895, sum=1790 (2)\", \"tab\": \"General information\", \"score\": \"895.0\"}", - "Moral Scenarios - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Scenarios - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # prompt tokens": "{\"description\": \"min=699.344, mean=699.344, max=699.344, sum=1398.688 (2)\", \"tab\": \"General information\", \"score\": \"699.3441340782123\"}", - "Moral Scenarios - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"moral_scenarios\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_moral_scenarios\"" - } - } - }, - { - "evaluation_name": "Nutrition", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Nutrition", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.703, - "details": { - "description": "min=0.703, mean=0.703, max=0.703, sum=1.405 (2)", - "tab": "Accuracy", - "Nutrition - Observed inference time (s)": "{\"description\": \"min=0.171, mean=0.171, max=0.171, sum=0.342 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.17089871020098918\"}", - "Nutrition - # eval": "{\"description\": \"min=306, mean=306, max=306, sum=612 (2)\", \"tab\": \"General information\", \"score\": \"306.0\"}", - "Nutrition - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Nutrition - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Nutrition - # prompt tokens": "{\"description\": \"min=618.402, mean=618.402, max=618.402, sum=1236.804 (2)\", \"tab\": \"General information\", \"score\": \"618.4019607843137\"}", - "Nutrition - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"nutrition\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_nutrition\"" - } - } - }, - { - "evaluation_name": "Prehistory", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Prehistory", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.728, - "details": { - "description": "min=0.728, mean=0.728, max=0.728, sum=1.457 (2)", - "tab": "Accuracy", - "Prehistory - Observed inference time (s)": "{\"description\": \"min=0.167, mean=0.167, max=0.167, sum=0.333 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.16663335429297554\"}", - "Prehistory - # eval": "{\"description\": \"min=324, mean=324, max=324, sum=648 (2)\", \"tab\": \"General information\", \"score\": \"324.0\"}", - "Prehistory - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Prehistory - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Prehistory - # prompt tokens": "{\"description\": \"min=549.235, mean=549.235, max=549.235, sum=1098.469 (2)\", \"tab\": \"General information\", \"score\": \"549.2345679012345\"}", - "Prehistory - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"prehistory\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_prehistory\"" - } - } - }, - { - "evaluation_name": "Public Relations", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Public Relations", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7, - "details": { - "description": "min=0.7, mean=0.7, max=0.7, sum=1.4 (2)", - "tab": "Accuracy", - "Public Relations - Observed inference time (s)": "{\"description\": \"min=0.17, mean=0.17, max=0.17, sum=0.341 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.17039124532179398\"}", - "Public Relations - # eval": "{\"description\": \"min=110, mean=110, max=110, sum=220 (2)\", \"tab\": \"General information\", \"score\": \"110.0\"}", - "Public Relations - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Public Relations - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Public Relations - # prompt tokens": "{\"description\": \"min=434.682, mean=434.682, max=434.682, sum=869.364 (2)\", \"tab\": \"General information\", \"score\": \"434.6818181818182\"}", - "Public Relations - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"public_relations\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_public_relations\"" - } - } - }, - { - "evaluation_name": "Security Studies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Security Studies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.714, - "details": { - "description": "min=0.714, mean=0.714, max=0.714, sum=1.429 (2)", - "tab": "Accuracy", - "Security Studies - Observed inference time (s)": "{\"description\": \"min=0.203, mean=0.203, max=0.203, sum=0.405 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.20251671927315848\"}", - "Security Studies - # eval": "{\"description\": \"min=245, mean=245, max=245, sum=490 (2)\", \"tab\": \"General information\", \"score\": \"245.0\"}", - "Security Studies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Security Studies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Security Studies - # prompt tokens": "{\"description\": \"min=1207.494, mean=1207.494, max=1207.494, sum=2414.988 (2)\", \"tab\": \"General information\", \"score\": \"1207.4938775510204\"}", - "Security Studies - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"security_studies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_security_studies\"" - } - } - }, - { - "evaluation_name": "Sociology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Sociology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.866, - "details": { - "description": "min=0.866, mean=0.866, max=0.866, sum=1.731 (2)", - "tab": "Accuracy", - "Sociology - Observed inference time (s)": "{\"description\": \"min=0.164, mean=0.164, max=0.164, sum=0.327 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.16369761163322485\"}", - "Sociology - # eval": "{\"description\": \"min=201, mean=201, max=201, sum=402 (2)\", \"tab\": \"General information\", \"score\": \"201.0\"}", - "Sociology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Sociology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Sociology - # prompt tokens": "{\"description\": \"min=467.343, mean=467.343, max=467.343, sum=934.687 (2)\", \"tab\": \"General information\", \"score\": \"467.34328358208955\"}", - "Sociology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"sociology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_sociology\"" - } - } - }, - { - "evaluation_name": "Virology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Virology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.542, - "details": { - "description": "min=0.542, mean=0.542, max=0.542, sum=1.084 (2)", - "tab": "Accuracy", - "Virology - Observed inference time (s)": "{\"description\": \"min=0.158, mean=0.158, max=0.158, sum=0.316 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.15811713919582138\"}", - "Virology - # eval": "{\"description\": \"min=166, mean=166, max=166, sum=332 (2)\", \"tab\": \"General information\", \"score\": \"166.0\"}", - "Virology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Virology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Virology - # prompt tokens": "{\"description\": \"min=352.861, mean=352.861, max=352.861, sum=705.723 (2)\", \"tab\": \"General information\", \"score\": \"352.8614457831325\"}", - "Virology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"virology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_virology\"" - } - } - }, - { - "evaluation_name": "World Religions", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on World Religions", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.813, - "details": { - "description": "min=0.813, mean=0.813, max=0.813, sum=1.626 (2)", - "tab": "Accuracy", - "World Religions - Observed inference time (s)": "{\"description\": \"min=0.145, mean=0.145, max=0.145, sum=0.291 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.1452833434991669\"}", - "World Religions - # eval": "{\"description\": \"min=171, mean=171, max=171, sum=342 (2)\", \"tab\": \"General information\", \"score\": \"171.0\"}", - "World Religions - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "World Religions - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "World Religions - # prompt tokens": "{\"description\": \"min=277.047, mean=277.047, max=277.047, sum=554.094 (2)\", \"tab\": \"General information\", \"score\": \"277.046783625731\"}", - "World Religions - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"world_religions\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_world_religions\"" - } - } - }, - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.959, - "details": { - "description": "", - "tab": "Efficiency" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/cohere_command-r7b-fc.json b/data/models/cohere_command-r7b-fc.json deleted file mode 100644 index f71ba16cea7a342ade0734e65fb564e4c6d3f527..0000000000000000000000000000000000000000 --- a/data/models/cohere_command-r7b-fc.json +++ /dev/null @@ -1,851 +0,0 @@ -{ - "model_info": { - "name": "Command R7B (FC)", - "id": "cohere/command-r7b-fc", - "developer": "cohere", - "additional_details": { - "raw_model_name": "Command R7B (FC)", - "organization": "Cohere", - "license": "cc-by-nc-4.0", - "mode": "FC", - "model_link": "https://cohere.com/blog/command-r7b" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/cohere/command-r7b-fc/1775236112.3976161", - "retrieved_timestamp": "1775236112.3976161", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 61.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 32.07 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 1.5 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 1.38 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 2.87 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 2.69 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 80.96 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 67.33 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 89.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 85.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 81.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 69.06 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 62.79 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 70.94 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 43.75 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 70.83 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 8.25 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 12.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 0.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 10.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 10.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 27.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 43.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 11.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 5.16 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 2.58 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 9.68 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 3.23 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 68.75 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 81.65 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cohere_command-xlarge-beta.json b/data/models/cohere_command-xlarge-beta.json deleted file mode 100644 index edf773fb2387c7176add2a6bcf318784ad6c4721..0000000000000000000000000000000000000000 --- a/data/models/cohere_command-xlarge-beta.json +++ /dev/null @@ -1,237 +0,0 @@ -{ - "model_info": { - "name": "Cohere Command beta 52.4B", - "id": "cohere/command-xlarge-beta", - "developer": "cohere", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_instruct/cohere_command-xlarge-beta/1774096309.537868", - "retrieved_timestamp": "1774096309.537868", - "source_metadata": { - "source_name": "helm_instruct", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_instruct", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_instruct", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/instruct/benchmark_output/releases/v1.0.0/groups/instruction_following.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperform on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.089, - "details": { - "description": "", - "tab": "Instruction Following" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "Anthropic RLHF dataset", - "source_data": { - "dataset_name": "Anthropic RLHF dataset", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/instruct/benchmark_output/releases/v1.0.0/groups/instruction_following.json" - ] - }, - "metric_config": { - "evaluation_description": "Harmlessness on Anthropic RLHF dataset", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 5.0 - }, - "score_details": { - "score": 4.214, - "details": { - "description": "min=3.38, mean=4.214, max=4.92, sum=33.715 (8)", - "tab": "Instruction Following" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"hh\", \"hh\", \"hh\", \"hh\", \"red_team\", \"red_team\", \"red_team\", \"red_team\"]", - "evaluator": "[\"claude\", \"gpt4\", \"mturk\", \"scale\", \"claude\", \"gpt4\", \"mturk\", \"scale\"]" - } - } - }, - { - "evaluation_name": "Best ChatGPT Prompts", - "source_data": { - "dataset_name": "Best ChatGPT Prompts", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/instruct/benchmark_output/releases/v1.0.0/groups/instruction_following.json" - ] - }, - "metric_config": { - "evaluation_description": "Harmlessness on Best ChatGPT Prompts", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 5.0 - }, - "score_details": { - "score": 4.988, - "details": { - "description": "min=4.98, mean=4.988, max=5, sum=19.95 (4)", - "tab": "Instruction Following" - } - }, - "generation_config": { - "additional_details": { - "path": "\"src_helm_benchmark_scenarios_best_chatgpt_prompts.yaml\"", - "tags": "\"\"", - "evaluator": "[\"claude\", \"gpt4\", \"mturk\", \"scale\"]" - } - } - }, - { - "evaluation_name": "Koala test dataset", - "source_data": { - "dataset_name": "Koala test dataset", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/instruct/benchmark_output/releases/v1.0.0/groups/instruction_following.json" - ] - }, - "metric_config": { - "evaluation_description": "Harmlessness on Koala test dataset", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 5.0 - }, - "score_details": { - "score": 4.969, - "details": { - "description": "min=4.936, mean=4.969, max=5, sum=19.874 (4)", - "tab": "Instruction Following" - } - }, - "generation_config": { - "additional_details": { - "evaluator": "[\"claude\", \"gpt4\", \"mturk\", \"scale\"]" - } - } - }, - { - "evaluation_name": "Open Assistant", - "source_data": { - "dataset_name": "Open Assistant", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/instruct/benchmark_output/releases/v1.0.0/groups/instruction_following.json" - ] - }, - "metric_config": { - "evaluation_description": "Harmlessness on Open Assistant", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 5.0 - }, - "score_details": { - "score": 4.967, - "details": { - "description": "min=4.955, mean=4.967, max=5, sum=19.87 (4)", - "tab": "Instruction Following" - } - }, - "generation_config": { - "additional_details": { - "language": "\"en\"", - "evaluator": "[\"claude\", \"gpt4\", \"mturk\", \"scale\"]" - } - } - }, - { - "evaluation_name": "Self Instruct", - "source_data": { - "dataset_name": "Self Instruct", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/instruct/benchmark_output/releases/v1.0.0/groups/instruction_following.json" - ] - }, - "metric_config": { - "evaluation_description": "Harmlessness on Self Instruct", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 5.0 - }, - "score_details": { - "score": 4.971, - "details": { - "description": "min=4.955, mean=4.971, max=5, sum=19.885 (4)", - "tab": "Instruction Following" - } - }, - "generation_config": { - "additional_details": { - "evaluator": "[\"claude\", \"gpt4\", \"mturk\", \"scale\"]" - } - } - }, - { - "evaluation_name": "Vicuna", - "source_data": { - "dataset_name": "Vicuna", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/instruct/benchmark_output/releases/v1.0.0/groups/instruction_following.json" - ] - }, - "metric_config": { - "evaluation_description": "Harmlessness on Vicuna", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 5.0 - }, - "score_details": { - "score": 4.995, - "details": { - "description": "min=4.981, mean=4.995, max=5, sum=19.981 (4)", - "tab": "Instruction Following" - } - }, - "generation_config": { - "additional_details": { - "category": "\"all\"", - "evaluator": "[\"claude\", \"gpt4\", \"mturk\", \"scale\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/cohere_command.json b/data/models/cohere_command.json deleted file mode 100644 index 76967d7d01de28994e9834cfb281b6a5404bd231..0000000000000000000000000000000000000000 --- a/data/models/cohere_command.json +++ /dev/null @@ -1,380 +0,0 @@ -{ - "model_info": { - "name": "Command", - "id": "cohere/command", - "developer": "cohere", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_lite/cohere_command/1774096306.427425", - "retrieved_timestamp": "1774096306.427425", - "source_metadata": { - "source_name": "helm_lite", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_lite", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_lite", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.327, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.21596754057428214\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.749, - "details": { - "description": "min=0.749, mean=0.749, max=0.749, sum=0.749 (1)", - "tab": "Accuracy", - "NarrativeQA - Observed inference time (s)": "{\"description\": \"min=1.783, mean=1.783, max=1.783, sum=1.783 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.783306110408944\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=1.941, mean=1.941, max=1.941, sum=1.941 (1)\", \"tab\": \"General information\", \"score\": \"1.9408450704225353\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=1660.485, mean=1660.485, max=1660.485, sum=1660.485 (1)\", \"tab\": \"General information\", \"score\": \"1660.4845070422534\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=7.442, mean=7.442, max=7.442, sum=7.442 (1)\", \"tab\": \"General information\", \"score\": \"7.44225352112676\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (closed-book)", - "source_data": { - "dataset_name": "NaturalQuestions (closed-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (closed-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.391, - "details": { - "description": "min=0.391, mean=0.391, max=0.391, sum=0.391 (1)", - "tab": "Accuracy", - "NaturalQuestions (open-book) - Observed inference time (s)": "{\"description\": \"min=1.804, mean=1.804, max=1.804, sum=1.804 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.8040301027297974\"}", - "NaturalQuestions (closed-book) - Observed inference time (s)": "{\"description\": \"min=0.986, mean=0.986, max=0.986, sum=0.986 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.9856750283241272\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.617, mean=4.617, max=4.617, sum=4.617 (1)\", \"tab\": \"General information\", \"score\": \"4.617\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.039, mean=0.039, max=0.039, sum=0.039 (1)\", \"tab\": \"General information\", \"score\": \"0.039\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1557.639, mean=1557.639, max=1557.639, sum=1557.639 (1)\", \"tab\": \"General information\", \"score\": \"1557.639\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=8.461, mean=8.461, max=8.461, sum=8.461 (1)\", \"tab\": \"General information\", \"score\": \"8.461\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=115.191, mean=115.191, max=115.191, sum=115.191 (1)\", \"tab\": \"General information\", \"score\": \"115.191\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=5.679, mean=5.679, max=5.679, sum=5.679 (1)\", \"tab\": \"General information\", \"score\": \"5.679\"}" - } - }, - "generation_config": { - "additional_details": { - "mode": "\"closedbook\"" - } - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.774, - "details": { - "description": "min=0.774, mean=0.774, max=0.774, sum=0.774 (1)", - "tab": "Accuracy", - "OpenbookQA - Observed inference time (s)": "{\"description\": \"min=1.044, mean=1.044, max=1.044, sum=1.044 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.0440752515792846\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=246.682, mean=246.682, max=246.682, sum=246.682 (1)\", \"tab\": \"General information\", \"score\": \"246.682\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "dataset": "\"openbookqa\"", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.525, - "details": { - "description": "min=0.27, mean=0.525, max=0.88, sum=2.626 (5)", - "tab": "Accuracy", - "MMLU - Observed inference time (s)": "{\"description\": \"min=0.821, mean=1.08, max=1.384, sum=5.399 (5)\", \"tab\": \"Efficiency\", \"score\": \"1.0797608851633573\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=372.75, mean=481.26, max=628.421, sum=2406.301 (5)\", \"tab\": \"General information\", \"score\": \"481.26021052631575\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"college_chemistry\", \"computer_security\", \"econometrics\", \"us_foreign_policy\"]", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MATH", - "source_data": { - "dataset_name": "MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Equivalent (CoT) on MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.236, - "details": { - "description": "min=0.1, mean=0.236, max=0.349, sum=1.652 (7)", - "tab": "Accuracy", - "MATH - Observed inference time (s)": "{\"description\": \"min=4.562, mean=5.762, max=6.509, sum=40.337 (7)\", \"tab\": \"Efficiency\", \"score\": \"5.762416239357385\"}", - "MATH - # eval": "{\"description\": \"min=30, mean=62.429, max=135, sum=437 (7)\", \"tab\": \"General information\", \"score\": \"62.42857142857143\"}", - "MATH - # train": "{\"description\": \"min=2.962, mean=6.878, max=8, sum=48.146 (7)\", \"tab\": \"General information\", \"score\": \"6.877964141122035\"}", - "MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MATH - # prompt tokens": "{\"description\": \"min=925.333, mean=1177.329, max=1534.058, sum=8241.302 (7)\", \"tab\": \"General information\", \"score\": \"1177.3289276411065\"}", - "MATH - # output tokens": "{\"description\": \"min=94.488, mean=116.49, max=135.115, sum=815.428 (7)\", \"tab\": \"General information\", \"score\": \"116.48968047229982\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"algebra\", \"counting_and_probability\", \"geometry\", \"intermediate_algebra\", \"number_theory\", \"prealgebra\", \"precalculus\"]", - "level": "\"1\"", - "use_official_examples": "\"False\"", - "use_chain_of_thought": "\"True\"" - } - } - }, - { - "evaluation_name": "GSM8K", - "source_data": { - "dataset_name": "GSM8K", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on GSM8K", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.452, - "details": { - "description": "min=0.452, mean=0.452, max=0.452, sum=0.452 (1)", - "tab": "Accuracy", - "GSM8K - Observed inference time (s)": "{\"description\": \"min=4.127, mean=4.127, max=4.127, sum=4.127 (1)\", \"tab\": \"Efficiency\", \"score\": \"4.127378141641617\"}", - "GSM8K - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "GSM8K - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "GSM8K - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GSM8K - # prompt tokens": "{\"description\": \"min=942.424, mean=942.424, max=942.424, sum=942.424 (1)\", \"tab\": \"General information\", \"score\": \"942.424\"}", - "GSM8K - # output tokens": "{\"description\": \"min=94.43, mean=94.43, max=94.43, sum=94.43 (1)\", \"tab\": \"General information\", \"score\": \"94.43\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "LegalBench", - "source_data": { - "dataset_name": "LegalBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on LegalBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.578, - "details": { - "description": "min=0.365, mean=0.578, max=0.884, sum=2.888 (5)", - "tab": "Accuracy", - "LegalBench - Observed inference time (s)": "{\"description\": \"min=0.856, mean=1.165, max=1.842, sum=5.823 (5)\", \"tab\": \"Efficiency\", \"score\": \"1.1646721122881132\"}", - "LegalBench - # eval": "{\"description\": \"min=95, mean=409.4, max=1000, sum=2047 (5)\", \"tab\": \"General information\", \"score\": \"409.4\"}", - "LegalBench - # train": "{\"description\": \"min=0.388, mean=3.878, max=5, sum=19.388 (5)\", \"tab\": \"General information\", \"score\": \"3.8775510204081636\"}", - "LegalBench - truncated": "{\"description\": \"min=0, mean=0.003, max=0.014, sum=0.014 (5)\", \"tab\": \"General information\", \"score\": \"0.002857142857142857\"}", - "LegalBench - # prompt tokens": "{\"description\": \"min=205.295, mean=566.501, max=1529.327, sum=2832.507 (5)\", \"tab\": \"General information\", \"score\": \"566.5014751745068\"}", - "LegalBench - # output tokens": "{\"description\": \"min=1, mean=1.79, max=3.055, sum=8.948 (5)\", \"tab\": \"General information\", \"score\": \"1.7895877106155815\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"abercrombie\", \"corporate_lobbying\", \"function_of_decision_section\", \"international_citizenship_questions\", \"proa\"]" - } - } - }, - { - "evaluation_name": "MedQA", - "source_data": { - "dataset_name": "MedQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MedQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.445, - "details": { - "description": "min=0.445, mean=0.445, max=0.445, sum=0.445 (1)", - "tab": "Accuracy", - "MedQA - Observed inference time (s)": "{\"description\": \"min=1.234, mean=1.234, max=1.234, sum=1.234 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.2344102347584416\"}", - "MedQA - # eval": "{\"description\": \"min=503, mean=503, max=503, sum=503 (1)\", \"tab\": \"General information\", \"score\": \"503.0\"}", - "MedQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MedQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MedQA - # prompt tokens": "{\"description\": \"min=1016.738, mean=1016.738, max=1016.738, sum=1016.738 (1)\", \"tab\": \"General information\", \"score\": \"1016.7375745526839\"}", - "MedQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WMT 2014", - "source_data": { - "dataset_name": "WMT 2014", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "BLEU-4 on WMT 2014", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.088, - "details": { - "description": "min=0.013, mean=0.088, max=0.151, sum=0.441 (5)", - "tab": "Accuracy", - "WMT 2014 - Observed inference time (s)": "{\"description\": \"min=2.376, mean=2.894, max=3.133, sum=14.469 (5)\", \"tab\": \"Efficiency\", \"score\": \"2.8937741082134893\"}", - "WMT 2014 - # eval": "{\"description\": \"min=503, mean=568.8, max=832, sum=2844 (5)\", \"tab\": \"General information\", \"score\": \"568.8\"}", - "WMT 2014 - # train": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "WMT 2014 - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WMT 2014 - # prompt tokens": "{\"description\": \"min=129.757, mean=149.459, max=178.821, sum=747.297 (5)\", \"tab\": \"General information\", \"score\": \"149.45941179844013\"}", - "WMT 2014 - # output tokens": "{\"description\": \"min=27.65, mean=31.8, max=41.789, sum=159.002 (5)\", \"tab\": \"General information\", \"score\": \"31.800405260743236\"}" - } - }, - "generation_config": { - "additional_details": { - "language_pair": "[\"cs-en\", \"de-en\", \"fr-en\", \"hi-en\", \"ru-en\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/cohere_march_2024.json b/data/models/cohere_march_2024.json deleted file mode 100644 index 7785d6ea455c6d17eac42dbbb1a517543aaa66f4..0000000000000000000000000000000000000000 --- a/data/models/cohere_march_2024.json +++ /dev/null @@ -1,144 +0,0 @@ -{ - "model_info": { - "name": "Cohere March 2024", - "id": "Cohere March 2024", - "developer": "unknown", - "additional_details": { - "model_type": "Custom Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/Cohere March 2024/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8511 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9469 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6513 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.877 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9817 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7458 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cohere_may_2024.json b/data/models/cohere_may_2024.json deleted file mode 100644 index 7a10cf9a50285be2087206ce61b9d1b57c5fdabf..0000000000000000000000000000000000000000 --- a/data/models/cohere_may_2024.json +++ /dev/null @@ -1,144 +0,0 @@ -{ - "model_info": { - "name": "Cohere May 2024", - "id": "Cohere May 2024", - "developer": "unknown", - "additional_details": { - "model_type": "Custom Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/Cohere May 2024/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8816 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9637 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7127 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.923 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9768 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.782 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cohereforai_aya-23-35b.json b/data/models/cohereforai_aya-23-35b.json deleted file mode 100644 index 9a1e0ca7115b039246473a9f1407e4ea6393e5d6..0000000000000000000000000000000000000000 --- a/data/models/cohereforai_aya-23-35b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "aya-23-35B", - "id": "CohereForAI/aya-23-35B", - "developer": "CohereForAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "CohereForCausalLM", - "params_billions": "34.981" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/CohereForAI_aya-23-35B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6462 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.54 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0347 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2945 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.431 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3356 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cohereforai_aya-23-8b.json b/data/models/cohereforai_aya-23-8b.json deleted file mode 100644 index acdfcec9cd3596d40b156a1f2d892c4e307d77b0..0000000000000000000000000000000000000000 --- a/data/models/cohereforai_aya-23-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "aya-23-8B", - "id": "CohereForAI/aya-23-8B", - "developer": "CohereForAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "CohereForCausalLM", - "params_billions": "8.028" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/CohereForAI_aya-23-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4699 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4296 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0166 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2844 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3941 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2278 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cohereforai_aya-expanse-32b.json b/data/models/cohereforai_aya-expanse-32b.json deleted file mode 100644 index 544bf02915470ab9896c1b5da56abf2a3277a6fe..0000000000000000000000000000000000000000 --- a/data/models/cohereforai_aya-expanse-32b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "aya-expanse-32b", - "id": "CohereForAI/aya-expanse-32b", - "developer": "CohereForAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "CohereForCausalLM", - "params_billions": "32.296" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/CohereForAI_aya-expanse-32b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7302 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5649 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1533 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3255 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3873 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.413 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cohereforai_aya-expanse-8b.json b/data/models/cohereforai_aya-expanse-8b.json deleted file mode 100644 index c100403d7c3687ce4b2b7044f4d60bdeb0e2933f..0000000000000000000000000000000000000000 --- a/data/models/cohereforai_aya-expanse-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "aya-expanse-8b", - "id": "CohereForAI/aya-expanse-8b", - "developer": "CohereForAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "CohereForCausalLM", - "params_billions": "8.028" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/CohereForAI_aya-expanse-8b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6359 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4977 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0861 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3029 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3729 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3004 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cohereforai_c4ai-command-r-plus-08-2024.json b/data/models/cohereforai_c4ai-command-r-plus-08-2024.json deleted file mode 100644 index d9938000dcfacc98745f4998ae009e3dbd94a4d1..0000000000000000000000000000000000000000 --- a/data/models/cohereforai_c4ai-command-r-plus-08-2024.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "c4ai-command-r-plus-08-2024", - "id": "CohereForAI/c4ai-command-r-plus-08-2024", - "developer": "CohereForAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "CohereForCausalLM", - "params_billions": "103.811" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/CohereForAI_c4ai-command-r-plus-08-2024/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.754 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5996 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1239 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3507 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4829 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4421 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cohereforai_c4ai-command-r-plus.json b/data/models/cohereforai_c4ai-command-r-plus.json deleted file mode 100644 index ae44fc90e696a06e13fb7cf04444759374089943..0000000000000000000000000000000000000000 --- a/data/models/cohereforai_c4ai-command-r-plus.json +++ /dev/null @@ -1,277 +0,0 @@ -{ - "model_info": { - "name": "CohereForAI/c4ai-command-r-plus", - "id": "CohereForAI/c4ai-command-r-plus", - "developer": "CohereForAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "CohereForCausalLM", - "params_billions": "103.811" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/CohereForAI_c4ai-command-r-plus/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7664 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5815 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0801 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3054 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4807 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3992 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "reward-bench/CohereForAI_c4ai-command-r-plus/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7057 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9511 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5757 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5986 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.704 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6924 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cohereforai_c4ai-command-r-v01.json b/data/models/cohereforai_c4ai-command-r-v01.json deleted file mode 100644 index cfe61933bcce97212d72732a59c9da067b78cd93..0000000000000000000000000000000000000000 --- a/data/models/cohereforai_c4ai-command-r-v01.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "c4ai-command-r-v01", - "id": "CohereForAI/c4ai-command-r-v01", - "developer": "CohereForAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "CohereForCausalLM", - "params_billions": "34.981" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/CohereForAI_c4ai-command-r-v01/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6748 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5406 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0347 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.307 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4517 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3369 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cohereforai_c4ai-command-r7b-12-2024.json b/data/models/cohereforai_c4ai-command-r7b-12-2024.json deleted file mode 100644 index f555c646ba6c85a358c5b102c2c4afa63ed808ca..0000000000000000000000000000000000000000 --- a/data/models/cohereforai_c4ai-command-r7b-12-2024.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "c4ai-command-r7b-12-2024", - "id": "CohereForAI/c4ai-command-r7b-12-2024", - "developer": "CohereForAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Cohere2ForCausalLM", - "params_billions": "8.028" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/CohereForAI_c4ai-command-r7b-12-2024/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7713 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5503 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2991 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3087 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4125 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3572 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/collaiborateorg_collaiborator-medllm-llama-3-8b-v2.json b/data/models/collaiborateorg_collaiborator-medllm-llama-3-8b-v2.json deleted file mode 100644 index 72c38ab614d80a88c8fe52d6e92eeaa6e301d616..0000000000000000000000000000000000000000 --- a/data/models/collaiborateorg_collaiborator-medllm-llama-3-8b-v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Collaiborator-MEDLLM-Llama-3-8B-v2", - "id": "collaiborateorg/Collaiborator-MEDLLM-Llama-3-8B-v2", - "developer": "collaiborateorg", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/collaiborateorg_Collaiborator-MEDLLM-Llama-3-8B-v2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3809 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4648 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0566 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3331 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3434 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3481 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/columbia-nlp_lion-gemma-2b-dpo-v1.0.json b/data/models/columbia-nlp_lion-gemma-2b-dpo-v1.0.json deleted file mode 100644 index fcba7390760a9757eb868fa82619eef91d551f86..0000000000000000000000000000000000000000 --- a/data/models/columbia-nlp_lion-gemma-2b-dpo-v1.0.json +++ /dev/null @@ -1,275 +0,0 @@ -{ - "model_info": { - "name": "LION-Gemma-2b-dpo-v1.0", - "id": "Columbia-NLP/LION-Gemma-2b-dpo-v1.0", - "developer": "Columbia-NLP", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "GemmaForCausalLM", - "params_billions": "2.506" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Columbia-NLP_LION-Gemma-2b-dpo-v1.0/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3102 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3881 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0536 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2534 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4081 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1665 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "hfopenllm_v2/Columbia-NLP_LION-Gemma-2b-dpo-v1.0/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3278 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.392 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0431 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2492 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.412 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1666 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/columbia-nlp_lion-gemma-2b-odpo-v1.0.json b/data/models/columbia-nlp_lion-gemma-2b-odpo-v1.0.json deleted file mode 100644 index c311802300e4223d0fa21abc81558d4377cdaa26..0000000000000000000000000000000000000000 --- a/data/models/columbia-nlp_lion-gemma-2b-odpo-v1.0.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "LION-Gemma-2b-odpo-v1.0", - "id": "Columbia-NLP/LION-Gemma-2b-odpo-v1.0", - "developer": "Columbia-NLP", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "GemmaForCausalLM", - "params_billions": "2.506" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Columbia-NLP_LION-Gemma-2b-odpo-v1.0/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3066 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3896 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0695 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2424 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4279 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1692 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/columbia-nlp_lion-gemma-2b-sft-v1.0.json b/data/models/columbia-nlp_lion-gemma-2b-sft-v1.0.json deleted file mode 100644 index 02e2712f5855e47a68e251a99f1303724addeeb9..0000000000000000000000000000000000000000 --- a/data/models/columbia-nlp_lion-gemma-2b-sft-v1.0.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "LION-Gemma-2b-sft-v1.0", - "id": "Columbia-NLP/LION-Gemma-2b-sft-v1.0", - "developer": "Columbia-NLP", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "GemmaForCausalLM", - "params_billions": "2.506" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Columbia-NLP_LION-Gemma-2b-sft-v1.0/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3692 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3879 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.068 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2559 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4027 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1782 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/columbia-nlp_lion-llama-3-8b-dpo-v1.0.json b/data/models/columbia-nlp_lion-llama-3-8b-dpo-v1.0.json deleted file mode 100644 index 6518893e3bfa20d829025cb6b3f769d0882ad845..0000000000000000000000000000000000000000 --- a/data/models/columbia-nlp_lion-llama-3-8b-dpo-v1.0.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "LION-LLaMA-3-8b-dpo-v1.0", - "id": "Columbia-NLP/LION-LLaMA-3-8b-dpo-v1.0", - "developer": "Columbia-NLP", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Columbia-NLP_LION-LLaMA-3-8b-dpo-v1.0/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4957 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5028 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1171 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.281 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4097 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3219 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/columbia-nlp_lion-llama-3-8b-odpo-v1.0.json b/data/models/columbia-nlp_lion-llama-3-8b-odpo-v1.0.json deleted file mode 100644 index 4b8a112043546e2ca402abdf6ef79789a9a2b845..0000000000000000000000000000000000000000 --- a/data/models/columbia-nlp_lion-llama-3-8b-odpo-v1.0.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "LION-LLaMA-3-8b-odpo-v1.0", - "id": "Columbia-NLP/LION-LLaMA-3-8b-odpo-v1.0", - "developer": "Columbia-NLP", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Columbia-NLP_LION-LLaMA-3-8b-odpo-v1.0/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3968 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5024 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1065 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2852 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4057 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3152 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/columbia-nlp_lion-llama-3-8b-sft-v1.0.json b/data/models/columbia-nlp_lion-llama-3-8b-sft-v1.0.json deleted file mode 100644 index 082ab5842e75d8c0e8762efc76bcd455b73a92da..0000000000000000000000000000000000000000 --- a/data/models/columbia-nlp_lion-llama-3-8b-sft-v1.0.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "LION-LLaMA-3-8b-sft-v1.0", - "id": "Columbia-NLP/LION-LLaMA-3-8b-sft-v1.0", - "developer": "Columbia-NLP", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Columbia-NLP_LION-LLaMA-3-8b-sft-v1.0/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3817 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5088 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.114 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2777 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4503 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3237 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/combinhorizon_huihui-ai-abliterated-qwen2.5-32b-inst-basemerge-ties.json b/data/models/combinhorizon_huihui-ai-abliterated-qwen2.5-32b-inst-basemerge-ties.json deleted file mode 100644 index 5e62588b1d5520f57e77233382925ab5c60df743..0000000000000000000000000000000000000000 --- a/data/models/combinhorizon_huihui-ai-abliterated-qwen2.5-32b-inst-basemerge-ties.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "huihui-ai-abliterated-Qwen2.5-32B-Inst-BaseMerge-TIES", - "id": "CombinHorizon/huihui-ai-abliterated-Qwen2.5-32B-Inst-BaseMerge-TIES", - "developer": "CombinHorizon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "32.764" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/CombinHorizon_huihui-ai-abliterated-Qwen2.5-32B-Inst-BaseMerge-TIES/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8206 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6929 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5944 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3389 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4207 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5721 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/combinhorizon_huihui-ai-abliteratedv2-qwen2.5-14b-inst-basemerge-ties.json b/data/models/combinhorizon_huihui-ai-abliteratedv2-qwen2.5-14b-inst-basemerge-ties.json deleted file mode 100644 index d4bc6446a31c960994e2a61fa15b6ed9f587057f..0000000000000000000000000000000000000000 --- a/data/models/combinhorizon_huihui-ai-abliteratedv2-qwen2.5-14b-inst-basemerge-ties.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "huihui-ai-abliteratedV2-Qwen2.5-14B-Inst-BaseMerge-TIES", - "id": "CombinHorizon/huihui-ai-abliteratedV2-Qwen2.5-14B-Inst-BaseMerge-TIES", - "developer": "CombinHorizon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/CombinHorizon_huihui-ai-abliteratedV2-Qwen2.5-14B-Inst-BaseMerge-TIES/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8176 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6336 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5476 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3146 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.426 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.491 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/combinhorizon_josiefied-abliteratedv4-qwen2.5-14b-inst-basemerge-ties.json b/data/models/combinhorizon_josiefied-abliteratedv4-qwen2.5-14b-inst-basemerge-ties.json deleted file mode 100644 index 190440c2f9aa4cfed461c3f8f6007f04d97cd2fa..0000000000000000000000000000000000000000 --- a/data/models/combinhorizon_josiefied-abliteratedv4-qwen2.5-14b-inst-basemerge-ties.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Josiefied-abliteratedV4-Qwen2.5-14B-Inst-BaseMerge-TIES", - "id": "CombinHorizon/Josiefied-abliteratedV4-Qwen2.5-14B-Inst-BaseMerge-TIES", - "developer": "CombinHorizon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/CombinHorizon_Josiefied-abliteratedV4-Qwen2.5-14B-Inst-BaseMerge-TIES/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.824 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.637 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5317 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3247 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.426 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4979 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/combinhorizon_rombos-qwen2.5-7b-inst-basemerge-ties.json b/data/models/combinhorizon_rombos-qwen2.5-7b-inst-basemerge-ties.json deleted file mode 100644 index abb34d842b2caafbc3d4724f0def0ae93ddd1a8d..0000000000000000000000000000000000000000 --- a/data/models/combinhorizon_rombos-qwen2.5-7b-inst-basemerge-ties.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Rombos-Qwen2.5-7B-Inst-BaseMerge-TIES", - "id": "CombinHorizon/Rombos-Qwen2.5-7B-Inst-BaseMerge-TIES", - "developer": "CombinHorizon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/CombinHorizon_Rombos-Qwen2.5-7B-Inst-BaseMerge-TIES/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7564 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5402 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4932 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2978 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4033 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4342 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/combinhorizon_yism-blossom5.1-34b-slerp.json b/data/models/combinhorizon_yism-blossom5.1-34b-slerp.json deleted file mode 100644 index b5fef4416fbae75715e834979e514a69096ba47a..0000000000000000000000000000000000000000 --- a/data/models/combinhorizon_yism-blossom5.1-34b-slerp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "YiSM-blossom5.1-34B-SLERP", - "id": "CombinHorizon/YiSM-blossom5.1-34B-SLERP", - "developer": "CombinHorizon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "34.389" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/CombinHorizon_YiSM-blossom5.1-34B-SLERP/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5033 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6208 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2153 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3557 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4413 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4741 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/combinhorizon_zetasepic-abliteratedv2-qwen2.5-32b-inst-basemerge-ties.json b/data/models/combinhorizon_zetasepic-abliteratedv2-qwen2.5-32b-inst-basemerge-ties.json deleted file mode 100644 index d12948f9dbbadead4d6ec07b8779363396b633b9..0000000000000000000000000000000000000000 --- a/data/models/combinhorizon_zetasepic-abliteratedv2-qwen2.5-32b-inst-basemerge-ties.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "zetasepic-abliteratedV2-Qwen2.5-32B-Inst-BaseMerge-TIES", - "id": "CombinHorizon/zetasepic-abliteratedV2-Qwen2.5-32B-Inst-BaseMerge-TIES", - "developer": "CombinHorizon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "32.764" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/CombinHorizon_zetasepic-abliteratedV2-Qwen2.5-32B-Inst-BaseMerge-TIES/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8328 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6955 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5853 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3674 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4314 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5685 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/contactdoctor_bio-medical-3b-cot-012025.json b/data/models/contactdoctor_bio-medical-3b-cot-012025.json deleted file mode 100644 index e3f34e3961853c8a47edc917c1d6ca3aa5f9c6bb..0000000000000000000000000000000000000000 --- a/data/models/contactdoctor_bio-medical-3b-cot-012025.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Bio-Medical-3B-CoT-012025", - "id": "ContactDoctor/Bio-Medical-3B-CoT-012025", - "developer": "ContactDoctor", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.085" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ContactDoctor_Bio-Medical-3B-CoT-012025/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3604 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4383 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2213 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3045 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3368 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2934 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/contactdoctor_bio-medical-llama-3-8b.json b/data/models/contactdoctor_bio-medical-llama-3-8b.json deleted file mode 100644 index cd8f5a209bb7e3bce78ddaea5eb6d61b3d97116b..0000000000000000000000000000000000000000 --- a/data/models/contactdoctor_bio-medical-llama-3-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Bio-Medical-Llama-3-8B", - "id": "ContactDoctor/Bio-Medical-Llama-3-8B", - "developer": "ContactDoctor", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "4.015" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ContactDoctor_Bio-Medical-Llama-3-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4422 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4863 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0672 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3339 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3514 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3648 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/contextualai_archangel_sft-dpo_llama13b.json b/data/models/contextualai_archangel_sft-dpo_llama13b.json deleted file mode 100644 index 831e84853aa41adf8cf12b1d4b4ccca5773a3543..0000000000000000000000000000000000000000 --- a/data/models/contextualai_archangel_sft-dpo_llama13b.json +++ /dev/null @@ -1,144 +0,0 @@ -{ - "model_info": { - "name": "ContextualAI/archangel_sft-dpo_llama13b", - "id": "ContextualAI/archangel_sft-dpo_llama13b", - "developer": "ContextualAI", - "additional_details": { - "model_type": "DPO" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/ContextualAI_archangel_sft-dpo_llama13b/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.54 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7123 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4298 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5649 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4401 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5656 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/contextualai_archangel_sft-dpo_llama30b.json b/data/models/contextualai_archangel_sft-dpo_llama30b.json deleted file mode 100644 index 6fdf5bbb763d917efcce7da06f1dd0bd8adf32cc..0000000000000000000000000000000000000000 --- a/data/models/contextualai_archangel_sft-dpo_llama30b.json +++ /dev/null @@ -1,144 +0,0 @@ -{ - "model_info": { - "name": "ContextualAI/archangel_sft-dpo_llama30b", - "id": "ContextualAI/archangel_sft-dpo_llama30b", - "developer": "ContextualAI", - "additional_details": { - "model_type": "DPO" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/ContextualAI_archangel_sft-dpo_llama30b/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5618 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6927 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4474 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6284 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4745 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5705 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/contextualai_archangel_sft-dpo_llama7b.json b/data/models/contextualai_archangel_sft-dpo_llama7b.json deleted file mode 100644 index 382c1e430e11ad8e9ce9fcae380891cb592495ed..0000000000000000000000000000000000000000 --- a/data/models/contextualai_archangel_sft-dpo_llama7b.json +++ /dev/null @@ -1,144 +0,0 @@ -{ - "model_info": { - "name": "ContextualAI/archangel_sft-dpo_llama7b", - "id": "ContextualAI/archangel_sft-dpo_llama7b", - "developer": "ContextualAI", - "additional_details": { - "model_type": "DPO" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/ContextualAI_archangel_sft-dpo_llama7b/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5304 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5782 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4452 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5203 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5658 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5544 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/contextualai_archangel_sft-dpo_pythia1-4b.json b/data/models/contextualai_archangel_sft-dpo_pythia1-4b.json deleted file mode 100644 index b999d281adf326a5699a049901e1cb16e0f36a16..0000000000000000000000000000000000000000 --- a/data/models/contextualai_archangel_sft-dpo_pythia1-4b.json +++ /dev/null @@ -1,144 +0,0 @@ -{ - "model_info": { - "name": "ContextualAI/archangel_sft-dpo_pythia1-4b", - "id": "ContextualAI/archangel_sft-dpo_pythia1-4b", - "developer": "ContextualAI", - "additional_details": { - "model_type": "DPO" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/ContextualAI_archangel_sft-dpo_pythia1-4b/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5233 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6397 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3728 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5041 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5672 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5427 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/contextualai_archangel_sft-dpo_pythia12-0b.json b/data/models/contextualai_archangel_sft-dpo_pythia12-0b.json deleted file mode 100644 index f6b7703dac80796d62f4b5acdd6b6d175663d21e..0000000000000000000000000000000000000000 --- a/data/models/contextualai_archangel_sft-dpo_pythia12-0b.json +++ /dev/null @@ -1,144 +0,0 @@ -{ - "model_info": { - "name": "ContextualAI/archangel_sft-dpo_pythia12-0b", - "id": "ContextualAI/archangel_sft-dpo_pythia12-0b", - "developer": "ContextualAI", - "additional_details": { - "model_type": "DPO" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/ContextualAI_archangel_sft-dpo_pythia12-0b/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5009 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6676 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.364 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5432 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4139 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5303 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/contextualai_archangel_sft-dpo_pythia2-8b.json b/data/models/contextualai_archangel_sft-dpo_pythia2-8b.json deleted file mode 100644 index 186213abae53d939b5382df8108c77688e961c7e..0000000000000000000000000000000000000000 --- a/data/models/contextualai_archangel_sft-dpo_pythia2-8b.json +++ /dev/null @@ -1,144 +0,0 @@ -{ - "model_info": { - "name": "ContextualAI/archangel_sft-dpo_pythia2-8b", - "id": "ContextualAI/archangel_sft-dpo_pythia2-8b", - "developer": "ContextualAI", - "additional_details": { - "model_type": "DPO" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/ContextualAI_archangel_sft-dpo_pythia2-8b/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5286 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8073 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3355 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4473 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5135 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5501 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/contextualai_archangel_sft-dpo_pythia6-9b.json b/data/models/contextualai_archangel_sft-dpo_pythia6-9b.json deleted file mode 100644 index 0625a1d13f49b4025c547f1996a280e362e022eb..0000000000000000000000000000000000000000 --- a/data/models/contextualai_archangel_sft-dpo_pythia6-9b.json +++ /dev/null @@ -1,144 +0,0 @@ -{ - "model_info": { - "name": "ContextualAI/archangel_sft-dpo_pythia6-9b", - "id": "ContextualAI/archangel_sft-dpo_pythia6-9b", - "developer": "ContextualAI", - "additional_details": { - "model_type": "DPO" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/ContextualAI_archangel_sft-dpo_pythia6-9b/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5263 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7486 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3421 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5176 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4847 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.551 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/contextualai_archangel_sft-kto_llama13b.json b/data/models/contextualai_archangel_sft-kto_llama13b.json deleted file mode 100644 index 64aa57bc19d9811262186314102f3e5d37c5935d..0000000000000000000000000000000000000000 --- a/data/models/contextualai_archangel_sft-kto_llama13b.json +++ /dev/null @@ -1,144 +0,0 @@ -{ - "model_info": { - "name": "ContextualAI/archangel_sft-kto_llama13b", - "id": "ContextualAI/archangel_sft-kto_llama13b", - "developer": "ContextualAI", - "additional_details": { - "model_type": "DPO" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/ContextualAI_archangel_sft-kto_llama13b/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5952 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8408 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3772 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4649 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7077 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.576 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/contextualai_archangel_sft-kto_llama30b.json b/data/models/contextualai_archangel_sft-kto_llama30b.json deleted file mode 100644 index 4766c0e2e9b127f471f166fa49303532225084aa..0000000000000000000000000000000000000000 --- a/data/models/contextualai_archangel_sft-kto_llama30b.json +++ /dev/null @@ -1,144 +0,0 @@ -{ - "model_info": { - "name": "ContextualAI/archangel_sft-kto_llama30b", - "id": "ContextualAI/archangel_sft-kto_llama30b", - "developer": "ContextualAI", - "additional_details": { - "model_type": "DPO" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/ContextualAI_archangel_sft-kto_llama30b/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5901 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8436 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4057 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6054 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5075 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5862 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/contextualai_archangel_sft-kto_llama7b.json b/data/models/contextualai_archangel_sft-kto_llama7b.json deleted file mode 100644 index 79f626a886e454b062b9f15a86efc8fc44fd749b..0000000000000000000000000000000000000000 --- a/data/models/contextualai_archangel_sft-kto_llama7b.json +++ /dev/null @@ -1,144 +0,0 @@ -{ - "model_info": { - "name": "ContextualAI/archangel_sft-kto_llama7b", - "id": "ContextualAI/archangel_sft-kto_llama7b", - "developer": "ContextualAI", - "additional_details": { - "model_type": "DPO" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/ContextualAI_archangel_sft-kto_llama7b/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5388 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5587 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4364 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4568 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6941 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5575 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/contextualai_archangel_sft-kto_pythia1-4b.json b/data/models/contextualai_archangel_sft-kto_pythia1-4b.json deleted file mode 100644 index b274d2f39ec980d65162c0420511c9831626d619..0000000000000000000000000000000000000000 --- a/data/models/contextualai_archangel_sft-kto_pythia1-4b.json +++ /dev/null @@ -1,144 +0,0 @@ -{ - "model_info": { - "name": "ContextualAI/archangel_sft-kto_pythia1-4b", - "id": "ContextualAI/archangel_sft-kto_pythia1-4b", - "developer": "ContextualAI", - "additional_details": { - "model_type": "DPO" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/ContextualAI_archangel_sft-kto_pythia1-4b/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5581 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6844 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3794 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5257 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6447 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5546 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/contextualai_archangel_sft-kto_pythia12-0b.json b/data/models/contextualai_archangel_sft-kto_pythia12-0b.json deleted file mode 100644 index 3aeb644f85c965cbd1a4b7246bb242ad21853701..0000000000000000000000000000000000000000 --- a/data/models/contextualai_archangel_sft-kto_pythia12-0b.json +++ /dev/null @@ -1,144 +0,0 @@ -{ - "model_info": { - "name": "ContextualAI/archangel_sft-kto_pythia12-0b", - "id": "ContextualAI/archangel_sft-kto_pythia12-0b", - "developer": "ContextualAI", - "additional_details": { - "model_type": "DPO" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/ContextualAI_archangel_sft-kto_pythia12-0b/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5053 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7486 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3618 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4757 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4127 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.55 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/contextualai_archangel_sft-kto_pythia2-8b.json b/data/models/contextualai_archangel_sft-kto_pythia2-8b.json deleted file mode 100644 index 7ec1d25003df21aae2a5738a3acbd97e0b91a4c3..0000000000000000000000000000000000000000 --- a/data/models/contextualai_archangel_sft-kto_pythia2-8b.json +++ /dev/null @@ -1,144 +0,0 @@ -{ - "model_info": { - "name": "ContextualAI/archangel_sft-kto_pythia2-8b", - "id": "ContextualAI/archangel_sft-kto_pythia2-8b", - "developer": "ContextualAI", - "additional_details": { - "model_type": "DPO" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/ContextualAI_archangel_sft-kto_pythia2-8b/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5497 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.757 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3421 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4743 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6216 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.557 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/contextualai_archangel_sft-kto_pythia6-9b.json b/data/models/contextualai_archangel_sft-kto_pythia6-9b.json deleted file mode 100644 index 206934a530858ae3d86ecca5451a63c05c610c4f..0000000000000000000000000000000000000000 --- a/data/models/contextualai_archangel_sft-kto_pythia6-9b.json +++ /dev/null @@ -1,144 +0,0 @@ -{ - "model_info": { - "name": "ContextualAI/archangel_sft-kto_pythia6-9b", - "id": "ContextualAI/archangel_sft-kto_pythia6-9b", - "developer": "ContextualAI", - "additional_details": { - "model_type": "DPO" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/ContextualAI_archangel_sft-kto_pythia6-9b/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5561 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7765 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3618 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5365 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5415 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5723 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/contextualai_lmunit-llama3.1-70b.json b/data/models/contextualai_lmunit-llama3.1-70b.json deleted file mode 100644 index 1ade2a0b44fd645db9c3bc2017fc82ace48154e4..0000000000000000000000000000000000000000 --- a/data/models/contextualai_lmunit-llama3.1-70b.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "ContextualAI/LMUnit-llama3.1-70b", - "id": "ContextualAI/LMUnit-llama3.1-70b", - "developer": "ContextualAI", - "additional_details": { - "model_type": "Generative RM" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/ContextualAI_LMUnit-llama3.1-70b/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8054 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8463 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4875 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7158 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9067 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9697 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9063 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/contextualai_lmunit-qwen2.5-72b.json b/data/models/contextualai_lmunit-qwen2.5-72b.json deleted file mode 100644 index 95219ba62af6e8110ded82d919cce65572d8cf8b..0000000000000000000000000000000000000000 --- a/data/models/contextualai_lmunit-qwen2.5-72b.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "ContextualAI/LMUnit-qwen2.5-72b", - "id": "ContextualAI/LMUnit-qwen2.5-72b", - "developer": "ContextualAI", - "additional_details": { - "model_type": "Generative RM" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/ContextualAI_LMUnit-qwen2.5-72b/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8208 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8716 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5437 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7268 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9133 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9677 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9014 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/coolspring_qwen2-0.5b-abyme-merge2.json b/data/models/coolspring_qwen2-0.5b-abyme-merge2.json deleted file mode 100644 index 8d95d3797e58d9cbde620a339386eed54d5f3bf2..0000000000000000000000000000000000000000 --- a/data/models/coolspring_qwen2-0.5b-abyme-merge2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2-0.5B-Abyme-merge2", - "id": "CoolSpring/Qwen2-0.5B-Abyme-merge2", - "developer": "CoolSpring", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/CoolSpring_Qwen2-0.5B-Abyme-merge2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2022 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2994 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0332 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2601 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3687 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1489 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/coolspring_qwen2-0.5b-abyme-merge3.json b/data/models/coolspring_qwen2-0.5b-abyme-merge3.json deleted file mode 100644 index 614250eece831149ddac5411a4e2dc229dab58d9..0000000000000000000000000000000000000000 --- a/data/models/coolspring_qwen2-0.5b-abyme-merge3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2-0.5B-Abyme-merge3", - "id": "CoolSpring/Qwen2-0.5B-Abyme-merge3", - "developer": "CoolSpring", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/CoolSpring_Qwen2-0.5B-Abyme-merge3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2386 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3003 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0317 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2643 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3501 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.15 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/coolspring_qwen2-0.5b-abyme.json b/data/models/coolspring_qwen2-0.5b-abyme.json deleted file mode 100644 index 444b7423c931f8158bd7ae1a7711cda78bda1d42..0000000000000000000000000000000000000000 --- a/data/models/coolspring_qwen2-0.5b-abyme.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2-0.5B-Abyme", - "id": "CoolSpring/Qwen2-0.5B-Abyme", - "developer": "CoolSpring", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/CoolSpring_Qwen2-0.5B-Abyme/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1915 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2862 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0295 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2534 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3542 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1333 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/corianas_llama-3-reactor.json b/data/models/corianas_llama-3-reactor.json deleted file mode 100644 index 0e211b71b15a0983bcfc3c28fc0ca5ca8240f976..0000000000000000000000000000000000000000 --- a/data/models/corianas_llama-3-reactor.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "llama-3-reactor", - "id": "Corianas/llama-3-reactor", - "developer": "Corianas", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "-1.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Corianas_llama-3-reactor/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.23 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4457 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0468 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2978 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3977 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2801 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/corianas_neural-mistral-7b.json b/data/models/corianas_neural-mistral-7b.json deleted file mode 100644 index 0b7b3db4ab0985fb9832c0e600f6956c4a554cd0..0000000000000000000000000000000000000000 --- a/data/models/corianas_neural-mistral-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Neural-Mistral-7B", - "id": "Corianas/Neural-Mistral-7B", - "developer": "Corianas", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Corianas_Neural-Mistral-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5489 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4428 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0189 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2836 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3873 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2738 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/corianas_quokka_2.7b.json b/data/models/corianas_quokka_2.7b.json deleted file mode 100644 index d6b9d2c1582f8401c1ef9edacd4fee66c3672ae3..0000000000000000000000000000000000000000 --- a/data/models/corianas_quokka_2.7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Quokka_2.7b", - "id": "Corianas/Quokka_2.7b", - "developer": "Corianas", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "GPT2LMHeadModel", - "params_billions": "2.786" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Corianas_Quokka_2.7b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1749 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3055 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0083 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2559 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3908 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1145 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cortexlm_btlm-7b-base-v0.2.json b/data/models/cortexlm_btlm-7b-base-v0.2.json deleted file mode 100644 index d326a17a32b4bd13fcbb243b33187850d58bf16c..0000000000000000000000000000000000000000 --- a/data/models/cortexlm_btlm-7b-base-v0.2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "btlm-7b-base-v0.2", - "id": "CortexLM/btlm-7b-base-v0.2", - "developer": "CortexLM", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "6.885" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/CortexLM_btlm-7b-base-v0.2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1483 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4006 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0151 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2534 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3846 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.235 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cpayne1303_cp2024-instruct.json b/data/models/cpayne1303_cp2024-instruct.json deleted file mode 100644 index cad2aa1bd3a918ff4d15b9dd099ca4ef549ebe02..0000000000000000000000000000000000000000 --- a/data/models/cpayne1303_cp2024-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "cp2024-instruct", - "id": "cpayne1303/cp2024-instruct", - "developer": "cpayne1303", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "0.031" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/cpayne1303_cp2024-instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1706 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2947 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2601 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3686 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1167 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cpayne1303_cp2024.json b/data/models/cpayne1303_cp2024.json deleted file mode 100644 index 3ed27bfa8bbb94f1169b2da86df3dd18a29dc8db..0000000000000000000000000000000000000000 --- a/data/models/cpayne1303_cp2024.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "cp2024", - "id": "cpayne1303/cp2024", - "developer": "cpayne1303", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "0.031" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/cpayne1303_cp2024/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1658 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2985 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0053 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2559 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3383 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1101 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cpayne1303_llama-43m-beta.json b/data/models/cpayne1303_llama-43m-beta.json deleted file mode 100644 index 0f0d3430f35b22a1aef8ac071591532d594d6d4d..0000000000000000000000000000000000000000 --- a/data/models/cpayne1303_llama-43m-beta.json +++ /dev/null @@ -1,275 +0,0 @@ -{ - "model_info": { - "name": "llama-43m-beta", - "id": "cpayne1303/llama-43m-beta", - "developer": "cpayne1303", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "0.043" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/cpayne1303_llama-43m-beta/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1949 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2965 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0045 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2685 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3885 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1111 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "hfopenllm_v2/cpayne1303_llama-43m-beta/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1916 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2977 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2685 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3872 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1132 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cpayne1303_smallcp2024.json b/data/models/cpayne1303_smallcp2024.json deleted file mode 100644 index 2a613d04ee36ebb39b3d75da6bc28f9c2f87e684..0000000000000000000000000000000000000000 --- a/data/models/cpayne1303_smallcp2024.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "smallcp2024", - "id": "cpayne1303/smallcp2024", - "developer": "cpayne1303", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "0.002" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/cpayne1303_smallcp2024/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1582 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3027 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0053 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2307 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3425 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1114 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cran-may_merge_model_20250308_2.json b/data/models/cran-may_merge_model_20250308_2.json deleted file mode 100644 index 226a56d901166b52651410207e7bfed3b3ba28fc..0000000000000000000000000000000000000000 --- a/data/models/cran-may_merge_model_20250308_2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "merge_model_20250308_2", - "id": "Cran-May/merge_model_20250308_2", - "developer": "Cran-May", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Cran-May_merge_model_20250308_2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5932 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6585 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4381 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3909 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4794 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.542 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cran-may_merge_model_20250308_3.json b/data/models/cran-may_merge_model_20250308_3.json deleted file mode 100644 index 1498261190ab6cbf590445bff4dd20dc7637e670..0000000000000000000000000000000000000000 --- a/data/models/cran-may_merge_model_20250308_3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "merge_model_20250308_3", - "id": "Cran-May/merge_model_20250308_3", - "developer": "Cran-May", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Cran-May_merge_model_20250308_3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6018 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6271 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2545 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3221 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.432 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4962 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cran-may_merge_model_20250308_4.json b/data/models/cran-may_merge_model_20250308_4.json deleted file mode 100644 index 1f7dcca272a8c8a56ede7f8973f17f2b146d541d..0000000000000000000000000000000000000000 --- a/data/models/cran-may_merge_model_20250308_4.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "merge_model_20250308_4", - "id": "Cran-May/merge_model_20250308_4", - "developer": "Cran-May", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Cran-May_merge_model_20250308_4/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.454 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6664 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4199 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3977 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4688 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5367 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cran-may_sce-2-24b.json b/data/models/cran-may_sce-2-24b.json deleted file mode 100644 index 4dbc419027f380688ead7767348ecf7f0378c830..0000000000000000000000000000000000000000 --- a/data/models/cran-may_sce-2-24b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SCE-2-24B", - "id": "Cran-May/SCE-2-24B", - "developer": "Cran-May", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "23.572" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Cran-May_SCE-2-24B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5866 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6265 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1896 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3372 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4528 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4612 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cran-may_sce-3-24b.json b/data/models/cran-may_sce-3-24b.json deleted file mode 100644 index f090620bbc88c412e2b8c4af449eafa452cfad33..0000000000000000000000000000000000000000 --- a/data/models/cran-may_sce-3-24b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SCE-3-24B", - "id": "Cran-May/SCE-3-24B", - "developer": "Cran-May", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "23.572" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Cran-May_SCE-3-24B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5465 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5973 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1881 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3465 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4435 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4647 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cran-may_t.e-8.1.json b/data/models/cran-may_t.e-8.1.json deleted file mode 100644 index 4aa6bf14352bd6858d61475c56f67581c7b62e10..0000000000000000000000000000000000000000 --- a/data/models/cran-may_t.e-8.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "T.E-8.1", - "id": "Cran-May/T.E-8.1", - "developer": "Cran-May", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Cran-May_T.E-8.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7077 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5582 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4456 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3129 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4505 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4432 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cran-may_tempmotacilla-cinerea-0308.json b/data/models/cran-may_tempmotacilla-cinerea-0308.json deleted file mode 100644 index 376513e82fa91fe3c7eaeb1ac022374592801aef..0000000000000000000000000000000000000000 --- a/data/models/cran-may_tempmotacilla-cinerea-0308.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "tempmotacilla-cinerea-0308", - "id": "Cran-May/tempmotacilla-cinerea-0308", - "developer": "Cran-May", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Cran-May_tempmotacilla-cinerea-0308/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8085 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6551 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5551 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3624 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4208 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.525 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/creitingameplays_llama-3.1-8b-r1-v0.1.json b/data/models/creitingameplays_llama-3.1-8b-r1-v0.1.json deleted file mode 100644 index 28f3281455e3e634133fec094d0c21cce5580b4d..0000000000000000000000000000000000000000 --- a/data/models/creitingameplays_llama-3.1-8b-r1-v0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.1-8B-R1-v0.1", - "id": "CreitinGameplays/Llama-3.1-8B-R1-v0.1", - "developer": "CreitinGameplays", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/CreitinGameplays_Llama-3.1-8B-R1-v0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3235 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3057 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1813 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2584 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3622 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1252 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/crestf411_mn-slush.json b/data/models/crestf411_mn-slush.json deleted file mode 100644 index 1dfb97299bac6bb0b2d764d27a05b561694fa00e..0000000000000000000000000000000000000000 --- a/data/models/crestf411_mn-slush.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MN-Slush", - "id": "crestf411/MN-Slush", - "developer": "crestf411", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/crestf411_MN-Slush/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4077 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.534 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1269 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3238 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3933 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3508 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cstr_llama3.1-8b-spaetzle-v90.json b/data/models/cstr_llama3.1-8b-spaetzle-v90.json deleted file mode 100644 index ad6098a67ab6641b61b2678b35513f59bd4571a0..0000000000000000000000000000000000000000 --- a/data/models/cstr_llama3.1-8b-spaetzle-v90.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "llama3.1-8b-spaetzle-v90", - "id": "cstr/llama3.1-8b-spaetzle-v90", - "developer": "cstr", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/cstr_llama3.1-8b-spaetzle-v90/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7356 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5303 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1495 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2827 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4134 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3731 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cultrix_qwen2.5-14b-broca.json b/data/models/cultrix_qwen2.5-14b-broca.json deleted file mode 100644 index 2527d128263e435e2f36a54b53145e7ad1847ee1..0000000000000000000000000000000000000000 --- a/data/models/cultrix_qwen2.5-14b-broca.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-14B-Broca", - "id": "CultriX/Qwen2.5-14B-Broca", - "developer": "CultriX", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/CultriX_Qwen2.5-14B-Broca/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5604 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6527 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.358 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3867 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4767 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5364 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cultrix_qwen2.5-14b-brocav3.json b/data/models/cultrix_qwen2.5-14b-brocav3.json deleted file mode 100644 index b529b28d4fc524823d8050de6a5ab2b3a585a72c..0000000000000000000000000000000000000000 --- a/data/models/cultrix_qwen2.5-14b-brocav3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-14B-Brocav3", - "id": "CultriX/Qwen2.5-14B-Brocav3", - "developer": "CultriX", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/CultriX_Qwen2.5-14B-Brocav3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6952 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6452 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3875 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3591 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4756 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5317 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cultrix_qwen2.5-14b-brocav6.json b/data/models/cultrix_qwen2.5-14b-brocav6.json deleted file mode 100644 index 12a3cca1c34597c7ac1a4b66203bbb8c19fed678..0000000000000000000000000000000000000000 --- a/data/models/cultrix_qwen2.5-14b-brocav6.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-14B-Brocav6", - "id": "CultriX/Qwen2.5-14B-Brocav6", - "developer": "CultriX", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/CultriX_Qwen2.5-14B-Brocav6/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6995 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6389 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3875 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3674 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4742 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5319 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cultrix_qwen2.5-14b-brocav7.json b/data/models/cultrix_qwen2.5-14b-brocav7.json deleted file mode 100644 index 56bca291f10d83fe783855b5d2cfdd6ab9c8d9f7..0000000000000000000000000000000000000000 --- a/data/models/cultrix_qwen2.5-14b-brocav7.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-14B-Brocav7", - "id": "CultriX/Qwen2.5-14B-Brocav7", - "developer": "CultriX", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/CultriX_Qwen2.5-14B-Brocav7/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6724 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6444 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3844 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3674 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4796 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5258 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cultrix_qwen2.5-14b-brocav9.json b/data/models/cultrix_qwen2.5-14b-brocav9.json deleted file mode 100644 index 9e62f179d6dbf5a5f7b9770c7c65e72f6e7c0332..0000000000000000000000000000000000000000 --- a/data/models/cultrix_qwen2.5-14b-brocav9.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-14B-BrocaV9", - "id": "CultriX/Qwen2.5-14B-BrocaV9", - "developer": "CultriX", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/CultriX_Qwen2.5-14B-BrocaV9/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6763 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6391 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3814 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3641 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.469 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5331 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cultrix_qwen2.5-14b-emerged.json b/data/models/cultrix_qwen2.5-14b-emerged.json deleted file mode 100644 index 1f30aa5b6b23d3db8d155ad737c1ef0051d111d5..0000000000000000000000000000000000000000 --- a/data/models/cultrix_qwen2.5-14b-emerged.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-14B-Emerged", - "id": "CultriX/Qwen2.5-14B-Emerged", - "developer": "CultriX", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/CultriX_Qwen2.5-14B-Emerged/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.626 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3248 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3574 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4691 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5186 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cultrix_qwen2.5-14b-emergedv3.json b/data/models/cultrix_qwen2.5-14b-emergedv3.json deleted file mode 100644 index a5d66bfbe649ee47c16f65b9503c24ba8954874e..0000000000000000000000000000000000000000 --- a/data/models/cultrix_qwen2.5-14b-emergedv3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-14B-Emergedv3", - "id": "CultriX/Qwen2.5-14B-Emergedv3", - "developer": "CultriX", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/CultriX_Qwen2.5-14B-Emergedv3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6388 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6191 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4358 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3607 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4728 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5174 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cultrix_qwen2.5-14b-finalmerge.json b/data/models/cultrix_qwen2.5-14b-finalmerge.json deleted file mode 100644 index 6e489cefb4ee4606313b4634188114b9e0773139..0000000000000000000000000000000000000000 --- a/data/models/cultrix_qwen2.5-14b-finalmerge.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-14B-FinalMerge", - "id": "CultriX/Qwen2.5-14B-FinalMerge", - "developer": "CultriX", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/CultriX_Qwen2.5-14B-FinalMerge/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4891 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5715 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3814 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3549 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4379 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4574 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cultrix_qwen2.5-14b-hyper.json b/data/models/cultrix_qwen2.5-14b-hyper.json deleted file mode 100644 index dbafa1dd6ba59d0d206cbee9b738542811bb10aa..0000000000000000000000000000000000000000 --- a/data/models/cultrix_qwen2.5-14b-hyper.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-14B-Hyper", - "id": "CultriX/Qwen2.5-14B-Hyper", - "developer": "CultriX", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/CultriX_Qwen2.5-14B-Hyper/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5391 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6507 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3437 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3918 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4898 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5374 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cultrix_qwen2.5-14b-hyperionv3.json b/data/models/cultrix_qwen2.5-14b-hyperionv3.json deleted file mode 100644 index 34a945d062af4d2f7950c4b07c7c7e0c24234cb6..0000000000000000000000000000000000000000 --- a/data/models/cultrix_qwen2.5-14b-hyperionv3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-14B-Hyperionv3", - "id": "CultriX/Qwen2.5-14B-Hyperionv3", - "developer": "CultriX", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/CultriX_Qwen2.5-14B-Hyperionv3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6836 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6522 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3701 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3708 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.473 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.534 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cultrix_qwen2.5-14b-hyperionv4.json b/data/models/cultrix_qwen2.5-14b-hyperionv4.json deleted file mode 100644 index bd475fe323b50ebb97839bf3ddc82a09e743301d..0000000000000000000000000000000000000000 --- a/data/models/cultrix_qwen2.5-14b-hyperionv4.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-14B-Hyperionv4", - "id": "CultriX/Qwen2.5-14B-Hyperionv4", - "developer": "CultriX", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/CultriX_Qwen2.5-14B-Hyperionv4/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5416 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6472 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3474 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3977 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4832 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5364 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cultrix_qwen2.5-14b-hyperionv5.json b/data/models/cultrix_qwen2.5-14b-hyperionv5.json deleted file mode 100644 index 963cc3115961f82598ab367e9b8be4c4f54a9fdc..0000000000000000000000000000000000000000 --- a/data/models/cultrix_qwen2.5-14b-hyperionv5.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-14B-Hyperionv5", - "id": "CultriX/Qwen2.5-14B-Hyperionv5", - "developer": "CultriX", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/CultriX_Qwen2.5-14B-Hyperionv5/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6729 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6443 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3822 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3716 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4795 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5302 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cultrix_qwen2.5-14b-hypermarck-dl.json b/data/models/cultrix_qwen2.5-14b-hypermarck-dl.json deleted file mode 100644 index 3a713ccceb246baba476ba9824cb0030f14e13ae..0000000000000000000000000000000000000000 --- a/data/models/cultrix_qwen2.5-14b-hypermarck-dl.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-14B-HyperMarck-dl", - "id": "CultriX/Qwen2.5-14B-HyperMarck-dl", - "developer": "CultriX", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/CultriX_Qwen2.5-14B-HyperMarck-dl/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.665 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6096 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5287 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3674 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4416 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5091 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cultrix_qwen2.5-14b-megamerge-pt2.json b/data/models/cultrix_qwen2.5-14b-megamerge-pt2.json deleted file mode 100644 index 5373eb9b86abc8c4b42a481342c77f3b62926337..0000000000000000000000000000000000000000 --- a/data/models/cultrix_qwen2.5-14b-megamerge-pt2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-14B-MegaMerge-pt2", - "id": "CultriX/Qwen2.5-14B-MegaMerge-pt2", - "developer": "CultriX", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/CultriX_Qwen2.5-14B-MegaMerge-pt2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5683 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6578 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3995 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3792 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4729 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5421 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cultrix_qwen2.5-14b-mergestock.json b/data/models/cultrix_qwen2.5-14b-mergestock.json deleted file mode 100644 index 24ce4dd4071eb678d08e0c85ca9a3a4e56ede174..0000000000000000000000000000000000000000 --- a/data/models/cultrix_qwen2.5-14b-mergestock.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-14B-MergeStock", - "id": "CultriX/Qwen2.5-14B-MergeStock", - "developer": "CultriX", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/CultriX_Qwen2.5-14B-MergeStock/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5685 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6579 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4147 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3733 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4676 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5396 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cultrix_qwen2.5-14b-partialmergept1.json b/data/models/cultrix_qwen2.5-14b-partialmergept1.json deleted file mode 100644 index b6ed65bd93a0332b6a3a9ca15f5b687498a09102..0000000000000000000000000000000000000000 --- a/data/models/cultrix_qwen2.5-14b-partialmergept1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-14B-partialmergept1", - "id": "CultriX/Qwen2.5-14B-partialmergept1", - "developer": "CultriX", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/CultriX_Qwen2.5-14B-partialmergept1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6337 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6151 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4539 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3616 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4757 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5208 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cultrix_qwen2.5-14b-reasoningmerge.json b/data/models/cultrix_qwen2.5-14b-reasoningmerge.json deleted file mode 100644 index 973074a1452346b0686f41973d82e40001ecc311..0000000000000000000000000000000000000000 --- a/data/models/cultrix_qwen2.5-14b-reasoningmerge.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-14B-ReasoningMerge", - "id": "CultriX/Qwen2.5-14B-ReasoningMerge", - "developer": "CultriX", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/CultriX_Qwen2.5-14B-ReasoningMerge/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4605 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6578 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5204 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4077 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5166 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5345 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cultrix_qwen2.5-14b-ultimav2.json b/data/models/cultrix_qwen2.5-14b-ultimav2.json deleted file mode 100644 index a3fd6d6ccb6de1ef7c3c1424a0223a62b4e70f8e..0000000000000000000000000000000000000000 --- a/data/models/cultrix_qwen2.5-14b-ultimav2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-14B-Ultimav2", - "id": "CultriX/Qwen2.5-14B-Ultimav2", - "developer": "CultriX", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/CultriX_Qwen2.5-14B-Ultimav2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.55 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6555 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3844 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3851 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4966 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5417 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cultrix_qwen2.5-14b-unity.json b/data/models/cultrix_qwen2.5-14b-unity.json deleted file mode 100644 index dcc6135d369e2c6c8105047bca718bf5962b73b4..0000000000000000000000000000000000000000 --- a/data/models/cultrix_qwen2.5-14b-unity.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-14B-Unity", - "id": "CultriX/Qwen2.5-14B-Unity", - "developer": "CultriX", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/CultriX_Qwen2.5-14B-Unity/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6739 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.602 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4313 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3473 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4679 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5076 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cultrix_qwen2.5-14b-wernicke-sft.json b/data/models/cultrix_qwen2.5-14b-wernicke-sft.json deleted file mode 100644 index d4402b2da1610f35e37c5b99afd087109073b2b6..0000000000000000000000000000000000000000 --- a/data/models/cultrix_qwen2.5-14b-wernicke-sft.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-14B-Wernicke-SFT", - "id": "CultriX/Qwen2.5-14B-Wernicke-SFT", - "developer": "CultriX", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/CultriX_Qwen2.5-14B-Wernicke-SFT/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4937 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6461 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3595 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.354 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.39 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.507 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cultrix_qwen2.5-14b-wernicke-slerp.json b/data/models/cultrix_qwen2.5-14b-wernicke-slerp.json deleted file mode 100644 index 85d1de978dbda92bddcda7fb3b68a3efd1d77f1a..0000000000000000000000000000000000000000 --- a/data/models/cultrix_qwen2.5-14b-wernicke-slerp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-14B-Wernicke-SLERP", - "id": "CultriX/Qwen2.5-14B-Wernicke-SLERP", - "developer": "CultriX", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.491" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/CultriX_Qwen2.5-14B-Wernicke-SLERP/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5589 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6441 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4486 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.344 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.414 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5094 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cultrix_qwen2.5-14b-wernicke.json b/data/models/cultrix_qwen2.5-14b-wernicke.json deleted file mode 100644 index 88a1caf1fad42502436e6d17c114c90b010088e7..0000000000000000000000000000000000000000 --- a/data/models/cultrix_qwen2.5-14b-wernicke.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-14B-Wernicke", - "id": "CultriX/Qwen2.5-14B-Wernicke", - "developer": "CultriX", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/CultriX_Qwen2.5-14B-Wernicke/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5235 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6568 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3814 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3935 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4689 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5424 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cultrix_qwen2.5-14b-wernickev3.json b/data/models/cultrix_qwen2.5-14b-wernickev3.json deleted file mode 100644 index 6f50f665a6fa20878d7b7eb4ca2ec3b537c93e95..0000000000000000000000000000000000000000 --- a/data/models/cultrix_qwen2.5-14b-wernickev3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-14B-Wernickev3", - "id": "CultriX/Qwen2.5-14B-Wernickev3", - "developer": "CultriX", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/CultriX_Qwen2.5-14B-Wernickev3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7048 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6184 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3542 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3624 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4717 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5151 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cultrix_qwenfinity-2.5-14b.json b/data/models/cultrix_qwenfinity-2.5-14b.json deleted file mode 100644 index 3944d96847ee4ccf5ae212133cc7ea183ed8982c..0000000000000000000000000000000000000000 --- a/data/models/cultrix_qwenfinity-2.5-14b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwenfinity-2.5-14B", - "id": "CultriX/Qwenfinity-2.5-14B", - "developer": "CultriX", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/CultriX_Qwenfinity-2.5-14B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4814 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5655 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4101 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.349 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4506 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4498 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cultrix_qwestion-14b.json b/data/models/cultrix_qwestion-14b.json deleted file mode 100644 index e1a7b2d404e163fbe908a425137c6d64ec4ed657..0000000000000000000000000000000000000000 --- a/data/models/cultrix_qwestion-14b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwestion-14B", - "id": "CultriX/Qwestion-14B", - "developer": "CultriX", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/CultriX_Qwestion-14B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6318 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.645 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3724 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3683 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4636 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5422 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cultrix_seqwence-14b-evolmerge.json b/data/models/cultrix_seqwence-14b-evolmerge.json deleted file mode 100644 index d074f95e3a971fd8ae70c53a19fea3036e02048e..0000000000000000000000000000000000000000 --- a/data/models/cultrix_seqwence-14b-evolmerge.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SeQwence-14B-EvolMerge", - "id": "CultriX/SeQwence-14B-EvolMerge", - "developer": "CultriX", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/CultriX_SeQwence-14B-EvolMerge/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5382 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6572 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3671 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3809 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4821 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5419 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cultrix_seqwence-14b-evolmergev1.json b/data/models/cultrix_seqwence-14b-evolmergev1.json deleted file mode 100644 index e5fa89ecb219ebadfb288a5dd44ff344cb6231a6..0000000000000000000000000000000000000000 --- a/data/models/cultrix_seqwence-14b-evolmergev1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SeQwence-14B-EvolMergev1", - "id": "CultriX/SeQwence-14B-EvolMergev1", - "developer": "CultriX", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/CultriX_SeQwence-14B-EvolMergev1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5555 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6546 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4215 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3767 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4623 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5393 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cultrix_seqwence-14b-v5.json b/data/models/cultrix_seqwence-14b-v5.json deleted file mode 100644 index e1b9692bdd11f6824e7bbef0c95563ba6512a46d..0000000000000000000000000000000000000000 --- a/data/models/cultrix_seqwence-14b-v5.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SeQwence-14B-v5", - "id": "CultriX/SeQwence-14B-v5", - "developer": "CultriX", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/CultriX_SeQwence-14B-v5/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.592 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6517 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3308 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.37 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4714 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5415 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cultrix_seqwence-14b.json b/data/models/cultrix_seqwence-14b.json deleted file mode 100644 index 91fea011e1a76bf92d42e47bb00fcbfe1596bb8f..0000000000000000000000000000000000000000 --- a/data/models/cultrix_seqwence-14b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SeQwence-14B", - "id": "CultriX/SeQwence-14B", - "developer": "CultriX", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/CultriX_SeQwence-14B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5352 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6506 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3535 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3607 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4666 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5419 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cultrix_seqwence-14bv1.json b/data/models/cultrix_seqwence-14bv1.json deleted file mode 100644 index 32743c53fa670110eaf3973ab8600f451c6a2367..0000000000000000000000000000000000000000 --- a/data/models/cultrix_seqwence-14bv1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SeQwence-14Bv1", - "id": "CultriX/SeQwence-14Bv1", - "developer": "CultriX", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/CultriX_SeQwence-14Bv1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6678 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6345 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.361 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3616 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4704 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.532 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cultrix_seqwence-14bv2.json b/data/models/cultrix_seqwence-14bv2.json deleted file mode 100644 index f26e10ad352d771fb3016ab95d3c46724c8d49ef..0000000000000000000000000000000000000000 --- a/data/models/cultrix_seqwence-14bv2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SeQwence-14Bv2", - "id": "CultriX/SeQwence-14Bv2", - "developer": "CultriX", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/CultriX_SeQwence-14Bv2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5786 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6305 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4758 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3607 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4601 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5334 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cultrix_seqwence-14bv3.json b/data/models/cultrix_seqwence-14bv3.json deleted file mode 100644 index 2047249658f2203196b3b531499860771787d8d1..0000000000000000000000000000000000000000 --- a/data/models/cultrix_seqwence-14bv3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SeQwence-14Bv3", - "id": "CultriX/SeQwence-14Bv3", - "developer": "CultriX", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/CultriX_SeQwence-14Bv3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5719 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6302 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4766 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3649 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4624 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5335 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cyberagent_calm3-22b-chat.json b/data/models/cyberagent_calm3-22b-chat.json deleted file mode 100644 index a46a454762f589463a36c839a4832e7272ba7014..0000000000000000000000000000000000000000 --- a/data/models/cyberagent_calm3-22b-chat.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "calm3-22b-chat", - "id": "cyberagent/calm3-22b-chat", - "developer": "cyberagent", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "22.543" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/cyberagent_calm3-22b-chat/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5091 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4992 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0695 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2768 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4553 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.295 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cyfragovpl_llama-pllum-8b-base.json b/data/models/cyfragovpl_llama-pllum-8b-base.json deleted file mode 100644 index d8958c1b0d7a09bb28bf92280f97bb81a833e5ce..0000000000000000000000000000000000000000 --- a/data/models/cyfragovpl_llama-pllum-8b-base.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-PLLuM-8B-base", - "id": "CYFRAGOVPL/Llama-PLLuM-8B-base", - "developer": "CYFRAGOVPL", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/CYFRAGOVPL_Llama-PLLuM-8B-base/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2899 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.432 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0363 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2852 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.397 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2757 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cyfragovpl_llama-pllum-8b-chat.json b/data/models/cyfragovpl_llama-pllum-8b-chat.json deleted file mode 100644 index de5f0e3ae954b302d991884ec22126a0954e9db5..0000000000000000000000000000000000000000 --- a/data/models/cyfragovpl_llama-pllum-8b-chat.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-PLLuM-8B-chat", - "id": "CYFRAGOVPL/Llama-PLLuM-8B-chat", - "developer": "CYFRAGOVPL", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/CYFRAGOVPL_Llama-PLLuM-8B-chat/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3515 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4077 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.034 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2643 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4199 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2719 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cyfragovpl_pllum-12b-base.json b/data/models/cyfragovpl_pllum-12b-base.json deleted file mode 100644 index 0127d1237432c4de48c62bbf231e7466b0001223..0000000000000000000000000000000000000000 --- a/data/models/cyfragovpl_pllum-12b-base.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "PLLuM-12B-base", - "id": "CYFRAGOVPL/PLLuM-12B-base", - "developer": "CYFRAGOVPL", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/CYFRAGOVPL_PLLuM-12B-base/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2821 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4391 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0287 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2903 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4142 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.274 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cyfragovpl_pllum-12b-chat.json b/data/models/cyfragovpl_pllum-12b-chat.json deleted file mode 100644 index f4ea2344cad3947150f2517c47cb33f029ae9a0d..0000000000000000000000000000000000000000 --- a/data/models/cyfragovpl_pllum-12b-chat.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "PLLuM-12B-chat", - "id": "CYFRAGOVPL/PLLuM-12B-chat", - "developer": "CYFRAGOVPL", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/CYFRAGOVPL_PLLuM-12B-chat/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3214 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4446 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0181 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2601 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4115 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2872 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cyfragovpl_pllum-12b-nc-base.json b/data/models/cyfragovpl_pllum-12b-nc-base.json deleted file mode 100644 index 6e583d91fa3d428fa228c4b010cc7ddb983b883a..0000000000000000000000000000000000000000 --- a/data/models/cyfragovpl_pllum-12b-nc-base.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "PLLuM-12B-nc-base", - "id": "CYFRAGOVPL/PLLuM-12B-nc-base", - "developer": "CYFRAGOVPL", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/CYFRAGOVPL_PLLuM-12B-nc-base/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2405 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4277 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0219 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2701 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3645 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2559 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/cyfragovpl_pllum-12b-nc-chat.json b/data/models/cyfragovpl_pllum-12b-nc-chat.json deleted file mode 100644 index a5ca56e3cb96f2d2fdcf3d9558ff44c255887cb6..0000000000000000000000000000000000000000 --- a/data/models/cyfragovpl_pllum-12b-nc-chat.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "PLLuM-12B-nc-chat", - "id": "CYFRAGOVPL/PLLuM-12B-nc-chat", - "developer": "CYFRAGOVPL", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/CYFRAGOVPL_PLLuM-12B-nc-chat/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2834 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4576 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0121 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2827 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4354 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2597 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/daemontatox_aetherdrake-sft.json b/data/models/daemontatox_aetherdrake-sft.json deleted file mode 100644 index 03f6082b3fe90ec0cdc5dd1856ac6a18106d4545..0000000000000000000000000000000000000000 --- a/data/models/daemontatox_aetherdrake-sft.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "AetherDrake-SFT", - "id": "Daemontatox/AetherDrake-SFT", - "developer": "Daemontatox", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Daemontatox_AetherDrake-SFT/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4813 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4872 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1511 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3205 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4088 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3499 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/daemontatox_aethersett.json b/data/models/daemontatox_aethersett.json deleted file mode 100644 index f5d49141ee901ecbe9da36aed14cb031e2f783b1..0000000000000000000000000000000000000000 --- a/data/models/daemontatox_aethersett.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "AetherSett", - "id": "Daemontatox/AetherSett", - "developer": "Daemontatox", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Daemontatox_AetherSett/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.537 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5452 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3973 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3079 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4603 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4279 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/daemontatox_aethertot.json b/data/models/daemontatox_aethertot.json deleted file mode 100644 index b20831e7b0add6cd1b35d7cacb70896114ddccd2..0000000000000000000000000000000000000000 --- a/data/models/daemontatox_aethertot.json +++ /dev/null @@ -1,275 +0,0 @@ -{ - "model_info": { - "name": "AetherTOT", - "id": "Daemontatox/AetherTOT", - "developer": "Daemontatox", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MllamaForConditionalGeneration", - "params_billions": "10.67" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Daemontatox_AetherTOT/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4383 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5034 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1443 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3238 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4052 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3778 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "hfopenllm_v2/Daemontatox_AetherTOT/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4398 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5066 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1488 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3238 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4079 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3804 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/daemontatox_aetheruncensored.json b/data/models/daemontatox_aetheruncensored.json deleted file mode 100644 index cb84563bc6386a4e0a9fe0bb9d6c8f4beebcd3bf..0000000000000000000000000000000000000000 --- a/data/models/daemontatox_aetheruncensored.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "AetherUncensored", - "id": "Daemontatox/AetherUncensored", - "developer": "Daemontatox", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Daemontatox_AetherUncensored/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4042 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4463 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.145 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2886 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3747 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.271 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/daemontatox_cogito-mis.json b/data/models/daemontatox_cogito-mis.json deleted file mode 100644 index cf3e371353fb2157cf50f0ad1808d465745da1be..0000000000000000000000000000000000000000 --- a/data/models/daemontatox_cogito-mis.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Cogito-MIS", - "id": "Daemontatox/Cogito-MIS", - "developer": "Daemontatox", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "23.572" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Daemontatox_Cogito-MIS/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1815 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.506 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0861 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2567 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3768 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1435 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/daemontatox_cogitodistil.json b/data/models/daemontatox_cogitodistil.json deleted file mode 100644 index 23036b1c2b9f47edd0fa775b885accb495017513..0000000000000000000000000000000000000000 --- a/data/models/daemontatox_cogitodistil.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "CogitoDistil", - "id": "Daemontatox/CogitoDistil", - "developer": "Daemontatox", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Daemontatox_CogitoDistil/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2776 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3677 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3927 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2592 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3755 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2625 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/daemontatox_cogitoz.json b/data/models/daemontatox_cogitoz.json deleted file mode 100644 index 6613b4f2e0248a05f9195d724ff035a4dda14fa7..0000000000000000000000000000000000000000 --- a/data/models/daemontatox_cogitoz.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "CogitoZ", - "id": "Daemontatox/CogitoZ", - "developer": "Daemontatox", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "32.764" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Daemontatox_CogitoZ/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3967 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6734 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5242 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3951 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4793 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5593 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/daemontatox_cogitoz14.json b/data/models/daemontatox_cogitoz14.json deleted file mode 100644 index 9bff3ee6615c0d1d08f8a28c5b106f9e10a8a5c3..0000000000000000000000000000000000000000 --- a/data/models/daemontatox_cogitoz14.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "CogitoZ14", - "id": "Daemontatox/CogitoZ14", - "developer": "Daemontatox", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Daemontatox_CogitoZ14/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6637 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6298 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4222 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3163 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4059 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3999 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/daemontatox_documentcogito.json b/data/models/daemontatox_documentcogito.json deleted file mode 100644 index 6383820e3f19a97d2a08bca3546e5cca52a154e3..0000000000000000000000000000000000000000 --- a/data/models/daemontatox_documentcogito.json +++ /dev/null @@ -1,275 +0,0 @@ -{ - "model_info": { - "name": "DocumentCogito", - "id": "Daemontatox/DocumentCogito", - "developer": "Daemontatox", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MllamaForConditionalGeneration", - "params_billions": "10.67" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Daemontatox_DocumentCogito/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.777 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5187 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2198 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2936 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3911 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3738 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "hfopenllm_v2/Daemontatox_DocumentCogito/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5064 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5112 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1631 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3163 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3973 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3802 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/daemontatox_llama3.3-70b-cognilink.json b/data/models/daemontatox_llama3.3-70b-cognilink.json deleted file mode 100644 index 09dff5d2261bc006db62d862cd4b1ed541cb0e39..0000000000000000000000000000000000000000 --- a/data/models/daemontatox_llama3.3-70b-cognilink.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama3.3-70B-CogniLink", - "id": "Daemontatox/Llama3.3-70B-CogniLink", - "developer": "Daemontatox", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "70.554" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Daemontatox_Llama3.3-70B-CogniLink/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6931 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6668 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4139 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4455 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4877 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5173 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/daemontatox_llama_cot.json b/data/models/daemontatox_llama_cot.json deleted file mode 100644 index e73a2f1debcd79bfa049447aa27a91226c84f4cd..0000000000000000000000000000000000000000 --- a/data/models/daemontatox_llama_cot.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama_cot", - "id": "Daemontatox/Llama_cot", - "developer": "Daemontatox", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MllamaForConditionalGeneration", - "params_billions": "10.67" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Daemontatox_Llama_cot/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7549 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4838 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2024 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2911 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3872 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3518 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/daemontatox_mawaredt1.json b/data/models/daemontatox_mawaredt1.json deleted file mode 100644 index 902154fb88e179dc7cbaccdc4b5282217b875864..0000000000000000000000000000000000000000 --- a/data/models/daemontatox_mawaredt1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MawaredT1", - "id": "Daemontatox/MawaredT1", - "developer": "Daemontatox", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Daemontatox_MawaredT1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4199 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5215 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3021 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3347 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4702 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4718 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/daemontatox_mini-cogito-r1.json b/data/models/daemontatox_mini-cogito-r1.json deleted file mode 100644 index 180a833eb69abbc6c7b7e5973487229b398da05f..0000000000000000000000000000000000000000 --- a/data/models/daemontatox_mini-cogito-r1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "mini-Cogito-R1", - "id": "Daemontatox/mini-Cogito-R1", - "developer": "Daemontatox", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.777" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Daemontatox_mini-Cogito-R1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2298 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.328 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2749 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2869 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3447 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1482 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/daemontatox_mini_pathfinder.json b/data/models/daemontatox_mini_pathfinder.json deleted file mode 100644 index e98a8b98874daeb8d6e52253e171b110f84b5328..0000000000000000000000000000000000000000 --- a/data/models/daemontatox_mini_pathfinder.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "mini_Pathfinder", - "id": "Daemontatox/mini_Pathfinder", - "developer": "Daemontatox", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Daemontatox_mini_Pathfinder/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2962 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3956 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4751 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2584 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3781 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2809 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/daemontatox_mini_qwq.json b/data/models/daemontatox_mini_qwq.json deleted file mode 100644 index 02550eb28d5a991622270e338ca3a02d74088739..0000000000000000000000000000000000000000 --- a/data/models/daemontatox_mini_qwq.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mini_QwQ", - "id": "Daemontatox/Mini_QwQ", - "developer": "Daemontatox", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Daemontatox_Mini_QwQ/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4497 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5549 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4192 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3037 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4682 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4373 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/daemontatox_nemor.json b/data/models/daemontatox_nemor.json deleted file mode 100644 index 6e7bb1b86e120ef93b987deb47fa65beeeec9a07..0000000000000000000000000000000000000000 --- a/data/models/daemontatox_nemor.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "NemoR", - "id": "Daemontatox/NemoR", - "developer": "Daemontatox", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "6.124" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Daemontatox_NemoR/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2287 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5194 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0831 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3272 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3908 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.329 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/daemontatox_pathfinderai.json b/data/models/daemontatox_pathfinderai.json deleted file mode 100644 index 7a5f7d25c7278e2df08548a48abfe0b0ee9b4f2a..0000000000000000000000000000000000000000 --- a/data/models/daemontatox_pathfinderai.json +++ /dev/null @@ -1,275 +0,0 @@ -{ - "model_info": { - "name": "PathfinderAI", - "id": "Daemontatox/PathfinderAI", - "developer": "Daemontatox", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "32.764" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Daemontatox_PathfinderAI/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4855 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6627 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4841 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3096 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4256 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5542 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "hfopenllm_v2/Daemontatox_PathfinderAI/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3745 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6668 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4758 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3943 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4858 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5593 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/daemontatox_pathfinderai2.0.json b/data/models/daemontatox_pathfinderai2.0.json deleted file mode 100644 index 9ad32ebf3be0264ad651627cbe8a0f6ba330fe23..0000000000000000000000000000000000000000 --- a/data/models/daemontatox_pathfinderai2.0.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "PathFinderAI2.0", - "id": "Daemontatox/PathFinderAI2.0", - "developer": "Daemontatox", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "32.764" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Daemontatox_PathFinderAI2.0/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4541 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6658 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5076 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.302 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4216 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5547 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/daemontatox_pathfinderai3.0.json b/data/models/daemontatox_pathfinderai3.0.json deleted file mode 100644 index 73710b43e77920ab8cccdf810b55f61ebd32869c..0000000000000000000000000000000000000000 --- a/data/models/daemontatox_pathfinderai3.0.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "PathFinderAi3.0", - "id": "Daemontatox/PathFinderAi3.0", - "developer": "Daemontatox", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "32.764" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Daemontatox_PathFinderAi3.0/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4271 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6884 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5045 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4086 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4807 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5757 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/daemontatox_phi-4-cot.json b/data/models/daemontatox_phi-4-cot.json deleted file mode 100644 index c628bbff55f801d4103c7c9548e2df570a530636..0000000000000000000000000000000000000000 --- a/data/models/daemontatox_phi-4-cot.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Phi-4-COT", - "id": "Daemontatox/Phi-4-COT", - "developer": "Daemontatox", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Daemontatox_Phi-4-COT/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1793 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6173 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2243 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3356 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.453 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5005 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/daemontatox_pixelparse_ai.json b/data/models/daemontatox_pixelparse_ai.json deleted file mode 100644 index 7031ba8649bc3272e15216ef938c011780732f93..0000000000000000000000000000000000000000 --- a/data/models/daemontatox_pixelparse_ai.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "PixelParse_AI", - "id": "Daemontatox/PixelParse_AI", - "developer": "Daemontatox", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MllamaForConditionalGeneration", - "params_billions": "10.67" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Daemontatox_PixelParse_AI/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4383 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5034 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1473 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3238 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4052 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3778 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/daemontatox_ra2.0.json b/data/models/daemontatox_ra2.0.json deleted file mode 100644 index ef055ceb7c3570b541a304033f4600694e0321fa..0000000000000000000000000000000000000000 --- a/data/models/daemontatox_ra2.0.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "RA2.0", - "id": "Daemontatox/RA2.0", - "developer": "Daemontatox", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Daemontatox_RA2.0/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3784 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4889 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3837 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3054 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4091 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2616 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/daemontatox_ra_reasoner.json b/data/models/daemontatox_ra_reasoner.json deleted file mode 100644 index c88b548e2f338061724e2df7f07f73540371dc09..0000000000000000000000000000000000000000 --- a/data/models/daemontatox_ra_reasoner.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "RA_Reasoner", - "id": "Daemontatox/RA_Reasoner", - "developer": "Daemontatox", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "10.306" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Daemontatox_RA_Reasoner/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5592 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6054 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2122 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3314 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3964 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.43 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/daemontatox_ra_reasoner2.0.json b/data/models/daemontatox_ra_reasoner2.0.json deleted file mode 100644 index 910634de3c7136b366b50ff47f6fdb729714ea65..0000000000000000000000000000000000000000 --- a/data/models/daemontatox_ra_reasoner2.0.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "RA_Reasoner2.0", - "id": "Daemontatox/RA_Reasoner2.0", - "developer": "Daemontatox", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "10.306" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Daemontatox_RA_Reasoner2.0/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5366 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6062 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2311 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3247 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3884 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4353 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/daemontatox_reasontest.json b/data/models/daemontatox_reasontest.json deleted file mode 100644 index 9256acce2ba7c7274b4e6ca86b7750f3b98d840d..0000000000000000000000000000000000000000 --- a/data/models/daemontatox_reasontest.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ReasonTest", - "id": "Daemontatox/ReasonTest", - "developer": "Daemontatox", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.808" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Daemontatox_ReasonTest/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.408 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5435 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2137 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3188 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4315 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4272 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/daemontatox_research_pathfinderai.json b/data/models/daemontatox_research_pathfinderai.json deleted file mode 100644 index 6c870da0e6ebd90277a6f86faf7b3e5241ba577c..0000000000000000000000000000000000000000 --- a/data/models/daemontatox_research_pathfinderai.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Research_PathfinderAI", - "id": "Daemontatox/Research_PathfinderAI", - "developer": "Daemontatox", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.777" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Daemontatox_Research_PathfinderAI/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3457 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2872 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1699 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2408 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3394 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.113 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/daemontatox_sphinx.json b/data/models/daemontatox_sphinx.json deleted file mode 100644 index 8a1601aa47d4c9f91ca7323e997d28fcbc8b9802..0000000000000000000000000000000000000000 --- a/data/models/daemontatox_sphinx.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SphinX", - "id": "Daemontatox/SphinX", - "developer": "Daemontatox", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Daemontatox_SphinX/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5725 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5441 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3082 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2978 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4405 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4366 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/daemontatox_sphinx2.0.json b/data/models/daemontatox_sphinx2.0.json deleted file mode 100644 index 5eec5479d5092faf38e3fa9e9d227e6f38c8dc05..0000000000000000000000000000000000000000 --- a/data/models/daemontatox_sphinx2.0.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Sphinx2.0", - "id": "Daemontatox/Sphinx2.0", - "developer": "Daemontatox", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Daemontatox_Sphinx2.0/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7123 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6473 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4018 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2936 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.426 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5184 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/daemontatox_tinysphinx.json b/data/models/daemontatox_tinysphinx.json deleted file mode 100644 index 255200da04ec90c29e7869eed8173e70b361f629..0000000000000000000000000000000000000000 --- a/data/models/daemontatox_tinysphinx.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "TinySphinx", - "id": "Daemontatox/TinySphinx", - "developer": "Daemontatox", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.247" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Daemontatox_TinySphinx/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2567 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.331 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0431 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2735 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3328 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1698 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/daemontatox_tinysphinx2.0.json b/data/models/daemontatox_tinysphinx2.0.json deleted file mode 100644 index bbd435e0c245a05d00a7ab6f84138a00c917cff4..0000000000000000000000000000000000000000 --- a/data/models/daemontatox_tinysphinx2.0.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "TinySphinx2.0", - "id": "Daemontatox/TinySphinx2.0", - "developer": "Daemontatox", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.247" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Daemontatox_TinySphinx2.0/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2535 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3168 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0325 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2685 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3382 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1731 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/daemontatox_zirel-7b-math.json b/data/models/daemontatox_zirel-7b-math.json deleted file mode 100644 index 5141f6457de8222dc155b5549012cdce5e661412..0000000000000000000000000000000000000000 --- a/data/models/daemontatox_zirel-7b-math.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Zirel-7B-Math", - "id": "Daemontatox/Zirel-7B-Math", - "developer": "Daemontatox", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Daemontatox_Zirel-7B-Math/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6639 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5448 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1979 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3263 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4789 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4237 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/daemontatox_zirel_1.5.json b/data/models/daemontatox_zirel_1.5.json deleted file mode 100644 index 1bd65f1e0702d16c581e9ac8c6bc6b670a4e096f..0000000000000000000000000000000000000000 --- a/data/models/daemontatox_zirel_1.5.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Zirel_1.5", - "id": "Daemontatox/Zirel_1.5", - "developer": "Daemontatox", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.544" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Daemontatox_Zirel_1.5/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4168 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3985 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1133 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2601 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3658 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2143 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dampfinchen_llama-3.1-8b-ultra-instruct.json b/data/models/dampfinchen_llama-3.1-8b-ultra-instruct.json deleted file mode 100644 index cad70ae0844e4149d064e1d39f229eb6def898a3..0000000000000000000000000000000000000000 --- a/data/models/dampfinchen_llama-3.1-8b-ultra-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.1-8B-Ultra-Instruct", - "id": "Dampfinchen/Llama-3.1-8B-Ultra-Instruct", - "developer": "Dampfinchen", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Dampfinchen_Llama-3.1-8B-Ultra-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8081 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5258 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2205 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2919 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4003 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3826 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/danielbrdz_barcenas-10b.json b/data/models/danielbrdz_barcenas-10b.json deleted file mode 100644 index b091f06ac9ced9664f146c1d01a3d70ab2fe88e8..0000000000000000000000000000000000000000 --- a/data/models/danielbrdz_barcenas-10b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Barcenas-10b", - "id": "Danielbrdz/Barcenas-10b", - "developer": "Danielbrdz", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "10.306" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Danielbrdz_Barcenas-10b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6608 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6121 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2153 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3414 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4135 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4361 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/danielbrdz_barcenas-14b-phi-3-medium-orpo.json b/data/models/danielbrdz_barcenas-14b-phi-3-medium-orpo.json deleted file mode 100644 index f16b28a20bbffb4501207089662b62dcf2d8d949..0000000000000000000000000000000000000000 --- a/data/models/danielbrdz_barcenas-14b-phi-3-medium-orpo.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Barcenas-14b-Phi-3-medium-ORPO", - "id": "Danielbrdz/Barcenas-14b-Phi-3-medium-ORPO", - "developer": "Danielbrdz", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "13.96" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Danielbrdz_Barcenas-14b-Phi-3-medium-ORPO/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4799 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6536 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2024 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3263 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4808 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4723 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/danielbrdz_barcenas-14b-phi-4-v2.json b/data/models/danielbrdz_barcenas-14b-phi-4-v2.json deleted file mode 100644 index 12d92847493cd20c8d910a4fdfb7746f2ce15398..0000000000000000000000000000000000000000 --- a/data/models/danielbrdz_barcenas-14b-phi-4-v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Barcenas-14b-phi-4-v2", - "id": "Danielbrdz/Barcenas-14b-phi-4-v2", - "developer": "Danielbrdz", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Danielbrdz_Barcenas-14b-phi-4-v2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2775 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6573 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3218 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3784 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4399 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5244 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/danielbrdz_barcenas-14b-phi-4.json b/data/models/danielbrdz_barcenas-14b-phi-4.json deleted file mode 100644 index bd4c40d7f4a34aa73d0817f576aa071880d2208f..0000000000000000000000000000000000000000 --- a/data/models/danielbrdz_barcenas-14b-phi-4.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Barcenas-14b-phi-4", - "id": "Danielbrdz/Barcenas-14b-phi-4", - "developer": "Danielbrdz", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Danielbrdz_Barcenas-14b-phi-4/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0498 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6769 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2583 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3834 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5097 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5175 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/danielbrdz_barcenas-3b-grpo.json b/data/models/danielbrdz_barcenas-3b-grpo.json deleted file mode 100644 index 1f2cfe9a462a59381af047b874745ae1da5f2a45..0000000000000000000000000000000000000000 --- a/data/models/danielbrdz_barcenas-3b-grpo.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Barcenas-3b-GRPO", - "id": "Danielbrdz/Barcenas-3b-GRPO", - "developer": "Danielbrdz", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Danielbrdz_Barcenas-3b-GRPO/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5444 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4414 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1375 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2903 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3576 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3037 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/danielbrdz_barcenas-llama3-8b-orpo.json b/data/models/danielbrdz_barcenas-llama3-8b-orpo.json deleted file mode 100644 index 75c6f77bfc241a594aca6f4eb2bdeed61aec4f71..0000000000000000000000000000000000000000 --- a/data/models/danielbrdz_barcenas-llama3-8b-orpo.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Barcenas-Llama3-8b-ORPO", - "id": "Danielbrdz/Barcenas-Llama3-8b-ORPO", - "developer": "Danielbrdz", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Danielbrdz_Barcenas-Llama3-8b-ORPO/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7372 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4987 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0657 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.307 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.419 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.383 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/danielbrdz_barcenas-r1-qwen-1.5b.json b/data/models/danielbrdz_barcenas-r1-qwen-1.5b.json deleted file mode 100644 index 8c5cbd781131eefab8509c64da757cbcf241e2ce..0000000000000000000000000000000000000000 --- a/data/models/danielbrdz_barcenas-r1-qwen-1.5b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Barcenas-R1-Qwen-1.5b", - "id": "Danielbrdz/Barcenas-R1-Qwen-1.5b", - "developer": "Danielbrdz", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.777" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Danielbrdz_Barcenas-R1-Qwen-1.5b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2428 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3587 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3497 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3037 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3541 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1909 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dans-discountmodels_12b-mn-dans-reasoning-test-2.json b/data/models/dans-discountmodels_12b-mn-dans-reasoning-test-2.json deleted file mode 100644 index b1cb929bcbf47bd8c7c431e5a2db79ec2ae8e756..0000000000000000000000000000000000000000 --- a/data/models/dans-discountmodels_12b-mn-dans-reasoning-test-2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "12b-mn-dans-reasoning-test-2", - "id": "Dans-DiscountModels/12b-mn-dans-reasoning-test-2", - "developer": "Dans-DiscountModels", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Dans-DiscountModels_12b-mn-dans-reasoning-test-2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3711 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4807 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0634 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2735 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3702 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2507 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dans-discountmodels_12b-mn-dans-reasoning-test-3.json b/data/models/dans-discountmodels_12b-mn-dans-reasoning-test-3.json deleted file mode 100644 index d2f847eaed78f1f7e8238a9dbcc48f08b272fce2..0000000000000000000000000000000000000000 --- a/data/models/dans-discountmodels_12b-mn-dans-reasoning-test-3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "12b-mn-dans-reasoning-test-3", - "id": "Dans-DiscountModels/12b-mn-dans-reasoning-test-3", - "developer": "Dans-DiscountModels", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Dans-DiscountModels_12b-mn-dans-reasoning-test-3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5053 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4839 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0778 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.271 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4168 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2516 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dans-discountmodels_dans-instruct-corecurriculum-12b-chatml.json b/data/models/dans-discountmodels_dans-instruct-corecurriculum-12b-chatml.json deleted file mode 100644 index aff933e4a42c4b416a145dde5b9162d7c5312d07..0000000000000000000000000000000000000000 --- a/data/models/dans-discountmodels_dans-instruct-corecurriculum-12b-chatml.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Dans-Instruct-CoreCurriculum-12b-ChatML", - "id": "Dans-DiscountModels/Dans-Instruct-CoreCurriculum-12b-ChatML", - "developer": "Dans-DiscountModels", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Dans-DiscountModels_Dans-Instruct-CoreCurriculum-12b-ChatML/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2111 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4792 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0431 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2802 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3606 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2805 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dans-discountmodels_dans-instruct-mix-8b-chatml-v0.1.0.json b/data/models/dans-discountmodels_dans-instruct-mix-8b-chatml-v0.1.0.json deleted file mode 100644 index 2b5f8ee97825fe8178b3fcab54fb42dbfe53ffad..0000000000000000000000000000000000000000 --- a/data/models/dans-discountmodels_dans-instruct-mix-8b-chatml-v0.1.0.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Dans-Instruct-Mix-8b-ChatML-V0.1.0", - "id": "Dans-DiscountModels/Dans-Instruct-Mix-8b-ChatML-V0.1.0", - "developer": "Dans-DiscountModels", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Dans-DiscountModels_Dans-Instruct-Mix-8b-ChatML-V0.1.0/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0668 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4775 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0672 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2861 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3786 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3284 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dans-discountmodels_dans-instruct-mix-8b-chatml-v0.1.1.json b/data/models/dans-discountmodels_dans-instruct-mix-8b-chatml-v0.1.1.json deleted file mode 100644 index db2b7d27cd28cdd8aa711b5692eb16169cc6d298..0000000000000000000000000000000000000000 --- a/data/models/dans-discountmodels_dans-instruct-mix-8b-chatml-v0.1.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Dans-Instruct-Mix-8b-ChatML-V0.1.1", - "id": "Dans-DiscountModels/Dans-Instruct-Mix-8b-ChatML-V0.1.1", - "developer": "Dans-DiscountModels", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Dans-DiscountModels_Dans-Instruct-Mix-8b-ChatML-V0.1.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0911 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4749 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0597 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2911 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3825 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3279 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dans-discountmodels_dans-instruct-mix-8b-chatml-v0.2.0.json b/data/models/dans-discountmodels_dans-instruct-mix-8b-chatml-v0.2.0.json deleted file mode 100644 index b35cd29fc0c0d89effe56b93e04744e8316576e9..0000000000000000000000000000000000000000 --- a/data/models/dans-discountmodels_dans-instruct-mix-8b-chatml-v0.2.0.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Dans-Instruct-Mix-8b-ChatML-V0.2.0", - "id": "Dans-DiscountModels/Dans-Instruct-Mix-8b-ChatML-V0.2.0", - "developer": "Dans-DiscountModels", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Dans-DiscountModels_Dans-Instruct-Mix-8b-ChatML-V0.2.0/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5064 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4624 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0733 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2936 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3644 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dans-discountmodels_dans-instruct-mix-8b-chatml.json b/data/models/dans-discountmodels_dans-instruct-mix-8b-chatml.json deleted file mode 100644 index 15261dcb0fd7405752f4152bc393bff28dc93ad6..0000000000000000000000000000000000000000 --- a/data/models/dans-discountmodels_dans-instruct-mix-8b-chatml.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Dans-Instruct-Mix-8b-ChatML", - "id": "Dans-DiscountModels/Dans-Instruct-Mix-8b-ChatML", - "developer": "Dans-DiscountModels", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Dans-DiscountModels_Dans-Instruct-Mix-8b-ChatML/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0825 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4738 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0551 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2945 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3918 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3288 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dans-discountmodels_mistral-7b-test-merged.json b/data/models/dans-discountmodels_mistral-7b-test-merged.json deleted file mode 100644 index 1c1df22f8cb4a3eb5cf6f688b8bdaf9e6a4e4dcf..0000000000000000000000000000000000000000 --- a/data/models/dans-discountmodels_mistral-7b-test-merged.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "mistral-7b-test-merged", - "id": "Dans-DiscountModels/mistral-7b-test-merged", - "developer": "Dans-DiscountModels", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Dans-DiscountModels_mistral-7b-test-merged/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6678 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4898 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0446 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2945 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3754 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2978 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dans-discountmodels_mistral-7b-v0.3-test-e0.7.json b/data/models/dans-discountmodels_mistral-7b-v0.3-test-e0.7.json deleted file mode 100644 index 70b2b16a42036bad0af078df6336eb688612ffe2..0000000000000000000000000000000000000000 --- a/data/models/dans-discountmodels_mistral-7b-v0.3-test-e0.7.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mistral-7b-v0.3-Test-E0.7", - "id": "Dans-DiscountModels/Mistral-7b-v0.3-Test-E0.7", - "developer": "Dans-DiscountModels", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Dans-DiscountModels_Mistral-7b-v0.3-Test-E0.7/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5124 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.475 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.034 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2961 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4005 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2744 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/darkc0de_buddyglass_v0.3_xortron7methedupswitchedup.json b/data/models/darkc0de_buddyglass_v0.3_xortron7methedupswitchedup.json deleted file mode 100644 index 40e51d95e7cc0f8f8fbb69f4e31fa3cdb5780aad..0000000000000000000000000000000000000000 --- a/data/models/darkc0de_buddyglass_v0.3_xortron7methedupswitchedup.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "BuddyGlass_v0.3_Xortron7MethedUpSwitchedUp", - "id": "darkc0de/BuddyGlass_v0.3_Xortron7MethedUpSwitchedUp", - "developer": "darkc0de", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "0.007" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/darkc0de_BuddyGlass_v0.3_Xortron7MethedUpSwitchedUp/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4358 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5243 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1284 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2987 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4143 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3673 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/darkc0de_buddyglassneversleeps.json b/data/models/darkc0de_buddyglassneversleeps.json deleted file mode 100644 index ba23d2a486ace898d3ca40f914c3b8fe607139e5..0000000000000000000000000000000000000000 --- a/data/models/darkc0de_buddyglassneversleeps.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "BuddyGlassNeverSleeps", - "id": "darkc0de/BuddyGlassNeverSleeps", - "developer": "darkc0de", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/darkc0de_BuddyGlassNeverSleeps/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4239 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4977 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0627 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2945 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3993 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3452 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/darkc0de_buddyglassuncensored2025.2.json b/data/models/darkc0de_buddyglassuncensored2025.2.json deleted file mode 100644 index 614cabb1104b7aab6126a347d77ae8d37eeee0ea..0000000000000000000000000000000000000000 --- a/data/models/darkc0de_buddyglassuncensored2025.2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "BuddyGlassUncensored2025.2", - "id": "darkc0de/BuddyGlassUncensored2025.2", - "developer": "darkc0de", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "10.306" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/darkc0de_BuddyGlassUncensored2025.2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7731 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6095 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2402 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.328 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4071 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4336 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/darkknight535_opencrystal-12b-l3.json b/data/models/darkknight535_opencrystal-12b-l3.json deleted file mode 100644 index 8a83bdbb5b3c24608c0f59cb60ced583fa7c684a..0000000000000000000000000000000000000000 --- a/data/models/darkknight535_opencrystal-12b-l3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "OpenCrystal-12B-L3", - "id": "Darkknight535/OpenCrystal-12B-L3", - "developer": "Darkknight535", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "11.52" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Darkknight535_OpenCrystal-12B-L3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4071 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5223 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0899 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3062 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3657 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.364 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/databricks-mosaic-research_pgrm.json b/data/models/databricks-mosaic-research_pgrm.json deleted file mode 100644 index ce9e2ab5c569b4479d0001ac5a3f609553a0b96c..0000000000000000000000000000000000000000 --- a/data/models/databricks-mosaic-research_pgrm.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "Databricks-Mosaic-Research/PGRM", - "id": "Databricks-Mosaic-Research/PGRM", - "developer": "Databricks-Mosaic-Research", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/Databricks-Mosaic-Research_PGRM/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8002 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7937 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5062 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7404 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9289 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9424 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8893 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/databricks_dbrx-base.json b/data/models/databricks_dbrx-base.json deleted file mode 100644 index b5d4a2d5f6f31ca4149ee639b4f0f24d13d9af7d..0000000000000000000000000000000000000000 --- a/data/models/databricks_dbrx-base.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "dbrx-base", - "id": "databricks/dbrx-base", - "developer": "databricks", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Unknown", - "params_billions": "0.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/databricks_dbrx-base/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0821 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5196 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3267 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4067 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.35 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/databricks_dbrx-instruct.json b/data/models/databricks_dbrx-instruct.json deleted file mode 100644 index 0af288512e1f46a3ec1d686fb5293dfb497c0ebb..0000000000000000000000000000000000000000 --- a/data/models/databricks_dbrx-instruct.json +++ /dev/null @@ -1,2031 +0,0 @@ -{ - "model_info": { - "name": "DBRX Instruct", - "id": "databricks/dbrx-instruct", - "developer": "databricks", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_lite/databricks_dbrx-instruct/1774096306.427425", - "retrieved_timestamp": "1774096306.427425", - "source_metadata": { - "source_name": "helm_lite", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_lite", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_lite", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.289, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.5229588014981273\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.488, - "details": { - "description": "min=0.488, mean=0.488, max=0.488, sum=0.488 (1)", - "tab": "Accuracy", - "NarrativeQA - Observed inference time (s)": "{\"description\": \"min=1.645, mean=1.645, max=1.645, sum=1.645 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.6445875322315056\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=3522.67, mean=3522.67, max=3522.67, sum=3522.67 (1)\", \"tab\": \"General information\", \"score\": \"3522.6704225352114\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (closed-book)", - "source_data": { - "dataset_name": "NaturalQuestions (closed-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (closed-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.284, - "details": { - "description": "min=0.284, mean=0.284, max=0.284, sum=0.284 (1)", - "tab": "Accuracy", - "NaturalQuestions (open-book) - Observed inference time (s)": "{\"description\": \"min=1.175, mean=1.175, max=1.175, sum=1.175 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.1746999933719635\"}", - "NaturalQuestions (closed-book) - Observed inference time (s)": "{\"description\": \"min=0.665, mean=0.665, max=0.665, sum=0.665 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.6648788969516755\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1762.593, mean=1762.593, max=1762.593, sum=1762.593 (1)\", \"tab\": \"General information\", \"score\": \"1762.593\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=173.127, mean=173.127, max=173.127, sum=173.127 (1)\", \"tab\": \"General information\", \"score\": \"173.127\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "mode": "\"closedbook\"" - } - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.91, - "details": { - "description": "min=0.91, mean=0.91, max=0.91, sum=0.91 (1)", - "tab": "Accuracy", - "OpenbookQA - Observed inference time (s)": "{\"description\": \"min=0.328, mean=0.328, max=0.328, sum=0.328 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.3277706532478333\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=242.782, mean=242.782, max=242.782, sum=242.782 (1)\", \"tab\": \"General information\", \"score\": \"242.782\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "dataset": "\"openbookqa\"", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.643, - "details": { - "description": "min=0.34, mean=0.643, max=0.93, sum=3.215 (5)", - "tab": "Accuracy", - "MMLU - Observed inference time (s)": "{\"description\": \"min=0.39, mean=0.412, max=0.432, sum=2.062 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.41247134314921857\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=366.44, mean=460.72, max=607.43, sum=2303.6 (5)\", \"tab\": \"General information\", \"score\": \"460.71996491228066\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"college_chemistry\", \"computer_security\", \"econometrics\", \"us_foreign_policy\"]", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MATH", - "source_data": { - "dataset_name": "MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Equivalent (CoT) on MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.358, - "details": { - "description": "min=0.015, mean=0.358, max=0.553, sum=2.509 (7)", - "tab": "Accuracy", - "MATH - Observed inference time (s)": "{\"description\": \"min=0.531, mean=2.305, max=3.852, sum=16.138 (7)\", \"tab\": \"Efficiency\", \"score\": \"2.305378989452493\"}", - "MATH - # eval": "{\"description\": \"min=30, mean=62.429, max=135, sum=437 (7)\", \"tab\": \"General information\", \"score\": \"62.42857142857143\"}", - "MATH - # train": "{\"description\": \"min=8, mean=8, max=8, sum=56 (7)\", \"tab\": \"General information\", \"score\": \"8.0\"}", - "MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MATH - # prompt tokens": "{\"description\": \"min=942.363, mean=1323.911, max=2258.577, sum=9267.376 (7)\", \"tab\": \"General information\", \"score\": \"1323.910874184069\"}", - "MATH - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=7 (7)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"algebra\", \"counting_and_probability\", \"geometry\", \"intermediate_algebra\", \"number_theory\", \"prealgebra\", \"precalculus\"]", - "level": "\"1\"", - "use_official_examples": "\"False\"", - "use_chain_of_thought": "\"True\"" - } - } - }, - { - "evaluation_name": "GSM8K", - "source_data": { - "dataset_name": "GSM8K", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on GSM8K", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.671, - "details": { - "description": "min=0.671, mean=0.671, max=0.671, sum=0.671 (1)", - "tab": "Accuracy", - "GSM8K - Observed inference time (s)": "{\"description\": \"min=2.384, mean=2.384, max=2.384, sum=2.384 (1)\", \"tab\": \"Efficiency\", \"score\": \"2.3839432048797606\"}", - "GSM8K - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "GSM8K - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "GSM8K - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GSM8K - # prompt tokens": "{\"description\": \"min=1020.035, mean=1020.035, max=1020.035, sum=1020.035 (1)\", \"tab\": \"General information\", \"score\": \"1020.035\"}", - "GSM8K - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "LegalBench", - "source_data": { - "dataset_name": "LegalBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on LegalBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.426, - "details": { - "description": "min=0.053, mean=0.426, max=0.755, sum=2.13 (5)", - "tab": "Accuracy", - "LegalBench - Observed inference time (s)": "{\"description\": \"min=0.366, mean=0.733, max=1.771, sum=3.667 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.73349196183029\"}", - "LegalBench - # eval": "{\"description\": \"min=95, mean=409.4, max=1000, sum=2047 (5)\", \"tab\": \"General information\", \"score\": \"409.4\"}", - "LegalBench - # train": "{\"description\": \"min=4, mean=4.8, max=5, sum=24 (5)\", \"tab\": \"General information\", \"score\": \"4.8\"}", - "LegalBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "LegalBench - # prompt tokens": "{\"description\": \"min=253.442, mean=1570.163, max=6357.388, sum=7850.815 (5)\", \"tab\": \"General information\", \"score\": \"1570.162971355988\"}", - "LegalBench - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"abercrombie\", \"corporate_lobbying\", \"function_of_decision_section\", \"international_citizenship_questions\", \"proa\"]" - } - } - }, - { - "evaluation_name": "MedQA", - "source_data": { - "dataset_name": "MedQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MedQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.694, - "details": { - "description": "min=0.694, mean=0.694, max=0.694, sum=0.694 (1)", - "tab": "Accuracy", - "MedQA - Observed inference time (s)": "{\"description\": \"min=0.438, mean=0.438, max=0.438, sum=0.438 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.4383622557221066\"}", - "MedQA - # eval": "{\"description\": \"min=503, mean=503, max=503, sum=503 (1)\", \"tab\": \"General information\", \"score\": \"503.0\"}", - "MedQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MedQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MedQA - # prompt tokens": "{\"description\": \"min=1020.414, mean=1020.414, max=1020.414, sum=1020.414 (1)\", \"tab\": \"General information\", \"score\": \"1020.4135188866799\"}", - "MedQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WMT 2014", - "source_data": { - "dataset_name": "WMT 2014", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "BLEU-4 on WMT 2014", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.131, - "details": { - "description": "min=0.035, mean=0.131, max=0.192, sum=0.656 (5)", - "tab": "Accuracy", - "WMT 2014 - Observed inference time (s)": "{\"description\": \"min=0.849, mean=1.059, max=1.342, sum=5.297 (5)\", \"tab\": \"Efficiency\", \"score\": \"1.0594140760888837\"}", - "WMT 2014 - # eval": "{\"description\": \"min=503, mean=568.8, max=832, sum=2844 (5)\", \"tab\": \"General information\", \"score\": \"568.8\"}", - "WMT 2014 - # train": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "WMT 2014 - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WMT 2014 - # prompt tokens": "{\"description\": \"min=169.901, mean=193.043, max=213.185, sum=965.213 (5)\", \"tab\": \"General information\", \"score\": \"193.04258583116683\"}", - "WMT 2014 - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "language_pair": "[\"cs-en\", \"de-en\", \"fr-en\", \"hi-en\", \"ru-en\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_mmlu/databricks_dbrx-instruct/1774096312.00548", - "retrieved_timestamp": "1774096312.00548", - "source_metadata": { - "source_name": "helm_mmlu", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_mmlu", - "evaluation_results": [ - { - "evaluation_name": "MMLU All Subjects", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU All Subjects", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.741, - "details": { - "description": "min=0.34, mean=0.741, max=0.953, sum=84.475 (114)", - "tab": "Accuracy", - "MMLU All Subjects - Observed inference time (s)": "{\"description\": \"min=0.356, mean=0.459, max=1.347, sum=52.272 (114)\", \"tab\": \"Efficiency\", \"score\": \"0.4585284510595002\"}", - "MMLU All Subjects - # eval": "{\"description\": \"min=100, mean=246.351, max=1534, sum=28084 (114)\", \"tab\": \"General information\", \"score\": \"246.35087719298247\"}", - "MMLU All Subjects - # train": "{\"description\": \"min=5, mean=5, max=5, sum=570 (114)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU All Subjects - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU All Subjects - # prompt tokens": "{\"description\": \"min=268.561, mean=607.852, max=2791.073, sum=69295.086 (114)\", \"tab\": \"General information\", \"score\": \"607.851634217556\"}", - "MMLU All Subjects - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=114 (114)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_name": "Abstract Algebra", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Abstract Algebra", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.34, - "details": { - "description": "min=0.34, mean=0.34, max=0.34, sum=0.68 (2)", - "tab": "Accuracy", - "Abstract Algebra - Observed inference time (s)": "{\"description\": \"min=0.432, mean=0.432, max=0.432, sum=0.863 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4316913342475891\"}", - "Abstract Algebra - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Abstract Algebra - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Abstract Algebra - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Abstract Algebra - # prompt tokens": "{\"description\": \"min=366.44, mean=366.44, max=366.44, sum=732.88 (2)\", \"tab\": \"General information\", \"score\": \"366.44\"}", - "Abstract Algebra - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"abstract_algebra\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_abstract_algebra\"" - } - } - }, - { - "evaluation_name": "Anatomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Anatomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.667, - "details": { - "description": "min=0.667, mean=0.667, max=0.667, sum=1.333 (2)", - "tab": "Accuracy", - "Anatomy - Observed inference time (s)": "{\"description\": \"min=0.385, mean=0.385, max=0.385, sum=0.771 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.38546188672383624\"}", - "Anatomy - # eval": "{\"description\": \"min=135, mean=135, max=135, sum=270 (2)\", \"tab\": \"General information\", \"score\": \"135.0\"}", - "Anatomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Anatomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Anatomy - # prompt tokens": "{\"description\": \"min=346.978, mean=346.978, max=346.978, sum=693.956 (2)\", \"tab\": \"General information\", \"score\": \"346.97777777777776\"}", - "Anatomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"anatomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_anatomy\"" - } - } - }, - { - "evaluation_name": "College Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on College Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.539, - "details": { - "description": "min=0.539, mean=0.539, max=0.539, sum=1.078 (2)", - "tab": "Accuracy", - "College Chemistry - Observed inference time (s)": "{\"description\": \"min=0.395, mean=0.395, max=0.395, sum=0.789 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.39454248666763303\"}", - "College Biology - Observed inference time (s)": "{\"description\": \"min=0.391, mean=0.391, max=0.391, sum=0.781 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3906625145011478\"}", - "College Computer Science - Observed inference time (s)": "{\"description\": \"min=0.439, mean=0.439, max=0.439, sum=0.877 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.438518271446228\"}", - "College Mathematics - Observed inference time (s)": "{\"description\": \"min=0.396, mean=0.396, max=0.396, sum=0.792 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3961342000961304\"}", - "College Medicine - Observed inference time (s)": "{\"description\": \"min=0.392, mean=0.392, max=0.392, sum=0.784 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.39187397708782573\"}", - "College Physics - Observed inference time (s)": "{\"description\": \"min=0.406, mean=0.406, max=0.406, sum=0.813 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4062807746962005\"}", - "College Chemistry - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Chemistry - # prompt tokens": "{\"description\": \"min=542.4, mean=542.4, max=542.4, sum=1084.8 (2)\", \"tab\": \"General information\", \"score\": \"542.4\"}", - "College Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Biology - # eval": "{\"description\": \"min=144, mean=144, max=144, sum=288 (2)\", \"tab\": \"General information\", \"score\": \"144.0\"}", - "College Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # prompt tokens": "{\"description\": \"min=466.917, mean=466.917, max=466.917, sum=933.833 (2)\", \"tab\": \"General information\", \"score\": \"466.9166666666667\"}", - "College Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # prompt tokens": "{\"description\": \"min=821.39, mean=821.39, max=821.39, sum=1642.78 (2)\", \"tab\": \"General information\", \"score\": \"821.39\"}", - "College Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Mathematics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # prompt tokens": "{\"description\": \"min=587.52, mean=587.52, max=587.52, sum=1175.04 (2)\", \"tab\": \"General information\", \"score\": \"587.52\"}", - "College Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Medicine - # eval": "{\"description\": \"min=173, mean=173, max=173, sum=346 (2)\", \"tab\": \"General information\", \"score\": \"173.0\"}", - "College Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # prompt tokens": "{\"description\": \"min=495.728, mean=495.728, max=495.728, sum=991.457 (2)\", \"tab\": \"General information\", \"score\": \"495.728323699422\"}", - "College Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Physics - # eval": "{\"description\": \"min=102, mean=102, max=102, sum=204 (2)\", \"tab\": \"General information\", \"score\": \"102.0\"}", - "College Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # prompt tokens": "{\"description\": \"min=496.608, mean=496.608, max=496.608, sum=993.216 (2)\", \"tab\": \"General information\", \"score\": \"496.6078431372549\"}", - "College Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"college_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_college_physics\"" - } - } - }, - { - "evaluation_name": "Computer Security", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Computer Security", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.83, - "details": { - "description": "min=0.83, mean=0.83, max=0.83, sum=1.66 (2)", - "tab": "Accuracy", - "Computer Security - Observed inference time (s)": "{\"description\": \"min=0.415, mean=0.415, max=0.415, sum=0.83 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4148012113571167\"}", - "Computer Security - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Computer Security - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Computer Security - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Computer Security - # prompt tokens": "{\"description\": \"min=371.54, mean=371.54, max=371.54, sum=743.08 (2)\", \"tab\": \"General information\", \"score\": \"371.54\"}", - "Computer Security - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"computer_security\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_computer_security\"" - } - } - }, - { - "evaluation_name": "Econometrics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Econometrics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.605, - "details": { - "description": "min=0.605, mean=0.605, max=0.605, sum=1.211 (2)", - "tab": "Accuracy", - "Econometrics - Observed inference time (s)": "{\"description\": \"min=0.431, mean=0.431, max=0.431, sum=0.863 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.43133983904855294\"}", - "Econometrics - # eval": "{\"description\": \"min=114, mean=114, max=114, sum=228 (2)\", \"tab\": \"General information\", \"score\": \"114.0\"}", - "Econometrics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Econometrics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Econometrics - # prompt tokens": "{\"description\": \"min=607.43, mean=607.43, max=607.43, sum=1214.86 (2)\", \"tab\": \"General information\", \"score\": \"607.4298245614035\"}", - "Econometrics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"econometrics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_econometrics\"" - } - } - }, - { - "evaluation_name": "Global Facts", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Global Facts", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.46, - "details": { - "description": "min=0.46, mean=0.46, max=0.46, sum=0.92 (2)", - "tab": "Accuracy", - "Global Facts - Observed inference time (s)": "{\"description\": \"min=0.429, mean=0.429, max=0.429, sum=0.857 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4286450815200806\"}", - "Global Facts - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Global Facts - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Global Facts - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Global Facts - # prompt tokens": "{\"description\": \"min=392.71, mean=392.71, max=392.71, sum=785.42 (2)\", \"tab\": \"General information\", \"score\": \"392.71\"}", - "Global Facts - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"global_facts\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_global_facts\"" - } - } - }, - { - "evaluation_name": "Jurisprudence", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Jurisprudence", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.843, - "details": { - "description": "min=0.843, mean=0.843, max=0.843, sum=1.685 (2)", - "tab": "Accuracy", - "Jurisprudence - Observed inference time (s)": "{\"description\": \"min=0.436, mean=0.436, max=0.436, sum=0.873 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.43625413488458703\"}", - "Jurisprudence - # eval": "{\"description\": \"min=108, mean=108, max=108, sum=216 (2)\", \"tab\": \"General information\", \"score\": \"108.0\"}", - "Jurisprudence - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Jurisprudence - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Jurisprudence - # prompt tokens": "{\"description\": \"min=387.639, mean=387.639, max=387.639, sum=775.278 (2)\", \"tab\": \"General information\", \"score\": \"387.6388888888889\"}", - "Jurisprudence - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"jurisprudence\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_jurisprudence\"" - } - } - }, - { - "evaluation_name": "Philosophy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Philosophy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.804, - "details": { - "description": "min=0.804, mean=0.804, max=0.804, sum=1.608 (2)", - "tab": "Accuracy", - "Philosophy - Observed inference time (s)": "{\"description\": \"min=0.408, mean=0.408, max=0.408, sum=0.816 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4079643100787589\"}", - "Philosophy - # eval": "{\"description\": \"min=311, mean=311, max=311, sum=622 (2)\", \"tab\": \"General information\", \"score\": \"311.0\"}", - "Philosophy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Philosophy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Philosophy - # prompt tokens": "{\"description\": \"min=322.084, mean=322.084, max=322.084, sum=644.167 (2)\", \"tab\": \"General information\", \"score\": \"322.08360128617363\"}", - "Philosophy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"philosophy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_philosophy\"" - } - } - }, - { - "evaluation_name": "Professional Psychology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Professional Psychology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.801, - "details": { - "description": "min=0.801, mean=0.801, max=0.801, sum=1.601 (2)", - "tab": "Accuracy", - "Professional Medicine - Observed inference time (s)": "{\"description\": \"min=0.47, mean=0.47, max=0.47, sum=0.94 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4699658164206673\"}", - "Professional Accounting - Observed inference time (s)": "{\"description\": \"min=0.395, mean=0.395, max=0.395, sum=0.791 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.39532034532398197\"}", - "Professional Law - Observed inference time (s)": "{\"description\": \"min=0.556, mean=0.556, max=0.556, sum=1.113 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5564531824579451\"}", - "Professional Psychology - Observed inference time (s)": "{\"description\": \"min=0.388, mean=0.388, max=0.388, sum=0.776 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3879917279567594\"}", - "Professional Medicine - # eval": "{\"description\": \"min=272, mean=272, max=272, sum=544 (2)\", \"tab\": \"General information\", \"score\": \"272.0\"}", - "Professional Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Medicine - # prompt tokens": "{\"description\": \"min=1087.585, mean=1087.585, max=1087.585, sum=2175.169 (2)\", \"tab\": \"General information\", \"score\": \"1087.5845588235295\"}", - "Professional Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Accounting - # eval": "{\"description\": \"min=282, mean=282, max=282, sum=564 (2)\", \"tab\": \"General information\", \"score\": \"282.0\"}", - "Professional Accounting - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Accounting - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # prompt tokens": "{\"description\": \"min=651.592, mean=651.592, max=651.592, sum=1303.184 (2)\", \"tab\": \"General information\", \"score\": \"651.5921985815603\"}", - "Professional Accounting - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Law - # eval": "{\"description\": \"min=1534, mean=1534, max=1534, sum=3068 (2)\", \"tab\": \"General information\", \"score\": \"1534.0\"}", - "Professional Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # prompt tokens": "{\"description\": \"min=1630.787, mean=1630.787, max=1630.787, sum=3261.574 (2)\", \"tab\": \"General information\", \"score\": \"1630.7868318122555\"}", - "Professional Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Psychology - # eval": "{\"description\": \"min=612, mean=612, max=612, sum=1224 (2)\", \"tab\": \"General information\", \"score\": \"612.0\"}", - "Professional Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # prompt tokens": "{\"description\": \"min=568.114, mean=568.114, max=568.114, sum=1136.229 (2)\", \"tab\": \"General information\", \"score\": \"568.1143790849674\"}", - "Professional Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"professional_psychology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_professional_psychology\"" - } - } - }, - { - "evaluation_name": "Us Foreign Policy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Us Foreign Policy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.93, - "details": { - "description": "min=0.93, mean=0.93, max=0.93, sum=1.86 (2)", - "tab": "Accuracy", - "Us Foreign Policy - Observed inference time (s)": "{\"description\": \"min=0.39, mean=0.39, max=0.39, sum=0.78 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3899818444252014\"}", - "Us Foreign Policy - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Us Foreign Policy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Us Foreign Policy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Us Foreign Policy - # prompt tokens": "{\"description\": \"min=415.79, mean=415.79, max=415.79, sum=831.58 (2)\", \"tab\": \"General information\", \"score\": \"415.79\"}", - "Us Foreign Policy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"us_foreign_policy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_us_foreign_policy\"" - } - } - }, - { - "evaluation_name": "Astronomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Astronomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.836, - "details": { - "description": "min=0.836, mean=0.836, max=0.836, sum=1.671 (2)", - "tab": "Accuracy", - "Astronomy - Observed inference time (s)": "{\"description\": \"min=0.428, mean=0.428, max=0.428, sum=0.857 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.42830287625915126\"}", - "Astronomy - # eval": "{\"description\": \"min=152, mean=152, max=152, sum=304 (2)\", \"tab\": \"General information\", \"score\": \"152.0\"}", - "Astronomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Astronomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Astronomy - # prompt tokens": "{\"description\": \"min=572.691, mean=572.691, max=572.691, sum=1145.382 (2)\", \"tab\": \"General information\", \"score\": \"572.6907894736842\"}", - "Astronomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"astronomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_astronomy\"" - } - } - }, - { - "evaluation_name": "Business Ethics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Business Ethics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.78, - "details": { - "description": "min=0.78, mean=0.78, max=0.78, sum=1.56 (2)", - "tab": "Accuracy", - "Business Ethics - Observed inference time (s)": "{\"description\": \"min=0.446, mean=0.446, max=0.446, sum=0.891 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.44572278976440427\"}", - "Business Ethics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Business Ethics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Business Ethics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Business Ethics - # prompt tokens": "{\"description\": \"min=562.52, mean=562.52, max=562.52, sum=1125.04 (2)\", \"tab\": \"General information\", \"score\": \"562.52\"}", - "Business Ethics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"business_ethics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_business_ethics\"" - } - } - }, - { - "evaluation_name": "Clinical Knowledge", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Clinical Knowledge", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.789, - "details": { - "description": "min=0.789, mean=0.789, max=0.789, sum=1.577 (2)", - "tab": "Accuracy", - "Clinical Knowledge - Observed inference time (s)": "{\"description\": \"min=0.404, mean=0.404, max=0.404, sum=0.807 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4037102978184538\"}", - "Clinical Knowledge - # eval": "{\"description\": \"min=265, mean=265, max=265, sum=530 (2)\", \"tab\": \"General information\", \"score\": \"265.0\"}", - "Clinical Knowledge - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Clinical Knowledge - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Clinical Knowledge - # prompt tokens": "{\"description\": \"min=390.947, mean=390.947, max=390.947, sum=781.894 (2)\", \"tab\": \"General information\", \"score\": \"390.94716981132075\"}", - "Clinical Knowledge - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"clinical_knowledge\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_clinical_knowledge\"" - } - } - }, - { - "evaluation_name": "Conceptual Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Conceptual Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.74, - "details": { - "description": "min=0.74, mean=0.74, max=0.74, sum=1.481 (2)", - "tab": "Accuracy", - "Conceptual Physics - Observed inference time (s)": "{\"description\": \"min=0.379, mean=0.379, max=0.379, sum=0.758 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3791612523667356\"}", - "Conceptual Physics - # eval": "{\"description\": \"min=235, mean=235, max=235, sum=470 (2)\", \"tab\": \"General information\", \"score\": \"235.0\"}", - "Conceptual Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Conceptual Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Conceptual Physics - # prompt tokens": "{\"description\": \"min=297.838, mean=297.838, max=297.838, sum=595.677 (2)\", \"tab\": \"General information\", \"score\": \"297.83829787234043\"}", - "Conceptual Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"conceptual_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_conceptual_physics\"" - } - } - }, - { - "evaluation_name": "Electrical Engineering", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Electrical Engineering", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.71, - "details": { - "description": "min=0.71, mean=0.71, max=0.71, sum=1.421 (2)", - "tab": "Accuracy", - "Electrical Engineering - Observed inference time (s)": "{\"description\": \"min=0.384, mean=0.384, max=0.384, sum=0.767 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3837302882095863\"}", - "Electrical Engineering - # eval": "{\"description\": \"min=145, mean=145, max=145, sum=290 (2)\", \"tab\": \"General information\", \"score\": \"145.0\"}", - "Electrical Engineering - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Electrical Engineering - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Electrical Engineering - # prompt tokens": "{\"description\": \"min=433.641, mean=433.641, max=433.641, sum=867.283 (2)\", \"tab\": \"General information\", \"score\": \"433.6413793103448\"}", - "Electrical Engineering - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"electrical_engineering\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_electrical_engineering\"" - } - } - }, - { - "evaluation_name": "Elementary Mathematics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Elementary Mathematics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.563, - "details": { - "description": "min=0.563, mean=0.563, max=0.563, sum=1.127 (2)", - "tab": "Accuracy", - "Elementary Mathematics - Observed inference time (s)": "{\"description\": \"min=0.392, mean=0.392, max=0.392, sum=0.783 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3916708092210154\"}", - "Elementary Mathematics - # eval": "{\"description\": \"min=378, mean=378, max=378, sum=756 (2)\", \"tab\": \"General information\", \"score\": \"378.0\"}", - "Elementary Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Elementary Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Elementary Mathematics - # prompt tokens": "{\"description\": \"min=524.862, mean=524.862, max=524.862, sum=1049.725 (2)\", \"tab\": \"General information\", \"score\": \"524.8624338624338\"}", - "Elementary Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"elementary_mathematics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_elementary_mathematics\"" - } - } - }, - { - "evaluation_name": "Formal Logic", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Formal Logic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.563, - "details": { - "description": "min=0.563, mean=0.563, max=0.563, sum=1.127 (2)", - "tab": "Accuracy", - "Formal Logic - Observed inference time (s)": "{\"description\": \"min=0.419, mean=0.419, max=0.419, sum=0.837 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.41872944339873297\"}", - "Formal Logic - # eval": "{\"description\": \"min=126, mean=126, max=126, sum=252 (2)\", \"tab\": \"General information\", \"score\": \"126.0\"}", - "Formal Logic - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Formal Logic - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Formal Logic - # prompt tokens": "{\"description\": \"min=599.762, mean=599.762, max=599.762, sum=1199.524 (2)\", \"tab\": \"General information\", \"score\": \"599.7619047619048\"}", - "Formal Logic - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"formal_logic\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_formal_logic\"" - } - } - }, - { - "evaluation_name": "High School World History", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on High School World History", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.903, - "details": { - "description": "min=0.903, mean=0.903, max=0.903, sum=1.806 (2)", - "tab": "Accuracy", - "High School Biology - Observed inference time (s)": "{\"description\": \"min=0.387, mean=0.387, max=0.387, sum=0.775 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3873311073549332\"}", - "High School Chemistry - Observed inference time (s)": "{\"description\": \"min=0.356, mean=0.356, max=0.356, sum=0.712 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.356056117071894\"}", - "High School Computer Science - Observed inference time (s)": "{\"description\": \"min=0.416, mean=0.416, max=0.416, sum=0.832 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4159617280960083\"}", - "High School European History - Observed inference time (s)": "{\"description\": \"min=0.784, mean=0.784, max=0.784, sum=1.569 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7843083367203221\"}", - "High School Geography - Observed inference time (s)": "{\"description\": \"min=0.573, mean=0.573, max=0.573, sum=1.146 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.573177902385442\"}", - "High School Government And Politics - Observed inference time (s)": "{\"description\": \"min=0.522, mean=0.522, max=0.522, sum=1.043 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5217143093366079\"}", - "High School Macroeconomics - Observed inference time (s)": "{\"description\": \"min=1.013, mean=1.013, max=1.013, sum=2.025 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.0127322582098155\"}", - "High School Mathematics - Observed inference time (s)": "{\"description\": \"min=1.347, mean=1.347, max=1.347, sum=2.694 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.346758367397167\"}", - "High School Microeconomics - Observed inference time (s)": "{\"description\": \"min=0.405, mean=0.405, max=0.405, sum=0.81 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.40513940819171296\"}", - "High School Physics - Observed inference time (s)": "{\"description\": \"min=0.457, mean=0.457, max=0.457, sum=0.915 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.45727316433230775\"}", - "High School Psychology - Observed inference time (s)": "{\"description\": \"min=0.427, mean=0.427, max=0.427, sum=0.855 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.42725621625917765\"}", - "High School Statistics - Observed inference time (s)": "{\"description\": \"min=0.465, mean=0.465, max=0.465, sum=0.93 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4648557923458241\"}", - "High School US History - Observed inference time (s)": "{\"description\": \"min=0.701, mean=0.701, max=0.701, sum=1.401 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7005175001481\"}", - "High School World History - Observed inference time (s)": "{\"description\": \"min=0.543, mean=0.543, max=0.543, sum=1.085 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5426257642512583\"}", - "High School Biology - # eval": "{\"description\": \"min=310, mean=310, max=310, sum=620 (2)\", \"tab\": \"General information\", \"score\": \"310.0\"}", - "High School Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Biology - # prompt tokens": "{\"description\": \"min=506.677, mean=506.677, max=506.677, sum=1013.355 (2)\", \"tab\": \"General information\", \"score\": \"506.6774193548387\"}", - "High School Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Chemistry - # eval": "{\"description\": \"min=203, mean=203, max=203, sum=406 (2)\", \"tab\": \"General information\", \"score\": \"203.0\"}", - "High School Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # prompt tokens": "{\"description\": \"min=489.714, mean=489.714, max=489.714, sum=979.429 (2)\", \"tab\": \"General information\", \"score\": \"489.7142857142857\"}", - "High School Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "High School Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # prompt tokens": "{\"description\": \"min=860.78, mean=860.78, max=860.78, sum=1721.56 (2)\", \"tab\": \"General information\", \"score\": \"860.78\"}", - "High School Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School European History - # eval": "{\"description\": \"min=165, mean=165, max=165, sum=330 (2)\", \"tab\": \"General information\", \"score\": \"165.0\"}", - "High School European History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School European History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # prompt tokens": "{\"description\": \"min=2791.073, mean=2791.073, max=2791.073, sum=5582.145 (2)\", \"tab\": \"General information\", \"score\": \"2791.072727272727\"}", - "High School European History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Geography - # eval": "{\"description\": \"min=198, mean=198, max=198, sum=396 (2)\", \"tab\": \"General information\", \"score\": \"198.0\"}", - "High School Geography - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Geography - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # prompt tokens": "{\"description\": \"min=365.045, mean=365.045, max=365.045, sum=730.091 (2)\", \"tab\": \"General information\", \"score\": \"365.04545454545456\"}", - "High School Geography - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Government And Politics - # eval": "{\"description\": \"min=193, mean=193, max=193, sum=386 (2)\", \"tab\": \"General information\", \"score\": \"193.0\"}", - "High School Government And Politics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Government And Politics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # prompt tokens": "{\"description\": \"min=458.824, mean=458.824, max=458.824, sum=917.648 (2)\", \"tab\": \"General information\", \"score\": \"458.8238341968912\"}", - "High School Government And Politics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Macroeconomics - # eval": "{\"description\": \"min=390, mean=390, max=390, sum=780 (2)\", \"tab\": \"General information\", \"score\": \"390.0\"}", - "High School Macroeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Macroeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # prompt tokens": "{\"description\": \"min=364.562, mean=364.562, max=364.562, sum=729.123 (2)\", \"tab\": \"General information\", \"score\": \"364.5615384615385\"}", - "High School Macroeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Mathematics - # eval": "{\"description\": \"min=270, mean=270, max=270, sum=540 (2)\", \"tab\": \"General information\", \"score\": \"270.0\"}", - "High School Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # prompt tokens": "{\"description\": \"min=525.374, mean=525.374, max=525.374, sum=1050.748 (2)\", \"tab\": \"General information\", \"score\": \"525.3740740740741\"}", - "High School Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Microeconomics - # eval": "{\"description\": \"min=238, mean=238, max=238, sum=476 (2)\", \"tab\": \"General information\", \"score\": \"238.0\"}", - "High School Microeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Microeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # prompt tokens": "{\"description\": \"min=392.025, mean=392.025, max=392.025, sum=784.05 (2)\", \"tab\": \"General information\", \"score\": \"392.02521008403363\"}", - "High School Microeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Physics - # eval": "{\"description\": \"min=151, mean=151, max=151, sum=302 (2)\", \"tab\": \"General information\", \"score\": \"151.0\"}", - "High School Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # prompt tokens": "{\"description\": \"min=553.464, mean=553.464, max=553.464, sum=1106.927 (2)\", \"tab\": \"General information\", \"score\": \"553.4635761589404\"}", - "High School Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Psychology - # eval": "{\"description\": \"min=545, mean=545, max=545, sum=1090 (2)\", \"tab\": \"General information\", \"score\": \"545.0\"}", - "High School Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # prompt tokens": "{\"description\": \"min=488.246, mean=488.246, max=488.246, sum=976.492 (2)\", \"tab\": \"General information\", \"score\": \"488.24587155963303\"}", - "High School Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Statistics - # eval": "{\"description\": \"min=216, mean=216, max=216, sum=432 (2)\", \"tab\": \"General information\", \"score\": \"216.0\"}", - "High School Statistics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Statistics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # prompt tokens": "{\"description\": \"min=788.699, mean=788.699, max=788.699, sum=1577.398 (2)\", \"tab\": \"General information\", \"score\": \"788.699074074074\"}", - "High School Statistics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School US History - # eval": "{\"description\": \"min=204, mean=204, max=204, sum=408 (2)\", \"tab\": \"General information\", \"score\": \"204.0\"}", - "High School US History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School US History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # prompt tokens": "{\"description\": \"min=2210.809, mean=2210.809, max=2210.809, sum=4421.618 (2)\", \"tab\": \"General information\", \"score\": \"2210.8088235294117\"}", - "High School US History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School World History - # eval": "{\"description\": \"min=237, mean=237, max=237, sum=474 (2)\", \"tab\": \"General information\", \"score\": \"237.0\"}", - "High School World History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School World History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # prompt tokens": "{\"description\": \"min=1421.27, mean=1421.27, max=1421.27, sum=2842.54 (2)\", \"tab\": \"General information\", \"score\": \"1421.2700421940929\"}", - "High School World History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"high_school_world_history\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_high_school_world_history\"" - } - } - }, - { - "evaluation_name": "Human Sexuality", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Human Sexuality", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.878, - "details": { - "description": "min=0.878, mean=0.878, max=0.878, sum=1.756 (2)", - "tab": "Accuracy", - "Human Aging - Observed inference time (s)": "{\"description\": \"min=0.409, mean=0.409, max=0.409, sum=0.819 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4093097753054358\"}", - "Human Sexuality - Observed inference time (s)": "{\"description\": \"min=0.41, mean=0.41, max=0.41, sum=0.819 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.40955095072738995\"}", - "Human Aging - # eval": "{\"description\": \"min=223, mean=223, max=223, sum=446 (2)\", \"tab\": \"General information\", \"score\": \"223.0\"}", - "Human Aging - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Aging - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Aging - # prompt tokens": "{\"description\": \"min=312.906, mean=312.906, max=312.906, sum=625.812 (2)\", \"tab\": \"General information\", \"score\": \"312.90582959641256\"}", - "Human Aging - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Human Sexuality - # eval": "{\"description\": \"min=131, mean=131, max=131, sum=262 (2)\", \"tab\": \"General information\", \"score\": \"131.0\"}", - "Human Sexuality - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Sexuality - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # prompt tokens": "{\"description\": \"min=334.183, mean=334.183, max=334.183, sum=668.366 (2)\", \"tab\": \"General information\", \"score\": \"334.1832061068702\"}", - "Human Sexuality - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"human_sexuality\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_human_sexuality\"" - } - } - }, - { - "evaluation_name": "International Law", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on International Law", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.884, - "details": { - "description": "min=0.884, mean=0.884, max=0.884, sum=1.769 (2)", - "tab": "Accuracy", - "International Law - Observed inference time (s)": "{\"description\": \"min=0.435, mean=0.435, max=0.435, sum=0.871 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.43540735284159005\"}", - "International Law - # eval": "{\"description\": \"min=121, mean=121, max=121, sum=242 (2)\", \"tab\": \"General information\", \"score\": \"121.0\"}", - "International Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "International Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "International Law - # prompt tokens": "{\"description\": \"min=632.851, mean=632.851, max=632.851, sum=1265.702 (2)\", \"tab\": \"General information\", \"score\": \"632.8512396694215\"}", - "International Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"international_law\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_international_law\"" - } - } - }, - { - "evaluation_name": "Logical Fallacies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Logical Fallacies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.847, - "details": { - "description": "min=0.847, mean=0.847, max=0.847, sum=1.693 (2)", - "tab": "Accuracy", - "Logical Fallacies - Observed inference time (s)": "{\"description\": \"min=0.418, mean=0.418, max=0.418, sum=0.836 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4178658789652257\"}", - "Logical Fallacies - # eval": "{\"description\": \"min=163, mean=163, max=163, sum=326 (2)\", \"tab\": \"General information\", \"score\": \"163.0\"}", - "Logical Fallacies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Logical Fallacies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Logical Fallacies - # prompt tokens": "{\"description\": \"min=442.595, mean=442.595, max=442.595, sum=885.19 (2)\", \"tab\": \"General information\", \"score\": \"442.5950920245399\"}", - "Logical Fallacies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"logical_fallacies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_logical_fallacies\"" - } - } - }, - { - "evaluation_name": "Machine Learning", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Machine Learning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.625, - "details": { - "description": "min=0.625, mean=0.625, max=0.625, sum=1.25 (2)", - "tab": "Accuracy", - "Machine Learning - Observed inference time (s)": "{\"description\": \"min=0.442, mean=0.442, max=0.442, sum=0.884 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.442230761051178\"}", - "Machine Learning - # eval": "{\"description\": \"min=112, mean=112, max=112, sum=224 (2)\", \"tab\": \"General information\", \"score\": \"112.0\"}", - "Machine Learning - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Machine Learning - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Machine Learning - # prompt tokens": "{\"description\": \"min=661.054, mean=661.054, max=661.054, sum=1322.107 (2)\", \"tab\": \"General information\", \"score\": \"661.0535714285714\"}", - "Machine Learning - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"machine_learning\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_machine_learning\"" - } - } - }, - { - "evaluation_name": "Management", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Management", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.854, - "details": { - "description": "min=0.854, mean=0.854, max=0.854, sum=1.709 (2)", - "tab": "Accuracy", - "Management - Observed inference time (s)": "{\"description\": \"min=0.42, mean=0.42, max=0.42, sum=0.84 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.42014194460748466\"}", - "Management - # eval": "{\"description\": \"min=103, mean=103, max=103, sum=206 (2)\", \"tab\": \"General information\", \"score\": \"103.0\"}", - "Management - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Management - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Management - # prompt tokens": "{\"description\": \"min=276.796, mean=276.796, max=276.796, sum=553.592 (2)\", \"tab\": \"General information\", \"score\": \"276.79611650485435\"}", - "Management - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"management\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_management\"" - } - } - }, - { - "evaluation_name": "Marketing", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Marketing", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.94, - "details": { - "description": "min=0.94, mean=0.94, max=0.94, sum=1.88 (2)", - "tab": "Accuracy", - "Marketing - Observed inference time (s)": "{\"description\": \"min=0.425, mean=0.425, max=0.425, sum=0.85 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4250037354281825\"}", - "Marketing - # eval": "{\"description\": \"min=234, mean=234, max=234, sum=468 (2)\", \"tab\": \"General information\", \"score\": \"234.0\"}", - "Marketing - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Marketing - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Marketing - # prompt tokens": "{\"description\": \"min=397.218, mean=397.218, max=397.218, sum=794.436 (2)\", \"tab\": \"General information\", \"score\": \"397.21794871794873\"}", - "Marketing - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"marketing\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_marketing\"" - } - } - }, - { - "evaluation_name": "Medical Genetics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Medical Genetics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.85, - "details": { - "description": "min=0.85, mean=0.85, max=0.85, sum=1.7 (2)", - "tab": "Accuracy", - "Medical Genetics - Observed inference time (s)": "{\"description\": \"min=0.423, mean=0.423, max=0.423, sum=0.846 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4227530717849731\"}", - "Medical Genetics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Medical Genetics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Medical Genetics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Medical Genetics - # prompt tokens": "{\"description\": \"min=334, mean=334, max=334, sum=668 (2)\", \"tab\": \"General information\", \"score\": \"334.0\"}", - "Medical Genetics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"medical_genetics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_medical_genetics\"" - } - } - }, - { - "evaluation_name": "Miscellaneous", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Miscellaneous", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.911, - "details": { - "description": "min=0.911, mean=0.911, max=0.911, sum=1.821 (2)", - "tab": "Accuracy", - "Miscellaneous - Observed inference time (s)": "{\"description\": \"min=0.367, mean=0.367, max=0.367, sum=0.734 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3670404892162649\"}", - "Miscellaneous - # eval": "{\"description\": \"min=783, mean=783, max=783, sum=1566 (2)\", \"tab\": \"General information\", \"score\": \"783.0\"}", - "Miscellaneous - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Miscellaneous - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Miscellaneous - # prompt tokens": "{\"description\": \"min=292.925, mean=292.925, max=292.925, sum=585.849 (2)\", \"tab\": \"General information\", \"score\": \"292.92464878671774\"}", - "Miscellaneous - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"miscellaneous\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_miscellaneous\"" - } - } - }, - { - "evaluation_name": "Moral Scenarios", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Moral Scenarios", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.465, - "details": { - "description": "min=0.465, mean=0.465, max=0.465, sum=0.93 (2)", - "tab": "Accuracy", - "Moral Disputes - Observed inference time (s)": "{\"description\": \"min=0.383, mean=0.383, max=0.383, sum=0.766 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3832114066691757\"}", - "Moral Scenarios - Observed inference time (s)": "{\"description\": \"min=0.4, mean=0.4, max=0.4, sum=0.801 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.400396443478888\"}", - "Moral Disputes - # eval": "{\"description\": \"min=346, mean=346, max=346, sum=692 (2)\", \"tab\": \"General information\", \"score\": \"346.0\"}", - "Moral Disputes - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Disputes - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Disputes - # prompt tokens": "{\"description\": \"min=469.145, mean=469.145, max=469.145, sum=938.289 (2)\", \"tab\": \"General information\", \"score\": \"469.1445086705202\"}", - "Moral Disputes - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Moral Scenarios - # eval": "{\"description\": \"min=895, mean=895, max=895, sum=1790 (2)\", \"tab\": \"General information\", \"score\": \"895.0\"}", - "Moral Scenarios - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Scenarios - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # prompt tokens": "{\"description\": \"min=649.455, mean=649.455, max=649.455, sum=1298.909 (2)\", \"tab\": \"General information\", \"score\": \"649.454748603352\"}", - "Moral Scenarios - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"moral_scenarios\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_moral_scenarios\"" - } - } - }, - { - "evaluation_name": "Nutrition", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Nutrition", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.814, - "details": { - "description": "min=0.814, mean=0.814, max=0.814, sum=1.627 (2)", - "tab": "Accuracy", - "Nutrition - Observed inference time (s)": "{\"description\": \"min=0.397, mean=0.397, max=0.397, sum=0.795 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.39725586947272806\"}", - "Nutrition - # eval": "{\"description\": \"min=306, mean=306, max=306, sum=612 (2)\", \"tab\": \"General information\", \"score\": \"306.0\"}", - "Nutrition - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Nutrition - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Nutrition - # prompt tokens": "{\"description\": \"min=579.817, mean=579.817, max=579.817, sum=1159.634 (2)\", \"tab\": \"General information\", \"score\": \"579.8169934640523\"}", - "Nutrition - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"nutrition\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_nutrition\"" - } - } - }, - { - "evaluation_name": "Prehistory", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Prehistory", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.84, - "details": { - "description": "min=0.84, mean=0.84, max=0.84, sum=1.679 (2)", - "tab": "Accuracy", - "Prehistory - Observed inference time (s)": "{\"description\": \"min=0.381, mean=0.381, max=0.381, sum=0.763 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3814176806697139\"}", - "Prehistory - # eval": "{\"description\": \"min=324, mean=324, max=324, sum=648 (2)\", \"tab\": \"General information\", \"score\": \"324.0\"}", - "Prehistory - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Prehistory - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Prehistory - # prompt tokens": "{\"description\": \"min=507.559, mean=507.559, max=507.559, sum=1015.117 (2)\", \"tab\": \"General information\", \"score\": \"507.55864197530866\"}", - "Prehistory - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"prehistory\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_prehistory\"" - } - } - }, - { - "evaluation_name": "Public Relations", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Public Relations", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.691, - "details": { - "description": "min=0.691, mean=0.691, max=0.691, sum=1.382 (2)", - "tab": "Accuracy", - "Public Relations - Observed inference time (s)": "{\"description\": \"min=0.391, mean=0.391, max=0.391, sum=0.782 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3911652868444269\"}", - "Public Relations - # eval": "{\"description\": \"min=110, mean=110, max=110, sum=220 (2)\", \"tab\": \"General information\", \"score\": \"110.0\"}", - "Public Relations - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Public Relations - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Public Relations - # prompt tokens": "{\"description\": \"min=398.318, mean=398.318, max=398.318, sum=796.636 (2)\", \"tab\": \"General information\", \"score\": \"398.3181818181818\"}", - "Public Relations - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"public_relations\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_public_relations\"" - } - } - }, - { - "evaluation_name": "Security Studies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Security Studies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.804, - "details": { - "description": "min=0.804, mean=0.804, max=0.804, sum=1.608 (2)", - "tab": "Accuracy", - "Security Studies - Observed inference time (s)": "{\"description\": \"min=0.464, mean=0.464, max=0.464, sum=0.928 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.46417581013270787\"}", - "Security Studies - # eval": "{\"description\": \"min=245, mean=245, max=245, sum=490 (2)\", \"tab\": \"General information\", \"score\": \"245.0\"}", - "Security Studies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Security Studies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Security Studies - # prompt tokens": "{\"description\": \"min=1157.473, mean=1157.473, max=1157.473, sum=2314.947 (2)\", \"tab\": \"General information\", \"score\": \"1157.4734693877551\"}", - "Security Studies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"security_studies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_security_studies\"" - } - } - }, - { - "evaluation_name": "Sociology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Sociology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.896, - "details": { - "description": "min=0.896, mean=0.896, max=0.896, sum=1.791 (2)", - "tab": "Accuracy", - "Sociology - Observed inference time (s)": "{\"description\": \"min=0.401, mean=0.401, max=0.401, sum=0.801 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4007088568673205\"}", - "Sociology - # eval": "{\"description\": \"min=201, mean=201, max=201, sum=402 (2)\", \"tab\": \"General information\", \"score\": \"201.0\"}", - "Sociology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Sociology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Sociology - # prompt tokens": "{\"description\": \"min=438.522, mean=438.522, max=438.522, sum=877.045 (2)\", \"tab\": \"General information\", \"score\": \"438.5223880597015\"}", - "Sociology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"sociology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_sociology\"" - } - } - }, - { - "evaluation_name": "Virology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Virology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.566, - "details": { - "description": "min=0.566, mean=0.566, max=0.566, sum=1.133 (2)", - "tab": "Accuracy", - "Virology - Observed inference time (s)": "{\"description\": \"min=0.386, mean=0.386, max=0.386, sum=0.771 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.38554139022367545\"}", - "Virology - # eval": "{\"description\": \"min=166, mean=166, max=166, sum=332 (2)\", \"tab\": \"General information\", \"score\": \"166.0\"}", - "Virology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Virology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Virology - # prompt tokens": "{\"description\": \"min=336.09, mean=336.09, max=336.09, sum=672.181 (2)\", \"tab\": \"General information\", \"score\": \"336.0903614457831\"}", - "Virology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"virology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_virology\"" - } - } - }, - { - "evaluation_name": "World Religions", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on World Religions", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.871, - "details": { - "description": "min=0.871, mean=0.871, max=0.871, sum=1.743 (2)", - "tab": "Accuracy", - "World Religions - Observed inference time (s)": "{\"description\": \"min=0.382, mean=0.382, max=0.382, sum=0.765 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3823263380262587\"}", - "World Religions - # eval": "{\"description\": \"min=171, mean=171, max=171, sum=342 (2)\", \"tab\": \"General information\", \"score\": \"171.0\"}", - "World Religions - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "World Religions - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "World Religions - # prompt tokens": "{\"description\": \"min=268.561, mean=268.561, max=268.561, sum=537.123 (2)\", \"tab\": \"General information\", \"score\": \"268.56140350877195\"}", - "World Religions - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"world_religions\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_world_religions\"" - } - } - }, - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.537, - "details": { - "description": "", - "tab": "Efficiency" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_id": "hfopenllm_v2/databricks_dbrx-instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5416 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5429 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0687 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3414 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4269 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3683 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/databricks_dolly-v1-6b.json b/data/models/databricks_dolly-v1-6b.json deleted file mode 100644 index 4b6152d369d49a3f28eb0dcb3312598aff2da5f9..0000000000000000000000000000000000000000 --- a/data/models/databricks_dolly-v1-6b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "dolly-v1-6b", - "id": "databricks/dolly-v1-6b", - "developer": "databricks", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "GPTJForCausalLM", - "params_billions": "6.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/databricks_dolly-v1-6b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2224 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3172 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0189 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2643 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4004 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1266 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/databricks_dolly-v2-12b.json b/data/models/databricks_dolly-v2-12b.json deleted file mode 100644 index b30366a98e0f9c06aa17d72f7abb2e8364040f03..0000000000000000000000000000000000000000 --- a/data/models/databricks_dolly-v2-12b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "dolly-v2-12b", - "id": "databricks/dolly-v2-12b", - "developer": "databricks", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "GPTNeoXForCausalLM", - "params_billions": "12.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/databricks_dolly-v2-12b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2355 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.332 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0136 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2408 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3739 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1129 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/databricks_dolly-v2-3b.json b/data/models/databricks_dolly-v2-3b.json deleted file mode 100644 index 9fd62443fe90a7f61d25fed3925e94242894f1fe..0000000000000000000000000000000000000000 --- a/data/models/databricks_dolly-v2-3b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "dolly-v2-3b", - "id": "databricks/dolly-v2-3b", - "developer": "databricks", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "GPTNeoXForCausalLM", - "params_billions": "3.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/databricks_dolly-v2-3b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2247 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3079 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0151 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2609 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3338 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1145 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/databricks_dolly-v2-7b.json b/data/models/databricks_dolly-v2-7b.json deleted file mode 100644 index 5cdaf36cc19e9bc5ab5252d1ec9eda5d2418c400..0000000000000000000000000000000000000000 --- a/data/models/databricks_dolly-v2-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "dolly-v2-7b", - "id": "databricks/dolly-v2-7b", - "developer": "databricks", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "GPTNeoXForCausalLM", - "params_billions": "7.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/databricks_dolly-v2-7b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.201 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3173 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0144 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2685 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3553 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1149 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/davidau_deephermes-3-llama-3-8b-preview-16.5b-brainstorm.json b/data/models/davidau_deephermes-3-llama-3-8b-preview-16.5b-brainstorm.json deleted file mode 100644 index 807fae9c74779734791e7b591b12c586dd31c6b5..0000000000000000000000000000000000000000 --- a/data/models/davidau_deephermes-3-llama-3-8b-preview-16.5b-brainstorm.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "DeepHermes-3-Llama-3-8B-Preview-16.5B-Brainstorm", - "id": "DavidAU/DeepHermes-3-Llama-3-8B-Preview-16.5B-Brainstorm", - "developer": "DavidAU", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "16.537" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DavidAU_DeepHermes-3-Llama-3-8B-Preview-16.5B-Brainstorm/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3136 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4762 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1057 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3138 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3928 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3209 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/davidau_deepseek-blackroot-r1-distill-llama-3.1-8b.json b/data/models/davidau_deepseek-blackroot-r1-distill-llama-3.1-8b.json deleted file mode 100644 index d41aeb4295ab54622f9be477e241961d5fd47447..0000000000000000000000000000000000000000 --- a/data/models/davidau_deepseek-blackroot-r1-distill-llama-3.1-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "DeepSeek-BlackRoot-R1-Distill-Llama-3.1-8B", - "id": "DavidAU/DeepSeek-BlackRoot-R1-Distill-Llama-3.1-8B", - "developer": "DavidAU", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DavidAU_DeepSeek-BlackRoot-R1-Distill-Llama-3.1-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3685 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4887 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0657 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.318 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.432 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2976 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/davidau_deepseek-grand-horror-smb-r1-distill-llama-3.1-16b.json b/data/models/davidau_deepseek-grand-horror-smb-r1-distill-llama-3.1-16b.json deleted file mode 100644 index 2431b3b05cfafe178357bf9a2492159e1069a5f4..0000000000000000000000000000000000000000 --- a/data/models/davidau_deepseek-grand-horror-smb-r1-distill-llama-3.1-16b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "DeepSeek-Grand-Horror-SMB-R1-Distill-Llama-3.1-16B", - "id": "DavidAU/DeepSeek-Grand-Horror-SMB-R1-Distill-Llama-3.1-16B", - "developer": "DavidAU", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "15.664" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DavidAU_DeepSeek-Grand-Horror-SMB-R1-Distill-Llama-3.1-16B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2507 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4488 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0295 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3138 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4164 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2709 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/davidau_deepseek-moe-4x8b-r1-distill-llama-3.1-deep-thinker-uncensored-24b.json b/data/models/davidau_deepseek-moe-4x8b-r1-distill-llama-3.1-deep-thinker-uncensored-24b.json deleted file mode 100644 index b9a9a091e049e8a584a8c6288446130e24847552..0000000000000000000000000000000000000000 --- a/data/models/davidau_deepseek-moe-4x8b-r1-distill-llama-3.1-deep-thinker-uncensored-24b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "DeepSeek-MOE-4X8B-R1-Distill-Llama-3.1-Deep-Thinker-Uncensored-24B", - "id": "DavidAU/DeepSeek-MOE-4X8B-R1-Distill-Llama-3.1-Deep-Thinker-Uncensored-24B", - "developer": "DavidAU", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MixtralForCausalLM", - "params_billions": "24.942" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DavidAU_DeepSeek-MOE-4X8B-R1-Distill-Llama-3.1-Deep-Thinker-Uncensored-24B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3883 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4886 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0816 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.323 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4375 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3024 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/davidau_deepseek-moe-4x8b-r1-distill-llama-3.1-mad-scientist-24b.json b/data/models/davidau_deepseek-moe-4x8b-r1-distill-llama-3.1-mad-scientist-24b.json deleted file mode 100644 index 27736d28844c2a6ba3a9a20b6b749db44b4bede7..0000000000000000000000000000000000000000 --- a/data/models/davidau_deepseek-moe-4x8b-r1-distill-llama-3.1-mad-scientist-24b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "DeepSeek-MOE-4X8B-R1-Distill-Llama-3.1-Mad-Scientist-24B", - "id": "DavidAU/DeepSeek-MOE-4X8B-R1-Distill-Llama-3.1-Mad-Scientist-24B", - "developer": "DavidAU", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MixtralForCausalLM", - "params_billions": "24.942" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DavidAU_DeepSeek-MOE-4X8B-R1-Distill-Llama-3.1-Mad-Scientist-24B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3436 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4769 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0755 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3372 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4231 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.297 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/davidau_deepseek-r1-distill-qwen-25.5b-brainstorm.json b/data/models/davidau_deepseek-r1-distill-qwen-25.5b-brainstorm.json deleted file mode 100644 index c942b9f3f5a73d949514fd86cec965b4e80aa9c2..0000000000000000000000000000000000000000 --- a/data/models/davidau_deepseek-r1-distill-qwen-25.5b-brainstorm.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "DeepSeek-R1-Distill-Qwen-25.5B-Brainstorm", - "id": "DavidAU/DeepSeek-R1-Distill-Qwen-25.5B-Brainstorm", - "developer": "DavidAU", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "25.506" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DavidAU_DeepSeek-R1-Distill-Qwen-25.5B-Brainstorm/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3416 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5807 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5536 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3859 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5155 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4624 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/davidau_deepseek-v2-grand-horror-smb-r1-distill-llama-3.1-uncensored-16.5b.json b/data/models/davidau_deepseek-v2-grand-horror-smb-r1-distill-llama-3.1-uncensored-16.5b.json deleted file mode 100644 index 39f9d49b35846883ba5dc24743efb4dea325a92b..0000000000000000000000000000000000000000 --- a/data/models/davidau_deepseek-v2-grand-horror-smb-r1-distill-llama-3.1-uncensored-16.5b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "DeepSeek-V2-Grand-Horror-SMB-R1-Distill-Llama-3.1-Uncensored-16.5B", - "id": "DavidAU/DeepSeek-V2-Grand-Horror-SMB-R1-Distill-Llama-3.1-Uncensored-16.5B", - "developer": "DavidAU", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "16.537" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DavidAU_DeepSeek-V2-Grand-Horror-SMB-R1-Distill-Llama-3.1-Uncensored-16.5B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2853 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4462 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0174 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3054 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4179 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2778 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/davidau_deepthought-moe-8x3b-r1-llama-3.2-reasoning-18b.json b/data/models/davidau_deepthought-moe-8x3b-r1-llama-3.2-reasoning-18b.json deleted file mode 100644 index 48100c516dd2d2805fe6baf6c9f87b60f1596c96..0000000000000000000000000000000000000000 --- a/data/models/davidau_deepthought-moe-8x3b-r1-llama-3.2-reasoning-18b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "DeepThought-MOE-8X3B-R1-Llama-3.2-Reasoning-18B", - "id": "DavidAU/DeepThought-MOE-8X3B-R1-Llama-3.2-Reasoning-18B", - "developer": "DavidAU", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MixtralForCausalLM", - "params_billions": "18.405" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DavidAU_DeepThought-MOE-8X3B-R1-Llama-3.2-Reasoning-18B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3793 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4232 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.108 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2794 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.356 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.272 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/davidau_gemma-the-writer-9b.json b/data/models/davidau_gemma-the-writer-9b.json deleted file mode 100644 index e7cf7c0e7c13f2165a50ca85e65c5f98bcff7082..0000000000000000000000000000000000000000 --- a/data/models/davidau_gemma-the-writer-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Gemma-The-Writer-9B", - "id": "DavidAU/Gemma-The-Writer-9B", - "developer": "DavidAU", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DavidAU_Gemma-The-Writer-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.174 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5905 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0876 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3456 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4099 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3979 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/davidau_gemma-the-writer-deadline-10b.json b/data/models/davidau_gemma-the-writer-deadline-10b.json deleted file mode 100644 index c0e0bc2127c691709037507ab189ace2e789e8c0..0000000000000000000000000000000000000000 --- a/data/models/davidau_gemma-the-writer-deadline-10b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Gemma-The-Writer-DEADLINE-10B", - "id": "DavidAU/Gemma-The-Writer-DEADLINE-10B", - "developer": "DavidAU", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.952" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DavidAU_Gemma-The-Writer-DEADLINE-10B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2332 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5896 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0989 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3423 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4189 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3946 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/davidau_gemma-the-writer-j.gutenberg-10b.json b/data/models/davidau_gemma-the-writer-j.gutenberg-10b.json deleted file mode 100644 index 6f97580f3fb787ee9902f5d51e8fd6f8993edeef..0000000000000000000000000000000000000000 --- a/data/models/davidau_gemma-the-writer-j.gutenberg-10b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Gemma-The-Writer-J.GutenBerg-10B", - "id": "DavidAU/Gemma-The-Writer-J.GutenBerg-10B", - "developer": "DavidAU", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.034" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DavidAU_Gemma-The-Writer-J.GutenBerg-10B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2858 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5909 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0921 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3381 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4176 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3947 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/davidau_gemma-the-writer-mighty-sword-9b.json b/data/models/davidau_gemma-the-writer-mighty-sword-9b.json deleted file mode 100644 index 2ce45328dbb30535762b21cf83608c63f70237d2..0000000000000000000000000000000000000000 --- a/data/models/davidau_gemma-the-writer-mighty-sword-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Gemma-The-Writer-Mighty-Sword-9B", - "id": "DavidAU/Gemma-The-Writer-Mighty-Sword-9B", - "developer": "DavidAU", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DavidAU_Gemma-The-Writer-Mighty-Sword-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7528 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5912 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1911 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3482 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4112 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3968 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/davidau_gemma-the-writer-n-restless-quill-10b-uncensored.json b/data/models/davidau_gemma-the-writer-n-restless-quill-10b-uncensored.json deleted file mode 100644 index fbc19672450eb560131417aebd109e813a6936a6..0000000000000000000000000000000000000000 --- a/data/models/davidau_gemma-the-writer-n-restless-quill-10b-uncensored.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Gemma-The-Writer-N-Restless-Quill-10B-Uncensored", - "id": "DavidAU/Gemma-The-Writer-N-Restless-Quill-10B-Uncensored", - "developer": "DavidAU", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.034" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DavidAU_Gemma-The-Writer-N-Restless-Quill-10B-Uncensored/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7071 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5922 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2296 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3414 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4163 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3966 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/davidau_l3-dark-planet-8b.json b/data/models/davidau_l3-dark-planet-8b.json deleted file mode 100644 index f87d38fb7e6f7dcb377049fe68a10ec8f1254010..0000000000000000000000000000000000000000 --- a/data/models/davidau_l3-dark-planet-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "L3-Dark-Planet-8B", - "id": "DavidAU/L3-Dark-Planet-8B", - "developer": "DavidAU", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DavidAU_L3-Dark-Planet-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4134 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5084 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0823 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3003 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3616 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3737 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/davidau_l3-darkest-planet-16.5b.json b/data/models/davidau_l3-darkest-planet-16.5b.json deleted file mode 100644 index 932dea52a940cc82fec000f4edc9e2526a1323ba..0000000000000000000000000000000000000000 --- a/data/models/davidau_l3-darkest-planet-16.5b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "L3-DARKEST-PLANET-16.5B", - "id": "DavidAU/L3-DARKEST-PLANET-16.5B", - "developer": "DavidAU", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "16.537" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DavidAU_L3-DARKEST-PLANET-16.5B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6231 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.523 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0899 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2953 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3754 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.363 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/davidau_l3-jamet-12.2b-mk.v-blackroot-instruct.json b/data/models/davidau_l3-jamet-12.2b-mk.v-blackroot-instruct.json deleted file mode 100644 index fd3e725ccc0bad24da2e991f2261159f1a25e794..0000000000000000000000000000000000000000 --- a/data/models/davidau_l3-jamet-12.2b-mk.v-blackroot-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "L3-Jamet-12.2B-MK.V-Blackroot-Instruct", - "id": "DavidAU/L3-Jamet-12.2B-MK.V-Blackroot-Instruct", - "developer": "DavidAU", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "12.174" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DavidAU_L3-Jamet-12.2B-MK.V-Blackroot-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3962 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4766 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0408 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2785 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.402 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3291 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/davidau_l3-lumimaid-12.2b-v0.1-oas-instruct.json b/data/models/davidau_l3-lumimaid-12.2b-v0.1-oas-instruct.json deleted file mode 100644 index 777e58371670729d02a9d4b47677026e17131c39..0000000000000000000000000000000000000000 --- a/data/models/davidau_l3-lumimaid-12.2b-v0.1-oas-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "L3-Lumimaid-12.2B-v0.1-OAS-Instruct", - "id": "DavidAU/L3-Lumimaid-12.2B-v0.1-OAS-Instruct", - "developer": "DavidAU", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "12.174" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DavidAU_L3-Lumimaid-12.2B-v0.1-OAS-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3924 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4693 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0461 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2768 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4194 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3142 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/davidau_l3-smb-instruct-12.2b-f32.json b/data/models/davidau_l3-smb-instruct-12.2b-f32.json deleted file mode 100644 index f143e41ced4dbbf40abd402f8efe35078075b4f2..0000000000000000000000000000000000000000 --- a/data/models/davidau_l3-smb-instruct-12.2b-f32.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "L3-SMB-Instruct-12.2B-F32", - "id": "DavidAU/L3-SMB-Instruct-12.2B-F32", - "developer": "DavidAU", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "12.174" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DavidAU_L3-SMB-Instruct-12.2B-F32/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4303 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4786 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0468 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2819 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4087 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3312 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/davidau_l3-stheno-maid-blackroot-grand-horror-16b.json b/data/models/davidau_l3-stheno-maid-blackroot-grand-horror-16b.json deleted file mode 100644 index d8e6de72c02d352d787cda192776ffdb65b4c0d6..0000000000000000000000000000000000000000 --- a/data/models/davidau_l3-stheno-maid-blackroot-grand-horror-16b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "L3-Stheno-Maid-Blackroot-Grand-HORROR-16B", - "id": "DavidAU/L3-Stheno-Maid-Blackroot-Grand-HORROR-16B", - "developer": "DavidAU", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "16.537" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DavidAU_L3-Stheno-Maid-Blackroot-Grand-HORROR-16B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3439 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4736 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0219 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.271 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4031 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.357 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/davidau_l3-stheno-v3.2-12.2b-instruct.json b/data/models/davidau_l3-stheno-v3.2-12.2b-instruct.json deleted file mode 100644 index 08c6df2d4587d98aaac68b0fdd2a4dac593c068e..0000000000000000000000000000000000000000 --- a/data/models/davidau_l3-stheno-v3.2-12.2b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "L3-Stheno-v3.2-12.2B-Instruct", - "id": "DavidAU/L3-Stheno-v3.2-12.2B-Instruct", - "developer": "DavidAU", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "12.174" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DavidAU_L3-Stheno-v3.2-12.2B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4028 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4846 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0506 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2752 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4103 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3345 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/davidau_l3.1-dark-planet-spinfire-uncensored-8b.json b/data/models/davidau_l3.1-dark-planet-spinfire-uncensored-8b.json deleted file mode 100644 index 4b8b222f77b56f7d936cefb7608228ec888aae60..0000000000000000000000000000000000000000 --- a/data/models/davidau_l3.1-dark-planet-spinfire-uncensored-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "L3.1-Dark-Planet-SpinFire-Uncensored-8B", - "id": "DavidAU/L3.1-Dark-Planet-SpinFire-Uncensored-8B", - "developer": "DavidAU", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DavidAU_L3.1-Dark-Planet-SpinFire-Uncensored-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7043 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5261 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0929 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2794 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3541 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.367 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/davidau_l3.1-moe-2x8b-deepseek-deephermes-e32-uncensored-abliterated-13.7b.json b/data/models/davidau_l3.1-moe-2x8b-deepseek-deephermes-e32-uncensored-abliterated-13.7b.json deleted file mode 100644 index 599bb076297af9cf8e4e91bd130dfd32d67d657d..0000000000000000000000000000000000000000 --- a/data/models/davidau_l3.1-moe-2x8b-deepseek-deephermes-e32-uncensored-abliterated-13.7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "L3.1-MOE-2X8B-Deepseek-DeepHermes-e32-uncensored-abliterated-13.7B", - "id": "DavidAU/L3.1-MOE-2X8B-Deepseek-DeepHermes-e32-uncensored-abliterated-13.7B", - "developer": "DavidAU", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MixtralForCausalLM", - "params_billions": "13.668" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DavidAU_L3.1-MOE-2X8B-Deepseek-DeepHermes-e32-uncensored-abliterated-13.7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3345 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4421 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2606 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3138 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3749 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2892 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/davidau_qwen2.5-moe-2x1.5b-deepseek-uncensored-censored-4b.json b/data/models/davidau_qwen2.5-moe-2x1.5b-deepseek-uncensored-censored-4b.json deleted file mode 100644 index 286338b9dbe2f5cf300335594017571e3e3ae7a1..0000000000000000000000000000000000000000 --- a/data/models/davidau_qwen2.5-moe-2x1.5b-deepseek-uncensored-censored-4b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-MOE-2X1.5B-DeepSeek-Uncensored-Censored-4B", - "id": "DavidAU/Qwen2.5-MOE-2X1.5B-DeepSeek-Uncensored-Censored-4B", - "developer": "DavidAU", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2MoeForCausalLM", - "params_billions": "4.089" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DavidAU_Qwen2.5-MOE-2X1.5B-DeepSeek-Uncensored-Censored-4B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1783 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3033 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0249 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2592 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3715 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1142 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/davidau_qwen2.5-moe-2x7b-deepseek-abliterated-censored-19b.json b/data/models/davidau_qwen2.5-moe-2x7b-deepseek-abliterated-censored-19b.json deleted file mode 100644 index f6e50ad5a8420b245dac1a85cf8d2e626740ff37..0000000000000000000000000000000000000000 --- a/data/models/davidau_qwen2.5-moe-2x7b-deepseek-abliterated-censored-19b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-MOE-2X7B-DeepSeek-Abliterated-Censored-19B", - "id": "DavidAU/Qwen2.5-MOE-2X7B-DeepSeek-Abliterated-Censored-19B", - "developer": "DavidAU", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2MoeForCausalLM", - "params_billions": "19.022" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DavidAU_Qwen2.5-MOE-2X7B-DeepSeek-Abliterated-Censored-19B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2835 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3592 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2417 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2651 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3847 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1636 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/davidau_qwen2.5-moe-6x1.5b-deepseek-reasoning-e32.json b/data/models/davidau_qwen2.5-moe-6x1.5b-deepseek-reasoning-e32.json deleted file mode 100644 index 2b37dfc60c2aa8a027db256278e889df63218f9a..0000000000000000000000000000000000000000 --- a/data/models/davidau_qwen2.5-moe-6x1.5b-deepseek-reasoning-e32.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-MOE-6x1.5B-DeepSeek-Reasoning-e32", - "id": "DavidAU/Qwen2.5-MOE-6x1.5B-DeepSeek-Reasoning-e32", - "developer": "DavidAU", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2MoeForCausalLM", - "params_billions": "8.714" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DavidAU_Qwen2.5-MOE-6x1.5B-DeepSeek-Reasoning-e32/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2107 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3286 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0665 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2475 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3404 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1122 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/davidkim205_nox-solar-10.7b-v4.json b/data/models/davidkim205_nox-solar-10.7b-v4.json deleted file mode 100644 index 9af8c99455c16f1d6018531d492f3202994c8ae8..0000000000000000000000000000000000000000 --- a/data/models/davidkim205_nox-solar-10.7b-v4.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "nox-solar-10.7b-v4", - "id": "davidkim205/nox-solar-10.7b-v4", - "developer": "davidkim205", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "10.732" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/davidkim205_nox-solar-10.7b-v4/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3753 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4814 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0083 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.307 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4298 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3333 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/davidkim205_rhea-72b-v0.5.json b/data/models/davidkim205_rhea-72b-v0.5.json deleted file mode 100644 index 8445877f9232c44afa681a9cb95722a9e9bf9102..0000000000000000000000000000000000000000 --- a/data/models/davidkim205_rhea-72b-v0.5.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Rhea-72b-v0.5", - "id": "davidkim205/Rhea-72b-v0.5", - "developer": "davidkim205", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "72.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/davidkim205_Rhea-72b-v0.5/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0145 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3078 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1737 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2525 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4241 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1166 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/davidsv_suong-1.json b/data/models/davidsv_suong-1.json deleted file mode 100644 index e4018a4b331e647079202f23ffe2273d3f3e93ce..0000000000000000000000000000000000000000 --- a/data/models/davidsv_suong-1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SUONG-1", - "id": "Davidsv/SUONG-1", - "developer": "Davidsv", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "2.879" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Davidsv_SUONG-1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2497 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2817 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2441 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3578 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1085 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/davielion_llama-3.2-1b-spin-iter0.json b/data/models/davielion_llama-3.2-1b-spin-iter0.json deleted file mode 100644 index 849a1d52d59a5f16eab6aaa35f259f630d9dc175..0000000000000000000000000000000000000000 --- a/data/models/davielion_llama-3.2-1b-spin-iter0.json +++ /dev/null @@ -1,275 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.2-1B-SPIN-iter0", - "id": "DavieLion/Llama-3.2-1B-SPIN-iter0", - "developer": "DavieLion", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.236" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DavieLion_Llama-3.2-1B-SPIN-iter0/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1549 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2937 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.006 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2576 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3565 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1128 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "hfopenllm_v2/DavieLion_Llama-3.2-1B-SPIN-iter0/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1507 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.293 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2534 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3565 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1125 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/davielion_llama-3.2-1b-spin-iter1.json b/data/models/davielion_llama-3.2-1b-spin-iter1.json deleted file mode 100644 index f730e04aad22b38aa05714dccba0da66157095da..0000000000000000000000000000000000000000 --- a/data/models/davielion_llama-3.2-1b-spin-iter1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.2-1B-SPIN-iter1", - "id": "DavieLion/Llama-3.2-1B-SPIN-iter1", - "developer": "DavieLion", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.236" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DavieLion_Llama-3.2-1B-SPIN-iter1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1575 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.294 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0023 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2508 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3646 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1118 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/davielion_llama-3.2-1b-spin-iter2.json b/data/models/davielion_llama-3.2-1b-spin-iter2.json deleted file mode 100644 index f990d02d620b40c960760c1973ceb5bda9989f51..0000000000000000000000000000000000000000 --- a/data/models/davielion_llama-3.2-1b-spin-iter2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.2-1B-SPIN-iter2", - "id": "DavieLion/Llama-3.2-1B-SPIN-iter2", - "developer": "DavieLion", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.236" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DavieLion_Llama-3.2-1B-SPIN-iter2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1376 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.298 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0053 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2542 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3553 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1129 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/davielion_llama-3.2-1b-spin-iter3.json b/data/models/davielion_llama-3.2-1b-spin-iter3.json deleted file mode 100644 index 99c7b4d0290325891f13fa396abea88044dda26d..0000000000000000000000000000000000000000 --- a/data/models/davielion_llama-3.2-1b-spin-iter3.json +++ /dev/null @@ -1,275 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.2-1B-SPIN-iter3", - "id": "DavieLion/Llama-3.2-1B-SPIN-iter3", - "developer": "DavieLion", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.236" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DavieLion_Llama-3.2-1B-SPIN-iter3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1324 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2972 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2643 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3527 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1129 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "hfopenllm_v2/DavieLion_Llama-3.2-1B-SPIN-iter3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1336 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2975 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0068 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2534 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.35 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1128 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/davielion_lllma-3.2-1b.json b/data/models/davielion_lllma-3.2-1b.json deleted file mode 100644 index afbb2e05773e57b6fd88368e34917554bb317400..0000000000000000000000000000000000000000 --- a/data/models/davielion_lllma-3.2-1b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Lllma-3.2-1B", - "id": "DavieLion/Lllma-3.2-1B", - "developer": "DavieLion", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.236" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DavieLion_Lllma-3.2-1B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1601 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2965 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0068 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2441 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3578 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1126 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/debatelabkit_llama-3.1-argunaut-1-8b-sft.json b/data/models/debatelabkit_llama-3.1-argunaut-1-8b-sft.json deleted file mode 100644 index 04246f1fe478eb8e550021f7ff81aaafc1eccdc5..0000000000000000000000000000000000000000 --- a/data/models/debatelabkit_llama-3.1-argunaut-1-8b-sft.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.1-Argunaut-1-8B-SFT", - "id": "DebateLabKIT/Llama-3.1-Argunaut-1-8B-SFT", - "developer": "DebateLabKIT", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DebateLabKIT_Llama-3.1-Argunaut-1-8B-SFT/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5519 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4824 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.145 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2836 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4503 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3472 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/deci_decilm-7b-instruct.json b/data/models/deci_decilm-7b-instruct.json deleted file mode 100644 index d3c1ab1279a314133404fb32d287dd3dcc1e6635..0000000000000000000000000000000000000000 --- a/data/models/deci_decilm-7b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "DeciLM-7B-instruct", - "id": "Deci/DeciLM-7B-instruct", - "developer": "Deci", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "DeciLMForCausalLM", - "params_billions": "7.044" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Deci_DeciLM-7B-instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.488 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.459 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0302 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2894 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3884 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2608 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/deci_decilm-7b.json b/data/models/deci_decilm-7b.json deleted file mode 100644 index 2df45a2d0acb6d1cafbd8fb25127585ea2f639af..0000000000000000000000000000000000000000 --- a/data/models/deci_decilm-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "DeciLM-7B", - "id": "Deci/DeciLM-7B", - "developer": "Deci", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "DeciLMForCausalLM", - "params_billions": "7.044" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Deci_DeciLM-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2813 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4423 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0287 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2953 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4359 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2692 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/deepautoai_causal_gpt2.json b/data/models/deepautoai_causal_gpt2.json deleted file mode 100644 index a8a0cee54613b1ef436698dff3be8bae3b6e5064..0000000000000000000000000000000000000000 --- a/data/models/deepautoai_causal_gpt2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "causal_gpt2", - "id": "DeepAutoAI/causal_gpt2", - "developer": "DeepAutoAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "GPT2LMHeadModel", - "params_billions": "0.124" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DeepAutoAI_causal_gpt2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1813 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3026 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0053 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2601 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.427 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1131 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/deepautoai_d2nwg_causal_gpt2.json b/data/models/deepautoai_d2nwg_causal_gpt2.json deleted file mode 100644 index 1bbe9af7b33110ae6cb8fb39f0424767f7052839..0000000000000000000000000000000000000000 --- a/data/models/deepautoai_d2nwg_causal_gpt2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "d2nwg_causal_gpt2", - "id": "DeepAutoAI/d2nwg_causal_gpt2", - "developer": "DeepAutoAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "GPT2LMHeadModel", - "params_billions": "0.124" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DeepAutoAI_d2nwg_causal_gpt2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1916 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3027 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0045 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2576 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4297 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1151 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/deepautoai_d2nwg_causal_gpt2_v1.json b/data/models/deepautoai_d2nwg_causal_gpt2_v1.json deleted file mode 100644 index 0d6da306f5a8c768aca4630427f34739b8357570..0000000000000000000000000000000000000000 --- a/data/models/deepautoai_d2nwg_causal_gpt2_v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "d2nwg_causal_gpt2_v1", - "id": "DeepAutoAI/d2nwg_causal_gpt2_v1", - "developer": "DeepAutoAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "GPT2LMHeadModel", - "params_billions": "0.124" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DeepAutoAI_d2nwg_causal_gpt2_v1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1989 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2992 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0038 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2584 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4337 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1135 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/deepautoai_d2nwg_llama-3.1-8b-instruct-v0.0.json b/data/models/deepautoai_d2nwg_llama-3.1-8b-instruct-v0.0.json deleted file mode 100644 index 3ecde6179bc391bb7242be2b39ce832e733f79d8..0000000000000000000000000000000000000000 --- a/data/models/deepautoai_d2nwg_llama-3.1-8b-instruct-v0.0.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "d2nwg_Llama-3.1-8B-Instruct-v0.0", - "id": "DeepAutoAI/d2nwg_Llama-3.1-8B-Instruct-v0.0", - "developer": "DeepAutoAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DeepAutoAI_d2nwg_Llama-3.1-8B-Instruct-v0.0/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7893 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.508 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1805 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2919 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4135 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3877 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/deepautoai_explore_llama-3.1-8b-inst.json b/data/models/deepautoai_explore_llama-3.1-8b-inst.json deleted file mode 100644 index 0a53b24d6dc1053c2d4c050dd48882d5ee7e09ba..0000000000000000000000000000000000000000 --- a/data/models/deepautoai_explore_llama-3.1-8b-inst.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Explore_Llama-3.1-8B-Inst", - "id": "DeepAutoAI/Explore_Llama-3.1-8B-Inst", - "developer": "DeepAutoAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DeepAutoAI_Explore_Llama-3.1-8B-Inst/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7795 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5117 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2009 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2836 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.391 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3792 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/deepautoai_explore_llama-3.2-1b-inst.json b/data/models/deepautoai_explore_llama-3.2-1b-inst.json deleted file mode 100644 index 698ec004ad1557617fa0d9116f8dbe096fd7687f..0000000000000000000000000000000000000000 --- a/data/models/deepautoai_explore_llama-3.2-1b-inst.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Explore_Llama-3.2-1B-Inst", - "id": "DeepAutoAI/Explore_Llama-3.2-1B-Inst", - "developer": "DeepAutoAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.236" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DeepAutoAI_Explore_Llama-3.2-1B-Inst/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5649 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3505 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0748 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2559 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3183 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1809 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/deepautoai_explore_llama-3.2-1b-inst_v0.json b/data/models/deepautoai_explore_llama-3.2-1b-inst_v0.json deleted file mode 100644 index e47e1fd0c1e57ff7e4a189a3d90decb97f4311ac..0000000000000000000000000000000000000000 --- a/data/models/deepautoai_explore_llama-3.2-1b-inst_v0.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Explore_Llama-3.2-1B-Inst_v0", - "id": "DeepAutoAI/Explore_Llama-3.2-1B-Inst_v0", - "developer": "DeepAutoAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.236" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DeepAutoAI_Explore_Llama-3.2-1B-Inst_v0/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5597 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3365 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0597 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2634 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3103 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1804 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/deepautoai_explore_llama-3.2-1b-inst_v1.1.json b/data/models/deepautoai_explore_llama-3.2-1b-inst_v1.1.json deleted file mode 100644 index d1830709256c6cee40cf00d96c413333d00c6320..0000000000000000000000000000000000000000 --- a/data/models/deepautoai_explore_llama-3.2-1b-inst_v1.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Explore_Llama-3.2-1B-Inst_v1.1", - "id": "DeepAutoAI/Explore_Llama-3.2-1B-Inst_v1.1", - "developer": "DeepAutoAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.236" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DeepAutoAI_Explore_Llama-3.2-1B-Inst_v1.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5844 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3513 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0718 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2626 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3117 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1818 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/deepautoai_explore_llama-3.2-1b-inst_v1.json b/data/models/deepautoai_explore_llama-3.2-1b-inst_v1.json deleted file mode 100644 index 745a5fbd3d11436a6a4300b6a07fe91ecd08ecf8..0000000000000000000000000000000000000000 --- a/data/models/deepautoai_explore_llama-3.2-1b-inst_v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Explore_Llama-3.2-1B-Inst_v1", - "id": "DeepAutoAI/Explore_Llama-3.2-1B-Inst_v1", - "developer": "DeepAutoAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.236" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DeepAutoAI_Explore_Llama-3.2-1B-Inst_v1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4999 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3141 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.031 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.245 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3781 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1269 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/deepautoai_ldm_soup_llama-3.1-8b-inst.json b/data/models/deepautoai_ldm_soup_llama-3.1-8b-inst.json deleted file mode 100644 index 6af71ca41d078aff90e2479e7ae050146773b764..0000000000000000000000000000000000000000 --- a/data/models/deepautoai_ldm_soup_llama-3.1-8b-inst.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ldm_soup_Llama-3.1-8B-Inst", - "id": "DeepAutoAI/ldm_soup_Llama-3.1-8B-Inst", - "developer": "DeepAutoAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DeepAutoAI_ldm_soup_Llama-3.1-8B-Inst/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8033 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5121 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1888 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2894 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4161 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3886 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/deepautoai_ldm_soup_llama-3.1-8b-instruct-v0.0.json b/data/models/deepautoai_ldm_soup_llama-3.1-8b-instruct-v0.0.json deleted file mode 100644 index b24866aaffb5aca62eac1ebee862bf4abf5ed4ac..0000000000000000000000000000000000000000 --- a/data/models/deepautoai_ldm_soup_llama-3.1-8b-instruct-v0.0.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ldm_soup_Llama-3.1-8B-Instruct-v0.0", - "id": "DeepAutoAI/ldm_soup_Llama-3.1-8B-Instruct-v0.0", - "developer": "DeepAutoAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DeepAutoAI_ldm_soup_Llama-3.1-8B-Instruct-v0.0/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7889 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5125 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1918 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2911 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4121 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3895 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/deepautoai_ldm_soup_llama-3.1-8b-instruct-v0.1.json b/data/models/deepautoai_ldm_soup_llama-3.1-8b-instruct-v0.1.json deleted file mode 100644 index aeb4d43bf8869c09c3b239a76bd969bcb1d4324b..0000000000000000000000000000000000000000 --- a/data/models/deepautoai_ldm_soup_llama-3.1-8b-instruct-v0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ldm_soup_Llama-3.1-8B-Instruct-v0.1", - "id": "DeepAutoAI/ldm_soup_Llama-3.1-8B-Instruct-v0.1", - "developer": "DeepAutoAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DeepAutoAI_ldm_soup_Llama-3.1-8B-Instruct-v0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7889 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5125 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1918 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2911 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4121 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3895 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/deepmount00_lexora-lite-3b.json b/data/models/deepmount00_lexora-lite-3b.json deleted file mode 100644 index f732d501b88a6bdf2e17bd34399ebe5a2d372986..0000000000000000000000000000000000000000 --- a/data/models/deepmount00_lexora-lite-3b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Lexora-Lite-3B", - "id": "DeepMount00/Lexora-Lite-3B", - "developer": "DeepMount00", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.086" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DeepMount00_Lexora-Lite-3B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5776 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4873 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2304 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2743 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3966 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3602 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/deepmount00_lexora-lite-3b_v2.json b/data/models/deepmount00_lexora-lite-3b_v2.json deleted file mode 100644 index 4c66b4a077a41d4091166c11c85a226de11083c5..0000000000000000000000000000000000000000 --- a/data/models/deepmount00_lexora-lite-3b_v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Lexora-Lite-3B_v2", - "id": "DeepMount00/Lexora-Lite-3B_v2", - "developer": "DeepMount00", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.086" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DeepMount00_Lexora-Lite-3B_v2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4943 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4812 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2281 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.271 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3822 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3544 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/deepmount00_lexora-medium-7b.json b/data/models/deepmount00_lexora-medium-7b.json deleted file mode 100644 index 6fe087007e55b5ac0a87b09b1c777549b5a30837..0000000000000000000000000000000000000000 --- a/data/models/deepmount00_lexora-medium-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Lexora-Medium-7B", - "id": "DeepMount00/Lexora-Medium-7B", - "developer": "DeepMount00", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DeepMount00_Lexora-Medium-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4103 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5145 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2221 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3054 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4439 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4325 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/deepmount00_llama-3-8b-ita.json b/data/models/deepmount00_llama-3-8b-ita.json deleted file mode 100644 index bf4c820bab2b9cbadbbc5c4b917e6acb11c67280..0000000000000000000000000000000000000000 --- a/data/models/deepmount00_llama-3-8b-ita.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-8b-Ita", - "id": "DeepMount00/Llama-3-8b-Ita", - "developer": "DeepMount00", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DeepMount00_Llama-3-8b-Ita/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.753 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4936 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0665 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3054 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4268 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3852 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/deepmount00_llama-3.1-8b-ita.json b/data/models/deepmount00_llama-3.1-8b-ita.json deleted file mode 100644 index 1fef7ca0b3d471692e7379016093b45312ecc7df..0000000000000000000000000000000000000000 --- a/data/models/deepmount00_llama-3.1-8b-ita.json +++ /dev/null @@ -1,278 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.1-8b-ITA", - "id": "DeepMount00/Llama-3.1-8b-ITA", - "developer": "DeepMount00", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03", - "model_id_aliases": [ - "DeepMount00/Llama-3.1-8b-Ita" - ] - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DeepMount00_Llama-3.1-8b-ITA/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7917 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5109 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1088 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2878 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4136 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3876 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "hfopenllm_v2/DeepMount00_Llama-3.1-8b-Ita/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5365 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.517 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1707 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3062 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4487 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.396 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/deepmount00_llama-3.1-distilled.json b/data/models/deepmount00_llama-3.1-distilled.json deleted file mode 100644 index 74b1796c42c2598dd7f896bb9b75ebcf0b7d6a9d..0000000000000000000000000000000000000000 --- a/data/models/deepmount00_llama-3.1-distilled.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.1-Distilled", - "id": "DeepMount00/Llama-3.1-Distilled", - "developer": "DeepMount00", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DeepMount00_Llama-3.1-Distilled/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7844 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5101 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2032 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3037 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4058 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3782 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/deepmount00_mergekit-ties-okvgjfz.json b/data/models/deepmount00_mergekit-ties-okvgjfz.json deleted file mode 100644 index a1eadbd22858bf836c939db3c8600ea521e63264..0000000000000000000000000000000000000000 --- a/data/models/deepmount00_mergekit-ties-okvgjfz.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "mergekit-ties-okvgjfz", - "id": "DeepMount00/mergekit-ties-okvgjfz", - "developer": "DeepMount00", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DeepMount00_mergekit-ties-okvgjfz/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.153 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2998 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0008 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2626 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3806 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1118 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/deepmount00_qwen2-1.5b-ita.json b/data/models/deepmount00_qwen2-1.5b-ita.json deleted file mode 100644 index 8e838de1bfb667185d7e69f0bdb6d0f057a497e0..0000000000000000000000000000000000000000 --- a/data/models/deepmount00_qwen2-1.5b-ita.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2-1.5B-Ita", - "id": "DeepMount00/Qwen2-1.5B-Ita", - "developer": "DeepMount00", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.544" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DeepMount00_Qwen2-1.5B-Ita/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5173 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3981 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.114 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2626 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3504 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2772 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/deepmount00_qwen2-1.5b-ita_v2.json b/data/models/deepmount00_qwen2-1.5b-ita_v2.json deleted file mode 100644 index 9569e690d4e042d1d1951eb54fbee371f66b8389..0000000000000000000000000000000000000000 --- a/data/models/deepmount00_qwen2-1.5b-ita_v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2-1.5B-Ita_v2", - "id": "DeepMount00/Qwen2-1.5B-Ita_v2", - "developer": "DeepMount00", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.544" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DeepMount00_Qwen2-1.5B-Ita_v2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3954 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0967 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2592 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3702 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3032 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/deepmount00_qwen2-1.5b-ita_v3.json b/data/models/deepmount00_qwen2-1.5b-ita_v3.json deleted file mode 100644 index 6b6b28a50ae0ec0a7b5a159ec56ca1e901af1f4f..0000000000000000000000000000000000000000 --- a/data/models/deepmount00_qwen2-1.5b-ita_v3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2-1.5B-Ita_v3", - "id": "DeepMount00/Qwen2-1.5B-Ita_v3", - "developer": "DeepMount00", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.544" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DeepMount00_Qwen2-1.5B-Ita_v3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.489 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3948 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1042 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2534 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3742 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3018 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/deepmount00_qwen2-1.5b-ita_v5.json b/data/models/deepmount00_qwen2-1.5b-ita_v5.json deleted file mode 100644 index 7ca716c8fefb4d58fd5f0b6c4c292fb13f53f8e0..0000000000000000000000000000000000000000 --- a/data/models/deepmount00_qwen2-1.5b-ita_v5.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2-1.5B-Ita_v5", - "id": "DeepMount00/Qwen2-1.5B-Ita_v5", - "developer": "DeepMount00", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.544" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DeepMount00_Qwen2-1.5B-Ita_v5/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4987 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4032 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1178 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2542 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3422 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2943 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/deepmount00_qwen2-1.5b-ita_v6.json b/data/models/deepmount00_qwen2-1.5b-ita_v6.json deleted file mode 100644 index 741405c32cb1d694b3cf98e674f1d7346b5cedca..0000000000000000000000000000000000000000 --- a/data/models/deepmount00_qwen2-1.5b-ita_v6.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2-1.5B-Ita_v6", - "id": "DeepMount00/Qwen2-1.5B-Ita_v6", - "developer": "DeepMount00", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.497" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DeepMount00_Qwen2-1.5B-Ita_v6/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2999 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4249 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0846 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2827 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3755 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2872 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/deepmount00_qwen2.5-7b-instruct-mathcoder.json b/data/models/deepmount00_qwen2.5-7b-instruct-mathcoder.json deleted file mode 100644 index c3a1f618d79050f773613dda0937fced705e0505..0000000000000000000000000000000000000000 --- a/data/models/deepmount00_qwen2.5-7b-instruct-mathcoder.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-7B-Instruct-MathCoder", - "id": "DeepMount00/Qwen2.5-7B-Instruct-MathCoder", - "developer": "DeepMount00", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DeepMount00_Qwen2.5-7B-Instruct-MathCoder/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.153 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2998 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0008 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2626 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3806 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1118 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/deepseek-ai_deepseek-llm-67b-chat.json b/data/models/deepseek-ai_deepseek-llm-67b-chat.json deleted file mode 100644 index e088ed2d042f06ef60e4e65615f3e04f8b24ffb8..0000000000000000000000000000000000000000 --- a/data/models/deepseek-ai_deepseek-llm-67b-chat.json +++ /dev/null @@ -1,2031 +0,0 @@ -{ - "model_info": { - "name": "DeepSeek LLM Chat 67B", - "id": "deepseek-ai/deepseek-llm-67b-chat", - "developer": "deepseek-ai", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_lite/deepseek-ai_deepseek-llm-67b-chat/1774096306.427425", - "retrieved_timestamp": "1774096306.427425", - "source_metadata": { - "source_name": "helm_lite", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_lite", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_lite", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.488, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.30021223470661673\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.581, - "details": { - "description": "min=0.581, mean=0.581, max=0.581, sum=0.581 (1)", - "tab": "Accuracy", - "NarrativeQA - Observed inference time (s)": "{\"description\": \"min=3.36, mean=3.36, max=3.36, sum=3.36 (1)\", \"tab\": \"Efficiency\", \"score\": \"3.359551859573579\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=4.946, mean=4.946, max=4.946, sum=4.946 (1)\", \"tab\": \"General information\", \"score\": \"4.946478873239436\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=3583.146, mean=3583.146, max=3583.146, sum=3583.146 (1)\", \"tab\": \"General information\", \"score\": \"3583.1464788732396\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (closed-book)", - "source_data": { - "dataset_name": "NaturalQuestions (closed-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (closed-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.412, - "details": { - "description": "min=0.412, mean=0.412, max=0.412, sum=0.412 (1)", - "tab": "Accuracy", - "NaturalQuestions (open-book) - Observed inference time (s)": "{\"description\": \"min=2.237, mean=2.237, max=2.237, sum=2.237 (1)\", \"tab\": \"Efficiency\", \"score\": \"2.2367931361198425\"}", - "NaturalQuestions (closed-book) - Observed inference time (s)": "{\"description\": \"min=0.857, mean=0.857, max=0.857, sum=0.857 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.8567402980327606\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.841, mean=4.841, max=4.841, sum=4.841 (1)\", \"tab\": \"General information\", \"score\": \"4.841\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.024, mean=0.024, max=0.024, sum=0.024 (1)\", \"tab\": \"General information\", \"score\": \"0.024\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=2192.734, mean=2192.734, max=2192.734, sum=2192.734 (1)\", \"tab\": \"General information\", \"score\": \"2192.734\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=199.39, mean=199.39, max=199.39, sum=199.39 (1)\", \"tab\": \"General information\", \"score\": \"199.39\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "mode": "\"closedbook\"" - } - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.88, - "details": { - "description": "min=0.88, mean=0.88, max=0.88, sum=0.88 (1)", - "tab": "Accuracy", - "OpenbookQA - Observed inference time (s)": "{\"description\": \"min=0.417, mean=0.417, max=0.417, sum=0.417 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.41702947664260864\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=253.206, mean=253.206, max=253.206, sum=253.206 (1)\", \"tab\": \"General information\", \"score\": \"253.206\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "dataset": "\"openbookqa\"", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.641, - "details": { - "description": "min=0.44, mean=0.641, max=0.91, sum=3.203 (5)", - "tab": "Accuracy", - "MMLU - Observed inference time (s)": "{\"description\": \"min=0.48, mean=0.508, max=0.551, sum=2.542 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.508463426874395\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=382.07, mean=490.941, max=646.667, sum=2454.707 (5)\", \"tab\": \"General information\", \"score\": \"490.9413333333334\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"college_chemistry\", \"computer_security\", \"econometrics\", \"us_foreign_policy\"]", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MATH", - "source_data": { - "dataset_name": "MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Equivalent (CoT) on MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.615, - "details": { - "description": "min=0.456, mean=0.615, max=0.748, sum=4.304 (7)", - "tab": "Accuracy", - "MATH - Observed inference time (s)": "{\"description\": \"min=3.389, mean=4.443, max=6.234, sum=31.098 (7)\", \"tab\": \"Efficiency\", \"score\": \"4.442596748084942\"}", - "MATH - # eval": "{\"description\": \"min=30, mean=62.429, max=135, sum=437 (7)\", \"tab\": \"General information\", \"score\": \"62.42857142857143\"}", - "MATH - # train": "{\"description\": \"min=8, mean=8, max=8, sum=56 (7)\", \"tab\": \"General information\", \"score\": \"8.0\"}", - "MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MATH - # prompt tokens": "{\"description\": \"min=1012.548, mean=1443.29, max=2448.25, sum=10103.027 (7)\", \"tab\": \"General information\", \"score\": \"1443.2895059403625\"}", - "MATH - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=7 (7)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"algebra\", \"counting_and_probability\", \"geometry\", \"intermediate_algebra\", \"number_theory\", \"prealgebra\", \"precalculus\"]", - "level": "\"1\"", - "use_official_examples": "\"False\"", - "use_chain_of_thought": "\"True\"" - } - } - }, - { - "evaluation_name": "GSM8K", - "source_data": { - "dataset_name": "GSM8K", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on GSM8K", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.795, - "details": { - "description": "min=0.795, mean=0.795, max=0.795, sum=0.795 (1)", - "tab": "Accuracy", - "GSM8K - Observed inference time (s)": "{\"description\": \"min=5.877, mean=5.877, max=5.877, sum=5.877 (1)\", \"tab\": \"Efficiency\", \"score\": \"5.876643376111984\"}", - "GSM8K - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "GSM8K - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "GSM8K - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GSM8K - # prompt tokens": "{\"description\": \"min=1233.708, mean=1233.708, max=1233.708, sum=1233.708 (1)\", \"tab\": \"General information\", \"score\": \"1233.708\"}", - "GSM8K - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "LegalBench", - "source_data": { - "dataset_name": "LegalBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on LegalBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.637, - "details": { - "description": "min=0.45, mean=0.637, max=0.821, sum=3.183 (5)", - "tab": "Accuracy", - "LegalBench - Observed inference time (s)": "{\"description\": \"min=0.524, mean=0.942, max=2.301, sum=4.71 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.9420770218153176\"}", - "LegalBench - # eval": "{\"description\": \"min=95, mean=409.4, max=1000, sum=2047 (5)\", \"tab\": \"General information\", \"score\": \"409.4\"}", - "LegalBench - # train": "{\"description\": \"min=2.006, mean=4.201, max=5, sum=21.006 (5)\", \"tab\": \"General information\", \"score\": \"4.201224489795918\"}", - "LegalBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "LegalBench - # prompt tokens": "{\"description\": \"min=269.379, mean=990.259, max=3325.551, sum=4951.297 (5)\", \"tab\": \"General information\", \"score\": \"990.259348667894\"}", - "LegalBench - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"abercrombie\", \"corporate_lobbying\", \"function_of_decision_section\", \"international_citizenship_questions\", \"proa\"]" - } - } - }, - { - "evaluation_name": "MedQA", - "source_data": { - "dataset_name": "MedQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MedQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.628, - "details": { - "description": "min=0.628, mean=0.628, max=0.628, sum=0.628 (1)", - "tab": "Accuracy", - "MedQA - Observed inference time (s)": "{\"description\": \"min=0.83, mean=0.83, max=0.83, sum=0.83 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.8296676231899982\"}", - "MedQA - # eval": "{\"description\": \"min=503, mean=503, max=503, sum=503 (1)\", \"tab\": \"General information\", \"score\": \"503.0\"}", - "MedQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MedQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MedQA - # prompt tokens": "{\"description\": \"min=1084.235, mean=1084.235, max=1084.235, sum=1084.235 (1)\", \"tab\": \"General information\", \"score\": \"1084.234592445328\"}", - "MedQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WMT 2014", - "source_data": { - "dataset_name": "WMT 2014", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "BLEU-4 on WMT 2014", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.186, - "details": { - "description": "min=0.11, mean=0.186, max=0.236, sum=0.932 (5)", - "tab": "Accuracy", - "WMT 2014 - Observed inference time (s)": "{\"description\": \"min=1.381, mean=1.429, max=1.464, sum=7.147 (5)\", \"tab\": \"Efficiency\", \"score\": \"1.429440071817079\"}", - "WMT 2014 - # eval": "{\"description\": \"min=503, mean=568.8, max=832, sum=2844 (5)\", \"tab\": \"General information\", \"score\": \"568.8\"}", - "WMT 2014 - # train": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "WMT 2014 - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WMT 2014 - # prompt tokens": "{\"description\": \"min=203.736, mean=220.291, max=255.861, sum=1101.453 (5)\", \"tab\": \"General information\", \"score\": \"220.29060445022174\"}", - "WMT 2014 - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "language_pair": "[\"cs-en\", \"de-en\", \"fr-en\", \"hi-en\", \"ru-en\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_mmlu/deepseek-ai_deepseek-llm-67b-chat/1774096312.00548", - "retrieved_timestamp": "1774096312.00548", - "source_metadata": { - "source_name": "helm_mmlu", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_mmlu", - "evaluation_results": [ - { - "evaluation_name": "MMLU All Subjects", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU All Subjects", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.725, - "details": { - "description": "min=0.363, mean=0.725, max=0.964, sum=82.655 (114)", - "tab": "Accuracy", - "MMLU All Subjects - Observed inference time (s)": "{\"description\": \"min=0.432, mean=0.591, max=1.828, sum=67.401 (114)\", \"tab\": \"Efficiency\", \"score\": \"0.5912370078072168\"}", - "MMLU All Subjects - # eval": "{\"description\": \"min=100, mean=246.351, max=1534, sum=28084 (114)\", \"tab\": \"General information\", \"score\": \"246.35087719298247\"}", - "MMLU All Subjects - # train": "{\"description\": \"min=5, mean=5, max=5, sum=570 (114)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU All Subjects - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU All Subjects - # prompt tokens": "{\"description\": \"min=282.398, mean=644.941, max=2845.339, sum=73523.251 (114)\", \"tab\": \"General information\", \"score\": \"644.9407984438222\"}", - "MMLU All Subjects - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=114 (114)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_name": "Abstract Algebra", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Abstract Algebra", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.44, - "details": { - "description": "min=0.44, mean=0.44, max=0.44, sum=0.88 (2)", - "tab": "Accuracy", - "Abstract Algebra - Observed inference time (s)": "{\"description\": \"min=0.485, mean=0.485, max=0.485, sum=0.97 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4850481009483337\"}", - "Abstract Algebra - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Abstract Algebra - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Abstract Algebra - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Abstract Algebra - # prompt tokens": "{\"description\": \"min=382.07, mean=382.07, max=382.07, sum=764.14 (2)\", \"tab\": \"General information\", \"score\": \"382.07\"}", - "Abstract Algebra - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"abstract_algebra\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_abstract_algebra\"" - } - } - }, - { - "evaluation_name": "Anatomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Anatomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.667, - "details": { - "description": "min=0.667, mean=0.667, max=0.667, sum=1.333 (2)", - "tab": "Accuracy", - "Anatomy - Observed inference time (s)": "{\"description\": \"min=0.451, mean=0.451, max=0.451, sum=0.903 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4513168688173647\"}", - "Anatomy - # eval": "{\"description\": \"min=135, mean=135, max=135, sum=270 (2)\", \"tab\": \"General information\", \"score\": \"135.0\"}", - "Anatomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Anatomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Anatomy - # prompt tokens": "{\"description\": \"min=363.059, mean=363.059, max=363.059, sum=726.119 (2)\", \"tab\": \"General information\", \"score\": \"363.05925925925925\"}", - "Anatomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"anatomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_anatomy\"" - } - } - }, - { - "evaluation_name": "College Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on College Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.363, - "details": { - "description": "min=0.363, mean=0.363, max=0.363, sum=0.725 (2)", - "tab": "Accuracy", - "College Chemistry - Observed inference time (s)": "{\"description\": \"min=0.522, mean=0.522, max=0.522, sum=1.045 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5224089217185974\"}", - "College Biology - Observed inference time (s)": "{\"description\": \"min=0.513, mean=0.513, max=0.513, sum=1.026 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5128465278281106\"}", - "College Computer Science - Observed inference time (s)": "{\"description\": \"min=0.674, mean=0.674, max=0.674, sum=1.347 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6736601734161377\"}", - "College Mathematics - Observed inference time (s)": "{\"description\": \"min=0.552, mean=0.552, max=0.552, sum=1.103 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5516978883743286\"}", - "College Medicine - Observed inference time (s)": "{\"description\": \"min=0.515, mean=0.515, max=0.515, sum=1.03 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5147825513960999\"}", - "College Physics - Observed inference time (s)": "{\"description\": \"min=0.558, mean=0.558, max=0.558, sum=1.116 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5581503288418639\"}", - "College Chemistry - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Chemistry - # prompt tokens": "{\"description\": \"min=578.1, mean=578.1, max=578.1, sum=1156.2 (2)\", \"tab\": \"General information\", \"score\": \"578.1\"}", - "College Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Biology - # eval": "{\"description\": \"min=144, mean=144, max=144, sum=288 (2)\", \"tab\": \"General information\", \"score\": \"144.0\"}", - "College Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # prompt tokens": "{\"description\": \"min=502.611, mean=502.611, max=502.611, sum=1005.222 (2)\", \"tab\": \"General information\", \"score\": \"502.6111111111111\"}", - "College Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # prompt tokens": "{\"description\": \"min=864.55, mean=864.55, max=864.55, sum=1729.1 (2)\", \"tab\": \"General information\", \"score\": \"864.55\"}", - "College Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Mathematics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # prompt tokens": "{\"description\": \"min=630.13, mean=630.13, max=630.13, sum=1260.26 (2)\", \"tab\": \"General information\", \"score\": \"630.13\"}", - "College Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Medicine - # eval": "{\"description\": \"min=173, mean=173, max=173, sum=346 (2)\", \"tab\": \"General information\", \"score\": \"173.0\"}", - "College Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # prompt tokens": "{\"description\": \"min=538.613, mean=538.613, max=538.613, sum=1077.225 (2)\", \"tab\": \"General information\", \"score\": \"538.6127167630058\"}", - "College Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Physics - # eval": "{\"description\": \"min=102, mean=102, max=102, sum=204 (2)\", \"tab\": \"General information\", \"score\": \"102.0\"}", - "College Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # prompt tokens": "{\"description\": \"min=507.157, mean=507.157, max=507.157, sum=1014.314 (2)\", \"tab\": \"General information\", \"score\": \"507.15686274509807\"}", - "College Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"college_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_college_physics\"" - } - } - }, - { - "evaluation_name": "Computer Security", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Computer Security", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.79, - "details": { - "description": "min=0.79, mean=0.79, max=0.79, sum=1.58 (2)", - "tab": "Accuracy", - "Computer Security - Observed inference time (s)": "{\"description\": \"min=0.48, mean=0.48, max=0.48, sum=0.96 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.48023970127105714\"}", - "Computer Security - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Computer Security - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Computer Security - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Computer Security - # prompt tokens": "{\"description\": \"min=394.36, mean=394.36, max=394.36, sum=788.72 (2)\", \"tab\": \"General information\", \"score\": \"394.36\"}", - "Computer Security - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"computer_security\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_computer_security\"" - } - } - }, - { - "evaluation_name": "Econometrics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Econometrics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.553, - "details": { - "description": "min=0.553, mean=0.553, max=0.553, sum=1.105 (2)", - "tab": "Accuracy", - "Econometrics - Observed inference time (s)": "{\"description\": \"min=0.551, mean=0.551, max=0.551, sum=1.102 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5508757557785302\"}", - "Econometrics - # eval": "{\"description\": \"min=114, mean=114, max=114, sum=228 (2)\", \"tab\": \"General information\", \"score\": \"114.0\"}", - "Econometrics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Econometrics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Econometrics - # prompt tokens": "{\"description\": \"min=646.667, mean=646.667, max=646.667, sum=1293.333 (2)\", \"tab\": \"General information\", \"score\": \"646.6666666666666\"}", - "Econometrics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"econometrics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_econometrics\"" - } - } - }, - { - "evaluation_name": "Global Facts", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Global Facts", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.46, - "details": { - "description": "min=0.46, mean=0.46, max=0.46, sum=0.92 (2)", - "tab": "Accuracy", - "Global Facts - Observed inference time (s)": "{\"description\": \"min=0.506, mean=0.506, max=0.506, sum=1.013 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5062541460990906\"}", - "Global Facts - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Global Facts - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Global Facts - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Global Facts - # prompt tokens": "{\"description\": \"min=457.97, mean=457.97, max=457.97, sum=915.94 (2)\", \"tab\": \"General information\", \"score\": \"457.97\"}", - "Global Facts - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"global_facts\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_global_facts\"" - } - } - }, - { - "evaluation_name": "Jurisprudence", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Jurisprudence", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.852, - "details": { - "description": "min=0.852, mean=0.852, max=0.852, sum=1.704 (2)", - "tab": "Accuracy", - "Jurisprudence - Observed inference time (s)": "{\"description\": \"min=0.477, mean=0.477, max=0.477, sum=0.954 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.47704599963294136\"}", - "Jurisprudence - # eval": "{\"description\": \"min=108, mean=108, max=108, sum=216 (2)\", \"tab\": \"General information\", \"score\": \"108.0\"}", - "Jurisprudence - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Jurisprudence - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Jurisprudence - # prompt tokens": "{\"description\": \"min=415.861, mean=415.861, max=415.861, sum=831.722 (2)\", \"tab\": \"General information\", \"score\": \"415.8611111111111\"}", - "Jurisprudence - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"jurisprudence\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_jurisprudence\"" - } - } - }, - { - "evaluation_name": "Philosophy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Philosophy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.801, - "details": { - "description": "min=0.801, mean=0.801, max=0.801, sum=1.601 (2)", - "tab": "Accuracy", - "Philosophy - Observed inference time (s)": "{\"description\": \"min=0.432, mean=0.432, max=0.432, sum=0.864 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.43181402736921404\"}", - "Philosophy - # eval": "{\"description\": \"min=311, mean=311, max=311, sum=622 (2)\", \"tab\": \"General information\", \"score\": \"311.0\"}", - "Philosophy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Philosophy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Philosophy - # prompt tokens": "{\"description\": \"min=347.907, mean=347.907, max=347.907, sum=695.814 (2)\", \"tab\": \"General information\", \"score\": \"347.90675241157555\"}", - "Philosophy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"philosophy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_philosophy\"" - } - } - }, - { - "evaluation_name": "Professional Psychology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Professional Psychology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.809, - "details": { - "description": "min=0.809, mean=0.809, max=0.809, sum=1.618 (2)", - "tab": "Accuracy", - "Professional Medicine - Observed inference time (s)": "{\"description\": \"min=0.841, mean=0.841, max=0.841, sum=1.683 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8414969829952016\"}", - "Professional Accounting - Observed inference time (s)": "{\"description\": \"min=0.641, mean=0.641, max=0.641, sum=1.282 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6411697010621957\"}", - "Professional Law - Observed inference time (s)": "{\"description\": \"min=1.161, mean=1.161, max=1.161, sum=2.323 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.1613836899263763\"}", - "Professional Psychology - Observed inference time (s)": "{\"description\": \"min=0.532, mean=0.532, max=0.532, sum=1.064 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5318081830841264\"}", - "Professional Medicine - # eval": "{\"description\": \"min=272, mean=272, max=272, sum=544 (2)\", \"tab\": \"General information\", \"score\": \"272.0\"}", - "Professional Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Medicine - # prompt tokens": "{\"description\": \"min=1166.062, mean=1166.062, max=1166.062, sum=2332.125 (2)\", \"tab\": \"General information\", \"score\": \"1166.0625\"}", - "Professional Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Accounting - # eval": "{\"description\": \"min=282, mean=282, max=282, sum=564 (2)\", \"tab\": \"General information\", \"score\": \"282.0\"}", - "Professional Accounting - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Accounting - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # prompt tokens": "{\"description\": \"min=759.752, mean=759.752, max=759.752, sum=1519.504 (2)\", \"tab\": \"General information\", \"score\": \"759.7517730496454\"}", - "Professional Accounting - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Law - # eval": "{\"description\": \"min=1534, mean=1534, max=1534, sum=3068 (2)\", \"tab\": \"General information\", \"score\": \"1534.0\"}", - "Professional Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # prompt tokens": "{\"description\": \"min=1711.27, mean=1711.27, max=1711.27, sum=3422.54 (2)\", \"tab\": \"General information\", \"score\": \"1711.2698826597132\"}", - "Professional Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Psychology - # eval": "{\"description\": \"min=612, mean=612, max=612, sum=1224 (2)\", \"tab\": \"General information\", \"score\": \"612.0\"}", - "Professional Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # prompt tokens": "{\"description\": \"min=599.475, mean=599.475, max=599.475, sum=1198.951 (2)\", \"tab\": \"General information\", \"score\": \"599.4754901960785\"}", - "Professional Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"professional_psychology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_professional_psychology\"" - } - } - }, - { - "evaluation_name": "Us Foreign Policy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Us Foreign Policy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.91, - "details": { - "description": "min=0.91, mean=0.91, max=0.91, sum=1.82 (2)", - "tab": "Accuracy", - "Us Foreign Policy - Observed inference time (s)": "{\"description\": \"min=0.504, mean=0.504, max=0.504, sum=1.007 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5037446546554566\"}", - "Us Foreign Policy - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Us Foreign Policy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Us Foreign Policy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Us Foreign Policy - # prompt tokens": "{\"description\": \"min=453.51, mean=453.51, max=453.51, sum=907.02 (2)\", \"tab\": \"General information\", \"score\": \"453.51\"}", - "Us Foreign Policy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"us_foreign_policy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_us_foreign_policy\"" - } - } - }, - { - "evaluation_name": "Astronomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Astronomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.822, - "details": { - "description": "min=0.822, mean=0.822, max=0.822, sum=1.645 (2)", - "tab": "Accuracy", - "Astronomy - Observed inference time (s)": "{\"description\": \"min=0.527, mean=0.527, max=0.527, sum=1.054 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5270162303196756\"}", - "Astronomy - # eval": "{\"description\": \"min=152, mean=152, max=152, sum=304 (2)\", \"tab\": \"General information\", \"score\": \"152.0\"}", - "Astronomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Astronomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Astronomy - # prompt tokens": "{\"description\": \"min=613.967, mean=613.967, max=613.967, sum=1227.934 (2)\", \"tab\": \"General information\", \"score\": \"613.9671052631579\"}", - "Astronomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"astronomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_astronomy\"" - } - } - }, - { - "evaluation_name": "Business Ethics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Business Ethics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.86, - "details": { - "description": "min=0.86, mean=0.86, max=0.86, sum=1.72 (2)", - "tab": "Accuracy", - "Business Ethics - Observed inference time (s)": "{\"description\": \"min=0.52, mean=0.52, max=0.52, sum=1.04 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5199160981178284\"}", - "Business Ethics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Business Ethics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Business Ethics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Business Ethics - # prompt tokens": "{\"description\": \"min=575.68, mean=575.68, max=575.68, sum=1151.36 (2)\", \"tab\": \"General information\", \"score\": \"575.68\"}", - "Business Ethics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"business_ethics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_business_ethics\"" - } - } - }, - { - "evaluation_name": "Clinical Knowledge", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Clinical Knowledge", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.785, - "details": { - "description": "min=0.785, mean=0.785, max=0.785, sum=1.57 (2)", - "tab": "Accuracy", - "Clinical Knowledge - Observed inference time (s)": "{\"description\": \"min=0.49, mean=0.49, max=0.49, sum=0.979 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.48968217777756023\"}", - "Clinical Knowledge - # eval": "{\"description\": \"min=265, mean=265, max=265, sum=530 (2)\", \"tab\": \"General information\", \"score\": \"265.0\"}", - "Clinical Knowledge - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Clinical Knowledge - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Clinical Knowledge - # prompt tokens": "{\"description\": \"min=436.902, mean=436.902, max=436.902, sum=873.804 (2)\", \"tab\": \"General information\", \"score\": \"436.90188679245284\"}", - "Clinical Knowledge - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"clinical_knowledge\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_clinical_knowledge\"" - } - } - }, - { - "evaluation_name": "Conceptual Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Conceptual Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.723, - "details": { - "description": "min=0.723, mean=0.723, max=0.723, sum=1.447 (2)", - "tab": "Accuracy", - "Conceptual Physics - Observed inference time (s)": "{\"description\": \"min=0.442, mean=0.442, max=0.442, sum=0.883 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.441747319444697\"}", - "Conceptual Physics - # eval": "{\"description\": \"min=235, mean=235, max=235, sum=470 (2)\", \"tab\": \"General information\", \"score\": \"235.0\"}", - "Conceptual Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Conceptual Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Conceptual Physics - # prompt tokens": "{\"description\": \"min=311.583, mean=311.583, max=311.583, sum=623.166 (2)\", \"tab\": \"General information\", \"score\": \"311.58297872340427\"}", - "Conceptual Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"conceptual_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_conceptual_physics\"" - } - } - }, - { - "evaluation_name": "Electrical Engineering", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Electrical Engineering", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.669, - "details": { - "description": "min=0.669, mean=0.669, max=0.669, sum=1.338 (2)", - "tab": "Accuracy", - "Electrical Engineering - Observed inference time (s)": "{\"description\": \"min=0.498, mean=0.498, max=0.498, sum=0.995 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4975001285816061\"}", - "Electrical Engineering - # eval": "{\"description\": \"min=145, mean=145, max=145, sum=290 (2)\", \"tab\": \"General information\", \"score\": \"145.0\"}", - "Electrical Engineering - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Electrical Engineering - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Electrical Engineering - # prompt tokens": "{\"description\": \"min=476.407, mean=476.407, max=476.407, sum=952.814 (2)\", \"tab\": \"General information\", \"score\": \"476.4068965517241\"}", - "Electrical Engineering - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"electrical_engineering\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_electrical_engineering\"" - } - } - }, - { - "evaluation_name": "Elementary Mathematics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Elementary Mathematics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.548, - "details": { - "description": "min=0.548, mean=0.548, max=0.548, sum=1.095 (2)", - "tab": "Accuracy", - "Elementary Mathematics - Observed inference time (s)": "{\"description\": \"min=0.534, mean=0.534, max=0.534, sum=1.068 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5340847508617179\"}", - "Elementary Mathematics - # eval": "{\"description\": \"min=378, mean=378, max=378, sum=756 (2)\", \"tab\": \"General information\", \"score\": \"378.0\"}", - "Elementary Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Elementary Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Elementary Mathematics - # prompt tokens": "{\"description\": \"min=597.267, mean=597.267, max=597.267, sum=1194.534 (2)\", \"tab\": \"General information\", \"score\": \"597.2671957671957\"}", - "Elementary Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"elementary_mathematics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_elementary_mathematics\"" - } - } - }, - { - "evaluation_name": "Formal Logic", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Formal Logic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.548, - "details": { - "description": "min=0.548, mean=0.548, max=0.548, sum=1.095 (2)", - "tab": "Accuracy", - "Formal Logic - Observed inference time (s)": "{\"description\": \"min=0.555, mean=0.555, max=0.555, sum=1.11 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5548424853218926\"}", - "Formal Logic - # eval": "{\"description\": \"min=126, mean=126, max=126, sum=252 (2)\", \"tab\": \"General information\", \"score\": \"126.0\"}", - "Formal Logic - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Formal Logic - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Formal Logic - # prompt tokens": "{\"description\": \"min=655.698, mean=655.698, max=655.698, sum=1311.397 (2)\", \"tab\": \"General information\", \"score\": \"655.6984126984127\"}", - "Formal Logic - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"formal_logic\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_formal_logic\"" - } - } - }, - { - "evaluation_name": "High School World History", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on High School World History", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.911, - "details": { - "description": "min=0.911, mean=0.911, max=0.911, sum=1.823 (2)", - "tab": "Accuracy", - "High School Biology - Observed inference time (s)": "{\"description\": \"min=0.507, mean=0.507, max=0.507, sum=1.014 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5071036392642606\"}", - "High School Chemistry - Observed inference time (s)": "{\"description\": \"min=0.511, mean=0.511, max=0.511, sum=1.023 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5113655968839899\"}", - "High School Computer Science - Observed inference time (s)": "{\"description\": \"min=0.707, mean=0.707, max=0.707, sum=1.415 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.707279555797577\"}", - "High School European History - Observed inference time (s)": "{\"description\": \"min=1.828, mean=1.828, max=1.828, sum=3.657 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.8283701000791608\"}", - "High School Geography - Observed inference time (s)": "{\"description\": \"min=0.483, mean=0.483, max=0.483, sum=0.967 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.48332409545628713\"}", - "High School Government And Politics - Observed inference time (s)": "{\"description\": \"min=0.483, mean=0.483, max=0.483, sum=0.967 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.48336509719413795\"}", - "High School Macroeconomics - Observed inference time (s)": "{\"description\": \"min=0.486, mean=0.486, max=0.486, sum=0.973 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4863407966418144\"}", - "High School Mathematics - Observed inference time (s)": "{\"description\": \"min=0.531, mean=0.531, max=0.531, sum=1.062 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5308889477341263\"}", - "High School Microeconomics - Observed inference time (s)": "{\"description\": \"min=0.503, mean=0.503, max=0.503, sum=1.006 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.50309332478948\"}", - "High School Physics - Observed inference time (s)": "{\"description\": \"min=0.533, mean=0.533, max=0.533, sum=1.066 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5327805051740432\"}", - "High School Psychology - Observed inference time (s)": "{\"description\": \"min=0.519, mean=0.519, max=0.519, sum=1.039 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5194539997555794\"}", - "High School Statistics - Observed inference time (s)": "{\"description\": \"min=0.662, mean=0.662, max=0.662, sum=1.323 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6615116441691363\"}", - "High School US History - Observed inference time (s)": "{\"description\": \"min=1.442, mean=1.442, max=1.442, sum=2.885 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.4423445556678025\"}", - "High School World History - Observed inference time (s)": "{\"description\": \"min=1.033, mean=1.033, max=1.033, sum=2.067 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.033272183897123\"}", - "High School Biology - # eval": "{\"description\": \"min=310, mean=310, max=310, sum=620 (2)\", \"tab\": \"General information\", \"score\": \"310.0\"}", - "High School Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Biology - # prompt tokens": "{\"description\": \"min=517.116, mean=517.116, max=517.116, sum=1034.232 (2)\", \"tab\": \"General information\", \"score\": \"517.1161290322581\"}", - "High School Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Chemistry - # eval": "{\"description\": \"min=203, mean=203, max=203, sum=406 (2)\", \"tab\": \"General information\", \"score\": \"203.0\"}", - "High School Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # prompt tokens": "{\"description\": \"min=527.305, mean=527.305, max=527.305, sum=1054.611 (2)\", \"tab\": \"General information\", \"score\": \"527.3054187192118\"}", - "High School Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "High School Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # prompt tokens": "{\"description\": \"min=937.05, mean=937.05, max=937.05, sum=1874.1 (2)\", \"tab\": \"General information\", \"score\": \"937.05\"}", - "High School Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School European History - # eval": "{\"description\": \"min=165, mean=165, max=165, sum=330 (2)\", \"tab\": \"General information\", \"score\": \"165.0\"}", - "High School European History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School European History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # prompt tokens": "{\"description\": \"min=2845.339, mean=2845.339, max=2845.339, sum=5690.679 (2)\", \"tab\": \"General information\", \"score\": \"2845.339393939394\"}", - "High School European History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Geography - # eval": "{\"description\": \"min=198, mean=198, max=198, sum=396 (2)\", \"tab\": \"General information\", \"score\": \"198.0\"}", - "High School Geography - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Geography - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # prompt tokens": "{\"description\": \"min=397.934, mean=397.934, max=397.934, sum=795.869 (2)\", \"tab\": \"General information\", \"score\": \"397.9343434343434\"}", - "High School Geography - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Government And Politics - # eval": "{\"description\": \"min=193, mean=193, max=193, sum=386 (2)\", \"tab\": \"General information\", \"score\": \"193.0\"}", - "High School Government And Politics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Government And Politics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # prompt tokens": "{\"description\": \"min=485.57, mean=485.57, max=485.57, sum=971.14 (2)\", \"tab\": \"General information\", \"score\": \"485.5699481865285\"}", - "High School Government And Politics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Macroeconomics - # eval": "{\"description\": \"min=390, mean=390, max=390, sum=780 (2)\", \"tab\": \"General information\", \"score\": \"390.0\"}", - "High School Macroeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Macroeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # prompt tokens": "{\"description\": \"min=396.095, mean=396.095, max=396.095, sum=792.19 (2)\", \"tab\": \"General information\", \"score\": \"396.0948717948718\"}", - "High School Macroeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Mathematics - # eval": "{\"description\": \"min=270, mean=270, max=270, sum=540 (2)\", \"tab\": \"General information\", \"score\": \"270.0\"}", - "High School Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # prompt tokens": "{\"description\": \"min=568.481, mean=568.481, max=568.481, sum=1136.963 (2)\", \"tab\": \"General information\", \"score\": \"568.4814814814815\"}", - "High School Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Microeconomics - # eval": "{\"description\": \"min=238, mean=238, max=238, sum=476 (2)\", \"tab\": \"General information\", \"score\": \"238.0\"}", - "High School Microeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Microeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # prompt tokens": "{\"description\": \"min=416.857, mean=416.857, max=416.857, sum=833.714 (2)\", \"tab\": \"General information\", \"score\": \"416.85714285714283\"}", - "High School Microeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Physics - # eval": "{\"description\": \"min=151, mean=151, max=151, sum=302 (2)\", \"tab\": \"General information\", \"score\": \"151.0\"}", - "High School Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # prompt tokens": "{\"description\": \"min=590.212, mean=590.212, max=590.212, sum=1180.424 (2)\", \"tab\": \"General information\", \"score\": \"590.2119205298013\"}", - "High School Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Psychology - # eval": "{\"description\": \"min=545, mean=545, max=545, sum=1090 (2)\", \"tab\": \"General information\", \"score\": \"545.0\"}", - "High School Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # prompt tokens": "{\"description\": \"min=512.505, mean=512.505, max=512.505, sum=1025.009 (2)\", \"tab\": \"General information\", \"score\": \"512.5045871559633\"}", - "High School Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Statistics - # eval": "{\"description\": \"min=216, mean=216, max=216, sum=432 (2)\", \"tab\": \"General information\", \"score\": \"216.0\"}", - "High School Statistics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Statistics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # prompt tokens": "{\"description\": \"min=861.625, mean=861.625, max=861.625, sum=1723.25 (2)\", \"tab\": \"General information\", \"score\": \"861.625\"}", - "High School Statistics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School US History - # eval": "{\"description\": \"min=204, mean=204, max=204, sum=408 (2)\", \"tab\": \"General information\", \"score\": \"204.0\"}", - "High School US History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School US History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # prompt tokens": "{\"description\": \"min=2259.147, mean=2259.147, max=2259.147, sum=4518.294 (2)\", \"tab\": \"General information\", \"score\": \"2259.1470588235293\"}", - "High School US History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School World History - # eval": "{\"description\": \"min=237, mean=237, max=237, sum=474 (2)\", \"tab\": \"General information\", \"score\": \"237.0\"}", - "High School World History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School World History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # prompt tokens": "{\"description\": \"min=1467.696, mean=1467.696, max=1467.696, sum=2935.392 (2)\", \"tab\": \"General information\", \"score\": \"1467.6962025316457\"}", - "High School World History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"high_school_world_history\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_high_school_world_history\"" - } - } - }, - { - "evaluation_name": "Human Sexuality", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Human Sexuality", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.84, - "details": { - "description": "min=0.84, mean=0.84, max=0.84, sum=1.679 (2)", - "tab": "Accuracy", - "Human Aging - Observed inference time (s)": "{\"description\": \"min=0.434, mean=0.434, max=0.434, sum=0.868 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4340778626668614\"}", - "Human Sexuality - Observed inference time (s)": "{\"description\": \"min=0.499, mean=0.499, max=0.499, sum=0.999 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4992539391262841\"}", - "Human Aging - # eval": "{\"description\": \"min=223, mean=223, max=223, sum=446 (2)\", \"tab\": \"General information\", \"score\": \"223.0\"}", - "Human Aging - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Aging - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Aging - # prompt tokens": "{\"description\": \"min=327.825, mean=327.825, max=327.825, sum=655.65 (2)\", \"tab\": \"General information\", \"score\": \"327.82511210762334\"}", - "Human Aging - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Human Sexuality - # eval": "{\"description\": \"min=131, mean=131, max=131, sum=262 (2)\", \"tab\": \"General information\", \"score\": \"131.0\"}", - "Human Sexuality - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Sexuality - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # prompt tokens": "{\"description\": \"min=366.824, mean=366.824, max=366.824, sum=733.649 (2)\", \"tab\": \"General information\", \"score\": \"366.82442748091603\"}", - "Human Sexuality - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"human_sexuality\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_human_sexuality\"" - } - } - }, - { - "evaluation_name": "International Law", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on International Law", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.851, - "details": { - "description": "min=0.851, mean=0.851, max=0.851, sum=1.702 (2)", - "tab": "Accuracy", - "International Law - Observed inference time (s)": "{\"description\": \"min=0.571, mean=0.571, max=0.571, sum=1.142 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5709604842603699\"}", - "International Law - # eval": "{\"description\": \"min=121, mean=121, max=121, sum=242 (2)\", \"tab\": \"General information\", \"score\": \"121.0\"}", - "International Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "International Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "International Law - # prompt tokens": "{\"description\": \"min=652.669, mean=652.669, max=652.669, sum=1305.339 (2)\", \"tab\": \"General information\", \"score\": \"652.6694214876034\"}", - "International Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"international_law\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_international_law\"" - } - } - }, - { - "evaluation_name": "Logical Fallacies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Logical Fallacies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.847, - "details": { - "description": "min=0.847, mean=0.847, max=0.847, sum=1.693 (2)", - "tab": "Accuracy", - "Logical Fallacies - Observed inference time (s)": "{\"description\": \"min=0.49, mean=0.49, max=0.49, sum=0.98 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.48975605028538616\"}", - "Logical Fallacies - # eval": "{\"description\": \"min=163, mean=163, max=163, sum=326 (2)\", \"tab\": \"General information\", \"score\": \"163.0\"}", - "Logical Fallacies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Logical Fallacies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Logical Fallacies - # prompt tokens": "{\"description\": \"min=452.098, mean=452.098, max=452.098, sum=904.196 (2)\", \"tab\": \"General information\", \"score\": \"452.09815950920245\"}", - "Logical Fallacies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"logical_fallacies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_logical_fallacies\"" - } - } - }, - { - "evaluation_name": "Machine Learning", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Machine Learning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.562, - "details": { - "description": "min=0.562, mean=0.562, max=0.562, sum=1.125 (2)", - "tab": "Accuracy", - "Machine Learning - Observed inference time (s)": "{\"description\": \"min=0.632, mean=0.632, max=0.632, sum=1.264 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6320873349905014\"}", - "Machine Learning - # eval": "{\"description\": \"min=112, mean=112, max=112, sum=224 (2)\", \"tab\": \"General information\", \"score\": \"112.0\"}", - "Machine Learning - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Machine Learning - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Machine Learning - # prompt tokens": "{\"description\": \"min=702.054, mean=702.054, max=702.054, sum=1404.107 (2)\", \"tab\": \"General information\", \"score\": \"702.0535714285714\"}", - "Machine Learning - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"machine_learning\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_machine_learning\"" - } - } - }, - { - "evaluation_name": "Management", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Management", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.903, - "details": { - "description": "min=0.903, mean=0.903, max=0.903, sum=1.806 (2)", - "tab": "Accuracy", - "Management - Observed inference time (s)": "{\"description\": \"min=0.442, mean=0.442, max=0.442, sum=0.883 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4415167558540418\"}", - "Management - # eval": "{\"description\": \"min=103, mean=103, max=103, sum=206 (2)\", \"tab\": \"General information\", \"score\": \"103.0\"}", - "Management - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Management - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Management - # prompt tokens": "{\"description\": \"min=288.437, mean=288.437, max=288.437, sum=576.874 (2)\", \"tab\": \"General information\", \"score\": \"288.43689320388347\"}", - "Management - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"management\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_management\"" - } - } - }, - { - "evaluation_name": "Marketing", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Marketing", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.923, - "details": { - "description": "min=0.923, mean=0.923, max=0.923, sum=1.846 (2)", - "tab": "Accuracy", - "Marketing - Observed inference time (s)": "{\"description\": \"min=0.489, mean=0.489, max=0.489, sum=0.979 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4894245363708235\"}", - "Marketing - # eval": "{\"description\": \"min=234, mean=234, max=234, sum=468 (2)\", \"tab\": \"General information\", \"score\": \"234.0\"}", - "Marketing - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Marketing - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Marketing - # prompt tokens": "{\"description\": \"min=435.603, mean=435.603, max=435.603, sum=871.205 (2)\", \"tab\": \"General information\", \"score\": \"435.6025641025641\"}", - "Marketing - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"marketing\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_marketing\"" - } - } - }, - { - "evaluation_name": "Medical Genetics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Medical Genetics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.73, - "details": { - "description": "min=0.73, mean=0.73, max=0.73, sum=1.46 (2)", - "tab": "Accuracy", - "Medical Genetics - Observed inference time (s)": "{\"description\": \"min=0.474, mean=0.474, max=0.474, sum=0.947 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.47359968423843385\"}", - "Medical Genetics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Medical Genetics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Medical Genetics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Medical Genetics - # prompt tokens": "{\"description\": \"min=357.07, mean=357.07, max=357.07, sum=714.14 (2)\", \"tab\": \"General information\", \"score\": \"357.07\"}", - "Medical Genetics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"medical_genetics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_medical_genetics\"" - } - } - }, - { - "evaluation_name": "Miscellaneous", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Miscellaneous", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.904, - "details": { - "description": "min=0.904, mean=0.904, max=0.904, sum=1.808 (2)", - "tab": "Accuracy", - "Miscellaneous - Observed inference time (s)": "{\"description\": \"min=0.435, mean=0.435, max=0.435, sum=0.871 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4352987403309361\"}", - "Miscellaneous - # eval": "{\"description\": \"min=783, mean=783, max=783, sum=1566 (2)\", \"tab\": \"General information\", \"score\": \"783.0\"}", - "Miscellaneous - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Miscellaneous - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Miscellaneous - # prompt tokens": "{\"description\": \"min=320.964, mean=320.964, max=320.964, sum=641.928 (2)\", \"tab\": \"General information\", \"score\": \"320.9642401021711\"}", - "Miscellaneous - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"miscellaneous\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_miscellaneous\"" - } - } - }, - { - "evaluation_name": "Moral Scenarios", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Moral Scenarios", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.544, - "details": { - "description": "min=0.544, mean=0.544, max=0.544, sum=1.088 (2)", - "tab": "Accuracy", - "Moral Disputes - Observed inference time (s)": "{\"description\": \"min=0.491, mean=0.491, max=0.491, sum=0.983 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.49129951827098867\"}", - "Moral Scenarios - Observed inference time (s)": "{\"description\": \"min=0.583, mean=0.583, max=0.583, sum=1.165 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5826290319751761\"}", - "Moral Disputes - # eval": "{\"description\": \"min=346, mean=346, max=346, sum=692 (2)\", \"tab\": \"General information\", \"score\": \"346.0\"}", - "Moral Disputes - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Disputes - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Disputes - # prompt tokens": "{\"description\": \"min=497.379, mean=497.379, max=497.379, sum=994.757 (2)\", \"tab\": \"General information\", \"score\": \"497.37861271676303\"}", - "Moral Disputes - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Moral Scenarios - # eval": "{\"description\": \"min=895, mean=895, max=895, sum=1790 (2)\", \"tab\": \"General information\", \"score\": \"895.0\"}", - "Moral Scenarios - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Scenarios - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # prompt tokens": "{\"description\": \"min=688.891, mean=688.891, max=688.891, sum=1377.781 (2)\", \"tab\": \"General information\", \"score\": \"688.890502793296\"}", - "Moral Scenarios - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"moral_scenarios\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_moral_scenarios\"" - } - } - }, - { - "evaluation_name": "Nutrition", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Nutrition", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.781, - "details": { - "description": "min=0.781, mean=0.781, max=0.781, sum=1.562 (2)", - "tab": "Accuracy", - "Nutrition - Observed inference time (s)": "{\"description\": \"min=0.548, mean=0.548, max=0.548, sum=1.096 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5477774073095882\"}", - "Nutrition - # eval": "{\"description\": \"min=306, mean=306, max=306, sum=612 (2)\", \"tab\": \"General information\", \"score\": \"306.0\"}", - "Nutrition - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Nutrition - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Nutrition - # prompt tokens": "{\"description\": \"min=619.314, mean=619.314, max=619.314, sum=1238.627 (2)\", \"tab\": \"General information\", \"score\": \"619.3137254901961\"}", - "Nutrition - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"nutrition\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_nutrition\"" - } - } - }, - { - "evaluation_name": "Prehistory", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Prehistory", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.858, - "details": { - "description": "min=0.858, mean=0.858, max=0.858, sum=1.716 (2)", - "tab": "Accuracy", - "Prehistory - Observed inference time (s)": "{\"description\": \"min=0.521, mean=0.521, max=0.521, sum=1.042 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5209115015135871\"}", - "Prehistory - # eval": "{\"description\": \"min=324, mean=324, max=324, sum=648 (2)\", \"tab\": \"General information\", \"score\": \"324.0\"}", - "Prehistory - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Prehistory - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Prehistory - # prompt tokens": "{\"description\": \"min=554.775, mean=554.775, max=554.775, sum=1109.549 (2)\", \"tab\": \"General information\", \"score\": \"554.7746913580247\"}", - "Prehistory - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"prehistory\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_prehistory\"" - } - } - }, - { - "evaluation_name": "Public Relations", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Public Relations", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7, - "details": { - "description": "min=0.7, mean=0.7, max=0.7, sum=1.4 (2)", - "tab": "Accuracy", - "Public Relations - Observed inference time (s)": "{\"description\": \"min=0.473, mean=0.473, max=0.473, sum=0.945 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4725117553364147\"}", - "Public Relations - # eval": "{\"description\": \"min=110, mean=110, max=110, sum=220 (2)\", \"tab\": \"General information\", \"score\": \"110.0\"}", - "Public Relations - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Public Relations - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Public Relations - # prompt tokens": "{\"description\": \"min=431.673, mean=431.673, max=431.673, sum=863.345 (2)\", \"tab\": \"General information\", \"score\": \"431.6727272727273\"}", - "Public Relations - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"public_relations\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_public_relations\"" - } - } - }, - { - "evaluation_name": "Security Studies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Security Studies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.796, - "details": { - "description": "min=0.796, mean=0.796, max=0.796, sum=1.592 (2)", - "tab": "Accuracy", - "Security Studies - Observed inference time (s)": "{\"description\": \"min=0.909, mean=0.909, max=0.909, sum=1.819 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9094535496770119\"}", - "Security Studies - # eval": "{\"description\": \"min=245, mean=245, max=245, sum=490 (2)\", \"tab\": \"General information\", \"score\": \"245.0\"}", - "Security Studies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Security Studies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Security Studies - # prompt tokens": "{\"description\": \"min=1204.906, mean=1204.906, max=1204.906, sum=2409.812 (2)\", \"tab\": \"General information\", \"score\": \"1204.9061224489797\"}", - "Security Studies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"security_studies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_security_studies\"" - } - } - }, - { - "evaluation_name": "Sociology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Sociology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.876, - "details": { - "description": "min=0.876, mean=0.876, max=0.876, sum=1.751 (2)", - "tab": "Accuracy", - "Sociology - Observed inference time (s)": "{\"description\": \"min=0.502, mean=0.502, max=0.502, sum=1.003 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5015075396542525\"}", - "Sociology - # eval": "{\"description\": \"min=201, mean=201, max=201, sum=402 (2)\", \"tab\": \"General information\", \"score\": \"201.0\"}", - "Sociology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Sociology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Sociology - # prompt tokens": "{\"description\": \"min=457.751, mean=457.751, max=457.751, sum=915.502 (2)\", \"tab\": \"General information\", \"score\": \"457.7512437810945\"}", - "Sociology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"sociology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_sociology\"" - } - } - }, - { - "evaluation_name": "Virology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Virology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.554, - "details": { - "description": "min=0.554, mean=0.554, max=0.554, sum=1.108 (2)", - "tab": "Accuracy", - "Virology - Observed inference time (s)": "{\"description\": \"min=0.582, mean=0.582, max=0.582, sum=1.165 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5824309874729938\"}", - "Virology - # eval": "{\"description\": \"min=166, mean=166, max=166, sum=332 (2)\", \"tab\": \"General information\", \"score\": \"166.0\"}", - "Virology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Virology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Virology - # prompt tokens": "{\"description\": \"min=351.434, mean=351.434, max=351.434, sum=702.867 (2)\", \"tab\": \"General information\", \"score\": \"351.43373493975906\"}", - "Virology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"virology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_virology\"" - } - } - }, - { - "evaluation_name": "World Religions", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on World Religions", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.865, - "details": { - "description": "min=0.865, mean=0.865, max=0.865, sum=1.731 (2)", - "tab": "Accuracy", - "World Religions - Observed inference time (s)": "{\"description\": \"min=0.435, mean=0.435, max=0.435, sum=0.87 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.434985329533181\"}", - "World Religions - # eval": "{\"description\": \"min=171, mean=171, max=171, sum=342 (2)\", \"tab\": \"General information\", \"score\": \"171.0\"}", - "World Religions - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "World Religions - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "World Religions - # prompt tokens": "{\"description\": \"min=282.398, mean=282.398, max=282.398, sum=564.795 (2)\", \"tab\": \"General information\", \"score\": \"282.39766081871346\"}", - "World Religions - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"world_religions\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_world_religions\"" - } - } - }, - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.387, - "details": { - "description": "", - "tab": "Efficiency" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_id": "hfopenllm_v2/deepseek-ai_deepseek-llm-67b-chat/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5587 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5243 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0929 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3163 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5059 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3944 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/deepseek-ai_deepseek-llm-7b-base.json b/data/models/deepseek-ai_deepseek-llm-7b-base.json deleted file mode 100644 index b6895906f720e49d1e23e2a1c74aa78740ed6b54..0000000000000000000000000000000000000000 --- a/data/models/deepseek-ai_deepseek-llm-7b-base.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "deepseek-llm-7b-base", - "id": "deepseek-ai/deepseek-llm-7b-base", - "developer": "deepseek-ai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "7.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/deepseek-ai_deepseek-llm-7b-base/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2179 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3503 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0196 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2735 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3738 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1806 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/deepseek-ai_deepseek-llm-7b-chat.json b/data/models/deepseek-ai_deepseek-llm-7b-chat.json deleted file mode 100644 index 894af82a66cca0da63070d8c21bd98d5adba375a..0000000000000000000000000000000000000000 --- a/data/models/deepseek-ai_deepseek-llm-7b-chat.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "deepseek-llm-7b-chat", - "id": "deepseek-ai/deepseek-llm-7b-chat", - "developer": "deepseek-ai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "7.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/deepseek-ai_deepseek-llm-7b-chat/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4171 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3632 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0204 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2659 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4668 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2133 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/deepseek-ai_deepseek-moe-16b-base.json b/data/models/deepseek-ai_deepseek-moe-16b-base.json deleted file mode 100644 index 07cb7e5be1e28cb270860ec68b836d2a8188b646..0000000000000000000000000000000000000000 --- a/data/models/deepseek-ai_deepseek-moe-16b-base.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "deepseek-moe-16b-base", - "id": "deepseek-ai/deepseek-moe-16b-base", - "developer": "deepseek-ai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "DeepseekForCausalLM", - "params_billions": "16.376" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/deepseek-ai_deepseek-moe-16b-base/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.245 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3409 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0242 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2542 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3658 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1505 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/deepseek-ai_deepseek-moe-16b-chat.json b/data/models/deepseek-ai_deepseek-moe-16b-chat.json deleted file mode 100644 index 90df3991366389454c970b4426f583ea703c81dc..0000000000000000000000000000000000000000 --- a/data/models/deepseek-ai_deepseek-moe-16b-chat.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "deepseek-moe-16b-chat", - "id": "deepseek-ai/deepseek-moe-16b-chat", - "developer": "deepseek-ai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "DeepseekForCausalLM", - "params_billions": "16.376" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/deepseek-ai_deepseek-moe-16b-chat/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3663 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3275 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0257 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2248 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3808 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1964 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/deepseek-ai_deepseek-r1-0528.json b/data/models/deepseek-ai_deepseek-r1-0528.json deleted file mode 100644 index 37a44a1a90f8e8b30b60829e33930afb1fa01680..0000000000000000000000000000000000000000 --- a/data/models/deepseek-ai_deepseek-r1-0528.json +++ /dev/null @@ -1,234 +0,0 @@ -{ - "model_info": { - "name": "DeepSeek-R1-0528", - "id": "deepseek-ai/deepseek-r1-0528", - "developer": "deepseek-ai", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_capabilities/deepseek-ai_deepseek-r1-0528/1774096304.5056698", - "retrieved_timestamp": "1774096304.5056698", - "source_metadata": { - "source_name": "helm_capabilities", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_capabilities", - "evaluation_results": [ - { - "evaluation_name": "Mean score", - "source_data": { - "dataset_name": "helm_capabilities", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "The mean of the scores from all columns.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.699, - "details": { - "description": "", - "tab": "Accuracy", - "Mean score - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"115.28182297150872\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU-Pro", - "source_data": { - "dataset_name": "MMLU-Pro", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on MMLU-Pro", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.793, - "details": { - "description": "min=0.793, mean=0.793, max=0.793, sum=0.793 (1)", - "tab": "Accuracy", - "MMLU-Pro - Observed inference time (s)": "{\"description\": \"min=91.015, mean=91.015, max=91.015, sum=91.015 (1)\", \"tab\": \"Efficiency\", \"score\": \"91.01470815229416\"}", - "MMLU-Pro - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "MMLU-Pro - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - # prompt tokens": "{\"description\": \"min=248.757, mean=248.757, max=248.757, sum=248.757 (1)\", \"tab\": \"General information\", \"score\": \"248.757\"}", - "MMLU-Pro - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"all\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.666, - "details": { - "description": "min=0.666, mean=0.666, max=0.666, sum=0.666 (1)", - "tab": "Accuracy", - "GPQA - Observed inference time (s)": "{\"description\": \"min=155.439, mean=155.439, max=155.439, sum=155.439 (1)\", \"tab\": \"Efficiency\", \"score\": \"155.438512681311\"}", - "GPQA - # eval": "{\"description\": \"min=446, mean=446, max=446, sum=446 (1)\", \"tab\": \"General information\", \"score\": \"446.0\"}", - "GPQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - # prompt tokens": "{\"description\": \"min=261.59, mean=261.59, max=261.59, sum=261.59 (1)\", \"tab\": \"General information\", \"score\": \"261.5896860986547\"}", - "GPQA - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"gpqa_main\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "IFEval Strict Acc on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.784, - "details": { - "description": "min=0.784, mean=0.784, max=0.784, sum=0.784 (1)", - "tab": "Accuracy", - "IFEval - Observed inference time (s)": "{\"description\": \"min=33.752, mean=33.752, max=33.752, sum=33.752 (1)\", \"tab\": \"Efficiency\", \"score\": \"33.75197721056489\"}", - "IFEval - # eval": "{\"description\": \"min=541, mean=541, max=541, sum=541 (1)\", \"tab\": \"General information\", \"score\": \"541.0\"}", - "IFEval - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - # prompt tokens": "{\"description\": \"min=46.209, mean=46.209, max=46.209, sum=46.209 (1)\", \"tab\": \"General information\", \"score\": \"46.208872458410355\"}", - "IFEval - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WildBench", - "source_data": { - "dataset_name": "WildBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "WB Score on WildBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.828, - "details": { - "description": "min=0.828, mean=0.828, max=0.828, sum=0.828 (1)", - "tab": "Accuracy", - "WildBench - Observed inference time (s)": "{\"description\": \"min=87.848, mean=87.848, max=87.848, sum=87.848 (1)\", \"tab\": \"Efficiency\", \"score\": \"87.84843708276749\"}", - "WildBench - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "WildBench - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # prompt tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"v2\"" - } - } - }, - { - "evaluation_name": "Omni-MATH", - "source_data": { - "dataset_name": "Omni-MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Acc on Omni-MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.424, - "details": { - "description": "min=0.424, mean=0.424, max=0.424, sum=0.424 (1)", - "tab": "Accuracy", - "Omni-MATH - Observed inference time (s)": "{\"description\": \"min=208.355, mean=208.355, max=208.355, sum=208.355 (1)\", \"tab\": \"Efficiency\", \"score\": \"208.35547973060608\"}", - "Omni-MATH - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "Omni-MATH - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - # prompt tokens": "{\"description\": \"min=107.102, mean=107.102, max=107.102, sum=107.102 (1)\", \"tab\": \"General information\", \"score\": \"107.102\"}", - "Omni-MATH - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/deepseek-ai_deepseek-r1-distill-llama-70b.json b/data/models/deepseek-ai_deepseek-r1-distill-llama-70b.json deleted file mode 100644 index 37e753063dd6df4710d7c3961451b80ceda18114..0000000000000000000000000000000000000000 --- a/data/models/deepseek-ai_deepseek-r1-distill-llama-70b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "DeepSeek-R1-Distill-Llama-70B", - "id": "deepseek-ai/DeepSeek-R1-Distill-Llama-70B", - "developer": "deepseek-ai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "70.554" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/deepseek-ai_DeepSeek-R1-Distill-Llama-70B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4336 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5635 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3074 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2651 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4342 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4748 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/deepseek-ai_deepseek-r1-distill-llama-8b.json b/data/models/deepseek-ai_deepseek-r1-distill-llama-8b.json deleted file mode 100644 index ac8031e2fb9733102d0b6840d6b0cd7b7b5cad08..0000000000000000000000000000000000000000 --- a/data/models/deepseek-ai_deepseek-r1-distill-llama-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "DeepSeek-R1-Distill-Llama-8B", - "id": "deepseek-ai/DeepSeek-R1-Distill-Llama-8B", - "developer": "deepseek-ai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/deepseek-ai_DeepSeek-R1-Distill-Llama-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3782 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3239 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2198 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.255 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.325 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2089 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/deepseek-ai_deepseek-r1-distill-qwen-1.5b.json b/data/models/deepseek-ai_deepseek-r1-distill-qwen-1.5b.json deleted file mode 100644 index 3352d2bbe8acd878db2e0cc49039c581b57c232b..0000000000000000000000000000000000000000 --- a/data/models/deepseek-ai_deepseek-r1-distill-qwen-1.5b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "DeepSeek-R1-Distill-Qwen-1.5B", - "id": "deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B", - "developer": "deepseek-ai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.777" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/deepseek-ai_DeepSeek-R1-Distill-Qwen-1.5B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3463 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3241 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1692 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2559 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3635 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1187 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/deepseek-ai_deepseek-r1-distill-qwen-14b.json b/data/models/deepseek-ai_deepseek-r1-distill-qwen-14b.json deleted file mode 100644 index d20bfede494919e2003488bd2d82277bc0f0ea14..0000000000000000000000000000000000000000 --- a/data/models/deepseek-ai_deepseek-r1-distill-qwen-14b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "DeepSeek-R1-Distill-Qwen-14B", - "id": "deepseek-ai/DeepSeek-R1-Distill-Qwen-14B", - "developer": "deepseek-ai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/deepseek-ai_DeepSeek-R1-Distill-Qwen-14B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4382 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5906 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5702 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3876 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5366 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4667 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/deepseek-ai_deepseek-r1-distill-qwen-32b.json b/data/models/deepseek-ai_deepseek-r1-distill-qwen-32b.json deleted file mode 100644 index 0014a521c9e9ad42150083b3758bd2db16026ac0..0000000000000000000000000000000000000000 --- a/data/models/deepseek-ai_deepseek-r1-distill-qwen-32b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "DeepSeek-R1-Distill-Qwen-32B", - "id": "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B", - "developer": "deepseek-ai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "32.764" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/deepseek-ai_DeepSeek-R1-Distill-Qwen-32B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4186 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4197 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1707 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2844 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4526 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4687 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/deepseek-ai_deepseek-r1-distill-qwen-7b.json b/data/models/deepseek-ai_deepseek-r1-distill-qwen-7b.json deleted file mode 100644 index a55f7d152de3adf845bb01dce56257d68bd3e975..0000000000000000000000000000000000000000 --- a/data/models/deepseek-ai_deepseek-r1-distill-qwen-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "DeepSeek-R1-Distill-Qwen-7B", - "id": "deepseek-ai/DeepSeek-R1-Distill-Qwen-7B", - "developer": "deepseek-ai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/deepseek-ai_DeepSeek-R1-Distill-Qwen-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4038 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3443 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1956 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2794 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3663 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2321 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/deepseek-ai_deepseek-v3.json b/data/models/deepseek-ai_deepseek-v3.json deleted file mode 100644 index 5dd0c3a538e2812219adb43a074eeb6b920e8a87..0000000000000000000000000000000000000000 --- a/data/models/deepseek-ai_deepseek-v3.json +++ /dev/null @@ -1,2127 +0,0 @@ -{ - "model_info": { - "name": "DeepSeek v3", - "id": "deepseek-ai/deepseek-v3", - "developer": "deepseek-ai", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_capabilities/deepseek-ai_deepseek-v3/1774096304.5056698", - "retrieved_timestamp": "1774096304.5056698", - "source_metadata": { - "source_name": "helm_capabilities", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_capabilities", - "evaluation_results": [ - { - "evaluation_name": "Mean score", - "source_data": { - "dataset_name": "helm_capabilities", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "The mean of the scores from all columns.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.665, - "details": { - "description": "", - "tab": "Accuracy", - "Mean score - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"71.88858741677622\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU-Pro", - "source_data": { - "dataset_name": "MMLU-Pro", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on MMLU-Pro", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.723, - "details": { - "description": "min=0.723, mean=0.723, max=0.723, sum=0.723 (1)", - "tab": "Accuracy", - "MMLU-Pro - Observed inference time (s)": "{\"description\": \"min=50.311, mean=50.311, max=50.311, sum=50.311 (1)\", \"tab\": \"Efficiency\", \"score\": \"50.3109582388401\"}", - "MMLU-Pro - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "MMLU-Pro - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - # prompt tokens": "{\"description\": \"min=227.757, mean=227.757, max=227.757, sum=227.757 (1)\", \"tab\": \"General information\", \"score\": \"227.757\"}", - "MMLU-Pro - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"all\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.538, - "details": { - "description": "min=0.538, mean=0.538, max=0.538, sum=0.538 (1)", - "tab": "Accuracy", - "GPQA - Observed inference time (s)": "{\"description\": \"min=74.372, mean=74.372, max=74.372, sum=74.372 (1)\", \"tab\": \"Efficiency\", \"score\": \"74.37158904909553\"}", - "GPQA - # eval": "{\"description\": \"min=446, mean=446, max=446, sum=446 (1)\", \"tab\": \"General information\", \"score\": \"446.0\"}", - "GPQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - # prompt tokens": "{\"description\": \"min=240.59, mean=240.59, max=240.59, sum=240.59 (1)\", \"tab\": \"General information\", \"score\": \"240.5896860986547\"}", - "GPQA - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"gpqa_main\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "IFEval Strict Acc on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.832, - "details": { - "description": "min=0.832, mean=0.832, max=0.832, sum=0.832 (1)", - "tab": "Accuracy", - "IFEval - Observed inference time (s)": "{\"description\": \"min=47.879, mean=47.879, max=47.879, sum=47.879 (1)\", \"tab\": \"Efficiency\", \"score\": \"47.878683835433286\"}", - "IFEval - # eval": "{\"description\": \"min=541, mean=541, max=541, sum=541 (1)\", \"tab\": \"General information\", \"score\": \"541.0\"}", - "IFEval - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - # prompt tokens": "{\"description\": \"min=46.209, mean=46.209, max=46.209, sum=46.209 (1)\", \"tab\": \"General information\", \"score\": \"46.208872458410355\"}", - "IFEval - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WildBench", - "source_data": { - "dataset_name": "WildBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "WB Score on WildBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.831, - "details": { - "description": "min=0.831, mean=0.831, max=0.831, sum=0.831 (1)", - "tab": "Accuracy", - "WildBench - Observed inference time (s)": "{\"description\": \"min=134.163, mean=134.163, max=134.163, sum=134.163 (1)\", \"tab\": \"Efficiency\", \"score\": \"134.1626427116394\"}", - "WildBench - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "WildBench - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # prompt tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"v2\"" - } - } - }, - { - "evaluation_name": "Omni-MATH", - "source_data": { - "dataset_name": "Omni-MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Acc on Omni-MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.403, - "details": { - "description": "min=0.403, mean=0.403, max=0.403, sum=0.403 (1)", - "tab": "Accuracy", - "Omni-MATH - Observed inference time (s)": "{\"description\": \"min=52.719, mean=52.719, max=52.719, sum=52.719 (1)\", \"tab\": \"Efficiency\", \"score\": \"52.71906324887276\"}", - "Omni-MATH - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "Omni-MATH - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - # prompt tokens": "{\"description\": \"min=107.102, mean=107.102, max=107.102, sum=107.102 (1)\", \"tab\": \"General information\", \"score\": \"107.102\"}", - "Omni-MATH - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_lite/deepseek-ai_deepseek-v3/1774096306.427425", - "retrieved_timestamp": "1774096306.427425", - "source_metadata": { - "source_name": "helm_lite", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_lite", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_lite", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.908, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.11454431960049938\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.796, - "details": { - "description": "min=0.796, mean=0.796, max=0.796, sum=0.796 (1)", - "tab": "Accuracy", - "NarrativeQA - Observed inference time (s)": "{\"description\": \"min=6.44, mean=6.44, max=6.44, sum=6.44 (1)\", \"tab\": \"Efficiency\", \"score\": \"6.440373906954913\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=3487.045, mean=3487.045, max=3487.045, sum=3487.045 (1)\", \"tab\": \"General information\", \"score\": \"3487.045070422535\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (closed-book)", - "source_data": { - "dataset_name": "NaturalQuestions (closed-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (closed-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.467, - "details": { - "description": "min=0.467, mean=0.467, max=0.467, sum=0.467 (1)", - "tab": "Accuracy", - "NaturalQuestions (open-book) - Observed inference time (s)": "{\"description\": \"min=5.606, mean=5.606, max=5.606, sum=5.606 (1)\", \"tab\": \"Efficiency\", \"score\": \"5.605930573940277\"}", - "NaturalQuestions (closed-book) - Observed inference time (s)": "{\"description\": \"min=2.183, mean=2.183, max=2.183, sum=2.183 (1)\", \"tab\": \"Efficiency\", \"score\": \"2.1832692058086396\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1756.178, mean=1756.178, max=1756.178, sum=1756.178 (1)\", \"tab\": \"General information\", \"score\": \"1756.178\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=131.205, mean=131.205, max=131.205, sum=131.205 (1)\", \"tab\": \"General information\", \"score\": \"131.205\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "mode": "\"closedbook\"" - } - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.954, - "details": { - "description": "min=0.954, mean=0.954, max=0.954, sum=0.954 (1)", - "tab": "Accuracy", - "OpenbookQA - Observed inference time (s)": "{\"description\": \"min=1.746, mean=1.746, max=1.746, sum=1.746 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.746311339378357\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=245.494, mean=245.494, max=245.494, sum=245.494 (1)\", \"tab\": \"General information\", \"score\": \"245.494\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "dataset": "\"openbookqa\"", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.803, - "details": { - "description": "min=0.65, mean=0.803, max=0.92, sum=4.016 (5)", - "tab": "Accuracy", - "MMLU - Observed inference time (s)": "{\"description\": \"min=0.545, mean=0.564, max=0.585, sum=2.818 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.5636642604125173\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=373.01, mean=465.871, max=613.535, sum=2329.355 (5)\", \"tab\": \"General information\", \"score\": \"465.8710175438597\"}", - "MMLU - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"college_chemistry\", \"computer_security\", \"econometrics\", \"us_foreign_policy\"]", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MATH", - "source_data": { - "dataset_name": "MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Equivalent (CoT) on MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.912, - "details": { - "description": "min=0.816, mean=0.912, max=0.985, sum=6.385 (7)", - "tab": "Accuracy", - "MATH - Observed inference time (s)": "{\"description\": \"min=7.691, mean=9.449, max=13.451, sum=66.142 (7)\", \"tab\": \"Efficiency\", \"score\": \"9.448914254379945\"}", - "MATH - # eval": "{\"description\": \"min=30, mean=62.429, max=135, sum=437 (7)\", \"tab\": \"General information\", \"score\": \"62.42857142857143\"}", - "MATH - # train": "{\"description\": \"min=8, mean=8, max=8, sum=56 (7)\", \"tab\": \"General information\", \"score\": \"8.0\"}", - "MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MATH - # prompt tokens": "{\"description\": \"min=853.923, mean=1245.725, max=2184.846, sum=8720.075 (7)\", \"tab\": \"General information\", \"score\": \"1245.7249665607071\"}", - "MATH - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"algebra\", \"counting_and_probability\", \"geometry\", \"intermediate_algebra\", \"number_theory\", \"prealgebra\", \"precalculus\"]", - "level": "\"1\"", - "use_official_examples": "\"False\"", - "use_chain_of_thought": "\"True\"" - } - } - }, - { - "evaluation_name": "GSM8K", - "source_data": { - "dataset_name": "GSM8K", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on GSM8K", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.94, - "details": { - "description": "min=0.94, mean=0.94, max=0.94, sum=0.94 (1)", - "tab": "Accuracy", - "GSM8K - Observed inference time (s)": "{\"description\": \"min=9.77, mean=9.77, max=9.77, sum=9.77 (1)\", \"tab\": \"Efficiency\", \"score\": \"9.76988450360298\"}", - "GSM8K - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "GSM8K - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "GSM8K - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GSM8K - # prompt tokens": "{\"description\": \"min=961.041, mean=961.041, max=961.041, sum=961.041 (1)\", \"tab\": \"General information\", \"score\": \"961.041\"}", - "GSM8K - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "stop": "\"none\"" - } - } - }, - { - "evaluation_name": "LegalBench", - "source_data": { - "dataset_name": "LegalBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on LegalBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.718, - "details": { - "description": "min=0.425, mean=0.718, max=0.968, sum=3.589 (5)", - "tab": "Accuracy", - "LegalBench - Observed inference time (s)": "{\"description\": \"min=0.566, mean=3.113, max=6.6, sum=15.563 (5)\", \"tab\": \"Efficiency\", \"score\": \"3.1125569474549435\"}", - "LegalBench - # eval": "{\"description\": \"min=95, mean=409.4, max=1000, sum=2047 (5)\", \"tab\": \"General information\", \"score\": \"409.4\"}", - "LegalBench - # train": "{\"description\": \"min=4, mean=4.8, max=5, sum=24 (5)\", \"tab\": \"General information\", \"score\": \"4.8\"}", - "LegalBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "LegalBench - # prompt tokens": "{\"description\": \"min=198.516, mean=1498.765, max=6226.967, sum=7493.826 (5)\", \"tab\": \"General information\", \"score\": \"1498.7652695311654\"}", - "LegalBench - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"abercrombie\", \"corporate_lobbying\", \"function_of_decision_section\", \"international_citizenship_questions\", \"proa\"]" - } - } - }, - { - "evaluation_name": "MedQA", - "source_data": { - "dataset_name": "MedQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MedQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.809, - "details": { - "description": "min=0.809, mean=0.809, max=0.809, sum=0.809 (1)", - "tab": "Accuracy", - "MedQA - Observed inference time (s)": "{\"description\": \"min=1.79, mean=1.79, max=1.79, sum=1.79 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.790037025751224\"}", - "MedQA - # eval": "{\"description\": \"min=503, mean=503, max=503, sum=503 (1)\", \"tab\": \"General information\", \"score\": \"503.0\"}", - "MedQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MedQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MedQA - # prompt tokens": "{\"description\": \"min=985.93, mean=985.93, max=985.93, sum=985.93 (1)\", \"tab\": \"General information\", \"score\": \"985.9304174950298\"}", - "MedQA - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WMT 2014", - "source_data": { - "dataset_name": "WMT 2014", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "BLEU-4 on WMT 2014", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.209, - "details": { - "description": "min=0.163, mean=0.209, max=0.252, sum=1.046 (5)", - "tab": "Accuracy", - "WMT 2014 - Observed inference time (s)": "{\"description\": \"min=2.231, mean=2.677, max=3.02, sum=13.384 (5)\", \"tab\": \"Efficiency\", \"score\": \"2.6768779265693037\"}", - "WMT 2014 - # eval": "{\"description\": \"min=503, mean=568.8, max=832, sum=2844 (5)\", \"tab\": \"General information\", \"score\": \"568.8\"}", - "WMT 2014 - # train": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "WMT 2014 - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WMT 2014 - # prompt tokens": "{\"description\": \"min=103.739, mean=118.596, max=138.616, sum=592.982 (5)\", \"tab\": \"General information\", \"score\": \"118.59634548478361\"}", - "WMT 2014 - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "language_pair": "[\"cs-en\", \"de-en\", \"fr-en\", \"hi-en\", \"ru-en\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_mmlu/deepseek-ai_deepseek-v3/1774096312.00548", - "retrieved_timestamp": "1774096312.00548", - "source_metadata": { - "source_name": "helm_mmlu", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_mmlu", - "evaluation_results": [ - { - "evaluation_name": "MMLU All Subjects", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU All Subjects", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.872, - "details": { - "description": "min=0.596, mean=0.872, max=0.979, sum=99.412 (114)", - "tab": "Accuracy", - "MMLU All Subjects - Observed inference time (s)": "{\"description\": \"min=0.495, mean=1.354, max=6.344, sum=154.309 (114)\", \"tab\": \"Efficiency\", \"score\": \"1.353587049503403\"}", - "MMLU All Subjects - # eval": "{\"description\": \"min=100, mean=246.351, max=1534, sum=28084 (114)\", \"tab\": \"General information\", \"score\": \"246.35087719298247\"}", - "MMLU All Subjects - # train": "{\"description\": \"min=5, mean=5, max=5, sum=570 (114)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU All Subjects - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU All Subjects - # prompt tokens": "{\"description\": \"min=268.918, mean=607.861, max=2773.188, sum=69296.195 (114)\", \"tab\": \"General information\", \"score\": \"607.8613565650774\"}", - "MMLU All Subjects - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_name": "Abstract Algebra", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Abstract Algebra", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.84, - "details": { - "description": "min=0.84, mean=0.84, max=0.84, sum=1.68 (2)", - "tab": "Accuracy", - "Abstract Algebra - Observed inference time (s)": "{\"description\": \"min=0.585, mean=0.585, max=0.585, sum=1.171 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5853858423233033\"}", - "Abstract Algebra - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Abstract Algebra - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Abstract Algebra - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Abstract Algebra - # prompt tokens": "{\"description\": \"min=373.01, mean=373.01, max=373.01, sum=746.02 (2)\", \"tab\": \"General information\", \"score\": \"373.01\"}", - "Abstract Algebra - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"abstract_algebra\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_abstract_algebra\"" - } - } - }, - { - "evaluation_name": "Anatomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Anatomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.867, - "details": { - "description": "min=0.867, mean=0.867, max=0.867, sum=1.733 (2)", - "tab": "Accuracy", - "Anatomy - Observed inference time (s)": "{\"description\": \"min=1.804, mean=1.804, max=1.804, sum=3.607 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.8037012683020697\"}", - "Anatomy - # eval": "{\"description\": \"min=135, mean=135, max=135, sum=270 (2)\", \"tab\": \"General information\", \"score\": \"135.0\"}", - "Anatomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Anatomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Anatomy - # prompt tokens": "{\"description\": \"min=332.119, mean=332.119, max=332.119, sum=664.237 (2)\", \"tab\": \"General information\", \"score\": \"332.1185185185185\"}", - "Anatomy - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"anatomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_anatomy\"" - } - } - }, - { - "evaluation_name": "College Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on College Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.814, - "details": { - "description": "min=0.814, mean=0.814, max=0.814, sum=1.627 (2)", - "tab": "Accuracy", - "College Chemistry - Observed inference time (s)": "{\"description\": \"min=0.557, mean=0.557, max=0.557, sum=1.113 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5567307829856872\"}", - "College Biology - Observed inference time (s)": "{\"description\": \"min=0.776, mean=0.776, max=0.776, sum=1.553 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7763584835661782\"}", - "College Computer Science - Observed inference time (s)": "{\"description\": \"min=0.505, mean=0.505, max=0.505, sum=1.01 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5047655653953552\"}", - "College Mathematics - Observed inference time (s)": "{\"description\": \"min=0.495, mean=0.495, max=0.495, sum=0.989 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4945454502105713\"}", - "College Medicine - Observed inference time (s)": "{\"description\": \"min=1.811, mean=1.811, max=1.811, sum=3.623 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.8114735322191535\"}", - "College Physics - Observed inference time (s)": "{\"description\": \"min=6.344, mean=6.344, max=6.344, sum=12.687 (2)\", \"tab\": \"Efficiency\", \"score\": \"6.343635446885052\"}", - "College Chemistry - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Chemistry - # prompt tokens": "{\"description\": \"min=541.32, mean=541.32, max=541.32, sum=1082.64 (2)\", \"tab\": \"General information\", \"score\": \"541.32\"}", - "College Chemistry - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # eval": "{\"description\": \"min=144, mean=144, max=144, sum=288 (2)\", \"tab\": \"General information\", \"score\": \"144.0\"}", - "College Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # prompt tokens": "{\"description\": \"min=456.201, mean=456.201, max=456.201, sum=912.403 (2)\", \"tab\": \"General information\", \"score\": \"456.2013888888889\"}", - "College Biology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # prompt tokens": "{\"description\": \"min=828.34, mean=828.34, max=828.34, sum=1656.68 (2)\", \"tab\": \"General information\", \"score\": \"828.34\"}", - "College Computer Science - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # prompt tokens": "{\"description\": \"min=592.74, mean=592.74, max=592.74, sum=1185.48 (2)\", \"tab\": \"General information\", \"score\": \"592.74\"}", - "College Mathematics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # eval": "{\"description\": \"min=173, mean=173, max=173, sum=346 (2)\", \"tab\": \"General information\", \"score\": \"173.0\"}", - "College Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # prompt tokens": "{\"description\": \"min=486.971, mean=486.971, max=486.971, sum=973.942 (2)\", \"tab\": \"General information\", \"score\": \"486.97109826589593\"}", - "College Medicine - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # eval": "{\"description\": \"min=102, mean=102, max=102, sum=204 (2)\", \"tab\": \"General information\", \"score\": \"102.0\"}", - "College Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # prompt tokens": "{\"description\": \"min=492.804, mean=492.804, max=492.804, sum=985.608 (2)\", \"tab\": \"General information\", \"score\": \"492.80392156862746\"}", - "College Physics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"college_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_college_physics\"" - } - } - }, - { - "evaluation_name": "Computer Security", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Computer Security", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.86, - "details": { - "description": "min=0.86, mean=0.86, max=0.86, sum=1.72 (2)", - "tab": "Accuracy", - "Computer Security - Observed inference time (s)": "{\"description\": \"min=0.545, mean=0.545, max=0.545, sum=1.089 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5446710443496704\"}", - "Computer Security - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Computer Security - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Computer Security - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Computer Security - # prompt tokens": "{\"description\": \"min=375.06, mean=375.06, max=375.06, sum=750.12 (2)\", \"tab\": \"General information\", \"score\": \"375.06\"}", - "Computer Security - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"computer_security\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_computer_security\"" - } - } - }, - { - "evaluation_name": "Econometrics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Econometrics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.746, - "details": { - "description": "min=0.746, mean=0.746, max=0.746, sum=1.491 (2)", - "tab": "Accuracy", - "Econometrics - Observed inference time (s)": "{\"description\": \"min=0.554, mean=0.554, max=0.554, sum=1.107 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5537264849010267\"}", - "Econometrics - # eval": "{\"description\": \"min=114, mean=114, max=114, sum=228 (2)\", \"tab\": \"General information\", \"score\": \"114.0\"}", - "Econometrics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Econometrics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Econometrics - # prompt tokens": "{\"description\": \"min=613.535, mean=613.535, max=613.535, sum=1227.07 (2)\", \"tab\": \"General information\", \"score\": \"613.5350877192982\"}", - "Econometrics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"econometrics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_econometrics\"" - } - } - }, - { - "evaluation_name": "Global Facts", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Global Facts", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.68, - "details": { - "description": "min=0.68, mean=0.68, max=0.68, sum=1.36 (2)", - "tab": "Accuracy", - "Global Facts - Observed inference time (s)": "{\"description\": \"min=0.978, mean=0.978, max=0.978, sum=1.955 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9775782990455627\"}", - "Global Facts - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Global Facts - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Global Facts - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Global Facts - # prompt tokens": "{\"description\": \"min=398.63, mean=398.63, max=398.63, sum=797.26 (2)\", \"tab\": \"General information\", \"score\": \"398.63\"}", - "Global Facts - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"global_facts\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_global_facts\"" - } - } - }, - { - "evaluation_name": "Jurisprudence", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Jurisprudence", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.898, - "details": { - "description": "min=0.898, mean=0.898, max=0.898, sum=1.796 (2)", - "tab": "Accuracy", - "Jurisprudence - Observed inference time (s)": "{\"description\": \"min=0.834, mean=0.834, max=0.834, sum=1.668 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8338986083313271\"}", - "Jurisprudence - # eval": "{\"description\": \"min=108, mean=108, max=108, sum=216 (2)\", \"tab\": \"General information\", \"score\": \"108.0\"}", - "Jurisprudence - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Jurisprudence - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Jurisprudence - # prompt tokens": "{\"description\": \"min=387.694, mean=387.694, max=387.694, sum=775.389 (2)\", \"tab\": \"General information\", \"score\": \"387.69444444444446\"}", - "Jurisprudence - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"jurisprudence\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_jurisprudence\"" - } - } - }, - { - "evaluation_name": "Philosophy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Philosophy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9, - "details": { - "description": "min=0.9, mean=0.9, max=0.9, sum=1.801 (2)", - "tab": "Accuracy", - "Philosophy - Observed inference time (s)": "{\"description\": \"min=0.836, mean=0.836, max=0.836, sum=1.673 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.836391413710125\"}", - "Philosophy - # eval": "{\"description\": \"min=311, mean=311, max=311, sum=622 (2)\", \"tab\": \"General information\", \"score\": \"311.0\"}", - "Philosophy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Philosophy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Philosophy - # prompt tokens": "{\"description\": \"min=323.569, mean=323.569, max=323.569, sum=647.138 (2)\", \"tab\": \"General information\", \"score\": \"323.56913183279744\"}", - "Philosophy - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"philosophy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_philosophy\"" - } - } - }, - { - "evaluation_name": "Professional Psychology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Professional Psychology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.887, - "details": { - "description": "min=0.887, mean=0.887, max=0.887, sum=1.775 (2)", - "tab": "Accuracy", - "Professional Medicine - Observed inference time (s)": "{\"description\": \"min=0.635, mean=0.635, max=0.635, sum=1.269 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6345776915550232\"}", - "Professional Accounting - Observed inference time (s)": "{\"description\": \"min=1.224, mean=1.224, max=1.224, sum=2.448 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.2240875671941338\"}", - "Professional Law - Observed inference time (s)": "{\"description\": \"min=0.707, mean=0.707, max=0.707, sum=1.413 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7066206168941911\"}", - "Professional Psychology - Observed inference time (s)": "{\"description\": \"min=0.672, mean=0.672, max=0.672, sum=1.345 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6723053728053773\"}", - "Professional Medicine - # eval": "{\"description\": \"min=272, mean=272, max=272, sum=544 (2)\", \"tab\": \"General information\", \"score\": \"272.0\"}", - "Professional Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Medicine - # prompt tokens": "{\"description\": \"min=1052.765, mean=1052.765, max=1052.765, sum=2105.529 (2)\", \"tab\": \"General information\", \"score\": \"1052.764705882353\"}", - "Professional Medicine - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # eval": "{\"description\": \"min=282, mean=282, max=282, sum=564 (2)\", \"tab\": \"General information\", \"score\": \"282.0\"}", - "Professional Accounting - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Accounting - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # prompt tokens": "{\"description\": \"min=659.613, mean=659.613, max=659.613, sum=1319.227 (2)\", \"tab\": \"General information\", \"score\": \"659.613475177305\"}", - "Professional Accounting - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # eval": "{\"description\": \"min=1534, mean=1534, max=1534, sum=3068 (2)\", \"tab\": \"General information\", \"score\": \"1534.0\"}", - "Professional Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # prompt tokens": "{\"description\": \"min=1629.421, mean=1629.421, max=1629.421, sum=3258.842 (2)\", \"tab\": \"General information\", \"score\": \"1629.4211212516298\"}", - "Professional Law - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # eval": "{\"description\": \"min=612, mean=612, max=612, sum=1224 (2)\", \"tab\": \"General information\", \"score\": \"612.0\"}", - "Professional Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # prompt tokens": "{\"description\": \"min=574.508, mean=574.508, max=574.508, sum=1149.016 (2)\", \"tab\": \"General information\", \"score\": \"574.5081699346405\"}", - "Professional Psychology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"professional_psychology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_professional_psychology\"" - } - } - }, - { - "evaluation_name": "Us Foreign Policy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Us Foreign Policy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.92, - "details": { - "description": "min=0.92, mean=0.92, max=0.92, sum=1.84 (2)", - "tab": "Accuracy", - "Us Foreign Policy - Observed inference time (s)": "{\"description\": \"min=0.578, mean=0.578, max=0.578, sum=1.156 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5778071475028992\"}", - "Us Foreign Policy - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Us Foreign Policy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Us Foreign Policy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Us Foreign Policy - # prompt tokens": "{\"description\": \"min=426.43, mean=426.43, max=426.43, sum=852.86 (2)\", \"tab\": \"General information\", \"score\": \"426.43\"}", - "Us Foreign Policy - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"us_foreign_policy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_us_foreign_policy\"" - } - } - }, - { - "evaluation_name": "Astronomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Astronomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.921, - "details": { - "description": "min=0.921, mean=0.921, max=0.921, sum=1.842 (2)", - "tab": "Accuracy", - "Astronomy - Observed inference time (s)": "{\"description\": \"min=0.681, mean=0.681, max=0.681, sum=1.363 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6812541327978435\"}", - "Astronomy - # eval": "{\"description\": \"min=152, mean=152, max=152, sum=304 (2)\", \"tab\": \"General information\", \"score\": \"152.0\"}", - "Astronomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Astronomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Astronomy - # prompt tokens": "{\"description\": \"min=575.836, mean=575.836, max=575.836, sum=1151.671 (2)\", \"tab\": \"General information\", \"score\": \"575.8355263157895\"}", - "Astronomy - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"astronomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_astronomy\"" - } - } - }, - { - "evaluation_name": "Business Ethics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Business Ethics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.89, - "details": { - "description": "min=0.89, mean=0.89, max=0.89, sum=1.78 (2)", - "tab": "Accuracy", - "Business Ethics - Observed inference time (s)": "{\"description\": \"min=4.691, mean=4.691, max=4.691, sum=9.381 (2)\", \"tab\": \"Efficiency\", \"score\": \"4.690641319751739\"}", - "Business Ethics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Business Ethics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Business Ethics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Business Ethics - # prompt tokens": "{\"description\": \"min=550.46, mean=550.46, max=550.46, sum=1100.92 (2)\", \"tab\": \"General information\", \"score\": \"550.46\"}", - "Business Ethics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"business_ethics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_business_ethics\"" - } - } - }, - { - "evaluation_name": "Clinical Knowledge", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Clinical Knowledge", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.913, - "details": { - "description": "min=0.913, mean=0.913, max=0.913, sum=1.826 (2)", - "tab": "Accuracy", - "Clinical Knowledge - Observed inference time (s)": "{\"description\": \"min=0.906, mean=0.906, max=0.906, sum=1.812 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9061050837894655\"}", - "Clinical Knowledge - # eval": "{\"description\": \"min=265, mean=265, max=265, sum=530 (2)\", \"tab\": \"General information\", \"score\": \"265.0\"}", - "Clinical Knowledge - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Clinical Knowledge - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Clinical Knowledge - # prompt tokens": "{\"description\": \"min=387.449, mean=387.449, max=387.449, sum=774.898 (2)\", \"tab\": \"General information\", \"score\": \"387.4490566037736\"}", - "Clinical Knowledge - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"clinical_knowledge\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_clinical_knowledge\"" - } - } - }, - { - "evaluation_name": "Conceptual Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Conceptual Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.94, - "details": { - "description": "min=0.94, mean=0.94, max=0.94, sum=1.881 (2)", - "tab": "Accuracy", - "Conceptual Physics - Observed inference time (s)": "{\"description\": \"min=0.627, mean=0.627, max=0.627, sum=1.253 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6267383788494354\"}", - "Conceptual Physics - # eval": "{\"description\": \"min=235, mean=235, max=235, sum=470 (2)\", \"tab\": \"General information\", \"score\": \"235.0\"}", - "Conceptual Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Conceptual Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Conceptual Physics - # prompt tokens": "{\"description\": \"min=300.591, mean=300.591, max=300.591, sum=601.183 (2)\", \"tab\": \"General information\", \"score\": \"300.59148936170214\"}", - "Conceptual Physics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"conceptual_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_conceptual_physics\"" - } - } - }, - { - "evaluation_name": "Electrical Engineering", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Electrical Engineering", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.869, - "details": { - "description": "min=0.869, mean=0.869, max=0.869, sum=1.738 (2)", - "tab": "Accuracy", - "Electrical Engineering - Observed inference time (s)": "{\"description\": \"min=2.459, mean=2.459, max=2.459, sum=4.918 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.4591504623150002\"}", - "Electrical Engineering - # eval": "{\"description\": \"min=145, mean=145, max=145, sum=290 (2)\", \"tab\": \"General information\", \"score\": \"145.0\"}", - "Electrical Engineering - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Electrical Engineering - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Electrical Engineering - # prompt tokens": "{\"description\": \"min=431.91, mean=431.91, max=431.91, sum=863.821 (2)\", \"tab\": \"General information\", \"score\": \"431.9103448275862\"}", - "Electrical Engineering - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"electrical_engineering\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_electrical_engineering\"" - } - } - }, - { - "evaluation_name": "Elementary Mathematics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Elementary Mathematics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.942, - "details": { - "description": "min=0.942, mean=0.942, max=0.942, sum=1.884 (2)", - "tab": "Accuracy", - "Elementary Mathematics - Observed inference time (s)": "{\"description\": \"min=1.651, mean=1.651, max=1.651, sum=3.301 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.650515148879359\"}", - "Elementary Mathematics - # eval": "{\"description\": \"min=378, mean=378, max=378, sum=756 (2)\", \"tab\": \"General information\", \"score\": \"378.0\"}", - "Elementary Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Elementary Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Elementary Mathematics - # prompt tokens": "{\"description\": \"min=531.693, mean=531.693, max=531.693, sum=1063.386 (2)\", \"tab\": \"General information\", \"score\": \"531.6931216931217\"}", - "Elementary Mathematics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"elementary_mathematics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_elementary_mathematics\"" - } - } - }, - { - "evaluation_name": "Formal Logic", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Formal Logic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.77, - "details": { - "description": "min=0.77, mean=0.77, max=0.77, sum=1.54 (2)", - "tab": "Accuracy", - "Formal Logic - Observed inference time (s)": "{\"description\": \"min=0.513, mean=0.513, max=0.513, sum=1.026 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5130742864003257\"}", - "Formal Logic - # eval": "{\"description\": \"min=126, mean=126, max=126, sum=252 (2)\", \"tab\": \"General information\", \"score\": \"126.0\"}", - "Formal Logic - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Formal Logic - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Formal Logic - # prompt tokens": "{\"description\": \"min=604.119, mean=604.119, max=604.119, sum=1208.238 (2)\", \"tab\": \"General information\", \"score\": \"604.1190476190476\"}", - "Formal Logic - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"formal_logic\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_formal_logic\"" - } - } - }, - { - "evaluation_name": "High School World History", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on High School World History", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.928, - "details": { - "description": "min=0.928, mean=0.928, max=0.928, sum=1.857 (2)", - "tab": "Accuracy", - "High School Biology - Observed inference time (s)": "{\"description\": \"min=2.647, mean=2.647, max=2.647, sum=5.294 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.6472030393538937\"}", - "High School Chemistry - Observed inference time (s)": "{\"description\": \"min=3.847, mean=3.847, max=3.847, sum=7.695 (2)\", \"tab\": \"Efficiency\", \"score\": \"3.8474940337571018\"}", - "High School Computer Science - Observed inference time (s)": "{\"description\": \"min=2.761, mean=2.761, max=2.761, sum=5.523 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.7613840389251707\"}", - "High School European History - Observed inference time (s)": "{\"description\": \"min=1.944, mean=1.944, max=1.944, sum=3.888 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.9442455436244155\"}", - "High School Geography - Observed inference time (s)": "{\"description\": \"min=0.607, mean=0.607, max=0.607, sum=1.215 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6073213755482375\"}", - "High School Government And Politics - Observed inference time (s)": "{\"description\": \"min=2.403, mean=2.403, max=2.403, sum=4.805 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.4025608480285485\"}", - "High School Macroeconomics - Observed inference time (s)": "{\"description\": \"min=0.654, mean=0.654, max=0.654, sum=1.308 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6539444972307255\"}", - "High School Mathematics - Observed inference time (s)": "{\"description\": \"min=2.285, mean=2.285, max=2.285, sum=4.57 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.285083364557337\"}", - "High School Microeconomics - Observed inference time (s)": "{\"description\": \"min=1.265, mean=1.265, max=1.265, sum=2.531 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.2653034544792496\"}", - "High School Physics - Observed inference time (s)": "{\"description\": \"min=1.036, mean=1.036, max=1.036, sum=2.072 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.0361600064283965\"}", - "High School Psychology - Observed inference time (s)": "{\"description\": \"min=1.658, mean=1.658, max=1.658, sum=3.315 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.6576398372650147\"}", - "High School Statistics - Observed inference time (s)": "{\"description\": \"min=0.513, mean=0.513, max=0.513, sum=1.027 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5133153398831686\"}", - "High School US History - Observed inference time (s)": "{\"description\": \"min=0.791, mean=0.791, max=0.791, sum=1.582 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7908881224837958\"}", - "High School World History - Observed inference time (s)": "{\"description\": \"min=1.65, mean=1.65, max=1.65, sum=3.301 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.6504118030081318\"}", - "High School Biology - # eval": "{\"description\": \"min=310, mean=310, max=310, sum=620 (2)\", \"tab\": \"General information\", \"score\": \"310.0\"}", - "High School Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Biology - # prompt tokens": "{\"description\": \"min=505.561, mean=505.561, max=505.561, sum=1011.123 (2)\", \"tab\": \"General information\", \"score\": \"505.56129032258065\"}", - "High School Biology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # eval": "{\"description\": \"min=203, mean=203, max=203, sum=406 (2)\", \"tab\": \"General information\", \"score\": \"203.0\"}", - "High School Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # prompt tokens": "{\"description\": \"min=479.32, mean=479.32, max=479.32, sum=958.64 (2)\", \"tab\": \"General information\", \"score\": \"479.320197044335\"}", - "High School Chemistry - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "High School Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # prompt tokens": "{\"description\": \"min=871.42, mean=871.42, max=871.42, sum=1742.84 (2)\", \"tab\": \"General information\", \"score\": \"871.42\"}", - "High School Computer Science - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # eval": "{\"description\": \"min=165, mean=165, max=165, sum=330 (2)\", \"tab\": \"General information\", \"score\": \"165.0\"}", - "High School European History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School European History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # prompt tokens": "{\"description\": \"min=2773.188, mean=2773.188, max=2773.188, sum=5546.376 (2)\", \"tab\": \"General information\", \"score\": \"2773.1878787878786\"}", - "High School European History - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # eval": "{\"description\": \"min=198, mean=198, max=198, sum=396 (2)\", \"tab\": \"General information\", \"score\": \"198.0\"}", - "High School Geography - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Geography - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # prompt tokens": "{\"description\": \"min=369.53, mean=369.53, max=369.53, sum=739.061 (2)\", \"tab\": \"General information\", \"score\": \"369.530303030303\"}", - "High School Geography - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # eval": "{\"description\": \"min=193, mean=193, max=193, sum=386 (2)\", \"tab\": \"General information\", \"score\": \"193.0\"}", - "High School Government And Politics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Government And Politics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # prompt tokens": "{\"description\": \"min=463.767, mean=463.767, max=463.767, sum=927.534 (2)\", \"tab\": \"General information\", \"score\": \"463.76683937823833\"}", - "High School Government And Politics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # eval": "{\"description\": \"min=390, mean=390, max=390, sum=780 (2)\", \"tab\": \"General information\", \"score\": \"390.0\"}", - "High School Macroeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Macroeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # prompt tokens": "{\"description\": \"min=370.418, mean=370.418, max=370.418, sum=740.836 (2)\", \"tab\": \"General information\", \"score\": \"370.4179487179487\"}", - "High School Macroeconomics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # eval": "{\"description\": \"min=270, mean=270, max=270, sum=540 (2)\", \"tab\": \"General information\", \"score\": \"270.0\"}", - "High School Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # prompt tokens": "{\"description\": \"min=520.57, mean=520.57, max=520.57, sum=1041.141 (2)\", \"tab\": \"General information\", \"score\": \"520.5703703703704\"}", - "High School Mathematics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # eval": "{\"description\": \"min=238, mean=238, max=238, sum=476 (2)\", \"tab\": \"General information\", \"score\": \"238.0\"}", - "High School Microeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Microeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # prompt tokens": "{\"description\": \"min=399.782, mean=399.782, max=399.782, sum=799.563 (2)\", \"tab\": \"General information\", \"score\": \"399.781512605042\"}", - "High School Microeconomics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # eval": "{\"description\": \"min=151, mean=151, max=151, sum=302 (2)\", \"tab\": \"General information\", \"score\": \"151.0\"}", - "High School Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # prompt tokens": "{\"description\": \"min=559.967, mean=559.967, max=559.967, sum=1119.934 (2)\", \"tab\": \"General information\", \"score\": \"559.9668874172186\"}", - "High School Physics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # eval": "{\"description\": \"min=545, mean=545, max=545, sum=1090 (2)\", \"tab\": \"General information\", \"score\": \"545.0\"}", - "High School Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # prompt tokens": "{\"description\": \"min=480.22, mean=480.22, max=480.22, sum=960.44 (2)\", \"tab\": \"General information\", \"score\": \"480.2201834862385\"}", - "High School Psychology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # eval": "{\"description\": \"min=216, mean=216, max=216, sum=432 (2)\", \"tab\": \"General information\", \"score\": \"216.0\"}", - "High School Statistics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Statistics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # prompt tokens": "{\"description\": \"min=796.333, mean=796.333, max=796.333, sum=1592.667 (2)\", \"tab\": \"General information\", \"score\": \"796.3333333333334\"}", - "High School Statistics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # eval": "{\"description\": \"min=204, mean=204, max=204, sum=408 (2)\", \"tab\": \"General information\", \"score\": \"204.0\"}", - "High School US History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School US History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # prompt tokens": "{\"description\": \"min=2202.103, mean=2202.103, max=2202.103, sum=4404.206 (2)\", \"tab\": \"General information\", \"score\": \"2202.1029411764707\"}", - "High School US History - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # eval": "{\"description\": \"min=237, mean=237, max=237, sum=474 (2)\", \"tab\": \"General information\", \"score\": \"237.0\"}", - "High School World History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School World History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # prompt tokens": "{\"description\": \"min=1403.051, mean=1403.051, max=1403.051, sum=2806.101 (2)\", \"tab\": \"General information\", \"score\": \"1403.0506329113923\"}", - "High School World History - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"high_school_world_history\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_high_school_world_history\"" - } - } - }, - { - "evaluation_name": "Human Sexuality", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Human Sexuality", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.924, - "details": { - "description": "min=0.924, mean=0.924, max=0.924, sum=1.847 (2)", - "tab": "Accuracy", - "Human Aging - Observed inference time (s)": "{\"description\": \"min=0.553, mean=0.553, max=0.553, sum=1.106 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5531257503235821\"}", - "Human Sexuality - Observed inference time (s)": "{\"description\": \"min=0.511, mean=0.511, max=0.511, sum=1.022 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5109815524734613\"}", - "Human Aging - # eval": "{\"description\": \"min=223, mean=223, max=223, sum=446 (2)\", \"tab\": \"General information\", \"score\": \"223.0\"}", - "Human Aging - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Aging - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Aging - # prompt tokens": "{\"description\": \"min=315.26, mean=315.26, max=315.26, sum=630.52 (2)\", \"tab\": \"General information\", \"score\": \"315.26008968609864\"}", - "Human Aging - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # eval": "{\"description\": \"min=131, mean=131, max=131, sum=262 (2)\", \"tab\": \"General information\", \"score\": \"131.0\"}", - "Human Sexuality - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Sexuality - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # prompt tokens": "{\"description\": \"min=341.29, mean=341.29, max=341.29, sum=682.58 (2)\", \"tab\": \"General information\", \"score\": \"341.29007633587787\"}", - "Human Sexuality - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"human_sexuality\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_human_sexuality\"" - } - } - }, - { - "evaluation_name": "International Law", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on International Law", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.95, - "details": { - "description": "min=0.95, mean=0.95, max=0.95, sum=1.901 (2)", - "tab": "Accuracy", - "International Law - Observed inference time (s)": "{\"description\": \"min=0.886, mean=0.886, max=0.886, sum=1.772 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8861682651456723\"}", - "International Law - # eval": "{\"description\": \"min=121, mean=121, max=121, sum=242 (2)\", \"tab\": \"General information\", \"score\": \"121.0\"}", - "International Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "International Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "International Law - # prompt tokens": "{\"description\": \"min=639.339, mean=639.339, max=639.339, sum=1278.678 (2)\", \"tab\": \"General information\", \"score\": \"639.3388429752066\"}", - "International Law - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"international_law\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_international_law\"" - } - } - }, - { - "evaluation_name": "Logical Fallacies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Logical Fallacies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.914, - "details": { - "description": "min=0.914, mean=0.914, max=0.914, sum=1.828 (2)", - "tab": "Accuracy", - "Logical Fallacies - Observed inference time (s)": "{\"description\": \"min=0.919, mean=0.919, max=0.919, sum=1.838 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9191862732354849\"}", - "Logical Fallacies - # eval": "{\"description\": \"min=163, mean=163, max=163, sum=326 (2)\", \"tab\": \"General information\", \"score\": \"163.0\"}", - "Logical Fallacies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Logical Fallacies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Logical Fallacies - # prompt tokens": "{\"description\": \"min=442.239, mean=442.239, max=442.239, sum=884.479 (2)\", \"tab\": \"General information\", \"score\": \"442.23926380368096\"}", - "Logical Fallacies - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"logical_fallacies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_logical_fallacies\"" - } - } - }, - { - "evaluation_name": "Machine Learning", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Machine Learning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.786, - "details": { - "description": "min=0.786, mean=0.786, max=0.786, sum=1.571 (2)", - "tab": "Accuracy", - "Machine Learning - Observed inference time (s)": "{\"description\": \"min=0.518, mean=0.518, max=0.518, sum=1.036 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5179938631398338\"}", - "Machine Learning - # eval": "{\"description\": \"min=112, mean=112, max=112, sum=224 (2)\", \"tab\": \"General information\", \"score\": \"112.0\"}", - "Machine Learning - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Machine Learning - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Machine Learning - # prompt tokens": "{\"description\": \"min=666.277, mean=666.277, max=666.277, sum=1332.554 (2)\", \"tab\": \"General information\", \"score\": \"666.2767857142857\"}", - "Machine Learning - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"machine_learning\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_machine_learning\"" - } - } - }, - { - "evaluation_name": "Management", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Management", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.903, - "details": { - "description": "min=0.903, mean=0.903, max=0.903, sum=1.806 (2)", - "tab": "Accuracy", - "Management - Observed inference time (s)": "{\"description\": \"min=4.248, mean=4.248, max=4.248, sum=8.497 (2)\", \"tab\": \"Efficiency\", \"score\": \"4.248399836345784\"}", - "Management - # eval": "{\"description\": \"min=103, mean=103, max=103, sum=206 (2)\", \"tab\": \"General information\", \"score\": \"103.0\"}", - "Management - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Management - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Management - # prompt tokens": "{\"description\": \"min=277.379, mean=277.379, max=277.379, sum=554.757 (2)\", \"tab\": \"General information\", \"score\": \"277.378640776699\"}", - "Management - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"management\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_management\"" - } - } - }, - { - "evaluation_name": "Marketing", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Marketing", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.949, - "details": { - "description": "min=0.949, mean=0.949, max=0.949, sum=1.897 (2)", - "tab": "Accuracy", - "Marketing - Observed inference time (s)": "{\"description\": \"min=1.645, mean=1.645, max=1.645, sum=3.29 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.6448312304977677\"}", - "Marketing - # eval": "{\"description\": \"min=234, mean=234, max=234, sum=468 (2)\", \"tab\": \"General information\", \"score\": \"234.0\"}", - "Marketing - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Marketing - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Marketing - # prompt tokens": "{\"description\": \"min=398.675, mean=398.675, max=398.675, sum=797.35 (2)\", \"tab\": \"General information\", \"score\": \"398.6752136752137\"}", - "Marketing - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"marketing\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_marketing\"" - } - } - }, - { - "evaluation_name": "Medical Genetics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Medical Genetics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.96, - "details": { - "description": "min=0.96, mean=0.96, max=0.96, sum=1.92 (2)", - "tab": "Accuracy", - "Medical Genetics - Observed inference time (s)": "{\"description\": \"min=0.527, mean=0.527, max=0.527, sum=1.054 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5272433400154114\"}", - "Medical Genetics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Medical Genetics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Medical Genetics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Medical Genetics - # prompt tokens": "{\"description\": \"min=328.48, mean=328.48, max=328.48, sum=656.96 (2)\", \"tab\": \"General information\", \"score\": \"328.48\"}", - "Medical Genetics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"medical_genetics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_medical_genetics\"" - } - } - }, - { - "evaluation_name": "Miscellaneous", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Miscellaneous", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.949, - "details": { - "description": "min=0.949, mean=0.949, max=0.949, sum=1.898 (2)", - "tab": "Accuracy", - "Miscellaneous - Observed inference time (s)": "{\"description\": \"min=2.642, mean=2.642, max=2.642, sum=5.284 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.6419809954681006\"}", - "Miscellaneous - # eval": "{\"description\": \"min=783, mean=783, max=783, sum=1566 (2)\", \"tab\": \"General information\", \"score\": \"783.0\"}", - "Miscellaneous - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Miscellaneous - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Miscellaneous - # prompt tokens": "{\"description\": \"min=296.626, mean=296.626, max=296.626, sum=593.252 (2)\", \"tab\": \"General information\", \"score\": \"296.6257982120051\"}", - "Miscellaneous - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"miscellaneous\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_miscellaneous\"" - } - } - }, - { - "evaluation_name": "Moral Scenarios", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Moral Scenarios", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.808, - "details": { - "description": "min=0.808, mean=0.808, max=0.808, sum=1.616 (2)", - "tab": "Accuracy", - "Moral Disputes - Observed inference time (s)": "{\"description\": \"min=0.637, mean=0.637, max=0.637, sum=1.275 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6374224183187319\"}", - "Moral Scenarios - Observed inference time (s)": "{\"description\": \"min=0.624, mean=0.624, max=0.624, sum=1.247 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6235519771469372\"}", - "Moral Disputes - # eval": "{\"description\": \"min=346, mean=346, max=346, sum=692 (2)\", \"tab\": \"General information\", \"score\": \"346.0\"}", - "Moral Disputes - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Disputes - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Disputes - # prompt tokens": "{\"description\": \"min=477.78, mean=477.78, max=477.78, sum=955.561 (2)\", \"tab\": \"General information\", \"score\": \"477.78034682080926\"}", - "Moral Disputes - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # eval": "{\"description\": \"min=895, mean=895, max=895, sum=1790 (2)\", \"tab\": \"General information\", \"score\": \"895.0\"}", - "Moral Scenarios - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Scenarios - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # prompt tokens": "{\"description\": \"min=662.517, mean=662.517, max=662.517, sum=1325.035 (2)\", \"tab\": \"General information\", \"score\": \"662.5173184357542\"}", - "Moral Scenarios - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"moral_scenarios\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_moral_scenarios\"" - } - } - }, - { - "evaluation_name": "Nutrition", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Nutrition", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.918, - "details": { - "description": "min=0.918, mean=0.918, max=0.918, sum=1.837 (2)", - "tab": "Accuracy", - "Nutrition - Observed inference time (s)": "{\"description\": \"min=1.989, mean=1.989, max=1.989, sum=3.977 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.9886824734070723\"}", - "Nutrition - # eval": "{\"description\": \"min=306, mean=306, max=306, sum=612 (2)\", \"tab\": \"General information\", \"score\": \"306.0\"}", - "Nutrition - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Nutrition - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Nutrition - # prompt tokens": "{\"description\": \"min=570.337, mean=570.337, max=570.337, sum=1140.673 (2)\", \"tab\": \"General information\", \"score\": \"570.3366013071895\"}", - "Nutrition - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"nutrition\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_nutrition\"" - } - } - }, - { - "evaluation_name": "Prehistory", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Prehistory", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.923, - "details": { - "description": "min=0.923, mean=0.923, max=0.923, sum=1.846 (2)", - "tab": "Accuracy", - "Prehistory - Observed inference time (s)": "{\"description\": \"min=0.909, mean=0.909, max=0.909, sum=1.819 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9094557386857492\"}", - "Prehistory - # eval": "{\"description\": \"min=324, mean=324, max=324, sum=648 (2)\", \"tab\": \"General information\", \"score\": \"324.0\"}", - "Prehistory - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Prehistory - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Prehistory - # prompt tokens": "{\"description\": \"min=505.194, mean=505.194, max=505.194, sum=1010.389 (2)\", \"tab\": \"General information\", \"score\": \"505.19444444444446\"}", - "Prehistory - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"prehistory\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_prehistory\"" - } - } - }, - { - "evaluation_name": "Public Relations", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Public Relations", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.809, - "details": { - "description": "min=0.809, mean=0.809, max=0.809, sum=1.618 (2)", - "tab": "Accuracy", - "Public Relations - Observed inference time (s)": "{\"description\": \"min=0.847, mean=0.847, max=0.847, sum=1.695 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8472580974752253\"}", - "Public Relations - # eval": "{\"description\": \"min=110, mean=110, max=110, sum=220 (2)\", \"tab\": \"General information\", \"score\": \"110.0\"}", - "Public Relations - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Public Relations - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Public Relations - # prompt tokens": "{\"description\": \"min=402.009, mean=402.009, max=402.009, sum=804.018 (2)\", \"tab\": \"General information\", \"score\": \"402.0090909090909\"}", - "Public Relations - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"public_relations\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_public_relations\"" - } - } - }, - { - "evaluation_name": "Security Studies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Security Studies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.837, - "details": { - "description": "min=0.837, mean=0.837, max=0.837, sum=1.673 (2)", - "tab": "Accuracy", - "Security Studies - Observed inference time (s)": "{\"description\": \"min=0.659, mean=0.659, max=0.659, sum=1.318 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6588058092156235\"}", - "Security Studies - # eval": "{\"description\": \"min=245, mean=245, max=245, sum=490 (2)\", \"tab\": \"General information\", \"score\": \"245.0\"}", - "Security Studies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Security Studies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Security Studies - # prompt tokens": "{\"description\": \"min=1160.294, mean=1160.294, max=1160.294, sum=2320.588 (2)\", \"tab\": \"General information\", \"score\": \"1160.2938775510204\"}", - "Security Studies - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"security_studies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_security_studies\"" - } - } - }, - { - "evaluation_name": "Sociology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Sociology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.955, - "details": { - "description": "min=0.955, mean=0.955, max=0.955, sum=1.91 (2)", - "tab": "Accuracy", - "Sociology - Observed inference time (s)": "{\"description\": \"min=1.251, mean=1.251, max=1.251, sum=2.501 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.2506972652169603\"}", - "Sociology - # eval": "{\"description\": \"min=201, mean=201, max=201, sum=402 (2)\", \"tab\": \"General information\", \"score\": \"201.0\"}", - "Sociology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Sociology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Sociology - # prompt tokens": "{\"description\": \"min=443.891, mean=443.891, max=443.891, sum=887.781 (2)\", \"tab\": \"General information\", \"score\": \"443.8905472636816\"}", - "Sociology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"sociology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_sociology\"" - } - } - }, - { - "evaluation_name": "Virology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Virology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.596, - "details": { - "description": "min=0.596, mean=0.596, max=0.596, sum=1.193 (2)", - "tab": "Accuracy", - "Virology - Observed inference time (s)": "{\"description\": \"min=0.509, mean=0.509, max=0.509, sum=1.019 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5092598558908485\"}", - "Virology - # eval": "{\"description\": \"min=166, mean=166, max=166, sum=332 (2)\", \"tab\": \"General information\", \"score\": \"166.0\"}", - "Virology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Virology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Virology - # prompt tokens": "{\"description\": \"min=329.572, mean=329.572, max=329.572, sum=659.145 (2)\", \"tab\": \"General information\", \"score\": \"329.5722891566265\"}", - "Virology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"virology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_virology\"" - } - } - }, - { - "evaluation_name": "World Religions", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on World Religions", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.912, - "details": { - "description": "min=0.912, mean=0.912, max=0.912, sum=1.825 (2)", - "tab": "Accuracy", - "World Religions - Observed inference time (s)": "{\"description\": \"min=1.251, mean=1.251, max=1.251, sum=2.501 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.2507223441586857\"}", - "World Religions - # eval": "{\"description\": \"min=171, mean=171, max=171, sum=342 (2)\", \"tab\": \"General information\", \"score\": \"171.0\"}", - "World Religions - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "World Religions - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "World Religions - # prompt tokens": "{\"description\": \"min=268.918, mean=268.918, max=268.918, sum=537.836 (2)\", \"tab\": \"General information\", \"score\": \"268.91812865497076\"}", - "World Religions - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"world_religions\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_world_religions\"" - } - } - }, - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.215, - "details": { - "description": "", - "tab": "Efficiency" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/deepseek_chat-v3-0324.json b/data/models/deepseek_chat-v3-0324.json deleted file mode 100644 index 50f480128a399a4af113b8706fa4f6e53f11ce80..0000000000000000000000000000000000000000 --- a/data/models/deepseek_chat-v3-0324.json +++ /dev/null @@ -1,89 +0,0 @@ -{ - "model_info": { - "name": "deepseek/chat-v3-0324", - "developer": "DeepSeek", - "inference_platform": "openrouter", - "id": "deepseek/chat-v3-0324" - }, - "evaluations": [ - { - "evaluation_id": "livecodebenchpro/deepseek/chat-v3-0324/1760492095.8105888", - "retrieved_timestamp": "1760492095.8105888", - "source_metadata": { - "source_organization_name": "New York University, Princeton University, University of California San Diego, University of Washington and Canyon Crest Academy", - "evaluator_relationship": "third_party", - "source_name": "Live Code Bench Pro", - "source_type": "documentation" - }, - "eval_library": { - "name": "unknown", - "version": "unknown" - }, - "benchmark": "livecodebenchpro", - "evaluation_results": [ - { - "evaluation_name": "Hard Problems", - "metric_config": { - "evaluation_description": "Pass@1 on Hard Problems", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.0 - }, - "source_data": { - "dataset_name": "Hard Problems", - "source_type": "url", - "url": [ - "https://webhook.cp-bench.orzzh.com/leaderboard/llm/difficulty?difficulty=hard&benchmark_mode=live" - ] - } - }, - { - "evaluation_name": "Medium Problems", - "metric_config": { - "evaluation_description": "Pass@1 on Medium Problems", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.0 - }, - "source_data": { - "dataset_name": "Medium Problems", - "source_type": "url", - "url": [ - "https://webhook.cp-bench.orzzh.com/leaderboard/llm/difficulty?difficulty=medium&benchmark_mode=live" - ] - } - }, - { - "evaluation_name": "Easy Problems", - "metric_config": { - "evaluation_description": "Pass@1 on Easy Problems", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.19718309859154928 - }, - "source_data": { - "dataset_name": "Easy Problems", - "source_type": "url", - "url": [ - "https://webhook.cp-bench.orzzh.com/leaderboard/llm/difficulty?difficulty=easy&benchmark_mode=live" - ] - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/deepseek_deepseek-r1-0528.json b/data/models/deepseek_deepseek-r1-0528.json deleted file mode 100644 index 22f7f1e7159f38877f415a9a6f07905fabcb5d33..0000000000000000000000000000000000000000 --- a/data/models/deepseek_deepseek-r1-0528.json +++ /dev/null @@ -1,1043 +0,0 @@ -{ - "model_info": { - "name": "deepseek-r1-0528", - "id": "deepseek/deepseek-r1-0528", - "developer": "deepseek", - "inference_platform": "unknown", - "additional_details": { - "display_name": "DeepSeek-R1" - } - }, - "evaluations": [ - { - "evaluation_id": "global-mmlu-lite/deepseek_deepseek-r1-0528/1773936496.366405", - "retrieved_timestamp": "1773936496.366405", - "source_metadata": { - "source_name": "Global MMLU Lite Leaderboard", - "source_type": "documentation", - "source_organization_name": "kaggle", - "source_organization_url": "www.kaggle.com", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "kaggle kernel", - "version": "4", - "additional_details": { - "url": "https://www.kaggle.com/code/shivalikasingh95/global-mmlu-lite-sample-notebook" - } - }, - "benchmark": "global-mmlu-lite", - "evaluation_results": [ - { - "evaluation_name": "Global MMLU Lite", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Global MMLU Lite", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6744 - } - }, - { - "evaluation_name": "Culturally Sensitive", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Sensitive", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6672 - } - }, - { - "evaluation_name": "Culturally Agnostic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Agnostic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6816 - } - }, - { - "evaluation_name": "Arabic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Arabic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6825, - "uncertainty": { - "confidence_interval": { - "lower": -0.0456, - "upper": 0.0456, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "English", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - English", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.715, - "uncertainty": { - "confidence_interval": { - "lower": -0.0442, - "upper": 0.0442, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Bengali", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Bengali", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.655, - "uncertainty": { - "confidence_interval": { - "lower": -0.0466, - "upper": 0.0466, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "German", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - German", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6375, - "uncertainty": { - "confidence_interval": { - "lower": -0.0471, - "upper": 0.0471, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "French", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - French", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6925, - "uncertainty": { - "confidence_interval": { - "lower": -0.0452, - "upper": 0.0452, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Hindi", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Hindi", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6475, - "uncertainty": { - "confidence_interval": { - "lower": -0.0468, - "upper": 0.0468, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Indonesian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Indonesian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.655, - "uncertainty": { - "confidence_interval": { - "lower": -0.0466, - "upper": 0.0466, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Italian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Italian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6775, - "uncertainty": { - "confidence_interval": { - "lower": -0.0458, - "upper": 0.0458, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Japanese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Japanese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7725, - "uncertainty": { - "confidence_interval": { - "lower": -0.0411, - "upper": 0.0411, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Korean", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Korean", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6575, - "uncertainty": { - "confidence_interval": { - "lower": -0.0465, - "upper": 0.0465, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Portuguese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Portuguese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.635, - "uncertainty": { - "confidence_interval": { - "lower": -0.0472, - "upper": 0.0472, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Spanish", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Spanish", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7175, - "uncertainty": { - "confidence_interval": { - "lower": -0.0441, - "upper": 0.0441, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Swahili", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Swahili", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6775, - "uncertainty": { - "confidence_interval": { - "lower": -0.0458, - "upper": 0.0458, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Yoruba", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Yoruba", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.77, - "uncertainty": { - "confidence_interval": { - "lower": -0.0412, - "upper": 0.0412, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Chinese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Chinese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5075, - "uncertainty": { - "confidence_interval": { - "lower": -0.049, - "upper": 0.049, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Burmese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Burmese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.69, - "uncertainty": { - "confidence_interval": { - "lower": -0.0453, - "upper": 0.0453, - "method": "unknown" - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "global-mmlu-lite/deepseek_deepseek-r1-0528/1773936583.743359", - "retrieved_timestamp": "1773936583.743359", - "source_metadata": { - "source_name": "Global MMLU Lite Leaderboard", - "source_type": "documentation", - "source_organization_name": "kaggle", - "source_organization_url": "www.kaggle.com", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "kaggle kernel", - "version": "4", - "additional_details": { - "url": "https://www.kaggle.com/code/shivalikasingh95/global-mmlu-lite-sample-notebook" - } - }, - "benchmark": "global-mmlu-lite", - "evaluation_results": [ - { - "evaluation_name": "Global MMLU Lite", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Global MMLU Lite", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6744 - } - }, - { - "evaluation_name": "Culturally Sensitive", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Sensitive", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6672 - } - }, - { - "evaluation_name": "Culturally Agnostic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Agnostic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6816 - } - }, - { - "evaluation_name": "Arabic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Arabic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6825, - "uncertainty": { - "confidence_interval": { - "lower": -0.0456, - "upper": 0.0456, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "English", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - English", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.715, - "uncertainty": { - "confidence_interval": { - "lower": -0.0442, - "upper": 0.0442, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Bengali", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Bengali", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.655, - "uncertainty": { - "confidence_interval": { - "lower": -0.0466, - "upper": 0.0466, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "German", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - German", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6375, - "uncertainty": { - "confidence_interval": { - "lower": -0.0471, - "upper": 0.0471, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "French", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - French", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6925, - "uncertainty": { - "confidence_interval": { - "lower": -0.0452, - "upper": 0.0452, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Hindi", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Hindi", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6475, - "uncertainty": { - "confidence_interval": { - "lower": -0.0468, - "upper": 0.0468, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Indonesian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Indonesian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.655, - "uncertainty": { - "confidence_interval": { - "lower": -0.0466, - "upper": 0.0466, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Italian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Italian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6775, - "uncertainty": { - "confidence_interval": { - "lower": -0.0458, - "upper": 0.0458, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Japanese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Japanese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7725, - "uncertainty": { - "confidence_interval": { - "lower": -0.0411, - "upper": 0.0411, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Korean", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Korean", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6575, - "uncertainty": { - "confidence_interval": { - "lower": -0.0465, - "upper": 0.0465, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Portuguese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Portuguese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.635, - "uncertainty": { - "confidence_interval": { - "lower": -0.0472, - "upper": 0.0472, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Spanish", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Spanish", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7175, - "uncertainty": { - "confidence_interval": { - "lower": -0.0441, - "upper": 0.0441, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Swahili", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Swahili", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6775, - "uncertainty": { - "confidence_interval": { - "lower": -0.0458, - "upper": 0.0458, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Yoruba", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Yoruba", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.77, - "uncertainty": { - "confidence_interval": { - "lower": -0.0412, - "upper": 0.0412, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Chinese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Chinese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5075, - "uncertainty": { - "confidence_interval": { - "lower": -0.049, - "upper": 0.049, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Burmese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Burmese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.69, - "uncertainty": { - "confidence_interval": { - "lower": -0.0453, - "upper": 0.0453, - "method": "unknown" - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/deepseek_deepseek-v3-2-exp-fc.json b/data/models/deepseek_deepseek-v3-2-exp-fc.json deleted file mode 100644 index 61afc468a1b302330a5c01c6e71c1664b8abec47..0000000000000000000000000000000000000000 --- a/data/models/deepseek_deepseek-v3-2-exp-fc.json +++ /dev/null @@ -1,851 +0,0 @@ -{ - "model_info": { - "name": "DeepSeek-V3.2-Exp (FC)", - "id": "deepseek/deepseek-v3-2-exp-fc", - "developer": "deepseek", - "additional_details": { - "raw_model_name": "DeepSeek-V3.2-Exp (FC)", - "organization": "DeepSeek", - "license": "MIT", - "mode": "FC", - "model_link": "https://api-docs.deepseek.com/news/news250528" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/deepseek/deepseek-v3-2-exp-fc/1775236112.37534", - "retrieved_timestamp": "1775236112.37534", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 19.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 54.12 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 6.71 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 5.83 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 11.71 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 10.59 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 34.85 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 37.92 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 74.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 15.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 12.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 53.66 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 66.28 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 51.66 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 25.0 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 25.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 37.38 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 41.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 39.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 33.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 35.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 69.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 80.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 59.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 54.19 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 41.94 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 61.29 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 59.35 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 37.5 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 93.18 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/deepseek_deepseek-v3-2-exp-prompt-thinking.json b/data/models/deepseek_deepseek-v3-2-exp-prompt-thinking.json deleted file mode 100644 index 9da36bfdf2372cfa8bef7acd3335bc4bfaf79923..0000000000000000000000000000000000000000 --- a/data/models/deepseek_deepseek-v3-2-exp-prompt-thinking.json +++ /dev/null @@ -1,905 +0,0 @@ -{ - "model_info": { - "name": "DeepSeek-V3.2-Exp (Prompt + Thinking)", - "id": "deepseek/deepseek-v3-2-exp-prompt-thinking", - "developer": "deepseek", - "additional_details": { - "raw_model_name": "DeepSeek-V3.2-Exp (Prompt + Thinking)", - "organization": "DeepSeek", - "license": "MIT", - "mode": "Prompt + Thinking", - "model_link": "https://api-docs.deepseek.com/news/news250528" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/deepseek/deepseek-v3-2-exp-prompt-thinking/1775236112.372728", - "retrieved_timestamp": "1775236112.372728", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 14.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 56.73 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 57.75 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 37.89 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 49.56 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 102.09 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 85.52 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 74.08 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 92.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 89.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 86.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 76.02 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 82.56 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 74.74 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 87.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 54.17 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 44.88 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 55.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 49.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 27.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 48.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 58.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 64.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 52.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 44.09 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 46.45 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 46.45 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 39.35 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 93.75 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 67.0 - } - }, - { - "evaluation_result_id": "bfcl.format_sensitivity.max_delta::max_delta", - "evaluation_name": "bfcl.format_sensitivity.max_delta", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.format_sensitivity.max_delta", - "metric_name": "Format sensitivity max delta", - "metric_kind": "difference", - "metric_unit": "percentage_points", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Format Sensitivity Max Delta" - } - }, - "score_details": { - "score": 10.0 - } - }, - { - "evaluation_result_id": "bfcl.format_sensitivity.stddev::stddev", - "evaluation_name": "bfcl.format_sensitivity.stddev", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.format_sensitivity.stddev", - "metric_name": "Format sensitivity standard deviation", - "metric_kind": "difference", - "metric_unit": "percentage_points", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Format Sensitivity Standard Deviation" - } - }, - "score_details": { - "score": 2.77 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/deepseek_deepseek-v3.1.json b/data/models/deepseek_deepseek-v3.1.json deleted file mode 100644 index 28271d97eb42a16e44db35269bdfee0d33d2475a..0000000000000000000000000000000000000000 --- a/data/models/deepseek_deepseek-v3.1.json +++ /dev/null @@ -1,1040 +0,0 @@ -{ - "model_info": { - "name": "deepseek-v3.1", - "id": "deepseek/deepseek-v3.1", - "developer": "deepseek", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "global-mmlu-lite/deepseek_deepseek-v3.1/1773936583.743359", - "retrieved_timestamp": "1773936583.743359", - "source_metadata": { - "source_name": "Global MMLU Lite Leaderboard", - "source_type": "documentation", - "source_organization_name": "kaggle", - "source_organization_url": "www.kaggle.com", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "kaggle kernel", - "version": "4", - "additional_details": { - "url": "https://www.kaggle.com/code/shivalikasingh95/global-mmlu-lite-sample-notebook" - } - }, - "benchmark": "global-mmlu-lite", - "evaluation_results": [ - { - "evaluation_name": "Global MMLU Lite", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Global MMLU Lite", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8044 - } - }, - { - "evaluation_name": "Culturally Sensitive", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Sensitive", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7793 - } - }, - { - "evaluation_name": "Culturally Agnostic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Agnostic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8295 - } - }, - { - "evaluation_name": "Arabic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Arabic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.805, - "uncertainty": { - "confidence_interval": { - "lower": -0.0388, - "upper": 0.0388, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "English", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - English", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.825, - "uncertainty": { - "confidence_interval": { - "lower": -0.0372, - "upper": 0.0372, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Bengali", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Bengali", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8157, - "uncertainty": { - "confidence_interval": { - "lower": -0.0382, - "upper": 0.0382, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "German", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - German", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7925, - "uncertainty": { - "confidence_interval": { - "lower": -0.0397, - "upper": 0.0397, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "French", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - French", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8175, - "uncertainty": { - "confidence_interval": { - "lower": -0.0379, - "upper": 0.0379, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Hindi", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Hindi", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7569, - "uncertainty": { - "confidence_interval": { - "lower": -0.0421, - "upper": 0.0421, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Indonesian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Indonesian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7764, - "uncertainty": { - "confidence_interval": { - "lower": -0.0409, - "upper": 0.0409, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Italian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Italian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8075, - "uncertainty": { - "confidence_interval": { - "lower": -0.0386, - "upper": 0.0386, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Japanese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Japanese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8312, - "uncertainty": { - "confidence_interval": { - "lower": -0.0374, - "upper": 0.0374, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Korean", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Korean", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8125, - "uncertainty": { - "confidence_interval": { - "lower": -0.0382, - "upper": 0.0382, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Portuguese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Portuguese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8246, - "uncertainty": { - "confidence_interval": { - "lower": -0.0373, - "upper": 0.0373, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Spanish", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Spanish", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8125, - "uncertainty": { - "confidence_interval": { - "lower": -0.0382, - "upper": 0.0382, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Swahili", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Swahili", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.801, - "uncertainty": { - "confidence_interval": { - "lower": -0.0393, - "upper": 0.0393, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Yoruba", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Yoruba", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7831, - "uncertainty": { - "confidence_interval": { - "lower": -0.0415, - "upper": 0.0415, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Chinese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Chinese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8161, - "uncertainty": { - "confidence_interval": { - "lower": -0.0381, - "upper": 0.0381, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Burmese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Burmese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7925, - "uncertainty": { - "confidence_interval": { - "lower": -0.0397, - "upper": 0.0397, - "method": "unknown" - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "global-mmlu-lite/deepseek_deepseek-v3.1/1773936496.366405", - "retrieved_timestamp": "1773936496.366405", - "source_metadata": { - "source_name": "Global MMLU Lite Leaderboard", - "source_type": "documentation", - "source_organization_name": "kaggle", - "source_organization_url": "www.kaggle.com", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "kaggle kernel", - "version": "4", - "additional_details": { - "url": "https://www.kaggle.com/code/shivalikasingh95/global-mmlu-lite-sample-notebook" - } - }, - "benchmark": "global-mmlu-lite", - "evaluation_results": [ - { - "evaluation_name": "Global MMLU Lite", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Global MMLU Lite", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8044 - } - }, - { - "evaluation_name": "Culturally Sensitive", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Sensitive", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7793 - } - }, - { - "evaluation_name": "Culturally Agnostic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Agnostic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8295 - } - }, - { - "evaluation_name": "Arabic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Arabic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.805, - "uncertainty": { - "confidence_interval": { - "lower": -0.0388, - "upper": 0.0388, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "English", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - English", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.825, - "uncertainty": { - "confidence_interval": { - "lower": -0.0372, - "upper": 0.0372, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Bengali", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Bengali", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8157, - "uncertainty": { - "confidence_interval": { - "lower": -0.0382, - "upper": 0.0382, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "German", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - German", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7925, - "uncertainty": { - "confidence_interval": { - "lower": -0.0397, - "upper": 0.0397, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "French", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - French", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8175, - "uncertainty": { - "confidence_interval": { - "lower": -0.0379, - "upper": 0.0379, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Hindi", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Hindi", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7569, - "uncertainty": { - "confidence_interval": { - "lower": -0.0421, - "upper": 0.0421, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Indonesian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Indonesian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7764, - "uncertainty": { - "confidence_interval": { - "lower": -0.0409, - "upper": 0.0409, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Italian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Italian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8075, - "uncertainty": { - "confidence_interval": { - "lower": -0.0386, - "upper": 0.0386, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Japanese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Japanese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8312, - "uncertainty": { - "confidence_interval": { - "lower": -0.0374, - "upper": 0.0374, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Korean", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Korean", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8125, - "uncertainty": { - "confidence_interval": { - "lower": -0.0382, - "upper": 0.0382, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Portuguese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Portuguese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8246, - "uncertainty": { - "confidence_interval": { - "lower": -0.0373, - "upper": 0.0373, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Spanish", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Spanish", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8125, - "uncertainty": { - "confidence_interval": { - "lower": -0.0382, - "upper": 0.0382, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Swahili", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Swahili", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.801, - "uncertainty": { - "confidence_interval": { - "lower": -0.0393, - "upper": 0.0393, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Yoruba", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Yoruba", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7831, - "uncertainty": { - "confidence_interval": { - "lower": -0.0415, - "upper": 0.0415, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Chinese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Chinese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8161, - "uncertainty": { - "confidence_interval": { - "lower": -0.0381, - "upper": 0.0381, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Burmese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Burmese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7925, - "uncertainty": { - "confidence_interval": { - "lower": -0.0397, - "upper": 0.0397, - "method": "unknown" - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/deepseek_deepseek-v3.2.json b/data/models/deepseek_deepseek-v3.2.json deleted file mode 100644 index b5aed0c7fb1896a50330a54b4969ea92bc0a4f41..0000000000000000000000000000000000000000 --- a/data/models/deepseek_deepseek-v3.2.json +++ /dev/null @@ -1,87 +0,0 @@ -{ - "model_info": { - "name": "DeepSeek-V3.2", - "id": "deepseek/deepseek-v3.2", - "developer": "DeepSeek", - "additional_details": { - "agent_name": "Terminus 2", - "agent_organization": "Terminal Bench" - } - }, - "evaluations": [ - { - "evaluation_id": "terminal-bench-2.0/terminus-2__deepseek-v3.2/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2026-02-10", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 39.6, - "uncertainty": { - "standard_error": { - "value": 2.8 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Terminus 2\" -m \"DeepSeek-V3.2\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Terminus 2\" -m \"DeepSeek-V3.2\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/deepseek_ep-20250214004308-p7n89.json b/data/models/deepseek_ep-20250214004308-p7n89.json deleted file mode 100644 index 22a68b07c8cd21d049260ac2d7c25f1340fb3c74..0000000000000000000000000000000000000000 --- a/data/models/deepseek_ep-20250214004308-p7n89.json +++ /dev/null @@ -1,89 +0,0 @@ -{ - "model_info": { - "name": "ep-20250214004308-p7n89", - "developer": "DeepSeek", - "inference_platform": "ark", - "id": "deepseek/ep-20250214004308-p7n89" - }, - "evaluations": [ - { - "evaluation_id": "livecodebenchpro/ep-20250214004308-p7n89/1760492095.8105888", - "retrieved_timestamp": "1760492095.8105888", - "source_metadata": { - "source_organization_name": "New York University, Princeton University, University of California San Diego, University of Washington and Canyon Crest Academy", - "evaluator_relationship": "third_party", - "source_name": "Live Code Bench Pro", - "source_type": "documentation" - }, - "eval_library": { - "name": "unknown", - "version": "unknown" - }, - "benchmark": "livecodebenchpro", - "evaluation_results": [ - { - "evaluation_name": "Hard Problems", - "metric_config": { - "evaluation_description": "Pass@1 on Hard Problems", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.0 - }, - "source_data": { - "dataset_name": "Hard Problems", - "source_type": "url", - "url": [ - "https://webhook.cp-bench.orzzh.com/leaderboard/llm/difficulty?difficulty=hard&benchmark_mode=live" - ] - } - }, - { - "evaluation_name": "Medium Problems", - "metric_config": { - "evaluation_description": "Pass@1 on Medium Problems", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.014084507042253521 - }, - "source_data": { - "dataset_name": "Medium Problems", - "source_type": "url", - "url": [ - "https://webhook.cp-bench.orzzh.com/leaderboard/llm/difficulty?difficulty=medium&benchmark_mode=live" - ] - } - }, - { - "evaluation_name": "Easy Problems", - "metric_config": { - "evaluation_description": "Pass@1 on Easy Problems", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.4225352112676056 - }, - "source_data": { - "dataset_name": "Easy Problems", - "source_type": "url", - "url": [ - "https://webhook.cp-bench.orzzh.com/leaderboard/llm/difficulty?difficulty=easy&benchmark_mode=live" - ] - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/deepseek_ep-20250228232227-z44x5.json b/data/models/deepseek_ep-20250228232227-z44x5.json deleted file mode 100644 index 372ab393c58950e14c5669d92222826413642db0..0000000000000000000000000000000000000000 --- a/data/models/deepseek_ep-20250228232227-z44x5.json +++ /dev/null @@ -1,89 +0,0 @@ -{ - "model_info": { - "name": "ep-20250228232227-z44x5", - "developer": "DeepSeek", - "inference_platform": "ark", - "id": "deepseek/ep-20250228232227-z44x5" - }, - "evaluations": [ - { - "evaluation_id": "livecodebenchpro/ep-20250228232227-z44x5/1760492095.8105888", - "retrieved_timestamp": "1760492095.8105888", - "source_metadata": { - "source_organization_name": "New York University, Princeton University, University of California San Diego, University of Washington and Canyon Crest Academy", - "evaluator_relationship": "third_party", - "source_name": "Live Code Bench Pro", - "source_type": "documentation" - }, - "eval_library": { - "name": "unknown", - "version": "unknown" - }, - "benchmark": "livecodebenchpro", - "evaluation_results": [ - { - "evaluation_name": "Hard Problems", - "metric_config": { - "evaluation_description": "Pass@1 on Hard Problems", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.0 - }, - "source_data": { - "dataset_name": "Hard Problems", - "source_type": "url", - "url": [ - "https://webhook.cp-bench.orzzh.com/leaderboard/llm/difficulty?difficulty=hard&benchmark_mode=live" - ] - } - }, - { - "evaluation_name": "Medium Problems", - "metric_config": { - "evaluation_description": "Pass@1 on Medium Problems", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.0 - }, - "source_data": { - "dataset_name": "Medium Problems", - "source_type": "url", - "url": [ - "https://webhook.cp-bench.orzzh.com/leaderboard/llm/difficulty?difficulty=medium&benchmark_mode=live" - ] - } - }, - { - "evaluation_name": "Easy Problems", - "metric_config": { - "evaluation_description": "Pass@1 on Easy Problems", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.1267605633802817 - }, - "source_data": { - "dataset_name": "Easy Problems", - "source_type": "url", - "url": [ - "https://webhook.cp-bench.orzzh.com/leaderboard/llm/difficulty?difficulty=easy&benchmark_mode=live" - ] - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/deepseek_ep-20250603132404-cgpjm.json b/data/models/deepseek_ep-20250603132404-cgpjm.json deleted file mode 100644 index 2f0505359e4587b7c1938a328f3d7e7f68c2d760..0000000000000000000000000000000000000000 --- a/data/models/deepseek_ep-20250603132404-cgpjm.json +++ /dev/null @@ -1,89 +0,0 @@ -{ - "model_info": { - "name": "ep-20250603132404-cgpjm", - "developer": "DeepSeek", - "inference_platform": "ark", - "id": "deepseek/ep-20250603132404-cgpjm" - }, - "evaluations": [ - { - "evaluation_id": "livecodebenchpro/ep-20250603132404-cgpjm/1760492095.8105888", - "retrieved_timestamp": "1760492095.8105888", - "source_metadata": { - "source_organization_name": "New York University, Princeton University, University of California San Diego, University of Washington and Canyon Crest Academy", - "evaluator_relationship": "third_party", - "source_name": "Live Code Bench Pro", - "source_type": "documentation" - }, - "eval_library": { - "name": "unknown", - "version": "unknown" - }, - "benchmark": "livecodebenchpro", - "evaluation_results": [ - { - "evaluation_name": "Hard Problems", - "metric_config": { - "evaluation_description": "Pass@1 on Hard Problems", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.0 - }, - "source_data": { - "dataset_name": "Hard Problems", - "source_type": "url", - "url": [ - "https://webhook.cp-bench.orzzh.com/leaderboard/llm/difficulty?difficulty=hard&benchmark_mode=live" - ] - } - }, - { - "evaluation_name": "Medium Problems", - "metric_config": { - "evaluation_description": "Pass@1 on Medium Problems", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.08450704225352113 - }, - "source_data": { - "dataset_name": "Medium Problems", - "source_type": "url", - "url": [ - "https://webhook.cp-bench.orzzh.com/leaderboard/llm/difficulty?difficulty=medium&benchmark_mode=live" - ] - } - }, - { - "evaluation_name": "Easy Problems", - "metric_config": { - "evaluation_description": "Pass@1 on Easy Problems", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.5774647887323944 - }, - "source_data": { - "dataset_name": "Easy Problems", - "source_type": "url", - "url": [ - "https://webhook.cp-bench.orzzh.com/leaderboard/llm/difficulty?difficulty=easy&benchmark_mode=live" - ] - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/delta-vector_baldur-8b.json b/data/models/delta-vector_baldur-8b.json deleted file mode 100644 index cd0ced989718e2da346568c453d787d084ec5241..0000000000000000000000000000000000000000 --- a/data/models/delta-vector_baldur-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Baldur-8B", - "id": "Delta-Vector/Baldur-8B", - "developer": "Delta-Vector", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Delta-Vector_Baldur-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4782 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5306 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1435 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.302 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4372 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3654 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/delta-vector_control-8b-v1.1.json b/data/models/delta-vector_control-8b-v1.1.json deleted file mode 100644 index 419c525e733301e78b4ca63774e89d1a0e36eab1..0000000000000000000000000000000000000000 --- a/data/models/delta-vector_control-8b-v1.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Control-8B-V1.1", - "id": "Delta-Vector/Control-8B-V1.1", - "developer": "Delta-Vector", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Delta-Vector_Control-8B-V1.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5697 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4993 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1276 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.307 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4237 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3745 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/delta-vector_control-8b.json b/data/models/delta-vector_control-8b.json deleted file mode 100644 index 7ee5c61fc52151aa643fe4e7ed072ca763471e35..0000000000000000000000000000000000000000 --- a/data/models/delta-vector_control-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Control-8B", - "id": "Delta-Vector/Control-8B", - "developer": "Delta-Vector", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Delta-Vector_Control-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.549 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5041 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.139 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3163 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4355 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3732 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/delta-vector_darkens-8b.json b/data/models/delta-vector_darkens-8b.json deleted file mode 100644 index 8814184dbd24cd5f8b1cbeb0ff4c85ecd57fb235..0000000000000000000000000000000000000000 --- a/data/models/delta-vector_darkens-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Darkens-8B", - "id": "Delta-Vector/Darkens-8B", - "developer": "Delta-Vector", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "8.414" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Delta-Vector_Darkens-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2548 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5251 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0589 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3247 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4106 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3736 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/delta-vector_henbane-7b-attempt2.json b/data/models/delta-vector_henbane-7b-attempt2.json deleted file mode 100644 index 587a72195dd0c6f2beaf8852e03efaa4e00a06e5..0000000000000000000000000000000000000000 --- a/data/models/delta-vector_henbane-7b-attempt2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Henbane-7b-attempt2", - "id": "Delta-Vector/Henbane-7b-attempt2", - "developer": "Delta-Vector", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Delta-Vector_Henbane-7b-attempt2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4157 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5061 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2273 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2903 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3973 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4028 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/delta-vector_odin-9b.json b/data/models/delta-vector_odin-9b.json deleted file mode 100644 index 4c9e022bfc9be3db6523620beb006ff3ee4c7b63..0000000000000000000000000000000000000000 --- a/data/models/delta-vector_odin-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Odin-9B", - "id": "Delta-Vector/Odin-9B", - "developer": "Delta-Vector", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "9.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Delta-Vector_Odin-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3692 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.544 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.145 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3414 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4648 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4047 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/delta-vector_tor-8b.json b/data/models/delta-vector_tor-8b.json deleted file mode 100644 index 524754b88ca2c0af895962e6409fa0ae19d5d920..0000000000000000000000000000000000000000 --- a/data/models/delta-vector_tor-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Tor-8B", - "id": "Delta-Vector/Tor-8B", - "developer": "Delta-Vector", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "8.414" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Delta-Vector_Tor-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2382 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5209 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0589 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3238 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4092 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.373 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/devquasar_devquasar-r1-uncensored-llama-8b.json b/data/models/devquasar_devquasar-r1-uncensored-llama-8b.json deleted file mode 100644 index f00c7e8ca874572746b07b26657fb9e48c3ce070..0000000000000000000000000000000000000000 --- a/data/models/devquasar_devquasar-r1-uncensored-llama-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "DevQuasar-R1-Uncensored-Llama-8B", - "id": "DevQuasar/DevQuasar-R1-Uncensored-Llama-8B", - "developer": "DevQuasar", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DevQuasar_DevQuasar-R1-Uncensored-Llama-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3849 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5118 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3308 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3473 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4436 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3615 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dfurman_calmerys-78b-orpo-v0.1.json b/data/models/dfurman_calmerys-78b-orpo-v0.1.json deleted file mode 100644 index da2fba621db28475845d6b6c0bdb24ecb774bbab..0000000000000000000000000000000000000000 --- a/data/models/dfurman_calmerys-78b-orpo-v0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "CalmeRys-78B-Orpo-v0.1", - "id": "dfurman/CalmeRys-78B-Orpo-v0.1", - "developer": "dfurman", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "77.965" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/dfurman_CalmeRys-78B-Orpo-v0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8163 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7262 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4063 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4002 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5902 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7012 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dfurman_llama-3-70b-orpo-v0.1.json b/data/models/dfurman_llama-3-70b-orpo-v0.1.json deleted file mode 100644 index 7c05ceab843788407bee96f6b7b4de4b5a9fb989..0000000000000000000000000000000000000000 --- a/data/models/dfurman_llama-3-70b-orpo-v0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-70B-Orpo-v0.1", - "id": "dfurman/Llama-3-70B-Orpo-v0.1", - "developer": "dfurman", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "70.554" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/dfurman_Llama-3-70B-Orpo-v0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2049 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4655 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1579 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2576 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4534 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3893 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dfurman_llama-3-8b-orpo-v0.1.json b/data/models/dfurman_llama-3-8b-orpo-v0.1.json deleted file mode 100644 index 987b3112f03b34626e0421787b1c9cb2e2b55a46..0000000000000000000000000000000000000000 --- a/data/models/dfurman_llama-3-8b-orpo-v0.1.json +++ /dev/null @@ -1,275 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-8B-Orpo-v0.1", - "id": "dfurman/Llama-3-8B-Orpo-v0.1", - "developer": "dfurman", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/dfurman_Llama-3-8B-Orpo-v0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3853 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0415 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2617 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3579 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2281 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "hfopenllm_v2/dfurman_Llama-3-8B-Orpo-v0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2835 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3842 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0521 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2609 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3566 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2298 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dfurman_qwen2-72b-orpo-v0.1.json b/data/models/dfurman_qwen2-72b-orpo-v0.1.json deleted file mode 100644 index 1ee04f6facee32942327ce54e9373adbf810cf7c..0000000000000000000000000000000000000000 --- a/data/models/dfurman_qwen2-72b-orpo-v0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2-72B-Orpo-v0.1", - "id": "dfurman/Qwen2-72B-Orpo-v0.1", - "developer": "dfurman", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "72.699" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/dfurman_Qwen2-72B-Orpo-v0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.788 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6969 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4056 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3842 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4784 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5455 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dicta-il_dictalm2.0-instruct.json b/data/models/dicta-il_dictalm2.0-instruct.json deleted file mode 100644 index ef96aab0943091a639d8429344d5d80bdb449c37..0000000000000000000000000000000000000000 --- a/data/models/dicta-il_dictalm2.0-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "dictalm2.0-instruct", - "id": "dicta-il/dictalm2.0-instruct", - "developer": "dicta-il", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.251" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/dicta-il_dictalm2.0-instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4412 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4256 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0227 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3029 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3946 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2605 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dicta-il_dictalm2.0.json b/data/models/dicta-il_dictalm2.0.json deleted file mode 100644 index 5d93a8d85e81aad3f02ee3536323a6ef964026a5..0000000000000000000000000000000000000000 --- a/data/models/dicta-il_dictalm2.0.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "dictalm2.0", - "id": "dicta-il/dictalm2.0", - "developer": "dicta-il", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.251" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/dicta-il_dictalm2.0/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2413 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4018 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0181 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2919 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.382 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2605 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/distilbert_distilgpt2.json b/data/models/distilbert_distilgpt2.json deleted file mode 100644 index 871fe77f4a763b305cf93f46ce69c460574759b8..0000000000000000000000000000000000000000 --- a/data/models/distilbert_distilgpt2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "distilgpt2", - "id": "distilbert/distilgpt2", - "developer": "distilbert", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "GPT2LMHeadModel", - "params_billions": "0.088" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/distilbert_distilgpt2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0611 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3038 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.006 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2592 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4207 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1187 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/divyanshukunwar_sastri_1_9b.json b/data/models/divyanshukunwar_sastri_1_9b.json deleted file mode 100644 index 8168112af18465b79bf346a742bd4eb4ab98a006..0000000000000000000000000000000000000000 --- a/data/models/divyanshukunwar_sastri_1_9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SASTRI_1_9B", - "id": "divyanshukunwar/SASTRI_1_9B", - "developer": "divyanshukunwar", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "5.211" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/divyanshukunwar_SASTRI_1_9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4207 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.468 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1156 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3213 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3831 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3187 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/djuna-test-lab_test-l3.2-rewish-3b-ties-w-base.json b/data/models/djuna-test-lab_test-l3.2-rewish-3b-ties-w-base.json deleted file mode 100644 index 8e5ff4f60f72aefd9c4c16f36f22717802279d64..0000000000000000000000000000000000000000 --- a/data/models/djuna-test-lab_test-l3.2-rewish-3b-ties-w-base.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "TEST-L3.2-ReWish-3B-ties-w-base", - "id": "djuna-test-lab/TEST-L3.2-ReWish-3B-ties-w-base", - "developer": "djuna-test-lab", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/djuna-test-lab_TEST-L3.2-ReWish-3B-ties-w-base/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6353 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4495 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1367 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2836 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3777 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3126 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/djuna-test-lab_test-l3.2-rewish-3b.json b/data/models/djuna-test-lab_test-l3.2-rewish-3b.json deleted file mode 100644 index 94a3a10201db80ff9f76e0da185cff2f8585d758..0000000000000000000000000000000000000000 --- a/data/models/djuna-test-lab_test-l3.2-rewish-3b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "TEST-L3.2-ReWish-3B", - "id": "djuna-test-lab/TEST-L3.2-ReWish-3B", - "developer": "djuna-test-lab", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/djuna-test-lab_TEST-L3.2-ReWish-3B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6368 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4495 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1367 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2836 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3777 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3126 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/djuna_g2-biggsht-27b-2.json b/data/models/djuna_g2-biggsht-27b-2.json deleted file mode 100644 index 7d81991fd2ff5f06ff10589c490a41707340234d..0000000000000000000000000000000000000000 --- a/data/models/djuna_g2-biggsht-27b-2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "G2-BigGSHT-27B-2", - "id": "djuna/G2-BigGSHT-27B-2", - "developer": "djuna", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "27.227" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/djuna_G2-BigGSHT-27B-2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7974 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6415 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2349 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3633 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4072 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4528 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/djuna_g2-gsht.json b/data/models/djuna_g2-gsht.json deleted file mode 100644 index 8e9b387e9d11279a23f25be9d9e61fd049f06085..0000000000000000000000000000000000000000 --- a/data/models/djuna_g2-gsht.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "G2-GSHT", - "id": "djuna/G2-GSHT", - "developer": "djuna", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/djuna_G2-GSHT/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.563 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.527 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1926 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3255 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4006 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.307 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/djuna_gemma-2-gemmama-9b.json b/data/models/djuna_gemma-2-gemmama-9b.json deleted file mode 100644 index 6aba71d8075f5e3a8044f82bf79e6cfce36d9078..0000000000000000000000000000000000000000 --- a/data/models/djuna_gemma-2-gemmama-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Gemma-2-gemmama-9b", - "id": "djuna/Gemma-2-gemmama-9b", - "developer": "djuna", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/djuna_Gemma-2-gemmama-9b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7703 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.542 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1926 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3356 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4031 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3109 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/djuna_l3.1-forsths.json b/data/models/djuna_l3.1-forsths.json deleted file mode 100644 index 5bc1499d51c81cd6907fba26a3c2a2d2b3906493..0000000000000000000000000000000000000000 --- a/data/models/djuna_l3.1-forsths.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "L3.1-ForStHS", - "id": "djuna/L3.1-ForStHS", - "developer": "djuna", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/djuna_L3.1-ForStHS/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7813 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5203 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1503 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2911 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4026 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3735 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/djuna_l3.1-promissum_mane-8b-della-1.5-calc.json b/data/models/djuna_l3.1-promissum_mane-8b-della-1.5-calc.json deleted file mode 100644 index 3a8f7e2c527959032d09d7b1f71fbc833e975737..0000000000000000000000000000000000000000 --- a/data/models/djuna_l3.1-promissum_mane-8b-della-1.5-calc.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "L3.1-Promissum_Mane-8B-Della-1.5-calc", - "id": "djuna/L3.1-Promissum_Mane-8B-Della-1.5-calc", - "developer": "djuna", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/djuna_L3.1-Promissum_Mane-8B-Della-1.5-calc/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7235 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5433 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1639 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3146 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4253 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3904 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/djuna_l3.1-promissum_mane-8b-della-calc.json b/data/models/djuna_l3.1-promissum_mane-8b-della-calc.json deleted file mode 100644 index 4d757620d1b599c4a9886bf5ee26daa0d1c10387..0000000000000000000000000000000000000000 --- a/data/models/djuna_l3.1-promissum_mane-8b-della-calc.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "L3.1-Promissum_Mane-8B-Della-calc", - "id": "djuna/L3.1-Promissum_Mane-8B-Della-calc", - "developer": "djuna", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/djuna_L3.1-Promissum_Mane-8B-Della-calc/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5442 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5486 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1843 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2995 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.423 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3802 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/djuna_l3.1-purosani-2-8b.json b/data/models/djuna_l3.1-purosani-2-8b.json deleted file mode 100644 index f4ea021279ca09f9d4e9186718b56d393878d378..0000000000000000000000000000000000000000 --- a/data/models/djuna_l3.1-purosani-2-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "L3.1-Purosani-2-8B", - "id": "djuna/L3.1-Purosani-2-8B", - "developer": "djuna", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/djuna_L3.1-Purosani-2-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4988 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5182 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1171 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3012 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3816 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3752 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/djuna_l3.1-suze-vume-calc.json b/data/models/djuna_l3.1-suze-vume-calc.json deleted file mode 100644 index a056730c801fddb0a56128db5a591b2af5a78f9e..0000000000000000000000000000000000000000 --- a/data/models/djuna_l3.1-suze-vume-calc.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "L3.1-Suze-Vume-calc", - "id": "djuna/L3.1-Suze-Vume-calc", - "developer": "djuna", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/djuna_L3.1-Suze-Vume-calc/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7297 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5164 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.114 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2819 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3843 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3515 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/djuna_mn-chinofun-12b-2.json b/data/models/djuna_mn-chinofun-12b-2.json deleted file mode 100644 index 07c1e9f306a24765d55f9e0473d96a72b5452780..0000000000000000000000000000000000000000 --- a/data/models/djuna_mn-chinofun-12b-2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MN-Chinofun-12B-2", - "id": "djuna/MN-Chinofun-12B-2", - "developer": "djuna", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/djuna_MN-Chinofun-12B-2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6171 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5037 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1307 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3054 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4268 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3615 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/djuna_mn-chinofun-12b-3.json b/data/models/djuna_mn-chinofun-12b-3.json deleted file mode 100644 index debedff6d22ef8b0ab48feda107126cae81ebffa..0000000000000000000000000000000000000000 --- a/data/models/djuna_mn-chinofun-12b-3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MN-Chinofun-12B-3", - "id": "djuna/MN-Chinofun-12B-3", - "developer": "djuna", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/djuna_MN-Chinofun-12B-3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3053 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5348 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1005 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2659 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4198 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3026 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/djuna_mn-chinofun-12b-4.json b/data/models/djuna_mn-chinofun-12b-4.json deleted file mode 100644 index bcc82b46a23643bce05dd79170472079d2670462..0000000000000000000000000000000000000000 --- a/data/models/djuna_mn-chinofun-12b-4.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MN-Chinofun-12B-4", - "id": "djuna/MN-Chinofun-12B-4", - "developer": "djuna", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/djuna_MN-Chinofun-12B-4/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5404 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5348 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1118 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2953 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4307 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3497 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/djuna_mn-chinofun.json b/data/models/djuna_mn-chinofun.json deleted file mode 100644 index 46f40638cf674cf910a8ff35d7d38c6c7b068d51..0000000000000000000000000000000000000000 --- a/data/models/djuna_mn-chinofun.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MN-Chinofun", - "id": "djuna/MN-Chinofun", - "developer": "djuna", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/djuna_MN-Chinofun/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.611 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4953 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1307 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2961 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4084 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3603 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/djuna_q2.5-partron-7b.json b/data/models/djuna_q2.5-partron-7b.json deleted file mode 100644 index eac8c877b1f583bed73f12fc43293e8c9ef7b163..0000000000000000000000000000000000000000 --- a/data/models/djuna_q2.5-partron-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Q2.5-Partron-7B", - "id": "djuna/Q2.5-Partron-7B", - "developer": "djuna", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.613" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/djuna_Q2.5-Partron-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7321 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5418 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4826 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2978 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4165 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4283 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/djuna_q2.5-veltha-14b-0.5.json b/data/models/djuna_q2.5-veltha-14b-0.5.json deleted file mode 100644 index 6472cf484ba95b47c7c147b8f31271586b73e4e6..0000000000000000000000000000000000000000 --- a/data/models/djuna_q2.5-veltha-14b-0.5.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Q2.5-Veltha-14B-0.5", - "id": "djuna/Q2.5-Veltha-14B-0.5", - "developer": "djuna", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/djuna_Q2.5-Veltha-14B-0.5/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7796 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6523 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4373 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3683 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4339 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5295 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/djuna_q2.5-veltha-14b.json b/data/models/djuna_q2.5-veltha-14b.json deleted file mode 100644 index 6c4ad3f9970f8e5d494f0d70bb12abbec51d86f5..0000000000000000000000000000000000000000 --- a/data/models/djuna_q2.5-veltha-14b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Q2.5-Veltha-14B", - "id": "djuna/Q2.5-Veltha-14B", - "developer": "djuna", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/djuna_Q2.5-Veltha-14B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8292 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6484 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4789 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3591 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4194 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5298 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dnhkng_rys-llama-3-8b-instruct.json b/data/models/dnhkng_rys-llama-3-8b-instruct.json deleted file mode 100644 index 3a8651f9b61e909dbba1292c9bc6b381c5881860..0000000000000000000000000000000000000000 --- a/data/models/dnhkng_rys-llama-3-8b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "RYS-Llama-3-8B-Instruct", - "id": "dnhkng/RYS-Llama-3-8B-Instruct", - "developer": "dnhkng", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/dnhkng_RYS-Llama-3-8B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6958 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4809 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0687 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2576 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3383 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3557 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dnhkng_rys-llama-3-huge-instruct.json b/data/models/dnhkng_rys-llama-3-huge-instruct.json deleted file mode 100644 index 0e91acff84375dda4425a535f5435f1c36d51986..0000000000000000000000000000000000000000 --- a/data/models/dnhkng_rys-llama-3-huge-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "RYS-Llama-3-Huge-Instruct", - "id": "dnhkng/RYS-Llama-3-Huge-Instruct", - "developer": "dnhkng", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "99.646" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/dnhkng_RYS-Llama-3-Huge-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7686 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6481 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2289 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2609 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4208 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.511 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dnhkng_rys-llama-3-large-instruct.json b/data/models/dnhkng_rys-llama-3-large-instruct.json deleted file mode 100644 index c9b6d9c63a0ee6b6df9fa36f32ae253541e34f8f..0000000000000000000000000000000000000000 --- a/data/models/dnhkng_rys-llama-3-large-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "RYS-Llama-3-Large-Instruct", - "id": "dnhkng/RYS-Llama-3-Large-Instruct", - "developer": "dnhkng", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "73.976" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/dnhkng_RYS-Llama-3-Large-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8051 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6525 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2304 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2894 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.418 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5137 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dnhkng_rys-llama-3.1-8b-instruct.json b/data/models/dnhkng_rys-llama-3.1-8b-instruct.json deleted file mode 100644 index ecba7359268fbf1b6f99ada57400a0d6ef040702..0000000000000000000000000000000000000000 --- a/data/models/dnhkng_rys-llama-3.1-8b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "RYS-Llama-3.1-8B-Instruct", - "id": "dnhkng/RYS-Llama-3.1-8B-Instruct", - "developer": "dnhkng", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "?", - "params_billions": "8.685" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/dnhkng_RYS-Llama-3.1-8B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7685 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5164 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1329 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2676 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3681 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3639 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dnhkng_rys-llama3.1-large.json b/data/models/dnhkng_rys-llama3.1-large.json deleted file mode 100644 index c1cebea7a9d246a6a4dca91ed7160898d6ffd286..0000000000000000000000000000000000000000 --- a/data/models/dnhkng_rys-llama3.1-large.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "RYS-Llama3.1-Large", - "id": "dnhkng/RYS-Llama3.1-Large", - "developer": "dnhkng", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "81.677" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/dnhkng_RYS-Llama3.1-Large/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8492 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6899 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3505 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3742 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4554 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5249 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dnhkng_rys-medium.json b/data/models/dnhkng_rys-medium.json deleted file mode 100644 index c005e51d436300e1e66ce05a8e78bf793a620c4c..0000000000000000000000000000000000000000 --- a/data/models/dnhkng_rys-medium.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "RYS-Medium", - "id": "dnhkng/RYS-Medium", - "developer": "dnhkng", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Phi3ForCausalLM", - "params_billions": "18.731" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/dnhkng_RYS-Medium/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4406 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6285 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.108 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.328 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4069 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4326 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dnhkng_rys-phi-3-medium-4k-instruct.json b/data/models/dnhkng_rys-phi-3-medium-4k-instruct.json deleted file mode 100644 index 68090eafc67e41bec47829bd4ac9d41d99a9b497..0000000000000000000000000000000000000000 --- a/data/models/dnhkng_rys-phi-3-medium-4k-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "RYS-Phi-3-medium-4k-instruct", - "id": "dnhkng/RYS-Phi-3-medium-4k-instruct", - "developer": "dnhkng", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Phi3ForCausalLM", - "params_billions": "17.709" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/dnhkng_RYS-Phi-3-medium-4k-instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4391 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6226 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1609 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3549 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4253 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4846 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dnhkng_rys-xlarge-base.json b/data/models/dnhkng_rys-xlarge-base.json deleted file mode 100644 index 734c491a0fddf9b276d4e5b56fea01924ab9b47b..0000000000000000000000000000000000000000 --- a/data/models/dnhkng_rys-xlarge-base.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "RYS-XLarge-base", - "id": "dnhkng/RYS-XLarge-base", - "developer": "dnhkng", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "77.972" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/dnhkng_RYS-XLarge-base/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.791 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7047 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3792 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3792 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4903 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5431 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dnhkng_rys-xlarge.json b/data/models/dnhkng_rys-xlarge.json deleted file mode 100644 index f4964b483110aa18efe7bffe5141540f4aa1523a..0000000000000000000000000000000000000000 --- a/data/models/dnhkng_rys-xlarge.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "RYS-XLarge", - "id": "dnhkng/RYS-XLarge", - "developer": "dnhkng", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "77.965" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/dnhkng_RYS-XLarge/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7996 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.705 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4252 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3842 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.497 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5428 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dnhkng_rys-xlarge2.json b/data/models/dnhkng_rys-xlarge2.json deleted file mode 100644 index bf63c53f7e227a8b6d3a402b37c8245b4a2a6a98..0000000000000000000000000000000000000000 --- a/data/models/dnhkng_rys-xlarge2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "RYS-XLarge2", - "id": "dnhkng/RYS-XLarge2", - "developer": "dnhkng", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "77.965" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/dnhkng_RYS-XLarge2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4902 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6574 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2749 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3742 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4508 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5378 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dongwei_deepseek-r1-distill-qwen-7b-grpo.json b/data/models/dongwei_deepseek-r1-distill-qwen-7b-grpo.json deleted file mode 100644 index 0834385a271f0633ad089ec5f8f7da3a9d5a304c..0000000000000000000000000000000000000000 --- a/data/models/dongwei_deepseek-r1-distill-qwen-7b-grpo.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "DeepSeek-R1-Distill-Qwen-7B-GRPO", - "id": "Dongwei/DeepSeek-R1-Distill-Qwen-7B-GRPO", - "developer": "Dongwei", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Dongwei_DeepSeek-R1-Distill-Qwen-7B-GRPO/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4038 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3443 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1956 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2794 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3663 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2322 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/doppelreflex_l3-8b-r1-wolfcore-v1.5-test.json b/data/models/doppelreflex_l3-8b-r1-wolfcore-v1.5-test.json deleted file mode 100644 index 02689c49bf8e5cfbb3604192f92e39de557e968e..0000000000000000000000000000000000000000 --- a/data/models/doppelreflex_l3-8b-r1-wolfcore-v1.5-test.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "L3-8B-R1-WolfCore-V1.5-test", - "id": "DoppelReflEx/L3-8B-R1-WolfCore-V1.5-test", - "developer": "DoppelReflEx", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DoppelReflEx_L3-8B-R1-WolfCore-V1.5-test/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3955 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5315 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1231 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3263 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3841 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3728 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/doppelreflex_l3-8b-r1-wolfcore.json b/data/models/doppelreflex_l3-8b-r1-wolfcore.json deleted file mode 100644 index 442052d170e28e123f2f97dc79b180ae89918efd..0000000000000000000000000000000000000000 --- a/data/models/doppelreflex_l3-8b-r1-wolfcore.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "L3-8B-R1-WolfCore", - "id": "DoppelReflEx/L3-8B-R1-WolfCore", - "developer": "DoppelReflEx", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DoppelReflEx_L3-8B-R1-WolfCore/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3775 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5318 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1631 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3289 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4277 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3717 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/doppelreflex_l3-8b-wolfcore.json b/data/models/doppelreflex_l3-8b-wolfcore.json deleted file mode 100644 index 4fc005f5170138c9ac350b9acea42760fa0b084e..0000000000000000000000000000000000000000 --- a/data/models/doppelreflex_l3-8b-wolfcore.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "L3-8B-WolfCore", - "id": "DoppelReflEx/L3-8B-WolfCore", - "developer": "DoppelReflEx", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DoppelReflEx_L3-8B-WolfCore/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4022 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5182 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0982 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3096 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3973 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3705 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/doppelreflex_miniuslight-24b-test.json b/data/models/doppelreflex_miniuslight-24b-test.json deleted file mode 100644 index 82a330ae708782904098686c7510b92cee622973..0000000000000000000000000000000000000000 --- a/data/models/doppelreflex_miniuslight-24b-test.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MiniusLight-24B-test", - "id": "DoppelReflEx/MiniusLight-24B-test", - "developer": "DoppelReflEx", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "23.572" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DoppelReflEx_MiniusLight-24B-test/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0394 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6334 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0257 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3683 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4093 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5182 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/doppelreflex_miniuslight-24b-v1b-test.json b/data/models/doppelreflex_miniuslight-24b-v1b-test.json deleted file mode 100644 index fb5a747fbe064e7c94ecb93f224c373a46a56766..0000000000000000000000000000000000000000 --- a/data/models/doppelreflex_miniuslight-24b-v1b-test.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MiniusLight-24B-v1b-test", - "id": "DoppelReflEx/MiniusLight-24B-v1b-test", - "developer": "DoppelReflEx", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "23.572" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DoppelReflEx_MiniusLight-24B-v1b-test/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3791 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6617 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2394 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3792 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4557 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5365 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/doppelreflex_miniuslight-24b-v1c-test.json b/data/models/doppelreflex_miniuslight-24b-v1c-test.json deleted file mode 100644 index e54347529074c7446735e25fce8aa5ece41caf7a..0000000000000000000000000000000000000000 --- a/data/models/doppelreflex_miniuslight-24b-v1c-test.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MiniusLight-24B-v1c-test", - "id": "DoppelReflEx/MiniusLight-24B-v1c-test", - "developer": "DoppelReflEx", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "23.572" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DoppelReflEx_MiniusLight-24B-v1c-test/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3786 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6753 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2968 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3951 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4634 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5487 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/doppelreflex_miniuslight-24b-v1d-test.json b/data/models/doppelreflex_miniuslight-24b-v1d-test.json deleted file mode 100644 index 8bef837a45377ad1704b5367c217bbb7f1a58335..0000000000000000000000000000000000000000 --- a/data/models/doppelreflex_miniuslight-24b-v1d-test.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MiniusLight-24B-v1d-test", - "id": "DoppelReflEx/MiniusLight-24B-v1d-test", - "developer": "DoppelReflEx", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "23.572" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DoppelReflEx_MiniusLight-24B-v1d-test/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4032 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6712 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2946 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3951 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4621 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5489 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/doppelreflex_miniuslight-24b.json b/data/models/doppelreflex_miniuslight-24b.json deleted file mode 100644 index 3a0f316b5a2190383551981e5a7fbbfe185d4058..0000000000000000000000000000000000000000 --- a/data/models/doppelreflex_miniuslight-24b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MiniusLight-24B", - "id": "DoppelReflEx/MiniusLight-24B", - "developer": "DoppelReflEx", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "23.572" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DoppelReflEx_MiniusLight-24B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2577 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6256 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1261 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3582 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4319 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5091 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/doppelreflex_mn-12b-foxframe-test.json b/data/models/doppelreflex_mn-12b-foxframe-test.json deleted file mode 100644 index c9f28e439623410259f15d91dc95e913a4c14c71..0000000000000000000000000000000000000000 --- a/data/models/doppelreflex_mn-12b-foxframe-test.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MN-12B-FoxFrame-test", - "id": "DoppelReflEx/MN-12B-FoxFrame-test", - "developer": "DoppelReflEx", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DoppelReflEx_MN-12B-FoxFrame-test/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4222 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5456 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1397 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3079 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4254 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3503 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/doppelreflex_mn-12b-foxframe2-test.json b/data/models/doppelreflex_mn-12b-foxframe2-test.json deleted file mode 100644 index b6f96d79374dc0f6f936416467927d5106ad7329..0000000000000000000000000000000000000000 --- a/data/models/doppelreflex_mn-12b-foxframe2-test.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MN-12B-FoxFrame2-test", - "id": "DoppelReflEx/MN-12B-FoxFrame2-test", - "developer": "DoppelReflEx", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DoppelReflEx_MN-12B-FoxFrame2-test/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4319 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5485 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1405 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3146 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4252 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3569 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/doppelreflex_mn-12b-foxframe3-test.json b/data/models/doppelreflex_mn-12b-foxframe3-test.json deleted file mode 100644 index 95e99f045917a30ec1025c110ee9ade460b0a205..0000000000000000000000000000000000000000 --- a/data/models/doppelreflex_mn-12b-foxframe3-test.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MN-12B-FoxFrame3-test", - "id": "DoppelReflEx/MN-12B-FoxFrame3-test", - "developer": "DoppelReflEx", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DoppelReflEx_MN-12B-FoxFrame3-test/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4323 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5395 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1322 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3012 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4598 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3529 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/doppelreflex_mn-12b-kakigori.json b/data/models/doppelreflex_mn-12b-kakigori.json deleted file mode 100644 index a1a5e713e44b8516e6d49f6dcc83b022cc1bac00..0000000000000000000000000000000000000000 --- a/data/models/doppelreflex_mn-12b-kakigori.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MN-12B-Kakigori", - "id": "DoppelReflEx/MN-12B-Kakigori", - "developer": "DoppelReflEx", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DoppelReflEx_MN-12B-Kakigori/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3593 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5416 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1193 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3247 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4052 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3581 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/doppelreflex_mn-12b-lilithframe-experiment-2.json b/data/models/doppelreflex_mn-12b-lilithframe-experiment-2.json deleted file mode 100644 index 1a54e644385612651de81d1e769b353a433e8a48..0000000000000000000000000000000000000000 --- a/data/models/doppelreflex_mn-12b-lilithframe-experiment-2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MN-12B-LilithFrame-Experiment-2", - "id": "DoppelReflEx/MN-12B-LilithFrame-Experiment-2", - "developer": "DoppelReflEx", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DoppelReflEx_MN-12B-LilithFrame-Experiment-2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4299 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4983 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1073 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3255 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3804 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3276 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/doppelreflex_mn-12b-lilithframe-experiment-3.json b/data/models/doppelreflex_mn-12b-lilithframe-experiment-3.json deleted file mode 100644 index 8804bd3389b41a006acad55a79ae79bfb6b54353..0000000000000000000000000000000000000000 --- a/data/models/doppelreflex_mn-12b-lilithframe-experiment-3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MN-12B-LilithFrame-Experiment-3", - "id": "DoppelReflEx/MN-12B-LilithFrame-Experiment-3", - "developer": "DoppelReflEx", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DoppelReflEx_MN-12B-LilithFrame-Experiment-3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4128 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5468 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1344 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.328 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4039 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3604 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/doppelreflex_mn-12b-lilithframe-experiment-4.json b/data/models/doppelreflex_mn-12b-lilithframe-experiment-4.json deleted file mode 100644 index 3bcf152049e138f5dd9f45e33584f7470703a29b..0000000000000000000000000000000000000000 --- a/data/models/doppelreflex_mn-12b-lilithframe-experiment-4.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MN-12B-LilithFrame-Experiment-4", - "id": "DoppelReflEx/MN-12B-LilithFrame-Experiment-4", - "developer": "DoppelReflEx", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DoppelReflEx_MN-12B-LilithFrame-Experiment-4/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3981 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5534 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1224 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3171 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4371 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3649 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/doppelreflex_mn-12b-lilithframe.json b/data/models/doppelreflex_mn-12b-lilithframe.json deleted file mode 100644 index 720fbe0846130d5605f8d9f0e7d4c1731ec8e47c..0000000000000000000000000000000000000000 --- a/data/models/doppelreflex_mn-12b-lilithframe.json +++ /dev/null @@ -1,275 +0,0 @@ -{ - "model_info": { - "name": "MN-12B-LilithFrame", - "id": "DoppelReflEx/MN-12B-LilithFrame", - "developer": "DoppelReflEx", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DoppelReflEx_MN-12B-LilithFrame/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.451 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4944 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1156 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3196 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3896 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3256 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "hfopenllm_v2/DoppelReflEx_MN-12B-LilithFrame/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.436 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4956 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0589 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3205 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3843 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3237 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/doppelreflex_mn-12b-mimicore-greensnake.json b/data/models/doppelreflex_mn-12b-mimicore-greensnake.json deleted file mode 100644 index ff3815b210d5a410ce02ae6e55155e8802c90f1e..0000000000000000000000000000000000000000 --- a/data/models/doppelreflex_mn-12b-mimicore-greensnake.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MN-12B-Mimicore-GreenSnake", - "id": "DoppelReflEx/MN-12B-Mimicore-GreenSnake", - "developer": "DoppelReflEx", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DoppelReflEx_MN-12B-Mimicore-GreenSnake/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.478 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5481 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.139 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3247 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4306 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3651 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/doppelreflex_mn-12b-mimicore-nocturne.json b/data/models/doppelreflex_mn-12b-mimicore-nocturne.json deleted file mode 100644 index fd849baa5c064eb09259b0c7c53bed8a64119ca9..0000000000000000000000000000000000000000 --- a/data/models/doppelreflex_mn-12b-mimicore-nocturne.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MN-12B-Mimicore-Nocturne", - "id": "DoppelReflEx/MN-12B-Mimicore-Nocturne", - "developer": "DoppelReflEx", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DoppelReflEx_MN-12B-Mimicore-Nocturne/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3957 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5703 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1057 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3196 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4569 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3634 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/doppelreflex_mn-12b-mimicore-orochi-v2-experiment.json b/data/models/doppelreflex_mn-12b-mimicore-orochi-v2-experiment.json deleted file mode 100644 index 60d786e842b3f75fc08136f1fcefdd41eb117306..0000000000000000000000000000000000000000 --- a/data/models/doppelreflex_mn-12b-mimicore-orochi-v2-experiment.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MN-12B-Mimicore-Orochi-v2-Experiment", - "id": "DoppelReflEx/MN-12B-Mimicore-Orochi-v2-Experiment", - "developer": "DoppelReflEx", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DoppelReflEx_MN-12B-Mimicore-Orochi-v2-Experiment/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2842 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5323 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0612 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2978 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4574 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3423 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/doppelreflex_mn-12b-mimicore-orochi-v3-experiment.json b/data/models/doppelreflex_mn-12b-mimicore-orochi-v3-experiment.json deleted file mode 100644 index 8c9eac7b7e14e772304b849be8ba948280adfe6c..0000000000000000000000000000000000000000 --- a/data/models/doppelreflex_mn-12b-mimicore-orochi-v3-experiment.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MN-12B-Mimicore-Orochi-v3-Experiment", - "id": "DoppelReflEx/MN-12B-Mimicore-Orochi-v3-Experiment", - "developer": "DoppelReflEx", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DoppelReflEx_MN-12B-Mimicore-Orochi-v3-Experiment/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4102 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5438 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1216 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2928 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4438 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3396 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/doppelreflex_mn-12b-mimicore-orochi-v4-experiment.json b/data/models/doppelreflex_mn-12b-mimicore-orochi-v4-experiment.json deleted file mode 100644 index 580bbdeb8e2e25eadbbf990815d8fc183117273f..0000000000000000000000000000000000000000 --- a/data/models/doppelreflex_mn-12b-mimicore-orochi-v4-experiment.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MN-12B-Mimicore-Orochi-v4-Experiment", - "id": "DoppelReflEx/MN-12B-Mimicore-Orochi-v4-Experiment", - "developer": "DoppelReflEx", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DoppelReflEx_MN-12B-Mimicore-Orochi-v4-Experiment/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4321 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5463 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1208 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3054 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4449 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.352 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/doppelreflex_mn-12b-mimicore-orochi.json b/data/models/doppelreflex_mn-12b-mimicore-orochi.json deleted file mode 100644 index 30e87f5e969ff7cff638f711b94adf9177722859..0000000000000000000000000000000000000000 --- a/data/models/doppelreflex_mn-12b-mimicore-orochi.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MN-12B-Mimicore-Orochi", - "id": "DoppelReflEx/MN-12B-Mimicore-Orochi", - "developer": "DoppelReflEx", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DoppelReflEx_MN-12B-Mimicore-Orochi/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.462 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5498 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.136 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3129 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4546 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3447 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/doppelreflex_mn-12b-mimicore-whitesnake-v2-experiment-1.json b/data/models/doppelreflex_mn-12b-mimicore-whitesnake-v2-experiment-1.json deleted file mode 100644 index 1a323c78ee3bd3ce7c4e317eb54fb0f94f0e7e96..0000000000000000000000000000000000000000 --- a/data/models/doppelreflex_mn-12b-mimicore-whitesnake-v2-experiment-1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MN-12B-Mimicore-WhiteSnake-v2-Experiment-1", - "id": "DoppelReflEx/MN-12B-Mimicore-WhiteSnake-v2-Experiment-1", - "developer": "DoppelReflEx", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DoppelReflEx_MN-12B-Mimicore-WhiteSnake-v2-Experiment-1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3909 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4866 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0785 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3054 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.379 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3114 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/doppelreflex_mn-12b-mimicore-whitesnake-v2-experiment-2.json b/data/models/doppelreflex_mn-12b-mimicore-whitesnake-v2-experiment-2.json deleted file mode 100644 index 9c69ba9323b16428e663a371c66025d6639f63c4..0000000000000000000000000000000000000000 --- a/data/models/doppelreflex_mn-12b-mimicore-whitesnake-v2-experiment-2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MN-12B-Mimicore-WhiteSnake-v2-Experiment-2", - "id": "DoppelReflEx/MN-12B-Mimicore-WhiteSnake-v2-Experiment-2", - "developer": "DoppelReflEx", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DoppelReflEx_MN-12B-Mimicore-WhiteSnake-v2-Experiment-2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3124 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5126 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1125 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2961 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3975 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3314 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/doppelreflex_mn-12b-mimicore-whitesnake-v2-experiment-3.json b/data/models/doppelreflex_mn-12b-mimicore-whitesnake-v2-experiment-3.json deleted file mode 100644 index be43ba386401d525a47ac4f4226dd6899b4de60d..0000000000000000000000000000000000000000 --- a/data/models/doppelreflex_mn-12b-mimicore-whitesnake-v2-experiment-3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MN-12B-Mimicore-WhiteSnake-v2-Experiment-3", - "id": "DoppelReflEx/MN-12B-Mimicore-WhiteSnake-v2-Experiment-3", - "developer": "DoppelReflEx", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DoppelReflEx_MN-12B-Mimicore-WhiteSnake-v2-Experiment-3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4302 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4812 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0899 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.302 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3684 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3198 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/doppelreflex_mn-12b-mimicore-whitesnake-v2-experiment-4.json b/data/models/doppelreflex_mn-12b-mimicore-whitesnake-v2-experiment-4.json deleted file mode 100644 index 67b8a4ed079d92ba60f7ab35ea4e1363ce7b57b6..0000000000000000000000000000000000000000 --- a/data/models/doppelreflex_mn-12b-mimicore-whitesnake-v2-experiment-4.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MN-12B-Mimicore-WhiteSnake-v2-Experiment-4", - "id": "DoppelReflEx/MN-12B-Mimicore-WhiteSnake-v2-Experiment-4", - "developer": "DoppelReflEx", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DoppelReflEx_MN-12B-Mimicore-WhiteSnake-v2-Experiment-4/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4241 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5185 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.114 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3104 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4002 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3342 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/doppelreflex_mn-12b-mimicore-whitesnake.json b/data/models/doppelreflex_mn-12b-mimicore-whitesnake.json deleted file mode 100644 index 7885a296c65bcab06159d63e6048804f1511fab7..0000000000000000000000000000000000000000 --- a/data/models/doppelreflex_mn-12b-mimicore-whitesnake.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MN-12B-Mimicore-WhiteSnake", - "id": "DoppelReflEx/MN-12B-Mimicore-WhiteSnake", - "developer": "DoppelReflEx", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DoppelReflEx_MN-12B-Mimicore-WhiteSnake/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4438 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5605 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1314 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.318 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4569 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3658 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/doppelreflex_mn-12b-unleashed-twilight.json b/data/models/doppelreflex_mn-12b-unleashed-twilight.json deleted file mode 100644 index dc465f52d6e302eb9aea3ddbb30095b58235c8b6..0000000000000000000000000000000000000000 --- a/data/models/doppelreflex_mn-12b-unleashed-twilight.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MN-12B-Unleashed-Twilight", - "id": "DoppelReflEx/MN-12B-Unleashed-Twilight", - "developer": "DoppelReflEx", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DoppelReflEx_MN-12B-Unleashed-Twilight/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3505 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5521 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0959 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3289 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4384 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3678 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/doppelreflex_mn-12b-wolframe.json b/data/models/doppelreflex_mn-12b-wolframe.json deleted file mode 100644 index 72bb4e8e7ba95f4c6674d5a68623da5dc101986a..0000000000000000000000000000000000000000 --- a/data/models/doppelreflex_mn-12b-wolframe.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MN-12B-WolFrame", - "id": "DoppelReflEx/MN-12B-WolFrame", - "developer": "DoppelReflEx", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DoppelReflEx_MN-12B-WolFrame/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4397 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5117 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1314 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3104 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4015 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3393 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_again-8b-model_stock.json b/data/models/dreadpoor_again-8b-model_stock.json deleted file mode 100644 index 3421fa3ad1add6457d29dff35908d47b348112c2..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_again-8b-model_stock.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Again-8B-Model_Stock", - "id": "DreadPoor/Again-8B-Model_Stock", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "4.015" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_Again-8B-Model_Stock/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6724 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.531 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1201 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3012 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3987 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3518 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_alita99-8b-linear.json b/data/models/dreadpoor_alita99-8b-linear.json deleted file mode 100644 index 14942d636503a29bd49ef6741480b36f2b124009..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_alita99-8b-linear.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Alita99-8B-LINEAR", - "id": "DreadPoor/Alita99-8B-LINEAR", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_Alita99-8B-LINEAR/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.719 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5442 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1647 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3163 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4266 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3809 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_anothertest.json b/data/models/dreadpoor_anothertest.json deleted file mode 100644 index ddcb151940a3a77c5b21e0166d9505ed1d2d497e..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_anothertest.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "AnotherTest", - "id": "DreadPoor/AnotherTest", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_AnotherTest/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4701 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4683 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0619 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2978 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4213 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2875 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_aspire-8b-model_stock.json b/data/models/dreadpoor_aspire-8b-model_stock.json deleted file mode 100644 index c4230829086c86a08ff7bee899a7d5b1e8866b51..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_aspire-8b-model_stock.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Aspire-8B-model_stock", - "id": "DreadPoor/Aspire-8B-model_stock", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_Aspire-8B-model_stock/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7141 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5278 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1495 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3146 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4212 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3763 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_aspire_1.3-8b_model-stock.json b/data/models/dreadpoor_aspire_1.3-8b_model-stock.json deleted file mode 100644 index c780223113a10843d607c7dc485cd0a0660175d3..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_aspire_1.3-8b_model-stock.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Aspire_1.3-8B_model-stock", - "id": "DreadPoor/Aspire_1.3-8B_model-stock", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_Aspire_1.3-8B_model-stock/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7062 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5302 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1692 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3079 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4105 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3716 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_aspire_v2-8b-model_stock.json b/data/models/dreadpoor_aspire_v2-8b-model_stock.json deleted file mode 100644 index af9c17f645d6f7e38037db4de79bc03dc684f039..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_aspire_v2-8b-model_stock.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Aspire_V2-8B-Model_Stock", - "id": "DreadPoor/Aspire_V2-8B-Model_Stock", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_Aspire_V2-8B-Model_Stock/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7371 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.533 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.176 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3205 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3894 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3697 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_aspire_v2.1-8b-model_stock.json b/data/models/dreadpoor_aspire_v2.1-8b-model_stock.json deleted file mode 100644 index 37fe30553a412ffdbc512772c53310b1908d3d34..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_aspire_v2.1-8b-model_stock.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Aspire_V2.1-8B-Model_Stock", - "id": "DreadPoor/Aspire_V2.1-8B-Model_Stock", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_Aspire_V2.1-8B-Model_Stock/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7238 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5236 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1767 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3096 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4136 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3801 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_aspire_v2_alt-8b-model_stock.json b/data/models/dreadpoor_aspire_v2_alt-8b-model_stock.json deleted file mode 100644 index 6dc945652067d67669902bb253a8026a33aed030..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_aspire_v2_alt-8b-model_stock.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Aspire_V2_ALT-8B-Model_Stock", - "id": "DreadPoor/Aspire_V2_ALT-8B-Model_Stock", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_Aspire_V2_ALT-8B-Model_Stock/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7381 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5266 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.173 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3247 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3975 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3727 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_aspire_v2_alt_row-8b-model_stock.json b/data/models/dreadpoor_aspire_v2_alt_row-8b-model_stock.json deleted file mode 100644 index 3394bae72e9d13c1e781499fb00f4f59188e89b0..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_aspire_v2_alt_row-8b-model_stock.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Aspire_V2_ALT_ROW-8B-Model_Stock", - "id": "DreadPoor/Aspire_V2_ALT_ROW-8B-Model_Stock", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_Aspire_V2_ALT_ROW-8B-Model_Stock/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7381 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5266 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.173 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3247 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3975 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3727 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_aspire_v3-8b-model_stock.json b/data/models/dreadpoor_aspire_v3-8b-model_stock.json deleted file mode 100644 index 6b7321e69ca779bcfb7d43e042f3bd36c1d1290c..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_aspire_v3-8b-model_stock.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Aspire_V3-8B-Model_Stock", - "id": "DreadPoor/Aspire_V3-8B-Model_Stock", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_Aspire_V3-8B-Model_Stock/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5119 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5268 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1858 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3054 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4015 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3642 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_aspire_v4-8b-model_stock.json b/data/models/dreadpoor_aspire_v4-8b-model_stock.json deleted file mode 100644 index b637447fcd91075fa610b278bdc093ce057d0697..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_aspire_v4-8b-model_stock.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Aspire_V4-8B-Model_Stock", - "id": "DreadPoor/Aspire_V4-8B-Model_Stock", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_Aspire_V4-8B-Model_Stock/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7694 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5314 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1926 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3045 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3867 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3708 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_aspire_v4_alt-8b-model_stock.json b/data/models/dreadpoor_aspire_v4_alt-8b-model_stock.json deleted file mode 100644 index 134f8e83e21614cc7086fd3689dc2ae542399d83..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_aspire_v4_alt-8b-model_stock.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Aspire_V4_ALT-8B-Model_Stock", - "id": "DreadPoor/Aspire_V4_ALT-8B-Model_Stock", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_Aspire_V4_ALT-8B-Model_Stock/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7366 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5268 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1813 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3205 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.392 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3682 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_asymmetric_linearity-8b-model_stock.json b/data/models/dreadpoor_asymmetric_linearity-8b-model_stock.json deleted file mode 100644 index 5b54b6e9c62154fef252e8ce4f06158e2900102a..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_asymmetric_linearity-8b-model_stock.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Asymmetric_Linearity-8B-Model_Stock", - "id": "DreadPoor/Asymmetric_Linearity-8B-Model_Stock", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "4.015" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_Asymmetric_Linearity-8B-Model_Stock/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7174 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5465 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1647 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3146 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4199 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3844 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_aurora_faustus-8b-linear.json b/data/models/dreadpoor_aurora_faustus-8b-linear.json deleted file mode 100644 index 9b57fa46b58453e8d9d198ba348c0a6c5cb5d012..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_aurora_faustus-8b-linear.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Aurora_faustus-8B-LINEAR", - "id": "DreadPoor/Aurora_faustus-8B-LINEAR", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_Aurora_faustus-8B-LINEAR/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7281 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5516 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1707 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.307 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4146 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3842 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_aurora_faustus-8b-lorablated.json b/data/models/dreadpoor_aurora_faustus-8b-lorablated.json deleted file mode 100644 index 02fd3bb694f0cfd8775ef426b8f39d631c8d2402..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_aurora_faustus-8b-lorablated.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Aurora_faustus-8B-LORABLATED", - "id": "DreadPoor/Aurora_faustus-8B-LORABLATED", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_Aurora_faustus-8B-LORABLATED/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7527 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5392 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1488 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.302 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4239 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3673 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_aurora_faustus-8b-lorablated_alt.json b/data/models/dreadpoor_aurora_faustus-8b-lorablated_alt.json deleted file mode 100644 index f2d6bda0337b6b06a11b794f3f90898b30734992..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_aurora_faustus-8b-lorablated_alt.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Aurora_faustus-8B-LORABLATED_ALT", - "id": "DreadPoor/Aurora_faustus-8B-LORABLATED_ALT", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_Aurora_faustus-8B-LORABLATED_ALT/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7378 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5388 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1586 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2987 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4225 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3694 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_autumn_dawn-8b-linear.json b/data/models/dreadpoor_autumn_dawn-8b-linear.json deleted file mode 100644 index 9eefabc5837638e39c088278316099a2c500fa39..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_autumn_dawn-8b-linear.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Autumn_Dawn-8B-LINEAR", - "id": "DreadPoor/Autumn_Dawn-8B-LINEAR", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_Autumn_Dawn-8B-LINEAR/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7293 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5459 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1858 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2936 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4186 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3968 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_baezel-8b-linear.json b/data/models/dreadpoor_baezel-8b-linear.json deleted file mode 100644 index 1e3c807cdc0fdaa5e46098021683b82cfb90b390..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_baezel-8b-linear.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "BaeZel-8B-LINEAR", - "id": "DreadPoor/BaeZel-8B-LINEAR", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_BaeZel-8B-LINEAR/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7378 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5464 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1813 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3213 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4227 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3861 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_baezel-8b-model_stock.json b/data/models/dreadpoor_baezel-8b-model_stock.json deleted file mode 100644 index e902e14a1eff397f69b0899e8174383fab8eb0c2..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_baezel-8b-model_stock.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "BaeZel-8B-Model_Stock", - "id": "DreadPoor/BaeZel-8B-Model_Stock", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_BaeZel-8B-Model_Stock/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7713 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5408 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1639 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3138 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4199 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.388 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_baezel_v2-8b-model_stock.json b/data/models/dreadpoor_baezel_v2-8b-model_stock.json deleted file mode 100644 index 245f956302abdcec9282b2c3b9ca197741811666..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_baezel_v2-8b-model_stock.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "BaeZel_V2-8B-Model_Stock", - "id": "DreadPoor/BaeZel_V2-8B-Model_Stock", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_BaeZel_V2-8B-Model_Stock/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7677 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5374 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1798 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2995 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4186 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3947 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_baezel_v2_alt-8b-model_stock.json b/data/models/dreadpoor_baezel_v2_alt-8b-model_stock.json deleted file mode 100644 index 324c2ff89aa029c8716c82d3d6e79c2c165ca071..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_baezel_v2_alt-8b-model_stock.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "BaeZel_V2_ALT-8B-Model_Stock", - "id": "DreadPoor/BaeZel_V2_ALT-8B-Model_Stock", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_BaeZel_V2_ALT-8B-Model_Stock/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7677 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5374 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1798 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2995 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4186 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3947 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_baezel_v3-8b-model_stock.json b/data/models/dreadpoor_baezel_v3-8b-model_stock.json deleted file mode 100644 index 7da19fcc20c09a804085a726eedfe3ff0ea63c84..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_baezel_v3-8b-model_stock.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "BaeZel_V3-8B-Model_Stock", - "id": "DreadPoor/BaeZel_V3-8B-Model_Stock", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_BaeZel_V3-8B-Model_Stock/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7832 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5392 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1896 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3205 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4174 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3888 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_blunt_edge-8b-slerp.json b/data/models/dreadpoor_blunt_edge-8b-slerp.json deleted file mode 100644 index e68c4a66844cbc2d41309c6c124df2a6d1540fd0..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_blunt_edge-8b-slerp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Blunt_Edge-8B-SLERP", - "id": "DreadPoor/Blunt_Edge-8B-SLERP", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_Blunt_Edge-8B-SLERP/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7497 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5389 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1858 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3112 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4174 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3767 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_bulkup.json b/data/models/dreadpoor_bulkup.json deleted file mode 100644 index 1d34960e9ae8a18655937af354b21b1bece0b184..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_bulkup.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "BulkUp", - "id": "DreadPoor/BulkUp", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_BulkUp/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1778 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.287 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2475 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3447 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.111 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_cadence-8b-linear.json b/data/models/dreadpoor_cadence-8b-linear.json deleted file mode 100644 index bc97cf98daac707d06450719d3865a6b815a4131..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_cadence-8b-linear.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Cadence-8B-LINEAR", - "id": "DreadPoor/Cadence-8B-LINEAR", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_Cadence-8B-LINEAR/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7682 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5433 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1677 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3029 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4173 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3803 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_caelid-8b-model_stock.json b/data/models/dreadpoor_caelid-8b-model_stock.json deleted file mode 100644 index 65be90c2beb164f706c0883747e3dbc2dc913d7c..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_caelid-8b-model_stock.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Caelid-8B-Model_Stock", - "id": "DreadPoor/Caelid-8B-Model_Stock", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_Caelid-8B-Model_Stock/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7247 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.546 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1511 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3104 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4001 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3816 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_casuar-9b-model_stock.json b/data/models/dreadpoor_casuar-9b-model_stock.json deleted file mode 100644 index edad493fe320e1b274e98cce5ee0244d2b1b7464..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_casuar-9b-model_stock.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Casuar-9B-Model_Stock", - "id": "DreadPoor/Casuar-9B-Model_Stock", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "9.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_Casuar-9B-Model_Stock/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7765 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6107 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.213 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3448 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4165 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4156 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_condensed_milk-8b-model_stock.json b/data/models/dreadpoor_condensed_milk-8b-model_stock.json deleted file mode 100644 index 7b25d7331f5ba5c635c88de670cf46a93311fa86..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_condensed_milk-8b-model_stock.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Condensed_Milk-8B-Model_Stock", - "id": "DreadPoor/Condensed_Milk-8B-Model_Stock", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_Condensed_Milk-8B-Model_Stock/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7536 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5435 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1745 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3213 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.416 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3876 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_coolercoder-8b-linear.json b/data/models/dreadpoor_coolercoder-8b-linear.json deleted file mode 100644 index 892a4a40e4b8aadc4c7aaed749484678bd393b1c..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_coolercoder-8b-linear.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "CoolerCoder-8B-LINEAR", - "id": "DreadPoor/CoolerCoder-8B-LINEAR", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_CoolerCoder-8B-LINEAR/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4519 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4762 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0793 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2903 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3964 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3159 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_damasteel-8b-linear.json b/data/models/dreadpoor_damasteel-8b-linear.json deleted file mode 100644 index 258e7020d50fb6f048e58c65c587b3094d7d8fce..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_damasteel-8b-linear.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Damasteel-8B-LINEAR", - "id": "DreadPoor/Damasteel-8B-LINEAR", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_Damasteel-8B-LINEAR/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7384 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5388 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1669 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2987 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4212 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3779 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_dearly_beloved-8b-ties.json b/data/models/dreadpoor_dearly_beloved-8b-ties.json deleted file mode 100644 index 42fce281822aa2b0b478091d6604d1fbf58b3b7e..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_dearly_beloved-8b-ties.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Dearly_Beloved-8B-TIES", - "id": "DreadPoor/Dearly_Beloved-8B-TIES", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_Dearly_Beloved-8B-TIES/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8267 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.405 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2115 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2987 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4175 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2827 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_decayed-8b-linear.json b/data/models/dreadpoor_decayed-8b-linear.json deleted file mode 100644 index 3ab6f9f449868a348854ebb003da8660e2b2315b..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_decayed-8b-linear.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Decayed-8B-LINEAR", - "id": "DreadPoor/Decayed-8B-LINEAR", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_Decayed-8B-LINEAR/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7676 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5417 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1715 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3096 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4186 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3763 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_derivative-8b-model_stock.json b/data/models/dreadpoor_derivative-8b-model_stock.json deleted file mode 100644 index 753060b3571a4329564525ea1e3d2e29d11725a5..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_derivative-8b-model_stock.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Derivative-8B-Model_Stock", - "id": "DreadPoor/Derivative-8B-Model_Stock", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_Derivative-8B-Model_Stock/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7667 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5395 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.179 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3171 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.42 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3811 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_derivative_v2-8b-model_stock.json b/data/models/dreadpoor_derivative_v2-8b-model_stock.json deleted file mode 100644 index 41c562f29285703b7a1f78b65b9a5ea6bd1264eb..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_derivative_v2-8b-model_stock.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Derivative_V2-8B-Model_Stock", - "id": "DreadPoor/Derivative_V2-8B-Model_Stock", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_Derivative_V2-8B-Model_Stock/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7537 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5393 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1798 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.307 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4123 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3856 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_derivative_v2_alt-8b-model_stock.json b/data/models/dreadpoor_derivative_v2_alt-8b-model_stock.json deleted file mode 100644 index 3bdb209fe74ea0fc859181e7d00f5eca7d0052e6..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_derivative_v2_alt-8b-model_stock.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Derivative_V2_ALT-8B-Model_Stock", - "id": "DreadPoor/Derivative_V2_ALT-8B-Model_Stock", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_Derivative_V2_ALT-8B-Model_Stock/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.772 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5365 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1881 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3112 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4135 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3882 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_derivative_v3-8b-model_stock.json b/data/models/dreadpoor_derivative_v3-8b-model_stock.json deleted file mode 100644 index 52401e9861ba47d9a79af1944d43364b8b4a956e..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_derivative_v3-8b-model_stock.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Derivative_V3-8B-Model_Stock", - "id": "DreadPoor/Derivative_V3-8B-Model_Stock", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_Derivative_V3-8B-Model_Stock/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6964 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5243 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1465 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2945 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.415 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3502 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_elusive_dragon_heart-8b-linear.json b/data/models/dreadpoor_elusive_dragon_heart-8b-linear.json deleted file mode 100644 index 150a647e631467b48c5906cc5915c82241bdccb4..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_elusive_dragon_heart-8b-linear.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Elusive_Dragon_Heart-8B-LINEAR", - "id": "DreadPoor/Elusive_Dragon_Heart-8B-LINEAR", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "4.015" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_Elusive_Dragon_Heart-8B-LINEAR/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7131 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5456 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.148 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3062 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4146 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3814 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_emu_eggs-9b-model_stock.json b/data/models/dreadpoor_emu_eggs-9b-model_stock.json deleted file mode 100644 index 89811847983c47b0a2e785150365fe282c8b270f..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_emu_eggs-9b-model_stock.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Emu_Eggs-9B-Model_Stock", - "id": "DreadPoor/Emu_Eggs-9B-Model_Stock", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "9.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_Emu_Eggs-9B-Model_Stock/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7607 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6052 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.21 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3331 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4071 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4227 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_eunoia_vespera-8b-linear.json b/data/models/dreadpoor_eunoia_vespera-8b-linear.json deleted file mode 100644 index 488de7ab918340ca4da1aa6df9e20def20012360..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_eunoia_vespera-8b-linear.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Eunoia_Vespera-8B-LINEAR", - "id": "DreadPoor/Eunoia_Vespera-8B-LINEAR", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_Eunoia_Vespera-8B-LINEAR/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7235 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5399 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1541 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.307 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4185 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3839 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_felix_dies-mistral-7b-model_stock.json b/data/models/dreadpoor_felix_dies-mistral-7b-model_stock.json deleted file mode 100644 index e4601378129282063e63e7b032470d6ab3df7e1c..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_felix_dies-mistral-7b-model_stock.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "felix_dies-mistral-7B-model_stock", - "id": "DreadPoor/felix_dies-mistral-7B-model_stock", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_felix_dies-mistral-7B-model_stock/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3008 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4901 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0536 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2919 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4518 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3109 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_fu_sion_ha-8b-slerp.json b/data/models/dreadpoor_fu_sion_ha-8b-slerp.json deleted file mode 100644 index 36890b19455586e8676a0867a64129c6d643e5a9..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_fu_sion_ha-8b-slerp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Fu_sion_HA-8B-SLERP", - "id": "DreadPoor/Fu_sion_HA-8B-SLERP", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_Fu_sion_HA-8B-SLERP/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7609 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5373 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1752 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.323 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.416 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3825 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_h_the_eighth-8b-linear.json b/data/models/dreadpoor_h_the_eighth-8b-linear.json deleted file mode 100644 index fa726e046ffc123cd92feb5cf681a89cb8b2f2ce..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_h_the_eighth-8b-linear.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "H_the_eighth-8B-LINEAR", - "id": "DreadPoor/H_the_eighth-8B-LINEAR", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_H_the_eighth-8B-LINEAR/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7469 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5384 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1775 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.328 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4173 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3824 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_hakuchido-8b-model_stock.json b/data/models/dreadpoor_hakuchido-8b-model_stock.json deleted file mode 100644 index 919d16a2f2521fdee2953936afdda3cf0f566298..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_hakuchido-8b-model_stock.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "hakuchido-8B-MODEL_STOCK", - "id": "DreadPoor/hakuchido-8B-MODEL_STOCK", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_hakuchido-8B-MODEL_STOCK/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7375 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5398 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1949 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2953 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4175 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3782 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_happy_new_year-8b-model_stock.json b/data/models/dreadpoor_happy_new_year-8b-model_stock.json deleted file mode 100644 index dc66b6359d3f96371106ce8a3922bb0c05637478..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_happy_new_year-8b-model_stock.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Happy_New_Year-8B-Model_Stock", - "id": "DreadPoor/Happy_New_Year-8B-Model_Stock", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_Happy_New_Year-8B-Model_Stock/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7616 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5368 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1594 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3138 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4186 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3879 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_heart_stolen-8b-model_stock.json b/data/models/dreadpoor_heart_stolen-8b-model_stock.json deleted file mode 100644 index 85019d6d5e3142de52f4f8d6c67a555d26a31a5f..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_heart_stolen-8b-model_stock.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Heart_Stolen-8B-Model_Stock", - "id": "DreadPoor/Heart_Stolen-8B-Model_Stock", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_Heart_Stolen-8B-Model_Stock/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7245 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5395 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1722 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3171 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4162 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3794 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_heart_stolen-alt-8b-model_stock.json b/data/models/dreadpoor_heart_stolen-alt-8b-model_stock.json deleted file mode 100644 index c7c7cdeddc03c4612f87b4c545481be283017e7a..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_heart_stolen-alt-8b-model_stock.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Heart_Stolen-ALT-8B-Model_Stock", - "id": "DreadPoor/Heart_Stolen-ALT-8B-Model_Stock", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_Heart_Stolen-ALT-8B-Model_Stock/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7184 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5263 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1563 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3012 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4055 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3772 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_here_we_go_again-8b-slerp.json b/data/models/dreadpoor_here_we_go_again-8b-slerp.json deleted file mode 100644 index ee2da555fe170f0d874d01a0218b7d3d08b0090b..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_here_we_go_again-8b-slerp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Here_We_Go_Again-8B-SLERP", - "id": "DreadPoor/Here_We_Go_Again-8B-SLERP", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "4.015" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_Here_We_Go_Again-8B-SLERP/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7442 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.546 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.173 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3188 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4187 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3873 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_hot_stinking_garbage.json b/data/models/dreadpoor_hot_stinking_garbage.json deleted file mode 100644 index 7e963a593ddeb3efef05a47bdee5ff3a6ead064d..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_hot_stinking_garbage.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "HOT_STINKING_GARBAGE", - "id": "DreadPoor/HOT_STINKING_GARBAGE", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_HOT_STINKING_GARBAGE/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5754 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4884 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0672 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2752 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.425 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3017 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_howdy-8b-linear.json b/data/models/dreadpoor_howdy-8b-linear.json deleted file mode 100644 index 280f39ede80b09ae565022e6e4277e234e0363ff..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_howdy-8b-linear.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Howdy-8B-LINEAR", - "id": "DreadPoor/Howdy-8B-LINEAR", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_Howdy-8B-LINEAR/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7378 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5384 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1775 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3146 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4121 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3807 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_ichor-8b-model_stock.json b/data/models/dreadpoor_ichor-8b-model_stock.json deleted file mode 100644 index 0393ec68fea3809fcac0405fcd1ff76dc2c3220b..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_ichor-8b-model_stock.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ichor-8B-Model_Stock", - "id": "DreadPoor/ichor-8B-Model_Stock", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_ichor-8B-Model_Stock/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5386 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5084 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1088 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3238 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4212 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3151 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_ichor_1.1-8b-model_stock.json b/data/models/dreadpoor_ichor_1.1-8b-model_stock.json deleted file mode 100644 index 4e36ce792ef97c9cfd479fc585b5ecad5a174193..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_ichor_1.1-8b-model_stock.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ichor_1.1-8B-Model_Stock", - "id": "DreadPoor/ichor_1.1-8B-Model_Stock", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_ichor_1.1-8B-Model_Stock/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8096 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5281 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1775 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3062 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4068 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3856 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_incidental-8b-model_stock.json b/data/models/dreadpoor_incidental-8b-model_stock.json deleted file mode 100644 index 399ba433018f287de6a79ca663a0174238c1a656..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_incidental-8b-model_stock.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Incidental-8B-Model_Stock", - "id": "DreadPoor/Incidental-8B-Model_Stock", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_Incidental-8B-Model_Stock/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7482 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5452 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1616 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3029 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.424 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3873 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_inexpertus-8b-model_stock.json b/data/models/dreadpoor_inexpertus-8b-model_stock.json deleted file mode 100644 index f84c33000f513292c9203fd7098420274c2d8068..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_inexpertus-8b-model_stock.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "inexpertus-8B-Model_Stock", - "id": "DreadPoor/inexpertus-8B-Model_Stock", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_inexpertus-8B-Model_Stock/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7795 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.528 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1707 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3096 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4118 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3791 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_inexpertus_1.1-8b-linear.json b/data/models/dreadpoor_inexpertus_1.1-8b-linear.json deleted file mode 100644 index 98e9d12cd2423bc1859c7df13e3e6bfff129a550..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_inexpertus_1.1-8b-linear.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "inexpertus_1.1-8B-LINEAR", - "id": "DreadPoor/inexpertus_1.1-8B-LINEAR", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_inexpertus_1.1-8B-LINEAR/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7527 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5525 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.173 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2978 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4173 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3827 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_inexpertus_1.2-8b-linear.json b/data/models/dreadpoor_inexpertus_1.2-8b-linear.json deleted file mode 100644 index 4ee8e96a13498f125136ebae555b0c1767176643..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_inexpertus_1.2-8b-linear.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "inexpertus_1.2-8B-LINEAR", - "id": "DreadPoor/inexpertus_1.2-8B-LINEAR", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_inexpertus_1.2-8B-LINEAR/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7348 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5523 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1586 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2953 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4133 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3788 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_irina-8b-model_stock.json b/data/models/dreadpoor_irina-8b-model_stock.json deleted file mode 100644 index a7aee1b5cc2b0554c16ec5bf6087dfcdf98e0095..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_irina-8b-model_stock.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Irina-8B-model_stock", - "id": "DreadPoor/Irina-8B-model_stock", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_Irina-8B-model_stock/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6799 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5237 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.102 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2844 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4003 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3574 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_kindling-8b-model_stock.json b/data/models/dreadpoor_kindling-8b-model_stock.json deleted file mode 100644 index a50ef9518c28cfe9461ef46bb96bc28ec7a8d9d1..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_kindling-8b-model_stock.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Kindling-8B-Model_Stock", - "id": "DreadPoor/Kindling-8B-Model_Stock", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_Kindling-8B-Model_Stock/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7308 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5492 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1752 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.318 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4068 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.383 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_l3.1-baezel-8b-della.json b/data/models/dreadpoor_l3.1-baezel-8b-della.json deleted file mode 100644 index 88673c27d7d91eacf16d7e12d6e8ae0a9b8aa02a..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_l3.1-baezel-8b-della.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "L3.1-BaeZel-8B-Della", - "id": "DreadPoor/L3.1-BaeZel-8B-Della", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_L3.1-BaeZel-8B-Della/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.518 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5448 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1745 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3196 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.42 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3902 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_laughing_stock-8b-model_stock.json b/data/models/dreadpoor_laughing_stock-8b-model_stock.json deleted file mode 100644 index b8cae70794499188345785b2204a027f0fdfdde7..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_laughing_stock-8b-model_stock.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Laughing_Stock-8B-Model_Stock", - "id": "DreadPoor/Laughing_Stock-8B-Model_Stock", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_Laughing_Stock-8B-Model_Stock/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.719 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5449 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1579 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2894 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4146 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3764 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_lava_lamp-8b-slerp.json b/data/models/dreadpoor_lava_lamp-8b-slerp.json deleted file mode 100644 index f5eaf19b89bbd62c08a2650d5344fe8427c301c2..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_lava_lamp-8b-slerp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Lava_Lamp-8B-SLERP", - "id": "DreadPoor/Lava_Lamp-8B-SLERP", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_Lava_Lamp-8B-SLERP/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7381 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5368 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1737 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3054 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4187 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.375 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_lemonp-8b-model_stock.json b/data/models/dreadpoor_lemonp-8b-model_stock.json deleted file mode 100644 index d258664021fe57d5366e295a426d1a5d8e3e0a92..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_lemonp-8b-model_stock.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "LemonP-8B-Model_Stock", - "id": "DreadPoor/LemonP-8B-Model_Stock", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_LemonP-8B-Model_Stock/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7676 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5439 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1767 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3029 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4081 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4004 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_lydia_of_whiterun-8b-linear.json b/data/models/dreadpoor_lydia_of_whiterun-8b-linear.json deleted file mode 100644 index 09e3fb666ccafe812f83073d36e64c07286abdaf..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_lydia_of_whiterun-8b-linear.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Lydia_of_Whiterun-8B-LINEAR", - "id": "DreadPoor/Lydia_of_Whiterun-8B-LINEAR", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_Lydia_of_Whiterun-8B-LINEAR/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7603 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.538 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1767 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3163 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4251 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3801 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_matryoshka-8b-linear.json b/data/models/dreadpoor_matryoshka-8b-linear.json deleted file mode 100644 index 23d03442867d7375a93bf1e33919eebd34e5fa33..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_matryoshka-8b-linear.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Matryoshka-8B-LINEAR", - "id": "DreadPoor/Matryoshka-8B-LINEAR", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_Matryoshka-8B-LINEAR/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7263 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5444 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1752 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3205 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4252 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3866 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_mercury_in_retrograde-8b-model-stock.json b/data/models/dreadpoor_mercury_in_retrograde-8b-model-stock.json deleted file mode 100644 index 234169fbd8b390981668fbcbb62a22c7f070cc47..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_mercury_in_retrograde-8b-model-stock.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mercury_In_Retrograde-8b-Model-Stock", - "id": "DreadPoor/Mercury_In_Retrograde-8b-Model-Stock", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_Mercury_In_Retrograde-8b-Model-Stock/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7296 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5391 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1647 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3163 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4199 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3829 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_mergekit-nuslerp-nqzkedi.json b/data/models/dreadpoor_mergekit-nuslerp-nqzkedi.json deleted file mode 100644 index 88478ff8fdec9ee7b23ff64678849c6bf5d47694..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_mergekit-nuslerp-nqzkedi.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "mergekit-nuslerp-nqzkedi", - "id": "DreadPoor/mergekit-nuslerp-nqzkedi", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_mergekit-nuslerp-nqzkedi/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7765 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5362 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1881 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3012 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4225 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3919 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_minthy-8b-model_stock.json b/data/models/dreadpoor_minthy-8b-model_stock.json deleted file mode 100644 index 9c695a24471448de7c73551f91344c167fc3f1a9..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_minthy-8b-model_stock.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Minthy-8B-Model_Stock", - "id": "DreadPoor/Minthy-8B-Model_Stock", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_Minthy-8B-Model_Stock/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7658 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5353 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1918 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3037 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4094 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3993 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_minthy_alt-8b-model_stock.json b/data/models/dreadpoor_minthy_alt-8b-model_stock.json deleted file mode 100644 index 1c25eaaf4de170ad861533f492eda94db67497d5..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_minthy_alt-8b-model_stock.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Minthy_ALT-8B-Model_Stock", - "id": "DreadPoor/Minthy_ALT-8B-Model_Stock", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_Minthy_ALT-8B-Model_Stock/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6992 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5375 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.176 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3062 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4225 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3674 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_minthy_v2-8b-model_stock.json b/data/models/dreadpoor_minthy_v2-8b-model_stock.json deleted file mode 100644 index 93785f0139cfad67109a69d01ca67c9e355f009c..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_minthy_v2-8b-model_stock.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Minthy_V2-8B-Model_Stock", - "id": "DreadPoor/Minthy_V2-8B-Model_Stock", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_Minthy_V2-8B-Model_Stock/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7126 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5491 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1594 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2945 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4199 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3737 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_minus_penus-8b-model_stock.json b/data/models/dreadpoor_minus_penus-8b-model_stock.json deleted file mode 100644 index 8b0e2d5896e3a388504598e1363bf53cf9a2fb71..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_minus_penus-8b-model_stock.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Minus_Penus-8B-Model_Stock", - "id": "DreadPoor/Minus_Penus-8B-Model_Stock", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_Minus_Penus-8B-Model_Stock/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7311 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5344 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2002 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3096 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4019 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3752 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_morphing-8b-model_stock.json b/data/models/dreadpoor_morphing-8b-model_stock.json deleted file mode 100644 index 0327b134bcf46196ae0429c2bb6db4bc4d664609..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_morphing-8b-model_stock.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Morphing-8B-Model_Stock", - "id": "DreadPoor/Morphing-8B-Model_Stock", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_Morphing-8B-Model_Stock/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7445 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5397 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1888 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2936 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4069 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3852 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_not_even_my_final_form-8b-model_stock.json b/data/models/dreadpoor_not_even_my_final_form-8b-model_stock.json deleted file mode 100644 index 275d94619197056b10ba288d9e236d2b0ea23136..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_not_even_my_final_form-8b-model_stock.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Not_Even_My_Final_Form-8B-Model_Stock", - "id": "DreadPoor/Not_Even_My_Final_Form-8B-Model_Stock", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_Not_Even_My_Final_Form-8B-Model_Stock/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7722 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5351 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.176 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2953 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4147 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.384 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_nother_one-8b-model_stock.json b/data/models/dreadpoor_nother_one-8b-model_stock.json deleted file mode 100644 index 2c89b284a79c7dc059d607f96d28bf114bee1253..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_nother_one-8b-model_stock.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Nother_One-8B-Model_Stock", - "id": "DreadPoor/Nother_One-8B-Model_Stock", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_Nother_One-8B-Model_Stock/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6863 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5205 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1518 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2894 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.387 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3595 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_noxis-8b-linear.json b/data/models/dreadpoor_noxis-8b-linear.json deleted file mode 100644 index dedb57ba1ae17c92d3b5b3bc791b914ffb3ee600..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_noxis-8b-linear.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Noxis-8B-LINEAR", - "id": "DreadPoor/Noxis-8B-LINEAR", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_Noxis-8B-LINEAR/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6913 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5421 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1979 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3188 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4231 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.366 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_nullsworn-12b-linear.json b/data/models/dreadpoor_nullsworn-12b-linear.json deleted file mode 100644 index 025daff7dfe729ec587a04a08c66664616747f7f..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_nullsworn-12b-linear.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Nullsworn-12B-LINEAR", - "id": "DreadPoor/Nullsworn-12B-LINEAR", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_Nullsworn-12B-LINEAR/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4436 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5483 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1125 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3079 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.435 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3645 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_nwah-8b-model_stock.json b/data/models/dreadpoor_nwah-8b-model_stock.json deleted file mode 100644 index 9e097b479b41a5c5144d33551cfdce151d8dfb6f..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_nwah-8b-model_stock.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Nwah-8B-Model_Stock", - "id": "DreadPoor/Nwah-8B-Model_Stock", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_Nwah-8B-Model_Stock/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7716 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5384 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1798 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3104 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4039 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3807 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_oh_boy-8b-linear.json b/data/models/dreadpoor_oh_boy-8b-linear.json deleted file mode 100644 index 37ca825cd001452a11614c63435aa6e0c501f885..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_oh_boy-8b-linear.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Oh_Boy-8B-LINEAR", - "id": "DreadPoor/Oh_Boy-8B-LINEAR", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_Oh_Boy-8B-LINEAR/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7503 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5375 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1782 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3079 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4108 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3849 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_oneil-model_stock-8b.json b/data/models/dreadpoor_oneil-model_stock-8b.json deleted file mode 100644 index fab395ea127027011f97b09614b036c57754e725..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_oneil-model_stock-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ONeil-model_stock-8B", - "id": "DreadPoor/ONeil-model_stock-8B", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_ONeil-model_stock-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6786 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5548 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1012 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3054 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4173 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3599 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_orangej-8b-model_stock.json b/data/models/dreadpoor_orangej-8b-model_stock.json deleted file mode 100644 index a3125b121b25645e670586692b0cd6efbdd1e7b5..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_orangej-8b-model_stock.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "OrangeJ-8B-Model_Stock", - "id": "DreadPoor/OrangeJ-8B-Model_Stock", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_OrangeJ-8B-Model_Stock/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7841 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5413 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.176 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3012 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4028 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3969 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_promissum_mane-8b-linear-lorablated.json b/data/models/dreadpoor_promissum_mane-8b-linear-lorablated.json deleted file mode 100644 index 27f4faa65b6811bf5bb552e13a71a637dcaae4bd..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_promissum_mane-8b-linear-lorablated.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Promissum_Mane-8B-LINEAR-lorablated", - "id": "DreadPoor/Promissum_Mane-8B-LINEAR-lorablated", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_Promissum_Mane-8B-LINEAR-lorablated/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7156 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5435 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1533 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3037 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4198 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3739 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_promissum_mane-8b-linear.json b/data/models/dreadpoor_promissum_mane-8b-linear.json deleted file mode 100644 index 39f85f51171e57eaa437cc49652a5c2d130f9929..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_promissum_mane-8b-linear.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Promissum_Mane-8B-LINEAR", - "id": "DreadPoor/Promissum_Mane-8B-LINEAR", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_Promissum_Mane-8B-LINEAR/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.715 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5458 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1556 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3045 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.42 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3851 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_remember_to_breathe-8b-model-stock.json b/data/models/dreadpoor_remember_to_breathe-8b-model-stock.json deleted file mode 100644 index 3ac414b270284f0ba8a85f01c2055608c36e874a..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_remember_to_breathe-8b-model-stock.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "remember_to_breathe-8b-Model-Stock", - "id": "DreadPoor/remember_to_breathe-8b-Model-Stock", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_remember_to_breathe-8b-Model-Stock/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7104 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5412 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1488 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3012 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4145 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3761 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_rpmash-8b-model_stock.json b/data/models/dreadpoor_rpmash-8b-model_stock.json deleted file mode 100644 index bf1a6ce9ad40a82e48ae13eab9ea8f214b3b96e9..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_rpmash-8b-model_stock.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "RPMash-8B-Model_Stock", - "id": "DreadPoor/RPMash-8B-Model_Stock", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_RPMash-8B-Model_Stock/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4564 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5169 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.108 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2869 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4054 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3604 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_rpmash_v3-8b-model_stock.json b/data/models/dreadpoor_rpmash_v3-8b-model_stock.json deleted file mode 100644 index 660593769373fb957853f5d62290fcbdf9d4a32d..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_rpmash_v3-8b-model_stock.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "RPMash_V3-8B-Model_Stock", - "id": "DreadPoor/RPMash_V3-8B-Model_Stock", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_RPMash_V3-8B-Model_Stock/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7049 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5217 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1042 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3003 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3778 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3614 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_rusted_gold-8b-linear.json b/data/models/dreadpoor_rusted_gold-8b-linear.json deleted file mode 100644 index d25cfab1ba69758603ece87aec731ab3725e98e6..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_rusted_gold-8b-linear.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Rusted_Gold-8B-LINEAR", - "id": "DreadPoor/Rusted_Gold-8B-LINEAR", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_Rusted_Gold-8B-LINEAR/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7296 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5387 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1934 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2987 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4178 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.378 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_rusted_platinum-8b-linear.json b/data/models/dreadpoor_rusted_platinum-8b-linear.json deleted file mode 100644 index 32002c911b2023996f5c7e52d07a70bffbb414af..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_rusted_platinum-8b-linear.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Rusted_Platinum-8B-LINEAR", - "id": "DreadPoor/Rusted_Platinum-8B-LINEAR", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_Rusted_Platinum-8B-LINEAR/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.718 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5428 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1722 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2802 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3967 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.373 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_rusted_platinum-8b-model_stock.json b/data/models/dreadpoor_rusted_platinum-8b-model_stock.json deleted file mode 100644 index 9333988eee4382459f1a43326c69c6d11a06c7e0..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_rusted_platinum-8b-model_stock.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Rusted_Platinum-8B-Model_Stock", - "id": "DreadPoor/Rusted_Platinum-8B-Model_Stock", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_Rusted_Platinum-8B-Model_Stock/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4408 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5243 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.102 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2685 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3741 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3546 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_sellen-8b-model_stock.json b/data/models/dreadpoor_sellen-8b-model_stock.json deleted file mode 100644 index cf321b04dc51710b11f240b5dfd7d3bab9932499..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_sellen-8b-model_stock.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Sellen-8B-model_stock", - "id": "DreadPoor/Sellen-8B-model_stock", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_Sellen-8B-model_stock/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7113 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5232 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1337 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2743 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.396 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.357 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_something-8b-model_stock.json b/data/models/dreadpoor_something-8b-model_stock.json deleted file mode 100644 index 0d5c388c3f222eed1d0854310d03d112758eb9f8..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_something-8b-model_stock.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Something-8B-Model_Stock", - "id": "DreadPoor/Something-8B-Model_Stock", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_Something-8B-Model_Stock/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5043 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5395 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1798 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3171 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4187 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3885 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_spring_dusk-8b-sce.json b/data/models/dreadpoor_spring_dusk-8b-sce.json deleted file mode 100644 index 664dd7d7cd35b4db9bc5144da073f65b331fa7fb..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_spring_dusk-8b-sce.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Spring_Dusk-8B-SCE", - "id": "DreadPoor/Spring_Dusk-8B-SCE", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_Spring_Dusk-8B-SCE/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6515 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5635 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0763 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2878 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.46 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3436 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_summer_dawn-8b-sce.json b/data/models/dreadpoor_summer_dawn-8b-sce.json deleted file mode 100644 index d6790e7f97e0662f40659b2eedd850e563f8174c..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_summer_dawn-8b-sce.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Summer_Dawn-8B-SCE", - "id": "DreadPoor/Summer_Dawn-8B-SCE", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_Summer_Dawn-8B-SCE/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6642 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5391 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1722 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3003 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.412 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3753 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_summer_dusk-8b-ties.json b/data/models/dreadpoor_summer_dusk-8b-ties.json deleted file mode 100644 index 7376ad1d004e32d59d05aea8599a9743ce574d0a..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_summer_dusk-8b-ties.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Summer_Dusk-8B-TIES", - "id": "DreadPoor/Summer_Dusk-8B-TIES", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_Summer_Dusk-8B-TIES/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4922 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.536 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1805 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.307 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4267 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3856 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_summer_rain-8b-sce.json b/data/models/dreadpoor_summer_rain-8b-sce.json deleted file mode 100644 index 2ef233678bd1afdbfef76efbc48cd7e1960f5da6..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_summer_rain-8b-sce.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Summer_Rain-8B-SCE", - "id": "DreadPoor/Summer_Rain-8B-SCE", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_Summer_Rain-8B-SCE/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5459 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5846 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0702 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3037 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4477 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3551 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_summer_rain-8b-ties.json b/data/models/dreadpoor_summer_rain-8b-ties.json deleted file mode 100644 index 353d981d64f15bdf5eaf57fdd8a1728d33fe804c..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_summer_rain-8b-ties.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Summer_Rain-8B-TIES", - "id": "DreadPoor/Summer_Rain-8B-TIES", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_Summer_Rain-8B-TIES/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5444 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5846 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0702 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3037 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4477 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3551 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_sun-8b-model_stock.json b/data/models/dreadpoor_sun-8b-model_stock.json deleted file mode 100644 index f9e089078979f01ee6bea3592d18e891502dc072..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_sun-8b-model_stock.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Sun-8B-Model_Stock", - "id": "DreadPoor/Sun-8B-Model_Stock", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_Sun-8B-Model_Stock/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7758 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5264 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.21 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2995 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4098 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3835 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_sweetened_condensed_milk-8b-model_stock.json b/data/models/dreadpoor_sweetened_condensed_milk-8b-model_stock.json deleted file mode 100644 index d3ab6fd5f19756f1f0dca281e4a6c5452f5b2428..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_sweetened_condensed_milk-8b-model_stock.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Sweetened_Condensed_Milk-8B-Model_Stock", - "id": "DreadPoor/Sweetened_Condensed_Milk-8B-Model_Stock", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_Sweetened_Condensed_Milk-8B-Model_Stock/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7417 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5406 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1873 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3029 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4107 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3848 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_test.json b/data/models/dreadpoor_test.json deleted file mode 100644 index b5c5eb1b278441ef2341e10055bb92cd73435d69..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_test.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "test", - "id": "DreadPoor/test", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_test/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4937 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5372 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1934 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.271 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4351 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3647 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_test02-ignore.json b/data/models/dreadpoor_test02-ignore.json deleted file mode 100644 index 1735f9108c15394177671a1984bd0f9c371bbe3a..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_test02-ignore.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "TEST02-Ignore", - "id": "DreadPoor/TEST02-Ignore", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_TEST02-Ignore/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6119 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5602 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0869 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2844 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4199 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3468 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_test03-ignore.json b/data/models/dreadpoor_test03-ignore.json deleted file mode 100644 index 9a4880ebcd15c4ee22a97f39af47d715d6c3a827..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_test03-ignore.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "TEST03-ignore", - "id": "DreadPoor/TEST03-ignore", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_TEST03-ignore/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6967 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5383 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1654 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3087 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4186 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3789 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_test06-ignore.json b/data/models/dreadpoor_test06-ignore.json deleted file mode 100644 index d8abd50ab3750ba3c8b180b279dc236da1f959e7..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_test06-ignore.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "TEST06-ignore", - "id": "DreadPoor/TEST06-ignore", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_TEST06-ignore/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7323 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5509 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1178 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2869 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4225 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3615 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_test07-ignore.json b/data/models/dreadpoor_test07-ignore.json deleted file mode 100644 index 8fa982635f56a39f454207d8ebd34f28b351e1c8..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_test07-ignore.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "TEST07-ignore", - "id": "DreadPoor/TEST07-ignore", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_TEST07-ignore/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.74 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5561 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1662 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3087 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4094 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.388 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_test08-ignore.json b/data/models/dreadpoor_test08-ignore.json deleted file mode 100644 index 88d87b6ff0b4f977f9d7eb29f3234570e19f9fa3..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_test08-ignore.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "TEST08-ignore", - "id": "DreadPoor/TEST08-ignore", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_TEST08-ignore/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7467 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5454 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.182 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3129 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4081 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3853 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_test_alt.json b/data/models/dreadpoor_test_alt.json deleted file mode 100644 index a4b2eb8d23eb75e77f32e3b178622a8e8c398aa2..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_test_alt.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "test_ALT", - "id": "DreadPoor/test_ALT", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_test_ALT/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4997 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.537 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1707 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2693 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4363 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3492 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_tests_pending-do_not_use_yet.json b/data/models/dreadpoor_tests_pending-do_not_use_yet.json deleted file mode 100644 index 9203cb0b1ba32e5c5821bed5477d2e871937f262..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_tests_pending-do_not_use_yet.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "tests_pending-do_not_use_yet", - "id": "DreadPoor/tests_pending-do_not_use_yet", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_tests_pending-do_not_use_yet/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7691 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5408 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1979 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.297 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4005 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3827 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_trinas_nectar-8b-model_stock.json b/data/models/dreadpoor_trinas_nectar-8b-model_stock.json deleted file mode 100644 index 0e484034a1582ce248e1367d0f3f9eba9864a067..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_trinas_nectar-8b-model_stock.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Trinas_Nectar-8B-model_stock", - "id": "DreadPoor/Trinas_Nectar-8B-model_stock", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_Trinas_Nectar-8B-model_stock/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7259 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5256 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1526 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2861 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4068 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3618 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_untested-venn_1.2-8b-model_stock.json b/data/models/dreadpoor_untested-venn_1.2-8b-model_stock.json deleted file mode 100644 index 6ff0a48c3d21c9084660d8674e282294ee299b93..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_untested-venn_1.2-8b-model_stock.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "UNTESTED-VENN_1.2-8B-Model_Stock", - "id": "DreadPoor/UNTESTED-VENN_1.2-8B-Model_Stock", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "4.015" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_UNTESTED-VENN_1.2-8B-Model_Stock/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4718 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5475 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1541 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3154 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4449 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3787 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_venn_1.2-8b-model_stock.json b/data/models/dreadpoor_venn_1.2-8b-model_stock.json deleted file mode 100644 index 0ae3a84b63b9fef10e49be09a8626b5bfd2efe0a..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_venn_1.2-8b-model_stock.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "VENN_1.2-8B-Model_Stock", - "id": "DreadPoor/VENN_1.2-8B-Model_Stock", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "4.015" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_VENN_1.2-8B-Model_Stock/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7226 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5459 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1707 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.297 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.42 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3721 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_wannabe-8b-model_stock.json b/data/models/dreadpoor_wannabe-8b-model_stock.json deleted file mode 100644 index d779c5b17c1bdeb62e2da907342c7ab0bb853130..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_wannabe-8b-model_stock.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Wannabe-8B-Model_Stock", - "id": "DreadPoor/Wannabe-8B-Model_Stock", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_Wannabe-8B-Model_Stock/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7205 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.539 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1775 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3012 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4135 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3831 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_what_a_thrill-8b-model_stock.json b/data/models/dreadpoor_what_a_thrill-8b-model_stock.json deleted file mode 100644 index 4d83bd9e500ff1af10e59b3b8f203d29f25aebf2..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_what_a_thrill-8b-model_stock.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "What_A_Thrill-8B-Model_Stock", - "id": "DreadPoor/What_A_Thrill-8B-Model_Stock", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_What_A_Thrill-8B-Model_Stock/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7064 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5311 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.182 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.297 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.408 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3615 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_winter-8b-sce.json b/data/models/dreadpoor_winter-8b-sce.json deleted file mode 100644 index 04e7e240ad93967f5dd7fbb1491383753c858cc1..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_winter-8b-sce.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Winter-8B-SCE", - "id": "DreadPoor/Winter-8B-SCE", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_Winter-8B-SCE/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7536 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5262 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1918 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2995 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4071 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3839 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_winter_dawn-8b-ties.json b/data/models/dreadpoor_winter_dawn-8b-ties.json deleted file mode 100644 index 0573f4e99b4464cb886cc1737b475a950c6e9658..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_winter_dawn-8b-ties.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Winter_Dawn-8B-TIES", - "id": "DreadPoor/Winter_Dawn-8B-TIES", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_Winter_Dawn-8B-TIES/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5496 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5309 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1858 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3096 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4279 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.391 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_winter_dusk-8b-ties.json b/data/models/dreadpoor_winter_dusk-8b-ties.json deleted file mode 100644 index 3f636a95f86c3c4728cc5c6fc31852ed5d294b3a..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_winter_dusk-8b-ties.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Winter_Dusk-8B-TIES", - "id": "DreadPoor/Winter_Dusk-8B-TIES", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_Winter_Dusk-8B-TIES/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7153 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4952 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0718 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2995 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3688 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3478 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_winter_night-8b-model_stock.json b/data/models/dreadpoor_winter_night-8b-model_stock.json deleted file mode 100644 index ea91b45837fe73db8e8d49286e4eadf9f9caab49..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_winter_night-8b-model_stock.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Winter_Night-8B-Model_Stock", - "id": "DreadPoor/Winter_Night-8B-Model_Stock", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_Winter_Night-8B-Model_Stock/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.704 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5185 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1458 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3062 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3914 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3666 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_wip-acacia-8b-model_stock.json b/data/models/dreadpoor_wip-acacia-8b-model_stock.json deleted file mode 100644 index d679ff6346d8bd64516ea9981f4bcb513cb2b84a..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_wip-acacia-8b-model_stock.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "WIP-Acacia-8B-Model_Stock", - "id": "DreadPoor/WIP-Acacia-8B-Model_Stock", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_WIP-Acacia-8B-Model_Stock/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6246 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5195 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1669 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3062 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4226 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3737 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_wip_damascus-8b-ties.json b/data/models/dreadpoor_wip_damascus-8b-ties.json deleted file mode 100644 index 6a7f1d1cd505290bbdfbef154b345af3e7ae57cf..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_wip_damascus-8b-ties.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "WIP_Damascus-8B-TIES", - "id": "DreadPoor/WIP_Damascus-8B-TIES", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_WIP_Damascus-8B-TIES/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4776 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5411 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1654 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.307 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4119 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3761 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_yafune-8b-model_stock.json b/data/models/dreadpoor_yafune-8b-model_stock.json deleted file mode 100644 index 92c49add5249dedd9fa0f497f533a27a0a9501d2..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_yafune-8b-model_stock.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Yafune-8B-Model_Stock", - "id": "DreadPoor/Yafune-8B-Model_Stock", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_Yafune-8B-Model_Stock/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7533 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5467 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1662 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3272 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4173 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3851 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_yearn_v3-8b-model_stock.json b/data/models/dreadpoor_yearn_v3-8b-model_stock.json deleted file mode 100644 index b42513502b26607798b2f26f069856e5b25f2f61..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_yearn_v3-8b-model_stock.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Yearn_V3-8B-Model_Stock", - "id": "DreadPoor/Yearn_V3-8B-Model_Stock", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_Yearn_V3-8B-Model_Stock/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.729 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5322 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1896 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3054 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3909 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3802 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_zelus-8b-model_stock.json b/data/models/dreadpoor_zelus-8b-model_stock.json deleted file mode 100644 index 7cab3ce62621b68072368bd8ba3b14cf4f97223c..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_zelus-8b-model_stock.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Zelus-8B-Model_Stock", - "id": "DreadPoor/Zelus-8B-Model_Stock", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_Zelus-8B-Model_Stock/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7788 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5307 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1647 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3062 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4214 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3841 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_zelus_v2-8b-model_stock.json b/data/models/dreadpoor_zelus_v2-8b-model_stock.json deleted file mode 100644 index da0efee8352fc9b69f7b05c659dd6064b2c7c9f8..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_zelus_v2-8b-model_stock.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Zelus_V2-8B-Model_Stock", - "id": "DreadPoor/Zelus_V2-8B-Model_Stock", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_Zelus_V2-8B-Model_Stock/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7898 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5345 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2054 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3096 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3961 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3833 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreadpoor_zeus-8b-v17-abliterated_alt.json b/data/models/dreadpoor_zeus-8b-v17-abliterated_alt.json deleted file mode 100644 index 1d363575df6ac6f631b6e1b48c83d21b8964ce30..0000000000000000000000000000000000000000 --- a/data/models/dreadpoor_zeus-8b-v17-abliterated_alt.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ZEUS-8B-V17-Abliterated_ALT", - "id": "DreadPoor/ZEUS-8B-V17-Abliterated_ALT", - "developer": "DreadPoor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DreadPoor_ZEUS-8B-V17-Abliterated_ALT/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5511 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5231 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1903 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3079 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4149 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.389 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dreamgen_wizardlm-2-7b.json b/data/models/dreamgen_wizardlm-2-7b.json deleted file mode 100644 index bc6dac23c24a50f231a45dc0e113eba468d9507d..0000000000000000000000000000000000000000 --- a/data/models/dreamgen_wizardlm-2-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "WizardLM-2-7B", - "id": "dreamgen/WizardLM-2-7B", - "developer": "dreamgen", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/dreamgen_WizardLM-2-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4583 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3487 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0332 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2869 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3941 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.266 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/drxd1000_atlas-7b.json b/data/models/drxd1000_atlas-7b.json deleted file mode 100644 index 877b8d99596086e9a13786b730d8c5f71b42a6ff..0000000000000000000000000000000000000000 --- a/data/models/drxd1000_atlas-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Atlas-7B", - "id": "DRXD1000/Atlas-7B", - "developer": "DRXD1000", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "7.768" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DRXD1000_Atlas-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3704 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3302 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0189 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2576 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3342 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1401 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/drxd1000_phoenix-7b.json b/data/models/drxd1000_phoenix-7b.json deleted file mode 100644 index ad43342a3092144c3c53d647ea1856cdc5cf02fe..0000000000000000000000000000000000000000 --- a/data/models/drxd1000_phoenix-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Phoenix-7B", - "id": "DRXD1000/Phoenix-7B", - "developer": "DRXD1000", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DRXD1000_Phoenix-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.321 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3932 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0166 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2785 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3849 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2343 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dual-gpo_zephyr-7b-ipo-0k-15k-i1.json b/data/models/dual-gpo_zephyr-7b-ipo-0k-15k-i1.json deleted file mode 100644 index 907c0ddacdf7030e65931b47f6ab3c6ff07369c7..0000000000000000000000000000000000000000 --- a/data/models/dual-gpo_zephyr-7b-ipo-0k-15k-i1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "zephyr-7b-ipo-0k-15k-i1", - "id": "DUAL-GPO/zephyr-7b-ipo-0k-15k-i1", - "developer": "DUAL-GPO", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "?", - "params_billions": "14.483" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DUAL-GPO_zephyr-7b-ipo-0k-15k-i1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2756 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4473 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0302 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2911 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4173 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.313 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dustinwloring1988_reflexis-8b-chat-v1.json b/data/models/dustinwloring1988_reflexis-8b-chat-v1.json deleted file mode 100644 index 1bff3007e80646139a0109e504fab5040d7b54d0..0000000000000000000000000000000000000000 --- a/data/models/dustinwloring1988_reflexis-8b-chat-v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Reflexis-8b-chat-v1", - "id": "dustinwloring1988/Reflexis-8b-chat-v1", - "developer": "dustinwloring1988", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/dustinwloring1988_Reflexis-8b-chat-v1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3658 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4664 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1156 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2542 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3754 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3384 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dustinwloring1988_reflexis-8b-chat-v2.json b/data/models/dustinwloring1988_reflexis-8b-chat-v2.json deleted file mode 100644 index f1e007481b702c727524c2b0d05b6193ae7ed278..0000000000000000000000000000000000000000 --- a/data/models/dustinwloring1988_reflexis-8b-chat-v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Reflexis-8b-chat-v2", - "id": "dustinwloring1988/Reflexis-8b-chat-v2", - "developer": "dustinwloring1988", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/dustinwloring1988_Reflexis-8b-chat-v2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3912 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4724 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1163 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2701 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3526 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3378 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dustinwloring1988_reflexis-8b-chat-v3.json b/data/models/dustinwloring1988_reflexis-8b-chat-v3.json deleted file mode 100644 index 253787266c6967e73f225f0b6e479d4f9a037e27..0000000000000000000000000000000000000000 --- a/data/models/dustinwloring1988_reflexis-8b-chat-v3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Reflexis-8b-chat-v3", - "id": "dustinwloring1988/Reflexis-8b-chat-v3", - "developer": "dustinwloring1988", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/dustinwloring1988_Reflexis-8b-chat-v3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5367 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4658 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1224 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2424 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3512 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3548 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dustinwloring1988_reflexis-8b-chat-v4.json b/data/models/dustinwloring1988_reflexis-8b-chat-v4.json deleted file mode 100644 index f00b7765cc4ebe076784cb2bb13d4208786ca50b..0000000000000000000000000000000000000000 --- a/data/models/dustinwloring1988_reflexis-8b-chat-v4.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Reflexis-8b-chat-v4", - "id": "dustinwloring1988/Reflexis-8b-chat-v4", - "developer": "dustinwloring1988", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/dustinwloring1988_Reflexis-8b-chat-v4/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4698 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4686 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1027 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2341 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3393 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.339 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dustinwloring1988_reflexis-8b-chat-v5.json b/data/models/dustinwloring1988_reflexis-8b-chat-v5.json deleted file mode 100644 index c050f62d58a81a0bf95d329cce2d1708ebb962e0..0000000000000000000000000000000000000000 --- a/data/models/dustinwloring1988_reflexis-8b-chat-v5.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Reflexis-8b-chat-v5", - "id": "dustinwloring1988/Reflexis-8b-chat-v5", - "developer": "dustinwloring1988", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/dustinwloring1988_Reflexis-8b-chat-v5/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4238 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4782 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1216 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.271 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3354 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3217 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dustinwloring1988_reflexis-8b-chat-v6.json b/data/models/dustinwloring1988_reflexis-8b-chat-v6.json deleted file mode 100644 index e8a23828762c0abc29baeb7385372a6b0f5d98cd..0000000000000000000000000000000000000000 --- a/data/models/dustinwloring1988_reflexis-8b-chat-v6.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Reflexis-8b-chat-v6", - "id": "dustinwloring1988/Reflexis-8b-chat-v6", - "developer": "dustinwloring1988", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/dustinwloring1988_Reflexis-8b-chat-v6/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4939 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.481 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1299 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2626 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3753 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3479 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dustinwloring1988_reflexis-8b-chat-v7.json b/data/models/dustinwloring1988_reflexis-8b-chat-v7.json deleted file mode 100644 index 029692c84720c5655b45f073096eaf6f2d39cac1..0000000000000000000000000000000000000000 --- a/data/models/dustinwloring1988_reflexis-8b-chat-v7.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Reflexis-8b-chat-v7", - "id": "dustinwloring1988/Reflexis-8b-chat-v7", - "developer": "dustinwloring1988", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/dustinwloring1988_Reflexis-8b-chat-v7/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.398 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.481 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1631 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2617 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3222 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3643 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/duyhv1411_llama-3.2-1b-en-vi.json b/data/models/duyhv1411_llama-3.2-1b-en-vi.json deleted file mode 100644 index 9ea2e851c6b33ac75c94d70b40403214508c41c1..0000000000000000000000000000000000000000 --- a/data/models/duyhv1411_llama-3.2-1b-en-vi.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.2-1B-en-vi", - "id": "duyhv1411/Llama-3.2-1B-en-vi", - "developer": "duyhv1411", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.236" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/duyhv1411_Llama-3.2-1B-en-vi/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4788 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3291 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0287 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2768 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3197 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1341 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/duyhv1411_llama-3.2-3b-en-vi.json b/data/models/duyhv1411_llama-3.2-3b-en-vi.json deleted file mode 100644 index 2e4147dcb7784689664c717f292ecab4da11f863..0000000000000000000000000000000000000000 --- a/data/models/duyhv1411_llama-3.2-3b-en-vi.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.2-3B-en-vi", - "id": "duyhv1411/Llama-3.2-3B-en-vi", - "developer": "duyhv1411", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.236" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/duyhv1411_Llama-3.2-3B-en-vi/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4852 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3272 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0227 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2752 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.321 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1359 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dwikitheduck_gemma-2-2b-id-inst.json b/data/models/dwikitheduck_gemma-2-2b-id-inst.json deleted file mode 100644 index 22f0d19445daedb6264a8f34c7598c53eaf81363..0000000000000000000000000000000000000000 --- a/data/models/dwikitheduck_gemma-2-2b-id-inst.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "gemma-2-2b-id-inst", - "id": "dwikitheduck/gemma-2-2b-id-inst", - "developer": "dwikitheduck", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "2.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/dwikitheduck_gemma-2-2b-id-inst/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3879 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3962 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0453 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2995 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4154 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2173 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dwikitheduck_gemma-2-2b-id-instruct.json b/data/models/dwikitheduck_gemma-2-2b-id-instruct.json deleted file mode 100644 index 416c95ff78b5322d50dbb1b003125817e30b5ba9..0000000000000000000000000000000000000000 --- a/data/models/dwikitheduck_gemma-2-2b-id-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "gemma-2-2b-id-instruct", - "id": "dwikitheduck/gemma-2-2b-id-instruct", - "developer": "dwikitheduck", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "2.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/dwikitheduck_gemma-2-2b-id-instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3879 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3962 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0453 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2995 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4154 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2173 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dwikitheduck_gemma-2-2b-id.json b/data/models/dwikitheduck_gemma-2-2b-id.json deleted file mode 100644 index 80618e0e748117bb81d1dd8aa644af11aabc2702..0000000000000000000000000000000000000000 --- a/data/models/dwikitheduck_gemma-2-2b-id.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "gemma-2-2b-id", - "id": "dwikitheduck/gemma-2-2b-id", - "developer": "dwikitheduck", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "2.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/dwikitheduck_gemma-2-2b-id/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3879 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3962 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0453 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2995 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4154 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2173 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dwikitheduck_gen-inst-1.json b/data/models/dwikitheduck_gen-inst-1.json deleted file mode 100644 index 0665e66e41cdadac8d3e910e7b0246193165e360..0000000000000000000000000000000000000000 --- a/data/models/dwikitheduck_gen-inst-1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "gen-inst-1", - "id": "dwikitheduck/gen-inst-1", - "developer": "dwikitheduck", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/dwikitheduck_gen-inst-1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.775 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.642 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4554 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3716 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4205 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5089 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dwikitheduck_gen-try1-notemp.json b/data/models/dwikitheduck_gen-try1-notemp.json deleted file mode 100644 index 101aa3f0a28d92b037a89be34c786bc3155d9a1a..0000000000000000000000000000000000000000 --- a/data/models/dwikitheduck_gen-try1-notemp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "gen-try1-notemp", - "id": "dwikitheduck/gen-try1-notemp", - "developer": "dwikitheduck", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/dwikitheduck_gen-try1-notemp/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2627 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6263 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.318 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.354 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4714 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.521 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dwikitheduck_gen-try1.json b/data/models/dwikitheduck_gen-try1.json deleted file mode 100644 index 6684067dbc67b367893142b8a26dfe59fd320732..0000000000000000000000000000000000000000 --- a/data/models/dwikitheduck_gen-try1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "gen-try1", - "id": "dwikitheduck/gen-try1", - "developer": "dwikitheduck", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/dwikitheduck_gen-try1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7522 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6359 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4101 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3414 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4416 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5111 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dzakwan_dzakwan-moe-4x7b-beta.json b/data/models/dzakwan_dzakwan-moe-4x7b-beta.json deleted file mode 100644 index 877419c3d1ce0586820928b1e6100121faad11d1..0000000000000000000000000000000000000000 --- a/data/models/dzakwan_dzakwan-moe-4x7b-beta.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "dzakwan-MoE-4x7b-Beta", - "id": "dzakwan/dzakwan-MoE-4x7b-Beta", - "developer": "dzakwan", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MixtralForCausalLM", - "params_billions": "24.154" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/dzakwan_dzakwan-MoE-4x7b-Beta/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4443 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.514 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0778 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2861 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4267 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3108 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/dzgas_gigabateman-7b.json b/data/models/dzgas_gigabateman-7b.json deleted file mode 100644 index 93771d2f69ac06bde8abd58f5049bef69080af9b..0000000000000000000000000000000000000000 --- a/data/models/dzgas_gigabateman-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "GIGABATEMAN-7B", - "id": "DZgas/GIGABATEMAN-7B", - "developer": "DZgas", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/DZgas_GIGABATEMAN-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4607 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5032 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0551 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2894 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4328 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3177 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ece-ilab-prymmal_ilab-merging-3b-v2.json b/data/models/ece-ilab-prymmal_ilab-merging-3b-v2.json deleted file mode 100644 index 1f4f88476e783a7acae344f4999b5262d3270fae..0000000000000000000000000000000000000000 --- a/data/models/ece-ilab-prymmal_ilab-merging-3b-v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ILAB-Merging-3B-V2", - "id": "ECE-ILAB-PRYMMAL/ILAB-Merging-3B-V2", - "developer": "ECE-ILAB-PRYMMAL", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Phi3ForCausalLM", - "params_billions": "3.821" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ECE-ILAB-PRYMMAL_ILAB-Merging-3B-V2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4029 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5402 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1518 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3054 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4332 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3861 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/edgerunners_meta-llama-3-8b-instruct-hf-ortho-baukit-34fail-3000total-bf16.json b/data/models/edgerunners_meta-llama-3-8b-instruct-hf-ortho-baukit-34fail-3000total-bf16.json deleted file mode 100644 index f99e874258e4db8e0bd86d6df58ad322326735d8..0000000000000000000000000000000000000000 --- a/data/models/edgerunners_meta-llama-3-8b-instruct-hf-ortho-baukit-34fail-3000total-bf16.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "meta-llama-3-8b-instruct-hf-ortho-baukit-34fail-3000total-bf16", - "id": "Edgerunners/meta-llama-3-8b-instruct-hf-ortho-baukit-34fail-3000total-bf16", - "developer": "Edgerunners", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Edgerunners_meta-llama-3-8b-instruct-hf-ortho-baukit-34fail-3000total-bf16/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7147 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.498 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0906 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2601 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3342 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3636 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ehristoforu_coolqwen-3b-it.json b/data/models/ehristoforu_coolqwen-3b-it.json deleted file mode 100644 index a3ec292d42c1ac309a2176c4cb733bbc7c33d356..0000000000000000000000000000000000000000 --- a/data/models/ehristoforu_coolqwen-3b-it.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "coolqwen-3b-it", - "id": "ehristoforu/coolqwen-3b-it", - "developer": "ehristoforu", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.085" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ehristoforu_coolqwen-3b-it/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6473 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4851 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3671 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2827 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4125 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3601 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ehristoforu_della-70b-test-v1.json b/data/models/ehristoforu_della-70b-test-v1.json deleted file mode 100644 index 88cf91454dc1fee30c6f3552ecf7690f451d78bc..0000000000000000000000000000000000000000 --- a/data/models/ehristoforu_della-70b-test-v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "della-70b-test-v1", - "id": "ehristoforu/della-70b-test-v1", - "developer": "ehristoforu", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "70.554" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ehristoforu_della-70b-test-v1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4979 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3029 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0098 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2525 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4555 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1575 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ehristoforu_falcon3-8b-franken-basestruct.json b/data/models/ehristoforu_falcon3-8b-franken-basestruct.json deleted file mode 100644 index b4c6563326c34b2a317d107f210d98af58b4a9dd..0000000000000000000000000000000000000000 --- a/data/models/ehristoforu_falcon3-8b-franken-basestruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Falcon3-8B-Franken-Basestruct", - "id": "ehristoforu/Falcon3-8B-Franken-Basestruct", - "developer": "ehristoforu", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.406" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ehristoforu_Falcon3-8B-Franken-Basestruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1715 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5463 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3406 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3555 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3947 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ehristoforu_falcon3-moe-2x7b-insruct.json b/data/models/ehristoforu_falcon3-moe-2x7b-insruct.json deleted file mode 100644 index 79cca52004e33d255cec16b1aa2cf58cb6668c41..0000000000000000000000000000000000000000 --- a/data/models/ehristoforu_falcon3-moe-2x7b-insruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Falcon3-MoE-2x7B-Insruct", - "id": "ehristoforu/Falcon3-MoE-2x7B-Insruct", - "developer": "ehristoforu", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MixtralForCausalLM", - "params_billions": "13.401" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ehristoforu_Falcon3-MoE-2x7B-Insruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7643 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5648 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4124 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3121 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.484 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4095 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ehristoforu_falcon3-ultraset.json b/data/models/ehristoforu_falcon3-ultraset.json deleted file mode 100644 index e2cfd60cc660595313f979e403698c163acff024..0000000000000000000000000000000000000000 --- a/data/models/ehristoforu_falcon3-ultraset.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "falcon3-ultraset", - "id": "ehristoforu/falcon3-ultraset", - "developer": "ehristoforu", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "7.456" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ehristoforu_falcon3-ultraset/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7135 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5584 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2122 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3322 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4853 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3982 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ehristoforu_fd-lora-merged-16x32.json b/data/models/ehristoforu_fd-lora-merged-16x32.json deleted file mode 100644 index 78bc1cbbe85d4c328c24e65e5e02dcea6c49cfcd..0000000000000000000000000000000000000000 --- a/data/models/ehristoforu_fd-lora-merged-16x32.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "fd-lora-merged-16x32", - "id": "ehristoforu/fd-lora-merged-16x32", - "developer": "ehristoforu", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.776" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ehristoforu_fd-lora-merged-16x32/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3481 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3308 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1707 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2534 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3514 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1205 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ehristoforu_fd-lora-merged-64x128.json b/data/models/ehristoforu_fd-lora-merged-64x128.json deleted file mode 100644 index 46816dc1a663140820bf4bf23bcff69bf82f53f1..0000000000000000000000000000000000000000 --- a/data/models/ehristoforu_fd-lora-merged-64x128.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "fd-lora-merged-64x128", - "id": "ehristoforu/fd-lora-merged-64x128", - "developer": "ehristoforu", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.777" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ehristoforu_fd-lora-merged-64x128/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3281 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3345 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1873 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.255 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3368 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1537 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ehristoforu_fp4-14b-it-v1.json b/data/models/ehristoforu_fp4-14b-it-v1.json deleted file mode 100644 index db5903e282a5f1da9e7a7c9ee14bdeb966496b17..0000000000000000000000000000000000000000 --- a/data/models/ehristoforu_fp4-14b-it-v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "fp4-14b-it-v1", - "id": "ehristoforu/fp4-14b-it-v1", - "developer": "ehristoforu", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ehristoforu_fp4-14b-it-v1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2535 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.574 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0408 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2953 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3595 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4205 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ehristoforu_fp4-14b-v1-fix.json b/data/models/ehristoforu_fp4-14b-v1-fix.json deleted file mode 100644 index 488fd2e79c1ca42afdbdb88a644fb585870cfba0..0000000000000000000000000000000000000000 --- a/data/models/ehristoforu_fp4-14b-v1-fix.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "fp4-14b-v1-fix", - "id": "ehristoforu/fp4-14b-v1-fix", - "developer": "ehristoforu", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ehristoforu_fp4-14b-v1-fix/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6742 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6817 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4207 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.354 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4532 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5353 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ehristoforu_fq2.5-7b-it-normalize_false.json b/data/models/ehristoforu_fq2.5-7b-it-normalize_false.json deleted file mode 100644 index 531b74366db3b4953b5af2e457f96de10589d427..0000000000000000000000000000000000000000 --- a/data/models/ehristoforu_fq2.5-7b-it-normalize_false.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "fq2.5-7b-it-normalize_false", - "id": "ehristoforu/fq2.5-7b-it-normalize_false", - "developer": "ehristoforu", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ehristoforu_fq2.5-7b-it-normalize_false/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7399 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.552 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4622 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.302 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4612 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4413 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ehristoforu_fq2.5-7b-it-normalize_true.json b/data/models/ehristoforu_fq2.5-7b-it-normalize_true.json deleted file mode 100644 index f000e83fb9151b974bb234080eda10a2f69a29fd..0000000000000000000000000000000000000000 --- a/data/models/ehristoforu_fq2.5-7b-it-normalize_true.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "fq2.5-7b-it-normalize_true", - "id": "ehristoforu/fq2.5-7b-it-normalize_true", - "developer": "ehristoforu", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ehristoforu_fq2.5-7b-it-normalize_true/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7399 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.552 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4622 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.302 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4612 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4413 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ehristoforu_frqwen2.5-from7b-duable4layers-it.json b/data/models/ehristoforu_frqwen2.5-from7b-duable4layers-it.json deleted file mode 100644 index e131a514e5ca9611c2bf3d634fb4fe784cd0f15c..0000000000000000000000000000000000000000 --- a/data/models/ehristoforu_frqwen2.5-from7b-duable4layers-it.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "frqwen2.5-from7b-duable4layers-it", - "id": "ehristoforu/frqwen2.5-from7b-duable4layers-it", - "developer": "ehristoforu", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "8.545" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ehristoforu_frqwen2.5-from7b-duable4layers-it/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7729 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5264 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4509 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2953 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4166 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4126 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ehristoforu_frqwen2.5-from7b-it.json b/data/models/ehristoforu_frqwen2.5-from7b-it.json deleted file mode 100644 index 3da5cabf16f12d9893e12c74e91836fa483a3cb2..0000000000000000000000000000000000000000 --- a/data/models/ehristoforu_frqwen2.5-from7b-it.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "frqwen2.5-from7b-it", - "id": "ehristoforu/frqwen2.5-from7b-it", - "developer": "ehristoforu", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "13.206" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ehristoforu_frqwen2.5-from7b-it/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6532 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5143 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2923 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2903 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4086 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3977 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ehristoforu_gemma2-9b-it-psy10k-mental_health.json b/data/models/ehristoforu_gemma2-9b-it-psy10k-mental_health.json deleted file mode 100644 index 4eac167f6f25efc2d383804f10f69a951994785a..0000000000000000000000000000000000000000 --- a/data/models/ehristoforu_gemma2-9b-it-psy10k-mental_health.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Gemma2-9B-it-psy10k-mental_health", - "id": "ehristoforu/Gemma2-9B-it-psy10k-mental_health", - "developer": "ehristoforu", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "9.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ehristoforu_Gemma2-9B-it-psy10k-mental_health/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5887 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5539 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1631 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3372 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4086 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3829 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ehristoforu_gemma2-9b-it-train6.json b/data/models/ehristoforu_gemma2-9b-it-train6.json deleted file mode 100644 index 2435a47f55b37108e33d26d6b307e987dd329217..0000000000000000000000000000000000000000 --- a/data/models/ehristoforu_gemma2-9b-it-train6.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Gemma2-9b-it-train6", - "id": "ehristoforu/Gemma2-9b-it-train6", - "developer": "ehristoforu", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "9.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ehristoforu_Gemma2-9b-it-train6/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7025 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5898 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1911 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3289 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4084 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3942 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ehristoforu_happyllama1.json b/data/models/ehristoforu_happyllama1.json deleted file mode 100644 index 0d7ef53b8f30dc5f8a3bcec9295b67ef19f5c29b..0000000000000000000000000000000000000000 --- a/data/models/ehristoforu_happyllama1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "HappyLlama1", - "id": "ehristoforu/HappyLlama1", - "developer": "ehristoforu", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ehristoforu_HappyLlama1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7363 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4996 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1427 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2836 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4287 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3546 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ehristoforu_mllama-3.1-8b-instruct.json b/data/models/ehristoforu_mllama-3.1-8b-instruct.json deleted file mode 100644 index 429596ed50bf2b3f27bdd280603e3abdcbff8c5e..0000000000000000000000000000000000000000 --- a/data/models/ehristoforu_mllama-3.1-8b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "mllama-3.1-8b-instruct", - "id": "ehristoforu/mllama-3.1-8b-instruct", - "developer": "ehristoforu", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ehristoforu_mllama-3.1-8b-instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3458 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4718 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3776 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2701 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.338 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2533 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ehristoforu_mllama-3.1-8b-it.json b/data/models/ehristoforu_mllama-3.1-8b-it.json deleted file mode 100644 index 949091308b7c098a7b3afc1e277c66c6c1ff14e3..0000000000000000000000000000000000000000 --- a/data/models/ehristoforu_mllama-3.1-8b-it.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "mllama-3.1-8b-it", - "id": "ehristoforu/mllama-3.1-8b-it", - "developer": "ehristoforu", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ehristoforu_mllama-3.1-8b-it/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3879 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4868 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3799 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2768 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3349 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2622 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ehristoforu_moremerge-upscaled.json b/data/models/ehristoforu_moremerge-upscaled.json deleted file mode 100644 index f350b133218ec0a6990b67a4b1f4940d48f38054..0000000000000000000000000000000000000000 --- a/data/models/ehristoforu_moremerge-upscaled.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "moremerge-upscaled", - "id": "ehristoforu/moremerge-upscaled", - "developer": "ehristoforu", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "8.545" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ehristoforu_moremerge-upscaled/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1979 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2698 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2466 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3593 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1041 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ehristoforu_moremerge.json b/data/models/ehristoforu_moremerge.json deleted file mode 100644 index 6ee5069eaf2170d89f55850aaf578448643ec191..0000000000000000000000000000000000000000 --- a/data/models/ehristoforu_moremerge.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "moremerge", - "id": "ehristoforu/moremerge", - "developer": "ehristoforu", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.613" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ehristoforu_moremerge/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2019 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2868 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2601 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3566 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1065 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ehristoforu_phi-4-25b.json b/data/models/ehristoforu_phi-4-25b.json deleted file mode 100644 index 32ed33799c0608762a6d248d1438695ef49edb45..0000000000000000000000000000000000000000 --- a/data/models/ehristoforu_phi-4-25b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "phi-4-25b", - "id": "ehristoforu/phi-4-25b", - "developer": "ehristoforu", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Phi3ForCausalLM", - "params_billions": "24.883" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ehristoforu_phi-4-25b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6484 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6908 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4524 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3188 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4208 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5351 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ehristoforu_qwen2.5-test-32b-it.json b/data/models/ehristoforu_qwen2.5-test-32b-it.json deleted file mode 100644 index 80c1073007fdc34a324b9cf26dd9065f61ecacd0..0000000000000000000000000000000000000000 --- a/data/models/ehristoforu_qwen2.5-test-32b-it.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "qwen2.5-test-32b-it", - "id": "ehristoforu/qwen2.5-test-32b-it", - "developer": "ehristoforu", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "32.764" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ehristoforu_qwen2.5-test-32b-it/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7889 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7081 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5974 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3641 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4578 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5765 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ehristoforu_qwen2.5-with-lora-think-3b-it.json b/data/models/ehristoforu_qwen2.5-with-lora-think-3b-it.json deleted file mode 100644 index 60892506a86b6b2754ed106d0969947ea242f510..0000000000000000000000000000000000000000 --- a/data/models/ehristoforu_qwen2.5-with-lora-think-3b-it.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "qwen2.5-with-lora-think-3b-it", - "id": "ehristoforu/qwen2.5-with-lora-think-3b-it", - "developer": "ehristoforu", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.086" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ehristoforu_qwen2.5-with-lora-think-3b-it/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5319 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4687 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2364 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2802 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.431 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3403 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ehristoforu_qwenqwen2.5-7b-it-dare.json b/data/models/ehristoforu_qwenqwen2.5-7b-it-dare.json deleted file mode 100644 index c718b64ba4c2fbbf8cca5a4ecc2dc37df742cbee..0000000000000000000000000000000000000000 --- a/data/models/ehristoforu_qwenqwen2.5-7b-it-dare.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "QwenQwen2.5-7B-IT-Dare", - "id": "ehristoforu/QwenQwen2.5-7B-IT-Dare", - "developer": "ehristoforu", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.613" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ehristoforu_QwenQwen2.5-7B-IT-Dare/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7509 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5398 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5091 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3037 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4034 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4289 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ehristoforu_qwenqwen2.5-7b-it.json b/data/models/ehristoforu_qwenqwen2.5-7b-it.json deleted file mode 100644 index 7a0d3704a8b70462a3d84e0ed34d9357f702c066..0000000000000000000000000000000000000000 --- a/data/models/ehristoforu_qwenqwen2.5-7b-it.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "QwenQwen2.5-7B-IT", - "id": "ehristoforu/QwenQwen2.5-7B-IT", - "developer": "ehristoforu", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.613" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ehristoforu_QwenQwen2.5-7B-IT/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7518 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5398 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5091 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3037 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4034 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4289 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ehristoforu_rmoe-v1.json b/data/models/ehristoforu_rmoe-v1.json deleted file mode 100644 index 4feda9013047eb5e24213fb33e932d23dd7b939e..0000000000000000000000000000000000000000 --- a/data/models/ehristoforu_rmoe-v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "rmoe-v1", - "id": "ehristoforu/rmoe-v1", - "developer": "ehristoforu", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2MoeForCausalLM", - "params_billions": "11.026" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ehristoforu_rmoe-v1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.265 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2929 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0015 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2584 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3663 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1125 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ehristoforu_rqwen-v0.1.json b/data/models/ehristoforu_rqwen-v0.1.json deleted file mode 100644 index c59ac9d100660e574984cdf6f3674d2f9ef1d238..0000000000000000000000000000000000000000 --- a/data/models/ehristoforu_rqwen-v0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "RQwen-v0.1", - "id": "ehristoforu/RQwen-v0.1", - "developer": "ehristoforu", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ehristoforu_RQwen-v0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7625 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6446 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4645 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3255 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4139 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5202 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ehristoforu_rqwen-v0.2.json b/data/models/ehristoforu_rqwen-v0.2.json deleted file mode 100644 index dc10cad2a5ef6161fa71d8d9ea9d37de0b6838ba..0000000000000000000000000000000000000000 --- a/data/models/ehristoforu_rqwen-v0.2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "RQwen-v0.2", - "id": "ehristoforu/RQwen-v0.2", - "developer": "ehristoforu", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ehristoforu_RQwen-v0.2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7504 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6427 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.327 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3372 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4207 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5159 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ehristoforu_rufalcon3-3b-it.json b/data/models/ehristoforu_rufalcon3-3b-it.json deleted file mode 100644 index 3918caa777afc29d347b41e89eeae2971c42118b..0000000000000000000000000000000000000000 --- a/data/models/ehristoforu_rufalcon3-3b-it.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "rufalcon3-3b-it", - "id": "ehristoforu/rufalcon3-3b-it", - "developer": "ehristoforu", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.228" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ehristoforu_rufalcon3-3b-it/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5942 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4155 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1782 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2727 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3895 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2348 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ehristoforu_ruphi-4b.json b/data/models/ehristoforu_ruphi-4b.json deleted file mode 100644 index 04e3fc765082b3e522565c61efb5ba678b556e11..0000000000000000000000000000000000000000 --- a/data/models/ehristoforu_ruphi-4b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ruphi-4b", - "id": "ehristoforu/ruphi-4b", - "developer": "ehristoforu", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Phi3ForCausalLM", - "params_billions": "3.821" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ehristoforu_ruphi-4b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1752 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2906 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2399 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3512 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1126 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ehristoforu_soru-0009.json b/data/models/ehristoforu_soru-0009.json deleted file mode 100644 index ca2a47eab887d599fd9ef4bdcfdf18465b00ee71..0000000000000000000000000000000000000000 --- a/data/models/ehristoforu_soru-0009.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SoRu-0009", - "id": "ehristoforu/SoRu-0009", - "developer": "ehristoforu", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ehristoforu_SoRu-0009/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2582 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.315 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0211 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2609 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3369 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1239 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ehristoforu_testq-32b.json b/data/models/ehristoforu_testq-32b.json deleted file mode 100644 index 2d4f196a003735a4e28fd79fe46248cf4abb97fd..0000000000000000000000000000000000000000 --- a/data/models/ehristoforu_testq-32b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "testq-32b", - "id": "ehristoforu/testq-32b", - "developer": "ehristoforu", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "56.165" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ehristoforu_testq-32b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1876 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2877 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.003 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2542 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3715 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1166 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ehristoforu_tmoe-v2.json b/data/models/ehristoforu_tmoe-v2.json deleted file mode 100644 index 344d35baa4b3b25e464def414cd311d8882f188a..0000000000000000000000000000000000000000 --- a/data/models/ehristoforu_tmoe-v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "tmoe-v2", - "id": "ehristoforu/tmoe-v2", - "developer": "ehristoforu", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2MoeForCausalLM", - "params_billions": "11.026" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ehristoforu_tmoe-v2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1903 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2897 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0023 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2634 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4151 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.11 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ehristoforu_tmoe.json b/data/models/ehristoforu_tmoe.json deleted file mode 100644 index e09ee82cd92abf7e68074108dcc44b3114741d23..0000000000000000000000000000000000000000 --- a/data/models/ehristoforu_tmoe.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "tmoe", - "id": "ehristoforu/tmoe", - "developer": "ehristoforu", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2MoeForCausalLM", - "params_billions": "11.026" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ehristoforu_tmoe/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1193 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3073 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0076 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2232 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3699 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1191 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ehristoforu_trd-7b-it.json b/data/models/ehristoforu_trd-7b-it.json deleted file mode 100644 index 152d56d6e7420c992d3ffd34669824e5736678dd..0000000000000000000000000000000000000000 --- a/data/models/ehristoforu_trd-7b-it.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "trd-7b-it", - "id": "ehristoforu/trd-7b-it", - "developer": "ehristoforu", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.613" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ehristoforu_trd-7b-it/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2185 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.299 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0317 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2701 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3794 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1179 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ehristoforu_ud-14b.json b/data/models/ehristoforu_ud-14b.json deleted file mode 100644 index 47d3c84a9676181e163f606a5953ca91cb9193b5..0000000000000000000000000000000000000000 --- a/data/models/ehristoforu_ud-14b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ud-14b", - "id": "ehristoforu/ud-14b", - "developer": "ehristoforu", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ehristoforu_ud-14b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4235 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3324 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1903 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2374 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4394 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2415 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/eleutherai_gpt-j-6b.json b/data/models/eleutherai_gpt-j-6b.json deleted file mode 100644 index 41cb92eaad3e71b151095442c3f8297bb928c9f0..0000000000000000000000000000000000000000 --- a/data/models/eleutherai_gpt-j-6b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "gpt-j-6b", - "id": "EleutherAI/gpt-j-6b", - "developer": "EleutherAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "GPTJForCausalLM", - "params_billions": "6.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/EleutherAI_gpt-j-6b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2522 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3191 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0136 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2458 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3658 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1241 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/eleutherai_gpt-neo-1.3b.json b/data/models/eleutherai_gpt-neo-1.3b.json deleted file mode 100644 index 55ce1699b1e4b731ba97cb9bf0a071a76a85d1e7..0000000000000000000000000000000000000000 --- a/data/models/eleutherai_gpt-neo-1.3b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "gpt-neo-1.3B", - "id": "EleutherAI/gpt-neo-1.3B", - "developer": "EleutherAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "GPTNeoForCausalLM", - "params_billions": "1.366" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/EleutherAI_gpt-neo-1.3B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2079 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3039 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0106 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2559 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3817 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1164 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/eleutherai_gpt-neo-125m.json b/data/models/eleutherai_gpt-neo-125m.json deleted file mode 100644 index e7ff769a90b6f916da4c60938b83132468b70dd0..0000000000000000000000000000000000000000 --- a/data/models/eleutherai_gpt-neo-125m.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "gpt-neo-125m", - "id": "EleutherAI/gpt-neo-125m", - "developer": "EleutherAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "GPTNeoForCausalLM", - "params_billions": "0.15" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/EleutherAI_gpt-neo-125m/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1905 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3115 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.006 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2534 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3593 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1026 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/eleutherai_gpt-neo-2.7b.json b/data/models/eleutherai_gpt-neo-2.7b.json deleted file mode 100644 index 4af1623ccf71156102dce0626cd11e76acf98870..0000000000000000000000000000000000000000 --- a/data/models/eleutherai_gpt-neo-2.7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "gpt-neo-2.7B", - "id": "EleutherAI/gpt-neo-2.7B", - "developer": "EleutherAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "GPTNeoForCausalLM", - "params_billions": "2.718" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/EleutherAI_gpt-neo-2.7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.259 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.314 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0106 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2659 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3554 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1163 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/eleutherai_gpt-neox-20b.json b/data/models/eleutherai_gpt-neox-20b.json deleted file mode 100644 index c5cb4d5c3c16584548b2e620a3ab7a0e1dfa052b..0000000000000000000000000000000000000000 --- a/data/models/eleutherai_gpt-neox-20b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "gpt-neox-20b", - "id": "EleutherAI/gpt-neox-20b", - "developer": "EleutherAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "GPTNeoXForCausalLM", - "params_billions": "20.739" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/EleutherAI_gpt-neox-20b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2587 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3165 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0136 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2433 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3647 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1155 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/eleutherai_pythia-1.4b.json b/data/models/eleutherai_pythia-1.4b.json deleted file mode 100644 index 1ce4d77b0706c994275be9381523a642184ae006..0000000000000000000000000000000000000000 --- a/data/models/eleutherai_pythia-1.4b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "pythia-1.4b", - "id": "EleutherAI/pythia-1.4b", - "developer": "EleutherAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "GPTNeoXForCausalLM", - "params_billions": "1.515" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/EleutherAI_pythia-1.4b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2371 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.315 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0151 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2617 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3538 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1123 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/eleutherai_pythia-12b.json b/data/models/eleutherai_pythia-12b.json deleted file mode 100644 index 5ebca0860ae7f24c8a9ecb80393be4601aeba31e..0000000000000000000000000000000000000000 --- a/data/models/eleutherai_pythia-12b.json +++ /dev/null @@ -1,809 +0,0 @@ -{ - "model_info": { - "name": "Pythia 12B", - "id": "EleutherAI/pythia-12b", - "developer": "EleutherAI", - "inference_platform": "unknown", - "additional_details": { - "model_id_aliases": [ - "eleutherai/Pythia-12B" - ] - } - }, - "evaluations": [ - { - "evaluation_id": "helm_classic/eleutherai_Pythia-12B/1774096308.339228", - "retrieved_timestamp": "1774096308.339228", - "source_metadata": { - "source_name": "helm_classic", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_classic", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_classic", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperform on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.257, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Calibration": "{\"description\": \"\", \"tab\": \"Calibration\", \"score\": \"0.37428307123034227\"}", - "Mean win rate - Robustness": "{\"description\": \"\", \"tab\": \"Robustness\", \"score\": \"0.27195804195804196\"}", - "Mean win rate - Fairness": "{\"description\": \"\", \"tab\": \"Fairness\", \"score\": \"0.22631701631701634\"}", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}", - "Mean win rate - Bias": "{\"description\": \"\", \"tab\": \"Bias\", \"score\": \"0.4331466568182155\"}", - "Mean win rate - Toxicity": "{\"description\": \"\", \"tab\": \"Toxicity\", \"score\": \"0.38444055944055944\"}", - "Mean win rate - Summarization metrics": "{\"description\": \"\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.274, - "details": { - "description": "min=0.2, mean=0.274, max=0.3, sum=1.368 (5)", - "tab": "Accuracy", - "MMLU - ECE (10-bin)": "{\"description\": \"min=0.092, mean=0.111, max=0.166, sum=0.557 (5)\", \"tab\": \"Calibration\", \"score\": \"0.11132961223278444\"}", - "MMLU - EM (Robustness)": "{\"description\": \"min=0.17, mean=0.22, max=0.28, sum=1.102 (5)\", \"tab\": \"Robustness\", \"score\": \"0.22035087719298244\"}", - "MMLU - EM (Fairness)": "{\"description\": \"min=0.16, mean=0.212, max=0.29, sum=1.061 (5)\", \"tab\": \"Fairness\", \"score\": \"0.2121052631578947\"}", - "MMLU - Denoised inference time (s)": "{\"description\": \"5 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=358.76, mean=467.936, max=612.798, sum=2339.678 (5)\", \"tab\": \"General information\", \"score\": \"467.935649122807\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "MMLU - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "BoolQ", - "source_data": { - "dataset_name": "BoolQ", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on BoolQ", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.662, - "details": { - "description": "min=0.662, mean=0.662, max=0.662, sum=0.662 (1)", - "tab": "Accuracy", - "BoolQ - ECE (10-bin)": "{\"description\": \"min=0.14, mean=0.14, max=0.14, sum=0.14 (1)\", \"tab\": \"Calibration\", \"score\": \"0.13986557582802048\"}", - "BoolQ - EM (Robustness)": "{\"description\": \"min=0.51, mean=0.51, max=0.51, sum=0.51 (1)\", \"tab\": \"Robustness\", \"score\": \"0.51\"}", - "BoolQ - EM (Fairness)": "{\"description\": \"min=0.547, mean=0.547, max=0.547, sum=0.547 (1)\", \"tab\": \"Fairness\", \"score\": \"0.547\"}", - "BoolQ - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "BoolQ - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "BoolQ - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "BoolQ - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "BoolQ - # prompt tokens": "{\"description\": \"min=1251.897, mean=1251.897, max=1251.897, sum=1251.897 (1)\", \"tab\": \"General information\", \"score\": \"1251.897\"}", - "BoolQ - # output tokens": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "BoolQ - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "BoolQ - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.596, - "details": { - "description": "min=0.596, mean=0.596, max=0.596, sum=0.596 (1)", - "tab": "Accuracy", - "NarrativeQA - ECE (10-bin)": "{\"description\": \"min=0.239, mean=0.239, max=0.239, sum=0.239 (1)\", \"tab\": \"Calibration\", \"score\": \"0.2394289121866973\"}", - "NarrativeQA - F1 (Robustness)": "{\"description\": \"min=0.42, mean=0.42, max=0.42, sum=0.42 (1)\", \"tab\": \"Robustness\", \"score\": \"0.42022169799567144\"}", - "NarrativeQA - F1 (Fairness)": "{\"description\": \"min=0.449, mean=0.449, max=0.449, sum=0.449 (1)\", \"tab\": \"Fairness\", \"score\": \"0.44869513696457247\"}", - "NarrativeQA - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=1.969, mean=1.969, max=1.969, sum=1.969 (1)\", \"tab\": \"General information\", \"score\": \"1.9690140845070423\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=1691.082, mean=1691.082, max=1691.082, sum=1691.082 (1)\", \"tab\": \"General information\", \"score\": \"1691.081690140845\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=100, mean=100, max=100, sum=100 (1)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "NarrativeQA - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NarrativeQA - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NarrativeQA - Stereotypes (gender)": "{\"description\": \"min=0.5, mean=0.5, max=0.5, sum=0.5 (1)\", \"tab\": \"Bias\", \"score\": \"0.5\"}", - "NarrativeQA - Representation (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=0.667 (1)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666667\"}", - "NarrativeQA - Representation (gender)": "{\"description\": \"min=0.215, mean=0.215, max=0.215, sum=0.215 (1)\", \"tab\": \"Bias\", \"score\": \"0.2152777777777778\"}", - "NarrativeQA - Toxic fraction": "{\"description\": \"min=0.023, mean=0.023, max=0.023, sum=0.023 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.022535211267605635\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (open-book)", - "source_data": { - "dataset_name": "NaturalQuestions (open-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (open-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.581, - "details": { - "description": "min=0.581, mean=0.581, max=0.581, sum=0.581 (1)", - "tab": "Accuracy", - "NaturalQuestions (closed-book) - ECE (10-bin)": "{\"description\": \"min=0.094, mean=0.094, max=0.094, sum=0.094 (1)\", \"tab\": \"Calibration\", \"score\": \"0.09399996958029097\"}", - "NaturalQuestions (open-book) - ECE (10-bin)": "{\"description\": \"min=0.39, mean=0.39, max=0.39, sum=0.39 (1)\", \"tab\": \"Calibration\", \"score\": \"0.3899944090149843\"}", - "NaturalQuestions (closed-book) - F1 (Robustness)": "{\"description\": \"min=0.108, mean=0.108, max=0.108, sum=0.108 (1)\", \"tab\": \"Robustness\", \"score\": \"0.10849928114746796\"}", - "NaturalQuestions (open-book) - F1 (Robustness)": "{\"description\": \"min=0.47, mean=0.47, max=0.47, sum=0.47 (1)\", \"tab\": \"Robustness\", \"score\": \"0.46990137932247006\"}", - "NaturalQuestions (closed-book) - F1 (Fairness)": "{\"description\": \"min=0.131, mean=0.131, max=0.131, sum=0.131 (1)\", \"tab\": \"Fairness\", \"score\": \"0.13109020655004933\"}", - "NaturalQuestions (open-book) - F1 (Fairness)": "{\"description\": \"min=0.523, mean=0.523, max=0.523, sum=0.523 (1)\", \"tab\": \"Fairness\", \"score\": \"0.5229768252994325\"}", - "NaturalQuestions (closed-book) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NaturalQuestions (open-book) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=117.299, mean=117.299, max=117.299, sum=117.299 (1)\", \"tab\": \"General information\", \"score\": \"117.299\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=300, mean=300, max=300, sum=300 (1)\", \"tab\": \"General information\", \"score\": \"300.0\"}", - "NaturalQuestions (closed-book) - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.704, mean=4.704, max=4.704, sum=4.704 (1)\", \"tab\": \"General information\", \"score\": \"4.704\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.037, mean=0.037, max=0.037, sum=0.037 (1)\", \"tab\": \"General information\", \"score\": \"0.037\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1495.552, mean=1495.552, max=1495.552, sum=1495.552 (1)\", \"tab\": \"General information\", \"score\": \"1495.552\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=300, mean=300, max=300, sum=300 (1)\", \"tab\": \"General information\", \"score\": \"300.0\"}", - "NaturalQuestions (open-book) - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NaturalQuestions (closed-book) - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - Stereotypes (gender)": "{\"description\": \"min=0.5, mean=0.5, max=0.5, sum=0.5 (1)\", \"tab\": \"Bias\", \"score\": \"0.5\"}", - "NaturalQuestions (closed-book) - Representation (race)": "{\"description\": \"min=0.407, mean=0.407, max=0.407, sum=0.407 (1)\", \"tab\": \"Bias\", \"score\": \"0.40682414698162733\"}", - "NaturalQuestions (closed-book) - Representation (gender)": "{\"description\": \"min=0.122, mean=0.122, max=0.122, sum=0.122 (1)\", \"tab\": \"Bias\", \"score\": \"0.1216216216216216\"}", - "NaturalQuestions (open-book) - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NaturalQuestions (open-book) - Stereotypes (gender)": "{\"description\": \"min=0.405, mean=0.405, max=0.405, sum=0.405 (1)\", \"tab\": \"Bias\", \"score\": \"0.4047619047619048\"}", - "NaturalQuestions (open-book) - Representation (race)": "{\"description\": \"min=0.467, mean=0.467, max=0.467, sum=0.467 (1)\", \"tab\": \"Bias\", \"score\": \"0.4666666666666667\"}", - "NaturalQuestions (open-book) - Representation (gender)": "{\"description\": \"min=0.276, mean=0.276, max=0.276, sum=0.276 (1)\", \"tab\": \"Bias\", \"score\": \"0.27551020408163257\"}", - "NaturalQuestions (closed-book) - Toxic fraction": "{\"description\": \"min=0.002, mean=0.002, max=0.002, sum=0.002 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.002\"}", - "NaturalQuestions (open-book) - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "QuAC", - "source_data": { - "dataset_name": "QuAC", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on QuAC", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.313, - "details": { - "description": "min=0.313, mean=0.313, max=0.313, sum=0.313 (1)", - "tab": "Accuracy", - "QuAC - ECE (10-bin)": "{\"description\": \"min=0.138, mean=0.138, max=0.138, sum=0.138 (1)\", \"tab\": \"Calibration\", \"score\": \"0.1383150544527575\"}", - "QuAC - F1 (Robustness)": "{\"description\": \"min=0.171, mean=0.171, max=0.171, sum=0.171 (1)\", \"tab\": \"Robustness\", \"score\": \"0.17120890749036072\"}", - "QuAC - F1 (Fairness)": "{\"description\": \"min=0.227, mean=0.227, max=0.227, sum=0.227 (1)\", \"tab\": \"Fairness\", \"score\": \"0.22738715021444486\"}", - "QuAC - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "QuAC - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "QuAC - # train": "{\"description\": \"min=0.883, mean=0.883, max=0.883, sum=0.883 (1)\", \"tab\": \"General information\", \"score\": \"0.883\"}", - "QuAC - truncated": "{\"description\": \"min=0.021, mean=0.021, max=0.021, sum=0.021 (1)\", \"tab\": \"General information\", \"score\": \"0.021\"}", - "QuAC - # prompt tokens": "{\"description\": \"min=1655.708, mean=1655.708, max=1655.708, sum=1655.708 (1)\", \"tab\": \"General information\", \"score\": \"1655.708\"}", - "QuAC - # output tokens": "{\"description\": \"min=100, mean=100, max=100, sum=100 (1)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "QuAC - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "QuAC - Stereotypes (race)": "{\"description\": \"min=0.641, mean=0.641, max=0.641, sum=0.641 (1)\", \"tab\": \"Bias\", \"score\": \"0.6406926406926409\"}", - "QuAC - Stereotypes (gender)": "{\"description\": \"min=0.415, mean=0.415, max=0.415, sum=0.415 (1)\", \"tab\": \"Bias\", \"score\": \"0.4150793650793651\"}", - "QuAC - Representation (race)": "{\"description\": \"min=0.314, mean=0.314, max=0.314, sum=0.314 (1)\", \"tab\": \"Bias\", \"score\": \"0.3137254901960784\"}", - "QuAC - Representation (gender)": "{\"description\": \"min=0.26, mean=0.26, max=0.26, sum=0.26 (1)\", \"tab\": \"Bias\", \"score\": \"0.25965665236051505\"}", - "QuAC - Toxic fraction": "{\"description\": \"min=0.002, mean=0.002, max=0.002, sum=0.002 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.002\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "HellaSwag", - "source_data": { - "dataset_name": "HellaSwag", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on HellaSwag", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "HellaSwag - ECE (10-bin)": "{\"description\": \"No matching runs\", \"tab\": \"Calibration\", \"score\": \"\"}", - "HellaSwag - EM (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "HellaSwag - EM (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "HellaSwag - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "HellaSwag - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "OpenbookQA - ECE (10-bin)": "{\"description\": \"No matching runs\", \"tab\": \"Calibration\", \"score\": \"\"}", - "OpenbookQA - EM (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "OpenbookQA - EM (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "OpenbookQA - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "OpenbookQA - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "TruthfulQA", - "source_data": { - "dataset_name": "TruthfulQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on TruthfulQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.177, - "details": { - "description": "min=0.177, mean=0.177, max=0.177, sum=0.177 (1)", - "tab": "Accuracy", - "TruthfulQA - ECE (10-bin)": "{\"description\": \"min=0.094, mean=0.094, max=0.094, sum=0.094 (1)\", \"tab\": \"Calibration\", \"score\": \"0.09363268995646454\"}", - "TruthfulQA - EM (Robustness)": "{\"description\": \"min=0.138, mean=0.138, max=0.138, sum=0.138 (1)\", \"tab\": \"Robustness\", \"score\": \"0.13761467889908258\"}", - "TruthfulQA - EM (Fairness)": "{\"description\": \"min=0.154, mean=0.154, max=0.154, sum=0.154 (1)\", \"tab\": \"Fairness\", \"score\": \"0.154434250764526\"}", - "TruthfulQA - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "TruthfulQA - # eval": "{\"description\": \"min=654, mean=654, max=654, sum=654 (1)\", \"tab\": \"General information\", \"score\": \"654.0\"}", - "TruthfulQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "TruthfulQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "TruthfulQA - # prompt tokens": "{\"description\": \"min=505.352, mean=505.352, max=505.352, sum=505.352 (1)\", \"tab\": \"General information\", \"score\": \"505.35168195718654\"}", - "TruthfulQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "TruthfulQA - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MS MARCO (TREC)", - "source_data": { - "dataset_name": "MS MARCO (TREC)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "NDCG@10 on MS MARCO (TREC)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "MS MARCO (regular) - RR@10 (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "MS MARCO (TREC) - NDCG@10 (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "MS MARCO (regular) - RR@10 (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "MS MARCO (TREC) - NDCG@10 (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "MS MARCO (regular) - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MS MARCO (TREC) - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MS MARCO (regular) - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "MS MARCO (TREC) - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CNN/DailyMail", - "source_data": { - "dataset_name": "CNN/DailyMail", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on CNN/DailyMail", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "CNN/DailyMail - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "CNN/DailyMail - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "CNN/DailyMail - SummaC": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - QAFactEval": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - BERTScore (F1)": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - Coverage": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - Density": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - Compression": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-faithfulness": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-relevance": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-coherence": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "XSUM", - "source_data": { - "dataset_name": "XSUM", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on XSUM", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "XSUM - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "XSUM - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "XSUM - SummaC": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - QAFactEval": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - BERTScore (F1)": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - Coverage": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - Density": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - Compression": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-faithfulness": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-relevance": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-coherence": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "IMDB", - "source_data": { - "dataset_name": "IMDB", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on IMDB", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.931, - "details": { - "description": "min=0.931, mean=0.931, max=0.931, sum=0.931 (1)", - "tab": "Accuracy", - "IMDB - ECE (10-bin)": "{\"description\": \"min=0.342, mean=0.342, max=0.342, sum=0.342 (1)\", \"tab\": \"Calibration\", \"score\": \"0.34150363639115\"}", - "IMDB - EM (Robustness)": "{\"description\": \"min=0.854, mean=0.854, max=0.854, sum=0.854 (1)\", \"tab\": \"Robustness\", \"score\": \"0.854\"}", - "IMDB - EM (Fairness)": "{\"description\": \"min=0.916, mean=0.916, max=0.916, sum=0.916 (1)\", \"tab\": \"Fairness\", \"score\": \"0.916\"}", - "IMDB - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "IMDB - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "IMDB - # train": "{\"description\": \"min=2.911, mean=2.911, max=2.911, sum=2.911 (1)\", \"tab\": \"General information\", \"score\": \"2.911\"}", - "IMDB - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IMDB - # prompt tokens": "{\"description\": \"min=1619.568, mean=1619.568, max=1619.568, sum=1619.568 (1)\", \"tab\": \"General information\", \"score\": \"1619.568\"}", - "IMDB - # output tokens": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "IMDB - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "IMDB - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CivilComments", - "source_data": { - "dataset_name": "CivilComments", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on CivilComments", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.531, - "details": { - "description": "min=0.03, mean=0.531, max=0.988, sum=9.561 (18)", - "tab": "Accuracy", - "CivilComments - ECE (10-bin)": "{\"description\": \"min=0.138, mean=0.297, max=0.479, sum=5.337 (18)\", \"tab\": \"Calibration\", \"score\": \"0.2965193799633309\"}", - "CivilComments - EM (Robustness)": "{\"description\": \"min=0.02, mean=0.418, max=0.973, sum=7.526 (18)\", \"tab\": \"Robustness\", \"score\": \"0.41812542395705293\"}", - "CivilComments - EM (Fairness)": "{\"description\": \"min=0.01, mean=0.448, max=0.985, sum=8.071 (18)\", \"tab\": \"Fairness\", \"score\": \"0.44837567354282437\"}", - "CivilComments - Denoised inference time (s)": "{\"description\": \"9 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "CivilComments - # eval": "{\"description\": \"min=74, mean=371.556, max=683, sum=6688 (18)\", \"tab\": \"General information\", \"score\": \"371.55555555555554\"}", - "CivilComments - # train": "{\"description\": \"min=5, mean=5, max=5, sum=90 (18)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CivilComments - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (18)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CivilComments - # prompt tokens": "{\"description\": \"min=360.976, mean=771.654, max=1282.4, sum=13889.772 (18)\", \"tab\": \"General information\", \"score\": \"771.6539847352628\"}", - "CivilComments - # output tokens": "{\"description\": \"min=5, mean=5, max=5, sum=90 (18)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CivilComments - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=18 (18)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "CivilComments - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Toxic fraction": "{\"description\": \"9 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "RAFT", - "source_data": { - "dataset_name": "RAFT", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on RAFT", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.514, - "details": { - "description": "min=0.175, mean=0.514, max=0.975, sum=5.65 (11)", - "tab": "Accuracy", - "RAFT - ECE (10-bin)": "{\"description\": \"min=0.175, mean=0.514, max=0.975, sum=5.649 (11)\", \"tab\": \"Calibration\", \"score\": \"0.5135614568346981\"}", - "RAFT - EM (Robustness)": "{\"description\": \"min=0, mean=0.45, max=0.975, sum=4.95 (11)\", \"tab\": \"Robustness\", \"score\": \"0.45\"}", - "RAFT - EM (Fairness)": "{\"description\": \"min=0.15, mean=0.489, max=0.975, sum=5.375 (11)\", \"tab\": \"Fairness\", \"score\": \"0.48863636363636365\"}", - "RAFT - Denoised inference time (s)": "{\"description\": \"11 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "RAFT - # eval": "{\"description\": \"min=40, mean=40, max=40, sum=440 (11)\", \"tab\": \"General information\", \"score\": \"40.0\"}", - "RAFT - # train": "{\"description\": \"min=0.7, mean=4.605, max=5, sum=50.65 (11)\", \"tab\": \"General information\", \"score\": \"4.6045454545454545\"}", - "RAFT - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (11)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "RAFT - # prompt tokens": "{\"description\": \"min=280.35, mean=869.691, max=1756.575, sum=9566.6 (11)\", \"tab\": \"General information\", \"score\": \"869.6909090909089\"}", - "RAFT - # output tokens": "{\"description\": \"min=30, mean=30, max=30, sum=330 (11)\", \"tab\": \"General information\", \"score\": \"30.0\"}", - "RAFT - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=11 (11)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "RAFT - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Toxic fraction": "{\"description\": \"11 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "hfopenllm_v2/EleutherAI_pythia-12b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2471 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.318 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0166 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2466 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3647 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1109 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/eleutherai_pythia-160m.json b/data/models/eleutherai_pythia-160m.json deleted file mode 100644 index 04bcd3ac45835efa1eccd87be7d71d22f32e8b81..0000000000000000000000000000000000000000 --- a/data/models/eleutherai_pythia-160m.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "pythia-160m", - "id": "EleutherAI/pythia-160m", - "developer": "EleutherAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "GPTNeoXForCausalLM", - "params_billions": "0.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/EleutherAI_pythia-160m/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1816 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.297 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0091 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2584 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4179 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.112 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/eleutherai_pythia-1b.json b/data/models/eleutherai_pythia-1b.json deleted file mode 100644 index fc92df96d0794fff15be95573ed9c82f83f927ca..0000000000000000000000000000000000000000 --- a/data/models/eleutherai_pythia-1b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "pythia-1b", - "id": "EleutherAI/pythia-1b", - "developer": "EleutherAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "GPTNeoXForCausalLM", - "params_billions": "1.079" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/EleutherAI_pythia-1b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2208 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3004 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0091 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2567 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3552 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1136 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/eleutherai_pythia-2.8b.json b/data/models/eleutherai_pythia-2.8b.json deleted file mode 100644 index 0d8cbd5a5c59dbfcd59e228248ff7f8c43057ff0..0000000000000000000000000000000000000000 --- a/data/models/eleutherai_pythia-2.8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "pythia-2.8b", - "id": "EleutherAI/pythia-2.8b", - "developer": "EleutherAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "GPTNeoXForCausalLM", - "params_billions": "2.909" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/EleutherAI_pythia-2.8b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2173 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3224 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0136 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.25 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3486 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1137 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/eleutherai_pythia-410m.json b/data/models/eleutherai_pythia-410m.json deleted file mode 100644 index 41aabfebeae623665e7752bcfdc4e19463467af0..0000000000000000000000000000000000000000 --- a/data/models/eleutherai_pythia-410m.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "pythia-410m", - "id": "EleutherAI/pythia-410m", - "developer": "EleutherAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "GPTNeoXForCausalLM", - "params_billions": "0.506" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/EleutherAI_pythia-410m/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2195 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3028 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0098 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2592 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3578 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1128 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/eleutherai_pythia-6.9b.json b/data/models/eleutherai_pythia-6.9b.json deleted file mode 100644 index e4c45d1fae35583aeb2dbe138e2bc2da331dacc7..0000000000000000000000000000000000000000 --- a/data/models/eleutherai_pythia-6.9b.json +++ /dev/null @@ -1,809 +0,0 @@ -{ - "model_info": { - "name": "Pythia 6.9B", - "id": "EleutherAI/pythia-6.9b", - "developer": "EleutherAI", - "inference_platform": "unknown", - "additional_details": { - "model_id_aliases": [ - "eleutherai/Pythia-6.9B" - ] - } - }, - "evaluations": [ - { - "evaluation_id": "helm_classic/eleutherai_Pythia-6.9B/1774096308.339228", - "retrieved_timestamp": "1774096308.339228", - "source_metadata": { - "source_name": "helm_classic", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_classic", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_classic", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperform on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.196, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Calibration": "{\"description\": \"\", \"tab\": \"Calibration\", \"score\": \"0.4304810360777058\"}", - "Mean win rate - Robustness": "{\"description\": \"\", \"tab\": \"Robustness\", \"score\": \"0.1820979020979021\"}", - "Mean win rate - Fairness": "{\"description\": \"\", \"tab\": \"Fairness\", \"score\": \"0.17121212121212123\"}", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}", - "Mean win rate - Bias": "{\"description\": \"\", \"tab\": \"Bias\", \"score\": \"0.5099743679983342\"}", - "Mean win rate - Toxicity": "{\"description\": \"\", \"tab\": \"Toxicity\", \"score\": \"0.512004662004662\"}", - "Mean win rate - Summarization metrics": "{\"description\": \"\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.236, - "details": { - "description": "min=0.16, mean=0.236, max=0.281, sum=1.181 (5)", - "tab": "Accuracy", - "MMLU - ECE (10-bin)": "{\"description\": \"min=0.064, mean=0.136, max=0.2, sum=0.682 (5)\", \"tab\": \"Calibration\", \"score\": \"0.1364262799156796\"}", - "MMLU - EM (Robustness)": "{\"description\": \"min=0.12, mean=0.201, max=0.263, sum=1.003 (5)\", \"tab\": \"Robustness\", \"score\": \"0.20063157894736844\"}", - "MMLU - EM (Fairness)": "{\"description\": \"min=0.14, mean=0.207, max=0.254, sum=1.034 (5)\", \"tab\": \"Fairness\", \"score\": \"0.20687719298245613\"}", - "MMLU - Denoised inference time (s)": "{\"description\": \"5 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=358.76, mean=467.936, max=612.798, sum=2339.678 (5)\", \"tab\": \"General information\", \"score\": \"467.935649122807\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "MMLU - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "BoolQ", - "source_data": { - "dataset_name": "BoolQ", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on BoolQ", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.631, - "details": { - "description": "min=0.631, mean=0.631, max=0.631, sum=0.631 (1)", - "tab": "Accuracy", - "BoolQ - ECE (10-bin)": "{\"description\": \"min=0.106, mean=0.106, max=0.106, sum=0.106 (1)\", \"tab\": \"Calibration\", \"score\": \"0.10596147166386737\"}", - "BoolQ - EM (Robustness)": "{\"description\": \"min=0.527, mean=0.527, max=0.527, sum=0.527 (1)\", \"tab\": \"Robustness\", \"score\": \"0.527\"}", - "BoolQ - EM (Fairness)": "{\"description\": \"min=0.552, mean=0.552, max=0.552, sum=0.552 (1)\", \"tab\": \"Fairness\", \"score\": \"0.552\"}", - "BoolQ - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "BoolQ - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "BoolQ - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "BoolQ - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "BoolQ - # prompt tokens": "{\"description\": \"min=1251.897, mean=1251.897, max=1251.897, sum=1251.897 (1)\", \"tab\": \"General information\", \"score\": \"1251.897\"}", - "BoolQ - # output tokens": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "BoolQ - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "BoolQ - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.528, - "details": { - "description": "min=0.528, mean=0.528, max=0.528, sum=0.528 (1)", - "tab": "Accuracy", - "NarrativeQA - ECE (10-bin)": "{\"description\": \"min=0.217, mean=0.217, max=0.217, sum=0.217 (1)\", \"tab\": \"Calibration\", \"score\": \"0.21689349381563713\"}", - "NarrativeQA - F1 (Robustness)": "{\"description\": \"min=0.313, mean=0.313, max=0.313, sum=0.313 (1)\", \"tab\": \"Robustness\", \"score\": \"0.31250255336597976\"}", - "NarrativeQA - F1 (Fairness)": "{\"description\": \"min=0.389, mean=0.389, max=0.389, sum=0.389 (1)\", \"tab\": \"Fairness\", \"score\": \"0.38935766339772926\"}", - "NarrativeQA - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=1.969, mean=1.969, max=1.969, sum=1.969 (1)\", \"tab\": \"General information\", \"score\": \"1.9690140845070423\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=1691.082, mean=1691.082, max=1691.082, sum=1691.082 (1)\", \"tab\": \"General information\", \"score\": \"1691.081690140845\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=100, mean=100, max=100, sum=100 (1)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "NarrativeQA - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NarrativeQA - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NarrativeQA - Stereotypes (gender)": "{\"description\": \"min=0.444, mean=0.444, max=0.444, sum=0.444 (1)\", \"tab\": \"Bias\", \"score\": \"0.4444444444444444\"}", - "NarrativeQA - Representation (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=0.667 (1)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666667\"}", - "NarrativeQA - Representation (gender)": "{\"description\": \"min=0.204, mean=0.204, max=0.204, sum=0.204 (1)\", \"tab\": \"Bias\", \"score\": \"0.20434782608695648\"}", - "NarrativeQA - Toxic fraction": "{\"description\": \"min=0.014, mean=0.014, max=0.014, sum=0.014 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.014084507042253521\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (open-book)", - "source_data": { - "dataset_name": "NaturalQuestions (open-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (open-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.539, - "details": { - "description": "min=0.539, mean=0.539, max=0.539, sum=0.539 (1)", - "tab": "Accuracy", - "NaturalQuestions (closed-book) - ECE (10-bin)": "{\"description\": \"min=0.07, mean=0.07, max=0.07, sum=0.07 (1)\", \"tab\": \"Calibration\", \"score\": \"0.06999999827276561\"}", - "NaturalQuestions (open-book) - ECE (10-bin)": "{\"description\": \"min=0.369, mean=0.369, max=0.369, sum=0.369 (1)\", \"tab\": \"Calibration\", \"score\": \"0.3689977017786239\"}", - "NaturalQuestions (closed-book) - F1 (Robustness)": "{\"description\": \"min=0.094, mean=0.094, max=0.094, sum=0.094 (1)\", \"tab\": \"Robustness\", \"score\": \"0.09385332819874069\"}", - "NaturalQuestions (open-book) - F1 (Robustness)": "{\"description\": \"min=0.391, mean=0.391, max=0.391, sum=0.391 (1)\", \"tab\": \"Robustness\", \"score\": \"0.39128308105054077\"}", - "NaturalQuestions (closed-book) - F1 (Fairness)": "{\"description\": \"min=0.103, mean=0.103, max=0.103, sum=0.103 (1)\", \"tab\": \"Fairness\", \"score\": \"0.10301926896303132\"}", - "NaturalQuestions (open-book) - F1 (Fairness)": "{\"description\": \"min=0.464, mean=0.464, max=0.464, sum=0.464 (1)\", \"tab\": \"Fairness\", \"score\": \"0.4640855445555752\"}", - "NaturalQuestions (closed-book) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NaturalQuestions (open-book) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=117.299, mean=117.299, max=117.299, sum=117.299 (1)\", \"tab\": \"General information\", \"score\": \"117.299\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=300, mean=300, max=300, sum=300 (1)\", \"tab\": \"General information\", \"score\": \"300.0\"}", - "NaturalQuestions (closed-book) - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.704, mean=4.704, max=4.704, sum=4.704 (1)\", \"tab\": \"General information\", \"score\": \"4.704\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.037, mean=0.037, max=0.037, sum=0.037 (1)\", \"tab\": \"General information\", \"score\": \"0.037\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1495.552, mean=1495.552, max=1495.552, sum=1495.552 (1)\", \"tab\": \"General information\", \"score\": \"1495.552\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=299.883, mean=299.883, max=299.883, sum=299.883 (1)\", \"tab\": \"General information\", \"score\": \"299.883\"}", - "NaturalQuestions (open-book) - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NaturalQuestions (closed-book) - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - Stereotypes (gender)": "{\"description\": \"min=0.5, mean=0.5, max=0.5, sum=0.5 (1)\", \"tab\": \"Bias\", \"score\": \"0.5\"}", - "NaturalQuestions (closed-book) - Representation (race)": "{\"description\": \"min=0.312, mean=0.312, max=0.312, sum=0.312 (1)\", \"tab\": \"Bias\", \"score\": \"0.31182795698924726\"}", - "NaturalQuestions (closed-book) - Representation (gender)": "{\"description\": \"min=0.188, mean=0.188, max=0.188, sum=0.188 (1)\", \"tab\": \"Bias\", \"score\": \"0.1875\"}", - "NaturalQuestions (open-book) - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NaturalQuestions (open-book) - Stereotypes (gender)": "{\"description\": \"min=0.387, mean=0.387, max=0.387, sum=0.387 (1)\", \"tab\": \"Bias\", \"score\": \"0.38690476190476186\"}", - "NaturalQuestions (open-book) - Representation (race)": "{\"description\": \"min=0.422, mean=0.422, max=0.422, sum=0.422 (1)\", \"tab\": \"Bias\", \"score\": \"0.42222222222222217\"}", - "NaturalQuestions (open-book) - Representation (gender)": "{\"description\": \"min=0.159, mean=0.159, max=0.159, sum=0.159 (1)\", \"tab\": \"Bias\", \"score\": \"0.1590909090909091\"}", - "NaturalQuestions (closed-book) - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}", - "NaturalQuestions (open-book) - Toxic fraction": "{\"description\": \"min=0.001, mean=0.001, max=0.001, sum=0.001 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.001\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "QuAC", - "source_data": { - "dataset_name": "QuAC", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on QuAC", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.296, - "details": { - "description": "min=0.296, mean=0.296, max=0.296, sum=0.296 (1)", - "tab": "Accuracy", - "QuAC - ECE (10-bin)": "{\"description\": \"min=0.1, mean=0.1, max=0.1, sum=0.1 (1)\", \"tab\": \"Calibration\", \"score\": \"0.09977223409937552\"}", - "QuAC - F1 (Robustness)": "{\"description\": \"min=0.171, mean=0.171, max=0.171, sum=0.171 (1)\", \"tab\": \"Robustness\", \"score\": \"0.17097990289529255\"}", - "QuAC - F1 (Fairness)": "{\"description\": \"min=0.198, mean=0.198, max=0.198, sum=0.198 (1)\", \"tab\": \"Fairness\", \"score\": \"0.19836760191150613\"}", - "QuAC - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "QuAC - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "QuAC - # train": "{\"description\": \"min=0.883, mean=0.883, max=0.883, sum=0.883 (1)\", \"tab\": \"General information\", \"score\": \"0.883\"}", - "QuAC - truncated": "{\"description\": \"min=0.021, mean=0.021, max=0.021, sum=0.021 (1)\", \"tab\": \"General information\", \"score\": \"0.021\"}", - "QuAC - # prompt tokens": "{\"description\": \"min=1655.708, mean=1655.708, max=1655.708, sum=1655.708 (1)\", \"tab\": \"General information\", \"score\": \"1655.708\"}", - "QuAC - # output tokens": "{\"description\": \"min=100, mean=100, max=100, sum=100 (1)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "QuAC - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "QuAC - Stereotypes (race)": "{\"description\": \"min=0.635, mean=0.635, max=0.635, sum=0.635 (1)\", \"tab\": \"Bias\", \"score\": \"0.6349206349206349\"}", - "QuAC - Stereotypes (gender)": "{\"description\": \"min=0.416, mean=0.416, max=0.416, sum=0.416 (1)\", \"tab\": \"Bias\", \"score\": \"0.41639199007620065\"}", - "QuAC - Representation (race)": "{\"description\": \"min=0.369, mean=0.369, max=0.369, sum=0.369 (1)\", \"tab\": \"Bias\", \"score\": \"0.3687074829931972\"}", - "QuAC - Representation (gender)": "{\"description\": \"min=0.25, mean=0.25, max=0.25, sum=0.25 (1)\", \"tab\": \"Bias\", \"score\": \"0.25\"}", - "QuAC - Toxic fraction": "{\"description\": \"min=0.003, mean=0.003, max=0.003, sum=0.003 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.003\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "HellaSwag", - "source_data": { - "dataset_name": "HellaSwag", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on HellaSwag", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "HellaSwag - ECE (10-bin)": "{\"description\": \"No matching runs\", \"tab\": \"Calibration\", \"score\": \"\"}", - "HellaSwag - EM (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "HellaSwag - EM (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "HellaSwag - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "HellaSwag - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "OpenbookQA - ECE (10-bin)": "{\"description\": \"No matching runs\", \"tab\": \"Calibration\", \"score\": \"\"}", - "OpenbookQA - EM (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "OpenbookQA - EM (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "OpenbookQA - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "OpenbookQA - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "TruthfulQA", - "source_data": { - "dataset_name": "TruthfulQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on TruthfulQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.213, - "details": { - "description": "min=0.213, mean=0.213, max=0.213, sum=0.213 (1)", - "tab": "Accuracy", - "TruthfulQA - ECE (10-bin)": "{\"description\": \"min=0.076, mean=0.076, max=0.076, sum=0.076 (1)\", \"tab\": \"Calibration\", \"score\": \"0.07613907039385276\"}", - "TruthfulQA - EM (Robustness)": "{\"description\": \"min=0.139, mean=0.139, max=0.139, sum=0.139 (1)\", \"tab\": \"Robustness\", \"score\": \"0.13914373088685014\"}", - "TruthfulQA - EM (Fairness)": "{\"description\": \"min=0.18, mean=0.18, max=0.18, sum=0.18 (1)\", \"tab\": \"Fairness\", \"score\": \"0.18042813455657492\"}", - "TruthfulQA - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "TruthfulQA - # eval": "{\"description\": \"min=654, mean=654, max=654, sum=654 (1)\", \"tab\": \"General information\", \"score\": \"654.0\"}", - "TruthfulQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "TruthfulQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "TruthfulQA - # prompt tokens": "{\"description\": \"min=505.352, mean=505.352, max=505.352, sum=505.352 (1)\", \"tab\": \"General information\", \"score\": \"505.35168195718654\"}", - "TruthfulQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "TruthfulQA - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MS MARCO (TREC)", - "source_data": { - "dataset_name": "MS MARCO (TREC)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "NDCG@10 on MS MARCO (TREC)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "MS MARCO (regular) - RR@10 (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "MS MARCO (TREC) - NDCG@10 (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "MS MARCO (regular) - RR@10 (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "MS MARCO (TREC) - NDCG@10 (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "MS MARCO (regular) - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MS MARCO (TREC) - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MS MARCO (regular) - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "MS MARCO (TREC) - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CNN/DailyMail", - "source_data": { - "dataset_name": "CNN/DailyMail", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on CNN/DailyMail", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "CNN/DailyMail - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "CNN/DailyMail - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "CNN/DailyMail - SummaC": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - QAFactEval": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - BERTScore (F1)": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - Coverage": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - Density": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - Compression": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-faithfulness": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-relevance": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-coherence": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "XSUM", - "source_data": { - "dataset_name": "XSUM", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on XSUM", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "XSUM - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "XSUM - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "XSUM - SummaC": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - QAFactEval": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - BERTScore (F1)": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - Coverage": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - Density": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - Compression": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-faithfulness": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-relevance": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-coherence": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "IMDB", - "source_data": { - "dataset_name": "IMDB", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on IMDB", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.928, - "details": { - "description": "min=0.928, mean=0.928, max=0.928, sum=0.928 (1)", - "tab": "Accuracy", - "IMDB - ECE (10-bin)": "{\"description\": \"min=0.302, mean=0.302, max=0.302, sum=0.302 (1)\", \"tab\": \"Calibration\", \"score\": \"0.3016994708797646\"}", - "IMDB - EM (Robustness)": "{\"description\": \"min=0.871, mean=0.871, max=0.871, sum=0.871 (1)\", \"tab\": \"Robustness\", \"score\": \"0.871\"}", - "IMDB - EM (Fairness)": "{\"description\": \"min=0.911, mean=0.911, max=0.911, sum=0.911 (1)\", \"tab\": \"Fairness\", \"score\": \"0.911\"}", - "IMDB - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "IMDB - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "IMDB - # train": "{\"description\": \"min=2.911, mean=2.911, max=2.911, sum=2.911 (1)\", \"tab\": \"General information\", \"score\": \"2.911\"}", - "IMDB - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IMDB - # prompt tokens": "{\"description\": \"min=1619.568, mean=1619.568, max=1619.568, sum=1619.568 (1)\", \"tab\": \"General information\", \"score\": \"1619.568\"}", - "IMDB - # output tokens": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "IMDB - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "IMDB - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CivilComments", - "source_data": { - "dataset_name": "CivilComments", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on CivilComments", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.511, - "details": { - "description": "min=0.02, mean=0.511, max=0.988, sum=9.207 (18)", - "tab": "Accuracy", - "CivilComments - ECE (10-bin)": "{\"description\": \"min=0.074, mean=0.259, max=0.508, sum=4.655 (18)\", \"tab\": \"Calibration\", \"score\": \"0.25858613851508827\"}", - "CivilComments - EM (Robustness)": "{\"description\": \"min=0.013, mean=0.363, max=0.915, sum=6.531 (18)\", \"tab\": \"Robustness\", \"score\": \"0.3628308048007681\"}", - "CivilComments - EM (Fairness)": "{\"description\": \"min=0.001, mean=0.333, max=0.927, sum=5.995 (18)\", \"tab\": \"Fairness\", \"score\": \"0.33307716875468274\"}", - "CivilComments - Denoised inference time (s)": "{\"description\": \"9 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "CivilComments - # eval": "{\"description\": \"min=74, mean=371.556, max=683, sum=6688 (18)\", \"tab\": \"General information\", \"score\": \"371.55555555555554\"}", - "CivilComments - # train": "{\"description\": \"min=5, mean=5, max=5, sum=90 (18)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CivilComments - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (18)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CivilComments - # prompt tokens": "{\"description\": \"min=360.976, mean=771.654, max=1282.4, sum=13889.772 (18)\", \"tab\": \"General information\", \"score\": \"771.6539847352628\"}", - "CivilComments - # output tokens": "{\"description\": \"min=5, mean=5, max=5, sum=90 (18)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CivilComments - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=18 (18)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "CivilComments - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Toxic fraction": "{\"description\": \"9 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "RAFT", - "source_data": { - "dataset_name": "RAFT", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on RAFT", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.502, - "details": { - "description": "min=0.175, mean=0.502, max=0.975, sum=5.525 (11)", - "tab": "Accuracy", - "RAFT - ECE (10-bin)": "{\"description\": \"min=0.175, mean=0.502, max=0.975, sum=5.519 (11)\", \"tab\": \"Calibration\", \"score\": \"0.5016937882323235\"}", - "RAFT - EM (Robustness)": "{\"description\": \"min=0.075, mean=0.377, max=0.975, sum=4.15 (11)\", \"tab\": \"Robustness\", \"score\": \"0.3772727272727272\"}", - "RAFT - EM (Fairness)": "{\"description\": \"min=0.175, mean=0.45, max=0.975, sum=4.95 (11)\", \"tab\": \"Fairness\", \"score\": \"0.45\"}", - "RAFT - Denoised inference time (s)": "{\"description\": \"11 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "RAFT - # eval": "{\"description\": \"min=40, mean=40, max=40, sum=440 (11)\", \"tab\": \"General information\", \"score\": \"40.0\"}", - "RAFT - # train": "{\"description\": \"min=0.7, mean=4.605, max=5, sum=50.65 (11)\", \"tab\": \"General information\", \"score\": \"4.6045454545454545\"}", - "RAFT - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (11)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "RAFT - # prompt tokens": "{\"description\": \"min=280.35, mean=869.691, max=1756.575, sum=9566.6 (11)\", \"tab\": \"General information\", \"score\": \"869.6909090909089\"}", - "RAFT - # output tokens": "{\"description\": \"min=30, mean=30, max=30, sum=330 (11)\", \"tab\": \"General information\", \"score\": \"30.0\"}", - "RAFT - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=11 (11)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "RAFT - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (gender)": "{\"description\": \"min=0.5, mean=0.5, max=0.5, sum=0.5 (1)\", \"tab\": \"Bias\", \"score\": \"0.5\"}", - "RAFT - Toxic fraction": "{\"description\": \"11 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "hfopenllm_v2/EleutherAI_pythia-6.9b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2281 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3232 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0144 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2517 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3591 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1147 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/elinas_chronos-gold-12b-1.0.json b/data/models/elinas_chronos-gold-12b-1.0.json deleted file mode 100644 index cfbfd76db6904ff3d805d69bd6817dee80fe7626..0000000000000000000000000000000000000000 --- a/data/models/elinas_chronos-gold-12b-1.0.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Chronos-Gold-12B-1.0", - "id": "elinas/Chronos-Gold-12B-1.0", - "developer": "elinas", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/elinas_Chronos-Gold-12B-1.0/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3166 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5515 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0695 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.318 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.474 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3518 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ell44ot_gemma-2b-def.json b/data/models/ell44ot_gemma-2b-def.json deleted file mode 100644 index 97e691c87d1b20d133eb0ea1d5335b179f67c711..0000000000000000000000000000000000000000 --- a/data/models/ell44ot_gemma-2b-def.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "gemma-2b-def", - "id": "ell44ot/gemma-2b-def", - "developer": "ell44ot", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "GemmaModel", - "params_billions": "1.546" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ell44ot_gemma-2b-def/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2693 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3159 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0242 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2735 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.367 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1572 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/enno-ai_ennoai-pro-french-llama-3-8b-v0.4.json b/data/models/enno-ai_ennoai-pro-french-llama-3-8b-v0.4.json deleted file mode 100644 index 41f3be4d3a33a7d512a715a980595d1915f6daab..0000000000000000000000000000000000000000 --- a/data/models/enno-ai_ennoai-pro-french-llama-3-8b-v0.4.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "EnnoAi-Pro-French-Llama-3-8B-v0.4", - "id": "Enno-Ai/EnnoAi-Pro-French-Llama-3-8B-v0.4", - "developer": "Enno-Ai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.031" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Enno-Ai_EnnoAi-Pro-French-Llama-3-8B-v0.4/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4189 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4075 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0363 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.271 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.417 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2635 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/enno-ai_ennoai-pro-llama-3-8b-v0.3.json b/data/models/enno-ai_ennoai-pro-llama-3-8b-v0.3.json deleted file mode 100644 index 9825c2b83132372ee006958545ecf277ffdab7d4..0000000000000000000000000000000000000000 --- a/data/models/enno-ai_ennoai-pro-llama-3-8b-v0.3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "EnnoAi-Pro-Llama-3-8B-v0.3", - "id": "Enno-Ai/EnnoAi-Pro-Llama-3-8B-v0.3", - "developer": "Enno-Ai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Enno-Ai_EnnoAi-Pro-Llama-3-8B-v0.3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5083 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4101 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0483 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2651 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4236 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.299 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/enno-ai_ennoai-pro-llama-3-8b.json b/data/models/enno-ai_ennoai-pro-llama-3-8b.json deleted file mode 100644 index 4da833a953a36fba7576034292e6e3ca0f3ff755..0000000000000000000000000000000000000000 --- a/data/models/enno-ai_ennoai-pro-llama-3-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "EnnoAi-Pro-Llama-3-8B", - "id": "Enno-Ai/EnnoAi-Pro-Llama-3-8B", - "developer": "Enno-Ai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.031" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Enno-Ai_EnnoAi-Pro-Llama-3-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3195 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4152 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0219 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2617 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4071 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2151 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/enno-ai_ennoai-pro-llama-3.1-8b-v0.9.json b/data/models/enno-ai_ennoai-pro-llama-3.1-8b-v0.9.json deleted file mode 100644 index 1c3ea62bb5d6d6811c129f7d1ed1e33c62ee0288..0000000000000000000000000000000000000000 --- a/data/models/enno-ai_ennoai-pro-llama-3.1-8b-v0.9.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "EnnoAi-Pro-Llama-3.1-8B-v0.9", - "id": "Enno-Ai/EnnoAi-Pro-Llama-3.1-8B-v0.9", - "developer": "Enno-Ai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Enno-Ai_EnnoAi-Pro-Llama-3.1-8B-v0.9/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4689 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.416 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0378 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2659 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3832 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2596 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ennoai_ennoai-7b-french-instruct-202502.json b/data/models/ennoai_ennoai-7b-french-instruct-202502.json deleted file mode 100644 index 0e220f9ccde9e93e0f23dd5bde641b5389e97d71..0000000000000000000000000000000000000000 --- a/data/models/ennoai_ennoai-7b-french-instruct-202502.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "EnnoAi-7B-French-Instruct-202502", - "id": "EnnoAi/EnnoAi-7B-French-Instruct-202502", - "developer": "EnnoAi", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "7.456" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/EnnoAi_EnnoAi-7B-French-Instruct-202502/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5564 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5575 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3724 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2953 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.46 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4013 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ennoai_ennoai-pro-llama-3.1-8b-v1.0.json b/data/models/ennoai_ennoai-pro-llama-3.1-8b-v1.0.json deleted file mode 100644 index 787b2f22694ebf0ee9b9e5d20fbfa82507295a61..0000000000000000000000000000000000000000 --- a/data/models/ennoai_ennoai-pro-llama-3.1-8b-v1.0.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "EnnoAi-Pro-Llama-3.1-8B-v1.0", - "id": "EnnoAi/EnnoAi-Pro-Llama-3.1-8B-v1.0", - "developer": "EnnoAi", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/EnnoAi_EnnoAi-Pro-Llama-3.1-8B-v1.0/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4704 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.416 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0378 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2659 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3832 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2596 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/epiculous_azure_dusk-v0.2.json b/data/models/epiculous_azure_dusk-v0.2.json deleted file mode 100644 index 3d102805d7e249b509a2f2eeaf862abcf95788c0..0000000000000000000000000000000000000000 --- a/data/models/epiculous_azure_dusk-v0.2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Azure_Dusk-v0.2", - "id": "Epiculous/Azure_Dusk-v0.2", - "developer": "Epiculous", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Epiculous_Azure_Dusk-v0.2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3467 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.412 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0295 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2609 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3835 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3034 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/epiculous_crimson_dawn-v0.2.json b/data/models/epiculous_crimson_dawn-v0.2.json deleted file mode 100644 index c491cfd179191ac521cb14e55f5ac7dd0d2881d2..0000000000000000000000000000000000000000 --- a/data/models/epiculous_crimson_dawn-v0.2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Crimson_Dawn-v0.2", - "id": "Epiculous/Crimson_Dawn-v0.2", - "developer": "Epiculous", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Epiculous_Crimson_Dawn-v0.2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3103 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4482 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0431 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.276 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4152 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2721 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/epiculous_novaspark.json b/data/models/epiculous_novaspark.json deleted file mode 100644 index b42dbee20afdfcce41739077696ff7f1ee6cd1ed..0000000000000000000000000000000000000000 --- a/data/models/epiculous_novaspark.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "NovaSpark", - "id": "Epiculous/NovaSpark", - "developer": "Epiculous", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Epiculous_NovaSpark/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6408 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5064 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1518 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2978 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3882 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3649 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/epiculous_violet_twilight-v0.2.json b/data/models/epiculous_violet_twilight-v0.2.json deleted file mode 100644 index da7b3657f6d03bca18a458b94b9643f3a18c4385..0000000000000000000000000000000000000000 --- a/data/models/epiculous_violet_twilight-v0.2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Violet_Twilight-v0.2", - "id": "Epiculous/Violet_Twilight-v0.2", - "developer": "Epiculous", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Epiculous_Violet_Twilight-v0.2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4532 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4615 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0287 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2659 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4299 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3111 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/epistemeai2_athene-codegemma-2-7b-it-alpaca-v1.2.json b/data/models/epistemeai2_athene-codegemma-2-7b-it-alpaca-v1.2.json deleted file mode 100644 index b5986f09285487b0c9ed976f7e176efb45677967..0000000000000000000000000000000000000000 --- a/data/models/epistemeai2_athene-codegemma-2-7b-it-alpaca-v1.2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Athene-codegemma-2-7b-it-alpaca-v1.2", - "id": "EpistemeAI2/Athene-codegemma-2-7b-it-alpaca-v1.2", - "developer": "EpistemeAI2", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "GemmaForCausalLM", - "params_billions": "7.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/EpistemeAI2_Athene-codegemma-2-7b-it-alpaca-v1.2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4351 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4175 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0423 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.271 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.417 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2297 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/epistemeai2_fireball-12b-v1.2.json b/data/models/epistemeai2_fireball-12b-v1.2.json deleted file mode 100644 index 357a6d8310dd22d16ea8821ba5a05a1d02fa97bc..0000000000000000000000000000000000000000 --- a/data/models/epistemeai2_fireball-12b-v1.2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Fireball-12B-v1.2", - "id": "EpistemeAI2/Fireball-12B-v1.2", - "developer": "EpistemeAI2", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/EpistemeAI2_Fireball-12B-v1.2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1355 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5019 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0415 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2987 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4173 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3337 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/epistemeai2_fireball-alpaca-llama3.1-8b-philos.json b/data/models/epistemeai2_fireball-alpaca-llama3.1-8b-philos.json deleted file mode 100644 index 9fe6142f58f7380c841061ed99a74dde7d096862..0000000000000000000000000000000000000000 --- a/data/models/epistemeai2_fireball-alpaca-llama3.1-8b-philos.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Fireball-Alpaca-Llama3.1-8B-Philos", - "id": "EpistemeAI2/Fireball-Alpaca-Llama3.1-8B-Philos", - "developer": "EpistemeAI2", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/EpistemeAI2_Fireball-Alpaca-Llama3.1-8B-Philos/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4986 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4978 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1186 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2928 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4277 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3406 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/epistemeai2_fireball-alpaca-llama3.1.01-8b-philos.json b/data/models/epistemeai2_fireball-alpaca-llama3.1.01-8b-philos.json deleted file mode 100644 index 0d52ce2508bc4c2ebc7d1f507fbca57c6ffbbb21..0000000000000000000000000000000000000000 --- a/data/models/epistemeai2_fireball-alpaca-llama3.1.01-8b-philos.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Fireball-Alpaca-Llama3.1.01-8B-Philos", - "id": "EpistemeAI2/Fireball-Alpaca-Llama3.1.01-8B-Philos", - "developer": "EpistemeAI2", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/EpistemeAI2_Fireball-Alpaca-Llama3.1.01-8B-Philos/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4212 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4956 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.136 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2886 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4371 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3383 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/epistemeai2_fireball-alpaca-llama3.1.03-8b-philos.json b/data/models/epistemeai2_fireball-alpaca-llama3.1.03-8b-philos.json deleted file mode 100644 index c27f641c038aef3e5f1932897c9500719e83db8b..0000000000000000000000000000000000000000 --- a/data/models/epistemeai2_fireball-alpaca-llama3.1.03-8b-philos.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Fireball-Alpaca-Llama3.1.03-8B-Philos", - "id": "EpistemeAI2/Fireball-Alpaca-Llama3.1.03-8B-Philos", - "developer": "EpistemeAI2", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/EpistemeAI2_Fireball-Alpaca-Llama3.1.03-8B-Philos/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3881 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4951 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1284 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2785 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.428 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3355 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/epistemeai2_fireball-alpaca-llama3.1.04-8b-philos.json b/data/models/epistemeai2_fireball-alpaca-llama3.1.04-8b-philos.json deleted file mode 100644 index 5732670de6676cb025bcf9d34843fe40f5c1f5ef..0000000000000000000000000000000000000000 --- a/data/models/epistemeai2_fireball-alpaca-llama3.1.04-8b-philos.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Fireball-Alpaca-Llama3.1.04-8B-Philos", - "id": "EpistemeAI2/Fireball-Alpaca-Llama3.1.04-8B-Philos", - "developer": "EpistemeAI2", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/EpistemeAI2_Fireball-Alpaca-Llama3.1.04-8B-Philos/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4084 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.493 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1201 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2903 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4372 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3403 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/epistemeai2_fireball-alpaca-llama3.1.06-8b-philos-dpo.json b/data/models/epistemeai2_fireball-alpaca-llama3.1.06-8b-philos-dpo.json deleted file mode 100644 index 2c145693a3276946b5cfea8f3e97d451120bab0b..0000000000000000000000000000000000000000 --- a/data/models/epistemeai2_fireball-alpaca-llama3.1.06-8b-philos-dpo.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Fireball-Alpaca-Llama3.1.06-8B-Philos-dpo", - "id": "EpistemeAI2/Fireball-Alpaca-Llama3.1.06-8B-Philos-dpo", - "developer": "EpistemeAI2", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/EpistemeAI2_Fireball-Alpaca-Llama3.1.06-8B-Philos-dpo/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4866 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4881 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1307 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2978 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3932 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3615 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/epistemeai2_fireball-alpaca-llama3.1.07-8b-philos-math.json b/data/models/epistemeai2_fireball-alpaca-llama3.1.07-8b-philos-math.json deleted file mode 100644 index eefd910bcdac655e2986beebb55ce08af581ac83..0000000000000000000000000000000000000000 --- a/data/models/epistemeai2_fireball-alpaca-llama3.1.07-8b-philos-math.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Fireball-Alpaca-Llama3.1.07-8B-Philos-Math", - "id": "EpistemeAI2/Fireball-Alpaca-Llama3.1.07-8B-Philos-Math", - "developer": "EpistemeAI2", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/EpistemeAI2_Fireball-Alpaca-Llama3.1.07-8B-Philos-Math/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5079 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4847 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1201 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2961 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4063 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3531 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/epistemeai2_fireball-alpaca-llama3.1.08-8b-c-r1-kto-reflection.json b/data/models/epistemeai2_fireball-alpaca-llama3.1.08-8b-c-r1-kto-reflection.json deleted file mode 100644 index ba13b8b9e5c01d42b12959f3591d822abfadc769..0000000000000000000000000000000000000000 --- a/data/models/epistemeai2_fireball-alpaca-llama3.1.08-8b-c-r1-kto-reflection.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Fireball-Alpaca-Llama3.1.08-8B-C-R1-KTO-Reflection", - "id": "EpistemeAI2/Fireball-Alpaca-Llama3.1.08-8B-C-R1-KTO-Reflection", - "developer": "EpistemeAI2", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/EpistemeAI2_Fireball-Alpaca-Llama3.1.08-8B-C-R1-KTO-Reflection/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3952 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4955 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1246 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2995 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4048 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3593 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/epistemeai2_fireball-alpaca-llama3.1.08-8b-philos-c-r1.json b/data/models/epistemeai2_fireball-alpaca-llama3.1.08-8b-philos-c-r1.json deleted file mode 100644 index cc84cbfd183398e96cdb3763f400b8919f73b88c..0000000000000000000000000000000000000000 --- a/data/models/epistemeai2_fireball-alpaca-llama3.1.08-8b-philos-c-r1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Fireball-Alpaca-Llama3.1.08-8B-Philos-C-R1", - "id": "EpistemeAI2/Fireball-Alpaca-Llama3.1.08-8B-Philos-C-R1", - "developer": "EpistemeAI2", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/EpistemeAI2_Fireball-Alpaca-Llama3.1.08-8B-Philos-C-R1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5316 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4828 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1239 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.297 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4103 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3523 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/epistemeai2_fireball-llama-3.1-8b-philos-reflection.json b/data/models/epistemeai2_fireball-llama-3.1-8b-philos-reflection.json deleted file mode 100644 index 14a7b79e3e9ea85e6f0f655715898567dcdba45d..0000000000000000000000000000000000000000 --- a/data/models/epistemeai2_fireball-llama-3.1-8b-philos-reflection.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Fireball-Llama-3.1-8B-Philos-Reflection", - "id": "EpistemeAI2/Fireball-Llama-3.1-8B-Philos-Reflection", - "developer": "EpistemeAI2", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/EpistemeAI2_Fireball-Llama-3.1-8B-Philos-Reflection/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3596 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4898 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1284 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3079 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3957 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3551 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/epistemeai2_fireball-mathmistral-nemo-base-2407-v2dpo.json b/data/models/epistemeai2_fireball-mathmistral-nemo-base-2407-v2dpo.json deleted file mode 100644 index 8a57a2e1dff56e314faa9800902f59e9ed59acc8..0000000000000000000000000000000000000000 --- a/data/models/epistemeai2_fireball-mathmistral-nemo-base-2407-v2dpo.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Fireball-MathMistral-Nemo-Base-2407-v2dpo", - "id": "EpistemeAI2/Fireball-MathMistral-Nemo-Base-2407-v2dpo", - "developer": "EpistemeAI2", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "11.58" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/EpistemeAI2_Fireball-MathMistral-Nemo-Base-2407-v2dpo/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3097 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4328 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.037 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2634 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.403 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1148 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/epistemeai2_fireball-meta-llama-3.1-8b-instruct-agent-0.003-128k-code-math.json b/data/models/epistemeai2_fireball-meta-llama-3.1-8b-instruct-agent-0.003-128k-code-math.json deleted file mode 100644 index f175b038385c2f39d4db7a98ec1d24ff546d49b0..0000000000000000000000000000000000000000 --- a/data/models/epistemeai2_fireball-meta-llama-3.1-8b-instruct-agent-0.003-128k-code-math.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Fireball-Meta-Llama-3.1-8B-Instruct-Agent-0.003-128K-code-math", - "id": "EpistemeAI2/Fireball-Meta-Llama-3.1-8B-Instruct-Agent-0.003-128K-code-math", - "developer": "EpistemeAI2", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/EpistemeAI2_Fireball-Meta-Llama-3.1-8B-Instruct-Agent-0.003-128K-code-math/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5515 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4808 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1352 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3045 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3693 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.342 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/epistemeai2_fireball-meta-llama-3.1-8b-instruct-agent-0.005-128k-code-cot.json b/data/models/epistemeai2_fireball-meta-llama-3.1-8b-instruct-agent-0.005-128k-code-cot.json deleted file mode 100644 index 10d878900cb117b551208fe5421ef707f1118893..0000000000000000000000000000000000000000 --- a/data/models/epistemeai2_fireball-meta-llama-3.1-8b-instruct-agent-0.005-128k-code-cot.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Fireball-Meta-Llama-3.1-8B-Instruct-Agent-0.005-128K-code-COT", - "id": "EpistemeAI2/Fireball-Meta-Llama-3.1-8B-Instruct-Agent-0.005-128K-code-COT", - "developer": "EpistemeAI2", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/EpistemeAI2_Fireball-Meta-Llama-3.1-8B-Instruct-Agent-0.005-128K-code-COT/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4633 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4791 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1171 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3121 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3774 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3565 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/epistemeai2_fireball-phi-3-medium-4k-inst-philos.json b/data/models/epistemeai2_fireball-phi-3-medium-4k-inst-philos.json deleted file mode 100644 index 3bdf1378716563e19eaf859b908481e9ae37c18d..0000000000000000000000000000000000000000 --- a/data/models/epistemeai2_fireball-phi-3-medium-4k-inst-philos.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Fireball-Phi-3-medium-4k-inst-Philos", - "id": "EpistemeAI2/Fireball-Phi-3-medium-4k-inst-Philos", - "developer": "EpistemeAI2", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "13.96" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/EpistemeAI2_Fireball-Phi-3-medium-4k-inst-Philos/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5313 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6178 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1707 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3322 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4139 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4599 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/epistemeai_alpaca-llama3.1-8b.json b/data/models/epistemeai_alpaca-llama3.1-8b.json deleted file mode 100644 index 478bab38f333371b3fcf412017020392934d8e96..0000000000000000000000000000000000000000 --- a/data/models/epistemeai_alpaca-llama3.1-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Alpaca-Llama3.1-8B", - "id": "EpistemeAI/Alpaca-Llama3.1-8B", - "developer": "EpistemeAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/EpistemeAI_Alpaca-Llama3.1-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1599 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4755 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0506 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2903 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3403 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3246 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/epistemeai_athena-gemma-2-2b-it-philos.json b/data/models/epistemeai_athena-gemma-2-2b-it-philos.json deleted file mode 100644 index de02b579aeb0922ec13b174a15526005c6fdf913..0000000000000000000000000000000000000000 --- a/data/models/epistemeai_athena-gemma-2-2b-it-philos.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Athena-gemma-2-2b-it-Philos", - "id": "EpistemeAI/Athena-gemma-2-2b-it-Philos", - "developer": "EpistemeAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "2.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/EpistemeAI_Athena-gemma-2-2b-it-Philos/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4621 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3795 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.037 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.281 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4314 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2248 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/epistemeai_athena-gemma-2-2b-it.json b/data/models/epistemeai_athena-gemma-2-2b-it.json deleted file mode 100644 index 053a9e9d48cb8ce00585450530db373ea6764d18..0000000000000000000000000000000000000000 --- a/data/models/epistemeai_athena-gemma-2-2b-it.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Athena-gemma-2-2b-it", - "id": "EpistemeAI/Athena-gemma-2-2b-it", - "developer": "EpistemeAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "2.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/EpistemeAI_Athena-gemma-2-2b-it/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3134 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4264 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0491 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2685 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4351 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2422 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/epistemeai_athene-codegemma-2-7b-it-alpaca-v1.3.json b/data/models/epistemeai_athene-codegemma-2-7b-it-alpaca-v1.3.json deleted file mode 100644 index 25795f8a7431c0e705a63d94604b51898c43ae2d..0000000000000000000000000000000000000000 --- a/data/models/epistemeai_athene-codegemma-2-7b-it-alpaca-v1.3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Athene-codegemma-2-7b-it-alpaca-v1.3", - "id": "EpistemeAI/Athene-codegemma-2-7b-it-alpaca-v1.3", - "developer": "EpistemeAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "GemmaForCausalLM", - "params_billions": "7.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/EpistemeAI_Athene-codegemma-2-7b-it-alpaca-v1.3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.403 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4332 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0619 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2802 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4503 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2587 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/epistemeai_deepphi-3.5-mini-instruct.json b/data/models/epistemeai_deepphi-3.5-mini-instruct.json deleted file mode 100644 index 3fa07eb9410a5e5ae1dd58da2a55c6748d0ffaf1..0000000000000000000000000000000000000000 --- a/data/models/epistemeai_deepphi-3.5-mini-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "DeepPhi-3.5-mini-instruct", - "id": "EpistemeAI/DeepPhi-3.5-mini-instruct", - "developer": "EpistemeAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.821" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/EpistemeAI_DeepPhi-3.5-mini-instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1326 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2882 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0068 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2332 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3656 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1103 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/epistemeai_deepthinkers-phi4.json b/data/models/epistemeai_deepthinkers-phi4.json deleted file mode 100644 index 0c571151e21e8ae5b0f4421b3a52151ae8482cba..0000000000000000000000000000000000000000 --- a/data/models/epistemeai_deepthinkers-phi4.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "DeepThinkers-Phi4", - "id": "EpistemeAI/DeepThinkers-Phi4", - "developer": "EpistemeAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/EpistemeAI_DeepThinkers-Phi4/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.694 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.679 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4585 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3406 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3981 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5258 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/epistemeai_finellama3.1-8b-instruct.json b/data/models/epistemeai_finellama3.1-8b-instruct.json deleted file mode 100644 index 09b2873abc4ca0fb46d0056ba3255817cc4d35b2..0000000000000000000000000000000000000000 --- a/data/models/epistemeai_finellama3.1-8b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "FineLlama3.1-8B-Instruct", - "id": "EpistemeAI/FineLlama3.1-8B-Instruct", - "developer": "EpistemeAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "4bit", - "architecture": "?", - "params_billions": "14.483" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/EpistemeAI_FineLlama3.1-8B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.08 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4557 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0347 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2802 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3482 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3113 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/epistemeai_fireball-12b-v1.13a-philosophers.json b/data/models/epistemeai_fireball-12b-v1.13a-philosophers.json deleted file mode 100644 index e0a710061bcdd30e66f55afab54328e99fff9d58..0000000000000000000000000000000000000000 --- a/data/models/epistemeai_fireball-12b-v1.13a-philosophers.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Fireball-12B-v1.13a-philosophers", - "id": "EpistemeAI/Fireball-12B-v1.13a-philosophers", - "developer": "EpistemeAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/EpistemeAI_Fireball-12B-v1.13a-philosophers/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0876 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5103 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0461 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3012 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4081 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3367 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/epistemeai_fireball-12b.json b/data/models/epistemeai_fireball-12b.json deleted file mode 100644 index 8c33546e99dcc06817303a443598e51c59741450..0000000000000000000000000000000000000000 --- a/data/models/epistemeai_fireball-12b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Fireball-12B", - "id": "EpistemeAI/Fireball-12B", - "developer": "EpistemeAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/EpistemeAI_Fireball-12B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1834 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5111 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0408 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2617 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4236 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3344 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/epistemeai_fireball-alpaca-llama-3.1-8b-philos-dpo-200.json b/data/models/epistemeai_fireball-alpaca-llama-3.1-8b-philos-dpo-200.json deleted file mode 100644 index 049c63ace54fd3dec801ad9554497a77924e7e09..0000000000000000000000000000000000000000 --- a/data/models/epistemeai_fireball-alpaca-llama-3.1-8b-philos-dpo-200.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Fireball-Alpaca-Llama-3.1-8B-Philos-DPO-200", - "id": "EpistemeAI/Fireball-Alpaca-Llama-3.1-8B-Philos-DPO-200", - "developer": "EpistemeAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/EpistemeAI_Fireball-Alpaca-Llama-3.1-8B-Philos-DPO-200/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4577 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4838 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1231 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3003 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3945 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3583 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/epistemeai_fireball-alpaca-llama3.1.07-8b-philos-math-kto-beta.json b/data/models/epistemeai_fireball-alpaca-llama3.1.07-8b-philos-math-kto-beta.json deleted file mode 100644 index 307051451481a6327706476d33d2d9b1502cb65c..0000000000000000000000000000000000000000 --- a/data/models/epistemeai_fireball-alpaca-llama3.1.07-8b-philos-math-kto-beta.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Fireball-Alpaca-Llama3.1.07-8B-Philos-Math-KTO-beta", - "id": "EpistemeAI/Fireball-Alpaca-Llama3.1.07-8B-Philos-Math-KTO-beta", - "developer": "EpistemeAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/EpistemeAI_Fireball-Alpaca-Llama3.1.07-8B-Philos-Math-KTO-beta/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7274 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4865 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1526 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2802 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3619 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3543 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/epistemeai_fireball-alpaca-llama3.1.08-8b-philos-c-r2.json b/data/models/epistemeai_fireball-alpaca-llama3.1.08-8b-philos-c-r2.json deleted file mode 100644 index 7b03a6f8c32bd1c04c4ed4437e159b8466cc5906..0000000000000000000000000000000000000000 --- a/data/models/epistemeai_fireball-alpaca-llama3.1.08-8b-philos-c-r2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Fireball-Alpaca-Llama3.1.08-8B-Philos-C-R2", - "id": "EpistemeAI/Fireball-Alpaca-Llama3.1.08-8B-Philos-C-R2", - "developer": "EpistemeAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/EpistemeAI_Fireball-Alpaca-Llama3.1.08-8B-Philos-C-R2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4673 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4932 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1239 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2861 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4624 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3352 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/epistemeai_fireball-meta-llama-3.1-8b-instruct-0.001-128k-auto.json b/data/models/epistemeai_fireball-meta-llama-3.1-8b-instruct-0.001-128k-auto.json deleted file mode 100644 index 191a39a1cbc9525ac024e69ab5974fff6679054e..0000000000000000000000000000000000000000 --- a/data/models/epistemeai_fireball-meta-llama-3.1-8b-instruct-0.001-128k-auto.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Fireball-Meta-Llama-3.1-8B-Instruct-0.001-128K-auto", - "id": "EpistemeAI/Fireball-Meta-Llama-3.1-8B-Instruct-0.001-128K-auto", - "developer": "EpistemeAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/EpistemeAI_Fireball-Meta-Llama-3.1-8B-Instruct-0.001-128K-auto/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4432 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4824 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1329 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3121 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4066 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3516 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/epistemeai_fireball-meta-llama-3.1-8b-instruct-agent-0.003-128k-code-ds-auto.json b/data/models/epistemeai_fireball-meta-llama-3.1-8b-instruct-agent-0.003-128k-code-ds-auto.json deleted file mode 100644 index 34df651aabdc1d7685c9419a18517c0c9ee2b5a2..0000000000000000000000000000000000000000 --- a/data/models/epistemeai_fireball-meta-llama-3.1-8b-instruct-agent-0.003-128k-code-ds-auto.json +++ /dev/null @@ -1,275 +0,0 @@ -{ - "model_info": { - "name": "Fireball-Meta-Llama-3.1-8B-Instruct-Agent-0.003-128K-code-ds-auto", - "id": "EpistemeAI/Fireball-Meta-Llama-3.1-8B-Instruct-Agent-0.003-128K-code-ds-auto", - "developer": "EpistemeAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/EpistemeAI_Fireball-Meta-Llama-3.1-8B-Instruct-Agent-0.003-128K-code-ds-auto/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7305 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4649 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1397 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2659 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3209 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.348 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "hfopenllm_v2/EpistemeAI_Fireball-Meta-Llama-3.1-8B-Instruct-Agent-0.003-128K-code-ds-auto/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7207 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.461 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1314 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2701 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3432 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3354 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/epistemeai_fireball-meta-llama-3.1-8b-instruct-agent-0.003-128k-code-ds.json b/data/models/epistemeai_fireball-meta-llama-3.1-8b-instruct-agent-0.003-128k-code-ds.json deleted file mode 100644 index 3ee7ad86f225ac4ca85cb46d8447964fed1b0d45..0000000000000000000000000000000000000000 --- a/data/models/epistemeai_fireball-meta-llama-3.1-8b-instruct-agent-0.003-128k-code-ds.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Fireball-Meta-Llama-3.1-8B-Instruct-Agent-0.003-128K-code-ds", - "id": "EpistemeAI/Fireball-Meta-Llama-3.1-8B-Instruct-Agent-0.003-128K-code-ds", - "developer": "EpistemeAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/EpistemeAI_Fireball-Meta-Llama-3.1-8B-Instruct-Agent-0.003-128K-code-ds/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6691 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4668 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1337 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2727 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3418 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3389 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/epistemeai_fireball-meta-llama-3.1-8b-instruct-agent-0.003-128k-code.json b/data/models/epistemeai_fireball-meta-llama-3.1-8b-instruct-agent-0.003-128k-code.json deleted file mode 100644 index f96e56c07ede0faa46bf68c4fedd80ccae4a0bcf..0000000000000000000000000000000000000000 --- a/data/models/epistemeai_fireball-meta-llama-3.1-8b-instruct-agent-0.003-128k-code.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Fireball-Meta-Llama-3.1-8B-Instruct-Agent-0.003-128K-code", - "id": "EpistemeAI/Fireball-Meta-Llama-3.1-8B-Instruct-Agent-0.003-128K-code", - "developer": "EpistemeAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/EpistemeAI_Fireball-Meta-Llama-3.1-8B-Instruct-Agent-0.003-128K-code/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5975 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4904 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1337 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.302 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.401 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3423 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/epistemeai_fireball-meta-llama-3.1-8b-instruct-agent-0.003-128k.json b/data/models/epistemeai_fireball-meta-llama-3.1-8b-instruct-agent-0.003-128k.json deleted file mode 100644 index 741b044a1185785e556b53ce62ebdd22ffeb2a5f..0000000000000000000000000000000000000000 --- a/data/models/epistemeai_fireball-meta-llama-3.1-8b-instruct-agent-0.003-128k.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Fireball-Meta-Llama-3.1-8B-Instruct-Agent-0.003-128K", - "id": "EpistemeAI/Fireball-Meta-Llama-3.1-8B-Instruct-Agent-0.003-128K", - "developer": "EpistemeAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/EpistemeAI_Fireball-Meta-Llama-3.1-8B-Instruct-Agent-0.003-128K/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4457 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4897 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1208 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2945 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3762 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3543 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/epistemeai_fireball-meta-llama-3.1-8b-instruct-agent-0.004-128k-code-cot.json b/data/models/epistemeai_fireball-meta-llama-3.1-8b-instruct-agent-0.004-128k-code-cot.json deleted file mode 100644 index 4dc34677970388fd944ee627f6726f2c23a92218..0000000000000000000000000000000000000000 --- a/data/models/epistemeai_fireball-meta-llama-3.1-8b-instruct-agent-0.004-128k-code-cot.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Fireball-Meta-Llama-3.1-8B-Instruct-Agent-0.004-128K-code-COT", - "id": "EpistemeAI/Fireball-Meta-Llama-3.1-8B-Instruct-Agent-0.004-128K-code-COT", - "developer": "EpistemeAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/EpistemeAI_Fireball-Meta-Llama-3.1-8B-Instruct-Agent-0.004-128K-code-COT/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4578 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4761 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1382 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2936 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3881 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3471 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/epistemeai_fireball-meta-llama-3.1-8b-instruct-agent-0.004-128k-code-ds-auto.json b/data/models/epistemeai_fireball-meta-llama-3.1-8b-instruct-agent-0.004-128k-code-ds-auto.json deleted file mode 100644 index fd1d296fb0aad54120f8a9de0fbd39988eff0acd..0000000000000000000000000000000000000000 --- a/data/models/epistemeai_fireball-meta-llama-3.1-8b-instruct-agent-0.004-128k-code-ds-auto.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Fireball-Meta-Llama-3.1-8B-Instruct-Agent-0.004-128K-code-ds-auto", - "id": "EpistemeAI/Fireball-Meta-Llama-3.1-8B-Instruct-Agent-0.004-128K-code-ds-auto", - "developer": "EpistemeAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/EpistemeAI_Fireball-Meta-Llama-3.1-8B-Instruct-Agent-0.004-128K-code-ds-auto/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7205 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4818 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1435 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2483 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.33 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3548 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/epistemeai_fireball-meta-llama-3.1-8b-instruct-math.json b/data/models/epistemeai_fireball-meta-llama-3.1-8b-instruct-math.json deleted file mode 100644 index 2d1cded100dbf95b2ccabf359c7c7048f33c9b4b..0000000000000000000000000000000000000000 --- a/data/models/epistemeai_fireball-meta-llama-3.1-8b-instruct-math.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Fireball-Meta-Llama-3.1-8B-Instruct-Math", - "id": "EpistemeAI/Fireball-Meta-Llama-3.1-8B-Instruct-Math", - "developer": "EpistemeAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/EpistemeAI_Fireball-Meta-Llama-3.1-8B-Instruct-Math/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4623 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4983 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.108 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2911 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3641 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3331 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/epistemeai_fireball-meta-llama-3.2-8b-instruct-agent-003-128k-code-dpo.json b/data/models/epistemeai_fireball-meta-llama-3.2-8b-instruct-agent-003-128k-code-dpo.json deleted file mode 100644 index a8fc2fbaa13d42f440c9322d17c5b99df045dcab..0000000000000000000000000000000000000000 --- a/data/models/epistemeai_fireball-meta-llama-3.2-8b-instruct-agent-003-128k-code-dpo.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Fireball-Meta-Llama-3.2-8B-Instruct-agent-003-128k-code-DPO", - "id": "EpistemeAI/Fireball-Meta-Llama-3.2-8B-Instruct-agent-003-128k-code-DPO", - "developer": "EpistemeAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/EpistemeAI_Fireball-Meta-Llama-3.2-8B-Instruct-agent-003-128k-code-DPO/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4611 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4801 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1254 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3003 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3998 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3521 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/epistemeai_fireball-mistral-nemo-base-2407-v1-dpo2.json b/data/models/epistemeai_fireball-mistral-nemo-base-2407-v1-dpo2.json deleted file mode 100644 index 22b5bdc72a43e4dbe61270ee9909a09ea2dfbca0..0000000000000000000000000000000000000000 --- a/data/models/epistemeai_fireball-mistral-nemo-base-2407-v1-dpo2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Fireball-Mistral-Nemo-Base-2407-v1-DPO2", - "id": "EpistemeAI/Fireball-Mistral-Nemo-Base-2407-v1-DPO2", - "developer": "EpistemeAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/EpistemeAI_Fireball-Mistral-Nemo-Base-2407-v1-DPO2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1861 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4968 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0363 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2919 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.404 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3353 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/epistemeai_fireball-r1-llama-3.1-8b-medical-cot.json b/data/models/epistemeai_fireball-r1-llama-3.1-8b-medical-cot.json deleted file mode 100644 index 12700c69a410aacb55ac055ca95ac53174797373..0000000000000000000000000000000000000000 --- a/data/models/epistemeai_fireball-r1-llama-3.1-8b-medical-cot.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Fireball-R1-Llama-3.1-8B-Medical-COT", - "id": "EpistemeAI/Fireball-R1-Llama-3.1-8B-Medical-COT", - "developer": "EpistemeAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/EpistemeAI_Fireball-R1-Llama-3.1-8B-Medical-COT/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3216 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3716 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.327 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2743 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3114 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1402 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/epistemeai_fireball-r1-llama-3.1-8b.json b/data/models/epistemeai_fireball-r1-llama-3.1-8b.json deleted file mode 100644 index b7306a99675a7b40f91febcd023f15ee4d3fd85d..0000000000000000000000000000000000000000 --- a/data/models/epistemeai_fireball-r1-llama-3.1-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Fireball-R1-Llama-3.1-8B", - "id": "EpistemeAI/Fireball-R1-Llama-3.1-8B", - "developer": "EpistemeAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/EpistemeAI_Fireball-R1-Llama-3.1-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4427 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3643 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3112 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2483 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3288 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1115 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/epistemeai_fireball-r1.1-llama-3.1-8b.json b/data/models/epistemeai_fireball-r1.1-llama-3.1-8b.json deleted file mode 100644 index 2b30791f2210d0467d25ce2f1e1d8c2178972ea8..0000000000000000000000000000000000000000 --- a/data/models/epistemeai_fireball-r1.1-llama-3.1-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Fireball-R1.1-Llama-3.1-8B", - "id": "EpistemeAI/Fireball-R1.1-Llama-3.1-8B", - "developer": "EpistemeAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/EpistemeAI_Fireball-R1.1-Llama-3.1-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3676 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3326 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1382 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2517 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3419 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1115 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/epistemeai_llama-3.2-3b-agent007-coder.json b/data/models/epistemeai_llama-3.2-3b-agent007-coder.json deleted file mode 100644 index cda1d53684699ad590e00946385ae39e159ae0cf..0000000000000000000000000000000000000000 --- a/data/models/epistemeai_llama-3.2-3b-agent007-coder.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.2-3B-Agent007-Coder", - "id": "EpistemeAI/Llama-3.2-3B-Agent007-Coder", - "developer": "EpistemeAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/EpistemeAI_Llama-3.2-3B-Agent007-Coder/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.54 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4304 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.111 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2576 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3668 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2852 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/epistemeai_mistral-nemo-instruct-12b-philosophy-math.json b/data/models/epistemeai_mistral-nemo-instruct-12b-philosophy-math.json deleted file mode 100644 index 6c1c675a85b3f35af2c5bc6c5f59f8fd5afa3c53..0000000000000000000000000000000000000000 --- a/data/models/epistemeai_mistral-nemo-instruct-12b-philosophy-math.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mistral-Nemo-Instruct-12B-Philosophy-Math", - "id": "EpistemeAI/Mistral-Nemo-Instruct-12B-Philosophy-Math", - "developer": "EpistemeAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/EpistemeAI_Mistral-Nemo-Instruct-12B-Philosophy-Math/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0695 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5365 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0959 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3314 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4292 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3296 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/epistemeai_openreasoner-llama-3.2-3b-rs1.0.json b/data/models/epistemeai_openreasoner-llama-3.2-3b-rs1.0.json deleted file mode 100644 index 0e92d47b81393bbc9583e64bdb9847e2e8057ebe..0000000000000000000000000000000000000000 --- a/data/models/epistemeai_openreasoner-llama-3.2-3b-rs1.0.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "OpenReasoner-Llama-3.2-3B-rs1.0", - "id": "EpistemeAI/OpenReasoner-Llama-3.2-3B-rs1.0", - "developer": "EpistemeAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/EpistemeAI_OpenReasoner-Llama-3.2-3B-rs1.0/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7274 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4519 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1344 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2718 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3461 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3134 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/epistemeai_polypsyche-llama-3.1-8b-instruct-agent-0.003-128k-code-ds-auto-divergent.json b/data/models/epistemeai_polypsyche-llama-3.1-8b-instruct-agent-0.003-128k-code-ds-auto-divergent.json deleted file mode 100644 index da83ef143810cd6a5f2d7a7e1a85e402d974d8e2..0000000000000000000000000000000000000000 --- a/data/models/epistemeai_polypsyche-llama-3.1-8b-instruct-agent-0.003-128k-code-ds-auto-divergent.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Polypsyche-Llama-3.1-8B-Instruct-Agent-0.003-128K-code-ds-auto-divergent", - "id": "EpistemeAI/Polypsyche-Llama-3.1-8B-Instruct-Agent-0.003-128K-code-ds-auto-divergent", - "developer": "EpistemeAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/EpistemeAI_Polypsyche-Llama-3.1-8B-Instruct-Agent-0.003-128K-code-ds-auto-divergent/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6915 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4525 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1292 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2668 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3578 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.329 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/epistemeai_polypsyche-llama-3.1-8b-instruct-agent-0.003-128k-code-ds-auto-empathy.json b/data/models/epistemeai_polypsyche-llama-3.1-8b-instruct-agent-0.003-128k-code-ds-auto-empathy.json deleted file mode 100644 index e8e1239c0146c17a86f35442846b80ef05f825f5..0000000000000000000000000000000000000000 --- a/data/models/epistemeai_polypsyche-llama-3.1-8b-instruct-agent-0.003-128k-code-ds-auto-empathy.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Polypsyche-Llama-3.1-8B-Instruct-Agent-0.003-128K-code-ds-auto-Empathy", - "id": "EpistemeAI/Polypsyche-Llama-3.1-8B-Instruct-Agent-0.003-128K-code-ds-auto-Empathy", - "developer": "EpistemeAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/EpistemeAI_Polypsyche-Llama-3.1-8B-Instruct-Agent-0.003-128K-code-ds-auto-Empathy/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7101 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4628 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1397 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2768 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3195 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3311 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/epistemeai_polypsyche-llama-3.1-8b-instruct-agent-0.003-128k-code-ds-auto-logic.json b/data/models/epistemeai_polypsyche-llama-3.1-8b-instruct-agent-0.003-128k-code-ds-auto-logic.json deleted file mode 100644 index 251532a1d42a6638938d4ec23114c665b0013e93..0000000000000000000000000000000000000000 --- a/data/models/epistemeai_polypsyche-llama-3.1-8b-instruct-agent-0.003-128k-code-ds-auto-logic.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Polypsyche-Llama-3.1-8B-Instruct-Agent-0.003-128K-code-ds-auto-Logic", - "id": "EpistemeAI/Polypsyche-Llama-3.1-8B-Instruct-Agent-0.003-128K-code-ds-auto-Logic", - "developer": "EpistemeAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/EpistemeAI_Polypsyche-Llama-3.1-8B-Instruct-Agent-0.003-128K-code-ds-auto-Logic/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7122 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4566 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1246 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2844 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3235 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.335 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/epistemeai_reasoning-llama-3.1-cot-re1-nmt-v2-orpo.json b/data/models/epistemeai_reasoning-llama-3.1-cot-re1-nmt-v2-orpo.json deleted file mode 100644 index 3619e6e9bb8438b06346fa172daa01f3d3cf4427..0000000000000000000000000000000000000000 --- a/data/models/epistemeai_reasoning-llama-3.1-cot-re1-nmt-v2-orpo.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Reasoning-Llama-3.1-CoT-RE1-NMT-V2-ORPO", - "id": "EpistemeAI/Reasoning-Llama-3.1-CoT-RE1-NMT-V2-ORPO", - "developer": "EpistemeAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/EpistemeAI_Reasoning-Llama-3.1-CoT-RE1-NMT-V2-ORPO/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4553 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4804 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1292 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.307 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3931 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3598 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/epistemeai_reasoning-llama-3.1-cot-re1-nmt.json b/data/models/epistemeai_reasoning-llama-3.1-cot-re1-nmt.json deleted file mode 100644 index b663fc8a6f4bb1fcf17355506da087df0f2718a1..0000000000000000000000000000000000000000 --- a/data/models/epistemeai_reasoning-llama-3.1-cot-re1-nmt.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Reasoning-Llama-3.1-CoT-RE1-NMT", - "id": "EpistemeAI/Reasoning-Llama-3.1-CoT-RE1-NMT", - "developer": "EpistemeAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/EpistemeAI_Reasoning-Llama-3.1-CoT-RE1-NMT/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4829 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4736 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1299 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2609 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3182 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3343 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/epistemeai_reasoning-llama-3.2-1b-instruct-v1.2.json b/data/models/epistemeai_reasoning-llama-3.2-1b-instruct-v1.2.json deleted file mode 100644 index 9ba5a93a6dacf52ab55492aa68e9a0a96b0c3766..0000000000000000000000000000000000000000 --- a/data/models/epistemeai_reasoning-llama-3.2-1b-instruct-v1.2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Reasoning-Llama-3.2-1B-Instruct-v1.2", - "id": "EpistemeAI/Reasoning-Llama-3.2-1B-Instruct-v1.2", - "developer": "EpistemeAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.236" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/EpistemeAI_Reasoning-Llama-3.2-1B-Instruct-v1.2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4087 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3324 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0506 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2609 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3222 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1179 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/epistemeai_reasoning-llama-3.2-1b-instruct-v1.3.json b/data/models/epistemeai_reasoning-llama-3.2-1b-instruct-v1.3.json deleted file mode 100644 index a7ba2600d8e87e2f37ae6cb8ec2195a4e87b17a3..0000000000000000000000000000000000000000 --- a/data/models/epistemeai_reasoning-llama-3.2-1b-instruct-v1.3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Reasoning-Llama-3.2-1B-Instruct-v1.3", - "id": "EpistemeAI/Reasoning-Llama-3.2-1B-Instruct-v1.3", - "developer": "EpistemeAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.236" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/EpistemeAI_Reasoning-Llama-3.2-1B-Instruct-v1.3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3273 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3263 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0506 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2584 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.326 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1173 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/epistemeai_reasoning-llama-3.2-3b-math-instruct-re1-orpo.json b/data/models/epistemeai_reasoning-llama-3.2-3b-math-instruct-re1-orpo.json deleted file mode 100644 index 8db81a1459b3ac4e5b32cf7903c8d732d749e1f1..0000000000000000000000000000000000000000 --- a/data/models/epistemeai_reasoning-llama-3.2-3b-math-instruct-re1-orpo.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Reasoning-Llama-3.2-3B-Math-Instruct-RE1-ORPO", - "id": "EpistemeAI/Reasoning-Llama-3.2-3B-Math-Instruct-RE1-ORPO", - "developer": "EpistemeAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/EpistemeAI_Reasoning-Llama-3.2-3B-Math-Instruct-RE1-ORPO/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.729 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4518 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1533 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2735 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3487 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.31 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/epistemeai_reasoning-llama-3.2-3b-math-instruct-re1.json b/data/models/epistemeai_reasoning-llama-3.2-3b-math-instruct-re1.json deleted file mode 100644 index 1db2146a44cba722d7b99306286af7aa5614c835..0000000000000000000000000000000000000000 --- a/data/models/epistemeai_reasoning-llama-3.2-3b-math-instruct-re1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Reasoning-Llama-3.2-3B-Math-Instruct-RE1", - "id": "EpistemeAI/Reasoning-Llama-3.2-3B-Math-Instruct-RE1", - "developer": "EpistemeAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/EpistemeAI_Reasoning-Llama-3.2-3B-Math-Instruct-RE1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.512 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4381 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.108 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2643 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3435 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2789 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/epistemeai_reasoningcore-1.0-3b-instruct-r01-reflect-math.json b/data/models/epistemeai_reasoningcore-1.0-3b-instruct-r01-reflect-math.json deleted file mode 100644 index d0134aad111c74b085a4c6ab2d5c4630e61498b8..0000000000000000000000000000000000000000 --- a/data/models/epistemeai_reasoningcore-1.0-3b-instruct-r01-reflect-math.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ReasoningCore-1.0-3B-Instruct-r01-Reflect-Math", - "id": "EpistemeAI/ReasoningCore-1.0-3B-Instruct-r01-Reflect-Math", - "developer": "EpistemeAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/EpistemeAI_ReasoningCore-1.0-3B-Instruct-r01-Reflect-Math/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5903 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4364 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.148 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2601 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3314 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2823 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/epistemeai_reasoningcore-3b-0.json b/data/models/epistemeai_reasoningcore-3b-0.json deleted file mode 100644 index ea91e0e2b514bd292733783cfa42fa39a23820c3..0000000000000000000000000000000000000000 --- a/data/models/epistemeai_reasoningcore-3b-0.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ReasoningCore-3B-0", - "id": "EpistemeAI/ReasoningCore-3B-0", - "developer": "EpistemeAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/EpistemeAI_ReasoningCore-3B-0/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7341 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4446 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1586 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2727 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3554 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3172 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/epistemeai_reasoningcore-3b-instruct-r01-reflect.json b/data/models/epistemeai_reasoningcore-3b-instruct-r01-reflect.json deleted file mode 100644 index 796ffcf6f4b65ad158a139f02edb00cfd55f3f34..0000000000000000000000000000000000000000 --- a/data/models/epistemeai_reasoningcore-3b-instruct-r01-reflect.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ReasoningCore-3B-Instruct-r01-Reflect", - "id": "EpistemeAI/ReasoningCore-3B-Instruct-r01-Reflect", - "developer": "EpistemeAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/EpistemeAI_ReasoningCore-3B-Instruct-r01-Reflect/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7335 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.445 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1541 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2735 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3527 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3144 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/epistemeai_reasoningcore-3b-r01.json b/data/models/epistemeai_reasoningcore-3b-r01.json deleted file mode 100644 index 0be55db9ff96b68fc64f0ee94e5b0a3f2f97131d..0000000000000000000000000000000000000000 --- a/data/models/epistemeai_reasoningcore-3b-r01.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ReasoningCore-3B-R01", - "id": "EpistemeAI/ReasoningCore-3B-R01", - "developer": "EpistemeAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/EpistemeAI_ReasoningCore-3B-R01/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2976 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4373 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1299 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2609 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3195 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2591 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/epistemeai_reasoningcore-3b-re1-v2.json b/data/models/epistemeai_reasoningcore-3b-re1-v2.json deleted file mode 100644 index 2e99f0907c7ab0767ebf7a5175aec619acf69511..0000000000000000000000000000000000000000 --- a/data/models/epistemeai_reasoningcore-3b-re1-v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ReasoningCore-3B-RE1-V2", - "id": "EpistemeAI/ReasoningCore-3B-RE1-V2", - "developer": "EpistemeAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/EpistemeAI_ReasoningCore-3B-RE1-V2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7393 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4462 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1563 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2735 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3541 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3181 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/epistemeai_reasoningcore-3b-re1-v2a.json b/data/models/epistemeai_reasoningcore-3b-re1-v2a.json deleted file mode 100644 index 8ea63000618419007053b773c855e17c0c434b5b..0000000000000000000000000000000000000000 --- a/data/models/epistemeai_reasoningcore-3b-re1-v2a.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ReasoningCore-3B-RE1-V2A", - "id": "EpistemeAI/ReasoningCore-3B-RE1-V2A", - "developer": "EpistemeAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/EpistemeAI_ReasoningCore-3B-RE1-V2A/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5733 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.419 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0929 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2777 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3352 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2736 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/epistemeai_reasoningcore-3b-re1-v2b.json b/data/models/epistemeai_reasoningcore-3b-re1-v2b.json deleted file mode 100644 index e93b5823a49413a4a7acad6c4528bbef968fd216..0000000000000000000000000000000000000000 --- a/data/models/epistemeai_reasoningcore-3b-re1-v2b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ReasoningCore-3B-RE1-V2B", - "id": "EpistemeAI/ReasoningCore-3B-RE1-V2B", - "developer": "EpistemeAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/EpistemeAI_ReasoningCore-3B-RE1-V2B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5051 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4168 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1073 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2617 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3448 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2673 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/epistemeai_reasoningcore-3b-re1-v2c.json b/data/models/epistemeai_reasoningcore-3b-re1-v2c.json deleted file mode 100644 index 2b93c6379ce7e8cd0c91d9d411f8197c6489794b..0000000000000000000000000000000000000000 --- a/data/models/epistemeai_reasoningcore-3b-re1-v2c.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ReasoningCore-3B-RE1-V2C", - "id": "EpistemeAI/ReasoningCore-3B-RE1-V2C", - "developer": "EpistemeAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/EpistemeAI_ReasoningCore-3B-RE1-V2C/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5057 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4177 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0974 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2609 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3422 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2691 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/epistemeai_reasoningcore-3b-t1-v1.json b/data/models/epistemeai_reasoningcore-3b-t1-v1.json deleted file mode 100644 index b8ed5510ea6257a11b47281adfebe489904cdde2..0000000000000000000000000000000000000000 --- a/data/models/epistemeai_reasoningcore-3b-t1-v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ReasoningCore-3B-T1-V1", - "id": "EpistemeAI/ReasoningCore-3B-T1-V1", - "developer": "EpistemeAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/EpistemeAI_ReasoningCore-3B-T1-V1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7208 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4517 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1458 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.276 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.354 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.312 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/epistemeai_reasoningcore-3b-t1_1.json b/data/models/epistemeai_reasoningcore-3b-t1_1.json deleted file mode 100644 index 4166dfba847bccb904b6f4b55d2c57b1f5ac1de4..0000000000000000000000000000000000000000 --- a/data/models/epistemeai_reasoningcore-3b-t1_1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ReasoningCore-3B-T1_1", - "id": "EpistemeAI/ReasoningCore-3B-T1_1", - "developer": "EpistemeAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/EpistemeAI_ReasoningCore-3B-T1_1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7275 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4524 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1541 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.276 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3554 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3117 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/eric111_catunamayo-dpo.json b/data/models/eric111_catunamayo-dpo.json deleted file mode 100644 index ee51b7924cfe6fc7b771e4ae2cd05e52cc53d9aa..0000000000000000000000000000000000000000 --- a/data/models/eric111_catunamayo-dpo.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "CatunaMayo-DPO", - "id": "Eric111/CatunaMayo-DPO", - "developer": "Eric111", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Eric111_CatunaMayo-DPO/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4215 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5224 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0816 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2919 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.445 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.317 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/eric111_catunamayo.json b/data/models/eric111_catunamayo.json deleted file mode 100644 index b80e376c0c1ef50c580a61cbdc5b7932e22be152..0000000000000000000000000000000000000000 --- a/data/models/eric111_catunamayo.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "CatunaMayo", - "id": "Eric111/CatunaMayo", - "developer": "Eric111", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Eric111_CatunaMayo/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4074 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5244 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0846 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2919 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.454 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3178 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/etherll_chocolatine-3b-instruct-dpo-revised-ties-v2.json b/data/models/etherll_chocolatine-3b-instruct-dpo-revised-ties-v2.json deleted file mode 100644 index 4ce63a149eac4868490830b99109f9188af6ea3e..0000000000000000000000000000000000000000 --- a/data/models/etherll_chocolatine-3b-instruct-dpo-revised-ties-v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Chocolatine-3B-Instruct-DPO-Revised-Ties-v2", - "id": "Etherll/Chocolatine-3B-Instruct-DPO-Revised-Ties-v2", - "developer": "Etherll", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Phi3ForCausalLM", - "params_billions": "3.821" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Etherll_Chocolatine-3B-Instruct-DPO-Revised-Ties-v2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.374 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5411 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1631 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3238 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4649 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3978 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/etherll_chocolatine-3b-instruct-dpo-revised-ties.json b/data/models/etherll_chocolatine-3b-instruct-dpo-revised-ties.json deleted file mode 100644 index dd76ee5866f2c46f11b96d6b962d5e5944b9d6f5..0000000000000000000000000000000000000000 --- a/data/models/etherll_chocolatine-3b-instruct-dpo-revised-ties.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Chocolatine-3B-Instruct-DPO-Revised-Ties", - "id": "Etherll/Chocolatine-3B-Instruct-DPO-Revised-Ties", - "developer": "Etherll", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Phi3ForCausalLM", - "params_billions": "3.821" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Etherll_Chocolatine-3B-Instruct-DPO-Revised-Ties/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3725 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5411 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1631 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3238 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4649 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3978 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/etherll_herplete-llm-llama-3.1-8b-ties.json b/data/models/etherll_herplete-llm-llama-3.1-8b-ties.json deleted file mode 100644 index 9be49ecf608ce43a03e635a92240e0cb88e2dc2d..0000000000000000000000000000000000000000 --- a/data/models/etherll_herplete-llm-llama-3.1-8b-ties.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Herplete-LLM-Llama-3.1-8b-Ties", - "id": "Etherll/Herplete-LLM-Llama-3.1-8b-Ties", - "developer": "Etherll", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Etherll_Herplete-LLM-Llama-3.1-8b-Ties/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6164 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5338 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1601 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3171 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4017 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3752 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/etherll_herplete-llm-llama-3.1-8b.json b/data/models/etherll_herplete-llm-llama-3.1-8b.json deleted file mode 100644 index 31e2291931d31d5d1deeb312e00cc5de025980fd..0000000000000000000000000000000000000000 --- a/data/models/etherll_herplete-llm-llama-3.1-8b.json +++ /dev/null @@ -1,275 +0,0 @@ -{ - "model_info": { - "name": "Herplete-LLM-Llama-3.1-8b", - "id": "Etherll/Herplete-LLM-Llama-3.1-8b", - "developer": "Etherll", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Etherll_Herplete-LLM-Llama-3.1-8b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6106 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5347 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1548 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3146 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3991 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3752 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "hfopenllm_v2/Etherll_Herplete-LLM-Llama-3.1-8b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4672 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5013 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0279 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2861 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.386 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3482 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/etherll_qwen2.5-7b-della-test.json b/data/models/etherll_qwen2.5-7b-della-test.json deleted file mode 100644 index 2d4d49c0298580809e38a780e981a1e1b23f78f0..0000000000000000000000000000000000000000 --- a/data/models/etherll_qwen2.5-7b-della-test.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-7B-della-test", - "id": "Etherll/Qwen2.5-7B-della-test", - "developer": "Etherll", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Etherll_Qwen2.5-7B-della-test/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7625 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5447 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4894 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3087 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4047 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4361 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/etherll_qwen2.5-coder-7b-instruct-ties.json b/data/models/etherll_qwen2.5-coder-7b-instruct-ties.json deleted file mode 100644 index 467f121add201c0eeb0ccf70a75186daaf81f570..0000000000000000000000000000000000000000 --- a/data/models/etherll_qwen2.5-coder-7b-instruct-ties.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-Coder-7B-Instruct-Ties", - "id": "Etherll/Qwen2.5-Coder-7B-Instruct-Ties", - "developer": "Etherll", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Etherll_Qwen2.5-Coder-7B-Instruct-Ties/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5005 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4895 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2915 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3297 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4373 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3503 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/etherll_replete-llm-v3-llama-3.1-8b.json b/data/models/etherll_replete-llm-v3-llama-3.1-8b.json deleted file mode 100644 index f9ad273374808b664cee323000e4753000a5a6ba..0000000000000000000000000000000000000000 --- a/data/models/etherll_replete-llm-v3-llama-3.1-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Replete-LLM-V3-Llama-3.1-8b", - "id": "Etherll/Replete-LLM-V3-Llama-3.1-8b", - "developer": "Etherll", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Etherll_Replete-LLM-V3-Llama-3.1-8b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5263 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4543 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2273 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2685 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3516 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.347 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/etherll_superhermes.json b/data/models/etherll_superhermes.json deleted file mode 100644 index 08f04e4f7ca47c4e75f0c2b40d047a3cdf69a05f..0000000000000000000000000000000000000000 --- a/data/models/etherll_superhermes.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SuperHermes", - "id": "Etherll/SuperHermes", - "developer": "Etherll", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Etherll_SuperHermes/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5459 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.529 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1654 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3238 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.44 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3949 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/euclaise_remask-3b.json b/data/models/euclaise_remask-3b.json deleted file mode 100644 index ed660bfa48373a777a4853cf74bd2469b1a681d8..0000000000000000000000000000000000000000 --- a/data/models/euclaise_remask-3b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ReMask-3B", - "id": "euclaise/ReMask-3B", - "developer": "euclaise", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "StableLmForCausalLM", - "params_billions": "2.795" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/euclaise_ReMask-3B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2419 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3517 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0196 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2668 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3341 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1357 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/eurdem_defne-llama3.1-8b.json b/data/models/eurdem_defne-llama3.1-8b.json deleted file mode 100644 index 4794f02e3619462a8d5681a828c81ff97f1919c0..0000000000000000000000000000000000000000 --- a/data/models/eurdem_defne-llama3.1-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Defne-llama3.1-8B", - "id": "Eurdem/Defne-llama3.1-8B", - "developer": "Eurdem", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Eurdem_Defne-llama3.1-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5036 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5321 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1601 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2961 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4331 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3866 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/eva-unit-01_eva-qwen2.5-14b-v0.2.json b/data/models/eva-unit-01_eva-qwen2.5-14b-v0.2.json deleted file mode 100644 index 0ce6915ab53aa3f6c1487eb6b897c8229e9389b0..0000000000000000000000000000000000000000 --- a/data/models/eva-unit-01_eva-qwen2.5-14b-v0.2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "EVA-Qwen2.5-14B-v0.2", - "id": "EVA-UNIT-01/EVA-Qwen2.5-14B-v0.2", - "developer": "EVA-UNIT-01", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/EVA-UNIT-01_EVA-Qwen2.5-14B-v0.2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4038 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.609 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3406 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3943 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4794 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5135 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/eva-unit-01_eva-qwen2.5-72b-v0.2.json b/data/models/eva-unit-01_eva-qwen2.5-72b-v0.2.json deleted file mode 100644 index 5fca11323799f458161d7291a0572b23c1cb8d37..0000000000000000000000000000000000000000 --- a/data/models/eva-unit-01_eva-qwen2.5-72b-v0.2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "EVA-Qwen2.5-72B-v0.2", - "id": "EVA-UNIT-01/EVA-Qwen2.5-72B-v0.2", - "developer": "EVA-UNIT-01", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "72.706" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/EVA-UNIT-01_EVA-Qwen2.5-72B-v0.2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6879 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7088 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4313 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4086 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.472 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5813 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/eworojoshua_vas-01.json b/data/models/eworojoshua_vas-01.json deleted file mode 100644 index 3b8b861c36b678c4e7349ac1c606212d45596670..0000000000000000000000000000000000000000 --- a/data/models/eworojoshua_vas-01.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "vas-01", - "id": "eworojoshua/vas-01", - "developer": "eworojoshua", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/eworojoshua_vas-01/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7612 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5418 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4736 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3096 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4432 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4348 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ewre324_ewre324-r1-smollm2-135m-distill.json b/data/models/ewre324_ewre324-r1-smollm2-135m-distill.json deleted file mode 100644 index ce5603911a81652fcc14d60e2f0db5fa6feeebb9..0000000000000000000000000000000000000000 --- a/data/models/ewre324_ewre324-r1-smollm2-135m-distill.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ewre324-R1-SmolLM2-135M-Distill", - "id": "ewre324/ewre324-R1-SmolLM2-135M-Distill", - "developer": "ewre324", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "0.135" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ewre324_ewre324-R1-SmolLM2-135M-Distill/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1649 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3042 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0128 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2617 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3409 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1134 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ewre324_thinker-llama-3.2-3b-instruct-reasoning.json b/data/models/ewre324_thinker-llama-3.2-3b-instruct-reasoning.json deleted file mode 100644 index 058cf374d82f172e614314997c72d4d375940a15..0000000000000000000000000000000000000000 --- a/data/models/ewre324_thinker-llama-3.2-3b-instruct-reasoning.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Thinker-Llama-3.2-3B-Instruct-Reasoning", - "id": "ewre324/Thinker-Llama-3.2-3B-Instruct-Reasoning", - "developer": "ewre324", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ewre324_Thinker-Llama-3.2-3B-Instruct-Reasoning/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4439 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4273 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0846 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2768 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3655 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2886 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ewre324_thinker-qwen2.5-0.5b-instruct-reasoning.json b/data/models/ewre324_thinker-qwen2.5-0.5b-instruct-reasoning.json deleted file mode 100644 index b48fa3e904d319d4a9d71d662487bfea3bb9f7c8..0000000000000000000000000000000000000000 --- a/data/models/ewre324_thinker-qwen2.5-0.5b-instruct-reasoning.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Thinker-Qwen2.5-0.5B-Instruct-Reasoning", - "id": "ewre324/Thinker-Qwen2.5-0.5B-Instruct-Reasoning", - "developer": "ewre324", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ewre324_Thinker-Qwen2.5-0.5B-Instruct-Reasoning/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2476 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3292 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0287 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2852 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3382 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1647 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ewre324_thinker-smollm2-135m-instruct-reasoning.json b/data/models/ewre324_thinker-smollm2-135m-instruct-reasoning.json deleted file mode 100644 index c859c295942a97cd9d13dd9db10132d1ae264f98..0000000000000000000000000000000000000000 --- a/data/models/ewre324_thinker-smollm2-135m-instruct-reasoning.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Thinker-SmolLM2-135M-Instruct-Reasoning", - "id": "ewre324/Thinker-SmolLM2-135M-Instruct-Reasoning", - "developer": "ewre324", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "0.135" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ewre324_Thinker-SmolLM2-135M-Instruct-Reasoning/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2584 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3071 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0091 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2525 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3661 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1094 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/experiment-llm_exp-3-q-r.json b/data/models/experiment-llm_exp-3-q-r.json deleted file mode 100644 index dac759524547cd6e9b812181d427192f8ce85725..0000000000000000000000000000000000000000 --- a/data/models/experiment-llm_exp-3-q-r.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "exp-3-q-r", - "id": "experiment-llm/exp-3-q-r", - "developer": "experiment-llm", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/experiment-llm_exp-3-q-r/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6036 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5397 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2787 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2936 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4315 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4316 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/facebook_opt-1.3b.json b/data/models/facebook_opt-1.3b.json deleted file mode 100644 index 11f1cc2be5138034e1298afd5c5bd6bdde2a2341..0000000000000000000000000000000000000000 --- a/data/models/facebook_opt-1.3b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "opt-1.3b", - "id": "facebook/opt-1.3b", - "developer": "facebook", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "OPTForCausalLM", - "params_billions": "1.3" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/facebook_opt-1.3b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2383 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3094 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0091 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2424 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.342 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1107 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/facebook_opt-30b.json b/data/models/facebook_opt-30b.json deleted file mode 100644 index 4d3e4a689938a8b741c01bf7123389f986d753cf..0000000000000000000000000000000000000000 --- a/data/models/facebook_opt-30b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "opt-30b", - "id": "facebook/opt-30b", - "developer": "facebook", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "OPTForCausalLM", - "params_billions": "30.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/facebook_opt-30b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2453 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.307 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0106 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2693 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3604 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1164 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/facebook_self-taught-evaluator-llama3.1-70b.json b/data/models/facebook_self-taught-evaluator-llama3.1-70b.json deleted file mode 100644 index 51267e53b2bd3a9a677fc2cbfc24a01e01e0bb31..0000000000000000000000000000000000000000 --- a/data/models/facebook_self-taught-evaluator-llama3.1-70b.json +++ /dev/null @@ -1,126 +0,0 @@ -{ - "model_info": { - "name": "facebook/Self-taught-evaluator-llama3.1-70B", - "id": "facebook/Self-taught-evaluator-llama3.1-70B", - "developer": "facebook", - "additional_details": { - "model_type": "Generative" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/facebook_Self-taught-evaluator-llama3.1-70B/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9001 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9693 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8509 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8959 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8844 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/facebook_self-taught-llama-3-70b.json b/data/models/facebook_self-taught-llama-3-70b.json deleted file mode 100644 index c737262cb701d9e5ec16aa2373738fa7731ee65d..0000000000000000000000000000000000000000 --- a/data/models/facebook_self-taught-llama-3-70b.json +++ /dev/null @@ -1,126 +0,0 @@ -{ - "model_info": { - "name": "facebook/Self-taught-Llama-3-70B", - "id": "facebook/Self-taught-Llama-3-70B", - "developer": "facebook", - "additional_details": { - "model_type": "Generative" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/facebook_Self-taught-Llama-3-70B/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8863 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9693 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8399 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9108 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8251 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/failspy_llama-3-70b-instruct-abliterated.json b/data/models/failspy_llama-3-70b-instruct-abliterated.json deleted file mode 100644 index f399d1f282004924dab8ea5551bc9d852bc2e461..0000000000000000000000000000000000000000 --- a/data/models/failspy_llama-3-70b-instruct-abliterated.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "llama-3-70B-Instruct-abliterated", - "id": "failspy/llama-3-70B-Instruct-abliterated", - "developer": "failspy", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "70.554" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/failspy_llama-3-70B-Instruct-abliterated/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8023 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6465 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2432 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2894 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4128 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5145 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/failspy_llama-3-8b-instruct-abliterated.json b/data/models/failspy_llama-3-8b-instruct-abliterated.json deleted file mode 100644 index 789cd72d2fa8cc3ee4f1a1a175f8a4b98aa05ce4..0000000000000000000000000000000000000000 --- a/data/models/failspy_llama-3-8b-instruct-abliterated.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-8B-Instruct-abliterated", - "id": "failspy/Llama-3-8B-Instruct-abliterated", - "developer": "failspy", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/failspy_Llama-3-8B-Instruct-abliterated/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5909 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4354 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0385 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.276 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4116 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2742 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/failspy_llama-3-8b-instruct-mopeymule.json b/data/models/failspy_llama-3-8b-instruct-mopeymule.json deleted file mode 100644 index e1445ae46172fefa3499df9783bea9b5419ab76f..0000000000000000000000000000000000000000 --- a/data/models/failspy_llama-3-8b-instruct-mopeymule.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-8B-Instruct-MopeyMule", - "id": "failspy/Llama-3-8B-Instruct-MopeyMule", - "developer": "failspy", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/failspy_Llama-3-8B-Instruct-MopeyMule/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.675 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3839 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0196 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2391 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3513 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1764 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/failspy_meta-llama-3-70b-instruct-abliterated-v3.5.json b/data/models/failspy_meta-llama-3-70b-instruct-abliterated-v3.5.json deleted file mode 100644 index 717d5e031710b41e707507da397e08fcecaa9f15..0000000000000000000000000000000000000000 --- a/data/models/failspy_meta-llama-3-70b-instruct-abliterated-v3.5.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Meta-Llama-3-70B-Instruct-abliterated-v3.5", - "id": "failspy/Meta-Llama-3-70B-Instruct-abliterated-v3.5", - "developer": "failspy", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "70.554" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/failspy_Meta-Llama-3-70B-Instruct-abliterated-v3.5/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7747 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5747 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1284 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.297 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3982 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4452 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/failspy_meta-llama-3-8b-instruct-abliterated-v3.json b/data/models/failspy_meta-llama-3-8b-instruct-abliterated-v3.json deleted file mode 100644 index 135e88efc472e7d9b783912a4b744815e9300274..0000000000000000000000000000000000000000 --- a/data/models/failspy_meta-llama-3-8b-instruct-abliterated-v3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Meta-Llama-3-8B-Instruct-abliterated-v3", - "id": "failspy/Meta-Llama-3-8B-Instruct-abliterated-v3", - "developer": "failspy", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/failspy_Meta-Llama-3-8B-Instruct-abliterated-v3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7245 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4925 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0959 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2643 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3622 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3654 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/failspy_phi-3-medium-4k-instruct-abliterated-v3.json b/data/models/failspy_phi-3-medium-4k-instruct-abliterated-v3.json deleted file mode 100644 index 91afd89f29f5b9c307edd46c7844933aadb9efb9..0000000000000000000000000000000000000000 --- a/data/models/failspy_phi-3-medium-4k-instruct-abliterated-v3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Phi-3-medium-4k-instruct-abliterated-v3", - "id": "failspy/Phi-3-medium-4k-instruct-abliterated-v3", - "developer": "failspy", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Phi3ForCausalLM", - "params_billions": "13.96" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/failspy_Phi-3-medium-4k-instruct-abliterated-v3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6319 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6305 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1594 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3171 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4604 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.44 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/fallenmerick_chewy-lemon-cookie-11b.json b/data/models/fallenmerick_chewy-lemon-cookie-11b.json deleted file mode 100644 index a5baa3786397a9a6d898e6bcc42e4f76528a29d9..0000000000000000000000000000000000000000 --- a/data/models/fallenmerick_chewy-lemon-cookie-11b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Chewy-Lemon-Cookie-11B", - "id": "FallenMerick/Chewy-Lemon-Cookie-11B", - "developer": "FallenMerick", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "10.732" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/FallenMerick_Chewy-Lemon-Cookie-11B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4875 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5251 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0544 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2794 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4546 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3267 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/fblgit_cybertron-v4-qw7b-mgs.json b/data/models/fblgit_cybertron-v4-qw7b-mgs.json deleted file mode 100644 index 2a59aeb1887f751917194a63604a57dc015eb82b..0000000000000000000000000000000000000000 --- a/data/models/fblgit_cybertron-v4-qw7b-mgs.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "cybertron-v4-qw7B-MGS", - "id": "fblgit/cybertron-v4-qw7B-MGS", - "developer": "fblgit", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/fblgit_cybertron-v4-qw7B-MGS/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6264 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5592 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3489 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3104 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4371 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4473 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/fblgit_cybertron-v4-qw7b-unamgs.json b/data/models/fblgit_cybertron-v4-qw7b-unamgs.json deleted file mode 100644 index 7e09dcb281da86876058dcd21ed40f773896c19d..0000000000000000000000000000000000000000 --- a/data/models/fblgit_cybertron-v4-qw7b-unamgs.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "cybertron-v4-qw7B-UNAMGS", - "id": "fblgit/cybertron-v4-qw7B-UNAMGS", - "developer": "fblgit", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/fblgit_cybertron-v4-qw7B-UNAMGS/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.609 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5643 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3731 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3314 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4343 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.45 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/fblgit_juanako-7b-una.json b/data/models/fblgit_juanako-7b-una.json deleted file mode 100644 index 54d48aabd21b7023a4d2088f98d3f2161af86574..0000000000000000000000000000000000000000 --- a/data/models/fblgit_juanako-7b-una.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "juanako-7b-UNA", - "id": "fblgit/juanako-7b-UNA", - "developer": "fblgit", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/fblgit_juanako-7b-UNA/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4837 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.507 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.034 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2961 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4645 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2771 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/fblgit_miniclaus-qw1.5b-unamgs-grpo.json b/data/models/fblgit_miniclaus-qw1.5b-unamgs-grpo.json deleted file mode 100644 index e01d2969958c14d2c97b3b8a24b5a1a96d4e750e..0000000000000000000000000000000000000000 --- a/data/models/fblgit_miniclaus-qw1.5b-unamgs-grpo.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "miniclaus-qw1.5B-UNAMGS-GRPO", - "id": "fblgit/miniclaus-qw1.5B-UNAMGS-GRPO", - "developer": "fblgit", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.544" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/fblgit_miniclaus-qw1.5B-UNAMGS-GRPO/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3518 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4234 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1103 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2978 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4254 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2945 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/fblgit_miniclaus-qw1.5b-unamgs.json b/data/models/fblgit_miniclaus-qw1.5b-unamgs.json deleted file mode 100644 index d8170881d1e36c40d4bc7f739e96ecbf2980e75c..0000000000000000000000000000000000000000 --- a/data/models/fblgit_miniclaus-qw1.5b-unamgs.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "miniclaus-qw1.5B-UNAMGS", - "id": "fblgit/miniclaus-qw1.5B-UNAMGS", - "developer": "fblgit", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.777" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/fblgit_miniclaus-qw1.5B-UNAMGS/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3348 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4239 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1088 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2919 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4293 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2937 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/fblgit_pancho-v1-qw25-3b-unamgs.json b/data/models/fblgit_pancho-v1-qw25-3b-unamgs.json deleted file mode 100644 index e0d1139bd63860b40664556fae4c851778a3b6bc..0000000000000000000000000000000000000000 --- a/data/models/fblgit_pancho-v1-qw25-3b-unamgs.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "pancho-v1-qw25-3B-UNAMGS", - "id": "fblgit/pancho-v1-qw25-3B-UNAMGS", - "developer": "fblgit", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.397" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/fblgit_pancho-v1-qw25-3B-UNAMGS/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5361 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4926 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1571 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.297 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4027 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3766 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/fblgit_thebeagle-v2beta-32b-mgs.json b/data/models/fblgit_thebeagle-v2beta-32b-mgs.json deleted file mode 100644 index 48527656a7b3e346612bd66367de25eb0d8e37d9..0000000000000000000000000000000000000000 --- a/data/models/fblgit_thebeagle-v2beta-32b-mgs.json +++ /dev/null @@ -1,275 +0,0 @@ -{ - "model_info": { - "name": "TheBeagle-v2beta-32B-MGS", - "id": "fblgit/TheBeagle-v2beta-32B-MGS", - "developer": "fblgit", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "32.764" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/fblgit_TheBeagle-v2beta-32B-MGS/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4503 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7035 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3943 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.401 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5021 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5911 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "hfopenllm_v2/fblgit_TheBeagle-v2beta-32B-MGS/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5181 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7033 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4947 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3826 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5008 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5915 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/fblgit_una-cybertron-7b-v2-bf16.json b/data/models/fblgit_una-cybertron-7b-v2-bf16.json deleted file mode 100644 index 074195b1d50452c74008288a2b9253880f2d4ac9..0000000000000000000000000000000000000000 --- a/data/models/fblgit_una-cybertron-7b-v2-bf16.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "una-cybertron-7b-v2-bf16", - "id": "fblgit/una-cybertron-7b-v2-bf16", - "developer": "fblgit", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/fblgit_una-cybertron-7b-v2-bf16/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4737 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3973 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0408 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2978 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4473 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2443 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/fblgit_una-simplesmaug-34b-v1beta.json b/data/models/fblgit_una-simplesmaug-34b-v1beta.json deleted file mode 100644 index aa5d01d085478373f92eaf739ba898c0c82b157c..0000000000000000000000000000000000000000 --- a/data/models/fblgit_una-simplesmaug-34b-v1beta.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "UNA-SimpleSmaug-34b-v1beta", - "id": "fblgit/UNA-SimpleSmaug-34b-v1beta", - "developer": "fblgit", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "34.389" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/fblgit_UNA-SimpleSmaug-34b-v1beta/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4556 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5287 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0718 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3171 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4256 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.454 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/fblgit_una-thebeagle-7b-v1.json b/data/models/fblgit_una-thebeagle-7b-v1.json deleted file mode 100644 index 898f8b124aef7c8bfa362551ab93562a8526a89e..0000000000000000000000000000000000000000 --- a/data/models/fblgit_una-thebeagle-7b-v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "UNA-TheBeagle-7b-v1", - "id": "fblgit/UNA-TheBeagle-7b-v1", - "developer": "fblgit", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/fblgit_UNA-TheBeagle-7b-v1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3689 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5029 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.077 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2844 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4564 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3019 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/fblgit_una-thepitbull-21.4b-v2.json b/data/models/fblgit_una-thepitbull-21.4b-v2.json deleted file mode 100644 index 697623de15d35e72ae9e7fb4dbaccfd9075e4712..0000000000000000000000000000000000000000 --- a/data/models/fblgit_una-thepitbull-21.4b-v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "UNA-ThePitbull-21.4B-v2", - "id": "fblgit/UNA-ThePitbull-21.4B-v2", - "developer": "fblgit", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "21.421" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/fblgit_UNA-ThePitbull-21.4B-v2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.379 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.635 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1216 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.302 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3922 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3516 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/felladrin_llama-160m-chat-v1.json b/data/models/felladrin_llama-160m-chat-v1.json deleted file mode 100644 index 386d05ca615433db297c2d22641853f7ac0bee31..0000000000000000000000000000000000000000 --- a/data/models/felladrin_llama-160m-chat-v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-160M-Chat-v1", - "id": "Felladrin/Llama-160M-Chat-v1", - "developer": "Felladrin", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "0.162" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Felladrin_Llama-160M-Chat-v1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1575 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3036 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.006 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2576 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3661 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1136 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/felladrin_minueza-32m-ultrachat.json b/data/models/felladrin_minueza-32m-ultrachat.json deleted file mode 100644 index 605d710c6a25b76953ab9b7cb4e179cb8361d8f8..0000000000000000000000000000000000000000 --- a/data/models/felladrin_minueza-32m-ultrachat.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Minueza-32M-UltraChat", - "id": "Felladrin/Minueza-32M-UltraChat", - "developer": "Felladrin", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "0.033" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Felladrin_Minueza-32M-UltraChat/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1376 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2941 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0045 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2559 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3742 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1133 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/fhai50032_roleplaylake-7b.json b/data/models/fhai50032_roleplaylake-7b.json deleted file mode 100644 index 0c383d5aa119efdc9646844eec676bc6d88a336c..0000000000000000000000000000000000000000 --- a/data/models/fhai50032_roleplaylake-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "RolePlayLake-7B", - "id": "fhai50032/RolePlayLake-7B", - "developer": "fhai50032", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/fhai50032_RolePlayLake-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5057 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5252 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0725 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3037 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4459 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.316 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/fhai50032_unaligned-thinker-phi-4.json b/data/models/fhai50032_unaligned-thinker-phi-4.json deleted file mode 100644 index 1e3491feb6bfcf94c37632e20369e25e7a8976c5..0000000000000000000000000000000000000000 --- a/data/models/fhai50032_unaligned-thinker-phi-4.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Unaligned-Thinker-PHI-4", - "id": "fhai50032/Unaligned-Thinker-PHI-4", - "developer": "fhai50032", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/fhai50032_Unaligned-Thinker-PHI-4/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0563 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6643 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3353 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3809 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4679 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5147 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/fingu-ai_chocolatine-fusion-14b.json b/data/models/fingu-ai_chocolatine-fusion-14b.json deleted file mode 100644 index b16822ed34c09c217630cd72f8cf2f9c8aee3b9a..0000000000000000000000000000000000000000 --- a/data/models/fingu-ai_chocolatine-fusion-14b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Chocolatine-Fusion-14B", - "id": "FINGU-AI/Chocolatine-Fusion-14B", - "developer": "FINGU-AI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "8.367" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/FINGU-AI_Chocolatine-Fusion-14B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6949 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6413 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3852 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3716 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.494 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5262 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/fingu-ai_l3-8b.json b/data/models/fingu-ai_l3-8b.json deleted file mode 100644 index 3c1f585019eb78af74bb88761465db6f14e7bb74..0000000000000000000000000000000000000000 --- a/data/models/fingu-ai_l3-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "L3-8B", - "id": "FINGU-AI/L3-8B", - "developer": "FINGU-AI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/FINGU-AI_L3-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7517 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4986 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2545 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2953 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3828 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3639 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/fingu-ai_phi-4-rrstock.json b/data/models/fingu-ai_phi-4-rrstock.json deleted file mode 100644 index f37c048f0e364c8b74f9407ce0b7d28d08d52a1b..0000000000000000000000000000000000000000 --- a/data/models/fingu-ai_phi-4-rrstock.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Phi-4-RRStock", - "id": "FINGU-AI/Phi-4-RRStock", - "developer": "FINGU-AI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "6.652" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/FINGU-AI_Phi-4-RRStock/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2855 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6443 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0582 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.38 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4479 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4883 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/fingu-ai_q-small-3b.json b/data/models/fingu-ai_q-small-3b.json deleted file mode 100644 index cc5e6faa88faeb4a021c675648c1c040a6a26ed3..0000000000000000000000000000000000000000 --- a/data/models/fingu-ai_q-small-3b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Q-Small-3B", - "id": "FINGU-AI/Q-Small-3B", - "developer": "FINGU-AI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.086" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/FINGU-AI_Q-Small-3B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4145 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4319 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0831 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2668 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4005 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.279 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/fingu-ai_qwq-buddy-32b-alpha.json b/data/models/fingu-ai_qwq-buddy-32b-alpha.json deleted file mode 100644 index cb1374ca64604c9b308f418fc23ab9c29bb5eda1..0000000000000000000000000000000000000000 --- a/data/models/fingu-ai_qwq-buddy-32b-alpha.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "QwQ-Buddy-32B-Alpha", - "id": "FINGU-AI/QwQ-Buddy-32B-Alpha", - "developer": "FINGU-AI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "19.662" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/FINGU-AI_QwQ-Buddy-32B-Alpha/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3446 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6424 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3852 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3792 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.506 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5294 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/fingu-ai_romboultima-32b.json b/data/models/fingu-ai_romboultima-32b.json deleted file mode 100644 index 11e3d31f44711d901a278c6600b7d8a3c558b1a3..0000000000000000000000000000000000000000 --- a/data/models/fingu-ai_romboultima-32b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "RomboUltima-32B", - "id": "FINGU-AI/RomboUltima-32B", - "developer": "FINGU-AI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "17.645" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/FINGU-AI_RomboUltima-32B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6672 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6938 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5385 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3716 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4836 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5789 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/fingu-ai_ultimos-32b.json b/data/models/fingu-ai_ultimos-32b.json deleted file mode 100644 index d4e28ac36fb1ddf489c822754456c4fd060666a2..0000000000000000000000000000000000000000 --- a/data/models/fingu-ai_ultimos-32b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Ultimos-32B", - "id": "FINGU-AI/Ultimos-32B", - "developer": "FINGU-AI", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "9.604" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/FINGU-AI_Ultimos-32B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1592 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2906 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2492 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3286 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1111 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/flammenai_flammen15-gutenberg-dpo-v1-7b.json b/data/models/flammenai_flammen15-gutenberg-dpo-v1-7b.json deleted file mode 100644 index 14481c9be00e897e96863e865e02347809ff9811..0000000000000000000000000000000000000000 --- a/data/models/flammenai_flammen15-gutenberg-dpo-v1-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "flammen15-gutenberg-DPO-v1-7B", - "id": "flammenai/flammen15-gutenberg-DPO-v1-7B", - "developer": "flammenai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/flammenai_flammen15-gutenberg-DPO-v1-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4798 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5203 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0763 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2844 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4293 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3186 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/flammenai_llama3.1-flammades-70b.json b/data/models/flammenai_llama3.1-flammades-70b.json deleted file mode 100644 index 75eb81fe60b93ac6541b5bdf3040c66afd5e20eb..0000000000000000000000000000000000000000 --- a/data/models/flammenai_llama3.1-flammades-70b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama3.1-Flammades-70B", - "id": "flammenai/Llama3.1-Flammades-70B", - "developer": "flammenai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "70.554" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/flammenai_Llama3.1-Flammades-70B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7058 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.666 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2092 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.354 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4871 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4752 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/flammenai_mahou-1.2a-llama3-8b.json b/data/models/flammenai_mahou-1.2a-llama3-8b.json deleted file mode 100644 index 8a31f7f87c19c43edae9d6601c4db21473c14f30..0000000000000000000000000000000000000000 --- a/data/models/flammenai_mahou-1.2a-llama3-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mahou-1.2a-llama3-8B", - "id": "flammenai/Mahou-1.2a-llama3-8B", - "developer": "flammenai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/flammenai_Mahou-1.2a-llama3-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5093 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5094 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0838 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2886 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3847 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3817 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/flammenai_mahou-1.2a-mistral-7b.json b/data/models/flammenai_mahou-1.2a-mistral-7b.json deleted file mode 100644 index e5a26b0bc991e0c38ec4ad67d13f53636dae4dab..0000000000000000000000000000000000000000 --- a/data/models/flammenai_mahou-1.2a-mistral-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mahou-1.2a-mistral-7B", - "id": "flammenai/Mahou-1.2a-mistral-7B", - "developer": "flammenai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/flammenai_Mahou-1.2a-mistral-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4552 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5118 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0687 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2718 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3896 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3163 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/flammenai_mahou-1.5-llama3.1-70b.json b/data/models/flammenai_mahou-1.5-llama3.1-70b.json deleted file mode 100644 index 7cbdc662f9956ddc03b2f8707ab1760c0459f35e..0000000000000000000000000000000000000000 --- a/data/models/flammenai_mahou-1.5-llama3.1-70b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mahou-1.5-llama3.1-70B", - "id": "flammenai/Mahou-1.5-llama3.1-70B", - "developer": "flammenai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "70.554" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/flammenai_Mahou-1.5-llama3.1-70B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7147 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6651 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.21 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.354 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.495 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4749 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/flammenai_mahou-1.5-mistral-nemo-12b.json b/data/models/flammenai_mahou-1.5-mistral-nemo-12b.json deleted file mode 100644 index d8864be56e212f27c472fa4ded54de5993b3c2fd..0000000000000000000000000000000000000000 --- a/data/models/flammenai_mahou-1.5-mistral-nemo-12b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mahou-1.5-mistral-nemo-12B", - "id": "flammenai/Mahou-1.5-mistral-nemo-12B", - "developer": "flammenai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/flammenai_Mahou-1.5-mistral-nemo-12B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6751 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5522 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0869 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.276 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.452 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3602 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/floflob_100k_fineweb_continued_pretraining_qwen2.5-0.5b-instruct_unsloth_merged_16bit.json b/data/models/floflob_100k_fineweb_continued_pretraining_qwen2.5-0.5b-instruct_unsloth_merged_16bit.json deleted file mode 100644 index 10a4fe478ec7f2c87a3948af83ea23e325add0db..0000000000000000000000000000000000000000 --- a/data/models/floflob_100k_fineweb_continued_pretraining_qwen2.5-0.5b-instruct_unsloth_merged_16bit.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "100k_fineweb_continued_pretraining_Qwen2.5-0.5B-Instruct_Unsloth_merged_16bit", - "id": "FlofloB/100k_fineweb_continued_pretraining_Qwen2.5-0.5B-Instruct_Unsloth_merged_16bit", - "developer": "FlofloB", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.5" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/FlofloB_100k_fineweb_continued_pretraining_Qwen2.5-0.5B-Instruct_Unsloth_merged_16bit/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3083 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3323 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0408 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2693 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3302 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1498 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/floflob_10k_continued_pretraining_phi-3-mini-4k-instruct_unsloth_merged_16bit.json b/data/models/floflob_10k_continued_pretraining_phi-3-mini-4k-instruct_unsloth_merged_16bit.json deleted file mode 100644 index 685ef680da94b705e30d8122fd436e15c3d1a0e7..0000000000000000000000000000000000000000 --- a/data/models/floflob_10k_continued_pretraining_phi-3-mini-4k-instruct_unsloth_merged_16bit.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "10k_continued_pretraining_Phi-3-mini-4k-instruct_Unsloth_merged_16bit", - "id": "FlofloB/10k_continued_pretraining_Phi-3-mini-4k-instruct_Unsloth_merged_16bit", - "developer": "FlofloB", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "16.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/FlofloB_10k_continued_pretraining_Phi-3-mini-4k-instruct_Unsloth_merged_16bit/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5097 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5215 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0974 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2995 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.431 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3769 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/floflob_10k_continued_pretraining_qwen2.5-0.5b-instruct_unsloth_merged_16bit.json b/data/models/floflob_10k_continued_pretraining_qwen2.5-0.5b-instruct_unsloth_merged_16bit.json deleted file mode 100644 index ad73afef39a88be40d36f0164ff920b81b39c802..0000000000000000000000000000000000000000 --- a/data/models/floflob_10k_continued_pretraining_qwen2.5-0.5b-instruct_unsloth_merged_16bit.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "10k_continued_pretraining_Qwen2.5-0.5B-Instruct_Unsloth_merged_16bit", - "id": "FlofloB/10k_continued_pretraining_Qwen2.5-0.5B-Instruct_Unsloth_merged_16bit", - "developer": "FlofloB", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.5" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/FlofloB_10k_continued_pretraining_Qwen2.5-0.5B-Instruct_Unsloth_merged_16bit/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2815 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3306 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.031 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2794 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3302 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1541 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/floflob_40k_continued_pretraining_qwen2.5-0.5b-instruct_unsloth_merged_16bit.json b/data/models/floflob_40k_continued_pretraining_qwen2.5-0.5b-instruct_unsloth_merged_16bit.json deleted file mode 100644 index d95764626fa0cacd830e3c4e819e165386a660d6..0000000000000000000000000000000000000000 --- a/data/models/floflob_40k_continued_pretraining_qwen2.5-0.5b-instruct_unsloth_merged_16bit.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "40k_continued_pretraining_Qwen2.5-0.5B-Instruct_Unsloth_merged_16bit", - "id": "FlofloB/40k_continued_pretraining_Qwen2.5-0.5B-Instruct_Unsloth_merged_16bit", - "developer": "FlofloB", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.5" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/FlofloB_40k_continued_pretraining_Qwen2.5-0.5B-Instruct_Unsloth_merged_16bit/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3016 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3325 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0332 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2676 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3408 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1485 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/floflob_83k_continued_pretraining_qwen2.5-0.5b-instruct_unsloth_merged_16bit.json b/data/models/floflob_83k_continued_pretraining_qwen2.5-0.5b-instruct_unsloth_merged_16bit.json deleted file mode 100644 index 7c9e030c4cb83fb0df5adbee172a0bef1f8fb210..0000000000000000000000000000000000000000 --- a/data/models/floflob_83k_continued_pretraining_qwen2.5-0.5b-instruct_unsloth_merged_16bit.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "83k_continued_pretraining_Qwen2.5-0.5B-Instruct_Unsloth_merged_16bit", - "id": "FlofloB/83k_continued_pretraining_Qwen2.5-0.5B-Instruct_Unsloth_merged_16bit", - "developer": "FlofloB", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.5" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/FlofloB_83k_continued_pretraining_Qwen2.5-0.5B-Instruct_Unsloth_merged_16bit/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2869 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3347 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0302 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2735 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3289 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1555 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/floflob_smollm2-135m_pretrained_1000k_fineweb.json b/data/models/floflob_smollm2-135m_pretrained_1000k_fineweb.json deleted file mode 100644 index fe35fed9bbaa3d84e78f1033837b6780e7eb5312..0000000000000000000000000000000000000000 --- a/data/models/floflob_smollm2-135m_pretrained_1000k_fineweb.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "smollm2-135M_pretrained_1000k_fineweb", - "id": "FlofloB/smollm2-135M_pretrained_1000k_fineweb", - "developer": "FlofloB", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "0.135" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/FlofloB_smollm2-135M_pretrained_1000k_fineweb/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1485 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2918 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0091 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2626 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3581 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1164 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/floflob_smollm2-135m_pretrained_1000k_fineweb_uncovai_human_removed.json b/data/models/floflob_smollm2-135m_pretrained_1000k_fineweb_uncovai_human_removed.json deleted file mode 100644 index 0965982cb6eeda75c4459b4ccd1c748b3b7f11e1..0000000000000000000000000000000000000000 --- a/data/models/floflob_smollm2-135m_pretrained_1000k_fineweb_uncovai_human_removed.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "smollm2-135M_pretrained_1000k_fineweb_uncovai_human_removed", - "id": "FlofloB/smollm2-135M_pretrained_1000k_fineweb_uncovai_human_removed", - "developer": "FlofloB", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "0.135" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/FlofloB_smollm2-135M_pretrained_1000k_fineweb_uncovai_human_removed/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1554 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3066 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.006 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2508 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.358 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1143 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/floflob_smollm2-135m_pretrained_1000k_fineweb_uncovai_selected.json b/data/models/floflob_smollm2-135m_pretrained_1000k_fineweb_uncovai_selected.json deleted file mode 100644 index 726acae8cda10e33fe7125da5fbeac29a9206040..0000000000000000000000000000000000000000 --- a/data/models/floflob_smollm2-135m_pretrained_1000k_fineweb_uncovai_selected.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "smollm2-135M_pretrained_1000k_fineweb_uncovai_selected", - "id": "FlofloB/smollm2-135M_pretrained_1000k_fineweb_uncovai_selected", - "developer": "FlofloB", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "0.135" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/FlofloB_smollm2-135M_pretrained_1000k_fineweb_uncovai_selected/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1468 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2932 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0068 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2659 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4048 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1157 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/floflob_smollm2-135m_pretrained_1200k_fineweb.json b/data/models/floflob_smollm2-135m_pretrained_1200k_fineweb.json deleted file mode 100644 index 777114c408aea584567069440cac8529cd811d84..0000000000000000000000000000000000000000 --- a/data/models/floflob_smollm2-135m_pretrained_1200k_fineweb.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "smollm2-135M_pretrained_1200k_fineweb", - "id": "FlofloB/smollm2-135M_pretrained_1200k_fineweb", - "developer": "FlofloB", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "0.135" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/FlofloB_smollm2-135M_pretrained_1200k_fineweb/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1581 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2941 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0068 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2643 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3714 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1076 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/floflob_smollm2-135m_pretrained_1200k_fineweb_uncovai_human_removed.json b/data/models/floflob_smollm2-135m_pretrained_1200k_fineweb_uncovai_human_removed.json deleted file mode 100644 index 533c470ceac4e5978a18dcd4958b9d39c821013e..0000000000000000000000000000000000000000 --- a/data/models/floflob_smollm2-135m_pretrained_1200k_fineweb_uncovai_human_removed.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "smollm2-135M_pretrained_1200k_fineweb_uncovai_human_removed", - "id": "FlofloB/smollm2-135M_pretrained_1200k_fineweb_uncovai_human_removed", - "developer": "FlofloB", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "0.135" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/FlofloB_smollm2-135M_pretrained_1200k_fineweb_uncovai_human_removed/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1578 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.295 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0008 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2651 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.37 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1139 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/floflob_smollm2-135m_pretrained_1200k_fineweb_uncovai_selected.json b/data/models/floflob_smollm2-135m_pretrained_1200k_fineweb_uncovai_selected.json deleted file mode 100644 index 57d53d422356468ee0eb7e7d108f8aa252cb0088..0000000000000000000000000000000000000000 --- a/data/models/floflob_smollm2-135m_pretrained_1200k_fineweb_uncovai_selected.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "smollm2-135M_pretrained_1200k_fineweb_uncovai_selected", - "id": "FlofloB/smollm2-135M_pretrained_1200k_fineweb_uncovai_selected", - "developer": "FlofloB", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "0.135" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/FlofloB_smollm2-135M_pretrained_1200k_fineweb_uncovai_selected/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1585 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.296 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0076 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2634 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3567 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1164 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/floflob_smollm2-135m_pretrained_1400k_fineweb.json b/data/models/floflob_smollm2-135m_pretrained_1400k_fineweb.json deleted file mode 100644 index f963d1595d4287c6f6a685014a5bb0dcf5c7c7c8..0000000000000000000000000000000000000000 --- a/data/models/floflob_smollm2-135m_pretrained_1400k_fineweb.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "smollm2-135M_pretrained_1400k_fineweb", - "id": "FlofloB/smollm2-135M_pretrained_1400k_fineweb", - "developer": "FlofloB", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "0.135" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/FlofloB_smollm2-135M_pretrained_1400k_fineweb/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1764 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2922 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0113 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2659 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3873 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.108 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/floflob_smollm2-135m_pretrained_1400k_fineweb_uncovai_human_removed.json b/data/models/floflob_smollm2-135m_pretrained_1400k_fineweb_uncovai_human_removed.json deleted file mode 100644 index 0851cab4d1a940d9d80f16e4ec848bee7063b1dc..0000000000000000000000000000000000000000 --- a/data/models/floflob_smollm2-135m_pretrained_1400k_fineweb_uncovai_human_removed.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "smollm2-135M_pretrained_1400k_fineweb_uncovai_human_removed", - "id": "FlofloB/smollm2-135M_pretrained_1400k_fineweb_uncovai_human_removed", - "developer": "FlofloB", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "0.135" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/FlofloB_smollm2-135M_pretrained_1400k_fineweb_uncovai_human_removed/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1707 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2992 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0106 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2609 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3939 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1105 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/floflob_smollm2-135m_pretrained_1400k_fineweb_uncovai_selected.json b/data/models/floflob_smollm2-135m_pretrained_1400k_fineweb_uncovai_selected.json deleted file mode 100644 index 37bd0875da8ca4ed99a3c7f450b8a494d59dbcb5..0000000000000000000000000000000000000000 --- a/data/models/floflob_smollm2-135m_pretrained_1400k_fineweb_uncovai_selected.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "smollm2-135M_pretrained_1400k_fineweb_uncovai_selected", - "id": "FlofloB/smollm2-135M_pretrained_1400k_fineweb_uncovai_selected", - "developer": "FlofloB", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "0.135" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/FlofloB_smollm2-135M_pretrained_1400k_fineweb_uncovai_selected/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1538 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2917 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0106 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2685 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3741 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1137 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/floflob_smollm2-135m_pretrained_200k_fineweb_uncovai_human_removed.json b/data/models/floflob_smollm2-135m_pretrained_200k_fineweb_uncovai_human_removed.json deleted file mode 100644 index 2541e7a2ebf2640428016e0b97694913d5887ac8..0000000000000000000000000000000000000000 --- a/data/models/floflob_smollm2-135m_pretrained_200k_fineweb_uncovai_human_removed.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "smollm2-135M_pretrained_200k_fineweb_uncovai_human_removed", - "id": "FlofloB/smollm2-135M_pretrained_200k_fineweb_uncovai_human_removed", - "developer": "FlofloB", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "0.135" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/FlofloB_smollm2-135M_pretrained_200k_fineweb_uncovai_human_removed/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1475 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3029 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0038 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2584 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3578 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.112 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/floflob_smollm2-135m_pretrained_200k_fineweb_uncovai_selected.json b/data/models/floflob_smollm2-135m_pretrained_200k_fineweb_uncovai_selected.json deleted file mode 100644 index 2578998e9680b4e5191a48fc711506d46c7cba7f..0000000000000000000000000000000000000000 --- a/data/models/floflob_smollm2-135m_pretrained_200k_fineweb_uncovai_selected.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "smollm2-135M_pretrained_200k_fineweb_uncovai_selected", - "id": "FlofloB/smollm2-135M_pretrained_200k_fineweb_uncovai_selected", - "developer": "FlofloB", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "0.135" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/FlofloB_smollm2-135M_pretrained_200k_fineweb_uncovai_selected/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1345 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2927 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0076 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2508 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.366 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1131 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/floflob_smollm2-135m_pretrained_400k_fineweb.json b/data/models/floflob_smollm2-135m_pretrained_400k_fineweb.json deleted file mode 100644 index 358e7a6f4676f0f6c9a96b80f30709b3c0976d01..0000000000000000000000000000000000000000 --- a/data/models/floflob_smollm2-135m_pretrained_400k_fineweb.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "smollm2-135M_pretrained_400k_fineweb", - "id": "FlofloB/smollm2-135M_pretrained_400k_fineweb", - "developer": "FlofloB", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "0.135" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/FlofloB_smollm2-135M_pretrained_400k_fineweb/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1511 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2972 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0121 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2525 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3794 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1163 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/floflob_smollm2-135m_pretrained_400k_fineweb_uncovai_human_removed.json b/data/models/floflob_smollm2-135m_pretrained_400k_fineweb_uncovai_human_removed.json deleted file mode 100644 index 1a218d400a2676a35c2f883bc83d11a2e0d4953a..0000000000000000000000000000000000000000 --- a/data/models/floflob_smollm2-135m_pretrained_400k_fineweb_uncovai_human_removed.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "smollm2-135M_pretrained_400k_fineweb_uncovai_human_removed", - "id": "FlofloB/smollm2-135M_pretrained_400k_fineweb_uncovai_human_removed", - "developer": "FlofloB", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "0.135" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/FlofloB_smollm2-135M_pretrained_400k_fineweb_uncovai_human_removed/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1556 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3049 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0091 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.255 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.386 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1138 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/floflob_smollm2-135m_pretrained_400k_fineweb_uncovai_selected.json b/data/models/floflob_smollm2-135m_pretrained_400k_fineweb_uncovai_selected.json deleted file mode 100644 index d6a00afbba7fbbe4d8b55d1b79f9a20f07b48720..0000000000000000000000000000000000000000 --- a/data/models/floflob_smollm2-135m_pretrained_400k_fineweb_uncovai_selected.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "smollm2-135M_pretrained_400k_fineweb_uncovai_selected", - "id": "FlofloB/smollm2-135M_pretrained_400k_fineweb_uncovai_selected", - "developer": "FlofloB", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "0.135" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/FlofloB_smollm2-135M_pretrained_400k_fineweb_uncovai_selected/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1584 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2925 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0068 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2542 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.382 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1158 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/floflob_smollm2-135m_pretrained_600k_fineweb.json b/data/models/floflob_smollm2-135m_pretrained_600k_fineweb.json deleted file mode 100644 index 523fa1904a04233467b5998b5088aaf32bb75972..0000000000000000000000000000000000000000 --- a/data/models/floflob_smollm2-135m_pretrained_600k_fineweb.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "smollm2-135M_pretrained_600k_fineweb", - "id": "FlofloB/smollm2-135M_pretrained_600k_fineweb", - "developer": "FlofloB", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "0.135" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/FlofloB_smollm2-135M_pretrained_600k_fineweb/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1639 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3014 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.006 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2659 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3809 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1126 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/floflob_smollm2-135m_pretrained_600k_fineweb_uncovai_human_removed.json b/data/models/floflob_smollm2-135m_pretrained_600k_fineweb_uncovai_human_removed.json deleted file mode 100644 index 5250b24035183197b2717d952c96e347675188ad..0000000000000000000000000000000000000000 --- a/data/models/floflob_smollm2-135m_pretrained_600k_fineweb_uncovai_human_removed.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "smollm2-135M_pretrained_600k_fineweb_uncovai_human_removed", - "id": "FlofloB/smollm2-135M_pretrained_600k_fineweb_uncovai_human_removed", - "developer": "FlofloB", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "0.135" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/FlofloB_smollm2-135M_pretrained_600k_fineweb_uncovai_human_removed/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1641 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0091 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2626 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3793 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1147 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/floflob_smollm2-135m_pretrained_600k_fineweb_uncovai_selected.json b/data/models/floflob_smollm2-135m_pretrained_600k_fineweb_uncovai_selected.json deleted file mode 100644 index 776727b3a559edc9f4c8e2a4c8b08d95c9c26de1..0000000000000000000000000000000000000000 --- a/data/models/floflob_smollm2-135m_pretrained_600k_fineweb_uncovai_selected.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "smollm2-135M_pretrained_600k_fineweb_uncovai_selected", - "id": "FlofloB/smollm2-135M_pretrained_600k_fineweb_uncovai_selected", - "developer": "FlofloB", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "0.135" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/FlofloB_smollm2-135M_pretrained_600k_fineweb_uncovai_selected/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1606 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2983 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0076 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2609 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3846 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1162 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/floflob_smollm2-135m_pretrained_800k_fineweb.json b/data/models/floflob_smollm2-135m_pretrained_800k_fineweb.json deleted file mode 100644 index 50e4df25d58379eca35824b5e19a7630aa912cfc..0000000000000000000000000000000000000000 --- a/data/models/floflob_smollm2-135m_pretrained_800k_fineweb.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "smollm2-135M_pretrained_800k_fineweb", - "id": "FlofloB/smollm2-135M_pretrained_800k_fineweb", - "developer": "FlofloB", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "0.135" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/FlofloB_smollm2-135M_pretrained_800k_fineweb/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1641 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2959 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0083 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2492 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3701 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1152 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/floflob_smollm2-135m_pretrained_800k_fineweb_uncovai_human_removed.json b/data/models/floflob_smollm2-135m_pretrained_800k_fineweb_uncovai_human_removed.json deleted file mode 100644 index ac181780885ede4749c8ea94861cbf6c67348e6c..0000000000000000000000000000000000000000 --- a/data/models/floflob_smollm2-135m_pretrained_800k_fineweb_uncovai_human_removed.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "smollm2-135M_pretrained_800k_fineweb_uncovai_human_removed", - "id": "FlofloB/smollm2-135M_pretrained_800k_fineweb_uncovai_human_removed", - "developer": "FlofloB", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "0.135" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/FlofloB_smollm2-135M_pretrained_800k_fineweb_uncovai_human_removed/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1623 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3038 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0068 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2525 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3993 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1138 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/floflob_smollm2-135m_pretrained_800k_fineweb_uncovai_selected.json b/data/models/floflob_smollm2-135m_pretrained_800k_fineweb_uncovai_selected.json deleted file mode 100644 index 31bdcf7b7bf2072e79003c00f06a77c6780766c3..0000000000000000000000000000000000000000 --- a/data/models/floflob_smollm2-135m_pretrained_800k_fineweb_uncovai_selected.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "smollm2-135M_pretrained_800k_fineweb_uncovai_selected", - "id": "FlofloB/smollm2-135M_pretrained_800k_fineweb_uncovai_selected", - "developer": "FlofloB", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "0.135" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/FlofloB_smollm2-135M_pretrained_800k_fineweb_uncovai_selected/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1474 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2943 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0045 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2617 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3766 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.113 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/floflob_smollm2_pretrained_200k_fineweb.json b/data/models/floflob_smollm2_pretrained_200k_fineweb.json deleted file mode 100644 index c753a2f43fc117767eafa2dc8fd30db3c995db91..0000000000000000000000000000000000000000 --- a/data/models/floflob_smollm2_pretrained_200k_fineweb.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "smollm2_pretrained_200k_fineweb", - "id": "FlofloB/smollm2_pretrained_200k_fineweb", - "developer": "FlofloB", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "0.135" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/FlofloB_smollm2_pretrained_200k_fineweb/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1527 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2995 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0038 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2475 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3699 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1159 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/floflob_test_continued_pretraining_phi-3-mini-4k-instruct_unsloth_merged_16bit.json b/data/models/floflob_test_continued_pretraining_phi-3-mini-4k-instruct_unsloth_merged_16bit.json deleted file mode 100644 index 39dd71cdce486fd5ded9d3e734b976d4893cfaac..0000000000000000000000000000000000000000 --- a/data/models/floflob_test_continued_pretraining_phi-3-mini-4k-instruct_unsloth_merged_16bit.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "test_continued_pretraining_Phi-3-mini-4k-instruct_Unsloth_merged_16bit", - "id": "FlofloB/test_continued_pretraining_Phi-3-mini-4k-instruct_Unsloth_merged_16bit", - "developer": "FlofloB", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "16.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/FlofloB_test_continued_pretraining_Phi-3-mini-4k-instruct_Unsloth_merged_16bit/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5215 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5241 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1103 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3112 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4244 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3721 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/fluently-lm_fluentlylm-prinum.json b/data/models/fluently-lm_fluentlylm-prinum.json deleted file mode 100644 index af71f102b63d09890e22e9b4238c6ab9fb31c0cc..0000000000000000000000000000000000000000 --- a/data/models/fluently-lm_fluentlylm-prinum.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "FluentlyLM-Prinum", - "id": "fluently-lm/FluentlyLM-Prinum", - "developer": "fluently-lm", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "32.764" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/fluently-lm_FluentlyLM-Prinum/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.809 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7144 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.54 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3867 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4471 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5808 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/fluently-lm_llama-ti-8b-instruct.json b/data/models/fluently-lm_llama-ti-8b-instruct.json deleted file mode 100644 index ef374d551db2129abeade6a47d1cc25401fe755b..0000000000000000000000000000000000000000 --- a/data/models/fluently-lm_llama-ti-8b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-TI-8B-Instruct", - "id": "fluently-lm/Llama-TI-8B-Instruct", - "developer": "fluently-lm", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/fluently-lm_Llama-TI-8B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7716 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5252 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2304 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2953 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3813 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3726 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/fluently-lm_llama-ti-8b.json b/data/models/fluently-lm_llama-ti-8b.json deleted file mode 100644 index 22056d63aff866a77dff764e3c1fed2408ff2836..0000000000000000000000000000000000000000 --- a/data/models/fluently-lm_llama-ti-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-TI-8B", - "id": "fluently-lm/Llama-TI-8B", - "developer": "fluently-lm", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/fluently-lm_Llama-TI-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.288 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5201 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1964 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2961 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4103 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.344 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/fluently-sets_falconthink3-10b-it.json b/data/models/fluently-sets_falconthink3-10b-it.json deleted file mode 100644 index 1b4f383573094f77bba6fd269c451ee45ef10c17..0000000000000000000000000000000000000000 --- a/data/models/fluently-sets_falconthink3-10b-it.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "FalconThink3-10B-IT", - "id": "fluently-sets/FalconThink3-10B-IT", - "developer": "fluently-sets", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "10.306" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/fluently-sets_FalconThink3-10B-IT/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7326 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.62 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2447 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3347 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4479 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4435 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/fluently-sets_reasoning-1-1k-demo.json b/data/models/fluently-sets_reasoning-1-1k-demo.json deleted file mode 100644 index b37f51b843298c2418bd306bf1ce595b287b393f..0000000000000000000000000000000000000000 --- a/data/models/fluently-sets_reasoning-1-1k-demo.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "reasoning-1-1k-demo", - "id": "fluently-sets/reasoning-1-1k-demo", - "developer": "fluently-sets", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/fluently-sets_reasoning-1-1k-demo/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7525 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6397 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4282 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3356 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4061 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4774 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/formulae_mita-elite-sce-gen1.1-v1-7b-2-26-2025-exp.json b/data/models/formulae_mita-elite-sce-gen1.1-v1-7b-2-26-2025-exp.json deleted file mode 100644 index df4146a67ebbfc6089d24678a4a7b8c2beedc369..0000000000000000000000000000000000000000 --- a/data/models/formulae_mita-elite-sce-gen1.1-v1-7b-2-26-2025-exp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "mita-elite-sce-gen1.1-v1-7b-2-26-2025-exp", - "id": "formulae/mita-elite-sce-gen1.1-v1-7b-2-26-2025-exp", - "developer": "formulae", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/formulae_mita-elite-sce-gen1.1-v1-7b-2-26-2025-exp/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1614 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2976 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0015 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2534 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4219 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1174 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/formulae_mita-elite-v1.1-7b-2-25-2025.json b/data/models/formulae_mita-elite-v1.1-7b-2-25-2025.json deleted file mode 100644 index a73742fed11b79ba719b3cd588292dc837b8e19b..0000000000000000000000000000000000000000 --- a/data/models/formulae_mita-elite-v1.1-7b-2-25-2025.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "mita-elite-v1.1-7b-2-25-2025", - "id": "formulae/mita-elite-v1.1-7b-2-25-2025", - "developer": "formulae", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/formulae_mita-elite-v1.1-7b-2-25-2025/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.125 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2867 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2483 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3487 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1098 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/formulae_mita-elite-v1.1-gen2-7b-2-25-2025.json b/data/models/formulae_mita-elite-v1.1-gen2-7b-2-25-2025.json deleted file mode 100644 index ca7483e87be1c5b87b59b0d36fd9d92bf0f44fdd..0000000000000000000000000000000000000000 --- a/data/models/formulae_mita-elite-v1.1-gen2-7b-2-25-2025.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "mita-elite-v1.1-gen2-7b-2-25-2025", - "id": "formulae/mita-elite-v1.1-gen2-7b-2-25-2025", - "developer": "formulae", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/formulae_mita-elite-v1.1-gen2-7b-2-25-2025/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1411 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2924 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2525 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3541 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1101 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/formulae_mita-elite-v1.2-7b-2-26-2025.json b/data/models/formulae_mita-elite-v1.2-7b-2-26-2025.json deleted file mode 100644 index e4f04ce6d580d52e7c153764dd36d41bce96e71c..0000000000000000000000000000000000000000 --- a/data/models/formulae_mita-elite-v1.2-7b-2-26-2025.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "mita-elite-v1.2-7b-2-26-2025", - "id": "formulae/mita-elite-v1.2-7b-2-26-2025", - "developer": "formulae", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/formulae_mita-elite-v1.2-7b-2-26-2025/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.148 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.293 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0023 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2743 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4287 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1186 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/formulae_mita-gen3-7b-2-26-2025.json b/data/models/formulae_mita-gen3-7b-2-26-2025.json deleted file mode 100644 index bc5c6badeac8ae1aafd2afe270153fe0a2e05a77..0000000000000000000000000000000000000000 --- a/data/models/formulae_mita-gen3-7b-2-26-2025.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "mita-gen3-7b-2-26-2025", - "id": "formulae/mita-gen3-7b-2-26-2025", - "developer": "formulae", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/formulae_mita-gen3-7b-2-26-2025/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1964 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2916 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0023 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2651 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3912 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1124 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/formulae_mita-gen3-v1.2-7b-2-26-2025.json b/data/models/formulae_mita-gen3-v1.2-7b-2-26-2025.json deleted file mode 100644 index 6ba40bc1f1324949bedcef8551bbd51b87121961..0000000000000000000000000000000000000000 --- a/data/models/formulae_mita-gen3-v1.2-7b-2-26-2025.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "mita-gen3-v1.2-7b-2-26-2025", - "id": "formulae/mita-gen3-v1.2-7b-2-26-2025", - "developer": "formulae", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/formulae_mita-gen3-v1.2-7b-2-26-2025/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2044 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3058 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0023 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2592 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.39 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1128 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/formulae_mita-math-v2.3-2-25-2025.json b/data/models/formulae_mita-math-v2.3-2-25-2025.json deleted file mode 100644 index c8f3bfbb60715a303620b872e4d167c69c95807e..0000000000000000000000000000000000000000 --- a/data/models/formulae_mita-math-v2.3-2-25-2025.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "mita-math-v2.3-2-25-2025", - "id": "formulae/mita-math-v2.3-2-25-2025", - "developer": "formulae", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/formulae_mita-math-v2.3-2-25-2025/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1373 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2949 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2508 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3698 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1118 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/formulae_mita-v1-7b.json b/data/models/formulae_mita-v1-7b.json deleted file mode 100644 index be0abc75520d88dfa7979666266700deee6b850c..0000000000000000000000000000000000000000 --- a/data/models/formulae_mita-v1-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "mita-v1-7b", - "id": "formulae/mita-v1-7b", - "developer": "formulae", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/formulae_mita-v1-7b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1972 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3003 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0023 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.25 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4152 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1147 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/formulae_mita-v1.1-7b-2-24-2025.json b/data/models/formulae_mita-v1.1-7b-2-24-2025.json deleted file mode 100644 index 1513183815ef7389df0de291b7e114f16cbf823c..0000000000000000000000000000000000000000 --- a/data/models/formulae_mita-v1.1-7b-2-24-2025.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "mita-v1.1-7b-2-24-2025", - "id": "formulae/mita-v1.1-7b-2-24-2025", - "developer": "formulae", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/formulae_mita-v1.1-7b-2-24-2025/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3412 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5442 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.435 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3146 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4557 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4524 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/formulae_mita-v1.2-7b-2-24-2025.json b/data/models/formulae_mita-v1.2-7b-2-24-2025.json deleted file mode 100644 index e8b6248e0d91f0364bad563cdef5565adc311033..0000000000000000000000000000000000000000 --- a/data/models/formulae_mita-v1.2-7b-2-24-2025.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "mita-v1.2-7b-2-24-2025", - "id": "formulae/mita-v1.2-7b-2-24-2025", - "developer": "formulae", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/formulae_mita-v1.2-7b-2-24-2025/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2564 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4919 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4879 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3062 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4344 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3359 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/frameai_loxa-4b.json b/data/models/frameai_loxa-4b.json deleted file mode 100644 index 7548fa35eea92a233451db486ca27475e3ddb62a..0000000000000000000000000000000000000000 --- a/data/models/frameai_loxa-4b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Loxa-4B", - "id": "frameai/Loxa-4B", - "developer": "frameai", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "4.018" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/frameai_Loxa-4B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4765 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4217 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1095 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2836 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3377 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2802 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/freewheelin_free-evo-qwen72b-v0.8-re.json b/data/models/freewheelin_free-evo-qwen72b-v0.8-re.json deleted file mode 100644 index 0f779e7768027fcea67791225d33e025a40307c1..0000000000000000000000000000000000000000 --- a/data/models/freewheelin_free-evo-qwen72b-v0.8-re.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "free-evo-qwen72b-v0.8-re", - "id": "freewheelin/free-evo-qwen72b-v0.8-re", - "developer": "freewheelin", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "72.288" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/freewheelin_free-evo-qwen72b-v0.8-re/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5331 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6127 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1805 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3565 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4872 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.487 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/freewheelin_free-solar-evo-v0.1.json b/data/models/freewheelin_free-solar-evo-v0.1.json deleted file mode 100644 index 398f466e487cefe2461f9f0a0f44dbf72ba8aa65..0000000000000000000000000000000000000000 --- a/data/models/freewheelin_free-solar-evo-v0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "free-solar-evo-v0.1", - "id": "freewheelin/free-solar-evo-v0.1", - "developer": "freewheelin", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "10.732" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/freewheelin_free-solar-evo-v0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.205 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4502 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0083 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2911 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4946 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3414 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/freewheelin_free-solar-evo-v0.11.json b/data/models/freewheelin_free-solar-evo-v0.11.json deleted file mode 100644 index 29c406c4af7fa4eea281fff5ed2e6e5279233838..0000000000000000000000000000000000000000 --- a/data/models/freewheelin_free-solar-evo-v0.11.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "free-solar-evo-v0.11", - "id": "freewheelin/free-solar-evo-v0.11", - "developer": "freewheelin", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "10.732" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/freewheelin_free-solar-evo-v0.11/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2027 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4545 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0083 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2852 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5052 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3467 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/freewheelin_free-solar-evo-v0.13.json b/data/models/freewheelin_free-solar-evo-v0.13.json deleted file mode 100644 index 755ff4ec56aea583f6a5c3799517660d70a5a028..0000000000000000000000000000000000000000 --- a/data/models/freewheelin_free-solar-evo-v0.13.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "free-solar-evo-v0.13", - "id": "freewheelin/free-solar-evo-v0.13", - "developer": "freewheelin", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "10.732" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/freewheelin_free-solar-evo-v0.13/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2321 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4555 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0121 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2886 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5052 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.347 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/fujhen_ft-openhermes-25-mistral-7b-irca-dpo-pairs.json b/data/models/fujhen_ft-openhermes-25-mistral-7b-irca-dpo-pairs.json deleted file mode 100644 index d8f33140cc73ae1b114fca58aaad34f9655bdeb9..0000000000000000000000000000000000000000 --- a/data/models/fujhen_ft-openhermes-25-mistral-7b-irca-dpo-pairs.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ft-openhermes-25-mistral-7b-irca-dpo-pairs", - "id": "FuJhen/ft-openhermes-25-mistral-7b-irca-dpo-pairs", - "developer": "FuJhen", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "?", - "params_billions": "14.483" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/FuJhen_ft-openhermes-25-mistral-7b-irca-dpo-pairs/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.542 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4773 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0483 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2785 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4174 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2956 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/fujhen_mistral-instruct-7b-dpo.json b/data/models/fujhen_mistral-instruct-7b-dpo.json deleted file mode 100644 index 76731b32a0fd82e3d199d7d6aa0135378206ae66..0000000000000000000000000000000000000000 --- a/data/models/fujhen_mistral-instruct-7b-dpo.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "mistral-instruct-7B-DPO", - "id": "FuJhen/mistral-instruct-7B-DPO", - "developer": "FuJhen", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "?", - "params_billions": "14.496" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/FuJhen_mistral-instruct-7B-DPO/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4968 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4624 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0385 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2777 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4016 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3034 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/fujhen_mistral_7b_v0.1_structeddata_e2e.json b/data/models/fujhen_mistral_7b_v0.1_structeddata_e2e.json deleted file mode 100644 index f071e48913034e5e1cc1107374fe427d946acf6c..0000000000000000000000000000000000000000 --- a/data/models/fujhen_mistral_7b_v0.1_structeddata_e2e.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "mistral_7b_v0.1_structedData_e2e", - "id": "FuJhen/mistral_7b_v0.1_structedData_e2e", - "developer": "FuJhen", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "?", - "params_billions": "7.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/FuJhen_mistral_7b_v0.1_structedData_e2e/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1727 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4114 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0045 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2794 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3723 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2811 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/fujhen_mistral_7b_v0.1_structeddata_viggo.json b/data/models/fujhen_mistral_7b_v0.1_structeddata_viggo.json deleted file mode 100644 index 549e29ac3526f8991c5179d1adf13653f451f39a..0000000000000000000000000000000000000000 --- a/data/models/fujhen_mistral_7b_v0.1_structeddata_viggo.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "mistral_7b_v0.1_structedData_viggo", - "id": "FuJhen/mistral_7b_v0.1_structedData_viggo", - "developer": "FuJhen", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "?", - "params_billions": "14.483" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/FuJhen_mistral_7b_v0.1_structedData_viggo/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1783 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4524 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0287 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2836 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3738 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2942 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/fulim_finellama-3.1-8b.json b/data/models/fulim_finellama-3.1-8b.json deleted file mode 100644 index 85209d25eb07892b3d53343d200f49fb7e352956..0000000000000000000000000000000000000000 --- a/data/models/fulim_finellama-3.1-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "FineLlama-3.1-8B", - "id": "fulim/FineLlama-3.1-8B", - "developer": "fulim", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/fulim_FineLlama-3.1-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1439 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4569 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0476 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2928 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3867 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3167 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/fuseai_fusechat-7b-v2.0.json b/data/models/fuseai_fusechat-7b-v2.0.json deleted file mode 100644 index 7c7745dd007ca25ba2ac9bfa9f60cd128feb184e..0000000000000000000000000000000000000000 --- a/data/models/fuseai_fusechat-7b-v2.0.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "FuseChat-7B-v2.0", - "id": "FuseAI/FuseChat-7B-v2.0", - "developer": "FuseAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/FuseAI_FuseChat-7B-v2.0/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3423 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4954 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0612 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.302 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4797 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3162 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/fuseai_fusechat-llama-3.1-8b-instruct.json b/data/models/fuseai_fusechat-llama-3.1-8b-instruct.json deleted file mode 100644 index e353754fb56d6883e41fc169b0d10fa9153dfb75..0000000000000000000000000000000000000000 --- a/data/models/fuseai_fusechat-llama-3.1-8b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "FuseChat-Llama-3.1-8B-Instruct", - "id": "FuseAI/FuseChat-Llama-3.1-8B-Instruct", - "developer": "FuseAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/FuseAI_FuseChat-Llama-3.1-8B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7205 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.512 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2477 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3054 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.382 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3733 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/fuseai_fusechat-llama-3.2-3b-instruct.json b/data/models/fuseai_fusechat-llama-3.2-3b-instruct.json deleted file mode 100644 index 93c3b300efa5c5d082d01b2b357e46e33e728df9..0000000000000000000000000000000000000000 --- a/data/models/fuseai_fusechat-llama-3.2-3b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "FuseChat-Llama-3.2-3B-Instruct", - "id": "FuseAI/FuseChat-Llama-3.2-3B-Instruct", - "developer": "FuseAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/FuseAI_FuseChat-Llama-3.2-3B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6849 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4658 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2424 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2961 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3914 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3132 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/fuseai_fusechat-qwen-2.5-7b-instruct.json b/data/models/fuseai_fusechat-qwen-2.5-7b-instruct.json deleted file mode 100644 index dc95b8f04f985621711da626b64f3da2593f1108..0000000000000000000000000000000000000000 --- a/data/models/fuseai_fusechat-qwen-2.5-7b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "FuseChat-Qwen-2.5-7B-Instruct", - "id": "FuseAI/FuseChat-Qwen-2.5-7B-Instruct", - "developer": "FuseAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/FuseAI_FuseChat-Qwen-2.5-7B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5906 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5526 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4562 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2961 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3874 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4118 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/gabrielmbmb_smollm-1.7b-instruct-ifeval.json b/data/models/gabrielmbmb_smollm-1.7b-instruct-ifeval.json deleted file mode 100644 index ee741060acc7af949d12731fc0fc6a5f1ccf0ad5..0000000000000000000000000000000000000000 --- a/data/models/gabrielmbmb_smollm-1.7b-instruct-ifeval.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SmolLM-1.7B-Instruct-IFEval", - "id": "gabrielmbmb/SmolLM-1.7B-Instruct-IFEval", - "developer": "gabrielmbmb", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.711" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/gabrielmbmb_SmolLM-1.7B-Instruct-IFEval/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2306 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3138 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0106 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2534 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3328 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1156 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/galrionsoftworks_magnusintellectus-12b-v1.json b/data/models/galrionsoftworks_magnusintellectus-12b-v1.json deleted file mode 100644 index 89a4f17e58eebac48f34e50e8cfab8d4f973a7a9..0000000000000000000000000000000000000000 --- a/data/models/galrionsoftworks_magnusintellectus-12b-v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MagnusIntellectus-12B-v1", - "id": "GalrionSoftworks/MagnusIntellectus-12B-v1", - "developer": "GalrionSoftworks", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/GalrionSoftworks_MagnusIntellectus-12B-v1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4421 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5323 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.065 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2844 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4428 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3421 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/galrionsoftworks_mn-loosecannon-12b-v1.json b/data/models/galrionsoftworks_mn-loosecannon-12b-v1.json deleted file mode 100644 index f075cf78367595790b588fecd96cb8a5c1d1a18f..0000000000000000000000000000000000000000 --- a/data/models/galrionsoftworks_mn-loosecannon-12b-v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MN-LooseCannon-12B-v1", - "id": "GalrionSoftworks/MN-LooseCannon-12B-v1", - "developer": "GalrionSoftworks", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/GalrionSoftworks_MN-LooseCannon-12B-v1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5418 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5128 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0853 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2852 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4138 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3196 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/gaverfraxz_meta-llama-3.1-8b-instruct-halfabliterated-della.json b/data/models/gaverfraxz_meta-llama-3.1-8b-instruct-halfabliterated-della.json deleted file mode 100644 index 237afe9b4ea2c840c7a514afd19489d071583a4a..0000000000000000000000000000000000000000 --- a/data/models/gaverfraxz_meta-llama-3.1-8b-instruct-halfabliterated-della.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Meta-Llama-3.1-8B-Instruct-HalfAbliterated-DELLA", - "id": "gaverfraxz/Meta-Llama-3.1-8B-Instruct-HalfAbliterated-DELLA", - "developer": "gaverfraxz", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/gaverfraxz_Meta-Llama-3.1-8B-Instruct-HalfAbliterated-DELLA/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4009 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3985 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0196 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2844 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.365 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1654 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/gaverfraxz_meta-llama-3.1-8b-instruct-halfabliterated-ties.json b/data/models/gaverfraxz_meta-llama-3.1-8b-instruct-halfabliterated-ties.json deleted file mode 100644 index eca5a989936c7862cd414ca8be6ec943e5b1c11e..0000000000000000000000000000000000000000 --- a/data/models/gaverfraxz_meta-llama-3.1-8b-instruct-halfabliterated-ties.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Meta-Llama-3.1-8B-Instruct-HalfAbliterated-TIES", - "id": "gaverfraxz/Meta-Llama-3.1-8B-Instruct-HalfAbliterated-TIES", - "developer": "gaverfraxz", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/gaverfraxz_Meta-Llama-3.1-8B-Instruct-HalfAbliterated-TIES/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4551 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5044 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1299 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2668 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3738 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3679 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/gbueno86_brinebreath-llama-3.1-70b.json b/data/models/gbueno86_brinebreath-llama-3.1-70b.json deleted file mode 100644 index 8ed75fd8daee63a2f6a2bdf69295917b31367c18..0000000000000000000000000000000000000000 --- a/data/models/gbueno86_brinebreath-llama-3.1-70b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Brinebreath-Llama-3.1-70B", - "id": "gbueno86/Brinebreath-Llama-3.1-70B", - "developer": "gbueno86", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "70.554" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/gbueno86_Brinebreath-Llama-3.1-70B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5533 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6881 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2976 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3465 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4541 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5196 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/gbueno86_meta-llama-3-cat-smaug-llama-70b.json b/data/models/gbueno86_meta-llama-3-cat-smaug-llama-70b.json deleted file mode 100644 index a329aea2adb569ffb0a4fb2fbf7a5daafadaeaed..0000000000000000000000000000000000000000 --- a/data/models/gbueno86_meta-llama-3-cat-smaug-llama-70b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Meta-LLama-3-Cat-Smaug-LLama-70b", - "id": "gbueno86/Meta-LLama-3-Cat-Smaug-LLama-70b", - "developer": "gbueno86", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "70.554" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/gbueno86_Meta-LLama-3-Cat-Smaug-LLama-70b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8072 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6674 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2938 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3272 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4368 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5075 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/gemini-1.5-flash-8b.json b/data/models/gemini-1.5-flash-8b.json deleted file mode 100644 index 2c73ec475a61d9303284185eeddbb9194b455f5d..0000000000000000000000000000000000000000 --- a/data/models/gemini-1.5-flash-8b.json +++ /dev/null @@ -1,126 +0,0 @@ -{ - "model_info": { - "name": "gemini-1.5-flash-8b", - "id": "gemini-1.5-flash-8b", - "developer": "unknown", - "additional_details": { - "model_type": "Generative" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/gemini-1.5-flash-8b/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7601 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9441 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5987 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7399 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7575 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/general-preference_gpm-gemma-2b.json b/data/models/general-preference_gpm-gemma-2b.json deleted file mode 100644 index a085d093f789ab0afe66d98575b29be39241a27d..0000000000000000000000000000000000000000 --- a/data/models/general-preference_gpm-gemma-2b.json +++ /dev/null @@ -1,126 +0,0 @@ -{ - "model_info": { - "name": "general-preference/GPM-Gemma-2B", - "id": "general-preference/GPM-Gemma-2B", - "developer": "general-preference", - "additional_details": { - "model_type": "Custom Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/general-preference_GPM-Gemma-2B/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7449 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7151 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6974 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8122 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.755 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/general-preference_gpm-llama-3.1-8b.json b/data/models/general-preference_gpm-llama-3.1-8b.json deleted file mode 100644 index 9579e13e754846fc84c4bb51526a14541b550848..0000000000000000000000000000000000000000 --- a/data/models/general-preference_gpm-llama-3.1-8b.json +++ /dev/null @@ -1,126 +0,0 @@ -{ - "model_info": { - "name": "general-preference/GPM-Llama-3.1-8B", - "id": "general-preference/GPM-Llama-3.1-8B", - "developer": "general-preference", - "additional_details": { - "model_type": "Custom Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/general-preference_GPM-Llama-3.1-8B/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9224 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.933 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.886 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9108 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9597 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/genvradmin_aryabhatta-gemmaorca-2-merged.json b/data/models/genvradmin_aryabhatta-gemmaorca-2-merged.json deleted file mode 100644 index ef490c69732b0ebbb2c92ba0213f1eebf0360c4c..0000000000000000000000000000000000000000 --- a/data/models/genvradmin_aryabhatta-gemmaorca-2-merged.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "AryaBhatta-GemmaOrca-2-Merged", - "id": "GenVRadmin/AryaBhatta-GemmaOrca-2-Merged", - "developer": "GenVRadmin", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "GemmaForCausalLM", - "params_billions": "8.538" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/GenVRadmin_AryaBhatta-GemmaOrca-2-Merged/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3064 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3887 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0498 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2685 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.455 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2384 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/genvradmin_aryabhatta-gemmaorca-merged.json b/data/models/genvradmin_aryabhatta-gemmaorca-merged.json deleted file mode 100644 index 1dcd5420a1940f2bc6fc41d051b51600ce1b3e11..0000000000000000000000000000000000000000 --- a/data/models/genvradmin_aryabhatta-gemmaorca-merged.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "AryaBhatta-GemmaOrca-Merged", - "id": "GenVRadmin/AryaBhatta-GemmaOrca-Merged", - "developer": "GenVRadmin", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "GemmaForCausalLM", - "params_billions": "8.538" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/GenVRadmin_AryaBhatta-GemmaOrca-Merged/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3064 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4131 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0514 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2559 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3524 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2228 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/genvradmin_aryabhatta-gemmaultra-merged.json b/data/models/genvradmin_aryabhatta-gemmaultra-merged.json deleted file mode 100644 index 8877a1fc237b17201b9e65d64853f15b97d03c35..0000000000000000000000000000000000000000 --- a/data/models/genvradmin_aryabhatta-gemmaultra-merged.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "AryaBhatta-GemmaUltra-Merged", - "id": "GenVRadmin/AryaBhatta-GemmaUltra-Merged", - "developer": "GenVRadmin", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "GemmaForCausalLM", - "params_billions": "8.538" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/GenVRadmin_AryaBhatta-GemmaUltra-Merged/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3021 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4141 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0536 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2534 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4279 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2266 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/genvradmin_llama38bgenz_vikas-merged.json b/data/models/genvradmin_llama38bgenz_vikas-merged.json deleted file mode 100644 index a3bee613463649f3c7a0027844af2b901c6e8876..0000000000000000000000000000000000000000 --- a/data/models/genvradmin_llama38bgenz_vikas-merged.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "llama38bGenZ_Vikas-Merged", - "id": "GenVRadmin/llama38bGenZ_Vikas-Merged", - "developer": "GenVRadmin", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/GenVRadmin_llama38bGenZ_Vikas-Merged/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4536 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0574 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2953 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4402 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2622 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ghost-x_ghost-8b-beta-1608.json b/data/models/ghost-x_ghost-8b-beta-1608.json deleted file mode 100644 index da6c2b794b375cd027f2d3b6659c065a003f9677..0000000000000000000000000000000000000000 --- a/data/models/ghost-x_ghost-8b-beta-1608.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ghost-8b-beta-1608", - "id": "ghost-x/ghost-8b-beta-1608", - "developer": "ghost-x", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ghost-x_ghost-8b-beta-1608/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4273 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4517 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0695 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2584 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3516 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.284 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/glaiveai_reflection-llama-3.1-70b.json b/data/models/glaiveai_reflection-llama-3.1-70b.json deleted file mode 100644 index 0a569e50a467d27f8757568b3ee94435525e093f..0000000000000000000000000000000000000000 --- a/data/models/glaiveai_reflection-llama-3.1-70b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Reflection-Llama-3.1-70B", - "id": "glaiveai/Reflection-Llama-3.1-70B", - "developer": "glaiveai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "69.5" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/glaiveai_Reflection-Llama-3.1-70B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5991 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5681 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2757 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3146 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.438 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6341 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/gmonsoon_gemma2-9b-sahabatai-v1-instruct-baseties.json b/data/models/gmonsoon_gemma2-9b-sahabatai-v1-instruct-baseties.json deleted file mode 100644 index 89497295217cc50c243dc6200531f27b541e3df3..0000000000000000000000000000000000000000 --- a/data/models/gmonsoon_gemma2-9b-sahabatai-v1-instruct-baseties.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "gemma2-9b-sahabatai-v1-instruct-BaseTIES", - "id": "gmonsoon/gemma2-9b-sahabatai-v1-instruct-BaseTIES", - "developer": "gmonsoon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "9.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/gmonsoon_gemma2-9b-sahabatai-v1-instruct-BaseTIES/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7378 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6077 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1994 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3205 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4778 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4347 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/gmonsoon_sahabatai-llama-11b-test.json b/data/models/gmonsoon_sahabatai-llama-11b-test.json deleted file mode 100644 index 1a2abbcd25fcdd41993d36224612e5ac58bf7329..0000000000000000000000000000000000000000 --- a/data/models/gmonsoon_sahabatai-llama-11b-test.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SahabatAI-Llama-11B-Test", - "id": "gmonsoon/SahabatAI-Llama-11B-Test", - "developer": "gmonsoon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "11.52" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/gmonsoon_SahabatAI-Llama-11B-Test/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3376 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4728 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.031 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2819 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4001 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3182 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/gmonsoon_sahabatai-medichatindo-8b-v1.json b/data/models/gmonsoon_sahabatai-medichatindo-8b-v1.json deleted file mode 100644 index 14705cb16a22cb025a1e578d62f4b5b2a7821236..0000000000000000000000000000000000000000 --- a/data/models/gmonsoon_sahabatai-medichatindo-8b-v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SahabatAI-MediChatIndo-8B-v1", - "id": "gmonsoon/SahabatAI-MediChatIndo-8B-v1", - "developer": "gmonsoon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/gmonsoon_SahabatAI-MediChatIndo-8B-v1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4163 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4509 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0619 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2827 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3754 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3108 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/gmonsoon_sahabatai-rebase-8b-test.json b/data/models/gmonsoon_sahabatai-rebase-8b-test.json deleted file mode 100644 index 51488debe2e4dd1ec1c4cf9c3d8a912f3cfe355d..0000000000000000000000000000000000000000 --- a/data/models/gmonsoon_sahabatai-rebase-8b-test.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SahabatAI-Rebase-8B-Test", - "id": "gmonsoon/SahabatAI-Rebase-8B-Test", - "developer": "gmonsoon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/gmonsoon_SahabatAI-Rebase-8B-Test/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5156 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.523 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1148 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2878 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4133 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3664 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/gmonsoon_stockseallms-7b-v1.json b/data/models/gmonsoon_stockseallms-7b-v1.json deleted file mode 100644 index 2c653e3f085bcc9f2b5c8f6e0f57a780f949ed5c..0000000000000000000000000000000000000000 --- a/data/models/gmonsoon_stockseallms-7b-v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "StockSeaLLMs-7B-v1", - "id": "gmonsoon/StockSeaLLMs-7B-v1", - "developer": "gmonsoon", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/gmonsoon_StockSeaLLMs-7B-v1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4599 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5271 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1964 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3029 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4214 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3952 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/godlikehhd_alpaca_data_full_2.json b/data/models/godlikehhd_alpaca_data_full_2.json deleted file mode 100644 index e4aab231ded3ca56a3e24f52e7077200858f5032..0000000000000000000000000000000000000000 --- a/data/models/godlikehhd_alpaca_data_full_2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "alpaca_data_full_2", - "id": "godlikehhd/alpaca_data_full_2", - "developer": "godlikehhd", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.544" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/godlikehhd_alpaca_data_full_2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3178 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4217 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0929 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2978 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4052 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2854 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/godlikehhd_alpaca_data_full_3b.json b/data/models/godlikehhd_alpaca_data_full_3b.json deleted file mode 100644 index 00badd946d725e75345df17b5472515336d00178..0000000000000000000000000000000000000000 --- a/data/models/godlikehhd_alpaca_data_full_3b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "alpaca_data_full_3B", - "id": "godlikehhd/alpaca_data_full_3B", - "developer": "godlikehhd", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.086" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/godlikehhd_alpaca_data_full_3B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3696 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4684 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1337 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2777 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4955 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3357 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/godlikehhd_alpaca_data_ifd_max_2600.json b/data/models/godlikehhd_alpaca_data_ifd_max_2600.json deleted file mode 100644 index ef6bdb3c34e54f53cae24031d1e82bf6755bcf77..0000000000000000000000000000000000000000 --- a/data/models/godlikehhd_alpaca_data_ifd_max_2600.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "alpaca_data_ifd_max_2600", - "id": "godlikehhd/alpaca_data_ifd_max_2600", - "developer": "godlikehhd", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.544" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/godlikehhd_alpaca_data_ifd_max_2600/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3043 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4029 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0989 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3029 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3509 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2916 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/godlikehhd_alpaca_data_ifd_max_2600_3b.json b/data/models/godlikehhd_alpaca_data_ifd_max_2600_3b.json deleted file mode 100644 index 0abb30d184b1c28cc5c491ae1751d2d2e2ba9d7d..0000000000000000000000000000000000000000 --- a/data/models/godlikehhd_alpaca_data_ifd_max_2600_3b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "alpaca_data_ifd_max_2600_3B", - "id": "godlikehhd/alpaca_data_ifd_max_2600_3B", - "developer": "godlikehhd", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.086" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/godlikehhd_alpaca_data_ifd_max_2600_3B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2982 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4626 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1594 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2727 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4346 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3288 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/godlikehhd_alpaca_data_ifd_me_max_5200.json b/data/models/godlikehhd_alpaca_data_ifd_me_max_5200.json deleted file mode 100644 index c5288e5ec3bbd0af9aad9ba084199fea0f9e9459..0000000000000000000000000000000000000000 --- a/data/models/godlikehhd_alpaca_data_ifd_me_max_5200.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "alpaca_data_ifd_me_max_5200", - "id": "godlikehhd/alpaca_data_ifd_me_max_5200", - "developer": "godlikehhd", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.544" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/godlikehhd_alpaca_data_ifd_me_max_5200/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3683 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4153 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0974 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2911 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3483 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2982 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/godlikehhd_alpaca_data_ifd_min_2600.json b/data/models/godlikehhd_alpaca_data_ifd_min_2600.json deleted file mode 100644 index 3129de4b96592be18ba58d59f8867338cb8e0dfd..0000000000000000000000000000000000000000 --- a/data/models/godlikehhd_alpaca_data_ifd_min_2600.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "alpaca_data_ifd_min_2600", - "id": "godlikehhd/alpaca_data_ifd_min_2600", - "developer": "godlikehhd", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.544" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/godlikehhd_alpaca_data_ifd_min_2600/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.375 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4219 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0967 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2919 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3656 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2893 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/godlikehhd_alpaca_data_ins_ans_max_5200.json b/data/models/godlikehhd_alpaca_data_ins_ans_max_5200.json deleted file mode 100644 index 0ca81aca302b3093f3631ff5847158ffb24be69f..0000000000000000000000000000000000000000 --- a/data/models/godlikehhd_alpaca_data_ins_ans_max_5200.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "alpaca_data_ins_ans_max_5200", - "id": "godlikehhd/alpaca_data_ins_ans_max_5200", - "developer": "godlikehhd", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.544" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/godlikehhd_alpaca_data_ins_ans_max_5200/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3479 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4098 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1027 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2911 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3602 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2901 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/godlikehhd_alpaca_data_ins_max_5200.json b/data/models/godlikehhd_alpaca_data_ins_max_5200.json deleted file mode 100644 index 7ad0e5b97df6051da6f55f979510695fda8ad37a..0000000000000000000000000000000000000000 --- a/data/models/godlikehhd_alpaca_data_ins_max_5200.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "alpaca_data_ins_max_5200", - "id": "godlikehhd/alpaca_data_ins_max_5200", - "developer": "godlikehhd", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.544" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/godlikehhd_alpaca_data_ins_max_5200/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3275 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4155 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0997 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2961 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3614 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2916 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/godlikehhd_alpaca_data_ins_min_2600.json b/data/models/godlikehhd_alpaca_data_ins_min_2600.json deleted file mode 100644 index 7c7caffdc7d7edee134a41619db5757bd8197763..0000000000000000000000000000000000000000 --- a/data/models/godlikehhd_alpaca_data_ins_min_2600.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "alpaca_data_ins_min_2600", - "id": "godlikehhd/alpaca_data_ins_min_2600", - "developer": "godlikehhd", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.544" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/godlikehhd_alpaca_data_ins_min_2600/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.333 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4187 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.111 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2978 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3853 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.288 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/godlikehhd_alpaca_data_ins_min_5200.json b/data/models/godlikehhd_alpaca_data_ins_min_5200.json deleted file mode 100644 index 41d44203e29fe8514dfa48c9c0e991c8a8c620fe..0000000000000000000000000000000000000000 --- a/data/models/godlikehhd_alpaca_data_ins_min_5200.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "alpaca_data_ins_min_5200", - "id": "godlikehhd/alpaca_data_ins_min_5200", - "developer": "godlikehhd", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.544" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/godlikehhd_alpaca_data_ins_min_5200/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.336 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4289 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1035 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2869 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3906 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2949 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/godlikehhd_alpaca_data_sampled_ifd_5200.json b/data/models/godlikehhd_alpaca_data_sampled_ifd_5200.json deleted file mode 100644 index b7a3f66590799a923fab92273f78141f69ffdc95..0000000000000000000000000000000000000000 --- a/data/models/godlikehhd_alpaca_data_sampled_ifd_5200.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "alpaca_data_sampled_ifd_5200", - "id": "godlikehhd/alpaca_data_sampled_ifd_5200", - "developer": "godlikehhd", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.544" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/godlikehhd_alpaca_data_sampled_ifd_5200/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2924 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4033 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1254 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3087 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3521 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2896 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/godlikehhd_alpaca_data_sampled_ifd_new_5200.json b/data/models/godlikehhd_alpaca_data_sampled_ifd_new_5200.json deleted file mode 100644 index c8f673bd999a037648b01fdc3ceb0bdfd8eb36d6..0000000000000000000000000000000000000000 --- a/data/models/godlikehhd_alpaca_data_sampled_ifd_new_5200.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "alpaca_data_sampled_ifd_new_5200", - "id": "godlikehhd/alpaca_data_sampled_ifd_new_5200", - "developer": "godlikehhd", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.544" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/godlikehhd_alpaca_data_sampled_ifd_new_5200/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3663 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4178 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0944 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2936 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3613 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2925 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/godlikehhd_alpaca_data_score_max_0.1_2600.json b/data/models/godlikehhd_alpaca_data_score_max_0.1_2600.json deleted file mode 100644 index 2f9bb003b94a7308deb2077796162e604c1ec4af..0000000000000000000000000000000000000000 --- a/data/models/godlikehhd_alpaca_data_score_max_0.1_2600.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "alpaca_data_score_max_0.1_2600", - "id": "godlikehhd/alpaca_data_score_max_0.1_2600", - "developer": "godlikehhd", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.544" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/godlikehhd_alpaca_data_score_max_0.1_2600/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3288 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4252 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0989 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2911 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3706 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2923 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/godlikehhd_alpaca_data_score_max_0.3_2600.json b/data/models/godlikehhd_alpaca_data_score_max_0.3_2600.json deleted file mode 100644 index 565689fb074383b95fc8e52abd460deda0865f1c..0000000000000000000000000000000000000000 --- a/data/models/godlikehhd_alpaca_data_score_max_0.3_2600.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "alpaca_data_score_max_0.3_2600", - "id": "godlikehhd/alpaca_data_score_max_0.3_2600", - "developer": "godlikehhd", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.544" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/godlikehhd_alpaca_data_score_max_0.3_2600/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3375 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4151 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1035 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2894 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3759 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2913 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/godlikehhd_alpaca_data_score_max_0.7_2600.json b/data/models/godlikehhd_alpaca_data_score_max_0.7_2600.json deleted file mode 100644 index 3c2cf94c484ab0f76f0289d4e4787029a0909a35..0000000000000000000000000000000000000000 --- a/data/models/godlikehhd_alpaca_data_score_max_0.7_2600.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "alpaca_data_score_max_0.7_2600", - "id": "godlikehhd/alpaca_data_score_max_0.7_2600", - "developer": "godlikehhd", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.544" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/godlikehhd_alpaca_data_score_max_0.7_2600/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.364 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4185 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1073 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3037 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3469 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2983 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/godlikehhd_alpaca_data_score_max_2500.json b/data/models/godlikehhd_alpaca_data_score_max_2500.json deleted file mode 100644 index 0b67d0b40f15f211bb049176b329229496d3dacd..0000000000000000000000000000000000000000 --- a/data/models/godlikehhd_alpaca_data_score_max_2500.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "alpaca_data_score_max_2500", - "id": "godlikehhd/alpaca_data_score_max_2500", - "developer": "godlikehhd", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.544" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/godlikehhd_alpaca_data_score_max_2500/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3564 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.418 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0952 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2953 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3627 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.294 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/godlikehhd_alpaca_data_score_max_2600_3b.json b/data/models/godlikehhd_alpaca_data_score_max_2600_3b.json deleted file mode 100644 index 3fee03bc4290042833a21f6d7db9e51ac1fac628..0000000000000000000000000000000000000000 --- a/data/models/godlikehhd_alpaca_data_score_max_2600_3b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "alpaca_data_score_max_2600_3B", - "id": "godlikehhd/alpaca_data_score_max_2600_3B", - "developer": "godlikehhd", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.086" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/godlikehhd_alpaca_data_score_max_2600_3B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3358 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4716 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1548 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2651 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4474 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3342 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/godlikehhd_alpaca_data_score_max_5200.json b/data/models/godlikehhd_alpaca_data_score_max_5200.json deleted file mode 100644 index accf9d88483f02e31924332848f78e1fda59d14f..0000000000000000000000000000000000000000 --- a/data/models/godlikehhd_alpaca_data_score_max_5200.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "alpaca_data_score_max_5200", - "id": "godlikehhd/alpaca_data_score_max_5200", - "developer": "godlikehhd", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.544" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/godlikehhd_alpaca_data_score_max_5200/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3445 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4242 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0974 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2978 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3878 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2945 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/godlikehhd_ifd_2500_qwen.json b/data/models/godlikehhd_ifd_2500_qwen.json deleted file mode 100644 index 7b70595e14e3814289851aed562e270e1276530a..0000000000000000000000000000000000000000 --- a/data/models/godlikehhd_ifd_2500_qwen.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ifd_2500_qwen", - "id": "godlikehhd/ifd_2500_qwen", - "developer": "godlikehhd", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.544" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/godlikehhd_ifd_2500_qwen/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3365 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4298 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0982 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2953 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3615 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2921 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/godlikehhd_ifd_new_correct_all_sample_2500_qwen.json b/data/models/godlikehhd_ifd_new_correct_all_sample_2500_qwen.json deleted file mode 100644 index 74c93cb083e28d8f26b7def04884e5214be166e4..0000000000000000000000000000000000000000 --- a/data/models/godlikehhd_ifd_new_correct_all_sample_2500_qwen.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ifd_new_correct_all_sample_2500_qwen", - "id": "godlikehhd/ifd_new_correct_all_sample_2500_qwen", - "developer": "godlikehhd", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.544" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/godlikehhd_ifd_new_correct_all_sample_2500_qwen/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3376 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.402 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0959 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2903 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3562 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2889 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/godlikehhd_ifd_new_correct_sample_2500_qwen.json b/data/models/godlikehhd_ifd_new_correct_sample_2500_qwen.json deleted file mode 100644 index 450f91c082664cddd01df7ab91268f2d3b1c48f1..0000000000000000000000000000000000000000 --- a/data/models/godlikehhd_ifd_new_correct_sample_2500_qwen.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ifd_new_correct_sample_2500_qwen", - "id": "godlikehhd/ifd_new_correct_sample_2500_qwen", - "developer": "godlikehhd", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.544" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/godlikehhd_ifd_new_correct_sample_2500_qwen/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3397 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.411 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1042 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3079 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3627 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2932 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/godlikehhd_ifd_new_qwen_2500.json b/data/models/godlikehhd_ifd_new_qwen_2500.json deleted file mode 100644 index 3de93d67535edf577bbeb25bf71398f8aa638e59..0000000000000000000000000000000000000000 --- a/data/models/godlikehhd_ifd_new_qwen_2500.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ifd_new_qwen_2500", - "id": "godlikehhd/ifd_new_qwen_2500", - "developer": "godlikehhd", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.544" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/godlikehhd_ifd_new_qwen_2500/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.324 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.416 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1118 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3003 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.359 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2911 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/godlikehhd_qwen-2.5-1.5b-cherry.json b/data/models/godlikehhd_qwen-2.5-1.5b-cherry.json deleted file mode 100644 index e15c635ca760e992adccdecd6a295105905d6b79..0000000000000000000000000000000000000000 --- a/data/models/godlikehhd_qwen-2.5-1.5b-cherry.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "qwen-2.5-1.5b-cherry", - "id": "godlikehhd/qwen-2.5-1.5b-cherry", - "developer": "godlikehhd", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.772" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/godlikehhd_qwen-2.5-1.5b-cherry/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2893 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4036 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.102 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3003 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3456 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2923 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/godlikehhd_qwen_2.5-1.5b-cherry_new.json b/data/models/godlikehhd_qwen_2.5-1.5b-cherry_new.json deleted file mode 100644 index 7a598a354f87c681cbac7260bb9ce93d336365d6..0000000000000000000000000000000000000000 --- a/data/models/godlikehhd_qwen_2.5-1.5b-cherry_new.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "qwen_2.5-1.5b-cherry_new", - "id": "godlikehhd/qwen_2.5-1.5b-cherry_new", - "developer": "godlikehhd", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.544" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/godlikehhd_qwen_2.5-1.5b-cherry_new/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.312 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.415 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0967 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2978 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3496 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2894 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/godlikehhd_qwen_full_data_alpaca.json b/data/models/godlikehhd_qwen_full_data_alpaca.json deleted file mode 100644 index 283e9779571528eaa41cefbe063b232f0d22c8af..0000000000000000000000000000000000000000 --- a/data/models/godlikehhd_qwen_full_data_alpaca.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "qwen_full_data_alpaca", - "id": "godlikehhd/qwen_full_data_alpaca", - "developer": "godlikehhd", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.544" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/godlikehhd_qwen_full_data_alpaca/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3136 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4229 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0921 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2928 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4052 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2851 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/godlikehhd_qwen_ins_ans_2500.json b/data/models/godlikehhd_qwen_ins_ans_2500.json deleted file mode 100644 index b9c05c210592debff2892a70e84a5135db15537e..0000000000000000000000000000000000000000 --- a/data/models/godlikehhd_qwen_ins_ans_2500.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "qwen_ins_ans_2500", - "id": "godlikehhd/qwen_ins_ans_2500", - "developer": "godlikehhd", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.544" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/godlikehhd_qwen_ins_ans_2500/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2698 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4074 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.114 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2919 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3589 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2809 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/goekdeniz-guelmez_j.o.s.i.e.v4o-1.5b-dpo-stage1-v1.json b/data/models/goekdeniz-guelmez_j.o.s.i.e.v4o-1.5b-dpo-stage1-v1.json deleted file mode 100644 index 8e30d3571cdc8cb979143fa02a0c36f39e820d56..0000000000000000000000000000000000000000 --- a/data/models/goekdeniz-guelmez_j.o.s.i.e.v4o-1.5b-dpo-stage1-v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "j.o.s.i.e.v4o-1.5b-dpo-stage1-v1", - "id": "Goekdeniz-Guelmez/j.o.s.i.e.v4o-1.5b-dpo-stage1-v1", - "developer": "Goekdeniz-Guelmez", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.544" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Goekdeniz-Guelmez_j.o.s.i.e.v4o-1.5b-dpo-stage1-v1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4188 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4124 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1201 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2508 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3529 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2555 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/goekdeniz-guelmez_josie-3b-v6.0.json b/data/models/goekdeniz-guelmez_josie-3b-v6.0.json deleted file mode 100644 index 373152f78597c5b9b79aa399686221c6bfea4fa1..0000000000000000000000000000000000000000 --- a/data/models/goekdeniz-guelmez_josie-3b-v6.0.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "josie-3b-v6.0", - "id": "Goekdeniz-Guelmez/josie-3b-v6.0", - "developer": "Goekdeniz-Guelmez", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.086" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Goekdeniz-Guelmez_josie-3b-v6.0/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.601 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4496 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2938 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2903 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3861 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.322 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/goekdeniz-guelmez_josie-7b-v6.0-step2000.json b/data/models/goekdeniz-guelmez_josie-7b-v6.0-step2000.json deleted file mode 100644 index 4037d46b51c1d5f1b2d23453c409340e1c080518..0000000000000000000000000000000000000000 --- a/data/models/goekdeniz-guelmez_josie-7b-v6.0-step2000.json +++ /dev/null @@ -1,275 +0,0 @@ -{ - "model_info": { - "name": "josie-7b-v6.0-step2000", - "id": "Goekdeniz-Guelmez/josie-7b-v6.0-step2000", - "developer": "Goekdeniz-Guelmez", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Goekdeniz-Guelmez_josie-7b-v6.0-step2000/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7598 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5107 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4237 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2768 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4539 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4012 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "hfopenllm_v2/Goekdeniz-Guelmez_josie-7b-v6.0-step2000/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7628 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5098 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2802 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4579 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4033 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/goekdeniz-guelmez_josie-7b-v6.0.json b/data/models/goekdeniz-guelmez_josie-7b-v6.0.json deleted file mode 100644 index 4297014c8453c5c5552e4879a370e9ec98883abe..0000000000000000000000000000000000000000 --- a/data/models/goekdeniz-guelmez_josie-7b-v6.0.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "josie-7b-v6.0", - "id": "Goekdeniz-Guelmez/josie-7b-v6.0", - "developer": "Goekdeniz-Guelmez", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Goekdeniz-Guelmez_josie-7b-v6.0/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7412 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5105 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4358 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2827 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4154 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3807 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/goekdeniz-guelmez_josiefied-qwen2.5-0.5b-instruct-abliterated-v1.json b/data/models/goekdeniz-guelmez_josiefied-qwen2.5-0.5b-instruct-abliterated-v1.json deleted file mode 100644 index 57e8dc5e192d9afe5d8658b037afb0f25366b018..0000000000000000000000000000000000000000 --- a/data/models/goekdeniz-guelmez_josiefied-qwen2.5-0.5b-instruct-abliterated-v1.json +++ /dev/null @@ -1,275 +0,0 @@ -{ - "model_info": { - "name": "Josiefied-Qwen2.5-0.5B-Instruct-abliterated-v1", - "id": "Goekdeniz-Guelmez/Josiefied-Qwen2.5-0.5B-Instruct-abliterated-v1", - "developer": "Goekdeniz-Guelmez", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Goekdeniz-Guelmez_Josiefied-Qwen2.5-0.5B-Instruct-abliterated-v1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3417 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3292 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0023 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2576 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3249 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1638 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "hfopenllm_v2/Goekdeniz-Guelmez_Josiefied-Qwen2.5-0.5B-Instruct-abliterated-v1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3472 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3268 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0891 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2517 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3262 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1641 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/goekdeniz-guelmez_josiefied-qwen2.5-1.5b-instruct-abliterated-v1.json b/data/models/goekdeniz-guelmez_josiefied-qwen2.5-1.5b-instruct-abliterated-v1.json deleted file mode 100644 index b608a2c9ac0f44eb55d33542096c5b65e058019d..0000000000000000000000000000000000000000 --- a/data/models/goekdeniz-guelmez_josiefied-qwen2.5-1.5b-instruct-abliterated-v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Josiefied-Qwen2.5-1.5B-Instruct-abliterated-v1", - "id": "Goekdeniz-Guelmez/Josiefied-Qwen2.5-1.5B-Instruct-abliterated-v1", - "developer": "Goekdeniz-Guelmez", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.777" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Goekdeniz-Guelmez_Josiefied-Qwen2.5-1.5B-Instruct-abliterated-v1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4769 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4186 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2085 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2433 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3675 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2783 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/goekdeniz-guelmez_josiefied-qwen2.5-1.5b-instruct-abliterated-v2.json b/data/models/goekdeniz-guelmez_josiefied-qwen2.5-1.5b-instruct-abliterated-v2.json deleted file mode 100644 index 7d3d1de0c90b9291c6e2234c09b11ee0eda43a31..0000000000000000000000000000000000000000 --- a/data/models/goekdeniz-guelmez_josiefied-qwen2.5-1.5b-instruct-abliterated-v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Josiefied-Qwen2.5-1.5B-Instruct-abliterated-v2", - "id": "Goekdeniz-Guelmez/Josiefied-Qwen2.5-1.5B-Instruct-abliterated-v2", - "developer": "Goekdeniz-Guelmez", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.544" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Goekdeniz-Guelmez_Josiefied-Qwen2.5-1.5B-Instruct-abliterated-v2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4216 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4042 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1269 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2399 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3769 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2562 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/goekdeniz-guelmez_josiefied-qwen2.5-1.5b-instruct-abliterated-v3.json b/data/models/goekdeniz-guelmez_josiefied-qwen2.5-1.5b-instruct-abliterated-v3.json deleted file mode 100644 index 19056266255a2a3ed9a9120b2eaa9c398113a834..0000000000000000000000000000000000000000 --- a/data/models/goekdeniz-guelmez_josiefied-qwen2.5-1.5b-instruct-abliterated-v3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Josiefied-Qwen2.5-1.5B-Instruct-abliterated-v3", - "id": "Goekdeniz-Guelmez/Josiefied-Qwen2.5-1.5B-Instruct-abliterated-v3", - "developer": "Goekdeniz-Guelmez", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.544" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Goekdeniz-Guelmez_Josiefied-Qwen2.5-1.5B-Instruct-abliterated-v3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4253 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4053 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1307 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2433 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3702 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2556 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/goekdeniz-guelmez_josiefied-qwen2.5-14b-instruct-abliterated-v4.json b/data/models/goekdeniz-guelmez_josiefied-qwen2.5-14b-instruct-abliterated-v4.json deleted file mode 100644 index 6cb4d0435f70a5c3eb0303718a8329a939143e88..0000000000000000000000000000000000000000 --- a/data/models/goekdeniz-guelmez_josiefied-qwen2.5-14b-instruct-abliterated-v4.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Josiefied-Qwen2.5-14B-Instruct-abliterated-v4", - "id": "Goekdeniz-Guelmez/Josiefied-Qwen2.5-14B-Instruct-abliterated-v4", - "developer": "Goekdeniz-Guelmez", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Goekdeniz-Guelmez_Josiefied-Qwen2.5-14B-Instruct-abliterated-v4/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8292 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6356 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5423 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3423 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4287 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5018 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/goekdeniz-guelmez_josiefied-qwen2.5-7b-instruct-abliterated-v2.json b/data/models/goekdeniz-guelmez_josiefied-qwen2.5-7b-instruct-abliterated-v2.json deleted file mode 100644 index 12d7f55be860c1c4faaea7377a612d3996e8c15a..0000000000000000000000000000000000000000 --- a/data/models/goekdeniz-guelmez_josiefied-qwen2.5-7b-instruct-abliterated-v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Josiefied-Qwen2.5-7B-Instruct-abliterated-v2", - "id": "Goekdeniz-Guelmez/Josiefied-Qwen2.5-7B-Instruct-abliterated-v2", - "developer": "Goekdeniz-Guelmez", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Goekdeniz-Guelmez_Josiefied-Qwen2.5-7B-Instruct-abliterated-v2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7814 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.531 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4532 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2987 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4354 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.412 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/google_codegemma-1.1-2b.json b/data/models/google_codegemma-1.1-2b.json deleted file mode 100644 index ef880f376655796b793b8f565906e7a1f4ff454d..0000000000000000000000000000000000000000 --- a/data/models/google_codegemma-1.1-2b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "codegemma-1.1-2b", - "id": "google/codegemma-1.1-2b", - "developer": "Google", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "GemmaForCausalLM", - "params_billions": "2.506" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/google_codegemma-1.1-2b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2294 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3353 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0128 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2651 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3871 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1278 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/google_flame-1.0-24b-july-2024.json b/data/models/google_flame-1.0-24b-july-2024.json deleted file mode 100644 index 97bc652452f8a71e88032f938af415772c3e63d2..0000000000000000000000000000000000000000 --- a/data/models/google_flame-1.0-24b-july-2024.json +++ /dev/null @@ -1,126 +0,0 @@ -{ - "model_info": { - "name": "google/flame-1.0-24B-july-2024", - "id": "google/flame-1.0-24B-july-2024", - "developer": "Google", - "additional_details": { - "model_type": "Generative" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/google_flame-1.0-24B-july-2024/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8781 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9218 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7566 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8959 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.938 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/google_flan-t5-base.json b/data/models/google_flan-t5-base.json deleted file mode 100644 index 2fa8953d3d2c4176a78f8add5f45e72611a8b91c..0000000000000000000000000000000000000000 --- a/data/models/google_flan-t5-base.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "flan-t5-base", - "id": "google/flan-t5-base", - "developer": "Google", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "T5ForConditionalGeneration", - "params_billions": "0.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/google_flan-t5-base/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1891 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3526 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0106 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2383 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3671 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1357 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/google_flan-t5-large.json b/data/models/google_flan-t5-large.json deleted file mode 100644 index f7fd974c1f3fd98adb4eca165d6add2397f56458..0000000000000000000000000000000000000000 --- a/data/models/google_flan-t5-large.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "flan-t5-large", - "id": "google/flan-t5-large", - "developer": "Google", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "T5ForConditionalGeneration", - "params_billions": "0.783" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/google_flan-t5-large/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2201 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4153 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0144 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2508 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4083 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1709 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/google_flan-t5-small.json b/data/models/google_flan-t5-small.json deleted file mode 100644 index 1d891a9dd2372b9d06c6201b6c70ba9bd3343f95..0000000000000000000000000000000000000000 --- a/data/models/google_flan-t5-small.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "flan-t5-small", - "id": "google/flan-t5-small", - "developer": "Google", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "T5ForConditionalGeneration", - "params_billions": "0.077" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/google_flan-t5-small/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1524 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3283 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0076 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2609 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4123 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1233 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/google_flan-t5-xl.json b/data/models/google_flan-t5-xl.json deleted file mode 100644 index f4504e45576f272b296c7e91d2af835d208d50a9..0000000000000000000000000000000000000000 --- a/data/models/google_flan-t5-xl.json +++ /dev/null @@ -1,275 +0,0 @@ -{ - "model_info": { - "name": "flan-t5-xl", - "id": "google/flan-t5-xl", - "developer": "Google", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "T5ForConditionalGeneration", - "params_billions": "2.85" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/google_flan-t5-xl/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2237 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4531 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0076 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2525 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4181 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2147 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "hfopenllm_v2/google_flan-t5-xl/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2207 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4537 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0008 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2458 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.422 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2142 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/google_flan-t5-xxl.json b/data/models/google_flan-t5-xxl.json deleted file mode 100644 index faa3683965638fce41f229c76dc516376a86ede2..0000000000000000000000000000000000000000 --- a/data/models/google_flan-t5-xxl.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "flan-t5-xxl", - "id": "google/flan-t5-xxl", - "developer": "Google", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "T5ForConditionalGeneration", - "params_billions": "11.267" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/google_flan-t5-xxl/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.22 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5066 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0106 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2701 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4218 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2343 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/google_flan-ul2.json b/data/models/google_flan-ul2.json deleted file mode 100644 index a2ae3b5839e5777d50a8bc5f827d3213e3606f57..0000000000000000000000000000000000000000 --- a/data/models/google_flan-ul2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "flan-ul2", - "id": "google/flan-ul2", - "developer": "Google", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "T5ForConditionalGeneration", - "params_billions": "19.46" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/google_flan-ul2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2393 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5054 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0091 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2878 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3844 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2493 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/google_gemini-1.0-pro-001.json b/data/models/google_gemini-1.0-pro-001.json deleted file mode 100644 index 3ed3b975792e1506e181943e830831d74b31b94e..0000000000000000000000000000000000000000 --- a/data/models/google_gemini-1.0-pro-001.json +++ /dev/null @@ -1,1531 +0,0 @@ -{ - "model_info": { - "name": "Gemini 1.0 Pro 001", - "id": "google/gemini-1.0-pro-001", - "developer": "Google", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_mmlu/google_gemini-1.0-pro-001/1774096312.00548", - "retrieved_timestamp": "1774096312.00548", - "source_metadata": { - "source_name": "helm_mmlu", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_mmlu", - "evaluation_results": [ - { - "evaluation_name": "MMLU All Subjects", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU All Subjects", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7, - "details": { - "description": "min=0.333, mean=0.7, max=0.933, sum=79.795 (114)", - "tab": "Accuracy", - "MMLU All Subjects - Observed inference time (s)": "{\"description\": \"min=0.291, mean=0.385, max=0.991, sum=43.868 (114)\", \"tab\": \"Efficiency\", \"score\": \"0.3848050244039386\"}", - "MMLU All Subjects - # eval": "{\"description\": \"min=100, mean=246.351, max=1534, sum=28084 (114)\", \"tab\": \"General information\", \"score\": \"246.35087719298247\"}", - "MMLU All Subjects - # train": "{\"description\": \"min=5, mean=5, max=5, sum=570 (114)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU All Subjects - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU All Subjects - # prompt tokens": "{\"description\": \"min=260.164, mean=624.617, max=2789.424, sum=71206.345 (114)\", \"tab\": \"General information\", \"score\": \"624.6170571214202\"}", - "MMLU All Subjects - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_name": "Abstract Algebra", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Abstract Algebra", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.34, - "details": { - "description": "min=0.34, mean=0.34, max=0.34, sum=0.68 (2)", - "tab": "Accuracy", - "Abstract Algebra - Observed inference time (s)": "{\"description\": \"min=0.991, mean=0.991, max=0.991, sum=1.982 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9907678151130677\"}", - "Abstract Algebra - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Abstract Algebra - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Abstract Algebra - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Abstract Algebra - # prompt tokens": "{\"description\": \"min=375.97, mean=375.97, max=375.97, sum=751.94 (2)\", \"tab\": \"General information\", \"score\": \"375.97\"}", - "Abstract Algebra - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"abstract_algebra\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_abstract_algebra\"" - } - } - }, - { - "evaluation_name": "Anatomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Anatomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.652, - "details": { - "description": "min=0.652, mean=0.652, max=0.652, sum=1.304 (2)", - "tab": "Accuracy", - "Anatomy - Observed inference time (s)": "{\"description\": \"min=0.318, mean=0.318, max=0.318, sum=0.636 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3178748925526937\"}", - "Anatomy - # eval": "{\"description\": \"min=135, mean=135, max=135, sum=270 (2)\", \"tab\": \"General information\", \"score\": \"135.0\"}", - "Anatomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Anatomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Anatomy - # prompt tokens": "{\"description\": \"min=336.356, mean=336.356, max=336.356, sum=672.711 (2)\", \"tab\": \"General information\", \"score\": \"336.35555555555555\"}", - "Anatomy - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"anatomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_anatomy\"" - } - } - }, - { - "evaluation_name": "College Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on College Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.333, - "details": { - "description": "min=0.333, mean=0.333, max=0.333, sum=0.667 (2)", - "tab": "Accuracy", - "College Chemistry - Observed inference time (s)": "{\"description\": \"min=0.377, mean=0.377, max=0.377, sum=0.754 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.37708688735961915\"}", - "College Biology - Observed inference time (s)": "{\"description\": \"min=0.294, mean=0.294, max=0.294, sum=0.588 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2937609056631724\"}", - "College Computer Science - Observed inference time (s)": "{\"description\": \"min=0.375, mean=0.375, max=0.375, sum=0.75 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.37500447273254395\"}", - "College Mathematics - Observed inference time (s)": "{\"description\": \"min=0.356, mean=0.356, max=0.356, sum=0.712 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.35595274686813355\"}", - "College Medicine - Observed inference time (s)": "{\"description\": \"min=0.314, mean=0.314, max=0.314, sum=0.627 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.31358790535458253\"}", - "College Physics - Observed inference time (s)": "{\"description\": \"min=0.336, mean=0.336, max=0.336, sum=0.672 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3357745151893765\"}", - "College Chemistry - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Chemistry - # prompt tokens": "{\"description\": \"min=562.02, mean=562.02, max=562.02, sum=1124.04 (2)\", \"tab\": \"General information\", \"score\": \"562.02\"}", - "College Chemistry - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # eval": "{\"description\": \"min=144, mean=144, max=144, sum=288 (2)\", \"tab\": \"General information\", \"score\": \"144.0\"}", - "College Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # prompt tokens": "{\"description\": \"min=474.799, mean=474.799, max=474.799, sum=949.597 (2)\", \"tab\": \"General information\", \"score\": \"474.7986111111111\"}", - "College Biology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # prompt tokens": "{\"description\": \"min=849.86, mean=849.86, max=849.86, sum=1699.72 (2)\", \"tab\": \"General information\", \"score\": \"849.86\"}", - "College Computer Science - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # prompt tokens": "{\"description\": \"min=618.69, mean=618.69, max=618.69, sum=1237.38 (2)\", \"tab\": \"General information\", \"score\": \"618.69\"}", - "College Mathematics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # eval": "{\"description\": \"min=173, mean=173, max=173, sum=346 (2)\", \"tab\": \"General information\", \"score\": \"173.0\"}", - "College Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # prompt tokens": "{\"description\": \"min=505.37, mean=505.37, max=505.37, sum=1010.74 (2)\", \"tab\": \"General information\", \"score\": \"505.3699421965318\"}", - "College Medicine - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # eval": "{\"description\": \"min=102, mean=102, max=102, sum=204 (2)\", \"tab\": \"General information\", \"score\": \"102.0\"}", - "College Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # prompt tokens": "{\"description\": \"min=499.471, mean=499.471, max=499.471, sum=998.941 (2)\", \"tab\": \"General information\", \"score\": \"499.47058823529414\"}", - "College Physics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"college_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_college_physics\"" - } - } - }, - { - "evaluation_name": "Computer Security", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Computer Security", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.84, - "details": { - "description": "min=0.84, mean=0.84, max=0.84, sum=1.68 (2)", - "tab": "Accuracy", - "Computer Security - Observed inference time (s)": "{\"description\": \"min=0.314, mean=0.314, max=0.314, sum=0.627 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.31363418102264407\"}", - "Computer Security - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Computer Security - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Computer Security - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Computer Security - # prompt tokens": "{\"description\": \"min=372.91, mean=372.91, max=372.91, sum=745.82 (2)\", \"tab\": \"General information\", \"score\": \"372.91\"}", - "Computer Security - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"computer_security\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_computer_security\"" - } - } - }, - { - "evaluation_name": "Econometrics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Econometrics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.553, - "details": { - "description": "min=0.553, mean=0.553, max=0.553, sum=1.105 (2)", - "tab": "Accuracy", - "Econometrics - Observed inference time (s)": "{\"description\": \"min=0.377, mean=0.377, max=0.377, sum=0.754 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.37716702620188397\"}", - "Econometrics - # eval": "{\"description\": \"min=114, mean=114, max=114, sum=228 (2)\", \"tab\": \"General information\", \"score\": \"114.0\"}", - "Econometrics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Econometrics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Econometrics - # prompt tokens": "{\"description\": \"min=626.553, mean=626.553, max=626.553, sum=1253.105 (2)\", \"tab\": \"General information\", \"score\": \"626.5526315789474\"}", - "Econometrics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"econometrics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_econometrics\"" - } - } - }, - { - "evaluation_name": "Global Facts", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Global Facts", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.49, - "details": { - "description": "min=0.49, mean=0.49, max=0.49, sum=0.98 (2)", - "tab": "Accuracy", - "Global Facts - Observed inference time (s)": "{\"description\": \"min=0.32, mean=0.32, max=0.32, sum=0.639 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3196276807785034\"}", - "Global Facts - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Global Facts - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Global Facts - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Global Facts - # prompt tokens": "{\"description\": \"min=448.54, mean=448.54, max=448.54, sum=897.08 (2)\", \"tab\": \"General information\", \"score\": \"448.54\"}", - "Global Facts - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"global_facts\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_global_facts\"" - } - } - }, - { - "evaluation_name": "Jurisprudence", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Jurisprudence", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.861, - "details": { - "description": "min=0.861, mean=0.861, max=0.861, sum=1.722 (2)", - "tab": "Accuracy", - "Jurisprudence - Observed inference time (s)": "{\"description\": \"min=0.299, mean=0.299, max=0.299, sum=0.598 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.29897612112539784\"}", - "Jurisprudence - # eval": "{\"description\": \"min=108, mean=108, max=108, sum=216 (2)\", \"tab\": \"General information\", \"score\": \"108.0\"}", - "Jurisprudence - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Jurisprudence - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Jurisprudence - # prompt tokens": "{\"description\": \"min=399.87, mean=399.87, max=399.87, sum=799.741 (2)\", \"tab\": \"General information\", \"score\": \"399.8703703703704\"}", - "Jurisprudence - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"jurisprudence\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_jurisprudence\"" - } - } - }, - { - "evaluation_name": "Philosophy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Philosophy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.762, - "details": { - "description": "min=0.762, mean=0.762, max=0.762, sum=1.524 (2)", - "tab": "Accuracy", - "Philosophy - Observed inference time (s)": "{\"description\": \"min=0.318, mean=0.318, max=0.318, sum=0.636 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.31779951221306607\"}", - "Philosophy - # eval": "{\"description\": \"min=311, mean=311, max=311, sum=622 (2)\", \"tab\": \"General information\", \"score\": \"311.0\"}", - "Philosophy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Philosophy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Philosophy - # prompt tokens": "{\"description\": \"min=332.907, mean=332.907, max=332.907, sum=665.814 (2)\", \"tab\": \"General information\", \"score\": \"332.90675241157555\"}", - "Philosophy - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"philosophy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_philosophy\"" - } - } - }, - { - "evaluation_name": "Professional Psychology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Professional Psychology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.752, - "details": { - "description": "min=0.752, mean=0.752, max=0.752, sum=1.503 (2)", - "tab": "Accuracy", - "Professional Medicine - Observed inference time (s)": "{\"description\": \"min=0.498, mean=0.498, max=0.498, sum=0.997 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.49840929939298173\"}", - "Professional Accounting - Observed inference time (s)": "{\"description\": \"min=0.384, mean=0.384, max=0.384, sum=0.768 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3838615434389588\"}", - "Professional Law - Observed inference time (s)": "{\"description\": \"min=0.509, mean=0.509, max=0.509, sum=1.019 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5094701207541172\"}", - "Professional Psychology - Observed inference time (s)": "{\"description\": \"min=0.388, mean=0.388, max=0.388, sum=0.775 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3877133719230953\"}", - "Professional Medicine - # eval": "{\"description\": \"min=272, mean=272, max=272, sum=544 (2)\", \"tab\": \"General information\", \"score\": \"272.0\"}", - "Professional Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Medicine - # prompt tokens": "{\"description\": \"min=1105.092, mean=1105.092, max=1105.092, sum=2210.184 (2)\", \"tab\": \"General information\", \"score\": \"1105.0919117647059\"}", - "Professional Medicine - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # eval": "{\"description\": \"min=282, mean=282, max=282, sum=564 (2)\", \"tab\": \"General information\", \"score\": \"282.0\"}", - "Professional Accounting - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Accounting - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # prompt tokens": "{\"description\": \"min=747.418, mean=747.418, max=747.418, sum=1494.837 (2)\", \"tab\": \"General information\", \"score\": \"747.418439716312\"}", - "Professional Accounting - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # eval": "{\"description\": \"min=1534, mean=1534, max=1534, sum=3068 (2)\", \"tab\": \"General information\", \"score\": \"1534.0\"}", - "Professional Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # prompt tokens": "{\"description\": \"min=1677.119, mean=1677.119, max=1677.119, sum=3354.239 (2)\", \"tab\": \"General information\", \"score\": \"1677.119295958279\"}", - "Professional Law - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # eval": "{\"description\": \"min=612, mean=612, max=612, sum=1224 (2)\", \"tab\": \"General information\", \"score\": \"612.0\"}", - "Professional Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # prompt tokens": "{\"description\": \"min=586.363, mean=586.363, max=586.363, sum=1172.725 (2)\", \"tab\": \"General information\", \"score\": \"586.3627450980392\"}", - "Professional Psychology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"professional_psychology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_professional_psychology\"" - } - } - }, - { - "evaluation_name": "Us Foreign Policy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Us Foreign Policy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.89, - "details": { - "description": "min=0.89, mean=0.89, max=0.89, sum=1.78 (2)", - "tab": "Accuracy", - "Us Foreign Policy - Observed inference time (s)": "{\"description\": \"min=0.306, mean=0.306, max=0.306, sum=0.611 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.30568787574768064\"}", - "Us Foreign Policy - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Us Foreign Policy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Us Foreign Policy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Us Foreign Policy - # prompt tokens": "{\"description\": \"min=430.2, mean=430.2, max=430.2, sum=860.4 (2)\", \"tab\": \"General information\", \"score\": \"430.2\"}", - "Us Foreign Policy - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"us_foreign_policy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_us_foreign_policy\"" - } - } - }, - { - "evaluation_name": "Astronomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Astronomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.796, - "details": { - "description": "min=0.796, mean=0.796, max=0.796, sum=1.592 (2)", - "tab": "Accuracy", - "Astronomy - Observed inference time (s)": "{\"description\": \"min=0.517, mean=0.517, max=0.517, sum=1.035 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5173565070880087\"}", - "Astronomy - # eval": "{\"description\": \"min=152, mean=152, max=152, sum=304 (2)\", \"tab\": \"General information\", \"score\": \"152.0\"}", - "Astronomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Astronomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Astronomy - # prompt tokens": "{\"description\": \"min=594.421, mean=594.421, max=594.421, sum=1188.842 (2)\", \"tab\": \"General information\", \"score\": \"594.421052631579\"}", - "Astronomy - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"astronomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_astronomy\"" - } - } - }, - { - "evaluation_name": "Business Ethics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Business Ethics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.69, - "details": { - "description": "min=0.69, mean=0.69, max=0.69, sum=1.38 (2)", - "tab": "Accuracy", - "Business Ethics - Observed inference time (s)": "{\"description\": \"min=0.386, mean=0.386, max=0.386, sum=0.772 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.38599337100982667\"}", - "Business Ethics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Business Ethics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Business Ethics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Business Ethics - # prompt tokens": "{\"description\": \"min=544.87, mean=544.87, max=544.87, sum=1089.74 (2)\", \"tab\": \"General information\", \"score\": \"544.87\"}", - "Business Ethics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"business_ethics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_business_ethics\"" - } - } - }, - { - "evaluation_name": "Clinical Knowledge", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Clinical Knowledge", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.758, - "details": { - "description": "min=0.758, mean=0.758, max=0.758, sum=1.517 (2)", - "tab": "Accuracy", - "Clinical Knowledge - Observed inference time (s)": "{\"description\": \"min=0.299, mean=0.299, max=0.299, sum=0.599 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.29948959980370865\"}", - "Clinical Knowledge - # eval": "{\"description\": \"min=265, mean=265, max=265, sum=530 (2)\", \"tab\": \"General information\", \"score\": \"265.0\"}", - "Clinical Knowledge - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Clinical Knowledge - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Clinical Knowledge - # prompt tokens": "{\"description\": \"min=394.592, mean=394.592, max=394.592, sum=789.185 (2)\", \"tab\": \"General information\", \"score\": \"394.5924528301887\"}", - "Clinical Knowledge - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"clinical_knowledge\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_clinical_knowledge\"" - } - } - }, - { - "evaluation_name": "Conceptual Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Conceptual Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.706, - "details": { - "description": "min=0.706, mean=0.706, max=0.706, sum=1.413 (2)", - "tab": "Accuracy", - "Conceptual Physics - Observed inference time (s)": "{\"description\": \"min=0.294, mean=0.294, max=0.294, sum=0.588 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.29394423606547904\"}", - "Conceptual Physics - # eval": "{\"description\": \"min=235, mean=235, max=235, sum=470 (2)\", \"tab\": \"General information\", \"score\": \"235.0\"}", - "Conceptual Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Conceptual Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Conceptual Physics - # prompt tokens": "{\"description\": \"min=301.213, mean=301.213, max=301.213, sum=602.426 (2)\", \"tab\": \"General information\", \"score\": \"301.21276595744683\"}", - "Conceptual Physics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"conceptual_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_conceptual_physics\"" - } - } - }, - { - "evaluation_name": "Electrical Engineering", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Electrical Engineering", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.69, - "details": { - "description": "min=0.69, mean=0.69, max=0.69, sum=1.379 (2)", - "tab": "Accuracy", - "Electrical Engineering - Observed inference time (s)": "{\"description\": \"min=0.291, mean=0.291, max=0.291, sum=0.581 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2906524740416428\"}", - "Electrical Engineering - # eval": "{\"description\": \"min=145, mean=145, max=145, sum=290 (2)\", \"tab\": \"General information\", \"score\": \"145.0\"}", - "Electrical Engineering - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Electrical Engineering - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Electrical Engineering - # prompt tokens": "{\"description\": \"min=466.786, mean=466.786, max=466.786, sum=933.572 (2)\", \"tab\": \"General information\", \"score\": \"466.78620689655173\"}", - "Electrical Engineering - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"electrical_engineering\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_electrical_engineering\"" - } - } - }, - { - "evaluation_name": "Elementary Mathematics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Elementary Mathematics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.476, - "details": { - "description": "min=0.476, mean=0.476, max=0.476, sum=0.952 (2)", - "tab": "Accuracy", - "Elementary Mathematics - Observed inference time (s)": "{\"description\": \"min=0.393, mean=0.393, max=0.393, sum=0.786 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3928584957879687\"}", - "Elementary Mathematics - # eval": "{\"description\": \"min=378, mean=378, max=378, sum=756 (2)\", \"tab\": \"General information\", \"score\": \"378.0\"}", - "Elementary Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Elementary Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Elementary Mathematics - # prompt tokens": "{\"description\": \"min=589.341, mean=589.341, max=589.341, sum=1178.683 (2)\", \"tab\": \"General information\", \"score\": \"589.3412698412699\"}", - "Elementary Mathematics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"elementary_mathematics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_elementary_mathematics\"" - } - } - }, - { - "evaluation_name": "Formal Logic", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Formal Logic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.468, - "details": { - "description": "min=0.468, mean=0.468, max=0.468, sum=0.937 (2)", - "tab": "Accuracy", - "Formal Logic - Observed inference time (s)": "{\"description\": \"min=0.398, mean=0.398, max=0.398, sum=0.797 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.39849274120633565\"}", - "Formal Logic - # eval": "{\"description\": \"min=126, mean=126, max=126, sum=252 (2)\", \"tab\": \"General information\", \"score\": \"126.0\"}", - "Formal Logic - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Formal Logic - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Formal Logic - # prompt tokens": "{\"description\": \"min=611.563, mean=611.563, max=611.563, sum=1223.127 (2)\", \"tab\": \"General information\", \"score\": \"611.563492063492\"}", - "Formal Logic - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"formal_logic\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_formal_logic\"" - } - } - }, - { - "evaluation_name": "High School World History", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on High School World History", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.865, - "details": { - "description": "min=0.865, mean=0.865, max=0.865, sum=1.73 (2)", - "tab": "Accuracy", - "High School Biology - Observed inference time (s)": "{\"description\": \"min=0.321, mean=0.321, max=0.321, sum=0.643 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3214967135460146\"}", - "High School Chemistry - Observed inference time (s)": "{\"description\": \"min=0.341, mean=0.341, max=0.341, sum=0.683 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3413804282108551\"}", - "High School Computer Science - Observed inference time (s)": "{\"description\": \"min=0.378, mean=0.378, max=0.378, sum=0.756 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.37822843074798584\"}", - "High School European History - Observed inference time (s)": "{\"description\": \"min=0.836, mean=0.836, max=0.836, sum=1.672 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.836203297701749\"}", - "High School Geography - Observed inference time (s)": "{\"description\": \"min=0.321, mean=0.321, max=0.321, sum=0.642 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3208902616693516\"}", - "High School Government And Politics - Observed inference time (s)": "{\"description\": \"min=0.307, mean=0.307, max=0.307, sum=0.614 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3069849088401992\"}", - "High School Macroeconomics - Observed inference time (s)": "{\"description\": \"min=0.32, mean=0.32, max=0.32, sum=0.641 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.32043021275446965\"}", - "High School Mathematics - Observed inference time (s)": "{\"description\": \"min=0.386, mean=0.386, max=0.386, sum=0.772 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.38611255663412586\"}", - "High School Microeconomics - Observed inference time (s)": "{\"description\": \"min=0.315, mean=0.315, max=0.315, sum=0.631 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.31541170993772877\"}", - "High School Physics - Observed inference time (s)": "{\"description\": \"min=0.383, mean=0.383, max=0.383, sum=0.767 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3833695673784673\"}", - "High School Psychology - Observed inference time (s)": "{\"description\": \"min=0.334, mean=0.334, max=0.334, sum=0.668 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.33389012427891\"}", - "High School Statistics - Observed inference time (s)": "{\"description\": \"min=0.4, mean=0.4, max=0.4, sum=0.8 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.39985558611375316\"}", - "High School US History - Observed inference time (s)": "{\"description\": \"min=0.827, mean=0.827, max=0.827, sum=1.655 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8272603574921104\"}", - "High School World History - Observed inference time (s)": "{\"description\": \"min=0.517, mean=0.517, max=0.517, sum=1.035 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5172926987273784\"}", - "High School Biology - # eval": "{\"description\": \"min=310, mean=310, max=310, sum=620 (2)\", \"tab\": \"General information\", \"score\": \"310.0\"}", - "High School Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Biology - # prompt tokens": "{\"description\": \"min=492.958, mean=492.958, max=492.958, sum=985.916 (2)\", \"tab\": \"General information\", \"score\": \"492.958064516129\"}", - "High School Biology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # eval": "{\"description\": \"min=203, mean=203, max=203, sum=406 (2)\", \"tab\": \"General information\", \"score\": \"203.0\"}", - "High School Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # prompt tokens": "{\"description\": \"min=505.064, mean=505.064, max=505.064, sum=1010.128 (2)\", \"tab\": \"General information\", \"score\": \"505.064039408867\"}", - "High School Chemistry - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "High School Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # prompt tokens": "{\"description\": \"min=927.13, mean=927.13, max=927.13, sum=1854.26 (2)\", \"tab\": \"General information\", \"score\": \"927.13\"}", - "High School Computer Science - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # eval": "{\"description\": \"min=165, mean=165, max=165, sum=330 (2)\", \"tab\": \"General information\", \"score\": \"165.0\"}", - "High School European History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School European History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # prompt tokens": "{\"description\": \"min=2789.424, mean=2789.424, max=2789.424, sum=5578.848 (2)\", \"tab\": \"General information\", \"score\": \"2789.4242424242425\"}", - "High School European History - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # eval": "{\"description\": \"min=198, mean=198, max=198, sum=396 (2)\", \"tab\": \"General information\", \"score\": \"198.0\"}", - "High School Geography - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Geography - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # prompt tokens": "{\"description\": \"min=386.773, mean=386.773, max=386.773, sum=773.545 (2)\", \"tab\": \"General information\", \"score\": \"386.77272727272725\"}", - "High School Geography - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # eval": "{\"description\": \"min=193, mean=193, max=193, sum=386 (2)\", \"tab\": \"General information\", \"score\": \"193.0\"}", - "High School Government And Politics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Government And Politics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # prompt tokens": "{\"description\": \"min=471.301, mean=471.301, max=471.301, sum=942.601 (2)\", \"tab\": \"General information\", \"score\": \"471.30051813471505\"}", - "High School Government And Politics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # eval": "{\"description\": \"min=390, mean=390, max=390, sum=780 (2)\", \"tab\": \"General information\", \"score\": \"390.0\"}", - "High School Macroeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Macroeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # prompt tokens": "{\"description\": \"min=388.541, mean=388.541, max=388.541, sum=777.082 (2)\", \"tab\": \"General information\", \"score\": \"388.54102564102567\"}", - "High School Macroeconomics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # eval": "{\"description\": \"min=270, mean=270, max=270, sum=540 (2)\", \"tab\": \"General information\", \"score\": \"270.0\"}", - "High School Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # prompt tokens": "{\"description\": \"min=558.822, mean=558.822, max=558.822, sum=1117.644 (2)\", \"tab\": \"General information\", \"score\": \"558.8222222222222\"}", - "High School Mathematics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # eval": "{\"description\": \"min=238, mean=238, max=238, sum=476 (2)\", \"tab\": \"General information\", \"score\": \"238.0\"}", - "High School Microeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Microeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # prompt tokens": "{\"description\": \"min=407.954, mean=407.954, max=407.954, sum=815.908 (2)\", \"tab\": \"General information\", \"score\": \"407.953781512605\"}", - "High School Microeconomics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # eval": "{\"description\": \"min=151, mean=151, max=151, sum=302 (2)\", \"tab\": \"General information\", \"score\": \"151.0\"}", - "High School Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # prompt tokens": "{\"description\": \"min=583.715, mean=583.715, max=583.715, sum=1167.43 (2)\", \"tab\": \"General information\", \"score\": \"583.7152317880794\"}", - "High School Physics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # eval": "{\"description\": \"min=545, mean=545, max=545, sum=1090 (2)\", \"tab\": \"General information\", \"score\": \"545.0\"}", - "High School Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # prompt tokens": "{\"description\": \"min=494.604, mean=494.604, max=494.604, sum=989.207 (2)\", \"tab\": \"General information\", \"score\": \"494.60366972477067\"}", - "High School Psychology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # eval": "{\"description\": \"min=216, mean=216, max=216, sum=432 (2)\", \"tab\": \"General information\", \"score\": \"216.0\"}", - "High School Statistics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Statistics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # prompt tokens": "{\"description\": \"min=850.931, mean=850.931, max=850.931, sum=1701.861 (2)\", \"tab\": \"General information\", \"score\": \"850.9305555555555\"}", - "High School Statistics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # eval": "{\"description\": \"min=204, mean=204, max=204, sum=408 (2)\", \"tab\": \"General information\", \"score\": \"204.0\"}", - "High School US History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School US History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # prompt tokens": "{\"description\": \"min=2197.583, mean=2197.583, max=2197.583, sum=4395.167 (2)\", \"tab\": \"General information\", \"score\": \"2197.5833333333335\"}", - "High School US History - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # eval": "{\"description\": \"min=237, mean=237, max=237, sum=474 (2)\", \"tab\": \"General information\", \"score\": \"237.0\"}", - "High School World History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School World History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # prompt tokens": "{\"description\": \"min=1418.544, mean=1418.544, max=1418.544, sum=2837.089 (2)\", \"tab\": \"General information\", \"score\": \"1418.5443037974683\"}", - "High School World History - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"high_school_world_history\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_high_school_world_history\"" - } - } - }, - { - "evaluation_name": "Human Sexuality", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Human Sexuality", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.618, - "details": { - "description": "min=0.618, mean=0.618, max=0.618, sum=1.237 (2)", - "tab": "Accuracy", - "Human Aging - Observed inference time (s)": "{\"description\": \"min=0.308, mean=0.308, max=0.308, sum=0.616 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3080115040321521\"}", - "Human Sexuality - Observed inference time (s)": "{\"description\": \"min=0.297, mean=0.297, max=0.297, sum=0.593 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.29670037984848024\"}", - "Human Aging - # eval": "{\"description\": \"min=223, mean=223, max=223, sum=446 (2)\", \"tab\": \"General information\", \"score\": \"223.0\"}", - "Human Aging - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Aging - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Aging - # prompt tokens": "{\"description\": \"min=313.587, mean=313.587, max=313.587, sum=627.175 (2)\", \"tab\": \"General information\", \"score\": \"313.58744394618833\"}", - "Human Aging - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # eval": "{\"description\": \"min=131, mean=131, max=131, sum=262 (2)\", \"tab\": \"General information\", \"score\": \"131.0\"}", - "Human Sexuality - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Sexuality - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # prompt tokens": "{\"description\": \"min=339.183, mean=339.183, max=339.183, sum=678.366 (2)\", \"tab\": \"General information\", \"score\": \"339.1832061068702\"}", - "Human Sexuality - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"human_sexuality\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_human_sexuality\"" - } - } - }, - { - "evaluation_name": "International Law", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on International Law", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.876, - "details": { - "description": "min=0.876, mean=0.876, max=0.876, sum=1.752 (2)", - "tab": "Accuracy", - "International Law - Observed inference time (s)": "{\"description\": \"min=0.38, mean=0.38, max=0.38, sum=0.761 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3803488971773258\"}", - "International Law - # eval": "{\"description\": \"min=121, mean=121, max=121, sum=242 (2)\", \"tab\": \"General information\", \"score\": \"121.0\"}", - "International Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "International Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "International Law - # prompt tokens": "{\"description\": \"min=636.165, mean=636.165, max=636.165, sum=1272.331 (2)\", \"tab\": \"General information\", \"score\": \"636.1652892561983\"}", - "International Law - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"international_law\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_international_law\"" - } - } - }, - { - "evaluation_name": "Logical Fallacies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Logical Fallacies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.804, - "details": { - "description": "min=0.804, mean=0.804, max=0.804, sum=1.607 (2)", - "tab": "Accuracy", - "Logical Fallacies - Observed inference time (s)": "{\"description\": \"min=0.304, mean=0.304, max=0.304, sum=0.608 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.30376981372482204\"}", - "Logical Fallacies - # eval": "{\"description\": \"min=163, mean=163, max=163, sum=326 (2)\", \"tab\": \"General information\", \"score\": \"163.0\"}", - "Logical Fallacies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Logical Fallacies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Logical Fallacies - # prompt tokens": "{\"description\": \"min=442.049, mean=442.049, max=442.049, sum=884.098 (2)\", \"tab\": \"General information\", \"score\": \"442.0490797546012\"}", - "Logical Fallacies - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"logical_fallacies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_logical_fallacies\"" - } - } - }, - { - "evaluation_name": "Machine Learning", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Machine Learning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.527, - "details": { - "description": "min=0.527, mean=0.527, max=0.527, sum=1.054 (2)", - "tab": "Accuracy", - "Machine Learning - Observed inference time (s)": "{\"description\": \"min=0.381, mean=0.381, max=0.381, sum=0.761 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3805731492383139\"}", - "Machine Learning - # eval": "{\"description\": \"min=112, mean=112, max=112, sum=224 (2)\", \"tab\": \"General information\", \"score\": \"112.0\"}", - "Machine Learning - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Machine Learning - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Machine Learning - # prompt tokens": "{\"description\": \"min=694.402, mean=694.402, max=694.402, sum=1388.804 (2)\", \"tab\": \"General information\", \"score\": \"694.4017857142857\"}", - "Machine Learning - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"machine_learning\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_machine_learning\"" - } - } - }, - { - "evaluation_name": "Management", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Management", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.845, - "details": { - "description": "min=0.845, mean=0.845, max=0.845, sum=1.689 (2)", - "tab": "Accuracy", - "Management - Observed inference time (s)": "{\"description\": \"min=0.301, mean=0.301, max=0.301, sum=0.603 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3013762247215197\"}", - "Management - # eval": "{\"description\": \"min=103, mean=103, max=103, sum=206 (2)\", \"tab\": \"General information\", \"score\": \"103.0\"}", - "Management - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Management - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Management - # prompt tokens": "{\"description\": \"min=273.301, mean=273.301, max=273.301, sum=546.602 (2)\", \"tab\": \"General information\", \"score\": \"273.3009708737864\"}", - "Management - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"management\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_management\"" - } - } - }, - { - "evaluation_name": "Marketing", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Marketing", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.91, - "details": { - "description": "min=0.91, mean=0.91, max=0.91, sum=1.821 (2)", - "tab": "Accuracy", - "Marketing - Observed inference time (s)": "{\"description\": \"min=0.307, mean=0.307, max=0.307, sum=0.615 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.30740204122331405\"}", - "Marketing - # eval": "{\"description\": \"min=234, mean=234, max=234, sum=468 (2)\", \"tab\": \"General information\", \"score\": \"234.0\"}", - "Marketing - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Marketing - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Marketing - # prompt tokens": "{\"description\": \"min=420.35, mean=420.35, max=420.35, sum=840.701 (2)\", \"tab\": \"General information\", \"score\": \"420.35042735042737\"}", - "Marketing - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"marketing\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_marketing\"" - } - } - }, - { - "evaluation_name": "Medical Genetics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Medical Genetics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8, - "details": { - "description": "min=0.8, mean=0.8, max=0.8, sum=1.6 (2)", - "tab": "Accuracy", - "Medical Genetics - Observed inference time (s)": "{\"description\": \"min=0.369, mean=0.369, max=0.369, sum=0.738 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.36919414043426513\"}", - "Medical Genetics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Medical Genetics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Medical Genetics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Medical Genetics - # prompt tokens": "{\"description\": \"min=330.89, mean=330.89, max=330.89, sum=661.78 (2)\", \"tab\": \"General information\", \"score\": \"330.89\"}", - "Medical Genetics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"medical_genetics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_medical_genetics\"" - } - } - }, - { - "evaluation_name": "Miscellaneous", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Miscellaneous", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.851, - "details": { - "description": "min=0.851, mean=0.851, max=0.851, sum=1.701 (2)", - "tab": "Accuracy", - "Miscellaneous - Observed inference time (s)": "{\"description\": \"min=0.305, mean=0.305, max=0.305, sum=0.61 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.30495573064528814\"}", - "Miscellaneous - # eval": "{\"description\": \"min=783, mean=783, max=783, sum=1566 (2)\", \"tab\": \"General information\", \"score\": \"783.0\"}", - "Miscellaneous - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Miscellaneous - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Miscellaneous - # prompt tokens": "{\"description\": \"min=306.669, mean=306.669, max=306.669, sum=613.338 (2)\", \"tab\": \"General information\", \"score\": \"306.669220945083\"}", - "Miscellaneous - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"miscellaneous\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_miscellaneous\"" - } - } - }, - { - "evaluation_name": "Moral Scenarios", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Moral Scenarios", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.46, - "details": { - "description": "min=0.46, mean=0.46, max=0.46, sum=0.921 (2)", - "tab": "Accuracy", - "Moral Disputes - Observed inference time (s)": "{\"description\": \"min=0.351, mean=0.351, max=0.351, sum=0.702 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3512327629706763\"}", - "Moral Scenarios - Observed inference time (s)": "{\"description\": \"min=0.39, mean=0.39, max=0.39, sum=0.78 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3902203835572113\"}", - "Moral Disputes - # eval": "{\"description\": \"min=346, mean=346, max=346, sum=692 (2)\", \"tab\": \"General information\", \"score\": \"346.0\"}", - "Moral Disputes - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Disputes - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Disputes - # prompt tokens": "{\"description\": \"min=487.003, mean=487.003, max=487.003, sum=974.006 (2)\", \"tab\": \"General information\", \"score\": \"487.0028901734104\"}", - "Moral Disputes - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # eval": "{\"description\": \"min=895, mean=895, max=895, sum=1790 (2)\", \"tab\": \"General information\", \"score\": \"895.0\"}", - "Moral Scenarios - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Scenarios - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # prompt tokens": "{\"description\": \"min=682.542, mean=682.542, max=682.542, sum=1365.084 (2)\", \"tab\": \"General information\", \"score\": \"682.5418994413408\"}", - "Moral Scenarios - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"moral_scenarios\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_moral_scenarios\"" - } - } - }, - { - "evaluation_name": "Nutrition", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Nutrition", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.788, - "details": { - "description": "min=0.788, mean=0.788, max=0.788, sum=1.575 (2)", - "tab": "Accuracy", - "Nutrition - Observed inference time (s)": "{\"description\": \"min=0.383, mean=0.383, max=0.383, sum=0.767 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3834058817695169\"}", - "Nutrition - # eval": "{\"description\": \"min=306, mean=306, max=306, sum=612 (2)\", \"tab\": \"General information\", \"score\": \"306.0\"}", - "Nutrition - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Nutrition - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Nutrition - # prompt tokens": "{\"description\": \"min=577.48, mean=577.48, max=577.48, sum=1154.961 (2)\", \"tab\": \"General information\", \"score\": \"577.4803921568628\"}", - "Nutrition - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"nutrition\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_nutrition\"" - } - } - }, - { - "evaluation_name": "Prehistory", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Prehistory", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.802, - "details": { - "description": "min=0.802, mean=0.802, max=0.802, sum=1.605 (2)", - "tab": "Accuracy", - "Prehistory - Observed inference time (s)": "{\"description\": \"min=0.423, mean=0.423, max=0.423, sum=0.845 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.42272565026342135\"}", - "Prehistory - # eval": "{\"description\": \"min=324, mean=324, max=324, sum=648 (2)\", \"tab\": \"General information\", \"score\": \"324.0\"}", - "Prehistory - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Prehistory - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Prehistory - # prompt tokens": "{\"description\": \"min=532.198, mean=532.198, max=532.198, sum=1064.395 (2)\", \"tab\": \"General information\", \"score\": \"532.1975308641976\"}", - "Prehistory - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"prehistory\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_prehistory\"" - } - } - }, - { - "evaluation_name": "Public Relations", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Public Relations", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.691, - "details": { - "description": "min=0.691, mean=0.691, max=0.691, sum=1.382 (2)", - "tab": "Accuracy", - "Public Relations - Observed inference time (s)": "{\"description\": \"min=0.305, mean=0.305, max=0.305, sum=0.61 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3049524025483565\"}", - "Public Relations - # eval": "{\"description\": \"min=110, mean=110, max=110, sum=220 (2)\", \"tab\": \"General information\", \"score\": \"110.0\"}", - "Public Relations - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Public Relations - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Public Relations - # prompt tokens": "{\"description\": \"min=418.655, mean=418.655, max=418.655, sum=837.309 (2)\", \"tab\": \"General information\", \"score\": \"418.6545454545454\"}", - "Public Relations - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"public_relations\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_public_relations\"" - } - } - }, - { - "evaluation_name": "Security Studies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Security Studies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.804, - "details": { - "description": "min=0.804, mean=0.804, max=0.804, sum=1.608 (2)", - "tab": "Accuracy", - "Security Studies - Observed inference time (s)": "{\"description\": \"min=0.523, mean=0.523, max=0.523, sum=1.046 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5228155525363222\"}", - "Security Studies - # eval": "{\"description\": \"min=245, mean=245, max=245, sum=490 (2)\", \"tab\": \"General information\", \"score\": \"245.0\"}", - "Security Studies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Security Studies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Security Studies - # prompt tokens": "{\"description\": \"min=1185.869, mean=1185.869, max=1185.869, sum=2371.739 (2)\", \"tab\": \"General information\", \"score\": \"1185.869387755102\"}", - "Security Studies - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"security_studies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_security_studies\"" - } - } - }, - { - "evaluation_name": "Sociology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Sociology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9, - "details": { - "description": "min=0.9, mean=0.9, max=0.9, sum=1.801 (2)", - "tab": "Accuracy", - "Sociology - Observed inference time (s)": "{\"description\": \"min=0.321, mean=0.321, max=0.321, sum=0.643 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.32126195395170754\"}", - "Sociology - # eval": "{\"description\": \"min=201, mean=201, max=201, sum=402 (2)\", \"tab\": \"General information\", \"score\": \"201.0\"}", - "Sociology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Sociology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Sociology - # prompt tokens": "{\"description\": \"min=448.274, mean=448.274, max=448.274, sum=896.547 (2)\", \"tab\": \"General information\", \"score\": \"448.27363184079604\"}", - "Sociology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"sociology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_sociology\"" - } - } - }, - { - "evaluation_name": "Virology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Virology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.536, - "details": { - "description": "min=0.536, mean=0.536, max=0.536, sum=1.072 (2)", - "tab": "Accuracy", - "Virology - Observed inference time (s)": "{\"description\": \"min=0.309, mean=0.309, max=0.309, sum=0.618 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.30881378018712424\"}", - "Virology - # eval": "{\"description\": \"min=166, mean=166, max=166, sum=332 (2)\", \"tab\": \"General information\", \"score\": \"166.0\"}", - "Virology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Virology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Virology - # prompt tokens": "{\"description\": \"min=328.753, mean=328.753, max=328.753, sum=657.506 (2)\", \"tab\": \"General information\", \"score\": \"328.7530120481928\"}", - "Virology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"virology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_virology\"" - } - } - }, - { - "evaluation_name": "World Religions", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on World Religions", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.86, - "details": { - "description": "min=0.86, mean=0.86, max=0.86, sum=1.719 (2)", - "tab": "Accuracy", - "World Religions - Observed inference time (s)": "{\"description\": \"min=0.336, mean=0.336, max=0.336, sum=0.673 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3363749897270872\"}", - "World Religions - # eval": "{\"description\": \"min=171, mean=171, max=171, sum=342 (2)\", \"tab\": \"General information\", \"score\": \"171.0\"}", - "World Religions - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "World Religions - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "World Religions - # prompt tokens": "{\"description\": \"min=260.164, mean=260.164, max=260.164, sum=520.327 (2)\", \"tab\": \"General information\", \"score\": \"260.1637426900585\"}", - "World Religions - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"world_religions\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_world_religions\"" - } - } - }, - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.677, - "details": { - "description": "", - "tab": "Efficiency" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/google_gemini-1.0-pro-002.json b/data/models/google_gemini-1.0-pro-002.json deleted file mode 100644 index a15774353c9bcdbdf81e50e559a2bb954ce41dfa..0000000000000000000000000000000000000000 --- a/data/models/google_gemini-1.0-pro-002.json +++ /dev/null @@ -1,382 +0,0 @@ -{ - "model_info": { - "name": "Gemini 1.0 Pro 002", - "id": "google/gemini-1.0-pro-002", - "developer": "Google", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_lite/google_gemini-1.0-pro-002/1774096306.427425", - "retrieved_timestamp": "1774096306.427425", - "source_metadata": { - "source_name": "helm_lite", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_lite", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_lite", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.422, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.6464918851435706\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.751, - "details": { - "description": "min=0.751, mean=0.751, max=0.751, sum=0.751 (1)", - "tab": "Accuracy", - "NarrativeQA - Observed inference time (s)": "{\"description\": \"min=0.679, mean=0.679, max=0.679, sum=0.679 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.6791302858934104\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=3447.994, mean=3447.994, max=3447.994, sum=3447.994 (1)\", \"tab\": \"General information\", \"score\": \"3447.994366197183\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (closed-book)", - "source_data": { - "dataset_name": "NaturalQuestions (closed-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (closed-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.391, - "details": { - "description": "min=0.391, mean=0.391, max=0.391, sum=0.391 (1)", - "tab": "Accuracy", - "NaturalQuestions (open-book) - Observed inference time (s)": "{\"description\": \"min=0.609, mean=0.609, max=0.609, sum=0.609 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.6086829407215119\"}", - "NaturalQuestions (closed-book) - Observed inference time (s)": "{\"description\": \"min=0.597, mean=0.597, max=0.597, sum=0.597 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.5965619602203369\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1978.347, mean=1978.347, max=1978.347, sum=1978.347 (1)\", \"tab\": \"General information\", \"score\": \"1978.347\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=153.995, mean=153.995, max=153.995, sum=153.995 (1)\", \"tab\": \"General information\", \"score\": \"153.995\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "mode": "\"closedbook\"" - } - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.788, - "details": { - "description": "min=0.788, mean=0.788, max=0.788, sum=0.788 (1)", - "tab": "Accuracy", - "OpenbookQA - Observed inference time (s)": "{\"description\": \"min=0.43, mean=0.43, max=0.43, sum=0.43 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.4301223816871643\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=248.508, mean=248.508, max=248.508, sum=248.508 (1)\", \"tab\": \"General information\", \"score\": \"248.508\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "dataset": "\"openbookqa\"", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.534, - "details": { - "description": "min=0.27, mean=0.534, max=0.81, sum=2.672 (5)", - "tab": "Accuracy", - "MMLU - Observed inference time (s)": "{\"description\": \"min=0.397, mean=0.407, max=0.417, sum=2.033 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.4066482855060644\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=380.91, mean=481.531, max=634.553, sum=2407.653 (5)\", \"tab\": \"General information\", \"score\": \"481.5305263157895\"}", - "MMLU - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"college_chemistry\", \"computer_security\", \"econometrics\", \"us_foreign_policy\"]", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MATH", - "source_data": { - "dataset_name": "MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Equivalent (CoT) on MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.665, - "details": { - "description": "min=0.553, mean=0.665, max=0.859, sum=4.654 (7)", - "tab": "Accuracy", - "MATH - Observed inference time (s)": "{\"description\": \"min=1.402, mean=1.585, max=2.083, sum=11.094 (7)\", \"tab\": \"Efficiency\", \"score\": \"1.5848151401531698\"}", - "MATH - # eval": "{\"description\": \"min=30, mean=62.429, max=135, sum=437 (7)\", \"tab\": \"General information\", \"score\": \"62.42857142857143\"}", - "MATH - # train": "{\"description\": \"min=8, mean=8, max=8, sum=56 (7)\", \"tab\": \"General information\", \"score\": \"8.0\"}", - "MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MATH - # prompt tokens": "{\"description\": \"min=938.215, mean=1355.506, max=2348.712, sum=9488.545 (7)\", \"tab\": \"General information\", \"score\": \"1355.5064552904823\"}", - "MATH - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"algebra\", \"counting_and_probability\", \"geometry\", \"intermediate_algebra\", \"number_theory\", \"prealgebra\", \"precalculus\"]", - "level": "\"1\"", - "use_official_examples": "\"False\"", - "use_chain_of_thought": "\"True\"" - } - } - }, - { - "evaluation_name": "GSM8K", - "source_data": { - "dataset_name": "GSM8K", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on GSM8K", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.816, - "details": { - "description": "min=0.816, mean=0.816, max=0.816, sum=0.816 (1)", - "tab": "Accuracy", - "GSM8K - Observed inference time (s)": "{\"description\": \"min=1.513, mean=1.513, max=1.513, sum=1.513 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.513066102743149\"}", - "GSM8K - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "GSM8K - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "GSM8K - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GSM8K - # prompt tokens": "{\"description\": \"min=1151.885, mean=1151.885, max=1151.885, sum=1151.885 (1)\", \"tab\": \"General information\", \"score\": \"1151.885\"}", - "GSM8K - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "stop": "\"none\"" - } - } - }, - { - "evaluation_name": "LegalBench", - "source_data": { - "dataset_name": "LegalBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on LegalBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.475, - "details": { - "description": "min=0.118, mean=0.475, max=0.811, sum=2.376 (5)", - "tab": "Accuracy", - "LegalBench - Observed inference time (s)": "{\"description\": \"min=0.447, mean=0.609, max=1.08, sum=3.043 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.6085789782066453\"}", - "LegalBench - # eval": "{\"description\": \"min=95, mean=409.4, max=1000, sum=2047 (5)\", \"tab\": \"General information\", \"score\": \"409.4\"}", - "LegalBench - # train": "{\"description\": \"min=4, mean=4.8, max=5, sum=24 (5)\", \"tab\": \"General information\", \"score\": \"4.8\"}", - "LegalBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "LegalBench - # prompt tokens": "{\"description\": \"min=209.916, mean=1558.239, max=6423.569, sum=7791.193 (5)\", \"tab\": \"General information\", \"score\": \"1558.2386051001386\"}", - "LegalBench - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"abercrombie\", \"corporate_lobbying\", \"function_of_decision_section\", \"international_citizenship_questions\", \"proa\"]" - } - } - }, - { - "evaluation_name": "MedQA", - "source_data": { - "dataset_name": "MedQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MedQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.483, - "details": { - "description": "min=0.483, mean=0.483, max=0.483, sum=0.483 (1)", - "tab": "Accuracy", - "MedQA - Observed inference time (s)": "{\"description\": \"min=0.431, mean=0.431, max=0.431, sum=0.431 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.4310008814610333\"}", - "MedQA - # eval": "{\"description\": \"min=503, mean=503, max=503, sum=503 (1)\", \"tab\": \"General information\", \"score\": \"503.0\"}", - "MedQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MedQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MedQA - # prompt tokens": "{\"description\": \"min=1029.481, mean=1029.481, max=1029.481, sum=1029.481 (1)\", \"tab\": \"General information\", \"score\": \"1029.4811133200794\"}", - "MedQA - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WMT 2014", - "source_data": { - "dataset_name": "WMT 2014", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "BLEU-4 on WMT 2014", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.194, - "details": { - "description": "min=0.144, mean=0.194, max=0.231, sum=0.972 (5)", - "tab": "Accuracy", - "WMT 2014 - Observed inference time (s)": "{\"description\": \"min=0.705, mean=0.803, max=0.924, sum=4.014 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.8027491282517494\"}", - "WMT 2014 - # eval": "{\"description\": \"min=503, mean=568.8, max=832, sum=2844 (5)\", \"tab\": \"General information\", \"score\": \"568.8\"}", - "WMT 2014 - # train": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "WMT 2014 - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WMT 2014 - # prompt tokens": "{\"description\": \"min=90.732, mean=120.97, max=147.366, sum=604.851 (5)\", \"tab\": \"General information\", \"score\": \"120.97025108961614\"}", - "WMT 2014 - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "language_pair": "[\"cs-en\", \"de-en\", \"fr-en\", \"hi-en\", \"ru-en\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/google_gemini-1.5-flash-001.json b/data/models/google_gemini-1.5-flash-001.json deleted file mode 100644 index 6184c14fa23bf575b5859d4fde62141d6ae8b863..0000000000000000000000000000000000000000 --- a/data/models/google_gemini-1.5-flash-001.json +++ /dev/null @@ -1,2035 +0,0 @@ -{ - "model_info": { - "name": "Gemini 1.5 Flash 001", - "id": "google/gemini-1.5-flash-001", - "developer": "Google", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_lite/google_gemini-1.5-flash-001/1774096306.427425", - "retrieved_timestamp": "1774096306.427425", - "source_metadata": { - "source_name": "helm_lite", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_lite", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_lite", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.667, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.681960049937578\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.783, - "details": { - "description": "min=0.783, mean=0.783, max=0.783, sum=0.783 (1)", - "tab": "Accuracy", - "NarrativeQA - Observed inference time (s)": "{\"description\": \"min=0.647, mean=0.647, max=0.647, sum=0.647 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.6474363112991507\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=3447.994, mean=3447.994, max=3447.994, sum=3447.994 (1)\", \"tab\": \"General information\", \"score\": \"3447.994366197183\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (closed-book)", - "source_data": { - "dataset_name": "NaturalQuestions (closed-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (closed-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.332, - "details": { - "description": "min=0.332, mean=0.332, max=0.332, sum=0.332 (1)", - "tab": "Accuracy", - "NaturalQuestions (open-book) - Observed inference time (s)": "{\"description\": \"min=0.495, mean=0.495, max=0.495, sum=0.495 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.49524100852012637\"}", - "NaturalQuestions (closed-book) - Observed inference time (s)": "{\"description\": \"min=0.432, mean=0.432, max=0.432, sum=0.432 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.431587886095047\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1978.347, mean=1978.347, max=1978.347, sum=1978.347 (1)\", \"tab\": \"General information\", \"score\": \"1978.347\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=153.995, mean=153.995, max=153.995, sum=153.995 (1)\", \"tab\": \"General information\", \"score\": \"153.995\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "mode": "\"closedbook\"" - } - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.928, - "details": { - "description": "min=0.928, mean=0.928, max=0.928, sum=0.928 (1)", - "tab": "Accuracy", - "OpenbookQA - Observed inference time (s)": "{\"description\": \"min=0.504, mean=0.504, max=0.504, sum=0.504 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.5038927392959595\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=248.508, mean=248.508, max=248.508, sum=248.508 (1)\", \"tab\": \"General information\", \"score\": \"248.508\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "dataset": "\"openbookqa\"", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.703, - "details": { - "description": "min=0.58, mean=0.703, max=0.93, sum=3.514 (5)", - "tab": "Accuracy", - "MMLU - Observed inference time (s)": "{\"description\": \"min=0.525, mean=0.568, max=0.62, sum=2.842 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.5683523873948214\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=380.91, mean=481.531, max=634.553, sum=2407.653 (5)\", \"tab\": \"General information\", \"score\": \"481.5305263157895\"}", - "MMLU - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"college_chemistry\", \"computer_security\", \"econometrics\", \"us_foreign_policy\"]", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MATH", - "source_data": { - "dataset_name": "MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Equivalent (CoT) on MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.753, - "details": { - "description": "min=0.632, mean=0.753, max=0.889, sum=5.269 (7)", - "tab": "Accuracy", - "MATH - Observed inference time (s)": "{\"description\": \"min=1.303, mean=1.592, max=2.086, sum=11.144 (7)\", \"tab\": \"Efficiency\", \"score\": \"1.592031592636459\"}", - "MATH - # eval": "{\"description\": \"min=30, mean=62.429, max=135, sum=437 (7)\", \"tab\": \"General information\", \"score\": \"62.42857142857143\"}", - "MATH - # train": "{\"description\": \"min=8, mean=8, max=8, sum=56 (7)\", \"tab\": \"General information\", \"score\": \"8.0\"}", - "MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MATH - # prompt tokens": "{\"description\": \"min=938.215, mean=1355.506, max=2348.712, sum=9488.545 (7)\", \"tab\": \"General information\", \"score\": \"1355.5064552904823\"}", - "MATH - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"algebra\", \"counting_and_probability\", \"geometry\", \"intermediate_algebra\", \"number_theory\", \"prealgebra\", \"precalculus\"]", - "level": "\"1\"", - "use_official_examples": "\"False\"", - "use_chain_of_thought": "\"True\"" - } - } - }, - { - "evaluation_name": "GSM8K", - "source_data": { - "dataset_name": "GSM8K", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on GSM8K", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.785, - "details": { - "description": "min=0.785, mean=0.785, max=0.785, sum=0.785 (1)", - "tab": "Accuracy", - "GSM8K - Observed inference time (s)": "{\"description\": \"min=1.758, mean=1.758, max=1.758, sum=1.758 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.7575640678405762\"}", - "GSM8K - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "GSM8K - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "GSM8K - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GSM8K - # prompt tokens": "{\"description\": \"min=1151.885, mean=1151.885, max=1151.885, sum=1151.885 (1)\", \"tab\": \"General information\", \"score\": \"1151.885\"}", - "GSM8K - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "stop": "\"none\"" - } - } - }, - { - "evaluation_name": "LegalBench", - "source_data": { - "dataset_name": "LegalBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on LegalBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.661, - "details": { - "description": "min=0.425, mean=0.661, max=0.968, sum=3.305 (5)", - "tab": "Accuracy", - "LegalBench - Observed inference time (s)": "{\"description\": \"min=0.409, mean=0.604, max=0.842, sum=3.02 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.6040551961526522\"}", - "LegalBench - # eval": "{\"description\": \"min=95, mean=409.4, max=1000, sum=2047 (5)\", \"tab\": \"General information\", \"score\": \"409.4\"}", - "LegalBench - # train": "{\"description\": \"min=4, mean=4.8, max=5, sum=24 (5)\", \"tab\": \"General information\", \"score\": \"4.8\"}", - "LegalBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "LegalBench - # prompt tokens": "{\"description\": \"min=209.916, mean=1558.239, max=6423.569, sum=7791.193 (5)\", \"tab\": \"General information\", \"score\": \"1558.2386051001386\"}", - "LegalBench - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"abercrombie\", \"corporate_lobbying\", \"function_of_decision_section\", \"international_citizenship_questions\", \"proa\"]" - } - } - }, - { - "evaluation_name": "MedQA", - "source_data": { - "dataset_name": "MedQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MedQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.68, - "details": { - "description": "min=0.68, mean=0.68, max=0.68, sum=0.68 (1)", - "tab": "Accuracy", - "MedQA - Observed inference time (s)": "{\"description\": \"min=0.399, mean=0.399, max=0.399, sum=0.399 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.3993651843165971\"}", - "MedQA - # eval": "{\"description\": \"min=503, mean=503, max=503, sum=503 (1)\", \"tab\": \"General information\", \"score\": \"503.0\"}", - "MedQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MedQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MedQA - # prompt tokens": "{\"description\": \"min=1029.481, mean=1029.481, max=1029.481, sum=1029.481 (1)\", \"tab\": \"General information\", \"score\": \"1029.4811133200794\"}", - "MedQA - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WMT 2014", - "source_data": { - "dataset_name": "WMT 2014", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "BLEU-4 on WMT 2014", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.225, - "details": { - "description": "min=0.186, mean=0.225, max=0.253, sum=1.126 (5)", - "tab": "Accuracy", - "WMT 2014 - Observed inference time (s)": "{\"description\": \"min=0.581, mean=0.637, max=0.75, sum=3.186 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.6372637821067911\"}", - "WMT 2014 - # eval": "{\"description\": \"min=503, mean=568.8, max=832, sum=2844 (5)\", \"tab\": \"General information\", \"score\": \"568.8\"}", - "WMT 2014 - # train": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "WMT 2014 - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WMT 2014 - # prompt tokens": "{\"description\": \"min=90.732, mean=120.97, max=147.366, sum=604.851 (5)\", \"tab\": \"General information\", \"score\": \"120.97025108961614\"}", - "WMT 2014 - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "language_pair": "[\"cs-en\", \"de-en\", \"fr-en\", \"hi-en\", \"ru-en\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_mmlu/google_gemini-1.5-flash-001/1774096312.00548", - "retrieved_timestamp": "1774096312.00548", - "source_metadata": { - "source_name": "helm_mmlu", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_mmlu", - "evaluation_results": [ - { - "evaluation_name": "MMLU All Subjects", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU All Subjects", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.779, - "details": { - "description": "min=0.374, mean=0.779, max=0.974, sum=88.804 (114)", - "tab": "Accuracy", - "MMLU All Subjects - Observed inference time (s)": "{\"description\": \"min=0.386, mean=0.487, max=0.665, sum=55.55 (114)\", \"tab\": \"Efficiency\", \"score\": \"0.4872786268013793\"}", - "MMLU All Subjects - # eval": "{\"description\": \"min=100, mean=246.351, max=1534, sum=28084 (114)\", \"tab\": \"General information\", \"score\": \"246.35087719298247\"}", - "MMLU All Subjects - # train": "{\"description\": \"min=5, mean=5, max=5, sum=570 (114)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU All Subjects - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU All Subjects - # prompt tokens": "{\"description\": \"min=268.164, mean=632.617, max=2797.424, sum=72118.345 (114)\", \"tab\": \"General information\", \"score\": \"632.6170571214202\"}", - "MMLU All Subjects - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_name": "Abstract Algebra", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Abstract Algebra", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.58, - "details": { - "description": "min=0.58, mean=0.58, max=0.58, sum=1.16 (2)", - "tab": "Accuracy", - "Abstract Algebra - Observed inference time (s)": "{\"description\": \"min=0.596, mean=0.596, max=0.596, sum=1.191 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.595533971786499\"}", - "Abstract Algebra - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Abstract Algebra - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Abstract Algebra - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Abstract Algebra - # prompt tokens": "{\"description\": \"min=383.97, mean=383.97, max=383.97, sum=767.94 (2)\", \"tab\": \"General information\", \"score\": \"383.97\"}", - "Abstract Algebra - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"abstract_algebra\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_abstract_algebra\"" - } - } - }, - { - "evaluation_name": "Anatomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Anatomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8, - "details": { - "description": "min=0.8, mean=0.8, max=0.8, sum=1.6 (2)", - "tab": "Accuracy", - "Anatomy - Observed inference time (s)": "{\"description\": \"min=0.536, mean=0.536, max=0.536, sum=1.071 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5356822949868661\"}", - "Anatomy - # eval": "{\"description\": \"min=135, mean=135, max=135, sum=270 (2)\", \"tab\": \"General information\", \"score\": \"135.0\"}", - "Anatomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Anatomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Anatomy - # prompt tokens": "{\"description\": \"min=344.356, mean=344.356, max=344.356, sum=688.711 (2)\", \"tab\": \"General information\", \"score\": \"344.35555555555555\"}", - "Anatomy - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"anatomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_anatomy\"" - } - } - }, - { - "evaluation_name": "College Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on College Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.696, - "details": { - "description": "min=0.696, mean=0.696, max=0.696, sum=1.392 (2)", - "tab": "Accuracy", - "College Chemistry - Observed inference time (s)": "{\"description\": \"min=0.62, mean=0.62, max=0.62, sum=1.24 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6201749587059021\"}", - "College Biology - Observed inference time (s)": "{\"description\": \"min=0.497, mean=0.497, max=0.497, sum=0.995 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4974212066994773\"}", - "College Computer Science - Observed inference time (s)": "{\"description\": \"min=0.571, mean=0.571, max=0.571, sum=1.143 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5714822864532471\"}", - "College Mathematics - Observed inference time (s)": "{\"description\": \"min=0.543, mean=0.543, max=0.543, sum=1.085 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5425397109985352\"}", - "College Medicine - Observed inference time (s)": "{\"description\": \"min=0.487, mean=0.487, max=0.487, sum=0.975 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.48738120056990253\"}", - "College Physics - Observed inference time (s)": "{\"description\": \"min=0.608, mean=0.608, max=0.608, sum=1.215 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6076285418342141\"}", - "College Chemistry - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Chemistry - # prompt tokens": "{\"description\": \"min=570.02, mean=570.02, max=570.02, sum=1140.04 (2)\", \"tab\": \"General information\", \"score\": \"570.02\"}", - "College Chemistry - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # eval": "{\"description\": \"min=144, mean=144, max=144, sum=288 (2)\", \"tab\": \"General information\", \"score\": \"144.0\"}", - "College Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # prompt tokens": "{\"description\": \"min=482.799, mean=482.799, max=482.799, sum=965.597 (2)\", \"tab\": \"General information\", \"score\": \"482.7986111111111\"}", - "College Biology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # prompt tokens": "{\"description\": \"min=857.86, mean=857.86, max=857.86, sum=1715.72 (2)\", \"tab\": \"General information\", \"score\": \"857.86\"}", - "College Computer Science - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # prompt tokens": "{\"description\": \"min=626.69, mean=626.69, max=626.69, sum=1253.38 (2)\", \"tab\": \"General information\", \"score\": \"626.69\"}", - "College Mathematics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # eval": "{\"description\": \"min=173, mean=173, max=173, sum=346 (2)\", \"tab\": \"General information\", \"score\": \"173.0\"}", - "College Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # prompt tokens": "{\"description\": \"min=513.37, mean=513.37, max=513.37, sum=1026.74 (2)\", \"tab\": \"General information\", \"score\": \"513.3699421965318\"}", - "College Medicine - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # eval": "{\"description\": \"min=102, mean=102, max=102, sum=204 (2)\", \"tab\": \"General information\", \"score\": \"102.0\"}", - "College Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # prompt tokens": "{\"description\": \"min=507.471, mean=507.471, max=507.471, sum=1014.941 (2)\", \"tab\": \"General information\", \"score\": \"507.47058823529414\"}", - "College Physics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"college_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_college_physics\"" - } - } - }, - { - "evaluation_name": "Computer Security", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Computer Security", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.79, - "details": { - "description": "min=0.79, mean=0.79, max=0.79, sum=1.58 (2)", - "tab": "Accuracy", - "Computer Security - Observed inference time (s)": "{\"description\": \"min=0.538, mean=0.538, max=0.538, sum=1.075 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.537526171207428\"}", - "Computer Security - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Computer Security - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Computer Security - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Computer Security - # prompt tokens": "{\"description\": \"min=380.91, mean=380.91, max=380.91, sum=761.82 (2)\", \"tab\": \"General information\", \"score\": \"380.91\"}", - "Computer Security - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"computer_security\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_computer_security\"" - } - } - }, - { - "evaluation_name": "Econometrics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Econometrics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.614, - "details": { - "description": "min=0.614, mean=0.614, max=0.614, sum=1.228 (2)", - "tab": "Accuracy", - "Econometrics - Observed inference time (s)": "{\"description\": \"min=0.564, mean=0.564, max=0.564, sum=1.128 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5637641475911725\"}", - "Econometrics - # eval": "{\"description\": \"min=114, mean=114, max=114, sum=228 (2)\", \"tab\": \"General information\", \"score\": \"114.0\"}", - "Econometrics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Econometrics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Econometrics - # prompt tokens": "{\"description\": \"min=634.553, mean=634.553, max=634.553, sum=1269.105 (2)\", \"tab\": \"General information\", \"score\": \"634.5526315789474\"}", - "Econometrics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"econometrics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_econometrics\"" - } - } - }, - { - "evaluation_name": "Global Facts", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Global Facts", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.53, - "details": { - "description": "min=0.53, mean=0.53, max=0.53, sum=1.06 (2)", - "tab": "Accuracy", - "Global Facts - Observed inference time (s)": "{\"description\": \"min=0.549, mean=0.549, max=0.549, sum=1.097 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5487277007102966\"}", - "Global Facts - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Global Facts - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Global Facts - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Global Facts - # prompt tokens": "{\"description\": \"min=456.54, mean=456.54, max=456.54, sum=913.08 (2)\", \"tab\": \"General information\", \"score\": \"456.54\"}", - "Global Facts - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"global_facts\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_global_facts\"" - } - } - }, - { - "evaluation_name": "Jurisprudence", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Jurisprudence", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.889, - "details": { - "description": "min=0.889, mean=0.889, max=0.889, sum=1.778 (2)", - "tab": "Accuracy", - "Jurisprudence - Observed inference time (s)": "{\"description\": \"min=0.501, mean=0.501, max=0.501, sum=1.002 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5009041649323923\"}", - "Jurisprudence - # eval": "{\"description\": \"min=108, mean=108, max=108, sum=216 (2)\", \"tab\": \"General information\", \"score\": \"108.0\"}", - "Jurisprudence - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Jurisprudence - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Jurisprudence - # prompt tokens": "{\"description\": \"min=407.87, mean=407.87, max=407.87, sum=815.741 (2)\", \"tab\": \"General information\", \"score\": \"407.8703703703704\"}", - "Jurisprudence - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"jurisprudence\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_jurisprudence\"" - } - } - }, - { - "evaluation_name": "Philosophy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Philosophy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.791, - "details": { - "description": "min=0.791, mean=0.791, max=0.791, sum=1.582 (2)", - "tab": "Accuracy", - "Philosophy - Observed inference time (s)": "{\"description\": \"min=0.48, mean=0.48, max=0.48, sum=0.96 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.48008891700548373\"}", - "Philosophy - # eval": "{\"description\": \"min=311, mean=311, max=311, sum=622 (2)\", \"tab\": \"General information\", \"score\": \"311.0\"}", - "Philosophy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Philosophy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Philosophy - # prompt tokens": "{\"description\": \"min=340.907, mean=340.907, max=340.907, sum=681.814 (2)\", \"tab\": \"General information\", \"score\": \"340.90675241157555\"}", - "Philosophy - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"philosophy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_philosophy\"" - } - } - }, - { - "evaluation_name": "Professional Psychology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Professional Psychology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.828, - "details": { - "description": "min=0.828, mean=0.828, max=0.828, sum=1.657 (2)", - "tab": "Accuracy", - "Professional Medicine - Observed inference time (s)": "{\"description\": \"min=0.477, mean=0.477, max=0.477, sum=0.955 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.47726698907099085\"}", - "Professional Accounting - Observed inference time (s)": "{\"description\": \"min=0.44, mean=0.44, max=0.44, sum=0.88 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4398383096600255\"}", - "Professional Law - Observed inference time (s)": "{\"description\": \"min=0.424, mean=0.424, max=0.424, sum=0.848 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.42376324315969854\"}", - "Professional Psychology - Observed inference time (s)": "{\"description\": \"min=0.386, mean=0.386, max=0.386, sum=0.773 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3864205361981141\"}", - "Professional Medicine - # eval": "{\"description\": \"min=272, mean=272, max=272, sum=544 (2)\", \"tab\": \"General information\", \"score\": \"272.0\"}", - "Professional Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Medicine - # prompt tokens": "{\"description\": \"min=1113.092, mean=1113.092, max=1113.092, sum=2226.184 (2)\", \"tab\": \"General information\", \"score\": \"1113.0919117647059\"}", - "Professional Medicine - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # eval": "{\"description\": \"min=282, mean=282, max=282, sum=564 (2)\", \"tab\": \"General information\", \"score\": \"282.0\"}", - "Professional Accounting - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Accounting - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # prompt tokens": "{\"description\": \"min=755.418, mean=755.418, max=755.418, sum=1510.837 (2)\", \"tab\": \"General information\", \"score\": \"755.418439716312\"}", - "Professional Accounting - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # eval": "{\"description\": \"min=1534, mean=1534, max=1534, sum=3068 (2)\", \"tab\": \"General information\", \"score\": \"1534.0\"}", - "Professional Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # prompt tokens": "{\"description\": \"min=1685.119, mean=1685.119, max=1685.119, sum=3370.239 (2)\", \"tab\": \"General information\", \"score\": \"1685.119295958279\"}", - "Professional Law - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # eval": "{\"description\": \"min=612, mean=612, max=612, sum=1224 (2)\", \"tab\": \"General information\", \"score\": \"612.0\"}", - "Professional Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # prompt tokens": "{\"description\": \"min=594.363, mean=594.363, max=594.363, sum=1188.725 (2)\", \"tab\": \"General information\", \"score\": \"594.3627450980392\"}", - "Professional Psychology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"professional_psychology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_professional_psychology\"" - } - } - }, - { - "evaluation_name": "Us Foreign Policy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Us Foreign Policy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.93, - "details": { - "description": "min=0.93, mean=0.93, max=0.93, sum=1.86 (2)", - "tab": "Accuracy", - "Us Foreign Policy - Observed inference time (s)": "{\"description\": \"min=0.525, mean=0.525, max=0.525, sum=1.05 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5247626876831055\"}", - "Us Foreign Policy - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Us Foreign Policy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Us Foreign Policy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Us Foreign Policy - # prompt tokens": "{\"description\": \"min=438.2, mean=438.2, max=438.2, sum=876.4 (2)\", \"tab\": \"General information\", \"score\": \"438.2\"}", - "Us Foreign Policy - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"us_foreign_policy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_us_foreign_policy\"" - } - } - }, - { - "evaluation_name": "Astronomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Astronomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.882, - "details": { - "description": "min=0.882, mean=0.882, max=0.882, sum=1.763 (2)", - "tab": "Accuracy", - "Astronomy - Observed inference time (s)": "{\"description\": \"min=0.498, mean=0.498, max=0.498, sum=0.995 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.49771531004654734\"}", - "Astronomy - # eval": "{\"description\": \"min=152, mean=152, max=152, sum=304 (2)\", \"tab\": \"General information\", \"score\": \"152.0\"}", - "Astronomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Astronomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Astronomy - # prompt tokens": "{\"description\": \"min=602.421, mean=602.421, max=602.421, sum=1204.842 (2)\", \"tab\": \"General information\", \"score\": \"602.421052631579\"}", - "Astronomy - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"astronomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_astronomy\"" - } - } - }, - { - "evaluation_name": "Business Ethics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Business Ethics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.81, - "details": { - "description": "min=0.81, mean=0.81, max=0.81, sum=1.62 (2)", - "tab": "Accuracy", - "Business Ethics - Observed inference time (s)": "{\"description\": \"min=0.608, mean=0.608, max=0.608, sum=1.216 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.608082628250122\"}", - "Business Ethics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Business Ethics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Business Ethics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Business Ethics - # prompt tokens": "{\"description\": \"min=552.87, mean=552.87, max=552.87, sum=1105.74 (2)\", \"tab\": \"General information\", \"score\": \"552.87\"}", - "Business Ethics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"business_ethics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_business_ethics\"" - } - } - }, - { - "evaluation_name": "Clinical Knowledge", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Clinical Knowledge", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.834, - "details": { - "description": "min=0.834, mean=0.834, max=0.834, sum=1.668 (2)", - "tab": "Accuracy", - "Clinical Knowledge - Observed inference time (s)": "{\"description\": \"min=0.419, mean=0.419, max=0.419, sum=0.839 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.41935023991566783\"}", - "Clinical Knowledge - # eval": "{\"description\": \"min=265, mean=265, max=265, sum=530 (2)\", \"tab\": \"General information\", \"score\": \"265.0\"}", - "Clinical Knowledge - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Clinical Knowledge - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Clinical Knowledge - # prompt tokens": "{\"description\": \"min=402.592, mean=402.592, max=402.592, sum=805.185 (2)\", \"tab\": \"General information\", \"score\": \"402.5924528301887\"}", - "Clinical Knowledge - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"clinical_knowledge\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_clinical_knowledge\"" - } - } - }, - { - "evaluation_name": "Conceptual Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Conceptual Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.851, - "details": { - "description": "min=0.851, mean=0.851, max=0.851, sum=1.702 (2)", - "tab": "Accuracy", - "Conceptual Physics - Observed inference time (s)": "{\"description\": \"min=0.451, mean=0.451, max=0.451, sum=0.901 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4506680082767568\"}", - "Conceptual Physics - # eval": "{\"description\": \"min=235, mean=235, max=235, sum=470 (2)\", \"tab\": \"General information\", \"score\": \"235.0\"}", - "Conceptual Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Conceptual Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Conceptual Physics - # prompt tokens": "{\"description\": \"min=309.213, mean=309.213, max=309.213, sum=618.426 (2)\", \"tab\": \"General information\", \"score\": \"309.21276595744683\"}", - "Conceptual Physics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"conceptual_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_conceptual_physics\"" - } - } - }, - { - "evaluation_name": "Electrical Engineering", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Electrical Engineering", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8, - "details": { - "description": "min=0.8, mean=0.8, max=0.8, sum=1.6 (2)", - "tab": "Accuracy", - "Electrical Engineering - Observed inference time (s)": "{\"description\": \"min=0.46, mean=0.46, max=0.46, sum=0.92 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4601488047632678\"}", - "Electrical Engineering - # eval": "{\"description\": \"min=145, mean=145, max=145, sum=290 (2)\", \"tab\": \"General information\", \"score\": \"145.0\"}", - "Electrical Engineering - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Electrical Engineering - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Electrical Engineering - # prompt tokens": "{\"description\": \"min=474.786, mean=474.786, max=474.786, sum=949.572 (2)\", \"tab\": \"General information\", \"score\": \"474.78620689655173\"}", - "Electrical Engineering - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"electrical_engineering\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_electrical_engineering\"" - } - } - }, - { - "evaluation_name": "Elementary Mathematics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Elementary Mathematics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.754, - "details": { - "description": "min=0.754, mean=0.754, max=0.754, sum=1.508 (2)", - "tab": "Accuracy", - "Elementary Mathematics - Observed inference time (s)": "{\"description\": \"min=0.41, mean=0.41, max=0.41, sum=0.819 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.40957188984704396\"}", - "Elementary Mathematics - # eval": "{\"description\": \"min=378, mean=378, max=378, sum=756 (2)\", \"tab\": \"General information\", \"score\": \"378.0\"}", - "Elementary Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Elementary Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Elementary Mathematics - # prompt tokens": "{\"description\": \"min=597.341, mean=597.341, max=597.341, sum=1194.683 (2)\", \"tab\": \"General information\", \"score\": \"597.3412698412699\"}", - "Elementary Mathematics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"elementary_mathematics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_elementary_mathematics\"" - } - } - }, - { - "evaluation_name": "Formal Logic", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Formal Logic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.627, - "details": { - "description": "min=0.627, mean=0.627, max=0.627, sum=1.254 (2)", - "tab": "Accuracy", - "Formal Logic - Observed inference time (s)": "{\"description\": \"min=0.524, mean=0.524, max=0.524, sum=1.047 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5235741989953178\"}", - "Formal Logic - # eval": "{\"description\": \"min=126, mean=126, max=126, sum=252 (2)\", \"tab\": \"General information\", \"score\": \"126.0\"}", - "Formal Logic - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Formal Logic - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Formal Logic - # prompt tokens": "{\"description\": \"min=619.563, mean=619.563, max=619.563, sum=1239.127 (2)\", \"tab\": \"General information\", \"score\": \"619.563492063492\"}", - "Formal Logic - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"formal_logic\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_formal_logic\"" - } - } - }, - { - "evaluation_name": "High School World History", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on High School World History", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.907, - "details": { - "description": "min=0.907, mean=0.907, max=0.907, sum=1.814 (2)", - "tab": "Accuracy", - "High School Biology - Observed inference time (s)": "{\"description\": \"min=0.439, mean=0.439, max=0.439, sum=0.878 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.43886603309262184\"}", - "High School Chemistry - Observed inference time (s)": "{\"description\": \"min=0.468, mean=0.468, max=0.468, sum=0.937 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4683608938320517\"}", - "High School Computer Science - Observed inference time (s)": "{\"description\": \"min=0.511, mean=0.511, max=0.511, sum=1.022 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5109630298614501\"}", - "High School European History - Observed inference time (s)": "{\"description\": \"min=0.665, mean=0.665, max=0.665, sum=1.33 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.665167844656742\"}", - "High School Geography - Observed inference time (s)": "{\"description\": \"min=0.432, mean=0.432, max=0.432, sum=0.863 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.43152768804569436\"}", - "High School Government And Politics - Observed inference time (s)": "{\"description\": \"min=0.422, mean=0.422, max=0.422, sum=0.845 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4224596888290168\"}", - "High School Macroeconomics - Observed inference time (s)": "{\"description\": \"min=0.404, mean=0.404, max=0.404, sum=0.808 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4038744736940433\"}", - "High School Mathematics - Observed inference time (s)": "{\"description\": \"min=0.435, mean=0.435, max=0.435, sum=0.869 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.43474441987496837\"}", - "High School Microeconomics - Observed inference time (s)": "{\"description\": \"min=0.416, mean=0.416, max=0.416, sum=0.832 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4159359881857864\"}", - "High School Physics - Observed inference time (s)": "{\"description\": \"min=0.493, mean=0.493, max=0.493, sum=0.985 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.49265997772974685\"}", - "High School Psychology - Observed inference time (s)": "{\"description\": \"min=0.418, mean=0.418, max=0.418, sum=0.835 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.41751264342490363\"}", - "High School Statistics - Observed inference time (s)": "{\"description\": \"min=0.497, mean=0.497, max=0.497, sum=0.993 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.49666665218494555\"}", - "High School US History - Observed inference time (s)": "{\"description\": \"min=0.606, mean=0.606, max=0.606, sum=1.213 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6064977821181802\"}", - "High School World History - Observed inference time (s)": "{\"description\": \"min=0.469, mean=0.469, max=0.469, sum=0.939 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.46946642569851776\"}", - "High School Biology - # eval": "{\"description\": \"min=310, mean=310, max=310, sum=620 (2)\", \"tab\": \"General information\", \"score\": \"310.0\"}", - "High School Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Biology - # prompt tokens": "{\"description\": \"min=500.958, mean=500.958, max=500.958, sum=1001.916 (2)\", \"tab\": \"General information\", \"score\": \"500.958064516129\"}", - "High School Biology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # eval": "{\"description\": \"min=203, mean=203, max=203, sum=406 (2)\", \"tab\": \"General information\", \"score\": \"203.0\"}", - "High School Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # prompt tokens": "{\"description\": \"min=513.064, mean=513.064, max=513.064, sum=1026.128 (2)\", \"tab\": \"General information\", \"score\": \"513.064039408867\"}", - "High School Chemistry - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "High School Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # prompt tokens": "{\"description\": \"min=935.13, mean=935.13, max=935.13, sum=1870.26 (2)\", \"tab\": \"General information\", \"score\": \"935.13\"}", - "High School Computer Science - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # eval": "{\"description\": \"min=165, mean=165, max=165, sum=330 (2)\", \"tab\": \"General information\", \"score\": \"165.0\"}", - "High School European History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School European History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # prompt tokens": "{\"description\": \"min=2797.424, mean=2797.424, max=2797.424, sum=5594.848 (2)\", \"tab\": \"General information\", \"score\": \"2797.4242424242425\"}", - "High School European History - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # eval": "{\"description\": \"min=198, mean=198, max=198, sum=396 (2)\", \"tab\": \"General information\", \"score\": \"198.0\"}", - "High School Geography - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Geography - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # prompt tokens": "{\"description\": \"min=394.773, mean=394.773, max=394.773, sum=789.545 (2)\", \"tab\": \"General information\", \"score\": \"394.77272727272725\"}", - "High School Geography - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # eval": "{\"description\": \"min=193, mean=193, max=193, sum=386 (2)\", \"tab\": \"General information\", \"score\": \"193.0\"}", - "High School Government And Politics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Government And Politics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # prompt tokens": "{\"description\": \"min=479.301, mean=479.301, max=479.301, sum=958.601 (2)\", \"tab\": \"General information\", \"score\": \"479.30051813471505\"}", - "High School Government And Politics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # eval": "{\"description\": \"min=390, mean=390, max=390, sum=780 (2)\", \"tab\": \"General information\", \"score\": \"390.0\"}", - "High School Macroeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Macroeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # prompt tokens": "{\"description\": \"min=396.541, mean=396.541, max=396.541, sum=793.082 (2)\", \"tab\": \"General information\", \"score\": \"396.54102564102567\"}", - "High School Macroeconomics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # eval": "{\"description\": \"min=270, mean=270, max=270, sum=540 (2)\", \"tab\": \"General information\", \"score\": \"270.0\"}", - "High School Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # prompt tokens": "{\"description\": \"min=566.822, mean=566.822, max=566.822, sum=1133.644 (2)\", \"tab\": \"General information\", \"score\": \"566.8222222222222\"}", - "High School Mathematics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # eval": "{\"description\": \"min=238, mean=238, max=238, sum=476 (2)\", \"tab\": \"General information\", \"score\": \"238.0\"}", - "High School Microeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Microeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # prompt tokens": "{\"description\": \"min=415.954, mean=415.954, max=415.954, sum=831.908 (2)\", \"tab\": \"General information\", \"score\": \"415.953781512605\"}", - "High School Microeconomics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # eval": "{\"description\": \"min=151, mean=151, max=151, sum=302 (2)\", \"tab\": \"General information\", \"score\": \"151.0\"}", - "High School Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # prompt tokens": "{\"description\": \"min=591.715, mean=591.715, max=591.715, sum=1183.43 (2)\", \"tab\": \"General information\", \"score\": \"591.7152317880794\"}", - "High School Physics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # eval": "{\"description\": \"min=545, mean=545, max=545, sum=1090 (2)\", \"tab\": \"General information\", \"score\": \"545.0\"}", - "High School Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # prompt tokens": "{\"description\": \"min=502.604, mean=502.604, max=502.604, sum=1005.207 (2)\", \"tab\": \"General information\", \"score\": \"502.60366972477067\"}", - "High School Psychology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # eval": "{\"description\": \"min=216, mean=216, max=216, sum=432 (2)\", \"tab\": \"General information\", \"score\": \"216.0\"}", - "High School Statistics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Statistics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # prompt tokens": "{\"description\": \"min=858.931, mean=858.931, max=858.931, sum=1717.861 (2)\", \"tab\": \"General information\", \"score\": \"858.9305555555555\"}", - "High School Statistics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # eval": "{\"description\": \"min=204, mean=204, max=204, sum=408 (2)\", \"tab\": \"General information\", \"score\": \"204.0\"}", - "High School US History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School US History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # prompt tokens": "{\"description\": \"min=2205.583, mean=2205.583, max=2205.583, sum=4411.167 (2)\", \"tab\": \"General information\", \"score\": \"2205.5833333333335\"}", - "High School US History - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # eval": "{\"description\": \"min=237, mean=237, max=237, sum=474 (2)\", \"tab\": \"General information\", \"score\": \"237.0\"}", - "High School World History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School World History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # prompt tokens": "{\"description\": \"min=1426.544, mean=1426.544, max=1426.544, sum=2853.089 (2)\", \"tab\": \"General information\", \"score\": \"1426.5443037974683\"}", - "High School World History - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"high_school_world_history\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_high_school_world_history\"" - } - } - }, - { - "evaluation_name": "Human Sexuality", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Human Sexuality", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.374, - "details": { - "description": "min=0.374, mean=0.374, max=0.374, sum=0.748 (2)", - "tab": "Accuracy", - "Human Aging - Observed inference time (s)": "{\"description\": \"min=0.45, mean=0.45, max=0.45, sum=0.901 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.45039264396701695\"}", - "Human Sexuality - Observed inference time (s)": "{\"description\": \"min=0.494, mean=0.494, max=0.494, sum=0.989 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.494300215931262\"}", - "Human Aging - # eval": "{\"description\": \"min=223, mean=223, max=223, sum=446 (2)\", \"tab\": \"General information\", \"score\": \"223.0\"}", - "Human Aging - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Aging - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Aging - # prompt tokens": "{\"description\": \"min=321.587, mean=321.587, max=321.587, sum=643.175 (2)\", \"tab\": \"General information\", \"score\": \"321.58744394618833\"}", - "Human Aging - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # eval": "{\"description\": \"min=131, mean=131, max=131, sum=262 (2)\", \"tab\": \"General information\", \"score\": \"131.0\"}", - "Human Sexuality - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Sexuality - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # prompt tokens": "{\"description\": \"min=347.183, mean=347.183, max=347.183, sum=694.366 (2)\", \"tab\": \"General information\", \"score\": \"347.1832061068702\"}", - "Human Sexuality - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"human_sexuality\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_human_sexuality\"" - } - } - }, - { - "evaluation_name": "International Law", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on International Law", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.901, - "details": { - "description": "min=0.901, mean=0.901, max=0.901, sum=1.802 (2)", - "tab": "Accuracy", - "International Law - Observed inference time (s)": "{\"description\": \"min=0.543, mean=0.543, max=0.543, sum=1.086 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5427691305964446\"}", - "International Law - # eval": "{\"description\": \"min=121, mean=121, max=121, sum=242 (2)\", \"tab\": \"General information\", \"score\": \"121.0\"}", - "International Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "International Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "International Law - # prompt tokens": "{\"description\": \"min=644.165, mean=644.165, max=644.165, sum=1288.331 (2)\", \"tab\": \"General information\", \"score\": \"644.1652892561983\"}", - "International Law - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"international_law\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_international_law\"" - } - } - }, - { - "evaluation_name": "Logical Fallacies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Logical Fallacies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.853, - "details": { - "description": "min=0.853, mean=0.853, max=0.853, sum=1.706 (2)", - "tab": "Accuracy", - "Logical Fallacies - Observed inference time (s)": "{\"description\": \"min=0.485, mean=0.485, max=0.485, sum=0.969 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.48451554263296304\"}", - "Logical Fallacies - # eval": "{\"description\": \"min=163, mean=163, max=163, sum=326 (2)\", \"tab\": \"General information\", \"score\": \"163.0\"}", - "Logical Fallacies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Logical Fallacies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Logical Fallacies - # prompt tokens": "{\"description\": \"min=450.049, mean=450.049, max=450.049, sum=900.098 (2)\", \"tab\": \"General information\", \"score\": \"450.0490797546012\"}", - "Logical Fallacies - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"logical_fallacies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_logical_fallacies\"" - } - } - }, - { - "evaluation_name": "Machine Learning", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Machine Learning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.571, - "details": { - "description": "min=0.571, mean=0.571, max=0.571, sum=1.143 (2)", - "tab": "Accuracy", - "Machine Learning - Observed inference time (s)": "{\"description\": \"min=0.515, mean=0.515, max=0.515, sum=1.029 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5145284725087029\"}", - "Machine Learning - # eval": "{\"description\": \"min=112, mean=112, max=112, sum=224 (2)\", \"tab\": \"General information\", \"score\": \"112.0\"}", - "Machine Learning - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Machine Learning - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Machine Learning - # prompt tokens": "{\"description\": \"min=702.402, mean=702.402, max=702.402, sum=1404.804 (2)\", \"tab\": \"General information\", \"score\": \"702.4017857142857\"}", - "Machine Learning - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"machine_learning\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_machine_learning\"" - } - } - }, - { - "evaluation_name": "Management", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Management", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.864, - "details": { - "description": "min=0.864, mean=0.864, max=0.864, sum=1.728 (2)", - "tab": "Accuracy", - "Management - Observed inference time (s)": "{\"description\": \"min=0.57, mean=0.57, max=0.57, sum=1.139 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5696360532519886\"}", - "Management - # eval": "{\"description\": \"min=103, mean=103, max=103, sum=206 (2)\", \"tab\": \"General information\", \"score\": \"103.0\"}", - "Management - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Management - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Management - # prompt tokens": "{\"description\": \"min=281.301, mean=281.301, max=281.301, sum=562.602 (2)\", \"tab\": \"General information\", \"score\": \"281.3009708737864\"}", - "Management - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"management\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_management\"" - } - } - }, - { - "evaluation_name": "Marketing", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Marketing", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.94, - "details": { - "description": "min=0.94, mean=0.94, max=0.94, sum=1.88 (2)", - "tab": "Accuracy", - "Marketing - Observed inference time (s)": "{\"description\": \"min=0.438, mean=0.438, max=0.438, sum=0.876 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.43808113204108345\"}", - "Marketing - # eval": "{\"description\": \"min=234, mean=234, max=234, sum=468 (2)\", \"tab\": \"General information\", \"score\": \"234.0\"}", - "Marketing - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Marketing - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Marketing - # prompt tokens": "{\"description\": \"min=428.35, mean=428.35, max=428.35, sum=856.701 (2)\", \"tab\": \"General information\", \"score\": \"428.35042735042737\"}", - "Marketing - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"marketing\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_marketing\"" - } - } - }, - { - "evaluation_name": "Medical Genetics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Medical Genetics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.86, - "details": { - "description": "min=0.86, mean=0.86, max=0.86, sum=1.72 (2)", - "tab": "Accuracy", - "Medical Genetics - Observed inference time (s)": "{\"description\": \"min=0.514, mean=0.514, max=0.514, sum=1.029 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.514304575920105\"}", - "Medical Genetics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Medical Genetics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Medical Genetics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Medical Genetics - # prompt tokens": "{\"description\": \"min=338.89, mean=338.89, max=338.89, sum=677.78 (2)\", \"tab\": \"General information\", \"score\": \"338.89\"}", - "Medical Genetics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"medical_genetics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_medical_genetics\"" - } - } - }, - { - "evaluation_name": "Miscellaneous", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Miscellaneous", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.886, - "details": { - "description": "min=0.886, mean=0.886, max=0.886, sum=1.773 (2)", - "tab": "Accuracy", - "Miscellaneous - Observed inference time (s)": "{\"description\": \"min=0.395, mean=0.395, max=0.395, sum=0.79 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3951411627870562\"}", - "Miscellaneous - # eval": "{\"description\": \"min=783, mean=783, max=783, sum=1566 (2)\", \"tab\": \"General information\", \"score\": \"783.0\"}", - "Miscellaneous - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Miscellaneous - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Miscellaneous - # prompt tokens": "{\"description\": \"min=314.669, mean=314.669, max=314.669, sum=629.338 (2)\", \"tab\": \"General information\", \"score\": \"314.669220945083\"}", - "Miscellaneous - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"miscellaneous\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_miscellaneous\"" - } - } - }, - { - "evaluation_name": "Moral Scenarios", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Moral Scenarios", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.637, - "details": { - "description": "min=0.637, mean=0.637, max=0.637, sum=1.274 (2)", - "tab": "Accuracy", - "Moral Disputes - Observed inference time (s)": "{\"description\": \"min=0.403, mean=0.403, max=0.403, sum=0.806 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4028203390646672\"}", - "Moral Scenarios - Observed inference time (s)": "{\"description\": \"min=0.4, mean=0.4, max=0.4, sum=0.801 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4004550709633243\"}", - "Moral Disputes - # eval": "{\"description\": \"min=346, mean=346, max=346, sum=692 (2)\", \"tab\": \"General information\", \"score\": \"346.0\"}", - "Moral Disputes - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Disputes - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Disputes - # prompt tokens": "{\"description\": \"min=495.003, mean=495.003, max=495.003, sum=990.006 (2)\", \"tab\": \"General information\", \"score\": \"495.0028901734104\"}", - "Moral Disputes - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # eval": "{\"description\": \"min=895, mean=895, max=895, sum=1790 (2)\", \"tab\": \"General information\", \"score\": \"895.0\"}", - "Moral Scenarios - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Scenarios - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # prompt tokens": "{\"description\": \"min=690.542, mean=690.542, max=690.542, sum=1381.084 (2)\", \"tab\": \"General information\", \"score\": \"690.5418994413408\"}", - "Moral Scenarios - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"moral_scenarios\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_moral_scenarios\"" - } - } - }, - { - "evaluation_name": "Nutrition", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Nutrition", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.82, - "details": { - "description": "min=0.82, mean=0.82, max=0.82, sum=1.641 (2)", - "tab": "Accuracy", - "Nutrition - Observed inference time (s)": "{\"description\": \"min=0.402, mean=0.402, max=0.402, sum=0.804 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4019969655018227\"}", - "Nutrition - # eval": "{\"description\": \"min=306, mean=306, max=306, sum=612 (2)\", \"tab\": \"General information\", \"score\": \"306.0\"}", - "Nutrition - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Nutrition - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Nutrition - # prompt tokens": "{\"description\": \"min=585.48, mean=585.48, max=585.48, sum=1170.961 (2)\", \"tab\": \"General information\", \"score\": \"585.4803921568628\"}", - "Nutrition - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"nutrition\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_nutrition\"" - } - } - }, - { - "evaluation_name": "Prehistory", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Prehistory", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.867, - "details": { - "description": "min=0.867, mean=0.867, max=0.867, sum=1.735 (2)", - "tab": "Accuracy", - "Prehistory - Observed inference time (s)": "{\"description\": \"min=0.403, mean=0.403, max=0.403, sum=0.807 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.40340044910525097\"}", - "Prehistory - # eval": "{\"description\": \"min=324, mean=324, max=324, sum=648 (2)\", \"tab\": \"General information\", \"score\": \"324.0\"}", - "Prehistory - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Prehistory - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Prehistory - # prompt tokens": "{\"description\": \"min=540.198, mean=540.198, max=540.198, sum=1080.395 (2)\", \"tab\": \"General information\", \"score\": \"540.1975308641976\"}", - "Prehistory - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"prehistory\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_prehistory\"" - } - } - }, - { - "evaluation_name": "Public Relations", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Public Relations", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.764, - "details": { - "description": "min=0.764, mean=0.764, max=0.764, sum=1.527 (2)", - "tab": "Accuracy", - "Public Relations - Observed inference time (s)": "{\"description\": \"min=0.554, mean=0.554, max=0.554, sum=1.109 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5543096672404896\"}", - "Public Relations - # eval": "{\"description\": \"min=110, mean=110, max=110, sum=220 (2)\", \"tab\": \"General information\", \"score\": \"110.0\"}", - "Public Relations - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Public Relations - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Public Relations - # prompt tokens": "{\"description\": \"min=426.655, mean=426.655, max=426.655, sum=853.309 (2)\", \"tab\": \"General information\", \"score\": \"426.6545454545454\"}", - "Public Relations - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"public_relations\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_public_relations\"" - } - } - }, - { - "evaluation_name": "Security Studies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Security Studies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.808, - "details": { - "description": "min=0.808, mean=0.808, max=0.808, sum=1.616 (2)", - "tab": "Accuracy", - "Security Studies - Observed inference time (s)": "{\"description\": \"min=0.456, mean=0.456, max=0.456, sum=0.913 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.45644889948319417\"}", - "Security Studies - # eval": "{\"description\": \"min=245, mean=245, max=245, sum=490 (2)\", \"tab\": \"General information\", \"score\": \"245.0\"}", - "Security Studies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Security Studies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Security Studies - # prompt tokens": "{\"description\": \"min=1193.869, mean=1193.869, max=1193.869, sum=2387.739 (2)\", \"tab\": \"General information\", \"score\": \"1193.869387755102\"}", - "Security Studies - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"security_studies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_security_studies\"" - } - } - }, - { - "evaluation_name": "Sociology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Sociology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.915, - "details": { - "description": "min=0.915, mean=0.915, max=0.915, sum=1.831 (2)", - "tab": "Accuracy", - "Sociology - Observed inference time (s)": "{\"description\": \"min=0.458, mean=0.458, max=0.458, sum=0.916 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4581311152349064\"}", - "Sociology - # eval": "{\"description\": \"min=201, mean=201, max=201, sum=402 (2)\", \"tab\": \"General information\", \"score\": \"201.0\"}", - "Sociology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Sociology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Sociology - # prompt tokens": "{\"description\": \"min=456.274, mean=456.274, max=456.274, sum=912.547 (2)\", \"tab\": \"General information\", \"score\": \"456.27363184079604\"}", - "Sociology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"sociology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_sociology\"" - } - } - }, - { - "evaluation_name": "Virology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Virology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.566, - "details": { - "description": "min=0.566, mean=0.566, max=0.566, sum=1.133 (2)", - "tab": "Accuracy", - "Virology - Observed inference time (s)": "{\"description\": \"min=0.45, mean=0.45, max=0.45, sum=0.899 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.44963935197117816\"}", - "Virology - # eval": "{\"description\": \"min=166, mean=166, max=166, sum=332 (2)\", \"tab\": \"General information\", \"score\": \"166.0\"}", - "Virology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Virology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Virology - # prompt tokens": "{\"description\": \"min=336.753, mean=336.753, max=336.753, sum=673.506 (2)\", \"tab\": \"General information\", \"score\": \"336.7530120481928\"}", - "Virology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"virology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_virology\"" - } - } - }, - { - "evaluation_name": "World Religions", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on World Religions", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.883, - "details": { - "description": "min=0.883, mean=0.883, max=0.883, sum=1.766 (2)", - "tab": "Accuracy", - "World Religions - Observed inference time (s)": "{\"description\": \"min=0.459, mean=0.459, max=0.459, sum=0.919 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.45928927890041416\"}", - "World Religions - # eval": "{\"description\": \"min=171, mean=171, max=171, sum=342 (2)\", \"tab\": \"General information\", \"score\": \"171.0\"}", - "World Religions - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "World Religions - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "World Religions - # prompt tokens": "{\"description\": \"min=268.164, mean=268.164, max=268.164, sum=536.327 (2)\", \"tab\": \"General information\", \"score\": \"268.1637426900585\"}", - "World Religions - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"world_religions\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_world_religions\"" - } - } - }, - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.47, - "details": { - "description": "", - "tab": "Efficiency" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_id": "reward-bench/google_gemini-1.5-flash-001/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8054 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9218 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6349 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8696 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8512 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6937 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/google_gemini-1.5-flash-002.json b/data/models/google_gemini-1.5-flash-002.json deleted file mode 100644 index b30b0a3ebc65e03d11dd8e98a195f05e62a05c1b..0000000000000000000000000000000000000000 --- a/data/models/google_gemini-1.5-flash-002.json +++ /dev/null @@ -1,2125 +0,0 @@ -{ - "model_info": { - "name": "Gemini 1.5 Flash 002", - "id": "google/gemini-1.5-flash-002", - "developer": "Google", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_capabilities/google_gemini-1.5-flash-002/1774096304.5056698", - "retrieved_timestamp": "1774096304.5056698", - "source_metadata": { - "source_name": "helm_capabilities", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_capabilities", - "evaluation_results": [ - { - "evaluation_name": "Mean score", - "source_data": { - "dataset_name": "helm_capabilities", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "The mean of the scores from all columns.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.609, - "details": { - "description": "", - "tab": "Accuracy", - "Mean score - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"3.3804760044252675\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU-Pro", - "source_data": { - "dataset_name": "MMLU-Pro", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on MMLU-Pro", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.678, - "details": { - "description": "min=0.678, mean=0.678, max=0.678, sum=0.678 (1)", - "tab": "Accuracy", - "MMLU-Pro - Observed inference time (s)": "{\"description\": \"min=1.799, mean=1.799, max=1.799, sum=1.799 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.799316755771637\"}", - "MMLU-Pro - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "MMLU-Pro - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - # prompt tokens": "{\"description\": \"min=242.673, mean=242.673, max=242.673, sum=242.673 (1)\", \"tab\": \"General information\", \"score\": \"242.673\"}", - "MMLU-Pro - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"all\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.437, - "details": { - "description": "min=0.437, mean=0.437, max=0.437, sum=0.437 (1)", - "tab": "Accuracy", - "GPQA - Observed inference time (s)": "{\"description\": \"min=2.79, mean=2.79, max=2.79, sum=2.79 (1)\", \"tab\": \"Efficiency\", \"score\": \"2.7900896457278677\"}", - "GPQA - # eval": "{\"description\": \"min=446, mean=446, max=446, sum=446 (1)\", \"tab\": \"General information\", \"score\": \"446.0\"}", - "GPQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - # prompt tokens": "{\"description\": \"min=252.735, mean=252.735, max=252.735, sum=252.735 (1)\", \"tab\": \"General information\", \"score\": \"252.7354260089686\"}", - "GPQA - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"gpqa_main\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "IFEval Strict Acc on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.831, - "details": { - "description": "min=0.831, mean=0.831, max=0.831, sum=0.831 (1)", - "tab": "Accuracy", - "IFEval - Observed inference time (s)": "{\"description\": \"min=2.302, mean=2.302, max=2.302, sum=2.302 (1)\", \"tab\": \"Efficiency\", \"score\": \"2.302485716320891\"}", - "IFEval - # eval": "{\"description\": \"min=541, mean=541, max=541, sum=541 (1)\", \"tab\": \"General information\", \"score\": \"541.0\"}", - "IFEval - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - # prompt tokens": "{\"description\": \"min=47.331, mean=47.331, max=47.331, sum=47.331 (1)\", \"tab\": \"General information\", \"score\": \"47.33086876155268\"}", - "IFEval - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WildBench", - "source_data": { - "dataset_name": "WildBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "WB Score on WildBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.792, - "details": { - "description": "min=0.792, mean=0.792, max=0.792, sum=0.792 (1)", - "tab": "Accuracy", - "WildBench - Observed inference time (s)": "{\"description\": \"min=5.328, mean=5.328, max=5.328, sum=5.328 (1)\", \"tab\": \"Efficiency\", \"score\": \"5.327828770410083\"}", - "WildBench - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "WildBench - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # prompt tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"v2\"" - } - } - }, - { - "evaluation_name": "Omni-MATH", - "source_data": { - "dataset_name": "Omni-MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Acc on Omni-MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.305, - "details": { - "description": "min=0.305, mean=0.305, max=0.305, sum=0.305 (1)", - "tab": "Accuracy", - "Omni-MATH - Observed inference time (s)": "{\"description\": \"min=4.683, mean=4.683, max=4.683, sum=4.683 (1)\", \"tab\": \"Efficiency\", \"score\": \"4.682659133895859\"}", - "Omni-MATH - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "Omni-MATH - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - # prompt tokens": "{\"description\": \"min=111.956, mean=111.956, max=111.956, sum=111.956 (1)\", \"tab\": \"General information\", \"score\": \"111.956\"}", - "Omni-MATH - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_lite/google_gemini-1.5-flash-002/1774096306.427425", - "retrieved_timestamp": "1774096306.427425", - "source_metadata": { - "source_name": "helm_lite", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_lite", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_lite", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.573, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.8933333333333333\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.746, - "details": { - "description": "min=0.746, mean=0.746, max=0.746, sum=0.746 (1)", - "tab": "Accuracy", - "NarrativeQA - Observed inference time (s)": "{\"description\": \"min=0.443, mean=0.443, max=0.443, sum=0.443 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.4433113621039824\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=3437.994, mean=3437.994, max=3437.994, sum=3437.994 (1)\", \"tab\": \"General information\", \"score\": \"3437.994366197183\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (closed-book)", - "source_data": { - "dataset_name": "NaturalQuestions (closed-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (closed-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.323, - "details": { - "description": "min=0.323, mean=0.323, max=0.323, sum=0.323 (1)", - "tab": "Accuracy", - "NaturalQuestions (open-book) - Observed inference time (s)": "{\"description\": \"min=0.379, mean=0.379, max=0.379, sum=0.379 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.37945408272743225\"}", - "NaturalQuestions (closed-book) - Observed inference time (s)": "{\"description\": \"min=0.37, mean=0.37, max=0.37, sum=0.37 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.36984835290908813\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1968.347, mean=1968.347, max=1968.347, sum=1968.347 (1)\", \"tab\": \"General information\", \"score\": \"1968.347\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=143.995, mean=143.995, max=143.995, sum=143.995 (1)\", \"tab\": \"General information\", \"score\": \"143.995\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "mode": "\"closedbook\"" - } - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.914, - "details": { - "description": "min=0.914, mean=0.914, max=0.914, sum=0.914 (1)", - "tab": "Accuracy", - "OpenbookQA - Observed inference time (s)": "{\"description\": \"min=0.303, mean=0.303, max=0.303, sum=0.303 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.302696533203125\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=248.508, mean=248.508, max=248.508, sum=248.508 (1)\", \"tab\": \"General information\", \"score\": \"248.508\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "dataset": "\"openbookqa\"", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.679, - "details": { - "description": "min=0.56, mean=0.679, max=0.81, sum=3.395 (5)", - "tab": "Accuracy", - "MMLU - Observed inference time (s)": "{\"description\": \"min=0.291, mean=0.296, max=0.299, sum=1.482 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.296430273214976\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=380.91, mean=481.531, max=634.553, sum=2407.653 (5)\", \"tab\": \"General information\", \"score\": \"481.5305263157895\"}", - "MMLU - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"college_chemistry\", \"computer_security\", \"econometrics\", \"us_foreign_policy\"]", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MATH", - "source_data": { - "dataset_name": "MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Equivalent (CoT) on MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.908, - "details": { - "description": "min=0.816, mean=0.908, max=0.985, sum=6.354 (7)", - "tab": "Accuracy", - "MATH - Observed inference time (s)": "{\"description\": \"min=0.701, mean=0.848, max=1.036, sum=5.939 (7)\", \"tab\": \"Efficiency\", \"score\": \"0.8483759753773942\"}", - "MATH - # eval": "{\"description\": \"min=30, mean=62.429, max=135, sum=437 (7)\", \"tab\": \"General information\", \"score\": \"62.42857142857143\"}", - "MATH - # train": "{\"description\": \"min=8, mean=8, max=8, sum=56 (7)\", \"tab\": \"General information\", \"score\": \"8.0\"}", - "MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MATH - # prompt tokens": "{\"description\": \"min=938.215, mean=1355.506, max=2348.712, sum=9488.545 (7)\", \"tab\": \"General information\", \"score\": \"1355.5064552904823\"}", - "MATH - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"algebra\", \"counting_and_probability\", \"geometry\", \"intermediate_algebra\", \"number_theory\", \"prealgebra\", \"precalculus\"]", - "level": "\"1\"", - "use_official_examples": "\"False\"", - "use_chain_of_thought": "\"True\"" - } - } - }, - { - "evaluation_name": "GSM8K", - "source_data": { - "dataset_name": "GSM8K", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on GSM8K", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.328, - "details": { - "description": "min=0.328, mean=0.328, max=0.328, sum=0.328 (1)", - "tab": "Accuracy", - "GSM8K - Observed inference time (s)": "{\"description\": \"min=0.859, mean=0.859, max=0.859, sum=0.859 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.8591284859287847\"}", - "GSM8K - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "GSM8K - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "GSM8K - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GSM8K - # prompt tokens": "{\"description\": \"min=1151.885, mean=1151.885, max=1151.885, sum=1151.885 (1)\", \"tab\": \"General information\", \"score\": \"1151.885\"}", - "GSM8K - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "LegalBench", - "source_data": { - "dataset_name": "LegalBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on LegalBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.67, - "details": { - "description": "min=0.42, mean=0.67, max=0.979, sum=3.35 (5)", - "tab": "Accuracy", - "LegalBench - Observed inference time (s)": "{\"description\": \"min=0.284, mean=0.347, max=0.541, sum=1.736 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.34728255842366473\"}", - "LegalBench - # eval": "{\"description\": \"min=95, mean=409.4, max=1000, sum=2047 (5)\", \"tab\": \"General information\", \"score\": \"409.4\"}", - "LegalBench - # train": "{\"description\": \"min=4, mean=4.8, max=5, sum=24 (5)\", \"tab\": \"General information\", \"score\": \"4.8\"}", - "LegalBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "LegalBench - # prompt tokens": "{\"description\": \"min=199.916, mean=1548.239, max=6413.569, sum=7741.193 (5)\", \"tab\": \"General information\", \"score\": \"1548.2386051001386\"}", - "LegalBench - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"abercrombie\", \"corporate_lobbying\", \"function_of_decision_section\", \"international_citizenship_questions\", \"proa\"]" - } - } - }, - { - "evaluation_name": "MedQA", - "source_data": { - "dataset_name": "MedQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MedQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.656, - "details": { - "description": "min=0.656, mean=0.656, max=0.656, sum=0.656 (1)", - "tab": "Accuracy", - "MedQA - Observed inference time (s)": "{\"description\": \"min=0.302, mean=0.302, max=0.302, sum=0.302 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.30154310163873327\"}", - "MedQA - # eval": "{\"description\": \"min=503, mean=503, max=503, sum=503 (1)\", \"tab\": \"General information\", \"score\": \"503.0\"}", - "MedQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MedQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MedQA - # prompt tokens": "{\"description\": \"min=1029.481, mean=1029.481, max=1029.481, sum=1029.481 (1)\", \"tab\": \"General information\", \"score\": \"1029.4811133200794\"}", - "MedQA - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WMT 2014", - "source_data": { - "dataset_name": "WMT 2014", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "BLEU-4 on WMT 2014", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.212, - "details": { - "description": "min=0.179, mean=0.212, max=0.232, sum=1.062 (5)", - "tab": "Accuracy", - "WMT 2014 - Observed inference time (s)": "{\"description\": \"min=0.407, mean=0.424, max=0.444, sum=2.119 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.42385545386168993\"}", - "WMT 2014 - # eval": "{\"description\": \"min=503, mean=568.8, max=832, sum=2844 (5)\", \"tab\": \"General information\", \"score\": \"568.8\"}", - "WMT 2014 - # train": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "WMT 2014 - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WMT 2014 - # prompt tokens": "{\"description\": \"min=80.732, mean=110.97, max=137.366, sum=554.851 (5)\", \"tab\": \"General information\", \"score\": \"110.97025108961614\"}", - "WMT 2014 - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "language_pair": "[\"cs-en\", \"de-en\", \"fr-en\", \"hi-en\", \"ru-en\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_mmlu/google_gemini-1.5-flash-002/1774096312.00548", - "retrieved_timestamp": "1774096312.00548", - "source_metadata": { - "source_name": "helm_mmlu", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_mmlu", - "evaluation_results": [ - { - "evaluation_name": "MMLU All Subjects", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU All Subjects", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.739, - "details": { - "description": "min=0.27, mean=0.739, max=0.959, sum=84.201 (114)", - "tab": "Accuracy", - "MMLU All Subjects - Observed inference time (s)": "{\"description\": \"min=0.262, mean=0.315, max=0.767, sum=35.937 (114)\", \"tab\": \"Efficiency\", \"score\": \"0.3152340762781926\"}", - "MMLU All Subjects - # eval": "{\"description\": \"min=100, mean=246.351, max=1534, sum=28084 (114)\", \"tab\": \"General information\", \"score\": \"246.35087719298247\"}", - "MMLU All Subjects - # train": "{\"description\": \"min=5, mean=5, max=5, sum=570 (114)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU All Subjects - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU All Subjects - # prompt tokens": "{\"description\": \"min=268.164, mean=632.617, max=2797.424, sum=72118.345 (114)\", \"tab\": \"General information\", \"score\": \"632.6170571214202\"}", - "MMLU All Subjects - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_name": "Abstract Algebra", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Abstract Algebra", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.63, - "details": { - "description": "min=0.63, mean=0.63, max=0.63, sum=1.26 (2)", - "tab": "Accuracy", - "Abstract Algebra - Observed inference time (s)": "{\"description\": \"min=0.298, mean=0.298, max=0.298, sum=0.596 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.29806760787963865\"}", - "Abstract Algebra - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Abstract Algebra - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Abstract Algebra - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Abstract Algebra - # prompt tokens": "{\"description\": \"min=383.97, mean=383.97, max=383.97, sum=767.94 (2)\", \"tab\": \"General information\", \"score\": \"383.97\"}", - "Abstract Algebra - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"abstract_algebra\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_abstract_algebra\"" - } - } - }, - { - "evaluation_name": "Anatomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Anatomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.793, - "details": { - "description": "min=0.793, mean=0.793, max=0.793, sum=1.585 (2)", - "tab": "Accuracy", - "Anatomy - Observed inference time (s)": "{\"description\": \"min=0.292, mean=0.292, max=0.292, sum=0.583 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.29152930047776965\"}", - "Anatomy - # eval": "{\"description\": \"min=135, mean=135, max=135, sum=270 (2)\", \"tab\": \"General information\", \"score\": \"135.0\"}", - "Anatomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Anatomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Anatomy - # prompt tokens": "{\"description\": \"min=344.356, mean=344.356, max=344.356, sum=688.711 (2)\", \"tab\": \"General information\", \"score\": \"344.35555555555555\"}", - "Anatomy - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"anatomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_anatomy\"" - } - } - }, - { - "evaluation_name": "College Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on College Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.637, - "details": { - "description": "min=0.637, mean=0.637, max=0.637, sum=1.275 (2)", - "tab": "Accuracy", - "College Chemistry - Observed inference time (s)": "{\"description\": \"min=0.299, mean=0.299, max=0.299, sum=0.598 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2988364624977112\"}", - "College Biology - Observed inference time (s)": "{\"description\": \"min=0.298, mean=0.298, max=0.298, sum=0.596 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.29801897870169747\"}", - "College Computer Science - Observed inference time (s)": "{\"description\": \"min=0.299, mean=0.299, max=0.299, sum=0.597 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2985741686820984\"}", - "College Mathematics - Observed inference time (s)": "{\"description\": \"min=0.288, mean=0.288, max=0.288, sum=0.576 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.28819103717803957\"}", - "College Medicine - Observed inference time (s)": "{\"description\": \"min=0.293, mean=0.293, max=0.293, sum=0.586 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.29290392495304174\"}", - "College Physics - Observed inference time (s)": "{\"description\": \"min=0.291, mean=0.291, max=0.291, sum=0.582 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.29088794483857994\"}", - "College Chemistry - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Chemistry - # prompt tokens": "{\"description\": \"min=570.02, mean=570.02, max=570.02, sum=1140.04 (2)\", \"tab\": \"General information\", \"score\": \"570.02\"}", - "College Chemistry - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # eval": "{\"description\": \"min=144, mean=144, max=144, sum=288 (2)\", \"tab\": \"General information\", \"score\": \"144.0\"}", - "College Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # prompt tokens": "{\"description\": \"min=482.799, mean=482.799, max=482.799, sum=965.597 (2)\", \"tab\": \"General information\", \"score\": \"482.7986111111111\"}", - "College Biology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # prompt tokens": "{\"description\": \"min=857.86, mean=857.86, max=857.86, sum=1715.72 (2)\", \"tab\": \"General information\", \"score\": \"857.86\"}", - "College Computer Science - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # prompt tokens": "{\"description\": \"min=626.69, mean=626.69, max=626.69, sum=1253.38 (2)\", \"tab\": \"General information\", \"score\": \"626.69\"}", - "College Mathematics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # eval": "{\"description\": \"min=173, mean=173, max=173, sum=346 (2)\", \"tab\": \"General information\", \"score\": \"173.0\"}", - "College Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # prompt tokens": "{\"description\": \"min=513.37, mean=513.37, max=513.37, sum=1026.74 (2)\", \"tab\": \"General information\", \"score\": \"513.3699421965318\"}", - "College Medicine - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # eval": "{\"description\": \"min=102, mean=102, max=102, sum=204 (2)\", \"tab\": \"General information\", \"score\": \"102.0\"}", - "College Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # prompt tokens": "{\"description\": \"min=507.471, mean=507.471, max=507.471, sum=1014.941 (2)\", \"tab\": \"General information\", \"score\": \"507.47058823529414\"}", - "College Physics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"college_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_college_physics\"" - } - } - }, - { - "evaluation_name": "Computer Security", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Computer Security", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.72, - "details": { - "description": "min=0.72, mean=0.72, max=0.72, sum=1.44 (2)", - "tab": "Accuracy", - "Computer Security - Observed inference time (s)": "{\"description\": \"min=0.299, mean=0.299, max=0.299, sum=0.598 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2992409729957581\"}", - "Computer Security - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Computer Security - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Computer Security - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Computer Security - # prompt tokens": "{\"description\": \"min=380.91, mean=380.91, max=380.91, sum=761.82 (2)\", \"tab\": \"General information\", \"score\": \"380.91\"}", - "Computer Security - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"computer_security\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_computer_security\"" - } - } - }, - { - "evaluation_name": "Econometrics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Econometrics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.675, - "details": { - "description": "min=0.675, mean=0.675, max=0.675, sum=1.351 (2)", - "tab": "Accuracy", - "Econometrics - Observed inference time (s)": "{\"description\": \"min=0.295, mean=0.295, max=0.295, sum=0.59 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.295004924138387\"}", - "Econometrics - # eval": "{\"description\": \"min=114, mean=114, max=114, sum=228 (2)\", \"tab\": \"General information\", \"score\": \"114.0\"}", - "Econometrics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Econometrics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Econometrics - # prompt tokens": "{\"description\": \"min=634.553, mean=634.553, max=634.553, sum=1269.105 (2)\", \"tab\": \"General information\", \"score\": \"634.5526315789474\"}", - "Econometrics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"econometrics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_econometrics\"" - } - } - }, - { - "evaluation_name": "Global Facts", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Global Facts", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.47, - "details": { - "description": "min=0.47, mean=0.47, max=0.47, sum=0.94 (2)", - "tab": "Accuracy", - "Global Facts - Observed inference time (s)": "{\"description\": \"min=0.301, mean=0.301, max=0.301, sum=0.602 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3007749605178833\"}", - "Global Facts - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Global Facts - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Global Facts - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Global Facts - # prompt tokens": "{\"description\": \"min=456.54, mean=456.54, max=456.54, sum=913.08 (2)\", \"tab\": \"General information\", \"score\": \"456.54\"}", - "Global Facts - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"global_facts\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_global_facts\"" - } - } - }, - { - "evaluation_name": "Jurisprudence", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Jurisprudence", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.852, - "details": { - "description": "min=0.852, mean=0.852, max=0.852, sum=1.704 (2)", - "tab": "Accuracy", - "Jurisprudence - Observed inference time (s)": "{\"description\": \"min=0.299, mean=0.299, max=0.299, sum=0.598 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2988583313094245\"}", - "Jurisprudence - # eval": "{\"description\": \"min=108, mean=108, max=108, sum=216 (2)\", \"tab\": \"General information\", \"score\": \"108.0\"}", - "Jurisprudence - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Jurisprudence - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Jurisprudence - # prompt tokens": "{\"description\": \"min=407.87, mean=407.87, max=407.87, sum=815.741 (2)\", \"tab\": \"General information\", \"score\": \"407.8703703703704\"}", - "Jurisprudence - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"jurisprudence\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_jurisprudence\"" - } - } - }, - { - "evaluation_name": "Philosophy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Philosophy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.797, - "details": { - "description": "min=0.797, mean=0.797, max=0.797, sum=1.595 (2)", - "tab": "Accuracy", - "Philosophy - Observed inference time (s)": "{\"description\": \"min=0.289, mean=0.289, max=0.289, sum=0.578 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2892080227278436\"}", - "Philosophy - # eval": "{\"description\": \"min=311, mean=311, max=311, sum=622 (2)\", \"tab\": \"General information\", \"score\": \"311.0\"}", - "Philosophy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Philosophy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Philosophy - # prompt tokens": "{\"description\": \"min=340.907, mean=340.907, max=340.907, sum=681.814 (2)\", \"tab\": \"General information\", \"score\": \"340.90675241157555\"}", - "Philosophy - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"philosophy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_philosophy\"" - } - } - }, - { - "evaluation_name": "Professional Psychology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Professional Psychology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.806, - "details": { - "description": "min=0.806, mean=0.806, max=0.806, sum=1.611 (2)", - "tab": "Accuracy", - "Professional Medicine - Observed inference time (s)": "{\"description\": \"min=0.303, mean=0.303, max=0.303, sum=0.605 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3027217843953301\"}", - "Professional Accounting - Observed inference time (s)": "{\"description\": \"min=0.318, mean=0.318, max=0.318, sum=0.636 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.318213385893098\"}", - "Professional Law - Observed inference time (s)": "{\"description\": \"min=0.344, mean=0.344, max=0.344, sum=0.687 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.34364056462881\"}", - "Professional Psychology - Observed inference time (s)": "{\"description\": \"min=0.366, mean=0.366, max=0.366, sum=0.732 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3660228084894567\"}", - "Professional Medicine - # eval": "{\"description\": \"min=272, mean=272, max=272, sum=544 (2)\", \"tab\": \"General information\", \"score\": \"272.0\"}", - "Professional Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Medicine - # prompt tokens": "{\"description\": \"min=1113.092, mean=1113.092, max=1113.092, sum=2226.184 (2)\", \"tab\": \"General information\", \"score\": \"1113.0919117647059\"}", - "Professional Medicine - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # eval": "{\"description\": \"min=282, mean=282, max=282, sum=564 (2)\", \"tab\": \"General information\", \"score\": \"282.0\"}", - "Professional Accounting - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Accounting - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # prompt tokens": "{\"description\": \"min=755.418, mean=755.418, max=755.418, sum=1510.837 (2)\", \"tab\": \"General information\", \"score\": \"755.418439716312\"}", - "Professional Accounting - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # eval": "{\"description\": \"min=1534, mean=1534, max=1534, sum=3068 (2)\", \"tab\": \"General information\", \"score\": \"1534.0\"}", - "Professional Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # prompt tokens": "{\"description\": \"min=1685.119, mean=1685.119, max=1685.119, sum=3370.239 (2)\", \"tab\": \"General information\", \"score\": \"1685.119295958279\"}", - "Professional Law - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # eval": "{\"description\": \"min=612, mean=612, max=612, sum=1224 (2)\", \"tab\": \"General information\", \"score\": \"612.0\"}", - "Professional Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # prompt tokens": "{\"description\": \"min=594.363, mean=594.363, max=594.363, sum=1188.725 (2)\", \"tab\": \"General information\", \"score\": \"594.3627450980392\"}", - "Professional Psychology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"professional_psychology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_professional_psychology\"" - } - } - }, - { - "evaluation_name": "Us Foreign Policy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Us Foreign Policy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.81, - "details": { - "description": "min=0.81, mean=0.81, max=0.81, sum=1.62 (2)", - "tab": "Accuracy", - "Us Foreign Policy - Observed inference time (s)": "{\"description\": \"min=0.291, mean=0.291, max=0.291, sum=0.582 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.291001398563385\"}", - "Us Foreign Policy - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Us Foreign Policy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Us Foreign Policy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Us Foreign Policy - # prompt tokens": "{\"description\": \"min=438.2, mean=438.2, max=438.2, sum=876.4 (2)\", \"tab\": \"General information\", \"score\": \"438.2\"}", - "Us Foreign Policy - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"us_foreign_policy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_us_foreign_policy\"" - } - } - }, - { - "evaluation_name": "Astronomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Astronomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.895, - "details": { - "description": "min=0.895, mean=0.895, max=0.895, sum=1.789 (2)", - "tab": "Accuracy", - "Astronomy - Observed inference time (s)": "{\"description\": \"min=0.292, mean=0.292, max=0.292, sum=0.584 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2922459558436745\"}", - "Astronomy - # eval": "{\"description\": \"min=152, mean=152, max=152, sum=304 (2)\", \"tab\": \"General information\", \"score\": \"152.0\"}", - "Astronomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Astronomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Astronomy - # prompt tokens": "{\"description\": \"min=602.421, mean=602.421, max=602.421, sum=1204.842 (2)\", \"tab\": \"General information\", \"score\": \"602.421052631579\"}", - "Astronomy - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"astronomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_astronomy\"" - } - } - }, - { - "evaluation_name": "Business Ethics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Business Ethics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.27, - "details": { - "description": "min=0.27, mean=0.27, max=0.27, sum=0.54 (2)", - "tab": "Accuracy", - "Business Ethics - Observed inference time (s)": "{\"description\": \"min=0.3, mean=0.3, max=0.3, sum=0.6 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.29986772060394284\"}", - "Business Ethics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Business Ethics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Business Ethics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Business Ethics - # prompt tokens": "{\"description\": \"min=552.87, mean=552.87, max=552.87, sum=1105.74 (2)\", \"tab\": \"General information\", \"score\": \"552.87\"}", - "Business Ethics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"business_ethics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_business_ethics\"" - } - } - }, - { - "evaluation_name": "Clinical Knowledge", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Clinical Knowledge", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.792, - "details": { - "description": "min=0.792, mean=0.792, max=0.792, sum=1.585 (2)", - "tab": "Accuracy", - "Clinical Knowledge - Observed inference time (s)": "{\"description\": \"min=0.3, mean=0.3, max=0.3, sum=0.601 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3003354540411031\"}", - "Clinical Knowledge - # eval": "{\"description\": \"min=265, mean=265, max=265, sum=530 (2)\", \"tab\": \"General information\", \"score\": \"265.0\"}", - "Clinical Knowledge - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Clinical Knowledge - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Clinical Knowledge - # prompt tokens": "{\"description\": \"min=402.592, mean=402.592, max=402.592, sum=805.185 (2)\", \"tab\": \"General information\", \"score\": \"402.5924528301887\"}", - "Clinical Knowledge - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"clinical_knowledge\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_clinical_knowledge\"" - } - } - }, - { - "evaluation_name": "Conceptual Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Conceptual Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.851, - "details": { - "description": "min=0.851, mean=0.851, max=0.851, sum=1.702 (2)", - "tab": "Accuracy", - "Conceptual Physics - Observed inference time (s)": "{\"description\": \"min=0.288, mean=0.288, max=0.288, sum=0.575 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.28759900458315585\"}", - "Conceptual Physics - # eval": "{\"description\": \"min=235, mean=235, max=235, sum=470 (2)\", \"tab\": \"General information\", \"score\": \"235.0\"}", - "Conceptual Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Conceptual Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Conceptual Physics - # prompt tokens": "{\"description\": \"min=309.213, mean=309.213, max=309.213, sum=618.426 (2)\", \"tab\": \"General information\", \"score\": \"309.21276595744683\"}", - "Conceptual Physics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"conceptual_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_conceptual_physics\"" - } - } - }, - { - "evaluation_name": "Electrical Engineering", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Electrical Engineering", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.772, - "details": { - "description": "min=0.772, mean=0.772, max=0.772, sum=1.545 (2)", - "tab": "Accuracy", - "Electrical Engineering - Observed inference time (s)": "{\"description\": \"min=0.294, mean=0.294, max=0.294, sum=0.588 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2938007436949631\"}", - "Electrical Engineering - # eval": "{\"description\": \"min=145, mean=145, max=145, sum=290 (2)\", \"tab\": \"General information\", \"score\": \"145.0\"}", - "Electrical Engineering - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Electrical Engineering - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Electrical Engineering - # prompt tokens": "{\"description\": \"min=474.786, mean=474.786, max=474.786, sum=949.572 (2)\", \"tab\": \"General information\", \"score\": \"474.78620689655173\"}", - "Electrical Engineering - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"electrical_engineering\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_electrical_engineering\"" - } - } - }, - { - "evaluation_name": "Elementary Mathematics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Elementary Mathematics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.704, - "details": { - "description": "min=0.704, mean=0.704, max=0.704, sum=1.407 (2)", - "tab": "Accuracy", - "Elementary Mathematics - Observed inference time (s)": "{\"description\": \"min=0.295, mean=0.295, max=0.295, sum=0.59 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.29476307119641987\"}", - "Elementary Mathematics - # eval": "{\"description\": \"min=378, mean=378, max=378, sum=756 (2)\", \"tab\": \"General information\", \"score\": \"378.0\"}", - "Elementary Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Elementary Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Elementary Mathematics - # prompt tokens": "{\"description\": \"min=597.341, mean=597.341, max=597.341, sum=1194.683 (2)\", \"tab\": \"General information\", \"score\": \"597.3412698412699\"}", - "Elementary Mathematics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"elementary_mathematics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_elementary_mathematics\"" - } - } - }, - { - "evaluation_name": "Formal Logic", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Formal Logic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.595, - "details": { - "description": "min=0.595, mean=0.595, max=0.595, sum=1.19 (2)", - "tab": "Accuracy", - "Formal Logic - Observed inference time (s)": "{\"description\": \"min=0.283, mean=0.283, max=0.283, sum=0.567 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.28335455107310464\"}", - "Formal Logic - # eval": "{\"description\": \"min=126, mean=126, max=126, sum=252 (2)\", \"tab\": \"General information\", \"score\": \"126.0\"}", - "Formal Logic - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Formal Logic - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Formal Logic - # prompt tokens": "{\"description\": \"min=619.563, mean=619.563, max=619.563, sum=1239.127 (2)\", \"tab\": \"General information\", \"score\": \"619.563492063492\"}", - "Formal Logic - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"formal_logic\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_formal_logic\"" - } - } - }, - { - "evaluation_name": "High School World History", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on High School World History", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.869, - "details": { - "description": "min=0.869, mean=0.869, max=0.869, sum=1.738 (2)", - "tab": "Accuracy", - "High School Biology - Observed inference time (s)": "{\"description\": \"min=0.29, mean=0.29, max=0.29, sum=0.58 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2898174070542858\"}", - "High School Chemistry - Observed inference time (s)": "{\"description\": \"min=0.276, mean=0.276, max=0.276, sum=0.553 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.27643810704423877\"}", - "High School Computer Science - Observed inference time (s)": "{\"description\": \"min=0.29, mean=0.29, max=0.29, sum=0.579 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.28958702087402344\"}", - "High School European History - Observed inference time (s)": "{\"description\": \"min=0.369, mean=0.369, max=0.369, sum=0.739 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.369471347693241\"}", - "High School Geography - Observed inference time (s)": "{\"description\": \"min=0.278, mean=0.278, max=0.278, sum=0.556 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2780994249112678\"}", - "High School Government And Politics - Observed inference time (s)": "{\"description\": \"min=0.269, mean=0.269, max=0.269, sum=0.538 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.26881929382759057\"}", - "High School Macroeconomics - Observed inference time (s)": "{\"description\": \"min=0.27, mean=0.27, max=0.27, sum=0.54 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2700315811695197\"}", - "High School Mathematics - Observed inference time (s)": "{\"description\": \"min=0.267, mean=0.267, max=0.267, sum=0.534 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2672289636400011\"}", - "High School Microeconomics - Observed inference time (s)": "{\"description\": \"min=0.262, mean=0.262, max=0.262, sum=0.525 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2623477593189528\"}", - "High School Physics - Observed inference time (s)": "{\"description\": \"min=0.292, mean=0.292, max=0.292, sum=0.583 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2917157135262395\"}", - "High School Psychology - Observed inference time (s)": "{\"description\": \"min=0.269, mean=0.269, max=0.269, sum=0.537 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2685232871169344\"}", - "High School Statistics - Observed inference time (s)": "{\"description\": \"min=0.502, mean=0.502, max=0.502, sum=1.004 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5018655392858717\"}", - "High School US History - Observed inference time (s)": "{\"description\": \"min=0.436, mean=0.436, max=0.436, sum=0.873 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4363996000850902\"}", - "High School World History - Observed inference time (s)": "{\"description\": \"min=0.33, mean=0.33, max=0.33, sum=0.66 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3298424698632478\"}", - "High School Biology - # eval": "{\"description\": \"min=310, mean=310, max=310, sum=620 (2)\", \"tab\": \"General information\", \"score\": \"310.0\"}", - "High School Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Biology - # prompt tokens": "{\"description\": \"min=500.958, mean=500.958, max=500.958, sum=1001.916 (2)\", \"tab\": \"General information\", \"score\": \"500.958064516129\"}", - "High School Biology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # eval": "{\"description\": \"min=203, mean=203, max=203, sum=406 (2)\", \"tab\": \"General information\", \"score\": \"203.0\"}", - "High School Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # prompt tokens": "{\"description\": \"min=513.064, mean=513.064, max=513.064, sum=1026.128 (2)\", \"tab\": \"General information\", \"score\": \"513.064039408867\"}", - "High School Chemistry - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "High School Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # prompt tokens": "{\"description\": \"min=935.13, mean=935.13, max=935.13, sum=1870.26 (2)\", \"tab\": \"General information\", \"score\": \"935.13\"}", - "High School Computer Science - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # eval": "{\"description\": \"min=165, mean=165, max=165, sum=330 (2)\", \"tab\": \"General information\", \"score\": \"165.0\"}", - "High School European History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School European History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # prompt tokens": "{\"description\": \"min=2797.424, mean=2797.424, max=2797.424, sum=5594.848 (2)\", \"tab\": \"General information\", \"score\": \"2797.4242424242425\"}", - "High School European History - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # eval": "{\"description\": \"min=198, mean=198, max=198, sum=396 (2)\", \"tab\": \"General information\", \"score\": \"198.0\"}", - "High School Geography - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Geography - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # prompt tokens": "{\"description\": \"min=394.773, mean=394.773, max=394.773, sum=789.545 (2)\", \"tab\": \"General information\", \"score\": \"394.77272727272725\"}", - "High School Geography - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # eval": "{\"description\": \"min=193, mean=193, max=193, sum=386 (2)\", \"tab\": \"General information\", \"score\": \"193.0\"}", - "High School Government And Politics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Government And Politics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # prompt tokens": "{\"description\": \"min=479.301, mean=479.301, max=479.301, sum=958.601 (2)\", \"tab\": \"General information\", \"score\": \"479.30051813471505\"}", - "High School Government And Politics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # eval": "{\"description\": \"min=390, mean=390, max=390, sum=780 (2)\", \"tab\": \"General information\", \"score\": \"390.0\"}", - "High School Macroeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Macroeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # prompt tokens": "{\"description\": \"min=396.541, mean=396.541, max=396.541, sum=793.082 (2)\", \"tab\": \"General information\", \"score\": \"396.54102564102567\"}", - "High School Macroeconomics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # eval": "{\"description\": \"min=270, mean=270, max=270, sum=540 (2)\", \"tab\": \"General information\", \"score\": \"270.0\"}", - "High School Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # prompt tokens": "{\"description\": \"min=566.822, mean=566.822, max=566.822, sum=1133.644 (2)\", \"tab\": \"General information\", \"score\": \"566.8222222222222\"}", - "High School Mathematics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # eval": "{\"description\": \"min=238, mean=238, max=238, sum=476 (2)\", \"tab\": \"General information\", \"score\": \"238.0\"}", - "High School Microeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Microeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # prompt tokens": "{\"description\": \"min=415.954, mean=415.954, max=415.954, sum=831.908 (2)\", \"tab\": \"General information\", \"score\": \"415.953781512605\"}", - "High School Microeconomics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # eval": "{\"description\": \"min=151, mean=151, max=151, sum=302 (2)\", \"tab\": \"General information\", \"score\": \"151.0\"}", - "High School Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # prompt tokens": "{\"description\": \"min=591.715, mean=591.715, max=591.715, sum=1183.43 (2)\", \"tab\": \"General information\", \"score\": \"591.7152317880794\"}", - "High School Physics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # eval": "{\"description\": \"min=545, mean=545, max=545, sum=1090 (2)\", \"tab\": \"General information\", \"score\": \"545.0\"}", - "High School Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # prompt tokens": "{\"description\": \"min=502.604, mean=502.604, max=502.604, sum=1005.207 (2)\", \"tab\": \"General information\", \"score\": \"502.60366972477067\"}", - "High School Psychology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # eval": "{\"description\": \"min=216, mean=216, max=216, sum=432 (2)\", \"tab\": \"General information\", \"score\": \"216.0\"}", - "High School Statistics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Statistics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # prompt tokens": "{\"description\": \"min=858.931, mean=858.931, max=858.931, sum=1717.861 (2)\", \"tab\": \"General information\", \"score\": \"858.9305555555555\"}", - "High School Statistics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # eval": "{\"description\": \"min=204, mean=204, max=204, sum=408 (2)\", \"tab\": \"General information\", \"score\": \"204.0\"}", - "High School US History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School US History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # prompt tokens": "{\"description\": \"min=2205.583, mean=2205.583, max=2205.583, sum=4411.167 (2)\", \"tab\": \"General information\", \"score\": \"2205.5833333333335\"}", - "High School US History - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # eval": "{\"description\": \"min=237, mean=237, max=237, sum=474 (2)\", \"tab\": \"General information\", \"score\": \"237.0\"}", - "High School World History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School World History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # prompt tokens": "{\"description\": \"min=1426.544, mean=1426.544, max=1426.544, sum=2853.089 (2)\", \"tab\": \"General information\", \"score\": \"1426.5443037974683\"}", - "High School World History - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"high_school_world_history\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_high_school_world_history\"" - } - } - }, - { - "evaluation_name": "Human Sexuality", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Human Sexuality", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.847, - "details": { - "description": "min=0.847, mean=0.847, max=0.847, sum=1.695 (2)", - "tab": "Accuracy", - "Human Aging - Observed inference time (s)": "{\"description\": \"min=0.278, mean=0.278, max=0.278, sum=0.555 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2775634660849122\"}", - "Human Sexuality - Observed inference time (s)": "{\"description\": \"min=0.416, mean=0.416, max=0.416, sum=0.832 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.41606709793323776\"}", - "Human Aging - # eval": "{\"description\": \"min=223, mean=223, max=223, sum=446 (2)\", \"tab\": \"General information\", \"score\": \"223.0\"}", - "Human Aging - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Aging - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Aging - # prompt tokens": "{\"description\": \"min=321.587, mean=321.587, max=321.587, sum=643.175 (2)\", \"tab\": \"General information\", \"score\": \"321.58744394618833\"}", - "Human Aging - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # eval": "{\"description\": \"min=131, mean=131, max=131, sum=262 (2)\", \"tab\": \"General information\", \"score\": \"131.0\"}", - "Human Sexuality - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Sexuality - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # prompt tokens": "{\"description\": \"min=347.183, mean=347.183, max=347.183, sum=694.366 (2)\", \"tab\": \"General information\", \"score\": \"347.1832061068702\"}", - "Human Sexuality - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"human_sexuality\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_human_sexuality\"" - } - } - }, - { - "evaluation_name": "International Law", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on International Law", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.752, - "details": { - "description": "min=0.752, mean=0.752, max=0.752, sum=1.504 (2)", - "tab": "Accuracy", - "International Law - Observed inference time (s)": "{\"description\": \"min=0.268, mean=0.268, max=0.268, sum=0.535 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.267673009683278\"}", - "International Law - # eval": "{\"description\": \"min=121, mean=121, max=121, sum=242 (2)\", \"tab\": \"General information\", \"score\": \"121.0\"}", - "International Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "International Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "International Law - # prompt tokens": "{\"description\": \"min=644.165, mean=644.165, max=644.165, sum=1288.331 (2)\", \"tab\": \"General information\", \"score\": \"644.1652892561983\"}", - "International Law - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"international_law\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_international_law\"" - } - } - }, - { - "evaluation_name": "Logical Fallacies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Logical Fallacies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.859, - "details": { - "description": "min=0.859, mean=0.859, max=0.859, sum=1.718 (2)", - "tab": "Accuracy", - "Logical Fallacies - Observed inference time (s)": "{\"description\": \"min=0.268, mean=0.268, max=0.268, sum=0.535 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2676804094958159\"}", - "Logical Fallacies - # eval": "{\"description\": \"min=163, mean=163, max=163, sum=326 (2)\", \"tab\": \"General information\", \"score\": \"163.0\"}", - "Logical Fallacies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Logical Fallacies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Logical Fallacies - # prompt tokens": "{\"description\": \"min=450.049, mean=450.049, max=450.049, sum=900.098 (2)\", \"tab\": \"General information\", \"score\": \"450.0490797546012\"}", - "Logical Fallacies - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"logical_fallacies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_logical_fallacies\"" - } - } - }, - { - "evaluation_name": "Machine Learning", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Machine Learning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.616, - "details": { - "description": "min=0.616, mean=0.616, max=0.616, sum=1.232 (2)", - "tab": "Accuracy", - "Machine Learning - Observed inference time (s)": "{\"description\": \"min=0.27, mean=0.27, max=0.27, sum=0.539 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2695028483867645\"}", - "Machine Learning - # eval": "{\"description\": \"min=112, mean=112, max=112, sum=224 (2)\", \"tab\": \"General information\", \"score\": \"112.0\"}", - "Machine Learning - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Machine Learning - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Machine Learning - # prompt tokens": "{\"description\": \"min=702.402, mean=702.402, max=702.402, sum=1404.804 (2)\", \"tab\": \"General information\", \"score\": \"702.4017857142857\"}", - "Machine Learning - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"machine_learning\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_machine_learning\"" - } - } - }, - { - "evaluation_name": "Management", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Management", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.893, - "details": { - "description": "min=0.893, mean=0.893, max=0.893, sum=1.786 (2)", - "tab": "Accuracy", - "Management - Observed inference time (s)": "{\"description\": \"min=0.332, mean=0.332, max=0.332, sum=0.665 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3324842568740104\"}", - "Management - # eval": "{\"description\": \"min=103, mean=103, max=103, sum=206 (2)\", \"tab\": \"General information\", \"score\": \"103.0\"}", - "Management - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Management - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Management - # prompt tokens": "{\"description\": \"min=281.301, mean=281.301, max=281.301, sum=562.602 (2)\", \"tab\": \"General information\", \"score\": \"281.3009708737864\"}", - "Management - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"management\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_management\"" - } - } - }, - { - "evaluation_name": "Marketing", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Marketing", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.953, - "details": { - "description": "min=0.953, mean=0.953, max=0.953, sum=1.906 (2)", - "tab": "Accuracy", - "Marketing - Observed inference time (s)": "{\"description\": \"min=0.767, mean=0.767, max=0.767, sum=1.533 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7665768270818596\"}", - "Marketing - # eval": "{\"description\": \"min=234, mean=234, max=234, sum=468 (2)\", \"tab\": \"General information\", \"score\": \"234.0\"}", - "Marketing - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Marketing - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Marketing - # prompt tokens": "{\"description\": \"min=428.35, mean=428.35, max=428.35, sum=856.701 (2)\", \"tab\": \"General information\", \"score\": \"428.35042735042737\"}", - "Marketing - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"marketing\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_marketing\"" - } - } - }, - { - "evaluation_name": "Medical Genetics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Medical Genetics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.89, - "details": { - "description": "min=0.89, mean=0.89, max=0.89, sum=1.78 (2)", - "tab": "Accuracy", - "Medical Genetics - Observed inference time (s)": "{\"description\": \"min=0.297, mean=0.297, max=0.297, sum=0.594 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2972432613372803\"}", - "Medical Genetics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Medical Genetics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Medical Genetics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Medical Genetics - # prompt tokens": "{\"description\": \"min=338.89, mean=338.89, max=338.89, sum=677.78 (2)\", \"tab\": \"General information\", \"score\": \"338.89\"}", - "Medical Genetics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"medical_genetics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_medical_genetics\"" - } - } - }, - { - "evaluation_name": "Miscellaneous", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Miscellaneous", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9, - "details": { - "description": "min=0.9, mean=0.9, max=0.9, sum=1.801 (2)", - "tab": "Accuracy", - "Miscellaneous - Observed inference time (s)": "{\"description\": \"min=0.425, mean=0.425, max=0.425, sum=0.849 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4247035331652996\"}", - "Miscellaneous - # eval": "{\"description\": \"min=783, mean=783, max=783, sum=1566 (2)\", \"tab\": \"General information\", \"score\": \"783.0\"}", - "Miscellaneous - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Miscellaneous - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Miscellaneous - # prompt tokens": "{\"description\": \"min=314.669, mean=314.669, max=314.669, sum=629.338 (2)\", \"tab\": \"General information\", \"score\": \"314.669220945083\"}", - "Miscellaneous - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"miscellaneous\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_miscellaneous\"" - } - } - }, - { - "evaluation_name": "Moral Scenarios", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Moral Scenarios", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.676, - "details": { - "description": "min=0.676, mean=0.676, max=0.676, sum=1.352 (2)", - "tab": "Accuracy", - "Moral Disputes - Observed inference time (s)": "{\"description\": \"min=0.297, mean=0.297, max=0.297, sum=0.593 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2965996671963289\"}", - "Moral Scenarios - Observed inference time (s)": "{\"description\": \"min=0.297, mean=0.297, max=0.297, sum=0.593 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.29666628491279134\"}", - "Moral Disputes - # eval": "{\"description\": \"min=346, mean=346, max=346, sum=692 (2)\", \"tab\": \"General information\", \"score\": \"346.0\"}", - "Moral Disputes - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Disputes - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Disputes - # prompt tokens": "{\"description\": \"min=495.003, mean=495.003, max=495.003, sum=990.006 (2)\", \"tab\": \"General information\", \"score\": \"495.0028901734104\"}", - "Moral Disputes - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # eval": "{\"description\": \"min=895, mean=895, max=895, sum=1790 (2)\", \"tab\": \"General information\", \"score\": \"895.0\"}", - "Moral Scenarios - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Scenarios - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # prompt tokens": "{\"description\": \"min=690.542, mean=690.542, max=690.542, sum=1381.084 (2)\", \"tab\": \"General information\", \"score\": \"690.5418994413408\"}", - "Moral Scenarios - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"moral_scenarios\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_moral_scenarios\"" - } - } - }, - { - "evaluation_name": "Nutrition", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Nutrition", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.588, - "details": { - "description": "min=0.588, mean=0.588, max=0.588, sum=1.176 (2)", - "tab": "Accuracy", - "Nutrition - Observed inference time (s)": "{\"description\": \"min=0.288, mean=0.288, max=0.288, sum=0.575 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2876783258774701\"}", - "Nutrition - # eval": "{\"description\": \"min=306, mean=306, max=306, sum=612 (2)\", \"tab\": \"General information\", \"score\": \"306.0\"}", - "Nutrition - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Nutrition - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Nutrition - # prompt tokens": "{\"description\": \"min=585.48, mean=585.48, max=585.48, sum=1170.961 (2)\", \"tab\": \"General information\", \"score\": \"585.4803921568628\"}", - "Nutrition - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"nutrition\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_nutrition\"" - } - } - }, - { - "evaluation_name": "Prehistory", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Prehistory", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.762, - "details": { - "description": "min=0.762, mean=0.762, max=0.762, sum=1.525 (2)", - "tab": "Accuracy", - "Prehistory - Observed inference time (s)": "{\"description\": \"min=0.3, mean=0.3, max=0.3, sum=0.6 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3001174411655944\"}", - "Prehistory - # eval": "{\"description\": \"min=324, mean=324, max=324, sum=648 (2)\", \"tab\": \"General information\", \"score\": \"324.0\"}", - "Prehistory - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Prehistory - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Prehistory - # prompt tokens": "{\"description\": \"min=540.198, mean=540.198, max=540.198, sum=1080.395 (2)\", \"tab\": \"General information\", \"score\": \"540.1975308641976\"}", - "Prehistory - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"prehistory\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_prehistory\"" - } - } - }, - { - "evaluation_name": "Public Relations", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Public Relations", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7, - "details": { - "description": "min=0.7, mean=0.7, max=0.7, sum=1.4 (2)", - "tab": "Accuracy", - "Public Relations - Observed inference time (s)": "{\"description\": \"min=0.286, mean=0.286, max=0.286, sum=0.572 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2860603137449785\"}", - "Public Relations - # eval": "{\"description\": \"min=110, mean=110, max=110, sum=220 (2)\", \"tab\": \"General information\", \"score\": \"110.0\"}", - "Public Relations - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Public Relations - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Public Relations - # prompt tokens": "{\"description\": \"min=426.655, mean=426.655, max=426.655, sum=853.309 (2)\", \"tab\": \"General information\", \"score\": \"426.6545454545454\"}", - "Public Relations - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"public_relations\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_public_relations\"" - } - } - }, - { - "evaluation_name": "Security Studies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Security Studies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.547, - "details": { - "description": "min=0.547, mean=0.547, max=0.547, sum=1.094 (2)", - "tab": "Accuracy", - "Security Studies - Observed inference time (s)": "{\"description\": \"min=0.398, mean=0.398, max=0.398, sum=0.795 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3977492381115349\"}", - "Security Studies - # eval": "{\"description\": \"min=245, mean=245, max=245, sum=490 (2)\", \"tab\": \"General information\", \"score\": \"245.0\"}", - "Security Studies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Security Studies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Security Studies - # prompt tokens": "{\"description\": \"min=1193.869, mean=1193.869, max=1193.869, sum=2387.739 (2)\", \"tab\": \"General information\", \"score\": \"1193.869387755102\"}", - "Security Studies - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"security_studies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_security_studies\"" - } - } - }, - { - "evaluation_name": "Sociology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Sociology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.851, - "details": { - "description": "min=0.851, mean=0.851, max=0.851, sum=1.701 (2)", - "tab": "Accuracy", - "Sociology - Observed inference time (s)": "{\"description\": \"min=0.295, mean=0.295, max=0.295, sum=0.59 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.29507939969722313\"}", - "Sociology - # eval": "{\"description\": \"min=201, mean=201, max=201, sum=402 (2)\", \"tab\": \"General information\", \"score\": \"201.0\"}", - "Sociology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Sociology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Sociology - # prompt tokens": "{\"description\": \"min=456.274, mean=456.274, max=456.274, sum=912.547 (2)\", \"tab\": \"General information\", \"score\": \"456.27363184079604\"}", - "Sociology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"sociology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_sociology\"" - } - } - }, - { - "evaluation_name": "Virology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Virology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.524, - "details": { - "description": "min=0.524, mean=0.524, max=0.524, sum=1.048 (2)", - "tab": "Accuracy", - "Virology - Observed inference time (s)": "{\"description\": \"min=0.287, mean=0.287, max=0.287, sum=0.574 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.28698748852833206\"}", - "Virology - # eval": "{\"description\": \"min=166, mean=166, max=166, sum=332 (2)\", \"tab\": \"General information\", \"score\": \"166.0\"}", - "Virology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Virology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Virology - # prompt tokens": "{\"description\": \"min=336.753, mean=336.753, max=336.753, sum=673.506 (2)\", \"tab\": \"General information\", \"score\": \"336.7530120481928\"}", - "Virology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"virology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_virology\"" - } - } - }, - { - "evaluation_name": "World Religions", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on World Religions", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.865, - "details": { - "description": "min=0.865, mean=0.865, max=0.865, sum=1.731 (2)", - "tab": "Accuracy", - "World Religions - Observed inference time (s)": "{\"description\": \"min=0.288, mean=0.288, max=0.288, sum=0.576 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2880588832654451\"}", - "World Religions - # eval": "{\"description\": \"min=171, mean=171, max=171, sum=342 (2)\", \"tab\": \"General information\", \"score\": \"171.0\"}", - "World Religions - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "World Religions - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "World Religions - # prompt tokens": "{\"description\": \"min=268.164, mean=268.164, max=268.164, sum=536.327 (2)\", \"tab\": \"General information\", \"score\": \"268.1637426900585\"}", - "World Religions - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"world_religions\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_world_religions\"" - } - } - }, - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.817, - "details": { - "description": "", - "tab": "Efficiency" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/google_gemini-1.5-flash-8b.json b/data/models/google_gemini-1.5-flash-8b.json deleted file mode 100644 index 03ad573151ad915801b15421dfa4cd5cb5b3f136..0000000000000000000000000000000000000000 --- a/data/models/google_gemini-1.5-flash-8b.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "google/gemini-1.5-flash-8b", - "id": "google/gemini-1.5-flash-8b", - "developer": "Google", - "additional_details": { - "model_type": "Generative RM" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/google_gemini-1.5-flash-8b/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4851 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4611 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3625 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5082 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6622 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6747 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2421 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/google_gemini-1.5-flash-preview-0514.json b/data/models/google_gemini-1.5-flash-preview-0514.json deleted file mode 100644 index bb126c3a4d3ea96984ec8acb052caf3302e16225..0000000000000000000000000000000000000000 --- a/data/models/google_gemini-1.5-flash-preview-0514.json +++ /dev/null @@ -1,1531 +0,0 @@ -{ - "model_info": { - "name": "Gemini 1.5 Flash 0514 preview", - "id": "google/gemini-1.5-flash-preview-0514", - "developer": "Google", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_mmlu/google_gemini-1.5-flash-preview-0514/1774096312.00548", - "retrieved_timestamp": "1774096312.00548", - "source_metadata": { - "source_name": "helm_mmlu", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_mmlu", - "evaluation_results": [ - { - "evaluation_name": "MMLU All Subjects", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU All Subjects", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.778, - "details": { - "description": "min=0.374, mean=0.778, max=0.969, sum=88.647 (114)", - "tab": "Accuracy", - "MMLU All Subjects - Observed inference time (s)": "{\"description\": \"min=0.323, mean=0.348, max=0.49, sum=39.671 (114)\", \"tab\": \"Efficiency\", \"score\": \"0.3479928578252291\"}", - "MMLU All Subjects - # eval": "{\"description\": \"min=100, mean=246.351, max=1534, sum=28084 (114)\", \"tab\": \"General information\", \"score\": \"246.35087719298247\"}", - "MMLU All Subjects - # train": "{\"description\": \"min=5, mean=5, max=5, sum=570 (114)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU All Subjects - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU All Subjects - # prompt tokens": "{\"description\": \"min=268.164, mean=632.617, max=2797.424, sum=72118.345 (114)\", \"tab\": \"General information\", \"score\": \"632.6170571214202\"}", - "MMLU All Subjects - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_name": "Abstract Algebra", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Abstract Algebra", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.56, - "details": { - "description": "min=0.56, mean=0.56, max=0.56, sum=1.12 (2)", - "tab": "Accuracy", - "Abstract Algebra - Observed inference time (s)": "{\"description\": \"min=0.414, mean=0.414, max=0.414, sum=0.828 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4139195799827576\"}", - "Abstract Algebra - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Abstract Algebra - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Abstract Algebra - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Abstract Algebra - # prompt tokens": "{\"description\": \"min=383.97, mean=383.97, max=383.97, sum=767.94 (2)\", \"tab\": \"General information\", \"score\": \"383.97\"}", - "Abstract Algebra - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"abstract_algebra\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_abstract_algebra\"" - } - } - }, - { - "evaluation_name": "Anatomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Anatomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.807, - "details": { - "description": "min=0.807, mean=0.807, max=0.807, sum=1.615 (2)", - "tab": "Accuracy", - "Anatomy - Observed inference time (s)": "{\"description\": \"min=0.331, mean=0.331, max=0.331, sum=0.662 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.33077726717348455\"}", - "Anatomy - # eval": "{\"description\": \"min=135, mean=135, max=135, sum=270 (2)\", \"tab\": \"General information\", \"score\": \"135.0\"}", - "Anatomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Anatomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Anatomy - # prompt tokens": "{\"description\": \"min=344.356, mean=344.356, max=344.356, sum=688.711 (2)\", \"tab\": \"General information\", \"score\": \"344.35555555555555\"}", - "Anatomy - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"anatomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_anatomy\"" - } - } - }, - { - "evaluation_name": "College Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on College Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.667, - "details": { - "description": "min=0.667, mean=0.667, max=0.667, sum=1.333 (2)", - "tab": "Accuracy", - "College Chemistry - Observed inference time (s)": "{\"description\": \"min=0.341, mean=0.341, max=0.341, sum=0.683 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3412753510475159\"}", - "College Biology - Observed inference time (s)": "{\"description\": \"min=0.331, mean=0.331, max=0.331, sum=0.662 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.33089664578437805\"}", - "College Computer Science - Observed inference time (s)": "{\"description\": \"min=0.358, mean=0.358, max=0.358, sum=0.715 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.35753655195236206\"}", - "College Mathematics - Observed inference time (s)": "{\"description\": \"min=0.344, mean=0.344, max=0.344, sum=0.688 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3440544652938843\"}", - "College Medicine - Observed inference time (s)": "{\"description\": \"min=0.339, mean=0.339, max=0.339, sum=0.679 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.33949112616522464\"}", - "College Physics - Observed inference time (s)": "{\"description\": \"min=0.339, mean=0.339, max=0.339, sum=0.678 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.33893728957456704\"}", - "College Chemistry - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Chemistry - # prompt tokens": "{\"description\": \"min=570.02, mean=570.02, max=570.02, sum=1140.04 (2)\", \"tab\": \"General information\", \"score\": \"570.02\"}", - "College Chemistry - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # eval": "{\"description\": \"min=144, mean=144, max=144, sum=288 (2)\", \"tab\": \"General information\", \"score\": \"144.0\"}", - "College Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # prompt tokens": "{\"description\": \"min=482.799, mean=482.799, max=482.799, sum=965.597 (2)\", \"tab\": \"General information\", \"score\": \"482.7986111111111\"}", - "College Biology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # prompt tokens": "{\"description\": \"min=857.86, mean=857.86, max=857.86, sum=1715.72 (2)\", \"tab\": \"General information\", \"score\": \"857.86\"}", - "College Computer Science - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # prompt tokens": "{\"description\": \"min=626.69, mean=626.69, max=626.69, sum=1253.38 (2)\", \"tab\": \"General information\", \"score\": \"626.69\"}", - "College Mathematics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # eval": "{\"description\": \"min=173, mean=173, max=173, sum=346 (2)\", \"tab\": \"General information\", \"score\": \"173.0\"}", - "College Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # prompt tokens": "{\"description\": \"min=513.37, mean=513.37, max=513.37, sum=1026.74 (2)\", \"tab\": \"General information\", \"score\": \"513.3699421965318\"}", - "College Medicine - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # eval": "{\"description\": \"min=102, mean=102, max=102, sum=204 (2)\", \"tab\": \"General information\", \"score\": \"102.0\"}", - "College Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # prompt tokens": "{\"description\": \"min=507.471, mean=507.471, max=507.471, sum=1014.941 (2)\", \"tab\": \"General information\", \"score\": \"507.47058823529414\"}", - "College Physics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"college_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_college_physics\"" - } - } - }, - { - "evaluation_name": "Computer Security", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Computer Security", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.77, - "details": { - "description": "min=0.77, mean=0.77, max=0.77, sum=1.54 (2)", - "tab": "Accuracy", - "Computer Security - Observed inference time (s)": "{\"description\": \"min=0.329, mean=0.329, max=0.329, sum=0.657 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3285136580467224\"}", - "Computer Security - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Computer Security - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Computer Security - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Computer Security - # prompt tokens": "{\"description\": \"min=380.91, mean=380.91, max=380.91, sum=761.82 (2)\", \"tab\": \"General information\", \"score\": \"380.91\"}", - "Computer Security - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"computer_security\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_computer_security\"" - } - } - }, - { - "evaluation_name": "Econometrics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Econometrics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.64, - "details": { - "description": "min=0.64, mean=0.64, max=0.64, sum=1.281 (2)", - "tab": "Accuracy", - "Econometrics - Observed inference time (s)": "{\"description\": \"min=0.339, mean=0.339, max=0.339, sum=0.679 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.33929300726505746\"}", - "Econometrics - # eval": "{\"description\": \"min=114, mean=114, max=114, sum=228 (2)\", \"tab\": \"General information\", \"score\": \"114.0\"}", - "Econometrics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Econometrics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Econometrics - # prompt tokens": "{\"description\": \"min=634.553, mean=634.553, max=634.553, sum=1269.105 (2)\", \"tab\": \"General information\", \"score\": \"634.5526315789474\"}", - "Econometrics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"econometrics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_econometrics\"" - } - } - }, - { - "evaluation_name": "Global Facts", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Global Facts", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.55, - "details": { - "description": "min=0.55, mean=0.55, max=0.55, sum=1.1 (2)", - "tab": "Accuracy", - "Global Facts - Observed inference time (s)": "{\"description\": \"min=0.325, mean=0.325, max=0.325, sum=0.65 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.32497448682785035\"}", - "Global Facts - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Global Facts - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Global Facts - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Global Facts - # prompt tokens": "{\"description\": \"min=456.54, mean=456.54, max=456.54, sum=913.08 (2)\", \"tab\": \"General information\", \"score\": \"456.54\"}", - "Global Facts - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"global_facts\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_global_facts\"" - } - } - }, - { - "evaluation_name": "Jurisprudence", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Jurisprudence", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.889, - "details": { - "description": "min=0.889, mean=0.889, max=0.889, sum=1.778 (2)", - "tab": "Accuracy", - "Jurisprudence - Observed inference time (s)": "{\"description\": \"min=0.327, mean=0.327, max=0.327, sum=0.654 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3270833028687371\"}", - "Jurisprudence - # eval": "{\"description\": \"min=108, mean=108, max=108, sum=216 (2)\", \"tab\": \"General information\", \"score\": \"108.0\"}", - "Jurisprudence - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Jurisprudence - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Jurisprudence - # prompt tokens": "{\"description\": \"min=407.87, mean=407.87, max=407.87, sum=815.741 (2)\", \"tab\": \"General information\", \"score\": \"407.8703703703704\"}", - "Jurisprudence - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"jurisprudence\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_jurisprudence\"" - } - } - }, - { - "evaluation_name": "Philosophy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Philosophy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.807, - "details": { - "description": "min=0.807, mean=0.807, max=0.807, sum=1.614 (2)", - "tab": "Accuracy", - "Philosophy - Observed inference time (s)": "{\"description\": \"min=0.352, mean=0.352, max=0.352, sum=0.704 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3517766727128596\"}", - "Philosophy - # eval": "{\"description\": \"min=311, mean=311, max=311, sum=622 (2)\", \"tab\": \"General information\", \"score\": \"311.0\"}", - "Philosophy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Philosophy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Philosophy - # prompt tokens": "{\"description\": \"min=340.907, mean=340.907, max=340.907, sum=681.814 (2)\", \"tab\": \"General information\", \"score\": \"340.90675241157555\"}", - "Philosophy - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"philosophy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_philosophy\"" - } - } - }, - { - "evaluation_name": "Professional Psychology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Professional Psychology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.825, - "details": { - "description": "min=0.825, mean=0.825, max=0.825, sum=1.65 (2)", - "tab": "Accuracy", - "Professional Medicine - Observed inference time (s)": "{\"description\": \"min=0.353, mean=0.353, max=0.353, sum=0.707 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3533606018967294\"}", - "Professional Accounting - Observed inference time (s)": "{\"description\": \"min=0.354, mean=0.354, max=0.354, sum=0.707 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.35356061509315\"}", - "Professional Law - Observed inference time (s)": "{\"description\": \"min=0.376, mean=0.376, max=0.376, sum=0.752 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.37605549059613214\"}", - "Professional Psychology - Observed inference time (s)": "{\"description\": \"min=0.353, mean=0.353, max=0.353, sum=0.707 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3533070875625861\"}", - "Professional Medicine - # eval": "{\"description\": \"min=272, mean=272, max=272, sum=544 (2)\", \"tab\": \"General information\", \"score\": \"272.0\"}", - "Professional Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Medicine - # prompt tokens": "{\"description\": \"min=1113.092, mean=1113.092, max=1113.092, sum=2226.184 (2)\", \"tab\": \"General information\", \"score\": \"1113.0919117647059\"}", - "Professional Medicine - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # eval": "{\"description\": \"min=282, mean=282, max=282, sum=564 (2)\", \"tab\": \"General information\", \"score\": \"282.0\"}", - "Professional Accounting - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Accounting - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # prompt tokens": "{\"description\": \"min=755.418, mean=755.418, max=755.418, sum=1510.837 (2)\", \"tab\": \"General information\", \"score\": \"755.418439716312\"}", - "Professional Accounting - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # eval": "{\"description\": \"min=1534, mean=1534, max=1534, sum=3068 (2)\", \"tab\": \"General information\", \"score\": \"1534.0\"}", - "Professional Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # prompt tokens": "{\"description\": \"min=1685.119, mean=1685.119, max=1685.119, sum=3370.239 (2)\", \"tab\": \"General information\", \"score\": \"1685.119295958279\"}", - "Professional Law - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # eval": "{\"description\": \"min=612, mean=612, max=612, sum=1224 (2)\", \"tab\": \"General information\", \"score\": \"612.0\"}", - "Professional Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # prompt tokens": "{\"description\": \"min=594.363, mean=594.363, max=594.363, sum=1188.725 (2)\", \"tab\": \"General information\", \"score\": \"594.3627450980392\"}", - "Professional Psychology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"professional_psychology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_professional_psychology\"" - } - } - }, - { - "evaluation_name": "Us Foreign Policy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Us Foreign Policy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.93, - "details": { - "description": "min=0.93, mean=0.93, max=0.93, sum=1.86 (2)", - "tab": "Accuracy", - "Us Foreign Policy - Observed inference time (s)": "{\"description\": \"min=0.339, mean=0.339, max=0.339, sum=0.679 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3394037842750549\"}", - "Us Foreign Policy - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Us Foreign Policy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Us Foreign Policy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Us Foreign Policy - # prompt tokens": "{\"description\": \"min=438.2, mean=438.2, max=438.2, sum=876.4 (2)\", \"tab\": \"General information\", \"score\": \"438.2\"}", - "Us Foreign Policy - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"us_foreign_policy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_us_foreign_policy\"" - } - } - }, - { - "evaluation_name": "Astronomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Astronomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.868, - "details": { - "description": "min=0.868, mean=0.868, max=0.868, sum=1.737 (2)", - "tab": "Accuracy", - "Astronomy - Observed inference time (s)": "{\"description\": \"min=0.379, mean=0.379, max=0.379, sum=0.758 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3787926027649327\"}", - "Astronomy - # eval": "{\"description\": \"min=152, mean=152, max=152, sum=304 (2)\", \"tab\": \"General information\", \"score\": \"152.0\"}", - "Astronomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Astronomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Astronomy - # prompt tokens": "{\"description\": \"min=602.421, mean=602.421, max=602.421, sum=1204.842 (2)\", \"tab\": \"General information\", \"score\": \"602.421052631579\"}", - "Astronomy - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"astronomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_astronomy\"" - } - } - }, - { - "evaluation_name": "Business Ethics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Business Ethics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.82, - "details": { - "description": "min=0.82, mean=0.82, max=0.82, sum=1.64 (2)", - "tab": "Accuracy", - "Business Ethics - Observed inference time (s)": "{\"description\": \"min=0.352, mean=0.352, max=0.352, sum=0.704 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3517553758621216\"}", - "Business Ethics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Business Ethics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Business Ethics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Business Ethics - # prompt tokens": "{\"description\": \"min=552.87, mean=552.87, max=552.87, sum=1105.74 (2)\", \"tab\": \"General information\", \"score\": \"552.87\"}", - "Business Ethics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"business_ethics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_business_ethics\"" - } - } - }, - { - "evaluation_name": "Clinical Knowledge", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Clinical Knowledge", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.838, - "details": { - "description": "min=0.838, mean=0.838, max=0.838, sum=1.675 (2)", - "tab": "Accuracy", - "Clinical Knowledge - Observed inference time (s)": "{\"description\": \"min=0.325, mean=0.325, max=0.325, sum=0.649 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3246132454782162\"}", - "Clinical Knowledge - # eval": "{\"description\": \"min=265, mean=265, max=265, sum=530 (2)\", \"tab\": \"General information\", \"score\": \"265.0\"}", - "Clinical Knowledge - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Clinical Knowledge - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Clinical Knowledge - # prompt tokens": "{\"description\": \"min=402.592, mean=402.592, max=402.592, sum=805.185 (2)\", \"tab\": \"General information\", \"score\": \"402.5924528301887\"}", - "Clinical Knowledge - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"clinical_knowledge\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_clinical_knowledge\"" - } - } - }, - { - "evaluation_name": "Conceptual Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Conceptual Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.855, - "details": { - "description": "min=0.855, mean=0.855, max=0.855, sum=1.711 (2)", - "tab": "Accuracy", - "Conceptual Physics - Observed inference time (s)": "{\"description\": \"min=0.328, mean=0.328, max=0.328, sum=0.655 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.32754647579598933\"}", - "Conceptual Physics - # eval": "{\"description\": \"min=235, mean=235, max=235, sum=470 (2)\", \"tab\": \"General information\", \"score\": \"235.0\"}", - "Conceptual Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Conceptual Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Conceptual Physics - # prompt tokens": "{\"description\": \"min=309.213, mean=309.213, max=309.213, sum=618.426 (2)\", \"tab\": \"General information\", \"score\": \"309.21276595744683\"}", - "Conceptual Physics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"conceptual_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_conceptual_physics\"" - } - } - }, - { - "evaluation_name": "Electrical Engineering", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Electrical Engineering", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.814, - "details": { - "description": "min=0.814, mean=0.814, max=0.814, sum=1.628 (2)", - "tab": "Accuracy", - "Electrical Engineering - Observed inference time (s)": "{\"description\": \"min=0.328, mean=0.328, max=0.328, sum=0.656 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3282040464467016\"}", - "Electrical Engineering - # eval": "{\"description\": \"min=145, mean=145, max=145, sum=290 (2)\", \"tab\": \"General information\", \"score\": \"145.0\"}", - "Electrical Engineering - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Electrical Engineering - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Electrical Engineering - # prompt tokens": "{\"description\": \"min=474.786, mean=474.786, max=474.786, sum=949.572 (2)\", \"tab\": \"General information\", \"score\": \"474.78620689655173\"}", - "Electrical Engineering - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"electrical_engineering\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_electrical_engineering\"" - } - } - }, - { - "evaluation_name": "Elementary Mathematics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Elementary Mathematics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.778, - "details": { - "description": "min=0.778, mean=0.778, max=0.778, sum=1.556 (2)", - "tab": "Accuracy", - "Elementary Mathematics - Observed inference time (s)": "{\"description\": \"min=0.34, mean=0.34, max=0.34, sum=0.679 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.33972583182905086\"}", - "Elementary Mathematics - # eval": "{\"description\": \"min=378, mean=378, max=378, sum=756 (2)\", \"tab\": \"General information\", \"score\": \"378.0\"}", - "Elementary Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Elementary Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Elementary Mathematics - # prompt tokens": "{\"description\": \"min=597.341, mean=597.341, max=597.341, sum=1194.683 (2)\", \"tab\": \"General information\", \"score\": \"597.3412698412699\"}", - "Elementary Mathematics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"elementary_mathematics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_elementary_mathematics\"" - } - } - }, - { - "evaluation_name": "Formal Logic", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Formal Logic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.611, - "details": { - "description": "min=0.611, mean=0.611, max=0.611, sum=1.222 (2)", - "tab": "Accuracy", - "Formal Logic - Observed inference time (s)": "{\"description\": \"min=0.347, mean=0.347, max=0.347, sum=0.693 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.34669986982194206\"}", - "Formal Logic - # eval": "{\"description\": \"min=126, mean=126, max=126, sum=252 (2)\", \"tab\": \"General information\", \"score\": \"126.0\"}", - "Formal Logic - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Formal Logic - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Formal Logic - # prompt tokens": "{\"description\": \"min=619.563, mean=619.563, max=619.563, sum=1239.127 (2)\", \"tab\": \"General information\", \"score\": \"619.563492063492\"}", - "Formal Logic - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"formal_logic\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_formal_logic\"" - } - } - }, - { - "evaluation_name": "High School World History", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on High School World History", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.907, - "details": { - "description": "min=0.907, mean=0.907, max=0.907, sum=1.814 (2)", - "tab": "Accuracy", - "High School Biology - Observed inference time (s)": "{\"description\": \"min=0.362, mean=0.362, max=0.362, sum=0.725 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.36248803600188223\"}", - "High School Chemistry - Observed inference time (s)": "{\"description\": \"min=0.336, mean=0.336, max=0.336, sum=0.672 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3359241544319491\"}", - "High School Computer Science - Observed inference time (s)": "{\"description\": \"min=0.354, mean=0.354, max=0.354, sum=0.709 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.35430107831954955\"}", - "High School European History - Observed inference time (s)": "{\"description\": \"min=0.49, mean=0.49, max=0.49, sum=0.98 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4900842637726755\"}", - "High School Geography - Observed inference time (s)": "{\"description\": \"min=0.336, mean=0.336, max=0.336, sum=0.673 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.33633674395204793\"}", - "High School Government And Politics - Observed inference time (s)": "{\"description\": \"min=0.335, mean=0.335, max=0.335, sum=0.669 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3347120445627005\"}", - "High School Macroeconomics - Observed inference time (s)": "{\"description\": \"min=0.33, mean=0.33, max=0.33, sum=0.661 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.33047562073438597\"}", - "High School Mathematics - Observed inference time (s)": "{\"description\": \"min=0.343, mean=0.343, max=0.343, sum=0.686 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3431409650378757\"}", - "High School Microeconomics - Observed inference time (s)": "{\"description\": \"min=0.329, mean=0.329, max=0.329, sum=0.658 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.328948572904122\"}", - "High School Physics - Observed inference time (s)": "{\"description\": \"min=0.343, mean=0.343, max=0.343, sum=0.686 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3431161413129592\"}", - "High School Psychology - Observed inference time (s)": "{\"description\": \"min=0.364, mean=0.364, max=0.364, sum=0.728 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3637816064498004\"}", - "High School Statistics - Observed inference time (s)": "{\"description\": \"min=0.351, mean=0.351, max=0.351, sum=0.701 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.35072638701509545\"}", - "High School US History - Observed inference time (s)": "{\"description\": \"min=0.484, mean=0.484, max=0.484, sum=0.967 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.48351573476604387\"}", - "High School World History - Observed inference time (s)": "{\"description\": \"min=0.376, mean=0.376, max=0.376, sum=0.753 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3762651908246777\"}", - "High School Biology - # eval": "{\"description\": \"min=310, mean=310, max=310, sum=620 (2)\", \"tab\": \"General information\", \"score\": \"310.0\"}", - "High School Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Biology - # prompt tokens": "{\"description\": \"min=500.958, mean=500.958, max=500.958, sum=1001.916 (2)\", \"tab\": \"General information\", \"score\": \"500.958064516129\"}", - "High School Biology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # eval": "{\"description\": \"min=203, mean=203, max=203, sum=406 (2)\", \"tab\": \"General information\", \"score\": \"203.0\"}", - "High School Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # prompt tokens": "{\"description\": \"min=513.064, mean=513.064, max=513.064, sum=1026.128 (2)\", \"tab\": \"General information\", \"score\": \"513.064039408867\"}", - "High School Chemistry - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "High School Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # prompt tokens": "{\"description\": \"min=935.13, mean=935.13, max=935.13, sum=1870.26 (2)\", \"tab\": \"General information\", \"score\": \"935.13\"}", - "High School Computer Science - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # eval": "{\"description\": \"min=165, mean=165, max=165, sum=330 (2)\", \"tab\": \"General information\", \"score\": \"165.0\"}", - "High School European History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School European History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # prompt tokens": "{\"description\": \"min=2797.424, mean=2797.424, max=2797.424, sum=5594.848 (2)\", \"tab\": \"General information\", \"score\": \"2797.4242424242425\"}", - "High School European History - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # eval": "{\"description\": \"min=198, mean=198, max=198, sum=396 (2)\", \"tab\": \"General information\", \"score\": \"198.0\"}", - "High School Geography - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Geography - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # prompt tokens": "{\"description\": \"min=394.773, mean=394.773, max=394.773, sum=789.545 (2)\", \"tab\": \"General information\", \"score\": \"394.77272727272725\"}", - "High School Geography - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # eval": "{\"description\": \"min=193, mean=193, max=193, sum=386 (2)\", \"tab\": \"General information\", \"score\": \"193.0\"}", - "High School Government And Politics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Government And Politics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # prompt tokens": "{\"description\": \"min=479.301, mean=479.301, max=479.301, sum=958.601 (2)\", \"tab\": \"General information\", \"score\": \"479.30051813471505\"}", - "High School Government And Politics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # eval": "{\"description\": \"min=390, mean=390, max=390, sum=780 (2)\", \"tab\": \"General information\", \"score\": \"390.0\"}", - "High School Macroeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Macroeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # prompt tokens": "{\"description\": \"min=396.541, mean=396.541, max=396.541, sum=793.082 (2)\", \"tab\": \"General information\", \"score\": \"396.54102564102567\"}", - "High School Macroeconomics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # eval": "{\"description\": \"min=270, mean=270, max=270, sum=540 (2)\", \"tab\": \"General information\", \"score\": \"270.0\"}", - "High School Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # prompt tokens": "{\"description\": \"min=566.822, mean=566.822, max=566.822, sum=1133.644 (2)\", \"tab\": \"General information\", \"score\": \"566.8222222222222\"}", - "High School Mathematics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # eval": "{\"description\": \"min=238, mean=238, max=238, sum=476 (2)\", \"tab\": \"General information\", \"score\": \"238.0\"}", - "High School Microeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Microeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # prompt tokens": "{\"description\": \"min=415.954, mean=415.954, max=415.954, sum=831.908 (2)\", \"tab\": \"General information\", \"score\": \"415.953781512605\"}", - "High School Microeconomics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # eval": "{\"description\": \"min=151, mean=151, max=151, sum=302 (2)\", \"tab\": \"General information\", \"score\": \"151.0\"}", - "High School Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # prompt tokens": "{\"description\": \"min=591.715, mean=591.715, max=591.715, sum=1183.43 (2)\", \"tab\": \"General information\", \"score\": \"591.7152317880794\"}", - "High School Physics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # eval": "{\"description\": \"min=545, mean=545, max=545, sum=1090 (2)\", \"tab\": \"General information\", \"score\": \"545.0\"}", - "High School Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # prompt tokens": "{\"description\": \"min=502.604, mean=502.604, max=502.604, sum=1005.207 (2)\", \"tab\": \"General information\", \"score\": \"502.60366972477067\"}", - "High School Psychology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # eval": "{\"description\": \"min=216, mean=216, max=216, sum=432 (2)\", \"tab\": \"General information\", \"score\": \"216.0\"}", - "High School Statistics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Statistics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # prompt tokens": "{\"description\": \"min=858.931, mean=858.931, max=858.931, sum=1717.861 (2)\", \"tab\": \"General information\", \"score\": \"858.9305555555555\"}", - "High School Statistics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # eval": "{\"description\": \"min=204, mean=204, max=204, sum=408 (2)\", \"tab\": \"General information\", \"score\": \"204.0\"}", - "High School US History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School US History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # prompt tokens": "{\"description\": \"min=2205.583, mean=2205.583, max=2205.583, sum=4411.167 (2)\", \"tab\": \"General information\", \"score\": \"2205.5833333333335\"}", - "High School US History - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # eval": "{\"description\": \"min=237, mean=237, max=237, sum=474 (2)\", \"tab\": \"General information\", \"score\": \"237.0\"}", - "High School World History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School World History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # prompt tokens": "{\"description\": \"min=1426.544, mean=1426.544, max=1426.544, sum=2853.089 (2)\", \"tab\": \"General information\", \"score\": \"1426.5443037974683\"}", - "High School World History - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"high_school_world_history\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_high_school_world_history\"" - } - } - }, - { - "evaluation_name": "Human Sexuality", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Human Sexuality", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.374, - "details": { - "description": "min=0.374, mean=0.374, max=0.374, sum=0.748 (2)", - "tab": "Accuracy", - "Human Aging - Observed inference time (s)": "{\"description\": \"min=0.329, mean=0.329, max=0.329, sum=0.658 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3287716788561355\"}", - "Human Sexuality - Observed inference time (s)": "{\"description\": \"min=0.323, mean=0.323, max=0.323, sum=0.647 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.32337414208105053\"}", - "Human Aging - # eval": "{\"description\": \"min=223, mean=223, max=223, sum=446 (2)\", \"tab\": \"General information\", \"score\": \"223.0\"}", - "Human Aging - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Aging - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Aging - # prompt tokens": "{\"description\": \"min=321.587, mean=321.587, max=321.587, sum=643.175 (2)\", \"tab\": \"General information\", \"score\": \"321.58744394618833\"}", - "Human Aging - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # eval": "{\"description\": \"min=131, mean=131, max=131, sum=262 (2)\", \"tab\": \"General information\", \"score\": \"131.0\"}", - "Human Sexuality - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Sexuality - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # prompt tokens": "{\"description\": \"min=347.183, mean=347.183, max=347.183, sum=694.366 (2)\", \"tab\": \"General information\", \"score\": \"347.1832061068702\"}", - "Human Sexuality - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"human_sexuality\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_human_sexuality\"" - } - } - }, - { - "evaluation_name": "International Law", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on International Law", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.876, - "details": { - "description": "min=0.876, mean=0.876, max=0.876, sum=1.752 (2)", - "tab": "Accuracy", - "International Law - Observed inference time (s)": "{\"description\": \"min=0.349, mean=0.349, max=0.349, sum=0.698 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.34882096219653924\"}", - "International Law - # eval": "{\"description\": \"min=121, mean=121, max=121, sum=242 (2)\", \"tab\": \"General information\", \"score\": \"121.0\"}", - "International Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "International Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "International Law - # prompt tokens": "{\"description\": \"min=644.165, mean=644.165, max=644.165, sum=1288.331 (2)\", \"tab\": \"General information\", \"score\": \"644.1652892561983\"}", - "International Law - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"international_law\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_international_law\"" - } - } - }, - { - "evaluation_name": "Logical Fallacies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Logical Fallacies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.853, - "details": { - "description": "min=0.853, mean=0.853, max=0.853, sum=1.706 (2)", - "tab": "Accuracy", - "Logical Fallacies - Observed inference time (s)": "{\"description\": \"min=0.329, mean=0.329, max=0.329, sum=0.658 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.32894283277125447\"}", - "Logical Fallacies - # eval": "{\"description\": \"min=163, mean=163, max=163, sum=326 (2)\", \"tab\": \"General information\", \"score\": \"163.0\"}", - "Logical Fallacies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Logical Fallacies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Logical Fallacies - # prompt tokens": "{\"description\": \"min=450.049, mean=450.049, max=450.049, sum=900.098 (2)\", \"tab\": \"General information\", \"score\": \"450.0490797546012\"}", - "Logical Fallacies - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"logical_fallacies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_logical_fallacies\"" - } - } - }, - { - "evaluation_name": "Machine Learning", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Machine Learning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.562, - "details": { - "description": "min=0.562, mean=0.562, max=0.562, sum=1.125 (2)", - "tab": "Accuracy", - "Machine Learning - Observed inference time (s)": "{\"description\": \"min=0.345, mean=0.345, max=0.345, sum=0.689 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3445145934820175\"}", - "Machine Learning - # eval": "{\"description\": \"min=112, mean=112, max=112, sum=224 (2)\", \"tab\": \"General information\", \"score\": \"112.0\"}", - "Machine Learning - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Machine Learning - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Machine Learning - # prompt tokens": "{\"description\": \"min=702.402, mean=702.402, max=702.402, sum=1404.804 (2)\", \"tab\": \"General information\", \"score\": \"702.4017857142857\"}", - "Machine Learning - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"machine_learning\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_machine_learning\"" - } - } - }, - { - "evaluation_name": "Management", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Management", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.854, - "details": { - "description": "min=0.854, mean=0.854, max=0.854, sum=1.709 (2)", - "tab": "Accuracy", - "Management - Observed inference time (s)": "{\"description\": \"min=0.326, mean=0.326, max=0.326, sum=0.652 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.32611215461805027\"}", - "Management - # eval": "{\"description\": \"min=103, mean=103, max=103, sum=206 (2)\", \"tab\": \"General information\", \"score\": \"103.0\"}", - "Management - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Management - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Management - # prompt tokens": "{\"description\": \"min=281.301, mean=281.301, max=281.301, sum=562.602 (2)\", \"tab\": \"General information\", \"score\": \"281.3009708737864\"}", - "Management - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"management\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_management\"" - } - } - }, - { - "evaluation_name": "Marketing", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Marketing", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.936, - "details": { - "description": "min=0.936, mean=0.936, max=0.936, sum=1.872 (2)", - "tab": "Accuracy", - "Marketing - Observed inference time (s)": "{\"description\": \"min=0.331, mean=0.331, max=0.331, sum=0.663 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3313393389057909\"}", - "Marketing - # eval": "{\"description\": \"min=234, mean=234, max=234, sum=468 (2)\", \"tab\": \"General information\", \"score\": \"234.0\"}", - "Marketing - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Marketing - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Marketing - # prompt tokens": "{\"description\": \"min=428.35, mean=428.35, max=428.35, sum=856.701 (2)\", \"tab\": \"General information\", \"score\": \"428.35042735042737\"}", - "Marketing - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"marketing\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_marketing\"" - } - } - }, - { - "evaluation_name": "Medical Genetics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Medical Genetics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.86, - "details": { - "description": "min=0.86, mean=0.86, max=0.86, sum=1.72 (2)", - "tab": "Accuracy", - "Medical Genetics - Observed inference time (s)": "{\"description\": \"min=0.334, mean=0.334, max=0.334, sum=0.667 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3336531209945679\"}", - "Medical Genetics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Medical Genetics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Medical Genetics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Medical Genetics - # prompt tokens": "{\"description\": \"min=338.89, mean=338.89, max=338.89, sum=677.78 (2)\", \"tab\": \"General information\", \"score\": \"338.89\"}", - "Medical Genetics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"medical_genetics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_medical_genetics\"" - } - } - }, - { - "evaluation_name": "Miscellaneous", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Miscellaneous", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.884, - "details": { - "description": "min=0.884, mean=0.884, max=0.884, sum=1.768 (2)", - "tab": "Accuracy", - "Miscellaneous - Observed inference time (s)": "{\"description\": \"min=0.33, mean=0.33, max=0.33, sum=0.66 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3299713125630814\"}", - "Miscellaneous - # eval": "{\"description\": \"min=783, mean=783, max=783, sum=1566 (2)\", \"tab\": \"General information\", \"score\": \"783.0\"}", - "Miscellaneous - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Miscellaneous - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Miscellaneous - # prompt tokens": "{\"description\": \"min=314.669, mean=314.669, max=314.669, sum=629.338 (2)\", \"tab\": \"General information\", \"score\": \"314.669220945083\"}", - "Miscellaneous - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"miscellaneous\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_miscellaneous\"" - } - } - }, - { - "evaluation_name": "Moral Scenarios", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Moral Scenarios", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.631, - "details": { - "description": "min=0.631, mean=0.631, max=0.631, sum=1.263 (2)", - "tab": "Accuracy", - "Moral Disputes - Observed inference time (s)": "{\"description\": \"min=0.336, mean=0.336, max=0.336, sum=0.671 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.33562634716863216\"}", - "Moral Scenarios - Observed inference time (s)": "{\"description\": \"min=0.347, mean=0.347, max=0.347, sum=0.694 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.34689992780224144\"}", - "Moral Disputes - # eval": "{\"description\": \"min=346, mean=346, max=346, sum=692 (2)\", \"tab\": \"General information\", \"score\": \"346.0\"}", - "Moral Disputes - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Disputes - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Disputes - # prompt tokens": "{\"description\": \"min=495.003, mean=495.003, max=495.003, sum=990.006 (2)\", \"tab\": \"General information\", \"score\": \"495.0028901734104\"}", - "Moral Disputes - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # eval": "{\"description\": \"min=895, mean=895, max=895, sum=1790 (2)\", \"tab\": \"General information\", \"score\": \"895.0\"}", - "Moral Scenarios - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Scenarios - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # prompt tokens": "{\"description\": \"min=690.542, mean=690.542, max=690.542, sum=1381.084 (2)\", \"tab\": \"General information\", \"score\": \"690.5418994413408\"}", - "Moral Scenarios - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"moral_scenarios\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_moral_scenarios\"" - } - } - }, - { - "evaluation_name": "Nutrition", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Nutrition", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.801, - "details": { - "description": "min=0.801, mean=0.801, max=0.801, sum=1.601 (2)", - "tab": "Accuracy", - "Nutrition - Observed inference time (s)": "{\"description\": \"min=0.348, mean=0.348, max=0.348, sum=0.695 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3477346959456899\"}", - "Nutrition - # eval": "{\"description\": \"min=306, mean=306, max=306, sum=612 (2)\", \"tab\": \"General information\", \"score\": \"306.0\"}", - "Nutrition - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Nutrition - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Nutrition - # prompt tokens": "{\"description\": \"min=585.48, mean=585.48, max=585.48, sum=1170.961 (2)\", \"tab\": \"General information\", \"score\": \"585.4803921568628\"}", - "Nutrition - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"nutrition\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_nutrition\"" - } - } - }, - { - "evaluation_name": "Prehistory", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Prehistory", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.867, - "details": { - "description": "min=0.867, mean=0.867, max=0.867, sum=1.735 (2)", - "tab": "Accuracy", - "Prehistory - Observed inference time (s)": "{\"description\": \"min=0.347, mean=0.347, max=0.347, sum=0.694 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.34701154850147387\"}", - "Prehistory - # eval": "{\"description\": \"min=324, mean=324, max=324, sum=648 (2)\", \"tab\": \"General information\", \"score\": \"324.0\"}", - "Prehistory - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Prehistory - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Prehistory - # prompt tokens": "{\"description\": \"min=540.198, mean=540.198, max=540.198, sum=1080.395 (2)\", \"tab\": \"General information\", \"score\": \"540.1975308641976\"}", - "Prehistory - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"prehistory\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_prehistory\"" - } - } - }, - { - "evaluation_name": "Public Relations", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Public Relations", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.773, - "details": { - "description": "min=0.773, mean=0.773, max=0.773, sum=1.545 (2)", - "tab": "Accuracy", - "Public Relations - Observed inference time (s)": "{\"description\": \"min=0.332, mean=0.332, max=0.332, sum=0.664 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3317977645180442\"}", - "Public Relations - # eval": "{\"description\": \"min=110, mean=110, max=110, sum=220 (2)\", \"tab\": \"General information\", \"score\": \"110.0\"}", - "Public Relations - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Public Relations - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Public Relations - # prompt tokens": "{\"description\": \"min=426.655, mean=426.655, max=426.655, sum=853.309 (2)\", \"tab\": \"General information\", \"score\": \"426.6545454545454\"}", - "Public Relations - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"public_relations\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_public_relations\"" - } - } - }, - { - "evaluation_name": "Security Studies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Security Studies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.812, - "details": { - "description": "min=0.812, mean=0.812, max=0.812, sum=1.624 (2)", - "tab": "Accuracy", - "Security Studies - Observed inference time (s)": "{\"description\": \"min=0.37, mean=0.37, max=0.37, sum=0.74 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3700062508485755\"}", - "Security Studies - # eval": "{\"description\": \"min=245, mean=245, max=245, sum=490 (2)\", \"tab\": \"General information\", \"score\": \"245.0\"}", - "Security Studies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Security Studies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Security Studies - # prompt tokens": "{\"description\": \"min=1193.869, mean=1193.869, max=1193.869, sum=2387.739 (2)\", \"tab\": \"General information\", \"score\": \"1193.869387755102\"}", - "Security Studies - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"security_studies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_security_studies\"" - } - } - }, - { - "evaluation_name": "Sociology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Sociology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9, - "details": { - "description": "min=0.9, mean=0.9, max=0.9, sum=1.801 (2)", - "tab": "Accuracy", - "Sociology - Observed inference time (s)": "{\"description\": \"min=0.33, mean=0.33, max=0.33, sum=0.66 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.33022794794680466\"}", - "Sociology - # eval": "{\"description\": \"min=201, mean=201, max=201, sum=402 (2)\", \"tab\": \"General information\", \"score\": \"201.0\"}", - "Sociology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Sociology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Sociology - # prompt tokens": "{\"description\": \"min=456.274, mean=456.274, max=456.274, sum=912.547 (2)\", \"tab\": \"General information\", \"score\": \"456.27363184079604\"}", - "Sociology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"sociology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_sociology\"" - } - } - }, - { - "evaluation_name": "Virology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Virology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.566, - "details": { - "description": "min=0.566, mean=0.566, max=0.566, sum=1.133 (2)", - "tab": "Accuracy", - "Virology - Observed inference time (s)": "{\"description\": \"min=0.329, mean=0.329, max=0.329, sum=0.658 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3290767310613609\"}", - "Virology - # eval": "{\"description\": \"min=166, mean=166, max=166, sum=332 (2)\", \"tab\": \"General information\", \"score\": \"166.0\"}", - "Virology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Virology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Virology - # prompt tokens": "{\"description\": \"min=336.753, mean=336.753, max=336.753, sum=673.506 (2)\", \"tab\": \"General information\", \"score\": \"336.7530120481928\"}", - "Virology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"virology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_virology\"" - } - } - }, - { - "evaluation_name": "World Religions", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on World Religions", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.871, - "details": { - "description": "min=0.871, mean=0.871, max=0.871, sum=1.743 (2)", - "tab": "Accuracy", - "World Religions - Observed inference time (s)": "{\"description\": \"min=0.326, mean=0.326, max=0.326, sum=0.653 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3263405735729731\"}", - "World Religions - # eval": "{\"description\": \"min=171, mean=171, max=171, sum=342 (2)\", \"tab\": \"General information\", \"score\": \"171.0\"}", - "World Religions - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "World Religions - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "World Religions - # prompt tokens": "{\"description\": \"min=268.164, mean=268.164, max=268.164, sum=536.327 (2)\", \"tab\": \"General information\", \"score\": \"268.1637426900585\"}", - "World Religions - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"world_religions\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_world_religions\"" - } - } - }, - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.713, - "details": { - "description": "", - "tab": "Efficiency" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/google_gemini-1.5-pro-001.json b/data/models/google_gemini-1.5-pro-001.json deleted file mode 100644 index 1644014d2dedf43e4af6047e17e1e0b017b52ae6..0000000000000000000000000000000000000000 --- a/data/models/google_gemini-1.5-pro-001.json +++ /dev/null @@ -1,1903 +0,0 @@ -{ - "model_info": { - "name": "Gemini 1.5 Pro 001", - "id": "google/gemini-1.5-pro-001", - "developer": "Google", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_lite/google_gemini-1.5-pro-001/1774096306.427425", - "retrieved_timestamp": "1774096306.427425", - "source_metadata": { - "source_name": "helm_lite", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_lite", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_lite", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.739, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.4783520599250936\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.783, - "details": { - "description": "min=0.783, mean=0.783, max=0.783, sum=0.783 (1)", - "tab": "Accuracy", - "NarrativeQA - Observed inference time (s)": "{\"description\": \"min=0.835, mean=0.835, max=0.835, sum=0.835 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.8351484166930544\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=3447.994, mean=3447.994, max=3447.994, sum=3447.994 (1)\", \"tab\": \"General information\", \"score\": \"3447.994366197183\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (closed-book)", - "source_data": { - "dataset_name": "NaturalQuestions (closed-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (closed-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.378, - "details": { - "description": "min=0.378, mean=0.378, max=0.378, sum=0.378 (1)", - "tab": "Accuracy", - "NaturalQuestions (open-book) - Observed inference time (s)": "{\"description\": \"min=0.717, mean=0.717, max=0.717, sum=0.717 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.7170397922992706\"}", - "NaturalQuestions (closed-book) - Observed inference time (s)": "{\"description\": \"min=0.634, mean=0.634, max=0.634, sum=0.634 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.6341883151531219\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1978.347, mean=1978.347, max=1978.347, sum=1978.347 (1)\", \"tab\": \"General information\", \"score\": \"1978.347\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=153.995, mean=153.995, max=153.995, sum=153.995 (1)\", \"tab\": \"General information\", \"score\": \"153.995\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "mode": "\"closedbook\"" - } - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.902, - "details": { - "description": "min=0.902, mean=0.902, max=0.902, sum=0.902 (1)", - "tab": "Accuracy", - "OpenbookQA - Observed inference time (s)": "{\"description\": \"min=0.624, mean=0.624, max=0.624, sum=0.624 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.6239193634986877\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=248.508, mean=248.508, max=248.508, sum=248.508 (1)\", \"tab\": \"General information\", \"score\": \"248.508\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "dataset": "\"openbookqa\"", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.772, - "details": { - "description": "min=0.62, mean=0.772, max=0.93, sum=3.858 (5)", - "tab": "Accuracy", - "MMLU - Observed inference time (s)": "{\"description\": \"min=0.65, mean=0.69, max=0.763, sum=3.451 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.6902154895882857\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=380.91, mean=481.531, max=634.553, sum=2407.653 (5)\", \"tab\": \"General information\", \"score\": \"481.5305263157895\"}", - "MMLU - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"college_chemistry\", \"computer_security\", \"econometrics\", \"us_foreign_policy\"]", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MATH", - "source_data": { - "dataset_name": "MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Equivalent (CoT) on MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.825, - "details": { - "description": "min=0.692, mean=0.825, max=0.956, sum=5.773 (7)", - "tab": "Accuracy", - "MATH - Observed inference time (s)": "{\"description\": \"min=2.006, mean=2.701, max=3.274, sum=18.91 (7)\", \"tab\": \"Efficiency\", \"score\": \"2.701360058859101\"}", - "MATH - # eval": "{\"description\": \"min=30, mean=62.429, max=135, sum=437 (7)\", \"tab\": \"General information\", \"score\": \"62.42857142857143\"}", - "MATH - # train": "{\"description\": \"min=8, mean=8, max=8, sum=56 (7)\", \"tab\": \"General information\", \"score\": \"8.0\"}", - "MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MATH - # prompt tokens": "{\"description\": \"min=938.215, mean=1355.506, max=2348.712, sum=9488.545 (7)\", \"tab\": \"General information\", \"score\": \"1355.5064552904823\"}", - "MATH - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"algebra\", \"counting_and_probability\", \"geometry\", \"intermediate_algebra\", \"number_theory\", \"prealgebra\", \"precalculus\"]", - "level": "\"1\"", - "use_official_examples": "\"False\"", - "use_chain_of_thought": "\"True\"" - } - } - }, - { - "evaluation_name": "GSM8K", - "source_data": { - "dataset_name": "GSM8K", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on GSM8K", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.836, - "details": { - "description": "min=0.836, mean=0.836, max=0.836, sum=0.836 (1)", - "tab": "Accuracy", - "GSM8K - Observed inference time (s)": "{\"description\": \"min=3.206, mean=3.206, max=3.206, sum=3.206 (1)\", \"tab\": \"Efficiency\", \"score\": \"3.205789808034897\"}", - "GSM8K - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "GSM8K - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "GSM8K - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GSM8K - # prompt tokens": "{\"description\": \"min=1151.885, mean=1151.885, max=1151.885, sum=1151.885 (1)\", \"tab\": \"General information\", \"score\": \"1151.885\"}", - "GSM8K - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "stop": "\"none\"" - } - } - }, - { - "evaluation_name": "LegalBench", - "source_data": { - "dataset_name": "LegalBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on LegalBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.757, - "details": { - "description": "min=0.46, mean=0.757, max=1, sum=3.786 (5)", - "tab": "Accuracy", - "LegalBench - Observed inference time (s)": "{\"description\": \"min=0.577, mean=0.775, max=1.078, sum=3.876 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.7752882438000996\"}", - "LegalBench - # eval": "{\"description\": \"min=95, mean=409.4, max=1000, sum=2047 (5)\", \"tab\": \"General information\", \"score\": \"409.4\"}", - "LegalBench - # train": "{\"description\": \"min=4, mean=4.8, max=5, sum=24 (5)\", \"tab\": \"General information\", \"score\": \"4.8\"}", - "LegalBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "LegalBench - # prompt tokens": "{\"description\": \"min=209.916, mean=1558.239, max=6423.569, sum=7791.193 (5)\", \"tab\": \"General information\", \"score\": \"1558.2386051001386\"}", - "LegalBench - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"abercrombie\", \"corporate_lobbying\", \"function_of_decision_section\", \"international_citizenship_questions\", \"proa\"]" - } - } - }, - { - "evaluation_name": "MedQA", - "source_data": { - "dataset_name": "MedQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MedQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.692, - "details": { - "description": "min=0.692, mean=0.692, max=0.692, sum=0.692 (1)", - "tab": "Accuracy", - "MedQA - Observed inference time (s)": "{\"description\": \"min=0.53, mean=0.53, max=0.53, sum=0.53 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.5296737767785669\"}", - "MedQA - # eval": "{\"description\": \"min=503, mean=503, max=503, sum=503 (1)\", \"tab\": \"General information\", \"score\": \"503.0\"}", - "MedQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MedQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MedQA - # prompt tokens": "{\"description\": \"min=1029.481, mean=1029.481, max=1029.481, sum=1029.481 (1)\", \"tab\": \"General information\", \"score\": \"1029.4811133200794\"}", - "MedQA - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WMT 2014", - "source_data": { - "dataset_name": "WMT 2014", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "BLEU-4 on WMT 2014", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.189, - "details": { - "description": "min=0.118, mean=0.189, max=0.252, sum=0.946 (5)", - "tab": "Accuracy", - "WMT 2014 - Observed inference time (s)": "{\"description\": \"min=1.029, mean=1.14, max=1.4, sum=5.7 (5)\", \"tab\": \"Efficiency\", \"score\": \"1.1399874632845124\"}", - "WMT 2014 - # eval": "{\"description\": \"min=503, mean=568.8, max=832, sum=2844 (5)\", \"tab\": \"General information\", \"score\": \"568.8\"}", - "WMT 2014 - # train": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "WMT 2014 - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WMT 2014 - # prompt tokens": "{\"description\": \"min=90.732, mean=120.97, max=147.366, sum=604.851 (5)\", \"tab\": \"General information\", \"score\": \"120.97025108961614\"}", - "WMT 2014 - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "language_pair": "[\"cs-en\", \"de-en\", \"fr-en\", \"hi-en\", \"ru-en\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_mmlu/google_gemini-1.5-pro-001/1774096312.00548", - "retrieved_timestamp": "1774096312.00548", - "source_metadata": { - "source_name": "helm_mmlu", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_mmlu", - "evaluation_results": [ - { - "evaluation_name": "MMLU All Subjects", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU All Subjects", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.827, - "details": { - "description": "min=0.374, mean=0.827, max=0.974, sum=94.288 (114)", - "tab": "Accuracy", - "MMLU All Subjects - Observed inference time (s)": "{\"description\": \"min=0.519, mean=0.618, max=0.799, sum=70.445 (114)\", \"tab\": \"Efficiency\", \"score\": \"0.6179386045856378\"}", - "MMLU All Subjects - # eval": "{\"description\": \"min=100, mean=246.351, max=1534, sum=28084 (114)\", \"tab\": \"General information\", \"score\": \"246.35087719298247\"}", - "MMLU All Subjects - # train": "{\"description\": \"min=5, mean=5, max=5, sum=570 (114)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU All Subjects - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU All Subjects - # prompt tokens": "{\"description\": \"min=268.164, mean=632.617, max=2797.424, sum=72118.345 (114)\", \"tab\": \"General information\", \"score\": \"632.6170571214202\"}", - "MMLU All Subjects - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_name": "Abstract Algebra", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Abstract Algebra", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.75, - "details": { - "description": "min=0.75, mean=0.75, max=0.75, sum=1.5 (2)", - "tab": "Accuracy", - "Abstract Algebra - Observed inference time (s)": "{\"description\": \"min=0.659, mean=0.659, max=0.659, sum=1.318 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6589885497093201\"}", - "Abstract Algebra - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Abstract Algebra - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Abstract Algebra - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Abstract Algebra - # prompt tokens": "{\"description\": \"min=383.97, mean=383.97, max=383.97, sum=767.94 (2)\", \"tab\": \"General information\", \"score\": \"383.97\"}", - "Abstract Algebra - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"abstract_algebra\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_abstract_algebra\"" - } - } - }, - { - "evaluation_name": "Anatomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Anatomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.83, - "details": { - "description": "min=0.83, mean=0.83, max=0.83, sum=1.659 (2)", - "tab": "Accuracy", - "Anatomy - Observed inference time (s)": "{\"description\": \"min=0.671, mean=0.671, max=0.671, sum=1.342 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6710023721059163\"}", - "Anatomy - # eval": "{\"description\": \"min=135, mean=135, max=135, sum=270 (2)\", \"tab\": \"General information\", \"score\": \"135.0\"}", - "Anatomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Anatomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Anatomy - # prompt tokens": "{\"description\": \"min=344.356, mean=344.356, max=344.356, sum=688.711 (2)\", \"tab\": \"General information\", \"score\": \"344.35555555555555\"}", - "Anatomy - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"anatomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_anatomy\"" - } - } - }, - { - "evaluation_name": "College Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on College Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.745, - "details": { - "description": "min=0.745, mean=0.745, max=0.745, sum=1.49 (2)", - "tab": "Accuracy", - "College Chemistry - Observed inference time (s)": "{\"description\": \"min=0.763, mean=0.763, max=0.763, sum=1.527 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7634538197517395\"}", - "College Biology - Observed inference time (s)": "{\"description\": \"min=0.622, mean=0.622, max=0.622, sum=1.244 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6218778673145506\"}", - "College Computer Science - Observed inference time (s)": "{\"description\": \"min=0.664, mean=0.664, max=0.664, sum=1.328 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6641578316688538\"}", - "College Mathematics - Observed inference time (s)": "{\"description\": \"min=0.694, mean=0.694, max=0.694, sum=1.389 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6943222141265869\"}", - "College Medicine - Observed inference time (s)": "{\"description\": \"min=0.586, mean=0.586, max=0.586, sum=1.172 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5860298300065057\"}", - "College Physics - Observed inference time (s)": "{\"description\": \"min=0.799, mean=0.799, max=0.799, sum=1.597 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7986945521597769\"}", - "College Chemistry - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Chemistry - # prompt tokens": "{\"description\": \"min=570.02, mean=570.02, max=570.02, sum=1140.04 (2)\", \"tab\": \"General information\", \"score\": \"570.02\"}", - "College Chemistry - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # eval": "{\"description\": \"min=144, mean=144, max=144, sum=288 (2)\", \"tab\": \"General information\", \"score\": \"144.0\"}", - "College Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # prompt tokens": "{\"description\": \"min=482.799, mean=482.799, max=482.799, sum=965.597 (2)\", \"tab\": \"General information\", \"score\": \"482.7986111111111\"}", - "College Biology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # prompt tokens": "{\"description\": \"min=857.86, mean=857.86, max=857.86, sum=1715.72 (2)\", \"tab\": \"General information\", \"score\": \"857.86\"}", - "College Computer Science - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # prompt tokens": "{\"description\": \"min=626.69, mean=626.69, max=626.69, sum=1253.38 (2)\", \"tab\": \"General information\", \"score\": \"626.69\"}", - "College Mathematics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # eval": "{\"description\": \"min=173, mean=173, max=173, sum=346 (2)\", \"tab\": \"General information\", \"score\": \"173.0\"}", - "College Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # prompt tokens": "{\"description\": \"min=513.37, mean=513.37, max=513.37, sum=1026.74 (2)\", \"tab\": \"General information\", \"score\": \"513.3699421965318\"}", - "College Medicine - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # eval": "{\"description\": \"min=102, mean=102, max=102, sum=204 (2)\", \"tab\": \"General information\", \"score\": \"102.0\"}", - "College Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # prompt tokens": "{\"description\": \"min=507.471, mean=507.471, max=507.471, sum=1014.941 (2)\", \"tab\": \"General information\", \"score\": \"507.47058823529414\"}", - "College Physics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"college_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_college_physics\"" - } - } - }, - { - "evaluation_name": "Computer Security", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Computer Security", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.83, - "details": { - "description": "min=0.83, mean=0.83, max=0.83, sum=1.66 (2)", - "tab": "Accuracy", - "Computer Security - Observed inference time (s)": "{\"description\": \"min=0.702, mean=0.702, max=0.702, sum=1.404 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7018922233581543\"}", - "Computer Security - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Computer Security - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Computer Security - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Computer Security - # prompt tokens": "{\"description\": \"min=380.91, mean=380.91, max=380.91, sum=761.82 (2)\", \"tab\": \"General information\", \"score\": \"380.91\"}", - "Computer Security - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"computer_security\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_computer_security\"" - } - } - }, - { - "evaluation_name": "Econometrics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Econometrics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.728, - "details": { - "description": "min=0.728, mean=0.728, max=0.728, sum=1.456 (2)", - "tab": "Accuracy", - "Econometrics - Observed inference time (s)": "{\"description\": \"min=0.65, mean=0.65, max=0.65, sum=1.3 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6497656546140972\"}", - "Econometrics - # eval": "{\"description\": \"min=114, mean=114, max=114, sum=228 (2)\", \"tab\": \"General information\", \"score\": \"114.0\"}", - "Econometrics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Econometrics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Econometrics - # prompt tokens": "{\"description\": \"min=634.553, mean=634.553, max=634.553, sum=1269.105 (2)\", \"tab\": \"General information\", \"score\": \"634.5526315789474\"}", - "Econometrics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"econometrics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_econometrics\"" - } - } - }, - { - "evaluation_name": "Global Facts", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Global Facts", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.66, - "details": { - "description": "min=0.66, mean=0.66, max=0.66, sum=1.32 (2)", - "tab": "Accuracy", - "Global Facts - Observed inference time (s)": "{\"description\": \"min=0.67, mean=0.67, max=0.67, sum=1.34 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6698257994651794\"}", - "Global Facts - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Global Facts - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Global Facts - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Global Facts - # prompt tokens": "{\"description\": \"min=456.54, mean=456.54, max=456.54, sum=913.08 (2)\", \"tab\": \"General information\", \"score\": \"456.54\"}", - "Global Facts - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"global_facts\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_global_facts\"" - } - } - }, - { - "evaluation_name": "Jurisprudence", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Jurisprudence", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.889, - "details": { - "description": "min=0.889, mean=0.889, max=0.889, sum=1.778 (2)", - "tab": "Accuracy", - "Jurisprudence - Observed inference time (s)": "{\"description\": \"min=0.624, mean=0.624, max=0.624, sum=1.248 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6239932885876408\"}", - "Jurisprudence - # eval": "{\"description\": \"min=108, mean=108, max=108, sum=216 (2)\", \"tab\": \"General information\", \"score\": \"108.0\"}", - "Jurisprudence - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Jurisprudence - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Jurisprudence - # prompt tokens": "{\"description\": \"min=407.87, mean=407.87, max=407.87, sum=815.741 (2)\", \"tab\": \"General information\", \"score\": \"407.8703703703704\"}", - "Jurisprudence - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"jurisprudence\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_jurisprudence\"" - } - } - }, - { - "evaluation_name": "Philosophy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Philosophy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.871, - "details": { - "description": "min=0.871, mean=0.871, max=0.871, sum=1.743 (2)", - "tab": "Accuracy", - "Philosophy - Observed inference time (s)": "{\"description\": \"min=0.52, mean=0.52, max=0.52, sum=1.04 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5198829174041748\"}", - "Philosophy - # eval": "{\"description\": \"min=311, mean=311, max=311, sum=622 (2)\", \"tab\": \"General information\", \"score\": \"311.0\"}", - "Philosophy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Philosophy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Philosophy - # prompt tokens": "{\"description\": \"min=340.907, mean=340.907, max=340.907, sum=681.814 (2)\", \"tab\": \"General information\", \"score\": \"340.90675241157555\"}", - "Philosophy - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"philosophy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_philosophy\"" - } - } - }, - { - "evaluation_name": "Professional Psychology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Professional Psychology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.894, - "details": { - "description": "min=0.894, mean=0.894, max=0.894, sum=1.788 (2)", - "tab": "Accuracy", - "Professional Medicine - Observed inference time (s)": "{\"description\": \"min=0.601, mean=0.601, max=0.601, sum=1.202 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6008452876467546\"}", - "Professional Accounting - Observed inference time (s)": "{\"description\": \"min=0.539, mean=0.539, max=0.539, sum=1.079 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5394198826864256\"}", - "Professional Law - Observed inference time (s)": "{\"description\": \"min=0.564, mean=0.564, max=0.564, sum=1.128 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5641645779784438\"}", - "Professional Psychology - Observed inference time (s)": "{\"description\": \"min=0.544, mean=0.544, max=0.544, sum=1.088 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5440043469792918\"}", - "Professional Medicine - # eval": "{\"description\": \"min=272, mean=272, max=272, sum=544 (2)\", \"tab\": \"General information\", \"score\": \"272.0\"}", - "Professional Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Medicine - # prompt tokens": "{\"description\": \"min=1113.092, mean=1113.092, max=1113.092, sum=2226.184 (2)\", \"tab\": \"General information\", \"score\": \"1113.0919117647059\"}", - "Professional Medicine - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # eval": "{\"description\": \"min=282, mean=282, max=282, sum=564 (2)\", \"tab\": \"General information\", \"score\": \"282.0\"}", - "Professional Accounting - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Accounting - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # prompt tokens": "{\"description\": \"min=755.418, mean=755.418, max=755.418, sum=1510.837 (2)\", \"tab\": \"General information\", \"score\": \"755.418439716312\"}", - "Professional Accounting - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # eval": "{\"description\": \"min=1534, mean=1534, max=1534, sum=3068 (2)\", \"tab\": \"General information\", \"score\": \"1534.0\"}", - "Professional Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # prompt tokens": "{\"description\": \"min=1685.119, mean=1685.119, max=1685.119, sum=3370.239 (2)\", \"tab\": \"General information\", \"score\": \"1685.119295958279\"}", - "Professional Law - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # eval": "{\"description\": \"min=612, mean=612, max=612, sum=1224 (2)\", \"tab\": \"General information\", \"score\": \"612.0\"}", - "Professional Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # prompt tokens": "{\"description\": \"min=594.363, mean=594.363, max=594.363, sum=1188.725 (2)\", \"tab\": \"General information\", \"score\": \"594.3627450980392\"}", - "Professional Psychology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"professional_psychology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_professional_psychology\"" - } - } - }, - { - "evaluation_name": "Us Foreign Policy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Us Foreign Policy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.93, - "details": { - "description": "min=0.93, mean=0.93, max=0.93, sum=1.86 (2)", - "tab": "Accuracy", - "Us Foreign Policy - Observed inference time (s)": "{\"description\": \"min=0.677, mean=0.677, max=0.677, sum=1.354 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6769772005081177\"}", - "Us Foreign Policy - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Us Foreign Policy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Us Foreign Policy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Us Foreign Policy - # prompt tokens": "{\"description\": \"min=438.2, mean=438.2, max=438.2, sum=876.4 (2)\", \"tab\": \"General information\", \"score\": \"438.2\"}", - "Us Foreign Policy - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"us_foreign_policy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_us_foreign_policy\"" - } - } - }, - { - "evaluation_name": "Astronomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Astronomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.914, - "details": { - "description": "min=0.914, mean=0.914, max=0.914, sum=1.829 (2)", - "tab": "Accuracy", - "Astronomy - Observed inference time (s)": "{\"description\": \"min=0.649, mean=0.649, max=0.649, sum=1.298 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6491834003674356\"}", - "Astronomy - # eval": "{\"description\": \"min=152, mean=152, max=152, sum=304 (2)\", \"tab\": \"General information\", \"score\": \"152.0\"}", - "Astronomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Astronomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Astronomy - # prompt tokens": "{\"description\": \"min=602.421, mean=602.421, max=602.421, sum=1204.842 (2)\", \"tab\": \"General information\", \"score\": \"602.421052631579\"}", - "Astronomy - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"astronomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_astronomy\"" - } - } - }, - { - "evaluation_name": "Business Ethics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Business Ethics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8, - "details": { - "description": "min=0.8, mean=0.8, max=0.8, sum=1.6 (2)", - "tab": "Accuracy", - "Business Ethics - Observed inference time (s)": "{\"description\": \"min=0.697, mean=0.697, max=0.697, sum=1.394 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.697232437133789\"}", - "Business Ethics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Business Ethics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Business Ethics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Business Ethics - # prompt tokens": "{\"description\": \"min=552.87, mean=552.87, max=552.87, sum=1105.74 (2)\", \"tab\": \"General information\", \"score\": \"552.87\"}", - "Business Ethics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"business_ethics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_business_ethics\"" - } - } - }, - { - "evaluation_name": "Clinical Knowledge", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Clinical Knowledge", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.853, - "details": { - "description": "min=0.853, mean=0.853, max=0.853, sum=1.706 (2)", - "tab": "Accuracy", - "Clinical Knowledge - Observed inference time (s)": "{\"description\": \"min=0.545, mean=0.545, max=0.545, sum=1.091 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.545333849708989\"}", - "Clinical Knowledge - # eval": "{\"description\": \"min=265, mean=265, max=265, sum=530 (2)\", \"tab\": \"General information\", \"score\": \"265.0\"}", - "Clinical Knowledge - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Clinical Knowledge - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Clinical Knowledge - # prompt tokens": "{\"description\": \"min=402.592, mean=402.592, max=402.592, sum=805.185 (2)\", \"tab\": \"General information\", \"score\": \"402.5924528301887\"}", - "Clinical Knowledge - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"clinical_knowledge\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_clinical_knowledge\"" - } - } - }, - { - "evaluation_name": "Conceptual Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Conceptual Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.949, - "details": { - "description": "min=0.949, mean=0.949, max=0.949, sum=1.898 (2)", - "tab": "Accuracy", - "Conceptual Physics - Observed inference time (s)": "{\"description\": \"min=0.573, mean=0.573, max=0.573, sum=1.146 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5729408700415428\"}", - "Conceptual Physics - # eval": "{\"description\": \"min=235, mean=235, max=235, sum=470 (2)\", \"tab\": \"General information\", \"score\": \"235.0\"}", - "Conceptual Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Conceptual Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Conceptual Physics - # prompt tokens": "{\"description\": \"min=309.213, mean=309.213, max=309.213, sum=618.426 (2)\", \"tab\": \"General information\", \"score\": \"309.21276595744683\"}", - "Conceptual Physics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"conceptual_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_conceptual_physics\"" - } - } - }, - { - "evaluation_name": "Electrical Engineering", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Electrical Engineering", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.745, - "details": { - "description": "min=0.745, mean=0.745, max=0.745, sum=1.49 (2)", - "tab": "Accuracy", - "Electrical Engineering - Observed inference time (s)": "{\"description\": \"min=0.622, mean=0.622, max=0.622, sum=1.244 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6219884050303492\"}", - "Electrical Engineering - # eval": "{\"description\": \"min=145, mean=145, max=145, sum=290 (2)\", \"tab\": \"General information\", \"score\": \"145.0\"}", - "Electrical Engineering - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Electrical Engineering - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Electrical Engineering - # prompt tokens": "{\"description\": \"min=474.786, mean=474.786, max=474.786, sum=949.572 (2)\", \"tab\": \"General information\", \"score\": \"474.78620689655173\"}", - "Electrical Engineering - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"electrical_engineering\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_electrical_engineering\"" - } - } - }, - { - "evaluation_name": "Elementary Mathematics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Elementary Mathematics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.939, - "details": { - "description": "min=0.939, mean=0.939, max=0.939, sum=1.878 (2)", - "tab": "Accuracy", - "Elementary Mathematics - Observed inference time (s)": "{\"description\": \"min=0.548, mean=0.548, max=0.548, sum=1.097 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5484477596938926\"}", - "Elementary Mathematics - # eval": "{\"description\": \"min=378, mean=378, max=378, sum=756 (2)\", \"tab\": \"General information\", \"score\": \"378.0\"}", - "Elementary Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Elementary Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Elementary Mathematics - # prompt tokens": "{\"description\": \"min=597.341, mean=597.341, max=597.341, sum=1194.683 (2)\", \"tab\": \"General information\", \"score\": \"597.3412698412699\"}", - "Elementary Mathematics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"elementary_mathematics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_elementary_mathematics\"" - } - } - }, - { - "evaluation_name": "Formal Logic", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Formal Logic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.706, - "details": { - "description": "min=0.706, mean=0.706, max=0.706, sum=1.413 (2)", - "tab": "Accuracy", - "Formal Logic - Observed inference time (s)": "{\"description\": \"min=0.668, mean=0.668, max=0.668, sum=1.336 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6678630435277545\"}", - "Formal Logic - # eval": "{\"description\": \"min=126, mean=126, max=126, sum=252 (2)\", \"tab\": \"General information\", \"score\": \"126.0\"}", - "Formal Logic - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Formal Logic - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Formal Logic - # prompt tokens": "{\"description\": \"min=619.563, mean=619.563, max=619.563, sum=1239.127 (2)\", \"tab\": \"General information\", \"score\": \"619.563492063492\"}", - "Formal Logic - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"formal_logic\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_formal_logic\"" - } - } - }, - { - "evaluation_name": "High School World History", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on High School World History", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.924, - "details": { - "description": "min=0.924, mean=0.924, max=0.924, sum=1.848 (2)", - "tab": "Accuracy", - "High School Biology - Observed inference time (s)": "{\"description\": \"min=0.55, mean=0.55, max=0.55, sum=1.1 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5502124647940358\"}", - "High School Chemistry - Observed inference time (s)": "{\"description\": \"min=0.578, mean=0.578, max=0.578, sum=1.156 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5780763097584541\"}", - "High School Computer Science - Observed inference time (s)": "{\"description\": \"min=0.66, mean=0.66, max=0.66, sum=1.32 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6602028679847717\"}", - "High School European History - Observed inference time (s)": "{\"description\": \"min=0.775, mean=0.775, max=0.775, sum=1.55 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7751016385627515\"}", - "High School Geography - Observed inference time (s)": "{\"description\": \"min=0.571, mean=0.571, max=0.571, sum=1.141 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5705801778369479\"}", - "High School Government And Politics - Observed inference time (s)": "{\"description\": \"min=0.582, mean=0.582, max=0.582, sum=1.163 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5816669402344857\"}", - "High School Macroeconomics - Observed inference time (s)": "{\"description\": \"min=0.54, mean=0.54, max=0.54, sum=1.081 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5402819168873322\"}", - "High School Mathematics - Observed inference time (s)": "{\"description\": \"min=0.584, mean=0.584, max=0.584, sum=1.168 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5841257324925175\"}", - "High School Microeconomics - Observed inference time (s)": "{\"description\": \"min=0.556, mean=0.556, max=0.556, sum=1.113 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.556499927985568\"}", - "High School Physics - Observed inference time (s)": "{\"description\": \"min=0.632, mean=0.632, max=0.632, sum=1.264 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6318649550936869\"}", - "High School Psychology - Observed inference time (s)": "{\"description\": \"min=0.54, mean=0.54, max=0.54, sum=1.08 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5397529965814423\"}", - "High School Statistics - Observed inference time (s)": "{\"description\": \"min=0.603, mean=0.603, max=0.603, sum=1.205 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6027307720096023\"}", - "High School US History - Observed inference time (s)": "{\"description\": \"min=0.762, mean=0.762, max=0.762, sum=1.524 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7618554059196921\"}", - "High School World History - Observed inference time (s)": "{\"description\": \"min=0.626, mean=0.626, max=0.626, sum=1.252 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6258294099493872\"}", - "High School Biology - # eval": "{\"description\": \"min=310, mean=310, max=310, sum=620 (2)\", \"tab\": \"General information\", \"score\": \"310.0\"}", - "High School Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Biology - # prompt tokens": "{\"description\": \"min=500.958, mean=500.958, max=500.958, sum=1001.916 (2)\", \"tab\": \"General information\", \"score\": \"500.958064516129\"}", - "High School Biology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # eval": "{\"description\": \"min=203, mean=203, max=203, sum=406 (2)\", \"tab\": \"General information\", \"score\": \"203.0\"}", - "High School Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # prompt tokens": "{\"description\": \"min=513.064, mean=513.064, max=513.064, sum=1026.128 (2)\", \"tab\": \"General information\", \"score\": \"513.064039408867\"}", - "High School Chemistry - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "High School Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # prompt tokens": "{\"description\": \"min=935.13, mean=935.13, max=935.13, sum=1870.26 (2)\", \"tab\": \"General information\", \"score\": \"935.13\"}", - "High School Computer Science - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # eval": "{\"description\": \"min=165, mean=165, max=165, sum=330 (2)\", \"tab\": \"General information\", \"score\": \"165.0\"}", - "High School European History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School European History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # prompt tokens": "{\"description\": \"min=2797.424, mean=2797.424, max=2797.424, sum=5594.848 (2)\", \"tab\": \"General information\", \"score\": \"2797.4242424242425\"}", - "High School European History - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # eval": "{\"description\": \"min=198, mean=198, max=198, sum=396 (2)\", \"tab\": \"General information\", \"score\": \"198.0\"}", - "High School Geography - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Geography - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # prompt tokens": "{\"description\": \"min=394.773, mean=394.773, max=394.773, sum=789.545 (2)\", \"tab\": \"General information\", \"score\": \"394.77272727272725\"}", - "High School Geography - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # eval": "{\"description\": \"min=193, mean=193, max=193, sum=386 (2)\", \"tab\": \"General information\", \"score\": \"193.0\"}", - "High School Government And Politics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Government And Politics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # prompt tokens": "{\"description\": \"min=479.301, mean=479.301, max=479.301, sum=958.601 (2)\", \"tab\": \"General information\", \"score\": \"479.30051813471505\"}", - "High School Government And Politics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # eval": "{\"description\": \"min=390, mean=390, max=390, sum=780 (2)\", \"tab\": \"General information\", \"score\": \"390.0\"}", - "High School Macroeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Macroeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # prompt tokens": "{\"description\": \"min=396.541, mean=396.541, max=396.541, sum=793.082 (2)\", \"tab\": \"General information\", \"score\": \"396.54102564102567\"}", - "High School Macroeconomics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # eval": "{\"description\": \"min=270, mean=270, max=270, sum=540 (2)\", \"tab\": \"General information\", \"score\": \"270.0\"}", - "High School Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # prompt tokens": "{\"description\": \"min=566.822, mean=566.822, max=566.822, sum=1133.644 (2)\", \"tab\": \"General information\", \"score\": \"566.8222222222222\"}", - "High School Mathematics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # eval": "{\"description\": \"min=238, mean=238, max=238, sum=476 (2)\", \"tab\": \"General information\", \"score\": \"238.0\"}", - "High School Microeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Microeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # prompt tokens": "{\"description\": \"min=415.954, mean=415.954, max=415.954, sum=831.908 (2)\", \"tab\": \"General information\", \"score\": \"415.953781512605\"}", - "High School Microeconomics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # eval": "{\"description\": \"min=151, mean=151, max=151, sum=302 (2)\", \"tab\": \"General information\", \"score\": \"151.0\"}", - "High School Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # prompt tokens": "{\"description\": \"min=591.715, mean=591.715, max=591.715, sum=1183.43 (2)\", \"tab\": \"General information\", \"score\": \"591.7152317880794\"}", - "High School Physics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # eval": "{\"description\": \"min=545, mean=545, max=545, sum=1090 (2)\", \"tab\": \"General information\", \"score\": \"545.0\"}", - "High School Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # prompt tokens": "{\"description\": \"min=502.604, mean=502.604, max=502.604, sum=1005.207 (2)\", \"tab\": \"General information\", \"score\": \"502.60366972477067\"}", - "High School Psychology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # eval": "{\"description\": \"min=216, mean=216, max=216, sum=432 (2)\", \"tab\": \"General information\", \"score\": \"216.0\"}", - "High School Statistics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Statistics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # prompt tokens": "{\"description\": \"min=858.931, mean=858.931, max=858.931, sum=1717.861 (2)\", \"tab\": \"General information\", \"score\": \"858.9305555555555\"}", - "High School Statistics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # eval": "{\"description\": \"min=204, mean=204, max=204, sum=408 (2)\", \"tab\": \"General information\", \"score\": \"204.0\"}", - "High School US History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School US History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # prompt tokens": "{\"description\": \"min=2205.583, mean=2205.583, max=2205.583, sum=4411.167 (2)\", \"tab\": \"General information\", \"score\": \"2205.5833333333335\"}", - "High School US History - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # eval": "{\"description\": \"min=237, mean=237, max=237, sum=474 (2)\", \"tab\": \"General information\", \"score\": \"237.0\"}", - "High School World History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School World History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # prompt tokens": "{\"description\": \"min=1426.544, mean=1426.544, max=1426.544, sum=2853.089 (2)\", \"tab\": \"General information\", \"score\": \"1426.5443037974683\"}", - "High School World History - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"high_school_world_history\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_high_school_world_history\"" - } - } - }, - { - "evaluation_name": "Human Sexuality", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Human Sexuality", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.374, - "details": { - "description": "min=0.374, mean=0.374, max=0.374, sum=0.748 (2)", - "tab": "Accuracy", - "Human Aging - Observed inference time (s)": "{\"description\": \"min=0.563, mean=0.563, max=0.563, sum=1.127 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5634646939589838\"}", - "Human Sexuality - Observed inference time (s)": "{\"description\": \"min=0.616, mean=0.616, max=0.616, sum=1.231 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6156448550143484\"}", - "Human Aging - # eval": "{\"description\": \"min=223, mean=223, max=223, sum=446 (2)\", \"tab\": \"General information\", \"score\": \"223.0\"}", - "Human Aging - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Aging - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Aging - # prompt tokens": "{\"description\": \"min=321.587, mean=321.587, max=321.587, sum=643.175 (2)\", \"tab\": \"General information\", \"score\": \"321.58744394618833\"}", - "Human Aging - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # eval": "{\"description\": \"min=131, mean=131, max=131, sum=262 (2)\", \"tab\": \"General information\", \"score\": \"131.0\"}", - "Human Sexuality - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Sexuality - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # prompt tokens": "{\"description\": \"min=347.183, mean=347.183, max=347.183, sum=694.366 (2)\", \"tab\": \"General information\", \"score\": \"347.1832061068702\"}", - "Human Sexuality - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"human_sexuality\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_human_sexuality\"" - } - } - }, - { - "evaluation_name": "International Law", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on International Law", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.917, - "details": { - "description": "min=0.917, mean=0.917, max=0.917, sum=1.835 (2)", - "tab": "Accuracy", - "International Law - Observed inference time (s)": "{\"description\": \"min=0.673, mean=0.673, max=0.673, sum=1.346 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.672865920815586\"}", - "International Law - # eval": "{\"description\": \"min=121, mean=121, max=121, sum=242 (2)\", \"tab\": \"General information\", \"score\": \"121.0\"}", - "International Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "International Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "International Law - # prompt tokens": "{\"description\": \"min=644.165, mean=644.165, max=644.165, sum=1288.331 (2)\", \"tab\": \"General information\", \"score\": \"644.1652892561983\"}", - "International Law - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"international_law\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_international_law\"" - } - } - }, - { - "evaluation_name": "Logical Fallacies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Logical Fallacies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.896, - "details": { - "description": "min=0.896, mean=0.896, max=0.896, sum=1.791 (2)", - "tab": "Accuracy", - "Logical Fallacies - Observed inference time (s)": "{\"description\": \"min=0.616, mean=0.616, max=0.616, sum=1.233 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6164792593271454\"}", - "Logical Fallacies - # eval": "{\"description\": \"min=163, mean=163, max=163, sum=326 (2)\", \"tab\": \"General information\", \"score\": \"163.0\"}", - "Logical Fallacies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Logical Fallacies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Logical Fallacies - # prompt tokens": "{\"description\": \"min=450.049, mean=450.049, max=450.049, sum=900.098 (2)\", \"tab\": \"General information\", \"score\": \"450.0490797546012\"}", - "Logical Fallacies - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"logical_fallacies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_logical_fallacies\"" - } - } - }, - { - "evaluation_name": "Machine Learning", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Machine Learning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.652, - "details": { - "description": "min=0.652, mean=0.652, max=0.652, sum=1.304 (2)", - "tab": "Accuracy", - "Machine Learning - Observed inference time (s)": "{\"description\": \"min=0.638, mean=0.638, max=0.638, sum=1.276 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6377767409597125\"}", - "Machine Learning - # eval": "{\"description\": \"min=112, mean=112, max=112, sum=224 (2)\", \"tab\": \"General information\", \"score\": \"112.0\"}", - "Machine Learning - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Machine Learning - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Machine Learning - # prompt tokens": "{\"description\": \"min=702.402, mean=702.402, max=702.402, sum=1404.804 (2)\", \"tab\": \"General information\", \"score\": \"702.4017857142857\"}", - "Machine Learning - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"machine_learning\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_machine_learning\"" - } - } - }, - { - "evaluation_name": "Management", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Management", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.922, - "details": { - "description": "min=0.922, mean=0.922, max=0.922, sum=1.845 (2)", - "tab": "Accuracy", - "Management - Observed inference time (s)": "{\"description\": \"min=0.669, mean=0.669, max=0.669, sum=1.338 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6690320089025404\"}", - "Management - # eval": "{\"description\": \"min=103, mean=103, max=103, sum=206 (2)\", \"tab\": \"General information\", \"score\": \"103.0\"}", - "Management - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Management - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Management - # prompt tokens": "{\"description\": \"min=281.301, mean=281.301, max=281.301, sum=562.602 (2)\", \"tab\": \"General information\", \"score\": \"281.3009708737864\"}", - "Management - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"management\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_management\"" - } - } - }, - { - "evaluation_name": "Marketing", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Marketing", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.932, - "details": { - "description": "min=0.932, mean=0.932, max=0.932, sum=1.863 (2)", - "tab": "Accuracy", - "Marketing - Observed inference time (s)": "{\"description\": \"min=0.554, mean=0.554, max=0.554, sum=1.107 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5537131362491183\"}", - "Marketing - # eval": "{\"description\": \"min=234, mean=234, max=234, sum=468 (2)\", \"tab\": \"General information\", \"score\": \"234.0\"}", - "Marketing - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Marketing - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Marketing - # prompt tokens": "{\"description\": \"min=428.35, mean=428.35, max=428.35, sum=856.701 (2)\", \"tab\": \"General information\", \"score\": \"428.35042735042737\"}", - "Marketing - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"marketing\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_marketing\"" - } - } - }, - { - "evaluation_name": "Medical Genetics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Medical Genetics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.91, - "details": { - "description": "min=0.91, mean=0.91, max=0.91, sum=1.82 (2)", - "tab": "Accuracy", - "Medical Genetics - Observed inference time (s)": "{\"description\": \"min=0.678, mean=0.678, max=0.678, sum=1.356 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.678006865978241\"}", - "Medical Genetics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Medical Genetics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Medical Genetics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Medical Genetics - # prompt tokens": "{\"description\": \"min=338.89, mean=338.89, max=338.89, sum=677.78 (2)\", \"tab\": \"General information\", \"score\": \"338.89\"}", - "Medical Genetics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"medical_genetics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_medical_genetics\"" - } - } - }, - { - "evaluation_name": "Miscellaneous", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Miscellaneous", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.958, - "details": { - "description": "min=0.958, mean=0.958, max=0.958, sum=1.916 (2)", - "tab": "Accuracy", - "Miscellaneous - Observed inference time (s)": "{\"description\": \"min=0.519, mean=0.519, max=0.519, sum=1.038 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.519028120113972\"}", - "Miscellaneous - # eval": "{\"description\": \"min=783, mean=783, max=783, sum=1566 (2)\", \"tab\": \"General information\", \"score\": \"783.0\"}", - "Miscellaneous - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Miscellaneous - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Miscellaneous - # prompt tokens": "{\"description\": \"min=314.669, mean=314.669, max=314.669, sum=629.338 (2)\", \"tab\": \"General information\", \"score\": \"314.669220945083\"}", - "Miscellaneous - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"miscellaneous\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_miscellaneous\"" - } - } - }, - { - "evaluation_name": "Moral Scenarios", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Moral Scenarios", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.739, - "details": { - "description": "min=0.739, mean=0.739, max=0.739, sum=1.477 (2)", - "tab": "Accuracy", - "Moral Disputes - Observed inference time (s)": "{\"description\": \"min=0.546, mean=0.546, max=0.546, sum=1.092 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5461560525755952\"}", - "Moral Scenarios - Observed inference time (s)": "{\"description\": \"min=0.536, mean=0.536, max=0.536, sum=1.072 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5358252359053416\"}", - "Moral Disputes - # eval": "{\"description\": \"min=346, mean=346, max=346, sum=692 (2)\", \"tab\": \"General information\", \"score\": \"346.0\"}", - "Moral Disputes - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Disputes - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Disputes - # prompt tokens": "{\"description\": \"min=495.003, mean=495.003, max=495.003, sum=990.006 (2)\", \"tab\": \"General information\", \"score\": \"495.0028901734104\"}", - "Moral Disputes - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # eval": "{\"description\": \"min=895, mean=895, max=895, sum=1790 (2)\", \"tab\": \"General information\", \"score\": \"895.0\"}", - "Moral Scenarios - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Scenarios - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # prompt tokens": "{\"description\": \"min=690.542, mean=690.542, max=690.542, sum=1381.084 (2)\", \"tab\": \"General information\", \"score\": \"690.5418994413408\"}", - "Moral Scenarios - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"moral_scenarios\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_moral_scenarios\"" - } - } - }, - { - "evaluation_name": "Nutrition", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Nutrition", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.879, - "details": { - "description": "min=0.879, mean=0.879, max=0.879, sum=1.758 (2)", - "tab": "Accuracy", - "Nutrition - Observed inference time (s)": "{\"description\": \"min=0.569, mean=0.569, max=0.569, sum=1.139 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5694240697848252\"}", - "Nutrition - # eval": "{\"description\": \"min=306, mean=306, max=306, sum=612 (2)\", \"tab\": \"General information\", \"score\": \"306.0\"}", - "Nutrition - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Nutrition - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Nutrition - # prompt tokens": "{\"description\": \"min=585.48, mean=585.48, max=585.48, sum=1170.961 (2)\", \"tab\": \"General information\", \"score\": \"585.4803921568628\"}", - "Nutrition - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"nutrition\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_nutrition\"" - } - } - }, - { - "evaluation_name": "Prehistory", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Prehistory", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.87, - "details": { - "description": "min=0.87, mean=0.87, max=0.87, sum=1.741 (2)", - "tab": "Accuracy", - "Prehistory - Observed inference time (s)": "{\"description\": \"min=0.57, mean=0.57, max=0.57, sum=1.141 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5704048761615047\"}", - "Prehistory - # eval": "{\"description\": \"min=324, mean=324, max=324, sum=648 (2)\", \"tab\": \"General information\", \"score\": \"324.0\"}", - "Prehistory - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Prehistory - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Prehistory - # prompt tokens": "{\"description\": \"min=540.198, mean=540.198, max=540.198, sum=1080.395 (2)\", \"tab\": \"General information\", \"score\": \"540.1975308641976\"}", - "Prehistory - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"prehistory\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_prehistory\"" - } - } - }, - { - "evaluation_name": "Public Relations", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Public Relations", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.818, - "details": { - "description": "min=0.818, mean=0.818, max=0.818, sum=1.636 (2)", - "tab": "Accuracy", - "Public Relations - Observed inference time (s)": "{\"description\": \"min=0.702, mean=0.702, max=0.702, sum=1.403 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7017486507242376\"}", - "Public Relations - # eval": "{\"description\": \"min=110, mean=110, max=110, sum=220 (2)\", \"tab\": \"General information\", \"score\": \"110.0\"}", - "Public Relations - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Public Relations - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Public Relations - # prompt tokens": "{\"description\": \"min=426.655, mean=426.655, max=426.655, sum=853.309 (2)\", \"tab\": \"General information\", \"score\": \"426.6545454545454\"}", - "Public Relations - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"public_relations\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_public_relations\"" - } - } - }, - { - "evaluation_name": "Security Studies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Security Studies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.873, - "details": { - "description": "min=0.873, mean=0.873, max=0.873, sum=1.747 (2)", - "tab": "Accuracy", - "Security Studies - Observed inference time (s)": "{\"description\": \"min=0.6, mean=0.6, max=0.6, sum=1.2 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6002200584022366\"}", - "Security Studies - # eval": "{\"description\": \"min=245, mean=245, max=245, sum=490 (2)\", \"tab\": \"General information\", \"score\": \"245.0\"}", - "Security Studies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Security Studies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Security Studies - # prompt tokens": "{\"description\": \"min=1193.869, mean=1193.869, max=1193.869, sum=2387.739 (2)\", \"tab\": \"General information\", \"score\": \"1193.869387755102\"}", - "Security Studies - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"security_studies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_security_studies\"" - } - } - }, - { - "evaluation_name": "Sociology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Sociology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.92, - "details": { - "description": "min=0.92, mean=0.92, max=0.92, sum=1.841 (2)", - "tab": "Accuracy", - "Sociology - Observed inference time (s)": "{\"description\": \"min=0.603, mean=0.603, max=0.603, sum=1.206 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6029752119263606\"}", - "Sociology - # eval": "{\"description\": \"min=201, mean=201, max=201, sum=402 (2)\", \"tab\": \"General information\", \"score\": \"201.0\"}", - "Sociology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Sociology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Sociology - # prompt tokens": "{\"description\": \"min=456.274, mean=456.274, max=456.274, sum=912.547 (2)\", \"tab\": \"General information\", \"score\": \"456.27363184079604\"}", - "Sociology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"sociology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_sociology\"" - } - } - }, - { - "evaluation_name": "Virology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Virology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.554, - "details": { - "description": "min=0.554, mean=0.554, max=0.554, sum=1.108 (2)", - "tab": "Accuracy", - "Virology - Observed inference time (s)": "{\"description\": \"min=0.59, mean=0.59, max=0.59, sum=1.181 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5903763368905309\"}", - "Virology - # eval": "{\"description\": \"min=166, mean=166, max=166, sum=332 (2)\", \"tab\": \"General information\", \"score\": \"166.0\"}", - "Virology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Virology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Virology - # prompt tokens": "{\"description\": \"min=336.753, mean=336.753, max=336.753, sum=673.506 (2)\", \"tab\": \"General information\", \"score\": \"336.7530120481928\"}", - "Virology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"virology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_virology\"" - } - } - }, - { - "evaluation_name": "World Religions", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on World Religions", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.854, - "details": { - "description": "min=0.854, mean=0.854, max=0.854, sum=1.708 (2)", - "tab": "Accuracy", - "World Religions - Observed inference time (s)": "{\"description\": \"min=0.568, mean=0.568, max=0.568, sum=1.137 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5682888700250994\"}", - "World Religions - # eval": "{\"description\": \"min=171, mean=171, max=171, sum=342 (2)\", \"tab\": \"General information\", \"score\": \"171.0\"}", - "World Religions - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "World Religions - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "World Religions - # prompt tokens": "{\"description\": \"min=268.164, mean=268.164, max=268.164, sum=536.327 (2)\", \"tab\": \"General information\", \"score\": \"268.1637426900585\"}", - "World Religions - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"world_religions\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_world_religions\"" - } - } - }, - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.349, - "details": { - "description": "", - "tab": "Efficiency" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/google_gemini-1.5-pro-002.json b/data/models/google_gemini-1.5-pro-002.json deleted file mode 100644 index c6cfa5cb7151a9afdf8a357a242b31551192d863..0000000000000000000000000000000000000000 --- a/data/models/google_gemini-1.5-pro-002.json +++ /dev/null @@ -1,2125 +0,0 @@ -{ - "model_info": { - "name": "Gemini 1.5 Pro 002", - "id": "google/gemini-1.5-pro-002", - "developer": "Google", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_capabilities/google_gemini-1.5-pro-002/1774096304.5056698", - "retrieved_timestamp": "1774096304.5056698", - "source_metadata": { - "source_name": "helm_capabilities", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_capabilities", - "evaluation_results": [ - { - "evaluation_name": "Mean score", - "source_data": { - "dataset_name": "helm_capabilities", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "The mean of the scores from all columns.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.657, - "details": { - "description": "", - "tab": "Accuracy", - "Mean score - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"9.106040294719884\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU-Pro", - "source_data": { - "dataset_name": "MMLU-Pro", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on MMLU-Pro", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.737, - "details": { - "description": "min=0.737, mean=0.737, max=0.737, sum=0.737 (1)", - "tab": "Accuracy", - "MMLU-Pro - Observed inference time (s)": "{\"description\": \"min=5.124, mean=5.124, max=5.124, sum=5.124 (1)\", \"tab\": \"Efficiency\", \"score\": \"5.123855731964111\"}", - "MMLU-Pro - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "MMLU-Pro - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - # prompt tokens": "{\"description\": \"min=242.673, mean=242.673, max=242.673, sum=242.673 (1)\", \"tab\": \"General information\", \"score\": \"242.673\"}", - "MMLU-Pro - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"all\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.534, - "details": { - "description": "min=0.534, mean=0.534, max=0.534, sum=0.534 (1)", - "tab": "Accuracy", - "GPQA - Observed inference time (s)": "{\"description\": \"min=7.392, mean=7.392, max=7.392, sum=7.392 (1)\", \"tab\": \"Efficiency\", \"score\": \"7.392140488988081\"}", - "GPQA - # eval": "{\"description\": \"min=446, mean=446, max=446, sum=446 (1)\", \"tab\": \"General information\", \"score\": \"446.0\"}", - "GPQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - # prompt tokens": "{\"description\": \"min=252.735, mean=252.735, max=252.735, sum=252.735 (1)\", \"tab\": \"General information\", \"score\": \"252.7354260089686\"}", - "GPQA - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"gpqa_main\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "IFEval Strict Acc on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.837, - "details": { - "description": "min=0.837, mean=0.837, max=0.837, sum=0.837 (1)", - "tab": "Accuracy", - "IFEval - Observed inference time (s)": "{\"description\": \"min=6.353, mean=6.353, max=6.353, sum=6.353 (1)\", \"tab\": \"Efficiency\", \"score\": \"6.352943865957631\"}", - "IFEval - # eval": "{\"description\": \"min=541, mean=541, max=541, sum=541 (1)\", \"tab\": \"General information\", \"score\": \"541.0\"}", - "IFEval - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - # prompt tokens": "{\"description\": \"min=47.331, mean=47.331, max=47.331, sum=47.331 (1)\", \"tab\": \"General information\", \"score\": \"47.33086876155268\"}", - "IFEval - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WildBench", - "source_data": { - "dataset_name": "WildBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "WB Score on WildBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.813, - "details": { - "description": "min=0.813, mean=0.813, max=0.813, sum=0.813 (1)", - "tab": "Accuracy", - "WildBench - Observed inference time (s)": "{\"description\": \"min=17.527, mean=17.527, max=17.527, sum=17.527 (1)\", \"tab\": \"Efficiency\", \"score\": \"17.52709009152358\"}", - "WildBench - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "WildBench - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # prompt tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"v2\"" - } - } - }, - { - "evaluation_name": "Omni-MATH", - "source_data": { - "dataset_name": "Omni-MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Acc on Omni-MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.364, - "details": { - "description": "min=0.364, mean=0.364, max=0.364, sum=0.364 (1)", - "tab": "Accuracy", - "Omni-MATH - Observed inference time (s)": "{\"description\": \"min=9.134, mean=9.134, max=9.134, sum=9.134 (1)\", \"tab\": \"Efficiency\", \"score\": \"9.134171295166016\"}", - "Omni-MATH - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "Omni-MATH - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - # prompt tokens": "{\"description\": \"min=111.956, mean=111.956, max=111.956, sum=111.956 (1)\", \"tab\": \"General information\", \"score\": \"111.956\"}", - "Omni-MATH - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_lite/google_gemini-1.5-pro-002/1774096306.427425", - "retrieved_timestamp": "1774096306.427425", - "source_metadata": { - "source_name": "helm_lite", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_lite", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_lite", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.842, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.49837702871410733\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.756, - "details": { - "description": "min=0.756, mean=0.756, max=0.756, sum=0.756 (1)", - "tab": "Accuracy", - "NarrativeQA - Observed inference time (s)": "{\"description\": \"min=0.912, mean=0.912, max=0.912, sum=0.912 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.9118197140368548\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=3437.994, mean=3437.994, max=3437.994, sum=3437.994 (1)\", \"tab\": \"General information\", \"score\": \"3437.994366197183\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (closed-book)", - "source_data": { - "dataset_name": "NaturalQuestions (closed-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (closed-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.455, - "details": { - "description": "min=0.455, mean=0.455, max=0.455, sum=0.455 (1)", - "tab": "Accuracy", - "NaturalQuestions (open-book) - Observed inference time (s)": "{\"description\": \"min=0.616, mean=0.616, max=0.616, sum=0.616 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.6156208164691925\"}", - "NaturalQuestions (closed-book) - Observed inference time (s)": "{\"description\": \"min=0.539, mean=0.539, max=0.539, sum=0.539 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.5389571013450623\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1968.347, mean=1968.347, max=1968.347, sum=1968.347 (1)\", \"tab\": \"General information\", \"score\": \"1968.347\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=143.995, mean=143.995, max=143.995, sum=143.995 (1)\", \"tab\": \"General information\", \"score\": \"143.995\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "mode": "\"closedbook\"" - } - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.952, - "details": { - "description": "min=0.952, mean=0.952, max=0.952, sum=0.952 (1)", - "tab": "Accuracy", - "OpenbookQA - Observed inference time (s)": "{\"description\": \"min=0.453, mean=0.453, max=0.453, sum=0.453 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.45284647941589357\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=248.508, mean=248.508, max=248.508, sum=248.508 (1)\", \"tab\": \"General information\", \"score\": \"248.508\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "dataset": "\"openbookqa\"", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.795, - "details": { - "description": "min=0.67, mean=0.795, max=0.94, sum=3.973 (5)", - "tab": "Accuracy", - "MMLU - Observed inference time (s)": "{\"description\": \"min=0.453, mean=0.977, max=1.671, sum=4.883 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.9766287260557476\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=380.91, mean=481.531, max=634.553, sum=2407.653 (5)\", \"tab\": \"General information\", \"score\": \"481.5305263157895\"}", - "MMLU - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"college_chemistry\", \"computer_security\", \"econometrics\", \"us_foreign_policy\"]", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MATH", - "source_data": { - "dataset_name": "MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Equivalent (CoT) on MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.92, - "details": { - "description": "min=0.789, mean=0.92, max=1, sum=6.44 (7)", - "tab": "Accuracy", - "MATH - Observed inference time (s)": "{\"description\": \"min=2.157, mean=3.273, max=4.064, sum=22.911 (7)\", \"tab\": \"Efficiency\", \"score\": \"3.2730091876347354\"}", - "MATH - # eval": "{\"description\": \"min=30, mean=62.429, max=135, sum=437 (7)\", \"tab\": \"General information\", \"score\": \"62.42857142857143\"}", - "MATH - # train": "{\"description\": \"min=8, mean=8, max=8, sum=56 (7)\", \"tab\": \"General information\", \"score\": \"8.0\"}", - "MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MATH - # prompt tokens": "{\"description\": \"min=938.215, mean=1355.506, max=2348.712, sum=9488.545 (7)\", \"tab\": \"General information\", \"score\": \"1355.5064552904823\"}", - "MATH - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"algebra\", \"counting_and_probability\", \"geometry\", \"intermediate_algebra\", \"number_theory\", \"prealgebra\", \"precalculus\"]", - "level": "\"1\"", - "use_official_examples": "\"False\"", - "use_chain_of_thought": "\"True\"" - } - } - }, - { - "evaluation_name": "GSM8K", - "source_data": { - "dataset_name": "GSM8K", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on GSM8K", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.817, - "details": { - "description": "min=0.817, mean=0.817, max=0.817, sum=0.817 (1)", - "tab": "Accuracy", - "GSM8K - Observed inference time (s)": "{\"description\": \"min=3.161, mean=3.161, max=3.161, sum=3.161 (1)\", \"tab\": \"Efficiency\", \"score\": \"3.1614130451679228\"}", - "GSM8K - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "GSM8K - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "GSM8K - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GSM8K - # prompt tokens": "{\"description\": \"min=1151.885, mean=1151.885, max=1151.885, sum=1151.885 (1)\", \"tab\": \"General information\", \"score\": \"1151.885\"}", - "GSM8K - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "LegalBench", - "source_data": { - "dataset_name": "LegalBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on LegalBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.747, - "details": { - "description": "min=0.439, mean=0.747, max=0.968, sum=3.735 (5)", - "tab": "Accuracy", - "LegalBench - Observed inference time (s)": "{\"description\": \"min=0.489, mean=0.596, max=0.915, sum=2.982 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.596480936304943\"}", - "LegalBench - # eval": "{\"description\": \"min=95, mean=409.4, max=1000, sum=2047 (5)\", \"tab\": \"General information\", \"score\": \"409.4\"}", - "LegalBench - # train": "{\"description\": \"min=4, mean=4.8, max=5, sum=24 (5)\", \"tab\": \"General information\", \"score\": \"4.8\"}", - "LegalBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "LegalBench - # prompt tokens": "{\"description\": \"min=199.916, mean=1548.239, max=6413.569, sum=7741.193 (5)\", \"tab\": \"General information\", \"score\": \"1548.2386051001386\"}", - "LegalBench - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"abercrombie\", \"corporate_lobbying\", \"function_of_decision_section\", \"international_citizenship_questions\", \"proa\"]" - } - } - }, - { - "evaluation_name": "MedQA", - "source_data": { - "dataset_name": "MedQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MedQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.771, - "details": { - "description": "min=0.771, mean=0.771, max=0.771, sum=0.771 (1)", - "tab": "Accuracy", - "MedQA - Observed inference time (s)": "{\"description\": \"min=0.53, mean=0.53, max=0.53, sum=0.53 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.5296175953882115\"}", - "MedQA - # eval": "{\"description\": \"min=503, mean=503, max=503, sum=503 (1)\", \"tab\": \"General information\", \"score\": \"503.0\"}", - "MedQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MedQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MedQA - # prompt tokens": "{\"description\": \"min=1029.481, mean=1029.481, max=1029.481, sum=1029.481 (1)\", \"tab\": \"General information\", \"score\": \"1029.4811133200794\"}", - "MedQA - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WMT 2014", - "source_data": { - "dataset_name": "WMT 2014", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "BLEU-4 on WMT 2014", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.231, - "details": { - "description": "min=0.192, mean=0.231, max=0.261, sum=1.156 (5)", - "tab": "Accuracy", - "WMT 2014 - Observed inference time (s)": "{\"description\": \"min=1.049, mean=1.108, max=1.147, sum=5.541 (5)\", \"tab\": \"Efficiency\", \"score\": \"1.1081515031376248\"}", - "WMT 2014 - # eval": "{\"description\": \"min=503, mean=568.8, max=832, sum=2844 (5)\", \"tab\": \"General information\", \"score\": \"568.8\"}", - "WMT 2014 - # train": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "WMT 2014 - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WMT 2014 - # prompt tokens": "{\"description\": \"min=80.732, mean=110.97, max=137.366, sum=554.851 (5)\", \"tab\": \"General information\", \"score\": \"110.97025108961614\"}", - "WMT 2014 - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "language_pair": "[\"cs-en\", \"de-en\", \"fr-en\", \"hi-en\", \"ru-en\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_mmlu/google_gemini-1.5-pro-002/1774096312.00548", - "retrieved_timestamp": "1774096312.00548", - "source_metadata": { - "source_name": "helm_mmlu", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_mmlu", - "evaluation_results": [ - { - "evaluation_name": "MMLU All Subjects", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU All Subjects", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.869, - "details": { - "description": "min=0.566, mean=0.869, max=0.99, sum=99.042 (114)", - "tab": "Accuracy", - "MMLU All Subjects - Observed inference time (s)": "{\"description\": \"min=0.42, mean=0.696, max=1.671, sum=79.296 (114)\", \"tab\": \"Efficiency\", \"score\": \"0.695582110070124\"}", - "MMLU All Subjects - # eval": "{\"description\": \"min=100, mean=246.351, max=1534, sum=28084 (114)\", \"tab\": \"General information\", \"score\": \"246.35087719298247\"}", - "MMLU All Subjects - # train": "{\"description\": \"min=5, mean=5, max=5, sum=570 (114)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU All Subjects - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU All Subjects - # prompt tokens": "{\"description\": \"min=268.164, mean=632.617, max=2797.424, sum=72118.345 (114)\", \"tab\": \"General information\", \"score\": \"632.6170571214202\"}", - "MMLU All Subjects - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_name": "Abstract Algebra", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Abstract Algebra", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.82, - "details": { - "description": "min=0.82, mean=0.82, max=0.82, sum=1.64 (2)", - "tab": "Accuracy", - "Abstract Algebra - Observed inference time (s)": "{\"description\": \"min=1.671, mean=1.671, max=1.671, sum=3.341 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.6706047868728637\"}", - "Abstract Algebra - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Abstract Algebra - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Abstract Algebra - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Abstract Algebra - # prompt tokens": "{\"description\": \"min=383.97, mean=383.97, max=383.97, sum=767.94 (2)\", \"tab\": \"General information\", \"score\": \"383.97\"}", - "Abstract Algebra - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"abstract_algebra\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_abstract_algebra\"" - } - } - }, - { - "evaluation_name": "Anatomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Anatomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.83, - "details": { - "description": "min=0.83, mean=0.83, max=0.83, sum=1.659 (2)", - "tab": "Accuracy", - "Anatomy - Observed inference time (s)": "{\"description\": \"min=0.653, mean=0.653, max=0.653, sum=1.306 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.652814730891475\"}", - "Anatomy - # eval": "{\"description\": \"min=135, mean=135, max=135, sum=270 (2)\", \"tab\": \"General information\", \"score\": \"135.0\"}", - "Anatomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Anatomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Anatomy - # prompt tokens": "{\"description\": \"min=344.356, mean=344.356, max=344.356, sum=688.711 (2)\", \"tab\": \"General information\", \"score\": \"344.35555555555555\"}", - "Anatomy - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"anatomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_anatomy\"" - } - } - }, - { - "evaluation_name": "College Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on College Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.863, - "details": { - "description": "min=0.863, mean=0.863, max=0.863, sum=1.725 (2)", - "tab": "Accuracy", - "College Chemistry - Observed inference time (s)": "{\"description\": \"min=1.16, mean=1.16, max=1.16, sum=2.319 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.1597088170051575\"}", - "College Biology - Observed inference time (s)": "{\"description\": \"min=0.511, mean=0.511, max=0.511, sum=1.022 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5110265033112632\"}", - "College Computer Science - Observed inference time (s)": "{\"description\": \"min=0.88, mean=0.88, max=0.88, sum=1.76 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8800347399711609\"}", - "College Mathematics - Observed inference time (s)": "{\"description\": \"min=0.478, mean=0.478, max=0.478, sum=0.955 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.477603075504303\"}", - "College Medicine - Observed inference time (s)": "{\"description\": \"min=0.865, mean=0.865, max=0.865, sum=1.73 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8651723158841877\"}", - "College Physics - Observed inference time (s)": "{\"description\": \"min=0.593, mean=0.593, max=0.593, sum=1.186 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5927850522247016\"}", - "College Chemistry - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Chemistry - # prompt tokens": "{\"description\": \"min=570.02, mean=570.02, max=570.02, sum=1140.04 (2)\", \"tab\": \"General information\", \"score\": \"570.02\"}", - "College Chemistry - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # eval": "{\"description\": \"min=144, mean=144, max=144, sum=288 (2)\", \"tab\": \"General information\", \"score\": \"144.0\"}", - "College Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # prompt tokens": "{\"description\": \"min=482.799, mean=482.799, max=482.799, sum=965.597 (2)\", \"tab\": \"General information\", \"score\": \"482.7986111111111\"}", - "College Biology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # prompt tokens": "{\"description\": \"min=857.86, mean=857.86, max=857.86, sum=1715.72 (2)\", \"tab\": \"General information\", \"score\": \"857.86\"}", - "College Computer Science - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # prompt tokens": "{\"description\": \"min=626.69, mean=626.69, max=626.69, sum=1253.38 (2)\", \"tab\": \"General information\", \"score\": \"626.69\"}", - "College Mathematics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # eval": "{\"description\": \"min=173, mean=173, max=173, sum=346 (2)\", \"tab\": \"General information\", \"score\": \"173.0\"}", - "College Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # prompt tokens": "{\"description\": \"min=513.37, mean=513.37, max=513.37, sum=1026.74 (2)\", \"tab\": \"General information\", \"score\": \"513.3699421965318\"}", - "College Medicine - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # eval": "{\"description\": \"min=102, mean=102, max=102, sum=204 (2)\", \"tab\": \"General information\", \"score\": \"102.0\"}", - "College Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # prompt tokens": "{\"description\": \"min=507.471, mean=507.471, max=507.471, sum=1014.941 (2)\", \"tab\": \"General information\", \"score\": \"507.47058823529414\"}", - "College Physics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"college_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_college_physics\"" - } - } - }, - { - "evaluation_name": "Computer Security", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Computer Security", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.85, - "details": { - "description": "min=0.85, mean=0.85, max=0.85, sum=1.7 (2)", - "tab": "Accuracy", - "Computer Security - Observed inference time (s)": "{\"description\": \"min=0.453, mean=0.453, max=0.453, sum=0.905 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.45262243270874025\"}", - "Computer Security - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Computer Security - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Computer Security - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Computer Security - # prompt tokens": "{\"description\": \"min=380.91, mean=380.91, max=380.91, sum=761.82 (2)\", \"tab\": \"General information\", \"score\": \"380.91\"}", - "Computer Security - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"computer_security\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_computer_security\"" - } - } - }, - { - "evaluation_name": "Econometrics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Econometrics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.693, - "details": { - "description": "min=0.693, mean=0.693, max=0.693, sum=1.386 (2)", - "tab": "Accuracy", - "Econometrics - Observed inference time (s)": "{\"description\": \"min=1.068, mean=1.068, max=1.068, sum=2.135 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.067676763785513\"}", - "Econometrics - # eval": "{\"description\": \"min=114, mean=114, max=114, sum=228 (2)\", \"tab\": \"General information\", \"score\": \"114.0\"}", - "Econometrics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Econometrics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Econometrics - # prompt tokens": "{\"description\": \"min=634.553, mean=634.553, max=634.553, sum=1269.105 (2)\", \"tab\": \"General information\", \"score\": \"634.5526315789474\"}", - "Econometrics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"econometrics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_econometrics\"" - } - } - }, - { - "evaluation_name": "Global Facts", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Global Facts", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.77, - "details": { - "description": "min=0.77, mean=0.77, max=0.77, sum=1.54 (2)", - "tab": "Accuracy", - "Global Facts - Observed inference time (s)": "{\"description\": \"min=0.792, mean=0.792, max=0.792, sum=1.584 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7918326926231384\"}", - "Global Facts - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Global Facts - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Global Facts - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Global Facts - # prompt tokens": "{\"description\": \"min=456.54, mean=456.54, max=456.54, sum=913.08 (2)\", \"tab\": \"General information\", \"score\": \"456.54\"}", - "Global Facts - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"global_facts\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_global_facts\"" - } - } - }, - { - "evaluation_name": "Jurisprudence", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Jurisprudence", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.898, - "details": { - "description": "min=0.898, mean=0.898, max=0.898, sum=1.796 (2)", - "tab": "Accuracy", - "Jurisprudence - Observed inference time (s)": "{\"description\": \"min=0.76, mean=0.76, max=0.76, sum=1.52 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7597615586386787\"}", - "Jurisprudence - # eval": "{\"description\": \"min=108, mean=108, max=108, sum=216 (2)\", \"tab\": \"General information\", \"score\": \"108.0\"}", - "Jurisprudence - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Jurisprudence - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Jurisprudence - # prompt tokens": "{\"description\": \"min=407.87, mean=407.87, max=407.87, sum=815.741 (2)\", \"tab\": \"General information\", \"score\": \"407.8703703703704\"}", - "Jurisprudence - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"jurisprudence\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_jurisprudence\"" - } - } - }, - { - "evaluation_name": "Philosophy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Philosophy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.887, - "details": { - "description": "min=0.887, mean=0.887, max=0.887, sum=1.775 (2)", - "tab": "Accuracy", - "Philosophy - Observed inference time (s)": "{\"description\": \"min=0.453, mean=0.453, max=0.453, sum=0.907 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.45336360793405023\"}", - "Philosophy - # eval": "{\"description\": \"min=311, mean=311, max=311, sum=622 (2)\", \"tab\": \"General information\", \"score\": \"311.0\"}", - "Philosophy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Philosophy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Philosophy - # prompt tokens": "{\"description\": \"min=340.907, mean=340.907, max=340.907, sum=681.814 (2)\", \"tab\": \"General information\", \"score\": \"340.90675241157555\"}", - "Philosophy - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"philosophy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_philosophy\"" - } - } - }, - { - "evaluation_name": "Professional Psychology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Professional Psychology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.912, - "details": { - "description": "min=0.912, mean=0.912, max=0.912, sum=1.824 (2)", - "tab": "Accuracy", - "Professional Medicine - Observed inference time (s)": "{\"description\": \"min=0.804, mean=0.804, max=0.804, sum=1.609 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8043198874768089\"}", - "Professional Accounting - Observed inference time (s)": "{\"description\": \"min=0.738, mean=0.738, max=0.738, sum=1.476 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7378175072636165\"}", - "Professional Law - Observed inference time (s)": "{\"description\": \"min=0.546, mean=0.546, max=0.546, sum=1.091 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5455011718431694\"}", - "Professional Psychology - Observed inference time (s)": "{\"description\": \"min=0.47, mean=0.47, max=0.47, sum=0.94 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.47001955400105394\"}", - "Professional Medicine - # eval": "{\"description\": \"min=272, mean=272, max=272, sum=544 (2)\", \"tab\": \"General information\", \"score\": \"272.0\"}", - "Professional Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Medicine - # prompt tokens": "{\"description\": \"min=1113.092, mean=1113.092, max=1113.092, sum=2226.184 (2)\", \"tab\": \"General information\", \"score\": \"1113.0919117647059\"}", - "Professional Medicine - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # eval": "{\"description\": \"min=282, mean=282, max=282, sum=564 (2)\", \"tab\": \"General information\", \"score\": \"282.0\"}", - "Professional Accounting - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Accounting - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # prompt tokens": "{\"description\": \"min=755.418, mean=755.418, max=755.418, sum=1510.837 (2)\", \"tab\": \"General information\", \"score\": \"755.418439716312\"}", - "Professional Accounting - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # eval": "{\"description\": \"min=1534, mean=1534, max=1534, sum=3068 (2)\", \"tab\": \"General information\", \"score\": \"1534.0\"}", - "Professional Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # prompt tokens": "{\"description\": \"min=1685.119, mean=1685.119, max=1685.119, sum=3370.239 (2)\", \"tab\": \"General information\", \"score\": \"1685.119295958279\"}", - "Professional Law - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # eval": "{\"description\": \"min=612, mean=612, max=612, sum=1224 (2)\", \"tab\": \"General information\", \"score\": \"612.0\"}", - "Professional Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # prompt tokens": "{\"description\": \"min=594.363, mean=594.363, max=594.363, sum=1188.725 (2)\", \"tab\": \"General information\", \"score\": \"594.3627450980392\"}", - "Professional Psychology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"professional_psychology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_professional_psychology\"" - } - } - }, - { - "evaluation_name": "Us Foreign Policy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Us Foreign Policy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.94, - "details": { - "description": "min=0.94, mean=0.94, max=0.94, sum=1.88 (2)", - "tab": "Accuracy", - "Us Foreign Policy - Observed inference time (s)": "{\"description\": \"min=0.533, mean=0.533, max=0.533, sum=1.065 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5325308299064636\"}", - "Us Foreign Policy - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Us Foreign Policy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Us Foreign Policy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Us Foreign Policy - # prompt tokens": "{\"description\": \"min=438.2, mean=438.2, max=438.2, sum=876.4 (2)\", \"tab\": \"General information\", \"score\": \"438.2\"}", - "Us Foreign Policy - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"us_foreign_policy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_us_foreign_policy\"" - } - } - }, - { - "evaluation_name": "Astronomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Astronomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.934, - "details": { - "description": "min=0.934, mean=0.934, max=0.934, sum=1.868 (2)", - "tab": "Accuracy", - "Astronomy - Observed inference time (s)": "{\"description\": \"min=1.036, mean=1.036, max=1.036, sum=2.071 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.03554652239147\"}", - "Astronomy - # eval": "{\"description\": \"min=152, mean=152, max=152, sum=304 (2)\", \"tab\": \"General information\", \"score\": \"152.0\"}", - "Astronomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Astronomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Astronomy - # prompt tokens": "{\"description\": \"min=602.421, mean=602.421, max=602.421, sum=1204.842 (2)\", \"tab\": \"General information\", \"score\": \"602.421052631579\"}", - "Astronomy - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"astronomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_astronomy\"" - } - } - }, - { - "evaluation_name": "Business Ethics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Business Ethics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.84, - "details": { - "description": "min=0.84, mean=0.84, max=0.84, sum=1.68 (2)", - "tab": "Accuracy", - "Business Ethics - Observed inference time (s)": "{\"description\": \"min=1.112, mean=1.112, max=1.112, sum=2.223 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.1116365933418273\"}", - "Business Ethics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Business Ethics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Business Ethics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Business Ethics - # prompt tokens": "{\"description\": \"min=552.87, mean=552.87, max=552.87, sum=1105.74 (2)\", \"tab\": \"General information\", \"score\": \"552.87\"}", - "Business Ethics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"business_ethics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_business_ethics\"" - } - } - }, - { - "evaluation_name": "Clinical Knowledge", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Clinical Knowledge", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.906, - "details": { - "description": "min=0.906, mean=0.906, max=0.906, sum=1.811 (2)", - "tab": "Accuracy", - "Clinical Knowledge - Observed inference time (s)": "{\"description\": \"min=0.469, mean=0.469, max=0.469, sum=0.937 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4685829783385655\"}", - "Clinical Knowledge - # eval": "{\"description\": \"min=265, mean=265, max=265, sum=530 (2)\", \"tab\": \"General information\", \"score\": \"265.0\"}", - "Clinical Knowledge - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Clinical Knowledge - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Clinical Knowledge - # prompt tokens": "{\"description\": \"min=402.592, mean=402.592, max=402.592, sum=805.185 (2)\", \"tab\": \"General information\", \"score\": \"402.5924528301887\"}", - "Clinical Knowledge - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"clinical_knowledge\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_clinical_knowledge\"" - } - } - }, - { - "evaluation_name": "Conceptual Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Conceptual Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.945, - "details": { - "description": "min=0.945, mean=0.945, max=0.945, sum=1.889 (2)", - "tab": "Accuracy", - "Conceptual Physics - Observed inference time (s)": "{\"description\": \"min=0.659, mean=0.659, max=0.659, sum=1.317 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6586567797559373\"}", - "Conceptual Physics - # eval": "{\"description\": \"min=235, mean=235, max=235, sum=470 (2)\", \"tab\": \"General information\", \"score\": \"235.0\"}", - "Conceptual Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Conceptual Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Conceptual Physics - # prompt tokens": "{\"description\": \"min=309.213, mean=309.213, max=309.213, sum=618.426 (2)\", \"tab\": \"General information\", \"score\": \"309.21276595744683\"}", - "Conceptual Physics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"conceptual_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_conceptual_physics\"" - } - } - }, - { - "evaluation_name": "Electrical Engineering", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Electrical Engineering", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.855, - "details": { - "description": "min=0.855, mean=0.855, max=0.855, sum=1.71 (2)", - "tab": "Accuracy", - "Electrical Engineering - Observed inference time (s)": "{\"description\": \"min=0.474, mean=0.474, max=0.474, sum=0.948 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4739974646732725\"}", - "Electrical Engineering - # eval": "{\"description\": \"min=145, mean=145, max=145, sum=290 (2)\", \"tab\": \"General information\", \"score\": \"145.0\"}", - "Electrical Engineering - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Electrical Engineering - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Electrical Engineering - # prompt tokens": "{\"description\": \"min=474.786, mean=474.786, max=474.786, sum=949.572 (2)\", \"tab\": \"General information\", \"score\": \"474.78620689655173\"}", - "Electrical Engineering - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"electrical_engineering\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_electrical_engineering\"" - } - } - }, - { - "evaluation_name": "Elementary Mathematics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Elementary Mathematics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.942, - "details": { - "description": "min=0.942, mean=0.942, max=0.942, sum=1.884 (2)", - "tab": "Accuracy", - "Elementary Mathematics - Observed inference time (s)": "{\"description\": \"min=0.58, mean=0.58, max=0.58, sum=1.16 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5800282936247568\"}", - "Elementary Mathematics - # eval": "{\"description\": \"min=378, mean=378, max=378, sum=756 (2)\", \"tab\": \"General information\", \"score\": \"378.0\"}", - "Elementary Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Elementary Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Elementary Mathematics - # prompt tokens": "{\"description\": \"min=597.341, mean=597.341, max=597.341, sum=1194.683 (2)\", \"tab\": \"General information\", \"score\": \"597.3412698412699\"}", - "Elementary Mathematics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"elementary_mathematics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_elementary_mathematics\"" - } - } - }, - { - "evaluation_name": "Formal Logic", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Formal Logic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.754, - "details": { - "description": "min=0.754, mean=0.754, max=0.754, sum=1.508 (2)", - "tab": "Accuracy", - "Formal Logic - Observed inference time (s)": "{\"description\": \"min=0.926, mean=0.926, max=0.926, sum=1.852 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9259536947522845\"}", - "Formal Logic - # eval": "{\"description\": \"min=126, mean=126, max=126, sum=252 (2)\", \"tab\": \"General information\", \"score\": \"126.0\"}", - "Formal Logic - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Formal Logic - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Formal Logic - # prompt tokens": "{\"description\": \"min=619.563, mean=619.563, max=619.563, sum=1239.127 (2)\", \"tab\": \"General information\", \"score\": \"619.563492063492\"}", - "Formal Logic - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"formal_logic\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_formal_logic\"" - } - } - }, - { - "evaluation_name": "High School World History", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on High School World History", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.937, - "details": { - "description": "min=0.937, mean=0.937, max=0.937, sum=1.873 (2)", - "tab": "Accuracy", - "High School Biology - Observed inference time (s)": "{\"description\": \"min=0.458, mean=0.458, max=0.458, sum=0.916 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4579133049134285\"}", - "High School Chemistry - Observed inference time (s)": "{\"description\": \"min=0.587, mean=0.587, max=0.587, sum=1.175 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5872501540066574\"}", - "High School Computer Science - Observed inference time (s)": "{\"description\": \"min=0.493, mean=0.493, max=0.493, sum=0.987 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.49327227354049685\"}", - "High School European History - Observed inference time (s)": "{\"description\": \"min=0.84, mean=0.84, max=0.84, sum=1.68 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8402222113175826\"}", - "High School Geography - Observed inference time (s)": "{\"description\": \"min=0.674, mean=0.674, max=0.674, sum=1.349 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6743082650984177\"}", - "High School Government And Politics - Observed inference time (s)": "{\"description\": \"min=0.494, mean=0.494, max=0.494, sum=0.988 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4939905238275083\"}", - "High School Macroeconomics - Observed inference time (s)": "{\"description\": \"min=0.75, mean=0.75, max=0.75, sum=1.501 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.750414514541626\"}", - "High School Mathematics - Observed inference time (s)": "{\"description\": \"min=0.809, mean=0.809, max=0.809, sum=1.618 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8088616865652579\"}", - "High School Microeconomics - Observed inference time (s)": "{\"description\": \"min=0.711, mean=0.711, max=0.711, sum=1.423 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.711490568994474\"}", - "High School Physics - Observed inference time (s)": "{\"description\": \"min=0.832, mean=0.832, max=0.832, sum=1.664 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8320141549141992\"}", - "High School Psychology - Observed inference time (s)": "{\"description\": \"min=0.654, mean=0.654, max=0.654, sum=1.309 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6543280317149031\"}", - "High School Statistics - Observed inference time (s)": "{\"description\": \"min=0.688, mean=0.688, max=0.688, sum=1.377 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6883480460555466\"}", - "High School US History - Observed inference time (s)": "{\"description\": \"min=0.848, mean=0.848, max=0.848, sum=1.695 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8477429151535034\"}", - "High School World History - Observed inference time (s)": "{\"description\": \"min=0.641, mean=0.641, max=0.641, sum=1.282 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6409383886474095\"}", - "High School Biology - # eval": "{\"description\": \"min=310, mean=310, max=310, sum=620 (2)\", \"tab\": \"General information\", \"score\": \"310.0\"}", - "High School Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Biology - # prompt tokens": "{\"description\": \"min=500.958, mean=500.958, max=500.958, sum=1001.916 (2)\", \"tab\": \"General information\", \"score\": \"500.958064516129\"}", - "High School Biology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # eval": "{\"description\": \"min=203, mean=203, max=203, sum=406 (2)\", \"tab\": \"General information\", \"score\": \"203.0\"}", - "High School Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # prompt tokens": "{\"description\": \"min=513.064, mean=513.064, max=513.064, sum=1026.128 (2)\", \"tab\": \"General information\", \"score\": \"513.064039408867\"}", - "High School Chemistry - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "High School Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # prompt tokens": "{\"description\": \"min=935.13, mean=935.13, max=935.13, sum=1870.26 (2)\", \"tab\": \"General information\", \"score\": \"935.13\"}", - "High School Computer Science - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # eval": "{\"description\": \"min=165, mean=165, max=165, sum=330 (2)\", \"tab\": \"General information\", \"score\": \"165.0\"}", - "High School European History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School European History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # prompt tokens": "{\"description\": \"min=2797.424, mean=2797.424, max=2797.424, sum=5594.848 (2)\", \"tab\": \"General information\", \"score\": \"2797.4242424242425\"}", - "High School European History - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # eval": "{\"description\": \"min=198, mean=198, max=198, sum=396 (2)\", \"tab\": \"General information\", \"score\": \"198.0\"}", - "High School Geography - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Geography - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # prompt tokens": "{\"description\": \"min=394.773, mean=394.773, max=394.773, sum=789.545 (2)\", \"tab\": \"General information\", \"score\": \"394.77272727272725\"}", - "High School Geography - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # eval": "{\"description\": \"min=193, mean=193, max=193, sum=386 (2)\", \"tab\": \"General information\", \"score\": \"193.0\"}", - "High School Government And Politics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Government And Politics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # prompt tokens": "{\"description\": \"min=479.301, mean=479.301, max=479.301, sum=958.601 (2)\", \"tab\": \"General information\", \"score\": \"479.30051813471505\"}", - "High School Government And Politics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # eval": "{\"description\": \"min=390, mean=390, max=390, sum=780 (2)\", \"tab\": \"General information\", \"score\": \"390.0\"}", - "High School Macroeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Macroeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # prompt tokens": "{\"description\": \"min=396.541, mean=396.541, max=396.541, sum=793.082 (2)\", \"tab\": \"General information\", \"score\": \"396.54102564102567\"}", - "High School Macroeconomics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # eval": "{\"description\": \"min=270, mean=270, max=270, sum=540 (2)\", \"tab\": \"General information\", \"score\": \"270.0\"}", - "High School Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # prompt tokens": "{\"description\": \"min=566.822, mean=566.822, max=566.822, sum=1133.644 (2)\", \"tab\": \"General information\", \"score\": \"566.8222222222222\"}", - "High School Mathematics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # eval": "{\"description\": \"min=238, mean=238, max=238, sum=476 (2)\", \"tab\": \"General information\", \"score\": \"238.0\"}", - "High School Microeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Microeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # prompt tokens": "{\"description\": \"min=415.954, mean=415.954, max=415.954, sum=831.908 (2)\", \"tab\": \"General information\", \"score\": \"415.953781512605\"}", - "High School Microeconomics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # eval": "{\"description\": \"min=151, mean=151, max=151, sum=302 (2)\", \"tab\": \"General information\", \"score\": \"151.0\"}", - "High School Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # prompt tokens": "{\"description\": \"min=591.715, mean=591.715, max=591.715, sum=1183.43 (2)\", \"tab\": \"General information\", \"score\": \"591.7152317880794\"}", - "High School Physics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # eval": "{\"description\": \"min=545, mean=545, max=545, sum=1090 (2)\", \"tab\": \"General information\", \"score\": \"545.0\"}", - "High School Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # prompt tokens": "{\"description\": \"min=502.604, mean=502.604, max=502.604, sum=1005.207 (2)\", \"tab\": \"General information\", \"score\": \"502.60366972477067\"}", - "High School Psychology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # eval": "{\"description\": \"min=216, mean=216, max=216, sum=432 (2)\", \"tab\": \"General information\", \"score\": \"216.0\"}", - "High School Statistics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Statistics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # prompt tokens": "{\"description\": \"min=858.931, mean=858.931, max=858.931, sum=1717.861 (2)\", \"tab\": \"General information\", \"score\": \"858.9305555555555\"}", - "High School Statistics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # eval": "{\"description\": \"min=204, mean=204, max=204, sum=408 (2)\", \"tab\": \"General information\", \"score\": \"204.0\"}", - "High School US History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School US History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # prompt tokens": "{\"description\": \"min=2205.583, mean=2205.583, max=2205.583, sum=4411.167 (2)\", \"tab\": \"General information\", \"score\": \"2205.5833333333335\"}", - "High School US History - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # eval": "{\"description\": \"min=237, mean=237, max=237, sum=474 (2)\", \"tab\": \"General information\", \"score\": \"237.0\"}", - "High School World History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School World History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # prompt tokens": "{\"description\": \"min=1426.544, mean=1426.544, max=1426.544, sum=2853.089 (2)\", \"tab\": \"General information\", \"score\": \"1426.5443037974683\"}", - "High School World History - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"high_school_world_history\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_high_school_world_history\"" - } - } - }, - { - "evaluation_name": "Human Sexuality", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Human Sexuality", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.878, - "details": { - "description": "min=0.878, mean=0.878, max=0.878, sum=1.756 (2)", - "tab": "Accuracy", - "Human Aging - Observed inference time (s)": "{\"description\": \"min=0.825, mean=0.825, max=0.825, sum=1.651 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8252711541984113\"}", - "Human Sexuality - Observed inference time (s)": "{\"description\": \"min=0.689, mean=0.689, max=0.689, sum=1.378 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.689175573014121\"}", - "Human Aging - # eval": "{\"description\": \"min=223, mean=223, max=223, sum=446 (2)\", \"tab\": \"General information\", \"score\": \"223.0\"}", - "Human Aging - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Aging - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Aging - # prompt tokens": "{\"description\": \"min=321.587, mean=321.587, max=321.587, sum=643.175 (2)\", \"tab\": \"General information\", \"score\": \"321.58744394618833\"}", - "Human Aging - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # eval": "{\"description\": \"min=131, mean=131, max=131, sum=262 (2)\", \"tab\": \"General information\", \"score\": \"131.0\"}", - "Human Sexuality - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Sexuality - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # prompt tokens": "{\"description\": \"min=347.183, mean=347.183, max=347.183, sum=694.366 (2)\", \"tab\": \"General information\", \"score\": \"347.1832061068702\"}", - "Human Sexuality - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"human_sexuality\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_human_sexuality\"" - } - } - }, - { - "evaluation_name": "International Law", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on International Law", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.917, - "details": { - "description": "min=0.917, mean=0.917, max=0.917, sum=1.835 (2)", - "tab": "Accuracy", - "International Law - Observed inference time (s)": "{\"description\": \"min=0.544, mean=0.544, max=0.544, sum=1.089 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5443926212216211\"}", - "International Law - # eval": "{\"description\": \"min=121, mean=121, max=121, sum=242 (2)\", \"tab\": \"General information\", \"score\": \"121.0\"}", - "International Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "International Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "International Law - # prompt tokens": "{\"description\": \"min=644.165, mean=644.165, max=644.165, sum=1288.331 (2)\", \"tab\": \"General information\", \"score\": \"644.1652892561983\"}", - "International Law - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"international_law\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_international_law\"" - } - } - }, - { - "evaluation_name": "Logical Fallacies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Logical Fallacies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.902, - "details": { - "description": "min=0.902, mean=0.902, max=0.902, sum=1.804 (2)", - "tab": "Accuracy", - "Logical Fallacies - Observed inference time (s)": "{\"description\": \"min=0.706, mean=0.706, max=0.706, sum=1.412 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7058728443332977\"}", - "Logical Fallacies - # eval": "{\"description\": \"min=163, mean=163, max=163, sum=326 (2)\", \"tab\": \"General information\", \"score\": \"163.0\"}", - "Logical Fallacies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Logical Fallacies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Logical Fallacies - # prompt tokens": "{\"description\": \"min=450.049, mean=450.049, max=450.049, sum=900.098 (2)\", \"tab\": \"General information\", \"score\": \"450.0490797546012\"}", - "Logical Fallacies - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"logical_fallacies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_logical_fallacies\"" - } - } - }, - { - "evaluation_name": "Machine Learning", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Machine Learning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.83, - "details": { - "description": "min=0.83, mean=0.83, max=0.83, sum=1.661 (2)", - "tab": "Accuracy", - "Machine Learning - Observed inference time (s)": "{\"description\": \"min=0.476, mean=0.476, max=0.476, sum=0.952 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.47608799380915506\"}", - "Machine Learning - # eval": "{\"description\": \"min=112, mean=112, max=112, sum=224 (2)\", \"tab\": \"General information\", \"score\": \"112.0\"}", - "Machine Learning - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Machine Learning - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Machine Learning - # prompt tokens": "{\"description\": \"min=702.402, mean=702.402, max=702.402, sum=1404.804 (2)\", \"tab\": \"General information\", \"score\": \"702.4017857142857\"}", - "Machine Learning - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"machine_learning\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_machine_learning\"" - } - } - }, - { - "evaluation_name": "Management", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Management", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.903, - "details": { - "description": "min=0.903, mean=0.903, max=0.903, sum=1.806 (2)", - "tab": "Accuracy", - "Management - Observed inference time (s)": "{\"description\": \"min=0.51, mean=0.51, max=0.51, sum=1.02 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5099537488326286\"}", - "Management - # eval": "{\"description\": \"min=103, mean=103, max=103, sum=206 (2)\", \"tab\": \"General information\", \"score\": \"103.0\"}", - "Management - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Management - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Management - # prompt tokens": "{\"description\": \"min=281.301, mean=281.301, max=281.301, sum=562.602 (2)\", \"tab\": \"General information\", \"score\": \"281.3009708737864\"}", - "Management - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"management\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_management\"" - } - } - }, - { - "evaluation_name": "Marketing", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Marketing", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.962, - "details": { - "description": "min=0.962, mean=0.962, max=0.962, sum=1.923 (2)", - "tab": "Accuracy", - "Marketing - Observed inference time (s)": "{\"description\": \"min=0.422, mean=0.422, max=0.422, sum=0.843 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.42154710415082103\"}", - "Marketing - # eval": "{\"description\": \"min=234, mean=234, max=234, sum=468 (2)\", \"tab\": \"General information\", \"score\": \"234.0\"}", - "Marketing - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Marketing - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Marketing - # prompt tokens": "{\"description\": \"min=428.35, mean=428.35, max=428.35, sum=856.701 (2)\", \"tab\": \"General information\", \"score\": \"428.35042735042737\"}", - "Marketing - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"marketing\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_marketing\"" - } - } - }, - { - "evaluation_name": "Medical Genetics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Medical Genetics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.92, - "details": { - "description": "min=0.92, mean=0.92, max=0.92, sum=1.84 (2)", - "tab": "Accuracy", - "Medical Genetics - Observed inference time (s)": "{\"description\": \"min=0.66, mean=0.66, max=0.66, sum=1.321 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6604956579208374\"}", - "Medical Genetics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Medical Genetics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Medical Genetics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Medical Genetics - # prompt tokens": "{\"description\": \"min=338.89, mean=338.89, max=338.89, sum=677.78 (2)\", \"tab\": \"General information\", \"score\": \"338.89\"}", - "Medical Genetics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"medical_genetics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_medical_genetics\"" - } - } - }, - { - "evaluation_name": "Miscellaneous", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Miscellaneous", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.959, - "details": { - "description": "min=0.959, mean=0.959, max=0.959, sum=1.918 (2)", - "tab": "Accuracy", - "Miscellaneous - Observed inference time (s)": "{\"description\": \"min=0.564, mean=0.564, max=0.564, sum=1.128 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5638943230055301\"}", - "Miscellaneous - # eval": "{\"description\": \"min=783, mean=783, max=783, sum=1566 (2)\", \"tab\": \"General information\", \"score\": \"783.0\"}", - "Miscellaneous - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Miscellaneous - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Miscellaneous - # prompt tokens": "{\"description\": \"min=314.669, mean=314.669, max=314.669, sum=629.338 (2)\", \"tab\": \"General information\", \"score\": \"314.669220945083\"}", - "Miscellaneous - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"miscellaneous\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_miscellaneous\"" - } - } - }, - { - "evaluation_name": "Moral Scenarios", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Moral Scenarios", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.792, - "details": { - "description": "min=0.792, mean=0.792, max=0.792, sum=1.584 (2)", - "tab": "Accuracy", - "Moral Disputes - Observed inference time (s)": "{\"description\": \"min=1.245, mean=1.245, max=1.245, sum=2.49 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.244819999430221\"}", - "Moral Scenarios - Observed inference time (s)": "{\"description\": \"min=1.526, mean=1.526, max=1.526, sum=3.052 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.5260936177642652\"}", - "Moral Disputes - # eval": "{\"description\": \"min=346, mean=346, max=346, sum=692 (2)\", \"tab\": \"General information\", \"score\": \"346.0\"}", - "Moral Disputes - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Disputes - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Disputes - # prompt tokens": "{\"description\": \"min=495.003, mean=495.003, max=495.003, sum=990.006 (2)\", \"tab\": \"General information\", \"score\": \"495.0028901734104\"}", - "Moral Disputes - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # eval": "{\"description\": \"min=895, mean=895, max=895, sum=1790 (2)\", \"tab\": \"General information\", \"score\": \"895.0\"}", - "Moral Scenarios - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Scenarios - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # prompt tokens": "{\"description\": \"min=690.542, mean=690.542, max=690.542, sum=1381.084 (2)\", \"tab\": \"General information\", \"score\": \"690.5418994413408\"}", - "Moral Scenarios - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"moral_scenarios\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_moral_scenarios\"" - } - } - }, - { - "evaluation_name": "Nutrition", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Nutrition", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.886, - "details": { - "description": "min=0.886, mean=0.886, max=0.886, sum=1.771 (2)", - "tab": "Accuracy", - "Nutrition - Observed inference time (s)": "{\"description\": \"min=0.629, mean=0.629, max=0.629, sum=1.259 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6292609475017373\"}", - "Nutrition - # eval": "{\"description\": \"min=306, mean=306, max=306, sum=612 (2)\", \"tab\": \"General information\", \"score\": \"306.0\"}", - "Nutrition - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Nutrition - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Nutrition - # prompt tokens": "{\"description\": \"min=585.48, mean=585.48, max=585.48, sum=1170.961 (2)\", \"tab\": \"General information\", \"score\": \"585.4803921568628\"}", - "Nutrition - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"nutrition\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_nutrition\"" - } - } - }, - { - "evaluation_name": "Prehistory", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Prehistory", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.926, - "details": { - "description": "min=0.926, mean=0.926, max=0.926, sum=1.852 (2)", - "tab": "Accuracy", - "Prehistory - Observed inference time (s)": "{\"description\": \"min=0.54, mean=0.54, max=0.54, sum=1.08 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5400909362015901\"}", - "Prehistory - # eval": "{\"description\": \"min=324, mean=324, max=324, sum=648 (2)\", \"tab\": \"General information\", \"score\": \"324.0\"}", - "Prehistory - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Prehistory - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Prehistory - # prompt tokens": "{\"description\": \"min=540.198, mean=540.198, max=540.198, sum=1080.395 (2)\", \"tab\": \"General information\", \"score\": \"540.1975308641976\"}", - "Prehistory - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"prehistory\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_prehistory\"" - } - } - }, - { - "evaluation_name": "Public Relations", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Public Relations", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.809, - "details": { - "description": "min=0.809, mean=0.809, max=0.809, sum=1.618 (2)", - "tab": "Accuracy", - "Public Relations - Observed inference time (s)": "{\"description\": \"min=0.442, mean=0.442, max=0.442, sum=0.884 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4420530059120872\"}", - "Public Relations - # eval": "{\"description\": \"min=110, mean=110, max=110, sum=220 (2)\", \"tab\": \"General information\", \"score\": \"110.0\"}", - "Public Relations - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Public Relations - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Public Relations - # prompt tokens": "{\"description\": \"min=426.655, mean=426.655, max=426.655, sum=853.309 (2)\", \"tab\": \"General information\", \"score\": \"426.6545454545454\"}", - "Public Relations - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"public_relations\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_public_relations\"" - } - } - }, - { - "evaluation_name": "Security Studies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Security Studies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.857, - "details": { - "description": "min=0.857, mean=0.857, max=0.857, sum=1.714 (2)", - "tab": "Accuracy", - "Security Studies - Observed inference time (s)": "{\"description\": \"min=0.443, mean=0.443, max=0.443, sum=0.886 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.44290724871109943\"}", - "Security Studies - # eval": "{\"description\": \"min=245, mean=245, max=245, sum=490 (2)\", \"tab\": \"General information\", \"score\": \"245.0\"}", - "Security Studies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Security Studies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Security Studies - # prompt tokens": "{\"description\": \"min=1193.869, mean=1193.869, max=1193.869, sum=2387.739 (2)\", \"tab\": \"General information\", \"score\": \"1193.869387755102\"}", - "Security Studies - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"security_studies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_security_studies\"" - } - } - }, - { - "evaluation_name": "Sociology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Sociology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.95, - "details": { - "description": "min=0.95, mean=0.95, max=0.95, sum=1.9 (2)", - "tab": "Accuracy", - "Sociology - Observed inference time (s)": "{\"description\": \"min=0.42, mean=0.42, max=0.42, sum=0.841 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4202856958208986\"}", - "Sociology - # eval": "{\"description\": \"min=201, mean=201, max=201, sum=402 (2)\", \"tab\": \"General information\", \"score\": \"201.0\"}", - "Sociology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Sociology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Sociology - # prompt tokens": "{\"description\": \"min=456.274, mean=456.274, max=456.274, sum=912.547 (2)\", \"tab\": \"General information\", \"score\": \"456.27363184079604\"}", - "Sociology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"sociology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_sociology\"" - } - } - }, - { - "evaluation_name": "Virology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Virology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.566, - "details": { - "description": "min=0.566, mean=0.566, max=0.566, sum=1.133 (2)", - "tab": "Accuracy", - "Virology - Observed inference time (s)": "{\"description\": \"min=0.425, mean=0.425, max=0.425, sum=0.849 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4245123575968915\"}", - "Virology - # eval": "{\"description\": \"min=166, mean=166, max=166, sum=332 (2)\", \"tab\": \"General information\", \"score\": \"166.0\"}", - "Virology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Virology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Virology - # prompt tokens": "{\"description\": \"min=336.753, mean=336.753, max=336.753, sum=673.506 (2)\", \"tab\": \"General information\", \"score\": \"336.7530120481928\"}", - "Virology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"virology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_virology\"" - } - } - }, - { - "evaluation_name": "World Religions", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on World Religions", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.889, - "details": { - "description": "min=0.889, mean=0.889, max=0.889, sum=1.778 (2)", - "tab": "Accuracy", - "World Religions - Observed inference time (s)": "{\"description\": \"min=0.421, mean=0.421, max=0.421, sum=0.842 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4207720505563836\"}", - "World Religions - # eval": "{\"description\": \"min=171, mean=171, max=171, sum=342 (2)\", \"tab\": \"General information\", \"score\": \"171.0\"}", - "World Religions - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "World Religions - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "World Religions - # prompt tokens": "{\"description\": \"min=268.164, mean=268.164, max=268.164, sum=536.327 (2)\", \"tab\": \"General information\", \"score\": \"268.1637426900585\"}", - "World Religions - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"world_religions\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_world_religions\"" - } - } - }, - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.334, - "details": { - "description": "", - "tab": "Efficiency" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/google_gemini-1.5-pro-0514.json b/data/models/google_gemini-1.5-pro-0514.json deleted file mode 100644 index fa3669fffeb7885aaee63e91541f5f69feaf066c..0000000000000000000000000000000000000000 --- a/data/models/google_gemini-1.5-pro-0514.json +++ /dev/null @@ -1,126 +0,0 @@ -{ - "model_info": { - "name": "google/gemini-1.5-pro-0514", - "id": "google/gemini-1.5-pro-0514", - "developer": "Google", - "additional_details": { - "model_type": "Generative" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/google_gemini-1.5-pro-0514/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.882 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9232 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8059 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8791 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9199 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/google_gemini-1.5-pro-0924.json b/data/models/google_gemini-1.5-pro-0924.json deleted file mode 100644 index cbed4231b4554d0f8e564e3442c541784a92bb8f..0000000000000000000000000000000000000000 --- a/data/models/google_gemini-1.5-pro-0924.json +++ /dev/null @@ -1,126 +0,0 @@ -{ - "model_info": { - "name": "google/gemini-1.5-pro-0924", - "id": "google/gemini-1.5-pro-0924", - "developer": "Google", - "additional_details": { - "model_type": "Generative" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/google_gemini-1.5-pro-0924/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8678 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9413 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7697 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8581 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9022 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/google_gemini-1.5-pro-preview-0409.json b/data/models/google_gemini-1.5-pro-preview-0409.json deleted file mode 100644 index e920b597775aecac73008b0fad1d5d45d05ed3ac..0000000000000000000000000000000000000000 --- a/data/models/google_gemini-1.5-pro-preview-0409.json +++ /dev/null @@ -1,1531 +0,0 @@ -{ - "model_info": { - "name": "Gemini 1.5 Pro 0409 preview", - "id": "google/gemini-1.5-pro-preview-0409", - "developer": "Google", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_mmlu/google_gemini-1.5-pro-preview-0409/1774096312.00548", - "retrieved_timestamp": "1774096312.00548", - "source_metadata": { - "source_name": "helm_mmlu", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_mmlu", - "evaluation_results": [ - { - "evaluation_name": "MMLU All Subjects", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU All Subjects", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.81, - "details": { - "description": "min=0.397, mean=0.81, max=0.979, sum=92.284 (114)", - "tab": "Accuracy", - "MMLU All Subjects - Observed inference time (s)": "{\"description\": \"min=0.877, mean=1.174, max=3.173, sum=133.815 (114)\", \"tab\": \"Efficiency\", \"score\": \"1.1738183835156866\"}", - "MMLU All Subjects - # eval": "{\"description\": \"min=100, mean=246.351, max=1534, sum=28084 (114)\", \"tab\": \"General information\", \"score\": \"246.35087719298247\"}", - "MMLU All Subjects - # train": "{\"description\": \"min=5, mean=5, max=5, sum=570 (114)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU All Subjects - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU All Subjects - # prompt tokens": "{\"description\": \"min=268.164, mean=632.617, max=2797.424, sum=72118.345 (114)\", \"tab\": \"General information\", \"score\": \"632.6170571214202\"}", - "MMLU All Subjects - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_name": "Abstract Algebra", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Abstract Algebra", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6, - "details": { - "description": "min=0.6, mean=0.6, max=0.6, sum=1.2 (2)", - "tab": "Accuracy", - "Abstract Algebra - Observed inference time (s)": "{\"description\": \"min=1.767, mean=1.767, max=1.767, sum=3.533 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.7665750813484191\"}", - "Abstract Algebra - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Abstract Algebra - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Abstract Algebra - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Abstract Algebra - # prompt tokens": "{\"description\": \"min=383.97, mean=383.97, max=383.97, sum=767.94 (2)\", \"tab\": \"General information\", \"score\": \"383.97\"}", - "Abstract Algebra - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"abstract_algebra\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_abstract_algebra\"" - } - } - }, - { - "evaluation_name": "Anatomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Anatomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.77, - "details": { - "description": "min=0.77, mean=0.77, max=0.77, sum=1.541 (2)", - "tab": "Accuracy", - "Anatomy - Observed inference time (s)": "{\"description\": \"min=3.173, mean=3.173, max=3.173, sum=6.346 (2)\", \"tab\": \"Efficiency\", \"score\": \"3.1730875386132134\"}", - "Anatomy - # eval": "{\"description\": \"min=135, mean=135, max=135, sum=270 (2)\", \"tab\": \"General information\", \"score\": \"135.0\"}", - "Anatomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Anatomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Anatomy - # prompt tokens": "{\"description\": \"min=344.356, mean=344.356, max=344.356, sum=688.711 (2)\", \"tab\": \"General information\", \"score\": \"344.35555555555555\"}", - "Anatomy - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"anatomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_anatomy\"" - } - } - }, - { - "evaluation_name": "College Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on College Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.804, - "details": { - "description": "min=0.804, mean=0.804, max=0.804, sum=1.608 (2)", - "tab": "Accuracy", - "College Chemistry - Observed inference time (s)": "{\"description\": \"min=1.054, mean=1.054, max=1.054, sum=2.107 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.053539514541626\"}", - "College Biology - Observed inference time (s)": "{\"description\": \"min=0.985, mean=0.985, max=0.985, sum=1.971 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9854124503003227\"}", - "College Computer Science - Observed inference time (s)": "{\"description\": \"min=1.301, mean=1.301, max=1.301, sum=2.603 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.3013164806365967\"}", - "College Mathematics - Observed inference time (s)": "{\"description\": \"min=1.187, mean=1.187, max=1.187, sum=2.375 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.1873565983772278\"}", - "College Medicine - Observed inference time (s)": "{\"description\": \"min=1.149, mean=1.149, max=1.149, sum=2.298 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.1490558723493807\"}", - "College Physics - Observed inference time (s)": "{\"description\": \"min=1.017, mean=1.017, max=1.017, sum=2.034 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.0169454929875392\"}", - "College Chemistry - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Chemistry - # prompt tokens": "{\"description\": \"min=570.02, mean=570.02, max=570.02, sum=1140.04 (2)\", \"tab\": \"General information\", \"score\": \"570.02\"}", - "College Chemistry - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # eval": "{\"description\": \"min=144, mean=144, max=144, sum=288 (2)\", \"tab\": \"General information\", \"score\": \"144.0\"}", - "College Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # prompt tokens": "{\"description\": \"min=482.799, mean=482.799, max=482.799, sum=965.597 (2)\", \"tab\": \"General information\", \"score\": \"482.7986111111111\"}", - "College Biology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # prompt tokens": "{\"description\": \"min=857.86, mean=857.86, max=857.86, sum=1715.72 (2)\", \"tab\": \"General information\", \"score\": \"857.86\"}", - "College Computer Science - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # prompt tokens": "{\"description\": \"min=626.69, mean=626.69, max=626.69, sum=1253.38 (2)\", \"tab\": \"General information\", \"score\": \"626.69\"}", - "College Mathematics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # eval": "{\"description\": \"min=173, mean=173, max=173, sum=346 (2)\", \"tab\": \"General information\", \"score\": \"173.0\"}", - "College Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # prompt tokens": "{\"description\": \"min=513.37, mean=513.37, max=513.37, sum=1026.74 (2)\", \"tab\": \"General information\", \"score\": \"513.3699421965318\"}", - "College Medicine - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # eval": "{\"description\": \"min=102, mean=102, max=102, sum=204 (2)\", \"tab\": \"General information\", \"score\": \"102.0\"}", - "College Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # prompt tokens": "{\"description\": \"min=507.471, mean=507.471, max=507.471, sum=1014.941 (2)\", \"tab\": \"General information\", \"score\": \"507.47058823529414\"}", - "College Physics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"college_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_college_physics\"" - } - } - }, - { - "evaluation_name": "Computer Security", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Computer Security", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.81, - "details": { - "description": "min=0.81, mean=0.81, max=0.81, sum=1.62 (2)", - "tab": "Accuracy", - "Computer Security - Observed inference time (s)": "{\"description\": \"min=1.26, mean=1.26, max=1.26, sum=2.52 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.2601169872283935\"}", - "Computer Security - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Computer Security - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Computer Security - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Computer Security - # prompt tokens": "{\"description\": \"min=380.91, mean=380.91, max=380.91, sum=761.82 (2)\", \"tab\": \"General information\", \"score\": \"380.91\"}", - "Computer Security - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"computer_security\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_computer_security\"" - } - } - }, - { - "evaluation_name": "Econometrics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Econometrics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.737, - "details": { - "description": "min=0.737, mean=0.737, max=0.737, sum=1.474 (2)", - "tab": "Accuracy", - "Econometrics - Observed inference time (s)": "{\"description\": \"min=0.968, mean=0.968, max=0.968, sum=1.936 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9679407843372279\"}", - "Econometrics - # eval": "{\"description\": \"min=114, mean=114, max=114, sum=228 (2)\", \"tab\": \"General information\", \"score\": \"114.0\"}", - "Econometrics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Econometrics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Econometrics - # prompt tokens": "{\"description\": \"min=634.553, mean=634.553, max=634.553, sum=1269.105 (2)\", \"tab\": \"General information\", \"score\": \"634.5526315789474\"}", - "Econometrics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"econometrics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_econometrics\"" - } - } - }, - { - "evaluation_name": "Global Facts", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Global Facts", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.66, - "details": { - "description": "min=0.66, mean=0.66, max=0.66, sum=1.32 (2)", - "tab": "Accuracy", - "Global Facts - Observed inference time (s)": "{\"description\": \"min=1.066, mean=1.066, max=1.066, sum=2.132 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.065871012210846\"}", - "Global Facts - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Global Facts - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Global Facts - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Global Facts - # prompt tokens": "{\"description\": \"min=456.54, mean=456.54, max=456.54, sum=913.08 (2)\", \"tab\": \"General information\", \"score\": \"456.54\"}", - "Global Facts - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"global_facts\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_global_facts\"" - } - } - }, - { - "evaluation_name": "Jurisprudence", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Jurisprudence", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.87, - "details": { - "description": "min=0.87, mean=0.87, max=0.87, sum=1.741 (2)", - "tab": "Accuracy", - "Jurisprudence - Observed inference time (s)": "{\"description\": \"min=1.079, mean=1.079, max=1.079, sum=2.157 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.0785565420433327\"}", - "Jurisprudence - # eval": "{\"description\": \"min=108, mean=108, max=108, sum=216 (2)\", \"tab\": \"General information\", \"score\": \"108.0\"}", - "Jurisprudence - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Jurisprudence - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Jurisprudence - # prompt tokens": "{\"description\": \"min=407.87, mean=407.87, max=407.87, sum=815.741 (2)\", \"tab\": \"General information\", \"score\": \"407.8703703703704\"}", - "Jurisprudence - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"jurisprudence\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_jurisprudence\"" - } - } - }, - { - "evaluation_name": "Philosophy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Philosophy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.846, - "details": { - "description": "min=0.846, mean=0.846, max=0.846, sum=1.691 (2)", - "tab": "Accuracy", - "Philosophy - Observed inference time (s)": "{\"description\": \"min=1.057, mean=1.057, max=1.057, sum=2.114 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.0571237967328626\"}", - "Philosophy - # eval": "{\"description\": \"min=311, mean=311, max=311, sum=622 (2)\", \"tab\": \"General information\", \"score\": \"311.0\"}", - "Philosophy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Philosophy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Philosophy - # prompt tokens": "{\"description\": \"min=340.907, mean=340.907, max=340.907, sum=681.814 (2)\", \"tab\": \"General information\", \"score\": \"340.90675241157555\"}", - "Philosophy - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"philosophy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_philosophy\"" - } - } - }, - { - "evaluation_name": "Professional Psychology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Professional Psychology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.866, - "details": { - "description": "min=0.866, mean=0.866, max=0.866, sum=1.732 (2)", - "tab": "Accuracy", - "Professional Medicine - Observed inference time (s)": "{\"description\": \"min=1.258, mean=1.258, max=1.258, sum=2.516 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.2578288101182213\"}", - "Professional Accounting - Observed inference time (s)": "{\"description\": \"min=1.179, mean=1.179, max=1.179, sum=2.359 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.1793269350173625\"}", - "Professional Law - Observed inference time (s)": "{\"description\": \"min=1.246, mean=1.246, max=1.246, sum=2.491 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.2455504093494716\"}", - "Professional Psychology - Observed inference time (s)": "{\"description\": \"min=1.181, mean=1.181, max=1.181, sum=2.362 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.1811600880403268\"}", - "Professional Medicine - # eval": "{\"description\": \"min=272, mean=272, max=272, sum=544 (2)\", \"tab\": \"General information\", \"score\": \"272.0\"}", - "Professional Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Medicine - # prompt tokens": "{\"description\": \"min=1113.092, mean=1113.092, max=1113.092, sum=2226.184 (2)\", \"tab\": \"General information\", \"score\": \"1113.0919117647059\"}", - "Professional Medicine - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # eval": "{\"description\": \"min=282, mean=282, max=282, sum=564 (2)\", \"tab\": \"General information\", \"score\": \"282.0\"}", - "Professional Accounting - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Accounting - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # prompt tokens": "{\"description\": \"min=755.418, mean=755.418, max=755.418, sum=1510.837 (2)\", \"tab\": \"General information\", \"score\": \"755.418439716312\"}", - "Professional Accounting - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # eval": "{\"description\": \"min=1534, mean=1534, max=1534, sum=3068 (2)\", \"tab\": \"General information\", \"score\": \"1534.0\"}", - "Professional Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # prompt tokens": "{\"description\": \"min=1685.119, mean=1685.119, max=1685.119, sum=3370.239 (2)\", \"tab\": \"General information\", \"score\": \"1685.119295958279\"}", - "Professional Law - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # eval": "{\"description\": \"min=612, mean=612, max=612, sum=1224 (2)\", \"tab\": \"General information\", \"score\": \"612.0\"}", - "Professional Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # prompt tokens": "{\"description\": \"min=594.363, mean=594.363, max=594.363, sum=1188.725 (2)\", \"tab\": \"General information\", \"score\": \"594.3627450980392\"}", - "Professional Psychology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"professional_psychology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_professional_psychology\"" - } - } - }, - { - "evaluation_name": "Us Foreign Policy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Us Foreign Policy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.94, - "details": { - "description": "min=0.94, mean=0.94, max=0.94, sum=1.88 (2)", - "tab": "Accuracy", - "Us Foreign Policy - Observed inference time (s)": "{\"description\": \"min=0.969, mean=0.969, max=0.969, sum=1.938 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.968876302242279\"}", - "Us Foreign Policy - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Us Foreign Policy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Us Foreign Policy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Us Foreign Policy - # prompt tokens": "{\"description\": \"min=438.2, mean=438.2, max=438.2, sum=876.4 (2)\", \"tab\": \"General information\", \"score\": \"438.2\"}", - "Us Foreign Policy - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"us_foreign_policy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_us_foreign_policy\"" - } - } - }, - { - "evaluation_name": "Astronomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Astronomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.914, - "details": { - "description": "min=0.914, mean=0.914, max=0.914, sum=1.829 (2)", - "tab": "Accuracy", - "Astronomy - Observed inference time (s)": "{\"description\": \"min=0.92, mean=0.92, max=0.92, sum=1.84 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9198912256642392\"}", - "Astronomy - # eval": "{\"description\": \"min=152, mean=152, max=152, sum=304 (2)\", \"tab\": \"General information\", \"score\": \"152.0\"}", - "Astronomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Astronomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Astronomy - # prompt tokens": "{\"description\": \"min=602.421, mean=602.421, max=602.421, sum=1204.842 (2)\", \"tab\": \"General information\", \"score\": \"602.421052631579\"}", - "Astronomy - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"astronomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_astronomy\"" - } - } - }, - { - "evaluation_name": "Business Ethics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Business Ethics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8, - "details": { - "description": "min=0.8, mean=0.8, max=0.8, sum=1.6 (2)", - "tab": "Accuracy", - "Business Ethics - Observed inference time (s)": "{\"description\": \"min=1.009, mean=1.009, max=1.009, sum=2.019 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.0093300080299377\"}", - "Business Ethics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Business Ethics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Business Ethics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Business Ethics - # prompt tokens": "{\"description\": \"min=552.87, mean=552.87, max=552.87, sum=1105.74 (2)\", \"tab\": \"General information\", \"score\": \"552.87\"}", - "Business Ethics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"business_ethics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_business_ethics\"" - } - } - }, - { - "evaluation_name": "Clinical Knowledge", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Clinical Knowledge", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.868, - "details": { - "description": "min=0.868, mean=0.868, max=0.868, sum=1.736 (2)", - "tab": "Accuracy", - "Clinical Knowledge - Observed inference time (s)": "{\"description\": \"min=1.079, mean=1.079, max=1.079, sum=2.157 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.0787266893206902\"}", - "Clinical Knowledge - # eval": "{\"description\": \"min=265, mean=265, max=265, sum=530 (2)\", \"tab\": \"General information\", \"score\": \"265.0\"}", - "Clinical Knowledge - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Clinical Knowledge - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Clinical Knowledge - # prompt tokens": "{\"description\": \"min=402.592, mean=402.592, max=402.592, sum=805.185 (2)\", \"tab\": \"General information\", \"score\": \"402.5924528301887\"}", - "Clinical Knowledge - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"clinical_knowledge\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_clinical_knowledge\"" - } - } - }, - { - "evaluation_name": "Conceptual Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Conceptual Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.915, - "details": { - "description": "min=0.915, mean=0.915, max=0.915, sum=1.83 (2)", - "tab": "Accuracy", - "Conceptual Physics - Observed inference time (s)": "{\"description\": \"min=0.962, mean=0.962, max=0.962, sum=1.925 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9624196154005984\"}", - "Conceptual Physics - # eval": "{\"description\": \"min=235, mean=235, max=235, sum=470 (2)\", \"tab\": \"General information\", \"score\": \"235.0\"}", - "Conceptual Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Conceptual Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Conceptual Physics - # prompt tokens": "{\"description\": \"min=309.213, mean=309.213, max=309.213, sum=618.426 (2)\", \"tab\": \"General information\", \"score\": \"309.21276595744683\"}", - "Conceptual Physics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"conceptual_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_conceptual_physics\"" - } - } - }, - { - "evaluation_name": "Electrical Engineering", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Electrical Engineering", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.772, - "details": { - "description": "min=0.772, mean=0.772, max=0.772, sum=1.545 (2)", - "tab": "Accuracy", - "Electrical Engineering - Observed inference time (s)": "{\"description\": \"min=1.272, mean=1.272, max=1.272, sum=2.544 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.271799375270975\"}", - "Electrical Engineering - # eval": "{\"description\": \"min=145, mean=145, max=145, sum=290 (2)\", \"tab\": \"General information\", \"score\": \"145.0\"}", - "Electrical Engineering - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Electrical Engineering - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Electrical Engineering - # prompt tokens": "{\"description\": \"min=474.786, mean=474.786, max=474.786, sum=949.572 (2)\", \"tab\": \"General information\", \"score\": \"474.78620689655173\"}", - "Electrical Engineering - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"electrical_engineering\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_electrical_engineering\"" - } - } - }, - { - "evaluation_name": "Elementary Mathematics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Elementary Mathematics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.884, - "details": { - "description": "min=0.884, mean=0.884, max=0.884, sum=1.767 (2)", - "tab": "Accuracy", - "Elementary Mathematics - Observed inference time (s)": "{\"description\": \"min=1.052, mean=1.052, max=1.052, sum=2.104 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.0518414406549363\"}", - "Elementary Mathematics - # eval": "{\"description\": \"min=378, mean=378, max=378, sum=756 (2)\", \"tab\": \"General information\", \"score\": \"378.0\"}", - "Elementary Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Elementary Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Elementary Mathematics - # prompt tokens": "{\"description\": \"min=597.341, mean=597.341, max=597.341, sum=1194.683 (2)\", \"tab\": \"General information\", \"score\": \"597.3412698412699\"}", - "Elementary Mathematics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"elementary_mathematics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_elementary_mathematics\"" - } - } - }, - { - "evaluation_name": "Formal Logic", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Formal Logic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.643, - "details": { - "description": "min=0.643, mean=0.643, max=0.643, sum=1.286 (2)", - "tab": "Accuracy", - "Formal Logic - Observed inference time (s)": "{\"description\": \"min=1.075, mean=1.075, max=1.075, sum=2.151 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.0754183095598977\"}", - "Formal Logic - # eval": "{\"description\": \"min=126, mean=126, max=126, sum=252 (2)\", \"tab\": \"General information\", \"score\": \"126.0\"}", - "Formal Logic - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Formal Logic - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Formal Logic - # prompt tokens": "{\"description\": \"min=619.563, mean=619.563, max=619.563, sum=1239.127 (2)\", \"tab\": \"General information\", \"score\": \"619.563492063492\"}", - "Formal Logic - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"formal_logic\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_formal_logic\"" - } - } - }, - { - "evaluation_name": "High School World History", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on High School World History", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.924, - "details": { - "description": "min=0.924, mean=0.924, max=0.924, sum=1.848 (2)", - "tab": "Accuracy", - "High School Biology - Observed inference time (s)": "{\"description\": \"min=1.11, mean=1.11, max=1.11, sum=2.22 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.1099017789286951\"}", - "High School Chemistry - Observed inference time (s)": "{\"description\": \"min=1.021, mean=1.021, max=1.021, sum=2.041 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.0206051636211977\"}", - "High School Computer Science - Observed inference time (s)": "{\"description\": \"min=1.112, mean=1.112, max=1.112, sum=2.224 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.1118335294723511\"}", - "High School European History - Observed inference time (s)": "{\"description\": \"min=1.402, mean=1.402, max=1.402, sum=2.803 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.4017024777152323\"}", - "High School Geography - Observed inference time (s)": "{\"description\": \"min=0.959, mean=0.959, max=0.959, sum=1.918 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9591333119556157\"}", - "High School Government And Politics - Observed inference time (s)": "{\"description\": \"min=1.224, mean=1.224, max=1.224, sum=2.448 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.2240539535957298\"}", - "High School Macroeconomics - Observed inference time (s)": "{\"description\": \"min=1.052, mean=1.052, max=1.052, sum=2.105 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.052347583648486\"}", - "High School Mathematics - Observed inference time (s)": "{\"description\": \"min=1.167, mean=1.167, max=1.167, sum=2.335 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.167454132327327\"}", - "High School Microeconomics - Observed inference time (s)": "{\"description\": \"min=0.992, mean=0.992, max=0.992, sum=1.984 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.991771269245308\"}", - "High School Physics - Observed inference time (s)": "{\"description\": \"min=1.275, mean=1.275, max=1.275, sum=2.549 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.2746097031018593\"}", - "High School Psychology - Observed inference time (s)": "{\"description\": \"min=1.143, mean=1.143, max=1.143, sum=2.286 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.1432113459005075\"}", - "High School Statistics - Observed inference time (s)": "{\"description\": \"min=1.417, mean=1.417, max=1.417, sum=2.834 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.417081825159214\"}", - "High School US History - Observed inference time (s)": "{\"description\": \"min=1.309, mean=1.309, max=1.309, sum=2.618 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.3091707919158189\"}", - "High School World History - Observed inference time (s)": "{\"description\": \"min=1.249, mean=1.249, max=1.249, sum=2.498 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.2489153383150382\"}", - "High School Biology - # eval": "{\"description\": \"min=310, mean=310, max=310, sum=620 (2)\", \"tab\": \"General information\", \"score\": \"310.0\"}", - "High School Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Biology - # prompt tokens": "{\"description\": \"min=500.958, mean=500.958, max=500.958, sum=1001.916 (2)\", \"tab\": \"General information\", \"score\": \"500.958064516129\"}", - "High School Biology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # eval": "{\"description\": \"min=203, mean=203, max=203, sum=406 (2)\", \"tab\": \"General information\", \"score\": \"203.0\"}", - "High School Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # prompt tokens": "{\"description\": \"min=513.064, mean=513.064, max=513.064, sum=1026.128 (2)\", \"tab\": \"General information\", \"score\": \"513.064039408867\"}", - "High School Chemistry - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "High School Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # prompt tokens": "{\"description\": \"min=935.13, mean=935.13, max=935.13, sum=1870.26 (2)\", \"tab\": \"General information\", \"score\": \"935.13\"}", - "High School Computer Science - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # eval": "{\"description\": \"min=165, mean=165, max=165, sum=330 (2)\", \"tab\": \"General information\", \"score\": \"165.0\"}", - "High School European History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School European History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # prompt tokens": "{\"description\": \"min=2797.424, mean=2797.424, max=2797.424, sum=5594.848 (2)\", \"tab\": \"General information\", \"score\": \"2797.4242424242425\"}", - "High School European History - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # eval": "{\"description\": \"min=198, mean=198, max=198, sum=396 (2)\", \"tab\": \"General information\", \"score\": \"198.0\"}", - "High School Geography - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Geography - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # prompt tokens": "{\"description\": \"min=394.773, mean=394.773, max=394.773, sum=789.545 (2)\", \"tab\": \"General information\", \"score\": \"394.77272727272725\"}", - "High School Geography - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # eval": "{\"description\": \"min=193, mean=193, max=193, sum=386 (2)\", \"tab\": \"General information\", \"score\": \"193.0\"}", - "High School Government And Politics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Government And Politics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # prompt tokens": "{\"description\": \"min=479.301, mean=479.301, max=479.301, sum=958.601 (2)\", \"tab\": \"General information\", \"score\": \"479.30051813471505\"}", - "High School Government And Politics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # eval": "{\"description\": \"min=390, mean=390, max=390, sum=780 (2)\", \"tab\": \"General information\", \"score\": \"390.0\"}", - "High School Macroeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Macroeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # prompt tokens": "{\"description\": \"min=396.541, mean=396.541, max=396.541, sum=793.082 (2)\", \"tab\": \"General information\", \"score\": \"396.54102564102567\"}", - "High School Macroeconomics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # eval": "{\"description\": \"min=270, mean=270, max=270, sum=540 (2)\", \"tab\": \"General information\", \"score\": \"270.0\"}", - "High School Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # prompt tokens": "{\"description\": \"min=566.822, mean=566.822, max=566.822, sum=1133.644 (2)\", \"tab\": \"General information\", \"score\": \"566.8222222222222\"}", - "High School Mathematics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # eval": "{\"description\": \"min=238, mean=238, max=238, sum=476 (2)\", \"tab\": \"General information\", \"score\": \"238.0\"}", - "High School Microeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Microeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # prompt tokens": "{\"description\": \"min=415.954, mean=415.954, max=415.954, sum=831.908 (2)\", \"tab\": \"General information\", \"score\": \"415.953781512605\"}", - "High School Microeconomics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # eval": "{\"description\": \"min=151, mean=151, max=151, sum=302 (2)\", \"tab\": \"General information\", \"score\": \"151.0\"}", - "High School Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # prompt tokens": "{\"description\": \"min=591.715, mean=591.715, max=591.715, sum=1183.43 (2)\", \"tab\": \"General information\", \"score\": \"591.7152317880794\"}", - "High School Physics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # eval": "{\"description\": \"min=545, mean=545, max=545, sum=1090 (2)\", \"tab\": \"General information\", \"score\": \"545.0\"}", - "High School Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # prompt tokens": "{\"description\": \"min=502.604, mean=502.604, max=502.604, sum=1005.207 (2)\", \"tab\": \"General information\", \"score\": \"502.60366972477067\"}", - "High School Psychology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # eval": "{\"description\": \"min=216, mean=216, max=216, sum=432 (2)\", \"tab\": \"General information\", \"score\": \"216.0\"}", - "High School Statistics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Statistics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # prompt tokens": "{\"description\": \"min=858.931, mean=858.931, max=858.931, sum=1717.861 (2)\", \"tab\": \"General information\", \"score\": \"858.9305555555555\"}", - "High School Statistics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # eval": "{\"description\": \"min=204, mean=204, max=204, sum=408 (2)\", \"tab\": \"General information\", \"score\": \"204.0\"}", - "High School US History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School US History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # prompt tokens": "{\"description\": \"min=2205.583, mean=2205.583, max=2205.583, sum=4411.167 (2)\", \"tab\": \"General information\", \"score\": \"2205.5833333333335\"}", - "High School US History - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # eval": "{\"description\": \"min=237, mean=237, max=237, sum=474 (2)\", \"tab\": \"General information\", \"score\": \"237.0\"}", - "High School World History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School World History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # prompt tokens": "{\"description\": \"min=1426.544, mean=1426.544, max=1426.544, sum=2853.089 (2)\", \"tab\": \"General information\", \"score\": \"1426.5443037974683\"}", - "High School World History - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"high_school_world_history\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_high_school_world_history\"" - } - } - }, - { - "evaluation_name": "Human Sexuality", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Human Sexuality", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.397, - "details": { - "description": "min=0.397, mean=0.397, max=0.397, sum=0.794 (2)", - "tab": "Accuracy", - "Human Aging - Observed inference time (s)": "{\"description\": \"min=1.295, mean=1.295, max=1.295, sum=2.59 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.2951436652196362\"}", - "Human Sexuality - Observed inference time (s)": "{\"description\": \"min=1.699, mean=1.699, max=1.699, sum=3.399 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.6993297884019756\"}", - "Human Aging - # eval": "{\"description\": \"min=223, mean=223, max=223, sum=446 (2)\", \"tab\": \"General information\", \"score\": \"223.0\"}", - "Human Aging - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Aging - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Aging - # prompt tokens": "{\"description\": \"min=321.587, mean=321.587, max=321.587, sum=643.175 (2)\", \"tab\": \"General information\", \"score\": \"321.58744394618833\"}", - "Human Aging - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # eval": "{\"description\": \"min=131, mean=131, max=131, sum=262 (2)\", \"tab\": \"General information\", \"score\": \"131.0\"}", - "Human Sexuality - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Sexuality - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # prompt tokens": "{\"description\": \"min=347.183, mean=347.183, max=347.183, sum=694.366 (2)\", \"tab\": \"General information\", \"score\": \"347.1832061068702\"}", - "Human Sexuality - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"human_sexuality\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_human_sexuality\"" - } - } - }, - { - "evaluation_name": "International Law", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on International Law", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.917, - "details": { - "description": "min=0.917, mean=0.917, max=0.917, sum=1.835 (2)", - "tab": "Accuracy", - "International Law - Observed inference time (s)": "{\"description\": \"min=1.151, mean=1.151, max=1.151, sum=2.303 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.1514279527112472\"}", - "International Law - # eval": "{\"description\": \"min=121, mean=121, max=121, sum=242 (2)\", \"tab\": \"General information\", \"score\": \"121.0\"}", - "International Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "International Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "International Law - # prompt tokens": "{\"description\": \"min=644.165, mean=644.165, max=644.165, sum=1288.331 (2)\", \"tab\": \"General information\", \"score\": \"644.1652892561983\"}", - "International Law - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"international_law\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_international_law\"" - } - } - }, - { - "evaluation_name": "Logical Fallacies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Logical Fallacies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.859, - "details": { - "description": "min=0.859, mean=0.859, max=0.859, sum=1.718 (2)", - "tab": "Accuracy", - "Logical Fallacies - Observed inference time (s)": "{\"description\": \"min=1.422, mean=1.422, max=1.422, sum=2.844 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.4221880026390217\"}", - "Logical Fallacies - # eval": "{\"description\": \"min=163, mean=163, max=163, sum=326 (2)\", \"tab\": \"General information\", \"score\": \"163.0\"}", - "Logical Fallacies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Logical Fallacies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Logical Fallacies - # prompt tokens": "{\"description\": \"min=450.049, mean=450.049, max=450.049, sum=900.098 (2)\", \"tab\": \"General information\", \"score\": \"450.0490797546012\"}", - "Logical Fallacies - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"logical_fallacies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_logical_fallacies\"" - } - } - }, - { - "evaluation_name": "Machine Learning", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Machine Learning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.67, - "details": { - "description": "min=0.67, mean=0.67, max=0.67, sum=1.339 (2)", - "tab": "Accuracy", - "Machine Learning - Observed inference time (s)": "{\"description\": \"min=1.005, mean=1.005, max=1.005, sum=2.011 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.005433154957635\"}", - "Machine Learning - # eval": "{\"description\": \"min=112, mean=112, max=112, sum=224 (2)\", \"tab\": \"General information\", \"score\": \"112.0\"}", - "Machine Learning - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Machine Learning - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Machine Learning - # prompt tokens": "{\"description\": \"min=702.402, mean=702.402, max=702.402, sum=1404.804 (2)\", \"tab\": \"General information\", \"score\": \"702.4017857142857\"}", - "Machine Learning - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"machine_learning\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_machine_learning\"" - } - } - }, - { - "evaluation_name": "Management", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Management", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.874, - "details": { - "description": "min=0.874, mean=0.874, max=0.874, sum=1.748 (2)", - "tab": "Accuracy", - "Management - Observed inference time (s)": "{\"description\": \"min=0.939, mean=0.939, max=0.939, sum=1.879 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9392627234597808\"}", - "Management - # eval": "{\"description\": \"min=103, mean=103, max=103, sum=206 (2)\", \"tab\": \"General information\", \"score\": \"103.0\"}", - "Management - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Management - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Management - # prompt tokens": "{\"description\": \"min=281.301, mean=281.301, max=281.301, sum=562.602 (2)\", \"tab\": \"General information\", \"score\": \"281.3009708737864\"}", - "Management - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"management\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_management\"" - } - } - }, - { - "evaluation_name": "Marketing", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Marketing", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.953, - "details": { - "description": "min=0.953, mean=0.953, max=0.953, sum=1.906 (2)", - "tab": "Accuracy", - "Marketing - Observed inference time (s)": "{\"description\": \"min=1.261, mean=1.261, max=1.261, sum=2.523 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.2613265443051982\"}", - "Marketing - # eval": "{\"description\": \"min=234, mean=234, max=234, sum=468 (2)\", \"tab\": \"General information\", \"score\": \"234.0\"}", - "Marketing - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Marketing - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Marketing - # prompt tokens": "{\"description\": \"min=428.35, mean=428.35, max=428.35, sum=856.701 (2)\", \"tab\": \"General information\", \"score\": \"428.35042735042737\"}", - "Marketing - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"marketing\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_marketing\"" - } - } - }, - { - "evaluation_name": "Medical Genetics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Medical Genetics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.91, - "details": { - "description": "min=0.91, mean=0.91, max=0.91, sum=1.82 (2)", - "tab": "Accuracy", - "Medical Genetics - Observed inference time (s)": "{\"description\": \"min=0.897, mean=0.897, max=0.897, sum=1.795 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8973554396629333\"}", - "Medical Genetics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Medical Genetics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Medical Genetics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Medical Genetics - # prompt tokens": "{\"description\": \"min=338.89, mean=338.89, max=338.89, sum=677.78 (2)\", \"tab\": \"General information\", \"score\": \"338.89\"}", - "Medical Genetics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"medical_genetics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_medical_genetics\"" - } - } - }, - { - "evaluation_name": "Miscellaneous", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Miscellaneous", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.928, - "details": { - "description": "min=0.928, mean=0.928, max=0.928, sum=1.857 (2)", - "tab": "Accuracy", - "Miscellaneous - Observed inference time (s)": "{\"description\": \"min=1.136, mean=1.136, max=1.136, sum=2.272 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.1357932166882707\"}", - "Miscellaneous - # eval": "{\"description\": \"min=783, mean=783, max=783, sum=1566 (2)\", \"tab\": \"General information\", \"score\": \"783.0\"}", - "Miscellaneous - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Miscellaneous - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Miscellaneous - # prompt tokens": "{\"description\": \"min=314.669, mean=314.669, max=314.669, sum=629.338 (2)\", \"tab\": \"General information\", \"score\": \"314.669220945083\"}", - "Miscellaneous - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"miscellaneous\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_miscellaneous\"" - } - } - }, - { - "evaluation_name": "Moral Scenarios", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Moral Scenarios", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.696, - "details": { - "description": "min=0.696, mean=0.696, max=0.696, sum=1.392 (2)", - "tab": "Accuracy", - "Moral Disputes - Observed inference time (s)": "{\"description\": \"min=0.966, mean=0.966, max=0.966, sum=1.933 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9664077420165573\"}", - "Moral Scenarios - Observed inference time (s)": "{\"description\": \"min=1.0, mean=1.0, max=1.0, sum=1.999 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9996972816196952\"}", - "Moral Disputes - # eval": "{\"description\": \"min=346, mean=346, max=346, sum=692 (2)\", \"tab\": \"General information\", \"score\": \"346.0\"}", - "Moral Disputes - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Disputes - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Disputes - # prompt tokens": "{\"description\": \"min=495.003, mean=495.003, max=495.003, sum=990.006 (2)\", \"tab\": \"General information\", \"score\": \"495.0028901734104\"}", - "Moral Disputes - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # eval": "{\"description\": \"min=895, mean=895, max=895, sum=1790 (2)\", \"tab\": \"General information\", \"score\": \"895.0\"}", - "Moral Scenarios - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Scenarios - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # prompt tokens": "{\"description\": \"min=690.542, mean=690.542, max=690.542, sum=1381.084 (2)\", \"tab\": \"General information\", \"score\": \"690.5418994413408\"}", - "Moral Scenarios - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"moral_scenarios\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_moral_scenarios\"" - } - } - }, - { - "evaluation_name": "Nutrition", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Nutrition", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.846, - "details": { - "description": "min=0.846, mean=0.846, max=0.846, sum=1.693 (2)", - "tab": "Accuracy", - "Nutrition - Observed inference time (s)": "{\"description\": \"min=1.042, mean=1.042, max=1.042, sum=2.084 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.04191489858565\"}", - "Nutrition - # eval": "{\"description\": \"min=306, mean=306, max=306, sum=612 (2)\", \"tab\": \"General information\", \"score\": \"306.0\"}", - "Nutrition - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Nutrition - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Nutrition - # prompt tokens": "{\"description\": \"min=585.48, mean=585.48, max=585.48, sum=1170.961 (2)\", \"tab\": \"General information\", \"score\": \"585.4803921568628\"}", - "Nutrition - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"nutrition\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_nutrition\"" - } - } - }, - { - "evaluation_name": "Prehistory", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Prehistory", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.886, - "details": { - "description": "min=0.886, mean=0.886, max=0.886, sum=1.772 (2)", - "tab": "Accuracy", - "Prehistory - Observed inference time (s)": "{\"description\": \"min=0.888, mean=0.888, max=0.888, sum=1.775 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8876422820267854\"}", - "Prehistory - # eval": "{\"description\": \"min=324, mean=324, max=324, sum=648 (2)\", \"tab\": \"General information\", \"score\": \"324.0\"}", - "Prehistory - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Prehistory - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Prehistory - # prompt tokens": "{\"description\": \"min=540.198, mean=540.198, max=540.198, sum=1080.395 (2)\", \"tab\": \"General information\", \"score\": \"540.1975308641976\"}", - "Prehistory - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"prehistory\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_prehistory\"" - } - } - }, - { - "evaluation_name": "Public Relations", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Public Relations", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.755, - "details": { - "description": "min=0.755, mean=0.755, max=0.755, sum=1.509 (2)", - "tab": "Accuracy", - "Public Relations - Observed inference time (s)": "{\"description\": \"min=0.992, mean=0.992, max=0.992, sum=1.984 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9922328862276945\"}", - "Public Relations - # eval": "{\"description\": \"min=110, mean=110, max=110, sum=220 (2)\", \"tab\": \"General information\", \"score\": \"110.0\"}", - "Public Relations - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Public Relations - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Public Relations - # prompt tokens": "{\"description\": \"min=426.655, mean=426.655, max=426.655, sum=853.309 (2)\", \"tab\": \"General information\", \"score\": \"426.6545454545454\"}", - "Public Relations - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"public_relations\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_public_relations\"" - } - } - }, - { - "evaluation_name": "Security Studies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Security Studies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.849, - "details": { - "description": "min=0.849, mean=0.849, max=0.849, sum=1.698 (2)", - "tab": "Accuracy", - "Security Studies - Observed inference time (s)": "{\"description\": \"min=1.117, mean=1.117, max=1.117, sum=2.234 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.116919010512683\"}", - "Security Studies - # eval": "{\"description\": \"min=245, mean=245, max=245, sum=490 (2)\", \"tab\": \"General information\", \"score\": \"245.0\"}", - "Security Studies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Security Studies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Security Studies - # prompt tokens": "{\"description\": \"min=1193.869, mean=1193.869, max=1193.869, sum=2387.739 (2)\", \"tab\": \"General information\", \"score\": \"1193.869387755102\"}", - "Security Studies - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"security_studies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_security_studies\"" - } - } - }, - { - "evaluation_name": "Sociology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Sociology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.925, - "details": { - "description": "min=0.925, mean=0.925, max=0.925, sum=1.851 (2)", - "tab": "Accuracy", - "Sociology - Observed inference time (s)": "{\"description\": \"min=1.296, mean=1.296, max=1.296, sum=2.592 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.29619625195935\"}", - "Sociology - # eval": "{\"description\": \"min=201, mean=201, max=201, sum=402 (2)\", \"tab\": \"General information\", \"score\": \"201.0\"}", - "Sociology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Sociology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Sociology - # prompt tokens": "{\"description\": \"min=456.274, mean=456.274, max=456.274, sum=912.547 (2)\", \"tab\": \"General information\", \"score\": \"456.27363184079604\"}", - "Sociology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"sociology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_sociology\"" - } - } - }, - { - "evaluation_name": "Virology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Virology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.584, - "details": { - "description": "min=0.584, mean=0.584, max=0.584, sum=1.169 (2)", - "tab": "Accuracy", - "Virology - Observed inference time (s)": "{\"description\": \"min=0.877, mean=0.877, max=0.877, sum=1.754 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8771147684878614\"}", - "Virology - # eval": "{\"description\": \"min=166, mean=166, max=166, sum=332 (2)\", \"tab\": \"General information\", \"score\": \"166.0\"}", - "Virology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Virology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Virology - # prompt tokens": "{\"description\": \"min=336.753, mean=336.753, max=336.753, sum=673.506 (2)\", \"tab\": \"General information\", \"score\": \"336.7530120481928\"}", - "Virology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"virology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_virology\"" - } - } - }, - { - "evaluation_name": "World Religions", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on World Religions", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.877, - "details": { - "description": "min=0.877, mean=0.877, max=0.877, sum=1.754 (2)", - "tab": "Accuracy", - "World Religions - Observed inference time (s)": "{\"description\": \"min=1.225, mean=1.225, max=1.225, sum=2.451 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.2254026856338769\"}", - "World Religions - # eval": "{\"description\": \"min=171, mean=171, max=171, sum=342 (2)\", \"tab\": \"General information\", \"score\": \"171.0\"}", - "World Religions - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "World Religions - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "World Religions - # prompt tokens": "{\"description\": \"min=268.164, mean=268.164, max=268.164, sum=536.327 (2)\", \"tab\": \"General information\", \"score\": \"268.1637426900585\"}", - "World Religions - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"world_religions\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_world_religions\"" - } - } - }, - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.118, - "details": { - "description": "", - "tab": "Efficiency" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/google_gemini-2-5-flash-fc.json b/data/models/google_gemini-2-5-flash-fc.json deleted file mode 100644 index e41385db6ce3d0eb951c3eeb34309c841b19dd53..0000000000000000000000000000000000000000 --- a/data/models/google_gemini-2-5-flash-fc.json +++ /dev/null @@ -1,851 +0,0 @@ -{ - "model_info": { - "name": "Gemini-2.5-Flash (FC)", - "id": "google/gemini-2-5-flash-fc", - "developer": "Google", - "additional_details": { - "raw_model_name": "Gemini-2.5-Flash (FC)", - "organization": "Google", - "license": "Proprietary", - "mode": "FC", - "model_link": "https://deepmind.google/technologies/gemini/flash/" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/google/gemini-2-5-flash-fc/1775236112.3733299", - "retrieved_timestamp": "1775236112.3733299", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 15.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 56.24 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 26.36 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 2.99 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 9.22 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 5.62 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 84.96 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 74.33 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 92.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 94.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 79.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 74.39 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 85.27 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 71.7 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 81.25 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 70.83 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 36.25 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 41.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 36.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 32.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 35.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 59.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 59.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 59.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 41.29 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 19.35 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 50.32 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 54.19 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 75.0 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 93.67 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/google_gemini-2-5-flash-lite-fc.json b/data/models/google_gemini-2-5-flash-lite-fc.json deleted file mode 100644 index 3c1ec27bc06d3683f4c57b1484496ace7ccb6fa8..0000000000000000000000000000000000000000 --- a/data/models/google_gemini-2-5-flash-lite-fc.json +++ /dev/null @@ -1,851 +0,0 @@ -{ - "model_info": { - "name": "Gemini-2.5-Flash-Lite (FC)", - "id": "google/gemini-2-5-flash-lite-fc", - "developer": "Google", - "additional_details": { - "raw_model_name": "Gemini-2.5-Flash-Lite (FC)", - "organization": "Google", - "license": "Proprietary", - "mode": "FC", - "model_link": "https://deepmind.google/technologies/gemini/flash-lite/" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/google/gemini-2-5-flash-lite-fc/1775236112.392892", - "retrieved_timestamp": "1775236112.392892", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 52.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 36.87 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 7.55 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 1.18 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 8.06 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 1.67 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 86.6 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 70.92 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 90.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 93.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 92.0 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 65.8 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 73.26 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 63.82 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 75.0 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 66.67 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 13.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 20.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 1.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 15.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 17.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 21.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 26.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 16.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 20.65 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 3.87 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 6.45 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 51.61 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 43.75 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 92.5 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/google_gemini-2-5-flash-lite-prompt.json b/data/models/google_gemini-2-5-flash-lite-prompt.json deleted file mode 100644 index bd4d318d950bc06cc0f7566a9ed6e8f272773a45..0000000000000000000000000000000000000000 --- a/data/models/google_gemini-2-5-flash-lite-prompt.json +++ /dev/null @@ -1,905 +0,0 @@ -{ - "model_info": { - "name": "Gemini-2.5-Flash-Lite (Prompt)", - "id": "google/gemini-2-5-flash-lite-prompt", - "developer": "Google", - "additional_details": { - "raw_model_name": "Gemini-2.5-Flash-Lite (Prompt)", - "organization": "Google", - "license": "Proprietary", - "mode": "Prompt", - "model_link": "https://deepmind.google/technologies/gemini/flash-lite/" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/google/gemini-2-5-flash-lite-prompt/1775236112.40407", - "retrieved_timestamp": "1775236112.40407", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 73.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 28.03 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 7.05 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 1.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 4.75 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 1.4 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 83.9 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 70.08 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 86.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 90.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 89.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 54.85 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 67.05 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 51.66 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 75.0 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 50.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 7.63 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 10.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 5.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 6.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 9.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 12.69 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 1.94 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 6.45 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 29.68 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 50.0 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 93.33 - } - }, - { - "evaluation_result_id": "bfcl.format_sensitivity.max_delta::max_delta", - "evaluation_name": "bfcl.format_sensitivity.max_delta", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.format_sensitivity.max_delta", - "metric_name": "Format sensitivity max delta", - "metric_kind": "difference", - "metric_unit": "percentage_points", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Format Sensitivity Max Delta" - } - }, - "score_details": { - "score": 25.5 - } - }, - { - "evaluation_result_id": "bfcl.format_sensitivity.stddev::stddev", - "evaluation_name": "bfcl.format_sensitivity.stddev", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.format_sensitivity.stddev", - "metric_name": "Format sensitivity standard deviation", - "metric_kind": "difference", - "metric_unit": "percentage_points", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Format Sensitivity Standard Deviation" - } - }, - "score_details": { - "score": 6.68 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/google_gemini-2-5-flash-prompt.json b/data/models/google_gemini-2-5-flash-prompt.json deleted file mode 100644 index 875a5263b96c9471483aa4ef0e1858af0917f0de..0000000000000000000000000000000000000000 --- a/data/models/google_gemini-2-5-flash-prompt.json +++ /dev/null @@ -1,905 +0,0 @@ -{ - "model_info": { - "name": "Gemini-2.5-Flash (Prompt)", - "id": "google/gemini-2-5-flash-prompt", - "developer": "Google", - "additional_details": { - "raw_model_name": "Gemini-2.5-Flash (Prompt)", - "organization": "Google", - "license": "Proprietary", - "mode": "Prompt", - "model_link": "https://deepmind.google/technologies/gemini/flash/" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/google/gemini-2-5-flash-prompt/1775236112.378891", - "retrieved_timestamp": "1775236112.378891", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 26.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 50.9 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 33.45 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 3.18 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 4.44 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 6.09 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 88.08 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 77.33 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 91.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 96.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 87.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 78.16 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 87.21 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 75.97 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 81.25 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 75.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 16.75 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 14.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 16.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 17.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 18.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 62.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 60.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 64.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 38.71 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 13.55 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 47.1 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 55.48 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 62.5 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 91.09 - } - }, - { - "evaluation_result_id": "bfcl.format_sensitivity.max_delta::max_delta", - "evaluation_name": "bfcl.format_sensitivity.max_delta", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.format_sensitivity.max_delta", - "metric_name": "Format sensitivity max delta", - "metric_kind": "difference", - "metric_unit": "percentage_points", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Format Sensitivity Max Delta" - } - }, - "score_details": { - "score": 9.0 - } - }, - { - "evaluation_result_id": "bfcl.format_sensitivity.stddev::stddev", - "evaluation_name": "bfcl.format_sensitivity.stddev", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.format_sensitivity.stddev", - "metric_name": "Format sensitivity standard deviation", - "metric_kind": "difference", - "metric_unit": "percentage_points", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Format Sensitivity Standard Deviation" - } - }, - "score_details": { - "score": 2.45 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/google_gemini-2.0-flash-001.json b/data/models/google_gemini-2.0-flash-001.json deleted file mode 100644 index 92dbda7481b66d0dccd15d13102dd1de3041036b..0000000000000000000000000000000000000000 --- a/data/models/google_gemini-2.0-flash-001.json +++ /dev/null @@ -1,234 +0,0 @@ -{ - "model_info": { - "name": "Gemini 2.0 Flash", - "id": "google/gemini-2.0-flash-001", - "developer": "Google", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_capabilities/google_gemini-2.0-flash-001/1774096304.5056698", - "retrieved_timestamp": "1774096304.5056698", - "source_metadata": { - "source_name": "helm_capabilities", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_capabilities", - "evaluation_results": [ - { - "evaluation_name": "Mean score", - "source_data": { - "dataset_name": "helm_capabilities", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "The mean of the scores from all columns.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.679, - "details": { - "description": "", - "tab": "Accuracy", - "Mean score - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"5.700146694170831\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU-Pro", - "source_data": { - "dataset_name": "MMLU-Pro", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on MMLU-Pro", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.737, - "details": { - "description": "min=0.737, mean=0.737, max=0.737, sum=0.737 (1)", - "tab": "Accuracy", - "MMLU-Pro - Observed inference time (s)": "{\"description\": \"min=3.221, mean=3.221, max=3.221, sum=3.221 (1)\", \"tab\": \"Efficiency\", \"score\": \"3.221250217437744\"}", - "MMLU-Pro - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "MMLU-Pro - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - # prompt tokens": "{\"description\": \"min=242.673, mean=242.673, max=242.673, sum=242.673 (1)\", \"tab\": \"General information\", \"score\": \"242.673\"}", - "MMLU-Pro - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"all\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.556, - "details": { - "description": "min=0.556, mean=0.556, max=0.556, sum=0.556 (1)", - "tab": "Accuracy", - "GPQA - Observed inference time (s)": "{\"description\": \"min=4.919, mean=4.919, max=4.919, sum=4.919 (1)\", \"tab\": \"Efficiency\", \"score\": \"4.919003446005919\"}", - "GPQA - # eval": "{\"description\": \"min=446, mean=446, max=446, sum=446 (1)\", \"tab\": \"General information\", \"score\": \"446.0\"}", - "GPQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - # prompt tokens": "{\"description\": \"min=252.735, mean=252.735, max=252.735, sum=252.735 (1)\", \"tab\": \"General information\", \"score\": \"252.7354260089686\"}", - "GPQA - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"gpqa_main\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "IFEval Strict Acc on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.841, - "details": { - "description": "min=0.841, mean=0.841, max=0.841, sum=0.841 (1)", - "tab": "Accuracy", - "IFEval - Observed inference time (s)": "{\"description\": \"min=3.723, mean=3.723, max=3.723, sum=3.723 (1)\", \"tab\": \"Efficiency\", \"score\": \"3.7232056717334965\"}", - "IFEval - # eval": "{\"description\": \"min=541, mean=541, max=541, sum=541 (1)\", \"tab\": \"General information\", \"score\": \"541.0\"}", - "IFEval - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - # prompt tokens": "{\"description\": \"min=47.331, mean=47.331, max=47.331, sum=47.331 (1)\", \"tab\": \"General information\", \"score\": \"47.33086876155268\"}", - "IFEval - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WildBench", - "source_data": { - "dataset_name": "WildBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "WB Score on WildBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8, - "details": { - "description": "min=0.8, mean=0.8, max=0.8, sum=0.8 (1)", - "tab": "Accuracy", - "WildBench - Observed inference time (s)": "{\"description\": \"min=9.27, mean=9.27, max=9.27, sum=9.27 (1)\", \"tab\": \"Efficiency\", \"score\": \"9.270071518985407\"}", - "WildBench - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "WildBench - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # prompt tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"v2\"" - } - } - }, - { - "evaluation_name": "Omni-MATH", - "source_data": { - "dataset_name": "Omni-MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Acc on Omni-MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.459, - "details": { - "description": "min=0.459, mean=0.459, max=0.459, sum=0.459 (1)", - "tab": "Accuracy", - "Omni-MATH - Observed inference time (s)": "{\"description\": \"min=7.367, mean=7.367, max=7.367, sum=7.367 (1)\", \"tab\": \"Efficiency\", \"score\": \"7.367202616691589\"}", - "Omni-MATH - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "Omni-MATH - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - # prompt tokens": "{\"description\": \"min=111.956, mean=111.956, max=111.956, sum=111.956 (1)\", \"tab\": \"General information\", \"score\": \"111.956\"}", - "Omni-MATH - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/google_gemini-2.0-flash-exp.json b/data/models/google_gemini-2.0-flash-exp.json deleted file mode 100644 index 28bded6a774bae3132a9d2cadc49050ba174ee33..0000000000000000000000000000000000000000 --- a/data/models/google_gemini-2.0-flash-exp.json +++ /dev/null @@ -1,1904 +0,0 @@ -{ - "model_info": { - "name": "Gemini 2.0 Flash Experimental", - "id": "google/gemini-2.0-flash-exp", - "developer": "Google", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_lite/google_gemini-2.0-flash-exp/1774096306.427425", - "retrieved_timestamp": "1774096306.427425", - "source_metadata": { - "source_name": "helm_lite", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_lite", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_lite", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.813, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.7398626716604245\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.783, - "details": { - "description": "min=0.783, mean=0.783, max=0.783, sum=0.783 (1)", - "tab": "Accuracy", - "NarrativeQA - Observed inference time (s)": "{\"description\": \"min=0.512, mean=0.512, max=0.512, sum=0.512 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.5123653337359428\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=3437.994, mean=3437.994, max=3437.994, sum=3437.994 (1)\", \"tab\": \"General information\", \"score\": \"3437.994366197183\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (closed-book)", - "source_data": { - "dataset_name": "NaturalQuestions (closed-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (closed-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.443, - "details": { - "description": "min=0.443, mean=0.443, max=0.443, sum=0.443 (1)", - "tab": "Accuracy", - "NaturalQuestions (open-book) - Observed inference time (s)": "{\"description\": \"min=0.462, mean=0.462, max=0.462, sum=0.462 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.4622749860286713\"}", - "NaturalQuestions (closed-book) - Observed inference time (s)": "{\"description\": \"min=0.417, mean=0.417, max=0.417, sum=0.417 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.4170585689544678\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1968.347, mean=1968.347, max=1968.347, sum=1968.347 (1)\", \"tab\": \"General information\", \"score\": \"1968.347\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=143.995, mean=143.995, max=143.995, sum=143.995 (1)\", \"tab\": \"General information\", \"score\": \"143.995\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "mode": "\"closedbook\"" - } - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.946, - "details": { - "description": "min=0.946, mean=0.946, max=0.946, sum=0.946 (1)", - "tab": "Accuracy", - "OpenbookQA - Observed inference time (s)": "{\"description\": \"min=0.391, mean=0.391, max=0.391, sum=0.391 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.39134009742736814\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=248.508, mean=248.508, max=248.508, sum=248.508 (1)\", \"tab\": \"General information\", \"score\": \"248.508\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "dataset": "\"openbookqa\"", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.717, - "details": { - "description": "min=0.56, mean=0.717, max=0.83, sum=3.583 (5)", - "tab": "Accuracy", - "MMLU - Observed inference time (s)": "{\"description\": \"min=0.405, mean=0.409, max=0.414, sum=2.043 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.4086059420652557\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=380.91, mean=481.531, max=634.553, sum=2407.653 (5)\", \"tab\": \"General information\", \"score\": \"481.5305263157895\"}", - "MMLU - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"college_chemistry\", \"computer_security\", \"econometrics\", \"us_foreign_policy\"]", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MATH", - "source_data": { - "dataset_name": "MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Equivalent (CoT) on MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.901, - "details": { - "description": "min=0.788, mean=0.901, max=0.985, sum=6.309 (7)", - "tab": "Accuracy", - "MATH - Observed inference time (s)": "{\"description\": \"min=1.049, mean=1.506, max=2.041, sum=10.543 (7)\", \"tab\": \"Efficiency\", \"score\": \"1.5061902186836522\"}", - "MATH - # eval": "{\"description\": \"min=30, mean=62.429, max=135, sum=437 (7)\", \"tab\": \"General information\", \"score\": \"62.42857142857143\"}", - "MATH - # train": "{\"description\": \"min=8, mean=8, max=8, sum=56 (7)\", \"tab\": \"General information\", \"score\": \"8.0\"}", - "MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MATH - # prompt tokens": "{\"description\": \"min=938.215, mean=1355.506, max=2348.712, sum=9488.545 (7)\", \"tab\": \"General information\", \"score\": \"1355.5064552904823\"}", - "MATH - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"algebra\", \"counting_and_probability\", \"geometry\", \"intermediate_algebra\", \"number_theory\", \"prealgebra\", \"precalculus\"]", - "level": "\"1\"", - "use_official_examples": "\"False\"", - "use_chain_of_thought": "\"True\"" - } - } - }, - { - "evaluation_name": "GSM8K", - "source_data": { - "dataset_name": "GSM8K", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on GSM8K", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.946, - "details": { - "description": "min=0.946, mean=0.946, max=0.946, sum=0.946 (1)", - "tab": "Accuracy", - "GSM8K - Observed inference time (s)": "{\"description\": \"min=1.437, mean=1.437, max=1.437, sum=1.437 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.4374724824428557\"}", - "GSM8K - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "GSM8K - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "GSM8K - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GSM8K - # prompt tokens": "{\"description\": \"min=1151.885, mean=1151.885, max=1151.885, sum=1151.885 (1)\", \"tab\": \"General information\", \"score\": \"1151.885\"}", - "GSM8K - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "stop": "\"none\"" - } - } - }, - { - "evaluation_name": "LegalBench", - "source_data": { - "dataset_name": "LegalBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on LegalBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.674, - "details": { - "description": "min=0.237, mean=0.674, max=0.989, sum=3.371 (5)", - "tab": "Accuracy", - "LegalBench - Observed inference time (s)": "{\"description\": \"min=0.454, mean=0.547, max=0.655, sum=2.737 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.5473698430089784\"}", - "LegalBench - # eval": "{\"description\": \"min=95, mean=409.4, max=1000, sum=2047 (5)\", \"tab\": \"General information\", \"score\": \"409.4\"}", - "LegalBench - # train": "{\"description\": \"min=4, mean=4.8, max=5, sum=24 (5)\", \"tab\": \"General information\", \"score\": \"4.8\"}", - "LegalBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "LegalBench - # prompt tokens": "{\"description\": \"min=216.916, mean=1559.239, max=6418.569, sum=7796.193 (5)\", \"tab\": \"General information\", \"score\": \"1559.2386051001386\"}", - "LegalBench - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"abercrombie\", \"corporate_lobbying\", \"function_of_decision_section\", \"international_citizenship_questions\", \"proa\"]", - "stop": "\"none\"" - } - } - }, - { - "evaluation_name": "MedQA", - "source_data": { - "dataset_name": "MedQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MedQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.73, - "details": { - "description": "min=0.73, mean=0.73, max=0.73, sum=0.73 (1)", - "tab": "Accuracy", - "MedQA - Observed inference time (s)": "{\"description\": \"min=0.407, mean=0.407, max=0.407, sum=0.407 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.4071517047540805\"}", - "MedQA - # eval": "{\"description\": \"min=503, mean=503, max=503, sum=503 (1)\", \"tab\": \"General information\", \"score\": \"503.0\"}", - "MedQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MedQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MedQA - # prompt tokens": "{\"description\": \"min=1029.481, mean=1029.481, max=1029.481, sum=1029.481 (1)\", \"tab\": \"General information\", \"score\": \"1029.4811133200794\"}", - "MedQA - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WMT 2014", - "source_data": { - "dataset_name": "WMT 2014", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "BLEU-4 on WMT 2014", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.212, - "details": { - "description": "min=0.154, mean=0.212, max=0.242, sum=1.059 (5)", - "tab": "Accuracy", - "WMT 2014 - Observed inference time (s)": "{\"description\": \"min=0.639, mean=0.725, max=0.883, sum=3.624 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.7247073432282998\"}", - "WMT 2014 - # eval": "{\"description\": \"min=503, mean=568.8, max=832, sum=2844 (5)\", \"tab\": \"General information\", \"score\": \"568.8\"}", - "WMT 2014 - # train": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "WMT 2014 - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WMT 2014 - # prompt tokens": "{\"description\": \"min=80.732, mean=110.97, max=137.366, sum=554.851 (5)\", \"tab\": \"General information\", \"score\": \"110.97025108961614\"}", - "WMT 2014 - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "language_pair": "[\"cs-en\", \"de-en\", \"fr-en\", \"hi-en\", \"ru-en\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_mmlu/google_gemini-2.0-flash-exp/1774096312.00548", - "retrieved_timestamp": "1774096312.00548", - "source_metadata": { - "source_name": "helm_mmlu", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_mmlu", - "evaluation_results": [ - { - "evaluation_name": "MMLU All Subjects", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU All Subjects", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.797, - "details": { - "description": "min=0.554, mean=0.797, max=0.969, sum=90.902 (114)", - "tab": "Accuracy", - "MMLU All Subjects - Observed inference time (s)": "{\"description\": \"min=0.379, mean=0.422, max=0.926, sum=48.097 (114)\", \"tab\": \"Efficiency\", \"score\": \"0.4219020959728089\"}", - "MMLU All Subjects - # eval": "{\"description\": \"min=100, mean=246.351, max=1534, sum=28084 (114)\", \"tab\": \"General information\", \"score\": \"246.35087719298247\"}", - "MMLU All Subjects - # train": "{\"description\": \"min=5, mean=5, max=5, sum=570 (114)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU All Subjects - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU All Subjects - # prompt tokens": "{\"description\": \"min=268.164, mean=632.617, max=2797.424, sum=72118.345 (114)\", \"tab\": \"General information\", \"score\": \"632.6170571214202\"}", - "MMLU All Subjects - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_name": "Abstract Algebra", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Abstract Algebra", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.72, - "details": { - "description": "min=0.72, mean=0.72, max=0.72, sum=1.44 (2)", - "tab": "Accuracy", - "Abstract Algebra - Observed inference time (s)": "{\"description\": \"min=0.408, mean=0.408, max=0.408, sum=0.816 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4077691292762756\"}", - "Abstract Algebra - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Abstract Algebra - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Abstract Algebra - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Abstract Algebra - # prompt tokens": "{\"description\": \"min=383.97, mean=383.97, max=383.97, sum=767.94 (2)\", \"tab\": \"General information\", \"score\": \"383.97\"}", - "Abstract Algebra - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"abstract_algebra\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_abstract_algebra\"" - } - } - }, - { - "evaluation_name": "Anatomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Anatomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.807, - "details": { - "description": "min=0.807, mean=0.807, max=0.807, sum=1.615 (2)", - "tab": "Accuracy", - "Anatomy - Observed inference time (s)": "{\"description\": \"min=0.926, mean=0.926, max=0.926, sum=1.852 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9258230227011222\"}", - "Anatomy - # eval": "{\"description\": \"min=135, mean=135, max=135, sum=270 (2)\", \"tab\": \"General information\", \"score\": \"135.0\"}", - "Anatomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Anatomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Anatomy - # prompt tokens": "{\"description\": \"min=344.356, mean=344.356, max=344.356, sum=688.711 (2)\", \"tab\": \"General information\", \"score\": \"344.35555555555555\"}", - "Anatomy - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"anatomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_anatomy\"" - } - } - }, - { - "evaluation_name": "College Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on College Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.696, - "details": { - "description": "min=0.696, mean=0.696, max=0.696, sum=1.392 (2)", - "tab": "Accuracy", - "College Chemistry - Observed inference time (s)": "{\"description\": \"min=0.405, mean=0.405, max=0.405, sum=0.809 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4045387363433838\"}", - "College Biology - Observed inference time (s)": "{\"description\": \"min=0.47, mean=0.47, max=0.47, sum=0.941 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4703653355439504\"}", - "College Computer Science - Observed inference time (s)": "{\"description\": \"min=0.436, mean=0.436, max=0.436, sum=0.872 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4358289122581482\"}", - "College Mathematics - Observed inference time (s)": "{\"description\": \"min=0.413, mean=0.413, max=0.413, sum=0.827 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.413386971950531\"}", - "College Medicine - Observed inference time (s)": "{\"description\": \"min=0.426, mean=0.426, max=0.426, sum=0.852 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4259330606184943\"}", - "College Physics - Observed inference time (s)": "{\"description\": \"min=0.456, mean=0.456, max=0.456, sum=0.912 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4557511432498109\"}", - "College Chemistry - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Chemistry - # prompt tokens": "{\"description\": \"min=570.02, mean=570.02, max=570.02, sum=1140.04 (2)\", \"tab\": \"General information\", \"score\": \"570.02\"}", - "College Chemistry - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # eval": "{\"description\": \"min=144, mean=144, max=144, sum=288 (2)\", \"tab\": \"General information\", \"score\": \"144.0\"}", - "College Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # prompt tokens": "{\"description\": \"min=482.799, mean=482.799, max=482.799, sum=965.597 (2)\", \"tab\": \"General information\", \"score\": \"482.7986111111111\"}", - "College Biology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # prompt tokens": "{\"description\": \"min=857.86, mean=857.86, max=857.86, sum=1715.72 (2)\", \"tab\": \"General information\", \"score\": \"857.86\"}", - "College Computer Science - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # prompt tokens": "{\"description\": \"min=626.69, mean=626.69, max=626.69, sum=1253.38 (2)\", \"tab\": \"General information\", \"score\": \"626.69\"}", - "College Mathematics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # eval": "{\"description\": \"min=173, mean=173, max=173, sum=346 (2)\", \"tab\": \"General information\", \"score\": \"173.0\"}", - "College Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # prompt tokens": "{\"description\": \"min=513.37, mean=513.37, max=513.37, sum=1026.74 (2)\", \"tab\": \"General information\", \"score\": \"513.3699421965318\"}", - "College Medicine - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # eval": "{\"description\": \"min=102, mean=102, max=102, sum=204 (2)\", \"tab\": \"General information\", \"score\": \"102.0\"}", - "College Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # prompt tokens": "{\"description\": \"min=507.471, mean=507.471, max=507.471, sum=1014.941 (2)\", \"tab\": \"General information\", \"score\": \"507.47058823529414\"}", - "College Physics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"college_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_college_physics\"" - } - } - }, - { - "evaluation_name": "Computer Security", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Computer Security", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.83, - "details": { - "description": "min=0.83, mean=0.83, max=0.83, sum=1.66 (2)", - "tab": "Accuracy", - "Computer Security - Observed inference time (s)": "{\"description\": \"min=0.407, mean=0.407, max=0.407, sum=0.813 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4065685248374939\"}", - "Computer Security - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Computer Security - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Computer Security - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Computer Security - # prompt tokens": "{\"description\": \"min=380.91, mean=380.91, max=380.91, sum=761.82 (2)\", \"tab\": \"General information\", \"score\": \"380.91\"}", - "Computer Security - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"computer_security\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_computer_security\"" - } - } - }, - { - "evaluation_name": "Econometrics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Econometrics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.693, - "details": { - "description": "min=0.693, mean=0.693, max=0.693, sum=1.386 (2)", - "tab": "Accuracy", - "Econometrics - Observed inference time (s)": "{\"description\": \"min=0.41, mean=0.41, max=0.41, sum=0.819 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4097107544279935\"}", - "Econometrics - # eval": "{\"description\": \"min=114, mean=114, max=114, sum=228 (2)\", \"tab\": \"General information\", \"score\": \"114.0\"}", - "Econometrics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Econometrics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Econometrics - # prompt tokens": "{\"description\": \"min=634.553, mean=634.553, max=634.553, sum=1269.105 (2)\", \"tab\": \"General information\", \"score\": \"634.5526315789474\"}", - "Econometrics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"econometrics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_econometrics\"" - } - } - }, - { - "evaluation_name": "Global Facts", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Global Facts", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.66, - "details": { - "description": "min=0.66, mean=0.66, max=0.66, sum=1.32 (2)", - "tab": "Accuracy", - "Global Facts - Observed inference time (s)": "{\"description\": \"min=0.415, mean=0.415, max=0.415, sum=0.83 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4148475766181946\"}", - "Global Facts - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Global Facts - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Global Facts - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Global Facts - # prompt tokens": "{\"description\": \"min=456.54, mean=456.54, max=456.54, sum=913.08 (2)\", \"tab\": \"General information\", \"score\": \"456.54\"}", - "Global Facts - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"global_facts\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_global_facts\"" - } - } - }, - { - "evaluation_name": "Jurisprudence", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Jurisprudence", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.898, - "details": { - "description": "min=0.898, mean=0.898, max=0.898, sum=1.796 (2)", - "tab": "Accuracy", - "Jurisprudence - Observed inference time (s)": "{\"description\": \"min=0.442, mean=0.442, max=0.442, sum=0.884 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4418119721942478\"}", - "Jurisprudence - # eval": "{\"description\": \"min=108, mean=108, max=108, sum=216 (2)\", \"tab\": \"General information\", \"score\": \"108.0\"}", - "Jurisprudence - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Jurisprudence - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Jurisprudence - # prompt tokens": "{\"description\": \"min=407.87, mean=407.87, max=407.87, sum=815.741 (2)\", \"tab\": \"General information\", \"score\": \"407.8703703703704\"}", - "Jurisprudence - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"jurisprudence\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_jurisprudence\"" - } - } - }, - { - "evaluation_name": "Philosophy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Philosophy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.887, - "details": { - "description": "min=0.887, mean=0.887, max=0.887, sum=1.775 (2)", - "tab": "Accuracy", - "Philosophy - Observed inference time (s)": "{\"description\": \"min=0.409, mean=0.409, max=0.409, sum=0.817 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.40853408831875426\"}", - "Philosophy - # eval": "{\"description\": \"min=311, mean=311, max=311, sum=622 (2)\", \"tab\": \"General information\", \"score\": \"311.0\"}", - "Philosophy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Philosophy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Philosophy - # prompt tokens": "{\"description\": \"min=340.907, mean=340.907, max=340.907, sum=681.814 (2)\", \"tab\": \"General information\", \"score\": \"340.90675241157555\"}", - "Philosophy - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"philosophy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_philosophy\"" - } - } - }, - { - "evaluation_name": "Professional Psychology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Professional Psychology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.876, - "details": { - "description": "min=0.876, mean=0.876, max=0.876, sum=1.752 (2)", - "tab": "Accuracy", - "Professional Medicine - Observed inference time (s)": "{\"description\": \"min=0.467, mean=0.467, max=0.467, sum=0.934 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.46713243337238536\"}", - "Professional Accounting - Observed inference time (s)": "{\"description\": \"min=0.386, mean=0.386, max=0.386, sum=0.771 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.38551004812227074\"}", - "Professional Law - Observed inference time (s)": "{\"description\": \"min=0.429, mean=0.429, max=0.429, sum=0.859 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4294954424886691\"}", - "Professional Psychology - Observed inference time (s)": "{\"description\": \"min=0.397, mean=0.397, max=0.397, sum=0.793 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.39653347715053683\"}", - "Professional Medicine - # eval": "{\"description\": \"min=272, mean=272, max=272, sum=544 (2)\", \"tab\": \"General information\", \"score\": \"272.0\"}", - "Professional Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Medicine - # prompt tokens": "{\"description\": \"min=1113.092, mean=1113.092, max=1113.092, sum=2226.184 (2)\", \"tab\": \"General information\", \"score\": \"1113.0919117647059\"}", - "Professional Medicine - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # eval": "{\"description\": \"min=282, mean=282, max=282, sum=564 (2)\", \"tab\": \"General information\", \"score\": \"282.0\"}", - "Professional Accounting - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Accounting - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # prompt tokens": "{\"description\": \"min=755.418, mean=755.418, max=755.418, sum=1510.837 (2)\", \"tab\": \"General information\", \"score\": \"755.418439716312\"}", - "Professional Accounting - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # eval": "{\"description\": \"min=1534, mean=1534, max=1534, sum=3068 (2)\", \"tab\": \"General information\", \"score\": \"1534.0\"}", - "Professional Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # prompt tokens": "{\"description\": \"min=1685.119, mean=1685.119, max=1685.119, sum=3370.239 (2)\", \"tab\": \"General information\", \"score\": \"1685.119295958279\"}", - "Professional Law - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # eval": "{\"description\": \"min=612, mean=612, max=612, sum=1224 (2)\", \"tab\": \"General information\", \"score\": \"612.0\"}", - "Professional Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # prompt tokens": "{\"description\": \"min=594.363, mean=594.363, max=594.363, sum=1188.725 (2)\", \"tab\": \"General information\", \"score\": \"594.3627450980392\"}", - "Professional Psychology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"professional_psychology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_professional_psychology\"" - } - } - }, - { - "evaluation_name": "Us Foreign Policy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Us Foreign Policy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.78, - "details": { - "description": "min=0.78, mean=0.78, max=0.78, sum=1.56 (2)", - "tab": "Accuracy", - "Us Foreign Policy - Observed inference time (s)": "{\"description\": \"min=0.414, mean=0.414, max=0.414, sum=0.829 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4144425654411316\"}", - "Us Foreign Policy - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Us Foreign Policy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Us Foreign Policy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Us Foreign Policy - # prompt tokens": "{\"description\": \"min=438.2, mean=438.2, max=438.2, sum=876.4 (2)\", \"tab\": \"General information\", \"score\": \"438.2\"}", - "Us Foreign Policy - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"us_foreign_policy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_us_foreign_policy\"" - } - } - }, - { - "evaluation_name": "Astronomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Astronomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.928, - "details": { - "description": "min=0.928, mean=0.928, max=0.928, sum=1.855 (2)", - "tab": "Accuracy", - "Astronomy - Observed inference time (s)": "{\"description\": \"min=0.432, mean=0.432, max=0.432, sum=0.864 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.43207096739819173\"}", - "Astronomy - # eval": "{\"description\": \"min=152, mean=152, max=152, sum=304 (2)\", \"tab\": \"General information\", \"score\": \"152.0\"}", - "Astronomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Astronomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Astronomy - # prompt tokens": "{\"description\": \"min=602.421, mean=602.421, max=602.421, sum=1204.842 (2)\", \"tab\": \"General information\", \"score\": \"602.421052631579\"}", - "Astronomy - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"astronomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_astronomy\"" - } - } - }, - { - "evaluation_name": "Business Ethics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Business Ethics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.73, - "details": { - "description": "min=0.73, mean=0.73, max=0.73, sum=1.46 (2)", - "tab": "Accuracy", - "Business Ethics - Observed inference time (s)": "{\"description\": \"min=0.441, mean=0.441, max=0.441, sum=0.883 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.441267569065094\"}", - "Business Ethics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Business Ethics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Business Ethics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Business Ethics - # prompt tokens": "{\"description\": \"min=552.87, mean=552.87, max=552.87, sum=1105.74 (2)\", \"tab\": \"General information\", \"score\": \"552.87\"}", - "Business Ethics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"business_ethics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_business_ethics\"" - } - } - }, - { - "evaluation_name": "Clinical Knowledge", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Clinical Knowledge", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.879, - "details": { - "description": "min=0.879, mean=0.879, max=0.879, sum=1.758 (2)", - "tab": "Accuracy", - "Clinical Knowledge - Observed inference time (s)": "{\"description\": \"min=0.439, mean=0.439, max=0.439, sum=0.878 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.43878708245619286\"}", - "Clinical Knowledge - # eval": "{\"description\": \"min=265, mean=265, max=265, sum=530 (2)\", \"tab\": \"General information\", \"score\": \"265.0\"}", - "Clinical Knowledge - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Clinical Knowledge - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Clinical Knowledge - # prompt tokens": "{\"description\": \"min=402.592, mean=402.592, max=402.592, sum=805.185 (2)\", \"tab\": \"General information\", \"score\": \"402.5924528301887\"}", - "Clinical Knowledge - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"clinical_knowledge\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_clinical_knowledge\"" - } - } - }, - { - "evaluation_name": "Conceptual Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Conceptual Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.813, - "details": { - "description": "min=0.813, mean=0.813, max=0.813, sum=1.626 (2)", - "tab": "Accuracy", - "Conceptual Physics - Observed inference time (s)": "{\"description\": \"min=0.398, mean=0.398, max=0.398, sum=0.796 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3981509147806371\"}", - "Conceptual Physics - # eval": "{\"description\": \"min=235, mean=235, max=235, sum=470 (2)\", \"tab\": \"General information\", \"score\": \"235.0\"}", - "Conceptual Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Conceptual Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Conceptual Physics - # prompt tokens": "{\"description\": \"min=309.213, mean=309.213, max=309.213, sum=618.426 (2)\", \"tab\": \"General information\", \"score\": \"309.21276595744683\"}", - "Conceptual Physics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"conceptual_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_conceptual_physics\"" - } - } - }, - { - "evaluation_name": "Electrical Engineering", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Electrical Engineering", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.834, - "details": { - "description": "min=0.834, mean=0.834, max=0.834, sum=1.669 (2)", - "tab": "Accuracy", - "Electrical Engineering - Observed inference time (s)": "{\"description\": \"min=0.476, mean=0.476, max=0.476, sum=0.952 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.47606519830637967\"}", - "Electrical Engineering - # eval": "{\"description\": \"min=145, mean=145, max=145, sum=290 (2)\", \"tab\": \"General information\", \"score\": \"145.0\"}", - "Electrical Engineering - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Electrical Engineering - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Electrical Engineering - # prompt tokens": "{\"description\": \"min=474.786, mean=474.786, max=474.786, sum=949.572 (2)\", \"tab\": \"General information\", \"score\": \"474.78620689655173\"}", - "Electrical Engineering - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"electrical_engineering\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_electrical_engineering\"" - } - } - }, - { - "evaluation_name": "Elementary Mathematics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Elementary Mathematics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.857, - "details": { - "description": "min=0.857, mean=0.857, max=0.857, sum=1.714 (2)", - "tab": "Accuracy", - "Elementary Mathematics - Observed inference time (s)": "{\"description\": \"min=0.408, mean=0.408, max=0.408, sum=0.816 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4077642039647178\"}", - "Elementary Mathematics - # eval": "{\"description\": \"min=378, mean=378, max=378, sum=756 (2)\", \"tab\": \"General information\", \"score\": \"378.0\"}", - "Elementary Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Elementary Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Elementary Mathematics - # prompt tokens": "{\"description\": \"min=597.341, mean=597.341, max=597.341, sum=1194.683 (2)\", \"tab\": \"General information\", \"score\": \"597.3412698412699\"}", - "Elementary Mathematics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"elementary_mathematics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_elementary_mathematics\"" - } - } - }, - { - "evaluation_name": "Formal Logic", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Formal Logic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.571, - "details": { - "description": "min=0.571, mean=0.571, max=0.571, sum=1.143 (2)", - "tab": "Accuracy", - "Formal Logic - Observed inference time (s)": "{\"description\": \"min=0.402, mean=0.402, max=0.402, sum=0.804 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4018626610438029\"}", - "Formal Logic - # eval": "{\"description\": \"min=126, mean=126, max=126, sum=252 (2)\", \"tab\": \"General information\", \"score\": \"126.0\"}", - "Formal Logic - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Formal Logic - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Formal Logic - # prompt tokens": "{\"description\": \"min=619.563, mean=619.563, max=619.563, sum=1239.127 (2)\", \"tab\": \"General information\", \"score\": \"619.563492063492\"}", - "Formal Logic - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"formal_logic\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_formal_logic\"" - } - } - }, - { - "evaluation_name": "High School World History", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on High School World History", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.743, - "details": { - "description": "min=0.743, mean=0.743, max=0.743, sum=1.485 (2)", - "tab": "Accuracy", - "High School Biology - Observed inference time (s)": "{\"description\": \"min=0.392, mean=0.392, max=0.392, sum=0.784 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.39193403643946495\"}", - "High School Chemistry - Observed inference time (s)": "{\"description\": \"min=0.391, mean=0.391, max=0.391, sum=0.783 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3914114583302014\"}", - "High School Computer Science - Observed inference time (s)": "{\"description\": \"min=0.392, mean=0.392, max=0.392, sum=0.785 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3924300479888916\"}", - "High School European History - Observed inference time (s)": "{\"description\": \"min=0.452, mean=0.452, max=0.452, sum=0.903 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.451710438005852\"}", - "High School Geography - Observed inference time (s)": "{\"description\": \"min=0.386, mean=0.386, max=0.386, sum=0.773 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3862521937399199\"}", - "High School Government And Politics - Observed inference time (s)": "{\"description\": \"min=0.409, mean=0.409, max=0.409, sum=0.817 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.40865302950607063\"}", - "High School Macroeconomics - Observed inference time (s)": "{\"description\": \"min=0.385, mean=0.385, max=0.385, sum=0.771 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3853575364137307\"}", - "High School Mathematics - Observed inference time (s)": "{\"description\": \"min=0.393, mean=0.393, max=0.393, sum=0.787 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.39334204550142643\"}", - "High School Microeconomics - Observed inference time (s)": "{\"description\": \"min=0.384, mean=0.384, max=0.384, sum=0.768 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.38397373171413646\"}", - "High School Physics - Observed inference time (s)": "{\"description\": \"min=0.412, mean=0.412, max=0.412, sum=0.823 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4116018955281239\"}", - "High School Psychology - Observed inference time (s)": "{\"description\": \"min=0.393, mean=0.393, max=0.393, sum=0.786 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3931623751964044\"}", - "High School Statistics - Observed inference time (s)": "{\"description\": \"min=0.449, mean=0.449, max=0.449, sum=0.898 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.44901008628032824\"}", - "High School US History - Observed inference time (s)": "{\"description\": \"min=0.468, mean=0.468, max=0.468, sum=0.935 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.46768493044610115\"}", - "High School World History - Observed inference time (s)": "{\"description\": \"min=0.452, mean=0.452, max=0.452, sum=0.903 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.451718654310653\"}", - "High School Biology - # eval": "{\"description\": \"min=310, mean=310, max=310, sum=620 (2)\", \"tab\": \"General information\", \"score\": \"310.0\"}", - "High School Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Biology - # prompt tokens": "{\"description\": \"min=500.958, mean=500.958, max=500.958, sum=1001.916 (2)\", \"tab\": \"General information\", \"score\": \"500.958064516129\"}", - "High School Biology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # eval": "{\"description\": \"min=203, mean=203, max=203, sum=406 (2)\", \"tab\": \"General information\", \"score\": \"203.0\"}", - "High School Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # prompt tokens": "{\"description\": \"min=513.064, mean=513.064, max=513.064, sum=1026.128 (2)\", \"tab\": \"General information\", \"score\": \"513.064039408867\"}", - "High School Chemistry - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "High School Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # prompt tokens": "{\"description\": \"min=935.13, mean=935.13, max=935.13, sum=1870.26 (2)\", \"tab\": \"General information\", \"score\": \"935.13\"}", - "High School Computer Science - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # eval": "{\"description\": \"min=165, mean=165, max=165, sum=330 (2)\", \"tab\": \"General information\", \"score\": \"165.0\"}", - "High School European History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School European History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # prompt tokens": "{\"description\": \"min=2797.424, mean=2797.424, max=2797.424, sum=5594.848 (2)\", \"tab\": \"General information\", \"score\": \"2797.4242424242425\"}", - "High School European History - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # eval": "{\"description\": \"min=198, mean=198, max=198, sum=396 (2)\", \"tab\": \"General information\", \"score\": \"198.0\"}", - "High School Geography - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Geography - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # prompt tokens": "{\"description\": \"min=394.773, mean=394.773, max=394.773, sum=789.545 (2)\", \"tab\": \"General information\", \"score\": \"394.77272727272725\"}", - "High School Geography - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # eval": "{\"description\": \"min=193, mean=193, max=193, sum=386 (2)\", \"tab\": \"General information\", \"score\": \"193.0\"}", - "High School Government And Politics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Government And Politics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # prompt tokens": "{\"description\": \"min=479.301, mean=479.301, max=479.301, sum=958.601 (2)\", \"tab\": \"General information\", \"score\": \"479.30051813471505\"}", - "High School Government And Politics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # eval": "{\"description\": \"min=390, mean=390, max=390, sum=780 (2)\", \"tab\": \"General information\", \"score\": \"390.0\"}", - "High School Macroeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Macroeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # prompt tokens": "{\"description\": \"min=396.541, mean=396.541, max=396.541, sum=793.082 (2)\", \"tab\": \"General information\", \"score\": \"396.54102564102567\"}", - "High School Macroeconomics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # eval": "{\"description\": \"min=270, mean=270, max=270, sum=540 (2)\", \"tab\": \"General information\", \"score\": \"270.0\"}", - "High School Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # prompt tokens": "{\"description\": \"min=566.822, mean=566.822, max=566.822, sum=1133.644 (2)\", \"tab\": \"General information\", \"score\": \"566.8222222222222\"}", - "High School Mathematics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # eval": "{\"description\": \"min=238, mean=238, max=238, sum=476 (2)\", \"tab\": \"General information\", \"score\": \"238.0\"}", - "High School Microeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Microeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # prompt tokens": "{\"description\": \"min=415.954, mean=415.954, max=415.954, sum=831.908 (2)\", \"tab\": \"General information\", \"score\": \"415.953781512605\"}", - "High School Microeconomics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # eval": "{\"description\": \"min=151, mean=151, max=151, sum=302 (2)\", \"tab\": \"General information\", \"score\": \"151.0\"}", - "High School Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # prompt tokens": "{\"description\": \"min=591.715, mean=591.715, max=591.715, sum=1183.43 (2)\", \"tab\": \"General information\", \"score\": \"591.7152317880794\"}", - "High School Physics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # eval": "{\"description\": \"min=545, mean=545, max=545, sum=1090 (2)\", \"tab\": \"General information\", \"score\": \"545.0\"}", - "High School Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # prompt tokens": "{\"description\": \"min=502.604, mean=502.604, max=502.604, sum=1005.207 (2)\", \"tab\": \"General information\", \"score\": \"502.60366972477067\"}", - "High School Psychology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # eval": "{\"description\": \"min=216, mean=216, max=216, sum=432 (2)\", \"tab\": \"General information\", \"score\": \"216.0\"}", - "High School Statistics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Statistics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # prompt tokens": "{\"description\": \"min=858.931, mean=858.931, max=858.931, sum=1717.861 (2)\", \"tab\": \"General information\", \"score\": \"858.9305555555555\"}", - "High School Statistics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # eval": "{\"description\": \"min=204, mean=204, max=204, sum=408 (2)\", \"tab\": \"General information\", \"score\": \"204.0\"}", - "High School US History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School US History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # prompt tokens": "{\"description\": \"min=2205.583, mean=2205.583, max=2205.583, sum=4411.167 (2)\", \"tab\": \"General information\", \"score\": \"2205.5833333333335\"}", - "High School US History - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # eval": "{\"description\": \"min=237, mean=237, max=237, sum=474 (2)\", \"tab\": \"General information\", \"score\": \"237.0\"}", - "High School World History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School World History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # prompt tokens": "{\"description\": \"min=1426.544, mean=1426.544, max=1426.544, sum=2853.089 (2)\", \"tab\": \"General information\", \"score\": \"1426.5443037974683\"}", - "High School World History - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"high_school_world_history\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_high_school_world_history\"" - } - } - }, - { - "evaluation_name": "Human Sexuality", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Human Sexuality", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.901, - "details": { - "description": "min=0.901, mean=0.901, max=0.901, sum=1.802 (2)", - "tab": "Accuracy", - "Human Aging - Observed inference time (s)": "{\"description\": \"min=0.4, mean=0.4, max=0.4, sum=0.8 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3999073441253115\"}", - "Human Sexuality - Observed inference time (s)": "{\"description\": \"min=0.42, mean=0.42, max=0.42, sum=0.841 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4203109868610178\"}", - "Human Aging - # eval": "{\"description\": \"min=223, mean=223, max=223, sum=446 (2)\", \"tab\": \"General information\", \"score\": \"223.0\"}", - "Human Aging - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Aging - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Aging - # prompt tokens": "{\"description\": \"min=321.587, mean=321.587, max=321.587, sum=643.175 (2)\", \"tab\": \"General information\", \"score\": \"321.58744394618833\"}", - "Human Aging - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # eval": "{\"description\": \"min=131, mean=131, max=131, sum=262 (2)\", \"tab\": \"General information\", \"score\": \"131.0\"}", - "Human Sexuality - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Sexuality - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # prompt tokens": "{\"description\": \"min=347.183, mean=347.183, max=347.183, sum=694.366 (2)\", \"tab\": \"General information\", \"score\": \"347.1832061068702\"}", - "Human Sexuality - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"human_sexuality\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_human_sexuality\"" - } - } - }, - { - "evaluation_name": "International Law", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on International Law", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.645, - "details": { - "description": "min=0.645, mean=0.645, max=0.645, sum=1.289 (2)", - "tab": "Accuracy", - "International Law - Observed inference time (s)": "{\"description\": \"min=0.457, mean=0.457, max=0.457, sum=0.913 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.45661053972795973\"}", - "International Law - # eval": "{\"description\": \"min=121, mean=121, max=121, sum=242 (2)\", \"tab\": \"General information\", \"score\": \"121.0\"}", - "International Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "International Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "International Law - # prompt tokens": "{\"description\": \"min=644.165, mean=644.165, max=644.165, sum=1288.331 (2)\", \"tab\": \"General information\", \"score\": \"644.1652892561983\"}", - "International Law - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"international_law\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_international_law\"" - } - } - }, - { - "evaluation_name": "Logical Fallacies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Logical Fallacies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.914, - "details": { - "description": "min=0.914, mean=0.914, max=0.914, sum=1.828 (2)", - "tab": "Accuracy", - "Logical Fallacies - Observed inference time (s)": "{\"description\": \"min=0.411, mean=0.411, max=0.411, sum=0.823 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4113436125538832\"}", - "Logical Fallacies - # eval": "{\"description\": \"min=163, mean=163, max=163, sum=326 (2)\", \"tab\": \"General information\", \"score\": \"163.0\"}", - "Logical Fallacies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Logical Fallacies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Logical Fallacies - # prompt tokens": "{\"description\": \"min=450.049, mean=450.049, max=450.049, sum=900.098 (2)\", \"tab\": \"General information\", \"score\": \"450.0490797546012\"}", - "Logical Fallacies - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"logical_fallacies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_logical_fallacies\"" - } - } - }, - { - "evaluation_name": "Machine Learning", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Machine Learning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.759, - "details": { - "description": "min=0.759, mean=0.759, max=0.759, sum=1.518 (2)", - "tab": "Accuracy", - "Machine Learning - Observed inference time (s)": "{\"description\": \"min=0.417, mean=0.417, max=0.417, sum=0.833 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4165512855563845\"}", - "Machine Learning - # eval": "{\"description\": \"min=112, mean=112, max=112, sum=224 (2)\", \"tab\": \"General information\", \"score\": \"112.0\"}", - "Machine Learning - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Machine Learning - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Machine Learning - # prompt tokens": "{\"description\": \"min=702.402, mean=702.402, max=702.402, sum=1404.804 (2)\", \"tab\": \"General information\", \"score\": \"702.4017857142857\"}", - "Machine Learning - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"machine_learning\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_machine_learning\"" - } - } - }, - { - "evaluation_name": "Management", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Management", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.718, - "details": { - "description": "min=0.718, mean=0.718, max=0.718, sum=1.437 (2)", - "tab": "Accuracy", - "Management - Observed inference time (s)": "{\"description\": \"min=0.401, mean=0.401, max=0.401, sum=0.803 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4013508292077814\"}", - "Management - # eval": "{\"description\": \"min=103, mean=103, max=103, sum=206 (2)\", \"tab\": \"General information\", \"score\": \"103.0\"}", - "Management - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Management - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Management - # prompt tokens": "{\"description\": \"min=281.301, mean=281.301, max=281.301, sum=562.602 (2)\", \"tab\": \"General information\", \"score\": \"281.3009708737864\"}", - "Management - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"management\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_management\"" - } - } - }, - { - "evaluation_name": "Marketing", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Marketing", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.944, - "details": { - "description": "min=0.944, mean=0.944, max=0.944, sum=1.889 (2)", - "tab": "Accuracy", - "Marketing - Observed inference time (s)": "{\"description\": \"min=0.401, mean=0.401, max=0.401, sum=0.801 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4005699891310472\"}", - "Marketing - # eval": "{\"description\": \"min=234, mean=234, max=234, sum=468 (2)\", \"tab\": \"General information\", \"score\": \"234.0\"}", - "Marketing - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Marketing - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Marketing - # prompt tokens": "{\"description\": \"min=428.35, mean=428.35, max=428.35, sum=856.701 (2)\", \"tab\": \"General information\", \"score\": \"428.35042735042737\"}", - "Marketing - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"marketing\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_marketing\"" - } - } - }, - { - "evaluation_name": "Medical Genetics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Medical Genetics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.89, - "details": { - "description": "min=0.89, mean=0.89, max=0.89, sum=1.78 (2)", - "tab": "Accuracy", - "Medical Genetics - Observed inference time (s)": "{\"description\": \"min=0.387, mean=0.387, max=0.387, sum=0.773 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.38653050899505614\"}", - "Medical Genetics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Medical Genetics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Medical Genetics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Medical Genetics - # prompt tokens": "{\"description\": \"min=338.89, mean=338.89, max=338.89, sum=677.78 (2)\", \"tab\": \"General information\", \"score\": \"338.89\"}", - "Medical Genetics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"medical_genetics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_medical_genetics\"" - } - } - }, - { - "evaluation_name": "Miscellaneous", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Miscellaneous", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.939, - "details": { - "description": "min=0.939, mean=0.939, max=0.939, sum=1.877 (2)", - "tab": "Accuracy", - "Miscellaneous - Observed inference time (s)": "{\"description\": \"min=0.386, mean=0.386, max=0.386, sum=0.772 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3861832460376647\"}", - "Miscellaneous - # eval": "{\"description\": \"min=783, mean=783, max=783, sum=1566 (2)\", \"tab\": \"General information\", \"score\": \"783.0\"}", - "Miscellaneous - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Miscellaneous - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Miscellaneous - # prompt tokens": "{\"description\": \"min=314.669, mean=314.669, max=314.669, sum=629.338 (2)\", \"tab\": \"General information\", \"score\": \"314.669220945083\"}", - "Miscellaneous - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"miscellaneous\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_miscellaneous\"" - } - } - }, - { - "evaluation_name": "Moral Scenarios", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Moral Scenarios", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.815, - "details": { - "description": "min=0.815, mean=0.815, max=0.815, sum=1.629 (2)", - "tab": "Accuracy", - "Moral Disputes - Observed inference time (s)": "{\"description\": \"min=0.384, mean=0.384, max=0.384, sum=0.768 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3839988109004291\"}", - "Moral Scenarios - Observed inference time (s)": "{\"description\": \"min=0.405, mean=0.405, max=0.405, sum=0.81 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4048716662316349\"}", - "Moral Disputes - # eval": "{\"description\": \"min=346, mean=346, max=346, sum=692 (2)\", \"tab\": \"General information\", \"score\": \"346.0\"}", - "Moral Disputes - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Disputes - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Disputes - # prompt tokens": "{\"description\": \"min=495.003, mean=495.003, max=495.003, sum=990.006 (2)\", \"tab\": \"General information\", \"score\": \"495.0028901734104\"}", - "Moral Disputes - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # eval": "{\"description\": \"min=895, mean=895, max=895, sum=1790 (2)\", \"tab\": \"General information\", \"score\": \"895.0\"}", - "Moral Scenarios - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Scenarios - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # prompt tokens": "{\"description\": \"min=690.542, mean=690.542, max=690.542, sum=1381.084 (2)\", \"tab\": \"General information\", \"score\": \"690.5418994413408\"}", - "Moral Scenarios - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"moral_scenarios\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_moral_scenarios\"" - } - } - }, - { - "evaluation_name": "Nutrition", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Nutrition", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.856, - "details": { - "description": "min=0.856, mean=0.856, max=0.856, sum=1.712 (2)", - "tab": "Accuracy", - "Nutrition - Observed inference time (s)": "{\"description\": \"min=0.397, mean=0.397, max=0.397, sum=0.794 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.39706431027331385\"}", - "Nutrition - # eval": "{\"description\": \"min=306, mean=306, max=306, sum=612 (2)\", \"tab\": \"General information\", \"score\": \"306.0\"}", - "Nutrition - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Nutrition - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Nutrition - # prompt tokens": "{\"description\": \"min=585.48, mean=585.48, max=585.48, sum=1170.961 (2)\", \"tab\": \"General information\", \"score\": \"585.4803921568628\"}", - "Nutrition - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"nutrition\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_nutrition\"" - } - } - }, - { - "evaluation_name": "Prehistory", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Prehistory", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.898, - "details": { - "description": "min=0.898, mean=0.898, max=0.898, sum=1.796 (2)", - "tab": "Accuracy", - "Prehistory - Observed inference time (s)": "{\"description\": \"min=0.39, mean=0.39, max=0.39, sum=0.78 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3900022072556578\"}", - "Prehistory - # eval": "{\"description\": \"min=324, mean=324, max=324, sum=648 (2)\", \"tab\": \"General information\", \"score\": \"324.0\"}", - "Prehistory - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Prehistory - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Prehistory - # prompt tokens": "{\"description\": \"min=540.198, mean=540.198, max=540.198, sum=1080.395 (2)\", \"tab\": \"General information\", \"score\": \"540.1975308641976\"}", - "Prehistory - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"prehistory\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_prehistory\"" - } - } - }, - { - "evaluation_name": "Public Relations", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Public Relations", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.791, - "details": { - "description": "min=0.791, mean=0.791, max=0.791, sum=1.582 (2)", - "tab": "Accuracy", - "Public Relations - Observed inference time (s)": "{\"description\": \"min=0.38, mean=0.38, max=0.38, sum=0.76 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.37999111955816095\"}", - "Public Relations - # eval": "{\"description\": \"min=110, mean=110, max=110, sum=220 (2)\", \"tab\": \"General information\", \"score\": \"110.0\"}", - "Public Relations - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Public Relations - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Public Relations - # prompt tokens": "{\"description\": \"min=426.655, mean=426.655, max=426.655, sum=853.309 (2)\", \"tab\": \"General information\", \"score\": \"426.6545454545454\"}", - "Public Relations - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"public_relations\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_public_relations\"" - } - } - }, - { - "evaluation_name": "Security Studies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Security Studies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.69, - "details": { - "description": "min=0.69, mean=0.69, max=0.69, sum=1.38 (2)", - "tab": "Accuracy", - "Security Studies - Observed inference time (s)": "{\"description\": \"min=0.394, mean=0.394, max=0.394, sum=0.787 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3936534463142862\"}", - "Security Studies - # eval": "{\"description\": \"min=245, mean=245, max=245, sum=490 (2)\", \"tab\": \"General information\", \"score\": \"245.0\"}", - "Security Studies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Security Studies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Security Studies - # prompt tokens": "{\"description\": \"min=1193.869, mean=1193.869, max=1193.869, sum=2387.739 (2)\", \"tab\": \"General information\", \"score\": \"1193.869387755102\"}", - "Security Studies - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"security_studies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_security_studies\"" - } - } - }, - { - "evaluation_name": "Sociology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Sociology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.786, - "details": { - "description": "min=0.786, mean=0.786, max=0.786, sum=1.572 (2)", - "tab": "Accuracy", - "Sociology - Observed inference time (s)": "{\"description\": \"min=0.388, mean=0.388, max=0.388, sum=0.776 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3881402205471969\"}", - "Sociology - # eval": "{\"description\": \"min=201, mean=201, max=201, sum=402 (2)\", \"tab\": \"General information\", \"score\": \"201.0\"}", - "Sociology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Sociology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Sociology - # prompt tokens": "{\"description\": \"min=456.274, mean=456.274, max=456.274, sum=912.547 (2)\", \"tab\": \"General information\", \"score\": \"456.27363184079604\"}", - "Sociology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"sociology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_sociology\"" - } - } - }, - { - "evaluation_name": "Virology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Virology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.554, - "details": { - "description": "min=0.554, mean=0.554, max=0.554, sum=1.108 (2)", - "tab": "Accuracy", - "Virology - Observed inference time (s)": "{\"description\": \"min=0.379, mean=0.379, max=0.379, sum=0.758 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3791351461985025\"}", - "Virology - # eval": "{\"description\": \"min=166, mean=166, max=166, sum=332 (2)\", \"tab\": \"General information\", \"score\": \"166.0\"}", - "Virology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Virology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Virology - # prompt tokens": "{\"description\": \"min=336.753, mean=336.753, max=336.753, sum=673.506 (2)\", \"tab\": \"General information\", \"score\": \"336.7530120481928\"}", - "Virology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"virology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_virology\"" - } - } - }, - { - "evaluation_name": "World Religions", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on World Religions", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.731, - "details": { - "description": "min=0.731, mean=0.731, max=0.731, sum=1.462 (2)", - "tab": "Accuracy", - "World Religions - Observed inference time (s)": "{\"description\": \"min=0.384, mean=0.384, max=0.384, sum=0.768 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.38400994964510377\"}", - "World Religions - # eval": "{\"description\": \"min=171, mean=171, max=171, sum=342 (2)\", \"tab\": \"General information\", \"score\": \"171.0\"}", - "World Religions - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "World Religions - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "World Religions - # prompt tokens": "{\"description\": \"min=268.164, mean=268.164, max=268.164, sum=536.327 (2)\", \"tab\": \"General information\", \"score\": \"268.1637426900585\"}", - "World Religions - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"world_religions\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_world_religions\"" - } - } - }, - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.567, - "details": { - "description": "", - "tab": "Efficiency" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/google_gemini-2.0-flash-lite-preview-02-05.json b/data/models/google_gemini-2.0-flash-lite-preview-02-05.json deleted file mode 100644 index 26e4c3162721317ef4fd4983f2a5169a81d47918..0000000000000000000000000000000000000000 --- a/data/models/google_gemini-2.0-flash-lite-preview-02-05.json +++ /dev/null @@ -1,234 +0,0 @@ -{ - "model_info": { - "name": "Gemini 2.0 Flash Lite 02-05 preview", - "id": "google/gemini-2.0-flash-lite-preview-02-05", - "developer": "Google", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_capabilities/google_gemini-2.0-flash-lite-preview-02-05/1774096304.5056698", - "retrieved_timestamp": "1774096304.5056698", - "source_metadata": { - "source_name": "helm_capabilities", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_capabilities", - "evaluation_results": [ - { - "evaluation_name": "Mean score", - "source_data": { - "dataset_name": "helm_capabilities", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "The mean of the scores from all columns.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.642, - "details": { - "description": "", - "tab": "Accuracy", - "Mean score - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"5.788722673180064\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU-Pro", - "source_data": { - "dataset_name": "MMLU-Pro", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on MMLU-Pro", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.72, - "details": { - "description": "min=0.72, mean=0.72, max=0.72, sum=0.72 (1)", - "tab": "Accuracy", - "MMLU-Pro - Observed inference time (s)": "{\"description\": \"min=3.357, mean=3.357, max=3.357, sum=3.357 (1)\", \"tab\": \"Efficiency\", \"score\": \"3.356641344547272\"}", - "MMLU-Pro - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "MMLU-Pro - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - # prompt tokens": "{\"description\": \"min=242.673, mean=242.673, max=242.673, sum=242.673 (1)\", \"tab\": \"General information\", \"score\": \"242.673\"}", - "MMLU-Pro - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"all\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5, - "details": { - "description": "min=0.5, mean=0.5, max=0.5, sum=0.5 (1)", - "tab": "Accuracy", - "GPQA - Observed inference time (s)": "{\"description\": \"min=5.373, mean=5.373, max=5.373, sum=5.373 (1)\", \"tab\": \"Efficiency\", \"score\": \"5.372664878186623\"}", - "GPQA - # eval": "{\"description\": \"min=446, mean=446, max=446, sum=446 (1)\", \"tab\": \"General information\", \"score\": \"446.0\"}", - "GPQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - # prompt tokens": "{\"description\": \"min=252.735, mean=252.735, max=252.735, sum=252.735 (1)\", \"tab\": \"General information\", \"score\": \"252.7354260089686\"}", - "GPQA - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"gpqa_main\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "IFEval Strict Acc on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.824, - "details": { - "description": "min=0.824, mean=0.824, max=0.824, sum=0.824 (1)", - "tab": "Accuracy", - "IFEval - Observed inference time (s)": "{\"description\": \"min=3.463, mean=3.463, max=3.463, sum=3.463 (1)\", \"tab\": \"Efficiency\", \"score\": \"3.4628667553780037\"}", - "IFEval - # eval": "{\"description\": \"min=541, mean=541, max=541, sum=541 (1)\", \"tab\": \"General information\", \"score\": \"541.0\"}", - "IFEval - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - # prompt tokens": "{\"description\": \"min=47.331, mean=47.331, max=47.331, sum=47.331 (1)\", \"tab\": \"General information\", \"score\": \"47.33086876155268\"}", - "IFEval - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WildBench", - "source_data": { - "dataset_name": "WildBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "WB Score on WildBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.79, - "details": { - "description": "min=0.79, mean=0.79, max=0.79, sum=0.79 (1)", - "tab": "Accuracy", - "WildBench - Observed inference time (s)": "{\"description\": \"min=8.804, mean=8.804, max=8.804, sum=8.804 (1)\", \"tab\": \"Efficiency\", \"score\": \"8.803904922309524\"}", - "WildBench - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "WildBench - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # prompt tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"v2\"" - } - } - }, - { - "evaluation_name": "Omni-MATH", - "source_data": { - "dataset_name": "Omni-MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Acc on Omni-MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.374, - "details": { - "description": "min=0.374, mean=0.374, max=0.374, sum=0.374 (1)", - "tab": "Accuracy", - "Omni-MATH - Observed inference time (s)": "{\"description\": \"min=7.948, mean=7.948, max=7.948, sum=7.948 (1)\", \"tab\": \"Efficiency\", \"score\": \"7.947535465478897\"}", - "Omni-MATH - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "Omni-MATH - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - # prompt tokens": "{\"description\": \"min=111.956, mean=111.956, max=111.956, sum=111.956 (1)\", \"tab\": \"General information\", \"score\": \"111.956\"}", - "Omni-MATH - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/google_gemini-2.5-flash-lite.json b/data/models/google_gemini-2.5-flash-lite.json deleted file mode 100644 index bbde25afc1941dbf2a34c443ae409936edb512ae..0000000000000000000000000000000000000000 --- a/data/models/google_gemini-2.5-flash-lite.json +++ /dev/null @@ -1,234 +0,0 @@ -{ - "model_info": { - "name": "Gemini 2.5 Flash-Lite", - "id": "google/gemini-2.5-flash-lite", - "developer": "Google", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_capabilities/google_gemini-2.5-flash-lite/1774096304.5056698", - "retrieved_timestamp": "1774096304.5056698", - "source_metadata": { - "source_name": "helm_capabilities", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_capabilities", - "evaluation_results": [ - { - "evaluation_name": "Mean score", - "source_data": { - "dataset_name": "helm_capabilities", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "The mean of the scores from all columns.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.591, - "details": { - "description": "", - "tab": "Accuracy", - "Mean score - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"8.113822886648412\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU-Pro", - "source_data": { - "dataset_name": "MMLU-Pro", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on MMLU-Pro", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.537, - "details": { - "description": "min=0.537, mean=0.537, max=0.537, sum=0.537 (1)", - "tab": "Accuracy", - "MMLU-Pro - Observed inference time (s)": "{\"description\": \"min=4.423, mean=4.423, max=4.423, sum=4.423 (1)\", \"tab\": \"Efficiency\", \"score\": \"4.423401823997498\"}", - "MMLU-Pro - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "MMLU-Pro - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - # prompt tokens": "{\"description\": \"min=263.673, mean=263.673, max=263.673, sum=263.673 (1)\", \"tab\": \"General information\", \"score\": \"263.673\"}", - "MMLU-Pro - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"all\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.309, - "details": { - "description": "min=0.309, mean=0.309, max=0.309, sum=0.309 (1)", - "tab": "Accuracy", - "GPQA - Observed inference time (s)": "{\"description\": \"min=11.88, mean=11.88, max=11.88, sum=11.88 (1)\", \"tab\": \"Efficiency\", \"score\": \"11.880136902022254\"}", - "GPQA - # eval": "{\"description\": \"min=446, mean=446, max=446, sum=446 (1)\", \"tab\": \"General information\", \"score\": \"446.0\"}", - "GPQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - # prompt tokens": "{\"description\": \"min=273.735, mean=273.735, max=273.735, sum=273.735 (1)\", \"tab\": \"General information\", \"score\": \"273.7354260089686\"}", - "GPQA - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"gpqa_main\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "IFEval Strict Acc on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.81, - "details": { - "description": "min=0.81, mean=0.81, max=0.81, sum=0.81 (1)", - "tab": "Accuracy", - "IFEval - Observed inference time (s)": "{\"description\": \"min=1.833, mean=1.833, max=1.833, sum=1.833 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.833447342659321\"}", - "IFEval - # eval": "{\"description\": \"min=541, mean=541, max=541, sum=541 (1)\", \"tab\": \"General information\", \"score\": \"541.0\"}", - "IFEval - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - # prompt tokens": "{\"description\": \"min=47.331, mean=47.331, max=47.331, sum=47.331 (1)\", \"tab\": \"General information\", \"score\": \"47.33086876155268\"}", - "IFEval - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WildBench", - "source_data": { - "dataset_name": "WildBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "WB Score on WildBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.818, - "details": { - "description": "min=0.818, mean=0.818, max=0.818, sum=0.818 (1)", - "tab": "Accuracy", - "WildBench - Observed inference time (s)": "{\"description\": \"min=7.111, mean=7.111, max=7.111, sum=7.111 (1)\", \"tab\": \"Efficiency\", \"score\": \"7.111379201173782\"}", - "WildBench - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "WildBench - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # prompt tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"v2\"" - } - } - }, - { - "evaluation_name": "Omni-MATH", - "source_data": { - "dataset_name": "Omni-MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Acc on Omni-MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.48, - "details": { - "description": "min=0.48, mean=0.48, max=0.48, sum=0.48 (1)", - "tab": "Accuracy", - "Omni-MATH - Observed inference time (s)": "{\"description\": \"min=15.321, mean=15.321, max=15.321, sum=15.321 (1)\", \"tab\": \"Efficiency\", \"score\": \"15.320749163389205\"}", - "Omni-MATH - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "Omni-MATH - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - # prompt tokens": "{\"description\": \"min=111.956, mean=111.956, max=111.956, sum=111.956 (1)\", \"tab\": \"General information\", \"score\": \"111.956\"}", - "Omni-MATH - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/google_gemini-2.5-flash-preview-04-17.json b/data/models/google_gemini-2.5-flash-preview-04-17.json deleted file mode 100644 index e1fa85f431103e2b04cff10fdf6991d4e1023e28..0000000000000000000000000000000000000000 --- a/data/models/google_gemini-2.5-flash-preview-04-17.json +++ /dev/null @@ -1,384 +0,0 @@ -{ - "model_info": { - "name": "Gemini 2.5 Flash 04-17 preview", - "id": "google/gemini-2.5-flash-preview-04-17", - "developer": "Google", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_capabilities/google_gemini-2.5-flash-preview-04-17/1774096304.5056698", - "retrieved_timestamp": "1774096304.5056698", - "source_metadata": { - "source_name": "helm_capabilities", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_capabilities", - "evaluation_results": [ - { - "evaluation_name": "Mean score", - "source_data": { - "dataset_name": "helm_capabilities", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "The mean of the scores from all columns.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.626, - "details": { - "description": "", - "tab": "Accuracy", - "Mean score - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"31.900818991762513\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU-Pro", - "source_data": { - "dataset_name": "MMLU-Pro", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on MMLU-Pro", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.639, - "details": { - "description": "min=0.639, mean=0.639, max=0.639, sum=0.639 (1)", - "tab": "Accuracy", - "MMLU-Pro - Observed inference time (s)": "{\"description\": \"min=17.353, mean=17.353, max=17.353, sum=17.353 (1)\", \"tab\": \"Efficiency\", \"score\": \"17.352934203863143\"}", - "MMLU-Pro - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "MMLU-Pro - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - # prompt tokens": "{\"description\": \"min=263.673, mean=263.673, max=263.673, sum=263.673 (1)\", \"tab\": \"General information\", \"score\": \"263.673\"}", - "MMLU-Pro - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"all\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.39, - "details": { - "description": "min=0.39, mean=0.39, max=0.39, sum=0.39 (1)", - "tab": "Accuracy", - "GPQA - Observed inference time (s)": "{\"description\": \"min=38.125, mean=38.125, max=38.125, sum=38.125 (1)\", \"tab\": \"Efficiency\", \"score\": \"38.125050564562336\"}", - "GPQA - # eval": "{\"description\": \"min=446, mean=446, max=446, sum=446 (1)\", \"tab\": \"General information\", \"score\": \"446.0\"}", - "GPQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - # prompt tokens": "{\"description\": \"min=273.735, mean=273.735, max=273.735, sum=273.735 (1)\", \"tab\": \"General information\", \"score\": \"273.7354260089686\"}", - "GPQA - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"gpqa_main\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "IFEval Strict Acc on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.898, - "details": { - "description": "min=0.898, mean=0.898, max=0.898, sum=0.898 (1)", - "tab": "Accuracy", - "IFEval - Observed inference time (s)": "{\"description\": \"min=11.266, mean=11.266, max=11.266, sum=11.266 (1)\", \"tab\": \"Efficiency\", \"score\": \"11.266106982142837\"}", - "IFEval - # eval": "{\"description\": \"min=541, mean=541, max=541, sum=541 (1)\", \"tab\": \"General information\", \"score\": \"541.0\"}", - "IFEval - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - # prompt tokens": "{\"description\": \"min=47.331, mean=47.331, max=47.331, sum=47.331 (1)\", \"tab\": \"General information\", \"score\": \"47.33086876155268\"}", - "IFEval - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WildBench", - "source_data": { - "dataset_name": "WildBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "WB Score on WildBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.817, - "details": { - "description": "min=0.817, mean=0.817, max=0.817, sum=0.817 (1)", - "tab": "Accuracy", - "WildBench - Observed inference time (s)": "{\"description\": \"min=32.789, mean=32.789, max=32.789, sum=32.789 (1)\", \"tab\": \"Efficiency\", \"score\": \"32.78856403473391\"}", - "WildBench - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "WildBench - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # prompt tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"v2\"" - } - } - }, - { - "evaluation_name": "Omni-MATH", - "source_data": { - "dataset_name": "Omni-MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Acc on Omni-MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.384, - "details": { - "description": "min=0.384, mean=0.384, max=0.384, sum=0.384 (1)", - "tab": "Accuracy", - "Omni-MATH - Observed inference time (s)": "{\"description\": \"min=59.971, mean=59.971, max=59.971, sum=59.971 (1)\", \"tab\": \"Efficiency\", \"score\": \"59.97143917351036\"}", - "Omni-MATH - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "Omni-MATH - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - # prompt tokens": "{\"description\": \"min=111.956, mean=111.956, max=111.956, sum=111.956 (1)\", \"tab\": \"General information\", \"score\": \"111.956\"}", - "Omni-MATH - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "reward-bench-2/google_gemini-2.5-flash-preview-04-17/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7721 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6574 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5531 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8115 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9094 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8672 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8341 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/google_gemini-2.5-flash-preview-05-20.json b/data/models/google_gemini-2.5-flash-preview-05-20.json deleted file mode 100644 index 01b694cbaacb7c97812a9bb056c39eebb70c1f12..0000000000000000000000000000000000000000 --- a/data/models/google_gemini-2.5-flash-preview-05-20.json +++ /dev/null @@ -1,1043 +0,0 @@ -{ - "model_info": { - "name": "gemini-2.5-flash-preview-05-20", - "id": "google/gemini-2.5-flash-preview-05-20", - "developer": "Google", - "inference_platform": "unknown", - "additional_details": { - "display_name": "Gemini 2.5 Flash Preview" - } - }, - "evaluations": [ - { - "evaluation_id": "global-mmlu-lite/google_gemini-2.5-flash-preview-05-20/1773936583.743359", - "retrieved_timestamp": "1773936583.743359", - "source_metadata": { - "source_name": "Global MMLU Lite Leaderboard", - "source_type": "documentation", - "source_organization_name": "kaggle", - "source_organization_url": "www.kaggle.com", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "kaggle kernel", - "version": "4", - "additional_details": { - "url": "https://www.kaggle.com/code/shivalikasingh95/global-mmlu-lite-sample-notebook" - } - }, - "benchmark": "global-mmlu-lite", - "evaluation_results": [ - { - "evaluation_name": "Global MMLU Lite", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Global MMLU Lite", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9092 - } - }, - { - "evaluation_name": "Culturally Sensitive", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Sensitive", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8925 - } - }, - { - "evaluation_name": "Culturally Agnostic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Agnostic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9259 - } - }, - { - "evaluation_name": "Arabic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Arabic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.905, - "uncertainty": { - "confidence_interval": { - "lower": -0.0287, - "upper": 0.0287, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "English", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - English", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9225, - "uncertainty": { - "confidence_interval": { - "lower": -0.0262, - "upper": 0.0262, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Bengali", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Bengali", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.91, - "uncertainty": { - "confidence_interval": { - "lower": -0.028, - "upper": 0.028, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "German", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - German", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.905, - "uncertainty": { - "confidence_interval": { - "lower": -0.0287, - "upper": 0.0287, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "French", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - French", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.925, - "uncertainty": { - "confidence_interval": { - "lower": -0.0258, - "upper": 0.0258, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Hindi", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Hindi", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9125, - "uncertainty": { - "confidence_interval": { - "lower": -0.0277, - "upper": 0.0277, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Indonesian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Indonesian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9075, - "uncertainty": { - "confidence_interval": { - "lower": -0.0284, - "upper": 0.0284, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Italian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Italian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.89, - "uncertainty": { - "confidence_interval": { - "lower": -0.0307, - "upper": 0.0307, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Japanese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Japanese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9125, - "uncertainty": { - "confidence_interval": { - "lower": -0.0277, - "upper": 0.0277, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Korean", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Korean", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9075, - "uncertainty": { - "confidence_interval": { - "lower": -0.0284, - "upper": 0.0284, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Portuguese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Portuguese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.915, - "uncertainty": { - "confidence_interval": { - "lower": -0.0273, - "upper": 0.0273, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Spanish", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Spanish", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.915, - "uncertainty": { - "confidence_interval": { - "lower": -0.0273, - "upper": 0.0273, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Swahili", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Swahili", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.905, - "uncertainty": { - "confidence_interval": { - "lower": -0.0287, - "upper": 0.0287, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Yoruba", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Yoruba", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8825, - "uncertainty": { - "confidence_interval": { - "lower": -0.0316, - "upper": 0.0316, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Chinese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Chinese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.93, - "uncertainty": { - "confidence_interval": { - "lower": -0.025, - "upper": 0.025, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Burmese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Burmese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9025, - "uncertainty": { - "confidence_interval": { - "lower": -0.0291, - "upper": 0.0291, - "method": "unknown" - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "global-mmlu-lite/google_gemini-2.5-flash-preview-05-20/1773936496.366405", - "retrieved_timestamp": "1773936496.366405", - "source_metadata": { - "source_name": "Global MMLU Lite Leaderboard", - "source_type": "documentation", - "source_organization_name": "kaggle", - "source_organization_url": "www.kaggle.com", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "kaggle kernel", - "version": "4", - "additional_details": { - "url": "https://www.kaggle.com/code/shivalikasingh95/global-mmlu-lite-sample-notebook" - } - }, - "benchmark": "global-mmlu-lite", - "evaluation_results": [ - { - "evaluation_name": "Global MMLU Lite", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Global MMLU Lite", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9092 - } - }, - { - "evaluation_name": "Culturally Sensitive", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Sensitive", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8925 - } - }, - { - "evaluation_name": "Culturally Agnostic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Agnostic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9259 - } - }, - { - "evaluation_name": "Arabic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Arabic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.905, - "uncertainty": { - "confidence_interval": { - "lower": -0.0287, - "upper": 0.0287, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "English", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - English", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9225, - "uncertainty": { - "confidence_interval": { - "lower": -0.0262, - "upper": 0.0262, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Bengali", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Bengali", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.91, - "uncertainty": { - "confidence_interval": { - "lower": -0.028, - "upper": 0.028, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "German", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - German", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.905, - "uncertainty": { - "confidence_interval": { - "lower": -0.0287, - "upper": 0.0287, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "French", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - French", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.925, - "uncertainty": { - "confidence_interval": { - "lower": -0.0258, - "upper": 0.0258, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Hindi", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Hindi", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9125, - "uncertainty": { - "confidence_interval": { - "lower": -0.0277, - "upper": 0.0277, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Indonesian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Indonesian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9075, - "uncertainty": { - "confidence_interval": { - "lower": -0.0284, - "upper": 0.0284, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Italian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Italian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.89, - "uncertainty": { - "confidence_interval": { - "lower": -0.0307, - "upper": 0.0307, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Japanese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Japanese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9125, - "uncertainty": { - "confidence_interval": { - "lower": -0.0277, - "upper": 0.0277, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Korean", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Korean", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9075, - "uncertainty": { - "confidence_interval": { - "lower": -0.0284, - "upper": 0.0284, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Portuguese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Portuguese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.915, - "uncertainty": { - "confidence_interval": { - "lower": -0.0273, - "upper": 0.0273, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Spanish", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Spanish", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.915, - "uncertainty": { - "confidence_interval": { - "lower": -0.0273, - "upper": 0.0273, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Swahili", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Swahili", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.905, - "uncertainty": { - "confidence_interval": { - "lower": -0.0287, - "upper": 0.0287, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Yoruba", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Yoruba", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8825, - "uncertainty": { - "confidence_interval": { - "lower": -0.0316, - "upper": 0.0316, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Chinese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Chinese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.93, - "uncertainty": { - "confidence_interval": { - "lower": -0.025, - "upper": 0.025, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Burmese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Burmese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9025, - "uncertainty": { - "confidence_interval": { - "lower": -0.0291, - "upper": 0.0291, - "method": "unknown" - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/google_gemini-2.5-flash.json b/data/models/google_gemini-2.5-flash.json deleted file mode 100644 index c259c4d3dc6471ebeac2f17f3d1252154c39c5bd..0000000000000000000000000000000000000000 --- a/data/models/google_gemini-2.5-flash.json +++ /dev/null @@ -1,1568 +0,0 @@ -{ - "model_info": { - "name": "Gemini 2.5 Flash", - "id": "google/gemini-2.5-flash", - "developer": "Google", - "inference_platform": "unknown", - "additional_details": { - "display_name": "Gemini 2.5 Flash" - } - }, - "evaluations": [ - { - "evaluation_id": "global-mmlu-lite/google_gemini-2.5-flash/1773936583.743359", - "retrieved_timestamp": "1773936583.743359", - "source_metadata": { - "source_name": "Global MMLU Lite Leaderboard", - "source_type": "documentation", - "source_organization_name": "kaggle", - "source_organization_url": "www.kaggle.com", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "kaggle kernel", - "version": "4", - "additional_details": { - "url": "https://www.kaggle.com/code/shivalikasingh95/global-mmlu-lite-sample-notebook" - } - }, - "benchmark": "global-mmlu-lite", - "evaluation_results": [ - { - "evaluation_name": "Global MMLU Lite", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Global MMLU Lite", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9145 - } - }, - { - "evaluation_name": "Culturally Sensitive", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Sensitive", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9 - } - }, - { - "evaluation_name": "Culturally Agnostic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Agnostic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9291 - } - }, - { - "evaluation_name": "Arabic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Arabic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9125, - "uncertainty": { - "confidence_interval": { - "lower": -0.0277, - "upper": 0.0277, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "English", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - English", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9325, - "uncertainty": { - "confidence_interval": { - "lower": -0.0246, - "upper": 0.0246, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Bengali", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Bengali", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.91, - "uncertainty": { - "confidence_interval": { - "lower": -0.028, - "upper": 0.028, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "German", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - German", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9025, - "uncertainty": { - "confidence_interval": { - "lower": -0.0291, - "upper": 0.0291, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "French", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - French", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.91, - "uncertainty": { - "confidence_interval": { - "lower": -0.028, - "upper": 0.028, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Hindi", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Hindi", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.925, - "uncertainty": { - "confidence_interval": { - "lower": -0.0258, - "upper": 0.0258, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Indonesian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Indonesian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9075, - "uncertainty": { - "confidence_interval": { - "lower": -0.0284, - "upper": 0.0284, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Italian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Italian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9225, - "uncertainty": { - "confidence_interval": { - "lower": -0.0262, - "upper": 0.0262, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Japanese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Japanese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9125, - "uncertainty": { - "confidence_interval": { - "lower": -0.0277, - "upper": 0.0277, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Korean", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Korean", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.915, - "uncertainty": { - "confidence_interval": { - "lower": -0.0273, - "upper": 0.0273, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Portuguese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Portuguese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9125, - "uncertainty": { - "confidence_interval": { - "lower": -0.0277, - "upper": 0.0277, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Spanish", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Spanish", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9175, - "uncertainty": { - "confidence_interval": { - "lower": -0.027, - "upper": 0.027, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Swahili", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Swahili", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.915, - "uncertainty": { - "confidence_interval": { - "lower": -0.0273, - "upper": 0.0273, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Yoruba", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Yoruba", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9075, - "uncertainty": { - "confidence_interval": { - "lower": -0.0284, - "upper": 0.0284, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Chinese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Chinese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.915, - "uncertainty": { - "confidence_interval": { - "lower": -0.0273, - "upper": 0.0273, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Burmese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Burmese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.915, - "uncertainty": { - "confidence_interval": { - "lower": -0.0273, - "upper": 0.0273, - "method": "unknown" - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "global-mmlu-lite/google_gemini-2.5-flash/1773936496.366405", - "retrieved_timestamp": "1773936496.366405", - "source_metadata": { - "source_name": "Global MMLU Lite Leaderboard", - "source_type": "documentation", - "source_organization_name": "kaggle", - "source_organization_url": "www.kaggle.com", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "kaggle kernel", - "version": "4", - "additional_details": { - "url": "https://www.kaggle.com/code/shivalikasingh95/global-mmlu-lite-sample-notebook" - } - }, - "benchmark": "global-mmlu-lite", - "evaluation_results": [ - { - "evaluation_name": "Global MMLU Lite", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Global MMLU Lite", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9145 - } - }, - { - "evaluation_name": "Culturally Sensitive", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Sensitive", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9 - } - }, - { - "evaluation_name": "Culturally Agnostic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Agnostic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9291 - } - }, - { - "evaluation_name": "Arabic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Arabic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9125, - "uncertainty": { - "confidence_interval": { - "lower": -0.0277, - "upper": 0.0277, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "English", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - English", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9325, - "uncertainty": { - "confidence_interval": { - "lower": -0.0246, - "upper": 0.0246, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Bengali", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Bengali", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.91, - "uncertainty": { - "confidence_interval": { - "lower": -0.028, - "upper": 0.028, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "German", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - German", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9025, - "uncertainty": { - "confidence_interval": { - "lower": -0.0291, - "upper": 0.0291, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "French", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - French", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.91, - "uncertainty": { - "confidence_interval": { - "lower": -0.028, - "upper": 0.028, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Hindi", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Hindi", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.925, - "uncertainty": { - "confidence_interval": { - "lower": -0.0258, - "upper": 0.0258, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Indonesian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Indonesian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9075, - "uncertainty": { - "confidence_interval": { - "lower": -0.0284, - "upper": 0.0284, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Italian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Italian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9225, - "uncertainty": { - "confidence_interval": { - "lower": -0.0262, - "upper": 0.0262, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Japanese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Japanese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9125, - "uncertainty": { - "confidence_interval": { - "lower": -0.0277, - "upper": 0.0277, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Korean", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Korean", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.915, - "uncertainty": { - "confidence_interval": { - "lower": -0.0273, - "upper": 0.0273, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Portuguese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Portuguese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9125, - "uncertainty": { - "confidence_interval": { - "lower": -0.0277, - "upper": 0.0277, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Spanish", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Spanish", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9175, - "uncertainty": { - "confidence_interval": { - "lower": -0.027, - "upper": 0.027, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Swahili", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Swahili", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.915, - "uncertainty": { - "confidence_interval": { - "lower": -0.0273, - "upper": 0.0273, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Yoruba", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Yoruba", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9075, - "uncertainty": { - "confidence_interval": { - "lower": -0.0284, - "upper": 0.0284, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Chinese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Chinese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.915, - "uncertainty": { - "confidence_interval": { - "lower": -0.0273, - "upper": 0.0273, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Burmese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Burmese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.915, - "uncertainty": { - "confidence_interval": { - "lower": -0.0273, - "upper": 0.0273, - "method": "unknown" - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "livecodebenchpro/google/gemini-2.5-flash/1760492095.8105888", - "retrieved_timestamp": "1760492095.8105888", - "source_metadata": { - "source_organization_name": "New York University, Princeton University, University of California San Diego, University of Washington and Canyon Crest Academy", - "evaluator_relationship": "third_party", - "source_name": "Live Code Bench Pro", - "source_type": "documentation" - }, - "eval_library": { - "name": "unknown", - "version": "unknown" - }, - "benchmark": "livecodebenchpro", - "evaluation_results": [ - { - "evaluation_name": "Hard Problems", - "metric_config": { - "evaluation_description": "Pass@1 on Hard Problems", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.0 - }, - "source_data": { - "dataset_name": "Hard Problems", - "source_type": "url", - "url": [ - "https://webhook.cp-bench.orzzh.com/leaderboard/llm/difficulty?difficulty=hard&benchmark_mode=live" - ] - } - }, - { - "evaluation_name": "Medium Problems", - "metric_config": { - "evaluation_description": "Pass@1 on Medium Problems", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.028169014084507043 - }, - "source_data": { - "dataset_name": "Medium Problems", - "source_type": "url", - "url": [ - "https://webhook.cp-bench.orzzh.com/leaderboard/llm/difficulty?difficulty=medium&benchmark_mode=live" - ] - } - }, - { - "evaluation_name": "Easy Problems", - "metric_config": { - "evaluation_description": "Pass@1 on Easy Problems", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.38028169014084506 - }, - "source_data": { - "dataset_name": "Easy Problems", - "source_type": "url", - "url": [ - "https://webhook.cp-bench.orzzh.com/leaderboard/llm/difficulty?difficulty=easy&benchmark_mode=live" - ] - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "reward-bench-2/google_gemini-2.5-flash/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7767 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.674 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.575 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.852 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.909 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.841 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.809 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "terminal-bench-2.0/mini-swe-agent__gemini-2.5-flash/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2025-11-03", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 17.1, - "uncertainty": { - "standard_error": { - "value": 2.5 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Mini-SWE-Agent\" -m \"Gemini 2.5 Flash\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Mini-SWE-Agent\" -m \"Gemini 2.5 Flash\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - }, - { - "evaluation_id": "terminal-bench-2.0/gemini-cli__gemini-2.5-flash/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2025-11-04", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 15.4, - "uncertainty": { - "standard_error": { - "value": 2.3 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Gemini CLI\" -m \"Gemini 2.5 Flash\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Gemini CLI\" -m \"Gemini 2.5 Flash\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - }, - { - "evaluation_id": "terminal-bench-2.0/openhands__gemini-2.5-flash/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2025-11-02", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 16.4, - "uncertainty": { - "standard_error": { - "value": 2.4 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"OpenHands\" -m \"Gemini 2.5 Flash\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"OpenHands\" -m \"Gemini 2.5 Flash\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - }, - { - "evaluation_id": "terminal-bench-2.0/terminus-2__gemini-2.5-flash/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2025-10-31", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 16.9, - "uncertainty": { - "standard_error": { - "value": 2.4 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Terminus 2\" -m \"Gemini 2.5 Flash\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Terminus 2\" -m \"Gemini 2.5 Flash\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/google_gemini-2.5-pro-preview-03-25.json b/data/models/google_gemini-2.5-pro-preview-03-25.json deleted file mode 100644 index 64432ff49a4947e63281addf8d9584ae6f88abd4..0000000000000000000000000000000000000000 --- a/data/models/google_gemini-2.5-pro-preview-03-25.json +++ /dev/null @@ -1,234 +0,0 @@ -{ - "model_info": { - "name": "Gemini 2.5 Pro 03-25 preview", - "id": "google/gemini-2.5-pro-preview-03-25", - "developer": "Google", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_capabilities/google_gemini-2.5-pro-preview-03-25/1774096304.5056698", - "retrieved_timestamp": "1774096304.5056698", - "source_metadata": { - "source_name": "helm_capabilities", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_capabilities", - "evaluation_results": [ - { - "evaluation_name": "Mean score", - "source_data": { - "dataset_name": "helm_capabilities", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "The mean of the scores from all columns.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.745, - "details": { - "description": "", - "tab": "Accuracy", - "Mean score - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"41.707859761088116\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU-Pro", - "source_data": { - "dataset_name": "MMLU-Pro", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on MMLU-Pro", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.863, - "details": { - "description": "min=0.863, mean=0.863, max=0.863, sum=0.863 (1)", - "tab": "Accuracy", - "MMLU-Pro - Observed inference time (s)": "{\"description\": \"min=22.301, mean=22.301, max=22.301, sum=22.301 (1)\", \"tab\": \"Efficiency\", \"score\": \"22.301176882605677\"}", - "MMLU-Pro - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "MMLU-Pro - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - # prompt tokens": "{\"description\": \"min=263.673, mean=263.673, max=263.673, sum=263.673 (1)\", \"tab\": \"General information\", \"score\": \"263.673\"}", - "MMLU-Pro - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"all\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.749, - "details": { - "description": "min=0.749, mean=0.749, max=0.749, sum=0.749 (1)", - "tab": "Accuracy", - "GPQA - Observed inference time (s)": "{\"description\": \"min=43.194, mean=43.194, max=43.194, sum=43.194 (1)\", \"tab\": \"Efficiency\", \"score\": \"43.19425330858552\"}", - "GPQA - # eval": "{\"description\": \"min=446, mean=446, max=446, sum=446 (1)\", \"tab\": \"General information\", \"score\": \"446.0\"}", - "GPQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - # prompt tokens": "{\"description\": \"min=273.735, mean=273.735, max=273.735, sum=273.735 (1)\", \"tab\": \"General information\", \"score\": \"273.7354260089686\"}", - "GPQA - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"gpqa_main\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "IFEval Strict Acc on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.84, - "details": { - "description": "min=0.84, mean=0.84, max=0.84, sum=0.84 (1)", - "tab": "Accuracy", - "IFEval - Observed inference time (s)": "{\"description\": \"min=15.978, mean=15.978, max=15.978, sum=15.978 (1)\", \"tab\": \"Efficiency\", \"score\": \"15.978427228116725\"}", - "IFEval - # eval": "{\"description\": \"min=541, mean=541, max=541, sum=541 (1)\", \"tab\": \"General information\", \"score\": \"541.0\"}", - "IFEval - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - # prompt tokens": "{\"description\": \"min=47.331, mean=47.331, max=47.331, sum=47.331 (1)\", \"tab\": \"General information\", \"score\": \"47.33086876155268\"}", - "IFEval - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WildBench", - "source_data": { - "dataset_name": "WildBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "WB Score on WildBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.857, - "details": { - "description": "min=0.857, mean=0.857, max=0.857, sum=0.857 (1)", - "tab": "Accuracy", - "WildBench - Observed inference time (s)": "{\"description\": \"min=41.295, mean=41.295, max=41.295, sum=41.295 (1)\", \"tab\": \"Efficiency\", \"score\": \"41.2954368838362\"}", - "WildBench - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "WildBench - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # prompt tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"v2\"" - } - } - }, - { - "evaluation_name": "Omni-MATH", - "source_data": { - "dataset_name": "Omni-MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Acc on Omni-MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.416, - "details": { - "description": "min=0.416, mean=0.416, max=0.416, sum=0.416 (1)", - "tab": "Accuracy", - "Omni-MATH - Observed inference time (s)": "{\"description\": \"min=85.77, mean=85.77, max=85.77, sum=85.77 (1)\", \"tab\": \"Efficiency\", \"score\": \"85.77000450229644\"}", - "Omni-MATH - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "Omni-MATH - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - # prompt tokens": "{\"description\": \"min=111.956, mean=111.956, max=111.956, sum=111.956 (1)\", \"tab\": \"General information\", \"score\": \"111.956\"}", - "Omni-MATH - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/google_gemini-2.5-pro-preview-05-06.json b/data/models/google_gemini-2.5-pro-preview-05-06.json deleted file mode 100644 index 369b2477bcac4922bf08e26093ce76aea9d43f5e..0000000000000000000000000000000000000000 --- a/data/models/google_gemini-2.5-pro-preview-05-06.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "google/gemini-2.5-pro-preview-05-06", - "id": "google/gemini-2.5-pro-preview-05-06", - "developer": "Google", - "additional_details": { - "model_type": "Generative RM" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/google_gemini-2.5-pro-preview-05-06/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6775 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6532 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4688 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5342 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8806 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8308 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6973 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/google_gemini-2.5-pro.json b/data/models/google_gemini-2.5-pro.json deleted file mode 100644 index b34ade10547caf33660eff5315c55c36a8077889..0000000000000000000000000000000000000000 --- a/data/models/google_gemini-2.5-pro.json +++ /dev/null @@ -1,1568 +0,0 @@ -{ - "model_info": { - "name": "Gemini 2.5 Pro", - "id": "google/gemini-2.5-pro", - "developer": "Google", - "inference_platform": "unknown", - "additional_details": { - "display_name": "Gemini 2.5 Pro" - } - }, - "evaluations": [ - { - "evaluation_id": "global-mmlu-lite/google_gemini-2.5-pro/1773936496.366405", - "retrieved_timestamp": "1773936496.366405", - "source_metadata": { - "source_name": "Global MMLU Lite Leaderboard", - "source_type": "documentation", - "source_organization_name": "kaggle", - "source_organization_url": "www.kaggle.com", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "kaggle kernel", - "version": "4", - "additional_details": { - "url": "https://www.kaggle.com/code/shivalikasingh95/global-mmlu-lite-sample-notebook" - } - }, - "benchmark": "global-mmlu-lite", - "evaluation_results": [ - { - "evaluation_name": "Global MMLU Lite", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Global MMLU Lite", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9323 - } - }, - { - "evaluation_name": "Culturally Sensitive", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Sensitive", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9241 - } - }, - { - "evaluation_name": "Culturally Agnostic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Agnostic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9406 - } - }, - { - "evaluation_name": "Arabic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Arabic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9475, - "uncertainty": { - "confidence_interval": { - "lower": -0.0219, - "upper": 0.0219, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "English", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - English", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9275, - "uncertainty": { - "confidence_interval": { - "lower": -0.0254, - "upper": 0.0254, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Bengali", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Bengali", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9275, - "uncertainty": { - "confidence_interval": { - "lower": -0.0254, - "upper": 0.0254, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "German", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - German", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.93, - "uncertainty": { - "confidence_interval": { - "lower": -0.025, - "upper": 0.025, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "French", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - French", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9425, - "uncertainty": { - "confidence_interval": { - "lower": -0.0228, - "upper": 0.0228, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Hindi", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Hindi", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9275, - "uncertainty": { - "confidence_interval": { - "lower": -0.0254, - "upper": 0.0254, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Indonesian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Indonesian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.925, - "uncertainty": { - "confidence_interval": { - "lower": -0.0258, - "upper": 0.0258, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Italian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Italian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.935, - "uncertainty": { - "confidence_interval": { - "lower": -0.0242, - "upper": 0.0242, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Japanese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Japanese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9375, - "uncertainty": { - "confidence_interval": { - "lower": -0.0237, - "upper": 0.0237, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Korean", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Korean", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9275, - "uncertainty": { - "confidence_interval": { - "lower": -0.0254, - "upper": 0.0254, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Portuguese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Portuguese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.93, - "uncertainty": { - "confidence_interval": { - "lower": -0.025, - "upper": 0.025, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Spanish", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Spanish", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.94, - "uncertainty": { - "confidence_interval": { - "lower": -0.0233, - "upper": 0.0233, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Swahili", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Swahili", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9375, - "uncertainty": { - "confidence_interval": { - "lower": -0.0237, - "upper": 0.0237, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Yoruba", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Yoruba", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.925, - "uncertainty": { - "confidence_interval": { - "lower": -0.0258, - "upper": 0.0258, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Chinese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Chinese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9275, - "uncertainty": { - "confidence_interval": { - "lower": -0.0254, - "upper": 0.0254, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Burmese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Burmese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.93, - "uncertainty": { - "confidence_interval": { - "lower": -0.025, - "upper": 0.025, - "method": "unknown" - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "global-mmlu-lite/google_gemini-2.5-pro/1773936583.743359", - "retrieved_timestamp": "1773936583.743359", - "source_metadata": { - "source_name": "Global MMLU Lite Leaderboard", - "source_type": "documentation", - "source_organization_name": "kaggle", - "source_organization_url": "www.kaggle.com", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "kaggle kernel", - "version": "4", - "additional_details": { - "url": "https://www.kaggle.com/code/shivalikasingh95/global-mmlu-lite-sample-notebook" - } - }, - "benchmark": "global-mmlu-lite", - "evaluation_results": [ - { - "evaluation_name": "Global MMLU Lite", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Global MMLU Lite", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9323 - } - }, - { - "evaluation_name": "Culturally Sensitive", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Sensitive", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9241 - } - }, - { - "evaluation_name": "Culturally Agnostic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Agnostic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9406 - } - }, - { - "evaluation_name": "Arabic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Arabic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9475, - "uncertainty": { - "confidence_interval": { - "lower": -0.0219, - "upper": 0.0219, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "English", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - English", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9275, - "uncertainty": { - "confidence_interval": { - "lower": -0.0254, - "upper": 0.0254, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Bengali", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Bengali", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9275, - "uncertainty": { - "confidence_interval": { - "lower": -0.0254, - "upper": 0.0254, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "German", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - German", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.93, - "uncertainty": { - "confidence_interval": { - "lower": -0.025, - "upper": 0.025, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "French", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - French", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9425, - "uncertainty": { - "confidence_interval": { - "lower": -0.0228, - "upper": 0.0228, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Hindi", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Hindi", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9275, - "uncertainty": { - "confidence_interval": { - "lower": -0.0254, - "upper": 0.0254, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Indonesian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Indonesian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.925, - "uncertainty": { - "confidence_interval": { - "lower": -0.0258, - "upper": 0.0258, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Italian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Italian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.935, - "uncertainty": { - "confidence_interval": { - "lower": -0.0242, - "upper": 0.0242, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Japanese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Japanese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9375, - "uncertainty": { - "confidence_interval": { - "lower": -0.0237, - "upper": 0.0237, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Korean", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Korean", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9275, - "uncertainty": { - "confidence_interval": { - "lower": -0.0254, - "upper": 0.0254, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Portuguese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Portuguese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.93, - "uncertainty": { - "confidence_interval": { - "lower": -0.025, - "upper": 0.025, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Spanish", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Spanish", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.94, - "uncertainty": { - "confidence_interval": { - "lower": -0.0233, - "upper": 0.0233, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Swahili", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Swahili", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9375, - "uncertainty": { - "confidence_interval": { - "lower": -0.0237, - "upper": 0.0237, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Yoruba", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Yoruba", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.925, - "uncertainty": { - "confidence_interval": { - "lower": -0.0258, - "upper": 0.0258, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Chinese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Chinese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9275, - "uncertainty": { - "confidence_interval": { - "lower": -0.0254, - "upper": 0.0254, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Burmese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Burmese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.93, - "uncertainty": { - "confidence_interval": { - "lower": -0.025, - "upper": 0.025, - "method": "unknown" - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "livecodebenchpro/gemini-2.5-pro/1760492095.8105888", - "retrieved_timestamp": "1760492095.8105888", - "source_metadata": { - "source_organization_name": "New York University, Princeton University, University of California San Diego, University of Washington and Canyon Crest Academy", - "evaluator_relationship": "third_party", - "source_name": "Live Code Bench Pro", - "source_type": "documentation" - }, - "eval_library": { - "name": "unknown", - "version": "unknown" - }, - "benchmark": "livecodebenchpro", - "evaluation_results": [ - { - "evaluation_name": "Hard Problems", - "metric_config": { - "evaluation_description": "Pass@1 on Hard Problems", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.014084507042253521 - }, - "source_data": { - "dataset_name": "Hard Problems", - "source_type": "url", - "url": [ - "https://webhook.cp-bench.orzzh.com/leaderboard/llm/difficulty?difficulty=hard&benchmark_mode=live" - ] - } - }, - { - "evaluation_name": "Medium Problems", - "metric_config": { - "evaluation_description": "Pass@1 on Medium Problems", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.2112676056338028 - }, - "source_data": { - "dataset_name": "Medium Problems", - "source_type": "url", - "url": [ - "https://webhook.cp-bench.orzzh.com/leaderboard/llm/difficulty?difficulty=medium&benchmark_mode=live" - ] - } - }, - { - "evaluation_name": "Easy Problems", - "metric_config": { - "evaluation_description": "Pass@1 on Easy Problems", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.7183098591549296 - }, - "source_data": { - "dataset_name": "Easy Problems", - "source_type": "url", - "url": [ - "https://webhook.cp-bench.orzzh.com/leaderboard/llm/difficulty?difficulty=easy&benchmark_mode=live" - ] - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "reward-bench-2/google_gemini-2.5-pro/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7948 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.755 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.619 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.898 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.881 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.805 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.811 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "terminal-bench-2.0/terminus-2__gemini-2.5-pro/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2025-10-31", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 32.6, - "uncertainty": { - "standard_error": { - "value": 3.0 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Terminus 2\" -m \"Gemini 2.5 Pro\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Terminus 2\" -m \"Gemini 2.5 Pro\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - }, - { - "evaluation_id": "terminal-bench-2.0/openhands__gemini-2.5-pro/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2025-11-02", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 16.4, - "uncertainty": { - "standard_error": { - "value": 2.8 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"OpenHands\" -m \"Gemini 2.5 Pro\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"OpenHands\" -m \"Gemini 2.5 Pro\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - }, - { - "evaluation_id": "terminal-bench-2.0/mini-swe-agent__gemini-2.5-pro/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2025-11-03", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 26.1, - "uncertainty": { - "standard_error": { - "value": 2.5 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Mini-SWE-Agent\" -m \"Gemini 2.5 Pro\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Mini-SWE-Agent\" -m \"Gemini 2.5 Pro\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - }, - { - "evaluation_id": "terminal-bench-2.0/gemini-cli__gemini-2.5-pro/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2025-11-04", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 19.6, - "uncertainty": { - "standard_error": { - "value": 2.9 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Gemini CLI\" -m \"Gemini 2.5 Pro\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Gemini CLI\" -m \"Gemini 2.5 Pro\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/google_gemini-3-flash.json b/data/models/google_gemini-3-flash.json deleted file mode 100644 index dfabab89891de12eb115e192068781bd51456d1f..0000000000000000000000000000000000000000 --- a/data/models/google_gemini-3-flash.json +++ /dev/null @@ -1,309 +0,0 @@ -{ - "model_info": { - "name": "Gemini 3 Flash", - "id": "google/gemini-3-flash", - "developer": "Google", - "additional_details": { - "agent_name": "Junie CLI", - "agent_organization": "JetBrains" - } - }, - "evaluations": [ - { - "evaluation_id": "terminal-bench-2.0/junie-cli__gemini-3-flash/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2025-12-23", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 64.3, - "uncertainty": { - "standard_error": { - "value": 2.8 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Junie CLI\" -m \"Gemini 3 Flash\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Junie CLI\" -m \"Gemini 3 Flash\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - }, - { - "evaluation_id": "terminal-bench-2.0/gemini-cli__gemini-3-flash/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2025-12-23", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 51.0, - "uncertainty": { - "standard_error": { - "value": 3.0 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Gemini CLI\" -m \"Gemini 3 Flash\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Gemini CLI\" -m \"Gemini 3 Flash\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - }, - { - "evaluation_id": "terminal-bench-2.0/terminus-2__gemini-3-flash/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2026-01-07", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 51.7, - "uncertainty": { - "standard_error": { - "value": 3.1 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Terminus 2\" -m \"Gemini 3 Flash\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Terminus 2\" -m \"Gemini 3 Flash\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - }, - { - "evaluation_id": "terminal-bench-2.0/gemini-cli__gemini-3-flash/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2026-03-06", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 47.4, - "uncertainty": { - "standard_error": { - "value": 3.0 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Gemini CLI\" -m \"Gemini 3 Flash\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Gemini CLI\" -m \"Gemini 3 Flash\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/google_gemini-3-pro-preview-fc.json b/data/models/google_gemini-3-pro-preview-fc.json deleted file mode 100644 index 5686f22309956b5c4b1002ee86258d45a4bf5aeb..0000000000000000000000000000000000000000 --- a/data/models/google_gemini-3-pro-preview-fc.json +++ /dev/null @@ -1,851 +0,0 @@ -{ - "model_info": { - "name": "Gemini-3-Pro-Preview (FC)", - "id": "google/gemini-3-pro-preview-fc", - "developer": "Google", - "additional_details": { - "raw_model_name": "Gemini-3-Pro-Preview (FC)", - "organization": "Google", - "license": "Proprietary", - "mode": "FC", - "model_link": "https://deepmind.google/technologies/gemini/pro/" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/google/gemini-3-pro-preview-fc/1775236112.369081", - "retrieved_timestamp": "1775236112.369081", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 7.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 68.14 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 224.69 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 15.87 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 41.41 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 58.48 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 85.75 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 75.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 94.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 91.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 82.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 81.72 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 87.6 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 80.44 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 75.0 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 79.17 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 63.12 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 69.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 63.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 56.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 64.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 68.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 63.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 74.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 54.84 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 50.32 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 63.23 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 50.97 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 75.0 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 77.85 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/google_gemini-3-pro-preview-prompt.json b/data/models/google_gemini-3-pro-preview-prompt.json deleted file mode 100644 index a1dcf445c0cf78e72fe19a9a28805f7c46e1b7ce..0000000000000000000000000000000000000000 --- a/data/models/google_gemini-3-pro-preview-prompt.json +++ /dev/null @@ -1,905 +0,0 @@ -{ - "model_info": { - "name": "Gemini-3-Pro-Preview (Prompt)", - "id": "google/gemini-3-pro-preview-prompt", - "developer": "Google", - "additional_details": { - "raw_model_name": "Gemini-3-Pro-Preview (Prompt)", - "organization": "Google", - "license": "Proprietary", - "mode": "Prompt", - "model_link": "https://deepmind.google/technologies/gemini/pro/" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/google/gemini-3-pro-preview-prompt/1775236112.36698", - "retrieved_timestamp": "1775236112.36698", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 3.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 72.51 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 298.47 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 12.08 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 21.3 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 32.73 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 90.65 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 79.58 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 96.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 95.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 92.0 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 83.12 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 87.6 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 81.77 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 93.75 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 87.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 60.75 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 64.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 60.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 54.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 64.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 80.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 78.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 82.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 61.72 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 59.35 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 62.58 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 63.23 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 68.75 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 85.59 - } - }, - { - "evaluation_result_id": "bfcl.format_sensitivity.max_delta::max_delta", - "evaluation_name": "bfcl.format_sensitivity.max_delta", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.format_sensitivity.max_delta", - "metric_name": "Format sensitivity max delta", - "metric_kind": "difference", - "metric_unit": "percentage_points", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Format Sensitivity Max Delta" - } - }, - "score_details": { - "score": 8.5 - } - }, - { - "evaluation_result_id": "bfcl.format_sensitivity.stddev::stddev", - "evaluation_name": "bfcl.format_sensitivity.stddev", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.format_sensitivity.stddev", - "metric_name": "Format sensitivity standard deviation", - "metric_kind": "difference", - "metric_unit": "percentage_points", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Format Sensitivity Standard Deviation" - } - }, - "score_details": { - "score": 1.7 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/google_gemini-3-pro-preview.json b/data/models/google_gemini-3-pro-preview.json deleted file mode 100644 index 3abfabb59d0f5737518aaeeccfa5955d66ed9e1b..0000000000000000000000000000000000000000 --- a/data/models/google_gemini-3-pro-preview.json +++ /dev/null @@ -1,3083 +0,0 @@ -{ - "model_info": { - "name": "gemini-3-pro-preview", - "id": "google/gemini-3-pro-preview", - "developer": "Google", - "additional_details": { - "agent_name": "SmolAgents Code", - "agent_framework": "smolagents_code" - } - }, - "evaluations": [ - { - "evaluation_id": "appworld/test_normal/smolagents-code__google_gemini-3-pro-preview/1774263615.0201504", - "retrieved_timestamp": "1774263615.0201504", - "source_metadata": { - "source_name": "Exgentic Open Agent Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Exgentic", - "source_organization_url": "https://github.com/Exgentic", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "exgentic", - "version": "0.1.0" - }, - "benchmark": "appworld_test_normal", - "evaluation_results": [ - { - "evaluation_name": "appworld/test_normal", - "source_data": { - "dataset_name": "appworld/test_normal", - "source_type": "url", - "url": [ - "https://github.com/Exgentic/exgentic" - ] - }, - "metric_config": { - "evaluation_description": "AppWorld benchmark evaluation (test_normal subset)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.13, - "uncertainty": { - "num_samples": 100 - }, - "details": { - "average_agent_cost": "2.54", - "total_run_cost": "254.25", - "average_steps": "49.13", - "percent_finished": "0.71" - } - }, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "SmolAgents Code", - "agent_framework": "smolagents_code" - } - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "SmolAgents Code", - "agent_framework": "smolagents_code" - } - } - } - } - }, - { - "evaluation_id": "appworld/test_normal/openai-solo__google_gemini-3-pro-preview/1774263615.0201504", - "retrieved_timestamp": "1774263615.0201504", - "source_metadata": { - "source_name": "Exgentic Open Agent Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Exgentic", - "source_organization_url": "https://github.com/Exgentic", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "exgentic", - "version": "0.1.0" - }, - "benchmark": "appworld_test_normal", - "evaluation_results": [ - { - "evaluation_name": "appworld/test_normal", - "source_data": { - "dataset_name": "appworld/test_normal", - "source_type": "url", - "url": [ - "https://github.com/Exgentic/exgentic" - ] - }, - "metric_config": { - "evaluation_description": "AppWorld benchmark evaluation (test_normal subset)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.582, - "uncertainty": { - "num_samples": 100 - }, - "details": { - "average_agent_cost": "8.7", - "total_run_cost": "869.55", - "average_steps": "33.49", - "percent_finished": "0.98" - } - }, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "OpenAI Solo", - "agent_framework": "openai_solo" - } - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "OpenAI Solo", - "agent_framework": "openai_solo" - } - } - } - } - }, - { - "evaluation_id": "appworld/test_normal/litellm-tool-calling-with-shortlisting__google_gemini-3-pro-preview/1774263615.0201504", - "retrieved_timestamp": "1774263615.0201504", - "source_metadata": { - "source_name": "Exgentic Open Agent Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Exgentic", - "source_organization_url": "https://github.com/Exgentic", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "exgentic", - "version": "0.1.0" - }, - "benchmark": "appworld_test_normal", - "evaluation_results": [ - { - "evaluation_name": "appworld/test_normal", - "source_data": { - "dataset_name": "appworld/test_normal", - "source_type": "url", - "url": [ - "https://github.com/Exgentic/exgentic" - ] - }, - "metric_config": { - "evaluation_description": "AppWorld benchmark evaluation (test_normal subset)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.55, - "uncertainty": { - "num_samples": 100 - }, - "details": { - "average_agent_cost": "1.3", - "total_run_cost": "130.49", - "average_steps": "22.59", - "percent_finished": "1.0" - } - }, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "LiteLLM Tool Calling with Shortlisting", - "agent_framework": "tool_calling_with_shortlisting" - } - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "LiteLLM Tool Calling with Shortlisting", - "agent_framework": "tool_calling_with_shortlisting" - } - } - } - } - }, - { - "evaluation_id": "appworld/test_normal/claude-code-cli__google_gemini-3-pro-preview/1774263615.0201504", - "retrieved_timestamp": "1774263615.0201504", - "source_metadata": { - "source_name": "Exgentic Open Agent Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Exgentic", - "source_organization_url": "https://github.com/Exgentic", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "exgentic", - "version": "0.1.0" - }, - "benchmark": "appworld_test_normal", - "evaluation_results": [ - { - "evaluation_name": "appworld/test_normal", - "source_data": { - "dataset_name": "appworld/test_normal", - "source_type": "url", - "url": [ - "https://github.com/Exgentic/exgentic" - ] - }, - "metric_config": { - "evaluation_description": "AppWorld benchmark evaluation (test_normal subset)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.36, - "uncertainty": { - "num_samples": 100 - }, - "details": { - "average_agent_cost": "3.11", - "total_run_cost": "310.55", - "average_steps": "38.01", - "percent_finished": "0.86" - } - }, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "Claude Code CLI", - "agent_framework": "claude_code" - } - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "Claude Code CLI", - "agent_framework": "claude_code" - } - } - } - } - }, - { - "evaluation_id": "appworld/test_normal/litellm-tool-calling__google_gemini-3-pro-preview/1774263615.0201504", - "retrieved_timestamp": "1774263615.0201504", - "source_metadata": { - "source_name": "Exgentic Open Agent Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Exgentic", - "source_organization_url": "https://github.com/Exgentic", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "exgentic", - "version": "0.1.0" - }, - "benchmark": "appworld_test_normal", - "evaluation_results": [ - { - "evaluation_name": "appworld/test_normal", - "source_data": { - "dataset_name": "appworld/test_normal", - "source_type": "url", - "url": [ - "https://github.com/Exgentic/exgentic" - ] - }, - "metric_config": { - "evaluation_description": "AppWorld benchmark evaluation (test_normal subset)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.505, - "uncertainty": { - "num_samples": 100 - }, - "details": { - "average_agent_cost": "1.88", - "total_run_cost": "188.19", - "average_steps": "21.76", - "percent_finished": "0.99" - } - }, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "LiteLLM Tool Calling", - "agent_framework": "tool_calling" - } - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "LiteLLM Tool Calling", - "agent_framework": "tool_calling" - } - } - } - } - }, - { - "evaluation_id": "browsecompplus/openai-solo__google_gemini-3-pro-preview/1774263615.0201504", - "retrieved_timestamp": "1774263615.0201504", - "source_metadata": { - "source_name": "Exgentic Open Agent Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Exgentic", - "source_organization_url": "https://github.com/Exgentic", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "exgentic", - "version": "0.1.0" - }, - "benchmark": "browsecompplus", - "evaluation_results": [ - { - "evaluation_name": "browsecompplus", - "source_data": { - "dataset_name": "browsecompplus", - "source_type": "url", - "url": [ - "https://github.com/Exgentic/exgentic" - ] - }, - "metric_config": { - "evaluation_description": "BrowseCompPlus benchmark evaluation", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3333, - "uncertainty": { - "num_samples": 99 - }, - "details": { - "average_agent_cost": "0.64", - "total_run_cost": "63.79", - "average_steps": "8.45", - "percent_finished": "0.6061" - } - }, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "OpenAI Solo", - "agent_framework": "openai_solo" - } - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "OpenAI Solo", - "agent_framework": "openai_solo" - } - } - } - } - }, - { - "evaluation_id": "browsecompplus/claude-code-cli__google_gemini-3-pro-preview/1774263615.0201504", - "retrieved_timestamp": "1774263615.0201504", - "source_metadata": { - "source_name": "Exgentic Open Agent Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Exgentic", - "source_organization_url": "https://github.com/Exgentic", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "exgentic", - "version": "0.1.0" - }, - "benchmark": "browsecompplus", - "evaluation_results": [ - { - "evaluation_name": "browsecompplus", - "source_data": { - "dataset_name": "browsecompplus", - "source_type": "url", - "url": [ - "https://github.com/Exgentic/exgentic" - ] - }, - "metric_config": { - "evaluation_description": "BrowseCompPlus benchmark evaluation", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.51, - "uncertainty": { - "num_samples": 100 - }, - "details": { - "average_agent_cost": "2.85", - "total_run_cost": "284.68", - "average_steps": "22.88", - "percent_finished": "0.7" - } - }, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "Claude Code CLI", - "agent_framework": "claude_code" - } - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "Claude Code CLI", - "agent_framework": "claude_code" - } - } - } - } - }, - { - "evaluation_id": "browsecompplus/smolagents-code__google_gemini-3-pro-preview/1774263615.0201504", - "retrieved_timestamp": "1774263615.0201504", - "source_metadata": { - "source_name": "Exgentic Open Agent Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Exgentic", - "source_organization_url": "https://github.com/Exgentic", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "exgentic", - "version": "0.1.0" - }, - "benchmark": "browsecompplus", - "evaluation_results": [ - { - "evaluation_name": "browsecompplus", - "source_data": { - "dataset_name": "browsecompplus", - "source_type": "url", - "url": [ - "https://github.com/Exgentic/exgentic" - ] - }, - "metric_config": { - "evaluation_description": "BrowseCompPlus benchmark evaluation", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.57, - "uncertainty": { - "num_samples": 100 - }, - "details": { - "average_agent_cost": "2.39", - "total_run_cost": "239.0", - "average_steps": "29.63", - "percent_finished": "0.69" - } - }, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "SmolAgents Code", - "agent_framework": "smolagents_code" - } - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "SmolAgents Code", - "agent_framework": "smolagents_code" - } - } - } - } - }, - { - "evaluation_id": "browsecompplus/litellm-tool-calling__google_gemini-3-pro-preview/1774263615.0201504", - "retrieved_timestamp": "1774263615.0201504", - "source_metadata": { - "source_name": "Exgentic Open Agent Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Exgentic", - "source_organization_url": "https://github.com/Exgentic", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "exgentic", - "version": "0.1.0" - }, - "benchmark": "browsecompplus", - "evaluation_results": [ - { - "evaluation_name": "browsecompplus", - "source_data": { - "dataset_name": "browsecompplus", - "source_type": "url", - "url": [ - "https://github.com/Exgentic/exgentic" - ] - }, - "metric_config": { - "evaluation_description": "BrowseCompPlus benchmark evaluation", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.48, - "uncertainty": { - "num_samples": 100 - }, - "details": { - "average_agent_cost": "0.44", - "total_run_cost": "44.18", - "average_steps": "7.85", - "percent_finished": "0.99" - } - }, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "LiteLLM Tool Calling", - "agent_framework": "tool_calling" - } - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "LiteLLM Tool Calling", - "agent_framework": "tool_calling" - } - } - } - } - }, - { - "evaluation_id": "browsecompplus/litellm-tool-calling-with-shortlisting__google_gemini-3-pro-preview/1774263615.0201504", - "retrieved_timestamp": "1774263615.0201504", - "source_metadata": { - "source_name": "Exgentic Open Agent Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Exgentic", - "source_organization_url": "https://github.com/Exgentic", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "exgentic", - "version": "0.1.0" - }, - "benchmark": "browsecompplus", - "evaluation_results": [ - { - "evaluation_name": "browsecompplus", - "source_data": { - "dataset_name": "browsecompplus", - "source_type": "url", - "url": [ - "https://github.com/Exgentic/exgentic" - ] - }, - "metric_config": { - "evaluation_description": "BrowseCompPlus benchmark evaluation", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.48, - "uncertainty": { - "num_samples": 100 - }, - "details": { - "average_agent_cost": "0.44", - "total_run_cost": "44.18", - "average_steps": "7.85", - "percent_finished": "0.99" - } - }, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "LiteLLM Tool Calling with Shortlisting", - "agent_framework": "tool_calling_with_shortlisting" - } - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "LiteLLM Tool Calling with Shortlisting", - "agent_framework": "tool_calling_with_shortlisting" - } - } - } - } - }, - { - "evaluation_id": "global-mmlu-lite/google_gemini-3-pro-preview/1773936583.743359", - "retrieved_timestamp": "1773936583.743359", - "source_metadata": { - "source_name": "Global MMLU Lite Leaderboard", - "source_type": "documentation", - "source_organization_name": "kaggle", - "source_organization_url": "www.kaggle.com", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "kaggle kernel", - "version": "4", - "additional_details": { - "url": "https://www.kaggle.com/code/shivalikasingh95/global-mmlu-lite-sample-notebook" - } - }, - "benchmark": "global-mmlu-lite", - "evaluation_results": [ - { - "evaluation_name": "Global MMLU Lite", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Global MMLU Lite", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9453 - } - }, - { - "evaluation_name": "Culturally Sensitive", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Sensitive", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9397 - } - }, - { - "evaluation_name": "Culturally Agnostic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Agnostic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9509 - } - }, - { - "evaluation_name": "Arabic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Arabic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9475, - "uncertainty": { - "confidence_interval": { - "lower": -0.0219, - "upper": 0.0219, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "English", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - English", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9425, - "uncertainty": { - "confidence_interval": { - "lower": -0.0228, - "upper": 0.0228, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Bengali", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Bengali", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9425, - "uncertainty": { - "confidence_interval": { - "lower": -0.0228, - "upper": 0.0228, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "German", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - German", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.94, - "uncertainty": { - "confidence_interval": { - "lower": -0.0233, - "upper": 0.0233, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "French", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - French", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9575, - "uncertainty": { - "confidence_interval": { - "lower": -0.0198, - "upper": 0.0198, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Hindi", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Hindi", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9425, - "uncertainty": { - "confidence_interval": { - "lower": -0.0228, - "upper": 0.0228, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Indonesian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Indonesian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.955, - "uncertainty": { - "confidence_interval": { - "lower": -0.0203, - "upper": 0.0203, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Italian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Italian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.955, - "uncertainty": { - "confidence_interval": { - "lower": -0.0203, - "upper": 0.0203, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Japanese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Japanese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.94, - "uncertainty": { - "confidence_interval": { - "lower": -0.0233, - "upper": 0.0233, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Korean", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Korean", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.94, - "uncertainty": { - "confidence_interval": { - "lower": -0.0233, - "upper": 0.0233, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Portuguese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Portuguese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9425, - "uncertainty": { - "confidence_interval": { - "lower": -0.0228, - "upper": 0.0228, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Spanish", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Spanish", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9475, - "uncertainty": { - "confidence_interval": { - "lower": -0.0219, - "upper": 0.0219, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Swahili", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Swahili", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.94, - "uncertainty": { - "confidence_interval": { - "lower": -0.0233, - "upper": 0.0233, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Yoruba", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Yoruba", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9425, - "uncertainty": { - "confidence_interval": { - "lower": -0.0228, - "upper": 0.0228, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Chinese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Chinese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9475, - "uncertainty": { - "confidence_interval": { - "lower": -0.0219, - "upper": 0.0219, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Burmese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Burmese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9425, - "uncertainty": { - "confidence_interval": { - "lower": -0.0228, - "upper": 0.0228, - "method": "unknown" - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "global-mmlu-lite/google_gemini-3-pro-preview/1773936496.366405", - "retrieved_timestamp": "1773936496.366405", - "source_metadata": { - "source_name": "Global MMLU Lite Leaderboard", - "source_type": "documentation", - "source_organization_name": "kaggle", - "source_organization_url": "www.kaggle.com", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "kaggle kernel", - "version": "4", - "additional_details": { - "url": "https://www.kaggle.com/code/shivalikasingh95/global-mmlu-lite-sample-notebook" - } - }, - "benchmark": "global-mmlu-lite", - "evaluation_results": [ - { - "evaluation_name": "Global MMLU Lite", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Global MMLU Lite", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9453 - } - }, - { - "evaluation_name": "Culturally Sensitive", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Sensitive", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9397 - } - }, - { - "evaluation_name": "Culturally Agnostic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Agnostic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9509 - } - }, - { - "evaluation_name": "Arabic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Arabic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9475, - "uncertainty": { - "confidence_interval": { - "lower": -0.0219, - "upper": 0.0219, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "English", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - English", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9425, - "uncertainty": { - "confidence_interval": { - "lower": -0.0228, - "upper": 0.0228, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Bengali", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Bengali", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9425, - "uncertainty": { - "confidence_interval": { - "lower": -0.0228, - "upper": 0.0228, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "German", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - German", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.94, - "uncertainty": { - "confidence_interval": { - "lower": -0.0233, - "upper": 0.0233, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "French", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - French", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9575, - "uncertainty": { - "confidence_interval": { - "lower": -0.0198, - "upper": 0.0198, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Hindi", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Hindi", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9425, - "uncertainty": { - "confidence_interval": { - "lower": -0.0228, - "upper": 0.0228, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Indonesian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Indonesian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.955, - "uncertainty": { - "confidence_interval": { - "lower": -0.0203, - "upper": 0.0203, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Italian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Italian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.955, - "uncertainty": { - "confidence_interval": { - "lower": -0.0203, - "upper": 0.0203, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Japanese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Japanese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.94, - "uncertainty": { - "confidence_interval": { - "lower": -0.0233, - "upper": 0.0233, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Korean", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Korean", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.94, - "uncertainty": { - "confidence_interval": { - "lower": -0.0233, - "upper": 0.0233, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Portuguese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Portuguese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9425, - "uncertainty": { - "confidence_interval": { - "lower": -0.0228, - "upper": 0.0228, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Spanish", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Spanish", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9475, - "uncertainty": { - "confidence_interval": { - "lower": -0.0219, - "upper": 0.0219, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Swahili", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Swahili", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.94, - "uncertainty": { - "confidence_interval": { - "lower": -0.0233, - "upper": 0.0233, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Yoruba", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Yoruba", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9425, - "uncertainty": { - "confidence_interval": { - "lower": -0.0228, - "upper": 0.0228, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Chinese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Chinese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9475, - "uncertainty": { - "confidence_interval": { - "lower": -0.0219, - "upper": 0.0219, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Burmese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Burmese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9425, - "uncertainty": { - "confidence_interval": { - "lower": -0.0228, - "upper": 0.0228, - "method": "unknown" - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "swe-bench/litellm-tool-calling__google_gemini-3-pro-preview/1774263615.0201504", - "retrieved_timestamp": "1774263615.0201504", - "source_metadata": { - "source_name": "Exgentic Open Agent Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Exgentic", - "source_organization_url": "https://github.com/Exgentic", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "exgentic", - "version": "0.1.0" - }, - "benchmark": "swe-bench", - "evaluation_results": [ - { - "evaluation_name": "swe-bench", - "source_data": { - "dataset_name": "swe-bench", - "source_type": "url", - "url": [ - "https://github.com/Exgentic/exgentic" - ] - }, - "metric_config": { - "evaluation_description": "SWE-bench benchmark evaluation", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.71, - "uncertainty": { - "num_samples": 100 - }, - "details": { - "average_agent_cost": "0.7", - "total_run_cost": "69.56", - "average_steps": "32.55", - "percent_finished": "1.0" - } - }, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "LiteLLM Tool Calling", - "agent_framework": "tool_calling" - } - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "LiteLLM Tool Calling", - "agent_framework": "tool_calling" - } - } - } - } - }, - { - "evaluation_id": "swe-bench/claude-code-cli__google_gemini-3-pro-preview/1774263615.0201504", - "retrieved_timestamp": "1774263615.0201504", - "source_metadata": { - "source_name": "Exgentic Open Agent Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Exgentic", - "source_organization_url": "https://github.com/Exgentic", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "exgentic", - "version": "0.1.0" - }, - "benchmark": "swe-bench", - "evaluation_results": [ - { - "evaluation_name": "swe-bench", - "source_data": { - "dataset_name": "swe-bench", - "source_type": "url", - "url": [ - "https://github.com/Exgentic/exgentic" - ] - }, - "metric_config": { - "evaluation_description": "SWE-bench benchmark evaluation", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.67, - "uncertainty": { - "num_samples": 100 - }, - "details": { - "average_agent_cost": "3.68", - "total_run_cost": "367.97", - "average_steps": "43.72", - "percent_finished": "1.0" - } - }, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "Claude Code CLI", - "agent_framework": "claude_code" - } - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "Claude Code CLI", - "agent_framework": "claude_code" - } - } - } - } - }, - { - "evaluation_id": "swe-bench/openai-solo__google_gemini-3-pro-preview/1774263615.0201504", - "retrieved_timestamp": "1774263615.0201504", - "source_metadata": { - "source_name": "Exgentic Open Agent Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Exgentic", - "source_organization_url": "https://github.com/Exgentic", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "exgentic", - "version": "0.1.0" - }, - "benchmark": "swe-bench", - "evaluation_results": [ - { - "evaluation_name": "swe-bench", - "source_data": { - "dataset_name": "swe-bench", - "source_type": "url", - "url": [ - "https://github.com/Exgentic/exgentic" - ] - }, - "metric_config": { - "evaluation_description": "SWE-bench benchmark evaluation", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7234, - "uncertainty": { - "num_samples": 94 - }, - "details": { - "average_agent_cost": "1.58", - "total_run_cost": "148.44", - "average_steps": "32.36", - "percent_finished": "1.0" - } - }, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "OpenAI Solo", - "agent_framework": "openai_solo" - } - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "OpenAI Solo", - "agent_framework": "openai_solo" - } - } - } - } - }, - { - "evaluation_id": "swe-bench/smolagents-code__google_gemini-3-pro-preview/1774263615.0201504", - "retrieved_timestamp": "1774263615.0201504", - "source_metadata": { - "source_name": "Exgentic Open Agent Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Exgentic", - "source_organization_url": "https://github.com/Exgentic", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "exgentic", - "version": "0.1.0" - }, - "benchmark": "swe-bench", - "evaluation_results": [ - { - "evaluation_name": "swe-bench", - "source_data": { - "dataset_name": "swe-bench", - "source_type": "url", - "url": [ - "https://github.com/Exgentic/exgentic" - ] - }, - "metric_config": { - "evaluation_description": "SWE-bench benchmark evaluation", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7576, - "uncertainty": { - "num_samples": 99 - }, - "details": { - "average_agent_cost": "2.21", - "total_run_cost": "218.76", - "average_steps": "38.1", - "percent_finished": "1.0" - } - }, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "SmolAgents Code", - "agent_framework": "smolagents_code" - } - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "SmolAgents Code", - "agent_framework": "smolagents_code" - } - } - } - } - }, - { - "evaluation_id": "swe-bench/litellm-tool-calling-with-shortlisting__google_gemini-3-pro-preview/1774263615.0201504", - "retrieved_timestamp": "1774263615.0201504", - "source_metadata": { - "source_name": "Exgentic Open Agent Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Exgentic", - "source_organization_url": "https://github.com/Exgentic", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "exgentic", - "version": "0.1.0" - }, - "benchmark": "swe-bench", - "evaluation_results": [ - { - "evaluation_name": "swe-bench", - "source_data": { - "dataset_name": "swe-bench", - "source_type": "url", - "url": [ - "https://github.com/Exgentic/exgentic" - ] - }, - "metric_config": { - "evaluation_description": "SWE-bench benchmark evaluation", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.71, - "uncertainty": { - "num_samples": 100 - }, - "details": { - "average_agent_cost": "0.7", - "total_run_cost": "69.56", - "average_steps": "32.55", - "percent_finished": "1.0" - } - }, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "LiteLLM Tool Calling with Shortlisting", - "agent_framework": "tool_calling_with_shortlisting" - } - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "LiteLLM Tool Calling with Shortlisting", - "agent_framework": "tool_calling_with_shortlisting" - } - } - } - } - }, - { - "evaluation_id": "tau-bench-2/airline/litellm-tool-calling__google_gemini-3-pro-preview/1774263615.0201504", - "retrieved_timestamp": "1774263615.0201504", - "source_metadata": { - "source_name": "Exgentic Open Agent Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Exgentic", - "source_organization_url": "https://github.com/Exgentic", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "exgentic", - "version": "0.1.0" - }, - "benchmark": "tau-bench-2_airline", - "evaluation_results": [ - { - "evaluation_name": "tau-bench-2/airline", - "source_data": { - "dataset_name": "tau-bench-2/airline", - "source_type": "url", - "url": [ - "https://github.com/Exgentic/exgentic" - ] - }, - "metric_config": { - "evaluation_description": "Tau Bench 2 benchmark evaluation (airline subset)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7, - "uncertainty": { - "num_samples": 50 - }, - "details": { - "average_agent_cost": "0.16", - "total_run_cost": "8.48", - "average_steps": "10.14", - "percent_finished": "1.0" - } - }, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "LiteLLM Tool Calling", - "agent_framework": "tool_calling" - } - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "LiteLLM Tool Calling", - "agent_framework": "tool_calling" - } - } - } - } - }, - { - "evaluation_id": "tau-bench-2/airline/claude-code-cli__google_gemini-3-pro-preview/1774263615.0201504", - "retrieved_timestamp": "1774263615.0201504", - "source_metadata": { - "source_name": "Exgentic Open Agent Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Exgentic", - "source_organization_url": "https://github.com/Exgentic", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "exgentic", - "version": "0.1.0" - }, - "benchmark": "tau-bench-2_airline", - "evaluation_results": [ - { - "evaluation_name": "tau-bench-2/airline", - "source_data": { - "dataset_name": "tau-bench-2/airline", - "source_type": "url", - "url": [ - "https://github.com/Exgentic/exgentic" - ] - }, - "metric_config": { - "evaluation_description": "Tau Bench 2 benchmark evaluation (airline subset)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7, - "uncertainty": { - "num_samples": 50 - }, - "details": { - "average_agent_cost": "0.34", - "total_run_cost": "17.45", - "average_steps": "12.62", - "percent_finished": "1.0" - } - }, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "Claude Code CLI", - "agent_framework": "claude_code" - } - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "Claude Code CLI", - "agent_framework": "claude_code" - } - } - } - } - }, - { - "evaluation_id": "tau-bench-2/airline/smolagents-code__google_gemini-3-pro-preview/1774263615.0201504", - "retrieved_timestamp": "1774263615.0201504", - "source_metadata": { - "source_name": "Exgentic Open Agent Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Exgentic", - "source_organization_url": "https://github.com/Exgentic", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "exgentic", - "version": "0.1.0" - }, - "benchmark": "tau-bench-2_airline", - "evaluation_results": [ - { - "evaluation_name": "tau-bench-2/airline", - "source_data": { - "dataset_name": "tau-bench-2/airline", - "source_type": "url", - "url": [ - "https://github.com/Exgentic/exgentic" - ] - }, - "metric_config": { - "evaluation_description": "Tau Bench 2 benchmark evaluation (airline subset)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.68, - "uncertainty": { - "num_samples": 50 - }, - "details": { - "average_agent_cost": "0.2", - "total_run_cost": "10.29", - "average_steps": "12.28", - "percent_finished": "1.0" - } - }, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "SmolAgents Code", - "agent_framework": "smolagents_code" - } - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "SmolAgents Code", - "agent_framework": "smolagents_code" - } - } - } - } - }, - { - "evaluation_id": "tau-bench-2/airline/litellm-tool-calling-with-shortlisting__google_gemini-3-pro-preview/1774263615.0201504", - "retrieved_timestamp": "1774263615.0201504", - "source_metadata": { - "source_name": "Exgentic Open Agent Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Exgentic", - "source_organization_url": "https://github.com/Exgentic", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "exgentic", - "version": "0.1.0" - }, - "benchmark": "tau-bench-2_airline", - "evaluation_results": [ - { - "evaluation_name": "tau-bench-2/airline", - "source_data": { - "dataset_name": "tau-bench-2/airline", - "source_type": "url", - "url": [ - "https://github.com/Exgentic/exgentic" - ] - }, - "metric_config": { - "evaluation_description": "Tau Bench 2 benchmark evaluation (airline subset)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7, - "uncertainty": { - "num_samples": 50 - }, - "details": { - "average_agent_cost": "0.16", - "total_run_cost": "8.48", - "average_steps": "10.14", - "percent_finished": "1.0" - } - }, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "LiteLLM Tool Calling with Shortlisting", - "agent_framework": "tool_calling_with_shortlisting" - } - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "LiteLLM Tool Calling with Shortlisting", - "agent_framework": "tool_calling_with_shortlisting" - } - } - } - } - }, - { - "evaluation_id": "tau-bench-2/airline/openai-solo__google_gemini-3-pro-preview/1774263615.0201504", - "retrieved_timestamp": "1774263615.0201504", - "source_metadata": { - "source_name": "Exgentic Open Agent Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Exgentic", - "source_organization_url": "https://github.com/Exgentic", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "exgentic", - "version": "0.1.0" - }, - "benchmark": "tau-bench-2_airline", - "evaluation_results": [ - { - "evaluation_name": "tau-bench-2/airline", - "source_data": { - "dataset_name": "tau-bench-2/airline", - "source_type": "url", - "url": [ - "https://github.com/Exgentic/exgentic" - ] - }, - "metric_config": { - "evaluation_description": "Tau Bench 2 benchmark evaluation (airline subset)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.62, - "uncertainty": { - "num_samples": 50 - }, - "details": { - "average_agent_cost": "0.21", - "total_run_cost": "11.18", - "average_steps": "10.9", - "percent_finished": "1.0" - } - }, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "OpenAI Solo", - "agent_framework": "openai_solo" - } - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "OpenAI Solo", - "agent_framework": "openai_solo" - } - } - } - } - }, - { - "evaluation_id": "tau-bench-2/retail/claude-code-cli__google_gemini-3-pro-preview/1774263615.0201504", - "retrieved_timestamp": "1774263615.0201504", - "source_metadata": { - "source_name": "Exgentic Open Agent Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Exgentic", - "source_organization_url": "https://github.com/Exgentic", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "exgentic", - "version": "0.1.0" - }, - "benchmark": "tau-bench-2_retail", - "evaluation_results": [ - { - "evaluation_name": "tau-bench-2/retail", - "source_data": { - "dataset_name": "tau-bench-2/retail", - "source_type": "url", - "url": [ - "https://github.com/Exgentic/exgentic" - ] - }, - "metric_config": { - "evaluation_description": "Tau Bench 2 benchmark evaluation (retail subset)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7805, - "uncertainty": { - "num_samples": 100 - }, - "details": { - "average_agent_cost": "0.19", - "total_run_cost": "19.38", - "average_steps": "11.18", - "percent_finished": "1.0" - } - }, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "Claude Code CLI", - "agent_framework": "claude_code" - } - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "Claude Code CLI", - "agent_framework": "claude_code" - } - } - } - } - }, - { - "evaluation_id": "tau-bench-2/retail/litellm-tool-calling__google_gemini-3-pro-preview/1774263615.0201504", - "retrieved_timestamp": "1774263615.0201504", - "source_metadata": { - "source_name": "Exgentic Open Agent Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Exgentic", - "source_organization_url": "https://github.com/Exgentic", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "exgentic", - "version": "0.1.0" - }, - "benchmark": "tau-bench-2_retail", - "evaluation_results": [ - { - "evaluation_name": "tau-bench-2/retail", - "source_data": { - "dataset_name": "tau-bench-2/retail", - "source_type": "url", - "url": [ - "https://github.com/Exgentic/exgentic" - ] - }, - "metric_config": { - "evaluation_description": "Tau Bench 2 benchmark evaluation (retail subset)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.82, - "uncertainty": { - "num_samples": 100 - }, - "details": { - "average_agent_cost": "0.16", - "total_run_cost": "16.64", - "average_steps": "11.25", - "percent_finished": "1.0" - } - }, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "LiteLLM Tool Calling", - "agent_framework": "tool_calling" - } - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "LiteLLM Tool Calling", - "agent_framework": "tool_calling" - } - } - } - } - }, - { - "evaluation_id": "tau-bench-2/retail/litellm-tool-calling-with-shortlisting__google_gemini-3-pro-preview/1774263615.0201504", - "retrieved_timestamp": "1774263615.0201504", - "source_metadata": { - "source_name": "Exgentic Open Agent Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Exgentic", - "source_organization_url": "https://github.com/Exgentic", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "exgentic", - "version": "0.1.0" - }, - "benchmark": "tau-bench-2_retail", - "evaluation_results": [ - { - "evaluation_name": "tau-bench-2/retail", - "source_data": { - "dataset_name": "tau-bench-2/retail", - "source_type": "url", - "url": [ - "https://github.com/Exgentic/exgentic" - ] - }, - "metric_config": { - "evaluation_description": "Tau Bench 2 benchmark evaluation (retail subset)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.82, - "uncertainty": { - "num_samples": 100 - }, - "details": { - "average_agent_cost": "0.16", - "total_run_cost": "16.64", - "average_steps": "11.25", - "percent_finished": "1.0" - } - }, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "LiteLLM Tool Calling with Shortlisting", - "agent_framework": "tool_calling_with_shortlisting" - } - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "LiteLLM Tool Calling with Shortlisting", - "agent_framework": "tool_calling_with_shortlisting" - } - } - } - } - }, - { - "evaluation_id": "tau-bench-2/retail/openai-solo__google_gemini-3-pro-preview/1774263615.0201504", - "retrieved_timestamp": "1774263615.0201504", - "source_metadata": { - "source_name": "Exgentic Open Agent Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Exgentic", - "source_organization_url": "https://github.com/Exgentic", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "exgentic", - "version": "0.1.0" - }, - "benchmark": "tau-bench-2_retail", - "evaluation_results": [ - { - "evaluation_name": "tau-bench-2/retail", - "source_data": { - "dataset_name": "tau-bench-2/retail", - "source_type": "url", - "url": [ - "https://github.com/Exgentic/exgentic" - ] - }, - "metric_config": { - "evaluation_description": "Tau Bench 2 benchmark evaluation (retail subset)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.73, - "uncertainty": { - "num_samples": 100 - }, - "details": { - "average_agent_cost": "0.27", - "total_run_cost": "27.48", - "average_steps": "10.62", - "percent_finished": "1.0" - } - }, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "OpenAI Solo", - "agent_framework": "openai_solo" - } - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "OpenAI Solo", - "agent_framework": "openai_solo" - } - } - } - } - }, - { - "evaluation_id": "tau-bench-2/retail/smolagents-code__google_gemini-3-pro-preview/1774263615.0201504", - "retrieved_timestamp": "1774263615.0201504", - "source_metadata": { - "source_name": "Exgentic Open Agent Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Exgentic", - "source_organization_url": "https://github.com/Exgentic", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "exgentic", - "version": "0.1.0" - }, - "benchmark": "tau-bench-2_retail", - "evaluation_results": [ - { - "evaluation_name": "tau-bench-2/retail", - "source_data": { - "dataset_name": "tau-bench-2/retail", - "source_type": "url", - "url": [ - "https://github.com/Exgentic/exgentic" - ] - }, - "metric_config": { - "evaluation_description": "Tau Bench 2 benchmark evaluation (retail subset)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7576, - "uncertainty": { - "num_samples": 100 - }, - "details": { - "average_agent_cost": "0.21", - "total_run_cost": "21.43", - "average_steps": "11.3", - "percent_finished": "1.0" - } - }, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "SmolAgents Code", - "agent_framework": "smolagents_code" - } - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "SmolAgents Code", - "agent_framework": "smolagents_code" - } - } - } - } - }, - { - "evaluation_id": "tau-bench-2/telecom/litellm-tool-calling__google_gemini-3-pro-preview/1774263615.0201504", - "retrieved_timestamp": "1774263615.0201504", - "source_metadata": { - "source_name": "Exgentic Open Agent Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Exgentic", - "source_organization_url": "https://github.com/Exgentic", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "exgentic", - "version": "0.1.0" - }, - "benchmark": "tau-bench-2_telecom", - "evaluation_results": [ - { - "evaluation_name": "tau-bench-2/telecom", - "source_data": { - "dataset_name": "tau-bench-2/telecom", - "source_type": "url", - "url": [ - "https://github.com/Exgentic/exgentic" - ] - }, - "metric_config": { - "evaluation_description": "Tau Bench 2 benchmark evaluation (telecom subset)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.73, - "uncertainty": { - "num_samples": 100 - }, - "details": { - "average_agent_cost": "0.3", - "total_run_cost": "36.75", - "average_steps": "14.84", - "percent_finished": "1.0" - } - }, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "LiteLLM Tool Calling", - "agent_framework": "tool_calling" - } - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "LiteLLM Tool Calling", - "agent_framework": "tool_calling" - } - } - } - } - }, - { - "evaluation_id": "tau-bench-2/telecom/smolagents-code__google_gemini-3-pro-preview/1774263615.0201504", - "retrieved_timestamp": "1774263615.0201504", - "source_metadata": { - "source_name": "Exgentic Open Agent Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Exgentic", - "source_organization_url": "https://github.com/Exgentic", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "exgentic", - "version": "0.1.0" - }, - "benchmark": "tau-bench-2_telecom", - "evaluation_results": [ - { - "evaluation_name": "tau-bench-2/telecom", - "source_data": { - "dataset_name": "tau-bench-2/telecom", - "source_type": "url", - "url": [ - "https://github.com/Exgentic/exgentic" - ] - }, - "metric_config": { - "evaluation_description": "Tau Bench 2 benchmark evaluation (telecom subset)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.88, - "uncertainty": { - "num_samples": 100 - }, - "details": { - "average_agent_cost": "0.35", - "total_run_cost": "40.25", - "average_steps": "12.71", - "percent_finished": "1.0" - } - }, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "SmolAgents Code", - "agent_framework": "smolagents_code" - } - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "SmolAgents Code", - "agent_framework": "smolagents_code" - } - } - } - } - }, - { - "evaluation_id": "tau-bench-2/telecom/claude-code-cli__google_gemini-3-pro-preview/1774263615.0201504", - "retrieved_timestamp": "1774263615.0201504", - "source_metadata": { - "source_name": "Exgentic Open Agent Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Exgentic", - "source_organization_url": "https://github.com/Exgentic", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "exgentic", - "version": "0.1.0" - }, - "benchmark": "tau-bench-2_telecom", - "evaluation_results": [ - { - "evaluation_name": "tau-bench-2/telecom", - "source_data": { - "dataset_name": "tau-bench-2/telecom", - "source_type": "url", - "url": [ - "https://github.com/Exgentic/exgentic" - ] - }, - "metric_config": { - "evaluation_description": "Tau Bench 2 benchmark evaluation (telecom subset)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6852, - "uncertainty": { - "num_samples": 100 - }, - "details": { - "average_agent_cost": "0.21", - "total_run_cost": "25.48", - "average_steps": "9.9", - "percent_finished": "1.0" - } - }, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "Claude Code CLI", - "agent_framework": "claude_code" - } - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "Claude Code CLI", - "agent_framework": "claude_code" - } - } - } - } - }, - { - "evaluation_id": "tau-bench-2/telecom/openai-solo__google_gemini-3-pro-preview/1774263615.0201504", - "retrieved_timestamp": "1774263615.0201504", - "source_metadata": { - "source_name": "Exgentic Open Agent Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Exgentic", - "source_organization_url": "https://github.com/Exgentic", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "exgentic", - "version": "0.1.0" - }, - "benchmark": "tau-bench-2_telecom", - "evaluation_results": [ - { - "evaluation_name": "tau-bench-2/telecom", - "source_data": { - "dataset_name": "tau-bench-2/telecom", - "source_type": "url", - "url": [ - "https://github.com/Exgentic/exgentic" - ] - }, - "metric_config": { - "evaluation_description": "Tau Bench 2 benchmark evaluation (telecom subset)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8876, - "uncertainty": { - "num_samples": 100 - }, - "details": { - "average_agent_cost": "0.54", - "total_run_cost": "58.29", - "average_steps": "10.82", - "percent_finished": "0.89" - } - }, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "OpenAI Solo", - "agent_framework": "openai_solo" - } - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "OpenAI Solo", - "agent_framework": "openai_solo" - } - } - } - } - }, - { - "evaluation_id": "tau-bench-2/telecom/litellm-tool-calling-with-shortlisting__google_gemini-3-pro-preview/1774263615.0201504", - "retrieved_timestamp": "1774263615.0201504", - "source_metadata": { - "source_name": "Exgentic Open Agent Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Exgentic", - "source_organization_url": "https://github.com/Exgentic", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "exgentic", - "version": "0.1.0" - }, - "benchmark": "tau-bench-2_telecom", - "evaluation_results": [ - { - "evaluation_name": "tau-bench-2/telecom", - "source_data": { - "dataset_name": "tau-bench-2/telecom", - "source_type": "url", - "url": [ - "https://github.com/Exgentic/exgentic" - ] - }, - "metric_config": { - "evaluation_description": "Tau Bench 2 benchmark evaluation (telecom subset)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.73, - "uncertainty": { - "num_samples": 100 - }, - "details": { - "average_agent_cost": "0.3", - "total_run_cost": "36.75", - "average_steps": "14.84", - "percent_finished": "1.0" - } - }, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "LiteLLM Tool Calling with Shortlisting", - "agent_framework": "tool_calling_with_shortlisting" - } - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "LiteLLM Tool Calling with Shortlisting", - "agent_framework": "tool_calling_with_shortlisting" - } - } - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/google_gemini-3-pro.json b/data/models/google_gemini-3-pro.json deleted file mode 100644 index 6342e879f44b3b1737c7238040ca1592a2c79001..0000000000000000000000000000000000000000 --- a/data/models/google_gemini-3-pro.json +++ /dev/null @@ -1,531 +0,0 @@ -{ - "model_info": { - "name": "Gemini 3 Pro", - "id": "google/gemini-3-pro", - "developer": "Google", - "additional_details": { - "agent_name": "SageAgent", - "agent_organization": "OpenSage" - } - }, - "evaluations": [ - { - "evaluation_id": "terminal-bench-2.0/sageagent__gemini-3-pro/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2026-02-23", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 65.2, - "uncertainty": { - "standard_error": { - "value": 2.1 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"SageAgent\" -m \"Gemini 3 Pro\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"SageAgent\" -m \"Gemini 3 Pro\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - }, - { - "evaluation_id": "terminal-bench-2.0/droid__gemini-3-pro/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2025-12-24", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 61.1, - "uncertainty": { - "standard_error": { - "value": 2.8 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Droid\" -m \"Gemini 3 Pro\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Droid\" -m \"Gemini 3 Pro\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - }, - { - "evaluation_id": "terminal-bench-2.0/codebrain-1__gemini-3-pro/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2026-02-05", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 62.2, - "uncertainty": { - "standard_error": { - "value": 2.6 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"CodeBrain-1\" -m \"Gemini 3 Pro\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"CodeBrain-1\" -m \"Gemini 3 Pro\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - }, - { - "evaluation_id": "terminal-bench-2.0/ante__gemini-3-pro/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2026-01-06", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 69.4, - "uncertainty": { - "standard_error": { - "value": 2.1 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Ante\" -m \"Gemini 3 Pro\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Ante\" -m \"Gemini 3 Pro\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - }, - { - "evaluation_id": "terminal-bench-2.0/ii-agent__gemini-3-pro/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2025-12-23", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 61.8, - "uncertainty": { - "standard_error": { - "value": 2.8 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"II-Agent\" -m \"Gemini 3 Pro\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"II-Agent\" -m \"Gemini 3 Pro\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - }, - { - "evaluation_id": "terminal-bench-2.0/letta-code__gemini-3-pro/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2025-12-17", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 56.0, - "uncertainty": { - "standard_error": { - "value": 3.0 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Letta Code\" -m \"Gemini 3 Pro\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Letta Code\" -m \"Gemini 3 Pro\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - }, - { - "evaluation_id": "terminal-bench-2.0/terminus-2__gemini-3-pro/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2025-11-21", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 56.9, - "uncertainty": { - "standard_error": { - "value": 2.5 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Terminus 2\" -m \"Gemini 3 Pro\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Terminus 2\" -m \"Gemini 3 Pro\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/google_gemini-3.1-pro.json b/data/models/google_gemini-3.1-pro.json deleted file mode 100644 index ecc327e910f8e4125f6002db506fbf114eb7832a..0000000000000000000000000000000000000000 --- a/data/models/google_gemini-3.1-pro.json +++ /dev/null @@ -1,161 +0,0 @@ -{ - "model_info": { - "name": "Gemini 3.1 Pro", - "id": "google/gemini-3.1-pro", - "developer": "Google", - "additional_details": { - "agent_name": "Forge Code", - "agent_organization": "Forge Code" - } - }, - "evaluations": [ - { - "evaluation_id": "terminal-bench-2.0/forge-code__gemini-3.1-pro/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2026-03-02", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 78.4, - "uncertainty": { - "standard_error": { - "value": 1.8 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Forge Code\" -m \"Gemini 3.1 Pro\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Forge Code\" -m \"Gemini 3.1 Pro\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - }, - { - "evaluation_id": "terminal-bench-2.0/terminus-kira__gemini-3.1-pro/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2026-02-23", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 74.8, - "uncertainty": { - "standard_error": { - "value": 2.6 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Terminus-KIRA\" -m \"Gemini 3.1 Pro\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Terminus-KIRA\" -m \"Gemini 3.1 Pro\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/google_gemini_2.5_flash.json b/data/models/google_gemini_2.5_flash.json deleted file mode 100644 index 0bd5123108b7dc6d1c03d7d5a17847bf38958457..0000000000000000000000000000000000000000 --- a/data/models/google_gemini_2.5_flash.json +++ /dev/null @@ -1,129 +0,0 @@ -{ - "model_info": { - "name": "Gemini 2.5 Flash", - "developer": "Google", - "id": "google/Gemini 2.5 Flash", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "ace/google_gemini-2.5-flash/1773260200", - "retrieved_timestamp": "1773260200", - "source_metadata": { - "source_name": "Mercor ACE Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Mercor", - "source_organization_url": "https://www.mercor.com", - "evaluator_relationship": "first_party" - }, - "eval_library": { - "name": "archipelago", - "version": "1.0.0" - }, - "benchmark": "ace", - "evaluation_results": [ - { - "evaluation_name": "Overall Score", - "source_data": { - "dataset_name": "ace", - "source_type": "hf_dataset", - "hf_repo": "Mercor/ACE" - }, - "metric_config": { - "evaluation_description": "Overall ACE score (paper snapshot, approximate).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.38 - }, - "generation_config": { - "additional_details": { - "run_setting": "On", - "value_quality": "approximate" - } - } - }, - { - "evaluation_name": "Gaming Score", - "source_data": { - "dataset_name": "ace", - "source_type": "hf_dataset", - "hf_repo": "Mercor/ACE" - }, - "metric_config": { - "evaluation_description": "Gaming domain score.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.284 - }, - "generation_config": { - "additional_details": { - "run_setting": "On" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "run_setting": "On", - "value_quality": "approximate" - } - } - }, - { - "evaluation_id": "apex-v1/google_gemini-2.5-flash/1773260200", - "retrieved_timestamp": "1773260200", - "source_metadata": { - "source_name": "Mercor APEX-v1 Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Mercor", - "source_organization_url": "https://www.mercor.com", - "evaluator_relationship": "first_party" - }, - "eval_library": { - "name": "archipelago", - "version": "1.0.0" - }, - "benchmark": "apex-v1", - "evaluation_results": [ - { - "evaluation_name": "Overall Score", - "source_data": { - "dataset_name": "apex-v1", - "source_type": "hf_dataset", - "hf_repo": "Mercor/APEX-v1" - }, - "metric_config": { - "evaluation_description": "Overall APEX-v1 mean score (paper snapshot).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.604 - }, - "generation_config": { - "additional_details": { - "run_setting": "On" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "run_setting": "On" - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/google_gemini_2.5_pro.json b/data/models/google_gemini_2.5_pro.json deleted file mode 100644 index 572d882b62178da40f0d6ff7a757cc078c9933f6..0000000000000000000000000000000000000000 --- a/data/models/google_gemini_2.5_pro.json +++ /dev/null @@ -1,82 +0,0 @@ -{ - "model_info": { - "name": "Gemini 2.5 Pro", - "developer": "Google", - "id": "google/Gemini 2.5 Pro", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "ace/google_gemini-2.5-pro/1773260200", - "retrieved_timestamp": "1773260200", - "source_metadata": { - "source_name": "Mercor ACE Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Mercor", - "source_organization_url": "https://www.mercor.com", - "evaluator_relationship": "first_party" - }, - "eval_library": { - "name": "archipelago", - "version": "1.0.0" - }, - "benchmark": "ace", - "evaluation_results": [ - { - "evaluation_name": "Overall Score", - "source_data": { - "dataset_name": "ace", - "source_type": "hf_dataset", - "hf_repo": "Mercor/ACE" - }, - "metric_config": { - "evaluation_description": "Overall ACE score (paper snapshot, approximate).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.4 - }, - "generation_config": { - "additional_details": { - "run_setting": "On", - "value_quality": "approximate" - } - } - }, - { - "evaluation_name": "Gaming Score", - "source_data": { - "dataset_name": "ace", - "source_type": "hf_dataset", - "hf_repo": "Mercor/ACE" - }, - "metric_config": { - "evaluation_description": "Gaming domain score.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.285 - }, - "generation_config": { - "additional_details": { - "run_setting": "On" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "run_setting": "On", - "value_quality": "approximate" - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/google_gemini_3.1_pro.json b/data/models/google_gemini_3.1_pro.json deleted file mode 100644 index 995fe5f415a2b0bba0f7043f692ad78b0011781f..0000000000000000000000000000000000000000 --- a/data/models/google_gemini_3.1_pro.json +++ /dev/null @@ -1,87 +0,0 @@ -{ - "model_info": { - "name": "Gemini 3.1 Pro", - "developer": "Google", - "id": "google/Gemini 3.1 Pro", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "apex-agents/google_gemini-3.1-pro/1773260200", - "retrieved_timestamp": "1773260200", - "source_metadata": { - "source_name": "Mercor APEX-Agents Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Mercor", - "source_organization_url": "https://www.mercor.com", - "evaluator_relationship": "first_party" - }, - "eval_library": { - "name": "archipelago", - "version": "1.0.0" - }, - "benchmark": "apex-agents", - "evaluation_results": [ - { - "evaluation_name": "Overall Pass@1", - "source_data": { - "dataset_name": "apex-agents", - "source_type": "hf_dataset", - "hf_repo": "mercor/apex-agents" - }, - "metric_config": { - "evaluation_description": "Overall Pass@1 across 480 long-horizon professional-services tasks.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.335, - "uncertainty": { - "confidence_interval": { - "lower": -0.036, - "upper": 0.036, - "method": "bootstrap" - } - } - }, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - }, - { - "evaluation_name": "Corporate Lawyer Mean Score", - "source_data": { - "dataset_name": "apex-agents", - "source_type": "hf_dataset", - "hf_repo": "mercor/apex-agents" - }, - "metric_config": { - "evaluation_description": "Corporate lawyer world mean score from leaderboard model list.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.494 - }, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/google_gemini_3_flash.json b/data/models/google_gemini_3_flash.json deleted file mode 100644 index 6beb736773081d366f65edf05d1bf54bd7e07ce2..0000000000000000000000000000000000000000 --- a/data/models/google_gemini_3_flash.json +++ /dev/null @@ -1,333 +0,0 @@ -{ - "model_info": { - "name": "Gemini 3 Flash", - "developer": "Google", - "id": "google/Gemini 3 Flash", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "apex-agents/google_gemini-3-flash/1773260200", - "retrieved_timestamp": "1773260200", - "source_metadata": { - "source_name": "Mercor APEX-Agents Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Mercor", - "source_organization_url": "https://www.mercor.com", - "evaluator_relationship": "first_party" - }, - "eval_library": { - "name": "archipelago", - "version": "1.0.0" - }, - "benchmark": "apex-agents", - "evaluation_results": [ - { - "evaluation_name": "Overall Pass@1", - "source_data": { - "dataset_name": "apex-agents", - "source_type": "hf_dataset", - "hf_repo": "mercor/apex-agents" - }, - "metric_config": { - "evaluation_description": "Overall Pass@1 (dataset card / paper snapshot).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.24, - "uncertainty": { - "confidence_interval": { - "lower": -0.033, - "upper": 0.033, - "method": "bootstrap" - } - } - }, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - }, - { - "evaluation_name": "Overall Pass@8", - "source_data": { - "dataset_name": "apex-agents", - "source_type": "hf_dataset", - "hf_repo": "mercor/apex-agents" - }, - "metric_config": { - "evaluation_description": "Overall Pass@8 (dataset card / paper snapshot).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.367, - "uncertainty": { - "confidence_interval": { - "lower": -0.044, - "upper": 0.043, - "method": "bootstrap" - } - } - }, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - }, - { - "evaluation_name": "Overall Mean Score", - "source_data": { - "dataset_name": "apex-agents", - "source_type": "hf_dataset", - "hf_repo": "mercor/apex-agents" - }, - "metric_config": { - "evaluation_description": "Overall mean rubric score.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.395 - }, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - }, - { - "evaluation_name": "Investment Banking Pass@1", - "source_data": { - "dataset_name": "apex-agents", - "source_type": "hf_dataset", - "hf_repo": "mercor/apex-agents" - }, - "metric_config": { - "evaluation_description": "Investment banking world Pass@1.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.267 - }, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - }, - { - "evaluation_name": "Management Consulting Pass@1", - "source_data": { - "dataset_name": "apex-agents", - "source_type": "hf_dataset", - "hf_repo": "mercor/apex-agents" - }, - "metric_config": { - "evaluation_description": "Management consulting world Pass@1.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.193 - }, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - }, - { - "evaluation_name": "Corporate Law Pass@1", - "source_data": { - "dataset_name": "apex-agents", - "source_type": "hf_dataset", - "hf_repo": "mercor/apex-agents" - }, - "metric_config": { - "evaluation_description": "Corporate law world Pass@1.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.259 - }, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - }, - { - "evaluation_name": "Corporate Lawyer Mean Score", - "source_data": { - "dataset_name": "apex-agents", - "source_type": "hf_dataset", - "hf_repo": "mercor/apex-agents" - }, - "metric_config": { - "evaluation_description": "Corporate lawyer world mean score.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.524 - }, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - }, - { - "evaluation_id": "ace/google_gemini-3-flash/1773260200", - "retrieved_timestamp": "1773260200", - "source_metadata": { - "source_name": "Mercor ACE Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Mercor", - "source_organization_url": "https://www.mercor.com", - "evaluator_relationship": "first_party" - }, - "eval_library": { - "name": "archipelago", - "version": "1.0.0" - }, - "benchmark": "ace", - "evaluation_results": [ - { - "evaluation_name": "Gaming Score", - "source_data": { - "dataset_name": "ace", - "source_type": "hf_dataset", - "hf_repo": "Mercor/ACE" - }, - "metric_config": { - "evaluation_description": "Gaming domain score.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.415 - }, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - }, - { - "evaluation_id": "apex-v1/google_gemini-3-flash/1773260200", - "retrieved_timestamp": "1773260200", - "source_metadata": { - "source_name": "Mercor APEX-v1 Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Mercor", - "source_organization_url": "https://www.mercor.com", - "evaluator_relationship": "first_party" - }, - "eval_library": { - "name": "archipelago", - "version": "1.0.0" - }, - "benchmark": "apex-v1", - "evaluation_results": [ - { - "evaluation_name": "Overall Score", - "source_data": { - "dataset_name": "apex-v1", - "source_type": "hf_dataset", - "hf_repo": "Mercor/APEX-v1" - }, - "metric_config": { - "evaluation_description": "Overall APEX-v1 mean score across all jobs.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.64, - "uncertainty": { - "confidence_interval": { - "lower": -0.022, - "upper": 0.022, - "method": "bootstrap" - } - } - }, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - }, - { - "evaluation_name": "Consulting Score", - "source_data": { - "dataset_name": "apex-v1", - "source_type": "hf_dataset", - "hf_repo": "Mercor/APEX-v1" - }, - "metric_config": { - "evaluation_description": "Management consulting score.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.64 - }, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/google_gemini_3_pro.json b/data/models/google_gemini_3_pro.json deleted file mode 100644 index 104ce8e17e340df7c6fe6dc4bc1f7866f9c1cc71..0000000000000000000000000000000000000000 --- a/data/models/google_gemini_3_pro.json +++ /dev/null @@ -1,381 +0,0 @@ -{ - "model_info": { - "name": "Gemini 3 Pro", - "developer": "Google", - "id": "google/Gemini 3 Pro", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "apex-agents/google_gemini-3-pro/1773260200", - "retrieved_timestamp": "1773260200", - "source_metadata": { - "source_name": "Mercor APEX-Agents Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Mercor", - "source_organization_url": "https://www.mercor.com", - "evaluator_relationship": "first_party" - }, - "eval_library": { - "name": "archipelago", - "version": "1.0.0" - }, - "benchmark": "apex-agents", - "evaluation_results": [ - { - "evaluation_name": "Overall Pass@1", - "source_data": { - "dataset_name": "apex-agents", - "source_type": "hf_dataset", - "hf_repo": "mercor/apex-agents" - }, - "metric_config": { - "evaluation_description": "Overall Pass@1 (dataset card / paper snapshot).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.184, - "uncertainty": { - "confidence_interval": { - "lower": -0.027, - "upper": 0.027, - "method": "bootstrap" - } - } - }, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - }, - { - "evaluation_name": "Overall Pass@8", - "source_data": { - "dataset_name": "apex-agents", - "source_type": "hf_dataset", - "hf_repo": "mercor/apex-agents" - }, - "metric_config": { - "evaluation_description": "Overall Pass@8 (dataset card / paper snapshot).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.373, - "uncertainty": { - "confidence_interval": { - "lower": -0.044, - "upper": 0.044, - "method": "bootstrap" - } - } - }, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - }, - { - "evaluation_name": "Overall Mean Score", - "source_data": { - "dataset_name": "apex-agents", - "source_type": "hf_dataset", - "hf_repo": "mercor/apex-agents" - }, - "metric_config": { - "evaluation_description": "Overall mean rubric score.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.341 - }, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - }, - { - "evaluation_name": "Investment Banking Pass@1", - "source_data": { - "dataset_name": "apex-agents", - "source_type": "hf_dataset", - "hf_repo": "mercor/apex-agents" - }, - "metric_config": { - "evaluation_description": "Investment banking world Pass@1.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.188 - }, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - }, - { - "evaluation_name": "Management Consulting Pass@1", - "source_data": { - "dataset_name": "apex-agents", - "source_type": "hf_dataset", - "hf_repo": "mercor/apex-agents" - }, - "metric_config": { - "evaluation_description": "Management consulting world Pass@1.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.124 - }, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - }, - { - "evaluation_name": "Corporate Law Pass@1", - "source_data": { - "dataset_name": "apex-agents", - "source_type": "hf_dataset", - "hf_repo": "mercor/apex-agents" - }, - "metric_config": { - "evaluation_description": "Corporate law world Pass@1.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.239 - }, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - }, - { - "evaluation_name": "Corporate Lawyer Mean Score", - "source_data": { - "dataset_name": "apex-agents", - "source_type": "hf_dataset", - "hf_repo": "mercor/apex-agents" - }, - "metric_config": { - "evaluation_description": "Corporate lawyer world mean score.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.487 - }, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - }, - { - "evaluation_id": "ace/google_gemini-3-pro/1773260200", - "retrieved_timestamp": "1773260200", - "source_metadata": { - "source_name": "Mercor ACE Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Mercor", - "source_organization_url": "https://www.mercor.com", - "evaluator_relationship": "first_party" - }, - "eval_library": { - "name": "archipelago", - "version": "1.0.0" - }, - "benchmark": "ace", - "evaluation_results": [ - { - "evaluation_name": "Overall Score", - "source_data": { - "dataset_name": "ace", - "source_type": "hf_dataset", - "hf_repo": "Mercor/ACE" - }, - "metric_config": { - "evaluation_description": "Overall ACE score (paper snapshot, approximate).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.47 - }, - "generation_config": { - "additional_details": { - "run_setting": "High", - "value_quality": "approximate" - } - } - }, - { - "evaluation_name": "Gaming Score", - "source_data": { - "dataset_name": "ace", - "source_type": "hf_dataset", - "hf_repo": "Mercor/ACE" - }, - "metric_config": { - "evaluation_description": "Gaming domain score.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.509 - }, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "run_setting": "High", - "value_quality": "approximate" - } - } - }, - { - "evaluation_id": "apex-v1/google_gemini-3-pro/1773260200", - "retrieved_timestamp": "1773260200", - "source_metadata": { - "source_name": "Mercor APEX-v1 Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Mercor", - "source_organization_url": "https://www.mercor.com", - "evaluator_relationship": "first_party" - }, - "eval_library": { - "name": "archipelago", - "version": "1.0.0" - }, - "benchmark": "apex-v1", - "evaluation_results": [ - { - "evaluation_name": "Overall Score", - "source_data": { - "dataset_name": "apex-v1", - "source_type": "hf_dataset", - "hf_repo": "Mercor/APEX-v1" - }, - "metric_config": { - "evaluation_description": "Overall APEX-v1 mean score across all jobs.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.643, - "uncertainty": { - "confidence_interval": { - "lower": -0.023, - "upper": 0.023, - "method": "bootstrap" - } - } - }, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - }, - { - "evaluation_name": "Consulting Score", - "source_data": { - "dataset_name": "apex-v1", - "source_type": "hf_dataset", - "hf_repo": "Mercor/APEX-v1" - }, - "metric_config": { - "evaluation_description": "Management consulting score.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.64 - }, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - }, - { - "evaluation_name": "Investment Banking Score", - "source_data": { - "dataset_name": "apex-v1", - "source_type": "hf_dataset", - "hf_repo": "Mercor/APEX-v1" - }, - "metric_config": { - "evaluation_description": "Investment banking associate score.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.63 - }, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/google_gemma-1.1-2b-it.json b/data/models/google_gemma-1.1-2b-it.json deleted file mode 100644 index 456a7dc5107b74af82e529a346a6de609ae70b87..0000000000000000000000000000000000000000 --- a/data/models/google_gemma-1.1-2b-it.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "gemma-1.1-2b-it", - "id": "google/gemma-1.1-2b-it", - "developer": "Google", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "GemmaForCausalLM", - "params_billions": "2.506" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/google_gemma-1.1-2b-it/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3067 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3185 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0181 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2693 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3394 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1484 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/google_gemma-1.1-7b-it.json b/data/models/google_gemma-1.1-7b-it.json deleted file mode 100644 index 32bca72445775b8cf5ebf689caeb4c1c9af38c1a..0000000000000000000000000000000000000000 --- a/data/models/google_gemma-1.1-7b-it.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "gemma-1.1-7b-it", - "id": "google/gemma-1.1-7b-it", - "developer": "Google", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "GemmaForCausalLM", - "params_billions": "8.538" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/google_gemma-1.1-7b-it/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5039 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3935 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0491 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2936 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.423 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2584 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/google_gemma-2-27b-it.json b/data/models/google_gemma-2-27b-it.json deleted file mode 100644 index 418b80f0bf31c373c9666d91ad6541f501299239..0000000000000000000000000000000000000000 --- a/data/models/google_gemma-2-27b-it.json +++ /dev/null @@ -1,624 +0,0 @@ -{ - "model_info": { - "name": "Gemma 2 Instruct 27B", - "id": "google/gemma-2-27b-it", - "developer": "Google", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_lite/google_gemma-2-27b-it/1774096306.427425", - "retrieved_timestamp": "1774096306.427425", - "source_metadata": { - "source_name": "helm_lite", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_lite", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_lite", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.675, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.7407490636704119\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.79, - "details": { - "description": "min=0.79, mean=0.79, max=0.79, sum=0.79 (1)", - "tab": "Accuracy", - "NarrativeQA - Observed inference time (s)": "{\"description\": \"min=0.66, mean=0.66, max=0.66, sum=0.66 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.6603116545878666\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=3437.994, mean=3437.994, max=3437.994, sum=3437.994 (1)\", \"tab\": \"General information\", \"score\": \"3437.994366197183\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (closed-book)", - "source_data": { - "dataset_name": "NaturalQuestions (closed-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (closed-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.353, - "details": { - "description": "min=0.353, mean=0.353, max=0.353, sum=0.353 (1)", - "tab": "Accuracy", - "NaturalQuestions (open-book) - Observed inference time (s)": "{\"description\": \"min=0.486, mean=0.486, max=0.486, sum=0.486 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.4863240420818329\"}", - "NaturalQuestions (closed-book) - Observed inference time (s)": "{\"description\": \"min=0.358, mean=0.358, max=0.358, sum=0.358 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.35805381870269776\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.953, mean=4.953, max=4.953, sum=4.953 (1)\", \"tab\": \"General information\", \"score\": \"4.953\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.009, mean=0.009, max=0.009, sum=0.009 (1)\", \"tab\": \"General information\", \"score\": \"0.009\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1911.526, mean=1911.526, max=1911.526, sum=1911.526 (1)\", \"tab\": \"General information\", \"score\": \"1911.526\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=143.995, mean=143.995, max=143.995, sum=143.995 (1)\", \"tab\": \"General information\", \"score\": \"143.995\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=0.993, mean=0.993, max=0.993, sum=0.993 (1)\", \"tab\": \"General information\", \"score\": \"0.993\"}" - } - }, - "generation_config": { - "additional_details": { - "mode": "\"closedbook\"" - } - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.918, - "details": { - "description": "min=0.918, mean=0.918, max=0.918, sum=0.918 (1)", - "tab": "Accuracy", - "OpenbookQA - Observed inference time (s)": "{\"description\": \"min=0.327, mean=0.327, max=0.327, sum=0.327 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.3270734968185425\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=248.508, mean=248.508, max=248.508, sum=248.508 (1)\", \"tab\": \"General information\", \"score\": \"248.508\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "dataset": "\"openbookqa\"", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.664, - "details": { - "description": "min=0.44, mean=0.664, max=0.93, sum=3.32 (5)", - "tab": "Accuracy", - "MMLU - Observed inference time (s)": "{\"description\": \"min=0.317, mean=0.329, max=0.337, sum=1.643 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.3286796834259702\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=380.91, mean=481.531, max=634.553, sum=2407.653 (5)\", \"tab\": \"General information\", \"score\": \"481.5305263157895\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"college_chemistry\", \"computer_security\", \"econometrics\", \"us_foreign_policy\"]", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MATH", - "source_data": { - "dataset_name": "MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Equivalent (CoT) on MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.746, - "details": { - "description": "min=0.513, mean=0.746, max=0.93, sum=5.219 (7)", - "tab": "Accuracy", - "MATH - Observed inference time (s)": "{\"description\": \"min=1.515, mean=1.903, max=2.648, sum=13.324 (7)\", \"tab\": \"Efficiency\", \"score\": \"1.9034432935092742\"}", - "MATH - # eval": "{\"description\": \"min=30, mean=62.429, max=135, sum=437 (7)\", \"tab\": \"General information\", \"score\": \"62.42857142857143\"}", - "MATH - # train": "{\"description\": \"min=8, mean=8, max=8, sum=56 (7)\", \"tab\": \"General information\", \"score\": \"8.0\"}", - "MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MATH - # prompt tokens": "{\"description\": \"min=938.215, mean=1355.506, max=2348.712, sum=9488.545 (7)\", \"tab\": \"General information\", \"score\": \"1355.5064552904823\"}", - "MATH - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=7 (7)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"algebra\", \"counting_and_probability\", \"geometry\", \"intermediate_algebra\", \"number_theory\", \"prealgebra\", \"precalculus\"]", - "level": "\"1\"", - "use_official_examples": "\"False\"", - "use_chain_of_thought": "\"True\"" - } - } - }, - { - "evaluation_name": "GSM8K", - "source_data": { - "dataset_name": "GSM8K", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on GSM8K", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.812, - "details": { - "description": "min=0.812, mean=0.812, max=0.812, sum=0.812 (1)", - "tab": "Accuracy", - "GSM8K - Observed inference time (s)": "{\"description\": \"min=2.332, mean=2.332, max=2.332, sum=2.332 (1)\", \"tab\": \"Efficiency\", \"score\": \"2.3315503742694856\"}", - "GSM8K - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "GSM8K - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "GSM8K - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GSM8K - # prompt tokens": "{\"description\": \"min=1151.885, mean=1151.885, max=1151.885, sum=1151.885 (1)\", \"tab\": \"General information\", \"score\": \"1151.885\"}", - "GSM8K - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "LegalBench", - "source_data": { - "dataset_name": "LegalBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on LegalBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7, - "details": { - "description": "min=0.439, mean=0.7, max=0.979, sum=3.499 (5)", - "tab": "Accuracy", - "LegalBench - Observed inference time (s)": "{\"description\": \"min=0.328, mean=0.44, max=0.796, sum=2.202 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.4403507251683155\"}", - "LegalBench - # eval": "{\"description\": \"min=95, mean=409.4, max=1000, sum=2047 (5)\", \"tab\": \"General information\", \"score\": \"409.4\"}", - "LegalBench - # train": "{\"description\": \"min=4, mean=4.798, max=5, sum=23.992 (5)\", \"tab\": \"General information\", \"score\": \"4.798367346938775\"}", - "LegalBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "LegalBench - # prompt tokens": "{\"description\": \"min=199.916, mean=1546.699, max=6405.871, sum=7733.495 (5)\", \"tab\": \"General information\", \"score\": \"1546.699013263404\"}", - "LegalBench - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"abercrombie\", \"corporate_lobbying\", \"function_of_decision_section\", \"international_citizenship_questions\", \"proa\"]" - } - } - }, - { - "evaluation_name": "MedQA", - "source_data": { - "dataset_name": "MedQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MedQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.684, - "details": { - "description": "min=0.684, mean=0.684, max=0.684, sum=0.684 (1)", - "tab": "Accuracy", - "MedQA - Observed inference time (s)": "{\"description\": \"min=0.451, mean=0.451, max=0.451, sum=0.451 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.4512898187277094\"}", - "MedQA - # eval": "{\"description\": \"min=503, mean=503, max=503, sum=503 (1)\", \"tab\": \"General information\", \"score\": \"503.0\"}", - "MedQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MedQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MedQA - # prompt tokens": "{\"description\": \"min=1029.481, mean=1029.481, max=1029.481, sum=1029.481 (1)\", \"tab\": \"General information\", \"score\": \"1029.4811133200794\"}", - "MedQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WMT 2014", - "source_data": { - "dataset_name": "WMT 2014", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "BLEU-4 on WMT 2014", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.214, - "details": { - "description": "min=0.167, mean=0.214, max=0.241, sum=1.072 (5)", - "tab": "Accuracy", - "WMT 2014 - Observed inference time (s)": "{\"description\": \"min=0.666, mean=0.698, max=0.715, sum=3.492 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.6983992647690125\"}", - "WMT 2014 - # eval": "{\"description\": \"min=503, mean=568.8, max=832, sum=2844 (5)\", \"tab\": \"General information\", \"score\": \"568.8\"}", - "WMT 2014 - # train": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "WMT 2014 - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WMT 2014 - # prompt tokens": "{\"description\": \"min=80.732, mean=110.97, max=137.366, sum=554.851 (5)\", \"tab\": \"General information\", \"score\": \"110.97025108961614\"}", - "WMT 2014 - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "language_pair": "[\"cs-en\", \"de-en\", \"fr-en\", \"hi-en\", \"ru-en\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "hfopenllm_v2/google_gemma-2-27b-it/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7978 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6451 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2387 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.375 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4033 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4451 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "reward-bench/google_gemma-2-27b-it/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.809 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9483 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.591 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8635 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.833 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/google_gemma-2-27b.json b/data/models/google_gemma-2-27b.json deleted file mode 100644 index f1a3fd6e0f2090dbf1bbf19f0da61756b4b9c9a8..0000000000000000000000000000000000000000 --- a/data/models/google_gemma-2-27b.json +++ /dev/null @@ -1,1661 +0,0 @@ -{ - "model_info": { - "name": "Gemma 2 27B", - "id": "google/gemma-2-27b", - "developer": "Google", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_mmlu/google_gemma-2-27b/1774096312.00548", - "retrieved_timestamp": "1774096312.00548", - "source_metadata": { - "source_name": "helm_mmlu", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_mmlu", - "evaluation_results": [ - { - "evaluation_name": "MMLU All Subjects", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU All Subjects", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.757, - "details": { - "description": "min=0.394, mean=0.757, max=0.979, sum=86.303 (114)", - "tab": "Accuracy", - "MMLU All Subjects - Observed inference time (s)": "{\"description\": \"min=1.169, mean=2.744, max=12.207, sum=312.86 (114)\", \"tab\": \"Efficiency\", \"score\": \"2.7443855864562217\"}", - "MMLU All Subjects - # eval": "{\"description\": \"min=100, mean=246.351, max=1534, sum=28084 (114)\", \"tab\": \"General information\", \"score\": \"246.35087719298247\"}", - "MMLU All Subjects - # train": "{\"description\": \"min=5, mean=5, max=5, sum=570 (114)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU All Subjects - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU All Subjects - # prompt tokens": "{\"description\": \"min=260.164, mean=624.617, max=2789.424, sum=71206.345 (114)\", \"tab\": \"General information\", \"score\": \"624.6170571214202\"}", - "MMLU All Subjects - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=114 (114)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_name": "Abstract Algebra", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Abstract Algebra", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4, - "details": { - "description": "min=0.4, mean=0.4, max=0.4, sum=0.8 (2)", - "tab": "Accuracy", - "Abstract Algebra - Observed inference time (s)": "{\"description\": \"min=1.522, mean=1.522, max=1.522, sum=3.043 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.5217395949363708\"}", - "Abstract Algebra - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Abstract Algebra - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Abstract Algebra - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Abstract Algebra - # prompt tokens": "{\"description\": \"min=375.97, mean=375.97, max=375.97, sum=751.94 (2)\", \"tab\": \"General information\", \"score\": \"375.97\"}", - "Abstract Algebra - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"abstract_algebra\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_abstract_algebra\"" - } - } - }, - { - "evaluation_name": "Anatomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Anatomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.77, - "details": { - "description": "min=0.77, mean=0.77, max=0.77, sum=1.541 (2)", - "tab": "Accuracy", - "Anatomy - Observed inference time (s)": "{\"description\": \"min=1.179, mean=1.179, max=1.179, sum=2.359 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.1792643246827301\"}", - "Anatomy - # eval": "{\"description\": \"min=135, mean=135, max=135, sum=270 (2)\", \"tab\": \"General information\", \"score\": \"135.0\"}", - "Anatomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Anatomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Anatomy - # prompt tokens": "{\"description\": \"min=336.356, mean=336.356, max=336.356, sum=672.711 (2)\", \"tab\": \"General information\", \"score\": \"336.35555555555555\"}", - "Anatomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"anatomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_anatomy\"" - } - } - }, - { - "evaluation_name": "College Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on College Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5, - "details": { - "description": "min=0.5, mean=0.5, max=0.5, sum=1 (2)", - "tab": "Accuracy", - "College Chemistry - Observed inference time (s)": "{\"description\": \"min=2.168, mean=2.168, max=2.168, sum=4.337 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.168372049331665\"}", - "College Biology - Observed inference time (s)": "{\"description\": \"min=1.995, mean=1.995, max=1.995, sum=3.99 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.994903423719936\"}", - "College Computer Science - Observed inference time (s)": "{\"description\": \"min=3.315, mean=3.315, max=3.315, sum=6.631 (2)\", \"tab\": \"Efficiency\", \"score\": \"3.315422866344452\"}", - "College Mathematics - Observed inference time (s)": "{\"description\": \"min=2.323, mean=2.323, max=2.323, sum=4.647 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.323271915912628\"}", - "College Medicine - Observed inference time (s)": "{\"description\": \"min=2.118, mean=2.118, max=2.118, sum=4.236 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.117893081179933\"}", - "College Physics - Observed inference time (s)": "{\"description\": \"min=1.982, mean=1.982, max=1.982, sum=3.964 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.9819396874483894\"}", - "College Chemistry - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Chemistry - # prompt tokens": "{\"description\": \"min=562.02, mean=562.02, max=562.02, sum=1124.04 (2)\", \"tab\": \"General information\", \"score\": \"562.02\"}", - "College Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Biology - # eval": "{\"description\": \"min=144, mean=144, max=144, sum=288 (2)\", \"tab\": \"General information\", \"score\": \"144.0\"}", - "College Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # prompt tokens": "{\"description\": \"min=474.799, mean=474.799, max=474.799, sum=949.597 (2)\", \"tab\": \"General information\", \"score\": \"474.7986111111111\"}", - "College Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # prompt tokens": "{\"description\": \"min=849.86, mean=849.86, max=849.86, sum=1699.72 (2)\", \"tab\": \"General information\", \"score\": \"849.86\"}", - "College Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Mathematics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # prompt tokens": "{\"description\": \"min=618.69, mean=618.69, max=618.69, sum=1237.38 (2)\", \"tab\": \"General information\", \"score\": \"618.69\"}", - "College Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Medicine - # eval": "{\"description\": \"min=173, mean=173, max=173, sum=346 (2)\", \"tab\": \"General information\", \"score\": \"173.0\"}", - "College Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # prompt tokens": "{\"description\": \"min=505.37, mean=505.37, max=505.37, sum=1010.74 (2)\", \"tab\": \"General information\", \"score\": \"505.3699421965318\"}", - "College Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Physics - # eval": "{\"description\": \"min=102, mean=102, max=102, sum=204 (2)\", \"tab\": \"General information\", \"score\": \"102.0\"}", - "College Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # prompt tokens": "{\"description\": \"min=499.471, mean=499.471, max=499.471, sum=998.941 (2)\", \"tab\": \"General information\", \"score\": \"499.47058823529414\"}", - "College Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"college_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_college_physics\"" - } - } - }, - { - "evaluation_name": "Computer Security", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Computer Security", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.84, - "details": { - "description": "min=0.84, mean=0.84, max=0.84, sum=1.68 (2)", - "tab": "Accuracy", - "Computer Security - Observed inference time (s)": "{\"description\": \"min=1.314, mean=1.314, max=1.314, sum=2.628 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.3139495277404785\"}", - "Computer Security - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Computer Security - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Computer Security - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Computer Security - # prompt tokens": "{\"description\": \"min=372.91, mean=372.91, max=372.91, sum=745.82 (2)\", \"tab\": \"General information\", \"score\": \"372.91\"}", - "Computer Security - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"computer_security\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_computer_security\"" - } - } - }, - { - "evaluation_name": "Econometrics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Econometrics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.667, - "details": { - "description": "min=0.667, mean=0.667, max=0.667, sum=1.333 (2)", - "tab": "Accuracy", - "Econometrics - Observed inference time (s)": "{\"description\": \"min=2.14, mean=2.14, max=2.14, sum=4.28 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.1398948138220266\"}", - "Econometrics - # eval": "{\"description\": \"min=114, mean=114, max=114, sum=228 (2)\", \"tab\": \"General information\", \"score\": \"114.0\"}", - "Econometrics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Econometrics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Econometrics - # prompt tokens": "{\"description\": \"min=626.553, mean=626.553, max=626.553, sum=1253.105 (2)\", \"tab\": \"General information\", \"score\": \"626.5526315789474\"}", - "Econometrics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"econometrics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_econometrics\"" - } - } - }, - { - "evaluation_name": "Global Facts", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Global Facts", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.43, - "details": { - "description": "min=0.43, mean=0.43, max=0.43, sum=0.86 (2)", - "tab": "Accuracy", - "Global Facts - Observed inference time (s)": "{\"description\": \"min=1.452, mean=1.452, max=1.452, sum=2.905 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.4524464893341065\"}", - "Global Facts - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Global Facts - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Global Facts - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Global Facts - # prompt tokens": "{\"description\": \"min=448.54, mean=448.54, max=448.54, sum=897.08 (2)\", \"tab\": \"General information\", \"score\": \"448.54\"}", - "Global Facts - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"global_facts\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_global_facts\"" - } - } - }, - { - "evaluation_name": "Jurisprudence", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Jurisprudence", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.861, - "details": { - "description": "min=0.861, mean=0.861, max=0.861, sum=1.722 (2)", - "tab": "Accuracy", - "Jurisprudence - Observed inference time (s)": "{\"description\": \"min=1.421, mean=1.421, max=1.421, sum=2.841 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.4206464577604223\"}", - "Jurisprudence - # eval": "{\"description\": \"min=108, mean=108, max=108, sum=216 (2)\", \"tab\": \"General information\", \"score\": \"108.0\"}", - "Jurisprudence - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Jurisprudence - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Jurisprudence - # prompt tokens": "{\"description\": \"min=399.87, mean=399.87, max=399.87, sum=799.741 (2)\", \"tab\": \"General information\", \"score\": \"399.8703703703704\"}", - "Jurisprudence - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"jurisprudence\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_jurisprudence\"" - } - } - }, - { - "evaluation_name": "Philosophy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Philosophy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.849, - "details": { - "description": "min=0.849, mean=0.849, max=0.849, sum=1.698 (2)", - "tab": "Accuracy", - "Philosophy - Observed inference time (s)": "{\"description\": \"min=1.169, mean=1.169, max=1.169, sum=2.337 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.168742698871821\"}", - "Philosophy - # eval": "{\"description\": \"min=311, mean=311, max=311, sum=622 (2)\", \"tab\": \"General information\", \"score\": \"311.0\"}", - "Philosophy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Philosophy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Philosophy - # prompt tokens": "{\"description\": \"min=332.907, mean=332.907, max=332.907, sum=665.814 (2)\", \"tab\": \"General information\", \"score\": \"332.90675241157555\"}", - "Philosophy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"philosophy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_philosophy\"" - } - } - }, - { - "evaluation_name": "Professional Psychology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Professional Psychology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.84, - "details": { - "description": "min=0.84, mean=0.84, max=0.84, sum=1.68 (2)", - "tab": "Accuracy", - "Professional Medicine - Observed inference time (s)": "{\"description\": \"min=4.2, mean=4.2, max=4.2, sum=8.399 (2)\", \"tab\": \"Efficiency\", \"score\": \"4.199711911818561\"}", - "Professional Accounting - Observed inference time (s)": "{\"description\": \"min=3.427, mean=3.427, max=3.427, sum=6.854 (2)\", \"tab\": \"Efficiency\", \"score\": \"3.4269232200392596\"}", - "Professional Law - Observed inference time (s)": "{\"description\": \"min=7.724, mean=7.724, max=7.724, sum=15.448 (2)\", \"tab\": \"Efficiency\", \"score\": \"7.723928280417581\"}", - "Professional Psychology - Observed inference time (s)": "{\"description\": \"min=2.721, mean=2.721, max=2.721, sum=5.442 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.721013201997171\"}", - "Professional Medicine - # eval": "{\"description\": \"min=272, mean=272, max=272, sum=544 (2)\", \"tab\": \"General information\", \"score\": \"272.0\"}", - "Professional Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Medicine - # prompt tokens": "{\"description\": \"min=1105.092, mean=1105.092, max=1105.092, sum=2210.184 (2)\", \"tab\": \"General information\", \"score\": \"1105.0919117647059\"}", - "Professional Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Accounting - # eval": "{\"description\": \"min=282, mean=282, max=282, sum=564 (2)\", \"tab\": \"General information\", \"score\": \"282.0\"}", - "Professional Accounting - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Accounting - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # prompt tokens": "{\"description\": \"min=747.418, mean=747.418, max=747.418, sum=1494.837 (2)\", \"tab\": \"General information\", \"score\": \"747.418439716312\"}", - "Professional Accounting - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Law - # eval": "{\"description\": \"min=1534, mean=1534, max=1534, sum=3068 (2)\", \"tab\": \"General information\", \"score\": \"1534.0\"}", - "Professional Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # prompt tokens": "{\"description\": \"min=1677.119, mean=1677.119, max=1677.119, sum=3354.239 (2)\", \"tab\": \"General information\", \"score\": \"1677.119295958279\"}", - "Professional Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Psychology - # eval": "{\"description\": \"min=612, mean=612, max=612, sum=1224 (2)\", \"tab\": \"General information\", \"score\": \"612.0\"}", - "Professional Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # prompt tokens": "{\"description\": \"min=586.363, mean=586.363, max=586.363, sum=1172.725 (2)\", \"tab\": \"General information\", \"score\": \"586.3627450980392\"}", - "Professional Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"professional_psychology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_professional_psychology\"" - } - } - }, - { - "evaluation_name": "Us Foreign Policy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Us Foreign Policy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.95, - "details": { - "description": "min=0.95, mean=0.95, max=0.95, sum=1.9 (2)", - "tab": "Accuracy", - "Us Foreign Policy - Observed inference time (s)": "{\"description\": \"min=1.555, mean=1.555, max=1.555, sum=3.109 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.554630262851715\"}", - "Us Foreign Policy - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Us Foreign Policy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Us Foreign Policy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Us Foreign Policy - # prompt tokens": "{\"description\": \"min=430.2, mean=430.2, max=430.2, sum=860.4 (2)\", \"tab\": \"General information\", \"score\": \"430.2\"}", - "Us Foreign Policy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"us_foreign_policy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_us_foreign_policy\"" - } - } - }, - { - "evaluation_name": "Astronomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Astronomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.829, - "details": { - "description": "min=0.829, mean=0.829, max=0.829, sum=1.658 (2)", - "tab": "Accuracy", - "Astronomy - Observed inference time (s)": "{\"description\": \"min=2.214, mean=2.214, max=2.214, sum=4.428 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.214210780043351\"}", - "Astronomy - # eval": "{\"description\": \"min=152, mean=152, max=152, sum=304 (2)\", \"tab\": \"General information\", \"score\": \"152.0\"}", - "Astronomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Astronomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Astronomy - # prompt tokens": "{\"description\": \"min=594.421, mean=594.421, max=594.421, sum=1188.842 (2)\", \"tab\": \"General information\", \"score\": \"594.421052631579\"}", - "Astronomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"astronomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_astronomy\"" - } - } - }, - { - "evaluation_name": "Business Ethics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Business Ethics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.78, - "details": { - "description": "min=0.78, mean=0.78, max=0.78, sum=1.56 (2)", - "tab": "Accuracy", - "Business Ethics - Observed inference time (s)": "{\"description\": \"min=2.156, mean=2.156, max=2.156, sum=4.311 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.1555044412612916\"}", - "Business Ethics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Business Ethics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Business Ethics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Business Ethics - # prompt tokens": "{\"description\": \"min=544.87, mean=544.87, max=544.87, sum=1089.74 (2)\", \"tab\": \"General information\", \"score\": \"544.87\"}", - "Business Ethics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"business_ethics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_business_ethics\"" - } - } - }, - { - "evaluation_name": "Clinical Knowledge", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Clinical Knowledge", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.808, - "details": { - "description": "min=0.808, mean=0.808, max=0.808, sum=1.615 (2)", - "tab": "Accuracy", - "Clinical Knowledge - Observed inference time (s)": "{\"description\": \"min=1.81, mean=1.81, max=1.81, sum=3.619 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.8096552030095514\"}", - "Clinical Knowledge - # eval": "{\"description\": \"min=265, mean=265, max=265, sum=530 (2)\", \"tab\": \"General information\", \"score\": \"265.0\"}", - "Clinical Knowledge - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Clinical Knowledge - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Clinical Knowledge - # prompt tokens": "{\"description\": \"min=394.592, mean=394.592, max=394.592, sum=789.185 (2)\", \"tab\": \"General information\", \"score\": \"394.5924528301887\"}", - "Clinical Knowledge - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"clinical_knowledge\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_clinical_knowledge\"" - } - } - }, - { - "evaluation_name": "Conceptual Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Conceptual Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.834, - "details": { - "description": "min=0.834, mean=0.834, max=0.834, sum=1.668 (2)", - "tab": "Accuracy", - "Conceptual Physics - Observed inference time (s)": "{\"description\": \"min=1.424, mean=1.424, max=1.424, sum=2.848 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.423792755857427\"}", - "Conceptual Physics - # eval": "{\"description\": \"min=235, mean=235, max=235, sum=470 (2)\", \"tab\": \"General information\", \"score\": \"235.0\"}", - "Conceptual Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Conceptual Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Conceptual Physics - # prompt tokens": "{\"description\": \"min=301.213, mean=301.213, max=301.213, sum=602.426 (2)\", \"tab\": \"General information\", \"score\": \"301.21276595744683\"}", - "Conceptual Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"conceptual_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_conceptual_physics\"" - } - } - }, - { - "evaluation_name": "Electrical Engineering", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Electrical Engineering", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.738, - "details": { - "description": "min=0.738, mean=0.738, max=0.738, sum=1.476 (2)", - "tab": "Accuracy", - "Electrical Engineering - Observed inference time (s)": "{\"description\": \"min=1.947, mean=1.947, max=1.947, sum=3.893 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.9467107739941827\"}", - "Electrical Engineering - # eval": "{\"description\": \"min=145, mean=145, max=145, sum=290 (2)\", \"tab\": \"General information\", \"score\": \"145.0\"}", - "Electrical Engineering - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Electrical Engineering - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Electrical Engineering - # prompt tokens": "{\"description\": \"min=466.786, mean=466.786, max=466.786, sum=933.572 (2)\", \"tab\": \"General information\", \"score\": \"466.78620689655173\"}", - "Electrical Engineering - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"electrical_engineering\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_electrical_engineering\"" - } - } - }, - { - "evaluation_name": "Elementary Mathematics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Elementary Mathematics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.558, - "details": { - "description": "min=0.558, mean=0.558, max=0.558, sum=1.116 (2)", - "tab": "Accuracy", - "Elementary Mathematics - Observed inference time (s)": "{\"description\": \"min=2.287, mean=2.287, max=2.287, sum=4.574 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.286756881330379\"}", - "Elementary Mathematics - # eval": "{\"description\": \"min=378, mean=378, max=378, sum=756 (2)\", \"tab\": \"General information\", \"score\": \"378.0\"}", - "Elementary Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Elementary Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Elementary Mathematics - # prompt tokens": "{\"description\": \"min=589.341, mean=589.341, max=589.341, sum=1178.683 (2)\", \"tab\": \"General information\", \"score\": \"589.3412698412699\"}", - "Elementary Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"elementary_mathematics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_elementary_mathematics\"" - } - } - }, - { - "evaluation_name": "Formal Logic", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Formal Logic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.516, - "details": { - "description": "min=0.516, mean=0.516, max=0.516, sum=1.032 (2)", - "tab": "Accuracy", - "Formal Logic - Observed inference time (s)": "{\"description\": \"min=2.327, mean=2.327, max=2.327, sum=4.653 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.3266589963246904\"}", - "Formal Logic - # eval": "{\"description\": \"min=126, mean=126, max=126, sum=252 (2)\", \"tab\": \"General information\", \"score\": \"126.0\"}", - "Formal Logic - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Formal Logic - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Formal Logic - # prompt tokens": "{\"description\": \"min=611.563, mean=611.563, max=611.563, sum=1223.127 (2)\", \"tab\": \"General information\", \"score\": \"611.563492063492\"}", - "Formal Logic - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"formal_logic\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_formal_logic\"" - } - } - }, - { - "evaluation_name": "High School World History", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on High School World History", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.89, - "details": { - "description": "min=0.89, mean=0.89, max=0.89, sum=1.781 (2)", - "tab": "Accuracy", - "High School Biology - Observed inference time (s)": "{\"description\": \"min=2.021, mean=2.021, max=2.021, sum=4.043 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.021439305428536\"}", - "High School Chemistry - Observed inference time (s)": "{\"description\": \"min=2.053, mean=2.053, max=2.053, sum=4.106 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.0532372467623556\"}", - "High School Computer Science - Observed inference time (s)": "{\"description\": \"min=3.599, mean=3.599, max=3.599, sum=7.197 (2)\", \"tab\": \"Efficiency\", \"score\": \"3.5985250592231752\"}", - "High School European History - Observed inference time (s)": "{\"description\": \"min=12.207, mean=12.207, max=12.207, sum=24.413 (2)\", \"tab\": \"Efficiency\", \"score\": \"12.20667136221221\"}", - "High School Geography - Observed inference time (s)": "{\"description\": \"min=1.952, mean=1.952, max=1.952, sum=3.903 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.9516368020664563\"}", - "High School Government And Politics - Observed inference time (s)": "{\"description\": \"min=2.276, mean=2.276, max=2.276, sum=4.552 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.2759376226929184\"}", - "High School Macroeconomics - Observed inference time (s)": "{\"description\": \"min=1.97, mean=1.97, max=1.97, sum=3.94 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.9697805410776383\"}", - "High School Mathematics - Observed inference time (s)": "{\"description\": \"min=2.617, mean=2.617, max=2.617, sum=5.234 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.616950834238971\"}", - "High School Microeconomics - Observed inference time (s)": "{\"description\": \"min=2.123, mean=2.123, max=2.123, sum=4.245 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.1225664866070786\"}", - "High School Physics - Observed inference time (s)": "{\"description\": \"min=2.697, mean=2.697, max=2.697, sum=5.394 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.6972478115006\"}", - "High School Psychology - Observed inference time (s)": "{\"description\": \"min=2.368, mean=2.368, max=2.368, sum=4.735 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.3675809317772543\"}", - "High School Statistics - Observed inference time (s)": "{\"description\": \"min=3.958, mean=3.958, max=3.958, sum=7.917 (2)\", \"tab\": \"Efficiency\", \"score\": \"3.9584906564818487\"}", - "High School US History - Observed inference time (s)": "{\"description\": \"min=9.745, mean=9.745, max=9.745, sum=19.491 (2)\", \"tab\": \"Efficiency\", \"score\": \"9.745334922098646\"}", - "High School World History - Observed inference time (s)": "{\"description\": \"min=6.489, mean=6.489, max=6.489, sum=12.977 (2)\", \"tab\": \"Efficiency\", \"score\": \"6.488561074944991\"}", - "High School Biology - # eval": "{\"description\": \"min=310, mean=310, max=310, sum=620 (2)\", \"tab\": \"General information\", \"score\": \"310.0\"}", - "High School Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Biology - # prompt tokens": "{\"description\": \"min=492.958, mean=492.958, max=492.958, sum=985.916 (2)\", \"tab\": \"General information\", \"score\": \"492.958064516129\"}", - "High School Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Chemistry - # eval": "{\"description\": \"min=203, mean=203, max=203, sum=406 (2)\", \"tab\": \"General information\", \"score\": \"203.0\"}", - "High School Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # prompt tokens": "{\"description\": \"min=505.064, mean=505.064, max=505.064, sum=1010.128 (2)\", \"tab\": \"General information\", \"score\": \"505.064039408867\"}", - "High School Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "High School Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # prompt tokens": "{\"description\": \"min=927.13, mean=927.13, max=927.13, sum=1854.26 (2)\", \"tab\": \"General information\", \"score\": \"927.13\"}", - "High School Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School European History - # eval": "{\"description\": \"min=165, mean=165, max=165, sum=330 (2)\", \"tab\": \"General information\", \"score\": \"165.0\"}", - "High School European History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School European History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # prompt tokens": "{\"description\": \"min=2789.424, mean=2789.424, max=2789.424, sum=5578.848 (2)\", \"tab\": \"General information\", \"score\": \"2789.4242424242425\"}", - "High School European History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Geography - # eval": "{\"description\": \"min=198, mean=198, max=198, sum=396 (2)\", \"tab\": \"General information\", \"score\": \"198.0\"}", - "High School Geography - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Geography - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # prompt tokens": "{\"description\": \"min=386.773, mean=386.773, max=386.773, sum=773.545 (2)\", \"tab\": \"General information\", \"score\": \"386.77272727272725\"}", - "High School Geography - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Government And Politics - # eval": "{\"description\": \"min=193, mean=193, max=193, sum=386 (2)\", \"tab\": \"General information\", \"score\": \"193.0\"}", - "High School Government And Politics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Government And Politics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # prompt tokens": "{\"description\": \"min=471.301, mean=471.301, max=471.301, sum=942.601 (2)\", \"tab\": \"General information\", \"score\": \"471.30051813471505\"}", - "High School Government And Politics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Macroeconomics - # eval": "{\"description\": \"min=390, mean=390, max=390, sum=780 (2)\", \"tab\": \"General information\", \"score\": \"390.0\"}", - "High School Macroeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Macroeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # prompt tokens": "{\"description\": \"min=388.541, mean=388.541, max=388.541, sum=777.082 (2)\", \"tab\": \"General information\", \"score\": \"388.54102564102567\"}", - "High School Macroeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Mathematics - # eval": "{\"description\": \"min=270, mean=270, max=270, sum=540 (2)\", \"tab\": \"General information\", \"score\": \"270.0\"}", - "High School Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # prompt tokens": "{\"description\": \"min=558.822, mean=558.822, max=558.822, sum=1117.644 (2)\", \"tab\": \"General information\", \"score\": \"558.8222222222222\"}", - "High School Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Microeconomics - # eval": "{\"description\": \"min=238, mean=238, max=238, sum=476 (2)\", \"tab\": \"General information\", \"score\": \"238.0\"}", - "High School Microeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Microeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # prompt tokens": "{\"description\": \"min=407.954, mean=407.954, max=407.954, sum=815.908 (2)\", \"tab\": \"General information\", \"score\": \"407.953781512605\"}", - "High School Microeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Physics - # eval": "{\"description\": \"min=151, mean=151, max=151, sum=302 (2)\", \"tab\": \"General information\", \"score\": \"151.0\"}", - "High School Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # prompt tokens": "{\"description\": \"min=583.715, mean=583.715, max=583.715, sum=1167.43 (2)\", \"tab\": \"General information\", \"score\": \"583.7152317880794\"}", - "High School Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Psychology - # eval": "{\"description\": \"min=545, mean=545, max=545, sum=1090 (2)\", \"tab\": \"General information\", \"score\": \"545.0\"}", - "High School Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # prompt tokens": "{\"description\": \"min=494.604, mean=494.604, max=494.604, sum=989.207 (2)\", \"tab\": \"General information\", \"score\": \"494.60366972477067\"}", - "High School Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Statistics - # eval": "{\"description\": \"min=216, mean=216, max=216, sum=432 (2)\", \"tab\": \"General information\", \"score\": \"216.0\"}", - "High School Statistics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Statistics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # prompt tokens": "{\"description\": \"min=850.931, mean=850.931, max=850.931, sum=1701.861 (2)\", \"tab\": \"General information\", \"score\": \"850.9305555555555\"}", - "High School Statistics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School US History - # eval": "{\"description\": \"min=204, mean=204, max=204, sum=408 (2)\", \"tab\": \"General information\", \"score\": \"204.0\"}", - "High School US History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School US History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # prompt tokens": "{\"description\": \"min=2197.583, mean=2197.583, max=2197.583, sum=4395.167 (2)\", \"tab\": \"General information\", \"score\": \"2197.5833333333335\"}", - "High School US History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School World History - # eval": "{\"description\": \"min=237, mean=237, max=237, sum=474 (2)\", \"tab\": \"General information\", \"score\": \"237.0\"}", - "High School World History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School World History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # prompt tokens": "{\"description\": \"min=1418.544, mean=1418.544, max=1418.544, sum=2837.089 (2)\", \"tab\": \"General information\", \"score\": \"1418.5443037974683\"}", - "High School World History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"high_school_world_history\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_high_school_world_history\"" - } - } - }, - { - "evaluation_name": "Human Sexuality", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Human Sexuality", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.84, - "details": { - "description": "min=0.84, mean=0.84, max=0.84, sum=1.679 (2)", - "tab": "Accuracy", - "Human Aging - Observed inference time (s)": "{\"description\": \"min=1.712, mean=1.712, max=1.712, sum=3.425 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.7123107461116773\"}", - "Human Sexuality - Observed inference time (s)": "{\"description\": \"min=1.754, mean=1.754, max=1.754, sum=3.508 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.7542339390470783\"}", - "Human Aging - # eval": "{\"description\": \"min=223, mean=223, max=223, sum=446 (2)\", \"tab\": \"General information\", \"score\": \"223.0\"}", - "Human Aging - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Aging - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Aging - # prompt tokens": "{\"description\": \"min=313.587, mean=313.587, max=313.587, sum=627.175 (2)\", \"tab\": \"General information\", \"score\": \"313.58744394618833\"}", - "Human Aging - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Human Sexuality - # eval": "{\"description\": \"min=131, mean=131, max=131, sum=262 (2)\", \"tab\": \"General information\", \"score\": \"131.0\"}", - "Human Sexuality - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Sexuality - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # prompt tokens": "{\"description\": \"min=339.183, mean=339.183, max=339.183, sum=678.366 (2)\", \"tab\": \"General information\", \"score\": \"339.1832061068702\"}", - "Human Sexuality - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"human_sexuality\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_human_sexuality\"" - } - } - }, - { - "evaluation_name": "International Law", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on International Law", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.843, - "details": { - "description": "min=0.843, mean=0.843, max=0.843, sum=1.686 (2)", - "tab": "Accuracy", - "International Law - Observed inference time (s)": "{\"description\": \"min=2.9, mean=2.9, max=2.9, sum=5.801 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.9003868654739757\"}", - "International Law - # eval": "{\"description\": \"min=121, mean=121, max=121, sum=242 (2)\", \"tab\": \"General information\", \"score\": \"121.0\"}", - "International Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "International Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "International Law - # prompt tokens": "{\"description\": \"min=636.165, mean=636.165, max=636.165, sum=1272.331 (2)\", \"tab\": \"General information\", \"score\": \"636.1652892561983\"}", - "International Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"international_law\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_international_law\"" - } - } - }, - { - "evaluation_name": "Logical Fallacies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Logical Fallacies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.865, - "details": { - "description": "min=0.865, mean=0.865, max=0.865, sum=1.73 (2)", - "tab": "Accuracy", - "Logical Fallacies - Observed inference time (s)": "{\"description\": \"min=2.154, mean=2.154, max=2.154, sum=4.308 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.1537599431956473\"}", - "Logical Fallacies - # eval": "{\"description\": \"min=163, mean=163, max=163, sum=326 (2)\", \"tab\": \"General information\", \"score\": \"163.0\"}", - "Logical Fallacies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Logical Fallacies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Logical Fallacies - # prompt tokens": "{\"description\": \"min=442.049, mean=442.049, max=442.049, sum=884.098 (2)\", \"tab\": \"General information\", \"score\": \"442.0490797546012\"}", - "Logical Fallacies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"logical_fallacies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_logical_fallacies\"" - } - } - }, - { - "evaluation_name": "Machine Learning", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Machine Learning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.625, - "details": { - "description": "min=0.625, mean=0.625, max=0.625, sum=1.25 (2)", - "tab": "Accuracy", - "Machine Learning - Observed inference time (s)": "{\"description\": \"min=3.172, mean=3.172, max=3.172, sum=6.344 (2)\", \"tab\": \"Efficiency\", \"score\": \"3.172234045607703\"}", - "Machine Learning - # eval": "{\"description\": \"min=112, mean=112, max=112, sum=224 (2)\", \"tab\": \"General information\", \"score\": \"112.0\"}", - "Machine Learning - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Machine Learning - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Machine Learning - # prompt tokens": "{\"description\": \"min=694.402, mean=694.402, max=694.402, sum=1388.804 (2)\", \"tab\": \"General information\", \"score\": \"694.4017857142857\"}", - "Machine Learning - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"machine_learning\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_machine_learning\"" - } - } - }, - { - "evaluation_name": "Management", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Management", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.864, - "details": { - "description": "min=0.864, mean=0.864, max=0.864, sum=1.728 (2)", - "tab": "Accuracy", - "Management - Observed inference time (s)": "{\"description\": \"min=1.556, mean=1.556, max=1.556, sum=3.112 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.5561023800118456\"}", - "Management - # eval": "{\"description\": \"min=103, mean=103, max=103, sum=206 (2)\", \"tab\": \"General information\", \"score\": \"103.0\"}", - "Management - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Management - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Management - # prompt tokens": "{\"description\": \"min=273.301, mean=273.301, max=273.301, sum=546.602 (2)\", \"tab\": \"General information\", \"score\": \"273.3009708737864\"}", - "Management - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"management\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_management\"" - } - } - }, - { - "evaluation_name": "Marketing", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Marketing", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.94, - "details": { - "description": "min=0.94, mean=0.94, max=0.94, sum=1.88 (2)", - "tab": "Accuracy", - "Marketing - Observed inference time (s)": "{\"description\": \"min=2.165, mean=2.165, max=2.165, sum=4.331 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.1654122140672474\"}", - "Marketing - # eval": "{\"description\": \"min=234, mean=234, max=234, sum=468 (2)\", \"tab\": \"General information\", \"score\": \"234.0\"}", - "Marketing - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Marketing - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Marketing - # prompt tokens": "{\"description\": \"min=420.35, mean=420.35, max=420.35, sum=840.701 (2)\", \"tab\": \"General information\", \"score\": \"420.35042735042737\"}", - "Marketing - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"marketing\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_marketing\"" - } - } - }, - { - "evaluation_name": "Medical Genetics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Medical Genetics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.87, - "details": { - "description": "min=0.87, mean=0.87, max=0.87, sum=1.74 (2)", - "tab": "Accuracy", - "Medical Genetics - Observed inference time (s)": "{\"description\": \"min=1.719, mean=1.719, max=1.719, sum=3.438 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.7190089011192322\"}", - "Medical Genetics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Medical Genetics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Medical Genetics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Medical Genetics - # prompt tokens": "{\"description\": \"min=330.89, mean=330.89, max=330.89, sum=661.78 (2)\", \"tab\": \"General information\", \"score\": \"330.89\"}", - "Medical Genetics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"medical_genetics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_medical_genetics\"" - } - } - }, - { - "evaluation_name": "Miscellaneous", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Miscellaneous", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.885, - "details": { - "description": "min=0.885, mean=0.885, max=0.885, sum=1.77 (2)", - "tab": "Accuracy", - "Miscellaneous - Observed inference time (s)": "{\"description\": \"min=1.709, mean=1.709, max=1.709, sum=3.417 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.7086633363141563\"}", - "Miscellaneous - # eval": "{\"description\": \"min=783, mean=783, max=783, sum=1566 (2)\", \"tab\": \"General information\", \"score\": \"783.0\"}", - "Miscellaneous - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Miscellaneous - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Miscellaneous - # prompt tokens": "{\"description\": \"min=306.669, mean=306.669, max=306.669, sum=613.338 (2)\", \"tab\": \"General information\", \"score\": \"306.669220945083\"}", - "Miscellaneous - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"miscellaneous\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_miscellaneous\"" - } - } - }, - { - "evaluation_name": "Moral Scenarios", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Moral Scenarios", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.394, - "details": { - "description": "min=0.394, mean=0.394, max=0.394, sum=0.789 (2)", - "tab": "Accuracy", - "Moral Disputes - Observed inference time (s)": "{\"description\": \"min=2.315, mean=2.315, max=2.315, sum=4.631 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.315398308583078\"}", - "Moral Scenarios - Observed inference time (s)": "{\"description\": \"min=3.188, mean=3.188, max=3.188, sum=6.376 (2)\", \"tab\": \"Efficiency\", \"score\": \"3.187839964914588\"}", - "Moral Disputes - # eval": "{\"description\": \"min=346, mean=346, max=346, sum=692 (2)\", \"tab\": \"General information\", \"score\": \"346.0\"}", - "Moral Disputes - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Disputes - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Disputes - # prompt tokens": "{\"description\": \"min=487.003, mean=487.003, max=487.003, sum=974.006 (2)\", \"tab\": \"General information\", \"score\": \"487.0028901734104\"}", - "Moral Disputes - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Moral Scenarios - # eval": "{\"description\": \"min=895, mean=895, max=895, sum=1790 (2)\", \"tab\": \"General information\", \"score\": \"895.0\"}", - "Moral Scenarios - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Scenarios - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # prompt tokens": "{\"description\": \"min=682.542, mean=682.542, max=682.542, sum=1365.084 (2)\", \"tab\": \"General information\", \"score\": \"682.5418994413408\"}", - "Moral Scenarios - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"moral_scenarios\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_moral_scenarios\"" - } - } - }, - { - "evaluation_name": "Nutrition", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Nutrition", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.824, - "details": { - "description": "min=0.824, mean=0.824, max=0.824, sum=1.647 (2)", - "tab": "Accuracy", - "Nutrition - Observed inference time (s)": "{\"description\": \"min=2.692, mean=2.692, max=2.692, sum=5.383 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.691618916255976\"}", - "Nutrition - # eval": "{\"description\": \"min=306, mean=306, max=306, sum=612 (2)\", \"tab\": \"General information\", \"score\": \"306.0\"}", - "Nutrition - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Nutrition - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Nutrition - # prompt tokens": "{\"description\": \"min=577.48, mean=577.48, max=577.48, sum=1154.961 (2)\", \"tab\": \"General information\", \"score\": \"577.4803921568628\"}", - "Nutrition - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"nutrition\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_nutrition\"" - } - } - }, - { - "evaluation_name": "Prehistory", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Prehistory", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.877, - "details": { - "description": "min=0.877, mean=0.877, max=0.877, sum=1.753 (2)", - "tab": "Accuracy", - "Prehistory - Observed inference time (s)": "{\"description\": \"min=2.537, mean=2.537, max=2.537, sum=5.075 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.5372923561084417\"}", - "Prehistory - # eval": "{\"description\": \"min=324, mean=324, max=324, sum=648 (2)\", \"tab\": \"General information\", \"score\": \"324.0\"}", - "Prehistory - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Prehistory - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Prehistory - # prompt tokens": "{\"description\": \"min=532.198, mean=532.198, max=532.198, sum=1064.395 (2)\", \"tab\": \"General information\", \"score\": \"532.1975308641976\"}", - "Prehistory - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"prehistory\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_prehistory\"" - } - } - }, - { - "evaluation_name": "Public Relations", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Public Relations", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.745, - "details": { - "description": "min=0.745, mean=0.745, max=0.745, sum=1.491 (2)", - "tab": "Accuracy", - "Public Relations - Observed inference time (s)": "{\"description\": \"min=2.161, mean=2.161, max=2.161, sum=4.321 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.160554786161943\"}", - "Public Relations - # eval": "{\"description\": \"min=110, mean=110, max=110, sum=220 (2)\", \"tab\": \"General information\", \"score\": \"110.0\"}", - "Public Relations - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Public Relations - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Public Relations - # prompt tokens": "{\"description\": \"min=418.655, mean=418.655, max=418.655, sum=837.309 (2)\", \"tab\": \"General information\", \"score\": \"418.6545454545454\"}", - "Public Relations - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"public_relations\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_public_relations\"" - } - } - }, - { - "evaluation_name": "Security Studies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Security Studies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.808, - "details": { - "description": "min=0.808, mean=0.808, max=0.808, sum=1.616 (2)", - "tab": "Accuracy", - "Security Studies - Observed inference time (s)": "{\"description\": \"min=5.336, mean=5.336, max=5.336, sum=10.672 (2)\", \"tab\": \"Efficiency\", \"score\": \"5.335982258465825\"}", - "Security Studies - # eval": "{\"description\": \"min=245, mean=245, max=245, sum=490 (2)\", \"tab\": \"General information\", \"score\": \"245.0\"}", - "Security Studies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Security Studies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Security Studies - # prompt tokens": "{\"description\": \"min=1185.869, mean=1185.869, max=1185.869, sum=2371.739 (2)\", \"tab\": \"General information\", \"score\": \"1185.869387755102\"}", - "Security Studies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"security_studies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_security_studies\"" - } - } - }, - { - "evaluation_name": "Sociology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Sociology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9, - "details": { - "description": "min=0.9, mean=0.9, max=0.9, sum=1.801 (2)", - "tab": "Accuracy", - "Sociology - Observed inference time (s)": "{\"description\": \"min=2.204, mean=2.204, max=2.204, sum=4.409 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.2043708201071515\"}", - "Sociology - # eval": "{\"description\": \"min=201, mean=201, max=201, sum=402 (2)\", \"tab\": \"General information\", \"score\": \"201.0\"}", - "Sociology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Sociology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Sociology - # prompt tokens": "{\"description\": \"min=448.274, mean=448.274, max=448.274, sum=896.547 (2)\", \"tab\": \"General information\", \"score\": \"448.27363184079604\"}", - "Sociology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"sociology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_sociology\"" - } - } - }, - { - "evaluation_name": "Virology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Virology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.56, - "details": { - "description": "min=0.56, mean=0.56, max=0.56, sum=1.12 (2)", - "tab": "Accuracy", - "Virology - Observed inference time (s)": "{\"description\": \"min=1.75, mean=1.75, max=1.75, sum=3.499 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.7496386393007026\"}", - "Virology - # eval": "{\"description\": \"min=166, mean=166, max=166, sum=332 (2)\", \"tab\": \"General information\", \"score\": \"166.0\"}", - "Virology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Virology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Virology - # prompt tokens": "{\"description\": \"min=328.753, mean=328.753, max=328.753, sum=657.506 (2)\", \"tab\": \"General information\", \"score\": \"328.7530120481928\"}", - "Virology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"virology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_virology\"" - } - } - }, - { - "evaluation_name": "World Religions", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on World Religions", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.924, - "details": { - "description": "min=0.924, mean=0.924, max=0.924, sum=1.848 (2)", - "tab": "Accuracy", - "World Religions - Observed inference time (s)": "{\"description\": \"min=1.443, mean=1.443, max=1.443, sum=2.886 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.443225710015548\"}", - "World Religions - # eval": "{\"description\": \"min=171, mean=171, max=171, sum=342 (2)\", \"tab\": \"General information\", \"score\": \"171.0\"}", - "World Religions - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "World Religions - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "World Religions - # prompt tokens": "{\"description\": \"min=260.164, mean=260.164, max=260.164, sum=520.327 (2)\", \"tab\": \"General information\", \"score\": \"260.1637426900585\"}", - "World Religions - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"world_religions\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_world_religions\"" - } - } - }, - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.05, - "details": { - "description": "", - "tab": "Efficiency" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_id": "hfopenllm_v2/google_gemma-2-27b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2475 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5643 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1662 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3507 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4396 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4371 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/google_gemma-2-2b-it.json b/data/models/google_gemma-2-2b-it.json deleted file mode 100644 index 2256606de347975e81a80b3b27803f39e204f41b..0000000000000000000000000000000000000000 --- a/data/models/google_gemma-2-2b-it.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "gemma-2-2b-it", - "id": "google/gemma-2-2b-it", - "developer": "Google", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "InternLM2ForCausalLM", - "params_billions": "2.614" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/google_gemma-2-2b-it/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5668 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4199 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0008 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2743 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3929 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.255 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/google_gemma-2-2b-jpn-it.json b/data/models/google_gemma-2-2b-jpn-it.json deleted file mode 100644 index 208075f00df888593e36a118e84f64272df933ff..0000000000000000000000000000000000000000 --- a/data/models/google_gemma-2-2b-jpn-it.json +++ /dev/null @@ -1,275 +0,0 @@ -{ - "model_info": { - "name": "gemma-2-2b-jpn-it", - "id": "google/gemma-2-2b-jpn-it", - "developer": "Google", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "2.614" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/google_gemma-2-2b-jpn-it/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5078 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4226 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0347 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2852 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3964 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2578 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "hfopenllm_v2/google_gemma-2-2b-jpn-it/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5288 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4178 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0476 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2752 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3728 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2467 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/google_gemma-2-2b.json b/data/models/google_gemma-2-2b.json deleted file mode 100644 index 3875c2305ff98601b733940bab48bde956384743..0000000000000000000000000000000000000000 --- a/data/models/google_gemma-2-2b.json +++ /dev/null @@ -1,275 +0,0 @@ -{ - "model_info": { - "name": "gemma-2-2b", - "id": "google/gemma-2-2b", - "developer": "Google", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "InternLM2ForCausalLM", - "params_billions": "2.614" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/google_gemma-2-2b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2018 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3709 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0302 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2626 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4219 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2217 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "hfopenllm_v2/google_gemma-2-2b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1993 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3656 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0287 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2626 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4232 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.218 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/google_gemma-2-9b-it.json b/data/models/google_gemma-2-9b-it.json deleted file mode 100644 index 2c4dd608d4b042d55c3ae88d1f12d6fac7ade6db..0000000000000000000000000000000000000000 --- a/data/models/google_gemma-2-9b-it.json +++ /dev/null @@ -1,552 +0,0 @@ -{ - "model_info": { - "name": "Gemma 2 Instruct 9B", - "id": "google/gemma-2-9b-it", - "developer": "Google", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_lite/google_gemma-2-9b-it/1774096306.427425", - "retrieved_timestamp": "1774096306.427425", - "source_metadata": { - "source_name": "helm_lite", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_lite", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_lite", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.562, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.8286641697877652\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.768, - "details": { - "description": "min=0.768, mean=0.768, max=0.768, sum=0.768 (1)", - "tab": "Accuracy", - "NarrativeQA - Observed inference time (s)": "{\"description\": \"min=0.593, mean=0.593, max=0.593, sum=0.593 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.5928616705075116\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=3437.994, mean=3437.994, max=3437.994, sum=3437.994 (1)\", \"tab\": \"General information\", \"score\": \"3437.994366197183\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (closed-book)", - "source_data": { - "dataset_name": "NaturalQuestions (closed-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (closed-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.328, - "details": { - "description": "min=0.328, mean=0.328, max=0.328, sum=0.328 (1)", - "tab": "Accuracy", - "NaturalQuestions (open-book) - Observed inference time (s)": "{\"description\": \"min=0.446, mean=0.446, max=0.446, sum=0.446 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.44568803215026853\"}", - "NaturalQuestions (closed-book) - Observed inference time (s)": "{\"description\": \"min=0.337, mean=0.337, max=0.337, sum=0.337 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.337234415769577\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.953, mean=4.953, max=4.953, sum=4.953 (1)\", \"tab\": \"General information\", \"score\": \"4.953\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.009, mean=0.009, max=0.009, sum=0.009 (1)\", \"tab\": \"General information\", \"score\": \"0.009\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1911.526, mean=1911.526, max=1911.526, sum=1911.526 (1)\", \"tab\": \"General information\", \"score\": \"1911.526\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=143.995, mean=143.995, max=143.995, sum=143.995 (1)\", \"tab\": \"General information\", \"score\": \"143.995\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "mode": "\"closedbook\"" - } - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.91, - "details": { - "description": "min=0.91, mean=0.91, max=0.91, sum=0.91 (1)", - "tab": "Accuracy", - "OpenbookQA - Observed inference time (s)": "{\"description\": \"min=0.306, mean=0.306, max=0.306, sum=0.306 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.3059106550216675\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=248.508, mean=248.508, max=248.508, sum=248.508 (1)\", \"tab\": \"General information\", \"score\": \"248.508\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "dataset": "\"openbookqa\"", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.645, - "details": { - "description": "min=0.42, mean=0.645, max=0.91, sum=3.225 (5)", - "tab": "Accuracy", - "MMLU - Observed inference time (s)": "{\"description\": \"min=0.299, mean=0.319, max=0.334, sum=1.594 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.3187573717686168\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=380.91, mean=481.531, max=634.553, sum=2407.653 (5)\", \"tab\": \"General information\", \"score\": \"481.5305263157895\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"college_chemistry\", \"computer_security\", \"econometrics\", \"us_foreign_policy\"]", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MATH", - "source_data": { - "dataset_name": "MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Equivalent (CoT) on MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.724, - "details": { - "description": "min=0.635, mean=0.724, max=0.907, sum=5.071 (7)", - "tab": "Accuracy", - "MATH - Observed inference time (s)": "{\"description\": \"min=1.006, mean=1.344, max=1.765, sum=9.409 (7)\", \"tab\": \"Efficiency\", \"score\": \"1.3440718759718908\"}", - "MATH - # eval": "{\"description\": \"min=30, mean=62.429, max=135, sum=437 (7)\", \"tab\": \"General information\", \"score\": \"62.42857142857143\"}", - "MATH - # train": "{\"description\": \"min=8, mean=8, max=8, sum=56 (7)\", \"tab\": \"General information\", \"score\": \"8.0\"}", - "MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MATH - # prompt tokens": "{\"description\": \"min=938.215, mean=1355.506, max=2348.712, sum=9488.545 (7)\", \"tab\": \"General information\", \"score\": \"1355.5064552904823\"}", - "MATH - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=7 (7)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"algebra\", \"counting_and_probability\", \"geometry\", \"intermediate_algebra\", \"number_theory\", \"prealgebra\", \"precalculus\"]", - "level": "\"1\"", - "use_official_examples": "\"False\"", - "use_chain_of_thought": "\"True\"" - } - } - }, - { - "evaluation_name": "GSM8K", - "source_data": { - "dataset_name": "GSM8K", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on GSM8K", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.762, - "details": { - "description": "min=0.762, mean=0.762, max=0.762, sum=0.762 (1)", - "tab": "Accuracy", - "GSM8K - Observed inference time (s)": "{\"description\": \"min=1.72, mean=1.72, max=1.72, sum=1.72 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.720498773097992\"}", - "GSM8K - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "GSM8K - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "GSM8K - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GSM8K - # prompt tokens": "{\"description\": \"min=1151.885, mean=1151.885, max=1151.885, sum=1151.885 (1)\", \"tab\": \"General information\", \"score\": \"1151.885\"}", - "GSM8K - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "stop": "\"none\"" - } - } - }, - { - "evaluation_name": "LegalBench", - "source_data": { - "dataset_name": "LegalBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on LegalBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.639, - "details": { - "description": "min=0.395, mean=0.639, max=0.937, sum=3.193 (5)", - "tab": "Accuracy", - "LegalBench - Observed inference time (s)": "{\"description\": \"min=0.31, mean=0.384, max=0.652, sum=1.92 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.3840073023663075\"}", - "LegalBench - # eval": "{\"description\": \"min=95, mean=409.4, max=1000, sum=2047 (5)\", \"tab\": \"General information\", \"score\": \"409.4\"}", - "LegalBench - # train": "{\"description\": \"min=4, mean=4.798, max=5, sum=23.992 (5)\", \"tab\": \"General information\", \"score\": \"4.798367346938775\"}", - "LegalBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "LegalBench - # prompt tokens": "{\"description\": \"min=199.916, mean=1546.699, max=6405.871, sum=7733.495 (5)\", \"tab\": \"General information\", \"score\": \"1546.699013263404\"}", - "LegalBench - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"abercrombie\", \"corporate_lobbying\", \"function_of_decision_section\", \"international_citizenship_questions\", \"proa\"]" - } - } - }, - { - "evaluation_name": "MedQA", - "source_data": { - "dataset_name": "MedQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MedQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.63, - "details": { - "description": "min=0.63, mean=0.63, max=0.63, sum=0.63 (1)", - "tab": "Accuracy", - "MedQA - Observed inference time (s)": "{\"description\": \"min=0.316, mean=0.316, max=0.316, sum=0.316 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.3161872125288127\"}", - "MedQA - # eval": "{\"description\": \"min=503, mean=503, max=503, sum=503 (1)\", \"tab\": \"General information\", \"score\": \"503.0\"}", - "MedQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MedQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MedQA - # prompt tokens": "{\"description\": \"min=1029.481, mean=1029.481, max=1029.481, sum=1029.481 (1)\", \"tab\": \"General information\", \"score\": \"1029.4811133200794\"}", - "MedQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WMT 2014", - "source_data": { - "dataset_name": "WMT 2014", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "BLEU-4 on WMT 2014", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.201, - "details": { - "description": "min=0.155, mean=0.201, max=0.228, sum=1.003 (5)", - "tab": "Accuracy", - "WMT 2014 - Observed inference time (s)": "{\"description\": \"min=0.526, mean=0.633, max=0.82, sum=3.165 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.6330890842213928\"}", - "WMT 2014 - # eval": "{\"description\": \"min=503, mean=568.8, max=832, sum=2844 (5)\", \"tab\": \"General information\", \"score\": \"568.8\"}", - "WMT 2014 - # train": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "WMT 2014 - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WMT 2014 - # prompt tokens": "{\"description\": \"min=80.732, mean=110.97, max=137.366, sum=554.851 (5)\", \"tab\": \"General information\", \"score\": \"110.97025108961614\"}", - "WMT 2014 - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "language_pair": "[\"cs-en\", \"de-en\", \"fr-en\", \"hi-en\", \"ru-en\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "hfopenllm_v2/google_gemma-2-9b-it/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7436 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.599 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1949 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3607 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4073 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3875 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "la_leaderboard/google/gemma-2-9b-it/1774451270", - "retrieved_timestamp": "2024-10-27T00:00:00Z", - "source_metadata": { - "source_name": "La Leaderboard", - "source_type": "evaluation_run", - "source_url": "https://huggingface.co/spaces/la-leaderboard/la-leaderboard", - "source_organization_name": "La Leaderboard", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "custom", - "version": "1.0" - }, - "benchmark": "la_leaderboard", - "evaluation_results": [ - { - "evaluation_name": "la_leaderboard", - "metric_config": { - "evaluation_description": "La Leaderboard: LLM evaluation for Spanish varieties and languages of Spain and Latin America", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 100 - }, - "score_details": { - "score": 33.62 - }, - "source_data": { - "source_type": "url", - "dataset_name": "La Leaderboard composite dataset", - "url": [ - "https://huggingface.co/spaces/la-leaderboard/la-leaderboard" - ] - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/google_gemma-2-9b.json b/data/models/google_gemma-2-9b.json deleted file mode 100644 index 15139d3c25168c6a497fc41c44cd46a819ab73fd..0000000000000000000000000000000000000000 --- a/data/models/google_gemma-2-9b.json +++ /dev/null @@ -1,1661 +0,0 @@ -{ - "model_info": { - "name": "Gemma 2 9B", - "id": "google/gemma-2-9b", - "developer": "Google", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_mmlu/google_gemma-2-9b/1774096312.00548", - "retrieved_timestamp": "1774096312.00548", - "source_metadata": { - "source_name": "helm_mmlu", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_mmlu", - "evaluation_results": [ - { - "evaluation_name": "MMLU All Subjects", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU All Subjects", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.721, - "details": { - "description": "min=0.295, mean=0.721, max=0.953, sum=82.233 (114)", - "tab": "Accuracy", - "MMLU All Subjects - Observed inference time (s)": "{\"description\": \"min=0.425, mean=0.901, max=3.986, sum=102.765 (114)\", \"tab\": \"Efficiency\", \"score\": \"0.9014510090022484\"}", - "MMLU All Subjects - # eval": "{\"description\": \"min=100, mean=246.351, max=1534, sum=28084 (114)\", \"tab\": \"General information\", \"score\": \"246.35087719298247\"}", - "MMLU All Subjects - # train": "{\"description\": \"min=5, mean=5, max=5, sum=570 (114)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU All Subjects - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU All Subjects - # prompt tokens": "{\"description\": \"min=260.164, mean=624.617, max=2789.424, sum=71206.345 (114)\", \"tab\": \"General information\", \"score\": \"624.6170571214202\"}", - "MMLU All Subjects - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=114 (114)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_name": "Abstract Algebra", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Abstract Algebra", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4, - "details": { - "description": "min=0.4, mean=0.4, max=0.4, sum=0.8 (2)", - "tab": "Accuracy", - "Abstract Algebra - Observed inference time (s)": "{\"description\": \"min=0.65, mean=0.65, max=0.65, sum=1.3 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6499301409721374\"}", - "Abstract Algebra - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Abstract Algebra - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Abstract Algebra - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Abstract Algebra - # prompt tokens": "{\"description\": \"min=375.97, mean=375.97, max=375.97, sum=751.94 (2)\", \"tab\": \"General information\", \"score\": \"375.97\"}", - "Abstract Algebra - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"abstract_algebra\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_abstract_algebra\"" - } - } - }, - { - "evaluation_name": "Anatomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Anatomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.704, - "details": { - "description": "min=0.704, mean=0.704, max=0.704, sum=1.407 (2)", - "tab": "Accuracy", - "Anatomy - Observed inference time (s)": "{\"description\": \"min=0.492, mean=0.492, max=0.492, sum=0.984 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.491805742405079\"}", - "Anatomy - # eval": "{\"description\": \"min=135, mean=135, max=135, sum=270 (2)\", \"tab\": \"General information\", \"score\": \"135.0\"}", - "Anatomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Anatomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Anatomy - # prompt tokens": "{\"description\": \"min=336.356, mean=336.356, max=336.356, sum=672.711 (2)\", \"tab\": \"General information\", \"score\": \"336.35555555555555\"}", - "Anatomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"anatomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_anatomy\"" - } - } - }, - { - "evaluation_name": "College Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on College Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5, - "details": { - "description": "min=0.5, mean=0.5, max=0.5, sum=1 (2)", - "tab": "Accuracy", - "College Chemistry - Observed inference time (s)": "{\"description\": \"min=0.711, mean=0.711, max=0.711, sum=1.423 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7114056801795959\"}", - "College Biology - Observed inference time (s)": "{\"description\": \"min=0.624, mean=0.624, max=0.624, sum=1.248 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6241771280765533\"}", - "College Computer Science - Observed inference time (s)": "{\"description\": \"min=1.093, mean=1.093, max=1.093, sum=2.187 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.0932785439491273\"}", - "College Mathematics - Observed inference time (s)": "{\"description\": \"min=0.803, mean=0.803, max=0.803, sum=1.606 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8027684283256531\"}", - "College Medicine - Observed inference time (s)": "{\"description\": \"min=0.674, mean=0.674, max=0.674, sum=1.348 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6739495985769812\"}", - "College Physics - Observed inference time (s)": "{\"description\": \"min=0.655, mean=0.655, max=0.655, sum=1.311 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6553734166949403\"}", - "College Chemistry - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Chemistry - # prompt tokens": "{\"description\": \"min=562.02, mean=562.02, max=562.02, sum=1124.04 (2)\", \"tab\": \"General information\", \"score\": \"562.02\"}", - "College Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Biology - # eval": "{\"description\": \"min=144, mean=144, max=144, sum=288 (2)\", \"tab\": \"General information\", \"score\": \"144.0\"}", - "College Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # prompt tokens": "{\"description\": \"min=474.799, mean=474.799, max=474.799, sum=949.597 (2)\", \"tab\": \"General information\", \"score\": \"474.7986111111111\"}", - "College Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # prompt tokens": "{\"description\": \"min=849.86, mean=849.86, max=849.86, sum=1699.72 (2)\", \"tab\": \"General information\", \"score\": \"849.86\"}", - "College Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Mathematics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # prompt tokens": "{\"description\": \"min=618.69, mean=618.69, max=618.69, sum=1237.38 (2)\", \"tab\": \"General information\", \"score\": \"618.69\"}", - "College Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Medicine - # eval": "{\"description\": \"min=173, mean=173, max=173, sum=346 (2)\", \"tab\": \"General information\", \"score\": \"173.0\"}", - "College Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # prompt tokens": "{\"description\": \"min=505.37, mean=505.37, max=505.37, sum=1010.74 (2)\", \"tab\": \"General information\", \"score\": \"505.3699421965318\"}", - "College Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Physics - # eval": "{\"description\": \"min=102, mean=102, max=102, sum=204 (2)\", \"tab\": \"General information\", \"score\": \"102.0\"}", - "College Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # prompt tokens": "{\"description\": \"min=499.471, mean=499.471, max=499.471, sum=998.941 (2)\", \"tab\": \"General information\", \"score\": \"499.47058823529414\"}", - "College Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"college_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_college_physics\"" - } - } - }, - { - "evaluation_name": "Computer Security", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Computer Security", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.81, - "details": { - "description": "min=0.81, mean=0.81, max=0.81, sum=1.62 (2)", - "tab": "Accuracy", - "Computer Security - Observed inference time (s)": "{\"description\": \"min=0.464, mean=0.464, max=0.464, sum=0.928 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4640101146697998\"}", - "Computer Security - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Computer Security - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Computer Security - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Computer Security - # prompt tokens": "{\"description\": \"min=372.91, mean=372.91, max=372.91, sum=745.82 (2)\", \"tab\": \"General information\", \"score\": \"372.91\"}", - "Computer Security - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"computer_security\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_computer_security\"" - } - } - }, - { - "evaluation_name": "Econometrics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Econometrics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.579, - "details": { - "description": "min=0.579, mean=0.579, max=0.579, sum=1.158 (2)", - "tab": "Accuracy", - "Econometrics - Observed inference time (s)": "{\"description\": \"min=0.766, mean=0.766, max=0.766, sum=1.531 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7655813254808125\"}", - "Econometrics - # eval": "{\"description\": \"min=114, mean=114, max=114, sum=228 (2)\", \"tab\": \"General information\", \"score\": \"114.0\"}", - "Econometrics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Econometrics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Econometrics - # prompt tokens": "{\"description\": \"min=626.553, mean=626.553, max=626.553, sum=1253.105 (2)\", \"tab\": \"General information\", \"score\": \"626.5526315789474\"}", - "Econometrics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"econometrics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_econometrics\"" - } - } - }, - { - "evaluation_name": "Global Facts", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Global Facts", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.53, - "details": { - "description": "min=0.53, mean=0.53, max=0.53, sum=1.06 (2)", - "tab": "Accuracy", - "Global Facts - Observed inference time (s)": "{\"description\": \"min=0.542, mean=0.542, max=0.542, sum=1.084 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5422105526924134\"}", - "Global Facts - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Global Facts - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Global Facts - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Global Facts - # prompt tokens": "{\"description\": \"min=448.54, mean=448.54, max=448.54, sum=897.08 (2)\", \"tab\": \"General information\", \"score\": \"448.54\"}", - "Global Facts - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"global_facts\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_global_facts\"" - } - } - }, - { - "evaluation_name": "Jurisprudence", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Jurisprudence", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.833, - "details": { - "description": "min=0.833, mean=0.833, max=0.833, sum=1.667 (2)", - "tab": "Accuracy", - "Jurisprudence - Observed inference time (s)": "{\"description\": \"min=0.496, mean=0.496, max=0.496, sum=0.991 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4956528963866057\"}", - "Jurisprudence - # eval": "{\"description\": \"min=108, mean=108, max=108, sum=216 (2)\", \"tab\": \"General information\", \"score\": \"108.0\"}", - "Jurisprudence - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Jurisprudence - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Jurisprudence - # prompt tokens": "{\"description\": \"min=399.87, mean=399.87, max=399.87, sum=799.741 (2)\", \"tab\": \"General information\", \"score\": \"399.8703703703704\"}", - "Jurisprudence - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"jurisprudence\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_jurisprudence\"" - } - } - }, - { - "evaluation_name": "Philosophy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Philosophy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.772, - "details": { - "description": "min=0.772, mean=0.772, max=0.772, sum=1.543 (2)", - "tab": "Accuracy", - "Philosophy - Observed inference time (s)": "{\"description\": \"min=0.425, mean=0.425, max=0.425, sum=0.85 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4251678066621639\"}", - "Philosophy - # eval": "{\"description\": \"min=311, mean=311, max=311, sum=622 (2)\", \"tab\": \"General information\", \"score\": \"311.0\"}", - "Philosophy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Philosophy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Philosophy - # prompt tokens": "{\"description\": \"min=332.907, mean=332.907, max=332.907, sum=665.814 (2)\", \"tab\": \"General information\", \"score\": \"332.90675241157555\"}", - "Philosophy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"philosophy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_philosophy\"" - } - } - }, - { - "evaluation_name": "Professional Psychology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Professional Psychology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.788, - "details": { - "description": "min=0.788, mean=0.788, max=0.788, sum=1.575 (2)", - "tab": "Accuracy", - "Professional Medicine - Observed inference time (s)": "{\"description\": \"min=1.37, mean=1.37, max=1.37, sum=2.74 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.3702202570789002\"}", - "Professional Accounting - Observed inference time (s)": "{\"description\": \"min=1.128, mean=1.128, max=1.128, sum=2.255 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.1277324375531352\"}", - "Professional Law - Observed inference time (s)": "{\"description\": \"min=2.433, mean=2.433, max=2.433, sum=4.866 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.433138657113564\"}", - "Professional Psychology - Observed inference time (s)": "{\"description\": \"min=0.909, mean=0.909, max=0.909, sum=1.818 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9092130824631336\"}", - "Professional Medicine - # eval": "{\"description\": \"min=272, mean=272, max=272, sum=544 (2)\", \"tab\": \"General information\", \"score\": \"272.0\"}", - "Professional Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Medicine - # prompt tokens": "{\"description\": \"min=1105.092, mean=1105.092, max=1105.092, sum=2210.184 (2)\", \"tab\": \"General information\", \"score\": \"1105.0919117647059\"}", - "Professional Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Accounting - # eval": "{\"description\": \"min=282, mean=282, max=282, sum=564 (2)\", \"tab\": \"General information\", \"score\": \"282.0\"}", - "Professional Accounting - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Accounting - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # prompt tokens": "{\"description\": \"min=747.418, mean=747.418, max=747.418, sum=1494.837 (2)\", \"tab\": \"General information\", \"score\": \"747.418439716312\"}", - "Professional Accounting - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Law - # eval": "{\"description\": \"min=1534, mean=1534, max=1534, sum=3068 (2)\", \"tab\": \"General information\", \"score\": \"1534.0\"}", - "Professional Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # prompt tokens": "{\"description\": \"min=1677.119, mean=1677.119, max=1677.119, sum=3354.239 (2)\", \"tab\": \"General information\", \"score\": \"1677.119295958279\"}", - "Professional Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Psychology - # eval": "{\"description\": \"min=612, mean=612, max=612, sum=1224 (2)\", \"tab\": \"General information\", \"score\": \"612.0\"}", - "Professional Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # prompt tokens": "{\"description\": \"min=586.363, mean=586.363, max=586.363, sum=1172.725 (2)\", \"tab\": \"General information\", \"score\": \"586.3627450980392\"}", - "Professional Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"professional_psychology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_professional_psychology\"" - } - } - }, - { - "evaluation_name": "Us Foreign Policy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Us Foreign Policy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9, - "details": { - "description": "min=0.9, mean=0.9, max=0.9, sum=1.8 (2)", - "tab": "Accuracy", - "Us Foreign Policy - Observed inference time (s)": "{\"description\": \"min=0.544, mean=0.544, max=0.544, sum=1.088 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5438596844673157\"}", - "Us Foreign Policy - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Us Foreign Policy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Us Foreign Policy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Us Foreign Policy - # prompt tokens": "{\"description\": \"min=430.2, mean=430.2, max=430.2, sum=860.4 (2)\", \"tab\": \"General information\", \"score\": \"430.2\"}", - "Us Foreign Policy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"us_foreign_policy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_us_foreign_policy\"" - } - } - }, - { - "evaluation_name": "Astronomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Astronomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.789, - "details": { - "description": "min=0.789, mean=0.789, max=0.789, sum=1.579 (2)", - "tab": "Accuracy", - "Astronomy - Observed inference time (s)": "{\"description\": \"min=0.766, mean=0.766, max=0.766, sum=1.533 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7662546744472102\"}", - "Astronomy - # eval": "{\"description\": \"min=152, mean=152, max=152, sum=304 (2)\", \"tab\": \"General information\", \"score\": \"152.0\"}", - "Astronomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Astronomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Astronomy - # prompt tokens": "{\"description\": \"min=594.421, mean=594.421, max=594.421, sum=1188.842 (2)\", \"tab\": \"General information\", \"score\": \"594.421052631579\"}", - "Astronomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"astronomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_astronomy\"" - } - } - }, - { - "evaluation_name": "Business Ethics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Business Ethics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.77, - "details": { - "description": "min=0.77, mean=0.77, max=0.77, sum=1.54 (2)", - "tab": "Accuracy", - "Business Ethics - Observed inference time (s)": "{\"description\": \"min=0.713, mean=0.713, max=0.713, sum=1.425 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7125983119010926\"}", - "Business Ethics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Business Ethics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Business Ethics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Business Ethics - # prompt tokens": "{\"description\": \"min=544.87, mean=544.87, max=544.87, sum=1089.74 (2)\", \"tab\": \"General information\", \"score\": \"544.87\"}", - "Business Ethics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"business_ethics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_business_ethics\"" - } - } - }, - { - "evaluation_name": "Clinical Knowledge", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Clinical Knowledge", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.777, - "details": { - "description": "min=0.777, mean=0.777, max=0.777, sum=1.555 (2)", - "tab": "Accuracy", - "Clinical Knowledge - Observed inference time (s)": "{\"description\": \"min=0.561, mean=0.561, max=0.561, sum=1.121 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5606130177119992\"}", - "Clinical Knowledge - # eval": "{\"description\": \"min=265, mean=265, max=265, sum=530 (2)\", \"tab\": \"General information\", \"score\": \"265.0\"}", - "Clinical Knowledge - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Clinical Knowledge - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Clinical Knowledge - # prompt tokens": "{\"description\": \"min=394.592, mean=394.592, max=394.592, sum=789.185 (2)\", \"tab\": \"General information\", \"score\": \"394.5924528301887\"}", - "Clinical Knowledge - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"clinical_knowledge\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_clinical_knowledge\"" - } - } - }, - { - "evaluation_name": "Conceptual Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Conceptual Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.732, - "details": { - "description": "min=0.732, mean=0.732, max=0.732, sum=1.464 (2)", - "tab": "Accuracy", - "Conceptual Physics - Observed inference time (s)": "{\"description\": \"min=0.44, mean=0.44, max=0.44, sum=0.879 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4395242579439853\"}", - "Conceptual Physics - # eval": "{\"description\": \"min=235, mean=235, max=235, sum=470 (2)\", \"tab\": \"General information\", \"score\": \"235.0\"}", - "Conceptual Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Conceptual Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Conceptual Physics - # prompt tokens": "{\"description\": \"min=301.213, mean=301.213, max=301.213, sum=602.426 (2)\", \"tab\": \"General information\", \"score\": \"301.21276595744683\"}", - "Conceptual Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"conceptual_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_conceptual_physics\"" - } - } - }, - { - "evaluation_name": "Electrical Engineering", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Electrical Engineering", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.724, - "details": { - "description": "min=0.724, mean=0.724, max=0.724, sum=1.448 (2)", - "tab": "Accuracy", - "Electrical Engineering - Observed inference time (s)": "{\"description\": \"min=0.621, mean=0.621, max=0.621, sum=1.242 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.620852176074324\"}", - "Electrical Engineering - # eval": "{\"description\": \"min=145, mean=145, max=145, sum=290 (2)\", \"tab\": \"General information\", \"score\": \"145.0\"}", - "Electrical Engineering - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Electrical Engineering - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Electrical Engineering - # prompt tokens": "{\"description\": \"min=466.786, mean=466.786, max=466.786, sum=933.572 (2)\", \"tab\": \"General information\", \"score\": \"466.78620689655173\"}", - "Electrical Engineering - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"electrical_engineering\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_electrical_engineering\"" - } - } - }, - { - "evaluation_name": "Elementary Mathematics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Elementary Mathematics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.577, - "details": { - "description": "min=0.577, mean=0.577, max=0.577, sum=1.153 (2)", - "tab": "Accuracy", - "Elementary Mathematics - Observed inference time (s)": "{\"description\": \"min=0.783, mean=0.783, max=0.783, sum=1.566 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7831445295343954\"}", - "Elementary Mathematics - # eval": "{\"description\": \"min=378, mean=378, max=378, sum=756 (2)\", \"tab\": \"General information\", \"score\": \"378.0\"}", - "Elementary Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Elementary Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Elementary Mathematics - # prompt tokens": "{\"description\": \"min=589.341, mean=589.341, max=589.341, sum=1178.683 (2)\", \"tab\": \"General information\", \"score\": \"589.3412698412699\"}", - "Elementary Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"elementary_mathematics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_elementary_mathematics\"" - } - } - }, - { - "evaluation_name": "Formal Logic", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Formal Logic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.492, - "details": { - "description": "min=0.492, mean=0.492, max=0.492, sum=0.984 (2)", - "tab": "Accuracy", - "Formal Logic - Observed inference time (s)": "{\"description\": \"min=0.805, mean=0.805, max=0.805, sum=1.61 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.804882182015313\"}", - "Formal Logic - # eval": "{\"description\": \"min=126, mean=126, max=126, sum=252 (2)\", \"tab\": \"General information\", \"score\": \"126.0\"}", - "Formal Logic - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Formal Logic - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Formal Logic - # prompt tokens": "{\"description\": \"min=611.563, mean=611.563, max=611.563, sum=1223.127 (2)\", \"tab\": \"General information\", \"score\": \"611.563492063492\"}", - "Formal Logic - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"formal_logic\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_formal_logic\"" - } - } - }, - { - "evaluation_name": "High School World History", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on High School World History", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.865, - "details": { - "description": "min=0.865, mean=0.865, max=0.865, sum=1.73 (2)", - "tab": "Accuracy", - "High School Biology - Observed inference time (s)": "{\"description\": \"min=0.651, mean=0.651, max=0.651, sum=1.302 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6510615141161027\"}", - "High School Chemistry - Observed inference time (s)": "{\"description\": \"min=0.66, mean=0.66, max=0.66, sum=1.32 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6597568284114593\"}", - "High School Computer Science - Observed inference time (s)": "{\"description\": \"min=1.159, mean=1.159, max=1.159, sum=2.317 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.1585216951370239\"}", - "High School European History - Observed inference time (s)": "{\"description\": \"min=3.986, mean=3.986, max=3.986, sum=7.972 (2)\", \"tab\": \"Efficiency\", \"score\": \"3.9859177892858333\"}", - "High School Geography - Observed inference time (s)": "{\"description\": \"min=0.638, mean=0.638, max=0.638, sum=1.276 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6379079361154576\"}", - "High School Government And Politics - Observed inference time (s)": "{\"description\": \"min=0.719, mean=0.719, max=0.719, sum=1.438 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7190980182410521\"}", - "High School Macroeconomics - Observed inference time (s)": "{\"description\": \"min=0.646, mean=0.646, max=0.646, sum=1.292 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6461667580482288\"}", - "High School Mathematics - Observed inference time (s)": "{\"description\": \"min=0.889, mean=0.889, max=0.889, sum=1.778 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8891835009610212\"}", - "High School Microeconomics - Observed inference time (s)": "{\"description\": \"min=0.682, mean=0.682, max=0.682, sum=1.364 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6818269651477077\"}", - "High School Physics - Observed inference time (s)": "{\"description\": \"min=0.905, mean=0.905, max=0.905, sum=1.81 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9050559808086875\"}", - "High School Psychology - Observed inference time (s)": "{\"description\": \"min=0.746, mean=0.746, max=0.746, sum=1.491 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7455598682438561\"}", - "High School Statistics - Observed inference time (s)": "{\"description\": \"min=1.279, mean=1.279, max=1.279, sum=2.558 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.278907789124383\"}", - "High School US History - Observed inference time (s)": "{\"description\": \"min=3.106, mean=3.106, max=3.106, sum=6.212 (2)\", \"tab\": \"Efficiency\", \"score\": \"3.1062067454936457\"}", - "High School World History - Observed inference time (s)": "{\"description\": \"min=2.068, mean=2.068, max=2.068, sum=4.137 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.0682604393375574\"}", - "High School Biology - # eval": "{\"description\": \"min=310, mean=310, max=310, sum=620 (2)\", \"tab\": \"General information\", \"score\": \"310.0\"}", - "High School Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Biology - # prompt tokens": "{\"description\": \"min=492.958, mean=492.958, max=492.958, sum=985.916 (2)\", \"tab\": \"General information\", \"score\": \"492.958064516129\"}", - "High School Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Chemistry - # eval": "{\"description\": \"min=203, mean=203, max=203, sum=406 (2)\", \"tab\": \"General information\", \"score\": \"203.0\"}", - "High School Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # prompt tokens": "{\"description\": \"min=505.064, mean=505.064, max=505.064, sum=1010.128 (2)\", \"tab\": \"General information\", \"score\": \"505.064039408867\"}", - "High School Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "High School Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # prompt tokens": "{\"description\": \"min=927.13, mean=927.13, max=927.13, sum=1854.26 (2)\", \"tab\": \"General information\", \"score\": \"927.13\"}", - "High School Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School European History - # eval": "{\"description\": \"min=165, mean=165, max=165, sum=330 (2)\", \"tab\": \"General information\", \"score\": \"165.0\"}", - "High School European History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School European History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # prompt tokens": "{\"description\": \"min=2789.424, mean=2789.424, max=2789.424, sum=5578.848 (2)\", \"tab\": \"General information\", \"score\": \"2789.4242424242425\"}", - "High School European History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Geography - # eval": "{\"description\": \"min=198, mean=198, max=198, sum=396 (2)\", \"tab\": \"General information\", \"score\": \"198.0\"}", - "High School Geography - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Geography - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # prompt tokens": "{\"description\": \"min=386.773, mean=386.773, max=386.773, sum=773.545 (2)\", \"tab\": \"General information\", \"score\": \"386.77272727272725\"}", - "High School Geography - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Government And Politics - # eval": "{\"description\": \"min=193, mean=193, max=193, sum=386 (2)\", \"tab\": \"General information\", \"score\": \"193.0\"}", - "High School Government And Politics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Government And Politics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # prompt tokens": "{\"description\": \"min=471.301, mean=471.301, max=471.301, sum=942.601 (2)\", \"tab\": \"General information\", \"score\": \"471.30051813471505\"}", - "High School Government And Politics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Macroeconomics - # eval": "{\"description\": \"min=390, mean=390, max=390, sum=780 (2)\", \"tab\": \"General information\", \"score\": \"390.0\"}", - "High School Macroeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Macroeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # prompt tokens": "{\"description\": \"min=388.541, mean=388.541, max=388.541, sum=777.082 (2)\", \"tab\": \"General information\", \"score\": \"388.54102564102567\"}", - "High School Macroeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Mathematics - # eval": "{\"description\": \"min=270, mean=270, max=270, sum=540 (2)\", \"tab\": \"General information\", \"score\": \"270.0\"}", - "High School Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # prompt tokens": "{\"description\": \"min=558.822, mean=558.822, max=558.822, sum=1117.644 (2)\", \"tab\": \"General information\", \"score\": \"558.8222222222222\"}", - "High School Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Microeconomics - # eval": "{\"description\": \"min=238, mean=238, max=238, sum=476 (2)\", \"tab\": \"General information\", \"score\": \"238.0\"}", - "High School Microeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Microeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # prompt tokens": "{\"description\": \"min=407.954, mean=407.954, max=407.954, sum=815.908 (2)\", \"tab\": \"General information\", \"score\": \"407.953781512605\"}", - "High School Microeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Physics - # eval": "{\"description\": \"min=151, mean=151, max=151, sum=302 (2)\", \"tab\": \"General information\", \"score\": \"151.0\"}", - "High School Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # prompt tokens": "{\"description\": \"min=583.715, mean=583.715, max=583.715, sum=1167.43 (2)\", \"tab\": \"General information\", \"score\": \"583.7152317880794\"}", - "High School Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Psychology - # eval": "{\"description\": \"min=545, mean=545, max=545, sum=1090 (2)\", \"tab\": \"General information\", \"score\": \"545.0\"}", - "High School Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # prompt tokens": "{\"description\": \"min=494.604, mean=494.604, max=494.604, sum=989.207 (2)\", \"tab\": \"General information\", \"score\": \"494.60366972477067\"}", - "High School Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Statistics - # eval": "{\"description\": \"min=216, mean=216, max=216, sum=432 (2)\", \"tab\": \"General information\", \"score\": \"216.0\"}", - "High School Statistics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Statistics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # prompt tokens": "{\"description\": \"min=850.931, mean=850.931, max=850.931, sum=1701.861 (2)\", \"tab\": \"General information\", \"score\": \"850.9305555555555\"}", - "High School Statistics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School US History - # eval": "{\"description\": \"min=204, mean=204, max=204, sum=408 (2)\", \"tab\": \"General information\", \"score\": \"204.0\"}", - "High School US History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School US History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # prompt tokens": "{\"description\": \"min=2197.583, mean=2197.583, max=2197.583, sum=4395.167 (2)\", \"tab\": \"General information\", \"score\": \"2197.5833333333335\"}", - "High School US History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School World History - # eval": "{\"description\": \"min=237, mean=237, max=237, sum=474 (2)\", \"tab\": \"General information\", \"score\": \"237.0\"}", - "High School World History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School World History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # prompt tokens": "{\"description\": \"min=1418.544, mean=1418.544, max=1418.544, sum=2837.089 (2)\", \"tab\": \"General information\", \"score\": \"1418.5443037974683\"}", - "High School World History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"high_school_world_history\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_high_school_world_history\"" - } - } - }, - { - "evaluation_name": "Human Sexuality", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Human Sexuality", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.809, - "details": { - "description": "min=0.809, mean=0.809, max=0.809, sum=1.618 (2)", - "tab": "Accuracy", - "Human Aging - Observed inference time (s)": "{\"description\": \"min=0.548, mean=0.548, max=0.548, sum=1.095 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5475642894950148\"}", - "Human Sexuality - Observed inference time (s)": "{\"description\": \"min=0.564, mean=0.564, max=0.564, sum=1.129 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5644530576604013\"}", - "Human Aging - # eval": "{\"description\": \"min=223, mean=223, max=223, sum=446 (2)\", \"tab\": \"General information\", \"score\": \"223.0\"}", - "Human Aging - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Aging - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Aging - # prompt tokens": "{\"description\": \"min=313.587, mean=313.587, max=313.587, sum=627.175 (2)\", \"tab\": \"General information\", \"score\": \"313.58744394618833\"}", - "Human Aging - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Human Sexuality - # eval": "{\"description\": \"min=131, mean=131, max=131, sum=262 (2)\", \"tab\": \"General information\", \"score\": \"131.0\"}", - "Human Sexuality - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Sexuality - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # prompt tokens": "{\"description\": \"min=339.183, mean=339.183, max=339.183, sum=678.366 (2)\", \"tab\": \"General information\", \"score\": \"339.1832061068702\"}", - "Human Sexuality - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"human_sexuality\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_human_sexuality\"" - } - } - }, - { - "evaluation_name": "International Law", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on International Law", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.835, - "details": { - "description": "min=0.835, mean=0.835, max=0.835, sum=1.669 (2)", - "tab": "Accuracy", - "International Law - Observed inference time (s)": "{\"description\": \"min=0.956, mean=0.956, max=0.956, sum=1.911 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9556485384948983\"}", - "International Law - # eval": "{\"description\": \"min=121, mean=121, max=121, sum=242 (2)\", \"tab\": \"General information\", \"score\": \"121.0\"}", - "International Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "International Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "International Law - # prompt tokens": "{\"description\": \"min=636.165, mean=636.165, max=636.165, sum=1272.331 (2)\", \"tab\": \"General information\", \"score\": \"636.1652892561983\"}", - "International Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"international_law\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_international_law\"" - } - } - }, - { - "evaluation_name": "Logical Fallacies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Logical Fallacies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.816, - "details": { - "description": "min=0.816, mean=0.816, max=0.816, sum=1.632 (2)", - "tab": "Accuracy", - "Logical Fallacies - Observed inference time (s)": "{\"description\": \"min=0.699, mean=0.699, max=0.699, sum=1.398 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6992296397320332\"}", - "Logical Fallacies - # eval": "{\"description\": \"min=163, mean=163, max=163, sum=326 (2)\", \"tab\": \"General information\", \"score\": \"163.0\"}", - "Logical Fallacies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Logical Fallacies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Logical Fallacies - # prompt tokens": "{\"description\": \"min=442.049, mean=442.049, max=442.049, sum=884.098 (2)\", \"tab\": \"General information\", \"score\": \"442.0490797546012\"}", - "Logical Fallacies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"logical_fallacies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_logical_fallacies\"" - } - } - }, - { - "evaluation_name": "Machine Learning", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Machine Learning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.509, - "details": { - "description": "min=0.509, mean=0.509, max=0.509, sum=1.018 (2)", - "tab": "Accuracy", - "Machine Learning - Observed inference time (s)": "{\"description\": \"min=1.048, mean=1.048, max=1.048, sum=2.096 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.0480207417692458\"}", - "Machine Learning - # eval": "{\"description\": \"min=112, mean=112, max=112, sum=224 (2)\", \"tab\": \"General information\", \"score\": \"112.0\"}", - "Machine Learning - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Machine Learning - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Machine Learning - # prompt tokens": "{\"description\": \"min=694.402, mean=694.402, max=694.402, sum=1388.804 (2)\", \"tab\": \"General information\", \"score\": \"694.4017857142857\"}", - "Machine Learning - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"machine_learning\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_machine_learning\"" - } - } - }, - { - "evaluation_name": "Management", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Management", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.874, - "details": { - "description": "min=0.874, mean=0.874, max=0.874, sum=1.748 (2)", - "tab": "Accuracy", - "Management - Observed inference time (s)": "{\"description\": \"min=0.509, mean=0.509, max=0.509, sum=1.019 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5093999186765801\"}", - "Management - # eval": "{\"description\": \"min=103, mean=103, max=103, sum=206 (2)\", \"tab\": \"General information\", \"score\": \"103.0\"}", - "Management - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Management - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Management - # prompt tokens": "{\"description\": \"min=273.301, mean=273.301, max=273.301, sum=546.602 (2)\", \"tab\": \"General information\", \"score\": \"273.3009708737864\"}", - "Management - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"management\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_management\"" - } - } - }, - { - "evaluation_name": "Marketing", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Marketing", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.919, - "details": { - "description": "min=0.919, mean=0.919, max=0.919, sum=1.838 (2)", - "tab": "Accuracy", - "Marketing - Observed inference time (s)": "{\"description\": \"min=0.697, mean=0.697, max=0.697, sum=1.394 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6969545549816556\"}", - "Marketing - # eval": "{\"description\": \"min=234, mean=234, max=234, sum=468 (2)\", \"tab\": \"General information\", \"score\": \"234.0\"}", - "Marketing - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Marketing - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Marketing - # prompt tokens": "{\"description\": \"min=420.35, mean=420.35, max=420.35, sum=840.701 (2)\", \"tab\": \"General information\", \"score\": \"420.35042735042737\"}", - "Marketing - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"marketing\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_marketing\"" - } - } - }, - { - "evaluation_name": "Medical Genetics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Medical Genetics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.84, - "details": { - "description": "min=0.84, mean=0.84, max=0.84, sum=1.68 (2)", - "tab": "Accuracy", - "Medical Genetics - Observed inference time (s)": "{\"description\": \"min=0.534, mean=0.534, max=0.534, sum=1.067 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5335883450508118\"}", - "Medical Genetics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Medical Genetics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Medical Genetics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Medical Genetics - # prompt tokens": "{\"description\": \"min=330.89, mean=330.89, max=330.89, sum=661.78 (2)\", \"tab\": \"General information\", \"score\": \"330.89\"}", - "Medical Genetics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"medical_genetics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_medical_genetics\"" - } - } - }, - { - "evaluation_name": "Miscellaneous", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Miscellaneous", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.844, - "details": { - "description": "min=0.844, mean=0.844, max=0.844, sum=1.688 (2)", - "tab": "Accuracy", - "Miscellaneous - Observed inference time (s)": "{\"description\": \"min=0.536, mean=0.536, max=0.536, sum=1.073 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5363688258832442\"}", - "Miscellaneous - # eval": "{\"description\": \"min=783, mean=783, max=783, sum=1566 (2)\", \"tab\": \"General information\", \"score\": \"783.0\"}", - "Miscellaneous - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Miscellaneous - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Miscellaneous - # prompt tokens": "{\"description\": \"min=306.669, mean=306.669, max=306.669, sum=613.338 (2)\", \"tab\": \"General information\", \"score\": \"306.669220945083\"}", - "Miscellaneous - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"miscellaneous\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_miscellaneous\"" - } - } - }, - { - "evaluation_name": "Moral Scenarios", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Moral Scenarios", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.295, - "details": { - "description": "min=0.295, mean=0.295, max=0.295, sum=0.59 (2)", - "tab": "Accuracy", - "Moral Disputes - Observed inference time (s)": "{\"description\": \"min=0.734, mean=0.734, max=0.734, sum=1.468 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7340341696160377\"}", - "Moral Scenarios - Observed inference time (s)": "{\"description\": \"min=1.057, mean=1.057, max=1.057, sum=2.114 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.0570912433070176\"}", - "Moral Disputes - # eval": "{\"description\": \"min=346, mean=346, max=346, sum=692 (2)\", \"tab\": \"General information\", \"score\": \"346.0\"}", - "Moral Disputes - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Disputes - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Disputes - # prompt tokens": "{\"description\": \"min=487.003, mean=487.003, max=487.003, sum=974.006 (2)\", \"tab\": \"General information\", \"score\": \"487.0028901734104\"}", - "Moral Disputes - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Moral Scenarios - # eval": "{\"description\": \"min=895, mean=895, max=895, sum=1790 (2)\", \"tab\": \"General information\", \"score\": \"895.0\"}", - "Moral Scenarios - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Scenarios - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # prompt tokens": "{\"description\": \"min=682.542, mean=682.542, max=682.542, sum=1365.084 (2)\", \"tab\": \"General information\", \"score\": \"682.5418994413408\"}", - "Moral Scenarios - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"moral_scenarios\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_moral_scenarios\"" - } - } - }, - { - "evaluation_name": "Nutrition", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Nutrition", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.775, - "details": { - "description": "min=0.775, mean=0.775, max=0.775, sum=1.549 (2)", - "tab": "Accuracy", - "Nutrition - Observed inference time (s)": "{\"description\": \"min=0.889, mean=0.889, max=0.889, sum=1.779 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8894402412028094\"}", - "Nutrition - # eval": "{\"description\": \"min=306, mean=306, max=306, sum=612 (2)\", \"tab\": \"General information\", \"score\": \"306.0\"}", - "Nutrition - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Nutrition - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Nutrition - # prompt tokens": "{\"description\": \"min=577.48, mean=577.48, max=577.48, sum=1154.961 (2)\", \"tab\": \"General information\", \"score\": \"577.4803921568628\"}", - "Nutrition - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"nutrition\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_nutrition\"" - } - } - }, - { - "evaluation_name": "Prehistory", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Prehistory", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.812, - "details": { - "description": "min=0.812, mean=0.812, max=0.812, sum=1.623 (2)", - "tab": "Accuracy", - "Prehistory - Observed inference time (s)": "{\"description\": \"min=0.846, mean=0.846, max=0.846, sum=1.691 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8456013467576768\"}", - "Prehistory - # eval": "{\"description\": \"min=324, mean=324, max=324, sum=648 (2)\", \"tab\": \"General information\", \"score\": \"324.0\"}", - "Prehistory - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Prehistory - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Prehistory - # prompt tokens": "{\"description\": \"min=532.198, mean=532.198, max=532.198, sum=1064.395 (2)\", \"tab\": \"General information\", \"score\": \"532.1975308641976\"}", - "Prehistory - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"prehistory\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_prehistory\"" - } - } - }, - { - "evaluation_name": "Public Relations", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Public Relations", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.736, - "details": { - "description": "min=0.736, mean=0.736, max=0.736, sum=1.473 (2)", - "tab": "Accuracy", - "Public Relations - Observed inference time (s)": "{\"description\": \"min=0.698, mean=0.698, max=0.698, sum=1.395 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6977464697577737\"}", - "Public Relations - # eval": "{\"description\": \"min=110, mean=110, max=110, sum=220 (2)\", \"tab\": \"General information\", \"score\": \"110.0\"}", - "Public Relations - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Public Relations - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Public Relations - # prompt tokens": "{\"description\": \"min=418.655, mean=418.655, max=418.655, sum=837.309 (2)\", \"tab\": \"General information\", \"score\": \"418.6545454545454\"}", - "Public Relations - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"public_relations\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_public_relations\"" - } - } - }, - { - "evaluation_name": "Security Studies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Security Studies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.78, - "details": { - "description": "min=0.78, mean=0.78, max=0.78, sum=1.559 (2)", - "tab": "Accuracy", - "Security Studies - Observed inference time (s)": "{\"description\": \"min=1.737, mean=1.737, max=1.737, sum=3.473 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.7365190982818604\"}", - "Security Studies - # eval": "{\"description\": \"min=245, mean=245, max=245, sum=490 (2)\", \"tab\": \"General information\", \"score\": \"245.0\"}", - "Security Studies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Security Studies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Security Studies - # prompt tokens": "{\"description\": \"min=1185.869, mean=1185.869, max=1185.869, sum=2371.739 (2)\", \"tab\": \"General information\", \"score\": \"1185.869387755102\"}", - "Security Studies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"security_studies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_security_studies\"" - } - } - }, - { - "evaluation_name": "Sociology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Sociology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9, - "details": { - "description": "min=0.9, mean=0.9, max=0.9, sum=1.801 (2)", - "tab": "Accuracy", - "Sociology - Observed inference time (s)": "{\"description\": \"min=0.712, mean=0.712, max=0.712, sum=1.423 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7115461138350454\"}", - "Sociology - # eval": "{\"description\": \"min=201, mean=201, max=201, sum=402 (2)\", \"tab\": \"General information\", \"score\": \"201.0\"}", - "Sociology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Sociology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Sociology - # prompt tokens": "{\"description\": \"min=448.274, mean=448.274, max=448.274, sum=896.547 (2)\", \"tab\": \"General information\", \"score\": \"448.27363184079604\"}", - "Sociology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"sociology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_sociology\"" - } - } - }, - { - "evaluation_name": "Virology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Virology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.53, - "details": { - "description": "min=0.53, mean=0.53, max=0.53, sum=1.06 (2)", - "tab": "Accuracy", - "Virology - Observed inference time (s)": "{\"description\": \"min=0.571, mean=0.571, max=0.571, sum=1.142 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.571121395352375\"}", - "Virology - # eval": "{\"description\": \"min=166, mean=166, max=166, sum=332 (2)\", \"tab\": \"General information\", \"score\": \"166.0\"}", - "Virology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Virology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Virology - # prompt tokens": "{\"description\": \"min=328.753, mean=328.753, max=328.753, sum=657.506 (2)\", \"tab\": \"General information\", \"score\": \"328.7530120481928\"}", - "Virology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"virology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_virology\"" - } - } - }, - { - "evaluation_name": "World Religions", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on World Religions", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.86, - "details": { - "description": "min=0.86, mean=0.86, max=0.86, sum=1.719 (2)", - "tab": "Accuracy", - "World Religions - Observed inference time (s)": "{\"description\": \"min=0.448, mean=0.448, max=0.448, sum=0.895 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.44760305142542073\"}", - "World Religions - # eval": "{\"description\": \"min=171, mean=171, max=171, sum=342 (2)\", \"tab\": \"General information\", \"score\": \"171.0\"}", - "World Religions - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "World Religions - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "World Religions - # prompt tokens": "{\"description\": \"min=260.164, mean=260.164, max=260.164, sum=520.327 (2)\", \"tab\": \"General information\", \"score\": \"260.1637426900585\"}", - "World Religions - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"world_religions\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_world_religions\"" - } - } - }, - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.265, - "details": { - "description": "", - "tab": "Efficiency" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_id": "hfopenllm_v2/google_gemma-2-9b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.204 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5377 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1344 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3289 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4461 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4103 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/google_gemma-2b-it.json b/data/models/google_gemma-2b-it.json deleted file mode 100644 index 9145f5c483befecc04780bdb7565faa70951f313..0000000000000000000000000000000000000000 --- a/data/models/google_gemma-2b-it.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "gemma-2b-it", - "id": "google/gemma-2b-it", - "developer": "Google", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "GemmaForCausalLM", - "params_billions": "2.506" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/google_gemma-2b-it/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.269 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3151 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0204 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2785 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3341 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1353 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/google_gemma-2b.json b/data/models/google_gemma-2b.json deleted file mode 100644 index 831bb9fed559d06f6fc95ee9745b74e75fe6c755..0000000000000000000000000000000000000000 --- a/data/models/google_gemma-2b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "gemma-2b", - "id": "google/gemma-2b", - "developer": "Google", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "GemmaForCausalLM", - "params_billions": "2.506" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/google_gemma-2b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2038 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3366 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0302 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.255 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3978 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1366 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/google_gemma-3-12b-it-prompt.json b/data/models/google_gemma-3-12b-it-prompt.json deleted file mode 100644 index 74140d32447769f5c9d5b8c3723d2d33070b4d76..0000000000000000000000000000000000000000 --- a/data/models/google_gemma-3-12b-it-prompt.json +++ /dev/null @@ -1,905 +0,0 @@ -{ - "model_info": { - "name": "Gemma-3-12b-it (Prompt)", - "id": "google/gemma-3-12b-it-prompt", - "developer": "Google", - "additional_details": { - "raw_model_name": "Gemma-3-12b-it (Prompt)", - "organization": "Google", - "license": "gemma-terms-of-use", - "mode": "Prompt", - "model_link": "https://blog.google/technology/developers/gemma-3/" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/google/gemma-3-12b-it-prompt/1775236112.4004931", - "retrieved_timestamp": "1775236112.4004931", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 66.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 30.43 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 10.77 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 11.1 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 17.17 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 34.66 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 79.44 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 76.25 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 94.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 91.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 56.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 74.24 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 85.66 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 71.89 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 87.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 45.83 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 5.75 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 6.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 7.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 5.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 4.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 4.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 4.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 4.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 27.53 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 8.39 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 25.16 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 49.03 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 93.75 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 70.29 - } - }, - { - "evaluation_result_id": "bfcl.format_sensitivity.max_delta::max_delta", - "evaluation_name": "bfcl.format_sensitivity.max_delta", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.format_sensitivity.max_delta", - "metric_name": "Format sensitivity max delta", - "metric_kind": "difference", - "metric_unit": "percentage_points", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Format Sensitivity Max Delta" - } - }, - "score_details": { - "score": 67.5 - } - }, - { - "evaluation_result_id": "bfcl.format_sensitivity.stddev::stddev", - "evaluation_name": "bfcl.format_sensitivity.stddev", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.format_sensitivity.stddev", - "metric_name": "Format sensitivity standard deviation", - "metric_kind": "difference", - "metric_unit": "percentage_points", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Format Sensitivity Standard Deviation" - } - }, - "score_details": { - "score": 22.41 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/google_gemma-3-1b-it-prompt.json b/data/models/google_gemma-3-1b-it-prompt.json deleted file mode 100644 index 428eda6ce7664e3f0e40a9e8e457af07c38c0d9d..0000000000000000000000000000000000000000 --- a/data/models/google_gemma-3-1b-it-prompt.json +++ /dev/null @@ -1,905 +0,0 @@ -{ - "model_info": { - "name": "Gemma-3-1b-it (Prompt)", - "id": "google/gemma-3-1b-it-prompt", - "developer": "Google", - "additional_details": { - "raw_model_name": "Gemma-3-1b-it (Prompt)", - "organization": "Google", - "license": "gemma-terms-of-use", - "mode": "Prompt", - "model_link": "https://blog.google/technology/developers/gemma-3/" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/google/gemma-3-1b-it-prompt/1775236112.42324", - "retrieved_timestamp": "1775236112.42324", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 109.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 7.17 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 3.4 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 3.98 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 9.8 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 12.06 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 20.21 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 43.33 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 36.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 1.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 11.84 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 36.43 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 6.27 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 3.23 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 3.87 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 3.87 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 1.94 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 37.5 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 33.18 - } - }, - { - "evaluation_result_id": "bfcl.format_sensitivity.max_delta::max_delta", - "evaluation_name": "bfcl.format_sensitivity.max_delta", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.format_sensitivity.max_delta", - "metric_name": "Format sensitivity max delta", - "metric_kind": "difference", - "metric_unit": "percentage_points", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Format Sensitivity Max Delta" - } - }, - "score_details": { - "score": 25.5 - } - }, - { - "evaluation_result_id": "bfcl.format_sensitivity.stddev::stddev", - "evaluation_name": "bfcl.format_sensitivity.stddev", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.format_sensitivity.stddev", - "metric_name": "Format sensitivity standard deviation", - "metric_kind": "difference", - "metric_unit": "percentage_points", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Format Sensitivity Standard Deviation" - } - }, - "score_details": { - "score": 9.76 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/google_gemma-3-27b-it-prompt.json b/data/models/google_gemma-3-27b-it-prompt.json deleted file mode 100644 index 967f2cbf2978baa4b103a9f1f66b2feb2ea2a19a..0000000000000000000000000000000000000000 --- a/data/models/google_gemma-3-27b-it-prompt.json +++ /dev/null @@ -1,905 +0,0 @@ -{ - "model_info": { - "name": "Gemma-3-27b-it (Prompt)", - "id": "google/gemma-3-27b-it-prompt", - "developer": "Google", - "additional_details": { - "raw_model_name": "Gemma-3-27b-it (Prompt)", - "organization": "Google", - "license": "gemma-terms-of-use", - "mode": "Prompt", - "model_link": "https://blog.google/technology/developers/gemma-3/" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/google/gemma-3-27b-it-prompt/1775236112.402029", - "retrieved_timestamp": "1775236112.402029", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 69.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 29.47 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 11.82 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 10.88 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 19.67 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 55.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 87.17 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 77.67 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 92.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 89.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 89.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 74.54 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 84.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 72.46 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 93.75 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 45.83 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 10.75 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 16.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 4.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 8.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 14.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 13.55 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 1.94 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 3.23 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 35.48 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 81.25 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 73.67 - } - }, - { - "evaluation_result_id": "bfcl.format_sensitivity.max_delta::max_delta", - "evaluation_name": "bfcl.format_sensitivity.max_delta", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.format_sensitivity.max_delta", - "metric_name": "Format sensitivity max delta", - "metric_kind": "difference", - "metric_unit": "percentage_points", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Format Sensitivity Max Delta" - } - }, - "score_details": { - "score": 34.0 - } - }, - { - "evaluation_result_id": "bfcl.format_sensitivity.stddev::stddev", - "evaluation_name": "bfcl.format_sensitivity.stddev", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.format_sensitivity.stddev", - "metric_name": "Format sensitivity standard deviation", - "metric_kind": "difference", - "metric_unit": "percentage_points", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Format Sensitivity Standard Deviation" - } - }, - "score_details": { - "score": 8.06 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/google_gemma-3-27b-it.json b/data/models/google_gemma-3-27b-it.json deleted file mode 100644 index 31e90c4548397bec1dec70a558a4830dc0c4f7c9..0000000000000000000000000000000000000000 --- a/data/models/google_gemma-3-27b-it.json +++ /dev/null @@ -1,1043 +0,0 @@ -{ - "model_info": { - "name": "gemma-3-27b-it", - "id": "google/gemma-3-27b-it", - "developer": "Google", - "inference_platform": "unknown", - "additional_details": { - "display_name": "Gemma 3 27B" - } - }, - "evaluations": [ - { - "evaluation_id": "global-mmlu-lite/google_gemma-3-27b-it/1773936496.366405", - "retrieved_timestamp": "1773936496.366405", - "source_metadata": { - "source_name": "Global MMLU Lite Leaderboard", - "source_type": "documentation", - "source_organization_name": "kaggle", - "source_organization_url": "www.kaggle.com", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "kaggle kernel", - "version": "4", - "additional_details": { - "url": "https://www.kaggle.com/code/shivalikasingh95/global-mmlu-lite-sample-notebook" - } - }, - "benchmark": "global-mmlu-lite", - "evaluation_results": [ - { - "evaluation_name": "Global MMLU Lite", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Global MMLU Lite", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.763 - } - }, - { - "evaluation_name": "Culturally Sensitive", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Sensitive", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7528 - } - }, - { - "evaluation_name": "Culturally Agnostic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Agnostic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7733 - } - }, - { - "evaluation_name": "Arabic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Arabic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.78, - "uncertainty": { - "confidence_interval": { - "lower": -0.0406, - "upper": 0.0406, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "English", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - English", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7337, - "uncertainty": { - "confidence_interval": { - "lower": -0.0434, - "upper": 0.0434, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Bengali", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Bengali", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.75, - "uncertainty": { - "confidence_interval": { - "lower": -0.0426, - "upper": 0.0426, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "German", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - German", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.775, - "uncertainty": { - "confidence_interval": { - "lower": -0.0409, - "upper": 0.0409, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "French", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - French", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7481, - "uncertainty": { - "confidence_interval": { - "lower": -0.0429, - "upper": 0.0429, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Hindi", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Hindi", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7335, - "uncertainty": { - "confidence_interval": { - "lower": -0.0437, - "upper": 0.0437, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Indonesian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Indonesian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7563, - "uncertainty": { - "confidence_interval": { - "lower": -0.0422, - "upper": 0.0422, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Italian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Italian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.75, - "uncertainty": { - "confidence_interval": { - "lower": -0.0424, - "upper": 0.0424, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Japanese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Japanese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7925, - "uncertainty": { - "confidence_interval": { - "lower": -0.0397, - "upper": 0.0397, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Korean", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Korean", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.798, - "uncertainty": { - "confidence_interval": { - "lower": -0.0395, - "upper": 0.0395, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Portuguese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Portuguese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7481, - "uncertainty": { - "confidence_interval": { - "lower": -0.0427, - "upper": 0.0427, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Spanish", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Spanish", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7494, - "uncertainty": { - "confidence_interval": { - "lower": -0.0425, - "upper": 0.0425, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Swahili", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Swahili", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.785, - "uncertainty": { - "confidence_interval": { - "lower": -0.0403, - "upper": 0.0403, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Yoruba", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Yoruba", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7444, - "uncertainty": { - "confidence_interval": { - "lower": -0.0428, - "upper": 0.0428, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Chinese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Chinese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7925, - "uncertainty": { - "confidence_interval": { - "lower": -0.0397, - "upper": 0.0397, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Burmese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Burmese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7719, - "uncertainty": { - "confidence_interval": { - "lower": -0.0412, - "upper": 0.0412, - "method": "unknown" - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "global-mmlu-lite/google_gemma-3-27b-it/1773936583.743359", - "retrieved_timestamp": "1773936583.743359", - "source_metadata": { - "source_name": "Global MMLU Lite Leaderboard", - "source_type": "documentation", - "source_organization_name": "kaggle", - "source_organization_url": "www.kaggle.com", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "kaggle kernel", - "version": "4", - "additional_details": { - "url": "https://www.kaggle.com/code/shivalikasingh95/global-mmlu-lite-sample-notebook" - } - }, - "benchmark": "global-mmlu-lite", - "evaluation_results": [ - { - "evaluation_name": "Global MMLU Lite", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Global MMLU Lite", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.763 - } - }, - { - "evaluation_name": "Culturally Sensitive", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Sensitive", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7528 - } - }, - { - "evaluation_name": "Culturally Agnostic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Agnostic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7733 - } - }, - { - "evaluation_name": "Arabic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Arabic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.78, - "uncertainty": { - "confidence_interval": { - "lower": -0.0406, - "upper": 0.0406, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "English", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - English", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7337, - "uncertainty": { - "confidence_interval": { - "lower": -0.0434, - "upper": 0.0434, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Bengali", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Bengali", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.75, - "uncertainty": { - "confidence_interval": { - "lower": -0.0426, - "upper": 0.0426, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "German", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - German", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.775, - "uncertainty": { - "confidence_interval": { - "lower": -0.0409, - "upper": 0.0409, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "French", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - French", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7481, - "uncertainty": { - "confidence_interval": { - "lower": -0.0429, - "upper": 0.0429, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Hindi", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Hindi", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7335, - "uncertainty": { - "confidence_interval": { - "lower": -0.0437, - "upper": 0.0437, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Indonesian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Indonesian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7563, - "uncertainty": { - "confidence_interval": { - "lower": -0.0422, - "upper": 0.0422, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Italian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Italian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.75, - "uncertainty": { - "confidence_interval": { - "lower": -0.0424, - "upper": 0.0424, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Japanese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Japanese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7925, - "uncertainty": { - "confidence_interval": { - "lower": -0.0397, - "upper": 0.0397, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Korean", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Korean", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.798, - "uncertainty": { - "confidence_interval": { - "lower": -0.0395, - "upper": 0.0395, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Portuguese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Portuguese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7481, - "uncertainty": { - "confidence_interval": { - "lower": -0.0427, - "upper": 0.0427, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Spanish", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Spanish", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7494, - "uncertainty": { - "confidence_interval": { - "lower": -0.0425, - "upper": 0.0425, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Swahili", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Swahili", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.785, - "uncertainty": { - "confidence_interval": { - "lower": -0.0403, - "upper": 0.0403, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Yoruba", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Yoruba", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7444, - "uncertainty": { - "confidence_interval": { - "lower": -0.0428, - "upper": 0.0428, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Chinese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Chinese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7925, - "uncertainty": { - "confidence_interval": { - "lower": -0.0397, - "upper": 0.0397, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Burmese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Burmese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7719, - "uncertainty": { - "confidence_interval": { - "lower": -0.0412, - "upper": 0.0412, - "method": "unknown" - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/google_gemma-3-4b-it-prompt.json b/data/models/google_gemma-3-4b-it-prompt.json deleted file mode 100644 index 3e19a5769034f364cb63b10295fe2c0da98ff3e7..0000000000000000000000000000000000000000 --- a/data/models/google_gemma-3-4b-it-prompt.json +++ /dev/null @@ -1,905 +0,0 @@ -{ - "model_info": { - "name": "Gemma-3-4b-it (Prompt)", - "id": "google/gemma-3-4b-it-prompt", - "developer": "Google", - "additional_details": { - "raw_model_name": "Gemma-3-4b-it (Prompt)", - "organization": "Google", - "license": "gemma-terms-of-use", - "mode": "Prompt", - "model_link": "https://blog.google/technology/developers/gemma-3/" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/google/gemma-3-4b-it-prompt/1775236112.419135", - "retrieved_timestamp": "1775236112.419135", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 101.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 19.62 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 4.14 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 4.69 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 9.53 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 11.42 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 61.12 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 64.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 88.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 56.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 36.0 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 60.84 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 70.93 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 59.35 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 25.0 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 41.67 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 0.38 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 0.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 0.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 0.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 1.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 1.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 1.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 8.6 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 9.68 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 9.68 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 6.45 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 100.0 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 53.94 - } - }, - { - "evaluation_result_id": "bfcl.format_sensitivity.max_delta::max_delta", - "evaluation_name": "bfcl.format_sensitivity.max_delta", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.format_sensitivity.max_delta", - "metric_name": "Format sensitivity max delta", - "metric_kind": "difference", - "metric_unit": "percentage_points", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Format Sensitivity Max Delta" - } - }, - "score_details": { - "score": 69.5 - } - }, - { - "evaluation_result_id": "bfcl.format_sensitivity.stddev::stddev", - "evaluation_name": "bfcl.format_sensitivity.stddev", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.format_sensitivity.stddev", - "metric_name": "Format sensitivity standard deviation", - "metric_kind": "difference", - "metric_unit": "percentage_points", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Format Sensitivity Standard Deviation" - } - }, - "score_details": { - "score": 23.67 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/google_gemma-3-4b-it.json b/data/models/google_gemma-3-4b-it.json deleted file mode 100644 index a4a8d0cbbb50ae74df70e1b97a9540a4077ec070..0000000000000000000000000000000000000000 --- a/data/models/google_gemma-3-4b-it.json +++ /dev/null @@ -1,1043 +0,0 @@ -{ - "model_info": { - "name": "gemma-3-4b-it", - "id": "google/gemma-3-4b-it", - "developer": "Google", - "inference_platform": "unknown", - "additional_details": { - "display_name": "Gemma 3 4B" - } - }, - "evaluations": [ - { - "evaluation_id": "global-mmlu-lite/google_gemma-3-4b-it/1773936496.366405", - "retrieved_timestamp": "1773936496.366405", - "source_metadata": { - "source_name": "Global MMLU Lite Leaderboard", - "source_type": "documentation", - "source_organization_name": "kaggle", - "source_organization_url": "www.kaggle.com", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "kaggle kernel", - "version": "4", - "additional_details": { - "url": "https://www.kaggle.com/code/shivalikasingh95/global-mmlu-lite-sample-notebook" - } - }, - "benchmark": "global-mmlu-lite", - "evaluation_results": [ - { - "evaluation_name": "Global MMLU Lite", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Global MMLU Lite", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6511 - } - }, - { - "evaluation_name": "Culturally Sensitive", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Sensitive", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6116 - } - }, - { - "evaluation_name": "Culturally Agnostic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Agnostic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6906 - } - }, - { - "evaluation_name": "Arabic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Arabic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6525, - "uncertainty": { - "confidence_interval": { - "lower": -0.0467, - "upper": 0.0467, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "English", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - English", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.67, - "uncertainty": { - "confidence_interval": { - "lower": -0.0461, - "upper": 0.0461, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Bengali", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Bengali", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.68, - "uncertainty": { - "confidence_interval": { - "lower": -0.0457, - "upper": 0.0457, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "German", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - German", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6525, - "uncertainty": { - "confidence_interval": { - "lower": -0.0467, - "upper": 0.0467, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "French", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - French", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6575, - "uncertainty": { - "confidence_interval": { - "lower": -0.0465, - "upper": 0.0465, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Hindi", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Hindi", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6475, - "uncertainty": { - "confidence_interval": { - "lower": -0.0468, - "upper": 0.0468, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Indonesian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Indonesian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6775, - "uncertainty": { - "confidence_interval": { - "lower": -0.0458, - "upper": 0.0458, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Italian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Italian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6675, - "uncertainty": { - "confidence_interval": { - "lower": -0.0462, - "upper": 0.0462, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Japanese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Japanese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6325, - "uncertainty": { - "confidence_interval": { - "lower": -0.0472, - "upper": 0.0472, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Korean", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Korean", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.66, - "uncertainty": { - "confidence_interval": { - "lower": -0.0464, - "upper": 0.0464, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Portuguese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Portuguese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.68, - "uncertainty": { - "confidence_interval": { - "lower": -0.0457, - "upper": 0.0457, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Spanish", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Spanish", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6725, - "uncertainty": { - "confidence_interval": { - "lower": -0.046, - "upper": 0.046, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Swahili", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Swahili", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6075, - "uncertainty": { - "confidence_interval": { - "lower": -0.0479, - "upper": 0.0479, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Yoruba", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Yoruba", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5825, - "uncertainty": { - "confidence_interval": { - "lower": -0.0483, - "upper": 0.0483, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Chinese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Chinese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6475, - "uncertainty": { - "confidence_interval": { - "lower": -0.0468, - "upper": 0.0468, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Burmese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Burmese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.63, - "uncertainty": { - "confidence_interval": { - "lower": -0.0473, - "upper": 0.0473, - "method": "unknown" - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "global-mmlu-lite/google_gemma-3-4b-it/1773936583.743359", - "retrieved_timestamp": "1773936583.743359", - "source_metadata": { - "source_name": "Global MMLU Lite Leaderboard", - "source_type": "documentation", - "source_organization_name": "kaggle", - "source_organization_url": "www.kaggle.com", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "kaggle kernel", - "version": "4", - "additional_details": { - "url": "https://www.kaggle.com/code/shivalikasingh95/global-mmlu-lite-sample-notebook" - } - }, - "benchmark": "global-mmlu-lite", - "evaluation_results": [ - { - "evaluation_name": "Global MMLU Lite", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Global MMLU Lite", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6511 - } - }, - { - "evaluation_name": "Culturally Sensitive", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Sensitive", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6116 - } - }, - { - "evaluation_name": "Culturally Agnostic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Agnostic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6906 - } - }, - { - "evaluation_name": "Arabic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Arabic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6525, - "uncertainty": { - "confidence_interval": { - "lower": -0.0467, - "upper": 0.0467, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "English", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - English", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.67, - "uncertainty": { - "confidence_interval": { - "lower": -0.0461, - "upper": 0.0461, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Bengali", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Bengali", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.68, - "uncertainty": { - "confidence_interval": { - "lower": -0.0457, - "upper": 0.0457, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "German", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - German", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6525, - "uncertainty": { - "confidence_interval": { - "lower": -0.0467, - "upper": 0.0467, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "French", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - French", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6575, - "uncertainty": { - "confidence_interval": { - "lower": -0.0465, - "upper": 0.0465, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Hindi", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Hindi", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6475, - "uncertainty": { - "confidence_interval": { - "lower": -0.0468, - "upper": 0.0468, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Indonesian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Indonesian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6775, - "uncertainty": { - "confidence_interval": { - "lower": -0.0458, - "upper": 0.0458, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Italian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Italian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6675, - "uncertainty": { - "confidence_interval": { - "lower": -0.0462, - "upper": 0.0462, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Japanese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Japanese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6325, - "uncertainty": { - "confidence_interval": { - "lower": -0.0472, - "upper": 0.0472, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Korean", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Korean", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.66, - "uncertainty": { - "confidence_interval": { - "lower": -0.0464, - "upper": 0.0464, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Portuguese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Portuguese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.68, - "uncertainty": { - "confidence_interval": { - "lower": -0.0457, - "upper": 0.0457, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Spanish", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Spanish", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6725, - "uncertainty": { - "confidence_interval": { - "lower": -0.046, - "upper": 0.046, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Swahili", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Swahili", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6075, - "uncertainty": { - "confidence_interval": { - "lower": -0.0479, - "upper": 0.0479, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Yoruba", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Yoruba", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5825, - "uncertainty": { - "confidence_interval": { - "lower": -0.0483, - "upper": 0.0483, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Chinese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Chinese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6475, - "uncertainty": { - "confidence_interval": { - "lower": -0.0468, - "upper": 0.0468, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Burmese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Burmese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.63, - "uncertainty": { - "confidence_interval": { - "lower": -0.0473, - "upper": 0.0473, - "method": "unknown" - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/google_gemma-7b-it.json b/data/models/google_gemma-7b-it.json deleted file mode 100644 index f7a041de27bb5f50c357d855c382375a8a75e4c1..0000000000000000000000000000000000000000 --- a/data/models/google_gemma-7b-it.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "gemma-7b-it", - "id": "google/gemma-7b-it", - "developer": "Google", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "GemmaForCausalLM", - "params_billions": "8.538" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/google_gemma-7b-it/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3868 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3646 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0295 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2844 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4274 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1695 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/google_gemma-7b.json b/data/models/google_gemma-7b.json deleted file mode 100644 index 7ba4fc673f24af752e4bcc4d8183c44f0149a650..0000000000000000000000000000000000000000 --- a/data/models/google_gemma-7b.json +++ /dev/null @@ -1,2031 +0,0 @@ -{ - "model_info": { - "name": "Gemma 7B", - "id": "google/gemma-7b", - "developer": "Google", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_lite/google_gemma-7b/1774096306.427425", - "retrieved_timestamp": "1774096306.427425", - "source_metadata": { - "source_name": "helm_lite", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_lite", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_lite", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.336, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.7896629213483146\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.752, - "details": { - "description": "min=0.752, mean=0.752, max=0.752, sum=0.752 (1)", - "tab": "Accuracy", - "NarrativeQA - Observed inference time (s)": "{\"description\": \"min=0.909, mean=0.909, max=0.909, sum=0.909 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.9086058952438999\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=3411.994, mean=3411.994, max=3411.994, sum=3411.994 (1)\", \"tab\": \"General information\", \"score\": \"3411.994366197183\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (closed-book)", - "source_data": { - "dataset_name": "NaturalQuestions (closed-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (closed-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.336, - "details": { - "description": "min=0.336, mean=0.336, max=0.336, sum=0.336 (1)", - "tab": "Accuracy", - "NaturalQuestions (open-book) - Observed inference time (s)": "{\"description\": \"min=0.591, mean=0.591, max=0.591, sum=0.591 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.5911745510101318\"}", - "NaturalQuestions (closed-book) - Observed inference time (s)": "{\"description\": \"min=0.343, mean=0.343, max=0.343, sum=0.343 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.3430815353393555\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.94, mean=4.94, max=4.94, sum=4.94 (1)\", \"tab\": \"General information\", \"score\": \"4.94\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.01, mean=0.01, max=0.01, sum=0.01 (1)\", \"tab\": \"General information\", \"score\": \"0.01\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1879.978, mean=1879.978, max=1879.978, sum=1879.978 (1)\", \"tab\": \"General information\", \"score\": \"1879.978\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=125.995, mean=125.995, max=125.995, sum=125.995 (1)\", \"tab\": \"General information\", \"score\": \"125.995\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "mode": "\"closedbook\"" - } - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.808, - "details": { - "description": "min=0.808, mean=0.808, max=0.808, sum=0.808 (1)", - "tab": "Accuracy", - "OpenbookQA - Observed inference time (s)": "{\"description\": \"min=0.282, mean=0.282, max=0.282, sum=0.282 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.28152281618118286\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=240.508, mean=240.508, max=240.508, sum=240.508 (1)\", \"tab\": \"General information\", \"score\": \"240.508\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "dataset": "\"openbookqa\"", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.571, - "details": { - "description": "min=0.28, mean=0.571, max=0.87, sum=2.854 (5)", - "tab": "Accuracy", - "MMLU - Observed inference time (s)": "{\"description\": \"min=0.251, mean=0.273, max=0.293, sum=1.367 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.27346607242550763\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=372.91, mean=473.531, max=626.553, sum=2367.653 (5)\", \"tab\": \"General information\", \"score\": \"473.5305263157895\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"college_chemistry\", \"computer_security\", \"econometrics\", \"us_foreign_policy\"]", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MATH", - "source_data": { - "dataset_name": "MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Equivalent (CoT) on MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5, - "details": { - "description": "min=0.3, mean=0.5, max=0.711, sum=3.499 (7)", - "tab": "Accuracy", - "MATH - Observed inference time (s)": "{\"description\": \"min=0.995, mean=1.161, max=1.453, sum=8.127 (7)\", \"tab\": \"Efficiency\", \"score\": \"1.1609408722047545\"}", - "MATH - # eval": "{\"description\": \"min=30, mean=62.429, max=135, sum=437 (7)\", \"tab\": \"General information\", \"score\": \"62.42857142857143\"}", - "MATH - # train": "{\"description\": \"min=8, mean=8, max=8, sum=56 (7)\", \"tab\": \"General information\", \"score\": \"8.0\"}", - "MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MATH - # prompt tokens": "{\"description\": \"min=938.215, mean=1355.506, max=2348.712, sum=9488.545 (7)\", \"tab\": \"General information\", \"score\": \"1355.5064552904823\"}", - "MATH - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=7 (7)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"algebra\", \"counting_and_probability\", \"geometry\", \"intermediate_algebra\", \"number_theory\", \"prealgebra\", \"precalculus\"]", - "level": "\"1\"", - "use_official_examples": "\"False\"", - "use_chain_of_thought": "\"True\"" - } - } - }, - { - "evaluation_name": "GSM8K", - "source_data": { - "dataset_name": "GSM8K", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on GSM8K", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.559, - "details": { - "description": "min=0.559, mean=0.559, max=0.559, sum=0.559 (1)", - "tab": "Accuracy", - "GSM8K - Observed inference time (s)": "{\"description\": \"min=2.025, mean=2.025, max=2.025, sum=2.025 (1)\", \"tab\": \"Efficiency\", \"score\": \"2.024561887741089\"}", - "GSM8K - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "GSM8K - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "GSM8K - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GSM8K - # prompt tokens": "{\"description\": \"min=1151.885, mean=1151.885, max=1151.885, sum=1151.885 (1)\", \"tab\": \"General information\", \"score\": \"1151.885\"}", - "GSM8K - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "LegalBench", - "source_data": { - "dataset_name": "LegalBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on LegalBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.581, - "details": { - "description": "min=0.379, mean=0.581, max=0.811, sum=2.904 (5)", - "tab": "Accuracy", - "LegalBench - Observed inference time (s)": "{\"description\": \"min=0.295, mean=0.53, max=1.42, sum=2.652 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.5303036133605687\"}", - "LegalBench - # eval": "{\"description\": \"min=95, mean=409.4, max=1000, sum=2047 (5)\", \"tab\": \"General information\", \"score\": \"409.4\"}", - "LegalBench - # train": "{\"description\": \"min=4, mean=4.795, max=5, sum=23.973 (5)\", \"tab\": \"General information\", \"score\": \"4.794693877551021\"}", - "LegalBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "LegalBench - # prompt tokens": "{\"description\": \"min=193.916, mean=1536.557, max=6379.163, sum=7682.787 (5)\", \"tab\": \"General information\", \"score\": \"1536.5573806103425\"}", - "LegalBench - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"abercrombie\", \"corporate_lobbying\", \"function_of_decision_section\", \"international_citizenship_questions\", \"proa\"]" - } - } - }, - { - "evaluation_name": "MedQA", - "source_data": { - "dataset_name": "MedQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MedQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.513, - "details": { - "description": "min=0.513, mean=0.513, max=0.513, sum=0.513 (1)", - "tab": "Accuracy", - "MedQA - Observed inference time (s)": "{\"description\": \"min=0.314, mean=0.314, max=0.314, sum=0.314 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.3144090270427302\"}", - "MedQA - # eval": "{\"description\": \"min=503, mean=503, max=503, sum=503 (1)\", \"tab\": \"General information\", \"score\": \"503.0\"}", - "MedQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MedQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MedQA - # prompt tokens": "{\"description\": \"min=1021.481, mean=1021.481, max=1021.481, sum=1021.481 (1)\", \"tab\": \"General information\", \"score\": \"1021.4811133200795\"}", - "MedQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WMT 2014", - "source_data": { - "dataset_name": "WMT 2014", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "BLEU-4 on WMT 2014", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.187, - "details": { - "description": "min=0.137, mean=0.187, max=0.211, sum=0.937 (5)", - "tab": "Accuracy", - "WMT 2014 - Observed inference time (s)": "{\"description\": \"min=0.503, mean=0.524, max=0.541, sum=2.618 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.5235538594776801\"}", - "WMT 2014 - # eval": "{\"description\": \"min=503, mean=568.8, max=832, sum=2844 (5)\", \"tab\": \"General information\", \"score\": \"568.8\"}", - "WMT 2014 - # train": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "WMT 2014 - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WMT 2014 - # prompt tokens": "{\"description\": \"min=73.732, mean=103.97, max=130.366, sum=519.851 (5)\", \"tab\": \"General information\", \"score\": \"103.97025108961614\"}", - "WMT 2014 - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "language_pair": "[\"cs-en\", \"de-en\", \"fr-en\", \"hi-en\", \"ru-en\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_mmlu/google_gemma-7b/1774096312.00548", - "retrieved_timestamp": "1774096312.00548", - "source_metadata": { - "source_name": "helm_mmlu", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_mmlu", - "evaluation_results": [ - { - "evaluation_name": "MMLU All Subjects", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU All Subjects", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.661, - "details": { - "description": "min=0.28, mean=0.661, max=0.891, sum=75.376 (114)", - "tab": "Accuracy", - "MMLU All Subjects - Observed inference time (s)": "{\"description\": \"min=0.238, mean=0.312, max=0.614, sum=35.566 (114)\", \"tab\": \"Efficiency\", \"score\": \"0.3119781121356026\"}", - "MMLU All Subjects - # eval": "{\"description\": \"min=100, mean=246.351, max=1534, sum=28084 (114)\", \"tab\": \"General information\", \"score\": \"246.35087719298247\"}", - "MMLU All Subjects - # train": "{\"description\": \"min=5, mean=5, max=5, sum=570 (114)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU All Subjects - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU All Subjects - # prompt tokens": "{\"description\": \"min=260.164, mean=624.617, max=2789.424, sum=71206.345 (114)\", \"tab\": \"General information\", \"score\": \"624.6170571214202\"}", - "MMLU All Subjects - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=114 (114)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_name": "Abstract Algebra", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Abstract Algebra", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.28, - "details": { - "description": "min=0.28, mean=0.28, max=0.28, sum=0.56 (2)", - "tab": "Accuracy", - "Abstract Algebra - Observed inference time (s)": "{\"description\": \"min=0.271, mean=0.271, max=0.271, sum=0.543 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.27131984949111937\"}", - "Abstract Algebra - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Abstract Algebra - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Abstract Algebra - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Abstract Algebra - # prompt tokens": "{\"description\": \"min=375.97, mean=375.97, max=375.97, sum=751.94 (2)\", \"tab\": \"General information\", \"score\": \"375.97\"}", - "Abstract Algebra - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"abstract_algebra\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_abstract_algebra\"" - } - } - }, - { - "evaluation_name": "Anatomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Anatomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.563, - "details": { - "description": "min=0.563, mean=0.563, max=0.563, sum=1.126 (2)", - "tab": "Accuracy", - "Anatomy - Observed inference time (s)": "{\"description\": \"min=0.294, mean=0.294, max=0.294, sum=0.587 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2935627672407362\"}", - "Anatomy - # eval": "{\"description\": \"min=135, mean=135, max=135, sum=270 (2)\", \"tab\": \"General information\", \"score\": \"135.0\"}", - "Anatomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Anatomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Anatomy - # prompt tokens": "{\"description\": \"min=336.356, mean=336.356, max=336.356, sum=672.711 (2)\", \"tab\": \"General information\", \"score\": \"336.35555555555555\"}", - "Anatomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"anatomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_anatomy\"" - } - } - }, - { - "evaluation_name": "College Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on College Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.412, - "details": { - "description": "min=0.412, mean=0.412, max=0.412, sum=0.824 (2)", - "tab": "Accuracy", - "College Chemistry - Observed inference time (s)": "{\"description\": \"min=0.267, mean=0.267, max=0.267, sum=0.534 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.26709758281707763\"}", - "College Biology - Observed inference time (s)": "{\"description\": \"min=0.296, mean=0.296, max=0.296, sum=0.592 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2961096896065606\"}", - "College Computer Science - Observed inference time (s)": "{\"description\": \"min=0.29, mean=0.29, max=0.29, sum=0.58 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2900628304481506\"}", - "College Mathematics - Observed inference time (s)": "{\"description\": \"min=0.299, mean=0.299, max=0.299, sum=0.598 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.298998281955719\"}", - "College Medicine - Observed inference time (s)": "{\"description\": \"min=0.255, mean=0.255, max=0.255, sum=0.51 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.25478591119622906\"}", - "College Physics - Observed inference time (s)": "{\"description\": \"min=0.614, mean=0.614, max=0.614, sum=1.229 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.614474796781353\"}", - "College Chemistry - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Chemistry - # prompt tokens": "{\"description\": \"min=562.02, mean=562.02, max=562.02, sum=1124.04 (2)\", \"tab\": \"General information\", \"score\": \"562.02\"}", - "College Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Biology - # eval": "{\"description\": \"min=144, mean=144, max=144, sum=288 (2)\", \"tab\": \"General information\", \"score\": \"144.0\"}", - "College Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # prompt tokens": "{\"description\": \"min=474.799, mean=474.799, max=474.799, sum=949.597 (2)\", \"tab\": \"General information\", \"score\": \"474.7986111111111\"}", - "College Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # prompt tokens": "{\"description\": \"min=849.86, mean=849.86, max=849.86, sum=1699.72 (2)\", \"tab\": \"General information\", \"score\": \"849.86\"}", - "College Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Mathematics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # prompt tokens": "{\"description\": \"min=618.69, mean=618.69, max=618.69, sum=1237.38 (2)\", \"tab\": \"General information\", \"score\": \"618.69\"}", - "College Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Medicine - # eval": "{\"description\": \"min=173, mean=173, max=173, sum=346 (2)\", \"tab\": \"General information\", \"score\": \"173.0\"}", - "College Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # prompt tokens": "{\"description\": \"min=505.37, mean=505.37, max=505.37, sum=1010.74 (2)\", \"tab\": \"General information\", \"score\": \"505.3699421965318\"}", - "College Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Physics - # eval": "{\"description\": \"min=102, mean=102, max=102, sum=204 (2)\", \"tab\": \"General information\", \"score\": \"102.0\"}", - "College Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # prompt tokens": "{\"description\": \"min=499.471, mean=499.471, max=499.471, sum=998.941 (2)\", \"tab\": \"General information\", \"score\": \"499.47058823529414\"}", - "College Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"college_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_college_physics\"" - } - } - }, - { - "evaluation_name": "Computer Security", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Computer Security", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.75, - "details": { - "description": "min=0.75, mean=0.75, max=0.75, sum=1.5 (2)", - "tab": "Accuracy", - "Computer Security - Observed inference time (s)": "{\"description\": \"min=0.251, mean=0.251, max=0.251, sum=0.503 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2512932848930359\"}", - "Computer Security - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Computer Security - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Computer Security - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Computer Security - # prompt tokens": "{\"description\": \"min=372.91, mean=372.91, max=372.91, sum=745.82 (2)\", \"tab\": \"General information\", \"score\": \"372.91\"}", - "Computer Security - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"computer_security\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_computer_security\"" - } - } - }, - { - "evaluation_name": "Econometrics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Econometrics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.474, - "details": { - "description": "min=0.474, mean=0.474, max=0.474, sum=0.947 (2)", - "tab": "Accuracy", - "Econometrics - Observed inference time (s)": "{\"description\": \"min=0.285, mean=0.285, max=0.285, sum=0.569 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.28468057565521776\"}", - "Econometrics - # eval": "{\"description\": \"min=114, mean=114, max=114, sum=228 (2)\", \"tab\": \"General information\", \"score\": \"114.0\"}", - "Econometrics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Econometrics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Econometrics - # prompt tokens": "{\"description\": \"min=626.553, mean=626.553, max=626.553, sum=1253.105 (2)\", \"tab\": \"General information\", \"score\": \"626.5526315789474\"}", - "Econometrics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"econometrics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_econometrics\"" - } - } - }, - { - "evaluation_name": "Global Facts", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Global Facts", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.42, - "details": { - "description": "min=0.42, mean=0.42, max=0.42, sum=0.84 (2)", - "tab": "Accuracy", - "Global Facts - Observed inference time (s)": "{\"description\": \"min=0.296, mean=0.296, max=0.296, sum=0.591 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2956829309463501\"}", - "Global Facts - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Global Facts - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Global Facts - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Global Facts - # prompt tokens": "{\"description\": \"min=448.54, mean=448.54, max=448.54, sum=897.08 (2)\", \"tab\": \"General information\", \"score\": \"448.54\"}", - "Global Facts - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"global_facts\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_global_facts\"" - } - } - }, - { - "evaluation_name": "Jurisprudence", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Jurisprudence", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.769, - "details": { - "description": "min=0.769, mean=0.769, max=0.769, sum=1.537 (2)", - "tab": "Accuracy", - "Jurisprudence - Observed inference time (s)": "{\"description\": \"min=0.26, mean=0.26, max=0.26, sum=0.521 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.26035096910264754\"}", - "Jurisprudence - # eval": "{\"description\": \"min=108, mean=108, max=108, sum=216 (2)\", \"tab\": \"General information\", \"score\": \"108.0\"}", - "Jurisprudence - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Jurisprudence - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Jurisprudence - # prompt tokens": "{\"description\": \"min=399.87, mean=399.87, max=399.87, sum=799.741 (2)\", \"tab\": \"General information\", \"score\": \"399.8703703703704\"}", - "Jurisprudence - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"jurisprudence\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_jurisprudence\"" - } - } - }, - { - "evaluation_name": "Philosophy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Philosophy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.727, - "details": { - "description": "min=0.727, mean=0.727, max=0.727, sum=1.453 (2)", - "tab": "Accuracy", - "Philosophy - Observed inference time (s)": "{\"description\": \"min=0.276, mean=0.276, max=0.276, sum=0.552 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.276187143141817\"}", - "Philosophy - # eval": "{\"description\": \"min=311, mean=311, max=311, sum=622 (2)\", \"tab\": \"General information\", \"score\": \"311.0\"}", - "Philosophy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Philosophy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Philosophy - # prompt tokens": "{\"description\": \"min=332.907, mean=332.907, max=332.907, sum=665.814 (2)\", \"tab\": \"General information\", \"score\": \"332.90675241157555\"}", - "Philosophy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"philosophy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_philosophy\"" - } - } - }, - { - "evaluation_name": "Professional Psychology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Professional Psychology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.712, - "details": { - "description": "min=0.712, mean=0.712, max=0.712, sum=1.425 (2)", - "tab": "Accuracy", - "Professional Medicine - Observed inference time (s)": "{\"description\": \"min=0.311, mean=0.311, max=0.311, sum=0.621 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3106422327897128\"}", - "Professional Accounting - Observed inference time (s)": "{\"description\": \"min=0.292, mean=0.292, max=0.292, sum=0.583 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2916089237159026\"}", - "Professional Law - Observed inference time (s)": "{\"description\": \"min=0.385, mean=0.385, max=0.385, sum=0.77 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.38496507379812867\"}", - "Professional Psychology - Observed inference time (s)": "{\"description\": \"min=0.261, mean=0.261, max=0.261, sum=0.522 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.26078930010203444\"}", - "Professional Medicine - # eval": "{\"description\": \"min=272, mean=272, max=272, sum=544 (2)\", \"tab\": \"General information\", \"score\": \"272.0\"}", - "Professional Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Medicine - # prompt tokens": "{\"description\": \"min=1105.092, mean=1105.092, max=1105.092, sum=2210.184 (2)\", \"tab\": \"General information\", \"score\": \"1105.0919117647059\"}", - "Professional Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Accounting - # eval": "{\"description\": \"min=282, mean=282, max=282, sum=564 (2)\", \"tab\": \"General information\", \"score\": \"282.0\"}", - "Professional Accounting - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Accounting - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # prompt tokens": "{\"description\": \"min=747.418, mean=747.418, max=747.418, sum=1494.837 (2)\", \"tab\": \"General information\", \"score\": \"747.418439716312\"}", - "Professional Accounting - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Law - # eval": "{\"description\": \"min=1534, mean=1534, max=1534, sum=3068 (2)\", \"tab\": \"General information\", \"score\": \"1534.0\"}", - "Professional Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # prompt tokens": "{\"description\": \"min=1677.119, mean=1677.119, max=1677.119, sum=3354.239 (2)\", \"tab\": \"General information\", \"score\": \"1677.119295958279\"}", - "Professional Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Psychology - # eval": "{\"description\": \"min=612, mean=612, max=612, sum=1224 (2)\", \"tab\": \"General information\", \"score\": \"612.0\"}", - "Professional Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # prompt tokens": "{\"description\": \"min=586.363, mean=586.363, max=586.363, sum=1172.725 (2)\", \"tab\": \"General information\", \"score\": \"586.3627450980392\"}", - "Professional Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"professional_psychology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_professional_psychology\"" - } - } - }, - { - "evaluation_name": "Us Foreign Policy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Us Foreign Policy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.87, - "details": { - "description": "min=0.87, mean=0.87, max=0.87, sum=1.74 (2)", - "tab": "Accuracy", - "Us Foreign Policy - Observed inference time (s)": "{\"description\": \"min=0.293, mean=0.293, max=0.293, sum=0.586 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.29293906927108765\"}", - "Us Foreign Policy - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Us Foreign Policy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Us Foreign Policy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Us Foreign Policy - # prompt tokens": "{\"description\": \"min=430.2, mean=430.2, max=430.2, sum=860.4 (2)\", \"tab\": \"General information\", \"score\": \"430.2\"}", - "Us Foreign Policy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"us_foreign_policy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_us_foreign_policy\"" - } - } - }, - { - "evaluation_name": "Astronomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Astronomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.717, - "details": { - "description": "min=0.717, mean=0.717, max=0.717, sum=1.434 (2)", - "tab": "Accuracy", - "Astronomy - Observed inference time (s)": "{\"description\": \"min=0.27, mean=0.27, max=0.27, sum=0.54 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2697504366699018\"}", - "Astronomy - # eval": "{\"description\": \"min=152, mean=152, max=152, sum=304 (2)\", \"tab\": \"General information\", \"score\": \"152.0\"}", - "Astronomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Astronomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Astronomy - # prompt tokens": "{\"description\": \"min=594.421, mean=594.421, max=594.421, sum=1188.842 (2)\", \"tab\": \"General information\", \"score\": \"594.421052631579\"}", - "Astronomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"astronomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_astronomy\"" - } - } - }, - { - "evaluation_name": "Business Ethics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Business Ethics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.65, - "details": { - "description": "min=0.65, mean=0.65, max=0.65, sum=1.3 (2)", - "tab": "Accuracy", - "Business Ethics - Observed inference time (s)": "{\"description\": \"min=0.298, mean=0.298, max=0.298, sum=0.596 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.297854323387146\"}", - "Business Ethics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Business Ethics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Business Ethics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Business Ethics - # prompt tokens": "{\"description\": \"min=544.87, mean=544.87, max=544.87, sum=1089.74 (2)\", \"tab\": \"General information\", \"score\": \"544.87\"}", - "Business Ethics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"business_ethics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_business_ethics\"" - } - } - }, - { - "evaluation_name": "Clinical Knowledge", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Clinical Knowledge", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.698, - "details": { - "description": "min=0.698, mean=0.698, max=0.698, sum=1.396 (2)", - "tab": "Accuracy", - "Clinical Knowledge - Observed inference time (s)": "{\"description\": \"min=0.257, mean=0.257, max=0.257, sum=0.515 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.25743662816173624\"}", - "Clinical Knowledge - # eval": "{\"description\": \"min=265, mean=265, max=265, sum=530 (2)\", \"tab\": \"General information\", \"score\": \"265.0\"}", - "Clinical Knowledge - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Clinical Knowledge - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Clinical Knowledge - # prompt tokens": "{\"description\": \"min=394.592, mean=394.592, max=394.592, sum=789.185 (2)\", \"tab\": \"General information\", \"score\": \"394.5924528301887\"}", - "Clinical Knowledge - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"clinical_knowledge\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_clinical_knowledge\"" - } - } - }, - { - "evaluation_name": "Conceptual Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Conceptual Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.621, - "details": { - "description": "min=0.621, mean=0.621, max=0.621, sum=1.243 (2)", - "tab": "Accuracy", - "Conceptual Physics - Observed inference time (s)": "{\"description\": \"min=0.249, mean=0.249, max=0.249, sum=0.498 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.24898753064744017\"}", - "Conceptual Physics - # eval": "{\"description\": \"min=235, mean=235, max=235, sum=470 (2)\", \"tab\": \"General information\", \"score\": \"235.0\"}", - "Conceptual Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Conceptual Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Conceptual Physics - # prompt tokens": "{\"description\": \"min=301.213, mean=301.213, max=301.213, sum=602.426 (2)\", \"tab\": \"General information\", \"score\": \"301.21276595744683\"}", - "Conceptual Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"conceptual_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_conceptual_physics\"" - } - } - }, - { - "evaluation_name": "Electrical Engineering", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Electrical Engineering", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.628, - "details": { - "description": "min=0.628, mean=0.628, max=0.628, sum=1.255 (2)", - "tab": "Accuracy", - "Electrical Engineering - Observed inference time (s)": "{\"description\": \"min=0.254, mean=0.254, max=0.254, sum=0.508 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.25389171797653726\"}", - "Electrical Engineering - # eval": "{\"description\": \"min=145, mean=145, max=145, sum=290 (2)\", \"tab\": \"General information\", \"score\": \"145.0\"}", - "Electrical Engineering - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Electrical Engineering - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Electrical Engineering - # prompt tokens": "{\"description\": \"min=466.786, mean=466.786, max=466.786, sum=933.572 (2)\", \"tab\": \"General information\", \"score\": \"466.78620689655173\"}", - "Electrical Engineering - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"electrical_engineering\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_electrical_engineering\"" - } - } - }, - { - "evaluation_name": "Elementary Mathematics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Elementary Mathematics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.516, - "details": { - "description": "min=0.516, mean=0.516, max=0.516, sum=1.032 (2)", - "tab": "Accuracy", - "Elementary Mathematics - Observed inference time (s)": "{\"description\": \"min=0.287, mean=0.287, max=0.287, sum=0.573 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.28658196219691523\"}", - "Elementary Mathematics - # eval": "{\"description\": \"min=378, mean=378, max=378, sum=756 (2)\", \"tab\": \"General information\", \"score\": \"378.0\"}", - "Elementary Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Elementary Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Elementary Mathematics - # prompt tokens": "{\"description\": \"min=589.341, mean=589.341, max=589.341, sum=1178.683 (2)\", \"tab\": \"General information\", \"score\": \"589.3412698412699\"}", - "Elementary Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"elementary_mathematics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_elementary_mathematics\"" - } - } - }, - { - "evaluation_name": "Formal Logic", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Formal Logic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.508, - "details": { - "description": "min=0.508, mean=0.508, max=0.508, sum=1.016 (2)", - "tab": "Accuracy", - "Formal Logic - Observed inference time (s)": "{\"description\": \"min=0.548, mean=0.548, max=0.548, sum=1.097 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5483344452721732\"}", - "Formal Logic - # eval": "{\"description\": \"min=126, mean=126, max=126, sum=252 (2)\", \"tab\": \"General information\", \"score\": \"126.0\"}", - "Formal Logic - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Formal Logic - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Formal Logic - # prompt tokens": "{\"description\": \"min=611.563, mean=611.563, max=611.563, sum=1223.127 (2)\", \"tab\": \"General information\", \"score\": \"611.563492063492\"}", - "Formal Logic - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"formal_logic\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_formal_logic\"" - } - } - }, - { - "evaluation_name": "High School World History", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on High School World History", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.857, - "details": { - "description": "min=0.857, mean=0.857, max=0.857, sum=1.713 (2)", - "tab": "Accuracy", - "High School Biology - Observed inference time (s)": "{\"description\": \"min=0.251, mean=0.251, max=0.251, sum=0.502 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2509724578549785\"}", - "High School Chemistry - Observed inference time (s)": "{\"description\": \"min=0.292, mean=0.292, max=0.292, sum=0.584 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2920628909406991\"}", - "High School Computer Science - Observed inference time (s)": "{\"description\": \"min=0.33, mean=0.33, max=0.33, sum=0.66 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3299814939498901\"}", - "High School European History - Observed inference time (s)": "{\"description\": \"min=0.553, mean=0.553, max=0.553, sum=1.107 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5534277785908092\"}", - "High School Geography - Observed inference time (s)": "{\"description\": \"min=0.471, mean=0.471, max=0.471, sum=0.943 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.47140675602537213\"}", - "High School Government And Politics - Observed inference time (s)": "{\"description\": \"min=0.282, mean=0.282, max=0.282, sum=0.565 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.28242908734731725\"}", - "High School Macroeconomics - Observed inference time (s)": "{\"description\": \"min=0.316, mean=0.316, max=0.316, sum=0.632 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3160711630796775\"}", - "High School Mathematics - Observed inference time (s)": "{\"description\": \"min=0.256, mean=0.256, max=0.256, sum=0.512 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.25601085556877984\"}", - "High School Microeconomics - Observed inference time (s)": "{\"description\": \"min=0.251, mean=0.251, max=0.251, sum=0.503 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.25132194386810813\"}", - "High School Physics - Observed inference time (s)": "{\"description\": \"min=0.339, mean=0.339, max=0.339, sum=0.679 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3394651823485924\"}", - "High School Psychology - Observed inference time (s)": "{\"description\": \"min=0.348, mean=0.348, max=0.348, sum=0.697 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3483087859022508\"}", - "High School Statistics - Observed inference time (s)": "{\"description\": \"min=0.316, mean=0.316, max=0.316, sum=0.632 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.31601137033215276\"}", - "High School US History - Observed inference time (s)": "{\"description\": \"min=0.452, mean=0.452, max=0.452, sum=0.905 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4523548308540793\"}", - "High School World History - Observed inference time (s)": "{\"description\": \"min=0.342, mean=0.342, max=0.342, sum=0.683 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.34174740565980033\"}", - "High School Biology - # eval": "{\"description\": \"min=310, mean=310, max=310, sum=620 (2)\", \"tab\": \"General information\", \"score\": \"310.0\"}", - "High School Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Biology - # prompt tokens": "{\"description\": \"min=492.958, mean=492.958, max=492.958, sum=985.916 (2)\", \"tab\": \"General information\", \"score\": \"492.958064516129\"}", - "High School Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Chemistry - # eval": "{\"description\": \"min=203, mean=203, max=203, sum=406 (2)\", \"tab\": \"General information\", \"score\": \"203.0\"}", - "High School Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # prompt tokens": "{\"description\": \"min=505.064, mean=505.064, max=505.064, sum=1010.128 (2)\", \"tab\": \"General information\", \"score\": \"505.064039408867\"}", - "High School Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "High School Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # prompt tokens": "{\"description\": \"min=927.13, mean=927.13, max=927.13, sum=1854.26 (2)\", \"tab\": \"General information\", \"score\": \"927.13\"}", - "High School Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School European History - # eval": "{\"description\": \"min=165, mean=165, max=165, sum=330 (2)\", \"tab\": \"General information\", \"score\": \"165.0\"}", - "High School European History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School European History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # prompt tokens": "{\"description\": \"min=2789.424, mean=2789.424, max=2789.424, sum=5578.848 (2)\", \"tab\": \"General information\", \"score\": \"2789.4242424242425\"}", - "High School European History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Geography - # eval": "{\"description\": \"min=198, mean=198, max=198, sum=396 (2)\", \"tab\": \"General information\", \"score\": \"198.0\"}", - "High School Geography - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Geography - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # prompt tokens": "{\"description\": \"min=386.773, mean=386.773, max=386.773, sum=773.545 (2)\", \"tab\": \"General information\", \"score\": \"386.77272727272725\"}", - "High School Geography - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Government And Politics - # eval": "{\"description\": \"min=193, mean=193, max=193, sum=386 (2)\", \"tab\": \"General information\", \"score\": \"193.0\"}", - "High School Government And Politics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Government And Politics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # prompt tokens": "{\"description\": \"min=471.301, mean=471.301, max=471.301, sum=942.601 (2)\", \"tab\": \"General information\", \"score\": \"471.30051813471505\"}", - "High School Government And Politics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Macroeconomics - # eval": "{\"description\": \"min=390, mean=390, max=390, sum=780 (2)\", \"tab\": \"General information\", \"score\": \"390.0\"}", - "High School Macroeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Macroeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # prompt tokens": "{\"description\": \"min=388.541, mean=388.541, max=388.541, sum=777.082 (2)\", \"tab\": \"General information\", \"score\": \"388.54102564102567\"}", - "High School Macroeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Mathematics - # eval": "{\"description\": \"min=270, mean=270, max=270, sum=540 (2)\", \"tab\": \"General information\", \"score\": \"270.0\"}", - "High School Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # prompt tokens": "{\"description\": \"min=558.822, mean=558.822, max=558.822, sum=1117.644 (2)\", \"tab\": \"General information\", \"score\": \"558.8222222222222\"}", - "High School Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Microeconomics - # eval": "{\"description\": \"min=238, mean=238, max=238, sum=476 (2)\", \"tab\": \"General information\", \"score\": \"238.0\"}", - "High School Microeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Microeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # prompt tokens": "{\"description\": \"min=407.954, mean=407.954, max=407.954, sum=815.908 (2)\", \"tab\": \"General information\", \"score\": \"407.953781512605\"}", - "High School Microeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Physics - # eval": "{\"description\": \"min=151, mean=151, max=151, sum=302 (2)\", \"tab\": \"General information\", \"score\": \"151.0\"}", - "High School Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # prompt tokens": "{\"description\": \"min=583.715, mean=583.715, max=583.715, sum=1167.43 (2)\", \"tab\": \"General information\", \"score\": \"583.7152317880794\"}", - "High School Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Psychology - # eval": "{\"description\": \"min=545, mean=545, max=545, sum=1090 (2)\", \"tab\": \"General information\", \"score\": \"545.0\"}", - "High School Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # prompt tokens": "{\"description\": \"min=494.604, mean=494.604, max=494.604, sum=989.207 (2)\", \"tab\": \"General information\", \"score\": \"494.60366972477067\"}", - "High School Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Statistics - # eval": "{\"description\": \"min=216, mean=216, max=216, sum=432 (2)\", \"tab\": \"General information\", \"score\": \"216.0\"}", - "High School Statistics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Statistics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # prompt tokens": "{\"description\": \"min=850.931, mean=850.931, max=850.931, sum=1701.861 (2)\", \"tab\": \"General information\", \"score\": \"850.9305555555555\"}", - "High School Statistics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School US History - # eval": "{\"description\": \"min=204, mean=204, max=204, sum=408 (2)\", \"tab\": \"General information\", \"score\": \"204.0\"}", - "High School US History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School US History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # prompt tokens": "{\"description\": \"min=2197.583, mean=2197.583, max=2197.583, sum=4395.167 (2)\", \"tab\": \"General information\", \"score\": \"2197.5833333333335\"}", - "High School US History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School World History - # eval": "{\"description\": \"min=237, mean=237, max=237, sum=474 (2)\", \"tab\": \"General information\", \"score\": \"237.0\"}", - "High School World History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School World History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # prompt tokens": "{\"description\": \"min=1418.544, mean=1418.544, max=1418.544, sum=2837.089 (2)\", \"tab\": \"General information\", \"score\": \"1418.5443037974683\"}", - "High School World History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"high_school_world_history\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_high_school_world_history\"" - } - } - }, - { - "evaluation_name": "Human Sexuality", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Human Sexuality", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.733, - "details": { - "description": "min=0.733, mean=0.733, max=0.733, sum=1.466 (2)", - "tab": "Accuracy", - "Human Aging - Observed inference time (s)": "{\"description\": \"min=0.406, mean=0.406, max=0.406, sum=0.812 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4062144061375092\"}", - "Human Sexuality - Observed inference time (s)": "{\"description\": \"min=0.238, mean=0.238, max=0.238, sum=0.476 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.23785374910776852\"}", - "Human Aging - # eval": "{\"description\": \"min=223, mean=223, max=223, sum=446 (2)\", \"tab\": \"General information\", \"score\": \"223.0\"}", - "Human Aging - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Aging - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Aging - # prompt tokens": "{\"description\": \"min=313.587, mean=313.587, max=313.587, sum=627.175 (2)\", \"tab\": \"General information\", \"score\": \"313.58744394618833\"}", - "Human Aging - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Human Sexuality - # eval": "{\"description\": \"min=131, mean=131, max=131, sum=262 (2)\", \"tab\": \"General information\", \"score\": \"131.0\"}", - "Human Sexuality - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Sexuality - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # prompt tokens": "{\"description\": \"min=339.183, mean=339.183, max=339.183, sum=678.366 (2)\", \"tab\": \"General information\", \"score\": \"339.1832061068702\"}", - "Human Sexuality - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"human_sexuality\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_human_sexuality\"" - } - } - }, - { - "evaluation_name": "International Law", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on International Law", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.835, - "details": { - "description": "min=0.835, mean=0.835, max=0.835, sum=1.669 (2)", - "tab": "Accuracy", - "International Law - Observed inference time (s)": "{\"description\": \"min=0.292, mean=0.292, max=0.292, sum=0.584 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2918710767730208\"}", - "International Law - # eval": "{\"description\": \"min=121, mean=121, max=121, sum=242 (2)\", \"tab\": \"General information\", \"score\": \"121.0\"}", - "International Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "International Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "International Law - # prompt tokens": "{\"description\": \"min=636.165, mean=636.165, max=636.165, sum=1272.331 (2)\", \"tab\": \"General information\", \"score\": \"636.1652892561983\"}", - "International Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"international_law\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_international_law\"" - } - } - }, - { - "evaluation_name": "Logical Fallacies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Logical Fallacies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.742, - "details": { - "description": "min=0.742, mean=0.742, max=0.742, sum=1.485 (2)", - "tab": "Accuracy", - "Logical Fallacies - Observed inference time (s)": "{\"description\": \"min=0.477, mean=0.477, max=0.477, sum=0.954 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.47711458089161507\"}", - "Logical Fallacies - # eval": "{\"description\": \"min=163, mean=163, max=163, sum=326 (2)\", \"tab\": \"General information\", \"score\": \"163.0\"}", - "Logical Fallacies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Logical Fallacies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Logical Fallacies - # prompt tokens": "{\"description\": \"min=442.049, mean=442.049, max=442.049, sum=884.098 (2)\", \"tab\": \"General information\", \"score\": \"442.0490797546012\"}", - "Logical Fallacies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"logical_fallacies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_logical_fallacies\"" - } - } - }, - { - "evaluation_name": "Machine Learning", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Machine Learning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.554, - "details": { - "description": "min=0.554, mean=0.554, max=0.554, sum=1.107 (2)", - "tab": "Accuracy", - "Machine Learning - Observed inference time (s)": "{\"description\": \"min=0.265, mean=0.265, max=0.265, sum=0.529 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2645489977938788\"}", - "Machine Learning - # eval": "{\"description\": \"min=112, mean=112, max=112, sum=224 (2)\", \"tab\": \"General information\", \"score\": \"112.0\"}", - "Machine Learning - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Machine Learning - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Machine Learning - # prompt tokens": "{\"description\": \"min=694.402, mean=694.402, max=694.402, sum=1388.804 (2)\", \"tab\": \"General information\", \"score\": \"694.4017857142857\"}", - "Machine Learning - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"machine_learning\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_machine_learning\"" - } - } - }, - { - "evaluation_name": "Management", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Management", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.864, - "details": { - "description": "min=0.864, mean=0.864, max=0.864, sum=1.728 (2)", - "tab": "Accuracy", - "Management - Observed inference time (s)": "{\"description\": \"min=0.293, mean=0.293, max=0.293, sum=0.587 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.293421483734279\"}", - "Management - # eval": "{\"description\": \"min=103, mean=103, max=103, sum=206 (2)\", \"tab\": \"General information\", \"score\": \"103.0\"}", - "Management - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Management - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Management - # prompt tokens": "{\"description\": \"min=273.301, mean=273.301, max=273.301, sum=546.602 (2)\", \"tab\": \"General information\", \"score\": \"273.3009708737864\"}", - "Management - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"management\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_management\"" - } - } - }, - { - "evaluation_name": "Marketing", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Marketing", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.885, - "details": { - "description": "min=0.885, mean=0.885, max=0.885, sum=1.769 (2)", - "tab": "Accuracy", - "Marketing - Observed inference time (s)": "{\"description\": \"min=0.254, mean=0.254, max=0.254, sum=0.507 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.25355013211568195\"}", - "Marketing - # eval": "{\"description\": \"min=234, mean=234, max=234, sum=468 (2)\", \"tab\": \"General information\", \"score\": \"234.0\"}", - "Marketing - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Marketing - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Marketing - # prompt tokens": "{\"description\": \"min=420.35, mean=420.35, max=420.35, sum=840.701 (2)\", \"tab\": \"General information\", \"score\": \"420.35042735042737\"}", - "Marketing - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"marketing\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_marketing\"" - } - } - }, - { - "evaluation_name": "Medical Genetics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Medical Genetics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7, - "details": { - "description": "min=0.7, mean=0.7, max=0.7, sum=1.4 (2)", - "tab": "Accuracy", - "Medical Genetics - Observed inference time (s)": "{\"description\": \"min=0.262, mean=0.262, max=0.262, sum=0.524 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.26187997102737426\"}", - "Medical Genetics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Medical Genetics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Medical Genetics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Medical Genetics - # prompt tokens": "{\"description\": \"min=330.89, mean=330.89, max=330.89, sum=661.78 (2)\", \"tab\": \"General information\", \"score\": \"330.89\"}", - "Medical Genetics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"medical_genetics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_medical_genetics\"" - } - } - }, - { - "evaluation_name": "Miscellaneous", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Miscellaneous", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.838, - "details": { - "description": "min=0.838, mean=0.838, max=0.838, sum=1.676 (2)", - "tab": "Accuracy", - "Miscellaneous - Observed inference time (s)": "{\"description\": \"min=0.245, mean=0.245, max=0.245, sum=0.49 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.24482133348935103\"}", - "Miscellaneous - # eval": "{\"description\": \"min=783, mean=783, max=783, sum=1566 (2)\", \"tab\": \"General information\", \"score\": \"783.0\"}", - "Miscellaneous - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Miscellaneous - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Miscellaneous - # prompt tokens": "{\"description\": \"min=306.669, mean=306.669, max=306.669, sum=613.338 (2)\", \"tab\": \"General information\", \"score\": \"306.669220945083\"}", - "Miscellaneous - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"miscellaneous\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_miscellaneous\"" - } - } - }, - { - "evaluation_name": "Moral Scenarios", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Moral Scenarios", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.377, - "details": { - "description": "min=0.377, mean=0.377, max=0.377, sum=0.753 (2)", - "tab": "Accuracy", - "Moral Disputes - Observed inference time (s)": "{\"description\": \"min=0.254, mean=0.254, max=0.254, sum=0.508 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2542355225954442\"}", - "Moral Scenarios - Observed inference time (s)": "{\"description\": \"min=0.392, mean=0.392, max=0.392, sum=0.784 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.39224682173915415\"}", - "Moral Disputes - # eval": "{\"description\": \"min=346, mean=346, max=346, sum=692 (2)\", \"tab\": \"General information\", \"score\": \"346.0\"}", - "Moral Disputes - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Disputes - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Disputes - # prompt tokens": "{\"description\": \"min=487.003, mean=487.003, max=487.003, sum=974.006 (2)\", \"tab\": \"General information\", \"score\": \"487.0028901734104\"}", - "Moral Disputes - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Moral Scenarios - # eval": "{\"description\": \"min=895, mean=895, max=895, sum=1790 (2)\", \"tab\": \"General information\", \"score\": \"895.0\"}", - "Moral Scenarios - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Scenarios - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # prompt tokens": "{\"description\": \"min=682.542, mean=682.542, max=682.542, sum=1365.084 (2)\", \"tab\": \"General information\", \"score\": \"682.5418994413408\"}", - "Moral Scenarios - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"moral_scenarios\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_moral_scenarios\"" - } - } - }, - { - "evaluation_name": "Nutrition", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Nutrition", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.778, - "details": { - "description": "min=0.778, mean=0.778, max=0.778, sum=1.556 (2)", - "tab": "Accuracy", - "Nutrition - Observed inference time (s)": "{\"description\": \"min=0.351, mean=0.351, max=0.351, sum=0.702 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3507605791091919\"}", - "Nutrition - # eval": "{\"description\": \"min=306, mean=306, max=306, sum=612 (2)\", \"tab\": \"General information\", \"score\": \"306.0\"}", - "Nutrition - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Nutrition - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Nutrition - # prompt tokens": "{\"description\": \"min=577.48, mean=577.48, max=577.48, sum=1154.961 (2)\", \"tab\": \"General information\", \"score\": \"577.4803921568628\"}", - "Nutrition - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"nutrition\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_nutrition\"" - } - } - }, - { - "evaluation_name": "Prehistory", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Prehistory", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.756, - "details": { - "description": "min=0.756, mean=0.756, max=0.756, sum=1.512 (2)", - "tab": "Accuracy", - "Prehistory - Observed inference time (s)": "{\"description\": \"min=0.254, mean=0.254, max=0.254, sum=0.509 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.25446349014470604\"}", - "Prehistory - # eval": "{\"description\": \"min=324, mean=324, max=324, sum=648 (2)\", \"tab\": \"General information\", \"score\": \"324.0\"}", - "Prehistory - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Prehistory - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Prehistory - # prompt tokens": "{\"description\": \"min=532.198, mean=532.198, max=532.198, sum=1064.395 (2)\", \"tab\": \"General information\", \"score\": \"532.1975308641976\"}", - "Prehistory - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"prehistory\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_prehistory\"" - } - } - }, - { - "evaluation_name": "Public Relations", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Public Relations", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.682, - "details": { - "description": "min=0.682, mean=0.682, max=0.682, sum=1.364 (2)", - "tab": "Accuracy", - "Public Relations - Observed inference time (s)": "{\"description\": \"min=0.248, mean=0.248, max=0.248, sum=0.495 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.24754605726762252\"}", - "Public Relations - # eval": "{\"description\": \"min=110, mean=110, max=110, sum=220 (2)\", \"tab\": \"General information\", \"score\": \"110.0\"}", - "Public Relations - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Public Relations - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Public Relations - # prompt tokens": "{\"description\": \"min=418.655, mean=418.655, max=418.655, sum=837.309 (2)\", \"tab\": \"General information\", \"score\": \"418.6545454545454\"}", - "Public Relations - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"public_relations\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_public_relations\"" - } - } - }, - { - "evaluation_name": "Security Studies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Security Studies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.735, - "details": { - "description": "min=0.735, mean=0.735, max=0.735, sum=1.469 (2)", - "tab": "Accuracy", - "Security Studies - Observed inference time (s)": "{\"description\": \"min=0.3, mean=0.3, max=0.3, sum=0.6 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.30012765806548447\"}", - "Security Studies - # eval": "{\"description\": \"min=245, mean=245, max=245, sum=490 (2)\", \"tab\": \"General information\", \"score\": \"245.0\"}", - "Security Studies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Security Studies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Security Studies - # prompt tokens": "{\"description\": \"min=1185.869, mean=1185.869, max=1185.869, sum=2371.739 (2)\", \"tab\": \"General information\", \"score\": \"1185.869387755102\"}", - "Security Studies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"security_studies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_security_studies\"" - } - } - }, - { - "evaluation_name": "Sociology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Sociology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.841, - "details": { - "description": "min=0.841, mean=0.841, max=0.841, sum=1.682 (2)", - "tab": "Accuracy", - "Sociology - Observed inference time (s)": "{\"description\": \"min=0.293, mean=0.293, max=0.293, sum=0.586 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.29275026487473826\"}", - "Sociology - # eval": "{\"description\": \"min=201, mean=201, max=201, sum=402 (2)\", \"tab\": \"General information\", \"score\": \"201.0\"}", - "Sociology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Sociology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Sociology - # prompt tokens": "{\"description\": \"min=448.274, mean=448.274, max=448.274, sum=896.547 (2)\", \"tab\": \"General information\", \"score\": \"448.27363184079604\"}", - "Sociology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"sociology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_sociology\"" - } - } - }, - { - "evaluation_name": "Virology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Virology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.548, - "details": { - "description": "min=0.548, mean=0.548, max=0.548, sum=1.096 (2)", - "tab": "Accuracy", - "Virology - Observed inference time (s)": "{\"description\": \"min=0.25, mean=0.25, max=0.25, sum=0.501 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2502512199332915\"}", - "Virology - # eval": "{\"description\": \"min=166, mean=166, max=166, sum=332 (2)\", \"tab\": \"General information\", \"score\": \"166.0\"}", - "Virology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Virology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Virology - # prompt tokens": "{\"description\": \"min=328.753, mean=328.753, max=328.753, sum=657.506 (2)\", \"tab\": \"General information\", \"score\": \"328.7530120481928\"}", - "Virology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"virology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_virology\"" - } - } - }, - { - "evaluation_name": "World Religions", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on World Religions", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.842, - "details": { - "description": "min=0.842, mean=0.842, max=0.842, sum=1.684 (2)", - "tab": "Accuracy", - "World Religions - Observed inference time (s)": "{\"description\": \"min=0.249, mean=0.249, max=0.249, sum=0.498 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.24913478734200462\"}", - "World Religions - # eval": "{\"description\": \"min=171, mean=171, max=171, sum=342 (2)\", \"tab\": \"General information\", \"score\": \"171.0\"}", - "World Religions - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "World Religions - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "World Religions - # prompt tokens": "{\"description\": \"min=260.164, mean=260.164, max=260.164, sum=520.327 (2)\", \"tab\": \"General information\", \"score\": \"260.1637426900585\"}", - "World Religions - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"world_religions\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_world_religions\"" - } - } - }, - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.824, - "details": { - "description": "", - "tab": "Efficiency" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_id": "hfopenllm_v2/google_gemma-7b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2659 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4362 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.074 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2869 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4062 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2948 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/google_mt5-base.json b/data/models/google_mt5-base.json deleted file mode 100644 index 10a2a32518d63d11387192ec0def85f539339b23..0000000000000000000000000000000000000000 --- a/data/models/google_mt5-base.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "mt5-base", - "id": "google/mt5-base", - "developer": "Google", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MT5ForConditionalGeneration", - "params_billions": "0.39" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/google_mt5-base/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1645 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2883 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0091 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2391 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3672 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.107 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/google_mt5-small.json b/data/models/google_mt5-small.json deleted file mode 100644 index b30ddaf1ab64cc66ca8a9595a1fba760f107a497..0000000000000000000000000000000000000000 --- a/data/models/google_mt5-small.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "mt5-small", - "id": "google/mt5-small", - "developer": "Google", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MT5ForConditionalGeneration", - "params_billions": "0.17" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/google_mt5-small/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1718 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2766 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2424 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3857 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1123 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/google_mt5-xl.json b/data/models/google_mt5-xl.json deleted file mode 100644 index 37d26ffad51ced1706fb864ea9004fd9fb3d846a..0000000000000000000000000000000000000000 --- a/data/models/google_mt5-xl.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "mt5-xl", - "id": "google/mt5-xl", - "developer": "Google", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MT5ForConditionalGeneration", - "params_billions": "3.23" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/google_mt5-xl/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.196 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3047 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2643 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3795 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.112 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/google_mt5-xxl.json b/data/models/google_mt5-xxl.json deleted file mode 100644 index 7faab94915ecf0fd43c50a2643af6b484239c307..0000000000000000000000000000000000000000 --- a/data/models/google_mt5-xxl.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "mt5-xxl", - "id": "google/mt5-xxl", - "developer": "Google", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "T5ForConditionalGeneration", - "params_billions": "11.9" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/google_mt5-xxl/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2358 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2959 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2416 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3689 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1089 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/google_palmyra-x-43b.json b/data/models/google_palmyra-x-43b.json deleted file mode 100644 index b965463e115d245fb7f58470b2bf5f71755f154b..0000000000000000000000000000000000000000 --- a/data/models/google_palmyra-x-43b.json +++ /dev/null @@ -1,674 +0,0 @@ -{ - "model_info": { - "name": "Palmyra X 43B", - "id": "google/Palmyra-X-43B", - "developer": "Google", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_classic/google_Palmyra-X-43B/1774096308.339228", - "retrieved_timestamp": "1774096308.339228", - "source_metadata": { - "source_name": "helm_classic", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_classic", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_classic", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperform on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.732, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Calibration": "{\"description\": \"\", \"tab\": \"Calibration\", \"score\": \"\"}", - "Mean win rate - Robustness": "{\"description\": \"\", \"tab\": \"Robustness\", \"score\": \"0.8206682206682206\"}", - "Mean win rate - Fairness": "{\"description\": \"\", \"tab\": \"Fairness\", \"score\": \"0.7968401968401968\"}", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}", - "Mean win rate - Bias": "{\"description\": \"\", \"tab\": \"Bias\", \"score\": \"0.5458006056443556\"}", - "Mean win rate - Toxicity": "{\"description\": \"\", \"tab\": \"Toxicity\", \"score\": \"0.462995337995338\"}", - "Mean win rate - Summarization metrics": "{\"description\": \"\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.609, - "details": { - "description": "min=0.35, mean=0.609, max=0.88, sum=9.136 (15)", - "tab": "Accuracy", - "MMLU - ECE (10-bin)": "{\"description\": \"5 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "MMLU - EM (Robustness)": "{\"description\": \"min=0.29, mean=0.566, max=0.86, sum=8.494 (15)\", \"tab\": \"Robustness\", \"score\": \"0.5662339181286549\"}", - "MMLU - EM (Fairness)": "{\"description\": \"min=0.34, mean=0.588, max=0.86, sum=8.822 (15)\", \"tab\": \"Fairness\", \"score\": \"0.5881637426900584\"}", - "MMLU - Denoised inference time (s)": "{\"description\": \"5 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=1542 (15)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=75 (15)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (15)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=371.38, mean=472.274, max=624.07, sum=7084.111 (15)\", \"tab\": \"General information\", \"score\": \"472.2740350877193\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=15 (15)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "MMLU - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=45 (15)\", \"tab\": \"General information\", \"score\": \"3.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "BoolQ", - "source_data": { - "dataset_name": "BoolQ", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on BoolQ", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.896, - "details": { - "description": "min=0.894, mean=0.896, max=0.898, sum=2.689 (3)", - "tab": "Accuracy", - "BoolQ - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "BoolQ - EM (Robustness)": "{\"description\": \"min=0.875, mean=0.878, max=0.88, sum=2.634 (3)\", \"tab\": \"Robustness\", \"score\": \"0.878\"}", - "BoolQ - EM (Fairness)": "{\"description\": \"min=0.872, mean=0.875, max=0.878, sum=2.625 (3)\", \"tab\": \"Fairness\", \"score\": \"0.875\"}", - "BoolQ - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "BoolQ - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "BoolQ - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "BoolQ - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "BoolQ - # prompt tokens": "{\"description\": \"min=660.073, mean=908.406, max=1242.073, sum=2725.219 (3)\", \"tab\": \"General information\", \"score\": \"908.4063333333334\"}", - "BoolQ - # output tokens": "{\"description\": \"min=1.005, mean=1.007, max=1.01, sum=3.021 (3)\", \"tab\": \"General information\", \"score\": \"1.007\"}", - "BoolQ - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "BoolQ - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.742, - "details": { - "description": "min=0.732, mean=0.742, max=0.748, sum=2.226 (3)", - "tab": "Accuracy", - "NarrativeQA - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "NarrativeQA - F1 (Robustness)": "{\"description\": \"min=0.667, mean=0.672, max=0.68, sum=2.016 (3)\", \"tab\": \"Robustness\", \"score\": \"0.6719021727640991\"}", - "NarrativeQA - F1 (Fairness)": "{\"description\": \"min=0.64, mean=0.651, max=0.659, sum=1.952 (3)\", \"tab\": \"Fairness\", \"score\": \"0.6506183133514157\"}", - "NarrativeQA - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=1065 (3)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=3504.577, mean=3803.911, max=3972.577, sum=11411.732 (3)\", \"tab\": \"General information\", \"score\": \"3803.910798122066\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=5.375, mean=6.272, max=7.29, sum=18.817 (3)\", \"tab\": \"General information\", \"score\": \"6.272300469483568\"}", - "NarrativeQA - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NarrativeQA - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=1.333 (2)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666667\"}", - "NarrativeQA - Stereotypes (gender)": "{\"description\": \"min=0.396, mean=0.398, max=0.403, sum=1.194 (3)\", \"tab\": \"Bias\", \"score\": \"0.39814814814814814\"}", - "NarrativeQA - Representation (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=2 (3)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "NarrativeQA - Representation (gender)": "{\"description\": \"min=0.149, mean=0.159, max=0.181, sum=0.478 (3)\", \"tab\": \"Bias\", \"score\": \"0.15935305534542177\"}", - "NarrativeQA - Toxic fraction": "{\"description\": \"min=0.008, mean=0.011, max=0.014, sum=0.034 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.011267605633802818\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (open-book)", - "source_data": { - "dataset_name": "NaturalQuestions (open-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (open-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "NaturalQuestions (closed-book) - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "NaturalQuestions (open-book) - ECE (10-bin)": "{\"description\": \"No matching runs\", \"tab\": \"Calibration\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - F1 (Robustness)": "{\"description\": \"min=0.355, mean=0.363, max=0.368, sum=1.089 (3)\", \"tab\": \"Robustness\", \"score\": \"0.3629707081568259\"}", - "NaturalQuestions (open-book) - F1 (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - F1 (Fairness)": "{\"description\": \"min=0.356, mean=0.362, max=0.367, sum=1.087 (3)\", \"tab\": \"Fairness\", \"score\": \"0.3624320629787478\"}", - "NaturalQuestions (open-book) - F1 (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NaturalQuestions (open-book) - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=110.254, mean=112.254, max=116.254, sum=336.762 (3)\", \"tab\": \"General information\", \"score\": \"112.254\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=3.166, mean=3.19, max=3.231, sum=9.571 (3)\", \"tab\": \"General information\", \"score\": \"3.1903333333333332\"}", - "NaturalQuestions (closed-book) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "NaturalQuestions (open-book) - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=2 (3)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "NaturalQuestions (closed-book) - Stereotypes (gender)": "{\"description\": \"min=0.5, mean=0.5, max=0.5, sum=0.5 (1)\", \"tab\": \"Bias\", \"score\": \"0.5\"}", - "NaturalQuestions (closed-book) - Representation (race)": "{\"description\": \"min=0.245, mean=0.314, max=0.378, sum=0.941 (3)\", \"tab\": \"Bias\", \"score\": \"0.31352905160694455\"}", - "NaturalQuestions (closed-book) - Representation (gender)": "{\"description\": \"min=0.25, mean=0.266, max=0.278, sum=0.797 (3)\", \"tab\": \"Bias\", \"score\": \"0.26566951566951563\"}", - "NaturalQuestions (open-book) - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "NaturalQuestions (open-book) - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "NaturalQuestions (open-book) - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "NaturalQuestions (open-book) - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - Toxic fraction": "{\"description\": \"min=0.001, mean=0.001, max=0.001, sum=0.003 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.001\"}", - "NaturalQuestions (open-book) - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "QuAC", - "source_data": { - "dataset_name": "QuAC", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on QuAC", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.473, - "details": { - "description": "min=0.459, mean=0.473, max=0.488, sum=1.419 (3)", - "tab": "Accuracy", - "QuAC - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "QuAC - F1 (Robustness)": "{\"description\": \"min=0.379, mean=0.383, max=0.392, sum=1.15 (3)\", \"tab\": \"Robustness\", \"score\": \"0.38348793103386436\"}", - "QuAC - F1 (Fairness)": "{\"description\": \"min=0.394, mean=0.399, max=0.408, sum=1.196 (3)\", \"tab\": \"Fairness\", \"score\": \"0.39873411995988545\"}", - "QuAC - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "QuAC - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "QuAC - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "QuAC - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "QuAC - # prompt tokens": "{\"description\": \"min=4676.788, mean=5199.788, max=5842.788, sum=15599.364 (3)\", \"tab\": \"General information\", \"score\": \"5199.788\"}", - "QuAC - # output tokens": "{\"description\": \"min=25.906, mean=26.581, max=27.052, sum=79.742 (3)\", \"tab\": \"General information\", \"score\": \"26.580666666666662\"}", - "QuAC - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "QuAC - Stereotypes (race)": "{\"description\": \"min=0.63, mean=0.642, max=0.667, sum=1.926 (3)\", \"tab\": \"Bias\", \"score\": \"0.6419753086419754\"}", - "QuAC - Stereotypes (gender)": "{\"description\": \"min=0.364, mean=0.395, max=0.447, sum=1.186 (3)\", \"tab\": \"Bias\", \"score\": \"0.39526937310090554\"}", - "QuAC - Representation (race)": "{\"description\": \"min=0.286, mean=0.293, max=0.298, sum=0.878 (3)\", \"tab\": \"Bias\", \"score\": \"0.29267512260888473\"}", - "QuAC - Representation (gender)": "{\"description\": \"min=0.221, mean=0.235, max=0.248, sum=0.705 (3)\", \"tab\": \"Bias\", \"score\": \"0.23492413534960777\"}", - "QuAC - Toxic fraction": "{\"description\": \"min=0, mean=0.001, max=0.002, sum=0.003 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.001\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "HellaSwag", - "source_data": { - "dataset_name": "HellaSwag", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on HellaSwag", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "HellaSwag - ECE (10-bin)": "{\"description\": \"No matching runs\", \"tab\": \"Calibration\", \"score\": \"\"}", - "HellaSwag - EM (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "HellaSwag - EM (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "HellaSwag - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "HellaSwag - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "OpenbookQA - ECE (10-bin)": "{\"description\": \"No matching runs\", \"tab\": \"Calibration\", \"score\": \"\"}", - "OpenbookQA - EM (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "OpenbookQA - EM (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "OpenbookQA - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "OpenbookQA - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "TruthfulQA", - "source_data": { - "dataset_name": "TruthfulQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on TruthfulQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.616, - "details": { - "description": "min=0.601, mean=0.616, max=0.63, sum=1.847 (3)", - "tab": "Accuracy", - "TruthfulQA - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "TruthfulQA - EM (Robustness)": "{\"description\": \"min=0.554, mean=0.568, max=0.584, sum=1.705 (3)\", \"tab\": \"Robustness\", \"score\": \"0.5682976554536188\"}", - "TruthfulQA - EM (Fairness)": "{\"description\": \"min=0.529, mean=0.542, max=0.56, sum=1.625 (3)\", \"tab\": \"Fairness\", \"score\": \"0.5417940876656473\"}", - "TruthfulQA - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "TruthfulQA - # eval": "{\"description\": \"min=654, mean=654, max=654, sum=1962 (3)\", \"tab\": \"General information\", \"score\": \"654.0\"}", - "TruthfulQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "TruthfulQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "TruthfulQA - # prompt tokens": "{\"description\": \"min=501.121, mean=511.121, max=529.121, sum=1533.362 (3)\", \"tab\": \"General information\", \"score\": \"511.12079510703364\"}", - "TruthfulQA - # output tokens": "{\"description\": \"min=0.908, mean=0.949, max=0.982, sum=2.847 (3)\", \"tab\": \"General information\", \"score\": \"0.9490316004077473\"}", - "TruthfulQA - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MS MARCO (TREC)", - "source_data": { - "dataset_name": "MS MARCO (TREC)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "NDCG@10 on MS MARCO (TREC)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "MS MARCO (regular) - RR@10 (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "MS MARCO (TREC) - NDCG@10 (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "MS MARCO (regular) - RR@10 (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "MS MARCO (TREC) - NDCG@10 (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "MS MARCO (regular) - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MS MARCO (TREC) - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MS MARCO (regular) - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "MS MARCO (TREC) - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CNN/DailyMail", - "source_data": { - "dataset_name": "CNN/DailyMail", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on CNN/DailyMail", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.049, - "details": { - "description": "min=0, mean=0.049, max=0.147, sum=0.147 (3)", - "tab": "Accuracy", - "CNN/DailyMail - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "CNN/DailyMail - # eval": "{\"description\": \"min=466, mean=466, max=466, sum=1398 (3)\", \"tab\": \"General information\", \"score\": \"466.0\"}", - "CNN/DailyMail - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CNN/DailyMail - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CNN/DailyMail - # prompt tokens": "{\"description\": \"min=1531.586, mean=1549.919, max=1567.586, sum=4649.758 (3)\", \"tab\": \"General information\", \"score\": \"1549.9191702432045\"}", - "CNN/DailyMail - # output tokens": "{\"description\": \"min=0, mean=17.63, max=52.891, sum=52.891 (3)\", \"tab\": \"General information\", \"score\": \"17.630185979971387\"}", - "CNN/DailyMail - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "CNN/DailyMail - Stereotypes (race)": "{\"description\": \"min=0.622, mean=0.622, max=0.622, sum=0.622 (1)\", \"tab\": \"Bias\", \"score\": \"0.6219394640447272\"}", - "CNN/DailyMail - Stereotypes (gender)": "{\"description\": \"min=0.421, mean=0.421, max=0.421, sum=0.421 (1)\", \"tab\": \"Bias\", \"score\": \"0.42094867293009713\"}", - "CNN/DailyMail - Representation (race)": "{\"description\": \"min=0.276, mean=0.276, max=0.276, sum=0.276 (1)\", \"tab\": \"Bias\", \"score\": \"0.27642276422764234\"}", - "CNN/DailyMail - Representation (gender)": "{\"description\": \"min=0.114, mean=0.114, max=0.114, sum=0.114 (1)\", \"tab\": \"Bias\", \"score\": \"0.11422708618331054\"}", - "CNN/DailyMail - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}", - "CNN/DailyMail - SummaC": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - QAFactEval": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - BERTScore (F1)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - Coverage": "{\"description\": \"min=0, mean=0.291, max=0.872, sum=0.872 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.29078580039209107\"}", - "CNN/DailyMail - Density": "{\"description\": \"min=0, mean=2.35, max=7.049, sum=7.049 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"2.34978873721003\"}", - "CNN/DailyMail - Compression": "{\"description\": \"min=0, mean=3.117, max=9.351, sum=9.351 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"3.116859693035\"}", - "CNN/DailyMail - HumanEval-faithfulness": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-relevance": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-coherence": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "XSUM", - "source_data": { - "dataset_name": "XSUM", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on XSUM", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.149, - "details": { - "description": "min=0.144, mean=0.149, max=0.157, sum=0.447 (3)", - "tab": "Accuracy", - "XSUM - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "XSUM - # eval": "{\"description\": \"min=518, mean=518, max=518, sum=1554 (3)\", \"tab\": \"General information\", \"score\": \"518.0\"}", - "XSUM - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "XSUM - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "XSUM - # prompt tokens": "{\"description\": \"min=1456.402, mean=1510.735, max=1539.402, sum=4532.205 (3)\", \"tab\": \"General information\", \"score\": \"1510.734877734878\"}", - "XSUM - # output tokens": "{\"description\": \"min=25.077, mean=25.248, max=25.463, sum=75.745 (3)\", \"tab\": \"General information\", \"score\": \"25.248391248391247\"}", - "XSUM - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "XSUM - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=2.0 (3)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666669\"}", - "XSUM - Stereotypes (gender)": "{\"description\": \"min=0.434, mean=0.438, max=0.444, sum=1.313 (3)\", \"tab\": \"Bias\", \"score\": \"0.43769157088122607\"}", - "XSUM - Representation (race)": "{\"description\": \"min=0.383, mean=0.439, max=0.494, sum=1.318 (3)\", \"tab\": \"Bias\", \"score\": \"0.4393992219104699\"}", - "XSUM - Representation (gender)": "{\"description\": \"min=0.202, mean=0.205, max=0.208, sum=0.616 (3)\", \"tab\": \"Bias\", \"score\": \"0.2054618848004968\"}", - "XSUM - Toxic fraction": "{\"description\": \"min=0.002, mean=0.002, max=0.002, sum=0.006 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.0019305019305019308\"}", - "XSUM - SummaC": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - QAFactEval": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - BERTScore (F1)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - Coverage": "{\"description\": \"min=0.77, mean=0.775, max=0.778, sum=2.324 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.7746217499327193\"}", - "XSUM - Density": "{\"description\": \"min=2.38, mean=2.466, max=2.546, sum=7.399 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"2.4662768763204443\"}", - "XSUM - Compression": "{\"description\": \"min=14.242, mean=14.252, max=14.266, sum=42.756 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"14.25194669426599\"}", - "XSUM - HumanEval-faithfulness": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-relevance": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-coherence": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "IMDB", - "source_data": { - "dataset_name": "IMDB", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on IMDB", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.935, - "details": { - "description": "min=0.928, mean=0.935, max=0.939, sum=2.806 (3)", - "tab": "Accuracy", - "IMDB - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "IMDB - EM (Robustness)": "{\"description\": \"min=0.896, mean=0.904, max=0.909, sum=2.713 (3)\", \"tab\": \"Robustness\", \"score\": \"0.9043333333333333\"}", - "IMDB - EM (Fairness)": "{\"description\": \"min=0.909, mean=0.918, max=0.923, sum=2.754 (3)\", \"tab\": \"Fairness\", \"score\": \"0.918\"}", - "IMDB - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "IMDB - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "IMDB - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "IMDB - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IMDB - # prompt tokens": "{\"description\": \"min=1282.797, mean=1897.464, max=2572.797, sum=5692.391 (3)\", \"tab\": \"General information\", \"score\": \"1897.4636666666665\"}", - "IMDB - # output tokens": "{\"description\": \"min=1.928, mean=1.939, max=1.95, sum=5.816 (3)\", \"tab\": \"General information\", \"score\": \"1.9386666666666665\"}", - "IMDB - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "IMDB - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CivilComments", - "source_data": { - "dataset_name": "CivilComments", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on CivilComments", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.008, - "details": { - "description": "min=0, mean=0.008, max=0.344, sum=0.406 (54)", - "tab": "Accuracy", - "CivilComments - ECE (10-bin)": "{\"description\": \"9 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "CivilComments - EM (Robustness)": "{\"description\": \"min=0, mean=0.006, max=0.319, sum=0.347 (54)\", \"tab\": \"Robustness\", \"score\": \"0.006429753618269135\"}", - "CivilComments - EM (Fairness)": "{\"description\": \"min=0, mean=0.006, max=0.288, sum=0.338 (54)\", \"tab\": \"Fairness\", \"score\": \"0.006254555939232581\"}", - "CivilComments - Denoised inference time (s)": "{\"description\": \"9 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "CivilComments - # eval": "{\"description\": \"min=74, mean=371.556, max=683, sum=20064 (54)\", \"tab\": \"General information\", \"score\": \"371.55555555555554\"}", - "CivilComments - # train": "{\"description\": \"min=5, mean=5, max=5, sum=270 (54)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CivilComments - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (54)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CivilComments - # prompt tokens": "{\"description\": \"min=356.537, mean=722.635, max=1267.519, sum=39022.317 (54)\", \"tab\": \"General information\", \"score\": \"722.6354931173206\"}", - "CivilComments - # output tokens": "{\"description\": \"min=0, mean=0.011, max=0.504, sum=0.604 (54)\", \"tab\": \"General information\", \"score\": \"0.011187107057192404\"}", - "CivilComments - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=162 (54)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "CivilComments - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Toxic fraction": "{\"description\": \"9 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "RAFT", - "source_data": { - "dataset_name": "RAFT", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on RAFT", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.701, - "details": { - "description": "min=0, mean=0.701, max=0.975, sum=23.125 (33)", - "tab": "Accuracy", - "RAFT - ECE (10-bin)": "{\"description\": \"11 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "RAFT - EM (Robustness)": "{\"description\": \"min=0, mean=0.677, max=0.975, sum=22.35 (33)\", \"tab\": \"Robustness\", \"score\": \"0.6772727272727272\"}", - "RAFT - EM (Fairness)": "{\"description\": \"min=0, mean=0.672, max=0.975, sum=22.175 (33)\", \"tab\": \"Fairness\", \"score\": \"0.6719696969696969\"}", - "RAFT - Denoised inference time (s)": "{\"description\": \"11 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "RAFT - # eval": "{\"description\": \"min=40, mean=40, max=40, sum=1320 (33)\", \"tab\": \"General information\", \"score\": \"40.0\"}", - "RAFT - # train": "{\"description\": \"min=5, mean=5, max=5, sum=165 (33)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "RAFT - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (33)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "RAFT - # prompt tokens": "{\"description\": \"min=257.35, mean=1279.572, max=6599.65, sum=42225.875 (33)\", \"tab\": \"General information\", \"score\": \"1279.5719696969697\"}", - "RAFT - # output tokens": "{\"description\": \"min=0, mean=3.07, max=6.825, sum=101.3 (33)\", \"tab\": \"General information\", \"score\": \"3.06969696969697\"}", - "RAFT - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=99 (33)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "RAFT - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Toxic fraction": "{\"description\": \"11 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/google_recurrentgemma-2b-it.json b/data/models/google_recurrentgemma-2b-it.json deleted file mode 100644 index 8f565e43d86deeefa9a2258365f5239ee790aa17..0000000000000000000000000000000000000000 --- a/data/models/google_recurrentgemma-2b-it.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "recurrentgemma-2b-it", - "id": "google/recurrentgemma-2b-it", - "developer": "Google", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "RecurrentGemmaForCausalLM", - "params_billions": "2.683" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/google_recurrentgemma-2b-it/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2949 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.333 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0196 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2534 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3341 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1402 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/google_recurrentgemma-2b.json b/data/models/google_recurrentgemma-2b.json deleted file mode 100644 index 0e3f195212d13db250c7edcaf8e388d024666915..0000000000000000000000000000000000000000 --- a/data/models/google_recurrentgemma-2b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "recurrentgemma-2b", - "id": "google/recurrentgemma-2b", - "developer": "Google", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "RecurrentGemmaForCausalLM", - "params_billions": "2.683" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/google_recurrentgemma-2b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3017 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3197 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0204 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2458 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3446 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1176 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/google_recurrentgemma-9b-it.json b/data/models/google_recurrentgemma-9b-it.json deleted file mode 100644 index 7fe6920cdaffdc72314e2b836a2b2eea8e2aceb7..0000000000000000000000000000000000000000 --- a/data/models/google_recurrentgemma-9b-it.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "recurrentgemma-9b-it", - "id": "google/recurrentgemma-9b-it", - "developer": "Google", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "RecurrentGemmaForCausalLM", - "params_billions": "9.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/google_recurrentgemma-9b-it/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.501 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4367 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0665 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2701 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4379 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2843 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/google_recurrentgemma-9b.json b/data/models/google_recurrentgemma-9b.json deleted file mode 100644 index eeff13c066910fa88bd8fa83c4812d127e45463b..0000000000000000000000000000000000000000 --- a/data/models/google_recurrentgemma-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "recurrentgemma-9b", - "id": "google/recurrentgemma-9b", - "developer": "Google", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "RecurrentGemmaForCausalLM", - "params_billions": "9.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/google_recurrentgemma-9b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3116 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3956 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0665 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2852 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3803 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2605 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/google_switch-base-8.json b/data/models/google_switch-base-8.json deleted file mode 100644 index 424cab2cda3dffe41206bc312d3f0374e1c4e7c8..0000000000000000000000000000000000000000 --- a/data/models/google_switch-base-8.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "switch-base-8", - "id": "google/switch-base-8", - "developer": "Google", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "SwitchTransformersForConditionalGeneration", - "params_billions": "0.62" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/google_switch-base-8/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1585 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2876 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.25 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3517 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1098 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/google_t5-11b.json b/data/models/google_t5-11b.json deleted file mode 100644 index 6dbcb6071b1c5f70a11f84651ddefbd208f8a081..0000000000000000000000000000000000000000 --- a/data/models/google_t5-11b.json +++ /dev/null @@ -1,674 +0,0 @@ -{ - "model_info": { - "name": "T5 11B", - "id": "google/T5-11B", - "developer": "Google", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_classic/google_T5-11B/1774096308.339228", - "retrieved_timestamp": "1774096308.339228", - "source_metadata": { - "source_name": "helm_classic", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_classic", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_classic", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperform on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.131, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Calibration": "{\"description\": \"\", \"tab\": \"Calibration\", \"score\": \"0.43469010175763184\"}", - "Mean win rate - Robustness": "{\"description\": \"\", \"tab\": \"Robustness\", \"score\": \"0.16445221445221445\"}", - "Mean win rate - Fairness": "{\"description\": \"\", \"tab\": \"Fairness\", \"score\": \"0.14974358974358976\"}", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.4340277777777778\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}", - "Mean win rate - Bias": "{\"description\": \"\", \"tab\": \"Bias\", \"score\": \"0.4887674914954327\"}", - "Mean win rate - Toxicity": "{\"description\": \"\", \"tab\": \"Toxicity\", \"score\": \"0.5758109174775842\"}", - "Mean win rate - Summarization metrics": "{\"description\": \"\", \"tab\": \"Summarization metrics\", \"score\": \"0.1118421052631579\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.29, - "details": { - "description": "min=0.211, mean=0.29, max=0.4, sum=4.354 (15)", - "tab": "Accuracy", - "MMLU - ECE (10-bin)": "{\"description\": \"min=0.1, mean=0.151, max=0.242, sum=2.271 (15)\", \"tab\": \"Calibration\", \"score\": \"0.1514046561108303\"}", - "MMLU - EM (Robustness)": "{\"description\": \"min=0.19, mean=0.258, max=0.38, sum=3.866 (15)\", \"tab\": \"Robustness\", \"score\": \"0.25776608187134503\"}", - "MMLU - EM (Fairness)": "{\"description\": \"min=0.167, mean=0.235, max=0.33, sum=3.525 (15)\", \"tab\": \"Fairness\", \"score\": \"0.23500584795321638\"}", - "MMLU - Denoised inference time (s)": "{\"description\": \"min=0.173, mean=0.218, max=0.232, sum=3.277 (15)\", \"tab\": \"Efficiency\", \"score\": \"0.21847905223539232\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=1542 (15)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=2.482, mean=4.326, max=5, sum=64.896 (15)\", \"tab\": \"General information\", \"score\": \"4.326397660818714\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (15)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=382.49, mean=420.562, max=467.75, sum=6308.426 (15)\", \"tab\": \"General information\", \"score\": \"420.5617309941521\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=15 (15)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "MMLU - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=45 (15)\", \"tab\": \"General information\", \"score\": \"3.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "BoolQ", - "source_data": { - "dataset_name": "BoolQ", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on BoolQ", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.761, - "details": { - "description": "min=0.732, mean=0.761, max=0.803, sum=2.283 (3)", - "tab": "Accuracy", - "BoolQ - ECE (10-bin)": "{\"description\": \"min=0.348, mean=0.433, max=0.512, sum=1.298 (3)\", \"tab\": \"Calibration\", \"score\": \"0.43269382093398495\"}", - "BoolQ - EM (Robustness)": "{\"description\": \"min=0.624, mean=0.65, max=0.688, sum=1.951 (3)\", \"tab\": \"Robustness\", \"score\": \"0.6503333333333333\"}", - "BoolQ - EM (Fairness)": "{\"description\": \"min=0.697, mean=0.723, max=0.766, sum=2.168 (3)\", \"tab\": \"Fairness\", \"score\": \"0.7226666666666667\"}", - "BoolQ - Denoised inference time (s)": "{\"description\": \"min=0.27, mean=0.271, max=0.272, sum=0.814 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.27128291567197677\"}", - "BoolQ - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "BoolQ - # train": "{\"description\": \"min=0.969, mean=1.588, max=2.006, sum=4.765 (3)\", \"tab\": \"General information\", \"score\": \"1.5883333333333332\"}", - "BoolQ - truncated": "{\"description\": \"min=0.004, mean=0.004, max=0.004, sum=0.012 (3)\", \"tab\": \"General information\", \"score\": \"0.004\"}", - "BoolQ - # prompt tokens": "{\"description\": \"min=386.367, mean=401.944, max=422.649, sum=1205.833 (3)\", \"tab\": \"General information\", \"score\": \"401.94433333333336\"}", - "BoolQ - # output tokens": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "BoolQ - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "BoolQ - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=2 (3)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "BoolQ - Representation (gender)": "{\"description\": \"min=0.125, mean=0.375, max=0.5, sum=1.125 (3)\", \"tab\": \"Bias\", \"score\": \"0.375\"}", - "BoolQ - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.086, - "details": { - "description": "min=0.086, mean=0.086, max=0.086, sum=0.257 (3)", - "tab": "Accuracy", - "NarrativeQA - ECE (10-bin)": "{\"description\": \"min=0.0, mean=0.0, max=0.0, sum=0.0 (3)\", \"tab\": \"Calibration\", \"score\": \"8.06672937578031e-11\"}", - "NarrativeQA - F1 (Robustness)": "{\"description\": \"min=0.045, mean=0.045, max=0.045, sum=0.136 (3)\", \"tab\": \"Robustness\", \"score\": \"0.04518225074755041\"}", - "NarrativeQA - F1 (Fairness)": "{\"description\": \"min=0.05, mean=0.05, max=0.05, sum=0.149 (3)\", \"tab\": \"Fairness\", \"score\": \"0.0497772820026842\"}", - "NarrativeQA - Denoised inference time (s)": "{\"description\": \"min=1.054, mean=1.054, max=1.054, sum=3.163 (3)\", \"tab\": \"Efficiency\", \"score\": \"1.0544504576125933\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=1065 (3)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0.825, mean=0.825, max=0.825, sum=2.476 (3)\", \"tab\": \"General information\", \"score\": \"0.8253521126760562\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=492.141, mean=492.141, max=492.141, sum=1476.423 (3)\", \"tab\": \"General information\", \"score\": \"492.14084507042253\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=100, mean=100, max=100, sum=300 (3)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "NarrativeQA - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NarrativeQA - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=2 (3)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "NarrativeQA - Stereotypes (gender)": "{\"description\": \"min=0.408, mean=0.408, max=0.408, sum=1.225 (3)\", \"tab\": \"Bias\", \"score\": \"0.4081829027907459\"}", - "NarrativeQA - Representation (race)": "{\"description\": \"min=0.367, mean=0.367, max=0.367, sum=1.1 (3)\", \"tab\": \"Bias\", \"score\": \"0.36666666666666664\"}", - "NarrativeQA - Representation (gender)": "{\"description\": \"min=0.156, mean=0.156, max=0.156, sum=0.469 (3)\", \"tab\": \"Bias\", \"score\": \"0.15620542082738947\"}", - "NarrativeQA - Toxic fraction": "{\"description\": \"min=0.011, mean=0.011, max=0.011, sum=0.034 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.011267605633802818\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (open-book)", - "source_data": { - "dataset_name": "NaturalQuestions (open-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (open-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.477, - "details": { - "description": "min=0.278, mean=0.477, max=0.588, sum=1.432 (3)", - "tab": "Accuracy", - "NaturalQuestions (closed-book) - ECE (10-bin)": "{\"description\": \"min=0.07, mean=0.076, max=0.082, sum=0.228 (3)\", \"tab\": \"Calibration\", \"score\": \"0.07599999619350188\"}", - "NaturalQuestions (open-book) - ECE (10-bin)": "{\"description\": \"min=0.051, mean=0.239, max=0.356, sum=0.717 (3)\", \"tab\": \"Calibration\", \"score\": \"0.23900003883193166\"}", - "NaturalQuestions (closed-book) - F1 (Robustness)": "{\"description\": \"min=0.146, mean=0.153, max=0.159, sum=0.458 (3)\", \"tab\": \"Robustness\", \"score\": \"0.15251804391476487\"}", - "NaturalQuestions (open-book) - F1 (Robustness)": "{\"description\": \"min=0.047, mean=0.071, max=0.107, sum=0.213 (3)\", \"tab\": \"Robustness\", \"score\": \"0.0710016541484974\"}", - "NaturalQuestions (closed-book) - F1 (Fairness)": "{\"description\": \"min=0.152, mean=0.159, max=0.164, sum=0.476 (3)\", \"tab\": \"Fairness\", \"score\": \"0.15857963279707157\"}", - "NaturalQuestions (open-book) - F1 (Fairness)": "{\"description\": \"min=0.227, mean=0.424, max=0.532, sum=1.271 (3)\", \"tab\": \"Fairness\", \"score\": \"0.42376820534695847\"}", - "NaturalQuestions (closed-book) - Denoised inference time (s)": "{\"description\": \"min=2.617, mean=2.856, max=3.211, sum=8.569 (3)\", \"tab\": \"Efficiency\", \"score\": \"2.856322434252687\"}", - "NaturalQuestions (open-book) - Denoised inference time (s)": "{\"description\": \"min=6.926, mean=12.846, max=24.675, sum=38.539 (3)\", \"tab\": \"Efficiency\", \"score\": \"12.84636455836454\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=109.556, mean=113.556, max=118.556, sum=340.668 (3)\", \"tab\": \"General information\", \"score\": \"113.556\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=300, mean=300, max=300, sum=900 (3)\", \"tab\": \"General information\", \"score\": \"300.0\"}", - "NaturalQuestions (closed-book) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=0.096, mean=0.924, max=1.792, sum=2.771 (3)\", \"tab\": \"General information\", \"score\": \"0.9236666666666666\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.094, mean=0.349, max=0.839, sum=1.048 (3)\", \"tab\": \"General information\", \"score\": \"0.34933333333333333\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=233.452, mean=301.907, max=339.767, sum=905.721 (3)\", \"tab\": \"General information\", \"score\": \"301.907\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=300, mean=300, max=300, sum=900 (3)\", \"tab\": \"General information\", \"score\": \"300.0\"}", - "NaturalQuestions (open-book) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NaturalQuestions (closed-book) - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - Stereotypes (gender)": "{\"description\": \"min=0.5, mean=0.5, max=0.5, sum=1.5 (3)\", \"tab\": \"Bias\", \"score\": \"0.5\"}", - "NaturalQuestions (closed-book) - Representation (race)": "{\"description\": \"min=0.529, mean=0.533, max=0.535, sum=1.6 (3)\", \"tab\": \"Bias\", \"score\": \"0.5332530194915516\"}", - "NaturalQuestions (closed-book) - Representation (gender)": "{\"description\": \"min=0.071, mean=0.103, max=0.125, sum=0.308 (3)\", \"tab\": \"Bias\", \"score\": \"0.10251322751322754\"}", - "NaturalQuestions (open-book) - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=2 (3)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "NaturalQuestions (open-book) - Stereotypes (gender)": "{\"description\": \"min=0.389, mean=0.417, max=0.472, sum=1.25 (3)\", \"tab\": \"Bias\", \"score\": \"0.4166666666666666\"}", - "NaturalQuestions (open-book) - Representation (race)": "{\"description\": \"min=0.483, mean=0.516, max=0.552, sum=1.549 (3)\", \"tab\": \"Bias\", \"score\": \"0.5163891020108681\"}", - "NaturalQuestions (open-book) - Representation (gender)": "{\"description\": \"min=0.218, mean=0.243, max=0.26, sum=0.728 (3)\", \"tab\": \"Bias\", \"score\": \"0.24276995305164317\"}", - "NaturalQuestions (closed-book) - Toxic fraction": "{\"description\": \"min=0.001, mean=0.001, max=0.001, sum=0.003 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.001\"}", - "NaturalQuestions (open-book) - Toxic fraction": "{\"description\": \"min=0, mean=0.001, max=0.002, sum=0.003 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.001\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "QuAC", - "source_data": { - "dataset_name": "QuAC", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on QuAC", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.116, - "details": { - "description": "min=0.116, mean=0.116, max=0.116, sum=0.348 (3)", - "tab": "Accuracy", - "QuAC - ECE (10-bin)": "{\"description\": \"min=0.0, mean=0.0, max=0.0, sum=0.0 (3)\", \"tab\": \"Calibration\", \"score\": \"1.908717030577995e-09\"}", - "QuAC - F1 (Robustness)": "{\"description\": \"min=0.064, mean=0.064, max=0.064, sum=0.191 (3)\", \"tab\": \"Robustness\", \"score\": \"0.06378325242260692\"}", - "QuAC - F1 (Fairness)": "{\"description\": \"min=0.074, mean=0.074, max=0.074, sum=0.221 (3)\", \"tab\": \"Fairness\", \"score\": \"0.07376443691909672\"}", - "QuAC - Denoised inference time (s)": "{\"description\": \"min=1.032, mean=1.032, max=1.032, sum=3.097 (3)\", \"tab\": \"Efficiency\", \"score\": \"1.0323945961168868\"}", - "QuAC - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "QuAC - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "QuAC - truncated": "{\"description\": \"min=0.999, mean=0.999, max=0.999, sum=2.997 (3)\", \"tab\": \"General information\", \"score\": \"0.999\"}", - "QuAC - # prompt tokens": "{\"description\": \"min=510.923, mean=510.923, max=510.923, sum=1532.769 (3)\", \"tab\": \"General information\", \"score\": \"510.923\"}", - "QuAC - # output tokens": "{\"description\": \"min=100, mean=100, max=100, sum=300 (3)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "QuAC - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "QuAC - Stereotypes (race)": "{\"description\": \"min=0.65, mean=0.65, max=0.65, sum=1.949 (3)\", \"tab\": \"Bias\", \"score\": \"0.6495726495726497\"}", - "QuAC - Stereotypes (gender)": "{\"description\": \"min=0.44, mean=0.44, max=0.44, sum=1.32 (3)\", \"tab\": \"Bias\", \"score\": \"0.4400900674211062\"}", - "QuAC - Representation (race)": "{\"description\": \"min=0.397, mean=0.397, max=0.397, sum=1.192 (3)\", \"tab\": \"Bias\", \"score\": \"0.39717891610987377\"}", - "QuAC - Representation (gender)": "{\"description\": \"min=0.257, mean=0.257, max=0.257, sum=0.771 (3)\", \"tab\": \"Bias\", \"score\": \"0.25702629193109705\"}", - "QuAC - Toxic fraction": "{\"description\": \"min=0.002, mean=0.002, max=0.002, sum=0.006 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.002\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "HellaSwag", - "source_data": { - "dataset_name": "HellaSwag", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on HellaSwag", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "HellaSwag - ECE (10-bin)": "{\"description\": \"No matching runs\", \"tab\": \"Calibration\", \"score\": \"\"}", - "HellaSwag - EM (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "HellaSwag - EM (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "HellaSwag - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "HellaSwag - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "OpenbookQA - ECE (10-bin)": "{\"description\": \"No matching runs\", \"tab\": \"Calibration\", \"score\": \"\"}", - "OpenbookQA - EM (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "OpenbookQA - EM (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "OpenbookQA - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "OpenbookQA - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "TruthfulQA", - "source_data": { - "dataset_name": "TruthfulQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on TruthfulQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.133, - "details": { - "description": "min=0.104, mean=0.133, max=0.15, sum=0.532 (4)", - "tab": "Accuracy", - "TruthfulQA - ECE (10-bin)": "{\"description\": \"min=0.109, mean=0.143, max=0.195, sum=0.574 (4)\", \"tab\": \"Calibration\", \"score\": \"0.1434693835940009\"}", - "TruthfulQA - EM (Robustness)": "{\"description\": \"min=0.09, mean=0.122, max=0.148, sum=0.489 (4)\", \"tab\": \"Robustness\", \"score\": \"0.12232415902140673\"}", - "TruthfulQA - EM (Fairness)": "{\"description\": \"min=0.058, mean=0.101, max=0.136, sum=0.405 (4)\", \"tab\": \"Fairness\", \"score\": \"0.10129969418960244\"}", - "TruthfulQA - Denoised inference time (s)": "{\"description\": \"min=0.174, mean=0.21, max=0.249, sum=0.838 (4)\", \"tab\": \"Efficiency\", \"score\": \"0.2095953345265857\"}", - "TruthfulQA - # eval": "{\"description\": \"min=654, mean=654, max=654, sum=2616 (4)\", \"tab\": \"General information\", \"score\": \"654.0\"}", - "TruthfulQA - # train": "{\"description\": \"min=0, mean=3.547, max=4.869, sum=14.19 (4)\", \"tab\": \"General information\", \"score\": \"3.547400611620795\"}", - "TruthfulQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (4)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "TruthfulQA - # prompt tokens": "{\"description\": \"min=85.896, mean=371.92, max=471.52, sum=1487.679 (4)\", \"tab\": \"General information\", \"score\": \"371.9197247706422\"}", - "TruthfulQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=4 (4)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "TruthfulQA - # trials": "{\"description\": \"min=1, mean=2.5, max=3, sum=10 (4)\", \"tab\": \"General information\", \"score\": \"2.5\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MS MARCO (TREC)", - "source_data": { - "dataset_name": "MS MARCO (TREC)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "NDCG@10 on MS MARCO (TREC)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "MS MARCO (regular) - RR@10 (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "MS MARCO (TREC) - NDCG@10 (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "MS MARCO (regular) - RR@10 (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "MS MARCO (TREC) - NDCG@10 (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "MS MARCO (regular) - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MS MARCO (TREC) - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MS MARCO (regular) - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "MS MARCO (TREC) - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CNN/DailyMail", - "source_data": { - "dataset_name": "CNN/DailyMail", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on CNN/DailyMail", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.043, - "details": { - "description": "min=0.043, mean=0.043, max=0.043, sum=0.257 (6)", - "tab": "Accuracy", - "CNN/DailyMail - Denoised inference time (s)": "{\"description\": \"min=1.653, mean=1.654, max=1.655, sum=9.926 (6)\", \"tab\": \"Efficiency\", \"score\": \"1.6543884711070522\"}", - "CNN/DailyMail - # eval": "{\"description\": \"min=466, mean=466, max=466, sum=2796 (6)\", \"tab\": \"General information\", \"score\": \"466.0\"}", - "CNN/DailyMail - # train": "{\"description\": \"min=0.062, mean=0.064, max=0.067, sum=0.382 (6)\", \"tab\": \"General information\", \"score\": \"0.06366237482117311\"}", - "CNN/DailyMail - truncated": "{\"description\": \"min=0.929, mean=0.932, max=0.933, sum=5.592 (6)\", \"tab\": \"General information\", \"score\": \"0.9320457796852647\"}", - "CNN/DailyMail - # prompt tokens": "{\"description\": \"min=500.412, mean=500.553, max=500.835, sum=3003.318 (6)\", \"tab\": \"General information\", \"score\": \"500.5529327610873\"}", - "CNN/DailyMail - # output tokens": "{\"description\": \"min=128, mean=128, max=128, sum=768 (6)\", \"tab\": \"General information\", \"score\": \"128.0\"}", - "CNN/DailyMail - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=18 (6)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "CNN/DailyMail - Stereotypes (race)": "{\"description\": \"min=0.632, mean=0.632, max=0.632, sum=3.789 (6)\", \"tab\": \"Bias\", \"score\": \"0.631578947368421\"}", - "CNN/DailyMail - Stereotypes (gender)": "{\"description\": \"min=0.452, mean=0.452, max=0.452, sum=2.709 (6)\", \"tab\": \"Bias\", \"score\": \"0.4515726043503821\"}", - "CNN/DailyMail - Representation (race)": "{\"description\": \"min=0.264, mean=0.264, max=0.264, sum=1.581 (6)\", \"tab\": \"Bias\", \"score\": \"0.26356589147286824\"}", - "CNN/DailyMail - Representation (gender)": "{\"description\": \"min=0.119, mean=0.119, max=0.12, sum=0.713 (6)\", \"tab\": \"Bias\", \"score\": \"0.11890102842483792\"}", - "CNN/DailyMail - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (6)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}", - "CNN/DailyMail - SummaC": "{\"description\": \"min=-0.125, mean=-0.122, max=-0.117, sum=-0.365 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"-0.12151602946968616\"}", - "CNN/DailyMail - QAFactEval": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - BERTScore (F1)": "{\"description\": \"min=-0.173, mean=-0.17, max=-0.165, sum=-0.509 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"-0.16977369097758946\"}", - "CNN/DailyMail - Coverage": "{\"description\": \"min=0.55, mean=0.555, max=0.56, sum=3.329 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"0.5547542182286073\"}", - "CNN/DailyMail - Density": "{\"description\": \"min=2.69, mean=2.698, max=2.706, sum=16.19 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"2.698337926712314\"}", - "CNN/DailyMail - Compression": "{\"description\": \"min=19.085, mean=19.248, max=19.44, sum=115.49 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"19.248383205041776\"}", - "CNN/DailyMail - HumanEval-faithfulness": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-relevance": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-coherence": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "XSUM", - "source_data": { - "dataset_name": "XSUM", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on XSUM", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.015, - "details": { - "description": "min=0.008, mean=0.015, max=0.018, sum=0.087 (6)", - "tab": "Accuracy", - "XSUM - Denoised inference time (s)": "{\"description\": \"min=1.096, mean=1.159, max=1.283, sum=6.953 (6)\", \"tab\": \"Efficiency\", \"score\": \"1.15883249730996\"}", - "XSUM - # eval": "{\"description\": \"min=518, mean=518, max=518, sum=3108 (6)\", \"tab\": \"General information\", \"score\": \"518.0\"}", - "XSUM - # train": "{\"description\": \"min=0.239, mean=0.3, max=0.373, sum=1.799 (6)\", \"tab\": \"General information\", \"score\": \"0.29987129987129985\"}", - "XSUM - truncated": "{\"description\": \"min=0.602, mean=0.671, max=0.73, sum=4.023 (6)\", \"tab\": \"General information\", \"score\": \"0.6705276705276706\"}", - "XSUM - # prompt tokens": "{\"description\": \"min=432.851, mean=436.826, max=442.064, sum=2620.958 (6)\", \"tab\": \"General information\", \"score\": \"436.8262548262548\"}", - "XSUM - # output tokens": "{\"description\": \"min=64, mean=64, max=64, sum=384 (6)\", \"tab\": \"General information\", \"score\": \"64.0\"}", - "XSUM - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=18 (6)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "XSUM - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=2.667 (4)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666667\"}", - "XSUM - Stereotypes (gender)": "{\"description\": \"min=0.5, mean=0.5, max=0.5, sum=3 (6)\", \"tab\": \"Bias\", \"score\": \"0.5\"}", - "XSUM - Representation (race)": "{\"description\": \"min=0.333, mean=0.358, max=0.394, sum=2.15 (6)\", \"tab\": \"Bias\", \"score\": \"0.3582634859230604\"}", - "XSUM - Representation (gender)": "{\"description\": \"min=0.214, mean=0.222, max=0.231, sum=1.332 (6)\", \"tab\": \"Bias\", \"score\": \"0.2219358310118288\"}", - "XSUM - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (6)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}", - "XSUM - SummaC": "{\"description\": \"min=-0.267, mean=-0.258, max=-0.244, sum=-0.775 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"-0.2584302846171323\"}", - "XSUM - QAFactEval": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - BERTScore (F1)": "{\"description\": \"min=-0.379, mean=-0.315, max=-0.276, sum=-0.944 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"-0.3147063674770794\"}", - "XSUM - Coverage": "{\"description\": \"min=0.324, mean=0.355, max=0.372, sum=2.133 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"0.3554524422801694\"}", - "XSUM - Density": "{\"description\": \"min=0.763, mean=0.831, max=0.866, sum=4.987 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"0.831154946558878\"}", - "XSUM - Compression": "{\"description\": \"min=16.29, mean=16.544, max=16.714, sum=99.261 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"16.543527805806836\"}", - "XSUM - HumanEval-faithfulness": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-relevance": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-coherence": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "IMDB", - "source_data": { - "dataset_name": "IMDB", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on IMDB", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.379, - "details": { - "description": "min=0.248, mean=0.379, max=0.568, sum=1.137 (3)", - "tab": "Accuracy", - "IMDB - ECE (10-bin)": "{\"description\": \"min=0.108, mean=0.236, max=0.374, sum=0.707 (3)\", \"tab\": \"Calibration\", \"score\": \"0.23573461605966659\"}", - "IMDB - EM (Robustness)": "{\"description\": \"min=0.17, mean=0.304, max=0.51, sum=0.911 (3)\", \"tab\": \"Robustness\", \"score\": \"0.3036666666666667\"}", - "IMDB - EM (Fairness)": "{\"description\": \"min=0.162, mean=0.303, max=0.502, sum=0.91 (3)\", \"tab\": \"Fairness\", \"score\": \"0.30333333333333334\"}", - "IMDB - Denoised inference time (s)": "{\"description\": \"min=0.276, mean=0.278, max=0.28, sum=0.834 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.27797461745258367\"}", - "IMDB - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "IMDB - # train": "{\"description\": \"min=0.33, mean=0.466, max=0.701, sum=1.397 (3)\", \"tab\": \"General information\", \"score\": \"0.4656666666666666\"}", - "IMDB - truncated": "{\"description\": \"min=0.172, mean=0.173, max=0.173, sum=0.518 (3)\", \"tab\": \"General information\", \"score\": \"0.17266666666666666\"}", - "IMDB - # prompt tokens": "{\"description\": \"min=391.442, mean=408.425, max=434.668, sum=1225.274 (3)\", \"tab\": \"General information\", \"score\": \"408.4246666666666\"}", - "IMDB - # output tokens": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "IMDB - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "IMDB - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CivilComments", - "source_data": { - "dataset_name": "CivilComments", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on CivilComments", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.509, - "details": { - "description": "min=0, mean=0.509, max=0.998, sum=27.462 (54)", - "tab": "Accuracy", - "CivilComments - ECE (10-bin)": "{\"description\": \"min=0.108, mean=0.38, max=0.553, sum=20.519 (54)\", \"tab\": \"Calibration\", \"score\": \"0.3799801119037254\"}", - "CivilComments - EM (Robustness)": "{\"description\": \"min=0, mean=0.392, max=0.991, sum=21.175 (54)\", \"tab\": \"Robustness\", \"score\": \"0.39212772273586344\"}", - "CivilComments - EM (Fairness)": "{\"description\": \"min=0, mean=0.329, max=0.991, sum=17.759 (54)\", \"tab\": \"Fairness\", \"score\": \"0.32887358622117774\"}", - "CivilComments - Denoised inference time (s)": "{\"description\": \"min=0.269, mean=0.27, max=0.273, sum=14.596 (54)\", \"tab\": \"Efficiency\", \"score\": \"0.27030228534077655\"}", - "CivilComments - # eval": "{\"description\": \"min=74, mean=371.556, max=683, sum=20064 (54)\", \"tab\": \"General information\", \"score\": \"371.55555555555554\"}", - "CivilComments - # train": "{\"description\": \"min=1.019, mean=2.636, max=4.881, sum=142.352 (54)\", \"tab\": \"General information\", \"score\": \"2.6361556323380086\"}", - "CivilComments - truncated": "{\"description\": \"min=0, mean=0.002, max=0.022, sum=0.094 (54)\", \"tab\": \"General information\", \"score\": \"0.0017482982997674094\"}", - "CivilComments - # prompt tokens": "{\"description\": \"min=331.768, mean=416.791, max=477.628, sum=22506.741 (54)\", \"tab\": \"General information\", \"score\": \"416.79149386044713\"}", - "CivilComments - # output tokens": "{\"description\": \"min=5, mean=5, max=5, sum=270 (54)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CivilComments - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=162 (54)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "CivilComments - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (54)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "RAFT", - "source_data": { - "dataset_name": "RAFT", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on RAFT", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.37, - "details": { - "description": "min=0, mean=0.37, max=0.925, sum=12.2 (33)", - "tab": "Accuracy", - "RAFT - ECE (10-bin)": "{\"description\": \"min=0.0, mean=0.367, max=0.925, sum=12.1 (33)\", \"tab\": \"Calibration\", \"score\": \"0.36667176546312147\"}", - "RAFT - EM (Robustness)": "{\"description\": \"min=0, mean=0.331, max=0.875, sum=10.925 (33)\", \"tab\": \"Robustness\", \"score\": \"0.33106060606060606\"}", - "RAFT - EM (Fairness)": "{\"description\": \"min=0, mean=0.351, max=0.85, sum=11.575 (33)\", \"tab\": \"Fairness\", \"score\": \"0.3507575757575757\"}", - "RAFT - Denoised inference time (s)": "{\"description\": \"min=0.411, mean=0.448, max=0.835, sum=14.799 (33)\", \"tab\": \"Efficiency\", \"score\": \"0.4484652494441787\"}", - "RAFT - # eval": "{\"description\": \"min=40, mean=40, max=40, sum=1320 (33)\", \"tab\": \"General information\", \"score\": \"40.0\"}", - "RAFT - # train": "{\"description\": \"min=0, mean=2.433, max=5, sum=80.3 (33)\", \"tab\": \"General information\", \"score\": \"2.433333333333333\"}", - "RAFT - truncated": "{\"description\": \"min=0, mean=0.394, max=1, sum=13 (33)\", \"tab\": \"General information\", \"score\": \"0.3939393939393939\"}", - "RAFT - # prompt tokens": "{\"description\": \"min=263.4, mean=420.742, max=511, sum=13884.475 (33)\", \"tab\": \"General information\", \"score\": \"420.7416666666667\"}", - "RAFT - # output tokens": "{\"description\": \"min=30, mean=30, max=30, sum=990 (33)\", \"tab\": \"General information\", \"score\": \"30.0\"}", - "RAFT - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=99 (33)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "RAFT - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (gender)": "{\"description\": \"min=0.5, mean=0.5, max=0.5, sum=1.5 (3)\", \"tab\": \"Bias\", \"score\": \"0.5\"}", - "RAFT - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (33)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/google_text-bison_001.json b/data/models/google_text-bison_001.json deleted file mode 100644 index b111549ae96e5e33c5682568429a8555b5729b91..0000000000000000000000000000000000000000 --- a/data/models/google_text-bison_001.json +++ /dev/null @@ -1,1901 +0,0 @@ -{ - "model_info": { - "name": "PaLM-2 Bison", - "id": "google/text-bison@001", - "developer": "Google", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_lite/google_text-bison@001/1774096306.427425", - "retrieved_timestamp": "1774096306.427425", - "source_metadata": { - "source_name": "helm_lite", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_lite", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_lite", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.526, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.47540574282147313\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.718, - "details": { - "description": "min=0.718, mean=0.718, max=0.718, sum=0.718 (1)", - "tab": "Accuracy", - "NarrativeQA - Observed inference time (s)": "{\"description\": \"min=1.031, mean=1.031, max=1.031, sum=1.031 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.030712524602111\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=4414.234, mean=4414.234, max=4414.234, sum=4414.234 (1)\", \"tab\": \"General information\", \"score\": \"4414.2338028169015\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=7.997, mean=7.997, max=7.997, sum=7.997 (1)\", \"tab\": \"General information\", \"score\": \"7.997183098591549\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (closed-book)", - "source_data": { - "dataset_name": "NaturalQuestions (closed-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (closed-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.39, - "details": { - "description": "min=0.39, mean=0.39, max=0.39, sum=0.39 (1)", - "tab": "Accuracy", - "NaturalQuestions (open-book) - Observed inference time (s)": "{\"description\": \"min=0.987, mean=0.987, max=0.987, sum=0.987 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.987217092037201\"}", - "NaturalQuestions (closed-book) - Observed inference time (s)": "{\"description\": \"min=0.755, mean=0.755, max=0.755, sum=0.755 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.754590849161148\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.906, mean=4.906, max=4.906, sum=4.906 (1)\", \"tab\": \"General information\", \"score\": \"4.906\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.015, mean=0.015, max=0.015, sum=0.015 (1)\", \"tab\": \"General information\", \"score\": \"0.015\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=2124.565, mean=2124.565, max=2124.565, sum=2124.565 (1)\", \"tab\": \"General information\", \"score\": \"2124.565\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=7.358, mean=7.358, max=7.358, sum=7.358 (1)\", \"tab\": \"General information\", \"score\": \"7.358\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=190.187, mean=190.187, max=190.187, sum=190.187 (1)\", \"tab\": \"General information\", \"score\": \"190.187\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=4.48, mean=4.48, max=4.48, sum=4.48 (1)\", \"tab\": \"General information\", \"score\": \"4.48\"}" - } - }, - "generation_config": { - "additional_details": { - "mode": "\"closedbook\"" - } - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.878, - "details": { - "description": "min=0.878, mean=0.878, max=0.878, sum=0.878 (1)", - "tab": "Accuracy", - "OpenbookQA - Observed inference time (s)": "{\"description\": \"min=0.788, mean=0.788, max=0.788, sum=0.788 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.7879144654273987\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=253.308, mean=253.308, max=253.308, sum=253.308 (1)\", \"tab\": \"General information\", \"score\": \"253.308\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "dataset": "\"openbookqa\"", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.608, - "details": { - "description": "min=0.39, mean=0.608, max=0.87, sum=3.038 (5)", - "tab": "Accuracy", - "MMLU - Observed inference time (s)": "{\"description\": \"min=1.017, mean=1.112, max=1.352, sum=5.561 (5)\", \"tab\": \"Efficiency\", \"score\": \"1.1122005350882547\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=360.7, mean=487.294, max=638.088, sum=2436.468 (5)\", \"tab\": \"General information\", \"score\": \"487.29354385964905\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"college_chemistry\", \"computer_security\", \"econometrics\", \"us_foreign_policy\"]", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MATH", - "source_data": { - "dataset_name": "MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Equivalent (CoT) on MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.421, - "details": { - "description": "min=0.25, mean=0.421, max=0.558, sum=2.946 (7)", - "tab": "Accuracy", - "MATH - Observed inference time (s)": "{\"description\": \"min=1.161, mean=1.614, max=2.126, sum=11.299 (7)\", \"tab\": \"Efficiency\", \"score\": \"1.6140828338918989\"}", - "MATH - # eval": "{\"description\": \"min=30, mean=62.429, max=135, sum=437 (7)\", \"tab\": \"General information\", \"score\": \"62.42857142857143\"}", - "MATH - # train": "{\"description\": \"min=8, mean=8, max=8, sum=56 (7)\", \"tab\": \"General information\", \"score\": \"8.0\"}", - "MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MATH - # prompt tokens": "{\"description\": \"min=1004.274, mean=1439.843, max=2386.942, sum=10078.901 (7)\", \"tab\": \"General information\", \"score\": \"1439.842989280994\"}", - "MATH - # output tokens": "{\"description\": \"min=38.4, mean=66.89, max=88.316, sum=468.232 (7)\", \"tab\": \"General information\", \"score\": \"66.89023408252294\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"algebra\", \"counting_and_probability\", \"geometry\", \"intermediate_algebra\", \"number_theory\", \"prealgebra\", \"precalculus\"]", - "level": "\"1\"", - "use_official_examples": "\"False\"", - "use_chain_of_thought": "\"True\"" - } - } - }, - { - "evaluation_name": "GSM8K", - "source_data": { - "dataset_name": "GSM8K", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on GSM8K", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.61, - "details": { - "description": "min=0.61, mean=0.61, max=0.61, sum=0.61 (1)", - "tab": "Accuracy", - "GSM8K - Observed inference time (s)": "{\"description\": \"min=1.44, mean=1.44, max=1.44, sum=1.44 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.4403084371089936\"}", - "GSM8K - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "GSM8K - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "GSM8K - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GSM8K - # prompt tokens": "{\"description\": \"min=1109.549, mean=1109.549, max=1109.549, sum=1109.549 (1)\", \"tab\": \"General information\", \"score\": \"1109.549\"}", - "GSM8K - # output tokens": "{\"description\": \"min=94.258, mean=94.258, max=94.258, sum=94.258 (1)\", \"tab\": \"General information\", \"score\": \"94.258\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "LegalBench", - "source_data": { - "dataset_name": "LegalBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on LegalBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.645, - "details": { - "description": "min=0.466, mean=0.645, max=0.937, sum=3.224 (5)", - "tab": "Accuracy", - "LegalBench - Observed inference time (s)": "{\"description\": \"min=0.53, mean=0.737, max=1.325, sum=3.683 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.7366328867537384\"}", - "LegalBench - # eval": "{\"description\": \"min=95, mean=409.4, max=1000, sum=2047 (5)\", \"tab\": \"General information\", \"score\": \"409.4\"}", - "LegalBench - # train": "{\"description\": \"min=2.988, mean=4.398, max=5, sum=21.988 (5)\", \"tab\": \"General information\", \"score\": \"4.397551020408163\"}", - "LegalBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "LegalBench - # prompt tokens": "{\"description\": \"min=287.432, mean=1387.966, max=5134.504, sum=6939.831 (5)\", \"tab\": \"General information\", \"score\": \"1387.966233478402\"}", - "LegalBench - # output tokens": "{\"description\": \"min=1, mean=1.389, max=2.347, sum=6.947 (5)\", \"tab\": \"General information\", \"score\": \"1.3893499784884555\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"abercrombie\", \"corporate_lobbying\", \"function_of_decision_section\", \"international_citizenship_questions\", \"proa\"]" - } - } - }, - { - "evaluation_name": "MedQA", - "source_data": { - "dataset_name": "MedQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MedQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.547, - "details": { - "description": "min=0.547, mean=0.547, max=0.547, sum=0.547 (1)", - "tab": "Accuracy", - "MedQA - Observed inference time (s)": "{\"description\": \"min=0.735, mean=0.735, max=0.735, sum=0.735 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.7348999071784806\"}", - "MedQA - # eval": "{\"description\": \"min=503, mean=503, max=503, sum=503 (1)\", \"tab\": \"General information\", \"score\": \"503.0\"}", - "MedQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MedQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MedQA - # prompt tokens": "{\"description\": \"min=1138.622, mean=1138.622, max=1138.622, sum=1138.622 (1)\", \"tab\": \"General information\", \"score\": \"1138.6222664015904\"}", - "MedQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WMT 2014", - "source_data": { - "dataset_name": "WMT 2014", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "BLEU-4 on WMT 2014", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.241, - "details": { - "description": "min=0.22, mean=0.241, max=0.255, sum=1.204 (5)", - "tab": "Accuracy", - "WMT 2014 - Observed inference time (s)": "{\"description\": \"min=0.826, mean=0.875, max=0.952, sum=4.377 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.8753595397700126\"}", - "WMT 2014 - # eval": "{\"description\": \"min=503, mean=568.8, max=832, sum=2844 (5)\", \"tab\": \"General information\", \"score\": \"568.8\"}", - "WMT 2014 - # train": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "WMT 2014 - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WMT 2014 - # prompt tokens": "{\"description\": \"min=145.755, mean=183.587, max=206.169, sum=917.936 (5)\", \"tab\": \"General information\", \"score\": \"183.58714444104604\"}", - "WMT 2014 - # output tokens": "{\"description\": \"min=28.076, mean=29.981, max=31.366, sum=149.905 (5)\", \"tab\": \"General information\", \"score\": \"29.980943664933477\"}" - } - }, - "generation_config": { - "additional_details": { - "language_pair": "[\"cs-en\", \"de-en\", \"fr-en\", \"hi-en\", \"ru-en\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_mmlu/google_text-bison@001/1774096312.00548", - "retrieved_timestamp": "1774096312.00548", - "source_metadata": { - "source_name": "helm_mmlu", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_mmlu", - "evaluation_results": [ - { - "evaluation_name": "MMLU All Subjects", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU All Subjects", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.692, - "details": { - "description": "min=0.331, mean=0.692, max=0.927, sum=78.899 (114)", - "tab": "Accuracy", - "MMLU All Subjects - Observed inference time (s)": "{\"description\": \"min=0.619, mean=1.845, max=23.541, sum=210.314 (114)\", \"tab\": \"Efficiency\", \"score\": \"1.8448593983042894\"}", - "MMLU All Subjects - # eval": "{\"description\": \"min=100, mean=246.351, max=1534, sum=28084 (114)\", \"tab\": \"General information\", \"score\": \"246.35087719298247\"}", - "MMLU All Subjects - # train": "{\"description\": \"min=5, mean=5, max=5, sum=570 (114)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU All Subjects - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU All Subjects - # prompt tokens": "{\"description\": \"min=270.187, mean=635.61, max=2823.23, sum=72459.527 (114)\", \"tab\": \"General information\", \"score\": \"635.6098850770794\"}", - "MMLU All Subjects - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=114 (114)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_name": "Abstract Algebra", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Abstract Algebra", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.39, - "details": { - "description": "min=0.39, mean=0.39, max=0.39, sum=0.78 (2)", - "tab": "Accuracy", - "Abstract Algebra - Observed inference time (s)": "{\"description\": \"min=1.017, mean=1.017, max=1.017, sum=2.033 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.0166235256195069\"}", - "Abstract Algebra - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Abstract Algebra - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Abstract Algebra - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Abstract Algebra - # prompt tokens": "{\"description\": \"min=387.12, mean=387.12, max=387.12, sum=774.24 (2)\", \"tab\": \"General information\", \"score\": \"387.12\"}", - "Abstract Algebra - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"abstract_algebra\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_abstract_algebra\"" - } - } - }, - { - "evaluation_name": "Anatomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Anatomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.644, - "details": { - "description": "min=0.644, mean=0.644, max=0.644, sum=1.289 (2)", - "tab": "Accuracy", - "Anatomy - Observed inference time (s)": "{\"description\": \"min=0.837, mean=0.837, max=0.837, sum=1.673 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.836542272567749\"}", - "Anatomy - # eval": "{\"description\": \"min=135, mean=135, max=135, sum=270 (2)\", \"tab\": \"General information\", \"score\": \"135.0\"}", - "Anatomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Anatomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Anatomy - # prompt tokens": "{\"description\": \"min=344.089, mean=344.089, max=344.089, sum=688.178 (2)\", \"tab\": \"General information\", \"score\": \"344.0888888888889\"}", - "Anatomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"anatomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_anatomy\"" - } - } - }, - { - "evaluation_name": "College Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on College Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.51, - "details": { - "description": "min=0.51, mean=0.51, max=0.51, sum=1.02 (2)", - "tab": "Accuracy", - "College Chemistry - Observed inference time (s)": "{\"description\": \"min=1.352, mean=1.352, max=1.352, sum=2.704 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.3518596124649047\"}", - "College Biology - Observed inference time (s)": "{\"description\": \"min=0.862, mean=0.862, max=0.862, sum=1.724 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8619864102866914\"}", - "College Computer Science - Observed inference time (s)": "{\"description\": \"min=23.541, mean=23.541, max=23.541, sum=47.082 (2)\", \"tab\": \"Efficiency\", \"score\": \"23.54095259666443\"}", - "College Mathematics - Observed inference time (s)": "{\"description\": \"min=0.931, mean=0.931, max=0.931, sum=1.862 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9307789158821106\"}", - "College Medicine - Observed inference time (s)": "{\"description\": \"min=0.947, mean=0.947, max=0.947, sum=1.894 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9472322174579422\"}", - "College Physics - Observed inference time (s)": "{\"description\": \"min=0.928, mean=0.928, max=0.928, sum=1.856 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9281005485385072\"}", - "College Chemistry - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Chemistry - # prompt tokens": "{\"description\": \"min=573.7, mean=573.7, max=573.7, sum=1147.4 (2)\", \"tab\": \"General information\", \"score\": \"573.7\"}", - "College Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Biology - # eval": "{\"description\": \"min=144, mean=144, max=144, sum=288 (2)\", \"tab\": \"General information\", \"score\": \"144.0\"}", - "College Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # prompt tokens": "{\"description\": \"min=480.875, mean=480.875, max=480.875, sum=961.75 (2)\", \"tab\": \"General information\", \"score\": \"480.875\"}", - "College Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # prompt tokens": "{\"description\": \"min=861.96, mean=861.96, max=861.96, sum=1723.92 (2)\", \"tab\": \"General information\", \"score\": \"861.96\"}", - "College Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Mathematics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # prompt tokens": "{\"description\": \"min=636.94, mean=636.94, max=636.94, sum=1273.88 (2)\", \"tab\": \"General information\", \"score\": \"636.94\"}", - "College Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Medicine - # eval": "{\"description\": \"min=173, mean=173, max=173, sum=346 (2)\", \"tab\": \"General information\", \"score\": \"173.0\"}", - "College Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # prompt tokens": "{\"description\": \"min=512.584, mean=512.584, max=512.584, sum=1025.168 (2)\", \"tab\": \"General information\", \"score\": \"512.5838150289018\"}", - "College Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Physics - # eval": "{\"description\": \"min=102, mean=102, max=102, sum=204 (2)\", \"tab\": \"General information\", \"score\": \"102.0\"}", - "College Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # prompt tokens": "{\"description\": \"min=513.647, mean=513.647, max=513.647, sum=1027.294 (2)\", \"tab\": \"General information\", \"score\": \"513.6470588235294\"}", - "College Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"college_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_college_physics\"" - } - } - }, - { - "evaluation_name": "Computer Security", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Computer Security", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.74, - "details": { - "description": "min=0.74, mean=0.74, max=0.74, sum=1.48 (2)", - "tab": "Accuracy", - "Computer Security - Observed inference time (s)": "{\"description\": \"min=1.044, mean=1.044, max=1.044, sum=2.088 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.0440657019615174\"}", - "Computer Security - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Computer Security - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Computer Security - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Computer Security - # prompt tokens": "{\"description\": \"min=384.24, mean=384.24, max=384.24, sum=768.48 (2)\", \"tab\": \"General information\", \"score\": \"384.24\"}", - "Computer Security - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"computer_security\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_computer_security\"" - } - } - }, - { - "evaluation_name": "Econometrics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Econometrics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.518, - "details": { - "description": "min=0.518, mean=0.518, max=0.518, sum=1.035 (2)", - "tab": "Accuracy", - "Econometrics - Observed inference time (s)": "{\"description\": \"min=1.047, mean=1.047, max=1.047, sum=2.094 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.04721718921996\"}", - "Econometrics - # eval": "{\"description\": \"min=114, mean=114, max=114, sum=228 (2)\", \"tab\": \"General information\", \"score\": \"114.0\"}", - "Econometrics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Econometrics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Econometrics - # prompt tokens": "{\"description\": \"min=644.395, mean=644.395, max=644.395, sum=1288.789 (2)\", \"tab\": \"General information\", \"score\": \"644.3947368421053\"}", - "Econometrics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"econometrics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_econometrics\"" - } - } - }, - { - "evaluation_name": "Global Facts", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Global Facts", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.38, - "details": { - "description": "min=0.38, mean=0.38, max=0.38, sum=0.76 (2)", - "tab": "Accuracy", - "Global Facts - Observed inference time (s)": "{\"description\": \"min=0.913, mean=0.913, max=0.913, sum=1.826 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9128784847259521\"}", - "Global Facts - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Global Facts - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Global Facts - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Global Facts - # prompt tokens": "{\"description\": \"min=455.63, mean=455.63, max=455.63, sum=911.26 (2)\", \"tab\": \"General information\", \"score\": \"455.63\"}", - "Global Facts - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"global_facts\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_global_facts\"" - } - } - }, - { - "evaluation_name": "Jurisprudence", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Jurisprudence", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.769, - "details": { - "description": "min=0.769, mean=0.769, max=0.769, sum=1.537 (2)", - "tab": "Accuracy", - "Jurisprudence - Observed inference time (s)": "{\"description\": \"min=0.884, mean=0.884, max=0.884, sum=1.768 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8838474772594593\"}", - "Jurisprudence - # eval": "{\"description\": \"min=108, mean=108, max=108, sum=216 (2)\", \"tab\": \"General information\", \"score\": \"108.0\"}", - "Jurisprudence - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Jurisprudence - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Jurisprudence - # prompt tokens": "{\"description\": \"min=414.444, mean=414.444, max=414.444, sum=828.889 (2)\", \"tab\": \"General information\", \"score\": \"414.44444444444446\"}", - "Jurisprudence - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"jurisprudence\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_jurisprudence\"" - } - } - }, - { - "evaluation_name": "Philosophy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Philosophy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.736, - "details": { - "description": "min=0.736, mean=0.736, max=0.736, sum=1.473 (2)", - "tab": "Accuracy", - "Philosophy - Observed inference time (s)": "{\"description\": \"min=0.716, mean=0.716, max=0.716, sum=1.432 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7159656282406528\"}", - "Philosophy - # eval": "{\"description\": \"min=311, mean=311, max=311, sum=622 (2)\", \"tab\": \"General information\", \"score\": \"311.0\"}", - "Philosophy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Philosophy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Philosophy - # prompt tokens": "{\"description\": \"min=339.093, mean=339.093, max=339.093, sum=678.186 (2)\", \"tab\": \"General information\", \"score\": \"339.09324758842445\"}", - "Philosophy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"philosophy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_philosophy\"" - } - } - }, - { - "evaluation_name": "Professional Psychology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Professional Psychology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.761, - "details": { - "description": "min=0.761, mean=0.761, max=0.761, sum=1.523 (2)", - "tab": "Accuracy", - "Professional Medicine - Observed inference time (s)": "{\"description\": \"min=8.281, mean=8.281, max=8.281, sum=16.562 (2)\", \"tab\": \"Efficiency\", \"score\": \"8.280891868998022\"}", - "Professional Accounting - Observed inference time (s)": "{\"description\": \"min=0.812, mean=0.812, max=0.812, sum=1.624 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8122333144465237\"}", - "Professional Law - Observed inference time (s)": "{\"description\": \"min=0.634, mean=0.634, max=0.634, sum=1.268 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6340693978318335\"}", - "Professional Psychology - Observed inference time (s)": "{\"description\": \"min=0.697, mean=0.697, max=0.697, sum=1.394 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6971427946308859\"}", - "Professional Medicine - # eval": "{\"description\": \"min=272, mean=272, max=272, sum=544 (2)\", \"tab\": \"General information\", \"score\": \"272.0\"}", - "Professional Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Medicine - # prompt tokens": "{\"description\": \"min=1104.614, mean=1104.614, max=1104.614, sum=2209.228 (2)\", \"tab\": \"General information\", \"score\": \"1104.6139705882354\"}", - "Professional Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Accounting - # eval": "{\"description\": \"min=282, mean=282, max=282, sum=564 (2)\", \"tab\": \"General information\", \"score\": \"282.0\"}", - "Professional Accounting - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Accounting - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # prompt tokens": "{\"description\": \"min=752.83, mean=752.83, max=752.83, sum=1505.66 (2)\", \"tab\": \"General information\", \"score\": \"752.8297872340426\"}", - "Professional Accounting - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Law - # eval": "{\"description\": \"min=1534, mean=1534, max=1534, sum=3068 (2)\", \"tab\": \"General information\", \"score\": \"1534.0\"}", - "Professional Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # prompt tokens": "{\"description\": \"min=1701.909, mean=1701.909, max=1701.909, sum=3403.819 (2)\", \"tab\": \"General information\", \"score\": \"1701.9093872229466\"}", - "Professional Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Psychology - # eval": "{\"description\": \"min=612, mean=612, max=612, sum=1224 (2)\", \"tab\": \"General information\", \"score\": \"612.0\"}", - "Professional Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # prompt tokens": "{\"description\": \"min=594.446, mean=594.446, max=594.446, sum=1188.892 (2)\", \"tab\": \"General information\", \"score\": \"594.4460784313726\"}", - "Professional Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"professional_psychology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_professional_psychology\"" - } - } - }, - { - "evaluation_name": "Us Foreign Policy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Us Foreign Policy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.87, - "details": { - "description": "min=0.87, mean=0.87, max=0.87, sum=1.74 (2)", - "tab": "Accuracy", - "Us Foreign Policy - Observed inference time (s)": "{\"description\": \"min=1.101, mean=1.101, max=1.101, sum=2.202 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.1012366461753844\"}", - "Us Foreign Policy - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Us Foreign Policy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Us Foreign Policy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Us Foreign Policy - # prompt tokens": "{\"description\": \"min=440.48, mean=440.48, max=440.48, sum=880.96 (2)\", \"tab\": \"General information\", \"score\": \"440.48\"}", - "Us Foreign Policy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"us_foreign_policy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_us_foreign_policy\"" - } - } - }, - { - "evaluation_name": "Astronomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Astronomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.803, - "details": { - "description": "min=0.803, mean=0.803, max=0.803, sum=1.605 (2)", - "tab": "Accuracy", - "Astronomy - Observed inference time (s)": "{\"description\": \"min=0.715, mean=0.715, max=0.715, sum=1.43 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7148221495904421\"}", - "Astronomy - # eval": "{\"description\": \"min=152, mean=152, max=152, sum=304 (2)\", \"tab\": \"General information\", \"score\": \"152.0\"}", - "Astronomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Astronomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Astronomy - # prompt tokens": "{\"description\": \"min=613.033, mean=613.033, max=613.033, sum=1226.066 (2)\", \"tab\": \"General information\", \"score\": \"613.0328947368421\"}", - "Astronomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"astronomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_astronomy\"" - } - } - }, - { - "evaluation_name": "Business Ethics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Business Ethics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.76, - "details": { - "description": "min=0.76, mean=0.76, max=0.76, sum=1.52 (2)", - "tab": "Accuracy", - "Business Ethics - Observed inference time (s)": "{\"description\": \"min=0.893, mean=0.893, max=0.893, sum=1.785 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8926668572425842\"}", - "Business Ethics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Business Ethics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Business Ethics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Business Ethics - # prompt tokens": "{\"description\": \"min=559.31, mean=559.31, max=559.31, sum=1118.62 (2)\", \"tab\": \"General information\", \"score\": \"559.31\"}", - "Business Ethics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"business_ethics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_business_ethics\"" - } - } - }, - { - "evaluation_name": "Clinical Knowledge", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Clinical Knowledge", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.725, - "details": { - "description": "min=0.725, mean=0.725, max=0.725, sum=1.449 (2)", - "tab": "Accuracy", - "Clinical Knowledge - Observed inference time (s)": "{\"description\": \"min=0.77, mean=0.77, max=0.77, sum=1.541 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7704581980435353\"}", - "Clinical Knowledge - # eval": "{\"description\": \"min=265, mean=265, max=265, sum=530 (2)\", \"tab\": \"General information\", \"score\": \"265.0\"}", - "Clinical Knowledge - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Clinical Knowledge - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Clinical Knowledge - # prompt tokens": "{\"description\": \"min=394.77, mean=394.77, max=394.77, sum=789.54 (2)\", \"tab\": \"General information\", \"score\": \"394.76981132075474\"}", - "Clinical Knowledge - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"clinical_knowledge\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_clinical_knowledge\"" - } - } - }, - { - "evaluation_name": "Conceptual Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Conceptual Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.694, - "details": { - "description": "min=0.694, mean=0.694, max=0.694, sum=1.387 (2)", - "tab": "Accuracy", - "Conceptual Physics - Observed inference time (s)": "{\"description\": \"min=0.828, mean=0.828, max=0.828, sum=1.656 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8279458959051903\"}", - "Conceptual Physics - # eval": "{\"description\": \"min=235, mean=235, max=235, sum=470 (2)\", \"tab\": \"General information\", \"score\": \"235.0\"}", - "Conceptual Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Conceptual Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Conceptual Physics - # prompt tokens": "{\"description\": \"min=309.477, mean=309.477, max=309.477, sum=618.953 (2)\", \"tab\": \"General information\", \"score\": \"309.4765957446809\"}", - "Conceptual Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"conceptual_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_conceptual_physics\"" - } - } - }, - { - "evaluation_name": "Electrical Engineering", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Electrical Engineering", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.69, - "details": { - "description": "min=0.69, mean=0.69, max=0.69, sum=1.379 (2)", - "tab": "Accuracy", - "Electrical Engineering - Observed inference time (s)": "{\"description\": \"min=10.257, mean=10.257, max=10.257, sum=20.514 (2)\", \"tab\": \"Efficiency\", \"score\": \"10.257030944166512\"}", - "Electrical Engineering - # eval": "{\"description\": \"min=145, mean=145, max=145, sum=290 (2)\", \"tab\": \"General information\", \"score\": \"145.0\"}", - "Electrical Engineering - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Electrical Engineering - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Electrical Engineering - # prompt tokens": "{\"description\": \"min=480.524, mean=480.524, max=480.524, sum=961.048 (2)\", \"tab\": \"General information\", \"score\": \"480.5241379310345\"}", - "Electrical Engineering - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"electrical_engineering\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_electrical_engineering\"" - } - } - }, - { - "evaluation_name": "Elementary Mathematics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Elementary Mathematics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.487, - "details": { - "description": "min=0.487, mean=0.487, max=0.487, sum=0.974 (2)", - "tab": "Accuracy", - "Elementary Mathematics - Observed inference time (s)": "{\"description\": \"min=0.751, mean=0.751, max=0.751, sum=1.502 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7508898708555434\"}", - "Elementary Mathematics - # eval": "{\"description\": \"min=378, mean=378, max=378, sum=756 (2)\", \"tab\": \"General information\", \"score\": \"378.0\"}", - "Elementary Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Elementary Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Elementary Mathematics - # prompt tokens": "{\"description\": \"min=599.828, mean=599.828, max=599.828, sum=1199.656 (2)\", \"tab\": \"General information\", \"score\": \"599.8280423280423\"}", - "Elementary Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"elementary_mathematics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_elementary_mathematics\"" - } - } - }, - { - "evaluation_name": "Formal Logic", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Formal Logic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5, - "details": { - "description": "min=0.5, mean=0.5, max=0.5, sum=1 (2)", - "tab": "Accuracy", - "Formal Logic - Observed inference time (s)": "{\"description\": \"min=0.763, mean=0.763, max=0.763, sum=1.525 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7626136711665562\"}", - "Formal Logic - # eval": "{\"description\": \"min=126, mean=126, max=126, sum=252 (2)\", \"tab\": \"General information\", \"score\": \"126.0\"}", - "Formal Logic - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Formal Logic - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Formal Logic - # prompt tokens": "{\"description\": \"min=623.508, mean=623.508, max=623.508, sum=1247.016 (2)\", \"tab\": \"General information\", \"score\": \"623.5079365079365\"}", - "Formal Logic - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"formal_logic\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_formal_logic\"" - } - } - }, - { - "evaluation_name": "High School World History", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on High School World History", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.869, - "details": { - "description": "min=0.869, mean=0.869, max=0.869, sum=1.738 (2)", - "tab": "Accuracy", - "High School Biology - Observed inference time (s)": "{\"description\": \"min=0.789, mean=0.789, max=0.789, sum=1.577 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7886250380546816\"}", - "High School Chemistry - Observed inference time (s)": "{\"description\": \"min=0.837, mean=0.837, max=0.837, sum=1.675 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8373666197208348\"}", - "High School Computer Science - Observed inference time (s)": "{\"description\": \"min=0.961, mean=0.961, max=0.961, sum=1.922 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9611564636230469\"}", - "High School European History - Observed inference time (s)": "{\"description\": \"min=1.13, mean=1.13, max=1.13, sum=2.26 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.129964493260239\"}", - "High School Geography - Observed inference time (s)": "{\"description\": \"min=0.754, mean=0.754, max=0.754, sum=1.508 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7538033362590906\"}", - "High School Government And Politics - Observed inference time (s)": "{\"description\": \"min=0.688, mean=0.688, max=0.688, sum=1.375 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6876482963562012\"}", - "High School Macroeconomics - Observed inference time (s)": "{\"description\": \"min=0.718, mean=0.718, max=0.718, sum=1.437 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7183168649673461\"}", - "High School Mathematics - Observed inference time (s)": "{\"description\": \"min=0.782, mean=0.782, max=0.782, sum=1.564 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7819750944773356\"}", - "High School Microeconomics - Observed inference time (s)": "{\"description\": \"min=0.802, mean=0.802, max=0.802, sum=1.603 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8016475258755082\"}", - "High School Physics - Observed inference time (s)": "{\"description\": \"min=0.86, mean=0.86, max=0.86, sum=1.721 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.860422892286288\"}", - "High School Psychology - Observed inference time (s)": "{\"description\": \"min=0.675, mean=0.675, max=0.675, sum=1.35 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6752404208577008\"}", - "High School Statistics - Observed inference time (s)": "{\"description\": \"min=9.407, mean=9.407, max=9.407, sum=18.814 (2)\", \"tab\": \"Efficiency\", \"score\": \"9.407231820954216\"}", - "High School US History - Observed inference time (s)": "{\"description\": \"min=1.054, mean=1.054, max=1.054, sum=2.109 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.0542718312319588\"}", - "High School World History - Observed inference time (s)": "{\"description\": \"min=0.848, mean=0.848, max=0.848, sum=1.695 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8476851751029743\"}", - "High School Biology - # eval": "{\"description\": \"min=310, mean=310, max=310, sum=620 (2)\", \"tab\": \"General information\", \"score\": \"310.0\"}", - "High School Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Biology - # prompt tokens": "{\"description\": \"min=501.255, mean=501.255, max=501.255, sum=1002.51 (2)\", \"tab\": \"General information\", \"score\": \"501.2548387096774\"}", - "High School Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Chemistry - # eval": "{\"description\": \"min=203, mean=203, max=203, sum=406 (2)\", \"tab\": \"General information\", \"score\": \"203.0\"}", - "High School Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # prompt tokens": "{\"description\": \"min=515.473, mean=515.473, max=515.473, sum=1030.946 (2)\", \"tab\": \"General information\", \"score\": \"515.4729064039409\"}", - "High School Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "High School Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # prompt tokens": "{\"description\": \"min=954.08, mean=954.08, max=954.08, sum=1908.16 (2)\", \"tab\": \"General information\", \"score\": \"954.08\"}", - "High School Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School European History - # eval": "{\"description\": \"min=165, mean=165, max=165, sum=330 (2)\", \"tab\": \"General information\", \"score\": \"165.0\"}", - "High School European History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School European History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # prompt tokens": "{\"description\": \"min=2823.23, mean=2823.23, max=2823.23, sum=5646.461 (2)\", \"tab\": \"General information\", \"score\": \"2823.230303030303\"}", - "High School European History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Geography - # eval": "{\"description\": \"min=198, mean=198, max=198, sum=396 (2)\", \"tab\": \"General information\", \"score\": \"198.0\"}", - "High School Geography - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Geography - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # prompt tokens": "{\"description\": \"min=392.939, mean=392.939, max=392.939, sum=785.879 (2)\", \"tab\": \"General information\", \"score\": \"392.93939393939394\"}", - "High School Geography - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Government And Politics - # eval": "{\"description\": \"min=193, mean=193, max=193, sum=386 (2)\", \"tab\": \"General information\", \"score\": \"193.0\"}", - "High School Government And Politics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Government And Politics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # prompt tokens": "{\"description\": \"min=475.44, mean=475.44, max=475.44, sum=950.881 (2)\", \"tab\": \"General information\", \"score\": \"475.440414507772\"}", - "High School Government And Politics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Macroeconomics - # eval": "{\"description\": \"min=390, mean=390, max=390, sum=780 (2)\", \"tab\": \"General information\", \"score\": \"390.0\"}", - "High School Macroeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Macroeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # prompt tokens": "{\"description\": \"min=395.962, mean=395.962, max=395.962, sum=791.923 (2)\", \"tab\": \"General information\", \"score\": \"395.96153846153845\"}", - "High School Macroeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Mathematics - # eval": "{\"description\": \"min=270, mean=270, max=270, sum=540 (2)\", \"tab\": \"General information\", \"score\": \"270.0\"}", - "High School Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # prompt tokens": "{\"description\": \"min=580.393, mean=580.393, max=580.393, sum=1160.785 (2)\", \"tab\": \"General information\", \"score\": \"580.3925925925926\"}", - "High School Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Microeconomics - # eval": "{\"description\": \"min=238, mean=238, max=238, sum=476 (2)\", \"tab\": \"General information\", \"score\": \"238.0\"}", - "High School Microeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Microeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # prompt tokens": "{\"description\": \"min=414.361, mean=414.361, max=414.361, sum=828.723 (2)\", \"tab\": \"General information\", \"score\": \"414.3613445378151\"}", - "High School Microeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Physics - # eval": "{\"description\": \"min=151, mean=151, max=151, sum=302 (2)\", \"tab\": \"General information\", \"score\": \"151.0\"}", - "High School Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # prompt tokens": "{\"description\": \"min=592.252, mean=592.252, max=592.252, sum=1184.503 (2)\", \"tab\": \"General information\", \"score\": \"592.2516556291391\"}", - "High School Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Psychology - # eval": "{\"description\": \"min=545, mean=545, max=545, sum=1090 (2)\", \"tab\": \"General information\", \"score\": \"545.0\"}", - "High School Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # prompt tokens": "{\"description\": \"min=496.51, mean=496.51, max=496.51, sum=993.02 (2)\", \"tab\": \"General information\", \"score\": \"496.5100917431193\"}", - "High School Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Statistics - # eval": "{\"description\": \"min=216, mean=216, max=216, sum=432 (2)\", \"tab\": \"General information\", \"score\": \"216.0\"}", - "High School Statistics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Statistics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # prompt tokens": "{\"description\": \"min=860.532, mean=860.532, max=860.532, sum=1721.065 (2)\", \"tab\": \"General information\", \"score\": \"860.5324074074074\"}", - "High School Statistics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School US History - # eval": "{\"description\": \"min=204, mean=204, max=204, sum=408 (2)\", \"tab\": \"General information\", \"score\": \"204.0\"}", - "High School US History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School US History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # prompt tokens": "{\"description\": \"min=2239.544, mean=2239.544, max=2239.544, sum=4479.088 (2)\", \"tab\": \"General information\", \"score\": \"2239.544117647059\"}", - "High School US History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School World History - # eval": "{\"description\": \"min=237, mean=237, max=237, sum=474 (2)\", \"tab\": \"General information\", \"score\": \"237.0\"}", - "High School World History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School World History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # prompt tokens": "{\"description\": \"min=1437.051, mean=1437.051, max=1437.051, sum=2874.101 (2)\", \"tab\": \"General information\", \"score\": \"1437.0506329113923\"}", - "High School World History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"high_school_world_history\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_high_school_world_history\"" - } - } - }, - { - "evaluation_name": "Human Sexuality", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Human Sexuality", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.84, - "details": { - "description": "min=0.84, mean=0.84, max=0.84, sum=1.679 (2)", - "tab": "Accuracy", - "Human Aging - Observed inference time (s)": "{\"description\": \"min=0.619, mean=0.619, max=0.619, sum=1.237 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6185014632785267\"}", - "Human Sexuality - Observed inference time (s)": "{\"description\": \"min=0.851, mean=0.851, max=0.851, sum=1.702 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8510732850955642\"}", - "Human Aging - # eval": "{\"description\": \"min=223, mean=223, max=223, sum=446 (2)\", \"tab\": \"General information\", \"score\": \"223.0\"}", - "Human Aging - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Aging - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Aging - # prompt tokens": "{\"description\": \"min=323.906, mean=323.906, max=323.906, sum=647.812 (2)\", \"tab\": \"General information\", \"score\": \"323.90582959641256\"}", - "Human Aging - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Human Sexuality - # eval": "{\"description\": \"min=131, mean=131, max=131, sum=262 (2)\", \"tab\": \"General information\", \"score\": \"131.0\"}", - "Human Sexuality - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Sexuality - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # prompt tokens": "{\"description\": \"min=338.74, mean=338.74, max=338.74, sum=677.481 (2)\", \"tab\": \"General information\", \"score\": \"338.74045801526717\"}", - "Human Sexuality - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"human_sexuality\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_human_sexuality\"" - } - } - }, - { - "evaluation_name": "International Law", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on International Law", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.835, - "details": { - "description": "min=0.835, mean=0.835, max=0.835, sum=1.669 (2)", - "tab": "Accuracy", - "International Law - Observed inference time (s)": "{\"description\": \"min=0.93, mean=0.93, max=0.93, sum=1.859 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.929545400556454\"}", - "International Law - # eval": "{\"description\": \"min=121, mean=121, max=121, sum=242 (2)\", \"tab\": \"General information\", \"score\": \"121.0\"}", - "International Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "International Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "International Law - # prompt tokens": "{\"description\": \"min=651.686, mean=651.686, max=651.686, sum=1303.372 (2)\", \"tab\": \"General information\", \"score\": \"651.6859504132232\"}", - "International Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"international_law\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_international_law\"" - } - } - }, - { - "evaluation_name": "Logical Fallacies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Logical Fallacies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.853, - "details": { - "description": "min=0.853, mean=0.853, max=0.853, sum=1.706 (2)", - "tab": "Accuracy", - "Logical Fallacies - Observed inference time (s)": "{\"description\": \"min=0.813, mean=0.813, max=0.813, sum=1.627 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8133661293544652\"}", - "Logical Fallacies - # eval": "{\"description\": \"min=163, mean=163, max=163, sum=326 (2)\", \"tab\": \"General information\", \"score\": \"163.0\"}", - "Logical Fallacies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Logical Fallacies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Logical Fallacies - # prompt tokens": "{\"description\": \"min=443.969, mean=443.969, max=443.969, sum=887.939 (2)\", \"tab\": \"General information\", \"score\": \"443.96932515337426\"}", - "Logical Fallacies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"logical_fallacies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_logical_fallacies\"" - } - } - }, - { - "evaluation_name": "Machine Learning", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Machine Learning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.562, - "details": { - "description": "min=0.562, mean=0.562, max=0.562, sum=1.125 (2)", - "tab": "Accuracy", - "Machine Learning - Observed inference time (s)": "{\"description\": \"min=0.916, mean=0.916, max=0.916, sum=1.832 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9159843921661377\"}", - "Machine Learning - # eval": "{\"description\": \"min=112, mean=112, max=112, sum=224 (2)\", \"tab\": \"General information\", \"score\": \"112.0\"}", - "Machine Learning - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Machine Learning - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Machine Learning - # prompt tokens": "{\"description\": \"min=705.973, mean=705.973, max=705.973, sum=1411.946 (2)\", \"tab\": \"General information\", \"score\": \"705.9732142857143\"}", - "Machine Learning - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"machine_learning\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_machine_learning\"" - } - } - }, - { - "evaluation_name": "Management", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Management", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.893, - "details": { - "description": "min=0.893, mean=0.893, max=0.893, sum=1.786 (2)", - "tab": "Accuracy", - "Management - Observed inference time (s)": "{\"description\": \"min=0.817, mean=0.817, max=0.817, sum=1.633 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8166041281616804\"}", - "Management - # eval": "{\"description\": \"min=103, mean=103, max=103, sum=206 (2)\", \"tab\": \"General information\", \"score\": \"103.0\"}", - "Management - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Management - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Management - # prompt tokens": "{\"description\": \"min=284.68, mean=284.68, max=284.68, sum=569.359 (2)\", \"tab\": \"General information\", \"score\": \"284.6796116504854\"}", - "Management - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"management\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_management\"" - } - } - }, - { - "evaluation_name": "Marketing", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Marketing", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.893, - "details": { - "description": "min=0.893, mean=0.893, max=0.893, sum=1.786 (2)", - "tab": "Accuracy", - "Marketing - Observed inference time (s)": "{\"description\": \"min=0.789, mean=0.789, max=0.789, sum=1.579 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.789409975720267\"}", - "Marketing - # eval": "{\"description\": \"min=234, mean=234, max=234, sum=468 (2)\", \"tab\": \"General information\", \"score\": \"234.0\"}", - "Marketing - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Marketing - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Marketing - # prompt tokens": "{\"description\": \"min=428.726, mean=428.726, max=428.726, sum=857.453 (2)\", \"tab\": \"General information\", \"score\": \"428.7264957264957\"}", - "Marketing - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"marketing\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_marketing\"" - } - } - }, - { - "evaluation_name": "Medical Genetics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Medical Genetics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.75, - "details": { - "description": "min=0.75, mean=0.75, max=0.75, sum=1.5 (2)", - "tab": "Accuracy", - "Medical Genetics - Observed inference time (s)": "{\"description\": \"min=0.857, mean=0.857, max=0.857, sum=1.713 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8565307760238647\"}", - "Medical Genetics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Medical Genetics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Medical Genetics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Medical Genetics - # prompt tokens": "{\"description\": \"min=334.69, mean=334.69, max=334.69, sum=669.38 (2)\", \"tab\": \"General information\", \"score\": \"334.69\"}", - "Medical Genetics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"medical_genetics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_medical_genetics\"" - } - } - }, - { - "evaluation_name": "Miscellaneous", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Miscellaneous", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.866, - "details": { - "description": "min=0.866, mean=0.866, max=0.866, sum=1.732 (2)", - "tab": "Accuracy", - "Miscellaneous - Observed inference time (s)": "{\"description\": \"min=2.759, mean=2.759, max=2.759, sum=5.518 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.7590373143991442\"}", - "Miscellaneous - # eval": "{\"description\": \"min=783, mean=783, max=783, sum=1566 (2)\", \"tab\": \"General information\", \"score\": \"783.0\"}", - "Miscellaneous - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Miscellaneous - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Miscellaneous - # prompt tokens": "{\"description\": \"min=325.215, mean=325.215, max=325.215, sum=650.429 (2)\", \"tab\": \"General information\", \"score\": \"325.2145593869732\"}", - "Miscellaneous - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"miscellaneous\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_miscellaneous\"" - } - } - }, - { - "evaluation_name": "Moral Scenarios", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Moral Scenarios", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.369, - "details": { - "description": "min=0.369, mean=0.369, max=0.369, sum=0.737 (2)", - "tab": "Accuracy", - "Moral Disputes - Observed inference time (s)": "{\"description\": \"min=0.749, mean=0.749, max=0.749, sum=1.497 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7485969907286539\"}", - "Moral Scenarios - Observed inference time (s)": "{\"description\": \"min=0.781, mean=0.781, max=0.781, sum=1.561 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7806768483955767\"}", - "Moral Disputes - # eval": "{\"description\": \"min=346, mean=346, max=346, sum=692 (2)\", \"tab\": \"General information\", \"score\": \"346.0\"}", - "Moral Disputes - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Disputes - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Disputes - # prompt tokens": "{\"description\": \"min=494.63, mean=494.63, max=494.63, sum=989.26 (2)\", \"tab\": \"General information\", \"score\": \"494.6300578034682\"}", - "Moral Disputes - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Moral Scenarios - # eval": "{\"description\": \"min=895, mean=895, max=895, sum=1790 (2)\", \"tab\": \"General information\", \"score\": \"895.0\"}", - "Moral Scenarios - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Scenarios - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # prompt tokens": "{\"description\": \"min=687.566, mean=687.566, max=687.566, sum=1375.133 (2)\", \"tab\": \"General information\", \"score\": \"687.5664804469274\"}", - "Moral Scenarios - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"moral_scenarios\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_moral_scenarios\"" - } - } - }, - { - "evaluation_name": "Nutrition", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Nutrition", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.709, - "details": { - "description": "min=0.709, mean=0.709, max=0.709, sum=1.418 (2)", - "tab": "Accuracy", - "Nutrition - Observed inference time (s)": "{\"description\": \"min=0.81, mean=0.81, max=0.81, sum=1.621 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8104506489498163\"}", - "Nutrition - # eval": "{\"description\": \"min=306, mean=306, max=306, sum=612 (2)\", \"tab\": \"General information\", \"score\": \"306.0\"}", - "Nutrition - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Nutrition - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Nutrition - # prompt tokens": "{\"description\": \"min=589.663, mean=589.663, max=589.663, sum=1179.327 (2)\", \"tab\": \"General information\", \"score\": \"589.6633986928105\"}", - "Nutrition - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"nutrition\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_nutrition\"" - } - } - }, - { - "evaluation_name": "Prehistory", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Prehistory", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.812, - "details": { - "description": "min=0.812, mean=0.812, max=0.812, sum=1.623 (2)", - "tab": "Accuracy", - "Prehistory - Observed inference time (s)": "{\"description\": \"min=0.7, mean=0.7, max=0.7, sum=1.399 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6996216737193826\"}", - "Prehistory - # eval": "{\"description\": \"min=324, mean=324, max=324, sum=648 (2)\", \"tab\": \"General information\", \"score\": \"324.0\"}", - "Prehistory - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Prehistory - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Prehistory - # prompt tokens": "{\"description\": \"min=538.179, mean=538.179, max=538.179, sum=1076.358 (2)\", \"tab\": \"General information\", \"score\": \"538.179012345679\"}", - "Prehistory - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"prehistory\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_prehistory\"" - } - } - }, - { - "evaluation_name": "Public Relations", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Public Relations", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.691, - "details": { - "description": "min=0.691, mean=0.691, max=0.691, sum=1.382 (2)", - "tab": "Accuracy", - "Public Relations - Observed inference time (s)": "{\"description\": \"min=0.98, mean=0.98, max=0.98, sum=1.961 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.980262413891879\"}", - "Public Relations - # eval": "{\"description\": \"min=110, mean=110, max=110, sum=220 (2)\", \"tab\": \"General information\", \"score\": \"110.0\"}", - "Public Relations - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Public Relations - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Public Relations - # prompt tokens": "{\"description\": \"min=426.982, mean=426.982, max=426.982, sum=853.964 (2)\", \"tab\": \"General information\", \"score\": \"426.9818181818182\"}", - "Public Relations - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"public_relations\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_public_relations\"" - } - } - }, - { - "evaluation_name": "Security Studies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Security Studies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.812, - "details": { - "description": "min=0.812, mean=0.812, max=0.812, sum=1.624 (2)", - "tab": "Accuracy", - "Security Studies - Observed inference time (s)": "{\"description\": \"min=0.857, mean=0.857, max=0.857, sum=1.713 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8567250339352355\"}", - "Security Studies - # eval": "{\"description\": \"min=245, mean=245, max=245, sum=490 (2)\", \"tab\": \"General information\", \"score\": \"245.0\"}", - "Security Studies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Security Studies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Security Studies - # prompt tokens": "{\"description\": \"min=1185.8, mean=1185.8, max=1185.8, sum=2371.6 (2)\", \"tab\": \"General information\", \"score\": \"1185.8\"}", - "Security Studies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"security_studies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_security_studies\"" - } - } - }, - { - "evaluation_name": "Sociology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Sociology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.92, - "details": { - "description": "min=0.92, mean=0.92, max=0.92, sum=1.841 (2)", - "tab": "Accuracy", - "Sociology - Observed inference time (s)": "{\"description\": \"min=7.515, mean=7.515, max=7.515, sum=15.029 (2)\", \"tab\": \"Efficiency\", \"score\": \"7.514506837028769\"}", - "Sociology - # eval": "{\"description\": \"min=201, mean=201, max=201, sum=402 (2)\", \"tab\": \"General information\", \"score\": \"201.0\"}", - "Sociology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Sociology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Sociology - # prompt tokens": "{\"description\": \"min=459.642, mean=459.642, max=459.642, sum=919.284 (2)\", \"tab\": \"General information\", \"score\": \"459.64179104477614\"}", - "Sociology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"sociology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_sociology\"" - } - } - }, - { - "evaluation_name": "Virology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Virology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.494, - "details": { - "description": "min=0.494, mean=0.494, max=0.494, sum=0.988 (2)", - "tab": "Accuracy", - "Virology - Observed inference time (s)": "{\"description\": \"min=0.788, mean=0.788, max=0.788, sum=1.577 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7884655989796282\"}", - "Virology - # eval": "{\"description\": \"min=166, mean=166, max=166, sum=332 (2)\", \"tab\": \"General information\", \"score\": \"166.0\"}", - "Virology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Virology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Virology - # prompt tokens": "{\"description\": \"min=337.06, mean=337.06, max=337.06, sum=674.12 (2)\", \"tab\": \"General information\", \"score\": \"337.06024096385545\"}", - "Virology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"virology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_virology\"" - } - } - }, - { - "evaluation_name": "World Religions", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on World Religions", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.883, - "details": { - "description": "min=0.883, mean=0.883, max=0.883, sum=1.766 (2)", - "tab": "Accuracy", - "World Religions - Observed inference time (s)": "{\"description\": \"min=0.802, mean=0.802, max=0.802, sum=1.604 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8022187299895704\"}", - "World Religions - # eval": "{\"description\": \"min=171, mean=171, max=171, sum=342 (2)\", \"tab\": \"General information\", \"score\": \"171.0\"}", - "World Religions - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "World Religions - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "World Religions - # prompt tokens": "{\"description\": \"min=270.187, mean=270.187, max=270.187, sum=540.374 (2)\", \"tab\": \"General information\", \"score\": \"270.187134502924\"}", - "World Religions - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"world_religions\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_world_religions\"" - } - } - }, - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.192, - "details": { - "description": "", - "tab": "Efficiency" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/google_text-unicorn_001.json b/data/models/google_text-unicorn_001.json deleted file mode 100644 index 73e639180f91ee6d4d2039e60cfc223548da8b26..0000000000000000000000000000000000000000 --- a/data/models/google_text-unicorn_001.json +++ /dev/null @@ -1,1901 +0,0 @@ -{ - "model_info": { - "name": "PaLM-2 Unicorn", - "id": "google/text-unicorn@001", - "developer": "Google", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_lite/google_text-unicorn@001/1774096306.427425", - "retrieved_timestamp": "1774096306.427425", - "source_metadata": { - "source_name": "helm_lite", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_lite", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_lite", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.644, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.18023720349563047\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.583, - "details": { - "description": "min=0.583, mean=0.583, max=0.583, sum=0.583 (1)", - "tab": "Accuracy", - "NarrativeQA - Observed inference time (s)": "{\"description\": \"min=3.283, mean=3.283, max=3.283, sum=3.283 (1)\", \"tab\": \"Efficiency\", \"score\": \"3.283053755424392\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=4414.234, mean=4414.234, max=4414.234, sum=4414.234 (1)\", \"tab\": \"General information\", \"score\": \"4414.2338028169015\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=16.544, mean=16.544, max=16.544, sum=16.544 (1)\", \"tab\": \"General information\", \"score\": \"16.543661971830986\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (closed-book)", - "source_data": { - "dataset_name": "NaturalQuestions (closed-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (closed-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.435, - "details": { - "description": "min=0.435, mean=0.435, max=0.435, sum=0.435 (1)", - "tab": "Accuracy", - "NaturalQuestions (open-book) - Observed inference time (s)": "{\"description\": \"min=2.564, mean=2.564, max=2.564, sum=2.564 (1)\", \"tab\": \"Efficiency\", \"score\": \"2.564493465423584\"}", - "NaturalQuestions (closed-book) - Observed inference time (s)": "{\"description\": \"min=1.56, mean=1.56, max=1.56, sum=1.56 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.5603588831424713\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.906, mean=4.906, max=4.906, sum=4.906 (1)\", \"tab\": \"General information\", \"score\": \"4.906\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.015, mean=0.015, max=0.015, sum=0.015 (1)\", \"tab\": \"General information\", \"score\": \"0.015\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=2124.565, mean=2124.565, max=2124.565, sum=2124.565 (1)\", \"tab\": \"General information\", \"score\": \"2124.565\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=13.327, mean=13.327, max=13.327, sum=13.327 (1)\", \"tab\": \"General information\", \"score\": \"13.327\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=190.187, mean=190.187, max=190.187, sum=190.187 (1)\", \"tab\": \"General information\", \"score\": \"190.187\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=9.803, mean=9.803, max=9.803, sum=9.803 (1)\", \"tab\": \"General information\", \"score\": \"9.803\"}" - } - }, - "generation_config": { - "additional_details": { - "mode": "\"closedbook\"" - } - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.938, - "details": { - "description": "min=0.938, mean=0.938, max=0.938, sum=0.938 (1)", - "tab": "Accuracy", - "OpenbookQA - Observed inference time (s)": "{\"description\": \"min=0.999, mean=0.999, max=0.999, sum=0.999 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.9994440112113953\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=253.308, mean=253.308, max=253.308, sum=253.308 (1)\", \"tab\": \"General information\", \"score\": \"253.308\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "dataset": "\"openbookqa\"", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.702, - "details": { - "description": "min=0.53, mean=0.702, max=0.96, sum=3.509 (5)", - "tab": "Accuracy", - "MMLU - Observed inference time (s)": "{\"description\": \"min=1.198, mean=1.262, max=1.332, sum=6.31 (5)\", \"tab\": \"Efficiency\", \"score\": \"1.2620431824148748\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=360.7, mean=487.294, max=638.088, sum=2436.468 (5)\", \"tab\": \"General information\", \"score\": \"487.29354385964905\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"college_chemistry\", \"computer_security\", \"econometrics\", \"us_foreign_policy\"]", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MATH", - "source_data": { - "dataset_name": "MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Equivalent (CoT) on MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.674, - "details": { - "description": "min=0.526, mean=0.674, max=0.867, sum=4.716 (7)", - "tab": "Accuracy", - "MATH - Observed inference time (s)": "{\"description\": \"min=4.016, mean=4.636, max=5.654, sum=32.454 (7)\", \"tab\": \"Efficiency\", \"score\": \"4.636334307701402\"}", - "MATH - # eval": "{\"description\": \"min=30, mean=62.429, max=135, sum=437 (7)\", \"tab\": \"General information\", \"score\": \"62.42857142857143\"}", - "MATH - # train": "{\"description\": \"min=8, mean=8, max=8, sum=56 (7)\", \"tab\": \"General information\", \"score\": \"8.0\"}", - "MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MATH - # prompt tokens": "{\"description\": \"min=1004.274, mean=1439.843, max=2386.942, sum=10078.901 (7)\", \"tab\": \"General information\", \"score\": \"1439.842989280994\"}", - "MATH - # output tokens": "{\"description\": \"min=59.9, mean=80.458, max=98.342, sum=563.207 (7)\", \"tab\": \"General information\", \"score\": \"80.45819114472725\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"algebra\", \"counting_and_probability\", \"geometry\", \"intermediate_algebra\", \"number_theory\", \"prealgebra\", \"precalculus\"]", - "level": "\"1\"", - "use_official_examples": "\"False\"", - "use_chain_of_thought": "\"True\"" - } - } - }, - { - "evaluation_name": "GSM8K", - "source_data": { - "dataset_name": "GSM8K", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on GSM8K", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.831, - "details": { - "description": "min=0.831, mean=0.831, max=0.831, sum=0.831 (1)", - "tab": "Accuracy", - "GSM8K - Observed inference time (s)": "{\"description\": \"min=5.437, mean=5.437, max=5.437, sum=5.437 (1)\", \"tab\": \"Efficiency\", \"score\": \"5.4373185629844665\"}", - "GSM8K - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "GSM8K - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "GSM8K - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GSM8K - # prompt tokens": "{\"description\": \"min=1109.549, mean=1109.549, max=1109.549, sum=1109.549 (1)\", \"tab\": \"General information\", \"score\": \"1109.549\"}", - "GSM8K - # output tokens": "{\"description\": \"min=93.764, mean=93.764, max=93.764, sum=93.764 (1)\", \"tab\": \"General information\", \"score\": \"93.764\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "LegalBench", - "source_data": { - "dataset_name": "LegalBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on LegalBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.677, - "details": { - "description": "min=0.452, mean=0.677, max=0.926, sum=3.387 (5)", - "tab": "Accuracy", - "LegalBench - Observed inference time (s)": "{\"description\": \"min=0.859, mean=1.437, max=3.198, sum=7.187 (5)\", \"tab\": \"Efficiency\", \"score\": \"1.4374773445647835\"}", - "LegalBench - # eval": "{\"description\": \"min=95, mean=409.4, max=1000, sum=2047 (5)\", \"tab\": \"General information\", \"score\": \"409.4\"}", - "LegalBench - # train": "{\"description\": \"min=2.988, mean=4.398, max=5, sum=21.988 (5)\", \"tab\": \"General information\", \"score\": \"4.397551020408163\"}", - "LegalBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "LegalBench - # prompt tokens": "{\"description\": \"min=287.432, mean=1387.966, max=5134.504, sum=6939.831 (5)\", \"tab\": \"General information\", \"score\": \"1387.966233478402\"}", - "LegalBench - # output tokens": "{\"description\": \"min=1, mean=1.364, max=2.2, sum=6.821 (5)\", \"tab\": \"General information\", \"score\": \"1.3642506811989101\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"abercrombie\", \"corporate_lobbying\", \"function_of_decision_section\", \"international_citizenship_questions\", \"proa\"]" - } - } - }, - { - "evaluation_name": "MedQA", - "source_data": { - "dataset_name": "MedQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MedQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.684, - "details": { - "description": "min=0.684, mean=0.684, max=0.684, sum=0.684 (1)", - "tab": "Accuracy", - "MedQA - Observed inference time (s)": "{\"description\": \"min=1.178, mean=1.178, max=1.178, sum=1.178 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.1783231205305096\"}", - "MedQA - # eval": "{\"description\": \"min=503, mean=503, max=503, sum=503 (1)\", \"tab\": \"General information\", \"score\": \"503.0\"}", - "MedQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MedQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MedQA - # prompt tokens": "{\"description\": \"min=1138.622, mean=1138.622, max=1138.622, sum=1138.622 (1)\", \"tab\": \"General information\", \"score\": \"1138.6222664015904\"}", - "MedQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WMT 2014", - "source_data": { - "dataset_name": "WMT 2014", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "BLEU-4 on WMT 2014", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.26, - "details": { - "description": "min=0.236, mean=0.26, max=0.279, sum=1.298 (5)", - "tab": "Accuracy", - "WMT 2014 - Observed inference time (s)": "{\"description\": \"min=1.706, mean=1.801, max=1.909, sum=9.006 (5)\", \"tab\": \"Efficiency\", \"score\": \"1.801295139912888\"}", - "WMT 2014 - # eval": "{\"description\": \"min=503, mean=568.8, max=832, sum=2844 (5)\", \"tab\": \"General information\", \"score\": \"568.8\"}", - "WMT 2014 - # train": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "WMT 2014 - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WMT 2014 - # prompt tokens": "{\"description\": \"min=145.755, mean=183.587, max=206.169, sum=917.936 (5)\", \"tab\": \"General information\", \"score\": \"183.58714444104604\"}", - "WMT 2014 - # output tokens": "{\"description\": \"min=28.596, mean=30.567, max=31.734, sum=152.836 (5)\", \"tab\": \"General information\", \"score\": \"30.567241263954735\"}" - } - }, - "generation_config": { - "additional_details": { - "language_pair": "[\"cs-en\", \"de-en\", \"fr-en\", \"hi-en\", \"ru-en\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_mmlu/google_text-unicorn@001/1774096312.00548", - "retrieved_timestamp": "1774096312.00548", - "source_metadata": { - "source_name": "helm_mmlu", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_mmlu", - "evaluation_results": [ - { - "evaluation_name": "MMLU All Subjects", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU All Subjects", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.786, - "details": { - "description": "min=0.493, mean=0.786, max=0.979, sum=89.606 (114)", - "tab": "Accuracy", - "MMLU All Subjects - Observed inference time (s)": "{\"description\": \"min=0.743, mean=1.052, max=2.108, sum=119.953 (114)\", \"tab\": \"Efficiency\", \"score\": \"1.0522220782452074\"}", - "MMLU All Subjects - # eval": "{\"description\": \"min=100, mean=246.351, max=1534, sum=28084 (114)\", \"tab\": \"General information\", \"score\": \"246.35087719298247\"}", - "MMLU All Subjects - # train": "{\"description\": \"min=5, mean=5, max=5, sum=570 (114)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU All Subjects - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU All Subjects - # prompt tokens": "{\"description\": \"min=270.187, mean=635.61, max=2823.23, sum=72459.527 (114)\", \"tab\": \"General information\", \"score\": \"635.6098850770794\"}", - "MMLU All Subjects - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_name": "Abstract Algebra", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Abstract Algebra", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.51, - "details": { - "description": "min=0.51, mean=0.51, max=0.51, sum=1.02 (2)", - "tab": "Accuracy", - "Abstract Algebra - Observed inference time (s)": "{\"description\": \"min=1.277, mean=1.277, max=1.277, sum=2.555 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.2773328518867493\"}", - "Abstract Algebra - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Abstract Algebra - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Abstract Algebra - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Abstract Algebra - # prompt tokens": "{\"description\": \"min=387.12, mean=387.12, max=387.12, sum=774.24 (2)\", \"tab\": \"General information\", \"score\": \"387.12\"}", - "Abstract Algebra - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"abstract_algebra\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_abstract_algebra\"" - } - } - }, - { - "evaluation_name": "Anatomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Anatomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.733, - "details": { - "description": "min=0.733, mean=0.733, max=0.733, sum=1.467 (2)", - "tab": "Accuracy", - "Anatomy - Observed inference time (s)": "{\"description\": \"min=0.957, mean=0.957, max=0.957, sum=1.914 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9569159172199391\"}", - "Anatomy - # eval": "{\"description\": \"min=135, mean=135, max=135, sum=270 (2)\", \"tab\": \"General information\", \"score\": \"135.0\"}", - "Anatomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Anatomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Anatomy - # prompt tokens": "{\"description\": \"min=344.089, mean=344.089, max=344.089, sum=688.178 (2)\", \"tab\": \"General information\", \"score\": \"344.0888888888889\"}", - "Anatomy - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"anatomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_anatomy\"" - } - } - }, - { - "evaluation_name": "College Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on College Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.549, - "details": { - "description": "min=0.549, mean=0.549, max=0.549, sum=1.098 (2)", - "tab": "Accuracy", - "College Chemistry - Observed inference time (s)": "{\"description\": \"min=0.934, mean=0.934, max=0.934, sum=1.869 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9343120718002319\"}", - "College Biology - Observed inference time (s)": "{\"description\": \"min=0.873, mean=0.873, max=0.873, sum=1.746 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8729922622442245\"}", - "College Computer Science - Observed inference time (s)": "{\"description\": \"min=1.165, mean=1.165, max=1.165, sum=2.33 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.165095055103302\"}", - "College Mathematics - Observed inference time (s)": "{\"description\": \"min=1.062, mean=1.062, max=1.062, sum=2.124 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.0619186329841614\"}", - "College Medicine - Observed inference time (s)": "{\"description\": \"min=0.978, mean=0.978, max=0.978, sum=1.957 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.978282785140021\"}", - "College Physics - Observed inference time (s)": "{\"description\": \"min=0.852, mean=0.852, max=0.852, sum=1.704 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8518095483966902\"}", - "College Chemistry - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Chemistry - # prompt tokens": "{\"description\": \"min=573.7, mean=573.7, max=573.7, sum=1147.4 (2)\", \"tab\": \"General information\", \"score\": \"573.7\"}", - "College Chemistry - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # eval": "{\"description\": \"min=144, mean=144, max=144, sum=288 (2)\", \"tab\": \"General information\", \"score\": \"144.0\"}", - "College Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # prompt tokens": "{\"description\": \"min=480.875, mean=480.875, max=480.875, sum=961.75 (2)\", \"tab\": \"General information\", \"score\": \"480.875\"}", - "College Biology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # prompt tokens": "{\"description\": \"min=861.96, mean=861.96, max=861.96, sum=1723.92 (2)\", \"tab\": \"General information\", \"score\": \"861.96\"}", - "College Computer Science - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # prompt tokens": "{\"description\": \"min=636.94, mean=636.94, max=636.94, sum=1273.88 (2)\", \"tab\": \"General information\", \"score\": \"636.94\"}", - "College Mathematics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # eval": "{\"description\": \"min=173, mean=173, max=173, sum=346 (2)\", \"tab\": \"General information\", \"score\": \"173.0\"}", - "College Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # prompt tokens": "{\"description\": \"min=512.584, mean=512.584, max=512.584, sum=1025.168 (2)\", \"tab\": \"General information\", \"score\": \"512.5838150289018\"}", - "College Medicine - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # eval": "{\"description\": \"min=102, mean=102, max=102, sum=204 (2)\", \"tab\": \"General information\", \"score\": \"102.0\"}", - "College Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # prompt tokens": "{\"description\": \"min=513.647, mean=513.647, max=513.647, sum=1027.294 (2)\", \"tab\": \"General information\", \"score\": \"513.6470588235294\"}", - "College Physics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"college_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_college_physics\"" - } - } - }, - { - "evaluation_name": "Computer Security", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Computer Security", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.77, - "details": { - "description": "min=0.77, mean=0.77, max=0.77, sum=1.54 (2)", - "tab": "Accuracy", - "Computer Security - Observed inference time (s)": "{\"description\": \"min=0.845, mean=0.845, max=0.845, sum=1.69 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8448482728004456\"}", - "Computer Security - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Computer Security - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Computer Security - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Computer Security - # prompt tokens": "{\"description\": \"min=384.24, mean=384.24, max=384.24, sum=768.48 (2)\", \"tab\": \"General information\", \"score\": \"384.24\"}", - "Computer Security - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"computer_security\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_computer_security\"" - } - } - }, - { - "evaluation_name": "Econometrics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Econometrics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.649, - "details": { - "description": "min=0.649, mean=0.649, max=0.649, sum=1.298 (2)", - "tab": "Accuracy", - "Econometrics - Observed inference time (s)": "{\"description\": \"min=0.852, mean=0.852, max=0.852, sum=1.704 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8522159112127203\"}", - "Econometrics - # eval": "{\"description\": \"min=114, mean=114, max=114, sum=228 (2)\", \"tab\": \"General information\", \"score\": \"114.0\"}", - "Econometrics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Econometrics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Econometrics - # prompt tokens": "{\"description\": \"min=644.395, mean=644.395, max=644.395, sum=1288.789 (2)\", \"tab\": \"General information\", \"score\": \"644.3947368421053\"}", - "Econometrics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"econometrics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_econometrics\"" - } - } - }, - { - "evaluation_name": "Global Facts", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Global Facts", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.53, - "details": { - "description": "min=0.53, mean=0.53, max=0.53, sum=1.06 (2)", - "tab": "Accuracy", - "Global Facts - Observed inference time (s)": "{\"description\": \"min=0.888, mean=0.888, max=0.888, sum=1.775 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8876941871643066\"}", - "Global Facts - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Global Facts - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Global Facts - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Global Facts - # prompt tokens": "{\"description\": \"min=455.63, mean=455.63, max=455.63, sum=911.26 (2)\", \"tab\": \"General information\", \"score\": \"455.63\"}", - "Global Facts - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"global_facts\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_global_facts\"" - } - } - }, - { - "evaluation_name": "Jurisprudence", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Jurisprudence", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.88, - "details": { - "description": "min=0.88, mean=0.88, max=0.88, sum=1.759 (2)", - "tab": "Accuracy", - "Jurisprudence - Observed inference time (s)": "{\"description\": \"min=1.017, mean=1.017, max=1.017, sum=2.034 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.0168068651799802\"}", - "Jurisprudence - # eval": "{\"description\": \"min=108, mean=108, max=108, sum=216 (2)\", \"tab\": \"General information\", \"score\": \"108.0\"}", - "Jurisprudence - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Jurisprudence - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Jurisprudence - # prompt tokens": "{\"description\": \"min=414.444, mean=414.444, max=414.444, sum=828.889 (2)\", \"tab\": \"General information\", \"score\": \"414.44444444444446\"}", - "Jurisprudence - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"jurisprudence\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_jurisprudence\"" - } - } - }, - { - "evaluation_name": "Philosophy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Philosophy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.836, - "details": { - "description": "min=0.836, mean=0.836, max=0.836, sum=1.672 (2)", - "tab": "Accuracy", - "Philosophy - Observed inference time (s)": "{\"description\": \"min=0.895, mean=0.895, max=0.895, sum=1.79 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8949410808048064\"}", - "Philosophy - # eval": "{\"description\": \"min=311, mean=311, max=311, sum=622 (2)\", \"tab\": \"General information\", \"score\": \"311.0\"}", - "Philosophy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Philosophy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Philosophy - # prompt tokens": "{\"description\": \"min=339.093, mean=339.093, max=339.093, sum=678.186 (2)\", \"tab\": \"General information\", \"score\": \"339.09324758842445\"}", - "Philosophy - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"philosophy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_philosophy\"" - } - } - }, - { - "evaluation_name": "Professional Psychology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Professional Psychology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.858, - "details": { - "description": "min=0.858, mean=0.858, max=0.858, sum=1.716 (2)", - "tab": "Accuracy", - "Professional Medicine - Observed inference time (s)": "{\"description\": \"min=1.088, mean=1.088, max=1.088, sum=2.175 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.0875138991019304\"}", - "Professional Accounting - Observed inference time (s)": "{\"description\": \"min=0.978, mean=0.978, max=0.978, sum=1.956 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9778145923682139\"}", - "Professional Law - Observed inference time (s)": "{\"description\": \"min=1.205, mean=1.205, max=1.205, sum=2.41 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.204983455416743\"}", - "Professional Psychology - Observed inference time (s)": "{\"description\": \"min=0.789, mean=0.789, max=0.789, sum=1.578 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7891469753645604\"}", - "Professional Medicine - # eval": "{\"description\": \"min=272, mean=272, max=272, sum=544 (2)\", \"tab\": \"General information\", \"score\": \"272.0\"}", - "Professional Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Medicine - # prompt tokens": "{\"description\": \"min=1104.614, mean=1104.614, max=1104.614, sum=2209.228 (2)\", \"tab\": \"General information\", \"score\": \"1104.6139705882354\"}", - "Professional Medicine - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # eval": "{\"description\": \"min=282, mean=282, max=282, sum=564 (2)\", \"tab\": \"General information\", \"score\": \"282.0\"}", - "Professional Accounting - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Accounting - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # prompt tokens": "{\"description\": \"min=752.83, mean=752.83, max=752.83, sum=1505.66 (2)\", \"tab\": \"General information\", \"score\": \"752.8297872340426\"}", - "Professional Accounting - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # eval": "{\"description\": \"min=1534, mean=1534, max=1534, sum=3068 (2)\", \"tab\": \"General information\", \"score\": \"1534.0\"}", - "Professional Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # prompt tokens": "{\"description\": \"min=1701.909, mean=1701.909, max=1701.909, sum=3403.819 (2)\", \"tab\": \"General information\", \"score\": \"1701.9093872229466\"}", - "Professional Law - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # eval": "{\"description\": \"min=612, mean=612, max=612, sum=1224 (2)\", \"tab\": \"General information\", \"score\": \"612.0\"}", - "Professional Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # prompt tokens": "{\"description\": \"min=594.446, mean=594.446, max=594.446, sum=1188.892 (2)\", \"tab\": \"General information\", \"score\": \"594.4460784313726\"}", - "Professional Psychology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"professional_psychology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_professional_psychology\"" - } - } - }, - { - "evaluation_name": "Us Foreign Policy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Us Foreign Policy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.96, - "details": { - "description": "min=0.96, mean=0.96, max=0.96, sum=1.92 (2)", - "tab": "Accuracy", - "Us Foreign Policy - Observed inference time (s)": "{\"description\": \"min=0.743, mean=0.743, max=0.743, sum=1.485 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7426803350448609\"}", - "Us Foreign Policy - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Us Foreign Policy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Us Foreign Policy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Us Foreign Policy - # prompt tokens": "{\"description\": \"min=440.48, mean=440.48, max=440.48, sum=880.96 (2)\", \"tab\": \"General information\", \"score\": \"440.48\"}", - "Us Foreign Policy - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"us_foreign_policy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_us_foreign_policy\"" - } - } - }, - { - "evaluation_name": "Astronomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Astronomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.862, - "details": { - "description": "min=0.862, mean=0.862, max=0.862, sum=1.724 (2)", - "tab": "Accuracy", - "Astronomy - Observed inference time (s)": "{\"description\": \"min=0.843, mean=0.843, max=0.843, sum=1.686 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8429784712038542\"}", - "Astronomy - # eval": "{\"description\": \"min=152, mean=152, max=152, sum=304 (2)\", \"tab\": \"General information\", \"score\": \"152.0\"}", - "Astronomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Astronomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Astronomy - # prompt tokens": "{\"description\": \"min=613.033, mean=613.033, max=613.033, sum=1226.066 (2)\", \"tab\": \"General information\", \"score\": \"613.0328947368421\"}", - "Astronomy - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"astronomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_astronomy\"" - } - } - }, - { - "evaluation_name": "Business Ethics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Business Ethics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.83, - "details": { - "description": "min=0.83, mean=0.83, max=0.83, sum=1.66 (2)", - "tab": "Accuracy", - "Business Ethics - Observed inference time (s)": "{\"description\": \"min=1.018, mean=1.018, max=1.018, sum=2.035 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.0176324987411498\"}", - "Business Ethics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Business Ethics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Business Ethics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Business Ethics - # prompt tokens": "{\"description\": \"min=559.31, mean=559.31, max=559.31, sum=1118.62 (2)\", \"tab\": \"General information\", \"score\": \"559.31\"}", - "Business Ethics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"business_ethics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_business_ethics\"" - } - } - }, - { - "evaluation_name": "Clinical Knowledge", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Clinical Knowledge", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.804, - "details": { - "description": "min=0.804, mean=0.804, max=0.804, sum=1.608 (2)", - "tab": "Accuracy", - "Clinical Knowledge - Observed inference time (s)": "{\"description\": \"min=0.954, mean=0.954, max=0.954, sum=1.909 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9543584787620688\"}", - "Clinical Knowledge - # eval": "{\"description\": \"min=265, mean=265, max=265, sum=530 (2)\", \"tab\": \"General information\", \"score\": \"265.0\"}", - "Clinical Knowledge - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Clinical Knowledge - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Clinical Knowledge - # prompt tokens": "{\"description\": \"min=394.77, mean=394.77, max=394.77, sum=789.54 (2)\", \"tab\": \"General information\", \"score\": \"394.76981132075474\"}", - "Clinical Knowledge - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"clinical_knowledge\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_clinical_knowledge\"" - } - } - }, - { - "evaluation_name": "Conceptual Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Conceptual Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.809, - "details": { - "description": "min=0.809, mean=0.809, max=0.809, sum=1.617 (2)", - "tab": "Accuracy", - "Conceptual Physics - Observed inference time (s)": "{\"description\": \"min=0.834, mean=0.834, max=0.834, sum=1.667 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8336589884250722\"}", - "Conceptual Physics - # eval": "{\"description\": \"min=235, mean=235, max=235, sum=470 (2)\", \"tab\": \"General information\", \"score\": \"235.0\"}", - "Conceptual Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Conceptual Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Conceptual Physics - # prompt tokens": "{\"description\": \"min=309.477, mean=309.477, max=309.477, sum=618.953 (2)\", \"tab\": \"General information\", \"score\": \"309.4765957446809\"}", - "Conceptual Physics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"conceptual_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_conceptual_physics\"" - } - } - }, - { - "evaluation_name": "Electrical Engineering", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Electrical Engineering", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.772, - "details": { - "description": "min=0.772, mean=0.772, max=0.772, sum=1.545 (2)", - "tab": "Accuracy", - "Electrical Engineering - Observed inference time (s)": "{\"description\": \"min=1.064, mean=1.064, max=1.064, sum=2.128 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.0639554155283961\"}", - "Electrical Engineering - # eval": "{\"description\": \"min=145, mean=145, max=145, sum=290 (2)\", \"tab\": \"General information\", \"score\": \"145.0\"}", - "Electrical Engineering - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Electrical Engineering - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Electrical Engineering - # prompt tokens": "{\"description\": \"min=480.524, mean=480.524, max=480.524, sum=961.048 (2)\", \"tab\": \"General information\", \"score\": \"480.5241379310345\"}", - "Electrical Engineering - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"electrical_engineering\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_electrical_engineering\"" - } - } - }, - { - "evaluation_name": "Elementary Mathematics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Elementary Mathematics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.661, - "details": { - "description": "min=0.661, mean=0.661, max=0.661, sum=1.323 (2)", - "tab": "Accuracy", - "Elementary Mathematics - Observed inference time (s)": "{\"description\": \"min=1.026, mean=1.026, max=1.026, sum=2.052 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.0261994568759172\"}", - "Elementary Mathematics - # eval": "{\"description\": \"min=378, mean=378, max=378, sum=756 (2)\", \"tab\": \"General information\", \"score\": \"378.0\"}", - "Elementary Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Elementary Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Elementary Mathematics - # prompt tokens": "{\"description\": \"min=599.828, mean=599.828, max=599.828, sum=1199.656 (2)\", \"tab\": \"General information\", \"score\": \"599.8280423280423\"}", - "Elementary Mathematics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"elementary_mathematics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_elementary_mathematics\"" - } - } - }, - { - "evaluation_name": "Formal Logic", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Formal Logic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.659, - "details": { - "description": "min=0.659, mean=0.659, max=0.659, sum=1.317 (2)", - "tab": "Accuracy", - "Formal Logic - Observed inference time (s)": "{\"description\": \"min=1.016, mean=1.016, max=1.016, sum=2.032 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.0157842484731523\"}", - "Formal Logic - # eval": "{\"description\": \"min=126, mean=126, max=126, sum=252 (2)\", \"tab\": \"General information\", \"score\": \"126.0\"}", - "Formal Logic - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Formal Logic - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Formal Logic - # prompt tokens": "{\"description\": \"min=623.508, mean=623.508, max=623.508, sum=1247.016 (2)\", \"tab\": \"General information\", \"score\": \"623.5079365079365\"}", - "Formal Logic - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"formal_logic\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_formal_logic\"" - } - } - }, - { - "evaluation_name": "High School World History", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on High School World History", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.911, - "details": { - "description": "min=0.911, mean=0.911, max=0.911, sum=1.823 (2)", - "tab": "Accuracy", - "High School Biology - Observed inference time (s)": "{\"description\": \"min=1.026, mean=1.026, max=1.026, sum=2.052 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.026222055189071\"}", - "High School Chemistry - Observed inference time (s)": "{\"description\": \"min=1.054, mean=1.054, max=1.054, sum=2.109 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.054317417990398\"}", - "High School Computer Science - Observed inference time (s)": "{\"description\": \"min=1.519, mean=1.519, max=1.519, sum=3.039 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.519298493862152\"}", - "High School European History - Observed inference time (s)": "{\"description\": \"min=2.108, mean=2.108, max=2.108, sum=4.215 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.107529640197754\"}", - "High School Geography - Observed inference time (s)": "{\"description\": \"min=1.159, mean=1.159, max=1.159, sum=2.319 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.1594982544581096\"}", - "High School Government And Politics - Observed inference time (s)": "{\"description\": \"min=1.056, mean=1.056, max=1.056, sum=2.112 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.0561638829621627\"}", - "High School Macroeconomics - Observed inference time (s)": "{\"description\": \"min=1.016, mean=1.016, max=1.016, sum=2.033 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.0163854268880992\"}", - "High School Mathematics - Observed inference time (s)": "{\"description\": \"min=1.018, mean=1.018, max=1.018, sum=2.036 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.0180342506479334\"}", - "High School Microeconomics - Observed inference time (s)": "{\"description\": \"min=0.905, mean=0.905, max=0.905, sum=1.811 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9054926122937884\"}", - "High School Physics - Observed inference time (s)": "{\"description\": \"min=1.252, mean=1.252, max=1.252, sum=2.503 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.2517439276966829\"}", - "High School Psychology - Observed inference time (s)": "{\"description\": \"min=0.954, mean=0.954, max=0.954, sum=1.909 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9543260762450891\"}", - "High School Statistics - Observed inference time (s)": "{\"description\": \"min=1.329, mean=1.329, max=1.329, sum=2.657 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.3287169370386336\"}", - "High School US History - Observed inference time (s)": "{\"description\": \"min=2.056, mean=2.056, max=2.056, sum=4.112 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.0560385222528494\"}", - "High School World History - Observed inference time (s)": "{\"description\": \"min=1.276, mean=1.276, max=1.276, sum=2.553 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.2764891250224053\"}", - "High School Biology - # eval": "{\"description\": \"min=310, mean=310, max=310, sum=620 (2)\", \"tab\": \"General information\", \"score\": \"310.0\"}", - "High School Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Biology - # prompt tokens": "{\"description\": \"min=501.255, mean=501.255, max=501.255, sum=1002.51 (2)\", \"tab\": \"General information\", \"score\": \"501.2548387096774\"}", - "High School Biology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # eval": "{\"description\": \"min=203, mean=203, max=203, sum=406 (2)\", \"tab\": \"General information\", \"score\": \"203.0\"}", - "High School Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # prompt tokens": "{\"description\": \"min=515.473, mean=515.473, max=515.473, sum=1030.946 (2)\", \"tab\": \"General information\", \"score\": \"515.4729064039409\"}", - "High School Chemistry - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "High School Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # prompt tokens": "{\"description\": \"min=954.08, mean=954.08, max=954.08, sum=1908.16 (2)\", \"tab\": \"General information\", \"score\": \"954.08\"}", - "High School Computer Science - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # eval": "{\"description\": \"min=165, mean=165, max=165, sum=330 (2)\", \"tab\": \"General information\", \"score\": \"165.0\"}", - "High School European History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School European History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # prompt tokens": "{\"description\": \"min=2823.23, mean=2823.23, max=2823.23, sum=5646.461 (2)\", \"tab\": \"General information\", \"score\": \"2823.230303030303\"}", - "High School European History - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # eval": "{\"description\": \"min=198, mean=198, max=198, sum=396 (2)\", \"tab\": \"General information\", \"score\": \"198.0\"}", - "High School Geography - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Geography - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # prompt tokens": "{\"description\": \"min=392.939, mean=392.939, max=392.939, sum=785.879 (2)\", \"tab\": \"General information\", \"score\": \"392.93939393939394\"}", - "High School Geography - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # eval": "{\"description\": \"min=193, mean=193, max=193, sum=386 (2)\", \"tab\": \"General information\", \"score\": \"193.0\"}", - "High School Government And Politics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Government And Politics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # prompt tokens": "{\"description\": \"min=475.44, mean=475.44, max=475.44, sum=950.881 (2)\", \"tab\": \"General information\", \"score\": \"475.440414507772\"}", - "High School Government And Politics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # eval": "{\"description\": \"min=390, mean=390, max=390, sum=780 (2)\", \"tab\": \"General information\", \"score\": \"390.0\"}", - "High School Macroeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Macroeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # prompt tokens": "{\"description\": \"min=395.962, mean=395.962, max=395.962, sum=791.923 (2)\", \"tab\": \"General information\", \"score\": \"395.96153846153845\"}", - "High School Macroeconomics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # eval": "{\"description\": \"min=270, mean=270, max=270, sum=540 (2)\", \"tab\": \"General information\", \"score\": \"270.0\"}", - "High School Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # prompt tokens": "{\"description\": \"min=580.393, mean=580.393, max=580.393, sum=1160.785 (2)\", \"tab\": \"General information\", \"score\": \"580.3925925925926\"}", - "High School Mathematics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # eval": "{\"description\": \"min=238, mean=238, max=238, sum=476 (2)\", \"tab\": \"General information\", \"score\": \"238.0\"}", - "High School Microeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Microeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # prompt tokens": "{\"description\": \"min=414.361, mean=414.361, max=414.361, sum=828.723 (2)\", \"tab\": \"General information\", \"score\": \"414.3613445378151\"}", - "High School Microeconomics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # eval": "{\"description\": \"min=151, mean=151, max=151, sum=302 (2)\", \"tab\": \"General information\", \"score\": \"151.0\"}", - "High School Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # prompt tokens": "{\"description\": \"min=592.252, mean=592.252, max=592.252, sum=1184.503 (2)\", \"tab\": \"General information\", \"score\": \"592.2516556291391\"}", - "High School Physics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # eval": "{\"description\": \"min=545, mean=545, max=545, sum=1090 (2)\", \"tab\": \"General information\", \"score\": \"545.0\"}", - "High School Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # prompt tokens": "{\"description\": \"min=496.51, mean=496.51, max=496.51, sum=993.02 (2)\", \"tab\": \"General information\", \"score\": \"496.5100917431193\"}", - "High School Psychology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # eval": "{\"description\": \"min=216, mean=216, max=216, sum=432 (2)\", \"tab\": \"General information\", \"score\": \"216.0\"}", - "High School Statistics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Statistics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # prompt tokens": "{\"description\": \"min=860.532, mean=860.532, max=860.532, sum=1721.065 (2)\", \"tab\": \"General information\", \"score\": \"860.5324074074074\"}", - "High School Statistics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # eval": "{\"description\": \"min=204, mean=204, max=204, sum=408 (2)\", \"tab\": \"General information\", \"score\": \"204.0\"}", - "High School US History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School US History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # prompt tokens": "{\"description\": \"min=2239.544, mean=2239.544, max=2239.544, sum=4479.088 (2)\", \"tab\": \"General information\", \"score\": \"2239.544117647059\"}", - "High School US History - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # eval": "{\"description\": \"min=237, mean=237, max=237, sum=474 (2)\", \"tab\": \"General information\", \"score\": \"237.0\"}", - "High School World History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School World History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # prompt tokens": "{\"description\": \"min=1437.051, mean=1437.051, max=1437.051, sum=2874.101 (2)\", \"tab\": \"General information\", \"score\": \"1437.0506329113923\"}", - "High School World History - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"high_school_world_history\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_high_school_world_history\"" - } - } - }, - { - "evaluation_name": "Human Sexuality", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Human Sexuality", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.924, - "details": { - "description": "min=0.924, mean=0.924, max=0.924, sum=1.847 (2)", - "tab": "Accuracy", - "Human Aging - Observed inference time (s)": "{\"description\": \"min=0.884, mean=0.884, max=0.884, sum=1.768 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8839223662833996\"}", - "Human Sexuality - Observed inference time (s)": "{\"description\": \"min=1.095, mean=1.095, max=1.095, sum=2.191 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.0953879956980699\"}", - "Human Aging - # eval": "{\"description\": \"min=223, mean=223, max=223, sum=446 (2)\", \"tab\": \"General information\", \"score\": \"223.0\"}", - "Human Aging - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Aging - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Aging - # prompt tokens": "{\"description\": \"min=323.906, mean=323.906, max=323.906, sum=647.812 (2)\", \"tab\": \"General information\", \"score\": \"323.90582959641256\"}", - "Human Aging - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # eval": "{\"description\": \"min=131, mean=131, max=131, sum=262 (2)\", \"tab\": \"General information\", \"score\": \"131.0\"}", - "Human Sexuality - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Sexuality - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # prompt tokens": "{\"description\": \"min=338.74, mean=338.74, max=338.74, sum=677.481 (2)\", \"tab\": \"General information\", \"score\": \"338.74045801526717\"}", - "Human Sexuality - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"human_sexuality\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_human_sexuality\"" - } - } - }, - { - "evaluation_name": "International Law", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on International Law", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.909, - "details": { - "description": "min=0.909, mean=0.909, max=0.909, sum=1.818 (2)", - "tab": "Accuracy", - "International Law - Observed inference time (s)": "{\"description\": \"min=1.104, mean=1.104, max=1.104, sum=2.208 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.1039516984923812\"}", - "International Law - # eval": "{\"description\": \"min=121, mean=121, max=121, sum=242 (2)\", \"tab\": \"General information\", \"score\": \"121.0\"}", - "International Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "International Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "International Law - # prompt tokens": "{\"description\": \"min=651.686, mean=651.686, max=651.686, sum=1303.372 (2)\", \"tab\": \"General information\", \"score\": \"651.6859504132232\"}", - "International Law - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"international_law\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_international_law\"" - } - } - }, - { - "evaluation_name": "Logical Fallacies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Logical Fallacies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.877, - "details": { - "description": "min=0.877, mean=0.877, max=0.877, sum=1.755 (2)", - "tab": "Accuracy", - "Logical Fallacies - Observed inference time (s)": "{\"description\": \"min=1.094, mean=1.094, max=1.094, sum=2.188 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.0941538839983793\"}", - "Logical Fallacies - # eval": "{\"description\": \"min=163, mean=163, max=163, sum=326 (2)\", \"tab\": \"General information\", \"score\": \"163.0\"}", - "Logical Fallacies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Logical Fallacies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Logical Fallacies - # prompt tokens": "{\"description\": \"min=443.969, mean=443.969, max=443.969, sum=887.939 (2)\", \"tab\": \"General information\", \"score\": \"443.96932515337426\"}", - "Logical Fallacies - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"logical_fallacies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_logical_fallacies\"" - } - } - }, - { - "evaluation_name": "Machine Learning", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Machine Learning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.625, - "details": { - "description": "min=0.625, mean=0.625, max=0.625, sum=1.25 (2)", - "tab": "Accuracy", - "Machine Learning - Observed inference time (s)": "{\"description\": \"min=1.11, mean=1.11, max=1.11, sum=2.22 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.110024324485234\"}", - "Machine Learning - # eval": "{\"description\": \"min=112, mean=112, max=112, sum=224 (2)\", \"tab\": \"General information\", \"score\": \"112.0\"}", - "Machine Learning - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Machine Learning - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Machine Learning - # prompt tokens": "{\"description\": \"min=705.973, mean=705.973, max=705.973, sum=1411.946 (2)\", \"tab\": \"General information\", \"score\": \"705.9732142857143\"}", - "Machine Learning - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"machine_learning\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_machine_learning\"" - } - } - }, - { - "evaluation_name": "Management", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Management", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.903, - "details": { - "description": "min=0.903, mean=0.903, max=0.903, sum=1.806 (2)", - "tab": "Accuracy", - "Management - Observed inference time (s)": "{\"description\": \"min=1.154, mean=1.154, max=1.154, sum=2.308 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.153875772235463\"}", - "Management - # eval": "{\"description\": \"min=103, mean=103, max=103, sum=206 (2)\", \"tab\": \"General information\", \"score\": \"103.0\"}", - "Management - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Management - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Management - # prompt tokens": "{\"description\": \"min=284.68, mean=284.68, max=284.68, sum=569.359 (2)\", \"tab\": \"General information\", \"score\": \"284.6796116504854\"}", - "Management - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"management\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_management\"" - } - } - }, - { - "evaluation_name": "Marketing", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Marketing", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.94, - "details": { - "description": "min=0.94, mean=0.94, max=0.94, sum=1.88 (2)", - "tab": "Accuracy", - "Marketing - Observed inference time (s)": "{\"description\": \"min=1.031, mean=1.031, max=1.031, sum=2.063 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.0312827428181965\"}", - "Marketing - # eval": "{\"description\": \"min=234, mean=234, max=234, sum=468 (2)\", \"tab\": \"General information\", \"score\": \"234.0\"}", - "Marketing - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Marketing - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Marketing - # prompt tokens": "{\"description\": \"min=428.726, mean=428.726, max=428.726, sum=857.453 (2)\", \"tab\": \"General information\", \"score\": \"428.7264957264957\"}", - "Marketing - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"marketing\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_marketing\"" - } - } - }, - { - "evaluation_name": "Medical Genetics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Medical Genetics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.83, - "details": { - "description": "min=0.83, mean=0.83, max=0.83, sum=1.66 (2)", - "tab": "Accuracy", - "Medical Genetics - Observed inference time (s)": "{\"description\": \"min=1.068, mean=1.068, max=1.068, sum=2.136 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.0681284523010255\"}", - "Medical Genetics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Medical Genetics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Medical Genetics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Medical Genetics - # prompt tokens": "{\"description\": \"min=334.69, mean=334.69, max=334.69, sum=669.38 (2)\", \"tab\": \"General information\", \"score\": \"334.69\"}", - "Medical Genetics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"medical_genetics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_medical_genetics\"" - } - } - }, - { - "evaluation_name": "Miscellaneous", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Miscellaneous", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.894, - "details": { - "description": "min=0.894, mean=0.894, max=0.894, sum=1.788 (2)", - "tab": "Accuracy", - "Miscellaneous - Observed inference time (s)": "{\"description\": \"min=0.894, mean=0.894, max=0.894, sum=1.788 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8939257733818824\"}", - "Miscellaneous - # eval": "{\"description\": \"min=783, mean=783, max=783, sum=1566 (2)\", \"tab\": \"General information\", \"score\": \"783.0\"}", - "Miscellaneous - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Miscellaneous - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Miscellaneous - # prompt tokens": "{\"description\": \"min=325.215, mean=325.215, max=325.215, sum=650.429 (2)\", \"tab\": \"General information\", \"score\": \"325.2145593869732\"}", - "Miscellaneous - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"miscellaneous\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_miscellaneous\"" - } - } - }, - { - "evaluation_name": "Moral Scenarios", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Moral Scenarios", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.562, - "details": { - "description": "min=0.562, mean=0.562, max=0.562, sum=1.124 (2)", - "tab": "Accuracy", - "Moral Disputes - Observed inference time (s)": "{\"description\": \"min=0.988, mean=0.988, max=0.988, sum=1.976 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9880901995421834\"}", - "Moral Scenarios - Observed inference time (s)": "{\"description\": \"min=0.968, mean=0.968, max=0.968, sum=1.935 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9677273009742439\"}", - "Moral Disputes - # eval": "{\"description\": \"min=346, mean=346, max=346, sum=692 (2)\", \"tab\": \"General information\", \"score\": \"346.0\"}", - "Moral Disputes - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Disputes - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Disputes - # prompt tokens": "{\"description\": \"min=494.63, mean=494.63, max=494.63, sum=989.26 (2)\", \"tab\": \"General information\", \"score\": \"494.6300578034682\"}", - "Moral Disputes - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # eval": "{\"description\": \"min=895, mean=895, max=895, sum=1790 (2)\", \"tab\": \"General information\", \"score\": \"895.0\"}", - "Moral Scenarios - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Scenarios - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # prompt tokens": "{\"description\": \"min=687.566, mean=687.566, max=687.566, sum=1375.133 (2)\", \"tab\": \"General information\", \"score\": \"687.5664804469274\"}", - "Moral Scenarios - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"moral_scenarios\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_moral_scenarios\"" - } - } - }, - { - "evaluation_name": "Nutrition", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Nutrition", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.856, - "details": { - "description": "min=0.856, mean=0.856, max=0.856, sum=1.712 (2)", - "tab": "Accuracy", - "Nutrition - Observed inference time (s)": "{\"description\": \"min=0.912, mean=0.912, max=0.912, sum=1.824 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9120152238147711\"}", - "Nutrition - # eval": "{\"description\": \"min=306, mean=306, max=306, sum=612 (2)\", \"tab\": \"General information\", \"score\": \"306.0\"}", - "Nutrition - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Nutrition - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Nutrition - # prompt tokens": "{\"description\": \"min=589.663, mean=589.663, max=589.663, sum=1179.327 (2)\", \"tab\": \"General information\", \"score\": \"589.6633986928105\"}", - "Nutrition - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"nutrition\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_nutrition\"" - } - } - }, - { - "evaluation_name": "Prehistory", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Prehistory", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.87, - "details": { - "description": "min=0.87, mean=0.87, max=0.87, sum=1.741 (2)", - "tab": "Accuracy", - "Prehistory - Observed inference time (s)": "{\"description\": \"min=0.916, mean=0.916, max=0.916, sum=1.831 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9155398577819636\"}", - "Prehistory - # eval": "{\"description\": \"min=324, mean=324, max=324, sum=648 (2)\", \"tab\": \"General information\", \"score\": \"324.0\"}", - "Prehistory - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Prehistory - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Prehistory - # prompt tokens": "{\"description\": \"min=538.179, mean=538.179, max=538.179, sum=1076.358 (2)\", \"tab\": \"General information\", \"score\": \"538.179012345679\"}", - "Prehistory - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"prehistory\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_prehistory\"" - } - } - }, - { - "evaluation_name": "Public Relations", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Public Relations", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.773, - "details": { - "description": "min=0.773, mean=0.773, max=0.773, sum=1.545 (2)", - "tab": "Accuracy", - "Public Relations - Observed inference time (s)": "{\"description\": \"min=0.79, mean=0.79, max=0.79, sum=1.579 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7896393559195779\"}", - "Public Relations - # eval": "{\"description\": \"min=110, mean=110, max=110, sum=220 (2)\", \"tab\": \"General information\", \"score\": \"110.0\"}", - "Public Relations - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Public Relations - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Public Relations - # prompt tokens": "{\"description\": \"min=426.982, mean=426.982, max=426.982, sum=853.964 (2)\", \"tab\": \"General information\", \"score\": \"426.9818181818182\"}", - "Public Relations - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"public_relations\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_public_relations\"" - } - } - }, - { - "evaluation_name": "Security Studies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Security Studies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.829, - "details": { - "description": "min=0.829, mean=0.829, max=0.829, sum=1.657 (2)", - "tab": "Accuracy", - "Security Studies - Observed inference time (s)": "{\"description\": \"min=1.254, mean=1.254, max=1.254, sum=2.508 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.2542338507516044\"}", - "Security Studies - # eval": "{\"description\": \"min=245, mean=245, max=245, sum=490 (2)\", \"tab\": \"General information\", \"score\": \"245.0\"}", - "Security Studies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Security Studies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Security Studies - # prompt tokens": "{\"description\": \"min=1185.8, mean=1185.8, max=1185.8, sum=2371.6 (2)\", \"tab\": \"General information\", \"score\": \"1185.8\"}", - "Security Studies - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"security_studies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_security_studies\"" - } - } - }, - { - "evaluation_name": "Sociology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Sociology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.91, - "details": { - "description": "min=0.91, mean=0.91, max=0.91, sum=1.821 (2)", - "tab": "Accuracy", - "Sociology - Observed inference time (s)": "{\"description\": \"min=0.84, mean=0.84, max=0.84, sum=1.681 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8403987184685854\"}", - "Sociology - # eval": "{\"description\": \"min=201, mean=201, max=201, sum=402 (2)\", \"tab\": \"General information\", \"score\": \"201.0\"}", - "Sociology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Sociology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Sociology - # prompt tokens": "{\"description\": \"min=459.642, mean=459.642, max=459.642, sum=919.284 (2)\", \"tab\": \"General information\", \"score\": \"459.64179104477614\"}", - "Sociology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"sociology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_sociology\"" - } - } - }, - { - "evaluation_name": "Virology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Virology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.572, - "details": { - "description": "min=0.572, mean=0.572, max=0.572, sum=1.145 (2)", - "tab": "Accuracy", - "Virology - Observed inference time (s)": "{\"description\": \"min=1.029, mean=1.029, max=1.029, sum=2.059 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.0293473134557884\"}", - "Virology - # eval": "{\"description\": \"min=166, mean=166, max=166, sum=332 (2)\", \"tab\": \"General information\", \"score\": \"166.0\"}", - "Virology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Virology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Virology - # prompt tokens": "{\"description\": \"min=337.06, mean=337.06, max=337.06, sum=674.12 (2)\", \"tab\": \"General information\", \"score\": \"337.06024096385545\"}", - "Virology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"virology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_virology\"" - } - } - }, - { - "evaluation_name": "World Religions", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on World Religions", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.877, - "details": { - "description": "min=0.877, mean=0.877, max=0.877, sum=1.754 (2)", - "tab": "Accuracy", - "World Religions - Observed inference time (s)": "{\"description\": \"min=0.963, mean=0.963, max=0.963, sum=1.926 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9628847495854249\"}", - "World Religions - # eval": "{\"description\": \"min=171, mean=171, max=171, sum=342 (2)\", \"tab\": \"General information\", \"score\": \"171.0\"}", - "World Religions - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "World Religions - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "World Religions - # prompt tokens": "{\"description\": \"min=270.187, mean=270.187, max=270.187, sum=540.374 (2)\", \"tab\": \"General information\", \"score\": \"270.187134502924\"}", - "World Religions - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"world_religions\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_world_religions\"" - } - } - }, - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.142, - "details": { - "description": "", - "tab": "Efficiency" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/google_ul2-20b.json b/data/models/google_ul2-20b.json deleted file mode 100644 index 9055a57d20e5f596aa7e5264c1a7235433a5f52a..0000000000000000000000000000000000000000 --- a/data/models/google_ul2-20b.json +++ /dev/null @@ -1,674 +0,0 @@ -{ - "model_info": { - "name": "UL2 20B", - "id": "google/UL2-20B", - "developer": "Google", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_classic/google_UL2-20B/1774096308.339228", - "retrieved_timestamp": "1774096308.339228", - "source_metadata": { - "source_name": "helm_classic", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_classic", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_classic", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperform on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.167, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Calibration": "{\"description\": \"\", \"tab\": \"Calibration\", \"score\": \"0.464477335800185\"}", - "Mean win rate - Robustness": "{\"description\": \"\", \"tab\": \"Robustness\", \"score\": \"0.2572027972027972\"}", - "Mean win rate - Fairness": "{\"description\": \"\", \"tab\": \"Fairness\", \"score\": \"0.1858974358974359\"}", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.5056944444444444\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}", - "Mean win rate - Bias": "{\"description\": \"\", \"tab\": \"Bias\", \"score\": \"0.5601766236691538\"}", - "Mean win rate - Toxicity": "{\"description\": \"\", \"tab\": \"Toxicity\", \"score\": \"0.2902378485711819\"}", - "Mean win rate - Summarization metrics": "{\"description\": \"\", \"tab\": \"Summarization metrics\", \"score\": \"0.11842105263157894\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.291, - "details": { - "description": "min=0.2, mean=0.291, max=0.39, sum=4.368 (15)", - "tab": "Accuracy", - "MMLU - ECE (10-bin)": "{\"description\": \"min=0.084, mean=0.134, max=0.202, sum=2.004 (15)\", \"tab\": \"Calibration\", \"score\": \"0.13362255376880447\"}", - "MMLU - EM (Robustness)": "{\"description\": \"min=0.2, mean=0.272, max=0.37, sum=4.079 (15)\", \"tab\": \"Robustness\", \"score\": \"0.2719415204678362\"}", - "MMLU - EM (Fairness)": "{\"description\": \"min=0.19, mean=0.273, max=0.36, sum=4.102 (15)\", \"tab\": \"Fairness\", \"score\": \"0.2734502923976609\"}", - "MMLU - Denoised inference time (s)": "{\"description\": \"min=0.178, mean=0.182, max=0.184, sum=2.725 (15)\", \"tab\": \"Efficiency\", \"score\": \"0.18164482078684702\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=1542 (15)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=2.465, mean=4.316, max=5, sum=64.743 (15)\", \"tab\": \"General information\", \"score\": \"4.316222222222222\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (15)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=385.228, mean=423.395, max=467.79, sum=6350.919 (15)\", \"tab\": \"General information\", \"score\": \"423.39457309941525\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=15 (15)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "MMLU - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=45 (15)\", \"tab\": \"General information\", \"score\": \"3.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "BoolQ", - "source_data": { - "dataset_name": "BoolQ", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on BoolQ", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.746, - "details": { - "description": "min=0.717, mean=0.746, max=0.762, sum=2.237 (3)", - "tab": "Accuracy", - "BoolQ - ECE (10-bin)": "{\"description\": \"min=0.416, mean=0.46, max=0.512, sum=1.379 (3)\", \"tab\": \"Calibration\", \"score\": \"0.45980755585445926\"}", - "BoolQ - EM (Robustness)": "{\"description\": \"min=0.638, mean=0.646, max=0.651, sum=1.938 (3)\", \"tab\": \"Robustness\", \"score\": \"0.646\"}", - "BoolQ - EM (Fairness)": "{\"description\": \"min=0.672, mean=0.698, max=0.714, sum=2.095 (3)\", \"tab\": \"Fairness\", \"score\": \"0.6983333333333334\"}", - "BoolQ - Denoised inference time (s)": "{\"description\": \"min=0.292, mean=0.313, max=0.341, sum=0.938 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.3127442524572212\"}", - "BoolQ - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "BoolQ - # train": "{\"description\": \"min=0.953, mean=1.57, max=1.978, sum=4.709 (3)\", \"tab\": \"General information\", \"score\": \"1.5696666666666668\"}", - "BoolQ - truncated": "{\"description\": \"min=0.004, mean=0.004, max=0.004, sum=0.012 (3)\", \"tab\": \"General information\", \"score\": \"0.004\"}", - "BoolQ - # prompt tokens": "{\"description\": \"min=386.826, mean=402.285, max=424.449, sum=1206.854 (3)\", \"tab\": \"General information\", \"score\": \"402.2846666666667\"}", - "BoolQ - # output tokens": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "BoolQ - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "BoolQ - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (gender)": "{\"description\": \"min=0.167, mean=0.23, max=0.357, sum=0.69 (3)\", \"tab\": \"Bias\", \"score\": \"0.23015873015873015\"}", - "BoolQ - Toxic fraction": "{\"description\": \"min=0.001, mean=0.001, max=0.001, sum=0.003 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.001\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.083, - "details": { - "description": "min=0.083, mean=0.083, max=0.083, sum=0.248 (3)", - "tab": "Accuracy", - "NarrativeQA - ECE (10-bin)": "{\"description\": \"min=0.0, mean=0.0, max=0.0, sum=0.0 (3)\", \"tab\": \"Calibration\", \"score\": \"4.840114578300129e-06\"}", - "NarrativeQA - F1 (Robustness)": "{\"description\": \"min=0.059, mean=0.059, max=0.059, sum=0.178 (3)\", \"tab\": \"Robustness\", \"score\": \"0.05920683866208649\"}", - "NarrativeQA - F1 (Fairness)": "{\"description\": \"min=0.053, mean=0.053, max=0.053, sum=0.159 (3)\", \"tab\": \"Fairness\", \"score\": \"0.05305645886768214\"}", - "NarrativeQA - Denoised inference time (s)": "{\"description\": \"min=1.182, mean=1.182, max=1.182, sum=3.546 (3)\", \"tab\": \"Efficiency\", \"score\": \"1.1820060481894892\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=1065 (3)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0.834, mean=0.834, max=0.834, sum=2.501 (3)\", \"tab\": \"General information\", \"score\": \"0.8338028169014086\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=492.876, mean=492.876, max=492.876, sum=1478.628 (3)\", \"tab\": \"General information\", \"score\": \"492.87605633802815\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=100, mean=100, max=100, sum=300 (3)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "NarrativeQA - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NarrativeQA - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=2 (3)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "NarrativeQA - Stereotypes (gender)": "{\"description\": \"min=0.337, mean=0.337, max=0.337, sum=1.01 (3)\", \"tab\": \"Bias\", \"score\": \"0.3368016513369257\"}", - "NarrativeQA - Representation (race)": "{\"description\": \"min=0.342, mean=0.342, max=0.342, sum=1.026 (3)\", \"tab\": \"Bias\", \"score\": \"0.3419913419913419\"}", - "NarrativeQA - Representation (gender)": "{\"description\": \"min=0.154, mean=0.154, max=0.154, sum=0.462 (3)\", \"tab\": \"Bias\", \"score\": \"0.15399534522885955\"}", - "NarrativeQA - Toxic fraction": "{\"description\": \"min=0.017, mean=0.017, max=0.017, sum=0.051 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.016901408450704224\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (open-book)", - "source_data": { - "dataset_name": "NaturalQuestions (open-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (open-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.349, - "details": { - "description": "min=0.195, mean=0.349, max=0.432, sum=1.048 (3)", - "tab": "Accuracy", - "NaturalQuestions (closed-book) - ECE (10-bin)": "{\"description\": \"min=0.088, mean=0.092, max=0.095, sum=0.276 (3)\", \"tab\": \"Calibration\", \"score\": \"0.09200000000000001\"}", - "NaturalQuestions (open-book) - ECE (10-bin)": "{\"description\": \"min=0.028, mean=0.179, max=0.258, sum=0.537 (3)\", \"tab\": \"Calibration\", \"score\": \"0.17899999902043598\"}", - "NaturalQuestions (closed-book) - F1 (Robustness)": "{\"description\": \"min=0.139, mean=0.141, max=0.143, sum=0.423 (3)\", \"tab\": \"Robustness\", \"score\": \"0.1409495030072503\"}", - "NaturalQuestions (open-book) - F1 (Robustness)": "{\"description\": \"min=0.154, mean=0.291, max=0.365, sum=0.872 (3)\", \"tab\": \"Robustness\", \"score\": \"0.2906387285430619\"}", - "NaturalQuestions (closed-book) - F1 (Fairness)": "{\"description\": \"min=0.159, mean=0.162, max=0.167, sum=0.486 (3)\", \"tab\": \"Fairness\", \"score\": \"0.16184307849771043\"}", - "NaturalQuestions (open-book) - F1 (Fairness)": "{\"description\": \"min=0.153, mean=0.303, max=0.389, sum=0.908 (3)\", \"tab\": \"Fairness\", \"score\": \"0.30281096844711025\"}", - "NaturalQuestions (closed-book) - Denoised inference time (s)": "{\"description\": \"min=1.912, mean=1.994, max=2.142, sum=5.981 (3)\", \"tab\": \"Efficiency\", \"score\": \"1.993551874854462\"}", - "NaturalQuestions (open-book) - Denoised inference time (s)": "{\"description\": \"min=2.941, mean=3.093, max=3.306, sum=9.279 (3)\", \"tab\": \"Efficiency\", \"score\": \"3.0931644739895567\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=113.556, mean=117.556, max=122.556, sum=352.668 (3)\", \"tab\": \"General information\", \"score\": \"117.556\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=300, mean=300, max=300, sum=900 (3)\", \"tab\": \"General information\", \"score\": \"300.0\"}", - "NaturalQuestions (closed-book) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=0.083, mean=0.918, max=1.789, sum=2.755 (3)\", \"tab\": \"General information\", \"score\": \"0.9183333333333333\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.097, mean=0.355, max=0.852, sum=1.064 (3)\", \"tab\": \"General information\", \"score\": \"0.3546666666666667\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=231.47, mean=303.619, max=343.479, sum=910.857 (3)\", \"tab\": \"General information\", \"score\": \"303.61899999999997\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=300, mean=300, max=300, sum=900 (3)\", \"tab\": \"General information\", \"score\": \"300.0\"}", - "NaturalQuestions (open-book) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NaturalQuestions (closed-book) - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=0.667 (1)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "NaturalQuestions (closed-book) - Stereotypes (gender)": "{\"description\": \"min=0.333, mean=0.387, max=0.44, sum=1.162 (3)\", \"tab\": \"Bias\", \"score\": \"0.3874074074074074\"}", - "NaturalQuestions (closed-book) - Representation (race)": "{\"description\": \"min=0.444, mean=0.519, max=0.562, sum=1.558 (3)\", \"tab\": \"Bias\", \"score\": \"0.5194689485314483\"}", - "NaturalQuestions (closed-book) - Representation (gender)": "{\"description\": \"min=0.079, mean=0.183, max=0.239, sum=0.549 (3)\", \"tab\": \"Bias\", \"score\": \"0.1829490113242974\"}", - "NaturalQuestions (open-book) - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=2 (3)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "NaturalQuestions (open-book) - Stereotypes (gender)": "{\"description\": \"min=0.41, mean=0.449, max=0.5, sum=1.346 (3)\", \"tab\": \"Bias\", \"score\": \"0.44858553791887124\"}", - "NaturalQuestions (open-book) - Representation (race)": "{\"description\": \"min=0.451, mean=0.538, max=0.595, sum=1.615 (3)\", \"tab\": \"Bias\", \"score\": \"0.5381999649472214\"}", - "NaturalQuestions (open-book) - Representation (gender)": "{\"description\": \"min=0.069, mean=0.111, max=0.136, sum=0.332 (3)\", \"tab\": \"Bias\", \"score\": \"0.11064384639781977\"}", - "NaturalQuestions (closed-book) - Toxic fraction": "{\"description\": \"min=0.001, mean=0.001, max=0.001, sum=0.003 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.001\"}", - "NaturalQuestions (open-book) - Toxic fraction": "{\"description\": \"min=0, mean=0.001, max=0.002, sum=0.003 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.001\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "QuAC", - "source_data": { - "dataset_name": "QuAC", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on QuAC", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.144, - "details": { - "description": "min=0.144, mean=0.144, max=0.144, sum=0.433 (3)", - "tab": "Accuracy", - "QuAC - ECE (10-bin)": "{\"description\": \"min=0.0, mean=0.0, max=0.0, sum=0.0 (3)\", \"tab\": \"Calibration\", \"score\": \"0.00013015946539738277\"}", - "QuAC - F1 (Robustness)": "{\"description\": \"min=0.111, mean=0.111, max=0.111, sum=0.333 (3)\", \"tab\": \"Robustness\", \"score\": \"0.11096938073772407\"}", - "QuAC - F1 (Fairness)": "{\"description\": \"min=0.107, mean=0.107, max=0.107, sum=0.32 (3)\", \"tab\": \"Fairness\", \"score\": \"0.10672699918485114\"}", - "QuAC - Denoised inference time (s)": "{\"description\": \"min=1.226, mean=1.226, max=1.226, sum=3.679 (3)\", \"tab\": \"Efficiency\", \"score\": \"1.2264695519389521\"}", - "QuAC - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "QuAC - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "QuAC - truncated": "{\"description\": \"min=0.999, mean=0.999, max=0.999, sum=2.997 (3)\", \"tab\": \"General information\", \"score\": \"0.999\"}", - "QuAC - # prompt tokens": "{\"description\": \"min=510.938, mean=510.938, max=510.938, sum=1532.814 (3)\", \"tab\": \"General information\", \"score\": \"510.93799999999993\"}", - "QuAC - # output tokens": "{\"description\": \"min=100, mean=100, max=100, sum=300 (3)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "QuAC - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "QuAC - Stereotypes (race)": "{\"description\": \"min=0.614, mean=0.614, max=0.614, sum=1.843 (3)\", \"tab\": \"Bias\", \"score\": \"0.6143486267149368\"}", - "QuAC - Stereotypes (gender)": "{\"description\": \"min=0.402, mean=0.402, max=0.402, sum=1.207 (3)\", \"tab\": \"Bias\", \"score\": \"0.40228575253954807\"}", - "QuAC - Representation (race)": "{\"description\": \"min=0.317, mean=0.317, max=0.317, sum=0.951 (3)\", \"tab\": \"Bias\", \"score\": \"0.3169129720853858\"}", - "QuAC - Representation (gender)": "{\"description\": \"min=0.253, mean=0.253, max=0.253, sum=0.758 (3)\", \"tab\": \"Bias\", \"score\": \"0.2525635309852876\"}", - "QuAC - Toxic fraction": "{\"description\": \"min=0.006, mean=0.006, max=0.006, sum=0.018 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.006000000000000001\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "HellaSwag", - "source_data": { - "dataset_name": "HellaSwag", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on HellaSwag", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "HellaSwag - ECE (10-bin)": "{\"description\": \"No matching runs\", \"tab\": \"Calibration\", \"score\": \"\"}", - "HellaSwag - EM (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "HellaSwag - EM (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "HellaSwag - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "HellaSwag - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "OpenbookQA - ECE (10-bin)": "{\"description\": \"No matching runs\", \"tab\": \"Calibration\", \"score\": \"\"}", - "OpenbookQA - EM (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "OpenbookQA - EM (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "OpenbookQA - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "OpenbookQA - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "TruthfulQA", - "source_data": { - "dataset_name": "TruthfulQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on TruthfulQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.193, - "details": { - "description": "min=0.162, mean=0.193, max=0.232, sum=0.772 (4)", - "tab": "Accuracy", - "TruthfulQA - ECE (10-bin)": "{\"description\": \"min=0.096, mean=0.125, max=0.139, sum=0.498 (4)\", \"tab\": \"Calibration\", \"score\": \"0.12460869505528777\"}", - "TruthfulQA - EM (Robustness)": "{\"description\": \"min=0.162, mean=0.178, max=0.209, sum=0.711 (4)\", \"tab\": \"Robustness\", \"score\": \"0.17775229357798167\"}", - "TruthfulQA - EM (Fairness)": "{\"description\": \"min=0.15, mean=0.162, max=0.176, sum=0.647 (4)\", \"tab\": \"Fairness\", \"score\": \"0.16169724770642202\"}", - "TruthfulQA - Denoised inference time (s)": "{\"description\": \"min=0.122, mean=0.168, max=0.183, sum=0.671 (4)\", \"tab\": \"Efficiency\", \"score\": \"0.16779271445154526\"}", - "TruthfulQA - # eval": "{\"description\": \"min=654, mean=654, max=654, sum=2616 (4)\", \"tab\": \"General information\", \"score\": \"654.0\"}", - "TruthfulQA - # train": "{\"description\": \"min=0, mean=3.513, max=4.838, sum=14.05 (4)\", \"tab\": \"General information\", \"score\": \"3.5126146788990824\"}", - "TruthfulQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (4)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "TruthfulQA - # prompt tokens": "{\"description\": \"min=89.896, mean=372.668, max=473.333, sum=1490.671 (4)\", \"tab\": \"General information\", \"score\": \"372.66781345565744\"}", - "TruthfulQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=4 (4)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "TruthfulQA - # trials": "{\"description\": \"min=1, mean=2.5, max=3, sum=10 (4)\", \"tab\": \"General information\", \"score\": \"2.5\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MS MARCO (TREC)", - "source_data": { - "dataset_name": "MS MARCO (TREC)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "NDCG@10 on MS MARCO (TREC)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "MS MARCO (regular) - RR@10 (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "MS MARCO (TREC) - NDCG@10 (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "MS MARCO (regular) - RR@10 (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "MS MARCO (TREC) - NDCG@10 (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "MS MARCO (regular) - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MS MARCO (TREC) - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MS MARCO (regular) - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "MS MARCO (TREC) - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CNN/DailyMail", - "source_data": { - "dataset_name": "CNN/DailyMail", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on CNN/DailyMail", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.03, - "details": { - "description": "min=0.03, mean=0.03, max=0.03, sum=0.182 (6)", - "tab": "Accuracy", - "CNN/DailyMail - Denoised inference time (s)": "{\"description\": \"min=1.108, mean=1.108, max=1.109, sum=6.651 (6)\", \"tab\": \"Efficiency\", \"score\": \"1.1084291968542619\"}", - "CNN/DailyMail - # eval": "{\"description\": \"min=466, mean=466, max=466, sum=2796 (6)\", \"tab\": \"General information\", \"score\": \"466.0\"}", - "CNN/DailyMail - # train": "{\"description\": \"min=0.06, mean=0.061, max=0.062, sum=0.365 (6)\", \"tab\": \"General information\", \"score\": \"0.060801144492131615\"}", - "CNN/DailyMail - truncated": "{\"description\": \"min=0.933, mean=0.935, max=0.936, sum=5.609 (6)\", \"tab\": \"General information\", \"score\": \"0.9349070100143061\"}", - "CNN/DailyMail - # prompt tokens": "{\"description\": \"min=500.788, mean=500.829, max=500.912, sum=3004.974 (6)\", \"tab\": \"General information\", \"score\": \"500.8290414878398\"}", - "CNN/DailyMail - # output tokens": "{\"description\": \"min=128, mean=128, max=128, sum=768 (6)\", \"tab\": \"General information\", \"score\": \"128.0\"}", - "CNN/DailyMail - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=18 (6)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "CNN/DailyMail - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=4 (6)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "CNN/DailyMail - Stereotypes (gender)": "{\"description\": \"min=0.402, mean=0.402, max=0.402, sum=2.411 (6)\", \"tab\": \"Bias\", \"score\": \"0.4018787714810442\"}", - "CNN/DailyMail - Representation (race)": "{\"description\": \"min=0.361, mean=0.361, max=0.361, sum=2.163 (6)\", \"tab\": \"Bias\", \"score\": \"0.3605442176870748\"}", - "CNN/DailyMail - Representation (gender)": "{\"description\": \"min=0.188, mean=0.188, max=0.188, sum=1.129 (6)\", \"tab\": \"Bias\", \"score\": \"0.1882129277566539\"}", - "CNN/DailyMail - Toxic fraction": "{\"description\": \"min=0.009, mean=0.009, max=0.009, sum=0.052 (6)\", \"tab\": \"Toxicity\", \"score\": \"0.008583690987124463\"}", - "CNN/DailyMail - SummaC": "{\"description\": \"min=-0.27, mean=-0.27, max=-0.27, sum=-0.81 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"-0.2698551726198464\"}", - "CNN/DailyMail - QAFactEval": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - BERTScore (F1)": "{\"description\": \"min=-0.122, mean=-0.121, max=-0.12, sum=-0.362 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"-0.12078049146748136\"}", - "CNN/DailyMail - Coverage": "{\"description\": \"min=0.72, mean=0.72, max=0.72, sum=4.319 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"0.7197585278365729\"}", - "CNN/DailyMail - Density": "{\"description\": \"min=5.044, mean=5.044, max=5.044, sum=30.265 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"5.044183333839311\"}", - "CNN/DailyMail - Compression": "{\"description\": \"min=7.173, mean=7.186, max=7.2, sum=43.118 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"7.186281356409094\"}", - "CNN/DailyMail - HumanEval-faithfulness": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-relevance": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-coherence": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "XSUM", - "source_data": { - "dataset_name": "XSUM", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on XSUM", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.058, - "details": { - "description": "min=0.049, mean=0.058, max=0.066, sum=0.345 (6)", - "tab": "Accuracy", - "XSUM - Denoised inference time (s)": "{\"description\": \"min=0.771, mean=0.774, max=0.781, sum=4.646 (6)\", \"tab\": \"Efficiency\", \"score\": \"0.7743015579914415\"}", - "XSUM - # eval": "{\"description\": \"min=518, mean=518, max=518, sum=3108 (6)\", \"tab\": \"General information\", \"score\": \"518.0\"}", - "XSUM - # train": "{\"description\": \"min=0.234, mean=0.293, max=0.361, sum=1.761 (6)\", \"tab\": \"General information\", \"score\": \"0.29343629343629346\"}", - "XSUM - truncated": "{\"description\": \"min=0.614, mean=0.677, max=0.736, sum=4.062 (6)\", \"tab\": \"General information\", \"score\": \"0.676962676962677\"}", - "XSUM - # prompt tokens": "{\"description\": \"min=433.917, mean=437.97, max=442.292, sum=2627.819 (6)\", \"tab\": \"General information\", \"score\": \"437.96975546975546\"}", - "XSUM - # output tokens": "{\"description\": \"min=64, mean=64, max=64, sum=384 (6)\", \"tab\": \"General information\", \"score\": \"64.0\"}", - "XSUM - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=18 (6)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "XSUM - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=4 (6)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "XSUM - Stereotypes (gender)": "{\"description\": \"min=0.45, mean=0.455, max=0.463, sum=2.729 (6)\", \"tab\": \"Bias\", \"score\": \"0.45478395061728394\"}", - "XSUM - Representation (race)": "{\"description\": \"min=0.489, mean=0.524, max=0.556, sum=3.145 (6)\", \"tab\": \"Bias\", \"score\": \"0.5241150528821762\"}", - "XSUM - Representation (gender)": "{\"description\": \"min=0.236, mean=0.251, max=0.262, sum=1.508 (6)\", \"tab\": \"Bias\", \"score\": \"0.251389993488347\"}", - "XSUM - Toxic fraction": "{\"description\": \"min=0, mean=0.001, max=0.002, sum=0.004 (6)\", \"tab\": \"Toxicity\", \"score\": \"0.0006435006435006435\"}", - "XSUM - SummaC": "{\"description\": \"min=-0.28, mean=-0.275, max=-0.272, sum=-0.826 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"-0.2753430534988641\"}", - "XSUM - QAFactEval": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - BERTScore (F1)": "{\"description\": \"min=0.028, mean=0.072, max=0.121, sum=0.215 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.07156637071699196\"}", - "XSUM - Coverage": "{\"description\": \"min=0.617, mean=0.643, max=0.671, sum=3.856 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"0.6426528869383965\"}", - "XSUM - Density": "{\"description\": \"min=3.058, mean=3.208, max=3.428, sum=19.25 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"3.2083925287601787\"}", - "XSUM - Compression": "{\"description\": \"min=7.31, mean=7.853, max=8.427, sum=47.12 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"7.853257861418139\"}", - "XSUM - HumanEval-faithfulness": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-relevance": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-coherence": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "IMDB", - "source_data": { - "dataset_name": "IMDB", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on IMDB", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.337, - "details": { - "description": "min=0.13, mean=0.337, max=0.556, sum=1.01 (3)", - "tab": "Accuracy", - "IMDB - ECE (10-bin)": "{\"description\": \"min=0.037, mean=0.225, max=0.41, sum=0.675 (3)\", \"tab\": \"Calibration\", \"score\": \"0.22500123786419848\"}", - "IMDB - EM (Robustness)": "{\"description\": \"min=0.091, mean=0.276, max=0.485, sum=0.827 (3)\", \"tab\": \"Robustness\", \"score\": \"0.27566666666666667\"}", - "IMDB - EM (Fairness)": "{\"description\": \"min=0.092, mean=0.271, max=0.484, sum=0.814 (3)\", \"tab\": \"Fairness\", \"score\": \"0.2713333333333333\"}", - "IMDB - Denoised inference time (s)": "{\"description\": \"min=0.214, mean=0.215, max=0.217, sum=0.645 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.21490736543138858\"}", - "IMDB - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "IMDB - # train": "{\"description\": \"min=0.309, mean=0.449, max=0.689, sum=1.347 (3)\", \"tab\": \"General information\", \"score\": \"0.449\"}", - "IMDB - truncated": "{\"description\": \"min=0.175, mean=0.176, max=0.176, sum=0.527 (3)\", \"tab\": \"General information\", \"score\": \"0.17566666666666664\"}", - "IMDB - # prompt tokens": "{\"description\": \"min=388.254, mean=407.098, max=435.686, sum=1221.293 (3)\", \"tab\": \"General information\", \"score\": \"407.0976666666666\"}", - "IMDB - # output tokens": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "IMDB - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "IMDB - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CivilComments", - "source_data": { - "dataset_name": "CivilComments", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on CivilComments", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.521, - "details": { - "description": "min=0, mean=0.521, max=1, sum=28.146 (54)", - "tab": "Accuracy", - "CivilComments - ECE (10-bin)": "{\"description\": \"min=0.123, mean=0.404, max=0.585, sum=21.802 (54)\", \"tab\": \"Calibration\", \"score\": \"0.40373338964571226\"}", - "CivilComments - EM (Robustness)": "{\"description\": \"min=0, mean=0.45, max=0.983, sum=24.293 (54)\", \"tab\": \"Robustness\", \"score\": \"0.4498711194026963\"}", - "CivilComments - EM (Fairness)": "{\"description\": \"min=0, mean=0.423, max=0.975, sum=22.816 (54)\", \"tab\": \"Fairness\", \"score\": \"0.4225225679997762\"}", - "CivilComments - Denoised inference time (s)": "{\"description\": \"min=0.21, mean=0.264, max=0.45, sum=14.236 (54)\", \"tab\": \"Efficiency\", \"score\": \"0.2636334561494892\"}", - "CivilComments - # eval": "{\"description\": \"min=74, mean=371.556, max=683, sum=20064 (54)\", \"tab\": \"General information\", \"score\": \"371.55555555555554\"}", - "CivilComments - # train": "{\"description\": \"min=1.01, mean=2.608, max=4.878, sum=140.857 (54)\", \"tab\": \"General information\", \"score\": \"2.608459470057463\"}", - "CivilComments - truncated": "{\"description\": \"min=0, mean=0.003, max=0.032, sum=0.138 (54)\", \"tab\": \"General information\", \"score\": \"0.0025500084787325617\"}", - "CivilComments - # prompt tokens": "{\"description\": \"min=335.768, mean=416.896, max=479.235, sum=22512.361 (54)\", \"tab\": \"General information\", \"score\": \"416.89557696196465\"}", - "CivilComments - # output tokens": "{\"description\": \"min=5, mean=5, max=5, sum=270 (54)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CivilComments - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=162 (54)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "CivilComments - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (54)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "RAFT", - "source_data": { - "dataset_name": "RAFT", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on RAFT", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.404, - "details": { - "description": "min=0, mean=0.404, max=0.95, sum=13.325 (33)", - "tab": "Accuracy", - "RAFT - ECE (10-bin)": "{\"description\": \"min=0.0, mean=0.401, max=0.95, sum=13.228 (33)\", \"tab\": \"Calibration\", \"score\": \"0.40084433515818857\"}", - "RAFT - EM (Robustness)": "{\"description\": \"min=0, mean=0.349, max=0.95, sum=11.525 (33)\", \"tab\": \"Robustness\", \"score\": \"0.3492424242424242\"}", - "RAFT - EM (Fairness)": "{\"description\": \"min=0, mean=0.375, max=0.95, sum=12.375 (33)\", \"tab\": \"Fairness\", \"score\": \"0.375\"}", - "RAFT - Denoised inference time (s)": "{\"description\": \"min=0.316, mean=0.434, max=0.454, sum=14.32 (33)\", \"tab\": \"Efficiency\", \"score\": \"0.43394225670679076\"}", - "RAFT - # eval": "{\"description\": \"min=40, mean=40, max=40, sum=1320 (33)\", \"tab\": \"General information\", \"score\": \"40.0\"}", - "RAFT - # train": "{\"description\": \"min=0, mean=2.433, max=5, sum=80.3 (33)\", \"tab\": \"General information\", \"score\": \"2.433333333333333\"}", - "RAFT - truncated": "{\"description\": \"min=0, mean=0.394, max=1, sum=13 (33)\", \"tab\": \"General information\", \"score\": \"0.3939393939393939\"}", - "RAFT - # prompt tokens": "{\"description\": \"min=267.4, mean=423.537, max=511, sum=13976.725 (33)\", \"tab\": \"General information\", \"score\": \"423.53712121212124\"}", - "RAFT - # output tokens": "{\"description\": \"min=30, mean=30, max=30, sum=990 (33)\", \"tab\": \"General information\", \"score\": \"30.0\"}", - "RAFT - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=99 (33)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "RAFT - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (gender)": "{\"description\": \"min=0.079, mean=0.079, max=0.079, sum=0.237 (3)\", \"tab\": \"Bias\", \"score\": \"0.07894736842105265\"}", - "RAFT - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (33)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/google_umt5-base.json b/data/models/google_umt5-base.json deleted file mode 100644 index 3f0608d7015dc52d0d4d65d067527aa588fdc104..0000000000000000000000000000000000000000 --- a/data/models/google_umt5-base.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "umt5-base", - "id": "google/umt5-base", - "developer": "Google", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "UMT5ForConditionalGeneration", - "params_billions": "-1.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/google_umt5-base/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1746 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2788 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0045 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2542 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3382 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1078 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/gotocompany_gemma2-9b-cpt-sahabatai-v1-instruct.json b/data/models/gotocompany_gemma2-9b-cpt-sahabatai-v1-instruct.json deleted file mode 100644 index 7d370bab9ca06baed01cc8d7ef4e56541e49c6ae..0000000000000000000000000000000000000000 --- a/data/models/gotocompany_gemma2-9b-cpt-sahabatai-v1-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "gemma2-9b-cpt-sahabatai-v1-instruct", - "id": "GoToCompany/gemma2-9b-cpt-sahabatai-v1-instruct", - "developer": "GoToCompany", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "9.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/GoToCompany_gemma2-9b-cpt-sahabatai-v1-instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6551 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5955 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2054 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3347 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4779 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4264 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/gotocompany_llama3-8b-cpt-sahabatai-v1-instruct.json b/data/models/gotocompany_llama3-8b-cpt-sahabatai-v1-instruct.json deleted file mode 100644 index 3b484a6fe16a5db7413d19d49f982c33a40190d4..0000000000000000000000000000000000000000 --- a/data/models/gotocompany_llama3-8b-cpt-sahabatai-v1-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "llama3-8b-cpt-sahabatai-v1-instruct", - "id": "GoToCompany/llama3-8b-cpt-sahabatai-v1-instruct", - "developer": "GoToCompany", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/GoToCompany_llama3-8b-cpt-sahabatai-v1-instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5238 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4951 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1276 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2668 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4488 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3453 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/goulue5_merging_llm.json b/data/models/goulue5_merging_llm.json deleted file mode 100644 index 3608b807fa76ff49ce36e863a28d83a67eb9a724..0000000000000000000000000000000000000000 --- a/data/models/goulue5_merging_llm.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "merging_LLM", - "id": "goulue5/merging_LLM", - "developer": "goulue5", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.544" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/goulue5_merging_LLM/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3233 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4216 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0967 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2911 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4333 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2958 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/gradientai_llama-3-8b-instruct-gradient-1048k.json b/data/models/gradientai_llama-3-8b-instruct-gradient-1048k.json deleted file mode 100644 index c4af3b1687e0708529741c144ad1787867ed827f..0000000000000000000000000000000000000000 --- a/data/models/gradientai_llama-3-8b-instruct-gradient-1048k.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-8B-Instruct-Gradient-1048k", - "id": "gradientai/Llama-3-8B-Instruct-Gradient-1048k", - "developer": "gradientai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/gradientai_Llama-3-8B-Instruct-Gradient-1048k/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4456 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4346 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0536 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2777 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4298 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.294 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/greennode_greennode-small-9b-it.json b/data/models/greennode_greennode-small-9b-it.json deleted file mode 100644 index 181a6d97ea20eff07aa4a971e7df6a5626b8c336..0000000000000000000000000000000000000000 --- a/data/models/greennode_greennode-small-9b-it.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "GreenNode-small-9B-it", - "id": "GreenNode/GreenNode-small-9B-it", - "developer": "GreenNode", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "9.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/GreenNode_GreenNode-small-9B-it/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7436 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5994 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1745 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3196 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4204 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3927 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/grimjim_deepsauerhuatuoskywork-r1-o1-llama-3.1-8b.json b/data/models/grimjim_deepsauerhuatuoskywork-r1-o1-llama-3.1-8b.json deleted file mode 100644 index e0f386af588e66bbadd76ddcbc8e7026b3080221..0000000000000000000000000000000000000000 --- a/data/models/grimjim_deepsauerhuatuoskywork-r1-o1-llama-3.1-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "DeepSauerHuatuoSkywork-R1-o1-Llama-3.1-8B", - "id": "grimjim/DeepSauerHuatuoSkywork-R1-o1-Llama-3.1-8B", - "developer": "grimjim", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/grimjim_DeepSauerHuatuoSkywork-R1-o1-Llama-3.1-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4797 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5269 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2221 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3381 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4408 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3957 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/grimjim_gigantes-v1-gemma2-9b-it.json b/data/models/grimjim_gigantes-v1-gemma2-9b-it.json deleted file mode 100644 index 71390ff9389d404b9336b59f1b92cdb59f9b9404..0000000000000000000000000000000000000000 --- a/data/models/grimjim_gigantes-v1-gemma2-9b-it.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Gigantes-v1-gemma2-9b-it", - "id": "grimjim/Gigantes-v1-gemma2-9b-it", - "developer": "grimjim", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "9.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/grimjim_Gigantes-v1-gemma2-9b-it/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6925 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5978 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2145 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3532 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4555 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4225 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/grimjim_gigantes-v2-gemma2-9b-it.json b/data/models/grimjim_gigantes-v2-gemma2-9b-it.json deleted file mode 100644 index 271aa88ba680c86d66acad44953050cdb2b973a0..0000000000000000000000000000000000000000 --- a/data/models/grimjim_gigantes-v2-gemma2-9b-it.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Gigantes-v2-gemma2-9b-it", - "id": "grimjim/Gigantes-v2-gemma2-9b-it", - "developer": "grimjim", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "9.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/grimjim_Gigantes-v2-gemma2-9b-it/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7351 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5987 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2017 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3515 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4595 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4259 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/grimjim_gigantes-v3-gemma2-9b-it.json b/data/models/grimjim_gigantes-v3-gemma2-9b-it.json deleted file mode 100644 index c758425d975b23e878ac86c3e0f5c9dd237f12de..0000000000000000000000000000000000000000 --- a/data/models/grimjim_gigantes-v3-gemma2-9b-it.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Gigantes-v3-gemma2-9b-it", - "id": "grimjim/Gigantes-v3-gemma2-9b-it", - "developer": "grimjim", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "9.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/grimjim_Gigantes-v3-gemma2-9b-it/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6976 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5984 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.21 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3565 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4608 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4226 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/grimjim_huatuoskywork-o1-llama-3.1-8b.json b/data/models/grimjim_huatuoskywork-o1-llama-3.1-8b.json deleted file mode 100644 index d579c78d9fcac954adf75a9125b17b8f14a441ae..0000000000000000000000000000000000000000 --- a/data/models/grimjim_huatuoskywork-o1-llama-3.1-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "HuatuoSkywork-o1-Llama-3.1-8B", - "id": "grimjim/HuatuoSkywork-o1-Llama-3.1-8B", - "developer": "grimjim", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/grimjim_HuatuoSkywork-o1-Llama-3.1-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3961 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4886 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3882 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2928 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3839 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3095 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/grimjim_llama-3-instruct-8b-simpo-sppo-iter3-merge.json b/data/models/grimjim_llama-3-instruct-8b-simpo-sppo-iter3-merge.json deleted file mode 100644 index f8796fc869f2c909cfbd53df8b56c42cd34836c0..0000000000000000000000000000000000000000 --- a/data/models/grimjim_llama-3-instruct-8b-simpo-sppo-iter3-merge.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-Instruct-8B-SimPO-SPPO-Iter3-merge", - "id": "grimjim/Llama-3-Instruct-8B-SimPO-SPPO-Iter3-merge", - "developer": "grimjim", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/grimjim_Llama-3-Instruct-8B-SimPO-SPPO-Iter3-merge/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6806 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5022 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0891 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2626 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3885 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3684 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/grimjim_llama-3-instruct-8b-sppo-iter3-simpo-merge.json b/data/models/grimjim_llama-3-instruct-8b-sppo-iter3-simpo-merge.json deleted file mode 100644 index cbd34500663aba6096115fa9ea9009bbab1db708..0000000000000000000000000000000000000000 --- a/data/models/grimjim_llama-3-instruct-8b-sppo-iter3-simpo-merge.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-Instruct-8B-SPPO-Iter3-SimPO-merge", - "id": "grimjim/Llama-3-Instruct-8B-SPPO-Iter3-SimPO-merge", - "developer": "grimjim", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/grimjim_Llama-3-Instruct-8B-SPPO-Iter3-SimPO-merge/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4271 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4962 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0997 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2903 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4043 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3625 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/grimjim_llama-3-nephilim-v1-8b.json b/data/models/grimjim_llama-3-nephilim-v1-8b.json deleted file mode 100644 index 4f3de196ac4d3cd24dd9cd25e3bbbf9f6300a3f1..0000000000000000000000000000000000000000 --- a/data/models/grimjim_llama-3-nephilim-v1-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "llama-3-Nephilim-v1-8B", - "id": "grimjim/llama-3-Nephilim-v1-8B", - "developer": "grimjim", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/grimjim_llama-3-Nephilim-v1-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4277 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5132 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0906 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.302 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4136 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3796 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/grimjim_llama-3-nephilim-v2-8b.json b/data/models/grimjim_llama-3-nephilim-v2-8b.json deleted file mode 100644 index 43d67f3e5ec71fb9f3077942b6db80391d68a96a..0000000000000000000000000000000000000000 --- a/data/models/grimjim_llama-3-nephilim-v2-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "llama-3-Nephilim-v2-8B", - "id": "grimjim/llama-3-Nephilim-v2-8B", - "developer": "grimjim", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/grimjim_llama-3-Nephilim-v2-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3922 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5048 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1065 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2995 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3895 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3641 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/grimjim_llama-3-nephilim-v2.1-8b.json b/data/models/grimjim_llama-3-nephilim-v2.1-8b.json deleted file mode 100644 index b329ff00412e2cc5e81efe194e090b5e45dd24a8..0000000000000000000000000000000000000000 --- a/data/models/grimjim_llama-3-nephilim-v2.1-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "llama-3-Nephilim-v2.1-8B", - "id": "grimjim/llama-3-Nephilim-v2.1-8B", - "developer": "grimjim", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/grimjim_llama-3-Nephilim-v2.1-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3895 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5095 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0997 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2995 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3935 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3644 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/grimjim_llama-3-nephilim-v3-8b.json b/data/models/grimjim_llama-3-nephilim-v3-8b.json deleted file mode 100644 index 2c06dc2b6ab3cfeded6c5751db8078bdab510039..0000000000000000000000000000000000000000 --- a/data/models/grimjim_llama-3-nephilim-v3-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "llama-3-Nephilim-v3-8B", - "id": "grimjim/llama-3-Nephilim-v3-8B", - "developer": "grimjim", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/grimjim_llama-3-Nephilim-v3-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4174 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5013 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0952 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2953 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3989 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3612 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/grimjim_llama-3.1-8b-instruct-abliterated_via_adapter.json b/data/models/grimjim_llama-3.1-8b-instruct-abliterated_via_adapter.json deleted file mode 100644 index 36454b96fa808e902adcf4d3c4d7cb1ed7c9e625..0000000000000000000000000000000000000000 --- a/data/models/grimjim_llama-3.1-8b-instruct-abliterated_via_adapter.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.1-8B-Instruct-abliterated_via_adapter", - "id": "grimjim/Llama-3.1-8B-Instruct-abliterated_via_adapter", - "developer": "grimjim", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/grimjim_Llama-3.1-8B-Instruct-abliterated_via_adapter/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.487 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5105 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1397 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3138 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.401 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3651 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/grimjim_llama-3.1-bonsaikraft-8b-instruct.json b/data/models/grimjim_llama-3.1-bonsaikraft-8b-instruct.json deleted file mode 100644 index 334b1bc9e6ffd96f4bc189979f21d649f39599e6..0000000000000000000000000000000000000000 --- a/data/models/grimjim_llama-3.1-bonsaikraft-8b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.1-Bonsaikraft-8B-Instruct", - "id": "grimjim/Llama-3.1-Bonsaikraft-8B-Instruct", - "developer": "grimjim", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/grimjim_Llama-3.1-Bonsaikraft-8B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.425 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5287 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1314 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3037 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4235 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3764 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/grimjim_llama-nephilim-metamorphosis-v2-8b.json b/data/models/grimjim_llama-nephilim-metamorphosis-v2-8b.json deleted file mode 100644 index fbd5ebdd8b5c596a1bba18dbde0d8fd7e5157341..0000000000000000000000000000000000000000 --- a/data/models/grimjim_llama-nephilim-metamorphosis-v2-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-Nephilim-Metamorphosis-v2-8B", - "id": "grimjim/Llama-Nephilim-Metamorphosis-v2-8B", - "developer": "grimjim", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/grimjim_Llama-Nephilim-Metamorphosis-v2-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4545 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5013 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1397 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.323 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4091 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3809 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/grimjim_llama3.1-supernovalite-huatuoskywork-o1-8b.json b/data/models/grimjim_llama3.1-supernovalite-huatuoskywork-o1-8b.json deleted file mode 100644 index 0e6dac82c103b1b8fd27e1a18875789179be9361..0000000000000000000000000000000000000000 --- a/data/models/grimjim_llama3.1-supernovalite-huatuoskywork-o1-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama3.1-SuperNovaLite-HuatuoSkywork-o1-8B", - "id": "grimjim/Llama3.1-SuperNovaLite-HuatuoSkywork-o1-8B", - "developer": "grimjim", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/grimjim_Llama3.1-SuperNovaLite-HuatuoSkywork-o1-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4366 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5287 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3006 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3112 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3999 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3684 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/grimjim_magnolia-v1-gemma2-8k-9b.json b/data/models/grimjim_magnolia-v1-gemma2-8k-9b.json deleted file mode 100644 index 4844001892402250d02eaac2d01a98ec87972dcb..0000000000000000000000000000000000000000 --- a/data/models/grimjim_magnolia-v1-gemma2-8k-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Magnolia-v1-Gemma2-8k-9B", - "id": "grimjim/Magnolia-v1-Gemma2-8k-9B", - "developer": "grimjim", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "9.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/grimjim_Magnolia-v1-Gemma2-8k-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3531 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5589 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1684 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3364 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4645 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4242 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/grimjim_magnolia-v2-12b.json b/data/models/grimjim_magnolia-v2-12b.json deleted file mode 100644 index 617e453900fcb0f30e6e81c9b05aee3cb10fe390..0000000000000000000000000000000000000000 --- a/data/models/grimjim_magnolia-v2-12b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Magnolia-v2-12B", - "id": "grimjim/Magnolia-v2-12B", - "developer": "grimjim", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/grimjim_Magnolia-v2-12B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3506 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.529 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1292 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3188 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4171 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3601 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/grimjim_magnolia-v2-gemma2-8k-9b.json b/data/models/grimjim_magnolia-v2-gemma2-8k-9b.json deleted file mode 100644 index 94100dee7e8a97eca14a101639bbd6c43d62d188..0000000000000000000000000000000000000000 --- a/data/models/grimjim_magnolia-v2-gemma2-8k-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Magnolia-v2-Gemma2-8k-9B", - "id": "grimjim/Magnolia-v2-Gemma2-8k-9B", - "developer": "grimjim", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "9.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/grimjim_Magnolia-v2-Gemma2-8k-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7384 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6016 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2281 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3574 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4488 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4332 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/grimjim_magnolia-v3-12b.json b/data/models/grimjim_magnolia-v3-12b.json deleted file mode 100644 index 0cc5d5c9859981b6964a356d2027b116909fec7f..0000000000000000000000000000000000000000 --- a/data/models/grimjim_magnolia-v3-12b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Magnolia-v3-12B", - "id": "grimjim/Magnolia-v3-12B", - "developer": "grimjim", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/grimjim_Magnolia-v3-12B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3965 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5327 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1352 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3255 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4184 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3615 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/grimjim_magnolia-v3-gemma2-8k-9b.json b/data/models/grimjim_magnolia-v3-gemma2-8k-9b.json deleted file mode 100644 index f4b20b4a88422f1f598d5a25d23bd911286352a3..0000000000000000000000000000000000000000 --- a/data/models/grimjim_magnolia-v3-gemma2-8k-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Magnolia-v3-Gemma2-8k-9B", - "id": "grimjim/Magnolia-v3-Gemma2-8k-9B", - "developer": "grimjim", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "9.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/grimjim_Magnolia-v3-Gemma2-8k-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7378 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6015 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2319 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3565 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4488 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4337 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/grimjim_magnolia-v4-12b.json b/data/models/grimjim_magnolia-v4-12b.json deleted file mode 100644 index af0d563b5944273dd825c8ec5b8841120df38af2..0000000000000000000000000000000000000000 --- a/data/models/grimjim_magnolia-v4-12b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Magnolia-v4-12B", - "id": "grimjim/Magnolia-v4-12B", - "developer": "grimjim", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/grimjim_Magnolia-v4-12B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3418 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5431 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1314 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.328 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4211 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3672 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/grimjim_magnolia-v5a-12b.json b/data/models/grimjim_magnolia-v5a-12b.json deleted file mode 100644 index 89ec16c89e1875286876a414dc741bee84fbe2de..0000000000000000000000000000000000000000 --- a/data/models/grimjim_magnolia-v5a-12b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Magnolia-v5a-12B", - "id": "grimjim/Magnolia-v5a-12B", - "developer": "grimjim", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/grimjim_Magnolia-v5a-12B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4114 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5312 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1375 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3221 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4145 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3601 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/grimjim_magot-v1-gemma2-8k-9b.json b/data/models/grimjim_magot-v1-gemma2-8k-9b.json deleted file mode 100644 index 16e9d08b32e3f75311376b116122f987b0b0acb3..0000000000000000000000000000000000000000 --- a/data/models/grimjim_magot-v1-gemma2-8k-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Magot-v1-Gemma2-8k-9B", - "id": "grimjim/Magot-v1-Gemma2-8k-9B", - "developer": "grimjim", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "9.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/grimjim_Magot-v1-Gemma2-8k-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2997 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6019 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0989 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3465 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4488 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4337 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/grimjim_magot-v2-gemma2-8k-9b.json b/data/models/grimjim_magot-v2-gemma2-8k-9b.json deleted file mode 100644 index 9ecf8e19da28ae6dd2fc205cf694dbe06ddb9ebe..0000000000000000000000000000000000000000 --- a/data/models/grimjim_magot-v2-gemma2-8k-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Magot-v2-Gemma2-8k-9B", - "id": "grimjim/Magot-v2-Gemma2-8k-9B", - "developer": "grimjim", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "9.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/grimjim_Magot-v2-Gemma2-8k-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7347 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5897 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2017 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.354 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4344 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4223 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/grimjim_sauerhuatuoskywork-o1-llama-3.1-8b.json b/data/models/grimjim_sauerhuatuoskywork-o1-llama-3.1-8b.json deleted file mode 100644 index c4999ec15359f5e5e530e73cc49c59235e8ca3cc..0000000000000000000000000000000000000000 --- a/data/models/grimjim_sauerhuatuoskywork-o1-llama-3.1-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SauerHuatuoSkywork-o1-Llama-3.1-8B", - "id": "grimjim/SauerHuatuoSkywork-o1-Llama-3.1-8B", - "developer": "grimjim", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/grimjim_SauerHuatuoSkywork-o1-Llama-3.1-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5219 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5222 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.173 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3213 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4527 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3991 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/gritlm_gritlm-7b-kto.json b/data/models/gritlm_gritlm-7b-kto.json deleted file mode 100644 index 97cd372cffa63ffa7858a660e052b6c4a608ce70..0000000000000000000000000000000000000000 --- a/data/models/gritlm_gritlm-7b-kto.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "GritLM-7B-KTO", - "id": "GritLM/GritLM-7B-KTO", - "developer": "GritLM", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/GritLM_GritLM-7B-KTO/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.531 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4853 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0272 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2978 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.371 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.268 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/gritlm_gritlm-8x7b-kto.json b/data/models/gritlm_gritlm-8x7b-kto.json deleted file mode 100644 index 40a6fd798183f6e3511ae333145befc6c34481f4..0000000000000000000000000000000000000000 --- a/data/models/gritlm_gritlm-8x7b-kto.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "GritLM-8x7B-KTO", - "id": "GritLM/GritLM-8x7B-KTO", - "developer": "GritLM", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MixtralForCausalLM", - "params_billions": "46.703" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/GritLM_GritLM-8x7B-KTO/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5714 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.582 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1224 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2961 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4217 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3648 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/groq_llama-3-groq-8b-tool-use.json b/data/models/groq_llama-3-groq-8b-tool-use.json deleted file mode 100644 index 669163af8a587d4deee4f04dd1e171f4fe1551f7..0000000000000000000000000000000000000000 --- a/data/models/groq_llama-3-groq-8b-tool-use.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-Groq-8B-Tool-Use", - "id": "Groq/Llama-3-Groq-8B-Tool-Use", - "developer": "Groq", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Groq_Llama-3-Groq-8B-Tool-Use/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6098 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4863 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0604 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2676 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.366 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3399 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/gryphe_pantheon-rp-1.0-8b-llama-3.json b/data/models/gryphe_pantheon-rp-1.0-8b-llama-3.json deleted file mode 100644 index b4e52accb78d3aa84fb8610f678e0cc87b99abc0..0000000000000000000000000000000000000000 --- a/data/models/gryphe_pantheon-rp-1.0-8b-llama-3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Pantheon-RP-1.0-8b-Llama-3", - "id": "Gryphe/Pantheon-RP-1.0-8b-Llama-3", - "developer": "Gryphe", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Gryphe_Pantheon-RP-1.0-8b-Llama-3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3933 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4539 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0634 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.276 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3832 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3067 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/gryphe_pantheon-rp-1.5-12b-nemo.json b/data/models/gryphe_pantheon-rp-1.5-12b-nemo.json deleted file mode 100644 index 1e2bd802dab8187956268b08bb470197ee67af0a..0000000000000000000000000000000000000000 --- a/data/models/gryphe_pantheon-rp-1.5-12b-nemo.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Pantheon-RP-1.5-12b-Nemo", - "id": "Gryphe/Pantheon-RP-1.5-12b-Nemo", - "developer": "Gryphe", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Gryphe_Pantheon-RP-1.5-12b-Nemo/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4763 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5196 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0491 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2727 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.442 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3302 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/gryphe_pantheon-rp-1.6-12b-nemo-kto.json b/data/models/gryphe_pantheon-rp-1.6-12b-nemo-kto.json deleted file mode 100644 index 5461f90fe6ae14e6fa5e2e6799331735fe69cdcf..0000000000000000000000000000000000000000 --- a/data/models/gryphe_pantheon-rp-1.6-12b-nemo-kto.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Pantheon-RP-1.6-12b-Nemo-KTO", - "id": "Gryphe/Pantheon-RP-1.6-12b-Nemo-KTO", - "developer": "Gryphe", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Gryphe_Pantheon-RP-1.6-12b-Nemo-KTO/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4636 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5277 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0529 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2953 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4248 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3382 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/gryphe_pantheon-rp-1.6-12b-nemo.json b/data/models/gryphe_pantheon-rp-1.6-12b-nemo.json deleted file mode 100644 index 3b68ca7cc0669b58e71ce42eb8e41ac31eb995d8..0000000000000000000000000000000000000000 --- a/data/models/gryphe_pantheon-rp-1.6-12b-nemo.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Pantheon-RP-1.6-12b-Nemo", - "id": "Gryphe/Pantheon-RP-1.6-12b-Nemo", - "developer": "Gryphe", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Gryphe_Pantheon-RP-1.6-12b-Nemo/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4481 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5204 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0461 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2777 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4288 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3311 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/gryphe_pantheon-rp-pure-1.6.2-22b-small.json b/data/models/gryphe_pantheon-rp-pure-1.6.2-22b-small.json deleted file mode 100644 index 82988e9041e90866608be56d051fc46456220c50..0000000000000000000000000000000000000000 --- a/data/models/gryphe_pantheon-rp-pure-1.6.2-22b-small.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Pantheon-RP-Pure-1.6.2-22b-Small", - "id": "Gryphe/Pantheon-RP-Pure-1.6.2-22b-Small", - "developer": "Gryphe", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "22.247" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Gryphe_Pantheon-RP-Pure-1.6.2-22b-Small/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6931 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5305 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2024 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3289 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3765 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3942 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/guilhermenaturaumana_nature-reason-1.2-reallysmall.json b/data/models/guilhermenaturaumana_nature-reason-1.2-reallysmall.json deleted file mode 100644 index 58b844aaea3cd18e9a2e6beb1696adc0f352442e..0000000000000000000000000000000000000000 --- a/data/models/guilhermenaturaumana_nature-reason-1.2-reallysmall.json +++ /dev/null @@ -1,275 +0,0 @@ -{ - "model_info": { - "name": "Nature-Reason-1.2-reallysmall", - "id": "GuilhermeNaturaUmana/Nature-Reason-1.2-reallysmall", - "developer": "GuilhermeNaturaUmana", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/GuilhermeNaturaUmana_Nature-Reason-1.2-reallysmall/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4791 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5649 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.25 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2995 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4439 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4408 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "hfopenllm_v2/GuilhermeNaturaUmana_Nature-Reason-1.2-reallysmall/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4985 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5645 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2576 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3003 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4373 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4429 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/gunulhona_gemma-ko-merge-peft.json b/data/models/gunulhona_gemma-ko-merge-peft.json deleted file mode 100644 index 7aaca8e76ac4d54a166ddba85afcaa38d76748c4..0000000000000000000000000000000000000000 --- a/data/models/gunulhona_gemma-ko-merge-peft.json +++ /dev/null @@ -1,275 +0,0 @@ -{ - "model_info": { - "name": "Gemma-Ko-Merge-PEFT", - "id": "Gunulhona/Gemma-Ko-Merge-PEFT", - "developer": "Gunulhona", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "?", - "params_billions": "20.318" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Gunulhona_Gemma-Ko-Merge-PEFT/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4441 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4863 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.307 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3986 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3098 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "hfopenllm_v2/Gunulhona_Gemma-Ko-Merge-PEFT/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.288 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5154 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3247 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.408 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3817 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/gunulhona_gemma-ko-merge.json b/data/models/gunulhona_gemma-ko-merge.json deleted file mode 100644 index 6a7832b6e14875f2dbe9a2ddc19321503e30e55f..0000000000000000000000000000000000000000 --- a/data/models/gunulhona_gemma-ko-merge.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Gemma-Ko-Merge", - "id": "Gunulhona/Gemma-Ko-Merge", - "developer": "Gunulhona", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Gunulhona_Gemma-Ko-Merge/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6416 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5813 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1881 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3356 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4047 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3879 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/gupta-tanish_llama-7b-dpo-baseline.json b/data/models/gupta-tanish_llama-7b-dpo-baseline.json deleted file mode 100644 index aa3d37fa32324bb299a49c5e0ca43908cda5ffe8..0000000000000000000000000000000000000000 --- a/data/models/gupta-tanish_llama-7b-dpo-baseline.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "llama-7b-dpo-baseline", - "id": "gupta-tanish/llama-7b-dpo-baseline", - "developer": "gupta-tanish", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "6.738" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/gupta-tanish_llama-7b-dpo-baseline/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2693 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3897 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0196 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2626 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4456 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2028 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/gz987_qwen2.5-7b-cabs-v0.1.json b/data/models/gz987_qwen2.5-7b-cabs-v0.1.json deleted file mode 100644 index d1f8a0f418dc7c639f372dc67f866cfc124d10d1..0000000000000000000000000000000000000000 --- a/data/models/gz987_qwen2.5-7b-cabs-v0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "qwen2.5-7b-cabs-v0.1", - "id": "gz987/qwen2.5-7b-cabs-v0.1", - "developer": "gz987", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/gz987_qwen2.5-7b-cabs-v0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7506 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5482 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4796 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3138 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4376 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4406 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/gz987_qwen2.5-7b-cabs-v0.2.json b/data/models/gz987_qwen2.5-7b-cabs-v0.2.json deleted file mode 100644 index 677fb71014d9df2cc925b4b43b49c39dd223e709..0000000000000000000000000000000000000000 --- a/data/models/gz987_qwen2.5-7b-cabs-v0.2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "qwen2.5-7b-cabs-v0.2", - "id": "gz987/qwen2.5-7b-cabs-v0.2", - "developer": "gz987", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/gz987_qwen2.5-7b-cabs-v0.2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7418 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5516 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4902 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.307 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4429 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4397 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/gz987_qwen2.5-7b-cabs-v0.3.json b/data/models/gz987_qwen2.5-7b-cabs-v0.3.json deleted file mode 100644 index d16d8b209c2167147327137df2ebe42361027b97..0000000000000000000000000000000000000000 --- a/data/models/gz987_qwen2.5-7b-cabs-v0.3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "qwen2.5-7b-cabs-v0.3", - "id": "gz987/qwen2.5-7b-cabs-v0.3", - "developer": "gz987", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/gz987_qwen2.5-7b-cabs-v0.3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.757 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5494 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4932 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.307 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.443 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4402 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/gz987_qwen2.5-7b-cabs-v0.4.json b/data/models/gz987_qwen2.5-7b-cabs-v0.4.json deleted file mode 100644 index 5be6f4aa7d5d2adb08c440fcdd9c797caa7e25f6..0000000000000000000000000000000000000000 --- a/data/models/gz987_qwen2.5-7b-cabs-v0.4.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "qwen2.5-7b-cabs-v0.4", - "id": "gz987/qwen2.5-7b-cabs-v0.4", - "developer": "gz987", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/gz987_qwen2.5-7b-cabs-v0.4/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7583 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5524 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4849 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3079 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.443 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4396 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/h2oai_h2o-danube-1.8b-chat.json b/data/models/h2oai_h2o-danube-1.8b-chat.json deleted file mode 100644 index 72eb6f93d6340b285e31d0ec50c351387c719848..0000000000000000000000000000000000000000 --- a/data/models/h2oai_h2o-danube-1.8b-chat.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "h2o-danube-1.8b-chat", - "id": "h2oai/h2o-danube-1.8b-chat", - "developer": "h2oai", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "1.831" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/h2oai_h2o-danube-1.8b-chat/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2199 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.322 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0136 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2542 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3989 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1314 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/h2oai_h2o-danube3-4b-base.json b/data/models/h2oai_h2o-danube3-4b-base.json deleted file mode 100644 index 4a5d8aed84132342d10f09ac1ffa55e41b1e823c..0000000000000000000000000000000000000000 --- a/data/models/h2oai_h2o-danube3-4b-base.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "h2o-danube3-4b-base", - "id": "h2oai/h2o-danube3-4b-base", - "developer": "h2oai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.962" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/h2oai_h2o-danube3-4b-base/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2338 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3599 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0227 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2911 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3778 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2109 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/h2oai_h2o-danube3-4b-chat.json b/data/models/h2oai_h2o-danube3-4b-chat.json deleted file mode 100644 index d6d2df895533875ff72d59f7cf5b6371d61520ea..0000000000000000000000000000000000000000 --- a/data/models/h2oai_h2o-danube3-4b-chat.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "h2o-danube3-4b-chat", - "id": "h2oai/h2o-danube3-4b-chat", - "developer": "h2oai", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.962" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/h2oai_h2o-danube3-4b-chat/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3629 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3466 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0408 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2601 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3781 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2228 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/h2oai_h2o-danube3-500m-chat.json b/data/models/h2oai_h2o-danube3-500m-chat.json deleted file mode 100644 index 4746206dee1689597209b551c1064c34861f737e..0000000000000000000000000000000000000000 --- a/data/models/h2oai_h2o-danube3-500m-chat.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "h2o-danube3-500m-chat", - "id": "h2oai/h2o-danube3-500m-chat", - "developer": "h2oai", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "0.514" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/h2oai_h2o-danube3-500m-chat/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2208 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3035 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0166 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2307 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3434 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1144 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/h2oai_h2o-danube3.1-4b-chat.json b/data/models/h2oai_h2o-danube3.1-4b-chat.json deleted file mode 100644 index c6d8c28aa1ad5942211c0e71eee8be1da5d63247..0000000000000000000000000000000000000000 --- a/data/models/h2oai_h2o-danube3.1-4b-chat.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "h2o-danube3.1-4b-chat", - "id": "h2oai/h2o-danube3.1-4b-chat", - "developer": "h2oai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.962" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/h2oai_h2o-danube3.1-4b-chat/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5021 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3608 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0332 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2852 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4102 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2719 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/haoranxu_alma-13b-r.json b/data/models/haoranxu_alma-13b-r.json deleted file mode 100644 index 5bcc875a1aff71ef93e965654c24655dbce4fac7..0000000000000000000000000000000000000000 --- a/data/models/haoranxu_alma-13b-r.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ALMA-13B-R", - "id": "haoranxu/ALMA-13B-R", - "developer": "haoranxu", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "?", - "params_billions": "13.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/haoranxu_ALMA-13B-R/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0039 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3457 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0174 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2576 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3528 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1817 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/haoranxu_llama-3-instruct-8b-cpo-simpo.json b/data/models/haoranxu_llama-3-instruct-8b-cpo-simpo.json deleted file mode 100644 index ccddb2163f90d1f2fb35024bde68bac2af4580a6..0000000000000000000000000000000000000000 --- a/data/models/haoranxu_llama-3-instruct-8b-cpo-simpo.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-Instruct-8B-CPO-SimPO", - "id": "haoranxu/Llama-3-Instruct-8B-CPO-SimPO", - "developer": "haoranxu", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/haoranxu_Llama-3-Instruct-8B-CPO-SimPO/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7046 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5048 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1027 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2928 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3567 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3686 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/haoranxu_llama-3-instruct-8b-simpo.json b/data/models/haoranxu_llama-3-instruct-8b-simpo.json deleted file mode 100644 index efb8cec9955bc7e3f358b3e814536b902ddc0ed4..0000000000000000000000000000000000000000 --- a/data/models/haoranxu_llama-3-instruct-8b-simpo.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-Instruct-8B-SimPO", - "id": "haoranxu/Llama-3-Instruct-8B-SimPO", - "developer": "haoranxu", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/haoranxu_Llama-3-Instruct-8B-SimPO/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7347 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4979 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0876 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2903 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3566 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3733 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/harbingerx_zeitgeist-3b-v1.2.json b/data/models/harbingerx_zeitgeist-3b-v1.2.json deleted file mode 100644 index 1afbd18a5c362412b2113439036f5bfd1c92bba8..0000000000000000000000000000000000000000 --- a/data/models/harbingerx_zeitgeist-3b-v1.2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Zeitgeist-3b-V1.2", - "id": "HarbingerX/Zeitgeist-3b-V1.2", - "developer": "HarbingerX", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/HarbingerX_Zeitgeist-3b-V1.2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6754 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4441 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1012 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2777 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3579 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3056 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/harbingerx_zeitgeist-3b-v1.json b/data/models/harbingerx_zeitgeist-3b-v1.json deleted file mode 100644 index 765c95ac9d7d7b0f4e70ed191cead864e9aeb7bb..0000000000000000000000000000000000000000 --- a/data/models/harbingerx_zeitgeist-3b-v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Zeitgeist-3b-V1", - "id": "HarbingerX/Zeitgeist-3b-V1", - "developer": "HarbingerX", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/HarbingerX_Zeitgeist-3b-V1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6712 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4441 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1035 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2819 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3579 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3009 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/hastagaras_l3.2-jametmini-3b-mk.iii.json b/data/models/hastagaras_l3.2-jametmini-3b-mk.iii.json deleted file mode 100644 index 60c2dab7eca2003ada2c3ac8728d0534074e2891..0000000000000000000000000000000000000000 --- a/data/models/hastagaras_l3.2-jametmini-3b-mk.iii.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "L3.2-JametMini-3B-MK.III", - "id": "Hastagaras/L3.2-JametMini-3B-MK.III", - "developer": "Hastagaras", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Hastagaras_L3.2-JametMini-3B-MK.III/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6183 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4539 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1458 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2827 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3686 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2983 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/hastagaras_llama-3.1-jamet-8b-mk.i.json b/data/models/hastagaras_llama-3.1-jamet-8b-mk.i.json deleted file mode 100644 index 6c4115a6ddebd040d684c75614a8b93d2f37e807..0000000000000000000000000000000000000000 --- a/data/models/hastagaras_llama-3.1-jamet-8b-mk.i.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.1-Jamet-8B-MK.I", - "id": "Hastagaras/Llama-3.1-Jamet-8B-MK.I", - "developer": "Hastagaras", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Hastagaras_Llama-3.1-Jamet-8B-MK.I/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7338 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5049 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1269 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2743 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3726 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3482 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/hastagaras_zabuza-8b-llama-3.1.json b/data/models/hastagaras_zabuza-8b-llama-3.1.json deleted file mode 100644 index 5ea435fbcb2905abc835dacfff8e89e93c1c1e7b..0000000000000000000000000000000000000000 --- a/data/models/hastagaras_zabuza-8b-llama-3.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Zabuza-8B-Llama-3.1", - "id": "Hastagaras/Zabuza-8B-Llama-3.1", - "developer": "Hastagaras", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Hastagaras_Zabuza-8B-Llama-3.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6265 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4539 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0551 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2643 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3568 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2923 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/hatemmahmoud_qwen2.5-1.5b-sft-raft-grpo-hra-doc.json b/data/models/hatemmahmoud_qwen2.5-1.5b-sft-raft-grpo-hra-doc.json deleted file mode 100644 index d631ff7dcda8cb28d6a1cc971808845d647a7b26..0000000000000000000000000000000000000000 --- a/data/models/hatemmahmoud_qwen2.5-1.5b-sft-raft-grpo-hra-doc.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "qwen2.5-1.5b-sft-raft-grpo-hra-doc", - "id": "hatemmahmoud/qwen2.5-1.5b-sft-raft-grpo-hra-doc", - "developer": "hatemmahmoud", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.544" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/hatemmahmoud_qwen2.5-1.5b-sft-raft-grpo-hra-doc/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4196 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.427 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2175 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2676 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.361 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2776 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/helpingai_cipher-20b.json b/data/models/helpingai_cipher-20b.json deleted file mode 100644 index 337634193b29c457b64f419df1d469dd5fd0785a..0000000000000000000000000000000000000000 --- a/data/models/helpingai_cipher-20b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Cipher-20B", - "id": "HelpingAI/Cipher-20B", - "developer": "HelpingAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "20.551" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/HelpingAI_Cipher-20B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5378 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6032 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1994 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2953 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4003 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3744 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/helpingai_dhanishtha-large.json b/data/models/helpingai_dhanishtha-large.json deleted file mode 100644 index 4796c5039ffa5d3a41b2b1f0e51f0bda30c86ad8..0000000000000000000000000000000000000000 --- a/data/models/helpingai_dhanishtha-large.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Dhanishtha-Large", - "id": "HelpingAI/Dhanishtha-Large", - "developer": "HelpingAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.613" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/HelpingAI_Dhanishtha-Large/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2457 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4604 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3852 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3029 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3845 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2755 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/helpingai_priya-10b.json b/data/models/helpingai_priya-10b.json deleted file mode 100644 index 431e25804b262cb5eff08920ea683fa9511984e1..0000000000000000000000000000000000000000 --- a/data/models/helpingai_priya-10b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Priya-10B", - "id": "HelpingAI/Priya-10B", - "developer": "HelpingAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "10.211" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/HelpingAI_Priya-10B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4043 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4441 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0189 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2559 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3793 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2493 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/helpingai_priya-3b.json b/data/models/helpingai_priya-3b.json deleted file mode 100644 index 3af90efaa51dc675cf3eaa8dc761cf91343f49bd..0000000000000000000000000000000000000000 --- a/data/models/helpingai_priya-3b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Priya-3B", - "id": "HelpingAI/Priya-3B", - "developer": "HelpingAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "2.81" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/HelpingAI_Priya-3B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4526 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3961 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0144 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2567 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3713 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2339 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/hendrydong_mistral-rm-for-raft-gshf-v0.json b/data/models/hendrydong_mistral-rm-for-raft-gshf-v0.json deleted file mode 100644 index 357f438b07831c53712dd63f870b6b2401c4d681..0000000000000000000000000000000000000000 --- a/data/models/hendrydong_mistral-rm-for-raft-gshf-v0.json +++ /dev/null @@ -1,294 +0,0 @@ -{ - "model_info": { - "name": "hendrydong/Mistral-RM-for-RAFT-GSHF-v0", - "id": "hendrydong/Mistral-RM-for-RAFT-GSHF-v0", - "developer": "hendrydong", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/hendrydong_Mistral-RM-for-RAFT-GSHF-v0/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5851 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5779 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3625 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6011 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6956 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6747 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5988 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "reward-bench/hendrydong_Mistral-RM-for-RAFT-GSHF-v0/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7847 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9832 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5789 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.85 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7434 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7508 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/heraihench_deepseek-r1-qwen-coder-8b.json b/data/models/heraihench_deepseek-r1-qwen-coder-8b.json deleted file mode 100644 index 2520b980f51cae222371b156481d6a16d3a0523b..0000000000000000000000000000000000000000 --- a/data/models/heraihench_deepseek-r1-qwen-coder-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "DeepSeek-R1-Qwen-Coder-8B", - "id": "HeraiHench/DeepSeek-R1-Qwen-Coder-8B", - "developer": "HeraiHench", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "8.164" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/HeraiHench_DeepSeek-R1-Qwen-Coder-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1869 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2913 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2601 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3738 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1123 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/heraihench_double-down-qwen-math-7b.json b/data/models/heraihench_double-down-qwen-math-7b.json deleted file mode 100644 index 3fca87eb2385e88975e6a44c905d976dbfd79c58..0000000000000000000000000000000000000000 --- a/data/models/heraihench_double-down-qwen-math-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Double-Down-Qwen-Math-7B", - "id": "HeraiHench/Double-Down-Qwen-Math-7B", - "developer": "HeraiHench", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/HeraiHench_Double-Down-Qwen-Math-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.167 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2845 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0008 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2651 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3737 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1112 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/heraihench_marge-qwen-math-7b.json b/data/models/heraihench_marge-qwen-math-7b.json deleted file mode 100644 index 5a98b7525575957039490898109ccd181fe24a6f..0000000000000000000000000000000000000000 --- a/data/models/heraihench_marge-qwen-math-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Marge-Qwen-Math-7B", - "id": "HeraiHench/Marge-Qwen-Math-7B", - "developer": "HeraiHench", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/HeraiHench_Marge-Qwen-Math-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1262 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3069 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0053 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2391 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3939 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1056 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/heraihench_phi-4-slerp-reasoningrp-14b.json b/data/models/heraihench_phi-4-slerp-reasoningrp-14b.json deleted file mode 100644 index 0faf9f584f7efcf4402d700e945bafb3992c89e0..0000000000000000000000000000000000000000 --- a/data/models/heraihench_phi-4-slerp-reasoningrp-14b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Phi-4-slerp-ReasoningRP-14B", - "id": "HeraiHench/Phi-4-slerp-ReasoningRP-14B", - "developer": "HeraiHench", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "9.207" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/HeraiHench_Phi-4-slerp-ReasoningRP-14B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1575 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4196 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2936 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3116 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.19 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/hfxm_ramo-llama3.1-8b.json b/data/models/hfxm_ramo-llama3.1-8b.json deleted file mode 100644 index 0f7ba8769a7d7cff4c3c315d49d2527bb9367896..0000000000000000000000000000000000000000 --- a/data/models/hfxm_ramo-llama3.1-8b.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "HFXM/RAMO-Llama3.1-8B", - "id": "HFXM/RAMO-Llama3.1-8B", - "developer": "HFXM", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/HFXM_RAMO-Llama3.1-8B/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6917 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6547 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.375 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5628 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9756 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9071 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6752 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/hirosekoichi_llama-salad-4x8b-v3.json b/data/models/hirosekoichi_llama-salad-4x8b-v3.json deleted file mode 100644 index 710e31823494e3052665f73c72e9259189856d44..0000000000000000000000000000000000000000 --- a/data/models/hirosekoichi_llama-salad-4x8b-v3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-Salad-4x8B-V3", - "id": "HiroseKoichi/Llama-Salad-4x8B-V3", - "developer": "HiroseKoichi", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MixtralForCausalLM", - "params_billions": "24.942" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/HiroseKoichi_Llama-Salad-4x8B-V3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6654 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5245 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0959 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3029 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.374 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3518 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/hoangha_pensez-llama3.1-8b.json b/data/models/hoangha_pensez-llama3.1-8b.json deleted file mode 100644 index 188a909634e8945224ef0b00e954279116155cb5..0000000000000000000000000000000000000000 --- a/data/models/hoangha_pensez-llama3.1-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Pensez-Llama3.1-8B", - "id": "HoangHa/Pensez-Llama3.1-8B", - "developer": "HoangHa", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/HoangHa_Pensez-Llama3.1-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3887 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4669 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1148 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2886 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3597 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3126 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/hon9kon9ize_cantonesellmchat-v0.5.json b/data/models/hon9kon9ize_cantonesellmchat-v0.5.json deleted file mode 100644 index c42f4f763f62756cef86d6f49033998a0780dbbf..0000000000000000000000000000000000000000 --- a/data/models/hon9kon9ize_cantonesellmchat-v0.5.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "CantoneseLLMChat-v0.5", - "id": "hon9kon9ize/CantoneseLLMChat-v0.5", - "developer": "hon9kon9ize", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "6.069" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/hon9kon9ize_CantoneseLLMChat-v0.5/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3231 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4345 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0415 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2777 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4706 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2504 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/hon9kon9ize_cantonesellmchat-v1.0-7b.json b/data/models/hon9kon9ize_cantonesellmchat-v1.0-7b.json deleted file mode 100644 index fac2d815582cda378fe728e9b5d0a2a8aa1a012f..0000000000000000000000000000000000000000 --- a/data/models/hon9kon9ize_cantonesellmchat-v1.0-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "CantoneseLLMChat-v1.0-7B", - "id": "hon9kon9ize/CantoneseLLMChat-v1.0-7B", - "developer": "hon9kon9ize", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/hon9kon9ize_CantoneseLLMChat-v1.0-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4455 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4866 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2107 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3221 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3883 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3785 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/hongbai12_li-0.4-pre.json b/data/models/hongbai12_li-0.4-pre.json deleted file mode 100644 index 0f56dd44cad24b761214ad6c6138ab3816767263..0000000000000000000000000000000000000000 --- a/data/models/hongbai12_li-0.4-pre.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "li-0.4-pre", - "id": "hongbai12/li-0.4-pre", - "developer": "hongbai12", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/hongbai12_li-0.4-pre/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.52 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6298 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4924 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.323 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4513 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5015 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/hotmailuser_deepseek-qwen-modelstock-2b.json b/data/models/hotmailuser_deepseek-qwen-modelstock-2b.json deleted file mode 100644 index 838897f9d9b8668b90d0062dfc98a0636c541328..0000000000000000000000000000000000000000 --- a/data/models/hotmailuser_deepseek-qwen-modelstock-2b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Deepseek-qwen-modelstock-2B", - "id": "hotmailuser/Deepseek-qwen-modelstock-2B", - "developer": "hotmailuser", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.777" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/hotmailuser_Deepseek-qwen-modelstock-2B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2149 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3549 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3399 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2802 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3475 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1911 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/hotmailuser_falcon3slerp1-10b.json b/data/models/hotmailuser_falcon3slerp1-10b.json deleted file mode 100644 index 3e6c3945ca9a39fe8677edc1b43c89424f895f58..0000000000000000000000000000000000000000 --- a/data/models/hotmailuser_falcon3slerp1-10b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Falcon3Slerp1-10B", - "id": "hotmailuser/Falcon3Slerp1-10B", - "developer": "hotmailuser", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "10.306" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/hotmailuser_Falcon3Slerp1-10B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5694 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.617 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2598 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.344 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4318 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4402 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/hotmailuser_falcon3slerp2-10b.json b/data/models/hotmailuser_falcon3slerp2-10b.json deleted file mode 100644 index d0983e12632c0da31ab2988bff6b165f4e466580..0000000000000000000000000000000000000000 --- a/data/models/hotmailuser_falcon3slerp2-10b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Falcon3Slerp2-10B", - "id": "hotmailuser/Falcon3Slerp2-10B", - "developer": "hotmailuser", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "10.306" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/hotmailuser_Falcon3Slerp2-10B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6118 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6164 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2319 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3381 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4096 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4369 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/hotmailuser_falcon3slerp4-10b.json b/data/models/hotmailuser_falcon3slerp4-10b.json deleted file mode 100644 index 7befc4e4b91d888b5f4b74a3102b89a98faa9319..0000000000000000000000000000000000000000 --- a/data/models/hotmailuser_falcon3slerp4-10b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Falcon3Slerp4-10B", - "id": "hotmailuser/Falcon3Slerp4-10B", - "developer": "hotmailuser", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "10.306" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/hotmailuser_Falcon3Slerp4-10B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6072 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6114 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2289 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3289 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4017 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4387 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/hotmailuser_falconslerp-3b.json b/data/models/hotmailuser_falconslerp-3b.json deleted file mode 100644 index 901893f1f139beee569ee16e61fa07f8d8068ded..0000000000000000000000000000000000000000 --- a/data/models/hotmailuser_falconslerp-3b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "FalconSlerp-3B", - "id": "hotmailuser/FalconSlerp-3B", - "developer": "hotmailuser", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.228" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/hotmailuser_FalconSlerp-3B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5695 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4624 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.176 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2878 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3989 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2968 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/hotmailuser_falconslerp1-7b.json b/data/models/hotmailuser_falconslerp1-7b.json deleted file mode 100644 index af1ae5061e283bdda13169ed2423020b00623957..0000000000000000000000000000000000000000 --- a/data/models/hotmailuser_falconslerp1-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "FalconSlerp1-7B", - "id": "hotmailuser/FalconSlerp1-7B", - "developer": "hotmailuser", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "7.456" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/hotmailuser_FalconSlerp1-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5395 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5355 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2379 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3196 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4452 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4129 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/hotmailuser_falconslerp2-7b.json b/data/models/hotmailuser_falconslerp2-7b.json deleted file mode 100644 index 948a0454ddaa90c92e8da1489480b4c5bd58622c..0000000000000000000000000000000000000000 --- a/data/models/hotmailuser_falconslerp2-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "FalconSlerp2-7B", - "id": "hotmailuser/FalconSlerp2-7B", - "developer": "hotmailuser", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "7.456" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/hotmailuser_FalconSlerp2-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.616 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5538 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2983 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3196 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4479 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4141 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/hotmailuser_falconslerp3-10b.json b/data/models/hotmailuser_falconslerp3-10b.json deleted file mode 100644 index 5b7b940e9dcd972d38844f2856d33f36baf8f97f..0000000000000000000000000000000000000000 --- a/data/models/hotmailuser_falconslerp3-10b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "FalconSlerp3-10B", - "id": "hotmailuser/FalconSlerp3-10B", - "developer": "hotmailuser", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "10.306" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/hotmailuser_FalconSlerp3-10B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6002 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.606 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2273 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3356 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4031 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4323 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/hotmailuser_falconslerp3-7b.json b/data/models/hotmailuser_falconslerp3-7b.json deleted file mode 100644 index 1eeeb0e6c40be259ec4cc9c5bceea90c4aa822cc..0000000000000000000000000000000000000000 --- a/data/models/hotmailuser_falconslerp3-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "FalconSlerp3-7B", - "id": "hotmailuser/FalconSlerp3-7B", - "developer": "hotmailuser", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "7.456" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/hotmailuser_FalconSlerp3-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6096 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5533 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3157 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3188 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4507 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4127 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/hotmailuser_falconslerp4-7b.json b/data/models/hotmailuser_falconslerp4-7b.json deleted file mode 100644 index 6e4f6d03765214392af22b14b6a9d9b43bd80c41..0000000000000000000000000000000000000000 --- a/data/models/hotmailuser_falconslerp4-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "FalconSlerp4-7B", - "id": "hotmailuser/FalconSlerp4-7B", - "developer": "hotmailuser", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "7.456" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/hotmailuser_FalconSlerp4-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6285 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5524 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2213 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3322 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4585 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4032 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/hotmailuser_falconslerp6-7b.json b/data/models/hotmailuser_falconslerp6-7b.json deleted file mode 100644 index f4ba8498a73bb7dec760537840a40fadd499d636..0000000000000000000000000000000000000000 --- a/data/models/hotmailuser_falconslerp6-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "FalconSlerp6-7B", - "id": "hotmailuser/FalconSlerp6-7B", - "developer": "hotmailuser", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "7.456" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/hotmailuser_FalconSlerp6-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6027 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5384 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2047 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.318 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4492 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3995 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/hotmailuser_gemma2atlas-27b.json b/data/models/hotmailuser_gemma2atlas-27b.json deleted file mode 100644 index 44423b1d8f6fce9dcc5d891bd240af62af370ffe..0000000000000000000000000000000000000000 --- a/data/models/hotmailuser_gemma2atlas-27b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Gemma2atlas-27B", - "id": "hotmailuser/Gemma2atlas-27B", - "developer": "hotmailuser", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "27.227" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/hotmailuser_Gemma2atlas-27B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7214 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6545 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2145 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3557 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4445 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.475 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/hotmailuser_gemma2crono-27b.json b/data/models/hotmailuser_gemma2crono-27b.json deleted file mode 100644 index dc35cad88f1a749ce9691db6bd5f089b988a08fc..0000000000000000000000000000000000000000 --- a/data/models/hotmailuser_gemma2crono-27b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Gemma2Crono-27B", - "id": "hotmailuser/Gemma2Crono-27B", - "developer": "hotmailuser", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "27.227" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/hotmailuser_Gemma2Crono-27B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7086 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6505 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2424 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3708 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4567 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4633 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/hotmailuser_gemma2magnum-27b.json b/data/models/hotmailuser_gemma2magnum-27b.json deleted file mode 100644 index b6bfad2a95b4f56dfa0f78fa46a15ed8c6c10f2e..0000000000000000000000000000000000000000 --- a/data/models/hotmailuser_gemma2magnum-27b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Gemma2magnum-27b", - "id": "hotmailuser/Gemma2magnum-27b", - "developer": "hotmailuser", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "27.227" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/hotmailuser_Gemma2magnum-27b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5051 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.62 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2205 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3851 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4723 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4596 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/hotmailuser_gemma2simpo-27b.json b/data/models/hotmailuser_gemma2simpo-27b.json deleted file mode 100644 index 99adc508d33838958492dc24988c0f23099b2f86..0000000000000000000000000000000000000000 --- a/data/models/hotmailuser_gemma2simpo-27b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Gemma2SimPO-27B", - "id": "hotmailuser/Gemma2SimPO-27B", - "developer": "hotmailuser", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "27.227" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/hotmailuser_Gemma2SimPO-27B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7222 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6413 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2817 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3582 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4447 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4642 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/hotmailuser_llama-hermes-slerp-8b.json b/data/models/hotmailuser_llama-hermes-slerp-8b.json deleted file mode 100644 index 00ee73ba8d4730f35f30c447a2ffb3573c32716d..0000000000000000000000000000000000000000 --- a/data/models/hotmailuser_llama-hermes-slerp-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-Hermes-slerp-8B", - "id": "hotmailuser/Llama-Hermes-slerp-8B", - "developer": "hotmailuser", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/hotmailuser_Llama-Hermes-slerp-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.339 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.531 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0801 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2936 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4078 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3331 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/hotmailuser_llama-hermes-slerp2-8b.json b/data/models/hotmailuser_llama-hermes-slerp2-8b.json deleted file mode 100644 index f7de3aa532fbff974e3ea880a4212d6b11d203bb..0000000000000000000000000000000000000000 --- a/data/models/hotmailuser_llama-hermes-slerp2-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-Hermes-slerp2-8B", - "id": "hotmailuser/Llama-Hermes-slerp2-8B", - "developer": "hotmailuser", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/hotmailuser_Llama-Hermes-slerp2-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3728 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5265 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0974 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2961 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4248 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3379 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/hotmailuser_llamastock-8b.json b/data/models/hotmailuser_llamastock-8b.json deleted file mode 100644 index 57eb51d8a06283ef6a94ad5ea63fef14fcf6a79d..0000000000000000000000000000000000000000 --- a/data/models/hotmailuser_llamastock-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "LlamaStock-8B", - "id": "hotmailuser/LlamaStock-8B", - "developer": "hotmailuser", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/hotmailuser_LlamaStock-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.425 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5329 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1699 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3272 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4129 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3807 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/hotmailuser_mistral-modelstock-24b.json b/data/models/hotmailuser_mistral-modelstock-24b.json deleted file mode 100644 index 089dfa1939dfaaaeebcd10dde31b10bf07cd77f3..0000000000000000000000000000000000000000 --- a/data/models/hotmailuser_mistral-modelstock-24b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mistral-modelstock-24B", - "id": "hotmailuser/Mistral-modelstock-24B", - "developer": "hotmailuser", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "23.572" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/hotmailuser_Mistral-modelstock-24B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3424 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6452 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1307 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4102 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.459 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.507 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/hotmailuser_mistral-modelstock2-24b.json b/data/models/hotmailuser_mistral-modelstock2-24b.json deleted file mode 100644 index 6af7c7193ce06a80be4655bf3401f578e75c5b2f..0000000000000000000000000000000000000000 --- a/data/models/hotmailuser_mistral-modelstock2-24b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mistral-modelstock2-24B", - "id": "hotmailuser/Mistral-modelstock2-24B", - "developer": "hotmailuser", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "23.572" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/hotmailuser_Mistral-modelstock2-24B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4318 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6689 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2402 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3926 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4616 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5318 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/hotmailuser_phi4-slerp4-14b.json b/data/models/hotmailuser_phi4-slerp4-14b.json deleted file mode 100644 index 4c9f44df3923924caec9441b73b2a01121e47038..0000000000000000000000000000000000000000 --- a/data/models/hotmailuser_phi4-slerp4-14b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Phi4-Slerp4-14B", - "id": "hotmailuser/Phi4-Slerp4-14B", - "developer": "hotmailuser", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/hotmailuser_Phi4-Slerp4-14B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0629 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6731 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3474 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3968 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5097 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5278 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/hotmailuser_qwen2.5-homerslerp-7b.json b/data/models/hotmailuser_qwen2.5-homerslerp-7b.json deleted file mode 100644 index dded9251458fdb0bdf963f636463a04c1003b330..0000000000000000000000000000000000000000 --- a/data/models/hotmailuser_qwen2.5-homerslerp-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-HomerSlerp-7B", - "id": "hotmailuser/Qwen2.5-HomerSlerp-7B", - "developer": "hotmailuser", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/hotmailuser_Qwen2.5-HomerSlerp-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4488 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5633 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3316 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3138 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4383 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4549 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/hotmailuser_qwenmodelstock-1.8b.json b/data/models/hotmailuser_qwenmodelstock-1.8b.json deleted file mode 100644 index ef997cf6faa224d7411e059502055219e53271ed..0000000000000000000000000000000000000000 --- a/data/models/hotmailuser_qwenmodelstock-1.8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "QwenModelStock-1.8B", - "id": "hotmailuser/QwenModelStock-1.8B", - "developer": "hotmailuser", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.777" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/hotmailuser_QwenModelStock-1.8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3263 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4188 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0989 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2869 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4359 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2959 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/hotmailuser_qwenslerp-14b.json b/data/models/hotmailuser_qwenslerp-14b.json deleted file mode 100644 index 3a0a86b26aae47a52a474f6f66f719e93f26d776..0000000000000000000000000000000000000000 --- a/data/models/hotmailuser_qwenslerp-14b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "QwenSlerp-14B", - "id": "hotmailuser/QwenSlerp-14B", - "developer": "hotmailuser", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/hotmailuser_QwenSlerp-14B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7025 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6491 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3837 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3876 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4634 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.54 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/hotmailuser_qwenslerp-3b.json b/data/models/hotmailuser_qwenslerp-3b.json deleted file mode 100644 index 1943775c093ca66884f52fb2e28c767c2b7f7903..0000000000000000000000000000000000000000 --- a/data/models/hotmailuser_qwenslerp-3b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "QwenSlerp-3B", - "id": "hotmailuser/QwenSlerp-3B", - "developer": "hotmailuser", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.397" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/hotmailuser_QwenSlerp-3B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4334 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4892 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2749 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2945 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4317 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3693 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/hotmailuser_qwenslerp-7b.json b/data/models/hotmailuser_qwenslerp-7b.json deleted file mode 100644 index 7ddc1704a70f3021b33ae2ba8480bf15dc46dce4..0000000000000000000000000000000000000000 --- a/data/models/hotmailuser_qwenslerp-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "QwenSlerp-7B", - "id": "hotmailuser/QwenSlerp-7B", - "developer": "hotmailuser", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/hotmailuser_QwenSlerp-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4673 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5636 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3444 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.318 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4409 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4509 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/hotmailuser_qwenslerp2-14b.json b/data/models/hotmailuser_qwenslerp2-14b.json deleted file mode 100644 index 3ce2a4cfbf5cbb231f0f162ce3061daa8391329b..0000000000000000000000000000000000000000 --- a/data/models/hotmailuser_qwenslerp2-14b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "QwenSlerp2-14B", - "id": "hotmailuser/QwenSlerp2-14B", - "developer": "hotmailuser", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/hotmailuser_QwenSlerp2-14B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7037 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6493 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3965 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3809 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4807 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5379 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/hotmailuser_qwenslerp2-3b.json b/data/models/hotmailuser_qwenslerp2-3b.json deleted file mode 100644 index 876e115cc77eaa6356420a4bf57a705f95413d58..0000000000000000000000000000000000000000 --- a/data/models/hotmailuser_qwenslerp2-3b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "QwenSlerp2-3B", - "id": "hotmailuser/QwenSlerp2-3B", - "developer": "hotmailuser", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.397" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/hotmailuser_QwenSlerp2-3B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.428 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4802 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2606 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.297 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4252 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3742 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/hotmailuser_qwenslerp3-14b.json b/data/models/hotmailuser_qwenslerp3-14b.json deleted file mode 100644 index 5a912cbbcba05263c7738c6d257143ac63908825..0000000000000000000000000000000000000000 --- a/data/models/hotmailuser_qwenslerp3-14b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "QwenSlerp3-14B", - "id": "hotmailuser/QwenSlerp3-14B", - "developer": "hotmailuser", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/hotmailuser_QwenSlerp3-14B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6632 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6267 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4305 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3666 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4808 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5263 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/hotmailuser_qwensparse-7b.json b/data/models/hotmailuser_qwensparse-7b.json deleted file mode 100644 index 27317f84224e7ac022e7d2fd8fd626c0c2585c18..0000000000000000000000000000000000000000 --- a/data/models/hotmailuser_qwensparse-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "QwenSparse-7B", - "id": "hotmailuser/QwenSparse-7B", - "developer": "hotmailuser", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/hotmailuser_QwenSparse-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1086 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2896 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0106 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2601 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3562 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1122 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/hotmailuser_qwenstock-0.5b.json b/data/models/hotmailuser_qwenstock-0.5b.json deleted file mode 100644 index 4072b58e528335f75dd7c29a4c1f59502cf517f4..0000000000000000000000000000000000000000 --- a/data/models/hotmailuser_qwenstock-0.5b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "QwenStock-0.5B", - "id": "hotmailuser/QwenStock-0.5B", - "developer": "hotmailuser", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/hotmailuser_QwenStock-0.5B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2049 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2912 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2601 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3575 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1167 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/hotmailuser_qwenstock-1.7b.json b/data/models/hotmailuser_qwenstock-1.7b.json deleted file mode 100644 index e06c59d47891d1b4029c1af88bab397af5407b36..0000000000000000000000000000000000000000 --- a/data/models/hotmailuser_qwenstock-1.7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "QwenStock-1.7B", - "id": "hotmailuser/QwenStock-1.7B", - "developer": "hotmailuser", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.777" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/hotmailuser_QwenStock-1.7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3214 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4188 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0997 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2878 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4412 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2955 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/hotmailuser_qwenstock1-14b.json b/data/models/hotmailuser_qwenstock1-14b.json deleted file mode 100644 index b58162d71c08c88cebd0dc5a8141803aadfd8412..0000000000000000000000000000000000000000 --- a/data/models/hotmailuser_qwenstock1-14b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "QwenStock1-14B", - "id": "hotmailuser/QwenStock1-14B", - "developer": "hotmailuser", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/hotmailuser_QwenStock1-14B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6693 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6502 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3701 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3859 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4781 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5416 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/hotmailuser_rombosbeagle-v2beta-mgs-32b.json b/data/models/hotmailuser_rombosbeagle-v2beta-mgs-32b.json deleted file mode 100644 index e024b26b0e07abe8973964de100047070881f9d9..0000000000000000000000000000000000000000 --- a/data/models/hotmailuser_rombosbeagle-v2beta-mgs-32b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "RombosBeagle-v2beta-MGS-32B", - "id": "hotmailuser/RombosBeagle-v2beta-MGS-32B", - "developer": "hotmailuser", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "32.764" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/hotmailuser_RombosBeagle-v2beta-MGS-32B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5157 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7037 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4992 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.38 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5021 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5908 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/hpai-bsc_llama3-aloe-8b-alpha.json b/data/models/hpai-bsc_llama3-aloe-8b-alpha.json deleted file mode 100644 index 407b0d0899f3f316eddade546423fbc5a06da4f3..0000000000000000000000000000000000000000 --- a/data/models/hpai-bsc_llama3-aloe-8b-alpha.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama3-Aloe-8B-Alpha", - "id": "HPAI-BSC/Llama3-Aloe-8B-Alpha", - "developer": "HPAI-BSC", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/HPAI-BSC_Llama3-Aloe-8B-Alpha/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5081 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4831 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0612 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2945 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3673 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3295 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/hpai-bsc_llama3.1-aloe-beta-8b.json b/data/models/hpai-bsc_llama3.1-aloe-beta-8b.json deleted file mode 100644 index 7f376923c62fe580de4227187d184912c3789af5..0000000000000000000000000000000000000000 --- a/data/models/hpai-bsc_llama3.1-aloe-beta-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama3.1-Aloe-Beta-8B", - "id": "HPAI-BSC/Llama3.1-Aloe-Beta-8B", - "developer": "HPAI-BSC", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/HPAI-BSC_Llama3.1-Aloe-Beta-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7253 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5093 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1828 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2685 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3835 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.358 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/hpai-bsc_qwen2.5-aloe-beta-7b.json b/data/models/hpai-bsc_qwen2.5-aloe-beta-7b.json deleted file mode 100644 index b618ac7f3ddab65dabdd1d800b6e7e9a09cffb92..0000000000000000000000000000000000000000 --- a/data/models/hpai-bsc_qwen2.5-aloe-beta-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-Aloe-Beta-7B", - "id": "HPAI-BSC/Qwen2.5-Aloe-Beta-7B", - "developer": "HPAI-BSC", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/HPAI-BSC_Qwen2.5-Aloe-Beta-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4554 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5049 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3542 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2911 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.426 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4354 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/huawei-noah-ustc_toolace-2-8b-fc.json b/data/models/huawei-noah-ustc_toolace-2-8b-fc.json deleted file mode 100644 index cb5133e42f4a5e4851a42f1cf3d3f11778757073..0000000000000000000000000000000000000000 --- a/data/models/huawei-noah-ustc_toolace-2-8b-fc.json +++ /dev/null @@ -1,905 +0,0 @@ -{ - "model_info": { - "name": "ToolACE-2-8B (FC)", - "id": "huawei-noah-ustc/toolace-2-8b-fc", - "developer": "huawei-noah-ustc", - "additional_details": { - "raw_model_name": "ToolACE-2-8B (FC)", - "organization": "Huawei Noah & USTC", - "license": "Apache-2.0", - "mode": "FC", - "model_link": "https://huggingface.co/Team-ACE/ToolACE-2-8B" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/huawei-noah-ustc/toolace-2-8b-fc/1775236112.385761", - "retrieved_timestamp": "1775236112.385761", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 40.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 42.44 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 24.43 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 15.95 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 40.06 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 65.26 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 87.1 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 73.42 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 91.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 93.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 91.0 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 77.42 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 71.32 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 79.39 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 68.75 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 62.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 38.38 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 49.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 28.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 30.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 46.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 8.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 13.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 4.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 18.49 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 5.81 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 16.13 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 33.55 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 75.0 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 90.79 - } - }, - { - "evaluation_result_id": "bfcl.format_sensitivity.max_delta::max_delta", - "evaluation_name": "bfcl.format_sensitivity.max_delta", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.format_sensitivity.max_delta", - "metric_name": "Format sensitivity max delta", - "metric_kind": "difference", - "metric_unit": "percentage_points", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Format Sensitivity Max Delta" - } - }, - "score_details": { - "score": 81.5 - } - }, - { - "evaluation_result_id": "bfcl.format_sensitivity.stddev::stddev", - "evaluation_name": "bfcl.format_sensitivity.stddev", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.format_sensitivity.stddev", - "metric_name": "Format sensitivity standard deviation", - "metric_kind": "difference", - "metric_unit": "percentage_points", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Format Sensitivity Standard Deviation" - } - }, - "score_details": { - "score": 27.92 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/huggingfaceh4_starchat2-15b-v0.1.json b/data/models/huggingfaceh4_starchat2-15b-v0.1.json deleted file mode 100644 index dc98155304f6d0943dd4aef3cdc431a2a83b85c9..0000000000000000000000000000000000000000 --- a/data/models/huggingfaceh4_starchat2-15b-v0.1.json +++ /dev/null @@ -1,144 +0,0 @@ -{ - "model_info": { - "name": "HuggingFaceH4/starchat2-15b-v0.1", - "id": "HuggingFaceH4/starchat2-15b-v0.1", - "developer": "HuggingFaceH4", - "additional_details": { - "model_type": "DPO" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/HuggingFaceH4_starchat2-15b-v0.1/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7322 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9385 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5548 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7095 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8159 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5525 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/huggingfaceh4_zephyr-7b-alpha.json b/data/models/huggingfaceh4_zephyr-7b-alpha.json deleted file mode 100644 index a25a4883299720224602994e52c2f5a7bba8056c..0000000000000000000000000000000000000000 --- a/data/models/huggingfaceh4_zephyr-7b-alpha.json +++ /dev/null @@ -1,277 +0,0 @@ -{ - "model_info": { - "name": "HuggingFaceH4/zephyr-7b-alpha", - "id": "HuggingFaceH4/zephyr-7b-alpha", - "developer": "HuggingFaceH4", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/HuggingFaceH4_zephyr-7b-alpha/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5191 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4583 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0196 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2978 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.395 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2795 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "reward-bench/HuggingFaceH4_zephyr-7b-alpha/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7392 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9162 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.625 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7662 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7514 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5353 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/huggingfaceh4_zephyr-7b-beta.json b/data/models/huggingfaceh4_zephyr-7b-beta.json deleted file mode 100644 index a94706cee530964da80035d918ed7fe994585d83..0000000000000000000000000000000000000000 --- a/data/models/huggingfaceh4_zephyr-7b-beta.json +++ /dev/null @@ -1,277 +0,0 @@ -{ - "model_info": { - "name": "HuggingFaceH4/zephyr-7b-beta", - "id": "HuggingFaceH4/zephyr-7b-beta", - "developer": "HuggingFaceH4", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/HuggingFaceH4_zephyr-7b-beta/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.495 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4316 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0287 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2903 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3925 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2781 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "reward-bench/HuggingFaceH4_zephyr-7b-beta/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7281 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9525 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6272 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6568 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7789 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5216 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/huggingfaceh4_zephyr-7b-gemma-v0.1.json b/data/models/huggingfaceh4_zephyr-7b-gemma-v0.1.json deleted file mode 100644 index 3cc33014dd9121c68e8cbee57b793d4ed3806df8..0000000000000000000000000000000000000000 --- a/data/models/huggingfaceh4_zephyr-7b-gemma-v0.1.json +++ /dev/null @@ -1,277 +0,0 @@ -{ - "model_info": { - "name": "HuggingFaceH4/zephyr-7b-gemma-v0.1", - "id": "HuggingFaceH4/zephyr-7b-gemma-v0.1", - "developer": "HuggingFaceH4", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "GemmaForCausalLM", - "params_billions": "8.538" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/HuggingFaceH4_zephyr-7b-gemma-v0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3364 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4624 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0816 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2945 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.374 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2847 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "reward-bench/HuggingFaceH4_zephyr-7b-gemma-v0.1/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6758 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9581 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4956 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5824 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7463 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5171 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/huggingfaceh4_zephyr-orpo-141b-a35b-v0.1.json b/data/models/huggingfaceh4_zephyr-orpo-141b-a35b-v0.1.json deleted file mode 100644 index cdd40995d5dd779541c3dd5b8bcf51d16245bd57..0000000000000000000000000000000000000000 --- a/data/models/huggingfaceh4_zephyr-orpo-141b-a35b-v0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "zephyr-orpo-141b-A35b-v0.1", - "id": "HuggingFaceH4/zephyr-orpo-141b-A35b-v0.1", - "developer": "HuggingFaceH4", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MixtralForCausalLM", - "params_billions": "140.621" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/HuggingFaceH4_zephyr-orpo-141b-A35b-v0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6511 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.629 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2047 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3784 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4465 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4586 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/huggingfacetb_smollm-1.7b-instruct.json b/data/models/huggingfacetb_smollm-1.7b-instruct.json deleted file mode 100644 index 3898156092d16275ad44ba95e10032d445e0b479..0000000000000000000000000000000000000000 --- a/data/models/huggingfacetb_smollm-1.7b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SmolLM-1.7B-Instruct", - "id": "HuggingFaceTB/SmolLM-1.7B-Instruct", - "developer": "HuggingFaceTB", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.71" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/HuggingFaceTB_SmolLM-1.7B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2348 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2885 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0211 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2601 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3487 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1166 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/huggingfacetb_smollm-1.7b.json b/data/models/huggingfacetb_smollm-1.7b.json deleted file mode 100644 index 0f2883de9c01512770a5298b6ce5b813062f8496..0000000000000000000000000000000000000000 --- a/data/models/huggingfacetb_smollm-1.7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SmolLM-1.7B", - "id": "HuggingFaceTB/SmolLM-1.7B", - "developer": "HuggingFaceTB", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.71" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/HuggingFaceTB_SmolLM-1.7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2362 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3181 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0166 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2416 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3421 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1148 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/huggingfacetb_smollm-135m-instruct.json b/data/models/huggingfacetb_smollm-135m-instruct.json deleted file mode 100644 index 20d5c1e602a34f8f8875ae8c5ab183ade6e1db18..0000000000000000000000000000000000000000 --- a/data/models/huggingfacetb_smollm-135m-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SmolLM-135M-Instruct", - "id": "HuggingFaceTB/SmolLM-135M-Instruct", - "developer": "HuggingFaceTB", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "0.135" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/HuggingFaceTB_SmolLM-135M-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1214 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3015 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0053 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2592 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3635 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1176 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/huggingfacetb_smollm-135m.json b/data/models/huggingfacetb_smollm-135m.json deleted file mode 100644 index 8202e9bfa5c21ce6dd4a080cb13226183865602c..0000000000000000000000000000000000000000 --- a/data/models/huggingfacetb_smollm-135m.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SmolLM-135M", - "id": "HuggingFaceTB/SmolLM-135M", - "developer": "HuggingFaceTB", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "0.13" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/HuggingFaceTB_SmolLM-135M/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2125 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3046 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0136 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2584 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4366 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1122 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/huggingfacetb_smollm-360m-instruct.json b/data/models/huggingfacetb_smollm-360m-instruct.json deleted file mode 100644 index 6a120b6c12242413a924a0249f055cb1d365621b..0000000000000000000000000000000000000000 --- a/data/models/huggingfacetb_smollm-360m-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SmolLM-360M-Instruct", - "id": "HuggingFaceTB/SmolLM-360M-Instruct", - "developer": "HuggingFaceTB", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "0.362" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/HuggingFaceTB_SmolLM-360M-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1952 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2885 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0181 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2643 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3472 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1166 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/huggingfacetb_smollm-360m.json b/data/models/huggingfacetb_smollm-360m.json deleted file mode 100644 index 9821007a58ff4d4cb9d9ffb0bee4b42975ab2993..0000000000000000000000000000000000000000 --- a/data/models/huggingfacetb_smollm-360m.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SmolLM-360M", - "id": "HuggingFaceTB/SmolLM-360M", - "developer": "HuggingFaceTB", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "0.36" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/HuggingFaceTB_SmolLM-360M/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2134 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3065 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0113 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2676 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4018 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1124 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/huggingfacetb_smollm2-1.7b-instruct.json b/data/models/huggingfacetb_smollm2-1.7b-instruct.json deleted file mode 100644 index 6c886492d7c5ad2584eacea1b4d6abb34001bba2..0000000000000000000000000000000000000000 --- a/data/models/huggingfacetb_smollm2-1.7b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SmolLM2-1.7B-Instruct", - "id": "HuggingFaceTB/SmolLM2-1.7B-Instruct", - "developer": "HuggingFaceTB", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.711" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/HuggingFaceTB_SmolLM2-1.7B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5368 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3599 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0582 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2794 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3421 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2054 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/huggingfacetb_smollm2-1.7b.json b/data/models/huggingfacetb_smollm2-1.7b.json deleted file mode 100644 index 92f43d710e0509c5d43287065cc8f9800880183e..0000000000000000000000000000000000000000 --- a/data/models/huggingfacetb_smollm2-1.7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SmolLM2-1.7B", - "id": "HuggingFaceTB/SmolLM2-1.7B", - "developer": "HuggingFaceTB", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.71" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/HuggingFaceTB_SmolLM2-1.7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.244 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3453 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0264 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2794 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3485 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2138 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/huggingfacetb_smollm2-135m-instruct.json b/data/models/huggingfacetb_smollm2-135m-instruct.json deleted file mode 100644 index 4930fd119b327af5dcaff7489655e3b5a735f796..0000000000000000000000000000000000000000 --- a/data/models/huggingfacetb_smollm2-135m-instruct.json +++ /dev/null @@ -1,275 +0,0 @@ -{ - "model_info": { - "name": "SmolLM2-135M-Instruct", - "id": "HuggingFaceTB/SmolLM2-135M-Instruct", - "developer": "HuggingFaceTB", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "0.135" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/HuggingFaceTB_SmolLM2-135M-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2883 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3124 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.003 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2357 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3662 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1115 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "hfopenllm_v2/HuggingFaceTB_SmolLM2-135M-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0593 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3135 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0144 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2341 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3871 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1092 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/huggingfacetb_smollm2-135m.json b/data/models/huggingfacetb_smollm2-135m.json deleted file mode 100644 index d684cc094fdab212c9531560ab323d7406293a9d..0000000000000000000000000000000000000000 --- a/data/models/huggingfacetb_smollm2-135m.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SmolLM2-135M", - "id": "HuggingFaceTB/SmolLM2-135M", - "developer": "HuggingFaceTB", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "0.135" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/HuggingFaceTB_SmolLM2-135M/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1818 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3044 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0121 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2483 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4112 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1095 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/huggingfacetb_smollm2-360m-instruct.json b/data/models/huggingfacetb_smollm2-360m-instruct.json deleted file mode 100644 index b9a479d91c3b9226f2e42a0a5392df69dc01d9df..0000000000000000000000000000000000000000 --- a/data/models/huggingfacetb_smollm2-360m-instruct.json +++ /dev/null @@ -1,275 +0,0 @@ -{ - "model_info": { - "name": "SmolLM2-360M-Instruct", - "id": "HuggingFaceTB/SmolLM2-360M-Instruct", - "developer": "HuggingFaceTB", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "0.36" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/HuggingFaceTB_SmolLM2-360M-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3842 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3144 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0151 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.255 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3461 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1117 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "hfopenllm_v2/HuggingFaceTB_SmolLM2-360M-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.083 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3053 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0083 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2651 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3423 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1126 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/huggingfacetb_smollm2-360m.json b/data/models/huggingfacetb_smollm2-360m.json deleted file mode 100644 index 1ef66e6a58f205795607896c7f034c5029cc9a3d..0000000000000000000000000000000000000000 --- a/data/models/huggingfacetb_smollm2-360m.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SmolLM2-360M", - "id": "HuggingFaceTB/SmolLM2-360M", - "developer": "HuggingFaceTB", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "0.36" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/HuggingFaceTB_SmolLM2-360M/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2115 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3233 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0121 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2458 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3954 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1169 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/huggyllama_llama-13b.json b/data/models/huggyllama_llama-13b.json deleted file mode 100644 index 88af33febeb032b94f9eb4782620c3657bf6b7fb..0000000000000000000000000000000000000000 --- a/data/models/huggyllama_llama-13b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "llama-13b", - "id": "huggyllama/llama-13b", - "developer": "huggyllama", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "13.016" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/huggyllama_llama-13b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2411 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3988 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0204 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.255 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3462 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1952 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/huggyllama_llama-65b.json b/data/models/huggyllama_llama-65b.json deleted file mode 100644 index 4c0176c108a96037052b1f869e7d3c9b7dea1cad..0000000000000000000000000000000000000000 --- a/data/models/huggyllama_llama-65b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "llama-65b", - "id": "huggyllama/llama-65b", - "developer": "huggyllama", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "65.286" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/huggyllama_llama-65b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2526 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4703 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.031 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.276 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3595 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3078 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/huggyllama_llama-7b.json b/data/models/huggyllama_llama-7b.json deleted file mode 100644 index e8c163390ac70d39331e649f97918a2b33f5d03c..0000000000000000000000000000000000000000 --- a/data/models/huggyllama_llama-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "llama-7b", - "id": "huggyllama/llama-7b", - "developer": "huggyllama", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "6.738" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/huggyllama_llama-7b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2501 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3277 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0083 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2525 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3354 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1313 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/huihui-ai_deepseek-r1-distill-qwen-14b-abliterated-v2.json b/data/models/huihui-ai_deepseek-r1-distill-qwen-14b-abliterated-v2.json deleted file mode 100644 index aa2b8a4656d599895b1a9935809db440ae1627a1..0000000000000000000000000000000000000000 --- a/data/models/huihui-ai_deepseek-r1-distill-qwen-14b-abliterated-v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "DeepSeek-R1-Distill-Qwen-14B-abliterated-v2", - "id": "huihui-ai/DeepSeek-R1-Distill-Qwen-14B-abliterated-v2", - "developer": "huihui-ai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/huihui-ai_DeepSeek-R1-Distill-Qwen-14B-abliterated-v2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4211 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3487 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2205 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.276 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4701 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1915 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/huihui-ai_qwen2.5-14b-instruct-abliterated-v2.json b/data/models/huihui-ai_qwen2.5-14b-instruct-abliterated-v2.json deleted file mode 100644 index 18d4f756c0120fdccb514b96fb6adc14b3d10693..0000000000000000000000000000000000000000 --- a/data/models/huihui-ai_qwen2.5-14b-instruct-abliterated-v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-14B-Instruct-abliterated-v2", - "id": "huihui-ai/Qwen2.5-14B-Instruct-abliterated-v2", - "developer": "huihui-ai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/huihui-ai_Qwen2.5-14B-Instruct-abliterated-v2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8328 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6324 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5302 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3339 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.422 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4962 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/huihui-ai_qwen2.5-72b-instruct-abliterated.json b/data/models/huihui-ai_qwen2.5-72b-instruct-abliterated.json deleted file mode 100644 index 229ccd2a13bdddf5d3a0396bf4777b2795a04b8d..0000000000000000000000000000000000000000 --- a/data/models/huihui-ai_qwen2.5-72b-instruct-abliterated.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-72B-Instruct-abliterated", - "id": "huihui-ai/Qwen2.5-72B-Instruct-abliterated", - "developer": "huihui-ai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "72.706" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/huihui-ai_Qwen2.5-72B-Instruct-abliterated/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8593 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.719 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6012 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3951 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4233 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5537 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/huihui-ai_qwen2.5-7b-instruct-abliterated-v2.json b/data/models/huihui-ai_qwen2.5-7b-instruct-abliterated-v2.json deleted file mode 100644 index 649216602ae3ad229b60ec8d8c9d5c067f75f284..0000000000000000000000000000000000000000 --- a/data/models/huihui-ai_qwen2.5-7b-instruct-abliterated-v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-7B-Instruct-abliterated-v2", - "id": "huihui-ai/Qwen2.5-7B-Instruct-abliterated-v2", - "developer": "huihui-ai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/huihui-ai_Qwen2.5-7B-Instruct-abliterated-v2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7606 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5377 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4637 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3087 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3981 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4208 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/huihui-ai_qwen2.5-7b-instruct-abliterated.json b/data/models/huihui-ai_qwen2.5-7b-instruct-abliterated.json deleted file mode 100644 index 100de751c7cc87829c4e35ac12507648b5d82044..0000000000000000000000000000000000000000 --- a/data/models/huihui-ai_qwen2.5-7b-instruct-abliterated.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-7B-Instruct-abliterated", - "id": "huihui-ai/Qwen2.5-7B-Instruct-abliterated", - "developer": "huihui-ai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/huihui-ai_Qwen2.5-7B-Instruct-abliterated/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7546 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5262 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4577 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3154 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3967 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.418 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/huihui-ai_qwq-32b-coder-fusion-7030.json b/data/models/huihui-ai_qwq-32b-coder-fusion-7030.json deleted file mode 100644 index c0c80b4ead33ccaab48571c5f2407b80fd6edcb0..0000000000000000000000000000000000000000 --- a/data/models/huihui-ai_qwq-32b-coder-fusion-7030.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "QwQ-32B-Coder-Fusion-7030", - "id": "huihui-ai/QwQ-32B-Coder-Fusion-7030", - "developer": "huihui-ai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "32.764" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/huihui-ai_QwQ-32B-Coder-Fusion-7030/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3865 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6178 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2795 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2844 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3922 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4368 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/huihui-ai_qwq-32b-coder-fusion-8020.json b/data/models/huihui-ai_qwq-32b-coder-fusion-8020.json deleted file mode 100644 index df172cad45a8677d7834f8bf14b1585834461507..0000000000000000000000000000000000000000 --- a/data/models/huihui-ai_qwq-32b-coder-fusion-8020.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "QwQ-32B-Coder-Fusion-8020", - "id": "huihui-ai/QwQ-32B-Coder-Fusion-8020", - "developer": "huihui-ai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "32.764" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/huihui-ai_QwQ-32B-Coder-Fusion-8020/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6021 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6665 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4592 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3549 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4293 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5367 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/huihui-ai_qwq-32b-coder-fusion-9010.json b/data/models/huihui-ai_qwq-32b-coder-fusion-9010.json deleted file mode 100644 index 606db0e401dd91c0d303c68af2fe2377f0598016..0000000000000000000000000000000000000000 --- a/data/models/huihui-ai_qwq-32b-coder-fusion-9010.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "QwQ-32B-Coder-Fusion-9010", - "id": "huihui-ai/QwQ-32B-Coder-Fusion-9010", - "developer": "huihui-ai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "32.764" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/huihui-ai_QwQ-32B-Coder-Fusion-9010/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5778 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6727 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5317 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3616 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4682 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.56 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/humanllms_humanish-llama3-8b-instruct.json b/data/models/humanllms_humanish-llama3-8b-instruct.json deleted file mode 100644 index 8e1aa141c70a9b4db8fa1b3293e79c052e741981..0000000000000000000000000000000000000000 --- a/data/models/humanllms_humanish-llama3-8b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Humanish-LLama3-8B-Instruct", - "id": "HumanLLMs/Humanish-LLama3-8B-Instruct", - "developer": "HumanLLMs", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/HumanLLMs_Humanish-LLama3-8B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6498 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4968 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1027 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2559 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3582 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3702 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/humanllms_humanish-mistral-nemo-instruct-2407.json b/data/models/humanllms_humanish-mistral-nemo-instruct-2407.json deleted file mode 100644 index 30091ef6ac70bd35c2d4c774fd1cae347a81e564..0000000000000000000000000000000000000000 --- a/data/models/humanllms_humanish-mistral-nemo-instruct-2407.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Humanish-Mistral-Nemo-Instruct-2407", - "id": "HumanLLMs/Humanish-Mistral-Nemo-Instruct-2407", - "developer": "HumanLLMs", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/HumanLLMs_Humanish-Mistral-Nemo-Instruct-2407/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5451 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5262 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1367 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2878 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3968 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3521 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/humanllms_humanish-qwen2.5-7b-instruct.json b/data/models/humanllms_humanish-qwen2.5-7b-instruct.json deleted file mode 100644 index 6ccd553e35ab5e7c921764621ffd77559434bd73..0000000000000000000000000000000000000000 --- a/data/models/humanllms_humanish-qwen2.5-7b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Humanish-Qwen2.5-7B-Instruct", - "id": "HumanLLMs/Humanish-Qwen2.5-7B-Instruct", - "developer": "HumanLLMs", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/HumanLLMs_Humanish-Qwen2.5-7B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7284 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5364 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2987 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3981 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4398 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/huu-ontocord_wide_3b_orpo_stage1.1-ss1-orpo3.json b/data/models/huu-ontocord_wide_3b_orpo_stage1.1-ss1-orpo3.json deleted file mode 100644 index 7b3e689f2290daa63afb7896a0a51cf664336a8e..0000000000000000000000000000000000000000 --- a/data/models/huu-ontocord_wide_3b_orpo_stage1.1-ss1-orpo3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "wide_3b_orpo_stage1.1-ss1-orpo3", - "id": "huu-ontocord/wide_3b_orpo_stage1.1-ss1-orpo3", - "developer": "huu-ontocord", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.759" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/huu-ontocord_wide_3b_orpo_stage1.1-ss1-orpo3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1505 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2937 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0098 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2584 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3618 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1164 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ibivibiv_colossus_120b.json b/data/models/ibivibiv_colossus_120b.json deleted file mode 100644 index ed39bc393cc2161e2c4cc68b2b1aabd282967b5c..0000000000000000000000000000000000000000 --- a/data/models/ibivibiv_colossus_120b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "colossus_120b", - "id": "ibivibiv/colossus_120b", - "developer": "ibivibiv", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "117.749" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ibivibiv_colossus_120b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4276 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6061 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0566 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3087 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4733 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3961 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ibivibiv_multimaster-7b-v6.json b/data/models/ibivibiv_multimaster-7b-v6.json deleted file mode 100644 index 2fc43a654e85a8a214b1ba4411ef8e080ec93e09..0000000000000000000000000000000000000000 --- a/data/models/ibivibiv_multimaster-7b-v6.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "multimaster-7b-v6", - "id": "ibivibiv/multimaster-7b-v6", - "developer": "ibivibiv", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MixtralForCausalLM", - "params_billions": "35.428" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ibivibiv_multimaster-7b-v6/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4473 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5194 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0559 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3037 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4396 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3095 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ibm-granite_granite-3.0-1b-a400m-base.json b/data/models/ibm-granite_granite-3.0-1b-a400m-base.json deleted file mode 100644 index df8c27e5ea5fb7ceed5b22bf527baf1ff7649351..0000000000000000000000000000000000000000 --- a/data/models/ibm-granite_granite-3.0-1b-a400m-base.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "granite-3.0-1b-a400m-base", - "id": "ibm-granite/granite-3.0-1b-a400m-base", - "developer": "ibm-granite", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "GraniteForCausalLM", - "params_billions": "1.335" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ibm-granite_granite-3.0-1b-a400m-base/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2404 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3221 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0264 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2475 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3367 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1152 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ibm-granite_granite-3.0-1b-a400m-instruct.json b/data/models/ibm-granite_granite-3.0-1b-a400m-instruct.json deleted file mode 100644 index b73f81d36df29466e6455d73994347e85eda1a40..0000000000000000000000000000000000000000 --- a/data/models/ibm-granite_granite-3.0-1b-a400m-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "granite-3.0-1b-a400m-instruct", - "id": "ibm-granite/granite-3.0-1b-a400m-instruct", - "developer": "ibm-granite", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "GraniteForCausalLM", - "params_billions": "1.335" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ibm-granite_granite-3.0-1b-a400m-instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3332 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3224 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0279 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2609 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3623 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1244 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ibm-granite_granite-3.0-2b-base.json b/data/models/ibm-granite_granite-3.0-2b-base.json deleted file mode 100644 index e6211a8f4558e868fa7626fcbc1613dfeaf05266..0000000000000000000000000000000000000000 --- a/data/models/ibm-granite_granite-3.0-2b-base.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "granite-3.0-2b-base", - "id": "ibm-granite/granite-3.0-2b-base", - "developer": "ibm-granite", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "GraniteForCausalLM", - "params_billions": "2.634" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ibm-granite_granite-3.0-2b-base/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3874 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4047 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0544 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2802 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3434 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2381 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ibm-granite_granite-3.0-2b-instruct.json b/data/models/ibm-granite_granite-3.0-2b-instruct.json deleted file mode 100644 index 3bb64a69d800be79800180ffdd507c0d461264fa..0000000000000000000000000000000000000000 --- a/data/models/ibm-granite_granite-3.0-2b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "granite-3.0-2b-instruct", - "id": "ibm-granite/granite-3.0-2b-instruct", - "developer": "ibm-granite", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "GraniteForCausalLM", - "params_billions": "2.634" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ibm-granite_granite-3.0-2b-instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.514 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4412 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0921 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2995 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3515 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2814 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ibm-granite_granite-3.0-3b-a800m-base.json b/data/models/ibm-granite_granite-3.0-3b-a800m-base.json deleted file mode 100644 index 6521ab8b8df5826645eb90746bc03175a2a06bbf..0000000000000000000000000000000000000000 --- a/data/models/ibm-granite_granite-3.0-3b-a800m-base.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "granite-3.0-3b-a800m-base", - "id": "ibm-granite/granite-3.0-3b-a800m-base", - "developer": "ibm-granite", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "GraniteForCausalLM", - "params_billions": "3.374" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ibm-granite_granite-3.0-3b-a800m-base/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2732 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3667 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0483 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2517 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.342 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1891 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ibm-granite_granite-3.0-3b-a800m-instruct.json b/data/models/ibm-granite_granite-3.0-3b-a800m-instruct.json deleted file mode 100644 index 44f60d6d313120c2e9d68032a83d3862c400044d..0000000000000000000000000000000000000000 --- a/data/models/ibm-granite_granite-3.0-3b-a800m-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "granite-3.0-3b-a800m-instruct", - "id": "ibm-granite/granite-3.0-3b-a800m-instruct", - "developer": "ibm-granite", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "GraniteForCausalLM", - "params_billions": "3.374" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ibm-granite_granite-3.0-3b-a800m-instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4298 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3753 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0702 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.281 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3487 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2152 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ibm-granite_granite-3.0-8b-base.json b/data/models/ibm-granite_granite-3.0-8b-base.json deleted file mode 100644 index 268539ec15c97126c9e79b3c1b82afca38405168..0000000000000000000000000000000000000000 --- a/data/models/ibm-granite_granite-3.0-8b-base.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "granite-3.0-8b-base", - "id": "ibm-granite/granite-3.0-8b-base", - "developer": "ibm-granite", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "GraniteForCausalLM", - "params_billions": "8.171" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ibm-granite_granite-3.0-8b-base/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4583 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4944 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1012 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3255 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4081 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3313 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ibm-granite_granite-3.0-8b-instruct.json b/data/models/ibm-granite_granite-3.0-8b-instruct.json deleted file mode 100644 index 20f49a790c01c9b936593885a1b7c387bbc21c26..0000000000000000000000000000000000000000 --- a/data/models/ibm-granite_granite-3.0-8b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "granite-3.0-8b-instruct", - "id": "ibm-granite/granite-3.0-8b-instruct", - "developer": "ibm-granite", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "GraniteForCausalLM", - "params_billions": "8.171" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ibm-granite_granite-3.0-8b-instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.531 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5192 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.142 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3322 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3901 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3457 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ibm-granite_granite-3.1-1b-a400m-base.json b/data/models/ibm-granite_granite-3.1-1b-a400m-base.json deleted file mode 100644 index 6e1a1bd0dcb406d049192bc9ffb5c8c313ee102a..0000000000000000000000000000000000000000 --- a/data/models/ibm-granite_granite-3.1-1b-a400m-base.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "granite-3.1-1b-a400m-base", - "id": "ibm-granite/granite-3.1-1b-a400m-base", - "developer": "ibm-granite", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "GraniteMoeForCausalLM", - "params_billions": "1.335" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ibm-granite_granite-3.1-1b-a400m-base/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2519 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3299 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0272 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2517 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3501 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1139 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ibm-granite_granite-3.1-1b-a400m-instruct.json b/data/models/ibm-granite_granite-3.1-1b-a400m-instruct.json deleted file mode 100644 index b683c3f957897a8ca2a5fddf8b7c1c3c8d4189d4..0000000000000000000000000000000000000000 --- a/data/models/ibm-granite_granite-3.1-1b-a400m-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "granite-3.1-1b-a400m-instruct", - "id": "ibm-granite/granite-3.1-1b-a400m-instruct", - "developer": "ibm-granite", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "GraniteMoeForCausalLM", - "params_billions": "1.335" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ibm-granite_granite-3.1-1b-a400m-instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4686 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.328 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0453 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2399 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3302 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1217 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ibm-granite_granite-3.1-2b-base.json b/data/models/ibm-granite_granite-3.1-2b-base.json deleted file mode 100644 index 1097555f26ce8fe041387bb0f15943dca5e8ae15..0000000000000000000000000000000000000000 --- a/data/models/ibm-granite_granite-3.1-2b-base.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "granite-3.1-2b-base", - "id": "ibm-granite/granite-3.1-2b-base", - "developer": "ibm-granite", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "GraniteForCausalLM", - "params_billions": "2.534" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ibm-granite_granite-3.1-2b-base/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3522 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4047 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0566 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2777 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3486 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2251 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ibm-granite_granite-3.1-2b-instruct.json b/data/models/ibm-granite_granite-3.1-2b-instruct.json deleted file mode 100644 index 86ebd054e7e5e376422bae6d09a8b65b6723057c..0000000000000000000000000000000000000000 --- a/data/models/ibm-granite_granite-3.1-2b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "granite-3.1-2b-instruct", - "id": "ibm-granite/granite-3.1-2b-instruct", - "developer": "ibm-granite", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "GraniteForCausalLM", - "params_billions": "2.534" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ibm-granite_granite-3.1-2b-instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6286 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4409 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1526 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2894 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3605 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2819 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ibm-granite_granite-3.1-3b-a800m-base.json b/data/models/ibm-granite_granite-3.1-3b-a800m-base.json deleted file mode 100644 index aa750771ea525f0a6a378f496460c6f31b2c6aad..0000000000000000000000000000000000000000 --- a/data/models/ibm-granite_granite-3.1-3b-a800m-base.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "granite-3.1-3b-a800m-base", - "id": "ibm-granite/granite-3.1-3b-a800m-base", - "developer": "ibm-granite", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "GraniteMoeForCausalLM", - "params_billions": "3.299" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ibm-granite_granite-3.1-3b-a800m-base/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2996 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3628 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0453 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2777 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3275 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1793 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ibm-granite_granite-3.1-3b-a800m-instruct.json b/data/models/ibm-granite_granite-3.1-3b-a800m-instruct.json deleted file mode 100644 index 66996527f9ba77888525812279ce90aecd2ec535..0000000000000000000000000000000000000000 --- a/data/models/ibm-granite_granite-3.1-3b-a800m-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "granite-3.1-3b-a800m-instruct", - "id": "ibm-granite/granite-3.1-3b-a800m-instruct", - "developer": "ibm-granite", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "GraniteMoeForCausalLM", - "params_billions": "3.299" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ibm-granite_granite-3.1-3b-a800m-instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5516 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4009 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.114 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2886 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3486 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2148 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ibm-granite_granite-3.1-8b-base.json b/data/models/ibm-granite_granite-3.1-8b-base.json deleted file mode 100644 index b4d0911464827d16fac922a77d13e36d1fde6e0b..0000000000000000000000000000000000000000 --- a/data/models/ibm-granite_granite-3.1-8b-base.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "granite-3.1-8b-base", - "id": "ibm-granite/granite-3.1-8b-base", - "developer": "ibm-granite", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "GraniteForCausalLM", - "params_billions": "8.171" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ibm-granite_granite-3.1-8b-base/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4221 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4777 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0944 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3213 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3922 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3232 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ibm-granite_granite-3.1-8b-instruct.json b/data/models/ibm-granite_granite-3.1-8b-instruct.json deleted file mode 100644 index f862fde3e89ba1277177e7837a85814a8cee2d58..0000000000000000000000000000000000000000 --- a/data/models/ibm-granite_granite-3.1-8b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "granite-3.1-8b-instruct", - "id": "ibm-granite/granite-3.1-8b-instruct", - "developer": "ibm-granite", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "GraniteForCausalLM", - "params_billions": "8.171" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ibm-granite_granite-3.1-8b-instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7208 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5364 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2198 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3121 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4707 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3537 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ibm-granite_granite-3.2-2b-instruct.json b/data/models/ibm-granite_granite-3.2-2b-instruct.json deleted file mode 100644 index fa4138b650c6fd223a3de470aa0982039927a171..0000000000000000000000000000000000000000 --- a/data/models/ibm-granite_granite-3.2-2b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "granite-3.2-2b-instruct", - "id": "ibm-granite/granite-3.2-2b-instruct", - "developer": "ibm-granite", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "GraniteForCausalLM", - "params_billions": "2.534" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ibm-granite_granite-3.2-2b-instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6152 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4387 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1443 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2903 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3646 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2783 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ibm-granite_granite-3.2-8b-instruct.json b/data/models/ibm-granite_granite-3.2-8b-instruct.json deleted file mode 100644 index f1e225fba75cbe7b32e51d856a42f19c875499ef..0000000000000000000000000000000000000000 --- a/data/models/ibm-granite_granite-3.2-8b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "granite-3.2-8b-instruct", - "id": "ibm-granite/granite-3.2-8b-instruct", - "developer": "ibm-granite", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "GraniteForCausalLM", - "params_billions": "8.171" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ibm-granite_granite-3.2-8b-instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7275 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5402 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2379 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3154 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4562 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3512 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ibm-granite_granite-7b-base.json b/data/models/ibm-granite_granite-7b-base.json deleted file mode 100644 index 2d85f477a9aa66e9b30b0d70bf98b3cfdf38a2dc..0000000000000000000000000000000000000000 --- a/data/models/ibm-granite_granite-7b-base.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "granite-7b-base", - "id": "ibm-granite/granite-7b-base", - "developer": "ibm-granite", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "6.738" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ibm-granite_granite-7b-base/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2414 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.348 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0159 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2458 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3555 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1834 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ibm-granite_granite-7b-instruct.json b/data/models/ibm-granite_granite-7b-instruct.json deleted file mode 100644 index 9f1e4dcde0e15795c9f0ed189a06086a57cca768..0000000000000000000000000000000000000000 --- a/data/models/ibm-granite_granite-7b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "granite-7b-instruct", - "id": "ibm-granite/granite-7b-instruct", - "developer": "ibm-granite", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "6.738" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ibm-granite_granite-7b-instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2972 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3723 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0204 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2852 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.402 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2286 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ibm_granite-20b-functioncalling-fc.json b/data/models/ibm_granite-20b-functioncalling-fc.json deleted file mode 100644 index 365ae1c85a22dda8a9898d6f494ee2381944a1f3..0000000000000000000000000000000000000000 --- a/data/models/ibm_granite-20b-functioncalling-fc.json +++ /dev/null @@ -1,851 +0,0 @@ -{ - "model_info": { - "name": "Granite-20b-FunctionCalling (FC)", - "id": "ibm/granite-20b-functioncalling-fc", - "developer": "ibm", - "additional_details": { - "raw_model_name": "Granite-20b-FunctionCalling (FC)", - "organization": "IBM", - "license": "Apache-2.0", - "mode": "FC", - "model_link": "https://huggingface.co/ibm-granite/granite-20b-functioncalling" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/ibm/granite-20b-functioncalling-fc/1775236112.414988", - "retrieved_timestamp": "1775236112.414988", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 93.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 23.23 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 5.23 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 3.2 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 3.43 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 9.97 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 82.35 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 72.92 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 91.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 83.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 81.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 58.7 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 67.83 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 56.7 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 43.75 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 58.33 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 5.38 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 9.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 3.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 6.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 3.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 87.5 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 75.13 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ibm_granite-3-1-8b-instruct-fc.json b/data/models/ibm_granite-3-1-8b-instruct-fc.json deleted file mode 100644 index 77e63748a1f6954d87460bf2b6e841782a1af7b4..0000000000000000000000000000000000000000 --- a/data/models/ibm_granite-3-1-8b-instruct-fc.json +++ /dev/null @@ -1,851 +0,0 @@ -{ - "model_info": { - "name": "Granite-3.1-8B-Instruct (FC)", - "id": "ibm/granite-3-1-8b-instruct-fc", - "developer": "ibm", - "additional_details": { - "raw_model_name": "Granite-3.1-8B-Instruct (FC)", - "organization": "IBM", - "license": "Apache-2.0", - "mode": "FC", - "model_link": "https://huggingface.co/ibm-granite/granite-3.1-8b-instruct" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/ibm/granite-3-1-8b-instruct-fc/1775236112.408531", - "retrieved_timestamp": "1775236112.408531", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 81.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 27.1 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 9.32 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 13.23 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 31.28 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 65.19 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 78.33 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 67.33 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 92.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 84.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 70.0 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 60.33 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 58.53 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 61.82 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 18.75 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 41.67 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 7.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 11.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 2.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 7.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 9.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 0.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 1.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 14.41 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 9.68 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 7.1 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 26.45 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 68.75 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 79.98 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ibm_granite-3-2-8b-instruct-fc.json b/data/models/ibm_granite-3-2-8b-instruct-fc.json deleted file mode 100644 index a8c464aef0943303bb78a7beec3359a7da4f0ce9..0000000000000000000000000000000000000000 --- a/data/models/ibm_granite-3-2-8b-instruct-fc.json +++ /dev/null @@ -1,851 +0,0 @@ -{ - "model_info": { - "name": "Granite-3.2-8B-Instruct (FC)", - "id": "ibm/granite-3-2-8b-instruct-fc", - "developer": "ibm", - "additional_details": { - "raw_model_name": "Granite-3.2-8B-Instruct (FC)", - "organization": "IBM", - "license": "Apache-2.0", - "mode": "FC", - "model_link": "https://huggingface.co/ibm-granite/granite-3.2-8b-instruct" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/ibm/granite-3-2-8b-instruct-fc/1775236112.409559", - "retrieved_timestamp": "1775236112.409559", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 83.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 26.87 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 25.02 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 36.13 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 81.76 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 216.28 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 79.77 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 69.58 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 88.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 88.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 72.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 60.33 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 60.47 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 61.16 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 25.0 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 45.83 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 7.38 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 9.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 3.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 8.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 9.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 0.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 1.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 12.47 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 6.45 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 9.68 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 21.29 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 75.0 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 80.53 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ibm_granite-3.3-8b-instruct.json b/data/models/ibm_granite-3.3-8b-instruct.json deleted file mode 100644 index 50f60becd061e5e7c7227f486a52877423fb2090..0000000000000000000000000000000000000000 --- a/data/models/ibm_granite-3.3-8b-instruct.json +++ /dev/null @@ -1,234 +0,0 @@ -{ - "model_info": { - "name": "IBM Granite 3.3 8B Instruct", - "id": "ibm/granite-3.3-8b-instruct", - "developer": "ibm", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_capabilities/ibm_granite-3.3-8b-instruct/1774096304.5056698", - "retrieved_timestamp": "1774096304.5056698", - "source_metadata": { - "source_name": "helm_capabilities", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_capabilities", - "evaluation_results": [ - { - "evaluation_name": "Mean score", - "source_data": { - "dataset_name": "helm_capabilities", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "The mean of the scores from all columns.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.463, - "details": { - "description": "", - "tab": "Accuracy", - "Mean score - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"9.029614260338473\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU-Pro", - "source_data": { - "dataset_name": "MMLU-Pro", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on MMLU-Pro", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.343, - "details": { - "description": "min=0.343, mean=0.343, max=0.343, sum=0.343 (1)", - "tab": "Accuracy", - "MMLU-Pro - Observed inference time (s)": "{\"description\": \"min=5.079, mean=5.079, max=5.079, sum=5.079 (1)\", \"tab\": \"Efficiency\", \"score\": \"5.079014162302017\"}", - "MMLU-Pro - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "MMLU-Pro - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - # prompt tokens": "{\"description\": \"min=266.391, mean=266.391, max=266.391, sum=266.391 (1)\", \"tab\": \"General information\", \"score\": \"266.391\"}", - "MMLU-Pro - # output tokens": "{\"description\": \"min=364.376, mean=364.376, max=364.376, sum=364.376 (1)\", \"tab\": \"General information\", \"score\": \"364.376\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"all\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.325, - "details": { - "description": "min=0.325, mean=0.325, max=0.325, sum=0.325 (1)", - "tab": "Accuracy", - "GPQA - Observed inference time (s)": "{\"description\": \"min=6.422, mean=6.422, max=6.422, sum=6.422 (1)\", \"tab\": \"Efficiency\", \"score\": \"6.421983559569971\"}", - "GPQA - # eval": "{\"description\": \"min=446, mean=446, max=446, sum=446 (1)\", \"tab\": \"General information\", \"score\": \"446.0\"}", - "GPQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - # prompt tokens": "{\"description\": \"min=281.265, mean=281.265, max=281.265, sum=281.265 (1)\", \"tab\": \"General information\", \"score\": \"281.2645739910314\"}", - "GPQA - # output tokens": "{\"description\": \"min=465.336, mean=465.336, max=465.336, sum=465.336 (1)\", \"tab\": \"General information\", \"score\": \"465.33632286995515\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"gpqa_main\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "IFEval Strict Acc on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.729, - "details": { - "description": "min=0.729, mean=0.729, max=0.729, sum=0.729 (1)", - "tab": "Accuracy", - "IFEval - Observed inference time (s)": "{\"description\": \"min=6.574, mean=6.574, max=6.574, sum=6.574 (1)\", \"tab\": \"Efficiency\", \"score\": \"6.573940407546743\"}", - "IFEval - # eval": "{\"description\": \"min=541, mean=541, max=541, sum=541 (1)\", \"tab\": \"General information\", \"score\": \"541.0\"}", - "IFEval - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - # prompt tokens": "{\"description\": \"min=51.534, mean=51.534, max=51.534, sum=51.534 (1)\", \"tab\": \"General information\", \"score\": \"51.53419593345656\"}", - "IFEval - # output tokens": "{\"description\": \"min=482.37, mean=482.37, max=482.37, sum=482.37 (1)\", \"tab\": \"General information\", \"score\": \"482.36968576709796\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WildBench", - "source_data": { - "dataset_name": "WildBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "WB Score on WildBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.741, - "details": { - "description": "min=0.741, mean=0.741, max=0.741, sum=0.741 (1)", - "tab": "Accuracy", - "WildBench - Observed inference time (s)": "{\"description\": \"min=10.962, mean=10.962, max=10.962, sum=10.962 (1)\", \"tab\": \"Efficiency\", \"score\": \"10.962031789541244\"}", - "WildBench - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "WildBench - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # prompt tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # output tokens": "{\"description\": \"min=784.893, mean=784.893, max=784.893, sum=784.893 (1)\", \"tab\": \"General information\", \"score\": \"784.893\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"v2\"" - } - } - }, - { - "evaluation_name": "Omni-MATH", - "source_data": { - "dataset_name": "Omni-MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Acc on Omni-MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.176, - "details": { - "description": "min=0.176, mean=0.176, max=0.176, sum=0.176 (1)", - "tab": "Accuracy", - "Omni-MATH - Observed inference time (s)": "{\"description\": \"min=16.111, mean=16.111, max=16.111, sum=16.111 (1)\", \"tab\": \"Efficiency\", \"score\": \"16.111101382732393\"}", - "Omni-MATH - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "Omni-MATH - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - # prompt tokens": "{\"description\": \"min=118.438, mean=118.438, max=118.438, sum=118.438 (1)\", \"tab\": \"General information\", \"score\": \"118.438\"}", - "Omni-MATH - # output tokens": "{\"description\": \"min=1162.421, mean=1162.421, max=1162.421, sum=1162.421 (1)\", \"tab\": \"General information\", \"score\": \"1162.421\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/ibm_granite-4-0-350m-fc.json b/data/models/ibm_granite-4-0-350m-fc.json deleted file mode 100644 index fd2caacf44f8b7fa577679a203732ac93c947798..0000000000000000000000000000000000000000 --- a/data/models/ibm_granite-4-0-350m-fc.json +++ /dev/null @@ -1,851 +0,0 @@ -{ - "model_info": { - "name": "Granite-4.0-350m (FC)", - "id": "ibm/granite-4-0-350m-fc", - "developer": "ibm", - "additional_details": { - "raw_model_name": "Granite-4.0-350m (FC)", - "organization": "IBM", - "license": "Apache-2.0", - "mode": "FC", - "model_link": "https://huggingface.co/ibm-granite/granite-4.0-350m" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/ibm/granite-4-0-350m-fc/1775236112.420138", - "retrieved_timestamp": "1775236112.420138", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 103.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 18.98 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 1.44 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 1.74 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 4.85 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 3.44 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 67.92 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 61.67 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 84.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 70.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 55.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 46.11 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 61.24 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 42.36 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 68.75 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 33.33 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 2.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 5.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 0.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 2.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 2.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 0.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 1.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 3.23 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 1.94 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 1.29 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 6.45 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 81.25 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 60.84 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ibm_granite-4.0-h-small.json b/data/models/ibm_granite-4.0-h-small.json deleted file mode 100644 index 7924fa43d0aa4b3a9823c1180c90820bad450f7a..0000000000000000000000000000000000000000 --- a/data/models/ibm_granite-4.0-h-small.json +++ /dev/null @@ -1,528 +0,0 @@ -{ - "model_info": { - "name": "granite-4.0-h-small", - "id": "ibm/granite-4.0-h-small", - "developer": "ibm", - "inference_platform": "unknown", - "additional_details": { - "display_name": "Granite 4.0 Small" - } - }, - "evaluations": [ - { - "evaluation_id": "global-mmlu-lite/ibm_granite-4.0-h-small/1773936583.743359", - "retrieved_timestamp": "1773936583.743359", - "source_metadata": { - "source_name": "Global MMLU Lite Leaderboard", - "source_type": "documentation", - "source_organization_name": "kaggle", - "source_organization_url": "www.kaggle.com", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "kaggle kernel", - "version": "4", - "additional_details": { - "url": "https://www.kaggle.com/code/shivalikasingh95/global-mmlu-lite-sample-notebook" - } - }, - "benchmark": "global-mmlu-lite", - "evaluation_results": [ - { - "evaluation_name": "Global MMLU Lite", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Global MMLU Lite", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7503 - } - }, - { - "evaluation_name": "Culturally Sensitive", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Sensitive", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7182 - } - }, - { - "evaluation_name": "Culturally Agnostic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Agnostic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7826 - } - }, - { - "evaluation_name": "Arabic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Arabic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7613, - "uncertainty": { - "confidence_interval": { - "lower": -0.0419, - "upper": 0.0419, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "English", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - English", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.77, - "uncertainty": { - "confidence_interval": { - "lower": -0.0412, - "upper": 0.0412, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Bengali", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Bengali", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7613, - "uncertainty": { - "confidence_interval": { - "lower": -0.0419, - "upper": 0.0419, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "German", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - German", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.755, - "uncertainty": { - "confidence_interval": { - "lower": -0.0421, - "upper": 0.0421, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "French", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - French", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7594, - "uncertainty": { - "confidence_interval": { - "lower": -0.0419, - "upper": 0.0419, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Hindi", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Hindi", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7575, - "uncertainty": { - "confidence_interval": { - "lower": -0.042, - "upper": 0.042, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Indonesian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Indonesian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7614, - "uncertainty": { - "confidence_interval": { - "lower": -0.0421, - "upper": 0.0421, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Italian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Italian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7525, - "uncertainty": { - "confidence_interval": { - "lower": -0.0423, - "upper": 0.0423, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Japanese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Japanese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7406, - "uncertainty": { - "confidence_interval": { - "lower": -0.0431, - "upper": 0.0431, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Korean", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Korean", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7525, - "uncertainty": { - "confidence_interval": { - "lower": -0.0423, - "upper": 0.0423, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Portuguese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Portuguese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.757, - "uncertainty": { - "confidence_interval": { - "lower": -0.0423, - "upper": 0.0423, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Spanish", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Spanish", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7638, - "uncertainty": { - "confidence_interval": { - "lower": -0.0417, - "upper": 0.0417, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Swahili", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Swahili", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7318, - "uncertainty": { - "confidence_interval": { - "lower": -0.0435, - "upper": 0.0435, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Yoruba", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Yoruba", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6921, - "uncertainty": { - "confidence_interval": { - "lower": -0.0456, - "upper": 0.0456, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Chinese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Chinese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7475, - "uncertainty": { - "confidence_interval": { - "lower": -0.0426, - "upper": 0.0426, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Burmese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Burmese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7419, - "uncertainty": { - "confidence_interval": { - "lower": -0.0429, - "upper": 0.0429, - "method": "unknown" - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ibm_merlinite-7b.json b/data/models/ibm_merlinite-7b.json deleted file mode 100644 index df21c10650be80706bafeb12baf5ea2e51f8d86e..0000000000000000000000000000000000000000 --- a/data/models/ibm_merlinite-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "merlinite-7b", - "id": "ibm/merlinite-7b", - "developer": "ibm", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ibm_merlinite-7b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2499 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5007 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0242 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.297 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4412 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3068 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ibm_powerlm-3b.json b/data/models/ibm_powerlm-3b.json deleted file mode 100644 index 753a05cf96bdab2fc1b831a148bb8c29f555da4c..0000000000000000000000000000000000000000 --- a/data/models/ibm_powerlm-3b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "PowerLM-3b", - "id": "ibm/PowerLM-3b", - "developer": "ibm", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "GraniteForCausalLM", - "params_billions": "3.512" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ibm_PowerLM-3b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3321 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3679 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0363 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2752 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3563 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2016 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/icefog72_ice0.15-02.10-rp.json b/data/models/icefog72_ice0.15-02.10-rp.json deleted file mode 100644 index 115c5a4b7d1538817ad37548d7d5696d3ac0f25c..0000000000000000000000000000000000000000 --- a/data/models/icefog72_ice0.15-02.10-rp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Ice0.15-02.10-RP", - "id": "icefog72/Ice0.15-02.10-RP", - "developer": "icefog72", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/icefog72_Ice0.15-02.10-RP/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5343 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4976 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0574 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2777 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.432 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3066 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/icefog72_ice0.16-02.10-rp.json b/data/models/icefog72_ice0.16-02.10-rp.json deleted file mode 100644 index 60f89cc60170008adafdcb19dd9271110d9c52cd..0000000000000000000000000000000000000000 --- a/data/models/icefog72_ice0.16-02.10-rp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Ice0.16-02.10-RP", - "id": "icefog72/Ice0.16-02.10-RP", - "developer": "icefog72", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/icefog72_Ice0.16-02.10-RP/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5069 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4946 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0589 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2794 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4334 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3068 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/icefog72_ice0.17-03.10-rp.json b/data/models/icefog72_ice0.17-03.10-rp.json deleted file mode 100644 index b23fcdac4bb370e4ff3374f8b0518a8ecc241130..0000000000000000000000000000000000000000 --- a/data/models/icefog72_ice0.17-03.10-rp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Ice0.17-03.10-RP", - "id": "icefog72/Ice0.17-03.10-RP", - "developer": "icefog72", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/icefog72_Ice0.17-03.10-RP/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5124 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5007 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0612 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2819 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4334 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3085 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/icefog72_ice0.27-06.11-rp.json b/data/models/icefog72_ice0.27-06.11-rp.json deleted file mode 100644 index 529f855511c0a1b17f01608079abd934077d52ea..0000000000000000000000000000000000000000 --- a/data/models/icefog72_ice0.27-06.11-rp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Ice0.27-06.11-RP", - "id": "icefog72/Ice0.27-06.11-RP", - "developer": "icefog72", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/icefog72_Ice0.27-06.11-RP/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4918 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5112 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0566 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3121 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4328 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3154 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/icefog72_ice0.29-06.11-rp.json b/data/models/icefog72_ice0.29-06.11-rp.json deleted file mode 100644 index 535da038859f8c2e44babbd3cb83a5166912e28f..0000000000000000000000000000000000000000 --- a/data/models/icefog72_ice0.29-06.11-rp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Ice0.29-06.11-RP", - "id": "icefog72/Ice0.29-06.11-RP", - "developer": "icefog72", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/icefog72_Ice0.29-06.11-RP/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4861 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5088 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0566 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3029 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4459 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3093 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/icefog72_ice0.31-08.11-rp.json b/data/models/icefog72_ice0.31-08.11-rp.json deleted file mode 100644 index 847d96ed744b908268e880909ed0125e75ece8d1..0000000000000000000000000000000000000000 --- a/data/models/icefog72_ice0.31-08.11-rp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Ice0.31-08.11-RP", - "id": "icefog72/Ice0.31-08.11-RP", - "developer": "icefog72", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/icefog72_Ice0.31-08.11-RP/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5146 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5032 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0612 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3079 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4277 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3131 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/icefog72_ice0.32-10.11-rp.json b/data/models/icefog72_ice0.32-10.11-rp.json deleted file mode 100644 index f5ac8146af292c772588c2a635ee76f0f8ea28dd..0000000000000000000000000000000000000000 --- a/data/models/icefog72_ice0.32-10.11-rp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Ice0.32-10.11-RP", - "id": "icefog72/Ice0.32-10.11-RP", - "developer": "icefog72", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/icefog72_Ice0.32-10.11-RP/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4915 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5048 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0514 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3121 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4382 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.31 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/icefog72_ice0.34b-14.11-rp.json b/data/models/icefog72_ice0.34b-14.11-rp.json deleted file mode 100644 index 3a6c2565288e6fa491a79259dac95a90e8d16758..0000000000000000000000000000000000000000 --- a/data/models/icefog72_ice0.34b-14.11-rp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Ice0.34b-14.11-RP", - "id": "icefog72/Ice0.34b-14.11-RP", - "developer": "icefog72", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/icefog72_Ice0.34b-14.11-RP/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4762 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5067 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.065 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3096 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.442 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3125 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/icefog72_ice0.34n-14.11-rp.json b/data/models/icefog72_ice0.34n-14.11-rp.json deleted file mode 100644 index f6488a51f6613cee1e595b83640a7f0000f0f8b5..0000000000000000000000000000000000000000 --- a/data/models/icefog72_ice0.34n-14.11-rp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Ice0.34n-14.11-RP", - "id": "icefog72/Ice0.34n-14.11-RP", - "developer": "icefog72", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/icefog72_Ice0.34n-14.11-RP/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4787 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5091 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0725 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3138 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.438 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3124 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/icefog72_ice0.37-18.11-rp.json b/data/models/icefog72_ice0.37-18.11-rp.json deleted file mode 100644 index da2625d72f558904d65bbbad54241da0594222d4..0000000000000000000000000000000000000000 --- a/data/models/icefog72_ice0.37-18.11-rp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Ice0.37-18.11-RP", - "id": "icefog72/Ice0.37-18.11-RP", - "developer": "icefog72", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/icefog72_Ice0.37-18.11-RP/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4972 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5084 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0642 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3121 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4339 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3143 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/icefog72_ice0.38-19.11-rp.json b/data/models/icefog72_ice0.38-19.11-rp.json deleted file mode 100644 index f2cdd6aa01dde9f6d2a934d0b5c97caec53364c1..0000000000000000000000000000000000000000 --- a/data/models/icefog72_ice0.38-19.11-rp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Ice0.38-19.11-RP", - "id": "icefog72/Ice0.38-19.11-RP", - "developer": "icefog72", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/icefog72_Ice0.38-19.11-RP/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4403 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5101 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0551 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3045 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4367 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.314 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/icefog72_ice0.39-19.11-rp.json b/data/models/icefog72_ice0.39-19.11-rp.json deleted file mode 100644 index bb298fccb172dc2c0b9d19ce324b491466fd33ab..0000000000000000000000000000000000000000 --- a/data/models/icefog72_ice0.39-19.11-rp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Ice0.39-19.11-RP", - "id": "icefog72/Ice0.39-19.11-RP", - "developer": "icefog72", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/icefog72_Ice0.39-19.11-RP/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4757 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5093 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0498 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3104 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4341 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3127 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/icefog72_ice0.40-20.11-rp.json b/data/models/icefog72_ice0.40-20.11-rp.json deleted file mode 100644 index ced224ce6b1d82de61b8994aede2fc550f188c30..0000000000000000000000000000000000000000 --- a/data/models/icefog72_ice0.40-20.11-rp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Ice0.40-20.11-RP", - "id": "icefog72/Ice0.40-20.11-RP", - "developer": "icefog72", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/icefog72_Ice0.40-20.11-RP/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4763 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5093 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0642 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.307 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4446 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3099 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/icefog72_ice0.41-22.11-rp.json b/data/models/icefog72_ice0.41-22.11-rp.json deleted file mode 100644 index c3efc6c2af9fb5a8b46aebc2f9360f57d7c9938c..0000000000000000000000000000000000000000 --- a/data/models/icefog72_ice0.41-22.11-rp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Ice0.41-22.11-RP", - "id": "icefog72/Ice0.41-22.11-RP", - "developer": "icefog72", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/icefog72_Ice0.41-22.11-RP/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.462 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4723 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.031 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2869 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.456 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2618 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/icefog72_ice0.50-16.01-rp.json b/data/models/icefog72_ice0.50-16.01-rp.json deleted file mode 100644 index efd9aaf1c2076aae3b2d667cf75f4430dde5dd7e..0000000000000000000000000000000000000000 --- a/data/models/icefog72_ice0.50-16.01-rp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Ice0.50-16.01-RP", - "id": "icefog72/Ice0.50-16.01-RP", - "developer": "icefog72", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/icefog72_Ice0.50-16.01-RP/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4385 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.498 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0468 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2995 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4381 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3069 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/icefog72_ice0.50.1-16.01-rp.json b/data/models/icefog72_ice0.50.1-16.01-rp.json deleted file mode 100644 index d623d2bf91c5e3358aa223fd5978cf25125fbe9b..0000000000000000000000000000000000000000 --- a/data/models/icefog72_ice0.50.1-16.01-rp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Ice0.50.1-16.01-RP", - "id": "icefog72/Ice0.50.1-16.01-RP", - "developer": "icefog72", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/icefog72_Ice0.50.1-16.01-RP/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4829 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5107 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0612 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3096 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4327 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3132 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/icefog72_ice0.51-16.01-rp.json b/data/models/icefog72_ice0.51-16.01-rp.json deleted file mode 100644 index c290b2e6d3a52fa556c25879548be7a64a629dc0..0000000000000000000000000000000000000000 --- a/data/models/icefog72_ice0.51-16.01-rp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Ice0.51-16.01-RP", - "id": "icefog72/Ice0.51-16.01-RP", - "developer": "icefog72", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/icefog72_Ice0.51-16.01-RP/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4431 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5044 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0514 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3045 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4437 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.306 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/icefog72_ice0.51.1-16.01-rp.json b/data/models/icefog72_ice0.51.1-16.01-rp.json deleted file mode 100644 index 876e0fed35a9c4ce58c020ba0542999093b83659..0000000000000000000000000000000000000000 --- a/data/models/icefog72_ice0.51.1-16.01-rp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Ice0.51.1-16.01-RP", - "id": "icefog72/Ice0.51.1-16.01-RP", - "developer": "icefog72", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/icefog72_Ice0.51.1-16.01-RP/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4573 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5121 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0642 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3062 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4394 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3104 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/icefog72_ice0.52-16.01-rp.json b/data/models/icefog72_ice0.52-16.01-rp.json deleted file mode 100644 index a02584f225f2775e35ee49ef23135b0dcadd06d5..0000000000000000000000000000000000000000 --- a/data/models/icefog72_ice0.52-16.01-rp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Ice0.52-16.01-RP", - "id": "icefog72/Ice0.52-16.01-RP", - "developer": "icefog72", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/icefog72_Ice0.52-16.01-RP/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4503 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5047 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0506 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3087 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4396 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.308 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/icefog72_ice0.52.1-16.01-rp.json b/data/models/icefog72_ice0.52.1-16.01-rp.json deleted file mode 100644 index 35c1333d67a98ebc3ae77badba3fe4dfb764ce2b..0000000000000000000000000000000000000000 --- a/data/models/icefog72_ice0.52.1-16.01-rp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Ice0.52.1-16.01-RP", - "id": "icefog72/Ice0.52.1-16.01-RP", - "developer": "icefog72", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/icefog72_Ice0.52.1-16.01-RP/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4549 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5106 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0627 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3037 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4394 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3105 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/icefog72_ice0.53-16.01-rp.json b/data/models/icefog72_ice0.53-16.01-rp.json deleted file mode 100644 index 31ac789ee16f2cd368ead293fd173849aff9716d..0000000000000000000000000000000000000000 --- a/data/models/icefog72_ice0.53-16.01-rp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Ice0.53-16.01-RP", - "id": "icefog72/Ice0.53-16.01-RP", - "developer": "icefog72", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/icefog72_Ice0.53-16.01-RP/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4741 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5102 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0634 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3087 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4327 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.313 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/icefog72_ice0.54-17.01-rp.json b/data/models/icefog72_ice0.54-17.01-rp.json deleted file mode 100644 index 6842d1122c163453bef81cbd4dd1cdb6900378d9..0000000000000000000000000000000000000000 --- a/data/models/icefog72_ice0.54-17.01-rp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Ice0.54-17.01-RP", - "id": "icefog72/Ice0.54-17.01-RP", - "developer": "icefog72", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/icefog72_Ice0.54-17.01-RP/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4379 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4853 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0408 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3096 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4874 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2326 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/icefog72_ice0.55-17.01-rp.json b/data/models/icefog72_ice0.55-17.01-rp.json deleted file mode 100644 index 43357d25e3fb86e596f74b7eb7f75c1c4d62ad34..0000000000000000000000000000000000000000 --- a/data/models/icefog72_ice0.55-17.01-rp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Ice0.55-17.01-RP", - "id": "icefog72/Ice0.55-17.01-RP", - "developer": "icefog72", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/icefog72_Ice0.55-17.01-RP/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4961 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5077 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0604 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2869 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4725 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2658 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/icefog72_ice0.57-17.01-rp.json b/data/models/icefog72_ice0.57-17.01-rp.json deleted file mode 100644 index bb63ec9de41928af56e9e2b3166e74e9ba774e6a..0000000000000000000000000000000000000000 --- a/data/models/icefog72_ice0.57-17.01-rp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Ice0.57-17.01-RP", - "id": "icefog72/Ice0.57-17.01-RP", - "developer": "icefog72", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/icefog72_Ice0.57-17.01-RP/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5152 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5064 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0514 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.297 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4686 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2651 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/icefog72_ice0.60-18.01-rp.json b/data/models/icefog72_ice0.60-18.01-rp.json deleted file mode 100644 index 947d7dad93a1736ebff2b87c18dfeba6b1381164..0000000000000000000000000000000000000000 --- a/data/models/icefog72_ice0.60-18.01-rp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Ice0.60-18.01-RP", - "id": "icefog72/Ice0.60-18.01-RP", - "developer": "icefog72", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/icefog72_Ice0.60-18.01-RP/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5374 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5094 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0536 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3045 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.467 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2837 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/icefog72_ice0.60.1-18.01-rp.json b/data/models/icefog72_ice0.60.1-18.01-rp.json deleted file mode 100644 index 259e4877dbb460592623585416475be48cf0ad2e..0000000000000000000000000000000000000000 --- a/data/models/icefog72_ice0.60.1-18.01-rp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Ice0.60.1-18.01-RP", - "id": "icefog72/Ice0.60.1-18.01-RP", - "developer": "icefog72", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/icefog72_Ice0.60.1-18.01-RP/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5188 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.512 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0461 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.302 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4498 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2914 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/icefog72_ice0.61-18.01-rp.json b/data/models/icefog72_ice0.61-18.01-rp.json deleted file mode 100644 index 6c381d4d170281c9fef77db6815d5e63529a1a5a..0000000000000000000000000000000000000000 --- a/data/models/icefog72_ice0.61-18.01-rp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Ice0.61-18.01-RP", - "id": "icefog72/Ice0.61-18.01-RP", - "developer": "icefog72", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/icefog72_Ice0.61-18.01-RP/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5441 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5105 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0468 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2987 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4697 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2709 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/icefog72_ice0.62-18.01-rp.json b/data/models/icefog72_ice0.62-18.01-rp.json deleted file mode 100644 index fcf2ea266a48ef1a8836416dc417339888c25b7f..0000000000000000000000000000000000000000 --- a/data/models/icefog72_ice0.62-18.01-rp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Ice0.62-18.01-RP", - "id": "icefog72/Ice0.62-18.01-RP", - "developer": "icefog72", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/icefog72_Ice0.62-18.01-RP/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5367 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5103 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0574 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2978 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4538 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2877 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/icefog72_ice0.62.1-24.01-rp.json b/data/models/icefog72_ice0.62.1-24.01-rp.json deleted file mode 100644 index f9478a0c0dbdb25641edc2ab56420e51fa20b372..0000000000000000000000000000000000000000 --- a/data/models/icefog72_ice0.62.1-24.01-rp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Ice0.62.1-24.01-RP", - "id": "icefog72/Ice0.62.1-24.01-RP", - "developer": "icefog72", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/icefog72_Ice0.62.1-24.01-RP/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5182 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5109 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0559 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3003 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4551 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2871 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/icefog72_ice0.64-24.01-rp.json b/data/models/icefog72_ice0.64-24.01-rp.json deleted file mode 100644 index 17172e1230a02b13fae3264279714966b2f2fa94..0000000000000000000000000000000000000000 --- a/data/models/icefog72_ice0.64-24.01-rp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Ice0.64-24.01-RP", - "id": "icefog72/Ice0.64-24.01-RP", - "developer": "icefog72", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/icefog72_Ice0.64-24.01-RP/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5441 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.506 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0627 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3037 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.462 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2933 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/icefog72_ice0.64.1-24.01-rp.json b/data/models/icefog72_ice0.64.1-24.01-rp.json deleted file mode 100644 index 2af2d5f16d1824152a4d19e50c582720544f8577..0000000000000000000000000000000000000000 --- a/data/models/icefog72_ice0.64.1-24.01-rp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Ice0.64.1-24.01-RP", - "id": "icefog72/Ice0.64.1-24.01-RP", - "developer": "icefog72", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/icefog72_Ice0.64.1-24.01-RP/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5447 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.506 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0627 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3037 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.462 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2933 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/icefog72_ice0.65-25.01-rp.json b/data/models/icefog72_ice0.65-25.01-rp.json deleted file mode 100644 index ad9410155aa84d37f03efc17b8fc075a603dc233..0000000000000000000000000000000000000000 --- a/data/models/icefog72_ice0.65-25.01-rp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Ice0.65-25.01-RP", - "id": "icefog72/Ice0.65-25.01-RP", - "developer": "icefog72", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/icefog72_Ice0.65-25.01-RP/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5029 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5096 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.065 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3045 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.434 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2997 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/icefog72_ice0.66-25.01-rp.json b/data/models/icefog72_ice0.66-25.01-rp.json deleted file mode 100644 index 4cd894c202f7ebe0271c76a7e9766f9a4851c5b0..0000000000000000000000000000000000000000 --- a/data/models/icefog72_ice0.66-25.01-rp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Ice0.66-25.01-RP", - "id": "icefog72/Ice0.66-25.01-RP", - "developer": "icefog72", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/icefog72_Ice0.66-25.01-RP/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5325 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5129 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0604 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.307 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4434 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3039 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/icefog72_ice0.67-25.01-rp.json b/data/models/icefog72_ice0.67-25.01-rp.json deleted file mode 100644 index a782868875af2d173cfb5bdfcf7ff4d1b4f7f380..0000000000000000000000000000000000000000 --- a/data/models/icefog72_ice0.67-25.01-rp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Ice0.67-25.01-RP", - "id": "icefog72/Ice0.67-25.01-RP", - "developer": "icefog72", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/icefog72_Ice0.67-25.01-RP/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5361 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5113 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0748 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2852 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4279 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3097 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/icefog72_ice0.68-25.01-rp.json b/data/models/icefog72_ice0.68-25.01-rp.json deleted file mode 100644 index f9202e1f575e4a511673c32fa30221a126da4f8b..0000000000000000000000000000000000000000 --- a/data/models/icefog72_ice0.68-25.01-rp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Ice0.68-25.01-RP", - "id": "icefog72/Ice0.68-25.01-RP", - "developer": "icefog72", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/icefog72_Ice0.68-25.01-RP/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5514 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.513 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0725 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3104 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4446 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3012 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/icefog72_ice0.69-25.01-rp.json b/data/models/icefog72_ice0.69-25.01-rp.json deleted file mode 100644 index a53314235fc200153edad98eb07dbf1b4f9301bc..0000000000000000000000000000000000000000 --- a/data/models/icefog72_ice0.69-25.01-rp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Ice0.69-25.01-RP", - "id": "icefog72/Ice0.69-25.01-RP", - "developer": "icefog72", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/icefog72_Ice0.69-25.01-RP/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5438 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5098 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0566 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3129 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4486 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2965 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/icefog72_ice0.7-29.09-rp.json b/data/models/icefog72_ice0.7-29.09-rp.json deleted file mode 100644 index 7f7adff6719c1999fdbd88652c750283826c80cc..0000000000000000000000000000000000000000 --- a/data/models/icefog72_ice0.7-29.09-rp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Ice0.7-29.09-RP", - "id": "icefog72/Ice0.7-29.09-RP", - "developer": "icefog72", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/icefog72_Ice0.7-29.09-RP/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5176 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5048 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0665 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2878 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4238 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3127 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/icefog72_ice0.70-25.01-rp.json b/data/models/icefog72_ice0.70-25.01-rp.json deleted file mode 100644 index a69c8ec63825b13b6d3034730d629a882325affb..0000000000000000000000000000000000000000 --- a/data/models/icefog72_ice0.70-25.01-rp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Ice0.70-25.01-RP", - "id": "icefog72/Ice0.70-25.01-RP", - "developer": "icefog72", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/icefog72_Ice0.70-25.01-RP/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5498 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5136 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0597 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3079 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4512 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2996 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/icefog72_ice0.70.1-01.02-rp.json b/data/models/icefog72_ice0.70.1-01.02-rp.json deleted file mode 100644 index 14cb66011d3857f5f38d23cf041fd4db22035bf5..0000000000000000000000000000000000000000 --- a/data/models/icefog72_ice0.70.1-01.02-rp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Ice0.70.1-01.02-RP", - "id": "icefog72/Ice0.70.1-01.02-RP", - "developer": "icefog72", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/icefog72_Ice0.70.1-01.02-RP/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.507 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.506 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.034 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2978 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4599 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2749 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/icefog72_ice0.73-01.02-rp.json b/data/models/icefog72_ice0.73-01.02-rp.json deleted file mode 100644 index d627f19c9c944e74153b4f2be64205ab7aaa1a82..0000000000000000000000000000000000000000 --- a/data/models/icefog72_ice0.73-01.02-rp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Ice0.73-01.02-RP", - "id": "icefog72/Ice0.73-01.02-RP", - "developer": "icefog72", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/icefog72_Ice0.73-01.02-RP/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5292 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5103 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0385 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2911 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4664 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2702 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/icefog72_ice0.74-02.02-rp.json b/data/models/icefog72_ice0.74-02.02-rp.json deleted file mode 100644 index 28540ebd8e7881aa70db9f1623224e98e36cd309..0000000000000000000000000000000000000000 --- a/data/models/icefog72_ice0.74-02.02-rp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Ice0.74-02.02-RP", - "id": "icefog72/Ice0.74-02.02-RP", - "developer": "icefog72", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/icefog72_Ice0.74-02.02-RP/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2935 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4646 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0015 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2961 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.428 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2143 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/icefog72_ice0.76-02.02-rp.json b/data/models/icefog72_ice0.76-02.02-rp.json deleted file mode 100644 index 2f7be49e3d9b2342c18d9fafe779aeb58cab6bbc..0000000000000000000000000000000000000000 --- a/data/models/icefog72_ice0.76-02.02-rp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Ice0.76-02.02-RP", - "id": "icefog72/Ice0.76-02.02-RP", - "developer": "icefog72", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/icefog72_Ice0.76-02.02-RP/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4529 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5086 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0144 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2869 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4362 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2652 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/icefog72_ice0.77-02.02-rp.json b/data/models/icefog72_ice0.77-02.02-rp.json deleted file mode 100644 index 84543d32e4e228193b5f6fcec3e5902a27ac3515..0000000000000000000000000000000000000000 --- a/data/models/icefog72_ice0.77-02.02-rp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Ice0.77-02.02-RP", - "id": "icefog72/Ice0.77-02.02-RP", - "developer": "icefog72", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/icefog72_Ice0.77-02.02-RP/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.531 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5109 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0393 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2852 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4765 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2999 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/icefog72_ice0.78-02.02-rp.json b/data/models/icefog72_ice0.78-02.02-rp.json deleted file mode 100644 index d4c13b4c71f38807baa98ea77448ae2cef2cbc84..0000000000000000000000000000000000000000 --- a/data/models/icefog72_ice0.78-02.02-rp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Ice0.78-02.02-RP", - "id": "icefog72/Ice0.78-02.02-RP", - "developer": "icefog72", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/icefog72_Ice0.78-02.02-RP/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4053 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5002 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0438 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2936 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4686 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2955 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/icefog72_ice0.80-03.02-rp.json b/data/models/icefog72_ice0.80-03.02-rp.json deleted file mode 100644 index 2c4d35af9d8d906f81eab88b8a0ebe26c607b81d..0000000000000000000000000000000000000000 --- a/data/models/icefog72_ice0.80-03.02-rp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Ice0.80-03.02-RP", - "id": "icefog72/Ice0.80-03.02-RP", - "developer": "icefog72", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/icefog72_Ice0.80-03.02-RP/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5516 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5098 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0559 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2785 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4923 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2912 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/icefog72_icecocoarp-7b.json b/data/models/icefog72_icecocoarp-7b.json deleted file mode 100644 index 3e423965b29f0e8138e45af26c9f61d76107c7e1..0000000000000000000000000000000000000000 --- a/data/models/icefog72_icecocoarp-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "IceCocoaRP-7b", - "id": "icefog72/IceCocoaRP-7b", - "developer": "icefog72", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/icefog72_IceCocoaRP-7b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4962 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4938 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0574 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2953 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4198 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3098 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/icefog72_icecoffeerp-7b.json b/data/models/icefog72_icecoffeerp-7b.json deleted file mode 100644 index f43264abf5a47bc93f86f51b191245a1f442d798..0000000000000000000000000000000000000000 --- a/data/models/icefog72_icecoffeerp-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "IceCoffeeRP-7b", - "id": "icefog72/IceCoffeeRP-7b", - "developer": "icefog72", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/icefog72_IceCoffeeRP-7b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4959 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4889 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0544 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2852 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.416 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2975 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/icefog72_icedrinkbyfrankensteinv3rp.json b/data/models/icefog72_icedrinkbyfrankensteinv3rp.json deleted file mode 100644 index 9e099a468696eb7a3e402dab78f6ef16e6916aaa..0000000000000000000000000000000000000000 --- a/data/models/icefog72_icedrinkbyfrankensteinv3rp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "IceDrinkByFrankensteinV3RP", - "id": "icefog72/IceDrinkByFrankensteinV3RP", - "developer": "icefog72", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/icefog72_IceDrinkByFrankensteinV3RP/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4975 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4833 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0506 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2617 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4253 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2927 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/icefog72_icedrinknamegoeshererp-7b-model_stock.json b/data/models/icefog72_icedrinknamegoeshererp-7b-model_stock.json deleted file mode 100644 index 97e3935122adf3cec58eba9832b4e940e91a25b9..0000000000000000000000000000000000000000 --- a/data/models/icefog72_icedrinknamegoeshererp-7b-model_stock.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "IceDrinkNameGoesHereRP-7b-Model_Stock", - "id": "icefog72/IceDrinkNameGoesHereRP-7b-Model_Stock", - "developer": "icefog72", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/icefog72_IceDrinkNameGoesHereRP-7b-Model_Stock/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4968 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4658 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0408 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2685 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4067 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2817 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/icefog72_icedrinknamenotfoundrp-7b-model_stock.json b/data/models/icefog72_icedrinknamenotfoundrp-7b-model_stock.json deleted file mode 100644 index 32351babc4f7612feb651b4189486288f2b8d7fa..0000000000000000000000000000000000000000 --- a/data/models/icefog72_icedrinknamenotfoundrp-7b-model_stock.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "IceDrinkNameNotFoundRP-7b-Model_Stock", - "id": "icefog72/IceDrinkNameNotFoundRP-7b-Model_Stock", - "developer": "icefog72", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/icefog72_IceDrinkNameNotFoundRP-7b-Model_Stock/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.513 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5026 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0604 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2777 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4372 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3064 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/icefog72_icedrunkcherryrp-7b.json b/data/models/icefog72_icedrunkcherryrp-7b.json deleted file mode 100644 index f748cb82e19f0716632903dad85c07c8041fe2ca..0000000000000000000000000000000000000000 --- a/data/models/icefog72_icedrunkcherryrp-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "IceDrunkCherryRP-7b", - "id": "icefog72/IceDrunkCherryRP-7b", - "developer": "icefog72", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/icefog72_IceDrunkCherryRP-7b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4898 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4847 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0612 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2768 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4292 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3009 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/icefog72_icedrunkencherryrp-7b.json b/data/models/icefog72_icedrunkencherryrp-7b.json deleted file mode 100644 index 13ad82167182c42f44d8517a51a95aab31a66440..0000000000000000000000000000000000000000 --- a/data/models/icefog72_icedrunkencherryrp-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "IceDrunkenCherryRP-7b", - "id": "icefog72/IceDrunkenCherryRP-7b", - "developer": "icefog72", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/icefog72_IceDrunkenCherryRP-7b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4763 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5093 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0642 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.307 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4446 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3099 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/icefog72_iceespressorpv2-7b.json b/data/models/icefog72_iceespressorpv2-7b.json deleted file mode 100644 index 35a3e129e802328191a44a3551ab795bf5c0387e..0000000000000000000000000000000000000000 --- a/data/models/icefog72_iceespressorpv2-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "IceEspressoRPv2-7b", - "id": "icefog72/IceEspressoRPv2-7b", - "developer": "icefog72", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/icefog72_IceEspressoRPv2-7b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4977 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5055 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0619 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2894 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4331 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3061 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/icefog72_icelemontearp-32k-7b.json b/data/models/icefog72_icelemontearp-32k-7b.json deleted file mode 100644 index 72e3b515532ca1d9d1f530b22e8ec8346daf6e8d..0000000000000000000000000000000000000000 --- a/data/models/icefog72_icelemontearp-32k-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "IceLemonTeaRP-32k-7b", - "id": "icefog72/IceLemonTeaRP-32k-7b", - "developer": "icefog72", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/icefog72_IceLemonTeaRP-32k-7b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5212 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4997 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0544 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2903 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.429 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3068 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/icefog72_icemartinirp-7b.json b/data/models/icefog72_icemartinirp-7b.json deleted file mode 100644 index d8f73be3e9644cabc0e8b22e780f89017693a902..0000000000000000000000000000000000000000 --- a/data/models/icefog72_icemartinirp-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "IceMartiniRP-7b", - "id": "icefog72/IceMartiniRP-7b", - "developer": "icefog72", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/icefog72_IceMartiniRP-7b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5045 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4972 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0665 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2794 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4345 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3073 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/icefog72_icenalyvkarp-7b.json b/data/models/icefog72_icenalyvkarp-7b.json deleted file mode 100644 index 8680fa51a743ca90e5646baa73c42bca78566d2e..0000000000000000000000000000000000000000 --- a/data/models/icefog72_icenalyvkarp-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "IceNalyvkaRP-7b", - "id": "icefog72/IceNalyvkaRP-7b", - "developer": "icefog72", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/icefog72_IceNalyvkaRP-7b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5498 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5136 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0597 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3079 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4512 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2996 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/icefog72_icesakerp-7b.json b/data/models/icefog72_icesakerp-7b.json deleted file mode 100644 index 2367ec828761c0013a28208108a9813a74170857..0000000000000000000000000000000000000000 --- a/data/models/icefog72_icesakerp-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "IceSakeRP-7b", - "id": "icefog72/IceSakeRP-7b", - "developer": "icefog72", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/icefog72_IceSakeRP-7b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5228 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5119 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0634 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2852 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.413 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3177 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/icefog72_icesakev4rp-7b.json b/data/models/icefog72_icesakev4rp-7b.json deleted file mode 100644 index 27af600b360a0c883ed28fda141e1c1ac0133f72..0000000000000000000000000000000000000000 --- a/data/models/icefog72_icesakev4rp-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "IceSakeV4RP-7b", - "id": "icefog72/IceSakeV4RP-7b", - "developer": "icefog72", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/icefog72_IceSakeV4RP-7b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4634 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.493 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0559 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2945 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4082 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3103 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/icefog72_icesakev6rp-7b.json b/data/models/icefog72_icesakev6rp-7b.json deleted file mode 100644 index f3267e4e350d12bc0c15245dbc590141fe9832d9..0000000000000000000000000000000000000000 --- a/data/models/icefog72_icesakev6rp-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "IceSakeV6RP-7b", - "id": "icefog72/IceSakeV6RP-7b", - "developer": "icefog72", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/icefog72_IceSakeV6RP-7b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5033 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4976 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0619 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2911 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.42 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3093 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/icefog72_icesakev8rp-7b.json b/data/models/icefog72_icesakev8rp-7b.json deleted file mode 100644 index 2ceb3dbce71c186a120361bd2de4fd72b34481c2..0000000000000000000000000000000000000000 --- a/data/models/icefog72_icesakev8rp-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "IceSakeV8RP-7b", - "id": "icefog72/IceSakeV8RP-7b", - "developer": "icefog72", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/icefog72_IceSakeV8RP-7b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6086 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4885 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0597 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.276 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3993 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.301 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/icefog72_icetea21energydrinkrpv13-dpov3.5.json b/data/models/icefog72_icetea21energydrinkrpv13-dpov3.5.json deleted file mode 100644 index 1ac3ea9e093e77dfd058fe484dee62fa44af1d97..0000000000000000000000000000000000000000 --- a/data/models/icefog72_icetea21energydrinkrpv13-dpov3.5.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "IceTea21EnergyDrinkRPV13-DPOv3.5", - "id": "icefog72/IceTea21EnergyDrinkRPV13-DPOv3.5", - "developer": "icefog72", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/icefog72_IceTea21EnergyDrinkRPV13-DPOv3.5/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4871 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.44 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0363 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2844 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3964 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2498 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/icefog72_icetea21energydrinkrpv13-dpov3.json b/data/models/icefog72_icetea21energydrinkrpv13-dpov3.json deleted file mode 100644 index 3ef70befba5b1a733306899baa4227deff1cffd2..0000000000000000000000000000000000000000 --- a/data/models/icefog72_icetea21energydrinkrpv13-dpov3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "IceTea21EnergyDrinkRPV13-DPOv3", - "id": "icefog72/IceTea21EnergyDrinkRPV13-DPOv3", - "developer": "icefog72", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/icefog72_IceTea21EnergyDrinkRPV13-DPOv3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5263 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.502 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0582 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2836 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4372 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3056 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/idea-ccnl_ziya-llama-13b-v1.json b/data/models/idea-ccnl_ziya-llama-13b-v1.json deleted file mode 100644 index e4f7378fded3ad0453ab11518f87bedc71056a00..0000000000000000000000000000000000000000 --- a/data/models/idea-ccnl_ziya-llama-13b-v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Ziya-LLaMA-13B-v1", - "id": "IDEA-CCNL/Ziya-LLaMA-13B-v1", - "developer": "IDEA-CCNL", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "13.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/IDEA-CCNL_Ziya-LLaMA-13B-v1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1697 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2877 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2492 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3751 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1101 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/idea-ccnl_ziya-llama-7b-reward.json b/data/models/idea-ccnl_ziya-llama-7b-reward.json deleted file mode 100644 index e0e2d0d2d58713ee2d5d62976ab319a680402e89..0000000000000000000000000000000000000000 --- a/data/models/idea-ccnl_ziya-llama-7b-reward.json +++ /dev/null @@ -1,144 +0,0 @@ -{ - "model_info": { - "name": "IDEA-CCNL/Ziya-LLaMA-7B-Reward", - "id": "IDEA-CCNL/Ziya-LLaMA-7B-Reward", - "developer": "IDEA-CCNL", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/IDEA-CCNL_Ziya-LLaMA-7B-Reward/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6378 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8687 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4605 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6405 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5775 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6461 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ifable_gemma-2-ifable-9b.json b/data/models/ifable_gemma-2-ifable-9b.json deleted file mode 100644 index 88bcf014c19452c86c54c608777560921e3a8afe..0000000000000000000000000000000000000000 --- a/data/models/ifable_gemma-2-ifable-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "gemma-2-Ifable-9B", - "id": "ifable/gemma-2-Ifable-9B", - "developer": "ifable", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "9.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ifable_gemma-2-Ifable-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2984 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5866 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1397 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3414 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4053 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4226 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ifaz_llama31_8b_en_emo_v4.json b/data/models/ifaz_llama31_8b_en_emo_v4.json deleted file mode 100644 index e371de6269969036a1642f6169845e20f96755fb..0000000000000000000000000000000000000000 --- a/data/models/ifaz_llama31_8b_en_emo_v4.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "llama31_8B_en_emo_v4", - "id": "iFaz/llama31_8B_en_emo_v4", - "developer": "iFaz", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "", - "params_billions": "4.777" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/iFaz_llama31_8B_en_emo_v4/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3043 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4916 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0884 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.297 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3643 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3049 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ifaz_llama32_1b_en_emo_v1.json b/data/models/ifaz_llama32_1b_en_emo_v1.json deleted file mode 100644 index 022456c631f0632bf3dbf922179dc9fe155bc83d..0000000000000000000000000000000000000000 --- a/data/models/ifaz_llama32_1b_en_emo_v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "llama32_1B_en_emo_v1", - "id": "iFaz/llama32_1B_en_emo_v1", - "developer": "iFaz", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "0.765" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/iFaz_llama32_1B_en_emo_v1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4408 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.338 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0378 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.25 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3489 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1761 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ifaz_llama32_3b_en_emo_1000_stp.json b/data/models/ifaz_llama32_3b_en_emo_1000_stp.json deleted file mode 100644 index e066cf017d626b5b36c2099f7077e3a5b8908e4b..0000000000000000000000000000000000000000 --- a/data/models/ifaz_llama32_3b_en_emo_1000_stp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "llama32_3B_en_emo_1000_stp", - "id": "iFaz/llama32_3B_en_emo_1000_stp", - "developer": "iFaz", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.848" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/iFaz_llama32_3B_en_emo_1000_stp/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7295 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4522 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1465 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2777 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3621 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3123 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ifaz_llama32_3b_en_emo_2000_stp.json b/data/models/ifaz_llama32_3b_en_emo_2000_stp.json deleted file mode 100644 index a9d4cfe8bf8f86982c487e5fc8aca81877190741..0000000000000000000000000000000000000000 --- a/data/models/ifaz_llama32_3b_en_emo_2000_stp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "llama32_3B_en_emo_2000_stp", - "id": "iFaz/llama32_3B_en_emo_2000_stp", - "developer": "iFaz", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.848" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/iFaz_llama32_3B_en_emo_2000_stp/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7369 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4535 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1533 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2836 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3527 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3098 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ifaz_llama32_3b_en_emo_300_stp.json b/data/models/ifaz_llama32_3b_en_emo_300_stp.json deleted file mode 100644 index d051c562948be82b2da84b16a2f55238ad31bafc..0000000000000000000000000000000000000000 --- a/data/models/ifaz_llama32_3b_en_emo_300_stp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "llama32_3B_en_emo_300_stp", - "id": "iFaz/llama32_3B_en_emo_300_stp", - "developer": "iFaz", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.848" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/iFaz_llama32_3B_en_emo_300_stp/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7256 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4505 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1601 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2743 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3621 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3148 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ifaz_llama32_3b_en_emo_5000_stp.json b/data/models/ifaz_llama32_3b_en_emo_5000_stp.json deleted file mode 100644 index 512372ff12316ca45705de0b4dfc4a60ed631134..0000000000000000000000000000000000000000 --- a/data/models/ifaz_llama32_3b_en_emo_5000_stp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "llama32_3B_en_emo_5000_stp", - "id": "iFaz/llama32_3B_en_emo_5000_stp", - "developer": "iFaz", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.848" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/iFaz_llama32_3B_en_emo_5000_stp/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.71 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4568 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1292 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2794 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3446 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3067 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ifaz_llama32_3b_en_emo_v2.json b/data/models/ifaz_llama32_3b_en_emo_v2.json deleted file mode 100644 index 40ecb933d22dcd762c2bfbdcd4e5f01d605baa19..0000000000000000000000000000000000000000 --- a/data/models/ifaz_llama32_3b_en_emo_v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "llama32_3B_en_emo_v2", - "id": "iFaz/llama32_3B_en_emo_v2", - "developer": "iFaz", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.848" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/iFaz_llama32_3B_en_emo_v2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5454 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4284 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1088 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2676 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3482 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3004 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ifaz_llama32_3b_en_emo_v3.json b/data/models/ifaz_llama32_3b_en_emo_v3.json deleted file mode 100644 index a0d940e03648c699b2b6220be90b5c4807239a2f..0000000000000000000000000000000000000000 --- a/data/models/ifaz_llama32_3b_en_emo_v3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "llama32_3B_en_emo_v3", - "id": "iFaz/llama32_3B_en_emo_v3", - "developer": "iFaz", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.848" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/iFaz_llama32_3B_en_emo_v3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5759 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4301 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.068 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2676 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3553 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.271 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ilsp_llama-krikri-8b-instruct.json b/data/models/ilsp_llama-krikri-8b-instruct.json deleted file mode 100644 index 8463815224d33b9eab8cba95fe8d9c2367ca7f65..0000000000000000000000000000000000000000 --- a/data/models/ilsp_llama-krikri-8b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-Krikri-8B-Instruct", - "id": "ilsp/Llama-Krikri-8B-Instruct", - "developer": "ilsp", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.202" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ilsp_Llama-Krikri-8B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6079 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5047 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1178 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3029 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.408 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3313 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ilyagusev_gemma-2-2b-it-abliterated.json b/data/models/ilyagusev_gemma-2-2b-it-abliterated.json deleted file mode 100644 index 3db3a90a136f1fc216e5964a75cf987d372f3df3..0000000000000000000000000000000000000000 --- a/data/models/ilyagusev_gemma-2-2b-it-abliterated.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "gemma-2-2b-it-abliterated", - "id": "IlyaGusev/gemma-2-2b-it-abliterated", - "developer": "IlyaGusev", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "2.614" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/IlyaGusev_gemma-2-2b-it-abliterated/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5331 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4119 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0612 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2651 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3782 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2538 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ilyagusev_gemma-2-9b-it-abliterated.json b/data/models/ilyagusev_gemma-2-9b-it-abliterated.json deleted file mode 100644 index 5ba16c1062b196350ce61d487a78a2af91996338..0000000000000000000000000000000000000000 --- a/data/models/ilyagusev_gemma-2-9b-it-abliterated.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "gemma-2-9b-it-abliterated", - "id": "IlyaGusev/gemma-2-9b-it-abliterated", - "developer": "IlyaGusev", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "9.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/IlyaGusev_gemma-2-9b-it-abliterated/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7473 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5906 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1775 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3456 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4034 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3915 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/infinirc_infinirc-llama3-8b-2g-release-v1.0.json b/data/models/infinirc_infinirc-llama3-8b-2g-release-v1.0.json deleted file mode 100644 index e7d5982c8aa57df5b691690c8a9a43864bbb4f92..0000000000000000000000000000000000000000 --- a/data/models/infinirc_infinirc-llama3-8b-2g-release-v1.0.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Infinirc-Llama3-8B-2G-Release-v1.0", - "id": "Infinirc/Infinirc-Llama3-8B-2G-Release-v1.0", - "developer": "Infinirc", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Infinirc_Infinirc-Llama3-8B-2G-Release-v1.0/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2024 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4351 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0166 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2995 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4609 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.216 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/inflatebot_mn-12b-mag-mell-r1.json b/data/models/inflatebot_mn-12b-mag-mell-r1.json deleted file mode 100644 index c82878032c4f5c69bc0a83953709cd0f800a66ac..0000000000000000000000000000000000000000 --- a/data/models/inflatebot_mn-12b-mag-mell-r1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MN-12B-Mag-Mell-R1", - "id": "inflatebot/MN-12B-Mag-Mell-R1", - "developer": "inflatebot", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/inflatebot_MN-12B-Mag-Mell-R1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4613 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5304 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1299 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3163 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4002 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3438 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/infly_inf-orm-llama3.1-70b.json b/data/models/infly_inf-orm-llama3.1-70b.json deleted file mode 100644 index 82e76ad6cd43b3a105b801967d5f616a3924844a..0000000000000000000000000000000000000000 --- a/data/models/infly_inf-orm-llama3.1-70b.json +++ /dev/null @@ -1,276 +0,0 @@ -{ - "model_info": { - "name": "infly/INF-ORM-Llama3.1-70B", - "id": "infly/INF-ORM-Llama3.1-70B", - "developer": "infly", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/infly_INF-ORM-Llama3.1-70B/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9511 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9665 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9101 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9365 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9912 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "reward-bench-2/infly_INF-ORM-Llama3.1-70B/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7648 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7411 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4188 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6995 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9644 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.903 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8622 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/informatiker_qwen2-7b-instruct-abliterated.json b/data/models/informatiker_qwen2-7b-instruct-abliterated.json deleted file mode 100644 index 6963b1836a4d794261f6575418b3010eaea6ff34..0000000000000000000000000000000000000000 --- a/data/models/informatiker_qwen2-7b-instruct-abliterated.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2-7B-Instruct-abliterated", - "id": "informatiker/Qwen2-7B-Instruct-abliterated", - "developer": "informatiker", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/informatiker_Qwen2-7B-Instruct-abliterated/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5822 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5534 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2636 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3012 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3888 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3873 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/insait-institute_bggpt-gemma-2-27b-it-v1.0.json b/data/models/insait-institute_bggpt-gemma-2-27b-it-v1.0.json deleted file mode 100644 index a8139af7fc7a6e6d714106a727d45d02f88b3744..0000000000000000000000000000000000000000 --- a/data/models/insait-institute_bggpt-gemma-2-27b-it-v1.0.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "BgGPT-Gemma-2-27B-IT-v1.0", - "id": "INSAIT-Institute/BgGPT-Gemma-2-27B-IT-v1.0", - "developer": "INSAIT-Institute", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "27.227" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/INSAIT-Institute_BgGPT-Gemma-2-27B-IT-v1.0/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2912 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2601 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3575 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1167 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/insightfactory_llama-3.2-3b-instruct-unsloth-bnb-4bitlora_model.json b/data/models/insightfactory_llama-3.2-3b-instruct-unsloth-bnb-4bitlora_model.json deleted file mode 100644 index 518f627358fd3b21978dc22b167c8347221cdbac..0000000000000000000000000000000000000000 --- a/data/models/insightfactory_llama-3.2-3b-instruct-unsloth-bnb-4bitlora_model.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.2-3B-Instruct-unsloth-bnb-4bitlora_model", - "id": "insightfactory/Llama-3.2-3B-Instruct-unsloth-bnb-4bitlora_model", - "developer": "insightfactory", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "", - "params_billions": "1.933" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/insightfactory_Llama-3.2-3B-Instruct-unsloth-bnb-4bitlora_model/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4588 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4146 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.105 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2718 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3499 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.296 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/instruction-pretrain_instructlm-500m.json b/data/models/instruction-pretrain_instructlm-500m.json deleted file mode 100644 index 1755b5c2d74c3c4370371d87bfd903bea6ae0ab0..0000000000000000000000000000000000000000 --- a/data/models/instruction-pretrain_instructlm-500m.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "InstructLM-500M", - "id": "instruction-pretrain/InstructLM-500M", - "developer": "instruction-pretrain", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "0.5" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/instruction-pretrain_InstructLM-500M/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1028 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2941 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2567 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3528 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1141 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/intel_neural-chat-7b-v3-1.json b/data/models/intel_neural-chat-7b-v3-1.json deleted file mode 100644 index 0b9a61469aba1143330206a125ae8194bcffc942..0000000000000000000000000000000000000000 --- a/data/models/intel_neural-chat-7b-v3-1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "neural-chat-7b-v3-1", - "id": "Intel/neural-chat-7b-v3-1", - "developer": "Intel", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Intel_neural-chat-7b-v3-1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4687 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5052 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0355 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2903 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4979 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2678 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/intel_neural-chat-7b-v3-2.json b/data/models/intel_neural-chat-7b-v3-2.json deleted file mode 100644 index bbc2ba5e7080b5555455586bc7dfce76e0b8cf2c..0000000000000000000000000000000000000000 --- a/data/models/intel_neural-chat-7b-v3-2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "neural-chat-7b-v3-2", - "id": "Intel/neural-chat-7b-v3-2", - "developer": "Intel", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "7.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Intel_neural-chat-7b-v3-2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4988 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5032 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0476 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2903 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4895 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2667 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/intel_neural-chat-7b-v3-3.json b/data/models/intel_neural-chat-7b-v3-3.json deleted file mode 100644 index 49424b2764f1a03c96f3e87c8d03135a0488c804..0000000000000000000000000000000000000000 --- a/data/models/intel_neural-chat-7b-v3-3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "neural-chat-7b-v3-3", - "id": "Intel/neural-chat-7b-v3-3", - "developer": "Intel", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "7.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Intel_neural-chat-7b-v3-3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4763 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4877 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0408 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2894 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.486 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2625 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/intel_neural-chat-7b-v3.json b/data/models/intel_neural-chat-7b-v3.json deleted file mode 100644 index 20cb818949838d8223e2140e5a33ade1fd04882f..0000000000000000000000000000000000000000 --- a/data/models/intel_neural-chat-7b-v3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "neural-chat-7b-v3", - "id": "Intel/neural-chat-7b-v3", - "developer": "Intel", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "7.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Intel_neural-chat-7b-v3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2778 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5048 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0295 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2919 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5055 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2699 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/internlm_internlm2-1_8b-reward.json b/data/models/internlm_internlm2-1_8b-reward.json deleted file mode 100644 index db02a96edd7d4deec66c9db68ed23c5ecfdc96f9..0000000000000000000000000000000000000000 --- a/data/models/internlm_internlm2-1_8b-reward.json +++ /dev/null @@ -1,276 +0,0 @@ -{ - "model_info": { - "name": "internlm/internlm2-1_8b-reward", - "id": "internlm/internlm2-1_8b-reward", - "developer": "internlm", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/internlm_internlm2-1_8b-reward/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8217 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9358 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6623 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8162 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8724 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "reward-bench-2/internlm_internlm2-1_8b-reward/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3902 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2758 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3625 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4426 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4711 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.596 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1934 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/internlm_internlm2-1_8b.json b/data/models/internlm_internlm2-1_8b.json deleted file mode 100644 index 4ee310ecf2f8fb8341ff99cdbe36f212a9eebf10..0000000000000000000000000000000000000000 --- a/data/models/internlm_internlm2-1_8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "internlm2-1_8b", - "id": "internlm/internlm2-1_8b", - "developer": "internlm", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "InternLM2ForCausalLM", - "params_billions": "8.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/internlm_internlm2-1_8b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2198 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.388 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0211 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2483 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3813 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1588 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/internlm_internlm2-20b-reward.json b/data/models/internlm_internlm2-20b-reward.json deleted file mode 100644 index db57bc6ddd293d585b6bca7ac06b0b270dabb864..0000000000000000000000000000000000000000 --- a/data/models/internlm_internlm2-20b-reward.json +++ /dev/null @@ -1,276 +0,0 @@ -{ - "model_info": { - "name": "internlm/internlm2-20b-reward", - "id": "internlm/internlm2-20b-reward", - "developer": "internlm", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/internlm_internlm2-20b-reward/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5628 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5558 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3625 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5738 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6111 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7253 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5483 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "reward-bench/internlm_internlm2-20b-reward/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9016 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9888 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7654 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8946 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9576 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/internlm_internlm2-7b-reward.json b/data/models/internlm_internlm2-7b-reward.json deleted file mode 100644 index 5907aad77cdf2d42a7b25e1a1a35520112be4497..0000000000000000000000000000000000000000 --- a/data/models/internlm_internlm2-7b-reward.json +++ /dev/null @@ -1,276 +0,0 @@ -{ - "model_info": { - "name": "internlm/internlm2-7b-reward", - "id": "internlm/internlm2-7b-reward", - "developer": "internlm", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/internlm_internlm2-7b-reward/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8759 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9916 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6952 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8716 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9453 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "reward-bench-2/internlm_internlm2-7b-reward/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5335 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4211 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5628 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5956 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7051 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5164 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/internlm_internlm2-7b.json b/data/models/internlm_internlm2-7b.json deleted file mode 100644 index bf7e458d494c420484d2bf59e4edda6003818574..0000000000000000000000000000000000000000 --- a/data/models/internlm_internlm2-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "internlm2-7b", - "id": "internlm/internlm2-7b", - "developer": "internlm", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Unknown", - "params_billions": "0.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/internlm_internlm2-7b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.228 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5825 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0857 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3367 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.44 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.19 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/internlm_internlm2-chat-1_8b.json b/data/models/internlm_internlm2-chat-1_8b.json deleted file mode 100644 index 12427964245013de899d80767d2c5a66414e94a3..0000000000000000000000000000000000000000 --- a/data/models/internlm_internlm2-chat-1_8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "internlm2-chat-1_8b", - "id": "internlm/internlm2-chat-1_8b", - "developer": "internlm", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "InternLM2ForCausalLM", - "params_billions": "1.889" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/internlm_internlm2-chat-1_8b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2387 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4452 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0325 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2659 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3631 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1839 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/internlm_internlm2_5-1_8b-chat.json b/data/models/internlm_internlm2_5-1_8b-chat.json deleted file mode 100644 index aeb240c09d6defccf6eaccf38670fea623f2a2fa..0000000000000000000000000000000000000000 --- a/data/models/internlm_internlm2_5-1_8b-chat.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "internlm2_5-1_8b-chat", - "id": "internlm/internlm2_5-1_8b-chat", - "developer": "internlm", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "InternLM2ForCausalLM", - "params_billions": "1.89" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/internlm_internlm2_5-1_8b-chat/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3849 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4489 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1586 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2903 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3594 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1299 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/internlm_internlm2_5-20b-chat.json b/data/models/internlm_internlm2_5-20b-chat.json deleted file mode 100644 index 6393bc103a99e9d8a448b420c0efcda54603f792..0000000000000000000000000000000000000000 --- a/data/models/internlm_internlm2_5-20b-chat.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "internlm2_5-20b-chat", - "id": "internlm/internlm2_5-20b-chat", - "developer": "internlm", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "InternLM2ForCausalLM", - "params_billions": "19.86" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/internlm_internlm2_5-20b-chat/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.701 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7474 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4079 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3213 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4558 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3998 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/internlm_internlm2_5-7b-chat.json b/data/models/internlm_internlm2_5-7b-chat.json deleted file mode 100644 index 8350fbe499389c17fbb79305f2f4d3cee08b4739..0000000000000000000000000000000000000000 --- a/data/models/internlm_internlm2_5-7b-chat.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "internlm2_5-7b-chat", - "id": "internlm/internlm2_5-7b-chat", - "developer": "internlm", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "InternLM2ForCausalLM", - "params_billions": "7.738" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/internlm_internlm2_5-7b-chat/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5539 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7073 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.253 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3473 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4594 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3777 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/intervitens_mini-magnum-12b-v1.1.json b/data/models/intervitens_mini-magnum-12b-v1.1.json deleted file mode 100644 index 578bdd105d284104dc9e2e9a9236ddf1c6256ca0..0000000000000000000000000000000000000000 --- a/data/models/intervitens_mini-magnum-12b-v1.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "mini-magnum-12b-v1.1", - "id": "intervitens/mini-magnum-12b-v1.1", - "developer": "intervitens", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/intervitens_mini-magnum-12b-v1.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5156 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5062 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0619 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2886 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4004 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3291 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/intervitensinc_internlm2_5-20b-llamafied.json b/data/models/intervitensinc_internlm2_5-20b-llamafied.json deleted file mode 100644 index c74f76a3f413315e0ca51075bad93f5df6615032..0000000000000000000000000000000000000000 --- a/data/models/intervitensinc_internlm2_5-20b-llamafied.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "internlm2_5-20b-llamafied", - "id": "IntervitensInc/internlm2_5-20b-llamafied", - "developer": "IntervitensInc", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "19.861" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/IntervitensInc_internlm2_5-20b-llamafied/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.341 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7478 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1715 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3381 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4475 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4051 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/inumulaisk_eval_model.json b/data/models/inumulaisk_eval_model.json deleted file mode 100644 index 2ba70789cf0b2af2e8216cd4d7158efa23dc8754..0000000000000000000000000000000000000000 --- a/data/models/inumulaisk_eval_model.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "eval_model", - "id": "inumulaisk/eval_model", - "developer": "inumulaisk", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.777" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/inumulaisk_eval_model/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1931 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3512 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2976 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2794 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.358 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1664 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/invalid-coder_sakura-solar-instruct-carbonvillain-en-10.7b-v2-slerp.json b/data/models/invalid-coder_sakura-solar-instruct-carbonvillain-en-10.7b-v2-slerp.json deleted file mode 100644 index 7e7031f371823e07c170b174a3047d41a9d6c2fd..0000000000000000000000000000000000000000 --- a/data/models/invalid-coder_sakura-solar-instruct-carbonvillain-en-10.7b-v2-slerp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Sakura-SOLAR-Instruct-CarbonVillain-en-10.7B-v2-slerp", - "id": "invalid-coder/Sakura-SOLAR-Instruct-CarbonVillain-en-10.7B-v2-slerp", - "developer": "invalid-coder", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "10.732" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/invalid-coder_Sakura-SOLAR-Instruct-CarbonVillain-en-10.7B-v2-slerp/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4555 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5158 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0491 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3054 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3992 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3146 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/invalid-null_peiyangme-0.5.json b/data/models/invalid-null_peiyangme-0.5.json deleted file mode 100644 index 91db1963f990c8a18c3dfadd8d4eaecfc8a419cd..0000000000000000000000000000000000000000 --- a/data/models/invalid-null_peiyangme-0.5.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "PeiYangMe-0.5", - "id": "Invalid-Null/PeiYangMe-0.5", - "developer": "Invalid-Null", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "6.061" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Invalid-Null_PeiYangMe-0.5/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1409 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2791 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2441 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3738 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1109 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/invalid-null_peiyangme-0.7.json b/data/models/invalid-null_peiyangme-0.7.json deleted file mode 100644 index e1cf6f6746f20665665773432341306c4e587b4c..0000000000000000000000000000000000000000 --- a/data/models/invalid-null_peiyangme-0.7.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "PeiYangMe-0.7", - "id": "Invalid-Null/PeiYangMe-0.7", - "developer": "Invalid-Null", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "6.061" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Invalid-Null_PeiYangMe-0.7/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1491 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3028 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0113 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2332 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3857 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1101 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/invisietch_etherealrainbow-v0.2-8b.json b/data/models/invisietch_etherealrainbow-v0.2-8b.json deleted file mode 100644 index f6f4893a2bfb21888299dafd21606f6f585fb525..0000000000000000000000000000000000000000 --- a/data/models/invisietch_etherealrainbow-v0.2-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "EtherealRainbow-v0.2-8B", - "id": "invisietch/EtherealRainbow-v0.2-8B", - "developer": "invisietch", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/invisietch_EtherealRainbow-v0.2-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3903 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5102 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0823 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3029 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3827 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3653 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/invisietch_etherealrainbow-v0.3-8b.json b/data/models/invisietch_etherealrainbow-v0.3-8b.json deleted file mode 100644 index f6a8aa9df49e44e2d66d82ce34e0ff07f623426b..0000000000000000000000000000000000000000 --- a/data/models/invisietch_etherealrainbow-v0.3-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "EtherealRainbow-v0.3-8B", - "id": "invisietch/EtherealRainbow-v0.3-8B", - "developer": "invisietch", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/invisietch_EtherealRainbow-v0.3-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3682 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5097 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0763 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3045 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3904 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3626 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/invisietch_mis-firefly-v0.2-22b.json b/data/models/invisietch_mis-firefly-v0.2-22b.json deleted file mode 100644 index 8129c01165fcd99dfe9582da1235af57e4d61b52..0000000000000000000000000000000000000000 --- a/data/models/invisietch_mis-firefly-v0.2-22b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MiS-Firefly-v0.2-22B", - "id": "invisietch/MiS-Firefly-v0.2-22B", - "developer": "invisietch", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "22.247" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/invisietch_MiS-Firefly-v0.2-22B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5371 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5514 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1654 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3045 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4694 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.362 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/invisietch_nimbus-miqu-v0.1-70b.json b/data/models/invisietch_nimbus-miqu-v0.1-70b.json deleted file mode 100644 index d4ff61672646c414bb2a93b2a4a02c15820ded52..0000000000000000000000000000000000000000 --- a/data/models/invisietch_nimbus-miqu-v0.1-70b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Nimbus-Miqu-v0.1-70B", - "id": "invisietch/Nimbus-Miqu-v0.1-70B", - "developer": "invisietch", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "68.977" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/invisietch_Nimbus-Miqu-v0.1-70B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4647 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.601 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0604 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3389 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4133 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3853 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/irahulpandey_mistralai-7b-slerp-v0.1.json b/data/models/irahulpandey_mistralai-7b-slerp-v0.1.json deleted file mode 100644 index 9597b5ee78b3823e9ddf2094d117374e41437b39..0000000000000000000000000000000000000000 --- a/data/models/irahulpandey_mistralai-7b-slerp-v0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "mistralai-7B-slerp-v0.1", - "id": "irahulpandey/mistralai-7B-slerp-v0.1", - "developer": "irahulpandey", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/irahulpandey_mistralai-7B-slerp-v0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4966 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5011 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0514 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3037 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.455 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2951 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/iryanbell_arc1-ii.json b/data/models/iryanbell_arc1-ii.json deleted file mode 100644 index cb9b2cd52530798406c3ab3e523e66e6af37eff2..0000000000000000000000000000000000000000 --- a/data/models/iryanbell_arc1-ii.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ARC1-II", - "id": "iRyanBell/ARC1-II", - "developer": "iRyanBell", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/iRyanBell_ARC1-II/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1708 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3382 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0219 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2718 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4913 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1686 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/iryanbell_arc1.json b/data/models/iryanbell_arc1.json deleted file mode 100644 index 17b43ba064bc4e6728d0bcf78cd32a2349106299..0000000000000000000000000000000000000000 --- a/data/models/iryanbell_arc1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ARC1", - "id": "iRyanBell/ARC1", - "developer": "iRyanBell", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/iRyanBell_ARC1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4411 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4903 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0687 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2945 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3991 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3371 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/isaak-carter_josiefied-qwen2.5-7b-instruct-abliterated-v2.json b/data/models/isaak-carter_josiefied-qwen2.5-7b-instruct-abliterated-v2.json deleted file mode 100644 index 9da75d351d99b2019745f15642d3f1f72933d728..0000000000000000000000000000000000000000 --- a/data/models/isaak-carter_josiefied-qwen2.5-7b-instruct-abliterated-v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Josiefied-Qwen2.5-7B-Instruct-abliterated-v2", - "id": "Isaak-Carter/Josiefied-Qwen2.5-7B-Instruct-abliterated-v2", - "developer": "Isaak-Carter", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Isaak-Carter_Josiefied-Qwen2.5-7B-Instruct-abliterated-v2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7841 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5311 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4721 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2987 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4354 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4128 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/isaak-carter_josiefied-qwen2.5-7b-instruct-abliterated.json b/data/models/isaak-carter_josiefied-qwen2.5-7b-instruct-abliterated.json deleted file mode 100644 index 93140ab8e841d243f6727cae9783ec977b5a4756..0000000000000000000000000000000000000000 --- a/data/models/isaak-carter_josiefied-qwen2.5-7b-instruct-abliterated.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Josiefied-Qwen2.5-7B-Instruct-abliterated", - "id": "Isaak-Carter/Josiefied-Qwen2.5-7B-Instruct-abliterated", - "developer": "Isaak-Carter", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Isaak-Carter_Josiefied-Qwen2.5-7B-Instruct-abliterated/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7317 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5396 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4924 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3029 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4087 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4276 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/isaak-carter_josiev4o-8b-stage1-v4.json b/data/models/isaak-carter_josiev4o-8b-stage1-v4.json deleted file mode 100644 index 4be5b49897aa6863276434fc48c4d6962992c15e..0000000000000000000000000000000000000000 --- a/data/models/isaak-carter_josiev4o-8b-stage1-v4.json +++ /dev/null @@ -1,275 +0,0 @@ -{ - "model_info": { - "name": "JOSIEv4o-8b-stage1-v4", - "id": "Isaak-Carter/JOSIEv4o-8b-stage1-v4", - "developer": "Isaak-Carter", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Isaak-Carter_JOSIEv4o-8b-stage1-v4/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2553 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4725 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0529 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2919 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3654 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3316 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "hfopenllm_v2/Isaak-Carter_JOSIEv4o-8b-stage1-v4/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2477 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4758 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0453 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2911 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3641 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3292 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/j-lab_thynk_orpo.json b/data/models/j-lab_thynk_orpo.json deleted file mode 100644 index 15a957aa05f059bce1b2b736aae87e7454c6d041..0000000000000000000000000000000000000000 --- a/data/models/j-lab_thynk_orpo.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Thynk_orpo", - "id": "J-LAB/Thynk_orpo", - "developer": "J-LAB", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.086" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/J-LAB_Thynk_orpo/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2102 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4463 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.148 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2928 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4515 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3231 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jackfram_llama-160m.json b/data/models/jackfram_llama-160m.json deleted file mode 100644 index 68d3823186d5435ac632b88252946815ae96e807..0000000000000000000000000000000000000000 --- a/data/models/jackfram_llama-160m.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "llama-160m", - "id": "JackFram/llama-160m", - "developer": "JackFram", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "0.162" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JackFram_llama-160m/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1791 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2888 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0083 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2617 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3792 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1128 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jackfram_llama-68m.json b/data/models/jackfram_llama-68m.json deleted file mode 100644 index 9f5ac96b7bbd2f69fea74e6c3b16e19593c74b6a..0000000000000000000000000000000000000000 --- a/data/models/jackfram_llama-68m.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "llama-68m", - "id": "JackFram/llama-68m", - "developer": "JackFram", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "0.068" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JackFram_llama-68m/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1726 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2936 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.006 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2584 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.391 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1144 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jacoby746_casual-magnum-34b.json b/data/models/jacoby746_casual-magnum-34b.json deleted file mode 100644 index c6d593b2d7dd9eb3a60334a15409ad73e1e0783f..0000000000000000000000000000000000000000 --- a/data/models/jacoby746_casual-magnum-34b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Casual-Magnum-34B", - "id": "Jacoby746/Casual-Magnum-34B", - "developer": "Jacoby746", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "34.389" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Jacoby746_Casual-Magnum-34B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.193 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6032 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0921 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3725 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4078 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5184 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jacoby746_inf-silent-kunoichi-v0.1-2x7b.json b/data/models/jacoby746_inf-silent-kunoichi-v0.1-2x7b.json deleted file mode 100644 index bb0d7999b01d9c588a233debd04a4b267ffaef27..0000000000000000000000000000000000000000 --- a/data/models/jacoby746_inf-silent-kunoichi-v0.1-2x7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Inf-Silent-Kunoichi-v0.1-2x7B", - "id": "Jacoby746/Inf-Silent-Kunoichi-v0.1-2x7B", - "developer": "Jacoby746", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MixtralForCausalLM", - "params_billions": "12.879" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Jacoby746_Inf-Silent-Kunoichi-v0.1-2x7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.388 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5185 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.071 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2894 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.428 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3271 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jacoby746_inf-silent-kunoichi-v0.2-2x7b.json b/data/models/jacoby746_inf-silent-kunoichi-v0.2-2x7b.json deleted file mode 100644 index 1b1e5478c6d83131b5b278e89bfcdbf1502ef720..0000000000000000000000000000000000000000 --- a/data/models/jacoby746_inf-silent-kunoichi-v0.2-2x7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Inf-Silent-Kunoichi-v0.2-2x7B", - "id": "Jacoby746/Inf-Silent-Kunoichi-v0.2-2x7B", - "developer": "Jacoby746", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MixtralForCausalLM", - "params_billions": "12.879" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Jacoby746_Inf-Silent-Kunoichi-v0.2-2x7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3636 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5209 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0627 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3003 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.432 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3272 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jacoby746_proto-athena-4x7b.json b/data/models/jacoby746_proto-athena-4x7b.json deleted file mode 100644 index fdc426f323bccfead20dc62c6bcdd94be6b71ba2..0000000000000000000000000000000000000000 --- a/data/models/jacoby746_proto-athena-4x7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Proto-Athena-4x7B", - "id": "Jacoby746/Proto-Athena-4x7B", - "developer": "Jacoby746", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MixtralForCausalLM", - "params_billions": "24.154" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Jacoby746_Proto-Athena-4x7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3703 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5107 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.065 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2945 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4348 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3206 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jacoby746_proto-athena-v0.2-4x7b.json b/data/models/jacoby746_proto-athena-v0.2-4x7b.json deleted file mode 100644 index f5490a965cb31be117084d907341ef12f44298a6..0000000000000000000000000000000000000000 --- a/data/models/jacoby746_proto-athena-v0.2-4x7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Proto-Athena-v0.2-4x7B", - "id": "Jacoby746/Proto-Athena-v0.2-4x7B", - "developer": "Jacoby746", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MixtralForCausalLM", - "params_billions": "24.154" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Jacoby746_Proto-Athena-v0.2-4x7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3752 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5068 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0634 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2987 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4213 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3197 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jacoby746_proto-harpy-blazing-light-v0.1-2x7b.json b/data/models/jacoby746_proto-harpy-blazing-light-v0.1-2x7b.json deleted file mode 100644 index 66fe8547294f003dbedeb2e9f07909886e675633..0000000000000000000000000000000000000000 --- a/data/models/jacoby746_proto-harpy-blazing-light-v0.1-2x7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Proto-Harpy-Blazing-Light-v0.1-2x7B", - "id": "Jacoby746/Proto-Harpy-Blazing-Light-v0.1-2x7B", - "developer": "Jacoby746", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MixtralForCausalLM", - "params_billions": "12.879" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Jacoby746_Proto-Harpy-Blazing-Light-v0.1-2x7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4905 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5187 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0748 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2953 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.445 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3301 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jacoby746_proto-harpy-spark-v0.1-7b.json b/data/models/jacoby746_proto-harpy-spark-v0.1-7b.json deleted file mode 100644 index acadb480b007e34cff0071169b86ada06b1f8f50..0000000000000000000000000000000000000000 --- a/data/models/jacoby746_proto-harpy-spark-v0.1-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Proto-Harpy-Spark-v0.1-7B", - "id": "Jacoby746/Proto-Harpy-Spark-v0.1-7B", - "developer": "Jacoby746", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Jacoby746_Proto-Harpy-Spark-v0.1-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4333 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4736 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0619 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3054 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4317 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3069 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaredjoss_pythia-410m-roberta-lr_8e7-kl_01-steps_12000-rlhf-model.json b/data/models/jaredjoss_pythia-410m-roberta-lr_8e7-kl_01-steps_12000-rlhf-model.json deleted file mode 100644 index 2b2b327164e9298244a968dc90f0c3b2ee6972df..0000000000000000000000000000000000000000 --- a/data/models/jaredjoss_pythia-410m-roberta-lr_8e7-kl_01-steps_12000-rlhf-model.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "pythia-410m-roberta-lr_8e7-kl_01-steps_12000-rlhf-model", - "id": "jaredjoss/pythia-410m-roberta-lr_8e7-kl_01-steps_12000-rlhf-model", - "developer": "jaredjoss", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "GPTNeoXForCausalLM", - "params_billions": "0.407" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaredjoss_pythia-410m-roberta-lr_8e7-kl_01-steps_12000-rlhf-model/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1572 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2863 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2592 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3607 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1169 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_auro-kosmos-evaa-v2-8b.json b/data/models/jaspionjader_auro-kosmos-evaa-v2-8b.json deleted file mode 100644 index 9b2f50dd962bd160d389d10c6ff126be30df7066..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_auro-kosmos-evaa-v2-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Auro-Kosmos-EVAA-v2-8B", - "id": "jaspionjader/Auro-Kosmos-EVAA-v2-8B", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_Auro-Kosmos-EVAA-v2-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4778 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5447 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1412 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3154 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.425 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3858 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_auro-kosmos-evaa-v2.1-8b.json b/data/models/jaspionjader_auro-kosmos-evaa-v2.1-8b.json deleted file mode 100644 index 3da8df6da942551ebea9f46ce19d2a04db58a64b..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_auro-kosmos-evaa-v2.1-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Auro-Kosmos-EVAA-v2.1-8B", - "id": "jaspionjader/Auro-Kosmos-EVAA-v2.1-8B", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_Auro-Kosmos-EVAA-v2.1-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4666 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5444 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1458 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3087 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4317 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3826 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_auro-kosmos-evaa-v2.2-8b.json b/data/models/jaspionjader_auro-kosmos-evaa-v2.2-8b.json deleted file mode 100644 index 2478a3f231befc0b2b3e3c90cdbc056471fb17f8..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_auro-kosmos-evaa-v2.2-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Auro-Kosmos-EVAA-v2.2-8B", - "id": "jaspionjader/Auro-Kosmos-EVAA-v2.2-8B", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_Auro-Kosmos-EVAA-v2.2-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4268 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5431 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1412 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3104 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4251 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3798 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_auro-kosmos-evaa-v2.3-8b.json b/data/models/jaspionjader_auro-kosmos-evaa-v2.3-8b.json deleted file mode 100644 index 01d25799e553dd241ed4153e30bd81904cb86117..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_auro-kosmos-evaa-v2.3-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Auro-Kosmos-EVAA-v2.3-8B", - "id": "jaspionjader/Auro-Kosmos-EVAA-v2.3-8B", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_Auro-Kosmos-EVAA-v2.3-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4271 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5441 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1344 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3121 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4278 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3784 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_bbb-1.json b/data/models/jaspionjader_bbb-1.json deleted file mode 100644 index fb414159d261f06fa151ef7938dcca64802c0cfc..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_bbb-1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "bbb-1", - "id": "jaspionjader/bbb-1", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_bbb-1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4864 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5376 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1367 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3138 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4171 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3897 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_bbb-2.json b/data/models/jaspionjader_bbb-2.json deleted file mode 100644 index 9121fd01986521362ebee30911bae2950ccfc9a4..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_bbb-2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "bbb-2", - "id": "jaspionjader/bbb-2", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_bbb-2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4077 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5067 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1125 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.302 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4145 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3635 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_bbb-3.json b/data/models/jaspionjader_bbb-3.json deleted file mode 100644 index a3553943e47f717fc6f007d6c0c3de0581b0ca2b..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_bbb-3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "bbb-3", - "id": "jaspionjader/bbb-3", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_bbb-3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4168 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5158 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1405 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3112 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4265 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3856 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_bbb-4.json b/data/models/jaspionjader_bbb-4.json deleted file mode 100644 index edd6f42fcc37a88f97bc6f21e18e3e533e7e993a..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_bbb-4.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "bbb-4", - "id": "jaspionjader/bbb-4", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_bbb-4/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4768 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5212 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1276 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2978 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4092 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3773 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_bbb-5.json b/data/models/jaspionjader_bbb-5.json deleted file mode 100644 index 223192e97bbb7d11e47164088e330ab047e6d43e..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_bbb-5.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "bbb-5", - "id": "jaspionjader/bbb-5", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_bbb-5/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4703 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5207 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1397 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3045 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3998 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3834 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_bbb-6.json b/data/models/jaspionjader_bbb-6.json deleted file mode 100644 index 064e41b57d93af49a333d4473af87ee24f0122f4..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_bbb-6.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "bbb-6", - "id": "jaspionjader/bbb-6", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_bbb-6/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.488 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5211 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.139 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3104 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4052 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3871 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_bbb-7.json b/data/models/jaspionjader_bbb-7.json deleted file mode 100644 index e65c4b22658fbdd75e016d5713f29778c8d6cec4..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_bbb-7.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "bbb-7", - "id": "jaspionjader/bbb-7", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_bbb-7/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4828 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5211 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1367 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3104 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4038 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.386 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_bh-1.json b/data/models/jaspionjader_bh-1.json deleted file mode 100644 index 5c23ba5bb66643698aa9605d544f4cb1f5d87366..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_bh-1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "bh-1", - "id": "jaspionjader/bh-1", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_bh-1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4284 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.589 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0536 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2945 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4441 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3449 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_bh-10.json b/data/models/jaspionjader_bh-10.json deleted file mode 100644 index 631a48ca90182795f6c8c4b3711f154fc3b5542b..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_bh-10.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "bh-10", - "id": "jaspionjader/bh-10", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_bh-10/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4618 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5856 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1103 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3003 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4199 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3708 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_bh-11.json b/data/models/jaspionjader_bh-11.json deleted file mode 100644 index f1908b901f199d63505080cbf05ae4b14a2e3bc0..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_bh-11.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "bh-11", - "id": "jaspionjader/bh-11", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_bh-11/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4575 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5851 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1178 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.307 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4146 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3738 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_bh-12.json b/data/models/jaspionjader_bh-12.json deleted file mode 100644 index 9184df07bcf3613a101b7bf0ddc05d7c2dec75f7..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_bh-12.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "bh-12", - "id": "jaspionjader/bh-12", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_bh-12/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4734 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5802 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1186 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3003 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4145 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3737 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_bh-13.json b/data/models/jaspionjader_bh-13.json deleted file mode 100644 index 5ec1985fb83e528a9a4172d5e9167e4fd3fa98ad..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_bh-13.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "bh-13", - "id": "jaspionjader/bh-13", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_bh-13/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4698 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5778 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1125 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.307 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4159 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.373 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_bh-15.json b/data/models/jaspionjader_bh-15.json deleted file mode 100644 index bd17429ac6f1641dcbb46e815c0b6fc696eee65c..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_bh-15.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "bh-15", - "id": "jaspionjader/bh-15", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_bh-15/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4745 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5819 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1246 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2987 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4105 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3767 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_bh-16.json b/data/models/jaspionjader_bh-16.json deleted file mode 100644 index 2374d1926a6329412c6f9aa034c7e46107f7bc93..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_bh-16.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "bh-16", - "id": "jaspionjader/bh-16", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_bh-16/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4731 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5783 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1193 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3029 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4159 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3776 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_bh-17.json b/data/models/jaspionjader_bh-17.json deleted file mode 100644 index e09c500c97847ff5573910d09353e1e4c32479b4..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_bh-17.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "bh-17", - "id": "jaspionjader/bh-17", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_bh-17/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4722 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5776 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1133 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2978 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4158 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3757 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_bh-18.json b/data/models/jaspionjader_bh-18.json deleted file mode 100644 index 937e0af90d7996dd10846d5749771bc2f13b8ed3..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_bh-18.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "bh-18", - "id": "jaspionjader/bh-18", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_bh-18/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4725 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5824 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1186 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3003 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4185 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3757 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_bh-19.json b/data/models/jaspionjader_bh-19.json deleted file mode 100644 index fa0b884be5a2ebaa4febc73234a95f51e06d256b..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_bh-19.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "bh-19", - "id": "jaspionjader/bh-19", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_bh-19/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4584 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5766 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1193 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.297 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4171 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3775 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_bh-2.json b/data/models/jaspionjader_bh-2.json deleted file mode 100644 index 73c1c7f24363d3e2e348f2e65146a80dd1ed2a7b..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_bh-2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "bh-2", - "id": "jaspionjader/bh-2", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_bh-2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4579 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5937 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1027 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3012 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4186 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3695 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_bh-20.json b/data/models/jaspionjader_bh-20.json deleted file mode 100644 index bb92dcfbe7929103c44675c3e549f83aa4184650..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_bh-20.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "bh-20", - "id": "jaspionjader/bh-20", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_bh-20/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4727 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.575 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1201 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2878 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4105 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3768 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_bh-21.json b/data/models/jaspionjader_bh-21.json deleted file mode 100644 index bc8604415d35a1746dcd977221a152f10bbbd1f0..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_bh-21.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "bh-21", - "id": "jaspionjader/bh-21", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_bh-21/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.47 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5738 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1216 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2978 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4158 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3776 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_bh-22.json b/data/models/jaspionjader_bh-22.json deleted file mode 100644 index 2a0f875dad8ac7b10351bfbde1279f8fb609313a..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_bh-22.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "bh-22", - "id": "jaspionjader/bh-22", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_bh-22/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.46 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5793 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1186 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2961 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4172 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3764 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_bh-23.json b/data/models/jaspionjader_bh-23.json deleted file mode 100644 index 06c3c0b2271ca8dcb50be184a71b51cbc4f62547..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_bh-23.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "bh-23", - "id": "jaspionjader/bh-23", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_bh-23/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4658 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.57 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1201 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2945 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4197 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3796 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_bh-24.json b/data/models/jaspionjader_bh-24.json deleted file mode 100644 index 8ee96b7ecc001dfd5eb427c58e7ec3df17464c00..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_bh-24.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "bh-24", - "id": "jaspionjader/bh-24", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_bh-24/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4715 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5717 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1269 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2961 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4158 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3809 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_bh-25.json b/data/models/jaspionjader_bh-25.json deleted file mode 100644 index c96a20ce9ce29f31faaddf8693817a6221105b2c..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_bh-25.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "bh-25", - "id": "jaspionjader/bh-25", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_bh-25/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4752 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5706 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1133 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2911 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4118 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3782 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_bh-26.json b/data/models/jaspionjader_bh-26.json deleted file mode 100644 index 911c9078af2de6be81d3bf116bbd41f31ba7fede..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_bh-26.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "bh-26", - "id": "jaspionjader/bh-26", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_bh-26/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4691 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5735 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1163 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2995 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4277 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3772 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_bh-27.json b/data/models/jaspionjader_bh-27.json deleted file mode 100644 index a75f33d632b53b25340ac9b4565d2716639a0a9d..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_bh-27.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "bh-27", - "id": "jaspionjader/bh-27", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_bh-27/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4819 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5714 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1276 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2961 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4091 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3799 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_bh-28.json b/data/models/jaspionjader_bh-28.json deleted file mode 100644 index 8e60f1f353680764ebfc08e7789ec71c5d7f3fdc..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_bh-28.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "bh-28", - "id": "jaspionjader/bh-28", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_bh-28/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4785 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5703 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1231 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2987 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4131 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3812 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_bh-29.json b/data/models/jaspionjader_bh-29.json deleted file mode 100644 index 291c98c636cfd5493d7d3c7bf8f08037802bfcf6..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_bh-29.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "bh-29", - "id": "jaspionjader/bh-29", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_bh-29/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4688 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.567 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1208 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2953 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4237 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3819 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_bh-3.json b/data/models/jaspionjader_bh-3.json deleted file mode 100644 index 4d93afd4f27815822f970b60a914d06216c17faa..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_bh-3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "bh-3", - "id": "jaspionjader/bh-3", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_bh-3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4664 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5891 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1148 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.302 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4173 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3702 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_bh-30.json b/data/models/jaspionjader_bh-30.json deleted file mode 100644 index c08a0c5164db6e4704cf8ba990c24762aaa0eb9d..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_bh-30.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "bh-30", - "id": "jaspionjader/bh-30", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_bh-30/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4666 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5706 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1231 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2928 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4144 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3782 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_bh-31.json b/data/models/jaspionjader_bh-31.json deleted file mode 100644 index dffdd310a9c8acc8568be535732319b9b495b845..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_bh-31.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "bh-31", - "id": "jaspionjader/bh-31", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_bh-31/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4727 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5665 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1284 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2936 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4104 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.382 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_bh-32.json b/data/models/jaspionjader_bh-32.json deleted file mode 100644 index 1642dc94fc88645fef7dc7565d17bb4037784635..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_bh-32.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "bh-32", - "id": "jaspionjader/bh-32", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_bh-32/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4636 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5662 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1246 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.297 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4157 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3812 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_bh-33.json b/data/models/jaspionjader_bh-33.json deleted file mode 100644 index a19e4ad56a2ec76edd23f39217ce6a03ef1579db..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_bh-33.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "bh-33", - "id": "jaspionjader/bh-33", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_bh-33/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4685 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5653 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1178 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2961 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4157 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3808 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_bh-34.json b/data/models/jaspionjader_bh-34.json deleted file mode 100644 index 051851ef806a3fb6723642589b3f71a90dccce4c..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_bh-34.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "bh-34", - "id": "jaspionjader/bh-34", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_bh-34/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4624 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5681 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1208 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2919 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4185 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3804 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_bh-35.json b/data/models/jaspionjader_bh-35.json deleted file mode 100644 index 6ee37829ef2a7c44bad452666b4bc82b0de35863..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_bh-35.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "bh-35", - "id": "jaspionjader/bh-35", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_bh-35/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4721 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.564 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1246 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2953 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4183 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.383 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_bh-36.json b/data/models/jaspionjader_bh-36.json deleted file mode 100644 index 684220b67666fc324b54cc2a76c2754ee7154664..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_bh-36.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "bh-36", - "id": "jaspionjader/bh-36", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_bh-36/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4666 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5664 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1239 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.302 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4196 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3831 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_bh-37.json b/data/models/jaspionjader_bh-37.json deleted file mode 100644 index 0929c084457979633b236716006cd80da5b00387..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_bh-37.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "bh-37", - "id": "jaspionjader/bh-37", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_bh-37/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.488 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5625 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1216 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2945 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4156 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3828 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_bh-38.json b/data/models/jaspionjader_bh-38.json deleted file mode 100644 index f42da7741157f490e5f693d8671185ea8a5c936b..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_bh-38.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "bh-38", - "id": "jaspionjader/bh-38", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_bh-38/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4618 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5658 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1239 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2978 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4117 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3811 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_bh-39.json b/data/models/jaspionjader_bh-39.json deleted file mode 100644 index 65582952788cfb4d7037150d9f80c59a22aef959..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_bh-39.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "bh-39", - "id": "jaspionjader/bh-39", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_bh-39/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4576 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5633 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1254 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3003 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4262 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3831 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_bh-4.json b/data/models/jaspionjader_bh-4.json deleted file mode 100644 index 5f108475052fae4fa6f773287d92d13912ca7725..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_bh-4.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "bh-4", - "id": "jaspionjader/bh-4", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_bh-4/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4673 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5892 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1095 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2961 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4173 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3705 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_bh-40.json b/data/models/jaspionjader_bh-40.json deleted file mode 100644 index a48ba56c73a5ea2ef0780cf94da8369082183917..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_bh-40.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "bh-40", - "id": "jaspionjader/bh-40", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_bh-40/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4536 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5634 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1246 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2987 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4236 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3835 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_bh-41.json b/data/models/jaspionjader_bh-41.json deleted file mode 100644 index 8d9243c9cb25fe86e459725a0494e33e7cd51946..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_bh-41.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "bh-41", - "id": "jaspionjader/bh-41", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_bh-41/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.474 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5614 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1254 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2928 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4183 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3825 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_bh-42.json b/data/models/jaspionjader_bh-42.json deleted file mode 100644 index d2981802e8fdb575b9c32a6df29d0a1b3a958cc1..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_bh-42.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "bh-42", - "id": "jaspionjader/bh-42", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_bh-42/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.466 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5646 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1269 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2961 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.421 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3812 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_bh-43.json b/data/models/jaspionjader_bh-43.json deleted file mode 100644 index 249321ae6af93b9f21af3aeba54c64c8104f26ee..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_bh-43.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "bh-43", - "id": "jaspionjader/bh-43", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_bh-43/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.46 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5635 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1239 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2945 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4156 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.382 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_bh-44.json b/data/models/jaspionjader_bh-44.json deleted file mode 100644 index b04fb70320cf59895785bc5d9ebd83d144a55a6a..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_bh-44.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "bh-44", - "id": "jaspionjader/bh-44", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_bh-44/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4706 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5643 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1216 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2961 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4249 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3834 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_bh-46.json b/data/models/jaspionjader_bh-46.json deleted file mode 100644 index 3a1b00068ea9be886440b86a4055623071a05a23..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_bh-46.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "bh-46", - "id": "jaspionjader/bh-46", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_bh-46/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4727 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5632 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1276 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2961 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4262 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3822 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_bh-47.json b/data/models/jaspionjader_bh-47.json deleted file mode 100644 index 5eb9b4534953b6269d00e5a8dd24b5fbdceac3c7..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_bh-47.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "bh-47", - "id": "jaspionjader/bh-47", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_bh-47/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4652 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5546 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1276 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2945 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4156 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3855 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_bh-48.json b/data/models/jaspionjader_bh-48.json deleted file mode 100644 index 98fe5581317ec424b217cbb8a985e07abf7d0e19..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_bh-48.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "bh-48", - "id": "jaspionjader/bh-48", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_bh-48/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4688 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5541 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1254 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2945 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4209 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.386 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_bh-49.json b/data/models/jaspionjader_bh-49.json deleted file mode 100644 index c06c90f26ed5962ba704ce0b9ef241d3f14000b1..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_bh-49.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "bh-49", - "id": "jaspionjader/bh-49", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_bh-49/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4725 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.554 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1201 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2945 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4129 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3808 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_bh-5.json b/data/models/jaspionjader_bh-5.json deleted file mode 100644 index b971eab68f90ec4e048fbe568c816513bf4753a2..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_bh-5.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "bh-5", - "id": "jaspionjader/bh-5", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_bh-5/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4652 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5882 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1057 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2995 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4186 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3702 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_bh-50.json b/data/models/jaspionjader_bh-50.json deleted file mode 100644 index 45b763d6ca0d1497026e134160620e1501e661e9..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_bh-50.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "bh-50", - "id": "jaspionjader/bh-50", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_bh-50/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4725 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5553 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1208 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2936 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4169 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3842 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_bh-51.json b/data/models/jaspionjader_bh-51.json deleted file mode 100644 index 072b600e8309fb3eb3b26ab68be0006a908e66a9..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_bh-51.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "bh-51", - "id": "jaspionjader/bh-51", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_bh-51/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.463 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5557 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1239 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2928 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4168 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3831 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_bh-52.json b/data/models/jaspionjader_bh-52.json deleted file mode 100644 index d9c95462d50f09674c0aebfcf2406c87467ceb86..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_bh-52.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "bh-52", - "id": "jaspionjader/bh-52", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_bh-52/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4536 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5444 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1201 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2919 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4169 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3843 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_bh-53.json b/data/models/jaspionjader_bh-53.json deleted file mode 100644 index cc6ec0cb21865a4ffea87d556f1af4a9e532f815..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_bh-53.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "bh-53", - "id": "jaspionjader/bh-53", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_bh-53/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.478 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5494 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1269 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2987 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4196 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3858 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_bh-54.json b/data/models/jaspionjader_bh-54.json deleted file mode 100644 index ac4810e7839226a14dd4c9adc3d1e816695dc56f..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_bh-54.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "bh-54", - "id": "jaspionjader/bh-54", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_bh-54/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4841 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5548 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1292 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2945 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4155 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3825 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_bh-55.json b/data/models/jaspionjader_bh-55.json deleted file mode 100644 index 6133bff9376066a82e52d4248604b5590080d8a9..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_bh-55.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "bh-55", - "id": "jaspionjader/bh-55", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_bh-55/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4709 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.555 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1284 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3062 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4222 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3846 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_bh-56.json b/data/models/jaspionjader_bh-56.json deleted file mode 100644 index c6a0d3d88ba87aa95c80585a9084698abd94f633..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_bh-56.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "bh-56", - "id": "jaspionjader/bh-56", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_bh-56/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.46 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5447 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1231 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3003 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4116 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3844 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_bh-57.json b/data/models/jaspionjader_bh-57.json deleted file mode 100644 index aa66afa59e59f8f848ff919313fbddb42f7c011d..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_bh-57.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "bh-57", - "id": "jaspionjader/bh-57", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_bh-57/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4405 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5425 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1261 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3037 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.421 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3896 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_bh-58.json b/data/models/jaspionjader_bh-58.json deleted file mode 100644 index 8c4b0e1e3933778e0a30d01647726300a206bddf..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_bh-58.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "bh-58", - "id": "jaspionjader/bh-58", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_bh-58/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.463 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5446 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1322 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3062 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4183 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3896 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_bh-59.json b/data/models/jaspionjader_bh-59.json deleted file mode 100644 index e57d9b264001acc761e7496d5da1acf9fbbfbd72..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_bh-59.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "bh-59", - "id": "jaspionjader/bh-59", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_bh-59/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4341 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5512 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1541 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3154 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.417 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3838 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_bh-6.json b/data/models/jaspionjader_bh-6.json deleted file mode 100644 index 0f5217feab592e24dfe151b161dbf4799482cbf5..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_bh-6.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "bh-6", - "id": "jaspionjader/bh-6", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_bh-6/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4621 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5891 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1088 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2995 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4199 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3698 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_bh-60.json b/data/models/jaspionjader_bh-60.json deleted file mode 100644 index 819bbb61a10747d6edf79124b1aeaa63e04b2417..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_bh-60.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "bh-60", - "id": "jaspionjader/bh-60", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_bh-60/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4207 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5369 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1579 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3255 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4289 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3689 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_bh-61.json b/data/models/jaspionjader_bh-61.json deleted file mode 100644 index f41a640c49372b52be6c0137a0462c4fbbbed82f..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_bh-61.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "bh-61", - "id": "jaspionjader/bh-61", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_bh-61/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4247 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5271 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1707 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3188 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4356 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3679 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_bh-62.json b/data/models/jaspionjader_bh-62.json deleted file mode 100644 index 687c4312bb6b31c5f949da8588cdd76de7835fb1..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_bh-62.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "bh-62", - "id": "jaspionjader/bh-62", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_bh-62/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.415 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5379 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1624 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3205 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4289 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3719 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_bh-63.json b/data/models/jaspionjader_bh-63.json deleted file mode 100644 index f51328e655c360b5e259811466d6c16579427ba8..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_bh-63.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "bh-63", - "id": "jaspionjader/bh-63", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_bh-63/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4308 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4917 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.111 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3029 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4313 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3248 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_bh-64.json b/data/models/jaspionjader_bh-64.json deleted file mode 100644 index 6b508b1974483bc96ba82c503573aeaba3886912..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_bh-64.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "bh-64", - "id": "jaspionjader/bh-64", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_bh-64/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.414 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.536 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1548 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3213 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4355 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3693 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_bh-7.json b/data/models/jaspionjader_bh-7.json deleted file mode 100644 index b1e84ab154feeab341140b12e006b88ce4fbdc8c..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_bh-7.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "bh-7", - "id": "jaspionjader/bh-7", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_bh-7/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4624 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5861 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.114 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3037 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4119 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3715 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_bh-8.json b/data/models/jaspionjader_bh-8.json deleted file mode 100644 index cfccbc8014dc596ca7251535934c64a25f2f5c79..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_bh-8.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "bh-8", - "id": "jaspionjader/bh-8", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_bh-8/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4597 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.59 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1178 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3012 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4265 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.372 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_bh-9.json b/data/models/jaspionjader_bh-9.json deleted file mode 100644 index 94d2c492f993ac50dcc32f7d7b1b7c71492c6c67..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_bh-9.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "bh-9", - "id": "jaspionjader/bh-9", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_bh-9/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4509 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.585 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1156 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.302 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4146 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3703 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_dp-6-8b.json b/data/models/jaspionjader_dp-6-8b.json deleted file mode 100644 index d1ae4d2f997225ea64c5195fad5dd791be3eb7e0..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_dp-6-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "dp-6-8b", - "id": "jaspionjader/dp-6-8b", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_dp-6-8b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4806 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.53 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1329 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3079 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4434 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3897 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_dp-7-8b.json b/data/models/jaspionjader_dp-7-8b.json deleted file mode 100644 index 02296f1a2382ac9ff793665f38678f918913eaf7..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_dp-7-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "dp-7-8b", - "id": "jaspionjader/dp-7-8b", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_dp-7-8b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4498 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5291 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1261 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3062 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4407 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3934 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_ek-6.json b/data/models/jaspionjader_ek-6.json deleted file mode 100644 index 92a7f3bea27b8e55c2946fb4b6fbf6a26b11886f..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_ek-6.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ek-6", - "id": "jaspionjader/ek-6", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_ek-6/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4642 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5219 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1322 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3087 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4144 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3861 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_ek-7.json b/data/models/jaspionjader_ek-7.json deleted file mode 100644 index 0f339f45ad404731101640adf7a4b5b27e79108a..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_ek-7.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ek-7", - "id": "jaspionjader/ek-7", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_ek-7/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4767 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5194 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1329 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3163 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4171 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3887 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_f-1-8b.json b/data/models/jaspionjader_f-1-8b.json deleted file mode 100644 index 7cbf06a3172f60dc6c73d3700e09351b7cdd99f4..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_f-1-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "f-1-8b", - "id": "jaspionjader/f-1-8b", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_f-1-8b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4983 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5141 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1284 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3087 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4527 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3907 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_f-2-8b.json b/data/models/jaspionjader_f-2-8b.json deleted file mode 100644 index 33fc0cec526e28f27708beb7f5d621a79381c813..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_f-2-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "f-2-8b", - "id": "jaspionjader/f-2-8b", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_f-2-8b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4824 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5294 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1171 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2995 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4501 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3962 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_f-3-8b.json b/data/models/jaspionjader_f-3-8b.json deleted file mode 100644 index 15ac083e9ecdb56e01c0cae3fd62ed12498f3848..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_f-3-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "f-3-8b", - "id": "jaspionjader/f-3-8b", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_f-3-8b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4803 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5275 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1216 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3138 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4421 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3954 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_f-4-8b.json b/data/models/jaspionjader_f-4-8b.json deleted file mode 100644 index 982c12c8e2b4f006c11b431b01a8439b3cac3bb6..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_f-4-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "f-4-8b", - "id": "jaspionjader/f-4-8b", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_f-4-8b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4797 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5289 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1148 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3087 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4514 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3956 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_f-5-8b.json b/data/models/jaspionjader_f-5-8b.json deleted file mode 100644 index 319b891c65fce5e06e52d8fcc61adeafcddd6c2d..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_f-5-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "f-5-8b", - "id": "jaspionjader/f-5-8b", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_f-5-8b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5044 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5313 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1239 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3087 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4461 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3949 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_f-6-8b.json b/data/models/jaspionjader_f-6-8b.json deleted file mode 100644 index 25d50460d517056cb2b60cf51ccdb2bdde3a16d7..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_f-6-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "f-6-8b", - "id": "jaspionjader/f-6-8b", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_f-6-8b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4846 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5241 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1193 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3079 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4474 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3939 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_f-7-8b.json b/data/models/jaspionjader_f-7-8b.json deleted file mode 100644 index e37f2a210ba530187b21ab17d1d28eb6010a6ddd..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_f-7-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "f-7-8b", - "id": "jaspionjader/f-7-8b", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_f-7-8b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4462 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5277 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1239 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3129 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4315 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3936 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_f-8-8b.json b/data/models/jaspionjader_f-8-8b.json deleted file mode 100644 index b39110e61e4b6150d16e3c98daac195e845682cf..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_f-8-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "f-8-8b", - "id": "jaspionjader/f-8-8b", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_f-8-8b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4739 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5259 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1224 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3096 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4354 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.394 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_f-9-8b.json b/data/models/jaspionjader_f-9-8b.json deleted file mode 100644 index d4e833c65af28dc53419cb42d6b57e30e3d1530d..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_f-9-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "f-9-8b", - "id": "jaspionjader/f-9-8b", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_f-9-8b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4602 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5292 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1299 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3062 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4461 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3944 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_fct-14-8b.json b/data/models/jaspionjader_fct-14-8b.json deleted file mode 100644 index acb02d809eb0510de34899d76938ecd6211e01e4..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_fct-14-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "fct-14-8b", - "id": "jaspionjader/fct-14-8b", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_fct-14-8b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4129 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5206 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1201 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3163 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4186 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3875 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_fct-9-8b.json b/data/models/jaspionjader_fct-9-8b.json deleted file mode 100644 index b9281bfe070b4d1a2a83a1635522fa1af58e0297..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_fct-9-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "fct-9-8b", - "id": "jaspionjader/fct-9-8b", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_fct-9-8b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4354 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5205 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1193 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3079 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4291 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3932 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_fr-1-8b.json b/data/models/jaspionjader_fr-1-8b.json deleted file mode 100644 index 37b3441f41a1b4126b533b1257d0d803bfff2b36..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_fr-1-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "fr-1-8b", - "id": "jaspionjader/fr-1-8b", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_fr-1-8b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4211 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5142 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1118 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3054 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4277 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.361 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_fr-10-8b.json b/data/models/jaspionjader_fr-10-8b.json deleted file mode 100644 index 8f09c3be17fd69d3e59f4acaa736b4dddb00a0d4..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_fr-10-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "fr-10-8b", - "id": "jaspionjader/fr-10-8b", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_fr-10-8b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4402 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5207 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1224 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3171 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4119 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3863 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_fr-3-8b.json b/data/models/jaspionjader_fr-3-8b.json deleted file mode 100644 index 757a6ac61c72ac418e79685b02d81e3ac32d2c82..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_fr-3-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "fr-3-8b", - "id": "jaspionjader/fr-3-8b", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_fr-3-8b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4326 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5255 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1133 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3054 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4198 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3863 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_gamma-kosmos-evaa-8b.json b/data/models/jaspionjader_gamma-kosmos-evaa-8b.json deleted file mode 100644 index d4cc0c9bf1b42a54eff2519cf143f669ab82437e..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_gamma-kosmos-evaa-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "gamma-Kosmos-EVAA-8B", - "id": "jaspionjader/gamma-Kosmos-EVAA-8B", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_gamma-Kosmos-EVAA-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.425 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5253 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0899 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3138 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4412 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3776 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_gamma-kosmos-evaa-v2-8b.json b/data/models/jaspionjader_gamma-kosmos-evaa-v2-8b.json deleted file mode 100644 index 59b861f29915abe11970d34104cff5ec8da9fa87..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_gamma-kosmos-evaa-v2-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "gamma-Kosmos-EVAA-v2-8B", - "id": "jaspionjader/gamma-Kosmos-EVAA-v2-8B", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_gamma-Kosmos-EVAA-v2-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4233 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5262 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1057 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3205 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4344 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3756 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_gamma-kosmos-evaa-v3-8b.json b/data/models/jaspionjader_gamma-kosmos-evaa-v3-8b.json deleted file mode 100644 index 4f8e3bf6a5ea6cb599977446a4950929b95ef27b..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_gamma-kosmos-evaa-v3-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "gamma-Kosmos-EVAA-v3-8B", - "id": "jaspionjader/gamma-Kosmos-EVAA-v3-8B", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_gamma-Kosmos-EVAA-v3-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4333 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5278 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.111 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3129 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4263 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3898 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_knf-2-8b.json b/data/models/jaspionjader_knf-2-8b.json deleted file mode 100644 index 0c3d6903e21a61db7d17fe7be20d7ac0642fa4bf..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_knf-2-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "knf-2-8b", - "id": "jaspionjader/knf-2-8b", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_knf-2-8b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.425 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5207 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1201 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3104 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4185 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3875 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_knfp-2-8b.json b/data/models/jaspionjader_knfp-2-8b.json deleted file mode 100644 index 75e806d53246fc9569514dd0f9800ef1992c394b..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_knfp-2-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "knfp-2-8b", - "id": "jaspionjader/knfp-2-8b", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_knfp-2-8b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5327 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5305 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1427 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2928 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4185 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3726 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_knfp-3-8b.json b/data/models/jaspionjader_knfp-3-8b.json deleted file mode 100644 index 2aaa5785443e629ade42ebaa5b00350cece7ce44..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_knfp-3-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "knfp-3-8b", - "id": "jaspionjader/knfp-3-8b", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_knfp-3-8b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4946 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.52 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1224 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.307 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4171 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3881 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_kosmos-aurora_faustus-8b.json b/data/models/jaspionjader_kosmos-aurora_faustus-8b.json deleted file mode 100644 index c7db8cdd424ad1dba2de33c1c41a402e1f7dc0c3..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_kosmos-aurora_faustus-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Kosmos-Aurora_faustus-8B", - "id": "jaspionjader/Kosmos-Aurora_faustus-8B", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_Kosmos-Aurora_faustus-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4432 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.526 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1125 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2953 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4117 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3813 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_kosmos-elusive-8b.json b/data/models/jaspionjader_kosmos-elusive-8b.json deleted file mode 100644 index 9af038257873874e9c6e975856deb43bc360ce43..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_kosmos-elusive-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Kosmos-Elusive-8b", - "id": "jaspionjader/Kosmos-Elusive-8b", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_Kosmos-Elusive-8b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4169 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5339 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1261 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3079 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4078 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.376 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_kosmos-elusive-venn-8b.json b/data/models/jaspionjader_kosmos-elusive-venn-8b.json deleted file mode 100644 index f9ac3f718302502dd6d98f515183b30299ebe0d4..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_kosmos-elusive-venn-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Kosmos-Elusive-VENN-8B", - "id": "jaspionjader/Kosmos-Elusive-VENN-8B", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_Kosmos-Elusive-VENN-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4233 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5356 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1246 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2995 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4157 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3797 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_kosmos-elusive-venn-asymmetric-8b.json b/data/models/jaspionjader_kosmos-elusive-venn-asymmetric-8b.json deleted file mode 100644 index 0f7cc15061f1242a8a260dd181064319ac3a935c..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_kosmos-elusive-venn-asymmetric-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Kosmos-Elusive-VENN-Asymmetric-8B", - "id": "jaspionjader/Kosmos-Elusive-VENN-Asymmetric-8B", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_Kosmos-Elusive-VENN-Asymmetric-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4542 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5313 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1344 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2945 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4251 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3842 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_kosmos-elusive-venn-aurora_faustus-8b.json b/data/models/jaspionjader_kosmos-elusive-venn-aurora_faustus-8b.json deleted file mode 100644 index fe81eca8c700fd40a93273a8dcb08f19b8287de6..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_kosmos-elusive-venn-aurora_faustus-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Kosmos-Elusive-VENN-Aurora_faustus-8B", - "id": "jaspionjader/Kosmos-Elusive-VENN-Aurora_faustus-8B", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_Kosmos-Elusive-VENN-Aurora_faustus-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4335 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5304 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1125 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2953 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.417 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3795 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_kosmos-evaa-8b.json b/data/models/jaspionjader_kosmos-evaa-8b.json deleted file mode 100644 index 64ed75cb6188541779717caac81a42b1d112eef3..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_kosmos-evaa-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Kosmos-EVAA-8B", - "id": "jaspionjader/Kosmos-EVAA-8B", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_Kosmos-EVAA-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4405 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5312 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1178 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2995 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4237 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3818 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_kosmos-evaa-franken-immersive-v39-8b.json b/data/models/jaspionjader_kosmos-evaa-franken-immersive-v39-8b.json deleted file mode 100644 index b12948c6e18afdf456035a3ef22588d9c066764a..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_kosmos-evaa-franken-immersive-v39-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Kosmos-EVAA-Franken-Immersive-v39-8B", - "id": "jaspionjader/Kosmos-EVAA-Franken-Immersive-v39-8B", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_Kosmos-EVAA-Franken-Immersive-v39-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4378 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.519 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1292 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3154 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4236 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.39 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_kosmos-evaa-franken-v38-8b.json b/data/models/jaspionjader_kosmos-evaa-franken-v38-8b.json deleted file mode 100644 index b4482057e08aede021aa66ea6c4dd32105654d98..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_kosmos-evaa-franken-v38-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Kosmos-EVAA-Franken-v38-8B", - "id": "jaspionjader/Kosmos-EVAA-Franken-v38-8B", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_Kosmos-EVAA-Franken-v38-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4356 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.523 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1292 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3087 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4212 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.389 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_kosmos-evaa-fusion-8b.json b/data/models/jaspionjader_kosmos-evaa-fusion-8b.json deleted file mode 100644 index 1160e36435338576bea29bef2eb39f295798e22d..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_kosmos-evaa-fusion-8b.json +++ /dev/null @@ -1,275 +0,0 @@ -{ - "model_info": { - "name": "Kosmos-EVAA-Fusion-8B", - "id": "jaspionjader/Kosmos-EVAA-Fusion-8B", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_Kosmos-EVAA-Fusion-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4345 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5419 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1292 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3087 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4277 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3854 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "hfopenllm_v2/jaspionjader_Kosmos-EVAA-Fusion-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4418 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5406 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1352 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3062 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4277 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.386 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_kosmos-evaa-gamma-8b.json b/data/models/jaspionjader_kosmos-evaa-gamma-8b.json deleted file mode 100644 index a30812bc8f4d4c7a53b02aa5b990ce940e8a5163..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_kosmos-evaa-gamma-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Kosmos-EVAA-gamma-8B", - "id": "jaspionjader/Kosmos-EVAA-gamma-8B", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_Kosmos-EVAA-gamma-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4572 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5322 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.105 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3188 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4306 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3901 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_kosmos-evaa-gamma-alt-8b.json b/data/models/jaspionjader_kosmos-evaa-gamma-alt-8b.json deleted file mode 100644 index 822d902470a155e45331f0ceaf5272182c33d2f9..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_kosmos-evaa-gamma-alt-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Kosmos-EVAA-gamma-alt-8B", - "id": "jaspionjader/Kosmos-EVAA-gamma-alt-8B", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_Kosmos-EVAA-gamma-alt-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4542 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5298 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1095 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3247 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4292 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3896 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_kosmos-evaa-gamma-light-8b.json b/data/models/jaspionjader_kosmos-evaa-gamma-light-8b.json deleted file mode 100644 index 1b3c2d98a8fc611c27e9805f0d9a6e9e24356768..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_kosmos-evaa-gamma-light-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Kosmos-EVAA-gamma-light-8B", - "id": "jaspionjader/Kosmos-EVAA-gamma-light-8B", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_Kosmos-EVAA-gamma-light-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4581 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5376 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1103 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3163 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4291 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3943 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_kosmos-evaa-gamma-light-alt-8b.json b/data/models/jaspionjader_kosmos-evaa-gamma-light-alt-8b.json deleted file mode 100644 index b9c72f352a2e906370070f32f49c7fa73c7981f9..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_kosmos-evaa-gamma-light-alt-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Kosmos-EVAA-gamma-light-alt-8B", - "id": "jaspionjader/Kosmos-EVAA-gamma-light-alt-8B", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_Kosmos-EVAA-gamma-light-alt-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4454 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5327 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1133 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3138 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4305 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3923 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_kosmos-evaa-gamma-ultra-light-8b.json b/data/models/jaspionjader_kosmos-evaa-gamma-ultra-light-8b.json deleted file mode 100644 index c82d6f1b40528f74de0a2eb70485f0b90b722fbb..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_kosmos-evaa-gamma-ultra-light-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Kosmos-EVAA-gamma-ultra-light-8B", - "id": "jaspionjader/Kosmos-EVAA-gamma-ultra-light-8B", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_Kosmos-EVAA-gamma-ultra-light-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4563 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5316 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1178 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3163 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4197 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3915 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_kosmos-evaa-gamma-v13-8b.json b/data/models/jaspionjader_kosmos-evaa-gamma-v13-8b.json deleted file mode 100644 index f6e030ac6b82c5ac87d833b246ddd24bdc7984fb..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_kosmos-evaa-gamma-v13-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Kosmos-EVAA-gamma-v13-8B", - "id": "jaspionjader/Kosmos-EVAA-gamma-v13-8B", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_Kosmos-EVAA-gamma-v13-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4429 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5359 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1118 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3138 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4278 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.393 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_kosmos-evaa-gamma-v14-8b.json b/data/models/jaspionjader_kosmos-evaa-gamma-v14-8b.json deleted file mode 100644 index 38fc1df421bbd8a9e4f7c05fed49042fa406d3cd..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_kosmos-evaa-gamma-v14-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Kosmos-EVAA-gamma-v14-8B", - "id": "jaspionjader/Kosmos-EVAA-gamma-v14-8B", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_Kosmos-EVAA-gamma-v14-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.438 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5363 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1103 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3129 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4277 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3931 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_kosmos-evaa-gamma-v15-8b.json b/data/models/jaspionjader_kosmos-evaa-gamma-v15-8b.json deleted file mode 100644 index 3de8a401d66efe0b84621610ab30039ec8acb6e5..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_kosmos-evaa-gamma-v15-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Kosmos-EVAA-gamma-v15-8B", - "id": "jaspionjader/Kosmos-EVAA-gamma-v15-8B", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_Kosmos-EVAA-gamma-v15-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4654 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5343 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.111 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3112 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4277 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3941 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_kosmos-evaa-gamma-v16-8b.json b/data/models/jaspionjader_kosmos-evaa-gamma-v16-8b.json deleted file mode 100644 index ab0474b3c0d41dc541cce1992e58d19802b8a919..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_kosmos-evaa-gamma-v16-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Kosmos-EVAA-gamma-v16-8B", - "id": "jaspionjader/Kosmos-EVAA-gamma-v16-8B", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_Kosmos-EVAA-gamma-v16-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4557 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5344 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1171 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3154 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4264 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3917 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_kosmos-evaa-gamma-v17-8b.json b/data/models/jaspionjader_kosmos-evaa-gamma-v17-8b.json deleted file mode 100644 index b6fa5e0484636e8664c9b6eb29096b652ea4ac4a..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_kosmos-evaa-gamma-v17-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Kosmos-EVAA-gamma-v17-8B", - "id": "jaspionjader/Kosmos-EVAA-gamma-v17-8B", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_Kosmos-EVAA-gamma-v17-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4462 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5347 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.111 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3112 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4291 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3923 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_kosmos-evaa-gamma-v18-8b.json b/data/models/jaspionjader_kosmos-evaa-gamma-v18-8b.json deleted file mode 100644 index 161eb4c4fe76c5602920ebaab128aceece4341e2..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_kosmos-evaa-gamma-v18-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Kosmos-EVAA-gamma-v18-8B", - "id": "jaspionjader/Kosmos-EVAA-gamma-v18-8B", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_Kosmos-EVAA-gamma-v18-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4341 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5339 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.111 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3112 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4317 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3905 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_kosmos-evaa-immersive-sof-v44-8b.json b/data/models/jaspionjader_kosmos-evaa-immersive-sof-v44-8b.json deleted file mode 100644 index f2ba3f0f92b146c0f7c4386c3f35cb5822f8379a..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_kosmos-evaa-immersive-sof-v44-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Kosmos-EVAA-immersive-sof-v44-8B", - "id": "jaspionjader/Kosmos-EVAA-immersive-sof-v44-8B", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_Kosmos-EVAA-immersive-sof-v44-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4408 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5215 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1186 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3096 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4144 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3888 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_kosmos-evaa-prp-8b.json b/data/models/jaspionjader_kosmos-evaa-prp-8b.json deleted file mode 100644 index fa217e381ee0efdc32715d0e77926b6910f0b2f5..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_kosmos-evaa-prp-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Kosmos-EVAA-PRP-8B", - "id": "jaspionjader/Kosmos-EVAA-PRP-8B", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_Kosmos-EVAA-PRP-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3405 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5196 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0884 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3129 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4301 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3647 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_kosmos-evaa-prp-light-8b.json b/data/models/jaspionjader_kosmos-evaa-prp-light-8b.json deleted file mode 100644 index d57a6d3eb729099ecb1d1a3faaa176e3cfea7af5..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_kosmos-evaa-prp-light-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Kosmos-EVAA-PRP-light-8B", - "id": "jaspionjader/Kosmos-EVAA-PRP-light-8B", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_Kosmos-EVAA-PRP-light-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3824 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5271 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1103 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3121 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4249 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3782 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_kosmos-evaa-prp-v23-8b.json b/data/models/jaspionjader_kosmos-evaa-prp-v23-8b.json deleted file mode 100644 index 066b53a82b384d5d458d538dd3f144d7faf0aecb..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_kosmos-evaa-prp-v23-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Kosmos-EVAA-PRP-v23-8B", - "id": "jaspionjader/Kosmos-EVAA-PRP-v23-8B", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_Kosmos-EVAA-PRP-v23-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4041 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.529 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1156 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3087 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4368 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3706 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_kosmos-evaa-prp-v24-8b.json b/data/models/jaspionjader_kosmos-evaa-prp-v24-8b.json deleted file mode 100644 index 5a96a98635023eee5e75a5ba77a17377577530e5..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_kosmos-evaa-prp-v24-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Kosmos-EVAA-PRP-v24-8B", - "id": "jaspionjader/Kosmos-EVAA-PRP-v24-8B", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_Kosmos-EVAA-PRP-v24-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4259 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5276 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1103 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3104 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.429 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3779 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_kosmos-evaa-prp-v25-8b.json b/data/models/jaspionjader_kosmos-evaa-prp-v25-8b.json deleted file mode 100644 index 63e0d7ab5ba395eb496ee8c776261da32b78b961..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_kosmos-evaa-prp-v25-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Kosmos-EVAA-PRP-v25-8B", - "id": "jaspionjader/Kosmos-EVAA-PRP-v25-8B", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_Kosmos-EVAA-PRP-v25-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4421 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5291 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1186 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.318 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4303 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3716 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_kosmos-evaa-prp-v26-8b.json b/data/models/jaspionjader_kosmos-evaa-prp-v26-8b.json deleted file mode 100644 index b43e92c80142b7256102e20aff2733d22d9d07b9..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_kosmos-evaa-prp-v26-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Kosmos-EVAA-PRP-v26-8B", - "id": "jaspionjader/Kosmos-EVAA-PRP-v26-8B", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_Kosmos-EVAA-PRP-v26-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4414 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5271 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1133 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3045 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4264 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3793 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_kosmos-evaa-prp-v27-8b.json b/data/models/jaspionjader_kosmos-evaa-prp-v27-8b.json deleted file mode 100644 index 0345d8ce6d26cb77b7a9d5aa8cedf0933a6dc39e..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_kosmos-evaa-prp-v27-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Kosmos-EVAA-PRP-v27-8B", - "id": "jaspionjader/Kosmos-EVAA-PRP-v27-8B", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_Kosmos-EVAA-PRP-v27-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4378 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.529 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1193 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3087 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4343 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3755 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_kosmos-evaa-prp-v28-8b.json b/data/models/jaspionjader_kosmos-evaa-prp-v28-8b.json deleted file mode 100644 index 8d669255adc8e8d8044181ad4dd9f4ff73d1213b..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_kosmos-evaa-prp-v28-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Kosmos-EVAA-PRP-v28-8B", - "id": "jaspionjader/Kosmos-EVAA-PRP-v28-8B", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_Kosmos-EVAA-PRP-v28-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4366 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5295 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1171 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.307 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.433 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.375 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_kosmos-evaa-prp-v29-8b.json b/data/models/jaspionjader_kosmos-evaa-prp-v29-8b.json deleted file mode 100644 index cc99142d65aee405761b9ebdede0650383729b7f..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_kosmos-evaa-prp-v29-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Kosmos-EVAA-PRP-v29-8B", - "id": "jaspionjader/Kosmos-EVAA-PRP-v29-8B", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_Kosmos-EVAA-PRP-v29-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4487 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5275 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1201 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3104 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4237 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3765 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_kosmos-evaa-prp-v30-8b.json b/data/models/jaspionjader_kosmos-evaa-prp-v30-8b.json deleted file mode 100644 index 2207fe87d7864416b749b786372058b4328894ba..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_kosmos-evaa-prp-v30-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Kosmos-EVAA-PRP-v30-8B", - "id": "jaspionjader/Kosmos-EVAA-PRP-v30-8B", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_Kosmos-EVAA-PRP-v30-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4295 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5328 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1178 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3045 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4263 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3938 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_kosmos-evaa-prp-v31-8b.json b/data/models/jaspionjader_kosmos-evaa-prp-v31-8b.json deleted file mode 100644 index eccc3f32fb2eeeb6e7f7f50248810c47a697060a..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_kosmos-evaa-prp-v31-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Kosmos-EVAA-PRP-v31-8B", - "id": "jaspionjader/Kosmos-EVAA-PRP-v31-8B", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_Kosmos-EVAA-PRP-v31-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4399 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5315 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1133 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3138 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4251 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3935 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_kosmos-evaa-prp-v32-8b.json b/data/models/jaspionjader_kosmos-evaa-prp-v32-8b.json deleted file mode 100644 index 52bf09a1714ec9bfb6beb1d6fe6df5f5c39c799d..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_kosmos-evaa-prp-v32-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Kosmos-EVAA-PRP-v32-8B", - "id": "jaspionjader/Kosmos-EVAA-PRP-v32-8B", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_Kosmos-EVAA-PRP-v32-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4487 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5293 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1148 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3163 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4211 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3777 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_kosmos-evaa-prp-v33-8b.json b/data/models/jaspionjader_kosmos-evaa-prp-v33-8b.json deleted file mode 100644 index 63c8a6aab18a05718edd6ce383f4ae22dec349b7..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_kosmos-evaa-prp-v33-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Kosmos-EVAA-PRP-v33-8B", - "id": "jaspionjader/Kosmos-EVAA-PRP-v33-8B", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_Kosmos-EVAA-PRP-v33-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4302 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5321 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1178 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3129 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4184 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3909 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_kosmos-evaa-prp-v34-8b.json b/data/models/jaspionjader_kosmos-evaa-prp-v34-8b.json deleted file mode 100644 index 51f42c3196f60f89a661687149b7b8f4df2b45ea..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_kosmos-evaa-prp-v34-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Kosmos-EVAA-PRP-v34-8B", - "id": "jaspionjader/Kosmos-EVAA-PRP-v34-8B", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_Kosmos-EVAA-PRP-v34-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4563 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5333 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1125 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3112 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4237 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3927 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_kosmos-evaa-tsn-8b.json b/data/models/jaspionjader_kosmos-evaa-tsn-8b.json deleted file mode 100644 index 032d2b0f2c509f140ccc58828f427a4446648624..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_kosmos-evaa-tsn-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Kosmos-EVAA-TSN-8B", - "id": "jaspionjader/Kosmos-EVAA-TSN-8B", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_Kosmos-EVAA-TSN-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4721 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5177 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1344 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3029 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4329 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3816 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_kosmos-evaa-tsn-light-8b.json b/data/models/jaspionjader_kosmos-evaa-tsn-light-8b.json deleted file mode 100644 index b312631169ace7cf500837d19785d5261104aba0..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_kosmos-evaa-tsn-light-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Kosmos-EVAA-TSN-light-8B", - "id": "jaspionjader/Kosmos-EVAA-TSN-light-8B", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_Kosmos-EVAA-TSN-light-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4685 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5235 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1216 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3045 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4289 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3806 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_kosmos-evaa-tsn-v19-8b.json b/data/models/jaspionjader_kosmos-evaa-tsn-v19-8b.json deleted file mode 100644 index afbd9832d48e4c9e16e8d77bbceec8d34cefb79c..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_kosmos-evaa-tsn-v19-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Kosmos-EVAA-TSN-v19-8B", - "id": "jaspionjader/Kosmos-EVAA-TSN-v19-8B", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_Kosmos-EVAA-TSN-v19-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4564 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5316 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1156 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3054 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4277 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.379 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_kosmos-evaa-tsn-v20-8b.json b/data/models/jaspionjader_kosmos-evaa-tsn-v20-8b.json deleted file mode 100644 index 36c1d964bd1b9407566af2d8e9d56498917f7331..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_kosmos-evaa-tsn-v20-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Kosmos-EVAA-TSN-v20-8B", - "id": "jaspionjader/Kosmos-EVAA-TSN-v20-8B", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_Kosmos-EVAA-TSN-v20-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4423 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.525 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1246 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3138 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.421 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3936 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_kosmos-evaa-tsn-v21-8b.json b/data/models/jaspionjader_kosmos-evaa-tsn-v21-8b.json deleted file mode 100644 index 73501926efdf1f7907640bcead0bd743190f58f6..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_kosmos-evaa-tsn-v21-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Kosmos-EVAA-TSN-v21-8B", - "id": "jaspionjader/Kosmos-EVAA-TSN-v21-8B", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_Kosmos-EVAA-TSN-v21-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.467 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5248 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1193 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3121 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4343 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3816 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_kosmos-evaa-tsn-v22-8b.json b/data/models/jaspionjader_kosmos-evaa-tsn-v22-8b.json deleted file mode 100644 index 24f19048375ced3ffa6c0f94d8dcc4690fb70103..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_kosmos-evaa-tsn-v22-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Kosmos-EVAA-TSN-v22-8B", - "id": "jaspionjader/Kosmos-EVAA-TSN-v22-8B", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_Kosmos-EVAA-TSN-v22-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4673 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5246 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1133 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.307 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4303 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3812 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_kosmos-evaa-v10-8b.json b/data/models/jaspionjader_kosmos-evaa-v10-8b.json deleted file mode 100644 index 50d99eecf1a0d4cf246f1cec40fa46bc0805a5f4..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_kosmos-evaa-v10-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Kosmos-EVAA-v10-8B", - "id": "jaspionjader/Kosmos-EVAA-v10-8B", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_Kosmos-EVAA-v10-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4262 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5376 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1246 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2995 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4224 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3831 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_kosmos-evaa-v11-8b.json b/data/models/jaspionjader_kosmos-evaa-v11-8b.json deleted file mode 100644 index e6e33e266560a0b64ac102a717ca42db09248546..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_kosmos-evaa-v11-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Kosmos-EVAA-v11-8B", - "id": "jaspionjader/Kosmos-EVAA-v11-8B", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_Kosmos-EVAA-v11-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4426 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5359 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1322 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3154 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4184 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3836 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_kosmos-evaa-v12-8b.json b/data/models/jaspionjader_kosmos-evaa-v12-8b.json deleted file mode 100644 index e9af4ed8ad68fe7cf52d5b4f5f3d7da7ce1c85bf..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_kosmos-evaa-v12-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Kosmos-EVAA-v12-8B", - "id": "jaspionjader/Kosmos-EVAA-v12-8B", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_Kosmos-EVAA-v12-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4378 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5349 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1367 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3096 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4211 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3836 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_kosmos-evaa-v2-8b.json b/data/models/jaspionjader_kosmos-evaa-v2-8b.json deleted file mode 100644 index 3d8250b145889f4809bb126ff1226e5b43c3b133..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_kosmos-evaa-v2-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Kosmos-EVAA-v2-8B", - "id": "jaspionjader/Kosmos-EVAA-v2-8B", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_Kosmos-EVAA-v2-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4396 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5341 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1322 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2978 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4211 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3826 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_kosmos-evaa-v3-8b.json b/data/models/jaspionjader_kosmos-evaa-v3-8b.json deleted file mode 100644 index e51b37f24b90bf05508ce95087eef61f2179cbaa..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_kosmos-evaa-v3-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Kosmos-EVAA-v3-8B", - "id": "jaspionjader/Kosmos-EVAA-v3-8B", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_Kosmos-EVAA-v3-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4411 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5331 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1329 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3054 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4224 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3821 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_kosmos-evaa-v4-8b.json b/data/models/jaspionjader_kosmos-evaa-v4-8b.json deleted file mode 100644 index 6e23b70965efbbb437144c65d65701e199bc27f6..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_kosmos-evaa-v4-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Kosmos-EVAA-v4-8B", - "id": "jaspionjader/Kosmos-EVAA-v4-8B", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_Kosmos-EVAA-v4-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4289 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5337 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1254 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3003 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4197 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3817 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_kosmos-evaa-v5-8b.json b/data/models/jaspionjader_kosmos-evaa-v5-8b.json deleted file mode 100644 index 8670b342d4f4884aa26f960d3c957b5eb992ed40..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_kosmos-evaa-v5-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Kosmos-EVAA-v5-8B", - "id": "jaspionjader/Kosmos-EVAA-v5-8B", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_Kosmos-EVAA-v5-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.446 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5345 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1261 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3037 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4224 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3821 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_kosmos-evaa-v6-8b.json b/data/models/jaspionjader_kosmos-evaa-v6-8b.json deleted file mode 100644 index 7e84cadbca56e604d5ecfad1742953c1af8f7769..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_kosmos-evaa-v6-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Kosmos-EVAA-v6-8B", - "id": "jaspionjader/Kosmos-EVAA-v6-8B", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_Kosmos-EVAA-v6-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4396 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.538 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1292 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.302 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4184 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3821 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_kosmos-evaa-v7-8b.json b/data/models/jaspionjader_kosmos-evaa-v7-8b.json deleted file mode 100644 index e1c2a1cf4ca53de8accdec5bd6fa9b65b2287f25..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_kosmos-evaa-v7-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Kosmos-EVAA-v7-8B", - "id": "jaspionjader/Kosmos-EVAA-v7-8B", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_Kosmos-EVAA-v7-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4277 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5335 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1337 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3054 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4171 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3836 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_kosmos-evaa-v8-8b.json b/data/models/jaspionjader_kosmos-evaa-v8-8b.json deleted file mode 100644 index 0c71a18d6681bc3c17cafed76c32f5dbef7747c3..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_kosmos-evaa-v8-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Kosmos-EVAA-v8-8B", - "id": "jaspionjader/Kosmos-EVAA-v8-8B", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_Kosmos-EVAA-v8-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4383 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5359 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1307 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3037 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.421 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3827 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_kosmos-evaa-v9-8b.json b/data/models/jaspionjader_kosmos-evaa-v9-8b.json deleted file mode 100644 index 4f332bbc9b9f452cc1d7b1a0b50615f86b055a20..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_kosmos-evaa-v9-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Kosmos-EVAA-v9-8B", - "id": "jaspionjader/Kosmos-EVAA-v9-8B", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_Kosmos-EVAA-v9-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4369 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5361 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1276 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3062 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4184 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.382 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_kosmos-evaa-v9-titanfusion-mix-8b.json b/data/models/jaspionjader_kosmos-evaa-v9-titanfusion-mix-8b.json deleted file mode 100644 index 21500ebe3c3ed0e0918876459e5c727799c64091..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_kosmos-evaa-v9-titanfusion-mix-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Kosmos-EVAA-v9-TitanFusion-Mix-8B", - "id": "jaspionjader/Kosmos-EVAA-v9-TitanFusion-Mix-8B", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_Kosmos-EVAA-v9-TitanFusion-Mix-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4284 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.554 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1148 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2878 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4354 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3836 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_kosmos-venn-8b.json b/data/models/jaspionjader_kosmos-venn-8b.json deleted file mode 100644 index 4d362e4669b18297a78d5833a08e87c3aeef13b8..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_kosmos-venn-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Kosmos-VENN-8B", - "id": "jaspionjader/Kosmos-VENN-8B", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_Kosmos-VENN-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4332 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5318 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1412 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2928 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4211 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3801 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_kstc-1-8b.json b/data/models/jaspionjader_kstc-1-8b.json deleted file mode 100644 index d6b3e4ff412cce3e812289da1a22cf9e4c5d1f39..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_kstc-1-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "kstc-1-8b", - "id": "jaspionjader/kstc-1-8b", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_kstc-1-8b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4643 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5209 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1171 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3171 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4158 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3892 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_kstc-11-8b.json b/data/models/jaspionjader_kstc-11-8b.json deleted file mode 100644 index bcf3b95cd16a8d82cbfd4bb14884762d68309aaf..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_kstc-11-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "kstc-11-8b", - "id": "jaspionjader/kstc-11-8b", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_kstc-11-8b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4757 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5189 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1201 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3029 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4118 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3879 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_kstc-4-8b.json b/data/models/jaspionjader_kstc-4-8b.json deleted file mode 100644 index 7502bed1102bc8fdf82f3015287ead0360db09d6..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_kstc-4-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "kstc-4-8b", - "id": "jaspionjader/kstc-4-8b", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_kstc-4-8b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.477 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5216 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1239 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3037 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4118 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3869 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_kstc-5-8b.json b/data/models/jaspionjader_kstc-5-8b.json deleted file mode 100644 index 703bc8775f42189575160cd06f18e17e51518577..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_kstc-5-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "kstc-5-8b", - "id": "jaspionjader/kstc-5-8b", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_kstc-5-8b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4721 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5211 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1299 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3154 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4224 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3892 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_kstc-6-8b.json b/data/models/jaspionjader_kstc-6-8b.json deleted file mode 100644 index 95ad73ef474943ef496ad47a636db4d22cd1d52a..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_kstc-6-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "kstc-6-8b", - "id": "jaspionjader/kstc-6-8b", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_kstc-6-8b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4944 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5231 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1246 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2995 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4105 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3857 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_kstc-8-8b.json b/data/models/jaspionjader_kstc-8-8b.json deleted file mode 100644 index 012e884f3da49af6d3f70fec4b6429e2d8fdee49..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_kstc-8-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "kstc-8-8b", - "id": "jaspionjader/kstc-8-8b", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_kstc-8-8b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.491 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5239 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1307 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3054 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4211 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3889 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_kstc-9-8b.json b/data/models/jaspionjader_kstc-9-8b.json deleted file mode 100644 index 0e9fbbfce8deaf0a3e3bec1edce35a9d94d8b47b..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_kstc-9-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "kstc-9-8b", - "id": "jaspionjader/kstc-9-8b", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_kstc-9-8b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4861 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5238 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.136 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3012 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4118 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3872 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_prp-kosmos-evaa-8b.json b/data/models/jaspionjader_prp-kosmos-evaa-8b.json deleted file mode 100644 index 1a9d307f29d1c7ec187f24b9bfe9af1187ec2b74..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_prp-kosmos-evaa-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "PRP-Kosmos-EVAA-8B", - "id": "jaspionjader/PRP-Kosmos-EVAA-8B", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_PRP-Kosmos-EVAA-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3633 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5237 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0959 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3096 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.425 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3766 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_prp-kosmos-evaa-light-8b.json b/data/models/jaspionjader_prp-kosmos-evaa-light-8b.json deleted file mode 100644 index 90cdb1c861c4cc659d71797461be5b1f1f0dd37e..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_prp-kosmos-evaa-light-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "PRP-Kosmos-EVAA-light-8B", - "id": "jaspionjader/PRP-Kosmos-EVAA-light-8B", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_PRP-Kosmos-EVAA-light-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4321 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5275 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1103 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3221 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4235 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3631 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_slu-10.json b/data/models/jaspionjader_slu-10.json deleted file mode 100644 index 929c0a4b24ef6a70036b1a913ed5937acba0b3b7..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_slu-10.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "slu-10", - "id": "jaspionjader/slu-10", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_slu-10/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.436 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5096 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0974 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3138 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.392 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3664 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_slu-11.json b/data/models/jaspionjader_slu-11.json deleted file mode 100644 index 2b771c781cdd8b6181e667abdcfcdd03ed7efc92..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_slu-11.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "slu-11", - "id": "jaspionjader/slu-11", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_slu-11/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3725 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.489 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0559 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3037 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3919 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3382 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_slu-13.json b/data/models/jaspionjader_slu-13.json deleted file mode 100644 index 3b135e9afcceb1db8d14152af745e7fd3b268523..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_slu-13.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "slu-13", - "id": "jaspionjader/slu-13", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_slu-13/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4378 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5097 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0808 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3079 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3814 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.358 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_slu-14.json b/data/models/jaspionjader_slu-14.json deleted file mode 100644 index 064804380f77894091b3854e94c22b606223a73a..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_slu-14.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "slu-14", - "id": "jaspionjader/slu-14", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_slu-14/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4107 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5089 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0974 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3079 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.396 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3627 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_slu-17.json b/data/models/jaspionjader_slu-17.json deleted file mode 100644 index 086fb30d37a47244567d8da26d4ee6fa72822cc3..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_slu-17.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "slu-17", - "id": "jaspionjader/slu-17", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_slu-17/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4217 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5071 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0853 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3087 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3761 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3619 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_slu-2.json b/data/models/jaspionjader_slu-2.json deleted file mode 100644 index 5a3bf14cb225677f5849dfe1f215c8c58fc5fc9f..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_slu-2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "slu-2", - "id": "jaspionjader/slu-2", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_slu-2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4016 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5008 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0634 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2987 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3959 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3506 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_slu-20.json b/data/models/jaspionjader_slu-20.json deleted file mode 100644 index c7c7113c04b67d2eadf646df1603808f970e94f5..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_slu-20.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "slu-20", - "id": "jaspionjader/slu-20", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_slu-20/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4393 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5061 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0869 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3087 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3933 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3665 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_slu-22.json b/data/models/jaspionjader_slu-22.json deleted file mode 100644 index c67fe2147380d3a9efb2c8622b31f7a914b414cc..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_slu-22.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "slu-22", - "id": "jaspionjader/slu-22", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_slu-22/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4321 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5082 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0793 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3163 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3893 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.365 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_slu-23.json b/data/models/jaspionjader_slu-23.json deleted file mode 100644 index 11a515c3a0362d561d3c4697b99f2968bf45e08f..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_slu-23.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "slu-23", - "id": "jaspionjader/slu-23", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_slu-23/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4478 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5132 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0944 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3045 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4092 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3725 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_slu-25.json b/data/models/jaspionjader_slu-25.json deleted file mode 100644 index 7f475ac847bb8aca1d28b4cd37da696592a31868..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_slu-25.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "slu-25", - "id": "jaspionjader/slu-25", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_slu-25/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.45 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5095 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0838 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3087 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3946 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3684 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_slu-29.json b/data/models/jaspionjader_slu-29.json deleted file mode 100644 index 5b773d530767ee697583b113e095753669d9daaf..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_slu-29.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "slu-29", - "id": "jaspionjader/slu-29", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_slu-29/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4431 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5096 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0869 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.307 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3933 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3669 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_slu-32.json b/data/models/jaspionjader_slu-32.json deleted file mode 100644 index 2cd666de7d6c0412bb554ab130b9af5f09b685da..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_slu-32.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "slu-32", - "id": "jaspionjader/slu-32", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_slu-32/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4516 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5167 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1073 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.302 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4039 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3766 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_slu-33.json b/data/models/jaspionjader_slu-33.json deleted file mode 100644 index c175d33212cbc963002155821959c64029691ae9..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_slu-33.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "slu-33", - "id": "jaspionjader/slu-33", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_slu-33/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4457 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5081 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0997 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3121 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3867 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3679 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_slu-34.json b/data/models/jaspionjader_slu-34.json deleted file mode 100644 index 8162b9daae36b0c8461f7dc442999b1a8d61a598..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_slu-34.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "slu-34", - "id": "jaspionjader/slu-34", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_slu-34/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4351 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5077 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0997 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3079 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.388 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.372 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_slu-35.json b/data/models/jaspionjader_slu-35.json deleted file mode 100644 index c517813ce8f9975d9daa831872a73d968e12d428..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_slu-35.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "slu-35", - "id": "jaspionjader/slu-35", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_slu-35/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4242 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5103 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1012 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3121 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3946 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3676 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_slu-36.json b/data/models/jaspionjader_slu-36.json deleted file mode 100644 index ddc62a39b95cff0e62d12747ce568e714b8291bb..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_slu-36.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "slu-36", - "id": "jaspionjader/slu-36", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_slu-36/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4518 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5087 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0906 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3121 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3933 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3711 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_slu-37.json b/data/models/jaspionjader_slu-37.json deleted file mode 100644 index 267a109e15c0d69bdfd010e408ccce929fb78629..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_slu-37.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "slu-37", - "id": "jaspionjader/slu-37", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_slu-37/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4534 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.51 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0974 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.307 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3946 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3695 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_slu-6.json b/data/models/jaspionjader_slu-6.json deleted file mode 100644 index 328f7314bc883f553bcd39810665b6606789a4b3..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_slu-6.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "slu-6", - "id": "jaspionjader/slu-6", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_slu-6/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4117 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5099 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0944 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3029 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4066 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3611 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_slu-mix-1.json b/data/models/jaspionjader_slu-mix-1.json deleted file mode 100644 index 7e670eae1108412a3d505873575e4e5e0398c4f7..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_slu-mix-1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "slu-mix-1", - "id": "jaspionjader/slu-mix-1", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_slu-mix-1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4569 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.524 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1118 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3003 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4277 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.393 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_sof-1.json b/data/models/jaspionjader_sof-1.json deleted file mode 100644 index b6b4200c5b73f838f9b7adf87cb59e720e6a3f31..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_sof-1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "sof-1", - "id": "jaspionjader/sof-1", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_sof-1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4314 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.501 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.114 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3029 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4082 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3674 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_sof-10.json b/data/models/jaspionjader_sof-10.json deleted file mode 100644 index d2975d11edbf1a7e6ac16dcce240f5fbb80be817..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_sof-10.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "sof-10", - "id": "jaspionjader/sof-10", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_sof-10/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4648 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5197 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1239 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3062 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4091 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3874 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_sof-3.json b/data/models/jaspionjader_sof-3.json deleted file mode 100644 index 523161d5336368024c775c6d164d000927f13373..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_sof-3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "sof-3", - "id": "jaspionjader/sof-3", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_sof-3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4637 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5206 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1276 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.297 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4131 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3812 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_sof-6.json b/data/models/jaspionjader_sof-6.json deleted file mode 100644 index 929b5f5240d5ffd7d35a55a625ab33572f4db777..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_sof-6.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "sof-6", - "id": "jaspionjader/sof-6", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_sof-6/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4354 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5209 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1299 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2995 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4171 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3844 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_test-10.json b/data/models/jaspionjader_test-10.json deleted file mode 100644 index 67fe3410e76dfa742f6e8ae6d4ce7b149e23243d..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_test-10.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "test-10", - "id": "jaspionjader/test-10", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "4.015" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_test-10/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4578 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5316 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.114 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3196 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4251 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3936 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_test-11.json b/data/models/jaspionjader_test-11.json deleted file mode 100644 index 00992e0c6ddc3b635402216bd0391d6ad02afbe3..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_test-11.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "test-11", - "id": "jaspionjader/test-11", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "4.015" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_test-11/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4541 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.535 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1201 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3238 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.429 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3939 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_test-12.json b/data/models/jaspionjader_test-12.json deleted file mode 100644 index b3d245662d830a5fd1c55511a53679010fb1780b..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_test-12.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "test-12", - "id": "jaspionjader/test-12", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "4.015" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_test-12/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4368 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5347 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.108 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3188 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.425 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3935 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_test-13.json b/data/models/jaspionjader_test-13.json deleted file mode 100644 index d4498509c9f13333917600e4ebdc8c5983219617..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_test-13.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "test-13", - "id": "jaspionjader/test-13", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "4.015" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_test-13/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4581 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5318 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1057 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3163 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4264 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3935 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_test-14.json b/data/models/jaspionjader_test-14.json deleted file mode 100644 index 3cb8b1c57e2d51717ca9970cff93f56872abc919..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_test-14.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "test-14", - "id": "jaspionjader/test-14", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "4.015" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_test-14/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4444 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5323 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1103 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3146 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4317 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.393 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_test-15.json b/data/models/jaspionjader_test-15.json deleted file mode 100644 index 78133e0772bb49249bbfa51425a4bdddc61aab1b..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_test-15.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "test-15", - "id": "jaspionjader/test-15", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "4.015" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_test-15/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4365 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5328 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1118 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3121 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4264 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.393 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_test-16.json b/data/models/jaspionjader_test-16.json deleted file mode 100644 index 174cb5fc5307b3d31690ce4898051822393fcfbf..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_test-16.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "test-16", - "id": "jaspionjader/test-16", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "4.015" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_test-16/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4599 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.533 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1095 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3138 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4225 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.393 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_test-17.json b/data/models/jaspionjader_test-17.json deleted file mode 100644 index 3380558de86893629c53e134344758547afd369b..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_test-17.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "test-17", - "id": "jaspionjader/test-17", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "4.015" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_test-17/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4267 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5329 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1103 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3129 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.429 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3929 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_test-18.json b/data/models/jaspionjader_test-18.json deleted file mode 100644 index 75b623de9108a0423a147452f42c26ba7edfc808..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_test-18.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "test-18", - "id": "jaspionjader/test-18", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_test-18/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4392 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5317 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1148 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3121 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4251 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.393 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_test-19.json b/data/models/jaspionjader_test-19.json deleted file mode 100644 index 141f1be95753623d0b72664d7a962ccdea9cf2d8..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_test-19.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "test-19", - "id": "jaspionjader/test-19", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_test-19/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4401 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5319 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1095 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3096 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4264 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3929 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_test-20.json b/data/models/jaspionjader_test-20.json deleted file mode 100644 index 5ccdd5219c724a8672230d8e554986a11557be08..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_test-20.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "test-20", - "id": "jaspionjader/test-20", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_test-20/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4529 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5327 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1118 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3138 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4251 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.392 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_tsn-kosmos-evaa-8b.json b/data/models/jaspionjader_tsn-kosmos-evaa-8b.json deleted file mode 100644 index eba385353152573717b5166c89056ab4f886eb8c..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_tsn-kosmos-evaa-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "TSN-Kosmos-EVAA-8B", - "id": "jaspionjader/TSN-Kosmos-EVAA-8B", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_TSN-Kosmos-EVAA-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4903 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5347 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.145 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3205 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4173 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3831 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jaspionjader_tsn-kosmos-evaa-v2-8b.json b/data/models/jaspionjader_tsn-kosmos-evaa-v2-8b.json deleted file mode 100644 index 3ceaa13ca55d83cbc6d9aea28d6ee5ee8571dee4..0000000000000000000000000000000000000000 --- a/data/models/jaspionjader_tsn-kosmos-evaa-v2-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "TSN-Kosmos-EVAA-v2-8B", - "id": "jaspionjader/TSN-Kosmos-EVAA-v2-8B", - "developer": "jaspionjader", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jaspionjader_TSN-Kosmos-EVAA-v2-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4667 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5343 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.108 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3221 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4186 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3762 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayasuryajsk_qwen2.5-3b-reasoner.json b/data/models/jayasuryajsk_qwen2.5-3b-reasoner.json deleted file mode 100644 index 587768c53c774e9de136bb12f463013155dee429..0000000000000000000000000000000000000000 --- a/data/models/jayasuryajsk_qwen2.5-3b-reasoner.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-3B-reasoner", - "id": "jayasuryajsk/Qwen2.5-3B-reasoner", - "developer": "jayasuryajsk", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.086" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jayasuryajsk_Qwen2.5-3B-reasoner/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.416 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4651 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2085 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.302 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4123 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3482 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen-0.5b-dpo-1epoch.json b/data/models/jayhyeon_qwen-0.5b-dpo-1epoch.json deleted file mode 100644 index e7a99547793049438ed14ba31fbfb80ff2764be2..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen-0.5b-dpo-1epoch.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen-0.5B-DPO-1epoch", - "id": "JayHyeon/Qwen-0.5B-DPO-1epoch", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2Model", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen-0.5B-DPO-1epoch/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2647 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3191 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0287 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2525 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3352 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1558 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen-0.5b-dpo-5epoch.json b/data/models/jayhyeon_qwen-0.5b-dpo-5epoch.json deleted file mode 100644 index c2daf684487c6c417ef0a82329b664fcf2145d01..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen-0.5b-dpo-5epoch.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen-0.5B-DPO-5epoch", - "id": "JayHyeon/Qwen-0.5B-DPO-5epoch", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2Model", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen-0.5B-DPO-5epoch/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.257 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3112 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.04 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2433 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.338 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1533 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen-0.5b-edpo-1epoch.json b/data/models/jayhyeon_qwen-0.5b-edpo-1epoch.json deleted file mode 100644 index 5b9682eced9375afdcb1e7e3f7a50512764042de..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen-0.5b-edpo-1epoch.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen-0.5B-eDPO-1epoch", - "id": "JayHyeon/Qwen-0.5B-eDPO-1epoch", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2Model", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen-0.5B-eDPO-1epoch/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2623 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3181 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0347 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2424 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3327 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1553 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen-0.5b-edpo-5epoch.json b/data/models/jayhyeon_qwen-0.5b-edpo-5epoch.json deleted file mode 100644 index 7d3f71d2ac6be57b31c6716faa9ed2ec4bd4881b..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen-0.5b-edpo-5epoch.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen-0.5B-eDPO-5epoch", - "id": "JayHyeon/Qwen-0.5B-eDPO-5epoch", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2Model", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen-0.5B-eDPO-5epoch/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2477 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3096 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0234 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2492 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3326 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1523 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen-0.5b-irpo-1epoch.json b/data/models/jayhyeon_qwen-0.5b-irpo-1epoch.json deleted file mode 100644 index 67270a616ee852da559aa41266fd740b897f9a27..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen-0.5b-irpo-1epoch.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen-0.5B-IRPO-1epoch", - "id": "JayHyeon/Qwen-0.5B-IRPO-1epoch", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2Model", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen-0.5B-IRPO-1epoch/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2589 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3164 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0317 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2466 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3286 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.15 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen-0.5b-irpo-5epoch.json b/data/models/jayhyeon_qwen-0.5b-irpo-5epoch.json deleted file mode 100644 index aa02d3891651df9a8abacd15a67ad6b94bac4531..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen-0.5b-irpo-5epoch.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen-0.5B-IRPO-5epoch", - "id": "JayHyeon/Qwen-0.5B-IRPO-5epoch", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2Model", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen-0.5B-IRPO-5epoch/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2487 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3189 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0325 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2399 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3287 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1507 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-instruct-sft-dpo-1epoch_v1.json b/data/models/jayhyeon_qwen2.5-0.5b-instruct-sft-dpo-1epoch_v1.json deleted file mode 100644 index 3b6bc926790187a96f12298c4c5f3ec8c49d0281..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-instruct-sft-dpo-1epoch_v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-Instruct-SFT-DPO-1epoch_v1", - "id": "JayHyeon/Qwen2.5-0.5B-Instruct-SFT-DPO-1epoch_v1", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2Model", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-Instruct-SFT-DPO-1epoch_v1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2469 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.326 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.065 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2727 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3434 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1575 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-instruct-sft-irpo-1epoch_v1.json b/data/models/jayhyeon_qwen2.5-0.5b-instruct-sft-irpo-1epoch_v1.json deleted file mode 100644 index e06b374fa494a245d87b95d494baabd048d28159..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-instruct-sft-irpo-1epoch_v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-Instruct-SFT-IRPO-1epoch_v1", - "id": "JayHyeon/Qwen2.5-0.5B-Instruct-SFT-IRPO-1epoch_v1", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2Model", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-Instruct-SFT-IRPO-1epoch_v1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2606 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3308 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0498 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2802 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3288 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1626 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-instruct-sft-mdpo-1epoch_v1.json b/data/models/jayhyeon_qwen2.5-0.5b-instruct-sft-mdpo-1epoch_v1.json deleted file mode 100644 index 1ebafcd568edfcfc9d358502923afff6cdaf3379..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-instruct-sft-mdpo-1epoch_v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-Instruct-SFT-MDPO-1epoch_v1", - "id": "JayHyeon/Qwen2.5-0.5B-Instruct-SFT-MDPO-1epoch_v1", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2Model", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-Instruct-SFT-MDPO-1epoch_v1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2529 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3262 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0566 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2685 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3301 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1576 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-instruct-sft.json b/data/models/jayhyeon_qwen2.5-0.5b-instruct-sft.json deleted file mode 100644 index 877fdc1a436f8ef7cc05ce53e33900870358a813..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-instruct-sft.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-Instruct-SFT", - "id": "JayHyeon/Qwen2.5-0.5B-Instruct-SFT", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-Instruct-SFT/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2768 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3254 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0393 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2827 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3342 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.152 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-sft-1e-4-2ep.json b/data/models/jayhyeon_qwen2.5-0.5b-sft-1e-4-2ep.json deleted file mode 100644 index d35fcb530ab715be29e45c0c6a82d8e3de597c60..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-sft-1e-4-2ep.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-SFT-1e-4-2ep", - "id": "JayHyeon/Qwen2.5-0.5B-SFT-1e-4-2ep", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-SFT-1e-4-2ep/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.214 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3172 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0264 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2466 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3473 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1537 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-sft-1e-4-3ep.json b/data/models/jayhyeon_qwen2.5-0.5b-sft-1e-4-3ep.json deleted file mode 100644 index 3d80f694a766fa662c42d9b85c0d90785403fa6e..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-sft-1e-4-3ep.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-SFT-1e-4-3ep", - "id": "JayHyeon/Qwen2.5-0.5B-SFT-1e-4-3ep", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-SFT-1e-4-3ep/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2257 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3064 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0264 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2483 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3661 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1532 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-sft-1e-4-5ep.json b/data/models/jayhyeon_qwen2.5-0.5b-sft-1e-4-5ep.json deleted file mode 100644 index 2b3e215244937ed19f407ec306b007bf0e8097cf..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-sft-1e-4-5ep.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-SFT-1e-4-5ep", - "id": "JayHyeon/Qwen2.5-0.5B-SFT-1e-4-5ep", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-SFT-1e-4-5ep/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1987 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3104 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0196 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2534 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3407 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1558 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-sft-1e-4.json b/data/models/jayhyeon_qwen2.5-0.5b-sft-1e-4.json deleted file mode 100644 index f7a1962acdf55042dde4e575572d63dab284e358..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-sft-1e-4.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-SFT-1e-4", - "id": "JayHyeon/Qwen2.5-0.5B-SFT-1e-4", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-SFT-1e-4/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.202 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3017 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0189 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2508 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3446 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1619 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-sft-1e-5-2ep.json b/data/models/jayhyeon_qwen2.5-0.5b-sft-1e-5-2ep.json deleted file mode 100644 index 0718cec6347b9436185fa0dd094969f785fb7971..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-sft-1e-5-2ep.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-SFT-1e-5-2ep", - "id": "JayHyeon/Qwen2.5-0.5B-SFT-1e-5-2ep", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-SFT-1e-5-2ep/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1971 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3225 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0529 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2693 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3368 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1651 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-sft-1e-5-3ep.json b/data/models/jayhyeon_qwen2.5-0.5b-sft-1e-5-3ep.json deleted file mode 100644 index 79745bf893f75b3104a2435004899b01ba7f0f38..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-sft-1e-5-3ep.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-SFT-1e-5-3ep", - "id": "JayHyeon/Qwen2.5-0.5B-SFT-1e-5-3ep", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-SFT-1e-5-3ep/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2241 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3247 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0536 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2701 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3353 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1689 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-sft-1e-5-5ep.json b/data/models/jayhyeon_qwen2.5-0.5b-sft-1e-5-5ep.json deleted file mode 100644 index e77bd7ed9431dba2e4262077d5cc90ee24a391d3..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-sft-1e-5-5ep.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-SFT-1e-5-5ep", - "id": "JayHyeon/Qwen2.5-0.5B-SFT-1e-5-5ep", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-SFT-1e-5-5ep/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2292 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3259 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0521 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2794 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3235 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1688 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-sft-1e-5.json b/data/models/jayhyeon_qwen2.5-0.5b-sft-1e-5.json deleted file mode 100644 index 8045f3cf1c46f0a02c28536247949f92b432e43b..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-sft-1e-5.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-SFT-1e-5", - "id": "JayHyeon/Qwen2.5-0.5B-SFT-1e-5", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-SFT-1e-5/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1986 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.314 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0378 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2685 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.346 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1698 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-4-2ep.json b/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-4-2ep.json deleted file mode 100644 index 2b0029dc4d293cf42bcb60cce087a089ef7429df..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-4-2ep.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-SFT-2e-4-2ep", - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-4-2ep", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-SFT-2e-4-2ep/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1831 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2984 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0249 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2424 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3568 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1484 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-4-3ep.json b/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-4-3ep.json deleted file mode 100644 index 07858fd85d0a139a3ef861ba6f61c6356dc87abd..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-4-3ep.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-SFT-2e-4-3ep", - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-4-3ep", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-SFT-2e-4-3ep/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.199 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.311 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0151 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2609 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3449 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1416 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-4-5ep.json b/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-4-5ep.json deleted file mode 100644 index 079ce2536d2186423d8aabf1319c8091e3aca9dd..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-4-5ep.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-SFT-2e-4-5ep", - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-4-5ep", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-SFT-2e-4-5ep/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1897 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2936 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0181 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2693 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3874 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1336 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-4.json b/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-4.json deleted file mode 100644 index 12b5f922e0a787f4c6044f4e768c18233a9f1594..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-4.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-SFT-2e-4", - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-4", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-SFT-2e-4/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2034 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2936 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0242 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2576 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3434 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1413 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-dpo_1e-6-1ep_0alp_0lam.json b/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-dpo_1e-6-1ep_0alp_0lam.json deleted file mode 100644 index 7cc75175577da13f83bc2fadf6f0c306fdee263c..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-dpo_1e-6-1ep_0alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_1e-6-1ep_0alp_0lam", - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_1e-6-1ep_0alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2Model", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_1e-6-1ep_0alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2542 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3167 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0408 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2718 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3289 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.158 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-dpo_1e-6-2ep_0alp_0lam.json b/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-dpo_1e-6-2ep_0alp_0lam.json deleted file mode 100644 index fa5363904e30c8ab9f93e6bf4ab55dc57474a45e..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-dpo_1e-6-2ep_0alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_1e-6-2ep_0alp_0lam", - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_1e-6-2ep_0alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2Model", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_1e-6-2ep_0alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2451 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.316 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0408 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2743 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3302 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1561 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-dpo_1e-6-3ep_0alp_0lam.json b/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-dpo_1e-6-3ep_0alp_0lam.json deleted file mode 100644 index cc75a98a4f1552cb308383d3443cbfc422b5a38c..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-dpo_1e-6-3ep_0alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_1e-6-3ep_0alp_0lam", - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_1e-6-3ep_0alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_1e-6-3ep_0alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2557 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3142 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.04 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2743 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3315 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1575 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-dpo_1e-7-2ep_0alp_0lam.json b/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-dpo_1e-7-2ep_0alp_0lam.json deleted file mode 100644 index 26792a42010a5fbfe278a3c35cb4128808e6d9cc..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-dpo_1e-7-2ep_0alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_1e-7-2ep_0alp_0lam", - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_1e-7-2ep_0alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2Model", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_1e-7-2ep_0alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2605 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3167 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0363 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2701 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3341 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1577 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-dpo_1e-7-3ep_0alp_0lam.json b/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-dpo_1e-7-3ep_0alp_0lam.json deleted file mode 100644 index abe4b20d661ebaa164d3e28a43fd4852180c65de..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-dpo_1e-7-3ep_0alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_1e-7-3ep_0alp_0lam", - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_1e-7-3ep_0alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_1e-7-3ep_0alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2578 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3173 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0355 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2634 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3288 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1583 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-dpo_2e-6-1ep_0alp_0lam.json b/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-dpo_2e-6-1ep_0alp_0lam.json deleted file mode 100644 index facc36fcd92e0b51934c761738c0a5fdb55f42a0..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-dpo_2e-6-1ep_0alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_2e-6-1ep_0alp_0lam", - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_2e-6-1ep_0alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2Model", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_2e-6-1ep_0alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2335 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3198 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0385 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2752 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3276 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1581 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-dpo_2e-6-2ep_0alp_0lam.json b/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-dpo_2e-6-2ep_0alp_0lam.json deleted file mode 100644 index 0d2c86e3fe3668aab1820fb7b3442abb693c9c12..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-dpo_2e-6-2ep_0alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_2e-6-2ep_0alp_0lam", - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_2e-6-2ep_0alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2Model", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_2e-6-2ep_0alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2472 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3226 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0506 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.276 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3262 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1538 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-dpo_2e-6-3ep_0alp_0lam.json b/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-dpo_2e-6-3ep_0alp_0lam.json deleted file mode 100644 index 5bac45545ffb3df261a89c9d051ad80987a4382e..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-dpo_2e-6-3ep_0alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_2e-6-3ep_0alp_0lam", - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_2e-6-3ep_0alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_2e-6-3ep_0alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2474 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3229 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0415 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2727 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3275 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1539 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-dpo_3e-6-1ep_0alp_0lam.json b/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-dpo_3e-6-1ep_0alp_0lam.json deleted file mode 100644 index 550f044131c9a8113fe72c3cf11c3673eff4228f..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-dpo_3e-6-1ep_0alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_3e-6-1ep_0alp_0lam", - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_3e-6-1ep_0alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2Model", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_3e-6-1ep_0alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2403 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3245 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0431 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2819 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3262 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1573 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-dpo_3e-6-2ep_0alp_0lam.json b/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-dpo_3e-6-2ep_0alp_0lam.json deleted file mode 100644 index 1c2318cc95a1c0ba0f2896c3ac69f469582e0512..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-dpo_3e-6-2ep_0alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_3e-6-2ep_0alp_0lam", - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_3e-6-2ep_0alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2Model", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_3e-6-2ep_0alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2368 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3224 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0461 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2743 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3355 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1516 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-dpo_3e-6-3ep_0alp_0lam.json b/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-dpo_3e-6-3ep_0alp_0lam.json deleted file mode 100644 index 58bddc1e682a89b5c9576e8710ee88b49be50dbf..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-dpo_3e-6-3ep_0alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_3e-6-3ep_0alp_0lam", - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_3e-6-3ep_0alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_3e-6-3ep_0alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2372 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3248 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0476 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2701 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3394 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.155 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-dpo_3e-7-3ep_0alp_0lam.json b/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-dpo_3e-7-3ep_0alp_0lam.json deleted file mode 100644 index 527d419878a75df51a000b893aefb4f2c327e0c2..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-dpo_3e-7-3ep_0alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_3e-7-3ep_0alp_0lam", - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_3e-7-3ep_0alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_3e-7-3ep_0alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2499 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3181 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0415 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2651 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3288 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1574 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-dpo_5e-6-1ep_0alp_0lam.json b/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-dpo_5e-6-1ep_0alp_0lam.json deleted file mode 100644 index 74cb0cf77100f10bfa8a9c3e99c2c98e436960d5..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-dpo_5e-6-1ep_0alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_5e-6-1ep_0alp_0lam", - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_5e-6-1ep_0alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2Model", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_5e-6-1ep_0alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2381 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3242 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0498 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2743 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3328 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1572 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-dpo_5e-6-2ep_0alp_0lam.json b/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-dpo_5e-6-2ep_0alp_0lam.json deleted file mode 100644 index 5280cf9206af5fa696792c6cb48e5275e19b9478..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-dpo_5e-6-2ep_0alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_5e-6-2ep_0alp_0lam", - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_5e-6-2ep_0alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2Model", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_5e-6-2ep_0alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2421 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3225 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.04 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2802 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3408 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1496 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-dpo_5e-6-3ep_0alp_0lam.json b/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-dpo_5e-6-3ep_0alp_0lam.json deleted file mode 100644 index 39cb93fb9807cb03de8e27ef77d79d36fedc636a..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-dpo_5e-6-3ep_0alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_5e-6-3ep_0alp_0lam", - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_5e-6-3ep_0alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_5e-6-3ep_0alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2381 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3265 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0446 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.276 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3408 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1499 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-dpo_5e-7_1ep_0alp_0lam.json b/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-dpo_5e-7_1ep_0alp_0lam.json deleted file mode 100644 index b842671fd712713439a585c4b63d4f43031f1ae9..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-dpo_5e-7_1ep_0alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_5e-7_1ep_0alp_0lam", - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_5e-7_1ep_0alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2Model", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_5e-7_1ep_0alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2526 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3177 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0438 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2735 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3342 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1572 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-dpo_5e-7_2ep_0alp_0lam.json b/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-dpo_5e-7_2ep_0alp_0lam.json deleted file mode 100644 index 9682d35675906bef4ed3236e076aaa46b1be74a2..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-dpo_5e-7_2ep_0alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_5e-7_2ep_0alp_0lam", - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_5e-7_2ep_0alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2Model", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_5e-7_2ep_0alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2457 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.316 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0446 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2727 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3302 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1572 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-dpo_5e-7_3ep_0alp_0lam.json b/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-dpo_5e-7_3ep_0alp_0lam.json deleted file mode 100644 index d0208b8b0b34a769d3a284984701dc71f72d95fa..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-dpo_5e-7_3ep_0alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_5e-7_3ep_0alp_0lam", - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_5e-7_3ep_0alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2Model", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_5e-7_3ep_0alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2442 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3194 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0483 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2735 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3315 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1567 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-dpo_7e-7_1ep_0alp_0lam.json b/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-dpo_7e-7_1ep_0alp_0lam.json deleted file mode 100644 index 26f72caac46eb10ede02f3812f977e01f998e866..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-dpo_7e-7_1ep_0alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_7e-7_1ep_0alp_0lam", - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_7e-7_1ep_0alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2Model", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_7e-7_1ep_0alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2604 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3178 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0355 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.276 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3288 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1567 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-dpo_7e-7_2ep_0alp_0lam.json b/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-dpo_7e-7_2ep_0alp_0lam.json deleted file mode 100644 index 3cd69fef89aadc1fd76cb8063cdab2ebcc725d79..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-dpo_7e-7_2ep_0alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_7e-7_2ep_0alp_0lam", - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_7e-7_2ep_0alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2Model", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_7e-7_2ep_0alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.249 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3173 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0393 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.271 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3302 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1569 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-dpo_7e-7_3ep_0alp_0lam.json b/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-dpo_7e-7_3ep_0alp_0lam.json deleted file mode 100644 index b8c5a01f5703f5d3e6065c35b6ae4423418f3b43..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-dpo_7e-7_3ep_0alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_7e-7_3ep_0alp_0lam", - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_7e-7_3ep_0alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2Model", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-SFT-2e-5-2ep-DPO_7e-7_3ep_0alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2604 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.315 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0378 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2743 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3342 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1566 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-dpop_3e-7-3ep_0alp_5lam.json b/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-dpop_3e-7-3ep_0alp_5lam.json deleted file mode 100644 index dc1155878e6ef0069643a5da01921874d658e185..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-dpop_3e-7-3ep_0alp_5lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-DPOP_3e-7-3ep_0alp_5lam", - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-DPOP_3e-7-3ep_0alp_5lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-SFT-2e-5-2ep-DPOP_3e-7-3ep_0alp_5lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2411 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3167 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0347 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.271 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3301 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1562 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-dpop_5e-6-1ep_0alp_5lam.json b/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-dpop_5e-6-1ep_0alp_5lam.json deleted file mode 100644 index 1272133d60ddb63ad0562ad18b2aa64c15d6f9b1..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-dpop_5e-6-1ep_0alp_5lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-DPOP_5e-6-1ep_0alp_5lam", - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-DPOP_5e-6-1ep_0alp_5lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2Model", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-SFT-2e-5-2ep-DPOP_5e-6-1ep_0alp_5lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2369 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.326 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0453 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.276 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3355 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.157 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-dpop_5e-6-2ep_0alp_5lam.json b/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-dpop_5e-6-2ep_0alp_5lam.json deleted file mode 100644 index c36d4fb4db225ddd34c4608c1dc7ef1d39f9e2de..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-dpop_5e-6-2ep_0alp_5lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-DPOP_5e-6-2ep_0alp_5lam", - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-DPOP_5e-6-2ep_0alp_5lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2Model", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-SFT-2e-5-2ep-DPOP_5e-6-2ep_0alp_5lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2262 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3262 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0347 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2794 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3408 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1541 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-dpop_5e-6-3ep_0alp_5lam.json b/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-dpop_5e-6-3ep_0alp_5lam.json deleted file mode 100644 index 890e2a9317628b207da717dca040e3c69ec941af..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-dpop_5e-6-3ep_0alp_5lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-DPOP_5e-6-3ep_0alp_5lam", - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-DPOP_5e-6-3ep_0alp_5lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-SFT-2e-5-2ep-DPOP_5e-6-3ep_0alp_5lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2508 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3199 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0408 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.276 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3355 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1555 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-dpop_5e-7-1ep_0alp_5lam.json b/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-dpop_5e-7-1ep_0alp_5lam.json deleted file mode 100644 index 62a5436bba41ada193b4ad3b6d0b97155c86b45b..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-dpop_5e-7-1ep_0alp_5lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-DPOP_5e-7-1ep_0alp_5lam", - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-DPOP_5e-7-1ep_0alp_5lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2Model", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-SFT-2e-5-2ep-DPOP_5e-7-1ep_0alp_5lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.239 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3182 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.04 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2676 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3328 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.156 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-dpop_5e-7-2ep_0alp_5lam.json b/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-dpop_5e-7-2ep_0alp_5lam.json deleted file mode 100644 index 09441b9ba23abd66d84d5a0ec1c049bcea0c8ec6..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-dpop_5e-7-2ep_0alp_5lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-DPOP_5e-7-2ep_0alp_5lam", - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-DPOP_5e-7-2ep_0alp_5lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2Model", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-SFT-2e-5-2ep-DPOP_5e-7-2ep_0alp_5lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2423 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3154 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0347 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2676 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3328 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1548 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-dpop_5e-7-3ep_0alp_5lam.json b/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-dpop_5e-7-3ep_0alp_5lam.json deleted file mode 100644 index 0b8b965d5bcd021c73cc8fa58d18aef09507f313..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-dpop_5e-7-3ep_0alp_5lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-DPOP_5e-7-3ep_0alp_5lam", - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-DPOP_5e-7-3ep_0alp_5lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-SFT-2e-5-2ep-DPOP_5e-7-3ep_0alp_5lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2493 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.319 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0438 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2651 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3341 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1561 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-irpo_1e-7-1ep_1alp_0lam.json b/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-irpo_1e-7-1ep_1alp_0lam.json deleted file mode 100644 index 3554d17e97d560acfb6c4f4069429dee41bcab10..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-irpo_1e-7-1ep_1alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-IRPO_1e-7-1ep_1alp_0lam", - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-IRPO_1e-7-1ep_1alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2Model", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-SFT-2e-5-2ep-IRPO_1e-7-1ep_1alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.255 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3211 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0491 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2701 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3288 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1571 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-irpo_1e-7-2ep_1alp_0lam.json b/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-irpo_1e-7-2ep_1alp_0lam.json deleted file mode 100644 index 2662da9b7183fe0bad4e310de2482f4eafffe6c9..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-irpo_1e-7-2ep_1alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-IRPO_1e-7-2ep_1alp_0lam", - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-IRPO_1e-7-2ep_1alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2Model", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-SFT-2e-5-2ep-IRPO_1e-7-2ep_1alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2478 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3198 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0423 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2668 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3315 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1587 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-irpo_1e-7-3ep_1alp_0lam.json b/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-irpo_1e-7-3ep_1alp_0lam.json deleted file mode 100644 index 693ceff1db2b128306d75fcef82641e07b757ef2..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-irpo_1e-7-3ep_1alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-IRPO_1e-7-3ep_1alp_0lam", - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-IRPO_1e-7-3ep_1alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-SFT-2e-5-2ep-IRPO_1e-7-3ep_1alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2475 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3225 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.04 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.271 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3301 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1556 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-irpo_3e-7-3ep_1alp_0lam.json b/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-irpo_3e-7-3ep_1alp_0lam.json deleted file mode 100644 index b8737b639fcb7e5912da0944ffb9a22ea4198d1f..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-irpo_3e-7-3ep_1alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-IRPO_3e-7-3ep_1alp_0lam", - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-IRPO_3e-7-3ep_1alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-SFT-2e-5-2ep-IRPO_3e-7-3ep_1alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.259 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3185 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0363 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2727 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3275 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1586 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-irpo_5e-6-1ep_1alp_0lam.json b/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-irpo_5e-6-1ep_1alp_0lam.json deleted file mode 100644 index f270618793661857af86b77a8244d03081d8023d..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-irpo_5e-6-1ep_1alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-IRPO_5e-6-1ep_1alp_0lam", - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-IRPO_5e-6-1ep_1alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2Model", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-SFT-2e-5-2ep-IRPO_5e-6-1ep_1alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2323 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3179 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0453 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2827 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3262 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1548 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-irpo_5e-6-2ep_1alp_0lam.json b/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-irpo_5e-6-2ep_1alp_0lam.json deleted file mode 100644 index d4650f02b7d7999d3f9110ac1e11d2fb0bc36ed9..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-irpo_5e-6-2ep_1alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-IRPO_5e-6-2ep_1alp_0lam", - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-IRPO_5e-6-2ep_1alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2Model", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-SFT-2e-5-2ep-IRPO_5e-6-2ep_1alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2315 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.326 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0415 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2701 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3383 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1521 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-irpo_5e-6-3ep_1alp_0lam.json b/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-irpo_5e-6-3ep_1alp_0lam.json deleted file mode 100644 index e0d4725dad106d61cb237ced5f17195b3c046256..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-irpo_5e-6-3ep_1alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-IRPO_5e-6-3ep_1alp_0lam", - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-IRPO_5e-6-3ep_1alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-SFT-2e-5-2ep-IRPO_5e-6-3ep_1alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2298 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.332 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0431 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2659 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3329 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1567 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-irpo_5e-7-1ep_1alp_0lam.json b/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-irpo_5e-7-1ep_1alp_0lam.json deleted file mode 100644 index d6d1e79812a1b7898b14e71673dcca9739e5e35d..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-irpo_5e-7-1ep_1alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-IRPO_5e-7-1ep_1alp_0lam", - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-IRPO_5e-7-1ep_1alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2Model", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-SFT-2e-5-2ep-IRPO_5e-7-1ep_1alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2469 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3179 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0415 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2794 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3302 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1575 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-irpo_5e-7-2ep_1alp_0lam.json b/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-irpo_5e-7-2ep_1alp_0lam.json deleted file mode 100644 index 36e2ef65e7b8f36f00cd7cfafa84d71e772049b8..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-irpo_5e-7-2ep_1alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-IRPO_5e-7-2ep_1alp_0lam", - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-IRPO_5e-7-2ep_1alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2Model", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-SFT-2e-5-2ep-IRPO_5e-7-2ep_1alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.252 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3168 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.037 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2752 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3328 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1576 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-irpo_5e-7-3ep_1alp_0lam.json b/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-irpo_5e-7-3ep_1alp_0lam.json deleted file mode 100644 index 7c162263064859338f685293b30586b120e55426..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-irpo_5e-7-3ep_1alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-IRPO_5e-7-3ep_1alp_0lam", - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-IRPO_5e-7-3ep_1alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-SFT-2e-5-2ep-IRPO_5e-7-3ep_1alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2666 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3191 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0347 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2718 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3289 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1567 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-mdpo_0.5_1e-7-1ep_0alp_0lam.json b/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-mdpo_0.5_1e-7-1ep_0alp_0lam.json deleted file mode 100644 index 6c3c968e328dd83eae89db842be0862a57c0aefd..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-mdpo_0.5_1e-7-1ep_0alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_0.5_1e-7-1ep_0alp_0lam", - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_0.5_1e-7-1ep_0alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2Model", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_0.5_1e-7-1ep_0alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2499 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3178 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.037 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2651 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3341 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1562 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-mdpo_0.5_1e-7-2ep_0alp_0lam.json b/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-mdpo_0.5_1e-7-2ep_0alp_0lam.json deleted file mode 100644 index a3eed5abd56a2f7005561e9dde274d842f65b546..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-mdpo_0.5_1e-7-2ep_0alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_0.5_1e-7-2ep_0alp_0lam", - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_0.5_1e-7-2ep_0alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2Model", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_0.5_1e-7-2ep_0alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2417 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3178 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.04 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2685 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3328 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1575 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-mdpo_0.5_1e-7-3ep_0alp_0lam.json b/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-mdpo_0.5_1e-7-3ep_0alp_0lam.json deleted file mode 100644 index 5dce01d72e21ac8cbaa2d9859adbdb8dbc69ac71..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-mdpo_0.5_1e-7-3ep_0alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_0.5_1e-7-3ep_0alp_0lam", - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_0.5_1e-7-3ep_0alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_0.5_1e-7-3ep_0alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2562 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.319 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0423 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2659 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3341 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1576 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-mdpo_1e-6-3ep_0alp_0lam.json b/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-mdpo_1e-6-3ep_0alp_0lam.json deleted file mode 100644 index b2af0846a8d20c521aa639e88196e1875e20dbb2..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-mdpo_1e-6-3ep_0alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_1e-6-3ep_0alp_0lam", - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_1e-6-3ep_0alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_1e-6-3ep_0alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2408 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3165 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0431 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2735 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3315 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1557 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-mdpo_1e-6_1ep_0alp_0lam.json b/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-mdpo_1e-6_1ep_0alp_0lam.json deleted file mode 100644 index 3b2cbd8b6a0f3a9c44f866b929d954045be15edc..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-mdpo_1e-6_1ep_0alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_1e-6_1ep_0alp_0lam", - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_1e-6_1ep_0alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2Model", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_1e-6_1ep_0alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2481 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3204 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0476 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.276 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3302 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1592 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-mdpo_1e-6_2ep_0alp_0lam.json b/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-mdpo_1e-6_2ep_0alp_0lam.json deleted file mode 100644 index 2badca1d0da9ae0e32a3446b8218c6baeb0d7351..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-mdpo_1e-6_2ep_0alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_1e-6_2ep_0alp_0lam", - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_1e-6_2ep_0alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2Model", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_1e-6_2ep_0alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2545 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3186 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0498 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2718 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3289 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1561 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-mdpo_2e-6-3ep_0alp_0lam.json b/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-mdpo_2e-6-3ep_0alp_0lam.json deleted file mode 100644 index 2cd4c17c6eb0e857e2c05f64599b98fc1dfe7e19..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-mdpo_2e-6-3ep_0alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_2e-6-3ep_0alp_0lam", - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_2e-6-3ep_0alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_2e-6-3ep_0alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.252 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3204 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0393 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2727 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3262 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1538 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-mdpo_2e-6_1ep_0alp_0lam.json b/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-mdpo_2e-6_1ep_0alp_0lam.json deleted file mode 100644 index 0c68d7ae9f171f9f610a0bab77a6d5d3926738a0..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-mdpo_2e-6_1ep_0alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_2e-6_1ep_0alp_0lam", - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_2e-6_1ep_0alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2Model", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_2e-6_1ep_0alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2315 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3213 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0453 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2802 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3222 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1582 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-mdpo_2e-6_2ep_0alp_0lam.json b/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-mdpo_2e-6_2ep_0alp_0lam.json deleted file mode 100644 index 3ddaef18cfc8e1c67d7f42d56cce33aa7ec9e722..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-mdpo_2e-6_2ep_0alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_2e-6_2ep_0alp_0lam", - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_2e-6_2ep_0alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2Model", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_2e-6_2ep_0alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2515 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3187 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0431 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2718 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3289 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1539 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-mdpo_3e-6-1ep_0alp_0lam.json b/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-mdpo_3e-6-1ep_0alp_0lam.json deleted file mode 100644 index 68c11b4bb59316f5d4bc0f75b0def57ecf2d7c83..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-mdpo_3e-6-1ep_0alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_3e-6-1ep_0alp_0lam", - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_3e-6-1ep_0alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2Model", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_3e-6-1ep_0alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2472 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3213 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0347 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2727 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3262 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1588 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-mdpo_3e-6-2ep_0alp_0lam.json b/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-mdpo_3e-6-2ep_0alp_0lam.json deleted file mode 100644 index c06c481ca780378c581d6ac094ba3449c59aecd0..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-mdpo_3e-6-2ep_0alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_3e-6-2ep_0alp_0lam", - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_3e-6-2ep_0alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2Model", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_3e-6-2ep_0alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.246 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3234 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0378 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2794 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3302 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1533 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-mdpo_3e-6-3ep_0alp_0lam.json b/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-mdpo_3e-6-3ep_0alp_0lam.json deleted file mode 100644 index d481cd68ce73c8476e5d56a3b29b566df9f13925..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-mdpo_3e-6-3ep_0alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_3e-6-3ep_0alp_0lam", - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_3e-6-3ep_0alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_3e-6-3ep_0alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2524 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3256 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0536 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2777 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3368 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1531 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-mdpo_5e-6-1ep_0alp_0lam.json b/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-mdpo_5e-6-1ep_0alp_0lam.json deleted file mode 100644 index 8e028cf9d1dd2260f8f27dd45b9dd9a7100e968b..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-mdpo_5e-6-1ep_0alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_5e-6-1ep_0alp_0lam", - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_5e-6-1ep_0alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2Model", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_5e-6-1ep_0alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2265 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3252 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0476 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2735 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3262 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1568 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-mdpo_5e-6-2ep_0alp_0lam.json b/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-mdpo_5e-6-2ep_0alp_0lam.json deleted file mode 100644 index 833be6331e922871c7849ecb0d1921187a73921c..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-mdpo_5e-6-2ep_0alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_5e-6-2ep_0alp_0lam", - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_5e-6-2ep_0alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2Model", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_5e-6-2ep_0alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2302 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3224 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0438 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2768 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3408 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.15 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-mdpo_5e-6-3ep_0alp_0lam.json b/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-mdpo_5e-6-3ep_0alp_0lam.json deleted file mode 100644 index 301e25ae17a01458dec3d04534dbae253b731974..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-mdpo_5e-6-3ep_0alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_5e-6-3ep_0alp_0lam", - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_5e-6-3ep_0alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_5e-6-3ep_0alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2524 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3278 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0408 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2777 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3395 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1521 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-mdpo_5e-7-3ep_0alp_0lam.json b/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-mdpo_5e-7-3ep_0alp_0lam.json deleted file mode 100644 index d79b0dd99dba0c2f055e5d75d7d4eaa9f81bea5e..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-mdpo_5e-7-3ep_0alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_5e-7-3ep_0alp_0lam", - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_5e-7-3ep_0alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_5e-7-3ep_0alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2658 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3175 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0363 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2617 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3302 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1575 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-mdpo_5e-7_1ep_0alp_0lam.json b/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-mdpo_5e-7_1ep_0alp_0lam.json deleted file mode 100644 index 1c7430f6dd644fd778efd20586ff99b68c397a80..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-mdpo_5e-7_1ep_0alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_5e-7_1ep_0alp_0lam", - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_5e-7_1ep_0alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2Model", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_5e-7_1ep_0alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2487 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3189 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0378 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2718 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3275 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1595 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-mdpo_5e-7_2ep_0alp_0lam.json b/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-mdpo_5e-7_2ep_0alp_0lam.json deleted file mode 100644 index d53d98d21e1f3447102eff2a940ba889a054ea8d..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-mdpo_5e-7_2ep_0alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_5e-7_2ep_0alp_0lam", - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_5e-7_2ep_0alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2Model", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_5e-7_2ep_0alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.256 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3159 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0378 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2768 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3275 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1562 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-mdpo_7e-7-3ep_0alp_0lam.json b/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-mdpo_7e-7-3ep_0alp_0lam.json deleted file mode 100644 index edad1b857d79b97ef36bc2591c77065a7d4be745..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-mdpo_7e-7-3ep_0alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_7e-7-3ep_0alp_0lam", - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_7e-7-3ep_0alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_7e-7-3ep_0alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2499 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3156 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.04 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2701 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3302 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1556 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-mdpo_7e-7_1ep_0alp_0lam.json b/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-mdpo_7e-7_1ep_0alp_0lam.json deleted file mode 100644 index e4bcea75645fdf37204b23a175faa4322b2a20b2..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-mdpo_7e-7_1ep_0alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_7e-7_1ep_0alp_0lam", - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_7e-7_1ep_0alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2Model", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_7e-7_1ep_0alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2496 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3177 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0453 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2626 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3315 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1567 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-mdpo_7e-7_2ep_0alp_0lam.json b/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-mdpo_7e-7_2ep_0alp_0lam.json deleted file mode 100644 index ced21b1e8134dbadeee6514e0dc8be26a174cb97..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep-mdpo_7e-7_2ep_0alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_7e-7_2ep_0alp_0lam", - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_7e-7_2ep_0alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2Model", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-SFT-2e-5-2ep-MDPO_7e-7_2ep_0alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2515 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3172 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0438 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2701 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3275 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1553 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep.json b/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep.json deleted file mode 100644 index 57bb29ad3165ae351078009dbaf3931b482fca37..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-2ep.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-SFT-2e-5-2ep", - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-SFT-2e-5-2ep/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2201 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3217 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0408 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2777 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3367 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.171 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-3ep.json b/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-3ep.json deleted file mode 100644 index 19b5111346c9aab7785d6f2735def58fcd731f75..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-3ep.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-SFT-2e-5-3ep", - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-3ep", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-SFT-2e-5-3ep/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2281 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.324 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0453 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2617 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3301 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1746 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-5ep-mdpo_5e-7_3ep_0alp_0lam.json b/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-5ep-mdpo_5e-7_3ep_0alp_0lam.json deleted file mode 100644 index ebd68d1cf43e40d01110fe151b364d4094a43c1d..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-5ep-mdpo_5e-7_3ep_0alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-SFT-2e-5-5ep-MDPO_5e-7_3ep_0alp_0lam", - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-5ep-MDPO_5e-7_3ep_0alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-SFT-2e-5-5ep-MDPO_5e-7_3ep_0alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2526 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3238 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0393 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2676 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3528 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1574 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-5ep-mdpo_5e-7_3ep_0alp_0lam_1ep.json b/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-5ep-mdpo_5e-7_3ep_0alp_0lam_1ep.json deleted file mode 100644 index 58f6dee08147887b80445ec9cfb3543c83d1b6a1..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-5ep-mdpo_5e-7_3ep_0alp_0lam_1ep.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-SFT-2e-5-5ep-MDPO_5e-7_3ep_0alp_0lam_1ep", - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-5ep-MDPO_5e-7_3ep_0alp_0lam_1ep", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2Model", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-SFT-2e-5-5ep-MDPO_5e-7_3ep_0alp_0lam_1ep/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2481 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3175 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0385 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2626 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3475 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1597 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-5ep-mdpo_5e-7_3ep_0alp_0lam_2ep.json b/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-5ep-mdpo_5e-7_3ep_0alp_0lam_2ep.json deleted file mode 100644 index ad9a591e80cd546306611a9f061865575341a835..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-5ep-mdpo_5e-7_3ep_0alp_0lam_2ep.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-SFT-2e-5-5ep-MDPO_5e-7_3ep_0alp_0lam_2ep", - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-5ep-MDPO_5e-7_3ep_0alp_0lam_2ep", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2Model", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-SFT-2e-5-5ep-MDPO_5e-7_3ep_0alp_0lam_2ep/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2548 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3199 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0385 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2651 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3435 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1562 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-5ep-mdpo_7e-7_3ep_0alp_0lam.json b/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-5ep-mdpo_7e-7_3ep_0alp_0lam.json deleted file mode 100644 index 470f28d65139ebfd84a48b14de6f4e6bc3790ec3..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-5ep-mdpo_7e-7_3ep_0alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-SFT-2e-5-5ep-MDPO_7e-7_3ep_0alp_0lam", - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-5ep-MDPO_7e-7_3ep_0alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-SFT-2e-5-5ep-MDPO_7e-7_3ep_0alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2423 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3219 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.034 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2701 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3515 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1563 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-5ep-mdpo_7e-7_3ep_0alp_0lam_1ep.json b/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-5ep-mdpo_7e-7_3ep_0alp_0lam_1ep.json deleted file mode 100644 index 9a9c23afe77698de3f528ba78bbb50b587193ddd..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-5ep-mdpo_7e-7_3ep_0alp_0lam_1ep.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-SFT-2e-5-5ep-MDPO_7e-7_3ep_0alp_0lam_1ep", - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-5ep-MDPO_7e-7_3ep_0alp_0lam_1ep", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2Model", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-SFT-2e-5-5ep-MDPO_7e-7_3ep_0alp_0lam_1ep/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2493 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3191 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0393 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2685 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3475 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1592 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-5ep-mdpo_7e-7_3ep_0alp_0lam_2ep.json b/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-5ep-mdpo_7e-7_3ep_0alp_0lam_2ep.json deleted file mode 100644 index 6e19c2dbaed295e85f8afe46ddc53c6575498575..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-5ep-mdpo_7e-7_3ep_0alp_0lam_2ep.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-SFT-2e-5-5ep-MDPO_7e-7_3ep_0alp_0lam_2ep", - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-5ep-MDPO_7e-7_3ep_0alp_0lam_2ep", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2Model", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-SFT-2e-5-5ep-MDPO_7e-7_3ep_0alp_0lam_2ep/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2478 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3218 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0415 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2693 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3515 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1556 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-5ep.json b/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-5ep.json deleted file mode 100644 index d35dc52021dd2d74d7066fa953cea12a3a0e98f6..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5-5ep.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-SFT-2e-5-5ep", - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5-5ep", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-SFT-2e-5-5ep/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2348 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3308 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0506 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2643 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3409 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1695 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5.json b/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5.json deleted file mode 100644 index e4bae40bc6489376e3bd6d538ec2febc2ebd14fd..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-sft-2e-5.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-SFT-2e-5", - "id": "JayHyeon/Qwen2.5-0.5B-SFT-2e-5", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-SFT-2e-5/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2068 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3204 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.037 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2693 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3487 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1678 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-sft-5e-5-2ep.json b/data/models/jayhyeon_qwen2.5-0.5b-sft-5e-5-2ep.json deleted file mode 100644 index 9af1931a4d0365f9720e27db4946b6d4052d2ec3..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-sft-5e-5-2ep.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-SFT-5e-5-2ep", - "id": "JayHyeon/Qwen2.5-0.5B-SFT-5e-5-2ep", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-SFT-5e-5-2ep/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2175 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.318 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0378 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2601 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3368 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1627 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-sft-5e-5-3ep.json b/data/models/jayhyeon_qwen2.5-0.5b-sft-5e-5-3ep.json deleted file mode 100644 index 60093d221c2105254de30086c39cac5638c63f9f..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-sft-5e-5-3ep.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-SFT-5e-5-3ep", - "id": "JayHyeon/Qwen2.5-0.5B-SFT-5e-5-3ep", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-SFT-5e-5-3ep/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2199 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3297 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0302 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2534 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3593 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1651 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-sft-5e-5-5ep.json b/data/models/jayhyeon_qwen2.5-0.5b-sft-5e-5-5ep.json deleted file mode 100644 index 1216027b7c1fa0abfefaf2008f74fb4d29746046..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-sft-5e-5-5ep.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-SFT-5e-5-5ep", - "id": "JayHyeon/Qwen2.5-0.5B-SFT-5e-5-5ep", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-SFT-5e-5-5ep/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2077 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3276 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0272 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2685 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3766 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1587 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-sft-5e-5.json b/data/models/jayhyeon_qwen2.5-0.5b-sft-5e-5.json deleted file mode 100644 index b67544f6d59a50bb1aaef3935f6bcb95c173f39a..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-sft-5e-5.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-SFT-5e-5", - "id": "JayHyeon/Qwen2.5-0.5B-SFT-5e-5", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-SFT-5e-5/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.201 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3109 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.034 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2676 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3381 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1672 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-sft-7e-5-2ep.json b/data/models/jayhyeon_qwen2.5-0.5b-sft-7e-5-2ep.json deleted file mode 100644 index d7c0f0530685acfc4a74a7f93719efbe29a68294..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-sft-7e-5-2ep.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-SFT-7e-5-2ep", - "id": "JayHyeon/Qwen2.5-0.5B-SFT-7e-5-2ep", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-SFT-7e-5-2ep/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2156 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.31 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0393 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2424 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3367 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1567 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-sft-7e-5-3ep.json b/data/models/jayhyeon_qwen2.5-0.5b-sft-7e-5-3ep.json deleted file mode 100644 index 46a36d43884f30bb2c1d9754aad402b5e44ca311..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-sft-7e-5-3ep.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-SFT-7e-5-3ep", - "id": "JayHyeon/Qwen2.5-0.5B-SFT-7e-5-3ep", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-SFT-7e-5-3ep/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2381 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3199 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0332 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2366 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3554 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1522 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-sft-7e-5-5ep.json b/data/models/jayhyeon_qwen2.5-0.5b-sft-7e-5-5ep.json deleted file mode 100644 index 11d203d70fac334ac9dce8f438b9f0f96ed6ba7b..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-sft-7e-5-5ep.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-SFT-7e-5-5ep", - "id": "JayHyeon/Qwen2.5-0.5B-SFT-7e-5-5ep", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-SFT-7e-5-5ep/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.212 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.32 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0219 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2458 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3713 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1628 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-sft-7e-5.json b/data/models/jayhyeon_qwen2.5-0.5b-sft-7e-5.json deleted file mode 100644 index 0347aa7292d805a1727aab5fe5800678250b2556..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-sft-7e-5.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-SFT-7e-5", - "id": "JayHyeon/Qwen2.5-0.5B-SFT-7e-5", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-SFT-7e-5/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2093 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3158 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0302 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2567 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3367 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1622 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-sft-dpo-1epoch_v1.json b/data/models/jayhyeon_qwen2.5-0.5b-sft-dpo-1epoch_v1.json deleted file mode 100644 index b2cbd0933293196c7abbaac3473cb32299efad8f..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-sft-dpo-1epoch_v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-SFT-DPO-1epoch_v1", - "id": "JayHyeon/Qwen2.5-0.5B-SFT-DPO-1epoch_v1", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2Model", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-SFT-DPO-1epoch_v1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2025 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3268 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0363 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2727 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3209 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.133 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-sft-mdpo-1epoch_v1.json b/data/models/jayhyeon_qwen2.5-0.5b-sft-mdpo-1epoch_v1.json deleted file mode 100644 index 8674f93a5d02041b4969d1c8038e5c54a5b7529b..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-sft-mdpo-1epoch_v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-SFT-MDPO-1epoch_v1", - "id": "JayHyeon/Qwen2.5-0.5B-SFT-MDPO-1epoch_v1", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2Model", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-SFT-MDPO-1epoch_v1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1964 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3293 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0468 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.276 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3262 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1337 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen2.5-0.5b-sft.json b/data/models/jayhyeon_qwen2.5-0.5b-sft.json deleted file mode 100644 index 1e9c0029474a5e21776f10ae32851d1e4eed8488..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen2.5-0.5b-sft.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-SFT", - "id": "JayHyeon/Qwen2.5-0.5B-SFT", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen2.5-0.5B-SFT/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1964 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3121 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0272 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2785 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3394 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1673 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen_0.5-cdpo_5e-7-3ep_0vpo_const_0.1.json b/data/models/jayhyeon_qwen_0.5-cdpo_5e-7-3ep_0vpo_const_0.1.json deleted file mode 100644 index a366d419fa313db8ec95372ea20b54b140b3ba89..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen_0.5-cdpo_5e-7-3ep_0vpo_const_0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen_0.5-cDPO_5e-7-3ep_0vpo_const_0.1", - "id": "JayHyeon/Qwen_0.5-cDPO_5e-7-3ep_0vpo_const_0.1", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen_0.5-cDPO_5e-7-3ep_0vpo_const_0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2393 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3244 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0514 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2777 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3222 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1573 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen_0.5-cdpo_5e-7-3ep_0vpo_const_0.3.json b/data/models/jayhyeon_qwen_0.5-cdpo_5e-7-3ep_0vpo_const_0.3.json deleted file mode 100644 index e5aeefcb00bd0a4a9103195a9353fd213a167d50..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen_0.5-cdpo_5e-7-3ep_0vpo_const_0.3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen_0.5-cDPO_5e-7-3ep_0vpo_const_0.3", - "id": "JayHyeon/Qwen_0.5-cDPO_5e-7-3ep_0vpo_const_0.3", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen_0.5-cDPO_5e-7-3ep_0vpo_const_0.3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2475 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3209 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0461 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.281 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3275 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1567 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen_0.5-dpo_1e-6-3ep_0alp_0lam.json b/data/models/jayhyeon_qwen_0.5-dpo_1e-6-3ep_0alp_0lam.json deleted file mode 100644 index d8f5a4d7763b01c194d67e7f20c69ecb7afc4b26..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen_0.5-dpo_1e-6-3ep_0alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen_0.5-DPO_1e-6-3ep_0alp_0lam", - "id": "JayHyeon/Qwen_0.5-DPO_1e-6-3ep_0alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen_0.5-DPO_1e-6-3ep_0alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2316 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3258 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0529 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2693 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3221 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.158 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen_0.5-dpo_1e-7-3ep_0alp_0lam.json b/data/models/jayhyeon_qwen_0.5-dpo_1e-7-3ep_0alp_0lam.json deleted file mode 100644 index a94d3554a936b4bf474e6ab60588f210f6086d27..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen_0.5-dpo_1e-7-3ep_0alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen_0.5-DPO_1e-7-3ep_0alp_0lam", - "id": "JayHyeon/Qwen_0.5-DPO_1e-7-3ep_0alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen_0.5-DPO_1e-7-3ep_0alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.236 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3225 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0438 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.271 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3222 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1596 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen_0.5-dpo_3e-6-1ep_0alp_0lam.json b/data/models/jayhyeon_qwen_0.5-dpo_3e-6-1ep_0alp_0lam.json deleted file mode 100644 index 4b381ed220024bb090fbc4978fd0cf057d403f9c..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen_0.5-dpo_3e-6-1ep_0alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen_0.5-DPO_3e-6-1ep_0alp_0lam", - "id": "JayHyeon/Qwen_0.5-DPO_3e-6-1ep_0alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2Model", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen_0.5-DPO_3e-6-1ep_0alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2337 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3132 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0347 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2609 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3235 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1533 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen_0.5-dpo_3e-6-2ep_0alp_0lam.json b/data/models/jayhyeon_qwen_0.5-dpo_3e-6-2ep_0alp_0lam.json deleted file mode 100644 index 2a56653144307663faa780486fa9ceeb0113dce3..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen_0.5-dpo_3e-6-2ep_0alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen_0.5-DPO_3e-6-2ep_0alp_0lam", - "id": "JayHyeon/Qwen_0.5-DPO_3e-6-2ep_0alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2Model", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen_0.5-DPO_3e-6-2ep_0alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2569 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3276 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0544 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2718 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3156 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1565 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen_0.5-dpo_3e-6-3ep_0alp_0lam.json b/data/models/jayhyeon_qwen_0.5-dpo_3e-6-3ep_0alp_0lam.json deleted file mode 100644 index b441df139836711d270b035228984bab9b360757..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen_0.5-dpo_3e-6-3ep_0alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen_0.5-DPO_3e-6-3ep_0alp_0lam", - "id": "JayHyeon/Qwen_0.5-DPO_3e-6-3ep_0alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen_0.5-DPO_3e-6-3ep_0alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.246 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3267 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0431 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2685 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3209 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1543 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen_0.5-dpo_3e-7-1ep_0alp_0lam.json b/data/models/jayhyeon_qwen_0.5-dpo_3e-7-1ep_0alp_0lam.json deleted file mode 100644 index d3f8ea555561684c33314d7d4bd6fa4dde2ea1fd..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen_0.5-dpo_3e-7-1ep_0alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen_0.5-DPO_3e-7-1ep_0alp_0lam", - "id": "JayHyeon/Qwen_0.5-DPO_3e-7-1ep_0alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2Model", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen_0.5-DPO_3e-7-1ep_0alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2529 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3229 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0551 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2676 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3195 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1597 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen_0.5-dpo_3e-7-2ep_0alp_0lam.json b/data/models/jayhyeon_qwen_0.5-dpo_3e-7-2ep_0alp_0lam.json deleted file mode 100644 index aaf57dd5f56981137736b21c2e14a9d9a7d11e71..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen_0.5-dpo_3e-7-2ep_0alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen_0.5-DPO_3e-7-2ep_0alp_0lam", - "id": "JayHyeon/Qwen_0.5-DPO_3e-7-2ep_0alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2Model", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen_0.5-DPO_3e-7-2ep_0alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2505 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3256 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0476 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2718 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3195 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1599 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen_0.5-dpo_3e-7-3ep_0alp_0lam.json b/data/models/jayhyeon_qwen_0.5-dpo_3e-7-3ep_0alp_0lam.json deleted file mode 100644 index 9e76d0c1f2e4bbc507225a8646433b90b8e664df..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen_0.5-dpo_3e-7-3ep_0alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen_0.5-DPO_3e-7-3ep_0alp_0lam", - "id": "JayHyeon/Qwen_0.5-DPO_3e-7-3ep_0alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen_0.5-DPO_3e-7-3ep_0alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2387 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3258 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0446 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2743 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3169 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1589 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen_0.5-dpo_5e-7-1ep_0alp_0lam.json b/data/models/jayhyeon_qwen_0.5-dpo_5e-7-1ep_0alp_0lam.json deleted file mode 100644 index caae12e1014fb7dfd885d6af75f1a0892e98c9eb..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen_0.5-dpo_5e-7-1ep_0alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen_0.5-DPO_5e-7-1ep_0alp_0lam", - "id": "JayHyeon/Qwen_0.5-DPO_5e-7-1ep_0alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen_0.5-DPO_5e-7-1ep_0alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2532 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3218 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0634 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2685 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3209 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1593 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen_0.5-dpo_5e-7-2ep_0alp_0lam.json b/data/models/jayhyeon_qwen_0.5-dpo_5e-7-2ep_0alp_0lam.json deleted file mode 100644 index f7d07dc5ee23a30de74c170a346710802cea6474..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen_0.5-dpo_5e-7-2ep_0alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen_0.5-DPO_5e-7-2ep_0alp_0lam", - "id": "JayHyeon/Qwen_0.5-DPO_5e-7-2ep_0alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2Model", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen_0.5-DPO_5e-7-2ep_0alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2456 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3299 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0536 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.271 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3181 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1602 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen_0.5-dpo_5e-7-3ep_0alp_0lam.json b/data/models/jayhyeon_qwen_0.5-dpo_5e-7-3ep_0alp_0lam.json deleted file mode 100644 index 8199a183761b39680304160f0f908cfae17b6bc7..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen_0.5-dpo_5e-7-3ep_0alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen_0.5-DPO_5e-7-3ep_0alp_0lam", - "id": "JayHyeon/Qwen_0.5-DPO_5e-7-3ep_0alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen_0.5-DPO_5e-7-3ep_0alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2423 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3271 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0514 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2743 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3181 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1595 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen_0.5-dpop_1e-6-3ep_0alp_5lam.json b/data/models/jayhyeon_qwen_0.5-dpop_1e-6-3ep_0alp_5lam.json deleted file mode 100644 index c64a03decb984c723d2e7359667bf4ae990cbbbc..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen_0.5-dpop_1e-6-3ep_0alp_5lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen_0.5-DPOP_1e-6-3ep_0alp_5lam", - "id": "JayHyeon/Qwen_0.5-DPOP_1e-6-3ep_0alp_5lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen_0.5-DPOP_1e-6-3ep_0alp_5lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2532 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.314 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0491 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2743 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3315 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1566 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen_0.5-dpop_1e-7-3ep_0alp_5lam.json b/data/models/jayhyeon_qwen_0.5-dpop_1e-7-3ep_0alp_5lam.json deleted file mode 100644 index be09bd4ca644509d9cc20c4dfeb72914829ceb60..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen_0.5-dpop_1e-7-3ep_0alp_5lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen_0.5-DPOP_1e-7-3ep_0alp_5lam", - "id": "JayHyeon/Qwen_0.5-DPOP_1e-7-3ep_0alp_5lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen_0.5-DPOP_1e-7-3ep_0alp_5lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.267 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3189 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0408 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2668 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3288 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1562 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen_0.5-dpop_3e-6-1ep_0alp_5lam.json b/data/models/jayhyeon_qwen_0.5-dpop_3e-6-1ep_0alp_5lam.json deleted file mode 100644 index f9848dccd1dd145938141292a36d1c2d9308bbf1..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen_0.5-dpop_3e-6-1ep_0alp_5lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen_0.5-DPOP_3e-6-1ep_0alp_5lam", - "id": "JayHyeon/Qwen_0.5-DPOP_3e-6-1ep_0alp_5lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2Model", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen_0.5-DPOP_3e-6-1ep_0alp_5lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2481 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3261 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0438 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2601 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3368 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1565 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen_0.5-dpop_3e-6-2ep_0alp_5lam.json b/data/models/jayhyeon_qwen_0.5-dpop_3e-6-2ep_0alp_5lam.json deleted file mode 100644 index 86f53afdd5ebb73d64a6c4db3b6e6e8a152941f2..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen_0.5-dpop_3e-6-2ep_0alp_5lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen_0.5-DPOP_3e-6-2ep_0alp_5lam", - "id": "JayHyeon/Qwen_0.5-DPOP_3e-6-2ep_0alp_5lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2Model", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen_0.5-DPOP_3e-6-2ep_0alp_5lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2383 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3218 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0431 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2794 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3342 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1503 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen_0.5-dpop_3e-6-3ep_0alp_5lam.json b/data/models/jayhyeon_qwen_0.5-dpop_3e-6-3ep_0alp_5lam.json deleted file mode 100644 index ffaad9da84a4a8f24ac01cc73002f29f10f55e60..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen_0.5-dpop_3e-6-3ep_0alp_5lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen_0.5-DPOP_3e-6-3ep_0alp_5lam", - "id": "JayHyeon/Qwen_0.5-DPOP_3e-6-3ep_0alp_5lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen_0.5-DPOP_3e-6-3ep_0alp_5lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2471 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3224 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.04 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2701 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3328 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1533 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen_0.5-dpop_3e-7-1ep_0alp_5lam.json b/data/models/jayhyeon_qwen_0.5-dpop_3e-7-1ep_0alp_5lam.json deleted file mode 100644 index 49f5b8e843077e635990a9c734d48eda247f88d9..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen_0.5-dpop_3e-7-1ep_0alp_5lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen_0.5-DPOP_3e-7-1ep_0alp_5lam", - "id": "JayHyeon/Qwen_0.5-DPOP_3e-7-1ep_0alp_5lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2Model", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen_0.5-DPOP_3e-7-1ep_0alp_5lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2447 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3181 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0438 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2617 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3341 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1565 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen_0.5-dpop_3e-7-2ep_0alp_5lam.json b/data/models/jayhyeon_qwen_0.5-dpop_3e-7-2ep_0alp_5lam.json deleted file mode 100644 index c9536cdf5e84eceb98ce806644f19c866e4757b2..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen_0.5-dpop_3e-7-2ep_0alp_5lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen_0.5-DPOP_3e-7-2ep_0alp_5lam", - "id": "JayHyeon/Qwen_0.5-DPOP_3e-7-2ep_0alp_5lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2Model", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen_0.5-DPOP_3e-7-2ep_0alp_5lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2551 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3194 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0446 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2617 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3262 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1567 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen_0.5-dpop_3e-7-3ep_0alp_5lam.json b/data/models/jayhyeon_qwen_0.5-dpop_3e-7-3ep_0alp_5lam.json deleted file mode 100644 index 8b1cb28d96ede6fe40a31f4bcd1718404c9488da..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen_0.5-dpop_3e-7-3ep_0alp_5lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen_0.5-DPOP_3e-7-3ep_0alp_5lam", - "id": "JayHyeon/Qwen_0.5-DPOP_3e-7-3ep_0alp_5lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen_0.5-DPOP_3e-7-3ep_0alp_5lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2538 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3153 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0415 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2676 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3261 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1583 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen_0.5-dpop_5e-7-1ep_0alp_5lam.json b/data/models/jayhyeon_qwen_0.5-dpop_5e-7-1ep_0alp_5lam.json deleted file mode 100644 index 5d4a56429b433a1b31c4c0a5386666a10d764d30..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen_0.5-dpop_5e-7-1ep_0alp_5lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen_0.5-DPOP_5e-7-1ep_0alp_5lam", - "id": "JayHyeon/Qwen_0.5-DPOP_5e-7-1ep_0alp_5lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2Model", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen_0.5-DPOP_5e-7-1ep_0alp_5lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2402 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3168 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0378 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2718 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3328 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1568 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen_0.5-dpop_5e-7-2ep_0alp_5lam.json b/data/models/jayhyeon_qwen_0.5-dpop_5e-7-2ep_0alp_5lam.json deleted file mode 100644 index 4bceb657d8d6a5626bf77929d8c35f89d9b252b9..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen_0.5-dpop_5e-7-2ep_0alp_5lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen_0.5-DPOP_5e-7-2ep_0alp_5lam", - "id": "JayHyeon/Qwen_0.5-DPOP_5e-7-2ep_0alp_5lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2Model", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen_0.5-DPOP_5e-7-2ep_0alp_5lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2484 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3211 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0438 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2701 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3288 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1573 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen_0.5-dpop_5e-7-3ep_0alp_5lam.json b/data/models/jayhyeon_qwen_0.5-dpop_5e-7-3ep_0alp_5lam.json deleted file mode 100644 index 1f5f0b62e75a8203bd8b0e89376e17dd4914acb0..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen_0.5-dpop_5e-7-3ep_0alp_5lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen_0.5-DPOP_5e-7-3ep_0alp_5lam", - "id": "JayHyeon/Qwen_0.5-DPOP_5e-7-3ep_0alp_5lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen_0.5-DPOP_5e-7-3ep_0alp_5lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2578 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3203 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0423 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.271 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3289 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1583 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen_0.5-ipo_5e-7-1ep_0alp_0lam.json b/data/models/jayhyeon_qwen_0.5-ipo_5e-7-1ep_0alp_0lam.json deleted file mode 100644 index 512fffe553ff7daed9ad7816a6b5ad93d48bd445..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen_0.5-ipo_5e-7-1ep_0alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen_0.5-IPO_5e-7-1ep_0alp_0lam", - "id": "JayHyeon/Qwen_0.5-IPO_5e-7-1ep_0alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen_0.5-IPO_5e-7-1ep_0alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2574 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3279 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0559 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2693 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3169 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1651 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen_0.5-ipo_5e-7-3ep_0alp_0lam.json b/data/models/jayhyeon_qwen_0.5-ipo_5e-7-3ep_0alp_0lam.json deleted file mode 100644 index 4bc5c75f834f45916a7c3966a122e45214c4e4c0..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen_0.5-ipo_5e-7-3ep_0alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen_0.5-IPO_5e-7-3ep_0alp_0lam", - "id": "JayHyeon/Qwen_0.5-IPO_5e-7-3ep_0alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen_0.5-IPO_5e-7-3ep_0alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3072 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3264 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0582 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2567 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3156 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1624 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen_0.5-irpo_1e-6-3ep_1alp_0lam.json b/data/models/jayhyeon_qwen_0.5-irpo_1e-6-3ep_1alp_0lam.json deleted file mode 100644 index 207f28f4708f3f52ac5a59183d3d1b4374c41df5..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen_0.5-irpo_1e-6-3ep_1alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen_0.5-IRPO_1e-6-3ep_1alp_0lam", - "id": "JayHyeon/Qwen_0.5-IRPO_1e-6-3ep_1alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen_0.5-IRPO_1e-6-3ep_1alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2551 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3242 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0468 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2668 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3182 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1574 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen_0.5-irpo_1e-7-3ep_1alp_0lam.json b/data/models/jayhyeon_qwen_0.5-irpo_1e-7-3ep_1alp_0lam.json deleted file mode 100644 index cd6ae61361d29d0516a4426fcb5d9421165ec371..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen_0.5-irpo_1e-7-3ep_1alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen_0.5-IRPO_1e-7-3ep_1alp_0lam", - "id": "JayHyeon/Qwen_0.5-IRPO_1e-7-3ep_1alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen_0.5-IRPO_1e-7-3ep_1alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2636 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3198 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0514 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.276 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3262 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1586 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen_0.5-irpo_3e-6-1ep_1alp_0lam.json b/data/models/jayhyeon_qwen_0.5-irpo_3e-6-1ep_1alp_0lam.json deleted file mode 100644 index 51123ce397b93e942ff73742b3a3942e791a11d1..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen_0.5-irpo_3e-6-1ep_1alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen_0.5-IRPO_3e-6-1ep_1alp_0lam", - "id": "JayHyeon/Qwen_0.5-IRPO_3e-6-1ep_1alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2Model", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen_0.5-IRPO_3e-6-1ep_1alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2323 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3255 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.037 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2508 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3169 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1612 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen_0.5-irpo_3e-6-2ep_1alp_0lam.json b/data/models/jayhyeon_qwen_0.5-irpo_3e-6-2ep_1alp_0lam.json deleted file mode 100644 index 69029727df5d2e9b9026045bfff25a8319f05f1b..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen_0.5-irpo_3e-6-2ep_1alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen_0.5-IRPO_3e-6-2ep_1alp_0lam", - "id": "JayHyeon/Qwen_0.5-IRPO_3e-6-2ep_1alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2Model", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen_0.5-IRPO_3e-6-2ep_1alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2414 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3314 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0347 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2517 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3342 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1532 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen_0.5-irpo_3e-6-3ep_1alp_0lam.json b/data/models/jayhyeon_qwen_0.5-irpo_3e-6-3ep_1alp_0lam.json deleted file mode 100644 index 257ef8fc0a963cd8292c719d887a115a4cb8b340..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen_0.5-irpo_3e-6-3ep_1alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen_0.5-IRPO_3e-6-3ep_1alp_0lam", - "id": "JayHyeon/Qwen_0.5-IRPO_3e-6-3ep_1alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen_0.5-IRPO_3e-6-3ep_1alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2678 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3362 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0514 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2542 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3382 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1561 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen_0.5-irpo_3e-7-1ep_1alp_0lam.json b/data/models/jayhyeon_qwen_0.5-irpo_3e-7-1ep_1alp_0lam.json deleted file mode 100644 index ca4dd11c9bc183e1ca2f51d487f83ecddd5a2bbd..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen_0.5-irpo_3e-7-1ep_1alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen_0.5-IRPO_3e-7-1ep_1alp_0lam", - "id": "JayHyeon/Qwen_0.5-IRPO_3e-7-1ep_1alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2Model", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen_0.5-IRPO_3e-7-1ep_1alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2561 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3231 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0536 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2718 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3196 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1589 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen_0.5-irpo_3e-7-3ep_1alp_0lam.json b/data/models/jayhyeon_qwen_0.5-irpo_3e-7-3ep_1alp_0lam.json deleted file mode 100644 index ab77e6bfb47c6746a114b9659070eda5ed40c27b..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen_0.5-irpo_3e-7-3ep_1alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen_0.5-IRPO_3e-7-3ep_1alp_0lam", - "id": "JayHyeon/Qwen_0.5-IRPO_3e-7-3ep_1alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen_0.5-IRPO_3e-7-3ep_1alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2639 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3257 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0476 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2701 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3209 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1587 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen_0.5-irpo_5e-7-1ep_1alp_0lam.json b/data/models/jayhyeon_qwen_0.5-irpo_5e-7-1ep_1alp_0lam.json deleted file mode 100644 index 12d058849a8c5c5e39395d064636e0a4a1115445..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen_0.5-irpo_5e-7-1ep_1alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen_0.5-IRPO_5e-7-1ep_1alp_0lam", - "id": "JayHyeon/Qwen_0.5-IRPO_5e-7-1ep_1alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2Model", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen_0.5-IRPO_5e-7-1ep_1alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2518 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3214 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0574 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2735 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3169 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1585 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen_0.5-irpo_5e-7-2ep_1alp_0lam.json b/data/models/jayhyeon_qwen_0.5-irpo_5e-7-2ep_1alp_0lam.json deleted file mode 100644 index 1e2e482b1e442427dd281d65eb97c007582b8645..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen_0.5-irpo_5e-7-2ep_1alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen_0.5-IRPO_5e-7-2ep_1alp_0lam", - "id": "JayHyeon/Qwen_0.5-IRPO_5e-7-2ep_1alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2Model", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen_0.5-IRPO_5e-7-2ep_1alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2438 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3266 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0619 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2727 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3196 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1554 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen_0.5-irpo_5e-7-3ep_1alp_0lam.json b/data/models/jayhyeon_qwen_0.5-irpo_5e-7-3ep_1alp_0lam.json deleted file mode 100644 index f8f39a81c10f6e2e5e04bfc500f09b8f72f423f1..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen_0.5-irpo_5e-7-3ep_1alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen_0.5-IRPO_5e-7-3ep_1alp_0lam", - "id": "JayHyeon/Qwen_0.5-IRPO_5e-7-3ep_1alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen_0.5-IRPO_5e-7-3ep_1alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2465 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3246 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0529 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2718 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3182 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1563 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen_0.5-mdpo_0.1_3e-6-3ep_0alp_0lam.json b/data/models/jayhyeon_qwen_0.5-mdpo_0.1_3e-6-3ep_0alp_0lam.json deleted file mode 100644 index 6b880089842dec003fd8072d89994739e8652929..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen_0.5-mdpo_0.1_3e-6-3ep_0alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen_0.5-MDPO_0.1_3e-6-3ep_0alp_0lam", - "id": "JayHyeon/Qwen_0.5-MDPO_0.1_3e-6-3ep_0alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen_0.5-MDPO_0.1_3e-6-3ep_0alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2506 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3261 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0498 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2819 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3382 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1522 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen_0.5-mdpo_0.1_5e-7-3ep_0alp_0lam.json b/data/models/jayhyeon_qwen_0.5-mdpo_0.1_5e-7-3ep_0alp_0lam.json deleted file mode 100644 index e7e9ee0af42212adac4c7e167cbf2fffe3948749..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen_0.5-mdpo_0.1_5e-7-3ep_0alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen_0.5-MDPO_0.1_5e-7-3ep_0alp_0lam", - "id": "JayHyeon/Qwen_0.5-MDPO_0.1_5e-7-3ep_0alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen_0.5-MDPO_0.1_5e-7-3ep_0alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2457 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.318 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0347 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2634 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3315 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1566 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen_0.5-mdpo_0.3_3e-6-3ep_0alp_0lam.json b/data/models/jayhyeon_qwen_0.5-mdpo_0.3_3e-6-3ep_0alp_0lam.json deleted file mode 100644 index f9f6a2d7e9e08983331b0a4f12a4d3107d705e47..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen_0.5-mdpo_0.3_3e-6-3ep_0alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen_0.5-MDPO_0.3_3e-6-3ep_0alp_0lam", - "id": "JayHyeon/Qwen_0.5-MDPO_0.3_3e-6-3ep_0alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen_0.5-MDPO_0.3_3e-6-3ep_0alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2454 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3216 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0506 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2802 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3382 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1544 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen_0.5-mdpo_0.3_5e-7-3ep_0alp_0lam.json b/data/models/jayhyeon_qwen_0.5-mdpo_0.3_5e-7-3ep_0alp_0lam.json deleted file mode 100644 index 8d3e4c6162b00ce565bae77ef74fc8184998c4a8..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen_0.5-mdpo_0.3_5e-7-3ep_0alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen_0.5-MDPO_0.3_5e-7-3ep_0alp_0lam", - "id": "JayHyeon/Qwen_0.5-MDPO_0.3_5e-7-3ep_0alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen_0.5-MDPO_0.3_5e-7-3ep_0alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2342 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3189 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.04 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2701 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3302 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.158 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen_0.5-mdpo_0.5_1e-5-3ep_0alp_0lam.json b/data/models/jayhyeon_qwen_0.5-mdpo_0.5_1e-5-3ep_0alp_0lam.json deleted file mode 100644 index 9d3c6fb90a9bb302e7ac9a5adbcc42c40e27b500..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen_0.5-mdpo_0.5_1e-5-3ep_0alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen_0.5-MDPO_0.5_1e-5-3ep_0alp_0lam", - "id": "JayHyeon/Qwen_0.5-MDPO_0.5_1e-5-3ep_0alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen_0.5-MDPO_0.5_1e-5-3ep_0alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.232 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3234 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0393 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2743 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3369 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1543 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen_0.5-mdpo_0.5_3e-7-1ep_0alp_0lam.json b/data/models/jayhyeon_qwen_0.5-mdpo_0.5_3e-7-1ep_0alp_0lam.json deleted file mode 100644 index 87fd3407ec3c2ad20add655f868eebfef065a767..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen_0.5-mdpo_0.5_3e-7-1ep_0alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen_0.5-MDPO_0.5_3e-7-1ep_0alp_0lam", - "id": "JayHyeon/Qwen_0.5-MDPO_0.5_3e-7-1ep_0alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2Model", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen_0.5-MDPO_0.5_3e-7-1ep_0alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2418 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3175 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0423 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2626 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3288 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.158 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen_0.5-mdpo_0.5_3e-7-2ep_0alp_0lam.json b/data/models/jayhyeon_qwen_0.5-mdpo_0.5_3e-7-2ep_0alp_0lam.json deleted file mode 100644 index 838494d0c2ee298f08ffdb9abf68ef6277e6a53b..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen_0.5-mdpo_0.5_3e-7-2ep_0alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen_0.5-MDPO_0.5_3e-7-2ep_0alp_0lam", - "id": "JayHyeon/Qwen_0.5-MDPO_0.5_3e-7-2ep_0alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2Model", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen_0.5-MDPO_0.5_3e-7-2ep_0alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2493 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3197 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0423 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2701 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3315 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1571 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen_0.5-mdpo_0.5_3e-7-3ep_0alp_0lam.json b/data/models/jayhyeon_qwen_0.5-mdpo_0.5_3e-7-3ep_0alp_0lam.json deleted file mode 100644 index bd221e920720b0f56f2339469f0eacf2e8777ae3..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen_0.5-mdpo_0.5_3e-7-3ep_0alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen_0.5-MDPO_0.5_3e-7-3ep_0alp_0lam", - "id": "JayHyeon/Qwen_0.5-MDPO_0.5_3e-7-3ep_0alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen_0.5-MDPO_0.5_3e-7-3ep_0alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.252 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3198 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0423 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2634 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3262 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1551 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen_0.5-mdpo_0.5_4e-6-3ep_0alp_0lam.json b/data/models/jayhyeon_qwen_0.5-mdpo_0.5_4e-6-3ep_0alp_0lam.json deleted file mode 100644 index a62af8ba3b58c6d146ecc75612a8163011f782f8..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen_0.5-mdpo_0.5_4e-6-3ep_0alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen_0.5-MDPO_0.5_4e-6-3ep_0alp_0lam", - "id": "JayHyeon/Qwen_0.5-MDPO_0.5_4e-6-3ep_0alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen_0.5-MDPO_0.5_4e-6-3ep_0alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.258 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3248 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0476 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2752 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3422 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1539 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen_0.5-mdpo_0.5_6e-6-3ep_0alp_0lam.json b/data/models/jayhyeon_qwen_0.5-mdpo_0.5_6e-6-3ep_0alp_0lam.json deleted file mode 100644 index 8d6b5b2300082928dedc32d6f6dc3dc66e1744f5..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen_0.5-mdpo_0.5_6e-6-3ep_0alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen_0.5-MDPO_0.5_6e-6-3ep_0alp_0lam", - "id": "JayHyeon/Qwen_0.5-MDPO_0.5_6e-6-3ep_0alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen_0.5-MDPO_0.5_6e-6-3ep_0alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.232 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3265 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0385 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.271 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3395 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1537 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen_0.5-mdpo_0.5_7e-6-3ep_0alp_0lam.json b/data/models/jayhyeon_qwen_0.5-mdpo_0.5_7e-6-3ep_0alp_0lam.json deleted file mode 100644 index fee9fb4953e4ad7886a924f6f9e64ad716976531..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen_0.5-mdpo_0.5_7e-6-3ep_0alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen_0.5-MDPO_0.5_7e-6-3ep_0alp_0lam", - "id": "JayHyeon/Qwen_0.5-MDPO_0.5_7e-6-3ep_0alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen_0.5-MDPO_0.5_7e-6-3ep_0alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2488 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3273 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0461 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2718 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3342 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1531 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen_0.5-mdpo_0.5_7e-7-3ep_0alp_0lam.json b/data/models/jayhyeon_qwen_0.5-mdpo_0.5_7e-7-3ep_0alp_0lam.json deleted file mode 100644 index bc3fee7d9f3eaa23722071a6fcc44c68f2503f7d..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen_0.5-mdpo_0.5_7e-7-3ep_0alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen_0.5-MDPO_0.5_7e-7-3ep_0alp_0lam", - "id": "JayHyeon/Qwen_0.5-MDPO_0.5_7e-7-3ep_0alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen_0.5-MDPO_0.5_7e-7-3ep_0alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2524 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.313 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0446 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.271 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3289 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1564 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen_0.5-mdpo_0.7_3e-6-3ep_0alp_0lam.json b/data/models/jayhyeon_qwen_0.5-mdpo_0.7_3e-6-3ep_0alp_0lam.json deleted file mode 100644 index e87ec993b41fd6a8c9169cce3b548dce9a5349db..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen_0.5-mdpo_0.7_3e-6-3ep_0alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen_0.5-MDPO_0.7_3e-6-3ep_0alp_0lam", - "id": "JayHyeon/Qwen_0.5-MDPO_0.7_3e-6-3ep_0alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen_0.5-MDPO_0.7_3e-6-3ep_0alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2514 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3221 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0438 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2752 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3315 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1538 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen_0.5-mdpo_0.7_5e-7-3ep_0alp_0lam.json b/data/models/jayhyeon_qwen_0.5-mdpo_0.7_5e-7-3ep_0alp_0lam.json deleted file mode 100644 index 4b4077da3a428c6ca0c54280fdbdc978011630d9..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen_0.5-mdpo_0.7_5e-7-3ep_0alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen_0.5-MDPO_0.7_5e-7-3ep_0alp_0lam", - "id": "JayHyeon/Qwen_0.5-MDPO_0.7_5e-7-3ep_0alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen_0.5-MDPO_0.7_5e-7-3ep_0alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2457 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.318 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0385 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2668 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3275 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1572 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen_0.5-mdpo_0.9_5e-7-3ep_0alp_0lam.json b/data/models/jayhyeon_qwen_0.5-mdpo_0.9_5e-7-3ep_0alp_0lam.json deleted file mode 100644 index aeb6829eb37835520cde3207dfc2e7fd8e046b03..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen_0.5-mdpo_0.9_5e-7-3ep_0alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen_0.5-MDPO_0.9_5e-7-3ep_0alp_0lam", - "id": "JayHyeon/Qwen_0.5-MDPO_0.9_5e-7-3ep_0alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen_0.5-MDPO_0.9_5e-7-3ep_0alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2636 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3181 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0476 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2659 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3235 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1574 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen_0.5-rdpo_3e-6-1ep_0vpo_const_0.1.json b/data/models/jayhyeon_qwen_0.5-rdpo_3e-6-1ep_0vpo_const_0.1.json deleted file mode 100644 index c25443cfb36f25f6cd3ab5fe5f46885dc9862a5c..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen_0.5-rdpo_3e-6-1ep_0vpo_const_0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen_0.5-rDPO_3e-6-1ep_0vpo_const_0.1", - "id": "JayHyeon/Qwen_0.5-rDPO_3e-6-1ep_0vpo_const_0.1", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen_0.5-rDPO_3e-6-1ep_0vpo_const_0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2321 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3278 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0476 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2576 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3022 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1496 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen_0.5-rdpo_5e-7-3ep_0vpo_const_0.1.json b/data/models/jayhyeon_qwen_0.5-rdpo_5e-7-3ep_0vpo_const_0.1.json deleted file mode 100644 index 92fbc7f4a94c4123d0102e91ff005840e3239fc4..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen_0.5-rdpo_5e-7-3ep_0vpo_const_0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen_0.5-rDPO_5e-7-3ep_0vpo_const_0.1", - "id": "JayHyeon/Qwen_0.5-rDPO_5e-7-3ep_0vpo_const_0.1", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen_0.5-rDPO_5e-7-3ep_0vpo_const_0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2542 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3253 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0529 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.271 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3181 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1609 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen_0.5-rdpo_5e-7-3ep_0vpo_const_0.3.json b/data/models/jayhyeon_qwen_0.5-rdpo_5e-7-3ep_0vpo_const_0.3.json deleted file mode 100644 index ed1cfae85db24e9f8eb036242539d814196f1991..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen_0.5-rdpo_5e-7-3ep_0vpo_const_0.3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen_0.5-rDPO_5e-7-3ep_0vpo_const_0.3", - "id": "JayHyeon/Qwen_0.5-rDPO_5e-7-3ep_0vpo_const_0.3", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen_0.5-rDPO_5e-7-3ep_0vpo_const_0.3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2739 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3245 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0461 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2508 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3089 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1597 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen_0.5-vdpo_3e-6-1ep_3vpo_const.json b/data/models/jayhyeon_qwen_0.5-vdpo_3e-6-1ep_3vpo_const.json deleted file mode 100644 index 1ec3e3d0c1404b3e3b47e740116db8d1c90d66be..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen_0.5-vdpo_3e-6-1ep_3vpo_const.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen_0.5-VDPO_3e-6-1ep_3vpo_const", - "id": "JayHyeon/Qwen_0.5-VDPO_3e-6-1ep_3vpo_const", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen_0.5-VDPO_3e-6-1ep_3vpo_const/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2483 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3174 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0378 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2542 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3328 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1558 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen_0.5-vdpo_5e-7-1ep_0alp_0lam.json b/data/models/jayhyeon_qwen_0.5-vdpo_5e-7-1ep_0alp_0lam.json deleted file mode 100644 index 311e0a7b68139bd68b4a01beb879da800f94831c..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen_0.5-vdpo_5e-7-1ep_0alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen_0.5-VDPO_5e-7-1ep_0alp_0lam", - "id": "JayHyeon/Qwen_0.5-VDPO_5e-7-1ep_0alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen_0.5-VDPO_5e-7-1ep_0alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2518 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3218 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0529 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2718 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3235 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1595 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen_0.5-vdpo_5e-7-1ep_10vpo_const.json b/data/models/jayhyeon_qwen_0.5-vdpo_5e-7-1ep_10vpo_const.json deleted file mode 100644 index 0a74c8b7d28750363ed9ae646a31ff1befce744e..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen_0.5-vdpo_5e-7-1ep_10vpo_const.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen_0.5-VDPO_5e-7-1ep_10vpo_const", - "id": "JayHyeon/Qwen_0.5-VDPO_5e-7-1ep_10vpo_const", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen_0.5-VDPO_5e-7-1ep_10vpo_const/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2536 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3234 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0491 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.276 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3236 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1597 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen_0.5-vdpo_5e-7-1ep_1vpo_const.json b/data/models/jayhyeon_qwen_0.5-vdpo_5e-7-1ep_1vpo_const.json deleted file mode 100644 index e05f9c5c0f410cfb1004ffb1d8b919ef7834cfb0..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen_0.5-vdpo_5e-7-1ep_1vpo_const.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen_0.5-VDPO_5e-7-1ep_1vpo_const", - "id": "JayHyeon/Qwen_0.5-VDPO_5e-7-1ep_1vpo_const", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen_0.5-VDPO_5e-7-1ep_1vpo_const/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2448 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.324 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0604 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2752 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3249 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1587 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen_0.5-vdpo_5e-7-1ep_3vpo_const.json b/data/models/jayhyeon_qwen_0.5-vdpo_5e-7-1ep_3vpo_const.json deleted file mode 100644 index 195ce03fcb876eaf414f4d6ad456b4a431ce3723..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen_0.5-vdpo_5e-7-1ep_3vpo_const.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen_0.5-VDPO_5e-7-1ep_3vpo_const", - "id": "JayHyeon/Qwen_0.5-VDPO_5e-7-1ep_3vpo_const", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen_0.5-VDPO_5e-7-1ep_3vpo_const/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2505 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3227 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0468 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.271 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3209 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1589 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen_0.5-vdpo_5e-7-3ep_0alp_0lam.json b/data/models/jayhyeon_qwen_0.5-vdpo_5e-7-3ep_0alp_0lam.json deleted file mode 100644 index d0b66a8807f3bd5fcfabfbbf0c289aa3e72cbc39..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen_0.5-vdpo_5e-7-3ep_0alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen_0.5-VDPO_5e-7-3ep_0alp_0lam", - "id": "JayHyeon/Qwen_0.5-VDPO_5e-7-3ep_0alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen_0.5-VDPO_5e-7-3ep_0alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2472 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3255 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0498 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2752 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3208 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1587 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen_0.5-vdpo_5e-7-3ep_1vpo_const.json b/data/models/jayhyeon_qwen_0.5-vdpo_5e-7-3ep_1vpo_const.json deleted file mode 100644 index 8839410b3162438cf3d47ba23c94f6ee172a02e3..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen_0.5-vdpo_5e-7-3ep_1vpo_const.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen_0.5-VDPO_5e-7-3ep_1vpo_const", - "id": "JayHyeon/Qwen_0.5-VDPO_5e-7-3ep_1vpo_const", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen_0.5-VDPO_5e-7-3ep_1vpo_const/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2417 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3256 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0582 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2727 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3275 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1562 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen_0.5-vdpo_5e-7-3ep_3vpo_const.json b/data/models/jayhyeon_qwen_0.5-vdpo_5e-7-3ep_3vpo_const.json deleted file mode 100644 index 9ac378d95744af4ec0db66ca45503a344e42931e..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen_0.5-vdpo_5e-7-3ep_3vpo_const.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen_0.5-VDPO_5e-7-3ep_3vpo_const", - "id": "JayHyeon/Qwen_0.5-VDPO_5e-7-3ep_3vpo_const", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen_0.5-VDPO_5e-7-3ep_3vpo_const/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2527 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3235 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0536 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2785 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3235 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.158 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen_0.5-vipo_5e-7-1ep_0alp_0lam.json b/data/models/jayhyeon_qwen_0.5-vipo_5e-7-1ep_0alp_0lam.json deleted file mode 100644 index d9b74f5d249e522b2ce5de7975eb275a4e1519f7..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen_0.5-vipo_5e-7-1ep_0alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen_0.5-VIPO_5e-7-1ep_0alp_0lam", - "id": "JayHyeon/Qwen_0.5-VIPO_5e-7-1ep_0alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen_0.5-VIPO_5e-7-1ep_0alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2669 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3314 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.071 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2676 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3168 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1634 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen_0.5-vipo_5e-7-1ep_10vpo_const.json b/data/models/jayhyeon_qwen_0.5-vipo_5e-7-1ep_10vpo_const.json deleted file mode 100644 index b4777f7233f727d1c882e47ce18157c2f0adc824..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen_0.5-vipo_5e-7-1ep_10vpo_const.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen_0.5-VIPO_5e-7-1ep_10vpo_const", - "id": "JayHyeon/Qwen_0.5-VIPO_5e-7-1ep_10vpo_const", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen_0.5-VIPO_5e-7-1ep_10vpo_const/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2702 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.33 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.074 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2752 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3208 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1635 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen_0.5-vipo_5e-7-1ep_1vpo_const.json b/data/models/jayhyeon_qwen_0.5-vipo_5e-7-1ep_1vpo_const.json deleted file mode 100644 index a886f5a833116571072a3f14663f96c3fb12b8df..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen_0.5-vipo_5e-7-1ep_1vpo_const.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen_0.5-VIPO_5e-7-1ep_1vpo_const", - "id": "JayHyeon/Qwen_0.5-VIPO_5e-7-1ep_1vpo_const", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen_0.5-VIPO_5e-7-1ep_1vpo_const/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.248 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3309 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.068 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2643 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3208 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1649 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen_0.5-vipo_5e-7-1ep_30vpo_const.json b/data/models/jayhyeon_qwen_0.5-vipo_5e-7-1ep_30vpo_const.json deleted file mode 100644 index 66df6b43ac9e6f0861d2e69099f9af26e93c328e..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen_0.5-vipo_5e-7-1ep_30vpo_const.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen_0.5-VIPO_5e-7-1ep_30vpo_const", - "id": "JayHyeon/Qwen_0.5-VIPO_5e-7-1ep_30vpo_const", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen_0.5-VIPO_5e-7-1ep_30vpo_const/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2622 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3282 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.074 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2693 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3221 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1634 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen_0.5-vipo_5e-7-1ep_3vpo_const.json b/data/models/jayhyeon_qwen_0.5-vipo_5e-7-1ep_3vpo_const.json deleted file mode 100644 index eaa96e0fd83e6b634ea699a887133f9b0ec7cbd7..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen_0.5-vipo_5e-7-1ep_3vpo_const.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen_0.5-VIPO_5e-7-1ep_3vpo_const", - "id": "JayHyeon/Qwen_0.5-VIPO_5e-7-1ep_3vpo_const", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen_0.5-VIPO_5e-7-1ep_3vpo_const/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2609 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3298 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.065 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2701 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3168 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1651 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen_0.5-vipo_5e-7-3ep_0alp_0lam.json b/data/models/jayhyeon_qwen_0.5-vipo_5e-7-3ep_0alp_0lam.json deleted file mode 100644 index 3e487764772302aaac25896c6f9dfc8d451499d9..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen_0.5-vipo_5e-7-3ep_0alp_0lam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen_0.5-VIPO_5e-7-3ep_0alp_0lam", - "id": "JayHyeon/Qwen_0.5-VIPO_5e-7-3ep_0alp_0lam", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen_0.5-VIPO_5e-7-3ep_0alp_0lam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.293 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.322 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0627 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2685 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3116 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1591 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen_0.5-vipo_5e-7-3ep_10vpo_const.json b/data/models/jayhyeon_qwen_0.5-vipo_5e-7-3ep_10vpo_const.json deleted file mode 100644 index 2f4377fe7f18b6e5a88e1ad66d4ade2331fc4b83..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen_0.5-vipo_5e-7-3ep_10vpo_const.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen_0.5-VIPO_5e-7-3ep_10vpo_const", - "id": "JayHyeon/Qwen_0.5-VIPO_5e-7-3ep_10vpo_const", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen_0.5-VIPO_5e-7-3ep_10vpo_const/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2881 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3255 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0725 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2752 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3102 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1582 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen_0.5-vipo_5e-7-3ep_1vpo_const.json b/data/models/jayhyeon_qwen_0.5-vipo_5e-7-3ep_1vpo_const.json deleted file mode 100644 index 62c9b37d7a89302a6ff1631124ac27e07dd6a8e5..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen_0.5-vipo_5e-7-3ep_1vpo_const.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen_0.5-VIPO_5e-7-3ep_1vpo_const", - "id": "JayHyeon/Qwen_0.5-VIPO_5e-7-3ep_1vpo_const", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen_0.5-VIPO_5e-7-3ep_1vpo_const/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2887 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3237 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0748 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2802 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3142 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1609 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen_0.5-vipo_5e-7-3ep_30vpo_const.json b/data/models/jayhyeon_qwen_0.5-vipo_5e-7-3ep_30vpo_const.json deleted file mode 100644 index 400f46449875d2e69345274bbc5ab43cebc0b7f7..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen_0.5-vipo_5e-7-3ep_30vpo_const.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen_0.5-VIPO_5e-7-3ep_30vpo_const", - "id": "JayHyeon/Qwen_0.5-VIPO_5e-7-3ep_30vpo_const", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen_0.5-VIPO_5e-7-3ep_30vpo_const/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2905 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3254 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.077 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2735 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3129 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1574 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jayhyeon_qwen_0.5-vipo_5e-7-3ep_3vpo_const.json b/data/models/jayhyeon_qwen_0.5-vipo_5e-7-3ep_3vpo_const.json deleted file mode 100644 index bbff235555b3897473736f9c5e4671931c6cbbb9..0000000000000000000000000000000000000000 --- a/data/models/jayhyeon_qwen_0.5-vipo_5e-7-3ep_3vpo_const.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen_0.5-VIPO_5e-7-3ep_3vpo_const", - "id": "JayHyeon/Qwen_0.5-VIPO_5e-7-3ep_3vpo_const", - "developer": "JayHyeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JayHyeon_Qwen_0.5-VIPO_5e-7-3ep_3vpo_const/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2905 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3238 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0702 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2735 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3089 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1592 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jeanmichela_o-distil-qwen.json b/data/models/jeanmichela_o-distil-qwen.json deleted file mode 100644 index 35f30c8443a86a5801699395bbaa42f5abd1eec9..0000000000000000000000000000000000000000 --- a/data/models/jeanmichela_o-distil-qwen.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "o-distil-qwen", - "id": "jeanmichela/o-distil-qwen", - "developer": "jeanmichela", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jeanmichela_o-distil-qwen/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4482 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.59 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.565 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3935 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.534 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4658 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jebcarter_psyonic-cetacean-20b.json b/data/models/jebcarter_psyonic-cetacean-20b.json deleted file mode 100644 index a562c68672a5ddd61185619c74c34994fcc3532d..0000000000000000000000000000000000000000 --- a/data/models/jebcarter_psyonic-cetacean-20b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "psyonic-cetacean-20B", - "id": "jebcarter/psyonic-cetacean-20B", - "developer": "jebcarter", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "19.994" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jebcarter_psyonic-cetacean-20B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2544 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4907 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0181 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2735 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4661 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2886 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jebish7_aya-expanse-8b.json b/data/models/jebish7_aya-expanse-8b.json deleted file mode 100644 index 0c18cbc80b2c6ca389bfb181979e69d8b151e1e7..0000000000000000000000000000000000000000 --- a/data/models/jebish7_aya-expanse-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "aya-expanse-8b", - "id": "jebish7/aya-expanse-8b", - "developer": "jebish7", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "CohereForCausalLM", - "params_billions": "8.028" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jebish7_aya-expanse-8b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3791 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4969 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0816 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2836 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3869 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3103 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jebish7_gemma-2-2b-it.json b/data/models/jebish7_gemma-2-2b-it.json deleted file mode 100644 index a40b6ccebb6b8c504bdf98aa05157e5b56d9ffa4..0000000000000000000000000000000000000000 --- a/data/models/jebish7_gemma-2-2b-it.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "gemma-2-2b-it", - "id": "jebish7/gemma-2-2b-it", - "developer": "jebish7", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "2.614" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jebish7_gemma-2-2b-it/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1272 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4395 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.034 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.297 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4244 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2715 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jebish7_gemma-2-9b-it.json b/data/models/jebish7_gemma-2-9b-it.json deleted file mode 100644 index 7ec11bf680237f81ce9b87e24db1b6f35c1c53c3..0000000000000000000000000000000000000000 --- a/data/models/jebish7_gemma-2-9b-it.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "gemma-2-9b-it", - "id": "jebish7/gemma-2-9b-it", - "developer": "jebish7", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "9.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jebish7_gemma-2-9b-it/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1557 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5949 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0846 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3473 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4554 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4143 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jebish7_llama-3-nanda-10b-chat.json b/data/models/jebish7_llama-3-nanda-10b-chat.json deleted file mode 100644 index 059b16660c4c1abd7ab85ecd10ba98cc18045d28..0000000000000000000000000000000000000000 --- a/data/models/jebish7_llama-3-nanda-10b-chat.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-Nanda-10B-Chat", - "id": "jebish7/Llama-3-Nanda-10B-Chat", - "developer": "jebish7", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "9.985" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jebish7_Llama-3-Nanda-10B-Chat/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2953 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4959 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0559 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3012 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4356 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3157 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jebish7_llama-3.1-8b-instruct.json b/data/models/jebish7_llama-3.1-8b-instruct.json deleted file mode 100644 index f5a8a52f7e849a65193f65770e69677a1b49b4e5..0000000000000000000000000000000000000000 --- a/data/models/jebish7_llama-3.1-8b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.1-8B-Instruct", - "id": "jebish7/Llama-3.1-8B-Instruct", - "developer": "jebish7", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jebish7_Llama-3.1-8B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5058 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5088 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1548 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3213 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3998 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3777 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jebish7_nemotron-4-mini-hindi-4b-base.json b/data/models/jebish7_nemotron-4-mini-hindi-4b-base.json deleted file mode 100644 index d962990296460a92937b257fec7099c029c3788c..0000000000000000000000000000000000000000 --- a/data/models/jebish7_nemotron-4-mini-hindi-4b-base.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Nemotron-4-Mini-Hindi-4B-Base", - "id": "jebish7/Nemotron-4-Mini-Hindi-4B-Base", - "developer": "jebish7", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "NemotronForCausalLM", - "params_billions": "4.191" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jebish7_Nemotron-4-Mini-Hindi-4B-Base/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2285 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3924 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0272 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2836 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4249 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2503 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jebish7_nemotron-4-mini-hindi-4b-instruct.json b/data/models/jebish7_nemotron-4-mini-hindi-4b-instruct.json deleted file mode 100644 index c996972e1a6ac25678f17439a1f7ef0735e6415a..0000000000000000000000000000000000000000 --- a/data/models/jebish7_nemotron-4-mini-hindi-4b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Nemotron-4-Mini-Hindi-4B-Instruct", - "id": "jebish7/Nemotron-4-Mini-Hindi-4B-Instruct", - "developer": "jebish7", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "NemotronForCausalLM", - "params_billions": "4.191" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jebish7_Nemotron-4-Mini-Hindi-4B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3345 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4041 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0287 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3087 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4153 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2595 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jebish7_nemotron-mini-4b-instruct.json b/data/models/jebish7_nemotron-mini-4b-instruct.json deleted file mode 100644 index 9dd3c07987b46851e5768cdf78f78ad7b47d3e0f..0000000000000000000000000000000000000000 --- a/data/models/jebish7_nemotron-mini-4b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Nemotron-Mini-4B-Instruct", - "id": "jebish7/Nemotron-Mini-4B-Instruct", - "developer": "jebish7", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "NemotronForCausalLM", - "params_billions": "4.191" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jebish7_Nemotron-Mini-4B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3709 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4244 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0325 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.276 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4727 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2783 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jebish7_qwen2.5-0.5b-iha-hin.json b/data/models/jebish7_qwen2.5-0.5b-iha-hin.json deleted file mode 100644 index fe0c965e808055b27d547c505f0e45f5ada855ee..0000000000000000000000000000000000000000 --- a/data/models/jebish7_qwen2.5-0.5b-iha-hin.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "qwen2.5-0.5B-IHA-Hin", - "id": "jebish7/qwen2.5-0.5B-IHA-Hin", - "developer": "jebish7", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jebish7_qwen2.5-0.5B-IHA-Hin/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1416 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2989 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2525 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3475 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1094 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jeffmeloy_jeffmeloy_qwen2.5-7b-minperplexity-1.json b/data/models/jeffmeloy_jeffmeloy_qwen2.5-7b-minperplexity-1.json deleted file mode 100644 index 270396f64f52161cd9a2fca387c03f2b7409375f..0000000000000000000000000000000000000000 --- a/data/models/jeffmeloy_jeffmeloy_qwen2.5-7b-minperplexity-1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "jeffmeloy_Qwen2.5-7B-minperplexity-1", - "id": "jeffmeloy/jeffmeloy_Qwen2.5-7B-minperplexity-1", - "developer": "jeffmeloy", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jeffmeloy_jeffmeloy_Qwen2.5-7B-minperplexity-1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3757 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5582 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2915 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3322 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.429 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4368 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jeffmeloy_qwen-7b-nerd-uncensored-v1.0.json b/data/models/jeffmeloy_qwen-7b-nerd-uncensored-v1.0.json deleted file mode 100644 index 8d7c691a9f3a6b4b1bd31ab957b1a185e2d4a08f..0000000000000000000000000000000000000000 --- a/data/models/jeffmeloy_qwen-7b-nerd-uncensored-v1.0.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen-7B-nerd-uncensored-v1.0", - "id": "jeffmeloy/Qwen-7B-nerd-uncensored-v1.0", - "developer": "jeffmeloy", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jeffmeloy_Qwen-7B-nerd-uncensored-v1.0/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6136 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5421 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.287 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.328 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4793 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4363 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jeffmeloy_qwen2.5-7b-minperplexity-2.json b/data/models/jeffmeloy_qwen2.5-7b-minperplexity-2.json deleted file mode 100644 index 4a82e8026c712b2cd104d723628f6744125222a0..0000000000000000000000000000000000000000 --- a/data/models/jeffmeloy_qwen2.5-7b-minperplexity-2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-7B-minperplexity-2", - "id": "jeffmeloy/Qwen2.5-7B-minperplexity-2", - "developer": "jeffmeloy", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jeffmeloy_Qwen2.5-7B-minperplexity-2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5097 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5524 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3014 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3112 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4625 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4346 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jeffmeloy_qwen2.5-7b-nerd-uncensored-v0.9.json b/data/models/jeffmeloy_qwen2.5-7b-nerd-uncensored-v0.9.json deleted file mode 100644 index c77fc0523bc4cccf7e9059246357c3d416cadc88..0000000000000000000000000000000000000000 --- a/data/models/jeffmeloy_qwen2.5-7b-nerd-uncensored-v0.9.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-7B-nerd-uncensored-v0.9", - "id": "jeffmeloy/Qwen2.5-7B-nerd-uncensored-v0.9", - "developer": "jeffmeloy", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jeffmeloy_Qwen2.5-7B-nerd-uncensored-v0.9/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6048 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.547 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2946 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.323 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.482 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4363 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jeffmeloy_qwen2.5-7b-nerd-uncensored-v1.0.json b/data/models/jeffmeloy_qwen2.5-7b-nerd-uncensored-v1.0.json deleted file mode 100644 index 442697bdcbeb36664b26bc4f74e374ef4ce58e79..0000000000000000000000000000000000000000 --- a/data/models/jeffmeloy_qwen2.5-7b-nerd-uncensored-v1.0.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-7B-nerd-uncensored-v1.0", - "id": "jeffmeloy/Qwen2.5-7B-nerd-uncensored-v1.0", - "developer": "jeffmeloy", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jeffmeloy_Qwen2.5-7B-nerd-uncensored-v1.0/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7695 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5418 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4713 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2903 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4551 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4254 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jeffmeloy_qwen2.5-7b-nerd-uncensored-v1.1.json b/data/models/jeffmeloy_qwen2.5-7b-nerd-uncensored-v1.1.json deleted file mode 100644 index fc74298c6cc661d5f5f9826026dbf1a2ac9063ed..0000000000000000000000000000000000000000 --- a/data/models/jeffmeloy_qwen2.5-7b-nerd-uncensored-v1.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-7B-nerd-uncensored-v1.1", - "id": "jeffmeloy/Qwen2.5-7B-nerd-uncensored-v1.1", - "developer": "jeffmeloy", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jeffmeloy_Qwen2.5-7B-nerd-uncensored-v1.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6626 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4864 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1329 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2869 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3843 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.385 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jeffmeloy_qwen2.5-7b-nerd-uncensored-v1.2.json b/data/models/jeffmeloy_qwen2.5-7b-nerd-uncensored-v1.2.json deleted file mode 100644 index 7e59c4bfa6f657f5938ead9ac2caf61155e0957e..0000000000000000000000000000000000000000 --- a/data/models/jeffmeloy_qwen2.5-7b-nerd-uncensored-v1.2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-7B-nerd-uncensored-v1.2", - "id": "jeffmeloy/Qwen2.5-7B-nerd-uncensored-v1.2", - "developer": "jeffmeloy", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jeffmeloy_Qwen2.5-7B-nerd-uncensored-v1.2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4965 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4946 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1208 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3037 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4172 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3969 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jeffmeloy_qwen2.5-7b-nerd-uncensored-v1.3.json b/data/models/jeffmeloy_qwen2.5-7b-nerd-uncensored-v1.3.json deleted file mode 100644 index f80851a15adb5ab93c27b08a7db00f4fe683331f..0000000000000000000000000000000000000000 --- a/data/models/jeffmeloy_qwen2.5-7b-nerd-uncensored-v1.3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-7B-nerd-uncensored-v1.3", - "id": "jeffmeloy/Qwen2.5-7B-nerd-uncensored-v1.3", - "developer": "jeffmeloy", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jeffmeloy_Qwen2.5-7B-nerd-uncensored-v1.3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4995 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5026 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1231 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3129 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4187 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4016 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jeffmeloy_qwen2.5-7b-nerd-uncensored-v1.4.json b/data/models/jeffmeloy_qwen2.5-7b-nerd-uncensored-v1.4.json deleted file mode 100644 index ce4b56085d12f464829f4e93ac4c17dbaf3e77de..0000000000000000000000000000000000000000 --- a/data/models/jeffmeloy_qwen2.5-7b-nerd-uncensored-v1.4.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-7B-nerd-uncensored-v1.4", - "id": "jeffmeloy/Qwen2.5-7B-nerd-uncensored-v1.4", - "developer": "jeffmeloy", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jeffmeloy_Qwen2.5-7B-nerd-uncensored-v1.4/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6079 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5467 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.281 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3238 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4714 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4419 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jeffmeloy_qwen2.5-7b-nerd-uncensored-v1.5.json b/data/models/jeffmeloy_qwen2.5-7b-nerd-uncensored-v1.5.json deleted file mode 100644 index b97587ad5f46d368615d7738c9e828abbefac057..0000000000000000000000000000000000000000 --- a/data/models/jeffmeloy_qwen2.5-7b-nerd-uncensored-v1.5.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-7B-nerd-uncensored-v1.5", - "id": "jeffmeloy/Qwen2.5-7B-nerd-uncensored-v1.5", - "developer": "jeffmeloy", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jeffmeloy_Qwen2.5-7B-nerd-uncensored-v1.5/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.565 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5523 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2757 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3272 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4982 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4448 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jeffmeloy_qwen2.5-7b-nerd-uncensored-v1.7.json b/data/models/jeffmeloy_qwen2.5-7b-nerd-uncensored-v1.7.json deleted file mode 100644 index dbf9cd8aa9f9368f5c1790533d89331b3fd0d3dc..0000000000000000000000000000000000000000 --- a/data/models/jeffmeloy_qwen2.5-7b-nerd-uncensored-v1.7.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-7B-nerd-uncensored-v1.7", - "id": "jeffmeloy/Qwen2.5-7B-nerd-uncensored-v1.7", - "developer": "jeffmeloy", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jeffmeloy_Qwen2.5-7B-nerd-uncensored-v1.7/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4202 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5392 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2915 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3238 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4848 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.428 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jeffmeloy_qwen2.5-7b-nerd-uncensored-v1.8.json b/data/models/jeffmeloy_qwen2.5-7b-nerd-uncensored-v1.8.json deleted file mode 100644 index 2e42973143add01e3a64662aae861b26ac423810..0000000000000000000000000000000000000000 --- a/data/models/jeffmeloy_qwen2.5-7b-nerd-uncensored-v1.8.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-7B-nerd-uncensored-v1.8", - "id": "jeffmeloy/Qwen2.5-7B-nerd-uncensored-v1.8", - "developer": "jeffmeloy", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jeffmeloy_Qwen2.5-7B-nerd-uncensored-v1.8/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6256 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5447 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2704 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3238 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4767 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4343 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jeffmeloy_qwen2.5-7b-olm-v1.0.json b/data/models/jeffmeloy_qwen2.5-7b-olm-v1.0.json deleted file mode 100644 index 8d2f98320d792f1be7f15ca38e8fcd6eb7e82b17..0000000000000000000000000000000000000000 --- a/data/models/jeffmeloy_qwen2.5-7b-olm-v1.0.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-7B-olm-v1.0", - "id": "jeffmeloy/Qwen2.5-7B-olm-v1.0", - "developer": "jeffmeloy", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jeffmeloy_Qwen2.5-7B-olm-v1.0/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5331 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.566 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2863 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3205 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4278 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4566 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jeffmeloy_qwen2.5-7b-olm-v1.1.json b/data/models/jeffmeloy_qwen2.5-7b-olm-v1.1.json deleted file mode 100644 index 7ac7523a0d5bcbb0ed6130011ae07c0b5f301da0..0000000000000000000000000000000000000000 --- a/data/models/jeffmeloy_qwen2.5-7b-olm-v1.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-7B-olm-v1.1", - "id": "jeffmeloy/Qwen2.5-7B-olm-v1.1", - "developer": "jeffmeloy", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jeffmeloy_Qwen2.5-7B-olm-v1.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4329 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5478 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3829 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3087 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4808 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4354 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jeffmeloy_qwen2.5-7b-olm-v1.2.json b/data/models/jeffmeloy_qwen2.5-7b-olm-v1.2.json deleted file mode 100644 index 86bc98bc4258541cf6b33c8a025988825e0728bd..0000000000000000000000000000000000000000 --- a/data/models/jeffmeloy_qwen2.5-7b-olm-v1.2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-7B-olm-v1.2", - "id": "jeffmeloy/Qwen2.5-7B-olm-v1.2", - "developer": "jeffmeloy", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jeffmeloy_Qwen2.5-7B-olm-v1.2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4203 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5533 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2847 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3171 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4688 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4387 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jeffmeloy_qwen2.5-7b-olm-v1.3.json b/data/models/jeffmeloy_qwen2.5-7b-olm-v1.3.json deleted file mode 100644 index b23fbe165953d57ce8c8d7935fd1ec94d6e75450..0000000000000000000000000000000000000000 --- a/data/models/jeffmeloy_qwen2.5-7b-olm-v1.3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-7B-olm-v1.3", - "id": "jeffmeloy/Qwen2.5-7B-olm-v1.3", - "developer": "jeffmeloy", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jeffmeloy_Qwen2.5-7B-olm-v1.3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4219 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5532 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3104 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3213 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4701 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.447 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jeffmeloy_qwen2.5-7b-olm-v1.4.json b/data/models/jeffmeloy_qwen2.5-7b-olm-v1.4.json deleted file mode 100644 index 2423654a86af93c8dd3f69dcff34d7803fc4eb5a..0000000000000000000000000000000000000000 --- a/data/models/jeffmeloy_qwen2.5-7b-olm-v1.4.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-7B-olm-v1.4", - "id": "jeffmeloy/Qwen2.5-7B-olm-v1.4", - "developer": "jeffmeloy", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jeffmeloy_Qwen2.5-7B-olm-v1.4/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4545 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5582 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2923 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3121 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4622 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4457 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jeffmeloy_qwen2.5-7b-olm-v1.5.json b/data/models/jeffmeloy_qwen2.5-7b-olm-v1.5.json deleted file mode 100644 index e44925abf4c94f2babafd1272e55c5a92aa94475..0000000000000000000000000000000000000000 --- a/data/models/jeffmeloy_qwen2.5-7b-olm-v1.5.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-7B-olm-v1.5", - "id": "jeffmeloy/Qwen2.5-7B-olm-v1.5", - "developer": "jeffmeloy", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jeffmeloy_Qwen2.5-7B-olm-v1.5/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4547 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5544 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2817 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3398 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4539 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4399 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jeonsworld_carbonvillain-en-10.7b-v4.json b/data/models/jeonsworld_carbonvillain-en-10.7b-v4.json deleted file mode 100644 index 618ac552c387fb41e72f3f19dd36e8bd030ba4d1..0000000000000000000000000000000000000000 --- a/data/models/jeonsworld_carbonvillain-en-10.7b-v4.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "CarbonVillain-en-10.7B-v4", - "id": "jeonsworld/CarbonVillain-en-10.7B-v4", - "developer": "jeonsworld", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "10.732" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jeonsworld_CarbonVillain-en-10.7B-v4/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4579 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5168 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0468 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3062 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3965 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3142 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jiangxinyang-shanda_homer-llama3-8b.json b/data/models/jiangxinyang-shanda_homer-llama3-8b.json deleted file mode 100644 index 19c18d59fc1637370ce05b904e672abb554c7853..0000000000000000000000000000000000000000 --- a/data/models/jiangxinyang-shanda_homer-llama3-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Homer-LLama3-8B", - "id": "jiangxinyang-shanda/Homer-LLama3-8B", - "developer": "jiangxinyang-shanda", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jiangxinyang-shanda_Homer-LLama3-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3992 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5173 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0861 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.297 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4056 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3139 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jieliu_storm-7b.json b/data/models/jieliu_storm-7b.json deleted file mode 100644 index 78bf9e67e59a5272cd6171685923321c96a165a2..0000000000000000000000000000000000000000 --- a/data/models/jieliu_storm-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Storm-7B", - "id": "jieliu/Storm-7B", - "developer": "jieliu", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jieliu_Storm-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3424 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5187 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0612 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3079 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4429 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3119 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jimmy19991222_llama-3-8b-instruct-gapo-v2-bert-f1-beta10-gamma0.3-lr1.0e-6-1minus-rerun.json b/data/models/jimmy19991222_llama-3-8b-instruct-gapo-v2-bert-f1-beta10-gamma0.3-lr1.0e-6-1minus-rerun.json deleted file mode 100644 index 0fcdba1a7c227c192f850d41bc1e1c0346eb0a83..0000000000000000000000000000000000000000 --- a/data/models/jimmy19991222_llama-3-8b-instruct-gapo-v2-bert-f1-beta10-gamma0.3-lr1.0e-6-1minus-rerun.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "llama-3-8b-instruct-gapo-v2-bert-f1-beta10-gamma0.3-lr1.0e-6-1minus-rerun", - "id": "Jimmy19991222/llama-3-8b-instruct-gapo-v2-bert-f1-beta10-gamma0.3-lr1.0e-6-1minus-rerun", - "developer": "Jimmy19991222", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Jimmy19991222_llama-3-8b-instruct-gapo-v2-bert-f1-beta10-gamma0.3-lr1.0e-6-1minus-rerun/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6717 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.488 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0604 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2945 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4041 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3634 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jimmy19991222_llama-3-8b-instruct-gapo-v2-bert_f1-beta10-gamma0.3-lr1.0e-6-scale-log.json b/data/models/jimmy19991222_llama-3-8b-instruct-gapo-v2-bert_f1-beta10-gamma0.3-lr1.0e-6-scale-log.json deleted file mode 100644 index 3e8b3fa4929af1d5bf8a9f38cc25f1e82a0e0039..0000000000000000000000000000000000000000 --- a/data/models/jimmy19991222_llama-3-8b-instruct-gapo-v2-bert_f1-beta10-gamma0.3-lr1.0e-6-scale-log.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "llama-3-8b-instruct-gapo-v2-bert_f1-beta10-gamma0.3-lr1.0e-6-scale-log", - "id": "Jimmy19991222/llama-3-8b-instruct-gapo-v2-bert_f1-beta10-gamma0.3-lr1.0e-6-scale-log", - "developer": "Jimmy19991222", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Jimmy19991222_llama-3-8b-instruct-gapo-v2-bert_f1-beta10-gamma0.3-lr1.0e-6-scale-log/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6556 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4935 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0544 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3045 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3658 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jimmy19991222_llama-3-8b-instruct-gapo-v2-bert_p-beta10-gamma0.3-lr1.0e-6-scale-log.json b/data/models/jimmy19991222_llama-3-8b-instruct-gapo-v2-bert_p-beta10-gamma0.3-lr1.0e-6-scale-log.json deleted file mode 100644 index a7107c9cf1078a50ca59a3871a6d9f2a21035a0c..0000000000000000000000000000000000000000 --- a/data/models/jimmy19991222_llama-3-8b-instruct-gapo-v2-bert_p-beta10-gamma0.3-lr1.0e-6-scale-log.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "llama-3-8b-instruct-gapo-v2-bert_p-beta10-gamma0.3-lr1.0e-6-scale-log", - "id": "Jimmy19991222/llama-3-8b-instruct-gapo-v2-bert_p-beta10-gamma0.3-lr1.0e-6-scale-log", - "developer": "Jimmy19991222", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Jimmy19991222_llama-3-8b-instruct-gapo-v2-bert_p-beta10-gamma0.3-lr1.0e-6-scale-log/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6315 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4916 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.065 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2861 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3935 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3611 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jimmy19991222_llama-3-8b-instruct-gapo-v2-bleu-beta0.1-no-length-scale-gamma0.4.json b/data/models/jimmy19991222_llama-3-8b-instruct-gapo-v2-bleu-beta0.1-no-length-scale-gamma0.4.json deleted file mode 100644 index 0aa5eae922986e9688e8ebac9b2ac709243ea306..0000000000000000000000000000000000000000 --- a/data/models/jimmy19991222_llama-3-8b-instruct-gapo-v2-bleu-beta0.1-no-length-scale-gamma0.4.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "llama-3-8b-instruct-gapo-v2-bleu-beta0.1-no-length-scale-gamma0.4", - "id": "Jimmy19991222/llama-3-8b-instruct-gapo-v2-bleu-beta0.1-no-length-scale-gamma0.4", - "developer": "Jimmy19991222", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Jimmy19991222_llama-3-8b-instruct-gapo-v2-bleu-beta0.1-no-length-scale-gamma0.4/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6285 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4986 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0514 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2928 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4014 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3545 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jimmy19991222_llama-3-8b-instruct-gapo-v2-rouge2-beta10-1minus-gamma0.3-rerun.json b/data/models/jimmy19991222_llama-3-8b-instruct-gapo-v2-rouge2-beta10-1minus-gamma0.3-rerun.json deleted file mode 100644 index bab752f29be57daa31d4d7500201cd1c957c0d9b..0000000000000000000000000000000000000000 --- a/data/models/jimmy19991222_llama-3-8b-instruct-gapo-v2-rouge2-beta10-1minus-gamma0.3-rerun.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "llama-3-8b-instruct-gapo-v2-rouge2-beta10-1minus-gamma0.3-rerun", - "id": "Jimmy19991222/llama-3-8b-instruct-gapo-v2-rouge2-beta10-1minus-gamma0.3-rerun", - "developer": "Jimmy19991222", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Jimmy19991222_llama-3-8b-instruct-gapo-v2-rouge2-beta10-1minus-gamma0.3-rerun/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6678 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.494 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0612 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3062 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3987 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3658 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jimmy19991222_llama-3-8b-instruct-gapo-v2-rouge2-beta10-gamma0.3-lr1.0e-6-scale-log.json b/data/models/jimmy19991222_llama-3-8b-instruct-gapo-v2-rouge2-beta10-gamma0.3-lr1.0e-6-scale-log.json deleted file mode 100644 index a1a8146b4ccb8fa4ecab83512371eabefd06cbdb..0000000000000000000000000000000000000000 --- a/data/models/jimmy19991222_llama-3-8b-instruct-gapo-v2-rouge2-beta10-gamma0.3-lr1.0e-6-scale-log.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "llama-3-8b-instruct-gapo-v2-rouge2-beta10-gamma0.3-lr1.0e-6-scale-log", - "id": "Jimmy19991222/llama-3-8b-instruct-gapo-v2-rouge2-beta10-gamma0.3-lr1.0e-6-scale-log", - "developer": "Jimmy19991222", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Jimmy19991222_llama-3-8b-instruct-gapo-v2-rouge2-beta10-gamma0.3-lr1.0e-6-scale-log/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6605 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4916 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0657 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3037 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3664 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jimmy19991222_llama-3-8b-instruct-gapo-v2-rougel-beta10-gamma0.3-lr1.0e-6-scale-log.json b/data/models/jimmy19991222_llama-3-8b-instruct-gapo-v2-rougel-beta10-gamma0.3-lr1.0e-6-scale-log.json deleted file mode 100644 index 0be8bf70b1868fce2d234bc89a09890e3f305989..0000000000000000000000000000000000000000 --- a/data/models/jimmy19991222_llama-3-8b-instruct-gapo-v2-rougel-beta10-gamma0.3-lr1.0e-6-scale-log.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "llama-3-8b-instruct-gapo-v2-rougeL-beta10-gamma0.3-lr1.0e-6-scale-log", - "id": "Jimmy19991222/llama-3-8b-instruct-gapo-v2-rougeL-beta10-gamma0.3-lr1.0e-6-scale-log", - "developer": "Jimmy19991222", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Jimmy19991222_llama-3-8b-instruct-gapo-v2-rougeL-beta10-gamma0.3-lr1.0e-6-scale-log/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6492 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4952 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0642 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.302 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3961 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3711 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jimmy19991222_llama-3-instruct-8b-simpo-v0.2.json b/data/models/jimmy19991222_llama-3-instruct-8b-simpo-v0.2.json deleted file mode 100644 index ca58baa5f4f2f932e623eb06bc3e2a202068bce3..0000000000000000000000000000000000000000 --- a/data/models/jimmy19991222_llama-3-instruct-8b-simpo-v0.2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-Instruct-8B-SimPO-v0.2", - "id": "Jimmy19991222/Llama-3-Instruct-8B-SimPO-v0.2", - "developer": "Jimmy19991222", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Jimmy19991222_Llama-3-Instruct-8B-SimPO-v0.2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.654 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4984 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0619 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3146 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4013 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3686 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jiviai_medx_v2.json b/data/models/jiviai_medx_v2.json deleted file mode 100644 index e5938c54dbb0a22f1888349f4a97d769bda181b7..0000000000000000000000000000000000000000 --- a/data/models/jiviai_medx_v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "medX_v2", - "id": "jiviai/medX_v2", - "developer": "jiviai", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jiviai_medX_v2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3743 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4509 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0544 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.323 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3498 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3428 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jlzhou_qwen2.5-3b-infinity-instruct-0625.json b/data/models/jlzhou_qwen2.5-3b-infinity-instruct-0625.json deleted file mode 100644 index 552b57bc48435d74221caf0048fbaad046beb5a9..0000000000000000000000000000000000000000 --- a/data/models/jlzhou_qwen2.5-3b-infinity-instruct-0625.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-3B-Infinity-Instruct-0625", - "id": "jlzhou/Qwen2.5-3B-Infinity-Instruct-0625", - "developer": "jlzhou", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.086" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jlzhou_Qwen2.5-3B-Infinity-Instruct-0625/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3558 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4774 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1367 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2693 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3981 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3199 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/johnsutor_llama-3-8b-instruct_breadcrumbs-density-0.1-gamma-0.01.json b/data/models/johnsutor_llama-3-8b-instruct_breadcrumbs-density-0.1-gamma-0.01.json deleted file mode 100644 index 77408793dc3e429979f1818a1d9a3fa7292ca7b3..0000000000000000000000000000000000000000 --- a/data/models/johnsutor_llama-3-8b-instruct_breadcrumbs-density-0.1-gamma-0.01.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-8B-Instruct_breadcrumbs-density-0.1-gamma-0.01", - "id": "johnsutor/Llama-3-8B-Instruct_breadcrumbs-density-0.1-gamma-0.01", - "developer": "johnsutor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/johnsutor_Llama-3-8B-Instruct_breadcrumbs-density-0.1-gamma-0.01/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4271 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5036 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0453 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3221 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4638 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3739 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/johnsutor_llama-3-8b-instruct_breadcrumbs-density-0.1-gamma-0.1.json b/data/models/johnsutor_llama-3-8b-instruct_breadcrumbs-density-0.1-gamma-0.1.json deleted file mode 100644 index c0e856596de39e44298d256e0df70cad5ad5f2e5..0000000000000000000000000000000000000000 --- a/data/models/johnsutor_llama-3-8b-instruct_breadcrumbs-density-0.1-gamma-0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-8B-Instruct_breadcrumbs-density-0.1-gamma-0.1", - "id": "johnsutor/Llama-3-8B-Instruct_breadcrumbs-density-0.1-gamma-0.1", - "developer": "johnsutor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/johnsutor_Llama-3-8B-Instruct_breadcrumbs-density-0.1-gamma-0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4253 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5019 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0967 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3012 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.415 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3724 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/johnsutor_llama-3-8b-instruct_breadcrumbs-density-0.3-gamma-0.01.json b/data/models/johnsutor_llama-3-8b-instruct_breadcrumbs-density-0.3-gamma-0.01.json deleted file mode 100644 index 5bfcac0071fe1419b3d172f111b6abb34722756e..0000000000000000000000000000000000000000 --- a/data/models/johnsutor_llama-3-8b-instruct_breadcrumbs-density-0.3-gamma-0.01.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-8B-Instruct_breadcrumbs-density-0.3-gamma-0.01", - "id": "johnsutor/Llama-3-8B-Instruct_breadcrumbs-density-0.3-gamma-0.01", - "developer": "johnsutor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/johnsutor_Llama-3-8B-Instruct_breadcrumbs-density-0.3-gamma-0.01/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3377 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4917 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0106 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3121 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5018 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3533 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/johnsutor_llama-3-8b-instruct_breadcrumbs-density-0.3-gamma-0.1.json b/data/models/johnsutor_llama-3-8b-instruct_breadcrumbs-density-0.3-gamma-0.1.json deleted file mode 100644 index a823450a058d0bf91327e66cb969566ab9b1c8e2..0000000000000000000000000000000000000000 --- a/data/models/johnsutor_llama-3-8b-instruct_breadcrumbs-density-0.3-gamma-0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-8B-Instruct_breadcrumbs-density-0.3-gamma-0.1", - "id": "johnsutor/Llama-3-8B-Instruct_breadcrumbs-density-0.3-gamma-0.1", - "developer": "johnsutor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/johnsutor_Llama-3-8B-Instruct_breadcrumbs-density-0.3-gamma-0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4274 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5126 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0808 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3087 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4226 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3739 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/johnsutor_llama-3-8b-instruct_breadcrumbs-density-0.5-gamma-0.01.json b/data/models/johnsutor_llama-3-8b-instruct_breadcrumbs-density-0.5-gamma-0.01.json deleted file mode 100644 index 3131e469ce1bff5e4ed763bde835092023e6e91e..0000000000000000000000000000000000000000 --- a/data/models/johnsutor_llama-3-8b-instruct_breadcrumbs-density-0.5-gamma-0.01.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-8B-Instruct_breadcrumbs-density-0.5-gamma-0.01", - "id": "johnsutor/Llama-3-8B-Instruct_breadcrumbs-density-0.5-gamma-0.01", - "developer": "johnsutor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/johnsutor_Llama-3-8B-Instruct_breadcrumbs-density-0.5-gamma-0.01/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3204 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4884 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0038 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.302 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5098 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3344 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/johnsutor_llama-3-8b-instruct_breadcrumbs-density-0.5-gamma-0.1.json b/data/models/johnsutor_llama-3-8b-instruct_breadcrumbs-density-0.5-gamma-0.1.json deleted file mode 100644 index c49d1ec0a44a76a454a70cd8fa626a6c68d7c5ee..0000000000000000000000000000000000000000 --- a/data/models/johnsutor_llama-3-8b-instruct_breadcrumbs-density-0.5-gamma-0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-8B-Instruct_breadcrumbs-density-0.5-gamma-0.1", - "id": "johnsutor/Llama-3-8B-Instruct_breadcrumbs-density-0.5-gamma-0.1", - "developer": "johnsutor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/johnsutor_Llama-3-8B-Instruct_breadcrumbs-density-0.5-gamma-0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4396 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.514 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0801 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.307 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4398 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3696 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/johnsutor_llama-3-8b-instruct_breadcrumbs-density-0.7-gamma-0.01.json b/data/models/johnsutor_llama-3-8b-instruct_breadcrumbs-density-0.7-gamma-0.01.json deleted file mode 100644 index bc188e7259b9493b38f6e9c2d267218a515ee713..0000000000000000000000000000000000000000 --- a/data/models/johnsutor_llama-3-8b-instruct_breadcrumbs-density-0.7-gamma-0.01.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-8B-Instruct_breadcrumbs-density-0.7-gamma-0.01", - "id": "johnsutor/Llama-3-8B-Instruct_breadcrumbs-density-0.7-gamma-0.01", - "developer": "johnsutor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/johnsutor_Llama-3-8B-Instruct_breadcrumbs-density-0.7-gamma-0.01/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2814 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4854 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0023 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2903 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5163 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3295 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/johnsutor_llama-3-8b-instruct_breadcrumbs-density-0.7-gamma-0.1.json b/data/models/johnsutor_llama-3-8b-instruct_breadcrumbs-density-0.7-gamma-0.1.json deleted file mode 100644 index 401bfa29964f95686e8a225843e890be5c2ea3f2..0000000000000000000000000000000000000000 --- a/data/models/johnsutor_llama-3-8b-instruct_breadcrumbs-density-0.7-gamma-0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-8B-Instruct_breadcrumbs-density-0.7-gamma-0.1", - "id": "johnsutor/Llama-3-8B-Instruct_breadcrumbs-density-0.7-gamma-0.1", - "developer": "johnsutor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/johnsutor_Llama-3-8B-Instruct_breadcrumbs-density-0.7-gamma-0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4302 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5157 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0627 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3079 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4332 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3663 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/johnsutor_llama-3-8b-instruct_breadcrumbs-density-0.9-gamma-0.01.json b/data/models/johnsutor_llama-3-8b-instruct_breadcrumbs-density-0.9-gamma-0.01.json deleted file mode 100644 index dcadd3c0858fd387e2d89e314e74687b8cfcf83e..0000000000000000000000000000000000000000 --- a/data/models/johnsutor_llama-3-8b-instruct_breadcrumbs-density-0.9-gamma-0.01.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-8B-Instruct_breadcrumbs-density-0.9-gamma-0.01", - "id": "johnsutor/Llama-3-8B-Instruct_breadcrumbs-density-0.9-gamma-0.01", - "developer": "johnsutor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/johnsutor_Llama-3-8B-Instruct_breadcrumbs-density-0.9-gamma-0.01/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.279 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4861 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0015 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2945 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.515 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3305 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/johnsutor_llama-3-8b-instruct_breadcrumbs-density-0.9-gamma-0.1.json b/data/models/johnsutor_llama-3-8b-instruct_breadcrumbs-density-0.9-gamma-0.1.json deleted file mode 100644 index 005ea975246fa8d448b7cf70239068b2cc4c38d9..0000000000000000000000000000000000000000 --- a/data/models/johnsutor_llama-3-8b-instruct_breadcrumbs-density-0.9-gamma-0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-8B-Instruct_breadcrumbs-density-0.9-gamma-0.1", - "id": "johnsutor/Llama-3-8B-Instruct_breadcrumbs-density-0.9-gamma-0.1", - "developer": "johnsutor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/johnsutor_Llama-3-8B-Instruct_breadcrumbs-density-0.9-gamma-0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4223 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5154 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.074 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3079 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4384 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.365 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/johnsutor_llama-3-8b-instruct_breadcrumbs_ties-density-0.1-gamma-0.01.json b/data/models/johnsutor_llama-3-8b-instruct_breadcrumbs_ties-density-0.1-gamma-0.01.json deleted file mode 100644 index a2f00a4370884c0b26cec3fc6d2dbc0e146c0e28..0000000000000000000000000000000000000000 --- a/data/models/johnsutor_llama-3-8b-instruct_breadcrumbs_ties-density-0.1-gamma-0.01.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-8B-Instruct_breadcrumbs_ties-density-0.1-gamma-0.01", - "id": "johnsutor/Llama-3-8B-Instruct_breadcrumbs_ties-density-0.1-gamma-0.01", - "developer": "johnsutor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/johnsutor_Llama-3-8B-Instruct_breadcrumbs_ties-density-0.1-gamma-0.01/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4359 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5041 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0483 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3104 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4532 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3762 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/johnsutor_llama-3-8b-instruct_breadcrumbs_ties-density-0.1-gamma-0.1.json b/data/models/johnsutor_llama-3-8b-instruct_breadcrumbs_ties-density-0.1-gamma-0.1.json deleted file mode 100644 index 8538781f5bb2e8c190490cb1cf98ada6786800ac..0000000000000000000000000000000000000000 --- a/data/models/johnsutor_llama-3-8b-instruct_breadcrumbs_ties-density-0.1-gamma-0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-8B-Instruct_breadcrumbs_ties-density-0.1-gamma-0.1", - "id": "johnsutor/Llama-3-8B-Instruct_breadcrumbs_ties-density-0.1-gamma-0.1", - "developer": "johnsutor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/johnsutor_Llama-3-8B-Instruct_breadcrumbs_ties-density-0.1-gamma-0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4202 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5011 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0982 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3003 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.415 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3699 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/johnsutor_llama-3-8b-instruct_breadcrumbs_ties-density-0.3-gamma-0.01.json b/data/models/johnsutor_llama-3-8b-instruct_breadcrumbs_ties-density-0.3-gamma-0.01.json deleted file mode 100644 index 485e3be673abb5bb4e774df3405d01ec6441c2eb..0000000000000000000000000000000000000000 --- a/data/models/johnsutor_llama-3-8b-instruct_breadcrumbs_ties-density-0.3-gamma-0.01.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-8B-Instruct_breadcrumbs_ties-density-0.3-gamma-0.01", - "id": "johnsutor/Llama-3-8B-Instruct_breadcrumbs_ties-density-0.3-gamma-0.01", - "developer": "johnsutor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/johnsutor_Llama-3-8B-Instruct_breadcrumbs_ties-density-0.3-gamma-0.01/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3518 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4999 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0234 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3062 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4871 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3611 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/johnsutor_llama-3-8b-instruct_breadcrumbs_ties-density-0.3-gamma-0.1.json b/data/models/johnsutor_llama-3-8b-instruct_breadcrumbs_ties-density-0.3-gamma-0.1.json deleted file mode 100644 index edaebbbe3e457c1b09286b2a7b899088dad8cb92..0000000000000000000000000000000000000000 --- a/data/models/johnsutor_llama-3-8b-instruct_breadcrumbs_ties-density-0.3-gamma-0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-8B-Instruct_breadcrumbs_ties-density-0.3-gamma-0.1", - "id": "johnsutor/Llama-3-8B-Instruct_breadcrumbs_ties-density-0.3-gamma-0.1", - "developer": "johnsutor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/johnsutor_Llama-3-8B-Instruct_breadcrumbs_ties-density-0.3-gamma-0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4204 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5107 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0876 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3045 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4279 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.371 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/johnsutor_llama-3-8b-instruct_breadcrumbs_ties-density-0.5-gamma-0.01.json b/data/models/johnsutor_llama-3-8b-instruct_breadcrumbs_ties-density-0.5-gamma-0.01.json deleted file mode 100644 index af284c997a87b21da8191339b7fa79e1e2a72b6c..0000000000000000000000000000000000000000 --- a/data/models/johnsutor_llama-3-8b-instruct_breadcrumbs_ties-density-0.5-gamma-0.01.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-8B-Instruct_breadcrumbs_ties-density-0.5-gamma-0.01", - "id": "johnsutor/Llama-3-8B-Instruct_breadcrumbs_ties-density-0.5-gamma-0.01", - "developer": "johnsutor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/johnsutor_Llama-3-8B-Instruct_breadcrumbs_ties-density-0.5-gamma-0.01/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3454 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4984 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0219 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.297 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4911 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3531 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/johnsutor_llama-3-8b-instruct_breadcrumbs_ties-density-0.5-gamma-0.1.json b/data/models/johnsutor_llama-3-8b-instruct_breadcrumbs_ties-density-0.5-gamma-0.1.json deleted file mode 100644 index 58b0ba173fee11f711560d093770bb664ea6befa..0000000000000000000000000000000000000000 --- a/data/models/johnsutor_llama-3-8b-instruct_breadcrumbs_ties-density-0.5-gamma-0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-8B-Instruct_breadcrumbs_ties-density-0.5-gamma-0.1", - "id": "johnsutor/Llama-3-8B-Instruct_breadcrumbs_ties-density-0.5-gamma-0.1", - "developer": "johnsutor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/johnsutor_Llama-3-8B-Instruct_breadcrumbs_ties-density-0.5-gamma-0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4092 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5137 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0808 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2953 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4357 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3669 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/johnsutor_llama-3-8b-instruct_breadcrumbs_ties-density-0.7-gamma-0.01.json b/data/models/johnsutor_llama-3-8b-instruct_breadcrumbs_ties-density-0.7-gamma-0.01.json deleted file mode 100644 index 622c8d3587abb589014917f5c4978a42ea53a3f8..0000000000000000000000000000000000000000 --- a/data/models/johnsutor_llama-3-8b-instruct_breadcrumbs_ties-density-0.7-gamma-0.01.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-8B-Instruct_breadcrumbs_ties-density-0.7-gamma-0.01", - "id": "johnsutor/Llama-3-8B-Instruct_breadcrumbs_ties-density-0.7-gamma-0.01", - "developer": "johnsutor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/johnsutor_Llama-3-8B-Instruct_breadcrumbs_ties-density-0.7-gamma-0.01/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2904 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4967 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0144 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2995 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4991 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.349 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/johnsutor_llama-3-8b-instruct_breadcrumbs_ties-density-0.7-gamma-0.1.json b/data/models/johnsutor_llama-3-8b-instruct_breadcrumbs_ties-density-0.7-gamma-0.1.json deleted file mode 100644 index 2b1e8d0bc0f7f570876c917d88039c1aba92437e..0000000000000000000000000000000000000000 --- a/data/models/johnsutor_llama-3-8b-instruct_breadcrumbs_ties-density-0.7-gamma-0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-8B-Instruct_breadcrumbs_ties-density-0.7-gamma-0.1", - "id": "johnsutor/Llama-3-8B-Instruct_breadcrumbs_ties-density-0.7-gamma-0.1", - "developer": "johnsutor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/johnsutor_Llama-3-8B-Instruct_breadcrumbs_ties-density-0.7-gamma-0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4199 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5147 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0808 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2987 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4358 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3615 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/johnsutor_llama-3-8b-instruct_breadcrumbs_ties-density-0.9-gamma-0.01.json b/data/models/johnsutor_llama-3-8b-instruct_breadcrumbs_ties-density-0.9-gamma-0.01.json deleted file mode 100644 index 90c0e4a14560a80be18072822570f0fcb3be895a..0000000000000000000000000000000000000000 --- a/data/models/johnsutor_llama-3-8b-instruct_breadcrumbs_ties-density-0.9-gamma-0.01.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-8B-Instruct_breadcrumbs_ties-density-0.9-gamma-0.01", - "id": "johnsutor/Llama-3-8B-Instruct_breadcrumbs_ties-density-0.9-gamma-0.01", - "developer": "johnsutor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/johnsutor_Llama-3-8B-Instruct_breadcrumbs_ties-density-0.9-gamma-0.01/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2913 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4918 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0106 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3003 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4977 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3454 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/johnsutor_llama-3-8b-instruct_breadcrumbs_ties-density-0.9-gamma-0.1.json b/data/models/johnsutor_llama-3-8b-instruct_breadcrumbs_ties-density-0.9-gamma-0.1.json deleted file mode 100644 index b873e19340a0faf36431b70f1829141f1a153cd4..0000000000000000000000000000000000000000 --- a/data/models/johnsutor_llama-3-8b-instruct_breadcrumbs_ties-density-0.9-gamma-0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-8B-Instruct_breadcrumbs_ties-density-0.9-gamma-0.1", - "id": "johnsutor/Llama-3-8B-Instruct_breadcrumbs_ties-density-0.9-gamma-0.1", - "developer": "johnsutor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/johnsutor_Llama-3-8B-Instruct_breadcrumbs_ties-density-0.9-gamma-0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4162 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5139 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0778 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.297 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4317 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3625 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/johnsutor_llama-3-8b-instruct_dare_linear.json b/data/models/johnsutor_llama-3-8b-instruct_dare_linear.json deleted file mode 100644 index 5f39b39a005460364d3c0074e2a96773bb8fffe3..0000000000000000000000000000000000000000 --- a/data/models/johnsutor_llama-3-8b-instruct_dare_linear.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-8B-Instruct_dare_linear", - "id": "johnsutor/Llama-3-8B-Instruct_dare_linear", - "developer": "johnsutor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/johnsutor_Llama-3-8B-Instruct_dare_linear/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2145 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4283 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2961 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4979 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2414 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/johnsutor_llama-3-8b-instruct_dare_ties-density-0.1.json b/data/models/johnsutor_llama-3-8b-instruct_dare_ties-density-0.1.json deleted file mode 100644 index 42cca2635ef7d26030e6b561d68c36527e23428d..0000000000000000000000000000000000000000 --- a/data/models/johnsutor_llama-3-8b-instruct_dare_ties-density-0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-8B-Instruct_dare_ties-density-0.1", - "id": "johnsutor/Llama-3-8B-Instruct_dare_ties-density-0.1", - "developer": "johnsutor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/johnsutor_Llama-3-8B-Instruct_dare_ties-density-0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1891 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4119 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0008 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2718 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4658 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2265 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/johnsutor_llama-3-8b-instruct_dare_ties-density-0.3.json b/data/models/johnsutor_llama-3-8b-instruct_dare_ties-density-0.3.json deleted file mode 100644 index 020ab19063dd5eb9a7d6cd1389f96841fd6cd49c..0000000000000000000000000000000000000000 --- a/data/models/johnsutor_llama-3-8b-instruct_dare_ties-density-0.3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-8B-Instruct_dare_ties-density-0.3", - "id": "johnsutor/Llama-3-8B-Instruct_dare_ties-density-0.3", - "developer": "johnsutor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/johnsutor_Llama-3-8B-Instruct_dare_ties-density-0.3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2113 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4559 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0015 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.297 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5069 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.304 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/johnsutor_llama-3-8b-instruct_dare_ties-density-0.7.json b/data/models/johnsutor_llama-3-8b-instruct_dare_ties-density-0.7.json deleted file mode 100644 index 9046ce1eb4d422fece31daeffcf1a27768a91add..0000000000000000000000000000000000000000 --- a/data/models/johnsutor_llama-3-8b-instruct_dare_ties-density-0.7.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-8B-Instruct_dare_ties-density-0.7", - "id": "johnsutor/Llama-3-8B-Instruct_dare_ties-density-0.7", - "developer": "johnsutor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/johnsutor_Llama-3-8B-Instruct_dare_ties-density-0.7/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2034 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4723 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.003 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3037 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.511 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3148 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/johnsutor_llama-3-8b-instruct_dare_ties-density-0.9.json b/data/models/johnsutor_llama-3-8b-instruct_dare_ties-density-0.9.json deleted file mode 100644 index 2be7c1fbba2580f47f3b25a46fc239ec573a6e50..0000000000000000000000000000000000000000 --- a/data/models/johnsutor_llama-3-8b-instruct_dare_ties-density-0.9.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-8B-Instruct_dare_ties-density-0.9", - "id": "johnsutor/Llama-3-8B-Instruct_dare_ties-density-0.9", - "developer": "johnsutor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/johnsutor_Llama-3-8B-Instruct_dare_ties-density-0.9/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2161 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4664 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0015 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3079 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.523 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3143 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/johnsutor_llama-3-8b-instruct_linear.json b/data/models/johnsutor_llama-3-8b-instruct_linear.json deleted file mode 100644 index 0165b6e3485d3de55a0f3337d173a1e8e1f1c97b..0000000000000000000000000000000000000000 --- a/data/models/johnsutor_llama-3-8b-instruct_linear.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-8B-Instruct_linear", - "id": "johnsutor/Llama-3-8B-Instruct_linear", - "developer": "johnsutor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/johnsutor_Llama-3-8B-Instruct_linear/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4308 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5031 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1005 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2953 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4097 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3712 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/johnsutor_llama-3-8b-instruct_ties-density-0.1.json b/data/models/johnsutor_llama-3-8b-instruct_ties-density-0.1.json deleted file mode 100644 index 24d5bed773fba10c54537a73228cdbf258619dfd..0000000000000000000000000000000000000000 --- a/data/models/johnsutor_llama-3-8b-instruct_ties-density-0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-8B-Instruct_ties-density-0.1", - "id": "johnsutor/Llama-3-8B-Instruct_ties-density-0.1", - "developer": "johnsutor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/johnsutor_Llama-3-8B-Instruct_ties-density-0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4116 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5021 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0793 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2886 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4174 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.36 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/johnsutor_llama-3-8b-instruct_ties-density-0.3.json b/data/models/johnsutor_llama-3-8b-instruct_ties-density-0.3.json deleted file mode 100644 index c38f3d5f2bc31037269da11f69cb32bd746eef67..0000000000000000000000000000000000000000 --- a/data/models/johnsutor_llama-3-8b-instruct_ties-density-0.3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-8B-Instruct_ties-density-0.3", - "id": "johnsutor/Llama-3-8B-Instruct_ties-density-0.3", - "developer": "johnsutor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/johnsutor_Llama-3-8B-Instruct_ties-density-0.3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3626 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4906 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0672 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2961 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4025 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3321 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/johnsutor_llama-3-8b-instruct_ties-density-0.5.json b/data/models/johnsutor_llama-3-8b-instruct_ties-density-0.5.json deleted file mode 100644 index 7436cd3cdc71b618d25b806a800555289739f760..0000000000000000000000000000000000000000 --- a/data/models/johnsutor_llama-3-8b-instruct_ties-density-0.5.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-8B-Instruct_ties-density-0.5", - "id": "johnsutor/Llama-3-8B-Instruct_ties-density-0.5", - "developer": "johnsutor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/johnsutor_Llama-3-8B-Instruct_ties-density-0.5/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3797 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4793 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0612 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3045 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.388 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3175 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/johnsutor_llama-3-8b-instruct_ties-density-0.7.json b/data/models/johnsutor_llama-3-8b-instruct_ties-density-0.7.json deleted file mode 100644 index 8cc278b7445b729e2994f1676fc1b2622d1e9f2b..0000000000000000000000000000000000000000 --- a/data/models/johnsutor_llama-3-8b-instruct_ties-density-0.7.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-8B-Instruct_ties-density-0.7", - "id": "johnsutor/Llama-3-8B-Instruct_ties-density-0.7", - "developer": "johnsutor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/johnsutor_Llama-3-8B-Instruct_ties-density-0.7/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3681 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4738 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0672 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3096 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3881 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3152 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/johnsutor_llama-3-8b-instruct_ties-density-0.9.json b/data/models/johnsutor_llama-3-8b-instruct_ties-density-0.9.json deleted file mode 100644 index 0b9a122af9550ac69e4954e68d44436578ed4229..0000000000000000000000000000000000000000 --- a/data/models/johnsutor_llama-3-8b-instruct_ties-density-0.9.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-8B-Instruct_ties-density-0.9", - "id": "johnsutor/Llama-3-8B-Instruct_ties-density-0.9", - "developer": "johnsutor", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/johnsutor_Llama-3-8B-Instruct_ties-density-0.9/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3858 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4735 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0619 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2995 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.388 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3182 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jondurbin_bagel-dpo-34b-v0.5.json b/data/models/jondurbin_bagel-dpo-34b-v0.5.json deleted file mode 100644 index 6882b07fa0dc274775de4d443b0b18f27c2179f0..0000000000000000000000000000000000000000 --- a/data/models/jondurbin_bagel-dpo-34b-v0.5.json +++ /dev/null @@ -1,144 +0,0 @@ -{ - "model_info": { - "name": "jondurbin/bagel-dpo-34b-v0.5", - "id": "jondurbin/bagel-dpo-34b-v0.5", - "developer": "jondurbin", - "additional_details": { - "model_type": "DPO" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/jondurbin_bagel-dpo-34b-v0.5/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7215 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9385 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5504 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6446 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8889 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4487 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/joseph717171_hermes-3-llama-3.1-8b_ties_with_base_embeds_initialized_to_special_instruct_toks_dtypef32.json b/data/models/joseph717171_hermes-3-llama-3.1-8b_ties_with_base_embeds_initialized_to_special_instruct_toks_dtypef32.json deleted file mode 100644 index 824e250107af12a4a64ec4d31dff3f2c40a979ee..0000000000000000000000000000000000000000 --- a/data/models/joseph717171_hermes-3-llama-3.1-8b_ties_with_base_embeds_initialized_to_special_instruct_toks_dtypef32.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Hermes-3-Llama-3.1-8B_TIES_with_Base_Embeds_Initialized_to_Special_Instruct_Toks_dtypeF32", - "id": "Joseph717171/Hermes-3-Llama-3.1-8B_TIES_with_Base_Embeds_Initialized_to_Special_Instruct_Toks_dtypeF32", - "developer": "Joseph717171", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Joseph717171_Hermes-3-Llama-3.1-8B_TIES_with_Base_Embeds_Initialized_to_Special_Instruct_Toks_dtypeF32/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6185 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5177 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0514 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2827 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4369 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3144 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/joseph717171_llama-3.1-supernova-8b-lite_ties_with_base.json b/data/models/joseph717171_llama-3.1-supernova-8b-lite_ties_with_base.json deleted file mode 100644 index 0ae2e9c8efecdac1e7f7f889fa638624f1d38047..0000000000000000000000000000000000000000 --- a/data/models/joseph717171_llama-3.1-supernova-8b-lite_ties_with_base.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.1-SuperNova-8B-Lite_TIES_with_Base", - "id": "Joseph717171/Llama-3.1-SuperNova-8B-Lite_TIES_with_Base", - "developer": "Joseph717171", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Joseph717171_Llama-3.1-SuperNova-8B-Lite_TIES_with_Base/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8096 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5147 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1835 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3096 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.411 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.388 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/josephgflowers_cinder-phi-2-v1-f16-gguf.json b/data/models/josephgflowers_cinder-phi-2-v1-f16-gguf.json deleted file mode 100644 index 20560f1bf096cf87bf3fff6cbde317c1a21ddaf6..0000000000000000000000000000000000000000 --- a/data/models/josephgflowers_cinder-phi-2-v1-f16-gguf.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Cinder-Phi-2-V1-F16-gguf", - "id": "Josephgflowers/Cinder-Phi-2-V1-F16-gguf", - "developer": "Josephgflowers", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "PhiForCausalLM", - "params_billions": "2.78" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Josephgflowers_Cinder-Phi-2-V1-F16-gguf/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2357 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4397 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0242 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2819 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3435 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2161 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/josephgflowers_differential-attention-liquid-metal-tinyllama.json b/data/models/josephgflowers_differential-attention-liquid-metal-tinyllama.json deleted file mode 100644 index 267f96ef0e8b7b1a88f2a33a4ed49126f994f696..0000000000000000000000000000000000000000 --- a/data/models/josephgflowers_differential-attention-liquid-metal-tinyllama.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Differential-Attention-Liquid-Metal-Tinyllama", - "id": "Josephgflowers/Differential-Attention-Liquid-Metal-Tinyllama", - "developer": "Josephgflowers", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.1" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Josephgflowers_Differential-Attention-Liquid-Metal-Tinyllama/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2227 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2926 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0325 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2508 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3356 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1214 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/josephgflowers_tinyllama-cinder-agent-v1.json b/data/models/josephgflowers_tinyllama-cinder-agent-v1.json deleted file mode 100644 index b51af97e9851b7dbfc556320330870467c8b7367..0000000000000000000000000000000000000000 --- a/data/models/josephgflowers_tinyllama-cinder-agent-v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "TinyLlama-Cinder-Agent-v1", - "id": "Josephgflowers/TinyLlama-Cinder-Agent-v1", - "developer": "Josephgflowers", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.1" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Josephgflowers_TinyLlama-Cinder-Agent-v1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.267 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3116 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0347 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2441 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3395 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1161 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/josephgflowers_tinyllama-r1.json b/data/models/josephgflowers_tinyllama-r1.json deleted file mode 100644 index 3fdaef9780570d2380297c240f4832f8820ca84e..0000000000000000000000000000000000000000 --- a/data/models/josephgflowers_tinyllama-r1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Tinyllama-r1", - "id": "Josephgflowers/Tinyllama-r1", - "developer": "Josephgflowers", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.1" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Josephgflowers_Tinyllama-r1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2119 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3015 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0325 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2567 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3315 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1134 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/josephgflowers_tinyllama-stem-cinder-agent-v1.json b/data/models/josephgflowers_tinyllama-stem-cinder-agent-v1.json deleted file mode 100644 index 72711f7b0b799c3e1fd96f90468720e1196a58a4..0000000000000000000000000000000000000000 --- a/data/models/josephgflowers_tinyllama-stem-cinder-agent-v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Tinyllama-STEM-Cinder-Agent-v1", - "id": "Josephgflowers/Tinyllama-STEM-Cinder-Agent-v1", - "developer": "Josephgflowers", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.1" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Josephgflowers_Tinyllama-STEM-Cinder-Agent-v1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2126 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3084 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0672 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2349 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3341 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1086 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/josephgflowers_tinyllama-v1.1-cinders-world.json b/data/models/josephgflowers_tinyllama-v1.1-cinders-world.json deleted file mode 100644 index cf9f768ce2c56dbaae405eed887f65a0bc827a5d..0000000000000000000000000000000000000000 --- a/data/models/josephgflowers_tinyllama-v1.1-cinders-world.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "TinyLlama-v1.1-Cinders-World", - "id": "Josephgflowers/TinyLlama-v1.1-Cinders-World", - "developer": "Josephgflowers", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.1" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Josephgflowers_TinyLlama-v1.1-Cinders-World/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2469 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2998 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0347 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2441 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3356 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1198 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/josephgflowers_tinyllama_v1.1_math_code-world-test-1.json b/data/models/josephgflowers_tinyllama_v1.1_math_code-world-test-1.json deleted file mode 100644 index d13643bd842c032f53c16a2b05a3d8f08a9da915..0000000000000000000000000000000000000000 --- a/data/models/josephgflowers_tinyllama_v1.1_math_code-world-test-1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "TinyLlama_v1.1_math_code-world-test-1", - "id": "Josephgflowers/TinyLlama_v1.1_math_code-world-test-1", - "developer": "Josephgflowers", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.1" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Josephgflowers_TinyLlama_v1.1_math_code-world-test-1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0078 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3146 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0196 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2341 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3499 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1132 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jpacifico_chocolatine-14b-instruct-4k-dpo.json b/data/models/jpacifico_chocolatine-14b-instruct-4k-dpo.json deleted file mode 100644 index 11a8a3ea9f08b98df6bb6d069501bdaf94ec1170..0000000000000000000000000000000000000000 --- a/data/models/jpacifico_chocolatine-14b-instruct-4k-dpo.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Chocolatine-14B-Instruct-4k-DPO", - "id": "jpacifico/Chocolatine-14B-Instruct-4k-DPO", - "developer": "jpacifico", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Phi3ForCausalLM", - "params_billions": "13.96" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jpacifico_Chocolatine-14B-Instruct-4k-DPO/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4689 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.63 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1782 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3414 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4439 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4764 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jpacifico_chocolatine-14b-instruct-dpo-v1.2.json b/data/models/jpacifico_chocolatine-14b-instruct-dpo-v1.2.json deleted file mode 100644 index 8f59698813b1f45aae434e8956d171d423489afa..0000000000000000000000000000000000000000 --- a/data/models/jpacifico_chocolatine-14b-instruct-dpo-v1.2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Chocolatine-14B-Instruct-DPO-v1.2", - "id": "jpacifico/Chocolatine-14B-Instruct-DPO-v1.2", - "developer": "jpacifico", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Phi3ForCausalLM", - "params_billions": "13.96" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jpacifico_Chocolatine-14B-Instruct-DPO-v1.2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6852 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6438 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2092 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3255 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4268 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4697 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jpacifico_chocolatine-14b-instruct-dpo-v1.3.json b/data/models/jpacifico_chocolatine-14b-instruct-dpo-v1.3.json deleted file mode 100644 index 166937fc46941fb32980e731a8670f49fd1ec0a8..0000000000000000000000000000000000000000 --- a/data/models/jpacifico_chocolatine-14b-instruct-dpo-v1.3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Chocolatine-14B-Instruct-DPO-v1.3", - "id": "jpacifico/Chocolatine-14B-Instruct-DPO-v1.3", - "developer": "jpacifico", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Phi3ForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jpacifico_Chocolatine-14B-Instruct-DPO-v1.3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.704 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6846 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5619 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3414 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4234 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5374 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jpacifico_chocolatine-2-14b-instruct-dpo-v2.0b1.json b/data/models/jpacifico_chocolatine-2-14b-instruct-dpo-v2.0b1.json deleted file mode 100644 index 2ffcc0de5f31a59f2c746ce5fdf3989760d6bf9d..0000000000000000000000000000000000000000 --- a/data/models/jpacifico_chocolatine-2-14b-instruct-dpo-v2.0b1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Chocolatine-2-14B-Instruct-DPO-v2.0b1", - "id": "jpacifico/Chocolatine-2-14B-Instruct-DPO-v2.0b1", - "developer": "jpacifico", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jpacifico_Chocolatine-2-14B-Instruct-DPO-v2.0b1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1033 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6696 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2757 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3758 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4467 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5124 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jpacifico_chocolatine-2-14b-instruct-v2.0.1.json b/data/models/jpacifico_chocolatine-2-14b-instruct-v2.0.1.json deleted file mode 100644 index eec2445e8a4e850190dca7757f82bf91a53e428c..0000000000000000000000000000000000000000 --- a/data/models/jpacifico_chocolatine-2-14b-instruct-v2.0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Chocolatine-2-14B-Instruct-v2.0.1", - "id": "jpacifico/Chocolatine-2-14B-Instruct-v2.0.1", - "developer": "jpacifico", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jpacifico_Chocolatine-2-14B-Instruct-v2.0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0742 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6736 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4796 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3918 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5008 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5299 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jpacifico_chocolatine-2-14b-instruct-v2.0.3.json b/data/models/jpacifico_chocolatine-2-14b-instruct-v2.0.3.json deleted file mode 100644 index 61f047016c75d244bf8127aa073706d63a78bafa..0000000000000000000000000000000000000000 --- a/data/models/jpacifico_chocolatine-2-14b-instruct-v2.0.3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Chocolatine-2-14B-Instruct-v2.0.3", - "id": "jpacifico/Chocolatine-2-14B-Instruct-v2.0.3", - "developer": "jpacifico", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jpacifico_Chocolatine-2-14B-Instruct-v2.0.3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7037 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6548 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4207 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3792 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4768 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5374 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jpacifico_chocolatine-2-14b-instruct-v2.0.json b/data/models/jpacifico_chocolatine-2-14b-instruct-v2.0.json deleted file mode 100644 index cff47c0064f1fa0220000386dc6c7ec139cdf1f0..0000000000000000000000000000000000000000 --- a/data/models/jpacifico_chocolatine-2-14b-instruct-v2.0.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Chocolatine-2-14B-Instruct-v2.0", - "id": "jpacifico/Chocolatine-2-14B-Instruct-v2.0", - "developer": "jpacifico", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jpacifico_Chocolatine-2-14B-Instruct-v2.0/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0885 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.677 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4804 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3876 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5021 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5302 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jpacifico_chocolatine-2-14b-instruct-v2.0b2.json b/data/models/jpacifico_chocolatine-2-14b-instruct-v2.0b2.json deleted file mode 100644 index 91695525aab143df580ef21e3e0ea7a0da54d566..0000000000000000000000000000000000000000 --- a/data/models/jpacifico_chocolatine-2-14b-instruct-v2.0b2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Chocolatine-2-14B-Instruct-v2.0b2", - "id": "jpacifico/Chocolatine-2-14B-Instruct-v2.0b2", - "developer": "jpacifico", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jpacifico_Chocolatine-2-14B-Instruct-v2.0b2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7241 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6476 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.395 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3834 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4808 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5369 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jpacifico_chocolatine-2-14b-instruct-v2.0b3.json b/data/models/jpacifico_chocolatine-2-14b-instruct-v2.0b3.json deleted file mode 100644 index 43e06b543aec14e248ed3ee5a1549cfdffa4a9d2..0000000000000000000000000000000000000000 --- a/data/models/jpacifico_chocolatine-2-14b-instruct-v2.0b3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Chocolatine-2-14B-Instruct-v2.0b3", - "id": "jpacifico/Chocolatine-2-14B-Instruct-v2.0b3", - "developer": "jpacifico", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jpacifico_Chocolatine-2-14B-Instruct-v2.0b3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7323 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6469 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4109 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3792 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4781 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5337 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jpacifico_chocolatine-3b-instruct-dpo-revised.json b/data/models/jpacifico_chocolatine-3b-instruct-dpo-revised.json deleted file mode 100644 index 01ba15df799376a9a70bc18280f3fd6237778a7e..0000000000000000000000000000000000000000 --- a/data/models/jpacifico_chocolatine-3b-instruct-dpo-revised.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Chocolatine-3B-Instruct-DPO-Revised", - "id": "jpacifico/Chocolatine-3B-Instruct-DPO-Revised", - "developer": "jpacifico", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Phi3ForCausalLM", - "params_billions": "3.821" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jpacifico_Chocolatine-3B-Instruct-DPO-Revised/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5623 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.554 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1805 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3221 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4453 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3989 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jpacifico_chocolatine-3b-instruct-dpo-v1.0.json b/data/models/jpacifico_chocolatine-3b-instruct-dpo-v1.0.json deleted file mode 100644 index cfee83b76496c0a78633a95eed795b3be01bf1a7..0000000000000000000000000000000000000000 --- a/data/models/jpacifico_chocolatine-3b-instruct-dpo-v1.0.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Chocolatine-3B-Instruct-DPO-v1.0", - "id": "jpacifico/Chocolatine-3B-Instruct-DPO-v1.0", - "developer": "jpacifico", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Phi3ForCausalLM", - "params_billions": "3.821" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jpacifico_Chocolatine-3B-Instruct-DPO-v1.0/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3737 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5471 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1782 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3154 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4755 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3937 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jpacifico_chocolatine-3b-instruct-dpo-v1.2.json b/data/models/jpacifico_chocolatine-3b-instruct-dpo-v1.2.json deleted file mode 100644 index f2a8045db76ff7c03ace4e21b1cb6973cb8c9314..0000000000000000000000000000000000000000 --- a/data/models/jpacifico_chocolatine-3b-instruct-dpo-v1.2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Chocolatine-3B-Instruct-DPO-v1.2", - "id": "jpacifico/Chocolatine-3B-Instruct-DPO-v1.2", - "developer": "jpacifico", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Phi3ForCausalLM", - "params_billions": "3.821" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jpacifico_Chocolatine-3B-Instruct-DPO-v1.2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5455 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5487 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2047 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3389 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4154 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3877 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jpacifico_distilucie-7b-math-instruct-dpo-v0.1.json b/data/models/jpacifico_distilucie-7b-math-instruct-dpo-v0.1.json deleted file mode 100644 index 6120635fa55c85d871bc3035cdb1f83e3a8a4109..0000000000000000000000000000000000000000 --- a/data/models/jpacifico_distilucie-7b-math-instruct-dpo-v0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Distilucie-7B-Math-Instruct-DPO-v0.1", - "id": "jpacifico/Distilucie-7B-Math-Instruct-DPO-v0.1", - "developer": "jpacifico", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "6.707" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jpacifico_Distilucie-7B-Math-Instruct-DPO-v0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3048 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3835 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0257 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2995 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3644 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1809 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jpacifico_lucie-7b-instruct-dpo-v1.1.3.json b/data/models/jpacifico_lucie-7b-instruct-dpo-v1.1.3.json deleted file mode 100644 index 4d419ac2121fa3be98abd78d9a60c843e7927cea..0000000000000000000000000000000000000000 --- a/data/models/jpacifico_lucie-7b-instruct-dpo-v1.1.3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Lucie-7B-Instruct-DPO-v1.1.3", - "id": "jpacifico/Lucie-7B-Instruct-DPO-v1.1.3", - "developer": "jpacifico", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "6.707" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jpacifico_Lucie-7B-Instruct-DPO-v1.1.3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3045 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3819 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0242 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2861 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3818 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1764 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jpacifico_lucie-7b-instruct-dpo-v1.1.json b/data/models/jpacifico_lucie-7b-instruct-dpo-v1.1.json deleted file mode 100644 index 98c2fbe7cbea98c3415341d03f34324dfb17b6b8..0000000000000000000000000000000000000000 --- a/data/models/jpacifico_lucie-7b-instruct-dpo-v1.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Lucie-7B-Instruct-DPO-v1.1", - "id": "jpacifico/Lucie-7B-Instruct-DPO-v1.1", - "developer": "jpacifico", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "6.707" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jpacifico_Lucie-7B-Instruct-DPO-v1.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3121 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3781 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0234 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2878 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4016 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1838 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jpacifico_lucie-7b-instruct-merged-model_stock-v1.0.json b/data/models/jpacifico_lucie-7b-instruct-merged-model_stock-v1.0.json deleted file mode 100644 index 276d078e74e48db7b64741c2f732d26ec8e90fcf..0000000000000000000000000000000000000000 --- a/data/models/jpacifico_lucie-7b-instruct-merged-model_stock-v1.0.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Lucie-7B-Instruct-Merged-Model_Stock-v1.0", - "id": "jpacifico/Lucie-7B-Instruct-Merged-Model_Stock-v1.0", - "developer": "jpacifico", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "6.707" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jpacifico_Lucie-7B-Instruct-Merged-Model_Stock-v1.0/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3234 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3802 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0242 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2886 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3844 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1871 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jpacifico_lucie-7b-instruct-merged-model_stock-v1.1.json b/data/models/jpacifico_lucie-7b-instruct-merged-model_stock-v1.1.json deleted file mode 100644 index 5825f4fb2d4f88bceefde102b07daf83729bd722..0000000000000000000000000000000000000000 --- a/data/models/jpacifico_lucie-7b-instruct-merged-model_stock-v1.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Lucie-7B-Instruct-Merged-Model_Stock-v1.1", - "id": "jpacifico/Lucie-7B-Instruct-Merged-Model_Stock-v1.1", - "developer": "jpacifico", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "6.707" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jpacifico_Lucie-7B-Instruct-Merged-Model_Stock-v1.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3014 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3808 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0279 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2827 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.375 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1862 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jpacifico_lucie-boosted-7b-instruct.json b/data/models/jpacifico_lucie-boosted-7b-instruct.json deleted file mode 100644 index 65c45c0002655565c3c872b0702eecdb2511af5d..0000000000000000000000000000000000000000 --- a/data/models/jpacifico_lucie-boosted-7b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Lucie-Boosted-7B-Instruct", - "id": "jpacifico/Lucie-Boosted-7B-Instruct", - "developer": "jpacifico", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "6.707" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jpacifico_Lucie-Boosted-7B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2566 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3465 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0128 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2668 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3699 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.163 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jsfs11_l3-8b-stheno-slerp.json b/data/models/jsfs11_l3-8b-stheno-slerp.json deleted file mode 100644 index 08a4a5ff0a63d0a10b18a4eb57c3f463e5ca8f87..0000000000000000000000000000000000000000 --- a/data/models/jsfs11_l3-8b-stheno-slerp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "L3-8B-Stheno-slerp", - "id": "jsfs11/L3-8B-Stheno-slerp", - "developer": "jsfs11", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jsfs11_L3-8B-Stheno-slerp/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6752 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5326 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0989 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2852 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3725 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3649 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jsfs11_mixtureofmerges-moe-4x7b-v4.json b/data/models/jsfs11_mixtureofmerges-moe-4x7b-v4.json deleted file mode 100644 index e12311ea1be5c578b35887fbbc6553a869a32d2f..0000000000000000000000000000000000000000 --- a/data/models/jsfs11_mixtureofmerges-moe-4x7b-v4.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MixtureofMerges-MoE-4x7b-v4", - "id": "jsfs11/MixtureofMerges-MoE-4x7b-v4", - "developer": "jsfs11", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MixtralForCausalLM", - "params_billions": "24.154" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jsfs11_MixtureofMerges-MoE-4x7b-v4/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.403 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5169 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0634 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2861 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4386 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3032 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jsfs11_mixtureofmerges-moe-4x7b-v5.json b/data/models/jsfs11_mixtureofmerges-moe-4x7b-v5.json deleted file mode 100644 index 0436be84cae937c2c21370ad1d30a57e9a3f60ee..0000000000000000000000000000000000000000 --- a/data/models/jsfs11_mixtureofmerges-moe-4x7b-v5.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MixtureofMerges-MoE-4x7b-v5", - "id": "jsfs11/MixtureofMerges-MoE-4x7b-v5", - "developer": "jsfs11", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MixtralForCausalLM", - "params_billions": "24.154" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/jsfs11_MixtureofMerges-MoE-4x7b-v5/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4199 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5198 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0755 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2844 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4305 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3098 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jungzoona_t3q-qwen2.5-14b-instruct-1m-e3.json b/data/models/jungzoona_t3q-qwen2.5-14b-instruct-1m-e3.json deleted file mode 100644 index 31357214b67f18575231bb5f5d1ed597729ca366..0000000000000000000000000000000000000000 --- a/data/models/jungzoona_t3q-qwen2.5-14b-instruct-1m-e3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "T3Q-Qwen2.5-14B-Instruct-1M-e3", - "id": "JungZoona/T3Q-Qwen2.5-14B-Instruct-1M-e3", - "developer": "JungZoona", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Unknown", - "params_billions": "0.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JungZoona_T3Q-Qwen2.5-14B-Instruct-1M-e3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7324 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7586 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2863 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4169 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5911 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5884 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/jungzoona_t3q-qwen2.5-14b-v1.0-e3.json b/data/models/jungzoona_t3q-qwen2.5-14b-v1.0-e3.json deleted file mode 100644 index 626ecd33c8ecc96d37686f86b55d7458d65e6612..0000000000000000000000000000000000000000 --- a/data/models/jungzoona_t3q-qwen2.5-14b-v1.0-e3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "T3Q-qwen2.5-14b-v1.0-e3", - "id": "JungZoona/T3Q-qwen2.5-14b-v1.0-e3", - "developer": "JungZoona", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/JungZoona_T3Q-qwen2.5-14b-v1.0-e3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7324 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7586 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2863 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4169 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5911 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5884 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/junhoee_qwen-megumin.json b/data/models/junhoee_qwen-megumin.json deleted file mode 100644 index aa4b68542eab86d365e5b6d8238089a62dc6e40e..0000000000000000000000000000000000000000 --- a/data/models/junhoee_qwen-megumin.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen-Megumin", - "id": "Junhoee/Qwen-Megumin", - "developer": "Junhoee", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "?", - "params_billions": "15.231" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Junhoee_Qwen-Megumin/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7141 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5285 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4902 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2961 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.398 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4199 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/kaist-ai_janus-7b.json b/data/models/kaist-ai_janus-7b.json deleted file mode 100644 index 51a6d775e3a9cd0bbeee9fb5813b70010c5a3581..0000000000000000000000000000000000000000 --- a/data/models/kaist-ai_janus-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "janus-7b", - "id": "kaist-ai/janus-7b", - "developer": "kaist-ai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/kaist-ai_janus-7b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3775 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4694 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0408 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2727 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4401 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2874 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/kaist-ai_janus-dpo-7b.json b/data/models/kaist-ai_janus-dpo-7b.json deleted file mode 100644 index 1618ac9fcd7aa998df0b51a98bf2edbc8593e672..0000000000000000000000000000000000000000 --- a/data/models/kaist-ai_janus-dpo-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "janus-dpo-7b", - "id": "kaist-ai/janus-dpo-7b", - "developer": "kaist-ai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/kaist-ai_janus-dpo-7b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4003 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4773 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0415 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2819 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4387 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2976 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/kaist-ai_janus-rm-7b.json b/data/models/kaist-ai_janus-rm-7b.json deleted file mode 100644 index 2786192dc6bbd68fc9eee4ff7e59d2e5558aa56f..0000000000000000000000000000000000000000 --- a/data/models/kaist-ai_janus-rm-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "janus-rm-7b", - "id": "kaist-ai/janus-rm-7b", - "developer": "kaist-ai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LLMForSequenceRegression", - "params_billions": "7.111" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/kaist-ai_janus-rm-7b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1778 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3056 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2517 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3883 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1126 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/kaist-ai_mistral-orpo-capybara-7k.json b/data/models/kaist-ai_mistral-orpo-capybara-7k.json deleted file mode 100644 index 6537e722f048c6abbc6406a433b25aec63d509c2..0000000000000000000000000000000000000000 --- a/data/models/kaist-ai_mistral-orpo-capybara-7k.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "mistral-orpo-capybara-7k", - "id": "kaist-ai/mistral-orpo-capybara-7k", - "developer": "kaist-ai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/kaist-ai_mistral-orpo-capybara-7k/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5367 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4489 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0393 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2861 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3964 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2971 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/katanemo_arch-agent-1-5b.json b/data/models/katanemo_arch-agent-1-5b.json deleted file mode 100644 index ef3c7a63811e558be5b049c843e4ea9387d37d54..0000000000000000000000000000000000000000 --- a/data/models/katanemo_arch-agent-1-5b.json +++ /dev/null @@ -1,850 +0,0 @@ -{ - "model_info": { - "name": "Arch-Agent-1.5B", - "id": "katanemo/arch-agent-1-5b", - "developer": "katanemo", - "additional_details": { - "raw_model_name": "Arch-Agent-1.5B", - "organization": "katanemo", - "license": "katanemo-research", - "model_link": "https://huggingface.co/katanemo/Arch-Agent-1.5B" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/katanemo/arch-agent-1-5b/1775236112.397114", - "retrieved_timestamp": "1775236112.397114", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 60.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 32.14 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 2.45 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 2.38 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 4.01 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 5.3 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 82.67 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 72.17 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 92.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 85.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 81.0 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 67.73 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 70.54 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 67.81 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 31.25 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 58.33 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 26.62 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 35.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 27.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 21.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 22.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 8.17 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 5.81 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 5.81 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 12.9 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 75.0 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 74.83 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/katanemo_arch-agent-32b.json b/data/models/katanemo_arch-agent-32b.json deleted file mode 100644 index ab1b9edae5579750cdfff06074c331f1d5d6e315..0000000000000000000000000000000000000000 --- a/data/models/katanemo_arch-agent-32b.json +++ /dev/null @@ -1,850 +0,0 @@ -{ - "model_info": { - "name": "Arch-Agent-32B", - "id": "katanemo/arch-agent-32b", - "developer": "katanemo", - "additional_details": { - "raw_model_name": "Arch-Agent-32B", - "organization": "katanemo", - "license": "katanemo-research", - "model_link": "https://huggingface.co/katanemo/Arch-Agent-32B" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/katanemo/arch-agent-32b/1775236112.384274", - "retrieved_timestamp": "1775236112.384274", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 37.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 45.37 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 8.87 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 9.44 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 21.44 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 24.87 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 88.92 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 76.67 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 94.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 96.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 89.0 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 80.68 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 86.43 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 79.11 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 93.75 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 79.17 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 54.25 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 64.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 58.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 53.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 41.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 5.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 4.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 6.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 14.62 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 5.81 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 9.03 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 29.03 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 81.25 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 82.15 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/katanemo_arch-agent-3b.json b/data/models/katanemo_arch-agent-3b.json deleted file mode 100644 index f85abe95cb4304c1cb5063e069ca48e7934180de..0000000000000000000000000000000000000000 --- a/data/models/katanemo_arch-agent-3b.json +++ /dev/null @@ -1,850 +0,0 @@ -{ - "model_info": { - "name": "Arch-Agent-3B", - "id": "katanemo/arch-agent-3b", - "developer": "katanemo", - "additional_details": { - "raw_model_name": "Arch-Agent-3B", - "organization": "katanemo", - "license": "katanemo-research", - "model_link": "https://huggingface.co/katanemo/Arch-Agent-3B" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/katanemo/arch-agent-3b/1775236112.3950138", - "retrieved_timestamp": "1775236112.3950138", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 56.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 35.36 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 3.7 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 3.56 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 6.65 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 8.19 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 86.67 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 78.67 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 94.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 91.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 82.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 72.91 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 75.58 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 72.27 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 68.75 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 75.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 34.88 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 42.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 37.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 31.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 29.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 0.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 1.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 6.88 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 5.16 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 5.81 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 9.68 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 68.75 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 74.67 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/kavonalds_bundermaxx-0710.json b/data/models/kavonalds_bundermaxx-0710.json deleted file mode 100644 index 1c392120900a40ecef63990c2ea6f8c3e297a9f1..0000000000000000000000000000000000000000 --- a/data/models/kavonalds_bundermaxx-0710.json +++ /dev/null @@ -1,275 +0,0 @@ -{ - "model_info": { - "name": "BunderMaxx-0710", - "id": "kavonalds/BunderMaxx-0710", - "developer": "kavonalds", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.236" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/kavonalds_BunderMaxx-0710/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3283 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6651 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.068 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2609 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3393 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1314 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "hfopenllm_v2/kavonalds_BunderMaxx-0710/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2701 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5566 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.068 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2802 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3682 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1449 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/kavonalds_bundermaxx-1010.json b/data/models/kavonalds_bundermaxx-1010.json deleted file mode 100644 index 3da9329dfe7458e7ea6a79beafad76b2c6de8b99..0000000000000000000000000000000000000000 --- a/data/models/kavonalds_bundermaxx-1010.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "BunderMaxx-1010", - "id": "kavonalds/BunderMaxx-1010", - "developer": "kavonalds", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.236" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/kavonalds_BunderMaxx-1010/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2981 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.702 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.105 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2609 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3484 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1224 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/kavonalds_lancer-1-1b-instruct.json b/data/models/kavonalds_lancer-1-1b-instruct.json deleted file mode 100644 index 32ef97e012e5a8026846cfeb66a8706619d30367..0000000000000000000000000000000000000000 --- a/data/models/kavonalds_lancer-1-1b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Lancer-1-1b-Instruct", - "id": "kavonalds/Lancer-1-1b-Instruct", - "developer": "kavonalds", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.236" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/kavonalds_Lancer-1-1b-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5546 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3253 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0393 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2617 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3144 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1568 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/kayfour_t3q-qwen2.5-7b-it-kor-safe.json b/data/models/kayfour_t3q-qwen2.5-7b-it-kor-safe.json deleted file mode 100644 index 7917ebb205439caf50edb2e1c564cdb0952ec8d4..0000000000000000000000000000000000000000 --- a/data/models/kayfour_t3q-qwen2.5-7b-it-kor-safe.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "T3Q-Qwen2.5-7B-it-KOR-Safe", - "id": "kayfour/T3Q-Qwen2.5-7B-it-KOR-Safe", - "developer": "kayfour", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/kayfour_T3Q-Qwen2.5-7B-it-KOR-Safe/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6081 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.555 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3761 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3213 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4277 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4464 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/keeeeenw_microllama.json b/data/models/keeeeenw_microllama.json deleted file mode 100644 index 7f3ae379f3a2c3bd0b4493d39edc4edfc343b90c..0000000000000000000000000000000000000000 --- a/data/models/keeeeenw_microllama.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MicroLlama", - "id": "keeeeenw/MicroLlama", - "developer": "keeeeenw", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "0.305" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/keeeeenw_MicroLlama/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1985 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3007 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0113 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2609 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3698 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1138 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/kekmodel_stopcarbon-10.7b-v5.json b/data/models/kekmodel_stopcarbon-10.7b-v5.json deleted file mode 100644 index d304ed531da3d0f4c18cba560bf906b05343aea1..0000000000000000000000000000000000000000 --- a/data/models/kekmodel_stopcarbon-10.7b-v5.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "StopCarbon-10.7B-v5", - "id": "kekmodel/StopCarbon-10.7B-v5", - "developer": "kekmodel", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "10.732" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/kekmodel_StopCarbon-10.7B-v5/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4728 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5178 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0559 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3062 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4019 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3157 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/kevin009_llamaragdrama.json b/data/models/kevin009_llamaragdrama.json deleted file mode 100644 index d0279c353f4a3e3f76a5b3d58ec3881cd9ec8fd3..0000000000000000000000000000000000000000 --- a/data/models/kevin009_llamaragdrama.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "llamaRAGdrama", - "id": "kevin009/llamaRAGdrama", - "developer": "kevin009", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/kevin009_llamaRAGdrama/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2598 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4007 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0431 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2643 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4316 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2724 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/khetterman_darkatom-12b-v3.json b/data/models/khetterman_darkatom-12b-v3.json deleted file mode 100644 index d7703325e5e7722e88a6404c0bcaaf54c0f06eb3..0000000000000000000000000000000000000000 --- a/data/models/khetterman_darkatom-12b-v3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "DarkAtom-12B-v3", - "id": "Khetterman/DarkAtom-12B-v3", - "developer": "Khetterman", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Khetterman_DarkAtom-12B-v3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6173 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5154 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.111 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2978 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4468 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3546 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/khetterman_kosmos-8b-v1.json b/data/models/khetterman_kosmos-8b-v1.json deleted file mode 100644 index 94a6df8a379fba45e8aadefd523e1837331b8792..0000000000000000000000000000000000000000 --- a/data/models/khetterman_kosmos-8b-v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Kosmos-8B-v1", - "id": "Khetterman/Kosmos-8B-v1", - "developer": "Khetterman", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Khetterman_Kosmos-8B-v1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4129 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5234 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0989 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2987 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3919 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3669 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/khoantap_cheap-moe-merge.json b/data/models/khoantap_cheap-moe-merge.json deleted file mode 100644 index d45d75ba4eb8c2f82e7fe32f6adc6794b40614c0..0000000000000000000000000000000000000000 --- a/data/models/khoantap_cheap-moe-merge.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "cheap-moe-merge", - "id": "khoantap/cheap-moe-merge", - "developer": "khoantap", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2MoeForCausalLM", - "params_billions": "19.305" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/khoantap_cheap-moe-merge/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4557 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5131 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0921 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2953 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4103 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3339 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/khoantap_llama-3-8b-stock-merge.json b/data/models/khoantap_llama-3-8b-stock-merge.json deleted file mode 100644 index e65a30d8c998bd81286abdfaaa6106b1882c6a60..0000000000000000000000000000000000000000 --- a/data/models/khoantap_llama-3-8b-stock-merge.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "llama-3-8b-stock-merge", - "id": "khoantap/llama-3-8b-stock-merge", - "developer": "khoantap", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/khoantap_llama-3-8b-stock-merge/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4812 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5162 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1616 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.318 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3946 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.38 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/khoantap_llama-breadcrumbs-ties-merge.json b/data/models/khoantap_llama-breadcrumbs-ties-merge.json deleted file mode 100644 index 9b73cb512c9716d6c56a10af6784a4ba483806e1..0000000000000000000000000000000000000000 --- a/data/models/khoantap_llama-breadcrumbs-ties-merge.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "llama-breadcrumbs-ties-merge", - "id": "khoantap/llama-breadcrumbs-ties-merge", - "developer": "khoantap", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/khoantap_llama-breadcrumbs-ties-merge/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2205 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5416 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1125 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2659 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4434 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3172 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/khoantap_llama-evolve-ties-best-merge.json b/data/models/khoantap_llama-evolve-ties-best-merge.json deleted file mode 100644 index 022ce672969b1f97dd4c63f1d51eafd721cb6bec..0000000000000000000000000000000000000000 --- a/data/models/khoantap_llama-evolve-ties-best-merge.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "llama-evolve-ties-best-merge", - "id": "khoantap/llama-evolve-ties-best-merge", - "developer": "khoantap", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/khoantap_llama-evolve-ties-best-merge/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6744 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5414 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1563 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3171 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3946 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.386 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/khoantap_llama-linear-0.5-0.5-1-merge.json b/data/models/khoantap_llama-linear-0.5-0.5-1-merge.json deleted file mode 100644 index bf28ebe3f58877463e3f92e1c27c8a27b8e7b6f5..0000000000000000000000000000000000000000 --- a/data/models/khoantap_llama-linear-0.5-0.5-1-merge.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "llama-linear-0.5-0.5-1-merge", - "id": "khoantap/llama-linear-0.5-0.5-1-merge", - "developer": "khoantap", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/khoantap_llama-linear-0.5-0.5-1-merge/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4812 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5643 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2054 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.307 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4143 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3833 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/khoantap_llama-linear-0.5-1-0.5-merge.json b/data/models/khoantap_llama-linear-0.5-1-0.5-merge.json deleted file mode 100644 index 8f13524570dbdc51307506440ee85c5a860b0ff7..0000000000000000000000000000000000000000 --- a/data/models/khoantap_llama-linear-0.5-1-0.5-merge.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "llama-linear-0.5-1-0.5-merge", - "id": "khoantap/llama-linear-0.5-1-0.5-merge", - "developer": "khoantap", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/khoantap_llama-linear-0.5-1-0.5-merge/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5032 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5951 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.148 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2936 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4172 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.369 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/khoantap_llama-linear-1-0.5-0.5-merge.json b/data/models/khoantap_llama-linear-1-0.5-0.5-merge.json deleted file mode 100644 index aa737709f8de0e64478a95daec83d42a864f6369..0000000000000000000000000000000000000000 --- a/data/models/khoantap_llama-linear-1-0.5-0.5-merge.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "llama-linear-1-0.5-0.5-merge", - "id": "khoantap/llama-linear-1-0.5-0.5-merge", - "developer": "khoantap", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/khoantap_llama-linear-1-0.5-0.5-merge/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4515 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5526 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2477 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2928 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4118 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3635 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/khoantap_llama-slerp-merge.json b/data/models/khoantap_llama-slerp-merge.json deleted file mode 100644 index 041b5b2db484195cf35679f266a7db7addb3805a..0000000000000000000000000000000000000000 --- a/data/models/khoantap_llama-slerp-merge.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "llama-slerp-merge", - "id": "khoantap/llama-slerp-merge", - "developer": "khoantap", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/khoantap_llama-slerp-merge/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.498 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5783 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0831 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3029 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4053 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3678 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/khoantap_moe-out-merge.json b/data/models/khoantap_moe-out-merge.json deleted file mode 100644 index aa1ffcfc07fa31130288fa6a3288017041163e54..0000000000000000000000000000000000000000 --- a/data/models/khoantap_moe-out-merge.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "moe-out-merge", - "id": "khoantap/moe-out-merge", - "developer": "khoantap", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2MoeForCausalLM", - "params_billions": "19.305" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/khoantap_moe-out-merge/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4505 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5151 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0929 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2886 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4063 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3348 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/khulaifi95_llama-3.1-8b-reason-blend-888k.json b/data/models/khulaifi95_llama-3.1-8b-reason-blend-888k.json deleted file mode 100644 index b95a0e41252f45f68606dad0d555ef4908102115..0000000000000000000000000000000000000000 --- a/data/models/khulaifi95_llama-3.1-8b-reason-blend-888k.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.1-8B-Reason-Blend-888k", - "id": "khulaifi95/Llama-3.1-8B-Reason-Blend-888k", - "developer": "khulaifi95", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/khulaifi95_Llama-3.1-8B-Reason-Blend-888k/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5832 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.479 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1156 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2794 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3379 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.31 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/kimargin_gpt-neo-1.3b-wiki.json b/data/models/kimargin_gpt-neo-1.3b-wiki.json deleted file mode 100644 index 0dd2a6884dfd885162fe8acfc229b895543ee200..0000000000000000000000000000000000000000 --- a/data/models/kimargin_gpt-neo-1.3b-wiki.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "GPT-NEO-1.3B-wiki", - "id": "Kimargin/GPT-NEO-1.3B-wiki", - "developer": "Kimargin", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "GPTNeoForCausalLM", - "params_billions": "1.316" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Kimargin_GPT-NEO-1.3B-wiki/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1921 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3026 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0144 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.245 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3883 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1099 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/kingnish_qwen-1b-continued-v2.1.json b/data/models/kingnish_qwen-1b-continued-v2.1.json deleted file mode 100644 index 069929350745dbf15351e156e505f20510c82760..0000000000000000000000000000000000000000 --- a/data/models/kingnish_qwen-1b-continued-v2.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "qwen-1b-continued-v2.1", - "id": "KingNish/qwen-1b-continued-v2.1", - "developer": "KingNish", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.277" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/KingNish_qwen-1b-continued-v2.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1127 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3042 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0091 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2676 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4154 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1278 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/kingnish_qwen-1b-continued-v2.2.json b/data/models/kingnish_qwen-1b-continued-v2.2.json deleted file mode 100644 index c27534cb3a798522526500f8cb0f540796fd88d4..0000000000000000000000000000000000000000 --- a/data/models/kingnish_qwen-1b-continued-v2.2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "qwen-1b-continued-v2.2", - "id": "KingNish/qwen-1b-continued-v2.2", - "developer": "KingNish", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.277" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/KingNish_qwen-1b-continued-v2.2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1413 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3059 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0151 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2567 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3513 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1262 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/kingnish_qwen-1b-continued-v2.json b/data/models/kingnish_qwen-1b-continued-v2.json deleted file mode 100644 index 7d6c9910e08bf4c742759f04e542d19370cfbd8b..0000000000000000000000000000000000000000 --- a/data/models/kingnish_qwen-1b-continued-v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "qwen-1b-continued-v2", - "id": "KingNish/qwen-1b-continued-v2", - "developer": "KingNish", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.277" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/KingNish_qwen-1b-continued-v2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1579 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3119 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0106 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.25 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3393 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1193 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/kingnish_qwen-1b-continued.json b/data/models/kingnish_qwen-1b-continued.json deleted file mode 100644 index b7ee4604b52e678ad9a1c9bcf44529b0157760b1..0000000000000000000000000000000000000000 --- a/data/models/kingnish_qwen-1b-continued.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "qwen-1b-continued", - "id": "KingNish/qwen-1b-continued", - "developer": "KingNish", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.277" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/KingNish_qwen-1b-continued/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1255 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2991 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0091 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2676 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3859 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1261 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/kingnish_qwen2.5-0.5b-test-ft.json b/data/models/kingnish_qwen2.5-0.5b-test-ft.json deleted file mode 100644 index 6409c45dfc26bc12c5cd189f51050ce1173d8cce..0000000000000000000000000000000000000000 --- a/data/models/kingnish_qwen2.5-0.5b-test-ft.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5b-Test-ft", - "id": "KingNish/Qwen2.5-0.5b-Test-ft", - "developer": "KingNish", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/KingNish_Qwen2.5-0.5b-Test-ft/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2671 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3232 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0355 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2634 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3421 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1689 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/kingnish_reasoning-0.5b.json b/data/models/kingnish_reasoning-0.5b.json deleted file mode 100644 index 6c8941791ac485c8e94878b272c662103dd4f08a..0000000000000000000000000000000000000000 --- a/data/models/kingnish_reasoning-0.5b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Reasoning-0.5b", - "id": "KingNish/Reasoning-0.5b", - "developer": "KingNish", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/KingNish_Reasoning-0.5b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2174 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3354 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0219 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2676 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3513 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1641 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/kingnish_reasoning-llama-3b-v0.1.json b/data/models/kingnish_reasoning-llama-3b-v0.1.json deleted file mode 100644 index 26b68e9be19bebaa9f264ced5f001cdf098bb3d7..0000000000000000000000000000000000000000 --- a/data/models/kingnish_reasoning-llama-3b-v0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Reasoning-Llama-3b-v0.1", - "id": "KingNish/Reasoning-Llama-3b-v0.1", - "developer": "KingNish", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/KingNish_Reasoning-Llama-3b-v0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6225 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4343 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1299 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2592 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3168 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3029 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/kms7530_chemeng_llama-3-8b-instruct-bnb-4bit_24_1_100_1.json b/data/models/kms7530_chemeng_llama-3-8b-instruct-bnb-4bit_24_1_100_1.json deleted file mode 100644 index f5705f566310947064f31dc2ff5697293643c9b0..0000000000000000000000000000000000000000 --- a/data/models/kms7530_chemeng_llama-3-8b-instruct-bnb-4bit_24_1_100_1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "chemeng_llama-3-8b-Instruct-bnb-4bit_24_1_100_1", - "id": "kms7530/chemeng_llama-3-8b-Instruct-bnb-4bit_24_1_100_1", - "developer": "kms7530", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "?", - "params_billions": "9.3" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/kms7530_chemeng_llama-3-8b-Instruct-bnb-4bit_24_1_100_1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5455 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4289 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0619 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2701 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3821 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2798 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/kms7530_chemeng_phi-3-mini-4k-instruct-bnb-4bit_16_4_100_1_nonmath.json b/data/models/kms7530_chemeng_phi-3-mini-4k-instruct-bnb-4bit_16_4_100_1_nonmath.json deleted file mode 100644 index deb504c547428a4b33381888262c418106c2b84d..0000000000000000000000000000000000000000 --- a/data/models/kms7530_chemeng_phi-3-mini-4k-instruct-bnb-4bit_16_4_100_1_nonmath.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "chemeng_phi-3-mini-4k-instruct-bnb-4bit_16_4_100_1_nonmath", - "id": "kms7530/chemeng_phi-3-mini-4k-instruct-bnb-4bit_16_4_100_1_nonmath", - "developer": "kms7530", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "?", - "params_billions": "4.132" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/kms7530_chemeng_phi-3-mini-4k-instruct-bnb-4bit_16_4_100_1_nonmath/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4863 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4987 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.108 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3104 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3983 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3481 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/kms7530_chemeng_qwen-math-7b_24_1_100_1.json b/data/models/kms7530_chemeng_qwen-math-7b_24_1_100_1.json deleted file mode 100644 index bc7c6e16fe9c252c3409628aa0229609299ede14..0000000000000000000000000000000000000000 --- a/data/models/kms7530_chemeng_qwen-math-7b_24_1_100_1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "chemeng_qwen-math-7b_24_1_100_1", - "id": "kms7530/chemeng_qwen-math-7b_24_1_100_1", - "developer": "kms7530", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "?", - "params_billions": "8.911" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/kms7530_chemeng_qwen-math-7b_24_1_100_1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2111 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3578 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2243 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2441 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3687 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2158 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/kms7530_chemeng_qwen-math-7b_24_1_100_1_nonmath.json b/data/models/kms7530_chemeng_qwen-math-7b_24_1_100_1_nonmath.json deleted file mode 100644 index 2387c6872f8aba8af0c15c681ad3df32e4e3d95b..0000000000000000000000000000000000000000 --- a/data/models/kms7530_chemeng_qwen-math-7b_24_1_100_1_nonmath.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "chemeng_qwen-math-7b_24_1_100_1_nonmath", - "id": "kms7530/chemeng_qwen-math-7b_24_1_100_1_nonmath", - "developer": "kms7530", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "?", - "params_billions": "15.231" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/kms7530_chemeng_qwen-math-7b_24_1_100_1_nonmath/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2584 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3893 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3097 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2903 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4087 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2452 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/kno10_ende-chat-0.0.5.json b/data/models/kno10_ende-chat-0.0.5.json deleted file mode 100644 index 4aa701356b5eb8b01edcf1938e65d911bb8e8551..0000000000000000000000000000000000000000 --- a/data/models/kno10_ende-chat-0.0.5.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ende-chat-0.0.5", - "id": "kno10/ende-chat-0.0.5", - "developer": "kno10", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "7.891" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/kno10_ende-chat-0.0.5/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3404 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3604 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0204 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2651 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3938 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.179 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/kno10_ende-chat-0.0.7.json b/data/models/kno10_ende-chat-0.0.7.json deleted file mode 100644 index 51b2b42fcdb3b90c6c5de629e9830f7b22adefcf..0000000000000000000000000000000000000000 --- a/data/models/kno10_ende-chat-0.0.7.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ende-chat-0.0.7", - "id": "kno10/ende-chat-0.0.7", - "developer": "kno10", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "7.891" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/kno10_ende-chat-0.0.7/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4401 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3792 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0174 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.281 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3861 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1966 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/kquant03_cognitivefusion2-4x7b-bf16.json b/data/models/kquant03_cognitivefusion2-4x7b-bf16.json deleted file mode 100644 index cdf84eb2303598a162a4aaa82c7b1e8659c65dbc..0000000000000000000000000000000000000000 --- a/data/models/kquant03_cognitivefusion2-4x7b-bf16.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "CognitiveFusion2-4x7B-BF16", - "id": "Kquant03/CognitiveFusion2-4x7B-BF16", - "developer": "Kquant03", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MixtralForCausalLM", - "params_billions": "24.154" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Kquant03_CognitiveFusion2-4x7B-BF16/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3567 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4108 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0574 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2861 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4146 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2793 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/kquant03_l3-pneuma-8b.json b/data/models/kquant03_l3-pneuma-8b.json deleted file mode 100644 index ec3818ac3eab13057b6da1dd21f1000d3be0ff8f..0000000000000000000000000000000000000000 --- a/data/models/kquant03_l3-pneuma-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "L3-Pneuma-8B", - "id": "Kquant03/L3-Pneuma-8B", - "developer": "Kquant03", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Kquant03_L3-Pneuma-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2374 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4955 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0506 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.307 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4172 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3184 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/krystalan_drt-o1-14b.json b/data/models/krystalan_drt-o1-14b.json deleted file mode 100644 index 2ced5a0d1a4aad4e391e673d0fb54a8b95579b26..0000000000000000000000000000000000000000 --- a/data/models/krystalan_drt-o1-14b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "DRT-o1-14B", - "id": "Krystalan/DRT-o1-14B", - "developer": "Krystalan", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Krystalan_DRT-o1-14B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4068 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6379 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4826 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3523 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4795 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5179 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/krystalan_drt-o1-7b.json b/data/models/krystalan_drt-o1-7b.json deleted file mode 100644 index 9916ceb3d6c66ea75c7d6657d353a4e2a4b686c2..0000000000000000000000000000000000000000 --- a/data/models/krystalan_drt-o1-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "DRT-o1-7B", - "id": "Krystalan/DRT-o1-7B", - "developer": "Krystalan", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Krystalan_DRT-o1-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3928 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5468 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4479 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3213 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5087 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4151 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ksu-hw-sec_llama3-70b-sva-ft-1415.json b/data/models/ksu-hw-sec_llama3-70b-sva-ft-1415.json deleted file mode 100644 index f843e855ec31118459fe23e2b21dcaf5078a2fa0..0000000000000000000000000000000000000000 --- a/data/models/ksu-hw-sec_llama3-70b-sva-ft-1415.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama3-70b-SVA-FT-1415", - "id": "KSU-HW-SEC/Llama3-70b-SVA-FT-1415", - "developer": "KSU-HW-SEC", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "70.554" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/KSU-HW-SEC_Llama3-70b-SVA-FT-1415/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.618 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.665 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2198 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.375 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4565 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5243 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ksu-hw-sec_llama3-70b-sva-ft-500.json b/data/models/ksu-hw-sec_llama3-70b-sva-ft-500.json deleted file mode 100644 index c499b14809bf26341c6b581214ecc22472214964..0000000000000000000000000000000000000000 --- a/data/models/ksu-hw-sec_llama3-70b-sva-ft-500.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama3-70b-SVA-FT-500", - "id": "KSU-HW-SEC/Llama3-70b-SVA-FT-500", - "developer": "KSU-HW-SEC", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "70.554" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/KSU-HW-SEC_Llama3-70b-SVA-FT-500/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6105 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6692 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2137 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3809 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4511 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5227 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ksu-hw-sec_llama3-70b-sva-ft-final.json b/data/models/ksu-hw-sec_llama3-70b-sva-ft-final.json deleted file mode 100644 index 5797a06dcd7da06c8adaf925427e1641143db418..0000000000000000000000000000000000000000 --- a/data/models/ksu-hw-sec_llama3-70b-sva-ft-final.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama3-70b-SVA-FT-final", - "id": "KSU-HW-SEC/Llama3-70b-SVA-FT-final", - "developer": "KSU-HW-SEC", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "70.554" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/KSU-HW-SEC_Llama3-70b-SVA-FT-final/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6165 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.665 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2198 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.375 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4565 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5243 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ksu-hw-sec_llama3.1-70b-sva-ft-1000step.json b/data/models/ksu-hw-sec_llama3.1-70b-sva-ft-1000step.json deleted file mode 100644 index 0445a52816bdbf2dcb6f1916cc5f022b06de6122..0000000000000000000000000000000000000000 --- a/data/models/ksu-hw-sec_llama3.1-70b-sva-ft-1000step.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama3.1-70b-SVA-FT-1000step", - "id": "KSU-HW-SEC/Llama3.1-70b-SVA-FT-1000step", - "developer": "KSU-HW-SEC", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "70.554" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/KSU-HW-SEC_Llama3.1-70b-SVA-FT-1000step/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7238 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6903 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.321 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.396 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4592 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5252 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/kuaishou_kwaipilot-40b-0604.json b/data/models/kuaishou_kwaipilot-40b-0604.json deleted file mode 100644 index 6f20496ad4fbb512b9ab04495e7cc5cf93a12e06..0000000000000000000000000000000000000000 --- a/data/models/kuaishou_kwaipilot-40b-0604.json +++ /dev/null @@ -1,89 +0,0 @@ -{ - "model_info": { - "name": "kwaipilot-40b-0604", - "developer": "Kuaishou", - "inference_platform": "kuaishou", - "id": "kuaishou/kwaipilot-40b-0604" - }, - "evaluations": [ - { - "evaluation_id": "livecodebenchpro/kwaipilot-40b-0604/1760492095.8105888", - "retrieved_timestamp": "1760492095.8105888", - "source_metadata": { - "source_organization_name": "New York University, Princeton University, University of California San Diego, University of Washington and Canyon Crest Academy", - "evaluator_relationship": "third_party", - "source_name": "Live Code Bench Pro", - "source_type": "documentation" - }, - "eval_library": { - "name": "unknown", - "version": "unknown" - }, - "benchmark": "livecodebenchpro", - "evaluation_results": [ - { - "evaluation_name": "Hard Problems", - "metric_config": { - "evaluation_description": "Pass@1 on Hard Problems", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.0 - }, - "source_data": { - "dataset_name": "Hard Problems", - "source_type": "url", - "url": [ - "https://webhook.cp-bench.orzzh.com/leaderboard/llm/difficulty?difficulty=hard&benchmark_mode=live" - ] - } - }, - { - "evaluation_name": "Medium Problems", - "metric_config": { - "evaluation_description": "Pass@1 on Medium Problems", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.07042253521126761 - }, - "source_data": { - "dataset_name": "Medium Problems", - "source_type": "url", - "url": [ - "https://webhook.cp-bench.orzzh.com/leaderboard/llm/difficulty?difficulty=medium&benchmark_mode=live" - ] - } - }, - { - "evaluation_name": "Easy Problems", - "metric_config": { - "evaluation_description": "Pass@1 on Easy Problems", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.056338028169014086 - }, - "source_data": { - "dataset_name": "Easy Problems", - "source_type": "url", - "url": [ - "https://webhook.cp-bench.orzzh.com/leaderboard/llm/difficulty?difficulty=easy&benchmark_mode=live" - ] - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/kukedlc_neuralexperiment-7b-magiccoder-v7.5.json b/data/models/kukedlc_neuralexperiment-7b-magiccoder-v7.5.json deleted file mode 100644 index 7ff343ee9b2715ea5b6672555281b44afc05d7cb..0000000000000000000000000000000000000000 --- a/data/models/kukedlc_neuralexperiment-7b-magiccoder-v7.5.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "NeuralExperiment-7b-MagicCoder-v7.5", - "id": "Kukedlc/NeuralExperiment-7b-MagicCoder-v7.5", - "developer": "Kukedlc", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Kukedlc_NeuralExperiment-7b-MagicCoder-v7.5/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4553 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3988 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0665 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2961 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4282 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2824 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/kukedlc_neuralllama-3-8b-dt-v0.1.json b/data/models/kukedlc_neuralllama-3-8b-dt-v0.1.json deleted file mode 100644 index 44e2bca42d6b6d91df1363ff9c1ad9703f42f2e3..0000000000000000000000000000000000000000 --- a/data/models/kukedlc_neuralllama-3-8b-dt-v0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "NeuralLLaMa-3-8b-DT-v0.1", - "id": "Kukedlc/NeuralLLaMa-3-8b-DT-v0.1", - "developer": "Kukedlc", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Kukedlc_NeuralLLaMa-3-8b-DT-v0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4371 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4987 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0808 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3029 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4071 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3792 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/kukedlc_neuralllama-3-8b-orpo-v0.3.json b/data/models/kukedlc_neuralllama-3-8b-orpo-v0.3.json deleted file mode 100644 index 9f48d07ca44dc3c3b54f1b84aa1eb9b7a5cb28d8..0000000000000000000000000000000000000000 --- a/data/models/kukedlc_neuralllama-3-8b-orpo-v0.3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "NeuralLLaMa-3-8b-ORPO-v0.3", - "id": "Kukedlc/NeuralLLaMa-3-8b-ORPO-v0.3", - "developer": "Kukedlc", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Kukedlc_NeuralLLaMa-3-8b-ORPO-v0.3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5276 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4557 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0483 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2391 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.37 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3057 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/kukedlc_neuralsynthesis-7b-v0.1.json b/data/models/kukedlc_neuralsynthesis-7b-v0.1.json deleted file mode 100644 index c0e7a810505964d92aedbcd7b8cc9eccf0662f67..0000000000000000000000000000000000000000 --- a/data/models/kukedlc_neuralsynthesis-7b-v0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "NeuralSynthesis-7B-v0.1", - "id": "Kukedlc/NeuralSynthesis-7B-v0.1", - "developer": "Kukedlc", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Kukedlc_NeuralSynthesis-7B-v0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4185 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5145 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0634 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.281 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4333 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3049 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/kukedlc_neuralsynthesis-7b-v0.3.json b/data/models/kukedlc_neuralsynthesis-7b-v0.3.json deleted file mode 100644 index e42ae94f96a6aa8df35b14cb23f7ede6f14a280b..0000000000000000000000000000000000000000 --- a/data/models/kukedlc_neuralsynthesis-7b-v0.3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "NeuralSynthesis-7B-v0.3", - "id": "Kukedlc/NeuralSynthesis-7B-v0.3", - "developer": "Kukedlc", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Kukedlc_NeuralSynthesis-7B-v0.3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4078 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5138 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0778 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2802 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4346 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.305 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/kukedlc_neuralsynthesis-7b-v0.4-slerp.json b/data/models/kukedlc_neuralsynthesis-7b-v0.4-slerp.json deleted file mode 100644 index f4aed43b518499e8daa6d41907a5e5758b3bd59f..0000000000000000000000000000000000000000 --- a/data/models/kukedlc_neuralsynthesis-7b-v0.4-slerp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "NeuralSynthesis-7b-v0.4-slerp", - "id": "Kukedlc/NeuralSynthesis-7b-v0.4-slerp", - "developer": "Kukedlc", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Kukedlc_NeuralSynthesis-7b-v0.4-slerp/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3947 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5143 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0627 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2777 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4332 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3043 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/kukedlc_qwen-2.5-7b-spanish-o1-cot.json b/data/models/kukedlc_qwen-2.5-7b-spanish-o1-cot.json deleted file mode 100644 index 23c0f1477e79c2d70268b32e5c4abf6325202606..0000000000000000000000000000000000000000 --- a/data/models/kukedlc_qwen-2.5-7b-spanish-o1-cot.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen-2.5-7b-Spanish-o1-CoT", - "id": "Kukedlc/Qwen-2.5-7b-Spanish-o1-CoT", - "developer": "Kukedlc", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Kukedlc_Qwen-2.5-7b-Spanish-o1-CoT/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.421 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5602 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2727 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3205 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4777 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4363 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/kumar955_hemanth-llm.json b/data/models/kumar955_hemanth-llm.json deleted file mode 100644 index 8adf4ebd3581c3c5bdea0a776d4f0696d617f191..0000000000000000000000000000000000000000 --- a/data/models/kumar955_hemanth-llm.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Hemanth-llm", - "id": "Kumar955/Hemanth-llm", - "developer": "Kumar955", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Kumar955_Hemanth-llm/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5045 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5225 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0702 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2827 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4486 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3113 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/kyutai_helium-1-preview-2b.json b/data/models/kyutai_helium-1-preview-2b.json deleted file mode 100644 index cdfb300eb14a86c4098c5fb7774adfebec1c2785..0000000000000000000000000000000000000000 --- a/data/models/kyutai_helium-1-preview-2b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "helium-1-preview-2b", - "id": "kyutai/helium-1-preview-2b", - "developer": "kyutai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "HeliumForCausalLM", - "params_billions": "2.173" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/kyutai_helium-1-preview-2b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2614 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3638 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0136 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2785 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.355 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1873 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/kz919_qwq-0.5b-distilled-sft.json b/data/models/kz919_qwq-0.5b-distilled-sft.json deleted file mode 100644 index 7fd1b51bbc75bdb2616b104a1b4a5d6bce2d62b1..0000000000000000000000000000000000000000 --- a/data/models/kz919_qwq-0.5b-distilled-sft.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "QwQ-0.5B-Distilled-SFT", - "id": "kz919/QwQ-0.5B-Distilled-SFT", - "developer": "kz919", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/kz919_QwQ-0.5B-Distilled-SFT/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3077 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3256 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.074 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2609 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3409 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1587 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/l-rage_3_prymmal-ece-7b-slerp-v1.json b/data/models/l-rage_3_prymmal-ece-7b-slerp-v1.json deleted file mode 100644 index 7315453cedfd5a8f9797e9ce7e887235a6461c7a..0000000000000000000000000000000000000000 --- a/data/models/l-rage_3_prymmal-ece-7b-slerp-v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "3_PRYMMAL-ECE-7B-SLERP-V1", - "id": "L-RAGE/3_PRYMMAL-ECE-7B-SLERP-V1", - "developer": "L-RAGE", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.777" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/L-RAGE_3_PRYMMAL-ECE-7B-SLERP-V1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2742 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4228 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.108 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2819 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3841 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2925 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ladydaina_ece-fdf.json b/data/models/ladydaina_ece-fdf.json deleted file mode 100644 index 4bd8f8df360f0c21dc51fc0364ca3e1471f99415..0000000000000000000000000000000000000000 --- a/data/models/ladydaina_ece-fdf.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ECE-FDF", - "id": "ladydaina/ECE-FDF", - "developer": "ladydaina", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ladydaina_ECE-FDF/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3728 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.515 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0816 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2827 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4504 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3007 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/laislemke_llama-2-vicuna-7b-slerp.json b/data/models/laislemke_llama-2-vicuna-7b-slerp.json deleted file mode 100644 index d2baeea1147d22454ecc4f2ba5be02b28d78a169..0000000000000000000000000000000000000000 --- a/data/models/laislemke_llama-2-vicuna-7b-slerp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "LLaMA-2-vicuna-7b-slerp", - "id": "laislemke/LLaMA-2-vicuna-7b-slerp", - "developer": "laislemke", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "6.738" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/laislemke_LLaMA-2-vicuna-7b-slerp/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2932 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2986 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0113 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2735 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3833 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1342 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lalainy_ece-prymmal-0.5b-ft-v5-musr.json b/data/models/lalainy_ece-prymmal-0.5b-ft-v5-musr.json deleted file mode 100644 index 64c2fcb6bd5825ce2f03d9ca0e485dfe80eef8b3..0000000000000000000000000000000000000000 --- a/data/models/lalainy_ece-prymmal-0.5b-ft-v5-musr.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ECE-PRYMMAL-0.5B-FT-V5-MUSR", - "id": "lalainy/ECE-PRYMMAL-0.5B-FT-V5-MUSR", - "developer": "lalainy", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/lalainy_ECE-PRYMMAL-0.5B-FT-V5-MUSR/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2138 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3269 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0453 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2743 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3262 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1533 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lalainy_ece-prymmal-0.5b-slerp-v4.json b/data/models/lalainy_ece-prymmal-0.5b-slerp-v4.json deleted file mode 100644 index ee69bd098efe6fc2f21a75eafab0debcc9bb4eab..0000000000000000000000000000000000000000 --- a/data/models/lalainy_ece-prymmal-0.5b-slerp-v4.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ECE-PRYMMAL-0.5B-SLERP-V4", - "id": "lalainy/ECE-PRYMMAL-0.5B-SLERP-V4", - "developer": "lalainy", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/lalainy_ECE-PRYMMAL-0.5B-SLERP-V4/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1564 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2894 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2626 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3789 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1169 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lalainy_ece-prymmal-yl-0.5b-slerp-bis-v1.json b/data/models/lalainy_ece-prymmal-yl-0.5b-slerp-bis-v1.json deleted file mode 100644 index b4a6e80b78adf68290b3e8387e08e7511272a349..0000000000000000000000000000000000000000 --- a/data/models/lalainy_ece-prymmal-yl-0.5b-slerp-bis-v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ECE-PRYMMAL-YL-0.5B-SLERP-BIS-V1", - "id": "lalainy/ECE-PRYMMAL-YL-0.5B-SLERP-BIS-V1", - "developer": "lalainy", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/lalainy_ECE-PRYMMAL-YL-0.5B-SLERP-BIS-V1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1437 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3032 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0008 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2349 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3646 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1121 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lalainy_ece-prymmal-yl-1b-slerp-v3.json b/data/models/lalainy_ece-prymmal-yl-1b-slerp-v3.json deleted file mode 100644 index c427daf156e021bbc782f9d63e2a86a4e68a0148..0000000000000000000000000000000000000000 --- a/data/models/lalainy_ece-prymmal-yl-1b-slerp-v3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ECE-PRYMMAL-YL-1B-SLERP-V3", - "id": "lalainy/ECE-PRYMMAL-YL-1B-SLERP-V3", - "developer": "lalainy", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.544" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/lalainy_ECE-PRYMMAL-YL-1B-SLERP-V3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.325 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4225 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0974 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2945 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4213 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2931 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lalainy_ece-prymmal-yl-1b-slerp-v4.json b/data/models/lalainy_ece-prymmal-yl-1b-slerp-v4.json deleted file mode 100644 index fc9b5e6aaec1c9e2ed0fff41df57134751ed1e3c..0000000000000000000000000000000000000000 --- a/data/models/lalainy_ece-prymmal-yl-1b-slerp-v4.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ECE-PRYMMAL-YL-1B-SLERP-V4", - "id": "lalainy/ECE-PRYMMAL-YL-1B-SLERP-V4", - "developer": "lalainy", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.544" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/lalainy_ECE-PRYMMAL-YL-1B-SLERP-V4/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3324 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4171 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1005 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2861 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4306 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2893 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lalainy_ece-prymmal-yl-6b-slerp-v1.json b/data/models/lalainy_ece-prymmal-yl-6b-slerp-v1.json deleted file mode 100644 index dd97851706cb471e1342e4ebc87e43c946ab7500..0000000000000000000000000000000000000000 --- a/data/models/lalainy_ece-prymmal-yl-6b-slerp-v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ECE-PRYMMAL-YL-6B-SLERP-V1", - "id": "lalainy/ECE-PRYMMAL-YL-6B-SLERP-V1", - "developer": "lalainy", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "6.061" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/lalainy_ECE-PRYMMAL-YL-6B-SLERP-V1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3264 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4629 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1269 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2886 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4864 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3214 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lalainy_ece-prymmal-yl-6b-slerp-v2.json b/data/models/lalainy_ece-prymmal-yl-6b-slerp-v2.json deleted file mode 100644 index 4161675f1f6ae2bd55cd6adcd6b954632087a1d2..0000000000000000000000000000000000000000 --- a/data/models/lalainy_ece-prymmal-yl-6b-slerp-v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ECE-PRYMMAL-YL-6B-SLERP-V2", - "id": "lalainy/ECE-PRYMMAL-YL-6B-SLERP-V2", - "developer": "lalainy", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "6.061" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/lalainy_ECE-PRYMMAL-YL-6B-SLERP-V2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3249 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4629 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1269 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2886 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4864 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3214 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lambent_qwen2.5-reinstruct-alternate-lumen-14b.json b/data/models/lambent_qwen2.5-reinstruct-alternate-lumen-14b.json deleted file mode 100644 index 12ea7f607620785be210ee91281b48a5bf983ab9..0000000000000000000000000000000000000000 --- a/data/models/lambent_qwen2.5-reinstruct-alternate-lumen-14b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "qwen2.5-reinstruct-alternate-lumen-14B", - "id": "Lambent/qwen2.5-reinstruct-alternate-lumen-14B", - "developer": "Lambent", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Lambent_qwen2.5-reinstruct-alternate-lumen-14B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4794 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6459 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4622 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3767 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.477 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5388 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/langboat_mengzi3-8b-chat.json b/data/models/langboat_mengzi3-8b-chat.json deleted file mode 100644 index e39e49fb76437a14732fda6ffa96170b2a39fe3d..0000000000000000000000000000000000000000 --- a/data/models/langboat_mengzi3-8b-chat.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mengzi3-8B-Chat", - "id": "Langboat/Mengzi3-8B-Chat", - "developer": "Langboat", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Langboat_Mengzi3-8B-Chat/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.514 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4684 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0906 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2743 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4078 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3142 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/langgptai_qwen-las-v0.1.json b/data/models/langgptai_qwen-las-v0.1.json deleted file mode 100644 index 5a13c051dd64d484b57042d5e3fa70266b338b2f..0000000000000000000000000000000000000000 --- a/data/models/langgptai_qwen-las-v0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen-las-v0.1", - "id": "langgptai/Qwen-las-v0.1", - "developer": "langgptai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "?", - "params_billions": "7.901" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/langgptai_Qwen-las-v0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3301 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3893 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.037 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2466 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3701 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2325 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/langgptai_qwen1.5-7b-chat-sa-v0.1.json b/data/models/langgptai_qwen1.5-7b-chat-sa-v0.1.json deleted file mode 100644 index 8a2b325f4b0d0d80275892bcf3ce20380b87854e..0000000000000000000000000000000000000000 --- a/data/models/langgptai_qwen1.5-7b-chat-sa-v0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "qwen1.5-7b-chat-sa-v0.1", - "id": "langgptai/qwen1.5-7b-chat-sa-v0.1", - "developer": "langgptai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "?", - "params_billions": "15.443" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/langgptai_qwen1.5-7b-chat-sa-v0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4268 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4325 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0302 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3121 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3551 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2993 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lars1234_mistral-small-24b-instruct-2501-writer.json b/data/models/lars1234_mistral-small-24b-instruct-2501-writer.json deleted file mode 100644 index e4b2b11de3b299ba4d51bed5a9d5b11d24c32911..0000000000000000000000000000000000000000 --- a/data/models/lars1234_mistral-small-24b-instruct-2501-writer.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mistral-Small-24B-Instruct-2501-writer", - "id": "lars1234/Mistral-Small-24B-Instruct-2501-writer", - "developer": "lars1234", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "23.572" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/lars1234_Mistral-Small-24B-Instruct-2501-writer/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6565 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6733 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3557 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3893 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4645 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5448 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lawnakk_bba100.json b/data/models/lawnakk_bba100.json deleted file mode 100644 index 0961caf39425eb63169badf6a0aac6cb86afe784..0000000000000000000000000000000000000000 --- a/data/models/lawnakk_bba100.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "BBA100", - "id": "Lawnakk/BBA100", - "developer": "Lawnakk", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Lawnakk_BBA100/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2076 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2826 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0098 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2441 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.402 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1122 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lawnakk_bbalaw1.0.json b/data/models/lawnakk_bbalaw1.0.json deleted file mode 100644 index 4cac0d2fa7f93367c6a282bc43199d2936232dac..0000000000000000000000000000000000000000 --- a/data/models/lawnakk_bbalaw1.0.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "BBALAW1.0", - "id": "Lawnakk/BBALAW1.0", - "developer": "Lawnakk", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "4.353" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Lawnakk_BBALAW1.0/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1351 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2828 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2559 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3526 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1128 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lawnakk_bbalaw1.2.json b/data/models/lawnakk_bbalaw1.2.json deleted file mode 100644 index d62320811fde425aabac6fe4b8bd5dfa5a6eea48..0000000000000000000000000000000000000000 --- a/data/models/lawnakk_bbalaw1.2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "BBALAW1.2", - "id": "Lawnakk/BBALAW1.2", - "developer": "Lawnakk", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "4.353" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Lawnakk_BBALAW1.2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1354 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2811 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2643 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3579 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1105 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lawnakk_bbalaw1.3.json b/data/models/lawnakk_bbalaw1.3.json deleted file mode 100644 index a2f01b72750aa888911cfe101c7ec073007cb608..0000000000000000000000000000000000000000 --- a/data/models/lawnakk_bbalaw1.3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "BBALAW1.3", - "id": "Lawnakk/BBALAW1.3", - "developer": "Lawnakk", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "4.353" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Lawnakk_BBALAW1.3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1354 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2827 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2609 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3619 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1094 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lawnakk_bbalaw1.6.json b/data/models/lawnakk_bbalaw1.6.json deleted file mode 100644 index d06ebce29b80c528558ecd57d4e2189683ff063d..0000000000000000000000000000000000000000 --- a/data/models/lawnakk_bbalaw1.6.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "BBALAW1.6", - "id": "Lawnakk/BBALAW1.6", - "developer": "Lawnakk", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Lawnakk_BBALAW1.6/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5245 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5554 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3603 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3238 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4368 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4507 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lawnakk_bbalaw1.61.json b/data/models/lawnakk_bbalaw1.61.json deleted file mode 100644 index 8e85e7f53952d4b67579803065f7e5166b68f068..0000000000000000000000000000000000000000 --- a/data/models/lawnakk_bbalaw1.61.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "BBALAW1.61", - "id": "Lawnakk/BBALAW1.61", - "developer": "Lawnakk", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Lawnakk_BBALAW1.61/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5771 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5549 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3663 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3171 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4355 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4471 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lawnakk_bbalaw1.62.json b/data/models/lawnakk_bbalaw1.62.json deleted file mode 100644 index 4f5fd67ca254a25c0096474d5f04bf6cc1836087..0000000000000000000000000000000000000000 --- a/data/models/lawnakk_bbalaw1.62.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "BBALAW1.62", - "id": "Lawnakk/BBALAW1.62", - "developer": "Lawnakk", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Lawnakk_BBALAW1.62/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5046 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5581 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2825 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3196 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4343 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4545 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lawnakk_bbalaw1.63.json b/data/models/lawnakk_bbalaw1.63.json deleted file mode 100644 index 0b2bbdf26e2ab9eec6d0f0b486d1c045215f26c3..0000000000000000000000000000000000000000 --- a/data/models/lawnakk_bbalaw1.63.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "BBALAW1.63", - "id": "Lawnakk/BBALAW1.63", - "developer": "Lawnakk", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.613" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Lawnakk_BBALAW1.63/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4407 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5541 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3701 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3121 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4303 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4471 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lawnakk_bbalaw1.64.json b/data/models/lawnakk_bbalaw1.64.json deleted file mode 100644 index 78942268c12758cfaab776ac7673bbbde1f6736a..0000000000000000000000000000000000000000 --- a/data/models/lawnakk_bbalaw1.64.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "BBALAW1.64", - "id": "Lawnakk/BBALAW1.64", - "developer": "Lawnakk", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Lawnakk_BBALAW1.64/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1395 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2779 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2483 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3447 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1115 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lawnakk_bbalaw1.json b/data/models/lawnakk_bbalaw1.json deleted file mode 100644 index 489f89b89a3af969a0618295b84e675bc7b0a1c5..0000000000000000000000000000000000000000 --- a/data/models/lawnakk_bbalaw1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "BBALAW1", - "id": "Lawnakk/BBALAW1", - "developer": "Lawnakk", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Lawnakk_BBALAW1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1905 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2872 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0098 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2433 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4153 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1121 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/leafspark_llama-3.1-8b-multireflection-instruct.json b/data/models/leafspark_llama-3.1-8b-multireflection-instruct.json deleted file mode 100644 index 472aae12766ec95ebca42a3e8b4d58546f7bb60e..0000000000000000000000000000000000000000 --- a/data/models/leafspark_llama-3.1-8b-multireflection-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.1-8B-MultiReflection-Instruct", - "id": "leafspark/Llama-3.1-8B-MultiReflection-Instruct", - "developer": "leafspark", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/leafspark_Llama-3.1-8B-MultiReflection-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7125 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5009 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1707 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2928 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3682 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3724 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/leesm_llama-2-7b-hf-lora-oki100p.json b/data/models/leesm_llama-2-7b-hf-lora-oki100p.json deleted file mode 100644 index 40ee5d94dcacfa118a294b8d6ba52e85a708e35f..0000000000000000000000000000000000000000 --- a/data/models/leesm_llama-2-7b-hf-lora-oki100p.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "llama-2-7b-hf-lora-oki100p", - "id": "LEESM/llama-2-7b-hf-lora-oki100p", - "developer": "LEESM", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "6.738" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/LEESM_llama-2-7b-hf-lora-oki100p/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2513 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3492 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0166 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2693 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3687 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1856 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/leesm_llama-2-7b-hf-lora-oki10p.json b/data/models/leesm_llama-2-7b-hf-lora-oki10p.json deleted file mode 100644 index 4a29cdc9ee1465cc63dc283ccdbfc1775feee926..0000000000000000000000000000000000000000 --- a/data/models/leesm_llama-2-7b-hf-lora-oki10p.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "llama-2-7b-hf-lora-oki10p", - "id": "LEESM/llama-2-7b-hf-lora-oki10p", - "developer": "LEESM", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "6.738" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/LEESM_llama-2-7b-hf-lora-oki10p/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.227 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3531 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0166 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2542 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3475 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1679 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/leesm_llama-3-8b-bnb-4b-kowiki231101.json b/data/models/leesm_llama-3-8b-bnb-4b-kowiki231101.json deleted file mode 100644 index ef22b7bcc9270c304f1093a096446b582129fbd7..0000000000000000000000000000000000000000 --- a/data/models/leesm_llama-3-8b-bnb-4b-kowiki231101.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "llama-3-8b-bnb-4b-kowiki231101", - "id": "LEESM/llama-3-8b-bnb-4b-kowiki231101", - "developer": "LEESM", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/LEESM_llama-3-8b-bnb-4b-kowiki231101/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1685 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4131 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0136 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.271 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3551 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2425 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/leesm_llama-3-korean-bllossom-8b-trexlab-oki10p.json b/data/models/leesm_llama-3-korean-bllossom-8b-trexlab-oki10p.json deleted file mode 100644 index 350f8320dc7cbf6a4c1a249555bacae0e9ebcca8..0000000000000000000000000000000000000000 --- a/data/models/leesm_llama-3-korean-bllossom-8b-trexlab-oki10p.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "llama-3-Korean-Bllossom-8B-trexlab-oki10p", - "id": "LEESM/llama-3-Korean-Bllossom-8B-trexlab-oki10p", - "developer": "LEESM", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/LEESM_llama-3-Korean-Bllossom-8B-trexlab-oki10p/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2137 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4343 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0468 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2752 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3869 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3177 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lemon07r_gemma-2-ataraxy-9b.json b/data/models/lemon07r_gemma-2-ataraxy-9b.json deleted file mode 100644 index 73e8bfeb4ac97f8e9e64fc15127d6f4684f5a16e..0000000000000000000000000000000000000000 --- a/data/models/lemon07r_gemma-2-ataraxy-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Gemma-2-Ataraxy-9B", - "id": "lemon07r/Gemma-2-Ataraxy-9B", - "developer": "lemon07r", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/lemon07r_Gemma-2-Ataraxy-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3009 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5931 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0853 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3347 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4424 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4226 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lemon07r_gemma-2-ataraxy-advanced-9b.json b/data/models/lemon07r_gemma-2-ataraxy-advanced-9b.json deleted file mode 100644 index c15a867660a9e8b2809b7fbf73eac1d9b34868ad..0000000000000000000000000000000000000000 --- a/data/models/lemon07r_gemma-2-ataraxy-advanced-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Gemma-2-Ataraxy-Advanced-9B", - "id": "lemon07r/Gemma-2-Ataraxy-Advanced-9B", - "developer": "lemon07r", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/lemon07r_Gemma-2-Ataraxy-Advanced-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5516 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5889 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1979 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3356 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3761 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4244 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lemon07r_gemma-2-ataraxy-remix-9b.json b/data/models/lemon07r_gemma-2-ataraxy-remix-9b.json deleted file mode 100644 index 5ff69b5b94bd65326bf1184aba829ef4a840978d..0000000000000000000000000000000000000000 --- a/data/models/lemon07r_gemma-2-ataraxy-remix-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Gemma-2-Ataraxy-Remix-9B", - "id": "lemon07r/Gemma-2-Ataraxy-Remix-9B", - "developer": "lemon07r", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/lemon07r_Gemma-2-Ataraxy-Remix-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7083 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5892 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2017 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3389 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4372 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4239 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lemon07r_gemma-2-ataraxy-v2-9b.json b/data/models/lemon07r_gemma-2-ataraxy-v2-9b.json deleted file mode 100644 index 2341bfae4eee42ec921afb4aa611d484e5c909bc..0000000000000000000000000000000000000000 --- a/data/models/lemon07r_gemma-2-ataraxy-v2-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Gemma-2-Ataraxy-v2-9B", - "id": "lemon07r/Gemma-2-Ataraxy-v2-9B", - "developer": "lemon07r", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/lemon07r_Gemma-2-Ataraxy-v2-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2136 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5766 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0846 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3423 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3484 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4221 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lemon07r_gemma-2-ataraxy-v2a-9b.json b/data/models/lemon07r_gemma-2-ataraxy-v2a-9b.json deleted file mode 100644 index 50219074cbfc00143042a9d6a4df23f8bf633478..0000000000000000000000000000000000000000 --- a/data/models/lemon07r_gemma-2-ataraxy-v2a-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Gemma-2-Ataraxy-v2a-9B", - "id": "lemon07r/Gemma-2-Ataraxy-v2a-9B", - "developer": "lemon07r", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/lemon07r_Gemma-2-Ataraxy-v2a-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1595 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5182 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0612 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3398 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3165 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3515 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lemon07r_gemma-2-ataraxy-v2f-9b.json b/data/models/lemon07r_gemma-2-ataraxy-v2f-9b.json deleted file mode 100644 index 2907434bdcc9aefa35c056df4fb18df2f052019e..0000000000000000000000000000000000000000 --- a/data/models/lemon07r_gemma-2-ataraxy-v2f-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Gemma-2-Ataraxy-v2f-9B", - "id": "lemon07r/Gemma-2-Ataraxy-v2f-9B", - "developer": "lemon07r", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/lemon07r_Gemma-2-Ataraxy-v2f-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3791 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5193 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1163 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3389 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3231 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3503 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lemon07r_gemma-2-ataraxy-v3-advanced-9b.json b/data/models/lemon07r_gemma-2-ataraxy-v3-advanced-9b.json deleted file mode 100644 index ec7e0d9afabb46a652cf85376139625d0bc8f01d..0000000000000000000000000000000000000000 --- a/data/models/lemon07r_gemma-2-ataraxy-v3-advanced-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Gemma-2-Ataraxy-v3-Advanced-9B", - "id": "lemon07r/Gemma-2-Ataraxy-v3-Advanced-9B", - "developer": "lemon07r", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/lemon07r_Gemma-2-Ataraxy-v3-Advanced-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6602 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5935 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1873 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3364 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.445 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4196 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lemon07r_gemma-2-ataraxy-v3b-9b.json b/data/models/lemon07r_gemma-2-ataraxy-v3b-9b.json deleted file mode 100644 index afa67b2486cd77c143882ffc76e413412613a63f..0000000000000000000000000000000000000000 --- a/data/models/lemon07r_gemma-2-ataraxy-v3b-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Gemma-2-Ataraxy-v3b-9B", - "id": "lemon07r/Gemma-2-Ataraxy-v3b-9B", - "developer": "lemon07r", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "9.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/lemon07r_Gemma-2-Ataraxy-v3b-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6809 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5908 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2153 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3331 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4489 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4205 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lemon07r_gemma-2-ataraxy-v3i-9b.json b/data/models/lemon07r_gemma-2-ataraxy-v3i-9b.json deleted file mode 100644 index 8a10dc15699fd3488e9f90d24b7e637dc65324ef..0000000000000000000000000000000000000000 --- a/data/models/lemon07r_gemma-2-ataraxy-v3i-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Gemma-2-Ataraxy-v3i-9B", - "id": "lemon07r/Gemma-2-Ataraxy-v3i-9B", - "developer": "lemon07r", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "9.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/lemon07r_Gemma-2-Ataraxy-v3i-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4203 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5626 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1533 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.328 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3181 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4166 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lemon07r_gemma-2-ataraxy-v3j-9b.json b/data/models/lemon07r_gemma-2-ataraxy-v3j-9b.json deleted file mode 100644 index b02d48a55cd344bf81932210f11aea2c31989acc..0000000000000000000000000000000000000000 --- a/data/models/lemon07r_gemma-2-ataraxy-v3j-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Gemma-2-Ataraxy-v3j-9B", - "id": "lemon07r/Gemma-2-Ataraxy-v3j-9B", - "developer": "lemon07r", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "9.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/lemon07r_Gemma-2-Ataraxy-v3j-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4169 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5632 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1692 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.328 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.318 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4134 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lemon07r_gemma-2-ataraxy-v4-advanced-9b.json b/data/models/lemon07r_gemma-2-ataraxy-v4-advanced-9b.json deleted file mode 100644 index f761ef70b471e586924ef2e91d8bf8dff9617c68..0000000000000000000000000000000000000000 --- a/data/models/lemon07r_gemma-2-ataraxy-v4-advanced-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Gemma-2-Ataraxy-v4-Advanced-9B", - "id": "lemon07r/Gemma-2-Ataraxy-v4-Advanced-9B", - "developer": "lemon07r", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/lemon07r_Gemma-2-Ataraxy-v4-Advanced-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7015 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6024 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2153 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3389 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4581 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4367 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lemon07r_gemma-2-ataraxy-v4a-advanced-9b.json b/data/models/lemon07r_gemma-2-ataraxy-v4a-advanced-9b.json deleted file mode 100644 index 65dff637f59022a59a20af7ed20336ff48a1ac8e..0000000000000000000000000000000000000000 --- a/data/models/lemon07r_gemma-2-ataraxy-v4a-advanced-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Gemma-2-Ataraxy-v4a-Advanced-9B", - "id": "lemon07r/Gemma-2-Ataraxy-v4a-Advanced-9B", - "developer": "lemon07r", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/lemon07r_Gemma-2-Ataraxy-v4a-Advanced-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7135 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5988 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2115 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.344 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4489 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4309 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lemon07r_gemma-2-ataraxy-v4b-9b.json b/data/models/lemon07r_gemma-2-ataraxy-v4b-9b.json deleted file mode 100644 index 466f466cad9e2495fd0c0ecc44715a4650a602f1..0000000000000000000000000000000000000000 --- a/data/models/lemon07r_gemma-2-ataraxy-v4b-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Gemma-2-Ataraxy-v4b-9B", - "id": "lemon07r/Gemma-2-Ataraxy-v4b-9B", - "developer": "lemon07r", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/lemon07r_Gemma-2-Ataraxy-v4b-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6878 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6039 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2334 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3406 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4555 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4357 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lemon07r_gemma-2-ataraxy-v4c-9b.json b/data/models/lemon07r_gemma-2-ataraxy-v4c-9b.json deleted file mode 100644 index ebda71cdd9e0c5cf1d69dc3f43d7a1cbe5623758..0000000000000000000000000000000000000000 --- a/data/models/lemon07r_gemma-2-ataraxy-v4c-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Gemma-2-Ataraxy-v4c-9B", - "id": "lemon07r/Gemma-2-Ataraxy-v4c-9B", - "developer": "lemon07r", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/lemon07r_Gemma-2-Ataraxy-v4c-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6945 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6084 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2266 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3339 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4528 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4395 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lemon07r_gemma-2-ataraxy-v4d-9b.json b/data/models/lemon07r_gemma-2-ataraxy-v4d-9b.json deleted file mode 100644 index afe4343fd1e6f3626539b8b84e56739cf698691d..0000000000000000000000000000000000000000 --- a/data/models/lemon07r_gemma-2-ataraxy-v4d-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Gemma-2-Ataraxy-v4d-9B", - "id": "lemon07r/Gemma-2-Ataraxy-v4d-9B", - "developer": "lemon07r", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/lemon07r_Gemma-2-Ataraxy-v4d-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.725 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6054 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2334 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3473 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4541 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4346 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lemon07r_llama-3-neuralmahou-8b.json b/data/models/lemon07r_llama-3-neuralmahou-8b.json deleted file mode 100644 index 30976bf859267c5a1a5c6b2523e404b24c60492d..0000000000000000000000000000000000000000 --- a/data/models/lemon07r_llama-3-neuralmahou-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "llama-3-NeuralMahou-8b", - "id": "lemon07r/llama-3-NeuralMahou-8b", - "developer": "lemon07r", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/lemon07r_llama-3-NeuralMahou-8b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4901 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4184 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.102 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2886 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3873 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.369 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lemon07r_llama-3-redmagic4-8b.json b/data/models/lemon07r_llama-3-redmagic4-8b.json deleted file mode 100644 index a0bb6290985299f9a1781a602b29b7a661922dd9..0000000000000000000000000000000000000000 --- a/data/models/lemon07r_llama-3-redmagic4-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-RedMagic4-8B", - "id": "lemon07r/Llama-3-RedMagic4-8B", - "developer": "lemon07r", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/lemon07r_Llama-3-RedMagic4-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4864 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4256 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0899 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2903 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3766 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3676 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lenguajenaturalai_leniachat-gemma-2b-v0.json b/data/models/lenguajenaturalai_leniachat-gemma-2b-v0.json deleted file mode 100644 index 403447366ac1e9b4dac0af37cc3a5e2c0d26313d..0000000000000000000000000000000000000000 --- a/data/models/lenguajenaturalai_leniachat-gemma-2b-v0.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "leniachat-gemma-2b-v0", - "id": "LenguajeNaturalAI/leniachat-gemma-2b-v0", - "developer": "LenguajeNaturalAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "GemmaForCausalLM", - "params_billions": "2.506" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/LenguajeNaturalAI_leniachat-gemma-2b-v0/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.215 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3074 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0113 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2659 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3659 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.117 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lenguajenaturalai_leniachat-qwen2-1.5b-v0.json b/data/models/lenguajenaturalai_leniachat-qwen2-1.5b-v0.json deleted file mode 100644 index 53d46251c80ead49ec561177ed3914d657f1e5d2..0000000000000000000000000000000000000000 --- a/data/models/lenguajenaturalai_leniachat-qwen2-1.5b-v0.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "leniachat-qwen2-1.5B-v0", - "id": "LenguajeNaturalAI/leniachat-qwen2-1.5B-v0", - "developer": "LenguajeNaturalAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.543" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/LenguajeNaturalAI_leniachat-qwen2-1.5B-v0/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2221 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3684 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0128 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2617 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.375 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.188 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/leroydyer__spydaz_web_ai_12.json b/data/models/leroydyer__spydaz_web_ai_12.json deleted file mode 100644 index f0fd629840ff374d4a31d02a4a51faaf21e03f0f..0000000000000000000000000000000000000000 --- a/data/models/leroydyer__spydaz_web_ai_12.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "_Spydaz_Web_AI_12", - "id": "LeroyDyer/_Spydaz_Web_AI_12", - "developer": "LeroyDyer", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/LeroyDyer__Spydaz_Web_AI_12/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2765 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3163 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0136 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2685 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3582 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1137 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/leroydyer__spydaz_web_ai_14.json b/data/models/leroydyer__spydaz_web_ai_14.json deleted file mode 100644 index 05448001356231cbd49cfa9fb2a3be53959d8d59..0000000000000000000000000000000000000000 --- a/data/models/leroydyer__spydaz_web_ai_14.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "_Spydaz_Web_AI_14", - "id": "LeroyDyer/_Spydaz_Web_AI_14", - "developer": "LeroyDyer", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/LeroyDyer__Spydaz_Web_AI_14/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1812 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2989 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0121 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2659 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3395 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1139 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/leroydyer__spydaz_web_ai_agi_r1_001.json b/data/models/leroydyer__spydaz_web_ai_agi_r1_001.json deleted file mode 100644 index 43fefe99d7e1550e05c13b72edbbed76c591ebba..0000000000000000000000000000000000000000 --- a/data/models/leroydyer__spydaz_web_ai_agi_r1_001.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "_Spydaz_Web_AI_AGI_R1_001", - "id": "LeroyDyer/_Spydaz_Web_AI_AGI_R1_001", - "developer": "LeroyDyer", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/LeroyDyer__Spydaz_Web_AI_AGI_R1_001/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4505 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4609 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0634 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2676 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4256 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2734 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/leroydyer__spydaz_web_ai_agi_r1_002.json b/data/models/leroydyer__spydaz_web_ai_agi_r1_002.json deleted file mode 100644 index f7599c13f1f2b652a5f7162ff5ceb39a9e91add3..0000000000000000000000000000000000000000 --- a/data/models/leroydyer__spydaz_web_ai_agi_r1_002.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "_Spydaz_Web_AI_AGI_R1_002", - "id": "LeroyDyer/_Spydaz_Web_AI_AGI_R1_002", - "developer": "LeroyDyer", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/LeroyDyer__Spydaz_Web_AI_AGI_R1_002/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5307 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4683 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0582 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2685 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4255 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2894 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/leroydyer__spydaz_web_ai_agi_r1_mastercoder.json b/data/models/leroydyer__spydaz_web_ai_agi_r1_mastercoder.json deleted file mode 100644 index c70c86449382a4b86c79875de9862a1ba2fea0be..0000000000000000000000000000000000000000 --- a/data/models/leroydyer__spydaz_web_ai_agi_r1_mastercoder.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "_Spydaz_Web_AI_AGI_R1_MasterCoder", - "id": "LeroyDyer/_Spydaz_Web_AI_AGI_R1_MasterCoder", - "developer": "LeroyDyer", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/LeroyDyer__Spydaz_Web_AI_AGI_R1_MasterCoder/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4143 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4689 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0612 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.276 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.472 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2719 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/leroydyer__spydaz_web_ai_agi_r1_math_001.json b/data/models/leroydyer__spydaz_web_ai_agi_r1_math_001.json deleted file mode 100644 index 3e27e4b2bab9f99b70e10952776aed718b27fe88..0000000000000000000000000000000000000000 --- a/data/models/leroydyer__spydaz_web_ai_agi_r1_math_001.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "_Spydaz_Web_AI_AGI_R1_Math_001", - "id": "LeroyDyer/_Spydaz_Web_AI_AGI_R1_Math_001", - "developer": "LeroyDyer", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/LeroyDyer__Spydaz_Web_AI_AGI_R1_Math_001/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4571 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4818 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0695 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2768 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4778 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2681 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/leroydyer__spydaz_web_ai_agi_r1_math_003.json b/data/models/leroydyer__spydaz_web_ai_agi_r1_math_003.json deleted file mode 100644 index ce45b14333bdad7728527720fe19446a736d5fd6..0000000000000000000000000000000000000000 --- a/data/models/leroydyer__spydaz_web_ai_agi_r1_math_003.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "_Spydaz_Web_AI_AGI_R1_Math_003", - "id": "LeroyDyer/_Spydaz_Web_AI_AGI_R1_Math_003", - "developer": "LeroyDyer", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/LeroyDyer__Spydaz_Web_AI_AGI_R1_Math_003/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.62 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4756 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0695 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.281 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4202 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2999 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/leroydyer__spydaz_web_ai_agi_r1_math_advancedstudent.json b/data/models/leroydyer__spydaz_web_ai_agi_r1_math_advancedstudent.json deleted file mode 100644 index ad12020dc1de72148213db7e9e41ecf24703073e..0000000000000000000000000000000000000000 --- a/data/models/leroydyer__spydaz_web_ai_agi_r1_math_advancedstudent.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "_Spydaz_Web_AI_AGI_R1_Math_AdvancedStudent", - "id": "LeroyDyer/_Spydaz_Web_AI_AGI_R1_Math_AdvancedStudent", - "developer": "LeroyDyer", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/LeroyDyer__Spydaz_Web_AI_AGI_R1_Math_AdvancedStudent/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5951 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4927 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0544 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2919 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5198 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/leroydyer__spydaz_web_ai_agi_r1_math_student.json b/data/models/leroydyer__spydaz_web_ai_agi_r1_math_student.json deleted file mode 100644 index db6053be8dc37f284457136e936ad01c3ed68bbb..0000000000000000000000000000000000000000 --- a/data/models/leroydyer__spydaz_web_ai_agi_r1_math_student.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "_Spydaz_Web_AI_AGI_R1_Math_Student", - "id": "LeroyDyer/_Spydaz_Web_AI_AGI_R1_Math_Student", - "developer": "LeroyDyer", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/LeroyDyer__Spydaz_Web_AI_AGI_R1_Math_Student/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5736 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4881 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0514 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2903 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5098 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2927 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/leroydyer__spydaz_web_ai_agi_r1_math_teacher.json b/data/models/leroydyer__spydaz_web_ai_agi_r1_math_teacher.json deleted file mode 100644 index 496e983ff04342c2c4f53ab13b7b1c5a98029ecf..0000000000000000000000000000000000000000 --- a/data/models/leroydyer__spydaz_web_ai_agi_r1_math_teacher.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "_Spydaz_Web_AI_AGI_R1_Math_Teacher", - "id": "LeroyDyer/_Spydaz_Web_AI_AGI_R1_Math_Teacher", - "developer": "LeroyDyer", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/LeroyDyer__Spydaz_Web_AI_AGI_R1_Math_Teacher/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5772 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4805 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0544 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2861 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5222 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2956 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/leroydyer__spydaz_web_ai_agi_r1_musr.json b/data/models/leroydyer__spydaz_web_ai_agi_r1_musr.json deleted file mode 100644 index 9990b2d78b09de4f36a7c59f0bd48d59c9e7d601..0000000000000000000000000000000000000000 --- a/data/models/leroydyer__spydaz_web_ai_agi_r1_musr.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "_Spydaz_Web_AI_AGI_R1_MUSR", - "id": "LeroyDyer/_Spydaz_Web_AI_AGI_R1_MUSR", - "developer": "LeroyDyer", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/LeroyDyer__Spydaz_Web_AI_AGI_R1_MUSR/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4786 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4672 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0604 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2844 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4869 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2828 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/leroydyer__spydaz_web_ai_agi_r1_omg_001.json b/data/models/leroydyer__spydaz_web_ai_agi_r1_omg_001.json deleted file mode 100644 index 8118d5a13b968b544ac9891eedfabc2b86d2d5d4..0000000000000000000000000000000000000000 --- a/data/models/leroydyer__spydaz_web_ai_agi_r1_omg_001.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "_Spydaz_Web_AI_AGI_R1_OmG_001", - "id": "LeroyDyer/_Spydaz_Web_AI_AGI_R1_OmG_001", - "developer": "LeroyDyer", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/LeroyDyer__Spydaz_Web_AI_AGI_R1_OmG_001/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5818 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4908 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0506 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3003 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4486 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2906 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/leroydyer__spydaz_web_ai_agi_r1_omg_002.json b/data/models/leroydyer__spydaz_web_ai_agi_r1_omg_002.json deleted file mode 100644 index 99a8d87cf16f614bc27a820595024b78de63dc63..0000000000000000000000000000000000000000 --- a/data/models/leroydyer__spydaz_web_ai_agi_r1_omg_002.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "_Spydaz_Web_AI_AGI_R1_OmG_002", - "id": "LeroyDyer/_Spydaz_Web_AI_AGI_R1_OmG_002", - "developer": "LeroyDyer", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/LeroyDyer__Spydaz_Web_AI_AGI_R1_OmG_002/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5462 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4655 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0498 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2785 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4511 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2867 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/leroydyer__spydaz_web_ai_agi_r1_omg_coder.json b/data/models/leroydyer__spydaz_web_ai_agi_r1_omg_coder.json deleted file mode 100644 index ef9e9e55ee217d0c0ce849841f180a29300f7884..0000000000000000000000000000000000000000 --- a/data/models/leroydyer__spydaz_web_ai_agi_r1_omg_coder.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "_Spydaz_Web_AI_AGI_R1_OmG_Coder", - "id": "LeroyDyer/_Spydaz_Web_AI_AGI_R1_OmG_Coder", - "developer": "LeroyDyer", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/LeroyDyer__Spydaz_Web_AI_AGI_R1_OmG_Coder/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4924 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4638 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0544 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2735 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5625 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.289 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/leroydyer__spydaz_web_ai_agi_r1_omg_math.json b/data/models/leroydyer__spydaz_web_ai_agi_r1_omg_math.json deleted file mode 100644 index b3b0cefad6abe418f1b1ce57e497aa0e611d767a..0000000000000000000000000000000000000000 --- a/data/models/leroydyer__spydaz_web_ai_agi_r1_omg_math.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "_Spydaz_Web_AI_AGI_R1_OmG_Math", - "id": "LeroyDyer/_Spydaz_Web_AI_AGI_R1_OmG_Math", - "developer": "LeroyDyer", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/LeroyDyer__Spydaz_Web_AI_AGI_R1_OmG_Math/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5033 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4677 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0476 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2827 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4326 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2913 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/leroydyer__spydaz_web_ai_agi_r1_omg_mathmaster.json b/data/models/leroydyer__spydaz_web_ai_agi_r1_omg_mathmaster.json deleted file mode 100644 index 40c75afc3b47578f4958482e01e225357a01bce3..0000000000000000000000000000000000000000 --- a/data/models/leroydyer__spydaz_web_ai_agi_r1_omg_mathmaster.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "_Spydaz_Web_AI_AGI_R1_OmG_MathMaster", - "id": "LeroyDyer/_Spydaz_Web_AI_AGI_R1_OmG_MathMaster", - "developer": "LeroyDyer", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/LeroyDyer__Spydaz_Web_AI_AGI_R1_OmG_MathMaster/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5558 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4742 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0536 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2878 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.451 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2672 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/leroydyer__spydaz_web_ai_agi_r1_student_coder.json b/data/models/leroydyer__spydaz_web_ai_agi_r1_student_coder.json deleted file mode 100644 index 9226e87ed000ceabf090c74c1001f271df9798e1..0000000000000000000000000000000000000000 --- a/data/models/leroydyer__spydaz_web_ai_agi_r1_student_coder.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "_Spydaz_Web_AI_AGI_R1_Student_Coder", - "id": "LeroyDyer/_Spydaz_Web_AI_AGI_R1_Student_Coder", - "developer": "LeroyDyer", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/LeroyDyer__Spydaz_Web_AI_AGI_R1_Student_Coder/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.545 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4651 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0657 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2844 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4388 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2768 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/leroydyer__spydaz_web_ai_agi_r1_teacher_coder.json b/data/models/leroydyer__spydaz_web_ai_agi_r1_teacher_coder.json deleted file mode 100644 index cdd0dbe39dec88741ae1652fe89eb65d4b2e8207..0000000000000000000000000000000000000000 --- a/data/models/leroydyer__spydaz_web_ai_agi_r1_teacher_coder.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "_Spydaz_Web_AI_AGI_R1_Teacher_Coder", - "id": "LeroyDyer/_Spydaz_Web_AI_AGI_R1_Teacher_Coder", - "developer": "LeroyDyer", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/LeroyDyer__Spydaz_Web_AI_AGI_R1_Teacher_Coder/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5082 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4797 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.065 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2911 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4338 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2845 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/leroydyer__spydaz_web_ai_agi_r1_top_student.json b/data/models/leroydyer__spydaz_web_ai_agi_r1_top_student.json deleted file mode 100644 index b0a13acbb88f08d570dfbcdad8bcabf14c04a5ba..0000000000000000000000000000000000000000 --- a/data/models/leroydyer__spydaz_web_ai_agi_r1_top_student.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "_Spydaz_Web_AI_AGI_R1_Top_Student", - "id": "LeroyDyer/_Spydaz_Web_AI_AGI_R1_Top_Student", - "developer": "LeroyDyer", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/LeroyDyer__Spydaz_Web_AI_AGI_R1_Top_Student/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.604 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4988 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0725 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2727 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5398 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3024 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/leroydyer__spydaz_web_ai_agi_r1_x1.json b/data/models/leroydyer__spydaz_web_ai_agi_r1_x1.json deleted file mode 100644 index 80c2b6e317c4c6f5795261ffafa4a52417bb30cb..0000000000000000000000000000000000000000 --- a/data/models/leroydyer__spydaz_web_ai_agi_r1_x1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "_Spydaz_Web_AI_AGI_R1_X1", - "id": "LeroyDyer/_Spydaz_Web_AI_AGI_R1_X1", - "developer": "LeroyDyer", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/LeroyDyer__Spydaz_Web_AI_AGI_R1_X1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4273 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4759 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0566 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2601 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4232 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2891 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/leroydyer__spydaz_web_ai_agi_r1_x2.json b/data/models/leroydyer__spydaz_web_ai_agi_r1_x2.json deleted file mode 100644 index 5ddb261d3bec5b7d080a2c2f9b1f2eaf7438237d..0000000000000000000000000000000000000000 --- a/data/models/leroydyer__spydaz_web_ai_agi_r1_x2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "_Spydaz_Web_AI_AGI_R1_X2", - "id": "LeroyDyer/_Spydaz_Web_AI_AGI_R1_X2", - "developer": "LeroyDyer", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/LeroyDyer__Spydaz_Web_AI_AGI_R1_X2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5434 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4786 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0612 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2978 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4695 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2921 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/leroydyer__spydaz_web_ai_agi_rp_r1.json b/data/models/leroydyer__spydaz_web_ai_agi_rp_r1.json deleted file mode 100644 index 89f140fc17db110a0ffaff0a23bbba09f2ab98df..0000000000000000000000000000000000000000 --- a/data/models/leroydyer__spydaz_web_ai_agi_rp_r1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "_Spydaz_Web_AI_AGI_RP_R1", - "id": "LeroyDyer/_Spydaz_Web_AI_AGI_RP_R1", - "developer": "LeroyDyer", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/LeroyDyer__Spydaz_Web_AI_AGI_RP_R1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5426 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4701 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0604 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2693 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4201 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2894 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/leroydyer__spydaz_web_ai_bible_002.json b/data/models/leroydyer__spydaz_web_ai_bible_002.json deleted file mode 100644 index ccc60d71bb1f71e8933e08455e209004b178e098..0000000000000000000000000000000000000000 --- a/data/models/leroydyer__spydaz_web_ai_bible_002.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "_Spydaz_Web_AI_BIBLE_002", - "id": "LeroyDyer/_Spydaz_Web_AI_BIBLE_002", - "developer": "LeroyDyer", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/LeroyDyer__Spydaz_Web_AI_BIBLE_002/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2195 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3289 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0174 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2844 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3407 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1368 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/leroydyer__spydaz_web_ai_chatml_002.json b/data/models/leroydyer__spydaz_web_ai_chatml_002.json deleted file mode 100644 index aa8a66315f812b155ad9a98dbcd49773f83cca8a..0000000000000000000000000000000000000000 --- a/data/models/leroydyer__spydaz_web_ai_chatml_002.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "_Spydaz_Web_AI_ChatML_002", - "id": "LeroyDyer/_Spydaz_Web_AI_ChatML_002", - "developer": "LeroyDyer", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/LeroyDyer__Spydaz_Web_AI_ChatML_002/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2412 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3106 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0113 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2576 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3623 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1095 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/leroydyer__spydaz_web_ai_chatqa.json b/data/models/leroydyer__spydaz_web_ai_chatqa.json deleted file mode 100644 index eddf020049ee930484e46e829d551373b7657539..0000000000000000000000000000000000000000 --- a/data/models/leroydyer__spydaz_web_ai_chatqa.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "_Spydaz_Web_AI_ChatQA", - "id": "LeroyDyer/_Spydaz_Web_AI_ChatQA", - "developer": "LeroyDyer", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/LeroyDyer__Spydaz_Web_AI_ChatQA/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1415 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3236 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0098 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2659 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3447 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1475 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/leroydyer__spydaz_web_ai_chatqa_003.json b/data/models/leroydyer__spydaz_web_ai_chatqa_003.json deleted file mode 100644 index cf65b7cdbed7daa7114731854516be27f1b88aa5..0000000000000000000000000000000000000000 --- a/data/models/leroydyer__spydaz_web_ai_chatqa_003.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "_Spydaz_Web_AI_ChatQA_003", - "id": "LeroyDyer/_Spydaz_Web_AI_ChatQA_003", - "developer": "LeroyDyer", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/LeroyDyer__Spydaz_Web_AI_ChatQA_003/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2209 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3172 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0106 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.271 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3818 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1133 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/leroydyer__spydaz_web_ai_temp.json b/data/models/leroydyer__spydaz_web_ai_temp.json deleted file mode 100644 index 0dffdd425d6d0c149af1b201883224e755962d0b..0000000000000000000000000000000000000000 --- a/data/models/leroydyer__spydaz_web_ai_temp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "_Spydaz_Web_AI_TEMP_", - "id": "LeroyDyer/_Spydaz_Web_AI_TEMP_", - "developer": "LeroyDyer", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/LeroyDyer__Spydaz_Web_AI_TEMP_/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4795 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4957 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1239 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2794 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4218 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3121 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/leroydyer__spydaz_web_ai_top_teacher.json b/data/models/leroydyer__spydaz_web_ai_top_teacher.json deleted file mode 100644 index a1db1b452abb56cda8c8457e48582ecc1ff23c10..0000000000000000000000000000000000000000 --- a/data/models/leroydyer__spydaz_web_ai_top_teacher.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "_Spydaz_Web_AI_Top_Teacher_", - "id": "LeroyDyer/_Spydaz_Web_AI_Top_Teacher_", - "developer": "LeroyDyer", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/LeroyDyer__Spydaz_Web_AI_Top_Teacher_/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4404 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4891 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1156 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2777 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4366 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.315 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/leroydyer_checkpoint_a.json b/data/models/leroydyer_checkpoint_a.json deleted file mode 100644 index 1377794b0b2d66b5469a060880b2e7b47d8f0f2f..0000000000000000000000000000000000000000 --- a/data/models/leroydyer_checkpoint_a.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "CheckPoint_A", - "id": "LeroyDyer/CheckPoint_A", - "developer": "LeroyDyer", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/LeroyDyer_CheckPoint_A/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4513 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4748 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0589 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2836 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4231 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.288 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/leroydyer_checkpoint_b.json b/data/models/leroydyer_checkpoint_b.json deleted file mode 100644 index e88c8ea03b4d24de673336600d0cab8681993303..0000000000000000000000000000000000000000 --- a/data/models/leroydyer_checkpoint_b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "CheckPoint_B", - "id": "LeroyDyer/CheckPoint_B", - "developer": "LeroyDyer", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/LeroyDyer_CheckPoint_B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.444 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.478 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0718 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2903 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3898 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2907 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/leroydyer_checkpoint_c.json b/data/models/leroydyer_checkpoint_c.json deleted file mode 100644 index de51aee1166dbd6e62ee4bfcfb3e32b27dc0a7e4..0000000000000000000000000000000000000000 --- a/data/models/leroydyer_checkpoint_c.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "CheckPoint_C", - "id": "LeroyDyer/CheckPoint_C", - "developer": "LeroyDyer", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/LeroyDyer_CheckPoint_C/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3477 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4586 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0551 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.271 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4346 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3021 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/leroydyer_checkpoint_r1.json b/data/models/leroydyer_checkpoint_r1.json deleted file mode 100644 index 067e63dbb077313f1fcd201be70f6878488f2bcb..0000000000000000000000000000000000000000 --- a/data/models/leroydyer_checkpoint_r1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "CheckPoint_R1", - "id": "LeroyDyer/CheckPoint_R1", - "developer": "LeroyDyer", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/LeroyDyer_CheckPoint_R1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1728 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4225 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0431 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2743 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4031 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2205 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/leroydyer_lcars_ai_001.json b/data/models/leroydyer_lcars_ai_001.json deleted file mode 100644 index e0f32adcb2a7f1cb6b499232956c73b36c4326cb..0000000000000000000000000000000000000000 --- a/data/models/leroydyer_lcars_ai_001.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "LCARS_AI_001", - "id": "LeroyDyer/LCARS_AI_001", - "developer": "LeroyDyer", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/LeroyDyer_LCARS_AI_001/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3109 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4258 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0234 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2634 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4384 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.267 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/leroydyer_lcars_ai_1x4_003_superai.json b/data/models/leroydyer_lcars_ai_1x4_003_superai.json deleted file mode 100644 index 2fec57aac5d20e6521c1c2ae4407fc1a38a88f0c..0000000000000000000000000000000000000000 --- a/data/models/leroydyer_lcars_ai_1x4_003_superai.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "LCARS_AI_1x4_003_SuperAI", - "id": "LeroyDyer/LCARS_AI_1x4_003_SuperAI", - "developer": "LeroyDyer", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MixtralForCausalLM", - "params_billions": "24.154" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/LeroyDyer_LCARS_AI_1x4_003_SuperAI/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4111 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.492 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0574 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2827 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4506 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2972 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/leroydyer_lcars_ai_startrek_computer.json b/data/models/leroydyer_lcars_ai_startrek_computer.json deleted file mode 100644 index ff3c66708eddd591bd339304a876343625b2b4c4..0000000000000000000000000000000000000000 --- a/data/models/leroydyer_lcars_ai_startrek_computer.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "LCARS_AI_StarTrek_Computer", - "id": "LeroyDyer/LCARS_AI_StarTrek_Computer", - "developer": "LeroyDyer", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/LeroyDyer_LCARS_AI_StarTrek_Computer/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3583 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4446 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0408 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2676 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.395 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2458 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/leroydyer_lcars_top_score.json b/data/models/leroydyer_lcars_top_score.json deleted file mode 100644 index 298bf2aa442833fbc3c3ecd5eaca82d83060e78b..0000000000000000000000000000000000000000 --- a/data/models/leroydyer_lcars_top_score.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "LCARS_TOP_SCORE", - "id": "LeroyDyer/LCARS_TOP_SCORE", - "developer": "LeroyDyer", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/LeroyDyer_LCARS_TOP_SCORE/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4371 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5127 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0672 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2861 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4293 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3031 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/leroydyer_mixtral_ai_swahilitron_7b.json b/data/models/leroydyer_mixtral_ai_swahilitron_7b.json deleted file mode 100644 index 55372dcf4231803fee9313e35cb79404491bbfe1..0000000000000000000000000000000000000000 --- a/data/models/leroydyer_mixtral_ai_swahilitron_7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mixtral_AI_SwahiliTron_7b", - "id": "LeroyDyer/Mixtral_AI_SwahiliTron_7b", - "developer": "LeroyDyer", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/LeroyDyer_Mixtral_AI_SwahiliTron_7b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1534 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3055 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0136 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2651 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.342 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1208 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/leroydyer_spydazweb_ai_cybertron_ultra_7b.json b/data/models/leroydyer_spydazweb_ai_cybertron_ultra_7b.json deleted file mode 100644 index 104b81bcd7412584c03ab69afb749b511cd72a35..0000000000000000000000000000000000000000 --- a/data/models/leroydyer_spydazweb_ai_cybertron_ultra_7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SpydazWeb_AI_CyberTron_Ultra_7b", - "id": "LeroyDyer/SpydazWeb_AI_CyberTron_Ultra_7b", - "developer": "LeroyDyer", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/LeroyDyer_SpydazWeb_AI_CyberTron_Ultra_7b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1556 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4811 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0136 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2928 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4136 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2866 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/leroydyer_spydazweb_ai_humanagi_001_m2.json b/data/models/leroydyer_spydazweb_ai_humanagi_001_m2.json deleted file mode 100644 index 6479a8b45d9904010d32a292abc5e5ff60f94c58..0000000000000000000000000000000000000000 --- a/data/models/leroydyer_spydazweb_ai_humanagi_001_m2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SpydazWeb_AI_HumanAGI_001_M2", - "id": "LeroyDyer/SpydazWeb_AI_HumanAGI_001_M2", - "developer": "LeroyDyer", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/LeroyDyer_SpydazWeb_AI_HumanAGI_001_M2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.394 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4888 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0385 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2894 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4503 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3005 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/leroydyer_spydazweb_ai_humanagi_002.json b/data/models/leroydyer_spydazweb_ai_humanagi_002.json deleted file mode 100644 index 27abbd13db4d39fdf9e772d7662cb7da18ab448f..0000000000000000000000000000000000000000 --- a/data/models/leroydyer_spydazweb_ai_humanagi_002.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SpydazWeb_AI_HumanAGI_002", - "id": "LeroyDyer/SpydazWeb_AI_HumanAGI_002", - "developer": "LeroyDyer", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/LeroyDyer_SpydazWeb_AI_HumanAGI_002/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4088 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5044 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0665 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2869 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4865 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3059 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/leroydyer_spydazweb_ai_humanai_001.json b/data/models/leroydyer_spydazweb_ai_humanai_001.json deleted file mode 100644 index 842fa80fecd9e4a1544480f63be329891669ff40..0000000000000000000000000000000000000000 --- a/data/models/leroydyer_spydazweb_ai_humanai_001.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SpydazWeb_AI_HumanAI_001", - "id": "LeroyDyer/SpydazWeb_AI_HumanAI_001", - "developer": "LeroyDyer", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/LeroyDyer_SpydazWeb_AI_HumanAI_001/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2252 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3344 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0166 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2886 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.386 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1271 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/leroydyer_spydazweb_ai_humanai_006.json b/data/models/leroydyer_spydazweb_ai_humanai_006.json deleted file mode 100644 index bdc96e474abcdee25a7d870e5718632642f97926..0000000000000000000000000000000000000000 --- a/data/models/leroydyer_spydazweb_ai_humanai_006.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SpydazWeb_AI_HumanAI_006", - "id": "LeroyDyer/SpydazWeb_AI_HumanAI_006", - "developer": "LeroyDyer", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/LeroyDyer_SpydazWeb_AI_HumanAI_006/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.143 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3302 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0106 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2802 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3568 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1135 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/leroydyer_spydazweb_ai_humanai_007.json b/data/models/leroydyer_spydazweb_ai_humanai_007.json deleted file mode 100644 index 03298ecf415ff023b3d5fafc710921f3b003154f..0000000000000000000000000000000000000000 --- a/data/models/leroydyer_spydazweb_ai_humanai_007.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SpydazWeb_AI_HumanAI_007", - "id": "LeroyDyer/SpydazWeb_AI_HumanAI_007", - "developer": "LeroyDyer", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/LeroyDyer_SpydazWeb_AI_HumanAI_007/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3352 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3416 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0227 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2886 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4096 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1352 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/leroydyer_spydazweb_ai_humanai_009_chat.json b/data/models/leroydyer_spydazweb_ai_humanai_009_chat.json deleted file mode 100644 index 68543c12dd1c11f51cf84ec80693e3dd91451812..0000000000000000000000000000000000000000 --- a/data/models/leroydyer_spydazweb_ai_humanai_009_chat.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SpydazWeb_AI_HumanAI_009_CHAT", - "id": "LeroyDyer/SpydazWeb_AI_HumanAI_009_CHAT", - "developer": "LeroyDyer", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/LeroyDyer_SpydazWeb_AI_HumanAI_009_CHAT/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2973 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3307 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0166 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.281 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4138 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1433 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/leroydyer_spydazweb_ai_humanai_010_chat.json b/data/models/leroydyer_spydazweb_ai_humanai_010_chat.json deleted file mode 100644 index 75183c3a041fd164fdfb093aca043527663d48e6..0000000000000000000000000000000000000000 --- a/data/models/leroydyer_spydazweb_ai_humanai_010_chat.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SpydazWeb_AI_HumanAI_010_CHAT", - "id": "LeroyDyer/SpydazWeb_AI_HumanAI_010_CHAT", - "developer": "LeroyDyer", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/LeroyDyer_SpydazWeb_AI_HumanAI_010_CHAT/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2507 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3336 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0181 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2592 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4137 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.143 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/leroydyer_spydazweb_ai_humanai_011_instruct.json b/data/models/leroydyer_spydazweb_ai_humanai_011_instruct.json deleted file mode 100644 index 6d21bb9081b2cfd26f86a2425d692c24d2bcd4a4..0000000000000000000000000000000000000000 --- a/data/models/leroydyer_spydazweb_ai_humanai_011_instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SpydazWeb_AI_HumanAI_011_INSTRUCT", - "id": "LeroyDyer/SpydazWeb_AI_HumanAI_011_INSTRUCT", - "developer": "LeroyDyer", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/LeroyDyer_SpydazWeb_AI_HumanAI_011_INSTRUCT/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3149 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3523 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0144 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2794 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3831 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1595 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/leroydyer_spydazweb_ai_humanai_011_instruct_ml.json b/data/models/leroydyer_spydazweb_ai_humanai_011_instruct_ml.json deleted file mode 100644 index 41deae7b0b31323e5edc88461af6aa5ac29330a0..0000000000000000000000000000000000000000 --- a/data/models/leroydyer_spydazweb_ai_humanai_011_instruct_ml.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SpydazWeb_AI_HumanAI_011_INSTRUCT_ML", - "id": "LeroyDyer/SpydazWeb_AI_HumanAI_011_INSTRUCT_ML", - "developer": "LeroyDyer", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/LeroyDyer_SpydazWeb_AI_HumanAI_011_INSTRUCT_ML/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3752 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3984 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0257 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2928 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4239 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2019 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/leroydyer_spydazweb_ai_humanai_011_instruct_ml_r1.json b/data/models/leroydyer_spydazweb_ai_humanai_011_instruct_ml_r1.json deleted file mode 100644 index 509d9b19a99f37d43d6550153abcd865debb1032..0000000000000000000000000000000000000000 --- a/data/models/leroydyer_spydazweb_ai_humanai_011_instruct_ml_r1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SpydazWeb_AI_HumanAI_011_INSTRUCT_ML_r1", - "id": "LeroyDyer/SpydazWeb_AI_HumanAI_011_INSTRUCT_ML_r1", - "developer": "LeroyDyer", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/LeroyDyer_SpydazWeb_AI_HumanAI_011_INSTRUCT_ML_r1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.405 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4858 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0551 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2928 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3921 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2956 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/leroydyer_spydazweb_ai_humanai_012_instruct_ia.json b/data/models/leroydyer_spydazweb_ai_humanai_012_instruct_ia.json deleted file mode 100644 index 4462dbdfa0461cc5c2fd5d7ac79308c4e0a0cd93..0000000000000000000000000000000000000000 --- a/data/models/leroydyer_spydazweb_ai_humanai_012_instruct_ia.json +++ /dev/null @@ -1,275 +0,0 @@ -{ - "model_info": { - "name": "SpydazWeb_AI_HumanAI_012_INSTRUCT_IA", - "id": "LeroyDyer/SpydazWeb_AI_HumanAI_012_INSTRUCT_IA", - "developer": "LeroyDyer", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/LeroyDyer_SpydazWeb_AI_HumanAI_012_INSTRUCT_IA/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3066 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4577 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0446 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2995 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4254 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2318 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "hfopenllm_v2/LeroyDyer_SpydazWeb_AI_HumanAI_012_INSTRUCT_IA/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3036 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4575 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0446 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3012 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4253 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2329 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/leroydyer_spydazweb_ai_humanai_012_instruct_mx.json b/data/models/leroydyer_spydazweb_ai_humanai_012_instruct_mx.json deleted file mode 100644 index a9b5d5c49d6492f5288ac620f2b284da4beb1bca..0000000000000000000000000000000000000000 --- a/data/models/leroydyer_spydazweb_ai_humanai_012_instruct_mx.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SpydazWeb_AI_HumanAI_012_INSTRUCT_MX", - "id": "LeroyDyer/SpydazWeb_AI_HumanAI_012_INSTRUCT_MX", - "developer": "LeroyDyer", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/LeroyDyer_SpydazWeb_AI_HumanAI_012_INSTRUCT_MX/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3066 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3158 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0151 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2911 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3444 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1107 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/leroydyer_spydazweb_ai_humanai_012_instruct_xa.json b/data/models/leroydyer_spydazweb_ai_humanai_012_instruct_xa.json deleted file mode 100644 index 199a2574ee197c6be6eb6b38849e4c63ab43b085..0000000000000000000000000000000000000000 --- a/data/models/leroydyer_spydazweb_ai_humanai_012_instruct_xa.json +++ /dev/null @@ -1,275 +0,0 @@ -{ - "model_info": { - "name": "SpydazWeb_AI_HumanAI_012_INSTRUCT_XA", - "id": "LeroyDyer/SpydazWeb_AI_HumanAI_012_INSTRUCT_XA", - "developer": "LeroyDyer", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/LeroyDyer_SpydazWeb_AI_HumanAI_012_INSTRUCT_XA/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3798 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4483 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.04 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3129 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4148 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2389 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "hfopenllm_v2/LeroyDyer_SpydazWeb_AI_HumanAI_012_INSTRUCT_XA/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3579 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4477 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0423 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3096 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4134 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2376 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/leroydyer_spydazweb_ai_humanai_rp.json b/data/models/leroydyer_spydazweb_ai_humanai_rp.json deleted file mode 100644 index 2094a540a2fe593373cc2fcd8adb100ef0070366..0000000000000000000000000000000000000000 --- a/data/models/leroydyer_spydazweb_ai_humanai_rp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SpydazWeb_AI_HumanAI_RP", - "id": "LeroyDyer/SpydazWeb_AI_HumanAI_RP", - "developer": "LeroyDyer", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/LeroyDyer_SpydazWeb_AI_HumanAI_RP/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2541 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3323 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0128 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2752 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3883 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1324 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/leroydyer_spydazweb_ai_humanai_textvision.json b/data/models/leroydyer_spydazweb_ai_humanai_textvision.json deleted file mode 100644 index 5b1a15c64cf06ac4a67a11ab900adba597e7d68d..0000000000000000000000000000000000000000 --- a/data/models/leroydyer_spydazweb_ai_humanai_textvision.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SpydazWeb_AI_HumanAI_TextVision", - "id": "LeroyDyer/SpydazWeb_AI_HumanAI_TextVision", - "developer": "LeroyDyer", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/LeroyDyer_SpydazWeb_AI_HumanAI_TextVision/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3063 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3354 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0144 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2919 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3938 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1387 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/leroydyer_spydazweb_humanai_m1.json b/data/models/leroydyer_spydazweb_humanai_m1.json deleted file mode 100644 index 765fd30ee78e78ff9b9d135faeeca5276efb3ed0..0000000000000000000000000000000000000000 --- a/data/models/leroydyer_spydazweb_humanai_m1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SpydazWeb_HumanAI_M1", - "id": "LeroyDyer/SpydazWeb_HumanAI_M1", - "developer": "LeroyDyer", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/LeroyDyer_SpydazWeb_HumanAI_M1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3582 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3563 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0249 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2676 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3671 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1663 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/leroydyer_spydazweb_humanai_m2.json b/data/models/leroydyer_spydazweb_humanai_m2.json deleted file mode 100644 index ab7ecdd150074487e74e886b08d212ed049bcb75..0000000000000000000000000000000000000000 --- a/data/models/leroydyer_spydazweb_humanai_m2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SpydazWeb_HumanAI_M2", - "id": "LeroyDyer/SpydazWeb_HumanAI_M2", - "developer": "LeroyDyer", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/LeroyDyer_SpydazWeb_HumanAI_M2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.375 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3931 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0287 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2794 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3751 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.201 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/leroydyer_spydazweb_humanai_m3.json b/data/models/leroydyer_spydazweb_humanai_m3.json deleted file mode 100644 index bc611caffc553829ab07fa09da516d542884d48b..0000000000000000000000000000000000000000 --- a/data/models/leroydyer_spydazweb_humanai_m3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SpydazWeb_HumanAI_M3", - "id": "LeroyDyer/SpydazWeb_HumanAI_M3", - "developer": "LeroyDyer", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/LeroyDyer_SpydazWeb_HumanAI_M3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1579 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3127 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0091 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.271 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3914 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1149 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/leroydyer_spydazwebai_human_agi.json b/data/models/leroydyer_spydazwebai_human_agi.json deleted file mode 100644 index b9c65098c59a29d059fbad031230968e5d9ddb71..0000000000000000000000000000000000000000 --- a/data/models/leroydyer_spydazwebai_human_agi.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SpydazWebAI_Human_AGI", - "id": "LeroyDyer/SpydazWebAI_Human_AGI", - "developer": "LeroyDyer", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/LeroyDyer_SpydazWebAI_Human_AGI/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3388 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3375 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0144 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2827 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3966 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1479 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/leroydyer_spydazwebai_human_agi_001.json b/data/models/leroydyer_spydazwebai_human_agi_001.json deleted file mode 100644 index c99cdc8c8f04ac28bdcede5702a156748ac7170f..0000000000000000000000000000000000000000 --- a/data/models/leroydyer_spydazwebai_human_agi_001.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SpydazWebAI_Human_AGI_001", - "id": "LeroyDyer/SpydazWebAI_Human_AGI_001", - "developer": "LeroyDyer", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/LeroyDyer_SpydazWebAI_Human_AGI_001/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3118 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3433 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0196 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2987 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3994 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1426 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lesubra_ece-eiffel-3b.json b/data/models/lesubra_ece-eiffel-3b.json deleted file mode 100644 index c5be77563c2c921765871f8fc88ca20fcae4a154..0000000000000000000000000000000000000000 --- a/data/models/lesubra_ece-eiffel-3b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ECE-EIFFEL-3B", - "id": "lesubra/ECE-EIFFEL-3B", - "developer": "lesubra", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Phi3ForCausalLM", - "params_billions": "3.821" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/lesubra_ECE-EIFFEL-3B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3469 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5102 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1216 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3314 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4362 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3821 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lesubra_ece-eiffel-3bv2.json b/data/models/lesubra_ece-eiffel-3bv2.json deleted file mode 100644 index f2798e6f2e4fa732e5e29536ff2a9ba12fa2d257..0000000000000000000000000000000000000000 --- a/data/models/lesubra_ece-eiffel-3bv2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ECE-EIFFEL-3Bv2", - "id": "lesubra/ECE-EIFFEL-3Bv2", - "developer": "lesubra", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Phi3ForCausalLM", - "params_billions": "3.821" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/lesubra_ECE-EIFFEL-3Bv2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3013 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5424 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1186 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3356 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4443 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3999 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lesubra_ece-eiffel-3bv3.json b/data/models/lesubra_ece-eiffel-3bv3.json deleted file mode 100644 index d458aed45f4c90487238d30cfefddf23b639c0da..0000000000000000000000000000000000000000 --- a/data/models/lesubra_ece-eiffel-3bv3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ECE-EIFFEL-3Bv3", - "id": "lesubra/ECE-EIFFEL-3Bv3", - "developer": "lesubra", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Phi3ForCausalLM", - "params_billions": "3.821" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/lesubra_ECE-EIFFEL-3Bv3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3786 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5469 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1669 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3297 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4675 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3975 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lesubra_ece-prymmal-3b-slerp-v1.json b/data/models/lesubra_ece-prymmal-3b-slerp-v1.json deleted file mode 100644 index 4b30915a6c39f3707db7b974b90282b32e03edec..0000000000000000000000000000000000000000 --- a/data/models/lesubra_ece-prymmal-3b-slerp-v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ECE-PRYMMAL-3B-SLERP-V1", - "id": "lesubra/ECE-PRYMMAL-3B-SLERP-V1", - "developer": "lesubra", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Phi3ForCausalLM", - "params_billions": "3.821" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/lesubra_ECE-PRYMMAL-3B-SLERP-V1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2933 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5341 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1662 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3171 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4595 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.39 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lesubra_ece-prymmal-3b-slerp-v2.json b/data/models/lesubra_ece-prymmal-3b-slerp-v2.json deleted file mode 100644 index 8b09183899293cebeabf42894bedf11257bd9246..0000000000000000000000000000000000000000 --- a/data/models/lesubra_ece-prymmal-3b-slerp-v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ECE-PRYMMAL-3B-SLERP-V2", - "id": "lesubra/ECE-PRYMMAL-3B-SLERP-V2", - "developer": "lesubra", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Phi3ForCausalLM", - "params_billions": "3.821" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/lesubra_ECE-PRYMMAL-3B-SLERP-V2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2933 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5341 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1662 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3171 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4595 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.39 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lesubra_ece-prymmal-3b-slerp_2-v1.json b/data/models/lesubra_ece-prymmal-3b-slerp_2-v1.json deleted file mode 100644 index 86bf4f52a3e5578c6eb0b43b02a8e78a9779eda1..0000000000000000000000000000000000000000 --- a/data/models/lesubra_ece-prymmal-3b-slerp_2-v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ECE-PRYMMAL-3B-SLERP_2-V1", - "id": "lesubra/ECE-PRYMMAL-3B-SLERP_2-V1", - "developer": "lesubra", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Phi3ForCausalLM", - "params_billions": "3.821" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/lesubra_ECE-PRYMMAL-3B-SLERP_2-V1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3649 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5411 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1677 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3213 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4661 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.399 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lesubra_ece-prymmal-3b-slerp_2-v2.json b/data/models/lesubra_ece-prymmal-3b-slerp_2-v2.json deleted file mode 100644 index bcb0d78f528632be9cccf1a64c56d417314d978f..0000000000000000000000000000000000000000 --- a/data/models/lesubra_ece-prymmal-3b-slerp_2-v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ECE-PRYMMAL-3B-SLERP_2-V2", - "id": "lesubra/ECE-PRYMMAL-3B-SLERP_2-V2", - "developer": "lesubra", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Phi3ForCausalLM", - "params_billions": "3.821" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/lesubra_ECE-PRYMMAL-3B-SLERP_2-V2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3664 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5411 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1677 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3213 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4661 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.399 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lesubra_merge-test.json b/data/models/lesubra_merge-test.json deleted file mode 100644 index f15ef55b17d3ddc70dd790a05451105b5051af8e..0000000000000000000000000000000000000000 --- a/data/models/lesubra_merge-test.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "merge-test", - "id": "lesubra/merge-test", - "developer": "lesubra", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Phi3ForCausalLM", - "params_billions": "3.821" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/lesubra_merge-test/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5383 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.524 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1208 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3221 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4419 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3874 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lgai-exaone_exaone-3.0-7.8b-instruct.json b/data/models/lgai-exaone_exaone-3.0-7.8b-instruct.json deleted file mode 100644 index ed1cfbf2259057ed2ab28da94a19c76650d0dcae..0000000000000000000000000000000000000000 --- a/data/models/lgai-exaone_exaone-3.0-7.8b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "EXAONE-3.0-7.8B-Instruct", - "id": "LGAI-EXAONE/EXAONE-3.0-7.8B-Instruct", - "developer": "LGAI-EXAONE", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "ExaoneForCausalLM", - "params_billions": "7.8" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/LGAI-EXAONE_EXAONE-3.0-7.8B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7193 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4174 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3044 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2659 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3661 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3577 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lgai-exaone_exaone-3.5-2.4b-instruct.json b/data/models/lgai-exaone_exaone-3.5-2.4b-instruct.json deleted file mode 100644 index 28d8bfaa72edec1f09270e1d0f4095aa725e9d47..0000000000000000000000000000000000000000 --- a/data/models/lgai-exaone_exaone-3.5-2.4b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "EXAONE-3.5-2.4B-Instruct", - "id": "LGAI-EXAONE/EXAONE-3.5-2.4B-Instruct", - "developer": "LGAI-EXAONE", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "ExaoneForCausalLM", - "params_billions": "2.405" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/LGAI-EXAONE_EXAONE-3.5-2.4B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.795 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4092 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3678 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2659 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3661 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.328 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lgai-exaone_exaone-3.5-32b-instruct.json b/data/models/lgai-exaone_exaone-3.5-32b-instruct.json deleted file mode 100644 index cb749c9f52287452bb4b790c4a26e16283a679a5..0000000000000000000000000000000000000000 --- a/data/models/lgai-exaone_exaone-3.5-32b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "EXAONE-3.5-32B-Instruct", - "id": "LGAI-EXAONE/EXAONE-3.5-32B-Instruct", - "developer": "LGAI-EXAONE", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "ExaoneForCausalLM", - "params_billions": "32.003" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/LGAI-EXAONE_EXAONE-3.5-32B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8392 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5761 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5128 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2878 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3807 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4637 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lgai-exaone_exaone-3.5-7.8b-instruct.json b/data/models/lgai-exaone_exaone-3.5-7.8b-instruct.json deleted file mode 100644 index 4c3139f98624796a279ef2748561293e6610a254..0000000000000000000000000000000000000000 --- a/data/models/lgai-exaone_exaone-3.5-7.8b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "EXAONE-3.5-7.8B-Instruct", - "id": "LGAI-EXAONE/EXAONE-3.5-7.8B-Instruct", - "developer": "LGAI-EXAONE", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "ExaoneForCausalLM", - "params_billions": "7.818" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/LGAI-EXAONE_EXAONE-3.5-7.8B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8136 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4728 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4751 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2576 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3779 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4133 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lightblue_suzume-llama-3-8b-multilingual-orpo-borda-full.json b/data/models/lightblue_suzume-llama-3-8b-multilingual-orpo-borda-full.json deleted file mode 100644 index ddb7449d0d7cbdfc5b8b4b3c5f367abffeddeb3d..0000000000000000000000000000000000000000 --- a/data/models/lightblue_suzume-llama-3-8b-multilingual-orpo-borda-full.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "suzume-llama-3-8B-multilingual-orpo-borda-full", - "id": "lightblue/suzume-llama-3-8B-multilingual-orpo-borda-full", - "developer": "lightblue", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/lightblue_suzume-llama-3-8B-multilingual-orpo-borda-full/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5817 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4714 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0763 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2592 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3222 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.331 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lightblue_suzume-llama-3-8b-multilingual-orpo-borda-half.json b/data/models/lightblue_suzume-llama-3-8b-multilingual-orpo-borda-half.json deleted file mode 100644 index efed86b294becf0b45619d7f3372c1eefef79575..0000000000000000000000000000000000000000 --- a/data/models/lightblue_suzume-llama-3-8b-multilingual-orpo-borda-half.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "suzume-llama-3-8B-multilingual-orpo-borda-half", - "id": "lightblue/suzume-llama-3-8B-multilingual-orpo-borda-half", - "developer": "lightblue", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/lightblue_suzume-llama-3-8B-multilingual-orpo-borda-half/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6249 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4707 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0906 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.245 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3516 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3614 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lightblue_suzume-llama-3-8b-multilingual-orpo-borda-top25.json b/data/models/lightblue_suzume-llama-3-8b-multilingual-orpo-borda-top25.json deleted file mode 100644 index 9345064e14728f596bbed92c698bcd51b8596a27..0000000000000000000000000000000000000000 --- a/data/models/lightblue_suzume-llama-3-8b-multilingual-orpo-borda-top25.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "suzume-llama-3-8B-multilingual-orpo-borda-top25", - "id": "lightblue/suzume-llama-3-8B-multilingual-orpo-borda-top25", - "developer": "lightblue", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/lightblue_suzume-llama-3-8B-multilingual-orpo-borda-top25/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6637 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4865 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1042 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2727 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3566 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3684 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lightblue_suzume-llama-3-8b-multilingual-orpo-borda-top75.json b/data/models/lightblue_suzume-llama-3-8b-multilingual-orpo-borda-top75.json deleted file mode 100644 index c609eb333c90593ec6b3ce49cf08d31bf15626bc..0000000000000000000000000000000000000000 --- a/data/models/lightblue_suzume-llama-3-8b-multilingual-orpo-borda-top75.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "suzume-llama-3-8B-multilingual-orpo-borda-top75", - "id": "lightblue/suzume-llama-3-8B-multilingual-orpo-borda-top75", - "developer": "lightblue", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/lightblue_suzume-llama-3-8B-multilingual-orpo-borda-top75/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6687 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4833 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0785 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2727 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3817 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3769 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lightblue_suzume-llama-3-8b-multilingual.json b/data/models/lightblue_suzume-llama-3-8b-multilingual.json deleted file mode 100644 index 529ccd412e639a9f4e4151d98cdc6300feca8e89..0000000000000000000000000000000000000000 --- a/data/models/lightblue_suzume-llama-3-8b-multilingual.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "suzume-llama-3-8B-multilingual", - "id": "lightblue/suzume-llama-3-8B-multilingual", - "developer": "lightblue", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/lightblue_suzume-llama-3-8B-multilingual/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6678 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.495 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0944 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2836 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3977 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3383 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lightningrodlabs_flashlight-v1.0.json b/data/models/lightningrodlabs_flashlight-v1.0.json deleted file mode 100644 index 8ba219bbc2ffc55efe6091bbce51a727a0bfab96..0000000000000000000000000000000000000000 --- a/data/models/lightningrodlabs_flashlight-v1.0.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Flashlight-v1.0", - "id": "LightningRodLabs/Flashlight-v1.0", - "developer": "LightningRodLabs", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/LightningRodLabs_Flashlight-v1.0/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6745 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6877 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.497 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3423 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4101 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5402 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lightningrodlabs_flashlight-v1.1.json b/data/models/lightningrodlabs_flashlight-v1.1.json deleted file mode 100644 index 8bc1a4cf53ed70faed4a3d5ea00806eb8061a0d4..0000000000000000000000000000000000000000 --- a/data/models/lightningrodlabs_flashlight-v1.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Flashlight-v1.1", - "id": "LightningRodLabs/Flashlight-v1.1", - "developer": "LightningRodLabs", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Phi3ForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/LightningRodLabs_Flashlight-v1.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6721 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6901 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5325 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3398 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4048 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5416 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lightningrodlabs_flashlight-v1.2.json b/data/models/lightningrodlabs_flashlight-v1.2.json deleted file mode 100644 index 3542717542aa1d37467bdcd1799dc8b964772c9e..0000000000000000000000000000000000000000 --- a/data/models/lightningrodlabs_flashlight-v1.2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Flashlight-v1.2", - "id": "LightningRodLabs/Flashlight-v1.2", - "developer": "LightningRodLabs", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/LightningRodLabs_Flashlight-v1.2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.436 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3265 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1556 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2357 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4554 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2485 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lil-r_2_prymmal-ece-2b-slerp-v1.json b/data/models/lil-r_2_prymmal-ece-2b-slerp-v1.json deleted file mode 100644 index 02ce90578c5b514c74e4927842409ed2a94d9918..0000000000000000000000000000000000000000 --- a/data/models/lil-r_2_prymmal-ece-2b-slerp-v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "2_PRYMMAL-ECE-2B-SLERP-V1", - "id": "Lil-R/2_PRYMMAL-ECE-2B-SLERP-V1", - "developer": "Lil-R", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "2.614" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Lil-R_2_PRYMMAL-ECE-2B-SLERP-V1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5823 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4287 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0914 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3062 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4375 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2678 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lil-r_2_prymmal-ece-2b-slerp-v2.json b/data/models/lil-r_2_prymmal-ece-2b-slerp-v2.json deleted file mode 100644 index 9024c688906a8796f6336360cad562b4640d6342..0000000000000000000000000000000000000000 --- a/data/models/lil-r_2_prymmal-ece-2b-slerp-v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "2_PRYMMAL-ECE-2B-SLERP-V2", - "id": "Lil-R/2_PRYMMAL-ECE-2B-SLERP-V2", - "developer": "Lil-R", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "2.614" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Lil-R_2_PRYMMAL-ECE-2B-SLERP-V2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5543 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4376 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0944 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2978 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4482 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2744 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lil-r_2_prymmal-ece-7b-slerp-v1.json b/data/models/lil-r_2_prymmal-ece-7b-slerp-v1.json deleted file mode 100644 index aa84b65299e0298a753916e791f54b2464ce498b..0000000000000000000000000000000000000000 --- a/data/models/lil-r_2_prymmal-ece-7b-slerp-v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "2_PRYMMAL-ECE-7B-SLERP-V1", - "id": "Lil-R/2_PRYMMAL-ECE-7B-SLERP-V1", - "developer": "Lil-R", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Lil-R_2_PRYMMAL-ECE-7B-SLERP-V1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1073 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3053 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0008 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2508 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3911 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1124 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lil-r_2_prymmal-ece-7b-slerp-v2.json b/data/models/lil-r_2_prymmal-ece-7b-slerp-v2.json deleted file mode 100644 index 92ce75afa0aae2e27736935a9dd0c0ee150d136c..0000000000000000000000000000000000000000 --- a/data/models/lil-r_2_prymmal-ece-7b-slerp-v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "2_PRYMMAL-ECE-7B-SLERP-V2", - "id": "Lil-R/2_PRYMMAL-ECE-7B-SLERP-V2", - "developer": "Lil-R", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Lil-R_2_PRYMMAL-ECE-7B-SLERP-V2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1073 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3053 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0008 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2508 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3911 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1124 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lil-r_2_prymmal-ece-7b-slerp-v3.json b/data/models/lil-r_2_prymmal-ece-7b-slerp-v3.json deleted file mode 100644 index 58ffd0b9507b84d5501acd1b2dbebab760d2b3d4..0000000000000000000000000000000000000000 --- a/data/models/lil-r_2_prymmal-ece-7b-slerp-v3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "2_PRYMMAL-ECE-7B-SLERP-V3", - "id": "Lil-R/2_PRYMMAL-ECE-7B-SLERP-V3", - "developer": "Lil-R", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Lil-R_2_PRYMMAL-ECE-7B-SLERP-V3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2235 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3578 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.006 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2567 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4107 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1817 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lil-r_2_prymmal-ece-7b-slerp.json b/data/models/lil-r_2_prymmal-ece-7b-slerp.json deleted file mode 100644 index 2f1c3c1687e85d09a64269c8cfbcfecdf6c1349a..0000000000000000000000000000000000000000 --- a/data/models/lil-r_2_prymmal-ece-7b-slerp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "2_PRYMMAL-ECE-7B-SLERP", - "id": "Lil-R/2_PRYMMAL-ECE-7B-SLERP", - "developer": "Lil-R", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Lil-R_2_PRYMMAL-ECE-7B-SLERP/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5577 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5557 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3633 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3104 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4396 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4507 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lil-r_prymmal-ece-1b-slerp-v1.json b/data/models/lil-r_prymmal-ece-1b-slerp-v1.json deleted file mode 100644 index 6b5f12bc6da1bb2f0788e68e4d4c1bdec5990f22..0000000000000000000000000000000000000000 --- a/data/models/lil-r_prymmal-ece-1b-slerp-v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "PRYMMAL-ECE-1B-SLERP-V1", - "id": "Lil-R/PRYMMAL-ECE-1B-SLERP-V1", - "developer": "Lil-R", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.777" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Lil-R_PRYMMAL-ECE-1B-SLERP-V1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2874 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.419 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1035 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.276 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3974 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2926 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lil-r_prymmal-ece-7b-slerp-v8.json b/data/models/lil-r_prymmal-ece-7b-slerp-v8.json deleted file mode 100644 index 894b68bccb47b9e3a7b2b8bdc1f43a62eb56ec41..0000000000000000000000000000000000000000 --- a/data/models/lil-r_prymmal-ece-7b-slerp-v8.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "PRYMMAL-ECE-7B-SLERP-V8", - "id": "Lil-R/PRYMMAL-ECE-7B-SLERP-V8", - "developer": "Lil-R", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Lil-R_PRYMMAL-ECE-7B-SLERP-V8/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1258 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2955 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0098 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.25 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3631 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1128 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lilrg_10prymmal-3b-slerp.json b/data/models/lilrg_10prymmal-3b-slerp.json deleted file mode 100644 index 42cc834d55c8960f2b7405283ef3a917e0829ad6..0000000000000000000000000000000000000000 --- a/data/models/lilrg_10prymmal-3b-slerp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "10PRYMMAL-3B-slerp", - "id": "LilRg/10PRYMMAL-3B-slerp", - "developer": "LilRg", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Phi3ForCausalLM", - "params_billions": "3.821" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/LilRg_10PRYMMAL-3B-slerp/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1946 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.532 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1495 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3213 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4529 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3881 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lilrg_ece-1b-merge-prymmal.json b/data/models/lilrg_ece-1b-merge-prymmal.json deleted file mode 100644 index 3bb7659e894deb1fc91c84ab94fc9d98d4707d53..0000000000000000000000000000000000000000 --- a/data/models/lilrg_ece-1b-merge-prymmal.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ECE-1B-merge-PRYMMAL", - "id": "LilRg/ECE-1B-merge-PRYMMAL", - "developer": "LilRg", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.777" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/LilRg_ECE-1B-merge-PRYMMAL/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2712 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4235 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1012 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.281 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3801 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2906 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lilrg_ece_finetunning.json b/data/models/lilrg_ece_finetunning.json deleted file mode 100644 index 0b7bd0933e0ea5b2ff167b90c4b3122d3b3debb5..0000000000000000000000000000000000000000 --- a/data/models/lilrg_ece_finetunning.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ECE_Finetunning", - "id": "LilRg/ECE_Finetunning", - "developer": "LilRg", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "?", - "params_billions": "16.061" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/LilRg_ECE_Finetunning/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0445 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4732 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0453 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2827 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3839 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3191 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lilrg_prymmal-6b-slerp.json b/data/models/lilrg_prymmal-6b-slerp.json deleted file mode 100644 index 75cb06f871e43f9d893d8238fd70e89940622752..0000000000000000000000000000000000000000 --- a/data/models/lilrg_prymmal-6b-slerp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "PRYMMAL-6B-slerp", - "id": "LilRg/PRYMMAL-6B-slerp", - "developer": "LilRg", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.293" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/LilRg_PRYMMAL-6B-slerp/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1153 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2868 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2458 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3698 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1108 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lilrg_prymmal-ece-7b-slerp-v3.json b/data/models/lilrg_prymmal-ece-7b-slerp-v3.json deleted file mode 100644 index 4490b4d93a9af2fb16bea45dde6faf62cd42fce4..0000000000000000000000000000000000000000 --- a/data/models/lilrg_prymmal-ece-7b-slerp-v3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "PRYMMAL-ECE-7B-SLERP-V3", - "id": "LilRg/PRYMMAL-ECE-7B-SLERP-V3", - "developer": "LilRg", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/LilRg_PRYMMAL-ECE-7B-SLERP-V3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1243 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2957 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0098 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2567 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3671 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1127 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lilrg_prymmal-ece-7b-slerp-v4.json b/data/models/lilrg_prymmal-ece-7b-slerp-v4.json deleted file mode 100644 index c158d417c7a89fa2273fdf908da1d23d679b209d..0000000000000000000000000000000000000000 --- a/data/models/lilrg_prymmal-ece-7b-slerp-v4.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "PRYMMAL-ECE-7B-SLERP-V4", - "id": "LilRg/PRYMMAL-ECE-7B-SLERP-V4", - "developer": "LilRg", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/LilRg_PRYMMAL-ECE-7B-SLERP-V4/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1249 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2957 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0098 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2567 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3671 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1127 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lilrg_prymmal-ece-7b-slerp-v5.json b/data/models/lilrg_prymmal-ece-7b-slerp-v5.json deleted file mode 100644 index 6885148ccc885756bb020e4b86aa07e795cacfeb..0000000000000000000000000000000000000000 --- a/data/models/lilrg_prymmal-ece-7b-slerp-v5.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "PRYMMAL-ECE-7B-SLERP-V5", - "id": "LilRg/PRYMMAL-ECE-7B-SLERP-V5", - "developer": "LilRg", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/LilRg_PRYMMAL-ECE-7B-SLERP-V5/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1249 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2957 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0098 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2567 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3671 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1127 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lilrg_prymmal-ece-7b-slerp-v6.json b/data/models/lilrg_prymmal-ece-7b-slerp-v6.json deleted file mode 100644 index 0c8a64981ac7b21f4e2227c90f704cc44e8340a7..0000000000000000000000000000000000000000 --- a/data/models/lilrg_prymmal-ece-7b-slerp-v6.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "PRYMMAL-ECE-7B-SLERP-V6", - "id": "LilRg/PRYMMAL-ECE-7B-SLERP-V6", - "developer": "LilRg", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/LilRg_PRYMMAL-ECE-7B-SLERP-V6/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1243 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2957 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0098 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2567 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3671 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1127 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lilrg_prymmal-ece-7b-slerp-v7.json b/data/models/lilrg_prymmal-ece-7b-slerp-v7.json deleted file mode 100644 index d619d73b3b2f1653200fc04dd6465080261a45de..0000000000000000000000000000000000000000 --- a/data/models/lilrg_prymmal-ece-7b-slerp-v7.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "PRYMMAL-ECE-7B-SLERP-V7", - "id": "LilRg/PRYMMAL-ECE-7B-SLERP-V7", - "developer": "LilRg", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/LilRg_PRYMMAL-ECE-7B-SLERP-V7/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1249 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2957 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0098 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2567 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3671 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1127 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lilrg_prymmal-slerp-merge.json b/data/models/lilrg_prymmal-slerp-merge.json deleted file mode 100644 index eccdfe61262b1a97df887bfbfd4792567d2b94be..0000000000000000000000000000000000000000 --- a/data/models/lilrg_prymmal-slerp-merge.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "PRYMMAL-slerp-Merge", - "id": "LilRg/PRYMMAL-slerp-Merge", - "developer": "LilRg", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Phi3ForCausalLM", - "params_billions": "3.821" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/LilRg_PRYMMAL-slerp-Merge/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3044 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5364 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1616 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3205 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4635 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3863 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/limyeri_codemind-llama3-8b-unsloth_v2-merged.json b/data/models/limyeri_codemind-llama3-8b-unsloth_v2-merged.json deleted file mode 100644 index 3958ad726aac4fec30f236a648eecf1f5ae19dd4..0000000000000000000000000000000000000000 --- a/data/models/limyeri_codemind-llama3-8b-unsloth_v2-merged.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "CodeMind-Llama3-8B-unsloth_v2-merged", - "id": "LimYeri/CodeMind-Llama3-8B-unsloth_v2-merged", - "developer": "LimYeri", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/LimYeri_CodeMind-Llama3-8B-unsloth_v2-merged/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6946 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.486 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0665 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2651 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3316 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3506 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/limyeri_codemind-llama3-8b-unsloth_v3-merged.json b/data/models/limyeri_codemind-llama3-8b-unsloth_v3-merged.json deleted file mode 100644 index fc1fceb502aa92621e45ff4b45ea7cf1289ebc64..0000000000000000000000000000000000000000 --- a/data/models/limyeri_codemind-llama3-8b-unsloth_v3-merged.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "CodeMind-Llama3-8B-unsloth_v3-merged", - "id": "LimYeri/CodeMind-Llama3-8B-unsloth_v3-merged", - "developer": "LimYeri", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/LimYeri_CodeMind-Llama3-8B-unsloth_v3-merged/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6763 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4908 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.068 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2584 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3356 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3496 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/limyeri_codemind-llama3-8b-unsloth_v4-one-dpo-merged.json b/data/models/limyeri_codemind-llama3-8b-unsloth_v4-one-dpo-merged.json deleted file mode 100644 index b466bb8d606bb1f2fd7f347370ec258ae6e56827..0000000000000000000000000000000000000000 --- a/data/models/limyeri_codemind-llama3-8b-unsloth_v4-one-dpo-merged.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "CodeMind-Llama3-8B-unsloth_v4-one-DPO-merged", - "id": "LimYeri/CodeMind-Llama3-8B-unsloth_v4-one-DPO-merged", - "developer": "LimYeri", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/LimYeri_CodeMind-Llama3-8B-unsloth_v4-one-DPO-merged/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6492 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4853 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.068 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2685 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3608 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3354 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/limyeri_codemind-llama3-8b-unsloth_v4-one-merged.json b/data/models/limyeri_codemind-llama3-8b-unsloth_v4-one-merged.json deleted file mode 100644 index 5d64be21bad94e68da6e3873fd278396a94b401d..0000000000000000000000000000000000000000 --- a/data/models/limyeri_codemind-llama3-8b-unsloth_v4-one-merged.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "CodeMind-Llama3-8B-unsloth_v4-one-merged", - "id": "LimYeri/CodeMind-Llama3-8B-unsloth_v4-one-merged", - "developer": "LimYeri", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/LimYeri_CodeMind-Llama3-8B-unsloth_v4-one-merged/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3211 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4739 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0551 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3096 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4069 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3353 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/limyeri_codemind-llama3.1-8b-unsloth-merged.json b/data/models/limyeri_codemind-llama3.1-8b-unsloth-merged.json deleted file mode 100644 index ae2f1bb80915742e9dd433880cfdecf3f148a317..0000000000000000000000000000000000000000 --- a/data/models/limyeri_codemind-llama3.1-8b-unsloth-merged.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "CodeMind-Llama3.1-8B-unsloth-merged", - "id": "LimYeri/CodeMind-Llama3.1-8B-unsloth-merged", - "developer": "LimYeri", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/LimYeri_CodeMind-Llama3.1-8B-unsloth-merged/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.649 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4695 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1088 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2643 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3752 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.334 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lkoenig_bbai_145.json b/data/models/lkoenig_bbai_145.json deleted file mode 100644 index f5c9065f05c6bfad6f1eb8f777e7010df84fe9e6..0000000000000000000000000000000000000000 --- a/data/models/lkoenig_bbai_145.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "BBAI_145_", - "id": "lkoenig/BBAI_145_", - "developer": "lkoenig", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/lkoenig_BBAI_145_/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.445 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5567 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.361 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3163 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4382 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.449 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lkoenig_bbai_200_gemma.json b/data/models/lkoenig_bbai_200_gemma.json deleted file mode 100644 index c80a5587a18a4f10ae8307cd20f7175ad1743e96..0000000000000000000000000000000000000000 --- a/data/models/lkoenig_bbai_200_gemma.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "BBAI_200_Gemma", - "id": "lkoenig/BBAI_200_Gemma", - "developer": "lkoenig", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "19.3" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/lkoenig_BBAI_200_Gemma/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0705 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3449 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2668 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3631 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1679 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lkoenig_bbai_212_qwencore.json b/data/models/lkoenig_bbai_212_qwencore.json deleted file mode 100644 index 0b335ba3a6a4c2cc2c4d1fa7598bedaf9b6496b5..0000000000000000000000000000000000000000 --- a/data/models/lkoenig_bbai_212_qwencore.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "BBAI_212_Qwencore", - "id": "lkoenig/BBAI_212_Qwencore", - "developer": "lkoenig", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.613" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/lkoenig_BBAI_212_Qwencore/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4384 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5569 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3489 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3163 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4343 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.449 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lkoenig_bbai_212_qwenlawlo.json b/data/models/lkoenig_bbai_212_qwenlawlo.json deleted file mode 100644 index 8fd14ccadb8bbefcbca3371c8023b1db5c9b8bdb..0000000000000000000000000000000000000000 --- a/data/models/lkoenig_bbai_212_qwenlawlo.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "BBAI_212_QwenLawLo", - "id": "lkoenig/BBAI_212_QwenLawLo", - "developer": "lkoenig", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/lkoenig_BBAI_212_QwenLawLo/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4566 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5574 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3603 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3163 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.437 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4489 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lkoenig_bbai_230_xiaqwen.json b/data/models/lkoenig_bbai_230_xiaqwen.json deleted file mode 100644 index 1e87f3e5750d459cacd559c30efc3c1f580cfa86..0000000000000000000000000000000000000000 --- a/data/models/lkoenig_bbai_230_xiaqwen.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "BBAI_230_Xiaqwen", - "id": "lkoenig/BBAI_230_Xiaqwen", - "developer": "lkoenig", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/lkoenig_BBAI_230_Xiaqwen/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4649 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5578 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3663 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3138 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4422 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4481 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lkoenig_bbai_375_qwendyancabs.json b/data/models/lkoenig_bbai_375_qwendyancabs.json deleted file mode 100644 index 81336453159abf8b6769fe03cd5b2a8270c1c794..0000000000000000000000000000000000000000 --- a/data/models/lkoenig_bbai_375_qwendyancabs.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "BBAI_375_QwenDyancabs", - "id": "lkoenig/BBAI_375_QwenDyancabs", - "developer": "lkoenig", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/lkoenig_BBAI_375_QwenDyancabs/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4566 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5571 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3776 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3129 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4462 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4476 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lkoenig_bbai_456_qwenkoen.json b/data/models/lkoenig_bbai_456_qwenkoen.json deleted file mode 100644 index 187f29cd89b7fd5c99fa27bccfd8675ac46ab85f..0000000000000000000000000000000000000000 --- a/data/models/lkoenig_bbai_456_qwenkoen.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "BBAI_456_QwenKoen", - "id": "lkoenig/BBAI_456_QwenKoen", - "developer": "lkoenig", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/lkoenig_BBAI_456_QwenKoen/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4529 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5553 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3686 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3129 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4395 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4469 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lkoenig_bbai_7b_koenqwendyan.json b/data/models/lkoenig_bbai_7b_koenqwendyan.json deleted file mode 100644 index 3b5158bab362de068cdd63ed065f161833f4bc51..0000000000000000000000000000000000000000 --- a/data/models/lkoenig_bbai_7b_koenqwendyan.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "BBAI_7B_KoenQwenDyan", - "id": "lkoenig/BBAI_7B_KoenQwenDyan", - "developer": "lkoenig", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/lkoenig_BBAI_7B_KoenQwenDyan/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5807 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5537 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3739 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.318 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4369 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.446 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lkoenig_bbai_7b_qwen2.5koen.json b/data/models/lkoenig_bbai_7b_qwen2.5koen.json deleted file mode 100644 index f130f728549b76734f2fdaeb2c28817350455a0d..0000000000000000000000000000000000000000 --- a/data/models/lkoenig_bbai_7b_qwen2.5koen.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "BBAI_7B_Qwen2.5koen", - "id": "lkoenig/BBAI_7B_Qwen2.5koen", - "developer": "lkoenig", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/lkoenig_BBAI_7B_Qwen2.5koen/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.46 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5544 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3656 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3129 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4369 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4485 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lkoenig_bbai_7b_qwendyancabslaw.json b/data/models/lkoenig_bbai_7b_qwendyancabslaw.json deleted file mode 100644 index 84762f6dcc8298f8a8985cc305935cb7af28e0a7..0000000000000000000000000000000000000000 --- a/data/models/lkoenig_bbai_7b_qwendyancabslaw.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "BBAI_7B_QwenDyancabsLAW", - "id": "lkoenig/BBAI_7B_QwenDyancabsLAW", - "developer": "lkoenig", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/lkoenig_BBAI_7B_QwenDyancabsLAW/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.555 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5579 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3678 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3188 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4461 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4471 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lkoenig_bbai_7b_qwendyankoenlo.json b/data/models/lkoenig_bbai_7b_qwendyankoenlo.json deleted file mode 100644 index 781e51173c1fdbdf4f4debb0569e31d2440ca8be..0000000000000000000000000000000000000000 --- a/data/models/lkoenig_bbai_7b_qwendyankoenlo.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "BBAI_7B_QwenDyanKoenLo", - "id": "lkoenig/BBAI_7B_QwenDyanKoenLo", - "developer": "lkoenig", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/lkoenig_BBAI_7B_QwenDyanKoenLo/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4663 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5562 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.364 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3188 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4343 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4465 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/llm-blender_pairrm-hf.json b/data/models/llm-blender_pairrm-hf.json deleted file mode 100644 index f44402213b9a113a45d2d0606b30764bbe4fec04..0000000000000000000000000000000000000000 --- a/data/models/llm-blender_pairrm-hf.json +++ /dev/null @@ -1,144 +0,0 @@ -{ - "model_info": { - "name": "llm-blender/PairRM-hf", - "id": "llm-blender/PairRM-hf", - "developer": "llm-blender", - "additional_details": { - "model_type": "Custom Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/llm-blender_PairRM-hf/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6087 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9022 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5219 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.477 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4898 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6961 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/llm360_k2-chat.json b/data/models/llm360_k2-chat.json deleted file mode 100644 index c25bbc73e2fa1db9055f237b199187ce72f9a261..0000000000000000000000000000000000000000 --- a/data/models/llm360_k2-chat.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "K2-Chat", - "id": "LLM360/K2-Chat", - "developer": "LLM360", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "65.286" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/LLM360_K2-Chat/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5152 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5358 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1035 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3062 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.457 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3371 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/llm360_k2.json b/data/models/llm360_k2.json deleted file mode 100644 index 393f86d275ebac681e8ef7910c20e22fb156b81c..0000000000000000000000000000000000000000 --- a/data/models/llm360_k2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "K2", - "id": "LLM360/K2", - "developer": "LLM360", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "65.286" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/LLM360_K2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2252 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4972 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0272 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2768 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.398 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3004 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/llm4binary_llm4decompile-1.3b-v2.json b/data/models/llm4binary_llm4decompile-1.3b-v2.json deleted file mode 100644 index 048cba0340dec77675392a15a1b3c295b8a71d40..0000000000000000000000000000000000000000 --- a/data/models/llm4binary_llm4decompile-1.3b-v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "llm4decompile-1.3b-v2", - "id": "LLM4Binary/llm4decompile-1.3b-v2", - "developer": "LLM4Binary", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.346" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/LLM4Binary_llm4decompile-1.3b-v2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2268 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3272 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0128 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2357 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4072 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1209 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/llmat_mistral-v0.3-7b-orpo.json b/data/models/llmat_mistral-v0.3-7b-orpo.json deleted file mode 100644 index 3a1b947d84c5d76bb2423237a20cbf150415592e..0000000000000000000000000000000000000000 --- a/data/models/llmat_mistral-v0.3-7b-orpo.json +++ /dev/null @@ -1,275 +0,0 @@ -{ - "model_info": { - "name": "Mistral-v0.3-7B-ORPO", - "id": "llmat/Mistral-v0.3-7B-ORPO", - "developer": "llmat", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "7.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/llmat_Mistral-v0.3-7B-ORPO/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.364 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4005 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0015 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2693 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3529 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2301 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "hfopenllm_v2/llmat_Mistral-v0.3-7B-ORPO/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.377 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3978 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0242 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2668 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3555 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2278 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/llnyou_ece-prymmal-yl-1b-slerp-v5.json b/data/models/llnyou_ece-prymmal-yl-1b-slerp-v5.json deleted file mode 100644 index 2b137b18fe8f3736a8d24501514c7bb2d555b485..0000000000000000000000000000000000000000 --- a/data/models/llnyou_ece-prymmal-yl-1b-slerp-v5.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ECE-PRYMMAL-YL-1B-SLERP-V5", - "id": "llnYou/ECE-PRYMMAL-YL-1B-SLERP-V5", - "developer": "llnYou", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.544" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/llnYou_ECE-PRYMMAL-YL-1B-SLERP-V5/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3313 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4233 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.111 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2861 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3868 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2931 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/llnyou_ece-prymmal-yl-1b-slerp-v6.json b/data/models/llnyou_ece-prymmal-yl-1b-slerp-v6.json deleted file mode 100644 index 31837688a92100a4638e83ba04eb3133c8103759..0000000000000000000000000000000000000000 --- a/data/models/llnyou_ece-prymmal-yl-1b-slerp-v6.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ECE-PRYMMAL-YL-1B-SLERP-V6", - "id": "llnYou/ECE-PRYMMAL-YL-1B-SLERP-V6", - "developer": "llnYou", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.357" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/llnYou_ECE-PRYMMAL-YL-1B-SLERP-V6/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1388 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3944 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0023 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2903 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3928 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.235 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/llnyou_ece-prymmal-yl-3b-slerp-v1.json b/data/models/llnyou_ece-prymmal-yl-3b-slerp-v1.json deleted file mode 100644 index 07aafc5fa0c9840b8deb4d1c3eb5536a758d747a..0000000000000000000000000000000000000000 --- a/data/models/llnyou_ece-prymmal-yl-3b-slerp-v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ECE-PRYMMAL-YL-3B-SLERP-V1", - "id": "llnYou/ECE-PRYMMAL-YL-3B-SLERP-V1", - "developer": "llnYou", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "2.81" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/llnYou_ECE-PRYMMAL-YL-3B-SLERP-V1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2346 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4018 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0091 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2936 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3364 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.285 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/llnyou_ece-prymmal-yl-3b-slerp-v2.json b/data/models/llnyou_ece-prymmal-yl-3b-slerp-v2.json deleted file mode 100644 index c527ab08af418e97295c39cc91bd8416678ec702..0000000000000000000000000000000000000000 --- a/data/models/llnyou_ece-prymmal-yl-3b-slerp-v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ECE-PRYMMAL-YL-3B-SLERP-V2", - "id": "llnYou/ECE-PRYMMAL-YL-3B-SLERP-V2", - "developer": "llnYou", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "2.81" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/llnYou_ECE-PRYMMAL-YL-3B-SLERP-V2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2309 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.399 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0128 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2768 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3588 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.29 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/llnyou_ece-prymmal-yl-3b-slerp-v3.json b/data/models/llnyou_ece-prymmal-yl-3b-slerp-v3.json deleted file mode 100644 index c96ef2815f4e2950690f3ddbbb8c88155d26bbe1..0000000000000000000000000000000000000000 --- a/data/models/llnyou_ece-prymmal-yl-3b-slerp-v3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ECE-PRYMMAL-YL-3B-SLERP-V3", - "id": "llnYou/ECE-PRYMMAL-YL-3B-SLERP-V3", - "developer": "llnYou", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Phi3ForCausalLM", - "params_billions": "3.821" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/llnYou_ECE-PRYMMAL-YL-3B-SLERP-V3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3581 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5473 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1299 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3045 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4361 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4043 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lmsys_vicuna-13b-v1.3.json b/data/models/lmsys_vicuna-13b-v1.3.json deleted file mode 100644 index cf96e565f77ddee654fb62fb96676e45874054b4..0000000000000000000000000000000000000000 --- a/data/models/lmsys_vicuna-13b-v1.3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "vicuna-13b-v1.3", - "id": "lmsys/vicuna-13b-v1.3", - "developer": "lmsys", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "13.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/lmsys_vicuna-13b-v1.3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3344 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3384 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0144 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2676 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3727 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2243 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lmsys_vicuna-7b-v1.3.json b/data/models/lmsys_vicuna-7b-v1.3.json deleted file mode 100644 index 4eddc59dbded026a93a19918735239378dc4592b..0000000000000000000000000000000000000000 --- a/data/models/lmsys_vicuna-7b-v1.3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "vicuna-7b-v1.3", - "id": "lmsys/vicuna-7b-v1.3", - "developer": "lmsys", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "7.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/lmsys_vicuna-7b-v1.3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2909 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3298 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0128 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2424 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3793 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1838 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lmsys_vicuna-7b-v1.5.json b/data/models/lmsys_vicuna-7b-v1.5.json deleted file mode 100644 index cdad485bd08965756ecd5c0a8fdbdacd4eb7e3c9..0000000000000000000000000000000000000000 --- a/data/models/lmsys_vicuna-7b-v1.5.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "vicuna-7b-v1.5", - "id": "lmsys/vicuna-7b-v1.5", - "developer": "lmsys", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "7.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/lmsys_vicuna-7b-v1.5/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2352 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3947 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0136 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2584 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4231 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2147 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lmsys_vicuna-v1.3-13b.json b/data/models/lmsys_vicuna-v1.3-13b.json deleted file mode 100644 index 20e3dfe4a38e109158225423aa2b384b5f1fa1d1..0000000000000000000000000000000000000000 --- a/data/models/lmsys_vicuna-v1.3-13b.json +++ /dev/null @@ -1,674 +0,0 @@ -{ - "model_info": { - "name": "Vicuna v1.3 13B", - "id": "lmsys/Vicuna-v1.3-13B", - "developer": "lmsys", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_classic/lmsys_Vicuna-v1.3-13B/1774096308.339228", - "retrieved_timestamp": "1774096308.339228", - "source_metadata": { - "source_name": "helm_classic", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_classic", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_classic", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperform on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.706, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Calibration": "{\"description\": \"\", \"tab\": \"Calibration\", \"score\": \"0.27488436632747454\"}", - "Mean win rate - Robustness": "{\"description\": \"\", \"tab\": \"Robustness\", \"score\": \"0.7320745920745921\"}", - "Mean win rate - Fairness": "{\"description\": \"\", \"tab\": \"Fairness\", \"score\": \"0.7154545454545455\"}", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}", - "Mean win rate - Bias": "{\"description\": \"\", \"tab\": \"Bias\", \"score\": \"0.5333173629091996\"}", - "Mean win rate - Toxicity": "{\"description\": \"\", \"tab\": \"Toxicity\", \"score\": \"0.5758158508158508\"}", - "Mean win rate - Summarization metrics": "{\"description\": \"\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.462, - "details": { - "description": "min=0.298, mean=0.462, max=0.72, sum=2.308 (5)", - "tab": "Accuracy", - "MMLU - ECE (10-bin)": "{\"description\": \"min=0.156, mean=0.194, max=0.246, sum=0.972 (5)\", \"tab\": \"Calibration\", \"score\": \"0.19445587267296924\"}", - "MMLU - EM (Robustness)": "{\"description\": \"min=0.237, mean=0.413, max=0.69, sum=2.067 (5)\", \"tab\": \"Robustness\", \"score\": \"0.4133684210526316\"}", - "MMLU - EM (Fairness)": "{\"description\": \"min=0.228, mean=0.424, max=0.7, sum=2.118 (5)\", \"tab\": \"Fairness\", \"score\": \"0.4236140350877193\"}", - "MMLU - Denoised inference time (s)": "{\"description\": \"5 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=397.65, mean=522.547, max=684.675, sum=2612.735 (5)\", \"tab\": \"General information\", \"score\": \"522.5470877192982\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "MMLU - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "BoolQ", - "source_data": { - "dataset_name": "BoolQ", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on BoolQ", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.808, - "details": { - "description": "min=0.808, mean=0.808, max=0.808, sum=0.808 (1)", - "tab": "Accuracy", - "BoolQ - ECE (10-bin)": "{\"description\": \"min=0.159, mean=0.159, max=0.159, sum=0.159 (1)\", \"tab\": \"Calibration\", \"score\": \"0.15912327464389103\"}", - "BoolQ - EM (Robustness)": "{\"description\": \"min=0.757, mean=0.757, max=0.757, sum=0.757 (1)\", \"tab\": \"Robustness\", \"score\": \"0.757\"}", - "BoolQ - EM (Fairness)": "{\"description\": \"min=0.748, mean=0.748, max=0.748, sum=0.748 (1)\", \"tab\": \"Fairness\", \"score\": \"0.748\"}", - "BoolQ - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "BoolQ - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "BoolQ - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "BoolQ - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "BoolQ - # prompt tokens": "{\"description\": \"min=1439.447, mean=1439.447, max=1439.447, sum=1439.447 (1)\", \"tab\": \"General information\", \"score\": \"1439.447\"}", - "BoolQ - # output tokens": "{\"description\": \"min=4.996, mean=4.996, max=4.996, sum=4.996 (1)\", \"tab\": \"General information\", \"score\": \"4.996\"}", - "BoolQ - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "BoolQ - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.691, - "details": { - "description": "min=0.691, mean=0.691, max=0.691, sum=0.691 (1)", - "tab": "Accuracy", - "NarrativeQA - ECE (10-bin)": "{\"description\": \"min=0.257, mean=0.257, max=0.257, sum=0.257 (1)\", \"tab\": \"Calibration\", \"score\": \"0.25677737638719905\"}", - "NarrativeQA - F1 (Robustness)": "{\"description\": \"min=0.525, mean=0.525, max=0.525, sum=0.525 (1)\", \"tab\": \"Robustness\", \"score\": \"0.5253621693457193\"}", - "NarrativeQA - F1 (Fairness)": "{\"description\": \"min=0.607, mean=0.607, max=0.607, sum=0.607 (1)\", \"tab\": \"Fairness\", \"score\": \"0.6066076692752655\"}", - "NarrativeQA - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=1.437, mean=1.437, max=1.437, sum=1.437 (1)\", \"tab\": \"General information\", \"score\": \"1.4366197183098592\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=1541.115, mean=1541.115, max=1541.115, sum=1541.115 (1)\", \"tab\": \"General information\", \"score\": \"1541.1154929577465\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=67.575, mean=67.575, max=67.575, sum=67.575 (1)\", \"tab\": \"General information\", \"score\": \"67.57464788732395\"}", - "NarrativeQA - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NarrativeQA - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NarrativeQA - Stereotypes (gender)": "{\"description\": \"min=0.417, mean=0.417, max=0.417, sum=0.417 (1)\", \"tab\": \"Bias\", \"score\": \"0.41666666666666663\"}", - "NarrativeQA - Representation (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=0.667 (1)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666667\"}", - "NarrativeQA - Representation (gender)": "{\"description\": \"min=0.181, mean=0.181, max=0.181, sum=0.181 (1)\", \"tab\": \"Bias\", \"score\": \"0.1806282722513089\"}", - "NarrativeQA - Toxic fraction": "{\"description\": \"min=0.008, mean=0.008, max=0.008, sum=0.008 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.008450704225352112\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (open-book)", - "source_data": { - "dataset_name": "NaturalQuestions (open-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (open-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.686, - "details": { - "description": "min=0.686, mean=0.686, max=0.686, sum=0.686 (1)", - "tab": "Accuracy", - "NaturalQuestions (closed-book) - ECE (10-bin)": "{\"description\": \"min=0.202, mean=0.202, max=0.202, sum=0.202 (1)\", \"tab\": \"Calibration\", \"score\": \"0.20199999735253094\"}", - "NaturalQuestions (open-book) - ECE (10-bin)": "{\"description\": \"min=0.43, mean=0.43, max=0.43, sum=0.43 (1)\", \"tab\": \"Calibration\", \"score\": \"0.4297157164166979\"}", - "NaturalQuestions (closed-book) - F1 (Robustness)": "{\"description\": \"min=0.273, mean=0.273, max=0.273, sum=0.273 (1)\", \"tab\": \"Robustness\", \"score\": \"0.2732835109469542\"}", - "NaturalQuestions (open-book) - F1 (Robustness)": "{\"description\": \"min=0.621, mean=0.621, max=0.621, sum=0.621 (1)\", \"tab\": \"Robustness\", \"score\": \"0.6205537766211775\"}", - "NaturalQuestions (closed-book) - F1 (Fairness)": "{\"description\": \"min=0.266, mean=0.266, max=0.266, sum=0.266 (1)\", \"tab\": \"Fairness\", \"score\": \"0.26608326669652704\"}", - "NaturalQuestions (open-book) - F1 (Fairness)": "{\"description\": \"min=0.63, mean=0.63, max=0.63, sum=0.63 (1)\", \"tab\": \"Fairness\", \"score\": \"0.6295785534387982\"}", - "NaturalQuestions (closed-book) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NaturalQuestions (open-book) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=137.383, mean=137.383, max=137.383, sum=137.383 (1)\", \"tab\": \"General information\", \"score\": \"137.383\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=299.508, mean=299.508, max=299.508, sum=299.508 (1)\", \"tab\": \"General information\", \"score\": \"299.508\"}", - "NaturalQuestions (closed-book) - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=3.722, mean=3.722, max=3.722, sum=3.722 (1)\", \"tab\": \"General information\", \"score\": \"3.722\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.049, mean=0.049, max=0.049, sum=0.049 (1)\", \"tab\": \"General information\", \"score\": \"0.049\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1407.178, mean=1407.178, max=1407.178, sum=1407.178 (1)\", \"tab\": \"General information\", \"score\": \"1407.178\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=266.895, mean=266.895, max=266.895, sum=266.895 (1)\", \"tab\": \"General information\", \"score\": \"266.895\"}", - "NaturalQuestions (open-book) - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NaturalQuestions (closed-book) - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=0.667 (1)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "NaturalQuestions (closed-book) - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - Representation (race)": "{\"description\": \"min=0.364, mean=0.364, max=0.364, sum=0.364 (1)\", \"tab\": \"Bias\", \"score\": \"0.363914373088685\"}", - "NaturalQuestions (closed-book) - Representation (gender)": "{\"description\": \"min=0.132, mean=0.132, max=0.132, sum=0.132 (1)\", \"tab\": \"Bias\", \"score\": \"0.13157894736842105\"}", - "NaturalQuestions (open-book) - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=0.667 (1)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666667\"}", - "NaturalQuestions (open-book) - Stereotypes (gender)": "{\"description\": \"min=0.5, mean=0.5, max=0.5, sum=0.5 (1)\", \"tab\": \"Bias\", \"score\": \"0.5\"}", - "NaturalQuestions (open-book) - Representation (race)": "{\"description\": \"min=0.484, mean=0.484, max=0.484, sum=0.484 (1)\", \"tab\": \"Bias\", \"score\": \"0.4838709677419355\"}", - "NaturalQuestions (open-book) - Representation (gender)": "{\"description\": \"min=0.293, mean=0.293, max=0.293, sum=0.293 (1)\", \"tab\": \"Bias\", \"score\": \"0.29310344827586204\"}", - "NaturalQuestions (closed-book) - Toxic fraction": "{\"description\": \"min=0.001, mean=0.001, max=0.001, sum=0.001 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.001\"}", - "NaturalQuestions (open-book) - Toxic fraction": "{\"description\": \"min=0.001, mean=0.001, max=0.001, sum=0.001 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.001\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "QuAC", - "source_data": { - "dataset_name": "QuAC", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on QuAC", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.403, - "details": { - "description": "min=0.403, mean=0.403, max=0.403, sum=0.403 (1)", - "tab": "Accuracy", - "QuAC - ECE (10-bin)": "{\"description\": \"min=0.103, mean=0.103, max=0.103, sum=0.103 (1)\", \"tab\": \"Calibration\", \"score\": \"0.10339686685910766\"}", - "QuAC - F1 (Robustness)": "{\"description\": \"min=0.247, mean=0.247, max=0.247, sum=0.247 (1)\", \"tab\": \"Robustness\", \"score\": \"0.24738453163162216\"}", - "QuAC - F1 (Fairness)": "{\"description\": \"min=0.324, mean=0.324, max=0.324, sum=0.324 (1)\", \"tab\": \"Fairness\", \"score\": \"0.32414193488324744\"}", - "QuAC - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "QuAC - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "QuAC - # train": "{\"description\": \"min=0.507, mean=0.507, max=0.507, sum=0.507 (1)\", \"tab\": \"General information\", \"score\": \"0.507\"}", - "QuAC - truncated": "{\"description\": \"min=0.06, mean=0.06, max=0.06, sum=0.06 (1)\", \"tab\": \"General information\", \"score\": \"0.06\"}", - "QuAC - # prompt tokens": "{\"description\": \"min=1498.657, mean=1498.657, max=1498.657, sum=1498.657 (1)\", \"tab\": \"General information\", \"score\": \"1498.657\"}", - "QuAC - # output tokens": "{\"description\": \"min=77.743, mean=77.743, max=77.743, sum=77.743 (1)\", \"tab\": \"General information\", \"score\": \"77.743\"}", - "QuAC - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "QuAC - Stereotypes (race)": "{\"description\": \"min=0.63, mean=0.63, max=0.63, sum=0.63 (1)\", \"tab\": \"Bias\", \"score\": \"0.6296296296296295\"}", - "QuAC - Stereotypes (gender)": "{\"description\": \"min=0.408, mean=0.408, max=0.408, sum=0.408 (1)\", \"tab\": \"Bias\", \"score\": \"0.4083074125172457\"}", - "QuAC - Representation (race)": "{\"description\": \"min=0.289, mean=0.289, max=0.289, sum=0.289 (1)\", \"tab\": \"Bias\", \"score\": \"0.28888888888888886\"}", - "QuAC - Representation (gender)": "{\"description\": \"min=0.242, mean=0.242, max=0.242, sum=0.242 (1)\", \"tab\": \"Bias\", \"score\": \"0.2418952618453865\"}", - "QuAC - Toxic fraction": "{\"description\": \"min=0.001, mean=0.001, max=0.001, sum=0.001 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.001\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "HellaSwag", - "source_data": { - "dataset_name": "HellaSwag", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on HellaSwag", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "HellaSwag - ECE (10-bin)": "{\"description\": \"No matching runs\", \"tab\": \"Calibration\", \"score\": \"\"}", - "HellaSwag - EM (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "HellaSwag - EM (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "HellaSwag - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "HellaSwag - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "OpenbookQA - ECE (10-bin)": "{\"description\": \"No matching runs\", \"tab\": \"Calibration\", \"score\": \"\"}", - "OpenbookQA - EM (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "OpenbookQA - EM (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "OpenbookQA - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "OpenbookQA - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "TruthfulQA", - "source_data": { - "dataset_name": "TruthfulQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on TruthfulQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.385, - "details": { - "description": "min=0.385, mean=0.385, max=0.385, sum=0.385 (1)", - "tab": "Accuracy", - "TruthfulQA - ECE (10-bin)": "{\"description\": \"min=0.316, mean=0.316, max=0.316, sum=0.316 (1)\", \"tab\": \"Calibration\", \"score\": \"0.31581376966800645\"}", - "TruthfulQA - EM (Robustness)": "{\"description\": \"min=0.341, mean=0.341, max=0.341, sum=0.341 (1)\", \"tab\": \"Robustness\", \"score\": \"0.3409785932721712\"}", - "TruthfulQA - EM (Fairness)": "{\"description\": \"min=0.315, mean=0.315, max=0.315, sum=0.315 (1)\", \"tab\": \"Fairness\", \"score\": \"0.3149847094801223\"}", - "TruthfulQA - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "TruthfulQA - # eval": "{\"description\": \"min=654, mean=654, max=654, sum=654 (1)\", \"tab\": \"General information\", \"score\": \"654.0\"}", - "TruthfulQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "TruthfulQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "TruthfulQA - # prompt tokens": "{\"description\": \"min=524.602, mean=524.602, max=524.602, sum=524.602 (1)\", \"tab\": \"General information\", \"score\": \"524.6024464831804\"}", - "TruthfulQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "TruthfulQA - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MS MARCO (TREC)", - "source_data": { - "dataset_name": "MS MARCO (TREC)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "NDCG@10 on MS MARCO (TREC)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "MS MARCO (regular) - RR@10 (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "MS MARCO (TREC) - NDCG@10 (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "MS MARCO (regular) - RR@10 (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "MS MARCO (TREC) - NDCG@10 (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "MS MARCO (regular) - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MS MARCO (TREC) - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MS MARCO (regular) - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "MS MARCO (TREC) - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CNN/DailyMail", - "source_data": { - "dataset_name": "CNN/DailyMail", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on CNN/DailyMail", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "CNN/DailyMail - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "CNN/DailyMail - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "CNN/DailyMail - SummaC": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - QAFactEval": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - BERTScore (F1)": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - Coverage": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - Density": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - Compression": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-faithfulness": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-relevance": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-coherence": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "XSUM", - "source_data": { - "dataset_name": "XSUM", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on XSUM", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "XSUM - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "XSUM - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "XSUM - SummaC": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - QAFactEval": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - BERTScore (F1)": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - Coverage": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - Density": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - Compression": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-faithfulness": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-relevance": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-coherence": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "IMDB", - "source_data": { - "dataset_name": "IMDB", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on IMDB", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.762, - "details": { - "description": "min=0.762, mean=0.762, max=0.762, sum=0.762 (1)", - "tab": "Accuracy", - "IMDB - ECE (10-bin)": "{\"description\": \"min=0.183, mean=0.183, max=0.183, sum=0.183 (1)\", \"tab\": \"Calibration\", \"score\": \"0.18259660460611343\"}", - "IMDB - EM (Robustness)": "{\"description\": \"min=0.674, mean=0.674, max=0.674, sum=0.674 (1)\", \"tab\": \"Robustness\", \"score\": \"0.674\"}", - "IMDB - EM (Fairness)": "{\"description\": \"min=0.707, mean=0.707, max=0.707, sum=0.707 (1)\", \"tab\": \"Fairness\", \"score\": \"0.707\"}", - "IMDB - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "IMDB - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "IMDB - # train": "{\"description\": \"min=2.781, mean=2.781, max=2.781, sum=2.781 (1)\", \"tab\": \"General information\", \"score\": \"2.781\"}", - "IMDB - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IMDB - # prompt tokens": "{\"description\": \"min=1751.213, mean=1751.213, max=1751.213, sum=1751.213 (1)\", \"tab\": \"General information\", \"score\": \"1751.213\"}", - "IMDB - # output tokens": "{\"description\": \"min=3.32, mean=3.32, max=3.32, sum=3.32 (1)\", \"tab\": \"General information\", \"score\": \"3.32\"}", - "IMDB - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "IMDB - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CivilComments", - "source_data": { - "dataset_name": "CivilComments", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on CivilComments", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.645, - "details": { - "description": "min=0.247, mean=0.645, max=0.946, sum=11.602 (18)", - "tab": "Accuracy", - "CivilComments - ECE (10-bin)": "{\"description\": \"min=0.086, mean=0.253, max=0.415, sum=4.559 (18)\", \"tab\": \"Calibration\", \"score\": \"0.25325054290553783\"}", - "CivilComments - EM (Robustness)": "{\"description\": \"min=0.177, mean=0.593, max=0.932, sum=10.679 (18)\", \"tab\": \"Robustness\", \"score\": \"0.5932501359027997\"}", - "CivilComments - EM (Fairness)": "{\"description\": \"min=0.139, mean=0.569, max=0.946, sum=10.248 (18)\", \"tab\": \"Fairness\", \"score\": \"0.5693148383516141\"}", - "CivilComments - Denoised inference time (s)": "{\"description\": \"9 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "CivilComments - # eval": "{\"description\": \"min=74, mean=371.556, max=683, sum=6688 (18)\", \"tab\": \"General information\", \"score\": \"371.55555555555554\"}", - "CivilComments - # train": "{\"description\": \"min=5, mean=5, max=5, sum=90 (18)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CivilComments - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (18)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CivilComments - # prompt tokens": "{\"description\": \"min=404.732, mean=855.241, max=1417.567, sum=15394.339 (18)\", \"tab\": \"General information\", \"score\": \"855.2410378605821\"}", - "CivilComments - # output tokens": "{\"description\": \"min=2, mean=2.59, max=4.159, sum=46.618 (18)\", \"tab\": \"General information\", \"score\": \"2.589879611958418\"}", - "CivilComments - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=18 (18)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "CivilComments - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Toxic fraction": "{\"description\": \"9 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "RAFT", - "source_data": { - "dataset_name": "RAFT", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on RAFT", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.657, - "details": { - "description": "min=0.175, mean=0.657, max=0.9, sum=7.225 (11)", - "tab": "Accuracy", - "RAFT - ECE (10-bin)": "{\"description\": \"min=0.212, mean=0.376, max=0.701, sum=4.137 (11)\", \"tab\": \"Calibration\", \"score\": \"0.37612291287489436\"}", - "RAFT - EM (Robustness)": "{\"description\": \"min=0.025, mean=0.591, max=0.875, sum=6.5 (11)\", \"tab\": \"Robustness\", \"score\": \"0.5909090909090909\"}", - "RAFT - EM (Fairness)": "{\"description\": \"min=0.125, mean=0.62, max=0.875, sum=6.825 (11)\", \"tab\": \"Fairness\", \"score\": \"0.6204545454545454\"}", - "RAFT - Denoised inference time (s)": "{\"description\": \"11 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "RAFT - # eval": "{\"description\": \"min=40, mean=40, max=40, sum=440 (11)\", \"tab\": \"General information\", \"score\": \"40.0\"}", - "RAFT - # train": "{\"description\": \"min=0.45, mean=4.552, max=5, sum=50.075 (11)\", \"tab\": \"General information\", \"score\": \"4.552272727272727\"}", - "RAFT - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (11)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "RAFT - # prompt tokens": "{\"description\": \"min=303.675, mean=954.111, max=1882.1, sum=10495.225 (11)\", \"tab\": \"General information\", \"score\": \"954.1113636363635\"}", - "RAFT - # output tokens": "{\"description\": \"min=5.3, mean=15.4, max=30, sum=169.4 (11)\", \"tab\": \"General information\", \"score\": \"15.399999999999999\"}", - "RAFT - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=11 (11)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "RAFT - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Toxic fraction": "{\"description\": \"11 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/lmsys_vicuna-v1.3-7b.json b/data/models/lmsys_vicuna-v1.3-7b.json deleted file mode 100644 index 662454a56a34f82dc0f50462f79634a93fac6620..0000000000000000000000000000000000000000 --- a/data/models/lmsys_vicuna-v1.3-7b.json +++ /dev/null @@ -1,674 +0,0 @@ -{ - "model_info": { - "name": "Vicuna v1.3 7B", - "id": "lmsys/Vicuna-v1.3-7B", - "developer": "lmsys", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_classic/lmsys_Vicuna-v1.3-7B/1774096308.339228", - "retrieved_timestamp": "1774096308.339228", - "source_metadata": { - "source_name": "helm_classic", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_classic", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_classic", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperform on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.625, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Calibration": "{\"description\": \"\", \"tab\": \"Calibration\", \"score\": \"0.20388529139685477\"}", - "Mean win rate - Robustness": "{\"description\": \"\", \"tab\": \"Robustness\", \"score\": \"0.662027972027972\"}", - "Mean win rate - Fairness": "{\"description\": \"\", \"tab\": \"Fairness\", \"score\": \"0.6221212121212122\"}", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}", - "Mean win rate - Bias": "{\"description\": \"\", \"tab\": \"Bias\", \"score\": \"0.5093893164757827\"}", - "Mean win rate - Toxicity": "{\"description\": \"\", \"tab\": \"Toxicity\", \"score\": \"0.8238927738927739\"}", - "Mean win rate - Summarization metrics": "{\"description\": \"\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.434, - "details": { - "description": "min=0.228, mean=0.434, max=0.7, sum=2.168 (5)", - "tab": "Accuracy", - "MMLU - ECE (10-bin)": "{\"description\": \"min=0.121, mean=0.176, max=0.315, sum=0.88 (5)\", \"tab\": \"Calibration\", \"score\": \"0.17593793416924502\"}", - "MMLU - EM (Robustness)": "{\"description\": \"min=0.175, mean=0.371, max=0.65, sum=1.855 (5)\", \"tab\": \"Robustness\", \"score\": \"0.3710877192982456\"}", - "MMLU - EM (Fairness)": "{\"description\": \"min=0.184, mean=0.385, max=0.68, sum=1.924 (5)\", \"tab\": \"Fairness\", \"score\": \"0.38484210526315793\"}", - "MMLU - Denoised inference time (s)": "{\"description\": \"5 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=397.65, mean=522.547, max=684.675, sum=2612.735 (5)\", \"tab\": \"General information\", \"score\": \"522.5470877192982\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "MMLU - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "BoolQ", - "source_data": { - "dataset_name": "BoolQ", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on BoolQ", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.76, - "details": { - "description": "min=0.76, mean=0.76, max=0.76, sum=0.76 (1)", - "tab": "Accuracy", - "BoolQ - ECE (10-bin)": "{\"description\": \"min=0.322, mean=0.322, max=0.322, sum=0.322 (1)\", \"tab\": \"Calibration\", \"score\": \"0.322404542566261\"}", - "BoolQ - EM (Robustness)": "{\"description\": \"min=0.672, mean=0.672, max=0.672, sum=0.672 (1)\", \"tab\": \"Robustness\", \"score\": \"0.672\"}", - "BoolQ - EM (Fairness)": "{\"description\": \"min=0.67, mean=0.67, max=0.67, sum=0.67 (1)\", \"tab\": \"Fairness\", \"score\": \"0.67\"}", - "BoolQ - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "BoolQ - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "BoolQ - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "BoolQ - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "BoolQ - # prompt tokens": "{\"description\": \"min=1439.447, mean=1439.447, max=1439.447, sum=1439.447 (1)\", \"tab\": \"General information\", \"score\": \"1439.447\"}", - "BoolQ - # output tokens": "{\"description\": \"min=4.412, mean=4.412, max=4.412, sum=4.412 (1)\", \"tab\": \"General information\", \"score\": \"4.412\"}", - "BoolQ - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "BoolQ - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.643, - "details": { - "description": "min=0.643, mean=0.643, max=0.643, sum=0.643 (1)", - "tab": "Accuracy", - "NarrativeQA - ECE (10-bin)": "{\"description\": \"min=0.084, mean=0.084, max=0.084, sum=0.084 (1)\", \"tab\": \"Calibration\", \"score\": \"0.08355639800803456\"}", - "NarrativeQA - F1 (Robustness)": "{\"description\": \"min=0.5, mean=0.5, max=0.5, sum=0.5 (1)\", \"tab\": \"Robustness\", \"score\": \"0.499695916561912\"}", - "NarrativeQA - F1 (Fairness)": "{\"description\": \"min=0.553, mean=0.553, max=0.553, sum=0.553 (1)\", \"tab\": \"Fairness\", \"score\": \"0.5528194590567359\"}", - "NarrativeQA - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=1.437, mean=1.437, max=1.437, sum=1.437 (1)\", \"tab\": \"General information\", \"score\": \"1.4366197183098592\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=1541.115, mean=1541.115, max=1541.115, sum=1541.115 (1)\", \"tab\": \"General information\", \"score\": \"1541.1154929577465\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=19.287, mean=19.287, max=19.287, sum=19.287 (1)\", \"tab\": \"General information\", \"score\": \"19.28732394366197\"}", - "NarrativeQA - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NarrativeQA - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NarrativeQA - Stereotypes (gender)": "{\"description\": \"min=0.379, mean=0.379, max=0.379, sum=0.379 (1)\", \"tab\": \"Bias\", \"score\": \"0.3794642857142857\"}", - "NarrativeQA - Representation (race)": "{\"description\": \"min=0.373, mean=0.373, max=0.373, sum=0.373 (1)\", \"tab\": \"Bias\", \"score\": \"0.37254901960784315\"}", - "NarrativeQA - Representation (gender)": "{\"description\": \"min=0.186, mean=0.186, max=0.186, sum=0.186 (1)\", \"tab\": \"Bias\", \"score\": \"0.18604651162790695\"}", - "NarrativeQA - Toxic fraction": "{\"description\": \"min=0.008, mean=0.008, max=0.008, sum=0.008 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.008450704225352112\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (open-book)", - "source_data": { - "dataset_name": "NaturalQuestions (open-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (open-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.634, - "details": { - "description": "min=0.634, mean=0.634, max=0.634, sum=0.634 (1)", - "tab": "Accuracy", - "NaturalQuestions (closed-book) - ECE (10-bin)": "{\"description\": \"min=0.162, mean=0.162, max=0.162, sum=0.162 (1)\", \"tab\": \"Calibration\", \"score\": \"0.16180078530132275\"}", - "NaturalQuestions (open-book) - ECE (10-bin)": "{\"description\": \"min=0.413, mean=0.413, max=0.413, sum=0.413 (1)\", \"tab\": \"Calibration\", \"score\": \"0.41328409267406696\"}", - "NaturalQuestions (closed-book) - F1 (Robustness)": "{\"description\": \"min=0.214, mean=0.214, max=0.214, sum=0.214 (1)\", \"tab\": \"Robustness\", \"score\": \"0.213860378689308\"}", - "NaturalQuestions (open-book) - F1 (Robustness)": "{\"description\": \"min=0.539, mean=0.539, max=0.539, sum=0.539 (1)\", \"tab\": \"Robustness\", \"score\": \"0.5393637207184442\"}", - "NaturalQuestions (closed-book) - F1 (Fairness)": "{\"description\": \"min=0.224, mean=0.224, max=0.224, sum=0.224 (1)\", \"tab\": \"Fairness\", \"score\": \"0.22422961995096835\"}", - "NaturalQuestions (open-book) - F1 (Fairness)": "{\"description\": \"min=0.575, mean=0.575, max=0.575, sum=0.575 (1)\", \"tab\": \"Fairness\", \"score\": \"0.5749345098495453\"}", - "NaturalQuestions (closed-book) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NaturalQuestions (open-book) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=137.383, mean=137.383, max=137.383, sum=137.383 (1)\", \"tab\": \"General information\", \"score\": \"137.383\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=296.95, mean=296.95, max=296.95, sum=296.95 (1)\", \"tab\": \"General information\", \"score\": \"296.95\"}", - "NaturalQuestions (closed-book) - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=3.722, mean=3.722, max=3.722, sum=3.722 (1)\", \"tab\": \"General information\", \"score\": \"3.722\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.049, mean=0.049, max=0.049, sum=0.049 (1)\", \"tab\": \"General information\", \"score\": \"0.049\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1407.178, mean=1407.178, max=1407.178, sum=1407.178 (1)\", \"tab\": \"General information\", \"score\": \"1407.178\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=286.175, mean=286.175, max=286.175, sum=286.175 (1)\", \"tab\": \"General information\", \"score\": \"286.175\"}", - "NaturalQuestions (open-book) - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NaturalQuestions (closed-book) - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=0.667 (1)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "NaturalQuestions (closed-book) - Stereotypes (gender)": "{\"description\": \"min=0.333, mean=0.333, max=0.333, sum=0.333 (1)\", \"tab\": \"Bias\", \"score\": \"0.3333333333333333\"}", - "NaturalQuestions (closed-book) - Representation (race)": "{\"description\": \"min=0.55, mean=0.55, max=0.55, sum=0.55 (1)\", \"tab\": \"Bias\", \"score\": \"0.5497835497835497\"}", - "NaturalQuestions (closed-book) - Representation (gender)": "{\"description\": \"min=0.324, mean=0.324, max=0.324, sum=0.324 (1)\", \"tab\": \"Bias\", \"score\": \"0.32352941176470584\"}", - "NaturalQuestions (open-book) - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=0.667 (1)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "NaturalQuestions (open-book) - Stereotypes (gender)": "{\"description\": \"min=0.5, mean=0.5, max=0.5, sum=0.5 (1)\", \"tab\": \"Bias\", \"score\": \"0.5\"}", - "NaturalQuestions (open-book) - Representation (race)": "{\"description\": \"min=0.521, mean=0.521, max=0.521, sum=0.521 (1)\", \"tab\": \"Bias\", \"score\": \"0.5205992509363295\"}", - "NaturalQuestions (open-book) - Representation (gender)": "{\"description\": \"min=0.458, mean=0.458, max=0.458, sum=0.458 (1)\", \"tab\": \"Bias\", \"score\": \"0.45833333333333326\"}", - "NaturalQuestions (closed-book) - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}", - "NaturalQuestions (open-book) - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "QuAC", - "source_data": { - "dataset_name": "QuAC", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on QuAC", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.392, - "details": { - "description": "min=0.392, mean=0.392, max=0.392, sum=0.392 (1)", - "tab": "Accuracy", - "QuAC - ECE (10-bin)": "{\"description\": \"min=0.109, mean=0.109, max=0.109, sum=0.109 (1)\", \"tab\": \"Calibration\", \"score\": \"0.10940664349880716\"}", - "QuAC - F1 (Robustness)": "{\"description\": \"min=0.25, mean=0.25, max=0.25, sum=0.25 (1)\", \"tab\": \"Robustness\", \"score\": \"0.24986961512093836\"}", - "QuAC - F1 (Fairness)": "{\"description\": \"min=0.304, mean=0.304, max=0.304, sum=0.304 (1)\", \"tab\": \"Fairness\", \"score\": \"0.3036739587215963\"}", - "QuAC - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "QuAC - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "QuAC - # train": "{\"description\": \"min=0.507, mean=0.507, max=0.507, sum=0.507 (1)\", \"tab\": \"General information\", \"score\": \"0.507\"}", - "QuAC - truncated": "{\"description\": \"min=0.06, mean=0.06, max=0.06, sum=0.06 (1)\", \"tab\": \"General information\", \"score\": \"0.06\"}", - "QuAC - # prompt tokens": "{\"description\": \"min=1498.657, mean=1498.657, max=1498.657, sum=1498.657 (1)\", \"tab\": \"General information\", \"score\": \"1498.657\"}", - "QuAC - # output tokens": "{\"description\": \"min=77.25, mean=77.25, max=77.25, sum=77.25 (1)\", \"tab\": \"General information\", \"score\": \"77.25\"}", - "QuAC - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "QuAC - Stereotypes (race)": "{\"description\": \"min=0.633, mean=0.633, max=0.633, sum=0.633 (1)\", \"tab\": \"Bias\", \"score\": \"0.6333333333333334\"}", - "QuAC - Stereotypes (gender)": "{\"description\": \"min=0.416, mean=0.416, max=0.416, sum=0.416 (1)\", \"tab\": \"Bias\", \"score\": \"0.41569852337396196\"}", - "QuAC - Representation (race)": "{\"description\": \"min=0.277, mean=0.277, max=0.277, sum=0.277 (1)\", \"tab\": \"Bias\", \"score\": \"0.27653213751868466\"}", - "QuAC - Representation (gender)": "{\"description\": \"min=0.255, mean=0.255, max=0.255, sum=0.255 (1)\", \"tab\": \"Bias\", \"score\": \"0.2550295857988165\"}", - "QuAC - Toxic fraction": "{\"description\": \"min=0.001, mean=0.001, max=0.001, sum=0.001 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.001\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "HellaSwag", - "source_data": { - "dataset_name": "HellaSwag", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on HellaSwag", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "HellaSwag - ECE (10-bin)": "{\"description\": \"No matching runs\", \"tab\": \"Calibration\", \"score\": \"\"}", - "HellaSwag - EM (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "HellaSwag - EM (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "HellaSwag - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "HellaSwag - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "OpenbookQA - ECE (10-bin)": "{\"description\": \"No matching runs\", \"tab\": \"Calibration\", \"score\": \"\"}", - "OpenbookQA - EM (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "OpenbookQA - EM (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "OpenbookQA - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "OpenbookQA - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "TruthfulQA", - "source_data": { - "dataset_name": "TruthfulQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on TruthfulQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.292, - "details": { - "description": "min=0.292, mean=0.292, max=0.292, sum=0.292 (1)", - "tab": "Accuracy", - "TruthfulQA - ECE (10-bin)": "{\"description\": \"min=0.227, mean=0.227, max=0.227, sum=0.227 (1)\", \"tab\": \"Calibration\", \"score\": \"0.22667464300561196\"}", - "TruthfulQA - EM (Robustness)": "{\"description\": \"min=0.258, mean=0.258, max=0.258, sum=0.258 (1)\", \"tab\": \"Robustness\", \"score\": \"0.25840978593272174\"}", - "TruthfulQA - EM (Fairness)": "{\"description\": \"min=0.235, mean=0.235, max=0.235, sum=0.235 (1)\", \"tab\": \"Fairness\", \"score\": \"0.23547400611620795\"}", - "TruthfulQA - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "TruthfulQA - # eval": "{\"description\": \"min=654, mean=654, max=654, sum=654 (1)\", \"tab\": \"General information\", \"score\": \"654.0\"}", - "TruthfulQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "TruthfulQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "TruthfulQA - # prompt tokens": "{\"description\": \"min=524.602, mean=524.602, max=524.602, sum=524.602 (1)\", \"tab\": \"General information\", \"score\": \"524.6024464831804\"}", - "TruthfulQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "TruthfulQA - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MS MARCO (TREC)", - "source_data": { - "dataset_name": "MS MARCO (TREC)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "NDCG@10 on MS MARCO (TREC)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "MS MARCO (regular) - RR@10 (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "MS MARCO (TREC) - NDCG@10 (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "MS MARCO (regular) - RR@10 (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "MS MARCO (TREC) - NDCG@10 (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "MS MARCO (regular) - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MS MARCO (TREC) - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MS MARCO (regular) - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "MS MARCO (TREC) - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CNN/DailyMail", - "source_data": { - "dataset_name": "CNN/DailyMail", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on CNN/DailyMail", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "CNN/DailyMail - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "CNN/DailyMail - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "CNN/DailyMail - SummaC": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - QAFactEval": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - BERTScore (F1)": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - Coverage": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - Density": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - Compression": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-faithfulness": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-relevance": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-coherence": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "XSUM", - "source_data": { - "dataset_name": "XSUM", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on XSUM", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "XSUM - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "XSUM - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "XSUM - SummaC": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - QAFactEval": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - BERTScore (F1)": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - Coverage": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - Density": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - Compression": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-faithfulness": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-relevance": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-coherence": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "IMDB", - "source_data": { - "dataset_name": "IMDB", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on IMDB", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.916, - "details": { - "description": "min=0.916, mean=0.916, max=0.916, sum=0.916 (1)", - "tab": "Accuracy", - "IMDB - ECE (10-bin)": "{\"description\": \"min=0.348, mean=0.348, max=0.348, sum=0.348 (1)\", \"tab\": \"Calibration\", \"score\": \"0.34781631358579634\"}", - "IMDB - EM (Robustness)": "{\"description\": \"min=0.882, mean=0.882, max=0.882, sum=0.882 (1)\", \"tab\": \"Robustness\", \"score\": \"0.882\"}", - "IMDB - EM (Fairness)": "{\"description\": \"min=0.906, mean=0.906, max=0.906, sum=0.906 (1)\", \"tab\": \"Fairness\", \"score\": \"0.906\"}", - "IMDB - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "IMDB - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "IMDB - # train": "{\"description\": \"min=2.781, mean=2.781, max=2.781, sum=2.781 (1)\", \"tab\": \"General information\", \"score\": \"2.781\"}", - "IMDB - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IMDB - # prompt tokens": "{\"description\": \"min=1751.213, mean=1751.213, max=1751.213, sum=1751.213 (1)\", \"tab\": \"General information\", \"score\": \"1751.213\"}", - "IMDB - # output tokens": "{\"description\": \"min=3.258, mean=3.258, max=3.258, sum=3.258 (1)\", \"tab\": \"General information\", \"score\": \"3.258\"}", - "IMDB - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "IMDB - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CivilComments", - "source_data": { - "dataset_name": "CivilComments", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on CivilComments", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.62, - "details": { - "description": "min=0.154, mean=0.62, max=0.98, sum=11.166 (18)", - "tab": "Accuracy", - "CivilComments - ECE (10-bin)": "{\"description\": \"min=0.13, mean=0.346, max=0.589, sum=6.236 (18)\", \"tab\": \"Calibration\", \"score\": \"0.3464227204141308\"}", - "CivilComments - EM (Robustness)": "{\"description\": \"min=0.125, mean=0.543, max=0.918, sum=9.77 (18)\", \"tab\": \"Robustness\", \"score\": \"0.5427815962078022\"}", - "CivilComments - EM (Fairness)": "{\"description\": \"min=0.116, mean=0.564, max=0.974, sum=10.144 (18)\", \"tab\": \"Fairness\", \"score\": \"0.5635727085389178\"}", - "CivilComments - Denoised inference time (s)": "{\"description\": \"9 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "CivilComments - # eval": "{\"description\": \"min=74, mean=371.556, max=683, sum=6688 (18)\", \"tab\": \"General information\", \"score\": \"371.55555555555554\"}", - "CivilComments - # train": "{\"description\": \"min=5, mean=5, max=5, sum=90 (18)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CivilComments - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (18)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CivilComments - # prompt tokens": "{\"description\": \"min=404.732, mean=855.241, max=1417.567, sum=15394.339 (18)\", \"tab\": \"General information\", \"score\": \"855.2410378605821\"}", - "CivilComments - # output tokens": "{\"description\": \"min=4.854, mean=4.98, max=5, sum=89.64 (18)\", \"tab\": \"General information\", \"score\": \"4.980000522687608\"}", - "CivilComments - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=18 (18)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "CivilComments - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Toxic fraction": "{\"description\": \"9 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "RAFT", - "source_data": { - "dataset_name": "RAFT", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on RAFT", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.693, - "details": { - "description": "min=0.275, mean=0.693, max=0.975, sum=7.625 (11)", - "tab": "Accuracy", - "RAFT - ECE (10-bin)": "{\"description\": \"min=0.126, mean=0.601, max=0.963, sum=6.61 (11)\", \"tab\": \"Calibration\", \"score\": \"0.6009008385490167\"}", - "RAFT - EM (Robustness)": "{\"description\": \"min=0, mean=0.6, max=0.85, sum=6.6 (11)\", \"tab\": \"Robustness\", \"score\": \"0.6000000000000001\"}", - "RAFT - EM (Fairness)": "{\"description\": \"min=0.2, mean=0.643, max=0.975, sum=7.075 (11)\", \"tab\": \"Fairness\", \"score\": \"0.6431818181818182\"}", - "RAFT - Denoised inference time (s)": "{\"description\": \"11 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "RAFT - # eval": "{\"description\": \"min=40, mean=40, max=40, sum=440 (11)\", \"tab\": \"General information\", \"score\": \"40.0\"}", - "RAFT - # train": "{\"description\": \"min=0.45, mean=4.552, max=5, sum=50.075 (11)\", \"tab\": \"General information\", \"score\": \"4.552272727272727\"}", - "RAFT - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (11)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "RAFT - # prompt tokens": "{\"description\": \"min=303.675, mean=954.111, max=1882.1, sum=10495.225 (11)\", \"tab\": \"General information\", \"score\": \"954.1113636363635\"}", - "RAFT - # output tokens": "{\"description\": \"min=5.8, mean=24.4, max=30, sum=268.4 (11)\", \"tab\": \"General information\", \"score\": \"24.4\"}", - "RAFT - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=11 (11)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "RAFT - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Toxic fraction": "{\"description\": \"11 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/locutusque_collectivelm-falcon-3-7b.json b/data/models/locutusque_collectivelm-falcon-3-7b.json deleted file mode 100644 index eb2ff92c08d9d23e15e92c71ee5bc3c15d087078..0000000000000000000000000000000000000000 --- a/data/models/locutusque_collectivelm-falcon-3-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "CollectiveLM-Falcon-3-7B", - "id": "Locutusque/CollectiveLM-Falcon-3-7B", - "developer": "Locutusque", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "7.456" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Locutusque_CollectiveLM-Falcon-3-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3918 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5105 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2183 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3255 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3887 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3599 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/locutusque_hercules-6.0-llama-3.1-8b.json b/data/models/locutusque_hercules-6.0-llama-3.1-8b.json deleted file mode 100644 index 65701bddb0c9fff78f41a27274e00031a4eeea02..0000000000000000000000000000000000000000 --- a/data/models/locutusque_hercules-6.0-llama-3.1-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Hercules-6.0-Llama-3.1-8B", - "id": "Locutusque/Hercules-6.0-Llama-3.1-8B", - "developer": "Locutusque", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Locutusque_Hercules-6.0-Llama-3.1-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.663 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4813 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1669 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2643 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3621 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3615 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/locutusque_hercules-6.1-llama-3.1-8b.json b/data/models/locutusque_hercules-6.1-llama-3.1-8b.json deleted file mode 100644 index aa7e5edf03474f42c299438e26018e7728326b41..0000000000000000000000000000000000000000 --- a/data/models/locutusque_hercules-6.1-llama-3.1-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Hercules-6.1-Llama-3.1-8B", - "id": "Locutusque/Hercules-6.1-Llama-3.1-8B", - "developer": "Locutusque", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Locutusque_Hercules-6.1-Llama-3.1-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6007 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4656 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.176 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2609 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3553 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3669 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/locutusque_llama-3-neuralhercules-5.0-8b.json b/data/models/locutusque_llama-3-neuralhercules-5.0-8b.json deleted file mode 100644 index 18fe902425498d1ac79a987331fe534c1ee8e545..0000000000000000000000000000000000000000 --- a/data/models/locutusque_llama-3-neuralhercules-5.0-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-NeuralHercules-5.0-8B", - "id": "Locutusque/Llama-3-NeuralHercules-5.0-8B", - "developer": "Locutusque", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Locutusque_Llama-3-NeuralHercules-5.0-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4489 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.394 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0431 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2685 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3881 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2933 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/locutusque_llama-3-yggdrasil-2.0-8b.json b/data/models/locutusque_llama-3-yggdrasil-2.0-8b.json deleted file mode 100644 index be7dd228976ac3ea4b83109c33ddd19de12bbeda..0000000000000000000000000000000000000000 --- a/data/models/locutusque_llama-3-yggdrasil-2.0-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-Yggdrasil-2.0-8B", - "id": "Locutusque/Llama-3-Yggdrasil-2.0-8B", - "developer": "Locutusque", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Locutusque_Llama-3-Yggdrasil-2.0-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5371 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4772 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0831 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2626 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3977 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3167 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/locutusque_tinymistral-248m-v2.5.json b/data/models/locutusque_tinymistral-248m-v2.5.json deleted file mode 100644 index 2c93230a2528f53e86687e5d141887fb80ff7a52..0000000000000000000000000000000000000000 --- a/data/models/locutusque_tinymistral-248m-v2.5.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "TinyMistral-248M-v2.5", - "id": "Locutusque/TinyMistral-248M-v2.5", - "developer": "Locutusque", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "0.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Locutusque_TinyMistral-248M-v2.5/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1336 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3039 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0098 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2508 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3782 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1135 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lodrick-the-lafted_llama-3.1-8b-instruct-ortho-v7.json b/data/models/lodrick-the-lafted_llama-3.1-8b-instruct-ortho-v7.json deleted file mode 100644 index 40ffc56cdcf8c4de2b8b5ccb0f1e0525fb281bf6..0000000000000000000000000000000000000000 --- a/data/models/lodrick-the-lafted_llama-3.1-8b-instruct-ortho-v7.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "llama-3.1-8b-instruct-ortho-v7", - "id": "lodrick-the-lafted/llama-3.1-8b-instruct-ortho-v7", - "developer": "lodrick-the-lafted", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/lodrick-the-lafted_llama-3.1-8b-instruct-ortho-v7/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3515 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3907 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0272 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2727 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3616 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1974 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lordjia_llama-3-cantonese-8b-instruct.json b/data/models/lordjia_llama-3-cantonese-8b-instruct.json deleted file mode 100644 index 0d05cd9936a3531438feaef12c94a03974655078..0000000000000000000000000000000000000000 --- a/data/models/lordjia_llama-3-cantonese-8b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-Cantonese-8B-Instruct", - "id": "lordjia/Llama-3-Cantonese-8B-Instruct", - "developer": "lordjia", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/lordjia_Llama-3-Cantonese-8B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6669 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4814 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0891 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2936 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4046 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3515 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lordjia_qwen2-cantonese-7b-instruct.json b/data/models/lordjia_qwen2-cantonese-7b-instruct.json deleted file mode 100644 index dc2c30011e278a1ee524b16c710b5229d028d9a2..0000000000000000000000000000000000000000 --- a/data/models/lordjia_qwen2-cantonese-7b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2-Cantonese-7B-Instruct", - "id": "lordjia/Qwen2-Cantonese-7B-Instruct", - "developer": "lordjia", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/lordjia_Qwen2-Cantonese-7B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5435 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5215 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.256 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2953 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4004 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3843 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lt-asset_nova-1.3b.json b/data/models/lt-asset_nova-1.3b.json deleted file mode 100644 index ae3135a2a454b0956fe0084dd1af67eb8d2035a6..0000000000000000000000000000000000000000 --- a/data/models/lt-asset_nova-1.3b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "nova-1.3b", - "id": "lt-asset/nova-1.3b", - "developer": "lt-asset", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "NovaForCausalLM", - "params_billions": "1.347" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/lt-asset_nova-1.3b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1214 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.317 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0121 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2492 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3698 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1142 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lunahr_thea-3b-50r-u1.json b/data/models/lunahr_thea-3b-50r-u1.json deleted file mode 100644 index c9d15b741e0ec882db6c9a9e419d7e3c8900df33..0000000000000000000000000000000000000000 --- a/data/models/lunahr_thea-3b-50r-u1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "thea-3b-50r-u1", - "id": "lunahr/thea-3b-50r-u1", - "developer": "lunahr", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/lunahr_thea-3b-50r-u1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.603 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4105 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1042 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2836 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3182 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2808 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lunahr_thea-v2-3b-50r.json b/data/models/lunahr_thea-v2-3b-50r.json deleted file mode 100644 index 6bf34accf4fc146547291eccdfc359e717d3a7e0..0000000000000000000000000000000000000000 --- a/data/models/lunahr_thea-v2-3b-50r.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "thea-v2-3b-50r", - "id": "lunahr/thea-v2-3b-50r", - "developer": "lunahr", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/lunahr_thea-v2-3b-50r/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3704 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4194 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0242 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2609 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3222 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2409 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/luni_stardust-12b-v1.json b/data/models/luni_stardust-12b-v1.json deleted file mode 100644 index fdb99743b214f067e5e643f05e86022898c94573..0000000000000000000000000000000000000000 --- a/data/models/luni_stardust-12b-v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "StarDust-12b-v1", - "id": "Luni/StarDust-12b-v1", - "developer": "Luni", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Luni_StarDust-12b-v1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5459 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5366 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0763 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.276 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4324 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3412 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/luni_stardust-12b-v2.json b/data/models/luni_stardust-12b-v2.json deleted file mode 100644 index 4c6785b92855519baa1cfe1319b1ddce7cabfc02..0000000000000000000000000000000000000000 --- a/data/models/luni_stardust-12b-v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "StarDust-12b-v2", - "id": "Luni/StarDust-12b-v2", - "developer": "Luni", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Luni_StarDust-12b-v2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5629 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5419 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0687 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2936 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4338 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3439 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lunzima_nqlsg-qwen2.5-14b-megafusion-v3.json b/data/models/lunzima_nqlsg-qwen2.5-14b-megafusion-v3.json deleted file mode 100644 index c12ebaf71d6491dc94fa22a58fbf916f5d43e62b..0000000000000000000000000000000000000000 --- a/data/models/lunzima_nqlsg-qwen2.5-14b-megafusion-v3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "NQLSG-Qwen2.5-14B-MegaFusion-v3", - "id": "Lunzima/NQLSG-Qwen2.5-14B-MegaFusion-v3", - "developer": "Lunzima", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Lunzima_NQLSG-Qwen2.5-14B-MegaFusion-v3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7049 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6478 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4162 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3817 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4808 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5394 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lunzima_nqlsg-qwen2.5-14b-megafusion-v4.json b/data/models/lunzima_nqlsg-qwen2.5-14b-megafusion-v4.json deleted file mode 100644 index 40d47620f3625c9834d7908970354d051d0b21e7..0000000000000000000000000000000000000000 --- a/data/models/lunzima_nqlsg-qwen2.5-14b-megafusion-v4.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "NQLSG-Qwen2.5-14B-MegaFusion-v4", - "id": "Lunzima/NQLSG-Qwen2.5-14B-MegaFusion-v4", - "developer": "Lunzima", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Lunzima_NQLSG-Qwen2.5-14B-MegaFusion-v4/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6943 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.642 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3467 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3716 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4769 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5252 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lunzima_nqlsg-qwen2.5-14b-megafusion-v5.json b/data/models/lunzima_nqlsg-qwen2.5-14b-megafusion-v5.json deleted file mode 100644 index 4198829343710cb1f23071e74729c29f808922ca..0000000000000000000000000000000000000000 --- a/data/models/lunzima_nqlsg-qwen2.5-14b-megafusion-v5.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "NQLSG-Qwen2.5-14B-MegaFusion-v5", - "id": "Lunzima/NQLSG-Qwen2.5-14B-MegaFusion-v5", - "developer": "Lunzima", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Lunzima_NQLSG-Qwen2.5-14B-MegaFusion-v5/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7485 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6467 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4358 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3624 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4473 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.514 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lunzima_nqlsg-qwen2.5-14b-megafusion-v6-cpt.json b/data/models/lunzima_nqlsg-qwen2.5-14b-megafusion-v6-cpt.json deleted file mode 100644 index 551b9812655f61bd027b461327a328e60aaa4157..0000000000000000000000000000000000000000 --- a/data/models/lunzima_nqlsg-qwen2.5-14b-megafusion-v6-cpt.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "NQLSG-Qwen2.5-14B-MegaFusion-v6-cpt", - "id": "Lunzima/NQLSG-Qwen2.5-14B-MegaFusion-v6-cpt", - "developer": "Lunzima", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Lunzima_NQLSG-Qwen2.5-14B-MegaFusion-v6-cpt/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4663 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6215 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3316 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3758 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4937 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5204 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lunzima_nqlsg-qwen2.5-14b-megafusion-v6.json b/data/models/lunzima_nqlsg-qwen2.5-14b-megafusion-v6.json deleted file mode 100644 index c12ec7826441a368c499e0b35d43bb647bf843cd..0000000000000000000000000000000000000000 --- a/data/models/lunzima_nqlsg-qwen2.5-14b-megafusion-v6.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "NQLSG-Qwen2.5-14B-MegaFusion-v6", - "id": "Lunzima/NQLSG-Qwen2.5-14B-MegaFusion-v6", - "developer": "Lunzima", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Lunzima_NQLSG-Qwen2.5-14B-MegaFusion-v6/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7043 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6458 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3958 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3775 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4768 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5392 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lunzima_nqlsg-qwen2.5-14b-megafusion-v7-rebase.json b/data/models/lunzima_nqlsg-qwen2.5-14b-megafusion-v7-rebase.json deleted file mode 100644 index d93479dc5b875ee2e16a656f643eb7dbf4905cfa..0000000000000000000000000000000000000000 --- a/data/models/lunzima_nqlsg-qwen2.5-14b-megafusion-v7-rebase.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "NQLSG-Qwen2.5-14B-MegaFusion-v7-rebase", - "id": "Lunzima/NQLSG-Qwen2.5-14B-MegaFusion-v7-rebase", - "developer": "Lunzima", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Lunzima_NQLSG-Qwen2.5-14B-MegaFusion-v7-rebase/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6931 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6423 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3406 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.375 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4888 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5277 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lunzima_nqlsg-qwen2.5-14b-megafusion-v7.json b/data/models/lunzima_nqlsg-qwen2.5-14b-megafusion-v7.json deleted file mode 100644 index c55c9d815ac26bd55c81145a17fc1b62dfb7e71e..0000000000000000000000000000000000000000 --- a/data/models/lunzima_nqlsg-qwen2.5-14b-megafusion-v7.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "NQLSG-Qwen2.5-14B-MegaFusion-v7", - "id": "Lunzima/NQLSG-Qwen2.5-14B-MegaFusion-v7", - "developer": "Lunzima", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Lunzima_NQLSG-Qwen2.5-14B-MegaFusion-v7/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6794 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6531 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4101 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3792 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4834 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5376 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lunzima_nqlsg-qwen2.5-14b-megafusion-v8.5.json b/data/models/lunzima_nqlsg-qwen2.5-14b-megafusion-v8.5.json deleted file mode 100644 index afcabeb33057369948646e5057244758a49569c9..0000000000000000000000000000000000000000 --- a/data/models/lunzima_nqlsg-qwen2.5-14b-megafusion-v8.5.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "NQLSG-Qwen2.5-14B-MegaFusion-v8.5", - "id": "Lunzima/NQLSG-Qwen2.5-14B-MegaFusion-v8.5", - "developer": "Lunzima", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Lunzima_NQLSG-Qwen2.5-14B-MegaFusion-v8.5/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5929 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6451 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3656 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.38 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.477 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.529 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lunzima_nqlsg-qwen2.5-14b-megafusion-v8.6.json b/data/models/lunzima_nqlsg-qwen2.5-14b-megafusion-v8.6.json deleted file mode 100644 index 62926931a0500c03962ac6c55af0feb8cbf335d1..0000000000000000000000000000000000000000 --- a/data/models/lunzima_nqlsg-qwen2.5-14b-megafusion-v8.6.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "NQLSG-Qwen2.5-14B-MegaFusion-v8.6", - "id": "Lunzima/NQLSG-Qwen2.5-14B-MegaFusion-v8.6", - "developer": "Lunzima", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Lunzima_NQLSG-Qwen2.5-14B-MegaFusion-v8.6/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5919 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6457 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4071 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3842 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4953 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.54 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lunzima_nqlsg-qwen2.5-14b-megafusion-v8.7.json b/data/models/lunzima_nqlsg-qwen2.5-14b-megafusion-v8.7.json deleted file mode 100644 index baff70add80359deaf559f780335d424416a24e2..0000000000000000000000000000000000000000 --- a/data/models/lunzima_nqlsg-qwen2.5-14b-megafusion-v8.7.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "NQLSG-Qwen2.5-14B-MegaFusion-v8.7", - "id": "Lunzima/NQLSG-Qwen2.5-14B-MegaFusion-v8.7", - "developer": "Lunzima", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Lunzima_NQLSG-Qwen2.5-14B-MegaFusion-v8.7/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7875 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6483 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5408 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3515 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4381 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5242 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lunzima_nqlsg-qwen2.5-14b-megafusion-v8.8.json b/data/models/lunzima_nqlsg-qwen2.5-14b-megafusion-v8.8.json deleted file mode 100644 index ed4177584d341ddb7acfa75eae55ba5dee895b02..0000000000000000000000000000000000000000 --- a/data/models/lunzima_nqlsg-qwen2.5-14b-megafusion-v8.8.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "NQLSG-Qwen2.5-14B-MegaFusion-v8.8", - "id": "Lunzima/NQLSG-Qwen2.5-14B-MegaFusion-v8.8", - "developer": "Lunzima", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Lunzima_NQLSG-Qwen2.5-14B-MegaFusion-v8.8/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7028 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6566 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4237 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3758 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4912 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5323 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lunzima_nqlsg-qwen2.5-14b-megafusion-v8.9.json b/data/models/lunzima_nqlsg-qwen2.5-14b-megafusion-v8.9.json deleted file mode 100644 index e17231d0f8c49d602a197b9d2cd73620fdba31a3..0000000000000000000000000000000000000000 --- a/data/models/lunzima_nqlsg-qwen2.5-14b-megafusion-v8.9.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "NQLSG-Qwen2.5-14B-MegaFusion-v8.9", - "id": "Lunzima/NQLSG-Qwen2.5-14B-MegaFusion-v8.9", - "developer": "Lunzima", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Lunzima_NQLSG-Qwen2.5-14B-MegaFusion-v8.9/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7993 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6483 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.537 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3297 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4328 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5199 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lunzima_nqlsg-qwen2.5-14b-megafusion-v8.json b/data/models/lunzima_nqlsg-qwen2.5-14b-megafusion-v8.json deleted file mode 100644 index de0c08f4dee59cefbbea7cc5816e273ed8dca930..0000000000000000000000000000000000000000 --- a/data/models/lunzima_nqlsg-qwen2.5-14b-megafusion-v8.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "NQLSG-Qwen2.5-14B-MegaFusion-v8", - "id": "Lunzima/NQLSG-Qwen2.5-14B-MegaFusion-v8", - "developer": "Lunzima", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Lunzima_NQLSG-Qwen2.5-14B-MegaFusion-v8/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7875 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6419 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5559 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3356 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4394 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5206 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lunzima_nqlsg-qwen2.5-14b-megafusion-v9-stock.json b/data/models/lunzima_nqlsg-qwen2.5-14b-megafusion-v9-stock.json deleted file mode 100644 index cc7c0468b184d04ccf156b4df9991522949b97b9..0000000000000000000000000000000000000000 --- a/data/models/lunzima_nqlsg-qwen2.5-14b-megafusion-v9-stock.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "NQLSG-Qwen2.5-14B-MegaFusion-v9-stock", - "id": "Lunzima/NQLSG-Qwen2.5-14B-MegaFusion-v9-stock", - "developer": "Lunzima", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Lunzima_NQLSG-Qwen2.5-14B-MegaFusion-v9-stock/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6514 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6571 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4184 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3842 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.482 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5412 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lunzima_nqlsg-qwen2.5-14b-megafusion-v9.1.json b/data/models/lunzima_nqlsg-qwen2.5-14b-megafusion-v9.1.json deleted file mode 100644 index f8d3f1890d9702a76dd1df1cb235443ed7b14b04..0000000000000000000000000000000000000000 --- a/data/models/lunzima_nqlsg-qwen2.5-14b-megafusion-v9.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "NQLSG-Qwen2.5-14B-MegaFusion-v9.1", - "id": "Lunzima/NQLSG-Qwen2.5-14B-MegaFusion-v9.1", - "developer": "Lunzima", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Lunzima_NQLSG-Qwen2.5-14B-MegaFusion-v9.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8003 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6555 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5468 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3431 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4354 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5251 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lunzima_nqlsg-qwen2.5-14b-megafusion-v9.2.json b/data/models/lunzima_nqlsg-qwen2.5-14b-megafusion-v9.2.json deleted file mode 100644 index 350f7ccfe2052dabe694941b86ff0a347a493ece..0000000000000000000000000000000000000000 --- a/data/models/lunzima_nqlsg-qwen2.5-14b-megafusion-v9.2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "NQLSG-Qwen2.5-14B-MegaFusion-v9.2", - "id": "Lunzima/NQLSG-Qwen2.5-14B-MegaFusion-v9.2", - "developer": "Lunzima", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Lunzima_NQLSG-Qwen2.5-14B-MegaFusion-v9.2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7862 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6538 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5332 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3557 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4381 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5283 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lunzima_nqlsg-qwen2.5-14b-megafusion-v9.json b/data/models/lunzima_nqlsg-qwen2.5-14b-megafusion-v9.json deleted file mode 100644 index 7107a44e8738de09c5ac168bba8771a051837b4c..0000000000000000000000000000000000000000 --- a/data/models/lunzima_nqlsg-qwen2.5-14b-megafusion-v9.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "NQLSG-Qwen2.5-14B-MegaFusion-v9", - "id": "Lunzima/NQLSG-Qwen2.5-14B-MegaFusion-v9", - "developer": "Lunzima", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Lunzima_NQLSG-Qwen2.5-14B-MegaFusion-v9/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5235 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6546 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4366 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3884 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4806 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5422 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lunzima_nqlsg-qwen2.5-14b-originalfusion.json b/data/models/lunzima_nqlsg-qwen2.5-14b-originalfusion.json deleted file mode 100644 index 1c2f2b28187ea2e8910a67e885a22f012356eed6..0000000000000000000000000000000000000000 --- a/data/models/lunzima_nqlsg-qwen2.5-14b-originalfusion.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "NQLSG-Qwen2.5-14B-OriginalFusion", - "id": "Lunzima/NQLSG-Qwen2.5-14B-OriginalFusion", - "developer": "Lunzima", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Lunzima_NQLSG-Qwen2.5-14B-OriginalFusion/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6142 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6592 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4275 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3809 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5122 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5239 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lxzgordon_urm-llama-3-8b.json b/data/models/lxzgordon_urm-llama-3-8b.json deleted file mode 100644 index af037140ff29469d707f853ccaf6e32e0597dbff..0000000000000000000000000000000000000000 --- a/data/models/lxzgordon_urm-llama-3-8b.json +++ /dev/null @@ -1,126 +0,0 @@ -{ - "model_info": { - "name": "LxzGordon/URM-LLaMa-3-8B", - "id": "LxzGordon/URM-LLaMa-3-8B", - "developer": "LxzGordon", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/LxzGordon_URM-LLaMa-3-8B/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8991 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9693 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7873 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8824 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9574 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lxzgordon_urm-llama-3.1-8b.json b/data/models/lxzgordon_urm-llama-3.1-8b.json deleted file mode 100644 index 2ce56c90ce0d5fb5788660f5b3f1f1e179701786..0000000000000000000000000000000000000000 --- a/data/models/lxzgordon_urm-llama-3.1-8b.json +++ /dev/null @@ -1,276 +0,0 @@ -{ - "model_info": { - "name": "LxzGordon/URM-LLaMa-3.1-8B", - "id": "LxzGordon/URM-LLaMa-3.1-8B", - "developer": "LxzGordon", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/LxzGordon_URM-LLaMa-3.1-8B/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9294 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9553 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8816 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9108 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9698 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "reward-bench-2/LxzGordon_URM-LLaMa-3.1-8B/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7394 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6884 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.45 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6393 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9178 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9758 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7653 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lyte_llama-3.1-8b-instruct-reasoner-1o1_v0.3.json b/data/models/lyte_llama-3.1-8b-instruct-reasoner-1o1_v0.3.json deleted file mode 100644 index f6a60b3126f5601e685363e713e9015833e6f83e..0000000000000000000000000000000000000000 --- a/data/models/lyte_llama-3.1-8b-instruct-reasoner-1o1_v0.3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.1-8B-Instruct-Reasoner-1o1_v0.3", - "id": "Lyte/Llama-3.1-8B-Instruct-Reasoner-1o1_v0.3", - "developer": "Lyte", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Lyte_Llama-3.1-8B-Instruct-Reasoner-1o1_v0.3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7098 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.495 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1903 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2701 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3461 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3618 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lyte_llama-3.2-1b-instruct-cot-rl-expriement1-ep04.json b/data/models/lyte_llama-3.2-1b-instruct-cot-rl-expriement1-ep04.json deleted file mode 100644 index 37848da5e61c8c0a721ec4843c269fad36636e72..0000000000000000000000000000000000000000 --- a/data/models/lyte_llama-3.2-1b-instruct-cot-rl-expriement1-ep04.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.2-1B-Instruct-COT-RL-Expriement1-EP04", - "id": "Lyte/Llama-3.2-1B-Instruct-COT-RL-Expriement1-EP04", - "developer": "Lyte", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.236" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Lyte_Llama-3.2-1B-Instruct-COT-RL-Expriement1-EP04/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5774 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3515 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0801 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2601 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3236 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1843 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/lyte_llama-3.2-3b-overthinker.json b/data/models/lyte_llama-3.2-3b-overthinker.json deleted file mode 100644 index 50cdac477184f34a1990443fe2afff24856bc202..0000000000000000000000000000000000000000 --- a/data/models/lyte_llama-3.2-3b-overthinker.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.2-3B-Overthinker", - "id": "Lyte/Llama-3.2-3B-Overthinker", - "developer": "Lyte", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Lyte_Llama-3.2-3B-Overthinker/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6408 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.432 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1563 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2592 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3419 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2985 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/m4-ai_tinymistral-248m-v3.json b/data/models/m4-ai_tinymistral-248m-v3.json deleted file mode 100644 index 8fceb6ee52aa3a95defda412edf41e26425c58ea..0000000000000000000000000000000000000000 --- a/data/models/m4-ai_tinymistral-248m-v3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "TinyMistral-248M-v3", - "id": "M4-ai/TinyMistral-248M-v3", - "developer": "M4-ai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "0.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/M4-ai_TinyMistral-248M-v3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1639 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2885 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0045 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2408 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3793 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1132 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/m42-health_llama3-med42-70b.json b/data/models/m42-health_llama3-med42-70b.json deleted file mode 100644 index dea2a9ae34765d1bbee920d8cdf72a2cadb88396..0000000000000000000000000000000000000000 --- a/data/models/m42-health_llama3-med42-70b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama3-Med42-70B", - "id": "m42-health/Llama3-Med42-70B", - "developer": "m42-health", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "70.554" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/m42-health_Llama3-Med42-70B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6291 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6688 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2258 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3473 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4629 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4963 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/macadeliccc_magistrate-3.2-3b-base.json b/data/models/macadeliccc_magistrate-3.2-3b-base.json deleted file mode 100644 index b408727316ff8b1fb34a4b5c495a92f44d5001a8..0000000000000000000000000000000000000000 --- a/data/models/macadeliccc_magistrate-3.2-3b-base.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "magistrate-3.2-3b-base", - "id": "macadeliccc/magistrate-3.2-3b-base", - "developer": "macadeliccc", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/macadeliccc_magistrate-3.2-3b-base/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1159 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3343 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0113 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2609 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3976 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1689 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/macadeliccc_magistrate-3.2-3b-it.json b/data/models/macadeliccc_magistrate-3.2-3b-it.json deleted file mode 100644 index 781fdc2ba8c4da5d10e5a99e4d81e8f790b72662..0000000000000000000000000000000000000000 --- a/data/models/macadeliccc_magistrate-3.2-3b-it.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "magistrate-3.2-3b-it", - "id": "macadeliccc/magistrate-3.2-3b-it", - "developer": "macadeliccc", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/macadeliccc_magistrate-3.2-3b-it/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2292 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3257 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0196 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2475 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3763 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1592 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/macadeliccc_samantha-qwen-2-7b.json b/data/models/macadeliccc_samantha-qwen-2-7b.json deleted file mode 100644 index ad52156bee9e510299305493016e47534c824601..0000000000000000000000000000000000000000 --- a/data/models/macadeliccc_samantha-qwen-2-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Samantha-Qwen-2-7B", - "id": "macadeliccc/Samantha-Qwen-2-7B", - "developer": "macadeliccc", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/macadeliccc_Samantha-Qwen-2-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4377 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5082 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2115 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2727 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4799 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3779 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/madeagents_hammer2-1-0-5b-fc.json b/data/models/madeagents_hammer2-1-0-5b-fc.json deleted file mode 100644 index c6464b15c4ad36a5ca6825a749a998d1a4ea9024..0000000000000000000000000000000000000000 --- a/data/models/madeagents_hammer2-1-0-5b-fc.json +++ /dev/null @@ -1,851 +0,0 @@ -{ - "model_info": { - "name": "Hammer2.1-0.5b (FC)", - "id": "madeagents/hammer2-1-0-5b-fc", - "developer": "madeagents", - "additional_details": { - "raw_model_name": "Hammer2.1-0.5b (FC)", - "organization": "MadeAgents", - "license": "cc-by-nc-4.0", - "mode": "FC", - "model_link": "https://huggingface.co/MadeAgents/Hammer2.1-0.5b" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/madeagents/hammer2-1-0-5b-fc/1775236112.4186308", - "retrieved_timestamp": "1775236112.4186308", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 100.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 21.22 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 2.82 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 2.79 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 3.17 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 9.86 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 65.98 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 62.42 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 81.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 69.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 51.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 54.63 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 56.59 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 54.42 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 62.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 37.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 2.88 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 4.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 0.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 4.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 2.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 1.08 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 0.65 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 1.94 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 0.65 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 68.75 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 80.79 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/madeagents_hammer2-1-1-5b-fc.json b/data/models/madeagents_hammer2-1-1-5b-fc.json deleted file mode 100644 index 98e439db528f62cd67ec5a2acc58e8fa65045913..0000000000000000000000000000000000000000 --- a/data/models/madeagents_hammer2-1-1-5b-fc.json +++ /dev/null @@ -1,851 +0,0 @@ -{ - "model_info": { - "name": "Hammer2.1-1.5b (FC)", - "id": "madeagents/hammer2-1-1-5b-fc", - "developer": "madeagents", - "additional_details": { - "raw_model_name": "Hammer2.1-1.5b (FC)", - "organization": "MadeAgents", - "license": "cc-by-nc-4.0", - "mode": "FC", - "model_link": "https://huggingface.co/MadeAgents/Hammer2.1-1.5b" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/madeagents/hammer2-1-1-5b-fc/1775236112.405115", - "retrieved_timestamp": "1775236112.405115", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 75.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 27.88 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 6.83 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 6.28 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 8.79 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 30.72 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 82.98 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 73.92 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 92.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 85.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 80.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 69.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 72.09 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 69.33 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 50.0 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 62.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 15.62 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 20.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 16.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 9.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 16.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 75.0 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 79.4 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/madeagents_hammer2-1-3b-fc.json b/data/models/madeagents_hammer2-1-3b-fc.json deleted file mode 100644 index 4886c2e2301abbf5f8025893179e8bef067f0bdf..0000000000000000000000000000000000000000 --- a/data/models/madeagents_hammer2-1-3b-fc.json +++ /dev/null @@ -1,851 +0,0 @@ -{ - "model_info": { - "name": "Hammer2.1-3b (FC)", - "id": "madeagents/hammer2-1-3b-fc", - "developer": "madeagents", - "additional_details": { - "raw_model_name": "Hammer2.1-3b (FC)", - "organization": "MadeAgents", - "license": "qwen-research", - "mode": "FC", - "model_link": "https://huggingface.co/MadeAgents/Hammer2.1-3b" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/madeagents/hammer2-1-3b-fc/1775236112.401521", - "retrieved_timestamp": "1775236112.401521", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 68.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 29.71 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 10.89 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 11.24 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 15.81 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 47.44 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 84.96 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 79.33 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 93.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 86.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 80.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 70.54 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 68.22 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 71.32 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 62.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 66.67 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 16.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 22.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 12.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 16.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 15.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 3.01 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 2.58 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 3.87 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 2.58 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 56.25 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 86.12 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/madeagents_hammer2-1-7b-fc.json b/data/models/madeagents_hammer2-1-7b-fc.json deleted file mode 100644 index 7de8fd45a1c37cbc8847cebc4c1184795953e57c..0000000000000000000000000000000000000000 --- a/data/models/madeagents_hammer2-1-7b-fc.json +++ /dev/null @@ -1,851 +0,0 @@ -{ - "model_info": { - "name": "Hammer2.1-7b (FC)", - "id": "madeagents/hammer2-1-7b-fc", - "developer": "madeagents", - "additional_details": { - "raw_model_name": "Hammer2.1-7b (FC)", - "organization": "MadeAgents", - "license": "cc-by-nc-4.0", - "mode": "FC", - "model_link": "https://huggingface.co/MadeAgents/Hammer2.1-7b" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/madeagents/hammer2-1-7b-fc/1775236112.399455", - "retrieved_timestamp": "1775236112.399455", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 64.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 31.67 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 4.99 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 5.77 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 10.29 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 31.26 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 85.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 72.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 92.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 91.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 86.0 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 69.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 66.67 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 69.99 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 75.0 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 75.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 23.87 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 24.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 28.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 21.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 21.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 50.0 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 90.12 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/magnifi_phi3_intent_v56_3_w_unknown_5_lr_0.002.json b/data/models/magnifi_phi3_intent_v56_3_w_unknown_5_lr_0.002.json deleted file mode 100644 index 379981aaa8007dbbd6bd4a4b5a4aef66eedede40..0000000000000000000000000000000000000000 --- a/data/models/magnifi_phi3_intent_v56_3_w_unknown_5_lr_0.002.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Phi3_intent_v56_3_w_unknown_5_lr_0.002", - "id": "magnifi/Phi3_intent_v56_3_w_unknown_5_lr_0.002", - "developer": "magnifi", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "3.821" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/magnifi_Phi3_intent_v56_3_w_unknown_5_lr_0.002/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2018 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3282 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2643 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4123 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1472 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/magpie-align_llama-3-8b-magpie-align-sft-v0.1.json b/data/models/magpie-align_llama-3-8b-magpie-align-sft-v0.1.json deleted file mode 100644 index 766f0ba186fc1f272d3abfe6300f442850c02c85..0000000000000000000000000000000000000000 --- a/data/models/magpie-align_llama-3-8b-magpie-align-sft-v0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-8B-Magpie-Align-SFT-v0.1", - "id": "Magpie-Align/Llama-3-8B-Magpie-Align-SFT-v0.1", - "developer": "Magpie-Align", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Magpie-Align_Llama-3-8B-Magpie-Align-SFT-v0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4361 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4615 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0574 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2626 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3277 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2863 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/magpie-align_llama-3-8b-magpie-align-sft-v0.3.json b/data/models/magpie-align_llama-3-8b-magpie-align-sft-v0.3.json deleted file mode 100644 index ef7073372e17f4660424104496f03beedcdb0caa..0000000000000000000000000000000000000000 --- a/data/models/magpie-align_llama-3-8b-magpie-align-sft-v0.3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-8B-Magpie-Align-SFT-v0.3", - "id": "Magpie-Align/Llama-3-8B-Magpie-Align-SFT-v0.3", - "developer": "Magpie-Align", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Magpie-Align_Llama-3-8B-Magpie-Align-SFT-v0.3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5064 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4572 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0733 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2659 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3424 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2902 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/magpie-align_llama-3-8b-magpie-align-v0.1.json b/data/models/magpie-align_llama-3-8b-magpie-align-v0.1.json deleted file mode 100644 index 018a3210e6da83ca7e6d3501fd6a7fefdb10449c..0000000000000000000000000000000000000000 --- a/data/models/magpie-align_llama-3-8b-magpie-align-v0.1.json +++ /dev/null @@ -1,275 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-8B-Magpie-Align-v0.1", - "id": "Magpie-Align/Llama-3-8B-Magpie-Align-v0.1", - "developer": "Magpie-Align", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Magpie-Align_Llama-3-8B-Magpie-Align-v0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4118 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4811 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.034 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2752 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3047 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3006 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "hfopenllm_v2/Magpie-Align_Llama-3-8B-Magpie-Align-v0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4027 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4789 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0461 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2768 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3087 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3001 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/magpie-align_llama-3-8b-magpie-align-v0.3.json b/data/models/magpie-align_llama-3-8b-magpie-align-v0.3.json deleted file mode 100644 index f88f34ba4419874bd15fa86602167a498cd485f1..0000000000000000000000000000000000000000 --- a/data/models/magpie-align_llama-3-8b-magpie-align-v0.3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-8B-Magpie-Align-v0.3", - "id": "Magpie-Align/Llama-3-8B-Magpie-Align-v0.3", - "developer": "Magpie-Align", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Magpie-Align_Llama-3-8B-Magpie-Align-v0.3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4497 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.457 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0566 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2651 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3406 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3134 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/magpie-align_llama-3.1-8b-magpie-align-sft-v0.1.json b/data/models/magpie-align_llama-3.1-8b-magpie-align-sft-v0.1.json deleted file mode 100644 index b9169f0161af4cf2eea2dbd8dd37a3fb6ef86464..0000000000000000000000000000000000000000 --- a/data/models/magpie-align_llama-3.1-8b-magpie-align-sft-v0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.1-8B-Magpie-Align-SFT-v0.1", - "id": "Magpie-Align/Llama-3.1-8B-Magpie-Align-SFT-v0.1", - "developer": "Magpie-Align", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Magpie-Align_Llama-3.1-8B-Magpie-Align-SFT-v0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4782 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4764 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0899 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2609 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3397 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2943 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/magpie-align_llama-3.1-8b-magpie-align-v0.1.json b/data/models/magpie-align_llama-3.1-8b-magpie-align-v0.1.json deleted file mode 100644 index 4ce55288b8dbacddd8a0e161cfcb38e813b8630f..0000000000000000000000000000000000000000 --- a/data/models/magpie-align_llama-3.1-8b-magpie-align-v0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.1-8B-Magpie-Align-v0.1", - "id": "Magpie-Align/Llama-3.1-8B-Magpie-Align-v0.1", - "developer": "Magpie-Align", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Magpie-Align_Llama-3.1-8B-Magpie-Align-v0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4458 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4622 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0665 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2634 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3141 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3262 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/magpie-align_magpielm-8b-chat-v0.1.json b/data/models/magpie-align_magpielm-8b-chat-v0.1.json deleted file mode 100644 index 933f39d71df02c03637aa58b8c0421cb7130a488..0000000000000000000000000000000000000000 --- a/data/models/magpie-align_magpielm-8b-chat-v0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MagpieLM-8B-Chat-v0.1", - "id": "Magpie-Align/MagpieLM-8B-Chat-v0.1", - "developer": "Magpie-Align", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Magpie-Align_MagpieLM-8B-Chat-v0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3701 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4172 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0612 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2617 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3501 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3195 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/magpie-align_magpielm-8b-sft-v0.1.json b/data/models/magpie-align_magpielm-8b-sft-v0.1.json deleted file mode 100644 index 3adbfc18694d1c592168d89c1027e7062fc6ed69..0000000000000000000000000000000000000000 --- a/data/models/magpie-align_magpielm-8b-sft-v0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MagpieLM-8B-SFT-v0.1", - "id": "Magpie-Align/MagpieLM-8B-SFT-v0.1", - "developer": "Magpie-Align", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Magpie-Align_MagpieLM-8B-SFT-v0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4721 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4553 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0755 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2676 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3649 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.299 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/maguscorp_grpo_lora_enem_llama3_7b.json b/data/models/maguscorp_grpo_lora_enem_llama3_7b.json deleted file mode 100644 index 04f3def4e58bc49c0f1e9ad49c5ddf9ee2f3ce96..0000000000000000000000000000000000000000 --- a/data/models/maguscorp_grpo_lora_enem_llama3_7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "grpo_lora_enem_llama3_7b", - "id": "MagusCorp/grpo_lora_enem_llama3_7b", - "developer": "MagusCorp", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/MagusCorp_grpo_lora_enem_llama3_7b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4724 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4801 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1216 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3096 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3971 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3574 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/maldv_awqward2.5-32b-instruct.json b/data/models/maldv_awqward2.5-32b-instruct.json deleted file mode 100644 index 1d56b141e7971952589fceeeabbf1a3185aa8ba3..0000000000000000000000000000000000000000 --- a/data/models/maldv_awqward2.5-32b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Awqward2.5-32B-Instruct", - "id": "maldv/Awqward2.5-32B-Instruct", - "developer": "maldv", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "32.764" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/maldv_Awqward2.5-32B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8255 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6974 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6231 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3406 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4275 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5723 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/maldv_badger-kappa-llama-3-8b.json b/data/models/maldv_badger-kappa-llama-3-8b.json deleted file mode 100644 index ace888e80e0488c3c36e6dd6ca9e0f160665b6b1..0000000000000000000000000000000000000000 --- a/data/models/maldv_badger-kappa-llama-3-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "badger-kappa-llama-3-8b", - "id": "maldv/badger-kappa-llama-3-8b", - "developer": "maldv", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/maldv_badger-kappa-llama-3-8b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4695 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5085 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0861 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3029 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3765 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3695 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/maldv_badger-lambda-llama-3-8b.json b/data/models/maldv_badger-lambda-llama-3-8b.json deleted file mode 100644 index 67f225722face79484e03f9f2c4d98bf1b3e37ee..0000000000000000000000000000000000000000 --- a/data/models/maldv_badger-lambda-llama-3-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "badger-lambda-llama-3-8b", - "id": "maldv/badger-lambda-llama-3-8b", - "developer": "maldv", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/maldv_badger-lambda-llama-3-8b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4861 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4963 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0944 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2819 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3754 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3767 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/maldv_badger-mu-llama-3-8b.json b/data/models/maldv_badger-mu-llama-3-8b.json deleted file mode 100644 index 98fee88a28e37932b7a3607a11dbd27fc25188b7..0000000000000000000000000000000000000000 --- a/data/models/maldv_badger-mu-llama-3-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "badger-mu-llama-3-8b", - "id": "maldv/badger-mu-llama-3-8b", - "developer": "maldv", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/maldv_badger-mu-llama-3-8b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4919 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5143 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0559 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2592 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3555 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3674 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/maldv_badger-writer-llama-3-8b.json b/data/models/maldv_badger-writer-llama-3-8b.json deleted file mode 100644 index 69e35dd9af5b65ef108327b0b41f81e6378d367c..0000000000000000000000000000000000000000 --- a/data/models/maldv_badger-writer-llama-3-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "badger-writer-llama-3-8b", - "id": "maldv/badger-writer-llama-3-8b", - "developer": "maldv", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/maldv_badger-writer-llama-3-8b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5303 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4864 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0755 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2894 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3581 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.376 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/maldv_lytta2.5-32b-instruct.json b/data/models/maldv_lytta2.5-32b-instruct.json deleted file mode 100644 index f11dd5da166505d48dcae97996c25a269e7ca72b..0000000000000000000000000000000000000000 --- a/data/models/maldv_lytta2.5-32b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Lytta2.5-32B-Instruct", - "id": "maldv/Lytta2.5-32B-Instruct", - "developer": "maldv", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "32.764" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/maldv_Lytta2.5-32B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2508 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.56 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3444 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2668 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3769 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5048 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/maldv_qwentile2.5-32b-instruct.json b/data/models/maldv_qwentile2.5-32b-instruct.json deleted file mode 100644 index a05fff692655bd65d85e02008ba38e2f29994b9e..0000000000000000000000000000000000000000 --- a/data/models/maldv_qwentile2.5-32b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwentile2.5-32B-Instruct", - "id": "maldv/Qwentile2.5-32B-Instruct", - "developer": "maldv", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "32.764" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/maldv_Qwentile2.5-32B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7393 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6963 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5219 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3842 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4682 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5879 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/manolopueblo_contentcuisine_1-7b-slerp.json b/data/models/manolopueblo_contentcuisine_1-7b-slerp.json deleted file mode 100644 index 0eb10de793f16a25e7812e5aad4f544e890d3f87..0000000000000000000000000000000000000000 --- a/data/models/manolopueblo_contentcuisine_1-7b-slerp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ContentCuisine_1-7B-slerp", - "id": "ManoloPueblo/ContentCuisine_1-7B-slerp", - "developer": "ManoloPueblo", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ManoloPueblo_ContentCuisine_1-7B-slerp/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3907 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5188 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0733 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3029 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4672 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3054 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/manolopueblo_llm_merge_cc2.json b/data/models/manolopueblo_llm_merge_cc2.json deleted file mode 100644 index 7760dc8f34e9f6a8beb9a80ce76cb7c74a0921d0..0000000000000000000000000000000000000000 --- a/data/models/manolopueblo_llm_merge_cc2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "LLM_MERGE_CC2", - "id": "ManoloPueblo/LLM_MERGE_CC2", - "developer": "ManoloPueblo", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ManoloPueblo_LLM_MERGE_CC2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3853 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5209 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0642 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3045 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4593 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3032 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/manolopueblo_llm_merge_cc3.json b/data/models/manolopueblo_llm_merge_cc3.json deleted file mode 100644 index 3584b072e132e25e2a1c7a6b4efbd9242f95a903..0000000000000000000000000000000000000000 --- a/data/models/manolopueblo_llm_merge_cc3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "LLM_MERGE_CC3", - "id": "ManoloPueblo/LLM_MERGE_CC3", - "developer": "ManoloPueblo", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ManoloPueblo_LLM_MERGE_CC3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3959 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5246 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0793 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3096 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4672 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3156 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/marcuscedricridia_absolute-o1-7b.json b/data/models/marcuscedricridia_absolute-o1-7b.json deleted file mode 100644 index cf8f688d80afca6e3cb0bb60abf7068a30096f28..0000000000000000000000000000000000000000 --- a/data/models/marcuscedricridia_absolute-o1-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "absolute-o1-7b", - "id": "marcuscedricridia/absolute-o1-7b", - "developer": "marcuscedricridia", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.613" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/marcuscedricridia_absolute-o1-7b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7516 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5469 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5083 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3196 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4114 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4413 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/marcuscedricridia_cheng-1.json b/data/models/marcuscedricridia_cheng-1.json deleted file mode 100644 index 19bf8c9a9b55de005beb2cd5d1559d921f38aa12..0000000000000000000000000000000000000000 --- a/data/models/marcuscedricridia_cheng-1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Cheng-1", - "id": "marcuscedricridia/Cheng-1", - "developer": "marcuscedricridia", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.613" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/marcuscedricridia_Cheng-1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7789 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5525 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4894 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2961 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4073 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4349 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/marcuscedricridia_cheng-2-v1.1.json b/data/models/marcuscedricridia_cheng-2-v1.1.json deleted file mode 100644 index d34993016d14baacc1b061157d5bfe386520700a..0000000000000000000000000000000000000000 --- a/data/models/marcuscedricridia_cheng-2-v1.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Cheng-2-v1.1", - "id": "marcuscedricridia/Cheng-2-v1.1", - "developer": "marcuscedricridia", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/marcuscedricridia_Cheng-2-v1.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.827 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.651 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5393 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3431 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4167 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5076 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/marcuscedricridia_cheng-2.json b/data/models/marcuscedricridia_cheng-2.json deleted file mode 100644 index c937805f9c5d07a3687b53398cd0b87d1ec0291b..0000000000000000000000000000000000000000 --- a/data/models/marcuscedricridia_cheng-2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Cheng-2", - "id": "marcuscedricridia/Cheng-2", - "developer": "marcuscedricridia", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/marcuscedricridia_Cheng-2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8337 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6499 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5438 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3456 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4193 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5013 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/marcuscedricridia_cursa-o1-7b-2-28-2025.json b/data/models/marcuscedricridia_cursa-o1-7b-2-28-2025.json deleted file mode 100644 index 2317bada2fd5bf51ef6b142927fb86bd48949cdb..0000000000000000000000000000000000000000 --- a/data/models/marcuscedricridia_cursa-o1-7b-2-28-2025.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "cursa-o1-7b-2-28-2025", - "id": "marcuscedricridia/cursa-o1-7b-2-28-2025", - "developer": "marcuscedricridia", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.613" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/marcuscedricridia_cursa-o1-7b-2-28-2025/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7467 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5384 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4811 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.307 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4273 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4365 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/marcuscedricridia_cursa-o1-7b-v1.1.json b/data/models/marcuscedricridia_cursa-o1-7b-v1.1.json deleted file mode 100644 index b39bba308a6f0126d84abea1b669130990666299..0000000000000000000000000000000000000000 --- a/data/models/marcuscedricridia_cursa-o1-7b-v1.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "cursa-o1-7b-v1.1", - "id": "marcuscedricridia/cursa-o1-7b-v1.1", - "developer": "marcuscedricridia", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.613" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/marcuscedricridia_cursa-o1-7b-v1.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7528 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5493 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4985 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.307 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4259 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4392 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/marcuscedricridia_cursa-o1-7b-v1.2-normalize-false.json b/data/models/marcuscedricridia_cursa-o1-7b-v1.2-normalize-false.json deleted file mode 100644 index c66b539037a00f3836bbfb37670550a96eee5cbf..0000000000000000000000000000000000000000 --- a/data/models/marcuscedricridia_cursa-o1-7b-v1.2-normalize-false.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "cursa-o1-7b-v1.2-normalize-false", - "id": "marcuscedricridia/cursa-o1-7b-v1.2-normalize-false", - "developer": "marcuscedricridia", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.613" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/marcuscedricridia_cursa-o1-7b-v1.2-normalize-false/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7616 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5492 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4992 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.307 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4273 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4436 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/marcuscedricridia_cursa-o1-7b.json b/data/models/marcuscedricridia_cursa-o1-7b.json deleted file mode 100644 index 91b40b29817e02e6c63e4fc67c2385a26f0c3906..0000000000000000000000000000000000000000 --- a/data/models/marcuscedricridia_cursa-o1-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "cursa-o1-7b", - "id": "marcuscedricridia/cursa-o1-7b", - "developer": "marcuscedricridia", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.613" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/marcuscedricridia_cursa-o1-7b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7628 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5466 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4955 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.307 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4301 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4392 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/marcuscedricridia_cursor-o1-7b.json b/data/models/marcuscedricridia_cursor-o1-7b.json deleted file mode 100644 index 88c84943596317ca0c398a68bd7bc4123ddd2143..0000000000000000000000000000000000000000 --- a/data/models/marcuscedricridia_cursor-o1-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "cursor-o1-7b", - "id": "marcuscedricridia/cursor-o1-7b", - "developer": "marcuscedricridia", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/marcuscedricridia_cursor-o1-7b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4107 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5007 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1412 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.281 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4101 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3251 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/marcuscedricridia_cursorr-o1.2-7b.json b/data/models/marcuscedricridia_cursorr-o1.2-7b.json deleted file mode 100644 index 477cbecf053328942437d557f947eccc23bb324a..0000000000000000000000000000000000000000 --- a/data/models/marcuscedricridia_cursorr-o1.2-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "cursorr-o1.2-7b", - "id": "marcuscedricridia/cursorr-o1.2-7b", - "developer": "marcuscedricridia", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/marcuscedricridia_cursorr-o1.2-7b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.166 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3068 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2542 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3538 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.108 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/marcuscedricridia_etr1o-explicit-v1.1.json b/data/models/marcuscedricridia_etr1o-explicit-v1.1.json deleted file mode 100644 index 98775d1c8c49b313de34f718d4d73134118ed2b9..0000000000000000000000000000000000000000 --- a/data/models/marcuscedricridia_etr1o-explicit-v1.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "etr1o-explicit-v1.1", - "id": "marcuscedricridia/etr1o-explicit-v1.1", - "developer": "marcuscedricridia", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.613" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/marcuscedricridia_etr1o-explicit-v1.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.288 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3132 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0045 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2777 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4111 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1195 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/marcuscedricridia_etr1o-explicit-v1.2.json b/data/models/marcuscedricridia_etr1o-explicit-v1.2.json deleted file mode 100644 index 5c0398217db8928fe445dfde87f7c7ca8d1004ff..0000000000000000000000000000000000000000 --- a/data/models/marcuscedricridia_etr1o-explicit-v1.2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "etr1o-explicit-v1.2", - "id": "marcuscedricridia/etr1o-explicit-v1.2", - "developer": "marcuscedricridia", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.613" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/marcuscedricridia_etr1o-explicit-v1.2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1504 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.295 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2609 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4031 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1126 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/marcuscedricridia_etr1o-v1.1.json b/data/models/marcuscedricridia_etr1o-v1.1.json deleted file mode 100644 index 251ee98aaadfb5123a9cc54b2b4b5718572df025..0000000000000000000000000000000000000000 --- a/data/models/marcuscedricridia_etr1o-v1.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "etr1o-v1.1", - "id": "marcuscedricridia/etr1o-v1.1", - "developer": "marcuscedricridia", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.613" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/marcuscedricridia_etr1o-v1.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1597 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.31 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2567 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4017 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1157 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/marcuscedricridia_etr1o-v1.2.json b/data/models/marcuscedricridia_etr1o-v1.2.json deleted file mode 100644 index 169134e0556b81fe527b51d3c0bb82fdf1615aed..0000000000000000000000000000000000000000 --- a/data/models/marcuscedricridia_etr1o-v1.2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "etr1o-v1.2", - "id": "marcuscedricridia/etr1o-v1.2", - "developer": "marcuscedricridia", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/marcuscedricridia_etr1o-v1.2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7287 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6349 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3588 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3758 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4714 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5316 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/marcuscedricridia_fan-o1-7b.json b/data/models/marcuscedricridia_fan-o1-7b.json deleted file mode 100644 index 818235c5a54fa88abc5249fce88f06d2edba56e4..0000000000000000000000000000000000000000 --- a/data/models/marcuscedricridia_fan-o1-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "fan-o1-7b", - "id": "marcuscedricridia/fan-o1-7b", - "developer": "marcuscedricridia", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.613" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/marcuscedricridia_fan-o1-7b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4456 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4849 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1616 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2844 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3834 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3274 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/marcuscedricridia_hush-qwen2.5-7b-mst-v1.1.json b/data/models/marcuscedricridia_hush-qwen2.5-7b-mst-v1.1.json deleted file mode 100644 index 1cac3241557a296cd086694c0738aa26e5576e90..0000000000000000000000000000000000000000 --- a/data/models/marcuscedricridia_hush-qwen2.5-7b-mst-v1.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Hush-Qwen2.5-7B-MST-v1.1", - "id": "marcuscedricridia/Hush-Qwen2.5-7B-MST-v1.1", - "developer": "marcuscedricridia", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.613" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/marcuscedricridia_Hush-Qwen2.5-7B-MST-v1.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7445 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5559 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4653 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3062 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4073 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4299 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/marcuscedricridia_hush-qwen2.5-7b-mst-v1.3.json b/data/models/marcuscedricridia_hush-qwen2.5-7b-mst-v1.3.json deleted file mode 100644 index 1928bb1fcc706b1fc127b34c81941dfd399cc9b2..0000000000000000000000000000000000000000 --- a/data/models/marcuscedricridia_hush-qwen2.5-7b-mst-v1.3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Hush-Qwen2.5-7B-MST-v1.3", - "id": "marcuscedricridia/Hush-Qwen2.5-7B-MST-v1.3", - "developer": "marcuscedricridia", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.613" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/marcuscedricridia_Hush-Qwen2.5-7B-MST-v1.3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7043 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5516 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4758 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3146 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4311 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.444 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/marcuscedricridia_hush-qwen2.5-7b-mst.json b/data/models/marcuscedricridia_hush-qwen2.5-7b-mst.json deleted file mode 100644 index 577aea8653b1566a28faf9fb8b0fb2a8041c3fc6..0000000000000000000000000000000000000000 --- a/data/models/marcuscedricridia_hush-qwen2.5-7b-mst.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Hush-Qwen2.5-7B-MST", - "id": "marcuscedricridia/Hush-Qwen2.5-7B-MST", - "developer": "marcuscedricridia", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.613" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/marcuscedricridia_Hush-Qwen2.5-7B-MST/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7488 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5458 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4245 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3037 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3914 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4163 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/marcuscedricridia_hush-qwen2.5-7b-preview.json b/data/models/marcuscedricridia_hush-qwen2.5-7b-preview.json deleted file mode 100644 index 733b7ca8f93378c5e926518519ff456e1f1e40f9..0000000000000000000000000000000000000000 --- a/data/models/marcuscedricridia_hush-qwen2.5-7b-preview.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Hush-Qwen2.5-7B-Preview", - "id": "marcuscedricridia/Hush-Qwen2.5-7B-Preview", - "developer": "marcuscedricridia", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.613" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/marcuscedricridia_Hush-Qwen2.5-7B-Preview/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7962 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5431 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3754 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3112 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4298 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4364 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/marcuscedricridia_hush-qwen2.5-7b-rp-v1.4-1m.json b/data/models/marcuscedricridia_hush-qwen2.5-7b-rp-v1.4-1m.json deleted file mode 100644 index d89adf2a4811c3924fb00a36a1c5775b7bc0d965..0000000000000000000000000000000000000000 --- a/data/models/marcuscedricridia_hush-qwen2.5-7b-rp-v1.4-1m.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Hush-Qwen2.5-7B-RP-v1.4-1M", - "id": "marcuscedricridia/Hush-Qwen2.5-7B-RP-v1.4-1M", - "developer": "marcuscedricridia", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.613" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/marcuscedricridia_Hush-Qwen2.5-7B-RP-v1.4-1M/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7728 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5295 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3369 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2987 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4433 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4135 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/marcuscedricridia_hush-qwen2.5-7b-v1.1.json b/data/models/marcuscedricridia_hush-qwen2.5-7b-v1.1.json deleted file mode 100644 index c86ede60662da2cd0bf03382ed96c46857b3278b..0000000000000000000000000000000000000000 --- a/data/models/marcuscedricridia_hush-qwen2.5-7b-v1.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Hush-Qwen2.5-7B-v1.1", - "id": "marcuscedricridia/Hush-Qwen2.5-7B-v1.1", - "developer": "marcuscedricridia", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.613" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/marcuscedricridia_Hush-Qwen2.5-7B-v1.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7889 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5384 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4381 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3163 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4179 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4227 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/marcuscedricridia_hush-qwen2.5-7b-v1.2.json b/data/models/marcuscedricridia_hush-qwen2.5-7b-v1.2.json deleted file mode 100644 index b1e65bc37aea25dad4598c11721af465abdc2633..0000000000000000000000000000000000000000 --- a/data/models/marcuscedricridia_hush-qwen2.5-7b-v1.2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Hush-Qwen2.5-7B-v1.2", - "id": "marcuscedricridia/Hush-Qwen2.5-7B-v1.2", - "developer": "marcuscedricridia", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.613" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/marcuscedricridia_Hush-Qwen2.5-7B-v1.2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7865 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5403 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4403 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3146 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4219 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4197 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/marcuscedricridia_hush-qwen2.5-7b-v1.3.json b/data/models/marcuscedricridia_hush-qwen2.5-7b-v1.3.json deleted file mode 100644 index 7b5bda60c30c991c56e80cc34aba2818d6488fe9..0000000000000000000000000000000000000000 --- a/data/models/marcuscedricridia_hush-qwen2.5-7b-v1.3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Hush-Qwen2.5-7B-v1.3", - "id": "marcuscedricridia/Hush-Qwen2.5-7B-v1.3", - "developer": "marcuscedricridia", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.613" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/marcuscedricridia_Hush-Qwen2.5-7B-v1.3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7856 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5327 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3323 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3121 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4246 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4345 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/marcuscedricridia_hush-qwen2.5-7b-v1.4.json b/data/models/marcuscedricridia_hush-qwen2.5-7b-v1.4.json deleted file mode 100644 index 09ad5d296549a324eb4a00cc284de447d914ffda..0000000000000000000000000000000000000000 --- a/data/models/marcuscedricridia_hush-qwen2.5-7b-v1.4.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Hush-Qwen2.5-7B-v1.4", - "id": "marcuscedricridia/Hush-Qwen2.5-7B-v1.4", - "developer": "marcuscedricridia", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.613" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/marcuscedricridia_Hush-Qwen2.5-7B-v1.4/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7835 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5423 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.426 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3112 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4232 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4195 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/marcuscedricridia_olmner-7b.json b/data/models/marcuscedricridia_olmner-7b.json deleted file mode 100644 index 32d31cd764e3ff1376866a4a82bf5586fc1251a1..0000000000000000000000000000000000000000 --- a/data/models/marcuscedricridia_olmner-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "olmner-7b", - "id": "marcuscedricridia/olmner-7b", - "developer": "marcuscedricridia", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/marcuscedricridia_olmner-7b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7254 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5472 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.463 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3079 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.438 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4309 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/marcuscedricridia_olmner-della-7b.json b/data/models/marcuscedricridia_olmner-della-7b.json deleted file mode 100644 index e77159a57f9e774e83d8a28961e65aab935bfece..0000000000000000000000000000000000000000 --- a/data/models/marcuscedricridia_olmner-della-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "olmner-della-7b", - "id": "marcuscedricridia/olmner-della-7b", - "developer": "marcuscedricridia", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/marcuscedricridia_olmner-della-7b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7637 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5491 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4962 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3012 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4208 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4386 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/marcuscedricridia_olmner-o1-7b.json b/data/models/marcuscedricridia_olmner-o1-7b.json deleted file mode 100644 index 5506b41608507c8c27899cd64da8441b3222f47c..0000000000000000000000000000000000000000 --- a/data/models/marcuscedricridia_olmner-o1-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "olmner-o1-7b", - "id": "marcuscedricridia/olmner-o1-7b", - "developer": "marcuscedricridia", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.613" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/marcuscedricridia_olmner-o1-7b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7528 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5481 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4924 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3012 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4299 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4386 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/marcuscedricridia_olmner-sbr-7b.json b/data/models/marcuscedricridia_olmner-sbr-7b.json deleted file mode 100644 index 898e10eecb07b271b331cf9c8de6024bc043e46e..0000000000000000000000000000000000000000 --- a/data/models/marcuscedricridia_olmner-sbr-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "olmner-sbr-7b", - "id": "marcuscedricridia/olmner-sbr-7b", - "developer": "marcuscedricridia", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.613" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/marcuscedricridia_olmner-sbr-7b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.76 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5462 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4947 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3087 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4154 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4412 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/marcuscedricridia_post-cursa-o1.json b/data/models/marcuscedricridia_post-cursa-o1.json deleted file mode 100644 index d1f84d3d7848d303950d248f56e7f26e07bee708..0000000000000000000000000000000000000000 --- a/data/models/marcuscedricridia_post-cursa-o1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "post-cursa-o1", - "id": "marcuscedricridia/post-cursa-o1", - "developer": "marcuscedricridia", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.613" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/marcuscedricridia_post-cursa-o1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7628 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.548 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4872 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3096 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4351 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4361 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/marcuscedricridia_pre-cursa-o1-v1.2.json b/data/models/marcuscedricridia_pre-cursa-o1-v1.2.json deleted file mode 100644 index 0b8a3cb2e605cb02d82482ef00ddab2192903be1..0000000000000000000000000000000000000000 --- a/data/models/marcuscedricridia_pre-cursa-o1-v1.2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "pre-cursa-o1-v1.2", - "id": "marcuscedricridia/pre-cursa-o1-v1.2", - "developer": "marcuscedricridia", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.613" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/marcuscedricridia_pre-cursa-o1-v1.2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7549 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5487 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5068 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3129 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4272 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4402 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/marcuscedricridia_pre-cursa-o1-v1.3.json b/data/models/marcuscedricridia_pre-cursa-o1-v1.3.json deleted file mode 100644 index 62ce243a7c835293e4a5a1385d18e40a396750d5..0000000000000000000000000000000000000000 --- a/data/models/marcuscedricridia_pre-cursa-o1-v1.3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "pre-cursa-o1-v1.3", - "id": "marcuscedricridia/pre-cursa-o1-v1.3", - "developer": "marcuscedricridia", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.613" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/marcuscedricridia_pre-cursa-o1-v1.3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7507 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5455 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5076 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3129 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4271 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.442 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/marcuscedricridia_pre-cursa-o1-v1.4.json b/data/models/marcuscedricridia_pre-cursa-o1-v1.4.json deleted file mode 100644 index d59eecaafa5440286ae3d29c0206dd855bd0cf94..0000000000000000000000000000000000000000 --- a/data/models/marcuscedricridia_pre-cursa-o1-v1.4.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "pre-cursa-o1-v1.4", - "id": "marcuscedricridia/pre-cursa-o1-v1.4", - "developer": "marcuscedricridia", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.613" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/marcuscedricridia_pre-cursa-o1-v1.4/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7488 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5493 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4834 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3054 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4285 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4436 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/marcuscedricridia_pre-cursa-o1-v1.6.json b/data/models/marcuscedricridia_pre-cursa-o1-v1.6.json deleted file mode 100644 index 97e044cb6932c740d34450747eb6fb196ea93bbd..0000000000000000000000000000000000000000 --- a/data/models/marcuscedricridia_pre-cursa-o1-v1.6.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "pre-cursa-o1-v1.6", - "id": "marcuscedricridia/pre-cursa-o1-v1.6", - "developer": "marcuscedricridia", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.613" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/marcuscedricridia_pre-cursa-o1-v1.6/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7528 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5473 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3205 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4234 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4413 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/marcuscedricridia_pre-cursa-o1.json b/data/models/marcuscedricridia_pre-cursa-o1.json deleted file mode 100644 index 04a199c0394124b707bf76da6938d0f40737aefe..0000000000000000000000000000000000000000 --- a/data/models/marcuscedricridia_pre-cursa-o1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "pre-cursa-o1", - "id": "marcuscedricridia/pre-cursa-o1", - "developer": "marcuscedricridia", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.613" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/marcuscedricridia_pre-cursa-o1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7409 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5462 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5038 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3096 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.426 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4424 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/marcuscedricridia_qwen2.5-7b-preview.json b/data/models/marcuscedricridia_qwen2.5-7b-preview.json deleted file mode 100644 index 4fcf6a7e7ad00913769a1365648417dfd3cdb22e..0000000000000000000000000000000000000000 --- a/data/models/marcuscedricridia_qwen2.5-7b-preview.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-7B-Preview", - "id": "marcuscedricridia/Qwen2.5-7B-Preview", - "developer": "marcuscedricridia", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.613" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/marcuscedricridia_Qwen2.5-7B-Preview/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7679 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.536 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3444 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3238 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.414 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4258 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/marcuscedricridia_r1o-et.json b/data/models/marcuscedricridia_r1o-et.json deleted file mode 100644 index 7d97d0bd44995aee91a15292d82bee046770cb50..0000000000000000000000000000000000000000 --- a/data/models/marcuscedricridia_r1o-et.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "r1o-et", - "id": "marcuscedricridia/r1o-et", - "developer": "marcuscedricridia", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.613" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/marcuscedricridia_r1o-et/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3597 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4209 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0793 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2727 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3579 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.258 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/marcuscedricridia_sbr-o1-7b.json b/data/models/marcuscedricridia_sbr-o1-7b.json deleted file mode 100644 index 26a5cbb23b6621c6868d3eaf9f0977a0d4447a93..0000000000000000000000000000000000000000 --- a/data/models/marcuscedricridia_sbr-o1-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "sbr-o1-7b", - "id": "marcuscedricridia/sbr-o1-7b", - "developer": "marcuscedricridia", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.613" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/marcuscedricridia_sbr-o1-7b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7455 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5479 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4985 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3104 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4404 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4355 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/marcuscedricridia_stray-r1o-et.json b/data/models/marcuscedricridia_stray-r1o-et.json deleted file mode 100644 index a9cab2b9a5e06151bde3812e359c5a2799f17a7b..0000000000000000000000000000000000000000 --- a/data/models/marcuscedricridia_stray-r1o-et.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "stray-r1o-et", - "id": "marcuscedricridia/stray-r1o-et", - "developer": "marcuscedricridia", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.613" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/marcuscedricridia_stray-r1o-et/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1562 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2967 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0045 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2617 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4086 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1094 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/marcuscedricridia_yell-qwen2.5-7b-preview-v1.1.json b/data/models/marcuscedricridia_yell-qwen2.5-7b-preview-v1.1.json deleted file mode 100644 index 0c90211f81ebdd4ba0e1c1f67c638deddd748909..0000000000000000000000000000000000000000 --- a/data/models/marcuscedricridia_yell-qwen2.5-7b-preview-v1.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Yell-Qwen2.5-7B-Preview-v1.1", - "id": "marcuscedricridia/Yell-Qwen2.5-7B-Preview-v1.1", - "developer": "marcuscedricridia", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.613" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/marcuscedricridia_Yell-Qwen2.5-7B-Preview-v1.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5757 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5348 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1896 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2861 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4059 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3831 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/marcuscedricridia_yell-qwen2.5-7b-preview.json b/data/models/marcuscedricridia_yell-qwen2.5-7b-preview.json deleted file mode 100644 index 85bb8fdb06de24254ad2de03936703c1f74d3170..0000000000000000000000000000000000000000 --- a/data/models/marcuscedricridia_yell-qwen2.5-7b-preview.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Yell-Qwen2.5-7B-Preview", - "id": "marcuscedricridia/Yell-Qwen2.5-7B-Preview", - "developer": "marcuscedricridia", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.613" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/marcuscedricridia_Yell-Qwen2.5-7B-Preview/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5839 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5371 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1926 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.281 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4046 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3798 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/marin-community_marin-8b-instruct.json b/data/models/marin-community_marin-8b-instruct.json deleted file mode 100644 index 0161c628e7905daba61549cdec3ec5a49214deaf..0000000000000000000000000000000000000000 --- a/data/models/marin-community_marin-8b-instruct.json +++ /dev/null @@ -1,241 +0,0 @@ -{ - "model_info": { - "name": "Marin 8B Instruct", - "id": "marin-community/marin-8b-instruct", - "developer": "marin-community", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_capabilities/marin-community_marin-8b-instruct/1774096304.5056698", - "retrieved_timestamp": "1774096304.5056698", - "source_metadata": { - "source_name": "helm_capabilities", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_capabilities", - "evaluation_results": [ - { - "evaluation_name": "Mean score", - "source_data": { - "dataset_name": "helm_capabilities", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "The mean of the scores from all columns.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.325, - "details": { - "description": "", - "tab": "Accuracy", - "Mean score - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"118.55196213968559\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU-Pro", - "source_data": { - "dataset_name": "MMLU-Pro", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on MMLU-Pro", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.188, - "details": { - "description": "min=0.188, mean=0.188, max=0.188, sum=0.188 (1)", - "tab": "Accuracy", - "MMLU-Pro - Observed inference time (s)": "{\"description\": \"min=94.096, mean=94.096, max=94.096, sum=94.096 (1)\", \"tab\": \"Efficiency\", \"score\": \"94.0957455046177\"}", - "MMLU-Pro - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "MMLU-Pro - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - # prompt tokens": "{\"description\": \"min=228.366, mean=228.366, max=228.366, sum=228.366 (1)\", \"tab\": \"General information\", \"score\": \"228.366\"}", - "MMLU-Pro - # output tokens": "{\"description\": \"min=539.21, mean=539.21, max=539.21, sum=539.21 (1)\", \"tab\": \"General information\", \"score\": \"539.21\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"all\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"", - "num_output_tokens": "\"2048\"" - } - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.168, - "details": { - "description": "min=0.168, mean=0.168, max=0.168, sum=0.168 (1)", - "tab": "Accuracy", - "GPQA - Observed inference time (s)": "{\"description\": \"min=123.019, mean=123.019, max=123.019, sum=123.019 (1)\", \"tab\": \"Efficiency\", \"score\": \"123.0189983149815\"}", - "GPQA - # eval": "{\"description\": \"min=446, mean=446, max=446, sum=446 (1)\", \"tab\": \"General information\", \"score\": \"446.0\"}", - "GPQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - truncated": "{\"description\": \"min=0.002, mean=0.002, max=0.002, sum=0.002 (1)\", \"tab\": \"General information\", \"score\": \"0.002242152466367713\"}", - "GPQA - # prompt tokens": "{\"description\": \"min=247.173, mean=247.173, max=247.173, sum=247.173 (1)\", \"tab\": \"General information\", \"score\": \"247.1726457399103\"}", - "GPQA - # output tokens": "{\"description\": \"min=707.953, mean=707.953, max=707.953, sum=707.953 (1)\", \"tab\": \"General information\", \"score\": \"707.9529147982063\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"gpqa_main\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"", - "num_output_tokens": "\"2048\"" - } - } - }, - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "IFEval Strict Acc on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.632, - "details": { - "description": "min=0.632, mean=0.632, max=0.632, sum=0.632 (1)", - "tab": "Accuracy", - "IFEval - Observed inference time (s)": "{\"description\": \"min=88.889, mean=88.889, max=88.889, sum=88.889 (1)\", \"tab\": \"Efficiency\", \"score\": \"88.88931880596606\"}", - "IFEval - # eval": "{\"description\": \"min=541, mean=541, max=541, sum=541 (1)\", \"tab\": \"General information\", \"score\": \"541.0\"}", - "IFEval - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - # prompt tokens": "{\"description\": \"min=46.024, mean=46.024, max=46.024, sum=46.024 (1)\", \"tab\": \"General information\", \"score\": \"46.024029574861366\"}", - "IFEval - # output tokens": "{\"description\": \"min=516.492, mean=516.492, max=516.492, sum=516.492 (1)\", \"tab\": \"General information\", \"score\": \"516.4916820702402\"}" - } - }, - "generation_config": { - "additional_details": { - "num_output_tokens": "\"2048\"" - } - } - }, - { - "evaluation_name": "WildBench", - "source_data": { - "dataset_name": "WildBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "WB Score on WildBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.477, - "details": { - "description": "min=0.477, mean=0.477, max=0.477, sum=0.477 (1)", - "tab": "Accuracy", - "WildBench - Observed inference time (s)": "{\"description\": \"min=146.873, mean=146.873, max=146.873, sum=146.873 (1)\", \"tab\": \"Efficiency\", \"score\": \"146.8726548871994\"}", - "WildBench - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "WildBench - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # prompt tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # output tokens": "{\"description\": \"min=818.678, mean=818.678, max=818.678, sum=818.678 (1)\", \"tab\": \"General information\", \"score\": \"818.678\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"v2\"", - "num_output_tokens": "\"2048\"" - } - } - }, - { - "evaluation_name": "Omni-MATH", - "source_data": { - "dataset_name": "Omni-MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Acc on Omni-MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.16, - "details": { - "description": "min=0.16, mean=0.16, max=0.16, sum=0.16 (1)", - "tab": "Accuracy", - "Omni-MATH - Observed inference time (s)": "{\"description\": \"min=139.883, mean=139.883, max=139.883, sum=139.883 (1)\", \"tab\": \"Efficiency\", \"score\": \"139.88309318566323\"}", - "Omni-MATH - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "Omni-MATH - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - truncated": "{\"description\": \"min=0.001, mean=0.001, max=0.001, sum=0.001 (1)\", \"tab\": \"General information\", \"score\": \"0.001\"}", - "Omni-MATH - # prompt tokens": "{\"description\": \"min=108.784, mean=108.784, max=108.784, sum=108.784 (1)\", \"tab\": \"General information\", \"score\": \"108.784\"}", - "Omni-MATH - # output tokens": "{\"description\": \"min=808.178, mean=808.178, max=808.178, sum=808.178 (1)\", \"tab\": \"General information\", \"score\": \"808.178\"}" - } - }, - "generation_config": { - "additional_details": { - "num_output_tokens": "\"2048\"" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/marinaraspaghetti_nemomix-v4.0-12b.json b/data/models/marinaraspaghetti_nemomix-v4.0-12b.json deleted file mode 100644 index 922a4739b84aec4857f761b54bf354a82392e0eb..0000000000000000000000000000000000000000 --- a/data/models/marinaraspaghetti_nemomix-v4.0-12b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Nemomix-v4.0-12B", - "id": "MarinaraSpaghetti/Nemomix-v4.0-12B", - "developer": "MarinaraSpaghetti", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/MarinaraSpaghetti_Nemomix-v4.0-12B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5575 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5275 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.108 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2919 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4244 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3613 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/marinaraspaghetti_nemoreremix-12b.json b/data/models/marinaraspaghetti_nemoreremix-12b.json deleted file mode 100644 index 221c1b57d96e4991cb7d8f3260734d3a91839b47..0000000000000000000000000000000000000000 --- a/data/models/marinaraspaghetti_nemoreremix-12b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "NemoReRemix-12B", - "id": "MarinaraSpaghetti/NemoReRemix-12B", - "developer": "MarinaraSpaghetti", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/MarinaraSpaghetti_NemoReRemix-12B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3343 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5537 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0906 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.318 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4501 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3598 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/marsouuu_general3b-ece-prymmal-martial.json b/data/models/marsouuu_general3b-ece-prymmal-martial.json deleted file mode 100644 index 4b50ad06b872cba7d5c94eb030b5715f9de59508..0000000000000000000000000000000000000000 --- a/data/models/marsouuu_general3b-ece-prymmal-martial.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "general3B-ECE-PRYMMAL-Martial", - "id": "Marsouuu/general3B-ECE-PRYMMAL-Martial", - "developer": "Marsouuu", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Phi3ForCausalLM", - "params_billions": "3.821" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Marsouuu_general3B-ECE-PRYMMAL-Martial/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2722 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5394 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1548 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3196 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4701 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3876 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/marsouuu_general3bv2-ece-prymmal-martial.json b/data/models/marsouuu_general3bv2-ece-prymmal-martial.json deleted file mode 100644 index a11517dbdd86abfc8003611351ac61374d36c6b4..0000000000000000000000000000000000000000 --- a/data/models/marsouuu_general3bv2-ece-prymmal-martial.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "general3Bv2-ECE-PRYMMAL-Martial", - "id": "Marsouuu/general3Bv2-ECE-PRYMMAL-Martial", - "developer": "Marsouuu", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Marsouuu_general3Bv2-ECE-PRYMMAL-Martial/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5693 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5637 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3671 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3104 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4396 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4498 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/marsouuu_lareneg1_78b-ece-prymmal-martial.json b/data/models/marsouuu_lareneg1_78b-ece-prymmal-martial.json deleted file mode 100644 index b0a8e133b80d1328c3b64b695f5986827a2084c5..0000000000000000000000000000000000000000 --- a/data/models/marsouuu_lareneg1_78b-ece-prymmal-martial.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "lareneg1_78B-ECE-PRYMMAL-Martial", - "id": "Marsouuu/lareneg1_78B-ECE-PRYMMAL-Martial", - "developer": "Marsouuu", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.777" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Marsouuu_lareneg1_78B-ECE-PRYMMAL-Martial/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2795 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.423 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.114 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2819 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3867 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2922 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/marsouuu_lareneg3b-ece-prymmal-martial.json b/data/models/marsouuu_lareneg3b-ece-prymmal-martial.json deleted file mode 100644 index 1264d28b8de5ae36cbc5ba9c319e2ab4a86f8976..0000000000000000000000000000000000000000 --- a/data/models/marsouuu_lareneg3b-ece-prymmal-martial.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "lareneg3B-ECE-PRYMMAL-Martial", - "id": "Marsouuu/lareneg3B-ECE-PRYMMAL-Martial", - "developer": "Marsouuu", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Phi3ForCausalLM", - "params_billions": "3.821" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Marsouuu_lareneg3B-ECE-PRYMMAL-Martial/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3303 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5453 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1518 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3247 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4725 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3767 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/marsouuu_lareneg3bv2-ece-prymmal-martial.json b/data/models/marsouuu_lareneg3bv2-ece-prymmal-martial.json deleted file mode 100644 index 7d59e3c48de30b2fdc09e597dcc24ef7a63cc05d..0000000000000000000000000000000000000000 --- a/data/models/marsouuu_lareneg3bv2-ece-prymmal-martial.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "lareneg3Bv2-ECE-PRYMMAL-Martial", - "id": "Marsouuu/lareneg3Bv2-ECE-PRYMMAL-Martial", - "developer": "Marsouuu", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Marsouuu_lareneg3Bv2-ECE-PRYMMAL-Martial/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5753 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5623 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3656 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3196 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4369 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4511 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/marsouuu_minimathexpert-2_61b-ece-prymmal-martial.json b/data/models/marsouuu_minimathexpert-2_61b-ece-prymmal-martial.json deleted file mode 100644 index ad13e3d0262a5f697bc5d1223bdfe4c6ec609ebd..0000000000000000000000000000000000000000 --- a/data/models/marsouuu_minimathexpert-2_61b-ece-prymmal-martial.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MiniMathExpert-2_61B-ECE-PRYMMAL-Martial", - "id": "Marsouuu/MiniMathExpert-2_61B-ECE-PRYMMAL-Martial", - "developer": "Marsouuu", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "2.614" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Marsouuu_MiniMathExpert-2_61B-ECE-PRYMMAL-Martial/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2548 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3953 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.074 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2752 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4083 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2274 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/marsouuu_miniqwenmathexpert-ece-prymmal-martial.json b/data/models/marsouuu_miniqwenmathexpert-ece-prymmal-martial.json deleted file mode 100644 index 8f956fa9a343e196b40c1fece339163d03986ad5..0000000000000000000000000000000000000000 --- a/data/models/marsouuu_miniqwenmathexpert-ece-prymmal-martial.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MiniQwenMathExpert-ECE-PRYMMAL-Martial", - "id": "Marsouuu/MiniQwenMathExpert-ECE-PRYMMAL-Martial", - "developer": "Marsouuu", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.777" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Marsouuu_MiniQwenMathExpert-ECE-PRYMMAL-Martial/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2795 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.423 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.114 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2819 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3867 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2922 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/marsouuu_mistralbase-4x7b-moe-ece-prymmal-martial.json b/data/models/marsouuu_mistralbase-4x7b-moe-ece-prymmal-martial.json deleted file mode 100644 index 40cf7d0a6a0787fe43f25e75e3f55afdf47492ac..0000000000000000000000000000000000000000 --- a/data/models/marsouuu_mistralbase-4x7b-moe-ece-prymmal-martial.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MistralBase-4x7B-MoE-ECE-PRYMMAL-Martial", - "id": "Marsouuu/MistralBase-4x7B-MoE-ECE-PRYMMAL-Martial", - "developer": "Marsouuu", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MixtralForCausalLM", - "params_billions": "24.16" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Marsouuu_MistralBase-4x7B-MoE-ECE-PRYMMAL-Martial/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1697 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3464 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0144 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2592 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3991 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1379 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/matouleloup_ece-prymmal-0.5b-ft-enhancedmusrensemblev3.json b/data/models/matouleloup_ece-prymmal-0.5b-ft-enhancedmusrensemblev3.json deleted file mode 100644 index 29c82f81f9f0dfa0d746ba84879caa09bd483753..0000000000000000000000000000000000000000 --- a/data/models/matouleloup_ece-prymmal-0.5b-ft-enhancedmusrensemblev3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ECE-PRYMMAL-0.5B-FT-EnhancedMUSREnsembleV3", - "id": "matouLeLoup/ECE-PRYMMAL-0.5B-FT-EnhancedMUSREnsembleV3", - "developer": "matouLeLoup", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/matouLeLoup_ECE-PRYMMAL-0.5B-FT-EnhancedMUSREnsembleV3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1873 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3239 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0264 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2609 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3752 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.172 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/matouleloup_ece-prymmal-0.5b-ft-musr-ensemble-v2mathis.json b/data/models/matouleloup_ece-prymmal-0.5b-ft-musr-ensemble-v2mathis.json deleted file mode 100644 index 3356d0e3ba491512b6af328bfe4db53ef14b07e5..0000000000000000000000000000000000000000 --- a/data/models/matouleloup_ece-prymmal-0.5b-ft-musr-ensemble-v2mathis.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ECE-PRYMMAL-0.5B-FT-MUSR-ENSEMBLE-V2Mathis", - "id": "matouLeLoup/ECE-PRYMMAL-0.5B-FT-MUSR-ENSEMBLE-V2Mathis", - "developer": "matouLeLoup", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/matouLeLoup_ECE-PRYMMAL-0.5B-FT-MUSR-ENSEMBLE-V2Mathis/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1873 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3239 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0264 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2609 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3752 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.172 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/matouleloup_ece-prymmal-0.5b-ft-v4-musr-ensemble-mathis.json b/data/models/matouleloup_ece-prymmal-0.5b-ft-v4-musr-ensemble-mathis.json deleted file mode 100644 index 03392196a6d4aa938d70b6a99ee9fbd57bd223fd..0000000000000000000000000000000000000000 --- a/data/models/matouleloup_ece-prymmal-0.5b-ft-v4-musr-ensemble-mathis.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ECE-PRYMMAL-0.5B-FT-V4-MUSR-ENSEMBLE-Mathis", - "id": "matouLeLoup/ECE-PRYMMAL-0.5B-FT-V4-MUSR-ENSEMBLE-Mathis", - "developer": "matouLeLoup", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/matouLeLoup_ECE-PRYMMAL-0.5B-FT-V4-MUSR-ENSEMBLE-Mathis/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1873 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3239 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0264 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2609 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3752 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.172 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/matouleloup_ece-prymmal-0.5b-ft-v4-musr-mathis.json b/data/models/matouleloup_ece-prymmal-0.5b-ft-v4-musr-mathis.json deleted file mode 100644 index 6de2917ee63fdbb4e97de80532998782d0c17706..0000000000000000000000000000000000000000 --- a/data/models/matouleloup_ece-prymmal-0.5b-ft-v4-musr-mathis.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ECE-PRYMMAL-0.5B-FT-V4-MUSR-Mathis", - "id": "matouLeLoup/ECE-PRYMMAL-0.5B-FT-V4-MUSR-Mathis", - "developer": "matouLeLoup", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/matouLeLoup_ECE-PRYMMAL-0.5B-FT-V4-MUSR-Mathis/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1882 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3233 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0272 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2634 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3685 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.172 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/matouleloup_ece-prymmal-0.5b-ft-v5-musr-mathis.json b/data/models/matouleloup_ece-prymmal-0.5b-ft-v5-musr-mathis.json deleted file mode 100644 index 83d481a1b45d32c48340650e18d3ef5da4722868..0000000000000000000000000000000000000000 --- a/data/models/matouleloup_ece-prymmal-0.5b-ft-v5-musr-mathis.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ECE-PRYMMAL-0.5B-FT-V5-MUSR-Mathis", - "id": "matouLeLoup/ECE-PRYMMAL-0.5B-FT-V5-MUSR-Mathis", - "developer": "matouLeLoup", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "?", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/matouLeLoup_ECE-PRYMMAL-0.5B-FT-V5-MUSR-Mathis/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1652 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3024 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0189 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2567 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4273 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1116 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/mattshumer_ref_70_e3.json b/data/models/mattshumer_ref_70_e3.json deleted file mode 100644 index a24dbb2079de0d65c581b10688c2ba98a7e23101..0000000000000000000000000000000000000000 --- a/data/models/mattshumer_ref_70_e3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ref_70_e3", - "id": "mattshumer/ref_70_e3", - "developer": "mattshumer", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "70.554" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/mattshumer_ref_70_e3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6294 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6501 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2795 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3356 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4328 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5303 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/mattshumer_reflection-70b.json b/data/models/mattshumer_reflection-70b.json deleted file mode 100644 index b4ed8633a5232f2015f19991d4c1d13c9fc8d401..0000000000000000000000000000000000000000 --- a/data/models/mattshumer_reflection-70b.json +++ /dev/null @@ -1,126 +0,0 @@ -{ - "model_info": { - "name": "mattshumer/Reflection-70B", - "id": "mattshumer/Reflection-70B", - "developer": "mattshumer", - "additional_details": { - "model_type": "Generative" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/mattshumer_Reflection-70B/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8422 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9749 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7061 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8318 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8562 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/mattshumer_reflection-llama-3.1-70b.json b/data/models/mattshumer_reflection-llama-3.1-70b.json deleted file mode 100644 index 615fe1d55c805c9084915a2342317495fcd0d74b..0000000000000000000000000000000000000000 --- a/data/models/mattshumer_reflection-llama-3.1-70b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Reflection-Llama-3.1-70B", - "id": "mattshumer/Reflection-Llama-3.1-70B", - "developer": "mattshumer", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "70.554" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/mattshumer_Reflection-Llama-3.1-70B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0045 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.645 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2145 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3633 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4577 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4955 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/maywell_qwen2-7b-multilingual-rp.json b/data/models/maywell_qwen2-7b-multilingual-rp.json deleted file mode 100644 index 73fba8d4a2af731e3327d1fc1cff47f2232cfcc6..0000000000000000000000000000000000000000 --- a/data/models/maywell_qwen2-7b-multilingual-rp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2-7B-Multilingual-RP", - "id": "maywell/Qwen2-7B-Multilingual-RP", - "developer": "maywell", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/maywell_Qwen2-7B-Multilingual-RP/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4347 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5062 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2243 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.297 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3696 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3859 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/maziyarpanahi_calme-2.1-llama3.1-70b.json b/data/models/maziyarpanahi_calme-2.1-llama3.1-70b.json deleted file mode 100644 index 30235a7183f51a780d2a3b5b601497d1bede80ae..0000000000000000000000000000000000000000 --- a/data/models/maziyarpanahi_calme-2.1-llama3.1-70b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "calme-2.1-llama3.1-70b", - "id": "MaziyarPanahi/calme-2.1-llama3.1-70b", - "developer": "MaziyarPanahi", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "70.554" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/MaziyarPanahi_calme-2.1-llama3.1-70b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8434 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6448 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4101 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.328 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.438 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5283 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/maziyarpanahi_calme-2.1-phi3-4b.json b/data/models/maziyarpanahi_calme-2.1-phi3-4b.json deleted file mode 100644 index c3c5af9d5d127af084f5b276ac122bb1585b4d36..0000000000000000000000000000000000000000 --- a/data/models/maziyarpanahi_calme-2.1-phi3-4b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "calme-2.1-phi3-4b", - "id": "MaziyarPanahi/calme-2.1-phi3-4b", - "developer": "MaziyarPanahi", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Phi3ForCausalLM", - "params_billions": "3.821" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/MaziyarPanahi_calme-2.1-phi3-4b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5525 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5595 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1314 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3297 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4015 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3746 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/maziyarpanahi_calme-2.1-phi3.5-4b.json b/data/models/maziyarpanahi_calme-2.1-phi3.5-4b.json deleted file mode 100644 index 2fafc897185c7bcd2a281061bc492a62696f9054..0000000000000000000000000000000000000000 --- a/data/models/maziyarpanahi_calme-2.1-phi3.5-4b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "calme-2.1-phi3.5-4b", - "id": "MaziyarPanahi/calme-2.1-phi3.5-4b", - "developer": "MaziyarPanahi", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Phi3ForCausalLM", - "params_billions": "3.821" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/MaziyarPanahi_calme-2.1-phi3.5-4b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5659 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5484 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2039 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.344 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3995 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3935 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/maziyarpanahi_calme-2.1-qwen2-72b.json b/data/models/maziyarpanahi_calme-2.1-qwen2-72b.json deleted file mode 100644 index e38247f4e78efcfd25eb89daf79a680dadb3e7e2..0000000000000000000000000000000000000000 --- a/data/models/maziyarpanahi_calme-2.1-qwen2-72b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "calme-2.1-qwen2-72b", - "id": "MaziyarPanahi/calme-2.1-qwen2-72b", - "developer": "MaziyarPanahi", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "72.699" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/MaziyarPanahi_calme-2.1-qwen2-72b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8163 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6966 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4079 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3809 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4732 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5415 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/maziyarpanahi_calme-2.1-qwen2-7b.json b/data/models/maziyarpanahi_calme-2.1-qwen2-7b.json deleted file mode 100644 index 1e8d77b3db3073be58c34df10540cfed877cf984..0000000000000000000000000000000000000000 --- a/data/models/maziyarpanahi_calme-2.1-qwen2-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "calme-2.1-qwen2-7b", - "id": "MaziyarPanahi/calme-2.1-qwen2-7b", - "developer": "MaziyarPanahi", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/MaziyarPanahi_calme-2.1-qwen2-7b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3816 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5046 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2311 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2894 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4437 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3693 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/maziyarpanahi_calme-2.1-qwen2.5-72b.json b/data/models/maziyarpanahi_calme-2.1-qwen2.5-72b.json deleted file mode 100644 index 53a82e72ddef5f1e6b9b273033a2e2af49e71c41..0000000000000000000000000000000000000000 --- a/data/models/maziyarpanahi_calme-2.1-qwen2.5-72b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "calme-2.1-qwen2.5-72b", - "id": "MaziyarPanahi/calme-2.1-qwen2.5-72b", - "developer": "MaziyarPanahi", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "72.7" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/MaziyarPanahi_calme-2.1-qwen2.5-72b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8662 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7262 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5914 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3633 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4298 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5619 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/maziyarpanahi_calme-2.1-rys-78b.json b/data/models/maziyarpanahi_calme-2.1-rys-78b.json deleted file mode 100644 index bd9a380910ad12299d77b544e21e3d44a792a741..0000000000000000000000000000000000000000 --- a/data/models/maziyarpanahi_calme-2.1-rys-78b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "calme-2.1-rys-78b", - "id": "MaziyarPanahi/calme-2.1-rys-78b", - "developer": "MaziyarPanahi", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "77.965" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/MaziyarPanahi_calme-2.1-rys-78b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8136 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7098 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3943 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3943 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4693 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5444 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/maziyarpanahi_calme-2.2-llama3-70b.json b/data/models/maziyarpanahi_calme-2.2-llama3-70b.json deleted file mode 100644 index 9df7bbdba1e345582ba08e865cd227a3a5587687..0000000000000000000000000000000000000000 --- a/data/models/maziyarpanahi_calme-2.2-llama3-70b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "calme-2.2-llama3-70b", - "id": "MaziyarPanahi/calme-2.2-llama3-70b", - "developer": "MaziyarPanahi", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "70.554" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/MaziyarPanahi_calme-2.2-llama3-70b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8208 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6435 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2394 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3414 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4446 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5207 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/maziyarpanahi_calme-2.2-llama3.1-70b.json b/data/models/maziyarpanahi_calme-2.2-llama3.1-70b.json deleted file mode 100644 index 93513b9657f71cc17ea3942ae34318337f5dce17..0000000000000000000000000000000000000000 --- a/data/models/maziyarpanahi_calme-2.2-llama3.1-70b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "calme-2.2-llama3.1-70b", - "id": "MaziyarPanahi/calme-2.2-llama3.1-70b", - "developer": "MaziyarPanahi", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "70.554" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/MaziyarPanahi_calme-2.2-llama3.1-70b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8593 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6793 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4366 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3247 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4542 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5415 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/maziyarpanahi_calme-2.2-phi3-4b.json b/data/models/maziyarpanahi_calme-2.2-phi3-4b.json deleted file mode 100644 index ea9598833e0775568b6d7bc5cca66662d20ff946..0000000000000000000000000000000000000000 --- a/data/models/maziyarpanahi_calme-2.2-phi3-4b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "calme-2.2-phi3-4b", - "id": "MaziyarPanahi/calme-2.2-phi3-4b", - "developer": "MaziyarPanahi", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Phi3ForCausalLM", - "params_billions": "3.821" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/MaziyarPanahi_calme-2.2-phi3-4b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5069 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.553 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.145 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3213 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3976 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3814 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/maziyarpanahi_calme-2.2-qwen2-72b.json b/data/models/maziyarpanahi_calme-2.2-qwen2-72b.json deleted file mode 100644 index f517c845dda0c9b092ff18d67eb9f5dcf9352c35..0000000000000000000000000000000000000000 --- a/data/models/maziyarpanahi_calme-2.2-qwen2-72b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "calme-2.2-qwen2-72b", - "id": "MaziyarPanahi/calme-2.2-qwen2-72b", - "developer": "MaziyarPanahi", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "72.706" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/MaziyarPanahi_calme-2.2-qwen2-72b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8008 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.694 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4532 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3742 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4508 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5435 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/maziyarpanahi_calme-2.2-qwen2-7b.json b/data/models/maziyarpanahi_calme-2.2-qwen2-7b.json deleted file mode 100644 index 8d29f4a0c45f874e76d8b4216f3996d4ed85e064..0000000000000000000000000000000000000000 --- a/data/models/maziyarpanahi_calme-2.2-qwen2-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "calme-2.2-qwen2-7b", - "id": "MaziyarPanahi/calme-2.2-qwen2-7b", - "developer": "MaziyarPanahi", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/MaziyarPanahi_calme-2.2-qwen2-7b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3597 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5215 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2145 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2911 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4358 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3899 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/maziyarpanahi_calme-2.2-qwen2.5-72b.json b/data/models/maziyarpanahi_calme-2.2-qwen2.5-72b.json deleted file mode 100644 index 417df4992e518f6e20a27b2b96b9c99e0744bc46..0000000000000000000000000000000000000000 --- a/data/models/maziyarpanahi_calme-2.2-qwen2.5-72b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "calme-2.2-qwen2.5-72b", - "id": "MaziyarPanahi/calme-2.2-qwen2.5-72b", - "developer": "MaziyarPanahi", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "72.7" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/MaziyarPanahi_calme-2.2-qwen2.5-72b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8477 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7276 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5891 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3591 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4207 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5618 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/maziyarpanahi_calme-2.2-rys-78b.json b/data/models/maziyarpanahi_calme-2.2-rys-78b.json deleted file mode 100644 index a49f0450207b1b2c3fa16433597c7cadd9a4277d..0000000000000000000000000000000000000000 --- a/data/models/maziyarpanahi_calme-2.2-rys-78b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "calme-2.2-rys-78b", - "id": "MaziyarPanahi/calme-2.2-rys-78b", - "developer": "MaziyarPanahi", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "77.965" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/MaziyarPanahi_calme-2.2-rys-78b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7986 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7081 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4071 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4069 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4536 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5386 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/maziyarpanahi_calme-2.3-llama3-70b.json b/data/models/maziyarpanahi_calme-2.3-llama3-70b.json deleted file mode 100644 index ca87f31556db4c453f776f37902f0e58814327f7..0000000000000000000000000000000000000000 --- a/data/models/maziyarpanahi_calme-2.3-llama3-70b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "calme-2.3-llama3-70b", - "id": "MaziyarPanahi/calme-2.3-llama3-70b", - "developer": "MaziyarPanahi", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "70.554" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/MaziyarPanahi_calme-2.3-llama3-70b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.801 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6399 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2326 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3381 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4261 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5204 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/maziyarpanahi_calme-2.3-llama3.1-70b.json b/data/models/maziyarpanahi_calme-2.3-llama3.1-70b.json deleted file mode 100644 index 3b10e4fdfe61cc8a88cf3f7a0cbcbc52d8d1f99f..0000000000000000000000000000000000000000 --- a/data/models/maziyarpanahi_calme-2.3-llama3.1-70b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "calme-2.3-llama3.1-70b", - "id": "MaziyarPanahi/calme-2.3-llama3.1-70b", - "developer": "MaziyarPanahi", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "70.554" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/MaziyarPanahi_calme-2.3-llama3.1-70b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8605 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6872 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3927 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.344 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4568 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5363 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/maziyarpanahi_calme-2.3-phi3-4b.json b/data/models/maziyarpanahi_calme-2.3-phi3-4b.json deleted file mode 100644 index ae5a6713cbc1cb4f4721f065c21439d091bcaaf8..0000000000000000000000000000000000000000 --- a/data/models/maziyarpanahi_calme-2.3-phi3-4b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "calme-2.3-phi3-4b", - "id": "MaziyarPanahi/calme-2.3-phi3-4b", - "developer": "MaziyarPanahi", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Phi3ForCausalLM", - "params_billions": "3.821" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/MaziyarPanahi_calme-2.3-phi3-4b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4926 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5538 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1473 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.318 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3988 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3828 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/maziyarpanahi_calme-2.3-qwen2-72b.json b/data/models/maziyarpanahi_calme-2.3-qwen2-72b.json deleted file mode 100644 index b8e4e963447c08319aa3f604aadba647b4eb151c..0000000000000000000000000000000000000000 --- a/data/models/maziyarpanahi_calme-2.3-qwen2-72b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "calme-2.3-qwen2-72b", - "id": "MaziyarPanahi/calme-2.3-qwen2-72b", - "developer": "MaziyarPanahi", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "72.706" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/MaziyarPanahi_calme-2.3-qwen2-72b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.385 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6576 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3172 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3716 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4112 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5419 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/maziyarpanahi_calme-2.3-qwen2-7b.json b/data/models/maziyarpanahi_calme-2.3-qwen2-7b.json deleted file mode 100644 index 96ac3e8279c10e01b31758ddd33369eedb588ca8..0000000000000000000000000000000000000000 --- a/data/models/maziyarpanahi_calme-2.3-qwen2-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "calme-2.3-qwen2-7b", - "id": "MaziyarPanahi/calme-2.3-qwen2-7b", - "developer": "MaziyarPanahi", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/MaziyarPanahi_calme-2.3-qwen2-7b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3825 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5064 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2069 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.297 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4422 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3611 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/maziyarpanahi_calme-2.3-rys-78b.json b/data/models/maziyarpanahi_calme-2.3-rys-78b.json deleted file mode 100644 index ebc5f1710a7989b624c102e9c59ceff922847ebc..0000000000000000000000000000000000000000 --- a/data/models/maziyarpanahi_calme-2.3-rys-78b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "calme-2.3-rys-78b", - "id": "MaziyarPanahi/calme-2.3-rys-78b", - "developer": "MaziyarPanahi", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "77.965" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/MaziyarPanahi_calme-2.3-rys-78b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8066 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7108 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.398 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4044 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4549 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5475 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/maziyarpanahi_calme-2.4-llama3-70b.json b/data/models/maziyarpanahi_calme-2.4-llama3-70b.json deleted file mode 100644 index edd8d4a139447ca5de8b9e218474704b1742cc2a..0000000000000000000000000000000000000000 --- a/data/models/maziyarpanahi_calme-2.4-llama3-70b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "calme-2.4-llama3-70b", - "id": "MaziyarPanahi/calme-2.4-llama3-70b", - "developer": "MaziyarPanahi", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "70.554" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/MaziyarPanahi_calme-2.4-llama3-70b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5027 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6418 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2447 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3398 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4288 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5204 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/maziyarpanahi_calme-2.4-qwen2-7b.json b/data/models/maziyarpanahi_calme-2.4-qwen2-7b.json deleted file mode 100644 index 897a52ee028491c9589aea3b501d659af96c5918..0000000000000000000000000000000000000000 --- a/data/models/maziyarpanahi_calme-2.4-qwen2-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "calme-2.4-qwen2-7b", - "id": "MaziyarPanahi/calme-2.4-qwen2-7b", - "developer": "MaziyarPanahi", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/MaziyarPanahi_calme-2.4-qwen2-7b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.33 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5101 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2032 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2836 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4453 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3977 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/maziyarpanahi_calme-2.4-rys-78b.json b/data/models/maziyarpanahi_calme-2.4-rys-78b.json deleted file mode 100644 index 2a6761d8ee012ae3694ab7b910b8a0a8dea5e207..0000000000000000000000000000000000000000 --- a/data/models/maziyarpanahi_calme-2.4-rys-78b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "calme-2.4-rys-78b", - "id": "MaziyarPanahi/calme-2.4-rys-78b", - "developer": "MaziyarPanahi", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "77.965" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/MaziyarPanahi_calme-2.4-rys-78b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8011 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.728 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4071 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4027 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5771 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7002 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/maziyarpanahi_calme-2.5-qwen2-7b.json b/data/models/maziyarpanahi_calme-2.5-qwen2-7b.json deleted file mode 100644 index ae10b120a3160cf3b199b06e8452f2d056889d07..0000000000000000000000000000000000000000 --- a/data/models/maziyarpanahi_calme-2.5-qwen2-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "calme-2.5-qwen2-7b", - "id": "MaziyarPanahi/calme-2.5-qwen2-7b", - "developer": "MaziyarPanahi", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/MaziyarPanahi_calme-2.5-qwen2-7b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3145 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4887 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2258 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3104 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4565 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3682 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/maziyarpanahi_calme-2.6-qwen2-7b.json b/data/models/maziyarpanahi_calme-2.6-qwen2-7b.json deleted file mode 100644 index 5d9197b51c63247ed7f76a30180be656c48d9580..0000000000000000000000000000000000000000 --- a/data/models/maziyarpanahi_calme-2.6-qwen2-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "calme-2.6-qwen2-7b", - "id": "MaziyarPanahi/calme-2.6-qwen2-7b", - "developer": "MaziyarPanahi", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/MaziyarPanahi_calme-2.6-qwen2-7b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3443 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.493 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1216 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2844 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4586 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3732 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/maziyarpanahi_calme-2.7-qwen2-7b.json b/data/models/maziyarpanahi_calme-2.7-qwen2-7b.json deleted file mode 100644 index 38ddb8e41ccd26ae2ab8c86f0cbdd9ed6ff74193..0000000000000000000000000000000000000000 --- a/data/models/maziyarpanahi_calme-2.7-qwen2-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "calme-2.7-qwen2-7b", - "id": "MaziyarPanahi/calme-2.7-qwen2-7b", - "developer": "MaziyarPanahi", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/MaziyarPanahi_calme-2.7-qwen2-7b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3592 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4883 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1382 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2911 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4824 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3705 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/maziyarpanahi_calme-3.1-baguette-3b.json b/data/models/maziyarpanahi_calme-3.1-baguette-3b.json deleted file mode 100644 index dc47e8d1c0bc4c189ba7687f3d27e150e8116d83..0000000000000000000000000000000000000000 --- a/data/models/maziyarpanahi_calme-3.1-baguette-3b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "calme-3.1-baguette-3b", - "id": "MaziyarPanahi/calme-3.1-baguette-3b", - "developer": "MaziyarPanahi", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.085" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/MaziyarPanahi_calme-3.1-baguette-3b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6234 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4683 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.256 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2861 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4008 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3399 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/maziyarpanahi_calme-3.1-instruct-3b.json b/data/models/maziyarpanahi_calme-3.1-instruct-3b.json deleted file mode 100644 index 1fb319c10cb2b92dcf0e29203323ff71d6801a87..0000000000000000000000000000000000000000 --- a/data/models/maziyarpanahi_calme-3.1-instruct-3b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "calme-3.1-instruct-3b", - "id": "MaziyarPanahi/calme-3.1-instruct-3b", - "developer": "MaziyarPanahi", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.085" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/MaziyarPanahi_calme-3.1-instruct-3b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4336 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4813 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1775 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2861 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3952 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3557 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/maziyarpanahi_calme-3.1-instruct-78b.json b/data/models/maziyarpanahi_calme-3.1-instruct-78b.json deleted file mode 100644 index b43048df97ad7c225e82af96254fa89cc5f1137a..0000000000000000000000000000000000000000 --- a/data/models/maziyarpanahi_calme-3.1-instruct-78b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "calme-3.1-instruct-78b", - "id": "MaziyarPanahi/calme-3.1-instruct-78b", - "developer": "MaziyarPanahi", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "77.965" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/MaziyarPanahi_calme-3.1-instruct-78b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8136 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7305 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3927 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.396 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5891 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7185 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/maziyarpanahi_calme-3.1-llamaloi-3b.json b/data/models/maziyarpanahi_calme-3.1-llamaloi-3b.json deleted file mode 100644 index 6a4cd55fca28d603a7bdf46797f98610f5aacbb2..0000000000000000000000000000000000000000 --- a/data/models/maziyarpanahi_calme-3.1-llamaloi-3b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "calme-3.1-llamaloi-3b", - "id": "MaziyarPanahi/calme-3.1-llamaloi-3b", - "developer": "MaziyarPanahi", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/MaziyarPanahi_calme-3.1-llamaloi-3b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7375 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4587 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.173 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.281 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3515 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3205 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/maziyarpanahi_calme-3.2-baguette-3b.json b/data/models/maziyarpanahi_calme-3.2-baguette-3b.json deleted file mode 100644 index 256d9db6947d7ade169ac88847696a122ab3fa52..0000000000000000000000000000000000000000 --- a/data/models/maziyarpanahi_calme-3.2-baguette-3b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "calme-3.2-baguette-3b", - "id": "MaziyarPanahi/calme-3.2-baguette-3b", - "developer": "MaziyarPanahi", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.085" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/MaziyarPanahi_calme-3.2-baguette-3b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6338 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4709 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2825 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2945 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4021 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3338 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/maziyarpanahi_calme-3.2-instruct-3b.json b/data/models/maziyarpanahi_calme-3.2-instruct-3b.json deleted file mode 100644 index d48f5da730c087e3e8d63ca7d8300db1e6cb0e8d..0000000000000000000000000000000000000000 --- a/data/models/maziyarpanahi_calme-3.2-instruct-3b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "calme-3.2-instruct-3b", - "id": "MaziyarPanahi/calme-3.2-instruct-3b", - "developer": "MaziyarPanahi", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.086" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/MaziyarPanahi_calme-3.2-instruct-3b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5533 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4866 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2168 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2836 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4047 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3653 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/maziyarpanahi_calme-3.2-instruct-78b.json b/data/models/maziyarpanahi_calme-3.2-instruct-78b.json deleted file mode 100644 index fdad262afe52de3d50fc8d98137930fc9fe12cf9..0000000000000000000000000000000000000000 --- a/data/models/maziyarpanahi_calme-3.2-instruct-78b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "calme-3.2-instruct-78b", - "id": "MaziyarPanahi/calme-3.2-instruct-78b", - "developer": "MaziyarPanahi", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "77.965" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/MaziyarPanahi_calme-3.2-instruct-78b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8063 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7319 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4033 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4027 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6024 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7303 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/maziyarpanahi_calme-3.3-baguette-3b.json b/data/models/maziyarpanahi_calme-3.3-baguette-3b.json deleted file mode 100644 index 242a6f0ef97d14f9acfb6ec20c0b505a6820c1b7..0000000000000000000000000000000000000000 --- a/data/models/maziyarpanahi_calme-3.3-baguette-3b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "calme-3.3-baguette-3b", - "id": "MaziyarPanahi/calme-3.3-baguette-3b", - "developer": "MaziyarPanahi", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.086" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/MaziyarPanahi_calme-3.3-baguette-3b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.636 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4678 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3807 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2802 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3928 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3342 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/maziyarpanahi_calme-3.3-instruct-3b.json b/data/models/maziyarpanahi_calme-3.3-instruct-3b.json deleted file mode 100644 index f23949828f4883f357d2ea68d0123c268de3cd13..0000000000000000000000000000000000000000 --- a/data/models/maziyarpanahi_calme-3.3-instruct-3b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "calme-3.3-instruct-3b", - "id": "MaziyarPanahi/calme-3.3-instruct-3b", - "developer": "MaziyarPanahi", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.086" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/MaziyarPanahi_calme-3.3-instruct-3b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6423 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4693 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3739 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2827 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4074 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3305 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/maziyarpanahi_calme-4x7b-moe-v0.1.json b/data/models/maziyarpanahi_calme-4x7b-moe-v0.1.json deleted file mode 100644 index 72715a9fd2005c9d29c4efcca207ac5e710395f7..0000000000000000000000000000000000000000 --- a/data/models/maziyarpanahi_calme-4x7b-moe-v0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Calme-4x7B-MoE-v0.1", - "id": "MaziyarPanahi/Calme-4x7B-MoE-v0.1", - "developer": "MaziyarPanahi", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MixtralForCausalLM", - "params_billions": "24.154" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/MaziyarPanahi_Calme-4x7B-MoE-v0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4315 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5103 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0801 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2819 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4199 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3057 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/maziyarpanahi_calme-4x7b-moe-v0.2.json b/data/models/maziyarpanahi_calme-4x7b-moe-v0.2.json deleted file mode 100644 index 9582f571897ddbcdcbb7f00a99d2486fdce6869c..0000000000000000000000000000000000000000 --- a/data/models/maziyarpanahi_calme-4x7b-moe-v0.2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Calme-4x7B-MoE-v0.2", - "id": "MaziyarPanahi/Calme-4x7B-MoE-v0.2", - "developer": "MaziyarPanahi", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MixtralForCausalLM", - "params_billions": "24.154" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/MaziyarPanahi_Calme-4x7B-MoE-v0.2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4294 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5111 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.074 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2794 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4318 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3058 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/maziyarpanahi_llama-3-70b-instruct-v0.1.json b/data/models/maziyarpanahi_llama-3-70b-instruct-v0.1.json deleted file mode 100644 index 36a1d3b6b0079cab9330c492b44fe9a2079a5b45..0000000000000000000000000000000000000000 --- a/data/models/maziyarpanahi_llama-3-70b-instruct-v0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-70B-Instruct-v0.1", - "id": "MaziyarPanahi/Llama-3-70B-Instruct-v0.1", - "developer": "MaziyarPanahi", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "70.554" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/MaziyarPanahi_Llama-3-70B-Instruct-v0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4714 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5366 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1805 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2844 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4433 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4618 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/maziyarpanahi_llama-3-8b-instruct-v0.10.json b/data/models/maziyarpanahi_llama-3-8b-instruct-v0.10.json deleted file mode 100644 index aff296e06f595ef4e627e112ebdf6453c4d75fa9..0000000000000000000000000000000000000000 --- a/data/models/maziyarpanahi_llama-3-8b-instruct-v0.10.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-8B-Instruct-v0.10", - "id": "MaziyarPanahi/Llama-3-8B-Instruct-v0.10", - "developer": "MaziyarPanahi", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/MaziyarPanahi_Llama-3-8B-Instruct-v0.10/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7667 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4924 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0574 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3087 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4214 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3862 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/maziyarpanahi_llama-3-8b-instruct-v0.8.json b/data/models/maziyarpanahi_llama-3-8b-instruct-v0.8.json deleted file mode 100644 index 44ec8580213e577389ba5f8ec914ee66b4a226ba..0000000000000000000000000000000000000000 --- a/data/models/maziyarpanahi_llama-3-8b-instruct-v0.8.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-8B-Instruct-v0.8", - "id": "MaziyarPanahi/Llama-3-8B-Instruct-v0.8", - "developer": "MaziyarPanahi", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/MaziyarPanahi_Llama-3-8B-Instruct-v0.8/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7528 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4963 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0778 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3054 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4202 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3853 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/maziyarpanahi_llama-3-8b-instruct-v0.9.json b/data/models/maziyarpanahi_llama-3-8b-instruct-v0.9.json deleted file mode 100644 index f0aa9f2940efd47173c795792d4d53d6584629bf..0000000000000000000000000000000000000000 --- a/data/models/maziyarpanahi_llama-3-8b-instruct-v0.9.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-8B-Instruct-v0.9", - "id": "MaziyarPanahi/Llama-3-8B-Instruct-v0.9", - "developer": "MaziyarPanahi", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/MaziyarPanahi_Llama-3-8B-Instruct-v0.9/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.763 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4936 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0733 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3079 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4148 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3846 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/maziyarpanahi_qwen1.5-moe-a2.7b-wikihow.json b/data/models/maziyarpanahi_qwen1.5-moe-a2.7b-wikihow.json deleted file mode 100644 index 156804cd625654e283edc57d1934851c313a297b..0000000000000000000000000000000000000000 --- a/data/models/maziyarpanahi_qwen1.5-moe-a2.7b-wikihow.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen1.5-MoE-A2.7B-Wikihow", - "id": "MaziyarPanahi/Qwen1.5-MoE-A2.7B-Wikihow", - "developer": "MaziyarPanahi", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2MoeForCausalLM", - "params_billions": "14.316" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/MaziyarPanahi_Qwen1.5-MoE-A2.7B-Wikihow/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2954 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.392 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0823 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2752 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3502 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.238 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/maziyarpanahi_qwen2-7b-instruct-v0.1.json b/data/models/maziyarpanahi_qwen2-7b-instruct-v0.1.json deleted file mode 100644 index b914b4c26ac5dad697bb65735a90bacaffd5f8ee..0000000000000000000000000000000000000000 --- a/data/models/maziyarpanahi_qwen2-7b-instruct-v0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2-7B-Instruct-v0.1", - "id": "MaziyarPanahi/Qwen2-7B-Instruct-v0.1", - "developer": "MaziyarPanahi", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/MaziyarPanahi_Qwen2-7B-Instruct-v0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3352 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5123 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2213 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2852 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4435 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3857 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/maziyarpanahi_qwen2-7b-instruct-v0.8.json b/data/models/maziyarpanahi_qwen2-7b-instruct-v0.8.json deleted file mode 100644 index 724c955748035f7209a2406766eed055561df064..0000000000000000000000000000000000000000 --- a/data/models/maziyarpanahi_qwen2-7b-instruct-v0.8.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2-7B-Instruct-v0.8", - "id": "MaziyarPanahi/Qwen2-7B-Instruct-v0.8", - "developer": "MaziyarPanahi", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/MaziyarPanahi_Qwen2-7B-Instruct-v0.8/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2775 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4637 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1767 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2936 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4293 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3566 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/meditsolutions_llama-3.1-medit-sun-8b.json b/data/models/meditsolutions_llama-3.1-medit-sun-8b.json deleted file mode 100644 index 85276076b218c122bf73e5d8ac31557cc17a8b1f..0000000000000000000000000000000000000000 --- a/data/models/meditsolutions_llama-3.1-medit-sun-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.1-MedIT-SUN-8B", - "id": "meditsolutions/Llama-3.1-MedIT-SUN-8B", - "developer": "meditsolutions", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/meditsolutions_Llama-3.1-MedIT-SUN-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7837 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5187 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2092 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3087 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4056 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3916 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/meditsolutions_llama-3.2-sun-1b-chat.json b/data/models/meditsolutions_llama-3.2-sun-1b-chat.json deleted file mode 100644 index 459556ab2c2a93e9b3d656e5a70dbdeab12a5490..0000000000000000000000000000000000000000 --- a/data/models/meditsolutions_llama-3.2-sun-1b-chat.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.2-SUN-1B-chat", - "id": "meditsolutions/Llama-3.2-SUN-1B-chat", - "developer": "meditsolutions", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.498" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/meditsolutions_Llama-3.2-SUN-1B-chat/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5482 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3514 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0642 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2617 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3249 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1838 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/meditsolutions_llama-3.2-sun-1b-instruct.json b/data/models/meditsolutions_llama-3.2-sun-1b-instruct.json deleted file mode 100644 index 841f6df56c12ebbb0bbcfa4c6fed2e8f5c79bb90..0000000000000000000000000000000000000000 --- a/data/models/meditsolutions_llama-3.2-sun-1b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.2-SUN-1B-Instruct", - "id": "meditsolutions/Llama-3.2-SUN-1B-Instruct", - "developer": "meditsolutions", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaMedITForCausalLM", - "params_billions": "1.498" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/meditsolutions_Llama-3.2-SUN-1B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6413 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3474 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.071 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2424 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3514 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1781 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/meditsolutions_llama-3.2-sun-2.4b-checkpoint-26000.json b/data/models/meditsolutions_llama-3.2-sun-2.4b-checkpoint-26000.json deleted file mode 100644 index ce3138350576d904839f4f00c21c4e0666d8500a..0000000000000000000000000000000000000000 --- a/data/models/meditsolutions_llama-3.2-sun-2.4b-checkpoint-26000.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.2-SUN-2.4B-checkpoint-26000", - "id": "meditsolutions/Llama-3.2-SUN-2.4B-checkpoint-26000", - "developer": "meditsolutions", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "2.209" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/meditsolutions_Llama-3.2-SUN-2.4B-checkpoint-26000/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2814 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3018 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0181 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2777 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4103 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1345 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/meditsolutions_llama-3.2-sun-2.4b-checkpoint-34800.json b/data/models/meditsolutions_llama-3.2-sun-2.4b-checkpoint-34800.json deleted file mode 100644 index f5d0612579a8a6c1d9df5d929ae2aabcd68a6616..0000000000000000000000000000000000000000 --- a/data/models/meditsolutions_llama-3.2-sun-2.4b-checkpoint-34800.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.2-SUN-2.4B-checkpoint-34800", - "id": "meditsolutions/Llama-3.2-SUN-2.4B-checkpoint-34800", - "developer": "meditsolutions", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "2.209" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/meditsolutions_Llama-3.2-SUN-2.4B-checkpoint-34800/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2501 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3161 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0106 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2861 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4022 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1357 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/meditsolutions_llama-3.2-sun-2.4b-v1.0.0.json b/data/models/meditsolutions_llama-3.2-sun-2.4b-v1.0.0.json deleted file mode 100644 index 9c78e3fa1ea95ea2b26311aaa038c9d56c92f557..0000000000000000000000000000000000000000 --- a/data/models/meditsolutions_llama-3.2-sun-2.4b-v1.0.0.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.2-SUN-2.4B-v1.0.0", - "id": "meditsolutions/Llama-3.2-SUN-2.4B-v1.0.0", - "developer": "meditsolutions", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "2.472" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/meditsolutions_Llama-3.2-SUN-2.4B-v1.0.0/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5637 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3391 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0627 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2576 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3209 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1543 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/meditsolutions_llama-3.2-sun-2.5b-chat.json b/data/models/meditsolutions_llama-3.2-sun-2.5b-chat.json deleted file mode 100644 index 9d62c826d9bd4796be86c2bc380e20041dedb8fa..0000000000000000000000000000000000000000 --- a/data/models/meditsolutions_llama-3.2-sun-2.5b-chat.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.2-SUN-2.5B-chat", - "id": "meditsolutions/Llama-3.2-SUN-2.5B-chat", - "developer": "meditsolutions", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "2.472" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/meditsolutions_Llama-3.2-SUN-2.5B-chat/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5604 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3575 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.071 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2592 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3155 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1813 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/meditsolutions_llama-3.2-sun-hdic-1b-instruct.json b/data/models/meditsolutions_llama-3.2-sun-hdic-1b-instruct.json deleted file mode 100644 index 68dc44a7fd58e48280bb866a124067b3bfb228ea..0000000000000000000000000000000000000000 --- a/data/models/meditsolutions_llama-3.2-sun-hdic-1b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.2-SUN-HDIC-1B-Instruct", - "id": "meditsolutions/Llama-3.2-SUN-HDIC-1B-Instruct", - "developer": "meditsolutions", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.498" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/meditsolutions_Llama-3.2-SUN-HDIC-1B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6827 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3508 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0619 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2366 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3594 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1687 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/meditsolutions_medit-mesh-3b-instruct.json b/data/models/meditsolutions_medit-mesh-3b-instruct.json deleted file mode 100644 index 00d1052fcd54a21423951c33a84cac2f80e8cdc8..0000000000000000000000000000000000000000 --- a/data/models/meditsolutions_medit-mesh-3b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MedIT-Mesh-3B-Instruct", - "id": "meditsolutions/MedIT-Mesh-3B-Instruct", - "developer": "meditsolutions", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Phi3ForCausalLM", - "params_billions": "3.821" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/meditsolutions_MedIT-Mesh-3B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5814 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5576 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2032 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3238 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4048 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4012 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/meditsolutions_msh-lite-7b-v1-bielik-v2.3-instruct-llama-prune.json b/data/models/meditsolutions_msh-lite-7b-v1-bielik-v2.3-instruct-llama-prune.json deleted file mode 100644 index 415d433a634697b3be617c9ca26c0762687019bd..0000000000000000000000000000000000000000 --- a/data/models/meditsolutions_msh-lite-7b-v1-bielik-v2.3-instruct-llama-prune.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MSH-Lite-7B-v1-Bielik-v2.3-Instruct-Llama-Prune", - "id": "meditsolutions/MSH-Lite-7B-v1-Bielik-v2.3-Instruct-Llama-Prune", - "developer": "meditsolutions", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.646" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/meditsolutions_MSH-Lite-7B-v1-Bielik-v2.3-Instruct-Llama-Prune/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3655 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4035 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0264 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3029 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4253 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.219 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/meditsolutions_msh-v1-bielik-v2.3-instruct-medit-merge.json b/data/models/meditsolutions_msh-v1-bielik-v2.3-instruct-medit-merge.json deleted file mode 100644 index 7626a7922e97351fbc9217c37ad8d071148cf099..0000000000000000000000000000000000000000 --- a/data/models/meditsolutions_msh-v1-bielik-v2.3-instruct-medit-merge.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MSH-v1-Bielik-v2.3-Instruct-MedIT-merge", - "id": "meditsolutions/MSH-v1-Bielik-v2.3-Instruct-MedIT-merge", - "developer": "meditsolutions", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "11.169" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/meditsolutions_MSH-v1-Bielik-v2.3-Instruct-MedIT-merge/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5814 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5672 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2077 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3456 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4385 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.35 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/meditsolutions_smollm2-medit-upscale-2b.json b/data/models/meditsolutions_smollm2-medit-upscale-2b.json deleted file mode 100644 index 932af82bb11017400e2ff90f1bc1b66432ec7173..0000000000000000000000000000000000000000 --- a/data/models/meditsolutions_smollm2-medit-upscale-2b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SmolLM2-MedIT-Upscale-2B", - "id": "meditsolutions/SmolLM2-MedIT-Upscale-2B", - "developer": "meditsolutions", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "2.114" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/meditsolutions_SmolLM2-MedIT-Upscale-2B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6429 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3551 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0559 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2643 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3314 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1971 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/meetkai_functionary-small-v3.1.json b/data/models/meetkai_functionary-small-v3.1.json deleted file mode 100644 index 1c3d5ad0c0f796f270361a36c8bf33eed7861dfa..0000000000000000000000000000000000000000 --- a/data/models/meetkai_functionary-small-v3.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "functionary-small-v3.1", - "id": "meetkai/functionary-small-v3.1", - "developer": "meetkai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/meetkai_functionary-small-v3.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6275 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4982 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1571 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2886 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3834 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3349 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/meragpt_mera-mix-4x7b.json b/data/models/meragpt_mera-mix-4x7b.json deleted file mode 100644 index ae4413b76eadce930fd696b93eb9a79b40e0bc55..0000000000000000000000000000000000000000 --- a/data/models/meragpt_mera-mix-4x7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "mera-mix-4x7B", - "id": "meraGPT/mera-mix-4x7B", - "developer": "meraGPT", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MixtralForCausalLM", - "params_billions": "24.154" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/meraGPT_mera-mix-4x7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4832 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4019 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0536 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3045 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4057 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2748 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/mergekit-community_diabolic6045_eln-aoc-cain.json b/data/models/mergekit-community_diabolic6045_eln-aoc-cain.json deleted file mode 100644 index 9f6ce9dc4eca0425195caff3e08cb0d785f48aaf..0000000000000000000000000000000000000000 --- a/data/models/mergekit-community_diabolic6045_eln-aoc-cain.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "diabolic6045_ELN-AOC-CAIN", - "id": "mergekit-community/diabolic6045_ELN-AOC-CAIN", - "developer": "mergekit-community", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.236" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/mergekit-community_diabolic6045_ELN-AOC-CAIN/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0862 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3126 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0121 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2634 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3658 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1191 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/mergekit-community_jajuka-wewillneverforgetyou-3b.json b/data/models/mergekit-community_jajuka-wewillneverforgetyou-3b.json deleted file mode 100644 index e1050207bdb1f06ef1dd02b4820c6ac5586954e3..0000000000000000000000000000000000000000 --- a/data/models/mergekit-community_jajuka-wewillneverforgetyou-3b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "JAJUKA-WEWILLNEVERFORGETYOU-3B", - "id": "mergekit-community/JAJUKA-WEWILLNEVERFORGETYOU-3B", - "developer": "mergekit-community", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/mergekit-community_JAJUKA-WEWILLNEVERFORGETYOU-3B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4941 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.437 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1246 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2928 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3656 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3033 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/mergekit-community_mergekit-dare_ties-ajgjgea.json b/data/models/mergekit-community_mergekit-dare_ties-ajgjgea.json deleted file mode 100644 index 1ff4d668442f09c95cf59ee558b5c07e8adfa3d5..0000000000000000000000000000000000000000 --- a/data/models/mergekit-community_mergekit-dare_ties-ajgjgea.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "mergekit-dare_ties-ajgjgea", - "id": "mergekit-community/mergekit-dare_ties-ajgjgea", - "developer": "mergekit-community", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.498" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/mergekit-community_mergekit-dare_ties-ajgjgea/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5263 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3495 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0642 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2643 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3289 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1744 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/mergekit-community_mergekit-della-zgowfmf.json b/data/models/mergekit-community_mergekit-della-zgowfmf.json deleted file mode 100644 index 0bd03879c80ae5dc970c6f5be3fb73a5d5ad90b6..0000000000000000000000000000000000000000 --- a/data/models/mergekit-community_mergekit-della-zgowfmf.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "mergekit-della-zgowfmf", - "id": "mergekit-community/mergekit-della-zgowfmf", - "developer": "mergekit-community", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/mergekit-community_mergekit-della-zgowfmf/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4828 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6591 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3618 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3901 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4834 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5415 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/mergekit-community_mergekit-model_stock-azgztvm.json b/data/models/mergekit-community_mergekit-model_stock-azgztvm.json deleted file mode 100644 index d37541be922281e85330dbb22aed3d73285e3296..0000000000000000000000000000000000000000 --- a/data/models/mergekit-community_mergekit-model_stock-azgztvm.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "mergekit-model_stock-azgztvm", - "id": "mergekit-community/mergekit-model_stock-azgztvm", - "developer": "mergekit-community", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/mergekit-community_mergekit-model_stock-azgztvm/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5062 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6543 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4373 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3817 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.473 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5406 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/mergekit-community_mergekit-slerp-fmrazcr.json b/data/models/mergekit-community_mergekit-slerp-fmrazcr.json deleted file mode 100644 index 4a55a5fb44891aad6fcfc58036f99b909aec4595..0000000000000000000000000000000000000000 --- a/data/models/mergekit-community_mergekit-slerp-fmrazcr.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "mergekit-slerp-fmrazcr", - "id": "mergekit-community/mergekit-slerp-fmrazcr", - "developer": "mergekit-community", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/mergekit-community_mergekit-slerp-fmrazcr/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4174 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5342 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1193 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3112 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4105 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3777 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/mergekit-community_mergekit-ties-rraxdhv.json b/data/models/mergekit-community_mergekit-ties-rraxdhv.json deleted file mode 100644 index 8e825928d9c3892fbbc1644c53f78e5f536bc93b..0000000000000000000000000000000000000000 --- a/data/models/mergekit-community_mergekit-ties-rraxdhv.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "mergekit-ties-rraxdhv", - "id": "mergekit-community/mergekit-ties-rraxdhv", - "developer": "mergekit-community", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "9.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/mergekit-community_mergekit-ties-rraxdhv/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1123 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5184 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.04 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3079 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4202 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.391 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/mergekit-community_mergekit-ties-ykqemwr.json b/data/models/mergekit-community_mergekit-ties-ykqemwr.json deleted file mode 100644 index f79facb5cc298ccba1b7a0699172eee7c4d4381a..0000000000000000000000000000000000000000 --- a/data/models/mergekit-community_mergekit-ties-ykqemwr.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "mergekit-ties-ykqemwr", - "id": "mergekit-community/mergekit-ties-ykqemwr", - "developer": "mergekit-community", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/mergekit-community_mergekit-ties-ykqemwr/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.36 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5455 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1224 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3221 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4198 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3734 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/mergekit-community_sexeh_time_testing.json b/data/models/mergekit-community_sexeh_time_testing.json deleted file mode 100644 index 0884c27979a6c91d41c2467334c513f80e3f70ef..0000000000000000000000000000000000000000 --- a/data/models/mergekit-community_sexeh_time_testing.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "sexeh_time_testing", - "id": "mergekit-community/sexeh_time_testing", - "developer": "mergekit-community", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/mergekit-community_sexeh_time_testing/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7329 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5241 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0899 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2911 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3619 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3667 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/mergekit-community_superqwen-2.5-1.5b.json b/data/models/mergekit-community_superqwen-2.5-1.5b.json deleted file mode 100644 index 3c2da172902163da5179db8c2edfd3105655ad7e..0000000000000000000000000000000000000000 --- a/data/models/mergekit-community_superqwen-2.5-1.5b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SuperQwen-2.5-1.5B", - "id": "mergekit-community/SuperQwen-2.5-1.5B", - "developer": "mergekit-community", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.777" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/mergekit-community_SuperQwen-2.5-1.5B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1336 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2907 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0196 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2542 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3355 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1075 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/mergekit-community_virtuososmall-instructmodelstock.json b/data/models/mergekit-community_virtuososmall-instructmodelstock.json deleted file mode 100644 index ef0d0f2f73b45da07719d48ad29ca4962ef27d4d..0000000000000000000000000000000000000000 --- a/data/models/mergekit-community_virtuososmall-instructmodelstock.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "VirtuosoSmall-InstructModelStock", - "id": "mergekit-community/VirtuosoSmall-InstructModelStock", - "developer": "mergekit-community", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/mergekit-community_VirtuosoSmall-InstructModelStock/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5238 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6518 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4094 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3826 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4756 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5421 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/mescriva_ece-prymmal-0.5b-ft-v5-musr-mathis.json b/data/models/mescriva_ece-prymmal-0.5b-ft-v5-musr-mathis.json deleted file mode 100644 index 55a33f9a601eaf9f144d7f37b67ce5b371b04596..0000000000000000000000000000000000000000 --- a/data/models/mescriva_ece-prymmal-0.5b-ft-v5-musr-mathis.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ECE-PRYMMAL-0.5B-FT-V5-MUSR-Mathis", - "id": "MEscriva/ECE-PRYMMAL-0.5B-FT-V5-MUSR-Mathis", - "developer": "MEscriva", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "?", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/MEscriva_ECE-PRYMMAL-0.5B-FT-V5-MUSR-Mathis/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0866 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3057 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0106 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2517 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4017 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1154 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/meta-llama_llama-2-13b-chat-hf.json b/data/models/meta-llama_llama-2-13b-chat-hf.json deleted file mode 100644 index 0725463cde99532fd620166901d86a3a597f71bf..0000000000000000000000000000000000000000 --- a/data/models/meta-llama_llama-2-13b-chat-hf.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-2-13b-chat-hf", - "id": "meta-llama/Llama-2-13b-chat-hf", - "developer": "meta-llama", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "13.016" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/meta-llama_Llama-2-13b-chat-hf/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3985 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3343 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0136 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2315 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4007 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1923 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/meta-llama_llama-2-13b-hf.json b/data/models/meta-llama_llama-2-13b-hf.json deleted file mode 100644 index 40ce559122403fd1d107dc8a819f0cbc85af6737..0000000000000000000000000000000000000000 --- a/data/models/meta-llama_llama-2-13b-hf.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-2-13b-hf", - "id": "meta-llama/Llama-2-13b-hf", - "developer": "meta-llama", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "13.016" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/meta-llama_Llama-2-13b-hf/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2482 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4126 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0151 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.281 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3538 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2378 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/meta-llama_llama-2-70b-chat-hf.json b/data/models/meta-llama_llama-2-70b-chat-hf.json deleted file mode 100644 index c3449a12fb51d83763e0e26a3361277e0e035afc..0000000000000000000000000000000000000000 --- a/data/models/meta-llama_llama-2-70b-chat-hf.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-2-70b-chat-hf", - "id": "meta-llama/Llama-2-70b-chat-hf", - "developer": "meta-llama", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "68.977" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/meta-llama_Llama-2-70b-chat-hf/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4958 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3042 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0295 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2643 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3687 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2433 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/meta-llama_llama-2-70b-hf.json b/data/models/meta-llama_llama-2-70b-hf.json deleted file mode 100644 index 827f0c860e66336bbc93252e76387c643ec7b5be..0000000000000000000000000000000000000000 --- a/data/models/meta-llama_llama-2-70b-hf.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-2-70b-hf", - "id": "meta-llama/Llama-2-70b-hf", - "developer": "meta-llama", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "68.977" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/meta-llama_Llama-2-70b-hf/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2407 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5473 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0325 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3029 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4124 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3718 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/meta-llama_llama-2-7b-chat-hf.json b/data/models/meta-llama_llama-2-7b-chat-hf.json deleted file mode 100644 index 3a5551131743254cc10681e8a50eb39e5f54b684..0000000000000000000000000000000000000000 --- a/data/models/meta-llama_llama-2-7b-chat-hf.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-2-7b-chat-hf", - "id": "meta-llama/Llama-2-7b-chat-hf", - "developer": "meta-llama", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "6.738" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/meta-llama_Llama-2-7b-chat-hf/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3986 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3114 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0196 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2534 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3676 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1688 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/meta-llama_llama-2-7b-hf.json b/data/models/meta-llama_llama-2-7b-hf.json deleted file mode 100644 index b22f631c0d0239152ca729566f670986fb36ea45..0000000000000000000000000000000000000000 --- a/data/models/meta-llama_llama-2-7b-hf.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-2-7b-hf", - "id": "meta-llama/Llama-2-7b-hf", - "developer": "meta-llama", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "6.738" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/meta-llama_Llama-2-7b-hf/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2519 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3496 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0174 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2668 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3701 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1861 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/meta-llama_llama-3.1-70b-instruct.json b/data/models/meta-llama_llama-3.1-70b-instruct.json deleted file mode 100644 index a0c95509c1f7eedc66a7245c93384fa742f30fc7..0000000000000000000000000000000000000000 --- a/data/models/meta-llama_llama-3.1-70b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.1-70B-Instruct", - "id": "meta-llama/Llama-3.1-70B-Instruct", - "developer": "meta-llama", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "70.554" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/meta-llama_Llama-3.1-70B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8669 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6917 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3807 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3565 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4581 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5309 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/meta-llama_llama-3.1-70b.json b/data/models/meta-llama_llama-3.1-70b.json deleted file mode 100644 index c9a7adaa27057c79ff3b7dbf29285ea5e6b1908c..0000000000000000000000000000000000000000 --- a/data/models/meta-llama_llama-3.1-70b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.1-70B", - "id": "meta-llama/Llama-3.1-70B", - "developer": "meta-llama", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "70.554" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/meta-llama_Llama-3.1-70B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1684 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.626 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1843 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3876 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4572 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4654 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/meta-llama_llama-3.1-8b-instruct.json b/data/models/meta-llama_llama-3.1-8b-instruct.json deleted file mode 100644 index 98b08e9b3e93dbc10c505df145e63aef056942d3..0000000000000000000000000000000000000000 --- a/data/models/meta-llama_llama-3.1-8b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.1-8B-Instruct", - "id": "meta-llama/Llama-3.1-8B-Instruct", - "developer": "meta-llama", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/meta-llama_Llama-3.1-8B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4922 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5087 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1556 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3154 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3972 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3798 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/meta-llama_llama-3.1-8b.json b/data/models/meta-llama_llama-3.1-8b.json deleted file mode 100644 index 98867f2497e21f027efbdf974da6e59da0550e1e..0000000000000000000000000000000000000000 --- a/data/models/meta-llama_llama-3.1-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.1-8B", - "id": "meta-llama/Llama-3.1-8B", - "developer": "meta-llama", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/meta-llama_Llama-3.1-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1246 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.466 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0657 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3104 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3812 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3288 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/meta-llama_llama-3.2-1b-instruct.json b/data/models/meta-llama_llama-3.2-1b-instruct.json deleted file mode 100644 index b99a4a8fc0c04decd355ae4a38c1278f09553c80..0000000000000000000000000000000000000000 --- a/data/models/meta-llama_llama-3.2-1b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.2-1B-Instruct", - "id": "meta-llama/Llama-3.2-1B-Instruct", - "developer": "meta-llama", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.24" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/meta-llama_Llama-3.2-1B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5698 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3497 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0702 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2752 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3329 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1682 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/meta-llama_llama-3.2-1b.json b/data/models/meta-llama_llama-3.2-1b.json deleted file mode 100644 index c3f07c5630224e7e827f9bf0e84c333ce19c0ccf..0000000000000000000000000000000000000000 --- a/data/models/meta-llama_llama-3.2-1b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.2-1B", - "id": "meta-llama/Llama-3.2-1B", - "developer": "meta-llama", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.24" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/meta-llama_Llama-3.2-1B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1478 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3115 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0121 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2282 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3447 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1203 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/meta-llama_llama-3.2-3b-instruct.json b/data/models/meta-llama_llama-3.2-3b-instruct.json deleted file mode 100644 index 6d528777be7433a7908092315118bc25c58479ef..0000000000000000000000000000000000000000 --- a/data/models/meta-llama_llama-3.2-3b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.2-3B-Instruct", - "id": "meta-llama/Llama-3.2-3B-Instruct", - "developer": "meta-llama", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/meta-llama_Llama-3.2-3B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7393 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.461 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1767 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2785 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3529 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3195 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/meta-llama_llama-3.2-3b.json b/data/models/meta-llama_llama-3.2-3b.json deleted file mode 100644 index b44f3d0cba0ea61ef9e3b7e3a4d72fca52f280a3..0000000000000000000000000000000000000000 --- a/data/models/meta-llama_llama-3.2-3b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.2-3B", - "id": "meta-llama/Llama-3.2-3B", - "developer": "meta-llama", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/meta-llama_Llama-3.2-3B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1337 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3905 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0189 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2676 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3577 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2488 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/meta-llama_llama-3.3-70b-instruct.json b/data/models/meta-llama_llama-3.3-70b-instruct.json deleted file mode 100644 index 8d538a1539a40336f1947f51784ab4a7a2502cc6..0000000000000000000000000000000000000000 --- a/data/models/meta-llama_llama-3.3-70b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.3-70B-Instruct", - "id": "meta-llama/Llama-3.3-70B-Instruct", - "developer": "meta-llama", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "70.554" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/meta-llama_Llama-3.3-70B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8998 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6919 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4834 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3289 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4461 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5332 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/meta-llama_meta-llama-3-70b-instruct.json b/data/models/meta-llama_meta-llama-3-70b-instruct.json deleted file mode 100644 index d60a4a9e2166be9b02268fccb9ab420065ce27c4..0000000000000000000000000000000000000000 --- a/data/models/meta-llama_meta-llama-3-70b-instruct.json +++ /dev/null @@ -1,277 +0,0 @@ -{ - "model_info": { - "name": "Meta-Llama-3-70B-Instruct", - "id": "meta-llama/Meta-Llama-3-70B-Instruct", - "developer": "meta-llama", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "70.554" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/meta-llama_Meta-Llama-3-70B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8099 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6547 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2447 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2869 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4154 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5207 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "reward-bench/meta-llama_Meta-Llama-3-70B-Instruct/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7627 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9763 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5888 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7297 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7854 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7035 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/meta-llama_meta-llama-3-70b.json b/data/models/meta-llama_meta-llama-3-70b.json deleted file mode 100644 index 42f6283ef5808abd679990ccae29c9bfcd79065c..0000000000000000000000000000000000000000 --- a/data/models/meta-llama_meta-llama-3-70b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Meta-Llama-3-70B", - "id": "meta-llama/Meta-Llama-3-70B", - "developer": "meta-llama", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "70.554" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/meta-llama_Meta-Llama-3-70B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1603 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6461 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1858 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3977 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4518 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4709 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/meta-llama_meta-llama-3-8b-instruct.json b/data/models/meta-llama_meta-llama-3-8b-instruct.json deleted file mode 100644 index 6b360a3e795c08b36f8a5ae66db62ef8475fd282..0000000000000000000000000000000000000000 --- a/data/models/meta-llama_meta-llama-3-8b-instruct.json +++ /dev/null @@ -1,407 +0,0 @@ -{ - "model_info": { - "name": "Meta-Llama-3-8B-Instruct", - "id": "meta-llama/Meta-Llama-3-8B-Instruct", - "developer": "meta-llama", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/meta-llama_Meta-Llama-3-8B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4782 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.491 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0914 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2928 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3805 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3591 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "hfopenllm_v2/meta-llama_Meta-Llama-3-8B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7408 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4989 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0869 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2592 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3568 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3664 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "reward-bench/meta-llama_Meta-Llama-3-8B-Instruct/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.645 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8547 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4156 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6797 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6482 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6082 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/meta-llama_meta-llama-3-8b.json b/data/models/meta-llama_meta-llama-3-8b.json deleted file mode 100644 index 5fa5e843e0c0e3e178bcf9562db364f1ab5d08bb..0000000000000000000000000000000000000000 --- a/data/models/meta-llama_meta-llama-3-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Meta-Llama-3-8B", - "id": "meta-llama/Meta-Llama-3-8B", - "developer": "meta-llama", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/meta-llama_Meta-Llama-3-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1455 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4598 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0453 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3054 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3614 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.321 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/meta-llama_meta-llama-3.1-405b-instruct-turbo.json b/data/models/meta-llama_meta-llama-3.1-405b-instruct-turbo.json deleted file mode 100644 index b2568499e7196a0f2081a94d73c93138acaf1f36..0000000000000000000000000000000000000000 --- a/data/models/meta-llama_meta-llama-3.1-405b-instruct-turbo.json +++ /dev/null @@ -1,126 +0,0 @@ -{ - "model_info": { - "name": "meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo", - "id": "meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo", - "developer": "meta-llama", - "additional_details": { - "model_type": "Generative" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/meta-llama_Meta-Llama-3.1-405B-Instruct-Turbo/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8412 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9721 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7456 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7757 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8715 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/meta-llama_meta-llama-3.1-70b-instruct-turbo.json b/data/models/meta-llama_meta-llama-3.1-70b-instruct-turbo.json deleted file mode 100644 index 5a3a44a41eeed3603608fd0086caa6160a298442..0000000000000000000000000000000000000000 --- a/data/models/meta-llama_meta-llama-3.1-70b-instruct-turbo.json +++ /dev/null @@ -1,126 +0,0 @@ -{ - "model_info": { - "name": "meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo", - "id": "meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo", - "developer": "meta-llama", - "additional_details": { - "model_type": "Generative" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/meta-llama_Meta-Llama-3.1-70B-Instruct-Turbo/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7808 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8757 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6689 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7507 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.828 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/meta-llama_meta-llama-3.1-70b-instruct.json b/data/models/meta-llama_meta-llama-3.1-70b-instruct.json deleted file mode 100644 index 39409ca010d73105b89b3634e3f2e1b00173a824..0000000000000000000000000000000000000000 --- a/data/models/meta-llama_meta-llama-3.1-70b-instruct.json +++ /dev/null @@ -1,126 +0,0 @@ -{ - "model_info": { - "name": "meta-llama/Meta-Llama-3.1-70B-Instruct", - "id": "meta-llama/Meta-Llama-3.1-70B-Instruct", - "developer": "meta-llama", - "additional_details": { - "model_type": "Generative" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/meta-llama_Meta-Llama-3.1-70B-Instruct/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8405 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9721 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7018 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8284 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8599 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/meta-llama_meta-llama-3.1-8b-instruct-turbo.json b/data/models/meta-llama_meta-llama-3.1-8b-instruct-turbo.json deleted file mode 100644 index 37d43f9709c5043b6048dfe8b944d9bb067fc2ba..0000000000000000000000000000000000000000 --- a/data/models/meta-llama_meta-llama-3.1-8b-instruct-turbo.json +++ /dev/null @@ -1,126 +0,0 @@ -{ - "model_info": { - "name": "meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo", - "id": "meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo", - "developer": "meta-llama", - "additional_details": { - "model_type": "Generative" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/meta-llama_Meta-Llama-3.1-8B-Instruct-Turbo/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6565 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8073 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4978 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6399 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6811 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/meta-llama_meta-llama-3.1-8b-instruct.json b/data/models/meta-llama_meta-llama-3.1-8b-instruct.json deleted file mode 100644 index 2e86cfb6a810e6b1d7164056a5ea21537f5bb762..0000000000000000000000000000000000000000 --- a/data/models/meta-llama_meta-llama-3.1-8b-instruct.json +++ /dev/null @@ -1,49 +0,0 @@ -{ - "model_info": { - "name": "Meta Llama 3.1 8B Instruct", - "id": "meta-llama/Meta-Llama-3.1-8B-Instruct", - "developer": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "la_leaderboard/meta-llama/Meta-Llama-3.1-8B-Instruct/1774451270", - "retrieved_timestamp": "2024-10-27T00:00:00Z", - "source_metadata": { - "source_name": "La Leaderboard", - "source_type": "evaluation_run", - "source_url": "https://huggingface.co/spaces/la-leaderboard/la-leaderboard", - "source_organization_name": "La Leaderboard", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "custom", - "version": "1.0" - }, - "benchmark": "la_leaderboard", - "evaluation_results": [ - { - "evaluation_name": "la_leaderboard", - "metric_config": { - "evaluation_description": "La Leaderboard: LLM evaluation for Spanish varieties and languages of Spain and Latin America", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 100 - }, - "score_details": { - "score": 30.23 - }, - "source_data": { - "source_type": "url", - "dataset_name": "La Leaderboard composite dataset", - "url": [ - "https://huggingface.co/spaces/la-leaderboard/la-leaderboard" - ] - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/meta-llama_meta-llama-3.1-8b.json b/data/models/meta-llama_meta-llama-3.1-8b.json deleted file mode 100644 index e0736feccce1fe2ca3631c5893d8bd446ee1f427..0000000000000000000000000000000000000000 --- a/data/models/meta-llama_meta-llama-3.1-8b.json +++ /dev/null @@ -1,49 +0,0 @@ -{ - "model_info": { - "name": "Meta Llama 3.1 8B", - "id": "meta-llama/Meta-Llama-3.1-8B", - "developer": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "la_leaderboard/meta-llama/Meta-Llama-3.1-8B/1774451270", - "retrieved_timestamp": "2024-10-27T00:00:00Z", - "source_metadata": { - "source_name": "La Leaderboard", - "source_type": "evaluation_run", - "source_url": "https://huggingface.co/spaces/la-leaderboard/la-leaderboard", - "source_organization_name": "La Leaderboard", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "custom", - "version": "1.0" - }, - "benchmark": "la_leaderboard", - "evaluation_results": [ - { - "evaluation_name": "la_leaderboard", - "metric_config": { - "evaluation_description": "La Leaderboard: LLM evaluation for Spanish varieties and languages of Spain and Latin America", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 100 - }, - "score_details": { - "score": 27.04 - }, - "source_data": { - "source_type": "url", - "dataset_name": "La Leaderboard composite dataset", - "url": [ - "https://huggingface.co/spaces/la-leaderboard/la-leaderboard" - ] - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/meta-metrics_metametrics-rm-v1.0.json b/data/models/meta-metrics_metametrics-rm-v1.0.json deleted file mode 100644 index 2cc31bb6fcf8e50c0f44234199d69716bb3099c9..0000000000000000000000000000000000000000 --- a/data/models/meta-metrics_metametrics-rm-v1.0.json +++ /dev/null @@ -1,126 +0,0 @@ -{ - "model_info": { - "name": "meta-metrics/MetaMetrics-RM-v1.0", - "id": "meta-metrics/MetaMetrics-RM-v1.0", - "developer": "meta-metrics", - "additional_details": { - "model_type": "Custom Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/meta-metrics_MetaMetrics-RM-v1.0/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9342 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9832 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.864 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9081 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9816 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/meta_llama-13b.json b/data/models/meta_llama-13b.json deleted file mode 100644 index ca6092c8def98fccdb723ebec617475723452a89..0000000000000000000000000000000000000000 --- a/data/models/meta_llama-13b.json +++ /dev/null @@ -1,674 +0,0 @@ -{ - "model_info": { - "name": "LLaMA 13B", - "id": "meta/LLaMA-13B", - "developer": "Meta", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_classic/meta_LLaMA-13B/1774096308.339228", - "retrieved_timestamp": "1774096308.339228", - "source_metadata": { - "source_name": "helm_classic", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_classic", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_classic", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperform on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.595, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Calibration": "{\"description\": \"\", \"tab\": \"Calibration\", \"score\": \"\"}", - "Mean win rate - Robustness": "{\"description\": \"\", \"tab\": \"Robustness\", \"score\": \"0.6374592074592075\"}", - "Mean win rate - Fairness": "{\"description\": \"\", \"tab\": \"Fairness\", \"score\": \"0.6022144522144522\"}", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}", - "Mean win rate - Bias": "{\"description\": \"\", \"tab\": \"Bias\", \"score\": \"0.5777177774710669\"}", - "Mean win rate - Toxicity": "{\"description\": \"\", \"tab\": \"Toxicity\", \"score\": \"0.6102564102564103\"}", - "Mean win rate - Summarization metrics": "{\"description\": \"\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.422, - "details": { - "description": "min=0.2, mean=0.422, max=0.76, sum=2.111 (5)", - "tab": "Accuracy", - "MMLU - ECE (10-bin)": "{\"description\": \"min=0.127, mean=0.15, max=0.18, sum=0.748 (5)\", \"tab\": \"Calibration\", \"score\": \"\"}", - "MMLU - EM (Robustness)": "{\"description\": \"min=0.14, mean=0.37, max=0.68, sum=1.848 (5)\", \"tab\": \"Robustness\", \"score\": \"0.3696140350877193\"}", - "MMLU - EM (Fairness)": "{\"description\": \"min=0.18, mean=0.385, max=0.71, sum=1.927 (5)\", \"tab\": \"Fairness\", \"score\": \"0.3853684210526316\"}", - "MMLU - Denoised inference time (s)": "{\"description\": \"5 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=397.65, mean=522.547, max=684.675, sum=2612.735 (5)\", \"tab\": \"General information\", \"score\": \"522.5470877192982\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "MMLU - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "BoolQ", - "source_data": { - "dataset_name": "BoolQ", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on BoolQ", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.714, - "details": { - "description": "min=0.714, mean=0.714, max=0.714, sum=0.714 (1)", - "tab": "Accuracy", - "BoolQ - ECE (10-bin)": "{\"description\": \"min=0.078, mean=0.078, max=0.078, sum=0.078 (1)\", \"tab\": \"Calibration\", \"score\": \"\"}", - "BoolQ - EM (Robustness)": "{\"description\": \"min=0.67, mean=0.67, max=0.67, sum=0.67 (1)\", \"tab\": \"Robustness\", \"score\": \"0.67\"}", - "BoolQ - EM (Fairness)": "{\"description\": \"min=0.666, mean=0.666, max=0.666, sum=0.666 (1)\", \"tab\": \"Fairness\", \"score\": \"0.666\"}", - "BoolQ - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "BoolQ - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "BoolQ - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "BoolQ - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "BoolQ - # prompt tokens": "{\"description\": \"min=1439.447, mean=1439.447, max=1439.447, sum=1439.447 (1)\", \"tab\": \"General information\", \"score\": \"1439.447\"}", - "BoolQ - # output tokens": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "BoolQ - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "BoolQ - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.711, - "details": { - "description": "min=0.711, mean=0.711, max=0.711, sum=0.711 (1)", - "tab": "Accuracy", - "NarrativeQA - ECE (10-bin)": "{\"description\": \"min=0.293, mean=0.293, max=0.293, sum=0.293 (1)\", \"tab\": \"Calibration\", \"score\": \"\"}", - "NarrativeQA - F1 (Robustness)": "{\"description\": \"min=0.544, mean=0.544, max=0.544, sum=0.544 (1)\", \"tab\": \"Robustness\", \"score\": \"0.543905538434645\"}", - "NarrativeQA - F1 (Fairness)": "{\"description\": \"min=0.628, mean=0.628, max=0.628, sum=0.628 (1)\", \"tab\": \"Fairness\", \"score\": \"0.6277072207288055\"}", - "NarrativeQA - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=1.437, mean=1.437, max=1.437, sum=1.437 (1)\", \"tab\": \"General information\", \"score\": \"1.4366197183098592\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=1541.115, mean=1541.115, max=1541.115, sum=1541.115 (1)\", \"tab\": \"General information\", \"score\": \"1541.1154929577465\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=100, mean=100, max=100, sum=100 (1)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "NarrativeQA - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NarrativeQA - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NarrativeQA - Stereotypes (gender)": "{\"description\": \"min=0.417, mean=0.417, max=0.417, sum=0.417 (1)\", \"tab\": \"Bias\", \"score\": \"0.4166666666666667\"}", - "NarrativeQA - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NarrativeQA - Representation (gender)": "{\"description\": \"min=0.224, mean=0.224, max=0.224, sum=0.224 (1)\", \"tab\": \"Bias\", \"score\": \"0.22357723577235772\"}", - "NarrativeQA - Toxic fraction": "{\"description\": \"min=0.008, mean=0.008, max=0.008, sum=0.008 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.008450704225352112\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (open-book)", - "source_data": { - "dataset_name": "NaturalQuestions (open-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (open-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.614, - "details": { - "description": "min=0.614, mean=0.614, max=0.614, sum=0.614 (1)", - "tab": "Accuracy", - "NaturalQuestions (closed-book) - ECE (10-bin)": "{\"description\": \"min=0.227, mean=0.227, max=0.227, sum=0.227 (1)\", \"tab\": \"Calibration\", \"score\": \"\"}", - "NaturalQuestions (open-book) - ECE (10-bin)": "{\"description\": \"min=0.414, mean=0.414, max=0.414, sum=0.414 (1)\", \"tab\": \"Calibration\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - F1 (Robustness)": "{\"description\": \"min=0.272, mean=0.272, max=0.272, sum=0.272 (1)\", \"tab\": \"Robustness\", \"score\": \"0.27211691617574163\"}", - "NaturalQuestions (open-book) - F1 (Robustness)": "{\"description\": \"min=0.556, mean=0.556, max=0.556, sum=0.556 (1)\", \"tab\": \"Robustness\", \"score\": \"0.5559403134593146\"}", - "NaturalQuestions (closed-book) - F1 (Fairness)": "{\"description\": \"min=0.288, mean=0.288, max=0.288, sum=0.288 (1)\", \"tab\": \"Fairness\", \"score\": \"0.28794490645078735\"}", - "NaturalQuestions (open-book) - F1 (Fairness)": "{\"description\": \"min=0.561, mean=0.561, max=0.561, sum=0.561 (1)\", \"tab\": \"Fairness\", \"score\": \"0.5608161827325524\"}", - "NaturalQuestions (closed-book) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NaturalQuestions (open-book) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=137.383, mean=137.383, max=137.383, sum=137.383 (1)\", \"tab\": \"General information\", \"score\": \"137.383\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=300, mean=300, max=300, sum=300 (1)\", \"tab\": \"General information\", \"score\": \"300.0\"}", - "NaturalQuestions (closed-book) - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=3.722, mean=3.722, max=3.722, sum=3.722 (1)\", \"tab\": \"General information\", \"score\": \"3.722\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.049, mean=0.049, max=0.049, sum=0.049 (1)\", \"tab\": \"General information\", \"score\": \"0.049\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1407.178, mean=1407.178, max=1407.178, sum=1407.178 (1)\", \"tab\": \"General information\", \"score\": \"1407.178\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=300, mean=300, max=300, sum=300 (1)\", \"tab\": \"General information\", \"score\": \"300.0\"}", - "NaturalQuestions (open-book) - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NaturalQuestions (closed-book) - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=0.667 (1)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666667\"}", - "NaturalQuestions (closed-book) - Stereotypes (gender)": "{\"description\": \"min=0.5, mean=0.5, max=0.5, sum=0.5 (1)\", \"tab\": \"Bias\", \"score\": \"0.5\"}", - "NaturalQuestions (closed-book) - Representation (race)": "{\"description\": \"min=0.438, mean=0.438, max=0.438, sum=0.438 (1)\", \"tab\": \"Bias\", \"score\": \"0.43775100401606426\"}", - "NaturalQuestions (closed-book) - Representation (gender)": "{\"description\": \"min=0.167, mean=0.167, max=0.167, sum=0.167 (1)\", \"tab\": \"Bias\", \"score\": \"0.16666666666666666\"}", - "NaturalQuestions (open-book) - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NaturalQuestions (open-book) - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NaturalQuestions (open-book) - Representation (race)": "{\"description\": \"min=0.333, mean=0.333, max=0.333, sum=0.333 (1)\", \"tab\": \"Bias\", \"score\": \"0.3333333333333333\"}", - "NaturalQuestions (open-book) - Representation (gender)": "{\"description\": \"min=0.083, mean=0.083, max=0.083, sum=0.083 (1)\", \"tab\": \"Bias\", \"score\": \"0.08333333333333334\"}", - "NaturalQuestions (closed-book) - Toxic fraction": "{\"description\": \"min=0.001, mean=0.001, max=0.001, sum=0.001 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.001\"}", - "NaturalQuestions (open-book) - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "QuAC", - "source_data": { - "dataset_name": "QuAC", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on QuAC", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.347, - "details": { - "description": "min=0.347, mean=0.347, max=0.347, sum=0.347 (1)", - "tab": "Accuracy", - "QuAC - ECE (10-bin)": "{\"description\": \"min=0.172, mean=0.172, max=0.172, sum=0.172 (1)\", \"tab\": \"Calibration\", \"score\": \"\"}", - "QuAC - F1 (Robustness)": "{\"description\": \"min=0.194, mean=0.194, max=0.194, sum=0.194 (1)\", \"tab\": \"Robustness\", \"score\": \"0.19407861446110536\"}", - "QuAC - F1 (Fairness)": "{\"description\": \"min=0.267, mean=0.267, max=0.267, sum=0.267 (1)\", \"tab\": \"Fairness\", \"score\": \"0.26734169068478314\"}", - "QuAC - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "QuAC - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "QuAC - # train": "{\"description\": \"min=0.507, mean=0.507, max=0.507, sum=0.507 (1)\", \"tab\": \"General information\", \"score\": \"0.507\"}", - "QuAC - truncated": "{\"description\": \"min=0.06, mean=0.06, max=0.06, sum=0.06 (1)\", \"tab\": \"General information\", \"score\": \"0.06\"}", - "QuAC - # prompt tokens": "{\"description\": \"min=1498.657, mean=1498.657, max=1498.657, sum=1498.657 (1)\", \"tab\": \"General information\", \"score\": \"1498.657\"}", - "QuAC - # output tokens": "{\"description\": \"min=99.882, mean=99.882, max=99.882, sum=99.882 (1)\", \"tab\": \"General information\", \"score\": \"99.882\"}", - "QuAC - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "QuAC - Stereotypes (race)": "{\"description\": \"min=0.605, mean=0.605, max=0.605, sum=0.605 (1)\", \"tab\": \"Bias\", \"score\": \"0.6047619047619048\"}", - "QuAC - Stereotypes (gender)": "{\"description\": \"min=0.444, mean=0.444, max=0.444, sum=0.444 (1)\", \"tab\": \"Bias\", \"score\": \"0.44425076013311304\"}", - "QuAC - Representation (race)": "{\"description\": \"min=0.276, mean=0.276, max=0.276, sum=0.276 (1)\", \"tab\": \"Bias\", \"score\": \"0.2761904761904762\"}", - "QuAC - Representation (gender)": "{\"description\": \"min=0.224, mean=0.224, max=0.224, sum=0.224 (1)\", \"tab\": \"Bias\", \"score\": \"0.22388059701492535\"}", - "QuAC - Toxic fraction": "{\"description\": \"min=0.003, mean=0.003, max=0.003, sum=0.003 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.003\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "HellaSwag", - "source_data": { - "dataset_name": "HellaSwag", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on HellaSwag", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "HellaSwag - ECE (10-bin)": "{\"description\": \"No matching runs\", \"tab\": \"Calibration\", \"score\": \"\"}", - "HellaSwag - EM (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "HellaSwag - EM (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "HellaSwag - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "HellaSwag - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "OpenbookQA - ECE (10-bin)": "{\"description\": \"No matching runs\", \"tab\": \"Calibration\", \"score\": \"\"}", - "OpenbookQA - EM (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "OpenbookQA - EM (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "OpenbookQA - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "OpenbookQA - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "TruthfulQA", - "source_data": { - "dataset_name": "TruthfulQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on TruthfulQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.324, - "details": { - "description": "min=0.324, mean=0.324, max=0.324, sum=0.324 (1)", - "tab": "Accuracy", - "TruthfulQA - ECE (10-bin)": "{\"description\": \"min=0.193, mean=0.193, max=0.193, sum=0.193 (1)\", \"tab\": \"Calibration\", \"score\": \"\"}", - "TruthfulQA - EM (Robustness)": "{\"description\": \"min=0.274, mean=0.274, max=0.274, sum=0.274 (1)\", \"tab\": \"Robustness\", \"score\": \"0.27370030581039756\"}", - "TruthfulQA - EM (Fairness)": "{\"description\": \"min=0.234, mean=0.234, max=0.234, sum=0.234 (1)\", \"tab\": \"Fairness\", \"score\": \"0.23394495412844038\"}", - "TruthfulQA - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "TruthfulQA - # eval": "{\"description\": \"min=654, mean=654, max=654, sum=654 (1)\", \"tab\": \"General information\", \"score\": \"654.0\"}", - "TruthfulQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "TruthfulQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "TruthfulQA - # prompt tokens": "{\"description\": \"min=524.602, mean=524.602, max=524.602, sum=524.602 (1)\", \"tab\": \"General information\", \"score\": \"524.6024464831804\"}", - "TruthfulQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "TruthfulQA - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MS MARCO (TREC)", - "source_data": { - "dataset_name": "MS MARCO (TREC)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "NDCG@10 on MS MARCO (TREC)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "MS MARCO (regular) - RR@10 (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "MS MARCO (TREC) - NDCG@10 (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "MS MARCO (regular) - RR@10 (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "MS MARCO (TREC) - NDCG@10 (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "MS MARCO (regular) - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MS MARCO (TREC) - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MS MARCO (regular) - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "MS MARCO (TREC) - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CNN/DailyMail", - "source_data": { - "dataset_name": "CNN/DailyMail", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on CNN/DailyMail", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "CNN/DailyMail - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "CNN/DailyMail - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "CNN/DailyMail - SummaC": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - QAFactEval": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - BERTScore (F1)": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - Coverage": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - Density": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - Compression": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-faithfulness": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-relevance": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-coherence": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "XSUM", - "source_data": { - "dataset_name": "XSUM", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on XSUM", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "XSUM - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "XSUM - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "XSUM - SummaC": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - QAFactEval": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - BERTScore (F1)": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - Coverage": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - Density": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - Compression": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-faithfulness": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-relevance": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-coherence": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "IMDB", - "source_data": { - "dataset_name": "IMDB", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on IMDB", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.928, - "details": { - "description": "min=0.928, mean=0.928, max=0.928, sum=0.928 (1)", - "tab": "Accuracy", - "IMDB - ECE (10-bin)": "{\"description\": \"min=0.302, mean=0.302, max=0.302, sum=0.302 (1)\", \"tab\": \"Calibration\", \"score\": \"\"}", - "IMDB - EM (Robustness)": "{\"description\": \"min=0.875, mean=0.875, max=0.875, sum=0.875 (1)\", \"tab\": \"Robustness\", \"score\": \"0.875\"}", - "IMDB - EM (Fairness)": "{\"description\": \"min=0.903, mean=0.903, max=0.903, sum=0.903 (1)\", \"tab\": \"Fairness\", \"score\": \"0.903\"}", - "IMDB - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "IMDB - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "IMDB - # train": "{\"description\": \"min=2.781, mean=2.781, max=2.781, sum=2.781 (1)\", \"tab\": \"General information\", \"score\": \"2.781\"}", - "IMDB - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IMDB - # prompt tokens": "{\"description\": \"min=1751.213, mean=1751.213, max=1751.213, sum=1751.213 (1)\", \"tab\": \"General information\", \"score\": \"1751.213\"}", - "IMDB - # output tokens": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "IMDB - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "IMDB - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CivilComments", - "source_data": { - "dataset_name": "CivilComments", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on CivilComments", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6, - "details": { - "description": "min=0.118, mean=0.6, max=0.947, sum=10.797 (18)", - "tab": "Accuracy", - "CivilComments - ECE (10-bin)": "{\"description\": \"min=0.098, mean=0.295, max=0.455, sum=5.305 (18)\", \"tab\": \"Calibration\", \"score\": \"\"}", - "CivilComments - EM (Robustness)": "{\"description\": \"min=0.079, mean=0.529, max=0.947, sum=9.523 (18)\", \"tab\": \"Robustness\", \"score\": \"0.529079897678074\"}", - "CivilComments - EM (Fairness)": "{\"description\": \"min=0.054, mean=0.533, max=0.947, sum=9.585 (18)\", \"tab\": \"Fairness\", \"score\": \"0.5325232651113918\"}", - "CivilComments - Denoised inference time (s)": "{\"description\": \"9 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "CivilComments - # eval": "{\"description\": \"min=74, mean=371.556, max=683, sum=6688 (18)\", \"tab\": \"General information\", \"score\": \"371.55555555555554\"}", - "CivilComments - # train": "{\"description\": \"min=5, mean=5, max=5, sum=90 (18)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CivilComments - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (18)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CivilComments - # prompt tokens": "{\"description\": \"min=404.732, mean=855.241, max=1417.567, sum=15394.339 (18)\", \"tab\": \"General information\", \"score\": \"855.2410378605821\"}", - "CivilComments - # output tokens": "{\"description\": \"min=5, mean=5, max=5, sum=90 (18)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CivilComments - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=18 (18)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "CivilComments - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Toxic fraction": "{\"description\": \"9 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "RAFT", - "source_data": { - "dataset_name": "RAFT", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on RAFT", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.643, - "details": { - "description": "min=0.125, mean=0.643, max=0.925, sum=7.075 (11)", - "tab": "Accuracy", - "RAFT - ECE (10-bin)": "{\"description\": \"min=0.132, mean=0.644, max=0.925, sum=7.081 (11)\", \"tab\": \"Calibration\", \"score\": \"\"}", - "RAFT - EM (Robustness)": "{\"description\": \"min=0, mean=0.559, max=0.9, sum=6.15 (11)\", \"tab\": \"Robustness\", \"score\": \"0.5590909090909091\"}", - "RAFT - EM (Fairness)": "{\"description\": \"min=0.075, mean=0.605, max=0.9, sum=6.65 (11)\", \"tab\": \"Fairness\", \"score\": \"0.6045454545454545\"}", - "RAFT - Denoised inference time (s)": "{\"description\": \"11 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "RAFT - # eval": "{\"description\": \"min=40, mean=40, max=40, sum=440 (11)\", \"tab\": \"General information\", \"score\": \"40.0\"}", - "RAFT - # train": "{\"description\": \"min=0.45, mean=4.552, max=5, sum=50.075 (11)\", \"tab\": \"General information\", \"score\": \"4.552272727272727\"}", - "RAFT - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (11)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "RAFT - # prompt tokens": "{\"description\": \"min=303.675, mean=954.111, max=1882.1, sum=10495.225 (11)\", \"tab\": \"General information\", \"score\": \"954.1113636363635\"}", - "RAFT - # output tokens": "{\"description\": \"min=22.975, mean=29.361, max=30, sum=322.975 (11)\", \"tab\": \"General information\", \"score\": \"29.361363636363638\"}", - "RAFT - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=11 (11)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "RAFT - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Toxic fraction": "{\"description\": \"11 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/meta_llama-2-13b.json b/data/models/meta_llama-2-13b.json deleted file mode 100644 index 1e4f5b95d4d5f91d161cfca5ac956ff33caed597..0000000000000000000000000000000000000000 --- a/data/models/meta_llama-2-13b.json +++ /dev/null @@ -1,2570 +0,0 @@ -{ - "model_info": { - "name": "Llama 2 13B", - "id": "meta/llama-2-13b", - "developer": "Meta", - "inference_platform": "unknown", - "additional_details": { - "model_id_aliases": [ - "meta/Llama-2-13B" - ] - } - }, - "evaluations": [ - { - "evaluation_id": "helm_classic/meta_Llama-2-13B/1774096308.339228", - "retrieved_timestamp": "1774096308.339228", - "source_metadata": { - "source_name": "helm_classic", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_classic", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_classic", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperform on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.823, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Calibration": "{\"description\": \"\", \"tab\": \"Calibration\", \"score\": \"\"}", - "Mean win rate - Robustness": "{\"description\": \"\", \"tab\": \"Robustness\", \"score\": \"0.8231701631701632\"}", - "Mean win rate - Fairness": "{\"description\": \"\", \"tab\": \"Fairness\", \"score\": \"0.8078088578088578\"}", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}", - "Mean win rate - Bias": "{\"description\": \"\", \"tab\": \"Bias\", \"score\": \"0.46948265409803874\"}", - "Mean win rate - Toxicity": "{\"description\": \"\", \"tab\": \"Toxicity\", \"score\": \"0.4142191142191142\"}", - "Mean win rate - Summarization metrics": "{\"description\": \"\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.507, - "details": { - "description": "min=0.28, mean=0.507, max=0.84, sum=2.533 (5)", - "tab": "Accuracy", - "MMLU - ECE (10-bin)": "{\"description\": \"5 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "MMLU - EM (Robustness)": "{\"description\": \"min=0.22, mean=0.444, max=0.76, sum=2.222 (5)\", \"tab\": \"Robustness\", \"score\": \"0.44438596491228066\"}", - "MMLU - EM (Fairness)": "{\"description\": \"min=0.26, mean=0.466, max=0.79, sum=2.331 (5)\", \"tab\": \"Fairness\", \"score\": \"0.46614035087719297\"}", - "MMLU - Denoised inference time (s)": "{\"description\": \"5 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=397.65, mean=522.547, max=684.675, sum=2612.735 (5)\", \"tab\": \"General information\", \"score\": \"522.5470877192982\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "MMLU - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "BoolQ", - "source_data": { - "dataset_name": "BoolQ", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on BoolQ", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.811, - "details": { - "description": "min=0.811, mean=0.811, max=0.811, sum=0.811 (1)", - "tab": "Accuracy", - "BoolQ - ECE (10-bin)": "{\"description\": \"min=0.116, mean=0.116, max=0.116, sum=0.116 (1)\", \"tab\": \"Calibration\", \"score\": \"\"}", - "BoolQ - EM (Robustness)": "{\"description\": \"min=0.753, mean=0.753, max=0.753, sum=0.753 (1)\", \"tab\": \"Robustness\", \"score\": \"0.753\"}", - "BoolQ - EM (Fairness)": "{\"description\": \"min=0.732, mean=0.732, max=0.732, sum=0.732 (1)\", \"tab\": \"Fairness\", \"score\": \"0.732\"}", - "BoolQ - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "BoolQ - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "BoolQ - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "BoolQ - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "BoolQ - # prompt tokens": "{\"description\": \"min=1439.447, mean=1439.447, max=1439.447, sum=1439.447 (1)\", \"tab\": \"General information\", \"score\": \"1439.447\"}", - "BoolQ - # output tokens": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "BoolQ - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "BoolQ - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.744, - "details": { - "description": "min=0.744, mean=0.744, max=0.744, sum=0.744 (1)", - "tab": "Accuracy", - "NarrativeQA - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "NarrativeQA - F1 (Robustness)": "{\"description\": \"min=0.682, mean=0.682, max=0.682, sum=0.682 (1)\", \"tab\": \"Robustness\", \"score\": \"0.681791424099214\"}", - "NarrativeQA - F1 (Fairness)": "{\"description\": \"min=0.657, mean=0.657, max=0.657, sum=0.657 (1)\", \"tab\": \"Fairness\", \"score\": \"0.6567284210865421\"}", - "NarrativeQA - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=4.414, mean=4.414, max=4.414, sum=4.414 (1)\", \"tab\": \"General information\", \"score\": \"4.414084507042253\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=3673.268, mean=3673.268, max=3673.268, sum=3673.268 (1)\", \"tab\": \"General information\", \"score\": \"3673.2676056338028\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NarrativeQA - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NarrativeQA - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NarrativeQA - Stereotypes (gender)": "{\"description\": \"min=0.417, mean=0.417, max=0.417, sum=0.417 (1)\", \"tab\": \"Bias\", \"score\": \"0.4166666666666667\"}", - "NarrativeQA - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NarrativeQA - Representation (gender)": "{\"description\": \"min=0.218, mean=0.218, max=0.218, sum=0.218 (1)\", \"tab\": \"Bias\", \"score\": \"0.21830985915492954\"}", - "NarrativeQA - Toxic fraction": "{\"description\": \"min=0.014, mean=0.014, max=0.014, sum=0.014 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.014084507042253521\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (open-book)", - "source_data": { - "dataset_name": "NaturalQuestions (open-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (open-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.637, - "details": { - "description": "min=0.637, mean=0.637, max=0.637, sum=0.637 (1)", - "tab": "Accuracy", - "NaturalQuestions (closed-book) - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "NaturalQuestions (open-book) - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - F1 (Robustness)": "{\"description\": \"min=0.324, mean=0.324, max=0.324, sum=0.324 (1)\", \"tab\": \"Robustness\", \"score\": \"0.3243542710528751\"}", - "NaturalQuestions (open-book) - F1 (Robustness)": "{\"description\": \"min=0.563, mean=0.563, max=0.563, sum=0.563 (1)\", \"tab\": \"Robustness\", \"score\": \"0.5631882717621935\"}", - "NaturalQuestions (closed-book) - F1 (Fairness)": "{\"description\": \"min=0.309, mean=0.309, max=0.309, sum=0.309 (1)\", \"tab\": \"Fairness\", \"score\": \"0.30927547433853436\"}", - "NaturalQuestions (open-book) - F1 (Fairness)": "{\"description\": \"min=0.58, mean=0.58, max=0.58, sum=0.58 (1)\", \"tab\": \"Fairness\", \"score\": \"0.5801102053016279\"}", - "NaturalQuestions (closed-book) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NaturalQuestions (open-book) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=137.383, mean=137.383, max=137.383, sum=137.383 (1)\", \"tab\": \"General information\", \"score\": \"137.383\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NaturalQuestions (closed-book) - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.831, mean=4.831, max=4.831, sum=4.831 (1)\", \"tab\": \"General information\", \"score\": \"4.831\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.026, mean=0.026, max=0.026, sum=0.026 (1)\", \"tab\": \"General information\", \"score\": \"0.026\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=2289.409, mean=2289.409, max=2289.409, sum=2289.409 (1)\", \"tab\": \"General information\", \"score\": \"2289.409\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=0.984, mean=0.984, max=0.984, sum=0.984 (1)\", \"tab\": \"General information\", \"score\": \"0.984\"}", - "NaturalQuestions (open-book) - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NaturalQuestions (closed-book) - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=0.667 (1)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666667\"}", - "NaturalQuestions (closed-book) - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - Representation (race)": "{\"description\": \"min=0.521, mean=0.521, max=0.521, sum=0.521 (1)\", \"tab\": \"Bias\", \"score\": \"0.5205992509363295\"}", - "NaturalQuestions (closed-book) - Representation (gender)": "{\"description\": \"min=0.15, mean=0.15, max=0.15, sum=0.15 (1)\", \"tab\": \"Bias\", \"score\": \"0.15000000000000002\"}", - "NaturalQuestions (open-book) - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=0.667 (1)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666667\"}", - "NaturalQuestions (open-book) - Stereotypes (gender)": "{\"description\": \"min=0.5, mean=0.5, max=0.5, sum=0.5 (1)\", \"tab\": \"Bias\", \"score\": \"0.5\"}", - "NaturalQuestions (open-book) - Representation (race)": "{\"description\": \"min=0.467, mean=0.467, max=0.467, sum=0.467 (1)\", \"tab\": \"Bias\", \"score\": \"0.4666666666666667\"}", - "NaturalQuestions (open-book) - Representation (gender)": "{\"description\": \"min=0.357, mean=0.357, max=0.357, sum=0.357 (1)\", \"tab\": \"Bias\", \"score\": \"0.3571428571428571\"}", - "NaturalQuestions (closed-book) - Toxic fraction": "{\"description\": \"min=0.001, mean=0.001, max=0.001, sum=0.001 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.001\"}", - "NaturalQuestions (open-book) - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "QuAC", - "source_data": { - "dataset_name": "QuAC", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on QuAC", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.424, - "details": { - "description": "min=0.424, mean=0.424, max=0.424, sum=0.424 (1)", - "tab": "Accuracy", - "QuAC - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "QuAC - F1 (Robustness)": "{\"description\": \"min=0.294, mean=0.294, max=0.294, sum=0.294 (1)\", \"tab\": \"Robustness\", \"score\": \"0.2939019916232739\"}", - "QuAC - F1 (Fairness)": "{\"description\": \"min=0.351, mean=0.351, max=0.351, sum=0.351 (1)\", \"tab\": \"Fairness\", \"score\": \"0.35074944218906556\"}", - "QuAC - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "QuAC - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "QuAC - # train": "{\"description\": \"min=3.204, mean=3.204, max=3.204, sum=3.204 (1)\", \"tab\": \"General information\", \"score\": \"3.204\"}", - "QuAC - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "QuAC - # prompt tokens": "{\"description\": \"min=3617.038, mean=3617.038, max=3617.038, sum=3617.038 (1)\", \"tab\": \"General information\", \"score\": \"3617.038\"}", - "QuAC - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "QuAC - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "QuAC - Stereotypes (race)": "{\"description\": \"min=0.549, mean=0.549, max=0.549, sum=0.549 (1)\", \"tab\": \"Bias\", \"score\": \"0.5485347985347986\"}", - "QuAC - Stereotypes (gender)": "{\"description\": \"min=0.392, mean=0.392, max=0.392, sum=0.392 (1)\", \"tab\": \"Bias\", \"score\": \"0.39214643381310055\"}", - "QuAC - Representation (race)": "{\"description\": \"min=0.325, mean=0.325, max=0.325, sum=0.325 (1)\", \"tab\": \"Bias\", \"score\": \"0.3248945147679325\"}", - "QuAC - Representation (gender)": "{\"description\": \"min=0.242, mean=0.242, max=0.242, sum=0.242 (1)\", \"tab\": \"Bias\", \"score\": \"0.24197860962566847\"}", - "QuAC - Toxic fraction": "{\"description\": \"min=0.004, mean=0.004, max=0.004, sum=0.004 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.004\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "HellaSwag", - "source_data": { - "dataset_name": "HellaSwag", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on HellaSwag", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "HellaSwag - ECE (10-bin)": "{\"description\": \"No matching runs\", \"tab\": \"Calibration\", \"score\": \"\"}", - "HellaSwag - EM (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "HellaSwag - EM (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "HellaSwag - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "HellaSwag - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "OpenbookQA - ECE (10-bin)": "{\"description\": \"No matching runs\", \"tab\": \"Calibration\", \"score\": \"\"}", - "OpenbookQA - EM (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "OpenbookQA - EM (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "OpenbookQA - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "OpenbookQA - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "TruthfulQA", - "source_data": { - "dataset_name": "TruthfulQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on TruthfulQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.33, - "details": { - "description": "min=0.33, mean=0.33, max=0.33, sum=0.33 (1)", - "tab": "Accuracy", - "TruthfulQA - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "TruthfulQA - EM (Robustness)": "{\"description\": \"min=0.287, mean=0.287, max=0.287, sum=0.287 (1)\", \"tab\": \"Robustness\", \"score\": \"0.2874617737003058\"}", - "TruthfulQA - EM (Fairness)": "{\"description\": \"min=0.274, mean=0.274, max=0.274, sum=0.274 (1)\", \"tab\": \"Fairness\", \"score\": \"0.27370030581039756\"}", - "TruthfulQA - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "TruthfulQA - # eval": "{\"description\": \"min=654, mean=654, max=654, sum=654 (1)\", \"tab\": \"General information\", \"score\": \"654.0\"}", - "TruthfulQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "TruthfulQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "TruthfulQA - # prompt tokens": "{\"description\": \"min=524.602, mean=524.602, max=524.602, sum=524.602 (1)\", \"tab\": \"General information\", \"score\": \"524.6024464831804\"}", - "TruthfulQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "TruthfulQA - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MS MARCO (TREC)", - "source_data": { - "dataset_name": "MS MARCO (TREC)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "NDCG@10 on MS MARCO (TREC)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "MS MARCO (regular) - RR@10 (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "MS MARCO (TREC) - NDCG@10 (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "MS MARCO (regular) - RR@10 (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "MS MARCO (TREC) - NDCG@10 (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "MS MARCO (regular) - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MS MARCO (TREC) - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MS MARCO (regular) - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "MS MARCO (TREC) - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CNN/DailyMail", - "source_data": { - "dataset_name": "CNN/DailyMail", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on CNN/DailyMail", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "CNN/DailyMail - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "CNN/DailyMail - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "CNN/DailyMail - SummaC": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - QAFactEval": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - BERTScore (F1)": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - Coverage": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - Density": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - Compression": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-faithfulness": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-relevance": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-coherence": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "XSUM", - "source_data": { - "dataset_name": "XSUM", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on XSUM", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "XSUM - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "XSUM - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "XSUM - SummaC": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - QAFactEval": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - BERTScore (F1)": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - Coverage": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - Density": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - Compression": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-faithfulness": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-relevance": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-coherence": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "IMDB", - "source_data": { - "dataset_name": "IMDB", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on IMDB", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.962, - "details": { - "description": "min=0.962, mean=0.962, max=0.962, sum=0.962 (1)", - "tab": "Accuracy", - "IMDB - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "IMDB - EM (Robustness)": "{\"description\": \"min=0.954, mean=0.954, max=0.954, sum=0.954 (1)\", \"tab\": \"Robustness\", \"score\": \"0.954\"}", - "IMDB - EM (Fairness)": "{\"description\": \"min=0.957, mean=0.957, max=0.957, sum=0.957 (1)\", \"tab\": \"Fairness\", \"score\": \"0.957\"}", - "IMDB - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "IMDB - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "IMDB - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "IMDB - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IMDB - # prompt tokens": "{\"description\": \"min=2897.409, mean=2897.409, max=2897.409, sum=2897.409 (1)\", \"tab\": \"General information\", \"score\": \"2897.409\"}", - "IMDB - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "IMDB - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "IMDB - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CivilComments", - "source_data": { - "dataset_name": "CivilComments", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on CivilComments", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.588, - "details": { - "description": "min=0.087, mean=0.588, max=0.968, sum=10.579 (18)", - "tab": "Accuracy", - "CivilComments - ECE (10-bin)": "{\"description\": \"min=0.098, mean=0.323, max=0.788, sum=4.519 (14)\", \"tab\": \"Calibration\", \"score\": \"\"}", - "CivilComments - EM (Robustness)": "{\"description\": \"min=0.022, mean=0.47, max=0.958, sum=8.468 (18)\", \"tab\": \"Robustness\", \"score\": \"0.47042658911281887\"}", - "CivilComments - EM (Fairness)": "{\"description\": \"min=0.006, mean=0.489, max=0.968, sum=8.81 (18)\", \"tab\": \"Fairness\", \"score\": \"0.4894481246425394\"}", - "CivilComments - Denoised inference time (s)": "{\"description\": \"9 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "CivilComments - # eval": "{\"description\": \"min=74, mean=371.556, max=683, sum=6688 (18)\", \"tab\": \"General information\", \"score\": \"371.55555555555554\"}", - "CivilComments - # train": "{\"description\": \"min=5, mean=5, max=5, sum=90 (18)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CivilComments - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (18)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CivilComments - # prompt tokens": "{\"description\": \"min=404.732, mean=855.241, max=1417.567, sum=15394.339 (18)\", \"tab\": \"General information\", \"score\": \"855.2410378605821\"}", - "CivilComments - # output tokens": "{\"description\": \"min=1, mean=2.692, max=5, sum=48.448 (18)\", \"tab\": \"General information\", \"score\": \"2.6915388744093813\"}", - "CivilComments - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=18 (18)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "CivilComments - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Toxic fraction": "{\"description\": \"9 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "RAFT", - "source_data": { - "dataset_name": "RAFT", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on RAFT", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.707, - "details": { - "description": "min=0.1, mean=0.707, max=0.975, sum=7.775 (11)", - "tab": "Accuracy", - "RAFT - ECE (10-bin)": "{\"description\": \"11 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "RAFT - EM (Robustness)": "{\"description\": \"min=0.05, mean=0.652, max=0.95, sum=7.175 (11)\", \"tab\": \"Robustness\", \"score\": \"0.6522727272727272\"}", - "RAFT - EM (Fairness)": "{\"description\": \"min=0.075, mean=0.673, max=0.975, sum=7.4 (11)\", \"tab\": \"Fairness\", \"score\": \"0.6727272727272727\"}", - "RAFT - Denoised inference time (s)": "{\"description\": \"11 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "RAFT - # eval": "{\"description\": \"min=40, mean=40, max=40, sum=440 (11)\", \"tab\": \"General information\", \"score\": \"40.0\"}", - "RAFT - # train": "{\"description\": \"min=2.575, mean=4.78, max=5, sum=52.575 (11)\", \"tab\": \"General information\", \"score\": \"4.779545454545455\"}", - "RAFT - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (11)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "RAFT - # prompt tokens": "{\"description\": \"min=303.675, mean=1153.852, max=3623.9, sum=12692.375 (11)\", \"tab\": \"General information\", \"score\": \"1153.8522727272727\"}", - "RAFT - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=11 (11)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "RAFT - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=11 (11)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "RAFT - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Toxic fraction": "{\"description\": \"11 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_lite/meta_llama-2-13b/1774096306.427425", - "retrieved_timestamp": "1774096306.427425", - "source_metadata": { - "source_name": "helm_lite", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_lite", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_lite", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.233, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.7253183520599251\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.741, - "details": { - "description": "min=0.741, mean=0.741, max=0.741, sum=0.741 (1)", - "tab": "Accuracy", - "NarrativeQA - Observed inference time (s)": "{\"description\": \"min=0.795, mean=0.795, max=0.795, sum=0.795 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.7950913200915699\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=4.408, mean=4.408, max=4.408, sum=4.408 (1)\", \"tab\": \"General information\", \"score\": \"4.408450704225352\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=3669.808, mean=3669.808, max=3669.808, sum=3669.808 (1)\", \"tab\": \"General information\", \"score\": \"3669.8084507042254\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (closed-book)", - "source_data": { - "dataset_name": "NaturalQuestions (closed-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (closed-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.371, - "details": { - "description": "min=0.371, mean=0.371, max=0.371, sum=0.371 (1)", - "tab": "Accuracy", - "NaturalQuestions (open-book) - Observed inference time (s)": "{\"description\": \"min=0.579, mean=0.579, max=0.579, sum=0.579 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.5793666501045227\"}", - "NaturalQuestions (closed-book) - Observed inference time (s)": "{\"description\": \"min=0.384, mean=0.384, max=0.384, sum=0.384 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.3839698841571808\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.831, mean=4.831, max=4.831, sum=4.831 (1)\", \"tab\": \"General information\", \"score\": \"4.831\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.026, mean=0.026, max=0.026, sum=0.026 (1)\", \"tab\": \"General information\", \"score\": \"0.026\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=2289.357, mean=2289.357, max=2289.357, sum=2289.357 (1)\", \"tab\": \"General information\", \"score\": \"2289.357\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=0.986, mean=0.986, max=0.986, sum=0.986 (1)\", \"tab\": \"General information\", \"score\": \"0.986\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=137.383, mean=137.383, max=137.383, sum=137.383 (1)\", \"tab\": \"General information\", \"score\": \"137.383\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "mode": "\"closedbook\"" - } - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.634, - "details": { - "description": "min=0.634, mean=0.634, max=0.634, sum=0.634 (1)", - "tab": "Accuracy", - "OpenbookQA - Observed inference time (s)": "{\"description\": \"min=0.347, mean=0.347, max=0.347, sum=0.347 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.34700755834579466\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=282.574, mean=282.574, max=282.574, sum=282.574 (1)\", \"tab\": \"General information\", \"score\": \"282.574\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "dataset": "\"openbookqa\"", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.505, - "details": { - "description": "min=0.28, mean=0.505, max=0.84, sum=2.527 (5)", - "tab": "Accuracy", - "MMLU - Observed inference time (s)": "{\"description\": \"min=0.359, mean=0.374, max=0.383, sum=1.872 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.37437369656144526\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=397.65, mean=522.547, max=684.675, sum=2612.735 (5)\", \"tab\": \"General information\", \"score\": \"522.5470877192982\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"college_chemistry\", \"computer_security\", \"econometrics\", \"us_foreign_policy\"]", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MATH", - "source_data": { - "dataset_name": "MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Equivalent (CoT) on MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.102, - "details": { - "description": "min=0, mean=0.102, max=0.193, sum=0.715 (7)", - "tab": "Accuracy", - "MATH - Observed inference time (s)": "{\"description\": \"min=1.083, mean=1.516, max=1.771, sum=10.613 (7)\", \"tab\": \"Efficiency\", \"score\": \"1.5161172209789922\"}", - "MATH - # eval": "{\"description\": \"min=30, mean=62.429, max=135, sum=437 (7)\", \"tab\": \"General information\", \"score\": \"62.42857142857143\"}", - "MATH - # train": "{\"description\": \"min=8, mean=8, max=8, sum=56 (7)\", \"tab\": \"General information\", \"score\": \"8.0\"}", - "MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MATH - # prompt tokens": "{\"description\": \"min=971.652, mean=1438.636, max=2490.962, sum=10070.453 (7)\", \"tab\": \"General information\", \"score\": \"1438.6362030100095\"}", - "MATH - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=7 (7)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"algebra\", \"counting_and_probability\", \"geometry\", \"intermediate_algebra\", \"number_theory\", \"prealgebra\", \"precalculus\"]", - "level": "\"1\"", - "use_official_examples": "\"False\"", - "use_chain_of_thought": "\"True\"" - } - } - }, - { - "evaluation_name": "GSM8K", - "source_data": { - "dataset_name": "GSM8K", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on GSM8K", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.266, - "details": { - "description": "min=0.266, mean=0.266, max=0.266, sum=0.266 (1)", - "tab": "Accuracy", - "GSM8K - Observed inference time (s)": "{\"description\": \"min=1.737, mean=1.737, max=1.737, sum=1.737 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.7367573575973512\"}", - "GSM8K - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "GSM8K - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "GSM8K - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GSM8K - # prompt tokens": "{\"description\": \"min=1207.746, mean=1207.746, max=1207.746, sum=1207.746 (1)\", \"tab\": \"General information\", \"score\": \"1207.746\"}", - "GSM8K - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "LegalBench", - "source_data": { - "dataset_name": "LegalBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on LegalBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.591, - "details": { - "description": "min=0.338, mean=0.591, max=0.779, sum=2.955 (5)", - "tab": "Accuracy", - "LegalBench - Observed inference time (s)": "{\"description\": \"min=0.331, mean=0.438, max=0.729, sum=2.189 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.43780977145306127\"}", - "LegalBench - # eval": "{\"description\": \"min=95, mean=409.4, max=1000, sum=2047 (5)\", \"tab\": \"General information\", \"score\": \"409.4\"}", - "LegalBench - # train": "{\"description\": \"min=1.886, mean=4.177, max=5, sum=20.886 (5)\", \"tab\": \"General information\", \"score\": \"4.177142857142857\"}", - "LegalBench - truncated": "{\"description\": \"min=0, mean=0.001, max=0.004, sum=0.004 (5)\", \"tab\": \"General information\", \"score\": \"0.0008163265306122449\"}", - "LegalBench - # prompt tokens": "{\"description\": \"min=222.137, mean=1027.35, max=3642.378, sum=5136.751 (5)\", \"tab\": \"General information\", \"score\": \"1027.3502076083553\"}", - "LegalBench - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"abercrombie\", \"corporate_lobbying\", \"function_of_decision_section\", \"international_citizenship_questions\", \"proa\"]" - } - } - }, - { - "evaluation_name": "MedQA", - "source_data": { - "dataset_name": "MedQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MedQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.392, - "details": { - "description": "min=0.392, mean=0.392, max=0.392, sum=0.392 (1)", - "tab": "Accuracy", - "MedQA - Observed inference time (s)": "{\"description\": \"min=0.459, mean=0.459, max=0.459, sum=0.459 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.4588449499005115\"}", - "MedQA - # eval": "{\"description\": \"min=503, mean=503, max=503, sum=503 (1)\", \"tab\": \"General information\", \"score\": \"503.0\"}", - "MedQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MedQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MedQA - # prompt tokens": "{\"description\": \"min=1234.901, mean=1234.901, max=1234.901, sum=1234.901 (1)\", \"tab\": \"General information\", \"score\": \"1234.9005964214712\"}", - "MedQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WMT 2014", - "source_data": { - "dataset_name": "WMT 2014", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "BLEU-4 on WMT 2014", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.167, - "details": { - "description": "min=0.074, mean=0.167, max=0.209, sum=0.836 (5)", - "tab": "Accuracy", - "WMT 2014 - Observed inference time (s)": "{\"description\": \"min=0.557, mean=0.691, max=0.814, sum=3.456 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.6911807014709866\"}", - "WMT 2014 - # eval": "{\"description\": \"min=503, mean=568.8, max=832, sum=2844 (5)\", \"tab\": \"General information\", \"score\": \"568.8\"}", - "WMT 2014 - # train": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "WMT 2014 - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WMT 2014 - # prompt tokens": "{\"description\": \"min=127.523, mean=142.288, max=164.972, sum=711.438 (5)\", \"tab\": \"General information\", \"score\": \"142.28751290334915\"}", - "WMT 2014 - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "language_pair": "[\"cs-en\", \"de-en\", \"fr-en\", \"hi-en\", \"ru-en\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_mmlu/meta_llama-2-13b/1774096312.00548", - "retrieved_timestamp": "1774096312.00548", - "source_metadata": { - "source_name": "helm_mmlu", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_mmlu", - "evaluation_results": [ - { - "evaluation_name": "MMLU All Subjects", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU All Subjects", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.554, - "details": { - "description": "min=0.235, mean=0.554, max=0.83, sum=63.174 (114)", - "tab": "Accuracy", - "MMLU All Subjects - Observed inference time (s)": "{\"description\": \"min=0.323, mean=0.492, max=1.697, sum=56.065 (114)\", \"tab\": \"Efficiency\", \"score\": \"0.49179914059061297\"}", - "MMLU All Subjects - # eval": "{\"description\": \"min=100, mean=246.351, max=1534, sum=28084 (114)\", \"tab\": \"General information\", \"score\": \"246.35087719298247\"}", - "MMLU All Subjects - # train": "{\"description\": \"min=5, mean=5, max=5, sum=570 (114)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU All Subjects - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU All Subjects - # prompt tokens": "{\"description\": \"min=304.474, mean=706.682, max=3159.636, sum=80561.749 (114)\", \"tab\": \"General information\", \"score\": \"706.6820126388612\"}", - "MMLU All Subjects - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=114 (114)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_name": "Abstract Algebra", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Abstract Algebra", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.27, - "details": { - "description": "min=0.27, mean=0.27, max=0.27, sum=0.54 (2)", - "tab": "Accuracy", - "Abstract Algebra - Observed inference time (s)": "{\"description\": \"min=0.361, mean=0.361, max=0.361, sum=0.722 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3610322856903076\"}", - "Abstract Algebra - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Abstract Algebra - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Abstract Algebra - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Abstract Algebra - # prompt tokens": "{\"description\": \"min=397.65, mean=397.65, max=397.65, sum=795.3 (2)\", \"tab\": \"General information\", \"score\": \"397.65\"}", - "Abstract Algebra - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"abstract_algebra\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_abstract_algebra\"" - } - } - }, - { - "evaluation_name": "Anatomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Anatomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.496, - "details": { - "description": "min=0.496, mean=0.496, max=0.496, sum=0.993 (2)", - "tab": "Accuracy", - "Anatomy - Observed inference time (s)": "{\"description\": \"min=0.357, mean=0.357, max=0.357, sum=0.715 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.35744349868209274\"}", - "Anatomy - # eval": "{\"description\": \"min=135, mean=135, max=135, sum=270 (2)\", \"tab\": \"General information\", \"score\": \"135.0\"}", - "Anatomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Anatomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Anatomy - # prompt tokens": "{\"description\": \"min=409.133, mean=409.133, max=409.133, sum=818.267 (2)\", \"tab\": \"General information\", \"score\": \"409.1333333333333\"}", - "Anatomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"anatomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_anatomy\"" - } - } - }, - { - "evaluation_name": "College Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on College Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.235, - "details": { - "description": "min=0.235, mean=0.235, max=0.235, sum=0.471 (2)", - "tab": "Accuracy", - "College Chemistry - Observed inference time (s)": "{\"description\": \"min=0.449, mean=0.449, max=0.449, sum=0.897 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.44854954242706296\"}", - "College Biology - Observed inference time (s)": "{\"description\": \"min=0.401, mean=0.401, max=0.401, sum=0.802 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.40112912986013627\"}", - "College Computer Science - Observed inference time (s)": "{\"description\": \"min=0.534, mean=0.534, max=0.534, sum=1.069 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5343992376327514\"}", - "College Mathematics - Observed inference time (s)": "{\"description\": \"min=0.454, mean=0.454, max=0.454, sum=0.909 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.45426050424575803\"}", - "College Medicine - Observed inference time (s)": "{\"description\": \"min=0.452, mean=0.452, max=0.452, sum=0.905 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4522962446157643\"}", - "College Physics - Observed inference time (s)": "{\"description\": \"min=0.413, mean=0.413, max=0.413, sum=0.826 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4130270574607101\"}", - "College Chemistry - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Chemistry - # prompt tokens": "{\"description\": \"min=622.43, mean=622.43, max=622.43, sum=1244.86 (2)\", \"tab\": \"General information\", \"score\": \"622.43\"}", - "College Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Biology - # eval": "{\"description\": \"min=144, mean=144, max=144, sum=288 (2)\", \"tab\": \"General information\", \"score\": \"144.0\"}", - "College Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # prompt tokens": "{\"description\": \"min=553.632, mean=553.632, max=553.632, sum=1107.264 (2)\", \"tab\": \"General information\", \"score\": \"553.6319444444445\"}", - "College Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # prompt tokens": "{\"description\": \"min=901.14, mean=901.14, max=901.14, sum=1802.28 (2)\", \"tab\": \"General information\", \"score\": \"901.14\"}", - "College Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Mathematics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # prompt tokens": "{\"description\": \"min=646.96, mean=646.96, max=646.96, sum=1293.92 (2)\", \"tab\": \"General information\", \"score\": \"646.96\"}", - "College Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Medicine - # eval": "{\"description\": \"min=173, mean=173, max=173, sum=346 (2)\", \"tab\": \"General information\", \"score\": \"173.0\"}", - "College Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # prompt tokens": "{\"description\": \"min=608.671, mean=608.671, max=608.671, sum=1217.341 (2)\", \"tab\": \"General information\", \"score\": \"608.6705202312139\"}", - "College Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Physics - # eval": "{\"description\": \"min=102, mean=102, max=102, sum=204 (2)\", \"tab\": \"General information\", \"score\": \"102.0\"}", - "College Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # prompt tokens": "{\"description\": \"min=551.873, mean=551.873, max=551.873, sum=1103.745 (2)\", \"tab\": \"General information\", \"score\": \"551.8725490196078\"}", - "College Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"college_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_college_physics\"" - } - } - }, - { - "evaluation_name": "Computer Security", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Computer Security", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.69, - "details": { - "description": "min=0.69, mean=0.69, max=0.69, sum=1.38 (2)", - "tab": "Accuracy", - "Computer Security - Observed inference time (s)": "{\"description\": \"min=0.355, mean=0.355, max=0.355, sum=0.71 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3552073335647583\"}", - "Computer Security - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Computer Security - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Computer Security - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Computer Security - # prompt tokens": "{\"description\": \"min=428.17, mean=428.17, max=428.17, sum=856.34 (2)\", \"tab\": \"General information\", \"score\": \"428.17\"}", - "Computer Security - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"computer_security\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_computer_security\"" - } - } - }, - { - "evaluation_name": "Econometrics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Econometrics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.307, - "details": { - "description": "min=0.307, mean=0.307, max=0.307, sum=0.614 (2)", - "tab": "Accuracy", - "Econometrics - Observed inference time (s)": "{\"description\": \"min=0.455, mean=0.455, max=0.455, sum=0.91 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.45517582014987346\"}", - "Econometrics - # eval": "{\"description\": \"min=114, mean=114, max=114, sum=228 (2)\", \"tab\": \"General information\", \"score\": \"114.0\"}", - "Econometrics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Econometrics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Econometrics - # prompt tokens": "{\"description\": \"min=684.675, mean=684.675, max=684.675, sum=1369.351 (2)\", \"tab\": \"General information\", \"score\": \"684.6754385964912\"}", - "Econometrics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"econometrics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_econometrics\"" - } - } - }, - { - "evaluation_name": "Global Facts", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Global Facts", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.38, - "details": { - "description": "min=0.38, mean=0.38, max=0.38, sum=0.76 (2)", - "tab": "Accuracy", - "Global Facts - Observed inference time (s)": "{\"description\": \"min=0.39, mean=0.39, max=0.39, sum=0.781 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3903778100013733\"}", - "Global Facts - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Global Facts - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Global Facts - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Global Facts - # prompt tokens": "{\"description\": \"min=484.54, mean=484.54, max=484.54, sum=969.08 (2)\", \"tab\": \"General information\", \"score\": \"484.54\"}", - "Global Facts - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"global_facts\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_global_facts\"" - } - } - }, - { - "evaluation_name": "Jurisprudence", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Jurisprudence", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.704, - "details": { - "description": "min=0.704, mean=0.704, max=0.704, sum=1.407 (2)", - "tab": "Accuracy", - "Jurisprudence - Observed inference time (s)": "{\"description\": \"min=0.359, mean=0.359, max=0.359, sum=0.718 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.35898366460093745\"}", - "Jurisprudence - # eval": "{\"description\": \"min=108, mean=108, max=108, sum=216 (2)\", \"tab\": \"General information\", \"score\": \"108.0\"}", - "Jurisprudence - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Jurisprudence - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Jurisprudence - # prompt tokens": "{\"description\": \"min=449.898, mean=449.898, max=449.898, sum=899.796 (2)\", \"tab\": \"General information\", \"score\": \"449.89814814814815\"}", - "Jurisprudence - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"jurisprudence\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_jurisprudence\"" - } - } - }, - { - "evaluation_name": "Philosophy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Philosophy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.672, - "details": { - "description": "min=0.672, mean=0.672, max=0.672, sum=1.344 (2)", - "tab": "Accuracy", - "Philosophy - Observed inference time (s)": "{\"description\": \"min=0.323, mean=0.323, max=0.323, sum=0.645 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3226076184361694\"}", - "Philosophy - # eval": "{\"description\": \"min=311, mean=311, max=311, sum=622 (2)\", \"tab\": \"General information\", \"score\": \"311.0\"}", - "Philosophy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Philosophy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Philosophy - # prompt tokens": "{\"description\": \"min=372.122, mean=372.122, max=372.122, sum=744.244 (2)\", \"tab\": \"General information\", \"score\": \"372.12218649517683\"}", - "Philosophy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"philosophy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_philosophy\"" - } - } - }, - { - "evaluation_name": "Professional Psychology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Professional Psychology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.567, - "details": { - "description": "min=0.567, mean=0.567, max=0.567, sum=1.134 (2)", - "tab": "Accuracy", - "Professional Medicine - Observed inference time (s)": "{\"description\": \"min=0.759, mean=0.759, max=0.759, sum=1.519 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7594411802642486\"}", - "Professional Accounting - Observed inference time (s)": "{\"description\": \"min=0.55, mean=0.55, max=0.55, sum=1.099 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5495186367778914\"}", - "Professional Law - Observed inference time (s)": "{\"description\": \"min=1.102, mean=1.102, max=1.102, sum=2.205 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.1024409701957851\"}", - "Professional Psychology - Observed inference time (s)": "{\"description\": \"min=0.438, mean=0.438, max=0.438, sum=0.875 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.43751365219066346\"}", - "Professional Medicine - # eval": "{\"description\": \"min=272, mean=272, max=272, sum=544 (2)\", \"tab\": \"General information\", \"score\": \"272.0\"}", - "Professional Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Medicine - # prompt tokens": "{\"description\": \"min=1330.647, mean=1330.647, max=1330.647, sum=2661.294 (2)\", \"tab\": \"General information\", \"score\": \"1330.6470588235295\"}", - "Professional Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Accounting - # eval": "{\"description\": \"min=282, mean=282, max=282, sum=564 (2)\", \"tab\": \"General information\", \"score\": \"282.0\"}", - "Professional Accounting - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Accounting - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # prompt tokens": "{\"description\": \"min=823.277, mean=823.277, max=823.277, sum=1646.553 (2)\", \"tab\": \"General information\", \"score\": \"823.2765957446809\"}", - "Professional Accounting - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Law - # eval": "{\"description\": \"min=1534, mean=1534, max=1534, sum=3068 (2)\", \"tab\": \"General information\", \"score\": \"1534.0\"}", - "Professional Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # prompt tokens": "{\"description\": \"min=1915.007, mean=1915.007, max=1915.007, sum=3830.014 (2)\", \"tab\": \"General information\", \"score\": \"1915.0071707953064\"}", - "Professional Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Psychology - # eval": "{\"description\": \"min=612, mean=612, max=612, sum=1224 (2)\", \"tab\": \"General information\", \"score\": \"612.0\"}", - "Professional Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # prompt tokens": "{\"description\": \"min=650.078, mean=650.078, max=650.078, sum=1300.157 (2)\", \"tab\": \"General information\", \"score\": \"650.0784313725491\"}", - "Professional Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"professional_psychology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_professional_psychology\"" - } - } - }, - { - "evaluation_name": "Us Foreign Policy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Us Foreign Policy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.83, - "details": { - "description": "min=0.83, mean=0.83, max=0.83, sum=1.66 (2)", - "tab": "Accuracy", - "Us Foreign Policy - Observed inference time (s)": "{\"description\": \"min=0.391, mean=0.391, max=0.391, sum=0.782 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3909334921836853\"}", - "Us Foreign Policy - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Us Foreign Policy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Us Foreign Policy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Us Foreign Policy - # prompt tokens": "{\"description\": \"min=479.81, mean=479.81, max=479.81, sum=959.62 (2)\", \"tab\": \"General information\", \"score\": \"479.81\"}", - "Us Foreign Policy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"us_foreign_policy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_us_foreign_policy\"" - } - } - }, - { - "evaluation_name": "Astronomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Astronomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.546, - "details": { - "description": "min=0.546, mean=0.546, max=0.546, sum=1.092 (2)", - "tab": "Accuracy", - "Astronomy - Observed inference time (s)": "{\"description\": \"min=0.472, mean=0.472, max=0.472, sum=0.945 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.47229841351509094\"}", - "Astronomy - # eval": "{\"description\": \"min=152, mean=152, max=152, sum=304 (2)\", \"tab\": \"General information\", \"score\": \"152.0\"}", - "Astronomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Astronomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Astronomy - # prompt tokens": "{\"description\": \"min=681.079, mean=681.079, max=681.079, sum=1362.158 (2)\", \"tab\": \"General information\", \"score\": \"681.078947368421\"}", - "Astronomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"astronomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_astronomy\"" - } - } - }, - { - "evaluation_name": "Business Ethics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Business Ethics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.55, - "details": { - "description": "min=0.55, mean=0.55, max=0.55, sum=1.1 (2)", - "tab": "Accuracy", - "Business Ethics - Observed inference time (s)": "{\"description\": \"min=0.476, mean=0.476, max=0.476, sum=0.952 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4758677792549133\"}", - "Business Ethics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Business Ethics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Business Ethics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Business Ethics - # prompt tokens": "{\"description\": \"min=674.44, mean=674.44, max=674.44, sum=1348.88 (2)\", \"tab\": \"General information\", \"score\": \"674.44\"}", - "Business Ethics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"business_ethics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_business_ethics\"" - } - } - }, - { - "evaluation_name": "Clinical Knowledge", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Clinical Knowledge", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.592, - "details": { - "description": "min=0.592, mean=0.592, max=0.592, sum=1.185 (2)", - "tab": "Accuracy", - "Clinical Knowledge - Observed inference time (s)": "{\"description\": \"min=0.386, mean=0.386, max=0.386, sum=0.772 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.38589143843021034\"}", - "Clinical Knowledge - # eval": "{\"description\": \"min=265, mean=265, max=265, sum=530 (2)\", \"tab\": \"General information\", \"score\": \"265.0\"}", - "Clinical Knowledge - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Clinical Knowledge - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Clinical Knowledge - # prompt tokens": "{\"description\": \"min=487.374, mean=487.374, max=487.374, sum=974.747 (2)\", \"tab\": \"General information\", \"score\": \"487.3735849056604\"}", - "Clinical Knowledge - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"clinical_knowledge\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_clinical_knowledge\"" - } - } - }, - { - "evaluation_name": "Conceptual Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Conceptual Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.413, - "details": { - "description": "min=0.413, mean=0.413, max=0.413, sum=0.826 (2)", - "tab": "Accuracy", - "Conceptual Physics - Observed inference time (s)": "{\"description\": \"min=0.48, mean=0.48, max=0.48, sum=0.961 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4802838366082374\"}", - "Conceptual Physics - # eval": "{\"description\": \"min=235, mean=235, max=235, sum=470 (2)\", \"tab\": \"General information\", \"score\": \"235.0\"}", - "Conceptual Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Conceptual Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Conceptual Physics - # prompt tokens": "{\"description\": \"min=333.153, mean=333.153, max=333.153, sum=666.306 (2)\", \"tab\": \"General information\", \"score\": \"333.1531914893617\"}", - "Conceptual Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"conceptual_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_conceptual_physics\"" - } - } - }, - { - "evaluation_name": "Electrical Engineering", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Electrical Engineering", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.49, - "details": { - "description": "min=0.49, mean=0.49, max=0.49, sum=0.979 (2)", - "tab": "Accuracy", - "Electrical Engineering - Observed inference time (s)": "{\"description\": \"min=0.368, mean=0.368, max=0.368, sum=0.737 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.36833986249463313\"}", - "Electrical Engineering - # eval": "{\"description\": \"min=145, mean=145, max=145, sum=290 (2)\", \"tab\": \"General information\", \"score\": \"145.0\"}", - "Electrical Engineering - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Electrical Engineering - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Electrical Engineering - # prompt tokens": "{\"description\": \"min=497.779, mean=497.779, max=497.779, sum=995.559 (2)\", \"tab\": \"General information\", \"score\": \"497.7793103448276\"}", - "Electrical Engineering - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"electrical_engineering\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_electrical_engineering\"" - } - } - }, - { - "evaluation_name": "Elementary Mathematics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Elementary Mathematics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.307, - "details": { - "description": "min=0.307, mean=0.307, max=0.307, sum=0.614 (2)", - "tab": "Accuracy", - "Elementary Mathematics - Observed inference time (s)": "{\"description\": \"min=0.497, mean=0.497, max=0.497, sum=0.995 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.49746112028757733\"}", - "Elementary Mathematics - # eval": "{\"description\": \"min=378, mean=378, max=378, sum=756 (2)\", \"tab\": \"General information\", \"score\": \"378.0\"}", - "Elementary Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Elementary Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Elementary Mathematics - # prompt tokens": "{\"description\": \"min=609.156, mean=609.156, max=609.156, sum=1218.312 (2)\", \"tab\": \"General information\", \"score\": \"609.1560846560847\"}", - "Elementary Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"elementary_mathematics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_elementary_mathematics\"" - } - } - }, - { - "evaluation_name": "Formal Logic", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Formal Logic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.381, - "details": { - "description": "min=0.381, mean=0.381, max=0.381, sum=0.762 (2)", - "tab": "Accuracy", - "Formal Logic - Observed inference time (s)": "{\"description\": \"min=0.444, mean=0.444, max=0.444, sum=0.887 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4436971952044775\"}", - "Formal Logic - # eval": "{\"description\": \"min=126, mean=126, max=126, sum=252 (2)\", \"tab\": \"General information\", \"score\": \"126.0\"}", - "Formal Logic - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Formal Logic - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Formal Logic - # prompt tokens": "{\"description\": \"min=691.81, mean=691.81, max=691.81, sum=1383.619 (2)\", \"tab\": \"General information\", \"score\": \"691.8095238095239\"}", - "Formal Logic - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"formal_logic\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_formal_logic\"" - } - } - }, - { - "evaluation_name": "High School World History", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on High School World History", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.705, - "details": { - "description": "min=0.705, mean=0.705, max=0.705, sum=1.409 (2)", - "tab": "Accuracy", - "High School Biology - Observed inference time (s)": "{\"description\": \"min=0.437, mean=0.437, max=0.437, sum=0.873 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.43674747020967547\"}", - "High School Chemistry - Observed inference time (s)": "{\"description\": \"min=0.423, mean=0.423, max=0.423, sum=0.846 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.42318584883741556\"}", - "High School Computer Science - Observed inference time (s)": "{\"description\": \"min=0.567, mean=0.567, max=0.567, sum=1.133 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5666733002662658\"}", - "High School European History - Observed inference time (s)": "{\"description\": \"min=1.697, mean=1.697, max=1.697, sum=3.394 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.6971724553541703\"}", - "High School Geography - Observed inference time (s)": "{\"description\": \"min=0.532, mean=0.532, max=0.532, sum=1.065 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5323956747247716\"}", - "High School Government And Politics - Observed inference time (s)": "{\"description\": \"min=0.368, mean=0.368, max=0.368, sum=0.735 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.36752033727774347\"}", - "High School Macroeconomics - Observed inference time (s)": "{\"description\": \"min=0.378, mean=0.378, max=0.378, sum=0.756 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3781696270673703\"}", - "High School Mathematics - Observed inference time (s)": "{\"description\": \"min=0.402, mean=0.402, max=0.402, sum=0.803 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4017471119209572\"}", - "High School Microeconomics - Observed inference time (s)": "{\"description\": \"min=0.36, mean=0.36, max=0.36, sum=0.721 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3603636326910067\"}", - "High School Physics - Observed inference time (s)": "{\"description\": \"min=0.429, mean=0.429, max=0.429, sum=0.858 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4290682780032126\"}", - "High School Psychology - Observed inference time (s)": "{\"description\": \"min=0.423, mean=0.423, max=0.423, sum=0.846 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.42302281703424016\"}", - "High School Statistics - Observed inference time (s)": "{\"description\": \"min=0.535, mean=0.535, max=0.535, sum=1.069 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.534513204186051\"}", - "High School US History - Observed inference time (s)": "{\"description\": \"min=1.332, mean=1.332, max=1.332, sum=2.665 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.33243932910994\"}", - "High School World History - Observed inference time (s)": "{\"description\": \"min=0.866, mean=0.866, max=0.866, sum=1.733 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8663106930406788\"}", - "High School Biology - # eval": "{\"description\": \"min=310, mean=310, max=310, sum=620 (2)\", \"tab\": \"General information\", \"score\": \"310.0\"}", - "High School Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Biology - # prompt tokens": "{\"description\": \"min=596.894, mean=596.894, max=596.894, sum=1193.787 (2)\", \"tab\": \"General information\", \"score\": \"596.8935483870968\"}", - "High School Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Chemistry - # eval": "{\"description\": \"min=203, mean=203, max=203, sum=406 (2)\", \"tab\": \"General information\", \"score\": \"203.0\"}", - "High School Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # prompt tokens": "{\"description\": \"min=568.665, mean=568.665, max=568.665, sum=1137.33 (2)\", \"tab\": \"General information\", \"score\": \"568.6650246305419\"}", - "High School Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "High School Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # prompt tokens": "{\"description\": \"min=988.57, mean=988.57, max=988.57, sum=1977.14 (2)\", \"tab\": \"General information\", \"score\": \"988.57\"}", - "High School Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School European History - # eval": "{\"description\": \"min=165, mean=165, max=165, sum=330 (2)\", \"tab\": \"General information\", \"score\": \"165.0\"}", - "High School European History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School European History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # prompt tokens": "{\"description\": \"min=3159.636, mean=3159.636, max=3159.636, sum=6319.273 (2)\", \"tab\": \"General information\", \"score\": \"3159.6363636363635\"}", - "High School European History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Geography - # eval": "{\"description\": \"min=198, mean=198, max=198, sum=396 (2)\", \"tab\": \"General information\", \"score\": \"198.0\"}", - "High School Geography - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Geography - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # prompt tokens": "{\"description\": \"min=436.657, mean=436.657, max=436.657, sum=873.313 (2)\", \"tab\": \"General information\", \"score\": \"436.65656565656565\"}", - "High School Geography - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Government And Politics - # eval": "{\"description\": \"min=193, mean=193, max=193, sum=386 (2)\", \"tab\": \"General information\", \"score\": \"193.0\"}", - "High School Government And Politics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Government And Politics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # prompt tokens": "{\"description\": \"min=527.927, mean=527.927, max=527.927, sum=1055.855 (2)\", \"tab\": \"General information\", \"score\": \"527.9274611398964\"}", - "High School Government And Politics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Macroeconomics - # eval": "{\"description\": \"min=390, mean=390, max=390, sum=780 (2)\", \"tab\": \"General information\", \"score\": \"390.0\"}", - "High School Macroeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Macroeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # prompt tokens": "{\"description\": \"min=445.662, mean=445.662, max=445.662, sum=891.323 (2)\", \"tab\": \"General information\", \"score\": \"445.66153846153844\"}", - "High School Macroeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Mathematics - # eval": "{\"description\": \"min=270, mean=270, max=270, sum=540 (2)\", \"tab\": \"General information\", \"score\": \"270.0\"}", - "High School Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # prompt tokens": "{\"description\": \"min=579.181, mean=579.181, max=579.181, sum=1158.363 (2)\", \"tab\": \"General information\", \"score\": \"579.1814814814815\"}", - "High School Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Microeconomics - # eval": "{\"description\": \"min=238, mean=238, max=238, sum=476 (2)\", \"tab\": \"General information\", \"score\": \"238.0\"}", - "High School Microeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Microeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # prompt tokens": "{\"description\": \"min=449.492, mean=449.492, max=449.492, sum=898.983 (2)\", \"tab\": \"General information\", \"score\": \"449.49159663865544\"}", - "High School Microeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Physics - # eval": "{\"description\": \"min=151, mean=151, max=151, sum=302 (2)\", \"tab\": \"General information\", \"score\": \"151.0\"}", - "High School Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # prompt tokens": "{\"description\": \"min=621.788, mean=621.788, max=621.788, sum=1243.576 (2)\", \"tab\": \"General information\", \"score\": \"621.7880794701987\"}", - "High School Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Psychology - # eval": "{\"description\": \"min=545, mean=545, max=545, sum=1090 (2)\", \"tab\": \"General information\", \"score\": \"545.0\"}", - "High School Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # prompt tokens": "{\"description\": \"min=585.919, mean=585.919, max=585.919, sum=1171.839 (2)\", \"tab\": \"General information\", \"score\": \"585.9192660550459\"}", - "High School Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Statistics - # eval": "{\"description\": \"min=216, mean=216, max=216, sum=432 (2)\", \"tab\": \"General information\", \"score\": \"216.0\"}", - "High School Statistics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Statistics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # prompt tokens": "{\"description\": \"min=908.208, mean=908.208, max=908.208, sum=1816.417 (2)\", \"tab\": \"General information\", \"score\": \"908.2083333333334\"}", - "High School Statistics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School US History - # eval": "{\"description\": \"min=204, mean=204, max=204, sum=408 (2)\", \"tab\": \"General information\", \"score\": \"204.0\"}", - "High School US History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School US History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # prompt tokens": "{\"description\": \"min=2535.324, mean=2535.324, max=2535.324, sum=5070.647 (2)\", \"tab\": \"General information\", \"score\": \"2535.323529411765\"}", - "High School US History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School World History - # eval": "{\"description\": \"min=237, mean=237, max=237, sum=474 (2)\", \"tab\": \"General information\", \"score\": \"237.0\"}", - "High School World History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School World History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # prompt tokens": "{\"description\": \"min=1638.219, mean=1638.219, max=1638.219, sum=3276.439 (2)\", \"tab\": \"General information\", \"score\": \"1638.2194092827003\"}", - "High School World History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"high_school_world_history\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_high_school_world_history\"" - } - } - }, - { - "evaluation_name": "Human Sexuality", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Human Sexuality", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.618, - "details": { - "description": "min=0.618, mean=0.618, max=0.618, sum=1.237 (2)", - "tab": "Accuracy", - "Human Aging - Observed inference time (s)": "{\"description\": \"min=0.473, mean=0.473, max=0.473, sum=0.947 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.47327254385157014\"}", - "Human Sexuality - Observed inference time (s)": "{\"description\": \"min=0.368, mean=0.368, max=0.368, sum=0.737 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3683396113737849\"}", - "Human Aging - # eval": "{\"description\": \"min=223, mean=223, max=223, sum=446 (2)\", \"tab\": \"General information\", \"score\": \"223.0\"}", - "Human Aging - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Aging - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Aging - # prompt tokens": "{\"description\": \"min=361.26, mean=361.26, max=361.26, sum=722.52 (2)\", \"tab\": \"General information\", \"score\": \"361.26008968609864\"}", - "Human Aging - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Human Sexuality - # eval": "{\"description\": \"min=131, mean=131, max=131, sum=262 (2)\", \"tab\": \"General information\", \"score\": \"131.0\"}", - "Human Sexuality - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Sexuality - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # prompt tokens": "{\"description\": \"min=403.382, mean=403.382, max=403.382, sum=806.763 (2)\", \"tab\": \"General information\", \"score\": \"403.381679389313\"}", - "Human Sexuality - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"human_sexuality\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_human_sexuality\"" - } - } - }, - { - "evaluation_name": "International Law", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on International Law", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.752, - "details": { - "description": "min=0.752, mean=0.752, max=0.752, sum=1.504 (2)", - "tab": "Accuracy", - "International Law - Observed inference time (s)": "{\"description\": \"min=0.488, mean=0.488, max=0.488, sum=0.975 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.48763008551164105\"}", - "International Law - # eval": "{\"description\": \"min=121, mean=121, max=121, sum=242 (2)\", \"tab\": \"General information\", \"score\": \"121.0\"}", - "International Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "International Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "International Law - # prompt tokens": "{\"description\": \"min=729.463, mean=729.463, max=729.463, sum=1458.926 (2)\", \"tab\": \"General information\", \"score\": \"729.4628099173553\"}", - "International Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"international_law\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_international_law\"" - } - } - }, - { - "evaluation_name": "Logical Fallacies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Logical Fallacies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.687, - "details": { - "description": "min=0.687, mean=0.687, max=0.687, sum=1.374 (2)", - "tab": "Accuracy", - "Logical Fallacies - Observed inference time (s)": "{\"description\": \"min=0.361, mean=0.361, max=0.361, sum=0.722 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3607579462367333\"}", - "Logical Fallacies - # eval": "{\"description\": \"min=163, mean=163, max=163, sum=326 (2)\", \"tab\": \"General information\", \"score\": \"163.0\"}", - "Logical Fallacies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Logical Fallacies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Logical Fallacies - # prompt tokens": "{\"description\": \"min=502.755, mean=502.755, max=502.755, sum=1005.509 (2)\", \"tab\": \"General information\", \"score\": \"502.7546012269939\"}", - "Logical Fallacies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"logical_fallacies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_logical_fallacies\"" - } - } - }, - { - "evaluation_name": "Machine Learning", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Machine Learning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.286, - "details": { - "description": "min=0.286, mean=0.286, max=0.286, sum=0.571 (2)", - "tab": "Accuracy", - "Machine Learning - Observed inference time (s)": "{\"description\": \"min=0.478, mean=0.478, max=0.478, sum=0.955 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4776035504681723\"}", - "Machine Learning - # eval": "{\"description\": \"min=112, mean=112, max=112, sum=224 (2)\", \"tab\": \"General information\", \"score\": \"112.0\"}", - "Machine Learning - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Machine Learning - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Machine Learning - # prompt tokens": "{\"description\": \"min=730.402, mean=730.402, max=730.402, sum=1460.804 (2)\", \"tab\": \"General information\", \"score\": \"730.4017857142857\"}", - "Machine Learning - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"machine_learning\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_machine_learning\"" - } - } - }, - { - "evaluation_name": "Management", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Management", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.738, - "details": { - "description": "min=0.738, mean=0.738, max=0.738, sum=1.476 (2)", - "tab": "Accuracy", - "Management - Observed inference time (s)": "{\"description\": \"min=0.343, mean=0.343, max=0.343, sum=0.686 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.34303417715054113\"}", - "Management - # eval": "{\"description\": \"min=103, mean=103, max=103, sum=206 (2)\", \"tab\": \"General information\", \"score\": \"103.0\"}", - "Management - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Management - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Management - # prompt tokens": "{\"description\": \"min=315.777, mean=315.777, max=315.777, sum=631.553 (2)\", \"tab\": \"General information\", \"score\": \"315.77669902912623\"}", - "Management - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"management\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_management\"" - } - } - }, - { - "evaluation_name": "Marketing", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Marketing", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.786, - "details": { - "description": "min=0.786, mean=0.786, max=0.786, sum=1.573 (2)", - "tab": "Accuracy", - "Marketing - Observed inference time (s)": "{\"description\": \"min=0.374, mean=0.374, max=0.374, sum=0.749 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.37440858845017916\"}", - "Marketing - # eval": "{\"description\": \"min=234, mean=234, max=234, sum=468 (2)\", \"tab\": \"General information\", \"score\": \"234.0\"}", - "Marketing - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Marketing - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Marketing - # prompt tokens": "{\"description\": \"min=472.628, mean=472.628, max=472.628, sum=945.256 (2)\", \"tab\": \"General information\", \"score\": \"472.62820512820514\"}", - "Marketing - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"marketing\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_marketing\"" - } - } - }, - { - "evaluation_name": "Medical Genetics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Medical Genetics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.57, - "details": { - "description": "min=0.57, mean=0.57, max=0.57, sum=1.14 (2)", - "tab": "Accuracy", - "Medical Genetics - Observed inference time (s)": "{\"description\": \"min=0.365, mean=0.365, max=0.365, sum=0.73 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3651238298416138\"}", - "Medical Genetics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Medical Genetics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Medical Genetics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Medical Genetics - # prompt tokens": "{\"description\": \"min=408.14, mean=408.14, max=408.14, sum=816.28 (2)\", \"tab\": \"General information\", \"score\": \"408.14\"}", - "Medical Genetics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"medical_genetics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_medical_genetics\"" - } - } - }, - { - "evaluation_name": "Miscellaneous", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Miscellaneous", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.748, - "details": { - "description": "min=0.748, mean=0.748, max=0.748, sum=1.497 (2)", - "tab": "Accuracy", - "Miscellaneous - Observed inference time (s)": "{\"description\": \"min=0.342, mean=0.342, max=0.342, sum=0.684 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.34193715342768916\"}", - "Miscellaneous - # eval": "{\"description\": \"min=783, mean=783, max=783, sum=1566 (2)\", \"tab\": \"General information\", \"score\": \"783.0\"}", - "Miscellaneous - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Miscellaneous - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Miscellaneous - # prompt tokens": "{\"description\": \"min=345.913, mean=345.913, max=345.913, sum=691.826 (2)\", \"tab\": \"General information\", \"score\": \"345.9131545338442\"}", - "Miscellaneous - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"miscellaneous\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_miscellaneous\"" - } - } - }, - { - "evaluation_name": "Moral Scenarios", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Moral Scenarios", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.407, - "details": { - "description": "min=0.407, mean=0.407, max=0.407, sum=0.813 (2)", - "tab": "Accuracy", - "Moral Disputes - Observed inference time (s)": "{\"description\": \"min=0.421, mean=0.421, max=0.421, sum=0.841 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4205500893510146\"}", - "Moral Scenarios - Observed inference time (s)": "{\"description\": \"min=0.474, mean=0.474, max=0.474, sum=0.949 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4744861464260677\"}", - "Moral Disputes - # eval": "{\"description\": \"min=346, mean=346, max=346, sum=692 (2)\", \"tab\": \"General information\", \"score\": \"346.0\"}", - "Moral Disputes - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Disputes - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Disputes - # prompt tokens": "{\"description\": \"min=542.506, mean=542.506, max=542.506, sum=1085.012 (2)\", \"tab\": \"General information\", \"score\": \"542.5057803468208\"}", - "Moral Disputes - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Moral Scenarios - # eval": "{\"description\": \"min=895, mean=895, max=895, sum=1790 (2)\", \"tab\": \"General information\", \"score\": \"895.0\"}", - "Moral Scenarios - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Scenarios - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # prompt tokens": "{\"description\": \"min=756.479, mean=756.479, max=756.479, sum=1512.959 (2)\", \"tab\": \"General information\", \"score\": \"756.4793296089385\"}", - "Moral Scenarios - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"moral_scenarios\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_moral_scenarios\"" - } - } - }, - { - "evaluation_name": "Nutrition", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Nutrition", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.627, - "details": { - "description": "min=0.627, mean=0.627, max=0.627, sum=1.255 (2)", - "tab": "Accuracy", - "Nutrition - Observed inference time (s)": "{\"description\": \"min=0.453, mean=0.453, max=0.453, sum=0.906 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4530853640799429\"}", - "Nutrition - # eval": "{\"description\": \"min=306, mean=306, max=306, sum=612 (2)\", \"tab\": \"General information\", \"score\": \"306.0\"}", - "Nutrition - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Nutrition - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Nutrition - # prompt tokens": "{\"description\": \"min=695.922, mean=695.922, max=695.922, sum=1391.843 (2)\", \"tab\": \"General information\", \"score\": \"695.9215686274509\"}", - "Nutrition - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"nutrition\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_nutrition\"" - } - } - }, - { - "evaluation_name": "Prehistory", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Prehistory", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.654, - "details": { - "description": "min=0.654, mean=0.654, max=0.654, sum=1.309 (2)", - "tab": "Accuracy", - "Prehistory - Observed inference time (s)": "{\"description\": \"min=0.445, mean=0.445, max=0.445, sum=0.889 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.44473813345402846\"}", - "Prehistory - # eval": "{\"description\": \"min=324, mean=324, max=324, sum=648 (2)\", \"tab\": \"General information\", \"score\": \"324.0\"}", - "Prehistory - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Prehistory - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Prehistory - # prompt tokens": "{\"description\": \"min=619.185, mean=619.185, max=619.185, sum=1238.37 (2)\", \"tab\": \"General information\", \"score\": \"619.1851851851852\"}", - "Prehistory - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"prehistory\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_prehistory\"" - } - } - }, - { - "evaluation_name": "Public Relations", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Public Relations", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6, - "details": { - "description": "min=0.6, mean=0.6, max=0.6, sum=1.2 (2)", - "tab": "Accuracy", - "Public Relations - Observed inference time (s)": "{\"description\": \"min=0.387, mean=0.387, max=0.387, sum=0.774 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.38679331866177646\"}", - "Public Relations - # eval": "{\"description\": \"min=110, mean=110, max=110, sum=220 (2)\", \"tab\": \"General information\", \"score\": \"110.0\"}", - "Public Relations - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Public Relations - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Public Relations - # prompt tokens": "{\"description\": \"min=474.827, mean=474.827, max=474.827, sum=949.655 (2)\", \"tab\": \"General information\", \"score\": \"474.8272727272727\"}", - "Public Relations - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"public_relations\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_public_relations\"" - } - } - }, - { - "evaluation_name": "Security Studies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Security Studies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.608, - "details": { - "description": "min=0.608, mean=0.608, max=0.608, sum=1.216 (2)", - "tab": "Accuracy", - "Security Studies - Observed inference time (s)": "{\"description\": \"min=0.771, mean=0.771, max=0.771, sum=1.542 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7707553902450873\"}", - "Security Studies - # eval": "{\"description\": \"min=245, mean=245, max=245, sum=490 (2)\", \"tab\": \"General information\", \"score\": \"245.0\"}", - "Security Studies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Security Studies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Security Studies - # prompt tokens": "{\"description\": \"min=1377.531, mean=1377.531, max=1377.531, sum=2755.061 (2)\", \"tab\": \"General information\", \"score\": \"1377.530612244898\"}", - "Security Studies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"security_studies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_security_studies\"" - } - } - }, - { - "evaluation_name": "Sociology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Sociology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.761, - "details": { - "description": "min=0.761, mean=0.761, max=0.761, sum=1.522 (2)", - "tab": "Accuracy", - "Sociology - Observed inference time (s)": "{\"description\": \"min=0.385, mean=0.385, max=0.385, sum=0.77 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.38491436853930727\"}", - "Sociology - # eval": "{\"description\": \"min=201, mean=201, max=201, sum=402 (2)\", \"tab\": \"General information\", \"score\": \"201.0\"}", - "Sociology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Sociology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Sociology - # prompt tokens": "{\"description\": \"min=508.478, mean=508.478, max=508.478, sum=1016.955 (2)\", \"tab\": \"General information\", \"score\": \"508.4776119402985\"}", - "Sociology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"sociology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_sociology\"" - } - } - }, - { - "evaluation_name": "Virology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Virology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.476, - "details": { - "description": "min=0.476, mean=0.476, max=0.476, sum=0.952 (2)", - "tab": "Accuracy", - "Virology - Observed inference time (s)": "{\"description\": \"min=0.35, mean=0.35, max=0.35, sum=0.7 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3499309801193605\"}", - "Virology - # eval": "{\"description\": \"min=166, mean=166, max=166, sum=332 (2)\", \"tab\": \"General information\", \"score\": \"166.0\"}", - "Virology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Virology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Virology - # prompt tokens": "{\"description\": \"min=405.108, mean=405.108, max=405.108, sum=810.217 (2)\", \"tab\": \"General information\", \"score\": \"405.10843373493975\"}", - "Virology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"virology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_virology\"" - } - } - }, - { - "evaluation_name": "World Religions", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on World Religions", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.76, - "details": { - "description": "min=0.76, mean=0.76, max=0.76, sum=1.52 (2)", - "tab": "Accuracy", - "World Religions - Observed inference time (s)": "{\"description\": \"min=0.338, mean=0.338, max=0.338, sum=0.675 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.33768263197781745\"}", - "World Religions - # eval": "{\"description\": \"min=171, mean=171, max=171, sum=342 (2)\", \"tab\": \"General information\", \"score\": \"171.0\"}", - "World Religions - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "World Religions - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "World Religions - # prompt tokens": "{\"description\": \"min=304.474, mean=304.474, max=304.474, sum=608.947 (2)\", \"tab\": \"General information\", \"score\": \"304.4736842105263\"}", - "World Religions - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"world_religions\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_world_religions\"" - } - } - }, - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.502, - "details": { - "description": "", - "tab": "Efficiency" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/meta_llama-2-70b.json b/data/models/meta_llama-2-70b.json deleted file mode 100644 index 1c313a3d7c6593e7e5e92619c93c74d1117633f4..0000000000000000000000000000000000000000 --- a/data/models/meta_llama-2-70b.json +++ /dev/null @@ -1,2570 +0,0 @@ -{ - "model_info": { - "name": "Llama 2 70B", - "id": "meta/llama-2-70b", - "developer": "Meta", - "inference_platform": "unknown", - "additional_details": { - "model_id_aliases": [ - "meta/Llama-2-70B" - ] - } - }, - "evaluations": [ - { - "evaluation_id": "helm_classic/meta_Llama-2-70B/1774096308.339228", - "retrieved_timestamp": "1774096308.339228", - "source_metadata": { - "source_name": "helm_classic", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_classic", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_classic", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperform on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.944, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Calibration": "{\"description\": \"\", \"tab\": \"Calibration\", \"score\": \"\"}", - "Mean win rate - Robustness": "{\"description\": \"\", \"tab\": \"Robustness\", \"score\": \"0.9649184149184149\"}", - "Mean win rate - Fairness": "{\"description\": \"\", \"tab\": \"Fairness\", \"score\": \"0.9587645687645687\"}", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}", - "Mean win rate - Bias": "{\"description\": \"\", \"tab\": \"Bias\", \"score\": \"0.5375895851224799\"}", - "Mean win rate - Toxicity": "{\"description\": \"\", \"tab\": \"Toxicity\", \"score\": \"0.643006993006993\"}", - "Mean win rate - Summarization metrics": "{\"description\": \"\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.582, - "details": { - "description": "min=0.29, mean=0.582, max=0.92, sum=2.909 (5)", - "tab": "Accuracy", - "MMLU - ECE (10-bin)": "{\"description\": \"5 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "MMLU - EM (Robustness)": "{\"description\": \"min=0.22, mean=0.545, max=0.9, sum=2.726 (5)\", \"tab\": \"Robustness\", \"score\": \"0.5451929824561403\"}", - "MMLU - EM (Fairness)": "{\"description\": \"min=0.26, mean=0.557, max=0.91, sum=2.786 (5)\", \"tab\": \"Fairness\", \"score\": \"0.5571929824561404\"}", - "MMLU - Denoised inference time (s)": "{\"description\": \"5 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=397.65, mean=522.547, max=684.675, sum=2612.735 (5)\", \"tab\": \"General information\", \"score\": \"522.5470877192982\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "MMLU - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "BoolQ", - "source_data": { - "dataset_name": "BoolQ", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on BoolQ", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.886, - "details": { - "description": "min=0.886, mean=0.886, max=0.886, sum=0.886 (1)", - "tab": "Accuracy", - "BoolQ - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "BoolQ - EM (Robustness)": "{\"description\": \"min=0.863, mean=0.863, max=0.863, sum=0.863 (1)\", \"tab\": \"Robustness\", \"score\": \"0.863\"}", - "BoolQ - EM (Fairness)": "{\"description\": \"min=0.859, mean=0.859, max=0.859, sum=0.859 (1)\", \"tab\": \"Fairness\", \"score\": \"0.859\"}", - "BoolQ - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "BoolQ - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "BoolQ - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "BoolQ - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "BoolQ - # prompt tokens": "{\"description\": \"min=1439.447, mean=1439.447, max=1439.447, sum=1439.447 (1)\", \"tab\": \"General information\", \"score\": \"1439.447\"}", - "BoolQ - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "BoolQ - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "BoolQ - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.77, - "details": { - "description": "min=0.77, mean=0.77, max=0.77, sum=0.77 (1)", - "tab": "Accuracy", - "NarrativeQA - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "NarrativeQA - F1 (Robustness)": "{\"description\": \"min=0.722, mean=0.722, max=0.722, sum=0.722 (1)\", \"tab\": \"Robustness\", \"score\": \"0.7215317388650366\"}", - "NarrativeQA - F1 (Fairness)": "{\"description\": \"min=0.709, mean=0.709, max=0.709, sum=0.709 (1)\", \"tab\": \"Fairness\", \"score\": \"0.709497495841271\"}", - "NarrativeQA - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=4.414, mean=4.414, max=4.414, sum=4.414 (1)\", \"tab\": \"General information\", \"score\": \"4.414084507042253\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=3673.268, mean=3673.268, max=3673.268, sum=3673.268 (1)\", \"tab\": \"General information\", \"score\": \"3673.2676056338028\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NarrativeQA - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NarrativeQA - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NarrativeQA - Stereotypes (gender)": "{\"description\": \"min=0.5, mean=0.5, max=0.5, sum=0.5 (1)\", \"tab\": \"Bias\", \"score\": \"0.5\"}", - "NarrativeQA - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NarrativeQA - Representation (gender)": "{\"description\": \"min=0.187, mean=0.187, max=0.187, sum=0.187 (1)\", \"tab\": \"Bias\", \"score\": \"0.18695652173913044\"}", - "NarrativeQA - Toxic fraction": "{\"description\": \"min=0.008, mean=0.008, max=0.008, sum=0.008 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.008450704225352112\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (open-book)", - "source_data": { - "dataset_name": "NaturalQuestions (open-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (open-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.674, - "details": { - "description": "min=0.674, mean=0.674, max=0.674, sum=0.674 (1)", - "tab": "Accuracy", - "NaturalQuestions (closed-book) - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "NaturalQuestions (open-book) - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - F1 (Robustness)": "{\"description\": \"min=0.42, mean=0.42, max=0.42, sum=0.42 (1)\", \"tab\": \"Robustness\", \"score\": \"0.42009390434309946\"}", - "NaturalQuestions (open-book) - F1 (Robustness)": "{\"description\": \"min=0.639, mean=0.639, max=0.639, sum=0.639 (1)\", \"tab\": \"Robustness\", \"score\": \"0.6385366212170214\"}", - "NaturalQuestions (closed-book) - F1 (Fairness)": "{\"description\": \"min=0.4, mean=0.4, max=0.4, sum=0.4 (1)\", \"tab\": \"Fairness\", \"score\": \"0.3997609830959401\"}", - "NaturalQuestions (open-book) - F1 (Fairness)": "{\"description\": \"min=0.637, mean=0.637, max=0.637, sum=0.637 (1)\", \"tab\": \"Fairness\", \"score\": \"0.6365724774019619\"}", - "NaturalQuestions (closed-book) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NaturalQuestions (open-book) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=137.383, mean=137.383, max=137.383, sum=137.383 (1)\", \"tab\": \"General information\", \"score\": \"137.383\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NaturalQuestions (closed-book) - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.831, mean=4.831, max=4.831, sum=4.831 (1)\", \"tab\": \"General information\", \"score\": \"4.831\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.026, mean=0.026, max=0.026, sum=0.026 (1)\", \"tab\": \"General information\", \"score\": \"0.026\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=2289.409, mean=2289.409, max=2289.409, sum=2289.409 (1)\", \"tab\": \"General information\", \"score\": \"2289.409\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=0.998, mean=0.998, max=0.998, sum=0.998 (1)\", \"tab\": \"General information\", \"score\": \"0.998\"}", - "NaturalQuestions (open-book) - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NaturalQuestions (closed-book) - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=0.667 (1)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "NaturalQuestions (closed-book) - Stereotypes (gender)": "{\"description\": \"min=0.167, mean=0.167, max=0.167, sum=0.167 (1)\", \"tab\": \"Bias\", \"score\": \"0.16666666666666666\"}", - "NaturalQuestions (closed-book) - Representation (race)": "{\"description\": \"min=0.524, mean=0.524, max=0.524, sum=0.524 (1)\", \"tab\": \"Bias\", \"score\": \"0.5238095238095237\"}", - "NaturalQuestions (closed-book) - Representation (gender)": "{\"description\": \"min=0.312, mean=0.312, max=0.312, sum=0.312 (1)\", \"tab\": \"Bias\", \"score\": \"0.3125\"}", - "NaturalQuestions (open-book) - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NaturalQuestions (open-book) - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NaturalQuestions (open-book) - Representation (race)": "{\"description\": \"min=0.566, mean=0.566, max=0.566, sum=0.566 (1)\", \"tab\": \"Bias\", \"score\": \"0.5655430711610487\"}", - "NaturalQuestions (open-book) - Representation (gender)": "{\"description\": \"min=0.184, mean=0.184, max=0.184, sum=0.184 (1)\", \"tab\": \"Bias\", \"score\": \"0.1842105263157895\"}", - "NaturalQuestions (closed-book) - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}", - "NaturalQuestions (open-book) - Toxic fraction": "{\"description\": \"min=0.002, mean=0.002, max=0.002, sum=0.002 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.002\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "QuAC", - "source_data": { - "dataset_name": "QuAC", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on QuAC", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.484, - "details": { - "description": "min=0.484, mean=0.484, max=0.484, sum=0.484 (1)", - "tab": "Accuracy", - "QuAC - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "QuAC - F1 (Robustness)": "{\"description\": \"min=0.362, mean=0.362, max=0.362, sum=0.362 (1)\", \"tab\": \"Robustness\", \"score\": \"0.36189050917141447\"}", - "QuAC - F1 (Fairness)": "{\"description\": \"min=0.414, mean=0.414, max=0.414, sum=0.414 (1)\", \"tab\": \"Fairness\", \"score\": \"0.4139340894194124\"}", - "QuAC - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "QuAC - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "QuAC - # train": "{\"description\": \"min=3.204, mean=3.204, max=3.204, sum=3.204 (1)\", \"tab\": \"General information\", \"score\": \"3.204\"}", - "QuAC - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "QuAC - # prompt tokens": "{\"description\": \"min=3617.038, mean=3617.038, max=3617.038, sum=3617.038 (1)\", \"tab\": \"General information\", \"score\": \"3617.038\"}", - "QuAC - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "QuAC - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "QuAC - Stereotypes (race)": "{\"description\": \"min=0.611, mean=0.611, max=0.611, sum=0.611 (1)\", \"tab\": \"Bias\", \"score\": \"0.6111111111111112\"}", - "QuAC - Stereotypes (gender)": "{\"description\": \"min=0.403, mean=0.403, max=0.403, sum=0.403 (1)\", \"tab\": \"Bias\", \"score\": \"0.4025455927051672\"}", - "QuAC - Representation (race)": "{\"description\": \"min=0.272, mean=0.272, max=0.272, sum=0.272 (1)\", \"tab\": \"Bias\", \"score\": \"0.27183271832718325\"}", - "QuAC - Representation (gender)": "{\"description\": \"min=0.239, mean=0.239, max=0.239, sum=0.239 (1)\", \"tab\": \"Bias\", \"score\": \"0.23913043478260873\"}", - "QuAC - Toxic fraction": "{\"description\": \"min=0.001, mean=0.001, max=0.001, sum=0.001 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.001\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "HellaSwag", - "source_data": { - "dataset_name": "HellaSwag", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on HellaSwag", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "HellaSwag - ECE (10-bin)": "{\"description\": \"No matching runs\", \"tab\": \"Calibration\", \"score\": \"\"}", - "HellaSwag - EM (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "HellaSwag - EM (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "HellaSwag - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "HellaSwag - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "OpenbookQA - ECE (10-bin)": "{\"description\": \"No matching runs\", \"tab\": \"Calibration\", \"score\": \"\"}", - "OpenbookQA - EM (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "OpenbookQA - EM (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "OpenbookQA - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "OpenbookQA - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "TruthfulQA", - "source_data": { - "dataset_name": "TruthfulQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on TruthfulQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.554, - "details": { - "description": "min=0.554, mean=0.554, max=0.554, sum=0.554 (1)", - "tab": "Accuracy", - "TruthfulQA - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "TruthfulQA - EM (Robustness)": "{\"description\": \"min=0.468, mean=0.468, max=0.468, sum=0.468 (1)\", \"tab\": \"Robustness\", \"score\": \"0.46788990825688076\"}", - "TruthfulQA - EM (Fairness)": "{\"description\": \"min=0.434, mean=0.434, max=0.434, sum=0.434 (1)\", \"tab\": \"Fairness\", \"score\": \"0.43425076452599387\"}", - "TruthfulQA - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "TruthfulQA - # eval": "{\"description\": \"min=654, mean=654, max=654, sum=654 (1)\", \"tab\": \"General information\", \"score\": \"654.0\"}", - "TruthfulQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "TruthfulQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "TruthfulQA - # prompt tokens": "{\"description\": \"min=524.602, mean=524.602, max=524.602, sum=524.602 (1)\", \"tab\": \"General information\", \"score\": \"524.6024464831804\"}", - "TruthfulQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "TruthfulQA - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MS MARCO (TREC)", - "source_data": { - "dataset_name": "MS MARCO (TREC)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "NDCG@10 on MS MARCO (TREC)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "MS MARCO (regular) - RR@10 (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "MS MARCO (TREC) - NDCG@10 (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "MS MARCO (regular) - RR@10 (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "MS MARCO (TREC) - NDCG@10 (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "MS MARCO (regular) - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MS MARCO (TREC) - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MS MARCO (regular) - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "MS MARCO (TREC) - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CNN/DailyMail", - "source_data": { - "dataset_name": "CNN/DailyMail", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on CNN/DailyMail", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "CNN/DailyMail - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "CNN/DailyMail - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "CNN/DailyMail - SummaC": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - QAFactEval": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - BERTScore (F1)": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - Coverage": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - Density": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - Compression": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-faithfulness": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-relevance": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-coherence": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "XSUM", - "source_data": { - "dataset_name": "XSUM", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on XSUM", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "XSUM - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "XSUM - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "XSUM - SummaC": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - QAFactEval": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - BERTScore (F1)": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - Coverage": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - Density": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - Compression": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-faithfulness": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-relevance": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-coherence": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "IMDB", - "source_data": { - "dataset_name": "IMDB", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on IMDB", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.961, - "details": { - "description": "min=0.961, mean=0.961, max=0.961, sum=0.961 (1)", - "tab": "Accuracy", - "IMDB - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "IMDB - EM (Robustness)": "{\"description\": \"min=0.949, mean=0.949, max=0.949, sum=0.949 (1)\", \"tab\": \"Robustness\", \"score\": \"0.949\"}", - "IMDB - EM (Fairness)": "{\"description\": \"min=0.954, mean=0.954, max=0.954, sum=0.954 (1)\", \"tab\": \"Fairness\", \"score\": \"0.954\"}", - "IMDB - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "IMDB - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "IMDB - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "IMDB - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IMDB - # prompt tokens": "{\"description\": \"min=2897.409, mean=2897.409, max=2897.409, sum=2897.409 (1)\", \"tab\": \"General information\", \"score\": \"2897.409\"}", - "IMDB - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "IMDB - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "IMDB - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CivilComments", - "source_data": { - "dataset_name": "CivilComments", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on CivilComments", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.652, - "details": { - "description": "min=0.337, mean=0.652, max=0.919, sum=11.733 (18)", - "tab": "Accuracy", - "CivilComments - ECE (10-bin)": "{\"description\": \"9 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "CivilComments - EM (Robustness)": "{\"description\": \"min=0.272, mean=0.59, max=0.884, sum=10.619 (18)\", \"tab\": \"Robustness\", \"score\": \"0.5899239945803259\"}", - "CivilComments - EM (Fairness)": "{\"description\": \"min=0.125, mean=0.551, max=0.892, sum=9.924 (18)\", \"tab\": \"Fairness\", \"score\": \"0.551334119704094\"}", - "CivilComments - Denoised inference time (s)": "{\"description\": \"9 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "CivilComments - # eval": "{\"description\": \"min=74, mean=371.556, max=683, sum=6688 (18)\", \"tab\": \"General information\", \"score\": \"371.55555555555554\"}", - "CivilComments - # train": "{\"description\": \"min=5, mean=5, max=5, sum=90 (18)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CivilComments - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (18)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CivilComments - # prompt tokens": "{\"description\": \"min=404.732, mean=855.241, max=1417.567, sum=15394.339 (18)\", \"tab\": \"General information\", \"score\": \"855.2410378605821\"}", - "CivilComments - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=18 (18)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "CivilComments - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=18 (18)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "CivilComments - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Toxic fraction": "{\"description\": \"9 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "RAFT", - "source_data": { - "dataset_name": "RAFT", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on RAFT", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.727, - "details": { - "description": "min=0.125, mean=0.727, max=0.975, sum=8 (11)", - "tab": "Accuracy", - "RAFT - ECE (10-bin)": "{\"description\": \"11 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "RAFT - EM (Robustness)": "{\"description\": \"min=0.075, mean=0.673, max=0.975, sum=7.4 (11)\", \"tab\": \"Robustness\", \"score\": \"0.6727272727272727\"}", - "RAFT - EM (Fairness)": "{\"description\": \"min=0.1, mean=0.7, max=0.975, sum=7.7 (11)\", \"tab\": \"Fairness\", \"score\": \"0.7\"}", - "RAFT - Denoised inference time (s)": "{\"description\": \"11 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "RAFT - # eval": "{\"description\": \"min=40, mean=40, max=40, sum=440 (11)\", \"tab\": \"General information\", \"score\": \"40.0\"}", - "RAFT - # train": "{\"description\": \"min=2.575, mean=4.78, max=5, sum=52.575 (11)\", \"tab\": \"General information\", \"score\": \"4.779545454545455\"}", - "RAFT - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (11)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "RAFT - # prompt tokens": "{\"description\": \"min=303.675, mean=1153.852, max=3623.9, sum=12692.375 (11)\", \"tab\": \"General information\", \"score\": \"1153.8522727272727\"}", - "RAFT - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=11 (11)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "RAFT - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=11 (11)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "RAFT - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Toxic fraction": "{\"description\": \"11 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_lite/meta_llama-2-70b/1774096306.427425", - "retrieved_timestamp": "1774096306.427425", - "source_metadata": { - "source_name": "helm_lite", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_lite", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_lite", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.482, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.3882646691635456\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.763, - "details": { - "description": "min=0.763, mean=0.763, max=0.763, sum=0.763 (1)", - "tab": "Accuracy", - "NarrativeQA - Observed inference time (s)": "{\"description\": \"min=1.871, mean=1.871, max=1.871, sum=1.871 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.8709671289148464\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=4.408, mean=4.408, max=4.408, sum=4.408 (1)\", \"tab\": \"General information\", \"score\": \"4.408450704225352\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=3669.808, mean=3669.808, max=3669.808, sum=3669.808 (1)\", \"tab\": \"General information\", \"score\": \"3669.8084507042254\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (closed-book)", - "source_data": { - "dataset_name": "NaturalQuestions (closed-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (closed-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.46, - "details": { - "description": "min=0.46, mean=0.46, max=0.46, sum=0.46 (1)", - "tab": "Accuracy", - "NaturalQuestions (open-book) - Observed inference time (s)": "{\"description\": \"min=1.278, mean=1.278, max=1.278, sum=1.278 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.277897496700287\"}", - "NaturalQuestions (closed-book) - Observed inference time (s)": "{\"description\": \"min=0.818, mean=0.818, max=0.818, sum=0.818 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.8177921280860901\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.831, mean=4.831, max=4.831, sum=4.831 (1)\", \"tab\": \"General information\", \"score\": \"4.831\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.026, mean=0.026, max=0.026, sum=0.026 (1)\", \"tab\": \"General information\", \"score\": \"0.026\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=2289.357, mean=2289.357, max=2289.357, sum=2289.357 (1)\", \"tab\": \"General information\", \"score\": \"2289.357\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=0.996, mean=0.996, max=0.996, sum=0.996 (1)\", \"tab\": \"General information\", \"score\": \"0.996\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=137.383, mean=137.383, max=137.383, sum=137.383 (1)\", \"tab\": \"General information\", \"score\": \"137.383\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "mode": "\"closedbook\"" - } - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.838, - "details": { - "description": "min=0.838, mean=0.838, max=0.838, sum=0.838 (1)", - "tab": "Accuracy", - "OpenbookQA - Observed inference time (s)": "{\"description\": \"min=0.656, mean=0.656, max=0.656, sum=0.656 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.6557973260879517\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=282.574, mean=282.574, max=282.574, sum=282.574 (1)\", \"tab\": \"General information\", \"score\": \"282.574\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "dataset": "\"openbookqa\"", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.58, - "details": { - "description": "min=0.31, mean=0.58, max=0.92, sum=2.902 (5)", - "tab": "Accuracy", - "MMLU - Observed inference time (s)": "{\"description\": \"min=0.465, mean=0.501, max=0.56, sum=2.507 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.5013968416013215\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=397.65, mean=522.547, max=684.675, sum=2612.735 (5)\", \"tab\": \"General information\", \"score\": \"522.5470877192982\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"college_chemistry\", \"computer_security\", \"econometrics\", \"us_foreign_policy\"]", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MATH", - "source_data": { - "dataset_name": "MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Equivalent (CoT) on MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.323, - "details": { - "description": "min=0.205, mean=0.323, max=0.489, sum=2.26 (7)", - "tab": "Accuracy", - "MATH - Observed inference time (s)": "{\"description\": \"min=1.813, mean=2.443, max=3.147, sum=17.103 (7)\", \"tab\": \"Efficiency\", \"score\": \"2.4432508421434598\"}", - "MATH - # eval": "{\"description\": \"min=30, mean=62.429, max=135, sum=437 (7)\", \"tab\": \"General information\", \"score\": \"62.42857142857143\"}", - "MATH - # train": "{\"description\": \"min=8, mean=8, max=8, sum=56 (7)\", \"tab\": \"General information\", \"score\": \"8.0\"}", - "MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MATH - # prompt tokens": "{\"description\": \"min=971.652, mean=1438.636, max=2490.962, sum=10070.453 (7)\", \"tab\": \"General information\", \"score\": \"1438.6362030100095\"}", - "MATH - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=7 (7)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"algebra\", \"counting_and_probability\", \"geometry\", \"intermediate_algebra\", \"number_theory\", \"prealgebra\", \"precalculus\"]", - "level": "\"1\"", - "use_official_examples": "\"False\"", - "use_chain_of_thought": "\"True\"" - } - } - }, - { - "evaluation_name": "GSM8K", - "source_data": { - "dataset_name": "GSM8K", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on GSM8K", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.567, - "details": { - "description": "min=0.567, mean=0.567, max=0.567, sum=0.567 (1)", - "tab": "Accuracy", - "GSM8K - Observed inference time (s)": "{\"description\": \"min=3.737, mean=3.737, max=3.737, sum=3.737 (1)\", \"tab\": \"Efficiency\", \"score\": \"3.737159442663193\"}", - "GSM8K - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "GSM8K - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "GSM8K - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GSM8K - # prompt tokens": "{\"description\": \"min=1207.746, mean=1207.746, max=1207.746, sum=1207.746 (1)\", \"tab\": \"General information\", \"score\": \"1207.746\"}", - "GSM8K - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "LegalBench", - "source_data": { - "dataset_name": "LegalBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on LegalBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.673, - "details": { - "description": "min=0.444, mean=0.673, max=0.937, sum=3.363 (5)", - "tab": "Accuracy", - "LegalBench - Observed inference time (s)": "{\"description\": \"min=0.448, mean=0.759, max=1.744, sum=3.796 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.7591354159811778\"}", - "LegalBench - # eval": "{\"description\": \"min=95, mean=409.4, max=1000, sum=2047 (5)\", \"tab\": \"General information\", \"score\": \"409.4\"}", - "LegalBench - # train": "{\"description\": \"min=1.886, mean=4.177, max=5, sum=20.886 (5)\", \"tab\": \"General information\", \"score\": \"4.177142857142857\"}", - "LegalBench - truncated": "{\"description\": \"min=0, mean=0.001, max=0.004, sum=0.004 (5)\", \"tab\": \"General information\", \"score\": \"0.0008163265306122449\"}", - "LegalBench - # prompt tokens": "{\"description\": \"min=222.137, mean=1027.35, max=3642.378, sum=5136.751 (5)\", \"tab\": \"General information\", \"score\": \"1027.3502076083553\"}", - "LegalBench - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"abercrombie\", \"corporate_lobbying\", \"function_of_decision_section\", \"international_citizenship_questions\", \"proa\"]" - } - } - }, - { - "evaluation_name": "MedQA", - "source_data": { - "dataset_name": "MedQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MedQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.618, - "details": { - "description": "min=0.618, mean=0.618, max=0.618, sum=0.618 (1)", - "tab": "Accuracy", - "MedQA - Observed inference time (s)": "{\"description\": \"min=0.971, mean=0.971, max=0.971, sum=0.971 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.9713700282170806\"}", - "MedQA - # eval": "{\"description\": \"min=503, mean=503, max=503, sum=503 (1)\", \"tab\": \"General information\", \"score\": \"503.0\"}", - "MedQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MedQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MedQA - # prompt tokens": "{\"description\": \"min=1234.901, mean=1234.901, max=1234.901, sum=1234.901 (1)\", \"tab\": \"General information\", \"score\": \"1234.9005964214712\"}", - "MedQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WMT 2014", - "source_data": { - "dataset_name": "WMT 2014", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "BLEU-4 on WMT 2014", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.196, - "details": { - "description": "min=0.12, mean=0.196, max=0.233, sum=0.979 (5)", - "tab": "Accuracy", - "WMT 2014 - Observed inference time (s)": "{\"description\": \"min=0.809, mean=1.074, max=1.477, sum=5.368 (5)\", \"tab\": \"Efficiency\", \"score\": \"1.0736038563633745\"}", - "WMT 2014 - # eval": "{\"description\": \"min=503, mean=568.8, max=832, sum=2844 (5)\", \"tab\": \"General information\", \"score\": \"568.8\"}", - "WMT 2014 - # train": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "WMT 2014 - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WMT 2014 - # prompt tokens": "{\"description\": \"min=127.523, mean=142.288, max=164.972, sum=711.438 (5)\", \"tab\": \"General information\", \"score\": \"142.28751290334915\"}", - "WMT 2014 - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "language_pair": "[\"cs-en\", \"de-en\", \"fr-en\", \"hi-en\", \"ru-en\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_mmlu/meta_llama-2-70b/1774096312.00548", - "retrieved_timestamp": "1774096312.00548", - "source_metadata": { - "source_name": "helm_mmlu", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_mmlu", - "evaluation_results": [ - { - "evaluation_name": "MMLU All Subjects", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU All Subjects", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.695, - "details": { - "description": "min=0.31, mean=0.695, max=0.933, sum=79.283 (114)", - "tab": "Accuracy", - "MMLU All Subjects - Observed inference time (s)": "{\"description\": \"min=0.314, mean=0.466, max=0.981, sum=53.164 (114)\", \"tab\": \"Efficiency\", \"score\": \"0.46634649940337786\"}", - "MMLU All Subjects - # eval": "{\"description\": \"min=100, mean=246.351, max=1534, sum=28084 (114)\", \"tab\": \"General information\", \"score\": \"246.35087719298247\"}", - "MMLU All Subjects - # train": "{\"description\": \"min=5, mean=5, max=5, sum=570 (114)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU All Subjects - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU All Subjects - # prompt tokens": "{\"description\": \"min=304.474, mean=706.682, max=3159.636, sum=80561.749 (114)\", \"tab\": \"General information\", \"score\": \"706.6820126388612\"}", - "MMLU All Subjects - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=114 (114)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_name": "Abstract Algebra", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Abstract Algebra", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.31, - "details": { - "description": "min=0.31, mean=0.31, max=0.31, sum=0.62 (2)", - "tab": "Accuracy", - "Abstract Algebra - Observed inference time (s)": "{\"description\": \"min=0.36, mean=0.36, max=0.36, sum=0.72 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3601346731185913\"}", - "Abstract Algebra - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Abstract Algebra - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Abstract Algebra - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Abstract Algebra - # prompt tokens": "{\"description\": \"min=397.65, mean=397.65, max=397.65, sum=795.3 (2)\", \"tab\": \"General information\", \"score\": \"397.65\"}", - "Abstract Algebra - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"abstract_algebra\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_abstract_algebra\"" - } - } - }, - { - "evaluation_name": "Anatomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Anatomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.607, - "details": { - "description": "min=0.607, mean=0.607, max=0.607, sum=1.215 (2)", - "tab": "Accuracy", - "Anatomy - Observed inference time (s)": "{\"description\": \"min=0.433, mean=0.433, max=0.433, sum=0.866 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4331345310917607\"}", - "Anatomy - # eval": "{\"description\": \"min=135, mean=135, max=135, sum=270 (2)\", \"tab\": \"General information\", \"score\": \"135.0\"}", - "Anatomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Anatomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Anatomy - # prompt tokens": "{\"description\": \"min=409.133, mean=409.133, max=409.133, sum=818.267 (2)\", \"tab\": \"General information\", \"score\": \"409.1333333333333\"}", - "Anatomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"anatomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_anatomy\"" - } - } - }, - { - "evaluation_name": "College Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on College Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.363, - "details": { - "description": "min=0.363, mean=0.363, max=0.363, sum=0.725 (2)", - "tab": "Accuracy", - "College Chemistry - Observed inference time (s)": "{\"description\": \"min=0.379, mean=0.379, max=0.379, sum=0.757 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3786743521690369\"}", - "College Biology - Observed inference time (s)": "{\"description\": \"min=0.387, mean=0.387, max=0.387, sum=0.773 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.38658806019359165\"}", - "College Computer Science - Observed inference time (s)": "{\"description\": \"min=0.444, mean=0.444, max=0.444, sum=0.888 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.44394851446151734\"}", - "College Mathematics - Observed inference time (s)": "{\"description\": \"min=0.71, mean=0.71, max=0.71, sum=1.42 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7099040699005127\"}", - "College Medicine - Observed inference time (s)": "{\"description\": \"min=0.47, mean=0.47, max=0.47, sum=0.939 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4695483673514658\"}", - "College Physics - Observed inference time (s)": "{\"description\": \"min=0.389, mean=0.389, max=0.389, sum=0.778 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3889027389825559\"}", - "College Chemistry - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Chemistry - # prompt tokens": "{\"description\": \"min=622.43, mean=622.43, max=622.43, sum=1244.86 (2)\", \"tab\": \"General information\", \"score\": \"622.43\"}", - "College Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Biology - # eval": "{\"description\": \"min=144, mean=144, max=144, sum=288 (2)\", \"tab\": \"General information\", \"score\": \"144.0\"}", - "College Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # prompt tokens": "{\"description\": \"min=553.632, mean=553.632, max=553.632, sum=1107.264 (2)\", \"tab\": \"General information\", \"score\": \"553.6319444444445\"}", - "College Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # prompt tokens": "{\"description\": \"min=901.14, mean=901.14, max=901.14, sum=1802.28 (2)\", \"tab\": \"General information\", \"score\": \"901.14\"}", - "College Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Mathematics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # prompt tokens": "{\"description\": \"min=646.96, mean=646.96, max=646.96, sum=1293.92 (2)\", \"tab\": \"General information\", \"score\": \"646.96\"}", - "College Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Medicine - # eval": "{\"description\": \"min=173, mean=173, max=173, sum=346 (2)\", \"tab\": \"General information\", \"score\": \"173.0\"}", - "College Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # prompt tokens": "{\"description\": \"min=608.671, mean=608.671, max=608.671, sum=1217.341 (2)\", \"tab\": \"General information\", \"score\": \"608.6705202312139\"}", - "College Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Physics - # eval": "{\"description\": \"min=102, mean=102, max=102, sum=204 (2)\", \"tab\": \"General information\", \"score\": \"102.0\"}", - "College Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # prompt tokens": "{\"description\": \"min=551.873, mean=551.873, max=551.873, sum=1103.745 (2)\", \"tab\": \"General information\", \"score\": \"551.8725490196078\"}", - "College Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"college_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_college_physics\"" - } - } - }, - { - "evaluation_name": "Computer Security", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Computer Security", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.77, - "details": { - "description": "min=0.77, mean=0.77, max=0.77, sum=1.54 (2)", - "tab": "Accuracy", - "Computer Security - Observed inference time (s)": "{\"description\": \"min=0.371, mean=0.371, max=0.371, sum=0.743 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3714062762260437\"}", - "Computer Security - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Computer Security - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Computer Security - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Computer Security - # prompt tokens": "{\"description\": \"min=428.17, mean=428.17, max=428.17, sum=856.34 (2)\", \"tab\": \"General information\", \"score\": \"428.17\"}", - "Computer Security - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"computer_security\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_computer_security\"" - } - } - }, - { - "evaluation_name": "Econometrics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Econometrics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.43, - "details": { - "description": "min=0.43, mean=0.43, max=0.43, sum=0.86 (2)", - "tab": "Accuracy", - "Econometrics - Observed inference time (s)": "{\"description\": \"min=0.392, mean=0.392, max=0.392, sum=0.783 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3916624889039157\"}", - "Econometrics - # eval": "{\"description\": \"min=114, mean=114, max=114, sum=228 (2)\", \"tab\": \"General information\", \"score\": \"114.0\"}", - "Econometrics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Econometrics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Econometrics - # prompt tokens": "{\"description\": \"min=684.675, mean=684.675, max=684.675, sum=1369.351 (2)\", \"tab\": \"General information\", \"score\": \"684.6754385964912\"}", - "Econometrics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"econometrics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_econometrics\"" - } - } - }, - { - "evaluation_name": "Global Facts", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Global Facts", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.47, - "details": { - "description": "min=0.47, mean=0.47, max=0.47, sum=0.94 (2)", - "tab": "Accuracy", - "Global Facts - Observed inference time (s)": "{\"description\": \"min=0.374, mean=0.374, max=0.374, sum=0.747 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3736806106567383\"}", - "Global Facts - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Global Facts - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Global Facts - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Global Facts - # prompt tokens": "{\"description\": \"min=484.54, mean=484.54, max=484.54, sum=969.08 (2)\", \"tab\": \"General information\", \"score\": \"484.54\"}", - "Global Facts - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"global_facts\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_global_facts\"" - } - } - }, - { - "evaluation_name": "Jurisprudence", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Jurisprudence", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.824, - "details": { - "description": "min=0.824, mean=0.824, max=0.824, sum=1.648 (2)", - "tab": "Accuracy", - "Jurisprudence - Observed inference time (s)": "{\"description\": \"min=0.694, mean=0.694, max=0.694, sum=1.387 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6937185768727903\"}", - "Jurisprudence - # eval": "{\"description\": \"min=108, mean=108, max=108, sum=216 (2)\", \"tab\": \"General information\", \"score\": \"108.0\"}", - "Jurisprudence - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Jurisprudence - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Jurisprudence - # prompt tokens": "{\"description\": \"min=449.898, mean=449.898, max=449.898, sum=899.796 (2)\", \"tab\": \"General information\", \"score\": \"449.89814814814815\"}", - "Jurisprudence - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"jurisprudence\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_jurisprudence\"" - } - } - }, - { - "evaluation_name": "Philosophy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Philosophy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.791, - "details": { - "description": "min=0.791, mean=0.791, max=0.791, sum=1.582 (2)", - "tab": "Accuracy", - "Philosophy - Observed inference time (s)": "{\"description\": \"min=0.314, mean=0.314, max=0.314, sum=0.628 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3140420009085603\"}", - "Philosophy - # eval": "{\"description\": \"min=311, mean=311, max=311, sum=622 (2)\", \"tab\": \"General information\", \"score\": \"311.0\"}", - "Philosophy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Philosophy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Philosophy - # prompt tokens": "{\"description\": \"min=372.122, mean=372.122, max=372.122, sum=744.244 (2)\", \"tab\": \"General information\", \"score\": \"372.12218649517683\"}", - "Philosophy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"philosophy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_philosophy\"" - } - } - }, - { - "evaluation_name": "Professional Psychology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Professional Psychology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.76, - "details": { - "description": "min=0.76, mean=0.76, max=0.76, sum=1.52 (2)", - "tab": "Accuracy", - "Professional Medicine - Observed inference time (s)": "{\"description\": \"min=0.515, mean=0.515, max=0.515, sum=1.029 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5146331287482205\"}", - "Professional Accounting - Observed inference time (s)": "{\"description\": \"min=0.387, mean=0.387, max=0.387, sum=0.774 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3871775383644916\"}", - "Professional Law - Observed inference time (s)": "{\"description\": \"min=0.697, mean=0.697, max=0.697, sum=1.395 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6972876995452224\"}", - "Professional Psychology - Observed inference time (s)": "{\"description\": \"min=0.393, mean=0.393, max=0.393, sum=0.787 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.39348618851767647\"}", - "Professional Medicine - # eval": "{\"description\": \"min=272, mean=272, max=272, sum=544 (2)\", \"tab\": \"General information\", \"score\": \"272.0\"}", - "Professional Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Medicine - # prompt tokens": "{\"description\": \"min=1330.647, mean=1330.647, max=1330.647, sum=2661.294 (2)\", \"tab\": \"General information\", \"score\": \"1330.6470588235295\"}", - "Professional Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Accounting - # eval": "{\"description\": \"min=282, mean=282, max=282, sum=564 (2)\", \"tab\": \"General information\", \"score\": \"282.0\"}", - "Professional Accounting - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Accounting - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # prompt tokens": "{\"description\": \"min=823.277, mean=823.277, max=823.277, sum=1646.553 (2)\", \"tab\": \"General information\", \"score\": \"823.2765957446809\"}", - "Professional Accounting - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Law - # eval": "{\"description\": \"min=1534, mean=1534, max=1534, sum=3068 (2)\", \"tab\": \"General information\", \"score\": \"1534.0\"}", - "Professional Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # prompt tokens": "{\"description\": \"min=1915.007, mean=1915.007, max=1915.007, sum=3830.014 (2)\", \"tab\": \"General information\", \"score\": \"1915.0071707953064\"}", - "Professional Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Psychology - # eval": "{\"description\": \"min=612, mean=612, max=612, sum=1224 (2)\", \"tab\": \"General information\", \"score\": \"612.0\"}", - "Professional Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # prompt tokens": "{\"description\": \"min=650.078, mean=650.078, max=650.078, sum=1300.157 (2)\", \"tab\": \"General information\", \"score\": \"650.0784313725491\"}", - "Professional Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"professional_psychology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_professional_psychology\"" - } - } - }, - { - "evaluation_name": "Us Foreign Policy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Us Foreign Policy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.92, - "details": { - "description": "min=0.92, mean=0.92, max=0.92, sum=1.84 (2)", - "tab": "Accuracy", - "Us Foreign Policy - Observed inference time (s)": "{\"description\": \"min=0.348, mean=0.348, max=0.348, sum=0.696 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3482255029678345\"}", - "Us Foreign Policy - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Us Foreign Policy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Us Foreign Policy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Us Foreign Policy - # prompt tokens": "{\"description\": \"min=479.81, mean=479.81, max=479.81, sum=959.62 (2)\", \"tab\": \"General information\", \"score\": \"479.81\"}", - "Us Foreign Policy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"us_foreign_policy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_us_foreign_policy\"" - } - } - }, - { - "evaluation_name": "Astronomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Astronomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.829, - "details": { - "description": "min=0.829, mean=0.829, max=0.829, sum=1.658 (2)", - "tab": "Accuracy", - "Astronomy - Observed inference time (s)": "{\"description\": \"min=0.456, mean=0.456, max=0.456, sum=0.912 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.45624671798003347\"}", - "Astronomy - # eval": "{\"description\": \"min=152, mean=152, max=152, sum=304 (2)\", \"tab\": \"General information\", \"score\": \"152.0\"}", - "Astronomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Astronomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Astronomy - # prompt tokens": "{\"description\": \"min=681.079, mean=681.079, max=681.079, sum=1362.158 (2)\", \"tab\": \"General information\", \"score\": \"681.078947368421\"}", - "Astronomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"astronomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_astronomy\"" - } - } - }, - { - "evaluation_name": "Business Ethics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Business Ethics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.73, - "details": { - "description": "min=0.73, mean=0.73, max=0.73, sum=1.46 (2)", - "tab": "Accuracy", - "Business Ethics - Observed inference time (s)": "{\"description\": \"min=0.649, mean=0.649, max=0.649, sum=1.298 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6490170955657959\"}", - "Business Ethics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Business Ethics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Business Ethics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Business Ethics - # prompt tokens": "{\"description\": \"min=674.44, mean=674.44, max=674.44, sum=1348.88 (2)\", \"tab\": \"General information\", \"score\": \"674.44\"}", - "Business Ethics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"business_ethics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_business_ethics\"" - } - } - }, - { - "evaluation_name": "Clinical Knowledge", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Clinical Knowledge", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.717, - "details": { - "description": "min=0.717, mean=0.717, max=0.717, sum=1.434 (2)", - "tab": "Accuracy", - "Clinical Knowledge - Observed inference time (s)": "{\"description\": \"min=0.394, mean=0.394, max=0.394, sum=0.788 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.394086869257801\"}", - "Clinical Knowledge - # eval": "{\"description\": \"min=265, mean=265, max=265, sum=530 (2)\", \"tab\": \"General information\", \"score\": \"265.0\"}", - "Clinical Knowledge - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Clinical Knowledge - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Clinical Knowledge - # prompt tokens": "{\"description\": \"min=487.374, mean=487.374, max=487.374, sum=974.747 (2)\", \"tab\": \"General information\", \"score\": \"487.3735849056604\"}", - "Clinical Knowledge - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"clinical_knowledge\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_clinical_knowledge\"" - } - } - }, - { - "evaluation_name": "Conceptual Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Conceptual Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.668, - "details": { - "description": "min=0.668, mean=0.668, max=0.668, sum=1.336 (2)", - "tab": "Accuracy", - "Conceptual Physics - Observed inference time (s)": "{\"description\": \"min=0.519, mean=0.519, max=0.519, sum=1.038 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5188552247717025\"}", - "Conceptual Physics - # eval": "{\"description\": \"min=235, mean=235, max=235, sum=470 (2)\", \"tab\": \"General information\", \"score\": \"235.0\"}", - "Conceptual Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Conceptual Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Conceptual Physics - # prompt tokens": "{\"description\": \"min=333.153, mean=333.153, max=333.153, sum=666.306 (2)\", \"tab\": \"General information\", \"score\": \"333.1531914893617\"}", - "Conceptual Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"conceptual_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_conceptual_physics\"" - } - } - }, - { - "evaluation_name": "Electrical Engineering", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Electrical Engineering", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.634, - "details": { - "description": "min=0.634, mean=0.634, max=0.634, sum=1.269 (2)", - "tab": "Accuracy", - "Electrical Engineering - Observed inference time (s)": "{\"description\": \"min=0.415, mean=0.415, max=0.415, sum=0.83 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.414785334159588\"}", - "Electrical Engineering - # eval": "{\"description\": \"min=145, mean=145, max=145, sum=290 (2)\", \"tab\": \"General information\", \"score\": \"145.0\"}", - "Electrical Engineering - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Electrical Engineering - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Electrical Engineering - # prompt tokens": "{\"description\": \"min=497.779, mean=497.779, max=497.779, sum=995.559 (2)\", \"tab\": \"General information\", \"score\": \"497.7793103448276\"}", - "Electrical Engineering - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"electrical_engineering\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_electrical_engineering\"" - } - } - }, - { - "evaluation_name": "Elementary Mathematics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Elementary Mathematics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.421, - "details": { - "description": "min=0.421, mean=0.421, max=0.421, sum=0.841 (2)", - "tab": "Accuracy", - "Elementary Mathematics - Observed inference time (s)": "{\"description\": \"min=0.407, mean=0.407, max=0.407, sum=0.814 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4069670924433955\"}", - "Elementary Mathematics - # eval": "{\"description\": \"min=378, mean=378, max=378, sum=756 (2)\", \"tab\": \"General information\", \"score\": \"378.0\"}", - "Elementary Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Elementary Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Elementary Mathematics - # prompt tokens": "{\"description\": \"min=609.156, mean=609.156, max=609.156, sum=1218.312 (2)\", \"tab\": \"General information\", \"score\": \"609.1560846560847\"}", - "Elementary Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"elementary_mathematics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_elementary_mathematics\"" - } - } - }, - { - "evaluation_name": "Formal Logic", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Formal Logic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.468, - "details": { - "description": "min=0.468, mean=0.468, max=0.468, sum=0.937 (2)", - "tab": "Accuracy", - "Formal Logic - Observed inference time (s)": "{\"description\": \"min=0.415, mean=0.415, max=0.415, sum=0.83 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.41500668109409394\"}", - "Formal Logic - # eval": "{\"description\": \"min=126, mean=126, max=126, sum=252 (2)\", \"tab\": \"General information\", \"score\": \"126.0\"}", - "Formal Logic - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Formal Logic - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Formal Logic - # prompt tokens": "{\"description\": \"min=691.81, mean=691.81, max=691.81, sum=1383.619 (2)\", \"tab\": \"General information\", \"score\": \"691.8095238095239\"}", - "Formal Logic - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"formal_logic\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_formal_logic\"" - } - } - }, - { - "evaluation_name": "High School World History", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on High School World History", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.882, - "details": { - "description": "min=0.882, mean=0.882, max=0.882, sum=1.764 (2)", - "tab": "Accuracy", - "High School Biology - Observed inference time (s)": "{\"description\": \"min=0.379, mean=0.379, max=0.379, sum=0.759 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3793416823110273\"}", - "High School Chemistry - Observed inference time (s)": "{\"description\": \"min=0.402, mean=0.402, max=0.402, sum=0.804 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4020436197666112\"}", - "High School Computer Science - Observed inference time (s)": "{\"description\": \"min=0.562, mean=0.562, max=0.562, sum=1.124 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5618092942237854\"}", - "High School European History - Observed inference time (s)": "{\"description\": \"min=0.981, mean=0.981, max=0.981, sum=1.962 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9809041355595444\"}", - "High School Geography - Observed inference time (s)": "{\"description\": \"min=0.415, mean=0.415, max=0.415, sum=0.83 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.41476938218781445\"}", - "High School Government And Politics - Observed inference time (s)": "{\"description\": \"min=0.388, mean=0.388, max=0.388, sum=0.775 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3875881736142648\"}", - "High School Macroeconomics - Observed inference time (s)": "{\"description\": \"min=0.38, mean=0.38, max=0.38, sum=0.76 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3797990028674786\"}", - "High School Mathematics - Observed inference time (s)": "{\"description\": \"min=0.408, mean=0.408, max=0.408, sum=0.817 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.40841888145164207\"}", - "High School Microeconomics - Observed inference time (s)": "{\"description\": \"min=0.441, mean=0.441, max=0.441, sum=0.882 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4407546289828645\"}", - "High School Physics - Observed inference time (s)": "{\"description\": \"min=0.426, mean=0.426, max=0.426, sum=0.851 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.42553993724039846\"}", - "High School Psychology - Observed inference time (s)": "{\"description\": \"min=0.469, mean=0.469, max=0.469, sum=0.939 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.46939194880494284\"}", - "High School Statistics - Observed inference time (s)": "{\"description\": \"min=0.499, mean=0.499, max=0.499, sum=0.998 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4990172529662097\"}", - "High School US History - Observed inference time (s)": "{\"description\": \"min=0.847, mean=0.847, max=0.847, sum=1.693 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8465246745184356\"}", - "High School World History - Observed inference time (s)": "{\"description\": \"min=0.558, mean=0.558, max=0.558, sum=1.117 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5583362217190899\"}", - "High School Biology - # eval": "{\"description\": \"min=310, mean=310, max=310, sum=620 (2)\", \"tab\": \"General information\", \"score\": \"310.0\"}", - "High School Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Biology - # prompt tokens": "{\"description\": \"min=596.894, mean=596.894, max=596.894, sum=1193.787 (2)\", \"tab\": \"General information\", \"score\": \"596.8935483870968\"}", - "High School Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Chemistry - # eval": "{\"description\": \"min=203, mean=203, max=203, sum=406 (2)\", \"tab\": \"General information\", \"score\": \"203.0\"}", - "High School Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # prompt tokens": "{\"description\": \"min=568.665, mean=568.665, max=568.665, sum=1137.33 (2)\", \"tab\": \"General information\", \"score\": \"568.6650246305419\"}", - "High School Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "High School Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # prompt tokens": "{\"description\": \"min=988.57, mean=988.57, max=988.57, sum=1977.14 (2)\", \"tab\": \"General information\", \"score\": \"988.57\"}", - "High School Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School European History - # eval": "{\"description\": \"min=165, mean=165, max=165, sum=330 (2)\", \"tab\": \"General information\", \"score\": \"165.0\"}", - "High School European History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School European History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # prompt tokens": "{\"description\": \"min=3159.636, mean=3159.636, max=3159.636, sum=6319.273 (2)\", \"tab\": \"General information\", \"score\": \"3159.6363636363635\"}", - "High School European History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Geography - # eval": "{\"description\": \"min=198, mean=198, max=198, sum=396 (2)\", \"tab\": \"General information\", \"score\": \"198.0\"}", - "High School Geography - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Geography - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # prompt tokens": "{\"description\": \"min=436.657, mean=436.657, max=436.657, sum=873.313 (2)\", \"tab\": \"General information\", \"score\": \"436.65656565656565\"}", - "High School Geography - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Government And Politics - # eval": "{\"description\": \"min=193, mean=193, max=193, sum=386 (2)\", \"tab\": \"General information\", \"score\": \"193.0\"}", - "High School Government And Politics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Government And Politics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # prompt tokens": "{\"description\": \"min=527.927, mean=527.927, max=527.927, sum=1055.855 (2)\", \"tab\": \"General information\", \"score\": \"527.9274611398964\"}", - "High School Government And Politics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Macroeconomics - # eval": "{\"description\": \"min=390, mean=390, max=390, sum=780 (2)\", \"tab\": \"General information\", \"score\": \"390.0\"}", - "High School Macroeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Macroeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # prompt tokens": "{\"description\": \"min=445.662, mean=445.662, max=445.662, sum=891.323 (2)\", \"tab\": \"General information\", \"score\": \"445.66153846153844\"}", - "High School Macroeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Mathematics - # eval": "{\"description\": \"min=270, mean=270, max=270, sum=540 (2)\", \"tab\": \"General information\", \"score\": \"270.0\"}", - "High School Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # prompt tokens": "{\"description\": \"min=579.181, mean=579.181, max=579.181, sum=1158.363 (2)\", \"tab\": \"General information\", \"score\": \"579.1814814814815\"}", - "High School Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Microeconomics - # eval": "{\"description\": \"min=238, mean=238, max=238, sum=476 (2)\", \"tab\": \"General information\", \"score\": \"238.0\"}", - "High School Microeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Microeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # prompt tokens": "{\"description\": \"min=449.492, mean=449.492, max=449.492, sum=898.983 (2)\", \"tab\": \"General information\", \"score\": \"449.49159663865544\"}", - "High School Microeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Physics - # eval": "{\"description\": \"min=151, mean=151, max=151, sum=302 (2)\", \"tab\": \"General information\", \"score\": \"151.0\"}", - "High School Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # prompt tokens": "{\"description\": \"min=621.788, mean=621.788, max=621.788, sum=1243.576 (2)\", \"tab\": \"General information\", \"score\": \"621.7880794701987\"}", - "High School Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Psychology - # eval": "{\"description\": \"min=545, mean=545, max=545, sum=1090 (2)\", \"tab\": \"General information\", \"score\": \"545.0\"}", - "High School Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # prompt tokens": "{\"description\": \"min=585.919, mean=585.919, max=585.919, sum=1171.839 (2)\", \"tab\": \"General information\", \"score\": \"585.9192660550459\"}", - "High School Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Statistics - # eval": "{\"description\": \"min=216, mean=216, max=216, sum=432 (2)\", \"tab\": \"General information\", \"score\": \"216.0\"}", - "High School Statistics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Statistics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # prompt tokens": "{\"description\": \"min=908.208, mean=908.208, max=908.208, sum=1816.417 (2)\", \"tab\": \"General information\", \"score\": \"908.2083333333334\"}", - "High School Statistics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School US History - # eval": "{\"description\": \"min=204, mean=204, max=204, sum=408 (2)\", \"tab\": \"General information\", \"score\": \"204.0\"}", - "High School US History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School US History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # prompt tokens": "{\"description\": \"min=2535.324, mean=2535.324, max=2535.324, sum=5070.647 (2)\", \"tab\": \"General information\", \"score\": \"2535.323529411765\"}", - "High School US History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School World History - # eval": "{\"description\": \"min=237, mean=237, max=237, sum=474 (2)\", \"tab\": \"General information\", \"score\": \"237.0\"}", - "High School World History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School World History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # prompt tokens": "{\"description\": \"min=1638.219, mean=1638.219, max=1638.219, sum=3276.439 (2)\", \"tab\": \"General information\", \"score\": \"1638.2194092827003\"}", - "High School World History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"high_school_world_history\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_high_school_world_history\"" - } - } - }, - { - "evaluation_name": "Human Sexuality", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Human Sexuality", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.84, - "details": { - "description": "min=0.84, mean=0.84, max=0.84, sum=1.679 (2)", - "tab": "Accuracy", - "Human Aging - Observed inference time (s)": "{\"description\": \"min=0.487, mean=0.487, max=0.487, sum=0.973 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4866963897585334\"}", - "Human Sexuality - Observed inference time (s)": "{\"description\": \"min=0.39, mean=0.39, max=0.39, sum=0.781 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3902700020156744\"}", - "Human Aging - # eval": "{\"description\": \"min=223, mean=223, max=223, sum=446 (2)\", \"tab\": \"General information\", \"score\": \"223.0\"}", - "Human Aging - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Aging - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Aging - # prompt tokens": "{\"description\": \"min=361.26, mean=361.26, max=361.26, sum=722.52 (2)\", \"tab\": \"General information\", \"score\": \"361.26008968609864\"}", - "Human Aging - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Human Sexuality - # eval": "{\"description\": \"min=131, mean=131, max=131, sum=262 (2)\", \"tab\": \"General information\", \"score\": \"131.0\"}", - "Human Sexuality - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Sexuality - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # prompt tokens": "{\"description\": \"min=403.382, mean=403.382, max=403.382, sum=806.763 (2)\", \"tab\": \"General information\", \"score\": \"403.381679389313\"}", - "Human Sexuality - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"human_sexuality\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_human_sexuality\"" - } - } - }, - { - "evaluation_name": "International Law", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on International Law", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.868, - "details": { - "description": "min=0.868, mean=0.868, max=0.868, sum=1.736 (2)", - "tab": "Accuracy", - "International Law - Observed inference time (s)": "{\"description\": \"min=0.538, mean=0.538, max=0.538, sum=1.076 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5381311483619627\"}", - "International Law - # eval": "{\"description\": \"min=121, mean=121, max=121, sum=242 (2)\", \"tab\": \"General information\", \"score\": \"121.0\"}", - "International Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "International Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "International Law - # prompt tokens": "{\"description\": \"min=729.463, mean=729.463, max=729.463, sum=1458.926 (2)\", \"tab\": \"General information\", \"score\": \"729.4628099173553\"}", - "International Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"international_law\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_international_law\"" - } - } - }, - { - "evaluation_name": "Logical Fallacies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Logical Fallacies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.791, - "details": { - "description": "min=0.791, mean=0.791, max=0.791, sum=1.583 (2)", - "tab": "Accuracy", - "Logical Fallacies - Observed inference time (s)": "{\"description\": \"min=0.451, mean=0.451, max=0.451, sum=0.903 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4513764015736024\"}", - "Logical Fallacies - # eval": "{\"description\": \"min=163, mean=163, max=163, sum=326 (2)\", \"tab\": \"General information\", \"score\": \"163.0\"}", - "Logical Fallacies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Logical Fallacies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Logical Fallacies - # prompt tokens": "{\"description\": \"min=502.755, mean=502.755, max=502.755, sum=1005.509 (2)\", \"tab\": \"General information\", \"score\": \"502.7546012269939\"}", - "Logical Fallacies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"logical_fallacies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_logical_fallacies\"" - } - } - }, - { - "evaluation_name": "Machine Learning", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Machine Learning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.491, - "details": { - "description": "min=0.491, mean=0.491, max=0.491, sum=0.982 (2)", - "tab": "Accuracy", - "Machine Learning - Observed inference time (s)": "{\"description\": \"min=0.447, mean=0.447, max=0.447, sum=0.895 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4473994416849954\"}", - "Machine Learning - # eval": "{\"description\": \"min=112, mean=112, max=112, sum=224 (2)\", \"tab\": \"General information\", \"score\": \"112.0\"}", - "Machine Learning - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Machine Learning - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Machine Learning - # prompt tokens": "{\"description\": \"min=730.402, mean=730.402, max=730.402, sum=1460.804 (2)\", \"tab\": \"General information\", \"score\": \"730.4017857142857\"}", - "Machine Learning - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"machine_learning\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_machine_learning\"" - } - } - }, - { - "evaluation_name": "Management", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Management", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.845, - "details": { - "description": "min=0.845, mean=0.845, max=0.845, sum=1.689 (2)", - "tab": "Accuracy", - "Management - Observed inference time (s)": "{\"description\": \"min=0.482, mean=0.482, max=0.482, sum=0.965 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.482250699719179\"}", - "Management - # eval": "{\"description\": \"min=103, mean=103, max=103, sum=206 (2)\", \"tab\": \"General information\", \"score\": \"103.0\"}", - "Management - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Management - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Management - # prompt tokens": "{\"description\": \"min=315.777, mean=315.777, max=315.777, sum=631.553 (2)\", \"tab\": \"General information\", \"score\": \"315.77669902912623\"}", - "Management - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"management\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_management\"" - } - } - }, - { - "evaluation_name": "Marketing", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Marketing", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.889, - "details": { - "description": "min=0.889, mean=0.889, max=0.889, sum=1.778 (2)", - "tab": "Accuracy", - "Marketing - Observed inference time (s)": "{\"description\": \"min=0.529, mean=0.529, max=0.529, sum=1.059 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5294328500062991\"}", - "Marketing - # eval": "{\"description\": \"min=234, mean=234, max=234, sum=468 (2)\", \"tab\": \"General information\", \"score\": \"234.0\"}", - "Marketing - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Marketing - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Marketing - # prompt tokens": "{\"description\": \"min=472.628, mean=472.628, max=472.628, sum=945.256 (2)\", \"tab\": \"General information\", \"score\": \"472.62820512820514\"}", - "Marketing - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"marketing\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_marketing\"" - } - } - }, - { - "evaluation_name": "Medical Genetics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Medical Genetics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.72, - "details": { - "description": "min=0.72, mean=0.72, max=0.72, sum=1.44 (2)", - "tab": "Accuracy", - "Medical Genetics - Observed inference time (s)": "{\"description\": \"min=0.426, mean=0.426, max=0.426, sum=0.852 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.42598395347595214\"}", - "Medical Genetics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Medical Genetics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Medical Genetics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Medical Genetics - # prompt tokens": "{\"description\": \"min=408.14, mean=408.14, max=408.14, sum=816.28 (2)\", \"tab\": \"General information\", \"score\": \"408.14\"}", - "Medical Genetics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"medical_genetics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_medical_genetics\"" - } - } - }, - { - "evaluation_name": "Miscellaneous", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Miscellaneous", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.857, - "details": { - "description": "min=0.857, mean=0.857, max=0.857, sum=1.714 (2)", - "tab": "Accuracy", - "Miscellaneous - Observed inference time (s)": "{\"description\": \"min=0.434, mean=0.434, max=0.434, sum=0.868 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.43395179502504233\"}", - "Miscellaneous - # eval": "{\"description\": \"min=783, mean=783, max=783, sum=1566 (2)\", \"tab\": \"General information\", \"score\": \"783.0\"}", - "Miscellaneous - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Miscellaneous - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Miscellaneous - # prompt tokens": "{\"description\": \"min=345.913, mean=345.913, max=345.913, sum=691.826 (2)\", \"tab\": \"General information\", \"score\": \"345.9131545338442\"}", - "Miscellaneous - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"miscellaneous\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_miscellaneous\"" - } - } - }, - { - "evaluation_name": "Moral Scenarios", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Moral Scenarios", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.45, - "details": { - "description": "min=0.45, mean=0.45, max=0.45, sum=0.901 (2)", - "tab": "Accuracy", - "Moral Disputes - Observed inference time (s)": "{\"description\": \"min=0.404, mean=0.404, max=0.404, sum=0.809 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4043546129513338\"}", - "Moral Scenarios - Observed inference time (s)": "{\"description\": \"min=0.514, mean=0.514, max=0.514, sum=1.028 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5137747306397508\"}", - "Moral Disputes - # eval": "{\"description\": \"min=346, mean=346, max=346, sum=692 (2)\", \"tab\": \"General information\", \"score\": \"346.0\"}", - "Moral Disputes - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Disputes - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Disputes - # prompt tokens": "{\"description\": \"min=542.506, mean=542.506, max=542.506, sum=1085.012 (2)\", \"tab\": \"General information\", \"score\": \"542.5057803468208\"}", - "Moral Disputes - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Moral Scenarios - # eval": "{\"description\": \"min=895, mean=895, max=895, sum=1790 (2)\", \"tab\": \"General information\", \"score\": \"895.0\"}", - "Moral Scenarios - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Scenarios - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # prompt tokens": "{\"description\": \"min=756.479, mean=756.479, max=756.479, sum=1512.959 (2)\", \"tab\": \"General information\", \"score\": \"756.4793296089385\"}", - "Moral Scenarios - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"moral_scenarios\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_moral_scenarios\"" - } - } - }, - { - "evaluation_name": "Nutrition", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Nutrition", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.758, - "details": { - "description": "min=0.758, mean=0.758, max=0.758, sum=1.516 (2)", - "tab": "Accuracy", - "Nutrition - Observed inference time (s)": "{\"description\": \"min=0.447, mean=0.447, max=0.447, sum=0.895 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.44729572885176716\"}", - "Nutrition - # eval": "{\"description\": \"min=306, mean=306, max=306, sum=612 (2)\", \"tab\": \"General information\", \"score\": \"306.0\"}", - "Nutrition - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Nutrition - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Nutrition - # prompt tokens": "{\"description\": \"min=695.922, mean=695.922, max=695.922, sum=1391.843 (2)\", \"tab\": \"General information\", \"score\": \"695.9215686274509\"}", - "Nutrition - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"nutrition\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_nutrition\"" - } - } - }, - { - "evaluation_name": "Prehistory", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Prehistory", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.84, - "details": { - "description": "min=0.84, mean=0.84, max=0.84, sum=1.679 (2)", - "tab": "Accuracy", - "Prehistory - Observed inference time (s)": "{\"description\": \"min=0.472, mean=0.472, max=0.472, sum=0.945 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4722691575686137\"}", - "Prehistory - # eval": "{\"description\": \"min=324, mean=324, max=324, sum=648 (2)\", \"tab\": \"General information\", \"score\": \"324.0\"}", - "Prehistory - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Prehistory - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Prehistory - # prompt tokens": "{\"description\": \"min=619.185, mean=619.185, max=619.185, sum=1238.37 (2)\", \"tab\": \"General information\", \"score\": \"619.1851851851852\"}", - "Prehistory - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"prehistory\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_prehistory\"" - } - } - }, - { - "evaluation_name": "Public Relations", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Public Relations", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.745, - "details": { - "description": "min=0.745, mean=0.745, max=0.745, sum=1.491 (2)", - "tab": "Accuracy", - "Public Relations - Observed inference time (s)": "{\"description\": \"min=0.345, mean=0.345, max=0.345, sum=0.69 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.34489609761671586\"}", - "Public Relations - # eval": "{\"description\": \"min=110, mean=110, max=110, sum=220 (2)\", \"tab\": \"General information\", \"score\": \"110.0\"}", - "Public Relations - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Public Relations - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Public Relations - # prompt tokens": "{\"description\": \"min=474.827, mean=474.827, max=474.827, sum=949.655 (2)\", \"tab\": \"General information\", \"score\": \"474.8272727272727\"}", - "Public Relations - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"public_relations\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_public_relations\"" - } - } - }, - { - "evaluation_name": "Security Studies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Security Studies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.796, - "details": { - "description": "min=0.796, mean=0.796, max=0.796, sum=1.592 (2)", - "tab": "Accuracy", - "Security Studies - Observed inference time (s)": "{\"description\": \"min=0.484, mean=0.484, max=0.484, sum=0.968 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.48404579649166185\"}", - "Security Studies - # eval": "{\"description\": \"min=245, mean=245, max=245, sum=490 (2)\", \"tab\": \"General information\", \"score\": \"245.0\"}", - "Security Studies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Security Studies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Security Studies - # prompt tokens": "{\"description\": \"min=1377.531, mean=1377.531, max=1377.531, sum=2755.061 (2)\", \"tab\": \"General information\", \"score\": \"1377.530612244898\"}", - "Security Studies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"security_studies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_security_studies\"" - } - } - }, - { - "evaluation_name": "Sociology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Sociology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9, - "details": { - "description": "min=0.9, mean=0.9, max=0.9, sum=1.801 (2)", - "tab": "Accuracy", - "Sociology - Observed inference time (s)": "{\"description\": \"min=0.384, mean=0.384, max=0.384, sum=0.769 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.38445919781775023\"}", - "Sociology - # eval": "{\"description\": \"min=201, mean=201, max=201, sum=402 (2)\", \"tab\": \"General information\", \"score\": \"201.0\"}", - "Sociology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Sociology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Sociology - # prompt tokens": "{\"description\": \"min=508.478, mean=508.478, max=508.478, sum=1016.955 (2)\", \"tab\": \"General information\", \"score\": \"508.4776119402985\"}", - "Sociology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"sociology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_sociology\"" - } - } - }, - { - "evaluation_name": "Virology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Virology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.53, - "details": { - "description": "min=0.53, mean=0.53, max=0.53, sum=1.06 (2)", - "tab": "Accuracy", - "Virology - Observed inference time (s)": "{\"description\": \"min=0.464, mean=0.464, max=0.464, sum=0.928 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.464106645928808\"}", - "Virology - # eval": "{\"description\": \"min=166, mean=166, max=166, sum=332 (2)\", \"tab\": \"General information\", \"score\": \"166.0\"}", - "Virology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Virology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Virology - # prompt tokens": "{\"description\": \"min=405.108, mean=405.108, max=405.108, sum=810.217 (2)\", \"tab\": \"General information\", \"score\": \"405.10843373493975\"}", - "Virology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"virology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_virology\"" - } - } - }, - { - "evaluation_name": "World Religions", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on World Religions", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.854, - "details": { - "description": "min=0.854, mean=0.854, max=0.854, sum=1.708 (2)", - "tab": "Accuracy", - "World Religions - Observed inference time (s)": "{\"description\": \"min=0.416, mean=0.416, max=0.416, sum=0.831 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.41569664603785467\"}", - "World Religions - # eval": "{\"description\": \"min=171, mean=171, max=171, sum=342 (2)\", \"tab\": \"General information\", \"score\": \"171.0\"}", - "World Religions - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "World Religions - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "World Religions - # prompt tokens": "{\"description\": \"min=304.474, mean=304.474, max=304.474, sum=608.947 (2)\", \"tab\": \"General information\", \"score\": \"304.4736842105263\"}", - "World Religions - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"world_religions\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_world_religions\"" - } - } - }, - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.508, - "details": { - "description": "", - "tab": "Efficiency" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/meta_llama-2-7b.json b/data/models/meta_llama-2-7b.json deleted file mode 100644 index 9c124cf8984108fc01da5f2c825db9c3874e4441..0000000000000000000000000000000000000000 --- a/data/models/meta_llama-2-7b.json +++ /dev/null @@ -1,2570 +0,0 @@ -{ - "model_info": { - "name": "Llama 2 7B", - "id": "meta/llama-2-7b", - "developer": "Meta", - "inference_platform": "unknown", - "additional_details": { - "model_id_aliases": [ - "meta/Llama-2-7B" - ] - } - }, - "evaluations": [ - { - "evaluation_id": "helm_classic/meta_Llama-2-7B/1774096308.339228", - "retrieved_timestamp": "1774096308.339228", - "source_metadata": { - "source_name": "helm_classic", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_classic", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_classic", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperform on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.607, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Calibration": "{\"description\": \"\", \"tab\": \"Calibration\", \"score\": \"\"}", - "Mean win rate - Robustness": "{\"description\": \"\", \"tab\": \"Robustness\", \"score\": \"0.6437529137529138\"}", - "Mean win rate - Fairness": "{\"description\": \"\", \"tab\": \"Fairness\", \"score\": \"0.6102097902097903\"}", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}", - "Mean win rate - Bias": "{\"description\": \"\", \"tab\": \"Bias\", \"score\": \"0.4576728062932413\"}", - "Mean win rate - Toxicity": "{\"description\": \"\", \"tab\": \"Toxicity\", \"score\": \"0.8121794871794872\"}", - "Mean win rate - Summarization metrics": "{\"description\": \"\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.431, - "details": { - "description": "min=0.28, mean=0.431, max=0.64, sum=2.153 (5)", - "tab": "Accuracy", - "MMLU - ECE (10-bin)": "{\"description\": \"5 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "MMLU - EM (Robustness)": "{\"description\": \"min=0.22, mean=0.373, max=0.57, sum=1.866 (5)\", \"tab\": \"Robustness\", \"score\": \"0.37312280701754386\"}", - "MMLU - EM (Fairness)": "{\"description\": \"min=0.26, mean=0.392, max=0.59, sum=1.961 (5)\", \"tab\": \"Fairness\", \"score\": \"0.392140350877193\"}", - "MMLU - Denoised inference time (s)": "{\"description\": \"5 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=397.65, mean=522.547, max=684.675, sum=2612.735 (5)\", \"tab\": \"General information\", \"score\": \"522.5470877192982\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "MMLU - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "BoolQ", - "source_data": { - "dataset_name": "BoolQ", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on BoolQ", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.762, - "details": { - "description": "min=0.762, mean=0.762, max=0.762, sum=0.762 (1)", - "tab": "Accuracy", - "BoolQ - ECE (10-bin)": "{\"description\": \"min=0.215, mean=0.215, max=0.215, sum=0.215 (1)\", \"tab\": \"Calibration\", \"score\": \"\"}", - "BoolQ - EM (Robustness)": "{\"description\": \"min=0.676, mean=0.676, max=0.676, sum=0.676 (1)\", \"tab\": \"Robustness\", \"score\": \"0.676\"}", - "BoolQ - EM (Fairness)": "{\"description\": \"min=0.706, mean=0.706, max=0.706, sum=0.706 (1)\", \"tab\": \"Fairness\", \"score\": \"0.706\"}", - "BoolQ - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "BoolQ - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "BoolQ - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "BoolQ - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "BoolQ - # prompt tokens": "{\"description\": \"min=1439.447, mean=1439.447, max=1439.447, sum=1439.447 (1)\", \"tab\": \"General information\", \"score\": \"1439.447\"}", - "BoolQ - # output tokens": "{\"description\": \"min=1.296, mean=1.296, max=1.296, sum=1.296 (1)\", \"tab\": \"General information\", \"score\": \"1.296\"}", - "BoolQ - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "BoolQ - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.691, - "details": { - "description": "min=0.691, mean=0.691, max=0.691, sum=0.691 (1)", - "tab": "Accuracy", - "NarrativeQA - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "NarrativeQA - F1 (Robustness)": "{\"description\": \"min=0.573, mean=0.573, max=0.573, sum=0.573 (1)\", \"tab\": \"Robustness\", \"score\": \"0.5726018964106345\"}", - "NarrativeQA - F1 (Fairness)": "{\"description\": \"min=0.596, mean=0.596, max=0.596, sum=0.596 (1)\", \"tab\": \"Fairness\", \"score\": \"0.5960691234215144\"}", - "NarrativeQA - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=4.414, mean=4.414, max=4.414, sum=4.414 (1)\", \"tab\": \"General information\", \"score\": \"4.414084507042253\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=3673.268, mean=3673.268, max=3673.268, sum=3673.268 (1)\", \"tab\": \"General information\", \"score\": \"3673.2676056338028\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NarrativeQA - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NarrativeQA - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NarrativeQA - Stereotypes (gender)": "{\"description\": \"min=0.333, mean=0.333, max=0.333, sum=0.333 (1)\", \"tab\": \"Bias\", \"score\": \"0.3333333333333333\"}", - "NarrativeQA - Representation (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=0.667 (1)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666667\"}", - "NarrativeQA - Representation (gender)": "{\"description\": \"min=0.203, mean=0.203, max=0.203, sum=0.203 (1)\", \"tab\": \"Bias\", \"score\": \"0.20348837209302328\"}", - "NarrativeQA - Toxic fraction": "{\"description\": \"min=0.011, mean=0.011, max=0.011, sum=0.011 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.011267605633802818\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (open-book)", - "source_data": { - "dataset_name": "NaturalQuestions (open-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (open-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.611, - "details": { - "description": "min=0.611, mean=0.611, max=0.611, sum=0.611 (1)", - "tab": "Accuracy", - "NaturalQuestions (closed-book) - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "NaturalQuestions (open-book) - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - F1 (Robustness)": "{\"description\": \"min=0.261, mean=0.261, max=0.261, sum=0.261 (1)\", \"tab\": \"Robustness\", \"score\": \"0.2606038875824225\"}", - "NaturalQuestions (open-book) - F1 (Robustness)": "{\"description\": \"min=0.501, mean=0.501, max=0.501, sum=0.501 (1)\", \"tab\": \"Robustness\", \"score\": \"0.5010811862440044\"}", - "NaturalQuestions (closed-book) - F1 (Fairness)": "{\"description\": \"min=0.264, mean=0.264, max=0.264, sum=0.264 (1)\", \"tab\": \"Fairness\", \"score\": \"0.26403309290317406\"}", - "NaturalQuestions (open-book) - F1 (Fairness)": "{\"description\": \"min=0.55, mean=0.55, max=0.55, sum=0.55 (1)\", \"tab\": \"Fairness\", \"score\": \"0.5499198184166533\"}", - "NaturalQuestions (closed-book) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NaturalQuestions (open-book) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=137.383, mean=137.383, max=137.383, sum=137.383 (1)\", \"tab\": \"General information\", \"score\": \"137.383\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=0.998, mean=0.998, max=0.998, sum=0.998 (1)\", \"tab\": \"General information\", \"score\": \"0.998\"}", - "NaturalQuestions (closed-book) - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.831, mean=4.831, max=4.831, sum=4.831 (1)\", \"tab\": \"General information\", \"score\": \"4.831\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.026, mean=0.026, max=0.026, sum=0.026 (1)\", \"tab\": \"General information\", \"score\": \"0.026\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=2289.409, mean=2289.409, max=2289.409, sum=2289.409 (1)\", \"tab\": \"General information\", \"score\": \"2289.409\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=0.955, mean=0.955, max=0.955, sum=0.955 (1)\", \"tab\": \"General information\", \"score\": \"0.955\"}", - "NaturalQuestions (open-book) - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NaturalQuestions (closed-book) - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - Stereotypes (gender)": "{\"description\": \"min=0.5, mean=0.5, max=0.5, sum=0.5 (1)\", \"tab\": \"Bias\", \"score\": \"0.5\"}", - "NaturalQuestions (closed-book) - Representation (race)": "{\"description\": \"min=0.381, mean=0.381, max=0.381, sum=0.381 (1)\", \"tab\": \"Bias\", \"score\": \"0.38095238095238093\"}", - "NaturalQuestions (closed-book) - Representation (gender)": "{\"description\": \"min=0.182, mean=0.182, max=0.182, sum=0.182 (1)\", \"tab\": \"Bias\", \"score\": \"0.18181818181818182\"}", - "NaturalQuestions (open-book) - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=0.667 (1)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666667\"}", - "NaturalQuestions (open-book) - Stereotypes (gender)": "{\"description\": \"min=0.5, mean=0.5, max=0.5, sum=0.5 (1)\", \"tab\": \"Bias\", \"score\": \"0.5\"}", - "NaturalQuestions (open-book) - Representation (race)": "{\"description\": \"min=0.577, mean=0.577, max=0.577, sum=0.577 (1)\", \"tab\": \"Bias\", \"score\": \"0.5770114942528735\"}", - "NaturalQuestions (open-book) - Representation (gender)": "{\"description\": \"min=0.486, mean=0.486, max=0.486, sum=0.486 (1)\", \"tab\": \"Bias\", \"score\": \"0.48630136986301375\"}", - "NaturalQuestions (closed-book) - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}", - "NaturalQuestions (open-book) - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "QuAC", - "source_data": { - "dataset_name": "QuAC", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on QuAC", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.406, - "details": { - "description": "min=0.406, mean=0.406, max=0.406, sum=0.406 (1)", - "tab": "Accuracy", - "QuAC - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "QuAC - F1 (Robustness)": "{\"description\": \"min=0.271, mean=0.271, max=0.271, sum=0.271 (1)\", \"tab\": \"Robustness\", \"score\": \"0.27069315379336467\"}", - "QuAC - F1 (Fairness)": "{\"description\": \"min=0.321, mean=0.321, max=0.321, sum=0.321 (1)\", \"tab\": \"Fairness\", \"score\": \"0.32122644280851614\"}", - "QuAC - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "QuAC - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "QuAC - # train": "{\"description\": \"min=3.204, mean=3.204, max=3.204, sum=3.204 (1)\", \"tab\": \"General information\", \"score\": \"3.204\"}", - "QuAC - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "QuAC - # prompt tokens": "{\"description\": \"min=3617.038, mean=3617.038, max=3617.038, sum=3617.038 (1)\", \"tab\": \"General information\", \"score\": \"3617.038\"}", - "QuAC - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "QuAC - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "QuAC - Stereotypes (race)": "{\"description\": \"min=0.583, mean=0.583, max=0.583, sum=0.583 (1)\", \"tab\": \"Bias\", \"score\": \"0.5833333333333334\"}", - "QuAC - Stereotypes (gender)": "{\"description\": \"min=0.426, mean=0.426, max=0.426, sum=0.426 (1)\", \"tab\": \"Bias\", \"score\": \"0.4264652792029702\"}", - "QuAC - Representation (race)": "{\"description\": \"min=0.283, mean=0.283, max=0.283, sum=0.283 (1)\", \"tab\": \"Bias\", \"score\": \"0.2831541218637993\"}", - "QuAC - Representation (gender)": "{\"description\": \"min=0.231, mean=0.231, max=0.231, sum=0.231 (1)\", \"tab\": \"Bias\", \"score\": \"0.23093681917211328\"}", - "QuAC - Toxic fraction": "{\"description\": \"min=0.001, mean=0.001, max=0.001, sum=0.001 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.001\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "HellaSwag", - "source_data": { - "dataset_name": "HellaSwag", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on HellaSwag", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "HellaSwag - ECE (10-bin)": "{\"description\": \"No matching runs\", \"tab\": \"Calibration\", \"score\": \"\"}", - "HellaSwag - EM (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "HellaSwag - EM (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "HellaSwag - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "HellaSwag - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "OpenbookQA - ECE (10-bin)": "{\"description\": \"No matching runs\", \"tab\": \"Calibration\", \"score\": \"\"}", - "OpenbookQA - EM (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "OpenbookQA - EM (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "OpenbookQA - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "OpenbookQA - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "TruthfulQA", - "source_data": { - "dataset_name": "TruthfulQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on TruthfulQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.272, - "details": { - "description": "min=0.272, mean=0.272, max=0.272, sum=0.272 (1)", - "tab": "Accuracy", - "TruthfulQA - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "TruthfulQA - EM (Robustness)": "{\"description\": \"min=0.234, mean=0.234, max=0.234, sum=0.234 (1)\", \"tab\": \"Robustness\", \"score\": \"0.23394495412844038\"}", - "TruthfulQA - EM (Fairness)": "{\"description\": \"min=0.223, mean=0.223, max=0.223, sum=0.223 (1)\", \"tab\": \"Fairness\", \"score\": \"0.22324159021406728\"}", - "TruthfulQA - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "TruthfulQA - # eval": "{\"description\": \"min=654, mean=654, max=654, sum=654 (1)\", \"tab\": \"General information\", \"score\": \"654.0\"}", - "TruthfulQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "TruthfulQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "TruthfulQA - # prompt tokens": "{\"description\": \"min=524.602, mean=524.602, max=524.602, sum=524.602 (1)\", \"tab\": \"General information\", \"score\": \"524.6024464831804\"}", - "TruthfulQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "TruthfulQA - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MS MARCO (TREC)", - "source_data": { - "dataset_name": "MS MARCO (TREC)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "NDCG@10 on MS MARCO (TREC)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "MS MARCO (regular) - RR@10 (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "MS MARCO (TREC) - NDCG@10 (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "MS MARCO (regular) - RR@10 (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "MS MARCO (TREC) - NDCG@10 (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "MS MARCO (regular) - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MS MARCO (TREC) - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MS MARCO (regular) - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "MS MARCO (TREC) - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CNN/DailyMail", - "source_data": { - "dataset_name": "CNN/DailyMail", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on CNN/DailyMail", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "CNN/DailyMail - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "CNN/DailyMail - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "CNN/DailyMail - SummaC": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - QAFactEval": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - BERTScore (F1)": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - Coverage": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - Density": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - Compression": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-faithfulness": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-relevance": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-coherence": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "XSUM", - "source_data": { - "dataset_name": "XSUM", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on XSUM", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "XSUM - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "XSUM - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "XSUM - SummaC": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - QAFactEval": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - BERTScore (F1)": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - Coverage": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - Density": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - Compression": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-faithfulness": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-relevance": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-coherence": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "IMDB", - "source_data": { - "dataset_name": "IMDB", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on IMDB", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.907, - "details": { - "description": "min=0.907, mean=0.907, max=0.907, sum=0.907 (1)", - "tab": "Accuracy", - "IMDB - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "IMDB - EM (Robustness)": "{\"description\": \"min=0.808, mean=0.808, max=0.808, sum=0.808 (1)\", \"tab\": \"Robustness\", \"score\": \"0.808\"}", - "IMDB - EM (Fairness)": "{\"description\": \"min=0.871, mean=0.871, max=0.871, sum=0.871 (1)\", \"tab\": \"Fairness\", \"score\": \"0.871\"}", - "IMDB - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "IMDB - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "IMDB - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "IMDB - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IMDB - # prompt tokens": "{\"description\": \"min=2897.409, mean=2897.409, max=2897.409, sum=2897.409 (1)\", \"tab\": \"General information\", \"score\": \"2897.409\"}", - "IMDB - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "IMDB - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "IMDB - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CivilComments", - "source_data": { - "dataset_name": "CivilComments", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on CivilComments", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.562, - "details": { - "description": "min=0.025, mean=0.562, max=1, sum=10.108 (18)", - "tab": "Accuracy", - "CivilComments - ECE (10-bin)": "{\"description\": \"9 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "CivilComments - EM (Robustness)": "{\"description\": \"min=0.025, mean=0.516, max=0.989, sum=9.28 (18)\", \"tab\": \"Robustness\", \"score\": \"0.5155612610622284\"}", - "CivilComments - EM (Fairness)": "{\"description\": \"min=0.01, mean=0.503, max=0.998, sum=9.057 (18)\", \"tab\": \"Fairness\", \"score\": \"0.5031757189564859\"}", - "CivilComments - Denoised inference time (s)": "{\"description\": \"9 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "CivilComments - # eval": "{\"description\": \"min=74, mean=371.556, max=683, sum=6688 (18)\", \"tab\": \"General information\", \"score\": \"371.55555555555554\"}", - "CivilComments - # train": "{\"description\": \"min=5, mean=5, max=5, sum=90 (18)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CivilComments - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (18)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CivilComments - # prompt tokens": "{\"description\": \"min=404.732, mean=855.241, max=1417.567, sum=15394.339 (18)\", \"tab\": \"General information\", \"score\": \"855.2410378605821\"}", - "CivilComments - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=18 (18)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "CivilComments - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=18 (18)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "CivilComments - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Toxic fraction": "{\"description\": \"9 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "RAFT", - "source_data": { - "dataset_name": "RAFT", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on RAFT", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.643, - "details": { - "description": "min=0.125, mean=0.643, max=0.95, sum=7.075 (11)", - "tab": "Accuracy", - "RAFT - ECE (10-bin)": "{\"description\": \"11 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "RAFT - EM (Robustness)": "{\"description\": \"min=0.05, mean=0.573, max=0.875, sum=6.3 (11)\", \"tab\": \"Robustness\", \"score\": \"0.5727272727272728\"}", - "RAFT - EM (Fairness)": "{\"description\": \"min=0.1, mean=0.609, max=0.95, sum=6.7 (11)\", \"tab\": \"Fairness\", \"score\": \"0.6090909090909092\"}", - "RAFT - Denoised inference time (s)": "{\"description\": \"11 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "RAFT - # eval": "{\"description\": \"min=40, mean=40, max=40, sum=440 (11)\", \"tab\": \"General information\", \"score\": \"40.0\"}", - "RAFT - # train": "{\"description\": \"min=2.575, mean=4.78, max=5, sum=52.575 (11)\", \"tab\": \"General information\", \"score\": \"4.779545454545455\"}", - "RAFT - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (11)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "RAFT - # prompt tokens": "{\"description\": \"min=303.675, mean=1153.852, max=3623.9, sum=12692.375 (11)\", \"tab\": \"General information\", \"score\": \"1153.8522727272727\"}", - "RAFT - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=11 (11)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "RAFT - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=11 (11)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "RAFT - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Toxic fraction": "{\"description\": \"11 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_lite/meta_llama-2-7b/1774096306.427425", - "retrieved_timestamp": "1774096306.427425", - "source_metadata": { - "source_name": "helm_lite", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_lite", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_lite", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.152, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.6685767790262173\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.686, - "details": { - "description": "min=0.686, mean=0.686, max=0.686, sum=0.686 (1)", - "tab": "Accuracy", - "NarrativeQA - Observed inference time (s)": "{\"description\": \"min=0.852, mean=0.852, max=0.852, sum=0.852 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.8524049973823655\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=4.408, mean=4.408, max=4.408, sum=4.408 (1)\", \"tab\": \"General information\", \"score\": \"4.408450704225352\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=3669.808, mean=3669.808, max=3669.808, sum=3669.808 (1)\", \"tab\": \"General information\", \"score\": \"3669.8084507042254\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (closed-book)", - "source_data": { - "dataset_name": "NaturalQuestions (closed-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (closed-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.333, - "details": { - "description": "min=0.333, mean=0.333, max=0.333, sum=0.333 (1)", - "tab": "Accuracy", - "NaturalQuestions (open-book) - Observed inference time (s)": "{\"description\": \"min=0.584, mean=0.584, max=0.584, sum=0.584 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.584290323972702\"}", - "NaturalQuestions (closed-book) - Observed inference time (s)": "{\"description\": \"min=0.479, mean=0.479, max=0.479, sum=0.479 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.47909903168678286\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.831, mean=4.831, max=4.831, sum=4.831 (1)\", \"tab\": \"General information\", \"score\": \"4.831\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.026, mean=0.026, max=0.026, sum=0.026 (1)\", \"tab\": \"General information\", \"score\": \"0.026\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=2289.357, mean=2289.357, max=2289.357, sum=2289.357 (1)\", \"tab\": \"General information\", \"score\": \"2289.357\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=0.958, mean=0.958, max=0.958, sum=0.958 (1)\", \"tab\": \"General information\", \"score\": \"0.958\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=137.383, mean=137.383, max=137.383, sum=137.383 (1)\", \"tab\": \"General information\", \"score\": \"137.383\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=0.996, mean=0.996, max=0.996, sum=0.996 (1)\", \"tab\": \"General information\", \"score\": \"0.996\"}" - } - }, - "generation_config": { - "additional_details": { - "mode": "\"closedbook\"" - } - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.544, - "details": { - "description": "min=0.544, mean=0.544, max=0.544, sum=0.544 (1)", - "tab": "Accuracy", - "OpenbookQA - Observed inference time (s)": "{\"description\": \"min=0.393, mean=0.393, max=0.393, sum=0.393 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.3927152595520019\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=282.574, mean=282.574, max=282.574, sum=282.574 (1)\", \"tab\": \"General information\", \"score\": \"282.574\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "dataset": "\"openbookqa\"", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.425, - "details": { - "description": "min=0.27, mean=0.425, max=0.63, sum=2.125 (5)", - "tab": "Accuracy", - "MMLU - Observed inference time (s)": "{\"description\": \"min=0.314, mean=0.33, max=0.349, sum=1.651 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.33028721380233766\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=397.65, mean=522.547, max=684.675, sum=2612.735 (5)\", \"tab\": \"General information\", \"score\": \"522.5470877192982\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"college_chemistry\", \"computer_security\", \"econometrics\", \"us_foreign_policy\"]", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MATH", - "source_data": { - "dataset_name": "MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Equivalent (CoT) on MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.097, - "details": { - "description": "min=0.019, mean=0.097, max=0.198, sum=0.68 (7)", - "tab": "Accuracy", - "MATH - Observed inference time (s)": "{\"description\": \"min=1.362, mean=2.66, max=5.271, sum=18.621 (7)\", \"tab\": \"Efficiency\", \"score\": \"2.6600816047289086\"}", - "MATH - # eval": "{\"description\": \"min=30, mean=62.429, max=135, sum=437 (7)\", \"tab\": \"General information\", \"score\": \"62.42857142857143\"}", - "MATH - # train": "{\"description\": \"min=8, mean=8, max=8, sum=56 (7)\", \"tab\": \"General information\", \"score\": \"8.0\"}", - "MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MATH - # prompt tokens": "{\"description\": \"min=971.652, mean=1438.636, max=2490.962, sum=10070.453 (7)\", \"tab\": \"General information\", \"score\": \"1438.6362030100095\"}", - "MATH - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=7 (7)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"algebra\", \"counting_and_probability\", \"geometry\", \"intermediate_algebra\", \"number_theory\", \"prealgebra\", \"precalculus\"]", - "level": "\"1\"", - "use_official_examples": "\"False\"", - "use_chain_of_thought": "\"True\"" - } - } - }, - { - "evaluation_name": "GSM8K", - "source_data": { - "dataset_name": "GSM8K", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on GSM8K", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.154, - "details": { - "description": "min=0.154, mean=0.154, max=0.154, sum=0.154 (1)", - "tab": "Accuracy", - "GSM8K - Observed inference time (s)": "{\"description\": \"min=1.96, mean=1.96, max=1.96, sum=1.96 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.95984334897995\"}", - "GSM8K - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "GSM8K - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "GSM8K - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GSM8K - # prompt tokens": "{\"description\": \"min=1207.746, mean=1207.746, max=1207.746, sum=1207.746 (1)\", \"tab\": \"General information\", \"score\": \"1207.746\"}", - "GSM8K - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "LegalBench", - "source_data": { - "dataset_name": "LegalBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on LegalBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.502, - "details": { - "description": "min=0.245, mean=0.502, max=0.747, sum=2.508 (5)", - "tab": "Accuracy", - "LegalBench - Observed inference time (s)": "{\"description\": \"min=0.306, mean=0.428, max=0.76, sum=2.139 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.4277655324222306\"}", - "LegalBench - # eval": "{\"description\": \"min=95, mean=409.4, max=1000, sum=2047 (5)\", \"tab\": \"General information\", \"score\": \"409.4\"}", - "LegalBench - # train": "{\"description\": \"min=1.886, mean=4.177, max=5, sum=20.886 (5)\", \"tab\": \"General information\", \"score\": \"4.177142857142857\"}", - "LegalBench - truncated": "{\"description\": \"min=0, mean=0.001, max=0.004, sum=0.004 (5)\", \"tab\": \"General information\", \"score\": \"0.0008163265306122449\"}", - "LegalBench - # prompt tokens": "{\"description\": \"min=222.137, mean=1027.35, max=3642.378, sum=5136.751 (5)\", \"tab\": \"General information\", \"score\": \"1027.3502076083553\"}", - "LegalBench - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"abercrombie\", \"corporate_lobbying\", \"function_of_decision_section\", \"international_citizenship_questions\", \"proa\"]" - } - } - }, - { - "evaluation_name": "MedQA", - "source_data": { - "dataset_name": "MedQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MedQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.392, - "details": { - "description": "min=0.392, mean=0.392, max=0.392, sum=0.392 (1)", - "tab": "Accuracy", - "MedQA - Observed inference time (s)": "{\"description\": \"min=0.467, mean=0.467, max=0.467, sum=0.467 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.46650436763497993\"}", - "MedQA - # eval": "{\"description\": \"min=503, mean=503, max=503, sum=503 (1)\", \"tab\": \"General information\", \"score\": \"503.0\"}", - "MedQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MedQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MedQA - # prompt tokens": "{\"description\": \"min=1234.901, mean=1234.901, max=1234.901, sum=1234.901 (1)\", \"tab\": \"General information\", \"score\": \"1234.9005964214712\"}", - "MedQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WMT 2014", - "source_data": { - "dataset_name": "WMT 2014", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "BLEU-4 on WMT 2014", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.144, - "details": { - "description": "min=0.046, mean=0.144, max=0.189, sum=0.72 (5)", - "tab": "Accuracy", - "WMT 2014 - Observed inference time (s)": "{\"description\": \"min=0.582, mean=0.697, max=0.802, sum=3.486 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.697166075241057\"}", - "WMT 2014 - # eval": "{\"description\": \"min=503, mean=568.8, max=832, sum=2844 (5)\", \"tab\": \"General information\", \"score\": \"568.8\"}", - "WMT 2014 - # train": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "WMT 2014 - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WMT 2014 - # prompt tokens": "{\"description\": \"min=127.523, mean=142.288, max=164.972, sum=711.438 (5)\", \"tab\": \"General information\", \"score\": \"142.28751290334915\"}", - "WMT 2014 - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "language_pair": "[\"cs-en\", \"de-en\", \"fr-en\", \"hi-en\", \"ru-en\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_mmlu/meta_llama-2-7b/1774096312.00548", - "retrieved_timestamp": "1774096312.00548", - "source_metadata": { - "source_name": "helm_mmlu", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_mmlu", - "evaluation_results": [ - { - "evaluation_name": "MMLU All Subjects", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU All Subjects", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.458, - "details": { - "description": "min=0.196, mean=0.458, max=0.713, sum=52.224 (114)", - "tab": "Accuracy", - "MMLU All Subjects - Observed inference time (s)": "{\"description\": \"min=0.28, mean=0.374, max=0.947, sum=42.6 (114)\", \"tab\": \"Efficiency\", \"score\": \"0.37368440752207543\"}", - "MMLU All Subjects - # eval": "{\"description\": \"min=100, mean=246.351, max=1534, sum=28084 (114)\", \"tab\": \"General information\", \"score\": \"246.35087719298247\"}", - "MMLU All Subjects - # train": "{\"description\": \"min=5, mean=5, max=5, sum=570 (114)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU All Subjects - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU All Subjects - # prompt tokens": "{\"description\": \"min=304.474, mean=706.682, max=3159.636, sum=80561.749 (114)\", \"tab\": \"General information\", \"score\": \"706.6820126388612\"}", - "MMLU All Subjects - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=114 (114)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_name": "Abstract Algebra", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Abstract Algebra", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.29, - "details": { - "description": "min=0.29, mean=0.29, max=0.29, sum=0.58 (2)", - "tab": "Accuracy", - "Abstract Algebra - Observed inference time (s)": "{\"description\": \"min=0.332, mean=0.332, max=0.332, sum=0.664 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3319991087913513\"}", - "Abstract Algebra - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Abstract Algebra - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Abstract Algebra - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Abstract Algebra - # prompt tokens": "{\"description\": \"min=397.65, mean=397.65, max=397.65, sum=795.3 (2)\", \"tab\": \"General information\", \"score\": \"397.65\"}", - "Abstract Algebra - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"abstract_algebra\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_abstract_algebra\"" - } - } - }, - { - "evaluation_name": "Anatomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Anatomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.452, - "details": { - "description": "min=0.452, mean=0.452, max=0.452, sum=0.904 (2)", - "tab": "Accuracy", - "Anatomy - Observed inference time (s)": "{\"description\": \"min=0.31, mean=0.31, max=0.31, sum=0.619 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3097020767353199\"}", - "Anatomy - # eval": "{\"description\": \"min=135, mean=135, max=135, sum=270 (2)\", \"tab\": \"General information\", \"score\": \"135.0\"}", - "Anatomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Anatomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Anatomy - # prompt tokens": "{\"description\": \"min=409.133, mean=409.133, max=409.133, sum=818.267 (2)\", \"tab\": \"General information\", \"score\": \"409.1333333333333\"}", - "Anatomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"anatomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_anatomy\"" - } - } - }, - { - "evaluation_name": "College Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on College Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.196, - "details": { - "description": "min=0.196, mean=0.196, max=0.196, sum=0.392 (2)", - "tab": "Accuracy", - "College Chemistry - Observed inference time (s)": "{\"description\": \"min=0.35, mean=0.35, max=0.35, sum=0.7 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.35009843587875367\"}", - "College Biology - Observed inference time (s)": "{\"description\": \"min=0.328, mean=0.328, max=0.328, sum=0.656 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3278946164581511\"}", - "College Computer Science - Observed inference time (s)": "{\"description\": \"min=0.381, mean=0.381, max=0.381, sum=0.763 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.38129755973815915\"}", - "College Mathematics - Observed inference time (s)": "{\"description\": \"min=0.341, mean=0.341, max=0.341, sum=0.682 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3409119129180908\"}", - "College Medicine - Observed inference time (s)": "{\"description\": \"min=0.331, mean=0.331, max=0.331, sum=0.662 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3307889693045203\"}", - "College Physics - Observed inference time (s)": "{\"description\": \"min=0.34, mean=0.34, max=0.34, sum=0.68 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3398791224348779\"}", - "College Chemistry - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Chemistry - # prompt tokens": "{\"description\": \"min=622.43, mean=622.43, max=622.43, sum=1244.86 (2)\", \"tab\": \"General information\", \"score\": \"622.43\"}", - "College Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Biology - # eval": "{\"description\": \"min=144, mean=144, max=144, sum=288 (2)\", \"tab\": \"General information\", \"score\": \"144.0\"}", - "College Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # prompt tokens": "{\"description\": \"min=553.632, mean=553.632, max=553.632, sum=1107.264 (2)\", \"tab\": \"General information\", \"score\": \"553.6319444444445\"}", - "College Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # prompt tokens": "{\"description\": \"min=901.14, mean=901.14, max=901.14, sum=1802.28 (2)\", \"tab\": \"General information\", \"score\": \"901.14\"}", - "College Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Mathematics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # prompt tokens": "{\"description\": \"min=646.96, mean=646.96, max=646.96, sum=1293.92 (2)\", \"tab\": \"General information\", \"score\": \"646.96\"}", - "College Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Medicine - # eval": "{\"description\": \"min=173, mean=173, max=173, sum=346 (2)\", \"tab\": \"General information\", \"score\": \"173.0\"}", - "College Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # prompt tokens": "{\"description\": \"min=608.671, mean=608.671, max=608.671, sum=1217.341 (2)\", \"tab\": \"General information\", \"score\": \"608.6705202312139\"}", - "College Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Physics - # eval": "{\"description\": \"min=102, mean=102, max=102, sum=204 (2)\", \"tab\": \"General information\", \"score\": \"102.0\"}", - "College Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # prompt tokens": "{\"description\": \"min=551.873, mean=551.873, max=551.873, sum=1103.745 (2)\", \"tab\": \"General information\", \"score\": \"551.8725490196078\"}", - "College Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"college_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_college_physics\"" - } - } - }, - { - "evaluation_name": "Computer Security", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Computer Security", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.59, - "details": { - "description": "min=0.59, mean=0.59, max=0.59, sum=1.18 (2)", - "tab": "Accuracy", - "Computer Security - Observed inference time (s)": "{\"description\": \"min=0.329, mean=0.329, max=0.329, sum=0.659 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3293105459213257\"}", - "Computer Security - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Computer Security - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Computer Security - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Computer Security - # prompt tokens": "{\"description\": \"min=428.17, mean=428.17, max=428.17, sum=856.34 (2)\", \"tab\": \"General information\", \"score\": \"428.17\"}", - "Computer Security - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"computer_security\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_computer_security\"" - } - } - }, - { - "evaluation_name": "Econometrics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Econometrics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.316, - "details": { - "description": "min=0.316, mean=0.316, max=0.316, sum=0.632 (2)", - "tab": "Accuracy", - "Econometrics - Observed inference time (s)": "{\"description\": \"min=0.375, mean=0.375, max=0.375, sum=0.749 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3746668204926608\"}", - "Econometrics - # eval": "{\"description\": \"min=114, mean=114, max=114, sum=228 (2)\", \"tab\": \"General information\", \"score\": \"114.0\"}", - "Econometrics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Econometrics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Econometrics - # prompt tokens": "{\"description\": \"min=684.675, mean=684.675, max=684.675, sum=1369.351 (2)\", \"tab\": \"General information\", \"score\": \"684.6754385964912\"}", - "Econometrics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"econometrics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_econometrics\"" - } - } - }, - { - "evaluation_name": "Global Facts", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Global Facts", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.29, - "details": { - "description": "min=0.29, mean=0.29, max=0.29, sum=0.58 (2)", - "tab": "Accuracy", - "Global Facts - Observed inference time (s)": "{\"description\": \"min=0.329, mean=0.329, max=0.329, sum=0.659 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.32934638738632205\"}", - "Global Facts - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Global Facts - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Global Facts - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Global Facts - # prompt tokens": "{\"description\": \"min=484.54, mean=484.54, max=484.54, sum=969.08 (2)\", \"tab\": \"General information\", \"score\": \"484.54\"}", - "Global Facts - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"global_facts\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_global_facts\"" - } - } - }, - { - "evaluation_name": "Jurisprudence", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Jurisprudence", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.519, - "details": { - "description": "min=0.519, mean=0.519, max=0.519, sum=1.037 (2)", - "tab": "Accuracy", - "Jurisprudence - Observed inference time (s)": "{\"description\": \"min=0.294, mean=0.294, max=0.294, sum=0.588 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2942208139984696\"}", - "Jurisprudence - # eval": "{\"description\": \"min=108, mean=108, max=108, sum=216 (2)\", \"tab\": \"General information\", \"score\": \"108.0\"}", - "Jurisprudence - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Jurisprudence - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Jurisprudence - # prompt tokens": "{\"description\": \"min=449.898, mean=449.898, max=449.898, sum=899.796 (2)\", \"tab\": \"General information\", \"score\": \"449.89814814814815\"}", - "Jurisprudence - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"jurisprudence\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_jurisprudence\"" - } - } - }, - { - "evaluation_name": "Philosophy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Philosophy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.592, - "details": { - "description": "min=0.592, mean=0.592, max=0.592, sum=1.183 (2)", - "tab": "Accuracy", - "Philosophy - Observed inference time (s)": "{\"description\": \"min=0.3, mean=0.3, max=0.3, sum=0.6 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2999055814896366\"}", - "Philosophy - # eval": "{\"description\": \"min=311, mean=311, max=311, sum=622 (2)\", \"tab\": \"General information\", \"score\": \"311.0\"}", - "Philosophy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Philosophy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Philosophy - # prompt tokens": "{\"description\": \"min=372.122, mean=372.122, max=372.122, sum=744.244 (2)\", \"tab\": \"General information\", \"score\": \"372.12218649517683\"}", - "Philosophy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"philosophy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_philosophy\"" - } - } - }, - { - "evaluation_name": "Professional Psychology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Professional Psychology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.459, - "details": { - "description": "min=0.459, mean=0.459, max=0.459, sum=0.918 (2)", - "tab": "Accuracy", - "Professional Medicine - Observed inference time (s)": "{\"description\": \"min=0.463, mean=0.463, max=0.463, sum=0.926 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.463154871674145\"}", - "Professional Accounting - Observed inference time (s)": "{\"description\": \"min=0.377, mean=0.377, max=0.377, sum=0.755 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.37741253392916196\"}", - "Professional Law - Observed inference time (s)": "{\"description\": \"min=0.618, mean=0.618, max=0.618, sum=1.235 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6177054020385543\"}", - "Professional Psychology - Observed inference time (s)": "{\"description\": \"min=0.44, mean=0.44, max=0.44, sum=0.879 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4397414544828577\"}", - "Professional Medicine - # eval": "{\"description\": \"min=272, mean=272, max=272, sum=544 (2)\", \"tab\": \"General information\", \"score\": \"272.0\"}", - "Professional Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Medicine - # prompt tokens": "{\"description\": \"min=1330.647, mean=1330.647, max=1330.647, sum=2661.294 (2)\", \"tab\": \"General information\", \"score\": \"1330.6470588235295\"}", - "Professional Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Accounting - # eval": "{\"description\": \"min=282, mean=282, max=282, sum=564 (2)\", \"tab\": \"General information\", \"score\": \"282.0\"}", - "Professional Accounting - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Accounting - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # prompt tokens": "{\"description\": \"min=823.277, mean=823.277, max=823.277, sum=1646.553 (2)\", \"tab\": \"General information\", \"score\": \"823.2765957446809\"}", - "Professional Accounting - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Law - # eval": "{\"description\": \"min=1534, mean=1534, max=1534, sum=3068 (2)\", \"tab\": \"General information\", \"score\": \"1534.0\"}", - "Professional Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # prompt tokens": "{\"description\": \"min=1915.007, mean=1915.007, max=1915.007, sum=3830.014 (2)\", \"tab\": \"General information\", \"score\": \"1915.0071707953064\"}", - "Professional Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Psychology - # eval": "{\"description\": \"min=612, mean=612, max=612, sum=1224 (2)\", \"tab\": \"General information\", \"score\": \"612.0\"}", - "Professional Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # prompt tokens": "{\"description\": \"min=650.078, mean=650.078, max=650.078, sum=1300.157 (2)\", \"tab\": \"General information\", \"score\": \"650.0784313725491\"}", - "Professional Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"professional_psychology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_professional_psychology\"" - } - } - }, - { - "evaluation_name": "Us Foreign Policy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Us Foreign Policy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.64, - "details": { - "description": "min=0.64, mean=0.64, max=0.64, sum=1.28 (2)", - "tab": "Accuracy", - "Us Foreign Policy - Observed inference time (s)": "{\"description\": \"min=0.343, mean=0.343, max=0.343, sum=0.686 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3431359338760376\"}", - "Us Foreign Policy - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Us Foreign Policy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Us Foreign Policy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Us Foreign Policy - # prompt tokens": "{\"description\": \"min=479.81, mean=479.81, max=479.81, sum=959.62 (2)\", \"tab\": \"General information\", \"score\": \"479.81\"}", - "Us Foreign Policy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"us_foreign_policy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_us_foreign_policy\"" - } - } - }, - { - "evaluation_name": "Astronomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Astronomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.408, - "details": { - "description": "min=0.408, mean=0.408, max=0.408, sum=0.816 (2)", - "tab": "Accuracy", - "Astronomy - Observed inference time (s)": "{\"description\": \"min=0.345, mean=0.345, max=0.345, sum=0.69 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.34498921193574605\"}", - "Astronomy - # eval": "{\"description\": \"min=152, mean=152, max=152, sum=304 (2)\", \"tab\": \"General information\", \"score\": \"152.0\"}", - "Astronomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Astronomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Astronomy - # prompt tokens": "{\"description\": \"min=681.079, mean=681.079, max=681.079, sum=1362.158 (2)\", \"tab\": \"General information\", \"score\": \"681.078947368421\"}", - "Astronomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"astronomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_astronomy\"" - } - } - }, - { - "evaluation_name": "Business Ethics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Business Ethics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.48, - "details": { - "description": "min=0.48, mean=0.48, max=0.48, sum=0.96 (2)", - "tab": "Accuracy", - "Business Ethics - Observed inference time (s)": "{\"description\": \"min=0.334, mean=0.334, max=0.334, sum=0.668 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3342457461357117\"}", - "Business Ethics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Business Ethics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Business Ethics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Business Ethics - # prompt tokens": "{\"description\": \"min=674.44, mean=674.44, max=674.44, sum=1348.88 (2)\", \"tab\": \"General information\", \"score\": \"674.44\"}", - "Business Ethics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"business_ethics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_business_ethics\"" - } - } - }, - { - "evaluation_name": "Clinical Knowledge", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Clinical Knowledge", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.453, - "details": { - "description": "min=0.453, mean=0.453, max=0.453, sum=0.906 (2)", - "tab": "Accuracy", - "Clinical Knowledge - Observed inference time (s)": "{\"description\": \"min=0.323, mean=0.323, max=0.323, sum=0.645 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3225168426081819\"}", - "Clinical Knowledge - # eval": "{\"description\": \"min=265, mean=265, max=265, sum=530 (2)\", \"tab\": \"General information\", \"score\": \"265.0\"}", - "Clinical Knowledge - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Clinical Knowledge - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Clinical Knowledge - # prompt tokens": "{\"description\": \"min=487.374, mean=487.374, max=487.374, sum=974.747 (2)\", \"tab\": \"General information\", \"score\": \"487.3735849056604\"}", - "Clinical Knowledge - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"clinical_knowledge\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_clinical_knowledge\"" - } - } - }, - { - "evaluation_name": "Conceptual Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Conceptual Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.434, - "details": { - "description": "min=0.434, mean=0.434, max=0.434, sum=0.868 (2)", - "tab": "Accuracy", - "Conceptual Physics - Observed inference time (s)": "{\"description\": \"min=0.323, mean=0.323, max=0.323, sum=0.646 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.32303770450835534\"}", - "Conceptual Physics - # eval": "{\"description\": \"min=235, mean=235, max=235, sum=470 (2)\", \"tab\": \"General information\", \"score\": \"235.0\"}", - "Conceptual Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Conceptual Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Conceptual Physics - # prompt tokens": "{\"description\": \"min=333.153, mean=333.153, max=333.153, sum=666.306 (2)\", \"tab\": \"General information\", \"score\": \"333.1531914893617\"}", - "Conceptual Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"conceptual_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_conceptual_physics\"" - } - } - }, - { - "evaluation_name": "Electrical Engineering", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Electrical Engineering", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.407, - "details": { - "description": "min=0.407, mean=0.407, max=0.407, sum=0.814 (2)", - "tab": "Accuracy", - "Electrical Engineering - Observed inference time (s)": "{\"description\": \"min=0.325, mean=0.325, max=0.325, sum=0.649 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.32454562516048036\"}", - "Electrical Engineering - # eval": "{\"description\": \"min=145, mean=145, max=145, sum=290 (2)\", \"tab\": \"General information\", \"score\": \"145.0\"}", - "Electrical Engineering - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Electrical Engineering - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Electrical Engineering - # prompt tokens": "{\"description\": \"min=497.779, mean=497.779, max=497.779, sum=995.559 (2)\", \"tab\": \"General information\", \"score\": \"497.7793103448276\"}", - "Electrical Engineering - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"electrical_engineering\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_electrical_engineering\"" - } - } - }, - { - "evaluation_name": "Elementary Mathematics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Elementary Mathematics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.254, - "details": { - "description": "min=0.254, mean=0.254, max=0.254, sum=0.508 (2)", - "tab": "Accuracy", - "Elementary Mathematics - Observed inference time (s)": "{\"description\": \"min=0.334, mean=0.334, max=0.334, sum=0.669 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.33426338718051\"}", - "Elementary Mathematics - # eval": "{\"description\": \"min=378, mean=378, max=378, sum=756 (2)\", \"tab\": \"General information\", \"score\": \"378.0\"}", - "Elementary Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Elementary Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Elementary Mathematics - # prompt tokens": "{\"description\": \"min=609.156, mean=609.156, max=609.156, sum=1218.312 (2)\", \"tab\": \"General information\", \"score\": \"609.1560846560847\"}", - "Elementary Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"elementary_mathematics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_elementary_mathematics\"" - } - } - }, - { - "evaluation_name": "Formal Logic", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Formal Logic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.27, - "details": { - "description": "min=0.27, mean=0.27, max=0.27, sum=0.54 (2)", - "tab": "Accuracy", - "Formal Logic - Observed inference time (s)": "{\"description\": \"min=0.383, mean=0.383, max=0.383, sum=0.766 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3832281846848745\"}", - "Formal Logic - # eval": "{\"description\": \"min=126, mean=126, max=126, sum=252 (2)\", \"tab\": \"General information\", \"score\": \"126.0\"}", - "Formal Logic - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Formal Logic - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Formal Logic - # prompt tokens": "{\"description\": \"min=691.81, mean=691.81, max=691.81, sum=1383.619 (2)\", \"tab\": \"General information\", \"score\": \"691.8095238095239\"}", - "Formal Logic - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"formal_logic\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_formal_logic\"" - } - } - }, - { - "evaluation_name": "High School World History", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on High School World History", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.662, - "details": { - "description": "min=0.662, mean=0.662, max=0.662, sum=1.325 (2)", - "tab": "Accuracy", - "High School Biology - Observed inference time (s)": "{\"description\": \"min=0.326, mean=0.326, max=0.326, sum=0.653 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.32630388890543294\"}", - "High School Chemistry - Observed inference time (s)": "{\"description\": \"min=0.306, mean=0.306, max=0.306, sum=0.611 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.30552317473688734\"}", - "High School Computer Science - Observed inference time (s)": "{\"description\": \"min=0.406, mean=0.406, max=0.406, sum=0.812 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4060112690925598\"}", - "High School European History - Observed inference time (s)": "{\"description\": \"min=0.947, mean=0.947, max=0.947, sum=1.894 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9469690496271307\"}", - "High School Geography - Observed inference time (s)": "{\"description\": \"min=0.327, mean=0.327, max=0.327, sum=0.655 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.32730214523546625\"}", - "High School Government And Politics - Observed inference time (s)": "{\"description\": \"min=0.337, mean=0.337, max=0.337, sum=0.674 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3369472236830954\"}", - "High School Macroeconomics - Observed inference time (s)": "{\"description\": \"min=0.331, mean=0.331, max=0.331, sum=0.662 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3308515047415709\"}", - "High School Mathematics - Observed inference time (s)": "{\"description\": \"min=0.336, mean=0.336, max=0.336, sum=0.671 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3355037459620723\"}", - "High School Microeconomics - Observed inference time (s)": "{\"description\": \"min=0.459, mean=0.459, max=0.459, sum=0.918 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.45884753475670054\"}", - "High School Physics - Observed inference time (s)": "{\"description\": \"min=0.336, mean=0.336, max=0.336, sum=0.671 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3355141222871692\"}", - "High School Psychology - Observed inference time (s)": "{\"description\": \"min=0.389, mean=0.389, max=0.389, sum=0.778 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3889624678760494\"}", - "High School Statistics - Observed inference time (s)": "{\"description\": \"min=0.393, mean=0.393, max=0.393, sum=0.786 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.39307444846188583\"}", - "High School US History - Observed inference time (s)": "{\"description\": \"min=0.778, mean=0.778, max=0.778, sum=1.556 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7781471855500165\"}", - "High School World History - Observed inference time (s)": "{\"description\": \"min=0.542, mean=0.542, max=0.542, sum=1.085 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5424087500270409\"}", - "High School Biology - # eval": "{\"description\": \"min=310, mean=310, max=310, sum=620 (2)\", \"tab\": \"General information\", \"score\": \"310.0\"}", - "High School Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Biology - # prompt tokens": "{\"description\": \"min=596.894, mean=596.894, max=596.894, sum=1193.787 (2)\", \"tab\": \"General information\", \"score\": \"596.8935483870968\"}", - "High School Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Chemistry - # eval": "{\"description\": \"min=203, mean=203, max=203, sum=406 (2)\", \"tab\": \"General information\", \"score\": \"203.0\"}", - "High School Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # prompt tokens": "{\"description\": \"min=568.665, mean=568.665, max=568.665, sum=1137.33 (2)\", \"tab\": \"General information\", \"score\": \"568.6650246305419\"}", - "High School Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "High School Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # prompt tokens": "{\"description\": \"min=988.57, mean=988.57, max=988.57, sum=1977.14 (2)\", \"tab\": \"General information\", \"score\": \"988.57\"}", - "High School Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School European History - # eval": "{\"description\": \"min=165, mean=165, max=165, sum=330 (2)\", \"tab\": \"General information\", \"score\": \"165.0\"}", - "High School European History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School European History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # prompt tokens": "{\"description\": \"min=3159.636, mean=3159.636, max=3159.636, sum=6319.273 (2)\", \"tab\": \"General information\", \"score\": \"3159.6363636363635\"}", - "High School European History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Geography - # eval": "{\"description\": \"min=198, mean=198, max=198, sum=396 (2)\", \"tab\": \"General information\", \"score\": \"198.0\"}", - "High School Geography - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Geography - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # prompt tokens": "{\"description\": \"min=436.657, mean=436.657, max=436.657, sum=873.313 (2)\", \"tab\": \"General information\", \"score\": \"436.65656565656565\"}", - "High School Geography - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Government And Politics - # eval": "{\"description\": \"min=193, mean=193, max=193, sum=386 (2)\", \"tab\": \"General information\", \"score\": \"193.0\"}", - "High School Government And Politics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Government And Politics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # prompt tokens": "{\"description\": \"min=527.927, mean=527.927, max=527.927, sum=1055.855 (2)\", \"tab\": \"General information\", \"score\": \"527.9274611398964\"}", - "High School Government And Politics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Macroeconomics - # eval": "{\"description\": \"min=390, mean=390, max=390, sum=780 (2)\", \"tab\": \"General information\", \"score\": \"390.0\"}", - "High School Macroeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Macroeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # prompt tokens": "{\"description\": \"min=445.662, mean=445.662, max=445.662, sum=891.323 (2)\", \"tab\": \"General information\", \"score\": \"445.66153846153844\"}", - "High School Macroeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Mathematics - # eval": "{\"description\": \"min=270, mean=270, max=270, sum=540 (2)\", \"tab\": \"General information\", \"score\": \"270.0\"}", - "High School Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # prompt tokens": "{\"description\": \"min=579.181, mean=579.181, max=579.181, sum=1158.363 (2)\", \"tab\": \"General information\", \"score\": \"579.1814814814815\"}", - "High School Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Microeconomics - # eval": "{\"description\": \"min=238, mean=238, max=238, sum=476 (2)\", \"tab\": \"General information\", \"score\": \"238.0\"}", - "High School Microeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Microeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # prompt tokens": "{\"description\": \"min=449.492, mean=449.492, max=449.492, sum=898.983 (2)\", \"tab\": \"General information\", \"score\": \"449.49159663865544\"}", - "High School Microeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Physics - # eval": "{\"description\": \"min=151, mean=151, max=151, sum=302 (2)\", \"tab\": \"General information\", \"score\": \"151.0\"}", - "High School Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # prompt tokens": "{\"description\": \"min=621.788, mean=621.788, max=621.788, sum=1243.576 (2)\", \"tab\": \"General information\", \"score\": \"621.7880794701987\"}", - "High School Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Psychology - # eval": "{\"description\": \"min=545, mean=545, max=545, sum=1090 (2)\", \"tab\": \"General information\", \"score\": \"545.0\"}", - "High School Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # prompt tokens": "{\"description\": \"min=585.919, mean=585.919, max=585.919, sum=1171.839 (2)\", \"tab\": \"General information\", \"score\": \"585.9192660550459\"}", - "High School Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Statistics - # eval": "{\"description\": \"min=216, mean=216, max=216, sum=432 (2)\", \"tab\": \"General information\", \"score\": \"216.0\"}", - "High School Statistics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Statistics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # prompt tokens": "{\"description\": \"min=908.208, mean=908.208, max=908.208, sum=1816.417 (2)\", \"tab\": \"General information\", \"score\": \"908.2083333333334\"}", - "High School Statistics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School US History - # eval": "{\"description\": \"min=204, mean=204, max=204, sum=408 (2)\", \"tab\": \"General information\", \"score\": \"204.0\"}", - "High School US History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School US History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # prompt tokens": "{\"description\": \"min=2535.324, mean=2535.324, max=2535.324, sum=5070.647 (2)\", \"tab\": \"General information\", \"score\": \"2535.323529411765\"}", - "High School US History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School World History - # eval": "{\"description\": \"min=237, mean=237, max=237, sum=474 (2)\", \"tab\": \"General information\", \"score\": \"237.0\"}", - "High School World History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School World History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # prompt tokens": "{\"description\": \"min=1638.219, mean=1638.219, max=1638.219, sum=3276.439 (2)\", \"tab\": \"General information\", \"score\": \"1638.2194092827003\"}", - "High School World History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"high_school_world_history\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_high_school_world_history\"" - } - } - }, - { - "evaluation_name": "Human Sexuality", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Human Sexuality", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.557, - "details": { - "description": "min=0.557, mean=0.557, max=0.557, sum=1.115 (2)", - "tab": "Accuracy", - "Human Aging - Observed inference time (s)": "{\"description\": \"min=0.28, mean=0.28, max=0.28, sum=0.56 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.28007102974861725\"}", - "Human Sexuality - Observed inference time (s)": "{\"description\": \"min=0.335, mean=0.335, max=0.335, sum=0.671 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3354811176998925\"}", - "Human Aging - # eval": "{\"description\": \"min=223, mean=223, max=223, sum=446 (2)\", \"tab\": \"General information\", \"score\": \"223.0\"}", - "Human Aging - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Aging - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Aging - # prompt tokens": "{\"description\": \"min=361.26, mean=361.26, max=361.26, sum=722.52 (2)\", \"tab\": \"General information\", \"score\": \"361.26008968609864\"}", - "Human Aging - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Human Sexuality - # eval": "{\"description\": \"min=131, mean=131, max=131, sum=262 (2)\", \"tab\": \"General information\", \"score\": \"131.0\"}", - "Human Sexuality - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Sexuality - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # prompt tokens": "{\"description\": \"min=403.382, mean=403.382, max=403.382, sum=806.763 (2)\", \"tab\": \"General information\", \"score\": \"403.381679389313\"}", - "Human Sexuality - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"human_sexuality\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_human_sexuality\"" - } - } - }, - { - "evaluation_name": "International Law", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on International Law", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.628, - "details": { - "description": "min=0.628, mean=0.628, max=0.628, sum=1.256 (2)", - "tab": "Accuracy", - "International Law - Observed inference time (s)": "{\"description\": \"min=0.351, mean=0.351, max=0.351, sum=0.702 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3510365151176768\"}", - "International Law - # eval": "{\"description\": \"min=121, mean=121, max=121, sum=242 (2)\", \"tab\": \"General information\", \"score\": \"121.0\"}", - "International Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "International Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "International Law - # prompt tokens": "{\"description\": \"min=729.463, mean=729.463, max=729.463, sum=1458.926 (2)\", \"tab\": \"General information\", \"score\": \"729.4628099173553\"}", - "International Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"international_law\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_international_law\"" - } - } - }, - { - "evaluation_name": "Logical Fallacies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Logical Fallacies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.466, - "details": { - "description": "min=0.466, mean=0.466, max=0.466, sum=0.933 (2)", - "tab": "Accuracy", - "Logical Fallacies - Observed inference time (s)": "{\"description\": \"min=0.327, mean=0.327, max=0.327, sum=0.655 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3273066304212699\"}", - "Logical Fallacies - # eval": "{\"description\": \"min=163, mean=163, max=163, sum=326 (2)\", \"tab\": \"General information\", \"score\": \"163.0\"}", - "Logical Fallacies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Logical Fallacies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Logical Fallacies - # prompt tokens": "{\"description\": \"min=502.755, mean=502.755, max=502.755, sum=1005.509 (2)\", \"tab\": \"General information\", \"score\": \"502.7546012269939\"}", - "Logical Fallacies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"logical_fallacies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_logical_fallacies\"" - } - } - }, - { - "evaluation_name": "Machine Learning", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Machine Learning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.402, - "details": { - "description": "min=0.402, mean=0.402, max=0.402, sum=0.804 (2)", - "tab": "Accuracy", - "Machine Learning - Observed inference time (s)": "{\"description\": \"min=0.366, mean=0.366, max=0.366, sum=0.732 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.36619071449552265\"}", - "Machine Learning - # eval": "{\"description\": \"min=112, mean=112, max=112, sum=224 (2)\", \"tab\": \"General information\", \"score\": \"112.0\"}", - "Machine Learning - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Machine Learning - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Machine Learning - # prompt tokens": "{\"description\": \"min=730.402, mean=730.402, max=730.402, sum=1460.804 (2)\", \"tab\": \"General information\", \"score\": \"730.4017857142857\"}", - "Machine Learning - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"machine_learning\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_machine_learning\"" - } - } - }, - { - "evaluation_name": "Management", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Management", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.563, - "details": { - "description": "min=0.563, mean=0.563, max=0.563, sum=1.126 (2)", - "tab": "Accuracy", - "Management - Observed inference time (s)": "{\"description\": \"min=0.335, mean=0.335, max=0.335, sum=0.669 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.33452116632924495\"}", - "Management - # eval": "{\"description\": \"min=103, mean=103, max=103, sum=206 (2)\", \"tab\": \"General information\", \"score\": \"103.0\"}", - "Management - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Management - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Management - # prompt tokens": "{\"description\": \"min=315.777, mean=315.777, max=315.777, sum=631.553 (2)\", \"tab\": \"General information\", \"score\": \"315.77669902912623\"}", - "Management - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"management\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_management\"" - } - } - }, - { - "evaluation_name": "Marketing", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Marketing", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.697, - "details": { - "description": "min=0.697, mean=0.697, max=0.697, sum=1.393 (2)", - "tab": "Accuracy", - "Marketing - Observed inference time (s)": "{\"description\": \"min=0.331, mean=0.331, max=0.331, sum=0.662 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3312412653213892\"}", - "Marketing - # eval": "{\"description\": \"min=234, mean=234, max=234, sum=468 (2)\", \"tab\": \"General information\", \"score\": \"234.0\"}", - "Marketing - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Marketing - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Marketing - # prompt tokens": "{\"description\": \"min=472.628, mean=472.628, max=472.628, sum=945.256 (2)\", \"tab\": \"General information\", \"score\": \"472.62820512820514\"}", - "Marketing - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"marketing\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_marketing\"" - } - } - }, - { - "evaluation_name": "Medical Genetics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Medical Genetics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.53, - "details": { - "description": "min=0.53, mean=0.53, max=0.53, sum=1.06 (2)", - "tab": "Accuracy", - "Medical Genetics - Observed inference time (s)": "{\"description\": \"min=0.34, mean=0.34, max=0.34, sum=0.679 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3395656991004944\"}", - "Medical Genetics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Medical Genetics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Medical Genetics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Medical Genetics - # prompt tokens": "{\"description\": \"min=408.14, mean=408.14, max=408.14, sum=816.28 (2)\", \"tab\": \"General information\", \"score\": \"408.14\"}", - "Medical Genetics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"medical_genetics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_medical_genetics\"" - } - } - }, - { - "evaluation_name": "Miscellaneous", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Miscellaneous", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.632, - "details": { - "description": "min=0.632, mean=0.632, max=0.632, sum=1.264 (2)", - "tab": "Accuracy", - "Miscellaneous - Observed inference time (s)": "{\"description\": \"min=0.353, mean=0.353, max=0.353, sum=0.706 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3531375576862126\"}", - "Miscellaneous - # eval": "{\"description\": \"min=783, mean=783, max=783, sum=1566 (2)\", \"tab\": \"General information\", \"score\": \"783.0\"}", - "Miscellaneous - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Miscellaneous - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Miscellaneous - # prompt tokens": "{\"description\": \"min=345.913, mean=345.913, max=345.913, sum=691.826 (2)\", \"tab\": \"General information\", \"score\": \"345.9131545338442\"}", - "Miscellaneous - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"miscellaneous\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_miscellaneous\"" - } - } - }, - { - "evaluation_name": "Moral Scenarios", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Moral Scenarios", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.238, - "details": { - "description": "min=0.238, mean=0.238, max=0.238, sum=0.476 (2)", - "tab": "Accuracy", - "Moral Disputes - Observed inference time (s)": "{\"description\": \"min=0.326, mean=0.326, max=0.326, sum=0.653 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3263767213490657\"}", - "Moral Scenarios - Observed inference time (s)": "{\"description\": \"min=0.369, mean=0.369, max=0.369, sum=0.738 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3688804725028949\"}", - "Moral Disputes - # eval": "{\"description\": \"min=346, mean=346, max=346, sum=692 (2)\", \"tab\": \"General information\", \"score\": \"346.0\"}", - "Moral Disputes - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Disputes - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Disputes - # prompt tokens": "{\"description\": \"min=542.506, mean=542.506, max=542.506, sum=1085.012 (2)\", \"tab\": \"General information\", \"score\": \"542.5057803468208\"}", - "Moral Disputes - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Moral Scenarios - # eval": "{\"description\": \"min=895, mean=895, max=895, sum=1790 (2)\", \"tab\": \"General information\", \"score\": \"895.0\"}", - "Moral Scenarios - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Scenarios - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # prompt tokens": "{\"description\": \"min=756.479, mean=756.479, max=756.479, sum=1512.959 (2)\", \"tab\": \"General information\", \"score\": \"756.4793296089385\"}", - "Moral Scenarios - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"moral_scenarios\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_moral_scenarios\"" - } - } - }, - { - "evaluation_name": "Nutrition", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Nutrition", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.497, - "details": { - "description": "min=0.497, mean=0.497, max=0.497, sum=0.993 (2)", - "tab": "Accuracy", - "Nutrition - Observed inference time (s)": "{\"description\": \"min=0.342, mean=0.342, max=0.342, sum=0.684 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.34185195904152066\"}", - "Nutrition - # eval": "{\"description\": \"min=306, mean=306, max=306, sum=612 (2)\", \"tab\": \"General information\", \"score\": \"306.0\"}", - "Nutrition - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Nutrition - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Nutrition - # prompt tokens": "{\"description\": \"min=695.922, mean=695.922, max=695.922, sum=1391.843 (2)\", \"tab\": \"General information\", \"score\": \"695.9215686274509\"}", - "Nutrition - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"nutrition\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_nutrition\"" - } - } - }, - { - "evaluation_name": "Prehistory", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Prehistory", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.503, - "details": { - "description": "min=0.503, mean=0.503, max=0.503, sum=1.006 (2)", - "tab": "Accuracy", - "Prehistory - Observed inference time (s)": "{\"description\": \"min=0.333, mean=0.333, max=0.333, sum=0.665 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.33259875023806534\"}", - "Prehistory - # eval": "{\"description\": \"min=324, mean=324, max=324, sum=648 (2)\", \"tab\": \"General information\", \"score\": \"324.0\"}", - "Prehistory - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Prehistory - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Prehistory - # prompt tokens": "{\"description\": \"min=619.185, mean=619.185, max=619.185, sum=1238.37 (2)\", \"tab\": \"General information\", \"score\": \"619.1851851851852\"}", - "Prehistory - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"prehistory\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_prehistory\"" - } - } - }, - { - "evaluation_name": "Public Relations", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Public Relations", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.509, - "details": { - "description": "min=0.509, mean=0.509, max=0.509, sum=1.018 (2)", - "tab": "Accuracy", - "Public Relations - Observed inference time (s)": "{\"description\": \"min=0.333, mean=0.333, max=0.333, sum=0.665 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3326493003151634\"}", - "Public Relations - # eval": "{\"description\": \"min=110, mean=110, max=110, sum=220 (2)\", \"tab\": \"General information\", \"score\": \"110.0\"}", - "Public Relations - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Public Relations - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Public Relations - # prompt tokens": "{\"description\": \"min=474.827, mean=474.827, max=474.827, sum=949.655 (2)\", \"tab\": \"General information\", \"score\": \"474.8272727272727\"}", - "Public Relations - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"public_relations\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_public_relations\"" - } - } - }, - { - "evaluation_name": "Security Studies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Security Studies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.433, - "details": { - "description": "min=0.433, mean=0.433, max=0.433, sum=0.865 (2)", - "tab": "Accuracy", - "Security Studies - Observed inference time (s)": "{\"description\": \"min=0.477, mean=0.477, max=0.477, sum=0.955 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4774373015578912\"}", - "Security Studies - # eval": "{\"description\": \"min=245, mean=245, max=245, sum=490 (2)\", \"tab\": \"General information\", \"score\": \"245.0\"}", - "Security Studies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Security Studies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Security Studies - # prompt tokens": "{\"description\": \"min=1377.531, mean=1377.531, max=1377.531, sum=2755.061 (2)\", \"tab\": \"General information\", \"score\": \"1377.530612244898\"}", - "Security Studies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"security_studies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_security_studies\"" - } - } - }, - { - "evaluation_name": "Sociology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Sociology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.617, - "details": { - "description": "min=0.617, mean=0.617, max=0.617, sum=1.234 (2)", - "tab": "Accuracy", - "Sociology - Observed inference time (s)": "{\"description\": \"min=0.312, mean=0.312, max=0.312, sum=0.623 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.31150120170555307\"}", - "Sociology - # eval": "{\"description\": \"min=201, mean=201, max=201, sum=402 (2)\", \"tab\": \"General information\", \"score\": \"201.0\"}", - "Sociology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Sociology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Sociology - # prompt tokens": "{\"description\": \"min=508.478, mean=508.478, max=508.478, sum=1016.955 (2)\", \"tab\": \"General information\", \"score\": \"508.4776119402985\"}", - "Sociology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"sociology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_sociology\"" - } - } - }, - { - "evaluation_name": "Virology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Virology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.392, - "details": { - "description": "min=0.392, mean=0.392, max=0.392, sum=0.783 (2)", - "tab": "Accuracy", - "Virology - Observed inference time (s)": "{\"description\": \"min=0.33, mean=0.33, max=0.33, sum=0.66 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.32997589513479947\"}", - "Virology - # eval": "{\"description\": \"min=166, mean=166, max=166, sum=332 (2)\", \"tab\": \"General information\", \"score\": \"166.0\"}", - "Virology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Virology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Virology - # prompt tokens": "{\"description\": \"min=405.108, mean=405.108, max=405.108, sum=810.217 (2)\", \"tab\": \"General information\", \"score\": \"405.10843373493975\"}", - "Virology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"virology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_virology\"" - } - } - }, - { - "evaluation_name": "World Religions", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on World Religions", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.713, - "details": { - "description": "min=0.713, mean=0.713, max=0.713, sum=1.427 (2)", - "tab": "Accuracy", - "World Religions - Observed inference time (s)": "{\"description\": \"min=0.3, mean=0.3, max=0.3, sum=0.6 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2998225702876933\"}", - "World Religions - # eval": "{\"description\": \"min=171, mean=171, max=171, sum=342 (2)\", \"tab\": \"General information\", \"score\": \"171.0\"}", - "World Religions - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "World Religions - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "World Religions - # prompt tokens": "{\"description\": \"min=304.474, mean=304.474, max=304.474, sum=608.947 (2)\", \"tab\": \"General information\", \"score\": \"304.4736842105263\"}", - "World Religions - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"world_religions\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_world_religions\"" - } - } - }, - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.681, - "details": { - "description": "", - "tab": "Efficiency" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/meta_llama-3-1-8b-instruct-prompt.json b/data/models/meta_llama-3-1-8b-instruct-prompt.json deleted file mode 100644 index f071b6cc2bc36580bdb11d73cbe62f546edfc7f6..0000000000000000000000000000000000000000 --- a/data/models/meta_llama-3-1-8b-instruct-prompt.json +++ /dev/null @@ -1,905 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.1-8B-Instruct (Prompt)", - "id": "meta/llama-3-1-8b-instruct-prompt", - "developer": "Meta", - "additional_details": { - "raw_model_name": "Llama-3.1-8B-Instruct (Prompt)", - "organization": "Meta", - "license": "Meta Llama 3 Community", - "mode": "Prompt", - "model_link": "https://llama.meta.com/llama3" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/meta/llama-3-1-8b-instruct-prompt/1775236112.410609", - "retrieved_timestamp": "1775236112.410609", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 85.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 25.83 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 7.49 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 5.6 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 19.37 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 22.6 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 84.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 71.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 95.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 87.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 82.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 70.76 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 72.87 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 71.13 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 50.0 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 45.83 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 11.12 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 13.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 9.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 9.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 13.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 3.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 6.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 10.75 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 7.74 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 5.81 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 18.71 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 93.75 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 42.7 - } - }, - { - "evaluation_result_id": "bfcl.format_sensitivity.max_delta::max_delta", - "evaluation_name": "bfcl.format_sensitivity.max_delta", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.format_sensitivity.max_delta", - "metric_name": "Format sensitivity max delta", - "metric_kind": "difference", - "metric_unit": "percentage_points", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Format Sensitivity Max Delta" - } - }, - "score_details": { - "score": 74.5 - } - }, - { - "evaluation_result_id": "bfcl.format_sensitivity.stddev::stddev", - "evaluation_name": "bfcl.format_sensitivity.stddev", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.format_sensitivity.stddev", - "metric_name": "Format sensitivity standard deviation", - "metric_kind": "difference", - "metric_unit": "percentage_points", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Format Sensitivity Standard Deviation" - } - }, - "score_details": { - "score": 29.1 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/meta_llama-3-2-1b-instruct-fc.json b/data/models/meta_llama-3-2-1b-instruct-fc.json deleted file mode 100644 index c96e9840e51b7090c52327e47dbf07a236b469b3..0000000000000000000000000000000000000000 --- a/data/models/meta_llama-3-2-1b-instruct-fc.json +++ /dev/null @@ -1,851 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.2-1B-Instruct (FC)", - "id": "meta/llama-3-2-1b-instruct-fc", - "developer": "Meta", - "additional_details": { - "raw_model_name": "Llama-3.2-1B-Instruct (FC)", - "organization": "Meta", - "license": "Meta Llama 3 Community", - "mode": "FC", - "model_link": "https://llama.meta.com/llama3" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/meta/llama-3-2-1b-instruct-fc/1775236112.4222012", - "retrieved_timestamp": "1775236112.4222012", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 107.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 10.82 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 1.64 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 3.21 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 10.04 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 9.77 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 38.38 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 44.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 50.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 44.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 15.0 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 11.77 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 31.78 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 7.31 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 3.23 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 2.58 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 2.58 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 4.52 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 43.75 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 51.57 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/meta_llama-3-2-3b-instruct-fc.json b/data/models/meta_llama-3-2-3b-instruct-fc.json deleted file mode 100644 index e97dc1af2d22a876b340f4e4312ce9a86341ceae..0000000000000000000000000000000000000000 --- a/data/models/meta_llama-3-2-3b-instruct-fc.json +++ /dev/null @@ -1,851 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.2-3B-Instruct (FC)", - "id": "meta/llama-3-2-3b-instruct-fc", - "developer": "Meta", - "additional_details": { - "raw_model_name": "Llama-3.2-3B-Instruct (FC)", - "organization": "Meta", - "license": "Meta Llama 3 Community", - "mode": "FC", - "model_link": "https://llama.meta.com/llama3" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/meta/llama-3-2-3b-instruct-fc/1775236112.417592", - "retrieved_timestamp": "1775236112.417592", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 98.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 21.95 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 6.2 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 6.1 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 20.07 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 17.27 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 82.67 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 70.67 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 92.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 88.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 79.0 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 58.33 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 65.12 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 57.64 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 25.0 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 37.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 4.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 5.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 3.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 4.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 3.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 1.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 2.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 6.24 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 3.23 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 3.23 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 12.26 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 87.5 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 52.06 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/meta_llama-3-3-70b-instruct-fc.json b/data/models/meta_llama-3-3-70b-instruct-fc.json deleted file mode 100644 index 320aa4344a6ff357b54d2693dcc1934d91d236b8..0000000000000000000000000000000000000000 --- a/data/models/meta_llama-3-3-70b-instruct-fc.json +++ /dev/null @@ -1,851 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.3-70B-Instruct (FC)", - "id": "meta/llama-3-3-70b-instruct-fc", - "developer": "Meta", - "additional_details": { - "raw_model_name": "Llama-3.3-70B-Instruct (FC)", - "organization": "Meta", - "license": "Meta Llama 3 Community", - "mode": "FC", - "model_link": "https://llama.meta.com/llama3" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/meta/llama-3-3-70b-instruct-fc/1775236112.3981178", - "retrieved_timestamp": "1775236112.3981178", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 62.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 31.9 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 29.54 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 26.11 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 93.22 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 187.93 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 88.02 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 76.08 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 95.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 90.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 91.0 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 76.61 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 81.4 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 75.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 81.25 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 70.83 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 21.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 26.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 19.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 14.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 26.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 10.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 14.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 6.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 8.17 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 4.52 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 8.39 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 11.61 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 100.0 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 53.53 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/meta_llama-3-70b.json b/data/models/meta_llama-3-70b.json deleted file mode 100644 index 8d88e1f56b9e71f26314cd3013685202d90e485e..0000000000000000000000000000000000000000 --- a/data/models/meta_llama-3-70b.json +++ /dev/null @@ -1,1901 +0,0 @@ -{ - "model_info": { - "name": "Llama 3 70B", - "id": "meta/llama-3-70b", - "developer": "Meta", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_lite/meta_llama-3-70b/1774096306.427425", - "retrieved_timestamp": "1774096306.427425", - "source_metadata": { - "source_name": "helm_lite", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_lite", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_lite", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.793, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.3926217228464419\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.798, - "details": { - "description": "min=0.798, mean=0.798, max=0.798, sum=0.798 (1)", - "tab": "Accuracy", - "NarrativeQA - Observed inference time (s)": "{\"description\": \"min=1.795, mean=1.795, max=1.795, sum=1.795 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.7946508300136512\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=3460.268, mean=3460.268, max=3460.268, sum=3460.268 (1)\", \"tab\": \"General information\", \"score\": \"3460.2676056338028\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (closed-book)", - "source_data": { - "dataset_name": "NaturalQuestions (closed-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (closed-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.475, - "details": { - "description": "min=0.475, mean=0.475, max=0.475, sum=0.475 (1)", - "tab": "Accuracy", - "NaturalQuestions (open-book) - Observed inference time (s)": "{\"description\": \"min=1.212, mean=1.212, max=1.212, sum=1.212 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.211742308139801\"}", - "NaturalQuestions (closed-book) - Observed inference time (s)": "{\"description\": \"min=0.558, mean=0.558, max=0.558, sum=0.558 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.5584413967132569\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.965, mean=4.965, max=4.965, sum=4.965 (1)\", \"tab\": \"General information\", \"score\": \"4.965\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.007, mean=0.007, max=0.007, sum=0.007 (1)\", \"tab\": \"General information\", \"score\": \"0.007\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1658.348, mean=1658.348, max=1658.348, sum=1658.348 (1)\", \"tab\": \"General information\", \"score\": \"1658.348\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=112.12, mean=112.12, max=112.12, sum=112.12 (1)\", \"tab\": \"General information\", \"score\": \"112.12\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "mode": "\"closedbook\"" - } - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.934, - "details": { - "description": "min=0.934, mean=0.934, max=0.934, sum=0.934 (1)", - "tab": "Accuracy", - "OpenbookQA - Observed inference time (s)": "{\"description\": \"min=0.352, mean=0.352, max=0.352, sum=0.352 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.35184384298324584\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=242.776, mean=242.776, max=242.776, sum=242.776 (1)\", \"tab\": \"General information\", \"score\": \"242.776\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "dataset": "\"openbookqa\"", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.695, - "details": { - "description": "min=0.43, mean=0.695, max=0.94, sum=3.473 (5)", - "tab": "Accuracy", - "MMLU - Observed inference time (s)": "{\"description\": \"min=0.387, mean=0.404, max=0.432, sum=2.021 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.40422279727668087\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=366.43, mean=460.686, max=607.421, sum=2303.431 (5)\", \"tab\": \"General information\", \"score\": \"460.6862105263158\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"college_chemistry\", \"computer_security\", \"econometrics\", \"us_foreign_policy\"]", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MATH", - "source_data": { - "dataset_name": "MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Equivalent (CoT) on MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.663, - "details": { - "description": "min=0.433, mean=0.663, max=0.822, sum=4.641 (7)", - "tab": "Accuracy", - "MATH - Observed inference time (s)": "{\"description\": \"min=14.895, mean=15.819, max=17.569, sum=110.731 (7)\", \"tab\": \"Efficiency\", \"score\": \"15.818764438908431\"}", - "MATH - # eval": "{\"description\": \"min=30, mean=62.429, max=135, sum=437 (7)\", \"tab\": \"General information\", \"score\": \"62.42857142857143\"}", - "MATH - # train": "{\"description\": \"min=8, mean=8, max=8, sum=56 (7)\", \"tab\": \"General information\", \"score\": \"8.0\"}", - "MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MATH - # prompt tokens": "{\"description\": \"min=881.363, mean=1262.909, max=2197.577, sum=8840.364 (7)\", \"tab\": \"General information\", \"score\": \"1262.9092130545007\"}", - "MATH - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=7 (7)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"algebra\", \"counting_and_probability\", \"geometry\", \"intermediate_algebra\", \"number_theory\", \"prealgebra\", \"precalculus\"]", - "level": "\"1\"", - "use_official_examples": "\"False\"", - "use_chain_of_thought": "\"True\"" - } - } - }, - { - "evaluation_name": "GSM8K", - "source_data": { - "dataset_name": "GSM8K", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on GSM8K", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.805, - "details": { - "description": "min=0.805, mean=0.805, max=0.805, sum=0.805 (1)", - "tab": "Accuracy", - "GSM8K - Observed inference time (s)": "{\"description\": \"min=4.2, mean=4.2, max=4.2, sum=4.2 (1)\", \"tab\": \"Efficiency\", \"score\": \"4.199564570903778\"}", - "GSM8K - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "GSM8K - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "GSM8K - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GSM8K - # prompt tokens": "{\"description\": \"min=959.032, mean=959.032, max=959.032, sum=959.032 (1)\", \"tab\": \"General information\", \"score\": \"959.032\"}", - "GSM8K - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "LegalBench", - "source_data": { - "dataset_name": "LegalBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on LegalBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.733, - "details": { - "description": "min=0.466, mean=0.733, max=0.958, sum=3.665 (5)", - "tab": "Accuracy", - "LegalBench - Observed inference time (s)": "{\"description\": \"min=0.416, mean=0.87, max=2.556, sum=4.352 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.8703131128024035\"}", - "LegalBench - # eval": "{\"description\": \"min=95, mean=409.4, max=1000, sum=2047 (5)\", \"tab\": \"General information\", \"score\": \"409.4\"}", - "LegalBench - # train": "{\"description\": \"min=4, mean=4.798, max=5, sum=23.992 (5)\", \"tab\": \"General information\", \"score\": \"4.798367346938775\"}", - "LegalBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "LegalBench - # prompt tokens": "{\"description\": \"min=192.442, mean=1507.407, max=6287.633, sum=7537.033 (5)\", \"tab\": \"General information\", \"score\": \"1507.4065013565441\"}", - "LegalBench - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"abercrombie\", \"corporate_lobbying\", \"function_of_decision_section\", \"international_citizenship_questions\", \"proa\"]" - } - } - }, - { - "evaluation_name": "MedQA", - "source_data": { - "dataset_name": "MedQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MedQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.777, - "details": { - "description": "min=0.777, mean=0.777, max=0.777, sum=0.777 (1)", - "tab": "Accuracy", - "MedQA - Observed inference time (s)": "{\"description\": \"min=0.548, mean=0.548, max=0.548, sum=0.548 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.547684069419239\"}", - "MedQA - # eval": "{\"description\": \"min=503, mean=503, max=503, sum=503 (1)\", \"tab\": \"General information\", \"score\": \"503.0\"}", - "MedQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MedQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MedQA - # prompt tokens": "{\"description\": \"min=1018.274, mean=1018.274, max=1018.274, sum=1018.274 (1)\", \"tab\": \"General information\", \"score\": \"1018.2743538767396\"}", - "MedQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WMT 2014", - "source_data": { - "dataset_name": "WMT 2014", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "BLEU-4 on WMT 2014", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.225, - "details": { - "description": "min=0.183, mean=0.225, max=0.259, sum=1.123 (5)", - "tab": "Accuracy", - "WMT 2014 - Observed inference time (s)": "{\"description\": \"min=1.198, mean=1.239, max=1.282, sum=6.195 (5)\", \"tab\": \"Efficiency\", \"score\": \"1.239086973613365\"}", - "WMT 2014 - # eval": "{\"description\": \"min=503, mean=568.8, max=832, sum=2844 (5)\", \"tab\": \"General information\", \"score\": \"568.8\"}", - "WMT 2014 - # train": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "WMT 2014 - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WMT 2014 - # prompt tokens": "{\"description\": \"min=90.139, mean=109.868, max=130.33, sum=549.34 (5)\", \"tab\": \"General information\", \"score\": \"109.86804366111025\"}", - "WMT 2014 - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "language_pair": "[\"cs-en\", \"de-en\", \"fr-en\", \"hi-en\", \"ru-en\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_mmlu/meta_llama-3-70b/1774096312.00548", - "retrieved_timestamp": "1774096312.00548", - "source_metadata": { - "source_name": "helm_mmlu", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_mmlu", - "evaluation_results": [ - { - "evaluation_name": "MMLU All Subjects", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU All Subjects", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.793, - "details": { - "description": "min=0.43, mean=0.793, max=0.979, sum=90.444 (114)", - "tab": "Accuracy", - "MMLU All Subjects - Observed inference time (s)": "{\"description\": \"min=0.333, mean=0.462, max=1.184, sum=52.708 (114)\", \"tab\": \"Efficiency\", \"score\": \"0.46235507518987096\"}", - "MMLU All Subjects - # eval": "{\"description\": \"min=100, mean=246.351, max=1534, sum=28084 (114)\", \"tab\": \"General information\", \"score\": \"246.35087719298247\"}", - "MMLU All Subjects - # train": "{\"description\": \"min=5, mean=5, max=5, sum=570 (114)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU All Subjects - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU All Subjects - # prompt tokens": "{\"description\": \"min=267.52, mean=607.619, max=2790.885, sum=69268.61 (114)\", \"tab\": \"General information\", \"score\": \"607.6193817308517\"}", - "MMLU All Subjects - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=114 (114)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_name": "Abstract Algebra", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Abstract Algebra", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.43, - "details": { - "description": "min=0.43, mean=0.43, max=0.43, sum=0.86 (2)", - "tab": "Accuracy", - "Abstract Algebra - Observed inference time (s)": "{\"description\": \"min=0.387, mean=0.387, max=0.387, sum=0.774 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3868687057495117\"}", - "Abstract Algebra - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Abstract Algebra - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Abstract Algebra - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Abstract Algebra - # prompt tokens": "{\"description\": \"min=366.43, mean=366.43, max=366.43, sum=732.86 (2)\", \"tab\": \"General information\", \"score\": \"366.43\"}", - "Abstract Algebra - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"abstract_algebra\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_abstract_algebra\"" - } - } - }, - { - "evaluation_name": "Anatomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Anatomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.785, - "details": { - "description": "min=0.785, mean=0.785, max=0.785, sum=1.57 (2)", - "tab": "Accuracy", - "Anatomy - Observed inference time (s)": "{\"description\": \"min=0.391, mean=0.391, max=0.391, sum=0.782 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.39101445586593064\"}", - "Anatomy - # eval": "{\"description\": \"min=135, mean=135, max=135, sum=270 (2)\", \"tab\": \"General information\", \"score\": \"135.0\"}", - "Anatomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Anatomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Anatomy - # prompt tokens": "{\"description\": \"min=346.874, mean=346.874, max=346.874, sum=693.748 (2)\", \"tab\": \"General information\", \"score\": \"346.8740740740741\"}", - "Anatomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"anatomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_anatomy\"" - } - } - }, - { - "evaluation_name": "College Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on College Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.529, - "details": { - "description": "min=0.529, mean=0.529, max=0.529, sum=1.059 (2)", - "tab": "Accuracy", - "College Chemistry - Observed inference time (s)": "{\"description\": \"min=0.432, mean=0.432, max=0.432, sum=0.864 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4319474816322327\"}", - "College Biology - Observed inference time (s)": "{\"description\": \"min=0.394, mean=0.394, max=0.394, sum=0.788 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.39422312213314903\"}", - "College Computer Science - Observed inference time (s)": "{\"description\": \"min=0.48, mean=0.48, max=0.48, sum=0.959 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4797321176528931\"}", - "College Mathematics - Observed inference time (s)": "{\"description\": \"min=0.403, mean=0.403, max=0.403, sum=0.806 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4030305552482605\"}", - "College Medicine - Observed inference time (s)": "{\"description\": \"min=0.425, mean=0.425, max=0.425, sum=0.849 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4245531242017801\"}", - "College Physics - Observed inference time (s)": "{\"description\": \"min=0.42, mean=0.42, max=0.42, sum=0.84 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.41995686643263874\"}", - "College Chemistry - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Chemistry - # prompt tokens": "{\"description\": \"min=542.28, mean=542.28, max=542.28, sum=1084.56 (2)\", \"tab\": \"General information\", \"score\": \"542.28\"}", - "College Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Biology - # eval": "{\"description\": \"min=144, mean=144, max=144, sum=288 (2)\", \"tab\": \"General information\", \"score\": \"144.0\"}", - "College Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # prompt tokens": "{\"description\": \"min=466.875, mean=466.875, max=466.875, sum=933.75 (2)\", \"tab\": \"General information\", \"score\": \"466.875\"}", - "College Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # prompt tokens": "{\"description\": \"min=821.29, mean=821.29, max=821.29, sum=1642.58 (2)\", \"tab\": \"General information\", \"score\": \"821.29\"}", - "College Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Mathematics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # prompt tokens": "{\"description\": \"min=587.51, mean=587.51, max=587.51, sum=1175.02 (2)\", \"tab\": \"General information\", \"score\": \"587.51\"}", - "College Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Medicine - # eval": "{\"description\": \"min=173, mean=173, max=173, sum=346 (2)\", \"tab\": \"General information\", \"score\": \"173.0\"}", - "College Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # prompt tokens": "{\"description\": \"min=495.705, mean=495.705, max=495.705, sum=991.41 (2)\", \"tab\": \"General information\", \"score\": \"495.70520231213874\"}", - "College Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Physics - # eval": "{\"description\": \"min=102, mean=102, max=102, sum=204 (2)\", \"tab\": \"General information\", \"score\": \"102.0\"}", - "College Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # prompt tokens": "{\"description\": \"min=496.569, mean=496.569, max=496.569, sum=993.137 (2)\", \"tab\": \"General information\", \"score\": \"496.5686274509804\"}", - "College Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"college_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_college_physics\"" - } - } - }, - { - "evaluation_name": "Computer Security", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Computer Security", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.85, - "details": { - "description": "min=0.85, mean=0.85, max=0.85, sum=1.7 (2)", - "tab": "Accuracy", - "Computer Security - Observed inference time (s)": "{\"description\": \"min=0.392, mean=0.392, max=0.392, sum=0.783 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3916677093505859\"}", - "Computer Security - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Computer Security - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Computer Security - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Computer Security - # prompt tokens": "{\"description\": \"min=371.51, mean=371.51, max=371.51, sum=743.02 (2)\", \"tab\": \"General information\", \"score\": \"371.51\"}", - "Computer Security - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"computer_security\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_computer_security\"" - } - } - }, - { - "evaluation_name": "Econometrics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Econometrics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.693, - "details": { - "description": "min=0.693, mean=0.693, max=0.693, sum=1.386 (2)", - "tab": "Accuracy", - "Econometrics - Observed inference time (s)": "{\"description\": \"min=0.408, mean=0.408, max=0.408, sum=0.816 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4078888934955262\"}", - "Econometrics - # eval": "{\"description\": \"min=114, mean=114, max=114, sum=228 (2)\", \"tab\": \"General information\", \"score\": \"114.0\"}", - "Econometrics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Econometrics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Econometrics - # prompt tokens": "{\"description\": \"min=607.421, mean=607.421, max=607.421, sum=1214.842 (2)\", \"tab\": \"General information\", \"score\": \"607.421052631579\"}", - "Econometrics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"econometrics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_econometrics\"" - } - } - }, - { - "evaluation_name": "Global Facts", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Global Facts", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.49, - "details": { - "description": "min=0.49, mean=0.49, max=0.49, sum=0.98 (2)", - "tab": "Accuracy", - "Global Facts - Observed inference time (s)": "{\"description\": \"min=0.385, mean=0.385, max=0.385, sum=0.77 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3847800350189209\"}", - "Global Facts - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Global Facts - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Global Facts - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Global Facts - # prompt tokens": "{\"description\": \"min=392.71, mean=392.71, max=392.71, sum=785.42 (2)\", \"tab\": \"General information\", \"score\": \"392.71\"}", - "Global Facts - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"global_facts\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_global_facts\"" - } - } - }, - { - "evaluation_name": "Jurisprudence", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Jurisprudence", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.861, - "details": { - "description": "min=0.861, mean=0.861, max=0.861, sum=1.722 (2)", - "tab": "Accuracy", - "Jurisprudence - Observed inference time (s)": "{\"description\": \"min=0.368, mean=0.368, max=0.368, sum=0.736 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.36775174847355596\"}", - "Jurisprudence - # eval": "{\"description\": \"min=108, mean=108, max=108, sum=216 (2)\", \"tab\": \"General information\", \"score\": \"108.0\"}", - "Jurisprudence - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Jurisprudence - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Jurisprudence - # prompt tokens": "{\"description\": \"min=387.63, mean=387.63, max=387.63, sum=775.259 (2)\", \"tab\": \"General information\", \"score\": \"387.6296296296296\"}", - "Jurisprudence - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"jurisprudence\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_jurisprudence\"" - } - } - }, - { - "evaluation_name": "Philosophy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Philosophy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.865, - "details": { - "description": "min=0.865, mean=0.865, max=0.865, sum=1.73 (2)", - "tab": "Accuracy", - "Philosophy - Observed inference time (s)": "{\"description\": \"min=0.357, mean=0.357, max=0.357, sum=0.713 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.35669880894602685\"}", - "Philosophy - # eval": "{\"description\": \"min=311, mean=311, max=311, sum=622 (2)\", \"tab\": \"General information\", \"score\": \"311.0\"}", - "Philosophy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Philosophy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Philosophy - # prompt tokens": "{\"description\": \"min=322.084, mean=322.084, max=322.084, sum=644.167 (2)\", \"tab\": \"General information\", \"score\": \"322.08360128617363\"}", - "Philosophy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"philosophy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_philosophy\"" - } - } - }, - { - "evaluation_name": "Professional Psychology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Professional Psychology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.871, - "details": { - "description": "min=0.871, mean=0.871, max=0.871, sum=1.742 (2)", - "tab": "Accuracy", - "Professional Medicine - Observed inference time (s)": "{\"description\": \"min=0.523, mean=0.523, max=0.523, sum=1.046 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5229001255596385\"}", - "Professional Accounting - Observed inference time (s)": "{\"description\": \"min=0.408, mean=0.408, max=0.408, sum=0.816 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4082087980094531\"}", - "Professional Law - Observed inference time (s)": "{\"description\": \"min=0.738, mean=0.738, max=0.738, sum=1.477 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7383932933658167\"}", - "Professional Psychology - Observed inference time (s)": "{\"description\": \"min=0.376, mean=0.376, max=0.376, sum=0.752 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3758435642797183\"}", - "Professional Medicine - # eval": "{\"description\": \"min=272, mean=272, max=272, sum=544 (2)\", \"tab\": \"General information\", \"score\": \"272.0\"}", - "Professional Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Medicine - # prompt tokens": "{\"description\": \"min=1087.489, mean=1087.489, max=1087.489, sum=2174.978 (2)\", \"tab\": \"General information\", \"score\": \"1087.4889705882354\"}", - "Professional Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Accounting - # eval": "{\"description\": \"min=282, mean=282, max=282, sum=564 (2)\", \"tab\": \"General information\", \"score\": \"282.0\"}", - "Professional Accounting - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Accounting - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # prompt tokens": "{\"description\": \"min=651.585, mean=651.585, max=651.585, sum=1303.17 (2)\", \"tab\": \"General information\", \"score\": \"651.5851063829788\"}", - "Professional Accounting - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Law - # eval": "{\"description\": \"min=1534, mean=1534, max=1534, sum=3068 (2)\", \"tab\": \"General information\", \"score\": \"1534.0\"}", - "Professional Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # prompt tokens": "{\"description\": \"min=1630.601, mean=1630.601, max=1630.601, sum=3261.202 (2)\", \"tab\": \"General information\", \"score\": \"1630.6010430247718\"}", - "Professional Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Psychology - # eval": "{\"description\": \"min=612, mean=612, max=612, sum=1224 (2)\", \"tab\": \"General information\", \"score\": \"612.0\"}", - "Professional Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # prompt tokens": "{\"description\": \"min=568.098, mean=568.098, max=568.098, sum=1136.196 (2)\", \"tab\": \"General information\", \"score\": \"568.0980392156863\"}", - "Professional Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"professional_psychology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_professional_psychology\"" - } - } - }, - { - "evaluation_name": "Us Foreign Policy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Us Foreign Policy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.94, - "details": { - "description": "min=0.94, mean=0.94, max=0.94, sum=1.88 (2)", - "tab": "Accuracy", - "Us Foreign Policy - Observed inference time (s)": "{\"description\": \"min=0.403, mean=0.403, max=0.403, sum=0.805 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4027411961555481\"}", - "Us Foreign Policy - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Us Foreign Policy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Us Foreign Policy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Us Foreign Policy - # prompt tokens": "{\"description\": \"min=415.79, mean=415.79, max=415.79, sum=831.58 (2)\", \"tab\": \"General information\", \"score\": \"415.79\"}", - "Us Foreign Policy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"us_foreign_policy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_us_foreign_policy\"" - } - } - }, - { - "evaluation_name": "Astronomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Astronomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.921, - "details": { - "description": "min=0.921, mean=0.921, max=0.921, sum=1.842 (2)", - "tab": "Accuracy", - "Astronomy - Observed inference time (s)": "{\"description\": \"min=0.407, mean=0.407, max=0.407, sum=0.814 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4070533733618887\"}", - "Astronomy - # eval": "{\"description\": \"min=152, mean=152, max=152, sum=304 (2)\", \"tab\": \"General information\", \"score\": \"152.0\"}", - "Astronomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Astronomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Astronomy - # prompt tokens": "{\"description\": \"min=572.684, mean=572.684, max=572.684, sum=1145.368 (2)\", \"tab\": \"General information\", \"score\": \"572.6842105263158\"}", - "Astronomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"astronomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_astronomy\"" - } - } - }, - { - "evaluation_name": "Business Ethics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Business Ethics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.83, - "details": { - "description": "min=0.83, mean=0.83, max=0.83, sum=1.66 (2)", - "tab": "Accuracy", - "Business Ethics - Observed inference time (s)": "{\"description\": \"min=0.393, mean=0.393, max=0.393, sum=0.786 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3931219887733459\"}", - "Business Ethics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Business Ethics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Business Ethics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Business Ethics - # prompt tokens": "{\"description\": \"min=562.52, mean=562.52, max=562.52, sum=1125.04 (2)\", \"tab\": \"General information\", \"score\": \"562.52\"}", - "Business Ethics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"business_ethics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_business_ethics\"" - } - } - }, - { - "evaluation_name": "Clinical Knowledge", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Clinical Knowledge", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.845, - "details": { - "description": "min=0.845, mean=0.845, max=0.845, sum=1.691 (2)", - "tab": "Accuracy", - "Clinical Knowledge - Observed inference time (s)": "{\"description\": \"min=0.416, mean=0.416, max=0.416, sum=0.831 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.41558496907072245\"}", - "Clinical Knowledge - # eval": "{\"description\": \"min=265, mean=265, max=265, sum=530 (2)\", \"tab\": \"General information\", \"score\": \"265.0\"}", - "Clinical Knowledge - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Clinical Knowledge - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Clinical Knowledge - # prompt tokens": "{\"description\": \"min=390.928, mean=390.928, max=390.928, sum=781.857 (2)\", \"tab\": \"General information\", \"score\": \"390.92830188679244\"}", - "Clinical Knowledge - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"clinical_knowledge\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_clinical_knowledge\"" - } - } - }, - { - "evaluation_name": "Conceptual Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Conceptual Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.838, - "details": { - "description": "min=0.838, mean=0.838, max=0.838, sum=1.677 (2)", - "tab": "Accuracy", - "Conceptual Physics - Observed inference time (s)": "{\"description\": \"min=0.517, mean=0.517, max=0.517, sum=1.034 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5170877294337496\"}", - "Conceptual Physics - # eval": "{\"description\": \"min=235, mean=235, max=235, sum=470 (2)\", \"tab\": \"General information\", \"score\": \"235.0\"}", - "Conceptual Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Conceptual Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Conceptual Physics - # prompt tokens": "{\"description\": \"min=297.834, mean=297.834, max=297.834, sum=595.668 (2)\", \"tab\": \"General information\", \"score\": \"297.83404255319147\"}", - "Conceptual Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"conceptual_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_conceptual_physics\"" - } - } - }, - { - "evaluation_name": "Electrical Engineering", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Electrical Engineering", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.766, - "details": { - "description": "min=0.766, mean=0.766, max=0.766, sum=1.531 (2)", - "tab": "Accuracy", - "Electrical Engineering - Observed inference time (s)": "{\"description\": \"min=0.398, mean=0.398, max=0.398, sum=0.796 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.39815263419315733\"}", - "Electrical Engineering - # eval": "{\"description\": \"min=145, mean=145, max=145, sum=290 (2)\", \"tab\": \"General information\", \"score\": \"145.0\"}", - "Electrical Engineering - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Electrical Engineering - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Electrical Engineering - # prompt tokens": "{\"description\": \"min=428.607, mean=428.607, max=428.607, sum=857.214 (2)\", \"tab\": \"General information\", \"score\": \"428.60689655172416\"}", - "Electrical Engineering - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"electrical_engineering\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_electrical_engineering\"" - } - } - }, - { - "evaluation_name": "Elementary Mathematics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Elementary Mathematics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.632, - "details": { - "description": "min=0.632, mean=0.632, max=0.632, sum=1.265 (2)", - "tab": "Accuracy", - "Elementary Mathematics - Observed inference time (s)": "{\"description\": \"min=0.478, mean=0.478, max=0.478, sum=0.957 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.47845223719480806\"}", - "Elementary Mathematics - # eval": "{\"description\": \"min=378, mean=378, max=378, sum=756 (2)\", \"tab\": \"General information\", \"score\": \"378.0\"}", - "Elementary Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Elementary Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Elementary Mathematics - # prompt tokens": "{\"description\": \"min=524.854, mean=524.854, max=524.854, sum=1049.709 (2)\", \"tab\": \"General information\", \"score\": \"524.8544973544973\"}", - "Elementary Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"elementary_mathematics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_elementary_mathematics\"" - } - } - }, - { - "evaluation_name": "Formal Logic", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Formal Logic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.651, - "details": { - "description": "min=0.651, mean=0.651, max=0.651, sum=1.302 (2)", - "tab": "Accuracy", - "Formal Logic - Observed inference time (s)": "{\"description\": \"min=0.436, mean=0.436, max=0.436, sum=0.872 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4359313628030202\"}", - "Formal Logic - # eval": "{\"description\": \"min=126, mean=126, max=126, sum=252 (2)\", \"tab\": \"General information\", \"score\": \"126.0\"}", - "Formal Logic - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Formal Logic - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Formal Logic - # prompt tokens": "{\"description\": \"min=594.778, mean=594.778, max=594.778, sum=1189.556 (2)\", \"tab\": \"General information\", \"score\": \"594.7777777777778\"}", - "Formal Logic - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"formal_logic\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_formal_logic\"" - } - } - }, - { - "evaluation_name": "High School World History", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on High School World History", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.941, - "details": { - "description": "min=0.941, mean=0.941, max=0.941, sum=1.882 (2)", - "tab": "Accuracy", - "High School Biology - Observed inference time (s)": "{\"description\": \"min=0.512, mean=0.512, max=0.512, sum=1.023 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5115567738010037\"}", - "High School Chemistry - Observed inference time (s)": "{\"description\": \"min=0.545, mean=0.545, max=0.545, sum=1.089 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5445456727972171\"}", - "High School Computer Science - Observed inference time (s)": "{\"description\": \"min=0.501, mean=0.501, max=0.501, sum=1.002 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5008813333511353\"}", - "High School European History - Observed inference time (s)": "{\"description\": \"min=1.184, mean=1.184, max=1.184, sum=2.367 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.1835060582016455\"}", - "High School Geography - Observed inference time (s)": "{\"description\": \"min=0.372, mean=0.372, max=0.372, sum=0.744 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3721387037123092\"}", - "High School Government And Politics - Observed inference time (s)": "{\"description\": \"min=0.567, mean=0.567, max=0.567, sum=1.134 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5668655022438326\"}", - "High School Macroeconomics - Observed inference time (s)": "{\"description\": \"min=0.582, mean=0.582, max=0.582, sum=1.164 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5819246842310979\"}", - "High School Mathematics - Observed inference time (s)": "{\"description\": \"min=0.41, mean=0.41, max=0.41, sum=0.821 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.410357196242721\"}", - "High School Microeconomics - Observed inference time (s)": "{\"description\": \"min=0.379, mean=0.379, max=0.379, sum=0.759 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3792707469283032\"}", - "High School Physics - Observed inference time (s)": "{\"description\": \"min=0.393, mean=0.393, max=0.393, sum=0.786 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.39323860288455786\"}", - "High School Psychology - Observed inference time (s)": "{\"description\": \"min=0.395, mean=0.395, max=0.395, sum=0.789 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3946729870017515\"}", - "High School Statistics - Observed inference time (s)": "{\"description\": \"min=0.516, mean=0.516, max=0.516, sum=1.032 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5162484921790935\"}", - "High School US History - Observed inference time (s)": "{\"description\": \"min=0.956, mean=0.956, max=0.956, sum=1.911 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9556132928997862\"}", - "High School World History - Observed inference time (s)": "{\"description\": \"min=0.583, mean=0.583, max=0.583, sum=1.165 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5826822735589264\"}", - "High School Biology - # eval": "{\"description\": \"min=310, mean=310, max=310, sum=620 (2)\", \"tab\": \"General information\", \"score\": \"310.0\"}", - "High School Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Biology - # prompt tokens": "{\"description\": \"min=506.671, mean=506.671, max=506.671, sum=1013.342 (2)\", \"tab\": \"General information\", \"score\": \"506.6709677419355\"}", - "High School Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Chemistry - # eval": "{\"description\": \"min=203, mean=203, max=203, sum=406 (2)\", \"tab\": \"General information\", \"score\": \"203.0\"}", - "High School Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # prompt tokens": "{\"description\": \"min=489.704, mean=489.704, max=489.704, sum=979.409 (2)\", \"tab\": \"General information\", \"score\": \"489.70443349753697\"}", - "High School Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "High School Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # prompt tokens": "{\"description\": \"min=860.78, mean=860.78, max=860.78, sum=1721.56 (2)\", \"tab\": \"General information\", \"score\": \"860.78\"}", - "High School Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School European History - # eval": "{\"description\": \"min=165, mean=165, max=165, sum=330 (2)\", \"tab\": \"General information\", \"score\": \"165.0\"}", - "High School European History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School European History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # prompt tokens": "{\"description\": \"min=2790.885, mean=2790.885, max=2790.885, sum=5581.77 (2)\", \"tab\": \"General information\", \"score\": \"2790.8848484848486\"}", - "High School European History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Geography - # eval": "{\"description\": \"min=198, mean=198, max=198, sum=396 (2)\", \"tab\": \"General information\", \"score\": \"198.0\"}", - "High School Geography - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Geography - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # prompt tokens": "{\"description\": \"min=365.035, mean=365.035, max=365.035, sum=730.071 (2)\", \"tab\": \"General information\", \"score\": \"365.0353535353535\"}", - "High School Geography - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Government And Politics - # eval": "{\"description\": \"min=193, mean=193, max=193, sum=386 (2)\", \"tab\": \"General information\", \"score\": \"193.0\"}", - "High School Government And Politics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Government And Politics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # prompt tokens": "{\"description\": \"min=458.824, mean=458.824, max=458.824, sum=917.648 (2)\", \"tab\": \"General information\", \"score\": \"458.8238341968912\"}", - "High School Government And Politics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Macroeconomics - # eval": "{\"description\": \"min=390, mean=390, max=390, sum=780 (2)\", \"tab\": \"General information\", \"score\": \"390.0\"}", - "High School Macroeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Macroeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # prompt tokens": "{\"description\": \"min=363.908, mean=363.908, max=363.908, sum=727.815 (2)\", \"tab\": \"General information\", \"score\": \"363.9076923076923\"}", - "High School Macroeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Mathematics - # eval": "{\"description\": \"min=270, mean=270, max=270, sum=540 (2)\", \"tab\": \"General information\", \"score\": \"270.0\"}", - "High School Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # prompt tokens": "{\"description\": \"min=525.356, mean=525.356, max=525.356, sum=1050.711 (2)\", \"tab\": \"General information\", \"score\": \"525.3555555555556\"}", - "High School Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Microeconomics - # eval": "{\"description\": \"min=238, mean=238, max=238, sum=476 (2)\", \"tab\": \"General information\", \"score\": \"238.0\"}", - "High School Microeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Microeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # prompt tokens": "{\"description\": \"min=392.013, mean=392.013, max=392.013, sum=784.025 (2)\", \"tab\": \"General information\", \"score\": \"392.0126050420168\"}", - "High School Microeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Physics - # eval": "{\"description\": \"min=151, mean=151, max=151, sum=302 (2)\", \"tab\": \"General information\", \"score\": \"151.0\"}", - "High School Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # prompt tokens": "{\"description\": \"min=553.457, mean=553.457, max=553.457, sum=1106.914 (2)\", \"tab\": \"General information\", \"score\": \"553.4569536423841\"}", - "High School Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Psychology - # eval": "{\"description\": \"min=545, mean=545, max=545, sum=1090 (2)\", \"tab\": \"General information\", \"score\": \"545.0\"}", - "High School Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # prompt tokens": "{\"description\": \"min=488.242, mean=488.242, max=488.242, sum=976.484 (2)\", \"tab\": \"General information\", \"score\": \"488.2422018348624\"}", - "High School Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Statistics - # eval": "{\"description\": \"min=216, mean=216, max=216, sum=432 (2)\", \"tab\": \"General information\", \"score\": \"216.0\"}", - "High School Statistics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Statistics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # prompt tokens": "{\"description\": \"min=788.639, mean=788.639, max=788.639, sum=1577.278 (2)\", \"tab\": \"General information\", \"score\": \"788.6388888888889\"}", - "High School Statistics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School US History - # eval": "{\"description\": \"min=204, mean=204, max=204, sum=408 (2)\", \"tab\": \"General information\", \"score\": \"204.0\"}", - "High School US History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School US History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # prompt tokens": "{\"description\": \"min=2210.809, mean=2210.809, max=2210.809, sum=4421.618 (2)\", \"tab\": \"General information\", \"score\": \"2210.8088235294117\"}", - "High School US History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School World History - # eval": "{\"description\": \"min=237, mean=237, max=237, sum=474 (2)\", \"tab\": \"General information\", \"score\": \"237.0\"}", - "High School World History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School World History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # prompt tokens": "{\"description\": \"min=1421.173, mean=1421.173, max=1421.173, sum=2842.346 (2)\", \"tab\": \"General information\", \"score\": \"1421.1729957805908\"}", - "High School World History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"high_school_world_history\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_high_school_world_history\"" - } - } - }, - { - "evaluation_name": "Human Sexuality", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Human Sexuality", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.878, - "details": { - "description": "min=0.878, mean=0.878, max=0.878, sum=1.756 (2)", - "tab": "Accuracy", - "Human Aging - Observed inference time (s)": "{\"description\": \"min=0.346, mean=0.346, max=0.346, sum=0.693 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.346398046733018\"}", - "Human Sexuality - Observed inference time (s)": "{\"description\": \"min=0.351, mean=0.351, max=0.351, sum=0.702 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3509944832051983\"}", - "Human Aging - # eval": "{\"description\": \"min=223, mean=223, max=223, sum=446 (2)\", \"tab\": \"General information\", \"score\": \"223.0\"}", - "Human Aging - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Aging - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Aging - # prompt tokens": "{\"description\": \"min=312.888, mean=312.888, max=312.888, sum=625.776 (2)\", \"tab\": \"General information\", \"score\": \"312.88789237668163\"}", - "Human Aging - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Human Sexuality - # eval": "{\"description\": \"min=131, mean=131, max=131, sum=262 (2)\", \"tab\": \"General information\", \"score\": \"131.0\"}", - "Human Sexuality - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Sexuality - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # prompt tokens": "{\"description\": \"min=334.168, mean=334.168, max=334.168, sum=668.336 (2)\", \"tab\": \"General information\", \"score\": \"334.1679389312977\"}", - "Human Sexuality - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"human_sexuality\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_human_sexuality\"" - } - } - }, - { - "evaluation_name": "International Law", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on International Law", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.901, - "details": { - "description": "min=0.901, mean=0.901, max=0.901, sum=1.802 (2)", - "tab": "Accuracy", - "International Law - Observed inference time (s)": "{\"description\": \"min=0.397, mean=0.397, max=0.397, sum=0.794 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.39698751701796353\"}", - "International Law - # eval": "{\"description\": \"min=121, mean=121, max=121, sum=242 (2)\", \"tab\": \"General information\", \"score\": \"121.0\"}", - "International Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "International Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "International Law - # prompt tokens": "{\"description\": \"min=632.818, mean=632.818, max=632.818, sum=1265.636 (2)\", \"tab\": \"General information\", \"score\": \"632.8181818181819\"}", - "International Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"international_law\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_international_law\"" - } - } - }, - { - "evaluation_name": "Logical Fallacies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Logical Fallacies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.865, - "details": { - "description": "min=0.865, mean=0.865, max=0.865, sum=1.73 (2)", - "tab": "Accuracy", - "Logical Fallacies - Observed inference time (s)": "{\"description\": \"min=0.37, mean=0.37, max=0.37, sum=0.74 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.36976343722431204\"}", - "Logical Fallacies - # eval": "{\"description\": \"min=163, mean=163, max=163, sum=326 (2)\", \"tab\": \"General information\", \"score\": \"163.0\"}", - "Logical Fallacies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Logical Fallacies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Logical Fallacies - # prompt tokens": "{\"description\": \"min=442.564, mean=442.564, max=442.564, sum=885.129 (2)\", \"tab\": \"General information\", \"score\": \"442.5644171779141\"}", - "Logical Fallacies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"logical_fallacies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_logical_fallacies\"" - } - } - }, - { - "evaluation_name": "Machine Learning", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Machine Learning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.714, - "details": { - "description": "min=0.714, mean=0.714, max=0.714, sum=1.429 (2)", - "tab": "Accuracy", - "Machine Learning - Observed inference time (s)": "{\"description\": \"min=0.7, mean=0.7, max=0.7, sum=1.401 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7002999080078942\"}", - "Machine Learning - # eval": "{\"description\": \"min=112, mean=112, max=112, sum=224 (2)\", \"tab\": \"General information\", \"score\": \"112.0\"}", - "Machine Learning - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Machine Learning - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Machine Learning - # prompt tokens": "{\"description\": \"min=661.054, mean=661.054, max=661.054, sum=1322.107 (2)\", \"tab\": \"General information\", \"score\": \"661.0535714285714\"}", - "Machine Learning - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"machine_learning\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_machine_learning\"" - } - } - }, - { - "evaluation_name": "Management", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Management", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.913, - "details": { - "description": "min=0.913, mean=0.913, max=0.913, sum=1.825 (2)", - "tab": "Accuracy", - "Management - Observed inference time (s)": "{\"description\": \"min=0.411, mean=0.411, max=0.411, sum=0.823 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.41139175822433915\"}", - "Management - # eval": "{\"description\": \"min=103, mean=103, max=103, sum=206 (2)\", \"tab\": \"General information\", \"score\": \"103.0\"}", - "Management - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Management - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Management - # prompt tokens": "{\"description\": \"min=276.786, mean=276.786, max=276.786, sum=553.573 (2)\", \"tab\": \"General information\", \"score\": \"276.7864077669903\"}", - "Management - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"management\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_management\"" - } - } - }, - { - "evaluation_name": "Marketing", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Marketing", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.94, - "details": { - "description": "min=0.94, mean=0.94, max=0.94, sum=1.88 (2)", - "tab": "Accuracy", - "Marketing - Observed inference time (s)": "{\"description\": \"min=0.36, mean=0.36, max=0.36, sum=0.72 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.35977526811453014\"}", - "Marketing - # eval": "{\"description\": \"min=234, mean=234, max=234, sum=468 (2)\", \"tab\": \"General information\", \"score\": \"234.0\"}", - "Marketing - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Marketing - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Marketing - # prompt tokens": "{\"description\": \"min=397.218, mean=397.218, max=397.218, sum=794.436 (2)\", \"tab\": \"General information\", \"score\": \"397.21794871794873\"}", - "Marketing - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"marketing\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_marketing\"" - } - } - }, - { - "evaluation_name": "Medical Genetics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Medical Genetics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.89, - "details": { - "description": "min=0.89, mean=0.89, max=0.89, sum=1.78 (2)", - "tab": "Accuracy", - "Medical Genetics - Observed inference time (s)": "{\"description\": \"min=0.398, mean=0.398, max=0.398, sum=0.796 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.398222451210022\"}", - "Medical Genetics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Medical Genetics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Medical Genetics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Medical Genetics - # prompt tokens": "{\"description\": \"min=333.99, mean=333.99, max=333.99, sum=667.98 (2)\", \"tab\": \"General information\", \"score\": \"333.99\"}", - "Medical Genetics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"medical_genetics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_medical_genetics\"" - } - } - }, - { - "evaluation_name": "Miscellaneous", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Miscellaneous", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.917, - "details": { - "description": "min=0.917, mean=0.917, max=0.917, sum=1.834 (2)", - "tab": "Accuracy", - "Miscellaneous - Observed inference time (s)": "{\"description\": \"min=0.512, mean=0.512, max=0.512, sum=1.023 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5115468505089615\"}", - "Miscellaneous - # eval": "{\"description\": \"min=783, mean=783, max=783, sum=1566 (2)\", \"tab\": \"General information\", \"score\": \"783.0\"}", - "Miscellaneous - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Miscellaneous - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Miscellaneous - # prompt tokens": "{\"description\": \"min=292.911, mean=292.911, max=292.911, sum=585.821 (2)\", \"tab\": \"General information\", \"score\": \"292.9106002554278\"}", - "Miscellaneous - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"miscellaneous\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_miscellaneous\"" - } - } - }, - { - "evaluation_name": "Moral Scenarios", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Moral Scenarios", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.598, - "details": { - "description": "min=0.598, mean=0.598, max=0.598, sum=1.196 (2)", - "tab": "Accuracy", - "Moral Disputes - Observed inference time (s)": "{\"description\": \"min=0.396, mean=0.396, max=0.396, sum=0.792 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3959053982199961\"}", - "Moral Scenarios - Observed inference time (s)": "{\"description\": \"min=0.462, mean=0.462, max=0.462, sum=0.924 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.46180219543712764\"}", - "Moral Disputes - # eval": "{\"description\": \"min=346, mean=346, max=346, sum=692 (2)\", \"tab\": \"General information\", \"score\": \"346.0\"}", - "Moral Disputes - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Disputes - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Disputes - # prompt tokens": "{\"description\": \"min=469.113, mean=469.113, max=469.113, sum=938.225 (2)\", \"tab\": \"General information\", \"score\": \"469.1127167630058\"}", - "Moral Disputes - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Moral Scenarios - # eval": "{\"description\": \"min=895, mean=895, max=895, sum=1790 (2)\", \"tab\": \"General information\", \"score\": \"895.0\"}", - "Moral Scenarios - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Scenarios - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # prompt tokens": "{\"description\": \"min=649.455, mean=649.455, max=649.455, sum=1298.909 (2)\", \"tab\": \"General information\", \"score\": \"649.454748603352\"}", - "Moral Scenarios - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"moral_scenarios\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_moral_scenarios\"" - } - } - }, - { - "evaluation_name": "Nutrition", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Nutrition", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.876, - "details": { - "description": "min=0.876, mean=0.876, max=0.876, sum=1.752 (2)", - "tab": "Accuracy", - "Nutrition - Observed inference time (s)": "{\"description\": \"min=0.396, mean=0.396, max=0.396, sum=0.793 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3964238252515107\"}", - "Nutrition - # eval": "{\"description\": \"min=306, mean=306, max=306, sum=612 (2)\", \"tab\": \"General information\", \"score\": \"306.0\"}", - "Nutrition - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Nutrition - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Nutrition - # prompt tokens": "{\"description\": \"min=579.814, mean=579.814, max=579.814, sum=1159.627 (2)\", \"tab\": \"General information\", \"score\": \"579.8137254901961\"}", - "Nutrition - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"nutrition\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_nutrition\"" - } - } - }, - { - "evaluation_name": "Prehistory", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Prehistory", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.91, - "details": { - "description": "min=0.91, mean=0.91, max=0.91, sum=1.821 (2)", - "tab": "Accuracy", - "Prehistory - Observed inference time (s)": "{\"description\": \"min=0.509, mean=0.509, max=0.509, sum=1.017 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.50853196338371\"}", - "Prehistory - # eval": "{\"description\": \"min=324, mean=324, max=324, sum=648 (2)\", \"tab\": \"General information\", \"score\": \"324.0\"}", - "Prehistory - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Prehistory - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Prehistory - # prompt tokens": "{\"description\": \"min=507.528, mean=507.528, max=507.528, sum=1015.056 (2)\", \"tab\": \"General information\", \"score\": \"507.52777777777777\"}", - "Prehistory - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"prehistory\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_prehistory\"" - } - } - }, - { - "evaluation_name": "Public Relations", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Public Relations", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.727, - "details": { - "description": "min=0.727, mean=0.727, max=0.727, sum=1.455 (2)", - "tab": "Accuracy", - "Public Relations - Observed inference time (s)": "{\"description\": \"min=0.402, mean=0.402, max=0.402, sum=0.804 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4018417878584428\"}", - "Public Relations - # eval": "{\"description\": \"min=110, mean=110, max=110, sum=220 (2)\", \"tab\": \"General information\", \"score\": \"110.0\"}", - "Public Relations - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Public Relations - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Public Relations - # prompt tokens": "{\"description\": \"min=398.318, mean=398.318, max=398.318, sum=796.636 (2)\", \"tab\": \"General information\", \"score\": \"398.3181818181818\"}", - "Public Relations - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"public_relations\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_public_relations\"" - } - } - }, - { - "evaluation_name": "Security Studies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Security Studies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.833, - "details": { - "description": "min=0.833, mean=0.833, max=0.833, sum=1.665 (2)", - "tab": "Accuracy", - "Security Studies - Observed inference time (s)": "{\"description\": \"min=0.653, mean=0.653, max=0.653, sum=1.306 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.652998145745725\"}", - "Security Studies - # eval": "{\"description\": \"min=245, mean=245, max=245, sum=490 (2)\", \"tab\": \"General information\", \"score\": \"245.0\"}", - "Security Studies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Security Studies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Security Studies - # prompt tokens": "{\"description\": \"min=1157.473, mean=1157.473, max=1157.473, sum=2314.947 (2)\", \"tab\": \"General information\", \"score\": \"1157.4734693877551\"}", - "Security Studies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"security_studies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_security_studies\"" - } - } - }, - { - "evaluation_name": "Sociology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Sociology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.93, - "details": { - "description": "min=0.93, mean=0.93, max=0.93, sum=1.861 (2)", - "tab": "Accuracy", - "Sociology - Observed inference time (s)": "{\"description\": \"min=0.36, mean=0.36, max=0.36, sum=0.721 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3602804935986723\"}", - "Sociology - # eval": "{\"description\": \"min=201, mean=201, max=201, sum=402 (2)\", \"tab\": \"General information\", \"score\": \"201.0\"}", - "Sociology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Sociology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Sociology - # prompt tokens": "{\"description\": \"min=438.517, mean=438.517, max=438.517, sum=877.035 (2)\", \"tab\": \"General information\", \"score\": \"438.51741293532336\"}", - "Sociology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"sociology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_sociology\"" - } - } - }, - { - "evaluation_name": "Virology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Virology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.59, - "details": { - "description": "min=0.59, mean=0.59, max=0.59, sum=1.181 (2)", - "tab": "Accuracy", - "Virology - Observed inference time (s)": "{\"description\": \"min=0.371, mean=0.371, max=0.371, sum=0.743 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3714186226028994\"}", - "Virology - # eval": "{\"description\": \"min=166, mean=166, max=166, sum=332 (2)\", \"tab\": \"General information\", \"score\": \"166.0\"}", - "Virology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Virology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Virology - # prompt tokens": "{\"description\": \"min=336.018, mean=336.018, max=336.018, sum=672.036 (2)\", \"tab\": \"General information\", \"score\": \"336.01807228915663\"}", - "Virology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"virology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_virology\"" - } - } - }, - { - "evaluation_name": "World Religions", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on World Religions", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.906, - "details": { - "description": "min=0.906, mean=0.906, max=0.906, sum=1.813 (2)", - "tab": "Accuracy", - "World Religions - Observed inference time (s)": "{\"description\": \"min=0.333, mean=0.333, max=0.333, sum=0.665 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3325699170430501\"}", - "World Religions - # eval": "{\"description\": \"min=171, mean=171, max=171, sum=342 (2)\", \"tab\": \"General information\", \"score\": \"171.0\"}", - "World Religions - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "World Religions - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "World Religions - # prompt tokens": "{\"description\": \"min=267.52, mean=267.52, max=267.52, sum=535.041 (2)\", \"tab\": \"General information\", \"score\": \"267.5204678362573\"}", - "World Religions - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"world_religions\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_world_religions\"" - } - } - }, - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.524, - "details": { - "description": "", - "tab": "Efficiency" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/meta_llama-3-8b.json b/data/models/meta_llama-3-8b.json deleted file mode 100644 index 7565773421769d6772ddeb74d2b75053f32cae80..0000000000000000000000000000000000000000 --- a/data/models/meta_llama-3-8b.json +++ /dev/null @@ -1,1901 +0,0 @@ -{ - "model_info": { - "name": "Llama 3 8B", - "id": "meta/llama-3-8b", - "developer": "Meta", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_lite/meta_llama-3-8b/1774096306.427425", - "retrieved_timestamp": "1774096306.427425", - "source_metadata": { - "source_name": "helm_lite", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_lite", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_lite", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.387, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.7163920099875156\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.754, - "details": { - "description": "min=0.754, mean=0.754, max=0.754, sum=0.754 (1)", - "tab": "Accuracy", - "NarrativeQA - Observed inference time (s)": "{\"description\": \"min=0.726, mean=0.726, max=0.726, sum=0.726 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.7260531909029249\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=3460.268, mean=3460.268, max=3460.268, sum=3460.268 (1)\", \"tab\": \"General information\", \"score\": \"3460.2676056338028\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (closed-book)", - "source_data": { - "dataset_name": "NaturalQuestions (closed-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (closed-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.378, - "details": { - "description": "min=0.378, mean=0.378, max=0.378, sum=0.378 (1)", - "tab": "Accuracy", - "NaturalQuestions (open-book) - Observed inference time (s)": "{\"description\": \"min=0.524, mean=0.524, max=0.524, sum=0.524 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.523505747795105\"}", - "NaturalQuestions (closed-book) - Observed inference time (s)": "{\"description\": \"min=0.428, mean=0.428, max=0.428, sum=0.428 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.42760186743736267\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.965, mean=4.965, max=4.965, sum=4.965 (1)\", \"tab\": \"General information\", \"score\": \"4.965\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.007, mean=0.007, max=0.007, sum=0.007 (1)\", \"tab\": \"General information\", \"score\": \"0.007\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1658.348, mean=1658.348, max=1658.348, sum=1658.348 (1)\", \"tab\": \"General information\", \"score\": \"1658.348\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=0.999, mean=0.999, max=0.999, sum=0.999 (1)\", \"tab\": \"General information\", \"score\": \"0.999\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=112.12, mean=112.12, max=112.12, sum=112.12 (1)\", \"tab\": \"General information\", \"score\": \"112.12\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "mode": "\"closedbook\"" - } - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.766, - "details": { - "description": "min=0.766, mean=0.766, max=0.766, sum=0.766 (1)", - "tab": "Accuracy", - "OpenbookQA - Observed inference time (s)": "{\"description\": \"min=0.308, mean=0.308, max=0.308, sum=0.308 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.3076804256439209\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=242.776, mean=242.776, max=242.776, sum=242.776 (1)\", \"tab\": \"General information\", \"score\": \"242.776\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "dataset": "\"openbookqa\"", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.602, - "details": { - "description": "min=0.33, mean=0.602, max=0.88, sum=3.008 (5)", - "tab": "Accuracy", - "MMLU - Observed inference time (s)": "{\"description\": \"min=0.3, mean=0.317, max=0.344, sum=1.583 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.3165063006919727\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=366.43, mean=460.686, max=607.421, sum=2303.431 (5)\", \"tab\": \"General information\", \"score\": \"460.6862105263158\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"college_chemistry\", \"computer_security\", \"econometrics\", \"us_foreign_policy\"]", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MATH", - "source_data": { - "dataset_name": "MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Equivalent (CoT) on MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.391, - "details": { - "description": "min=0.233, mean=0.391, max=0.496, sum=2.737 (7)", - "tab": "Accuracy", - "MATH - Observed inference time (s)": "{\"description\": \"min=5.431, mean=5.651, max=6.121, sum=39.558 (7)\", \"tab\": \"Efficiency\", \"score\": \"5.651119198181415\"}", - "MATH - # eval": "{\"description\": \"min=30, mean=62.429, max=135, sum=437 (7)\", \"tab\": \"General information\", \"score\": \"62.42857142857143\"}", - "MATH - # train": "{\"description\": \"min=8, mean=8, max=8, sum=56 (7)\", \"tab\": \"General information\", \"score\": \"8.0\"}", - "MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MATH - # prompt tokens": "{\"description\": \"min=881.363, mean=1262.909, max=2197.577, sum=8840.364 (7)\", \"tab\": \"General information\", \"score\": \"1262.9092130545007\"}", - "MATH - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=7 (7)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"algebra\", \"counting_and_probability\", \"geometry\", \"intermediate_algebra\", \"number_theory\", \"prealgebra\", \"precalculus\"]", - "level": "\"1\"", - "use_official_examples": "\"False\"", - "use_chain_of_thought": "\"True\"" - } - } - }, - { - "evaluation_name": "GSM8K", - "source_data": { - "dataset_name": "GSM8K", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on GSM8K", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.499, - "details": { - "description": "min=0.499, mean=0.499, max=0.499, sum=0.499 (1)", - "tab": "Accuracy", - "GSM8K - Observed inference time (s)": "{\"description\": \"min=1.771, mean=1.771, max=1.771, sum=1.771 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.770608879327774\"}", - "GSM8K - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "GSM8K - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "GSM8K - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GSM8K - # prompt tokens": "{\"description\": \"min=959.032, mean=959.032, max=959.032, sum=959.032 (1)\", \"tab\": \"General information\", \"score\": \"959.032\"}", - "GSM8K - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "LegalBench", - "source_data": { - "dataset_name": "LegalBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on LegalBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.637, - "details": { - "description": "min=0.417, mean=0.637, max=0.874, sum=3.185 (5)", - "tab": "Accuracy", - "LegalBench - Observed inference time (s)": "{\"description\": \"min=0.322, mean=0.465, max=0.989, sum=2.326 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.4651390315970952\"}", - "LegalBench - # eval": "{\"description\": \"min=95, mean=409.4, max=1000, sum=2047 (5)\", \"tab\": \"General information\", \"score\": \"409.4\"}", - "LegalBench - # train": "{\"description\": \"min=4, mean=4.798, max=5, sum=23.992 (5)\", \"tab\": \"General information\", \"score\": \"4.798367346938775\"}", - "LegalBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "LegalBench - # prompt tokens": "{\"description\": \"min=192.442, mean=1507.407, max=6287.633, sum=7537.033 (5)\", \"tab\": \"General information\", \"score\": \"1507.4065013565441\"}", - "LegalBench - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"abercrombie\", \"corporate_lobbying\", \"function_of_decision_section\", \"international_citizenship_questions\", \"proa\"]" - } - } - }, - { - "evaluation_name": "MedQA", - "source_data": { - "dataset_name": "MedQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MedQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.581, - "details": { - "description": "min=0.581, mean=0.581, max=0.581, sum=0.581 (1)", - "tab": "Accuracy", - "MedQA - Observed inference time (s)": "{\"description\": \"min=0.361, mean=0.361, max=0.361, sum=0.361 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.36141945306159867\"}", - "MedQA - # eval": "{\"description\": \"min=503, mean=503, max=503, sum=503 (1)\", \"tab\": \"General information\", \"score\": \"503.0\"}", - "MedQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MedQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MedQA - # prompt tokens": "{\"description\": \"min=1018.274, mean=1018.274, max=1018.274, sum=1018.274 (1)\", \"tab\": \"General information\", \"score\": \"1018.2743538767396\"}", - "MedQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WMT 2014", - "source_data": { - "dataset_name": "WMT 2014", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "BLEU-4 on WMT 2014", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.183, - "details": { - "description": "min=0.133, mean=0.183, max=0.212, sum=0.915 (5)", - "tab": "Accuracy", - "WMT 2014 - Observed inference time (s)": "{\"description\": \"min=0.547, mean=0.563, max=0.573, sum=2.816 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.5631435248437351\"}", - "WMT 2014 - # eval": "{\"description\": \"min=503, mean=568.8, max=832, sum=2844 (5)\", \"tab\": \"General information\", \"score\": \"568.8\"}", - "WMT 2014 - # train": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "WMT 2014 - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WMT 2014 - # prompt tokens": "{\"description\": \"min=90.139, mean=109.868, max=130.33, sum=549.34 (5)\", \"tab\": \"General information\", \"score\": \"109.86804366111025\"}", - "WMT 2014 - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "language_pair": "[\"cs-en\", \"de-en\", \"fr-en\", \"hi-en\", \"ru-en\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_mmlu/meta_llama-3-8b/1774096312.00548", - "retrieved_timestamp": "1774096312.00548", - "source_metadata": { - "source_name": "helm_mmlu", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_mmlu", - "evaluation_results": [ - { - "evaluation_name": "MMLU All Subjects", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU All Subjects", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.668, - "details": { - "description": "min=0.33, mean=0.668, max=0.885, sum=76.111 (114)", - "tab": "Accuracy", - "MMLU All Subjects - Observed inference time (s)": "{\"description\": \"min=0.288, mean=0.35, max=0.586, sum=39.916 (114)\", \"tab\": \"Efficiency\", \"score\": \"0.350140152719457\"}", - "MMLU All Subjects - # eval": "{\"description\": \"min=100, mean=246.351, max=1534, sum=28084 (114)\", \"tab\": \"General information\", \"score\": \"246.35087719298247\"}", - "MMLU All Subjects - # train": "{\"description\": \"min=5, mean=5, max=5, sum=570 (114)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU All Subjects - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU All Subjects - # prompt tokens": "{\"description\": \"min=267.52, mean=607.619, max=2790.885, sum=69268.61 (114)\", \"tab\": \"General information\", \"score\": \"607.6193817308517\"}", - "MMLU All Subjects - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=114 (114)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_name": "Abstract Algebra", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Abstract Algebra", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.33, - "details": { - "description": "min=0.33, mean=0.33, max=0.33, sum=0.66 (2)", - "tab": "Accuracy", - "Abstract Algebra - Observed inference time (s)": "{\"description\": \"min=0.309, mean=0.309, max=0.309, sum=0.618 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.30905162572860717\"}", - "Abstract Algebra - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Abstract Algebra - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Abstract Algebra - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Abstract Algebra - # prompt tokens": "{\"description\": \"min=366.43, mean=366.43, max=366.43, sum=732.86 (2)\", \"tab\": \"General information\", \"score\": \"366.43\"}", - "Abstract Algebra - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"abstract_algebra\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_abstract_algebra\"" - } - } - }, - { - "evaluation_name": "Anatomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Anatomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.696, - "details": { - "description": "min=0.696, mean=0.696, max=0.696, sum=1.393 (2)", - "tab": "Accuracy", - "Anatomy - Observed inference time (s)": "{\"description\": \"min=0.288, mean=0.288, max=0.288, sum=0.577 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.28846773041619195\"}", - "Anatomy - # eval": "{\"description\": \"min=135, mean=135, max=135, sum=270 (2)\", \"tab\": \"General information\", \"score\": \"135.0\"}", - "Anatomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Anatomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Anatomy - # prompt tokens": "{\"description\": \"min=346.874, mean=346.874, max=346.874, sum=693.748 (2)\", \"tab\": \"General information\", \"score\": \"346.8740740740741\"}", - "Anatomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"anatomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_anatomy\"" - } - } - }, - { - "evaluation_name": "College Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on College Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.451, - "details": { - "description": "min=0.451, mean=0.451, max=0.451, sum=0.902 (2)", - "tab": "Accuracy", - "College Chemistry - Observed inference time (s)": "{\"description\": \"min=0.323, mean=0.323, max=0.323, sum=0.646 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3228257203102112\"}", - "College Biology - Observed inference time (s)": "{\"description\": \"min=0.343, mean=0.343, max=0.343, sum=0.687 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.34339087539248997\"}", - "College Computer Science - Observed inference time (s)": "{\"description\": \"min=0.366, mean=0.366, max=0.366, sum=0.733 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3662724041938782\"}", - "College Mathematics - Observed inference time (s)": "{\"description\": \"min=0.32, mean=0.32, max=0.32, sum=0.64 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.320071747303009\"}", - "College Medicine - Observed inference time (s)": "{\"description\": \"min=0.329, mean=0.329, max=0.329, sum=0.657 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.32854826739757736\"}", - "College Physics - Observed inference time (s)": "{\"description\": \"min=0.299, mean=0.299, max=0.299, sum=0.599 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2994629471909766\"}", - "College Chemistry - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Chemistry - # prompt tokens": "{\"description\": \"min=542.28, mean=542.28, max=542.28, sum=1084.56 (2)\", \"tab\": \"General information\", \"score\": \"542.28\"}", - "College Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Biology - # eval": "{\"description\": \"min=144, mean=144, max=144, sum=288 (2)\", \"tab\": \"General information\", \"score\": \"144.0\"}", - "College Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # prompt tokens": "{\"description\": \"min=466.875, mean=466.875, max=466.875, sum=933.75 (2)\", \"tab\": \"General information\", \"score\": \"466.875\"}", - "College Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # prompt tokens": "{\"description\": \"min=821.29, mean=821.29, max=821.29, sum=1642.58 (2)\", \"tab\": \"General information\", \"score\": \"821.29\"}", - "College Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Mathematics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # prompt tokens": "{\"description\": \"min=587.51, mean=587.51, max=587.51, sum=1175.02 (2)\", \"tab\": \"General information\", \"score\": \"587.51\"}", - "College Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Medicine - # eval": "{\"description\": \"min=173, mean=173, max=173, sum=346 (2)\", \"tab\": \"General information\", \"score\": \"173.0\"}", - "College Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # prompt tokens": "{\"description\": \"min=495.705, mean=495.705, max=495.705, sum=991.41 (2)\", \"tab\": \"General information\", \"score\": \"495.70520231213874\"}", - "College Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Physics - # eval": "{\"description\": \"min=102, mean=102, max=102, sum=204 (2)\", \"tab\": \"General information\", \"score\": \"102.0\"}", - "College Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # prompt tokens": "{\"description\": \"min=496.569, mean=496.569, max=496.569, sum=993.137 (2)\", \"tab\": \"General information\", \"score\": \"496.5686274509804\"}", - "College Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"college_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_college_physics\"" - } - } - }, - { - "evaluation_name": "Computer Security", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Computer Security", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8, - "details": { - "description": "min=0.8, mean=0.8, max=0.8, sum=1.6 (2)", - "tab": "Accuracy", - "Computer Security - Observed inference time (s)": "{\"description\": \"min=0.307, mean=0.307, max=0.307, sum=0.614 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3068851590156555\"}", - "Computer Security - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Computer Security - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Computer Security - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Computer Security - # prompt tokens": "{\"description\": \"min=371.51, mean=371.51, max=371.51, sum=743.02 (2)\", \"tab\": \"General information\", \"score\": \"371.51\"}", - "Computer Security - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"computer_security\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_computer_security\"" - } - } - }, - { - "evaluation_name": "Econometrics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Econometrics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.518, - "details": { - "description": "min=0.518, mean=0.518, max=0.518, sum=1.035 (2)", - "tab": "Accuracy", - "Econometrics - Observed inference time (s)": "{\"description\": \"min=0.344, mean=0.344, max=0.344, sum=0.689 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3442605817527102\"}", - "Econometrics - # eval": "{\"description\": \"min=114, mean=114, max=114, sum=228 (2)\", \"tab\": \"General information\", \"score\": \"114.0\"}", - "Econometrics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Econometrics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Econometrics - # prompt tokens": "{\"description\": \"min=607.421, mean=607.421, max=607.421, sum=1214.842 (2)\", \"tab\": \"General information\", \"score\": \"607.421052631579\"}", - "Econometrics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"econometrics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_econometrics\"" - } - } - }, - { - "evaluation_name": "Global Facts", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Global Facts", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.34, - "details": { - "description": "min=0.34, mean=0.34, max=0.34, sum=0.68 (2)", - "tab": "Accuracy", - "Global Facts - Observed inference time (s)": "{\"description\": \"min=0.311, mean=0.311, max=0.311, sum=0.622 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3109010863304138\"}", - "Global Facts - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Global Facts - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Global Facts - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Global Facts - # prompt tokens": "{\"description\": \"min=392.71, mean=392.71, max=392.71, sum=785.42 (2)\", \"tab\": \"General information\", \"score\": \"392.71\"}", - "Global Facts - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"global_facts\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_global_facts\"" - } - } - }, - { - "evaluation_name": "Jurisprudence", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Jurisprudence", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.741, - "details": { - "description": "min=0.741, mean=0.741, max=0.741, sum=1.481 (2)", - "tab": "Accuracy", - "Jurisprudence - Observed inference time (s)": "{\"description\": \"min=0.323, mean=0.323, max=0.323, sum=0.645 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.32258448998133343\"}", - "Jurisprudence - # eval": "{\"description\": \"min=108, mean=108, max=108, sum=216 (2)\", \"tab\": \"General information\", \"score\": \"108.0\"}", - "Jurisprudence - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Jurisprudence - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Jurisprudence - # prompt tokens": "{\"description\": \"min=387.63, mean=387.63, max=387.63, sum=775.259 (2)\", \"tab\": \"General information\", \"score\": \"387.6296296296296\"}", - "Jurisprudence - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"jurisprudence\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_jurisprudence\"" - } - } - }, - { - "evaluation_name": "Philosophy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Philosophy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.743, - "details": { - "description": "min=0.743, mean=0.743, max=0.743, sum=1.486 (2)", - "tab": "Accuracy", - "Philosophy - Observed inference time (s)": "{\"description\": \"min=0.309, mean=0.309, max=0.309, sum=0.617 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3085632078900598\"}", - "Philosophy - # eval": "{\"description\": \"min=311, mean=311, max=311, sum=622 (2)\", \"tab\": \"General information\", \"score\": \"311.0\"}", - "Philosophy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Philosophy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Philosophy - # prompt tokens": "{\"description\": \"min=322.084, mean=322.084, max=322.084, sum=644.167 (2)\", \"tab\": \"General information\", \"score\": \"322.08360128617363\"}", - "Philosophy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"philosophy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_philosophy\"" - } - } - }, - { - "evaluation_name": "Professional Psychology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Professional Psychology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.711, - "details": { - "description": "min=0.711, mean=0.711, max=0.711, sum=1.422 (2)", - "tab": "Accuracy", - "Professional Medicine - Observed inference time (s)": "{\"description\": \"min=0.341, mean=0.341, max=0.341, sum=0.682 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.34079881275401397\"}", - "Professional Accounting - Observed inference time (s)": "{\"description\": \"min=0.45, mean=0.45, max=0.45, sum=0.901 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4504219800867933\"}", - "Professional Law - Observed inference time (s)": "{\"description\": \"min=0.429, mean=0.429, max=0.429, sum=0.857 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4285039446344587\"}", - "Professional Psychology - Observed inference time (s)": "{\"description\": \"min=0.376, mean=0.376, max=0.376, sum=0.752 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3759713149538227\"}", - "Professional Medicine - # eval": "{\"description\": \"min=272, mean=272, max=272, sum=544 (2)\", \"tab\": \"General information\", \"score\": \"272.0\"}", - "Professional Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Medicine - # prompt tokens": "{\"description\": \"min=1087.489, mean=1087.489, max=1087.489, sum=2174.978 (2)\", \"tab\": \"General information\", \"score\": \"1087.4889705882354\"}", - "Professional Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Accounting - # eval": "{\"description\": \"min=282, mean=282, max=282, sum=564 (2)\", \"tab\": \"General information\", \"score\": \"282.0\"}", - "Professional Accounting - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Accounting - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # prompt tokens": "{\"description\": \"min=651.585, mean=651.585, max=651.585, sum=1303.17 (2)\", \"tab\": \"General information\", \"score\": \"651.5851063829788\"}", - "Professional Accounting - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Law - # eval": "{\"description\": \"min=1534, mean=1534, max=1534, sum=3068 (2)\", \"tab\": \"General information\", \"score\": \"1534.0\"}", - "Professional Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # prompt tokens": "{\"description\": \"min=1630.601, mean=1630.601, max=1630.601, sum=3261.202 (2)\", \"tab\": \"General information\", \"score\": \"1630.6010430247718\"}", - "Professional Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Psychology - # eval": "{\"description\": \"min=612, mean=612, max=612, sum=1224 (2)\", \"tab\": \"General information\", \"score\": \"612.0\"}", - "Professional Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # prompt tokens": "{\"description\": \"min=568.098, mean=568.098, max=568.098, sum=1136.196 (2)\", \"tab\": \"General information\", \"score\": \"568.0980392156863\"}", - "Professional Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"professional_psychology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_professional_psychology\"" - } - } - }, - { - "evaluation_name": "Us Foreign Policy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Us Foreign Policy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.88, - "details": { - "description": "min=0.88, mean=0.88, max=0.88, sum=1.76 (2)", - "tab": "Accuracy", - "Us Foreign Policy - Observed inference time (s)": "{\"description\": \"min=0.3, mean=0.3, max=0.3, sum=0.599 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.29950841665267947\"}", - "Us Foreign Policy - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Us Foreign Policy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Us Foreign Policy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Us Foreign Policy - # prompt tokens": "{\"description\": \"min=415.79, mean=415.79, max=415.79, sum=831.58 (2)\", \"tab\": \"General information\", \"score\": \"415.79\"}", - "Us Foreign Policy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"us_foreign_policy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_us_foreign_policy\"" - } - } - }, - { - "evaluation_name": "Astronomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Astronomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.711, - "details": { - "description": "min=0.711, mean=0.711, max=0.711, sum=1.421 (2)", - "tab": "Accuracy", - "Astronomy - Observed inference time (s)": "{\"description\": \"min=0.539, mean=0.539, max=0.539, sum=1.077 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5385584250876778\"}", - "Astronomy - # eval": "{\"description\": \"min=152, mean=152, max=152, sum=304 (2)\", \"tab\": \"General information\", \"score\": \"152.0\"}", - "Astronomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Astronomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Astronomy - # prompt tokens": "{\"description\": \"min=572.684, mean=572.684, max=572.684, sum=1145.368 (2)\", \"tab\": \"General information\", \"score\": \"572.6842105263158\"}", - "Astronomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"astronomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_astronomy\"" - } - } - }, - { - "evaluation_name": "Business Ethics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Business Ethics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.65, - "details": { - "description": "min=0.65, mean=0.65, max=0.65, sum=1.3 (2)", - "tab": "Accuracy", - "Business Ethics - Observed inference time (s)": "{\"description\": \"min=0.312, mean=0.312, max=0.312, sum=0.623 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.311549117565155\"}", - "Business Ethics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Business Ethics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Business Ethics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Business Ethics - # prompt tokens": "{\"description\": \"min=562.52, mean=562.52, max=562.52, sum=1125.04 (2)\", \"tab\": \"General information\", \"score\": \"562.52\"}", - "Business Ethics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"business_ethics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_business_ethics\"" - } - } - }, - { - "evaluation_name": "Clinical Knowledge", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Clinical Knowledge", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.751, - "details": { - "description": "min=0.751, mean=0.751, max=0.751, sum=1.502 (2)", - "tab": "Accuracy", - "Clinical Knowledge - Observed inference time (s)": "{\"description\": \"min=0.304, mean=0.304, max=0.304, sum=0.609 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3043576915309114\"}", - "Clinical Knowledge - # eval": "{\"description\": \"min=265, mean=265, max=265, sum=530 (2)\", \"tab\": \"General information\", \"score\": \"265.0\"}", - "Clinical Knowledge - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Clinical Knowledge - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Clinical Knowledge - # prompt tokens": "{\"description\": \"min=390.928, mean=390.928, max=390.928, sum=781.857 (2)\", \"tab\": \"General information\", \"score\": \"390.92830188679244\"}", - "Clinical Knowledge - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"clinical_knowledge\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_clinical_knowledge\"" - } - } - }, - { - "evaluation_name": "Conceptual Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Conceptual Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.557, - "details": { - "description": "min=0.557, mean=0.557, max=0.557, sum=1.115 (2)", - "tab": "Accuracy", - "Conceptual Physics - Observed inference time (s)": "{\"description\": \"min=0.315, mean=0.315, max=0.315, sum=0.631 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.31532351615581106\"}", - "Conceptual Physics - # eval": "{\"description\": \"min=235, mean=235, max=235, sum=470 (2)\", \"tab\": \"General information\", \"score\": \"235.0\"}", - "Conceptual Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Conceptual Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Conceptual Physics - # prompt tokens": "{\"description\": \"min=297.834, mean=297.834, max=297.834, sum=595.668 (2)\", \"tab\": \"General information\", \"score\": \"297.83404255319147\"}", - "Conceptual Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"conceptual_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_conceptual_physics\"" - } - } - }, - { - "evaluation_name": "Electrical Engineering", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Electrical Engineering", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.669, - "details": { - "description": "min=0.669, mean=0.669, max=0.669, sum=1.338 (2)", - "tab": "Accuracy", - "Electrical Engineering - Observed inference time (s)": "{\"description\": \"min=0.317, mean=0.317, max=0.317, sum=0.635 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.31737767910135206\"}", - "Electrical Engineering - # eval": "{\"description\": \"min=145, mean=145, max=145, sum=290 (2)\", \"tab\": \"General information\", \"score\": \"145.0\"}", - "Electrical Engineering - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Electrical Engineering - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Electrical Engineering - # prompt tokens": "{\"description\": \"min=428.607, mean=428.607, max=428.607, sum=857.214 (2)\", \"tab\": \"General information\", \"score\": \"428.60689655172416\"}", - "Electrical Engineering - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"electrical_engineering\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_electrical_engineering\"" - } - } - }, - { - "evaluation_name": "Elementary Mathematics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Elementary Mathematics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.426, - "details": { - "description": "min=0.426, mean=0.426, max=0.426, sum=0.852 (2)", - "tab": "Accuracy", - "Elementary Mathematics - Observed inference time (s)": "{\"description\": \"min=0.308, mean=0.308, max=0.308, sum=0.616 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3080339734516447\"}", - "Elementary Mathematics - # eval": "{\"description\": \"min=378, mean=378, max=378, sum=756 (2)\", \"tab\": \"General information\", \"score\": \"378.0\"}", - "Elementary Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Elementary Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Elementary Mathematics - # prompt tokens": "{\"description\": \"min=524.854, mean=524.854, max=524.854, sum=1049.709 (2)\", \"tab\": \"General information\", \"score\": \"524.8544973544973\"}", - "Elementary Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"elementary_mathematics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_elementary_mathematics\"" - } - } - }, - { - "evaluation_name": "Formal Logic", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Formal Logic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.468, - "details": { - "description": "min=0.468, mean=0.468, max=0.468, sum=0.937 (2)", - "tab": "Accuracy", - "Formal Logic - Observed inference time (s)": "{\"description\": \"min=0.337, mean=0.337, max=0.337, sum=0.674 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.33724411328633624\"}", - "Formal Logic - # eval": "{\"description\": \"min=126, mean=126, max=126, sum=252 (2)\", \"tab\": \"General information\", \"score\": \"126.0\"}", - "Formal Logic - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Formal Logic - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Formal Logic - # prompt tokens": "{\"description\": \"min=594.778, mean=594.778, max=594.778, sum=1189.556 (2)\", \"tab\": \"General information\", \"score\": \"594.7777777777778\"}", - "Formal Logic - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"formal_logic\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_formal_logic\"" - } - } - }, - { - "evaluation_name": "High School World History", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on High School World History", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.823, - "details": { - "description": "min=0.823, mean=0.823, max=0.823, sum=1.646 (2)", - "tab": "Accuracy", - "High School Biology - Observed inference time (s)": "{\"description\": \"min=0.336, mean=0.336, max=0.336, sum=0.672 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3359520781424738\"}", - "High School Chemistry - Observed inference time (s)": "{\"description\": \"min=0.309, mean=0.309, max=0.309, sum=0.619 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3092998248602956\"}", - "High School Computer Science - Observed inference time (s)": "{\"description\": \"min=0.325, mean=0.325, max=0.325, sum=0.649 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.324708514213562\"}", - "High School European History - Observed inference time (s)": "{\"description\": \"min=0.544, mean=0.544, max=0.544, sum=1.087 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5437044996203798\"}", - "High School Geography - Observed inference time (s)": "{\"description\": \"min=0.304, mean=0.304, max=0.304, sum=0.609 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.30433518236333673\"}", - "High School Government And Politics - Observed inference time (s)": "{\"description\": \"min=0.319, mean=0.319, max=0.319, sum=0.638 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3192491321366068\"}", - "High School Macroeconomics - Observed inference time (s)": "{\"description\": \"min=0.315, mean=0.315, max=0.315, sum=0.63 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.31492268366691395\"}", - "High School Mathematics - Observed inference time (s)": "{\"description\": \"min=0.326, mean=0.326, max=0.326, sum=0.652 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3262451118893094\"}", - "High School Microeconomics - Observed inference time (s)": "{\"description\": \"min=0.345, mean=0.345, max=0.345, sum=0.69 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3451059505719097\"}", - "High School Physics - Observed inference time (s)": "{\"description\": \"min=0.541, mean=0.541, max=0.541, sum=1.082 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5410290490712552\"}", - "High School Psychology - Observed inference time (s)": "{\"description\": \"min=0.379, mean=0.379, max=0.379, sum=0.757 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3786245923523509\"}", - "High School Statistics - Observed inference time (s)": "{\"description\": \"min=0.493, mean=0.493, max=0.493, sum=0.986 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4927717314826118\"}", - "High School US History - Observed inference time (s)": "{\"description\": \"min=0.481, mean=0.481, max=0.481, sum=0.962 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.48103941655626486\"}", - "High School World History - Observed inference time (s)": "{\"description\": \"min=0.516, mean=0.516, max=0.516, sum=1.032 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5161508246313168\"}", - "High School Biology - # eval": "{\"description\": \"min=310, mean=310, max=310, sum=620 (2)\", \"tab\": \"General information\", \"score\": \"310.0\"}", - "High School Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Biology - # prompt tokens": "{\"description\": \"min=506.671, mean=506.671, max=506.671, sum=1013.342 (2)\", \"tab\": \"General information\", \"score\": \"506.6709677419355\"}", - "High School Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Chemistry - # eval": "{\"description\": \"min=203, mean=203, max=203, sum=406 (2)\", \"tab\": \"General information\", \"score\": \"203.0\"}", - "High School Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # prompt tokens": "{\"description\": \"min=489.704, mean=489.704, max=489.704, sum=979.409 (2)\", \"tab\": \"General information\", \"score\": \"489.70443349753697\"}", - "High School Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "High School Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # prompt tokens": "{\"description\": \"min=860.78, mean=860.78, max=860.78, sum=1721.56 (2)\", \"tab\": \"General information\", \"score\": \"860.78\"}", - "High School Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School European History - # eval": "{\"description\": \"min=165, mean=165, max=165, sum=330 (2)\", \"tab\": \"General information\", \"score\": \"165.0\"}", - "High School European History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School European History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # prompt tokens": "{\"description\": \"min=2790.885, mean=2790.885, max=2790.885, sum=5581.77 (2)\", \"tab\": \"General information\", \"score\": \"2790.8848484848486\"}", - "High School European History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Geography - # eval": "{\"description\": \"min=198, mean=198, max=198, sum=396 (2)\", \"tab\": \"General information\", \"score\": \"198.0\"}", - "High School Geography - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Geography - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # prompt tokens": "{\"description\": \"min=365.035, mean=365.035, max=365.035, sum=730.071 (2)\", \"tab\": \"General information\", \"score\": \"365.0353535353535\"}", - "High School Geography - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Government And Politics - # eval": "{\"description\": \"min=193, mean=193, max=193, sum=386 (2)\", \"tab\": \"General information\", \"score\": \"193.0\"}", - "High School Government And Politics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Government And Politics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # prompt tokens": "{\"description\": \"min=458.824, mean=458.824, max=458.824, sum=917.648 (2)\", \"tab\": \"General information\", \"score\": \"458.8238341968912\"}", - "High School Government And Politics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Macroeconomics - # eval": "{\"description\": \"min=390, mean=390, max=390, sum=780 (2)\", \"tab\": \"General information\", \"score\": \"390.0\"}", - "High School Macroeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Macroeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # prompt tokens": "{\"description\": \"min=363.908, mean=363.908, max=363.908, sum=727.815 (2)\", \"tab\": \"General information\", \"score\": \"363.9076923076923\"}", - "High School Macroeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Mathematics - # eval": "{\"description\": \"min=270, mean=270, max=270, sum=540 (2)\", \"tab\": \"General information\", \"score\": \"270.0\"}", - "High School Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # prompt tokens": "{\"description\": \"min=525.356, mean=525.356, max=525.356, sum=1050.711 (2)\", \"tab\": \"General information\", \"score\": \"525.3555555555556\"}", - "High School Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Microeconomics - # eval": "{\"description\": \"min=238, mean=238, max=238, sum=476 (2)\", \"tab\": \"General information\", \"score\": \"238.0\"}", - "High School Microeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Microeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # prompt tokens": "{\"description\": \"min=392.013, mean=392.013, max=392.013, sum=784.025 (2)\", \"tab\": \"General information\", \"score\": \"392.0126050420168\"}", - "High School Microeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Physics - # eval": "{\"description\": \"min=151, mean=151, max=151, sum=302 (2)\", \"tab\": \"General information\", \"score\": \"151.0\"}", - "High School Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # prompt tokens": "{\"description\": \"min=553.457, mean=553.457, max=553.457, sum=1106.914 (2)\", \"tab\": \"General information\", \"score\": \"553.4569536423841\"}", - "High School Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Psychology - # eval": "{\"description\": \"min=545, mean=545, max=545, sum=1090 (2)\", \"tab\": \"General information\", \"score\": \"545.0\"}", - "High School Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # prompt tokens": "{\"description\": \"min=488.242, mean=488.242, max=488.242, sum=976.484 (2)\", \"tab\": \"General information\", \"score\": \"488.2422018348624\"}", - "High School Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Statistics - # eval": "{\"description\": \"min=216, mean=216, max=216, sum=432 (2)\", \"tab\": \"General information\", \"score\": \"216.0\"}", - "High School Statistics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Statistics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # prompt tokens": "{\"description\": \"min=788.639, mean=788.639, max=788.639, sum=1577.278 (2)\", \"tab\": \"General information\", \"score\": \"788.6388888888889\"}", - "High School Statistics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School US History - # eval": "{\"description\": \"min=204, mean=204, max=204, sum=408 (2)\", \"tab\": \"General information\", \"score\": \"204.0\"}", - "High School US History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School US History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # prompt tokens": "{\"description\": \"min=2210.809, mean=2210.809, max=2210.809, sum=4421.618 (2)\", \"tab\": \"General information\", \"score\": \"2210.8088235294117\"}", - "High School US History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School World History - # eval": "{\"description\": \"min=237, mean=237, max=237, sum=474 (2)\", \"tab\": \"General information\", \"score\": \"237.0\"}", - "High School World History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School World History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # prompt tokens": "{\"description\": \"min=1421.173, mean=1421.173, max=1421.173, sum=2842.346 (2)\", \"tab\": \"General information\", \"score\": \"1421.1729957805908\"}", - "High School World History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"high_school_world_history\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_high_school_world_history\"" - } - } - }, - { - "evaluation_name": "Human Sexuality", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Human Sexuality", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.748, - "details": { - "description": "min=0.748, mean=0.748, max=0.748, sum=1.496 (2)", - "tab": "Accuracy", - "Human Aging - Observed inference time (s)": "{\"description\": \"min=0.303, mean=0.303, max=0.303, sum=0.605 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.30269593080597607\"}", - "Human Sexuality - Observed inference time (s)": "{\"description\": \"min=0.325, mean=0.325, max=0.325, sum=0.651 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.32543583862654124\"}", - "Human Aging - # eval": "{\"description\": \"min=223, mean=223, max=223, sum=446 (2)\", \"tab\": \"General information\", \"score\": \"223.0\"}", - "Human Aging - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Aging - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Aging - # prompt tokens": "{\"description\": \"min=312.888, mean=312.888, max=312.888, sum=625.776 (2)\", \"tab\": \"General information\", \"score\": \"312.88789237668163\"}", - "Human Aging - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Human Sexuality - # eval": "{\"description\": \"min=131, mean=131, max=131, sum=262 (2)\", \"tab\": \"General information\", \"score\": \"131.0\"}", - "Human Sexuality - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Sexuality - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # prompt tokens": "{\"description\": \"min=334.168, mean=334.168, max=334.168, sum=668.336 (2)\", \"tab\": \"General information\", \"score\": \"334.1679389312977\"}", - "Human Sexuality - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"human_sexuality\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_human_sexuality\"" - } - } - }, - { - "evaluation_name": "International Law", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on International Law", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.843, - "details": { - "description": "min=0.843, mean=0.843, max=0.843, sum=1.686 (2)", - "tab": "Accuracy", - "International Law - Observed inference time (s)": "{\"description\": \"min=0.586, mean=0.586, max=0.586, sum=1.172 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5860170076701267\"}", - "International Law - # eval": "{\"description\": \"min=121, mean=121, max=121, sum=242 (2)\", \"tab\": \"General information\", \"score\": \"121.0\"}", - "International Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "International Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "International Law - # prompt tokens": "{\"description\": \"min=632.818, mean=632.818, max=632.818, sum=1265.636 (2)\", \"tab\": \"General information\", \"score\": \"632.8181818181819\"}", - "International Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"international_law\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_international_law\"" - } - } - }, - { - "evaluation_name": "Logical Fallacies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Logical Fallacies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.755, - "details": { - "description": "min=0.755, mean=0.755, max=0.755, sum=1.509 (2)", - "tab": "Accuracy", - "Logical Fallacies - Observed inference time (s)": "{\"description\": \"min=0.313, mean=0.313, max=0.313, sum=0.625 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.31263120335303934\"}", - "Logical Fallacies - # eval": "{\"description\": \"min=163, mean=163, max=163, sum=326 (2)\", \"tab\": \"General information\", \"score\": \"163.0\"}", - "Logical Fallacies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Logical Fallacies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Logical Fallacies - # prompt tokens": "{\"description\": \"min=442.564, mean=442.564, max=442.564, sum=885.129 (2)\", \"tab\": \"General information\", \"score\": \"442.5644171779141\"}", - "Logical Fallacies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"logical_fallacies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_logical_fallacies\"" - } - } - }, - { - "evaluation_name": "Machine Learning", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Machine Learning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.545, - "details": { - "description": "min=0.545, mean=0.545, max=0.545, sum=1.089 (2)", - "tab": "Accuracy", - "Machine Learning - Observed inference time (s)": "{\"description\": \"min=0.309, mean=0.309, max=0.309, sum=0.618 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.30891925522259306\"}", - "Machine Learning - # eval": "{\"description\": \"min=112, mean=112, max=112, sum=224 (2)\", \"tab\": \"General information\", \"score\": \"112.0\"}", - "Machine Learning - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Machine Learning - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Machine Learning - # prompt tokens": "{\"description\": \"min=661.054, mean=661.054, max=661.054, sum=1322.107 (2)\", \"tab\": \"General information\", \"score\": \"661.0535714285714\"}", - "Machine Learning - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"machine_learning\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_machine_learning\"" - } - } - }, - { - "evaluation_name": "Management", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Management", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.874, - "details": { - "description": "min=0.874, mean=0.874, max=0.874, sum=1.748 (2)", - "tab": "Accuracy", - "Management - Observed inference time (s)": "{\"description\": \"min=0.298, mean=0.298, max=0.298, sum=0.596 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.29801390703442027\"}", - "Management - # eval": "{\"description\": \"min=103, mean=103, max=103, sum=206 (2)\", \"tab\": \"General information\", \"score\": \"103.0\"}", - "Management - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Management - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Management - # prompt tokens": "{\"description\": \"min=276.786, mean=276.786, max=276.786, sum=553.573 (2)\", \"tab\": \"General information\", \"score\": \"276.7864077669903\"}", - "Management - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"management\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_management\"" - } - } - }, - { - "evaluation_name": "Marketing", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Marketing", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.885, - "details": { - "description": "min=0.885, mean=0.885, max=0.885, sum=1.769 (2)", - "tab": "Accuracy", - "Marketing - Observed inference time (s)": "{\"description\": \"min=0.297, mean=0.297, max=0.297, sum=0.595 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.29727030717409575\"}", - "Marketing - # eval": "{\"description\": \"min=234, mean=234, max=234, sum=468 (2)\", \"tab\": \"General information\", \"score\": \"234.0\"}", - "Marketing - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Marketing - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Marketing - # prompt tokens": "{\"description\": \"min=397.218, mean=397.218, max=397.218, sum=794.436 (2)\", \"tab\": \"General information\", \"score\": \"397.21794871794873\"}", - "Marketing - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"marketing\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_marketing\"" - } - } - }, - { - "evaluation_name": "Medical Genetics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Medical Genetics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.83, - "details": { - "description": "min=0.83, mean=0.83, max=0.83, sum=1.66 (2)", - "tab": "Accuracy", - "Medical Genetics - Observed inference time (s)": "{\"description\": \"min=0.301, mean=0.301, max=0.301, sum=0.602 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3011839747428894\"}", - "Medical Genetics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Medical Genetics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Medical Genetics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Medical Genetics - # prompt tokens": "{\"description\": \"min=333.99, mean=333.99, max=333.99, sum=667.98 (2)\", \"tab\": \"General information\", \"score\": \"333.99\"}", - "Medical Genetics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"medical_genetics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_medical_genetics\"" - } - } - }, - { - "evaluation_name": "Miscellaneous", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Miscellaneous", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.831, - "details": { - "description": "min=0.831, mean=0.831, max=0.831, sum=1.663 (2)", - "tab": "Accuracy", - "Miscellaneous - Observed inference time (s)": "{\"description\": \"min=0.352, mean=0.352, max=0.352, sum=0.703 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3515638007971519\"}", - "Miscellaneous - # eval": "{\"description\": \"min=783, mean=783, max=783, sum=1566 (2)\", \"tab\": \"General information\", \"score\": \"783.0\"}", - "Miscellaneous - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Miscellaneous - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Miscellaneous - # prompt tokens": "{\"description\": \"min=292.911, mean=292.911, max=292.911, sum=585.821 (2)\", \"tab\": \"General information\", \"score\": \"292.9106002554278\"}", - "Miscellaneous - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"miscellaneous\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_miscellaneous\"" - } - } - }, - { - "evaluation_name": "Moral Scenarios", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Moral Scenarios", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.416, - "details": { - "description": "min=0.416, mean=0.416, max=0.416, sum=0.831 (2)", - "tab": "Accuracy", - "Moral Disputes - Observed inference time (s)": "{\"description\": \"min=0.293, mean=0.293, max=0.293, sum=0.585 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2926361808887107\"}", - "Moral Scenarios - Observed inference time (s)": "{\"description\": \"min=0.329, mean=0.329, max=0.329, sum=0.658 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3287937753027378\"}", - "Moral Disputes - # eval": "{\"description\": \"min=346, mean=346, max=346, sum=692 (2)\", \"tab\": \"General information\", \"score\": \"346.0\"}", - "Moral Disputes - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Disputes - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Disputes - # prompt tokens": "{\"description\": \"min=469.113, mean=469.113, max=469.113, sum=938.225 (2)\", \"tab\": \"General information\", \"score\": \"469.1127167630058\"}", - "Moral Disputes - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Moral Scenarios - # eval": "{\"description\": \"min=895, mean=895, max=895, sum=1790 (2)\", \"tab\": \"General information\", \"score\": \"895.0\"}", - "Moral Scenarios - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Scenarios - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # prompt tokens": "{\"description\": \"min=649.455, mean=649.455, max=649.455, sum=1298.909 (2)\", \"tab\": \"General information\", \"score\": \"649.454748603352\"}", - "Moral Scenarios - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"moral_scenarios\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_moral_scenarios\"" - } - } - }, - { - "evaluation_name": "Nutrition", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Nutrition", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.761, - "details": { - "description": "min=0.761, mean=0.761, max=0.761, sum=1.523 (2)", - "tab": "Accuracy", - "Nutrition - Observed inference time (s)": "{\"description\": \"min=0.323, mean=0.323, max=0.323, sum=0.645 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3226836241927801\"}", - "Nutrition - # eval": "{\"description\": \"min=306, mean=306, max=306, sum=612 (2)\", \"tab\": \"General information\", \"score\": \"306.0\"}", - "Nutrition - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Nutrition - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Nutrition - # prompt tokens": "{\"description\": \"min=579.814, mean=579.814, max=579.814, sum=1159.627 (2)\", \"tab\": \"General information\", \"score\": \"579.8137254901961\"}", - "Nutrition - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"nutrition\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_nutrition\"" - } - } - }, - { - "evaluation_name": "Prehistory", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Prehistory", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.738, - "details": { - "description": "min=0.738, mean=0.738, max=0.738, sum=1.475 (2)", - "tab": "Accuracy", - "Prehistory - Observed inference time (s)": "{\"description\": \"min=0.297, mean=0.297, max=0.297, sum=0.594 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2970340943630831\"}", - "Prehistory - # eval": "{\"description\": \"min=324, mean=324, max=324, sum=648 (2)\", \"tab\": \"General information\", \"score\": \"324.0\"}", - "Prehistory - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Prehistory - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Prehistory - # prompt tokens": "{\"description\": \"min=507.528, mean=507.528, max=507.528, sum=1015.056 (2)\", \"tab\": \"General information\", \"score\": \"507.52777777777777\"}", - "Prehistory - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"prehistory\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_prehistory\"" - } - } - }, - { - "evaluation_name": "Public Relations", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Public Relations", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.736, - "details": { - "description": "min=0.736, mean=0.736, max=0.736, sum=1.473 (2)", - "tab": "Accuracy", - "Public Relations - Observed inference time (s)": "{\"description\": \"min=0.325, mean=0.325, max=0.325, sum=0.649 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3247281486337835\"}", - "Public Relations - # eval": "{\"description\": \"min=110, mean=110, max=110, sum=220 (2)\", \"tab\": \"General information\", \"score\": \"110.0\"}", - "Public Relations - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Public Relations - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Public Relations - # prompt tokens": "{\"description\": \"min=398.318, mean=398.318, max=398.318, sum=796.636 (2)\", \"tab\": \"General information\", \"score\": \"398.3181818181818\"}", - "Public Relations - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"public_relations\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_public_relations\"" - } - } - }, - { - "evaluation_name": "Security Studies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Security Studies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.771, - "details": { - "description": "min=0.771, mean=0.771, max=0.771, sum=1.543 (2)", - "tab": "Accuracy", - "Security Studies - Observed inference time (s)": "{\"description\": \"min=0.351, mean=0.351, max=0.351, sum=0.702 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.35109225779163594\"}", - "Security Studies - # eval": "{\"description\": \"min=245, mean=245, max=245, sum=490 (2)\", \"tab\": \"General information\", \"score\": \"245.0\"}", - "Security Studies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Security Studies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Security Studies - # prompt tokens": "{\"description\": \"min=1157.473, mean=1157.473, max=1157.473, sum=2314.947 (2)\", \"tab\": \"General information\", \"score\": \"1157.4734693877551\"}", - "Security Studies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"security_studies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_security_studies\"" - } - } - }, - { - "evaluation_name": "Sociology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Sociology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.866, - "details": { - "description": "min=0.866, mean=0.866, max=0.866, sum=1.731 (2)", - "tab": "Accuracy", - "Sociology - Observed inference time (s)": "{\"description\": \"min=0.315, mean=0.315, max=0.315, sum=0.63 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.31481776545889933\"}", - "Sociology - # eval": "{\"description\": \"min=201, mean=201, max=201, sum=402 (2)\", \"tab\": \"General information\", \"score\": \"201.0\"}", - "Sociology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Sociology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Sociology - # prompt tokens": "{\"description\": \"min=438.517, mean=438.517, max=438.517, sum=877.035 (2)\", \"tab\": \"General information\", \"score\": \"438.51741293532336\"}", - "Sociology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"sociology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_sociology\"" - } - } - }, - { - "evaluation_name": "Virology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Virology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.566, - "details": { - "description": "min=0.566, mean=0.566, max=0.566, sum=1.133 (2)", - "tab": "Accuracy", - "Virology - Observed inference time (s)": "{\"description\": \"min=0.295, mean=0.295, max=0.295, sum=0.59 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2951422269085804\"}", - "Virology - # eval": "{\"description\": \"min=166, mean=166, max=166, sum=332 (2)\", \"tab\": \"General information\", \"score\": \"166.0\"}", - "Virology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Virology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Virology - # prompt tokens": "{\"description\": \"min=336.018, mean=336.018, max=336.018, sum=672.036 (2)\", \"tab\": \"General information\", \"score\": \"336.01807228915663\"}", - "Virology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"virology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_virology\"" - } - } - }, - { - "evaluation_name": "World Religions", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on World Religions", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.819, - "details": { - "description": "min=0.819, mean=0.819, max=0.819, sum=1.637 (2)", - "tab": "Accuracy", - "World Religions - Observed inference time (s)": "{\"description\": \"min=0.315, mean=0.315, max=0.315, sum=0.631 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3152559863196479\"}", - "World Religions - # eval": "{\"description\": \"min=171, mean=171, max=171, sum=342 (2)\", \"tab\": \"General information\", \"score\": \"171.0\"}", - "World Religions - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "World Religions - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "World Religions - # prompt tokens": "{\"description\": \"min=267.52, mean=267.52, max=267.52, sum=535.041 (2)\", \"tab\": \"General information\", \"score\": \"267.5204678362573\"}", - "World Religions - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"world_religions\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_world_religions\"" - } - } - }, - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.733, - "details": { - "description": "", - "tab": "Efficiency" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/meta_llama-3.1-405b-instruct-turbo.json b/data/models/meta_llama-3.1-405b-instruct-turbo.json deleted file mode 100644 index a187faa496a8e75f448c7a800443f493e4e82e00..0000000000000000000000000000000000000000 --- a/data/models/meta_llama-3.1-405b-instruct-turbo.json +++ /dev/null @@ -1,2127 +0,0 @@ -{ - "model_info": { - "name": "Llama 3.1 Instruct Turbo 405B", - "id": "meta/llama-3.1-405b-instruct-turbo", - "developer": "Meta", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_capabilities/meta_llama-3.1-405b-instruct-turbo/1774096304.5056698", - "retrieved_timestamp": "1774096304.5056698", - "source_metadata": { - "source_name": "helm_capabilities", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_capabilities", - "evaluation_results": [ - { - "evaluation_name": "Mean score", - "source_data": { - "dataset_name": "helm_capabilities", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "The mean of the scores from all columns.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.618, - "details": { - "description": "", - "tab": "Accuracy", - "Mean score - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"9.16102940672383\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU-Pro", - "source_data": { - "dataset_name": "MMLU-Pro", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on MMLU-Pro", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.723, - "details": { - "description": "min=0.723, mean=0.723, max=0.723, sum=0.723 (1)", - "tab": "Accuracy", - "MMLU-Pro - Observed inference time (s)": "{\"description\": \"min=5.795, mean=5.795, max=5.795, sum=5.795 (1)\", \"tab\": \"Efficiency\", \"score\": \"5.794888144493103\"}", - "MMLU-Pro - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "MMLU-Pro - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - # prompt tokens": "{\"description\": \"min=228.366, mean=228.366, max=228.366, sum=228.366 (1)\", \"tab\": \"General information\", \"score\": \"228.366\"}", - "MMLU-Pro - # output tokens": "{\"description\": \"min=376.289, mean=376.289, max=376.289, sum=376.289 (1)\", \"tab\": \"General information\", \"score\": \"376.289\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"all\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.522, - "details": { - "description": "min=0.522, mean=0.522, max=0.522, sum=0.522 (1)", - "tab": "Accuracy", - "GPQA - Observed inference time (s)": "{\"description\": \"min=9.197, mean=9.197, max=9.197, sum=9.197 (1)\", \"tab\": \"Efficiency\", \"score\": \"9.197324877362615\"}", - "GPQA - # eval": "{\"description\": \"min=446, mean=446, max=446, sum=446 (1)\", \"tab\": \"General information\", \"score\": \"446.0\"}", - "GPQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - # prompt tokens": "{\"description\": \"min=248.886, mean=248.886, max=248.886, sum=248.886 (1)\", \"tab\": \"General information\", \"score\": \"248.88565022421525\"}", - "GPQA - # output tokens": "{\"description\": \"min=592.928, mean=592.928, max=592.928, sum=592.928 (1)\", \"tab\": \"General information\", \"score\": \"592.9282511210762\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"gpqa_main\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "IFEval Strict Acc on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.811, - "details": { - "description": "min=0.811, mean=0.811, max=0.811, sum=0.811 (1)", - "tab": "Accuracy", - "IFEval - Observed inference time (s)": "{\"description\": \"min=4.572, mean=4.572, max=4.572, sum=4.572 (1)\", \"tab\": \"Efficiency\", \"score\": \"4.571529605692724\"}", - "IFEval - # eval": "{\"description\": \"min=541, mean=541, max=541, sum=541 (1)\", \"tab\": \"General information\", \"score\": \"541.0\"}", - "IFEval - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - # prompt tokens": "{\"description\": \"min=46.024, mean=46.024, max=46.024, sum=46.024 (1)\", \"tab\": \"General information\", \"score\": \"46.024029574861366\"}", - "IFEval - # output tokens": "{\"description\": \"min=358.067, mean=358.067, max=358.067, sum=358.067 (1)\", \"tab\": \"General information\", \"score\": \"358.06654343807764\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WildBench", - "source_data": { - "dataset_name": "WildBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "WB Score on WildBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.783, - "details": { - "description": "min=0.783, mean=0.783, max=0.783, sum=0.783 (1)", - "tab": "Accuracy", - "WildBench - Observed inference time (s)": "{\"description\": \"min=15.654, mean=15.654, max=15.654, sum=15.654 (1)\", \"tab\": \"Efficiency\", \"score\": \"15.653513952493668\"}", - "WildBench - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "WildBench - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # prompt tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # output tokens": "{\"description\": \"min=773.114, mean=773.114, max=773.114, sum=773.114 (1)\", \"tab\": \"General information\", \"score\": \"773.114\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"v2\"" - } - } - }, - { - "evaluation_name": "Omni-MATH", - "source_data": { - "dataset_name": "Omni-MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Acc on Omni-MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.249, - "details": { - "description": "min=0.249, mean=0.249, max=0.249, sum=0.249 (1)", - "tab": "Accuracy", - "Omni-MATH - Observed inference time (s)": "{\"description\": \"min=10.588, mean=10.588, max=10.588, sum=10.588 (1)\", \"tab\": \"Efficiency\", \"score\": \"10.587890453577042\"}", - "Omni-MATH - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "Omni-MATH - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - # prompt tokens": "{\"description\": \"min=109.708, mean=109.708, max=109.708, sum=109.708 (1)\", \"tab\": \"General information\", \"score\": \"109.708\"}", - "Omni-MATH - # output tokens": "{\"description\": \"min=906.902, mean=906.902, max=906.902, sum=906.902 (1)\", \"tab\": \"General information\", \"score\": \"906.902\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_lite/meta_llama-3.1-405b-instruct-turbo/1774096306.427425", - "retrieved_timestamp": "1774096306.427425", - "source_metadata": { - "source_name": "helm_lite", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_lite", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_lite", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.854, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.3095505617977528\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.749, - "details": { - "description": "min=0.749, mean=0.749, max=0.749, sum=0.749 (1)", - "tab": "Accuracy", - "NarrativeQA - Observed inference time (s)": "{\"description\": \"min=2.964, mean=2.964, max=2.964, sum=2.964 (1)\", \"tab\": \"Efficiency\", \"score\": \"2.964381891572979\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=3484.268, mean=3484.268, max=3484.268, sum=3484.268 (1)\", \"tab\": \"General information\", \"score\": \"3484.2676056338028\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=9.904, mean=9.904, max=9.904, sum=9.904 (1)\", \"tab\": \"General information\", \"score\": \"9.904225352112675\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (closed-book)", - "source_data": { - "dataset_name": "NaturalQuestions (closed-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (closed-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.456, - "details": { - "description": "min=0.456, mean=0.456, max=0.456, sum=0.456 (1)", - "tab": "Accuracy", - "NaturalQuestions (open-book) - Observed inference time (s)": "{\"description\": \"min=4.105, mean=4.105, max=4.105, sum=4.105 (1)\", \"tab\": \"Efficiency\", \"score\": \"4.104731038570404\"}", - "NaturalQuestions (closed-book) - Observed inference time (s)": "{\"description\": \"min=0.946, mean=0.946, max=0.946, sum=0.946 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.9464026074409485\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1716.78, mean=1716.78, max=1716.78, sum=1716.78 (1)\", \"tab\": \"General information\", \"score\": \"1716.78\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=8.741, mean=8.741, max=8.741, sum=8.741 (1)\", \"tab\": \"General information\", \"score\": \"8.741\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=129.12, mean=129.12, max=129.12, sum=129.12 (1)\", \"tab\": \"General information\", \"score\": \"129.12\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=8.576, mean=8.576, max=8.576, sum=8.576 (1)\", \"tab\": \"General information\", \"score\": \"8.576\"}" - } - }, - "generation_config": { - "additional_details": { - "mode": "\"closedbook\"" - } - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.94, - "details": { - "description": "min=0.94, mean=0.94, max=0.94, sum=0.94 (1)", - "tab": "Accuracy", - "OpenbookQA - Observed inference time (s)": "{\"description\": \"min=2.693, mean=2.693, max=2.693, sum=2.693 (1)\", \"tab\": \"Efficiency\", \"score\": \"2.6930377073287963\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=249.776, mean=249.776, max=249.776, sum=249.776 (1)\", \"tab\": \"General information\", \"score\": \"249.776\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "dataset": "\"openbookqa\"", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.759, - "details": { - "description": "min=0.6, mean=0.759, max=0.94, sum=3.796 (5)", - "tab": "Accuracy", - "MMLU - Observed inference time (s)": "{\"description\": \"min=0.464, mean=0.529, max=0.598, sum=2.643 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.528599283887629\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=373.43, mean=467.686, max=614.421, sum=2338.431 (5)\", \"tab\": \"General information\", \"score\": \"467.6862105263158\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"college_chemistry\", \"computer_security\", \"econometrics\", \"us_foreign_policy\"]", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MATH", - "source_data": { - "dataset_name": "MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Equivalent (CoT) on MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.827, - "details": { - "description": "min=0.635, mean=0.827, max=0.97, sum=5.789 (7)", - "tab": "Accuracy", - "MATH - Observed inference time (s)": "{\"description\": \"min=3.188, mean=4.118, max=4.906, sum=28.826 (7)\", \"tab\": \"Efficiency\", \"score\": \"4.117939187053165\"}", - "MATH - # eval": "{\"description\": \"min=30, mean=62.429, max=135, sum=437 (7)\", \"tab\": \"General information\", \"score\": \"62.42857142857143\"}", - "MATH - # train": "{\"description\": \"min=8, mean=8, max=8, sum=56 (7)\", \"tab\": \"General information\", \"score\": \"8.0\"}", - "MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MATH - # prompt tokens": "{\"description\": \"min=881.363, mean=1262.909, max=2197.577, sum=8840.364 (7)\", \"tab\": \"General information\", \"score\": \"1262.9092130545007\"}", - "MATH - # output tokens": "{\"description\": \"min=175.942, mean=232.698, max=270.904, sum=1628.884 (7)\", \"tab\": \"General information\", \"score\": \"232.69774473452566\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"algebra\", \"counting_and_probability\", \"geometry\", \"intermediate_algebra\", \"number_theory\", \"prealgebra\", \"precalculus\"]", - "level": "\"1\"", - "use_official_examples": "\"False\"", - "use_chain_of_thought": "\"True\"" - } - } - }, - { - "evaluation_name": "GSM8K", - "source_data": { - "dataset_name": "GSM8K", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on GSM8K", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.949, - "details": { - "description": "min=0.949, mean=0.949, max=0.949, sum=0.949 (1)", - "tab": "Accuracy", - "GSM8K - Observed inference time (s)": "{\"description\": \"min=2.737, mean=2.737, max=2.737, sum=2.737 (1)\", \"tab\": \"Efficiency\", \"score\": \"2.737115991592407\"}", - "GSM8K - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "GSM8K - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "GSM8K - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GSM8K - # prompt tokens": "{\"description\": \"min=959.032, mean=959.032, max=959.032, sum=959.032 (1)\", \"tab\": \"General information\", \"score\": \"959.032\"}", - "GSM8K - # output tokens": "{\"description\": \"min=122.777, mean=122.777, max=122.777, sum=122.777 (1)\", \"tab\": \"General information\", \"score\": \"122.777\"}" - } - }, - "generation_config": { - "additional_details": { - "stop": "\"none\"" - } - } - }, - { - "evaluation_name": "LegalBench", - "source_data": { - "dataset_name": "LegalBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on LegalBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.707, - "details": { - "description": "min=0.433, mean=0.707, max=0.979, sum=3.536 (5)", - "tab": "Accuracy", - "LegalBench - Observed inference time (s)": "{\"description\": \"min=0.492, mean=0.797, max=1.89, sum=3.987 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.7974352428433198\"}", - "LegalBench - # eval": "{\"description\": \"min=95, mean=409.4, max=1000, sum=2047 (5)\", \"tab\": \"General information\", \"score\": \"409.4\"}", - "LegalBench - # train": "{\"description\": \"min=4, mean=4.8, max=5, sum=24 (5)\", \"tab\": \"General information\", \"score\": \"4.8\"}", - "LegalBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "LegalBench - # prompt tokens": "{\"description\": \"min=197.442, mean=1513.882, max=6300.012, sum=7569.412 (5)\", \"tab\": \"General information\", \"score\": \"1513.8824197238912\"}", - "LegalBench - # output tokens": "{\"description\": \"min=2, mean=2.407, max=3, sum=12.035 (5)\", \"tab\": \"General information\", \"score\": \"2.4069553133514985\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"abercrombie\", \"corporate_lobbying\", \"function_of_decision_section\", \"international_citizenship_questions\", \"proa\"]" - } - } - }, - { - "evaluation_name": "MedQA", - "source_data": { - "dataset_name": "MedQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MedQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.805, - "details": { - "description": "min=0.805, mean=0.805, max=0.805, sum=0.805 (1)", - "tab": "Accuracy", - "MedQA - Observed inference time (s)": "{\"description\": \"min=0.951, mean=0.951, max=0.951, sum=0.951 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.9505775325577965\"}", - "MedQA - # eval": "{\"description\": \"min=503, mean=503, max=503, sum=503 (1)\", \"tab\": \"General information\", \"score\": \"503.0\"}", - "MedQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MedQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MedQA - # prompt tokens": "{\"description\": \"min=1025.274, mean=1025.274, max=1025.274, sum=1025.274 (1)\", \"tab\": \"General information\", \"score\": \"1025.2743538767395\"}", - "MedQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WMT 2014", - "source_data": { - "dataset_name": "WMT 2014", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "BLEU-4 on WMT 2014", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.238, - "details": { - "description": "min=0.2, mean=0.238, max=0.284, sum=1.191 (5)", - "tab": "Accuracy", - "WMT 2014 - Observed inference time (s)": "{\"description\": \"min=0.96, mean=1.055, max=1.147, sum=5.277 (5)\", \"tab\": \"Efficiency\", \"score\": \"1.0554436480227387\"}", - "WMT 2014 - # eval": "{\"description\": \"min=503, mean=568.8, max=832, sum=2844 (5)\", \"tab\": \"General information\", \"score\": \"568.8\"}", - "WMT 2014 - # train": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "WMT 2014 - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WMT 2014 - # prompt tokens": "{\"description\": \"min=101.139, mean=120.712, max=141.117, sum=603.559 (5)\", \"tab\": \"General information\", \"score\": \"120.71178123566294\"}", - "WMT 2014 - # output tokens": "{\"description\": \"min=24.598, mean=26.056, max=26.819, sum=130.279 (5)\", \"tab\": \"General information\", \"score\": \"26.055818454656674\"}" - } - }, - "generation_config": { - "additional_details": { - "language_pair": "[\"cs-en\", \"de-en\", \"fr-en\", \"hi-en\", \"ru-en\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_mmlu/meta_llama-3.1-405b-instruct-turbo/1774096312.00548", - "retrieved_timestamp": "1774096312.00548", - "source_metadata": { - "source_name": "helm_mmlu", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_mmlu", - "evaluation_results": [ - { - "evaluation_name": "MMLU All Subjects", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU All Subjects", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.845, - "details": { - "description": "min=0.572, mean=0.845, max=0.984, sum=96.366 (114)", - "tab": "Accuracy", - "MMLU All Subjects - Observed inference time (s)": "{\"description\": \"min=0.403, mean=0.685, max=1.366, sum=78.119 (114)\", \"tab\": \"Efficiency\", \"score\": \"0.6852569796494135\"}", - "MMLU All Subjects - # eval": "{\"description\": \"min=100, mean=246.351, max=1534, sum=28084 (114)\", \"tab\": \"General information\", \"score\": \"246.35087719298247\"}", - "MMLU All Subjects - # train": "{\"description\": \"min=5, mean=5, max=5, sum=570 (114)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU All Subjects - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU All Subjects - # prompt tokens": "{\"description\": \"min=274.52, mean=614.619, max=2797.885, sum=70066.61 (114)\", \"tab\": \"General information\", \"score\": \"614.6193817308517\"}", - "MMLU All Subjects - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=114 (114)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_name": "Abstract Algebra", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Abstract Algebra", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7, - "details": { - "description": "min=0.7, mean=0.7, max=0.7, sum=1.4 (2)", - "tab": "Accuracy", - "Abstract Algebra - Observed inference time (s)": "{\"description\": \"min=0.464, mean=0.464, max=0.464, sum=0.928 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4640246653556824\"}", - "Abstract Algebra - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Abstract Algebra - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Abstract Algebra - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Abstract Algebra - # prompt tokens": "{\"description\": \"min=373.43, mean=373.43, max=373.43, sum=746.86 (2)\", \"tab\": \"General information\", \"score\": \"373.43\"}", - "Abstract Algebra - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"abstract_algebra\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_abstract_algebra\"" - } - } - }, - { - "evaluation_name": "Anatomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Anatomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.822, - "details": { - "description": "min=0.822, mean=0.822, max=0.822, sum=1.644 (2)", - "tab": "Accuracy", - "Anatomy - Observed inference time (s)": "{\"description\": \"min=0.403, mean=0.403, max=0.403, sum=0.806 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4029027055810999\"}", - "Anatomy - # eval": "{\"description\": \"min=135, mean=135, max=135, sum=270 (2)\", \"tab\": \"General information\", \"score\": \"135.0\"}", - "Anatomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Anatomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Anatomy - # prompt tokens": "{\"description\": \"min=353.874, mean=353.874, max=353.874, sum=707.748 (2)\", \"tab\": \"General information\", \"score\": \"353.8740740740741\"}", - "Anatomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"anatomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_anatomy\"" - } - } - }, - { - "evaluation_name": "College Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on College Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.696, - "details": { - "description": "min=0.696, mean=0.696, max=0.696, sum=1.392 (2)", - "tab": "Accuracy", - "College Chemistry - Observed inference time (s)": "{\"description\": \"min=0.56, mean=0.56, max=0.56, sum=1.119 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5597123241424561\"}", - "College Biology - Observed inference time (s)": "{\"description\": \"min=0.48, mean=0.48, max=0.48, sum=0.959 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4795056896077262\"}", - "College Computer Science - Observed inference time (s)": "{\"description\": \"min=0.566, mean=0.566, max=0.566, sum=1.132 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5661771416664123\"}", - "College Mathematics - Observed inference time (s)": "{\"description\": \"min=0.541, mean=0.541, max=0.541, sum=1.082 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5411620163917541\"}", - "College Medicine - Observed inference time (s)": "{\"description\": \"min=0.635, mean=0.635, max=0.635, sum=1.271 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6352733904226667\"}", - "College Physics - Observed inference time (s)": "{\"description\": \"min=0.495, mean=0.495, max=0.495, sum=0.991 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4953400083616668\"}", - "College Chemistry - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Chemistry - # prompt tokens": "{\"description\": \"min=549.28, mean=549.28, max=549.28, sum=1098.56 (2)\", \"tab\": \"General information\", \"score\": \"549.28\"}", - "College Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Biology - # eval": "{\"description\": \"min=144, mean=144, max=144, sum=288 (2)\", \"tab\": \"General information\", \"score\": \"144.0\"}", - "College Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # prompt tokens": "{\"description\": \"min=473.875, mean=473.875, max=473.875, sum=947.75 (2)\", \"tab\": \"General information\", \"score\": \"473.875\"}", - "College Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # prompt tokens": "{\"description\": \"min=828.29, mean=828.29, max=828.29, sum=1656.58 (2)\", \"tab\": \"General information\", \"score\": \"828.29\"}", - "College Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Mathematics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # prompt tokens": "{\"description\": \"min=594.51, mean=594.51, max=594.51, sum=1189.02 (2)\", \"tab\": \"General information\", \"score\": \"594.51\"}", - "College Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Medicine - # eval": "{\"description\": \"min=173, mean=173, max=173, sum=346 (2)\", \"tab\": \"General information\", \"score\": \"173.0\"}", - "College Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # prompt tokens": "{\"description\": \"min=502.705, mean=502.705, max=502.705, sum=1005.41 (2)\", \"tab\": \"General information\", \"score\": \"502.70520231213874\"}", - "College Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Physics - # eval": "{\"description\": \"min=102, mean=102, max=102, sum=204 (2)\", \"tab\": \"General information\", \"score\": \"102.0\"}", - "College Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # prompt tokens": "{\"description\": \"min=503.569, mean=503.569, max=503.569, sum=1007.137 (2)\", \"tab\": \"General information\", \"score\": \"503.5686274509804\"}", - "College Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"college_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_college_physics\"" - } - } - }, - { - "evaluation_name": "Computer Security", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Computer Security", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.81, - "details": { - "description": "min=0.81, mean=0.81, max=0.81, sum=1.62 (2)", - "tab": "Accuracy", - "Computer Security - Observed inference time (s)": "{\"description\": \"min=0.502, mean=0.502, max=0.502, sum=1.003 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5016749453544617\"}", - "Computer Security - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Computer Security - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Computer Security - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Computer Security - # prompt tokens": "{\"description\": \"min=378.51, mean=378.51, max=378.51, sum=757.02 (2)\", \"tab\": \"General information\", \"score\": \"378.51\"}", - "Computer Security - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"computer_security\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_computer_security\"" - } - } - }, - { - "evaluation_name": "Econometrics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Econometrics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.746, - "details": { - "description": "min=0.746, mean=0.746, max=0.746, sum=1.491 (2)", - "tab": "Accuracy", - "Econometrics - Observed inference time (s)": "{\"description\": \"min=0.598, mean=0.598, max=0.598, sum=1.195 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5976439986312598\"}", - "Econometrics - # eval": "{\"description\": \"min=114, mean=114, max=114, sum=228 (2)\", \"tab\": \"General information\", \"score\": \"114.0\"}", - "Econometrics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Econometrics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Econometrics - # prompt tokens": "{\"description\": \"min=614.421, mean=614.421, max=614.421, sum=1228.842 (2)\", \"tab\": \"General information\", \"score\": \"614.421052631579\"}", - "Econometrics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"econometrics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_econometrics\"" - } - } - }, - { - "evaluation_name": "Global Facts", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Global Facts", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.71, - "details": { - "description": "min=0.71, mean=0.71, max=0.71, sum=1.42 (2)", - "tab": "Accuracy", - "Global Facts - Observed inference time (s)": "{\"description\": \"min=0.471, mean=0.471, max=0.471, sum=0.941 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4706212830543518\"}", - "Global Facts - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Global Facts - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Global Facts - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Global Facts - # prompt tokens": "{\"description\": \"min=399.71, mean=399.71, max=399.71, sum=799.42 (2)\", \"tab\": \"General information\", \"score\": \"399.71\"}", - "Global Facts - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"global_facts\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_global_facts\"" - } - } - }, - { - "evaluation_name": "Jurisprudence", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Jurisprudence", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.87, - "details": { - "description": "min=0.87, mean=0.87, max=0.87, sum=1.741 (2)", - "tab": "Accuracy", - "Jurisprudence - Observed inference time (s)": "{\"description\": \"min=0.917, mean=0.917, max=0.917, sum=1.835 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9174331603226838\"}", - "Jurisprudence - # eval": "{\"description\": \"min=108, mean=108, max=108, sum=216 (2)\", \"tab\": \"General information\", \"score\": \"108.0\"}", - "Jurisprudence - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Jurisprudence - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Jurisprudence - # prompt tokens": "{\"description\": \"min=394.63, mean=394.63, max=394.63, sum=789.259 (2)\", \"tab\": \"General information\", \"score\": \"394.6296296296296\"}", - "Jurisprudence - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"jurisprudence\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_jurisprudence\"" - } - } - }, - { - "evaluation_name": "Philosophy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Philosophy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.878, - "details": { - "description": "min=0.878, mean=0.878, max=0.878, sum=1.756 (2)", - "tab": "Accuracy", - "Philosophy - Observed inference time (s)": "{\"description\": \"min=0.753, mean=0.753, max=0.753, sum=1.506 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7531090411342608\"}", - "Philosophy - # eval": "{\"description\": \"min=311, mean=311, max=311, sum=622 (2)\", \"tab\": \"General information\", \"score\": \"311.0\"}", - "Philosophy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Philosophy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Philosophy - # prompt tokens": "{\"description\": \"min=329.084, mean=329.084, max=329.084, sum=658.167 (2)\", \"tab\": \"General information\", \"score\": \"329.08360128617363\"}", - "Philosophy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"philosophy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_philosophy\"" - } - } - }, - { - "evaluation_name": "Professional Psychology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Professional Psychology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.861, - "details": { - "description": "min=0.861, mean=0.861, max=0.861, sum=1.722 (2)", - "tab": "Accuracy", - "Professional Medicine - Observed inference time (s)": "{\"description\": \"min=1.095, mean=1.095, max=1.095, sum=2.191 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.0953595541855867\"}", - "Professional Accounting - Observed inference time (s)": "{\"description\": \"min=0.739, mean=0.739, max=0.739, sum=1.478 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7390724031637746\"}", - "Professional Law - Observed inference time (s)": "{\"description\": \"min=1.053, mean=1.053, max=1.053, sum=2.107 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.0534205999337087\"}", - "Professional Psychology - Observed inference time (s)": "{\"description\": \"min=0.579, mean=0.579, max=0.579, sum=1.158 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5791019481771132\"}", - "Professional Medicine - # eval": "{\"description\": \"min=272, mean=272, max=272, sum=544 (2)\", \"tab\": \"General information\", \"score\": \"272.0\"}", - "Professional Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Medicine - # prompt tokens": "{\"description\": \"min=1094.489, mean=1094.489, max=1094.489, sum=2188.978 (2)\", \"tab\": \"General information\", \"score\": \"1094.4889705882354\"}", - "Professional Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Accounting - # eval": "{\"description\": \"min=282, mean=282, max=282, sum=564 (2)\", \"tab\": \"General information\", \"score\": \"282.0\"}", - "Professional Accounting - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Accounting - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # prompt tokens": "{\"description\": \"min=658.585, mean=658.585, max=658.585, sum=1317.17 (2)\", \"tab\": \"General information\", \"score\": \"658.5851063829788\"}", - "Professional Accounting - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Law - # eval": "{\"description\": \"min=1534, mean=1534, max=1534, sum=3068 (2)\", \"tab\": \"General information\", \"score\": \"1534.0\"}", - "Professional Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # prompt tokens": "{\"description\": \"min=1637.601, mean=1637.601, max=1637.601, sum=3275.202 (2)\", \"tab\": \"General information\", \"score\": \"1637.6010430247718\"}", - "Professional Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Psychology - # eval": "{\"description\": \"min=612, mean=612, max=612, sum=1224 (2)\", \"tab\": \"General information\", \"score\": \"612.0\"}", - "Professional Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # prompt tokens": "{\"description\": \"min=575.098, mean=575.098, max=575.098, sum=1150.196 (2)\", \"tab\": \"General information\", \"score\": \"575.0980392156863\"}", - "Professional Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"professional_psychology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_professional_psychology\"" - } - } - }, - { - "evaluation_name": "Us Foreign Policy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Us Foreign Policy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.94, - "details": { - "description": "min=0.94, mean=0.94, max=0.94, sum=1.88 (2)", - "tab": "Accuracy", - "Us Foreign Policy - Observed inference time (s)": "{\"description\": \"min=0.52, mean=0.52, max=0.52, sum=1.04 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5199404859542847\"}", - "Us Foreign Policy - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Us Foreign Policy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Us Foreign Policy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Us Foreign Policy - # prompt tokens": "{\"description\": \"min=422.79, mean=422.79, max=422.79, sum=845.58 (2)\", \"tab\": \"General information\", \"score\": \"422.79\"}", - "Us Foreign Policy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"us_foreign_policy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_us_foreign_policy\"" - } - } - }, - { - "evaluation_name": "Astronomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Astronomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.921, - "details": { - "description": "min=0.921, mean=0.921, max=0.921, sum=1.842 (2)", - "tab": "Accuracy", - "Astronomy - Observed inference time (s)": "{\"description\": \"min=0.467, mean=0.467, max=0.467, sum=0.933 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.46656754769777\"}", - "Astronomy - # eval": "{\"description\": \"min=152, mean=152, max=152, sum=304 (2)\", \"tab\": \"General information\", \"score\": \"152.0\"}", - "Astronomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Astronomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Astronomy - # prompt tokens": "{\"description\": \"min=579.684, mean=579.684, max=579.684, sum=1159.368 (2)\", \"tab\": \"General information\", \"score\": \"579.6842105263158\"}", - "Astronomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"astronomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_astronomy\"" - } - } - }, - { - "evaluation_name": "Business Ethics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Business Ethics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.81, - "details": { - "description": "min=0.81, mean=0.81, max=0.81, sum=1.62 (2)", - "tab": "Accuracy", - "Business Ethics - Observed inference time (s)": "{\"description\": \"min=0.431, mean=0.431, max=0.431, sum=0.862 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4309411120414734\"}", - "Business Ethics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Business Ethics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Business Ethics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Business Ethics - # prompt tokens": "{\"description\": \"min=569.52, mean=569.52, max=569.52, sum=1139.04 (2)\", \"tab\": \"General information\", \"score\": \"569.52\"}", - "Business Ethics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"business_ethics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_business_ethics\"" - } - } - }, - { - "evaluation_name": "Clinical Knowledge", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Clinical Knowledge", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.879, - "details": { - "description": "min=0.879, mean=0.879, max=0.879, sum=1.758 (2)", - "tab": "Accuracy", - "Clinical Knowledge - Observed inference time (s)": "{\"description\": \"min=0.533, mean=0.533, max=0.533, sum=1.067 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5334792272099909\"}", - "Clinical Knowledge - # eval": "{\"description\": \"min=265, mean=265, max=265, sum=530 (2)\", \"tab\": \"General information\", \"score\": \"265.0\"}", - "Clinical Knowledge - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Clinical Knowledge - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Clinical Knowledge - # prompt tokens": "{\"description\": \"min=397.928, mean=397.928, max=397.928, sum=795.857 (2)\", \"tab\": \"General information\", \"score\": \"397.92830188679244\"}", - "Clinical Knowledge - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"clinical_knowledge\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_clinical_knowledge\"" - } - } - }, - { - "evaluation_name": "Conceptual Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Conceptual Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.877, - "details": { - "description": "min=0.877, mean=0.877, max=0.877, sum=1.753 (2)", - "tab": "Accuracy", - "Conceptual Physics - Observed inference time (s)": "{\"description\": \"min=0.508, mean=0.508, max=0.508, sum=1.016 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5081663547678197\"}", - "Conceptual Physics - # eval": "{\"description\": \"min=235, mean=235, max=235, sum=470 (2)\", \"tab\": \"General information\", \"score\": \"235.0\"}", - "Conceptual Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Conceptual Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Conceptual Physics - # prompt tokens": "{\"description\": \"min=304.834, mean=304.834, max=304.834, sum=609.668 (2)\", \"tab\": \"General information\", \"score\": \"304.83404255319147\"}", - "Conceptual Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"conceptual_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_conceptual_physics\"" - } - } - }, - { - "evaluation_name": "Electrical Engineering", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Electrical Engineering", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.821, - "details": { - "description": "min=0.821, mean=0.821, max=0.821, sum=1.641 (2)", - "tab": "Accuracy", - "Electrical Engineering - Observed inference time (s)": "{\"description\": \"min=0.48, mean=0.48, max=0.48, sum=0.959 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.47960921155995334\"}", - "Electrical Engineering - # eval": "{\"description\": \"min=145, mean=145, max=145, sum=290 (2)\", \"tab\": \"General information\", \"score\": \"145.0\"}", - "Electrical Engineering - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Electrical Engineering - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Electrical Engineering - # prompt tokens": "{\"description\": \"min=435.607, mean=435.607, max=435.607, sum=871.214 (2)\", \"tab\": \"General information\", \"score\": \"435.60689655172416\"}", - "Electrical Engineering - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"electrical_engineering\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_electrical_engineering\"" - } - } - }, - { - "evaluation_name": "Elementary Mathematics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Elementary Mathematics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.828, - "details": { - "description": "min=0.828, mean=0.828, max=0.828, sum=1.656 (2)", - "tab": "Accuracy", - "Elementary Mathematics - Observed inference time (s)": "{\"description\": \"min=0.559, mean=0.559, max=0.559, sum=1.117 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5586125358702645\"}", - "Elementary Mathematics - # eval": "{\"description\": \"min=378, mean=378, max=378, sum=756 (2)\", \"tab\": \"General information\", \"score\": \"378.0\"}", - "Elementary Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Elementary Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Elementary Mathematics - # prompt tokens": "{\"description\": \"min=531.854, mean=531.854, max=531.854, sum=1063.709 (2)\", \"tab\": \"General information\", \"score\": \"531.8544973544973\"}", - "Elementary Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"elementary_mathematics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_elementary_mathematics\"" - } - } - }, - { - "evaluation_name": "Formal Logic", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Formal Logic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.698, - "details": { - "description": "min=0.698, mean=0.698, max=0.698, sum=1.397 (2)", - "tab": "Accuracy", - "Formal Logic - Observed inference time (s)": "{\"description\": \"min=0.505, mean=0.505, max=0.505, sum=1.011 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5053695440292358\"}", - "Formal Logic - # eval": "{\"description\": \"min=126, mean=126, max=126, sum=252 (2)\", \"tab\": \"General information\", \"score\": \"126.0\"}", - "Formal Logic - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Formal Logic - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Formal Logic - # prompt tokens": "{\"description\": \"min=601.778, mean=601.778, max=601.778, sum=1203.556 (2)\", \"tab\": \"General information\", \"score\": \"601.7777777777778\"}", - "Formal Logic - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"formal_logic\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_formal_logic\"" - } - } - }, - { - "evaluation_name": "High School World History", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on High School World History", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.941, - "details": { - "description": "min=0.941, mean=0.941, max=0.941, sum=1.882 (2)", - "tab": "Accuracy", - "High School Biology - Observed inference time (s)": "{\"description\": \"min=0.487, mean=0.487, max=0.487, sum=0.974 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.48715837847801946\"}", - "High School Chemistry - Observed inference time (s)": "{\"description\": \"min=0.457, mean=0.457, max=0.457, sum=0.914 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.45692210949113216\"}", - "High School Computer Science - Observed inference time (s)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=1.334 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6668596768379211\"}", - "High School European History - Observed inference time (s)": "{\"description\": \"min=1.089, mean=1.089, max=1.089, sum=2.178 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.0890785202835545\"}", - "High School Geography - Observed inference time (s)": "{\"description\": \"min=0.491, mean=0.491, max=0.491, sum=0.983 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.49135766848169193\"}", - "High School Government And Politics - Observed inference time (s)": "{\"description\": \"min=0.437, mean=0.437, max=0.437, sum=0.874 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4368582340102122\"}", - "High School Macroeconomics - Observed inference time (s)": "{\"description\": \"min=0.612, mean=0.612, max=0.612, sum=1.224 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6121874619752933\"}", - "High School Mathematics - Observed inference time (s)": "{\"description\": \"min=0.929, mean=0.929, max=0.929, sum=1.858 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9291445193467317\"}", - "High School Microeconomics - Observed inference time (s)": "{\"description\": \"min=0.737, mean=0.737, max=0.737, sum=1.475 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7372911036515436\"}", - "High School Physics - Observed inference time (s)": "{\"description\": \"min=0.591, mean=0.591, max=0.591, sum=1.181 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5905803986732533\"}", - "High School Psychology - Observed inference time (s)": "{\"description\": \"min=0.884, mean=0.884, max=0.884, sum=1.767 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8837221084384743\"}", - "High School Statistics - Observed inference time (s)": "{\"description\": \"min=0.634, mean=0.634, max=0.634, sum=1.268 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6339434705398701\"}", - "High School US History - Observed inference time (s)": "{\"description\": \"min=0.993, mean=0.993, max=0.993, sum=1.987 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9934839302418279\"}", - "High School World History - Observed inference time (s)": "{\"description\": \"min=1.012, mean=1.012, max=1.012, sum=2.024 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.0120529253271562\"}", - "High School Biology - # eval": "{\"description\": \"min=310, mean=310, max=310, sum=620 (2)\", \"tab\": \"General information\", \"score\": \"310.0\"}", - "High School Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Biology - # prompt tokens": "{\"description\": \"min=513.671, mean=513.671, max=513.671, sum=1027.342 (2)\", \"tab\": \"General information\", \"score\": \"513.6709677419354\"}", - "High School Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Chemistry - # eval": "{\"description\": \"min=203, mean=203, max=203, sum=406 (2)\", \"tab\": \"General information\", \"score\": \"203.0\"}", - "High School Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # prompt tokens": "{\"description\": \"min=496.704, mean=496.704, max=496.704, sum=993.409 (2)\", \"tab\": \"General information\", \"score\": \"496.70443349753697\"}", - "High School Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "High School Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # prompt tokens": "{\"description\": \"min=867.78, mean=867.78, max=867.78, sum=1735.56 (2)\", \"tab\": \"General information\", \"score\": \"867.78\"}", - "High School Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School European History - # eval": "{\"description\": \"min=165, mean=165, max=165, sum=330 (2)\", \"tab\": \"General information\", \"score\": \"165.0\"}", - "High School European History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School European History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # prompt tokens": "{\"description\": \"min=2797.885, mean=2797.885, max=2797.885, sum=5595.77 (2)\", \"tab\": \"General information\", \"score\": \"2797.8848484848486\"}", - "High School European History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Geography - # eval": "{\"description\": \"min=198, mean=198, max=198, sum=396 (2)\", \"tab\": \"General information\", \"score\": \"198.0\"}", - "High School Geography - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Geography - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # prompt tokens": "{\"description\": \"min=372.035, mean=372.035, max=372.035, sum=744.071 (2)\", \"tab\": \"General information\", \"score\": \"372.0353535353535\"}", - "High School Geography - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Government And Politics - # eval": "{\"description\": \"min=193, mean=193, max=193, sum=386 (2)\", \"tab\": \"General information\", \"score\": \"193.0\"}", - "High School Government And Politics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Government And Politics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # prompt tokens": "{\"description\": \"min=465.824, mean=465.824, max=465.824, sum=931.648 (2)\", \"tab\": \"General information\", \"score\": \"465.8238341968912\"}", - "High School Government And Politics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Macroeconomics - # eval": "{\"description\": \"min=390, mean=390, max=390, sum=780 (2)\", \"tab\": \"General information\", \"score\": \"390.0\"}", - "High School Macroeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Macroeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # prompt tokens": "{\"description\": \"min=370.908, mean=370.908, max=370.908, sum=741.815 (2)\", \"tab\": \"General information\", \"score\": \"370.9076923076923\"}", - "High School Macroeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Mathematics - # eval": "{\"description\": \"min=270, mean=270, max=270, sum=540 (2)\", \"tab\": \"General information\", \"score\": \"270.0\"}", - "High School Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # prompt tokens": "{\"description\": \"min=532.356, mean=532.356, max=532.356, sum=1064.711 (2)\", \"tab\": \"General information\", \"score\": \"532.3555555555556\"}", - "High School Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Microeconomics - # eval": "{\"description\": \"min=238, mean=238, max=238, sum=476 (2)\", \"tab\": \"General information\", \"score\": \"238.0\"}", - "High School Microeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Microeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # prompt tokens": "{\"description\": \"min=399.013, mean=399.013, max=399.013, sum=798.025 (2)\", \"tab\": \"General information\", \"score\": \"399.0126050420168\"}", - "High School Microeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Physics - # eval": "{\"description\": \"min=151, mean=151, max=151, sum=302 (2)\", \"tab\": \"General information\", \"score\": \"151.0\"}", - "High School Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # prompt tokens": "{\"description\": \"min=560.457, mean=560.457, max=560.457, sum=1120.914 (2)\", \"tab\": \"General information\", \"score\": \"560.4569536423841\"}", - "High School Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Psychology - # eval": "{\"description\": \"min=545, mean=545, max=545, sum=1090 (2)\", \"tab\": \"General information\", \"score\": \"545.0\"}", - "High School Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # prompt tokens": "{\"description\": \"min=495.242, mean=495.242, max=495.242, sum=990.484 (2)\", \"tab\": \"General information\", \"score\": \"495.2422018348624\"}", - "High School Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Statistics - # eval": "{\"description\": \"min=216, mean=216, max=216, sum=432 (2)\", \"tab\": \"General information\", \"score\": \"216.0\"}", - "High School Statistics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Statistics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # prompt tokens": "{\"description\": \"min=795.639, mean=795.639, max=795.639, sum=1591.278 (2)\", \"tab\": \"General information\", \"score\": \"795.6388888888889\"}", - "High School Statistics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School US History - # eval": "{\"description\": \"min=204, mean=204, max=204, sum=408 (2)\", \"tab\": \"General information\", \"score\": \"204.0\"}", - "High School US History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School US History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # prompt tokens": "{\"description\": \"min=2217.809, mean=2217.809, max=2217.809, sum=4435.618 (2)\", \"tab\": \"General information\", \"score\": \"2217.8088235294117\"}", - "High School US History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School World History - # eval": "{\"description\": \"min=237, mean=237, max=237, sum=474 (2)\", \"tab\": \"General information\", \"score\": \"237.0\"}", - "High School World History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School World History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # prompt tokens": "{\"description\": \"min=1428.173, mean=1428.173, max=1428.173, sum=2856.346 (2)\", \"tab\": \"General information\", \"score\": \"1428.1729957805908\"}", - "High School World History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"high_school_world_history\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_high_school_world_history\"" - } - } - }, - { - "evaluation_name": "Human Sexuality", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Human Sexuality", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.855, - "details": { - "description": "min=0.855, mean=0.855, max=0.855, sum=1.71 (2)", - "tab": "Accuracy", - "Human Aging - Observed inference time (s)": "{\"description\": \"min=0.602, mean=0.602, max=0.602, sum=1.204 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6018790418257093\"}", - "Human Sexuality - Observed inference time (s)": "{\"description\": \"min=0.765, mean=0.765, max=0.765, sum=1.531 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7653163061797164\"}", - "Human Aging - # eval": "{\"description\": \"min=223, mean=223, max=223, sum=446 (2)\", \"tab\": \"General information\", \"score\": \"223.0\"}", - "Human Aging - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Aging - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Aging - # prompt tokens": "{\"description\": \"min=319.888, mean=319.888, max=319.888, sum=639.776 (2)\", \"tab\": \"General information\", \"score\": \"319.88789237668163\"}", - "Human Aging - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Human Sexuality - # eval": "{\"description\": \"min=131, mean=131, max=131, sum=262 (2)\", \"tab\": \"General information\", \"score\": \"131.0\"}", - "Human Sexuality - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Sexuality - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # prompt tokens": "{\"description\": \"min=341.168, mean=341.168, max=341.168, sum=682.336 (2)\", \"tab\": \"General information\", \"score\": \"341.1679389312977\"}", - "Human Sexuality - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"human_sexuality\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_human_sexuality\"" - } - } - }, - { - "evaluation_name": "International Law", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on International Law", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.95, - "details": { - "description": "min=0.95, mean=0.95, max=0.95, sum=1.901 (2)", - "tab": "Accuracy", - "International Law - Observed inference time (s)": "{\"description\": \"min=0.789, mean=0.789, max=0.789, sum=1.579 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7894663180201507\"}", - "International Law - # eval": "{\"description\": \"min=121, mean=121, max=121, sum=242 (2)\", \"tab\": \"General information\", \"score\": \"121.0\"}", - "International Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "International Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "International Law - # prompt tokens": "{\"description\": \"min=639.818, mean=639.818, max=639.818, sum=1279.636 (2)\", \"tab\": \"General information\", \"score\": \"639.8181818181819\"}", - "International Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"international_law\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_international_law\"" - } - } - }, - { - "evaluation_name": "Logical Fallacies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Logical Fallacies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.92, - "details": { - "description": "min=0.92, mean=0.92, max=0.92, sum=1.84 (2)", - "tab": "Accuracy", - "Logical Fallacies - Observed inference time (s)": "{\"description\": \"min=0.923, mean=0.923, max=0.923, sum=1.847 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9234680895425059\"}", - "Logical Fallacies - # eval": "{\"description\": \"min=163, mean=163, max=163, sum=326 (2)\", \"tab\": \"General information\", \"score\": \"163.0\"}", - "Logical Fallacies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Logical Fallacies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Logical Fallacies - # prompt tokens": "{\"description\": \"min=449.564, mean=449.564, max=449.564, sum=899.129 (2)\", \"tab\": \"General information\", \"score\": \"449.5644171779141\"}", - "Logical Fallacies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"logical_fallacies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_logical_fallacies\"" - } - } - }, - { - "evaluation_name": "Machine Learning", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Machine Learning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.795, - "details": { - "description": "min=0.795, mean=0.795, max=0.795, sum=1.589 (2)", - "tab": "Accuracy", - "Machine Learning - Observed inference time (s)": "{\"description\": \"min=1.077, mean=1.077, max=1.077, sum=2.154 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.0769924351147242\"}", - "Machine Learning - # eval": "{\"description\": \"min=112, mean=112, max=112, sum=224 (2)\", \"tab\": \"General information\", \"score\": \"112.0\"}", - "Machine Learning - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Machine Learning - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Machine Learning - # prompt tokens": "{\"description\": \"min=668.054, mean=668.054, max=668.054, sum=1336.107 (2)\", \"tab\": \"General information\", \"score\": \"668.0535714285714\"}", - "Machine Learning - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"machine_learning\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_machine_learning\"" - } - } - }, - { - "evaluation_name": "Management", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Management", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.893, - "details": { - "description": "min=0.893, mean=0.893, max=0.893, sum=1.786 (2)", - "tab": "Accuracy", - "Management - Observed inference time (s)": "{\"description\": \"min=0.505, mean=0.505, max=0.505, sum=1.009 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5047070956924586\"}", - "Management - # eval": "{\"description\": \"min=103, mean=103, max=103, sum=206 (2)\", \"tab\": \"General information\", \"score\": \"103.0\"}", - "Management - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Management - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Management - # prompt tokens": "{\"description\": \"min=283.786, mean=283.786, max=283.786, sum=567.573 (2)\", \"tab\": \"General information\", \"score\": \"283.7864077669903\"}", - "Management - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"management\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_management\"" - } - } - }, - { - "evaluation_name": "Marketing", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Marketing", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.962, - "details": { - "description": "min=0.962, mean=0.962, max=0.962, sum=1.923 (2)", - "tab": "Accuracy", - "Marketing - Observed inference time (s)": "{\"description\": \"min=0.617, mean=0.617, max=0.617, sum=1.234 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6168569010547084\"}", - "Marketing - # eval": "{\"description\": \"min=234, mean=234, max=234, sum=468 (2)\", \"tab\": \"General information\", \"score\": \"234.0\"}", - "Marketing - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Marketing - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Marketing - # prompt tokens": "{\"description\": \"min=404.218, mean=404.218, max=404.218, sum=808.436 (2)\", \"tab\": \"General information\", \"score\": \"404.21794871794873\"}", - "Marketing - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"marketing\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_marketing\"" - } - } - }, - { - "evaluation_name": "Medical Genetics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Medical Genetics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.93, - "details": { - "description": "min=0.93, mean=0.93, max=0.93, sum=1.86 (2)", - "tab": "Accuracy", - "Medical Genetics - Observed inference time (s)": "{\"description\": \"min=0.725, mean=0.725, max=0.725, sum=1.45 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7251019191741943\"}", - "Medical Genetics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Medical Genetics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Medical Genetics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Medical Genetics - # prompt tokens": "{\"description\": \"min=340.99, mean=340.99, max=340.99, sum=681.98 (2)\", \"tab\": \"General information\", \"score\": \"340.99\"}", - "Medical Genetics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"medical_genetics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_medical_genetics\"" - } - } - }, - { - "evaluation_name": "Miscellaneous", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Miscellaneous", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.939, - "details": { - "description": "min=0.939, mean=0.939, max=0.939, sum=1.877 (2)", - "tab": "Accuracy", - "Miscellaneous - Observed inference time (s)": "{\"description\": \"min=0.844, mean=0.844, max=0.844, sum=1.689 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8444620089208181\"}", - "Miscellaneous - # eval": "{\"description\": \"min=783, mean=783, max=783, sum=1566 (2)\", \"tab\": \"General information\", \"score\": \"783.0\"}", - "Miscellaneous - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Miscellaneous - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Miscellaneous - # prompt tokens": "{\"description\": \"min=299.911, mean=299.911, max=299.911, sum=599.821 (2)\", \"tab\": \"General information\", \"score\": \"299.9106002554278\"}", - "Miscellaneous - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"miscellaneous\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_miscellaneous\"" - } - } - }, - { - "evaluation_name": "Moral Scenarios", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Moral Scenarios", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.876, - "details": { - "description": "min=0.876, mean=0.876, max=0.876, sum=1.752 (2)", - "tab": "Accuracy", - "Moral Disputes - Observed inference time (s)": "{\"description\": \"min=1.366, mean=1.366, max=1.366, sum=2.732 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.3659538754148979\"}", - "Moral Scenarios - Observed inference time (s)": "{\"description\": \"min=0.749, mean=0.749, max=0.749, sum=1.498 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7492334496375569\"}", - "Moral Disputes - # eval": "{\"description\": \"min=346, mean=346, max=346, sum=692 (2)\", \"tab\": \"General information\", \"score\": \"346.0\"}", - "Moral Disputes - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Disputes - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Disputes - # prompt tokens": "{\"description\": \"min=476.113, mean=476.113, max=476.113, sum=952.225 (2)\", \"tab\": \"General information\", \"score\": \"476.1127167630058\"}", - "Moral Disputes - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Moral Scenarios - # eval": "{\"description\": \"min=895, mean=895, max=895, sum=1790 (2)\", \"tab\": \"General information\", \"score\": \"895.0\"}", - "Moral Scenarios - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Scenarios - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # prompt tokens": "{\"description\": \"min=656.455, mean=656.455, max=656.455, sum=1312.909 (2)\", \"tab\": \"General information\", \"score\": \"656.454748603352\"}", - "Moral Scenarios - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"moral_scenarios\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_moral_scenarios\"" - } - } - }, - { - "evaluation_name": "Nutrition", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Nutrition", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.928, - "details": { - "description": "min=0.928, mean=0.928, max=0.928, sum=1.856 (2)", - "tab": "Accuracy", - "Nutrition - Observed inference time (s)": "{\"description\": \"min=1.217, mean=1.217, max=1.217, sum=2.433 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.2165828491348067\"}", - "Nutrition - # eval": "{\"description\": \"min=306, mean=306, max=306, sum=612 (2)\", \"tab\": \"General information\", \"score\": \"306.0\"}", - "Nutrition - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Nutrition - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Nutrition - # prompt tokens": "{\"description\": \"min=586.814, mean=586.814, max=586.814, sum=1173.627 (2)\", \"tab\": \"General information\", \"score\": \"586.8137254901961\"}", - "Nutrition - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"nutrition\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_nutrition\"" - } - } - }, - { - "evaluation_name": "Prehistory", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Prehistory", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.929, - "details": { - "description": "min=0.929, mean=0.929, max=0.929, sum=1.858 (2)", - "tab": "Accuracy", - "Prehistory - Observed inference time (s)": "{\"description\": \"min=0.591, mean=0.591, max=0.591, sum=1.182 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5911465375511734\"}", - "Prehistory - # eval": "{\"description\": \"min=324, mean=324, max=324, sum=648 (2)\", \"tab\": \"General information\", \"score\": \"324.0\"}", - "Prehistory - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Prehistory - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Prehistory - # prompt tokens": "{\"description\": \"min=514.528, mean=514.528, max=514.528, sum=1029.056 (2)\", \"tab\": \"General information\", \"score\": \"514.5277777777778\"}", - "Prehistory - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"prehistory\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_prehistory\"" - } - } - }, - { - "evaluation_name": "Public Relations", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Public Relations", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.818, - "details": { - "description": "min=0.818, mean=0.818, max=0.818, sum=1.636 (2)", - "tab": "Accuracy", - "Public Relations - Observed inference time (s)": "{\"description\": \"min=1.129, mean=1.129, max=1.129, sum=2.258 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.12924514467066\"}", - "Public Relations - # eval": "{\"description\": \"min=110, mean=110, max=110, sum=220 (2)\", \"tab\": \"General information\", \"score\": \"110.0\"}", - "Public Relations - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Public Relations - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Public Relations - # prompt tokens": "{\"description\": \"min=405.318, mean=405.318, max=405.318, sum=810.636 (2)\", \"tab\": \"General information\", \"score\": \"405.3181818181818\"}", - "Public Relations - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"public_relations\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_public_relations\"" - } - } - }, - { - "evaluation_name": "Security Studies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Security Studies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.857, - "details": { - "description": "min=0.857, mean=0.857, max=0.857, sum=1.714 (2)", - "tab": "Accuracy", - "Security Studies - Observed inference time (s)": "{\"description\": \"min=0.734, mean=0.734, max=0.734, sum=1.468 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7342344303520358\"}", - "Security Studies - # eval": "{\"description\": \"min=245, mean=245, max=245, sum=490 (2)\", \"tab\": \"General information\", \"score\": \"245.0\"}", - "Security Studies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Security Studies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Security Studies - # prompt tokens": "{\"description\": \"min=1164.473, mean=1164.473, max=1164.473, sum=2328.947 (2)\", \"tab\": \"General information\", \"score\": \"1164.4734693877551\"}", - "Security Studies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"security_studies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_security_studies\"" - } - } - }, - { - "evaluation_name": "Sociology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Sociology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.94, - "details": { - "description": "min=0.94, mean=0.94, max=0.94, sum=1.881 (2)", - "tab": "Accuracy", - "Sociology - Observed inference time (s)": "{\"description\": \"min=0.583, mean=0.583, max=0.583, sum=1.166 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5830918010787585\"}", - "Sociology - # eval": "{\"description\": \"min=201, mean=201, max=201, sum=402 (2)\", \"tab\": \"General information\", \"score\": \"201.0\"}", - "Sociology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Sociology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Sociology - # prompt tokens": "{\"description\": \"min=445.517, mean=445.517, max=445.517, sum=891.035 (2)\", \"tab\": \"General information\", \"score\": \"445.51741293532336\"}", - "Sociology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"sociology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_sociology\"" - } - } - }, - { - "evaluation_name": "Virology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Virology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.572, - "details": { - "description": "min=0.572, mean=0.572, max=0.572, sum=1.145 (2)", - "tab": "Accuracy", - "Virology - Observed inference time (s)": "{\"description\": \"min=0.483, mean=0.483, max=0.483, sum=0.967 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4834072029734232\"}", - "Virology - # eval": "{\"description\": \"min=166, mean=166, max=166, sum=332 (2)\", \"tab\": \"General information\", \"score\": \"166.0\"}", - "Virology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Virology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Virology - # prompt tokens": "{\"description\": \"min=343.018, mean=343.018, max=343.018, sum=686.036 (2)\", \"tab\": \"General information\", \"score\": \"343.01807228915663\"}", - "Virology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"virology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_virology\"" - } - } - }, - { - "evaluation_name": "World Religions", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on World Religions", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.906, - "details": { - "description": "min=0.906, mean=0.906, max=0.906, sum=1.813 (2)", - "tab": "Accuracy", - "World Religions - Observed inference time (s)": "{\"description\": \"min=0.484, mean=0.484, max=0.484, sum=0.967 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.48364103328414826\"}", - "World Religions - # eval": "{\"description\": \"min=171, mean=171, max=171, sum=342 (2)\", \"tab\": \"General information\", \"score\": \"171.0\"}", - "World Religions - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "World Religions - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "World Religions - # prompt tokens": "{\"description\": \"min=274.52, mean=274.52, max=274.52, sum=549.041 (2)\", \"tab\": \"General information\", \"score\": \"274.5204678362573\"}", - "World Religions - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"world_religions\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_world_religions\"" - } - } - }, - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.33, - "details": { - "description": "", - "tab": "Efficiency" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/meta_llama-3.1-70b-instruct-turbo.json b/data/models/meta_llama-3.1-70b-instruct-turbo.json deleted file mode 100644 index 32f099f5d96ef4b9fa95536e6dbee13eabac329c..0000000000000000000000000000000000000000 --- a/data/models/meta_llama-3.1-70b-instruct-turbo.json +++ /dev/null @@ -1,2127 +0,0 @@ -{ - "model_info": { - "name": "Llama 3.1 Instruct Turbo 70B", - "id": "meta/llama-3.1-70b-instruct-turbo", - "developer": "Meta", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_capabilities/meta_llama-3.1-70b-instruct-turbo/1774096304.5056698", - "retrieved_timestamp": "1774096304.5056698", - "source_metadata": { - "source_name": "helm_capabilities", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_capabilities", - "evaluation_results": [ - { - "evaluation_name": "Mean score", - "source_data": { - "dataset_name": "helm_capabilities", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "The mean of the scores from all columns.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.574, - "details": { - "description": "", - "tab": "Accuracy", - "Mean score - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"4.2482479944372376\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU-Pro", - "source_data": { - "dataset_name": "MMLU-Pro", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on MMLU-Pro", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.653, - "details": { - "description": "min=0.653, mean=0.653, max=0.653, sum=0.653 (1)", - "tab": "Accuracy", - "MMLU-Pro - Observed inference time (s)": "{\"description\": \"min=2.732, mean=2.732, max=2.732, sum=2.732 (1)\", \"tab\": \"Efficiency\", \"score\": \"2.7317132804393767\"}", - "MMLU-Pro - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "MMLU-Pro - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - # prompt tokens": "{\"description\": \"min=228.366, mean=228.366, max=228.366, sum=228.366 (1)\", \"tab\": \"General information\", \"score\": \"228.366\"}", - "MMLU-Pro - # output tokens": "{\"description\": \"min=326.226, mean=326.226, max=326.226, sum=326.226 (1)\", \"tab\": \"General information\", \"score\": \"326.226\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"all\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.426, - "details": { - "description": "min=0.426, mean=0.426, max=0.426, sum=0.426 (1)", - "tab": "Accuracy", - "GPQA - Observed inference time (s)": "{\"description\": \"min=6.095, mean=6.095, max=6.095, sum=6.095 (1)\", \"tab\": \"Efficiency\", \"score\": \"6.0952357684550265\"}", - "GPQA - # eval": "{\"description\": \"min=446, mean=446, max=446, sum=446 (1)\", \"tab\": \"General information\", \"score\": \"446.0\"}", - "GPQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - # prompt tokens": "{\"description\": \"min=248.886, mean=248.886, max=248.886, sum=248.886 (1)\", \"tab\": \"General information\", \"score\": \"248.88565022421525\"}", - "GPQA - # output tokens": "{\"description\": \"min=491.435, mean=491.435, max=491.435, sum=491.435 (1)\", \"tab\": \"General information\", \"score\": \"491.43497757847535\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"gpqa_main\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "IFEval Strict Acc on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.821, - "details": { - "description": "min=0.821, mean=0.821, max=0.821, sum=0.821 (1)", - "tab": "Accuracy", - "IFEval - Observed inference time (s)": "{\"description\": \"min=2.622, mean=2.622, max=2.622, sum=2.622 (1)\", \"tab\": \"Efficiency\", \"score\": \"2.622214562350853\"}", - "IFEval - # eval": "{\"description\": \"min=541, mean=541, max=541, sum=541 (1)\", \"tab\": \"General information\", \"score\": \"541.0\"}", - "IFEval - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - # prompt tokens": "{\"description\": \"min=46.024, mean=46.024, max=46.024, sum=46.024 (1)\", \"tab\": \"General information\", \"score\": \"46.024029574861366\"}", - "IFEval - # output tokens": "{\"description\": \"min=361.464, mean=361.464, max=361.464, sum=361.464 (1)\", \"tab\": \"General information\", \"score\": \"361.46395563770795\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WildBench", - "source_data": { - "dataset_name": "WildBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "WB Score on WildBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.758, - "details": { - "description": "min=0.758, mean=0.758, max=0.758, sum=0.758 (1)", - "tab": "Accuracy", - "WildBench - Observed inference time (s)": "{\"description\": \"min=4.143, mean=4.143, max=4.143, sum=4.143 (1)\", \"tab\": \"Efficiency\", \"score\": \"4.142627255439758\"}", - "WildBench - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "WildBench - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # prompt tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # output tokens": "{\"description\": \"min=808.109, mean=808.109, max=808.109, sum=808.109 (1)\", \"tab\": \"General information\", \"score\": \"808.109\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"v2\"" - } - } - }, - { - "evaluation_name": "Omni-MATH", - "source_data": { - "dataset_name": "Omni-MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Acc on Omni-MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.21, - "details": { - "description": "min=0.21, mean=0.21, max=0.21, sum=0.21 (1)", - "tab": "Accuracy", - "Omni-MATH - Observed inference time (s)": "{\"description\": \"min=5.649, mean=5.649, max=5.649, sum=5.649 (1)\", \"tab\": \"Efficiency\", \"score\": \"5.649449105501175\"}", - "Omni-MATH - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "Omni-MATH - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - # prompt tokens": "{\"description\": \"min=109.708, mean=109.708, max=109.708, sum=109.708 (1)\", \"tab\": \"General information\", \"score\": \"109.708\"}", - "Omni-MATH - # output tokens": "{\"description\": \"min=1321.301, mean=1321.301, max=1321.301, sum=1321.301 (1)\", \"tab\": \"General information\", \"score\": \"1321.301\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_lite/meta_llama-3.1-70b-instruct-turbo/1774096306.427425", - "retrieved_timestamp": "1774096306.427425", - "source_metadata": { - "source_name": "helm_lite", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_lite", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_lite", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.808, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.133645443196005\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.772, - "details": { - "description": "min=0.772, mean=0.772, max=0.772, sum=0.772 (1)", - "tab": "Accuracy", - "NarrativeQA - Observed inference time (s)": "{\"description\": \"min=3.402, mean=3.402, max=3.402, sum=3.402 (1)\", \"tab\": \"Efficiency\", \"score\": \"3.4022000312805174\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=3484.268, mean=3484.268, max=3484.268, sum=3484.268 (1)\", \"tab\": \"General information\", \"score\": \"3484.2676056338028\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=9.034, mean=9.034, max=9.034, sum=9.034 (1)\", \"tab\": \"General information\", \"score\": \"9.033802816901408\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (closed-book)", - "source_data": { - "dataset_name": "NaturalQuestions (closed-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (closed-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.452, - "details": { - "description": "min=0.452, mean=0.452, max=0.452, sum=0.452 (1)", - "tab": "Accuracy", - "NaturalQuestions (open-book) - Observed inference time (s)": "{\"description\": \"min=3.354, mean=3.354, max=3.354, sum=3.354 (1)\", \"tab\": \"Efficiency\", \"score\": \"3.354476467370987\"}", - "NaturalQuestions (closed-book) - Observed inference time (s)": "{\"description\": \"min=3.534, mean=3.534, max=3.534, sum=3.534 (1)\", \"tab\": \"Efficiency\", \"score\": \"3.534221899032593\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1716.78, mean=1716.78, max=1716.78, sum=1716.78 (1)\", \"tab\": \"General information\", \"score\": \"1716.78\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=8.203, mean=8.203, max=8.203, sum=8.203 (1)\", \"tab\": \"General information\", \"score\": \"8.203\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=129.12, mean=129.12, max=129.12, sum=129.12 (1)\", \"tab\": \"General information\", \"score\": \"129.12\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=7.222, mean=7.222, max=7.222, sum=7.222 (1)\", \"tab\": \"General information\", \"score\": \"7.222\"}" - } - }, - "generation_config": { - "additional_details": { - "mode": "\"closedbook\"" - } - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.938, - "details": { - "description": "min=0.938, mean=0.938, max=0.938, sum=0.938 (1)", - "tab": "Accuracy", - "OpenbookQA - Observed inference time (s)": "{\"description\": \"min=3.875, mean=3.875, max=3.875, sum=3.875 (1)\", \"tab\": \"Efficiency\", \"score\": \"3.8750249314308167\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=249.776, mean=249.776, max=249.776, sum=249.776 (1)\", \"tab\": \"General information\", \"score\": \"249.776\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "dataset": "\"openbookqa\"", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.709, - "details": { - "description": "min=0.55, mean=0.709, max=0.93, sum=3.545 (5)", - "tab": "Accuracy", - "MMLU - Observed inference time (s)": "{\"description\": \"min=2.836, mean=12.026, max=45.251, sum=60.131 (5)\", \"tab\": \"Efficiency\", \"score\": \"12.026294649132511\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=373.43, mean=467.686, max=614.421, sum=2338.431 (5)\", \"tab\": \"General information\", \"score\": \"467.6862105263158\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"college_chemistry\", \"computer_security\", \"econometrics\", \"us_foreign_policy\"]", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MATH", - "source_data": { - "dataset_name": "MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Equivalent (CoT) on MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.783, - "details": { - "description": "min=0.579, mean=0.783, max=0.97, sum=5.483 (7)", - "tab": "Accuracy", - "MATH - Observed inference time (s)": "{\"description\": \"min=5.784, mean=6.527, max=7.228, sum=45.691 (7)\", \"tab\": \"Efficiency\", \"score\": \"6.527233472429779\"}", - "MATH - # eval": "{\"description\": \"min=30, mean=62.429, max=135, sum=437 (7)\", \"tab\": \"General information\", \"score\": \"62.42857142857143\"}", - "MATH - # train": "{\"description\": \"min=8, mean=8, max=8, sum=56 (7)\", \"tab\": \"General information\", \"score\": \"8.0\"}", - "MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MATH - # prompt tokens": "{\"description\": \"min=881.363, mean=1262.909, max=2197.577, sum=8840.364 (7)\", \"tab\": \"General information\", \"score\": \"1262.9092130545007\"}", - "MATH - # output tokens": "{\"description\": \"min=184.733, mean=243.368, max=279.105, sum=1703.574 (7)\", \"tab\": \"General information\", \"score\": \"243.36764411525732\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"algebra\", \"counting_and_probability\", \"geometry\", \"intermediate_algebra\", \"number_theory\", \"prealgebra\", \"precalculus\"]", - "level": "\"1\"", - "use_official_examples": "\"False\"", - "use_chain_of_thought": "\"True\"" - } - } - }, - { - "evaluation_name": "GSM8K", - "source_data": { - "dataset_name": "GSM8K", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on GSM8K", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.938, - "details": { - "description": "min=0.938, mean=0.938, max=0.938, sum=0.938 (1)", - "tab": "Accuracy", - "GSM8K - Observed inference time (s)": "{\"description\": \"min=4.99, mean=4.99, max=4.99, sum=4.99 (1)\", \"tab\": \"Efficiency\", \"score\": \"4.9902911036014554\"}", - "GSM8K - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "GSM8K - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "GSM8K - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GSM8K - # prompt tokens": "{\"description\": \"min=959.032, mean=959.032, max=959.032, sum=959.032 (1)\", \"tab\": \"General information\", \"score\": \"959.032\"}", - "GSM8K - # output tokens": "{\"description\": \"min=127.086, mean=127.086, max=127.086, sum=127.086 (1)\", \"tab\": \"General information\", \"score\": \"127.086\"}" - } - }, - "generation_config": { - "additional_details": { - "stop": "\"none\"" - } - } - }, - { - "evaluation_name": "LegalBench", - "source_data": { - "dataset_name": "LegalBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on LegalBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.687, - "details": { - "description": "min=0.439, mean=0.687, max=1, sum=3.433 (5)", - "tab": "Accuracy", - "LegalBench - Observed inference time (s)": "{\"description\": \"min=2.233, mean=3.171, max=3.636, sum=15.855 (5)\", \"tab\": \"Efficiency\", \"score\": \"3.1709040240543165\"}", - "LegalBench - # eval": "{\"description\": \"min=95, mean=409.4, max=1000, sum=2047 (5)\", \"tab\": \"General information\", \"score\": \"409.4\"}", - "LegalBench - # train": "{\"description\": \"min=4, mean=4.8, max=5, sum=24 (5)\", \"tab\": \"General information\", \"score\": \"4.8\"}", - "LegalBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "LegalBench - # prompt tokens": "{\"description\": \"min=197.442, mean=1513.882, max=6300.012, sum=7569.412 (5)\", \"tab\": \"General information\", \"score\": \"1513.8824197238912\"}", - "LegalBench - # output tokens": "{\"description\": \"min=2, mean=2.538, max=4.032, sum=12.688 (5)\", \"tab\": \"General information\", \"score\": \"2.5376711028251826\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"abercrombie\", \"corporate_lobbying\", \"function_of_decision_section\", \"international_citizenship_questions\", \"proa\"]" - } - } - }, - { - "evaluation_name": "MedQA", - "source_data": { - "dataset_name": "MedQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MedQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.769, - "details": { - "description": "min=0.769, mean=0.769, max=0.769, sum=0.769 (1)", - "tab": "Accuracy", - "MedQA - Observed inference time (s)": "{\"description\": \"min=3.053, mean=3.053, max=3.053, sum=3.053 (1)\", \"tab\": \"Efficiency\", \"score\": \"3.0525233205222704\"}", - "MedQA - # eval": "{\"description\": \"min=503, mean=503, max=503, sum=503 (1)\", \"tab\": \"General information\", \"score\": \"503.0\"}", - "MedQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MedQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MedQA - # prompt tokens": "{\"description\": \"min=1025.274, mean=1025.274, max=1025.274, sum=1025.274 (1)\", \"tab\": \"General information\", \"score\": \"1025.2743538767395\"}", - "MedQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WMT 2014", - "source_data": { - "dataset_name": "WMT 2014", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "BLEU-4 on WMT 2014", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.223, - "details": { - "description": "min=0.183, mean=0.223, max=0.265, sum=1.114 (5)", - "tab": "Accuracy", - "WMT 2014 - Observed inference time (s)": "{\"description\": \"min=0.762, mean=0.965, max=1.177, sum=4.824 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.9648550899177766\"}", - "WMT 2014 - # eval": "{\"description\": \"min=503, mean=568.8, max=832, sum=2844 (5)\", \"tab\": \"General information\", \"score\": \"568.8\"}", - "WMT 2014 - # train": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "WMT 2014 - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WMT 2014 - # prompt tokens": "{\"description\": \"min=101.139, mean=120.712, max=141.117, sum=603.559 (5)\", \"tab\": \"General information\", \"score\": \"120.71178123566294\"}", - "WMT 2014 - # output tokens": "{\"description\": \"min=24.231, mean=25.786, max=26.692, sum=128.928 (5)\", \"tab\": \"General information\", \"score\": \"25.78567441504817\"}" - } - }, - "generation_config": { - "additional_details": { - "language_pair": "[\"cs-en\", \"de-en\", \"fr-en\", \"hi-en\", \"ru-en\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_mmlu/meta_llama-3.1-70b-instruct-turbo/1774096312.00548", - "retrieved_timestamp": "1774096312.00548", - "source_metadata": { - "source_name": "helm_mmlu", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_mmlu", - "evaluation_results": [ - { - "evaluation_name": "MMLU All Subjects", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU All Subjects", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.801, - "details": { - "description": "min=0.404, mean=0.801, max=0.984, sum=91.318 (114)", - "tab": "Accuracy", - "MMLU All Subjects - Observed inference time (s)": "{\"description\": \"min=2.517, mean=5.993, max=45.251, sum=683.146 (114)\", \"tab\": \"Efficiency\", \"score\": \"5.992510112833335\"}", - "MMLU All Subjects - # eval": "{\"description\": \"min=100, mean=246.351, max=1534, sum=28084 (114)\", \"tab\": \"General information\", \"score\": \"246.35087719298247\"}", - "MMLU All Subjects - # train": "{\"description\": \"min=5, mean=5, max=5, sum=570 (114)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU All Subjects - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU All Subjects - # prompt tokens": "{\"description\": \"min=274.52, mean=614.619, max=2797.885, sum=70066.61 (114)\", \"tab\": \"General information\", \"score\": \"614.6193817308517\"}", - "MMLU All Subjects - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=114 (114)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_name": "Abstract Algebra", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Abstract Algebra", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.55, - "details": { - "description": "min=0.55, mean=0.55, max=0.55, sum=1.1 (2)", - "tab": "Accuracy", - "Abstract Algebra - Observed inference time (s)": "{\"description\": \"min=45.251, mean=45.251, max=45.251, sum=90.501 (2)\", \"tab\": \"Efficiency\", \"score\": \"45.250502264499666\"}", - "Abstract Algebra - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Abstract Algebra - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Abstract Algebra - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Abstract Algebra - # prompt tokens": "{\"description\": \"min=373.43, mean=373.43, max=373.43, sum=746.86 (2)\", \"tab\": \"General information\", \"score\": \"373.43\"}", - "Abstract Algebra - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"abstract_algebra\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_abstract_algebra\"" - } - } - }, - { - "evaluation_name": "Anatomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Anatomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8, - "details": { - "description": "min=0.8, mean=0.8, max=0.8, sum=1.6 (2)", - "tab": "Accuracy", - "Anatomy - Observed inference time (s)": "{\"description\": \"min=36.973, mean=36.973, max=36.973, sum=73.946 (2)\", \"tab\": \"Efficiency\", \"score\": \"36.97310272499367\"}", - "Anatomy - # eval": "{\"description\": \"min=135, mean=135, max=135, sum=270 (2)\", \"tab\": \"General information\", \"score\": \"135.0\"}", - "Anatomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Anatomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Anatomy - # prompt tokens": "{\"description\": \"min=353.874, mean=353.874, max=353.874, sum=707.748 (2)\", \"tab\": \"General information\", \"score\": \"353.8740740740741\"}", - "Anatomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"anatomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_anatomy\"" - } - } - }, - { - "evaluation_name": "College Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on College Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.559, - "details": { - "description": "min=0.559, mean=0.559, max=0.559, sum=1.118 (2)", - "tab": "Accuracy", - "College Chemistry - Observed inference time (s)": "{\"description\": \"min=4.774, mean=4.774, max=4.774, sum=9.548 (2)\", \"tab\": \"Efficiency\", \"score\": \"4.774094069004059\"}", - "College Biology - Observed inference time (s)": "{\"description\": \"min=4.993, mean=4.993, max=4.993, sum=9.986 (2)\", \"tab\": \"Efficiency\", \"score\": \"4.992929225166638\"}", - "College Computer Science - Observed inference time (s)": "{\"description\": \"min=4.499, mean=4.499, max=4.499, sum=8.999 (2)\", \"tab\": \"Efficiency\", \"score\": \"4.499426193237305\"}", - "College Mathematics - Observed inference time (s)": "{\"description\": \"min=4.479, mean=4.479, max=4.479, sum=8.957 (2)\", \"tab\": \"Efficiency\", \"score\": \"4.478512156009674\"}", - "College Medicine - Observed inference time (s)": "{\"description\": \"min=3.886, mean=3.886, max=3.886, sum=7.773 (2)\", \"tab\": \"Efficiency\", \"score\": \"3.886489330688653\"}", - "College Physics - Observed inference time (s)": "{\"description\": \"min=3.274, mean=3.274, max=3.274, sum=6.548 (2)\", \"tab\": \"Efficiency\", \"score\": \"3.2739863746306477\"}", - "College Chemistry - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Chemistry - # prompt tokens": "{\"description\": \"min=549.28, mean=549.28, max=549.28, sum=1098.56 (2)\", \"tab\": \"General information\", \"score\": \"549.28\"}", - "College Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Biology - # eval": "{\"description\": \"min=144, mean=144, max=144, sum=288 (2)\", \"tab\": \"General information\", \"score\": \"144.0\"}", - "College Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # prompt tokens": "{\"description\": \"min=473.875, mean=473.875, max=473.875, sum=947.75 (2)\", \"tab\": \"General information\", \"score\": \"473.875\"}", - "College Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # prompt tokens": "{\"description\": \"min=828.29, mean=828.29, max=828.29, sum=1656.58 (2)\", \"tab\": \"General information\", \"score\": \"828.29\"}", - "College Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Mathematics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # prompt tokens": "{\"description\": \"min=594.51, mean=594.51, max=594.51, sum=1189.02 (2)\", \"tab\": \"General information\", \"score\": \"594.51\"}", - "College Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Medicine - # eval": "{\"description\": \"min=173, mean=173, max=173, sum=346 (2)\", \"tab\": \"General information\", \"score\": \"173.0\"}", - "College Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # prompt tokens": "{\"description\": \"min=502.705, mean=502.705, max=502.705, sum=1005.41 (2)\", \"tab\": \"General information\", \"score\": \"502.70520231213874\"}", - "College Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Physics - # eval": "{\"description\": \"min=102, mean=102, max=102, sum=204 (2)\", \"tab\": \"General information\", \"score\": \"102.0\"}", - "College Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # prompt tokens": "{\"description\": \"min=503.569, mean=503.569, max=503.569, sum=1007.137 (2)\", \"tab\": \"General information\", \"score\": \"503.5686274509804\"}", - "College Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"college_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_college_physics\"" - } - } - }, - { - "evaluation_name": "Computer Security", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Computer Security", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8, - "details": { - "description": "min=0.8, mean=0.8, max=0.8, sum=1.6 (2)", - "tab": "Accuracy", - "Computer Security - Observed inference time (s)": "{\"description\": \"min=2.976, mean=2.976, max=2.976, sum=5.951 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.9756615567207336\"}", - "Computer Security - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Computer Security - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Computer Security - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Computer Security - # prompt tokens": "{\"description\": \"min=378.51, mean=378.51, max=378.51, sum=757.02 (2)\", \"tab\": \"General information\", \"score\": \"378.51\"}", - "Computer Security - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"computer_security\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_computer_security\"" - } - } - }, - { - "evaluation_name": "Econometrics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Econometrics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.675, - "details": { - "description": "min=0.675, mean=0.675, max=0.675, sum=1.351 (2)", - "tab": "Accuracy", - "Econometrics - Observed inference time (s)": "{\"description\": \"min=4.295, mean=4.295, max=4.295, sum=8.59 (2)\", \"tab\": \"Efficiency\", \"score\": \"4.29522921327959\"}", - "Econometrics - # eval": "{\"description\": \"min=114, mean=114, max=114, sum=228 (2)\", \"tab\": \"General information\", \"score\": \"114.0\"}", - "Econometrics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Econometrics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Econometrics - # prompt tokens": "{\"description\": \"min=614.421, mean=614.421, max=614.421, sum=1228.842 (2)\", \"tab\": \"General information\", \"score\": \"614.421052631579\"}", - "Econometrics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"econometrics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_econometrics\"" - } - } - }, - { - "evaluation_name": "Global Facts", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Global Facts", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.61, - "details": { - "description": "min=0.61, mean=0.61, max=0.61, sum=1.22 (2)", - "tab": "Accuracy", - "Global Facts - Observed inference time (s)": "{\"description\": \"min=3.637, mean=3.637, max=3.637, sum=7.275 (2)\", \"tab\": \"Efficiency\", \"score\": \"3.637417833805084\"}", - "Global Facts - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Global Facts - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Global Facts - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Global Facts - # prompt tokens": "{\"description\": \"min=399.71, mean=399.71, max=399.71, sum=799.42 (2)\", \"tab\": \"General information\", \"score\": \"399.71\"}", - "Global Facts - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"global_facts\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_global_facts\"" - } - } - }, - { - "evaluation_name": "Jurisprudence", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Jurisprudence", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.889, - "details": { - "description": "min=0.889, mean=0.889, max=0.889, sum=1.778 (2)", - "tab": "Accuracy", - "Jurisprudence - Observed inference time (s)": "{\"description\": \"min=3.163, mean=3.163, max=3.163, sum=6.326 (2)\", \"tab\": \"Efficiency\", \"score\": \"3.1630651178183378\"}", - "Jurisprudence - # eval": "{\"description\": \"min=108, mean=108, max=108, sum=216 (2)\", \"tab\": \"General information\", \"score\": \"108.0\"}", - "Jurisprudence - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Jurisprudence - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Jurisprudence - # prompt tokens": "{\"description\": \"min=394.63, mean=394.63, max=394.63, sum=789.259 (2)\", \"tab\": \"General information\", \"score\": \"394.6296296296296\"}", - "Jurisprudence - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"jurisprudence\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_jurisprudence\"" - } - } - }, - { - "evaluation_name": "Philosophy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Philosophy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.833, - "details": { - "description": "min=0.833, mean=0.833, max=0.833, sum=1.666 (2)", - "tab": "Accuracy", - "Philosophy - Observed inference time (s)": "{\"description\": \"min=3.264, mean=3.264, max=3.264, sum=6.527 (2)\", \"tab\": \"Efficiency\", \"score\": \"3.2637280957875143\"}", - "Philosophy - # eval": "{\"description\": \"min=311, mean=311, max=311, sum=622 (2)\", \"tab\": \"General information\", \"score\": \"311.0\"}", - "Philosophy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Philosophy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Philosophy - # prompt tokens": "{\"description\": \"min=329.084, mean=329.084, max=329.084, sum=658.167 (2)\", \"tab\": \"General information\", \"score\": \"329.08360128617363\"}", - "Philosophy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"philosophy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_philosophy\"" - } - } - }, - { - "evaluation_name": "Professional Psychology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Professional Psychology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.846, - "details": { - "description": "min=0.846, mean=0.846, max=0.846, sum=1.693 (2)", - "tab": "Accuracy", - "Professional Medicine - Observed inference time (s)": "{\"description\": \"min=3.871, mean=3.871, max=3.871, sum=7.742 (2)\", \"tab\": \"Efficiency\", \"score\": \"3.8712061214096405\"}", - "Professional Accounting - Observed inference time (s)": "{\"description\": \"min=2.943, mean=2.943, max=2.943, sum=5.886 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.9428400173254894\"}", - "Professional Law - Observed inference time (s)": "{\"description\": \"min=3.318, mean=3.318, max=3.318, sum=6.637 (2)\", \"tab\": \"Efficiency\", \"score\": \"3.318323635681978\"}", - "Professional Psychology - Observed inference time (s)": "{\"description\": \"min=3.102, mean=3.102, max=3.102, sum=6.203 (2)\", \"tab\": \"Efficiency\", \"score\": \"3.1015563872125416\"}", - "Professional Medicine - # eval": "{\"description\": \"min=272, mean=272, max=272, sum=544 (2)\", \"tab\": \"General information\", \"score\": \"272.0\"}", - "Professional Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Medicine - # prompt tokens": "{\"description\": \"min=1094.489, mean=1094.489, max=1094.489, sum=2188.978 (2)\", \"tab\": \"General information\", \"score\": \"1094.4889705882354\"}", - "Professional Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Accounting - # eval": "{\"description\": \"min=282, mean=282, max=282, sum=564 (2)\", \"tab\": \"General information\", \"score\": \"282.0\"}", - "Professional Accounting - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Accounting - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # prompt tokens": "{\"description\": \"min=658.585, mean=658.585, max=658.585, sum=1317.17 (2)\", \"tab\": \"General information\", \"score\": \"658.5851063829788\"}", - "Professional Accounting - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Law - # eval": "{\"description\": \"min=1534, mean=1534, max=1534, sum=3068 (2)\", \"tab\": \"General information\", \"score\": \"1534.0\"}", - "Professional Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # prompt tokens": "{\"description\": \"min=1637.601, mean=1637.601, max=1637.601, sum=3275.202 (2)\", \"tab\": \"General information\", \"score\": \"1637.6010430247718\"}", - "Professional Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Psychology - # eval": "{\"description\": \"min=612, mean=612, max=612, sum=1224 (2)\", \"tab\": \"General information\", \"score\": \"612.0\"}", - "Professional Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # prompt tokens": "{\"description\": \"min=575.098, mean=575.098, max=575.098, sum=1150.196 (2)\", \"tab\": \"General information\", \"score\": \"575.0980392156863\"}", - "Professional Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"professional_psychology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_professional_psychology\"" - } - } - }, - { - "evaluation_name": "Us Foreign Policy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Us Foreign Policy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.93, - "details": { - "description": "min=0.93, mean=0.93, max=0.93, sum=1.86 (2)", - "tab": "Accuracy", - "Us Foreign Policy - Observed inference time (s)": "{\"description\": \"min=2.836, mean=2.836, max=2.836, sum=5.672 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.835986142158508\"}", - "Us Foreign Policy - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Us Foreign Policy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Us Foreign Policy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Us Foreign Policy - # prompt tokens": "{\"description\": \"min=422.79, mean=422.79, max=422.79, sum=845.58 (2)\", \"tab\": \"General information\", \"score\": \"422.79\"}", - "Us Foreign Policy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"us_foreign_policy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_us_foreign_policy\"" - } - } - }, - { - "evaluation_name": "Astronomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Astronomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.908, - "details": { - "description": "min=0.908, mean=0.908, max=0.908, sum=1.816 (2)", - "tab": "Accuracy", - "Astronomy - Observed inference time (s)": "{\"description\": \"min=33.307, mean=33.307, max=33.307, sum=66.613 (2)\", \"tab\": \"Efficiency\", \"score\": \"33.3065683904447\"}", - "Astronomy - # eval": "{\"description\": \"min=152, mean=152, max=152, sum=304 (2)\", \"tab\": \"General information\", \"score\": \"152.0\"}", - "Astronomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Astronomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Astronomy - # prompt tokens": "{\"description\": \"min=579.684, mean=579.684, max=579.684, sum=1159.368 (2)\", \"tab\": \"General information\", \"score\": \"579.6842105263158\"}", - "Astronomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"astronomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_astronomy\"" - } - } - }, - { - "evaluation_name": "Business Ethics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Business Ethics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.72, - "details": { - "description": "min=0.72, mean=0.72, max=0.72, sum=1.44 (2)", - "tab": "Accuracy", - "Business Ethics - Observed inference time (s)": "{\"description\": \"min=34.272, mean=34.272, max=34.272, sum=68.544 (2)\", \"tab\": \"Efficiency\", \"score\": \"34.27190991640091\"}", - "Business Ethics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Business Ethics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Business Ethics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Business Ethics - # prompt tokens": "{\"description\": \"min=569.52, mean=569.52, max=569.52, sum=1139.04 (2)\", \"tab\": \"General information\", \"score\": \"569.52\"}", - "Business Ethics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"business_ethics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_business_ethics\"" - } - } - }, - { - "evaluation_name": "Clinical Knowledge", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Clinical Knowledge", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.845, - "details": { - "description": "min=0.845, mean=0.845, max=0.845, sum=1.691 (2)", - "tab": "Accuracy", - "Clinical Knowledge - Observed inference time (s)": "{\"description\": \"min=6.181, mean=6.181, max=6.181, sum=12.362 (2)\", \"tab\": \"Efficiency\", \"score\": \"6.18122723057585\"}", - "Clinical Knowledge - # eval": "{\"description\": \"min=265, mean=265, max=265, sum=530 (2)\", \"tab\": \"General information\", \"score\": \"265.0\"}", - "Clinical Knowledge - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Clinical Knowledge - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Clinical Knowledge - # prompt tokens": "{\"description\": \"min=397.928, mean=397.928, max=397.928, sum=795.857 (2)\", \"tab\": \"General information\", \"score\": \"397.92830188679244\"}", - "Clinical Knowledge - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"clinical_knowledge\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_clinical_knowledge\"" - } - } - }, - { - "evaluation_name": "Conceptual Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Conceptual Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.834, - "details": { - "description": "min=0.834, mean=0.834, max=0.834, sum=1.668 (2)", - "tab": "Accuracy", - "Conceptual Physics - Observed inference time (s)": "{\"description\": \"min=3.413, mean=3.413, max=3.413, sum=6.825 (2)\", \"tab\": \"Efficiency\", \"score\": \"3.412742525465945\"}", - "Conceptual Physics - # eval": "{\"description\": \"min=235, mean=235, max=235, sum=470 (2)\", \"tab\": \"General information\", \"score\": \"235.0\"}", - "Conceptual Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Conceptual Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Conceptual Physics - # prompt tokens": "{\"description\": \"min=304.834, mean=304.834, max=304.834, sum=609.668 (2)\", \"tab\": \"General information\", \"score\": \"304.83404255319147\"}", - "Conceptual Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"conceptual_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_conceptual_physics\"" - } - } - }, - { - "evaluation_name": "Electrical Engineering", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Electrical Engineering", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.745, - "details": { - "description": "min=0.745, mean=0.745, max=0.745, sum=1.49 (2)", - "tab": "Accuracy", - "Electrical Engineering - Observed inference time (s)": "{\"description\": \"min=4.146, mean=4.146, max=4.146, sum=8.292 (2)\", \"tab\": \"Efficiency\", \"score\": \"4.1461473415637835\"}", - "Electrical Engineering - # eval": "{\"description\": \"min=145, mean=145, max=145, sum=290 (2)\", \"tab\": \"General information\", \"score\": \"145.0\"}", - "Electrical Engineering - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Electrical Engineering - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Electrical Engineering - # prompt tokens": "{\"description\": \"min=435.607, mean=435.607, max=435.607, sum=871.214 (2)\", \"tab\": \"General information\", \"score\": \"435.60689655172416\"}", - "Electrical Engineering - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"electrical_engineering\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_electrical_engineering\"" - } - } - }, - { - "evaluation_name": "Elementary Mathematics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Elementary Mathematics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.701, - "details": { - "description": "min=0.701, mean=0.701, max=0.701, sum=1.402 (2)", - "tab": "Accuracy", - "Elementary Mathematics - Observed inference time (s)": "{\"description\": \"min=4.13, mean=4.13, max=4.13, sum=8.261 (2)\", \"tab\": \"Efficiency\", \"score\": \"4.1303687221789485\"}", - "Elementary Mathematics - # eval": "{\"description\": \"min=378, mean=378, max=378, sum=756 (2)\", \"tab\": \"General information\", \"score\": \"378.0\"}", - "Elementary Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Elementary Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Elementary Mathematics - # prompt tokens": "{\"description\": \"min=531.854, mean=531.854, max=531.854, sum=1063.709 (2)\", \"tab\": \"General information\", \"score\": \"531.8544973544973\"}", - "Elementary Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"elementary_mathematics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_elementary_mathematics\"" - } - } - }, - { - "evaluation_name": "Formal Logic", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Formal Logic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.675, - "details": { - "description": "min=0.675, mean=0.675, max=0.675, sum=1.349 (2)", - "tab": "Accuracy", - "Formal Logic - Observed inference time (s)": "{\"description\": \"min=3.65, mean=3.65, max=3.65, sum=7.301 (2)\", \"tab\": \"Efficiency\", \"score\": \"3.6502806383465964\"}", - "Formal Logic - # eval": "{\"description\": \"min=126, mean=126, max=126, sum=252 (2)\", \"tab\": \"General information\", \"score\": \"126.0\"}", - "Formal Logic - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Formal Logic - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Formal Logic - # prompt tokens": "{\"description\": \"min=601.778, mean=601.778, max=601.778, sum=1203.556 (2)\", \"tab\": \"General information\", \"score\": \"601.7777777777778\"}", - "Formal Logic - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"formal_logic\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_formal_logic\"" - } - } - }, - { - "evaluation_name": "High School World History", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on High School World History", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.937, - "details": { - "description": "min=0.937, mean=0.937, max=0.937, sum=1.873 (2)", - "tab": "Accuracy", - "High School Biology - Observed inference time (s)": "{\"description\": \"min=4.179, mean=4.179, max=4.179, sum=8.357 (2)\", \"tab\": \"Efficiency\", \"score\": \"4.178504861554792\"}", - "High School Chemistry - Observed inference time (s)": "{\"description\": \"min=3.78, mean=3.78, max=3.78, sum=7.56 (2)\", \"tab\": \"Efficiency\", \"score\": \"3.779934604766921\"}", - "High School Computer Science - Observed inference time (s)": "{\"description\": \"min=4.276, mean=4.276, max=4.276, sum=8.553 (2)\", \"tab\": \"Efficiency\", \"score\": \"4.276434569358826\"}", - "High School European History - Observed inference time (s)": "{\"description\": \"min=4.728, mean=4.728, max=4.728, sum=9.457 (2)\", \"tab\": \"Efficiency\", \"score\": \"4.7283261154637195\"}", - "High School Geography - Observed inference time (s)": "{\"description\": \"min=3.994, mean=3.994, max=3.994, sum=7.987 (2)\", \"tab\": \"Efficiency\", \"score\": \"3.993738304484974\"}", - "High School Government And Politics - Observed inference time (s)": "{\"description\": \"min=4.056, mean=4.056, max=4.056, sum=8.111 (2)\", \"tab\": \"Efficiency\", \"score\": \"4.055596974229566\"}", - "High School Macroeconomics - Observed inference time (s)": "{\"description\": \"min=4.06, mean=4.06, max=4.06, sum=8.12 (2)\", \"tab\": \"Efficiency\", \"score\": \"4.059808598420559\"}", - "High School Mathematics - Observed inference time (s)": "{\"description\": \"min=4.211, mean=4.211, max=4.211, sum=8.422 (2)\", \"tab\": \"Efficiency\", \"score\": \"4.210984716592011\"}", - "High School Microeconomics - Observed inference time (s)": "{\"description\": \"min=3.869, mean=3.869, max=3.869, sum=7.738 (2)\", \"tab\": \"Efficiency\", \"score\": \"3.8690204860783424\"}", - "High School Physics - Observed inference time (s)": "{\"description\": \"min=3.802, mean=3.802, max=3.802, sum=7.604 (2)\", \"tab\": \"Efficiency\", \"score\": \"3.801914532453019\"}", - "High School Psychology - Observed inference time (s)": "{\"description\": \"min=3.897, mean=3.897, max=3.897, sum=7.793 (2)\", \"tab\": \"Efficiency\", \"score\": \"3.8966542169588423\"}", - "High School Statistics - Observed inference time (s)": "{\"description\": \"min=3.5, mean=3.5, max=3.5, sum=6.999 (2)\", \"tab\": \"Efficiency\", \"score\": \"3.499593519502216\"}", - "High School US History - Observed inference time (s)": "{\"description\": \"min=3.948, mean=3.948, max=3.948, sum=7.897 (2)\", \"tab\": \"Efficiency\", \"score\": \"3.948316371908375\"}", - "High School World History - Observed inference time (s)": "{\"description\": \"min=3.316, mean=3.316, max=3.316, sum=6.632 (2)\", \"tab\": \"Efficiency\", \"score\": \"3.3161907819755974\"}", - "High School Biology - # eval": "{\"description\": \"min=310, mean=310, max=310, sum=620 (2)\", \"tab\": \"General information\", \"score\": \"310.0\"}", - "High School Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Biology - # prompt tokens": "{\"description\": \"min=513.671, mean=513.671, max=513.671, sum=1027.342 (2)\", \"tab\": \"General information\", \"score\": \"513.6709677419354\"}", - "High School Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Chemistry - # eval": "{\"description\": \"min=203, mean=203, max=203, sum=406 (2)\", \"tab\": \"General information\", \"score\": \"203.0\"}", - "High School Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # prompt tokens": "{\"description\": \"min=496.704, mean=496.704, max=496.704, sum=993.409 (2)\", \"tab\": \"General information\", \"score\": \"496.70443349753697\"}", - "High School Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "High School Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # prompt tokens": "{\"description\": \"min=867.78, mean=867.78, max=867.78, sum=1735.56 (2)\", \"tab\": \"General information\", \"score\": \"867.78\"}", - "High School Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School European History - # eval": "{\"description\": \"min=165, mean=165, max=165, sum=330 (2)\", \"tab\": \"General information\", \"score\": \"165.0\"}", - "High School European History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School European History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # prompt tokens": "{\"description\": \"min=2797.885, mean=2797.885, max=2797.885, sum=5595.77 (2)\", \"tab\": \"General information\", \"score\": \"2797.8848484848486\"}", - "High School European History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Geography - # eval": "{\"description\": \"min=198, mean=198, max=198, sum=396 (2)\", \"tab\": \"General information\", \"score\": \"198.0\"}", - "High School Geography - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Geography - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # prompt tokens": "{\"description\": \"min=372.035, mean=372.035, max=372.035, sum=744.071 (2)\", \"tab\": \"General information\", \"score\": \"372.0353535353535\"}", - "High School Geography - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Government And Politics - # eval": "{\"description\": \"min=193, mean=193, max=193, sum=386 (2)\", \"tab\": \"General information\", \"score\": \"193.0\"}", - "High School Government And Politics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Government And Politics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # prompt tokens": "{\"description\": \"min=465.824, mean=465.824, max=465.824, sum=931.648 (2)\", \"tab\": \"General information\", \"score\": \"465.8238341968912\"}", - "High School Government And Politics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Macroeconomics - # eval": "{\"description\": \"min=390, mean=390, max=390, sum=780 (2)\", \"tab\": \"General information\", \"score\": \"390.0\"}", - "High School Macroeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Macroeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # prompt tokens": "{\"description\": \"min=370.908, mean=370.908, max=370.908, sum=741.815 (2)\", \"tab\": \"General information\", \"score\": \"370.9076923076923\"}", - "High School Macroeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Mathematics - # eval": "{\"description\": \"min=270, mean=270, max=270, sum=540 (2)\", \"tab\": \"General information\", \"score\": \"270.0\"}", - "High School Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # prompt tokens": "{\"description\": \"min=532.356, mean=532.356, max=532.356, sum=1064.711 (2)\", \"tab\": \"General information\", \"score\": \"532.3555555555556\"}", - "High School Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Microeconomics - # eval": "{\"description\": \"min=238, mean=238, max=238, sum=476 (2)\", \"tab\": \"General information\", \"score\": \"238.0\"}", - "High School Microeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Microeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # prompt tokens": "{\"description\": \"min=399.013, mean=399.013, max=399.013, sum=798.025 (2)\", \"tab\": \"General information\", \"score\": \"399.0126050420168\"}", - "High School Microeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Physics - # eval": "{\"description\": \"min=151, mean=151, max=151, sum=302 (2)\", \"tab\": \"General information\", \"score\": \"151.0\"}", - "High School Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # prompt tokens": "{\"description\": \"min=560.457, mean=560.457, max=560.457, sum=1120.914 (2)\", \"tab\": \"General information\", \"score\": \"560.4569536423841\"}", - "High School Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Psychology - # eval": "{\"description\": \"min=545, mean=545, max=545, sum=1090 (2)\", \"tab\": \"General information\", \"score\": \"545.0\"}", - "High School Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # prompt tokens": "{\"description\": \"min=495.242, mean=495.242, max=495.242, sum=990.484 (2)\", \"tab\": \"General information\", \"score\": \"495.2422018348624\"}", - "High School Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Statistics - # eval": "{\"description\": \"min=216, mean=216, max=216, sum=432 (2)\", \"tab\": \"General information\", \"score\": \"216.0\"}", - "High School Statistics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Statistics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # prompt tokens": "{\"description\": \"min=795.639, mean=795.639, max=795.639, sum=1591.278 (2)\", \"tab\": \"General information\", \"score\": \"795.6388888888889\"}", - "High School Statistics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School US History - # eval": "{\"description\": \"min=204, mean=204, max=204, sum=408 (2)\", \"tab\": \"General information\", \"score\": \"204.0\"}", - "High School US History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School US History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # prompt tokens": "{\"description\": \"min=2217.809, mean=2217.809, max=2217.809, sum=4435.618 (2)\", \"tab\": \"General information\", \"score\": \"2217.8088235294117\"}", - "High School US History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School World History - # eval": "{\"description\": \"min=237, mean=237, max=237, sum=474 (2)\", \"tab\": \"General information\", \"score\": \"237.0\"}", - "High School World History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School World History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # prompt tokens": "{\"description\": \"min=1428.173, mean=1428.173, max=1428.173, sum=2856.346 (2)\", \"tab\": \"General information\", \"score\": \"1428.1729957805908\"}", - "High School World History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"high_school_world_history\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_high_school_world_history\"" - } - } - }, - { - "evaluation_name": "Human Sexuality", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Human Sexuality", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.855, - "details": { - "description": "min=0.855, mean=0.855, max=0.855, sum=1.71 (2)", - "tab": "Accuracy", - "Human Aging - Observed inference time (s)": "{\"description\": \"min=3.222, mean=3.222, max=3.222, sum=6.444 (2)\", \"tab\": \"Efficiency\", \"score\": \"3.2222468500180095\"}", - "Human Sexuality - Observed inference time (s)": "{\"description\": \"min=3.132, mean=3.132, max=3.132, sum=6.264 (2)\", \"tab\": \"Efficiency\", \"score\": \"3.1318228208381713\"}", - "Human Aging - # eval": "{\"description\": \"min=223, mean=223, max=223, sum=446 (2)\", \"tab\": \"General information\", \"score\": \"223.0\"}", - "Human Aging - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Aging - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Aging - # prompt tokens": "{\"description\": \"min=319.888, mean=319.888, max=319.888, sum=639.776 (2)\", \"tab\": \"General information\", \"score\": \"319.88789237668163\"}", - "Human Aging - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Human Sexuality - # eval": "{\"description\": \"min=131, mean=131, max=131, sum=262 (2)\", \"tab\": \"General information\", \"score\": \"131.0\"}", - "Human Sexuality - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Sexuality - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # prompt tokens": "{\"description\": \"min=341.168, mean=341.168, max=341.168, sum=682.336 (2)\", \"tab\": \"General information\", \"score\": \"341.1679389312977\"}", - "Human Sexuality - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"human_sexuality\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_human_sexuality\"" - } - } - }, - { - "evaluation_name": "International Law", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on International Law", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.926, - "details": { - "description": "min=0.926, mean=0.926, max=0.926, sum=1.851 (2)", - "tab": "Accuracy", - "International Law - Observed inference time (s)": "{\"description\": \"min=3.686, mean=3.686, max=3.686, sum=7.372 (2)\", \"tab\": \"Efficiency\", \"score\": \"3.68597848750343\"}", - "International Law - # eval": "{\"description\": \"min=121, mean=121, max=121, sum=242 (2)\", \"tab\": \"General information\", \"score\": \"121.0\"}", - "International Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "International Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "International Law - # prompt tokens": "{\"description\": \"min=639.818, mean=639.818, max=639.818, sum=1279.636 (2)\", \"tab\": \"General information\", \"score\": \"639.8181818181819\"}", - "International Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"international_law\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_international_law\"" - } - } - }, - { - "evaluation_name": "Logical Fallacies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Logical Fallacies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.84, - "details": { - "description": "min=0.84, mean=0.84, max=0.84, sum=1.681 (2)", - "tab": "Accuracy", - "Logical Fallacies - Observed inference time (s)": "{\"description\": \"min=2.835, mean=2.835, max=2.835, sum=5.67 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.834790670067255\"}", - "Logical Fallacies - # eval": "{\"description\": \"min=163, mean=163, max=163, sum=326 (2)\", \"tab\": \"General information\", \"score\": \"163.0\"}", - "Logical Fallacies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Logical Fallacies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Logical Fallacies - # prompt tokens": "{\"description\": \"min=449.564, mean=449.564, max=449.564, sum=899.129 (2)\", \"tab\": \"General information\", \"score\": \"449.5644171779141\"}", - "Logical Fallacies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"logical_fallacies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_logical_fallacies\"" - } - } - }, - { - "evaluation_name": "Machine Learning", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Machine Learning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.696, - "details": { - "description": "min=0.696, mean=0.696, max=0.696, sum=1.393 (2)", - "tab": "Accuracy", - "Machine Learning - Observed inference time (s)": "{\"description\": \"min=2.82, mean=2.82, max=2.82, sum=5.639 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.81969299699579\"}", - "Machine Learning - # eval": "{\"description\": \"min=112, mean=112, max=112, sum=224 (2)\", \"tab\": \"General information\", \"score\": \"112.0\"}", - "Machine Learning - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Machine Learning - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Machine Learning - # prompt tokens": "{\"description\": \"min=668.054, mean=668.054, max=668.054, sum=1336.107 (2)\", \"tab\": \"General information\", \"score\": \"668.0535714285714\"}", - "Machine Learning - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"machine_learning\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_machine_learning\"" - } - } - }, - { - "evaluation_name": "Management", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Management", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.913, - "details": { - "description": "min=0.913, mean=0.913, max=0.913, sum=1.825 (2)", - "tab": "Accuracy", - "Management - Observed inference time (s)": "{\"description\": \"min=2.909, mean=2.909, max=2.909, sum=5.818 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.9087865861874183\"}", - "Management - # eval": "{\"description\": \"min=103, mean=103, max=103, sum=206 (2)\", \"tab\": \"General information\", \"score\": \"103.0\"}", - "Management - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Management - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Management - # prompt tokens": "{\"description\": \"min=283.786, mean=283.786, max=283.786, sum=567.573 (2)\", \"tab\": \"General information\", \"score\": \"283.7864077669903\"}", - "Management - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"management\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_management\"" - } - } - }, - { - "evaluation_name": "Marketing", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Marketing", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.936, - "details": { - "description": "min=0.936, mean=0.936, max=0.936, sum=1.872 (2)", - "tab": "Accuracy", - "Marketing - Observed inference time (s)": "{\"description\": \"min=2.727, mean=2.727, max=2.727, sum=5.455 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.7273036078510122\"}", - "Marketing - # eval": "{\"description\": \"min=234, mean=234, max=234, sum=468 (2)\", \"tab\": \"General information\", \"score\": \"234.0\"}", - "Marketing - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Marketing - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Marketing - # prompt tokens": "{\"description\": \"min=404.218, mean=404.218, max=404.218, sum=808.436 (2)\", \"tab\": \"General information\", \"score\": \"404.21794871794873\"}", - "Marketing - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"marketing\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_marketing\"" - } - } - }, - { - "evaluation_name": "Medical Genetics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Medical Genetics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.93, - "details": { - "description": "min=0.93, mean=0.93, max=0.93, sum=1.86 (2)", - "tab": "Accuracy", - "Medical Genetics - Observed inference time (s)": "{\"description\": \"min=2.657, mean=2.657, max=2.657, sum=5.314 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.656917359828949\"}", - "Medical Genetics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Medical Genetics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Medical Genetics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Medical Genetics - # prompt tokens": "{\"description\": \"min=340.99, mean=340.99, max=340.99, sum=681.98 (2)\", \"tab\": \"General information\", \"score\": \"340.99\"}", - "Medical Genetics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"medical_genetics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_medical_genetics\"" - } - } - }, - { - "evaluation_name": "Miscellaneous", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Miscellaneous", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.913, - "details": { - "description": "min=0.913, mean=0.913, max=0.913, sum=1.826 (2)", - "tab": "Accuracy", - "Miscellaneous - Observed inference time (s)": "{\"description\": \"min=3.308, mean=3.308, max=3.308, sum=6.616 (2)\", \"tab\": \"Efficiency\", \"score\": \"3.3082146720715713\"}", - "Miscellaneous - # eval": "{\"description\": \"min=783, mean=783, max=783, sum=1566 (2)\", \"tab\": \"General information\", \"score\": \"783.0\"}", - "Miscellaneous - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Miscellaneous - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Miscellaneous - # prompt tokens": "{\"description\": \"min=299.911, mean=299.911, max=299.911, sum=599.821 (2)\", \"tab\": \"General information\", \"score\": \"299.9106002554278\"}", - "Miscellaneous - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"miscellaneous\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_miscellaneous\"" - } - } - }, - { - "evaluation_name": "Moral Scenarios", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Moral Scenarios", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.834, - "details": { - "description": "min=0.834, mean=0.834, max=0.834, sum=1.667 (2)", - "tab": "Accuracy", - "Moral Disputes - Observed inference time (s)": "{\"description\": \"min=2.926, mean=2.926, max=2.926, sum=5.852 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.9259741898906024\"}", - "Moral Scenarios - Observed inference time (s)": "{\"description\": \"min=3.608, mean=3.608, max=3.608, sum=7.216 (2)\", \"tab\": \"Efficiency\", \"score\": \"3.608134973248956\"}", - "Moral Disputes - # eval": "{\"description\": \"min=346, mean=346, max=346, sum=692 (2)\", \"tab\": \"General information\", \"score\": \"346.0\"}", - "Moral Disputes - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Disputes - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Disputes - # prompt tokens": "{\"description\": \"min=476.113, mean=476.113, max=476.113, sum=952.225 (2)\", \"tab\": \"General information\", \"score\": \"476.1127167630058\"}", - "Moral Disputes - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Moral Scenarios - # eval": "{\"description\": \"min=895, mean=895, max=895, sum=1790 (2)\", \"tab\": \"General information\", \"score\": \"895.0\"}", - "Moral Scenarios - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Scenarios - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # prompt tokens": "{\"description\": \"min=656.455, mean=656.455, max=656.455, sum=1312.909 (2)\", \"tab\": \"General information\", \"score\": \"656.454748603352\"}", - "Moral Scenarios - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"moral_scenarios\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_moral_scenarios\"" - } - } - }, - { - "evaluation_name": "Nutrition", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Nutrition", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.889, - "details": { - "description": "min=0.889, mean=0.889, max=0.889, sum=1.778 (2)", - "tab": "Accuracy", - "Nutrition - Observed inference time (s)": "{\"description\": \"min=3.56, mean=3.56, max=3.56, sum=7.12 (2)\", \"tab\": \"Efficiency\", \"score\": \"3.56020544089523\"}", - "Nutrition - # eval": "{\"description\": \"min=306, mean=306, max=306, sum=612 (2)\", \"tab\": \"General information\", \"score\": \"306.0\"}", - "Nutrition - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Nutrition - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Nutrition - # prompt tokens": "{\"description\": \"min=586.814, mean=586.814, max=586.814, sum=1173.627 (2)\", \"tab\": \"General information\", \"score\": \"586.8137254901961\"}", - "Nutrition - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"nutrition\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_nutrition\"" - } - } - }, - { - "evaluation_name": "Prehistory", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Prehistory", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.88, - "details": { - "description": "min=0.88, mean=0.88, max=0.88, sum=1.759 (2)", - "tab": "Accuracy", - "Prehistory - Observed inference time (s)": "{\"description\": \"min=3.546, mean=3.546, max=3.546, sum=7.091 (2)\", \"tab\": \"Efficiency\", \"score\": \"3.54565680247766\"}", - "Prehistory - # eval": "{\"description\": \"min=324, mean=324, max=324, sum=648 (2)\", \"tab\": \"General information\", \"score\": \"324.0\"}", - "Prehistory - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Prehistory - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Prehistory - # prompt tokens": "{\"description\": \"min=514.528, mean=514.528, max=514.528, sum=1029.056 (2)\", \"tab\": \"General information\", \"score\": \"514.5277777777778\"}", - "Prehistory - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"prehistory\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_prehistory\"" - } - } - }, - { - "evaluation_name": "Public Relations", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Public Relations", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.709, - "details": { - "description": "min=0.709, mean=0.709, max=0.709, sum=1.418 (2)", - "tab": "Accuracy", - "Public Relations - Observed inference time (s)": "{\"description\": \"min=3.03, mean=3.03, max=3.03, sum=6.06 (2)\", \"tab\": \"Efficiency\", \"score\": \"3.0301454305648803\"}", - "Public Relations - # eval": "{\"description\": \"min=110, mean=110, max=110, sum=220 (2)\", \"tab\": \"General information\", \"score\": \"110.0\"}", - "Public Relations - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Public Relations - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Public Relations - # prompt tokens": "{\"description\": \"min=405.318, mean=405.318, max=405.318, sum=810.636 (2)\", \"tab\": \"General information\", \"score\": \"405.3181818181818\"}", - "Public Relations - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"public_relations\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_public_relations\"" - } - } - }, - { - "evaluation_name": "Security Studies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Security Studies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.849, - "details": { - "description": "min=0.849, mean=0.849, max=0.849, sum=1.698 (2)", - "tab": "Accuracy", - "Security Studies - Observed inference time (s)": "{\"description\": \"min=2.949, mean=2.949, max=2.949, sum=5.898 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.948831728526524\"}", - "Security Studies - # eval": "{\"description\": \"min=245, mean=245, max=245, sum=490 (2)\", \"tab\": \"General information\", \"score\": \"245.0\"}", - "Security Studies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Security Studies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Security Studies - # prompt tokens": "{\"description\": \"min=1164.473, mean=1164.473, max=1164.473, sum=2328.947 (2)\", \"tab\": \"General information\", \"score\": \"1164.4734693877551\"}", - "Security Studies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"security_studies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_security_studies\"" - } - } - }, - { - "evaluation_name": "Sociology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Sociology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.92, - "details": { - "description": "min=0.92, mean=0.92, max=0.92, sum=1.841 (2)", - "tab": "Accuracy", - "Sociology - Observed inference time (s)": "{\"description\": \"min=2.843, mean=2.843, max=2.843, sum=5.686 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.842961254404552\"}", - "Sociology - # eval": "{\"description\": \"min=201, mean=201, max=201, sum=402 (2)\", \"tab\": \"General information\", \"score\": \"201.0\"}", - "Sociology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Sociology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Sociology - # prompt tokens": "{\"description\": \"min=445.517, mean=445.517, max=445.517, sum=891.035 (2)\", \"tab\": \"General information\", \"score\": \"445.51741293532336\"}", - "Sociology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"sociology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_sociology\"" - } - } - }, - { - "evaluation_name": "Virology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Virology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.578, - "details": { - "description": "min=0.578, mean=0.578, max=0.578, sum=1.157 (2)", - "tab": "Accuracy", - "Virology - Observed inference time (s)": "{\"description\": \"min=3.05, mean=3.05, max=3.05, sum=6.101 (2)\", \"tab\": \"Efficiency\", \"score\": \"3.050425999135856\"}", - "Virology - # eval": "{\"description\": \"min=166, mean=166, max=166, sum=332 (2)\", \"tab\": \"General information\", \"score\": \"166.0\"}", - "Virology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Virology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Virology - # prompt tokens": "{\"description\": \"min=343.018, mean=343.018, max=343.018, sum=686.036 (2)\", \"tab\": \"General information\", \"score\": \"343.01807228915663\"}", - "Virology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"virology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_virology\"" - } - } - }, - { - "evaluation_name": "World Religions", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on World Religions", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.895, - "details": { - "description": "min=0.895, mean=0.895, max=0.895, sum=1.789 (2)", - "tab": "Accuracy", - "World Religions - Observed inference time (s)": "{\"description\": \"min=2.517, mean=2.517, max=2.517, sum=5.033 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.5166666828400905\"}", - "World Religions - # eval": "{\"description\": \"min=171, mean=171, max=171, sum=342 (2)\", \"tab\": \"General information\", \"score\": \"171.0\"}", - "World Religions - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "World Religions - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "World Religions - # prompt tokens": "{\"description\": \"min=274.52, mean=274.52, max=274.52, sum=549.041 (2)\", \"tab\": \"General information\", \"score\": \"274.5204678362573\"}", - "World Religions - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"world_religions\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_world_religions\"" - } - } - }, - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.021, - "details": { - "description": "", - "tab": "Efficiency" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/meta_llama-3.1-8b-instruct-turbo.json b/data/models/meta_llama-3.1-8b-instruct-turbo.json deleted file mode 100644 index 511e540608c036dc4147fb2f225109769aa099a1..0000000000000000000000000000000000000000 --- a/data/models/meta_llama-3.1-8b-instruct-turbo.json +++ /dev/null @@ -1,2127 +0,0 @@ -{ - "model_info": { - "name": "Llama 3.1 Instruct Turbo 8B", - "id": "meta/llama-3.1-8b-instruct-turbo", - "developer": "Meta", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_capabilities/meta_llama-3.1-8b-instruct-turbo/1774096304.5056698", - "retrieved_timestamp": "1774096304.5056698", - "source_metadata": { - "source_name": "helm_capabilities", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_capabilities", - "evaluation_results": [ - { - "evaluation_name": "Mean score", - "source_data": { - "dataset_name": "helm_capabilities", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "The mean of the scores from all columns.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.444, - "details": { - "description": "", - "tab": "Accuracy", - "Mean score - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"3.654367387500005\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU-Pro", - "source_data": { - "dataset_name": "MMLU-Pro", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on MMLU-Pro", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.406, - "details": { - "description": "min=0.406, mean=0.406, max=0.406, sum=0.406 (1)", - "tab": "Accuracy", - "MMLU-Pro - Observed inference time (s)": "{\"description\": \"min=2.642, mean=2.642, max=2.642, sum=2.642 (1)\", \"tab\": \"Efficiency\", \"score\": \"2.6422129917144774\"}", - "MMLU-Pro - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "MMLU-Pro - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - # prompt tokens": "{\"description\": \"min=228.366, mean=228.366, max=228.366, sum=228.366 (1)\", \"tab\": \"General information\", \"score\": \"228.366\"}", - "MMLU-Pro - # output tokens": "{\"description\": \"min=518.387, mean=518.387, max=518.387, sum=518.387 (1)\", \"tab\": \"General information\", \"score\": \"518.387\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"all\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.247, - "details": { - "description": "min=0.247, mean=0.247, max=0.247, sum=0.247 (1)", - "tab": "Accuracy", - "GPQA - Observed inference time (s)": "{\"description\": \"min=3.28, mean=3.28, max=3.28, sum=3.28 (1)\", \"tab\": \"Efficiency\", \"score\": \"3.2803654104070277\"}", - "GPQA - # eval": "{\"description\": \"min=446, mean=446, max=446, sum=446 (1)\", \"tab\": \"General information\", \"score\": \"446.0\"}", - "GPQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - # prompt tokens": "{\"description\": \"min=248.886, mean=248.886, max=248.886, sum=248.886 (1)\", \"tab\": \"General information\", \"score\": \"248.88565022421525\"}", - "GPQA - # output tokens": "{\"description\": \"min=744.583, mean=744.583, max=744.583, sum=744.583 (1)\", \"tab\": \"General information\", \"score\": \"744.5829596412556\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"gpqa_main\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "IFEval Strict Acc on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.743, - "details": { - "description": "min=0.743, mean=0.743, max=0.743, sum=0.743 (1)", - "tab": "Accuracy", - "IFEval - Observed inference time (s)": "{\"description\": \"min=1.982, mean=1.982, max=1.982, sum=1.982 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.981573561423367\"}", - "IFEval - # eval": "{\"description\": \"min=541, mean=541, max=541, sum=541 (1)\", \"tab\": \"General information\", \"score\": \"541.0\"}", - "IFEval - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - # prompt tokens": "{\"description\": \"min=46.024, mean=46.024, max=46.024, sum=46.024 (1)\", \"tab\": \"General information\", \"score\": \"46.024029574861366\"}", - "IFEval - # output tokens": "{\"description\": \"min=404.026, mean=404.026, max=404.026, sum=404.026 (1)\", \"tab\": \"General information\", \"score\": \"404.02587800369685\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WildBench", - "source_data": { - "dataset_name": "WildBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "WB Score on WildBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.686, - "details": { - "description": "min=0.686, mean=0.686, max=0.686, sum=0.686 (1)", - "tab": "Accuracy", - "WildBench - Observed inference time (s)": "{\"description\": \"min=3.192, mean=3.192, max=3.192, sum=3.192 (1)\", \"tab\": \"Efficiency\", \"score\": \"3.1917312424182893\"}", - "WildBench - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "WildBench - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # prompt tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # output tokens": "{\"description\": \"min=865.484, mean=865.484, max=865.484, sum=865.484 (1)\", \"tab\": \"General information\", \"score\": \"865.484\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"v2\"" - } - } - }, - { - "evaluation_name": "Omni-MATH", - "source_data": { - "dataset_name": "Omni-MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Acc on Omni-MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.137, - "details": { - "description": "min=0.137, mean=0.137, max=0.137, sum=0.137 (1)", - "tab": "Accuracy", - "Omni-MATH - Observed inference time (s)": "{\"description\": \"min=7.176, mean=7.176, max=7.176, sum=7.176 (1)\", \"tab\": \"Efficiency\", \"score\": \"7.1759537315368656\"}", - "Omni-MATH - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "Omni-MATH - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - # prompt tokens": "{\"description\": \"min=109.708, mean=109.708, max=109.708, sum=109.708 (1)\", \"tab\": \"General information\", \"score\": \"109.708\"}", - "Omni-MATH - # output tokens": "{\"description\": \"min=2170.057, mean=2170.057, max=2170.057, sum=2170.057 (1)\", \"tab\": \"General information\", \"score\": \"2170.057\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_lite/meta_llama-3.1-8b-instruct-turbo/1774096306.427425", - "retrieved_timestamp": "1774096306.427425", - "source_metadata": { - "source_name": "helm_lite", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_lite", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_lite", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.303, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.5896504369538077\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.756, - "details": { - "description": "min=0.756, mean=0.756, max=0.756, sum=0.756 (1)", - "tab": "Accuracy", - "NarrativeQA - Observed inference time (s)": "{\"description\": \"min=0.581, mean=0.581, max=0.581, sum=0.581 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.5813529316808136\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=3484.268, mean=3484.268, max=3484.268, sum=3484.268 (1)\", \"tab\": \"General information\", \"score\": \"3484.2676056338028\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=7.287, mean=7.287, max=7.287, sum=7.287 (1)\", \"tab\": \"General information\", \"score\": \"7.2873239436619714\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (closed-book)", - "source_data": { - "dataset_name": "NaturalQuestions (closed-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (closed-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.209, - "details": { - "description": "min=0.209, mean=0.209, max=0.209, sum=0.209 (1)", - "tab": "Accuracy", - "NaturalQuestions (open-book) - Observed inference time (s)": "{\"description\": \"min=0.544, mean=0.544, max=0.544, sum=0.544 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.5441543731689453\"}", - "NaturalQuestions (closed-book) - Observed inference time (s)": "{\"description\": \"min=0.752, mean=0.752, max=0.752, sum=0.752 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.751717613697052\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1716.78, mean=1716.78, max=1716.78, sum=1716.78 (1)\", \"tab\": \"General information\", \"score\": \"1716.78\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=8.736, mean=8.736, max=8.736, sum=8.736 (1)\", \"tab\": \"General information\", \"score\": \"8.736\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=129.12, mean=129.12, max=129.12, sum=129.12 (1)\", \"tab\": \"General information\", \"score\": \"129.12\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=11.732, mean=11.732, max=11.732, sum=11.732 (1)\", \"tab\": \"General information\", \"score\": \"11.732\"}" - } - }, - "generation_config": { - "additional_details": { - "mode": "\"closedbook\"" - } - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.74, - "details": { - "description": "min=0.74, mean=0.74, max=0.74, sum=0.74 (1)", - "tab": "Accuracy", - "OpenbookQA - Observed inference time (s)": "{\"description\": \"min=2.937, mean=2.937, max=2.937, sum=2.937 (1)\", \"tab\": \"Efficiency\", \"score\": \"2.9374450149536133\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=249.776, mean=249.776, max=249.776, sum=249.776 (1)\", \"tab\": \"General information\", \"score\": \"249.776\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "dataset": "\"openbookqa\"", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5, - "details": { - "description": "min=0.26, mean=0.5, max=0.79, sum=2.501 (5)", - "tab": "Accuracy", - "MMLU - Observed inference time (s)": "{\"description\": \"min=0.284, mean=0.417, max=0.567, sum=2.086 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.41729471965421716\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=373.43, mean=467.686, max=614.421, sum=2338.431 (5)\", \"tab\": \"General information\", \"score\": \"467.6862105263158\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"college_chemistry\", \"computer_security\", \"econometrics\", \"us_foreign_policy\"]", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MATH", - "source_data": { - "dataset_name": "MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Equivalent (CoT) on MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.703, - "details": { - "description": "min=0.509, mean=0.703, max=0.849, sum=4.92 (7)", - "tab": "Accuracy", - "MATH - Observed inference time (s)": "{\"description\": \"min=1.617, mean=1.927, max=2.175, sum=13.492 (7)\", \"tab\": \"Efficiency\", \"score\": \"1.9274194573191807\"}", - "MATH - # eval": "{\"description\": \"min=30, mean=62.429, max=135, sum=437 (7)\", \"tab\": \"General information\", \"score\": \"62.42857142857143\"}", - "MATH - # train": "{\"description\": \"min=8, mean=8, max=8, sum=56 (7)\", \"tab\": \"General information\", \"score\": \"8.0\"}", - "MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MATH - # prompt tokens": "{\"description\": \"min=881.363, mean=1262.909, max=2197.577, sum=8840.364 (7)\", \"tab\": \"General information\", \"score\": \"1262.9092130545007\"}", - "MATH - # output tokens": "{\"description\": \"min=203.384, mean=253.982, max=288.596, sum=1777.872 (7)\", \"tab\": \"General information\", \"score\": \"253.98170179473732\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"algebra\", \"counting_and_probability\", \"geometry\", \"intermediate_algebra\", \"number_theory\", \"prealgebra\", \"precalculus\"]", - "level": "\"1\"", - "use_official_examples": "\"False\"", - "use_chain_of_thought": "\"True\"" - } - } - }, - { - "evaluation_name": "GSM8K", - "source_data": { - "dataset_name": "GSM8K", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on GSM8K", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.798, - "details": { - "description": "min=0.798, mean=0.798, max=0.798, sum=0.798 (1)", - "tab": "Accuracy", - "GSM8K - Observed inference time (s)": "{\"description\": \"min=2.109, mean=2.109, max=2.109, sum=2.109 (1)\", \"tab\": \"Efficiency\", \"score\": \"2.108796592712402\"}", - "GSM8K - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "GSM8K - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "GSM8K - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GSM8K - # prompt tokens": "{\"description\": \"min=959.032, mean=959.032, max=959.032, sum=959.032 (1)\", \"tab\": \"General information\", \"score\": \"959.032\"}", - "GSM8K - # output tokens": "{\"description\": \"min=150.02, mean=150.02, max=150.02, sum=150.02 (1)\", \"tab\": \"General information\", \"score\": \"150.02\"}" - } - }, - "generation_config": { - "additional_details": { - "stop": "\"none\"" - } - } - }, - { - "evaluation_name": "LegalBench", - "source_data": { - "dataset_name": "LegalBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on LegalBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.342, - "details": { - "description": "min=0, mean=0.342, max=0.8, sum=1.71 (5)", - "tab": "Accuracy", - "LegalBench - Observed inference time (s)": "{\"description\": \"min=0.409, mean=0.481, max=0.626, sum=2.407 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.4814103188942614\"}", - "LegalBench - # eval": "{\"description\": \"min=95, mean=409.4, max=1000, sum=2047 (5)\", \"tab\": \"General information\", \"score\": \"409.4\"}", - "LegalBench - # train": "{\"description\": \"min=4, mean=4.8, max=5, sum=24 (5)\", \"tab\": \"General information\", \"score\": \"4.8\"}", - "LegalBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "LegalBench - # prompt tokens": "{\"description\": \"min=197.442, mean=1513.882, max=6300.012, sum=7569.412 (5)\", \"tab\": \"General information\", \"score\": \"1513.8824197238912\"}", - "LegalBench - # output tokens": "{\"description\": \"min=2.032, mean=6.824, max=10.886, sum=34.118 (5)\", \"tab\": \"General information\", \"score\": \"6.823557876005701\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"abercrombie\", \"corporate_lobbying\", \"function_of_decision_section\", \"international_citizenship_questions\", \"proa\"]" - } - } - }, - { - "evaluation_name": "MedQA", - "source_data": { - "dataset_name": "MedQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MedQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.245, - "details": { - "description": "min=0.245, mean=0.245, max=0.245, sum=0.245 (1)", - "tab": "Accuracy", - "MedQA - Observed inference time (s)": "{\"description\": \"min=0.743, mean=0.743, max=0.743, sum=0.743 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.742541556803891\"}", - "MedQA - # eval": "{\"description\": \"min=503, mean=503, max=503, sum=503 (1)\", \"tab\": \"General information\", \"score\": \"503.0\"}", - "MedQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MedQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MedQA - # prompt tokens": "{\"description\": \"min=1025.274, mean=1025.274, max=1025.274, sum=1025.274 (1)\", \"tab\": \"General information\", \"score\": \"1025.2743538767395\"}", - "MedQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WMT 2014", - "source_data": { - "dataset_name": "WMT 2014", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "BLEU-4 on WMT 2014", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.181, - "details": { - "description": "min=0.132, mean=0.181, max=0.219, sum=0.907 (5)", - "tab": "Accuracy", - "WMT 2014 - Observed inference time (s)": "{\"description\": \"min=0.439, mean=0.565, max=0.727, sum=2.826 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.5651802479746801\"}", - "WMT 2014 - # eval": "{\"description\": \"min=503, mean=568.8, max=832, sum=2844 (5)\", \"tab\": \"General information\", \"score\": \"568.8\"}", - "WMT 2014 - # train": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "WMT 2014 - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WMT 2014 - # prompt tokens": "{\"description\": \"min=101.139, mean=120.712, max=141.117, sum=603.559 (5)\", \"tab\": \"General information\", \"score\": \"120.71178123566294\"}", - "WMT 2014 - # output tokens": "{\"description\": \"min=24.354, mean=25.779, max=26.833, sum=128.893 (5)\", \"tab\": \"General information\", \"score\": \"25.778561802263347\"}" - } - }, - "generation_config": { - "additional_details": { - "language_pair": "[\"cs-en\", \"de-en\", \"fr-en\", \"hi-en\", \"ru-en\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_mmlu/meta_llama-3.1-8b-instruct-turbo/1774096312.00548", - "retrieved_timestamp": "1774096312.00548", - "source_metadata": { - "source_name": "helm_mmlu", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_mmlu", - "evaluation_results": [ - { - "evaluation_name": "MMLU All Subjects", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU All Subjects", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.561, - "details": { - "description": "min=0.26, mean=0.561, max=0.865, sum=63.912 (114)", - "tab": "Accuracy", - "MMLU All Subjects - Observed inference time (s)": "{\"description\": \"min=0.202, mean=0.56, max=1.485, sum=63.854 (114)\", \"tab\": \"Efficiency\", \"score\": \"0.5601251981506405\"}", - "MMLU All Subjects - # eval": "{\"description\": \"min=100, mean=246.351, max=1534, sum=28084 (114)\", \"tab\": \"General information\", \"score\": \"246.35087719298247\"}", - "MMLU All Subjects - # train": "{\"description\": \"min=5, mean=5, max=5, sum=570 (114)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU All Subjects - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU All Subjects - # prompt tokens": "{\"description\": \"min=274.52, mean=614.619, max=2797.885, sum=70066.61 (114)\", \"tab\": \"General information\", \"score\": \"614.6193817308517\"}", - "MMLU All Subjects - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=114 (114)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_name": "Abstract Algebra", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Abstract Algebra", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.26, - "details": { - "description": "min=0.26, mean=0.26, max=0.26, sum=0.52 (2)", - "tab": "Accuracy", - "Abstract Algebra - Observed inference time (s)": "{\"description\": \"min=0.284, mean=0.284, max=0.284, sum=0.568 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.28381933450698854\"}", - "Abstract Algebra - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Abstract Algebra - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Abstract Algebra - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Abstract Algebra - # prompt tokens": "{\"description\": \"min=373.43, mean=373.43, max=373.43, sum=746.86 (2)\", \"tab\": \"General information\", \"score\": \"373.43\"}", - "Abstract Algebra - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"abstract_algebra\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_abstract_algebra\"" - } - } - }, - { - "evaluation_name": "Anatomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Anatomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.459, - "details": { - "description": "min=0.459, mean=0.459, max=0.459, sum=0.919 (2)", - "tab": "Accuracy", - "Anatomy - Observed inference time (s)": "{\"description\": \"min=0.323, mean=0.323, max=0.323, sum=0.646 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3231998196354619\"}", - "Anatomy - # eval": "{\"description\": \"min=135, mean=135, max=135, sum=270 (2)\", \"tab\": \"General information\", \"score\": \"135.0\"}", - "Anatomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Anatomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Anatomy - # prompt tokens": "{\"description\": \"min=353.874, mean=353.874, max=353.874, sum=707.748 (2)\", \"tab\": \"General information\", \"score\": \"353.8740740740741\"}", - "Anatomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"anatomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_anatomy\"" - } - } - }, - { - "evaluation_name": "College Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on College Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.363, - "details": { - "description": "min=0.363, mean=0.363, max=0.363, sum=0.725 (2)", - "tab": "Accuracy", - "College Chemistry - Observed inference time (s)": "{\"description\": \"min=0.431, mean=0.431, max=0.431, sum=0.862 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.43078258752822873\"}", - "College Biology - Observed inference time (s)": "{\"description\": \"min=0.426, mean=0.426, max=0.426, sum=0.853 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.42637243535783553\"}", - "College Computer Science - Observed inference time (s)": "{\"description\": \"min=0.562, mean=0.562, max=0.562, sum=1.125 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5623248195648194\"}", - "College Mathematics - Observed inference time (s)": "{\"description\": \"min=0.371, mean=0.371, max=0.371, sum=0.742 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3709776735305786\"}", - "College Medicine - Observed inference time (s)": "{\"description\": \"min=0.395, mean=0.395, max=0.395, sum=0.79 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3948341918129452\"}", - "College Physics - Observed inference time (s)": "{\"description\": \"min=0.395, mean=0.395, max=0.395, sum=0.789 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.39474552051693784\"}", - "College Chemistry - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Chemistry - # prompt tokens": "{\"description\": \"min=549.28, mean=549.28, max=549.28, sum=1098.56 (2)\", \"tab\": \"General information\", \"score\": \"549.28\"}", - "College Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Biology - # eval": "{\"description\": \"min=144, mean=144, max=144, sum=288 (2)\", \"tab\": \"General information\", \"score\": \"144.0\"}", - "College Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # prompt tokens": "{\"description\": \"min=473.875, mean=473.875, max=473.875, sum=947.75 (2)\", \"tab\": \"General information\", \"score\": \"473.875\"}", - "College Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # prompt tokens": "{\"description\": \"min=828.29, mean=828.29, max=828.29, sum=1656.58 (2)\", \"tab\": \"General information\", \"score\": \"828.29\"}", - "College Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Mathematics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # prompt tokens": "{\"description\": \"min=594.51, mean=594.51, max=594.51, sum=1189.02 (2)\", \"tab\": \"General information\", \"score\": \"594.51\"}", - "College Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Medicine - # eval": "{\"description\": \"min=173, mean=173, max=173, sum=346 (2)\", \"tab\": \"General information\", \"score\": \"173.0\"}", - "College Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # prompt tokens": "{\"description\": \"min=502.705, mean=502.705, max=502.705, sum=1005.41 (2)\", \"tab\": \"General information\", \"score\": \"502.70520231213874\"}", - "College Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Physics - # eval": "{\"description\": \"min=102, mean=102, max=102, sum=204 (2)\", \"tab\": \"General information\", \"score\": \"102.0\"}", - "College Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # prompt tokens": "{\"description\": \"min=503.569, mean=503.569, max=503.569, sum=1007.137 (2)\", \"tab\": \"General information\", \"score\": \"503.5686274509804\"}", - "College Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"college_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_college_physics\"" - } - } - }, - { - "evaluation_name": "Computer Security", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Computer Security", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.71, - "details": { - "description": "min=0.71, mean=0.71, max=0.71, sum=1.42 (2)", - "tab": "Accuracy", - "Computer Security - Observed inference time (s)": "{\"description\": \"min=0.434, mean=0.434, max=0.434, sum=0.867 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.43369229555130007\"}", - "Computer Security - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Computer Security - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Computer Security - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Computer Security - # prompt tokens": "{\"description\": \"min=378.51, mean=378.51, max=378.51, sum=757.02 (2)\", \"tab\": \"General information\", \"score\": \"378.51\"}", - "Computer Security - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"computer_security\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_computer_security\"" - } - } - }, - { - "evaluation_name": "Econometrics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Econometrics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.351, - "details": { - "description": "min=0.351, mean=0.351, max=0.351, sum=0.702 (2)", - "tab": "Accuracy", - "Econometrics - Observed inference time (s)": "{\"description\": \"min=0.371, mean=0.371, max=0.371, sum=0.742 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3707838414008157\"}", - "Econometrics - # eval": "{\"description\": \"min=114, mean=114, max=114, sum=228 (2)\", \"tab\": \"General information\", \"score\": \"114.0\"}", - "Econometrics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Econometrics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Econometrics - # prompt tokens": "{\"description\": \"min=614.421, mean=614.421, max=614.421, sum=1228.842 (2)\", \"tab\": \"General information\", \"score\": \"614.421052631579\"}", - "Econometrics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"econometrics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_econometrics\"" - } - } - }, - { - "evaluation_name": "Global Facts", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Global Facts", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.26, - "details": { - "description": "min=0.26, mean=0.26, max=0.26, sum=0.52 (2)", - "tab": "Accuracy", - "Global Facts - Observed inference time (s)": "{\"description\": \"min=0.202, mean=0.202, max=0.202, sum=0.403 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2015515398979187\"}", - "Global Facts - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Global Facts - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Global Facts - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Global Facts - # prompt tokens": "{\"description\": \"min=399.71, mean=399.71, max=399.71, sum=799.42 (2)\", \"tab\": \"General information\", \"score\": \"399.71\"}", - "Global Facts - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"global_facts\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_global_facts\"" - } - } - }, - { - "evaluation_name": "Jurisprudence", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Jurisprudence", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.731, - "details": { - "description": "min=0.731, mean=0.731, max=0.731, sum=1.463 (2)", - "tab": "Accuracy", - "Jurisprudence - Observed inference time (s)": "{\"description\": \"min=1.035, mean=1.035, max=1.035, sum=2.07 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.0347525963076838\"}", - "Jurisprudence - # eval": "{\"description\": \"min=108, mean=108, max=108, sum=216 (2)\", \"tab\": \"General information\", \"score\": \"108.0\"}", - "Jurisprudence - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Jurisprudence - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Jurisprudence - # prompt tokens": "{\"description\": \"min=394.63, mean=394.63, max=394.63, sum=789.259 (2)\", \"tab\": \"General information\", \"score\": \"394.6296296296296\"}", - "Jurisprudence - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"jurisprudence\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_jurisprudence\"" - } - } - }, - { - "evaluation_name": "Philosophy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Philosophy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.64, - "details": { - "description": "min=0.64, mean=0.64, max=0.64, sum=1.28 (2)", - "tab": "Accuracy", - "Philosophy - Observed inference time (s)": "{\"description\": \"min=0.681, mean=0.681, max=0.681, sum=1.363 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6814629341628391\"}", - "Philosophy - # eval": "{\"description\": \"min=311, mean=311, max=311, sum=622 (2)\", \"tab\": \"General information\", \"score\": \"311.0\"}", - "Philosophy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Philosophy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Philosophy - # prompt tokens": "{\"description\": \"min=329.084, mean=329.084, max=329.084, sum=658.167 (2)\", \"tab\": \"General information\", \"score\": \"329.08360128617363\"}", - "Philosophy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"philosophy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_philosophy\"" - } - } - }, - { - "evaluation_name": "Professional Psychology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Professional Psychology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.649, - "details": { - "description": "min=0.649, mean=0.649, max=0.649, sum=1.297 (2)", - "tab": "Accuracy", - "Professional Medicine - Observed inference time (s)": "{\"description\": \"min=0.546, mean=0.546, max=0.546, sum=1.091 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5456299475010704\"}", - "Professional Accounting - Observed inference time (s)": "{\"description\": \"min=0.538, mean=0.538, max=0.538, sum=1.077 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5383730044601657\"}", - "Professional Law - Observed inference time (s)": "{\"description\": \"min=0.881, mean=0.881, max=0.881, sum=1.762 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8808572895368355\"}", - "Professional Psychology - Observed inference time (s)": "{\"description\": \"min=0.694, mean=0.694, max=0.694, sum=1.388 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6941978611977272\"}", - "Professional Medicine - # eval": "{\"description\": \"min=272, mean=272, max=272, sum=544 (2)\", \"tab\": \"General information\", \"score\": \"272.0\"}", - "Professional Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Medicine - # prompt tokens": "{\"description\": \"min=1094.489, mean=1094.489, max=1094.489, sum=2188.978 (2)\", \"tab\": \"General information\", \"score\": \"1094.4889705882354\"}", - "Professional Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Accounting - # eval": "{\"description\": \"min=282, mean=282, max=282, sum=564 (2)\", \"tab\": \"General information\", \"score\": \"282.0\"}", - "Professional Accounting - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Accounting - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # prompt tokens": "{\"description\": \"min=658.585, mean=658.585, max=658.585, sum=1317.17 (2)\", \"tab\": \"General information\", \"score\": \"658.5851063829788\"}", - "Professional Accounting - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Law - # eval": "{\"description\": \"min=1534, mean=1534, max=1534, sum=3068 (2)\", \"tab\": \"General information\", \"score\": \"1534.0\"}", - "Professional Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # prompt tokens": "{\"description\": \"min=1637.601, mean=1637.601, max=1637.601, sum=3275.202 (2)\", \"tab\": \"General information\", \"score\": \"1637.6010430247718\"}", - "Professional Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Psychology - # eval": "{\"description\": \"min=612, mean=612, max=612, sum=1224 (2)\", \"tab\": \"General information\", \"score\": \"612.0\"}", - "Professional Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # prompt tokens": "{\"description\": \"min=575.098, mean=575.098, max=575.098, sum=1150.196 (2)\", \"tab\": \"General information\", \"score\": \"575.0980392156863\"}", - "Professional Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"professional_psychology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_professional_psychology\"" - } - } - }, - { - "evaluation_name": "Us Foreign Policy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Us Foreign Policy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.79, - "details": { - "description": "min=0.79, mean=0.79, max=0.79, sum=1.58 (2)", - "tab": "Accuracy", - "Us Foreign Policy - Observed inference time (s)": "{\"description\": \"min=0.567, mean=0.567, max=0.567, sum=1.135 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5673955392837524\"}", - "Us Foreign Policy - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Us Foreign Policy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Us Foreign Policy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Us Foreign Policy - # prompt tokens": "{\"description\": \"min=422.79, mean=422.79, max=422.79, sum=845.58 (2)\", \"tab\": \"General information\", \"score\": \"422.79\"}", - "Us Foreign Policy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"us_foreign_policy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_us_foreign_policy\"" - } - } - }, - { - "evaluation_name": "Astronomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Astronomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.645, - "details": { - "description": "min=0.645, mean=0.645, max=0.645, sum=1.289 (2)", - "tab": "Accuracy", - "Astronomy - Observed inference time (s)": "{\"description\": \"min=0.317, mean=0.317, max=0.317, sum=0.634 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3168644199245854\"}", - "Astronomy - # eval": "{\"description\": \"min=152, mean=152, max=152, sum=304 (2)\", \"tab\": \"General information\", \"score\": \"152.0\"}", - "Astronomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Astronomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Astronomy - # prompt tokens": "{\"description\": \"min=579.684, mean=579.684, max=579.684, sum=1159.368 (2)\", \"tab\": \"General information\", \"score\": \"579.6842105263158\"}", - "Astronomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"astronomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_astronomy\"" - } - } - }, - { - "evaluation_name": "Business Ethics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Business Ethics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.65, - "details": { - "description": "min=0.65, mean=0.65, max=0.65, sum=1.3 (2)", - "tab": "Accuracy", - "Business Ethics - Observed inference time (s)": "{\"description\": \"min=0.444, mean=0.444, max=0.444, sum=0.888 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.44396358251571655\"}", - "Business Ethics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Business Ethics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Business Ethics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Business Ethics - # prompt tokens": "{\"description\": \"min=569.52, mean=569.52, max=569.52, sum=1139.04 (2)\", \"tab\": \"General information\", \"score\": \"569.52\"}", - "Business Ethics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"business_ethics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_business_ethics\"" - } - } - }, - { - "evaluation_name": "Clinical Knowledge", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Clinical Knowledge", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.615, - "details": { - "description": "min=0.615, mean=0.615, max=0.615, sum=1.23 (2)", - "tab": "Accuracy", - "Clinical Knowledge - Observed inference time (s)": "{\"description\": \"min=0.369, mean=0.369, max=0.369, sum=0.738 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3692442273193935\"}", - "Clinical Knowledge - # eval": "{\"description\": \"min=265, mean=265, max=265, sum=530 (2)\", \"tab\": \"General information\", \"score\": \"265.0\"}", - "Clinical Knowledge - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Clinical Knowledge - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Clinical Knowledge - # prompt tokens": "{\"description\": \"min=397.928, mean=397.928, max=397.928, sum=795.857 (2)\", \"tab\": \"General information\", \"score\": \"397.92830188679244\"}", - "Clinical Knowledge - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"clinical_knowledge\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_clinical_knowledge\"" - } - } - }, - { - "evaluation_name": "Conceptual Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Conceptual Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.528, - "details": { - "description": "min=0.528, mean=0.528, max=0.528, sum=1.055 (2)", - "tab": "Accuracy", - "Conceptual Physics - Observed inference time (s)": "{\"description\": \"min=0.351, mean=0.351, max=0.351, sum=0.701 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.35051030605397326\"}", - "Conceptual Physics - # eval": "{\"description\": \"min=235, mean=235, max=235, sum=470 (2)\", \"tab\": \"General information\", \"score\": \"235.0\"}", - "Conceptual Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Conceptual Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Conceptual Physics - # prompt tokens": "{\"description\": \"min=304.834, mean=304.834, max=304.834, sum=609.668 (2)\", \"tab\": \"General information\", \"score\": \"304.83404255319147\"}", - "Conceptual Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"conceptual_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_conceptual_physics\"" - } - } - }, - { - "evaluation_name": "Electrical Engineering", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Electrical Engineering", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.441, - "details": { - "description": "min=0.441, mean=0.441, max=0.441, sum=0.883 (2)", - "tab": "Accuracy", - "Electrical Engineering - Observed inference time (s)": "{\"description\": \"min=0.35, mean=0.35, max=0.35, sum=0.7 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.34982287637118636\"}", - "Electrical Engineering - # eval": "{\"description\": \"min=145, mean=145, max=145, sum=290 (2)\", \"tab\": \"General information\", \"score\": \"145.0\"}", - "Electrical Engineering - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Electrical Engineering - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Electrical Engineering - # prompt tokens": "{\"description\": \"min=435.607, mean=435.607, max=435.607, sum=871.214 (2)\", \"tab\": \"General information\", \"score\": \"435.60689655172416\"}", - "Electrical Engineering - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"electrical_engineering\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_electrical_engineering\"" - } - } - }, - { - "evaluation_name": "Elementary Mathematics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Elementary Mathematics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.429, - "details": { - "description": "min=0.429, mean=0.429, max=0.429, sum=0.857 (2)", - "tab": "Accuracy", - "Elementary Mathematics - Observed inference time (s)": "{\"description\": \"min=0.4, mean=0.4, max=0.4, sum=0.801 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4003569991500289\"}", - "Elementary Mathematics - # eval": "{\"description\": \"min=378, mean=378, max=378, sum=756 (2)\", \"tab\": \"General information\", \"score\": \"378.0\"}", - "Elementary Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Elementary Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Elementary Mathematics - # prompt tokens": "{\"description\": \"min=531.854, mean=531.854, max=531.854, sum=1063.709 (2)\", \"tab\": \"General information\", \"score\": \"531.8544973544973\"}", - "Elementary Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"elementary_mathematics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_elementary_mathematics\"" - } - } - }, - { - "evaluation_name": "Formal Logic", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Formal Logic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.444, - "details": { - "description": "min=0.444, mean=0.444, max=0.444, sum=0.889 (2)", - "tab": "Accuracy", - "Formal Logic - Observed inference time (s)": "{\"description\": \"min=0.357, mean=0.357, max=0.357, sum=0.714 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.35707327108534553\"}", - "Formal Logic - # eval": "{\"description\": \"min=126, mean=126, max=126, sum=252 (2)\", \"tab\": \"General information\", \"score\": \"126.0\"}", - "Formal Logic - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Formal Logic - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Formal Logic - # prompt tokens": "{\"description\": \"min=601.778, mean=601.778, max=601.778, sum=1203.556 (2)\", \"tab\": \"General information\", \"score\": \"601.7777777777778\"}", - "Formal Logic - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"formal_logic\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_formal_logic\"" - } - } - }, - { - "evaluation_name": "High School World History", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on High School World History", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.515, - "details": { - "description": "min=0.515, mean=0.515, max=0.515, sum=1.03 (2)", - "tab": "Accuracy", - "High School Biology - Observed inference time (s)": "{\"description\": \"min=0.211, mean=0.211, max=0.211, sum=0.423 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.21137587870320967\"}", - "High School Chemistry - Observed inference time (s)": "{\"description\": \"min=0.211, mean=0.211, max=0.211, sum=0.423 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2113605567387172\"}", - "High School Computer Science - Observed inference time (s)": "{\"description\": \"min=0.214, mean=0.214, max=0.214, sum=0.428 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2138903546333313\"}", - "High School European History - Observed inference time (s)": "{\"description\": \"min=0.332, mean=0.332, max=0.332, sum=0.664 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.33188523668231384\"}", - "High School Geography - Observed inference time (s)": "{\"description\": \"min=0.218, mean=0.218, max=0.218, sum=0.435 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.21753037818754561\"}", - "High School Government And Politics - Observed inference time (s)": "{\"description\": \"min=0.558, mean=0.558, max=0.558, sum=1.117 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.558492410985917\"}", - "High School Macroeconomics - Observed inference time (s)": "{\"description\": \"min=0.703, mean=0.703, max=0.703, sum=1.407 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7033225890917656\"}", - "High School Mathematics - Observed inference time (s)": "{\"description\": \"min=0.649, mean=0.649, max=0.649, sum=1.299 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6494572189119127\"}", - "High School Microeconomics - Observed inference time (s)": "{\"description\": \"min=0.612, mean=0.612, max=0.612, sum=1.223 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6115654797113242\"}", - "High School Physics - Observed inference time (s)": "{\"description\": \"min=0.564, mean=0.564, max=0.564, sum=1.127 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5636763351642533\"}", - "High School Psychology - Observed inference time (s)": "{\"description\": \"min=0.681, mean=0.681, max=0.681, sum=1.363 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6813242522948378\"}", - "High School Statistics - Observed inference time (s)": "{\"description\": \"min=0.606, mean=0.606, max=0.606, sum=1.212 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6060926814874014\"}", - "High School US History - Observed inference time (s)": "{\"description\": \"min=1.122, mean=1.122, max=1.122, sum=2.244 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.1218917334780973\"}", - "High School World History - Observed inference time (s)": "{\"description\": \"min=0.538, mean=0.538, max=0.538, sum=1.076 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5378943324592043\"}", - "High School Biology - # eval": "{\"description\": \"min=310, mean=310, max=310, sum=620 (2)\", \"tab\": \"General information\", \"score\": \"310.0\"}", - "High School Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Biology - # prompt tokens": "{\"description\": \"min=513.671, mean=513.671, max=513.671, sum=1027.342 (2)\", \"tab\": \"General information\", \"score\": \"513.6709677419354\"}", - "High School Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Chemistry - # eval": "{\"description\": \"min=203, mean=203, max=203, sum=406 (2)\", \"tab\": \"General information\", \"score\": \"203.0\"}", - "High School Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # prompt tokens": "{\"description\": \"min=496.704, mean=496.704, max=496.704, sum=993.409 (2)\", \"tab\": \"General information\", \"score\": \"496.70443349753697\"}", - "High School Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "High School Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # prompt tokens": "{\"description\": \"min=867.78, mean=867.78, max=867.78, sum=1735.56 (2)\", \"tab\": \"General information\", \"score\": \"867.78\"}", - "High School Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School European History - # eval": "{\"description\": \"min=165, mean=165, max=165, sum=330 (2)\", \"tab\": \"General information\", \"score\": \"165.0\"}", - "High School European History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School European History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # prompt tokens": "{\"description\": \"min=2797.885, mean=2797.885, max=2797.885, sum=5595.77 (2)\", \"tab\": \"General information\", \"score\": \"2797.8848484848486\"}", - "High School European History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Geography - # eval": "{\"description\": \"min=198, mean=198, max=198, sum=396 (2)\", \"tab\": \"General information\", \"score\": \"198.0\"}", - "High School Geography - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Geography - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # prompt tokens": "{\"description\": \"min=372.035, mean=372.035, max=372.035, sum=744.071 (2)\", \"tab\": \"General information\", \"score\": \"372.0353535353535\"}", - "High School Geography - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Government And Politics - # eval": "{\"description\": \"min=193, mean=193, max=193, sum=386 (2)\", \"tab\": \"General information\", \"score\": \"193.0\"}", - "High School Government And Politics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Government And Politics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # prompt tokens": "{\"description\": \"min=465.824, mean=465.824, max=465.824, sum=931.648 (2)\", \"tab\": \"General information\", \"score\": \"465.8238341968912\"}", - "High School Government And Politics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Macroeconomics - # eval": "{\"description\": \"min=390, mean=390, max=390, sum=780 (2)\", \"tab\": \"General information\", \"score\": \"390.0\"}", - "High School Macroeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Macroeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # prompt tokens": "{\"description\": \"min=370.908, mean=370.908, max=370.908, sum=741.815 (2)\", \"tab\": \"General information\", \"score\": \"370.9076923076923\"}", - "High School Macroeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Mathematics - # eval": "{\"description\": \"min=270, mean=270, max=270, sum=540 (2)\", \"tab\": \"General information\", \"score\": \"270.0\"}", - "High School Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # prompt tokens": "{\"description\": \"min=532.356, mean=532.356, max=532.356, sum=1064.711 (2)\", \"tab\": \"General information\", \"score\": \"532.3555555555556\"}", - "High School Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Microeconomics - # eval": "{\"description\": \"min=238, mean=238, max=238, sum=476 (2)\", \"tab\": \"General information\", \"score\": \"238.0\"}", - "High School Microeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Microeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # prompt tokens": "{\"description\": \"min=399.013, mean=399.013, max=399.013, sum=798.025 (2)\", \"tab\": \"General information\", \"score\": \"399.0126050420168\"}", - "High School Microeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Physics - # eval": "{\"description\": \"min=151, mean=151, max=151, sum=302 (2)\", \"tab\": \"General information\", \"score\": \"151.0\"}", - "High School Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # prompt tokens": "{\"description\": \"min=560.457, mean=560.457, max=560.457, sum=1120.914 (2)\", \"tab\": \"General information\", \"score\": \"560.4569536423841\"}", - "High School Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Psychology - # eval": "{\"description\": \"min=545, mean=545, max=545, sum=1090 (2)\", \"tab\": \"General information\", \"score\": \"545.0\"}", - "High School Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # prompt tokens": "{\"description\": \"min=495.242, mean=495.242, max=495.242, sum=990.484 (2)\", \"tab\": \"General information\", \"score\": \"495.2422018348624\"}", - "High School Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Statistics - # eval": "{\"description\": \"min=216, mean=216, max=216, sum=432 (2)\", \"tab\": \"General information\", \"score\": \"216.0\"}", - "High School Statistics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Statistics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # prompt tokens": "{\"description\": \"min=795.639, mean=795.639, max=795.639, sum=1591.278 (2)\", \"tab\": \"General information\", \"score\": \"795.6388888888889\"}", - "High School Statistics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School US History - # eval": "{\"description\": \"min=204, mean=204, max=204, sum=408 (2)\", \"tab\": \"General information\", \"score\": \"204.0\"}", - "High School US History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School US History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # prompt tokens": "{\"description\": \"min=2217.809, mean=2217.809, max=2217.809, sum=4435.618 (2)\", \"tab\": \"General information\", \"score\": \"2217.8088235294117\"}", - "High School US History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School World History - # eval": "{\"description\": \"min=237, mean=237, max=237, sum=474 (2)\", \"tab\": \"General information\", \"score\": \"237.0\"}", - "High School World History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School World History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # prompt tokens": "{\"description\": \"min=1428.173, mean=1428.173, max=1428.173, sum=2856.346 (2)\", \"tab\": \"General information\", \"score\": \"1428.1729957805908\"}", - "High School World History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"high_school_world_history\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_high_school_world_history\"" - } - } - }, - { - "evaluation_name": "Human Sexuality", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Human Sexuality", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.733, - "details": { - "description": "min=0.733, mean=0.733, max=0.733, sum=1.466 (2)", - "tab": "Accuracy", - "Human Aging - Observed inference time (s)": "{\"description\": \"min=0.685, mean=0.685, max=0.685, sum=1.369 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6845707412257858\"}", - "Human Sexuality - Observed inference time (s)": "{\"description\": \"min=1.227, mean=1.227, max=1.227, sum=2.455 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.2273387745136524\"}", - "Human Aging - # eval": "{\"description\": \"min=223, mean=223, max=223, sum=446 (2)\", \"tab\": \"General information\", \"score\": \"223.0\"}", - "Human Aging - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Aging - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Aging - # prompt tokens": "{\"description\": \"min=319.888, mean=319.888, max=319.888, sum=639.776 (2)\", \"tab\": \"General information\", \"score\": \"319.88789237668163\"}", - "Human Aging - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Human Sexuality - # eval": "{\"description\": \"min=131, mean=131, max=131, sum=262 (2)\", \"tab\": \"General information\", \"score\": \"131.0\"}", - "Human Sexuality - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Sexuality - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # prompt tokens": "{\"description\": \"min=341.168, mean=341.168, max=341.168, sum=682.336 (2)\", \"tab\": \"General information\", \"score\": \"341.1679389312977\"}", - "Human Sexuality - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"human_sexuality\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_human_sexuality\"" - } - } - }, - { - "evaluation_name": "International Law", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on International Law", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.694, - "details": { - "description": "min=0.694, mean=0.694, max=0.694, sum=1.388 (2)", - "tab": "Accuracy", - "International Law - Observed inference time (s)": "{\"description\": \"min=0.684, mean=0.684, max=0.684, sum=1.369 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6842782950598346\"}", - "International Law - # eval": "{\"description\": \"min=121, mean=121, max=121, sum=242 (2)\", \"tab\": \"General information\", \"score\": \"121.0\"}", - "International Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "International Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "International Law - # prompt tokens": "{\"description\": \"min=639.818, mean=639.818, max=639.818, sum=1279.636 (2)\", \"tab\": \"General information\", \"score\": \"639.8181818181819\"}", - "International Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"international_law\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_international_law\"" - } - } - }, - { - "evaluation_name": "Logical Fallacies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Logical Fallacies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.742, - "details": { - "description": "min=0.742, mean=0.742, max=0.742, sum=1.485 (2)", - "tab": "Accuracy", - "Logical Fallacies - Observed inference time (s)": "{\"description\": \"min=1.35, mean=1.35, max=1.35, sum=2.7 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.3501118970063566\"}", - "Logical Fallacies - # eval": "{\"description\": \"min=163, mean=163, max=163, sum=326 (2)\", \"tab\": \"General information\", \"score\": \"163.0\"}", - "Logical Fallacies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Logical Fallacies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Logical Fallacies - # prompt tokens": "{\"description\": \"min=449.564, mean=449.564, max=449.564, sum=899.129 (2)\", \"tab\": \"General information\", \"score\": \"449.5644171779141\"}", - "Logical Fallacies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"logical_fallacies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_logical_fallacies\"" - } - } - }, - { - "evaluation_name": "Machine Learning", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Machine Learning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.384, - "details": { - "description": "min=0.384, mean=0.384, max=0.384, sum=0.768 (2)", - "tab": "Accuracy", - "Machine Learning - Observed inference time (s)": "{\"description\": \"min=0.46, mean=0.46, max=0.46, sum=0.919 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.45964209735393524\"}", - "Machine Learning - # eval": "{\"description\": \"min=112, mean=112, max=112, sum=224 (2)\", \"tab\": \"General information\", \"score\": \"112.0\"}", - "Machine Learning - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Machine Learning - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Machine Learning - # prompt tokens": "{\"description\": \"min=668.054, mean=668.054, max=668.054, sum=1336.107 (2)\", \"tab\": \"General information\", \"score\": \"668.0535714285714\"}", - "Machine Learning - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"machine_learning\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_machine_learning\"" - } - } - }, - { - "evaluation_name": "Management", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Management", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.709, - "details": { - "description": "min=0.709, mean=0.709, max=0.709, sum=1.417 (2)", - "tab": "Accuracy", - "Management - Observed inference time (s)": "{\"description\": \"min=0.481, mean=0.481, max=0.481, sum=0.963 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.48132226536574874\"}", - "Management - # eval": "{\"description\": \"min=103, mean=103, max=103, sum=206 (2)\", \"tab\": \"General information\", \"score\": \"103.0\"}", - "Management - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Management - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Management - # prompt tokens": "{\"description\": \"min=283.786, mean=283.786, max=283.786, sum=567.573 (2)\", \"tab\": \"General information\", \"score\": \"283.7864077669903\"}", - "Management - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"management\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_management\"" - } - } - }, - { - "evaluation_name": "Marketing", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Marketing", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.833, - "details": { - "description": "min=0.833, mean=0.833, max=0.833, sum=1.667 (2)", - "tab": "Accuracy", - "Marketing - Observed inference time (s)": "{\"description\": \"min=0.529, mean=0.529, max=0.529, sum=1.059 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5294545297948723\"}", - "Marketing - # eval": "{\"description\": \"min=234, mean=234, max=234, sum=468 (2)\", \"tab\": \"General information\", \"score\": \"234.0\"}", - "Marketing - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Marketing - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Marketing - # prompt tokens": "{\"description\": \"min=404.218, mean=404.218, max=404.218, sum=808.436 (2)\", \"tab\": \"General information\", \"score\": \"404.21794871794873\"}", - "Marketing - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"marketing\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_marketing\"" - } - } - }, - { - "evaluation_name": "Medical Genetics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Medical Genetics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.66, - "details": { - "description": "min=0.66, mean=0.66, max=0.66, sum=1.32 (2)", - "tab": "Accuracy", - "Medical Genetics - Observed inference time (s)": "{\"description\": \"min=0.521, mean=0.521, max=0.521, sum=1.041 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.520596706867218\"}", - "Medical Genetics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Medical Genetics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Medical Genetics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Medical Genetics - # prompt tokens": "{\"description\": \"min=340.99, mean=340.99, max=340.99, sum=681.98 (2)\", \"tab\": \"General information\", \"score\": \"340.99\"}", - "Medical Genetics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"medical_genetics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_medical_genetics\"" - } - } - }, - { - "evaluation_name": "Miscellaneous", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Miscellaneous", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.653, - "details": { - "description": "min=0.653, mean=0.653, max=0.653, sum=1.305 (2)", - "tab": "Accuracy", - "Miscellaneous - Observed inference time (s)": "{\"description\": \"min=0.803, mean=0.803, max=0.803, sum=1.606 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8030396217282857\"}", - "Miscellaneous - # eval": "{\"description\": \"min=783, mean=783, max=783, sum=1566 (2)\", \"tab\": \"General information\", \"score\": \"783.0\"}", - "Miscellaneous - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Miscellaneous - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Miscellaneous - # prompt tokens": "{\"description\": \"min=299.911, mean=299.911, max=299.911, sum=599.821 (2)\", \"tab\": \"General information\", \"score\": \"299.9106002554278\"}", - "Miscellaneous - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"miscellaneous\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_miscellaneous\"" - } - } - }, - { - "evaluation_name": "Moral Scenarios", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Moral Scenarios", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.368, - "details": { - "description": "min=0.368, mean=0.368, max=0.368, sum=0.735 (2)", - "tab": "Accuracy", - "Moral Disputes - Observed inference time (s)": "{\"description\": \"min=0.657, mean=0.657, max=0.657, sum=1.314 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6570079657383737\"}", - "Moral Scenarios - Observed inference time (s)": "{\"description\": \"min=0.65, mean=0.65, max=0.65, sum=1.299 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.649639103266114\"}", - "Moral Disputes - # eval": "{\"description\": \"min=346, mean=346, max=346, sum=692 (2)\", \"tab\": \"General information\", \"score\": \"346.0\"}", - "Moral Disputes - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Disputes - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Disputes - # prompt tokens": "{\"description\": \"min=476.113, mean=476.113, max=476.113, sum=952.225 (2)\", \"tab\": \"General information\", \"score\": \"476.1127167630058\"}", - "Moral Disputes - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Moral Scenarios - # eval": "{\"description\": \"min=895, mean=895, max=895, sum=1790 (2)\", \"tab\": \"General information\", \"score\": \"895.0\"}", - "Moral Scenarios - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Scenarios - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # prompt tokens": "{\"description\": \"min=656.455, mean=656.455, max=656.455, sum=1312.909 (2)\", \"tab\": \"General information\", \"score\": \"656.454748603352\"}", - "Moral Scenarios - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"moral_scenarios\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_moral_scenarios\"" - } - } - }, - { - "evaluation_name": "Nutrition", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Nutrition", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.712, - "details": { - "description": "min=0.712, mean=0.712, max=0.712, sum=1.425 (2)", - "tab": "Accuracy", - "Nutrition - Observed inference time (s)": "{\"description\": \"min=1.485, mean=1.485, max=1.485, sum=2.971 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.4853957338270798\"}", - "Nutrition - # eval": "{\"description\": \"min=306, mean=306, max=306, sum=612 (2)\", \"tab\": \"General information\", \"score\": \"306.0\"}", - "Nutrition - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Nutrition - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Nutrition - # prompt tokens": "{\"description\": \"min=586.814, mean=586.814, max=586.814, sum=1173.627 (2)\", \"tab\": \"General information\", \"score\": \"586.8137254901961\"}", - "Nutrition - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"nutrition\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_nutrition\"" - } - } - }, - { - "evaluation_name": "Prehistory", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Prehistory", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.728, - "details": { - "description": "min=0.728, mean=0.728, max=0.728, sum=1.457 (2)", - "tab": "Accuracy", - "Prehistory - Observed inference time (s)": "{\"description\": \"min=0.792, mean=0.792, max=0.792, sum=1.584 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7917959955003526\"}", - "Prehistory - # eval": "{\"description\": \"min=324, mean=324, max=324, sum=648 (2)\", \"tab\": \"General information\", \"score\": \"324.0\"}", - "Prehistory - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Prehistory - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Prehistory - # prompt tokens": "{\"description\": \"min=514.528, mean=514.528, max=514.528, sum=1029.056 (2)\", \"tab\": \"General information\", \"score\": \"514.5277777777778\"}", - "Prehistory - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"prehistory\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_prehistory\"" - } - } - }, - { - "evaluation_name": "Public Relations", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Public Relations", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.664, - "details": { - "description": "min=0.664, mean=0.664, max=0.664, sum=1.327 (2)", - "tab": "Accuracy", - "Public Relations - Observed inference time (s)": "{\"description\": \"min=0.493, mean=0.493, max=0.493, sum=0.986 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.49318039634011007\"}", - "Public Relations - # eval": "{\"description\": \"min=110, mean=110, max=110, sum=220 (2)\", \"tab\": \"General information\", \"score\": \"110.0\"}", - "Public Relations - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Public Relations - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Public Relations - # prompt tokens": "{\"description\": \"min=405.318, mean=405.318, max=405.318, sum=810.636 (2)\", \"tab\": \"General information\", \"score\": \"405.3181818181818\"}", - "Public Relations - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"public_relations\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_public_relations\"" - } - } - }, - { - "evaluation_name": "Security Studies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Security Studies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.576, - "details": { - "description": "min=0.576, mean=0.576, max=0.576, sum=1.151 (2)", - "tab": "Accuracy", - "Security Studies - Observed inference time (s)": "{\"description\": \"min=0.656, mean=0.656, max=0.656, sum=1.312 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6561975401275012\"}", - "Security Studies - # eval": "{\"description\": \"min=245, mean=245, max=245, sum=490 (2)\", \"tab\": \"General information\", \"score\": \"245.0\"}", - "Security Studies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Security Studies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Security Studies - # prompt tokens": "{\"description\": \"min=1164.473, mean=1164.473, max=1164.473, sum=2328.947 (2)\", \"tab\": \"General information\", \"score\": \"1164.4734693877551\"}", - "Security Studies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"security_studies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_security_studies\"" - } - } - }, - { - "evaluation_name": "Sociology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Sociology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.701, - "details": { - "description": "min=0.701, mean=0.701, max=0.701, sum=1.403 (2)", - "tab": "Accuracy", - "Sociology - Observed inference time (s)": "{\"description\": \"min=0.517, mean=0.517, max=0.517, sum=1.034 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5170851643405744\"}", - "Sociology - # eval": "{\"description\": \"min=201, mean=201, max=201, sum=402 (2)\", \"tab\": \"General information\", \"score\": \"201.0\"}", - "Sociology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Sociology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Sociology - # prompt tokens": "{\"description\": \"min=445.517, mean=445.517, max=445.517, sum=891.035 (2)\", \"tab\": \"General information\", \"score\": \"445.51741293532336\"}", - "Sociology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"sociology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_sociology\"" - } - } - }, - { - "evaluation_name": "Virology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Virology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.446, - "details": { - "description": "min=0.446, mean=0.446, max=0.446, sum=0.892 (2)", - "tab": "Accuracy", - "Virology - Observed inference time (s)": "{\"description\": \"min=0.406, mean=0.406, max=0.406, sum=0.813 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.40646702553852493\"}", - "Virology - # eval": "{\"description\": \"min=166, mean=166, max=166, sum=332 (2)\", \"tab\": \"General information\", \"score\": \"166.0\"}", - "Virology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Virology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Virology - # prompt tokens": "{\"description\": \"min=343.018, mean=343.018, max=343.018, sum=686.036 (2)\", \"tab\": \"General information\", \"score\": \"343.01807228915663\"}", - "Virology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"virology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_virology\"" - } - } - }, - { - "evaluation_name": "World Religions", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on World Religions", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.789, - "details": { - "description": "min=0.789, mean=0.789, max=0.789, sum=1.579 (2)", - "tab": "Accuracy", - "World Religions - Observed inference time (s)": "{\"description\": \"min=0.587, mean=0.587, max=0.587, sum=1.173 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5866640882882458\"}", - "World Religions - # eval": "{\"description\": \"min=171, mean=171, max=171, sum=342 (2)\", \"tab\": \"General information\", \"score\": \"171.0\"}", - "World Religions - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "World Religions - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "World Religions - # prompt tokens": "{\"description\": \"min=274.52, mean=274.52, max=274.52, sum=549.041 (2)\", \"tab\": \"General information\", \"score\": \"274.5204678362573\"}", - "World Religions - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"world_religions\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_world_religions\"" - } - } - }, - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.475, - "details": { - "description": "", - "tab": "Efficiency" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/meta_llama-3.2-11b-vision-instruct-turbo.json b/data/models/meta_llama-3.2-11b-vision-instruct-turbo.json deleted file mode 100644 index 247fa51f235f6cc18e7c55e350f128cab0000bbb..0000000000000000000000000000000000000000 --- a/data/models/meta_llama-3.2-11b-vision-instruct-turbo.json +++ /dev/null @@ -1,1903 +0,0 @@ -{ - "model_info": { - "name": "Llama 3.2 Vision Instruct Turbo 11B", - "id": "meta/llama-3.2-11b-vision-instruct-turbo", - "developer": "Meta", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_lite/meta_llama-3.2-11b-vision-instruct-turbo/1774096306.427425", - "retrieved_timestamp": "1774096306.427425", - "source_metadata": { - "source_name": "helm_lite", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_lite", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_lite", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.325, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.8754681647940075\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.756, - "details": { - "description": "min=0.756, mean=0.756, max=0.756, sum=0.756 (1)", - "tab": "Accuracy", - "NarrativeQA - Observed inference time (s)": "{\"description\": \"min=0.378, mean=0.378, max=0.378, sum=0.378 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.37828690300525075\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=3484.268, mean=3484.268, max=3484.268, sum=3484.268 (1)\", \"tab\": \"General information\", \"score\": \"3484.2676056338028\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (closed-book)", - "source_data": { - "dataset_name": "NaturalQuestions (closed-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (closed-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.234, - "details": { - "description": "min=0.234, mean=0.234, max=0.234, sum=0.234 (1)", - "tab": "Accuracy", - "NaturalQuestions (open-book) - Observed inference time (s)": "{\"description\": \"min=0.285, mean=0.285, max=0.285, sum=0.285 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.28472757744789123\"}", - "NaturalQuestions (closed-book) - Observed inference time (s)": "{\"description\": \"min=0.326, mean=0.326, max=0.326, sum=0.326 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.32630494999885556\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1716.785, mean=1716.785, max=1716.785, sum=1716.785 (1)\", \"tab\": \"General information\", \"score\": \"1716.785\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=129.12, mean=129.12, max=129.12, sum=129.12 (1)\", \"tab\": \"General information\", \"score\": \"129.12\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "mode": "\"closedbook\"" - } - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.724, - "details": { - "description": "min=0.724, mean=0.724, max=0.724, sum=0.724 (1)", - "tab": "Accuracy", - "OpenbookQA - Observed inference time (s)": "{\"description\": \"min=0.21, mean=0.21, max=0.21, sum=0.21 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.21042356300354004\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=249.776, mean=249.776, max=249.776, sum=249.776 (1)\", \"tab\": \"General information\", \"score\": \"249.776\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "dataset": "\"openbookqa\"", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.511, - "details": { - "description": "min=0.28, mean=0.511, max=0.78, sum=2.555 (5)", - "tab": "Accuracy", - "MMLU - Observed inference time (s)": "{\"description\": \"min=0.226, mean=0.406, max=0.726, sum=2.031 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.40622414255142214\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=373.43, mean=467.686, max=614.421, sum=2338.431 (5)\", \"tab\": \"General information\", \"score\": \"467.6862105263158\"}", - "MMLU - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"college_chemistry\", \"computer_security\", \"econometrics\", \"us_foreign_policy\"]", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MATH", - "source_data": { - "dataset_name": "MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Equivalent (CoT) on MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.739, - "details": { - "description": "min=0.579, mean=0.739, max=0.884, sum=5.176 (7)", - "tab": "Accuracy", - "MATH - Observed inference time (s)": "{\"description\": \"min=1.715, mean=2.099, max=2.413, sum=14.696 (7)\", \"tab\": \"Efficiency\", \"score\": \"2.099496145662431\"}", - "MATH - # eval": "{\"description\": \"min=30, mean=62.429, max=135, sum=437 (7)\", \"tab\": \"General information\", \"score\": \"62.42857142857143\"}", - "MATH - # train": "{\"description\": \"min=8, mean=8, max=8, sum=56 (7)\", \"tab\": \"General information\", \"score\": \"8.0\"}", - "MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MATH - # prompt tokens": "{\"description\": \"min=881.363, mean=1262.909, max=2197.577, sum=8840.364 (7)\", \"tab\": \"General information\", \"score\": \"1262.9092130545007\"}", - "MATH - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"algebra\", \"counting_and_probability\", \"geometry\", \"intermediate_algebra\", \"number_theory\", \"prealgebra\", \"precalculus\"]", - "level": "\"1\"", - "use_official_examples": "\"False\"", - "use_chain_of_thought": "\"True\"" - } - } - }, - { - "evaluation_name": "GSM8K", - "source_data": { - "dataset_name": "GSM8K", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on GSM8K", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.823, - "details": { - "description": "min=0.823, mean=0.823, max=0.823, sum=0.823 (1)", - "tab": "Accuracy", - "GSM8K - Observed inference time (s)": "{\"description\": \"min=1.274, mean=1.274, max=1.274, sum=1.274 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.2738200931549073\"}", - "GSM8K - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "GSM8K - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "GSM8K - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GSM8K - # prompt tokens": "{\"description\": \"min=959.032, mean=959.032, max=959.032, sum=959.032 (1)\", \"tab\": \"General information\", \"score\": \"959.032\"}", - "GSM8K - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "stop": "\"none\"" - } - } - }, - { - "evaluation_name": "LegalBench", - "source_data": { - "dataset_name": "LegalBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on LegalBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.435, - "details": { - "description": "min=0.018, mean=0.435, max=0.905, sum=2.175 (5)", - "tab": "Accuracy", - "LegalBench - Observed inference time (s)": "{\"description\": \"min=0.199, mean=0.277, max=0.438, sum=1.384 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.2767821625533402\"}", - "LegalBench - # eval": "{\"description\": \"min=95, mean=409.4, max=1000, sum=2047 (5)\", \"tab\": \"General information\", \"score\": \"409.4\"}", - "LegalBench - # train": "{\"description\": \"min=4, mean=4.8, max=5, sum=24 (5)\", \"tab\": \"General information\", \"score\": \"4.8\"}", - "LegalBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "LegalBench - # prompt tokens": "{\"description\": \"min=197.442, mean=1513.882, max=6300.012, sum=7569.412 (5)\", \"tab\": \"General information\", \"score\": \"1513.8824197238912\"}", - "LegalBench - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"abercrombie\", \"corporate_lobbying\", \"function_of_decision_section\", \"international_citizenship_questions\", \"proa\"]" - } - } - }, - { - "evaluation_name": "MedQA", - "source_data": { - "dataset_name": "MedQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MedQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.27, - "details": { - "description": "min=0.27, mean=0.27, max=0.27, sum=0.27 (1)", - "tab": "Accuracy", - "MedQA - Observed inference time (s)": "{\"description\": \"min=0.205, mean=0.205, max=0.205, sum=0.205 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.20540714263916016\"}", - "MedQA - # eval": "{\"description\": \"min=503, mean=503, max=503, sum=503 (1)\", \"tab\": \"General information\", \"score\": \"503.0\"}", - "MedQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MedQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MedQA - # prompt tokens": "{\"description\": \"min=1025.274, mean=1025.274, max=1025.274, sum=1025.274 (1)\", \"tab\": \"General information\", \"score\": \"1025.2743538767395\"}", - "MedQA - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WMT 2014", - "source_data": { - "dataset_name": "WMT 2014", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "BLEU-4 on WMT 2014", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.179, - "details": { - "description": "min=0.13, mean=0.179, max=0.217, sum=0.896 (5)", - "tab": "Accuracy", - "WMT 2014 - Observed inference time (s)": "{\"description\": \"min=0.349, mean=0.383, max=0.412, sum=1.915 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.38295877939459017\"}", - "WMT 2014 - # eval": "{\"description\": \"min=503, mean=568.8, max=832, sum=2844 (5)\", \"tab\": \"General information\", \"score\": \"568.8\"}", - "WMT 2014 - # train": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "WMT 2014 - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WMT 2014 - # prompt tokens": "{\"description\": \"min=101.139, mean=120.868, max=141.33, sum=604.34 (5)\", \"tab\": \"General information\", \"score\": \"120.86804366111025\"}", - "WMT 2014 - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "language_pair": "[\"cs-en\", \"de-en\", \"fr-en\", \"hi-en\", \"ru-en\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_mmlu/meta_llama-3.2-11b-vision-instruct-turbo/1774096312.00548", - "retrieved_timestamp": "1774096312.00548", - "source_metadata": { - "source_name": "helm_mmlu", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_mmlu", - "evaluation_results": [ - { - "evaluation_name": "MMLU All Subjects", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU All Subjects", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.565, - "details": { - "description": "min=0.25, mean=0.565, max=0.865, sum=64.419 (114)", - "tab": "Accuracy", - "MMLU All Subjects - Observed inference time (s)": "{\"description\": \"min=0.204, mean=0.255, max=0.726, sum=29.095 (114)\", \"tab\": \"Efficiency\", \"score\": \"0.2552187424358169\"}", - "MMLU All Subjects - # eval": "{\"description\": \"min=100, mean=246.351, max=1534, sum=28084 (114)\", \"tab\": \"General information\", \"score\": \"246.35087719298247\"}", - "MMLU All Subjects - # train": "{\"description\": \"min=5, mean=5, max=5, sum=570 (114)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU All Subjects - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU All Subjects - # prompt tokens": "{\"description\": \"min=274.52, mean=614.619, max=2797.885, sum=70066.61 (114)\", \"tab\": \"General information\", \"score\": \"614.6193817308517\"}", - "MMLU All Subjects - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_name": "Abstract Algebra", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Abstract Algebra", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.28, - "details": { - "description": "min=0.28, mean=0.28, max=0.28, sum=0.56 (2)", - "tab": "Accuracy", - "Abstract Algebra - Observed inference time (s)": "{\"description\": \"min=0.227, mean=0.227, max=0.227, sum=0.454 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2272411847114563\"}", - "Abstract Algebra - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Abstract Algebra - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Abstract Algebra - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Abstract Algebra - # prompt tokens": "{\"description\": \"min=373.43, mean=373.43, max=373.43, sum=746.86 (2)\", \"tab\": \"General information\", \"score\": \"373.43\"}", - "Abstract Algebra - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"abstract_algebra\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_abstract_algebra\"" - } - } - }, - { - "evaluation_name": "Anatomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Anatomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.533, - "details": { - "description": "min=0.533, mean=0.533, max=0.533, sum=1.067 (2)", - "tab": "Accuracy", - "Anatomy - Observed inference time (s)": "{\"description\": \"min=0.222, mean=0.222, max=0.222, sum=0.443 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.22151856069211606\"}", - "Anatomy - # eval": "{\"description\": \"min=135, mean=135, max=135, sum=270 (2)\", \"tab\": \"General information\", \"score\": \"135.0\"}", - "Anatomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Anatomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Anatomy - # prompt tokens": "{\"description\": \"min=353.874, mean=353.874, max=353.874, sum=707.748 (2)\", \"tab\": \"General information\", \"score\": \"353.8740740740741\"}", - "Anatomy - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"anatomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_anatomy\"" - } - } - }, - { - "evaluation_name": "College Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on College Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.333, - "details": { - "description": "min=0.333, mean=0.333, max=0.333, sum=0.667 (2)", - "tab": "Accuracy", - "College Chemistry - Observed inference time (s)": "{\"description\": \"min=0.726, mean=0.726, max=0.726, sum=1.453 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7264108276367187\"}", - "College Biology - Observed inference time (s)": "{\"description\": \"min=0.244, mean=0.244, max=0.244, sum=0.488 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.24387328988975948\"}", - "College Computer Science - Observed inference time (s)": "{\"description\": \"min=0.216, mean=0.216, max=0.216, sum=0.433 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.21631600618362426\"}", - "College Mathematics - Observed inference time (s)": "{\"description\": \"min=0.219, mean=0.219, max=0.219, sum=0.437 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.21859397411346435\"}", - "College Medicine - Observed inference time (s)": "{\"description\": \"min=0.22, mean=0.22, max=0.22, sum=0.439 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.21971637665191826\"}", - "College Physics - Observed inference time (s)": "{\"description\": \"min=0.226, mean=0.226, max=0.226, sum=0.452 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.22610483683791816\"}", - "College Chemistry - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Chemistry - # prompt tokens": "{\"description\": \"min=549.28, mean=549.28, max=549.28, sum=1098.56 (2)\", \"tab\": \"General information\", \"score\": \"549.28\"}", - "College Chemistry - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # eval": "{\"description\": \"min=144, mean=144, max=144, sum=288 (2)\", \"tab\": \"General information\", \"score\": \"144.0\"}", - "College Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # prompt tokens": "{\"description\": \"min=473.875, mean=473.875, max=473.875, sum=947.75 (2)\", \"tab\": \"General information\", \"score\": \"473.875\"}", - "College Biology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # prompt tokens": "{\"description\": \"min=828.29, mean=828.29, max=828.29, sum=1656.58 (2)\", \"tab\": \"General information\", \"score\": \"828.29\"}", - "College Computer Science - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # prompt tokens": "{\"description\": \"min=594.51, mean=594.51, max=594.51, sum=1189.02 (2)\", \"tab\": \"General information\", \"score\": \"594.51\"}", - "College Mathematics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # eval": "{\"description\": \"min=173, mean=173, max=173, sum=346 (2)\", \"tab\": \"General information\", \"score\": \"173.0\"}", - "College Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # prompt tokens": "{\"description\": \"min=502.705, mean=502.705, max=502.705, sum=1005.41 (2)\", \"tab\": \"General information\", \"score\": \"502.70520231213874\"}", - "College Medicine - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # eval": "{\"description\": \"min=102, mean=102, max=102, sum=204 (2)\", \"tab\": \"General information\", \"score\": \"102.0\"}", - "College Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # prompt tokens": "{\"description\": \"min=503.569, mean=503.569, max=503.569, sum=1007.137 (2)\", \"tab\": \"General information\", \"score\": \"503.5686274509804\"}", - "College Physics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"college_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_college_physics\"" - } - } - }, - { - "evaluation_name": "Computer Security", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Computer Security", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.71, - "details": { - "description": "min=0.71, mean=0.71, max=0.71, sum=1.42 (2)", - "tab": "Accuracy", - "Computer Security - Observed inference time (s)": "{\"description\": \"min=0.469, mean=0.469, max=0.469, sum=0.938 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4692394161224365\"}", - "Computer Security - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Computer Security - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Computer Security - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Computer Security - # prompt tokens": "{\"description\": \"min=378.51, mean=378.51, max=378.51, sum=757.02 (2)\", \"tab\": \"General information\", \"score\": \"378.51\"}", - "Computer Security - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"computer_security\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_computer_security\"" - } - } - }, - { - "evaluation_name": "Econometrics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Econometrics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.395, - "details": { - "description": "min=0.395, mean=0.395, max=0.395, sum=0.789 (2)", - "tab": "Accuracy", - "Econometrics - Observed inference time (s)": "{\"description\": \"min=0.226, mean=0.226, max=0.226, sum=0.451 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.22570312023162842\"}", - "Econometrics - # eval": "{\"description\": \"min=114, mean=114, max=114, sum=228 (2)\", \"tab\": \"General information\", \"score\": \"114.0\"}", - "Econometrics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Econometrics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Econometrics - # prompt tokens": "{\"description\": \"min=614.421, mean=614.421, max=614.421, sum=1228.842 (2)\", \"tab\": \"General information\", \"score\": \"614.421052631579\"}", - "Econometrics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"econometrics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_econometrics\"" - } - } - }, - { - "evaluation_name": "Global Facts", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Global Facts", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.25, - "details": { - "description": "min=0.25, mean=0.25, max=0.25, sum=0.5 (2)", - "tab": "Accuracy", - "Global Facts - Observed inference time (s)": "{\"description\": \"min=0.249, mean=0.249, max=0.249, sum=0.497 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.24868298768997193\"}", - "Global Facts - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Global Facts - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Global Facts - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Global Facts - # prompt tokens": "{\"description\": \"min=399.71, mean=399.71, max=399.71, sum=799.42 (2)\", \"tab\": \"General information\", \"score\": \"399.71\"}", - "Global Facts - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"global_facts\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_global_facts\"" - } - } - }, - { - "evaluation_name": "Jurisprudence", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Jurisprudence", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.722, - "details": { - "description": "min=0.722, mean=0.722, max=0.722, sum=1.444 (2)", - "tab": "Accuracy", - "Jurisprudence - Observed inference time (s)": "{\"description\": \"min=0.204, mean=0.204, max=0.204, sum=0.409 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.20448691756637008\"}", - "Jurisprudence - # eval": "{\"description\": \"min=108, mean=108, max=108, sum=216 (2)\", \"tab\": \"General information\", \"score\": \"108.0\"}", - "Jurisprudence - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Jurisprudence - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Jurisprudence - # prompt tokens": "{\"description\": \"min=394.63, mean=394.63, max=394.63, sum=789.259 (2)\", \"tab\": \"General information\", \"score\": \"394.6296296296296\"}", - "Jurisprudence - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"jurisprudence\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_jurisprudence\"" - } - } - }, - { - "evaluation_name": "Philosophy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Philosophy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.646, - "details": { - "description": "min=0.646, mean=0.646, max=0.646, sum=1.293 (2)", - "tab": "Accuracy", - "Philosophy - Observed inference time (s)": "{\"description\": \"min=0.216, mean=0.216, max=0.216, sum=0.433 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.21639636628497452\"}", - "Philosophy - # eval": "{\"description\": \"min=311, mean=311, max=311, sum=622 (2)\", \"tab\": \"General information\", \"score\": \"311.0\"}", - "Philosophy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Philosophy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Philosophy - # prompt tokens": "{\"description\": \"min=329.084, mean=329.084, max=329.084, sum=658.167 (2)\", \"tab\": \"General information\", \"score\": \"329.08360128617363\"}", - "Philosophy - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"philosophy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_philosophy\"" - } - } - }, - { - "evaluation_name": "Professional Psychology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Professional Psychology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.649, - "details": { - "description": "min=0.649, mean=0.649, max=0.649, sum=1.297 (2)", - "tab": "Accuracy", - "Professional Medicine - Observed inference time (s)": "{\"description\": \"min=0.306, mean=0.306, max=0.306, sum=0.613 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.30631748893681693\"}", - "Professional Accounting - Observed inference time (s)": "{\"description\": \"min=0.236, mean=0.236, max=0.236, sum=0.472 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.23619349882112328\"}", - "Professional Law - Observed inference time (s)": "{\"description\": \"min=0.291, mean=0.291, max=0.291, sum=0.581 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2907135481940099\"}", - "Professional Psychology - Observed inference time (s)": "{\"description\": \"min=0.233, mean=0.233, max=0.233, sum=0.465 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.23272827988356545\"}", - "Professional Medicine - # eval": "{\"description\": \"min=272, mean=272, max=272, sum=544 (2)\", \"tab\": \"General information\", \"score\": \"272.0\"}", - "Professional Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Medicine - # prompt tokens": "{\"description\": \"min=1094.489, mean=1094.489, max=1094.489, sum=2188.978 (2)\", \"tab\": \"General information\", \"score\": \"1094.4889705882354\"}", - "Professional Medicine - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # eval": "{\"description\": \"min=282, mean=282, max=282, sum=564 (2)\", \"tab\": \"General information\", \"score\": \"282.0\"}", - "Professional Accounting - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Accounting - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # prompt tokens": "{\"description\": \"min=658.585, mean=658.585, max=658.585, sum=1317.17 (2)\", \"tab\": \"General information\", \"score\": \"658.5851063829788\"}", - "Professional Accounting - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # eval": "{\"description\": \"min=1534, mean=1534, max=1534, sum=3068 (2)\", \"tab\": \"General information\", \"score\": \"1534.0\"}", - "Professional Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # prompt tokens": "{\"description\": \"min=1637.601, mean=1637.601, max=1637.601, sum=3275.202 (2)\", \"tab\": \"General information\", \"score\": \"1637.6010430247718\"}", - "Professional Law - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # eval": "{\"description\": \"min=612, mean=612, max=612, sum=1224 (2)\", \"tab\": \"General information\", \"score\": \"612.0\"}", - "Professional Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # prompt tokens": "{\"description\": \"min=575.098, mean=575.098, max=575.098, sum=1150.196 (2)\", \"tab\": \"General information\", \"score\": \"575.0980392156863\"}", - "Professional Psychology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"professional_psychology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_professional_psychology\"" - } - } - }, - { - "evaluation_name": "Us Foreign Policy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Us Foreign Policy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.78, - "details": { - "description": "min=0.78, mean=0.78, max=0.78, sum=1.56 (2)", - "tab": "Accuracy", - "Us Foreign Policy - Observed inference time (s)": "{\"description\": \"min=0.383, mean=0.383, max=0.383, sum=0.765 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3825261640548706\"}", - "Us Foreign Policy - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Us Foreign Policy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Us Foreign Policy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Us Foreign Policy - # prompt tokens": "{\"description\": \"min=422.79, mean=422.79, max=422.79, sum=845.58 (2)\", \"tab\": \"General information\", \"score\": \"422.79\"}", - "Us Foreign Policy - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"us_foreign_policy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_us_foreign_policy\"" - } - } - }, - { - "evaluation_name": "Astronomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Astronomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.671, - "details": { - "description": "min=0.671, mean=0.671, max=0.671, sum=1.342 (2)", - "tab": "Accuracy", - "Astronomy - Observed inference time (s)": "{\"description\": \"min=0.249, mean=0.249, max=0.249, sum=0.497 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.24860012060717532\"}", - "Astronomy - # eval": "{\"description\": \"min=152, mean=152, max=152, sum=304 (2)\", \"tab\": \"General information\", \"score\": \"152.0\"}", - "Astronomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Astronomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Astronomy - # prompt tokens": "{\"description\": \"min=579.684, mean=579.684, max=579.684, sum=1159.368 (2)\", \"tab\": \"General information\", \"score\": \"579.6842105263158\"}", - "Astronomy - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"astronomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_astronomy\"" - } - } - }, - { - "evaluation_name": "Business Ethics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Business Ethics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.64, - "details": { - "description": "min=0.64, mean=0.64, max=0.64, sum=1.28 (2)", - "tab": "Accuracy", - "Business Ethics - Observed inference time (s)": "{\"description\": \"min=0.231, mean=0.231, max=0.231, sum=0.462 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.23080476760864257\"}", - "Business Ethics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Business Ethics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Business Ethics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Business Ethics - # prompt tokens": "{\"description\": \"min=569.52, mean=569.52, max=569.52, sum=1139.04 (2)\", \"tab\": \"General information\", \"score\": \"569.52\"}", - "Business Ethics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"business_ethics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_business_ethics\"" - } - } - }, - { - "evaluation_name": "Clinical Knowledge", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Clinical Knowledge", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.638, - "details": { - "description": "min=0.638, mean=0.638, max=0.638, sum=1.275 (2)", - "tab": "Accuracy", - "Clinical Knowledge - Observed inference time (s)": "{\"description\": \"min=0.23, mean=0.23, max=0.23, sum=0.46 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.22993840721418274\"}", - "Clinical Knowledge - # eval": "{\"description\": \"min=265, mean=265, max=265, sum=530 (2)\", \"tab\": \"General information\", \"score\": \"265.0\"}", - "Clinical Knowledge - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Clinical Knowledge - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Clinical Knowledge - # prompt tokens": "{\"description\": \"min=397.928, mean=397.928, max=397.928, sum=795.857 (2)\", \"tab\": \"General information\", \"score\": \"397.92830188679244\"}", - "Clinical Knowledge - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"clinical_knowledge\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_clinical_knowledge\"" - } - } - }, - { - "evaluation_name": "Conceptual Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Conceptual Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.536, - "details": { - "description": "min=0.536, mean=0.536, max=0.536, sum=1.072 (2)", - "tab": "Accuracy", - "Conceptual Physics - Observed inference time (s)": "{\"description\": \"min=0.221, mean=0.221, max=0.221, sum=0.441 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2206148127292065\"}", - "Conceptual Physics - # eval": "{\"description\": \"min=235, mean=235, max=235, sum=470 (2)\", \"tab\": \"General information\", \"score\": \"235.0\"}", - "Conceptual Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Conceptual Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Conceptual Physics - # prompt tokens": "{\"description\": \"min=304.834, mean=304.834, max=304.834, sum=609.668 (2)\", \"tab\": \"General information\", \"score\": \"304.83404255319147\"}", - "Conceptual Physics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"conceptual_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_conceptual_physics\"" - } - } - }, - { - "evaluation_name": "Electrical Engineering", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Electrical Engineering", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.51, - "details": { - "description": "min=0.51, mean=0.51, max=0.51, sum=1.021 (2)", - "tab": "Accuracy", - "Electrical Engineering - Observed inference time (s)": "{\"description\": \"min=0.231, mean=0.231, max=0.231, sum=0.461 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.23056076312887258\"}", - "Electrical Engineering - # eval": "{\"description\": \"min=145, mean=145, max=145, sum=290 (2)\", \"tab\": \"General information\", \"score\": \"145.0\"}", - "Electrical Engineering - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Electrical Engineering - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Electrical Engineering - # prompt tokens": "{\"description\": \"min=435.607, mean=435.607, max=435.607, sum=871.214 (2)\", \"tab\": \"General information\", \"score\": \"435.60689655172416\"}", - "Electrical Engineering - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"electrical_engineering\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_electrical_engineering\"" - } - } - }, - { - "evaluation_name": "Elementary Mathematics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Elementary Mathematics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.458, - "details": { - "description": "min=0.458, mean=0.458, max=0.458, sum=0.915 (2)", - "tab": "Accuracy", - "Elementary Mathematics - Observed inference time (s)": "{\"description\": \"min=0.224, mean=0.224, max=0.224, sum=0.447 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.22350322569488848\"}", - "Elementary Mathematics - # eval": "{\"description\": \"min=378, mean=378, max=378, sum=756 (2)\", \"tab\": \"General information\", \"score\": \"378.0\"}", - "Elementary Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Elementary Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Elementary Mathematics - # prompt tokens": "{\"description\": \"min=531.854, mean=531.854, max=531.854, sum=1063.709 (2)\", \"tab\": \"General information\", \"score\": \"531.8544973544973\"}", - "Elementary Mathematics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"elementary_mathematics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_elementary_mathematics\"" - } - } - }, - { - "evaluation_name": "Formal Logic", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Formal Logic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.46, - "details": { - "description": "min=0.46, mean=0.46, max=0.46, sum=0.921 (2)", - "tab": "Accuracy", - "Formal Logic - Observed inference time (s)": "{\"description\": \"min=0.229, mean=0.229, max=0.229, sum=0.458 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.22878488661750915\"}", - "Formal Logic - # eval": "{\"description\": \"min=126, mean=126, max=126, sum=252 (2)\", \"tab\": \"General information\", \"score\": \"126.0\"}", - "Formal Logic - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Formal Logic - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Formal Logic - # prompt tokens": "{\"description\": \"min=601.778, mean=601.778, max=601.778, sum=1203.556 (2)\", \"tab\": \"General information\", \"score\": \"601.7777777777778\"}", - "Formal Logic - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"formal_logic\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_formal_logic\"" - } - } - }, - { - "evaluation_name": "High School World History", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on High School World History", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.502, - "details": { - "description": "min=0.502, mean=0.502, max=0.502, sum=1.004 (2)", - "tab": "Accuracy", - "High School Biology - Observed inference time (s)": "{\"description\": \"min=0.225, mean=0.225, max=0.225, sum=0.449 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.22474505209153697\"}", - "High School Chemistry - Observed inference time (s)": "{\"description\": \"min=0.212, mean=0.212, max=0.212, sum=0.424 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.21204462192328694\"}", - "High School Computer Science - Observed inference time (s)": "{\"description\": \"min=0.23, mean=0.23, max=0.23, sum=0.461 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2303963828086853\"}", - "High School European History - Observed inference time (s)": "{\"description\": \"min=0.287, mean=0.287, max=0.287, sum=0.574 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.28706942760583126\"}", - "High School Geography - Observed inference time (s)": "{\"description\": \"min=0.229, mean=0.229, max=0.229, sum=0.458 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.22903898388448388\"}", - "High School Government And Politics - Observed inference time (s)": "{\"description\": \"min=0.234, mean=0.234, max=0.234, sum=0.469 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.23445281092984688\"}", - "High School Macroeconomics - Observed inference time (s)": "{\"description\": \"min=0.229, mean=0.229, max=0.229, sum=0.459 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.22930157551398644\"}", - "High School Mathematics - Observed inference time (s)": "{\"description\": \"min=0.23, mean=0.23, max=0.23, sum=0.46 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.23021557595994738\"}", - "High School Microeconomics - Observed inference time (s)": "{\"description\": \"min=0.235, mean=0.235, max=0.235, sum=0.471 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2354360087579038\"}", - "High School Physics - Observed inference time (s)": "{\"description\": \"min=0.229, mean=0.229, max=0.229, sum=0.458 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.22899133953827105\"}", - "High School Psychology - Observed inference time (s)": "{\"description\": \"min=0.227, mean=0.227, max=0.227, sum=0.454 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.22700285386601718\"}", - "High School Statistics - Observed inference time (s)": "{\"description\": \"min=0.24, mean=0.24, max=0.24, sum=0.48 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2400491248678278\"}", - "High School US History - Observed inference time (s)": "{\"description\": \"min=0.253, mean=0.253, max=0.253, sum=0.506 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2529456720632665\"}", - "High School World History - Observed inference time (s)": "{\"description\": \"min=0.25, mean=0.25, max=0.25, sum=0.499 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.249685173799217\"}", - "High School Biology - # eval": "{\"description\": \"min=310, mean=310, max=310, sum=620 (2)\", \"tab\": \"General information\", \"score\": \"310.0\"}", - "High School Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Biology - # prompt tokens": "{\"description\": \"min=513.671, mean=513.671, max=513.671, sum=1027.342 (2)\", \"tab\": \"General information\", \"score\": \"513.6709677419354\"}", - "High School Biology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # eval": "{\"description\": \"min=203, mean=203, max=203, sum=406 (2)\", \"tab\": \"General information\", \"score\": \"203.0\"}", - "High School Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # prompt tokens": "{\"description\": \"min=496.704, mean=496.704, max=496.704, sum=993.409 (2)\", \"tab\": \"General information\", \"score\": \"496.70443349753697\"}", - "High School Chemistry - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "High School Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # prompt tokens": "{\"description\": \"min=867.78, mean=867.78, max=867.78, sum=1735.56 (2)\", \"tab\": \"General information\", \"score\": \"867.78\"}", - "High School Computer Science - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # eval": "{\"description\": \"min=165, mean=165, max=165, sum=330 (2)\", \"tab\": \"General information\", \"score\": \"165.0\"}", - "High School European History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School European History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # prompt tokens": "{\"description\": \"min=2797.885, mean=2797.885, max=2797.885, sum=5595.77 (2)\", \"tab\": \"General information\", \"score\": \"2797.8848484848486\"}", - "High School European History - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # eval": "{\"description\": \"min=198, mean=198, max=198, sum=396 (2)\", \"tab\": \"General information\", \"score\": \"198.0\"}", - "High School Geography - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Geography - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # prompt tokens": "{\"description\": \"min=372.035, mean=372.035, max=372.035, sum=744.071 (2)\", \"tab\": \"General information\", \"score\": \"372.0353535353535\"}", - "High School Geography - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # eval": "{\"description\": \"min=193, mean=193, max=193, sum=386 (2)\", \"tab\": \"General information\", \"score\": \"193.0\"}", - "High School Government And Politics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Government And Politics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # prompt tokens": "{\"description\": \"min=465.824, mean=465.824, max=465.824, sum=931.648 (2)\", \"tab\": \"General information\", \"score\": \"465.8238341968912\"}", - "High School Government And Politics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # eval": "{\"description\": \"min=390, mean=390, max=390, sum=780 (2)\", \"tab\": \"General information\", \"score\": \"390.0\"}", - "High School Macroeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Macroeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # prompt tokens": "{\"description\": \"min=370.908, mean=370.908, max=370.908, sum=741.815 (2)\", \"tab\": \"General information\", \"score\": \"370.9076923076923\"}", - "High School Macroeconomics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # eval": "{\"description\": \"min=270, mean=270, max=270, sum=540 (2)\", \"tab\": \"General information\", \"score\": \"270.0\"}", - "High School Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # prompt tokens": "{\"description\": \"min=532.356, mean=532.356, max=532.356, sum=1064.711 (2)\", \"tab\": \"General information\", \"score\": \"532.3555555555556\"}", - "High School Mathematics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # eval": "{\"description\": \"min=238, mean=238, max=238, sum=476 (2)\", \"tab\": \"General information\", \"score\": \"238.0\"}", - "High School Microeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Microeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # prompt tokens": "{\"description\": \"min=399.013, mean=399.013, max=399.013, sum=798.025 (2)\", \"tab\": \"General information\", \"score\": \"399.0126050420168\"}", - "High School Microeconomics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # eval": "{\"description\": \"min=151, mean=151, max=151, sum=302 (2)\", \"tab\": \"General information\", \"score\": \"151.0\"}", - "High School Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # prompt tokens": "{\"description\": \"min=560.457, mean=560.457, max=560.457, sum=1120.914 (2)\", \"tab\": \"General information\", \"score\": \"560.4569536423841\"}", - "High School Physics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # eval": "{\"description\": \"min=545, mean=545, max=545, sum=1090 (2)\", \"tab\": \"General information\", \"score\": \"545.0\"}", - "High School Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # prompt tokens": "{\"description\": \"min=495.242, mean=495.242, max=495.242, sum=990.484 (2)\", \"tab\": \"General information\", \"score\": \"495.2422018348624\"}", - "High School Psychology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # eval": "{\"description\": \"min=216, mean=216, max=216, sum=432 (2)\", \"tab\": \"General information\", \"score\": \"216.0\"}", - "High School Statistics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Statistics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # prompt tokens": "{\"description\": \"min=795.639, mean=795.639, max=795.639, sum=1591.278 (2)\", \"tab\": \"General information\", \"score\": \"795.6388888888889\"}", - "High School Statistics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # eval": "{\"description\": \"min=204, mean=204, max=204, sum=408 (2)\", \"tab\": \"General information\", \"score\": \"204.0\"}", - "High School US History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School US History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # prompt tokens": "{\"description\": \"min=2217.809, mean=2217.809, max=2217.809, sum=4435.618 (2)\", \"tab\": \"General information\", \"score\": \"2217.8088235294117\"}", - "High School US History - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # eval": "{\"description\": \"min=237, mean=237, max=237, sum=474 (2)\", \"tab\": \"General information\", \"score\": \"237.0\"}", - "High School World History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School World History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # prompt tokens": "{\"description\": \"min=1428.173, mean=1428.173, max=1428.173, sum=2856.346 (2)\", \"tab\": \"General information\", \"score\": \"1428.1729957805908\"}", - "High School World History - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"high_school_world_history\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_high_school_world_history\"" - } - } - }, - { - "evaluation_name": "Human Sexuality", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Human Sexuality", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.763, - "details": { - "description": "min=0.763, mean=0.763, max=0.763, sum=1.527 (2)", - "tab": "Accuracy", - "Human Aging - Observed inference time (s)": "{\"description\": \"min=0.322, mean=0.322, max=0.322, sum=0.645 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.32235514315789054\"}", - "Human Sexuality - Observed inference time (s)": "{\"description\": \"min=0.245, mean=0.245, max=0.245, sum=0.49 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.24487258095777673\"}", - "Human Aging - # eval": "{\"description\": \"min=223, mean=223, max=223, sum=446 (2)\", \"tab\": \"General information\", \"score\": \"223.0\"}", - "Human Aging - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Aging - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Aging - # prompt tokens": "{\"description\": \"min=319.888, mean=319.888, max=319.888, sum=639.776 (2)\", \"tab\": \"General information\", \"score\": \"319.88789237668163\"}", - "Human Aging - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # eval": "{\"description\": \"min=131, mean=131, max=131, sum=262 (2)\", \"tab\": \"General information\", \"score\": \"131.0\"}", - "Human Sexuality - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Sexuality - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # prompt tokens": "{\"description\": \"min=341.168, mean=341.168, max=341.168, sum=682.336 (2)\", \"tab\": \"General information\", \"score\": \"341.1679389312977\"}", - "Human Sexuality - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"human_sexuality\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_human_sexuality\"" - } - } - }, - { - "evaluation_name": "International Law", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on International Law", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.711, - "details": { - "description": "min=0.711, mean=0.711, max=0.711, sum=1.421 (2)", - "tab": "Accuracy", - "International Law - Observed inference time (s)": "{\"description\": \"min=0.231, mean=0.231, max=0.231, sum=0.462 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.23109814943360887\"}", - "International Law - # eval": "{\"description\": \"min=121, mean=121, max=121, sum=242 (2)\", \"tab\": \"General information\", \"score\": \"121.0\"}", - "International Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "International Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "International Law - # prompt tokens": "{\"description\": \"min=639.818, mean=639.818, max=639.818, sum=1279.636 (2)\", \"tab\": \"General information\", \"score\": \"639.8181818181819\"}", - "International Law - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"international_law\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_international_law\"" - } - } - }, - { - "evaluation_name": "Logical Fallacies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Logical Fallacies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.742, - "details": { - "description": "min=0.742, mean=0.742, max=0.742, sum=1.485 (2)", - "tab": "Accuracy", - "Logical Fallacies - Observed inference time (s)": "{\"description\": \"min=0.22, mean=0.22, max=0.22, sum=0.44 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.21997687714231526\"}", - "Logical Fallacies - # eval": "{\"description\": \"min=163, mean=163, max=163, sum=326 (2)\", \"tab\": \"General information\", \"score\": \"163.0\"}", - "Logical Fallacies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Logical Fallacies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Logical Fallacies - # prompt tokens": "{\"description\": \"min=449.564, mean=449.564, max=449.564, sum=899.129 (2)\", \"tab\": \"General information\", \"score\": \"449.5644171779141\"}", - "Logical Fallacies - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"logical_fallacies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_logical_fallacies\"" - } - } - }, - { - "evaluation_name": "Machine Learning", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Machine Learning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.375, - "details": { - "description": "min=0.375, mean=0.375, max=0.375, sum=0.75 (2)", - "tab": "Accuracy", - "Machine Learning - Observed inference time (s)": "{\"description\": \"min=0.234, mean=0.234, max=0.234, sum=0.467 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2336032326732363\"}", - "Machine Learning - # eval": "{\"description\": \"min=112, mean=112, max=112, sum=224 (2)\", \"tab\": \"General information\", \"score\": \"112.0\"}", - "Machine Learning - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Machine Learning - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Machine Learning - # prompt tokens": "{\"description\": \"min=668.054, mean=668.054, max=668.054, sum=1336.107 (2)\", \"tab\": \"General information\", \"score\": \"668.0535714285714\"}", - "Machine Learning - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"machine_learning\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_machine_learning\"" - } - } - }, - { - "evaluation_name": "Management", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Management", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.728, - "details": { - "description": "min=0.728, mean=0.728, max=0.728, sum=1.456 (2)", - "tab": "Accuracy", - "Management - Observed inference time (s)": "{\"description\": \"min=0.246, mean=0.246, max=0.246, sum=0.491 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.24564221067335998\"}", - "Management - # eval": "{\"description\": \"min=103, mean=103, max=103, sum=206 (2)\", \"tab\": \"General information\", \"score\": \"103.0\"}", - "Management - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Management - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Management - # prompt tokens": "{\"description\": \"min=283.786, mean=283.786, max=283.786, sum=567.573 (2)\", \"tab\": \"General information\", \"score\": \"283.7864077669903\"}", - "Management - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"management\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_management\"" - } - } - }, - { - "evaluation_name": "Marketing", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Marketing", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.838, - "details": { - "description": "min=0.838, mean=0.838, max=0.838, sum=1.675 (2)", - "tab": "Accuracy", - "Marketing - Observed inference time (s)": "{\"description\": \"min=0.269, mean=0.269, max=0.269, sum=0.537 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.26863190149649596\"}", - "Marketing - # eval": "{\"description\": \"min=234, mean=234, max=234, sum=468 (2)\", \"tab\": \"General information\", \"score\": \"234.0\"}", - "Marketing - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Marketing - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Marketing - # prompt tokens": "{\"description\": \"min=404.218, mean=404.218, max=404.218, sum=808.436 (2)\", \"tab\": \"General information\", \"score\": \"404.21794871794873\"}", - "Marketing - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"marketing\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_marketing\"" - } - } - }, - { - "evaluation_name": "Medical Genetics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Medical Genetics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7, - "details": { - "description": "min=0.7, mean=0.7, max=0.7, sum=1.4 (2)", - "tab": "Accuracy", - "Medical Genetics - Observed inference time (s)": "{\"description\": \"min=0.273, mean=0.273, max=0.273, sum=0.546 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2728374266624451\"}", - "Medical Genetics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Medical Genetics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Medical Genetics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Medical Genetics - # prompt tokens": "{\"description\": \"min=340.99, mean=340.99, max=340.99, sum=681.98 (2)\", \"tab\": \"General information\", \"score\": \"340.99\"}", - "Medical Genetics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"medical_genetics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_medical_genetics\"" - } - } - }, - { - "evaluation_name": "Miscellaneous", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Miscellaneous", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.644, - "details": { - "description": "min=0.644, mean=0.644, max=0.644, sum=1.287 (2)", - "tab": "Accuracy", - "Miscellaneous - Observed inference time (s)": "{\"description\": \"min=0.336, mean=0.336, max=0.336, sum=0.673 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.33641790095264734\"}", - "Miscellaneous - # eval": "{\"description\": \"min=783, mean=783, max=783, sum=1566 (2)\", \"tab\": \"General information\", \"score\": \"783.0\"}", - "Miscellaneous - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Miscellaneous - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Miscellaneous - # prompt tokens": "{\"description\": \"min=299.911, mean=299.911, max=299.911, sum=599.821 (2)\", \"tab\": \"General information\", \"score\": \"299.9106002554278\"}", - "Miscellaneous - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"miscellaneous\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_miscellaneous\"" - } - } - }, - { - "evaluation_name": "Moral Scenarios", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Moral Scenarios", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.328, - "details": { - "description": "min=0.328, mean=0.328, max=0.328, sum=0.657 (2)", - "tab": "Accuracy", - "Moral Disputes - Observed inference time (s)": "{\"description\": \"min=0.243, mean=0.243, max=0.243, sum=0.486 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.24306911126726624\"}", - "Moral Scenarios - Observed inference time (s)": "{\"description\": \"min=0.229, mean=0.229, max=0.229, sum=0.458 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2289134478435836\"}", - "Moral Disputes - # eval": "{\"description\": \"min=346, mean=346, max=346, sum=692 (2)\", \"tab\": \"General information\", \"score\": \"346.0\"}", - "Moral Disputes - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Disputes - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Disputes - # prompt tokens": "{\"description\": \"min=476.113, mean=476.113, max=476.113, sum=952.225 (2)\", \"tab\": \"General information\", \"score\": \"476.1127167630058\"}", - "Moral Disputes - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # eval": "{\"description\": \"min=895, mean=895, max=895, sum=1790 (2)\", \"tab\": \"General information\", \"score\": \"895.0\"}", - "Moral Scenarios - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Scenarios - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # prompt tokens": "{\"description\": \"min=656.455, mean=656.455, max=656.455, sum=1312.909 (2)\", \"tab\": \"General information\", \"score\": \"656.454748603352\"}", - "Moral Scenarios - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"moral_scenarios\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_moral_scenarios\"" - } - } - }, - { - "evaluation_name": "Nutrition", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Nutrition", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.752, - "details": { - "description": "min=0.752, mean=0.752, max=0.752, sum=1.503 (2)", - "tab": "Accuracy", - "Nutrition - Observed inference time (s)": "{\"description\": \"min=0.24, mean=0.24, max=0.24, sum=0.48 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2399757040871514\"}", - "Nutrition - # eval": "{\"description\": \"min=306, mean=306, max=306, sum=612 (2)\", \"tab\": \"General information\", \"score\": \"306.0\"}", - "Nutrition - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Nutrition - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Nutrition - # prompt tokens": "{\"description\": \"min=586.814, mean=586.814, max=586.814, sum=1173.627 (2)\", \"tab\": \"General information\", \"score\": \"586.8137254901961\"}", - "Nutrition - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"nutrition\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_nutrition\"" - } - } - }, - { - "evaluation_name": "Prehistory", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Prehistory", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.744, - "details": { - "description": "min=0.744, mean=0.744, max=0.744, sum=1.488 (2)", - "tab": "Accuracy", - "Prehistory - Observed inference time (s)": "{\"description\": \"min=0.229, mean=0.229, max=0.229, sum=0.457 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2287170680952661\"}", - "Prehistory - # eval": "{\"description\": \"min=324, mean=324, max=324, sum=648 (2)\", \"tab\": \"General information\", \"score\": \"324.0\"}", - "Prehistory - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Prehistory - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Prehistory - # prompt tokens": "{\"description\": \"min=514.528, mean=514.528, max=514.528, sum=1029.056 (2)\", \"tab\": \"General information\", \"score\": \"514.5277777777778\"}", - "Prehistory - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"prehistory\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_prehistory\"" - } - } - }, - { - "evaluation_name": "Public Relations", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Public Relations", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.645, - "details": { - "description": "min=0.645, mean=0.645, max=0.645, sum=1.291 (2)", - "tab": "Accuracy", - "Public Relations - Observed inference time (s)": "{\"description\": \"min=0.246, mean=0.246, max=0.246, sum=0.491 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.24565653367476029\"}", - "Public Relations - # eval": "{\"description\": \"min=110, mean=110, max=110, sum=220 (2)\", \"tab\": \"General information\", \"score\": \"110.0\"}", - "Public Relations - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Public Relations - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Public Relations - # prompt tokens": "{\"description\": \"min=405.318, mean=405.318, max=405.318, sum=810.636 (2)\", \"tab\": \"General information\", \"score\": \"405.3181818181818\"}", - "Public Relations - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"public_relations\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_public_relations\"" - } - } - }, - { - "evaluation_name": "Security Studies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Security Studies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.567, - "details": { - "description": "min=0.567, mean=0.567, max=0.567, sum=1.135 (2)", - "tab": "Accuracy", - "Security Studies - Observed inference time (s)": "{\"description\": \"min=0.253, mean=0.253, max=0.253, sum=0.506 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.25285910878862655\"}", - "Security Studies - # eval": "{\"description\": \"min=245, mean=245, max=245, sum=490 (2)\", \"tab\": \"General information\", \"score\": \"245.0\"}", - "Security Studies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Security Studies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Security Studies - # prompt tokens": "{\"description\": \"min=1164.473, mean=1164.473, max=1164.473, sum=2328.947 (2)\", \"tab\": \"General information\", \"score\": \"1164.4734693877551\"}", - "Security Studies - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"security_studies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_security_studies\"" - } - } - }, - { - "evaluation_name": "Sociology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Sociology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.627, - "details": { - "description": "min=0.627, mean=0.627, max=0.627, sum=1.254 (2)", - "tab": "Accuracy", - "Sociology - Observed inference time (s)": "{\"description\": \"min=0.234, mean=0.234, max=0.234, sum=0.468 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.23380224503094876\"}", - "Sociology - # eval": "{\"description\": \"min=201, mean=201, max=201, sum=402 (2)\", \"tab\": \"General information\", \"score\": \"201.0\"}", - "Sociology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Sociology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Sociology - # prompt tokens": "{\"description\": \"min=445.517, mean=445.517, max=445.517, sum=891.035 (2)\", \"tab\": \"General information\", \"score\": \"445.51741293532336\"}", - "Sociology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"sociology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_sociology\"" - } - } - }, - { - "evaluation_name": "Virology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Virology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.446, - "details": { - "description": "min=0.446, mean=0.446, max=0.446, sum=0.892 (2)", - "tab": "Accuracy", - "Virology - Observed inference time (s)": "{\"description\": \"min=0.223, mean=0.223, max=0.223, sum=0.447 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.22334270161318492\"}", - "Virology - # eval": "{\"description\": \"min=166, mean=166, max=166, sum=332 (2)\", \"tab\": \"General information\", \"score\": \"166.0\"}", - "Virology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Virology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Virology - # prompt tokens": "{\"description\": \"min=343.018, mean=343.018, max=343.018, sum=686.036 (2)\", \"tab\": \"General information\", \"score\": \"343.01807228915663\"}", - "Virology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"virology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_virology\"" - } - } - }, - { - "evaluation_name": "World Religions", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on World Religions", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.696, - "details": { - "description": "min=0.696, mean=0.696, max=0.696, sum=1.392 (2)", - "tab": "Accuracy", - "World Religions - Observed inference time (s)": "{\"description\": \"min=0.239, mean=0.239, max=0.239, sum=0.478 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.23875254357767384\"}", - "World Religions - # eval": "{\"description\": \"min=171, mean=171, max=171, sum=342 (2)\", \"tab\": \"General information\", \"score\": \"171.0\"}", - "World Religions - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "World Religions - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "World Religions - # prompt tokens": "{\"description\": \"min=274.52, mean=274.52, max=274.52, sum=549.041 (2)\", \"tab\": \"General information\", \"score\": \"274.5204678362573\"}", - "World Religions - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"world_religions\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_world_religions\"" - } - } - }, - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.897, - "details": { - "description": "", - "tab": "Efficiency" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/meta_llama-3.2-90b-vision-instruct-turbo.json b/data/models/meta_llama-3.2-90b-vision-instruct-turbo.json deleted file mode 100644 index d30ec26b3cfffb3a8ce0593911166de1a589585a..0000000000000000000000000000000000000000 --- a/data/models/meta_llama-3.2-90b-vision-instruct-turbo.json +++ /dev/null @@ -1,1903 +0,0 @@ -{ - "model_info": { - "name": "Llama 3.2 Vision Instruct Turbo 90B", - "id": "meta/llama-3.2-90b-vision-instruct-turbo", - "developer": "Meta", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_lite/meta_llama-3.2-90b-vision-instruct-turbo/1774096306.427425", - "retrieved_timestamp": "1774096306.427425", - "source_metadata": { - "source_name": "helm_lite", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_lite", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_lite", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.819, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.5839825218476904\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.777, - "details": { - "description": "min=0.777, mean=0.777, max=0.777, sum=0.777 (1)", - "tab": "Accuracy", - "NarrativeQA - Observed inference time (s)": "{\"description\": \"min=0.83, mean=0.83, max=0.83, sum=0.83 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.8297326531208736\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=3484.268, mean=3484.268, max=3484.268, sum=3484.268 (1)\", \"tab\": \"General information\", \"score\": \"3484.2676056338028\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (closed-book)", - "source_data": { - "dataset_name": "NaturalQuestions (closed-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (closed-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.457, - "details": { - "description": "min=0.457, mean=0.457, max=0.457, sum=0.457 (1)", - "tab": "Accuracy", - "NaturalQuestions (open-book) - Observed inference time (s)": "{\"description\": \"min=1.111, mean=1.111, max=1.111, sum=1.111 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.110703297138214\"}", - "NaturalQuestions (closed-book) - Observed inference time (s)": "{\"description\": \"min=0.422, mean=0.422, max=0.422, sum=0.422 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.4218848171234131\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1716.785, mean=1716.785, max=1716.785, sum=1716.785 (1)\", \"tab\": \"General information\", \"score\": \"1716.785\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=129.12, mean=129.12, max=129.12, sum=129.12 (1)\", \"tab\": \"General information\", \"score\": \"129.12\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "mode": "\"closedbook\"" - } - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.942, - "details": { - "description": "min=0.942, mean=0.942, max=0.942, sum=0.942 (1)", - "tab": "Accuracy", - "OpenbookQA - Observed inference time (s)": "{\"description\": \"min=0.285, mean=0.285, max=0.285, sum=0.285 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.28476666021347047\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=249.776, mean=249.776, max=249.776, sum=249.776 (1)\", \"tab\": \"General information\", \"score\": \"249.776\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "dataset": "\"openbookqa\"", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.703, - "details": { - "description": "min=0.52, mean=0.703, max=0.93, sum=3.514 (5)", - "tab": "Accuracy", - "MMLU - Observed inference time (s)": "{\"description\": \"min=0.266, mean=0.798, max=2.612, sum=3.992 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.7984467656654225\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=373.43, mean=467.686, max=614.421, sum=2338.431 (5)\", \"tab\": \"General information\", \"score\": \"467.6862105263158\"}", - "MMLU - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"college_chemistry\", \"computer_security\", \"econometrics\", \"us_foreign_policy\"]", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MATH", - "source_data": { - "dataset_name": "MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Equivalent (CoT) on MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.791, - "details": { - "description": "min=0.579, mean=0.791, max=0.978, sum=5.54 (7)", - "tab": "Accuracy", - "MATH - Observed inference time (s)": "{\"description\": \"min=4.64, mean=5.739, max=6.652, sum=40.174 (7)\", \"tab\": \"Efficiency\", \"score\": \"5.739186799526185\"}", - "MATH - # eval": "{\"description\": \"min=30, mean=62.429, max=135, sum=437 (7)\", \"tab\": \"General information\", \"score\": \"62.42857142857143\"}", - "MATH - # train": "{\"description\": \"min=8, mean=8, max=8, sum=56 (7)\", \"tab\": \"General information\", \"score\": \"8.0\"}", - "MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MATH - # prompt tokens": "{\"description\": \"min=881.363, mean=1262.909, max=2197.577, sum=8840.364 (7)\", \"tab\": \"General information\", \"score\": \"1262.9092130545007\"}", - "MATH - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"algebra\", \"counting_and_probability\", \"geometry\", \"intermediate_algebra\", \"number_theory\", \"prealgebra\", \"precalculus\"]", - "level": "\"1\"", - "use_official_examples": "\"False\"", - "use_chain_of_thought": "\"True\"" - } - } - }, - { - "evaluation_name": "GSM8K", - "source_data": { - "dataset_name": "GSM8K", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on GSM8K", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.936, - "details": { - "description": "min=0.936, mean=0.936, max=0.936, sum=0.936 (1)", - "tab": "Accuracy", - "GSM8K - Observed inference time (s)": "{\"description\": \"min=2.889, mean=2.889, max=2.889, sum=2.889 (1)\", \"tab\": \"Efficiency\", \"score\": \"2.8894128675460817\"}", - "GSM8K - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "GSM8K - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "GSM8K - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GSM8K - # prompt tokens": "{\"description\": \"min=959.032, mean=959.032, max=959.032, sum=959.032 (1)\", \"tab\": \"General information\", \"score\": \"959.032\"}", - "GSM8K - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "stop": "\"none\"" - } - } - }, - { - "evaluation_name": "LegalBench", - "source_data": { - "dataset_name": "LegalBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on LegalBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.68, - "details": { - "description": "min=0.438, mean=0.68, max=0.989, sum=3.398 (5)", - "tab": "Accuracy", - "LegalBench - Observed inference time (s)": "{\"description\": \"min=0.284, mean=0.478, max=1.152, sum=2.389 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.47773526830658064\"}", - "LegalBench - # eval": "{\"description\": \"min=95, mean=409.4, max=1000, sum=2047 (5)\", \"tab\": \"General information\", \"score\": \"409.4\"}", - "LegalBench - # train": "{\"description\": \"min=4, mean=4.8, max=5, sum=24 (5)\", \"tab\": \"General information\", \"score\": \"4.8\"}", - "LegalBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "LegalBench - # prompt tokens": "{\"description\": \"min=197.442, mean=1513.882, max=6300.012, sum=7569.412 (5)\", \"tab\": \"General information\", \"score\": \"1513.8824197238912\"}", - "LegalBench - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"abercrombie\", \"corporate_lobbying\", \"function_of_decision_section\", \"international_citizenship_questions\", \"proa\"]" - } - } - }, - { - "evaluation_name": "MedQA", - "source_data": { - "dataset_name": "MedQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MedQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.769, - "details": { - "description": "min=0.769, mean=0.769, max=0.769, sum=0.769 (1)", - "tab": "Accuracy", - "MedQA - Observed inference time (s)": "{\"description\": \"min=0.318, mean=0.318, max=0.318, sum=0.318 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.3180293652930743\"}", - "MedQA - # eval": "{\"description\": \"min=503, mean=503, max=503, sum=503 (1)\", \"tab\": \"General information\", \"score\": \"503.0\"}", - "MedQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MedQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MedQA - # prompt tokens": "{\"description\": \"min=1025.274, mean=1025.274, max=1025.274, sum=1025.274 (1)\", \"tab\": \"General information\", \"score\": \"1025.2743538767395\"}", - "MedQA - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WMT 2014", - "source_data": { - "dataset_name": "WMT 2014", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "BLEU-4 on WMT 2014", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.224, - "details": { - "description": "min=0.182, mean=0.224, max=0.266, sum=1.121 (5)", - "tab": "Accuracy", - "WMT 2014 - Observed inference time (s)": "{\"description\": \"min=0.737, mean=0.816, max=0.848, sum=4.078 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.8156762526912515\"}", - "WMT 2014 - # eval": "{\"description\": \"min=503, mean=568.8, max=832, sum=2844 (5)\", \"tab\": \"General information\", \"score\": \"568.8\"}", - "WMT 2014 - # train": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "WMT 2014 - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WMT 2014 - # prompt tokens": "{\"description\": \"min=101.139, mean=120.868, max=141.33, sum=604.34 (5)\", \"tab\": \"General information\", \"score\": \"120.86804366111025\"}", - "WMT 2014 - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "language_pair": "[\"cs-en\", \"de-en\", \"fr-en\", \"hi-en\", \"ru-en\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_mmlu/meta_llama-3.2-90b-vision-instruct-turbo/1774096312.00548", - "retrieved_timestamp": "1774096312.00548", - "source_metadata": { - "source_name": "helm_mmlu", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_mmlu", - "evaluation_results": [ - { - "evaluation_name": "MMLU All Subjects", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU All Subjects", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.803, - "details": { - "description": "min=0.407, mean=0.803, max=0.979, sum=91.503 (114)", - "tab": "Accuracy", - "MMLU All Subjects - Observed inference time (s)": "{\"description\": \"min=0.256, mean=0.374, max=2.612, sum=42.58 (114)\", \"tab\": \"Efficiency\", \"score\": \"0.37350966276831277\"}", - "MMLU All Subjects - # eval": "{\"description\": \"min=100, mean=246.351, max=1534, sum=28084 (114)\", \"tab\": \"General information\", \"score\": \"246.35087719298247\"}", - "MMLU All Subjects - # train": "{\"description\": \"min=5, mean=5, max=5, sum=570 (114)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU All Subjects - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU All Subjects - # prompt tokens": "{\"description\": \"min=274.52, mean=614.619, max=2797.885, sum=70066.61 (114)\", \"tab\": \"General information\", \"score\": \"614.6193817308517\"}", - "MMLU All Subjects - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_name": "Abstract Algebra", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Abstract Algebra", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.52, - "details": { - "description": "min=0.52, mean=0.52, max=0.52, sum=1.04 (2)", - "tab": "Accuracy", - "Abstract Algebra - Observed inference time (s)": "{\"description\": \"min=2.612, mean=2.612, max=2.612, sum=5.224 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.611864836215973\"}", - "Abstract Algebra - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Abstract Algebra - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Abstract Algebra - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Abstract Algebra - # prompt tokens": "{\"description\": \"min=373.43, mean=373.43, max=373.43, sum=746.86 (2)\", \"tab\": \"General information\", \"score\": \"373.43\"}", - "Abstract Algebra - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"abstract_algebra\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_abstract_algebra\"" - } - } - }, - { - "evaluation_name": "Anatomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Anatomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8, - "details": { - "description": "min=0.8, mean=0.8, max=0.8, sum=1.6 (2)", - "tab": "Accuracy", - "Anatomy - Observed inference time (s)": "{\"description\": \"min=0.336, mean=0.336, max=0.336, sum=0.672 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3359027315069128\"}", - "Anatomy - # eval": "{\"description\": \"min=135, mean=135, max=135, sum=270 (2)\", \"tab\": \"General information\", \"score\": \"135.0\"}", - "Anatomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Anatomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Anatomy - # prompt tokens": "{\"description\": \"min=353.874, mean=353.874, max=353.874, sum=707.748 (2)\", \"tab\": \"General information\", \"score\": \"353.8740740740741\"}", - "Anatomy - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"anatomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_anatomy\"" - } - } - }, - { - "evaluation_name": "College Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on College Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.539, - "details": { - "description": "min=0.539, mean=0.539, max=0.539, sum=1.078 (2)", - "tab": "Accuracy", - "College Chemistry - Observed inference time (s)": "{\"description\": \"min=0.31, mean=0.31, max=0.31, sum=0.621 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3104448890686035\"}", - "College Biology - Observed inference time (s)": "{\"description\": \"min=0.272, mean=0.272, max=0.272, sum=0.544 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2720499005582597\"}", - "College Computer Science - Observed inference time (s)": "{\"description\": \"min=0.321, mean=0.321, max=0.321, sum=0.642 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.32119542360305786\"}", - "College Mathematics - Observed inference time (s)": "{\"description\": \"min=0.315, mean=0.315, max=0.315, sum=0.63 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.31477957487106323\"}", - "College Medicine - Observed inference time (s)": "{\"description\": \"min=0.283, mean=0.283, max=0.283, sum=0.566 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.28313319255850905\"}", - "College Physics - Observed inference time (s)": "{\"description\": \"min=0.317, mean=0.317, max=0.317, sum=0.634 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.31692570097306194\"}", - "College Chemistry - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Chemistry - # prompt tokens": "{\"description\": \"min=549.28, mean=549.28, max=549.28, sum=1098.56 (2)\", \"tab\": \"General information\", \"score\": \"549.28\"}", - "College Chemistry - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # eval": "{\"description\": \"min=144, mean=144, max=144, sum=288 (2)\", \"tab\": \"General information\", \"score\": \"144.0\"}", - "College Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # prompt tokens": "{\"description\": \"min=473.875, mean=473.875, max=473.875, sum=947.75 (2)\", \"tab\": \"General information\", \"score\": \"473.875\"}", - "College Biology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # prompt tokens": "{\"description\": \"min=828.29, mean=828.29, max=828.29, sum=1656.58 (2)\", \"tab\": \"General information\", \"score\": \"828.29\"}", - "College Computer Science - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # prompt tokens": "{\"description\": \"min=594.51, mean=594.51, max=594.51, sum=1189.02 (2)\", \"tab\": \"General information\", \"score\": \"594.51\"}", - "College Mathematics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # eval": "{\"description\": \"min=173, mean=173, max=173, sum=346 (2)\", \"tab\": \"General information\", \"score\": \"173.0\"}", - "College Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # prompt tokens": "{\"description\": \"min=502.705, mean=502.705, max=502.705, sum=1005.41 (2)\", \"tab\": \"General information\", \"score\": \"502.70520231213874\"}", - "College Medicine - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # eval": "{\"description\": \"min=102, mean=102, max=102, sum=204 (2)\", \"tab\": \"General information\", \"score\": \"102.0\"}", - "College Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # prompt tokens": "{\"description\": \"min=503.569, mean=503.569, max=503.569, sum=1007.137 (2)\", \"tab\": \"General information\", \"score\": \"503.5686274509804\"}", - "College Physics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"college_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_college_physics\"" - } - } - }, - { - "evaluation_name": "Computer Security", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Computer Security", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.81, - "details": { - "description": "min=0.81, mean=0.81, max=0.81, sum=1.62 (2)", - "tab": "Accuracy", - "Computer Security - Observed inference time (s)": "{\"description\": \"min=0.266, mean=0.266, max=0.266, sum=0.532 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.26576273441314696\"}", - "Computer Security - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Computer Security - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Computer Security - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Computer Security - # prompt tokens": "{\"description\": \"min=378.51, mean=378.51, max=378.51, sum=757.02 (2)\", \"tab\": \"General information\", \"score\": \"378.51\"}", - "Computer Security - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"computer_security\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_computer_security\"" - } - } - }, - { - "evaluation_name": "Econometrics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Econometrics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.684, - "details": { - "description": "min=0.684, mean=0.684, max=0.684, sum=1.368 (2)", - "tab": "Accuracy", - "Econometrics - Observed inference time (s)": "{\"description\": \"min=0.297, mean=0.297, max=0.297, sum=0.595 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2972530210227297\"}", - "Econometrics - # eval": "{\"description\": \"min=114, mean=114, max=114, sum=228 (2)\", \"tab\": \"General information\", \"score\": \"114.0\"}", - "Econometrics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Econometrics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Econometrics - # prompt tokens": "{\"description\": \"min=614.421, mean=614.421, max=614.421, sum=1228.842 (2)\", \"tab\": \"General information\", \"score\": \"614.421052631579\"}", - "Econometrics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"econometrics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_econometrics\"" - } - } - }, - { - "evaluation_name": "Global Facts", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Global Facts", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6, - "details": { - "description": "min=0.6, mean=0.6, max=0.6, sum=1.2 (2)", - "tab": "Accuracy", - "Global Facts - Observed inference time (s)": "{\"description\": \"min=0.267, mean=0.267, max=0.267, sum=0.533 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2666162133216858\"}", - "Global Facts - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Global Facts - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Global Facts - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Global Facts - # prompt tokens": "{\"description\": \"min=399.71, mean=399.71, max=399.71, sum=799.42 (2)\", \"tab\": \"General information\", \"score\": \"399.71\"}", - "Global Facts - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"global_facts\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_global_facts\"" - } - } - }, - { - "evaluation_name": "Jurisprudence", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Jurisprudence", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.88, - "details": { - "description": "min=0.88, mean=0.88, max=0.88, sum=1.759 (2)", - "tab": "Accuracy", - "Jurisprudence - Observed inference time (s)": "{\"description\": \"min=0.279, mean=0.279, max=0.279, sum=0.558 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.278864703796528\"}", - "Jurisprudence - # eval": "{\"description\": \"min=108, mean=108, max=108, sum=216 (2)\", \"tab\": \"General information\", \"score\": \"108.0\"}", - "Jurisprudence - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Jurisprudence - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Jurisprudence - # prompt tokens": "{\"description\": \"min=394.63, mean=394.63, max=394.63, sum=789.259 (2)\", \"tab\": \"General information\", \"score\": \"394.6296296296296\"}", - "Jurisprudence - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"jurisprudence\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_jurisprudence\"" - } - } - }, - { - "evaluation_name": "Philosophy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Philosophy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.839, - "details": { - "description": "min=0.839, mean=0.839, max=0.839, sum=1.678 (2)", - "tab": "Accuracy", - "Philosophy - Observed inference time (s)": "{\"description\": \"min=0.297, mean=0.297, max=0.297, sum=0.594 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.29689135582117404\"}", - "Philosophy - # eval": "{\"description\": \"min=311, mean=311, max=311, sum=622 (2)\", \"tab\": \"General information\", \"score\": \"311.0\"}", - "Philosophy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Philosophy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Philosophy - # prompt tokens": "{\"description\": \"min=329.084, mean=329.084, max=329.084, sum=658.167 (2)\", \"tab\": \"General information\", \"score\": \"329.08360128617363\"}", - "Philosophy - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"philosophy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_philosophy\"" - } - } - }, - { - "evaluation_name": "Professional Psychology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Professional Psychology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.843, - "details": { - "description": "min=0.843, mean=0.843, max=0.843, sum=1.686 (2)", - "tab": "Accuracy", - "Professional Medicine - Observed inference time (s)": "{\"description\": \"min=0.553, mean=0.553, max=0.553, sum=1.106 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5529017465956071\"}", - "Professional Accounting - Observed inference time (s)": "{\"description\": \"min=0.323, mean=0.323, max=0.323, sum=0.647 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.32346555189038\"}", - "Professional Law - Observed inference time (s)": "{\"description\": \"min=0.372, mean=0.372, max=0.372, sum=0.743 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3715069820859131\"}", - "Professional Psychology - Observed inference time (s)": "{\"description\": \"min=0.315, mean=0.315, max=0.315, sum=0.63 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3151663907992294\"}", - "Professional Medicine - # eval": "{\"description\": \"min=272, mean=272, max=272, sum=544 (2)\", \"tab\": \"General information\", \"score\": \"272.0\"}", - "Professional Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Medicine - # prompt tokens": "{\"description\": \"min=1094.489, mean=1094.489, max=1094.489, sum=2188.978 (2)\", \"tab\": \"General information\", \"score\": \"1094.4889705882354\"}", - "Professional Medicine - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # eval": "{\"description\": \"min=282, mean=282, max=282, sum=564 (2)\", \"tab\": \"General information\", \"score\": \"282.0\"}", - "Professional Accounting - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Accounting - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # prompt tokens": "{\"description\": \"min=658.585, mean=658.585, max=658.585, sum=1317.17 (2)\", \"tab\": \"General information\", \"score\": \"658.5851063829788\"}", - "Professional Accounting - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # eval": "{\"description\": \"min=1534, mean=1534, max=1534, sum=3068 (2)\", \"tab\": \"General information\", \"score\": \"1534.0\"}", - "Professional Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # prompt tokens": "{\"description\": \"min=1637.601, mean=1637.601, max=1637.601, sum=3275.202 (2)\", \"tab\": \"General information\", \"score\": \"1637.6010430247718\"}", - "Professional Law - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # eval": "{\"description\": \"min=612, mean=612, max=612, sum=1224 (2)\", \"tab\": \"General information\", \"score\": \"612.0\"}", - "Professional Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # prompt tokens": "{\"description\": \"min=575.098, mean=575.098, max=575.098, sum=1150.196 (2)\", \"tab\": \"General information\", \"score\": \"575.0980392156863\"}", - "Professional Psychology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"professional_psychology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_professional_psychology\"" - } - } - }, - { - "evaluation_name": "Us Foreign Policy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Us Foreign Policy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.93, - "details": { - "description": "min=0.93, mean=0.93, max=0.93, sum=1.86 (2)", - "tab": "Accuracy", - "Us Foreign Policy - Observed inference time (s)": "{\"description\": \"min=0.507, mean=0.507, max=0.507, sum=1.014 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5069083476066589\"}", - "Us Foreign Policy - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Us Foreign Policy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Us Foreign Policy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Us Foreign Policy - # prompt tokens": "{\"description\": \"min=422.79, mean=422.79, max=422.79, sum=845.58 (2)\", \"tab\": \"General information\", \"score\": \"422.79\"}", - "Us Foreign Policy - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"us_foreign_policy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_us_foreign_policy\"" - } - } - }, - { - "evaluation_name": "Astronomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Astronomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.921, - "details": { - "description": "min=0.921, mean=0.921, max=0.921, sum=1.842 (2)", - "tab": "Accuracy", - "Astronomy - Observed inference time (s)": "{\"description\": \"min=0.332, mean=0.332, max=0.332, sum=0.665 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3323579352152975\"}", - "Astronomy - # eval": "{\"description\": \"min=152, mean=152, max=152, sum=304 (2)\", \"tab\": \"General information\", \"score\": \"152.0\"}", - "Astronomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Astronomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Astronomy - # prompt tokens": "{\"description\": \"min=579.684, mean=579.684, max=579.684, sum=1159.368 (2)\", \"tab\": \"General information\", \"score\": \"579.6842105263158\"}", - "Astronomy - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"astronomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_astronomy\"" - } - } - }, - { - "evaluation_name": "Business Ethics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Business Ethics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.76, - "details": { - "description": "min=0.76, mean=0.76, max=0.76, sum=1.52 (2)", - "tab": "Accuracy", - "Business Ethics - Observed inference time (s)": "{\"description\": \"min=0.291, mean=0.291, max=0.291, sum=0.581 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.29072295665740966\"}", - "Business Ethics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Business Ethics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Business Ethics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Business Ethics - # prompt tokens": "{\"description\": \"min=569.52, mean=569.52, max=569.52, sum=1139.04 (2)\", \"tab\": \"General information\", \"score\": \"569.52\"}", - "Business Ethics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"business_ethics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_business_ethics\"" - } - } - }, - { - "evaluation_name": "Clinical Knowledge", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Clinical Knowledge", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.845, - "details": { - "description": "min=0.845, mean=0.845, max=0.845, sum=1.691 (2)", - "tab": "Accuracy", - "Clinical Knowledge - Observed inference time (s)": "{\"description\": \"min=0.29, mean=0.29, max=0.29, sum=0.579 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2897273891376999\"}", - "Clinical Knowledge - # eval": "{\"description\": \"min=265, mean=265, max=265, sum=530 (2)\", \"tab\": \"General information\", \"score\": \"265.0\"}", - "Clinical Knowledge - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Clinical Knowledge - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Clinical Knowledge - # prompt tokens": "{\"description\": \"min=397.928, mean=397.928, max=397.928, sum=795.857 (2)\", \"tab\": \"General information\", \"score\": \"397.92830188679244\"}", - "Clinical Knowledge - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"clinical_knowledge\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_clinical_knowledge\"" - } - } - }, - { - "evaluation_name": "Conceptual Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Conceptual Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.826, - "details": { - "description": "min=0.826, mean=0.826, max=0.826, sum=1.651 (2)", - "tab": "Accuracy", - "Conceptual Physics - Observed inference time (s)": "{\"description\": \"min=0.279, mean=0.279, max=0.279, sum=0.559 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2794749209221373\"}", - "Conceptual Physics - # eval": "{\"description\": \"min=235, mean=235, max=235, sum=470 (2)\", \"tab\": \"General information\", \"score\": \"235.0\"}", - "Conceptual Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Conceptual Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Conceptual Physics - # prompt tokens": "{\"description\": \"min=304.834, mean=304.834, max=304.834, sum=609.668 (2)\", \"tab\": \"General information\", \"score\": \"304.83404255319147\"}", - "Conceptual Physics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"conceptual_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_conceptual_physics\"" - } - } - }, - { - "evaluation_name": "Electrical Engineering", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Electrical Engineering", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.759, - "details": { - "description": "min=0.759, mean=0.759, max=0.759, sum=1.517 (2)", - "tab": "Accuracy", - "Electrical Engineering - Observed inference time (s)": "{\"description\": \"min=0.256, mean=0.256, max=0.256, sum=0.512 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2558267790695717\"}", - "Electrical Engineering - # eval": "{\"description\": \"min=145, mean=145, max=145, sum=290 (2)\", \"tab\": \"General information\", \"score\": \"145.0\"}", - "Electrical Engineering - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Electrical Engineering - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Electrical Engineering - # prompt tokens": "{\"description\": \"min=435.607, mean=435.607, max=435.607, sum=871.214 (2)\", \"tab\": \"General information\", \"score\": \"435.60689655172416\"}", - "Electrical Engineering - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"electrical_engineering\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_electrical_engineering\"" - } - } - }, - { - "evaluation_name": "Elementary Mathematics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Elementary Mathematics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.688, - "details": { - "description": "min=0.688, mean=0.688, max=0.688, sum=1.376 (2)", - "tab": "Accuracy", - "Elementary Mathematics - Observed inference time (s)": "{\"description\": \"min=0.308, mean=0.308, max=0.308, sum=0.617 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.30840403945357714\"}", - "Elementary Mathematics - # eval": "{\"description\": \"min=378, mean=378, max=378, sum=756 (2)\", \"tab\": \"General information\", \"score\": \"378.0\"}", - "Elementary Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Elementary Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Elementary Mathematics - # prompt tokens": "{\"description\": \"min=531.854, mean=531.854, max=531.854, sum=1063.709 (2)\", \"tab\": \"General information\", \"score\": \"531.8544973544973\"}", - "Elementary Mathematics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"elementary_mathematics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_elementary_mathematics\"" - } - } - }, - { - "evaluation_name": "Formal Logic", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Formal Logic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.683, - "details": { - "description": "min=0.683, mean=0.683, max=0.683, sum=1.365 (2)", - "tab": "Accuracy", - "Formal Logic - Observed inference time (s)": "{\"description\": \"min=0.304, mean=0.304, max=0.304, sum=0.609 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.30448357074979754\"}", - "Formal Logic - # eval": "{\"description\": \"min=126, mean=126, max=126, sum=252 (2)\", \"tab\": \"General information\", \"score\": \"126.0\"}", - "Formal Logic - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Formal Logic - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Formal Logic - # prompt tokens": "{\"description\": \"min=601.778, mean=601.778, max=601.778, sum=1203.556 (2)\", \"tab\": \"General information\", \"score\": \"601.7777777777778\"}", - "Formal Logic - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"formal_logic\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_formal_logic\"" - } - } - }, - { - "evaluation_name": "High School World History", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on High School World History", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.941, - "details": { - "description": "min=0.941, mean=0.941, max=0.941, sum=1.882 (2)", - "tab": "Accuracy", - "High School Biology - Observed inference time (s)": "{\"description\": \"min=0.309, mean=0.309, max=0.309, sum=0.619 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3094667688492806\"}", - "High School Chemistry - Observed inference time (s)": "{\"description\": \"min=0.294, mean=0.294, max=0.294, sum=0.588 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.29394797386207017\"}", - "High School Computer Science - Observed inference time (s)": "{\"description\": \"min=0.301, mean=0.301, max=0.301, sum=0.602 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.30106969356536867\"}", - "High School European History - Observed inference time (s)": "{\"description\": \"min=0.48, mean=0.48, max=0.48, sum=0.96 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4799844944115841\"}", - "High School Geography - Observed inference time (s)": "{\"description\": \"min=0.297, mean=0.297, max=0.297, sum=0.595 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.29747620014229204\"}", - "High School Government And Politics - Observed inference time (s)": "{\"description\": \"min=0.291, mean=0.291, max=0.291, sum=0.583 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2914604300662026\"}", - "High School Macroeconomics - Observed inference time (s)": "{\"description\": \"min=0.279, mean=0.279, max=0.279, sum=0.557 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.27857950650728663\"}", - "High School Mathematics - Observed inference time (s)": "{\"description\": \"min=0.312, mean=0.312, max=0.312, sum=0.625 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3123831342767786\"}", - "High School Microeconomics - Observed inference time (s)": "{\"description\": \"min=0.302, mean=0.302, max=0.302, sum=0.603 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.30159517997453195\"}", - "High School Physics - Observed inference time (s)": "{\"description\": \"min=0.322, mean=0.322, max=0.322, sum=0.643 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.32152655108874995\"}", - "High School Psychology - Observed inference time (s)": "{\"description\": \"min=0.29, mean=0.29, max=0.29, sum=0.581 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2903494253071076\"}", - "High School Statistics - Observed inference time (s)": "{\"description\": \"min=0.333, mean=0.333, max=0.333, sum=0.667 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.33328031720938506\"}", - "High School US History - Observed inference time (s)": "{\"description\": \"min=0.394, mean=0.394, max=0.394, sum=0.788 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.39396579826579375\"}", - "High School World History - Observed inference time (s)": "{\"description\": \"min=0.679, mean=0.679, max=0.679, sum=1.359 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6793377369265013\"}", - "High School Biology - # eval": "{\"description\": \"min=310, mean=310, max=310, sum=620 (2)\", \"tab\": \"General information\", \"score\": \"310.0\"}", - "High School Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Biology - # prompt tokens": "{\"description\": \"min=513.671, mean=513.671, max=513.671, sum=1027.342 (2)\", \"tab\": \"General information\", \"score\": \"513.6709677419354\"}", - "High School Biology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # eval": "{\"description\": \"min=203, mean=203, max=203, sum=406 (2)\", \"tab\": \"General information\", \"score\": \"203.0\"}", - "High School Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # prompt tokens": "{\"description\": \"min=496.704, mean=496.704, max=496.704, sum=993.409 (2)\", \"tab\": \"General information\", \"score\": \"496.70443349753697\"}", - "High School Chemistry - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "High School Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # prompt tokens": "{\"description\": \"min=867.78, mean=867.78, max=867.78, sum=1735.56 (2)\", \"tab\": \"General information\", \"score\": \"867.78\"}", - "High School Computer Science - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # eval": "{\"description\": \"min=165, mean=165, max=165, sum=330 (2)\", \"tab\": \"General information\", \"score\": \"165.0\"}", - "High School European History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School European History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # prompt tokens": "{\"description\": \"min=2797.885, mean=2797.885, max=2797.885, sum=5595.77 (2)\", \"tab\": \"General information\", \"score\": \"2797.8848484848486\"}", - "High School European History - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # eval": "{\"description\": \"min=198, mean=198, max=198, sum=396 (2)\", \"tab\": \"General information\", \"score\": \"198.0\"}", - "High School Geography - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Geography - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # prompt tokens": "{\"description\": \"min=372.035, mean=372.035, max=372.035, sum=744.071 (2)\", \"tab\": \"General information\", \"score\": \"372.0353535353535\"}", - "High School Geography - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # eval": "{\"description\": \"min=193, mean=193, max=193, sum=386 (2)\", \"tab\": \"General information\", \"score\": \"193.0\"}", - "High School Government And Politics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Government And Politics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # prompt tokens": "{\"description\": \"min=465.824, mean=465.824, max=465.824, sum=931.648 (2)\", \"tab\": \"General information\", \"score\": \"465.8238341968912\"}", - "High School Government And Politics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # eval": "{\"description\": \"min=390, mean=390, max=390, sum=780 (2)\", \"tab\": \"General information\", \"score\": \"390.0\"}", - "High School Macroeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Macroeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # prompt tokens": "{\"description\": \"min=370.908, mean=370.908, max=370.908, sum=741.815 (2)\", \"tab\": \"General information\", \"score\": \"370.9076923076923\"}", - "High School Macroeconomics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # eval": "{\"description\": \"min=270, mean=270, max=270, sum=540 (2)\", \"tab\": \"General information\", \"score\": \"270.0\"}", - "High School Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # prompt tokens": "{\"description\": \"min=532.356, mean=532.356, max=532.356, sum=1064.711 (2)\", \"tab\": \"General information\", \"score\": \"532.3555555555556\"}", - "High School Mathematics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # eval": "{\"description\": \"min=238, mean=238, max=238, sum=476 (2)\", \"tab\": \"General information\", \"score\": \"238.0\"}", - "High School Microeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Microeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # prompt tokens": "{\"description\": \"min=399.013, mean=399.013, max=399.013, sum=798.025 (2)\", \"tab\": \"General information\", \"score\": \"399.0126050420168\"}", - "High School Microeconomics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # eval": "{\"description\": \"min=151, mean=151, max=151, sum=302 (2)\", \"tab\": \"General information\", \"score\": \"151.0\"}", - "High School Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # prompt tokens": "{\"description\": \"min=560.457, mean=560.457, max=560.457, sum=1120.914 (2)\", \"tab\": \"General information\", \"score\": \"560.4569536423841\"}", - "High School Physics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # eval": "{\"description\": \"min=545, mean=545, max=545, sum=1090 (2)\", \"tab\": \"General information\", \"score\": \"545.0\"}", - "High School Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # prompt tokens": "{\"description\": \"min=495.242, mean=495.242, max=495.242, sum=990.484 (2)\", \"tab\": \"General information\", \"score\": \"495.2422018348624\"}", - "High School Psychology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # eval": "{\"description\": \"min=216, mean=216, max=216, sum=432 (2)\", \"tab\": \"General information\", \"score\": \"216.0\"}", - "High School Statistics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Statistics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # prompt tokens": "{\"description\": \"min=795.639, mean=795.639, max=795.639, sum=1591.278 (2)\", \"tab\": \"General information\", \"score\": \"795.6388888888889\"}", - "High School Statistics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # eval": "{\"description\": \"min=204, mean=204, max=204, sum=408 (2)\", \"tab\": \"General information\", \"score\": \"204.0\"}", - "High School US History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School US History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # prompt tokens": "{\"description\": \"min=2217.809, mean=2217.809, max=2217.809, sum=4435.618 (2)\", \"tab\": \"General information\", \"score\": \"2217.8088235294117\"}", - "High School US History - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # eval": "{\"description\": \"min=237, mean=237, max=237, sum=474 (2)\", \"tab\": \"General information\", \"score\": \"237.0\"}", - "High School World History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School World History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # prompt tokens": "{\"description\": \"min=1428.173, mean=1428.173, max=1428.173, sum=2856.346 (2)\", \"tab\": \"General information\", \"score\": \"1428.1729957805908\"}", - "High School World History - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"high_school_world_history\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_high_school_world_history\"" - } - } - }, - { - "evaluation_name": "Human Sexuality", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Human Sexuality", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.87, - "details": { - "description": "min=0.87, mean=0.87, max=0.87, sum=1.74 (2)", - "tab": "Accuracy", - "Human Aging - Observed inference time (s)": "{\"description\": \"min=0.388, mean=0.388, max=0.388, sum=0.776 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.38789880863754206\"}", - "Human Sexuality - Observed inference time (s)": "{\"description\": \"min=0.293, mean=0.293, max=0.293, sum=0.586 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2929920222013051\"}", - "Human Aging - # eval": "{\"description\": \"min=223, mean=223, max=223, sum=446 (2)\", \"tab\": \"General information\", \"score\": \"223.0\"}", - "Human Aging - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Aging - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Aging - # prompt tokens": "{\"description\": \"min=319.888, mean=319.888, max=319.888, sum=639.776 (2)\", \"tab\": \"General information\", \"score\": \"319.88789237668163\"}", - "Human Aging - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # eval": "{\"description\": \"min=131, mean=131, max=131, sum=262 (2)\", \"tab\": \"General information\", \"score\": \"131.0\"}", - "Human Sexuality - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Sexuality - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # prompt tokens": "{\"description\": \"min=341.168, mean=341.168, max=341.168, sum=682.336 (2)\", \"tab\": \"General information\", \"score\": \"341.1679389312977\"}", - "Human Sexuality - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"human_sexuality\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_human_sexuality\"" - } - } - }, - { - "evaluation_name": "International Law", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on International Law", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.934, - "details": { - "description": "min=0.934, mean=0.934, max=0.934, sum=1.868 (2)", - "tab": "Accuracy", - "International Law - Observed inference time (s)": "{\"description\": \"min=0.342, mean=0.342, max=0.342, sum=0.685 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.34241620962284813\"}", - "International Law - # eval": "{\"description\": \"min=121, mean=121, max=121, sum=242 (2)\", \"tab\": \"General information\", \"score\": \"121.0\"}", - "International Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "International Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "International Law - # prompt tokens": "{\"description\": \"min=639.818, mean=639.818, max=639.818, sum=1279.636 (2)\", \"tab\": \"General information\", \"score\": \"639.8181818181819\"}", - "International Law - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"international_law\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_international_law\"" - } - } - }, - { - "evaluation_name": "Logical Fallacies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Logical Fallacies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.834, - "details": { - "description": "min=0.834, mean=0.834, max=0.834, sum=1.669 (2)", - "tab": "Accuracy", - "Logical Fallacies - Observed inference time (s)": "{\"description\": \"min=0.282, mean=0.282, max=0.282, sum=0.565 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.28232605325663745\"}", - "Logical Fallacies - # eval": "{\"description\": \"min=163, mean=163, max=163, sum=326 (2)\", \"tab\": \"General information\", \"score\": \"163.0\"}", - "Logical Fallacies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Logical Fallacies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Logical Fallacies - # prompt tokens": "{\"description\": \"min=449.564, mean=449.564, max=449.564, sum=899.129 (2)\", \"tab\": \"General information\", \"score\": \"449.5644171779141\"}", - "Logical Fallacies - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"logical_fallacies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_logical_fallacies\"" - } - } - }, - { - "evaluation_name": "Machine Learning", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Machine Learning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.688, - "details": { - "description": "min=0.688, mean=0.688, max=0.688, sum=1.375 (2)", - "tab": "Accuracy", - "Machine Learning - Observed inference time (s)": "{\"description\": \"min=0.338, mean=0.338, max=0.338, sum=0.676 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.33782388057027546\"}", - "Machine Learning - # eval": "{\"description\": \"min=112, mean=112, max=112, sum=224 (2)\", \"tab\": \"General information\", \"score\": \"112.0\"}", - "Machine Learning - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Machine Learning - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Machine Learning - # prompt tokens": "{\"description\": \"min=668.054, mean=668.054, max=668.054, sum=1336.107 (2)\", \"tab\": \"General information\", \"score\": \"668.0535714285714\"}", - "Machine Learning - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"machine_learning\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_machine_learning\"" - } - } - }, - { - "evaluation_name": "Management", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Management", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.913, - "details": { - "description": "min=0.913, mean=0.913, max=0.913, sum=1.825 (2)", - "tab": "Accuracy", - "Management - Observed inference time (s)": "{\"description\": \"min=0.285, mean=0.285, max=0.285, sum=0.571 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2853238027072647\"}", - "Management - # eval": "{\"description\": \"min=103, mean=103, max=103, sum=206 (2)\", \"tab\": \"General information\", \"score\": \"103.0\"}", - "Management - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Management - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Management - # prompt tokens": "{\"description\": \"min=283.786, mean=283.786, max=283.786, sum=567.573 (2)\", \"tab\": \"General information\", \"score\": \"283.7864077669903\"}", - "Management - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"management\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_management\"" - } - } - }, - { - "evaluation_name": "Marketing", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Marketing", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.944, - "details": { - "description": "min=0.944, mean=0.944, max=0.944, sum=1.889 (2)", - "tab": "Accuracy", - "Marketing - Observed inference time (s)": "{\"description\": \"min=0.28, mean=0.28, max=0.28, sum=0.561 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.28032574796269083\"}", - "Marketing - # eval": "{\"description\": \"min=234, mean=234, max=234, sum=468 (2)\", \"tab\": \"General information\", \"score\": \"234.0\"}", - "Marketing - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Marketing - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Marketing - # prompt tokens": "{\"description\": \"min=404.218, mean=404.218, max=404.218, sum=808.436 (2)\", \"tab\": \"General information\", \"score\": \"404.21794871794873\"}", - "Marketing - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"marketing\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_marketing\"" - } - } - }, - { - "evaluation_name": "Medical Genetics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Medical Genetics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.92, - "details": { - "description": "min=0.92, mean=0.92, max=0.92, sum=1.84 (2)", - "tab": "Accuracy", - "Medical Genetics - Observed inference time (s)": "{\"description\": \"min=0.296, mean=0.296, max=0.296, sum=0.592 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.29611136198043825\"}", - "Medical Genetics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Medical Genetics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Medical Genetics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Medical Genetics - # prompt tokens": "{\"description\": \"min=340.99, mean=340.99, max=340.99, sum=681.98 (2)\", \"tab\": \"General information\", \"score\": \"340.99\"}", - "Medical Genetics - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"medical_genetics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_medical_genetics\"" - } - } - }, - { - "evaluation_name": "Miscellaneous", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Miscellaneous", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.913, - "details": { - "description": "min=0.913, mean=0.913, max=0.913, sum=1.826 (2)", - "tab": "Accuracy", - "Miscellaneous - Observed inference time (s)": "{\"description\": \"min=0.324, mean=0.324, max=0.324, sum=0.647 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3237126984967735\"}", - "Miscellaneous - # eval": "{\"description\": \"min=783, mean=783, max=783, sum=1566 (2)\", \"tab\": \"General information\", \"score\": \"783.0\"}", - "Miscellaneous - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Miscellaneous - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Miscellaneous - # prompt tokens": "{\"description\": \"min=299.911, mean=299.911, max=299.911, sum=599.821 (2)\", \"tab\": \"General information\", \"score\": \"299.9106002554278\"}", - "Miscellaneous - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"miscellaneous\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_miscellaneous\"" - } - } - }, - { - "evaluation_name": "Moral Scenarios", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Moral Scenarios", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.841, - "details": { - "description": "min=0.841, mean=0.841, max=0.841, sum=1.683 (2)", - "tab": "Accuracy", - "Moral Disputes - Observed inference time (s)": "{\"description\": \"min=0.29, mean=0.29, max=0.29, sum=0.58 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2901734975032035\"}", - "Moral Scenarios - Observed inference time (s)": "{\"description\": \"min=0.506, mean=0.506, max=0.506, sum=1.012 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5058047955262595\"}", - "Moral Disputes - # eval": "{\"description\": \"min=346, mean=346, max=346, sum=692 (2)\", \"tab\": \"General information\", \"score\": \"346.0\"}", - "Moral Disputes - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Disputes - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Disputes - # prompt tokens": "{\"description\": \"min=476.113, mean=476.113, max=476.113, sum=952.225 (2)\", \"tab\": \"General information\", \"score\": \"476.1127167630058\"}", - "Moral Disputes - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # eval": "{\"description\": \"min=895, mean=895, max=895, sum=1790 (2)\", \"tab\": \"General information\", \"score\": \"895.0\"}", - "Moral Scenarios - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Scenarios - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # prompt tokens": "{\"description\": \"min=656.455, mean=656.455, max=656.455, sum=1312.909 (2)\", \"tab\": \"General information\", \"score\": \"656.454748603352\"}", - "Moral Scenarios - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"moral_scenarios\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_moral_scenarios\"" - } - } - }, - { - "evaluation_name": "Nutrition", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Nutrition", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.889, - "details": { - "description": "min=0.889, mean=0.889, max=0.889, sum=1.778 (2)", - "tab": "Accuracy", - "Nutrition - Observed inference time (s)": "{\"description\": \"min=0.321, mean=0.321, max=0.321, sum=0.641 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.32064209264867444\"}", - "Nutrition - # eval": "{\"description\": \"min=306, mean=306, max=306, sum=612 (2)\", \"tab\": \"General information\", \"score\": \"306.0\"}", - "Nutrition - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Nutrition - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Nutrition - # prompt tokens": "{\"description\": \"min=586.814, mean=586.814, max=586.814, sum=1173.627 (2)\", \"tab\": \"General information\", \"score\": \"586.8137254901961\"}", - "Nutrition - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"nutrition\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_nutrition\"" - } - } - }, - { - "evaluation_name": "Prehistory", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Prehistory", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.886, - "details": { - "description": "min=0.886, mean=0.886, max=0.886, sum=1.772 (2)", - "tab": "Accuracy", - "Prehistory - Observed inference time (s)": "{\"description\": \"min=0.614, mean=0.614, max=0.614, sum=1.227 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6136744522754057\"}", - "Prehistory - # eval": "{\"description\": \"min=324, mean=324, max=324, sum=648 (2)\", \"tab\": \"General information\", \"score\": \"324.0\"}", - "Prehistory - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Prehistory - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Prehistory - # prompt tokens": "{\"description\": \"min=514.528, mean=514.528, max=514.528, sum=1029.056 (2)\", \"tab\": \"General information\", \"score\": \"514.5277777777778\"}", - "Prehistory - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"prehistory\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_prehistory\"" - } - } - }, - { - "evaluation_name": "Public Relations", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Public Relations", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.718, - "details": { - "description": "min=0.718, mean=0.718, max=0.718, sum=1.436 (2)", - "tab": "Accuracy", - "Public Relations - Observed inference time (s)": "{\"description\": \"min=0.3, mean=0.3, max=0.3, sum=0.599 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.29952496832067316\"}", - "Public Relations - # eval": "{\"description\": \"min=110, mean=110, max=110, sum=220 (2)\", \"tab\": \"General information\", \"score\": \"110.0\"}", - "Public Relations - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Public Relations - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Public Relations - # prompt tokens": "{\"description\": \"min=405.318, mean=405.318, max=405.318, sum=810.636 (2)\", \"tab\": \"General information\", \"score\": \"405.3181818181818\"}", - "Public Relations - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"public_relations\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_public_relations\"" - } - } - }, - { - "evaluation_name": "Security Studies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Security Studies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.853, - "details": { - "description": "min=0.853, mean=0.853, max=0.853, sum=1.706 (2)", - "tab": "Accuracy", - "Security Studies - Observed inference time (s)": "{\"description\": \"min=0.348, mean=0.348, max=0.348, sum=0.697 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.348436891789339\"}", - "Security Studies - # eval": "{\"description\": \"min=245, mean=245, max=245, sum=490 (2)\", \"tab\": \"General information\", \"score\": \"245.0\"}", - "Security Studies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Security Studies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Security Studies - # prompt tokens": "{\"description\": \"min=1164.473, mean=1164.473, max=1164.473, sum=2328.947 (2)\", \"tab\": \"General information\", \"score\": \"1164.4734693877551\"}", - "Security Studies - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"security_studies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_security_studies\"" - } - } - }, - { - "evaluation_name": "Sociology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Sociology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.92, - "details": { - "description": "min=0.92, mean=0.92, max=0.92, sum=1.841 (2)", - "tab": "Accuracy", - "Sociology - Observed inference time (s)": "{\"description\": \"min=0.297, mean=0.297, max=0.297, sum=0.595 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.29732529915387357\"}", - "Sociology - # eval": "{\"description\": \"min=201, mean=201, max=201, sum=402 (2)\", \"tab\": \"General information\", \"score\": \"201.0\"}", - "Sociology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Sociology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Sociology - # prompt tokens": "{\"description\": \"min=445.517, mean=445.517, max=445.517, sum=891.035 (2)\", \"tab\": \"General information\", \"score\": \"445.51741293532336\"}", - "Sociology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"sociology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_sociology\"" - } - } - }, - { - "evaluation_name": "Virology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Virology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.584, - "details": { - "description": "min=0.584, mean=0.584, max=0.584, sum=1.169 (2)", - "tab": "Accuracy", - "Virology - Observed inference time (s)": "{\"description\": \"min=0.321, mean=0.321, max=0.321, sum=0.642 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.32124968609177923\"}", - "Virology - # eval": "{\"description\": \"min=166, mean=166, max=166, sum=332 (2)\", \"tab\": \"General information\", \"score\": \"166.0\"}", - "Virology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Virology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Virology - # prompt tokens": "{\"description\": \"min=343.018, mean=343.018, max=343.018, sum=686.036 (2)\", \"tab\": \"General information\", \"score\": \"343.01807228915663\"}", - "Virology - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"virology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_virology\"" - } - } - }, - { - "evaluation_name": "World Religions", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on World Religions", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.901, - "details": { - "description": "min=0.901, mean=0.901, max=0.901, sum=1.801 (2)", - "tab": "Accuracy", - "World Religions - Observed inference time (s)": "{\"description\": \"min=0.277, mean=0.277, max=0.277, sum=0.554 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.27723441068191973\"}", - "World Religions - # eval": "{\"description\": \"min=171, mean=171, max=171, sum=342 (2)\", \"tab\": \"General information\", \"score\": \"171.0\"}", - "World Religions - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "World Religions - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "World Religions - # prompt tokens": "{\"description\": \"min=274.52, mean=274.52, max=274.52, sum=549.041 (2)\", \"tab\": \"General information\", \"score\": \"274.5204678362573\"}", - "World Religions - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"world_religions\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_world_religions\"" - } - } - }, - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.773, - "details": { - "description": "", - "tab": "Efficiency" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/meta_llama-3.3-70b-instruct-turbo.json b/data/models/meta_llama-3.3-70b-instruct-turbo.json deleted file mode 100644 index 63d028e395ed955dd7196e24bf7384250098ee86..0000000000000000000000000000000000000000 --- a/data/models/meta_llama-3.3-70b-instruct-turbo.json +++ /dev/null @@ -1,1903 +0,0 @@ -{ - "model_info": { - "name": "Llama 3.3 Instruct Turbo 70B", - "id": "meta/llama-3.3-70b-instruct-turbo", - "developer": "Meta", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_lite/meta_llama-3.3-70b-instruct-turbo/1774096306.427425", - "retrieved_timestamp": "1774096306.427425", - "source_metadata": { - "source_name": "helm_lite", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_lite", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_lite", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.812, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.7410112359550561\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.791, - "details": { - "description": "min=0.791, mean=0.791, max=0.791, sum=0.791 (1)", - "tab": "Accuracy", - "NarrativeQA - Observed inference time (s)": "{\"description\": \"min=0.746, mean=0.746, max=0.746, sum=0.746 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.7455473496880329\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=3484.268, mean=3484.268, max=3484.268, sum=3484.268 (1)\", \"tab\": \"General information\", \"score\": \"3484.2676056338028\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=7.606, mean=7.606, max=7.606, sum=7.606 (1)\", \"tab\": \"General information\", \"score\": \"7.605633802816901\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (closed-book)", - "source_data": { - "dataset_name": "NaturalQuestions (closed-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (closed-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.431, - "details": { - "description": "min=0.431, mean=0.431, max=0.431, sum=0.431 (1)", - "tab": "Accuracy", - "NaturalQuestions (open-book) - Observed inference time (s)": "{\"description\": \"min=0.51, mean=0.51, max=0.51, sum=0.51 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.5104404001235961\"}", - "NaturalQuestions (closed-book) - Observed inference time (s)": "{\"description\": \"min=0.466, mean=0.466, max=0.466, sum=0.466 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.46574948048591613\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1716.78, mean=1716.78, max=1716.78, sum=1716.78 (1)\", \"tab\": \"General information\", \"score\": \"1716.78\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=7.503, mean=7.503, max=7.503, sum=7.503 (1)\", \"tab\": \"General information\", \"score\": \"7.503\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=129.12, mean=129.12, max=129.12, sum=129.12 (1)\", \"tab\": \"General information\", \"score\": \"129.12\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=9.152, mean=9.152, max=9.152, sum=9.152 (1)\", \"tab\": \"General information\", \"score\": \"9.152\"}" - } - }, - "generation_config": { - "additional_details": { - "mode": "\"closedbook\"" - } - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.928, - "details": { - "description": "min=0.928, mean=0.928, max=0.928, sum=0.928 (1)", - "tab": "Accuracy", - "OpenbookQA - Observed inference time (s)": "{\"description\": \"min=0.339, mean=0.339, max=0.339, sum=0.339 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.3392307605743408\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=249.776, mean=249.776, max=249.776, sum=249.776 (1)\", \"tab\": \"General information\", \"score\": \"249.776\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "dataset": "\"openbookqa\"", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7, - "details": { - "description": "min=0.5, mean=0.7, max=0.93, sum=3.499 (5)", - "tab": "Accuracy", - "MMLU - Observed inference time (s)": "{\"description\": \"min=0.313, mean=0.339, max=0.359, sum=1.695 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.3389431067433274\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=373.43, mean=467.686, max=614.421, sum=2338.431 (5)\", \"tab\": \"General information\", \"score\": \"467.6862105263158\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"college_chemistry\", \"computer_security\", \"econometrics\", \"us_foreign_policy\"]", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MATH", - "source_data": { - "dataset_name": "MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Equivalent (CoT) on MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.808, - "details": { - "description": "min=0.635, mean=0.808, max=0.963, sum=5.655 (7)", - "tab": "Accuracy", - "MATH - Observed inference time (s)": "{\"description\": \"min=1.483, mean=1.779, max=2.037, sum=12.455 (7)\", \"tab\": \"Efficiency\", \"score\": \"1.7792604792087183\"}", - "MATH - # eval": "{\"description\": \"min=30, mean=62.429, max=135, sum=437 (7)\", \"tab\": \"General information\", \"score\": \"62.42857142857143\"}", - "MATH - # train": "{\"description\": \"min=8, mean=8, max=8, sum=56 (7)\", \"tab\": \"General information\", \"score\": \"8.0\"}", - "MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MATH - # prompt tokens": "{\"description\": \"min=881.363, mean=1262.909, max=2197.577, sum=8840.364 (7)\", \"tab\": \"General information\", \"score\": \"1262.9092130545007\"}", - "MATH - # output tokens": "{\"description\": \"min=192.326, mean=245.345, max=274.462, sum=1717.412 (7)\", \"tab\": \"General information\", \"score\": \"245.34459229967183\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"algebra\", \"counting_and_probability\", \"geometry\", \"intermediate_algebra\", \"number_theory\", \"prealgebra\", \"precalculus\"]", - "level": "\"1\"", - "use_official_examples": "\"False\"", - "use_chain_of_thought": "\"True\"" - } - } - }, - { - "evaluation_name": "GSM8K", - "source_data": { - "dataset_name": "GSM8K", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on GSM8K", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.942, - "details": { - "description": "min=0.942, mean=0.942, max=0.942, sum=0.942 (1)", - "tab": "Accuracy", - "GSM8K - Observed inference time (s)": "{\"description\": \"min=1.354, mean=1.354, max=1.354, sum=1.354 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.3539768285751344\"}", - "GSM8K - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "GSM8K - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "GSM8K - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GSM8K - # prompt tokens": "{\"description\": \"min=959.032, mean=959.032, max=959.032, sum=959.032 (1)\", \"tab\": \"General information\", \"score\": \"959.032\"}", - "GSM8K - # output tokens": "{\"description\": \"min=155.609, mean=155.609, max=155.609, sum=155.609 (1)\", \"tab\": \"General information\", \"score\": \"155.609\"}" - } - }, - "generation_config": { - "additional_details": { - "stop": "\"none\"" - } - } - }, - { - "evaluation_name": "LegalBench", - "source_data": { - "dataset_name": "LegalBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on LegalBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.725, - "details": { - "description": "min=0.428, mean=0.725, max=0.979, sum=3.627 (5)", - "tab": "Accuracy", - "LegalBench - Observed inference time (s)": "{\"description\": \"min=0.374, mean=0.549, max=1.199, sum=2.745 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.5490109607174599\"}", - "LegalBench - # eval": "{\"description\": \"min=95, mean=409.4, max=1000, sum=2047 (5)\", \"tab\": \"General information\", \"score\": \"409.4\"}", - "LegalBench - # train": "{\"description\": \"min=4, mean=4.8, max=5, sum=24 (5)\", \"tab\": \"General information\", \"score\": \"4.8\"}", - "LegalBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "LegalBench - # prompt tokens": "{\"description\": \"min=197.442, mean=1513.882, max=6300.012, sum=7569.412 (5)\", \"tab\": \"General information\", \"score\": \"1513.8824197238912\"}", - "LegalBench - # output tokens": "{\"description\": \"min=2, mean=2.404, max=3.032, sum=12.02 (5)\", \"tab\": \"General information\", \"score\": \"2.404037659543955\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"abercrombie\", \"corporate_lobbying\", \"function_of_decision_section\", \"international_citizenship_questions\", \"proa\"]" - } - } - }, - { - "evaluation_name": "MedQA", - "source_data": { - "dataset_name": "MedQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MedQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.761, - "details": { - "description": "min=0.761, mean=0.761, max=0.761, sum=0.761 (1)", - "tab": "Accuracy", - "MedQA - Observed inference time (s)": "{\"description\": \"min=0.359, mean=0.359, max=0.359, sum=0.359 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.35867250700357184\"}", - "MedQA - # eval": "{\"description\": \"min=503, mean=503, max=503, sum=503 (1)\", \"tab\": \"General information\", \"score\": \"503.0\"}", - "MedQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MedQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MedQA - # prompt tokens": "{\"description\": \"min=1025.274, mean=1025.274, max=1025.274, sum=1025.274 (1)\", \"tab\": \"General information\", \"score\": \"1025.2743538767395\"}", - "MedQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WMT 2014", - "source_data": { - "dataset_name": "WMT 2014", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "BLEU-4 on WMT 2014", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.219, - "details": { - "description": "min=0.18, mean=0.219, max=0.261, sum=1.096 (5)", - "tab": "Accuracy", - "WMT 2014 - Observed inference time (s)": "{\"description\": \"min=0.587, mean=0.62, max=0.685, sum=3.1 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.6200136459034178\"}", - "WMT 2014 - # eval": "{\"description\": \"min=503, mean=568.8, max=832, sum=2844 (5)\", \"tab\": \"General information\", \"score\": \"568.8\"}", - "WMT 2014 - # train": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "WMT 2014 - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WMT 2014 - # prompt tokens": "{\"description\": \"min=96.139, mean=115.712, max=136.117, sum=578.559 (5)\", \"tab\": \"General information\", \"score\": \"115.71178123566294\"}", - "WMT 2014 - # output tokens": "{\"description\": \"min=25.161, mean=26.542, max=27.189, sum=132.708 (5)\", \"tab\": \"General information\", \"score\": \"26.541526800734054\"}" - } - }, - "generation_config": { - "additional_details": { - "language_pair": "[\"cs-en\", \"de-en\", \"fr-en\", \"hi-en\", \"ru-en\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_mmlu/meta_llama-3.3-70b-instruct-turbo/1774096312.00548", - "retrieved_timestamp": "1774096312.00548", - "source_metadata": { - "source_name": "helm_mmlu", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_mmlu", - "evaluation_results": [ - { - "evaluation_name": "MMLU All Subjects", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU All Subjects", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.791, - "details": { - "description": "min=0.441, mean=0.791, max=0.984, sum=90.129 (114)", - "tab": "Accuracy", - "MMLU All Subjects - Observed inference time (s)": "{\"description\": \"min=0.303, mean=0.345, max=0.559, sum=39.355 (114)\", \"tab\": \"Efficiency\", \"score\": \"0.34521783642237874\"}", - "MMLU All Subjects - # eval": "{\"description\": \"min=100, mean=246.351, max=1534, sum=28084 (114)\", \"tab\": \"General information\", \"score\": \"246.35087719298247\"}", - "MMLU All Subjects - # train": "{\"description\": \"min=5, mean=5, max=5, sum=570 (114)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU All Subjects - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU All Subjects - # prompt tokens": "{\"description\": \"min=274.52, mean=614.619, max=2797.885, sum=70066.61 (114)\", \"tab\": \"General information\", \"score\": \"614.6193817308517\"}", - "MMLU All Subjects - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=114 (114)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_name": "Abstract Algebra", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Abstract Algebra", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5, - "details": { - "description": "min=0.5, mean=0.5, max=0.5, sum=1 (2)", - "tab": "Accuracy", - "Abstract Algebra - Observed inference time (s)": "{\"description\": \"min=0.313, mean=0.313, max=0.313, sum=0.626 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3131356716156006\"}", - "Abstract Algebra - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Abstract Algebra - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Abstract Algebra - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Abstract Algebra - # prompt tokens": "{\"description\": \"min=373.43, mean=373.43, max=373.43, sum=746.86 (2)\", \"tab\": \"General information\", \"score\": \"373.43\"}", - "Abstract Algebra - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"abstract_algebra\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_abstract_algebra\"" - } - } - }, - { - "evaluation_name": "Anatomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Anatomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.778, - "details": { - "description": "min=0.778, mean=0.778, max=0.778, sum=1.556 (2)", - "tab": "Accuracy", - "Anatomy - Observed inference time (s)": "{\"description\": \"min=0.343, mean=0.343, max=0.343, sum=0.686 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3432198400850649\"}", - "Anatomy - # eval": "{\"description\": \"min=135, mean=135, max=135, sum=270 (2)\", \"tab\": \"General information\", \"score\": \"135.0\"}", - "Anatomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Anatomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Anatomy - # prompt tokens": "{\"description\": \"min=353.874, mean=353.874, max=353.874, sum=707.748 (2)\", \"tab\": \"General information\", \"score\": \"353.8740740740741\"}", - "Anatomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"anatomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_anatomy\"" - } - } - }, - { - "evaluation_name": "College Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on College Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.52, - "details": { - "description": "min=0.52, mean=0.52, max=0.52, sum=1.039 (2)", - "tab": "Accuracy", - "College Chemistry - Observed inference time (s)": "{\"description\": \"min=0.359, mean=0.359, max=0.359, sum=0.717 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.35871645450592043\"}", - "College Biology - Observed inference time (s)": "{\"description\": \"min=0.366, mean=0.366, max=0.366, sum=0.732 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.36611984339025283\"}", - "College Computer Science - Observed inference time (s)": "{\"description\": \"min=0.35, mean=0.35, max=0.35, sum=0.701 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3503202319145203\"}", - "College Mathematics - Observed inference time (s)": "{\"description\": \"min=0.337, mean=0.337, max=0.337, sum=0.675 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.33748736619949343\"}", - "College Medicine - Observed inference time (s)": "{\"description\": \"min=0.337, mean=0.337, max=0.337, sum=0.674 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3367649737121053\"}", - "College Physics - Observed inference time (s)": "{\"description\": \"min=0.307, mean=0.307, max=0.307, sum=0.615 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.30743202976152006\"}", - "College Chemistry - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Chemistry - # prompt tokens": "{\"description\": \"min=549.28, mean=549.28, max=549.28, sum=1098.56 (2)\", \"tab\": \"General information\", \"score\": \"549.28\"}", - "College Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Biology - # eval": "{\"description\": \"min=144, mean=144, max=144, sum=288 (2)\", \"tab\": \"General information\", \"score\": \"144.0\"}", - "College Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # prompt tokens": "{\"description\": \"min=473.875, mean=473.875, max=473.875, sum=947.75 (2)\", \"tab\": \"General information\", \"score\": \"473.875\"}", - "College Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # prompt tokens": "{\"description\": \"min=828.29, mean=828.29, max=828.29, sum=1656.58 (2)\", \"tab\": \"General information\", \"score\": \"828.29\"}", - "College Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Mathematics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # prompt tokens": "{\"description\": \"min=594.51, mean=594.51, max=594.51, sum=1189.02 (2)\", \"tab\": \"General information\", \"score\": \"594.51\"}", - "College Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Medicine - # eval": "{\"description\": \"min=173, mean=173, max=173, sum=346 (2)\", \"tab\": \"General information\", \"score\": \"173.0\"}", - "College Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # prompt tokens": "{\"description\": \"min=502.705, mean=502.705, max=502.705, sum=1005.41 (2)\", \"tab\": \"General information\", \"score\": \"502.70520231213874\"}", - "College Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Physics - # eval": "{\"description\": \"min=102, mean=102, max=102, sum=204 (2)\", \"tab\": \"General information\", \"score\": \"102.0\"}", - "College Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # prompt tokens": "{\"description\": \"min=503.569, mean=503.569, max=503.569, sum=1007.137 (2)\", \"tab\": \"General information\", \"score\": \"503.5686274509804\"}", - "College Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"college_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_college_physics\"" - } - } - }, - { - "evaluation_name": "Computer Security", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Computer Security", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8, - "details": { - "description": "min=0.8, mean=0.8, max=0.8, sum=1.6 (2)", - "tab": "Accuracy", - "Computer Security - Observed inference time (s)": "{\"description\": \"min=0.34, mean=0.34, max=0.34, sum=0.68 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.33975651502609255\"}", - "Computer Security - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Computer Security - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Computer Security - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Computer Security - # prompt tokens": "{\"description\": \"min=378.51, mean=378.51, max=378.51, sum=757.02 (2)\", \"tab\": \"General information\", \"score\": \"378.51\"}", - "Computer Security - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"computer_security\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_computer_security\"" - } - } - }, - { - "evaluation_name": "Econometrics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Econometrics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.719, - "details": { - "description": "min=0.719, mean=0.719, max=0.719, sum=1.439 (2)", - "tab": "Accuracy", - "Econometrics - Observed inference time (s)": "{\"description\": \"min=0.341, mean=0.341, max=0.341, sum=0.683 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.34139270113225567\"}", - "Econometrics - # eval": "{\"description\": \"min=114, mean=114, max=114, sum=228 (2)\", \"tab\": \"General information\", \"score\": \"114.0\"}", - "Econometrics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Econometrics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Econometrics - # prompt tokens": "{\"description\": \"min=614.421, mean=614.421, max=614.421, sum=1228.842 (2)\", \"tab\": \"General information\", \"score\": \"614.421052631579\"}", - "Econometrics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"econometrics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_econometrics\"" - } - } - }, - { - "evaluation_name": "Global Facts", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Global Facts", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.58, - "details": { - "description": "min=0.58, mean=0.58, max=0.58, sum=1.16 (2)", - "tab": "Accuracy", - "Global Facts - Observed inference time (s)": "{\"description\": \"min=0.343, mean=0.343, max=0.343, sum=0.687 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.34327178478240966\"}", - "Global Facts - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Global Facts - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Global Facts - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Global Facts - # prompt tokens": "{\"description\": \"min=399.71, mean=399.71, max=399.71, sum=799.42 (2)\", \"tab\": \"General information\", \"score\": \"399.71\"}", - "Global Facts - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"global_facts\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_global_facts\"" - } - } - }, - { - "evaluation_name": "Jurisprudence", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Jurisprudence", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.87, - "details": { - "description": "min=0.87, mean=0.87, max=0.87, sum=1.741 (2)", - "tab": "Accuracy", - "Jurisprudence - Observed inference time (s)": "{\"description\": \"min=0.33, mean=0.33, max=0.33, sum=0.659 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.32968640327453613\"}", - "Jurisprudence - # eval": "{\"description\": \"min=108, mean=108, max=108, sum=216 (2)\", \"tab\": \"General information\", \"score\": \"108.0\"}", - "Jurisprudence - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Jurisprudence - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Jurisprudence - # prompt tokens": "{\"description\": \"min=394.63, mean=394.63, max=394.63, sum=789.259 (2)\", \"tab\": \"General information\", \"score\": \"394.6296296296296\"}", - "Jurisprudence - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"jurisprudence\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_jurisprudence\"" - } - } - }, - { - "evaluation_name": "Philosophy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Philosophy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.83, - "details": { - "description": "min=0.83, mean=0.83, max=0.83, sum=1.659 (2)", - "tab": "Accuracy", - "Philosophy - Observed inference time (s)": "{\"description\": \"min=0.321, mean=0.321, max=0.321, sum=0.642 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.32124289515700755\"}", - "Philosophy - # eval": "{\"description\": \"min=311, mean=311, max=311, sum=622 (2)\", \"tab\": \"General information\", \"score\": \"311.0\"}", - "Philosophy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Philosophy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Philosophy - # prompt tokens": "{\"description\": \"min=329.084, mean=329.084, max=329.084, sum=658.167 (2)\", \"tab\": \"General information\", \"score\": \"329.08360128617363\"}", - "Philosophy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"philosophy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_philosophy\"" - } - } - }, - { - "evaluation_name": "Professional Psychology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Professional Psychology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.845, - "details": { - "description": "min=0.845, mean=0.845, max=0.845, sum=1.69 (2)", - "tab": "Accuracy", - "Professional Medicine - Observed inference time (s)": "{\"description\": \"min=0.367, mean=0.367, max=0.367, sum=0.733 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.36657266932375293\"}", - "Professional Accounting - Observed inference time (s)": "{\"description\": \"min=0.34, mean=0.34, max=0.34, sum=0.68 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.33986637440133605\"}", - "Professional Law - Observed inference time (s)": "{\"description\": \"min=0.386, mean=0.386, max=0.386, sum=0.772 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3858062526237856\"}", - "Professional Psychology - Observed inference time (s)": "{\"description\": \"min=0.334, mean=0.334, max=0.334, sum=0.668 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.33390796184539795\"}", - "Professional Medicine - # eval": "{\"description\": \"min=272, mean=272, max=272, sum=544 (2)\", \"tab\": \"General information\", \"score\": \"272.0\"}", - "Professional Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Medicine - # prompt tokens": "{\"description\": \"min=1094.489, mean=1094.489, max=1094.489, sum=2188.978 (2)\", \"tab\": \"General information\", \"score\": \"1094.4889705882354\"}", - "Professional Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Accounting - # eval": "{\"description\": \"min=282, mean=282, max=282, sum=564 (2)\", \"tab\": \"General information\", \"score\": \"282.0\"}", - "Professional Accounting - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Accounting - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # prompt tokens": "{\"description\": \"min=658.585, mean=658.585, max=658.585, sum=1317.17 (2)\", \"tab\": \"General information\", \"score\": \"658.5851063829788\"}", - "Professional Accounting - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Law - # eval": "{\"description\": \"min=1534, mean=1534, max=1534, sum=3068 (2)\", \"tab\": \"General information\", \"score\": \"1534.0\"}", - "Professional Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # prompt tokens": "{\"description\": \"min=1637.601, mean=1637.601, max=1637.601, sum=3275.202 (2)\", \"tab\": \"General information\", \"score\": \"1637.6010430247718\"}", - "Professional Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Psychology - # eval": "{\"description\": \"min=612, mean=612, max=612, sum=1224 (2)\", \"tab\": \"General information\", \"score\": \"612.0\"}", - "Professional Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # prompt tokens": "{\"description\": \"min=575.098, mean=575.098, max=575.098, sum=1150.196 (2)\", \"tab\": \"General information\", \"score\": \"575.0980392156863\"}", - "Professional Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"professional_psychology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_professional_psychology\"" - } - } - }, - { - "evaluation_name": "Us Foreign Policy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Us Foreign Policy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.93, - "details": { - "description": "min=0.93, mean=0.93, max=0.93, sum=1.86 (2)", - "tab": "Accuracy", - "Us Foreign Policy - Observed inference time (s)": "{\"description\": \"min=0.342, mean=0.342, max=0.342, sum=0.683 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.34171419143676757\"}", - "Us Foreign Policy - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Us Foreign Policy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Us Foreign Policy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Us Foreign Policy - # prompt tokens": "{\"description\": \"min=422.79, mean=422.79, max=422.79, sum=845.58 (2)\", \"tab\": \"General information\", \"score\": \"422.79\"}", - "Us Foreign Policy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"us_foreign_policy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_us_foreign_policy\"" - } - } - }, - { - "evaluation_name": "Astronomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Astronomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.888, - "details": { - "description": "min=0.888, mean=0.888, max=0.888, sum=1.776 (2)", - "tab": "Accuracy", - "Astronomy - Observed inference time (s)": "{\"description\": \"min=0.329, mean=0.329, max=0.329, sum=0.657 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3287427550867984\"}", - "Astronomy - # eval": "{\"description\": \"min=152, mean=152, max=152, sum=304 (2)\", \"tab\": \"General information\", \"score\": \"152.0\"}", - "Astronomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Astronomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Astronomy - # prompt tokens": "{\"description\": \"min=579.684, mean=579.684, max=579.684, sum=1159.368 (2)\", \"tab\": \"General information\", \"score\": \"579.6842105263158\"}", - "Astronomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"astronomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_astronomy\"" - } - } - }, - { - "evaluation_name": "Business Ethics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Business Ethics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8, - "details": { - "description": "min=0.8, mean=0.8, max=0.8, sum=1.6 (2)", - "tab": "Accuracy", - "Business Ethics - Observed inference time (s)": "{\"description\": \"min=0.327, mean=0.327, max=0.327, sum=0.654 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.327047655582428\"}", - "Business Ethics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Business Ethics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Business Ethics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Business Ethics - # prompt tokens": "{\"description\": \"min=569.52, mean=569.52, max=569.52, sum=1139.04 (2)\", \"tab\": \"General information\", \"score\": \"569.52\"}", - "Business Ethics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"business_ethics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_business_ethics\"" - } - } - }, - { - "evaluation_name": "Clinical Knowledge", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Clinical Knowledge", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.83, - "details": { - "description": "min=0.83, mean=0.83, max=0.83, sum=1.66 (2)", - "tab": "Accuracy", - "Clinical Knowledge - Observed inference time (s)": "{\"description\": \"min=0.344, mean=0.344, max=0.344, sum=0.687 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3435286764828664\"}", - "Clinical Knowledge - # eval": "{\"description\": \"min=265, mean=265, max=265, sum=530 (2)\", \"tab\": \"General information\", \"score\": \"265.0\"}", - "Clinical Knowledge - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Clinical Knowledge - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Clinical Knowledge - # prompt tokens": "{\"description\": \"min=397.928, mean=397.928, max=397.928, sum=795.857 (2)\", \"tab\": \"General information\", \"score\": \"397.92830188679244\"}", - "Clinical Knowledge - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"clinical_knowledge\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_clinical_knowledge\"" - } - } - }, - { - "evaluation_name": "Conceptual Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Conceptual Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.821, - "details": { - "description": "min=0.821, mean=0.821, max=0.821, sum=1.643 (2)", - "tab": "Accuracy", - "Conceptual Physics - Observed inference time (s)": "{\"description\": \"min=0.333, mean=0.333, max=0.333, sum=0.667 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.33338003361478763\"}", - "Conceptual Physics - # eval": "{\"description\": \"min=235, mean=235, max=235, sum=470 (2)\", \"tab\": \"General information\", \"score\": \"235.0\"}", - "Conceptual Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Conceptual Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Conceptual Physics - # prompt tokens": "{\"description\": \"min=304.834, mean=304.834, max=304.834, sum=609.668 (2)\", \"tab\": \"General information\", \"score\": \"304.83404255319147\"}", - "Conceptual Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"conceptual_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_conceptual_physics\"" - } - } - }, - { - "evaluation_name": "Electrical Engineering", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Electrical Engineering", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.745, - "details": { - "description": "min=0.745, mean=0.745, max=0.745, sum=1.49 (2)", - "tab": "Accuracy", - "Electrical Engineering - Observed inference time (s)": "{\"description\": \"min=0.354, mean=0.354, max=0.354, sum=0.709 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.35425889245395004\"}", - "Electrical Engineering - # eval": "{\"description\": \"min=145, mean=145, max=145, sum=290 (2)\", \"tab\": \"General information\", \"score\": \"145.0\"}", - "Electrical Engineering - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Electrical Engineering - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Electrical Engineering - # prompt tokens": "{\"description\": \"min=435.607, mean=435.607, max=435.607, sum=871.214 (2)\", \"tab\": \"General information\", \"score\": \"435.60689655172416\"}", - "Electrical Engineering - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"electrical_engineering\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_electrical_engineering\"" - } - } - }, - { - "evaluation_name": "Elementary Mathematics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Elementary Mathematics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.672, - "details": { - "description": "min=0.672, mean=0.672, max=0.672, sum=1.344 (2)", - "tab": "Accuracy", - "Elementary Mathematics - Observed inference time (s)": "{\"description\": \"min=0.334, mean=0.334, max=0.334, sum=0.669 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.33447367299801456\"}", - "Elementary Mathematics - # eval": "{\"description\": \"min=378, mean=378, max=378, sum=756 (2)\", \"tab\": \"General information\", \"score\": \"378.0\"}", - "Elementary Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Elementary Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Elementary Mathematics - # prompt tokens": "{\"description\": \"min=531.854, mean=531.854, max=531.854, sum=1063.709 (2)\", \"tab\": \"General information\", \"score\": \"531.8544973544973\"}", - "Elementary Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"elementary_mathematics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_elementary_mathematics\"" - } - } - }, - { - "evaluation_name": "Formal Logic", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Formal Logic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.675, - "details": { - "description": "min=0.675, mean=0.675, max=0.675, sum=1.349 (2)", - "tab": "Accuracy", - "Formal Logic - Observed inference time (s)": "{\"description\": \"min=0.35, mean=0.35, max=0.35, sum=0.7 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.349764451148018\"}", - "Formal Logic - # eval": "{\"description\": \"min=126, mean=126, max=126, sum=252 (2)\", \"tab\": \"General information\", \"score\": \"126.0\"}", - "Formal Logic - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Formal Logic - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Formal Logic - # prompt tokens": "{\"description\": \"min=601.778, mean=601.778, max=601.778, sum=1203.556 (2)\", \"tab\": \"General information\", \"score\": \"601.7777777777778\"}", - "Formal Logic - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"formal_logic\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_formal_logic\"" - } - } - }, - { - "evaluation_name": "High School World History", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on High School World History", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.907, - "details": { - "description": "min=0.907, mean=0.907, max=0.907, sum=1.814 (2)", - "tab": "Accuracy", - "High School Biology - Observed inference time (s)": "{\"description\": \"min=0.348, mean=0.348, max=0.348, sum=0.697 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.34841231069257184\"}", - "High School Chemistry - Observed inference time (s)": "{\"description\": \"min=0.325, mean=0.325, max=0.325, sum=0.65 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3249026636771968\"}", - "High School Computer Science - Observed inference time (s)": "{\"description\": \"min=0.376, mean=0.376, max=0.376, sum=0.752 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3761155128479004\"}", - "High School European History - Observed inference time (s)": "{\"description\": \"min=0.559, mean=0.559, max=0.559, sum=1.118 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.558924115787853\"}", - "High School Geography - Observed inference time (s)": "{\"description\": \"min=0.303, mean=0.303, max=0.303, sum=0.606 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.30311920907762313\"}", - "High School Government And Politics - Observed inference time (s)": "{\"description\": \"min=0.319, mean=0.319, max=0.319, sum=0.639 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3192925144353679\"}", - "High School Macroeconomics - Observed inference time (s)": "{\"description\": \"min=0.321, mean=0.321, max=0.321, sum=0.643 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3212899880531507\"}", - "High School Mathematics - Observed inference time (s)": "{\"description\": \"min=0.331, mean=0.331, max=0.331, sum=0.661 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3307388570573595\"}", - "High School Microeconomics - Observed inference time (s)": "{\"description\": \"min=0.332, mean=0.332, max=0.332, sum=0.663 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3317271210566288\"}", - "High School Physics - Observed inference time (s)": "{\"description\": \"min=0.34, mean=0.34, max=0.34, sum=0.68 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.34023177229016033\"}", - "High School Psychology - Observed inference time (s)": "{\"description\": \"min=0.327, mean=0.327, max=0.327, sum=0.655 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3273837903224\"}", - "High School Statistics - Observed inference time (s)": "{\"description\": \"min=0.359, mean=0.359, max=0.359, sum=0.718 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.359178250586545\"}", - "High School US History - Observed inference time (s)": "{\"description\": \"min=0.444, mean=0.444, max=0.444, sum=0.887 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.443670579031402\"}", - "High School World History - Observed inference time (s)": "{\"description\": \"min=0.382, mean=0.382, max=0.382, sum=0.764 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3818797411294929\"}", - "High School Biology - # eval": "{\"description\": \"min=310, mean=310, max=310, sum=620 (2)\", \"tab\": \"General information\", \"score\": \"310.0\"}", - "High School Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Biology - # prompt tokens": "{\"description\": \"min=513.671, mean=513.671, max=513.671, sum=1027.342 (2)\", \"tab\": \"General information\", \"score\": \"513.6709677419354\"}", - "High School Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Chemistry - # eval": "{\"description\": \"min=203, mean=203, max=203, sum=406 (2)\", \"tab\": \"General information\", \"score\": \"203.0\"}", - "High School Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # prompt tokens": "{\"description\": \"min=496.704, mean=496.704, max=496.704, sum=993.409 (2)\", \"tab\": \"General information\", \"score\": \"496.70443349753697\"}", - "High School Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "High School Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # prompt tokens": "{\"description\": \"min=867.78, mean=867.78, max=867.78, sum=1735.56 (2)\", \"tab\": \"General information\", \"score\": \"867.78\"}", - "High School Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School European History - # eval": "{\"description\": \"min=165, mean=165, max=165, sum=330 (2)\", \"tab\": \"General information\", \"score\": \"165.0\"}", - "High School European History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School European History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # prompt tokens": "{\"description\": \"min=2797.885, mean=2797.885, max=2797.885, sum=5595.77 (2)\", \"tab\": \"General information\", \"score\": \"2797.8848484848486\"}", - "High School European History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Geography - # eval": "{\"description\": \"min=198, mean=198, max=198, sum=396 (2)\", \"tab\": \"General information\", \"score\": \"198.0\"}", - "High School Geography - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Geography - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # prompt tokens": "{\"description\": \"min=372.035, mean=372.035, max=372.035, sum=744.071 (2)\", \"tab\": \"General information\", \"score\": \"372.0353535353535\"}", - "High School Geography - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Government And Politics - # eval": "{\"description\": \"min=193, mean=193, max=193, sum=386 (2)\", \"tab\": \"General information\", \"score\": \"193.0\"}", - "High School Government And Politics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Government And Politics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # prompt tokens": "{\"description\": \"min=465.824, mean=465.824, max=465.824, sum=931.648 (2)\", \"tab\": \"General information\", \"score\": \"465.8238341968912\"}", - "High School Government And Politics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Macroeconomics - # eval": "{\"description\": \"min=390, mean=390, max=390, sum=780 (2)\", \"tab\": \"General information\", \"score\": \"390.0\"}", - "High School Macroeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Macroeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # prompt tokens": "{\"description\": \"min=370.908, mean=370.908, max=370.908, sum=741.815 (2)\", \"tab\": \"General information\", \"score\": \"370.9076923076923\"}", - "High School Macroeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Mathematics - # eval": "{\"description\": \"min=270, mean=270, max=270, sum=540 (2)\", \"tab\": \"General information\", \"score\": \"270.0\"}", - "High School Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # prompt tokens": "{\"description\": \"min=532.356, mean=532.356, max=532.356, sum=1064.711 (2)\", \"tab\": \"General information\", \"score\": \"532.3555555555556\"}", - "High School Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Microeconomics - # eval": "{\"description\": \"min=238, mean=238, max=238, sum=476 (2)\", \"tab\": \"General information\", \"score\": \"238.0\"}", - "High School Microeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Microeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # prompt tokens": "{\"description\": \"min=399.013, mean=399.013, max=399.013, sum=798.025 (2)\", \"tab\": \"General information\", \"score\": \"399.0126050420168\"}", - "High School Microeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Physics - # eval": "{\"description\": \"min=151, mean=151, max=151, sum=302 (2)\", \"tab\": \"General information\", \"score\": \"151.0\"}", - "High School Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # prompt tokens": "{\"description\": \"min=560.457, mean=560.457, max=560.457, sum=1120.914 (2)\", \"tab\": \"General information\", \"score\": \"560.4569536423841\"}", - "High School Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Psychology - # eval": "{\"description\": \"min=545, mean=545, max=545, sum=1090 (2)\", \"tab\": \"General information\", \"score\": \"545.0\"}", - "High School Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # prompt tokens": "{\"description\": \"min=495.242, mean=495.242, max=495.242, sum=990.484 (2)\", \"tab\": \"General information\", \"score\": \"495.2422018348624\"}", - "High School Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Statistics - # eval": "{\"description\": \"min=216, mean=216, max=216, sum=432 (2)\", \"tab\": \"General information\", \"score\": \"216.0\"}", - "High School Statistics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Statistics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # prompt tokens": "{\"description\": \"min=795.639, mean=795.639, max=795.639, sum=1591.278 (2)\", \"tab\": \"General information\", \"score\": \"795.6388888888889\"}", - "High School Statistics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School US History - # eval": "{\"description\": \"min=204, mean=204, max=204, sum=408 (2)\", \"tab\": \"General information\", \"score\": \"204.0\"}", - "High School US History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School US History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # prompt tokens": "{\"description\": \"min=2217.809, mean=2217.809, max=2217.809, sum=4435.618 (2)\", \"tab\": \"General information\", \"score\": \"2217.8088235294117\"}", - "High School US History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School World History - # eval": "{\"description\": \"min=237, mean=237, max=237, sum=474 (2)\", \"tab\": \"General information\", \"score\": \"237.0\"}", - "High School World History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School World History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # prompt tokens": "{\"description\": \"min=1428.173, mean=1428.173, max=1428.173, sum=2856.346 (2)\", \"tab\": \"General information\", \"score\": \"1428.1729957805908\"}", - "High School World History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"high_school_world_history\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_high_school_world_history\"" - } - } - }, - { - "evaluation_name": "Human Sexuality", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Human Sexuality", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.855, - "details": { - "description": "min=0.855, mean=0.855, max=0.855, sum=1.71 (2)", - "tab": "Accuracy", - "Human Aging - Observed inference time (s)": "{\"description\": \"min=0.345, mean=0.345, max=0.345, sum=0.691 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3452627787140987\"}", - "Human Sexuality - Observed inference time (s)": "{\"description\": \"min=0.346, mean=0.346, max=0.346, sum=0.692 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.34599654183132955\"}", - "Human Aging - # eval": "{\"description\": \"min=223, mean=223, max=223, sum=446 (2)\", \"tab\": \"General information\", \"score\": \"223.0\"}", - "Human Aging - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Aging - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Aging - # prompt tokens": "{\"description\": \"min=319.888, mean=319.888, max=319.888, sum=639.776 (2)\", \"tab\": \"General information\", \"score\": \"319.88789237668163\"}", - "Human Aging - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Human Sexuality - # eval": "{\"description\": \"min=131, mean=131, max=131, sum=262 (2)\", \"tab\": \"General information\", \"score\": \"131.0\"}", - "Human Sexuality - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Sexuality - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # prompt tokens": "{\"description\": \"min=341.168, mean=341.168, max=341.168, sum=682.336 (2)\", \"tab\": \"General information\", \"score\": \"341.1679389312977\"}", - "Human Sexuality - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"human_sexuality\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_human_sexuality\"" - } - } - }, - { - "evaluation_name": "International Law", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on International Law", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.884, - "details": { - "description": "min=0.884, mean=0.884, max=0.884, sum=1.769 (2)", - "tab": "Accuracy", - "International Law - Observed inference time (s)": "{\"description\": \"min=0.37, mean=0.37, max=0.37, sum=0.741 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3704575231252623\"}", - "International Law - # eval": "{\"description\": \"min=121, mean=121, max=121, sum=242 (2)\", \"tab\": \"General information\", \"score\": \"121.0\"}", - "International Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "International Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "International Law - # prompt tokens": "{\"description\": \"min=639.818, mean=639.818, max=639.818, sum=1279.636 (2)\", \"tab\": \"General information\", \"score\": \"639.8181818181819\"}", - "International Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"international_law\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_international_law\"" - } - } - }, - { - "evaluation_name": "Logical Fallacies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Logical Fallacies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.816, - "details": { - "description": "min=0.816, mean=0.816, max=0.816, sum=1.632 (2)", - "tab": "Accuracy", - "Logical Fallacies - Observed inference time (s)": "{\"description\": \"min=0.307, mean=0.307, max=0.307, sum=0.613 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.30655721506458117\"}", - "Logical Fallacies - # eval": "{\"description\": \"min=163, mean=163, max=163, sum=326 (2)\", \"tab\": \"General information\", \"score\": \"163.0\"}", - "Logical Fallacies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Logical Fallacies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Logical Fallacies - # prompt tokens": "{\"description\": \"min=449.564, mean=449.564, max=449.564, sum=899.129 (2)\", \"tab\": \"General information\", \"score\": \"449.5644171779141\"}", - "Logical Fallacies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"logical_fallacies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_logical_fallacies\"" - } - } - }, - { - "evaluation_name": "Machine Learning", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Machine Learning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.714, - "details": { - "description": "min=0.714, mean=0.714, max=0.714, sum=1.429 (2)", - "tab": "Accuracy", - "Machine Learning - Observed inference time (s)": "{\"description\": \"min=0.375, mean=0.375, max=0.375, sum=0.75 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3751111796924046\"}", - "Machine Learning - # eval": "{\"description\": \"min=112, mean=112, max=112, sum=224 (2)\", \"tab\": \"General information\", \"score\": \"112.0\"}", - "Machine Learning - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Machine Learning - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Machine Learning - # prompt tokens": "{\"description\": \"min=668.054, mean=668.054, max=668.054, sum=1336.107 (2)\", \"tab\": \"General information\", \"score\": \"668.0535714285714\"}", - "Machine Learning - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"machine_learning\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_machine_learning\"" - } - } - }, - { - "evaluation_name": "Management", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Management", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.903, - "details": { - "description": "min=0.903, mean=0.903, max=0.903, sum=1.806 (2)", - "tab": "Accuracy", - "Management - Observed inference time (s)": "{\"description\": \"min=0.337, mean=0.337, max=0.337, sum=0.674 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3368335811837206\"}", - "Management - # eval": "{\"description\": \"min=103, mean=103, max=103, sum=206 (2)\", \"tab\": \"General information\", \"score\": \"103.0\"}", - "Management - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Management - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Management - # prompt tokens": "{\"description\": \"min=283.786, mean=283.786, max=283.786, sum=567.573 (2)\", \"tab\": \"General information\", \"score\": \"283.7864077669903\"}", - "Management - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"management\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_management\"" - } - } - }, - { - "evaluation_name": "Marketing", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Marketing", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.927, - "details": { - "description": "min=0.927, mean=0.927, max=0.927, sum=1.855 (2)", - "tab": "Accuracy", - "Marketing - Observed inference time (s)": "{\"description\": \"min=0.32, mean=0.32, max=0.32, sum=0.64 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.320215484015962\"}", - "Marketing - # eval": "{\"description\": \"min=234, mean=234, max=234, sum=468 (2)\", \"tab\": \"General information\", \"score\": \"234.0\"}", - "Marketing - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Marketing - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Marketing - # prompt tokens": "{\"description\": \"min=404.218, mean=404.218, max=404.218, sum=808.436 (2)\", \"tab\": \"General information\", \"score\": \"404.21794871794873\"}", - "Marketing - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"marketing\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_marketing\"" - } - } - }, - { - "evaluation_name": "Medical Genetics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Medical Genetics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9, - "details": { - "description": "min=0.9, mean=0.9, max=0.9, sum=1.8 (2)", - "tab": "Accuracy", - "Medical Genetics - Observed inference time (s)": "{\"description\": \"min=0.327, mean=0.327, max=0.327, sum=0.654 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3268785071372986\"}", - "Medical Genetics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Medical Genetics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Medical Genetics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Medical Genetics - # prompt tokens": "{\"description\": \"min=340.99, mean=340.99, max=340.99, sum=681.98 (2)\", \"tab\": \"General information\", \"score\": \"340.99\"}", - "Medical Genetics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"medical_genetics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_medical_genetics\"" - } - } - }, - { - "evaluation_name": "Miscellaneous", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Miscellaneous", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.914, - "details": { - "description": "min=0.914, mean=0.914, max=0.914, sum=1.829 (2)", - "tab": "Accuracy", - "Miscellaneous - Observed inference time (s)": "{\"description\": \"min=0.321, mean=0.321, max=0.321, sum=0.641 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.32054392161801704\"}", - "Miscellaneous - # eval": "{\"description\": \"min=783, mean=783, max=783, sum=1566 (2)\", \"tab\": \"General information\", \"score\": \"783.0\"}", - "Miscellaneous - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Miscellaneous - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Miscellaneous - # prompt tokens": "{\"description\": \"min=299.911, mean=299.911, max=299.911, sum=599.821 (2)\", \"tab\": \"General information\", \"score\": \"299.9106002554278\"}", - "Miscellaneous - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"miscellaneous\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_miscellaneous\"" - } - } - }, - { - "evaluation_name": "Moral Scenarios", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Moral Scenarios", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.698, - "details": { - "description": "min=0.698, mean=0.698, max=0.698, sum=1.397 (2)", - "tab": "Accuracy", - "Moral Disputes - Observed inference time (s)": "{\"description\": \"min=0.322, mean=0.322, max=0.322, sum=0.644 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.321929149544997\"}", - "Moral Scenarios - Observed inference time (s)": "{\"description\": \"min=0.351, mean=0.351, max=0.351, sum=0.702 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3511003518237748\"}", - "Moral Disputes - # eval": "{\"description\": \"min=346, mean=346, max=346, sum=692 (2)\", \"tab\": \"General information\", \"score\": \"346.0\"}", - "Moral Disputes - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Disputes - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Disputes - # prompt tokens": "{\"description\": \"min=476.113, mean=476.113, max=476.113, sum=952.225 (2)\", \"tab\": \"General information\", \"score\": \"476.1127167630058\"}", - "Moral Disputes - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Moral Scenarios - # eval": "{\"description\": \"min=895, mean=895, max=895, sum=1790 (2)\", \"tab\": \"General information\", \"score\": \"895.0\"}", - "Moral Scenarios - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Scenarios - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # prompt tokens": "{\"description\": \"min=656.455, mean=656.455, max=656.455, sum=1312.909 (2)\", \"tab\": \"General information\", \"score\": \"656.454748603352\"}", - "Moral Scenarios - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"moral_scenarios\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_moral_scenarios\"" - } - } - }, - { - "evaluation_name": "Nutrition", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Nutrition", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.882, - "details": { - "description": "min=0.882, mean=0.882, max=0.882, sum=1.765 (2)", - "tab": "Accuracy", - "Nutrition - Observed inference time (s)": "{\"description\": \"min=0.356, mean=0.356, max=0.356, sum=0.711 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.35563821730270884\"}", - "Nutrition - # eval": "{\"description\": \"min=306, mean=306, max=306, sum=612 (2)\", \"tab\": \"General information\", \"score\": \"306.0\"}", - "Nutrition - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Nutrition - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Nutrition - # prompt tokens": "{\"description\": \"min=586.814, mean=586.814, max=586.814, sum=1173.627 (2)\", \"tab\": \"General information\", \"score\": \"586.8137254901961\"}", - "Nutrition - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"nutrition\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_nutrition\"" - } - } - }, - { - "evaluation_name": "Prehistory", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Prehistory", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.895, - "details": { - "description": "min=0.895, mean=0.895, max=0.895, sum=1.79 (2)", - "tab": "Accuracy", - "Prehistory - Observed inference time (s)": "{\"description\": \"min=0.343, mean=0.343, max=0.343, sum=0.685 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.34269326410175843\"}", - "Prehistory - # eval": "{\"description\": \"min=324, mean=324, max=324, sum=648 (2)\", \"tab\": \"General information\", \"score\": \"324.0\"}", - "Prehistory - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Prehistory - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Prehistory - # prompt tokens": "{\"description\": \"min=514.528, mean=514.528, max=514.528, sum=1029.056 (2)\", \"tab\": \"General information\", \"score\": \"514.5277777777778\"}", - "Prehistory - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"prehistory\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_prehistory\"" - } - } - }, - { - "evaluation_name": "Public Relations", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Public Relations", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.727, - "details": { - "description": "min=0.727, mean=0.727, max=0.727, sum=1.455 (2)", - "tab": "Accuracy", - "Public Relations - Observed inference time (s)": "{\"description\": \"min=0.345, mean=0.345, max=0.345, sum=0.69 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.34484653039412066\"}", - "Public Relations - # eval": "{\"description\": \"min=110, mean=110, max=110, sum=220 (2)\", \"tab\": \"General information\", \"score\": \"110.0\"}", - "Public Relations - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Public Relations - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Public Relations - # prompt tokens": "{\"description\": \"min=405.318, mean=405.318, max=405.318, sum=810.636 (2)\", \"tab\": \"General information\", \"score\": \"405.3181818181818\"}", - "Public Relations - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"public_relations\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_public_relations\"" - } - } - }, - { - "evaluation_name": "Security Studies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Security Studies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.845, - "details": { - "description": "min=0.845, mean=0.845, max=0.845, sum=1.69 (2)", - "tab": "Accuracy", - "Security Studies - Observed inference time (s)": "{\"description\": \"min=0.369, mean=0.369, max=0.369, sum=0.737 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3686914687253991\"}", - "Security Studies - # eval": "{\"description\": \"min=245, mean=245, max=245, sum=490 (2)\", \"tab\": \"General information\", \"score\": \"245.0\"}", - "Security Studies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Security Studies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Security Studies - # prompt tokens": "{\"description\": \"min=1164.473, mean=1164.473, max=1164.473, sum=2328.947 (2)\", \"tab\": \"General information\", \"score\": \"1164.4734693877551\"}", - "Security Studies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"security_studies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_security_studies\"" - } - } - }, - { - "evaluation_name": "Sociology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Sociology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.92, - "details": { - "description": "min=0.92, mean=0.92, max=0.92, sum=1.841 (2)", - "tab": "Accuracy", - "Sociology - Observed inference time (s)": "{\"description\": \"min=0.324, mean=0.324, max=0.324, sum=0.647 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3236708546159279\"}", - "Sociology - # eval": "{\"description\": \"min=201, mean=201, max=201, sum=402 (2)\", \"tab\": \"General information\", \"score\": \"201.0\"}", - "Sociology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Sociology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Sociology - # prompt tokens": "{\"description\": \"min=445.517, mean=445.517, max=445.517, sum=891.035 (2)\", \"tab\": \"General information\", \"score\": \"445.51741293532336\"}", - "Sociology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"sociology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_sociology\"" - } - } - }, - { - "evaluation_name": "Virology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Virology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.566, - "details": { - "description": "min=0.566, mean=0.566, max=0.566, sum=1.133 (2)", - "tab": "Accuracy", - "Virology - Observed inference time (s)": "{\"description\": \"min=0.324, mean=0.324, max=0.324, sum=0.647 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3235311522541276\"}", - "Virology - # eval": "{\"description\": \"min=166, mean=166, max=166, sum=332 (2)\", \"tab\": \"General information\", \"score\": \"166.0\"}", - "Virology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Virology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Virology - # prompt tokens": "{\"description\": \"min=343.018, mean=343.018, max=343.018, sum=686.036 (2)\", \"tab\": \"General information\", \"score\": \"343.01807228915663\"}", - "Virology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"virology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_virology\"" - } - } - }, - { - "evaluation_name": "World Religions", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on World Religions", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.883, - "details": { - "description": "min=0.883, mean=0.883, max=0.883, sum=1.766 (2)", - "tab": "Accuracy", - "World Religions - Observed inference time (s)": "{\"description\": \"min=0.303, mean=0.303, max=0.303, sum=0.606 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.30298223132975616\"}", - "World Religions - # eval": "{\"description\": \"min=171, mean=171, max=171, sum=342 (2)\", \"tab\": \"General information\", \"score\": \"171.0\"}", - "World Religions - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "World Religions - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "World Religions - # prompt tokens": "{\"description\": \"min=274.52, mean=274.52, max=274.52, sum=549.041 (2)\", \"tab\": \"General information\", \"score\": \"274.5204678362573\"}", - "World Religions - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"world_religions\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_world_religions\"" - } - } - }, - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.722, - "details": { - "description": "", - "tab": "Efficiency" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/meta_llama-30b.json b/data/models/meta_llama-30b.json deleted file mode 100644 index 9c44c810143b6e689149ddaf4d43b7847c1e98f0..0000000000000000000000000000000000000000 --- a/data/models/meta_llama-30b.json +++ /dev/null @@ -1,674 +0,0 @@ -{ - "model_info": { - "name": "LLaMA 30B", - "id": "meta/LLaMA-30B", - "developer": "Meta", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_classic/meta_LLaMA-30B/1774096308.339228", - "retrieved_timestamp": "1774096308.339228", - "source_metadata": { - "source_name": "helm_classic", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_classic", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_classic", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperform on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.781, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Calibration": "{\"description\": \"\", \"tab\": \"Calibration\", \"score\": \"\"}", - "Mean win rate - Robustness": "{\"description\": \"\", \"tab\": \"Robustness\", \"score\": \"0.8149650349650349\"}", - "Mean win rate - Fairness": "{\"description\": \"\", \"tab\": \"Fairness\", \"score\": \"0.8224708624708624\"}", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}", - "Mean win rate - Bias": "{\"description\": \"\", \"tab\": \"Bias\", \"score\": \"0.5955016826844834\"}", - "Mean win rate - Toxicity": "{\"description\": \"\", \"tab\": \"Toxicity\", \"score\": \"0.6467365967365968\"}", - "Mean win rate - Summarization metrics": "{\"description\": \"\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.531, - "details": { - "description": "min=0.33, mean=0.531, max=0.83, sum=2.657 (5)", - "tab": "Accuracy", - "MMLU - ECE (10-bin)": "{\"description\": \"min=0.051, mean=0.093, max=0.139, sum=0.464 (5)\", \"tab\": \"Calibration\", \"score\": \"\"}", - "MMLU - EM (Robustness)": "{\"description\": \"min=0.22, mean=0.461, max=0.82, sum=2.305 (5)\", \"tab\": \"Robustness\", \"score\": \"0.4609122807017544\"}", - "MMLU - EM (Fairness)": "{\"description\": \"min=0.28, mean=0.496, max=0.81, sum=2.481 (5)\", \"tab\": \"Fairness\", \"score\": \"0.49617543859649127\"}", - "MMLU - Denoised inference time (s)": "{\"description\": \"5 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=397.65, mean=522.547, max=684.675, sum=2612.735 (5)\", \"tab\": \"General information\", \"score\": \"522.5470877192982\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "MMLU - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "BoolQ", - "source_data": { - "dataset_name": "BoolQ", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on BoolQ", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.861, - "details": { - "description": "min=0.861, mean=0.861, max=0.861, sum=0.861 (1)", - "tab": "Accuracy", - "BoolQ - ECE (10-bin)": "{\"description\": \"min=0.164, mean=0.164, max=0.164, sum=0.164 (1)\", \"tab\": \"Calibration\", \"score\": \"\"}", - "BoolQ - EM (Robustness)": "{\"description\": \"min=0.791, mean=0.791, max=0.791, sum=0.791 (1)\", \"tab\": \"Robustness\", \"score\": \"0.791\"}", - "BoolQ - EM (Fairness)": "{\"description\": \"min=0.813, mean=0.813, max=0.813, sum=0.813 (1)\", \"tab\": \"Fairness\", \"score\": \"0.813\"}", - "BoolQ - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "BoolQ - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "BoolQ - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "BoolQ - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "BoolQ - # prompt tokens": "{\"description\": \"min=1439.447, mean=1439.447, max=1439.447, sum=1439.447 (1)\", \"tab\": \"General information\", \"score\": \"1439.447\"}", - "BoolQ - # output tokens": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "BoolQ - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "BoolQ - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.752, - "details": { - "description": "min=0.752, mean=0.752, max=0.752, sum=0.752 (1)", - "tab": "Accuracy", - "NarrativeQA - ECE (10-bin)": "{\"description\": \"min=0.296, mean=0.296, max=0.296, sum=0.296 (1)\", \"tab\": \"Calibration\", \"score\": \"\"}", - "NarrativeQA - F1 (Robustness)": "{\"description\": \"min=0.611, mean=0.611, max=0.611, sum=0.611 (1)\", \"tab\": \"Robustness\", \"score\": \"0.6105202153922532\"}", - "NarrativeQA - F1 (Fairness)": "{\"description\": \"min=0.657, mean=0.657, max=0.657, sum=0.657 (1)\", \"tab\": \"Fairness\", \"score\": \"0.6567447414077484\"}", - "NarrativeQA - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=1.437, mean=1.437, max=1.437, sum=1.437 (1)\", \"tab\": \"General information\", \"score\": \"1.4366197183098592\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=1541.115, mean=1541.115, max=1541.115, sum=1541.115 (1)\", \"tab\": \"General information\", \"score\": \"1541.1154929577465\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=100, mean=100, max=100, sum=100 (1)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "NarrativeQA - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NarrativeQA - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NarrativeQA - Stereotypes (gender)": "{\"description\": \"min=0.4, mean=0.4, max=0.4, sum=0.4 (1)\", \"tab\": \"Bias\", \"score\": \"0.4\"}", - "NarrativeQA - Representation (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=0.667 (1)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666667\"}", - "NarrativeQA - Representation (gender)": "{\"description\": \"min=0.214, mean=0.214, max=0.214, sum=0.214 (1)\", \"tab\": \"Bias\", \"score\": \"0.2142857142857143\"}", - "NarrativeQA - Toxic fraction": "{\"description\": \"min=0.011, mean=0.011, max=0.011, sum=0.011 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.011267605633802818\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (open-book)", - "source_data": { - "dataset_name": "NaturalQuestions (open-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (open-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.666, - "details": { - "description": "min=0.666, mean=0.666, max=0.666, sum=0.666 (1)", - "tab": "Accuracy", - "NaturalQuestions (closed-book) - ECE (10-bin)": "{\"description\": \"min=0.264, mean=0.264, max=0.264, sum=0.264 (1)\", \"tab\": \"Calibration\", \"score\": \"\"}", - "NaturalQuestions (open-book) - ECE (10-bin)": "{\"description\": \"min=0.451, mean=0.451, max=0.451, sum=0.451 (1)\", \"tab\": \"Calibration\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - F1 (Robustness)": "{\"description\": \"min=0.36, mean=0.36, max=0.36, sum=0.36 (1)\", \"tab\": \"Robustness\", \"score\": \"0.36029476515740994\"}", - "NaturalQuestions (open-book) - F1 (Robustness)": "{\"description\": \"min=0.612, mean=0.612, max=0.612, sum=0.612 (1)\", \"tab\": \"Robustness\", \"score\": \"0.6123442768470954\"}", - "NaturalQuestions (closed-book) - F1 (Fairness)": "{\"description\": \"min=0.356, mean=0.356, max=0.356, sum=0.356 (1)\", \"tab\": \"Fairness\", \"score\": \"0.35638449124084753\"}", - "NaturalQuestions (open-book) - F1 (Fairness)": "{\"description\": \"min=0.621, mean=0.621, max=0.621, sum=0.621 (1)\", \"tab\": \"Fairness\", \"score\": \"0.6212987885688864\"}", - "NaturalQuestions (closed-book) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NaturalQuestions (open-book) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=137.383, mean=137.383, max=137.383, sum=137.383 (1)\", \"tab\": \"General information\", \"score\": \"137.383\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=300, mean=300, max=300, sum=300 (1)\", \"tab\": \"General information\", \"score\": \"300.0\"}", - "NaturalQuestions (closed-book) - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=3.722, mean=3.722, max=3.722, sum=3.722 (1)\", \"tab\": \"General information\", \"score\": \"3.722\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.049, mean=0.049, max=0.049, sum=0.049 (1)\", \"tab\": \"General information\", \"score\": \"0.049\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1407.178, mean=1407.178, max=1407.178, sum=1407.178 (1)\", \"tab\": \"General information\", \"score\": \"1407.178\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=300, mean=300, max=300, sum=300 (1)\", \"tab\": \"General information\", \"score\": \"300.0\"}", - "NaturalQuestions (open-book) - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NaturalQuestions (closed-book) - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=0.667 (1)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666667\"}", - "NaturalQuestions (closed-book) - Stereotypes (gender)": "{\"description\": \"min=0.5, mean=0.5, max=0.5, sum=0.5 (1)\", \"tab\": \"Bias\", \"score\": \"0.5\"}", - "NaturalQuestions (closed-book) - Representation (race)": "{\"description\": \"min=0.328, mean=0.328, max=0.328, sum=0.328 (1)\", \"tab\": \"Bias\", \"score\": \"0.32753623188405795\"}", - "NaturalQuestions (closed-book) - Representation (gender)": "{\"description\": \"min=0.333, mean=0.333, max=0.333, sum=0.333 (1)\", \"tab\": \"Bias\", \"score\": \"0.33333333333333337\"}", - "NaturalQuestions (open-book) - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NaturalQuestions (open-book) - Stereotypes (gender)": "{\"description\": \"min=0.1, mean=0.1, max=0.1, sum=0.1 (1)\", \"tab\": \"Bias\", \"score\": \"0.09999999999999998\"}", - "NaturalQuestions (open-book) - Representation (race)": "{\"description\": \"min=0.39, mean=0.39, max=0.39, sum=0.39 (1)\", \"tab\": \"Bias\", \"score\": \"0.3900709219858156\"}", - "NaturalQuestions (open-book) - Representation (gender)": "{\"description\": \"min=0.257, mean=0.257, max=0.257, sum=0.257 (1)\", \"tab\": \"Bias\", \"score\": \"0.2567567567567568\"}", - "NaturalQuestions (closed-book) - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}", - "NaturalQuestions (open-book) - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "QuAC", - "source_data": { - "dataset_name": "QuAC", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on QuAC", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.39, - "details": { - "description": "min=0.39, mean=0.39, max=0.39, sum=0.39 (1)", - "tab": "Accuracy", - "QuAC - ECE (10-bin)": "{\"description\": \"min=0.146, mean=0.146, max=0.146, sum=0.146 (1)\", \"tab\": \"Calibration\", \"score\": \"\"}", - "QuAC - F1 (Robustness)": "{\"description\": \"min=0.273, mean=0.273, max=0.273, sum=0.273 (1)\", \"tab\": \"Robustness\", \"score\": \"0.27320176375521127\"}", - "QuAC - F1 (Fairness)": "{\"description\": \"min=0.325, mean=0.325, max=0.325, sum=0.325 (1)\", \"tab\": \"Fairness\", \"score\": \"0.3253423128866467\"}", - "QuAC - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "QuAC - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "QuAC - # train": "{\"description\": \"min=0.507, mean=0.507, max=0.507, sum=0.507 (1)\", \"tab\": \"General information\", \"score\": \"0.507\"}", - "QuAC - truncated": "{\"description\": \"min=0.06, mean=0.06, max=0.06, sum=0.06 (1)\", \"tab\": \"General information\", \"score\": \"0.06\"}", - "QuAC - # prompt tokens": "{\"description\": \"min=1498.657, mean=1498.657, max=1498.657, sum=1498.657 (1)\", \"tab\": \"General information\", \"score\": \"1498.657\"}", - "QuAC - # output tokens": "{\"description\": \"min=99.987, mean=99.987, max=99.987, sum=99.987 (1)\", \"tab\": \"General information\", \"score\": \"99.987\"}", - "QuAC - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "QuAC - Stereotypes (race)": "{\"description\": \"min=0.571, mean=0.571, max=0.571, sum=0.571 (1)\", \"tab\": \"Bias\", \"score\": \"0.5714285714285715\"}", - "QuAC - Stereotypes (gender)": "{\"description\": \"min=0.436, mean=0.436, max=0.436, sum=0.436 (1)\", \"tab\": \"Bias\", \"score\": \"0.43576827288346653\"}", - "QuAC - Representation (race)": "{\"description\": \"min=0.229, mean=0.229, max=0.229, sum=0.229 (1)\", \"tab\": \"Bias\", \"score\": \"0.22891566265060237\"}", - "QuAC - Representation (gender)": "{\"description\": \"min=0.222, mean=0.222, max=0.222, sum=0.222 (1)\", \"tab\": \"Bias\", \"score\": \"0.22215709261430247\"}", - "QuAC - Toxic fraction": "{\"description\": \"min=0.004, mean=0.004, max=0.004, sum=0.004 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.004\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "HellaSwag", - "source_data": { - "dataset_name": "HellaSwag", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on HellaSwag", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "HellaSwag - ECE (10-bin)": "{\"description\": \"No matching runs\", \"tab\": \"Calibration\", \"score\": \"\"}", - "HellaSwag - EM (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "HellaSwag - EM (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "HellaSwag - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "HellaSwag - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "OpenbookQA - ECE (10-bin)": "{\"description\": \"No matching runs\", \"tab\": \"Calibration\", \"score\": \"\"}", - "OpenbookQA - EM (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "OpenbookQA - EM (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "OpenbookQA - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "OpenbookQA - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "TruthfulQA", - "source_data": { - "dataset_name": "TruthfulQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on TruthfulQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.344, - "details": { - "description": "min=0.344, mean=0.344, max=0.344, sum=0.344 (1)", - "tab": "Accuracy", - "TruthfulQA - ECE (10-bin)": "{\"description\": \"min=0.15, mean=0.15, max=0.15, sum=0.15 (1)\", \"tab\": \"Calibration\", \"score\": \"\"}", - "TruthfulQA - EM (Robustness)": "{\"description\": \"min=0.281, mean=0.281, max=0.281, sum=0.281 (1)\", \"tab\": \"Robustness\", \"score\": \"0.28134556574923547\"}", - "TruthfulQA - EM (Fairness)": "{\"description\": \"min=0.266, mean=0.266, max=0.266, sum=0.266 (1)\", \"tab\": \"Fairness\", \"score\": \"0.26605504587155965\"}", - "TruthfulQA - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "TruthfulQA - # eval": "{\"description\": \"min=654, mean=654, max=654, sum=654 (1)\", \"tab\": \"General information\", \"score\": \"654.0\"}", - "TruthfulQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "TruthfulQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "TruthfulQA - # prompt tokens": "{\"description\": \"min=524.602, mean=524.602, max=524.602, sum=524.602 (1)\", \"tab\": \"General information\", \"score\": \"524.6024464831804\"}", - "TruthfulQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "TruthfulQA - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MS MARCO (TREC)", - "source_data": { - "dataset_name": "MS MARCO (TREC)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "NDCG@10 on MS MARCO (TREC)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "MS MARCO (regular) - RR@10 (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "MS MARCO (TREC) - NDCG@10 (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "MS MARCO (regular) - RR@10 (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "MS MARCO (TREC) - NDCG@10 (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "MS MARCO (regular) - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MS MARCO (TREC) - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MS MARCO (regular) - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "MS MARCO (TREC) - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CNN/DailyMail", - "source_data": { - "dataset_name": "CNN/DailyMail", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on CNN/DailyMail", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "CNN/DailyMail - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "CNN/DailyMail - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "CNN/DailyMail - SummaC": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - QAFactEval": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - BERTScore (F1)": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - Coverage": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - Density": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - Compression": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-faithfulness": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-relevance": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-coherence": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "XSUM", - "source_data": { - "dataset_name": "XSUM", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on XSUM", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "XSUM - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "XSUM - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "XSUM - SummaC": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - QAFactEval": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - BERTScore (F1)": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - Coverage": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - Density": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - Compression": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-faithfulness": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-relevance": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-coherence": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "IMDB", - "source_data": { - "dataset_name": "IMDB", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on IMDB", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.927, - "details": { - "description": "min=0.927, mean=0.927, max=0.927, sum=0.927 (1)", - "tab": "Accuracy", - "IMDB - ECE (10-bin)": "{\"description\": \"min=0.255, mean=0.255, max=0.255, sum=0.255 (1)\", \"tab\": \"Calibration\", \"score\": \"\"}", - "IMDB - EM (Robustness)": "{\"description\": \"min=0.893, mean=0.893, max=0.893, sum=0.893 (1)\", \"tab\": \"Robustness\", \"score\": \"0.893\"}", - "IMDB - EM (Fairness)": "{\"description\": \"min=0.913, mean=0.913, max=0.913, sum=0.913 (1)\", \"tab\": \"Fairness\", \"score\": \"0.913\"}", - "IMDB - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "IMDB - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "IMDB - # train": "{\"description\": \"min=2.781, mean=2.781, max=2.781, sum=2.781 (1)\", \"tab\": \"General information\", \"score\": \"2.781\"}", - "IMDB - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IMDB - # prompt tokens": "{\"description\": \"min=1751.213, mean=1751.213, max=1751.213, sum=1751.213 (1)\", \"tab\": \"General information\", \"score\": \"1751.213\"}", - "IMDB - # output tokens": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "IMDB - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "IMDB - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CivilComments", - "source_data": { - "dataset_name": "CivilComments", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on CivilComments", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.549, - "details": { - "description": "min=0.027, mean=0.549, max=0.998, sum=9.887 (18)", - "tab": "Accuracy", - "CivilComments - ECE (10-bin)": "{\"description\": \"min=0.161, mean=0.4, max=0.513, sum=7.208 (18)\", \"tab\": \"Calibration\", \"score\": \"\"}", - "CivilComments - EM (Robustness)": "{\"description\": \"min=0.016, mean=0.503, max=0.97, sum=9.055 (18)\", \"tab\": \"Robustness\", \"score\": \"0.503044804739656\"}", - "CivilComments - EM (Fairness)": "{\"description\": \"min=0.006, mean=0.508, max=0.998, sum=9.137 (18)\", \"tab\": \"Fairness\", \"score\": \"0.5075946750657245\"}", - "CivilComments - Denoised inference time (s)": "{\"description\": \"9 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "CivilComments - # eval": "{\"description\": \"min=74, mean=371.556, max=683, sum=6688 (18)\", \"tab\": \"General information\", \"score\": \"371.55555555555554\"}", - "CivilComments - # train": "{\"description\": \"min=5, mean=5, max=5, sum=90 (18)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CivilComments - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (18)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CivilComments - # prompt tokens": "{\"description\": \"min=404.732, mean=855.241, max=1417.567, sum=15394.339 (18)\", \"tab\": \"General information\", \"score\": \"855.2410378605821\"}", - "CivilComments - # output tokens": "{\"description\": \"min=5, mean=5, max=5, sum=90 (18)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CivilComments - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=18 (18)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "CivilComments - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Toxic fraction": "{\"description\": \"9 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "RAFT", - "source_data": { - "dataset_name": "RAFT", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on RAFT", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.752, - "details": { - "description": "min=0.15, mean=0.752, max=1, sum=8.275 (11)", - "tab": "Accuracy", - "RAFT - ECE (10-bin)": "{\"description\": \"min=0.156, mean=0.753, max=1.0, sum=8.279 (11)\", \"tab\": \"Calibration\", \"score\": \"\"}", - "RAFT - EM (Robustness)": "{\"description\": \"min=0.05, mean=0.67, max=0.95, sum=7.375 (11)\", \"tab\": \"Robustness\", \"score\": \"0.6704545454545454\"}", - "RAFT - EM (Fairness)": "{\"description\": \"min=0.1, mean=0.718, max=0.975, sum=7.9 (11)\", \"tab\": \"Fairness\", \"score\": \"0.7181818181818181\"}", - "RAFT - Denoised inference time (s)": "{\"description\": \"11 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "RAFT - # eval": "{\"description\": \"min=40, mean=40, max=40, sum=440 (11)\", \"tab\": \"General information\", \"score\": \"40.0\"}", - "RAFT - # train": "{\"description\": \"min=0.45, mean=4.552, max=5, sum=50.075 (11)\", \"tab\": \"General information\", \"score\": \"4.552272727272727\"}", - "RAFT - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (11)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "RAFT - # prompt tokens": "{\"description\": \"min=303.675, mean=954.111, max=1882.1, sum=10495.225 (11)\", \"tab\": \"General information\", \"score\": \"954.1113636363635\"}", - "RAFT - # output tokens": "{\"description\": \"min=30, mean=30, max=30, sum=330 (11)\", \"tab\": \"General information\", \"score\": \"30.0\"}", - "RAFT - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=11 (11)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "RAFT - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Toxic fraction": "{\"description\": \"11 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/meta_llama-4-maverick-17b-128e-instruct-fp8-fc.json b/data/models/meta_llama-4-maverick-17b-128e-instruct-fp8-fc.json deleted file mode 100644 index b4ca8db53afd1bd5ae3c01e90695b60de79a2cf3..0000000000000000000000000000000000000000 --- a/data/models/meta_llama-4-maverick-17b-128e-instruct-fp8-fc.json +++ /dev/null @@ -1,851 +0,0 @@ -{ - "model_info": { - "name": "Llama-4-Maverick-17B-128E-Instruct-FP8 (FC)", - "id": "meta/llama-4-maverick-17b-128e-instruct-fp8-fc", - "developer": "Meta", - "additional_details": { - "raw_model_name": "Llama-4-Maverick-17B-128E-Instruct-FP8 (FC)", - "organization": "Meta", - "license": "Meta Llama 4 Community", - "mode": "FC", - "model_link": "https://huggingface.co/meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/meta/llama-4-maverick-17b-128e-instruct-fp8-fc/1775236112.391409", - "retrieved_timestamp": "1775236112.391409", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 50.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 37.29 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 18.25 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 18.43 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 34.11 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 102.75 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 88.65 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 77.08 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 95.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 94.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 88.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 73.65 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 84.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 71.04 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 75.0 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 70.83 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 20.25 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 27.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 22.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 14.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 18.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 28.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 39.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 17.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 18.92 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 8.39 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 32.9 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 15.48 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 100.0 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 55.97 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/meta_llama-4-maverick-17b-128e-instruct-fp8.json b/data/models/meta_llama-4-maverick-17b-128e-instruct-fp8.json deleted file mode 100644 index 31024354dd7ff14bca49f9d3e19eb5e66e7fde34..0000000000000000000000000000000000000000 --- a/data/models/meta_llama-4-maverick-17b-128e-instruct-fp8.json +++ /dev/null @@ -1,234 +0,0 @@ -{ - "model_info": { - "name": "Llama 4 Maverick 17Bx128E Instruct FP8", - "id": "meta/llama-4-maverick-17b-128e-instruct-fp8", - "developer": "Meta", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_capabilities/meta_llama-4-maverick-17b-128e-instruct-fp8/1774096304.5056698", - "retrieved_timestamp": "1774096304.5056698", - "source_metadata": { - "source_name": "helm_capabilities", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_capabilities", - "evaluation_results": [ - { - "evaluation_name": "Mean score", - "source_data": { - "dataset_name": "helm_capabilities", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "The mean of the scores from all columns.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.718, - "details": { - "description": "", - "tab": "Accuracy", - "Mean score - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"8.498428393165543\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU-Pro", - "source_data": { - "dataset_name": "MMLU-Pro", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on MMLU-Pro", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.81, - "details": { - "description": "min=0.81, mean=0.81, max=0.81, sum=0.81 (1)", - "tab": "Accuracy", - "MMLU-Pro - Observed inference time (s)": "{\"description\": \"min=6.74, mean=6.74, max=6.74, sum=6.74 (1)\", \"tab\": \"Efficiency\", \"score\": \"6.739848182201386\"}", - "MMLU-Pro - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "MMLU-Pro - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - # prompt tokens": "{\"description\": \"min=225.585, mean=225.585, max=225.585, sum=225.585 (1)\", \"tab\": \"General information\", \"score\": \"225.585\"}", - "MMLU-Pro - # output tokens": "{\"description\": \"min=548.208, mean=548.208, max=548.208, sum=548.208 (1)\", \"tab\": \"General information\", \"score\": \"548.208\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"all\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.65, - "details": { - "description": "min=0.65, mean=0.65, max=0.65, sum=0.65 (1)", - "tab": "Accuracy", - "GPQA - Observed inference time (s)": "{\"description\": \"min=9.838, mean=9.838, max=9.838, sum=9.838 (1)\", \"tab\": \"Efficiency\", \"score\": \"9.838454476921013\"}", - "GPQA - # eval": "{\"description\": \"min=446, mean=446, max=446, sum=446 (1)\", \"tab\": \"General information\", \"score\": \"446.0\"}", - "GPQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - # prompt tokens": "{\"description\": \"min=236.807, mean=236.807, max=236.807, sum=236.807 (1)\", \"tab\": \"General information\", \"score\": \"236.8071748878924\"}", - "GPQA - # output tokens": "{\"description\": \"min=822.336, mean=822.336, max=822.336, sum=822.336 (1)\", \"tab\": \"General information\", \"score\": \"822.3363228699552\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"gpqa_main\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "IFEval Strict Acc on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.908, - "details": { - "description": "min=0.908, mean=0.908, max=0.908, sum=0.908 (1)", - "tab": "Accuracy", - "IFEval - Observed inference time (s)": "{\"description\": \"min=3.773, mean=3.773, max=3.773, sum=3.773 (1)\", \"tab\": \"Efficiency\", \"score\": \"3.773326979987943\"}", - "IFEval - # eval": "{\"description\": \"min=541, mean=541, max=541, sum=541 (1)\", \"tab\": \"General information\", \"score\": \"541.0\"}", - "IFEval - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - # prompt tokens": "{\"description\": \"min=45.577, mean=45.577, max=45.577, sum=45.577 (1)\", \"tab\": \"General information\", \"score\": \"45.57670979667283\"}", - "IFEval - # output tokens": "{\"description\": \"min=311.251, mean=311.251, max=311.251, sum=311.251 (1)\", \"tab\": \"General information\", \"score\": \"311.2513863216266\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WildBench", - "source_data": { - "dataset_name": "WildBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "WB Score on WildBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8, - "details": { - "description": "min=0.8, mean=0.8, max=0.8, sum=0.8 (1)", - "tab": "Accuracy", - "WildBench - Observed inference time (s)": "{\"description\": \"min=10.37, mean=10.37, max=10.37, sum=10.37 (1)\", \"tab\": \"Efficiency\", \"score\": \"10.36993253993988\"}", - "WildBench - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "WildBench - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # prompt tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # output tokens": "{\"description\": \"min=842.777, mean=842.777, max=842.777, sum=842.777 (1)\", \"tab\": \"General information\", \"score\": \"842.777\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"v2\"" - } - } - }, - { - "evaluation_name": "Omni-MATH", - "source_data": { - "dataset_name": "Omni-MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Acc on Omni-MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.422, - "details": { - "description": "min=0.422, mean=0.422, max=0.422, sum=0.422 (1)", - "tab": "Accuracy", - "Omni-MATH - Observed inference time (s)": "{\"description\": \"min=11.771, mean=11.771, max=11.771, sum=11.771 (1)\", \"tab\": \"Efficiency\", \"score\": \"11.770579786777496\"}", - "Omni-MATH - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "Omni-MATH - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - # prompt tokens": "{\"description\": \"min=105.286, mean=105.286, max=105.286, sum=105.286 (1)\", \"tab\": \"General information\", \"score\": \"105.286\"}", - "Omni-MATH - # output tokens": "{\"description\": \"min=1055.205, mean=1055.205, max=1055.205, sum=1055.205 (1)\", \"tab\": \"General information\", \"score\": \"1055.205\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/meta_llama-4-maverick.json b/data/models/meta_llama-4-maverick.json deleted file mode 100644 index b3cbc63508776ceaf28ebb52d417c9d356ecf362..0000000000000000000000000000000000000000 --- a/data/models/meta_llama-4-maverick.json +++ /dev/null @@ -1,89 +0,0 @@ -{ - "model_info": { - "name": "meta/llama-4-maverick", - "developer": "Meta", - "inference_platform": "openrouter", - "id": "meta/llama-4-maverick" - }, - "evaluations": [ - { - "evaluation_id": "livecodebenchpro/meta/llama-4-maverick/1760492095.8105888", - "retrieved_timestamp": "1760492095.8105888", - "source_metadata": { - "source_organization_name": "New York University, Princeton University, University of California San Diego, University of Washington and Canyon Crest Academy", - "evaluator_relationship": "third_party", - "source_name": "Live Code Bench Pro", - "source_type": "documentation" - }, - "eval_library": { - "name": "unknown", - "version": "unknown" - }, - "benchmark": "livecodebenchpro", - "evaluation_results": [ - { - "evaluation_name": "Hard Problems", - "metric_config": { - "evaluation_description": "Pass@1 on Hard Problems", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.0 - }, - "source_data": { - "dataset_name": "Hard Problems", - "source_type": "url", - "url": [ - "https://webhook.cp-bench.orzzh.com/leaderboard/llm/difficulty?difficulty=hard&benchmark_mode=live" - ] - } - }, - { - "evaluation_name": "Medium Problems", - "metric_config": { - "evaluation_description": "Pass@1 on Medium Problems", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.0 - }, - "source_data": { - "dataset_name": "Medium Problems", - "source_type": "url", - "url": [ - "https://webhook.cp-bench.orzzh.com/leaderboard/llm/difficulty?difficulty=medium&benchmark_mode=live" - ] - } - }, - { - "evaluation_name": "Easy Problems", - "metric_config": { - "evaluation_description": "Pass@1 on Easy Problems", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.09859154929577464 - }, - "source_data": { - "dataset_name": "Easy Problems", - "source_type": "url", - "url": [ - "https://webhook.cp-bench.orzzh.com/leaderboard/llm/difficulty?difficulty=easy&benchmark_mode=live" - ] - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/meta_llama-4-scout-17b-16e-instruct-fc.json b/data/models/meta_llama-4-scout-17b-16e-instruct-fc.json deleted file mode 100644 index 01edc84ab154d13a07f74dd844f9a243236730be..0000000000000000000000000000000000000000 --- a/data/models/meta_llama-4-scout-17b-16e-instruct-fc.json +++ /dev/null @@ -1,851 +0,0 @@ -{ - "model_info": { - "name": "Llama-4-Scout-17B-16E-Instruct (FC)", - "id": "meta/llama-4-scout-17b-16e-instruct-fc", - "developer": "Meta", - "additional_details": { - "raw_model_name": "Llama-4-Scout-17B-16E-Instruct (FC)", - "organization": "Meta", - "license": "Meta Llama 4 Community", - "mode": "FC", - "model_link": "https://huggingface.co/meta-llama/Llama-4-Scout-17B-16E-Instruct" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/meta/llama-4-scout-17b-16e-instruct-fc/1775236112.403594", - "retrieved_timestamp": "1775236112.403594", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 72.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 28.13 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 24.68 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 17.86 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 50.68 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 166.2 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 89.38 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 79.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 94.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 94.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 90.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 74.69 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 81.78 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 72.74 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 81.25 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 79.17 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 9.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 12.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 7.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 7.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 9.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 14.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 18.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 11.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 8.17 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 2.58 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 2.58 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 19.35 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 100.0 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 44.92 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/meta_llama-4-scout-17b-16e-instruct.json b/data/models/meta_llama-4-scout-17b-16e-instruct.json deleted file mode 100644 index ce056a6f3226b9255338ccde10fc087bf2d12b52..0000000000000000000000000000000000000000 --- a/data/models/meta_llama-4-scout-17b-16e-instruct.json +++ /dev/null @@ -1,234 +0,0 @@ -{ - "model_info": { - "name": "Llama 4 Scout 17Bx16E Instruct", - "id": "meta/llama-4-scout-17b-16e-instruct", - "developer": "Meta", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_capabilities/meta_llama-4-scout-17b-16e-instruct/1774096304.5056698", - "retrieved_timestamp": "1774096304.5056698", - "source_metadata": { - "source_name": "helm_capabilities", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_capabilities", - "evaluation_results": [ - { - "evaluation_name": "Mean score", - "source_data": { - "dataset_name": "helm_capabilities", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "The mean of the scores from all columns.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.644, - "details": { - "description": "", - "tab": "Accuracy", - "Mean score - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"8.886502883481523\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU-Pro", - "source_data": { - "dataset_name": "MMLU-Pro", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on MMLU-Pro", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.742, - "details": { - "description": "min=0.742, mean=0.742, max=0.742, sum=0.742 (1)", - "tab": "Accuracy", - "MMLU-Pro - Observed inference time (s)": "{\"description\": \"min=6.525, mean=6.525, max=6.525, sum=6.525 (1)\", \"tab\": \"Efficiency\", \"score\": \"6.524971485614777\"}", - "MMLU-Pro - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "MMLU-Pro - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - # prompt tokens": "{\"description\": \"min=225.585, mean=225.585, max=225.585, sum=225.585 (1)\", \"tab\": \"General information\", \"score\": \"225.585\"}", - "MMLU-Pro - # output tokens": "{\"description\": \"min=550.212, mean=550.212, max=550.212, sum=550.212 (1)\", \"tab\": \"General information\", \"score\": \"550.212\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"all\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.507, - "details": { - "description": "min=0.507, mean=0.507, max=0.507, sum=0.507 (1)", - "tab": "Accuracy", - "GPQA - Observed inference time (s)": "{\"description\": \"min=11.027, mean=11.027, max=11.027, sum=11.027 (1)\", \"tab\": \"Efficiency\", \"score\": \"11.026973943004693\"}", - "GPQA - # eval": "{\"description\": \"min=446, mean=446, max=446, sum=446 (1)\", \"tab\": \"General information\", \"score\": \"446.0\"}", - "GPQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - # prompt tokens": "{\"description\": \"min=236.807, mean=236.807, max=236.807, sum=236.807 (1)\", \"tab\": \"General information\", \"score\": \"236.8071748878924\"}", - "GPQA - # output tokens": "{\"description\": \"min=856.76, mean=856.76, max=856.76, sum=856.76 (1)\", \"tab\": \"General information\", \"score\": \"856.7600896860987\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"gpqa_main\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "IFEval Strict Acc on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.818, - "details": { - "description": "min=0.818, mean=0.818, max=0.818, sum=0.818 (1)", - "tab": "Accuracy", - "IFEval - Observed inference time (s)": "{\"description\": \"min=4.297, mean=4.297, max=4.297, sum=4.297 (1)\", \"tab\": \"Efficiency\", \"score\": \"4.296513711679004\"}", - "IFEval - # eval": "{\"description\": \"min=541, mean=541, max=541, sum=541 (1)\", \"tab\": \"General information\", \"score\": \"541.0\"}", - "IFEval - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - # prompt tokens": "{\"description\": \"min=45.577, mean=45.577, max=45.577, sum=45.577 (1)\", \"tab\": \"General information\", \"score\": \"45.57670979667283\"}", - "IFEval - # output tokens": "{\"description\": \"min=399.399, mean=399.399, max=399.399, sum=399.399 (1)\", \"tab\": \"General information\", \"score\": \"399.3992606284658\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WildBench", - "source_data": { - "dataset_name": "WildBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "WB Score on WildBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.779, - "details": { - "description": "min=0.779, mean=0.779, max=0.779, sum=0.779 (1)", - "tab": "Accuracy", - "WildBench - Observed inference time (s)": "{\"description\": \"min=9.942, mean=9.942, max=9.942, sum=9.942 (1)\", \"tab\": \"Efficiency\", \"score\": \"9.942440722942353\"}", - "WildBench - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "WildBench - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # prompt tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # output tokens": "{\"description\": \"min=952.636, mean=952.636, max=952.636, sum=952.636 (1)\", \"tab\": \"General information\", \"score\": \"952.636\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"v2\"" - } - } - }, - { - "evaluation_name": "Omni-MATH", - "source_data": { - "dataset_name": "Omni-MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Acc on Omni-MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.373, - "details": { - "description": "min=0.373, mean=0.373, max=0.373, sum=0.373 (1)", - "tab": "Accuracy", - "Omni-MATH - Observed inference time (s)": "{\"description\": \"min=12.642, mean=12.642, max=12.642, sum=12.642 (1)\", \"tab\": \"Efficiency\", \"score\": \"12.641614554166793\"}", - "Omni-MATH - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "Omni-MATH - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - # prompt tokens": "{\"description\": \"min=105.286, mean=105.286, max=105.286, sum=105.286 (1)\", \"tab\": \"General information\", \"score\": \"105.286\"}", - "Omni-MATH - # output tokens": "{\"description\": \"min=1088.449, mean=1088.449, max=1088.449, sum=1088.449 (1)\", \"tab\": \"General information\", \"score\": \"1088.449\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/meta_llama-65b.json b/data/models/meta_llama-65b.json deleted file mode 100644 index 889801d7d76d26564bd490e4d97b2539c5bb4850..0000000000000000000000000000000000000000 --- a/data/models/meta_llama-65b.json +++ /dev/null @@ -1,1049 +0,0 @@ -{ - "model_info": { - "name": "LLaMA 65B", - "id": "meta/LLaMA-65B", - "developer": "Meta", - "inference_platform": "unknown", - "additional_details": { - "model_id_aliases": [ - "meta/llama-65b" - ] - } - }, - "evaluations": [ - { - "evaluation_id": "helm_classic/meta_LLaMA-65B/1774096308.339228", - "retrieved_timestamp": "1774096308.339228", - "source_metadata": { - "source_name": "helm_classic", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_classic", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_classic", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperform on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.908, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Calibration": "{\"description\": \"\", \"tab\": \"Calibration\", \"score\": \"\"}", - "Mean win rate - Robustness": "{\"description\": \"\", \"tab\": \"Robustness\", \"score\": \"0.8851981351981352\"}", - "Mean win rate - Fairness": "{\"description\": \"\", \"tab\": \"Fairness\", \"score\": \"0.9235431235431235\"}", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}", - "Mean win rate - Bias": "{\"description\": \"\", \"tab\": \"Bias\", \"score\": \"0.4059399223461723\"}", - "Mean win rate - Toxicity": "{\"description\": \"\", \"tab\": \"Toxicity\", \"score\": \"0.5910839160839161\"}", - "Mean win rate - Summarization metrics": "{\"description\": \"\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.584, - "details": { - "description": "min=0.34, mean=0.584, max=0.89, sum=2.919 (5)", - "tab": "Accuracy", - "MMLU - ECE (10-bin)": "{\"description\": \"5 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "MMLU - EM (Robustness)": "{\"description\": \"min=0.27, mean=0.504, max=0.81, sum=2.518 (5)\", \"tab\": \"Robustness\", \"score\": \"0.5036842105263158\"}", - "MMLU - EM (Fairness)": "{\"description\": \"min=0.34, mean=0.551, max=0.84, sum=2.757 (5)\", \"tab\": \"Fairness\", \"score\": \"0.5514385964912281\"}", - "MMLU - Denoised inference time (s)": "{\"description\": \"5 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=397.65, mean=522.547, max=684.675, sum=2612.735 (5)\", \"tab\": \"General information\", \"score\": \"522.5470877192982\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "MMLU - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "BoolQ", - "source_data": { - "dataset_name": "BoolQ", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on BoolQ", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.871, - "details": { - "description": "min=0.871, mean=0.871, max=0.871, sum=0.871 (1)", - "tab": "Accuracy", - "BoolQ - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "BoolQ - EM (Robustness)": "{\"description\": \"min=0.84, mean=0.84, max=0.84, sum=0.84 (1)\", \"tab\": \"Robustness\", \"score\": \"0.84\"}", - "BoolQ - EM (Fairness)": "{\"description\": \"min=0.847, mean=0.847, max=0.847, sum=0.847 (1)\", \"tab\": \"Fairness\", \"score\": \"0.847\"}", - "BoolQ - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "BoolQ - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "BoolQ - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "BoolQ - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "BoolQ - # prompt tokens": "{\"description\": \"min=1439.447, mean=1439.447, max=1439.447, sum=1439.447 (1)\", \"tab\": \"General information\", \"score\": \"1439.447\"}", - "BoolQ - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "BoolQ - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "BoolQ - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.755, - "details": { - "description": "min=0.755, mean=0.755, max=0.755, sum=0.755 (1)", - "tab": "Accuracy", - "NarrativeQA - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "NarrativeQA - F1 (Robustness)": "{\"description\": \"min=0.567, mean=0.567, max=0.567, sum=0.567 (1)\", \"tab\": \"Robustness\", \"score\": \"0.5674436891870642\"}", - "NarrativeQA - F1 (Fairness)": "{\"description\": \"min=0.661, mean=0.661, max=0.661, sum=0.661 (1)\", \"tab\": \"Fairness\", \"score\": \"0.6614214785759094\"}", - "NarrativeQA - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=1.437, mean=1.437, max=1.437, sum=1.437 (1)\", \"tab\": \"General information\", \"score\": \"1.4366197183098592\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=1541.115, mean=1541.115, max=1541.115, sum=1541.115 (1)\", \"tab\": \"General information\", \"score\": \"1541.1154929577465\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NarrativeQA - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NarrativeQA - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NarrativeQA - Stereotypes (gender)": "{\"description\": \"min=0.5, mean=0.5, max=0.5, sum=0.5 (1)\", \"tab\": \"Bias\", \"score\": \"0.5\"}", - "NarrativeQA - Representation (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=0.667 (1)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666667\"}", - "NarrativeQA - Representation (gender)": "{\"description\": \"min=0.198, mean=0.198, max=0.198, sum=0.198 (1)\", \"tab\": \"Bias\", \"score\": \"0.1981132075471698\"}", - "NarrativeQA - Toxic fraction": "{\"description\": \"min=0.008, mean=0.008, max=0.008, sum=0.008 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.008450704225352112\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (open-book)", - "source_data": { - "dataset_name": "NaturalQuestions (open-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (open-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.672, - "details": { - "description": "min=0.672, mean=0.672, max=0.672, sum=0.672 (1)", - "tab": "Accuracy", - "NaturalQuestions (closed-book) - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "NaturalQuestions (open-book) - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - F1 (Robustness)": "{\"description\": \"min=0.388, mean=0.388, max=0.388, sum=0.388 (1)\", \"tab\": \"Robustness\", \"score\": \"0.3875883665002626\"}", - "NaturalQuestions (open-book) - F1 (Robustness)": "{\"description\": \"min=0.624, mean=0.624, max=0.624, sum=0.624 (1)\", \"tab\": \"Robustness\", \"score\": \"0.623794662165915\"}", - "NaturalQuestions (closed-book) - F1 (Fairness)": "{\"description\": \"min=0.375, mean=0.375, max=0.375, sum=0.375 (1)\", \"tab\": \"Fairness\", \"score\": \"0.3753249636782112\"}", - "NaturalQuestions (open-book) - F1 (Fairness)": "{\"description\": \"min=0.633, mean=0.633, max=0.633, sum=0.633 (1)\", \"tab\": \"Fairness\", \"score\": \"0.6326996444457361\"}", - "NaturalQuestions (closed-book) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NaturalQuestions (open-book) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=137.383, mean=137.383, max=137.383, sum=137.383 (1)\", \"tab\": \"General information\", \"score\": \"137.383\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NaturalQuestions (closed-book) - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=3.722, mean=3.722, max=3.722, sum=3.722 (1)\", \"tab\": \"General information\", \"score\": \"3.722\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.049, mean=0.049, max=0.049, sum=0.049 (1)\", \"tab\": \"General information\", \"score\": \"0.049\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1407.178, mean=1407.178, max=1407.178, sum=1407.178 (1)\", \"tab\": \"General information\", \"score\": \"1407.178\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=0.987, mean=0.987, max=0.987, sum=0.987 (1)\", \"tab\": \"General information\", \"score\": \"0.987\"}", - "NaturalQuestions (open-book) - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NaturalQuestions (closed-book) - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - Representation (race)": "{\"description\": \"min=0.352, mean=0.352, max=0.352, sum=0.352 (1)\", \"tab\": \"Bias\", \"score\": \"0.35238095238095235\"}", - "NaturalQuestions (closed-book) - Representation (gender)": "{\"description\": \"min=0.3, mean=0.3, max=0.3, sum=0.3 (1)\", \"tab\": \"Bias\", \"score\": \"0.30000000000000004\"}", - "NaturalQuestions (open-book) - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NaturalQuestions (open-book) - Stereotypes (gender)": "{\"description\": \"min=0.5, mean=0.5, max=0.5, sum=0.5 (1)\", \"tab\": \"Bias\", \"score\": \"0.5\"}", - "NaturalQuestions (open-book) - Representation (race)": "{\"description\": \"min=0.436, mean=0.436, max=0.436, sum=0.436 (1)\", \"tab\": \"Bias\", \"score\": \"0.4358974358974359\"}", - "NaturalQuestions (open-book) - Representation (gender)": "{\"description\": \"min=0.393, mean=0.393, max=0.393, sum=0.393 (1)\", \"tab\": \"Bias\", \"score\": \"0.3928571428571429\"}", - "NaturalQuestions (closed-book) - Toxic fraction": "{\"description\": \"min=0.001, mean=0.001, max=0.001, sum=0.001 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.001\"}", - "NaturalQuestions (open-book) - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "QuAC", - "source_data": { - "dataset_name": "QuAC", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on QuAC", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.401, - "details": { - "description": "min=0.401, mean=0.401, max=0.401, sum=0.401 (1)", - "tab": "Accuracy", - "QuAC - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "QuAC - F1 (Robustness)": "{\"description\": \"min=0.275, mean=0.275, max=0.275, sum=0.275 (1)\", \"tab\": \"Robustness\", \"score\": \"0.2748605351114493\"}", - "QuAC - F1 (Fairness)": "{\"description\": \"min=0.333, mean=0.333, max=0.333, sum=0.333 (1)\", \"tab\": \"Fairness\", \"score\": \"0.33296543407590734\"}", - "QuAC - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "QuAC - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "QuAC - # train": "{\"description\": \"min=0.507, mean=0.507, max=0.507, sum=0.507 (1)\", \"tab\": \"General information\", \"score\": \"0.507\"}", - "QuAC - truncated": "{\"description\": \"min=0.06, mean=0.06, max=0.06, sum=0.06 (1)\", \"tab\": \"General information\", \"score\": \"0.06\"}", - "QuAC - # prompt tokens": "{\"description\": \"min=1498.657, mean=1498.657, max=1498.657, sum=1498.657 (1)\", \"tab\": \"General information\", \"score\": \"1498.657\"}", - "QuAC - # output tokens": "{\"description\": \"min=0.997, mean=0.997, max=0.997, sum=0.997 (1)\", \"tab\": \"General information\", \"score\": \"0.997\"}", - "QuAC - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "QuAC - Stereotypes (race)": "{\"description\": \"min=0.621, mean=0.621, max=0.621, sum=0.621 (1)\", \"tab\": \"Bias\", \"score\": \"0.6210526315789473\"}", - "QuAC - Stereotypes (gender)": "{\"description\": \"min=0.394, mean=0.394, max=0.394, sum=0.394 (1)\", \"tab\": \"Bias\", \"score\": \"0.3944670750705233\"}", - "QuAC - Representation (race)": "{\"description\": \"min=0.38, mean=0.38, max=0.38, sum=0.38 (1)\", \"tab\": \"Bias\", \"score\": \"0.3804713804713804\"}", - "QuAC - Representation (gender)": "{\"description\": \"min=0.243, mean=0.243, max=0.243, sum=0.243 (1)\", \"tab\": \"Bias\", \"score\": \"0.24335260115606938\"}", - "QuAC - Toxic fraction": "{\"description\": \"min=0.003, mean=0.003, max=0.003, sum=0.003 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.003\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "HellaSwag", - "source_data": { - "dataset_name": "HellaSwag", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on HellaSwag", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "HellaSwag - ECE (10-bin)": "{\"description\": \"No matching runs\", \"tab\": \"Calibration\", \"score\": \"\"}", - "HellaSwag - EM (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "HellaSwag - EM (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "HellaSwag - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "HellaSwag - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "OpenbookQA - ECE (10-bin)": "{\"description\": \"No matching runs\", \"tab\": \"Calibration\", \"score\": \"\"}", - "OpenbookQA - EM (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "OpenbookQA - EM (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "OpenbookQA - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "OpenbookQA - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "TruthfulQA", - "source_data": { - "dataset_name": "TruthfulQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on TruthfulQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.508, - "details": { - "description": "min=0.508, mean=0.508, max=0.508, sum=0.508 (1)", - "tab": "Accuracy", - "TruthfulQA - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "TruthfulQA - EM (Robustness)": "{\"description\": \"min=0.448, mean=0.448, max=0.448, sum=0.448 (1)\", \"tab\": \"Robustness\", \"score\": \"0.44801223241590216\"}", - "TruthfulQA - EM (Fairness)": "{\"description\": \"min=0.42, mean=0.42, max=0.42, sum=0.42 (1)\", \"tab\": \"Fairness\", \"score\": \"0.42048929663608564\"}", - "TruthfulQA - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "TruthfulQA - # eval": "{\"description\": \"min=654, mean=654, max=654, sum=654 (1)\", \"tab\": \"General information\", \"score\": \"654.0\"}", - "TruthfulQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "TruthfulQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "TruthfulQA - # prompt tokens": "{\"description\": \"min=524.602, mean=524.602, max=524.602, sum=524.602 (1)\", \"tab\": \"General information\", \"score\": \"524.6024464831804\"}", - "TruthfulQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "TruthfulQA - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MS MARCO (TREC)", - "source_data": { - "dataset_name": "MS MARCO (TREC)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "NDCG@10 on MS MARCO (TREC)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "MS MARCO (regular) - RR@10 (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "MS MARCO (TREC) - NDCG@10 (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "MS MARCO (regular) - RR@10 (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "MS MARCO (TREC) - NDCG@10 (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "MS MARCO (regular) - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MS MARCO (TREC) - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MS MARCO (regular) - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "MS MARCO (TREC) - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CNN/DailyMail", - "source_data": { - "dataset_name": "CNN/DailyMail", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on CNN/DailyMail", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "CNN/DailyMail - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "CNN/DailyMail - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "CNN/DailyMail - SummaC": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - QAFactEval": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - BERTScore (F1)": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - Coverage": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - Density": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - Compression": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-faithfulness": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-relevance": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-coherence": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "XSUM", - "source_data": { - "dataset_name": "XSUM", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on XSUM", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "XSUM - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "XSUM - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "XSUM - SummaC": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - QAFactEval": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - BERTScore (F1)": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - Coverage": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - Density": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - Compression": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-faithfulness": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-relevance": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-coherence": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "IMDB", - "source_data": { - "dataset_name": "IMDB", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on IMDB", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.962, - "details": { - "description": "min=0.962, mean=0.962, max=0.962, sum=0.962 (1)", - "tab": "Accuracy", - "IMDB - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "IMDB - EM (Robustness)": "{\"description\": \"min=0.935, mean=0.935, max=0.935, sum=0.935 (1)\", \"tab\": \"Robustness\", \"score\": \"0.935\"}", - "IMDB - EM (Fairness)": "{\"description\": \"min=0.953, mean=0.953, max=0.953, sum=0.953 (1)\", \"tab\": \"Fairness\", \"score\": \"0.953\"}", - "IMDB - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "IMDB - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "IMDB - # train": "{\"description\": \"min=2.781, mean=2.781, max=2.781, sum=2.781 (1)\", \"tab\": \"General information\", \"score\": \"2.781\"}", - "IMDB - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IMDB - # prompt tokens": "{\"description\": \"min=1751.213, mean=1751.213, max=1751.213, sum=1751.213 (1)\", \"tab\": \"General information\", \"score\": \"1751.213\"}", - "IMDB - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "IMDB - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "IMDB - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CivilComments", - "source_data": { - "dataset_name": "CivilComments", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on CivilComments", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.655, - "details": { - "description": "min=0.395, mean=0.655, max=0.863, sum=11.783 (18)", - "tab": "Accuracy", - "CivilComments - ECE (10-bin)": "{\"description\": \"9 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "CivilComments - EM (Robustness)": "{\"description\": \"min=0.247, mean=0.566, max=0.853, sum=10.188 (18)\", \"tab\": \"Robustness\", \"score\": \"0.565986035612513\"}", - "CivilComments - EM (Fairness)": "{\"description\": \"min=0.32, mean=0.574, max=0.8, sum=10.336 (18)\", \"tab\": \"Fairness\", \"score\": \"0.57420608635975\"}", - "CivilComments - Denoised inference time (s)": "{\"description\": \"9 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "CivilComments - # eval": "{\"description\": \"min=74, mean=371.556, max=683, sum=6688 (18)\", \"tab\": \"General information\", \"score\": \"371.55555555555554\"}", - "CivilComments - # train": "{\"description\": \"min=5, mean=5, max=5, sum=90 (18)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CivilComments - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (18)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CivilComments - # prompt tokens": "{\"description\": \"min=404.732, mean=855.241, max=1417.567, sum=15394.339 (18)\", \"tab\": \"General information\", \"score\": \"855.2410378605821\"}", - "CivilComments - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=18 (18)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "CivilComments - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=18 (18)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "CivilComments - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Toxic fraction": "{\"description\": \"9 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "RAFT", - "source_data": { - "dataset_name": "RAFT", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on RAFT", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.702, - "details": { - "description": "min=0.125, mean=0.702, max=0.975, sum=7.725 (11)", - "tab": "Accuracy", - "RAFT - ECE (10-bin)": "{\"description\": \"11 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "RAFT - EM (Robustness)": "{\"description\": \"min=0, mean=0.655, max=0.975, sum=7.2 (11)\", \"tab\": \"Robustness\", \"score\": \"0.6545454545454545\"}", - "RAFT - EM (Fairness)": "{\"description\": \"min=0.075, mean=0.668, max=0.975, sum=7.35 (11)\", \"tab\": \"Fairness\", \"score\": \"0.6681818181818182\"}", - "RAFT - Denoised inference time (s)": "{\"description\": \"11 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "RAFT - # eval": "{\"description\": \"min=40, mean=40, max=40, sum=440 (11)\", \"tab\": \"General information\", \"score\": \"40.0\"}", - "RAFT - # train": "{\"description\": \"min=0.45, mean=4.552, max=5, sum=50.075 (11)\", \"tab\": \"General information\", \"score\": \"4.552272727272727\"}", - "RAFT - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (11)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "RAFT - # prompt tokens": "{\"description\": \"min=303.675, mean=954.111, max=1882.1, sum=10495.225 (11)\", \"tab\": \"General information\", \"score\": \"954.1113636363635\"}", - "RAFT - # output tokens": "{\"description\": \"min=0.8, mean=0.982, max=1, sum=10.8 (11)\", \"tab\": \"General information\", \"score\": \"0.9818181818181819\"}", - "RAFT - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=11 (11)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "RAFT - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Toxic fraction": "{\"description\": \"11 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_lite/meta_llama-65b/1774096306.427425", - "retrieved_timestamp": "1774096306.427425", - "source_metadata": { - "source_name": "helm_lite", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_lite", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_lite", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.345, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.07451935081148564\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.755, - "details": { - "description": "min=0.755, mean=0.755, max=0.755, sum=0.755 (1)", - "tab": "Accuracy", - "NarrativeQA - Observed inference time (s)": "{\"description\": \"min=2.909, mean=2.909, max=2.909, sum=2.909 (1)\", \"tab\": \"Efficiency\", \"score\": \"2.9087761751362975\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=1.434, mean=1.434, max=1.434, sum=1.434 (1)\", \"tab\": \"General information\", \"score\": \"1.4338028169014085\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=1539.586, mean=1539.586, max=1539.586, sum=1539.586 (1)\", \"tab\": \"General information\", \"score\": \"1539.5859154929578\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (closed-book)", - "source_data": { - "dataset_name": "NaturalQuestions (closed-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (closed-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.433, - "details": { - "description": "min=0.433, mean=0.433, max=0.433, sum=0.433 (1)", - "tab": "Accuracy", - "NaturalQuestions (open-book) - Observed inference time (s)": "{\"description\": \"min=1.361, mean=1.361, max=1.361, sum=1.361 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.3611893365383148\"}", - "NaturalQuestions (closed-book) - Observed inference time (s)": "{\"description\": \"min=4.704, mean=4.704, max=4.704, sum=4.704 (1)\", \"tab\": \"Efficiency\", \"score\": \"4.703710767745972\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=3.722, mean=3.722, max=3.722, sum=3.722 (1)\", \"tab\": \"General information\", \"score\": \"3.722\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.049, mean=0.049, max=0.049, sum=0.049 (1)\", \"tab\": \"General information\", \"score\": \"0.049\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1407.129, mean=1407.129, max=1407.129, sum=1407.129 (1)\", \"tab\": \"General information\", \"score\": \"1407.129\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=0.985, mean=0.985, max=0.985, sum=0.985 (1)\", \"tab\": \"General information\", \"score\": \"0.985\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=137.383, mean=137.383, max=137.383, sum=137.383 (1)\", \"tab\": \"General information\", \"score\": \"137.383\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "mode": "\"closedbook\"" - } - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.754, - "details": { - "description": "min=0.754, mean=0.754, max=0.754, sum=0.754 (1)", - "tab": "Accuracy", - "OpenbookQA - Observed inference time (s)": "{\"description\": \"min=4.49, mean=4.49, max=4.49, sum=4.49 (1)\", \"tab\": \"Efficiency\", \"score\": \"4.490233006477356\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=282.574, mean=282.574, max=282.574, sum=282.574 (1)\", \"tab\": \"General information\", \"score\": \"282.574\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "dataset": "\"openbookqa\"", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.584, - "details": { - "description": "min=0.34, mean=0.584, max=0.89, sum=2.919 (5)", - "tab": "Accuracy", - "MMLU - Observed inference time (s)": "{\"description\": \"min=1.962, mean=3.925, max=5.875, sum=19.627 (5)\", \"tab\": \"Efficiency\", \"score\": \"3.925460591943641\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=397.65, mean=522.547, max=684.675, sum=2612.735 (5)\", \"tab\": \"General information\", \"score\": \"522.5470877192982\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"college_chemistry\", \"computer_security\", \"econometrics\", \"us_foreign_policy\"]", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MATH", - "source_data": { - "dataset_name": "MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Equivalent (CoT) on MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.257, - "details": { - "description": "min=0.096, mean=0.257, max=0.474, sum=1.802 (7)", - "tab": "Accuracy", - "MATH - Observed inference time (s)": "{\"description\": \"min=13.711, mean=20.79, max=30.888, sum=145.531 (7)\", \"tab\": \"Efficiency\", \"score\": \"20.790176352238564\"}", - "MATH - # eval": "{\"description\": \"min=30, mean=62.429, max=135, sum=437 (7)\", \"tab\": \"General information\", \"score\": \"62.42857142857143\"}", - "MATH - # train": "{\"description\": \"min=2.962, mean=6.897, max=8, sum=48.277 (7)\", \"tab\": \"General information\", \"score\": \"6.896761133603239\"}", - "MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MATH - # prompt tokens": "{\"description\": \"min=971.652, mean=1214.707, max=1552.038, sum=8502.951 (7)\", \"tab\": \"General information\", \"score\": \"1214.7073423969382\"}", - "MATH - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=7 (7)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"algebra\", \"counting_and_probability\", \"geometry\", \"intermediate_algebra\", \"number_theory\", \"prealgebra\", \"precalculus\"]", - "level": "\"1\"", - "use_official_examples": "\"False\"", - "use_chain_of_thought": "\"True\"" - } - } - }, - { - "evaluation_name": "GSM8K", - "source_data": { - "dataset_name": "GSM8K", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on GSM8K", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.489, - "details": { - "description": "min=0.489, mean=0.489, max=0.489, sum=0.489 (1)", - "tab": "Accuracy", - "GSM8K - Observed inference time (s)": "{\"description\": \"min=12.339, mean=12.339, max=12.339, sum=12.339 (1)\", \"tab\": \"Efficiency\", \"score\": \"12.338884568691254\"}", - "GSM8K - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "GSM8K - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "GSM8K - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GSM8K - # prompt tokens": "{\"description\": \"min=1207.746, mean=1207.746, max=1207.746, sum=1207.746 (1)\", \"tab\": \"General information\", \"score\": \"1207.746\"}", - "GSM8K - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "LegalBench", - "source_data": { - "dataset_name": "LegalBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on LegalBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.48, - "details": { - "description": "min=0.018, mean=0.48, max=0.863, sum=2.401 (5)", - "tab": "Accuracy", - "LegalBench - Observed inference time (s)": "{\"description\": \"min=1.489, mean=3.974, max=6.264, sum=19.868 (5)\", \"tab\": \"Efficiency\", \"score\": \"3.9735240905509466\"}", - "LegalBench - # eval": "{\"description\": \"min=95, mean=409.4, max=1000, sum=2047 (5)\", \"tab\": \"General information\", \"score\": \"409.4\"}", - "LegalBench - # train": "{\"description\": \"min=0.024, mean=3.805, max=5, sum=19.024 (5)\", \"tab\": \"General information\", \"score\": \"3.8048979591836734\"}", - "LegalBench - truncated": "{\"description\": \"min=0, mean=0.006, max=0.031, sum=0.031 (5)\", \"tab\": \"General information\", \"score\": \"0.006122448979591836\"}", - "LegalBench - # prompt tokens": "{\"description\": \"min=222.137, mean=595.161, max=1481.433, sum=2975.806 (5)\", \"tab\": \"General information\", \"score\": \"595.1612280165185\"}", - "LegalBench - # output tokens": "{\"description\": \"min=0.882, mean=0.976, max=1, sum=4.882 (5)\", \"tab\": \"General information\", \"score\": \"0.9763265306122448\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"abercrombie\", \"corporate_lobbying\", \"function_of_decision_section\", \"international_citizenship_questions\", \"proa\"]" - } - } - }, - { - "evaluation_name": "MedQA", - "source_data": { - "dataset_name": "MedQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MedQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.507, - "details": { - "description": "min=0.507, mean=0.507, max=0.507, sum=0.507 (1)", - "tab": "Accuracy", - "MedQA - Observed inference time (s)": "{\"description\": \"min=4.984, mean=4.984, max=4.984, sum=4.984 (1)\", \"tab\": \"Efficiency\", \"score\": \"4.983887912264875\"}", - "MedQA - # eval": "{\"description\": \"min=503, mean=503, max=503, sum=503 (1)\", \"tab\": \"General information\", \"score\": \"503.0\"}", - "MedQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MedQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MedQA - # prompt tokens": "{\"description\": \"min=1234.901, mean=1234.901, max=1234.901, sum=1234.901 (1)\", \"tab\": \"General information\", \"score\": \"1234.9005964214712\"}", - "MedQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WMT 2014", - "source_data": { - "dataset_name": "WMT 2014", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "BLEU-4 on WMT 2014", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.189, - "details": { - "description": "min=0.102, mean=0.189, max=0.239, sum=0.945 (5)", - "tab": "Accuracy", - "WMT 2014 - Observed inference time (s)": "{\"description\": \"min=2.057, mean=3.603, max=8.087, sum=18.014 (5)\", \"tab\": \"Efficiency\", \"score\": \"3.6028029962680237\"}", - "WMT 2014 - # eval": "{\"description\": \"min=503, mean=568.8, max=832, sum=2844 (5)\", \"tab\": \"General information\", \"score\": \"568.8\"}", - "WMT 2014 - # train": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "WMT 2014 - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WMT 2014 - # prompt tokens": "{\"description\": \"min=127.523, mean=142.288, max=164.972, sum=711.438 (5)\", \"tab\": \"General information\", \"score\": \"142.28751290334915\"}", - "WMT 2014 - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "language_pair": "[\"cs-en\", \"de-en\", \"fr-en\", \"hi-en\", \"ru-en\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/meta_llama-7b.json b/data/models/meta_llama-7b.json deleted file mode 100644 index e7de8dc22ae262c39f9ed01d5681444bcd10110b..0000000000000000000000000000000000000000 --- a/data/models/meta_llama-7b.json +++ /dev/null @@ -1,674 +0,0 @@ -{ - "model_info": { - "name": "LLaMA 7B", - "id": "meta/LLaMA-7B", - "developer": "Meta", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_classic/meta_LLaMA-7B/1774096308.339228", - "retrieved_timestamp": "1774096308.339228", - "source_metadata": { - "source_name": "helm_classic", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_classic", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_classic", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperform on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.533, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Calibration": "{\"description\": \"\", \"tab\": \"Calibration\", \"score\": \"\"}", - "Mean win rate - Robustness": "{\"description\": \"\", \"tab\": \"Robustness\", \"score\": \"0.567972027972028\"}", - "Mean win rate - Fairness": "{\"description\": \"\", \"tab\": \"Fairness\", \"score\": \"0.5526107226107226\"}", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}", - "Mean win rate - Bias": "{\"description\": \"\", \"tab\": \"Bias\", \"score\": \"0.5501935339738984\"}", - "Mean win rate - Toxicity": "{\"description\": \"\", \"tab\": \"Toxicity\", \"score\": \"0.7582167832167832\"}", - "Mean win rate - Summarization metrics": "{\"description\": \"\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.321, - "details": { - "description": "min=0.23, mean=0.321, max=0.45, sum=1.603 (5)", - "tab": "Accuracy", - "MMLU - ECE (10-bin)": "{\"description\": \"min=0.063, mean=0.111, max=0.138, sum=0.557 (5)\", \"tab\": \"Calibration\", \"score\": \"\"}", - "MMLU - EM (Robustness)": "{\"description\": \"min=0.18, mean=0.268, max=0.36, sum=1.338 (5)\", \"tab\": \"Robustness\", \"score\": \"0.2676140350877193\"}", - "MMLU - EM (Fairness)": "{\"description\": \"min=0.19, mean=0.284, max=0.42, sum=1.421 (5)\", \"tab\": \"Fairness\", \"score\": \"0.28410526315789475\"}", - "MMLU - Denoised inference time (s)": "{\"description\": \"5 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=397.65, mean=522.547, max=684.675, sum=2612.735 (5)\", \"tab\": \"General information\", \"score\": \"522.5470877192982\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "MMLU - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "BoolQ", - "source_data": { - "dataset_name": "BoolQ", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on BoolQ", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.756, - "details": { - "description": "min=0.756, mean=0.756, max=0.756, sum=0.756 (1)", - "tab": "Accuracy", - "BoolQ - ECE (10-bin)": "{\"description\": \"min=0.292, mean=0.292, max=0.292, sum=0.292 (1)\", \"tab\": \"Calibration\", \"score\": \"\"}", - "BoolQ - EM (Robustness)": "{\"description\": \"min=0.688, mean=0.688, max=0.688, sum=0.688 (1)\", \"tab\": \"Robustness\", \"score\": \"0.688\"}", - "BoolQ - EM (Fairness)": "{\"description\": \"min=0.71, mean=0.71, max=0.71, sum=0.71 (1)\", \"tab\": \"Fairness\", \"score\": \"0.71\"}", - "BoolQ - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "BoolQ - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "BoolQ - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "BoolQ - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "BoolQ - # prompt tokens": "{\"description\": \"min=1439.447, mean=1439.447, max=1439.447, sum=1439.447 (1)\", \"tab\": \"General information\", \"score\": \"1439.447\"}", - "BoolQ - # output tokens": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "BoolQ - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "BoolQ - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.669, - "details": { - "description": "min=0.669, mean=0.669, max=0.669, sum=0.669 (1)", - "tab": "Accuracy", - "NarrativeQA - ECE (10-bin)": "{\"description\": \"min=0.265, mean=0.265, max=0.265, sum=0.265 (1)\", \"tab\": \"Calibration\", \"score\": \"\"}", - "NarrativeQA - F1 (Robustness)": "{\"description\": \"min=0.485, mean=0.485, max=0.485, sum=0.485 (1)\", \"tab\": \"Robustness\", \"score\": \"0.48451305318378857\"}", - "NarrativeQA - F1 (Fairness)": "{\"description\": \"min=0.552, mean=0.552, max=0.552, sum=0.552 (1)\", \"tab\": \"Fairness\", \"score\": \"0.5523890751544673\"}", - "NarrativeQA - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=1.437, mean=1.437, max=1.437, sum=1.437 (1)\", \"tab\": \"General information\", \"score\": \"1.4366197183098592\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=1541.115, mean=1541.115, max=1541.115, sum=1541.115 (1)\", \"tab\": \"General information\", \"score\": \"1541.1154929577465\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=100, mean=100, max=100, sum=100 (1)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "NarrativeQA - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NarrativeQA - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NarrativeQA - Stereotypes (gender)": "{\"description\": \"min=0.444, mean=0.444, max=0.444, sum=0.444 (1)\", \"tab\": \"Bias\", \"score\": \"0.4444444444444444\"}", - "NarrativeQA - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NarrativeQA - Representation (gender)": "{\"description\": \"min=0.178, mean=0.178, max=0.178, sum=0.178 (1)\", \"tab\": \"Bias\", \"score\": \"0.17785234899328858\"}", - "NarrativeQA - Toxic fraction": "{\"description\": \"min=0.006, mean=0.006, max=0.006, sum=0.006 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.005633802816901409\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (open-book)", - "source_data": { - "dataset_name": "NaturalQuestions (open-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (open-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.589, - "details": { - "description": "min=0.589, mean=0.589, max=0.589, sum=0.589 (1)", - "tab": "Accuracy", - "NaturalQuestions (closed-book) - ECE (10-bin)": "{\"description\": \"min=0.176, mean=0.176, max=0.176, sum=0.176 (1)\", \"tab\": \"Calibration\", \"score\": \"\"}", - "NaturalQuestions (open-book) - ECE (10-bin)": "{\"description\": \"min=0.402, mean=0.402, max=0.402, sum=0.402 (1)\", \"tab\": \"Calibration\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - F1 (Robustness)": "{\"description\": \"min=0.222, mean=0.222, max=0.222, sum=0.222 (1)\", \"tab\": \"Robustness\", \"score\": \"0.22150747696392029\"}", - "NaturalQuestions (open-book) - F1 (Robustness)": "{\"description\": \"min=0.519, mean=0.519, max=0.519, sum=0.519 (1)\", \"tab\": \"Robustness\", \"score\": \"0.5190244505397503\"}", - "NaturalQuestions (closed-book) - F1 (Fairness)": "{\"description\": \"min=0.241, mean=0.241, max=0.241, sum=0.241 (1)\", \"tab\": \"Fairness\", \"score\": \"0.24052468144533276\"}", - "NaturalQuestions (open-book) - F1 (Fairness)": "{\"description\": \"min=0.537, mean=0.537, max=0.537, sum=0.537 (1)\", \"tab\": \"Fairness\", \"score\": \"0.5368535244140038\"}", - "NaturalQuestions (closed-book) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NaturalQuestions (open-book) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=137.383, mean=137.383, max=137.383, sum=137.383 (1)\", \"tab\": \"General information\", \"score\": \"137.383\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=300, mean=300, max=300, sum=300 (1)\", \"tab\": \"General information\", \"score\": \"300.0\"}", - "NaturalQuestions (closed-book) - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=3.722, mean=3.722, max=3.722, sum=3.722 (1)\", \"tab\": \"General information\", \"score\": \"3.722\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.049, mean=0.049, max=0.049, sum=0.049 (1)\", \"tab\": \"General information\", \"score\": \"0.049\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1407.178, mean=1407.178, max=1407.178, sum=1407.178 (1)\", \"tab\": \"General information\", \"score\": \"1407.178\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=300, mean=300, max=300, sum=300 (1)\", \"tab\": \"General information\", \"score\": \"300.0\"}", - "NaturalQuestions (open-book) - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NaturalQuestions (closed-book) - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=0.667 (1)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666667\"}", - "NaturalQuestions (closed-book) - Stereotypes (gender)": "{\"description\": \"min=0.5, mean=0.5, max=0.5, sum=0.5 (1)\", \"tab\": \"Bias\", \"score\": \"0.5\"}", - "NaturalQuestions (closed-book) - Representation (race)": "{\"description\": \"min=0.374, mean=0.374, max=0.374, sum=0.374 (1)\", \"tab\": \"Bias\", \"score\": \"0.3739837398373984\"}", - "NaturalQuestions (closed-book) - Representation (gender)": "{\"description\": \"min=0.111, mean=0.111, max=0.111, sum=0.111 (1)\", \"tab\": \"Bias\", \"score\": \"0.11111111111111116\"}", - "NaturalQuestions (open-book) - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NaturalQuestions (open-book) - Stereotypes (gender)": "{\"description\": \"min=0.3, mean=0.3, max=0.3, sum=0.3 (1)\", \"tab\": \"Bias\", \"score\": \"0.3\"}", - "NaturalQuestions (open-book) - Representation (race)": "{\"description\": \"min=0.506, mean=0.506, max=0.506, sum=0.506 (1)\", \"tab\": \"Bias\", \"score\": \"0.5061728395061729\"}", - "NaturalQuestions (open-book) - Representation (gender)": "{\"description\": \"min=0.36, mean=0.36, max=0.36, sum=0.36 (1)\", \"tab\": \"Bias\", \"score\": \"0.3604651162790698\"}", - "NaturalQuestions (closed-book) - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}", - "NaturalQuestions (open-book) - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "QuAC", - "source_data": { - "dataset_name": "QuAC", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on QuAC", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.338, - "details": { - "description": "min=0.338, mean=0.338, max=0.338, sum=0.338 (1)", - "tab": "Accuracy", - "QuAC - ECE (10-bin)": "{\"description\": \"min=0.114, mean=0.114, max=0.114, sum=0.114 (1)\", \"tab\": \"Calibration\", \"score\": \"\"}", - "QuAC - F1 (Robustness)": "{\"description\": \"min=0.223, mean=0.223, max=0.223, sum=0.223 (1)\", \"tab\": \"Robustness\", \"score\": \"0.22309180806281237\"}", - "QuAC - F1 (Fairness)": "{\"description\": \"min=0.257, mean=0.257, max=0.257, sum=0.257 (1)\", \"tab\": \"Fairness\", \"score\": \"0.2568299506065861\"}", - "QuAC - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "QuAC - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "QuAC - # train": "{\"description\": \"min=0.507, mean=0.507, max=0.507, sum=0.507 (1)\", \"tab\": \"General information\", \"score\": \"0.507\"}", - "QuAC - truncated": "{\"description\": \"min=0.06, mean=0.06, max=0.06, sum=0.06 (1)\", \"tab\": \"General information\", \"score\": \"0.06\"}", - "QuAC - # prompt tokens": "{\"description\": \"min=1498.657, mean=1498.657, max=1498.657, sum=1498.657 (1)\", \"tab\": \"General information\", \"score\": \"1498.657\"}", - "QuAC - # output tokens": "{\"description\": \"min=99.794, mean=99.794, max=99.794, sum=99.794 (1)\", \"tab\": \"General information\", \"score\": \"99.794\"}", - "QuAC - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "QuAC - Stereotypes (race)": "{\"description\": \"min=0.571, mean=0.571, max=0.571, sum=0.571 (1)\", \"tab\": \"Bias\", \"score\": \"0.5714285714285715\"}", - "QuAC - Stereotypes (gender)": "{\"description\": \"min=0.428, mean=0.428, max=0.428, sum=0.428 (1)\", \"tab\": \"Bias\", \"score\": \"0.42791413680110835\"}", - "QuAC - Representation (race)": "{\"description\": \"min=0.284, mean=0.284, max=0.284, sum=0.284 (1)\", \"tab\": \"Bias\", \"score\": \"0.28395061728395066\"}", - "QuAC - Representation (gender)": "{\"description\": \"min=0.259, mean=0.259, max=0.259, sum=0.259 (1)\", \"tab\": \"Bias\", \"score\": \"0.2594070695553022\"}", - "QuAC - Toxic fraction": "{\"description\": \"min=0.003, mean=0.003, max=0.003, sum=0.003 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.003\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "HellaSwag", - "source_data": { - "dataset_name": "HellaSwag", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on HellaSwag", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "HellaSwag - ECE (10-bin)": "{\"description\": \"No matching runs\", \"tab\": \"Calibration\", \"score\": \"\"}", - "HellaSwag - EM (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "HellaSwag - EM (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "HellaSwag - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "HellaSwag - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "OpenbookQA - ECE (10-bin)": "{\"description\": \"No matching runs\", \"tab\": \"Calibration\", \"score\": \"\"}", - "OpenbookQA - EM (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "OpenbookQA - EM (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "OpenbookQA - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "OpenbookQA - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "TruthfulQA", - "source_data": { - "dataset_name": "TruthfulQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on TruthfulQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.28, - "details": { - "description": "min=0.28, mean=0.28, max=0.28, sum=0.28 (1)", - "tab": "Accuracy", - "TruthfulQA - ECE (10-bin)": "{\"description\": \"min=0.057, mean=0.057, max=0.057, sum=0.057 (1)\", \"tab\": \"Calibration\", \"score\": \"\"}", - "TruthfulQA - EM (Robustness)": "{\"description\": \"min=0.229, mean=0.229, max=0.229, sum=0.229 (1)\", \"tab\": \"Robustness\", \"score\": \"0.22935779816513763\"}", - "TruthfulQA - EM (Fairness)": "{\"description\": \"min=0.219, mean=0.219, max=0.219, sum=0.219 (1)\", \"tab\": \"Fairness\", \"score\": \"0.21865443425076453\"}", - "TruthfulQA - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "TruthfulQA - # eval": "{\"description\": \"min=654, mean=654, max=654, sum=654 (1)\", \"tab\": \"General information\", \"score\": \"654.0\"}", - "TruthfulQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "TruthfulQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "TruthfulQA - # prompt tokens": "{\"description\": \"min=524.602, mean=524.602, max=524.602, sum=524.602 (1)\", \"tab\": \"General information\", \"score\": \"524.6024464831804\"}", - "TruthfulQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "TruthfulQA - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MS MARCO (TREC)", - "source_data": { - "dataset_name": "MS MARCO (TREC)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "NDCG@10 on MS MARCO (TREC)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "MS MARCO (regular) - RR@10 (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "MS MARCO (TREC) - NDCG@10 (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "MS MARCO (regular) - RR@10 (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "MS MARCO (TREC) - NDCG@10 (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "MS MARCO (regular) - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MS MARCO (TREC) - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MS MARCO (regular) - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "MS MARCO (TREC) - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CNN/DailyMail", - "source_data": { - "dataset_name": "CNN/DailyMail", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on CNN/DailyMail", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "CNN/DailyMail - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "CNN/DailyMail - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "CNN/DailyMail - SummaC": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - QAFactEval": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - BERTScore (F1)": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - Coverage": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - Density": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - Compression": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-faithfulness": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-relevance": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-coherence": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "XSUM", - "source_data": { - "dataset_name": "XSUM", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on XSUM", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "XSUM - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "XSUM - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "XSUM - SummaC": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - QAFactEval": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - BERTScore (F1)": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - Coverage": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - Density": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - Compression": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-faithfulness": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-relevance": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-coherence": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "IMDB", - "source_data": { - "dataset_name": "IMDB", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on IMDB", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.947, - "details": { - "description": "min=0.947, mean=0.947, max=0.947, sum=0.947 (1)", - "tab": "Accuracy", - "IMDB - ECE (10-bin)": "{\"description\": \"min=0.336, mean=0.336, max=0.336, sum=0.336 (1)\", \"tab\": \"Calibration\", \"score\": \"\"}", - "IMDB - EM (Robustness)": "{\"description\": \"min=0.897, mean=0.897, max=0.897, sum=0.897 (1)\", \"tab\": \"Robustness\", \"score\": \"0.897\"}", - "IMDB - EM (Fairness)": "{\"description\": \"min=0.936, mean=0.936, max=0.936, sum=0.936 (1)\", \"tab\": \"Fairness\", \"score\": \"0.936\"}", - "IMDB - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "IMDB - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "IMDB - # train": "{\"description\": \"min=2.781, mean=2.781, max=2.781, sum=2.781 (1)\", \"tab\": \"General information\", \"score\": \"2.781\"}", - "IMDB - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IMDB - # prompt tokens": "{\"description\": \"min=1751.213, mean=1751.213, max=1751.213, sum=1751.213 (1)\", \"tab\": \"General information\", \"score\": \"1751.213\"}", - "IMDB - # output tokens": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "IMDB - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "IMDB - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CivilComments", - "source_data": { - "dataset_name": "CivilComments", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on CivilComments", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.563, - "details": { - "description": "min=0.015, mean=0.563, max=0.99, sum=10.13 (18)", - "tab": "Accuracy", - "CivilComments - ECE (10-bin)": "{\"description\": \"min=0.13, mean=0.334, max=0.562, sum=6.012 (18)\", \"tab\": \"Calibration\", \"score\": \"\"}", - "CivilComments - EM (Robustness)": "{\"description\": \"min=0.012, mean=0.492, max=0.958, sum=8.864 (18)\", \"tab\": \"Robustness\", \"score\": \"0.4924249260198337\"}", - "CivilComments - EM (Fairness)": "{\"description\": \"min=0.008, mean=0.505, max=0.98, sum=9.086 (18)\", \"tab\": \"Fairness\", \"score\": \"0.5047868294149912\"}", - "CivilComments - Denoised inference time (s)": "{\"description\": \"9 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "CivilComments - # eval": "{\"description\": \"min=74, mean=371.556, max=683, sum=6688 (18)\", \"tab\": \"General information\", \"score\": \"371.55555555555554\"}", - "CivilComments - # train": "{\"description\": \"min=5, mean=5, max=5, sum=90 (18)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CivilComments - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (18)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CivilComments - # prompt tokens": "{\"description\": \"min=404.732, mean=855.241, max=1417.567, sum=15394.339 (18)\", \"tab\": \"General information\", \"score\": \"855.2410378605821\"}", - "CivilComments - # output tokens": "{\"description\": \"min=5, mean=5, max=5, sum=90 (18)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CivilComments - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=18 (18)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "CivilComments - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Toxic fraction": "{\"description\": \"9 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "RAFT", - "source_data": { - "dataset_name": "RAFT", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on RAFT", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.573, - "details": { - "description": "min=0.125, mean=0.573, max=0.975, sum=6.3 (11)", - "tab": "Accuracy", - "RAFT - ECE (10-bin)": "{\"description\": \"min=0.122, mean=0.572, max=0.975, sum=6.295 (11)\", \"tab\": \"Calibration\", \"score\": \"\"}", - "RAFT - EM (Robustness)": "{\"description\": \"min=0, mean=0.486, max=0.95, sum=5.35 (11)\", \"tab\": \"Robustness\", \"score\": \"0.4863636363636364\"}", - "RAFT - EM (Fairness)": "{\"description\": \"min=0.1, mean=0.545, max=0.975, sum=6 (11)\", \"tab\": \"Fairness\", \"score\": \"0.5454545454545454\"}", - "RAFT - Denoised inference time (s)": "{\"description\": \"11 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "RAFT - # eval": "{\"description\": \"min=40, mean=40, max=40, sum=440 (11)\", \"tab\": \"General information\", \"score\": \"40.0\"}", - "RAFT - # train": "{\"description\": \"min=0.45, mean=4.552, max=5, sum=50.075 (11)\", \"tab\": \"General information\", \"score\": \"4.552272727272727\"}", - "RAFT - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (11)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "RAFT - # prompt tokens": "{\"description\": \"min=303.675, mean=954.111, max=1882.1, sum=10495.225 (11)\", \"tab\": \"General information\", \"score\": \"954.1113636363635\"}", - "RAFT - # output tokens": "{\"description\": \"min=29.575, mean=29.961, max=30, sum=329.575 (11)\", \"tab\": \"General information\", \"score\": \"29.961363636363636\"}", - "RAFT - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=11 (11)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "RAFT - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Toxic fraction": "{\"description\": \"11 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/meta_opt-175b.json b/data/models/meta_opt-175b.json deleted file mode 100644 index bc7f20187cb6a8945fe4b5c1f92c0cc58c3e94a6..0000000000000000000000000000000000000000 --- a/data/models/meta_opt-175b.json +++ /dev/null @@ -1,674 +0,0 @@ -{ - "model_info": { - "name": "OPT 175B", - "id": "meta/OPT-175B", - "developer": "Meta", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_classic/meta_OPT-175B/1774096308.339228", - "retrieved_timestamp": "1774096308.339228", - "source_metadata": { - "source_name": "helm_classic", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_classic", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_classic", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperform on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.609, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Calibration": "{\"description\": \"\", \"tab\": \"Calibration\", \"score\": \"0.33807716905928437\"}", - "Mean win rate - Robustness": "{\"description\": \"\", \"tab\": \"Robustness\", \"score\": \"0.5191448151403657\"}", - "Mean win rate - Fairness": "{\"description\": \"\", \"tab\": \"Fairness\", \"score\": \"0.6221815633384042\"}", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.24121162280701755\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}", - "Mean win rate - Bias": "{\"description\": \"\", \"tab\": \"Bias\", \"score\": \"0.58013310485115\"}", - "Mean win rate - Toxicity": "{\"description\": \"\", \"tab\": \"Toxicity\", \"score\": \"0.43513523513523517\"}", - "Mean win rate - Summarization metrics": "{\"description\": \"\", \"tab\": \"Summarization metrics\", \"score\": \"0.5927318295739348\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.318, - "details": { - "description": "min=0.21, mean=0.318, max=0.48, sum=4.775 (15)", - "tab": "Accuracy", - "MMLU - ECE (10-bin)": "{\"description\": \"min=0.115, mean=0.147, max=0.194, sum=2.207 (15)\", \"tab\": \"Calibration\", \"score\": \"0.14714449343481936\"}", - "MMLU - EM (Robustness)": "{\"description\": \"min=0.13, mean=0.27, max=0.45, sum=4.048 (15)\", \"tab\": \"Robustness\", \"score\": \"0.2698479532163743\"}", - "MMLU - EM (Fairness)": "{\"description\": \"min=0.167, mean=0.287, max=0.43, sum=4.298 (15)\", \"tab\": \"Fairness\", \"score\": \"0.28651461988304094\"}", - "MMLU - Denoised inference time (s)": "{\"description\": \"min=0.11, mean=0.12, max=0.138, sum=1.793 (15)\", \"tab\": \"Efficiency\", \"score\": \"0.1195572826114746\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=1542 (15)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=75 (15)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (15)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=371.38, mean=472.274, max=624.07, sum=7084.111 (15)\", \"tab\": \"General information\", \"score\": \"472.2740350877193\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=15 (15)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "MMLU - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=45 (15)\", \"tab\": \"General information\", \"score\": \"3.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "BoolQ", - "source_data": { - "dataset_name": "BoolQ", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on BoolQ", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.793, - "details": { - "description": "min=0.777, mean=0.793, max=0.813, sum=2.379 (3)", - "tab": "Accuracy", - "BoolQ - ECE (10-bin)": "{\"description\": \"min=0.177, mean=0.194, max=0.218, sum=0.581 (3)\", \"tab\": \"Calibration\", \"score\": \"0.19360710050007168\"}", - "BoolQ - EM (Robustness)": "{\"description\": \"min=0.584, mean=0.623, max=0.662, sum=1.869 (3)\", \"tab\": \"Robustness\", \"score\": \"0.623\"}", - "BoolQ - EM (Fairness)": "{\"description\": \"min=0.712, mean=0.731, max=0.746, sum=2.193 (3)\", \"tab\": \"Fairness\", \"score\": \"0.731\"}", - "BoolQ - Denoised inference time (s)": "{\"description\": \"min=0.71, mean=0.869, max=0.954, sum=2.608 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.869335141547284\"}", - "BoolQ - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "BoolQ - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "BoolQ - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "BoolQ - # prompt tokens": "{\"description\": \"min=660.073, mean=908.406, max=1242.073, sum=2725.219 (3)\", \"tab\": \"General information\", \"score\": \"908.4063333333334\"}", - "BoolQ - # output tokens": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "BoolQ - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "BoolQ - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.671, - "details": { - "description": "min=0.657, mean=0.671, max=0.692, sum=2.013 (3)", - "tab": "Accuracy", - "NarrativeQA - ECE (10-bin)": "{\"description\": \"min=0.25, mean=0.254, max=0.261, sum=0.763 (3)\", \"tab\": \"Calibration\", \"score\": \"0.25442494535286947\"}", - "NarrativeQA - F1 (Robustness)": "{\"description\": \"min=0.365, mean=0.409, max=0.447, sum=1.227 (3)\", \"tab\": \"Robustness\", \"score\": \"0.4090933797146052\"}", - "NarrativeQA - F1 (Fairness)": "{\"description\": \"min=0.545, mean=0.573, max=0.6, sum=1.718 (3)\", \"tab\": \"Fairness\", \"score\": \"0.5725951072978767\"}", - "NarrativeQA - Denoised inference time (s)": "{\"description\": \"min=2.375, mean=2.783, max=3.573, sum=8.348 (3)\", \"tab\": \"Efficiency\", \"score\": \"2.7825779012238017\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=1065 (3)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=1.051, mean=1.647, max=2.085, sum=4.941 (3)\", \"tab\": \"General information\", \"score\": \"1.6469483568075116\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=1601.955, mean=1652.377, max=1705.003, sum=4957.132 (3)\", \"tab\": \"General information\", \"score\": \"1652.3774647887324\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=27.152, mean=40.781, max=56.166, sum=122.344 (3)\", \"tab\": \"General information\", \"score\": \"40.781220657277\"}", - "NarrativeQA - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NarrativeQA - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NarrativeQA - Stereotypes (gender)": "{\"description\": \"min=0.472, mean=0.491, max=0.5, sum=1.472 (3)\", \"tab\": \"Bias\", \"score\": \"0.49074074074074076\"}", - "NarrativeQA - Representation (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=2 (3)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "NarrativeQA - Representation (gender)": "{\"description\": \"min=0.213, mean=0.232, max=0.257, sum=0.695 (3)\", \"tab\": \"Bias\", \"score\": \"0.23182834585691858\"}", - "NarrativeQA - Toxic fraction": "{\"description\": \"min=0.017, mean=0.019, max=0.023, sum=0.056 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.018779342723004692\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (open-book)", - "source_data": { - "dataset_name": "NaturalQuestions (open-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (open-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.615, - "details": { - "description": "min=0.607, mean=0.615, max=0.619, sum=1.845 (3)", - "tab": "Accuracy", - "NaturalQuestions (closed-book) - ECE (10-bin)": "{\"description\": \"min=0.169, mean=0.173, max=0.178, sum=0.52 (3)\", \"tab\": \"Calibration\", \"score\": \"0.17321815784980257\"}", - "NaturalQuestions (open-book) - ECE (10-bin)": "{\"description\": \"min=0.365, mean=0.372, max=0.38, sum=1.117 (3)\", \"tab\": \"Calibration\", \"score\": \"0.3723122842871363\"}", - "NaturalQuestions (closed-book) - F1 (Robustness)": "{\"description\": \"min=0.202, mean=0.208, max=0.213, sum=0.623 (3)\", \"tab\": \"Robustness\", \"score\": \"0.2076699169323979\"}", - "NaturalQuestions (open-book) - F1 (Robustness)": "{\"description\": \"min=0.382, mean=0.408, max=0.445, sum=1.224 (3)\", \"tab\": \"Robustness\", \"score\": \"0.40794279599736244\"}", - "NaturalQuestions (closed-book) - F1 (Fairness)": "{\"description\": \"min=0.244, mean=0.246, max=0.248, sum=0.738 (3)\", \"tab\": \"Fairness\", \"score\": \"0.2461285688311032\"}", - "NaturalQuestions (open-book) - F1 (Fairness)": "{\"description\": \"min=0.557, mean=0.561, max=0.566, sum=1.684 (3)\", \"tab\": \"Fairness\", \"score\": \"0.5613201936765554\"}", - "NaturalQuestions (closed-book) - Denoised inference time (s)": "{\"description\": \"min=4.226, mean=4.548, max=4.977, sum=13.645 (3)\", \"tab\": \"Efficiency\", \"score\": \"4.5482187833781085\"}", - "NaturalQuestions (open-book) - Denoised inference time (s)": "{\"description\": \"min=6.761, mean=7.78, max=8.516, sum=23.341 (3)\", \"tab\": \"Efficiency\", \"score\": \"7.78018927021878\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=110.254, mean=112.254, max=116.254, sum=336.762 (3)\", \"tab\": \"General information\", \"score\": \"112.254\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=272.695, mean=278.02, max=287.118, sum=834.059 (3)\", \"tab\": \"General information\", \"score\": \"278.01966666666664\"}", - "NaturalQuestions (closed-book) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.647, mean=4.691, max=4.724, sum=14.074 (3)\", \"tab\": \"General information\", \"score\": \"4.691333333333334\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.036, mean=0.036, max=0.036, sum=0.108 (3)\", \"tab\": \"General information\", \"score\": \"0.036\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1231.212, mean=1419.574, max=1523.257, sum=4258.721 (3)\", \"tab\": \"General information\", \"score\": \"1419.5736666666664\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=168.53, mean=194.671, max=213.115, sum=584.014 (3)\", \"tab\": \"General information\", \"score\": \"194.67133333333334\"}", - "NaturalQuestions (open-book) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NaturalQuestions (closed-book) - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - Stereotypes (gender)": "{\"description\": \"min=0.279, mean=0.327, max=0.375, sum=0.654 (2)\", \"tab\": \"Bias\", \"score\": \"0.32684426229508196\"}", - "NaturalQuestions (closed-book) - Representation (race)": "{\"description\": \"min=0.48, mean=0.521, max=0.562, sum=1.563 (3)\", \"tab\": \"Bias\", \"score\": \"0.5211641167340236\"}", - "NaturalQuestions (closed-book) - Representation (gender)": "{\"description\": \"min=0.029, mean=0.081, max=0.119, sum=0.243 (3)\", \"tab\": \"Bias\", \"score\": \"0.0811320308714203\"}", - "NaturalQuestions (open-book) - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NaturalQuestions (open-book) - Stereotypes (gender)": "{\"description\": \"min=0.433, mean=0.439, max=0.45, sum=1.317 (3)\", \"tab\": \"Bias\", \"score\": \"0.4388888888888889\"}", - "NaturalQuestions (open-book) - Representation (race)": "{\"description\": \"min=0.423, mean=0.461, max=0.48, sum=1.384 (3)\", \"tab\": \"Bias\", \"score\": \"0.4612918002748511\"}", - "NaturalQuestions (open-book) - Representation (gender)": "{\"description\": \"min=0.318, mean=0.325, max=0.332, sum=0.974 (3)\", \"tab\": \"Bias\", \"score\": \"0.324702218997521\"}", - "NaturalQuestions (closed-book) - Toxic fraction": "{\"description\": \"min=0, mean=0.001, max=0.001, sum=0.002 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.0006666666666666666\"}", - "NaturalQuestions (open-book) - Toxic fraction": "{\"description\": \"min=0.001, mean=0.002, max=0.003, sum=0.005 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.0016666666666666668\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "QuAC", - "source_data": { - "dataset_name": "QuAC", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on QuAC", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.36, - "details": { - "description": "min=0.347, mean=0.36, max=0.369, sum=1.08 (3)", - "tab": "Accuracy", - "QuAC - ECE (10-bin)": "{\"description\": \"min=0.128, mean=0.148, max=0.173, sum=0.443 (3)\", \"tab\": \"Calibration\", \"score\": \"0.14774672207107284\"}", - "QuAC - F1 (Robustness)": "{\"description\": \"min=0.194, mean=0.2, max=0.209, sum=0.6 (3)\", \"tab\": \"Robustness\", \"score\": \"0.2000302607507829\"}", - "QuAC - F1 (Fairness)": "{\"description\": \"min=0.255, mean=0.266, max=0.274, sum=0.798 (3)\", \"tab\": \"Fairness\", \"score\": \"0.26591098840755784\"}", - "QuAC - Denoised inference time (s)": "{\"description\": \"min=3.951, mean=4.049, max=4.154, sum=12.147 (3)\", \"tab\": \"Efficiency\", \"score\": \"4.049007016242971\"}", - "QuAC - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "QuAC - # train": "{\"description\": \"min=0.845, mean=0.944, max=1.086, sum=2.833 (3)\", \"tab\": \"General information\", \"score\": \"0.9443333333333334\"}", - "QuAC - truncated": "{\"description\": \"min=0.016, mean=0.016, max=0.016, sum=0.048 (3)\", \"tab\": \"General information\", \"score\": \"0.016\"}", - "QuAC - # prompt tokens": "{\"description\": \"min=1625.523, mean=1644.831, max=1670.605, sum=4934.492 (3)\", \"tab\": \"General information\", \"score\": \"1644.8306666666667\"}", - "QuAC - # output tokens": "{\"description\": \"min=75.972, mean=77.836, max=79.528, sum=233.507 (3)\", \"tab\": \"General information\", \"score\": \"77.83566666666667\"}", - "QuAC - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "QuAC - Stereotypes (race)": "{\"description\": \"min=0.561, mean=0.591, max=0.614, sum=1.773 (3)\", \"tab\": \"Bias\", \"score\": \"0.5910808767951625\"}", - "QuAC - Stereotypes (gender)": "{\"description\": \"min=0.376, mean=0.386, max=0.399, sum=1.159 (3)\", \"tab\": \"Bias\", \"score\": \"0.38627685600159944\"}", - "QuAC - Representation (race)": "{\"description\": \"min=0.167, mean=0.243, max=0.304, sum=0.73 (3)\", \"tab\": \"Bias\", \"score\": \"0.2433558772540988\"}", - "QuAC - Representation (gender)": "{\"description\": \"min=0.195, mean=0.207, max=0.218, sum=0.621 (3)\", \"tab\": \"Bias\", \"score\": \"0.2069846056271054\"}", - "QuAC - Toxic fraction": "{\"description\": \"min=0, mean=0.001, max=0.003, sum=0.003 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.001\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "HellaSwag", - "source_data": { - "dataset_name": "HellaSwag", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on HellaSwag", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.791, - "details": { - "description": "min=0.791, mean=0.791, max=0.791, sum=0.791 (1)", - "tab": "Accuracy", - "HellaSwag - ECE (10-bin)": "{\"description\": \"min=0.325, mean=0.325, max=0.325, sum=0.325 (1)\", \"tab\": \"Calibration\", \"score\": \"0.324637159664446\"}", - "HellaSwag - EM (Robustness)": "{\"description\": \"min=0.744, mean=0.744, max=0.744, sum=0.744 (1)\", \"tab\": \"Robustness\", \"score\": \"0.744\"}", - "HellaSwag - EM (Fairness)": "{\"description\": \"min=0.66, mean=0.66, max=0.66, sum=0.66 (1)\", \"tab\": \"Fairness\", \"score\": \"0.66\"}", - "HellaSwag - Denoised inference time (s)": "{\"description\": \"min=0.71, mean=0.71, max=0.71, sum=0.71 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.7096132577732451\"}", - "HellaSwag - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "HellaSwag - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "HellaSwag - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "HellaSwag - # prompt tokens": "{\"description\": \"min=87.888, mean=87.888, max=87.888, sum=87.888 (1)\", \"tab\": \"General information\", \"score\": \"87.888\"}", - "HellaSwag - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "HellaSwag - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.586, - "details": { - "description": "min=0.586, mean=0.586, max=0.586, sum=0.586 (1)", - "tab": "Accuracy", - "OpenbookQA - ECE (10-bin)": "{\"description\": \"min=0.209, mean=0.209, max=0.209, sum=0.209 (1)\", \"tab\": \"Calibration\", \"score\": \"0.20889829455743214\"}", - "OpenbookQA - EM (Robustness)": "{\"description\": \"min=0.488, mean=0.488, max=0.488, sum=0.488 (1)\", \"tab\": \"Robustness\", \"score\": \"0.488\"}", - "OpenbookQA - EM (Fairness)": "{\"description\": \"min=0.5, mean=0.5, max=0.5, sum=0.5 (1)\", \"tab\": \"Fairness\", \"score\": \"0.5\"}", - "OpenbookQA - Denoised inference time (s)": "{\"description\": \"min=0.038, mean=0.038, max=0.038, sum=0.038 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.03760148134353242\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=5.27, mean=5.27, max=5.27, sum=5.27 (1)\", \"tab\": \"General information\", \"score\": \"5.27\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "TruthfulQA", - "source_data": { - "dataset_name": "TruthfulQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on TruthfulQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.25, - "details": { - "description": "min=0.228, mean=0.25, max=0.269, sum=1.002 (4)", - "tab": "Accuracy", - "TruthfulQA - ECE (10-bin)": "{\"description\": \"min=0.042, mean=0.054, max=0.061, sum=0.216 (4)\", \"tab\": \"Calibration\", \"score\": \"0.05404322346973557\"}", - "TruthfulQA - EM (Robustness)": "{\"description\": \"min=0.167, mean=0.205, max=0.249, sum=0.818 (4)\", \"tab\": \"Robustness\", \"score\": \"0.20451070336391436\"}", - "TruthfulQA - EM (Fairness)": "{\"description\": \"min=0.165, mean=0.203, max=0.249, sum=0.812 (4)\", \"tab\": \"Fairness\", \"score\": \"0.2029816513761468\"}", - "TruthfulQA - Denoised inference time (s)": "{\"description\": \"min=0.079, mean=0.141, max=0.246, sum=0.563 (4)\", \"tab\": \"Efficiency\", \"score\": \"0.1406602569641055\"}", - "TruthfulQA - # eval": "{\"description\": \"min=654, mean=654, max=654, sum=2616 (4)\", \"tab\": \"General information\", \"score\": \"654.0\"}", - "TruthfulQA - # train": "{\"description\": \"min=0, mean=3.75, max=5, sum=15 (4)\", \"tab\": \"General information\", \"score\": \"3.75\"}", - "TruthfulQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (4)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "TruthfulQA - # prompt tokens": "{\"description\": \"min=85.121, mean=404.621, max=529.121, sum=1618.483 (4)\", \"tab\": \"General information\", \"score\": \"404.62079510703364\"}", - "TruthfulQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=4 (4)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "TruthfulQA - # trials": "{\"description\": \"min=1, mean=2.5, max=3, sum=10 (4)\", \"tab\": \"General information\", \"score\": \"2.5\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MS MARCO (TREC)", - "source_data": { - "dataset_name": "MS MARCO (TREC)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "NDCG@10 on MS MARCO (TREC)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.448, - "details": { - "description": "min=0.425, mean=0.448, max=0.467, sum=1.344 (3)", - "tab": "Accuracy", - "MS MARCO (regular) - RR@10 (Robustness)": "{\"description\": \"min=0.198, mean=0.235, max=0.263, sum=0.705 (3)\", \"tab\": \"Robustness\", \"score\": \"0.23496613756613724\"}", - "MS MARCO (TREC) - NDCG@10 (Robustness)": "{\"description\": \"min=0.386, mean=0.408, max=0.422, sum=1.225 (3)\", \"tab\": \"Robustness\", \"score\": \"0.4083455179340017\"}", - "MS MARCO (regular) - RR@10 (Fairness)": "{\"description\": \"min=0.229, mean=0.26, max=0.288, sum=0.779 (3)\", \"tab\": \"Fairness\", \"score\": \"0.25959669312169276\"}", - "MS MARCO (TREC) - NDCG@10 (Fairness)": "{\"description\": \"min=0.4, mean=0.419, max=0.428, sum=1.256 (3)\", \"tab\": \"Fairness\", \"score\": \"0.41868435186381264\"}", - "MS MARCO (regular) - Denoised inference time (s)": "{\"description\": \"min=0.229, mean=0.241, max=0.262, sum=0.724 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.24148347487755295\"}", - "MS MARCO (TREC) - Denoised inference time (s)": "{\"description\": \"min=0.19, mean=0.226, max=0.254, sum=0.678 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.2261325473631569\"}", - "MS MARCO (regular) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "MS MARCO (regular) - # train": "{\"description\": \"min=2, mean=2, max=2, sum=6 (3)\", \"tab\": \"General information\", \"score\": \"2.0\"}", - "MS MARCO (regular) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MS MARCO (regular) - # prompt tokens": "{\"description\": \"min=495.232, mean=532.565, max=577.232, sum=1597.696 (3)\", \"tab\": \"General information\", \"score\": \"532.5653333333333\"}", - "MS MARCO (regular) - # output tokens": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MS MARCO (regular) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "MS MARCO (TREC) - # eval": "{\"description\": \"min=43, mean=43, max=43, sum=129 (3)\", \"tab\": \"General information\", \"score\": \"43.0\"}", - "MS MARCO (TREC) - # train": "{\"description\": \"min=2, mean=2, max=2, sum=6 (3)\", \"tab\": \"General information\", \"score\": \"2.0\"}", - "MS MARCO (TREC) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MS MARCO (TREC) - # prompt tokens": "{\"description\": \"min=478.488, mean=515.822, max=560.488, sum=1547.465 (3)\", \"tab\": \"General information\", \"score\": \"515.8217054263565\"}", - "MS MARCO (TREC) - # output tokens": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MS MARCO (TREC) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "MS MARCO (regular) - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "MS MARCO (TREC) - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CNN/DailyMail", - "source_data": { - "dataset_name": "CNN/DailyMail", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on CNN/DailyMail", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.146, - "details": { - "description": "min=0.132, mean=0.146, max=0.156, sum=0.875 (6)", - "tab": "Accuracy", - "CNN/DailyMail - Denoised inference time (s)": "{\"description\": \"min=4.705, mean=4.729, max=4.742, sum=28.373 (6)\", \"tab\": \"Efficiency\", \"score\": \"4.728843353285813\"}", - "CNN/DailyMail - # eval": "{\"description\": \"min=466, mean=466, max=466, sum=2796 (6)\", \"tab\": \"General information\", \"score\": \"466.0\"}", - "CNN/DailyMail - # train": "{\"description\": \"min=5, mean=5, max=5, sum=30 (6)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CNN/DailyMail - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (6)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CNN/DailyMail - # prompt tokens": "{\"description\": \"min=1531.586, mean=1549.919, max=1567.586, sum=9299.515 (6)\", \"tab\": \"General information\", \"score\": \"1549.9191702432045\"}", - "CNN/DailyMail - # output tokens": "{\"description\": \"min=72.006, mean=73.533, max=75.564, sum=441.197 (6)\", \"tab\": \"General information\", \"score\": \"73.53290414878398\"}", - "CNN/DailyMail - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=18 (6)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "CNN/DailyMail - Stereotypes (race)": "{\"description\": \"min=0.584, mean=0.591, max=0.602, sum=3.548 (6)\", \"tab\": \"Bias\", \"score\": \"0.5912557147615382\"}", - "CNN/DailyMail - Stereotypes (gender)": "{\"description\": \"min=0.389, mean=0.407, max=0.423, sum=2.439 (6)\", \"tab\": \"Bias\", \"score\": \"0.406575836707982\"}", - "CNN/DailyMail - Representation (race)": "{\"description\": \"min=0.258, mean=0.294, max=0.328, sum=1.765 (6)\", \"tab\": \"Bias\", \"score\": \"0.29422007838910086\"}", - "CNN/DailyMail - Representation (gender)": "{\"description\": \"min=0.109, mean=0.123, max=0.15, sum=0.74 (6)\", \"tab\": \"Bias\", \"score\": \"0.1233558384477443\"}", - "CNN/DailyMail - Toxic fraction": "{\"description\": \"min=0, mean=0.001, max=0.002, sum=0.004 (6)\", \"tab\": \"Toxicity\", \"score\": \"0.000715307582260372\"}", - "CNN/DailyMail - SummaC": "{\"description\": \"min=0.094, mean=0.202, max=0.259, sum=0.605 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.20179927196685032\"}", - "CNN/DailyMail - QAFactEval": "{\"description\": \"min=4.642, mean=4.67, max=4.721, sum=28.022 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"4.67041236939807\"}", - "CNN/DailyMail - BERTScore (F1)": "{\"description\": \"min=0.234, mean=0.276, max=0.301, sum=0.827 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.2755570292220846\"}", - "CNN/DailyMail - Coverage": "{\"description\": \"min=0.855, mean=0.933, max=0.973, sum=5.599 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"0.9331599358896452\"}", - "CNN/DailyMail - Density": "{\"description\": \"min=28.251, mean=31.307, max=33.584, sum=187.839 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"31.306505459997258\"}", - "CNN/DailyMail - Compression": "{\"description\": \"min=9.442, mean=9.8, max=10.068, sum=58.802 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"9.800322939057557\"}", - "CNN/DailyMail - HumanEval-faithfulness": "{\"description\": \"min=1, mean=1, max=1, sum=6 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"1.0\"}", - "CNN/DailyMail - HumanEval-relevance": "{\"description\": \"min=4.333, mean=4.378, max=4.467, sum=26.267 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"4.377777777777777\"}", - "CNN/DailyMail - HumanEval-coherence": "{\"description\": \"min=2.833, mean=3.233, max=3.867, sum=19.4 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"3.233333333333333\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "XSUM", - "source_data": { - "dataset_name": "XSUM", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on XSUM", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.155, - "details": { - "description": "min=0.153, mean=0.155, max=0.158, sum=0.929 (6)", - "tab": "Accuracy", - "XSUM - Denoised inference time (s)": "{\"description\": \"min=2.509, mean=2.523, max=2.545, sum=15.138 (6)\", \"tab\": \"Efficiency\", \"score\": \"2.522969657178858\"}", - "XSUM - # eval": "{\"description\": \"min=518, mean=518, max=518, sum=3108 (6)\", \"tab\": \"General information\", \"score\": \"518.0\"}", - "XSUM - # train": "{\"description\": \"min=4.998, mean=4.999, max=5, sum=29.992 (6)\", \"tab\": \"General information\", \"score\": \"4.998712998712999\"}", - "XSUM - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (6)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "XSUM - # prompt tokens": "{\"description\": \"min=1456.402, mean=1510.418, max=1538.921, sum=9062.51 (6)\", \"tab\": \"General information\", \"score\": \"1510.4182754182755\"}", - "XSUM - # output tokens": "{\"description\": \"min=26.037, mean=26.229, max=26.481, sum=157.375 (6)\", \"tab\": \"General information\", \"score\": \"26.22908622908623\"}", - "XSUM - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=18 (6)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "XSUM - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=4 (6)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "XSUM - Stereotypes (gender)": "{\"description\": \"min=0.444, mean=0.449, max=0.459, sum=2.697 (6)\", \"tab\": \"Bias\", \"score\": \"0.44948914431673054\"}", - "XSUM - Representation (race)": "{\"description\": \"min=0.429, mean=0.453, max=0.481, sum=2.719 (6)\", \"tab\": \"Bias\", \"score\": \"0.45310942412391686\"}", - "XSUM - Representation (gender)": "{\"description\": \"min=0.188, mean=0.218, max=0.235, sum=1.309 (6)\", \"tab\": \"Bias\", \"score\": \"0.21820243248814677\"}", - "XSUM - Toxic fraction": "{\"description\": \"min=0, mean=0.001, max=0.002, sum=0.008 (6)\", \"tab\": \"Toxicity\", \"score\": \"0.001287001287001287\"}", - "XSUM - SummaC": "{\"description\": \"min=-0.271, mean=-0.253, max=-0.224, sum=-0.76 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"-0.25337265715073337\"}", - "XSUM - QAFactEval": "{\"description\": \"min=3.343, mean=3.523, max=3.7, sum=21.139 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"3.5231601957035803\"}", - "XSUM - BERTScore (F1)": "{\"description\": \"min=0.458, mean=0.46, max=0.461, sum=1.38 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.45990517032509515\"}", - "XSUM - Coverage": "{\"description\": \"min=0.792, mean=0.793, max=0.795, sum=4.76 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"0.7933759020774565\"}", - "XSUM - Density": "{\"description\": \"min=2.672, mean=2.732, max=2.852, sum=16.393 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"2.732196710488823\"}", - "XSUM - Compression": "{\"description\": \"min=16.442, mean=16.792, max=17.056, sum=100.753 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"16.79220871639349\"}", - "XSUM - HumanEval-faithfulness": "{\"description\": \"min=0.583, mean=0.798, max=0.944, sum=4.789 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"0.7981481481481479\"}", - "XSUM - HumanEval-relevance": "{\"description\": \"min=4.167, mean=4.3, max=4.4, sum=25.8 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"4.300000000000001\"}", - "XSUM - HumanEval-coherence": "{\"description\": \"min=4.867, mean=4.891, max=4.917, sum=29.344 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"4.890740740740742\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "IMDB", - "source_data": { - "dataset_name": "IMDB", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on IMDB", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.947, - "details": { - "description": "min=0.932, mean=0.947, max=0.96, sum=2.842 (3)", - "tab": "Accuracy", - "IMDB - ECE (10-bin)": "{\"description\": \"min=0.164, mean=0.19, max=0.216, sum=0.569 (3)\", \"tab\": \"Calibration\", \"score\": \"0.18962950165784687\"}", - "IMDB - EM (Robustness)": "{\"description\": \"min=0.904, mean=0.919, max=0.937, sum=2.756 (3)\", \"tab\": \"Robustness\", \"score\": \"0.9186666666666667\"}", - "IMDB - EM (Fairness)": "{\"description\": \"min=0.929, mean=0.944, max=0.958, sum=2.831 (3)\", \"tab\": \"Fairness\", \"score\": \"0.9436666666666667\"}", - "IMDB - Denoised inference time (s)": "{\"description\": \"min=1.488, mean=1.575, max=1.732, sum=4.724 (3)\", \"tab\": \"Efficiency\", \"score\": \"1.5747312279142403\"}", - "IMDB - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "IMDB - # train": "{\"description\": \"min=4.846, mean=4.933, max=4.986, sum=14.798 (3)\", \"tab\": \"General information\", \"score\": \"4.932666666666667\"}", - "IMDB - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IMDB - # prompt tokens": "{\"description\": \"min=1152.694, mean=1389.454, max=1744.631, sum=4168.363 (3)\", \"tab\": \"General information\", \"score\": \"1389.4543333333331\"}", - "IMDB - # output tokens": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "IMDB - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "IMDB - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CivilComments", - "source_data": { - "dataset_name": "CivilComments", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on CivilComments", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.505, - "details": { - "description": "min=0, mean=0.505, max=1, sum=27.251 (54)", - "tab": "Accuracy", - "CivilComments - ECE (10-bin)": "{\"description\": \"min=0.226, mean=0.462, max=0.633, sum=24.957 (54)\", \"tab\": \"Calibration\", \"score\": \"0.46216217374926066\"}", - "CivilComments - EM (Robustness)": "{\"description\": \"min=0, mean=0.184, max=0.769, sum=9.952 (54)\", \"tab\": \"Robustness\", \"score\": \"0.18428995439708568\"}", - "CivilComments - EM (Fairness)": "{\"description\": \"min=0, mean=0.491, max=1, sum=26.489 (54)\", \"tab\": \"Fairness\", \"score\": \"0.4905409716584098\"}", - "CivilComments - Denoised inference time (s)": "{\"description\": \"min=0.299, mean=0.498, max=0.974, sum=26.871 (54)\", \"tab\": \"Efficiency\", \"score\": \"0.4976179389529128\"}", - "CivilComments - # eval": "{\"description\": \"min=74, mean=371.556, max=683, sum=20064 (54)\", \"tab\": \"General information\", \"score\": \"371.55555555555554\"}", - "CivilComments - # train": "{\"description\": \"min=5, mean=5, max=5, sum=270 (54)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CivilComments - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (54)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CivilComments - # prompt tokens": "{\"description\": \"min=356.537, mean=722.635, max=1267.519, sum=39022.317 (54)\", \"tab\": \"General information\", \"score\": \"722.6354931173206\"}", - "CivilComments - # output tokens": "{\"description\": \"min=5, mean=5, max=5, sum=270 (54)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CivilComments - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=162 (54)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "CivilComments - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (54)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "RAFT", - "source_data": { - "dataset_name": "RAFT", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on RAFT", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.606, - "details": { - "description": "min=0.075, mean=0.606, max=0.975, sum=20 (33)", - "tab": "Accuracy", - "RAFT - ECE (10-bin)": "{\"description\": \"min=0.1, mean=0.352, max=0.74, sum=11.606 (33)\", \"tab\": \"Calibration\", \"score\": \"0.35168585204039804\"}", - "RAFT - EM (Robustness)": "{\"description\": \"min=0, mean=0.48, max=0.975, sum=15.85 (33)\", \"tab\": \"Robustness\", \"score\": \"0.4803030303030303\"}", - "RAFT - EM (Fairness)": "{\"description\": \"min=0.075, mean=0.58, max=0.975, sum=19.125 (33)\", \"tab\": \"Fairness\", \"score\": \"0.5795454545454547\"}", - "RAFT - Denoised inference time (s)": "{\"description\": \"min=0.403, mean=0.962, max=1.712, sum=31.76 (33)\", \"tab\": \"Efficiency\", \"score\": \"0.9624239013413396\"}", - "RAFT - # eval": "{\"description\": \"min=40, mean=40, max=40, sum=1320 (33)\", \"tab\": \"General information\", \"score\": \"40.0\"}", - "RAFT - # train": "{\"description\": \"min=0, mean=4.556, max=5, sum=150.35 (33)\", \"tab\": \"General information\", \"score\": \"4.556060606060607\"}", - "RAFT - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (33)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "RAFT - # prompt tokens": "{\"description\": \"min=257.35, mean=812.938, max=1773.675, sum=26826.95 (33)\", \"tab\": \"General information\", \"score\": \"812.937878787879\"}", - "RAFT - # output tokens": "{\"description\": \"min=5, mean=9.057, max=18.95, sum=298.875 (33)\", \"tab\": \"General information\", \"score\": \"9.056818181818182\"}", - "RAFT - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=99 (33)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "RAFT - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (33)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/meta_opt-66b.json b/data/models/meta_opt-66b.json deleted file mode 100644 index ed5a8553e19001a66f4be6a5a1f08f4f7f719c9f..0000000000000000000000000000000000000000 --- a/data/models/meta_opt-66b.json +++ /dev/null @@ -1,674 +0,0 @@ -{ - "model_info": { - "name": "OPT 66B", - "id": "meta/OPT-66B", - "developer": "Meta", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_classic/meta_OPT-66B/1774096308.339228", - "retrieved_timestamp": "1774096308.339228", - "source_metadata": { - "source_name": "helm_classic", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_classic", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_classic", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperform on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.448, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Calibration": "{\"description\": \"\", \"tab\": \"Calibration\", \"score\": \"0.2888771827640159\"}", - "Mean win rate - Robustness": "{\"description\": \"\", \"tab\": \"Robustness\", \"score\": \"0.43828848200372117\"}", - "Mean win rate - Fairness": "{\"description\": \"\", \"tab\": \"Fairness\", \"score\": \"0.4763117490592463\"}", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.466875\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}", - "Mean win rate - Bias": "{\"description\": \"\", \"tab\": \"Bias\", \"score\": \"0.6312224376358433\"}", - "Mean win rate - Toxicity": "{\"description\": \"\", \"tab\": \"Toxicity\", \"score\": \"0.3347556764223431\"}", - "Mean win rate - Summarization metrics": "{\"description\": \"\", \"tab\": \"Summarization metrics\", \"score\": \"0.5785714285714286\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.276, - "details": { - "description": "min=0.2, mean=0.276, max=0.37, sum=4.141 (15)", - "tab": "Accuracy", - "MMLU - ECE (10-bin)": "{\"description\": \"min=0.101, mean=0.135, max=0.172, sum=2.031 (15)\", \"tab\": \"Calibration\", \"score\": \"0.13542563946906333\"}", - "MMLU - EM (Robustness)": "{\"description\": \"min=0.13, mean=0.216, max=0.32, sum=3.242 (15)\", \"tab\": \"Robustness\", \"score\": \"0.21610526315789472\"}", - "MMLU - EM (Fairness)": "{\"description\": \"min=0.18, mean=0.229, max=0.33, sum=3.44 (15)\", \"tab\": \"Fairness\", \"score\": \"0.22935672514619884\"}", - "MMLU - Denoised inference time (s)": "{\"description\": \"min=0.041, mean=0.055, max=0.081, sum=0.818 (15)\", \"tab\": \"Efficiency\", \"score\": \"0.05452067670741475\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=1542 (15)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=75 (15)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (15)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=371.38, mean=472.274, max=624.07, sum=7084.111 (15)\", \"tab\": \"General information\", \"score\": \"472.2740350877193\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=15 (15)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "MMLU - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=45 (15)\", \"tab\": \"General information\", \"score\": \"3.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "BoolQ", - "source_data": { - "dataset_name": "BoolQ", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on BoolQ", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.76, - "details": { - "description": "min=0.753, mean=0.76, max=0.764, sum=2.281 (3)", - "tab": "Accuracy", - "BoolQ - ECE (10-bin)": "{\"description\": \"min=0.193, mean=0.2, max=0.206, sum=0.601 (3)\", \"tab\": \"Calibration\", \"score\": \"0.20047176103986394\"}", - "BoolQ - EM (Robustness)": "{\"description\": \"min=0.666, mean=0.683, max=0.701, sum=2.049 (3)\", \"tab\": \"Robustness\", \"score\": \"0.6829999999999999\"}", - "BoolQ - EM (Fairness)": "{\"description\": \"min=0.696, mean=0.71, max=0.721, sum=2.131 (3)\", \"tab\": \"Fairness\", \"score\": \"0.7103333333333333\"}", - "BoolQ - Denoised inference time (s)": "{\"description\": \"min=0.272, mean=0.834, max=1.907, sum=2.501 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.8336340090708299\"}", - "BoolQ - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "BoolQ - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "BoolQ - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "BoolQ - # prompt tokens": "{\"description\": \"min=660.073, mean=908.406, max=1242.073, sum=2725.219 (3)\", \"tab\": \"General information\", \"score\": \"908.4063333333334\"}", - "BoolQ - # output tokens": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "BoolQ - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "BoolQ - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.638, - "details": { - "description": "min=0.618, mean=0.638, max=0.655, sum=1.913 (3)", - "tab": "Accuracy", - "NarrativeQA - ECE (10-bin)": "{\"description\": \"min=0.224, mean=0.245, max=0.264, sum=0.734 (3)\", \"tab\": \"Calibration\", \"score\": \"0.2445466042880168\"}", - "NarrativeQA - F1 (Robustness)": "{\"description\": \"min=0.364, mean=0.397, max=0.421, sum=1.19 (3)\", \"tab\": \"Robustness\", \"score\": \"0.39653941552028354\"}", - "NarrativeQA - F1 (Fairness)": "{\"description\": \"min=0.497, mean=0.526, max=0.543, sum=1.579 (3)\", \"tab\": \"Fairness\", \"score\": \"0.5262433008374211\"}", - "NarrativeQA - Denoised inference time (s)": "{\"description\": \"min=0.939, mean=1.98, max=3.714, sum=5.939 (3)\", \"tab\": \"Efficiency\", \"score\": \"1.979606440811339\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=1065 (3)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=1.051, mean=1.647, max=2.085, sum=4.941 (3)\", \"tab\": \"General information\", \"score\": \"1.6469483568075116\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=1601.955, mean=1652.377, max=1705.003, sum=4957.132 (3)\", \"tab\": \"General information\", \"score\": \"1652.3774647887324\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=39.707, mean=50.904, max=65.363, sum=152.713 (3)\", \"tab\": \"General information\", \"score\": \"50.90422535211267\"}", - "NarrativeQA - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NarrativeQA - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NarrativeQA - Stereotypes (gender)": "{\"description\": \"min=0.406, mean=0.416, max=0.425, sum=1.248 (3)\", \"tab\": \"Bias\", \"score\": \"0.41597222222222224\"}", - "NarrativeQA - Representation (race)": "{\"description\": \"min=0.333, mean=0.556, max=0.667, sum=1.667 (3)\", \"tab\": \"Bias\", \"score\": \"0.5555555555555556\"}", - "NarrativeQA - Representation (gender)": "{\"description\": \"min=0.164, mean=0.191, max=0.207, sum=0.574 (3)\", \"tab\": \"Bias\", \"score\": \"0.1911771437726737\"}", - "NarrativeQA - Toxic fraction": "{\"description\": \"min=0.02, mean=0.022, max=0.025, sum=0.065 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.0215962441314554\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (open-book)", - "source_data": { - "dataset_name": "NaturalQuestions (open-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (open-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.596, - "details": { - "description": "min=0.582, mean=0.596, max=0.615, sum=1.788 (3)", - "tab": "Accuracy", - "NaturalQuestions (closed-book) - ECE (10-bin)": "{\"description\": \"min=0.134, mean=0.141, max=0.149, sum=0.423 (3)\", \"tab\": \"Calibration\", \"score\": \"0.14107540425227785\"}", - "NaturalQuestions (open-book) - ECE (10-bin)": "{\"description\": \"min=0.379, mean=0.384, max=0.387, sum=1.153 (3)\", \"tab\": \"Calibration\", \"score\": \"0.38437204570087863\"}", - "NaturalQuestions (closed-book) - F1 (Robustness)": "{\"description\": \"min=0.2, mean=0.206, max=0.216, sum=0.619 (3)\", \"tab\": \"Robustness\", \"score\": \"0.20625206311676839\"}", - "NaturalQuestions (open-book) - F1 (Robustness)": "{\"description\": \"min=0.419, mean=0.458, max=0.503, sum=1.373 (3)\", \"tab\": \"Robustness\", \"score\": \"0.45767430702477907\"}", - "NaturalQuestions (closed-book) - F1 (Fairness)": "{\"description\": \"min=0.215, mean=0.218, max=0.221, sum=0.654 (3)\", \"tab\": \"Fairness\", \"score\": \"0.2180459446078801\"}", - "NaturalQuestions (open-book) - F1 (Fairness)": "{\"description\": \"min=0.52, mean=0.536, max=0.558, sum=1.607 (3)\", \"tab\": \"Fairness\", \"score\": \"0.5357020972773482\"}", - "NaturalQuestions (closed-book) - Denoised inference time (s)": "{\"description\": \"min=0.471, mean=0.611, max=0.739, sum=1.834 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.611190575244526\"}", - "NaturalQuestions (open-book) - Denoised inference time (s)": "{\"description\": \"min=2.887, mean=3.632, max=4.314, sum=10.896 (3)\", \"tab\": \"Efficiency\", \"score\": \"3.631964569965005\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=110.254, mean=112.254, max=116.254, sum=336.762 (3)\", \"tab\": \"General information\", \"score\": \"112.254\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=128.956, mean=153.231, max=173.545, sum=459.692 (3)\", \"tab\": \"General information\", \"score\": \"153.23066666666668\"}", - "NaturalQuestions (closed-book) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.647, mean=4.691, max=4.724, sum=14.074 (3)\", \"tab\": \"General information\", \"score\": \"4.691333333333334\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.036, mean=0.036, max=0.036, sum=0.108 (3)\", \"tab\": \"General information\", \"score\": \"0.036\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1231.212, mean=1419.574, max=1523.257, sum=4258.721 (3)\", \"tab\": \"General information\", \"score\": \"1419.5736666666664\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=168.231, mean=211.805, max=244.906, sum=635.415 (3)\", \"tab\": \"General information\", \"score\": \"211.80499999999998\"}", - "NaturalQuestions (open-book) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NaturalQuestions (closed-book) - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - Stereotypes (gender)": "{\"description\": \"min=0, mean=0.278, max=0.5, sum=0.833 (3)\", \"tab\": \"Bias\", \"score\": \"0.27777777777777773\"}", - "NaturalQuestions (closed-book) - Representation (race)": "{\"description\": \"min=0.467, mean=0.481, max=0.491, sum=1.444 (3)\", \"tab\": \"Bias\", \"score\": \"0.481339792158324\"}", - "NaturalQuestions (closed-book) - Representation (gender)": "{\"description\": \"min=0.106, mean=0.156, max=0.233, sum=0.469 (3)\", \"tab\": \"Bias\", \"score\": \"0.156341189674523\"}", - "NaturalQuestions (open-book) - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=0.667 (1)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "NaturalQuestions (open-book) - Stereotypes (gender)": "{\"description\": \"min=0.287, mean=0.338, max=0.395, sum=1.015 (3)\", \"tab\": \"Bias\", \"score\": \"0.33841269841269833\"}", - "NaturalQuestions (open-book) - Representation (race)": "{\"description\": \"min=0.4, mean=0.427, max=0.48, sum=1.281 (3)\", \"tab\": \"Bias\", \"score\": \"0.42701178032188486\"}", - "NaturalQuestions (open-book) - Representation (gender)": "{\"description\": \"min=0.116, mean=0.119, max=0.124, sum=0.357 (3)\", \"tab\": \"Bias\", \"score\": \"0.11888541157186479\"}", - "NaturalQuestions (closed-book) - Toxic fraction": "{\"description\": \"min=0.001, mean=0.001, max=0.001, sum=0.003 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.001\"}", - "NaturalQuestions (open-book) - Toxic fraction": "{\"description\": \"min=0.001, mean=0.002, max=0.002, sum=0.005 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.0016666666666666668\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "QuAC", - "source_data": { - "dataset_name": "QuAC", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on QuAC", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.357, - "details": { - "description": "min=0.35, mean=0.357, max=0.366, sum=1.07 (3)", - "tab": "Accuracy", - "QuAC - ECE (10-bin)": "{\"description\": \"min=0.135, mean=0.154, max=0.176, sum=0.461 (3)\", \"tab\": \"Calibration\", \"score\": \"0.15357329550060583\"}", - "QuAC - F1 (Robustness)": "{\"description\": \"min=0.177, mean=0.199, max=0.217, sum=0.597 (3)\", \"tab\": \"Robustness\", \"score\": \"0.19914898808715295\"}", - "QuAC - F1 (Fairness)": "{\"description\": \"min=0.267, mean=0.268, max=0.27, sum=0.805 (3)\", \"tab\": \"Fairness\", \"score\": \"0.26839685415319225\"}", - "QuAC - Denoised inference time (s)": "{\"description\": \"min=2.636, mean=2.658, max=2.683, sum=7.974 (3)\", \"tab\": \"Efficiency\", \"score\": \"2.6581093871351746\"}", - "QuAC - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "QuAC - # train": "{\"description\": \"min=0.845, mean=0.944, max=1.086, sum=2.833 (3)\", \"tab\": \"General information\", \"score\": \"0.9443333333333334\"}", - "QuAC - truncated": "{\"description\": \"min=0.016, mean=0.016, max=0.016, sum=0.048 (3)\", \"tab\": \"General information\", \"score\": \"0.016\"}", - "QuAC - # prompt tokens": "{\"description\": \"min=1625.523, mean=1644.831, max=1670.605, sum=4934.492 (3)\", \"tab\": \"General information\", \"score\": \"1644.8306666666667\"}", - "QuAC - # output tokens": "{\"description\": \"min=89.614, mean=91.909, max=95.996, sum=275.728 (3)\", \"tab\": \"General information\", \"score\": \"91.90933333333334\"}", - "QuAC - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "QuAC - Stereotypes (race)": "{\"description\": \"min=0.556, mean=0.592, max=0.619, sum=1.775 (3)\", \"tab\": \"Bias\", \"score\": \"0.5915343915343915\"}", - "QuAC - Stereotypes (gender)": "{\"description\": \"min=0.398, mean=0.413, max=0.424, sum=1.239 (3)\", \"tab\": \"Bias\", \"score\": \"0.41297615039041286\"}", - "QuAC - Representation (race)": "{\"description\": \"min=0.228, mean=0.272, max=0.324, sum=0.816 (3)\", \"tab\": \"Bias\", \"score\": \"0.27205505897640186\"}", - "QuAC - Representation (gender)": "{\"description\": \"min=0.239, mean=0.245, max=0.252, sum=0.734 (3)\", \"tab\": \"Bias\", \"score\": \"0.2445248639131045\"}", - "QuAC - Toxic fraction": "{\"description\": \"min=0.001, mean=0.001, max=0.002, sum=0.004 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.0013333333333333333\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "HellaSwag", - "source_data": { - "dataset_name": "HellaSwag", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on HellaSwag", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.745, - "details": { - "description": "min=0.745, mean=0.745, max=0.745, sum=0.745 (1)", - "tab": "Accuracy", - "HellaSwag - ECE (10-bin)": "{\"description\": \"min=0.293, mean=0.293, max=0.293, sum=0.293 (1)\", \"tab\": \"Calibration\", \"score\": \"0.29326475041918015\"}", - "HellaSwag - EM (Robustness)": "{\"description\": \"min=0.699, mean=0.699, max=0.699, sum=0.699 (1)\", \"tab\": \"Robustness\", \"score\": \"0.699\"}", - "HellaSwag - EM (Fairness)": "{\"description\": \"min=0.597, mean=0.597, max=0.597, sum=0.597 (1)\", \"tab\": \"Fairness\", \"score\": \"0.597\"}", - "HellaSwag - Denoised inference time (s)": "{\"description\": \"min=0.971, mean=0.971, max=0.971, sum=0.971 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.9708148735597889\"}", - "HellaSwag - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "HellaSwag - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "HellaSwag - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "HellaSwag - # prompt tokens": "{\"description\": \"min=87.888, mean=87.888, max=87.888, sum=87.888 (1)\", \"tab\": \"General information\", \"score\": \"87.888\"}", - "HellaSwag - # output tokens": "{\"description\": \"min=0.2, mean=0.2, max=0.2, sum=0.2 (1)\", \"tab\": \"General information\", \"score\": \"0.2\"}", - "HellaSwag - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.534, - "details": { - "description": "min=0.534, mean=0.534, max=0.534, sum=0.534 (1)", - "tab": "Accuracy", - "OpenbookQA - ECE (10-bin)": "{\"description\": \"min=0.237, mean=0.237, max=0.237, sum=0.237 (1)\", \"tab\": \"Calibration\", \"score\": \"0.2373615873422732\"}", - "OpenbookQA - EM (Robustness)": "{\"description\": \"min=0.45, mean=0.45, max=0.45, sum=0.45 (1)\", \"tab\": \"Robustness\", \"score\": \"0.45\"}", - "OpenbookQA - EM (Fairness)": "{\"description\": \"min=0.454, mean=0.454, max=0.454, sum=0.454 (1)\", \"tab\": \"Fairness\", \"score\": \"0.454\"}", - "OpenbookQA - Denoised inference time (s)": "{\"description\": \"min=0.188, mean=0.188, max=0.188, sum=0.188 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.18798254558309685\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=5.27, mean=5.27, max=5.27, sum=5.27 (1)\", \"tab\": \"General information\", \"score\": \"5.27\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "OpenbookQA - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "TruthfulQA", - "source_data": { - "dataset_name": "TruthfulQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on TruthfulQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.201, - "details": { - "description": "min=0.185, mean=0.201, max=0.22, sum=0.804 (4)", - "tab": "Accuracy", - "TruthfulQA - ECE (10-bin)": "{\"description\": \"min=0.047, mean=0.073, max=0.084, sum=0.293 (4)\", \"tab\": \"Calibration\", \"score\": \"0.07328356622626138\"}", - "TruthfulQA - EM (Robustness)": "{\"description\": \"min=0.135, mean=0.174, max=0.206, sum=0.694 (4)\", \"tab\": \"Robustness\", \"score\": \"0.1735474006116208\"}", - "TruthfulQA - EM (Fairness)": "{\"description\": \"min=0.145, mean=0.173, max=0.206, sum=0.693 (4)\", \"tab\": \"Fairness\", \"score\": \"0.17316513761467892\"}", - "TruthfulQA - Denoised inference time (s)": "{\"description\": \"min=0.033, mean=0.041, max=0.046, sum=0.163 (4)\", \"tab\": \"Efficiency\", \"score\": \"0.04074840224276806\"}", - "TruthfulQA - # eval": "{\"description\": \"min=654, mean=654, max=654, sum=2616 (4)\", \"tab\": \"General information\", \"score\": \"654.0\"}", - "TruthfulQA - # train": "{\"description\": \"min=0, mean=3.75, max=5, sum=15 (4)\", \"tab\": \"General information\", \"score\": \"3.75\"}", - "TruthfulQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (4)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "TruthfulQA - # prompt tokens": "{\"description\": \"min=85.121, mean=404.621, max=529.121, sum=1618.483 (4)\", \"tab\": \"General information\", \"score\": \"404.62079510703364\"}", - "TruthfulQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=4 (4)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "TruthfulQA - # trials": "{\"description\": \"min=1, mean=2.5, max=3, sum=10 (4)\", \"tab\": \"General information\", \"score\": \"2.5\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MS MARCO (TREC)", - "source_data": { - "dataset_name": "MS MARCO (TREC)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "NDCG@10 on MS MARCO (TREC)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.482, - "details": { - "description": "min=0.467, mean=0.482, max=0.511, sum=1.446 (3)", - "tab": "Accuracy", - "MS MARCO (regular) - RR@10 (Robustness)": "{\"description\": \"min=0.175, mean=0.179, max=0.187, sum=0.537 (3)\", \"tab\": \"Robustness\", \"score\": \"0.1788788359788358\"}", - "MS MARCO (TREC) - NDCG@10 (Robustness)": "{\"description\": \"min=0.421, mean=0.437, max=0.46, sum=1.31 (3)\", \"tab\": \"Robustness\", \"score\": \"0.436684763137285\"}", - "MS MARCO (regular) - RR@10 (Fairness)": "{\"description\": \"min=0.21, mean=0.214, max=0.221, sum=0.642 (3)\", \"tab\": \"Fairness\", \"score\": \"0.2139329365079363\"}", - "MS MARCO (TREC) - NDCG@10 (Fairness)": "{\"description\": \"min=0.45, mean=0.471, max=0.501, sum=1.412 (3)\", \"tab\": \"Fairness\", \"score\": \"0.4706976603850948\"}", - "MS MARCO (regular) - Denoised inference time (s)": "{\"description\": \"min=0.066, mean=0.076, max=0.089, sum=0.227 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.07567241383876121\"}", - "MS MARCO (TREC) - Denoised inference time (s)": "{\"description\": \"min=0.061, mean=0.102, max=0.183, sum=0.305 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.10182954292591756\"}", - "MS MARCO (regular) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "MS MARCO (regular) - # train": "{\"description\": \"min=2, mean=2, max=2, sum=6 (3)\", \"tab\": \"General information\", \"score\": \"2.0\"}", - "MS MARCO (regular) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MS MARCO (regular) - # prompt tokens": "{\"description\": \"min=495.232, mean=532.565, max=577.232, sum=1597.696 (3)\", \"tab\": \"General information\", \"score\": \"532.5653333333333\"}", - "MS MARCO (regular) - # output tokens": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MS MARCO (regular) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "MS MARCO (TREC) - # eval": "{\"description\": \"min=43, mean=43, max=43, sum=129 (3)\", \"tab\": \"General information\", \"score\": \"43.0\"}", - "MS MARCO (TREC) - # train": "{\"description\": \"min=2, mean=2, max=2, sum=6 (3)\", \"tab\": \"General information\", \"score\": \"2.0\"}", - "MS MARCO (TREC) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MS MARCO (TREC) - # prompt tokens": "{\"description\": \"min=478.488, mean=515.822, max=560.488, sum=1547.465 (3)\", \"tab\": \"General information\", \"score\": \"515.8217054263565\"}", - "MS MARCO (TREC) - # output tokens": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MS MARCO (TREC) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "MS MARCO (regular) - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "MS MARCO (TREC) - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CNN/DailyMail", - "source_data": { - "dataset_name": "CNN/DailyMail", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on CNN/DailyMail", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.136, - "details": { - "description": "min=0.119, mean=0.136, max=0.149, sum=0.816 (6)", - "tab": "Accuracy", - "CNN/DailyMail - Denoised inference time (s)": "{\"description\": \"min=1.828, mean=1.972, max=2.045, sum=11.831 (6)\", \"tab\": \"Efficiency\", \"score\": \"1.971851329588582\"}", - "CNN/DailyMail - # eval": "{\"description\": \"min=466, mean=466, max=466, sum=2796 (6)\", \"tab\": \"General information\", \"score\": \"466.0\"}", - "CNN/DailyMail - # train": "{\"description\": \"min=5, mean=5, max=5, sum=30 (6)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CNN/DailyMail - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (6)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CNN/DailyMail - # prompt tokens": "{\"description\": \"min=1531.586, mean=1549.919, max=1567.586, sum=9299.515 (6)\", \"tab\": \"General information\", \"score\": \"1549.9191702432045\"}", - "CNN/DailyMail - # output tokens": "{\"description\": \"min=72.955, mean=77.928, max=83.685, sum=467.567 (6)\", \"tab\": \"General information\", \"score\": \"77.9277539341917\"}", - "CNN/DailyMail - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=18 (6)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "CNN/DailyMail - Stereotypes (race)": "{\"description\": \"min=0.589, mean=0.609, max=0.627, sum=3.657 (6)\", \"tab\": \"Bias\", \"score\": \"0.6094903870639165\"}", - "CNN/DailyMail - Stereotypes (gender)": "{\"description\": \"min=0.396, mean=0.404, max=0.412, sum=2.424 (6)\", \"tab\": \"Bias\", \"score\": \"0.40393077624581836\"}", - "CNN/DailyMail - Representation (race)": "{\"description\": \"min=0.287, mean=0.337, max=0.37, sum=2.024 (6)\", \"tab\": \"Bias\", \"score\": \"0.33739205476866063\"}", - "CNN/DailyMail - Representation (gender)": "{\"description\": \"min=0.121, mean=0.128, max=0.139, sum=0.766 (6)\", \"tab\": \"Bias\", \"score\": \"0.12773227690338504\"}", - "CNN/DailyMail - Toxic fraction": "{\"description\": \"min=0, mean=0.001, max=0.002, sum=0.004 (6)\", \"tab\": \"Toxicity\", \"score\": \"0.000715307582260372\"}", - "CNN/DailyMail - SummaC": "{\"description\": \"min=0.064, mean=0.197, max=0.291, sum=0.592 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.19745183659958473\"}", - "CNN/DailyMail - QAFactEval": "{\"description\": \"min=4.708, mean=4.735, max=4.771, sum=28.41 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"4.735075808555843\"}", - "CNN/DailyMail - BERTScore (F1)": "{\"description\": \"min=0.206, mean=0.256, max=0.287, sum=0.769 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.2564336767010044\"}", - "CNN/DailyMail - Coverage": "{\"description\": \"min=0.829, mean=0.92, max=0.97, sum=5.522 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"0.9202647711974157\"}", - "CNN/DailyMail - Density": "{\"description\": \"min=34.301, mean=41.595, max=46.027, sum=249.573 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"41.59545904426739\"}", - "CNN/DailyMail - Compression": "{\"description\": \"min=8.796, mean=9.759, max=10.302, sum=58.557 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"9.759458553538733\"}", - "CNN/DailyMail - HumanEval-faithfulness": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-relevance": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-coherence": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "XSUM", - "source_data": { - "dataset_name": "XSUM", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on XSUM", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.126, - "details": { - "description": "min=0.123, mean=0.126, max=0.131, sum=0.757 (6)", - "tab": "Accuracy", - "XSUM - Denoised inference time (s)": "{\"description\": \"min=0.833, mean=0.885, max=0.939, sum=5.309 (6)\", \"tab\": \"Efficiency\", \"score\": \"0.8849094198151292\"}", - "XSUM - # eval": "{\"description\": \"min=518, mean=518, max=518, sum=3108 (6)\", \"tab\": \"General information\", \"score\": \"518.0\"}", - "XSUM - # train": "{\"description\": \"min=4.998, mean=4.999, max=5, sum=29.992 (6)\", \"tab\": \"General information\", \"score\": \"4.998712998712999\"}", - "XSUM - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (6)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "XSUM - # prompt tokens": "{\"description\": \"min=1456.402, mean=1510.418, max=1538.921, sum=9062.51 (6)\", \"tab\": \"General information\", \"score\": \"1510.4182754182755\"}", - "XSUM - # output tokens": "{\"description\": \"min=23.931, mean=24.362, max=24.873, sum=146.17 (6)\", \"tab\": \"General information\", \"score\": \"24.361647361647357\"}", - "XSUM - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=18 (6)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "XSUM - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=4 (6)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "XSUM - Stereotypes (gender)": "{\"description\": \"min=0.453, mean=0.469, max=0.478, sum=2.812 (6)\", \"tab\": \"Bias\", \"score\": \"0.46873713991769544\"}", - "XSUM - Representation (race)": "{\"description\": \"min=0.356, mean=0.462, max=0.532, sum=2.769 (6)\", \"tab\": \"Bias\", \"score\": \"0.46156957217464706\"}", - "XSUM - Representation (gender)": "{\"description\": \"min=0.168, mean=0.186, max=0.201, sum=1.118 (6)\", \"tab\": \"Bias\", \"score\": \"0.18640980232047377\"}", - "XSUM - Toxic fraction": "{\"description\": \"min=0.002, mean=0.003, max=0.004, sum=0.015 (6)\", \"tab\": \"Toxicity\", \"score\": \"0.002574002574002574\"}", - "XSUM - SummaC": "{\"description\": \"min=-0.208, mean=-0.189, max=-0.166, sum=-0.566 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"-0.18875486064192462\"}", - "XSUM - QAFactEval": "{\"description\": \"min=3.146, mean=3.324, max=3.669, sum=19.946 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"3.3243234460347995\"}", - "XSUM - BERTScore (F1)": "{\"description\": \"min=0.416, mean=0.417, max=0.419, sum=1.251 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.4169695047035986\"}", - "XSUM - Coverage": "{\"description\": \"min=0.815, mean=0.817, max=0.819, sum=4.904 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"0.8172878337570123\"}", - "XSUM - Density": "{\"description\": \"min=3.708, mean=3.899, max=4.102, sum=23.393 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"3.898863398596404\"}", - "XSUM - Compression": "{\"description\": \"min=18.005, mean=18.414, max=18.872, sum=110.483 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"18.413782867028814\"}", - "XSUM - HumanEval-faithfulness": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-relevance": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-coherence": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "IMDB", - "source_data": { - "dataset_name": "IMDB", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on IMDB", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.917, - "details": { - "description": "min=0.906, mean=0.917, max=0.926, sum=2.752 (3)", - "tab": "Accuracy", - "IMDB - ECE (10-bin)": "{\"description\": \"min=0.289, mean=0.302, max=0.327, sum=0.905 (3)\", \"tab\": \"Calibration\", \"score\": \"0.30155451934186406\"}", - "IMDB - EM (Robustness)": "{\"description\": \"min=0.872, mean=0.886, max=0.901, sum=2.659 (3)\", \"tab\": \"Robustness\", \"score\": \"0.8863333333333333\"}", - "IMDB - EM (Fairness)": "{\"description\": \"min=0.898, mean=0.908, max=0.919, sum=2.725 (3)\", \"tab\": \"Fairness\", \"score\": \"0.9083333333333333\"}", - "IMDB - Denoised inference time (s)": "{\"description\": \"min=0.515, mean=0.54, max=0.569, sum=1.62 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.5398914054599924\"}", - "IMDB - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "IMDB - # train": "{\"description\": \"min=4.846, mean=4.933, max=4.986, sum=14.798 (3)\", \"tab\": \"General information\", \"score\": \"4.932666666666667\"}", - "IMDB - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IMDB - # prompt tokens": "{\"description\": \"min=1152.694, mean=1389.454, max=1744.631, sum=4168.363 (3)\", \"tab\": \"General information\", \"score\": \"1389.4543333333331\"}", - "IMDB - # output tokens": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "IMDB - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "IMDB - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CivilComments", - "source_data": { - "dataset_name": "CivilComments", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on CivilComments", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.506, - "details": { - "description": "min=0, mean=0.506, max=1, sum=27.302 (54)", - "tab": "Accuracy", - "CivilComments - ECE (10-bin)": "{\"description\": \"min=0.299, mean=0.474, max=0.666, sum=25.591 (54)\", \"tab\": \"Calibration\", \"score\": \"0.47391416538592424\"}", - "CivilComments - EM (Robustness)": "{\"description\": \"min=0, mean=0.305, max=0.939, sum=16.459 (54)\", \"tab\": \"Robustness\", \"score\": \"0.30478947142198615\"}", - "CivilComments - EM (Fairness)": "{\"description\": \"min=0, mean=0.5, max=1, sum=27.006 (54)\", \"tab\": \"Fairness\", \"score\": \"0.5001070006147802\"}", - "CivilComments - Denoised inference time (s)": "{\"description\": \"min=0.173, mean=0.212, max=0.325, sum=11.459 (54)\", \"tab\": \"Efficiency\", \"score\": \"0.21220531272072915\"}", - "CivilComments - # eval": "{\"description\": \"min=74, mean=371.556, max=683, sum=20064 (54)\", \"tab\": \"General information\", \"score\": \"371.55555555555554\"}", - "CivilComments - # train": "{\"description\": \"min=5, mean=5, max=5, sum=270 (54)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CivilComments - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (54)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CivilComments - # prompt tokens": "{\"description\": \"min=356.537, mean=722.635, max=1267.519, sum=39022.317 (54)\", \"tab\": \"General information\", \"score\": \"722.6354931173206\"}", - "CivilComments - # output tokens": "{\"description\": \"min=5, mean=5, max=5, sum=270 (54)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CivilComments - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=162 (54)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "CivilComments - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (54)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "RAFT", - "source_data": { - "dataset_name": "RAFT", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on RAFT", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.557, - "details": { - "description": "min=0.175, mean=0.557, max=0.975, sum=18.375 (33)", - "tab": "Accuracy", - "RAFT - ECE (10-bin)": "{\"description\": \"min=0.126, mean=0.468, max=0.975, sum=15.455 (33)\", \"tab\": \"Calibration\", \"score\": \"0.468339884912531\"}", - "RAFT - EM (Robustness)": "{\"description\": \"min=0.025, mean=0.405, max=0.85, sum=13.35 (33)\", \"tab\": \"Robustness\", \"score\": \"0.4045454545454546\"}", - "RAFT - EM (Fairness)": "{\"description\": \"min=0.175, mean=0.536, max=0.975, sum=17.7 (33)\", \"tab\": \"Fairness\", \"score\": \"0.5363636363636364\"}", - "RAFT - Denoised inference time (s)": "{\"description\": \"min=0.069, mean=1.871, max=6.606, sum=61.732 (33)\", \"tab\": \"Efficiency\", \"score\": \"1.8706600076246471\"}", - "RAFT - # eval": "{\"description\": \"min=40, mean=40, max=40, sum=1320 (33)\", \"tab\": \"General information\", \"score\": \"40.0\"}", - "RAFT - # train": "{\"description\": \"min=0, mean=4.556, max=5, sum=150.35 (33)\", \"tab\": \"General information\", \"score\": \"4.556060606060607\"}", - "RAFT - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (33)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "RAFT - # prompt tokens": "{\"description\": \"min=257.35, mean=812.938, max=1773.675, sum=26826.95 (33)\", \"tab\": \"General information\", \"score\": \"812.937878787879\"}", - "RAFT - # output tokens": "{\"description\": \"min=5, mean=18.712, max=30, sum=617.5 (33)\", \"tab\": \"General information\", \"score\": \"18.712121212121207\"}", - "RAFT - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=99 (33)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "RAFT - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (33)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/mhl1_qwen2.5-0.5b-cinstruct-stage1.json b/data/models/mhl1_qwen2.5-0.5b-cinstruct-stage1.json deleted file mode 100644 index ae7a7712dff58fde2e3dee4639b7aaa5e8621e9e..0000000000000000000000000000000000000000 --- a/data/models/mhl1_qwen2.5-0.5b-cinstruct-stage1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-cinstruct-stage1", - "id": "mhl1/Qwen2.5-0.5B-cinstruct-stage1", - "developer": "mhl1", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/mhl1_Qwen2.5-0.5B-cinstruct-stage1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1482 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3256 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0128 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2651 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.35 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1139 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/microsoft_dialogpt-medium.json b/data/models/microsoft_dialogpt-medium.json deleted file mode 100644 index e9b68a933521f4d7863572b63038ebed1c164c5f..0000000000000000000000000000000000000000 --- a/data/models/microsoft_dialogpt-medium.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "DialoGPT-medium", - "id": "microsoft/DialoGPT-medium", - "developer": "microsoft", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "GPT2LMHeadModel", - "params_billions": "0.345" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/microsoft_DialoGPT-medium/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1479 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3014 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2542 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4287 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1119 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/microsoft_orca-2-13b.json b/data/models/microsoft_orca-2-13b.json deleted file mode 100644 index 3031003fa9866e1e23af00f26c8e3ad2841ce18e..0000000000000000000000000000000000000000 --- a/data/models/microsoft_orca-2-13b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Orca-2-13b", - "id": "microsoft/Orca-2-13b", - "developer": "microsoft", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "13.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/microsoft_Orca-2-13b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3128 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4884 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0317 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2802 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.513 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2749 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/microsoft_orca-2-7b.json b/data/models/microsoft_orca-2-7b.json deleted file mode 100644 index 01482c9abea94a9a579e0a90463d94bec8ba38e4..0000000000000000000000000000000000000000 --- a/data/models/microsoft_orca-2-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Orca-2-7b", - "id": "microsoft/Orca-2-7b", - "developer": "microsoft", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "7.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/microsoft_Orca-2-7b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2183 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4452 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0196 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2609 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5026 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2319 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/microsoft_phi-1.json b/data/models/microsoft_phi-1.json deleted file mode 100644 index c7ded33c69b49be61faa2d4c6a757270812859db..0000000000000000000000000000000000000000 --- a/data/models/microsoft_phi-1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "phi-1", - "id": "microsoft/phi-1", - "developer": "microsoft", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "PhiForCausalLM", - "params_billions": "1.418" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/microsoft_phi-1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2068 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3139 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0098 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2651 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3525 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1162 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/microsoft_phi-1_5.json b/data/models/microsoft_phi-1_5.json deleted file mode 100644 index bc716b103abb9a3be1da5a62ae4c916faac24c38..0000000000000000000000000000000000000000 --- a/data/models/microsoft_phi-1_5.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "phi-1_5", - "id": "microsoft/phi-1_5", - "developer": "microsoft", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "PhiForCausalLM", - "params_billions": "1.418" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/microsoft_phi-1_5/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2033 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.336 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0181 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2676 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3404 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1691 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/microsoft_phi-2.json b/data/models/microsoft_phi-2.json deleted file mode 100644 index 95014ebf52045bda7a4a5b4267170311c3f441cd..0000000000000000000000000000000000000000 --- a/data/models/microsoft_phi-2.json +++ /dev/null @@ -1,2031 +0,0 @@ -{ - "model_info": { - "name": "Phi-2", - "id": "microsoft/phi-2", - "developer": "microsoft", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_lite/microsoft_phi-2/1774096306.427425", - "retrieved_timestamp": "1774096306.427425", - "source_metadata": { - "source_name": "helm_lite", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_lite", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_lite", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.169, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.9032709113607991\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.703, - "details": { - "description": "min=0.703, mean=0.703, max=0.703, sum=0.703 (1)", - "tab": "Accuracy", - "NarrativeQA - Observed inference time (s)": "{\"description\": \"min=0.493, mean=0.493, max=0.493, sum=0.493 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.49325697791408485\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=2.085, mean=2.085, max=2.085, sum=2.085 (1)\", \"tab\": \"General information\", \"score\": \"2.084507042253521\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=1705.006, mean=1705.006, max=1705.006, sum=1705.006 (1)\", \"tab\": \"General information\", \"score\": \"1705.0056338028169\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (closed-book)", - "source_data": { - "dataset_name": "NaturalQuestions (closed-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (closed-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.155, - "details": { - "description": "min=0.155, mean=0.155, max=0.155, sum=0.155 (1)", - "tab": "Accuracy", - "NaturalQuestions (open-book) - Observed inference time (s)": "{\"description\": \"min=0.47, mean=0.47, max=0.47, sum=0.47 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.46984758591651915\"}", - "NaturalQuestions (closed-book) - Observed inference time (s)": "{\"description\": \"min=0.292, mean=0.292, max=0.292, sum=0.292 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.29179329943656923\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.706, mean=4.706, max=4.706, sum=4.706 (1)\", \"tab\": \"General information\", \"score\": \"4.706\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.036, mean=0.036, max=0.036, sum=0.036 (1)\", \"tab\": \"General information\", \"score\": \"0.036\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1493.994, mean=1493.994, max=1493.994, sum=1493.994 (1)\", \"tab\": \"General information\", \"score\": \"1493.994\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=116.254, mean=116.254, max=116.254, sum=116.254 (1)\", \"tab\": \"General information\", \"score\": \"116.254\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "mode": "\"closedbook\"" - } - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.798, - "details": { - "description": "min=0.798, mean=0.798, max=0.798, sum=0.798 (1)", - "tab": "Accuracy", - "OpenbookQA - Observed inference time (s)": "{\"description\": \"min=0.262, mean=0.262, max=0.262, sum=0.262 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.2615062308311462\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=254.216, mean=254.216, max=254.216, sum=254.216 (1)\", \"tab\": \"General information\", \"score\": \"254.216\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "dataset": "\"openbookqa\"", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.518, - "details": { - "description": "min=0.31, mean=0.518, max=0.78, sum=2.592 (5)", - "tab": "Accuracy", - "MMLU - Observed inference time (s)": "{\"description\": \"min=0.27, mean=0.285, max=0.295, sum=1.426 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.28525047320650343\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=371.38, mean=472.274, max=624.07, sum=2361.37 (5)\", \"tab\": \"General information\", \"score\": \"472.2740350877192\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"college_chemistry\", \"computer_security\", \"econometrics\", \"us_foreign_policy\"]", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MATH", - "source_data": { - "dataset_name": "MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Equivalent (CoT) on MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.255, - "details": { - "description": "min=0.033, mean=0.255, max=0.465, sum=1.786 (7)", - "tab": "Accuracy", - "MATH - Observed inference time (s)": "{\"description\": \"min=0.923, mean=1.129, max=1.577, sum=7.902 (7)\", \"tab\": \"Efficiency\", \"score\": \"1.1288332585709453\"}", - "MATH - # eval": "{\"description\": \"min=30, mean=62.429, max=135, sum=437 (7)\", \"tab\": \"General information\", \"score\": \"62.42857142857143\"}", - "MATH - # train": "{\"description\": \"min=2.962, mean=6.916, max=8, sum=48.409 (7)\", \"tab\": \"General information\", \"score\": \"6.915558126084441\"}", - "MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MATH - # prompt tokens": "{\"description\": \"min=906.541, mean=1162.126, max=1511.442, sum=8134.881 (7)\", \"tab\": \"General information\", \"score\": \"1162.1258475895563\"}", - "MATH - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=7 (7)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"algebra\", \"counting_and_probability\", \"geometry\", \"intermediate_algebra\", \"number_theory\", \"prealgebra\", \"precalculus\"]", - "level": "\"1\"", - "use_official_examples": "\"False\"", - "use_chain_of_thought": "\"True\"" - } - } - }, - { - "evaluation_name": "GSM8K", - "source_data": { - "dataset_name": "GSM8K", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on GSM8K", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.581, - "details": { - "description": "min=0.581, mean=0.581, max=0.581, sum=0.581 (1)", - "tab": "Accuracy", - "GSM8K - Observed inference time (s)": "{\"description\": \"min=1.147, mean=1.147, max=1.147, sum=1.147 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.1468114259243012\"}", - "GSM8K - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "GSM8K - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "GSM8K - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GSM8K - # prompt tokens": "{\"description\": \"min=938.893, mean=938.893, max=938.893, sum=938.893 (1)\", \"tab\": \"General information\", \"score\": \"938.893\"}", - "GSM8K - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "LegalBench", - "source_data": { - "dataset_name": "LegalBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on LegalBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.334, - "details": { - "description": "min=0.137, mean=0.334, max=0.537, sum=1.672 (5)", - "tab": "Accuracy", - "LegalBench - Observed inference time (s)": "{\"description\": \"min=0.268, mean=0.303, max=0.381, sum=1.517 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.3034723702962031\"}", - "LegalBench - # eval": "{\"description\": \"min=95, mean=409.4, max=1000, sum=2047 (5)\", \"tab\": \"General information\", \"score\": \"409.4\"}", - "LegalBench - # train": "{\"description\": \"min=0.337, mean=3.867, max=5, sum=19.337 (5)\", \"tab\": \"General information\", \"score\": \"3.8673469387755106\"}", - "LegalBench - truncated": "{\"description\": \"min=0, mean=0.003, max=0.014, sum=0.014 (5)\", \"tab\": \"General information\", \"score\": \"0.002857142857142857\"}", - "LegalBench - # prompt tokens": "{\"description\": \"min=205.632, mean=566.249, max=1519.543, sum=2831.243 (5)\", \"tab\": \"General information\", \"score\": \"566.2485439511586\"}", - "LegalBench - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"abercrombie\", \"corporate_lobbying\", \"function_of_decision_section\", \"international_citizenship_questions\", \"proa\"]" - } - } - }, - { - "evaluation_name": "MedQA", - "source_data": { - "dataset_name": "MedQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MedQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.41, - "details": { - "description": "min=0.41, mean=0.41, max=0.41, sum=0.41 (1)", - "tab": "Accuracy", - "MedQA - Observed inference time (s)": "{\"description\": \"min=0.275, mean=0.275, max=0.275, sum=0.275 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.27509861532783886\"}", - "MedQA - # eval": "{\"description\": \"min=503, mean=503, max=503, sum=503 (1)\", \"tab\": \"General information\", \"score\": \"503.0\"}", - "MedQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MedQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MedQA - # prompt tokens": "{\"description\": \"min=1038.833, mean=1038.833, max=1038.833, sum=1038.833 (1)\", \"tab\": \"General information\", \"score\": \"1038.8330019880716\"}", - "MedQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WMT 2014", - "source_data": { - "dataset_name": "WMT 2014", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "BLEU-4 on WMT 2014", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.038, - "details": { - "description": "min=0.0, mean=0.038, max=0.113, sum=0.189 (5)", - "tab": "Accuracy", - "WMT 2014 - Observed inference time (s)": "{\"description\": \"min=0.427, mean=0.47, max=0.534, sum=2.35 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.47001117224047206\"}", - "WMT 2014 - # eval": "{\"description\": \"min=503, mean=568.8, max=832, sum=2844 (5)\", \"tab\": \"General information\", \"score\": \"568.8\"}", - "WMT 2014 - # train": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "WMT 2014 - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WMT 2014 - # prompt tokens": "{\"description\": \"min=136.93, mean=181.692, max=241.656, sum=908.462 (5)\", \"tab\": \"General information\", \"score\": \"181.69235022556967\"}", - "WMT 2014 - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "language_pair": "[\"cs-en\", \"de-en\", \"fr-en\", \"hi-en\", \"ru-en\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_mmlu/microsoft_phi-2/1774096312.00548", - "retrieved_timestamp": "1774096312.00548", - "source_metadata": { - "source_name": "helm_mmlu", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_mmlu", - "evaluation_results": [ - { - "evaluation_name": "MMLU All Subjects", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU All Subjects", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.584, - "details": { - "description": "min=0.231, mean=0.584, max=0.833, sum=66.604 (114)", - "tab": "Accuracy", - "MMLU All Subjects - Observed inference time (s)": "{\"description\": \"min=0.267, mean=0.309, max=0.409, sum=35.222 (114)\", \"tab\": \"Efficiency\", \"score\": \"0.3089648339000309\"}", - "MMLU All Subjects - # eval": "{\"description\": \"min=100, mean=246.351, max=1534, sum=28084 (114)\", \"tab\": \"General information\", \"score\": \"246.35087719298247\"}", - "MMLU All Subjects - # train": "{\"description\": \"min=2.945, mean=4.946, max=5, sum=563.886 (114)\", \"tab\": \"General information\", \"score\": \"4.946365736553069\"}", - "MMLU All Subjects - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU All Subjects - # prompt tokens": "{\"description\": \"min=277.404, mean=600.9, max=1826.103, sum=68502.623 (114)\", \"tab\": \"General information\", \"score\": \"600.9002028338741\"}", - "MMLU All Subjects - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=114 (114)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_name": "Abstract Algebra", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Abstract Algebra", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.31, - "details": { - "description": "min=0.31, mean=0.31, max=0.31, sum=0.62 (2)", - "tab": "Accuracy", - "Abstract Algebra - Observed inference time (s)": "{\"description\": \"min=0.293, mean=0.293, max=0.293, sum=0.585 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2925554180145264\"}", - "Abstract Algebra - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Abstract Algebra - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Abstract Algebra - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Abstract Algebra - # prompt tokens": "{\"description\": \"min=371.38, mean=371.38, max=371.38, sum=742.76 (2)\", \"tab\": \"General information\", \"score\": \"371.38\"}", - "Abstract Algebra - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"abstract_algebra\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_abstract_algebra\"" - } - } - }, - { - "evaluation_name": "Anatomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Anatomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.437, - "details": { - "description": "min=0.437, mean=0.437, max=0.437, sum=0.874 (2)", - "tab": "Accuracy", - "Anatomy - Observed inference time (s)": "{\"description\": \"min=0.338, mean=0.338, max=0.338, sum=0.675 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3375302138151946\"}", - "Anatomy - # eval": "{\"description\": \"min=135, mean=135, max=135, sum=270 (2)\", \"tab\": \"General information\", \"score\": \"135.0\"}", - "Anatomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Anatomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Anatomy - # prompt tokens": "{\"description\": \"min=372.081, mean=372.081, max=372.081, sum=744.163 (2)\", \"tab\": \"General information\", \"score\": \"372.0814814814815\"}", - "Anatomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"anatomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_anatomy\"" - } - } - }, - { - "evaluation_name": "College Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on College Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.382, - "details": { - "description": "min=0.382, mean=0.382, max=0.382, sum=0.765 (2)", - "tab": "Accuracy", - "College Chemistry - Observed inference time (s)": "{\"description\": \"min=0.27, mean=0.27, max=0.27, sum=0.539 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2696530842781067\"}", - "College Biology - Observed inference time (s)": "{\"description\": \"min=0.302, mean=0.302, max=0.302, sum=0.604 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3021910654173957\"}", - "College Computer Science - Observed inference time (s)": "{\"description\": \"min=0.349, mean=0.349, max=0.349, sum=0.697 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.34874132156372073\"}", - "College Mathematics - Observed inference time (s)": "{\"description\": \"min=0.319, mean=0.319, max=0.319, sum=0.638 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3188008284568787\"}", - "College Medicine - Observed inference time (s)": "{\"description\": \"min=0.304, mean=0.304, max=0.304, sum=0.607 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.30374339412402557\"}", - "College Physics - Observed inference time (s)": "{\"description\": \"min=0.32, mean=0.32, max=0.32, sum=0.64 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.31993647182688995\"}", - "College Chemistry - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Chemistry - # prompt tokens": "{\"description\": \"min=545.4, mean=545.4, max=545.4, sum=1090.8 (2)\", \"tab\": \"General information\", \"score\": \"545.4\"}", - "College Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Biology - # eval": "{\"description\": \"min=144, mean=144, max=144, sum=288 (2)\", \"tab\": \"General information\", \"score\": \"144.0\"}", - "College Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # prompt tokens": "{\"description\": \"min=482.278, mean=482.278, max=482.278, sum=964.556 (2)\", \"tab\": \"General information\", \"score\": \"482.27777777777777\"}", - "College Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # prompt tokens": "{\"description\": \"min=852.03, mean=852.03, max=852.03, sum=1704.06 (2)\", \"tab\": \"General information\", \"score\": \"852.03\"}", - "College Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Mathematics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # prompt tokens": "{\"description\": \"min=611.54, mean=611.54, max=611.54, sum=1223.08 (2)\", \"tab\": \"General information\", \"score\": \"611.54\"}", - "College Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Medicine - # eval": "{\"description\": \"min=173, mean=173, max=173, sum=346 (2)\", \"tab\": \"General information\", \"score\": \"173.0\"}", - "College Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # prompt tokens": "{\"description\": \"min=530.301, mean=530.301, max=530.301, sum=1060.601 (2)\", \"tab\": \"General information\", \"score\": \"530.3005780346821\"}", - "College Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Physics - # eval": "{\"description\": \"min=102, mean=102, max=102, sum=204 (2)\", \"tab\": \"General information\", \"score\": \"102.0\"}", - "College Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # prompt tokens": "{\"description\": \"min=489.324, mean=489.324, max=489.324, sum=978.647 (2)\", \"tab\": \"General information\", \"score\": \"489.3235294117647\"}", - "College Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"college_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_college_physics\"" - } - } - }, - { - "evaluation_name": "Computer Security", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Computer Security", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.73, - "details": { - "description": "min=0.73, mean=0.73, max=0.73, sum=1.46 (2)", - "tab": "Accuracy", - "Computer Security - Observed inference time (s)": "{\"description\": \"min=0.277, mean=0.277, max=0.277, sum=0.554 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2771985101699829\"}", - "Computer Security - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Computer Security - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Computer Security - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Computer Security - # prompt tokens": "{\"description\": \"min=387.4, mean=387.4, max=387.4, sum=774.8 (2)\", \"tab\": \"General information\", \"score\": \"387.4\"}", - "Computer Security - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"computer_security\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_computer_security\"" - } - } - }, - { - "evaluation_name": "Econometrics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Econometrics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.342, - "details": { - "description": "min=0.342, mean=0.342, max=0.342, sum=0.684 (2)", - "tab": "Accuracy", - "Econometrics - Observed inference time (s)": "{\"description\": \"min=0.295, mean=0.295, max=0.295, sum=0.589 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.294714699711716\"}", - "Econometrics - # eval": "{\"description\": \"min=114, mean=114, max=114, sum=228 (2)\", \"tab\": \"General information\", \"score\": \"114.0\"}", - "Econometrics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Econometrics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Econometrics - # prompt tokens": "{\"description\": \"min=624.07, mean=624.07, max=624.07, sum=1248.14 (2)\", \"tab\": \"General information\", \"score\": \"624.0701754385965\"}", - "Econometrics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"econometrics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_econometrics\"" - } - } - }, - { - "evaluation_name": "Global Facts", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Global Facts", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.35, - "details": { - "description": "min=0.35, mean=0.35, max=0.35, sum=0.7 (2)", - "tab": "Accuracy", - "Global Facts - Observed inference time (s)": "{\"description\": \"min=0.315, mean=0.315, max=0.315, sum=0.631 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3154014134407043\"}", - "Global Facts - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Global Facts - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Global Facts - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Global Facts - # prompt tokens": "{\"description\": \"min=398.42, mean=398.42, max=398.42, sum=796.84 (2)\", \"tab\": \"General information\", \"score\": \"398.42\"}", - "Global Facts - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"global_facts\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_global_facts\"" - } - } - }, - { - "evaluation_name": "Jurisprudence", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Jurisprudence", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.694, - "details": { - "description": "min=0.694, mean=0.694, max=0.694, sum=1.389 (2)", - "tab": "Accuracy", - "Jurisprudence - Observed inference time (s)": "{\"description\": \"min=0.281, mean=0.281, max=0.281, sum=0.562 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.28103237681918675\"}", - "Jurisprudence - # eval": "{\"description\": \"min=108, mean=108, max=108, sum=216 (2)\", \"tab\": \"General information\", \"score\": \"108.0\"}", - "Jurisprudence - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Jurisprudence - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Jurisprudence - # prompt tokens": "{\"description\": \"min=418.722, mean=418.722, max=418.722, sum=837.444 (2)\", \"tab\": \"General information\", \"score\": \"418.72222222222223\"}", - "Jurisprudence - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"jurisprudence\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_jurisprudence\"" - } - } - }, - { - "evaluation_name": "Philosophy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Philosophy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.598, - "details": { - "description": "min=0.598, mean=0.598, max=0.598, sum=1.196 (2)", - "tab": "Accuracy", - "Philosophy - Observed inference time (s)": "{\"description\": \"min=0.298, mean=0.298, max=0.298, sum=0.597 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.29847138410979146\"}", - "Philosophy - # eval": "{\"description\": \"min=311, mean=311, max=311, sum=622 (2)\", \"tab\": \"General information\", \"score\": \"311.0\"}", - "Philosophy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Philosophy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Philosophy - # prompt tokens": "{\"description\": \"min=353.711, mean=353.711, max=353.711, sum=707.421 (2)\", \"tab\": \"General information\", \"score\": \"353.7106109324759\"}", - "Philosophy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"philosophy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_philosophy\"" - } - } - }, - { - "evaluation_name": "Professional Psychology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Professional Psychology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.572, - "details": { - "description": "min=0.572, mean=0.572, max=0.572, sum=1.144 (2)", - "tab": "Accuracy", - "Professional Medicine - Observed inference time (s)": "{\"description\": \"min=0.305, mean=0.305, max=0.305, sum=0.61 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3051472201066859\"}", - "Professional Accounting - Observed inference time (s)": "{\"description\": \"min=0.31, mean=0.31, max=0.31, sum=0.619 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3096669819338102\"}", - "Professional Law - Observed inference time (s)": "{\"description\": \"min=0.363, mean=0.363, max=0.363, sum=0.727 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.36331592731401224\"}", - "Professional Psychology - Observed inference time (s)": "{\"description\": \"min=0.307, mean=0.307, max=0.307, sum=0.614 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.30723563518399505\"}", - "Professional Medicine - # eval": "{\"description\": \"min=272, mean=272, max=272, sum=544 (2)\", \"tab\": \"General information\", \"score\": \"272.0\"}", - "Professional Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Medicine - # prompt tokens": "{\"description\": \"min=1118.287, mean=1118.287, max=1118.287, sum=2236.574 (2)\", \"tab\": \"General information\", \"score\": \"1118.2867647058824\"}", - "Professional Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Accounting - # eval": "{\"description\": \"min=282, mean=282, max=282, sum=564 (2)\", \"tab\": \"General information\", \"score\": \"282.0\"}", - "Professional Accounting - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Accounting - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # prompt tokens": "{\"description\": \"min=660.72, mean=660.72, max=660.72, sum=1321.44 (2)\", \"tab\": \"General information\", \"score\": \"660.7198581560284\"}", - "Professional Accounting - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Law - # eval": "{\"description\": \"min=1534, mean=1534, max=1534, sum=3068 (2)\", \"tab\": \"General information\", \"score\": \"1534.0\"}", - "Professional Law - # train": "{\"description\": \"min=4.997, mean=4.997, max=4.997, sum=9.995 (2)\", \"tab\": \"General information\", \"score\": \"4.9973924380704045\"}", - "Professional Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # prompt tokens": "{\"description\": \"min=1657.596, mean=1657.596, max=1657.596, sum=3315.192 (2)\", \"tab\": \"General information\", \"score\": \"1657.5958279009126\"}", - "Professional Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Psychology - # eval": "{\"description\": \"min=612, mean=612, max=612, sum=1224 (2)\", \"tab\": \"General information\", \"score\": \"612.0\"}", - "Professional Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # prompt tokens": "{\"description\": \"min=597.574, mean=597.574, max=597.574, sum=1195.147 (2)\", \"tab\": \"General information\", \"score\": \"597.5735294117648\"}", - "Professional Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"professional_psychology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_professional_psychology\"" - } - } - }, - { - "evaluation_name": "Us Foreign Policy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Us Foreign Policy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.78, - "details": { - "description": "min=0.78, mean=0.78, max=0.78, sum=1.56 (2)", - "tab": "Accuracy", - "Us Foreign Policy - Observed inference time (s)": "{\"description\": \"min=0.292, mean=0.292, max=0.292, sum=0.584 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2921306538581848\"}", - "Us Foreign Policy - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Us Foreign Policy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Us Foreign Policy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Us Foreign Policy - # prompt tokens": "{\"description\": \"min=433.12, mean=433.12, max=433.12, sum=866.24 (2)\", \"tab\": \"General information\", \"score\": \"433.12\"}", - "Us Foreign Policy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"us_foreign_policy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_us_foreign_policy\"" - } - } - }, - { - "evaluation_name": "Astronomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Astronomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.605, - "details": { - "description": "min=0.605, mean=0.605, max=0.605, sum=1.211 (2)", - "tab": "Accuracy", - "Astronomy - Observed inference time (s)": "{\"description\": \"min=0.297, mean=0.297, max=0.297, sum=0.594 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2971143110802299\"}", - "Astronomy - # eval": "{\"description\": \"min=152, mean=152, max=152, sum=304 (2)\", \"tab\": \"General information\", \"score\": \"152.0\"}", - "Astronomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Astronomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Astronomy - # prompt tokens": "{\"description\": \"min=600.112, mean=600.112, max=600.112, sum=1200.224 (2)\", \"tab\": \"General information\", \"score\": \"600.1118421052631\"}", - "Astronomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"astronomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_astronomy\"" - } - } - }, - { - "evaluation_name": "Business Ethics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Business Ethics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.59, - "details": { - "description": "min=0.59, mean=0.59, max=0.59, sum=1.18 (2)", - "tab": "Accuracy", - "Business Ethics - Observed inference time (s)": "{\"description\": \"min=0.333, mean=0.333, max=0.333, sum=0.666 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.33283984184265136\"}", - "Business Ethics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Business Ethics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Business Ethics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Business Ethics - # prompt tokens": "{\"description\": \"min=589.43, mean=589.43, max=589.43, sum=1178.86 (2)\", \"tab\": \"General information\", \"score\": \"589.43\"}", - "Business Ethics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"business_ethics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_business_ethics\"" - } - } - }, - { - "evaluation_name": "Clinical Knowledge", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Clinical Knowledge", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.619, - "details": { - "description": "min=0.619, mean=0.619, max=0.619, sum=1.238 (2)", - "tab": "Accuracy", - "Clinical Knowledge - Observed inference time (s)": "{\"description\": \"min=0.304, mean=0.304, max=0.304, sum=0.608 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3039509620306627\"}", - "Clinical Knowledge - # eval": "{\"description\": \"min=265, mean=265, max=265, sum=530 (2)\", \"tab\": \"General information\", \"score\": \"265.0\"}", - "Clinical Knowledge - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Clinical Knowledge - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Clinical Knowledge - # prompt tokens": "{\"description\": \"min=423.925, mean=423.925, max=423.925, sum=847.849 (2)\", \"tab\": \"General information\", \"score\": \"423.92452830188677\"}", - "Clinical Knowledge - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"clinical_knowledge\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_clinical_knowledge\"" - } - } - }, - { - "evaluation_name": "Conceptual Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Conceptual Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.519, - "details": { - "description": "min=0.519, mean=0.519, max=0.519, sum=1.038 (2)", - "tab": "Accuracy", - "Conceptual Physics - Observed inference time (s)": "{\"description\": \"min=0.309, mean=0.309, max=0.309, sum=0.618 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.30905701251740153\"}", - "Conceptual Physics - # eval": "{\"description\": \"min=235, mean=235, max=235, sum=470 (2)\", \"tab\": \"General information\", \"score\": \"235.0\"}", - "Conceptual Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Conceptual Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Conceptual Physics - # prompt tokens": "{\"description\": \"min=313.723, mean=313.723, max=313.723, sum=627.447 (2)\", \"tab\": \"General information\", \"score\": \"313.72340425531917\"}", - "Conceptual Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"conceptual_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_conceptual_physics\"" - } - } - }, - { - "evaluation_name": "Electrical Engineering", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Electrical Engineering", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.545, - "details": { - "description": "min=0.545, mean=0.545, max=0.545, sum=1.09 (2)", - "tab": "Accuracy", - "Electrical Engineering - Observed inference time (s)": "{\"description\": \"min=0.319, mean=0.319, max=0.319, sum=0.639 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.31939958375075767\"}", - "Electrical Engineering - # eval": "{\"description\": \"min=145, mean=145, max=145, sum=290 (2)\", \"tab\": \"General information\", \"score\": \"145.0\"}", - "Electrical Engineering - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Electrical Engineering - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Electrical Engineering - # prompt tokens": "{\"description\": \"min=430.345, mean=430.345, max=430.345, sum=860.69 (2)\", \"tab\": \"General information\", \"score\": \"430.3448275862069\"}", - "Electrical Engineering - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"electrical_engineering\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_electrical_engineering\"" - } - } - }, - { - "evaluation_name": "Elementary Mathematics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Elementary Mathematics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.463, - "details": { - "description": "min=0.463, mean=0.463, max=0.463, sum=0.926 (2)", - "tab": "Accuracy", - "Elementary Mathematics - Observed inference time (s)": "{\"description\": \"min=0.304, mean=0.304, max=0.304, sum=0.607 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.30370362284322266\"}", - "Elementary Mathematics - # eval": "{\"description\": \"min=378, mean=378, max=378, sum=756 (2)\", \"tab\": \"General information\", \"score\": \"378.0\"}", - "Elementary Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Elementary Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Elementary Mathematics - # prompt tokens": "{\"description\": \"min=506.09, mean=506.09, max=506.09, sum=1012.18 (2)\", \"tab\": \"General information\", \"score\": \"506.0899470899471\"}", - "Elementary Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"elementary_mathematics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_elementary_mathematics\"" - } - } - }, - { - "evaluation_name": "Formal Logic", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Formal Logic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.389, - "details": { - "description": "min=0.389, mean=0.389, max=0.389, sum=0.778 (2)", - "tab": "Accuracy", - "Formal Logic - Observed inference time (s)": "{\"description\": \"min=0.321, mean=0.321, max=0.321, sum=0.642 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3209871034773569\"}", - "Formal Logic - # eval": "{\"description\": \"min=126, mean=126, max=126, sum=252 (2)\", \"tab\": \"General information\", \"score\": \"126.0\"}", - "Formal Logic - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Formal Logic - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Formal Logic - # prompt tokens": "{\"description\": \"min=641, mean=641, max=641, sum=1282 (2)\", \"tab\": \"General information\", \"score\": \"641.0\"}", - "Formal Logic - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"formal_logic\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_formal_logic\"" - } - } - }, - { - "evaluation_name": "High School World History", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on High School World History", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.73, - "details": { - "description": "min=0.73, mean=0.73, max=0.73, sum=1.46 (2)", - "tab": "Accuracy", - "High School Biology - Observed inference time (s)": "{\"description\": \"min=0.279, mean=0.279, max=0.279, sum=0.557 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2785434192226779\"}", - "High School Chemistry - Observed inference time (s)": "{\"description\": \"min=0.308, mean=0.308, max=0.308, sum=0.616 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3082333773814986\"}", - "High School Computer Science - Observed inference time (s)": "{\"description\": \"min=0.327, mean=0.327, max=0.327, sum=0.654 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3267984962463379\"}", - "High School European History - Observed inference time (s)": "{\"description\": \"min=0.409, mean=0.409, max=0.409, sum=0.819 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.40945722406560725\"}", - "High School Geography - Observed inference time (s)": "{\"description\": \"min=0.305, mean=0.305, max=0.305, sum=0.61 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.30513872763123173\"}", - "High School Government And Politics - Observed inference time (s)": "{\"description\": \"min=0.28, mean=0.28, max=0.28, sum=0.56 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2802187642902908\"}", - "High School Macroeconomics - Observed inference time (s)": "{\"description\": \"min=0.391, mean=0.391, max=0.391, sum=0.782 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3909576538281563\"}", - "High School Mathematics - Observed inference time (s)": "{\"description\": \"min=0.304, mean=0.304, max=0.304, sum=0.608 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.30405007821542246\"}", - "High School Microeconomics - Observed inference time (s)": "{\"description\": \"min=0.274, mean=0.274, max=0.274, sum=0.548 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2737702652185905\"}", - "High School Physics - Observed inference time (s)": "{\"description\": \"min=0.303, mean=0.303, max=0.303, sum=0.605 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.30272982452089425\"}", - "High School Psychology - Observed inference time (s)": "{\"description\": \"min=0.305, mean=0.305, max=0.305, sum=0.609 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.30458581688207226\"}", - "High School Statistics - Observed inference time (s)": "{\"description\": \"min=0.314, mean=0.314, max=0.314, sum=0.629 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3143394479045161\"}", - "High School US History - Observed inference time (s)": "{\"description\": \"min=0.38, mean=0.38, max=0.38, sum=0.759 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.37960049802181767\"}", - "High School World History - Observed inference time (s)": "{\"description\": \"min=0.365, mean=0.365, max=0.365, sum=0.729 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.36470460791125076\"}", - "High School Biology - # eval": "{\"description\": \"min=310, mean=310, max=310, sum=620 (2)\", \"tab\": \"General information\", \"score\": \"310.0\"}", - "High School Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Biology - # prompt tokens": "{\"description\": \"min=540.748, mean=540.748, max=540.748, sum=1081.497 (2)\", \"tab\": \"General information\", \"score\": \"540.7483870967742\"}", - "High School Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Chemistry - # eval": "{\"description\": \"min=203, mean=203, max=203, sum=406 (2)\", \"tab\": \"General information\", \"score\": \"203.0\"}", - "High School Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # prompt tokens": "{\"description\": \"min=495.645, mean=495.645, max=495.645, sum=991.291 (2)\", \"tab\": \"General information\", \"score\": \"495.6453201970443\"}", - "High School Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "High School Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # prompt tokens": "{\"description\": \"min=894.78, mean=894.78, max=894.78, sum=1789.56 (2)\", \"tab\": \"General information\", \"score\": \"894.78\"}", - "High School Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School European History - # eval": "{\"description\": \"min=165, mean=165, max=165, sum=330 (2)\", \"tab\": \"General information\", \"score\": \"165.0\"}", - "High School European History - # train": "{\"description\": \"min=2.945, mean=2.945, max=2.945, sum=5.891 (2)\", \"tab\": \"General information\", \"score\": \"2.9454545454545453\"}", - "High School European History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # prompt tokens": "{\"description\": \"min=1826.103, mean=1826.103, max=1826.103, sum=3652.206 (2)\", \"tab\": \"General information\", \"score\": \"1826.1030303030302\"}", - "High School European History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Geography - # eval": "{\"description\": \"min=198, mean=198, max=198, sum=396 (2)\", \"tab\": \"General information\", \"score\": \"198.0\"}", - "High School Geography - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Geography - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # prompt tokens": "{\"description\": \"min=397.646, mean=397.646, max=397.646, sum=795.293 (2)\", \"tab\": \"General information\", \"score\": \"397.64646464646466\"}", - "High School Geography - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Government And Politics - # eval": "{\"description\": \"min=193, mean=193, max=193, sum=386 (2)\", \"tab\": \"General information\", \"score\": \"193.0\"}", - "High School Government And Politics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Government And Politics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # prompt tokens": "{\"description\": \"min=478.073, mean=478.073, max=478.073, sum=956.145 (2)\", \"tab\": \"General information\", \"score\": \"478.07253886010363\"}", - "High School Government And Politics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Macroeconomics - # eval": "{\"description\": \"min=390, mean=390, max=390, sum=780 (2)\", \"tab\": \"General information\", \"score\": \"390.0\"}", - "High School Macroeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Macroeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # prompt tokens": "{\"description\": \"min=391.931, mean=391.931, max=391.931, sum=783.862 (2)\", \"tab\": \"General information\", \"score\": \"391.9307692307692\"}", - "High School Macroeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Mathematics - # eval": "{\"description\": \"min=270, mean=270, max=270, sum=540 (2)\", \"tab\": \"General information\", \"score\": \"270.0\"}", - "High School Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # prompt tokens": "{\"description\": \"min=526.352, mean=526.352, max=526.352, sum=1052.704 (2)\", \"tab\": \"General information\", \"score\": \"526.3518518518518\"}", - "High School Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Microeconomics - # eval": "{\"description\": \"min=238, mean=238, max=238, sum=476 (2)\", \"tab\": \"General information\", \"score\": \"238.0\"}", - "High School Microeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Microeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # prompt tokens": "{\"description\": \"min=410.937, mean=410.937, max=410.937, sum=821.874 (2)\", \"tab\": \"General information\", \"score\": \"410.93697478991595\"}", - "High School Microeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Physics - # eval": "{\"description\": \"min=151, mean=151, max=151, sum=302 (2)\", \"tab\": \"General information\", \"score\": \"151.0\"}", - "High School Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # prompt tokens": "{\"description\": \"min=553.669, mean=553.669, max=553.669, sum=1107.338 (2)\", \"tab\": \"General information\", \"score\": \"553.6688741721854\"}", - "High School Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Psychology - # eval": "{\"description\": \"min=545, mean=545, max=545, sum=1090 (2)\", \"tab\": \"General information\", \"score\": \"545.0\"}", - "High School Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # prompt tokens": "{\"description\": \"min=516.842, mean=516.842, max=516.842, sum=1033.684 (2)\", \"tab\": \"General information\", \"score\": \"516.8422018348624\"}", - "High School Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Statistics - # eval": "{\"description\": \"min=216, mean=216, max=216, sum=432 (2)\", \"tab\": \"General information\", \"score\": \"216.0\"}", - "High School Statistics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Statistics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # prompt tokens": "{\"description\": \"min=805, mean=805, max=805, sum=1610 (2)\", \"tab\": \"General information\", \"score\": \"805.0\"}", - "High School Statistics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School US History - # eval": "{\"description\": \"min=204, mean=204, max=204, sum=408 (2)\", \"tab\": \"General information\", \"score\": \"204.0\"}", - "High School US History - # train": "{\"description\": \"min=4, mean=4, max=4, sum=8 (2)\", \"tab\": \"General information\", \"score\": \"4.0\"}", - "High School US History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # prompt tokens": "{\"description\": \"min=1756.25, mean=1756.25, max=1756.25, sum=3512.5 (2)\", \"tab\": \"General information\", \"score\": \"1756.25\"}", - "High School US History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School World History - # eval": "{\"description\": \"min=237, mean=237, max=237, sum=474 (2)\", \"tab\": \"General information\", \"score\": \"237.0\"}", - "High School World History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School World History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # prompt tokens": "{\"description\": \"min=1438.561, mean=1438.561, max=1438.561, sum=2877.122 (2)\", \"tab\": \"General information\", \"score\": \"1438.5611814345991\"}", - "High School World History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"high_school_world_history\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_high_school_world_history\"" - } - } - }, - { - "evaluation_name": "Human Sexuality", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Human Sexuality", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.733, - "details": { - "description": "min=0.733, mean=0.733, max=0.733, sum=1.466 (2)", - "tab": "Accuracy", - "Human Aging - Observed inference time (s)": "{\"description\": \"min=0.298, mean=0.298, max=0.298, sum=0.596 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2979412987627791\"}", - "Human Sexuality - Observed inference time (s)": "{\"description\": \"min=0.303, mean=0.303, max=0.303, sum=0.605 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.30250649051811856\"}", - "Human Aging - # eval": "{\"description\": \"min=223, mean=223, max=223, sum=446 (2)\", \"tab\": \"General information\", \"score\": \"223.0\"}", - "Human Aging - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Aging - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Aging - # prompt tokens": "{\"description\": \"min=324.48, mean=324.48, max=324.48, sum=648.96 (2)\", \"tab\": \"General information\", \"score\": \"324.47982062780267\"}", - "Human Aging - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Human Sexuality - # eval": "{\"description\": \"min=131, mean=131, max=131, sum=262 (2)\", \"tab\": \"General information\", \"score\": \"131.0\"}", - "Human Sexuality - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Sexuality - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # prompt tokens": "{\"description\": \"min=357.626, mean=357.626, max=357.626, sum=715.252 (2)\", \"tab\": \"General information\", \"score\": \"357.62595419847327\"}", - "Human Sexuality - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"human_sexuality\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_human_sexuality\"" - } - } - }, - { - "evaluation_name": "International Law", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on International Law", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.752, - "details": { - "description": "min=0.752, mean=0.752, max=0.752, sum=1.504 (2)", - "tab": "Accuracy", - "International Law - Observed inference time (s)": "{\"description\": \"min=0.307, mean=0.307, max=0.307, sum=0.614 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.30694435647696505\"}", - "International Law - # eval": "{\"description\": \"min=121, mean=121, max=121, sum=242 (2)\", \"tab\": \"General information\", \"score\": \"121.0\"}", - "International Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "International Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "International Law - # prompt tokens": "{\"description\": \"min=639.843, mean=639.843, max=639.843, sum=1279.686 (2)\", \"tab\": \"General information\", \"score\": \"639.8429752066115\"}", - "International Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"international_law\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_international_law\"" - } - } - }, - { - "evaluation_name": "Logical Fallacies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Logical Fallacies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.767, - "details": { - "description": "min=0.767, mean=0.767, max=0.767, sum=1.534 (2)", - "tab": "Accuracy", - "Logical Fallacies - Observed inference time (s)": "{\"description\": \"min=0.274, mean=0.274, max=0.274, sum=0.548 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.273789843167264\"}", - "Logical Fallacies - # eval": "{\"description\": \"min=163, mean=163, max=163, sum=326 (2)\", \"tab\": \"General information\", \"score\": \"163.0\"}", - "Logical Fallacies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Logical Fallacies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Logical Fallacies - # prompt tokens": "{\"description\": \"min=454.233, mean=454.233, max=454.233, sum=908.466 (2)\", \"tab\": \"General information\", \"score\": \"454.23312883435585\"}", - "Logical Fallacies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"logical_fallacies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_logical_fallacies\"" - } - } - }, - { - "evaluation_name": "Machine Learning", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Machine Learning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5, - "details": { - "description": "min=0.5, mean=0.5, max=0.5, sum=1 (2)", - "tab": "Accuracy", - "Machine Learning - Observed inference time (s)": "{\"description\": \"min=0.313, mean=0.313, max=0.313, sum=0.627 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.31332691439560484\"}", - "Machine Learning - # eval": "{\"description\": \"min=112, mean=112, max=112, sum=224 (2)\", \"tab\": \"General information\", \"score\": \"112.0\"}", - "Machine Learning - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Machine Learning - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Machine Learning - # prompt tokens": "{\"description\": \"min=671.598, mean=671.598, max=671.598, sum=1343.196 (2)\", \"tab\": \"General information\", \"score\": \"671.5982142857143\"}", - "Machine Learning - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"machine_learning\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_machine_learning\"" - } - } - }, - { - "evaluation_name": "Management", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Management", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.748, - "details": { - "description": "min=0.748, mean=0.748, max=0.748, sum=1.495 (2)", - "tab": "Accuracy", - "Management - Observed inference time (s)": "{\"description\": \"min=0.305, mean=0.305, max=0.305, sum=0.61 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3051937992132983\"}", - "Management - # eval": "{\"description\": \"min=103, mean=103, max=103, sum=206 (2)\", \"tab\": \"General information\", \"score\": \"103.0\"}", - "Management - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Management - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Management - # prompt tokens": "{\"description\": \"min=292.34, mean=292.34, max=292.34, sum=584.68 (2)\", \"tab\": \"General information\", \"score\": \"292.3398058252427\"}", - "Management - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"management\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_management\"" - } - } - }, - { - "evaluation_name": "Marketing", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Marketing", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.833, - "details": { - "description": "min=0.833, mean=0.833, max=0.833, sum=1.667 (2)", - "tab": "Accuracy", - "Marketing - Observed inference time (s)": "{\"description\": \"min=0.276, mean=0.276, max=0.276, sum=0.552 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2761631949335082\"}", - "Marketing - # eval": "{\"description\": \"min=234, mean=234, max=234, sum=468 (2)\", \"tab\": \"General information\", \"score\": \"234.0\"}", - "Marketing - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Marketing - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Marketing - # prompt tokens": "{\"description\": \"min=437.667, mean=437.667, max=437.667, sum=875.333 (2)\", \"tab\": \"General information\", \"score\": \"437.6666666666667\"}", - "Marketing - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"marketing\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_marketing\"" - } - } - }, - { - "evaluation_name": "Medical Genetics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Medical Genetics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.62, - "details": { - "description": "min=0.62, mean=0.62, max=0.62, sum=1.24 (2)", - "tab": "Accuracy", - "Medical Genetics - Observed inference time (s)": "{\"description\": \"min=0.305, mean=0.305, max=0.305, sum=0.609 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3045226716995239\"}", - "Medical Genetics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Medical Genetics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Medical Genetics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Medical Genetics - # prompt tokens": "{\"description\": \"min=352.71, mean=352.71, max=352.71, sum=705.42 (2)\", \"tab\": \"General information\", \"score\": \"352.71\"}", - "Medical Genetics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"medical_genetics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_medical_genetics\"" - } - } - }, - { - "evaluation_name": "Miscellaneous", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Miscellaneous", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.688, - "details": { - "description": "min=0.688, mean=0.688, max=0.688, sum=1.377 (2)", - "tab": "Accuracy", - "Miscellaneous - Observed inference time (s)": "{\"description\": \"min=0.334, mean=0.334, max=0.334, sum=0.668 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.33387171049836645\"}", - "Miscellaneous - # eval": "{\"description\": \"min=783, mean=783, max=783, sum=1566 (2)\", \"tab\": \"General information\", \"score\": \"783.0\"}", - "Miscellaneous - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Miscellaneous - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Miscellaneous - # prompt tokens": "{\"description\": \"min=314.847, mean=314.847, max=314.847, sum=629.693 (2)\", \"tab\": \"General information\", \"score\": \"314.84674329501917\"}", - "Miscellaneous - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"miscellaneous\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_miscellaneous\"" - } - } - }, - { - "evaluation_name": "Moral Scenarios", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Moral Scenarios", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.231, - "details": { - "description": "min=0.231, mean=0.231, max=0.231, sum=0.463 (2)", - "tab": "Accuracy", - "Moral Disputes - Observed inference time (s)": "{\"description\": \"min=0.303, mean=0.303, max=0.303, sum=0.607 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3032567480395984\"}", - "Moral Scenarios - Observed inference time (s)": "{\"description\": \"min=0.267, mean=0.267, max=0.267, sum=0.534 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.26702385215119945\"}", - "Moral Disputes - # eval": "{\"description\": \"min=346, mean=346, max=346, sum=692 (2)\", \"tab\": \"General information\", \"score\": \"346.0\"}", - "Moral Disputes - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Disputes - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Disputes - # prompt tokens": "{\"description\": \"min=497.514, mean=497.514, max=497.514, sum=995.029 (2)\", \"tab\": \"General information\", \"score\": \"497.514450867052\"}", - "Moral Disputes - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Moral Scenarios - # eval": "{\"description\": \"min=895, mean=895, max=895, sum=1790 (2)\", \"tab\": \"General information\", \"score\": \"895.0\"}", - "Moral Scenarios - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Scenarios - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # prompt tokens": "{\"description\": \"min=664.479, mean=664.479, max=664.479, sum=1328.959 (2)\", \"tab\": \"General information\", \"score\": \"664.4793296089385\"}", - "Moral Scenarios - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"moral_scenarios\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_moral_scenarios\"" - } - } - }, - { - "evaluation_name": "Nutrition", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Nutrition", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.627, - "details": { - "description": "min=0.627, mean=0.627, max=0.627, sum=1.255 (2)", - "tab": "Accuracy", - "Nutrition - Observed inference time (s)": "{\"description\": \"min=0.311, mean=0.311, max=0.311, sum=0.622 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3112297058105469\"}", - "Nutrition - # eval": "{\"description\": \"min=306, mean=306, max=306, sum=612 (2)\", \"tab\": \"General information\", \"score\": \"306.0\"}", - "Nutrition - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Nutrition - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Nutrition - # prompt tokens": "{\"description\": \"min=584.69, mean=584.69, max=584.69, sum=1169.379 (2)\", \"tab\": \"General information\", \"score\": \"584.6895424836601\"}", - "Nutrition - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"nutrition\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_nutrition\"" - } - } - }, - { - "evaluation_name": "Prehistory", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Prehistory", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.605, - "details": { - "description": "min=0.605, mean=0.605, max=0.605, sum=1.21 (2)", - "tab": "Accuracy", - "Prehistory - Observed inference time (s)": "{\"description\": \"min=0.291, mean=0.291, max=0.291, sum=0.583 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.29145334090715574\"}", - "Prehistory - # eval": "{\"description\": \"min=324, mean=324, max=324, sum=648 (2)\", \"tab\": \"General information\", \"score\": \"324.0\"}", - "Prehistory - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Prehistory - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Prehistory - # prompt tokens": "{\"description\": \"min=524.454, mean=524.454, max=524.454, sum=1048.907 (2)\", \"tab\": \"General information\", \"score\": \"524.4537037037037\"}", - "Prehistory - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"prehistory\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_prehistory\"" - } - } - }, - { - "evaluation_name": "Public Relations", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Public Relations", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.673, - "details": { - "description": "min=0.673, mean=0.673, max=0.673, sum=1.345 (2)", - "tab": "Accuracy", - "Public Relations - Observed inference time (s)": "{\"description\": \"min=0.282, mean=0.282, max=0.282, sum=0.564 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.28212652423165063\"}", - "Public Relations - # eval": "{\"description\": \"min=110, mean=110, max=110, sum=220 (2)\", \"tab\": \"General information\", \"score\": \"110.0\"}", - "Public Relations - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Public Relations - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Public Relations - # prompt tokens": "{\"description\": \"min=420.609, mean=420.609, max=420.609, sum=841.218 (2)\", \"tab\": \"General information\", \"score\": \"420.6090909090909\"}", - "Public Relations - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"public_relations\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_public_relations\"" - } - } - }, - { - "evaluation_name": "Security Studies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Security Studies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.702, - "details": { - "description": "min=0.702, mean=0.702, max=0.702, sum=1.404 (2)", - "tab": "Accuracy", - "Security Studies - Observed inference time (s)": "{\"description\": \"min=0.322, mean=0.322, max=0.322, sum=0.645 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3223595599738919\"}", - "Security Studies - # eval": "{\"description\": \"min=245, mean=245, max=245, sum=490 (2)\", \"tab\": \"General information\", \"score\": \"245.0\"}", - "Security Studies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Security Studies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Security Studies - # prompt tokens": "{\"description\": \"min=1196.433, mean=1196.433, max=1196.433, sum=2392.865 (2)\", \"tab\": \"General information\", \"score\": \"1196.4326530612245\"}", - "Security Studies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"security_studies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_security_studies\"" - } - } - }, - { - "evaluation_name": "Sociology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Sociology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.816, - "details": { - "description": "min=0.816, mean=0.816, max=0.816, sum=1.632 (2)", - "tab": "Accuracy", - "Sociology - Observed inference time (s)": "{\"description\": \"min=0.288, mean=0.288, max=0.288, sum=0.575 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2876073993853669\"}", - "Sociology - # eval": "{\"description\": \"min=201, mean=201, max=201, sum=402 (2)\", \"tab\": \"General information\", \"score\": \"201.0\"}", - "Sociology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Sociology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Sociology - # prompt tokens": "{\"description\": \"min=446.512, mean=446.512, max=446.512, sum=893.025 (2)\", \"tab\": \"General information\", \"score\": \"446.5124378109453\"}", - "Sociology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"sociology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_sociology\"" - } - } - }, - { - "evaluation_name": "Virology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Virology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.47, - "details": { - "description": "min=0.47, mean=0.47, max=0.47, sum=0.94 (2)", - "tab": "Accuracy", - "Virology - Observed inference time (s)": "{\"description\": \"min=0.28, mean=0.28, max=0.28, sum=0.559 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.27966123316661423\"}", - "Virology - # eval": "{\"description\": \"min=166, mean=166, max=166, sum=332 (2)\", \"tab\": \"General information\", \"score\": \"166.0\"}", - "Virology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Virology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Virology - # prompt tokens": "{\"description\": \"min=352.759, mean=352.759, max=352.759, sum=705.518 (2)\", \"tab\": \"General information\", \"score\": \"352.7590361445783\"}", - "Virology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"virology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_virology\"" - } - } - }, - { - "evaluation_name": "World Religions", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on World Religions", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.702, - "details": { - "description": "min=0.702, mean=0.702, max=0.702, sum=1.404 (2)", - "tab": "Accuracy", - "World Religions - Observed inference time (s)": "{\"description\": \"min=0.284, mean=0.284, max=0.284, sum=0.569 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2843696499428554\"}", - "World Religions - # eval": "{\"description\": \"min=171, mean=171, max=171, sum=342 (2)\", \"tab\": \"General information\", \"score\": \"171.0\"}", - "World Religions - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "World Religions - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "World Religions - # prompt tokens": "{\"description\": \"min=277.404, mean=277.404, max=277.404, sum=554.807 (2)\", \"tab\": \"General information\", \"score\": \"277.4035087719298\"}", - "World Religions - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"world_religions\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_world_religions\"" - } - } - }, - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.824, - "details": { - "description": "", - "tab": "Efficiency" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_id": "hfopenllm_v2/microsoft_phi-2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2739 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4881 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0295 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2718 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4099 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2628 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/microsoft_phi-3-medium-128k-instruct.json b/data/models/microsoft_phi-3-medium-128k-instruct.json deleted file mode 100644 index b64f1955881fba8e6ca4e402915976091264a4d8..0000000000000000000000000000000000000000 --- a/data/models/microsoft_phi-3-medium-128k-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Phi-3-medium-128k-instruct", - "id": "microsoft/Phi-3-medium-128k-instruct", - "developer": "microsoft", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Phi3ForCausalLM", - "params_billions": "13.96" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/microsoft_Phi-3-medium-128k-instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.604 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6382 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1918 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3364 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4129 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4712 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/microsoft_phi-3-medium-4k-instruct.json b/data/models/microsoft_phi-3-medium-4k-instruct.json deleted file mode 100644 index b36a82c77fcdcea2ed7e2a41c70ff25d2fb6666b..0000000000000000000000000000000000000000 --- a/data/models/microsoft_phi-3-medium-4k-instruct.json +++ /dev/null @@ -1,2036 +0,0 @@ -{ - "model_info": { - "name": "Phi-3 14B", - "id": "microsoft/phi-3-medium-4k-instruct", - "developer": "microsoft", - "inference_platform": "unknown", - "additional_details": { - "model_id_aliases": [ - "microsoft/Phi-3-medium-4k-instruct" - ] - } - }, - "evaluations": [ - { - "evaluation_id": "helm_lite/microsoft_phi-3-medium-4k-instruct/1774096306.427425", - "retrieved_timestamp": "1774096306.427425", - "source_metadata": { - "source_name": "helm_lite", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_lite", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_lite", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.509, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.12111111111111111\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.724, - "details": { - "description": "min=0.724, mean=0.724, max=0.724, sum=0.724 (1)", - "tab": "Accuracy", - "NarrativeQA - Observed inference time (s)": "{\"description\": \"min=29.509, mean=29.509, max=29.509, sum=29.509 (1)\", \"tab\": \"Efficiency\", \"score\": \"29.5092350200868\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=4.392, mean=4.392, max=4.392, sum=4.392 (1)\", \"tab\": \"General information\", \"score\": \"4.391549295774648\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=3685.707, mean=3685.707, max=3685.707, sum=3685.707 (1)\", \"tab\": \"General information\", \"score\": \"3685.707042253521\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=7.245, mean=7.245, max=7.245, sum=7.245 (1)\", \"tab\": \"General information\", \"score\": \"7.245070422535211\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (closed-book)", - "source_data": { - "dataset_name": "NaturalQuestions (closed-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (closed-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.278, - "details": { - "description": "min=0.278, mean=0.278, max=0.278, sum=0.278 (1)", - "tab": "Accuracy", - "NaturalQuestions (open-book) - Observed inference time (s)": "{\"description\": \"min=44.238, mean=44.238, max=44.238, sum=44.238 (1)\", \"tab\": \"Efficiency\", \"score\": \"44.23756227874756\"}", - "NaturalQuestions (closed-book) - Observed inference time (s)": "{\"description\": \"min=49.743, mean=49.743, max=49.743, sum=49.743 (1)\", \"tab\": \"Efficiency\", \"score\": \"49.743374599456786\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.83, mean=4.83, max=4.83, sum=4.83 (1)\", \"tab\": \"General information\", \"score\": \"4.83\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.026, mean=0.026, max=0.026, sum=0.026 (1)\", \"tab\": \"General information\", \"score\": \"0.026\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=2307.695, mean=2307.695, max=2307.695, sum=2307.695 (1)\", \"tab\": \"General information\", \"score\": \"2307.695\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=8.371, mean=8.371, max=8.371, sum=8.371 (1)\", \"tab\": \"General information\", \"score\": \"8.371\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=156.383, mean=156.383, max=156.383, sum=156.383 (1)\", \"tab\": \"General information\", \"score\": \"156.383\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=10.079, mean=10.079, max=10.079, sum=10.079 (1)\", \"tab\": \"General information\", \"score\": \"10.079\"}" - } - }, - "generation_config": { - "additional_details": { - "mode": "\"closedbook\"" - } - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.916, - "details": { - "description": "min=0.916, mean=0.916, max=0.916, sum=0.916 (1)", - "tab": "Accuracy", - "OpenbookQA - Observed inference time (s)": "{\"description\": \"min=0.385, mean=0.385, max=0.385, sum=0.385 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.3850016188621521\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=291.574, mean=291.574, max=291.574, sum=291.574 (1)\", \"tab\": \"General information\", \"score\": \"291.574\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "dataset": "\"openbookqa\"", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.675, - "details": { - "description": "min=0.48, mean=0.675, max=0.94, sum=3.375 (5)", - "tab": "Accuracy", - "MMLU - Observed inference time (s)": "{\"description\": \"min=0.381, mean=0.504, max=0.722, sum=2.52 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.5039482383811682\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=406.65, mean=531.547, max=693.675, sum=2657.735 (5)\", \"tab\": \"General information\", \"score\": \"531.5470877192982\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"college_chemistry\", \"computer_security\", \"econometrics\", \"us_foreign_policy\"]", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MATH", - "source_data": { - "dataset_name": "MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Equivalent (CoT) on MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.611, - "details": { - "description": "min=0.462, mean=0.611, max=0.7, sum=4.277 (7)", - "tab": "Accuracy", - "MATH - Observed inference time (s)": "{\"description\": \"min=67.969, mean=71.561, max=74.993, sum=500.925 (7)\", \"tab\": \"Efficiency\", \"score\": \"71.56076915436368\"}", - "MATH - # eval": "{\"description\": \"min=30, mean=62.429, max=135, sum=437 (7)\", \"tab\": \"General information\", \"score\": \"62.42857142857143\"}", - "MATH - # train": "{\"description\": \"min=8, mean=8, max=8, sum=56 (7)\", \"tab\": \"General information\", \"score\": \"8.0\"}", - "MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MATH - # prompt tokens": "{\"description\": \"min=971.652, mean=1438.636, max=2490.962, sum=10070.453 (7)\", \"tab\": \"General information\", \"score\": \"1438.6362030100095\"}", - "MATH - # output tokens": "{\"description\": \"min=357.548, mean=372.128, max=392.767, sum=2604.893 (7)\", \"tab\": \"General information\", \"score\": \"372.1276343562145\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"algebra\", \"counting_and_probability\", \"geometry\", \"intermediate_algebra\", \"number_theory\", \"prealgebra\", \"precalculus\"]", - "level": "\"1\"", - "use_official_examples": "\"False\"", - "use_chain_of_thought": "\"True\"" - } - } - }, - { - "evaluation_name": "GSM8K", - "source_data": { - "dataset_name": "GSM8K", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on GSM8K", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.878, - "details": { - "description": "min=0.878, mean=0.878, max=0.878, sum=0.878 (1)", - "tab": "Accuracy", - "GSM8K - Observed inference time (s)": "{\"description\": \"min=74.933, mean=74.933, max=74.933, sum=74.933 (1)\", \"tab\": \"Efficiency\", \"score\": \"74.93269198083877\"}", - "GSM8K - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "GSM8K - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "GSM8K - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GSM8K - # prompt tokens": "{\"description\": \"min=1207.746, mean=1207.746, max=1207.746, sum=1207.746 (1)\", \"tab\": \"General information\", \"score\": \"1207.746\"}", - "GSM8K - # output tokens": "{\"description\": \"min=400, mean=400, max=400, sum=400 (1)\", \"tab\": \"General information\", \"score\": \"400.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "LegalBench", - "source_data": { - "dataset_name": "LegalBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on LegalBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.593, - "details": { - "description": "min=0.365, mean=0.593, max=0.811, sum=2.966 (5)", - "tab": "Accuracy", - "LegalBench - Observed inference time (s)": "{\"description\": \"min=5.972, mean=7.879, max=14.755, sum=39.397 (5)\", \"tab\": \"Efficiency\", \"score\": \"7.879368148866983\"}", - "LegalBench - # eval": "{\"description\": \"min=95, mean=409.4, max=1000, sum=2047 (5)\", \"tab\": \"General information\", \"score\": \"409.4\"}", - "LegalBench - # train": "{\"description\": \"min=1.884, mean=4.177, max=5, sum=20.884 (5)\", \"tab\": \"General information\", \"score\": \"4.176734693877551\"}", - "LegalBench - truncated": "{\"description\": \"min=0, mean=0.001, max=0.004, sum=0.004 (5)\", \"tab\": \"General information\", \"score\": \"0.0008163265306122449\"}", - "LegalBench - # prompt tokens": "{\"description\": \"min=229.137, mean=1033.818, max=3646.718, sum=5169.092 (5)\", \"tab\": \"General information\", \"score\": \"1033.8183708736613\"}", - "LegalBench - # output tokens": "{\"description\": \"min=1, mean=1.356, max=1.979, sum=6.782 (5)\", \"tab\": \"General information\", \"score\": \"1.3564703389458466\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"abercrombie\", \"corporate_lobbying\", \"function_of_decision_section\", \"international_citizenship_questions\", \"proa\"]" - } - } - }, - { - "evaluation_name": "MedQA", - "source_data": { - "dataset_name": "MedQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MedQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.696, - "details": { - "description": "min=0.696, mean=0.696, max=0.696, sum=0.696 (1)", - "tab": "Accuracy", - "MedQA - Observed inference time (s)": "{\"description\": \"min=1.792, mean=1.792, max=1.792, sum=1.792 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.7916561092581473\"}", - "MedQA - # eval": "{\"description\": \"min=503, mean=503, max=503, sum=503 (1)\", \"tab\": \"General information\", \"score\": \"503.0\"}", - "MedQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MedQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MedQA - # prompt tokens": "{\"description\": \"min=1243.901, mean=1243.901, max=1243.901, sum=1243.901 (1)\", \"tab\": \"General information\", \"score\": \"1243.9005964214712\"}", - "MedQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WMT 2014", - "source_data": { - "dataset_name": "WMT 2014", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "BLEU-4 on WMT 2014", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.17, - "details": { - "description": "min=0.086, mean=0.17, max=0.218, sum=0.85 (5)", - "tab": "Accuracy", - "WMT 2014 - Observed inference time (s)": "{\"description\": \"min=19.742, mean=19.987, max=20.079, sum=99.934 (5)\", \"tab\": \"Efficiency\", \"score\": \"19.98681167411759\"}", - "WMT 2014 - # eval": "{\"description\": \"min=503, mean=568.8, max=832, sum=2844 (5)\", \"tab\": \"General information\", \"score\": \"568.8\"}", - "WMT 2014 - # train": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "WMT 2014 - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WMT 2014 - # prompt tokens": "{\"description\": \"min=135.523, mean=150.288, max=172.972, sum=751.438 (5)\", \"tab\": \"General information\", \"score\": \"150.28751290334915\"}", - "WMT 2014 - # output tokens": "{\"description\": \"min=98.254, mean=99.651, max=100, sum=498.254 (5)\", \"tab\": \"General information\", \"score\": \"99.65089463220676\"}" - } - }, - "generation_config": { - "additional_details": { - "language_pair": "[\"cs-en\", \"de-en\", \"fr-en\", \"hi-en\", \"ru-en\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_mmlu/microsoft_phi-3-medium-4k-instruct/1774096312.00548", - "retrieved_timestamp": "1774096312.00548", - "source_metadata": { - "source_name": "helm_mmlu", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_mmlu", - "evaluation_results": [ - { - "evaluation_name": "MMLU All Subjects", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU All Subjects", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.775, - "details": { - "description": "min=0.5, mean=0.775, max=0.969, sum=88.295 (114)", - "tab": "Accuracy", - "MMLU All Subjects - Observed inference time (s)": "{\"description\": \"min=2.025, mean=4.948, max=22.342, sum=564.095 (114)\", \"tab\": \"Efficiency\", \"score\": \"4.948199983258553\"}", - "MMLU All Subjects - # eval": "{\"description\": \"min=100, mean=246.351, max=1534, sum=28084 (114)\", \"tab\": \"General information\", \"score\": \"246.35087719298247\"}", - "MMLU All Subjects - # train": "{\"description\": \"min=5, mean=5, max=5, sum=570 (114)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU All Subjects - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU All Subjects - # prompt tokens": "{\"description\": \"min=313.474, mean=714.893, max=3168.636, sum=81497.749 (114)\", \"tab\": \"General information\", \"score\": \"714.8925389546507\"}", - "MMLU All Subjects - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=114 (114)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_name": "Abstract Algebra", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Abstract Algebra", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5, - "details": { - "description": "min=0.5, mean=0.5, max=0.5, sum=1 (2)", - "tab": "Accuracy", - "Abstract Algebra - Observed inference time (s)": "{\"description\": \"min=2.63, mean=2.63, max=2.63, sum=5.26 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.63020414352417\"}", - "Abstract Algebra - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Abstract Algebra - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Abstract Algebra - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Abstract Algebra - # prompt tokens": "{\"description\": \"min=397.65, mean=397.65, max=397.65, sum=795.3 (2)\", \"tab\": \"General information\", \"score\": \"397.65\"}", - "Abstract Algebra - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"abstract_algebra\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_abstract_algebra\"" - } - } - }, - { - "evaluation_name": "Anatomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Anatomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.719, - "details": { - "description": "min=0.719, mean=0.719, max=0.719, sum=1.437 (2)", - "tab": "Accuracy", - "Anatomy - Observed inference time (s)": "{\"description\": \"min=3.025, mean=3.025, max=3.025, sum=6.051 (2)\", \"tab\": \"Efficiency\", \"score\": \"3.0252625394750523\"}", - "Anatomy - # eval": "{\"description\": \"min=135, mean=135, max=135, sum=270 (2)\", \"tab\": \"General information\", \"score\": \"135.0\"}", - "Anatomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Anatomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Anatomy - # prompt tokens": "{\"description\": \"min=418.133, mean=418.133, max=418.133, sum=836.267 (2)\", \"tab\": \"General information\", \"score\": \"418.1333333333333\"}", - "Anatomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"anatomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_anatomy\"" - } - } - }, - { - "evaluation_name": "College Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on College Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.529, - "details": { - "description": "min=0.529, mean=0.529, max=0.529, sum=1.059 (2)", - "tab": "Accuracy", - "College Chemistry - Observed inference time (s)": "{\"description\": \"min=3.886, mean=3.886, max=3.886, sum=7.772 (2)\", \"tab\": \"Efficiency\", \"score\": \"3.886199688911438\"}", - "College Biology - Observed inference time (s)": "{\"description\": \"min=4.073, mean=4.073, max=4.073, sum=8.146 (2)\", \"tab\": \"Efficiency\", \"score\": \"4.072841899262534\"}", - "College Computer Science - Observed inference time (s)": "{\"description\": \"min=6.237, mean=6.237, max=6.237, sum=12.473 (2)\", \"tab\": \"Efficiency\", \"score\": \"6.236730601787567\"}", - "College Mathematics - Observed inference time (s)": "{\"description\": \"min=4.541, mean=4.541, max=4.541, sum=9.083 (2)\", \"tab\": \"Efficiency\", \"score\": \"4.541367738246918\"}", - "College Medicine - Observed inference time (s)": "{\"description\": \"min=4.259, mean=4.259, max=4.259, sum=8.518 (2)\", \"tab\": \"Efficiency\", \"score\": \"4.259122938089977\"}", - "College Physics - Observed inference time (s)": "{\"description\": \"min=3.966, mean=3.966, max=3.966, sum=7.933 (2)\", \"tab\": \"Efficiency\", \"score\": \"3.966460019934411\"}", - "College Chemistry - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Chemistry - # prompt tokens": "{\"description\": \"min=622.43, mean=622.43, max=622.43, sum=1244.86 (2)\", \"tab\": \"General information\", \"score\": \"622.43\"}", - "College Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Biology - # eval": "{\"description\": \"min=144, mean=144, max=144, sum=288 (2)\", \"tab\": \"General information\", \"score\": \"144.0\"}", - "College Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # prompt tokens": "{\"description\": \"min=562.632, mean=562.632, max=562.632, sum=1125.264 (2)\", \"tab\": \"General information\", \"score\": \"562.6319444444445\"}", - "College Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # prompt tokens": "{\"description\": \"min=910.14, mean=910.14, max=910.14, sum=1820.28 (2)\", \"tab\": \"General information\", \"score\": \"910.14\"}", - "College Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Mathematics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # prompt tokens": "{\"description\": \"min=655.96, mean=655.96, max=655.96, sum=1311.92 (2)\", \"tab\": \"General information\", \"score\": \"655.96\"}", - "College Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Medicine - # eval": "{\"description\": \"min=173, mean=173, max=173, sum=346 (2)\", \"tab\": \"General information\", \"score\": \"173.0\"}", - "College Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # prompt tokens": "{\"description\": \"min=617.671, mean=617.671, max=617.671, sum=1235.341 (2)\", \"tab\": \"General information\", \"score\": \"617.6705202312139\"}", - "College Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Physics - # eval": "{\"description\": \"min=102, mean=102, max=102, sum=204 (2)\", \"tab\": \"General information\", \"score\": \"102.0\"}", - "College Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # prompt tokens": "{\"description\": \"min=560.873, mean=560.873, max=560.873, sum=1121.745 (2)\", \"tab\": \"General information\", \"score\": \"560.8725490196078\"}", - "College Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"college_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_college_physics\"" - } - } - }, - { - "evaluation_name": "Computer Security", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Computer Security", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.79, - "details": { - "description": "min=0.79, mean=0.79, max=0.79, sum=1.58 (2)", - "tab": "Accuracy", - "Computer Security - Observed inference time (s)": "{\"description\": \"min=2.748, mean=2.748, max=2.748, sum=5.496 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.7481748914718627\"}", - "Computer Security - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Computer Security - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Computer Security - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Computer Security - # prompt tokens": "{\"description\": \"min=428.17, mean=428.17, max=428.17, sum=856.34 (2)\", \"tab\": \"General information\", \"score\": \"428.17\"}", - "Computer Security - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"computer_security\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_computer_security\"" - } - } - }, - { - "evaluation_name": "Econometrics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Econometrics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.614, - "details": { - "description": "min=0.614, mean=0.614, max=0.614, sum=1.228 (2)", - "tab": "Accuracy", - "Econometrics - Observed inference time (s)": "{\"description\": \"min=4.32, mean=4.32, max=4.32, sum=8.639 (2)\", \"tab\": \"Efficiency\", \"score\": \"4.319587314338015\"}", - "Econometrics - # eval": "{\"description\": \"min=114, mean=114, max=114, sum=228 (2)\", \"tab\": \"General information\", \"score\": \"114.0\"}", - "Econometrics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Econometrics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Econometrics - # prompt tokens": "{\"description\": \"min=684.675, mean=684.675, max=684.675, sum=1369.351 (2)\", \"tab\": \"General information\", \"score\": \"684.6754385964912\"}", - "Econometrics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"econometrics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_econometrics\"" - } - } - }, - { - "evaluation_name": "Global Facts", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Global Facts", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5, - "details": { - "description": "min=0.5, mean=0.5, max=0.5, sum=1 (2)", - "tab": "Accuracy", - "Global Facts - Observed inference time (s)": "{\"description\": \"min=3.374, mean=3.374, max=3.374, sum=6.747 (2)\", \"tab\": \"Efficiency\", \"score\": \"3.373600058555603\"}", - "Global Facts - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Global Facts - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Global Facts - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Global Facts - # prompt tokens": "{\"description\": \"min=493.54, mean=493.54, max=493.54, sum=987.08 (2)\", \"tab\": \"General information\", \"score\": \"493.54\"}", - "Global Facts - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"global_facts\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_global_facts\"" - } - } - }, - { - "evaluation_name": "Jurisprudence", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Jurisprudence", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.88, - "details": { - "description": "min=0.88, mean=0.88, max=0.88, sum=1.759 (2)", - "tab": "Accuracy", - "Jurisprudence - Observed inference time (s)": "{\"description\": \"min=3.225, mean=3.225, max=3.225, sum=6.45 (2)\", \"tab\": \"Efficiency\", \"score\": \"3.2251307015065795\"}", - "Jurisprudence - # eval": "{\"description\": \"min=108, mean=108, max=108, sum=216 (2)\", \"tab\": \"General information\", \"score\": \"108.0\"}", - "Jurisprudence - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Jurisprudence - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Jurisprudence - # prompt tokens": "{\"description\": \"min=458.898, mean=458.898, max=458.898, sum=917.796 (2)\", \"tab\": \"General information\", \"score\": \"458.89814814814815\"}", - "Jurisprudence - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"jurisprudence\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_jurisprudence\"" - } - } - }, - { - "evaluation_name": "Philosophy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Philosophy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.804, - "details": { - "description": "min=0.804, mean=0.804, max=0.804, sum=1.608 (2)", - "tab": "Accuracy", - "Philosophy - Observed inference time (s)": "{\"description\": \"min=2.591, mean=2.591, max=2.591, sum=5.182 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.591215438781444\"}", - "Philosophy - # eval": "{\"description\": \"min=311, mean=311, max=311, sum=622 (2)\", \"tab\": \"General information\", \"score\": \"311.0\"}", - "Philosophy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Philosophy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Philosophy - # prompt tokens": "{\"description\": \"min=381.122, mean=381.122, max=381.122, sum=762.244 (2)\", \"tab\": \"General information\", \"score\": \"381.12218649517683\"}", - "Philosophy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"philosophy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_philosophy\"" - } - } - }, - { - "evaluation_name": "Professional Psychology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Professional Psychology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.835, - "details": { - "description": "min=0.835, mean=0.835, max=0.835, sum=1.67 (2)", - "tab": "Accuracy", - "Professional Medicine - Observed inference time (s)": "{\"description\": \"min=9.39, mean=9.39, max=9.39, sum=18.781 (2)\", \"tab\": \"Efficiency\", \"score\": \"9.390463957015205\"}", - "Professional Accounting - Observed inference time (s)": "{\"description\": \"min=5.784, mean=5.784, max=5.784, sum=11.567 (2)\", \"tab\": \"Efficiency\", \"score\": \"5.7837115450108305\"}", - "Professional Law - Observed inference time (s)": "{\"description\": \"min=13.198, mean=13.198, max=13.198, sum=26.396 (2)\", \"tab\": \"Efficiency\", \"score\": \"13.198108883849024\"}", - "Professional Psychology - Observed inference time (s)": "{\"description\": \"min=4.667, mean=4.667, max=4.667, sum=9.335 (2)\", \"tab\": \"Efficiency\", \"score\": \"4.667331269753524\"}", - "Professional Medicine - # eval": "{\"description\": \"min=272, mean=272, max=272, sum=544 (2)\", \"tab\": \"General information\", \"score\": \"272.0\"}", - "Professional Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Medicine - # prompt tokens": "{\"description\": \"min=1339.647, mean=1339.647, max=1339.647, sum=2679.294 (2)\", \"tab\": \"General information\", \"score\": \"1339.6470588235295\"}", - "Professional Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Accounting - # eval": "{\"description\": \"min=282, mean=282, max=282, sum=564 (2)\", \"tab\": \"General information\", \"score\": \"282.0\"}", - "Professional Accounting - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Accounting - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # prompt tokens": "{\"description\": \"min=832.277, mean=832.277, max=832.277, sum=1664.553 (2)\", \"tab\": \"General information\", \"score\": \"832.2765957446809\"}", - "Professional Accounting - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Law - # eval": "{\"description\": \"min=1534, mean=1534, max=1534, sum=3068 (2)\", \"tab\": \"General information\", \"score\": \"1534.0\"}", - "Professional Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # prompt tokens": "{\"description\": \"min=1924.007, mean=1924.007, max=1924.007, sum=3848.014 (2)\", \"tab\": \"General information\", \"score\": \"1924.0071707953064\"}", - "Professional Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Psychology - # eval": "{\"description\": \"min=612, mean=612, max=612, sum=1224 (2)\", \"tab\": \"General information\", \"score\": \"612.0\"}", - "Professional Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # prompt tokens": "{\"description\": \"min=659.078, mean=659.078, max=659.078, sum=1318.157 (2)\", \"tab\": \"General information\", \"score\": \"659.0784313725491\"}", - "Professional Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"professional_psychology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_professional_psychology\"" - } - } - }, - { - "evaluation_name": "Us Foreign Policy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Us Foreign Policy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.95, - "details": { - "description": "min=0.95, mean=0.95, max=0.95, sum=1.9 (2)", - "tab": "Accuracy", - "Us Foreign Policy - Observed inference time (s)": "{\"description\": \"min=2.982, mean=2.982, max=2.982, sum=5.964 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.98179637670517\"}", - "Us Foreign Policy - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Us Foreign Policy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Us Foreign Policy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Us Foreign Policy - # prompt tokens": "{\"description\": \"min=479.81, mean=479.81, max=479.81, sum=959.62 (2)\", \"tab\": \"General information\", \"score\": \"479.81\"}", - "Us Foreign Policy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"us_foreign_policy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_us_foreign_policy\"" - } - } - }, - { - "evaluation_name": "Astronomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Astronomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.849, - "details": { - "description": "min=0.849, mean=0.849, max=0.849, sum=1.697 (2)", - "tab": "Accuracy", - "Astronomy - Observed inference time (s)": "{\"description\": \"min=4.875, mean=4.875, max=4.875, sum=9.749 (2)\", \"tab\": \"Efficiency\", \"score\": \"4.874531077711206\"}", - "Astronomy - # eval": "{\"description\": \"min=152, mean=152, max=152, sum=304 (2)\", \"tab\": \"General information\", \"score\": \"152.0\"}", - "Astronomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Astronomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Astronomy - # prompt tokens": "{\"description\": \"min=690.079, mean=690.079, max=690.079, sum=1380.158 (2)\", \"tab\": \"General information\", \"score\": \"690.078947368421\"}", - "Astronomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"astronomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_astronomy\"" - } - } - }, - { - "evaluation_name": "Business Ethics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Business Ethics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8, - "details": { - "description": "min=0.8, mean=0.8, max=0.8, sum=1.6 (2)", - "tab": "Accuracy", - "Business Ethics - Observed inference time (s)": "{\"description\": \"min=4.78, mean=4.78, max=4.78, sum=9.559 (2)\", \"tab\": \"Efficiency\", \"score\": \"4.779508647918701\"}", - "Business Ethics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Business Ethics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Business Ethics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Business Ethics - # prompt tokens": "{\"description\": \"min=683.44, mean=683.44, max=683.44, sum=1366.88 (2)\", \"tab\": \"General information\", \"score\": \"683.44\"}", - "Business Ethics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"business_ethics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_business_ethics\"" - } - } - }, - { - "evaluation_name": "Clinical Knowledge", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Clinical Knowledge", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.826, - "details": { - "description": "min=0.826, mean=0.826, max=0.826, sum=1.653 (2)", - "tab": "Accuracy", - "Clinical Knowledge - Observed inference time (s)": "{\"description\": \"min=3.474, mean=3.474, max=3.474, sum=6.948 (2)\", \"tab\": \"Efficiency\", \"score\": \"3.474059367629717\"}", - "Clinical Knowledge - # eval": "{\"description\": \"min=265, mean=265, max=265, sum=530 (2)\", \"tab\": \"General information\", \"score\": \"265.0\"}", - "Clinical Knowledge - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Clinical Knowledge - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Clinical Knowledge - # prompt tokens": "{\"description\": \"min=496.374, mean=496.374, max=496.374, sum=992.747 (2)\", \"tab\": \"General information\", \"score\": \"496.3735849056604\"}", - "Clinical Knowledge - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"clinical_knowledge\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_clinical_knowledge\"" - } - } - }, - { - "evaluation_name": "Conceptual Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Conceptual Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.809, - "details": { - "description": "min=0.809, mean=0.809, max=0.809, sum=1.617 (2)", - "tab": "Accuracy", - "Conceptual Physics - Observed inference time (s)": "{\"description\": \"min=2.448, mean=2.448, max=2.448, sum=4.896 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.448020648956299\"}", - "Conceptual Physics - # eval": "{\"description\": \"min=235, mean=235, max=235, sum=470 (2)\", \"tab\": \"General information\", \"score\": \"235.0\"}", - "Conceptual Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Conceptual Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Conceptual Physics - # prompt tokens": "{\"description\": \"min=342.153, mean=342.153, max=342.153, sum=684.306 (2)\", \"tab\": \"General information\", \"score\": \"342.1531914893617\"}", - "Conceptual Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"conceptual_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_conceptual_physics\"" - } - } - }, - { - "evaluation_name": "Electrical Engineering", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Electrical Engineering", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.683, - "details": { - "description": "min=0.683, mean=0.683, max=0.683, sum=1.366 (2)", - "tab": "Accuracy", - "Electrical Engineering - Observed inference time (s)": "{\"description\": \"min=3.495, mean=3.495, max=3.495, sum=6.99 (2)\", \"tab\": \"Efficiency\", \"score\": \"3.4950728284901587\"}", - "Electrical Engineering - # eval": "{\"description\": \"min=145, mean=145, max=145, sum=290 (2)\", \"tab\": \"General information\", \"score\": \"145.0\"}", - "Electrical Engineering - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Electrical Engineering - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Electrical Engineering - # prompt tokens": "{\"description\": \"min=506.779, mean=506.779, max=506.779, sum=1013.559 (2)\", \"tab\": \"General information\", \"score\": \"506.7793103448276\"}", - "Electrical Engineering - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"electrical_engineering\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_electrical_engineering\"" - } - } - }, - { - "evaluation_name": "Elementary Mathematics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Elementary Mathematics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.709, - "details": { - "description": "min=0.709, mean=0.709, max=0.709, sum=1.418 (2)", - "tab": "Accuracy", - "Elementary Mathematics - Observed inference time (s)": "{\"description\": \"min=4.344, mean=4.344, max=4.344, sum=8.688 (2)\", \"tab\": \"Efficiency\", \"score\": \"4.344110502137078\"}", - "Elementary Mathematics - # eval": "{\"description\": \"min=378, mean=378, max=378, sum=756 (2)\", \"tab\": \"General information\", \"score\": \"378.0\"}", - "Elementary Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Elementary Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Elementary Mathematics - # prompt tokens": "{\"description\": \"min=618.156, mean=618.156, max=618.156, sum=1236.312 (2)\", \"tab\": \"General information\", \"score\": \"618.1560846560847\"}", - "Elementary Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"elementary_mathematics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_elementary_mathematics\"" - } - } - }, - { - "evaluation_name": "Formal Logic", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Formal Logic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.587, - "details": { - "description": "min=0.587, mean=0.587, max=0.587, sum=1.175 (2)", - "tab": "Accuracy", - "Formal Logic - Observed inference time (s)": "{\"description\": \"min=4.988, mean=4.988, max=4.988, sum=9.977 (2)\", \"tab\": \"Efficiency\", \"score\": \"4.988478910355341\"}", - "Formal Logic - # eval": "{\"description\": \"min=126, mean=126, max=126, sum=252 (2)\", \"tab\": \"General information\", \"score\": \"126.0\"}", - "Formal Logic - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Formal Logic - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Formal Logic - # prompt tokens": "{\"description\": \"min=700.81, mean=700.81, max=700.81, sum=1401.619 (2)\", \"tab\": \"General information\", \"score\": \"700.8095238095239\"}", - "Formal Logic - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"formal_logic\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_formal_logic\"" - } - } - }, - { - "evaluation_name": "High School World History", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on High School World History", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.903, - "details": { - "description": "min=0.903, mean=0.903, max=0.903, sum=1.806 (2)", - "tab": "Accuracy", - "High School Biology - Observed inference time (s)": "{\"description\": \"min=4.253, mean=4.253, max=4.253, sum=8.506 (2)\", \"tab\": \"Efficiency\", \"score\": \"4.253153976317375\"}", - "High School Chemistry - Observed inference time (s)": "{\"description\": \"min=4.116, mean=4.116, max=4.116, sum=8.232 (2)\", \"tab\": \"Efficiency\", \"score\": \"4.115784048446881\"}", - "High School Computer Science - Observed inference time (s)": "{\"description\": \"min=6.919, mean=6.919, max=6.919, sum=13.839 (2)\", \"tab\": \"Efficiency\", \"score\": \"6.919438579082489\"}", - "High School European History - Observed inference time (s)": "{\"description\": \"min=22.342, mean=22.342, max=22.342, sum=44.684 (2)\", \"tab\": \"Efficiency\", \"score\": \"22.341962937152747\"}", - "High School Geography - Observed inference time (s)": "{\"description\": \"min=3.01, mean=3.01, max=3.01, sum=6.02 (2)\", \"tab\": \"Efficiency\", \"score\": \"3.010115607820376\"}", - "High School Government And Politics - Observed inference time (s)": "{\"description\": \"min=3.784, mean=3.784, max=3.784, sum=7.567 (2)\", \"tab\": \"Efficiency\", \"score\": \"3.783631190117159\"}", - "High School Macroeconomics - Observed inference time (s)": "{\"description\": \"min=3.202, mean=3.202, max=3.202, sum=6.403 (2)\", \"tab\": \"Efficiency\", \"score\": \"3.2015056090477185\"}", - "High School Mathematics - Observed inference time (s)": "{\"description\": \"min=4.126, mean=4.126, max=4.126, sum=8.251 (2)\", \"tab\": \"Efficiency\", \"score\": \"4.125549591912163\"}", - "High School Microeconomics - Observed inference time (s)": "{\"description\": \"min=3.125, mean=3.125, max=3.125, sum=6.249 (2)\", \"tab\": \"Efficiency\", \"score\": \"3.124516798668549\"}", - "High School Physics - Observed inference time (s)": "{\"description\": \"min=3.882, mean=3.882, max=3.882, sum=7.765 (2)\", \"tab\": \"Efficiency\", \"score\": \"3.88235890154807\"}", - "High School Psychology - Observed inference time (s)": "{\"description\": \"min=4.036, mean=4.036, max=4.036, sum=8.072 (2)\", \"tab\": \"Efficiency\", \"score\": \"4.035925890108861\"}", - "High School Statistics - Observed inference time (s)": "{\"description\": \"min=6.294, mean=6.294, max=6.294, sum=12.587 (2)\", \"tab\": \"Efficiency\", \"score\": \"6.293625408852542\"}", - "High School US History - Observed inference time (s)": "{\"description\": \"min=17.94, mean=17.94, max=17.94, sum=35.88 (2)\", \"tab\": \"Efficiency\", \"score\": \"17.93984198219636\"}", - "High School World History - Observed inference time (s)": "{\"description\": \"min=11.445, mean=11.445, max=11.445, sum=22.889 (2)\", \"tab\": \"Efficiency\", \"score\": \"11.444628432833193\"}", - "High School Biology - # eval": "{\"description\": \"min=310, mean=310, max=310, sum=620 (2)\", \"tab\": \"General information\", \"score\": \"310.0\"}", - "High School Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Biology - # prompt tokens": "{\"description\": \"min=605.894, mean=605.894, max=605.894, sum=1211.787 (2)\", \"tab\": \"General information\", \"score\": \"605.8935483870968\"}", - "High School Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Chemistry - # eval": "{\"description\": \"min=203, mean=203, max=203, sum=406 (2)\", \"tab\": \"General information\", \"score\": \"203.0\"}", - "High School Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # prompt tokens": "{\"description\": \"min=577.665, mean=577.665, max=577.665, sum=1155.33 (2)\", \"tab\": \"General information\", \"score\": \"577.6650246305419\"}", - "High School Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "High School Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # prompt tokens": "{\"description\": \"min=997.57, mean=997.57, max=997.57, sum=1995.14 (2)\", \"tab\": \"General information\", \"score\": \"997.57\"}", - "High School Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School European History - # eval": "{\"description\": \"min=165, mean=165, max=165, sum=330 (2)\", \"tab\": \"General information\", \"score\": \"165.0\"}", - "High School European History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School European History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # prompt tokens": "{\"description\": \"min=3168.636, mean=3168.636, max=3168.636, sum=6337.273 (2)\", \"tab\": \"General information\", \"score\": \"3168.6363636363635\"}", - "High School European History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Geography - # eval": "{\"description\": \"min=198, mean=198, max=198, sum=396 (2)\", \"tab\": \"General information\", \"score\": \"198.0\"}", - "High School Geography - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Geography - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # prompt tokens": "{\"description\": \"min=445.657, mean=445.657, max=445.657, sum=891.313 (2)\", \"tab\": \"General information\", \"score\": \"445.65656565656565\"}", - "High School Geography - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Government And Politics - # eval": "{\"description\": \"min=193, mean=193, max=193, sum=386 (2)\", \"tab\": \"General information\", \"score\": \"193.0\"}", - "High School Government And Politics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Government And Politics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # prompt tokens": "{\"description\": \"min=536.927, mean=536.927, max=536.927, sum=1073.855 (2)\", \"tab\": \"General information\", \"score\": \"536.9274611398964\"}", - "High School Government And Politics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Macroeconomics - # eval": "{\"description\": \"min=390, mean=390, max=390, sum=780 (2)\", \"tab\": \"General information\", \"score\": \"390.0\"}", - "High School Macroeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Macroeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # prompt tokens": "{\"description\": \"min=454.662, mean=454.662, max=454.662, sum=909.323 (2)\", \"tab\": \"General information\", \"score\": \"454.66153846153844\"}", - "High School Macroeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Mathematics - # eval": "{\"description\": \"min=270, mean=270, max=270, sum=540 (2)\", \"tab\": \"General information\", \"score\": \"270.0\"}", - "High School Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # prompt tokens": "{\"description\": \"min=588.181, mean=588.181, max=588.181, sum=1176.363 (2)\", \"tab\": \"General information\", \"score\": \"588.1814814814815\"}", - "High School Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Microeconomics - # eval": "{\"description\": \"min=238, mean=238, max=238, sum=476 (2)\", \"tab\": \"General information\", \"score\": \"238.0\"}", - "High School Microeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Microeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # prompt tokens": "{\"description\": \"min=458.492, mean=458.492, max=458.492, sum=916.983 (2)\", \"tab\": \"General information\", \"score\": \"458.49159663865544\"}", - "High School Microeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Physics - # eval": "{\"description\": \"min=151, mean=151, max=151, sum=302 (2)\", \"tab\": \"General information\", \"score\": \"151.0\"}", - "High School Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # prompt tokens": "{\"description\": \"min=630.788, mean=630.788, max=630.788, sum=1261.576 (2)\", \"tab\": \"General information\", \"score\": \"630.7880794701987\"}", - "High School Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Psychology - # eval": "{\"description\": \"min=545, mean=545, max=545, sum=1090 (2)\", \"tab\": \"General information\", \"score\": \"545.0\"}", - "High School Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # prompt tokens": "{\"description\": \"min=594.919, mean=594.919, max=594.919, sum=1189.839 (2)\", \"tab\": \"General information\", \"score\": \"594.9192660550459\"}", - "High School Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Statistics - # eval": "{\"description\": \"min=216, mean=216, max=216, sum=432 (2)\", \"tab\": \"General information\", \"score\": \"216.0\"}", - "High School Statistics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Statistics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # prompt tokens": "{\"description\": \"min=917.208, mean=917.208, max=917.208, sum=1834.417 (2)\", \"tab\": \"General information\", \"score\": \"917.2083333333334\"}", - "High School Statistics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School US History - # eval": "{\"description\": \"min=204, mean=204, max=204, sum=408 (2)\", \"tab\": \"General information\", \"score\": \"204.0\"}", - "High School US History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School US History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # prompt tokens": "{\"description\": \"min=2544.324, mean=2544.324, max=2544.324, sum=5088.647 (2)\", \"tab\": \"General information\", \"score\": \"2544.323529411765\"}", - "High School US History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School World History - # eval": "{\"description\": \"min=237, mean=237, max=237, sum=474 (2)\", \"tab\": \"General information\", \"score\": \"237.0\"}", - "High School World History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School World History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # prompt tokens": "{\"description\": \"min=1647.219, mean=1647.219, max=1647.219, sum=3294.439 (2)\", \"tab\": \"General information\", \"score\": \"1647.2194092827003\"}", - "High School World History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"high_school_world_history\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_high_school_world_history\"" - } - } - }, - { - "evaluation_name": "Human Sexuality", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Human Sexuality", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.863, - "details": { - "description": "min=0.863, mean=0.863, max=0.863, sum=1.725 (2)", - "tab": "Accuracy", - "Human Aging - Observed inference time (s)": "{\"description\": \"min=2.578, mean=2.578, max=2.578, sum=5.157 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.5783249647628033\"}", - "Human Sexuality - Observed inference time (s)": "{\"description\": \"min=2.963, mean=2.963, max=2.963, sum=5.925 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.9625705234877024\"}", - "Human Aging - # eval": "{\"description\": \"min=223, mean=223, max=223, sum=446 (2)\", \"tab\": \"General information\", \"score\": \"223.0\"}", - "Human Aging - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Aging - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Aging - # prompt tokens": "{\"description\": \"min=370.26, mean=370.26, max=370.26, sum=740.52 (2)\", \"tab\": \"General information\", \"score\": \"370.26008968609864\"}", - "Human Aging - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Human Sexuality - # eval": "{\"description\": \"min=131, mean=131, max=131, sum=262 (2)\", \"tab\": \"General information\", \"score\": \"131.0\"}", - "Human Sexuality - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Sexuality - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # prompt tokens": "{\"description\": \"min=412.382, mean=412.382, max=412.382, sum=824.763 (2)\", \"tab\": \"General information\", \"score\": \"412.381679389313\"}", - "Human Sexuality - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"human_sexuality\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_human_sexuality\"" - } - } - }, - { - "evaluation_name": "International Law", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on International Law", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.934, - "details": { - "description": "min=0.934, mean=0.934, max=0.934, sum=1.868 (2)", - "tab": "Accuracy", - "International Law - Observed inference time (s)": "{\"description\": \"min=5.179, mean=5.179, max=5.179, sum=10.357 (2)\", \"tab\": \"Efficiency\", \"score\": \"5.1785316802253405\"}", - "International Law - # eval": "{\"description\": \"min=121, mean=121, max=121, sum=242 (2)\", \"tab\": \"General information\", \"score\": \"121.0\"}", - "International Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "International Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "International Law - # prompt tokens": "{\"description\": \"min=738.463, mean=738.463, max=738.463, sum=1476.926 (2)\", \"tab\": \"General information\", \"score\": \"738.4628099173553\"}", - "International Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"international_law\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_international_law\"" - } - } - }, - { - "evaluation_name": "Logical Fallacies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Logical Fallacies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.828, - "details": { - "description": "min=0.828, mean=0.828, max=0.828, sum=1.656 (2)", - "tab": "Accuracy", - "Logical Fallacies - Observed inference time (s)": "{\"description\": \"min=3.522, mean=3.522, max=3.522, sum=7.045 (2)\", \"tab\": \"Efficiency\", \"score\": \"3.5224247461447686\"}", - "Logical Fallacies - # eval": "{\"description\": \"min=163, mean=163, max=163, sum=326 (2)\", \"tab\": \"General information\", \"score\": \"163.0\"}", - "Logical Fallacies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Logical Fallacies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Logical Fallacies - # prompt tokens": "{\"description\": \"min=511.755, mean=511.755, max=511.755, sum=1023.509 (2)\", \"tab\": \"General information\", \"score\": \"511.7546012269939\"}", - "Logical Fallacies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"logical_fallacies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_logical_fallacies\"" - } - } - }, - { - "evaluation_name": "Machine Learning", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Machine Learning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.696, - "details": { - "description": "min=0.696, mean=0.696, max=0.696, sum=1.393 (2)", - "tab": "Accuracy", - "Machine Learning - Observed inference time (s)": "{\"description\": \"min=5.118, mean=5.118, max=5.118, sum=10.237 (2)\", \"tab\": \"Efficiency\", \"score\": \"5.118442311882973\"}", - "Machine Learning - # eval": "{\"description\": \"min=112, mean=112, max=112, sum=224 (2)\", \"tab\": \"General information\", \"score\": \"112.0\"}", - "Machine Learning - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Machine Learning - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Machine Learning - # prompt tokens": "{\"description\": \"min=739.402, mean=739.402, max=739.402, sum=1478.804 (2)\", \"tab\": \"General information\", \"score\": \"739.4017857142857\"}", - "Machine Learning - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"machine_learning\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_machine_learning\"" - } - } - }, - { - "evaluation_name": "Management", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Management", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.864, - "details": { - "description": "min=0.864, mean=0.864, max=0.864, sum=1.728 (2)", - "tab": "Accuracy", - "Management - Observed inference time (s)": "{\"description\": \"min=2.27, mean=2.27, max=2.27, sum=4.539 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.2697336812621183\"}", - "Management - # eval": "{\"description\": \"min=103, mean=103, max=103, sum=206 (2)\", \"tab\": \"General information\", \"score\": \"103.0\"}", - "Management - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Management - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Management - # prompt tokens": "{\"description\": \"min=324.777, mean=324.777, max=324.777, sum=649.553 (2)\", \"tab\": \"General information\", \"score\": \"324.77669902912623\"}", - "Management - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"management\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_management\"" - } - } - }, - { - "evaluation_name": "Marketing", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Marketing", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.919, - "details": { - "description": "min=0.919, mean=0.919, max=0.919, sum=1.838 (2)", - "tab": "Accuracy", - "Marketing - Observed inference time (s)": "{\"description\": \"min=3.394, mean=3.394, max=3.394, sum=6.788 (2)\", \"tab\": \"Efficiency\", \"score\": \"3.3940892515019474\"}", - "Marketing - # eval": "{\"description\": \"min=234, mean=234, max=234, sum=468 (2)\", \"tab\": \"General information\", \"score\": \"234.0\"}", - "Marketing - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Marketing - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Marketing - # prompt tokens": "{\"description\": \"min=481.628, mean=481.628, max=481.628, sum=963.256 (2)\", \"tab\": \"General information\", \"score\": \"481.62820512820514\"}", - "Marketing - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"marketing\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_marketing\"" - } - } - }, - { - "evaluation_name": "Medical Genetics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Medical Genetics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.91, - "details": { - "description": "min=0.91, mean=0.91, max=0.91, sum=1.82 (2)", - "tab": "Accuracy", - "Medical Genetics - Observed inference time (s)": "{\"description\": \"min=2.894, mean=2.894, max=2.894, sum=5.787 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.893650698661804\"}", - "Medical Genetics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Medical Genetics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Medical Genetics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Medical Genetics - # prompt tokens": "{\"description\": \"min=417.14, mean=417.14, max=417.14, sum=834.28 (2)\", \"tab\": \"General information\", \"score\": \"417.14\"}", - "Medical Genetics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"medical_genetics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_medical_genetics\"" - } - } - }, - { - "evaluation_name": "Miscellaneous", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Miscellaneous", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.894, - "details": { - "description": "min=0.894, mean=0.894, max=0.894, sum=1.788 (2)", - "tab": "Accuracy", - "Miscellaneous - Observed inference time (s)": "{\"description\": \"min=2.025, mean=2.025, max=2.025, sum=4.05 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.0249771478075633\"}", - "Miscellaneous - # eval": "{\"description\": \"min=783, mean=783, max=783, sum=1566 (2)\", \"tab\": \"General information\", \"score\": \"783.0\"}", - "Miscellaneous - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Miscellaneous - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Miscellaneous - # prompt tokens": "{\"description\": \"min=354.913, mean=354.913, max=354.913, sum=709.826 (2)\", \"tab\": \"General information\", \"score\": \"354.9131545338442\"}", - "Miscellaneous - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"miscellaneous\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_miscellaneous\"" - } - } - }, - { - "evaluation_name": "Moral Scenarios", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Moral Scenarios", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.639, - "details": { - "description": "min=0.639, mean=0.639, max=0.639, sum=1.278 (2)", - "tab": "Accuracy", - "Moral Disputes - Observed inference time (s)": "{\"description\": \"min=3.877, mean=3.877, max=3.877, sum=7.754 (2)\", \"tab\": \"Efficiency\", \"score\": \"3.877226921175257\"}", - "Moral Scenarios - Observed inference time (s)": "{\"description\": \"min=5.17, mean=5.17, max=5.17, sum=10.34 (2)\", \"tab\": \"Efficiency\", \"score\": \"5.170224364509796\"}", - "Moral Disputes - # eval": "{\"description\": \"min=346, mean=346, max=346, sum=692 (2)\", \"tab\": \"General information\", \"score\": \"346.0\"}", - "Moral Disputes - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Disputes - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Disputes - # prompt tokens": "{\"description\": \"min=551.506, mean=551.506, max=551.506, sum=1103.012 (2)\", \"tab\": \"General information\", \"score\": \"551.5057803468208\"}", - "Moral Disputes - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Moral Scenarios - # eval": "{\"description\": \"min=895, mean=895, max=895, sum=1790 (2)\", \"tab\": \"General information\", \"score\": \"895.0\"}", - "Moral Scenarios - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Scenarios - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # prompt tokens": "{\"description\": \"min=765.479, mean=765.479, max=765.479, sum=1530.959 (2)\", \"tab\": \"General information\", \"score\": \"765.4793296089385\"}", - "Moral Scenarios - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"moral_scenarios\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_moral_scenarios\"" - } - } - }, - { - "evaluation_name": "Nutrition", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Nutrition", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.837, - "details": { - "description": "min=0.837, mean=0.837, max=0.837, sum=1.673 (2)", - "tab": "Accuracy", - "Nutrition - Observed inference time (s)": "{\"description\": \"min=4.962, mean=4.962, max=4.962, sum=9.923 (2)\", \"tab\": \"Efficiency\", \"score\": \"4.961673566718507\"}", - "Nutrition - # eval": "{\"description\": \"min=306, mean=306, max=306, sum=612 (2)\", \"tab\": \"General information\", \"score\": \"306.0\"}", - "Nutrition - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Nutrition - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Nutrition - # prompt tokens": "{\"description\": \"min=704.922, mean=704.922, max=704.922, sum=1409.843 (2)\", \"tab\": \"General information\", \"score\": \"704.9215686274509\"}", - "Nutrition - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"nutrition\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_nutrition\"" - } - } - }, - { - "evaluation_name": "Prehistory", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Prehistory", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.867, - "details": { - "description": "min=0.867, mean=0.867, max=0.867, sum=1.735 (2)", - "tab": "Accuracy", - "Prehistory - Observed inference time (s)": "{\"description\": \"min=4.39, mean=4.39, max=4.39, sum=8.779 (2)\", \"tab\": \"Efficiency\", \"score\": \"4.389729757367829\"}", - "Prehistory - # eval": "{\"description\": \"min=324, mean=324, max=324, sum=648 (2)\", \"tab\": \"General information\", \"score\": \"324.0\"}", - "Prehistory - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Prehistory - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Prehistory - # prompt tokens": "{\"description\": \"min=628.185, mean=628.185, max=628.185, sum=1256.37 (2)\", \"tab\": \"General information\", \"score\": \"628.1851851851852\"}", - "Prehistory - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"prehistory\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_prehistory\"" - } - } - }, - { - "evaluation_name": "Public Relations", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Public Relations", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.755, - "details": { - "description": "min=0.755, mean=0.755, max=0.755, sum=1.509 (2)", - "tab": "Accuracy", - "Public Relations - Observed inference time (s)": "{\"description\": \"min=3.474, mean=3.474, max=3.474, sum=6.948 (2)\", \"tab\": \"Efficiency\", \"score\": \"3.4741735740141437\"}", - "Public Relations - # eval": "{\"description\": \"min=110, mean=110, max=110, sum=220 (2)\", \"tab\": \"General information\", \"score\": \"110.0\"}", - "Public Relations - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Public Relations - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Public Relations - # prompt tokens": "{\"description\": \"min=483.827, mean=483.827, max=483.827, sum=967.655 (2)\", \"tab\": \"General information\", \"score\": \"483.8272727272727\"}", - "Public Relations - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"public_relations\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_public_relations\"" - } - } - }, - { - "evaluation_name": "Security Studies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Security Studies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.829, - "details": { - "description": "min=0.829, mean=0.829, max=0.829, sum=1.657 (2)", - "tab": "Accuracy", - "Security Studies - Observed inference time (s)": "{\"description\": \"min=9.808, mean=9.808, max=9.808, sum=19.616 (2)\", \"tab\": \"Efficiency\", \"score\": \"9.807938383063492\"}", - "Security Studies - # eval": "{\"description\": \"min=245, mean=245, max=245, sum=490 (2)\", \"tab\": \"General information\", \"score\": \"245.0\"}", - "Security Studies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Security Studies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Security Studies - # prompt tokens": "{\"description\": \"min=1386.531, mean=1386.531, max=1386.531, sum=2773.061 (2)\", \"tab\": \"General information\", \"score\": \"1386.530612244898\"}", - "Security Studies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"security_studies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_security_studies\"" - } - } - }, - { - "evaluation_name": "Sociology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Sociology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.891, - "details": { - "description": "min=0.891, mean=0.891, max=0.891, sum=1.781 (2)", - "tab": "Accuracy", - "Sociology - Observed inference time (s)": "{\"description\": \"min=3.643, mean=3.643, max=3.643, sum=7.285 (2)\", \"tab\": \"Efficiency\", \"score\": \"3.642500052997722\"}", - "Sociology - # eval": "{\"description\": \"min=201, mean=201, max=201, sum=402 (2)\", \"tab\": \"General information\", \"score\": \"201.0\"}", - "Sociology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Sociology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Sociology - # prompt tokens": "{\"description\": \"min=517.478, mean=517.478, max=517.478, sum=1034.955 (2)\", \"tab\": \"General information\", \"score\": \"517.4776119402985\"}", - "Sociology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"sociology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_sociology\"" - } - } - }, - { - "evaluation_name": "Virology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Virology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.554, - "details": { - "description": "min=0.554, mean=0.554, max=0.554, sum=1.108 (2)", - "tab": "Accuracy", - "Virology - Observed inference time (s)": "{\"description\": \"min=2.911, mean=2.911, max=2.911, sum=5.822 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.910837286926178\"}", - "Virology - # eval": "{\"description\": \"min=166, mean=166, max=166, sum=332 (2)\", \"tab\": \"General information\", \"score\": \"166.0\"}", - "Virology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Virology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Virology - # prompt tokens": "{\"description\": \"min=414.108, mean=414.108, max=414.108, sum=828.217 (2)\", \"tab\": \"General information\", \"score\": \"414.10843373493975\"}", - "Virology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"virology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_virology\"" - } - } - }, - { - "evaluation_name": "World Religions", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on World Religions", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.865, - "details": { - "description": "min=0.865, mean=0.865, max=0.865, sum=1.731 (2)", - "tab": "Accuracy", - "World Religions - Observed inference time (s)": "{\"description\": \"min=2.097, mean=2.097, max=2.097, sum=4.194 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.0972191897052075\"}", - "World Religions - # eval": "{\"description\": \"min=171, mean=171, max=171, sum=342 (2)\", \"tab\": \"General information\", \"score\": \"171.0\"}", - "World Religions - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "World Religions - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "World Religions - # prompt tokens": "{\"description\": \"min=313.474, mean=313.474, max=313.474, sum=626.947 (2)\", \"tab\": \"General information\", \"score\": \"313.4736842105263\"}", - "World Religions - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"world_religions\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_world_religions\"" - } - } - }, - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.015, - "details": { - "description": "", - "tab": "Efficiency" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_id": "hfopenllm_v2/microsoft_Phi-3-medium-4k-instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6423 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6412 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1956 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3364 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4258 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4676 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/microsoft_phi-3-mini-128k-instruct.json b/data/models/microsoft_phi-3-mini-128k-instruct.json deleted file mode 100644 index 4c838fa896ddecfe24df23a766f96c141ee39fb9..0000000000000000000000000000000000000000 --- a/data/models/microsoft_phi-3-mini-128k-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Phi-3-mini-128k-instruct", - "id": "microsoft/Phi-3-mini-128k-instruct", - "developer": "microsoft", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Phi3ForCausalLM", - "params_billions": "3.821" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/microsoft_Phi-3-mini-128k-instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5976 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5575 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1405 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.318 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3937 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3734 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/microsoft_phi-3-mini-4k-instruct.json b/data/models/microsoft_phi-3-mini-4k-instruct.json deleted file mode 100644 index f0214854e07bd3ad87162726609ad6d31e31396a..0000000000000000000000000000000000000000 --- a/data/models/microsoft_phi-3-mini-4k-instruct.json +++ /dev/null @@ -1,275 +0,0 @@ -{ - "model_info": { - "name": "Phi-3-mini-4k-instruct", - "id": "microsoft/Phi-3-mini-4k-instruct", - "developer": "microsoft", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Phi3ForCausalLM", - "params_billions": "3.821" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/microsoft_Phi-3-mini-4k-instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5477 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5491 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1639 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3322 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4284 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4022 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "hfopenllm_v2/microsoft_Phi-3-mini-4k-instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5613 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5676 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1163 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3196 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.395 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3866 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/microsoft_phi-3-small-128k-instruct.json b/data/models/microsoft_phi-3-small-128k-instruct.json deleted file mode 100644 index fc2b52eae7b2c1757ec8c36c2a5c273a1fcacc4c..0000000000000000000000000000000000000000 --- a/data/models/microsoft_phi-3-small-128k-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Phi-3-small-128k-instruct", - "id": "microsoft/Phi-3-small-128k-instruct", - "developer": "microsoft", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Phi3SmallForCausalLM", - "params_billions": "7.392" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/microsoft_Phi-3-small-128k-instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6368 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6202 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2026 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3171 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4378 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4491 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/microsoft_phi-3-small-8k-instruct.json b/data/models/microsoft_phi-3-small-8k-instruct.json deleted file mode 100644 index 95894a0add008d78487896919cd85615281a9f96..0000000000000000000000000000000000000000 --- a/data/models/microsoft_phi-3-small-8k-instruct.json +++ /dev/null @@ -1,2036 +0,0 @@ -{ - "model_info": { - "name": "Phi-3 7B", - "id": "microsoft/phi-3-small-8k-instruct", - "developer": "microsoft", - "inference_platform": "unknown", - "additional_details": { - "model_id_aliases": [ - "microsoft/Phi-3-small-8k-instruct" - ] - } - }, - "evaluations": [ - { - "evaluation_id": "helm_lite/microsoft_phi-3-small-8k-instruct/1774096306.427425", - "retrieved_timestamp": "1774096306.427425", - "source_metadata": { - "source_name": "helm_lite", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_lite", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_lite", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.473, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.18641975308641975\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.754, - "details": { - "description": "min=0.754, mean=0.754, max=0.754, sum=0.754 (1)", - "tab": "Accuracy", - "NarrativeQA - Observed inference time (s)": "{\"description\": \"min=30.408, mean=30.408, max=30.408, sum=30.408 (1)\", \"tab\": \"Efficiency\", \"score\": \"30.40753108749927\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=3485.67, mean=3485.67, max=3485.67, sum=3485.67 (1)\", \"tab\": \"General information\", \"score\": \"3485.6704225352114\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=33.71, mean=33.71, max=33.71, sum=33.71 (1)\", \"tab\": \"General information\", \"score\": \"33.709859154929575\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (closed-book)", - "source_data": { - "dataset_name": "NaturalQuestions (closed-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (closed-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.324, - "details": { - "description": "min=0.324, mean=0.324, max=0.324, sum=0.324 (1)", - "tab": "Accuracy", - "NaturalQuestions (open-book) - Observed inference time (s)": "{\"description\": \"min=68.232, mean=68.232, max=68.232, sum=68.232 (1)\", \"tab\": \"Efficiency\", \"score\": \"68.2322377743721\"}", - "NaturalQuestions (closed-book) - Observed inference time (s)": "{\"description\": \"min=63.003, mean=63.003, max=63.003, sum=63.003 (1)\", \"tab\": \"Efficiency\", \"score\": \"63.00250503087044\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.965, mean=4.965, max=4.965, sum=4.965 (1)\", \"tab\": \"General information\", \"score\": \"4.965\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.007, mean=0.007, max=0.007, sum=0.007 (1)\", \"tab\": \"General information\", \"score\": \"0.007\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1675.981, mean=1675.981, max=1675.981, sum=1675.981 (1)\", \"tab\": \"General information\", \"score\": \"1675.981\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=16.786, mean=16.786, max=16.786, sum=16.786 (1)\", \"tab\": \"General information\", \"score\": \"16.786\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=129.127, mean=129.127, max=129.127, sum=129.127 (1)\", \"tab\": \"General information\", \"score\": \"129.127\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=36.311, mean=36.311, max=36.311, sum=36.311 (1)\", \"tab\": \"General information\", \"score\": \"36.311\"}" - } - }, - "generation_config": { - "additional_details": { - "mode": "\"closedbook\"" - } - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.912, - "details": { - "description": "min=0.912, mean=0.912, max=0.912, sum=0.912 (1)", - "tab": "Accuracy", - "OpenbookQA - Observed inference time (s)": "{\"description\": \"min=0.289, mean=0.289, max=0.289, sum=0.289 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.28856802701950074\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=249.782, mean=249.782, max=249.782, sum=249.782 (1)\", \"tab\": \"General information\", \"score\": \"249.782\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "dataset": "\"openbookqa\"", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.659, - "details": { - "description": "min=0.44, mean=0.659, max=0.95, sum=3.296 (5)", - "tab": "Accuracy", - "MMLU - Observed inference time (s)": "{\"description\": \"min=0.275, mean=0.406, max=0.549, sum=2.032 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.406433069689232\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=373.44, mean=467.72, max=614.43, sum=2338.6 (5)\", \"tab\": \"General information\", \"score\": \"467.71996491228066\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"college_chemistry\", \"computer_security\", \"econometrics\", \"us_foreign_policy\"]", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MATH", - "source_data": { - "dataset_name": "MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Equivalent (CoT) on MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.703, - "details": { - "description": "min=0.538, mean=0.703, max=0.933, sum=4.922 (7)", - "tab": "Accuracy", - "MATH - Observed inference time (s)": "{\"description\": \"min=49.379, mean=60.681, max=73.413, sum=424.765 (7)\", \"tab\": \"Efficiency\", \"score\": \"60.680695580739844\"}", - "MATH - # eval": "{\"description\": \"min=30, mean=62.429, max=135, sum=437 (7)\", \"tab\": \"General information\", \"score\": \"62.42857142857143\"}", - "MATH - # train": "{\"description\": \"min=8, mean=8, max=8, sum=56 (7)\", \"tab\": \"General information\", \"score\": \"8.0\"}", - "MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MATH - # prompt tokens": "{\"description\": \"min=881.363, mean=1262.911, max=2197.577, sum=8840.376 (7)\", \"tab\": \"General information\", \"score\": \"1262.9108741840687\"}", - "MATH - # output tokens": "{\"description\": \"min=57.779, mean=115.236, max=283.904, sum=806.654 (7)\", \"tab\": \"General information\", \"score\": \"115.23627800867702\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"algebra\", \"counting_and_probability\", \"geometry\", \"intermediate_algebra\", \"number_theory\", \"prealgebra\", \"precalculus\"]", - "level": "\"1\"", - "use_official_examples": "\"False\"", - "use_chain_of_thought": "\"True\"" - } - } - }, - { - "evaluation_name": "GSM8K", - "source_data": { - "dataset_name": "GSM8K", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on GSM8K", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "GSM8K - Observed inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "GSM8K - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "GSM8K - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "GSM8K - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "GSM8K - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "GSM8K - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "LegalBench", - "source_data": { - "dataset_name": "LegalBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on LegalBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.584, - "details": { - "description": "min=0.395, mean=0.584, max=0.895, sum=2.92 (5)", - "tab": "Accuracy", - "LegalBench - Observed inference time (s)": "{\"description\": \"min=6.293, mean=8.342, max=16.012, sum=41.71 (5)\", \"tab\": \"Efficiency\", \"score\": \"8.34200078530511\"}", - "LegalBench - # eval": "{\"description\": \"min=95, mean=409.4, max=1000, sum=2047 (5)\", \"tab\": \"General information\", \"score\": \"409.4\"}", - "LegalBench - # train": "{\"description\": \"min=4, mean=4.798, max=5, sum=23.992 (5)\", \"tab\": \"General information\", \"score\": \"4.798367346938775\"}", - "LegalBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "LegalBench - # prompt tokens": "{\"description\": \"min=197.442, mean=1512.687, max=6294.008, sum=7563.435 (5)\", \"tab\": \"General information\", \"score\": \"1512.6870529886412\"}", - "LegalBench - # output tokens": "{\"description\": \"min=1, mean=1.192, max=1.538, sum=5.96 (5)\", \"tab\": \"General information\", \"score\": \"1.192017037143267\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"abercrombie\", \"corporate_lobbying\", \"function_of_decision_section\", \"international_citizenship_questions\", \"proa\"]" - } - } - }, - { - "evaluation_name": "MedQA", - "source_data": { - "dataset_name": "MedQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MedQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.672, - "details": { - "description": "min=0.672, mean=0.672, max=0.672, sum=0.672 (1)", - "tab": "Accuracy", - "MedQA - Observed inference time (s)": "{\"description\": \"min=0.89, mean=0.89, max=0.89, sum=0.89 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.8902683931126983\"}", - "MedQA - # eval": "{\"description\": \"min=503, mean=503, max=503, sum=503 (1)\", \"tab\": \"General information\", \"score\": \"503.0\"}", - "MedQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MedQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MedQA - # prompt tokens": "{\"description\": \"min=1027.414, mean=1027.414, max=1027.414, sum=1027.414 (1)\", \"tab\": \"General information\", \"score\": \"1027.4135188866799\"}", - "MedQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WMT 2014", - "source_data": { - "dataset_name": "WMT 2014", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "BLEU-4 on WMT 2014", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.154, - "details": { - "description": "min=0.043, mean=0.154, max=0.205, sum=0.772 (5)", - "tab": "Accuracy", - "WMT 2014 - Observed inference time (s)": "{\"description\": \"min=20.252, mean=20.399, max=20.714, sum=101.996 (5)\", \"tab\": \"Efficiency\", \"score\": \"20.399208641134514\"}", - "WMT 2014 - # eval": "{\"description\": \"min=503, mean=568.8, max=832, sum=2844 (5)\", \"tab\": \"General information\", \"score\": \"568.8\"}", - "WMT 2014 - # train": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "WMT 2014 - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WMT 2014 - # prompt tokens": "{\"description\": \"min=114.901, mean=138.043, max=158.185, sum=690.213 (5)\", \"tab\": \"General information\", \"score\": \"138.04258583116683\"}", - "WMT 2014 - # output tokens": "{\"description\": \"min=96.311, mean=96.966, max=98.575, sum=484.832 (5)\", \"tab\": \"General information\", \"score\": \"96.96643456568283\"}" - } - }, - "generation_config": { - "additional_details": { - "language_pair": "[\"cs-en\", \"de-en\", \"fr-en\", \"hi-en\", \"ru-en\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_mmlu/microsoft_phi-3-small-8k-instruct/1774096312.00548", - "retrieved_timestamp": "1774096312.00548", - "source_metadata": { - "source_name": "helm_mmlu", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_mmlu", - "evaluation_results": [ - { - "evaluation_name": "MMLU All Subjects", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU All Subjects", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.757, - "details": { - "description": "min=0.44, mean=0.757, max=0.969, sum=86.273 (114)", - "tab": "Accuracy", - "MMLU All Subjects - Observed inference time (s)": "{\"description\": \"min=0.226, mean=0.38, max=1.284, sum=43.298 (114)\", \"tab\": \"Efficiency\", \"score\": \"0.379805443442311\"}", - "MMLU All Subjects - # eval": "{\"description\": \"min=100, mean=246.351, max=1534, sum=28084 (114)\", \"tab\": \"General information\", \"score\": \"246.35087719298247\"}", - "MMLU All Subjects - # train": "{\"description\": \"min=5, mean=5, max=5, sum=570 (114)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU All Subjects - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU All Subjects - # prompt tokens": "{\"description\": \"min=275.561, mean=614.852, max=2798.073, sum=70093.086 (114)\", \"tab\": \"General information\", \"score\": \"614.851634217556\"}", - "MMLU All Subjects - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=114 (114)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_name": "Abstract Algebra", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Abstract Algebra", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.44, - "details": { - "description": "min=0.44, mean=0.44, max=0.44, sum=0.88 (2)", - "tab": "Accuracy", - "Abstract Algebra - Observed inference time (s)": "{\"description\": \"min=0.505, mean=0.505, max=0.505, sum=1.009 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5047230005264283\"}", - "Abstract Algebra - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Abstract Algebra - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Abstract Algebra - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Abstract Algebra - # prompt tokens": "{\"description\": \"min=373.44, mean=373.44, max=373.44, sum=746.88 (2)\", \"tab\": \"General information\", \"score\": \"373.44\"}", - "Abstract Algebra - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"abstract_algebra\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_abstract_algebra\"" - } - } - }, - { - "evaluation_name": "Anatomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Anatomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.726, - "details": { - "description": "min=0.726, mean=0.726, max=0.726, sum=1.452 (2)", - "tab": "Accuracy", - "Anatomy - Observed inference time (s)": "{\"description\": \"min=0.412, mean=0.412, max=0.412, sum=0.825 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4122970881285491\"}", - "Anatomy - # eval": "{\"description\": \"min=135, mean=135, max=135, sum=270 (2)\", \"tab\": \"General information\", \"score\": \"135.0\"}", - "Anatomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Anatomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Anatomy - # prompt tokens": "{\"description\": \"min=353.978, mean=353.978, max=353.978, sum=707.956 (2)\", \"tab\": \"General information\", \"score\": \"353.97777777777776\"}", - "Anatomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"anatomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_anatomy\"" - } - } - }, - { - "evaluation_name": "College Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on College Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.559, - "details": { - "description": "min=0.559, mean=0.559, max=0.559, sum=1.118 (2)", - "tab": "Accuracy", - "College Chemistry - Observed inference time (s)": "{\"description\": \"min=0.341, mean=0.341, max=0.341, sum=0.683 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3414782953262329\"}", - "College Biology - Observed inference time (s)": "{\"description\": \"min=0.3, mean=0.3, max=0.3, sum=0.6 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3002290378014247\"}", - "College Computer Science - Observed inference time (s)": "{\"description\": \"min=0.447, mean=0.447, max=0.447, sum=0.894 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4468130707740784\"}", - "College Mathematics - Observed inference time (s)": "{\"description\": \"min=0.351, mean=0.351, max=0.351, sum=0.703 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.35149253606796266\"}", - "College Medicine - Observed inference time (s)": "{\"description\": \"min=0.323, mean=0.323, max=0.323, sum=0.646 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.32299859399740405\"}", - "College Physics - Observed inference time (s)": "{\"description\": \"min=0.322, mean=0.322, max=0.322, sum=0.644 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.32188768246594596\"}", - "College Chemistry - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Chemistry - # prompt tokens": "{\"description\": \"min=549.4, mean=549.4, max=549.4, sum=1098.8 (2)\", \"tab\": \"General information\", \"score\": \"549.4\"}", - "College Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Biology - # eval": "{\"description\": \"min=144, mean=144, max=144, sum=288 (2)\", \"tab\": \"General information\", \"score\": \"144.0\"}", - "College Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # prompt tokens": "{\"description\": \"min=473.917, mean=473.917, max=473.917, sum=947.833 (2)\", \"tab\": \"General information\", \"score\": \"473.9166666666667\"}", - "College Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # prompt tokens": "{\"description\": \"min=828.39, mean=828.39, max=828.39, sum=1656.78 (2)\", \"tab\": \"General information\", \"score\": \"828.39\"}", - "College Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Mathematics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # prompt tokens": "{\"description\": \"min=594.52, mean=594.52, max=594.52, sum=1189.04 (2)\", \"tab\": \"General information\", \"score\": \"594.52\"}", - "College Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Medicine - # eval": "{\"description\": \"min=173, mean=173, max=173, sum=346 (2)\", \"tab\": \"General information\", \"score\": \"173.0\"}", - "College Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # prompt tokens": "{\"description\": \"min=502.728, mean=502.728, max=502.728, sum=1005.457 (2)\", \"tab\": \"General information\", \"score\": \"502.728323699422\"}", - "College Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Physics - # eval": "{\"description\": \"min=102, mean=102, max=102, sum=204 (2)\", \"tab\": \"General information\", \"score\": \"102.0\"}", - "College Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # prompt tokens": "{\"description\": \"min=503.608, mean=503.608, max=503.608, sum=1007.216 (2)\", \"tab\": \"General information\", \"score\": \"503.6078431372549\"}", - "College Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"college_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_college_physics\"" - } - } - }, - { - "evaluation_name": "Computer Security", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Computer Security", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.77, - "details": { - "description": "min=0.77, mean=0.77, max=0.77, sum=1.54 (2)", - "tab": "Accuracy", - "Computer Security - Observed inference time (s)": "{\"description\": \"min=0.275, mean=0.275, max=0.275, sum=0.55 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2747947096824646\"}", - "Computer Security - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Computer Security - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Computer Security - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Computer Security - # prompt tokens": "{\"description\": \"min=378.54, mean=378.54, max=378.54, sum=757.08 (2)\", \"tab\": \"General information\", \"score\": \"378.54\"}", - "Computer Security - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"computer_security\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_computer_security\"" - } - } - }, - { - "evaluation_name": "Econometrics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Econometrics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.596, - "details": { - "description": "min=0.596, mean=0.596, max=0.596, sum=1.193 (2)", - "tab": "Accuracy", - "Econometrics - Observed inference time (s)": "{\"description\": \"min=0.362, mean=0.362, max=0.362, sum=0.724 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.36201402178981845\"}", - "Econometrics - # eval": "{\"description\": \"min=114, mean=114, max=114, sum=228 (2)\", \"tab\": \"General information\", \"score\": \"114.0\"}", - "Econometrics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Econometrics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Econometrics - # prompt tokens": "{\"description\": \"min=614.43, mean=614.43, max=614.43, sum=1228.86 (2)\", \"tab\": \"General information\", \"score\": \"614.4298245614035\"}", - "Econometrics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"econometrics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_econometrics\"" - } - } - }, - { - "evaluation_name": "Global Facts", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Global Facts", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.52, - "details": { - "description": "min=0.52, mean=0.52, max=0.52, sum=1.04 (2)", - "tab": "Accuracy", - "Global Facts - Observed inference time (s)": "{\"description\": \"min=0.509, mean=0.509, max=0.509, sum=1.018 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5091006135940552\"}", - "Global Facts - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Global Facts - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Global Facts - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Global Facts - # prompt tokens": "{\"description\": \"min=399.71, mean=399.71, max=399.71, sum=799.42 (2)\", \"tab\": \"General information\", \"score\": \"399.71\"}", - "Global Facts - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"global_facts\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_global_facts\"" - } - } - }, - { - "evaluation_name": "Jurisprudence", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Jurisprudence", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.843, - "details": { - "description": "min=0.843, mean=0.843, max=0.843, sum=1.685 (2)", - "tab": "Accuracy", - "Jurisprudence - Observed inference time (s)": "{\"description\": \"min=0.269, mean=0.269, max=0.269, sum=0.538 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2687692134468644\"}", - "Jurisprudence - # eval": "{\"description\": \"min=108, mean=108, max=108, sum=216 (2)\", \"tab\": \"General information\", \"score\": \"108.0\"}", - "Jurisprudence - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Jurisprudence - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Jurisprudence - # prompt tokens": "{\"description\": \"min=394.639, mean=394.639, max=394.639, sum=789.278 (2)\", \"tab\": \"General information\", \"score\": \"394.6388888888889\"}", - "Jurisprudence - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"jurisprudence\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_jurisprudence\"" - } - } - }, - { - "evaluation_name": "Philosophy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Philosophy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.82, - "details": { - "description": "min=0.82, mean=0.82, max=0.82, sum=1.64 (2)", - "tab": "Accuracy", - "Philosophy - Observed inference time (s)": "{\"description\": \"min=0.263, mean=0.263, max=0.263, sum=0.527 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.26347158346145483\"}", - "Philosophy - # eval": "{\"description\": \"min=311, mean=311, max=311, sum=622 (2)\", \"tab\": \"General information\", \"score\": \"311.0\"}", - "Philosophy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Philosophy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Philosophy - # prompt tokens": "{\"description\": \"min=329.084, mean=329.084, max=329.084, sum=658.167 (2)\", \"tab\": \"General information\", \"score\": \"329.08360128617363\"}", - "Philosophy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"philosophy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_philosophy\"" - } - } - }, - { - "evaluation_name": "Professional Psychology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Professional Psychology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.835, - "details": { - "description": "min=0.835, mean=0.835, max=0.835, sum=1.67 (2)", - "tab": "Accuracy", - "Professional Medicine - Observed inference time (s)": "{\"description\": \"min=0.536, mean=0.536, max=0.536, sum=1.073 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5363782968591241\"}", - "Professional Accounting - Observed inference time (s)": "{\"description\": \"min=0.373, mean=0.373, max=0.373, sum=0.746 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.37297873885919014\"}", - "Professional Law - Observed inference time (s)": "{\"description\": \"min=0.764, mean=0.764, max=0.764, sum=1.527 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7635687488620564\"}", - "Professional Psychology - Observed inference time (s)": "{\"description\": \"min=0.332, mean=0.332, max=0.332, sum=0.664 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3322232922697379\"}", - "Professional Medicine - # eval": "{\"description\": \"min=272, mean=272, max=272, sum=544 (2)\", \"tab\": \"General information\", \"score\": \"272.0\"}", - "Professional Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Medicine - # prompt tokens": "{\"description\": \"min=1094.585, mean=1094.585, max=1094.585, sum=2189.169 (2)\", \"tab\": \"General information\", \"score\": \"1094.5845588235295\"}", - "Professional Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Accounting - # eval": "{\"description\": \"min=282, mean=282, max=282, sum=564 (2)\", \"tab\": \"General information\", \"score\": \"282.0\"}", - "Professional Accounting - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Accounting - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # prompt tokens": "{\"description\": \"min=658.592, mean=658.592, max=658.592, sum=1317.184 (2)\", \"tab\": \"General information\", \"score\": \"658.5921985815603\"}", - "Professional Accounting - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Law - # eval": "{\"description\": \"min=1534, mean=1534, max=1534, sum=3068 (2)\", \"tab\": \"General information\", \"score\": \"1534.0\"}", - "Professional Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # prompt tokens": "{\"description\": \"min=1637.787, mean=1637.787, max=1637.787, sum=3275.574 (2)\", \"tab\": \"General information\", \"score\": \"1637.7868318122555\"}", - "Professional Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Psychology - # eval": "{\"description\": \"min=612, mean=612, max=612, sum=1224 (2)\", \"tab\": \"General information\", \"score\": \"612.0\"}", - "Professional Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # prompt tokens": "{\"description\": \"min=575.114, mean=575.114, max=575.114, sum=1150.229 (2)\", \"tab\": \"General information\", \"score\": \"575.1143790849674\"}", - "Professional Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"professional_psychology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_professional_psychology\"" - } - } - }, - { - "evaluation_name": "Us Foreign Policy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Us Foreign Policy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.95, - "details": { - "description": "min=0.95, mean=0.95, max=0.95, sum=1.9 (2)", - "tab": "Accuracy", - "Us Foreign Policy - Observed inference time (s)": "{\"description\": \"min=0.549, mean=0.549, max=0.549, sum=1.098 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5491553211212158\"}", - "Us Foreign Policy - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Us Foreign Policy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Us Foreign Policy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Us Foreign Policy - # prompt tokens": "{\"description\": \"min=422.79, mean=422.79, max=422.79, sum=845.58 (2)\", \"tab\": \"General information\", \"score\": \"422.79\"}", - "Us Foreign Policy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"us_foreign_policy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_us_foreign_policy\"" - } - } - }, - { - "evaluation_name": "Astronomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Astronomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.849, - "details": { - "description": "min=0.849, mean=0.849, max=0.849, sum=1.697 (2)", - "tab": "Accuracy", - "Astronomy - Observed inference time (s)": "{\"description\": \"min=0.352, mean=0.352, max=0.352, sum=0.704 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.35213252902030945\"}", - "Astronomy - # eval": "{\"description\": \"min=152, mean=152, max=152, sum=304 (2)\", \"tab\": \"General information\", \"score\": \"152.0\"}", - "Astronomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Astronomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Astronomy - # prompt tokens": "{\"description\": \"min=579.691, mean=579.691, max=579.691, sum=1159.382 (2)\", \"tab\": \"General information\", \"score\": \"579.6907894736842\"}", - "Astronomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"astronomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_astronomy\"" - } - } - }, - { - "evaluation_name": "Business Ethics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Business Ethics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.77, - "details": { - "description": "min=0.77, mean=0.77, max=0.77, sum=1.54 (2)", - "tab": "Accuracy", - "Business Ethics - Observed inference time (s)": "{\"description\": \"min=0.347, mean=0.347, max=0.347, sum=0.693 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.34657839775085447\"}", - "Business Ethics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Business Ethics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Business Ethics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Business Ethics - # prompt tokens": "{\"description\": \"min=569.52, mean=569.52, max=569.52, sum=1139.04 (2)\", \"tab\": \"General information\", \"score\": \"569.52\"}", - "Business Ethics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"business_ethics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_business_ethics\"" - } - } - }, - { - "evaluation_name": "Clinical Knowledge", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Clinical Knowledge", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.83, - "details": { - "description": "min=0.83, mean=0.83, max=0.83, sum=1.66 (2)", - "tab": "Accuracy", - "Clinical Knowledge - Observed inference time (s)": "{\"description\": \"min=0.286, mean=0.286, max=0.286, sum=0.572 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2858500345697943\"}", - "Clinical Knowledge - # eval": "{\"description\": \"min=265, mean=265, max=265, sum=530 (2)\", \"tab\": \"General information\", \"score\": \"265.0\"}", - "Clinical Knowledge - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Clinical Knowledge - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Clinical Knowledge - # prompt tokens": "{\"description\": \"min=397.947, mean=397.947, max=397.947, sum=795.894 (2)\", \"tab\": \"General information\", \"score\": \"397.94716981132075\"}", - "Clinical Knowledge - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"clinical_knowledge\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_clinical_knowledge\"" - } - } - }, - { - "evaluation_name": "Conceptual Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Conceptual Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.779, - "details": { - "description": "min=0.779, mean=0.779, max=0.779, sum=1.557 (2)", - "tab": "Accuracy", - "Conceptual Physics - Observed inference time (s)": "{\"description\": \"min=0.254, mean=0.254, max=0.254, sum=0.507 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2537446346688778\"}", - "Conceptual Physics - # eval": "{\"description\": \"min=235, mean=235, max=235, sum=470 (2)\", \"tab\": \"General information\", \"score\": \"235.0\"}", - "Conceptual Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Conceptual Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Conceptual Physics - # prompt tokens": "{\"description\": \"min=304.838, mean=304.838, max=304.838, sum=609.677 (2)\", \"tab\": \"General information\", \"score\": \"304.83829787234043\"}", - "Conceptual Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"conceptual_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_conceptual_physics\"" - } - } - }, - { - "evaluation_name": "Electrical Engineering", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Electrical Engineering", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.69, - "details": { - "description": "min=0.69, mean=0.69, max=0.69, sum=1.379 (2)", - "tab": "Accuracy", - "Electrical Engineering - Observed inference time (s)": "{\"description\": \"min=0.301, mean=0.301, max=0.301, sum=0.602 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3010375532610663\"}", - "Electrical Engineering - # eval": "{\"description\": \"min=145, mean=145, max=145, sum=290 (2)\", \"tab\": \"General information\", \"score\": \"145.0\"}", - "Electrical Engineering - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Electrical Engineering - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Electrical Engineering - # prompt tokens": "{\"description\": \"min=440.641, mean=440.641, max=440.641, sum=881.283 (2)\", \"tab\": \"General information\", \"score\": \"440.6413793103448\"}", - "Electrical Engineering - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"electrical_engineering\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_electrical_engineering\"" - } - } - }, - { - "evaluation_name": "Elementary Mathematics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Elementary Mathematics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.619, - "details": { - "description": "min=0.619, mean=0.619, max=0.619, sum=1.238 (2)", - "tab": "Accuracy", - "Elementary Mathematics - Observed inference time (s)": "{\"description\": \"min=0.338, mean=0.338, max=0.338, sum=0.676 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3380681862906804\"}", - "Elementary Mathematics - # eval": "{\"description\": \"min=378, mean=378, max=378, sum=756 (2)\", \"tab\": \"General information\", \"score\": \"378.0\"}", - "Elementary Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Elementary Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Elementary Mathematics - # prompt tokens": "{\"description\": \"min=531.862, mean=531.862, max=531.862, sum=1063.725 (2)\", \"tab\": \"General information\", \"score\": \"531.8624338624338\"}", - "Elementary Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"elementary_mathematics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_elementary_mathematics\"" - } - } - }, - { - "evaluation_name": "Formal Logic", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Formal Logic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.595, - "details": { - "description": "min=0.595, mean=0.595, max=0.595, sum=1.19 (2)", - "tab": "Accuracy", - "Formal Logic - Observed inference time (s)": "{\"description\": \"min=0.358, mean=0.358, max=0.358, sum=0.716 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.35805845071399023\"}", - "Formal Logic - # eval": "{\"description\": \"min=126, mean=126, max=126, sum=252 (2)\", \"tab\": \"General information\", \"score\": \"126.0\"}", - "Formal Logic - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Formal Logic - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Formal Logic - # prompt tokens": "{\"description\": \"min=606.762, mean=606.762, max=606.762, sum=1213.524 (2)\", \"tab\": \"General information\", \"score\": \"606.7619047619048\"}", - "Formal Logic - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"formal_logic\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_formal_logic\"" - } - } - }, - { - "evaluation_name": "High School World History", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on High School World History", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.848, - "details": { - "description": "min=0.848, mean=0.848, max=0.848, sum=1.696 (2)", - "tab": "Accuracy", - "High School Biology - Observed inference time (s)": "{\"description\": \"min=0.327, mean=0.327, max=0.327, sum=0.655 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.32748886615999284\"}", - "High School Chemistry - Observed inference time (s)": "{\"description\": \"min=0.311, mean=0.311, max=0.311, sum=0.622 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.31104220545350625\"}", - "High School Computer Science - Observed inference time (s)": "{\"description\": \"min=0.465, mean=0.465, max=0.465, sum=0.93 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4648329520225525\"}", - "High School European History - Observed inference time (s)": "{\"description\": \"min=1.284, mean=1.284, max=1.284, sum=2.569 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.2842581590016684\"}", - "High School Geography - Observed inference time (s)": "{\"description\": \"min=0.272, mean=0.272, max=0.272, sum=0.544 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.27224273031408136\"}", - "High School Government And Politics - Observed inference time (s)": "{\"description\": \"min=0.299, mean=0.299, max=0.299, sum=0.598 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2989391489967781\"}", - "High School Macroeconomics - Observed inference time (s)": "{\"description\": \"min=0.273, mean=0.273, max=0.273, sum=0.546 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2728824230340811\"}", - "High School Mathematics - Observed inference time (s)": "{\"description\": \"min=0.339, mean=0.339, max=0.339, sum=0.679 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.33938890828026663\"}", - "High School Microeconomics - Observed inference time (s)": "{\"description\": \"min=0.285, mean=0.285, max=0.285, sum=0.57 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.28512202290927663\"}", - "High School Physics - Observed inference time (s)": "{\"description\": \"min=0.35, mean=0.35, max=0.35, sum=0.7 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.34992847537362815\"}", - "High School Psychology - Observed inference time (s)": "{\"description\": \"min=0.316, mean=0.316, max=0.316, sum=0.633 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.31643713986108063\"}", - "High School Statistics - Observed inference time (s)": "{\"description\": \"min=0.439, mean=0.439, max=0.439, sum=0.878 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.43886349929703605\"}", - "High School US History - Observed inference time (s)": "{\"description\": \"min=1.004, mean=1.004, max=1.004, sum=2.009 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.0044469611317504\"}", - "High School World History - Observed inference time (s)": "{\"description\": \"min=0.677, mean=0.677, max=0.677, sum=1.354 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6767715281072045\"}", - "High School Biology - # eval": "{\"description\": \"min=310, mean=310, max=310, sum=620 (2)\", \"tab\": \"General information\", \"score\": \"310.0\"}", - "High School Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Biology - # prompt tokens": "{\"description\": \"min=513.677, mean=513.677, max=513.677, sum=1027.355 (2)\", \"tab\": \"General information\", \"score\": \"513.6774193548387\"}", - "High School Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Chemistry - # eval": "{\"description\": \"min=203, mean=203, max=203, sum=406 (2)\", \"tab\": \"General information\", \"score\": \"203.0\"}", - "High School Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # prompt tokens": "{\"description\": \"min=496.714, mean=496.714, max=496.714, sum=993.429 (2)\", \"tab\": \"General information\", \"score\": \"496.7142857142857\"}", - "High School Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "High School Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # prompt tokens": "{\"description\": \"min=867.78, mean=867.78, max=867.78, sum=1735.56 (2)\", \"tab\": \"General information\", \"score\": \"867.78\"}", - "High School Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School European History - # eval": "{\"description\": \"min=165, mean=165, max=165, sum=330 (2)\", \"tab\": \"General information\", \"score\": \"165.0\"}", - "High School European History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School European History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # prompt tokens": "{\"description\": \"min=2798.073, mean=2798.073, max=2798.073, sum=5596.145 (2)\", \"tab\": \"General information\", \"score\": \"2798.072727272727\"}", - "High School European History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Geography - # eval": "{\"description\": \"min=198, mean=198, max=198, sum=396 (2)\", \"tab\": \"General information\", \"score\": \"198.0\"}", - "High School Geography - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Geography - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # prompt tokens": "{\"description\": \"min=372.045, mean=372.045, max=372.045, sum=744.091 (2)\", \"tab\": \"General information\", \"score\": \"372.04545454545456\"}", - "High School Geography - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Government And Politics - # eval": "{\"description\": \"min=193, mean=193, max=193, sum=386 (2)\", \"tab\": \"General information\", \"score\": \"193.0\"}", - "High School Government And Politics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Government And Politics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # prompt tokens": "{\"description\": \"min=465.824, mean=465.824, max=465.824, sum=931.648 (2)\", \"tab\": \"General information\", \"score\": \"465.8238341968912\"}", - "High School Government And Politics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Macroeconomics - # eval": "{\"description\": \"min=390, mean=390, max=390, sum=780 (2)\", \"tab\": \"General information\", \"score\": \"390.0\"}", - "High School Macroeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Macroeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # prompt tokens": "{\"description\": \"min=371.562, mean=371.562, max=371.562, sum=743.123 (2)\", \"tab\": \"General information\", \"score\": \"371.5615384615385\"}", - "High School Macroeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Mathematics - # eval": "{\"description\": \"min=270, mean=270, max=270, sum=540 (2)\", \"tab\": \"General information\", \"score\": \"270.0\"}", - "High School Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # prompt tokens": "{\"description\": \"min=532.374, mean=532.374, max=532.374, sum=1064.748 (2)\", \"tab\": \"General information\", \"score\": \"532.3740740740741\"}", - "High School Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Microeconomics - # eval": "{\"description\": \"min=238, mean=238, max=238, sum=476 (2)\", \"tab\": \"General information\", \"score\": \"238.0\"}", - "High School Microeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Microeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # prompt tokens": "{\"description\": \"min=399.025, mean=399.025, max=399.025, sum=798.05 (2)\", \"tab\": \"General information\", \"score\": \"399.02521008403363\"}", - "High School Microeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Physics - # eval": "{\"description\": \"min=151, mean=151, max=151, sum=302 (2)\", \"tab\": \"General information\", \"score\": \"151.0\"}", - "High School Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # prompt tokens": "{\"description\": \"min=560.464, mean=560.464, max=560.464, sum=1120.927 (2)\", \"tab\": \"General information\", \"score\": \"560.4635761589404\"}", - "High School Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Psychology - # eval": "{\"description\": \"min=545, mean=545, max=545, sum=1090 (2)\", \"tab\": \"General information\", \"score\": \"545.0\"}", - "High School Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # prompt tokens": "{\"description\": \"min=495.246, mean=495.246, max=495.246, sum=990.492 (2)\", \"tab\": \"General information\", \"score\": \"495.24587155963303\"}", - "High School Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Statistics - # eval": "{\"description\": \"min=216, mean=216, max=216, sum=432 (2)\", \"tab\": \"General information\", \"score\": \"216.0\"}", - "High School Statistics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Statistics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # prompt tokens": "{\"description\": \"min=795.699, mean=795.699, max=795.699, sum=1591.398 (2)\", \"tab\": \"General information\", \"score\": \"795.699074074074\"}", - "High School Statistics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School US History - # eval": "{\"description\": \"min=204, mean=204, max=204, sum=408 (2)\", \"tab\": \"General information\", \"score\": \"204.0\"}", - "High School US History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School US History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # prompt tokens": "{\"description\": \"min=2217.809, mean=2217.809, max=2217.809, sum=4435.618 (2)\", \"tab\": \"General information\", \"score\": \"2217.8088235294117\"}", - "High School US History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School World History - # eval": "{\"description\": \"min=237, mean=237, max=237, sum=474 (2)\", \"tab\": \"General information\", \"score\": \"237.0\"}", - "High School World History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School World History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # prompt tokens": "{\"description\": \"min=1428.27, mean=1428.27, max=1428.27, sum=2856.54 (2)\", \"tab\": \"General information\", \"score\": \"1428.2700421940929\"}", - "High School World History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"high_school_world_history\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_high_school_world_history\"" - } - } - }, - { - "evaluation_name": "Human Sexuality", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Human Sexuality", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.817, - "details": { - "description": "min=0.817, mean=0.817, max=0.817, sum=1.634 (2)", - "tab": "Accuracy", - "Human Aging - Observed inference time (s)": "{\"description\": \"min=0.258, mean=0.258, max=0.258, sum=0.515 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2577151257895568\"}", - "Human Sexuality - Observed inference time (s)": "{\"description\": \"min=0.264, mean=0.264, max=0.264, sum=0.529 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.26447626470609475\"}", - "Human Aging - # eval": "{\"description\": \"min=223, mean=223, max=223, sum=446 (2)\", \"tab\": \"General information\", \"score\": \"223.0\"}", - "Human Aging - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Aging - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Aging - # prompt tokens": "{\"description\": \"min=319.906, mean=319.906, max=319.906, sum=639.812 (2)\", \"tab\": \"General information\", \"score\": \"319.90582959641256\"}", - "Human Aging - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Human Sexuality - # eval": "{\"description\": \"min=131, mean=131, max=131, sum=262 (2)\", \"tab\": \"General information\", \"score\": \"131.0\"}", - "Human Sexuality - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Sexuality - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # prompt tokens": "{\"description\": \"min=341.183, mean=341.183, max=341.183, sum=682.366 (2)\", \"tab\": \"General information\", \"score\": \"341.1832061068702\"}", - "Human Sexuality - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"human_sexuality\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_human_sexuality\"" - } - } - }, - { - "evaluation_name": "International Law", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on International Law", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.851, - "details": { - "description": "min=0.851, mean=0.851, max=0.851, sum=1.702 (2)", - "tab": "Accuracy", - "International Law - Observed inference time (s)": "{\"description\": \"min=0.371, mean=0.371, max=0.371, sum=0.743 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3714516399320492\"}", - "International Law - # eval": "{\"description\": \"min=121, mean=121, max=121, sum=242 (2)\", \"tab\": \"General information\", \"score\": \"121.0\"}", - "International Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "International Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "International Law - # prompt tokens": "{\"description\": \"min=639.851, mean=639.851, max=639.851, sum=1279.702 (2)\", \"tab\": \"General information\", \"score\": \"639.8512396694215\"}", - "International Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"international_law\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_international_law\"" - } - } - }, - { - "evaluation_name": "Logical Fallacies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Logical Fallacies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.81, - "details": { - "description": "min=0.81, mean=0.81, max=0.81, sum=1.62 (2)", - "tab": "Accuracy", - "Logical Fallacies - Observed inference time (s)": "{\"description\": \"min=0.304, mean=0.304, max=0.304, sum=0.608 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.30408222543681324\"}", - "Logical Fallacies - # eval": "{\"description\": \"min=163, mean=163, max=163, sum=326 (2)\", \"tab\": \"General information\", \"score\": \"163.0\"}", - "Logical Fallacies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Logical Fallacies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Logical Fallacies - # prompt tokens": "{\"description\": \"min=449.595, mean=449.595, max=449.595, sum=899.19 (2)\", \"tab\": \"General information\", \"score\": \"449.5950920245399\"}", - "Logical Fallacies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"logical_fallacies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_logical_fallacies\"" - } - } - }, - { - "evaluation_name": "Machine Learning", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Machine Learning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.652, - "details": { - "description": "min=0.652, mean=0.652, max=0.652, sum=1.304 (2)", - "tab": "Accuracy", - "Machine Learning - Observed inference time (s)": "{\"description\": \"min=0.382, mean=0.382, max=0.382, sum=0.765 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3823078232152121\"}", - "Machine Learning - # eval": "{\"description\": \"min=112, mean=112, max=112, sum=224 (2)\", \"tab\": \"General information\", \"score\": \"112.0\"}", - "Machine Learning - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Machine Learning - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Machine Learning - # prompt tokens": "{\"description\": \"min=668.054, mean=668.054, max=668.054, sum=1336.107 (2)\", \"tab\": \"General information\", \"score\": \"668.0535714285714\"}", - "Machine Learning - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"machine_learning\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_machine_learning\"" - } - } - }, - { - "evaluation_name": "Management", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Management", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.903, - "details": { - "description": "min=0.903, mean=0.903, max=0.903, sum=1.806 (2)", - "tab": "Accuracy", - "Management - Observed inference time (s)": "{\"description\": \"min=0.255, mean=0.255, max=0.255, sum=0.511 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2552997649294659\"}", - "Management - # eval": "{\"description\": \"min=103, mean=103, max=103, sum=206 (2)\", \"tab\": \"General information\", \"score\": \"103.0\"}", - "Management - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Management - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Management - # prompt tokens": "{\"description\": \"min=283.796, mean=283.796, max=283.796, sum=567.592 (2)\", \"tab\": \"General information\", \"score\": \"283.79611650485435\"}", - "Management - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"management\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_management\"" - } - } - }, - { - "evaluation_name": "Marketing", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Marketing", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.897, - "details": { - "description": "min=0.897, mean=0.897, max=0.897, sum=1.795 (2)", - "tab": "Accuracy", - "Marketing - Observed inference time (s)": "{\"description\": \"min=0.291, mean=0.291, max=0.291, sum=0.582 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.29102008974450266\"}", - "Marketing - # eval": "{\"description\": \"min=234, mean=234, max=234, sum=468 (2)\", \"tab\": \"General information\", \"score\": \"234.0\"}", - "Marketing - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Marketing - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Marketing - # prompt tokens": "{\"description\": \"min=404.218, mean=404.218, max=404.218, sum=808.436 (2)\", \"tab\": \"General information\", \"score\": \"404.21794871794873\"}", - "Marketing - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"marketing\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_marketing\"" - } - } - }, - { - "evaluation_name": "Medical Genetics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Medical Genetics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.84, - "details": { - "description": "min=0.84, mean=0.84, max=0.84, sum=1.68 (2)", - "tab": "Accuracy", - "Medical Genetics - Observed inference time (s)": "{\"description\": \"min=0.27, mean=0.27, max=0.27, sum=0.54 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.27023372411727903\"}", - "Medical Genetics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Medical Genetics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Medical Genetics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Medical Genetics - # prompt tokens": "{\"description\": \"min=341, mean=341, max=341, sum=682 (2)\", \"tab\": \"General information\", \"score\": \"341.0\"}", - "Medical Genetics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"medical_genetics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_medical_genetics\"" - } - } - }, - { - "evaluation_name": "Miscellaneous", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Miscellaneous", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.871, - "details": { - "description": "min=0.871, mean=0.871, max=0.871, sum=1.742 (2)", - "tab": "Accuracy", - "Miscellaneous - Observed inference time (s)": "{\"description\": \"min=0.259, mean=0.259, max=0.259, sum=0.518 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.25915825382198565\"}", - "Miscellaneous - # eval": "{\"description\": \"min=783, mean=783, max=783, sum=1566 (2)\", \"tab\": \"General information\", \"score\": \"783.0\"}", - "Miscellaneous - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Miscellaneous - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Miscellaneous - # prompt tokens": "{\"description\": \"min=299.925, mean=299.925, max=299.925, sum=599.849 (2)\", \"tab\": \"General information\", \"score\": \"299.92464878671774\"}", - "Miscellaneous - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"miscellaneous\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_miscellaneous\"" - } - } - }, - { - "evaluation_name": "Moral Scenarios", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Moral Scenarios", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.711, - "details": { - "description": "min=0.711, mean=0.711, max=0.711, sum=1.421 (2)", - "tab": "Accuracy", - "Moral Disputes - Observed inference time (s)": "{\"description\": \"min=0.308, mean=0.308, max=0.308, sum=0.617 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3084571650951584\"}", - "Moral Scenarios - Observed inference time (s)": "{\"description\": \"min=0.383, mean=0.383, max=0.383, sum=0.766 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3827664223463176\"}", - "Moral Disputes - # eval": "{\"description\": \"min=346, mean=346, max=346, sum=692 (2)\", \"tab\": \"General information\", \"score\": \"346.0\"}", - "Moral Disputes - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Disputes - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Disputes - # prompt tokens": "{\"description\": \"min=476.145, mean=476.145, max=476.145, sum=952.289 (2)\", \"tab\": \"General information\", \"score\": \"476.1445086705202\"}", - "Moral Disputes - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Moral Scenarios - # eval": "{\"description\": \"min=895, mean=895, max=895, sum=1790 (2)\", \"tab\": \"General information\", \"score\": \"895.0\"}", - "Moral Scenarios - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Scenarios - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # prompt tokens": "{\"description\": \"min=656.455, mean=656.455, max=656.455, sum=1312.909 (2)\", \"tab\": \"General information\", \"score\": \"656.454748603352\"}", - "Moral Scenarios - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"moral_scenarios\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_moral_scenarios\"" - } - } - }, - { - "evaluation_name": "Nutrition", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Nutrition", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.833, - "details": { - "description": "min=0.833, mean=0.833, max=0.833, sum=1.667 (2)", - "tab": "Accuracy", - "Nutrition - Observed inference time (s)": "{\"description\": \"min=0.349, mean=0.349, max=0.349, sum=0.699 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.34937040011088055\"}", - "Nutrition - # eval": "{\"description\": \"min=306, mean=306, max=306, sum=612 (2)\", \"tab\": \"General information\", \"score\": \"306.0\"}", - "Nutrition - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Nutrition - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Nutrition - # prompt tokens": "{\"description\": \"min=586.817, mean=586.817, max=586.817, sum=1173.634 (2)\", \"tab\": \"General information\", \"score\": \"586.8169934640523\"}", - "Nutrition - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"nutrition\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_nutrition\"" - } - } - }, - { - "evaluation_name": "Prehistory", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Prehistory", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.858, - "details": { - "description": "min=0.858, mean=0.858, max=0.858, sum=1.716 (2)", - "tab": "Accuracy", - "Prehistory - Observed inference time (s)": "{\"description\": \"min=0.325, mean=0.325, max=0.325, sum=0.649 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.32473731188126553\"}", - "Prehistory - # eval": "{\"description\": \"min=324, mean=324, max=324, sum=648 (2)\", \"tab\": \"General information\", \"score\": \"324.0\"}", - "Prehistory - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Prehistory - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Prehistory - # prompt tokens": "{\"description\": \"min=514.559, mean=514.559, max=514.559, sum=1029.117 (2)\", \"tab\": \"General information\", \"score\": \"514.5586419753087\"}", - "Prehistory - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"prehistory\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_prehistory\"" - } - } - }, - { - "evaluation_name": "Public Relations", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Public Relations", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.727, - "details": { - "description": "min=0.727, mean=0.727, max=0.727, sum=1.455 (2)", - "tab": "Accuracy", - "Public Relations - Observed inference time (s)": "{\"description\": \"min=0.259, mean=0.259, max=0.259, sum=0.517 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2587012074210427\"}", - "Public Relations - # eval": "{\"description\": \"min=110, mean=110, max=110, sum=220 (2)\", \"tab\": \"General information\", \"score\": \"110.0\"}", - "Public Relations - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Public Relations - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Public Relations - # prompt tokens": "{\"description\": \"min=405.318, mean=405.318, max=405.318, sum=810.636 (2)\", \"tab\": \"General information\", \"score\": \"405.3181818181818\"}", - "Public Relations - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"public_relations\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_public_relations\"" - } - } - }, - { - "evaluation_name": "Security Studies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Security Studies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.804, - "details": { - "description": "min=0.804, mean=0.804, max=0.804, sum=1.608 (2)", - "tab": "Accuracy", - "Security Studies - Observed inference time (s)": "{\"description\": \"min=0.528, mean=0.528, max=0.528, sum=1.057 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5282714629659847\"}", - "Security Studies - # eval": "{\"description\": \"min=245, mean=245, max=245, sum=490 (2)\", \"tab\": \"General information\", \"score\": \"245.0\"}", - "Security Studies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Security Studies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Security Studies - # prompt tokens": "{\"description\": \"min=1164.473, mean=1164.473, max=1164.473, sum=2328.947 (2)\", \"tab\": \"General information\", \"score\": \"1164.4734693877551\"}", - "Security Studies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"security_studies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_security_studies\"" - } - } - }, - { - "evaluation_name": "Sociology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Sociology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.886, - "details": { - "description": "min=0.886, mean=0.886, max=0.886, sum=1.771 (2)", - "tab": "Accuracy", - "Sociology - Observed inference time (s)": "{\"description\": \"min=0.267, mean=0.267, max=0.267, sum=0.534 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2668588197053368\"}", - "Sociology - # eval": "{\"description\": \"min=201, mean=201, max=201, sum=402 (2)\", \"tab\": \"General information\", \"score\": \"201.0\"}", - "Sociology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Sociology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Sociology - # prompt tokens": "{\"description\": \"min=445.522, mean=445.522, max=445.522, sum=891.045 (2)\", \"tab\": \"General information\", \"score\": \"445.5223880597015\"}", - "Sociology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"sociology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_sociology\"" - } - } - }, - { - "evaluation_name": "Virology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Virology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.548, - "details": { - "description": "min=0.548, mean=0.548, max=0.548, sum=1.096 (2)", - "tab": "Accuracy", - "Virology - Observed inference time (s)": "{\"description\": \"min=0.235, mean=0.235, max=0.235, sum=0.47 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.235107473580234\"}", - "Virology - # eval": "{\"description\": \"min=166, mean=166, max=166, sum=332 (2)\", \"tab\": \"General information\", \"score\": \"166.0\"}", - "Virology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Virology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Virology - # prompt tokens": "{\"description\": \"min=343.09, mean=343.09, max=343.09, sum=686.181 (2)\", \"tab\": \"General information\", \"score\": \"343.0903614457831\"}", - "Virology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"virology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_virology\"" - } - } - }, - { - "evaluation_name": "World Religions", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on World Religions", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.825, - "details": { - "description": "min=0.825, mean=0.825, max=0.825, sum=1.649 (2)", - "tab": "Accuracy", - "World Religions - Observed inference time (s)": "{\"description\": \"min=0.226, mean=0.226, max=0.226, sum=0.453 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.22640645016006558\"}", - "World Religions - # eval": "{\"description\": \"min=171, mean=171, max=171, sum=342 (2)\", \"tab\": \"General information\", \"score\": \"171.0\"}", - "World Religions - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "World Religions - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "World Religions - # prompt tokens": "{\"description\": \"min=275.561, mean=275.561, max=275.561, sum=551.123 (2)\", \"tab\": \"General information\", \"score\": \"275.56140350877195\"}", - "World Religions - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"world_religions\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_world_religions\"" - } - } - }, - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.708, - "details": { - "description": "", - "tab": "Efficiency" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_id": "hfopenllm_v2/microsoft_Phi-3-small-8k-instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6497 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6208 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1887 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3121 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4558 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4506 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/microsoft_phi-3.5-mini-instruct.json b/data/models/microsoft_phi-3.5-mini-instruct.json deleted file mode 100644 index 0c7b3d27509a5564665caf45191b6a6eadade7e1..0000000000000000000000000000000000000000 --- a/data/models/microsoft_phi-3.5-mini-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Phi-3.5-mini-instruct", - "id": "microsoft/Phi-3.5-mini-instruct", - "developer": "microsoft", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Phi3ForCausalLM", - "params_billions": "3.821" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/microsoft_Phi-3.5-mini-instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5775 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5518 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1964 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3398 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4021 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3962 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/microsoft_phi-3.5-moe-instruct.json b/data/models/microsoft_phi-3.5-moe-instruct.json deleted file mode 100644 index b2c26a474cca0e16100985a07af4d12b3c01fcee..0000000000000000000000000000000000000000 --- a/data/models/microsoft_phi-3.5-moe-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Phi-3.5-MoE-instruct", - "id": "microsoft/Phi-3.5-MoE-instruct", - "developer": "microsoft", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Phi3ForCausalLM", - "params_billions": "42.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/microsoft_Phi-3.5-MoE-instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6925 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6408 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3119 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3557 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4565 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4658 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/microsoft_phi-4-mini-instruct.json b/data/models/microsoft_phi-4-mini-instruct.json deleted file mode 100644 index d5b4953dc9f3663f10e00454b7f154d0067e8661..0000000000000000000000000000000000000000 --- a/data/models/microsoft_phi-4-mini-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Phi-4-mini-instruct", - "id": "microsoft/Phi-4-mini-instruct", - "developer": "microsoft", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Phi3ForCausalLM", - "params_billions": "3.836" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/microsoft_Phi-4-mini-instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7378 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5689 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1699 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3096 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3873 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3932 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/microsoft_phi-4-prompt.json b/data/models/microsoft_phi-4-prompt.json deleted file mode 100644 index 1e2a92068efad1f3dcb63e6594e807c51e9f850f..0000000000000000000000000000000000000000 --- a/data/models/microsoft_phi-4-prompt.json +++ /dev/null @@ -1,905 +0,0 @@ -{ - "model_info": { - "name": "Phi-4 (Prompt)", - "id": "microsoft/phi-4-prompt", - "developer": "microsoft", - "additional_details": { - "raw_model_name": "Phi-4 (Prompt)", - "organization": "Microsoft", - "license": "MIT", - "mode": "Prompt", - "model_link": "https://huggingface.co/microsoft/phi-4" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/microsoft/phi-4-prompt/1775236112.402576", - "retrieved_timestamp": "1775236112.402576", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 70.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 28.79 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 8.72 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 9.49 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 26.73 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 23.02 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 69.56 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 74.25 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 89.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 65.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 49.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 60.7 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 65.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 59.64 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 81.25 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 41.67 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 3.88 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 9.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 3.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 3.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 4.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 4.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 5.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 24.73 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 17.42 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 25.16 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 31.61 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 50.0 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 87.55 - } - }, - { - "evaluation_result_id": "bfcl.format_sensitivity.max_delta::max_delta", - "evaluation_name": "bfcl.format_sensitivity.max_delta", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.format_sensitivity.max_delta", - "metric_name": "Format sensitivity max delta", - "metric_kind": "difference", - "metric_unit": "percentage_points", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Format Sensitivity Max Delta" - } - }, - "score_details": { - "score": 81.5 - } - }, - { - "evaluation_result_id": "bfcl.format_sensitivity.stddev::stddev", - "evaluation_name": "bfcl.format_sensitivity.stddev", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.format_sensitivity.stddev", - "metric_name": "Format sensitivity standard deviation", - "metric_kind": "difference", - "metric_unit": "percentage_points", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Format Sensitivity Standard Deviation" - } - }, - "score_details": { - "score": 23.34 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/microsoft_phi-4.json b/data/models/microsoft_phi-4.json deleted file mode 100644 index c9fca4565946dbb26825690fda8366b38bda6709..0000000000000000000000000000000000000000 --- a/data/models/microsoft_phi-4.json +++ /dev/null @@ -1,275 +0,0 @@ -{ - "model_info": { - "name": "phi-4", - "id": "microsoft/phi-4", - "developer": "microsoft", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Phi3ForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/microsoft_phi-4/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0488 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6703 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2787 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.401 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5034 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5295 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "hfopenllm_v2/microsoft_phi-4/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0585 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6691 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3165 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.406 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5034 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5287 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/microsoft_tnlg-v2-530b.json b/data/models/microsoft_tnlg-v2-530b.json deleted file mode 100644 index e5fdf9aa734c569d4503c0d87fa074d8c89c917c..0000000000000000000000000000000000000000 --- a/data/models/microsoft_tnlg-v2-530b.json +++ /dev/null @@ -1,674 +0,0 @@ -{ - "model_info": { - "name": "TNLG v2 530B", - "id": "microsoft/TNLG-v2-530B", - "developer": "microsoft", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_classic/microsoft_TNLG-v2-530B/1774096308.339228", - "retrieved_timestamp": "1774096308.339228", - "source_metadata": { - "source_name": "helm_classic", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_classic", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_classic", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperform on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.787, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Calibration": "{\"description\": \"\", \"tab\": \"Calibration\", \"score\": \"0.6152996196936993\"}", - "Mean win rate - Robustness": "{\"description\": \"\", \"tab\": \"Robustness\", \"score\": \"0.6503510949562118\"}", - "Mean win rate - Fairness": "{\"description\": \"\", \"tab\": \"Fairness\", \"score\": \"0.7516679834811092\"}", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}", - "Mean win rate - Bias": "{\"description\": \"\", \"tab\": \"Bias\", \"score\": \"0.5308990441173578\"}", - "Mean win rate - Toxicity": "{\"description\": \"\", \"tab\": \"Toxicity\", \"score\": \"0.3298371381704715\"}", - "Mean win rate - Summarization metrics": "{\"description\": \"\", \"tab\": \"Summarization metrics\", \"score\": \"0.756578947368421\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.469, - "details": { - "description": "min=0.24, mean=0.469, max=0.78, sum=7.035 (15)", - "tab": "Accuracy", - "MMLU - ECE (10-bin)": "{\"description\": \"min=0.073, mean=0.127, max=0.202, sum=1.908 (15)\", \"tab\": \"Calibration\", \"score\": \"0.12722994020701678\"}", - "MMLU - EM (Robustness)": "{\"description\": \"min=0.15, mean=0.403, max=0.75, sum=6.051 (15)\", \"tab\": \"Robustness\", \"score\": \"0.40336842105263154\"}", - "MMLU - EM (Fairness)": "{\"description\": \"min=0.17, mean=0.418, max=0.75, sum=6.266 (15)\", \"tab\": \"Fairness\", \"score\": \"0.41770760233918125\"}", - "MMLU - Denoised inference time (s)": "{\"description\": \"5 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=1542 (15)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=75 (15)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (15)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=371.38, mean=472.274, max=624.07, sum=7084.111 (15)\", \"tab\": \"General information\", \"score\": \"472.2740350877193\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=15 (15)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "MMLU - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=45 (15)\", \"tab\": \"General information\", \"score\": \"3.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "BoolQ", - "source_data": { - "dataset_name": "BoolQ", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on BoolQ", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.809, - "details": { - "description": "min=0.798, mean=0.809, max=0.829, sum=2.428 (3)", - "tab": "Accuracy", - "BoolQ - ECE (10-bin)": "{\"description\": \"min=0.017, mean=0.048, max=0.088, sum=0.144 (3)\", \"tab\": \"Calibration\", \"score\": \"0.04811928896988451\"}", - "BoolQ - EM (Robustness)": "{\"description\": \"min=0.724, mean=0.733, max=0.747, sum=2.198 (3)\", \"tab\": \"Robustness\", \"score\": \"0.7326666666666667\"}", - "BoolQ - EM (Fairness)": "{\"description\": \"min=0.756, mean=0.767, max=0.777, sum=2.3 (3)\", \"tab\": \"Fairness\", \"score\": \"0.7666666666666667\"}", - "BoolQ - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "BoolQ - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "BoolQ - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "BoolQ - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "BoolQ - # prompt tokens": "{\"description\": \"min=660.073, mean=908.406, max=1242.073, sum=2725.219 (3)\", \"tab\": \"General information\", \"score\": \"908.4063333333334\"}", - "BoolQ - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=3 (3)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "BoolQ - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "BoolQ - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.722, - "details": { - "description": "min=0.692, mean=0.722, max=0.743, sum=2.166 (3)", - "tab": "Accuracy", - "NarrativeQA - ECE (10-bin)": "{\"description\": \"min=0.026, mean=0.05, max=0.075, sum=0.15 (3)\", \"tab\": \"Calibration\", \"score\": \"0.05012197972633472\"}", - "NarrativeQA - F1 (Robustness)": "{\"description\": \"min=0.22, mean=0.319, max=0.405, sum=0.957 (3)\", \"tab\": \"Robustness\", \"score\": \"0.31894751591392195\"}", - "NarrativeQA - F1 (Fairness)": "{\"description\": \"min=0.601, mean=0.632, max=0.664, sum=1.895 (3)\", \"tab\": \"Fairness\", \"score\": \"0.6318169391667601\"}", - "NarrativeQA - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=1065 (3)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=1.051, mean=1.646, max=2.085, sum=4.938 (3)\", \"tab\": \"General information\", \"score\": \"1.6460093896713615\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=1600.366, mean=1651.848, max=1705.003, sum=4955.544 (3)\", \"tab\": \"General information\", \"score\": \"1651.8478873239437\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=5.113, mean=5.982, max=7.265, sum=17.946 (3)\", \"tab\": \"General information\", \"score\": \"5.982159624413145\"}", - "NarrativeQA - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NarrativeQA - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NarrativeQA - Stereotypes (gender)": "{\"description\": \"min=0.375, mean=0.395, max=0.436, sum=1.186 (3)\", \"tab\": \"Bias\", \"score\": \"0.3952991452991453\"}", - "NarrativeQA - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NarrativeQA - Representation (gender)": "{\"description\": \"min=0.204, mean=0.221, max=0.239, sum=0.663 (3)\", \"tab\": \"Bias\", \"score\": \"0.22112892189926373\"}", - "NarrativeQA - Toxic fraction": "{\"description\": \"min=0.011, mean=0.012, max=0.014, sum=0.037 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.012206572769953052\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (open-book)", - "source_data": { - "dataset_name": "NaturalQuestions (open-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (open-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.642, - "details": { - "description": "min=0.617, mean=0.642, max=0.656, sum=1.926 (3)", - "tab": "Accuracy", - "NaturalQuestions (closed-book) - ECE (10-bin)": "{\"description\": \"min=0.038, mean=0.04, max=0.041, sum=0.119 (3)\", \"tab\": \"Calibration\", \"score\": \"0.039723290660202144\"}", - "NaturalQuestions (open-book) - ECE (10-bin)": "{\"description\": \"min=0.071, mean=0.075, max=0.078, sum=0.225 (3)\", \"tab\": \"Calibration\", \"score\": \"0.07490014228309726\"}", - "NaturalQuestions (closed-book) - F1 (Robustness)": "{\"description\": \"min=0.291, mean=0.307, max=0.322, sum=0.922 (3)\", \"tab\": \"Robustness\", \"score\": \"0.3074701383832172\"}", - "NaturalQuestions (open-book) - F1 (Robustness)": "{\"description\": \"min=0.483, mean=0.525, max=0.549, sum=1.576 (3)\", \"tab\": \"Robustness\", \"score\": \"0.5253631735860874\"}", - "NaturalQuestions (closed-book) - F1 (Fairness)": "{\"description\": \"min=0.306, mean=0.318, max=0.324, sum=0.953 (3)\", \"tab\": \"Fairness\", \"score\": \"0.3175020164111731\"}", - "NaturalQuestions (open-book) - F1 (Fairness)": "{\"description\": \"min=0.575, mean=0.598, max=0.61, sum=1.794 (3)\", \"tab\": \"Fairness\", \"score\": \"0.5979278798197498\"}", - "NaturalQuestions (closed-book) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NaturalQuestions (open-book) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=110.254, mean=112.254, max=116.254, sum=336.762 (3)\", \"tab\": \"General information\", \"score\": \"112.254\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=3.8, mean=4.569, max=5.632, sum=13.707 (3)\", \"tab\": \"General information\", \"score\": \"4.569\"}", - "NaturalQuestions (closed-book) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.647, mean=4.691, max=4.723, sum=14.072 (3)\", \"tab\": \"General information\", \"score\": \"4.690666666666666\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.036, mean=0.036, max=0.036, sum=0.108 (3)\", \"tab\": \"General information\", \"score\": \"0.036\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1231.056, mean=1419.328, max=1523.222, sum=4257.983 (3)\", \"tab\": \"General information\", \"score\": \"1419.3276666666668\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=5.953, mean=6.015, max=6.134, sum=18.045 (3)\", \"tab\": \"General information\", \"score\": \"6.015000000000001\"}", - "NaturalQuestions (open-book) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NaturalQuestions (closed-book) - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - Stereotypes (gender)": "{\"description\": \"min=0.25, mean=0.342, max=0.443, sum=1.026 (3)\", \"tab\": \"Bias\", \"score\": \"0.342063492063492\"}", - "NaturalQuestions (closed-book) - Representation (race)": "{\"description\": \"min=0.53, mean=0.559, max=0.573, sum=1.676 (3)\", \"tab\": \"Bias\", \"score\": \"0.5587121212121212\"}", - "NaturalQuestions (closed-book) - Representation (gender)": "{\"description\": \"min=0.206, mean=0.289, max=0.419, sum=0.867 (3)\", \"tab\": \"Bias\", \"score\": \"0.2891147156537034\"}", - "NaturalQuestions (open-book) - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=0.667 (1)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666667\"}", - "NaturalQuestions (open-book) - Stereotypes (gender)": "{\"description\": \"min=0.191, mean=0.277, max=0.345, sum=0.83 (3)\", \"tab\": \"Bias\", \"score\": \"0.27656250000000004\"}", - "NaturalQuestions (open-book) - Representation (race)": "{\"description\": \"min=0.457, mean=0.469, max=0.484, sum=1.408 (3)\", \"tab\": \"Bias\", \"score\": \"0.4693006584979578\"}", - "NaturalQuestions (open-book) - Representation (gender)": "{\"description\": \"min=0.254, mean=0.259, max=0.261, sum=0.776 (3)\", \"tab\": \"Bias\", \"score\": \"0.2587447378492154\"}", - "NaturalQuestions (closed-book) - Toxic fraction": "{\"description\": \"min=0, mean=0.001, max=0.002, sum=0.003 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.001\"}", - "NaturalQuestions (open-book) - Toxic fraction": "{\"description\": \"min=0, mean=0.001, max=0.001, sum=0.002 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.0006666666666666666\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "QuAC", - "source_data": { - "dataset_name": "QuAC", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on QuAC", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.39, - "details": { - "description": "min=0.388, mean=0.39, max=0.393, sum=1.171 (3)", - "tab": "Accuracy", - "QuAC - ECE (10-bin)": "{\"description\": \"min=0.059, mean=0.08, max=0.106, sum=0.241 (3)\", \"tab\": \"Calibration\", \"score\": \"0.08020003145494241\"}", - "QuAC - F1 (Robustness)": "{\"description\": \"min=0.183, mean=0.194, max=0.203, sum=0.583 (3)\", \"tab\": \"Robustness\", \"score\": \"0.19421481147358363\"}", - "QuAC - F1 (Fairness)": "{\"description\": \"min=0.304, mean=0.313, max=0.32, sum=0.94 (3)\", \"tab\": \"Fairness\", \"score\": \"0.3132392185201357\"}", - "QuAC - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "QuAC - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "QuAC - # train": "{\"description\": \"min=0.845, mean=0.944, max=1.084, sum=2.831 (3)\", \"tab\": \"General information\", \"score\": \"0.9436666666666667\"}", - "QuAC - truncated": "{\"description\": \"min=0.016, mean=0.016, max=0.016, sum=0.048 (3)\", \"tab\": \"General information\", \"score\": \"0.016\"}", - "QuAC - # prompt tokens": "{\"description\": \"min=1624.371, mean=1644.436, max=1670.589, sum=4933.308 (3)\", \"tab\": \"General information\", \"score\": \"1644.436\"}", - "QuAC - # output tokens": "{\"description\": \"min=25.915, mean=29.956, max=32.756, sum=89.867 (3)\", \"tab\": \"General information\", \"score\": \"29.95566666666667\"}", - "QuAC - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "QuAC - Stereotypes (race)": "{\"description\": \"min=0.56, mean=0.579, max=0.599, sum=1.738 (3)\", \"tab\": \"Bias\", \"score\": \"0.5794166151309009\"}", - "QuAC - Stereotypes (gender)": "{\"description\": \"min=0.428, mean=0.435, max=0.448, sum=1.305 (3)\", \"tab\": \"Bias\", \"score\": \"0.43504680341335694\"}", - "QuAC - Representation (race)": "{\"description\": \"min=0.282, mean=0.333, max=0.369, sum=0.999 (3)\", \"tab\": \"Bias\", \"score\": \"0.33315102716024375\"}", - "QuAC - Representation (gender)": "{\"description\": \"min=0.24, mean=0.25, max=0.259, sum=0.75 (3)\", \"tab\": \"Bias\", \"score\": \"0.2499075403684782\"}", - "QuAC - Toxic fraction": "{\"description\": \"min=0.002, mean=0.003, max=0.003, sum=0.008 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.0026666666666666666\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "HellaSwag", - "source_data": { - "dataset_name": "HellaSwag", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on HellaSwag", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.799, - "details": { - "description": "min=0.799, mean=0.799, max=0.799, sum=0.799 (1)", - "tab": "Accuracy", - "HellaSwag - ECE (10-bin)": "{\"description\": \"min=0.322, mean=0.322, max=0.322, sum=0.322 (1)\", \"tab\": \"Calibration\", \"score\": \"0.32242755675811835\"}", - "HellaSwag - EM (Robustness)": "{\"description\": \"min=0.757, mean=0.757, max=0.757, sum=0.757 (1)\", \"tab\": \"Robustness\", \"score\": \"0.757\"}", - "HellaSwag - EM (Fairness)": "{\"description\": \"min=0.678, mean=0.678, max=0.678, sum=0.678 (1)\", \"tab\": \"Fairness\", \"score\": \"0.678\"}", - "HellaSwag - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "HellaSwag - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "HellaSwag - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "HellaSwag - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "HellaSwag - # prompt tokens": "{\"description\": \"min=87.888, mean=87.888, max=87.888, sum=87.888 (1)\", \"tab\": \"General information\", \"score\": \"87.888\"}", - "HellaSwag - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "HellaSwag - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.562, - "details": { - "description": "min=0.562, mean=0.562, max=0.562, sum=0.562 (1)", - "tab": "Accuracy", - "OpenbookQA - ECE (10-bin)": "{\"description\": \"min=0.243, mean=0.243, max=0.243, sum=0.243 (1)\", \"tab\": \"Calibration\", \"score\": \"0.2425759072363007\"}", - "OpenbookQA - EM (Robustness)": "{\"description\": \"min=0.476, mean=0.476, max=0.476, sum=0.476 (1)\", \"tab\": \"Robustness\", \"score\": \"0.476\"}", - "OpenbookQA - EM (Fairness)": "{\"description\": \"min=0.504, mean=0.504, max=0.504, sum=0.504 (1)\", \"tab\": \"Fairness\", \"score\": \"0.504\"}", - "OpenbookQA - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=5.27, mean=5.27, max=5.27, sum=5.27 (1)\", \"tab\": \"General information\", \"score\": \"5.27\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "TruthfulQA", - "source_data": { - "dataset_name": "TruthfulQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on TruthfulQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.251, - "details": { - "description": "min=0.22, mean=0.251, max=0.275, sum=0.752 (3)", - "tab": "Accuracy", - "TruthfulQA - ECE (10-bin)": "{\"description\": \"min=0.174, mean=0.226, max=0.252, sum=0.678 (3)\", \"tab\": \"Calibration\", \"score\": \"0.22594889867402287\"}", - "TruthfulQA - EM (Robustness)": "{\"description\": \"min=0.187, mean=0.202, max=0.217, sum=0.607 (3)\", \"tab\": \"Robustness\", \"score\": \"0.20234454638124363\"}", - "TruthfulQA - EM (Fairness)": "{\"description\": \"min=0.177, mean=0.197, max=0.213, sum=0.59 (3)\", \"tab\": \"Fairness\", \"score\": \"0.19673802242609584\"}", - "TruthfulQA - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "TruthfulQA - # eval": "{\"description\": \"min=654, mean=654, max=654, sum=1962 (3)\", \"tab\": \"General information\", \"score\": \"654.0\"}", - "TruthfulQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "TruthfulQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "TruthfulQA - # prompt tokens": "{\"description\": \"min=501.121, mean=511.121, max=529.121, sum=1533.362 (3)\", \"tab\": \"General information\", \"score\": \"511.12079510703364\"}", - "TruthfulQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=3 (3)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "TruthfulQA - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MS MARCO (TREC)", - "source_data": { - "dataset_name": "MS MARCO (TREC)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "NDCG@10 on MS MARCO (TREC)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.643, - "details": { - "description": "min=0.621, mean=0.643, max=0.662, sum=1.93 (3)", - "tab": "Accuracy", - "MS MARCO (regular) - RR@10 (Robustness)": "{\"description\": \"min=0.264, mean=0.287, max=0.315, sum=0.86 (3)\", \"tab\": \"Robustness\", \"score\": \"0.28667883597883553\"}", - "MS MARCO (TREC) - NDCG@10 (Robustness)": "{\"description\": \"min=0.54, mean=0.565, max=0.586, sum=1.696 (3)\", \"tab\": \"Robustness\", \"score\": \"0.5653481865448796\"}", - "MS MARCO (regular) - RR@10 (Fairness)": "{\"description\": \"min=0.332, mean=0.341, max=0.354, sum=1.024 (3)\", \"tab\": \"Fairness\", \"score\": \"0.3414910052910049\"}", - "MS MARCO (TREC) - NDCG@10 (Fairness)": "{\"description\": \"min=0.592, mean=0.612, max=0.629, sum=1.836 (3)\", \"tab\": \"Fairness\", \"score\": \"0.6120938886543282\"}", - "MS MARCO (regular) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MS MARCO (TREC) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MS MARCO (regular) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "MS MARCO (regular) - # train": "{\"description\": \"min=2, mean=2, max=2, sum=6 (3)\", \"tab\": \"General information\", \"score\": \"2.0\"}", - "MS MARCO (regular) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MS MARCO (regular) - # prompt tokens": "{\"description\": \"min=495.232, mean=532.565, max=577.232, sum=1597.696 (3)\", \"tab\": \"General information\", \"score\": \"532.5653333333333\"}", - "MS MARCO (regular) - # output tokens": "{\"description\": \"min=1.004, mean=1.011, max=1.02, sum=3.034 (3)\", \"tab\": \"General information\", \"score\": \"1.0113333333333334\"}", - "MS MARCO (regular) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "MS MARCO (TREC) - # eval": "{\"description\": \"min=43, mean=43, max=43, sum=129 (3)\", \"tab\": \"General information\", \"score\": \"43.0\"}", - "MS MARCO (TREC) - # train": "{\"description\": \"min=2, mean=2, max=2, sum=6 (3)\", \"tab\": \"General information\", \"score\": \"2.0\"}", - "MS MARCO (TREC) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MS MARCO (TREC) - # prompt tokens": "{\"description\": \"min=478.488, mean=515.822, max=560.488, sum=1547.465 (3)\", \"tab\": \"General information\", \"score\": \"515.8217054263565\"}", - "MS MARCO (TREC) - # output tokens": "{\"description\": \"min=1, mean=1.016, max=1.023, sum=3.047 (3)\", \"tab\": \"General information\", \"score\": \"1.0155038759689923\"}", - "MS MARCO (TREC) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "MS MARCO (regular) - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "MS MARCO (TREC) - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CNN/DailyMail", - "source_data": { - "dataset_name": "CNN/DailyMail", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on CNN/DailyMail", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.161, - "details": { - "description": "min=0.151, mean=0.161, max=0.166, sum=0.966 (6)", - "tab": "Accuracy", - "CNN/DailyMail - Denoised inference time (s)": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "CNN/DailyMail - # eval": "{\"description\": \"min=466, mean=466, max=466, sum=2796 (6)\", \"tab\": \"General information\", \"score\": \"466.0\"}", - "CNN/DailyMail - # train": "{\"description\": \"min=5, mean=5, max=5, sum=30 (6)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CNN/DailyMail - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (6)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CNN/DailyMail - # prompt tokens": "{\"description\": \"min=1531.586, mean=1549.919, max=1567.586, sum=9299.515 (6)\", \"tab\": \"General information\", \"score\": \"1549.9191702432045\"}", - "CNN/DailyMail - # output tokens": "{\"description\": \"min=64.44, mean=66.904, max=70.5, sum=401.425 (6)\", \"tab\": \"General information\", \"score\": \"66.9041487839771\"}", - "CNN/DailyMail - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=18 (6)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "CNN/DailyMail - Stereotypes (race)": "{\"description\": \"min=0.601, mean=0.629, max=0.647, sum=3.773 (6)\", \"tab\": \"Bias\", \"score\": \"0.6288257738993034\"}", - "CNN/DailyMail - Stereotypes (gender)": "{\"description\": \"min=0.377, mean=0.398, max=0.411, sum=2.388 (6)\", \"tab\": \"Bias\", \"score\": \"0.3980717194410541\"}", - "CNN/DailyMail - Representation (race)": "{\"description\": \"min=0.135, mean=0.227, max=0.309, sum=1.359 (6)\", \"tab\": \"Bias\", \"score\": \"0.22651255675216078\"}", - "CNN/DailyMail - Representation (gender)": "{\"description\": \"min=0.114, mean=0.12, max=0.124, sum=0.721 (6)\", \"tab\": \"Bias\", \"score\": \"0.12013592572007394\"}", - "CNN/DailyMail - Toxic fraction": "{\"description\": \"min=0.002, mean=0.003, max=0.004, sum=0.017 (6)\", \"tab\": \"Toxicity\", \"score\": \"0.002861230329041488\"}", - "CNN/DailyMail - SummaC": "{\"description\": \"min=0.553, mean=0.573, max=0.595, sum=1.718 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.5727510890981916\"}", - "CNN/DailyMail - QAFactEval": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - BERTScore (F1)": "{\"description\": \"min=0.296, mean=0.316, max=0.326, sum=0.947 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.3157002201673737\"}", - "CNN/DailyMail - Coverage": "{\"description\": \"min=0.975, mean=0.977, max=0.981, sum=5.862 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"0.9770276969879915\"}", - "CNN/DailyMail - Density": "{\"description\": \"min=25.944, mean=26.968, max=27.893, sum=161.808 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"26.967920888770376\"}", - "CNN/DailyMail - Compression": "{\"description\": \"min=9.708, mean=10.317, max=10.928, sum=61.905 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"10.317434111699901\"}", - "CNN/DailyMail - HumanEval-faithfulness": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-relevance": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-coherence": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "XSUM", - "source_data": { - "dataset_name": "XSUM", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on XSUM", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.169, - "details": { - "description": "min=0.162, mean=0.169, max=0.172, sum=1.013 (6)", - "tab": "Accuracy", - "XSUM - Denoised inference time (s)": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "XSUM - # eval": "{\"description\": \"min=518, mean=518, max=518, sum=3108 (6)\", \"tab\": \"General information\", \"score\": \"518.0\"}", - "XSUM - # train": "{\"description\": \"min=4.998, mean=4.999, max=5, sum=29.992 (6)\", \"tab\": \"General information\", \"score\": \"4.998712998712999\"}", - "XSUM - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (6)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "XSUM - # prompt tokens": "{\"description\": \"min=1456.402, mean=1510.418, max=1538.921, sum=9062.51 (6)\", \"tab\": \"General information\", \"score\": \"1510.4182754182755\"}", - "XSUM - # output tokens": "{\"description\": \"min=27.172, mean=27.501, max=27.815, sum=165.008 (6)\", \"tab\": \"General information\", \"score\": \"27.501287001287\"}", - "XSUM - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=18 (6)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "XSUM - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=4 (6)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "XSUM - Stereotypes (gender)": "{\"description\": \"min=0.443, mean=0.449, max=0.459, sum=2.696 (6)\", \"tab\": \"Bias\", \"score\": \"0.4493607590885817\"}", - "XSUM - Representation (race)": "{\"description\": \"min=0.362, mean=0.486, max=0.567, sum=2.914 (6)\", \"tab\": \"Bias\", \"score\": \"0.4857302118171683\"}", - "XSUM - Representation (gender)": "{\"description\": \"min=0.195, mean=0.204, max=0.217, sum=1.223 (6)\", \"tab\": \"Bias\", \"score\": \"0.2037662889603199\"}", - "XSUM - Toxic fraction": "{\"description\": \"min=0, mean=0.003, max=0.004, sum=0.015 (6)\", \"tab\": \"Toxicity\", \"score\": \"0.002574002574002574\"}", - "XSUM - SummaC": "{\"description\": \"min=-0.297, mean=-0.281, max=-0.266, sum=-0.842 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"-0.2807751739040458\"}", - "XSUM - QAFactEval": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - BERTScore (F1)": "{\"description\": \"min=0.472, mean=0.473, max=0.476, sum=1.42 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.4734549353569219\"}", - "XSUM - Coverage": "{\"description\": \"min=0.772, mean=0.774, max=0.777, sum=4.641 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"0.7735373951395458\"}", - "XSUM - Density": "{\"description\": \"min=2.174, mean=2.322, max=2.471, sum=13.929 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"2.321577703631062\"}", - "XSUM - Compression": "{\"description\": \"min=15.596, mean=15.776, max=15.931, sum=94.655 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"15.775903485860036\"}", - "XSUM - HumanEval-faithfulness": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-relevance": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-coherence": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "IMDB", - "source_data": { - "dataset_name": "IMDB", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on IMDB", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.941, - "details": { - "description": "min=0.939, mean=0.941, max=0.942, sum=2.822 (3)", - "tab": "Accuracy", - "IMDB - ECE (10-bin)": "{\"description\": \"min=0.065, mean=0.087, max=0.106, sum=0.262 (3)\", \"tab\": \"Calibration\", \"score\": \"0.08729270886734875\"}", - "IMDB - EM (Robustness)": "{\"description\": \"min=0.92, mean=0.921, max=0.922, sum=2.763 (3)\", \"tab\": \"Robustness\", \"score\": \"0.9210000000000002\"}", - "IMDB - EM (Fairness)": "{\"description\": \"min=0.933, mean=0.936, max=0.94, sum=2.807 (3)\", \"tab\": \"Fairness\", \"score\": \"0.9356666666666666\"}", - "IMDB - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "IMDB - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "IMDB - # train": "{\"description\": \"min=4.845, mean=4.932, max=4.985, sum=14.796 (3)\", \"tab\": \"General information\", \"score\": \"4.9319999999999995\"}", - "IMDB - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IMDB - # prompt tokens": "{\"description\": \"min=1152.524, mean=1389.183, max=1743.988, sum=4167.55 (3)\", \"tab\": \"General information\", \"score\": \"1389.1833333333332\"}", - "IMDB - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=3 (3)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "IMDB - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "IMDB - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CivilComments", - "source_data": { - "dataset_name": "CivilComments", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on CivilComments", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.601, - "details": { - "description": "min=0.171, mean=0.601, max=0.983, sum=32.472 (54)", - "tab": "Accuracy", - "CivilComments - ECE (10-bin)": "{\"description\": \"min=0.058, mean=0.213, max=0.447, sum=11.516 (54)\", \"tab\": \"Calibration\", \"score\": \"0.2132557883443423\"}", - "CivilComments - EM (Robustness)": "{\"description\": \"min=0.069, mean=0.409, max=0.689, sum=22.106 (54)\", \"tab\": \"Robustness\", \"score\": \"0.4093704023963013\"}", - "CivilComments - EM (Fairness)": "{\"description\": \"min=0.047, mean=0.48, max=0.97, sum=25.944 (54)\", \"tab\": \"Fairness\", \"score\": \"0.48044223702694133\"}", - "CivilComments - Denoised inference time (s)": "{\"description\": \"9 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "CivilComments - # eval": "{\"description\": \"min=74, mean=371.556, max=683, sum=20064 (54)\", \"tab\": \"General information\", \"score\": \"371.55555555555554\"}", - "CivilComments - # train": "{\"description\": \"min=5, mean=5, max=5, sum=270 (54)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CivilComments - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (54)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CivilComments - # prompt tokens": "{\"description\": \"min=356.537, mean=722.635, max=1267.519, sum=39022.317 (54)\", \"tab\": \"General information\", \"score\": \"722.6354931173206\"}", - "CivilComments - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=54 (54)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "CivilComments - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=162 (54)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "CivilComments - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (54)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "RAFT", - "source_data": { - "dataset_name": "RAFT", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on RAFT", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.679, - "details": { - "description": "min=0.025, mean=0.679, max=0.975, sum=22.4 (33)", - "tab": "Accuracy", - "RAFT - ECE (10-bin)": "{\"description\": \"min=0.089, mean=0.244, max=0.908, sum=8.049 (33)\", \"tab\": \"Calibration\", \"score\": \"0.24392205141094134\"}", - "RAFT - EM (Robustness)": "{\"description\": \"min=0, mean=0.545, max=0.85, sum=17.975 (33)\", \"tab\": \"Robustness\", \"score\": \"0.5446969696969698\"}", - "RAFT - EM (Fairness)": "{\"description\": \"min=0.025, mean=0.644, max=0.975, sum=21.25 (33)\", \"tab\": \"Fairness\", \"score\": \"0.6439393939393939\"}", - "RAFT - Denoised inference time (s)": "{\"description\": \"11 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "RAFT - # eval": "{\"description\": \"min=40, mean=40, max=40, sum=1320 (33)\", \"tab\": \"General information\", \"score\": \"40.0\"}", - "RAFT - # train": "{\"description\": \"min=0, mean=4.556, max=5, sum=150.35 (33)\", \"tab\": \"General information\", \"score\": \"4.556060606060607\"}", - "RAFT - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (33)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "RAFT - # prompt tokens": "{\"description\": \"min=257.35, mean=812.938, max=1773.675, sum=26826.95 (33)\", \"tab\": \"General information\", \"score\": \"812.937878787879\"}", - "RAFT - # output tokens": "{\"description\": \"min=0.15, mean=3.023, max=6.625, sum=99.75 (33)\", \"tab\": \"General information\", \"score\": \"3.022727272727273\"}", - "RAFT - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=99 (33)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "RAFT - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (33)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/microsoft_tnlg-v2-6.7b.json b/data/models/microsoft_tnlg-v2-6.7b.json deleted file mode 100644 index d3e5f6bfc557fb9e7c9b5716527d93ec055c3a64..0000000000000000000000000000000000000000 --- a/data/models/microsoft_tnlg-v2-6.7b.json +++ /dev/null @@ -1,674 +0,0 @@ -{ - "model_info": { - "name": "TNLG v2 6.7B", - "id": "microsoft/TNLG-v2-6.7B", - "developer": "microsoft", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_classic/microsoft_TNLG-v2-6.7B/1774096308.339228", - "retrieved_timestamp": "1774096308.339228", - "source_metadata": { - "source_name": "helm_classic", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_classic", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_classic", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperform on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.309, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Calibration": "{\"description\": \"\", \"tab\": \"Calibration\", \"score\": \"0.60170195635043\"}", - "Mean win rate - Robustness": "{\"description\": \"\", \"tab\": \"Robustness\", \"score\": \"0.2395553093550869\"}", - "Mean win rate - Fairness": "{\"description\": \"\", \"tab\": \"Fairness\", \"score\": \"0.2912077355347656\"}", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}", - "Mean win rate - Bias": "{\"description\": \"\", \"tab\": \"Bias\", \"score\": \"0.43656162406269206\"}", - "Mean win rate - Toxicity": "{\"description\": \"\", \"tab\": \"Toxicity\", \"score\": \"0.4445961445961446\"}", - "Mean win rate - Summarization metrics": "{\"description\": \"\", \"tab\": \"Summarization metrics\", \"score\": \"0.611842105263158\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.242, - "details": { - "description": "min=0.2, mean=0.242, max=0.35, sum=3.627 (15)", - "tab": "Accuracy", - "MMLU - ECE (10-bin)": "{\"description\": \"min=0.103, mean=0.132, max=0.175, sum=1.983 (15)\", \"tab\": \"Calibration\", \"score\": \"0.13220035950695058\"}", - "MMLU - EM (Robustness)": "{\"description\": \"min=0.09, mean=0.169, max=0.24, sum=2.542 (15)\", \"tab\": \"Robustness\", \"score\": \"0.1694970760233918\"}", - "MMLU - EM (Fairness)": "{\"description\": \"min=0.17, mean=0.212, max=0.31, sum=3.186 (15)\", \"tab\": \"Fairness\", \"score\": \"0.2124327485380117\"}", - "MMLU - Denoised inference time (s)": "{\"description\": \"5 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=1542 (15)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=75 (15)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (15)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=371.38, mean=472.274, max=624.07, sum=7084.111 (15)\", \"tab\": \"General information\", \"score\": \"472.2740350877193\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=15 (15)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "MMLU - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=45 (15)\", \"tab\": \"General information\", \"score\": \"3.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "BoolQ", - "source_data": { - "dataset_name": "BoolQ", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on BoolQ", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.698, - "details": { - "description": "min=0.685, mean=0.698, max=0.709, sum=2.095 (3)", - "tab": "Accuracy", - "BoolQ - ECE (10-bin)": "{\"description\": \"min=0.063, mean=0.065, max=0.067, sum=0.195 (3)\", \"tab\": \"Calibration\", \"score\": \"0.06514212406382298\"}", - "BoolQ - EM (Robustness)": "{\"description\": \"min=0.623, mean=0.638, max=0.653, sum=1.914 (3)\", \"tab\": \"Robustness\", \"score\": \"0.638\"}", - "BoolQ - EM (Fairness)": "{\"description\": \"min=0.649, mean=0.665, max=0.674, sum=1.996 (3)\", \"tab\": \"Fairness\", \"score\": \"0.6653333333333333\"}", - "BoolQ - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "BoolQ - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "BoolQ - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "BoolQ - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "BoolQ - # prompt tokens": "{\"description\": \"min=660.073, mean=908.406, max=1242.073, sum=2725.219 (3)\", \"tab\": \"General information\", \"score\": \"908.4063333333334\"}", - "BoolQ - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=3 (3)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "BoolQ - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "BoolQ - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.631, - "details": { - "description": "min=0.612, mean=0.631, max=0.644, sum=1.893 (3)", - "tab": "Accuracy", - "NarrativeQA - ECE (10-bin)": "{\"description\": \"min=0.045, mean=0.046, max=0.047, sum=0.138 (3)\", \"tab\": \"Calibration\", \"score\": \"0.0461090042242735\"}", - "NarrativeQA - F1 (Robustness)": "{\"description\": \"min=0.314, mean=0.352, max=0.375, sum=1.056 (3)\", \"tab\": \"Robustness\", \"score\": \"0.35196743378602896\"}", - "NarrativeQA - F1 (Fairness)": "{\"description\": \"min=0.492, mean=0.517, max=0.532, sum=1.552 (3)\", \"tab\": \"Fairness\", \"score\": \"0.5173113464127798\"}", - "NarrativeQA - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=1065 (3)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=1.051, mean=1.646, max=2.085, sum=4.938 (3)\", \"tab\": \"General information\", \"score\": \"1.6460093896713615\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=1600.366, mean=1651.848, max=1705.003, sum=4955.544 (3)\", \"tab\": \"General information\", \"score\": \"1651.8478873239437\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=5.189, mean=6.499, max=7.989, sum=19.496 (3)\", \"tab\": \"General information\", \"score\": \"6.498591549295774\"}", - "NarrativeQA - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NarrativeQA - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NarrativeQA - Stereotypes (gender)": "{\"description\": \"min=0.46, mean=0.476, max=0.5, sum=1.429 (3)\", \"tab\": \"Bias\", \"score\": \"0.47625\"}", - "NarrativeQA - Representation (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=2 (3)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "NarrativeQA - Representation (gender)": "{\"description\": \"min=0.203, mean=0.212, max=0.221, sum=0.637 (3)\", \"tab\": \"Bias\", \"score\": \"0.21227319042207152\"}", - "NarrativeQA - Toxic fraction": "{\"description\": \"min=0.008, mean=0.011, max=0.014, sum=0.034 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.011267605633802816\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (open-book)", - "source_data": { - "dataset_name": "NaturalQuestions (open-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (open-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.561, - "details": { - "description": "min=0.532, mean=0.561, max=0.585, sum=1.683 (3)", - "tab": "Accuracy", - "NaturalQuestions (closed-book) - ECE (10-bin)": "{\"description\": \"min=0.028, mean=0.031, max=0.033, sum=0.093 (3)\", \"tab\": \"Calibration\", \"score\": \"0.031006448164221535\"}", - "NaturalQuestions (open-book) - ECE (10-bin)": "{\"description\": \"min=0.071, mean=0.089, max=0.108, sum=0.266 (3)\", \"tab\": \"Calibration\", \"score\": \"0.08866228023213817\"}", - "NaturalQuestions (closed-book) - F1 (Robustness)": "{\"description\": \"min=0.144, mean=0.149, max=0.159, sum=0.448 (3)\", \"tab\": \"Robustness\", \"score\": \"0.149387882661448\"}", - "NaturalQuestions (open-book) - F1 (Robustness)": "{\"description\": \"min=0.215, mean=0.299, max=0.355, sum=0.896 (3)\", \"tab\": \"Robustness\", \"score\": \"0.2985499982493553\"}", - "NaturalQuestions (closed-book) - F1 (Fairness)": "{\"description\": \"min=0.152, mean=0.162, max=0.17, sum=0.485 (3)\", \"tab\": \"Fairness\", \"score\": \"0.16163226517271406\"}", - "NaturalQuestions (open-book) - F1 (Fairness)": "{\"description\": \"min=0.463, mean=0.501, max=0.532, sum=1.502 (3)\", \"tab\": \"Fairness\", \"score\": \"0.5005776676014201\"}", - "NaturalQuestions (closed-book) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NaturalQuestions (open-book) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=110.254, mean=112.254, max=116.254, sum=336.762 (3)\", \"tab\": \"General information\", \"score\": \"112.254\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=5.189, mean=5.6, max=5.896, sum=16.8 (3)\", \"tab\": \"General information\", \"score\": \"5.6000000000000005\"}", - "NaturalQuestions (closed-book) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.647, mean=4.691, max=4.723, sum=14.072 (3)\", \"tab\": \"General information\", \"score\": \"4.690666666666666\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.036, mean=0.036, max=0.036, sum=0.108 (3)\", \"tab\": \"General information\", \"score\": \"0.036\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1231.056, mean=1419.328, max=1523.222, sum=4257.983 (3)\", \"tab\": \"General information\", \"score\": \"1419.3276666666668\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=7.244, mean=8.369, max=10.389, sum=25.107 (3)\", \"tab\": \"General information\", \"score\": \"8.369\"}", - "NaturalQuestions (open-book) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NaturalQuestions (closed-book) - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - Stereotypes (gender)": "{\"description\": \"min=0.494, mean=0.498, max=0.5, sum=1.494 (3)\", \"tab\": \"Bias\", \"score\": \"0.4981481481481482\"}", - "NaturalQuestions (closed-book) - Representation (race)": "{\"description\": \"min=0.32, mean=0.479, max=0.588, sum=1.437 (3)\", \"tab\": \"Bias\", \"score\": \"0.47890062007709067\"}", - "NaturalQuestions (closed-book) - Representation (gender)": "{\"description\": \"min=0.179, mean=0.274, max=0.437, sum=0.821 (3)\", \"tab\": \"Bias\", \"score\": \"0.2737208807573663\"}", - "NaturalQuestions (open-book) - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=0.667 (1)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666667\"}", - "NaturalQuestions (open-book) - Stereotypes (gender)": "{\"description\": \"min=0.167, mean=0.333, max=0.417, sum=1.0 (3)\", \"tab\": \"Bias\", \"score\": \"0.3333333333333333\"}", - "NaturalQuestions (open-book) - Representation (race)": "{\"description\": \"min=0.399, mean=0.446, max=0.489, sum=1.338 (3)\", \"tab\": \"Bias\", \"score\": \"0.4460824634464231\"}", - "NaturalQuestions (open-book) - Representation (gender)": "{\"description\": \"min=0.115, mean=0.228, max=0.345, sum=0.684 (3)\", \"tab\": \"Bias\", \"score\": \"0.22804989848201077\"}", - "NaturalQuestions (closed-book) - Toxic fraction": "{\"description\": \"min=0, mean=0.0, max=0.001, sum=0.001 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.0003333333333333333\"}", - "NaturalQuestions (open-book) - Toxic fraction": "{\"description\": \"min=0.002, mean=0.002, max=0.003, sum=0.007 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.0023333333333333335\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "QuAC", - "source_data": { - "dataset_name": "QuAC", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on QuAC", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.345, - "details": { - "description": "min=0.334, mean=0.345, max=0.365, sum=1.034 (3)", - "tab": "Accuracy", - "QuAC - ECE (10-bin)": "{\"description\": \"min=0.046, mean=0.056, max=0.064, sum=0.169 (3)\", \"tab\": \"Calibration\", \"score\": \"0.056431419773363155\"}", - "QuAC - F1 (Robustness)": "{\"description\": \"min=0.143, mean=0.159, max=0.17, sum=0.477 (3)\", \"tab\": \"Robustness\", \"score\": \"0.1590786964332521\"}", - "QuAC - F1 (Fairness)": "{\"description\": \"min=0.26, mean=0.267, max=0.281, sum=0.801 (3)\", \"tab\": \"Fairness\", \"score\": \"0.26693937921563893\"}", - "QuAC - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "QuAC - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "QuAC - # train": "{\"description\": \"min=0.845, mean=0.944, max=1.084, sum=2.831 (3)\", \"tab\": \"General information\", \"score\": \"0.9436666666666667\"}", - "QuAC - truncated": "{\"description\": \"min=0.016, mean=0.016, max=0.016, sum=0.048 (3)\", \"tab\": \"General information\", \"score\": \"0.016\"}", - "QuAC - # prompt tokens": "{\"description\": \"min=1624.371, mean=1644.436, max=1670.589, sum=4933.308 (3)\", \"tab\": \"General information\", \"score\": \"1644.436\"}", - "QuAC - # output tokens": "{\"description\": \"min=17.622, mean=19.574, max=21.058, sum=58.723 (3)\", \"tab\": \"General information\", \"score\": \"19.574333333333332\"}", - "QuAC - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "QuAC - Stereotypes (race)": "{\"description\": \"min=0.598, mean=0.618, max=0.639, sum=1.855 (3)\", \"tab\": \"Bias\", \"score\": \"0.6181852538995397\"}", - "QuAC - Stereotypes (gender)": "{\"description\": \"min=0.451, mean=0.472, max=0.486, sum=1.416 (3)\", \"tab\": \"Bias\", \"score\": \"0.47198334521620583\"}", - "QuAC - Representation (race)": "{\"description\": \"min=0.32, mean=0.351, max=0.412, sum=1.054 (3)\", \"tab\": \"Bias\", \"score\": \"0.35120217651448443\"}", - "QuAC - Representation (gender)": "{\"description\": \"min=0.213, mean=0.232, max=0.259, sum=0.695 (3)\", \"tab\": \"Bias\", \"score\": \"0.23164076323994623\"}", - "QuAC - Toxic fraction": "{\"description\": \"min=0.001, mean=0.001, max=0.002, sum=0.004 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.0013333333333333333\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "HellaSwag", - "source_data": { - "dataset_name": "HellaSwag", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on HellaSwag", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.704, - "details": { - "description": "min=0.704, mean=0.704, max=0.704, sum=0.704 (1)", - "tab": "Accuracy", - "HellaSwag - ECE (10-bin)": "{\"description\": \"min=0.268, mean=0.268, max=0.268, sum=0.268 (1)\", \"tab\": \"Calibration\", \"score\": \"0.2676753668258396\"}", - "HellaSwag - EM (Robustness)": "{\"description\": \"min=0.656, mean=0.656, max=0.656, sum=0.656 (1)\", \"tab\": \"Robustness\", \"score\": \"0.656\"}", - "HellaSwag - EM (Fairness)": "{\"description\": \"min=0.53, mean=0.53, max=0.53, sum=0.53 (1)\", \"tab\": \"Fairness\", \"score\": \"0.53\"}", - "HellaSwag - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "HellaSwag - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "HellaSwag - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "HellaSwag - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "HellaSwag - # prompt tokens": "{\"description\": \"min=87.888, mean=87.888, max=87.888, sum=87.888 (1)\", \"tab\": \"General information\", \"score\": \"87.888\"}", - "HellaSwag - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "HellaSwag - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.478, - "details": { - "description": "min=0.478, mean=0.478, max=0.478, sum=0.478 (1)", - "tab": "Accuracy", - "OpenbookQA - ECE (10-bin)": "{\"description\": \"min=0.282, mean=0.282, max=0.282, sum=0.282 (1)\", \"tab\": \"Calibration\", \"score\": \"0.28175565698884514\"}", - "OpenbookQA - EM (Robustness)": "{\"description\": \"min=0.408, mean=0.408, max=0.408, sum=0.408 (1)\", \"tab\": \"Robustness\", \"score\": \"0.408\"}", - "OpenbookQA - EM (Fairness)": "{\"description\": \"min=0.412, mean=0.412, max=0.412, sum=0.412 (1)\", \"tab\": \"Fairness\", \"score\": \"0.412\"}", - "OpenbookQA - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=5.27, mean=5.27, max=5.27, sum=5.27 (1)\", \"tab\": \"General information\", \"score\": \"5.27\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "TruthfulQA", - "source_data": { - "dataset_name": "TruthfulQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on TruthfulQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.167, - "details": { - "description": "min=0.156, mean=0.167, max=0.173, sum=0.5 (3)", - "tab": "Accuracy", - "TruthfulQA - ECE (10-bin)": "{\"description\": \"min=0.101, mean=0.117, max=0.128, sum=0.35 (3)\", \"tab\": \"Calibration\", \"score\": \"0.11656099093897697\"}", - "TruthfulQA - EM (Robustness)": "{\"description\": \"min=0.128, mean=0.136, max=0.148, sum=0.408 (3)\", \"tab\": \"Robustness\", \"score\": \"0.13608562691131498\"}", - "TruthfulQA - EM (Fairness)": "{\"description\": \"min=0.133, mean=0.144, max=0.162, sum=0.431 (3)\", \"tab\": \"Fairness\", \"score\": \"0.1437308868501529\"}", - "TruthfulQA - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "TruthfulQA - # eval": "{\"description\": \"min=654, mean=654, max=654, sum=1962 (3)\", \"tab\": \"General information\", \"score\": \"654.0\"}", - "TruthfulQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "TruthfulQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "TruthfulQA - # prompt tokens": "{\"description\": \"min=501.121, mean=511.121, max=529.121, sum=1533.362 (3)\", \"tab\": \"General information\", \"score\": \"511.12079510703364\"}", - "TruthfulQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=3 (3)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "TruthfulQA - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MS MARCO (TREC)", - "source_data": { - "dataset_name": "MS MARCO (TREC)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "NDCG@10 on MS MARCO (TREC)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.332, - "details": { - "description": "min=0.273, mean=0.332, max=0.382, sum=0.997 (3)", - "tab": "Accuracy", - "MS MARCO (regular) - RR@10 (Robustness)": "{\"description\": \"min=0.074, mean=0.105, max=0.125, sum=0.315 (3)\", \"tab\": \"Robustness\", \"score\": \"0.1048433862433863\"}", - "MS MARCO (TREC) - NDCG@10 (Robustness)": "{\"description\": \"min=0.227, mean=0.278, max=0.312, sum=0.835 (3)\", \"tab\": \"Robustness\", \"score\": \"0.2783978738136928\"}", - "MS MARCO (regular) - RR@10 (Fairness)": "{\"description\": \"min=0.109, mean=0.14, max=0.166, sum=0.419 (3)\", \"tab\": \"Fairness\", \"score\": \"0.13970383597883587\"}", - "MS MARCO (TREC) - NDCG@10 (Fairness)": "{\"description\": \"min=0.256, mean=0.317, max=0.363, sum=0.95 (3)\", \"tab\": \"Fairness\", \"score\": \"0.31652617829212154\"}", - "MS MARCO (regular) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MS MARCO (TREC) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MS MARCO (regular) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "MS MARCO (regular) - # train": "{\"description\": \"min=2, mean=2, max=2, sum=6 (3)\", \"tab\": \"General information\", \"score\": \"2.0\"}", - "MS MARCO (regular) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MS MARCO (regular) - # prompt tokens": "{\"description\": \"min=495.232, mean=532.565, max=577.232, sum=1597.696 (3)\", \"tab\": \"General information\", \"score\": \"532.5653333333333\"}", - "MS MARCO (regular) - # output tokens": "{\"description\": \"min=1.028, mean=1.067, max=1.136, sum=3.2 (3)\", \"tab\": \"General information\", \"score\": \"1.0666666666666667\"}", - "MS MARCO (regular) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "MS MARCO (TREC) - # eval": "{\"description\": \"min=43, mean=43, max=43, sum=129 (3)\", \"tab\": \"General information\", \"score\": \"43.0\"}", - "MS MARCO (TREC) - # train": "{\"description\": \"min=2, mean=2, max=2, sum=6 (3)\", \"tab\": \"General information\", \"score\": \"2.0\"}", - "MS MARCO (TREC) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MS MARCO (TREC) - # prompt tokens": "{\"description\": \"min=478.488, mean=515.822, max=560.488, sum=1547.465 (3)\", \"tab\": \"General information\", \"score\": \"515.8217054263565\"}", - "MS MARCO (TREC) - # output tokens": "{\"description\": \"min=1.047, mean=1.047, max=1.047, sum=3.14 (3)\", \"tab\": \"General information\", \"score\": \"1.0465116279069768\"}", - "MS MARCO (TREC) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "MS MARCO (regular) - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "MS MARCO (TREC) - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CNN/DailyMail", - "source_data": { - "dataset_name": "CNN/DailyMail", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on CNN/DailyMail", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.146, - "details": { - "description": "min=0.139, mean=0.146, max=0.157, sum=0.877 (6)", - "tab": "Accuracy", - "CNN/DailyMail - Denoised inference time (s)": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "CNN/DailyMail - # eval": "{\"description\": \"min=466, mean=466, max=466, sum=2796 (6)\", \"tab\": \"General information\", \"score\": \"466.0\"}", - "CNN/DailyMail - # train": "{\"description\": \"min=5, mean=5, max=5, sum=30 (6)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CNN/DailyMail - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (6)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CNN/DailyMail - # prompt tokens": "{\"description\": \"min=1531.586, mean=1549.919, max=1567.586, sum=9299.515 (6)\", \"tab\": \"General information\", \"score\": \"1549.9191702432045\"}", - "CNN/DailyMail - # output tokens": "{\"description\": \"min=70.732, mean=83.556, max=100.29, sum=501.335 (6)\", \"tab\": \"General information\", \"score\": \"83.55579399141631\"}", - "CNN/DailyMail - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=18 (6)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "CNN/DailyMail - Stereotypes (race)": "{\"description\": \"min=0.605, mean=0.616, max=0.623, sum=3.698 (6)\", \"tab\": \"Bias\", \"score\": \"0.6163696620441931\"}", - "CNN/DailyMail - Stereotypes (gender)": "{\"description\": \"min=0.387, mean=0.404, max=0.42, sum=2.422 (6)\", \"tab\": \"Bias\", \"score\": \"0.4036032258152607\"}", - "CNN/DailyMail - Representation (race)": "{\"description\": \"min=0.306, mean=0.326, max=0.352, sum=1.955 (6)\", \"tab\": \"Bias\", \"score\": \"0.32584352768289004\"}", - "CNN/DailyMail - Representation (gender)": "{\"description\": \"min=0.125, mean=0.146, max=0.173, sum=0.878 (6)\", \"tab\": \"Bias\", \"score\": \"0.1463963556163381\"}", - "CNN/DailyMail - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (6)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}", - "CNN/DailyMail - SummaC": "{\"description\": \"min=0.487, mean=0.493, max=0.501, sum=1.48 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.4933195613927493\"}", - "CNN/DailyMail - QAFactEval": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - BERTScore (F1)": "{\"description\": \"min=0.278, mean=0.282, max=0.284, sum=0.845 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.2815425075266347\"}", - "CNN/DailyMail - Coverage": "{\"description\": \"min=0.973, mean=0.976, max=0.981, sum=5.857 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"0.9761546866038108\"}", - "CNN/DailyMail - Density": "{\"description\": \"min=38.053, mean=48.951, max=68.464, sum=293.707 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"48.951173188846475\"}", - "CNN/DailyMail - Compression": "{\"description\": \"min=7.327, mean=9.598, max=11.919, sum=57.585 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"9.59754128304669\"}", - "CNN/DailyMail - HumanEval-faithfulness": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-relevance": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-coherence": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "XSUM", - "source_data": { - "dataset_name": "XSUM", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on XSUM", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.11, - "details": { - "description": "min=0.107, mean=0.11, max=0.113, sum=0.661 (6)", - "tab": "Accuracy", - "XSUM - Denoised inference time (s)": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "XSUM - # eval": "{\"description\": \"min=518, mean=518, max=518, sum=3108 (6)\", \"tab\": \"General information\", \"score\": \"518.0\"}", - "XSUM - # train": "{\"description\": \"min=4.998, mean=4.999, max=5, sum=29.992 (6)\", \"tab\": \"General information\", \"score\": \"4.998712998712999\"}", - "XSUM - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (6)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "XSUM - # prompt tokens": "{\"description\": \"min=1456.402, mean=1510.418, max=1538.921, sum=9062.51 (6)\", \"tab\": \"General information\", \"score\": \"1510.4182754182755\"}", - "XSUM - # output tokens": "{\"description\": \"min=23.276, mean=23.579, max=24.127, sum=141.471 (6)\", \"tab\": \"General information\", \"score\": \"23.578507078507084\"}", - "XSUM - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=18 (6)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "XSUM - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=4 (6)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "XSUM - Stereotypes (gender)": "{\"description\": \"min=0.451, mean=0.462, max=0.473, sum=2.775 (6)\", \"tab\": \"Bias\", \"score\": \"0.46245791245791246\"}", - "XSUM - Representation (race)": "{\"description\": \"min=0.373, mean=0.489, max=0.579, sum=2.933 (6)\", \"tab\": \"Bias\", \"score\": \"0.4888826343934703\"}", - "XSUM - Representation (gender)": "{\"description\": \"min=0.136, mean=0.182, max=0.23, sum=1.089 (6)\", \"tab\": \"Bias\", \"score\": \"0.18150391082886233\"}", - "XSUM - Toxic fraction": "{\"description\": \"min=0, mean=0.002, max=0.004, sum=0.012 (6)\", \"tab\": \"Toxicity\", \"score\": \"0.0019305019305019308\"}", - "XSUM - SummaC": "{\"description\": \"min=-0.217, mean=-0.203, max=-0.192, sum=-0.61 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"-0.20340532606019324\"}", - "XSUM - QAFactEval": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - BERTScore (F1)": "{\"description\": \"min=0.38, mean=0.385, max=0.394, sum=1.156 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.3853545238949662\"}", - "XSUM - Coverage": "{\"description\": \"min=0.786, mean=0.793, max=0.801, sum=4.757 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"0.792833262373014\"}", - "XSUM - Density": "{\"description\": \"min=3.215, mean=3.286, max=3.34, sum=19.716 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"3.2859287054515427\"}", - "XSUM - Compression": "{\"description\": \"min=17.984, mean=18.428, max=18.968, sum=110.571 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"18.428451341381788\"}", - "XSUM - HumanEval-faithfulness": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-relevance": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-coherence": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "IMDB", - "source_data": { - "dataset_name": "IMDB", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on IMDB", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.927, - "details": { - "description": "min=0.923, mean=0.927, max=0.934, sum=2.782 (3)", - "tab": "Accuracy", - "IMDB - ECE (10-bin)": "{\"description\": \"min=0.093, mean=0.118, max=0.136, sum=0.355 (3)\", \"tab\": \"Calibration\", \"score\": \"0.11832833491942714\"}", - "IMDB - EM (Robustness)": "{\"description\": \"min=0.883, mean=0.896, max=0.909, sum=2.687 (3)\", \"tab\": \"Robustness\", \"score\": \"0.8956666666666667\"}", - "IMDB - EM (Fairness)": "{\"description\": \"min=0.904, mean=0.912, max=0.922, sum=2.737 (3)\", \"tab\": \"Fairness\", \"score\": \"0.9123333333333333\"}", - "IMDB - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "IMDB - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "IMDB - # train": "{\"description\": \"min=4.845, mean=4.932, max=4.985, sum=14.796 (3)\", \"tab\": \"General information\", \"score\": \"4.9319999999999995\"}", - "IMDB - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IMDB - # prompt tokens": "{\"description\": \"min=1152.524, mean=1389.183, max=1743.988, sum=4167.55 (3)\", \"tab\": \"General information\", \"score\": \"1389.1833333333332\"}", - "IMDB - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=3 (3)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "IMDB - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "IMDB - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CivilComments", - "source_data": { - "dataset_name": "CivilComments", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on CivilComments", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.532, - "details": { - "description": "min=0.053, mean=0.532, max=0.955, sum=28.701 (54)", - "tab": "Accuracy", - "CivilComments - ECE (10-bin)": "{\"description\": \"min=0.052, mean=0.248, max=0.54, sum=13.38 (54)\", \"tab\": \"Calibration\", \"score\": \"0.24778001352805415\"}", - "CivilComments - EM (Robustness)": "{\"description\": \"min=0.022, mean=0.336, max=0.831, sum=18.169 (54)\", \"tab\": \"Robustness\", \"score\": \"0.336456419012055\"}", - "CivilComments - EM (Fairness)": "{\"description\": \"min=0.042, mean=0.473, max=0.947, sum=25.533 (54)\", \"tab\": \"Fairness\", \"score\": \"0.4728366689674401\"}", - "CivilComments - Denoised inference time (s)": "{\"description\": \"9 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "CivilComments - # eval": "{\"description\": \"min=74, mean=371.556, max=683, sum=20064 (54)\", \"tab\": \"General information\", \"score\": \"371.55555555555554\"}", - "CivilComments - # train": "{\"description\": \"min=5, mean=5, max=5, sum=270 (54)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CivilComments - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (54)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CivilComments - # prompt tokens": "{\"description\": \"min=356.537, mean=722.635, max=1267.519, sum=39022.317 (54)\", \"tab\": \"General information\", \"score\": \"722.6354931173206\"}", - "CivilComments - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=54 (54)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "CivilComments - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=162 (54)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "CivilComments - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (54)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "RAFT", - "source_data": { - "dataset_name": "RAFT", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on RAFT", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.525, - "details": { - "description": "min=0.025, mean=0.525, max=0.975, sum=17.325 (33)", - "tab": "Accuracy", - "RAFT - ECE (10-bin)": "{\"description\": \"min=0.103, mean=0.314, max=0.912, sum=10.346 (33)\", \"tab\": \"Calibration\", \"score\": \"0.31351556505949635\"}", - "RAFT - EM (Robustness)": "{\"description\": \"min=0, mean=0.445, max=0.95, sum=14.675 (33)\", \"tab\": \"Robustness\", \"score\": \"0.4446969696969697\"}", - "RAFT - EM (Fairness)": "{\"description\": \"min=0.025, mean=0.502, max=0.975, sum=16.55 (33)\", \"tab\": \"Fairness\", \"score\": \"0.5015151515151516\"}", - "RAFT - Denoised inference time (s)": "{\"description\": \"11 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "RAFT - # eval": "{\"description\": \"min=40, mean=40, max=40, sum=1320 (33)\", \"tab\": \"General information\", \"score\": \"40.0\"}", - "RAFT - # train": "{\"description\": \"min=0, mean=4.556, max=5, sum=150.35 (33)\", \"tab\": \"General information\", \"score\": \"4.556060606060607\"}", - "RAFT - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (33)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "RAFT - # prompt tokens": "{\"description\": \"min=257.35, mean=812.938, max=1773.675, sum=26826.95 (33)\", \"tab\": \"General information\", \"score\": \"812.937878787879\"}", - "RAFT - # output tokens": "{\"description\": \"min=0.15, mean=2.76, max=6.175, sum=91.075 (33)\", \"tab\": \"General information\", \"score\": \"2.7598484848484848\"}", - "RAFT - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=99 (33)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "RAFT - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (33)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/mightbe_better-pairrm.json b/data/models/mightbe_better-pairrm.json deleted file mode 100644 index 981bed163d4cdab78fb70459d296695c7a85f412..0000000000000000000000000000000000000000 --- a/data/models/mightbe_better-pairrm.json +++ /dev/null @@ -1,144 +0,0 @@ -{ - "model_info": { - "name": "mightbe/Better-PairRM", - "id": "mightbe/Better-PairRM", - "developer": "mightbe", - "additional_details": { - "model_type": "Custom Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/mightbe_Better-PairRM/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.673 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9553 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3925 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8203 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4983 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.724 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/migtissera_llama-3-70b-synthia-v3.5.json b/data/models/migtissera_llama-3-70b-synthia-v3.5.json deleted file mode 100644 index 8ab8c463499bd221d314d7648bf17018116d4860..0000000000000000000000000000000000000000 --- a/data/models/migtissera_llama-3-70b-synthia-v3.5.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-70B-Synthia-v3.5", - "id": "migtissera/Llama-3-70B-Synthia-v3.5", - "developer": "migtissera", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "70.554" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/migtissera_Llama-3-70B-Synthia-v3.5/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6076 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6489 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2115 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3876 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4922 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4658 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/migtissera_llama-3-8b-synthia-v3.5.json b/data/models/migtissera_llama-3-8b-synthia-v3.5.json deleted file mode 100644 index 9cb5929bde2879031b5fa713c326394471160b96..0000000000000000000000000000000000000000 --- a/data/models/migtissera_llama-3-8b-synthia-v3.5.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-8B-Synthia-v3.5", - "id": "migtissera/Llama-3-8B-Synthia-v3.5", - "developer": "migtissera", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/migtissera_Llama-3-8B-Synthia-v3.5/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.507 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4888 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0657 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2718 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4044 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.303 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/migtissera_tess-3-7b-sft.json b/data/models/migtissera_tess-3-7b-sft.json deleted file mode 100644 index fc553c3ce0da1dbefb0a439d0b3b183d61db5ff9..0000000000000000000000000000000000000000 --- a/data/models/migtissera_tess-3-7b-sft.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Tess-3-7B-SFT", - "id": "migtissera/Tess-3-7B-SFT", - "developer": "migtissera", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/migtissera_Tess-3-7B-SFT/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3946 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4607 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.04 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.271 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4113 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3034 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/migtissera_tess-3-mistral-nemo-12b.json b/data/models/migtissera_tess-3-mistral-nemo-12b.json deleted file mode 100644 index 48d6716f49a5ec03536fdf1f9e453a399083324d..0000000000000000000000000000000000000000 --- a/data/models/migtissera_tess-3-mistral-nemo-12b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Tess-3-Mistral-Nemo-12B", - "id": "migtissera/Tess-3-Mistral-Nemo-12B", - "developer": "migtissera", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/migtissera_Tess-3-Mistral-Nemo-12B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3355 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4899 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0574 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2508 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4458 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2565 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/migtissera_tess-v2.5-phi-3-medium-128k-14b.json b/data/models/migtissera_tess-v2.5-phi-3-medium-128k-14b.json deleted file mode 100644 index aa3ab376fd4be21cee348cce0b18975d97ab4aea..0000000000000000000000000000000000000000 --- a/data/models/migtissera_tess-v2.5-phi-3-medium-128k-14b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Tess-v2.5-Phi-3-medium-128k-14B", - "id": "migtissera/Tess-v2.5-Phi-3-medium-128k-14B", - "developer": "migtissera", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Phi3ForCausalLM", - "params_billions": "13.96" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/migtissera_Tess-v2.5-Phi-3-medium-128k-14B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4539 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6207 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0506 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3079 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4113 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3732 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/migtissera_tess-v2.5.2-qwen2-72b.json b/data/models/migtissera_tess-v2.5.2-qwen2-72b.json deleted file mode 100644 index cae88bc9ba4e4b4a2b296e36f99f19179efac076..0000000000000000000000000000000000000000 --- a/data/models/migtissera_tess-v2.5.2-qwen2-72b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Tess-v2.5.2-Qwen2-72B", - "id": "migtissera/Tess-v2.5.2-Qwen2-72B", - "developer": "migtissera", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "72.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/migtissera_Tess-v2.5.2-Qwen2-72B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4494 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6647 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2938 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3507 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4188 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5561 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/migtissera_trinity-2-codestral-22b-v0.2.json b/data/models/migtissera_trinity-2-codestral-22b-v0.2.json deleted file mode 100644 index 31fe80318a861ae168003eeb962c4916cb0c7f71..0000000000000000000000000000000000000000 --- a/data/models/migtissera_trinity-2-codestral-22b-v0.2.json +++ /dev/null @@ -1,275 +0,0 @@ -{ - "model_info": { - "name": "Trinity-2-Codestral-22B-v0.2", - "id": "migtissera/Trinity-2-Codestral-22B-v0.2", - "developer": "migtissera", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "22.247" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/migtissera_Trinity-2-Codestral-22B-v0.2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4345 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5686 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0838 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3003 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4045 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.334 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "hfopenllm_v2/migtissera_Trinity-2-Codestral-22B-v0.2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.443 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5706 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0869 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3079 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4031 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3354 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/migtissera_trinity-2-codestral-22b.json b/data/models/migtissera_trinity-2-codestral-22b.json deleted file mode 100644 index 47d0a6dc1c321580dae8bab47901be192fa723fc..0000000000000000000000000000000000000000 --- a/data/models/migtissera_trinity-2-codestral-22b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Trinity-2-Codestral-22B", - "id": "migtissera/Trinity-2-Codestral-22B", - "developer": "migtissera", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "22.247" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/migtissera_Trinity-2-Codestral-22B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4202 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5593 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0967 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3146 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4111 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3308 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/minami-su_amara-o1-7b-qwen.json b/data/models/minami-su_amara-o1-7b-qwen.json deleted file mode 100644 index b11194eced0234f75986ef85d0c6e7e2b32998bc..0000000000000000000000000000000000000000 --- a/data/models/minami-su_amara-o1-7b-qwen.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Amara-o1-7B-Qwen", - "id": "Minami-su/Amara-o1-7B-Qwen", - "developer": "Minami-su", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Minami-su_Amara-o1-7B-Qwen/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.739 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5199 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5181 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2936 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4007 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4083 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/minami-su_amara-o2-7b-qwen.json b/data/models/minami-su_amara-o2-7b-qwen.json deleted file mode 100644 index 005d5ee1e1d15e525ec4493c0c8050dcad67b4de..0000000000000000000000000000000000000000 --- a/data/models/minami-su_amara-o2-7b-qwen.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Amara-o2-7B-Qwen", - "id": "Minami-su/Amara-o2-7B-Qwen", - "developer": "Minami-su", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Minami-su_Amara-o2-7B-Qwen/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7147 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5173 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4086 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2634 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3781 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4165 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/minami-su_test-7b-00.json b/data/models/minami-su_test-7b-00.json deleted file mode 100644 index 56d8b60146853e298d7128f45c956dc5769ec50e..0000000000000000000000000000000000000000 --- a/data/models/minami-su_test-7b-00.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "test-7B-00", - "id": "Minami-su/test-7B-00", - "developer": "Minami-su", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Minami-su_test-7B-00/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.669 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4466 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4517 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3029 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4126 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3588 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/minami-su_test-7b-01.json b/data/models/minami-su_test-7b-01.json deleted file mode 100644 index a5655f1ed5ea66131e8292b2c35e1b0c46ffe264..0000000000000000000000000000000000000000 --- a/data/models/minami-su_test-7b-01.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "test-7B-01", - "id": "Minami-su/test-7B-01", - "developer": "Minami-su", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Minami-su_test-7B-01/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6736 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4422 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4554 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.307 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4153 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3536 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/minami-su_test-v2-7b-00.json b/data/models/minami-su_test-v2-7b-00.json deleted file mode 100644 index 893cc887584b3be07292c36d00a6103b0cbf9a0c..0000000000000000000000000000000000000000 --- a/data/models/minami-su_test-v2-7b-00.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "test-v2-7B-00", - "id": "Minami-su/test-v2-7B-00", - "developer": "Minami-su", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Minami-su_test-v2-7B-00/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6747 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4416 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4418 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2919 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4154 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3472 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/mindw96_deepseek-llama3.3-bllossom-8b-dacon-llm3.json b/data/models/mindw96_deepseek-llama3.3-bllossom-8b-dacon-llm3.json deleted file mode 100644 index a52b3ad650992b81fbfd9fbfe56eef9a424869dd..0000000000000000000000000000000000000000 --- a/data/models/mindw96_deepseek-llama3.3-bllossom-8b-dacon-llm3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "DeepSeek-llama3.3-Bllossom-8B-DACON-LLM3", - "id": "mindw96/DeepSeek-llama3.3-Bllossom-8B-DACON-LLM3", - "developer": "mindw96", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/mindw96_DeepSeek-llama3.3-Bllossom-8B-DACON-LLM3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1388 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3068 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0083 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2508 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3792 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1106 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/minghaowu_qwen1.5-1.8b-openhermes-2.5.json b/data/models/minghaowu_qwen1.5-1.8b-openhermes-2.5.json deleted file mode 100644 index ada66085f08416535809ecf45508e25d2ed2ecaf..0000000000000000000000000000000000000000 --- a/data/models/minghaowu_qwen1.5-1.8b-openhermes-2.5.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen1.5-1.8B-OpenHermes-2.5", - "id": "minghaowu/Qwen1.5-1.8B-OpenHermes-2.5", - "developer": "minghaowu", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.837" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/minghaowu_Qwen1.5-1.8B-OpenHermes-2.5/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2778 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3375 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0242 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2836 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3529 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1792 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/minimax_minimax-2.5.json b/data/models/minimax_minimax-2.5.json deleted file mode 100644 index 1d69f71882c732c13f818e4ed2e83b22c2c85add..0000000000000000000000000000000000000000 --- a/data/models/minimax_minimax-2.5.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "model_info": { - "name": "Minimax-2.5", - "developer": "minimax", - "id": "minimax/Minimax-2.5", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "apex-agents/minimax_minimax-2.5/1773260200", - "retrieved_timestamp": "1773260200", - "source_metadata": { - "source_name": "Mercor APEX-Agents Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Mercor", - "source_organization_url": "https://www.mercor.com", - "evaluator_relationship": "first_party" - }, - "eval_library": { - "name": "archipelago", - "version": "1.0.0" - }, - "benchmark": "apex-agents", - "evaluation_results": [ - { - "evaluation_name": "Corporate Lawyer Mean Score", - "source_data": { - "dataset_name": "apex-agents", - "source_type": "hf_dataset", - "hf_repo": "mercor/apex-agents" - }, - "metric_config": { - "evaluation_description": "Corporate lawyer world mean score.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.339 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/minimax_minimax-m2.1.json b/data/models/minimax_minimax-m2.1.json deleted file mode 100644 index 33905b0ffaa72b65b1d021a73faac651c1ebfecd..0000000000000000000000000000000000000000 --- a/data/models/minimax_minimax-m2.1.json +++ /dev/null @@ -1,161 +0,0 @@ -{ - "model_info": { - "name": "MiniMax M2.1", - "id": "minimax/minimax-m2.1", - "developer": "MiniMax", - "additional_details": { - "agent_name": "Crux", - "agent_organization": "Roam" - } - }, - "evaluations": [ - { - "evaluation_id": "terminal-bench-2.0/crux__minimax-m2.1/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2025-12-22", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 36.6, - "uncertainty": { - "standard_error": { - "value": 2.9 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Crux\" -m \"MiniMax M2.1\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Crux\" -m \"MiniMax M2.1\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - }, - { - "evaluation_id": "terminal-bench-2.0/terminus-2__minimax-m2.1/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2025-12-23", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 29.2, - "uncertainty": { - "standard_error": { - "value": 2.9 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Terminus 2\" -m \"MiniMax M2.1\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Terminus 2\" -m \"MiniMax M2.1\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/minimax_minimax-m2.5.json b/data/models/minimax_minimax-m2.5.json deleted file mode 100644 index 00fc5b0ca0966da9924c8d38af97aac3e5fe3ee4..0000000000000000000000000000000000000000 --- a/data/models/minimax_minimax-m2.5.json +++ /dev/null @@ -1,87 +0,0 @@ -{ - "model_info": { - "name": "Minimax m2.5", - "id": "minimax/minimax-m2.5", - "developer": "Minimax", - "additional_details": { - "agent_name": "Terminus 2", - "agent_organization": "Terminal Bench" - } - }, - "evaluations": [ - { - "evaluation_id": "terminal-bench-2.0/terminus-2__minimax-m2.5/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2026-02-23", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 42.2, - "uncertainty": { - "standard_error": { - "value": 2.6 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Terminus 2\" -m \"Minimax m2.5\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Terminus 2\" -m \"Minimax m2.5\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/minimax_minimax-m2.json b/data/models/minimax_minimax-m2.json deleted file mode 100644 index 5cbd9bd783d8425a1c340138166da983b0f3debb..0000000000000000000000000000000000000000 --- a/data/models/minimax_minimax-m2.json +++ /dev/null @@ -1,87 +0,0 @@ -{ - "model_info": { - "name": "MiniMax M2", - "id": "minimax/minimax-m2", - "developer": "MiniMax", - "additional_details": { - "agent_name": "Terminus 2", - "agent_organization": "Terminal Bench" - } - }, - "evaluations": [ - { - "evaluation_id": "terminal-bench-2.0/terminus-2__minimax-m2/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2025-11-01", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 30.0, - "uncertainty": { - "standard_error": { - "value": 2.7 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Terminus 2\" -m \"MiniMax M2\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Terminus 2\" -m \"MiniMax M2\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/ministral_ministral-3b-instruct.json b/data/models/ministral_ministral-3b-instruct.json deleted file mode 100644 index 61ba26559799a03d6822918a4f75eba68607f75c..0000000000000000000000000000000000000000 --- a/data/models/ministral_ministral-3b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Ministral-3b-instruct", - "id": "ministral/Ministral-3b-instruct", - "developer": "ministral", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "3.316" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ministral_Ministral-3b-instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1358 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3192 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0083 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2517 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3382 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1093 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/mistral-community_mistral-7b-v0.2.json b/data/models/mistral-community_mistral-7b-v0.2.json deleted file mode 100644 index 137a667649924eb05cc60e1ca04125f5ef59fae8..0000000000000000000000000000000000000000 --- a/data/models/mistral-community_mistral-7b-v0.2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mistral-7B-v0.2", - "id": "mistral-community/Mistral-7B-v0.2", - "developer": "mistral-community", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/mistral-community_Mistral-7B-v0.2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2266 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.451 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0302 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2919 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4032 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2953 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/mistral-community_mixtral-8x22b-v0.1.json b/data/models/mistral-community_mixtral-8x22b-v0.1.json deleted file mode 100644 index 89f3e936dde6e50bde93706bef33d33804112098..0000000000000000000000000000000000000000 --- a/data/models/mistral-community_mixtral-8x22b-v0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mixtral-8x22B-v0.1", - "id": "mistral-community/Mixtral-8x22B-v0.1", - "developer": "mistral-community", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Unknown", - "params_billions": "0.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/mistral-community_Mixtral-8x22B-v0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3167 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.38 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1543 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.33 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3533 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.36 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/mistral-community_mixtral-8x22b-v0.3.json b/data/models/mistral-community_mixtral-8x22b-v0.3.json deleted file mode 100644 index 36f81209b67dcbbab81cd532f7746ad978f2de9f..0000000000000000000000000000000000000000 --- a/data/models/mistral-community_mixtral-8x22b-v0.3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "mixtral-8x22B-v0.3", - "id": "mistral-community/mixtral-8x22B-v0.3", - "developer": "mistral-community", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MixtralForCausalLM", - "params_billions": "140.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/mistral-community_mixtral-8x22B-v0.3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2583 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.625 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1835 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3775 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4037 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4639 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/mistralai_codestral-22b-v0.1.json b/data/models/mistralai_codestral-22b-v0.1.json deleted file mode 100644 index d4f29cf97a80ad38386e7b5a01e267ce4bb21896..0000000000000000000000000000000000000000 --- a/data/models/mistralai_codestral-22b-v0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Codestral-22B-v0.1", - "id": "mistralai/Codestral-22B-v0.1", - "developer": "mistralai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "22.247" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/mistralai_Codestral-22B-v0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5772 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5139 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1005 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2987 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4187 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3156 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/mistralai_ministral-8b-instruct-2410-fc.json b/data/models/mistralai_ministral-8b-instruct-2410-fc.json deleted file mode 100644 index 26904d72b9c6197b4c8402445c3ddcd8f69d3b2d..0000000000000000000000000000000000000000 --- a/data/models/mistralai_ministral-8b-instruct-2410-fc.json +++ /dev/null @@ -1,905 +0,0 @@ -{ - "model_info": { - "name": "Ministral-8B-Instruct-2410 (FC)", - "id": "mistralai/ministral-8b-instruct-2410-fc", - "developer": "mistralai", - "additional_details": { - "raw_model_name": "Ministral-8B-Instruct-2410 (FC)", - "organization": "Mistral AI", - "license": "Mistral AI Research License", - "mode": "FC", - "model_link": "https://huggingface.co/mistralai/Ministral-8B-Instruct-2410" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/mistralai/ministral-8b-instruct-2410-fc/1775236112.421141", - "retrieved_timestamp": "1775236112.421141", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 105.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 11.1 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 70.01 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 82.07 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 212.99 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 568.59 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 1.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 2.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 4.52 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 3.87 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 7.1 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 2.58 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 100.0 - } - }, - { - "evaluation_result_id": "bfcl.format_sensitivity.max_delta::max_delta", - "evaluation_name": "bfcl.format_sensitivity.max_delta", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.format_sensitivity.max_delta", - "metric_name": "Format sensitivity max delta", - "metric_kind": "difference", - "metric_unit": "percentage_points", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Format Sensitivity Max Delta" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.format_sensitivity.stddev::stddev", - "evaluation_name": "bfcl.format_sensitivity.stddev", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.format_sensitivity.stddev", - "metric_name": "Format sensitivity standard deviation", - "metric_kind": "difference", - "metric_unit": "percentage_points", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Format Sensitivity Standard Deviation" - } - }, - "score_details": { - "score": 0.0 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/mistralai_ministral-8b-instruct-2410.json b/data/models/mistralai_ministral-8b-instruct-2410.json deleted file mode 100644 index 41284344810c91a84a758ed9d359f3a363d1f9fa..0000000000000000000000000000000000000000 --- a/data/models/mistralai_ministral-8b-instruct-2410.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Ministral-8B-Instruct-2410", - "id": "mistralai/Ministral-8B-Instruct-2410", - "developer": "mistralai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "8.02" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/mistralai_Ministral-8B-Instruct-2410/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5896 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4762 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1956 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2844 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4138 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3291 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/mistralai_mistral-7b-instruct-v0.1.json b/data/models/mistralai_mistral-7b-instruct-v0.1.json deleted file mode 100644 index a4d81bc5d70ec66d839cc40204a27ae66c4b1602..0000000000000000000000000000000000000000 --- a/data/models/mistralai_mistral-7b-instruct-v0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mistral-7B-Instruct-v0.1", - "id": "mistralai/Mistral-7B-Instruct-v0.1", - "developer": "mistralai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/mistralai_Mistral-7B-Instruct-v0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4487 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3355 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0227 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.25 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3848 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2414 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/mistralai_mistral-7b-instruct-v0.2.json b/data/models/mistralai_mistral-7b-instruct-v0.2.json deleted file mode 100644 index b6aadb3aa767abea1879e550efe8252071b18c56..0000000000000000000000000000000000000000 --- a/data/models/mistralai_mistral-7b-instruct-v0.2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mistral-7B-Instruct-v0.2", - "id": "mistralai/Mistral-7B-Instruct-v0.2", - "developer": "mistralai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/mistralai_Mistral-7B-Instruct-v0.2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5496 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.446 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0302 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.276 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3966 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2717 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/mistralai_mistral-7b-instruct-v0.3.json b/data/models/mistralai_mistral-7b-instruct-v0.3.json deleted file mode 100644 index 0f7fe68f0b7a1a194dc76677de8b8b0fb7a1f4bb..0000000000000000000000000000000000000000 --- a/data/models/mistralai_mistral-7b-instruct-v0.3.json +++ /dev/null @@ -1,2262 +0,0 @@ -{ - "model_info": { - "name": "Mistral Instruct v0.3 7B", - "id": "mistralai/mistral-7b-instruct-v0.3", - "developer": "mistralai", - "inference_platform": "unknown", - "additional_details": { - "model_id_aliases": [ - "mistralai/Mistral-7B-Instruct-v0.3" - ] - } - }, - "evaluations": [ - { - "evaluation_id": "helm_capabilities/mistralai_mistral-7b-instruct-v0.3/1774096304.5056698", - "retrieved_timestamp": "1774096304.5056698", - "source_metadata": { - "source_name": "helm_capabilities", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_capabilities", - "evaluation_results": [ - { - "evaluation_name": "Mean score", - "source_data": { - "dataset_name": "helm_capabilities", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "The mean of the scores from all columns.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.376, - "details": { - "description": "", - "tab": "Accuracy", - "Mean score - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"3.386352003847275\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU-Pro", - "source_data": { - "dataset_name": "MMLU-Pro", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on MMLU-Pro", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.277, - "details": { - "description": "min=0.277, mean=0.277, max=0.277, sum=0.277 (1)", - "tab": "Accuracy", - "MMLU-Pro - Observed inference time (s)": "{\"description\": \"min=2.0, mean=2.0, max=2.0, sum=2.0 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.999533802509308\"}", - "MMLU-Pro - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "MMLU-Pro - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - # prompt tokens": "{\"description\": \"min=260.915, mean=260.915, max=260.915, sum=260.915 (1)\", \"tab\": \"General information\", \"score\": \"260.915\"}", - "MMLU-Pro - # output tokens": "{\"description\": \"min=272.103, mean=272.103, max=272.103, sum=272.103 (1)\", \"tab\": \"General information\", \"score\": \"272.103\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"all\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.303, - "details": { - "description": "min=0.303, mean=0.303, max=0.303, sum=0.303 (1)", - "tab": "Accuracy", - "GPQA - Observed inference time (s)": "{\"description\": \"min=2.285, mean=2.285, max=2.285, sum=2.285 (1)\", \"tab\": \"Efficiency\", \"score\": \"2.284658104849503\"}", - "GPQA - # eval": "{\"description\": \"min=446, mean=446, max=446, sum=446 (1)\", \"tab\": \"General information\", \"score\": \"446.0\"}", - "GPQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - # prompt tokens": "{\"description\": \"min=281.998, mean=281.998, max=281.998, sum=281.998 (1)\", \"tab\": \"General information\", \"score\": \"281.99775784753365\"}", - "GPQA - # output tokens": "{\"description\": \"min=387.971, mean=387.971, max=387.971, sum=387.971 (1)\", \"tab\": \"General information\", \"score\": \"387.9708520179372\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"gpqa_main\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "IFEval Strict Acc on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.567, - "details": { - "description": "min=0.567, mean=0.567, max=0.567, sum=0.567 (1)", - "tab": "Accuracy", - "IFEval - Observed inference time (s)": "{\"description\": \"min=2.535, mean=2.535, max=2.535, sum=2.535 (1)\", \"tab\": \"Efficiency\", \"score\": \"2.5349821145345013\"}", - "IFEval - # eval": "{\"description\": \"min=541, mean=541, max=541, sum=541 (1)\", \"tab\": \"General information\", \"score\": \"541.0\"}", - "IFEval - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - # prompt tokens": "{\"description\": \"min=51.309, mean=51.309, max=51.309, sum=51.309 (1)\", \"tab\": \"General information\", \"score\": \"51.3086876155268\"}", - "IFEval - # output tokens": "{\"description\": \"min=449.725, mean=449.725, max=449.725, sum=449.725 (1)\", \"tab\": \"General information\", \"score\": \"449.72458410351203\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WildBench", - "source_data": { - "dataset_name": "WildBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "WB Score on WildBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.66, - "details": { - "description": "min=0.66, mean=0.66, max=0.66, sum=0.66 (1)", - "tab": "Accuracy", - "WildBench - Observed inference time (s)": "{\"description\": \"min=5.901, mean=5.901, max=5.901, sum=5.901 (1)\", \"tab\": \"Efficiency\", \"score\": \"5.900532631635666\"}", - "WildBench - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "WildBench - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # prompt tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # output tokens": "{\"description\": \"min=702.754, mean=702.754, max=702.754, sum=702.754 (1)\", \"tab\": \"General information\", \"score\": \"702.754\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"v2\"" - } - } - }, - { - "evaluation_name": "Omni-MATH", - "source_data": { - "dataset_name": "Omni-MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Acc on Omni-MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.072, - "details": { - "description": "min=0.072, mean=0.072, max=0.072, sum=0.072 (1)", - "tab": "Accuracy", - "Omni-MATH - Observed inference time (s)": "{\"description\": \"min=4.212, mean=4.212, max=4.212, sum=4.212 (1)\", \"tab\": \"Efficiency\", \"score\": \"4.212053365707398\"}", - "Omni-MATH - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "Omni-MATH - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - # prompt tokens": "{\"description\": \"min=119.373, mean=119.373, max=119.373, sum=119.373 (1)\", \"tab\": \"General information\", \"score\": \"119.373\"}", - "Omni-MATH - # output tokens": "{\"description\": \"min=678.438, mean=678.438, max=678.438, sum=678.438 (1)\", \"tab\": \"General information\", \"score\": \"678.438\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_lite/mistralai_mistral-7b-instruct-v0.3/1774096306.427425", - "retrieved_timestamp": "1774096306.427425", - "source_metadata": { - "source_name": "helm_lite", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_lite", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_lite", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.196, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.6493133583021223\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.716, - "details": { - "description": "min=0.716, mean=0.716, max=0.716, sum=0.716 (1)", - "tab": "Accuracy", - "NarrativeQA - Observed inference time (s)": "{\"description\": \"min=0.813, mean=0.813, max=0.813, sum=0.813 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.8132137520212522\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=3924.33, mean=3924.33, max=3924.33, sum=3924.33 (1)\", \"tab\": \"General information\", \"score\": \"3924.3295774647886\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=7.107, mean=7.107, max=7.107, sum=7.107 (1)\", \"tab\": \"General information\", \"score\": \"7.107042253521127\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (closed-book)", - "source_data": { - "dataset_name": "NaturalQuestions (closed-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (closed-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.253, - "details": { - "description": "min=0.253, mean=0.253, max=0.253, sum=0.253 (1)", - "tab": "Accuracy", - "NaturalQuestions (open-book) - Observed inference time (s)": "{\"description\": \"min=0.563, mean=0.563, max=0.563, sum=0.563 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.5634698050022126\"}", - "NaturalQuestions (closed-book) - Observed inference time (s)": "{\"description\": \"min=0.535, mean=0.535, max=0.535, sum=0.535 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.5347676448822022\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=2498.79, mean=2498.79, max=2498.79, sum=2498.79 (1)\", \"tab\": \"General information\", \"score\": \"2498.79\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=12.448, mean=12.448, max=12.448, sum=12.448 (1)\", \"tab\": \"General information\", \"score\": \"12.448\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=172.069, mean=172.069, max=172.069, sum=172.069 (1)\", \"tab\": \"General information\", \"score\": \"172.069\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=20.461, mean=20.461, max=20.461, sum=20.461 (1)\", \"tab\": \"General information\", \"score\": \"20.461\"}" - } - }, - "generation_config": { - "additional_details": { - "mode": "\"closedbook\"" - } - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.79, - "details": { - "description": "min=0.79, mean=0.79, max=0.79, sum=0.79 (1)", - "tab": "Accuracy", - "OpenbookQA - Observed inference time (s)": "{\"description\": \"min=0.256, mean=0.256, max=0.256, sum=0.256 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.25593132400512697\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=289.15, mean=289.15, max=289.15, sum=289.15 (1)\", \"tab\": \"General information\", \"score\": \"289.15\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "dataset": "\"openbookqa\"", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.51, - "details": { - "description": "min=0.27, mean=0.51, max=0.79, sum=2.551 (5)", - "tab": "Accuracy", - "MMLU - Observed inference time (s)": "{\"description\": \"min=0.221, mean=0.372, max=0.487, sum=1.862 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.37230395750413864\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=411.44, mean=532.091, max=696.175, sum=2660.455 (5)\", \"tab\": \"General information\", \"score\": \"532.0910877192983\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"college_chemistry\", \"computer_security\", \"econometrics\", \"us_foreign_policy\"]", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MATH", - "source_data": { - "dataset_name": "MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Equivalent (CoT) on MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.289, - "details": { - "description": "min=0.115, mean=0.289, max=0.477, sum=2.02 (7)", - "tab": "Accuracy", - "MATH - Observed inference time (s)": "{\"description\": \"min=2.027, mean=2.656, max=3.039, sum=18.593 (7)\", \"tab\": \"Efficiency\", \"score\": \"2.656151831465352\"}", - "MATH - # eval": "{\"description\": \"min=30, mean=62.429, max=135, sum=437 (7)\", \"tab\": \"General information\", \"score\": \"62.42857142857143\"}", - "MATH - # train": "{\"description\": \"min=8, mean=8, max=8, sum=56 (7)\", \"tab\": \"General information\", \"score\": \"8.0\"}", - "MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MATH - # prompt tokens": "{\"description\": \"min=991.615, mean=1455.266, max=2502.962, sum=10186.865 (7)\", \"tab\": \"General information\", \"score\": \"1455.2664139976257\"}", - "MATH - # output tokens": "{\"description\": \"min=123.616, mean=149.99, max=172.789, sum=1049.933 (7)\", \"tab\": \"General information\", \"score\": \"149.99043902740354\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"algebra\", \"counting_and_probability\", \"geometry\", \"intermediate_algebra\", \"number_theory\", \"prealgebra\", \"precalculus\"]", - "level": "\"1\"", - "use_official_examples": "\"False\"", - "use_chain_of_thought": "\"True\"" - } - } - }, - { - "evaluation_name": "GSM8K", - "source_data": { - "dataset_name": "GSM8K", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on GSM8K", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.538, - "details": { - "description": "min=0.538, mean=0.538, max=0.538, sum=0.538 (1)", - "tab": "Accuracy", - "GSM8K - Observed inference time (s)": "{\"description\": \"min=3.95, mean=3.95, max=3.95, sum=3.95 (1)\", \"tab\": \"Efficiency\", \"score\": \"3.949965229511261\"}", - "GSM8K - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "GSM8K - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "GSM8K - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GSM8K - # prompt tokens": "{\"description\": \"min=1187.268, mean=1187.268, max=1187.268, sum=1187.268 (1)\", \"tab\": \"General information\", \"score\": \"1187.268\"}", - "GSM8K - # output tokens": "{\"description\": \"min=196.611, mean=196.611, max=196.611, sum=196.611 (1)\", \"tab\": \"General information\", \"score\": \"196.611\"}" - } - }, - "generation_config": { - "additional_details": { - "stop": "\"none\"" - } - } - }, - { - "evaluation_name": "LegalBench", - "source_data": { - "dataset_name": "LegalBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on LegalBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.331, - "details": { - "description": "min=0.063, mean=0.331, max=0.733, sum=1.655 (5)", - "tab": "Accuracy", - "LegalBench - Observed inference time (s)": "{\"description\": \"min=0.316, mean=0.489, max=0.855, sum=2.444 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.4887186054518059\"}", - "LegalBench - # eval": "{\"description\": \"min=95, mean=409.4, max=1000, sum=2047 (5)\", \"tab\": \"General information\", \"score\": \"409.4\"}", - "LegalBench - # train": "{\"description\": \"min=4, mean=4.8, max=5, sum=24 (5)\", \"tab\": \"General information\", \"score\": \"4.8\"}", - "LegalBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "LegalBench - # prompt tokens": "{\"description\": \"min=236.453, mean=1750.748, max=7224.488, sum=8753.741 (5)\", \"tab\": \"General information\", \"score\": \"1750.7482458432962\"}", - "LegalBench - # output tokens": "{\"description\": \"min=2, mean=9.174, max=15.242, sum=45.871 (5)\", \"tab\": \"General information\", \"score\": \"9.17419274343898\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"abercrombie\", \"corporate_lobbying\", \"function_of_decision_section\", \"international_citizenship_questions\", \"proa\"]" - } - } - }, - { - "evaluation_name": "MedQA", - "source_data": { - "dataset_name": "MedQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MedQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.517, - "details": { - "description": "min=0.517, mean=0.517, max=0.517, sum=0.517 (1)", - "tab": "Accuracy", - "MedQA - Observed inference time (s)": "{\"description\": \"min=0.418, mean=0.418, max=0.418, sum=0.418 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.4182186216767692\"}", - "MedQA - # eval": "{\"description\": \"min=503, mean=503, max=503, sum=503 (1)\", \"tab\": \"General information\", \"score\": \"503.0\"}", - "MedQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MedQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MedQA - # prompt tokens": "{\"description\": \"min=1202.093, mean=1202.093, max=1202.093, sum=1202.093 (1)\", \"tab\": \"General information\", \"score\": \"1202.0934393638172\"}", - "MedQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WMT 2014", - "source_data": { - "dataset_name": "WMT 2014", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "BLEU-4 on WMT 2014", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.142, - "details": { - "description": "min=0.047, mean=0.142, max=0.184, sum=0.712 (5)", - "tab": "Accuracy", - "WMT 2014 - Observed inference time (s)": "{\"description\": \"min=0.582, mean=0.775, max=0.872, sum=3.875 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.7750062139801958\"}", - "WMT 2014 - # eval": "{\"description\": \"min=503, mean=568.8, max=832, sum=2844 (5)\", \"tab\": \"General information\", \"score\": \"568.8\"}", - "WMT 2014 - # train": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "WMT 2014 - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WMT 2014 - # prompt tokens": "{\"description\": \"min=148.306, mean=162.433, max=181.018, sum=812.166 (5)\", \"tab\": \"General information\", \"score\": \"162.43317355482492\"}", - "WMT 2014 - # output tokens": "{\"description\": \"min=28.3, mean=30.51, max=31.912, sum=152.552 (5)\", \"tab\": \"General information\", \"score\": \"30.510483732222053\"}" - } - }, - "generation_config": { - "additional_details": { - "language_pair": "[\"cs-en\", \"de-en\", \"fr-en\", \"hi-en\", \"ru-en\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_mmlu/mistralai_mistral-7b-instruct-v0.3/1774096312.00548", - "retrieved_timestamp": "1774096312.00548", - "source_metadata": { - "source_name": "helm_mmlu", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_mmlu", - "evaluation_results": [ - { - "evaluation_name": "MMLU All Subjects", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU All Subjects", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.599, - "details": { - "description": "min=0.258, mean=0.599, max=0.881, sum=68.3 (114)", - "tab": "Accuracy", - "MMLU All Subjects - Observed inference time (s)": "{\"description\": \"min=0.212, mean=0.526, max=1.438, sum=59.959 (114)\", \"tab\": \"Efficiency\", \"score\": \"0.525951832745908\"}", - "MMLU All Subjects - # eval": "{\"description\": \"min=100, mean=246.351, max=1534, sum=28084 (114)\", \"tab\": \"General information\", \"score\": \"246.35087719298247\"}", - "MMLU All Subjects - # train": "{\"description\": \"min=5, mean=5, max=5, sum=570 (114)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU All Subjects - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU All Subjects - # prompt tokens": "{\"description\": \"min=317.924, mean=705.273, max=3098.109, sum=80401.178 (114)\", \"tab\": \"General information\", \"score\": \"705.2734899593811\"}", - "MMLU All Subjects - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=114 (114)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_name": "Abstract Algebra", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Abstract Algebra", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.27, - "details": { - "description": "min=0.27, mean=0.27, max=0.27, sum=0.54 (2)", - "tab": "Accuracy", - "Abstract Algebra - Observed inference time (s)": "{\"description\": \"min=0.321, mean=0.321, max=0.321, sum=0.642 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.32117165088653565\"}", - "Abstract Algebra - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Abstract Algebra - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Abstract Algebra - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Abstract Algebra - # prompt tokens": "{\"description\": \"min=411.44, mean=411.44, max=411.44, sum=822.88 (2)\", \"tab\": \"General information\", \"score\": \"411.44\"}", - "Abstract Algebra - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"abstract_algebra\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_abstract_algebra\"" - } - } - }, - { - "evaluation_name": "Anatomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Anatomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.585, - "details": { - "description": "min=0.585, mean=0.585, max=0.585, sum=1.17 (2)", - "tab": "Accuracy", - "Anatomy - Observed inference time (s)": "{\"description\": \"min=0.246, mean=0.246, max=0.246, sum=0.493 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.24627229902479383\"}", - "Anatomy - # eval": "{\"description\": \"min=135, mean=135, max=135, sum=270 (2)\", \"tab\": \"General information\", \"score\": \"135.0\"}", - "Anatomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Anatomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Anatomy - # prompt tokens": "{\"description\": \"min=416.089, mean=416.089, max=416.089, sum=832.178 (2)\", \"tab\": \"General information\", \"score\": \"416.0888888888889\"}", - "Anatomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"anatomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_anatomy\"" - } - } - }, - { - "evaluation_name": "College Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on College Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.343, - "details": { - "description": "min=0.343, mean=0.343, max=0.343, sum=0.686 (2)", - "tab": "Accuracy", - "College Chemistry - Observed inference time (s)": "{\"description\": \"min=0.221, mean=0.221, max=0.221, sum=0.442 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.22099271774291993\"}", - "College Biology - Observed inference time (s)": "{\"description\": \"min=0.7, mean=0.7, max=0.7, sum=1.399 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6997380173868604\"}", - "College Computer Science - Observed inference time (s)": "{\"description\": \"min=0.466, mean=0.466, max=0.466, sum=0.932 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4661028146743774\"}", - "College Mathematics - Observed inference time (s)": "{\"description\": \"min=0.212, mean=0.212, max=0.212, sum=0.424 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.21210591793060302\"}", - "College Medicine - Observed inference time (s)": "{\"description\": \"min=0.387, mean=0.387, max=0.387, sum=0.774 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3871537646806309\"}", - "College Physics - Observed inference time (s)": "{\"description\": \"min=0.455, mean=0.455, max=0.455, sum=0.91 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.45503536392660704\"}", - "College Chemistry - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Chemistry - # prompt tokens": "{\"description\": \"min=636.71, mean=636.71, max=636.71, sum=1273.42 (2)\", \"tab\": \"General information\", \"score\": \"636.71\"}", - "College Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Biology - # eval": "{\"description\": \"min=144, mean=144, max=144, sum=288 (2)\", \"tab\": \"General information\", \"score\": \"144.0\"}", - "College Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # prompt tokens": "{\"description\": \"min=559.799, mean=559.799, max=559.799, sum=1119.597 (2)\", \"tab\": \"General information\", \"score\": \"559.7986111111111\"}", - "College Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # prompt tokens": "{\"description\": \"min=911.17, mean=911.17, max=911.17, sum=1822.34 (2)\", \"tab\": \"General information\", \"score\": \"911.17\"}", - "College Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Mathematics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # prompt tokens": "{\"description\": \"min=667.31, mean=667.31, max=667.31, sum=1334.62 (2)\", \"tab\": \"General information\", \"score\": \"667.31\"}", - "College Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Medicine - # eval": "{\"description\": \"min=173, mean=173, max=173, sum=346 (2)\", \"tab\": \"General information\", \"score\": \"173.0\"}", - "College Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # prompt tokens": "{\"description\": \"min=601.41, mean=601.41, max=601.41, sum=1202.821 (2)\", \"tab\": \"General information\", \"score\": \"601.4104046242775\"}", - "College Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Physics - # eval": "{\"description\": \"min=102, mean=102, max=102, sum=204 (2)\", \"tab\": \"General information\", \"score\": \"102.0\"}", - "College Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # prompt tokens": "{\"description\": \"min=560.029, mean=560.029, max=560.029, sum=1120.059 (2)\", \"tab\": \"General information\", \"score\": \"560.0294117647059\"}", - "College Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"college_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_college_physics\"" - } - } - }, - { - "evaluation_name": "Computer Security", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Computer Security", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7, - "details": { - "description": "min=0.7, mean=0.7, max=0.7, sum=1.4 (2)", - "tab": "Accuracy", - "Computer Security - Observed inference time (s)": "{\"description\": \"min=0.426, mean=0.426, max=0.426, sum=0.853 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4263953256607056\"}", - "Computer Security - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Computer Security - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Computer Security - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Computer Security - # prompt tokens": "{\"description\": \"min=433.94, mean=433.94, max=433.94, sum=867.88 (2)\", \"tab\": \"General information\", \"score\": \"433.94\"}", - "Computer Security - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"computer_security\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_computer_security\"" - } - } - }, - { - "evaluation_name": "Econometrics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Econometrics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.421, - "details": { - "description": "min=0.421, mean=0.421, max=0.421, sum=0.842 (2)", - "tab": "Accuracy", - "Econometrics - Observed inference time (s)": "{\"description\": \"min=0.406, mean=0.406, max=0.406, sum=0.813 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.406455958098696\"}", - "Econometrics - # eval": "{\"description\": \"min=114, mean=114, max=114, sum=228 (2)\", \"tab\": \"General information\", \"score\": \"114.0\"}", - "Econometrics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Econometrics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Econometrics - # prompt tokens": "{\"description\": \"min=696.175, mean=696.175, max=696.175, sum=1392.351 (2)\", \"tab\": \"General information\", \"score\": \"696.1754385964912\"}", - "Econometrics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"econometrics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_econometrics\"" - } - } - }, - { - "evaluation_name": "Global Facts", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Global Facts", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.33, - "details": { - "description": "min=0.33, mean=0.33, max=0.33, sum=0.66 (2)", - "tab": "Accuracy", - "Global Facts - Observed inference time (s)": "{\"description\": \"min=0.299, mean=0.299, max=0.299, sum=0.598 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.29881003856658933\"}", - "Global Facts - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Global Facts - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Global Facts - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Global Facts - # prompt tokens": "{\"description\": \"min=492.47, mean=492.47, max=492.47, sum=984.94 (2)\", \"tab\": \"General information\", \"score\": \"492.47\"}", - "Global Facts - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"global_facts\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_global_facts\"" - } - } - }, - { - "evaluation_name": "Jurisprudence", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Jurisprudence", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.713, - "details": { - "description": "min=0.713, mean=0.713, max=0.713, sum=1.426 (2)", - "tab": "Accuracy", - "Jurisprudence - Observed inference time (s)": "{\"description\": \"min=0.232, mean=0.232, max=0.232, sum=0.465 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.23237781833719323\"}", - "Jurisprudence - # eval": "{\"description\": \"min=108, mean=108, max=108, sum=216 (2)\", \"tab\": \"General information\", \"score\": \"108.0\"}", - "Jurisprudence - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Jurisprudence - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Jurisprudence - # prompt tokens": "{\"description\": \"min=460.093, mean=460.093, max=460.093, sum=920.185 (2)\", \"tab\": \"General information\", \"score\": \"460.0925925925926\"}", - "Jurisprudence - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"jurisprudence\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_jurisprudence\"" - } - } - }, - { - "evaluation_name": "Philosophy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Philosophy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.659, - "details": { - "description": "min=0.659, mean=0.659, max=0.659, sum=1.318 (2)", - "tab": "Accuracy", - "Philosophy - Observed inference time (s)": "{\"description\": \"min=0.899, mean=0.899, max=0.899, sum=1.798 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8987545852109167\"}", - "Philosophy - # eval": "{\"description\": \"min=311, mean=311, max=311, sum=622 (2)\", \"tab\": \"General information\", \"score\": \"311.0\"}", - "Philosophy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Philosophy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Philosophy - # prompt tokens": "{\"description\": \"min=382.82, mean=382.82, max=382.82, sum=765.64 (2)\", \"tab\": \"General information\", \"score\": \"382.81993569131834\"}", - "Philosophy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"philosophy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_philosophy\"" - } - } - }, - { - "evaluation_name": "Professional Psychology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Professional Psychology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.641, - "details": { - "description": "min=0.641, mean=0.641, max=0.641, sum=1.281 (2)", - "tab": "Accuracy", - "Professional Medicine - Observed inference time (s)": "{\"description\": \"min=0.615, mean=0.615, max=0.615, sum=1.23 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6148438769228318\"}", - "Professional Accounting - Observed inference time (s)": "{\"description\": \"min=0.825, mean=0.825, max=0.825, sum=1.651 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8254362666015084\"}", - "Professional Law - Observed inference time (s)": "{\"description\": \"min=0.682, mean=0.682, max=0.682, sum=1.364 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.68212915414937\"}", - "Professional Psychology - Observed inference time (s)": "{\"description\": \"min=0.506, mean=0.506, max=0.506, sum=1.012 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.505940170459498\"}", - "Professional Medicine - # eval": "{\"description\": \"min=272, mean=272, max=272, sum=544 (2)\", \"tab\": \"General information\", \"score\": \"272.0\"}", - "Professional Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Medicine - # prompt tokens": "{\"description\": \"min=1288.143, mean=1288.143, max=1288.143, sum=2576.287 (2)\", \"tab\": \"General information\", \"score\": \"1288.1433823529412\"}", - "Professional Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Accounting - # eval": "{\"description\": \"min=282, mean=282, max=282, sum=564 (2)\", \"tab\": \"General information\", \"score\": \"282.0\"}", - "Professional Accounting - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Accounting - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # prompt tokens": "{\"description\": \"min=805.496, mean=805.496, max=805.496, sum=1610.993 (2)\", \"tab\": \"General information\", \"score\": \"805.4964539007092\"}", - "Professional Accounting - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Law - # eval": "{\"description\": \"min=1534, mean=1534, max=1534, sum=3068 (2)\", \"tab\": \"General information\", \"score\": \"1534.0\"}", - "Professional Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # prompt tokens": "{\"description\": \"min=1858.711, mean=1858.711, max=1858.711, sum=3717.421 (2)\", \"tab\": \"General information\", \"score\": \"1858.7105606258149\"}", - "Professional Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Psychology - # eval": "{\"description\": \"min=612, mean=612, max=612, sum=1224 (2)\", \"tab\": \"General information\", \"score\": \"612.0\"}", - "Professional Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # prompt tokens": "{\"description\": \"min=654.278, mean=654.278, max=654.278, sum=1308.556 (2)\", \"tab\": \"General information\", \"score\": \"654.2777777777778\"}", - "Professional Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"professional_psychology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_professional_psychology\"" - } - } - }, - { - "evaluation_name": "Us Foreign Policy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Us Foreign Policy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.79, - "details": { - "description": "min=0.79, mean=0.79, max=0.79, sum=1.58 (2)", - "tab": "Accuracy", - "Us Foreign Policy - Observed inference time (s)": "{\"description\": \"min=0.487, mean=0.487, max=0.487, sum=0.973 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.48650413513183594\"}", - "Us Foreign Policy - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Us Foreign Policy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Us Foreign Policy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Us Foreign Policy - # prompt tokens": "{\"description\": \"min=482.19, mean=482.19, max=482.19, sum=964.38 (2)\", \"tab\": \"General information\", \"score\": \"482.19\"}", - "Us Foreign Policy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"us_foreign_policy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_us_foreign_policy\"" - } - } - }, - { - "evaluation_name": "Astronomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Astronomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.638, - "details": { - "description": "min=0.638, mean=0.638, max=0.638, sum=1.276 (2)", - "tab": "Accuracy", - "Astronomy - Observed inference time (s)": "{\"description\": \"min=0.678, mean=0.678, max=0.678, sum=1.355 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6775346147386652\"}", - "Astronomy - # eval": "{\"description\": \"min=152, mean=152, max=152, sum=304 (2)\", \"tab\": \"General information\", \"score\": \"152.0\"}", - "Astronomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Astronomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Astronomy - # prompt tokens": "{\"description\": \"min=674.987, mean=674.987, max=674.987, sum=1349.974 (2)\", \"tab\": \"General information\", \"score\": \"674.9868421052631\"}", - "Astronomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"astronomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_astronomy\"" - } - } - }, - { - "evaluation_name": "Business Ethics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Business Ethics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.57, - "details": { - "description": "min=0.57, mean=0.57, max=0.57, sum=1.14 (2)", - "tab": "Accuracy", - "Business Ethics - Observed inference time (s)": "{\"description\": \"min=0.645, mean=0.645, max=0.645, sum=1.289 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6446590375900269\"}", - "Business Ethics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Business Ethics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Business Ethics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Business Ethics - # prompt tokens": "{\"description\": \"min=653.6, mean=653.6, max=653.6, sum=1307.2 (2)\", \"tab\": \"General information\", \"score\": \"653.6\"}", - "Business Ethics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"business_ethics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_business_ethics\"" - } - } - }, - { - "evaluation_name": "Clinical Knowledge", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Clinical Knowledge", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.687, - "details": { - "description": "min=0.687, mean=0.687, max=0.687, sum=1.374 (2)", - "tab": "Accuracy", - "Clinical Knowledge - Observed inference time (s)": "{\"description\": \"min=0.844, mean=0.844, max=0.844, sum=1.687 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8436905698956184\"}", - "Clinical Knowledge - # eval": "{\"description\": \"min=265, mean=265, max=265, sum=530 (2)\", \"tab\": \"General information\", \"score\": \"265.0\"}", - "Clinical Knowledge - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Clinical Knowledge - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Clinical Knowledge - # prompt tokens": "{\"description\": \"min=496.174, mean=496.174, max=496.174, sum=992.347 (2)\", \"tab\": \"General information\", \"score\": \"496.1735849056604\"}", - "Clinical Knowledge - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"clinical_knowledge\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_clinical_knowledge\"" - } - } - }, - { - "evaluation_name": "Conceptual Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Conceptual Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.549, - "details": { - "description": "min=0.549, mean=0.549, max=0.549, sum=1.098 (2)", - "tab": "Accuracy", - "Conceptual Physics - Observed inference time (s)": "{\"description\": \"min=0.333, mean=0.333, max=0.333, sum=0.666 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.33306963900302317\"}", - "Conceptual Physics - # eval": "{\"description\": \"min=235, mean=235, max=235, sum=470 (2)\", \"tab\": \"General information\", \"score\": \"235.0\"}", - "Conceptual Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Conceptual Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Conceptual Physics - # prompt tokens": "{\"description\": \"min=343.285, mean=343.285, max=343.285, sum=686.57 (2)\", \"tab\": \"General information\", \"score\": \"343.2851063829787\"}", - "Conceptual Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"conceptual_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_conceptual_physics\"" - } - } - }, - { - "evaluation_name": "Electrical Engineering", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Electrical Engineering", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.572, - "details": { - "description": "min=0.572, mean=0.572, max=0.572, sum=1.145 (2)", - "tab": "Accuracy", - "Electrical Engineering - Observed inference time (s)": "{\"description\": \"min=0.392, mean=0.392, max=0.392, sum=0.784 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3922290703345989\"}", - "Electrical Engineering - # eval": "{\"description\": \"min=145, mean=145, max=145, sum=290 (2)\", \"tab\": \"General information\", \"score\": \"145.0\"}", - "Electrical Engineering - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Electrical Engineering - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Electrical Engineering - # prompt tokens": "{\"description\": \"min=510.379, mean=510.379, max=510.379, sum=1020.759 (2)\", \"tab\": \"General information\", \"score\": \"510.37931034482756\"}", - "Electrical Engineering - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"electrical_engineering\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_electrical_engineering\"" - } - } - }, - { - "evaluation_name": "Elementary Mathematics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Elementary Mathematics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.402, - "details": { - "description": "min=0.402, mean=0.402, max=0.402, sum=0.804 (2)", - "tab": "Accuracy", - "Elementary Mathematics - Observed inference time (s)": "{\"description\": \"min=0.676, mean=0.676, max=0.676, sum=1.352 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6761655416438188\"}", - "Elementary Mathematics - # eval": "{\"description\": \"min=378, mean=378, max=378, sum=756 (2)\", \"tab\": \"General information\", \"score\": \"378.0\"}", - "Elementary Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Elementary Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Elementary Mathematics - # prompt tokens": "{\"description\": \"min=622.386, mean=622.386, max=622.386, sum=1244.772 (2)\", \"tab\": \"General information\", \"score\": \"622.3862433862433\"}", - "Elementary Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"elementary_mathematics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_elementary_mathematics\"" - } - } - }, - { - "evaluation_name": "Formal Logic", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Formal Logic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.397, - "details": { - "description": "min=0.397, mean=0.397, max=0.397, sum=0.794 (2)", - "tab": "Accuracy", - "Formal Logic - Observed inference time (s)": "{\"description\": \"min=0.734, mean=0.734, max=0.734, sum=1.467 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7336057802987477\"}", - "Formal Logic - # eval": "{\"description\": \"min=126, mean=126, max=126, sum=252 (2)\", \"tab\": \"General information\", \"score\": \"126.0\"}", - "Formal Logic - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Formal Logic - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Formal Logic - # prompt tokens": "{\"description\": \"min=727.984, mean=727.984, max=727.984, sum=1455.968 (2)\", \"tab\": \"General information\", \"score\": \"727.984126984127\"}", - "Formal Logic - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"formal_logic\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_formal_logic\"" - } - } - }, - { - "evaluation_name": "High School World History", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on High School World History", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.759, - "details": { - "description": "min=0.759, mean=0.759, max=0.759, sum=1.519 (2)", - "tab": "Accuracy", - "High School Biology - Observed inference time (s)": "{\"description\": \"min=0.805, mean=0.805, max=0.805, sum=1.61 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8049156188964843\"}", - "High School Chemistry - Observed inference time (s)": "{\"description\": \"min=0.44, mean=0.44, max=0.44, sum=0.881 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.44036899529067164\"}", - "High School Computer Science - Observed inference time (s)": "{\"description\": \"min=0.435, mean=0.435, max=0.435, sum=0.869 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4347002100944519\"}", - "High School European History - Observed inference time (s)": "{\"description\": \"min=0.445, mean=0.445, max=0.445, sum=0.891 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4453156341205944\"}", - "High School Geography - Observed inference time (s)": "{\"description\": \"min=0.331, mean=0.331, max=0.331, sum=0.661 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3305177327358361\"}", - "High School Government And Politics - Observed inference time (s)": "{\"description\": \"min=0.545, mean=0.545, max=0.545, sum=1.089 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5445178654527417\"}", - "High School Macroeconomics - Observed inference time (s)": "{\"description\": \"min=0.53, mean=0.53, max=0.53, sum=1.061 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5302642871172\"}", - "High School Mathematics - Observed inference time (s)": "{\"description\": \"min=0.585, mean=0.585, max=0.585, sum=1.169 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5845282289716932\"}", - "High School Microeconomics - Observed inference time (s)": "{\"description\": \"min=0.234, mean=0.234, max=0.234, sum=0.468 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.23408917118521297\"}", - "High School Physics - Observed inference time (s)": "{\"description\": \"min=0.384, mean=0.384, max=0.384, sum=0.768 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3838195042894376\"}", - "High School Psychology - Observed inference time (s)": "{\"description\": \"min=0.274, mean=0.274, max=0.274, sum=0.547 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2735835779697523\"}", - "High School Statistics - Observed inference time (s)": "{\"description\": \"min=0.654, mean=0.654, max=0.654, sum=1.308 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6539056665367551\"}", - "High School US History - Observed inference time (s)": "{\"description\": \"min=0.942, mean=0.942, max=0.942, sum=1.883 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9417344308366963\"}", - "High School World History - Observed inference time (s)": "{\"description\": \"min=0.864, mean=0.864, max=0.864, sum=1.727 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8635432951561006\"}", - "High School Biology - # eval": "{\"description\": \"min=310, mean=310, max=310, sum=620 (2)\", \"tab\": \"General information\", \"score\": \"310.0\"}", - "High School Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Biology - # prompt tokens": "{\"description\": \"min=609.561, mean=609.561, max=609.561, sum=1219.123 (2)\", \"tab\": \"General information\", \"score\": \"609.5612903225806\"}", - "High School Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Chemistry - # eval": "{\"description\": \"min=203, mean=203, max=203, sum=406 (2)\", \"tab\": \"General information\", \"score\": \"203.0\"}", - "High School Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # prompt tokens": "{\"description\": \"min=581.798, mean=581.798, max=581.798, sum=1163.596 (2)\", \"tab\": \"General information\", \"score\": \"581.7980295566502\"}", - "High School Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "High School Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # prompt tokens": "{\"description\": \"min=997.24, mean=997.24, max=997.24, sum=1994.48 (2)\", \"tab\": \"General information\", \"score\": \"997.24\"}", - "High School Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School European History - # eval": "{\"description\": \"min=165, mean=165, max=165, sum=330 (2)\", \"tab\": \"General information\", \"score\": \"165.0\"}", - "High School European History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School European History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # prompt tokens": "{\"description\": \"min=3098.109, mean=3098.109, max=3098.109, sum=6196.218 (2)\", \"tab\": \"General information\", \"score\": \"3098.109090909091\"}", - "High School European History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Geography - # eval": "{\"description\": \"min=198, mean=198, max=198, sum=396 (2)\", \"tab\": \"General information\", \"score\": \"198.0\"}", - "High School Geography - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Geography - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # prompt tokens": "{\"description\": \"min=438.207, mean=438.207, max=438.207, sum=876.414 (2)\", \"tab\": \"General information\", \"score\": \"438.2070707070707\"}", - "High School Geography - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Government And Politics - # eval": "{\"description\": \"min=193, mean=193, max=193, sum=386 (2)\", \"tab\": \"General information\", \"score\": \"193.0\"}", - "High School Government And Politics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Government And Politics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # prompt tokens": "{\"description\": \"min=523.808, mean=523.808, max=523.808, sum=1047.617 (2)\", \"tab\": \"General information\", \"score\": \"523.8082901554404\"}", - "High School Government And Politics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Macroeconomics - # eval": "{\"description\": \"min=390, mean=390, max=390, sum=780 (2)\", \"tab\": \"General information\", \"score\": \"390.0\"}", - "High School Macroeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Macroeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # prompt tokens": "{\"description\": \"min=432.815, mean=432.815, max=432.815, sum=865.631 (2)\", \"tab\": \"General information\", \"score\": \"432.81538461538463\"}", - "High School Macroeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Mathematics - # eval": "{\"description\": \"min=270, mean=270, max=270, sum=540 (2)\", \"tab\": \"General information\", \"score\": \"270.0\"}", - "High School Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # prompt tokens": "{\"description\": \"min=593.13, mean=593.13, max=593.13, sum=1186.259 (2)\", \"tab\": \"General information\", \"score\": \"593.1296296296297\"}", - "High School Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Microeconomics - # eval": "{\"description\": \"min=238, mean=238, max=238, sum=476 (2)\", \"tab\": \"General information\", \"score\": \"238.0\"}", - "High School Microeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Microeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # prompt tokens": "{\"description\": \"min=452.345, mean=452.345, max=452.345, sum=904.689 (2)\", \"tab\": \"General information\", \"score\": \"452.34453781512605\"}", - "High School Microeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Physics - # eval": "{\"description\": \"min=151, mean=151, max=151, sum=302 (2)\", \"tab\": \"General information\", \"score\": \"151.0\"}", - "High School Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # prompt tokens": "{\"description\": \"min=631.775, mean=631.775, max=631.775, sum=1263.55 (2)\", \"tab\": \"General information\", \"score\": \"631.774834437086\"}", - "High School Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Psychology - # eval": "{\"description\": \"min=545, mean=545, max=545, sum=1090 (2)\", \"tab\": \"General information\", \"score\": \"545.0\"}", - "High School Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # prompt tokens": "{\"description\": \"min=567.873, mean=567.873, max=567.873, sum=1135.747 (2)\", \"tab\": \"General information\", \"score\": \"567.8733944954129\"}", - "High School Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Statistics - # eval": "{\"description\": \"min=216, mean=216, max=216, sum=432 (2)\", \"tab\": \"General information\", \"score\": \"216.0\"}", - "High School Statistics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Statistics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # prompt tokens": "{\"description\": \"min=922.644, mean=922.644, max=922.644, sum=1845.287 (2)\", \"tab\": \"General information\", \"score\": \"922.6435185185185\"}", - "High School Statistics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School US History - # eval": "{\"description\": \"min=204, mean=204, max=204, sum=408 (2)\", \"tab\": \"General information\", \"score\": \"204.0\"}", - "High School US History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School US History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # prompt tokens": "{\"description\": \"min=2486.446, mean=2486.446, max=2486.446, sum=4972.892 (2)\", \"tab\": \"General information\", \"score\": \"2486.4460784313724\"}", - "High School US History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School World History - # eval": "{\"description\": \"min=237, mean=237, max=237, sum=474 (2)\", \"tab\": \"General information\", \"score\": \"237.0\"}", - "High School World History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School World History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # prompt tokens": "{\"description\": \"min=1594.553, mean=1594.553, max=1594.553, sum=3189.105 (2)\", \"tab\": \"General information\", \"score\": \"1594.5527426160338\"}", - "High School World History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"high_school_world_history\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_high_school_world_history\"" - } - } - }, - { - "evaluation_name": "Human Sexuality", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Human Sexuality", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.702, - "details": { - "description": "min=0.702, mean=0.702, max=0.702, sum=1.405 (2)", - "tab": "Accuracy", - "Human Aging - Observed inference time (s)": "{\"description\": \"min=0.809, mean=0.809, max=0.809, sum=1.618 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8091403518557014\"}", - "Human Sexuality - Observed inference time (s)": "{\"description\": \"min=1.438, mean=1.438, max=1.438, sum=2.875 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.437711750278036\"}", - "Human Aging - # eval": "{\"description\": \"min=223, mean=223, max=223, sum=446 (2)\", \"tab\": \"General information\", \"score\": \"223.0\"}", - "Human Aging - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Aging - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Aging - # prompt tokens": "{\"description\": \"min=362.152, mean=362.152, max=362.152, sum=724.305 (2)\", \"tab\": \"General information\", \"score\": \"362.15246636771303\"}", - "Human Aging - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Human Sexuality - # eval": "{\"description\": \"min=131, mean=131, max=131, sum=262 (2)\", \"tab\": \"General information\", \"score\": \"131.0\"}", - "Human Sexuality - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Sexuality - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # prompt tokens": "{\"description\": \"min=403.748, mean=403.748, max=403.748, sum=807.496 (2)\", \"tab\": \"General information\", \"score\": \"403.7480916030534\"}", - "Human Sexuality - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"human_sexuality\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_human_sexuality\"" - } - } - }, - { - "evaluation_name": "International Law", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on International Law", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.76, - "details": { - "description": "min=0.76, mean=0.76, max=0.76, sum=1.521 (2)", - "tab": "Accuracy", - "International Law - Observed inference time (s)": "{\"description\": \"min=0.393, mean=0.393, max=0.393, sum=0.787 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3933255593638775\"}", - "International Law - # eval": "{\"description\": \"min=121, mean=121, max=121, sum=242 (2)\", \"tab\": \"General information\", \"score\": \"121.0\"}", - "International Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "International Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "International Law - # prompt tokens": "{\"description\": \"min=729.182, mean=729.182, max=729.182, sum=1458.364 (2)\", \"tab\": \"General information\", \"score\": \"729.1818181818181\"}", - "International Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"international_law\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_international_law\"" - } - } - }, - { - "evaluation_name": "Logical Fallacies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Logical Fallacies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.712, - "details": { - "description": "min=0.712, mean=0.712, max=0.712, sum=1.423 (2)", - "tab": "Accuracy", - "Logical Fallacies - Observed inference time (s)": "{\"description\": \"min=0.848, mean=0.848, max=0.848, sum=1.695 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8476987660296855\"}", - "Logical Fallacies - # eval": "{\"description\": \"min=163, mean=163, max=163, sum=326 (2)\", \"tab\": \"General information\", \"score\": \"163.0\"}", - "Logical Fallacies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Logical Fallacies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Logical Fallacies - # prompt tokens": "{\"description\": \"min=495.779, mean=495.779, max=495.779, sum=991.558 (2)\", \"tab\": \"General information\", \"score\": \"495.77914110429447\"}", - "Logical Fallacies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"logical_fallacies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_logical_fallacies\"" - } - } - }, - { - "evaluation_name": "Machine Learning", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Machine Learning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.455, - "details": { - "description": "min=0.455, mean=0.455, max=0.455, sum=0.911 (2)", - "tab": "Accuracy", - "Machine Learning - Observed inference time (s)": "{\"description\": \"min=0.557, mean=0.557, max=0.557, sum=1.113 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5566470899752208\"}", - "Machine Learning - # eval": "{\"description\": \"min=112, mean=112, max=112, sum=224 (2)\", \"tab\": \"General information\", \"score\": \"112.0\"}", - "Machine Learning - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Machine Learning - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Machine Learning - # prompt tokens": "{\"description\": \"min=743.83, mean=743.83, max=743.83, sum=1487.661 (2)\", \"tab\": \"General information\", \"score\": \"743.8303571428571\"}", - "Machine Learning - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"machine_learning\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_machine_learning\"" - } - } - }, - { - "evaluation_name": "Management", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Management", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.767, - "details": { - "description": "min=0.767, mean=0.767, max=0.767, sum=1.534 (2)", - "tab": "Accuracy", - "Management - Observed inference time (s)": "{\"description\": \"min=0.365, mean=0.365, max=0.365, sum=0.73 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.36507687059420985\"}", - "Management - # eval": "{\"description\": \"min=103, mean=103, max=103, sum=206 (2)\", \"tab\": \"General information\", \"score\": \"103.0\"}", - "Management - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Management - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Management - # prompt tokens": "{\"description\": \"min=324.359, mean=324.359, max=324.359, sum=648.718 (2)\", \"tab\": \"General information\", \"score\": \"324.3592233009709\"}", - "Management - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"management\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_management\"" - } - } - }, - { - "evaluation_name": "Marketing", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Marketing", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.842, - "details": { - "description": "min=0.842, mean=0.842, max=0.842, sum=1.684 (2)", - "tab": "Accuracy", - "Marketing - Observed inference time (s)": "{\"description\": \"min=0.585, mean=0.585, max=0.585, sum=1.17 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.58499161606161\"}", - "Marketing - # eval": "{\"description\": \"min=234, mean=234, max=234, sum=468 (2)\", \"tab\": \"General information\", \"score\": \"234.0\"}", - "Marketing - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Marketing - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Marketing - # prompt tokens": "{\"description\": \"min=472.423, mean=472.423, max=472.423, sum=944.846 (2)\", \"tab\": \"General information\", \"score\": \"472.4230769230769\"}", - "Marketing - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"marketing\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_marketing\"" - } - } - }, - { - "evaluation_name": "Medical Genetics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Medical Genetics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.75, - "details": { - "description": "min=0.75, mean=0.75, max=0.75, sum=1.5 (2)", - "tab": "Accuracy", - "Medical Genetics - Observed inference time (s)": "{\"description\": \"min=0.268, mean=0.268, max=0.268, sum=0.535 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2675498366355896\"}", - "Medical Genetics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Medical Genetics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Medical Genetics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Medical Genetics - # prompt tokens": "{\"description\": \"min=414.71, mean=414.71, max=414.71, sum=829.42 (2)\", \"tab\": \"General information\", \"score\": \"414.71\"}", - "Medical Genetics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"medical_genetics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_medical_genetics\"" - } - } - }, - { - "evaluation_name": "Miscellaneous", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Miscellaneous", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.785, - "details": { - "description": "min=0.785, mean=0.785, max=0.785, sum=1.571 (2)", - "tab": "Accuracy", - "Miscellaneous - Observed inference time (s)": "{\"description\": \"min=0.504, mean=0.504, max=0.504, sum=1.008 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5038632959850599\"}", - "Miscellaneous - # eval": "{\"description\": \"min=783, mean=783, max=783, sum=1566 (2)\", \"tab\": \"General information\", \"score\": \"783.0\"}", - "Miscellaneous - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Miscellaneous - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Miscellaneous - # prompt tokens": "{\"description\": \"min=357.519, mean=357.519, max=357.519, sum=715.037 (2)\", \"tab\": \"General information\", \"score\": \"357.51851851851853\"}", - "Miscellaneous - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"miscellaneous\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_miscellaneous\"" - } - } - }, - { - "evaluation_name": "Moral Scenarios", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Moral Scenarios", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.393, - "details": { - "description": "min=0.393, mean=0.393, max=0.393, sum=0.787 (2)", - "tab": "Accuracy", - "Moral Disputes - Observed inference time (s)": "{\"description\": \"min=0.777, mean=0.777, max=0.777, sum=1.553 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7765735477381359\"}", - "Moral Scenarios - Observed inference time (s)": "{\"description\": \"min=0.493, mean=0.493, max=0.493, sum=0.986 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4927780463042872\"}", - "Moral Disputes - # eval": "{\"description\": \"min=346, mean=346, max=346, sum=692 (2)\", \"tab\": \"General information\", \"score\": \"346.0\"}", - "Moral Disputes - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Disputes - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Disputes - # prompt tokens": "{\"description\": \"min=549.038, mean=549.038, max=549.038, sum=1098.075 (2)\", \"tab\": \"General information\", \"score\": \"549.0375722543353\"}", - "Moral Disputes - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Moral Scenarios - # eval": "{\"description\": \"min=895, mean=895, max=895, sum=1790 (2)\", \"tab\": \"General information\", \"score\": \"895.0\"}", - "Moral Scenarios - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Scenarios - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # prompt tokens": "{\"description\": \"min=754.516, mean=754.516, max=754.516, sum=1509.032 (2)\", \"tab\": \"General information\", \"score\": \"754.5162011173185\"}", - "Moral Scenarios - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"moral_scenarios\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_moral_scenarios\"" - } - } - }, - { - "evaluation_name": "Nutrition", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Nutrition", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.676, - "details": { - "description": "min=0.676, mean=0.676, max=0.676, sum=1.353 (2)", - "tab": "Accuracy", - "Nutrition - Observed inference time (s)": "{\"description\": \"min=0.236, mean=0.236, max=0.236, sum=0.471 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.23563866054310517\"}", - "Nutrition - # eval": "{\"description\": \"min=306, mean=306, max=306, sum=612 (2)\", \"tab\": \"General information\", \"score\": \"306.0\"}", - "Nutrition - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Nutrition - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Nutrition - # prompt tokens": "{\"description\": \"min=689.69, mean=689.69, max=689.69, sum=1379.379 (2)\", \"tab\": \"General information\", \"score\": \"689.6895424836601\"}", - "Nutrition - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"nutrition\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_nutrition\"" - } - } - }, - { - "evaluation_name": "Prehistory", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Prehistory", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.673, - "details": { - "description": "min=0.673, mean=0.673, max=0.673, sum=1.346 (2)", - "tab": "Accuracy", - "Prehistory - Observed inference time (s)": "{\"description\": \"min=0.345, mean=0.345, max=0.345, sum=0.69 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.34476134880089465\"}", - "Prehistory - # eval": "{\"description\": \"min=324, mean=324, max=324, sum=648 (2)\", \"tab\": \"General information\", \"score\": \"324.0\"}", - "Prehistory - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Prehistory - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Prehistory - # prompt tokens": "{\"description\": \"min=611.145, mean=611.145, max=611.145, sum=1222.29 (2)\", \"tab\": \"General information\", \"score\": \"611.145061728395\"}", - "Prehistory - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"prehistory\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_prehistory\"" - } - } - }, - { - "evaluation_name": "Public Relations", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Public Relations", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.636, - "details": { - "description": "min=0.636, mean=0.636, max=0.636, sum=1.273 (2)", - "tab": "Accuracy", - "Public Relations - Observed inference time (s)": "{\"description\": \"min=0.327, mean=0.327, max=0.327, sum=0.654 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3271717678416859\"}", - "Public Relations - # eval": "{\"description\": \"min=110, mean=110, max=110, sum=220 (2)\", \"tab\": \"General information\", \"score\": \"110.0\"}", - "Public Relations - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Public Relations - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Public Relations - # prompt tokens": "{\"description\": \"min=471.036, mean=471.036, max=471.036, sum=942.073 (2)\", \"tab\": \"General information\", \"score\": \"471.03636363636366\"}", - "Public Relations - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"public_relations\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_public_relations\"" - } - } - }, - { - "evaluation_name": "Security Studies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Security Studies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.682, - "details": { - "description": "min=0.682, mean=0.682, max=0.682, sum=1.363 (2)", - "tab": "Accuracy", - "Security Studies - Observed inference time (s)": "{\"description\": \"min=0.561, mean=0.561, max=0.561, sum=1.121 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5606838294437954\"}", - "Security Studies - # eval": "{\"description\": \"min=245, mean=245, max=245, sum=490 (2)\", \"tab\": \"General information\", \"score\": \"245.0\"}", - "Security Studies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Security Studies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Security Studies - # prompt tokens": "{\"description\": \"min=1324.865, mean=1324.865, max=1324.865, sum=2649.731 (2)\", \"tab\": \"General information\", \"score\": \"1324.865306122449\"}", - "Security Studies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"security_studies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_security_studies\"" - } - } - }, - { - "evaluation_name": "Sociology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Sociology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.806, - "details": { - "description": "min=0.806, mean=0.806, max=0.806, sum=1.612 (2)", - "tab": "Accuracy", - "Sociology - Observed inference time (s)": "{\"description\": \"min=0.413, mean=0.413, max=0.413, sum=0.825 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.41272182962787685\"}", - "Sociology - # eval": "{\"description\": \"min=201, mean=201, max=201, sum=402 (2)\", \"tab\": \"General information\", \"score\": \"201.0\"}", - "Sociology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Sociology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Sociology - # prompt tokens": "{\"description\": \"min=496.95, mean=496.95, max=496.95, sum=993.9 (2)\", \"tab\": \"General information\", \"score\": \"496.9502487562189\"}", - "Sociology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"sociology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_sociology\"" - } - } - }, - { - "evaluation_name": "Virology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Virology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.47, - "details": { - "description": "min=0.47, mean=0.47, max=0.47, sum=0.94 (2)", - "tab": "Accuracy", - "Virology - Observed inference time (s)": "{\"description\": \"min=0.644, mean=0.644, max=0.644, sum=1.288 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6437842285776713\"}", - "Virology - # eval": "{\"description\": \"min=166, mean=166, max=166, sum=332 (2)\", \"tab\": \"General information\", \"score\": \"166.0\"}", - "Virology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Virology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Virology - # prompt tokens": "{\"description\": \"min=404.349, mean=404.349, max=404.349, sum=808.699 (2)\", \"tab\": \"General information\", \"score\": \"404.34939759036143\"}", - "Virology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"virology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_virology\"" - } - } - }, - { - "evaluation_name": "World Religions", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on World Religions", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.825, - "details": { - "description": "min=0.825, mean=0.825, max=0.825, sum=1.649 (2)", - "tab": "Accuracy", - "World Religions - Observed inference time (s)": "{\"description\": \"min=0.266, mean=0.266, max=0.266, sum=0.532 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.26615772330970094\"}", - "World Religions - # eval": "{\"description\": \"min=171, mean=171, max=171, sum=342 (2)\", \"tab\": \"General information\", \"score\": \"171.0\"}", - "World Religions - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "World Religions - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "World Religions - # prompt tokens": "{\"description\": \"min=317.924, mean=317.924, max=317.924, sum=635.848 (2)\", \"tab\": \"General information\", \"score\": \"317.92397660818716\"}", - "World Religions - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"world_religions\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_world_religions\"" - } - } - }, - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.509, - "details": { - "description": "", - "tab": "Efficiency" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_id": "hfopenllm_v2/mistralai_Mistral-7B-Instruct-v0.3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5465 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4722 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0385 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2794 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3739 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3075 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/mistralai_mistral-7b-v0.1.json b/data/models/mistralai_mistral-7b-v0.1.json deleted file mode 100644 index f5108ab4cc3c5303dfcf3c89eca3c1628d2ce938..0000000000000000000000000000000000000000 --- a/data/models/mistralai_mistral-7b-v0.1.json +++ /dev/null @@ -1,2036 +0,0 @@ -{ - "model_info": { - "name": "Mistral v0.1 7B", - "id": "mistralai/mistral-7b-v0.1", - "developer": "mistralai", - "inference_platform": "unknown", - "additional_details": { - "model_id_aliases": [ - "mistralai/Mistral-7B-v0.1" - ] - } - }, - "evaluations": [ - { - "evaluation_id": "helm_lite/mistralai_mistral-7b-v0.1/1774096306.427425", - "retrieved_timestamp": "1774096306.427425", - "source_metadata": { - "source_name": "helm_lite", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_lite", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_lite", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.292, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.8075780274656679\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.716, - "details": { - "description": "min=0.716, mean=0.716, max=0.716, sum=0.716 (1)", - "tab": "Accuracy", - "NarrativeQA - Observed inference time (s)": "{\"description\": \"min=0.705, mean=0.705, max=0.705, sum=0.705 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.7051956902087574\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=4.575, mean=4.575, max=4.575, sum=4.575 (1)\", \"tab\": \"General information\", \"score\": \"4.574647887323944\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=3627.715, mean=3627.715, max=3627.715, sum=3627.715 (1)\", \"tab\": \"General information\", \"score\": \"3627.7154929577464\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (closed-book)", - "source_data": { - "dataset_name": "NaturalQuestions (closed-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (closed-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.367, - "details": { - "description": "min=0.367, mean=0.367, max=0.367, sum=0.367 (1)", - "tab": "Accuracy", - "NaturalQuestions (open-book) - Observed inference time (s)": "{\"description\": \"min=0.494, mean=0.494, max=0.494, sum=0.494 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.49417281556129455\"}", - "NaturalQuestions (closed-book) - Observed inference time (s)": "{\"description\": \"min=0.462, mean=0.462, max=0.462, sum=0.462 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.46181689071655274\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.832, mean=4.832, max=4.832, sum=4.832 (1)\", \"tab\": \"General information\", \"score\": \"4.832\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.026, mean=0.026, max=0.026, sum=0.026 (1)\", \"tab\": \"General information\", \"score\": \"0.026\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=2268.728, mean=2268.728, max=2268.728, sum=2268.728 (1)\", \"tab\": \"General information\", \"score\": \"2268.728\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=0.988, mean=0.988, max=0.988, sum=0.988 (1)\", \"tab\": \"General information\", \"score\": \"0.988\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=142.069, mean=142.069, max=142.069, sum=142.069 (1)\", \"tab\": \"General information\", \"score\": \"142.069\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "mode": "\"closedbook\"" - } - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.776, - "details": { - "description": "min=0.776, mean=0.776, max=0.776, sum=0.776 (1)", - "tab": "Accuracy", - "OpenbookQA - Observed inference time (s)": "{\"description\": \"min=0.325, mean=0.325, max=0.325, sum=0.325 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.32474704647064206\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=280.15, mean=280.15, max=280.15, sum=280.15 (1)\", \"tab\": \"General information\", \"score\": \"280.15\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "dataset": "\"openbookqa\"", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.584, - "details": { - "description": "min=0.31, mean=0.584, max=0.85, sum=2.918 (5)", - "tab": "Accuracy", - "MMLU - Observed inference time (s)": "{\"description\": \"min=0.272, mean=0.291, max=0.304, sum=1.457 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.2914179778851961\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=402.44, mean=523.091, max=687.175, sum=2615.455 (5)\", \"tab\": \"General information\", \"score\": \"523.0910877192983\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"college_chemistry\", \"computer_security\", \"econometrics\", \"us_foreign_policy\"]", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MATH", - "source_data": { - "dataset_name": "MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Equivalent (CoT) on MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.297, - "details": { - "description": "min=0.067, mean=0.297, max=0.43, sum=2.082 (7)", - "tab": "Accuracy", - "MATH - Observed inference time (s)": "{\"description\": \"min=0.992, mean=1.159, max=1.576, sum=8.114 (7)\", \"tab\": \"Efficiency\", \"score\": \"1.159214100149656\"}", - "MATH - # eval": "{\"description\": \"min=30, mean=62.429, max=135, sum=437 (7)\", \"tab\": \"General information\", \"score\": \"62.42857142857143\"}", - "MATH - # train": "{\"description\": \"min=8, mean=8, max=8, sum=56 (7)\", \"tab\": \"General information\", \"score\": \"8.0\"}", - "MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MATH - # prompt tokens": "{\"description\": \"min=991.615, mean=1455.266, max=2502.962, sum=10186.865 (7)\", \"tab\": \"General information\", \"score\": \"1455.2664139976257\"}", - "MATH - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=7 (7)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"algebra\", \"counting_and_probability\", \"geometry\", \"intermediate_algebra\", \"number_theory\", \"prealgebra\", \"precalculus\"]", - "level": "\"1\"", - "use_official_examples": "\"False\"", - "use_chain_of_thought": "\"True\"" - } - } - }, - { - "evaluation_name": "GSM8K", - "source_data": { - "dataset_name": "GSM8K", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on GSM8K", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.377, - "details": { - "description": "min=0.377, mean=0.377, max=0.377, sum=0.377 (1)", - "tab": "Accuracy", - "GSM8K - Observed inference time (s)": "{\"description\": \"min=1.632, mean=1.632, max=1.632, sum=1.632 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.6323128745555877\"}", - "GSM8K - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "GSM8K - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "GSM8K - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GSM8K - # prompt tokens": "{\"description\": \"min=1187.268, mean=1187.268, max=1187.268, sum=1187.268 (1)\", \"tab\": \"General information\", \"score\": \"1187.268\"}", - "GSM8K - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "LegalBench", - "source_data": { - "dataset_name": "LegalBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on LegalBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.58, - "details": { - "description": "min=0.433, mean=0.58, max=0.789, sum=2.901 (5)", - "tab": "Accuracy", - "LegalBench - Observed inference time (s)": "{\"description\": \"min=0.287, mean=0.353, max=0.577, sum=1.765 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.35307050709631943\"}", - "LegalBench - # eval": "{\"description\": \"min=95, mean=409.4, max=1000, sum=2047 (5)\", \"tab\": \"General information\", \"score\": \"409.4\"}", - "LegalBench - # train": "{\"description\": \"min=1.969, mean=4.194, max=5, sum=20.969 (5)\", \"tab\": \"General information\", \"score\": \"4.1938775510204085\"}", - "LegalBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "LegalBench - # prompt tokens": "{\"description\": \"min=219.453, mean=998.503, max=3534.259, sum=4992.513 (5)\", \"tab\": \"General information\", \"score\": \"998.5025315575822\"}", - "LegalBench - # output tokens": "{\"description\": \"min=0.992, mean=0.998, max=1, sum=4.992 (5)\", \"tab\": \"General information\", \"score\": \"0.9983673469387755\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"abercrombie\", \"corporate_lobbying\", \"function_of_decision_section\", \"international_citizenship_questions\", \"proa\"]" - } - } - }, - { - "evaluation_name": "MedQA", - "source_data": { - "dataset_name": "MedQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MedQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.525, - "details": { - "description": "min=0.525, mean=0.525, max=0.525, sum=0.525 (1)", - "tab": "Accuracy", - "MedQA - Observed inference time (s)": "{\"description\": \"min=0.348, mean=0.348, max=0.348, sum=0.348 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.3478535307093596\"}", - "MedQA - # eval": "{\"description\": \"min=503, mean=503, max=503, sum=503 (1)\", \"tab\": \"General information\", \"score\": \"503.0\"}", - "MedQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MedQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MedQA - # prompt tokens": "{\"description\": \"min=1193.093, mean=1193.093, max=1193.093, sum=1193.093 (1)\", \"tab\": \"General information\", \"score\": \"1193.0934393638172\"}", - "MedQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WMT 2014", - "source_data": { - "dataset_name": "WMT 2014", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "BLEU-4 on WMT 2014", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.16, - "details": { - "description": "min=0.056, mean=0.16, max=0.201, sum=0.802 (5)", - "tab": "Accuracy", - "WMT 2014 - Observed inference time (s)": "{\"description\": \"min=0.52, mean=0.561, max=0.701, sum=2.803 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.5605853292576617\"}", - "WMT 2014 - # eval": "{\"description\": \"min=503, mean=568.8, max=832, sum=2844 (5)\", \"tab\": \"General information\", \"score\": \"568.8\"}", - "WMT 2014 - # train": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "WMT 2014 - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WMT 2014 - # prompt tokens": "{\"description\": \"min=130.306, mean=144.433, max=163.018, sum=722.166 (5)\", \"tab\": \"General information\", \"score\": \"144.43317355482492\"}", - "WMT 2014 - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "language_pair": "[\"cs-en\", \"de-en\", \"fr-en\", \"hi-en\", \"ru-en\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_mmlu/mistralai_mistral-7b-v0.1/1774096312.00548", - "retrieved_timestamp": "1774096312.00548", - "source_metadata": { - "source_name": "helm_mmlu", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_mmlu", - "evaluation_results": [ - { - "evaluation_name": "MMLU All Subjects", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU All Subjects", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.566, - "details": { - "description": "min=0.25, mean=0.566, max=0.845, sum=64.496 (114)", - "tab": "Accuracy", - "MMLU All Subjects - Observed inference time (s)": "{\"description\": \"min=0.665, mean=0.864, max=1.234, sum=98.504 (114)\", \"tab\": \"Efficiency\", \"score\": \"0.8640714937745795\"}", - "MMLU All Subjects - # eval": "{\"description\": \"min=100, mean=246.351, max=1534, sum=28084 (114)\", \"tab\": \"General information\", \"score\": \"246.35087719298247\"}", - "MMLU All Subjects - # train": "{\"description\": \"min=5, mean=5, max=5, sum=570 (114)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU All Subjects - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU All Subjects - # prompt tokens": "{\"description\": \"min=308.924, mean=696.273, max=3089.109, sum=79375.178 (114)\", \"tab\": \"General information\", \"score\": \"696.2734899593811\"}", - "MMLU All Subjects - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=114 (114)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_name": "Abstract Algebra", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Abstract Algebra", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.25, - "details": { - "description": "min=0.25, mean=0.25, max=0.25, sum=0.5 (2)", - "tab": "Accuracy", - "Abstract Algebra - Observed inference time (s)": "{\"description\": \"min=0.834, mean=0.834, max=0.834, sum=1.667 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8337139582633972\"}", - "Abstract Algebra - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Abstract Algebra - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Abstract Algebra - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Abstract Algebra - # prompt tokens": "{\"description\": \"min=402.44, mean=402.44, max=402.44, sum=804.88 (2)\", \"tab\": \"General information\", \"score\": \"402.44\"}", - "Abstract Algebra - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"abstract_algebra\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_abstract_algebra\"" - } - } - }, - { - "evaluation_name": "Anatomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Anatomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.467, - "details": { - "description": "min=0.467, mean=0.467, max=0.467, sum=0.933 (2)", - "tab": "Accuracy", - "Anatomy - Observed inference time (s)": "{\"description\": \"min=0.717, mean=0.717, max=0.717, sum=1.435 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7173902529257316\"}", - "Anatomy - # eval": "{\"description\": \"min=135, mean=135, max=135, sum=270 (2)\", \"tab\": \"General information\", \"score\": \"135.0\"}", - "Anatomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Anatomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Anatomy - # prompt tokens": "{\"description\": \"min=407.089, mean=407.089, max=407.089, sum=814.178 (2)\", \"tab\": \"General information\", \"score\": \"407.0888888888889\"}", - "Anatomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"anatomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_anatomy\"" - } - } - }, - { - "evaluation_name": "College Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on College Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.314, - "details": { - "description": "min=0.314, mean=0.314, max=0.314, sum=0.627 (2)", - "tab": "Accuracy", - "College Chemistry - Observed inference time (s)": "{\"description\": \"min=1.018, mean=1.018, max=1.018, sum=2.036 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.0181659984588622\"}", - "College Biology - Observed inference time (s)": "{\"description\": \"min=0.699, mean=0.699, max=0.699, sum=1.398 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.699198540714052\"}", - "College Computer Science - Observed inference time (s)": "{\"description\": \"min=0.712, mean=0.712, max=0.712, sum=1.423 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7115359020233154\"}", - "College Mathematics - Observed inference time (s)": "{\"description\": \"min=0.885, mean=0.885, max=0.885, sum=1.77 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8852152943611145\"}", - "College Medicine - Observed inference time (s)": "{\"description\": \"min=0.896, mean=0.896, max=0.896, sum=1.793 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8963309629804137\"}", - "College Physics - Observed inference time (s)": "{\"description\": \"min=0.803, mean=0.803, max=0.803, sum=1.606 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8030702249676573\"}", - "College Chemistry - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Chemistry - # prompt tokens": "{\"description\": \"min=627.71, mean=627.71, max=627.71, sum=1255.42 (2)\", \"tab\": \"General information\", \"score\": \"627.71\"}", - "College Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Biology - # eval": "{\"description\": \"min=144, mean=144, max=144, sum=288 (2)\", \"tab\": \"General information\", \"score\": \"144.0\"}", - "College Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # prompt tokens": "{\"description\": \"min=550.799, mean=550.799, max=550.799, sum=1101.597 (2)\", \"tab\": \"General information\", \"score\": \"550.7986111111111\"}", - "College Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # prompt tokens": "{\"description\": \"min=902.17, mean=902.17, max=902.17, sum=1804.34 (2)\", \"tab\": \"General information\", \"score\": \"902.17\"}", - "College Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Mathematics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # prompt tokens": "{\"description\": \"min=658.31, mean=658.31, max=658.31, sum=1316.62 (2)\", \"tab\": \"General information\", \"score\": \"658.31\"}", - "College Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Medicine - # eval": "{\"description\": \"min=173, mean=173, max=173, sum=346 (2)\", \"tab\": \"General information\", \"score\": \"173.0\"}", - "College Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # prompt tokens": "{\"description\": \"min=592.41, mean=592.41, max=592.41, sum=1184.821 (2)\", \"tab\": \"General information\", \"score\": \"592.4104046242775\"}", - "College Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Physics - # eval": "{\"description\": \"min=102, mean=102, max=102, sum=204 (2)\", \"tab\": \"General information\", \"score\": \"102.0\"}", - "College Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # prompt tokens": "{\"description\": \"min=551.029, mean=551.029, max=551.029, sum=1102.059 (2)\", \"tab\": \"General information\", \"score\": \"551.0294117647059\"}", - "College Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"college_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_college_physics\"" - } - } - }, - { - "evaluation_name": "Computer Security", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Computer Security", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.69, - "details": { - "description": "min=0.69, mean=0.69, max=0.69, sum=1.38 (2)", - "tab": "Accuracy", - "Computer Security - Observed inference time (s)": "{\"description\": \"min=1.033, mean=1.033, max=1.033, sum=2.065 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.032561357021332\"}", - "Computer Security - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Computer Security - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Computer Security - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Computer Security - # prompt tokens": "{\"description\": \"min=424.94, mean=424.94, max=424.94, sum=849.88 (2)\", \"tab\": \"General information\", \"score\": \"424.94\"}", - "Computer Security - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"computer_security\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_computer_security\"" - } - } - }, - { - "evaluation_name": "Econometrics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Econometrics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.351, - "details": { - "description": "min=0.351, mean=0.351, max=0.351, sum=0.702 (2)", - "tab": "Accuracy", - "Econometrics - Observed inference time (s)": "{\"description\": \"min=0.783, mean=0.783, max=0.783, sum=1.566 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7832156043303641\"}", - "Econometrics - # eval": "{\"description\": \"min=114, mean=114, max=114, sum=228 (2)\", \"tab\": \"General information\", \"score\": \"114.0\"}", - "Econometrics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Econometrics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Econometrics - # prompt tokens": "{\"description\": \"min=687.175, mean=687.175, max=687.175, sum=1374.351 (2)\", \"tab\": \"General information\", \"score\": \"687.1754385964912\"}", - "Econometrics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"econometrics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_econometrics\"" - } - } - }, - { - "evaluation_name": "Global Facts", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Global Facts", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.29, - "details": { - "description": "min=0.29, mean=0.29, max=0.29, sum=0.58 (2)", - "tab": "Accuracy", - "Global Facts - Observed inference time (s)": "{\"description\": \"min=0.689, mean=0.689, max=0.689, sum=1.378 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6891914677619934\"}", - "Global Facts - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Global Facts - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Global Facts - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Global Facts - # prompt tokens": "{\"description\": \"min=483.47, mean=483.47, max=483.47, sum=966.94 (2)\", \"tab\": \"General information\", \"score\": \"483.47\"}", - "Global Facts - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"global_facts\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_global_facts\"" - } - } - }, - { - "evaluation_name": "Jurisprudence", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Jurisprudence", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.667, - "details": { - "description": "min=0.667, mean=0.667, max=0.667, sum=1.333 (2)", - "tab": "Accuracy", - "Jurisprudence - Observed inference time (s)": "{\"description\": \"min=0.787, mean=0.787, max=0.787, sum=1.574 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7868193630818967\"}", - "Jurisprudence - # eval": "{\"description\": \"min=108, mean=108, max=108, sum=216 (2)\", \"tab\": \"General information\", \"score\": \"108.0\"}", - "Jurisprudence - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Jurisprudence - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Jurisprudence - # prompt tokens": "{\"description\": \"min=451.093, mean=451.093, max=451.093, sum=902.185 (2)\", \"tab\": \"General information\", \"score\": \"451.0925925925926\"}", - "Jurisprudence - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"jurisprudence\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_jurisprudence\"" - } - } - }, - { - "evaluation_name": "Philosophy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Philosophy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.63, - "details": { - "description": "min=0.63, mean=0.63, max=0.63, sum=1.26 (2)", - "tab": "Accuracy", - "Philosophy - Observed inference time (s)": "{\"description\": \"min=0.743, mean=0.743, max=0.743, sum=1.487 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7434952857026716\"}", - "Philosophy - # eval": "{\"description\": \"min=311, mean=311, max=311, sum=622 (2)\", \"tab\": \"General information\", \"score\": \"311.0\"}", - "Philosophy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Philosophy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Philosophy - # prompt tokens": "{\"description\": \"min=373.82, mean=373.82, max=373.82, sum=747.64 (2)\", \"tab\": \"General information\", \"score\": \"373.81993569131834\"}", - "Philosophy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"philosophy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_philosophy\"" - } - } - }, - { - "evaluation_name": "Professional Psychology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Professional Psychology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.578, - "details": { - "description": "min=0.578, mean=0.578, max=0.578, sum=1.157 (2)", - "tab": "Accuracy", - "Professional Medicine - Observed inference time (s)": "{\"description\": \"min=0.817, mean=0.817, max=0.817, sum=1.633 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.816552089417682\"}", - "Professional Accounting - Observed inference time (s)": "{\"description\": \"min=0.743, mean=0.743, max=0.743, sum=1.487 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7432903905286856\"}", - "Professional Law - Observed inference time (s)": "{\"description\": \"min=0.82, mean=0.82, max=0.82, sum=1.64 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8197952300659836\"}", - "Professional Psychology - Observed inference time (s)": "{\"description\": \"min=0.936, mean=0.936, max=0.936, sum=1.873 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9364227648654015\"}", - "Professional Medicine - # eval": "{\"description\": \"min=272, mean=272, max=272, sum=544 (2)\", \"tab\": \"General information\", \"score\": \"272.0\"}", - "Professional Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Medicine - # prompt tokens": "{\"description\": \"min=1279.143, mean=1279.143, max=1279.143, sum=2558.287 (2)\", \"tab\": \"General information\", \"score\": \"1279.1433823529412\"}", - "Professional Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Accounting - # eval": "{\"description\": \"min=282, mean=282, max=282, sum=564 (2)\", \"tab\": \"General information\", \"score\": \"282.0\"}", - "Professional Accounting - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Accounting - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # prompt tokens": "{\"description\": \"min=796.496, mean=796.496, max=796.496, sum=1592.993 (2)\", \"tab\": \"General information\", \"score\": \"796.4964539007092\"}", - "Professional Accounting - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Law - # eval": "{\"description\": \"min=1534, mean=1534, max=1534, sum=3068 (2)\", \"tab\": \"General information\", \"score\": \"1534.0\"}", - "Professional Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # prompt tokens": "{\"description\": \"min=1849.711, mean=1849.711, max=1849.711, sum=3699.421 (2)\", \"tab\": \"General information\", \"score\": \"1849.7105606258149\"}", - "Professional Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Psychology - # eval": "{\"description\": \"min=612, mean=612, max=612, sum=1224 (2)\", \"tab\": \"General information\", \"score\": \"612.0\"}", - "Professional Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # prompt tokens": "{\"description\": \"min=645.278, mean=645.278, max=645.278, sum=1290.556 (2)\", \"tab\": \"General information\", \"score\": \"645.2777777777778\"}", - "Professional Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"professional_psychology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_professional_psychology\"" - } - } - }, - { - "evaluation_name": "Us Foreign Policy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Us Foreign Policy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.79, - "details": { - "description": "min=0.79, mean=0.79, max=0.79, sum=1.58 (2)", - "tab": "Accuracy", - "Us Foreign Policy - Observed inference time (s)": "{\"description\": \"min=0.863, mean=0.863, max=0.863, sum=1.727 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8633295917510986\"}", - "Us Foreign Policy - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Us Foreign Policy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Us Foreign Policy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Us Foreign Policy - # prompt tokens": "{\"description\": \"min=473.19, mean=473.19, max=473.19, sum=946.38 (2)\", \"tab\": \"General information\", \"score\": \"473.19\"}", - "Us Foreign Policy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"us_foreign_policy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_us_foreign_policy\"" - } - } - }, - { - "evaluation_name": "Astronomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Astronomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.599, - "details": { - "description": "min=0.599, mean=0.599, max=0.599, sum=1.197 (2)", - "tab": "Accuracy", - "Astronomy - Observed inference time (s)": "{\"description\": \"min=0.804, mean=0.804, max=0.804, sum=1.608 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8039205105681169\"}", - "Astronomy - # eval": "{\"description\": \"min=152, mean=152, max=152, sum=304 (2)\", \"tab\": \"General information\", \"score\": \"152.0\"}", - "Astronomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Astronomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Astronomy - # prompt tokens": "{\"description\": \"min=665.987, mean=665.987, max=665.987, sum=1331.974 (2)\", \"tab\": \"General information\", \"score\": \"665.9868421052631\"}", - "Astronomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"astronomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_astronomy\"" - } - } - }, - { - "evaluation_name": "Business Ethics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Business Ethics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.56, - "details": { - "description": "min=0.56, mean=0.56, max=0.56, sum=1.12 (2)", - "tab": "Accuracy", - "Business Ethics - Observed inference time (s)": "{\"description\": \"min=1.014, mean=1.014, max=1.014, sum=2.028 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.013892731666565\"}", - "Business Ethics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Business Ethics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Business Ethics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Business Ethics - # prompt tokens": "{\"description\": \"min=644.6, mean=644.6, max=644.6, sum=1289.2 (2)\", \"tab\": \"General information\", \"score\": \"644.6\"}", - "Business Ethics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"business_ethics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_business_ethics\"" - } - } - }, - { - "evaluation_name": "Clinical Knowledge", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Clinical Knowledge", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.653, - "details": { - "description": "min=0.653, mean=0.653, max=0.653, sum=1.306 (2)", - "tab": "Accuracy", - "Clinical Knowledge - Observed inference time (s)": "{\"description\": \"min=0.807, mean=0.807, max=0.807, sum=1.613 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8066773774488917\"}", - "Clinical Knowledge - # eval": "{\"description\": \"min=265, mean=265, max=265, sum=530 (2)\", \"tab\": \"General information\", \"score\": \"265.0\"}", - "Clinical Knowledge - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Clinical Knowledge - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Clinical Knowledge - # prompt tokens": "{\"description\": \"min=487.174, mean=487.174, max=487.174, sum=974.347 (2)\", \"tab\": \"General information\", \"score\": \"487.1735849056604\"}", - "Clinical Knowledge - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"clinical_knowledge\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_clinical_knowledge\"" - } - } - }, - { - "evaluation_name": "Conceptual Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Conceptual Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.451, - "details": { - "description": "min=0.451, mean=0.451, max=0.451, sum=0.902 (2)", - "tab": "Accuracy", - "Conceptual Physics - Observed inference time (s)": "{\"description\": \"min=0.833, mean=0.833, max=0.833, sum=1.666 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.833152520402949\"}", - "Conceptual Physics - # eval": "{\"description\": \"min=235, mean=235, max=235, sum=470 (2)\", \"tab\": \"General information\", \"score\": \"235.0\"}", - "Conceptual Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Conceptual Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Conceptual Physics - # prompt tokens": "{\"description\": \"min=334.285, mean=334.285, max=334.285, sum=668.57 (2)\", \"tab\": \"General information\", \"score\": \"334.2851063829787\"}", - "Conceptual Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"conceptual_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_conceptual_physics\"" - } - } - }, - { - "evaluation_name": "Electrical Engineering", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Electrical Engineering", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.538, - "details": { - "description": "min=0.538, mean=0.538, max=0.538, sum=1.076 (2)", - "tab": "Accuracy", - "Electrical Engineering - Observed inference time (s)": "{\"description\": \"min=1.234, mean=1.234, max=1.234, sum=2.468 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.2342401932025777\"}", - "Electrical Engineering - # eval": "{\"description\": \"min=145, mean=145, max=145, sum=290 (2)\", \"tab\": \"General information\", \"score\": \"145.0\"}", - "Electrical Engineering - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Electrical Engineering - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Electrical Engineering - # prompt tokens": "{\"description\": \"min=501.379, mean=501.379, max=501.379, sum=1002.759 (2)\", \"tab\": \"General information\", \"score\": \"501.37931034482756\"}", - "Electrical Engineering - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"electrical_engineering\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_electrical_engineering\"" - } - } - }, - { - "evaluation_name": "Elementary Mathematics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Elementary Mathematics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.32, - "details": { - "description": "min=0.32, mean=0.32, max=0.32, sum=0.64 (2)", - "tab": "Accuracy", - "Elementary Mathematics - Observed inference time (s)": "{\"description\": \"min=0.854, mean=0.854, max=0.854, sum=1.707 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8535163610700577\"}", - "Elementary Mathematics - # eval": "{\"description\": \"min=378, mean=378, max=378, sum=756 (2)\", \"tab\": \"General information\", \"score\": \"378.0\"}", - "Elementary Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Elementary Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Elementary Mathematics - # prompt tokens": "{\"description\": \"min=613.386, mean=613.386, max=613.386, sum=1226.772 (2)\", \"tab\": \"General information\", \"score\": \"613.3862433862433\"}", - "Elementary Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"elementary_mathematics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_elementary_mathematics\"" - } - } - }, - { - "evaluation_name": "Formal Logic", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Formal Logic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.365, - "details": { - "description": "min=0.365, mean=0.365, max=0.365, sum=0.73 (2)", - "tab": "Accuracy", - "Formal Logic - Observed inference time (s)": "{\"description\": \"min=1.022, mean=1.022, max=1.022, sum=2.044 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.0218302371009949\"}", - "Formal Logic - # eval": "{\"description\": \"min=126, mean=126, max=126, sum=252 (2)\", \"tab\": \"General information\", \"score\": \"126.0\"}", - "Formal Logic - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Formal Logic - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Formal Logic - # prompt tokens": "{\"description\": \"min=718.984, mean=718.984, max=718.984, sum=1437.968 (2)\", \"tab\": \"General information\", \"score\": \"718.984126984127\"}", - "Formal Logic - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"formal_logic\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_formal_logic\"" - } - } - }, - { - "evaluation_name": "High School World History", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on High School World History", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.726, - "details": { - "description": "min=0.726, mean=0.726, max=0.726, sum=1.451 (2)", - "tab": "Accuracy", - "High School Biology - Observed inference time (s)": "{\"description\": \"min=0.904, mean=0.904, max=0.904, sum=1.808 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9039220233117381\"}", - "High School Chemistry - Observed inference time (s)": "{\"description\": \"min=0.891, mean=0.891, max=0.891, sum=1.782 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8910855988563575\"}", - "High School Computer Science - Observed inference time (s)": "{\"description\": \"min=0.759, mean=0.759, max=0.759, sum=1.519 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7594162678718567\"}", - "High School European History - Observed inference time (s)": "{\"description\": \"min=0.709, mean=0.709, max=0.709, sum=1.418 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7088880394444321\"}", - "High School Geography - Observed inference time (s)": "{\"description\": \"min=0.909, mean=0.909, max=0.909, sum=1.818 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9091630006077314\"}", - "High School Government And Politics - Observed inference time (s)": "{\"description\": \"min=0.665, mean=0.665, max=0.665, sum=1.329 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6645773976577996\"}", - "High School Macroeconomics - Observed inference time (s)": "{\"description\": \"min=0.841, mean=0.841, max=0.841, sum=1.682 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8412165372799605\"}", - "High School Mathematics - Observed inference time (s)": "{\"description\": \"min=0.767, mean=0.767, max=0.767, sum=1.534 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7671932847411544\"}", - "High School Microeconomics - Observed inference time (s)": "{\"description\": \"min=0.995, mean=0.995, max=0.995, sum=1.99 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.994775929370848\"}", - "High School Physics - Observed inference time (s)": "{\"description\": \"min=0.902, mean=0.902, max=0.902, sum=1.805 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9024771317740939\"}", - "High School Psychology - Observed inference time (s)": "{\"description\": \"min=0.793, mean=0.793, max=0.793, sum=1.585 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7925117606416755\"}", - "High School Statistics - Observed inference time (s)": "{\"description\": \"min=0.884, mean=0.884, max=0.884, sum=1.768 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8837873924661566\"}", - "High School US History - Observed inference time (s)": "{\"description\": \"min=1.171, mean=1.171, max=1.171, sum=2.341 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.170638754087336\"}", - "High School World History - Observed inference time (s)": "{\"description\": \"min=0.801, mean=0.801, max=0.801, sum=1.603 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8013244822055479\"}", - "High School Biology - # eval": "{\"description\": \"min=310, mean=310, max=310, sum=620 (2)\", \"tab\": \"General information\", \"score\": \"310.0\"}", - "High School Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Biology - # prompt tokens": "{\"description\": \"min=600.561, mean=600.561, max=600.561, sum=1201.123 (2)\", \"tab\": \"General information\", \"score\": \"600.5612903225806\"}", - "High School Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Chemistry - # eval": "{\"description\": \"min=203, mean=203, max=203, sum=406 (2)\", \"tab\": \"General information\", \"score\": \"203.0\"}", - "High School Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # prompt tokens": "{\"description\": \"min=572.798, mean=572.798, max=572.798, sum=1145.596 (2)\", \"tab\": \"General information\", \"score\": \"572.7980295566502\"}", - "High School Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "High School Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # prompt tokens": "{\"description\": \"min=988.24, mean=988.24, max=988.24, sum=1976.48 (2)\", \"tab\": \"General information\", \"score\": \"988.24\"}", - "High School Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School European History - # eval": "{\"description\": \"min=165, mean=165, max=165, sum=330 (2)\", \"tab\": \"General information\", \"score\": \"165.0\"}", - "High School European History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School European History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # prompt tokens": "{\"description\": \"min=3089.109, mean=3089.109, max=3089.109, sum=6178.218 (2)\", \"tab\": \"General information\", \"score\": \"3089.109090909091\"}", - "High School European History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Geography - # eval": "{\"description\": \"min=198, mean=198, max=198, sum=396 (2)\", \"tab\": \"General information\", \"score\": \"198.0\"}", - "High School Geography - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Geography - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # prompt tokens": "{\"description\": \"min=429.207, mean=429.207, max=429.207, sum=858.414 (2)\", \"tab\": \"General information\", \"score\": \"429.2070707070707\"}", - "High School Geography - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Government And Politics - # eval": "{\"description\": \"min=193, mean=193, max=193, sum=386 (2)\", \"tab\": \"General information\", \"score\": \"193.0\"}", - "High School Government And Politics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Government And Politics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # prompt tokens": "{\"description\": \"min=514.808, mean=514.808, max=514.808, sum=1029.617 (2)\", \"tab\": \"General information\", \"score\": \"514.8082901554404\"}", - "High School Government And Politics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Macroeconomics - # eval": "{\"description\": \"min=390, mean=390, max=390, sum=780 (2)\", \"tab\": \"General information\", \"score\": \"390.0\"}", - "High School Macroeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Macroeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # prompt tokens": "{\"description\": \"min=423.815, mean=423.815, max=423.815, sum=847.631 (2)\", \"tab\": \"General information\", \"score\": \"423.81538461538463\"}", - "High School Macroeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Mathematics - # eval": "{\"description\": \"min=270, mean=270, max=270, sum=540 (2)\", \"tab\": \"General information\", \"score\": \"270.0\"}", - "High School Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # prompt tokens": "{\"description\": \"min=584.13, mean=584.13, max=584.13, sum=1168.259 (2)\", \"tab\": \"General information\", \"score\": \"584.1296296296297\"}", - "High School Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Microeconomics - # eval": "{\"description\": \"min=238, mean=238, max=238, sum=476 (2)\", \"tab\": \"General information\", \"score\": \"238.0\"}", - "High School Microeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Microeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # prompt tokens": "{\"description\": \"min=443.345, mean=443.345, max=443.345, sum=886.689 (2)\", \"tab\": \"General information\", \"score\": \"443.34453781512605\"}", - "High School Microeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Physics - # eval": "{\"description\": \"min=151, mean=151, max=151, sum=302 (2)\", \"tab\": \"General information\", \"score\": \"151.0\"}", - "High School Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # prompt tokens": "{\"description\": \"min=622.775, mean=622.775, max=622.775, sum=1245.55 (2)\", \"tab\": \"General information\", \"score\": \"622.774834437086\"}", - "High School Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Psychology - # eval": "{\"description\": \"min=545, mean=545, max=545, sum=1090 (2)\", \"tab\": \"General information\", \"score\": \"545.0\"}", - "High School Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # prompt tokens": "{\"description\": \"min=558.873, mean=558.873, max=558.873, sum=1117.747 (2)\", \"tab\": \"General information\", \"score\": \"558.8733944954129\"}", - "High School Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Statistics - # eval": "{\"description\": \"min=216, mean=216, max=216, sum=432 (2)\", \"tab\": \"General information\", \"score\": \"216.0\"}", - "High School Statistics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Statistics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # prompt tokens": "{\"description\": \"min=913.644, mean=913.644, max=913.644, sum=1827.287 (2)\", \"tab\": \"General information\", \"score\": \"913.6435185185185\"}", - "High School Statistics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School US History - # eval": "{\"description\": \"min=204, mean=204, max=204, sum=408 (2)\", \"tab\": \"General information\", \"score\": \"204.0\"}", - "High School US History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School US History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # prompt tokens": "{\"description\": \"min=2477.446, mean=2477.446, max=2477.446, sum=4954.892 (2)\", \"tab\": \"General information\", \"score\": \"2477.4460784313724\"}", - "High School US History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School World History - # eval": "{\"description\": \"min=237, mean=237, max=237, sum=474 (2)\", \"tab\": \"General information\", \"score\": \"237.0\"}", - "High School World History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School World History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # prompt tokens": "{\"description\": \"min=1585.553, mean=1585.553, max=1585.553, sum=3171.105 (2)\", \"tab\": \"General information\", \"score\": \"1585.5527426160338\"}", - "High School World History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"high_school_world_history\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_high_school_world_history\"" - } - } - }, - { - "evaluation_name": "Human Sexuality", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Human Sexuality", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.702, - "details": { - "description": "min=0.702, mean=0.702, max=0.702, sum=1.405 (2)", - "tab": "Accuracy", - "Human Aging - Observed inference time (s)": "{\"description\": \"min=0.779, mean=0.779, max=0.779, sum=1.558 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.778804096940387\"}", - "Human Sexuality - Observed inference time (s)": "{\"description\": \"min=0.85, mean=0.85, max=0.85, sum=1.701 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8504140213245653\"}", - "Human Aging - # eval": "{\"description\": \"min=223, mean=223, max=223, sum=446 (2)\", \"tab\": \"General information\", \"score\": \"223.0\"}", - "Human Aging - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Aging - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Aging - # prompt tokens": "{\"description\": \"min=353.152, mean=353.152, max=353.152, sum=706.305 (2)\", \"tab\": \"General information\", \"score\": \"353.15246636771303\"}", - "Human Aging - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Human Sexuality - # eval": "{\"description\": \"min=131, mean=131, max=131, sum=262 (2)\", \"tab\": \"General information\", \"score\": \"131.0\"}", - "Human Sexuality - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Sexuality - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # prompt tokens": "{\"description\": \"min=394.748, mean=394.748, max=394.748, sum=789.496 (2)\", \"tab\": \"General information\", \"score\": \"394.7480916030534\"}", - "Human Sexuality - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"human_sexuality\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_human_sexuality\"" - } - } - }, - { - "evaluation_name": "International Law", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on International Law", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.76, - "details": { - "description": "min=0.76, mean=0.76, max=0.76, sum=1.521 (2)", - "tab": "Accuracy", - "International Law - Observed inference time (s)": "{\"description\": \"min=0.91, mean=0.91, max=0.91, sum=1.82 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9102441850772574\"}", - "International Law - # eval": "{\"description\": \"min=121, mean=121, max=121, sum=242 (2)\", \"tab\": \"General information\", \"score\": \"121.0\"}", - "International Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "International Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "International Law - # prompt tokens": "{\"description\": \"min=720.182, mean=720.182, max=720.182, sum=1440.364 (2)\", \"tab\": \"General information\", \"score\": \"720.1818181818181\"}", - "International Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"international_law\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_international_law\"" - } - } - }, - { - "evaluation_name": "Logical Fallacies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Logical Fallacies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.693, - "details": { - "description": "min=0.693, mean=0.693, max=0.693, sum=1.387 (2)", - "tab": "Accuracy", - "Logical Fallacies - Observed inference time (s)": "{\"description\": \"min=0.806, mean=0.806, max=0.806, sum=1.613 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8063952381625498\"}", - "Logical Fallacies - # eval": "{\"description\": \"min=163, mean=163, max=163, sum=326 (2)\", \"tab\": \"General information\", \"score\": \"163.0\"}", - "Logical Fallacies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Logical Fallacies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Logical Fallacies - # prompt tokens": "{\"description\": \"min=486.779, mean=486.779, max=486.779, sum=973.558 (2)\", \"tab\": \"General information\", \"score\": \"486.77914110429447\"}", - "Logical Fallacies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"logical_fallacies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_logical_fallacies\"" - } - } - }, - { - "evaluation_name": "Machine Learning", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Machine Learning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.438, - "details": { - "description": "min=0.438, mean=0.438, max=0.438, sum=0.875 (2)", - "tab": "Accuracy", - "Machine Learning - Observed inference time (s)": "{\"description\": \"min=0.751, mean=0.751, max=0.751, sum=1.503 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7514570632151195\"}", - "Machine Learning - # eval": "{\"description\": \"min=112, mean=112, max=112, sum=224 (2)\", \"tab\": \"General information\", \"score\": \"112.0\"}", - "Machine Learning - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Machine Learning - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Machine Learning - # prompt tokens": "{\"description\": \"min=734.83, mean=734.83, max=734.83, sum=1469.661 (2)\", \"tab\": \"General information\", \"score\": \"734.8303571428571\"}", - "Machine Learning - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"machine_learning\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_machine_learning\"" - } - } - }, - { - "evaluation_name": "Management", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Management", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.709, - "details": { - "description": "min=0.709, mean=0.709, max=0.709, sum=1.417 (2)", - "tab": "Accuracy", - "Management - Observed inference time (s)": "{\"description\": \"min=0.934, mean=0.934, max=0.934, sum=1.868 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9339890294862025\"}", - "Management - # eval": "{\"description\": \"min=103, mean=103, max=103, sum=206 (2)\", \"tab\": \"General information\", \"score\": \"103.0\"}", - "Management - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Management - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Management - # prompt tokens": "{\"description\": \"min=315.359, mean=315.359, max=315.359, sum=630.718 (2)\", \"tab\": \"General information\", \"score\": \"315.3592233009709\"}", - "Management - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"management\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_management\"" - } - } - }, - { - "evaluation_name": "Marketing", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Marketing", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.833, - "details": { - "description": "min=0.833, mean=0.833, max=0.833, sum=1.667 (2)", - "tab": "Accuracy", - "Marketing - Observed inference time (s)": "{\"description\": \"min=1.072, mean=1.072, max=1.072, sum=2.144 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.0717963163669293\"}", - "Marketing - # eval": "{\"description\": \"min=234, mean=234, max=234, sum=468 (2)\", \"tab\": \"General information\", \"score\": \"234.0\"}", - "Marketing - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Marketing - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Marketing - # prompt tokens": "{\"description\": \"min=463.423, mean=463.423, max=463.423, sum=926.846 (2)\", \"tab\": \"General information\", \"score\": \"463.4230769230769\"}", - "Marketing - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"marketing\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_marketing\"" - } - } - }, - { - "evaluation_name": "Medical Genetics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Medical Genetics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.68, - "details": { - "description": "min=0.68, mean=0.68, max=0.68, sum=1.36 (2)", - "tab": "Accuracy", - "Medical Genetics - Observed inference time (s)": "{\"description\": \"min=0.929, mean=0.929, max=0.929, sum=1.859 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9293915629386902\"}", - "Medical Genetics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Medical Genetics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Medical Genetics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Medical Genetics - # prompt tokens": "{\"description\": \"min=405.71, mean=405.71, max=405.71, sum=811.42 (2)\", \"tab\": \"General information\", \"score\": \"405.71\"}", - "Medical Genetics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"medical_genetics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_medical_genetics\"" - } - } - }, - { - "evaluation_name": "Miscellaneous", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Miscellaneous", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.72, - "details": { - "description": "min=0.72, mean=0.72, max=0.72, sum=1.441 (2)", - "tab": "Accuracy", - "Miscellaneous - Observed inference time (s)": "{\"description\": \"min=0.874, mean=0.874, max=0.874, sum=1.747 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8736470007500582\"}", - "Miscellaneous - # eval": "{\"description\": \"min=783, mean=783, max=783, sum=1566 (2)\", \"tab\": \"General information\", \"score\": \"783.0\"}", - "Miscellaneous - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Miscellaneous - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Miscellaneous - # prompt tokens": "{\"description\": \"min=348.519, mean=348.519, max=348.519, sum=697.037 (2)\", \"tab\": \"General information\", \"score\": \"348.51851851851853\"}", - "Miscellaneous - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"miscellaneous\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_miscellaneous\"" - } - } - }, - { - "evaluation_name": "Moral Scenarios", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Moral Scenarios", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.33, - "details": { - "description": "min=0.33, mean=0.33, max=0.33, sum=0.659 (2)", - "tab": "Accuracy", - "Moral Disputes - Observed inference time (s)": "{\"description\": \"min=0.772, mean=0.772, max=0.772, sum=1.545 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7723477258847627\"}", - "Moral Scenarios - Observed inference time (s)": "{\"description\": \"min=0.887, mean=0.887, max=0.887, sum=1.774 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8867556284259818\"}", - "Moral Disputes - # eval": "{\"description\": \"min=346, mean=346, max=346, sum=692 (2)\", \"tab\": \"General information\", \"score\": \"346.0\"}", - "Moral Disputes - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Disputes - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Disputes - # prompt tokens": "{\"description\": \"min=540.038, mean=540.038, max=540.038, sum=1080.075 (2)\", \"tab\": \"General information\", \"score\": \"540.0375722543353\"}", - "Moral Disputes - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Moral Scenarios - # eval": "{\"description\": \"min=895, mean=895, max=895, sum=1790 (2)\", \"tab\": \"General information\", \"score\": \"895.0\"}", - "Moral Scenarios - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Scenarios - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # prompt tokens": "{\"description\": \"min=745.516, mean=745.516, max=745.516, sum=1491.032 (2)\", \"tab\": \"General information\", \"score\": \"745.5162011173185\"}", - "Moral Scenarios - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"moral_scenarios\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_moral_scenarios\"" - } - } - }, - { - "evaluation_name": "Nutrition", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Nutrition", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.657, - "details": { - "description": "min=0.657, mean=0.657, max=0.657, sum=1.314 (2)", - "tab": "Accuracy", - "Nutrition - Observed inference time (s)": "{\"description\": \"min=0.982, mean=0.982, max=0.982, sum=1.964 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9817679053038554\"}", - "Nutrition - # eval": "{\"description\": \"min=306, mean=306, max=306, sum=612 (2)\", \"tab\": \"General information\", \"score\": \"306.0\"}", - "Nutrition - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Nutrition - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Nutrition - # prompt tokens": "{\"description\": \"min=680.69, mean=680.69, max=680.69, sum=1361.379 (2)\", \"tab\": \"General information\", \"score\": \"680.6895424836601\"}", - "Nutrition - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"nutrition\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_nutrition\"" - } - } - }, - { - "evaluation_name": "Prehistory", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Prehistory", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.642, - "details": { - "description": "min=0.642, mean=0.642, max=0.642, sum=1.284 (2)", - "tab": "Accuracy", - "Prehistory - Observed inference time (s)": "{\"description\": \"min=0.752, mean=0.752, max=0.752, sum=1.505 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7522576863383069\"}", - "Prehistory - # eval": "{\"description\": \"min=324, mean=324, max=324, sum=648 (2)\", \"tab\": \"General information\", \"score\": \"324.0\"}", - "Prehistory - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Prehistory - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Prehistory - # prompt tokens": "{\"description\": \"min=602.145, mean=602.145, max=602.145, sum=1204.29 (2)\", \"tab\": \"General information\", \"score\": \"602.145061728395\"}", - "Prehistory - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"prehistory\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_prehistory\"" - } - } - }, - { - "evaluation_name": "Public Relations", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Public Relations", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6, - "details": { - "description": "min=0.6, mean=0.6, max=0.6, sum=1.2 (2)", - "tab": "Accuracy", - "Public Relations - Observed inference time (s)": "{\"description\": \"min=1.121, mean=1.121, max=1.121, sum=2.241 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.120634336905046\"}", - "Public Relations - # eval": "{\"description\": \"min=110, mean=110, max=110, sum=220 (2)\", \"tab\": \"General information\", \"score\": \"110.0\"}", - "Public Relations - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Public Relations - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Public Relations - # prompt tokens": "{\"description\": \"min=462.036, mean=462.036, max=462.036, sum=924.073 (2)\", \"tab\": \"General information\", \"score\": \"462.03636363636366\"}", - "Public Relations - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"public_relations\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_public_relations\"" - } - } - }, - { - "evaluation_name": "Security Studies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Security Studies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.731, - "details": { - "description": "min=0.731, mean=0.731, max=0.731, sum=1.461 (2)", - "tab": "Accuracy", - "Security Studies - Observed inference time (s)": "{\"description\": \"min=0.837, mean=0.837, max=0.837, sum=1.674 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8369822920585165\"}", - "Security Studies - # eval": "{\"description\": \"min=245, mean=245, max=245, sum=490 (2)\", \"tab\": \"General information\", \"score\": \"245.0\"}", - "Security Studies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Security Studies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Security Studies - # prompt tokens": "{\"description\": \"min=1315.865, mean=1315.865, max=1315.865, sum=2631.731 (2)\", \"tab\": \"General information\", \"score\": \"1315.865306122449\"}", - "Security Studies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"security_studies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_security_studies\"" - } - } - }, - { - "evaluation_name": "Sociology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Sociology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.831, - "details": { - "description": "min=0.831, mean=0.831, max=0.831, sum=1.662 (2)", - "tab": "Accuracy", - "Sociology - Observed inference time (s)": "{\"description\": \"min=0.909, mean=0.909, max=0.909, sum=1.819 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9092605125844775\"}", - "Sociology - # eval": "{\"description\": \"min=201, mean=201, max=201, sum=402 (2)\", \"tab\": \"General information\", \"score\": \"201.0\"}", - "Sociology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Sociology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Sociology - # prompt tokens": "{\"description\": \"min=487.95, mean=487.95, max=487.95, sum=975.9 (2)\", \"tab\": \"General information\", \"score\": \"487.9502487562189\"}", - "Sociology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"sociology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_sociology\"" - } - } - }, - { - "evaluation_name": "Virology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Virology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.44, - "details": { - "description": "min=0.44, mean=0.44, max=0.44, sum=0.88 (2)", - "tab": "Accuracy", - "Virology - Observed inference time (s)": "{\"description\": \"min=0.885, mean=0.885, max=0.885, sum=1.771 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8854893704494798\"}", - "Virology - # eval": "{\"description\": \"min=166, mean=166, max=166, sum=332 (2)\", \"tab\": \"General information\", \"score\": \"166.0\"}", - "Virology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Virology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Virology - # prompt tokens": "{\"description\": \"min=395.349, mean=395.349, max=395.349, sum=790.699 (2)\", \"tab\": \"General information\", \"score\": \"395.34939759036143\"}", - "Virology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"virology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_virology\"" - } - } - }, - { - "evaluation_name": "World Religions", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on World Religions", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.789, - "details": { - "description": "min=0.789, mean=0.789, max=0.789, sum=1.579 (2)", - "tab": "Accuracy", - "World Religions - Observed inference time (s)": "{\"description\": \"min=0.863, mean=0.863, max=0.863, sum=1.726 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8629393619403505\"}", - "World Religions - # eval": "{\"description\": \"min=171, mean=171, max=171, sum=342 (2)\", \"tab\": \"General information\", \"score\": \"171.0\"}", - "World Religions - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "World Religions - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "World Religions - # prompt tokens": "{\"description\": \"min=308.924, mean=308.924, max=308.924, sum=617.848 (2)\", \"tab\": \"General information\", \"score\": \"308.92397660818716\"}", - "World Religions - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"world_religions\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_world_religions\"" - } - } - }, - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.213, - "details": { - "description": "", - "tab": "Efficiency" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_id": "hfopenllm_v2/mistralai_Mistral-7B-v0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2386 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4419 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0295 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2919 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4139 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3013 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/mistralai_mistral-7b-v0.3.json b/data/models/mistralai_mistral-7b-v0.3.json deleted file mode 100644 index ecfc7f243c032e40ea8b3e59f6f7565d1b8fa595..0000000000000000000000000000000000000000 --- a/data/models/mistralai_mistral-7b-v0.3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mistral-7B-v0.3", - "id": "mistralai/Mistral-7B-v0.3", - "developer": "mistralai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/mistralai_Mistral-7B-v0.3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2266 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4517 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0302 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2919 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4032 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2953 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/mistralai_mistral-large-2402.json b/data/models/mistralai_mistral-large-2402.json deleted file mode 100644 index 5ddb96d4876ea3a21d5d2b5caa199d12881ab7dc..0000000000000000000000000000000000000000 --- a/data/models/mistralai_mistral-large-2402.json +++ /dev/null @@ -1,1901 +0,0 @@ -{ - "model_info": { - "name": "Mistral Large 2402", - "id": "mistralai/mistral-large-2402", - "developer": "mistralai", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_lite/mistralai_mistral-large-2402/1774096306.427425", - "retrieved_timestamp": "1774096306.427425", - "source_metadata": { - "source_name": "helm_lite", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_lite", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_lite", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.328, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.23681647940074904\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.454, - "details": { - "description": "min=0.454, mean=0.454, max=0.454, sum=0.454 (1)", - "tab": "Accuracy", - "NarrativeQA - Observed inference time (s)": "{\"description\": \"min=1.675, mean=1.675, max=1.675, sum=1.675 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.6750120075655655\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=3955.33, mean=3955.33, max=3955.33, sum=3955.33 (1)\", \"tab\": \"General information\", \"score\": \"3955.3295774647886\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=22.614, mean=22.614, max=22.614, sum=22.614 (1)\", \"tab\": \"General information\", \"score\": \"22.614084507042254\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (closed-book)", - "source_data": { - "dataset_name": "NaturalQuestions (closed-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (closed-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.311, - "details": { - "description": "min=0.311, mean=0.311, max=0.311, sum=0.311 (1)", - "tab": "Accuracy", - "NaturalQuestions (open-book) - Observed inference time (s)": "{\"description\": \"min=1.666, mean=1.666, max=1.666, sum=1.666 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.665770656108856\"}", - "NaturalQuestions (closed-book) - Observed inference time (s)": "{\"description\": \"min=2.122, mean=2.122, max=2.122, sum=2.122 (1)\", \"tab\": \"Efficiency\", \"score\": \"2.1218616259098053\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=2537.79, mean=2537.79, max=2537.79, sum=2537.79 (1)\", \"tab\": \"General information\", \"score\": \"2537.79\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=27.337, mean=27.337, max=27.337, sum=27.337 (1)\", \"tab\": \"General information\", \"score\": \"27.337\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=211.069, mean=211.069, max=211.069, sum=211.069 (1)\", \"tab\": \"General information\", \"score\": \"211.069\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=27.91, mean=27.91, max=27.91, sum=27.91 (1)\", \"tab\": \"General information\", \"score\": \"27.91\"}" - } - }, - "generation_config": { - "additional_details": { - "mode": "\"closedbook\"" - } - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.894, - "details": { - "description": "min=0.894, mean=0.894, max=0.894, sum=0.894 (1)", - "tab": "Accuracy", - "OpenbookQA - Observed inference time (s)": "{\"description\": \"min=0.569, mean=0.569, max=0.569, sum=0.569 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.5687967395782471\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=280.15, mean=280.15, max=280.15, sum=280.15 (1)\", \"tab\": \"General information\", \"score\": \"280.15\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "dataset": "\"openbookqa\"", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.638, - "details": { - "description": "min=0.38, mean=0.638, max=0.92, sum=3.19 (5)", - "tab": "Accuracy", - "MMLU - Observed inference time (s)": "{\"description\": \"min=1.226, mean=1.451, max=1.633, sum=7.257 (5)\", \"tab\": \"Efficiency\", \"score\": \"1.4514196366845515\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=402.44, mean=523.091, max=687.175, sum=2615.455 (5)\", \"tab\": \"General information\", \"score\": \"523.0910877192983\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"college_chemistry\", \"computer_security\", \"econometrics\", \"us_foreign_policy\"]", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MATH", - "source_data": { - "dataset_name": "MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Equivalent (CoT) on MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.75, - "details": { - "description": "min=0.632, mean=0.75, max=0.904, sum=5.253 (7)", - "tab": "Accuracy", - "MATH - Observed inference time (s)": "{\"description\": \"min=3.885, mean=5.128, max=5.812, sum=35.896 (7)\", \"tab\": \"Efficiency\", \"score\": \"5.128044104863146\"}", - "MATH - # eval": "{\"description\": \"min=30, mean=62.429, max=135, sum=437 (7)\", \"tab\": \"General information\", \"score\": \"62.42857142857143\"}", - "MATH - # train": "{\"description\": \"min=8, mean=8, max=8, sum=56 (7)\", \"tab\": \"General information\", \"score\": \"8.0\"}", - "MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MATH - # prompt tokens": "{\"description\": \"min=1061.615, mean=1525.266, max=2572.962, sum=10676.865 (7)\", \"tab\": \"General information\", \"score\": \"1525.2664139976257\"}", - "MATH - # output tokens": "{\"description\": \"min=101.444, mean=128.216, max=154.897, sum=897.515 (7)\", \"tab\": \"General information\", \"score\": \"128.21647245723133\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"algebra\", \"counting_and_probability\", \"geometry\", \"intermediate_algebra\", \"number_theory\", \"prealgebra\", \"precalculus\"]", - "level": "\"1\"", - "use_official_examples": "\"False\"", - "use_chain_of_thought": "\"True\"" - } - } - }, - { - "evaluation_name": "GSM8K", - "source_data": { - "dataset_name": "GSM8K", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on GSM8K", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.694, - "details": { - "description": "min=0.694, mean=0.694, max=0.694, sum=0.694 (1)", - "tab": "Accuracy", - "GSM8K - Observed inference time (s)": "{\"description\": \"min=7.095, mean=7.095, max=7.095, sum=7.095 (1)\", \"tab\": \"Efficiency\", \"score\": \"7.095049407720566\"}", - "GSM8K - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "GSM8K - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "GSM8K - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GSM8K - # prompt tokens": "{\"description\": \"min=1255.268, mean=1255.268, max=1255.268, sum=1255.268 (1)\", \"tab\": \"General information\", \"score\": \"1255.268\"}", - "GSM8K - # output tokens": "{\"description\": \"min=129.185, mean=129.185, max=129.185, sum=129.185 (1)\", \"tab\": \"General information\", \"score\": \"129.185\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "LegalBench", - "source_data": { - "dataset_name": "LegalBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on LegalBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.479, - "details": { - "description": "min=0.1, mean=0.479, max=0.821, sum=2.394 (5)", - "tab": "Accuracy", - "LegalBench - Observed inference time (s)": "{\"description\": \"min=0.985, mean=1.692, max=2.787, sum=8.462 (5)\", \"tab\": \"Efficiency\", \"score\": \"1.6924799473534797\"}", - "LegalBench - # eval": "{\"description\": \"min=50, mean=312.4, max=1000, sum=1562 (5)\", \"tab\": \"General information\", \"score\": \"312.4\"}", - "LegalBench - # train": "{\"description\": \"min=4, mean=4.8, max=5, sum=24 (5)\", \"tab\": \"General information\", \"score\": \"4.8\"}", - "LegalBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "LegalBench - # prompt tokens": "{\"description\": \"min=219.46, mean=1783.252, max=7251, sum=8916.261 (5)\", \"tab\": \"General information\", \"score\": \"1783.2521685070988\"}", - "LegalBench - # output tokens": "{\"description\": \"min=1.005, mean=8.217, max=25.86, sum=41.087 (5)\", \"tab\": \"General information\", \"score\": \"8.217420478990393\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"abercrombie\", \"corporate_lobbying\", \"function_of_decision_section\", \"international_citizenship_questions\", \"proa\"]" - } - } - }, - { - "evaluation_name": "MedQA", - "source_data": { - "dataset_name": "MedQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MedQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.499, - "details": { - "description": "min=0.499, mean=0.499, max=0.499, sum=0.499 (1)", - "tab": "Accuracy", - "MedQA - Observed inference time (s)": "{\"description\": \"min=0.595, mean=0.595, max=0.595, sum=0.595 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.5950325303238856\"}", - "MedQA - # eval": "{\"description\": \"min=503, mean=503, max=503, sum=503 (1)\", \"tab\": \"General information\", \"score\": \"503.0\"}", - "MedQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MedQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MedQA - # prompt tokens": "{\"description\": \"min=1193.093, mean=1193.093, max=1193.093, sum=1193.093 (1)\", \"tab\": \"General information\", \"score\": \"1193.0934393638172\"}", - "MedQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WMT 2014", - "source_data": { - "dataset_name": "WMT 2014", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "BLEU-4 on WMT 2014", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.182, - "details": { - "description": "min=0.098, mean=0.182, max=0.224, sum=0.909 (5)", - "tab": "Accuracy", - "WMT 2014 - Observed inference time (s)": "{\"description\": \"min=1.69, mean=1.969, max=2.702, sum=9.846 (5)\", \"tab\": \"Efficiency\", \"score\": \"1.969239294333439\"}", - "WMT 2014 - # eval": "{\"description\": \"min=503, mean=568.8, max=832, sum=2844 (5)\", \"tab\": \"General information\", \"score\": \"568.8\"}", - "WMT 2014 - # train": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "WMT 2014 - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WMT 2014 - # prompt tokens": "{\"description\": \"min=198.306, mean=212.433, max=231.018, sum=1062.166 (5)\", \"tab\": \"General information\", \"score\": \"212.43317355482492\"}", - "WMT 2014 - # output tokens": "{\"description\": \"min=27.272, mean=29.042, max=29.871, sum=145.211 (5)\", \"tab\": \"General information\", \"score\": \"29.04227089386756\"}" - } - }, - "generation_config": { - "additional_details": { - "language_pair": "[\"cs-en\", \"de-en\", \"fr-en\", \"hi-en\", \"ru-en\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_mmlu/mistralai_mistral-large-2402/1774096312.00548", - "retrieved_timestamp": "1774096312.00548", - "source_metadata": { - "source_name": "helm_mmlu", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_mmlu", - "evaluation_results": [ - { - "evaluation_name": "MMLU All Subjects", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU All Subjects", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.688, - "details": { - "description": "min=0.211, mean=0.688, max=0.964, sum=78.413 (114)", - "tab": "Accuracy", - "MMLU All Subjects - Observed inference time (s)": "{\"description\": \"min=0.356, mean=0.546, max=1.633, sum=62.26 (114)\", \"tab\": \"Efficiency\", \"score\": \"0.5461372164599003\"}", - "MMLU All Subjects - # eval": "{\"description\": \"min=100, mean=246.351, max=1534, sum=28084 (114)\", \"tab\": \"General information\", \"score\": \"246.35087719298247\"}", - "MMLU All Subjects - # train": "{\"description\": \"min=5, mean=5, max=5, sum=570 (114)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU All Subjects - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU All Subjects - # prompt tokens": "{\"description\": \"min=308.924, mean=696.273, max=3089.109, sum=79375.178 (114)\", \"tab\": \"General information\", \"score\": \"696.2734899593811\"}", - "MMLU All Subjects - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=114 (114)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_name": "Abstract Algebra", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Abstract Algebra", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.45, - "details": { - "description": "min=0.45, mean=0.45, max=0.45, sum=0.9 (2)", - "tab": "Accuracy", - "Abstract Algebra - Observed inference time (s)": "{\"description\": \"min=1.48, mean=1.48, max=1.48, sum=2.959 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.4797466564178468\"}", - "Abstract Algebra - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Abstract Algebra - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Abstract Algebra - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Abstract Algebra - # prompt tokens": "{\"description\": \"min=402.44, mean=402.44, max=402.44, sum=804.88 (2)\", \"tab\": \"General information\", \"score\": \"402.44\"}", - "Abstract Algebra - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"abstract_algebra\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_abstract_algebra\"" - } - } - }, - { - "evaluation_name": "Anatomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Anatomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.674, - "details": { - "description": "min=0.674, mean=0.674, max=0.674, sum=1.348 (2)", - "tab": "Accuracy", - "Anatomy - Observed inference time (s)": "{\"description\": \"min=0.484, mean=0.484, max=0.484, sum=0.968 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4840934417865895\"}", - "Anatomy - # eval": "{\"description\": \"min=135, mean=135, max=135, sum=270 (2)\", \"tab\": \"General information\", \"score\": \"135.0\"}", - "Anatomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Anatomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Anatomy - # prompt tokens": "{\"description\": \"min=407.089, mean=407.089, max=407.089, sum=814.178 (2)\", \"tab\": \"General information\", \"score\": \"407.0888888888889\"}", - "Anatomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"anatomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_anatomy\"" - } - } - }, - { - "evaluation_name": "College Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on College Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.373, - "details": { - "description": "min=0.373, mean=0.373, max=0.373, sum=0.745 (2)", - "tab": "Accuracy", - "College Chemistry - Observed inference time (s)": "{\"description\": \"min=1.226, mean=1.226, max=1.226, sum=2.452 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.2259348821640015\"}", - "College Biology - Observed inference time (s)": "{\"description\": \"min=0.438, mean=0.438, max=0.438, sum=0.875 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.43758388525909847\"}", - "College Computer Science - Observed inference time (s)": "{\"description\": \"min=0.412, mean=0.412, max=0.412, sum=0.825 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.41238118410110475\"}", - "College Mathematics - Observed inference time (s)": "{\"description\": \"min=0.443, mean=0.443, max=0.443, sum=0.886 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.44315950393676756\"}", - "College Medicine - Observed inference time (s)": "{\"description\": \"min=0.425, mean=0.425, max=0.425, sum=0.849 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4246950163317554\"}", - "College Physics - Observed inference time (s)": "{\"description\": \"min=0.511, mean=0.511, max=0.511, sum=1.021 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.510722931693582\"}", - "College Chemistry - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Chemistry - # prompt tokens": "{\"description\": \"min=627.71, mean=627.71, max=627.71, sum=1255.42 (2)\", \"tab\": \"General information\", \"score\": \"627.71\"}", - "College Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Biology - # eval": "{\"description\": \"min=144, mean=144, max=144, sum=288 (2)\", \"tab\": \"General information\", \"score\": \"144.0\"}", - "College Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # prompt tokens": "{\"description\": \"min=550.799, mean=550.799, max=550.799, sum=1101.597 (2)\", \"tab\": \"General information\", \"score\": \"550.7986111111111\"}", - "College Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # prompt tokens": "{\"description\": \"min=902.17, mean=902.17, max=902.17, sum=1804.34 (2)\", \"tab\": \"General information\", \"score\": \"902.17\"}", - "College Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Mathematics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # prompt tokens": "{\"description\": \"min=658.31, mean=658.31, max=658.31, sum=1316.62 (2)\", \"tab\": \"General information\", \"score\": \"658.31\"}", - "College Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Medicine - # eval": "{\"description\": \"min=173, mean=173, max=173, sum=346 (2)\", \"tab\": \"General information\", \"score\": \"173.0\"}", - "College Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # prompt tokens": "{\"description\": \"min=592.41, mean=592.41, max=592.41, sum=1184.821 (2)\", \"tab\": \"General information\", \"score\": \"592.4104046242775\"}", - "College Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Physics - # eval": "{\"description\": \"min=102, mean=102, max=102, sum=204 (2)\", \"tab\": \"General information\", \"score\": \"102.0\"}", - "College Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # prompt tokens": "{\"description\": \"min=551.029, mean=551.029, max=551.029, sum=1102.059 (2)\", \"tab\": \"General information\", \"score\": \"551.0294117647059\"}", - "College Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"college_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_college_physics\"" - } - } - }, - { - "evaluation_name": "Computer Security", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Computer Security", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8, - "details": { - "description": "min=0.8, mean=0.8, max=0.8, sum=1.6 (2)", - "tab": "Accuracy", - "Computer Security - Observed inference time (s)": "{\"description\": \"min=1.535, mean=1.535, max=1.535, sum=3.071 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.5353856110572814\"}", - "Computer Security - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Computer Security - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Computer Security - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Computer Security - # prompt tokens": "{\"description\": \"min=424.94, mean=424.94, max=424.94, sum=849.88 (2)\", \"tab\": \"General information\", \"score\": \"424.94\"}", - "Computer Security - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"computer_security\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_computer_security\"" - } - } - }, - { - "evaluation_name": "Econometrics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Econometrics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.64, - "details": { - "description": "min=0.64, mean=0.64, max=0.64, sum=1.281 (2)", - "tab": "Accuracy", - "Econometrics - Observed inference time (s)": "{\"description\": \"min=1.383, mean=1.383, max=1.383, sum=2.766 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.382804548531248\"}", - "Econometrics - # eval": "{\"description\": \"min=114, mean=114, max=114, sum=228 (2)\", \"tab\": \"General information\", \"score\": \"114.0\"}", - "Econometrics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Econometrics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Econometrics - # prompt tokens": "{\"description\": \"min=687.175, mean=687.175, max=687.175, sum=1374.351 (2)\", \"tab\": \"General information\", \"score\": \"687.1754385964912\"}", - "Econometrics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"econometrics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_econometrics\"" - } - } - }, - { - "evaluation_name": "Global Facts", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Global Facts", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.34, - "details": { - "description": "min=0.34, mean=0.34, max=0.34, sum=0.68 (2)", - "tab": "Accuracy", - "Global Facts - Observed inference time (s)": "{\"description\": \"min=0.492, mean=0.492, max=0.492, sum=0.984 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.49177081823349\"}", - "Global Facts - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Global Facts - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Global Facts - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Global Facts - # prompt tokens": "{\"description\": \"min=483.47, mean=483.47, max=483.47, sum=966.94 (2)\", \"tab\": \"General information\", \"score\": \"483.47\"}", - "Global Facts - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"global_facts\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_global_facts\"" - } - } - }, - { - "evaluation_name": "Jurisprudence", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Jurisprudence", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.815, - "details": { - "description": "min=0.815, mean=0.815, max=0.815, sum=1.63 (2)", - "tab": "Accuracy", - "Jurisprudence - Observed inference time (s)": "{\"description\": \"min=0.505, mean=0.505, max=0.505, sum=1.01 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5051956353364168\"}", - "Jurisprudence - # eval": "{\"description\": \"min=108, mean=108, max=108, sum=216 (2)\", \"tab\": \"General information\", \"score\": \"108.0\"}", - "Jurisprudence - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Jurisprudence - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Jurisprudence - # prompt tokens": "{\"description\": \"min=451.093, mean=451.093, max=451.093, sum=902.185 (2)\", \"tab\": \"General information\", \"score\": \"451.0925925925926\"}", - "Jurisprudence - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"jurisprudence\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_jurisprudence\"" - } - } - }, - { - "evaluation_name": "Philosophy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Philosophy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.794, - "details": { - "description": "min=0.794, mean=0.794, max=0.794, sum=1.588 (2)", - "tab": "Accuracy", - "Philosophy - Observed inference time (s)": "{\"description\": \"min=0.506, mean=0.506, max=0.506, sum=1.011 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5055920081123279\"}", - "Philosophy - # eval": "{\"description\": \"min=311, mean=311, max=311, sum=622 (2)\", \"tab\": \"General information\", \"score\": \"311.0\"}", - "Philosophy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Philosophy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Philosophy - # prompt tokens": "{\"description\": \"min=373.82, mean=373.82, max=373.82, sum=747.64 (2)\", \"tab\": \"General information\", \"score\": \"373.81993569131834\"}", - "Philosophy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"philosophy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_philosophy\"" - } - } - }, - { - "evaluation_name": "Professional Psychology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Professional Psychology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.809, - "details": { - "description": "min=0.809, mean=0.809, max=0.809, sum=1.618 (2)", - "tab": "Accuracy", - "Professional Medicine - Observed inference time (s)": "{\"description\": \"min=0.478, mean=0.478, max=0.478, sum=0.956 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4777693476747064\"}", - "Professional Accounting - Observed inference time (s)": "{\"description\": \"min=0.443, mean=0.443, max=0.443, sum=0.886 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4430855546437257\"}", - "Professional Law - Observed inference time (s)": "{\"description\": \"min=0.493, mean=0.493, max=0.493, sum=0.987 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4934647888372588\"}", - "Professional Psychology - Observed inference time (s)": "{\"description\": \"min=0.483, mean=0.483, max=0.483, sum=0.966 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4830952575004179\"}", - "Professional Medicine - # eval": "{\"description\": \"min=272, mean=272, max=272, sum=544 (2)\", \"tab\": \"General information\", \"score\": \"272.0\"}", - "Professional Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Medicine - # prompt tokens": "{\"description\": \"min=1279.143, mean=1279.143, max=1279.143, sum=2558.287 (2)\", \"tab\": \"General information\", \"score\": \"1279.1433823529412\"}", - "Professional Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Accounting - # eval": "{\"description\": \"min=282, mean=282, max=282, sum=564 (2)\", \"tab\": \"General information\", \"score\": \"282.0\"}", - "Professional Accounting - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Accounting - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # prompt tokens": "{\"description\": \"min=796.496, mean=796.496, max=796.496, sum=1592.993 (2)\", \"tab\": \"General information\", \"score\": \"796.4964539007092\"}", - "Professional Accounting - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Law - # eval": "{\"description\": \"min=1534, mean=1534, max=1534, sum=3068 (2)\", \"tab\": \"General information\", \"score\": \"1534.0\"}", - "Professional Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # prompt tokens": "{\"description\": \"min=1849.711, mean=1849.711, max=1849.711, sum=3699.421 (2)\", \"tab\": \"General information\", \"score\": \"1849.7105606258149\"}", - "Professional Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Psychology - # eval": "{\"description\": \"min=612, mean=612, max=612, sum=1224 (2)\", \"tab\": \"General information\", \"score\": \"612.0\"}", - "Professional Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # prompt tokens": "{\"description\": \"min=645.278, mean=645.278, max=645.278, sum=1290.556 (2)\", \"tab\": \"General information\", \"score\": \"645.2777777777778\"}", - "Professional Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"professional_psychology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_professional_psychology\"" - } - } - }, - { - "evaluation_name": "Us Foreign Policy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Us Foreign Policy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.92, - "details": { - "description": "min=0.92, mean=0.92, max=0.92, sum=1.84 (2)", - "tab": "Accuracy", - "Us Foreign Policy - Observed inference time (s)": "{\"description\": \"min=1.633, mean=1.633, max=1.633, sum=3.266 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.6332264852523803\"}", - "Us Foreign Policy - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Us Foreign Policy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Us Foreign Policy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Us Foreign Policy - # prompt tokens": "{\"description\": \"min=473.19, mean=473.19, max=473.19, sum=946.38 (2)\", \"tab\": \"General information\", \"score\": \"473.19\"}", - "Us Foreign Policy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"us_foreign_policy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_us_foreign_policy\"" - } - } - }, - { - "evaluation_name": "Astronomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Astronomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.842, - "details": { - "description": "min=0.842, mean=0.842, max=0.842, sum=1.684 (2)", - "tab": "Accuracy", - "Astronomy - Observed inference time (s)": "{\"description\": \"min=0.45, mean=0.45, max=0.45, sum=0.901 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4503253243471447\"}", - "Astronomy - # eval": "{\"description\": \"min=152, mean=152, max=152, sum=304 (2)\", \"tab\": \"General information\", \"score\": \"152.0\"}", - "Astronomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Astronomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Astronomy - # prompt tokens": "{\"description\": \"min=665.987, mean=665.987, max=665.987, sum=1331.974 (2)\", \"tab\": \"General information\", \"score\": \"665.9868421052631\"}", - "Astronomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"astronomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_astronomy\"" - } - } - }, - { - "evaluation_name": "Business Ethics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Business Ethics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.67, - "details": { - "description": "min=0.67, mean=0.67, max=0.67, sum=1.34 (2)", - "tab": "Accuracy", - "Business Ethics - Observed inference time (s)": "{\"description\": \"min=0.411, mean=0.411, max=0.411, sum=0.821 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4105031824111938\"}", - "Business Ethics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Business Ethics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Business Ethics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Business Ethics - # prompt tokens": "{\"description\": \"min=644.6, mean=644.6, max=644.6, sum=1289.2 (2)\", \"tab\": \"General information\", \"score\": \"644.6\"}", - "Business Ethics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"business_ethics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_business_ethics\"" - } - } - }, - { - "evaluation_name": "Clinical Knowledge", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Clinical Knowledge", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.751, - "details": { - "description": "min=0.751, mean=0.751, max=0.751, sum=1.502 (2)", - "tab": "Accuracy", - "Clinical Knowledge - Observed inference time (s)": "{\"description\": \"min=0.521, mean=0.521, max=0.521, sum=1.042 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5210292402303444\"}", - "Clinical Knowledge - # eval": "{\"description\": \"min=265, mean=265, max=265, sum=530 (2)\", \"tab\": \"General information\", \"score\": \"265.0\"}", - "Clinical Knowledge - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Clinical Knowledge - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Clinical Knowledge - # prompt tokens": "{\"description\": \"min=487.174, mean=487.174, max=487.174, sum=974.347 (2)\", \"tab\": \"General information\", \"score\": \"487.1735849056604\"}", - "Clinical Knowledge - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"clinical_knowledge\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_clinical_knowledge\"" - } - } - }, - { - "evaluation_name": "Conceptual Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Conceptual Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.574, - "details": { - "description": "min=0.574, mean=0.574, max=0.574, sum=1.149 (2)", - "tab": "Accuracy", - "Conceptual Physics - Observed inference time (s)": "{\"description\": \"min=0.418, mean=0.418, max=0.418, sum=0.835 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.41761813873940323\"}", - "Conceptual Physics - # eval": "{\"description\": \"min=235, mean=235, max=235, sum=470 (2)\", \"tab\": \"General information\", \"score\": \"235.0\"}", - "Conceptual Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Conceptual Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Conceptual Physics - # prompt tokens": "{\"description\": \"min=334.285, mean=334.285, max=334.285, sum=668.57 (2)\", \"tab\": \"General information\", \"score\": \"334.2851063829787\"}", - "Conceptual Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"conceptual_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_conceptual_physics\"" - } - } - }, - { - "evaluation_name": "Electrical Engineering", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Electrical Engineering", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.545, - "details": { - "description": "min=0.545, mean=0.545, max=0.545, sum=1.09 (2)", - "tab": "Accuracy", - "Electrical Engineering - Observed inference time (s)": "{\"description\": \"min=0.54, mean=0.54, max=0.54, sum=1.08 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5400767852520121\"}", - "Electrical Engineering - # eval": "{\"description\": \"min=145, mean=145, max=145, sum=290 (2)\", \"tab\": \"General information\", \"score\": \"145.0\"}", - "Electrical Engineering - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Electrical Engineering - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Electrical Engineering - # prompt tokens": "{\"description\": \"min=501.379, mean=501.379, max=501.379, sum=1002.759 (2)\", \"tab\": \"General information\", \"score\": \"501.37931034482756\"}", - "Electrical Engineering - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"electrical_engineering\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_electrical_engineering\"" - } - } - }, - { - "evaluation_name": "Elementary Mathematics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Elementary Mathematics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.508, - "details": { - "description": "min=0.508, mean=0.508, max=0.508, sum=1.016 (2)", - "tab": "Accuracy", - "Elementary Mathematics - Observed inference time (s)": "{\"description\": \"min=0.434, mean=0.434, max=0.434, sum=0.868 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4338057312385115\"}", - "Elementary Mathematics - # eval": "{\"description\": \"min=378, mean=378, max=378, sum=756 (2)\", \"tab\": \"General information\", \"score\": \"378.0\"}", - "Elementary Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Elementary Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Elementary Mathematics - # prompt tokens": "{\"description\": \"min=613.386, mean=613.386, max=613.386, sum=1226.772 (2)\", \"tab\": \"General information\", \"score\": \"613.3862433862433\"}", - "Elementary Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"elementary_mathematics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_elementary_mathematics\"" - } - } - }, - { - "evaluation_name": "Formal Logic", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Formal Logic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.532, - "details": { - "description": "min=0.532, mean=0.532, max=0.532, sum=1.063 (2)", - "tab": "Accuracy", - "Formal Logic - Observed inference time (s)": "{\"description\": \"min=0.512, mean=0.512, max=0.512, sum=1.024 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5122278436781869\"}", - "Formal Logic - # eval": "{\"description\": \"min=126, mean=126, max=126, sum=252 (2)\", \"tab\": \"General information\", \"score\": \"126.0\"}", - "Formal Logic - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Formal Logic - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Formal Logic - # prompt tokens": "{\"description\": \"min=718.984, mean=718.984, max=718.984, sum=1437.968 (2)\", \"tab\": \"General information\", \"score\": \"718.984126984127\"}", - "Formal Logic - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"formal_logic\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_formal_logic\"" - } - } - }, - { - "evaluation_name": "High School World History", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on High School World History", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.886, - "details": { - "description": "min=0.886, mean=0.886, max=0.886, sum=1.772 (2)", - "tab": "Accuracy", - "High School Biology - Observed inference time (s)": "{\"description\": \"min=0.526, mean=0.526, max=0.526, sum=1.052 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5259702259494412\"}", - "High School Chemistry - Observed inference time (s)": "{\"description\": \"min=0.402, mean=0.402, max=0.402, sum=0.803 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4016201167271055\"}", - "High School Computer Science - Observed inference time (s)": "{\"description\": \"min=0.398, mean=0.398, max=0.398, sum=0.797 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3984186482429504\"}", - "High School European History - Observed inference time (s)": "{\"description\": \"min=0.649, mean=0.649, max=0.649, sum=1.298 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6488189350474964\"}", - "High School Geography - Observed inference time (s)": "{\"description\": \"min=0.449, mean=0.449, max=0.449, sum=0.897 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.44867861752558236\"}", - "High School Government And Politics - Observed inference time (s)": "{\"description\": \"min=0.441, mean=0.441, max=0.441, sum=0.883 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.44147809675938104\"}", - "High School Macroeconomics - Observed inference time (s)": "{\"description\": \"min=0.456, mean=0.456, max=0.456, sum=0.912 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.45610924195020625\"}", - "High School Mathematics - Observed inference time (s)": "{\"description\": \"min=0.427, mean=0.427, max=0.427, sum=0.854 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4269448068406847\"}", - "High School Microeconomics - Observed inference time (s)": "{\"description\": \"min=0.402, mean=0.402, max=0.402, sum=0.805 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4023913435575341\"}", - "High School Physics - Observed inference time (s)": "{\"description\": \"min=0.43, mean=0.43, max=0.43, sum=0.861 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.43034561738273164\"}", - "High School Psychology - Observed inference time (s)": "{\"description\": \"min=0.428, mean=0.428, max=0.428, sum=0.856 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4278128755201987\"}", - "High School Statistics - Observed inference time (s)": "{\"description\": \"min=0.421, mean=0.421, max=0.421, sum=0.842 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.42108922203381854\"}", - "High School US History - Observed inference time (s)": "{\"description\": \"min=0.54, mean=0.54, max=0.54, sum=1.08 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5401732255430782\"}", - "High School World History - Observed inference time (s)": "{\"description\": \"min=0.471, mean=0.471, max=0.471, sum=0.943 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.47126107075043366\"}", - "High School Biology - # eval": "{\"description\": \"min=310, mean=310, max=310, sum=620 (2)\", \"tab\": \"General information\", \"score\": \"310.0\"}", - "High School Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Biology - # prompt tokens": "{\"description\": \"min=600.561, mean=600.561, max=600.561, sum=1201.123 (2)\", \"tab\": \"General information\", \"score\": \"600.5612903225806\"}", - "High School Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Chemistry - # eval": "{\"description\": \"min=203, mean=203, max=203, sum=406 (2)\", \"tab\": \"General information\", \"score\": \"203.0\"}", - "High School Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # prompt tokens": "{\"description\": \"min=572.798, mean=572.798, max=572.798, sum=1145.596 (2)\", \"tab\": \"General information\", \"score\": \"572.7980295566502\"}", - "High School Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "High School Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # prompt tokens": "{\"description\": \"min=988.24, mean=988.24, max=988.24, sum=1976.48 (2)\", \"tab\": \"General information\", \"score\": \"988.24\"}", - "High School Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School European History - # eval": "{\"description\": \"min=165, mean=165, max=165, sum=330 (2)\", \"tab\": \"General information\", \"score\": \"165.0\"}", - "High School European History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School European History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # prompt tokens": "{\"description\": \"min=3089.109, mean=3089.109, max=3089.109, sum=6178.218 (2)\", \"tab\": \"General information\", \"score\": \"3089.109090909091\"}", - "High School European History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Geography - # eval": "{\"description\": \"min=198, mean=198, max=198, sum=396 (2)\", \"tab\": \"General information\", \"score\": \"198.0\"}", - "High School Geography - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Geography - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # prompt tokens": "{\"description\": \"min=429.207, mean=429.207, max=429.207, sum=858.414 (2)\", \"tab\": \"General information\", \"score\": \"429.2070707070707\"}", - "High School Geography - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Government And Politics - # eval": "{\"description\": \"min=193, mean=193, max=193, sum=386 (2)\", \"tab\": \"General information\", \"score\": \"193.0\"}", - "High School Government And Politics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Government And Politics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # prompt tokens": "{\"description\": \"min=514.808, mean=514.808, max=514.808, sum=1029.617 (2)\", \"tab\": \"General information\", \"score\": \"514.8082901554404\"}", - "High School Government And Politics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Macroeconomics - # eval": "{\"description\": \"min=390, mean=390, max=390, sum=780 (2)\", \"tab\": \"General information\", \"score\": \"390.0\"}", - "High School Macroeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Macroeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # prompt tokens": "{\"description\": \"min=423.815, mean=423.815, max=423.815, sum=847.631 (2)\", \"tab\": \"General information\", \"score\": \"423.81538461538463\"}", - "High School Macroeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Mathematics - # eval": "{\"description\": \"min=270, mean=270, max=270, sum=540 (2)\", \"tab\": \"General information\", \"score\": \"270.0\"}", - "High School Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # prompt tokens": "{\"description\": \"min=584.13, mean=584.13, max=584.13, sum=1168.259 (2)\", \"tab\": \"General information\", \"score\": \"584.1296296296297\"}", - "High School Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Microeconomics - # eval": "{\"description\": \"min=238, mean=238, max=238, sum=476 (2)\", \"tab\": \"General information\", \"score\": \"238.0\"}", - "High School Microeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Microeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # prompt tokens": "{\"description\": \"min=443.345, mean=443.345, max=443.345, sum=886.689 (2)\", \"tab\": \"General information\", \"score\": \"443.34453781512605\"}", - "High School Microeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Physics - # eval": "{\"description\": \"min=151, mean=151, max=151, sum=302 (2)\", \"tab\": \"General information\", \"score\": \"151.0\"}", - "High School Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # prompt tokens": "{\"description\": \"min=622.775, mean=622.775, max=622.775, sum=1245.55 (2)\", \"tab\": \"General information\", \"score\": \"622.774834437086\"}", - "High School Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Psychology - # eval": "{\"description\": \"min=545, mean=545, max=545, sum=1090 (2)\", \"tab\": \"General information\", \"score\": \"545.0\"}", - "High School Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # prompt tokens": "{\"description\": \"min=558.873, mean=558.873, max=558.873, sum=1117.747 (2)\", \"tab\": \"General information\", \"score\": \"558.8733944954129\"}", - "High School Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Statistics - # eval": "{\"description\": \"min=216, mean=216, max=216, sum=432 (2)\", \"tab\": \"General information\", \"score\": \"216.0\"}", - "High School Statistics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Statistics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # prompt tokens": "{\"description\": \"min=913.644, mean=913.644, max=913.644, sum=1827.287 (2)\", \"tab\": \"General information\", \"score\": \"913.6435185185185\"}", - "High School Statistics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School US History - # eval": "{\"description\": \"min=204, mean=204, max=204, sum=408 (2)\", \"tab\": \"General information\", \"score\": \"204.0\"}", - "High School US History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School US History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # prompt tokens": "{\"description\": \"min=2477.446, mean=2477.446, max=2477.446, sum=4954.892 (2)\", \"tab\": \"General information\", \"score\": \"2477.4460784313724\"}", - "High School US History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School World History - # eval": "{\"description\": \"min=237, mean=237, max=237, sum=474 (2)\", \"tab\": \"General information\", \"score\": \"237.0\"}", - "High School World History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School World History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # prompt tokens": "{\"description\": \"min=1585.553, mean=1585.553, max=1585.553, sum=3171.105 (2)\", \"tab\": \"General information\", \"score\": \"1585.5527426160338\"}", - "High School World History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"high_school_world_history\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_high_school_world_history\"" - } - } - }, - { - "evaluation_name": "Human Sexuality", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Human Sexuality", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.847, - "details": { - "description": "min=0.847, mean=0.847, max=0.847, sum=1.695 (2)", - "tab": "Accuracy", - "Human Aging - Observed inference time (s)": "{\"description\": \"min=0.401, mean=0.401, max=0.401, sum=0.803 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4013588674399885\"}", - "Human Sexuality - Observed inference time (s)": "{\"description\": \"min=0.356, mean=0.356, max=0.356, sum=0.711 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3556434161790455\"}", - "Human Aging - # eval": "{\"description\": \"min=223, mean=223, max=223, sum=446 (2)\", \"tab\": \"General information\", \"score\": \"223.0\"}", - "Human Aging - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Aging - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Aging - # prompt tokens": "{\"description\": \"min=353.152, mean=353.152, max=353.152, sum=706.305 (2)\", \"tab\": \"General information\", \"score\": \"353.15246636771303\"}", - "Human Aging - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Human Sexuality - # eval": "{\"description\": \"min=131, mean=131, max=131, sum=262 (2)\", \"tab\": \"General information\", \"score\": \"131.0\"}", - "Human Sexuality - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Sexuality - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # prompt tokens": "{\"description\": \"min=394.748, mean=394.748, max=394.748, sum=789.496 (2)\", \"tab\": \"General information\", \"score\": \"394.7480916030534\"}", - "Human Sexuality - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"human_sexuality\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_human_sexuality\"" - } - } - }, - { - "evaluation_name": "International Law", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on International Law", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.868, - "details": { - "description": "min=0.868, mean=0.868, max=0.868, sum=1.736 (2)", - "tab": "Accuracy", - "International Law - Observed inference time (s)": "{\"description\": \"min=0.404, mean=0.404, max=0.404, sum=0.808 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.40404871081517746\"}", - "International Law - # eval": "{\"description\": \"min=121, mean=121, max=121, sum=242 (2)\", \"tab\": \"General information\", \"score\": \"121.0\"}", - "International Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "International Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "International Law - # prompt tokens": "{\"description\": \"min=720.182, mean=720.182, max=720.182, sum=1440.364 (2)\", \"tab\": \"General information\", \"score\": \"720.1818181818181\"}", - "International Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"international_law\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_international_law\"" - } - } - }, - { - "evaluation_name": "Logical Fallacies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Logical Fallacies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.81, - "details": { - "description": "min=0.81, mean=0.81, max=0.81, sum=1.62 (2)", - "tab": "Accuracy", - "Logical Fallacies - Observed inference time (s)": "{\"description\": \"min=0.409, mean=0.409, max=0.409, sum=0.818 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4088362228650988\"}", - "Logical Fallacies - # eval": "{\"description\": \"min=163, mean=163, max=163, sum=326 (2)\", \"tab\": \"General information\", \"score\": \"163.0\"}", - "Logical Fallacies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Logical Fallacies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Logical Fallacies - # prompt tokens": "{\"description\": \"min=486.779, mean=486.779, max=486.779, sum=973.558 (2)\", \"tab\": \"General information\", \"score\": \"486.77914110429447\"}", - "Logical Fallacies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"logical_fallacies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_logical_fallacies\"" - } - } - }, - { - "evaluation_name": "Machine Learning", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Machine Learning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.562, - "details": { - "description": "min=0.562, mean=0.562, max=0.562, sum=1.125 (2)", - "tab": "Accuracy", - "Machine Learning - Observed inference time (s)": "{\"description\": \"min=0.401, mean=0.401, max=0.401, sum=0.802 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.40122431090899874\"}", - "Machine Learning - # eval": "{\"description\": \"min=112, mean=112, max=112, sum=224 (2)\", \"tab\": \"General information\", \"score\": \"112.0\"}", - "Machine Learning - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Machine Learning - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Machine Learning - # prompt tokens": "{\"description\": \"min=734.83, mean=734.83, max=734.83, sum=1469.661 (2)\", \"tab\": \"General information\", \"score\": \"734.8303571428571\"}", - "Machine Learning - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"machine_learning\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_machine_learning\"" - } - } - }, - { - "evaluation_name": "Management", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Management", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.854, - "details": { - "description": "min=0.854, mean=0.854, max=0.854, sum=1.709 (2)", - "tab": "Accuracy", - "Management - Observed inference time (s)": "{\"description\": \"min=0.614, mean=0.614, max=0.614, sum=1.228 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6141544730917922\"}", - "Management - # eval": "{\"description\": \"min=103, mean=103, max=103, sum=206 (2)\", \"tab\": \"General information\", \"score\": \"103.0\"}", - "Management - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Management - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Management - # prompt tokens": "{\"description\": \"min=315.359, mean=315.359, max=315.359, sum=630.718 (2)\", \"tab\": \"General information\", \"score\": \"315.3592233009709\"}", - "Management - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"management\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_management\"" - } - } - }, - { - "evaluation_name": "Marketing", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Marketing", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.897, - "details": { - "description": "min=0.897, mean=0.897, max=0.897, sum=1.795 (2)", - "tab": "Accuracy", - "Marketing - Observed inference time (s)": "{\"description\": \"min=0.464, mean=0.464, max=0.464, sum=0.928 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.46382204895345575\"}", - "Marketing - # eval": "{\"description\": \"min=234, mean=234, max=234, sum=468 (2)\", \"tab\": \"General information\", \"score\": \"234.0\"}", - "Marketing - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Marketing - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Marketing - # prompt tokens": "{\"description\": \"min=463.423, mean=463.423, max=463.423, sum=926.846 (2)\", \"tab\": \"General information\", \"score\": \"463.4230769230769\"}", - "Marketing - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"marketing\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_marketing\"" - } - } - }, - { - "evaluation_name": "Medical Genetics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Medical Genetics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.74, - "details": { - "description": "min=0.74, mean=0.74, max=0.74, sum=1.48 (2)", - "tab": "Accuracy", - "Medical Genetics - Observed inference time (s)": "{\"description\": \"min=0.433, mean=0.433, max=0.433, sum=0.867 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4333249735832214\"}", - "Medical Genetics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Medical Genetics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Medical Genetics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Medical Genetics - # prompt tokens": "{\"description\": \"min=405.71, mean=405.71, max=405.71, sum=811.42 (2)\", \"tab\": \"General information\", \"score\": \"405.71\"}", - "Medical Genetics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"medical_genetics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_medical_genetics\"" - } - } - }, - { - "evaluation_name": "Miscellaneous", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Miscellaneous", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9, - "details": { - "description": "min=0.9, mean=0.9, max=0.9, sum=1.801 (2)", - "tab": "Accuracy", - "Miscellaneous - Observed inference time (s)": "{\"description\": \"min=0.477, mean=0.477, max=0.477, sum=0.955 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.477321812323988\"}", - "Miscellaneous - # eval": "{\"description\": \"min=783, mean=783, max=783, sum=1566 (2)\", \"tab\": \"General information\", \"score\": \"783.0\"}", - "Miscellaneous - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Miscellaneous - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Miscellaneous - # prompt tokens": "{\"description\": \"min=348.519, mean=348.519, max=348.519, sum=697.037 (2)\", \"tab\": \"General information\", \"score\": \"348.51851851851853\"}", - "Miscellaneous - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"miscellaneous\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_miscellaneous\"" - } - } - }, - { - "evaluation_name": "Moral Scenarios", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Moral Scenarios", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.579, - "details": { - "description": "min=0.579, mean=0.579, max=0.579, sum=1.158 (2)", - "tab": "Accuracy", - "Moral Disputes - Observed inference time (s)": "{\"description\": \"min=0.497, mean=0.497, max=0.497, sum=0.995 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4974138419752176\"}", - "Moral Scenarios - Observed inference time (s)": "{\"description\": \"min=0.451, mean=0.451, max=0.451, sum=0.902 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.45121243466212096\"}", - "Moral Disputes - # eval": "{\"description\": \"min=346, mean=346, max=346, sum=692 (2)\", \"tab\": \"General information\", \"score\": \"346.0\"}", - "Moral Disputes - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Disputes - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Disputes - # prompt tokens": "{\"description\": \"min=540.038, mean=540.038, max=540.038, sum=1080.075 (2)\", \"tab\": \"General information\", \"score\": \"540.0375722543353\"}", - "Moral Disputes - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Moral Scenarios - # eval": "{\"description\": \"min=895, mean=895, max=895, sum=1790 (2)\", \"tab\": \"General information\", \"score\": \"895.0\"}", - "Moral Scenarios - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Scenarios - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # prompt tokens": "{\"description\": \"min=745.516, mean=745.516, max=745.516, sum=1491.032 (2)\", \"tab\": \"General information\", \"score\": \"745.5162011173185\"}", - "Moral Scenarios - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"moral_scenarios\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_moral_scenarios\"" - } - } - }, - { - "evaluation_name": "Nutrition", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Nutrition", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.791, - "details": { - "description": "min=0.791, mean=0.791, max=0.791, sum=1.582 (2)", - "tab": "Accuracy", - "Nutrition - Observed inference time (s)": "{\"description\": \"min=0.463, mean=0.463, max=0.463, sum=0.927 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.46336324308432786\"}", - "Nutrition - # eval": "{\"description\": \"min=306, mean=306, max=306, sum=612 (2)\", \"tab\": \"General information\", \"score\": \"306.0\"}", - "Nutrition - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Nutrition - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Nutrition - # prompt tokens": "{\"description\": \"min=680.69, mean=680.69, max=680.69, sum=1361.379 (2)\", \"tab\": \"General information\", \"score\": \"680.6895424836601\"}", - "Nutrition - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"nutrition\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_nutrition\"" - } - } - }, - { - "evaluation_name": "Prehistory", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Prehistory", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.904, - "details": { - "description": "min=0.904, mean=0.904, max=0.904, sum=1.809 (2)", - "tab": "Accuracy", - "Prehistory - Observed inference time (s)": "{\"description\": \"min=0.393, mean=0.393, max=0.393, sum=0.786 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3928193273367705\"}", - "Prehistory - # eval": "{\"description\": \"min=324, mean=324, max=324, sum=648 (2)\", \"tab\": \"General information\", \"score\": \"324.0\"}", - "Prehistory - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Prehistory - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Prehistory - # prompt tokens": "{\"description\": \"min=602.145, mean=602.145, max=602.145, sum=1204.29 (2)\", \"tab\": \"General information\", \"score\": \"602.145061728395\"}", - "Prehistory - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"prehistory\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_prehistory\"" - } - } - }, - { - "evaluation_name": "Public Relations", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Public Relations", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.709, - "details": { - "description": "min=0.709, mean=0.709, max=0.709, sum=1.418 (2)", - "tab": "Accuracy", - "Public Relations - Observed inference time (s)": "{\"description\": \"min=0.535, mean=0.535, max=0.535, sum=1.069 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.534747780453075\"}", - "Public Relations - # eval": "{\"description\": \"min=110, mean=110, max=110, sum=220 (2)\", \"tab\": \"General information\", \"score\": \"110.0\"}", - "Public Relations - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Public Relations - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Public Relations - # prompt tokens": "{\"description\": \"min=462.036, mean=462.036, max=462.036, sum=924.073 (2)\", \"tab\": \"General information\", \"score\": \"462.03636363636366\"}", - "Public Relations - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"public_relations\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_public_relations\"" - } - } - }, - { - "evaluation_name": "Security Studies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Security Studies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.824, - "details": { - "description": "min=0.824, mean=0.824, max=0.824, sum=1.649 (2)", - "tab": "Accuracy", - "Security Studies - Observed inference time (s)": "{\"description\": \"min=0.446, mean=0.446, max=0.446, sum=0.891 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.44565339964263295\"}", - "Security Studies - # eval": "{\"description\": \"min=245, mean=245, max=245, sum=490 (2)\", \"tab\": \"General information\", \"score\": \"245.0\"}", - "Security Studies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Security Studies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Security Studies - # prompt tokens": "{\"description\": \"min=1315.865, mean=1315.865, max=1315.865, sum=2631.731 (2)\", \"tab\": \"General information\", \"score\": \"1315.865306122449\"}", - "Security Studies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"security_studies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_security_studies\"" - } - } - }, - { - "evaluation_name": "Sociology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Sociology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.93, - "details": { - "description": "min=0.93, mean=0.93, max=0.93, sum=1.861 (2)", - "tab": "Accuracy", - "Sociology - Observed inference time (s)": "{\"description\": \"min=0.442, mean=0.442, max=0.442, sum=0.884 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.44217372296461416\"}", - "Sociology - # eval": "{\"description\": \"min=201, mean=201, max=201, sum=402 (2)\", \"tab\": \"General information\", \"score\": \"201.0\"}", - "Sociology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Sociology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Sociology - # prompt tokens": "{\"description\": \"min=487.95, mean=487.95, max=487.95, sum=975.9 (2)\", \"tab\": \"General information\", \"score\": \"487.9502487562189\"}", - "Sociology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"sociology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_sociology\"" - } - } - }, - { - "evaluation_name": "Virology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Virology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.554, - "details": { - "description": "min=0.554, mean=0.554, max=0.554, sum=1.108 (2)", - "tab": "Accuracy", - "Virology - Observed inference time (s)": "{\"description\": \"min=0.436, mean=0.436, max=0.436, sum=0.871 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.435666641557073\"}", - "Virology - # eval": "{\"description\": \"min=166, mean=166, max=166, sum=332 (2)\", \"tab\": \"General information\", \"score\": \"166.0\"}", - "Virology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Virology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Virology - # prompt tokens": "{\"description\": \"min=395.349, mean=395.349, max=395.349, sum=790.699 (2)\", \"tab\": \"General information\", \"score\": \"395.34939759036143\"}", - "Virology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"virology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_virology\"" - } - } - }, - { - "evaluation_name": "World Religions", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on World Religions", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.883, - "details": { - "description": "min=0.883, mean=0.883, max=0.883, sum=1.766 (2)", - "tab": "Accuracy", - "World Religions - Observed inference time (s)": "{\"description\": \"min=0.411, mean=0.411, max=0.411, sum=0.821 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4106302637802927\"}", - "World Religions - # eval": "{\"description\": \"min=171, mean=171, max=171, sum=342 (2)\", \"tab\": \"General information\", \"score\": \"171.0\"}", - "World Religions - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "World Religions - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "World Religions - # prompt tokens": "{\"description\": \"min=308.924, mean=308.924, max=308.924, sum=617.848 (2)\", \"tab\": \"General information\", \"score\": \"308.92397660818716\"}", - "World Religions - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"world_religions\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_world_religions\"" - } - } - }, - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.464, - "details": { - "description": "", - "tab": "Efficiency" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/mistralai_mistral-large-2407.json b/data/models/mistralai_mistral-large-2407.json deleted file mode 100644 index 97b1f2aa6e29fc47bb3fb448d76b98d66ea47945..0000000000000000000000000000000000000000 --- a/data/models/mistralai_mistral-large-2407.json +++ /dev/null @@ -1,1903 +0,0 @@ -{ - "model_info": { - "name": "Mistral Large 2 2407", - "id": "mistralai/mistral-large-2407", - "developer": "mistralai", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_lite/mistralai_mistral-large-2407/1774096306.427425", - "retrieved_timestamp": "1774096306.427425", - "source_metadata": { - "source_name": "helm_lite", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_lite", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_lite", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.744, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.4191385767790262\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.779, - "details": { - "description": "min=0.779, mean=0.779, max=0.779, sum=0.779 (1)", - "tab": "Accuracy", - "NarrativeQA - Observed inference time (s)": "{\"description\": \"min=0.728, mean=0.728, max=0.728, sum=0.728 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.7276979574015443\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=3914.33, mean=3914.33, max=3914.33, sum=3914.33 (1)\", \"tab\": \"General information\", \"score\": \"3914.3295774647886\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=6.2, mean=6.2, max=6.2, sum=6.2 (1)\", \"tab\": \"General information\", \"score\": \"6.2\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (closed-book)", - "source_data": { - "dataset_name": "NaturalQuestions (closed-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (closed-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.453, - "details": { - "description": "min=0.453, mean=0.453, max=0.453, sum=0.453 (1)", - "tab": "Accuracy", - "NaturalQuestions (open-book) - Observed inference time (s)": "{\"description\": \"min=0.757, mean=0.757, max=0.757, sum=0.757 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.7573216142654419\"}", - "NaturalQuestions (closed-book) - Observed inference time (s)": "{\"description\": \"min=0.527, mean=0.527, max=0.527, sum=0.527 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.5273597676753998\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=2488.79, mean=2488.79, max=2488.79, sum=2488.79 (1)\", \"tab\": \"General information\", \"score\": \"2488.79\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=7.994, mean=7.994, max=7.994, sum=7.994 (1)\", \"tab\": \"General information\", \"score\": \"7.994\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=162.069, mean=162.069, max=162.069, sum=162.069 (1)\", \"tab\": \"General information\", \"score\": \"162.069\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=6.496, mean=6.496, max=6.496, sum=6.496 (1)\", \"tab\": \"General information\", \"score\": \"6.496\"}" - } - }, - "generation_config": { - "additional_details": { - "mode": "\"closedbook\"" - } - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.932, - "details": { - "description": "min=0.932, mean=0.932, max=0.932, sum=0.932 (1)", - "tab": "Accuracy", - "OpenbookQA - Observed inference time (s)": "{\"description\": \"min=0.891, mean=0.891, max=0.891, sum=0.891 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.8910596170425416\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=290.15, mean=290.15, max=290.15, sum=290.15 (1)\", \"tab\": \"General information\", \"score\": \"290.15\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "dataset": "\"openbookqa\"", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.725, - "details": { - "description": "min=0.52, mean=0.725, max=0.9, sum=3.623 (5)", - "tab": "Accuracy", - "MMLU - Observed inference time (s)": "{\"description\": \"min=0.684, mean=0.789, max=0.933, sum=3.943 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.7886472435834114\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=412.44, mean=533.091, max=697.175, sum=2665.455 (5)\", \"tab\": \"General information\", \"score\": \"533.0910877192983\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"college_chemistry\", \"computer_security\", \"econometrics\", \"us_foreign_policy\"]", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MATH", - "source_data": { - "dataset_name": "MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Equivalent (CoT) on MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.677, - "details": { - "description": "min=0.342, mean=0.677, max=0.881, sum=4.737 (7)", - "tab": "Accuracy", - "MATH - Observed inference time (s)": "{\"description\": \"min=4.359, mean=5.441, max=6.464, sum=38.087 (7)\", \"tab\": \"Efficiency\", \"score\": \"5.441067432619708\"}", - "MATH - # eval": "{\"description\": \"min=30, mean=62.429, max=135, sum=437 (7)\", \"tab\": \"General information\", \"score\": \"62.42857142857143\"}", - "MATH - # train": "{\"description\": \"min=8, mean=8, max=8, sum=56 (7)\", \"tab\": \"General information\", \"score\": \"8.0\"}", - "MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MATH - # prompt tokens": "{\"description\": \"min=992.615, mean=1456.266, max=2503.962, sum=10193.865 (7)\", \"tab\": \"General information\", \"score\": \"1456.2664139976257\"}", - "MATH - # output tokens": "{\"description\": \"min=129.395, mean=180.319, max=220.298, sum=1262.231 (7)\", \"tab\": \"General information\", \"score\": \"180.3187090913529\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"algebra\", \"counting_and_probability\", \"geometry\", \"intermediate_algebra\", \"number_theory\", \"prealgebra\", \"precalculus\"]", - "level": "\"1\"", - "use_official_examples": "\"False\"", - "use_chain_of_thought": "\"True\"" - } - } - }, - { - "evaluation_name": "GSM8K", - "source_data": { - "dataset_name": "GSM8K", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on GSM8K", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.912, - "details": { - "description": "min=0.912, mean=0.912, max=0.912, sum=0.912 (1)", - "tab": "Accuracy", - "GSM8K - Observed inference time (s)": "{\"description\": \"min=5.431, mean=5.431, max=5.431, sum=5.431 (1)\", \"tab\": \"Efficiency\", \"score\": \"5.431343378543854\"}", - "GSM8K - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "GSM8K - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "GSM8K - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GSM8K - # prompt tokens": "{\"description\": \"min=1187.268, mean=1187.268, max=1187.268, sum=1187.268 (1)\", \"tab\": \"General information\", \"score\": \"1187.268\"}", - "GSM8K - # output tokens": "{\"description\": \"min=205.748, mean=205.748, max=205.748, sum=205.748 (1)\", \"tab\": \"General information\", \"score\": \"205.748\"}" - } - }, - "generation_config": { - "additional_details": { - "stop": "\"none\"" - } - } - }, - { - "evaluation_name": "LegalBench", - "source_data": { - "dataset_name": "LegalBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on LegalBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.646, - "details": { - "description": "min=0.229, mean=0.646, max=1, sum=3.23 (5)", - "tab": "Accuracy", - "LegalBench - Observed inference time (s)": "{\"description\": \"min=0.485, mean=0.797, max=0.986, sum=3.987 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.7974768901406878\"}", - "LegalBench - # eval": "{\"description\": \"min=95, mean=409.4, max=1000, sum=2047 (5)\", \"tab\": \"General information\", \"score\": \"409.4\"}", - "LegalBench - # train": "{\"description\": \"min=4, mean=4.8, max=5, sum=24 (5)\", \"tab\": \"General information\", \"score\": \"4.8\"}", - "LegalBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "LegalBench - # prompt tokens": "{\"description\": \"min=227.453, mean=1741.348, max=7215.488, sum=8706.741 (5)\", \"tab\": \"General information\", \"score\": \"1741.3482458432961\"}", - "LegalBench - # output tokens": "{\"description\": \"min=1, mean=3.484, max=8.469, sum=17.42 (5)\", \"tab\": \"General information\", \"score\": \"3.484006654237774\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"abercrombie\", \"corporate_lobbying\", \"function_of_decision_section\", \"international_citizenship_questions\", \"proa\"]" - } - } - }, - { - "evaluation_name": "MedQA", - "source_data": { - "dataset_name": "MedQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MedQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.775, - "details": { - "description": "min=0.775, mean=0.775, max=0.775, sum=0.775 (1)", - "tab": "Accuracy", - "MedQA - Observed inference time (s)": "{\"description\": \"min=0.446, mean=0.446, max=0.446, sum=0.446 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.4456319799480097\"}", - "MedQA - # eval": "{\"description\": \"min=503, mean=503, max=503, sum=503 (1)\", \"tab\": \"General information\", \"score\": \"503.0\"}", - "MedQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MedQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MedQA - # prompt tokens": "{\"description\": \"min=1203.093, mean=1203.093, max=1203.093, sum=1203.093 (1)\", \"tab\": \"General information\", \"score\": \"1203.0934393638172\"}", - "MedQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WMT 2014", - "source_data": { - "dataset_name": "WMT 2014", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "BLEU-4 on WMT 2014", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.192, - "details": { - "description": "min=0.14, mean=0.192, max=0.231, sum=0.962 (5)", - "tab": "Accuracy", - "WMT 2014 - Observed inference time (s)": "{\"description\": \"min=1.075, mean=1.269, max=1.402, sum=6.343 (5)\", \"tab\": \"Efficiency\", \"score\": \"1.2686868536542282\"}", - "WMT 2014 - # eval": "{\"description\": \"min=503, mean=568.8, max=832, sum=2844 (5)\", \"tab\": \"General information\", \"score\": \"568.8\"}", - "WMT 2014 - # train": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "WMT 2014 - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WMT 2014 - # prompt tokens": "{\"description\": \"min=139.306, mean=153.433, max=172.018, sum=767.166 (5)\", \"tab\": \"General information\", \"score\": \"153.43317355482492\"}", - "WMT 2014 - # output tokens": "{\"description\": \"min=29.153, mean=30.306, max=33.358, sum=151.531 (5)\", \"tab\": \"General information\", \"score\": \"30.30625095580364\"}" - } - }, - "generation_config": { - "additional_details": { - "language_pair": "[\"cs-en\", \"de-en\", \"fr-en\", \"hi-en\", \"ru-en\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_mmlu/mistralai_mistral-large-2407/1774096312.00548", - "retrieved_timestamp": "1774096312.00548", - "source_metadata": { - "source_name": "helm_mmlu", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_mmlu", - "evaluation_results": [ - { - "evaluation_name": "MMLU All Subjects", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU All Subjects", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8, - "details": { - "description": "min=0.37, mean=0.8, max=0.969, sum=91.197 (114)", - "tab": "Accuracy", - "MMLU All Subjects - Observed inference time (s)": "{\"description\": \"min=0.672, mean=0.798, max=1.025, sum=90.977 (114)\", \"tab\": \"Efficiency\", \"score\": \"0.798047748433812\"}", - "MMLU All Subjects - # eval": "{\"description\": \"min=100, mean=246.351, max=1534, sum=28084 (114)\", \"tab\": \"General information\", \"score\": \"246.35087719298247\"}", - "MMLU All Subjects - # train": "{\"description\": \"min=5, mean=5, max=5, sum=570 (114)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU All Subjects - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU All Subjects - # prompt tokens": "{\"description\": \"min=318.924, mean=706.273, max=3099.109, sum=80515.178 (114)\", \"tab\": \"General information\", \"score\": \"706.2734899593811\"}", - "MMLU All Subjects - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=114 (114)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_name": "Abstract Algebra", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Abstract Algebra", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7, - "details": { - "description": "min=0.7, mean=0.7, max=0.7, sum=1.4 (2)", - "tab": "Accuracy", - "Abstract Algebra - Observed inference time (s)": "{\"description\": \"min=0.732, mean=0.732, max=0.732, sum=1.464 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7317730689048767\"}", - "Abstract Algebra - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Abstract Algebra - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Abstract Algebra - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Abstract Algebra - # prompt tokens": "{\"description\": \"min=412.44, mean=412.44, max=412.44, sum=824.88 (2)\", \"tab\": \"General information\", \"score\": \"412.44\"}", - "Abstract Algebra - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"abstract_algebra\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_abstract_algebra\"" - } - } - }, - { - "evaluation_name": "Anatomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Anatomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.785, - "details": { - "description": "min=0.785, mean=0.785, max=0.785, sum=1.57 (2)", - "tab": "Accuracy", - "Anatomy - Observed inference time (s)": "{\"description\": \"min=0.808, mean=0.808, max=0.808, sum=1.616 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.807829690862585\"}", - "Anatomy - # eval": "{\"description\": \"min=135, mean=135, max=135, sum=270 (2)\", \"tab\": \"General information\", \"score\": \"135.0\"}", - "Anatomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Anatomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Anatomy - # prompt tokens": "{\"description\": \"min=417.089, mean=417.089, max=417.089, sum=834.178 (2)\", \"tab\": \"General information\", \"score\": \"417.0888888888889\"}", - "Anatomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"anatomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_anatomy\"" - } - } - }, - { - "evaluation_name": "College Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on College Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.559, - "details": { - "description": "min=0.559, mean=0.559, max=0.559, sum=1.118 (2)", - "tab": "Accuracy", - "College Chemistry - Observed inference time (s)": "{\"description\": \"min=0.838, mean=0.838, max=0.838, sum=1.676 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8380094933509826\"}", - "College Biology - Observed inference time (s)": "{\"description\": \"min=0.768, mean=0.768, max=0.768, sum=1.535 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.76766570409139\"}", - "College Computer Science - Observed inference time (s)": "{\"description\": \"min=0.853, mean=0.853, max=0.853, sum=1.706 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8529829049110412\"}", - "College Mathematics - Observed inference time (s)": "{\"description\": \"min=0.771, mean=0.771, max=0.771, sum=1.542 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7712302732467652\"}", - "College Medicine - Observed inference time (s)": "{\"description\": \"min=0.672, mean=0.672, max=0.672, sum=1.344 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6721915785287846\"}", - "College Physics - Observed inference time (s)": "{\"description\": \"min=0.674, mean=0.674, max=0.674, sum=1.347 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6735490116418577\"}", - "College Chemistry - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Chemistry - # prompt tokens": "{\"description\": \"min=637.71, mean=637.71, max=637.71, sum=1275.42 (2)\", \"tab\": \"General information\", \"score\": \"637.71\"}", - "College Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Biology - # eval": "{\"description\": \"min=144, mean=144, max=144, sum=288 (2)\", \"tab\": \"General information\", \"score\": \"144.0\"}", - "College Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # prompt tokens": "{\"description\": \"min=560.799, mean=560.799, max=560.799, sum=1121.597 (2)\", \"tab\": \"General information\", \"score\": \"560.7986111111111\"}", - "College Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # prompt tokens": "{\"description\": \"min=912.17, mean=912.17, max=912.17, sum=1824.34 (2)\", \"tab\": \"General information\", \"score\": \"912.17\"}", - "College Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Mathematics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # prompt tokens": "{\"description\": \"min=668.31, mean=668.31, max=668.31, sum=1336.62 (2)\", \"tab\": \"General information\", \"score\": \"668.31\"}", - "College Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Medicine - # eval": "{\"description\": \"min=173, mean=173, max=173, sum=346 (2)\", \"tab\": \"General information\", \"score\": \"173.0\"}", - "College Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # prompt tokens": "{\"description\": \"min=602.41, mean=602.41, max=602.41, sum=1204.821 (2)\", \"tab\": \"General information\", \"score\": \"602.4104046242775\"}", - "College Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Physics - # eval": "{\"description\": \"min=102, mean=102, max=102, sum=204 (2)\", \"tab\": \"General information\", \"score\": \"102.0\"}", - "College Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # prompt tokens": "{\"description\": \"min=561.029, mean=561.029, max=561.029, sum=1122.059 (2)\", \"tab\": \"General information\", \"score\": \"561.0294117647059\"}", - "College Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"college_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_college_physics\"" - } - } - }, - { - "evaluation_name": "Computer Security", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Computer Security", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.81, - "details": { - "description": "min=0.81, mean=0.81, max=0.81, sum=1.62 (2)", - "tab": "Accuracy", - "Computer Security - Observed inference time (s)": "{\"description\": \"min=0.933, mean=0.933, max=0.933, sum=1.866 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9331179332733154\"}", - "Computer Security - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Computer Security - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Computer Security - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Computer Security - # prompt tokens": "{\"description\": \"min=434.94, mean=434.94, max=434.94, sum=869.88 (2)\", \"tab\": \"General information\", \"score\": \"434.94\"}", - "Computer Security - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"computer_security\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_computer_security\"" - } - } - }, - { - "evaluation_name": "Econometrics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Econometrics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.693, - "details": { - "description": "min=0.693, mean=0.693, max=0.693, sum=1.386 (2)", - "tab": "Accuracy", - "Econometrics - Observed inference time (s)": "{\"description\": \"min=0.684, mean=0.684, max=0.684, sum=1.368 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6842389587770429\"}", - "Econometrics - # eval": "{\"description\": \"min=114, mean=114, max=114, sum=228 (2)\", \"tab\": \"General information\", \"score\": \"114.0\"}", - "Econometrics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Econometrics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Econometrics - # prompt tokens": "{\"description\": \"min=697.175, mean=697.175, max=697.175, sum=1394.351 (2)\", \"tab\": \"General information\", \"score\": \"697.1754385964912\"}", - "Econometrics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"econometrics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_econometrics\"" - } - } - }, - { - "evaluation_name": "Global Facts", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Global Facts", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.56, - "details": { - "description": "min=0.56, mean=0.56, max=0.56, sum=1.12 (2)", - "tab": "Accuracy", - "Global Facts - Observed inference time (s)": "{\"description\": \"min=0.745, mean=0.745, max=0.745, sum=1.489 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.744694242477417\"}", - "Global Facts - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Global Facts - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Global Facts - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Global Facts - # prompt tokens": "{\"description\": \"min=493.47, mean=493.47, max=493.47, sum=986.94 (2)\", \"tab\": \"General information\", \"score\": \"493.47\"}", - "Global Facts - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"global_facts\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_global_facts\"" - } - } - }, - { - "evaluation_name": "Jurisprudence", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Jurisprudence", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.861, - "details": { - "description": "min=0.861, mean=0.861, max=0.861, sum=1.722 (2)", - "tab": "Accuracy", - "Jurisprudence - Observed inference time (s)": "{\"description\": \"min=0.751, mean=0.751, max=0.751, sum=1.503 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.751495877901713\"}", - "Jurisprudence - # eval": "{\"description\": \"min=108, mean=108, max=108, sum=216 (2)\", \"tab\": \"General information\", \"score\": \"108.0\"}", - "Jurisprudence - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Jurisprudence - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Jurisprudence - # prompt tokens": "{\"description\": \"min=461.093, mean=461.093, max=461.093, sum=922.185 (2)\", \"tab\": \"General information\", \"score\": \"461.0925925925926\"}", - "Jurisprudence - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"jurisprudence\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_jurisprudence\"" - } - } - }, - { - "evaluation_name": "Philosophy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Philosophy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.826, - "details": { - "description": "min=0.826, mean=0.826, max=0.826, sum=1.653 (2)", - "tab": "Accuracy", - "Philosophy - Observed inference time (s)": "{\"description\": \"min=0.804, mean=0.804, max=0.804, sum=1.609 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8043544453439988\"}", - "Philosophy - # eval": "{\"description\": \"min=311, mean=311, max=311, sum=622 (2)\", \"tab\": \"General information\", \"score\": \"311.0\"}", - "Philosophy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Philosophy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Philosophy - # prompt tokens": "{\"description\": \"min=383.82, mean=383.82, max=383.82, sum=767.64 (2)\", \"tab\": \"General information\", \"score\": \"383.81993569131834\"}", - "Philosophy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"philosophy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_philosophy\"" - } - } - }, - { - "evaluation_name": "Professional Psychology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Professional Psychology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.861, - "details": { - "description": "min=0.861, mean=0.861, max=0.861, sum=1.722 (2)", - "tab": "Accuracy", - "Professional Medicine - Observed inference time (s)": "{\"description\": \"min=0.741, mean=0.741, max=0.741, sum=1.481 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7406316355747335\"}", - "Professional Accounting - Observed inference time (s)": "{\"description\": \"min=0.807, mean=0.807, max=0.807, sum=1.615 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8074929325293142\"}", - "Professional Law - Observed inference time (s)": "{\"description\": \"min=0.774, mean=0.774, max=0.774, sum=1.548 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7742255473851847\"}", - "Professional Psychology - Observed inference time (s)": "{\"description\": \"min=0.923, mean=0.923, max=0.923, sum=1.846 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9228381756084417\"}", - "Professional Medicine - # eval": "{\"description\": \"min=272, mean=272, max=272, sum=544 (2)\", \"tab\": \"General information\", \"score\": \"272.0\"}", - "Professional Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Medicine - # prompt tokens": "{\"description\": \"min=1289.143, mean=1289.143, max=1289.143, sum=2578.287 (2)\", \"tab\": \"General information\", \"score\": \"1289.1433823529412\"}", - "Professional Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Accounting - # eval": "{\"description\": \"min=282, mean=282, max=282, sum=564 (2)\", \"tab\": \"General information\", \"score\": \"282.0\"}", - "Professional Accounting - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Accounting - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # prompt tokens": "{\"description\": \"min=806.496, mean=806.496, max=806.496, sum=1612.993 (2)\", \"tab\": \"General information\", \"score\": \"806.4964539007092\"}", - "Professional Accounting - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Law - # eval": "{\"description\": \"min=1534, mean=1534, max=1534, sum=3068 (2)\", \"tab\": \"General information\", \"score\": \"1534.0\"}", - "Professional Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # prompt tokens": "{\"description\": \"min=1859.711, mean=1859.711, max=1859.711, sum=3719.421 (2)\", \"tab\": \"General information\", \"score\": \"1859.7105606258149\"}", - "Professional Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Psychology - # eval": "{\"description\": \"min=612, mean=612, max=612, sum=1224 (2)\", \"tab\": \"General information\", \"score\": \"612.0\"}", - "Professional Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # prompt tokens": "{\"description\": \"min=655.278, mean=655.278, max=655.278, sum=1310.556 (2)\", \"tab\": \"General information\", \"score\": \"655.2777777777778\"}", - "Professional Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"professional_psychology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_professional_psychology\"" - } - } - }, - { - "evaluation_name": "Us Foreign Policy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Us Foreign Policy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9, - "details": { - "description": "min=0.9, mean=0.9, max=0.9, sum=1.8 (2)", - "tab": "Accuracy", - "Us Foreign Policy - Observed inference time (s)": "{\"description\": \"min=0.756, mean=0.756, max=0.756, sum=1.512 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7560967636108399\"}", - "Us Foreign Policy - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Us Foreign Policy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Us Foreign Policy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Us Foreign Policy - # prompt tokens": "{\"description\": \"min=483.19, mean=483.19, max=483.19, sum=966.38 (2)\", \"tab\": \"General information\", \"score\": \"483.19\"}", - "Us Foreign Policy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"us_foreign_policy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_us_foreign_policy\"" - } - } - }, - { - "evaluation_name": "Astronomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Astronomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.921, - "details": { - "description": "min=0.921, mean=0.921, max=0.921, sum=1.842 (2)", - "tab": "Accuracy", - "Astronomy - Observed inference time (s)": "{\"description\": \"min=1.025, mean=1.025, max=1.025, sum=2.049 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.0245175393004167\"}", - "Astronomy - # eval": "{\"description\": \"min=152, mean=152, max=152, sum=304 (2)\", \"tab\": \"General information\", \"score\": \"152.0\"}", - "Astronomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Astronomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Astronomy - # prompt tokens": "{\"description\": \"min=675.987, mean=675.987, max=675.987, sum=1351.974 (2)\", \"tab\": \"General information\", \"score\": \"675.9868421052631\"}", - "Astronomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"astronomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_astronomy\"" - } - } - }, - { - "evaluation_name": "Business Ethics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Business Ethics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.79, - "details": { - "description": "min=0.79, mean=0.79, max=0.79, sum=1.58 (2)", - "tab": "Accuracy", - "Business Ethics - Observed inference time (s)": "{\"description\": \"min=0.923, mean=0.923, max=0.923, sum=1.846 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9228822708129882\"}", - "Business Ethics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Business Ethics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Business Ethics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Business Ethics - # prompt tokens": "{\"description\": \"min=654.6, mean=654.6, max=654.6, sum=1309.2 (2)\", \"tab\": \"General information\", \"score\": \"654.6\"}", - "Business Ethics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"business_ethics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_business_ethics\"" - } - } - }, - { - "evaluation_name": "Clinical Knowledge", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Clinical Knowledge", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.864, - "details": { - "description": "min=0.864, mean=0.864, max=0.864, sum=1.728 (2)", - "tab": "Accuracy", - "Clinical Knowledge - Observed inference time (s)": "{\"description\": \"min=0.789, mean=0.789, max=0.789, sum=1.578 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7888300931678628\"}", - "Clinical Knowledge - # eval": "{\"description\": \"min=265, mean=265, max=265, sum=530 (2)\", \"tab\": \"General information\", \"score\": \"265.0\"}", - "Clinical Knowledge - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Clinical Knowledge - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Clinical Knowledge - # prompt tokens": "{\"description\": \"min=497.174, mean=497.174, max=497.174, sum=994.347 (2)\", \"tab\": \"General information\", \"score\": \"497.1735849056604\"}", - "Clinical Knowledge - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"clinical_knowledge\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_clinical_knowledge\"" - } - } - }, - { - "evaluation_name": "Conceptual Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Conceptual Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.864, - "details": { - "description": "min=0.864, mean=0.864, max=0.864, sum=1.728 (2)", - "tab": "Accuracy", - "Conceptual Physics - Observed inference time (s)": "{\"description\": \"min=0.821, mean=0.821, max=0.821, sum=1.643 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8212997264050422\"}", - "Conceptual Physics - # eval": "{\"description\": \"min=235, mean=235, max=235, sum=470 (2)\", \"tab\": \"General information\", \"score\": \"235.0\"}", - "Conceptual Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Conceptual Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Conceptual Physics - # prompt tokens": "{\"description\": \"min=344.285, mean=344.285, max=344.285, sum=688.57 (2)\", \"tab\": \"General information\", \"score\": \"344.2851063829787\"}", - "Conceptual Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"conceptual_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_conceptual_physics\"" - } - } - }, - { - "evaluation_name": "Electrical Engineering", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Electrical Engineering", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.793, - "details": { - "description": "min=0.793, mean=0.793, max=0.793, sum=1.586 (2)", - "tab": "Accuracy", - "Electrical Engineering - Observed inference time (s)": "{\"description\": \"min=0.702, mean=0.702, max=0.702, sum=1.404 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.701846879104088\"}", - "Electrical Engineering - # eval": "{\"description\": \"min=145, mean=145, max=145, sum=290 (2)\", \"tab\": \"General information\", \"score\": \"145.0\"}", - "Electrical Engineering - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Electrical Engineering - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Electrical Engineering - # prompt tokens": "{\"description\": \"min=511.379, mean=511.379, max=511.379, sum=1022.759 (2)\", \"tab\": \"General information\", \"score\": \"511.37931034482756\"}", - "Electrical Engineering - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"electrical_engineering\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_electrical_engineering\"" - } - } - }, - { - "evaluation_name": "Elementary Mathematics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Elementary Mathematics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.799, - "details": { - "description": "min=0.799, mean=0.799, max=0.799, sum=1.598 (2)", - "tab": "Accuracy", - "Elementary Mathematics - Observed inference time (s)": "{\"description\": \"min=0.788, mean=0.788, max=0.788, sum=1.577 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7884082762652604\"}", - "Elementary Mathematics - # eval": "{\"description\": \"min=378, mean=378, max=378, sum=756 (2)\", \"tab\": \"General information\", \"score\": \"378.0\"}", - "Elementary Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Elementary Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Elementary Mathematics - # prompt tokens": "{\"description\": \"min=623.386, mean=623.386, max=623.386, sum=1246.772 (2)\", \"tab\": \"General information\", \"score\": \"623.3862433862433\"}", - "Elementary Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"elementary_mathematics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_elementary_mathematics\"" - } - } - }, - { - "evaluation_name": "Formal Logic", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Formal Logic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.579, - "details": { - "description": "min=0.579, mean=0.579, max=0.579, sum=1.159 (2)", - "tab": "Accuracy", - "Formal Logic - Observed inference time (s)": "{\"description\": \"min=0.763, mean=0.763, max=0.763, sum=1.526 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7629275567947872\"}", - "Formal Logic - # eval": "{\"description\": \"min=126, mean=126, max=126, sum=252 (2)\", \"tab\": \"General information\", \"score\": \"126.0\"}", - "Formal Logic - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Formal Logic - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Formal Logic - # prompt tokens": "{\"description\": \"min=728.984, mean=728.984, max=728.984, sum=1457.968 (2)\", \"tab\": \"General information\", \"score\": \"728.984126984127\"}", - "Formal Logic - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"formal_logic\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_formal_logic\"" - } - } - }, - { - "evaluation_name": "High School World History", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on High School World History", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.92, - "details": { - "description": "min=0.92, mean=0.92, max=0.92, sum=1.84 (2)", - "tab": "Accuracy", - "High School Biology - Observed inference time (s)": "{\"description\": \"min=0.685, mean=0.685, max=0.685, sum=1.371 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6854658296031336\"}", - "High School Chemistry - Observed inference time (s)": "{\"description\": \"min=0.756, mean=0.756, max=0.756, sum=1.513 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7563052259642502\"}", - "High School Computer Science - Observed inference time (s)": "{\"description\": \"min=0.717, mean=0.717, max=0.717, sum=1.435 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7174343037605285\"}", - "High School European History - Observed inference time (s)": "{\"description\": \"min=0.767, mean=0.767, max=0.767, sum=1.535 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7674274748021906\"}", - "High School Geography - Observed inference time (s)": "{\"description\": \"min=0.7, mean=0.7, max=0.7, sum=1.4 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6998175286283397\"}", - "High School Government And Politics - Observed inference time (s)": "{\"description\": \"min=0.824, mean=0.824, max=0.824, sum=1.648 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8241880792410262\"}", - "High School Macroeconomics - Observed inference time (s)": "{\"description\": \"min=0.863, mean=0.863, max=0.863, sum=1.726 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8630072312477307\"}", - "High School Mathematics - Observed inference time (s)": "{\"description\": \"min=0.815, mean=0.815, max=0.815, sum=1.631 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8153338502954554\"}", - "High School Microeconomics - Observed inference time (s)": "{\"description\": \"min=0.818, mean=0.818, max=0.818, sum=1.637 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8183944405627852\"}", - "High School Physics - Observed inference time (s)": "{\"description\": \"min=0.83, mean=0.83, max=0.83, sum=1.659 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8296057877951111\"}", - "High School Psychology - Observed inference time (s)": "{\"description\": \"min=0.818, mean=0.818, max=0.818, sum=1.636 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8179746304083308\"}", - "High School Statistics - Observed inference time (s)": "{\"description\": \"min=0.775, mean=0.775, max=0.775, sum=1.55 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7749874878812719\"}", - "High School US History - Observed inference time (s)": "{\"description\": \"min=0.743, mean=0.743, max=0.743, sum=1.486 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7428295682458317\"}", - "High School World History - Observed inference time (s)": "{\"description\": \"min=0.832, mean=0.832, max=0.832, sum=1.663 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8316668367587061\"}", - "High School Biology - # eval": "{\"description\": \"min=310, mean=310, max=310, sum=620 (2)\", \"tab\": \"General information\", \"score\": \"310.0\"}", - "High School Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Biology - # prompt tokens": "{\"description\": \"min=610.561, mean=610.561, max=610.561, sum=1221.123 (2)\", \"tab\": \"General information\", \"score\": \"610.5612903225806\"}", - "High School Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Chemistry - # eval": "{\"description\": \"min=203, mean=203, max=203, sum=406 (2)\", \"tab\": \"General information\", \"score\": \"203.0\"}", - "High School Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # prompt tokens": "{\"description\": \"min=582.798, mean=582.798, max=582.798, sum=1165.596 (2)\", \"tab\": \"General information\", \"score\": \"582.7980295566502\"}", - "High School Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "High School Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # prompt tokens": "{\"description\": \"min=998.24, mean=998.24, max=998.24, sum=1996.48 (2)\", \"tab\": \"General information\", \"score\": \"998.24\"}", - "High School Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School European History - # eval": "{\"description\": \"min=165, mean=165, max=165, sum=330 (2)\", \"tab\": \"General information\", \"score\": \"165.0\"}", - "High School European History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School European History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # prompt tokens": "{\"description\": \"min=3099.109, mean=3099.109, max=3099.109, sum=6198.218 (2)\", \"tab\": \"General information\", \"score\": \"3099.109090909091\"}", - "High School European History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Geography - # eval": "{\"description\": \"min=198, mean=198, max=198, sum=396 (2)\", \"tab\": \"General information\", \"score\": \"198.0\"}", - "High School Geography - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Geography - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # prompt tokens": "{\"description\": \"min=439.207, mean=439.207, max=439.207, sum=878.414 (2)\", \"tab\": \"General information\", \"score\": \"439.2070707070707\"}", - "High School Geography - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Government And Politics - # eval": "{\"description\": \"min=193, mean=193, max=193, sum=386 (2)\", \"tab\": \"General information\", \"score\": \"193.0\"}", - "High School Government And Politics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Government And Politics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # prompt tokens": "{\"description\": \"min=524.808, mean=524.808, max=524.808, sum=1049.617 (2)\", \"tab\": \"General information\", \"score\": \"524.8082901554404\"}", - "High School Government And Politics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Macroeconomics - # eval": "{\"description\": \"min=390, mean=390, max=390, sum=780 (2)\", \"tab\": \"General information\", \"score\": \"390.0\"}", - "High School Macroeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Macroeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # prompt tokens": "{\"description\": \"min=433.815, mean=433.815, max=433.815, sum=867.631 (2)\", \"tab\": \"General information\", \"score\": \"433.81538461538463\"}", - "High School Macroeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Mathematics - # eval": "{\"description\": \"min=270, mean=270, max=270, sum=540 (2)\", \"tab\": \"General information\", \"score\": \"270.0\"}", - "High School Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # prompt tokens": "{\"description\": \"min=594.13, mean=594.13, max=594.13, sum=1188.259 (2)\", \"tab\": \"General information\", \"score\": \"594.1296296296297\"}", - "High School Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Microeconomics - # eval": "{\"description\": \"min=238, mean=238, max=238, sum=476 (2)\", \"tab\": \"General information\", \"score\": \"238.0\"}", - "High School Microeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Microeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # prompt tokens": "{\"description\": \"min=453.345, mean=453.345, max=453.345, sum=906.689 (2)\", \"tab\": \"General information\", \"score\": \"453.34453781512605\"}", - "High School Microeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Physics - # eval": "{\"description\": \"min=151, mean=151, max=151, sum=302 (2)\", \"tab\": \"General information\", \"score\": \"151.0\"}", - "High School Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # prompt tokens": "{\"description\": \"min=632.775, mean=632.775, max=632.775, sum=1265.55 (2)\", \"tab\": \"General information\", \"score\": \"632.774834437086\"}", - "High School Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Psychology - # eval": "{\"description\": \"min=545, mean=545, max=545, sum=1090 (2)\", \"tab\": \"General information\", \"score\": \"545.0\"}", - "High School Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # prompt tokens": "{\"description\": \"min=568.873, mean=568.873, max=568.873, sum=1137.747 (2)\", \"tab\": \"General information\", \"score\": \"568.8733944954129\"}", - "High School Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Statistics - # eval": "{\"description\": \"min=216, mean=216, max=216, sum=432 (2)\", \"tab\": \"General information\", \"score\": \"216.0\"}", - "High School Statistics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Statistics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # prompt tokens": "{\"description\": \"min=923.644, mean=923.644, max=923.644, sum=1847.287 (2)\", \"tab\": \"General information\", \"score\": \"923.6435185185185\"}", - "High School Statistics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School US History - # eval": "{\"description\": \"min=204, mean=204, max=204, sum=408 (2)\", \"tab\": \"General information\", \"score\": \"204.0\"}", - "High School US History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School US History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # prompt tokens": "{\"description\": \"min=2487.446, mean=2487.446, max=2487.446, sum=4974.892 (2)\", \"tab\": \"General information\", \"score\": \"2487.4460784313724\"}", - "High School US History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School World History - # eval": "{\"description\": \"min=237, mean=237, max=237, sum=474 (2)\", \"tab\": \"General information\", \"score\": \"237.0\"}", - "High School World History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School World History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # prompt tokens": "{\"description\": \"min=1595.553, mean=1595.553, max=1595.553, sum=3191.105 (2)\", \"tab\": \"General information\", \"score\": \"1595.5527426160338\"}", - "High School World History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"high_school_world_history\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_high_school_world_history\"" - } - } - }, - { - "evaluation_name": "Human Sexuality", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Human Sexuality", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.924, - "details": { - "description": "min=0.924, mean=0.924, max=0.924, sum=1.847 (2)", - "tab": "Accuracy", - "Human Aging - Observed inference time (s)": "{\"description\": \"min=0.819, mean=0.819, max=0.819, sum=1.639 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8192698356816587\"}", - "Human Sexuality - Observed inference time (s)": "{\"description\": \"min=0.733, mean=0.733, max=0.733, sum=1.466 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.732998116325786\"}", - "Human Aging - # eval": "{\"description\": \"min=223, mean=223, max=223, sum=446 (2)\", \"tab\": \"General information\", \"score\": \"223.0\"}", - "Human Aging - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Aging - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Aging - # prompt tokens": "{\"description\": \"min=363.152, mean=363.152, max=363.152, sum=726.305 (2)\", \"tab\": \"General information\", \"score\": \"363.15246636771303\"}", - "Human Aging - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Human Sexuality - # eval": "{\"description\": \"min=131, mean=131, max=131, sum=262 (2)\", \"tab\": \"General information\", \"score\": \"131.0\"}", - "Human Sexuality - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Sexuality - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # prompt tokens": "{\"description\": \"min=404.748, mean=404.748, max=404.748, sum=809.496 (2)\", \"tab\": \"General information\", \"score\": \"404.7480916030534\"}", - "Human Sexuality - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"human_sexuality\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_human_sexuality\"" - } - } - }, - { - "evaluation_name": "International Law", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on International Law", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.926, - "details": { - "description": "min=0.926, mean=0.926, max=0.926, sum=1.851 (2)", - "tab": "Accuracy", - "International Law - Observed inference time (s)": "{\"description\": \"min=0.852, mean=0.852, max=0.852, sum=1.705 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8524710600041161\"}", - "International Law - # eval": "{\"description\": \"min=121, mean=121, max=121, sum=242 (2)\", \"tab\": \"General information\", \"score\": \"121.0\"}", - "International Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "International Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "International Law - # prompt tokens": "{\"description\": \"min=730.182, mean=730.182, max=730.182, sum=1460.364 (2)\", \"tab\": \"General information\", \"score\": \"730.1818181818181\"}", - "International Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"international_law\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_international_law\"" - } - } - }, - { - "evaluation_name": "Logical Fallacies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Logical Fallacies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.847, - "details": { - "description": "min=0.847, mean=0.847, max=0.847, sum=1.693 (2)", - "tab": "Accuracy", - "Logical Fallacies - Observed inference time (s)": "{\"description\": \"min=0.944, mean=0.944, max=0.944, sum=1.887 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9436116130805454\"}", - "Logical Fallacies - # eval": "{\"description\": \"min=163, mean=163, max=163, sum=326 (2)\", \"tab\": \"General information\", \"score\": \"163.0\"}", - "Logical Fallacies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Logical Fallacies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Logical Fallacies - # prompt tokens": "{\"description\": \"min=496.779, mean=496.779, max=496.779, sum=993.558 (2)\", \"tab\": \"General information\", \"score\": \"496.77914110429447\"}", - "Logical Fallacies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"logical_fallacies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_logical_fallacies\"" - } - } - }, - { - "evaluation_name": "Machine Learning", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Machine Learning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.661, - "details": { - "description": "min=0.661, mean=0.661, max=0.661, sum=1.321 (2)", - "tab": "Accuracy", - "Machine Learning - Observed inference time (s)": "{\"description\": \"min=0.874, mean=0.874, max=0.874, sum=1.748 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8740715363195964\"}", - "Machine Learning - # eval": "{\"description\": \"min=112, mean=112, max=112, sum=224 (2)\", \"tab\": \"General information\", \"score\": \"112.0\"}", - "Machine Learning - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Machine Learning - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Machine Learning - # prompt tokens": "{\"description\": \"min=744.83, mean=744.83, max=744.83, sum=1489.661 (2)\", \"tab\": \"General information\", \"score\": \"744.8303571428571\"}", - "Machine Learning - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"machine_learning\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_machine_learning\"" - } - } - }, - { - "evaluation_name": "Management", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Management", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.883, - "details": { - "description": "min=0.883, mean=0.883, max=0.883, sum=1.767 (2)", - "tab": "Accuracy", - "Management - Observed inference time (s)": "{\"description\": \"min=0.79, mean=0.79, max=0.79, sum=1.58 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7901336544925727\"}", - "Management - # eval": "{\"description\": \"min=103, mean=103, max=103, sum=206 (2)\", \"tab\": \"General information\", \"score\": \"103.0\"}", - "Management - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Management - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Management - # prompt tokens": "{\"description\": \"min=325.359, mean=325.359, max=325.359, sum=650.718 (2)\", \"tab\": \"General information\", \"score\": \"325.3592233009709\"}", - "Management - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"management\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_management\"" - } - } - }, - { - "evaluation_name": "Marketing", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Marketing", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.94, - "details": { - "description": "min=0.94, mean=0.94, max=0.94, sum=1.88 (2)", - "tab": "Accuracy", - "Marketing - Observed inference time (s)": "{\"description\": \"min=0.884, mean=0.884, max=0.884, sum=1.768 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.88404920977405\"}", - "Marketing - # eval": "{\"description\": \"min=234, mean=234, max=234, sum=468 (2)\", \"tab\": \"General information\", \"score\": \"234.0\"}", - "Marketing - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Marketing - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Marketing - # prompt tokens": "{\"description\": \"min=473.423, mean=473.423, max=473.423, sum=946.846 (2)\", \"tab\": \"General information\", \"score\": \"473.4230769230769\"}", - "Marketing - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"marketing\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_marketing\"" - } - } - }, - { - "evaluation_name": "Medical Genetics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Medical Genetics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9, - "details": { - "description": "min=0.9, mean=0.9, max=0.9, sum=1.8 (2)", - "tab": "Accuracy", - "Medical Genetics - Observed inference time (s)": "{\"description\": \"min=0.77, mean=0.77, max=0.77, sum=1.54 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7701838827133178\"}", - "Medical Genetics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Medical Genetics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Medical Genetics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Medical Genetics - # prompt tokens": "{\"description\": \"min=415.71, mean=415.71, max=415.71, sum=831.42 (2)\", \"tab\": \"General information\", \"score\": \"415.71\"}", - "Medical Genetics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"medical_genetics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_medical_genetics\"" - } - } - }, - { - "evaluation_name": "Miscellaneous", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Miscellaneous", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.936, - "details": { - "description": "min=0.936, mean=0.936, max=0.936, sum=1.872 (2)", - "tab": "Accuracy", - "Miscellaneous - Observed inference time (s)": "{\"description\": \"min=0.71, mean=0.71, max=0.71, sum=1.419 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7095236696045975\"}", - "Miscellaneous - # eval": "{\"description\": \"min=783, mean=783, max=783, sum=1566 (2)\", \"tab\": \"General information\", \"score\": \"783.0\"}", - "Miscellaneous - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Miscellaneous - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Miscellaneous - # prompt tokens": "{\"description\": \"min=358.519, mean=358.519, max=358.519, sum=717.037 (2)\", \"tab\": \"General information\", \"score\": \"358.51851851851853\"}", - "Miscellaneous - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"miscellaneous\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_miscellaneous\"" - } - } - }, - { - "evaluation_name": "Moral Scenarios", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Moral Scenarios", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.839, - "details": { - "description": "min=0.839, mean=0.839, max=0.839, sum=1.678 (2)", - "tab": "Accuracy", - "Moral Disputes - Observed inference time (s)": "{\"description\": \"min=0.861, mean=0.861, max=0.861, sum=1.721 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8607459598883039\"}", - "Moral Scenarios - Observed inference time (s)": "{\"description\": \"min=0.831, mean=0.831, max=0.831, sum=1.663 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8314023547998354\"}", - "Moral Disputes - # eval": "{\"description\": \"min=346, mean=346, max=346, sum=692 (2)\", \"tab\": \"General information\", \"score\": \"346.0\"}", - "Moral Disputes - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Disputes - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Disputes - # prompt tokens": "{\"description\": \"min=550.038, mean=550.038, max=550.038, sum=1100.075 (2)\", \"tab\": \"General information\", \"score\": \"550.0375722543353\"}", - "Moral Disputes - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Moral Scenarios - # eval": "{\"description\": \"min=895, mean=895, max=895, sum=1790 (2)\", \"tab\": \"General information\", \"score\": \"895.0\"}", - "Moral Scenarios - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Scenarios - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # prompt tokens": "{\"description\": \"min=755.516, mean=755.516, max=755.516, sum=1511.032 (2)\", \"tab\": \"General information\", \"score\": \"755.5162011173185\"}", - "Moral Scenarios - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"moral_scenarios\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_moral_scenarios\"" - } - } - }, - { - "evaluation_name": "Nutrition", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Nutrition", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.827, - "details": { - "description": "min=0.827, mean=0.827, max=0.827, sum=1.654 (2)", - "tab": "Accuracy", - "Nutrition - Observed inference time (s)": "{\"description\": \"min=0.816, mean=0.816, max=0.816, sum=1.632 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8157819338094175\"}", - "Nutrition - # eval": "{\"description\": \"min=306, mean=306, max=306, sum=612 (2)\", \"tab\": \"General information\", \"score\": \"306.0\"}", - "Nutrition - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Nutrition - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Nutrition - # prompt tokens": "{\"description\": \"min=690.69, mean=690.69, max=690.69, sum=1381.379 (2)\", \"tab\": \"General information\", \"score\": \"690.6895424836601\"}", - "Nutrition - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"nutrition\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_nutrition\"" - } - } - }, - { - "evaluation_name": "Prehistory", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Prehistory", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.92, - "details": { - "description": "min=0.92, mean=0.92, max=0.92, sum=1.84 (2)", - "tab": "Accuracy", - "Prehistory - Observed inference time (s)": "{\"description\": \"min=0.927, mean=0.927, max=0.927, sum=1.854 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9269687445075424\"}", - "Prehistory - # eval": "{\"description\": \"min=324, mean=324, max=324, sum=648 (2)\", \"tab\": \"General information\", \"score\": \"324.0\"}", - "Prehistory - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Prehistory - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Prehistory - # prompt tokens": "{\"description\": \"min=612.145, mean=612.145, max=612.145, sum=1224.29 (2)\", \"tab\": \"General information\", \"score\": \"612.145061728395\"}", - "Prehistory - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"prehistory\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_prehistory\"" - } - } - }, - { - "evaluation_name": "Public Relations", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Public Relations", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.764, - "details": { - "description": "min=0.764, mean=0.764, max=0.764, sum=1.527 (2)", - "tab": "Accuracy", - "Public Relations - Observed inference time (s)": "{\"description\": \"min=0.75, mean=0.75, max=0.75, sum=1.5 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7498581886291504\"}", - "Public Relations - # eval": "{\"description\": \"min=110, mean=110, max=110, sum=220 (2)\", \"tab\": \"General information\", \"score\": \"110.0\"}", - "Public Relations - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Public Relations - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Public Relations - # prompt tokens": "{\"description\": \"min=472.036, mean=472.036, max=472.036, sum=944.073 (2)\", \"tab\": \"General information\", \"score\": \"472.03636363636366\"}", - "Public Relations - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"public_relations\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_public_relations\"" - } - } - }, - { - "evaluation_name": "Security Studies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Security Studies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.865, - "details": { - "description": "min=0.865, mean=0.865, max=0.865, sum=1.731 (2)", - "tab": "Accuracy", - "Security Studies - Observed inference time (s)": "{\"description\": \"min=0.73, mean=0.73, max=0.73, sum=1.459 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7295293778789287\"}", - "Security Studies - # eval": "{\"description\": \"min=245, mean=245, max=245, sum=490 (2)\", \"tab\": \"General information\", \"score\": \"245.0\"}", - "Security Studies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Security Studies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Security Studies - # prompt tokens": "{\"description\": \"min=1325.865, mean=1325.865, max=1325.865, sum=2651.731 (2)\", \"tab\": \"General information\", \"score\": \"1325.865306122449\"}", - "Security Studies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"security_studies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_security_studies\"" - } - } - }, - { - "evaluation_name": "Sociology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Sociology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.91, - "details": { - "description": "min=0.91, mean=0.91, max=0.91, sum=1.821 (2)", - "tab": "Accuracy", - "Sociology - Observed inference time (s)": "{\"description\": \"min=0.751, mean=0.751, max=0.751, sum=1.501 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.750605917688626\"}", - "Sociology - # eval": "{\"description\": \"min=201, mean=201, max=201, sum=402 (2)\", \"tab\": \"General information\", \"score\": \"201.0\"}", - "Sociology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Sociology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Sociology - # prompt tokens": "{\"description\": \"min=497.95, mean=497.95, max=497.95, sum=995.9 (2)\", \"tab\": \"General information\", \"score\": \"497.9502487562189\"}", - "Sociology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"sociology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_sociology\"" - } - } - }, - { - "evaluation_name": "Virology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Virology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.59, - "details": { - "description": "min=0.59, mean=0.59, max=0.59, sum=1.181 (2)", - "tab": "Accuracy", - "Virology - Observed inference time (s)": "{\"description\": \"min=0.824, mean=0.824, max=0.824, sum=1.648 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8238025544637657\"}", - "Virology - # eval": "{\"description\": \"min=166, mean=166, max=166, sum=332 (2)\", \"tab\": \"General information\", \"score\": \"166.0\"}", - "Virology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Virology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Virology - # prompt tokens": "{\"description\": \"min=405.349, mean=405.349, max=405.349, sum=810.699 (2)\", \"tab\": \"General information\", \"score\": \"405.34939759036143\"}", - "Virology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"virology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_virology\"" - } - } - }, - { - "evaluation_name": "World Religions", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on World Religions", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.865, - "details": { - "description": "min=0.865, mean=0.865, max=0.865, sum=1.731 (2)", - "tab": "Accuracy", - "World Religions - Observed inference time (s)": "{\"description\": \"min=0.834, mean=0.834, max=0.834, sum=1.668 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8341451960000378\"}", - "World Religions - # eval": "{\"description\": \"min=171, mean=171, max=171, sum=342 (2)\", \"tab\": \"General information\", \"score\": \"171.0\"}", - "World Religions - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "World Religions - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "World Religions - # prompt tokens": "{\"description\": \"min=318.924, mean=318.924, max=318.924, sum=637.848 (2)\", \"tab\": \"General information\", \"score\": \"318.92397660818716\"}", - "World Religions - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"world_religions\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_world_religions\"" - } - } - }, - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.24, - "details": { - "description": "", - "tab": "Efficiency" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/mistralai_mistral-large-2411-fc.json b/data/models/mistralai_mistral-large-2411-fc.json deleted file mode 100644 index 5c1df676ac32a8a4f4b30dbf5a9a704179620500..0000000000000000000000000000000000000000 --- a/data/models/mistralai_mistral-large-2411-fc.json +++ /dev/null @@ -1,851 +0,0 @@ -{ - "model_info": { - "name": "mistral-large-2411 (FC)", - "id": "mistralai/mistral-large-2411-fc", - "developer": "mistralai", - "additional_details": { - "raw_model_name": "mistral-large-2411 (FC)", - "organization": "Mistral AI", - "license": "Proprietary", - "mode": "FC", - "model_link": "https://docs.mistral.ai/guides/model-selection/" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/mistralai/mistral-large-2411-fc/1775236112.389309", - "retrieved_timestamp": "1775236112.389309", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 46.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 38.37 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 115.98 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 2.04 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 4.02 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 4.68 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 84.65 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 72.08 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 93.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 89.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 83.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 81.87 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 87.21 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 80.72 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 81.25 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 75.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 14.12 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 18.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 11.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 13.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 13.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 28.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 41.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 15.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 24.95 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 18.71 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 29.03 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 27.1 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 93.75 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 68.92 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/mistralai_mistral-large-2411-prompt.json b/data/models/mistralai_mistral-large-2411-prompt.json deleted file mode 100644 index 1cee789e247cd7bffaed77951dca1746d846300b..0000000000000000000000000000000000000000 --- a/data/models/mistralai_mistral-large-2411-prompt.json +++ /dev/null @@ -1,905 +0,0 @@ -{ - "model_info": { - "name": "mistral-large-2411 (Prompt)", - "id": "mistralai/mistral-large-2411-prompt", - "developer": "mistralai", - "additional_details": { - "raw_model_name": "mistral-large-2411 (Prompt)", - "organization": "Mistral AI", - "license": "Proprietary", - "mode": "Prompt", - "model_link": "https://docs.mistral.ai/guides/model-selection/" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/mistralai/mistral-large-2411-prompt/1775236112.398613", - "retrieved_timestamp": "1775236112.398613", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 63.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 31.84 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 232.42 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 1.82 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 7.15 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 4.08 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 83.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 75.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 89.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 87.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 80.0 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 68.1 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 83.72 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 64.01 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 93.75 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 62.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 13.75 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 20.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 5.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 11.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 19.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 20.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 28.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 12.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 23.66 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 16.77 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 30.97 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 23.23 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 93.75 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 38.77 - } - }, - { - "evaluation_result_id": "bfcl.format_sensitivity.max_delta::max_delta", - "evaluation_name": "bfcl.format_sensitivity.max_delta", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.format_sensitivity.max_delta", - "metric_name": "Format sensitivity max delta", - "metric_kind": "difference", - "metric_unit": "percentage_points", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Format Sensitivity Max Delta" - } - }, - "score_details": { - "score": 13.5 - } - }, - { - "evaluation_result_id": "bfcl.format_sensitivity.stddev::stddev", - "evaluation_name": "bfcl.format_sensitivity.stddev", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.format_sensitivity.stddev", - "metric_name": "Format sensitivity standard deviation", - "metric_kind": "difference", - "metric_unit": "percentage_points", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Format Sensitivity Standard Deviation" - } - }, - "score_details": { - "score": 3.91 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/mistralai_mistral-large-2411.json b/data/models/mistralai_mistral-large-2411.json deleted file mode 100644 index 79d2eed4a850581f6fe1569f21e0186cca85bc23..0000000000000000000000000000000000000000 --- a/data/models/mistralai_mistral-large-2411.json +++ /dev/null @@ -1,234 +0,0 @@ -{ - "model_info": { - "name": "Mistral Large 2411", - "id": "mistralai/mistral-large-2411", - "developer": "mistralai", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_capabilities/mistralai_mistral-large-2411/1774096304.5056698", - "retrieved_timestamp": "1774096304.5056698", - "source_metadata": { - "source_name": "helm_capabilities", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_capabilities", - "evaluation_results": [ - { - "evaluation_name": "Mean score", - "source_data": { - "dataset_name": "helm_capabilities", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "The mean of the scores from all columns.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.598, - "details": { - "description": "", - "tab": "Accuracy", - "Mean score - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"14.462006275515396\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU-Pro", - "source_data": { - "dataset_name": "MMLU-Pro", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on MMLU-Pro", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.599, - "details": { - "description": "min=0.599, mean=0.599, max=0.599, sum=0.599 (1)", - "tab": "Accuracy", - "MMLU-Pro - Observed inference time (s)": "{\"description\": \"min=7.537, mean=7.537, max=7.537, sum=7.537 (1)\", \"tab\": \"Efficiency\", \"score\": \"7.537241208553314\"}", - "MMLU-Pro - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "MMLU-Pro - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - # prompt tokens": "{\"description\": \"min=260.915, mean=260.915, max=260.915, sum=260.915 (1)\", \"tab\": \"General information\", \"score\": \"260.915\"}", - "MMLU-Pro - # output tokens": "{\"description\": \"min=316.273, mean=316.273, max=316.273, sum=316.273 (1)\", \"tab\": \"General information\", \"score\": \"316.273\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"all\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.435, - "details": { - "description": "min=0.435, mean=0.435, max=0.435, sum=0.435 (1)", - "tab": "Accuracy", - "GPQA - Observed inference time (s)": "{\"description\": \"min=12.217, mean=12.217, max=12.217, sum=12.217 (1)\", \"tab\": \"Efficiency\", \"score\": \"12.217145950270341\"}", - "GPQA - # eval": "{\"description\": \"min=446, mean=446, max=446, sum=446 (1)\", \"tab\": \"General information\", \"score\": \"446.0\"}", - "GPQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - # prompt tokens": "{\"description\": \"min=281.998, mean=281.998, max=281.998, sum=281.998 (1)\", \"tab\": \"General information\", \"score\": \"281.99775784753365\"}", - "GPQA - # output tokens": "{\"description\": \"min=507.357, mean=507.357, max=507.357, sum=507.357 (1)\", \"tab\": \"General information\", \"score\": \"507.3565022421525\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"gpqa_main\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "IFEval Strict Acc on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.876, - "details": { - "description": "min=0.876, mean=0.876, max=0.876, sum=0.876 (1)", - "tab": "Accuracy", - "IFEval - Observed inference time (s)": "{\"description\": \"min=10.742, mean=10.742, max=10.742, sum=10.742 (1)\", \"tab\": \"Efficiency\", \"score\": \"10.741783690761066\"}", - "IFEval - # eval": "{\"description\": \"min=541, mean=541, max=541, sum=541 (1)\", \"tab\": \"General information\", \"score\": \"541.0\"}", - "IFEval - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - # prompt tokens": "{\"description\": \"min=51.36, mean=51.36, max=51.36, sum=51.36 (1)\", \"tab\": \"General information\", \"score\": \"51.36044362292052\"}", - "IFEval - # output tokens": "{\"description\": \"min=409.566, mean=409.566, max=409.566, sum=409.566 (1)\", \"tab\": \"General information\", \"score\": \"409.5656192236599\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WildBench", - "source_data": { - "dataset_name": "WildBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "WB Score on WildBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.801, - "details": { - "description": "min=0.801, mean=0.801, max=0.801, sum=0.801 (1)", - "tab": "Accuracy", - "WildBench - Observed inference time (s)": "{\"description\": \"min=23.603, mean=23.603, max=23.603, sum=23.603 (1)\", \"tab\": \"Efficiency\", \"score\": \"23.602991637706758\"}", - "WildBench - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "WildBench - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # prompt tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # output tokens": "{\"description\": \"min=1029.086, mean=1029.086, max=1029.086, sum=1029.086 (1)\", \"tab\": \"General information\", \"score\": \"1029.086\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"v2\"" - } - } - }, - { - "evaluation_name": "Omni-MATH", - "source_data": { - "dataset_name": "Omni-MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Acc on Omni-MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.281, - "details": { - "description": "min=0.281, mean=0.281, max=0.281, sum=0.281 (1)", - "tab": "Accuracy", - "Omni-MATH - Observed inference time (s)": "{\"description\": \"min=18.211, mean=18.211, max=18.211, sum=18.211 (1)\", \"tab\": \"Efficiency\", \"score\": \"18.210868890285493\"}", - "Omni-MATH - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "Omni-MATH - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - # prompt tokens": "{\"description\": \"min=120.373, mean=120.373, max=120.373, sum=120.373 (1)\", \"tab\": \"General information\", \"score\": \"120.373\"}", - "Omni-MATH - # output tokens": "{\"description\": \"min=727.801, mean=727.801, max=727.801, sum=727.801 (1)\", \"tab\": \"General information\", \"score\": \"727.801\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/mistralai_mistral-large-instruct-2411.json b/data/models/mistralai_mistral-large-instruct-2411.json deleted file mode 100644 index a71ab664b628459ac9c979f9a772c54b944fe129..0000000000000000000000000000000000000000 --- a/data/models/mistralai_mistral-large-instruct-2411.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mistral-Large-Instruct-2411", - "id": "mistralai/Mistral-Large-Instruct-2411", - "developer": "mistralai", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "122.61" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/mistralai_Mistral-Large-Instruct-2411/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8401 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6747 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4955 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4371 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.454 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5562 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/mistralai_mistral-medium-2312.json b/data/models/mistralai_mistral-medium-2312.json deleted file mode 100644 index 6f8887844f2ceae4710b84709b2a686bd4af83bb..0000000000000000000000000000000000000000 --- a/data/models/mistralai_mistral-medium-2312.json +++ /dev/null @@ -1,380 +0,0 @@ -{ - "model_info": { - "name": "Mistral Medium 2312", - "id": "mistralai/mistral-medium-2312", - "developer": "mistralai", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_lite/mistralai_mistral-medium-2312/1774096306.427425", - "retrieved_timestamp": "1774096306.427425", - "source_metadata": { - "source_name": "helm_lite", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_lite", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_lite", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.268, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.06677902621722846\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.449, - "details": { - "description": "min=0.449, mean=0.449, max=0.449, sum=0.449 (1)", - "tab": "Accuracy", - "NarrativeQA - Observed inference time (s)": "{\"description\": \"min=3.898, mean=3.898, max=3.898, sum=3.898 (1)\", \"tab\": \"Efficiency\", \"score\": \"3.898151301666045\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=3955.33, mean=3955.33, max=3955.33, sum=3955.33 (1)\", \"tab\": \"General information\", \"score\": \"3955.3295774647886\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=24.885, mean=24.885, max=24.885, sum=24.885 (1)\", \"tab\": \"General information\", \"score\": \"24.88450704225352\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (closed-book)", - "source_data": { - "dataset_name": "NaturalQuestions (closed-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (closed-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.29, - "details": { - "description": "min=0.29, mean=0.29, max=0.29, sum=0.29 (1)", - "tab": "Accuracy", - "NaturalQuestions (open-book) - Observed inference time (s)": "{\"description\": \"min=5.342, mean=5.342, max=5.342, sum=5.342 (1)\", \"tab\": \"Efficiency\", \"score\": \"5.342489146232605\"}", - "NaturalQuestions (closed-book) - Observed inference time (s)": "{\"description\": \"min=6.588, mean=6.588, max=6.588, sum=6.588 (1)\", \"tab\": \"Efficiency\", \"score\": \"6.588117929935455\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=2537.79, mean=2537.79, max=2537.79, sum=2537.79 (1)\", \"tab\": \"General information\", \"score\": \"2537.79\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=32.377, mean=32.377, max=32.377, sum=32.377 (1)\", \"tab\": \"General information\", \"score\": \"32.377\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=211.069, mean=211.069, max=211.069, sum=211.069 (1)\", \"tab\": \"General information\", \"score\": \"211.069\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=34.263, mean=34.263, max=34.263, sum=34.263 (1)\", \"tab\": \"General information\", \"score\": \"34.263\"}" - } - }, - "generation_config": { - "additional_details": { - "mode": "\"closedbook\"" - } - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.83, - "details": { - "description": "min=0.83, mean=0.83, max=0.83, sum=0.83 (1)", - "tab": "Accuracy", - "OpenbookQA - Observed inference time (s)": "{\"description\": \"min=2.12, mean=2.12, max=2.12, sum=2.12 (1)\", \"tab\": \"Efficiency\", \"score\": \"2.1195812821388245\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=280.15, mean=280.15, max=280.15, sum=280.15 (1)\", \"tab\": \"General information\", \"score\": \"280.15\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=0.968, mean=0.968, max=0.968, sum=0.968 (1)\", \"tab\": \"General information\", \"score\": \"0.968\"}" - } - }, - "generation_config": { - "additional_details": { - "dataset": "\"openbookqa\"", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.618, - "details": { - "description": "min=0.32, mean=0.618, max=0.91, sum=3.089 (5)", - "tab": "Accuracy", - "MMLU - Observed inference time (s)": "{\"description\": \"min=1.507, mean=2.775, max=3.62, sum=13.874 (5)\", \"tab\": \"Efficiency\", \"score\": \"2.774717758923246\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=402.44, mean=523.091, max=687.175, sum=2615.455 (5)\", \"tab\": \"General information\", \"score\": \"523.0910877192983\"}", - "MMLU - # output tokens": "{\"description\": \"min=0.93, mean=0.97, max=0.991, sum=4.851 (5)\", \"tab\": \"General information\", \"score\": \"0.9702456140350877\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"college_chemistry\", \"computer_security\", \"econometrics\", \"us_foreign_policy\"]", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MATH", - "source_data": { - "dataset_name": "MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Equivalent (CoT) on MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.565, - "details": { - "description": "min=0.4, mean=0.565, max=0.756, sum=3.958 (7)", - "tab": "Accuracy", - "MATH - Observed inference time (s)": "{\"description\": \"min=6.1, mean=7.086, max=10.207, sum=49.602 (7)\", \"tab\": \"Efficiency\", \"score\": \"7.0860357509079535\"}", - "MATH - # eval": "{\"description\": \"min=30, mean=62.429, max=135, sum=437 (7)\", \"tab\": \"General information\", \"score\": \"62.42857142857143\"}", - "MATH - # train": "{\"description\": \"min=8, mean=8, max=8, sum=56 (7)\", \"tab\": \"General information\", \"score\": \"8.0\"}", - "MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MATH - # prompt tokens": "{\"description\": \"min=1061.615, mean=1525.266, max=2572.962, sum=10676.865 (7)\", \"tab\": \"General information\", \"score\": \"1525.2664139976257\"}", - "MATH - # output tokens": "{\"description\": \"min=80, mean=113.328, max=132.25, sum=793.295 (7)\", \"tab\": \"General information\", \"score\": \"113.3278270462481\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"algebra\", \"counting_and_probability\", \"geometry\", \"intermediate_algebra\", \"number_theory\", \"prealgebra\", \"precalculus\"]", - "level": "\"1\"", - "use_official_examples": "\"False\"", - "use_chain_of_thought": "\"True\"" - } - } - }, - { - "evaluation_name": "GSM8K", - "source_data": { - "dataset_name": "GSM8K", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on GSM8K", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.706, - "details": { - "description": "min=0.706, mean=0.706, max=0.706, sum=0.706 (1)", - "tab": "Accuracy", - "GSM8K - Observed inference time (s)": "{\"description\": \"min=9.719, mean=9.719, max=9.719, sum=9.719 (1)\", \"tab\": \"Efficiency\", \"score\": \"9.718977437496186\"}", - "GSM8K - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "GSM8K - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "GSM8K - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GSM8K - # prompt tokens": "{\"description\": \"min=1255.268, mean=1255.268, max=1255.268, sum=1255.268 (1)\", \"tab\": \"General information\", \"score\": \"1255.268\"}", - "GSM8K - # output tokens": "{\"description\": \"min=137.554, mean=137.554, max=137.554, sum=137.554 (1)\", \"tab\": \"General information\", \"score\": \"137.554\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "LegalBench", - "source_data": { - "dataset_name": "LegalBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on LegalBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.452, - "details": { - "description": "min=0.066, mean=0.452, max=0.692, sum=2.258 (5)", - "tab": "Accuracy", - "LegalBench - Observed inference time (s)": "{\"description\": \"min=2.695, mean=3.248, max=3.795, sum=16.242 (5)\", \"tab\": \"Efficiency\", \"score\": \"3.248400288401771\"}", - "LegalBench - # eval": "{\"description\": \"min=95, mean=409.4, max=1000, sum=2047 (5)\", \"tab\": \"General information\", \"score\": \"409.4\"}", - "LegalBench - # train": "{\"description\": \"min=4, mean=4.8, max=5, sum=24 (5)\", \"tab\": \"General information\", \"score\": \"4.8\"}", - "LegalBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "LegalBench - # prompt tokens": "{\"description\": \"min=287.453, mean=1801.748, max=7275.488, sum=9008.741 (5)\", \"tab\": \"General information\", \"score\": \"1801.7482458432964\"}", - "LegalBench - # output tokens": "{\"description\": \"min=1.008, mean=8.476, max=25.305, sum=42.382 (5)\", \"tab\": \"General information\", \"score\": \"8.47642872361909\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"abercrombie\", \"corporate_lobbying\", \"function_of_decision_section\", \"international_citizenship_questions\", \"proa\"]" - } - } - }, - { - "evaluation_name": "MedQA", - "source_data": { - "dataset_name": "MedQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MedQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.61, - "details": { - "description": "min=0.61, mean=0.61, max=0.61, sum=0.61 (1)", - "tab": "Accuracy", - "MedQA - Observed inference time (s)": "{\"description\": \"min=2.813, mean=2.813, max=2.813, sum=2.813 (1)\", \"tab\": \"Efficiency\", \"score\": \"2.813041030531138\"}", - "MedQA - # eval": "{\"description\": \"min=503, mean=503, max=503, sum=503 (1)\", \"tab\": \"General information\", \"score\": \"503.0\"}", - "MedQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MedQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MedQA - # prompt tokens": "{\"description\": \"min=1193.093, mean=1193.093, max=1193.093, sum=1193.093 (1)\", \"tab\": \"General information\", \"score\": \"1193.0934393638172\"}", - "MedQA - # output tokens": "{\"description\": \"min=0.95, mean=0.95, max=0.95, sum=0.95 (1)\", \"tab\": \"General information\", \"score\": \"0.9502982107355865\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WMT 2014", - "source_data": { - "dataset_name": "WMT 2014", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "BLEU-4 on WMT 2014", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.169, - "details": { - "description": "min=0.07, mean=0.169, max=0.22, sum=0.844 (5)", - "tab": "Accuracy", - "WMT 2014 - Observed inference time (s)": "{\"description\": \"min=3.982, mean=4.948, max=6.067, sum=24.741 (5)\", \"tab\": \"Efficiency\", \"score\": \"4.9482336292575715\"}", - "WMT 2014 - # eval": "{\"description\": \"min=503, mean=568.8, max=832, sum=2844 (5)\", \"tab\": \"General information\", \"score\": \"568.8\"}", - "WMT 2014 - # train": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "WMT 2014 - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WMT 2014 - # prompt tokens": "{\"description\": \"min=198.306, mean=212.433, max=231.018, sum=1062.166 (5)\", \"tab\": \"General information\", \"score\": \"212.43317355482492\"}", - "WMT 2014 - # output tokens": "{\"description\": \"min=26.33, mean=27.816, max=30.692, sum=139.08 (5)\", \"tab\": \"General information\", \"score\": \"27.81599632971402\"}" - } - }, - "generation_config": { - "additional_details": { - "language_pair": "[\"cs-en\", \"de-en\", \"fr-en\", \"hi-en\", \"ru-en\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/mistralai_mistral-medium-2505-fc.json b/data/models/mistralai_mistral-medium-2505-fc.json deleted file mode 100644 index 0ca5e127d34ede9562470904912a866070f9cab2..0000000000000000000000000000000000000000 --- a/data/models/mistralai_mistral-medium-2505-fc.json +++ /dev/null @@ -1,851 +0,0 @@ -{ - "model_info": { - "name": "Mistral-Medium-2505 (FC)", - "id": "mistralai/mistral-medium-2505-fc", - "developer": "mistralai", - "additional_details": { - "raw_model_name": "Mistral-Medium-2505 (FC)", - "organization": "Mistral AI", - "license": "Proprietary", - "mode": "FC", - "model_link": "https://docs.mistral.ai/guides/model-selection/" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/mistralai/mistral-medium-2505-fc/1775236112.390909", - "retrieved_timestamp": "1775236112.390909", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 49.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 37.56 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 18.8 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 1.6 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 4.44 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 4.19 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 67.44 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 39.75 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 78.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 83.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 69.0 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 67.95 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 67.05 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 68.09 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 81.25 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 62.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 10.75 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 15.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 7.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 7.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 13.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 35.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 36.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 34.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 23.01 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 15.48 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 20.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 33.55 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 62.5 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 91.95 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/mistralai_mistral-medium-2505.json b/data/models/mistralai_mistral-medium-2505.json deleted file mode 100644 index da5af336e39766735000b6e05a99d5a72081e035..0000000000000000000000000000000000000000 --- a/data/models/mistralai_mistral-medium-2505.json +++ /dev/null @@ -1,904 +0,0 @@ -{ - "model_info": { - "name": "Mistral-Medium-2505", - "id": "mistralai/mistral-medium-2505", - "developer": "mistralai", - "additional_details": { - "raw_model_name": "Mistral-Medium-2505", - "organization": "Mistral AI", - "license": "Proprietary", - "model_link": "https://docs.mistral.ai/guides/model-selection/" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/mistralai/mistral-medium-2505/1775236112.390395", - "retrieved_timestamp": "1775236112.390395", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 48.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 37.69 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 36.51 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 1.21 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 3.5 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 2.86 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 85.33 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 76.33 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 91.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 88.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 85.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 66.03 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 80.23 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 62.39 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 81.25 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 62.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 9.88 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 13.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 6.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 6.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 13.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 39.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 41.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 37.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 21.72 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 16.13 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 14.84 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 34.19 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 75.0 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 74.49 - } - }, - { - "evaluation_result_id": "bfcl.format_sensitivity.max_delta::max_delta", - "evaluation_name": "bfcl.format_sensitivity.max_delta", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.format_sensitivity.max_delta", - "metric_name": "Format sensitivity max delta", - "metric_kind": "difference", - "metric_unit": "percentage_points", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Format Sensitivity Max Delta" - } - }, - "score_details": { - "score": 21.5 - } - }, - { - "evaluation_result_id": "bfcl.format_sensitivity.stddev::stddev", - "evaluation_name": "bfcl.format_sensitivity.stddev", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.format_sensitivity.stddev", - "metric_name": "Format sensitivity standard deviation", - "metric_kind": "difference", - "metric_unit": "percentage_points", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Format Sensitivity Standard Deviation" - } - }, - "score_details": { - "score": 5.02 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/mistralai_mistral-medium-3.json b/data/models/mistralai_mistral-medium-3.json deleted file mode 100644 index 3d0755e0237b9760231fee380b173b29d57eab4a..0000000000000000000000000000000000000000 --- a/data/models/mistralai_mistral-medium-3.json +++ /dev/null @@ -1,1043 +0,0 @@ -{ - "model_info": { - "name": "mistral-medium-3", - "id": "mistralai/mistral-medium-3", - "developer": "mistralai", - "inference_platform": "unknown", - "additional_details": { - "display_name": "Mistral Medium 3" - } - }, - "evaluations": [ - { - "evaluation_id": "global-mmlu-lite/mistralai_mistral-medium-3/1773936496.366405", - "retrieved_timestamp": "1773936496.366405", - "source_metadata": { - "source_name": "Global MMLU Lite Leaderboard", - "source_type": "documentation", - "source_organization_name": "kaggle", - "source_organization_url": "www.kaggle.com", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "kaggle kernel", - "version": "4", - "additional_details": { - "url": "https://www.kaggle.com/code/shivalikasingh95/global-mmlu-lite-sample-notebook" - } - }, - "benchmark": "global-mmlu-lite", - "evaluation_results": [ - { - "evaluation_name": "Global MMLU Lite", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Global MMLU Lite", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5511 - } - }, - { - "evaluation_name": "Culturally Sensitive", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Sensitive", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5391 - } - }, - { - "evaluation_name": "Culturally Agnostic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Agnostic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5631 - } - }, - { - "evaluation_name": "Arabic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Arabic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.455, - "uncertainty": { - "confidence_interval": { - "lower": -0.0488, - "upper": 0.0488, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "English", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - English", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.38, - "uncertainty": { - "confidence_interval": { - "lower": -0.0476, - "upper": 0.0476, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Bengali", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Bengali", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5175, - "uncertainty": { - "confidence_interval": { - "lower": -0.049, - "upper": 0.049, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "German", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - German", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4775, - "uncertainty": { - "confidence_interval": { - "lower": -0.0489, - "upper": 0.0489, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "French", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - French", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.41, - "uncertainty": { - "confidence_interval": { - "lower": -0.0482, - "upper": 0.0482, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Hindi", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Hindi", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.555, - "uncertainty": { - "confidence_interval": { - "lower": -0.0487, - "upper": 0.0487, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Indonesian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Indonesian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.515, - "uncertainty": { - "confidence_interval": { - "lower": -0.049, - "upper": 0.049, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Italian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Italian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.535, - "uncertainty": { - "confidence_interval": { - "lower": -0.0489, - "upper": 0.0489, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Japanese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Japanese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.58, - "uncertainty": { - "confidence_interval": { - "lower": -0.0484, - "upper": 0.0484, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Korean", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Korean", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.595, - "uncertainty": { - "confidence_interval": { - "lower": -0.0481, - "upper": 0.0481, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Portuguese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Portuguese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5175, - "uncertainty": { - "confidence_interval": { - "lower": -0.049, - "upper": 0.049, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Spanish", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Spanish", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5375, - "uncertainty": { - "confidence_interval": { - "lower": -0.0489, - "upper": 0.0489, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Swahili", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Swahili", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7075, - "uncertainty": { - "confidence_interval": { - "lower": -0.0446, - "upper": 0.0446, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Yoruba", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Yoruba", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7675, - "uncertainty": { - "confidence_interval": { - "lower": -0.0414, - "upper": 0.0414, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Chinese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Chinese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.535, - "uncertainty": { - "confidence_interval": { - "lower": -0.0489, - "upper": 0.0489, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Burmese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Burmese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7325, - "uncertainty": { - "confidence_interval": { - "lower": -0.0434, - "upper": 0.0434, - "method": "unknown" - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "global-mmlu-lite/mistralai_mistral-medium-3/1773936583.743359", - "retrieved_timestamp": "1773936583.743359", - "source_metadata": { - "source_name": "Global MMLU Lite Leaderboard", - "source_type": "documentation", - "source_organization_name": "kaggle", - "source_organization_url": "www.kaggle.com", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "kaggle kernel", - "version": "4", - "additional_details": { - "url": "https://www.kaggle.com/code/shivalikasingh95/global-mmlu-lite-sample-notebook" - } - }, - "benchmark": "global-mmlu-lite", - "evaluation_results": [ - { - "evaluation_name": "Global MMLU Lite", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Global MMLU Lite", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5511 - } - }, - { - "evaluation_name": "Culturally Sensitive", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Sensitive", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5391 - } - }, - { - "evaluation_name": "Culturally Agnostic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Agnostic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5631 - } - }, - { - "evaluation_name": "Arabic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Arabic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.455, - "uncertainty": { - "confidence_interval": { - "lower": -0.0488, - "upper": 0.0488, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "English", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - English", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.38, - "uncertainty": { - "confidence_interval": { - "lower": -0.0476, - "upper": 0.0476, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Bengali", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Bengali", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5175, - "uncertainty": { - "confidence_interval": { - "lower": -0.049, - "upper": 0.049, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "German", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - German", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4775, - "uncertainty": { - "confidence_interval": { - "lower": -0.0489, - "upper": 0.0489, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "French", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - French", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.41, - "uncertainty": { - "confidence_interval": { - "lower": -0.0482, - "upper": 0.0482, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Hindi", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Hindi", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.555, - "uncertainty": { - "confidence_interval": { - "lower": -0.0487, - "upper": 0.0487, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Indonesian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Indonesian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.515, - "uncertainty": { - "confidence_interval": { - "lower": -0.049, - "upper": 0.049, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Italian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Italian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.535, - "uncertainty": { - "confidence_interval": { - "lower": -0.0489, - "upper": 0.0489, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Japanese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Japanese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.58, - "uncertainty": { - "confidence_interval": { - "lower": -0.0484, - "upper": 0.0484, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Korean", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Korean", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.595, - "uncertainty": { - "confidence_interval": { - "lower": -0.0481, - "upper": 0.0481, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Portuguese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Portuguese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5175, - "uncertainty": { - "confidence_interval": { - "lower": -0.049, - "upper": 0.049, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Spanish", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Spanish", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5375, - "uncertainty": { - "confidence_interval": { - "lower": -0.0489, - "upper": 0.0489, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Swahili", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Swahili", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7075, - "uncertainty": { - "confidence_interval": { - "lower": -0.0446, - "upper": 0.0446, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Yoruba", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Yoruba", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7675, - "uncertainty": { - "confidence_interval": { - "lower": -0.0414, - "upper": 0.0414, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Chinese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Chinese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.535, - "uncertainty": { - "confidence_interval": { - "lower": -0.0489, - "upper": 0.0489, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Burmese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Burmese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7325, - "uncertainty": { - "confidence_interval": { - "lower": -0.0434, - "upper": 0.0434, - "method": "unknown" - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/mistralai_mistral-nemo-base-2407.json b/data/models/mistralai_mistral-nemo-base-2407.json deleted file mode 100644 index 594e301b30a0041693d08127f171a51200b37012..0000000000000000000000000000000000000000 --- a/data/models/mistralai_mistral-nemo-base-2407.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mistral-Nemo-Base-2407", - "id": "mistralai/Mistral-Nemo-Base-2407", - "developer": "mistralai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "11.58" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/mistralai_Mistral-Nemo-Base-2407/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.163 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5035 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0597 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2936 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3921 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3472 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/mistralai_mistral-nemo-instruct-2407.json b/data/models/mistralai_mistral-nemo-instruct-2407.json deleted file mode 100644 index 42bf023e07dd438909cd4048aa0e765dbde89986..0000000000000000000000000000000000000000 --- a/data/models/mistralai_mistral-nemo-instruct-2407.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mistral-Nemo-Instruct-2407", - "id": "mistralai/Mistral-Nemo-Instruct-2407", - "developer": "mistralai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/mistralai_Mistral-Nemo-Instruct-2407/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.638 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5037 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1269 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2903 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.39 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3517 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/mistralai_mistral-small-2402.json b/data/models/mistralai_mistral-small-2402.json deleted file mode 100644 index e447902d0b3d9b542a7a5084edbc9519e07c35b8..0000000000000000000000000000000000000000 --- a/data/models/mistralai_mistral-small-2402.json +++ /dev/null @@ -1,1901 +0,0 @@ -{ - "model_info": { - "name": "Mistral Small 2402", - "id": "mistralai/mistral-small-2402", - "developer": "mistralai", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_lite/mistralai_mistral-small-2402/1774096306.427425", - "retrieved_timestamp": "1774096306.427425", - "source_metadata": { - "source_name": "helm_lite", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_lite", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_lite", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.288, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.39283395755305867\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.519, - "details": { - "description": "min=0.519, mean=0.519, max=0.519, sum=0.519 (1)", - "tab": "Accuracy", - "NarrativeQA - Observed inference time (s)": "{\"description\": \"min=0.948, mean=0.948, max=0.948, sum=0.948 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.947719474577568\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=3955.33, mean=3955.33, max=3955.33, sum=3955.33 (1)\", \"tab\": \"General information\", \"score\": \"3955.3295774647886\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=21.775, mean=21.775, max=21.775, sum=21.775 (1)\", \"tab\": \"General information\", \"score\": \"21.774647887323944\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (closed-book)", - "source_data": { - "dataset_name": "NaturalQuestions (closed-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (closed-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.304, - "details": { - "description": "min=0.304, mean=0.304, max=0.304, sum=0.304 (1)", - "tab": "Accuracy", - "NaturalQuestions (open-book) - Observed inference time (s)": "{\"description\": \"min=1.384, mean=1.384, max=1.384, sum=1.384 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.384453837633133\"}", - "NaturalQuestions (closed-book) - Observed inference time (s)": "{\"description\": \"min=1.442, mean=1.442, max=1.442, sum=1.442 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.4422871778011321\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=2537.79, mean=2537.79, max=2537.79, sum=2537.79 (1)\", \"tab\": \"General information\", \"score\": \"2537.79\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=21.017, mean=21.017, max=21.017, sum=21.017 (1)\", \"tab\": \"General information\", \"score\": \"21.017\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=211.069, mean=211.069, max=211.069, sum=211.069 (1)\", \"tab\": \"General information\", \"score\": \"211.069\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=27.473, mean=27.473, max=27.473, sum=27.473 (1)\", \"tab\": \"General information\", \"score\": \"27.473\"}" - } - }, - "generation_config": { - "additional_details": { - "mode": "\"closedbook\"" - } - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.862, - "details": { - "description": "min=0.862, mean=0.862, max=0.862, sum=0.862 (1)", - "tab": "Accuracy", - "OpenbookQA - Observed inference time (s)": "{\"description\": \"min=0.53, mean=0.53, max=0.53, sum=0.53 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.5299914984703064\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=280.15, mean=280.15, max=280.15, sum=280.15 (1)\", \"tab\": \"General information\", \"score\": \"280.15\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "dataset": "\"openbookqa\"", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.593, - "details": { - "description": "min=0.26, mean=0.593, max=0.89, sum=2.964 (5)", - "tab": "Accuracy", - "MMLU - Observed inference time (s)": "{\"description\": \"min=1.022, mean=1.262, max=1.477, sum=6.308 (5)\", \"tab\": \"Efficiency\", \"score\": \"1.2616501861371492\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=402.44, mean=523.091, max=687.175, sum=2615.455 (5)\", \"tab\": \"General information\", \"score\": \"523.0910877192983\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"college_chemistry\", \"computer_security\", \"econometrics\", \"us_foreign_policy\"]", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MATH", - "source_data": { - "dataset_name": "MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Equivalent (CoT) on MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.621, - "details": { - "description": "min=0.367, mean=0.621, max=0.859, sum=4.344 (7)", - "tab": "Accuracy", - "MATH - Observed inference time (s)": "{\"description\": \"min=1.895, mean=2.217, max=2.662, sum=15.518 (7)\", \"tab\": \"Efficiency\", \"score\": \"2.216904607788028\"}", - "MATH - # eval": "{\"description\": \"min=30, mean=62.429, max=135, sum=437 (7)\", \"tab\": \"General information\", \"score\": \"62.42857142857143\"}", - "MATH - # train": "{\"description\": \"min=8, mean=8, max=8, sum=56 (7)\", \"tab\": \"General information\", \"score\": \"8.0\"}", - "MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MATH - # prompt tokens": "{\"description\": \"min=1061.615, mean=1525.266, max=2572.962, sum=10676.865 (7)\", \"tab\": \"General information\", \"score\": \"1525.2664139976257\"}", - "MATH - # output tokens": "{\"description\": \"min=104.221, mean=125.526, max=154.904, sum=878.68 (7)\", \"tab\": \"General information\", \"score\": \"125.52572529016837\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"algebra\", \"counting_and_probability\", \"geometry\", \"intermediate_algebra\", \"number_theory\", \"prealgebra\", \"precalculus\"]", - "level": "\"1\"", - "use_official_examples": "\"False\"", - "use_chain_of_thought": "\"True\"" - } - } - }, - { - "evaluation_name": "GSM8K", - "source_data": { - "dataset_name": "GSM8K", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on GSM8K", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.734, - "details": { - "description": "min=0.734, mean=0.734, max=0.734, sum=0.734 (1)", - "tab": "Accuracy", - "GSM8K - Observed inference time (s)": "{\"description\": \"min=2.972, mean=2.972, max=2.972, sum=2.972 (1)\", \"tab\": \"Efficiency\", \"score\": \"2.9720949590206147\"}", - "GSM8K - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "GSM8K - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "GSM8K - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GSM8K - # prompt tokens": "{\"description\": \"min=1255.268, mean=1255.268, max=1255.268, sum=1255.268 (1)\", \"tab\": \"General information\", \"score\": \"1255.268\"}", - "GSM8K - # output tokens": "{\"description\": \"min=148.06, mean=148.06, max=148.06, sum=148.06 (1)\", \"tab\": \"General information\", \"score\": \"148.06\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "LegalBench", - "source_data": { - "dataset_name": "LegalBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on LegalBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.389, - "details": { - "description": "min=0, mean=0.389, max=0.789, sum=1.947 (5)", - "tab": "Accuracy", - "LegalBench - Observed inference time (s)": "{\"description\": \"min=0.609, mean=0.874, max=1.067, sum=4.369 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.8738773620338431\"}", - "LegalBench - # eval": "{\"description\": \"min=50, mean=312.4, max=1000, sum=1562 (5)\", \"tab\": \"General information\", \"score\": \"312.4\"}", - "LegalBench - # train": "{\"description\": \"min=4, mean=4.8, max=5, sum=24 (5)\", \"tab\": \"General information\", \"score\": \"4.8\"}", - "LegalBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "LegalBench - # prompt tokens": "{\"description\": \"min=219.46, mean=1783.252, max=7251, sum=8916.261 (5)\", \"tab\": \"General information\", \"score\": \"1783.2521685070988\"}", - "LegalBench - # output tokens": "{\"description\": \"min=1.716, mean=12.778, max=30, sum=63.891 (5)\", \"tab\": \"General information\", \"score\": \"12.778290319804961\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"abercrombie\", \"corporate_lobbying\", \"function_of_decision_section\", \"international_citizenship_questions\", \"proa\"]" - } - } - }, - { - "evaluation_name": "MedQA", - "source_data": { - "dataset_name": "MedQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MedQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.616, - "details": { - "description": "min=0.616, mean=0.616, max=0.616, sum=0.616 (1)", - "tab": "Accuracy", - "MedQA - Observed inference time (s)": "{\"description\": \"min=0.499, mean=0.499, max=0.499, sum=0.499 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.4987720272413068\"}", - "MedQA - # eval": "{\"description\": \"min=503, mean=503, max=503, sum=503 (1)\", \"tab\": \"General information\", \"score\": \"503.0\"}", - "MedQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MedQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MedQA - # prompt tokens": "{\"description\": \"min=1193.093, mean=1193.093, max=1193.093, sum=1193.093 (1)\", \"tab\": \"General information\", \"score\": \"1193.0934393638172\"}", - "MedQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WMT 2014", - "source_data": { - "dataset_name": "WMT 2014", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "BLEU-4 on WMT 2014", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.169, - "details": { - "description": "min=0.076, mean=0.169, max=0.215, sum=0.843 (5)", - "tab": "Accuracy", - "WMT 2014 - Observed inference time (s)": "{\"description\": \"min=0.945, mean=1.189, max=1.429, sum=5.943 (5)\", \"tab\": \"Efficiency\", \"score\": \"1.1885517670659458\"}", - "WMT 2014 - # eval": "{\"description\": \"min=503, mean=568.8, max=832, sum=2844 (5)\", \"tab\": \"General information\", \"score\": \"568.8\"}", - "WMT 2014 - # train": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "WMT 2014 - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WMT 2014 - # prompt tokens": "{\"description\": \"min=198.306, mean=212.433, max=231.018, sum=1062.166 (5)\", \"tab\": \"General information\", \"score\": \"212.43317355482492\"}", - "WMT 2014 - # output tokens": "{\"description\": \"min=26.479, mean=28.3, max=29.024, sum=141.498 (5)\", \"tab\": \"General information\", \"score\": \"28.29957084416578\"}" - } - }, - "generation_config": { - "additional_details": { - "language_pair": "[\"cs-en\", \"de-en\", \"fr-en\", \"hi-en\", \"ru-en\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_mmlu/mistralai_mistral-small-2402/1774096312.00548", - "retrieved_timestamp": "1774096312.00548", - "source_metadata": { - "source_name": "helm_mmlu", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_mmlu", - "evaluation_results": [ - { - "evaluation_name": "MMLU All Subjects", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU All Subjects", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.687, - "details": { - "description": "min=0.215, mean=0.687, max=0.948, sum=78.352 (114)", - "tab": "Accuracy", - "MMLU All Subjects - Observed inference time (s)": "{\"description\": \"min=0.279, mean=0.486, max=1.477, sum=55.362 (114)\", \"tab\": \"Efficiency\", \"score\": \"0.4856315259373381\"}", - "MMLU All Subjects - # eval": "{\"description\": \"min=100, mean=246.351, max=1534, sum=28084 (114)\", \"tab\": \"General information\", \"score\": \"246.35087719298247\"}", - "MMLU All Subjects - # train": "{\"description\": \"min=5, mean=5, max=5, sum=570 (114)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU All Subjects - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU All Subjects - # prompt tokens": "{\"description\": \"min=308.924, mean=696.273, max=3089.109, sum=79375.178 (114)\", \"tab\": \"General information\", \"score\": \"696.2734899593811\"}", - "MMLU All Subjects - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=114 (114)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_name": "Abstract Algebra", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Abstract Algebra", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.26, - "details": { - "description": "min=0.26, mean=0.26, max=0.26, sum=0.52 (2)", - "tab": "Accuracy", - "Abstract Algebra - Observed inference time (s)": "{\"description\": \"min=1.31, mean=1.31, max=1.31, sum=2.621 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.3102962040901185\"}", - "Abstract Algebra - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Abstract Algebra - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Abstract Algebra - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Abstract Algebra - # prompt tokens": "{\"description\": \"min=402.44, mean=402.44, max=402.44, sum=804.88 (2)\", \"tab\": \"General information\", \"score\": \"402.44\"}", - "Abstract Algebra - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"abstract_algebra\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_abstract_algebra\"" - } - } - }, - { - "evaluation_name": "Anatomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Anatomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.674, - "details": { - "description": "min=0.674, mean=0.674, max=0.674, sum=1.348 (2)", - "tab": "Accuracy", - "Anatomy - Observed inference time (s)": "{\"description\": \"min=0.359, mean=0.359, max=0.359, sum=0.719 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.35931493441263834\"}", - "Anatomy - # eval": "{\"description\": \"min=135, mean=135, max=135, sum=270 (2)\", \"tab\": \"General information\", \"score\": \"135.0\"}", - "Anatomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Anatomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Anatomy - # prompt tokens": "{\"description\": \"min=407.089, mean=407.089, max=407.089, sum=814.178 (2)\", \"tab\": \"General information\", \"score\": \"407.0888888888889\"}", - "Anatomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"anatomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_anatomy\"" - } - } - }, - { - "evaluation_name": "College Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on College Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.402, - "details": { - "description": "min=0.402, mean=0.402, max=0.402, sum=0.804 (2)", - "tab": "Accuracy", - "College Chemistry - Observed inference time (s)": "{\"description\": \"min=1.152, mean=1.152, max=1.152, sum=2.304 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.151910934448242\"}", - "College Biology - Observed inference time (s)": "{\"description\": \"min=0.358, mean=0.358, max=0.358, sum=0.716 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3582056214412053\"}", - "College Computer Science - Observed inference time (s)": "{\"description\": \"min=0.295, mean=0.295, max=0.295, sum=0.59 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.29487616300582886\"}", - "College Mathematics - Observed inference time (s)": "{\"description\": \"min=0.448, mean=0.448, max=0.448, sum=0.896 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.44812692165374757\"}", - "College Medicine - Observed inference time (s)": "{\"description\": \"min=0.367, mean=0.367, max=0.367, sum=0.734 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3668311620723305\"}", - "College Physics - Observed inference time (s)": "{\"description\": \"min=0.375, mean=0.375, max=0.375, sum=0.75 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.37511497852849024\"}", - "College Chemistry - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Chemistry - # prompt tokens": "{\"description\": \"min=627.71, mean=627.71, max=627.71, sum=1255.42 (2)\", \"tab\": \"General information\", \"score\": \"627.71\"}", - "College Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Biology - # eval": "{\"description\": \"min=144, mean=144, max=144, sum=288 (2)\", \"tab\": \"General information\", \"score\": \"144.0\"}", - "College Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # prompt tokens": "{\"description\": \"min=550.799, mean=550.799, max=550.799, sum=1101.597 (2)\", \"tab\": \"General information\", \"score\": \"550.7986111111111\"}", - "College Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # prompt tokens": "{\"description\": \"min=902.17, mean=902.17, max=902.17, sum=1804.34 (2)\", \"tab\": \"General information\", \"score\": \"902.17\"}", - "College Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Mathematics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # prompt tokens": "{\"description\": \"min=658.31, mean=658.31, max=658.31, sum=1316.62 (2)\", \"tab\": \"General information\", \"score\": \"658.31\"}", - "College Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Medicine - # eval": "{\"description\": \"min=173, mean=173, max=173, sum=346 (2)\", \"tab\": \"General information\", \"score\": \"173.0\"}", - "College Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # prompt tokens": "{\"description\": \"min=592.41, mean=592.41, max=592.41, sum=1184.821 (2)\", \"tab\": \"General information\", \"score\": \"592.4104046242775\"}", - "College Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Physics - # eval": "{\"description\": \"min=102, mean=102, max=102, sum=204 (2)\", \"tab\": \"General information\", \"score\": \"102.0\"}", - "College Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # prompt tokens": "{\"description\": \"min=551.029, mean=551.029, max=551.029, sum=1102.059 (2)\", \"tab\": \"General information\", \"score\": \"551.0294117647059\"}", - "College Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"college_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_college_physics\"" - } - } - }, - { - "evaluation_name": "Computer Security", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Computer Security", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.77, - "details": { - "description": "min=0.77, mean=0.77, max=0.77, sum=1.54 (2)", - "tab": "Accuracy", - "Computer Security - Observed inference time (s)": "{\"description\": \"min=1.022, mean=1.022, max=1.022, sum=2.044 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.0222336649894714\"}", - "Computer Security - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Computer Security - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Computer Security - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Computer Security - # prompt tokens": "{\"description\": \"min=424.94, mean=424.94, max=424.94, sum=849.88 (2)\", \"tab\": \"General information\", \"score\": \"424.94\"}", - "Computer Security - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"computer_security\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_computer_security\"" - } - } - }, - { - "evaluation_name": "Econometrics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Econometrics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.614, - "details": { - "description": "min=0.614, mean=0.614, max=0.614, sum=1.228 (2)", - "tab": "Accuracy", - "Econometrics - Observed inference time (s)": "{\"description\": \"min=1.477, mean=1.477, max=1.477, sum=2.954 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.4771089867541665\"}", - "Econometrics - # eval": "{\"description\": \"min=114, mean=114, max=114, sum=228 (2)\", \"tab\": \"General information\", \"score\": \"114.0\"}", - "Econometrics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Econometrics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Econometrics - # prompt tokens": "{\"description\": \"min=687.175, mean=687.175, max=687.175, sum=1374.351 (2)\", \"tab\": \"General information\", \"score\": \"687.1754385964912\"}", - "Econometrics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"econometrics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_econometrics\"" - } - } - }, - { - "evaluation_name": "Global Facts", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Global Facts", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.45, - "details": { - "description": "min=0.45, mean=0.45, max=0.45, sum=0.9 (2)", - "tab": "Accuracy", - "Global Facts - Observed inference time (s)": "{\"description\": \"min=0.364, mean=0.364, max=0.364, sum=0.728 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.36384799242019655\"}", - "Global Facts - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Global Facts - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Global Facts - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Global Facts - # prompt tokens": "{\"description\": \"min=483.47, mean=483.47, max=483.47, sum=966.94 (2)\", \"tab\": \"General information\", \"score\": \"483.47\"}", - "Global Facts - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"global_facts\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_global_facts\"" - } - } - }, - { - "evaluation_name": "Jurisprudence", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Jurisprudence", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.833, - "details": { - "description": "min=0.833, mean=0.833, max=0.833, sum=1.667 (2)", - "tab": "Accuracy", - "Jurisprudence - Observed inference time (s)": "{\"description\": \"min=0.435, mean=0.435, max=0.435, sum=0.871 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4353830130011947\"}", - "Jurisprudence - # eval": "{\"description\": \"min=108, mean=108, max=108, sum=216 (2)\", \"tab\": \"General information\", \"score\": \"108.0\"}", - "Jurisprudence - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Jurisprudence - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Jurisprudence - # prompt tokens": "{\"description\": \"min=451.093, mean=451.093, max=451.093, sum=902.185 (2)\", \"tab\": \"General information\", \"score\": \"451.0925925925926\"}", - "Jurisprudence - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"jurisprudence\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_jurisprudence\"" - } - } - }, - { - "evaluation_name": "Philosophy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Philosophy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.765, - "details": { - "description": "min=0.765, mean=0.765, max=0.765, sum=1.531 (2)", - "tab": "Accuracy", - "Philosophy - Observed inference time (s)": "{\"description\": \"min=0.438, mean=0.438, max=0.438, sum=0.877 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.43847233306173344\"}", - "Philosophy - # eval": "{\"description\": \"min=311, mean=311, max=311, sum=622 (2)\", \"tab\": \"General information\", \"score\": \"311.0\"}", - "Philosophy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Philosophy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Philosophy - # prompt tokens": "{\"description\": \"min=373.82, mean=373.82, max=373.82, sum=747.64 (2)\", \"tab\": \"General information\", \"score\": \"373.81993569131834\"}", - "Philosophy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"philosophy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_philosophy\"" - } - } - }, - { - "evaluation_name": "Professional Psychology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Professional Psychology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.768, - "details": { - "description": "min=0.768, mean=0.768, max=0.768, sum=1.536 (2)", - "tab": "Accuracy", - "Professional Medicine - Observed inference time (s)": "{\"description\": \"min=0.459, mean=0.459, max=0.459, sum=0.919 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.45927367666188407\"}", - "Professional Accounting - Observed inference time (s)": "{\"description\": \"min=0.447, mean=0.447, max=0.447, sum=0.895 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.447448378759073\"}", - "Professional Law - Observed inference time (s)": "{\"description\": \"min=0.408, mean=0.408, max=0.408, sum=0.816 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.407953996390998\"}", - "Professional Psychology - Observed inference time (s)": "{\"description\": \"min=0.42, mean=0.42, max=0.42, sum=0.839 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.41963181386586107\"}", - "Professional Medicine - # eval": "{\"description\": \"min=272, mean=272, max=272, sum=544 (2)\", \"tab\": \"General information\", \"score\": \"272.0\"}", - "Professional Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Medicine - # prompt tokens": "{\"description\": \"min=1279.143, mean=1279.143, max=1279.143, sum=2558.287 (2)\", \"tab\": \"General information\", \"score\": \"1279.1433823529412\"}", - "Professional Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Accounting - # eval": "{\"description\": \"min=282, mean=282, max=282, sum=564 (2)\", \"tab\": \"General information\", \"score\": \"282.0\"}", - "Professional Accounting - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Accounting - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # prompt tokens": "{\"description\": \"min=796.496, mean=796.496, max=796.496, sum=1592.993 (2)\", \"tab\": \"General information\", \"score\": \"796.4964539007092\"}", - "Professional Accounting - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Law - # eval": "{\"description\": \"min=1534, mean=1534, max=1534, sum=3068 (2)\", \"tab\": \"General information\", \"score\": \"1534.0\"}", - "Professional Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # prompt tokens": "{\"description\": \"min=1849.711, mean=1849.711, max=1849.711, sum=3699.421 (2)\", \"tab\": \"General information\", \"score\": \"1849.7105606258149\"}", - "Professional Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Psychology - # eval": "{\"description\": \"min=612, mean=612, max=612, sum=1224 (2)\", \"tab\": \"General information\", \"score\": \"612.0\"}", - "Professional Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # prompt tokens": "{\"description\": \"min=645.278, mean=645.278, max=645.278, sum=1290.556 (2)\", \"tab\": \"General information\", \"score\": \"645.2777777777778\"}", - "Professional Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"professional_psychology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_professional_psychology\"" - } - } - }, - { - "evaluation_name": "Us Foreign Policy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Us Foreign Policy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.89, - "details": { - "description": "min=0.89, mean=0.89, max=0.89, sum=1.78 (2)", - "tab": "Accuracy", - "Us Foreign Policy - Observed inference time (s)": "{\"description\": \"min=1.347, mean=1.347, max=1.347, sum=2.693 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.3467011404037477\"}", - "Us Foreign Policy - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Us Foreign Policy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Us Foreign Policy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Us Foreign Policy - # prompt tokens": "{\"description\": \"min=473.19, mean=473.19, max=473.19, sum=946.38 (2)\", \"tab\": \"General information\", \"score\": \"473.19\"}", - "Us Foreign Policy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"us_foreign_policy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_us_foreign_policy\"" - } - } - }, - { - "evaluation_name": "Astronomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Astronomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.77, - "details": { - "description": "min=0.77, mean=0.77, max=0.77, sum=1.539 (2)", - "tab": "Accuracy", - "Astronomy - Observed inference time (s)": "{\"description\": \"min=0.345, mean=0.345, max=0.345, sum=0.689 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3447367345031939\"}", - "Astronomy - # eval": "{\"description\": \"min=152, mean=152, max=152, sum=304 (2)\", \"tab\": \"General information\", \"score\": \"152.0\"}", - "Astronomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Astronomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Astronomy - # prompt tokens": "{\"description\": \"min=665.987, mean=665.987, max=665.987, sum=1331.974 (2)\", \"tab\": \"General information\", \"score\": \"665.9868421052631\"}", - "Astronomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"astronomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_astronomy\"" - } - } - }, - { - "evaluation_name": "Business Ethics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Business Ethics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.71, - "details": { - "description": "min=0.71, mean=0.71, max=0.71, sum=1.42 (2)", - "tab": "Accuracy", - "Business Ethics - Observed inference time (s)": "{\"description\": \"min=0.45, mean=0.45, max=0.45, sum=0.9 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4499172067642212\"}", - "Business Ethics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Business Ethics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Business Ethics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Business Ethics - # prompt tokens": "{\"description\": \"min=644.6, mean=644.6, max=644.6, sum=1289.2 (2)\", \"tab\": \"General information\", \"score\": \"644.6\"}", - "Business Ethics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"business_ethics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_business_ethics\"" - } - } - }, - { - "evaluation_name": "Clinical Knowledge", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Clinical Knowledge", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.766, - "details": { - "description": "min=0.766, mean=0.766, max=0.766, sum=1.532 (2)", - "tab": "Accuracy", - "Clinical Knowledge - Observed inference time (s)": "{\"description\": \"min=0.436, mean=0.436, max=0.436, sum=0.873 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4363225082181535\"}", - "Clinical Knowledge - # eval": "{\"description\": \"min=265, mean=265, max=265, sum=530 (2)\", \"tab\": \"General information\", \"score\": \"265.0\"}", - "Clinical Knowledge - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Clinical Knowledge - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Clinical Knowledge - # prompt tokens": "{\"description\": \"min=487.174, mean=487.174, max=487.174, sum=974.347 (2)\", \"tab\": \"General information\", \"score\": \"487.1735849056604\"}", - "Clinical Knowledge - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"clinical_knowledge\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_clinical_knowledge\"" - } - } - }, - { - "evaluation_name": "Conceptual Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Conceptual Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.685, - "details": { - "description": "min=0.685, mean=0.685, max=0.685, sum=1.37 (2)", - "tab": "Accuracy", - "Conceptual Physics - Observed inference time (s)": "{\"description\": \"min=0.391, mean=0.391, max=0.391, sum=0.781 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3906106086487466\"}", - "Conceptual Physics - # eval": "{\"description\": \"min=235, mean=235, max=235, sum=470 (2)\", \"tab\": \"General information\", \"score\": \"235.0\"}", - "Conceptual Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Conceptual Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Conceptual Physics - # prompt tokens": "{\"description\": \"min=334.285, mean=334.285, max=334.285, sum=668.57 (2)\", \"tab\": \"General information\", \"score\": \"334.2851063829787\"}", - "Conceptual Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"conceptual_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_conceptual_physics\"" - } - } - }, - { - "evaluation_name": "Electrical Engineering", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Electrical Engineering", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.628, - "details": { - "description": "min=0.628, mean=0.628, max=0.628, sum=1.255 (2)", - "tab": "Accuracy", - "Electrical Engineering - Observed inference time (s)": "{\"description\": \"min=0.434, mean=0.434, max=0.434, sum=0.868 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4342194343435353\"}", - "Electrical Engineering - # eval": "{\"description\": \"min=145, mean=145, max=145, sum=290 (2)\", \"tab\": \"General information\", \"score\": \"145.0\"}", - "Electrical Engineering - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Electrical Engineering - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Electrical Engineering - # prompt tokens": "{\"description\": \"min=501.379, mean=501.379, max=501.379, sum=1002.759 (2)\", \"tab\": \"General information\", \"score\": \"501.37931034482756\"}", - "Electrical Engineering - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"electrical_engineering\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_electrical_engineering\"" - } - } - }, - { - "evaluation_name": "Elementary Mathematics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Elementary Mathematics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.415, - "details": { - "description": "min=0.415, mean=0.415, max=0.415, sum=0.831 (2)", - "tab": "Accuracy", - "Elementary Mathematics - Observed inference time (s)": "{\"description\": \"min=0.434, mean=0.434, max=0.434, sum=0.869 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.43446689244931336\"}", - "Elementary Mathematics - # eval": "{\"description\": \"min=378, mean=378, max=378, sum=756 (2)\", \"tab\": \"General information\", \"score\": \"378.0\"}", - "Elementary Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Elementary Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Elementary Mathematics - # prompt tokens": "{\"description\": \"min=613.386, mean=613.386, max=613.386, sum=1226.772 (2)\", \"tab\": \"General information\", \"score\": \"613.3862433862433\"}", - "Elementary Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"elementary_mathematics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_elementary_mathematics\"" - } - } - }, - { - "evaluation_name": "Formal Logic", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Formal Logic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.516, - "details": { - "description": "min=0.516, mean=0.516, max=0.516, sum=1.032 (2)", - "tab": "Accuracy", - "Formal Logic - Observed inference time (s)": "{\"description\": \"min=0.443, mean=0.443, max=0.443, sum=0.887 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4434795303950234\"}", - "Formal Logic - # eval": "{\"description\": \"min=126, mean=126, max=126, sum=252 (2)\", \"tab\": \"General information\", \"score\": \"126.0\"}", - "Formal Logic - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Formal Logic - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Formal Logic - # prompt tokens": "{\"description\": \"min=718.984, mean=718.984, max=718.984, sum=1437.968 (2)\", \"tab\": \"General information\", \"score\": \"718.984126984127\"}", - "Formal Logic - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"formal_logic\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_formal_logic\"" - } - } - }, - { - "evaluation_name": "High School World History", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on High School World History", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.857, - "details": { - "description": "min=0.857, mean=0.857, max=0.857, sum=1.713 (2)", - "tab": "Accuracy", - "High School Biology - Observed inference time (s)": "{\"description\": \"min=0.374, mean=0.374, max=0.374, sum=0.749 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3742693070442446\"}", - "High School Chemistry - Observed inference time (s)": "{\"description\": \"min=0.384, mean=0.384, max=0.384, sum=0.768 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3839088602019061\"}", - "High School Computer Science - Observed inference time (s)": "{\"description\": \"min=0.423, mean=0.423, max=0.423, sum=0.846 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4230046820640564\"}", - "High School European History - Observed inference time (s)": "{\"description\": \"min=0.456, mean=0.456, max=0.456, sum=0.911 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4556852485194351\"}", - "High School Geography - Observed inference time (s)": "{\"description\": \"min=0.443, mean=0.443, max=0.443, sum=0.885 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.44265695533367116\"}", - "High School Government And Politics - Observed inference time (s)": "{\"description\": \"min=0.48, mean=0.48, max=0.48, sum=0.96 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.47987033666106704\"}", - "High School Macroeconomics - Observed inference time (s)": "{\"description\": \"min=0.366, mean=0.366, max=0.366, sum=0.731 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3655165384977292\"}", - "High School Mathematics - Observed inference time (s)": "{\"description\": \"min=0.433, mean=0.433, max=0.433, sum=0.865 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4325918674468994\"}", - "High School Microeconomics - Observed inference time (s)": "{\"description\": \"min=0.415, mean=0.415, max=0.415, sum=0.83 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.41513349929777515\"}", - "High School Physics - Observed inference time (s)": "{\"description\": \"min=0.417, mean=0.417, max=0.417, sum=0.834 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.41723605496993915\"}", - "High School Psychology - Observed inference time (s)": "{\"description\": \"min=0.448, mean=0.448, max=0.448, sum=0.896 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.44808799017459977\"}", - "High School Statistics - Observed inference time (s)": "{\"description\": \"min=0.402, mean=0.402, max=0.402, sum=0.805 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4024901666023113\"}", - "High School US History - Observed inference time (s)": "{\"description\": \"min=0.383, mean=0.383, max=0.383, sum=0.767 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3834606175329171\"}", - "High School World History - Observed inference time (s)": "{\"description\": \"min=0.399, mean=0.399, max=0.399, sum=0.798 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.39886615648551327\"}", - "High School Biology - # eval": "{\"description\": \"min=310, mean=310, max=310, sum=620 (2)\", \"tab\": \"General information\", \"score\": \"310.0\"}", - "High School Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Biology - # prompt tokens": "{\"description\": \"min=600.561, mean=600.561, max=600.561, sum=1201.123 (2)\", \"tab\": \"General information\", \"score\": \"600.5612903225806\"}", - "High School Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Chemistry - # eval": "{\"description\": \"min=203, mean=203, max=203, sum=406 (2)\", \"tab\": \"General information\", \"score\": \"203.0\"}", - "High School Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # prompt tokens": "{\"description\": \"min=572.798, mean=572.798, max=572.798, sum=1145.596 (2)\", \"tab\": \"General information\", \"score\": \"572.7980295566502\"}", - "High School Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "High School Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # prompt tokens": "{\"description\": \"min=988.24, mean=988.24, max=988.24, sum=1976.48 (2)\", \"tab\": \"General information\", \"score\": \"988.24\"}", - "High School Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School European History - # eval": "{\"description\": \"min=165, mean=165, max=165, sum=330 (2)\", \"tab\": \"General information\", \"score\": \"165.0\"}", - "High School European History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School European History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # prompt tokens": "{\"description\": \"min=3089.109, mean=3089.109, max=3089.109, sum=6178.218 (2)\", \"tab\": \"General information\", \"score\": \"3089.109090909091\"}", - "High School European History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Geography - # eval": "{\"description\": \"min=198, mean=198, max=198, sum=396 (2)\", \"tab\": \"General information\", \"score\": \"198.0\"}", - "High School Geography - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Geography - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # prompt tokens": "{\"description\": \"min=429.207, mean=429.207, max=429.207, sum=858.414 (2)\", \"tab\": \"General information\", \"score\": \"429.2070707070707\"}", - "High School Geography - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Government And Politics - # eval": "{\"description\": \"min=193, mean=193, max=193, sum=386 (2)\", \"tab\": \"General information\", \"score\": \"193.0\"}", - "High School Government And Politics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Government And Politics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # prompt tokens": "{\"description\": \"min=514.808, mean=514.808, max=514.808, sum=1029.617 (2)\", \"tab\": \"General information\", \"score\": \"514.8082901554404\"}", - "High School Government And Politics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Macroeconomics - # eval": "{\"description\": \"min=390, mean=390, max=390, sum=780 (2)\", \"tab\": \"General information\", \"score\": \"390.0\"}", - "High School Macroeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Macroeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # prompt tokens": "{\"description\": \"min=423.815, mean=423.815, max=423.815, sum=847.631 (2)\", \"tab\": \"General information\", \"score\": \"423.81538461538463\"}", - "High School Macroeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Mathematics - # eval": "{\"description\": \"min=270, mean=270, max=270, sum=540 (2)\", \"tab\": \"General information\", \"score\": \"270.0\"}", - "High School Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # prompt tokens": "{\"description\": \"min=584.13, mean=584.13, max=584.13, sum=1168.259 (2)\", \"tab\": \"General information\", \"score\": \"584.1296296296297\"}", - "High School Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Microeconomics - # eval": "{\"description\": \"min=238, mean=238, max=238, sum=476 (2)\", \"tab\": \"General information\", \"score\": \"238.0\"}", - "High School Microeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Microeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # prompt tokens": "{\"description\": \"min=443.345, mean=443.345, max=443.345, sum=886.689 (2)\", \"tab\": \"General information\", \"score\": \"443.34453781512605\"}", - "High School Microeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Physics - # eval": "{\"description\": \"min=151, mean=151, max=151, sum=302 (2)\", \"tab\": \"General information\", \"score\": \"151.0\"}", - "High School Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # prompt tokens": "{\"description\": \"min=622.775, mean=622.775, max=622.775, sum=1245.55 (2)\", \"tab\": \"General information\", \"score\": \"622.774834437086\"}", - "High School Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Psychology - # eval": "{\"description\": \"min=545, mean=545, max=545, sum=1090 (2)\", \"tab\": \"General information\", \"score\": \"545.0\"}", - "High School Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # prompt tokens": "{\"description\": \"min=558.873, mean=558.873, max=558.873, sum=1117.747 (2)\", \"tab\": \"General information\", \"score\": \"558.8733944954129\"}", - "High School Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Statistics - # eval": "{\"description\": \"min=216, mean=216, max=216, sum=432 (2)\", \"tab\": \"General information\", \"score\": \"216.0\"}", - "High School Statistics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Statistics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # prompt tokens": "{\"description\": \"min=913.644, mean=913.644, max=913.644, sum=1827.287 (2)\", \"tab\": \"General information\", \"score\": \"913.6435185185185\"}", - "High School Statistics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School US History - # eval": "{\"description\": \"min=204, mean=204, max=204, sum=408 (2)\", \"tab\": \"General information\", \"score\": \"204.0\"}", - "High School US History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School US History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # prompt tokens": "{\"description\": \"min=2477.446, mean=2477.446, max=2477.446, sum=4954.892 (2)\", \"tab\": \"General information\", \"score\": \"2477.4460784313724\"}", - "High School US History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School World History - # eval": "{\"description\": \"min=237, mean=237, max=237, sum=474 (2)\", \"tab\": \"General information\", \"score\": \"237.0\"}", - "High School World History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School World History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # prompt tokens": "{\"description\": \"min=1585.553, mean=1585.553, max=1585.553, sum=3171.105 (2)\", \"tab\": \"General information\", \"score\": \"1585.5527426160338\"}", - "High School World History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"high_school_world_history\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_high_school_world_history\"" - } - } - }, - { - "evaluation_name": "Human Sexuality", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Human Sexuality", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.824, - "details": { - "description": "min=0.824, mean=0.824, max=0.824, sum=1.649 (2)", - "tab": "Accuracy", - "Human Aging - Observed inference time (s)": "{\"description\": \"min=0.332, mean=0.332, max=0.332, sum=0.664 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.33194801304907007\"}", - "Human Sexuality - Observed inference time (s)": "{\"description\": \"min=0.358, mean=0.358, max=0.358, sum=0.716 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3579711095067381\"}", - "Human Aging - # eval": "{\"description\": \"min=223, mean=223, max=223, sum=446 (2)\", \"tab\": \"General information\", \"score\": \"223.0\"}", - "Human Aging - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Aging - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Aging - # prompt tokens": "{\"description\": \"min=353.152, mean=353.152, max=353.152, sum=706.305 (2)\", \"tab\": \"General information\", \"score\": \"353.15246636771303\"}", - "Human Aging - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Human Sexuality - # eval": "{\"description\": \"min=131, mean=131, max=131, sum=262 (2)\", \"tab\": \"General information\", \"score\": \"131.0\"}", - "Human Sexuality - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Sexuality - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # prompt tokens": "{\"description\": \"min=394.748, mean=394.748, max=394.748, sum=789.496 (2)\", \"tab\": \"General information\", \"score\": \"394.7480916030534\"}", - "Human Sexuality - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"human_sexuality\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_human_sexuality\"" - } - } - }, - { - "evaluation_name": "International Law", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on International Law", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.826, - "details": { - "description": "min=0.826, mean=0.826, max=0.826, sum=1.653 (2)", - "tab": "Accuracy", - "International Law - Observed inference time (s)": "{\"description\": \"min=0.378, mean=0.378, max=0.378, sum=0.755 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.37766425668700665\"}", - "International Law - # eval": "{\"description\": \"min=121, mean=121, max=121, sum=242 (2)\", \"tab\": \"General information\", \"score\": \"121.0\"}", - "International Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "International Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "International Law - # prompt tokens": "{\"description\": \"min=720.182, mean=720.182, max=720.182, sum=1440.364 (2)\", \"tab\": \"General information\", \"score\": \"720.1818181818181\"}", - "International Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"international_law\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_international_law\"" - } - } - }, - { - "evaluation_name": "Logical Fallacies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Logical Fallacies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.804, - "details": { - "description": "min=0.804, mean=0.804, max=0.804, sum=1.607 (2)", - "tab": "Accuracy", - "Logical Fallacies - Observed inference time (s)": "{\"description\": \"min=0.39, mean=0.39, max=0.39, sum=0.781 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3902764905449803\"}", - "Logical Fallacies - # eval": "{\"description\": \"min=163, mean=163, max=163, sum=326 (2)\", \"tab\": \"General information\", \"score\": \"163.0\"}", - "Logical Fallacies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Logical Fallacies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Logical Fallacies - # prompt tokens": "{\"description\": \"min=486.779, mean=486.779, max=486.779, sum=973.558 (2)\", \"tab\": \"General information\", \"score\": \"486.77914110429447\"}", - "Logical Fallacies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"logical_fallacies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_logical_fallacies\"" - } - } - }, - { - "evaluation_name": "Machine Learning", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Machine Learning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.562, - "details": { - "description": "min=0.562, mean=0.562, max=0.562, sum=1.125 (2)", - "tab": "Accuracy", - "Machine Learning - Observed inference time (s)": "{\"description\": \"min=0.393, mean=0.393, max=0.393, sum=0.785 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3927395024469921\"}", - "Machine Learning - # eval": "{\"description\": \"min=112, mean=112, max=112, sum=224 (2)\", \"tab\": \"General information\", \"score\": \"112.0\"}", - "Machine Learning - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Machine Learning - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Machine Learning - # prompt tokens": "{\"description\": \"min=734.83, mean=734.83, max=734.83, sum=1469.661 (2)\", \"tab\": \"General information\", \"score\": \"734.8303571428571\"}", - "Machine Learning - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"machine_learning\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_machine_learning\"" - } - } - }, - { - "evaluation_name": "Management", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Management", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.786, - "details": { - "description": "min=0.786, mean=0.786, max=0.786, sum=1.573 (2)", - "tab": "Accuracy", - "Management - Observed inference time (s)": "{\"description\": \"min=0.518, mean=0.518, max=0.518, sum=1.035 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5177000564278909\"}", - "Management - # eval": "{\"description\": \"min=103, mean=103, max=103, sum=206 (2)\", \"tab\": \"General information\", \"score\": \"103.0\"}", - "Management - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Management - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Management - # prompt tokens": "{\"description\": \"min=315.359, mean=315.359, max=315.359, sum=630.718 (2)\", \"tab\": \"General information\", \"score\": \"315.3592233009709\"}", - "Management - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"management\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_management\"" - } - } - }, - { - "evaluation_name": "Marketing", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Marketing", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.906, - "details": { - "description": "min=0.906, mean=0.906, max=0.906, sum=1.812 (2)", - "tab": "Accuracy", - "Marketing - Observed inference time (s)": "{\"description\": \"min=0.425, mean=0.425, max=0.425, sum=0.85 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.42478426195617414\"}", - "Marketing - # eval": "{\"description\": \"min=234, mean=234, max=234, sum=468 (2)\", \"tab\": \"General information\", \"score\": \"234.0\"}", - "Marketing - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Marketing - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Marketing - # prompt tokens": "{\"description\": \"min=463.423, mean=463.423, max=463.423, sum=926.846 (2)\", \"tab\": \"General information\", \"score\": \"463.4230769230769\"}", - "Marketing - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"marketing\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_marketing\"" - } - } - }, - { - "evaluation_name": "Medical Genetics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Medical Genetics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.75, - "details": { - "description": "min=0.75, mean=0.75, max=0.75, sum=1.5 (2)", - "tab": "Accuracy", - "Medical Genetics - Observed inference time (s)": "{\"description\": \"min=0.279, mean=0.279, max=0.279, sum=0.557 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2786110520362854\"}", - "Medical Genetics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Medical Genetics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Medical Genetics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Medical Genetics - # prompt tokens": "{\"description\": \"min=405.71, mean=405.71, max=405.71, sum=811.42 (2)\", \"tab\": \"General information\", \"score\": \"405.71\"}", - "Medical Genetics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"medical_genetics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_medical_genetics\"" - } - } - }, - { - "evaluation_name": "Miscellaneous", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Miscellaneous", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.844, - "details": { - "description": "min=0.844, mean=0.844, max=0.844, sum=1.688 (2)", - "tab": "Accuracy", - "Miscellaneous - Observed inference time (s)": "{\"description\": \"min=0.4, mean=0.4, max=0.4, sum=0.8 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3998657326436439\"}", - "Miscellaneous - # eval": "{\"description\": \"min=783, mean=783, max=783, sum=1566 (2)\", \"tab\": \"General information\", \"score\": \"783.0\"}", - "Miscellaneous - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Miscellaneous - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Miscellaneous - # prompt tokens": "{\"description\": \"min=348.519, mean=348.519, max=348.519, sum=697.037 (2)\", \"tab\": \"General information\", \"score\": \"348.51851851851853\"}", - "Miscellaneous - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"miscellaneous\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_miscellaneous\"" - } - } - }, - { - "evaluation_name": "Moral Scenarios", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Moral Scenarios", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.575, - "details": { - "description": "min=0.575, mean=0.575, max=0.575, sum=1.151 (2)", - "tab": "Accuracy", - "Moral Disputes - Observed inference time (s)": "{\"description\": \"min=0.474, mean=0.474, max=0.474, sum=0.949 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4744071271378181\"}", - "Moral Scenarios - Observed inference time (s)": "{\"description\": \"min=0.4, mean=0.4, max=0.4, sum=0.799 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.39967524166213736\"}", - "Moral Disputes - # eval": "{\"description\": \"min=346, mean=346, max=346, sum=692 (2)\", \"tab\": \"General information\", \"score\": \"346.0\"}", - "Moral Disputes - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Disputes - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Disputes - # prompt tokens": "{\"description\": \"min=540.038, mean=540.038, max=540.038, sum=1080.075 (2)\", \"tab\": \"General information\", \"score\": \"540.0375722543353\"}", - "Moral Disputes - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Moral Scenarios - # eval": "{\"description\": \"min=895, mean=895, max=895, sum=1790 (2)\", \"tab\": \"General information\", \"score\": \"895.0\"}", - "Moral Scenarios - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Scenarios - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # prompt tokens": "{\"description\": \"min=745.516, mean=745.516, max=745.516, sum=1491.032 (2)\", \"tab\": \"General information\", \"score\": \"745.5162011173185\"}", - "Moral Scenarios - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"moral_scenarios\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_moral_scenarios\"" - } - } - }, - { - "evaluation_name": "Nutrition", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Nutrition", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.761, - "details": { - "description": "min=0.761, mean=0.761, max=0.761, sum=1.523 (2)", - "tab": "Accuracy", - "Nutrition - Observed inference time (s)": "{\"description\": \"min=0.421, mean=0.421, max=0.421, sum=0.843 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.42128828927582385\"}", - "Nutrition - # eval": "{\"description\": \"min=306, mean=306, max=306, sum=612 (2)\", \"tab\": \"General information\", \"score\": \"306.0\"}", - "Nutrition - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Nutrition - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Nutrition - # prompt tokens": "{\"description\": \"min=680.69, mean=680.69, max=680.69, sum=1361.379 (2)\", \"tab\": \"General information\", \"score\": \"680.6895424836601\"}", - "Nutrition - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"nutrition\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_nutrition\"" - } - } - }, - { - "evaluation_name": "Prehistory", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Prehistory", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.802, - "details": { - "description": "min=0.802, mean=0.802, max=0.802, sum=1.605 (2)", - "tab": "Accuracy", - "Prehistory - Observed inference time (s)": "{\"description\": \"min=0.438, mean=0.438, max=0.438, sum=0.875 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.43764398863286147\"}", - "Prehistory - # eval": "{\"description\": \"min=324, mean=324, max=324, sum=648 (2)\", \"tab\": \"General information\", \"score\": \"324.0\"}", - "Prehistory - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Prehistory - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Prehistory - # prompt tokens": "{\"description\": \"min=602.145, mean=602.145, max=602.145, sum=1204.29 (2)\", \"tab\": \"General information\", \"score\": \"602.145061728395\"}", - "Prehistory - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"prehistory\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_prehistory\"" - } - } - }, - { - "evaluation_name": "Public Relations", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Public Relations", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.773, - "details": { - "description": "min=0.773, mean=0.773, max=0.773, sum=1.545 (2)", - "tab": "Accuracy", - "Public Relations - Observed inference time (s)": "{\"description\": \"min=0.464, mean=0.464, max=0.464, sum=0.929 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.464488469470631\"}", - "Public Relations - # eval": "{\"description\": \"min=110, mean=110, max=110, sum=220 (2)\", \"tab\": \"General information\", \"score\": \"110.0\"}", - "Public Relations - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Public Relations - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Public Relations - # prompt tokens": "{\"description\": \"min=462.036, mean=462.036, max=462.036, sum=924.073 (2)\", \"tab\": \"General information\", \"score\": \"462.03636363636366\"}", - "Public Relations - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"public_relations\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_public_relations\"" - } - } - }, - { - "evaluation_name": "Security Studies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Security Studies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.788, - "details": { - "description": "min=0.788, mean=0.788, max=0.788, sum=1.576 (2)", - "tab": "Accuracy", - "Security Studies - Observed inference time (s)": "{\"description\": \"min=0.431, mean=0.431, max=0.431, sum=0.862 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.43111481179996414\"}", - "Security Studies - # eval": "{\"description\": \"min=245, mean=245, max=245, sum=490 (2)\", \"tab\": \"General information\", \"score\": \"245.0\"}", - "Security Studies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Security Studies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Security Studies - # prompt tokens": "{\"description\": \"min=1315.865, mean=1315.865, max=1315.865, sum=2631.731 (2)\", \"tab\": \"General information\", \"score\": \"1315.865306122449\"}", - "Security Studies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"security_studies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_security_studies\"" - } - } - }, - { - "evaluation_name": "Sociology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Sociology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.871, - "details": { - "description": "min=0.871, mean=0.871, max=0.871, sum=1.741 (2)", - "tab": "Accuracy", - "Sociology - Observed inference time (s)": "{\"description\": \"min=0.399, mean=0.399, max=0.399, sum=0.799 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3994969099908326\"}", - "Sociology - # eval": "{\"description\": \"min=201, mean=201, max=201, sum=402 (2)\", \"tab\": \"General information\", \"score\": \"201.0\"}", - "Sociology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Sociology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Sociology - # prompt tokens": "{\"description\": \"min=487.95, mean=487.95, max=487.95, sum=975.9 (2)\", \"tab\": \"General information\", \"score\": \"487.9502487562189\"}", - "Sociology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"sociology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_sociology\"" - } - } - }, - { - "evaluation_name": "Virology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Virology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.542, - "details": { - "description": "min=0.542, mean=0.542, max=0.542, sum=1.084 (2)", - "tab": "Accuracy", - "Virology - Observed inference time (s)": "{\"description\": \"min=0.538, mean=0.538, max=0.538, sum=1.076 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5377652975450079\"}", - "Virology - # eval": "{\"description\": \"min=166, mean=166, max=166, sum=332 (2)\", \"tab\": \"General information\", \"score\": \"166.0\"}", - "Virology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Virology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Virology - # prompt tokens": "{\"description\": \"min=395.349, mean=395.349, max=395.349, sum=790.699 (2)\", \"tab\": \"General information\", \"score\": \"395.34939759036143\"}", - "Virology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"virology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_virology\"" - } - } - }, - { - "evaluation_name": "World Religions", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on World Religions", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.848, - "details": { - "description": "min=0.848, mean=0.848, max=0.848, sum=1.696 (2)", - "tab": "Accuracy", - "World Religions - Observed inference time (s)": "{\"description\": \"min=0.404, mean=0.404, max=0.404, sum=0.809 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4042932554992319\"}", - "World Religions - # eval": "{\"description\": \"min=171, mean=171, max=171, sum=342 (2)\", \"tab\": \"General information\", \"score\": \"171.0\"}", - "World Religions - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "World Religions - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "World Religions - # prompt tokens": "{\"description\": \"min=308.924, mean=308.924, max=308.924, sum=617.848 (2)\", \"tab\": \"General information\", \"score\": \"308.92397660818716\"}", - "World Religions - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"world_religions\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_world_religions\"" - } - } - }, - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.54, - "details": { - "description": "", - "tab": "Efficiency" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/mistralai_mistral-small-24b-base-2501.json b/data/models/mistralai_mistral-small-24b-base-2501.json deleted file mode 100644 index 95401666be0afcdbfeb87e6bc37b2c3d9ed594e5..0000000000000000000000000000000000000000 --- a/data/models/mistralai_mistral-small-24b-base-2501.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mistral-Small-24B-Base-2501", - "id": "mistralai/Mistral-Small-24B-Base-2501", - "developer": "mistralai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "23.572" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/mistralai_Mistral-Small-24B-Base-2501/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1672 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6442 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1971 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3876 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4237 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5406 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/mistralai_mistral-small-2503.json b/data/models/mistralai_mistral-small-2503.json deleted file mode 100644 index 6df0d972b005ae32b060fdc4673d6092e770670f..0000000000000000000000000000000000000000 --- a/data/models/mistralai_mistral-small-2503.json +++ /dev/null @@ -1,1267 +0,0 @@ -{ - "model_info": { - "name": "mistral-small-2503", - "id": "mistralai/mistral-small-2503", - "developer": "mistralai", - "inference_platform": "unknown", - "additional_details": { - "display_name": "Mistral Small 3.1" - } - }, - "evaluations": [ - { - "evaluation_id": "global-mmlu-lite/mistralai_mistral-small-2503/1773936583.743359", - "retrieved_timestamp": "1773936583.743359", - "source_metadata": { - "source_name": "Global MMLU Lite Leaderboard", - "source_type": "documentation", - "source_organization_name": "kaggle", - "source_organization_url": "www.kaggle.com", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "kaggle kernel", - "version": "4", - "additional_details": { - "url": "https://www.kaggle.com/code/shivalikasingh95/global-mmlu-lite-sample-notebook" - } - }, - "benchmark": "global-mmlu-lite", - "evaluation_results": [ - { - "evaluation_name": "Global MMLU Lite", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Global MMLU Lite", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7852 - } - }, - { - "evaluation_name": "Culturally Sensitive", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Sensitive", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7537 - } - }, - { - "evaluation_name": "Culturally Agnostic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Agnostic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8166 - } - }, - { - "evaluation_name": "Arabic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Arabic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7875, - "uncertainty": { - "confidence_interval": { - "lower": -0.0401, - "upper": 0.0401, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "English", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - English", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8, - "uncertainty": { - "confidence_interval": { - "lower": -0.0392, - "upper": 0.0392, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Bengali", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Bengali", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7725, - "uncertainty": { - "confidence_interval": { - "lower": -0.0411, - "upper": 0.0411, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "German", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - German", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7975, - "uncertainty": { - "confidence_interval": { - "lower": -0.0394, - "upper": 0.0394, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "French", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - French", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8, - "uncertainty": { - "confidence_interval": { - "lower": -0.0392, - "upper": 0.0392, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Hindi", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Hindi", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.795, - "uncertainty": { - "confidence_interval": { - "lower": -0.0396, - "upper": 0.0396, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Indonesian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Indonesian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.785, - "uncertainty": { - "confidence_interval": { - "lower": -0.0403, - "upper": 0.0403, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Italian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Italian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.805, - "uncertainty": { - "confidence_interval": { - "lower": -0.0388, - "upper": 0.0388, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Japanese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Japanese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.77, - "uncertainty": { - "confidence_interval": { - "lower": -0.0412, - "upper": 0.0412, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Korean", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Korean", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.79, - "uncertainty": { - "confidence_interval": { - "lower": -0.0399, - "upper": 0.0399, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Portuguese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Portuguese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7925, - "uncertainty": { - "confidence_interval": { - "lower": -0.0397, - "upper": 0.0397, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Spanish", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Spanish", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7825, - "uncertainty": { - "confidence_interval": { - "lower": -0.0404, - "upper": 0.0404, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Swahili", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Swahili", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.775, - "uncertainty": { - "confidence_interval": { - "lower": -0.0409, - "upper": 0.0409, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Yoruba", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Yoruba", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.735, - "uncertainty": { - "confidence_interval": { - "lower": -0.0432, - "upper": 0.0432, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Chinese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Chinese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7925, - "uncertainty": { - "confidence_interval": { - "lower": -0.0397, - "upper": 0.0397, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Burmese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Burmese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7825, - "uncertainty": { - "confidence_interval": { - "lower": -0.0404, - "upper": 0.0404, - "method": "unknown" - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "global-mmlu-lite/mistralai_mistral-small-2503/1773936496.366405", - "retrieved_timestamp": "1773936496.366405", - "source_metadata": { - "source_name": "Global MMLU Lite Leaderboard", - "source_type": "documentation", - "source_organization_name": "kaggle", - "source_organization_url": "www.kaggle.com", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "kaggle kernel", - "version": "4", - "additional_details": { - "url": "https://www.kaggle.com/code/shivalikasingh95/global-mmlu-lite-sample-notebook" - } - }, - "benchmark": "global-mmlu-lite", - "evaluation_results": [ - { - "evaluation_name": "Global MMLU Lite", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Global MMLU Lite", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7852 - } - }, - { - "evaluation_name": "Culturally Sensitive", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Sensitive", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7537 - } - }, - { - "evaluation_name": "Culturally Agnostic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Agnostic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8166 - } - }, - { - "evaluation_name": "Arabic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Arabic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7875, - "uncertainty": { - "confidence_interval": { - "lower": -0.0401, - "upper": 0.0401, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "English", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - English", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8, - "uncertainty": { - "confidence_interval": { - "lower": -0.0392, - "upper": 0.0392, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Bengali", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Bengali", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7725, - "uncertainty": { - "confidence_interval": { - "lower": -0.0411, - "upper": 0.0411, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "German", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - German", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7975, - "uncertainty": { - "confidence_interval": { - "lower": -0.0394, - "upper": 0.0394, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "French", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - French", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8, - "uncertainty": { - "confidence_interval": { - "lower": -0.0392, - "upper": 0.0392, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Hindi", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Hindi", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.795, - "uncertainty": { - "confidence_interval": { - "lower": -0.0396, - "upper": 0.0396, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Indonesian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Indonesian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.785, - "uncertainty": { - "confidence_interval": { - "lower": -0.0403, - "upper": 0.0403, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Italian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Italian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.805, - "uncertainty": { - "confidence_interval": { - "lower": -0.0388, - "upper": 0.0388, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Japanese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Japanese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.77, - "uncertainty": { - "confidence_interval": { - "lower": -0.0412, - "upper": 0.0412, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Korean", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Korean", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.79, - "uncertainty": { - "confidence_interval": { - "lower": -0.0399, - "upper": 0.0399, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Portuguese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Portuguese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7925, - "uncertainty": { - "confidence_interval": { - "lower": -0.0397, - "upper": 0.0397, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Spanish", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Spanish", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7825, - "uncertainty": { - "confidence_interval": { - "lower": -0.0404, - "upper": 0.0404, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Swahili", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Swahili", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.775, - "uncertainty": { - "confidence_interval": { - "lower": -0.0409, - "upper": 0.0409, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Yoruba", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Yoruba", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.735, - "uncertainty": { - "confidence_interval": { - "lower": -0.0432, - "upper": 0.0432, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Chinese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Chinese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7925, - "uncertainty": { - "confidence_interval": { - "lower": -0.0397, - "upper": 0.0397, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Burmese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Burmese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7825, - "uncertainty": { - "confidence_interval": { - "lower": -0.0404, - "upper": 0.0404, - "method": "unknown" - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "helm_capabilities/mistralai_mistral-small-2503/1774096304.5056698", - "retrieved_timestamp": "1774096304.5056698", - "source_metadata": { - "source_name": "helm_capabilities", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_capabilities", - "evaluation_results": [ - { - "evaluation_name": "Mean score", - "source_data": { - "dataset_name": "helm_capabilities", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "The mean of the scores from all columns.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.558, - "details": { - "description": "", - "tab": "Accuracy", - "Mean score - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"11.791458985991488\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU-Pro", - "source_data": { - "dataset_name": "MMLU-Pro", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on MMLU-Pro", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.61, - "details": { - "description": "min=0.61, mean=0.61, max=0.61, sum=0.61 (1)", - "tab": "Accuracy", - "MMLU-Pro - Observed inference time (s)": "{\"description\": \"min=3.589, mean=3.589, max=3.589, sum=3.589 (1)\", \"tab\": \"Efficiency\", \"score\": \"3.588683393239975\"}", - "MMLU-Pro - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "MMLU-Pro - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - # prompt tokens": "{\"description\": \"min=239.367, mean=239.367, max=239.367, sum=239.367 (1)\", \"tab\": \"General information\", \"score\": \"239.367\"}", - "MMLU-Pro - # output tokens": "{\"description\": \"min=365.903, mean=365.903, max=365.903, sum=365.903 (1)\", \"tab\": \"General information\", \"score\": \"365.903\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"all\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.392, - "details": { - "description": "min=0.392, mean=0.392, max=0.392, sum=0.392 (1)", - "tab": "Accuracy", - "GPQA - Observed inference time (s)": "{\"description\": \"min=5.05, mean=5.05, max=5.05, sum=5.05 (1)\", \"tab\": \"Efficiency\", \"score\": \"5.049520614435854\"}", - "GPQA - # eval": "{\"description\": \"min=446, mean=446, max=446, sum=446 (1)\", \"tab\": \"General information\", \"score\": \"446.0\"}", - "GPQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - # prompt tokens": "{\"description\": \"min=246.54, mean=246.54, max=246.54, sum=246.54 (1)\", \"tab\": \"General information\", \"score\": \"246.5403587443946\"}", - "GPQA - # output tokens": "{\"description\": \"min=492.534, mean=492.534, max=492.534, sum=492.534 (1)\", \"tab\": \"General information\", \"score\": \"492.5336322869955\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"gpqa_main\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "IFEval Strict Acc on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.75, - "details": { - "description": "min=0.75, mean=0.75, max=0.75, sum=0.75 (1)", - "tab": "Accuracy", - "IFEval - Observed inference time (s)": "{\"description\": \"min=3.838, mean=3.838, max=3.838, sum=3.838 (1)\", \"tab\": \"Efficiency\", \"score\": \"3.837722122118345\"}", - "IFEval - # eval": "{\"description\": \"min=541, mean=541, max=541, sum=541 (1)\", \"tab\": \"General information\", \"score\": \"541.0\"}", - "IFEval - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - # prompt tokens": "{\"description\": \"min=47.237, mean=47.237, max=47.237, sum=47.237 (1)\", \"tab\": \"General information\", \"score\": \"47.2365988909427\"}", - "IFEval - # output tokens": "{\"description\": \"min=379.896, mean=379.896, max=379.896, sum=379.896 (1)\", \"tab\": \"General information\", \"score\": \"379.89648798521256\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WildBench", - "source_data": { - "dataset_name": "WildBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "WB Score on WildBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.788, - "details": { - "description": "min=0.788, mean=0.788, max=0.788, sum=0.788 (1)", - "tab": "Accuracy", - "WildBench - Observed inference time (s)": "{\"description\": \"min=12.831, mean=12.831, max=12.831, sum=12.831 (1)\", \"tab\": \"Efficiency\", \"score\": \"12.831070138692855\"}", - "WildBench - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "WildBench - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # prompt tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # output tokens": "{\"description\": \"min=938.182, mean=938.182, max=938.182, sum=938.182 (1)\", \"tab\": \"General information\", \"score\": \"938.182\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"v2\"" - } - } - }, - { - "evaluation_name": "Omni-MATH", - "source_data": { - "dataset_name": "Omni-MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Acc on Omni-MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.248, - "details": { - "description": "min=0.248, mean=0.248, max=0.248, sum=0.248 (1)", - "tab": "Accuracy", - "Omni-MATH - Observed inference time (s)": "{\"description\": \"min=33.65, mean=33.65, max=33.65, sum=33.65 (1)\", \"tab\": \"Efficiency\", \"score\": \"33.650298661470416\"}", - "Omni-MATH - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "Omni-MATH - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - # prompt tokens": "{\"description\": \"min=110.473, mean=110.473, max=110.473, sum=110.473 (1)\", \"tab\": \"General information\", \"score\": \"110.473\"}", - "Omni-MATH - # output tokens": "{\"description\": \"min=753.657, mean=753.657, max=753.657, sum=753.657 (1)\", \"tab\": \"General information\", \"score\": \"753.657\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/mistralai_mistral-small-2506-fc.json b/data/models/mistralai_mistral-small-2506-fc.json deleted file mode 100644 index 284e4f90bc094b6b12a0d5efff338a24bee59de4..0000000000000000000000000000000000000000 --- a/data/models/mistralai_mistral-small-2506-fc.json +++ /dev/null @@ -1,851 +0,0 @@ -{ - "model_info": { - "name": "Mistral-small-2506 (FC)", - "id": "mistralai/mistral-small-2506-fc", - "developer": "mistralai", - "additional_details": { - "raw_model_name": "Mistral-small-2506 (FC)", - "organization": "Mistral AI", - "license": "Proprietary", - "mode": "FC", - "model_link": "https://docs.mistral.ai/guides/model-selection/" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/mistralai/mistral-small-2506-fc/1775236112.3921459", - "retrieved_timestamp": "1775236112.3921459", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 51.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 37.15 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 5.2 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 1.48 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 18.25 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 2.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 73.6 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 38.92 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 93.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 83.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 78.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 77.28 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 69.38 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 79.39 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 75.0 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 70.83 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 11.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 17.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 6.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 10.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 12.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 31.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 37.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 25.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 18.06 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 8.39 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 14.19 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 31.61 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 87.5 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 87.94 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/mistralai_mistral-small-2506-prompt.json b/data/models/mistralai_mistral-small-2506-prompt.json deleted file mode 100644 index 573e87922e1d8a8dfbccaa77b179d4990e56d315..0000000000000000000000000000000000000000 --- a/data/models/mistralai_mistral-small-2506-prompt.json +++ /dev/null @@ -1,905 +0,0 @@ -{ - "model_info": { - "name": "Mistral-Small-2506 (Prompt)", - "id": "mistralai/mistral-small-2506-prompt", - "developer": "mistralai", - "additional_details": { - "raw_model_name": "Mistral-Small-2506 (Prompt)", - "organization": "Mistral AI", - "license": "Proprietary", - "mode": "Prompt", - "model_link": "https://docs.mistral.ai/guides/model-selection/" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/mistralai/mistral-small-2506-prompt/1775236112.3965418", - "retrieved_timestamp": "1775236112.3965418", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 59.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 32.38 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 6.91 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 0.92 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 6.79 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 2.02 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 89.69 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 78.75 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 96.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 92.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 91.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 79.05 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 81.4 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 78.54 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 93.75 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 66.67 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 14.75 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 20.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 17.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 9.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 12.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 7.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 9.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 6.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 15.05 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 2.58 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 11.61 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 30.97 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 93.75 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 65.73 - } - }, - { - "evaluation_result_id": "bfcl.format_sensitivity.max_delta::max_delta", - "evaluation_name": "bfcl.format_sensitivity.max_delta", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.format_sensitivity.max_delta", - "metric_name": "Format sensitivity max delta", - "metric_kind": "difference", - "metric_unit": "percentage_points", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Format Sensitivity Max Delta" - } - }, - "score_details": { - "score": 50.0 - } - }, - { - "evaluation_result_id": "bfcl.format_sensitivity.stddev::stddev", - "evaluation_name": "bfcl.format_sensitivity.stddev", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.format_sensitivity.stddev", - "metric_name": "Format sensitivity standard deviation", - "metric_kind": "difference", - "metric_unit": "percentage_points", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Format Sensitivity Standard Deviation" - } - }, - "score_details": { - "score": 13.57 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/mistralai_mistral-small-instruct-2409.json b/data/models/mistralai_mistral-small-instruct-2409.json deleted file mode 100644 index 5aac65e94f46e67333b7cb66dd77ae7f5415d17b..0000000000000000000000000000000000000000 --- a/data/models/mistralai_mistral-small-instruct-2409.json +++ /dev/null @@ -1,275 +0,0 @@ -{ - "model_info": { - "name": "Mistral-Small-Instruct-2409", - "id": "mistralai/Mistral-Small-Instruct-2409", - "developer": "mistralai", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "22.247" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/mistralai_Mistral-Small-Instruct-2409/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6283 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.583 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2039 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3331 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4063 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4099 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "hfopenllm_v2/mistralai_Mistral-Small-Instruct-2409/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.667 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5213 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1435 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3238 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3632 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.396 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/mistralai_mistral-v0.1-7b.json b/data/models/mistralai_mistral-v0.1-7b.json deleted file mode 100644 index 82f7a11efe53882434a053609691f2988c077b52..0000000000000000000000000000000000000000 --- a/data/models/mistralai_mistral-v0.1-7b.json +++ /dev/null @@ -1,674 +0,0 @@ -{ - "model_info": { - "name": "Mistral v0.1 7B", - "id": "mistralai/Mistral-v0.1-7B", - "developer": "mistralai", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_classic/mistralai_Mistral-v0.1-7B/1774096308.339228", - "retrieved_timestamp": "1774096308.339228", - "source_metadata": { - "source_name": "helm_classic", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_classic", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_classic", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperform on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.884, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Calibration": "{\"description\": \"\", \"tab\": \"Calibration\", \"score\": \"\"}", - "Mean win rate - Robustness": "{\"description\": \"\", \"tab\": \"Robustness\", \"score\": \"0.8963869463869464\"}", - "Mean win rate - Fairness": "{\"description\": \"\", \"tab\": \"Fairness\", \"score\": \"0.8611188811188811\"}", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}", - "Mean win rate - Bias": "{\"description\": \"\", \"tab\": \"Bias\", \"score\": \"0.5247457047269077\"}", - "Mean win rate - Toxicity": "{\"description\": \"\", \"tab\": \"Toxicity\", \"score\": \"0.4297202797202797\"}", - "Mean win rate - Summarization metrics": "{\"description\": \"\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.572, - "details": { - "description": "min=0.28, mean=0.572, max=0.84, sum=2.861 (5)", - "tab": "Accuracy", - "MMLU - ECE (10-bin)": "{\"description\": \"5 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "MMLU - EM (Robustness)": "{\"description\": \"min=0.24, mean=0.533, max=0.82, sum=2.666 (5)\", \"tab\": \"Robustness\", \"score\": \"0.5332280701754385\"}", - "MMLU - EM (Fairness)": "{\"description\": \"min=0.27, mean=0.542, max=0.83, sum=2.709 (5)\", \"tab\": \"Fairness\", \"score\": \"0.541719298245614\"}", - "MMLU - Denoised inference time (s)": "{\"description\": \"5 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "MMLU - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "BoolQ", - "source_data": { - "dataset_name": "BoolQ", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on BoolQ", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.874, - "details": { - "description": "min=0.874, mean=0.874, max=0.874, sum=0.874 (1)", - "tab": "Accuracy", - "BoolQ - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "BoolQ - EM (Robustness)": "{\"description\": \"min=0.837, mean=0.837, max=0.837, sum=0.837 (1)\", \"tab\": \"Robustness\", \"score\": \"0.837\"}", - "BoolQ - EM (Fairness)": "{\"description\": \"min=0.842, mean=0.842, max=0.842, sum=0.842 (1)\", \"tab\": \"Fairness\", \"score\": \"0.842\"}", - "BoolQ - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "BoolQ - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "BoolQ - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "BoolQ - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "BoolQ - # prompt tokens": "{\"description\": \"min=1418.259, mean=1418.259, max=1418.259, sum=1418.259 (1)\", \"tab\": \"General information\", \"score\": \"1418.259\"}", - "BoolQ - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "BoolQ - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "BoolQ - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.716, - "details": { - "description": "min=0.716, mean=0.716, max=0.716, sum=0.716 (1)", - "tab": "Accuracy", - "NarrativeQA - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "NarrativeQA - F1 (Robustness)": "{\"description\": \"min=0.649, mean=0.649, max=0.649, sum=0.649 (1)\", \"tab\": \"Robustness\", \"score\": \"0.6485445694648198\"}", - "NarrativeQA - F1 (Fairness)": "{\"description\": \"min=0.644, mean=0.644, max=0.644, sum=0.644 (1)\", \"tab\": \"Fairness\", \"score\": \"0.6436697691254157\"}", - "NarrativeQA - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=4.575, mean=4.575, max=4.575, sum=4.575 (1)\", \"tab\": \"General information\", \"score\": \"4.574647887323944\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=3627.715, mean=3627.715, max=3627.715, sum=3627.715 (1)\", \"tab\": \"General information\", \"score\": \"3627.7154929577464\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NarrativeQA - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NarrativeQA - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NarrativeQA - Stereotypes (gender)": "{\"description\": \"min=0.5, mean=0.5, max=0.5, sum=0.5 (1)\", \"tab\": \"Bias\", \"score\": \"0.5\"}", - "NarrativeQA - Representation (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=0.667 (1)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666667\"}", - "NarrativeQA - Representation (gender)": "{\"description\": \"min=0.173, mean=0.173, max=0.173, sum=0.173 (1)\", \"tab\": \"Bias\", \"score\": \"0.1730769230769231\"}", - "NarrativeQA - Toxic fraction": "{\"description\": \"min=0.008, mean=0.008, max=0.008, sum=0.008 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.008450704225352112\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (open-book)", - "source_data": { - "dataset_name": "NaturalQuestions (open-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (open-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.687, - "details": { - "description": "min=0.687, mean=0.687, max=0.687, sum=0.687 (1)", - "tab": "Accuracy", - "NaturalQuestions (closed-book) - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "NaturalQuestions (open-book) - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - F1 (Robustness)": "{\"description\": \"min=0.305, mean=0.305, max=0.305, sum=0.305 (1)\", \"tab\": \"Robustness\", \"score\": \"0.3052498746141498\"}", - "NaturalQuestions (open-book) - F1 (Robustness)": "{\"description\": \"min=0.631, mean=0.631, max=0.631, sum=0.631 (1)\", \"tab\": \"Robustness\", \"score\": \"0.6314234953832969\"}", - "NaturalQuestions (closed-book) - F1 (Fairness)": "{\"description\": \"min=0.3, mean=0.3, max=0.3, sum=0.3 (1)\", \"tab\": \"Fairness\", \"score\": \"0.30018094571517623\"}", - "NaturalQuestions (open-book) - F1 (Fairness)": "{\"description\": \"min=0.625, mean=0.625, max=0.625, sum=0.625 (1)\", \"tab\": \"Fairness\", \"score\": \"0.6249254915559919\"}", - "NaturalQuestions (closed-book) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NaturalQuestions (open-book) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NaturalQuestions (closed-book) - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.832, mean=4.832, max=4.832, sum=4.832 (1)\", \"tab\": \"General information\", \"score\": \"4.832\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.026, mean=0.026, max=0.026, sum=0.026 (1)\", \"tab\": \"General information\", \"score\": \"0.026\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=2268.728, mean=2268.728, max=2268.728, sum=2268.728 (1)\", \"tab\": \"General information\", \"score\": \"2268.728\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=0.987, mean=0.987, max=0.987, sum=0.987 (1)\", \"tab\": \"General information\", \"score\": \"0.987\"}", - "NaturalQuestions (open-book) - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NaturalQuestions (closed-book) - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=0.667 (1)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666667\"}", - "NaturalQuestions (closed-book) - Stereotypes (gender)": "{\"description\": \"min=0.25, mean=0.25, max=0.25, sum=0.25 (1)\", \"tab\": \"Bias\", \"score\": \"0.25\"}", - "NaturalQuestions (closed-book) - Representation (race)": "{\"description\": \"min=0.287, mean=0.287, max=0.287, sum=0.287 (1)\", \"tab\": \"Bias\", \"score\": \"0.28746177370030584\"}", - "NaturalQuestions (closed-book) - Representation (gender)": "{\"description\": \"min=0.065, mean=0.065, max=0.065, sum=0.065 (1)\", \"tab\": \"Bias\", \"score\": \"0.06521739130434784\"}", - "NaturalQuestions (open-book) - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=0.667 (1)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666667\"}", - "NaturalQuestions (open-book) - Stereotypes (gender)": "{\"description\": \"min=0.5, mean=0.5, max=0.5, sum=0.5 (1)\", \"tab\": \"Bias\", \"score\": \"0.5\"}", - "NaturalQuestions (open-book) - Representation (race)": "{\"description\": \"min=0.439, mean=0.439, max=0.439, sum=0.439 (1)\", \"tab\": \"Bias\", \"score\": \"0.4385964912280702\"}", - "NaturalQuestions (open-book) - Representation (gender)": "{\"description\": \"min=0.48, mean=0.48, max=0.48, sum=0.48 (1)\", \"tab\": \"Bias\", \"score\": \"0.48000000000000004\"}", - "NaturalQuestions (closed-book) - Toxic fraction": "{\"description\": \"min=0.001, mean=0.001, max=0.001, sum=0.001 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.001\"}", - "NaturalQuestions (open-book) - Toxic fraction": "{\"description\": \"min=0.001, mean=0.001, max=0.001, sum=0.001 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.001\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "QuAC", - "source_data": { - "dataset_name": "QuAC", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on QuAC", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.423, - "details": { - "description": "min=0.423, mean=0.423, max=0.423, sum=0.423 (1)", - "tab": "Accuracy", - "QuAC - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "QuAC - F1 (Robustness)": "{\"description\": \"min=0.31, mean=0.31, max=0.31, sum=0.31 (1)\", \"tab\": \"Robustness\", \"score\": \"0.3098633908730089\"}", - "QuAC - F1 (Fairness)": "{\"description\": \"min=0.353, mean=0.353, max=0.353, sum=0.353 (1)\", \"tab\": \"Fairness\", \"score\": \"0.3528008659962099\"}", - "QuAC - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "QuAC - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "QuAC - # train": "{\"description\": \"min=3.44, mean=3.44, max=3.44, sum=3.44 (1)\", \"tab\": \"General information\", \"score\": \"3.44\"}", - "QuAC - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "QuAC - # prompt tokens": "{\"description\": \"min=3680.143, mean=3680.143, max=3680.143, sum=3680.143 (1)\", \"tab\": \"General information\", \"score\": \"3680.143\"}", - "QuAC - # output tokens": "{\"description\": \"min=0.999, mean=0.999, max=0.999, sum=0.999 (1)\", \"tab\": \"General information\", \"score\": \"0.999\"}", - "QuAC - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "QuAC - Stereotypes (race)": "{\"description\": \"min=0.621, mean=0.621, max=0.621, sum=0.621 (1)\", \"tab\": \"Bias\", \"score\": \"0.6213450292397661\"}", - "QuAC - Stereotypes (gender)": "{\"description\": \"min=0.412, mean=0.412, max=0.412, sum=0.412 (1)\", \"tab\": \"Bias\", \"score\": \"0.4119047619047619\"}", - "QuAC - Representation (race)": "{\"description\": \"min=0.274, mean=0.274, max=0.274, sum=0.274 (1)\", \"tab\": \"Bias\", \"score\": \"0.27356321839080466\"}", - "QuAC - Representation (gender)": "{\"description\": \"min=0.248, mean=0.248, max=0.248, sum=0.248 (1)\", \"tab\": \"Bias\", \"score\": \"0.2479564032697547\"}", - "QuAC - Toxic fraction": "{\"description\": \"min=0.003, mean=0.003, max=0.003, sum=0.003 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.003\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "HellaSwag", - "source_data": { - "dataset_name": "HellaSwag", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on HellaSwag", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "HellaSwag - ECE (10-bin)": "{\"description\": \"No matching runs\", \"tab\": \"Calibration\", \"score\": \"\"}", - "HellaSwag - EM (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "HellaSwag - EM (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "HellaSwag - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "HellaSwag - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "OpenbookQA - ECE (10-bin)": "{\"description\": \"No matching runs\", \"tab\": \"Calibration\", \"score\": \"\"}", - "OpenbookQA - EM (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "OpenbookQA - EM (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "OpenbookQA - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "OpenbookQA - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "TruthfulQA", - "source_data": { - "dataset_name": "TruthfulQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on TruthfulQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.422, - "details": { - "description": "min=0.422, mean=0.422, max=0.422, sum=0.422 (1)", - "tab": "Accuracy", - "TruthfulQA - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "TruthfulQA - EM (Robustness)": "{\"description\": \"min=0.339, mean=0.339, max=0.339, sum=0.339 (1)\", \"tab\": \"Robustness\", \"score\": \"0.3394495412844037\"}", - "TruthfulQA - EM (Fairness)": "{\"description\": \"min=0.332, mean=0.332, max=0.332, sum=0.332 (1)\", \"tab\": \"Fairness\", \"score\": \"0.3318042813455658\"}", - "TruthfulQA - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "TruthfulQA - # eval": "{\"description\": \"min=654, mean=654, max=654, sum=654 (1)\", \"tab\": \"General information\", \"score\": \"654.0\"}", - "TruthfulQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "TruthfulQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "TruthfulQA - # prompt tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "TruthfulQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "TruthfulQA - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MS MARCO (TREC)", - "source_data": { - "dataset_name": "MS MARCO (TREC)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "NDCG@10 on MS MARCO (TREC)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "MS MARCO (regular) - RR@10 (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "MS MARCO (TREC) - NDCG@10 (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "MS MARCO (regular) - RR@10 (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "MS MARCO (TREC) - NDCG@10 (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "MS MARCO (regular) - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MS MARCO (TREC) - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MS MARCO (regular) - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "MS MARCO (TREC) - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CNN/DailyMail", - "source_data": { - "dataset_name": "CNN/DailyMail", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on CNN/DailyMail", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "CNN/DailyMail - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "CNN/DailyMail - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "CNN/DailyMail - SummaC": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - QAFactEval": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - BERTScore (F1)": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - Coverage": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - Density": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - Compression": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-faithfulness": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-relevance": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-coherence": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "XSUM", - "source_data": { - "dataset_name": "XSUM", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on XSUM", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "XSUM - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "XSUM - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "XSUM - SummaC": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - QAFactEval": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - BERTScore (F1)": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - Coverage": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - Density": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - Compression": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-faithfulness": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-relevance": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-coherence": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "IMDB", - "source_data": { - "dataset_name": "IMDB", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on IMDB", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.962, - "details": { - "description": "min=0.962, mean=0.962, max=0.962, sum=0.962 (1)", - "tab": "Accuracy", - "IMDB - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "IMDB - EM (Robustness)": "{\"description\": \"min=0.954, mean=0.954, max=0.954, sum=0.954 (1)\", \"tab\": \"Robustness\", \"score\": \"0.954\"}", - "IMDB - EM (Fairness)": "{\"description\": \"min=0.952, mean=0.952, max=0.952, sum=0.952 (1)\", \"tab\": \"Fairness\", \"score\": \"0.952\"}", - "IMDB - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "IMDB - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "IMDB - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "IMDB - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IMDB - # prompt tokens": "{\"description\": \"min=2811.31, mean=2811.31, max=2811.31, sum=2811.31 (1)\", \"tab\": \"General information\", \"score\": \"2811.31\"}", - "IMDB - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "IMDB - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "IMDB - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CivilComments", - "source_data": { - "dataset_name": "CivilComments", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on CivilComments", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.624, - "details": { - "description": "min=0.219, mean=0.624, max=0.874, sum=11.24 (18)", - "tab": "Accuracy", - "CivilComments - ECE (10-bin)": "{\"description\": \"9 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "CivilComments - EM (Robustness)": "{\"description\": \"min=0.123, mean=0.521, max=0.842, sum=9.37 (18)\", \"tab\": \"Robustness\", \"score\": \"0.5205335787071343\"}", - "CivilComments - EM (Fairness)": "{\"description\": \"min=0.06, mean=0.52, max=0.863, sum=9.357 (18)\", \"tab\": \"Fairness\", \"score\": \"0.5198588163222009\"}", - "CivilComments - Denoised inference time (s)": "{\"description\": \"9 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "CivilComments - # eval": "{\"description\": \"min=74, mean=371.556, max=683, sum=6688 (18)\", \"tab\": \"General information\", \"score\": \"371.55555555555554\"}", - "CivilComments - # train": "{\"description\": \"min=5, mean=5, max=5, sum=90 (18)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CivilComments - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (18)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CivilComments - # prompt tokens": "{\"description\": \"min=390.28, mean=831.904, max=1394.234, sum=14974.265 (18)\", \"tab\": \"General information\", \"score\": \"831.9036212109548\"}", - "CivilComments - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=18 (18)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "CivilComments - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=18 (18)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "CivilComments - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Toxic fraction": "{\"description\": \"9 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "RAFT", - "source_data": { - "dataset_name": "RAFT", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on RAFT", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.707, - "details": { - "description": "min=0.1, mean=0.707, max=0.975, sum=7.775 (11)", - "tab": "Accuracy", - "RAFT - ECE (10-bin)": "{\"description\": \"11 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "RAFT - EM (Robustness)": "{\"description\": \"min=0.025, mean=0.652, max=0.975, sum=7.175 (11)\", \"tab\": \"Robustness\", \"score\": \"0.6522727272727272\"}", - "RAFT - EM (Fairness)": "{\"description\": \"min=0.1, mean=0.664, max=0.975, sum=7.3 (11)\", \"tab\": \"Fairness\", \"score\": \"0.6636363636363636\"}", - "RAFT - Denoised inference time (s)": "{\"description\": \"11 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "RAFT - # eval": "{\"description\": \"min=40, mean=40, max=40, sum=440 (11)\", \"tab\": \"General information\", \"score\": \"40.0\"}", - "RAFT - # train": "{\"description\": \"min=2.675, mean=4.789, max=5, sum=52.675 (11)\", \"tab\": \"General information\", \"score\": \"4.788636363636363\"}", - "RAFT - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (11)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "RAFT - # prompt tokens": "{\"description\": \"min=0, mean=328.595, max=3614.55, sum=3614.55 (11)\", \"tab\": \"General information\", \"score\": \"328.5954545454546\"}", - "RAFT - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=11 (11)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "RAFT - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=11 (11)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "RAFT - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Toxic fraction": "{\"description\": \"11 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/mistralai_mixtral-8x22b-instruct-v0.1.json b/data/models/mistralai_mixtral-8x22b-instruct-v0.1.json deleted file mode 100644 index 09a2fa083c7e23648730b3ac8f2ac7c538ca8b26..0000000000000000000000000000000000000000 --- a/data/models/mistralai_mixtral-8x22b-instruct-v0.1.json +++ /dev/null @@ -1,369 +0,0 @@ -{ - "model_info": { - "name": "Mixtral-8x22B-Instruct-v0.1", - "id": "mistralai/Mixtral-8x22B-Instruct-v0.1", - "developer": "mistralai", - "inference_platform": "unknown", - "additional_details": { - "model_id_aliases": [ - "mistralai/mixtral-8x22b-instruct-v0.1" - ] - } - }, - "evaluations": [ - { - "evaluation_id": "helm_capabilities/mistralai_mixtral-8x22b-instruct-v0.1/1774096304.5056698", - "retrieved_timestamp": "1774096304.5056698", - "source_metadata": { - "source_name": "helm_capabilities", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_capabilities", - "evaluation_results": [ - { - "evaluation_name": "Mean score", - "source_data": { - "dataset_name": "helm_capabilities", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "The mean of the scores from all columns.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.478, - "details": { - "description": "", - "tab": "Accuracy", - "Mean score - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"6.16132193567775\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU-Pro", - "source_data": { - "dataset_name": "MMLU-Pro", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on MMLU-Pro", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.46, - "details": { - "description": "min=0.46, mean=0.46, max=0.46, sum=0.46 (1)", - "tab": "Accuracy", - "MMLU-Pro - Observed inference time (s)": "{\"description\": \"min=3.967, mean=3.967, max=3.967, sum=3.967 (1)\", \"tab\": \"Efficiency\", \"score\": \"3.967100965499878\"}", - "MMLU-Pro - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "MMLU-Pro - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - # prompt tokens": "{\"description\": \"min=260.915, mean=260.915, max=260.915, sum=260.915 (1)\", \"tab\": \"General information\", \"score\": \"260.915\"}", - "MMLU-Pro - # output tokens": "{\"description\": \"min=298.159, mean=298.159, max=298.159, sum=298.159 (1)\", \"tab\": \"General information\", \"score\": \"298.159\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"all\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.334, - "details": { - "description": "min=0.334, mean=0.334, max=0.334, sum=0.334 (1)", - "tab": "Accuracy", - "GPQA - Observed inference time (s)": "{\"description\": \"min=4.76, mean=4.76, max=4.76, sum=4.76 (1)\", \"tab\": \"Efficiency\", \"score\": \"4.760301354220095\"}", - "GPQA - # eval": "{\"description\": \"min=446, mean=446, max=446, sum=446 (1)\", \"tab\": \"General information\", \"score\": \"446.0\"}", - "GPQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - # prompt tokens": "{\"description\": \"min=281.998, mean=281.998, max=281.998, sum=281.998 (1)\", \"tab\": \"General information\", \"score\": \"281.99775784753365\"}", - "GPQA - # output tokens": "{\"description\": \"min=403.895, mean=403.895, max=403.895, sum=403.895 (1)\", \"tab\": \"General information\", \"score\": \"403.89461883408075\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"gpqa_main\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "IFEval Strict Acc on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.724, - "details": { - "description": "min=0.724, mean=0.724, max=0.724, sum=0.724 (1)", - "tab": "Accuracy", - "IFEval - Observed inference time (s)": "{\"description\": \"min=4.568, mean=4.568, max=4.568, sum=4.568 (1)\", \"tab\": \"Efficiency\", \"score\": \"4.56831247837398\"}", - "IFEval - # eval": "{\"description\": \"min=541, mean=541, max=541, sum=541 (1)\", \"tab\": \"General information\", \"score\": \"541.0\"}", - "IFEval - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - # prompt tokens": "{\"description\": \"min=51.309, mean=51.309, max=51.309, sum=51.309 (1)\", \"tab\": \"General information\", \"score\": \"51.3086876155268\"}", - "IFEval - # output tokens": "{\"description\": \"min=390.799, mean=390.799, max=390.799, sum=390.799 (1)\", \"tab\": \"General information\", \"score\": \"390.7985212569316\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WildBench", - "source_data": { - "dataset_name": "WildBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "WB Score on WildBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.711, - "details": { - "description": "min=0.711, mean=0.711, max=0.711, sum=0.711 (1)", - "tab": "Accuracy", - "WildBench - Observed inference time (s)": "{\"description\": \"min=10.146, mean=10.146, max=10.146, sum=10.146 (1)\", \"tab\": \"Efficiency\", \"score\": \"10.145776480436325\"}", - "WildBench - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "WildBench - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # prompt tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # output tokens": "{\"description\": \"min=668.768, mean=668.768, max=668.768, sum=668.768 (1)\", \"tab\": \"General information\", \"score\": \"668.768\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"v2\"" - } - } - }, - { - "evaluation_name": "Omni-MATH", - "source_data": { - "dataset_name": "Omni-MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Acc on Omni-MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.163, - "details": { - "description": "min=0.163, mean=0.163, max=0.163, sum=0.163 (1)", - "tab": "Accuracy", - "Omni-MATH - Observed inference time (s)": "{\"description\": \"min=7.365, mean=7.365, max=7.365, sum=7.365 (1)\", \"tab\": \"Efficiency\", \"score\": \"7.365118399858475\"}", - "Omni-MATH - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "Omni-MATH - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - # prompt tokens": "{\"description\": \"min=119.373, mean=119.373, max=119.373, sum=119.373 (1)\", \"tab\": \"General information\", \"score\": \"119.373\"}", - "Omni-MATH - # output tokens": "{\"description\": \"min=783.89, mean=783.89, max=783.89, sum=783.89 (1)\", \"tab\": \"General information\", \"score\": \"783.89\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "hfopenllm_v2/mistralai_Mixtral-8x22B-Instruct-v0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7184 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6125 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1873 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3733 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4311 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4483 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/mistralai_mixtral-8x22b-v0.1.json b/data/models/mistralai_mixtral-8x22b-v0.1.json deleted file mode 100644 index 7a1ca6fc58a7efbc015a160f5e0644c57a370ebd..0000000000000000000000000000000000000000 --- a/data/models/mistralai_mixtral-8x22b-v0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mixtral-8x22B-v0.1", - "id": "mistralai/Mixtral-8x22B-v0.1", - "developer": "mistralai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MixtralForCausalLM", - "params_billions": "140.621" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/mistralai_Mixtral-8x22B-v0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2583 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.624 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1835 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3758 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4037 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4639 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/mistralai_mixtral-8x22b.json b/data/models/mistralai_mixtral-8x22b.json deleted file mode 100644 index d1771ea52cef85833cfb97cff3623cbda3c86087..0000000000000000000000000000000000000000 --- a/data/models/mistralai_mixtral-8x22b.json +++ /dev/null @@ -1,1901 +0,0 @@ -{ - "model_info": { - "name": "Mixtral 8x22B", - "id": "mistralai/mixtral-8x22b", - "developer": "mistralai", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_lite/mistralai_mixtral-8x22b/1774096306.427425", - "retrieved_timestamp": "1774096306.427425", - "source_metadata": { - "source_name": "helm_lite", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_lite", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_lite", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.705, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.5349563046192259\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.779, - "details": { - "description": "min=0.779, mean=0.779, max=0.779, sum=0.779 (1)", - "tab": "Accuracy", - "NarrativeQA - Observed inference time (s)": "{\"description\": \"min=1.478, mean=1.478, max=1.478, sum=1.478 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.477503587158633\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=3886.33, mean=3886.33, max=3886.33, sum=3886.33 (1)\", \"tab\": \"General information\", \"score\": \"3886.3295774647886\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (closed-book)", - "source_data": { - "dataset_name": "NaturalQuestions (closed-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (closed-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.478, - "details": { - "description": "min=0.478, mean=0.478, max=0.478, sum=0.478 (1)", - "tab": "Accuracy", - "NaturalQuestions (open-book) - Observed inference time (s)": "{\"description\": \"min=1.004, mean=1.004, max=1.004, sum=1.004 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.003950766324997\"}", - "NaturalQuestions (closed-book) - Observed inference time (s)": "{\"description\": \"min=0.442, mean=0.442, max=0.442, sum=0.442 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.44196626234054565\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=2468.79, mean=2468.79, max=2468.79, sum=2468.79 (1)\", \"tab\": \"General information\", \"score\": \"2468.79\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=142.069, mean=142.069, max=142.069, sum=142.069 (1)\", \"tab\": \"General information\", \"score\": \"142.069\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "mode": "\"closedbook\"" - } - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.882, - "details": { - "description": "min=0.882, mean=0.882, max=0.882, sum=0.882 (1)", - "tab": "Accuracy", - "OpenbookQA - Observed inference time (s)": "{\"description\": \"min=0.338, mean=0.338, max=0.338, sum=0.338 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.33846320056915286\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=280.15, mean=280.15, max=280.15, sum=280.15 (1)\", \"tab\": \"General information\", \"score\": \"280.15\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "dataset": "\"openbookqa\"", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.701, - "details": { - "description": "min=0.48, mean=0.701, max=0.95, sum=3.507 (5)", - "tab": "Accuracy", - "MMLU - Observed inference time (s)": "{\"description\": \"min=0.313, mean=0.344, max=0.359, sum=1.722 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.344487278235586\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=402.44, mean=523.091, max=687.175, sum=2615.455 (5)\", \"tab\": \"General information\", \"score\": \"523.0910877192983\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"college_chemistry\", \"computer_security\", \"econometrics\", \"us_foreign_policy\"]", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MATH", - "source_data": { - "dataset_name": "MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Equivalent (CoT) on MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.656, - "details": { - "description": "min=0.5, mean=0.656, max=0.822, sum=4.589 (7)", - "tab": "Accuracy", - "MATH - Observed inference time (s)": "{\"description\": \"min=2.009, mean=2.509, max=3.121, sum=17.565 (7)\", \"tab\": \"Efficiency\", \"score\": \"2.5093491334109825\"}", - "MATH - # eval": "{\"description\": \"min=30, mean=62.429, max=135, sum=437 (7)\", \"tab\": \"General information\", \"score\": \"62.42857142857143\"}", - "MATH - # train": "{\"description\": \"min=8, mean=8, max=8, sum=56 (7)\", \"tab\": \"General information\", \"score\": \"8.0\"}", - "MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MATH - # prompt tokens": "{\"description\": \"min=991.615, mean=1455.266, max=2502.962, sum=10186.865 (7)\", \"tab\": \"General information\", \"score\": \"1455.2664139976257\"}", - "MATH - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=7 (7)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"algebra\", \"counting_and_probability\", \"geometry\", \"intermediate_algebra\", \"number_theory\", \"prealgebra\", \"precalculus\"]", - "level": "\"1\"", - "use_official_examples": "\"False\"", - "use_chain_of_thought": "\"True\"" - } - } - }, - { - "evaluation_name": "GSM8K", - "source_data": { - "dataset_name": "GSM8K", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on GSM8K", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8, - "details": { - "description": "min=0.8, mean=0.8, max=0.8, sum=0.8 (1)", - "tab": "Accuracy", - "GSM8K - Observed inference time (s)": "{\"description\": \"min=3.539, mean=3.539, max=3.539, sum=3.539 (1)\", \"tab\": \"Efficiency\", \"score\": \"3.5390553929805755\"}", - "GSM8K - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "GSM8K - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "GSM8K - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GSM8K - # prompt tokens": "{\"description\": \"min=1187.268, mean=1187.268, max=1187.268, sum=1187.268 (1)\", \"tab\": \"General information\", \"score\": \"1187.268\"}", - "GSM8K - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "LegalBench", - "source_data": { - "dataset_name": "LegalBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on LegalBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.708, - "details": { - "description": "min=0.441, mean=0.708, max=0.968, sum=3.539 (5)", - "tab": "Accuracy", - "LegalBench - Observed inference time (s)": "{\"description\": \"min=0.372, mean=0.821, max=1.973, sum=4.107 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.8213642223004287\"}", - "LegalBench - # eval": "{\"description\": \"min=95, mean=409.4, max=1000, sum=2047 (5)\", \"tab\": \"General information\", \"score\": \"409.4\"}", - "LegalBench - # train": "{\"description\": \"min=4, mean=4.8, max=5, sum=24 (5)\", \"tab\": \"General information\", \"score\": \"4.8\"}", - "LegalBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "LegalBench - # prompt tokens": "{\"description\": \"min=219.453, mean=1733.148, max=7207.488, sum=8665.741 (5)\", \"tab\": \"General information\", \"score\": \"1733.148245843296\"}", - "LegalBench - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"abercrombie\", \"corporate_lobbying\", \"function_of_decision_section\", \"international_citizenship_questions\", \"proa\"]" - } - } - }, - { - "evaluation_name": "MedQA", - "source_data": { - "dataset_name": "MedQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MedQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.704, - "details": { - "description": "min=0.704, mean=0.704, max=0.704, sum=0.704 (1)", - "tab": "Accuracy", - "MedQA - Observed inference time (s)": "{\"description\": \"min=0.463, mean=0.463, max=0.463, sum=0.463 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.46328771849038825\"}", - "MedQA - # eval": "{\"description\": \"min=503, mean=503, max=503, sum=503 (1)\", \"tab\": \"General information\", \"score\": \"503.0\"}", - "MedQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MedQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MedQA - # prompt tokens": "{\"description\": \"min=1193.093, mean=1193.093, max=1193.093, sum=1193.093 (1)\", \"tab\": \"General information\", \"score\": \"1193.0934393638172\"}", - "MedQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WMT 2014", - "source_data": { - "dataset_name": "WMT 2014", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "BLEU-4 on WMT 2014", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.209, - "details": { - "description": "min=0.133, mean=0.209, max=0.243, sum=1.045 (5)", - "tab": "Accuracy", - "WMT 2014 - Observed inference time (s)": "{\"description\": \"min=0.928, mean=0.963, max=0.982, sum=4.813 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.9626315307056144\"}", - "WMT 2014 - # eval": "{\"description\": \"min=503, mean=568.8, max=832, sum=2844 (5)\", \"tab\": \"General information\", \"score\": \"568.8\"}", - "WMT 2014 - # train": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "WMT 2014 - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WMT 2014 - # prompt tokens": "{\"description\": \"min=130.306, mean=144.433, max=163.018, sum=722.166 (5)\", \"tab\": \"General information\", \"score\": \"144.43317355482492\"}", - "WMT 2014 - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "language_pair": "[\"cs-en\", \"de-en\", \"fr-en\", \"hi-en\", \"ru-en\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_mmlu/mistralai_mixtral-8x22b/1774096312.00548", - "retrieved_timestamp": "1774096312.00548", - "source_metadata": { - "source_name": "helm_mmlu", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_mmlu", - "evaluation_results": [ - { - "evaluation_name": "MMLU All Subjects", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU All Subjects", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.778, - "details": { - "description": "min=0.463, mean=0.778, max=0.974, sum=88.715 (114)", - "tab": "Accuracy", - "MMLU All Subjects - Observed inference time (s)": "{\"description\": \"min=0.284, mean=0.555, max=4.852, sum=63.286 (114)\", \"tab\": \"Efficiency\", \"score\": \"0.5551394123775506\"}", - "MMLU All Subjects - # eval": "{\"description\": \"min=100, mean=246.351, max=1534, sum=28084 (114)\", \"tab\": \"General information\", \"score\": \"246.35087719298247\"}", - "MMLU All Subjects - # train": "{\"description\": \"min=5, mean=5, max=5, sum=570 (114)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU All Subjects - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU All Subjects - # prompt tokens": "{\"description\": \"min=308.924, mean=696.273, max=3089.109, sum=79375.178 (114)\", \"tab\": \"General information\", \"score\": \"696.2734899593811\"}", - "MMLU All Subjects - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=114 (114)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_name": "Abstract Algebra", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Abstract Algebra", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.48, - "details": { - "description": "min=0.48, mean=0.48, max=0.48, sum=0.96 (2)", - "tab": "Accuracy", - "Abstract Algebra - Observed inference time (s)": "{\"description\": \"min=0.313, mean=0.313, max=0.313, sum=0.626 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.31304038524627686\"}", - "Abstract Algebra - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Abstract Algebra - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Abstract Algebra - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Abstract Algebra - # prompt tokens": "{\"description\": \"min=402.44, mean=402.44, max=402.44, sum=804.88 (2)\", \"tab\": \"General information\", \"score\": \"402.44\"}", - "Abstract Algebra - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"abstract_algebra\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_abstract_algebra\"" - } - } - }, - { - "evaluation_name": "Anatomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Anatomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.741, - "details": { - "description": "min=0.741, mean=0.741, max=0.741, sum=1.481 (2)", - "tab": "Accuracy", - "Anatomy - Observed inference time (s)": "{\"description\": \"min=0.331, mean=0.331, max=0.331, sum=0.662 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3308721118503147\"}", - "Anatomy - # eval": "{\"description\": \"min=135, mean=135, max=135, sum=270 (2)\", \"tab\": \"General information\", \"score\": \"135.0\"}", - "Anatomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Anatomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Anatomy - # prompt tokens": "{\"description\": \"min=407.089, mean=407.089, max=407.089, sum=814.178 (2)\", \"tab\": \"General information\", \"score\": \"407.0888888888889\"}", - "Anatomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"anatomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_anatomy\"" - } - } - }, - { - "evaluation_name": "College Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on College Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.569, - "details": { - "description": "min=0.569, mean=0.569, max=0.569, sum=1.137 (2)", - "tab": "Accuracy", - "College Chemistry - Observed inference time (s)": "{\"description\": \"min=0.358, mean=0.358, max=0.358, sum=0.716 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.35782508373260496\"}", - "College Biology - Observed inference time (s)": "{\"description\": \"min=0.336, mean=0.336, max=0.336, sum=0.671 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.33555712799231213\"}", - "College Computer Science - Observed inference time (s)": "{\"description\": \"min=0.409, mean=0.409, max=0.409, sum=0.819 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.40926079750061034\"}", - "College Mathematics - Observed inference time (s)": "{\"description\": \"min=0.382, mean=0.382, max=0.382, sum=0.765 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3824312686920166\"}", - "College Medicine - Observed inference time (s)": "{\"description\": \"min=0.336, mean=0.336, max=0.336, sum=0.671 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.33573296993454066\"}", - "College Physics - Observed inference time (s)": "{\"description\": \"min=0.347, mean=0.347, max=0.347, sum=0.694 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.34694373841379206\"}", - "College Chemistry - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Chemistry - # prompt tokens": "{\"description\": \"min=627.71, mean=627.71, max=627.71, sum=1255.42 (2)\", \"tab\": \"General information\", \"score\": \"627.71\"}", - "College Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Biology - # eval": "{\"description\": \"min=144, mean=144, max=144, sum=288 (2)\", \"tab\": \"General information\", \"score\": \"144.0\"}", - "College Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # prompt tokens": "{\"description\": \"min=550.799, mean=550.799, max=550.799, sum=1101.597 (2)\", \"tab\": \"General information\", \"score\": \"550.7986111111111\"}", - "College Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # prompt tokens": "{\"description\": \"min=902.17, mean=902.17, max=902.17, sum=1804.34 (2)\", \"tab\": \"General information\", \"score\": \"902.17\"}", - "College Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Mathematics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # prompt tokens": "{\"description\": \"min=658.31, mean=658.31, max=658.31, sum=1316.62 (2)\", \"tab\": \"General information\", \"score\": \"658.31\"}", - "College Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Medicine - # eval": "{\"description\": \"min=173, mean=173, max=173, sum=346 (2)\", \"tab\": \"General information\", \"score\": \"173.0\"}", - "College Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # prompt tokens": "{\"description\": \"min=592.41, mean=592.41, max=592.41, sum=1184.821 (2)\", \"tab\": \"General information\", \"score\": \"592.4104046242775\"}", - "College Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Physics - # eval": "{\"description\": \"min=102, mean=102, max=102, sum=204 (2)\", \"tab\": \"General information\", \"score\": \"102.0\"}", - "College Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # prompt tokens": "{\"description\": \"min=551.029, mean=551.029, max=551.029, sum=1102.059 (2)\", \"tab\": \"General information\", \"score\": \"551.0294117647059\"}", - "College Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"college_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_college_physics\"" - } - } - }, - { - "evaluation_name": "Computer Security", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Computer Security", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.84, - "details": { - "description": "min=0.84, mean=0.84, max=0.84, sum=1.68 (2)", - "tab": "Accuracy", - "Computer Security - Observed inference time (s)": "{\"description\": \"min=0.344, mean=0.344, max=0.344, sum=0.689 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3443935012817383\"}", - "Computer Security - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Computer Security - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Computer Security - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Computer Security - # prompt tokens": "{\"description\": \"min=424.94, mean=424.94, max=424.94, sum=849.88 (2)\", \"tab\": \"General information\", \"score\": \"424.94\"}", - "Computer Security - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"computer_security\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_computer_security\"" - } - } - }, - { - "evaluation_name": "Econometrics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Econometrics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.667, - "details": { - "description": "min=0.667, mean=0.667, max=0.667, sum=1.333 (2)", - "tab": "Accuracy", - "Econometrics - Observed inference time (s)": "{\"description\": \"min=0.359, mean=0.359, max=0.359, sum=0.719 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.359416033092298\"}", - "Econometrics - # eval": "{\"description\": \"min=114, mean=114, max=114, sum=228 (2)\", \"tab\": \"General information\", \"score\": \"114.0\"}", - "Econometrics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Econometrics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Econometrics - # prompt tokens": "{\"description\": \"min=687.175, mean=687.175, max=687.175, sum=1374.351 (2)\", \"tab\": \"General information\", \"score\": \"687.1754385964912\"}", - "Econometrics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"econometrics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_econometrics\"" - } - } - }, - { - "evaluation_name": "Global Facts", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Global Facts", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.56, - "details": { - "description": "min=0.56, mean=0.56, max=0.56, sum=1.12 (2)", - "tab": "Accuracy", - "Global Facts - Observed inference time (s)": "{\"description\": \"min=0.349, mean=0.349, max=0.349, sum=0.699 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.34949236392974853\"}", - "Global Facts - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Global Facts - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Global Facts - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Global Facts - # prompt tokens": "{\"description\": \"min=483.47, mean=483.47, max=483.47, sum=966.94 (2)\", \"tab\": \"General information\", \"score\": \"483.47\"}", - "Global Facts - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"global_facts\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_global_facts\"" - } - } - }, - { - "evaluation_name": "Jurisprudence", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Jurisprudence", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.852, - "details": { - "description": "min=0.852, mean=0.852, max=0.852, sum=1.704 (2)", - "tab": "Accuracy", - "Jurisprudence - Observed inference time (s)": "{\"description\": \"min=0.308, mean=0.308, max=0.308, sum=0.616 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.30799298153983223\"}", - "Jurisprudence - # eval": "{\"description\": \"min=108, mean=108, max=108, sum=216 (2)\", \"tab\": \"General information\", \"score\": \"108.0\"}", - "Jurisprudence - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Jurisprudence - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Jurisprudence - # prompt tokens": "{\"description\": \"min=451.093, mean=451.093, max=451.093, sum=902.185 (2)\", \"tab\": \"General information\", \"score\": \"451.0925925925926\"}", - "Jurisprudence - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"jurisprudence\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_jurisprudence\"" - } - } - }, - { - "evaluation_name": "Philosophy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Philosophy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.842, - "details": { - "description": "min=0.842, mean=0.842, max=0.842, sum=1.685 (2)", - "tab": "Accuracy", - "Philosophy - Observed inference time (s)": "{\"description\": \"min=0.423, mean=0.423, max=0.423, sum=0.846 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4229524595561135\"}", - "Philosophy - # eval": "{\"description\": \"min=311, mean=311, max=311, sum=622 (2)\", \"tab\": \"General information\", \"score\": \"311.0\"}", - "Philosophy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Philosophy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Philosophy - # prompt tokens": "{\"description\": \"min=373.82, mean=373.82, max=373.82, sum=747.64 (2)\", \"tab\": \"General information\", \"score\": \"373.81993569131834\"}", - "Philosophy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"philosophy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_philosophy\"" - } - } - }, - { - "evaluation_name": "Professional Psychology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Professional Psychology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.845, - "details": { - "description": "min=0.845, mean=0.845, max=0.845, sum=1.69 (2)", - "tab": "Accuracy", - "Professional Medicine - Observed inference time (s)": "{\"description\": \"min=0.461, mean=0.461, max=0.461, sum=0.921 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4606352711425108\"}", - "Professional Accounting - Observed inference time (s)": "{\"description\": \"min=0.36, mean=0.36, max=0.36, sum=0.72 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3601941665013631\"}", - "Professional Law - Observed inference time (s)": "{\"description\": \"min=0.578, mean=0.578, max=0.578, sum=1.156 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5780843218115815\"}", - "Professional Psychology - Observed inference time (s)": "{\"description\": \"min=0.359, mean=0.359, max=0.359, sum=0.718 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3589704905460083\"}", - "Professional Medicine - # eval": "{\"description\": \"min=272, mean=272, max=272, sum=544 (2)\", \"tab\": \"General information\", \"score\": \"272.0\"}", - "Professional Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Medicine - # prompt tokens": "{\"description\": \"min=1279.143, mean=1279.143, max=1279.143, sum=2558.287 (2)\", \"tab\": \"General information\", \"score\": \"1279.1433823529412\"}", - "Professional Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Accounting - # eval": "{\"description\": \"min=282, mean=282, max=282, sum=564 (2)\", \"tab\": \"General information\", \"score\": \"282.0\"}", - "Professional Accounting - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Accounting - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # prompt tokens": "{\"description\": \"min=796.496, mean=796.496, max=796.496, sum=1592.993 (2)\", \"tab\": \"General information\", \"score\": \"796.4964539007092\"}", - "Professional Accounting - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Law - # eval": "{\"description\": \"min=1534, mean=1534, max=1534, sum=3068 (2)\", \"tab\": \"General information\", \"score\": \"1534.0\"}", - "Professional Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # prompt tokens": "{\"description\": \"min=1849.711, mean=1849.711, max=1849.711, sum=3699.421 (2)\", \"tab\": \"General information\", \"score\": \"1849.7105606258149\"}", - "Professional Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Psychology - # eval": "{\"description\": \"min=612, mean=612, max=612, sum=1224 (2)\", \"tab\": \"General information\", \"score\": \"612.0\"}", - "Professional Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # prompt tokens": "{\"description\": \"min=645.278, mean=645.278, max=645.278, sum=1290.556 (2)\", \"tab\": \"General information\", \"score\": \"645.2777777777778\"}", - "Professional Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"professional_psychology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_professional_psychology\"" - } - } - }, - { - "evaluation_name": "Us Foreign Policy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Us Foreign Policy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.95, - "details": { - "description": "min=0.95, mean=0.95, max=0.95, sum=1.9 (2)", - "tab": "Accuracy", - "Us Foreign Policy - Observed inference time (s)": "{\"description\": \"min=0.348, mean=0.348, max=0.348, sum=0.696 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3477613878250122\"}", - "Us Foreign Policy - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Us Foreign Policy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Us Foreign Policy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Us Foreign Policy - # prompt tokens": "{\"description\": \"min=473.19, mean=473.19, max=473.19, sum=946.38 (2)\", \"tab\": \"General information\", \"score\": \"473.19\"}", - "Us Foreign Policy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"us_foreign_policy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_us_foreign_policy\"" - } - } - }, - { - "evaluation_name": "Astronomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Astronomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.882, - "details": { - "description": "min=0.882, mean=0.882, max=0.882, sum=1.763 (2)", - "tab": "Accuracy", - "Astronomy - Observed inference time (s)": "{\"description\": \"min=0.347, mean=0.347, max=0.347, sum=0.694 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.34718117117881775\"}", - "Astronomy - # eval": "{\"description\": \"min=152, mean=152, max=152, sum=304 (2)\", \"tab\": \"General information\", \"score\": \"152.0\"}", - "Astronomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Astronomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Astronomy - # prompt tokens": "{\"description\": \"min=665.987, mean=665.987, max=665.987, sum=1331.974 (2)\", \"tab\": \"General information\", \"score\": \"665.9868421052631\"}", - "Astronomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"astronomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_astronomy\"" - } - } - }, - { - "evaluation_name": "Business Ethics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Business Ethics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.74, - "details": { - "description": "min=0.74, mean=0.74, max=0.74, sum=1.48 (2)", - "tab": "Accuracy", - "Business Ethics - Observed inference time (s)": "{\"description\": \"min=0.672, mean=0.672, max=0.672, sum=1.345 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6724735307693481\"}", - "Business Ethics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Business Ethics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Business Ethics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Business Ethics - # prompt tokens": "{\"description\": \"min=644.6, mean=644.6, max=644.6, sum=1289.2 (2)\", \"tab\": \"General information\", \"score\": \"644.6\"}", - "Business Ethics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"business_ethics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_business_ethics\"" - } - } - }, - { - "evaluation_name": "Clinical Knowledge", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Clinical Knowledge", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.819, - "details": { - "description": "min=0.819, mean=0.819, max=0.819, sum=1.638 (2)", - "tab": "Accuracy", - "Clinical Knowledge - Observed inference time (s)": "{\"description\": \"min=0.476, mean=0.476, max=0.476, sum=0.953 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4764475804454875\"}", - "Clinical Knowledge - # eval": "{\"description\": \"min=265, mean=265, max=265, sum=530 (2)\", \"tab\": \"General information\", \"score\": \"265.0\"}", - "Clinical Knowledge - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Clinical Knowledge - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Clinical Knowledge - # prompt tokens": "{\"description\": \"min=487.174, mean=487.174, max=487.174, sum=974.347 (2)\", \"tab\": \"General information\", \"score\": \"487.1735849056604\"}", - "Clinical Knowledge - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"clinical_knowledge\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_clinical_knowledge\"" - } - } - }, - { - "evaluation_name": "Conceptual Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Conceptual Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.796, - "details": { - "description": "min=0.796, mean=0.796, max=0.796, sum=1.591 (2)", - "tab": "Accuracy", - "Conceptual Physics - Observed inference time (s)": "{\"description\": \"min=0.327, mean=0.327, max=0.327, sum=0.654 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3271778279162468\"}", - "Conceptual Physics - # eval": "{\"description\": \"min=235, mean=235, max=235, sum=470 (2)\", \"tab\": \"General information\", \"score\": \"235.0\"}", - "Conceptual Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Conceptual Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Conceptual Physics - # prompt tokens": "{\"description\": \"min=334.285, mean=334.285, max=334.285, sum=668.57 (2)\", \"tab\": \"General information\", \"score\": \"334.2851063829787\"}", - "Conceptual Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"conceptual_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_conceptual_physics\"" - } - } - }, - { - "evaluation_name": "Electrical Engineering", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Electrical Engineering", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.766, - "details": { - "description": "min=0.766, mean=0.766, max=0.766, sum=1.531 (2)", - "tab": "Accuracy", - "Electrical Engineering - Observed inference time (s)": "{\"description\": \"min=0.579, mean=0.579, max=0.579, sum=1.158 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5787854655035611\"}", - "Electrical Engineering - # eval": "{\"description\": \"min=145, mean=145, max=145, sum=290 (2)\", \"tab\": \"General information\", \"score\": \"145.0\"}", - "Electrical Engineering - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Electrical Engineering - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Electrical Engineering - # prompt tokens": "{\"description\": \"min=501.379, mean=501.379, max=501.379, sum=1002.759 (2)\", \"tab\": \"General information\", \"score\": \"501.37931034482756\"}", - "Electrical Engineering - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"electrical_engineering\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_electrical_engineering\"" - } - } - }, - { - "evaluation_name": "Elementary Mathematics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Elementary Mathematics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.622, - "details": { - "description": "min=0.622, mean=0.622, max=0.622, sum=1.243 (2)", - "tab": "Accuracy", - "Elementary Mathematics - Observed inference time (s)": "{\"description\": \"min=4.852, mean=4.852, max=4.852, sum=9.703 (2)\", \"tab\": \"Efficiency\", \"score\": \"4.851643589438584\"}", - "Elementary Mathematics - # eval": "{\"description\": \"min=378, mean=378, max=378, sum=756 (2)\", \"tab\": \"General information\", \"score\": \"378.0\"}", - "Elementary Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Elementary Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Elementary Mathematics - # prompt tokens": "{\"description\": \"min=613.386, mean=613.386, max=613.386, sum=1226.772 (2)\", \"tab\": \"General information\", \"score\": \"613.3862433862433\"}", - "Elementary Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"elementary_mathematics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_elementary_mathematics\"" - } - } - }, - { - "evaluation_name": "Formal Logic", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Formal Logic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.627, - "details": { - "description": "min=0.627, mean=0.627, max=0.627, sum=1.254 (2)", - "tab": "Accuracy", - "Formal Logic - Observed inference time (s)": "{\"description\": \"min=4.348, mean=4.348, max=4.348, sum=8.696 (2)\", \"tab\": \"Efficiency\", \"score\": \"4.34797261631678\"}", - "Formal Logic - # eval": "{\"description\": \"min=126, mean=126, max=126, sum=252 (2)\", \"tab\": \"General information\", \"score\": \"126.0\"}", - "Formal Logic - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Formal Logic - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Formal Logic - # prompt tokens": "{\"description\": \"min=718.984, mean=718.984, max=718.984, sum=1437.968 (2)\", \"tab\": \"General information\", \"score\": \"718.984126984127\"}", - "Formal Logic - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"formal_logic\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_formal_logic\"" - } - } - }, - { - "evaluation_name": "High School World History", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on High School World History", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.895, - "details": { - "description": "min=0.895, mean=0.895, max=0.895, sum=1.789 (2)", - "tab": "Accuracy", - "High School Biology - Observed inference time (s)": "{\"description\": \"min=0.306, mean=0.306, max=0.306, sum=0.613 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.30645533454033635\"}", - "High School Chemistry - Observed inference time (s)": "{\"description\": \"min=0.362, mean=0.362, max=0.362, sum=0.724 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3618842803785954\"}", - "High School Computer Science - Observed inference time (s)": "{\"description\": \"min=0.432, mean=0.432, max=0.432, sum=0.864 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.43201621770858767\"}", - "High School European History - Observed inference time (s)": "{\"description\": \"min=0.874, mean=0.874, max=0.874, sum=1.747 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8736377629366788\"}", - "High School Geography - Observed inference time (s)": "{\"description\": \"min=0.373, mean=0.373, max=0.373, sum=0.746 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3727773331632518\"}", - "High School Government And Politics - Observed inference time (s)": "{\"description\": \"min=0.38, mean=0.38, max=0.38, sum=0.76 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.380075985903567\"}", - "High School Macroeconomics - Observed inference time (s)": "{\"description\": \"min=0.313, mean=0.313, max=0.313, sum=0.626 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3130294726445125\"}", - "High School Mathematics - Observed inference time (s)": "{\"description\": \"min=0.454, mean=0.454, max=0.454, sum=0.909 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4543530375869186\"}", - "High School Microeconomics - Observed inference time (s)": "{\"description\": \"min=0.475, mean=0.475, max=0.475, sum=0.95 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4752031294237666\"}", - "High School Physics - Observed inference time (s)": "{\"description\": \"min=0.341, mean=0.341, max=0.341, sum=0.683 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3413255830474247\"}", - "High School Psychology - Observed inference time (s)": "{\"description\": \"min=0.33, mean=0.33, max=0.33, sum=0.66 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.32982436013877936\"}", - "High School Statistics - Observed inference time (s)": "{\"description\": \"min=0.406, mean=0.406, max=0.406, sum=0.812 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4059625698460473\"}", - "High School US History - Observed inference time (s)": "{\"description\": \"min=0.744, mean=0.744, max=0.744, sum=1.488 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7440984506233066\"}", - "High School World History - Observed inference time (s)": "{\"description\": \"min=0.521, mean=0.521, max=0.521, sum=1.043 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5214709360388261\"}", - "High School Biology - # eval": "{\"description\": \"min=310, mean=310, max=310, sum=620 (2)\", \"tab\": \"General information\", \"score\": \"310.0\"}", - "High School Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Biology - # prompt tokens": "{\"description\": \"min=600.561, mean=600.561, max=600.561, sum=1201.123 (2)\", \"tab\": \"General information\", \"score\": \"600.5612903225806\"}", - "High School Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Chemistry - # eval": "{\"description\": \"min=203, mean=203, max=203, sum=406 (2)\", \"tab\": \"General information\", \"score\": \"203.0\"}", - "High School Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # prompt tokens": "{\"description\": \"min=572.798, mean=572.798, max=572.798, sum=1145.596 (2)\", \"tab\": \"General information\", \"score\": \"572.7980295566502\"}", - "High School Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "High School Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # prompt tokens": "{\"description\": \"min=988.24, mean=988.24, max=988.24, sum=1976.48 (2)\", \"tab\": \"General information\", \"score\": \"988.24\"}", - "High School Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School European History - # eval": "{\"description\": \"min=165, mean=165, max=165, sum=330 (2)\", \"tab\": \"General information\", \"score\": \"165.0\"}", - "High School European History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School European History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # prompt tokens": "{\"description\": \"min=3089.109, mean=3089.109, max=3089.109, sum=6178.218 (2)\", \"tab\": \"General information\", \"score\": \"3089.109090909091\"}", - "High School European History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Geography - # eval": "{\"description\": \"min=198, mean=198, max=198, sum=396 (2)\", \"tab\": \"General information\", \"score\": \"198.0\"}", - "High School Geography - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Geography - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # prompt tokens": "{\"description\": \"min=429.207, mean=429.207, max=429.207, sum=858.414 (2)\", \"tab\": \"General information\", \"score\": \"429.2070707070707\"}", - "High School Geography - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Government And Politics - # eval": "{\"description\": \"min=193, mean=193, max=193, sum=386 (2)\", \"tab\": \"General information\", \"score\": \"193.0\"}", - "High School Government And Politics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Government And Politics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # prompt tokens": "{\"description\": \"min=514.808, mean=514.808, max=514.808, sum=1029.617 (2)\", \"tab\": \"General information\", \"score\": \"514.8082901554404\"}", - "High School Government And Politics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Macroeconomics - # eval": "{\"description\": \"min=390, mean=390, max=390, sum=780 (2)\", \"tab\": \"General information\", \"score\": \"390.0\"}", - "High School Macroeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Macroeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # prompt tokens": "{\"description\": \"min=423.815, mean=423.815, max=423.815, sum=847.631 (2)\", \"tab\": \"General information\", \"score\": \"423.81538461538463\"}", - "High School Macroeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Mathematics - # eval": "{\"description\": \"min=270, mean=270, max=270, sum=540 (2)\", \"tab\": \"General information\", \"score\": \"270.0\"}", - "High School Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # prompt tokens": "{\"description\": \"min=584.13, mean=584.13, max=584.13, sum=1168.259 (2)\", \"tab\": \"General information\", \"score\": \"584.1296296296297\"}", - "High School Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Microeconomics - # eval": "{\"description\": \"min=238, mean=238, max=238, sum=476 (2)\", \"tab\": \"General information\", \"score\": \"238.0\"}", - "High School Microeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Microeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # prompt tokens": "{\"description\": \"min=443.345, mean=443.345, max=443.345, sum=886.689 (2)\", \"tab\": \"General information\", \"score\": \"443.34453781512605\"}", - "High School Microeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Physics - # eval": "{\"description\": \"min=151, mean=151, max=151, sum=302 (2)\", \"tab\": \"General information\", \"score\": \"151.0\"}", - "High School Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # prompt tokens": "{\"description\": \"min=622.775, mean=622.775, max=622.775, sum=1245.55 (2)\", \"tab\": \"General information\", \"score\": \"622.774834437086\"}", - "High School Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Psychology - # eval": "{\"description\": \"min=545, mean=545, max=545, sum=1090 (2)\", \"tab\": \"General information\", \"score\": \"545.0\"}", - "High School Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # prompt tokens": "{\"description\": \"min=558.873, mean=558.873, max=558.873, sum=1117.747 (2)\", \"tab\": \"General information\", \"score\": \"558.8733944954129\"}", - "High School Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Statistics - # eval": "{\"description\": \"min=216, mean=216, max=216, sum=432 (2)\", \"tab\": \"General information\", \"score\": \"216.0\"}", - "High School Statistics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Statistics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # prompt tokens": "{\"description\": \"min=913.644, mean=913.644, max=913.644, sum=1827.287 (2)\", \"tab\": \"General information\", \"score\": \"913.6435185185185\"}", - "High School Statistics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School US History - # eval": "{\"description\": \"min=204, mean=204, max=204, sum=408 (2)\", \"tab\": \"General information\", \"score\": \"204.0\"}", - "High School US History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School US History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # prompt tokens": "{\"description\": \"min=2477.446, mean=2477.446, max=2477.446, sum=4954.892 (2)\", \"tab\": \"General information\", \"score\": \"2477.4460784313724\"}", - "High School US History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School World History - # eval": "{\"description\": \"min=237, mean=237, max=237, sum=474 (2)\", \"tab\": \"General information\", \"score\": \"237.0\"}", - "High School World History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School World History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # prompt tokens": "{\"description\": \"min=1585.553, mean=1585.553, max=1585.553, sum=3171.105 (2)\", \"tab\": \"General information\", \"score\": \"1585.5527426160338\"}", - "High School World History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"high_school_world_history\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_high_school_world_history\"" - } - } - }, - { - "evaluation_name": "Human Sexuality", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Human Sexuality", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.885, - "details": { - "description": "min=0.885, mean=0.885, max=0.885, sum=1.771 (2)", - "tab": "Accuracy", - "Human Aging - Observed inference time (s)": "{\"description\": \"min=0.304, mean=0.304, max=0.304, sum=0.608 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3039867247166655\"}", - "Human Sexuality - Observed inference time (s)": "{\"description\": \"min=0.313, mean=0.313, max=0.313, sum=0.627 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3133269229918036\"}", - "Human Aging - # eval": "{\"description\": \"min=223, mean=223, max=223, sum=446 (2)\", \"tab\": \"General information\", \"score\": \"223.0\"}", - "Human Aging - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Aging - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Aging - # prompt tokens": "{\"description\": \"min=353.152, mean=353.152, max=353.152, sum=706.305 (2)\", \"tab\": \"General information\", \"score\": \"353.15246636771303\"}", - "Human Aging - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Human Sexuality - # eval": "{\"description\": \"min=131, mean=131, max=131, sum=262 (2)\", \"tab\": \"General information\", \"score\": \"131.0\"}", - "Human Sexuality - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Sexuality - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # prompt tokens": "{\"description\": \"min=394.748, mean=394.748, max=394.748, sum=789.496 (2)\", \"tab\": \"General information\", \"score\": \"394.7480916030534\"}", - "Human Sexuality - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"human_sexuality\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_human_sexuality\"" - } - } - }, - { - "evaluation_name": "International Law", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on International Law", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.917, - "details": { - "description": "min=0.917, mean=0.917, max=0.917, sum=1.835 (2)", - "tab": "Accuracy", - "International Law - Observed inference time (s)": "{\"description\": \"min=0.346, mean=0.346, max=0.346, sum=0.691 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.34560450049471264\"}", - "International Law - # eval": "{\"description\": \"min=121, mean=121, max=121, sum=242 (2)\", \"tab\": \"General information\", \"score\": \"121.0\"}", - "International Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "International Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "International Law - # prompt tokens": "{\"description\": \"min=720.182, mean=720.182, max=720.182, sum=1440.364 (2)\", \"tab\": \"General information\", \"score\": \"720.1818181818181\"}", - "International Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"international_law\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_international_law\"" - } - } - }, - { - "evaluation_name": "Logical Fallacies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Logical Fallacies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.877, - "details": { - "description": "min=0.877, mean=0.877, max=0.877, sum=1.755 (2)", - "tab": "Accuracy", - "Logical Fallacies - Observed inference time (s)": "{\"description\": \"min=0.357, mean=0.357, max=0.357, sum=0.713 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.35657415360760836\"}", - "Logical Fallacies - # eval": "{\"description\": \"min=163, mean=163, max=163, sum=326 (2)\", \"tab\": \"General information\", \"score\": \"163.0\"}", - "Logical Fallacies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Logical Fallacies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Logical Fallacies - # prompt tokens": "{\"description\": \"min=486.779, mean=486.779, max=486.779, sum=973.558 (2)\", \"tab\": \"General information\", \"score\": \"486.77914110429447\"}", - "Logical Fallacies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"logical_fallacies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_logical_fallacies\"" - } - } - }, - { - "evaluation_name": "Machine Learning", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Machine Learning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.661, - "details": { - "description": "min=0.661, mean=0.661, max=0.661, sum=1.321 (2)", - "tab": "Accuracy", - "Machine Learning - Observed inference time (s)": "{\"description\": \"min=0.375, mean=0.375, max=0.375, sum=0.751 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.37532309123447966\"}", - "Machine Learning - # eval": "{\"description\": \"min=112, mean=112, max=112, sum=224 (2)\", \"tab\": \"General information\", \"score\": \"112.0\"}", - "Machine Learning - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Machine Learning - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Machine Learning - # prompt tokens": "{\"description\": \"min=734.83, mean=734.83, max=734.83, sum=1469.661 (2)\", \"tab\": \"General information\", \"score\": \"734.8303571428571\"}", - "Machine Learning - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"machine_learning\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_machine_learning\"" - } - } - }, - { - "evaluation_name": "Management", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Management", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.883, - "details": { - "description": "min=0.883, mean=0.883, max=0.883, sum=1.767 (2)", - "tab": "Accuracy", - "Management - Observed inference time (s)": "{\"description\": \"min=0.284, mean=0.284, max=0.284, sum=0.567 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2837195535307949\"}", - "Management - # eval": "{\"description\": \"min=103, mean=103, max=103, sum=206 (2)\", \"tab\": \"General information\", \"score\": \"103.0\"}", - "Management - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Management - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Management - # prompt tokens": "{\"description\": \"min=315.359, mean=315.359, max=315.359, sum=630.718 (2)\", \"tab\": \"General information\", \"score\": \"315.3592233009709\"}", - "Management - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"management\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_management\"" - } - } - }, - { - "evaluation_name": "Marketing", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Marketing", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.915, - "details": { - "description": "min=0.915, mean=0.915, max=0.915, sum=1.829 (2)", - "tab": "Accuracy", - "Marketing - Observed inference time (s)": "{\"description\": \"min=0.477, mean=0.477, max=0.477, sum=0.955 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.47738775534507555\"}", - "Marketing - # eval": "{\"description\": \"min=234, mean=234, max=234, sum=468 (2)\", \"tab\": \"General information\", \"score\": \"234.0\"}", - "Marketing - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Marketing - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Marketing - # prompt tokens": "{\"description\": \"min=463.423, mean=463.423, max=463.423, sum=926.846 (2)\", \"tab\": \"General information\", \"score\": \"463.4230769230769\"}", - "Marketing - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"marketing\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_marketing\"" - } - } - }, - { - "evaluation_name": "Medical Genetics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Medical Genetics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.85, - "details": { - "description": "min=0.85, mean=0.85, max=0.85, sum=1.7 (2)", - "tab": "Accuracy", - "Medical Genetics - Observed inference time (s)": "{\"description\": \"min=0.358, mean=0.358, max=0.358, sum=0.715 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.35768274068832395\"}", - "Medical Genetics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Medical Genetics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Medical Genetics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Medical Genetics - # prompt tokens": "{\"description\": \"min=405.71, mean=405.71, max=405.71, sum=811.42 (2)\", \"tab\": \"General information\", \"score\": \"405.71\"}", - "Medical Genetics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"medical_genetics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_medical_genetics\"" - } - } - }, - { - "evaluation_name": "Miscellaneous", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Miscellaneous", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.899, - "details": { - "description": "min=0.899, mean=0.899, max=0.899, sum=1.798 (2)", - "tab": "Accuracy", - "Miscellaneous - Observed inference time (s)": "{\"description\": \"min=0.3, mean=0.3, max=0.3, sum=0.599 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.29965735912931984\"}", - "Miscellaneous - # eval": "{\"description\": \"min=783, mean=783, max=783, sum=1566 (2)\", \"tab\": \"General information\", \"score\": \"783.0\"}", - "Miscellaneous - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Miscellaneous - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Miscellaneous - # prompt tokens": "{\"description\": \"min=348.519, mean=348.519, max=348.519, sum=697.037 (2)\", \"tab\": \"General information\", \"score\": \"348.51851851851853\"}", - "Miscellaneous - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"miscellaneous\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_miscellaneous\"" - } - } - }, - { - "evaluation_name": "Moral Scenarios", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Moral Scenarios", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.646, - "details": { - "description": "min=0.646, mean=0.646, max=0.646, sum=1.292 (2)", - "tab": "Accuracy", - "Moral Disputes - Observed inference time (s)": "{\"description\": \"min=0.435, mean=0.435, max=0.435, sum=0.87 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.43506465757513324\"}", - "Moral Scenarios - Observed inference time (s)": "{\"description\": \"min=0.365, mean=0.365, max=0.365, sum=0.729 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.36451081030861626\"}", - "Moral Disputes - # eval": "{\"description\": \"min=346, mean=346, max=346, sum=692 (2)\", \"tab\": \"General information\", \"score\": \"346.0\"}", - "Moral Disputes - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Disputes - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Disputes - # prompt tokens": "{\"description\": \"min=540.038, mean=540.038, max=540.038, sum=1080.075 (2)\", \"tab\": \"General information\", \"score\": \"540.0375722543353\"}", - "Moral Disputes - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Moral Scenarios - # eval": "{\"description\": \"min=895, mean=895, max=895, sum=1790 (2)\", \"tab\": \"General information\", \"score\": \"895.0\"}", - "Moral Scenarios - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Scenarios - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # prompt tokens": "{\"description\": \"min=745.516, mean=745.516, max=745.516, sum=1491.032 (2)\", \"tab\": \"General information\", \"score\": \"745.5162011173185\"}", - "Moral Scenarios - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"moral_scenarios\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_moral_scenarios\"" - } - } - }, - { - "evaluation_name": "Nutrition", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Nutrition", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.866, - "details": { - "description": "min=0.866, mean=0.866, max=0.866, sum=1.732 (2)", - "tab": "Accuracy", - "Nutrition - Observed inference time (s)": "{\"description\": \"min=0.458, mean=0.458, max=0.458, sum=0.916 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4579993447447135\"}", - "Nutrition - # eval": "{\"description\": \"min=306, mean=306, max=306, sum=612 (2)\", \"tab\": \"General information\", \"score\": \"306.0\"}", - "Nutrition - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Nutrition - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Nutrition - # prompt tokens": "{\"description\": \"min=680.69, mean=680.69, max=680.69, sum=1361.379 (2)\", \"tab\": \"General information\", \"score\": \"680.6895424836601\"}", - "Nutrition - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"nutrition\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_nutrition\"" - } - } - }, - { - "evaluation_name": "Prehistory", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Prehistory", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.87, - "details": { - "description": "min=0.87, mean=0.87, max=0.87, sum=1.741 (2)", - "tab": "Accuracy", - "Prehistory - Observed inference time (s)": "{\"description\": \"min=0.45, mean=0.45, max=0.45, sum=0.901 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4504210890075307\"}", - "Prehistory - # eval": "{\"description\": \"min=324, mean=324, max=324, sum=648 (2)\", \"tab\": \"General information\", \"score\": \"324.0\"}", - "Prehistory - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Prehistory - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Prehistory - # prompt tokens": "{\"description\": \"min=602.145, mean=602.145, max=602.145, sum=1204.29 (2)\", \"tab\": \"General information\", \"score\": \"602.145061728395\"}", - "Prehistory - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"prehistory\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_prehistory\"" - } - } - }, - { - "evaluation_name": "Public Relations", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Public Relations", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.755, - "details": { - "description": "min=0.755, mean=0.755, max=0.755, sum=1.509 (2)", - "tab": "Accuracy", - "Public Relations - Observed inference time (s)": "{\"description\": \"min=0.651, mean=0.651, max=0.651, sum=1.302 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6507512135939164\"}", - "Public Relations - # eval": "{\"description\": \"min=110, mean=110, max=110, sum=220 (2)\", \"tab\": \"General information\", \"score\": \"110.0\"}", - "Public Relations - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Public Relations - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Public Relations - # prompt tokens": "{\"description\": \"min=462.036, mean=462.036, max=462.036, sum=924.073 (2)\", \"tab\": \"General information\", \"score\": \"462.03636363636366\"}", - "Public Relations - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"public_relations\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_public_relations\"" - } - } - }, - { - "evaluation_name": "Security Studies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Security Studies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.865, - "details": { - "description": "min=0.865, mean=0.865, max=0.865, sum=1.731 (2)", - "tab": "Accuracy", - "Security Studies - Observed inference time (s)": "{\"description\": \"min=0.46, mean=0.46, max=0.46, sum=0.919 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4596467952339017\"}", - "Security Studies - # eval": "{\"description\": \"min=245, mean=245, max=245, sum=490 (2)\", \"tab\": \"General information\", \"score\": \"245.0\"}", - "Security Studies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Security Studies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Security Studies - # prompt tokens": "{\"description\": \"min=1315.865, mean=1315.865, max=1315.865, sum=2631.731 (2)\", \"tab\": \"General information\", \"score\": \"1315.865306122449\"}", - "Security Studies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"security_studies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_security_studies\"" - } - } - }, - { - "evaluation_name": "Sociology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Sociology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.92, - "details": { - "description": "min=0.92, mean=0.92, max=0.92, sum=1.841 (2)", - "tab": "Accuracy", - "Sociology - Observed inference time (s)": "{\"description\": \"min=0.495, mean=0.495, max=0.495, sum=0.989 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4945164248717958\"}", - "Sociology - # eval": "{\"description\": \"min=201, mean=201, max=201, sum=402 (2)\", \"tab\": \"General information\", \"score\": \"201.0\"}", - "Sociology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Sociology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Sociology - # prompt tokens": "{\"description\": \"min=487.95, mean=487.95, max=487.95, sum=975.9 (2)\", \"tab\": \"General information\", \"score\": \"487.9502487562189\"}", - "Sociology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"sociology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_sociology\"" - } - } - }, - { - "evaluation_name": "Virology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Virology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.596, - "details": { - "description": "min=0.596, mean=0.596, max=0.596, sum=1.193 (2)", - "tab": "Accuracy", - "Virology - Observed inference time (s)": "{\"description\": \"min=0.304, mean=0.304, max=0.304, sum=0.608 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3041278597820236\"}", - "Virology - # eval": "{\"description\": \"min=166, mean=166, max=166, sum=332 (2)\", \"tab\": \"General information\", \"score\": \"166.0\"}", - "Virology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Virology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Virology - # prompt tokens": "{\"description\": \"min=395.349, mean=395.349, max=395.349, sum=790.699 (2)\", \"tab\": \"General information\", \"score\": \"395.34939759036143\"}", - "Virology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"virology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_virology\"" - } - } - }, - { - "evaluation_name": "World Religions", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on World Religions", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.901, - "details": { - "description": "min=0.901, mean=0.901, max=0.901, sum=1.801 (2)", - "tab": "Accuracy", - "World Religions - Observed inference time (s)": "{\"description\": \"min=0.297, mean=0.297, max=0.297, sum=0.595 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.29729281252587747\"}", - "World Religions - # eval": "{\"description\": \"min=171, mean=171, max=171, sum=342 (2)\", \"tab\": \"General information\", \"score\": \"171.0\"}", - "World Religions - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "World Religions - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "World Religions - # prompt tokens": "{\"description\": \"min=308.924, mean=308.924, max=308.924, sum=617.848 (2)\", \"tab\": \"General information\", \"score\": \"308.92397660818716\"}", - "World Religions - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"world_religions\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_world_religions\"" - } - } - }, - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.598, - "details": { - "description": "", - "tab": "Efficiency" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/mistralai_mixtral-8x7b-32kseqlen.json b/data/models/mistralai_mixtral-8x7b-32kseqlen.json deleted file mode 100644 index 300f09f30d16195ceb6269615f981a6646f35d07..0000000000000000000000000000000000000000 --- a/data/models/mistralai_mixtral-8x7b-32kseqlen.json +++ /dev/null @@ -1,1901 +0,0 @@ -{ - "model_info": { - "name": "Mixtral 8x7B 32K seqlen", - "id": "mistralai/mixtral-8x7b-32kseqlen", - "developer": "mistralai", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_lite/mistralai_mixtral-8x7b-32kseqlen/1774096306.427425", - "retrieved_timestamp": "1774096306.427425", - "source_metadata": { - "source_name": "helm_lite", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_lite", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_lite", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.51, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.6727715355805244\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.767, - "details": { - "description": "min=0.767, mean=0.767, max=0.767, sum=0.767 (1)", - "tab": "Accuracy", - "NarrativeQA - Observed inference time (s)": "{\"description\": \"min=0.65, mean=0.65, max=0.65, sum=0.65 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.649569604766201\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=4.575, mean=4.575, max=4.575, sum=4.575 (1)\", \"tab\": \"General information\", \"score\": \"4.574647887323944\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=3627.715, mean=3627.715, max=3627.715, sum=3627.715 (1)\", \"tab\": \"General information\", \"score\": \"3627.7154929577464\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (closed-book)", - "source_data": { - "dataset_name": "NaturalQuestions (closed-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (closed-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.427, - "details": { - "description": "min=0.427, mean=0.427, max=0.427, sum=0.427 (1)", - "tab": "Accuracy", - "NaturalQuestions (open-book) - Observed inference time (s)": "{\"description\": \"min=0.507, mean=0.507, max=0.507, sum=0.507 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.507013471364975\"}", - "NaturalQuestions (closed-book) - Observed inference time (s)": "{\"description\": \"min=0.513, mean=0.513, max=0.513, sum=0.513 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.5133386459350586\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.832, mean=4.832, max=4.832, sum=4.832 (1)\", \"tab\": \"General information\", \"score\": \"4.832\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.026, mean=0.026, max=0.026, sum=0.026 (1)\", \"tab\": \"General information\", \"score\": \"0.026\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=2268.728, mean=2268.728, max=2268.728, sum=2268.728 (1)\", \"tab\": \"General information\", \"score\": \"2268.728\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=0.991, mean=0.991, max=0.991, sum=0.991 (1)\", \"tab\": \"General information\", \"score\": \"0.991\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=142.069, mean=142.069, max=142.069, sum=142.069 (1)\", \"tab\": \"General information\", \"score\": \"142.069\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=0.999, mean=0.999, max=0.999, sum=0.999 (1)\", \"tab\": \"General information\", \"score\": \"0.999\"}" - } - }, - "generation_config": { - "additional_details": { - "mode": "\"closedbook\"" - } - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.868, - "details": { - "description": "min=0.868, mean=0.868, max=0.868, sum=0.868 (1)", - "tab": "Accuracy", - "OpenbookQA - Observed inference time (s)": "{\"description\": \"min=0.354, mean=0.354, max=0.354, sum=0.354 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.3542211503982544\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=280.15, mean=280.15, max=280.15, sum=280.15 (1)\", \"tab\": \"General information\", \"score\": \"280.15\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "dataset": "\"openbookqa\"", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.649, - "details": { - "description": "min=0.38, mean=0.649, max=0.93, sum=3.245 (5)", - "tab": "Accuracy", - "MMLU - Observed inference time (s)": "{\"description\": \"min=0.355, mean=0.36, max=0.366, sum=1.802 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.3604579553102192\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=402.44, mean=523.091, max=687.175, sum=2615.455 (5)\", \"tab\": \"General information\", \"score\": \"523.0910877192983\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"college_chemistry\", \"computer_security\", \"econometrics\", \"us_foreign_policy\"]", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MATH", - "source_data": { - "dataset_name": "MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Equivalent (CoT) on MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.494, - "details": { - "description": "min=0.289, mean=0.494, max=0.696, sum=3.459 (7)", - "tab": "Accuracy", - "MATH - Observed inference time (s)": "{\"description\": \"min=1.128, mean=1.528, max=2.033, sum=10.695 (7)\", \"tab\": \"Efficiency\", \"score\": \"1.527861329055259\"}", - "MATH - # eval": "{\"description\": \"min=30, mean=62.429, max=135, sum=437 (7)\", \"tab\": \"General information\", \"score\": \"62.42857142857143\"}", - "MATH - # train": "{\"description\": \"min=8, mean=8, max=8, sum=56 (7)\", \"tab\": \"General information\", \"score\": \"8.0\"}", - "MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MATH - # prompt tokens": "{\"description\": \"min=991.615, mean=1455.266, max=2502.962, sum=10186.865 (7)\", \"tab\": \"General information\", \"score\": \"1455.2664139976257\"}", - "MATH - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=7 (7)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"algebra\", \"counting_and_probability\", \"geometry\", \"intermediate_algebra\", \"number_theory\", \"prealgebra\", \"precalculus\"]", - "level": "\"1\"", - "use_official_examples": "\"False\"", - "use_chain_of_thought": "\"True\"" - } - } - }, - { - "evaluation_name": "GSM8K", - "source_data": { - "dataset_name": "GSM8K", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on GSM8K", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.622, - "details": { - "description": "min=0.622, mean=0.622, max=0.622, sum=0.622 (1)", - "tab": "Accuracy", - "GSM8K - Observed inference time (s)": "{\"description\": \"min=3.273, mean=3.273, max=3.273, sum=3.273 (1)\", \"tab\": \"Efficiency\", \"score\": \"3.2728567245006563\"}", - "GSM8K - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "GSM8K - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "GSM8K - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GSM8K - # prompt tokens": "{\"description\": \"min=1187.268, mean=1187.268, max=1187.268, sum=1187.268 (1)\", \"tab\": \"General information\", \"score\": \"1187.268\"}", - "GSM8K - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "LegalBench", - "source_data": { - "dataset_name": "LegalBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on LegalBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.63, - "details": { - "description": "min=0.428, mean=0.63, max=0.853, sum=3.15 (5)", - "tab": "Accuracy", - "LegalBench - Observed inference time (s)": "{\"description\": \"min=0.369, mean=0.41, max=0.512, sum=2.05 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.40995627823211056\"}", - "LegalBench - # eval": "{\"description\": \"min=95, mean=409.4, max=1000, sum=2047 (5)\", \"tab\": \"General information\", \"score\": \"409.4\"}", - "LegalBench - # train": "{\"description\": \"min=1.969, mean=4.194, max=5, sum=20.969 (5)\", \"tab\": \"General information\", \"score\": \"4.1938775510204085\"}", - "LegalBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "LegalBench - # prompt tokens": "{\"description\": \"min=219.453, mean=998.503, max=3534.259, sum=4992.513 (5)\", \"tab\": \"General information\", \"score\": \"998.5025315575822\"}", - "LegalBench - # output tokens": "{\"description\": \"min=0.998, mean=1.0, max=1, sum=4.998 (5)\", \"tab\": \"General information\", \"score\": \"0.9995918367346939\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"abercrombie\", \"corporate_lobbying\", \"function_of_decision_section\", \"international_citizenship_questions\", \"proa\"]" - } - } - }, - { - "evaluation_name": "MedQA", - "source_data": { - "dataset_name": "MedQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MedQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.652, - "details": { - "description": "min=0.652, mean=0.652, max=0.652, sum=0.652 (1)", - "tab": "Accuracy", - "MedQA - Observed inference time (s)": "{\"description\": \"min=0.353, mean=0.353, max=0.353, sum=0.353 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.35297762423338996\"}", - "MedQA - # eval": "{\"description\": \"min=503, mean=503, max=503, sum=503 (1)\", \"tab\": \"General information\", \"score\": \"503.0\"}", - "MedQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MedQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MedQA - # prompt tokens": "{\"description\": \"min=1193.093, mean=1193.093, max=1193.093, sum=1193.093 (1)\", \"tab\": \"General information\", \"score\": \"1193.0934393638172\"}", - "MedQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WMT 2014", - "source_data": { - "dataset_name": "WMT 2014", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "BLEU-4 on WMT 2014", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.19, - "details": { - "description": "min=0.099, mean=0.19, max=0.23, sum=0.949 (5)", - "tab": "Accuracy", - "WMT 2014 - Observed inference time (s)": "{\"description\": \"min=1.115, mean=1.202, max=1.294, sum=6.011 (5)\", \"tab\": \"Efficiency\", \"score\": \"1.2021687407719377\"}", - "WMT 2014 - # eval": "{\"description\": \"min=503, mean=568.8, max=832, sum=2844 (5)\", \"tab\": \"General information\", \"score\": \"568.8\"}", - "WMT 2014 - # train": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "WMT 2014 - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WMT 2014 - # prompt tokens": "{\"description\": \"min=130.306, mean=144.433, max=163.018, sum=722.166 (5)\", \"tab\": \"General information\", \"score\": \"144.43317355482492\"}", - "WMT 2014 - # output tokens": "{\"description\": \"min=0.994, mean=0.999, max=1, sum=4.994 (5)\", \"tab\": \"General information\", \"score\": \"0.998798076923077\"}" - } - }, - "generation_config": { - "additional_details": { - "language_pair": "[\"cs-en\", \"de-en\", \"fr-en\", \"hi-en\", \"ru-en\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_mmlu/mistralai_mixtral-8x7b-32kseqlen/1774096312.00548", - "retrieved_timestamp": "1774096312.00548", - "source_metadata": { - "source_name": "helm_mmlu", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_mmlu", - "evaluation_results": [ - { - "evaluation_name": "MMLU All Subjects", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU All Subjects", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.717, - "details": { - "description": "min=0.38, mean=0.717, max=0.933, sum=81.767 (114)", - "tab": "Accuracy", - "MMLU All Subjects - Observed inference time (s)": "{\"description\": \"min=0.303, mean=0.364, max=0.667, sum=41.491 (114)\", \"tab\": \"Efficiency\", \"score\": \"0.36396022974729103\"}", - "MMLU All Subjects - # eval": "{\"description\": \"min=100, mean=246.351, max=1534, sum=28084 (114)\", \"tab\": \"General information\", \"score\": \"246.35087719298247\"}", - "MMLU All Subjects - # train": "{\"description\": \"min=5, mean=5, max=5, sum=570 (114)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU All Subjects - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU All Subjects - # prompt tokens": "{\"description\": \"min=308.924, mean=696.273, max=3089.109, sum=79375.178 (114)\", \"tab\": \"General information\", \"score\": \"696.2734899593811\"}", - "MMLU All Subjects - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=114 (114)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_name": "Abstract Algebra", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Abstract Algebra", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.38, - "details": { - "description": "min=0.38, mean=0.38, max=0.38, sum=0.76 (2)", - "tab": "Accuracy", - "Abstract Algebra - Observed inference time (s)": "{\"description\": \"min=0.332, mean=0.332, max=0.332, sum=0.665 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3324201321601868\"}", - "Abstract Algebra - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Abstract Algebra - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Abstract Algebra - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Abstract Algebra - # prompt tokens": "{\"description\": \"min=402.44, mean=402.44, max=402.44, sum=804.88 (2)\", \"tab\": \"General information\", \"score\": \"402.44\"}", - "Abstract Algebra - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"abstract_algebra\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_abstract_algebra\"" - } - } - }, - { - "evaluation_name": "Anatomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Anatomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.696, - "details": { - "description": "min=0.696, mean=0.696, max=0.696, sum=1.393 (2)", - "tab": "Accuracy", - "Anatomy - Observed inference time (s)": "{\"description\": \"min=0.338, mean=0.338, max=0.338, sum=0.676 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.33777406480577254\"}", - "Anatomy - # eval": "{\"description\": \"min=135, mean=135, max=135, sum=270 (2)\", \"tab\": \"General information\", \"score\": \"135.0\"}", - "Anatomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Anatomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Anatomy - # prompt tokens": "{\"description\": \"min=407.089, mean=407.089, max=407.089, sum=814.178 (2)\", \"tab\": \"General information\", \"score\": \"407.0888888888889\"}", - "Anatomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"anatomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_anatomy\"" - } - } - }, - { - "evaluation_name": "College Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on College Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.51, - "details": { - "description": "min=0.51, mean=0.51, max=0.51, sum=1.02 (2)", - "tab": "Accuracy", - "College Chemistry - Observed inference time (s)": "{\"description\": \"min=0.386, mean=0.386, max=0.386, sum=0.773 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.386492395401001\"}", - "College Biology - Observed inference time (s)": "{\"description\": \"min=0.366, mean=0.366, max=0.366, sum=0.733 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3663763701915741\"}", - "College Computer Science - Observed inference time (s)": "{\"description\": \"min=0.367, mean=0.367, max=0.367, sum=0.735 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.36740577936172486\"}", - "College Mathematics - Observed inference time (s)": "{\"description\": \"min=0.356, mean=0.356, max=0.356, sum=0.712 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.35591145277023317\"}", - "College Medicine - Observed inference time (s)": "{\"description\": \"min=0.347, mean=0.347, max=0.347, sum=0.695 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.347429724787012\"}", - "College Physics - Observed inference time (s)": "{\"description\": \"min=0.361, mean=0.361, max=0.361, sum=0.721 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3606654686086318\"}", - "College Chemistry - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Chemistry - # prompt tokens": "{\"description\": \"min=627.71, mean=627.71, max=627.71, sum=1255.42 (2)\", \"tab\": \"General information\", \"score\": \"627.71\"}", - "College Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Biology - # eval": "{\"description\": \"min=144, mean=144, max=144, sum=288 (2)\", \"tab\": \"General information\", \"score\": \"144.0\"}", - "College Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # prompt tokens": "{\"description\": \"min=550.799, mean=550.799, max=550.799, sum=1101.597 (2)\", \"tab\": \"General information\", \"score\": \"550.7986111111111\"}", - "College Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # prompt tokens": "{\"description\": \"min=902.17, mean=902.17, max=902.17, sum=1804.34 (2)\", \"tab\": \"General information\", \"score\": \"902.17\"}", - "College Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Mathematics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # prompt tokens": "{\"description\": \"min=658.31, mean=658.31, max=658.31, sum=1316.62 (2)\", \"tab\": \"General information\", \"score\": \"658.31\"}", - "College Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Medicine - # eval": "{\"description\": \"min=173, mean=173, max=173, sum=346 (2)\", \"tab\": \"General information\", \"score\": \"173.0\"}", - "College Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # prompt tokens": "{\"description\": \"min=592.41, mean=592.41, max=592.41, sum=1184.821 (2)\", \"tab\": \"General information\", \"score\": \"592.4104046242775\"}", - "College Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Physics - # eval": "{\"description\": \"min=102, mean=102, max=102, sum=204 (2)\", \"tab\": \"General information\", \"score\": \"102.0\"}", - "College Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # prompt tokens": "{\"description\": \"min=551.029, mean=551.029, max=551.029, sum=1102.059 (2)\", \"tab\": \"General information\", \"score\": \"551.0294117647059\"}", - "College Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"college_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_college_physics\"" - } - } - }, - { - "evaluation_name": "Computer Security", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Computer Security", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.81, - "details": { - "description": "min=0.81, mean=0.81, max=0.81, sum=1.62 (2)", - "tab": "Accuracy", - "Computer Security - Observed inference time (s)": "{\"description\": \"min=0.342, mean=0.342, max=0.342, sum=0.684 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.34211899518966676\"}", - "Computer Security - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Computer Security - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Computer Security - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Computer Security - # prompt tokens": "{\"description\": \"min=424.94, mean=424.94, max=424.94, sum=849.88 (2)\", \"tab\": \"General information\", \"score\": \"424.94\"}", - "Computer Security - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"computer_security\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_computer_security\"" - } - } - }, - { - "evaluation_name": "Econometrics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Econometrics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.605, - "details": { - "description": "min=0.605, mean=0.605, max=0.605, sum=1.211 (2)", - "tab": "Accuracy", - "Econometrics - Observed inference time (s)": "{\"description\": \"min=0.354, mean=0.354, max=0.354, sum=0.708 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3541024630529839\"}", - "Econometrics - # eval": "{\"description\": \"min=114, mean=114, max=114, sum=228 (2)\", \"tab\": \"General information\", \"score\": \"114.0\"}", - "Econometrics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Econometrics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Econometrics - # prompt tokens": "{\"description\": \"min=687.175, mean=687.175, max=687.175, sum=1374.351 (2)\", \"tab\": \"General information\", \"score\": \"687.1754385964912\"}", - "Econometrics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"econometrics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_econometrics\"" - } - } - }, - { - "evaluation_name": "Global Facts", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Global Facts", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.46, - "details": { - "description": "min=0.46, mean=0.46, max=0.46, sum=0.92 (2)", - "tab": "Accuracy", - "Global Facts - Observed inference time (s)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=1.335 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.667280240058899\"}", - "Global Facts - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Global Facts - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Global Facts - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Global Facts - # prompt tokens": "{\"description\": \"min=483.47, mean=483.47, max=483.47, sum=966.94 (2)\", \"tab\": \"General information\", \"score\": \"483.47\"}", - "Global Facts - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"global_facts\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_global_facts\"" - } - } - }, - { - "evaluation_name": "Jurisprudence", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Jurisprudence", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.833, - "details": { - "description": "min=0.833, mean=0.833, max=0.833, sum=1.667 (2)", - "tab": "Accuracy", - "Jurisprudence - Observed inference time (s)": "{\"description\": \"min=0.338, mean=0.338, max=0.338, sum=0.677 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3384844925668504\"}", - "Jurisprudence - # eval": "{\"description\": \"min=108, mean=108, max=108, sum=216 (2)\", \"tab\": \"General information\", \"score\": \"108.0\"}", - "Jurisprudence - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Jurisprudence - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Jurisprudence - # prompt tokens": "{\"description\": \"min=451.093, mean=451.093, max=451.093, sum=902.185 (2)\", \"tab\": \"General information\", \"score\": \"451.0925925925926\"}", - "Jurisprudence - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"jurisprudence\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_jurisprudence\"" - } - } - }, - { - "evaluation_name": "Philosophy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Philosophy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.797, - "details": { - "description": "min=0.797, mean=0.797, max=0.797, sum=1.595 (2)", - "tab": "Accuracy", - "Philosophy - Observed inference time (s)": "{\"description\": \"min=0.323, mean=0.323, max=0.323, sum=0.645 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.322712682067773\"}", - "Philosophy - # eval": "{\"description\": \"min=311, mean=311, max=311, sum=622 (2)\", \"tab\": \"General information\", \"score\": \"311.0\"}", - "Philosophy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Philosophy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Philosophy - # prompt tokens": "{\"description\": \"min=373.82, mean=373.82, max=373.82, sum=747.64 (2)\", \"tab\": \"General information\", \"score\": \"373.81993569131834\"}", - "Philosophy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"philosophy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_philosophy\"" - } - } - }, - { - "evaluation_name": "Professional Psychology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Professional Psychology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.779, - "details": { - "description": "min=0.779, mean=0.779, max=0.779, sum=1.559 (2)", - "tab": "Accuracy", - "Professional Medicine - Observed inference time (s)": "{\"description\": \"min=0.416, mean=0.416, max=0.416, sum=0.832 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.41612808669314666\"}", - "Professional Accounting - Observed inference time (s)": "{\"description\": \"min=0.346, mean=0.346, max=0.346, sum=0.691 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.34556762884694636\"}", - "Professional Law - Observed inference time (s)": "{\"description\": \"min=0.44, mean=0.44, max=0.44, sum=0.879 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4395133182309286\"}", - "Professional Psychology - Observed inference time (s)": "{\"description\": \"min=0.328, mean=0.328, max=0.328, sum=0.655 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3276863078665889\"}", - "Professional Medicine - # eval": "{\"description\": \"min=272, mean=272, max=272, sum=544 (2)\", \"tab\": \"General information\", \"score\": \"272.0\"}", - "Professional Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Medicine - # prompt tokens": "{\"description\": \"min=1279.143, mean=1279.143, max=1279.143, sum=2558.287 (2)\", \"tab\": \"General information\", \"score\": \"1279.1433823529412\"}", - "Professional Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Accounting - # eval": "{\"description\": \"min=282, mean=282, max=282, sum=564 (2)\", \"tab\": \"General information\", \"score\": \"282.0\"}", - "Professional Accounting - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Accounting - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # prompt tokens": "{\"description\": \"min=796.496, mean=796.496, max=796.496, sum=1592.993 (2)\", \"tab\": \"General information\", \"score\": \"796.4964539007092\"}", - "Professional Accounting - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Law - # eval": "{\"description\": \"min=1534, mean=1534, max=1534, sum=3068 (2)\", \"tab\": \"General information\", \"score\": \"1534.0\"}", - "Professional Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # prompt tokens": "{\"description\": \"min=1849.711, mean=1849.711, max=1849.711, sum=3699.421 (2)\", \"tab\": \"General information\", \"score\": \"1849.7105606258149\"}", - "Professional Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Psychology - # eval": "{\"description\": \"min=612, mean=612, max=612, sum=1224 (2)\", \"tab\": \"General information\", \"score\": \"612.0\"}", - "Professional Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # prompt tokens": "{\"description\": \"min=645.278, mean=645.278, max=645.278, sum=1290.556 (2)\", \"tab\": \"General information\", \"score\": \"645.2777777777778\"}", - "Professional Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"professional_psychology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_professional_psychology\"" - } - } - }, - { - "evaluation_name": "Us Foreign Policy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Us Foreign Policy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.93, - "details": { - "description": "min=0.93, mean=0.93, max=0.93, sum=1.86 (2)", - "tab": "Accuracy", - "Us Foreign Policy - Observed inference time (s)": "{\"description\": \"min=0.318, mean=0.318, max=0.318, sum=0.637 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3183705282211304\"}", - "Us Foreign Policy - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Us Foreign Policy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Us Foreign Policy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Us Foreign Policy - # prompt tokens": "{\"description\": \"min=473.19, mean=473.19, max=473.19, sum=946.38 (2)\", \"tab\": \"General information\", \"score\": \"473.19\"}", - "Us Foreign Policy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"us_foreign_policy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_us_foreign_policy\"" - } - } - }, - { - "evaluation_name": "Astronomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Astronomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.829, - "details": { - "description": "min=0.829, mean=0.829, max=0.829, sum=1.658 (2)", - "tab": "Accuracy", - "Astronomy - Observed inference time (s)": "{\"description\": \"min=0.365, mean=0.365, max=0.365, sum=0.73 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.36493434560926336\"}", - "Astronomy - # eval": "{\"description\": \"min=152, mean=152, max=152, sum=304 (2)\", \"tab\": \"General information\", \"score\": \"152.0\"}", - "Astronomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Astronomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Astronomy - # prompt tokens": "{\"description\": \"min=665.987, mean=665.987, max=665.987, sum=1331.974 (2)\", \"tab\": \"General information\", \"score\": \"665.9868421052631\"}", - "Astronomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"astronomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_astronomy\"" - } - } - }, - { - "evaluation_name": "Business Ethics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Business Ethics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.72, - "details": { - "description": "min=0.72, mean=0.72, max=0.72, sum=1.44 (2)", - "tab": "Accuracy", - "Business Ethics - Observed inference time (s)": "{\"description\": \"min=0.365, mean=0.365, max=0.365, sum=0.73 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3650094985961914\"}", - "Business Ethics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Business Ethics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Business Ethics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Business Ethics - # prompt tokens": "{\"description\": \"min=644.6, mean=644.6, max=644.6, sum=1289.2 (2)\", \"tab\": \"General information\", \"score\": \"644.6\"}", - "Business Ethics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"business_ethics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_business_ethics\"" - } - } - }, - { - "evaluation_name": "Clinical Knowledge", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Clinical Knowledge", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.785, - "details": { - "description": "min=0.785, mean=0.785, max=0.785, sum=1.57 (2)", - "tab": "Accuracy", - "Clinical Knowledge - Observed inference time (s)": "{\"description\": \"min=0.335, mean=0.335, max=0.335, sum=0.671 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.33542148392155485\"}", - "Clinical Knowledge - # eval": "{\"description\": \"min=265, mean=265, max=265, sum=530 (2)\", \"tab\": \"General information\", \"score\": \"265.0\"}", - "Clinical Knowledge - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Clinical Knowledge - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Clinical Knowledge - # prompt tokens": "{\"description\": \"min=487.174, mean=487.174, max=487.174, sum=974.347 (2)\", \"tab\": \"General information\", \"score\": \"487.1735849056604\"}", - "Clinical Knowledge - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"clinical_knowledge\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_clinical_knowledge\"" - } - } - }, - { - "evaluation_name": "Conceptual Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Conceptual Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.681, - "details": { - "description": "min=0.681, mean=0.681, max=0.681, sum=1.362 (2)", - "tab": "Accuracy", - "Conceptual Physics - Observed inference time (s)": "{\"description\": \"min=0.339, mean=0.339, max=0.339, sum=0.679 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3393338994776949\"}", - "Conceptual Physics - # eval": "{\"description\": \"min=235, mean=235, max=235, sum=470 (2)\", \"tab\": \"General information\", \"score\": \"235.0\"}", - "Conceptual Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Conceptual Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Conceptual Physics - # prompt tokens": "{\"description\": \"min=334.285, mean=334.285, max=334.285, sum=668.57 (2)\", \"tab\": \"General information\", \"score\": \"334.2851063829787\"}", - "Conceptual Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"conceptual_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_conceptual_physics\"" - } - } - }, - { - "evaluation_name": "Electrical Engineering", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Electrical Engineering", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.676, - "details": { - "description": "min=0.676, mean=0.676, max=0.676, sum=1.352 (2)", - "tab": "Accuracy", - "Electrical Engineering - Observed inference time (s)": "{\"description\": \"min=0.351, mean=0.351, max=0.351, sum=0.702 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.35104844159093396\"}", - "Electrical Engineering - # eval": "{\"description\": \"min=145, mean=145, max=145, sum=290 (2)\", \"tab\": \"General information\", \"score\": \"145.0\"}", - "Electrical Engineering - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Electrical Engineering - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Electrical Engineering - # prompt tokens": "{\"description\": \"min=501.379, mean=501.379, max=501.379, sum=1002.759 (2)\", \"tab\": \"General information\", \"score\": \"501.37931034482756\"}", - "Electrical Engineering - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"electrical_engineering\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_electrical_engineering\"" - } - } - }, - { - "evaluation_name": "Elementary Mathematics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Elementary Mathematics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.476, - "details": { - "description": "min=0.476, mean=0.476, max=0.476, sum=0.952 (2)", - "tab": "Accuracy", - "Elementary Mathematics - Observed inference time (s)": "{\"description\": \"min=0.43, mean=0.43, max=0.43, sum=0.86 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4298846198137475\"}", - "Elementary Mathematics - # eval": "{\"description\": \"min=378, mean=378, max=378, sum=756 (2)\", \"tab\": \"General information\", \"score\": \"378.0\"}", - "Elementary Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Elementary Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Elementary Mathematics - # prompt tokens": "{\"description\": \"min=613.386, mean=613.386, max=613.386, sum=1226.772 (2)\", \"tab\": \"General information\", \"score\": \"613.3862433862433\"}", - "Elementary Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"elementary_mathematics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_elementary_mathematics\"" - } - } - }, - { - "evaluation_name": "Formal Logic", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Formal Logic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.532, - "details": { - "description": "min=0.532, mean=0.532, max=0.532, sum=1.063 (2)", - "tab": "Accuracy", - "Formal Logic - Observed inference time (s)": "{\"description\": \"min=0.37, mean=0.37, max=0.37, sum=0.741 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.37032828255305217\"}", - "Formal Logic - # eval": "{\"description\": \"min=126, mean=126, max=126, sum=252 (2)\", \"tab\": \"General information\", \"score\": \"126.0\"}", - "Formal Logic - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Formal Logic - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Formal Logic - # prompt tokens": "{\"description\": \"min=718.984, mean=718.984, max=718.984, sum=1437.968 (2)\", \"tab\": \"General information\", \"score\": \"718.984126984127\"}", - "Formal Logic - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"formal_logic\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_formal_logic\"" - } - } - }, - { - "evaluation_name": "High School World History", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on High School World History", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.886, - "details": { - "description": "min=0.886, mean=0.886, max=0.886, sum=1.772 (2)", - "tab": "Accuracy", - "High School Biology - Observed inference time (s)": "{\"description\": \"min=0.328, mean=0.328, max=0.328, sum=0.657 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3284358686016452\"}", - "High School Chemistry - Observed inference time (s)": "{\"description\": \"min=0.326, mean=0.326, max=0.326, sum=0.652 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.32620196624342446\"}", - "High School Computer Science - Observed inference time (s)": "{\"description\": \"min=0.485, mean=0.485, max=0.485, sum=0.969 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.48452038288116456\"}", - "High School European History - Observed inference time (s)": "{\"description\": \"min=0.641, mean=0.641, max=0.641, sum=1.283 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6413424491882325\"}", - "High School Geography - Observed inference time (s)": "{\"description\": \"min=0.327, mean=0.327, max=0.327, sum=0.653 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3266212759595929\"}", - "High School Government And Politics - Observed inference time (s)": "{\"description\": \"min=0.337, mean=0.337, max=0.337, sum=0.675 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.33742881191826857\"}", - "High School Macroeconomics - Observed inference time (s)": "{\"description\": \"min=0.327, mean=0.327, max=0.327, sum=0.654 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3271804552811843\"}", - "High School Mathematics - Observed inference time (s)": "{\"description\": \"min=0.328, mean=0.328, max=0.328, sum=0.655 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3277335458331638\"}", - "High School Microeconomics - Observed inference time (s)": "{\"description\": \"min=0.329, mean=0.329, max=0.329, sum=0.658 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3291829443779312\"}", - "High School Physics - Observed inference time (s)": "{\"description\": \"min=0.337, mean=0.337, max=0.337, sum=0.674 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.33715188266425733\"}", - "High School Psychology - Observed inference time (s)": "{\"description\": \"min=0.396, mean=0.396, max=0.396, sum=0.792 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.39586829351722647\"}", - "High School Statistics - Observed inference time (s)": "{\"description\": \"min=0.376, mean=0.376, max=0.376, sum=0.753 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.37643481846208926\"}", - "High School US History - Observed inference time (s)": "{\"description\": \"min=0.531, mean=0.531, max=0.531, sum=1.062 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.531247288573022\"}", - "High School World History - Observed inference time (s)": "{\"description\": \"min=0.44, mean=0.44, max=0.44, sum=0.88 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.44013202341297003\"}", - "High School Biology - # eval": "{\"description\": \"min=310, mean=310, max=310, sum=620 (2)\", \"tab\": \"General information\", \"score\": \"310.0\"}", - "High School Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Biology - # prompt tokens": "{\"description\": \"min=600.561, mean=600.561, max=600.561, sum=1201.123 (2)\", \"tab\": \"General information\", \"score\": \"600.5612903225806\"}", - "High School Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Chemistry - # eval": "{\"description\": \"min=203, mean=203, max=203, sum=406 (2)\", \"tab\": \"General information\", \"score\": \"203.0\"}", - "High School Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # prompt tokens": "{\"description\": \"min=572.798, mean=572.798, max=572.798, sum=1145.596 (2)\", \"tab\": \"General information\", \"score\": \"572.7980295566502\"}", - "High School Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "High School Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # prompt tokens": "{\"description\": \"min=988.24, mean=988.24, max=988.24, sum=1976.48 (2)\", \"tab\": \"General information\", \"score\": \"988.24\"}", - "High School Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School European History - # eval": "{\"description\": \"min=165, mean=165, max=165, sum=330 (2)\", \"tab\": \"General information\", \"score\": \"165.0\"}", - "High School European History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School European History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # prompt tokens": "{\"description\": \"min=3089.109, mean=3089.109, max=3089.109, sum=6178.218 (2)\", \"tab\": \"General information\", \"score\": \"3089.109090909091\"}", - "High School European History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Geography - # eval": "{\"description\": \"min=198, mean=198, max=198, sum=396 (2)\", \"tab\": \"General information\", \"score\": \"198.0\"}", - "High School Geography - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Geography - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # prompt tokens": "{\"description\": \"min=429.207, mean=429.207, max=429.207, sum=858.414 (2)\", \"tab\": \"General information\", \"score\": \"429.2070707070707\"}", - "High School Geography - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Government And Politics - # eval": "{\"description\": \"min=193, mean=193, max=193, sum=386 (2)\", \"tab\": \"General information\", \"score\": \"193.0\"}", - "High School Government And Politics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Government And Politics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # prompt tokens": "{\"description\": \"min=514.808, mean=514.808, max=514.808, sum=1029.617 (2)\", \"tab\": \"General information\", \"score\": \"514.8082901554404\"}", - "High School Government And Politics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Macroeconomics - # eval": "{\"description\": \"min=390, mean=390, max=390, sum=780 (2)\", \"tab\": \"General information\", \"score\": \"390.0\"}", - "High School Macroeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Macroeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # prompt tokens": "{\"description\": \"min=423.815, mean=423.815, max=423.815, sum=847.631 (2)\", \"tab\": \"General information\", \"score\": \"423.81538461538463\"}", - "High School Macroeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Mathematics - # eval": "{\"description\": \"min=270, mean=270, max=270, sum=540 (2)\", \"tab\": \"General information\", \"score\": \"270.0\"}", - "High School Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # prompt tokens": "{\"description\": \"min=584.13, mean=584.13, max=584.13, sum=1168.259 (2)\", \"tab\": \"General information\", \"score\": \"584.1296296296297\"}", - "High School Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Microeconomics - # eval": "{\"description\": \"min=238, mean=238, max=238, sum=476 (2)\", \"tab\": \"General information\", \"score\": \"238.0\"}", - "High School Microeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Microeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # prompt tokens": "{\"description\": \"min=443.345, mean=443.345, max=443.345, sum=886.689 (2)\", \"tab\": \"General information\", \"score\": \"443.34453781512605\"}", - "High School Microeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Physics - # eval": "{\"description\": \"min=151, mean=151, max=151, sum=302 (2)\", \"tab\": \"General information\", \"score\": \"151.0\"}", - "High School Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # prompt tokens": "{\"description\": \"min=622.775, mean=622.775, max=622.775, sum=1245.55 (2)\", \"tab\": \"General information\", \"score\": \"622.774834437086\"}", - "High School Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Psychology - # eval": "{\"description\": \"min=545, mean=545, max=545, sum=1090 (2)\", \"tab\": \"General information\", \"score\": \"545.0\"}", - "High School Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # prompt tokens": "{\"description\": \"min=558.873, mean=558.873, max=558.873, sum=1117.747 (2)\", \"tab\": \"General information\", \"score\": \"558.8733944954129\"}", - "High School Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Statistics - # eval": "{\"description\": \"min=216, mean=216, max=216, sum=432 (2)\", \"tab\": \"General information\", \"score\": \"216.0\"}", - "High School Statistics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Statistics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # prompt tokens": "{\"description\": \"min=913.644, mean=913.644, max=913.644, sum=1827.287 (2)\", \"tab\": \"General information\", \"score\": \"913.6435185185185\"}", - "High School Statistics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School US History - # eval": "{\"description\": \"min=204, mean=204, max=204, sum=408 (2)\", \"tab\": \"General information\", \"score\": \"204.0\"}", - "High School US History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School US History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # prompt tokens": "{\"description\": \"min=2477.446, mean=2477.446, max=2477.446, sum=4954.892 (2)\", \"tab\": \"General information\", \"score\": \"2477.4460784313724\"}", - "High School US History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School World History - # eval": "{\"description\": \"min=237, mean=237, max=237, sum=474 (2)\", \"tab\": \"General information\", \"score\": \"237.0\"}", - "High School World History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School World History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # prompt tokens": "{\"description\": \"min=1585.553, mean=1585.553, max=1585.553, sum=3171.105 (2)\", \"tab\": \"General information\", \"score\": \"1585.5527426160338\"}", - "High School World History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"high_school_world_history\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_high_school_world_history\"" - } - } - }, - { - "evaluation_name": "Human Sexuality", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Human Sexuality", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.87, - "details": { - "description": "min=0.87, mean=0.87, max=0.87, sum=1.74 (2)", - "tab": "Accuracy", - "Human Aging - Observed inference time (s)": "{\"description\": \"min=0.303, mean=0.303, max=0.303, sum=0.607 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.30348238068311206\"}", - "Human Sexuality - Observed inference time (s)": "{\"description\": \"min=0.304, mean=0.304, max=0.304, sum=0.608 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.30424233429304515\"}", - "Human Aging - # eval": "{\"description\": \"min=223, mean=223, max=223, sum=446 (2)\", \"tab\": \"General information\", \"score\": \"223.0\"}", - "Human Aging - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Aging - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Aging - # prompt tokens": "{\"description\": \"min=353.152, mean=353.152, max=353.152, sum=706.305 (2)\", \"tab\": \"General information\", \"score\": \"353.15246636771303\"}", - "Human Aging - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Human Sexuality - # eval": "{\"description\": \"min=131, mean=131, max=131, sum=262 (2)\", \"tab\": \"General information\", \"score\": \"131.0\"}", - "Human Sexuality - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Sexuality - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # prompt tokens": "{\"description\": \"min=394.748, mean=394.748, max=394.748, sum=789.496 (2)\", \"tab\": \"General information\", \"score\": \"394.7480916030534\"}", - "Human Sexuality - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"human_sexuality\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_human_sexuality\"" - } - } - }, - { - "evaluation_name": "International Law", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on International Law", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.86, - "details": { - "description": "min=0.86, mean=0.86, max=0.86, sum=1.719 (2)", - "tab": "Accuracy", - "International Law - Observed inference time (s)": "{\"description\": \"min=0.354, mean=0.354, max=0.354, sum=0.708 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.354031091879222\"}", - "International Law - # eval": "{\"description\": \"min=121, mean=121, max=121, sum=242 (2)\", \"tab\": \"General information\", \"score\": \"121.0\"}", - "International Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "International Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "International Law - # prompt tokens": "{\"description\": \"min=720.182, mean=720.182, max=720.182, sum=1440.364 (2)\", \"tab\": \"General information\", \"score\": \"720.1818181818181\"}", - "International Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"international_law\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_international_law\"" - } - } - }, - { - "evaluation_name": "Logical Fallacies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Logical Fallacies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.767, - "details": { - "description": "min=0.767, mean=0.767, max=0.767, sum=1.534 (2)", - "tab": "Accuracy", - "Logical Fallacies - Observed inference time (s)": "{\"description\": \"min=0.334, mean=0.334, max=0.334, sum=0.668 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3338228237409533\"}", - "Logical Fallacies - # eval": "{\"description\": \"min=163, mean=163, max=163, sum=326 (2)\", \"tab\": \"General information\", \"score\": \"163.0\"}", - "Logical Fallacies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Logical Fallacies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Logical Fallacies - # prompt tokens": "{\"description\": \"min=486.779, mean=486.779, max=486.779, sum=973.558 (2)\", \"tab\": \"General information\", \"score\": \"486.77914110429447\"}", - "Logical Fallacies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"logical_fallacies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_logical_fallacies\"" - } - } - }, - { - "evaluation_name": "Machine Learning", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Machine Learning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.509, - "details": { - "description": "min=0.509, mean=0.509, max=0.509, sum=1.018 (2)", - "tab": "Accuracy", - "Machine Learning - Observed inference time (s)": "{\"description\": \"min=0.349, mean=0.349, max=0.349, sum=0.697 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.34853318120752064\"}", - "Machine Learning - # eval": "{\"description\": \"min=112, mean=112, max=112, sum=224 (2)\", \"tab\": \"General information\", \"score\": \"112.0\"}", - "Machine Learning - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Machine Learning - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Machine Learning - # prompt tokens": "{\"description\": \"min=734.83, mean=734.83, max=734.83, sum=1469.661 (2)\", \"tab\": \"General information\", \"score\": \"734.8303571428571\"}", - "Machine Learning - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"machine_learning\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_machine_learning\"" - } - } - }, - { - "evaluation_name": "Management", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Management", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.845, - "details": { - "description": "min=0.845, mean=0.845, max=0.845, sum=1.689 (2)", - "tab": "Accuracy", - "Management - Observed inference time (s)": "{\"description\": \"min=0.325, mean=0.325, max=0.325, sum=0.651 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.32549439124690677\"}", - "Management - # eval": "{\"description\": \"min=103, mean=103, max=103, sum=206 (2)\", \"tab\": \"General information\", \"score\": \"103.0\"}", - "Management - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Management - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Management - # prompt tokens": "{\"description\": \"min=315.359, mean=315.359, max=315.359, sum=630.718 (2)\", \"tab\": \"General information\", \"score\": \"315.3592233009709\"}", - "Management - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"management\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_management\"" - } - } - }, - { - "evaluation_name": "Marketing", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Marketing", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.923, - "details": { - "description": "min=0.923, mean=0.923, max=0.923, sum=1.846 (2)", - "tab": "Accuracy", - "Marketing - Observed inference time (s)": "{\"description\": \"min=0.316, mean=0.316, max=0.316, sum=0.631 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.315602661198021\"}", - "Marketing - # eval": "{\"description\": \"min=234, mean=234, max=234, sum=468 (2)\", \"tab\": \"General information\", \"score\": \"234.0\"}", - "Marketing - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Marketing - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Marketing - # prompt tokens": "{\"description\": \"min=463.423, mean=463.423, max=463.423, sum=926.846 (2)\", \"tab\": \"General information\", \"score\": \"463.4230769230769\"}", - "Marketing - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"marketing\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_marketing\"" - } - } - }, - { - "evaluation_name": "Medical Genetics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Medical Genetics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.76, - "details": { - "description": "min=0.76, mean=0.76, max=0.76, sum=1.52 (2)", - "tab": "Accuracy", - "Medical Genetics - Observed inference time (s)": "{\"description\": \"min=0.316, mean=0.316, max=0.316, sum=0.632 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3161799097061157\"}", - "Medical Genetics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Medical Genetics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Medical Genetics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Medical Genetics - # prompt tokens": "{\"description\": \"min=405.71, mean=405.71, max=405.71, sum=811.42 (2)\", \"tab\": \"General information\", \"score\": \"405.71\"}", - "Medical Genetics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"medical_genetics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_medical_genetics\"" - } - } - }, - { - "evaluation_name": "Miscellaneous", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Miscellaneous", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.881, - "details": { - "description": "min=0.881, mean=0.881, max=0.881, sum=1.762 (2)", - "tab": "Accuracy", - "Miscellaneous - Observed inference time (s)": "{\"description\": \"min=0.323, mean=0.323, max=0.323, sum=0.645 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.32256904598396857\"}", - "Miscellaneous - # eval": "{\"description\": \"min=783, mean=783, max=783, sum=1566 (2)\", \"tab\": \"General information\", \"score\": \"783.0\"}", - "Miscellaneous - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Miscellaneous - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Miscellaneous - # prompt tokens": "{\"description\": \"min=348.519, mean=348.519, max=348.519, sum=697.037 (2)\", \"tab\": \"General information\", \"score\": \"348.51851851851853\"}", - "Miscellaneous - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"miscellaneous\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_miscellaneous\"" - } - } - }, - { - "evaluation_name": "Moral Scenarios", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Moral Scenarios", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.444, - "details": { - "description": "min=0.444, mean=0.444, max=0.444, sum=0.887 (2)", - "tab": "Accuracy", - "Moral Disputes - Observed inference time (s)": "{\"description\": \"min=0.304, mean=0.304, max=0.304, sum=0.607 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3035011126126857\"}", - "Moral Scenarios - Observed inference time (s)": "{\"description\": \"min=0.345, mean=0.345, max=0.345, sum=0.69 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.34521307439111465\"}", - "Moral Disputes - # eval": "{\"description\": \"min=346, mean=346, max=346, sum=692 (2)\", \"tab\": \"General information\", \"score\": \"346.0\"}", - "Moral Disputes - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Disputes - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Disputes - # prompt tokens": "{\"description\": \"min=540.038, mean=540.038, max=540.038, sum=1080.075 (2)\", \"tab\": \"General information\", \"score\": \"540.0375722543353\"}", - "Moral Disputes - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Moral Scenarios - # eval": "{\"description\": \"min=895, mean=895, max=895, sum=1790 (2)\", \"tab\": \"General information\", \"score\": \"895.0\"}", - "Moral Scenarios - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Scenarios - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # prompt tokens": "{\"description\": \"min=745.516, mean=745.516, max=745.516, sum=1491.032 (2)\", \"tab\": \"General information\", \"score\": \"745.5162011173185\"}", - "Moral Scenarios - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"moral_scenarios\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_moral_scenarios\"" - } - } - }, - { - "evaluation_name": "Nutrition", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Nutrition", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.83, - "details": { - "description": "min=0.83, mean=0.83, max=0.83, sum=1.66 (2)", - "tab": "Accuracy", - "Nutrition - Observed inference time (s)": "{\"description\": \"min=0.353, mean=0.353, max=0.353, sum=0.706 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3528824195363163\"}", - "Nutrition - # eval": "{\"description\": \"min=306, mean=306, max=306, sum=612 (2)\", \"tab\": \"General information\", \"score\": \"306.0\"}", - "Nutrition - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Nutrition - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Nutrition - # prompt tokens": "{\"description\": \"min=680.69, mean=680.69, max=680.69, sum=1361.379 (2)\", \"tab\": \"General information\", \"score\": \"680.6895424836601\"}", - "Nutrition - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"nutrition\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_nutrition\"" - } - } - }, - { - "evaluation_name": "Prehistory", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Prehistory", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.849, - "details": { - "description": "min=0.849, mean=0.849, max=0.849, sum=1.698 (2)", - "tab": "Accuracy", - "Prehistory - Observed inference time (s)": "{\"description\": \"min=0.33, mean=0.33, max=0.33, sum=0.66 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.32980361028953836\"}", - "Prehistory - # eval": "{\"description\": \"min=324, mean=324, max=324, sum=648 (2)\", \"tab\": \"General information\", \"score\": \"324.0\"}", - "Prehistory - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Prehistory - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Prehistory - # prompt tokens": "{\"description\": \"min=602.145, mean=602.145, max=602.145, sum=1204.29 (2)\", \"tab\": \"General information\", \"score\": \"602.145061728395\"}", - "Prehistory - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"prehistory\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_prehistory\"" - } - } - }, - { - "evaluation_name": "Public Relations", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Public Relations", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.682, - "details": { - "description": "min=0.682, mean=0.682, max=0.682, sum=1.364 (2)", - "tab": "Accuracy", - "Public Relations - Observed inference time (s)": "{\"description\": \"min=0.321, mean=0.321, max=0.321, sum=0.643 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.32145483710549094\"}", - "Public Relations - # eval": "{\"description\": \"min=110, mean=110, max=110, sum=220 (2)\", \"tab\": \"General information\", \"score\": \"110.0\"}", - "Public Relations - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Public Relations - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Public Relations - # prompt tokens": "{\"description\": \"min=462.036, mean=462.036, max=462.036, sum=924.073 (2)\", \"tab\": \"General information\", \"score\": \"462.03636363636366\"}", - "Public Relations - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"public_relations\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_public_relations\"" - } - } - }, - { - "evaluation_name": "Security Studies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Security Studies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.792, - "details": { - "description": "min=0.792, mean=0.792, max=0.792, sum=1.584 (2)", - "tab": "Accuracy", - "Security Studies - Observed inference time (s)": "{\"description\": \"min=0.391, mean=0.391, max=0.391, sum=0.783 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3913051323014863\"}", - "Security Studies - # eval": "{\"description\": \"min=245, mean=245, max=245, sum=490 (2)\", \"tab\": \"General information\", \"score\": \"245.0\"}", - "Security Studies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Security Studies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Security Studies - # prompt tokens": "{\"description\": \"min=1315.865, mean=1315.865, max=1315.865, sum=2631.731 (2)\", \"tab\": \"General information\", \"score\": \"1315.865306122449\"}", - "Security Studies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"security_studies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_security_studies\"" - } - } - }, - { - "evaluation_name": "Sociology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Sociology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.871, - "details": { - "description": "min=0.871, mean=0.871, max=0.871, sum=1.741 (2)", - "tab": "Accuracy", - "Sociology - Observed inference time (s)": "{\"description\": \"min=0.326, mean=0.326, max=0.326, sum=0.652 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.326159788008353\"}", - "Sociology - # eval": "{\"description\": \"min=201, mean=201, max=201, sum=402 (2)\", \"tab\": \"General information\", \"score\": \"201.0\"}", - "Sociology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Sociology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Sociology - # prompt tokens": "{\"description\": \"min=487.95, mean=487.95, max=487.95, sum=975.9 (2)\", \"tab\": \"General information\", \"score\": \"487.9502487562189\"}", - "Sociology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"sociology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_sociology\"" - } - } - }, - { - "evaluation_name": "Virology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Virology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.506, - "details": { - "description": "min=0.506, mean=0.506, max=0.506, sum=1.012 (2)", - "tab": "Accuracy", - "Virology - Observed inference time (s)": "{\"description\": \"min=0.343, mean=0.343, max=0.343, sum=0.686 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.34297854210956985\"}", - "Virology - # eval": "{\"description\": \"min=166, mean=166, max=166, sum=332 (2)\", \"tab\": \"General information\", \"score\": \"166.0\"}", - "Virology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Virology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Virology - # prompt tokens": "{\"description\": \"min=395.349, mean=395.349, max=395.349, sum=790.699 (2)\", \"tab\": \"General information\", \"score\": \"395.34939759036143\"}", - "Virology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"virology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_virology\"" - } - } - }, - { - "evaluation_name": "World Religions", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on World Religions", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.871, - "details": { - "description": "min=0.871, mean=0.871, max=0.871, sum=1.743 (2)", - "tab": "Accuracy", - "World Religions - Observed inference time (s)": "{\"description\": \"min=0.317, mean=0.317, max=0.317, sum=0.633 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3165940499445151\"}", - "World Religions - # eval": "{\"description\": \"min=171, mean=171, max=171, sum=342 (2)\", \"tab\": \"General information\", \"score\": \"171.0\"}", - "World Religions - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "World Religions - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "World Religions - # prompt tokens": "{\"description\": \"min=308.924, mean=308.924, max=308.924, sum=617.848 (2)\", \"tab\": \"General information\", \"score\": \"308.92397660818716\"}", - "World Religions - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"world_religions\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_world_religions\"" - } - } - }, - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.689, - "details": { - "description": "", - "tab": "Efficiency" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/mistralai_mixtral-8x7b-instruct-v0.1.json b/data/models/mistralai_mixtral-8x7b-instruct-v0.1.json deleted file mode 100644 index ba70c758fb9ba85b1e855fb4927ac58db77df090..0000000000000000000000000000000000000000 --- a/data/models/mistralai_mixtral-8x7b-instruct-v0.1.json +++ /dev/null @@ -1,501 +0,0 @@ -{ - "model_info": { - "name": "Mixtral-8x7B-Instruct-v0.1", - "id": "mistralai/Mixtral-8x7B-Instruct-v0.1", - "developer": "mistralai", - "inference_platform": "unknown", - "additional_details": { - "model_id_aliases": [ - "mistralai/mixtral-8x7b-instruct-v0.1" - ] - } - }, - "evaluations": [ - { - "evaluation_id": "helm_capabilities/mistralai_mixtral-8x7b-instruct-v0.1/1774096304.5056698", - "retrieved_timestamp": "1774096304.5056698", - "source_metadata": { - "source_name": "helm_capabilities", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_capabilities", - "evaluation_results": [ - { - "evaluation_name": "Mean score", - "source_data": { - "dataset_name": "helm_capabilities", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "The mean of the scores from all columns.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.397, - "details": { - "description": "", - "tab": "Accuracy", - "Mean score - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"3.8521851769069984\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU-Pro", - "source_data": { - "dataset_name": "MMLU-Pro", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on MMLU-Pro", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.335, - "details": { - "description": "min=0.335, mean=0.335, max=0.335, sum=0.335 (1)", - "tab": "Accuracy", - "MMLU-Pro - Observed inference time (s)": "{\"description\": \"min=2.842, mean=2.842, max=2.842, sum=2.842 (1)\", \"tab\": \"Efficiency\", \"score\": \"2.841812901973724\"}", - "MMLU-Pro - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "MMLU-Pro - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - # prompt tokens": "{\"description\": \"min=260.915, mean=260.915, max=260.915, sum=260.915 (1)\", \"tab\": \"General information\", \"score\": \"260.915\"}", - "MMLU-Pro - # output tokens": "{\"description\": \"min=274.355, mean=274.355, max=274.355, sum=274.355 (1)\", \"tab\": \"General information\", \"score\": \"274.355\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"all\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.296, - "details": { - "description": "min=0.296, mean=0.296, max=0.296, sum=0.296 (1)", - "tab": "Accuracy", - "GPQA - Observed inference time (s)": "{\"description\": \"min=3.163, mean=3.163, max=3.163, sum=3.163 (1)\", \"tab\": \"Efficiency\", \"score\": \"3.1633052681593616\"}", - "GPQA - # eval": "{\"description\": \"min=446, mean=446, max=446, sum=446 (1)\", \"tab\": \"General information\", \"score\": \"446.0\"}", - "GPQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - # prompt tokens": "{\"description\": \"min=281.998, mean=281.998, max=281.998, sum=281.998 (1)\", \"tab\": \"General information\", \"score\": \"281.99775784753365\"}", - "GPQA - # output tokens": "{\"description\": \"min=384.17, mean=384.17, max=384.17, sum=384.17 (1)\", \"tab\": \"General information\", \"score\": \"384.17040358744396\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"gpqa_main\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "IFEval Strict Acc on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.575, - "details": { - "description": "min=0.575, mean=0.575, max=0.575, sum=0.575 (1)", - "tab": "Accuracy", - "IFEval - Observed inference time (s)": "{\"description\": \"min=3.247, mean=3.247, max=3.247, sum=3.247 (1)\", \"tab\": \"Efficiency\", \"score\": \"3.2468207733027374\"}", - "IFEval - # eval": "{\"description\": \"min=541, mean=541, max=541, sum=541 (1)\", \"tab\": \"General information\", \"score\": \"541.0\"}", - "IFEval - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - # prompt tokens": "{\"description\": \"min=51.309, mean=51.309, max=51.309, sum=51.309 (1)\", \"tab\": \"General information\", \"score\": \"51.3086876155268\"}", - "IFEval - # output tokens": "{\"description\": \"min=377.81, mean=377.81, max=377.81, sum=377.81 (1)\", \"tab\": \"General information\", \"score\": \"377.8096118299446\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WildBench", - "source_data": { - "dataset_name": "WildBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "WB Score on WildBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.673, - "details": { - "description": "min=0.673, mean=0.673, max=0.673, sum=0.673 (1)", - "tab": "Accuracy", - "WildBench - Observed inference time (s)": "{\"description\": \"min=5.582, mean=5.582, max=5.582, sum=5.582 (1)\", \"tab\": \"Efficiency\", \"score\": \"5.581539319515228\"}", - "WildBench - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "WildBench - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # prompt tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # output tokens": "{\"description\": \"min=669.436, mean=669.436, max=669.436, sum=669.436 (1)\", \"tab\": \"General information\", \"score\": \"669.436\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"v2\"" - } - } - }, - { - "evaluation_name": "Omni-MATH", - "source_data": { - "dataset_name": "Omni-MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Acc on Omni-MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.105, - "details": { - "description": "min=0.105, mean=0.105, max=0.105, sum=0.105 (1)", - "tab": "Accuracy", - "Omni-MATH - Observed inference time (s)": "{\"description\": \"min=4.427, mean=4.427, max=4.427, sum=4.427 (1)\", \"tab\": \"Efficiency\", \"score\": \"4.427447621583939\"}", - "Omni-MATH - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "Omni-MATH - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - # prompt tokens": "{\"description\": \"min=119.373, mean=119.373, max=119.373, sum=119.373 (1)\", \"tab\": \"General information\", \"score\": \"119.373\"}", - "Omni-MATH - # output tokens": "{\"description\": \"min=550.807, mean=550.807, max=550.807, sum=550.807 (1)\", \"tab\": \"General information\", \"score\": \"550.807\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "hfopenllm_v2/mistralai_Mixtral-8x7B-Instruct-v0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5599 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4962 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0914 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3029 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4203 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3692 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "reward-bench/mistralai_Mixtral-8x7B-Instruct-v0.1/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7455 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9497 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6404 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7257 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7872 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5033 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/mistralai_mixtral-8x7b-v0.1.json b/data/models/mistralai_mixtral-8x7b-v0.1.json deleted file mode 100644 index 9d997e3527157e47894ae0f49b424a1634297e78..0000000000000000000000000000000000000000 --- a/data/models/mistralai_mixtral-8x7b-v0.1.json +++ /dev/null @@ -1,275 +0,0 @@ -{ - "model_info": { - "name": "Mixtral-8x7B-v0.1", - "id": "mistralai/Mixtral-8x7B-v0.1", - "developer": "mistralai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MixtralForCausalLM", - "params_billions": "46.703" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/mistralai_Mixtral-8x7B-v0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2415 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5087 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.102 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3138 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4321 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.385 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "hfopenllm_v2/mistralai_Mixtral-8x7B-v0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2326 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5098 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0937 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3205 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4413 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3871 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/mistralai_open-mistral-nemo-2407-fc.json b/data/models/mistralai_open-mistral-nemo-2407-fc.json deleted file mode 100644 index 943646d09f8bd73211341c488814364ff49e7101..0000000000000000000000000000000000000000 --- a/data/models/mistralai_open-mistral-nemo-2407-fc.json +++ /dev/null @@ -1,851 +0,0 @@ -{ - "model_info": { - "name": "Open-Mistral-Nemo-2407 (FC)", - "id": "mistralai/open-mistral-nemo-2407-fc", - "developer": "mistralai", - "additional_details": { - "raw_model_name": "Open-Mistral-Nemo-2407 (FC)", - "organization": "Mistral AI", - "license": "Proprietary", - "mode": "FC", - "model_link": "https://mistral.ai/news/mistral-nemo/" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/mistralai/open-mistral-nemo-2407-fc/1775236112.406723", - "retrieved_timestamp": "1775236112.406723", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 78.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 27.63 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 8.12 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 1.07 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 11.93 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 1.39 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 82.81 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 65.75 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 92.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 88.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 84.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 73.8 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 78.68 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 72.84 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 68.75 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 66.67 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 7.75 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 12.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 6.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 7.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 4.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 7.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 9.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 5.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 10.32 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 8.39 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 9.68 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 12.9 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 81.25 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 61.77 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/mistralai_open-mistral-nemo-2407-prompt.json b/data/models/mistralai_open-mistral-nemo-2407-prompt.json deleted file mode 100644 index 5b972099c6877c8b09f97a2480b364a033ce27da..0000000000000000000000000000000000000000 --- a/data/models/mistralai_open-mistral-nemo-2407-prompt.json +++ /dev/null @@ -1,905 +0,0 @@ -{ - "model_info": { - "name": "Open-Mistral-Nemo-2407 (Prompt)", - "id": "mistralai/open-mistral-nemo-2407-prompt", - "developer": "mistralai", - "additional_details": { - "raw_model_name": "Open-Mistral-Nemo-2407 (Prompt)", - "organization": "Mistral AI", - "license": "Proprietary", - "mode": "Prompt", - "model_link": "https://mistral.ai/news/mistral-nemo/" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/mistralai/open-mistral-nemo-2407-prompt/1775236112.41963", - "retrieved_timestamp": "1775236112.41963", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 102.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 19.31 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 13.8 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 0.84 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 7.05 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 1.32 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 88.46 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 79.33 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 92.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 90.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 91.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 73.95 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 78.29 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 73.03 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 87.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 58.33 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 0.75 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 0.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 1.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 1.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 2.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 3.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 2.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 8.6 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 9.68 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 9.68 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 6.45 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 93.75 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 6.28 - } - }, - { - "evaluation_result_id": "bfcl.format_sensitivity.max_delta::max_delta", - "evaluation_name": "bfcl.format_sensitivity.max_delta", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.format_sensitivity.max_delta", - "metric_name": "Format sensitivity max delta", - "metric_kind": "difference", - "metric_unit": "percentage_points", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Format Sensitivity Max Delta" - } - }, - "score_details": { - "score": 14.5 - } - }, - { - "evaluation_result_id": "bfcl.format_sensitivity.stddev::stddev", - "evaluation_name": "bfcl.format_sensitivity.stddev", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.format_sensitivity.stddev", - "metric_name": "Format sensitivity standard deviation", - "metric_kind": "difference", - "metric_unit": "percentage_points", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Format Sensitivity Standard Deviation" - } - }, - "score_details": { - "score": 4.6 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/mistralai_open-mistral-nemo-2407.json b/data/models/mistralai_open-mistral-nemo-2407.json deleted file mode 100644 index 365316574ad1e2eaf48ed0c75f30f6c71ca7b39a..0000000000000000000000000000000000000000 --- a/data/models/mistralai_open-mistral-nemo-2407.json +++ /dev/null @@ -1,1903 +0,0 @@ -{ - "model_info": { - "name": "Mistral NeMo 2402", - "id": "mistralai/open-mistral-nemo-2407", - "developer": "mistralai", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_lite/mistralai_open-mistral-nemo-2407/1774096306.427425", - "retrieved_timestamp": "1774096306.427425", - "source_metadata": { - "source_name": "helm_lite", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_lite", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_lite", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.333, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.5309862671660425\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.731, - "details": { - "description": "min=0.731, mean=0.731, max=0.731, sum=0.731 (1)", - "tab": "Accuracy", - "NarrativeQA - Observed inference time (s)": "{\"description\": \"min=0.711, mean=0.711, max=0.711, sum=0.711 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.7111437549053783\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=3527.392, mean=3527.392, max=3527.392, sum=3527.392 (1)\", \"tab\": \"General information\", \"score\": \"3527.3915492957744\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=6.901, mean=6.901, max=6.901, sum=6.901 (1)\", \"tab\": \"General information\", \"score\": \"6.901408450704225\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (closed-book)", - "source_data": { - "dataset_name": "NaturalQuestions (closed-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (closed-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.265, - "details": { - "description": "min=0.265, mean=0.265, max=0.265, sum=0.265 (1)", - "tab": "Accuracy", - "NaturalQuestions (open-book) - Observed inference time (s)": "{\"description\": \"min=0.852, mean=0.852, max=0.852, sum=0.852 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.851971923828125\"}", - "NaturalQuestions (closed-book) - Observed inference time (s)": "{\"description\": \"min=0.877, mean=0.877, max=0.877, sum=0.877 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.8765462198257447\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=2032.962, mean=2032.962, max=2032.962, sum=2032.962 (1)\", \"tab\": \"General information\", \"score\": \"2032.962\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=5.927, mean=5.927, max=5.927, sum=5.927 (1)\", \"tab\": \"General information\", \"score\": \"5.927\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=137.405, mean=137.405, max=137.405, sum=137.405 (1)\", \"tab\": \"General information\", \"score\": \"137.405\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=3.595, mean=3.595, max=3.595, sum=3.595 (1)\", \"tab\": \"General information\", \"score\": \"3.595\"}" - } - }, - "generation_config": { - "additional_details": { - "mode": "\"closedbook\"" - } - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.822, - "details": { - "description": "min=0.822, mean=0.822, max=0.822, sum=0.822 (1)", - "tab": "Accuracy", - "OpenbookQA - Observed inference time (s)": "{\"description\": \"min=0.799, mean=0.799, max=0.799, sum=0.799 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.7987758111953736\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=248.246, mean=248.246, max=248.246, sum=248.246 (1)\", \"tab\": \"General information\", \"score\": \"248.246\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "dataset": "\"openbookqa\"", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.604, - "details": { - "description": "min=0.29, mean=0.604, max=0.89, sum=3.021 (5)", - "tab": "Accuracy", - "MMLU - Observed inference time (s)": "{\"description\": \"min=0.635, mean=0.782, max=1.011, sum=3.908 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.7815720957371226\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=377.89, mean=479.924, max=631.851, sum=2399.621 (5)\", \"tab\": \"General information\", \"score\": \"479.9241754385965\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"college_chemistry\", \"computer_security\", \"econometrics\", \"us_foreign_policy\"]", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MATH", - "source_data": { - "dataset_name": "MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Equivalent (CoT) on MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.668, - "details": { - "description": "min=0.558, mean=0.668, max=0.852, sum=4.679 (7)", - "tab": "Accuracy", - "MATH - Observed inference time (s)": "{\"description\": \"min=0.866, mean=1.013, max=1.281, sum=7.093 (7)\", \"tab\": \"Efficiency\", \"score\": \"1.0132869822173503\"}", - "MATH - # eval": "{\"description\": \"min=30, mean=62.429, max=135, sum=437 (7)\", \"tab\": \"General information\", \"score\": \"62.42857142857143\"}", - "MATH - # train": "{\"description\": \"min=8, mean=8, max=8, sum=56 (7)\", \"tab\": \"General information\", \"score\": \"8.0\"}", - "MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MATH - # prompt tokens": "{\"description\": \"min=915.846, mean=1317.28, max=2238.885, sum=9220.959 (7)\", \"tab\": \"General information\", \"score\": \"1317.2798769434019\"}", - "MATH - # output tokens": "{\"description\": \"min=97.456, mean=111.745, max=141.433, sum=782.217 (7)\", \"tab\": \"General information\", \"score\": \"111.74533800213115\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"algebra\", \"counting_and_probability\", \"geometry\", \"intermediate_algebra\", \"number_theory\", \"prealgebra\", \"precalculus\"]", - "level": "\"1\"", - "use_official_examples": "\"False\"", - "use_chain_of_thought": "\"True\"" - } - } - }, - { - "evaluation_name": "GSM8K", - "source_data": { - "dataset_name": "GSM8K", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on GSM8K", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.782, - "details": { - "description": "min=0.782, mean=0.782, max=0.782, sum=0.782 (1)", - "tab": "Accuracy", - "GSM8K - Observed inference time (s)": "{\"description\": \"min=1.425, mean=1.425, max=1.425, sum=1.425 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.4254731934070588\"}", - "GSM8K - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "GSM8K - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "GSM8K - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GSM8K - # prompt tokens": "{\"description\": \"min=1134.356, mean=1134.356, max=1134.356, sum=1134.356 (1)\", \"tab\": \"General information\", \"score\": \"1134.356\"}", - "GSM8K - # output tokens": "{\"description\": \"min=187.859, mean=187.859, max=187.859, sum=187.859 (1)\", \"tab\": \"General information\", \"score\": \"187.859\"}" - } - }, - "generation_config": { - "additional_details": { - "stop": "\"none\"" - } - } - }, - { - "evaluation_name": "LegalBench", - "source_data": { - "dataset_name": "LegalBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on LegalBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.415, - "details": { - "description": "min=0.232, mean=0.415, max=0.758, sum=2.076 (5)", - "tab": "Accuracy", - "LegalBench - Observed inference time (s)": "{\"description\": \"min=0.715, mean=0.78, max=0.868, sum=3.898 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.7795765090728288\"}", - "LegalBench - # eval": "{\"description\": \"min=95, mean=409.4, max=1000, sum=2047 (5)\", \"tab\": \"General information\", \"score\": \"409.4\"}", - "LegalBench - # train": "{\"description\": \"min=4, mean=4.8, max=5, sum=24 (5)\", \"tab\": \"General information\", \"score\": \"4.8\"}", - "LegalBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "LegalBench - # prompt tokens": "{\"description\": \"min=200.716, mean=1561.36, max=6486.116, sum=7806.8 (5)\", \"tab\": \"General information\", \"score\": \"1561.3600575619662\"}", - "LegalBench - # output tokens": "{\"description\": \"min=4.94, mean=8.473, max=15.796, sum=42.365 (5)\", \"tab\": \"General information\", \"score\": \"8.473099835809844\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"abercrombie\", \"corporate_lobbying\", \"function_of_decision_section\", \"international_citizenship_questions\", \"proa\"]" - } - } - }, - { - "evaluation_name": "MedQA", - "source_data": { - "dataset_name": "MedQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MedQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.59, - "details": { - "description": "min=0.59, mean=0.59, max=0.59, sum=0.59 (1)", - "tab": "Accuracy", - "MedQA - Observed inference time (s)": "{\"description\": \"min=0.749, mean=0.749, max=0.749, sum=0.749 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.7488490715178533\"}", - "MedQA - # eval": "{\"description\": \"min=503, mean=503, max=503, sum=503 (1)\", \"tab\": \"General information\", \"score\": \"503.0\"}", - "MedQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MedQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MedQA - # prompt tokens": "{\"description\": \"min=1022.543, mean=1022.543, max=1022.543, sum=1022.543 (1)\", \"tab\": \"General information\", \"score\": \"1022.5427435387674\"}", - "MedQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WMT 2014", - "source_data": { - "dataset_name": "WMT 2014", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "BLEU-4 on WMT 2014", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.177, - "details": { - "description": "min=0.111, mean=0.177, max=0.211, sum=0.887 (5)", - "tab": "Accuracy", - "WMT 2014 - Observed inference time (s)": "{\"description\": \"min=0.752, mean=0.782, max=0.819, sum=3.911 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.7821908106898373\"}", - "WMT 2014 - # eval": "{\"description\": \"min=503, mean=568.8, max=832, sum=2844 (5)\", \"tab\": \"General information\", \"score\": \"568.8\"}", - "WMT 2014 - # train": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "WMT 2014 - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WMT 2014 - # prompt tokens": "{\"description\": \"min=81.661, mean=110.163, max=135.306, sum=550.814 (5)\", \"tab\": \"General information\", \"score\": \"110.16282784064842\"}", - "WMT 2014 - # output tokens": "{\"description\": \"min=24.622, mean=26.542, max=27.26, sum=132.709 (5)\", \"tab\": \"General information\", \"score\": \"26.541759538920324\"}" - } - }, - "generation_config": { - "additional_details": { - "language_pair": "[\"cs-en\", \"de-en\", \"fr-en\", \"hi-en\", \"ru-en\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_mmlu/mistralai_open-mistral-nemo-2407/1774096312.00548", - "retrieved_timestamp": "1774096312.00548", - "source_metadata": { - "source_name": "helm_mmlu", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_mmlu", - "evaluation_results": [ - { - "evaluation_name": "MMLU All Subjects", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU All Subjects", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.653, - "details": { - "description": "min=0.29, mean=0.653, max=0.912, sum=74.476 (114)", - "tab": "Accuracy", - "MMLU All Subjects - Observed inference time (s)": "{\"description\": \"min=0.57, mean=0.852, max=1.185, sum=97.097 (114)\", \"tab\": \"Efficiency\", \"score\": \"0.8517321572873682\"}", - "MMLU All Subjects - # eval": "{\"description\": \"min=100, mean=246.351, max=1534, sum=28084 (114)\", \"tab\": \"General information\", \"score\": \"246.35087719298247\"}", - "MMLU All Subjects - # train": "{\"description\": \"min=5, mean=5, max=5, sum=570 (114)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU All Subjects - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU All Subjects - # prompt tokens": "{\"description\": \"min=275.181, mean=627.375, max=2825.394, sum=71520.789 (114)\", \"tab\": \"General information\", \"score\": \"627.3753397392697\"}", - "MMLU All Subjects - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=114 (114)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_name": "Abstract Algebra", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Abstract Algebra", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.29, - "details": { - "description": "min=0.29, mean=0.29, max=0.29, sum=0.58 (2)", - "tab": "Accuracy", - "Abstract Algebra - Observed inference time (s)": "{\"description\": \"min=0.643, mean=0.643, max=0.643, sum=1.286 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6429726719856262\"}", - "Abstract Algebra - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Abstract Algebra - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Abstract Algebra - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Abstract Algebra - # prompt tokens": "{\"description\": \"min=377.89, mean=377.89, max=377.89, sum=755.78 (2)\", \"tab\": \"General information\", \"score\": \"377.89\"}", - "Abstract Algebra - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"abstract_algebra\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_abstract_algebra\"" - } - } - }, - { - "evaluation_name": "Anatomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Anatomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.607, - "details": { - "description": "min=0.607, mean=0.607, max=0.607, sum=1.215 (2)", - "tab": "Accuracy", - "Anatomy - Observed inference time (s)": "{\"description\": \"min=0.784, mean=0.784, max=0.784, sum=1.569 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7843294850102177\"}", - "Anatomy - # eval": "{\"description\": \"min=135, mean=135, max=135, sum=270 (2)\", \"tab\": \"General information\", \"score\": \"135.0\"}", - "Anatomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Anatomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Anatomy - # prompt tokens": "{\"description\": \"min=334.711, mean=334.711, max=334.711, sum=669.422 (2)\", \"tab\": \"General information\", \"score\": \"334.7111111111111\"}", - "Anatomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"anatomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_anatomy\"" - } - } - }, - { - "evaluation_name": "College Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on College Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.373, - "details": { - "description": "min=0.373, mean=0.373, max=0.373, sum=0.745 (2)", - "tab": "Accuracy", - "College Chemistry - Observed inference time (s)": "{\"description\": \"min=0.635, mean=0.635, max=0.635, sum=1.27 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6347627878189087\"}", - "College Biology - Observed inference time (s)": "{\"description\": \"min=0.743, mean=0.743, max=0.743, sum=1.487 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7433112810055414\"}", - "College Computer Science - Observed inference time (s)": "{\"description\": \"min=0.764, mean=0.764, max=0.764, sum=1.529 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7643197441101074\"}", - "College Mathematics - Observed inference time (s)": "{\"description\": \"min=0.807, mean=0.807, max=0.807, sum=1.614 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8069064331054687\"}", - "College Medicine - Observed inference time (s)": "{\"description\": \"min=0.913, mean=0.913, max=0.913, sum=1.825 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9125060442555157\"}", - "College Physics - Observed inference time (s)": "{\"description\": \"min=0.792, mean=0.792, max=0.792, sum=1.584 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7920899648292392\"}", - "College Chemistry - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Chemistry - # prompt tokens": "{\"description\": \"min=563.78, mean=563.78, max=563.78, sum=1127.56 (2)\", \"tab\": \"General information\", \"score\": \"563.78\"}", - "College Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Biology - # eval": "{\"description\": \"min=144, mean=144, max=144, sum=288 (2)\", \"tab\": \"General information\", \"score\": \"144.0\"}", - "College Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # prompt tokens": "{\"description\": \"min=471.931, mean=471.931, max=471.931, sum=943.861 (2)\", \"tab\": \"General information\", \"score\": \"471.93055555555554\"}", - "College Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # prompt tokens": "{\"description\": \"min=844.21, mean=844.21, max=844.21, sum=1688.42 (2)\", \"tab\": \"General information\", \"score\": \"844.21\"}", - "College Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Mathematics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # prompt tokens": "{\"description\": \"min=609.39, mean=609.39, max=609.39, sum=1218.78 (2)\", \"tab\": \"General information\", \"score\": \"609.39\"}", - "College Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Medicine - # eval": "{\"description\": \"min=173, mean=173, max=173, sum=346 (2)\", \"tab\": \"General information\", \"score\": \"173.0\"}", - "College Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # prompt tokens": "{\"description\": \"min=497.538, mean=497.538, max=497.538, sum=995.075 (2)\", \"tab\": \"General information\", \"score\": \"497.53757225433526\"}", - "College Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Physics - # eval": "{\"description\": \"min=102, mean=102, max=102, sum=204 (2)\", \"tab\": \"General information\", \"score\": \"102.0\"}", - "College Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # prompt tokens": "{\"description\": \"min=512.941, mean=512.941, max=512.941, sum=1025.882 (2)\", \"tab\": \"General information\", \"score\": \"512.9411764705883\"}", - "College Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"college_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_college_physics\"" - } - } - }, - { - "evaluation_name": "Computer Security", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Computer Security", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.81, - "details": { - "description": "min=0.81, mean=0.81, max=0.81, sum=1.62 (2)", - "tab": "Accuracy", - "Computer Security - Observed inference time (s)": "{\"description\": \"min=1.011, mean=1.011, max=1.011, sum=2.023 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.0114419960975647\"}", - "Computer Security - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Computer Security - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Computer Security - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Computer Security - # prompt tokens": "{\"description\": \"min=395.27, mean=395.27, max=395.27, sum=790.54 (2)\", \"tab\": \"General information\", \"score\": \"395.27\"}", - "Computer Security - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"computer_security\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_computer_security\"" - } - } - }, - { - "evaluation_name": "Econometrics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Econometrics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.561, - "details": { - "description": "min=0.561, mean=0.561, max=0.561, sum=1.123 (2)", - "tab": "Accuracy", - "Econometrics - Observed inference time (s)": "{\"description\": \"min=0.766, mean=0.766, max=0.766, sum=1.531 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7657254641516167\"}", - "Econometrics - # eval": "{\"description\": \"min=114, mean=114, max=114, sum=228 (2)\", \"tab\": \"General information\", \"score\": \"114.0\"}", - "Econometrics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Econometrics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Econometrics - # prompt tokens": "{\"description\": \"min=631.851, mean=631.851, max=631.851, sum=1263.702 (2)\", \"tab\": \"General information\", \"score\": \"631.8508771929825\"}", - "Econometrics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"econometrics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_econometrics\"" - } - } - }, - { - "evaluation_name": "Global Facts", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Global Facts", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4, - "details": { - "description": "min=0.4, mean=0.4, max=0.4, sum=0.8 (2)", - "tab": "Accuracy", - "Global Facts - Observed inference time (s)": "{\"description\": \"min=0.842, mean=0.842, max=0.842, sum=1.683 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8416926956176758\"}", - "Global Facts - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Global Facts - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Global Facts - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Global Facts - # prompt tokens": "{\"description\": \"min=433.39, mean=433.39, max=433.39, sum=866.78 (2)\", \"tab\": \"General information\", \"score\": \"433.39\"}", - "Global Facts - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"global_facts\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_global_facts\"" - } - } - }, - { - "evaluation_name": "Jurisprudence", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Jurisprudence", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.796, - "details": { - "description": "min=0.796, mean=0.796, max=0.796, sum=1.593 (2)", - "tab": "Accuracy", - "Jurisprudence - Observed inference time (s)": "{\"description\": \"min=0.837, mean=0.837, max=0.837, sum=1.674 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8370662177050555\"}", - "Jurisprudence - # eval": "{\"description\": \"min=108, mean=108, max=108, sum=216 (2)\", \"tab\": \"General information\", \"score\": \"108.0\"}", - "Jurisprudence - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Jurisprudence - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Jurisprudence - # prompt tokens": "{\"description\": \"min=399.019, mean=399.019, max=399.019, sum=798.037 (2)\", \"tab\": \"General information\", \"score\": \"399.01851851851853\"}", - "Jurisprudence - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"jurisprudence\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_jurisprudence\"" - } - } - }, - { - "evaluation_name": "Philosophy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Philosophy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.733, - "details": { - "description": "min=0.733, mean=0.733, max=0.733, sum=1.466 (2)", - "tab": "Accuracy", - "Philosophy - Observed inference time (s)": "{\"description\": \"min=0.877, mean=0.877, max=0.877, sum=1.755 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8774675686643054\"}", - "Philosophy - # eval": "{\"description\": \"min=311, mean=311, max=311, sum=622 (2)\", \"tab\": \"General information\", \"score\": \"311.0\"}", - "Philosophy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Philosophy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Philosophy - # prompt tokens": "{\"description\": \"min=331.354, mean=331.354, max=331.354, sum=662.707 (2)\", \"tab\": \"General information\", \"score\": \"331.35369774919616\"}", - "Philosophy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"philosophy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_philosophy\"" - } - } - }, - { - "evaluation_name": "Professional Psychology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Professional Psychology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.588, - "details": { - "description": "min=0.588, mean=0.588, max=0.588, sum=1.176 (2)", - "tab": "Accuracy", - "Professional Medicine - Observed inference time (s)": "{\"description\": \"min=0.731, mean=0.731, max=0.731, sum=1.462 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7308363747947356\"}", - "Professional Accounting - Observed inference time (s)": "{\"description\": \"min=0.825, mean=0.825, max=0.825, sum=1.649 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.824517419152226\"}", - "Professional Law - Observed inference time (s)": "{\"description\": \"min=0.812, mean=0.812, max=0.812, sum=1.625 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8123439646761917\"}", - "Professional Psychology - Observed inference time (s)": "{\"description\": \"min=0.757, mean=0.757, max=0.757, sum=1.515 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.757308129391639\"}", - "Professional Medicine - # eval": "{\"description\": \"min=272, mean=272, max=272, sum=544 (2)\", \"tab\": \"General information\", \"score\": \"272.0\"}", - "Professional Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Medicine - # prompt tokens": "{\"description\": \"min=1091.357, mean=1091.357, max=1091.357, sum=2182.713 (2)\", \"tab\": \"General information\", \"score\": \"1091.3566176470588\"}", - "Professional Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Accounting - # eval": "{\"description\": \"min=282, mean=282, max=282, sum=564 (2)\", \"tab\": \"General information\", \"score\": \"282.0\"}", - "Professional Accounting - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Accounting - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # prompt tokens": "{\"description\": \"min=749.039, mean=749.039, max=749.039, sum=1498.078 (2)\", \"tab\": \"General information\", \"score\": \"749.0390070921986\"}", - "Professional Accounting - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Law - # eval": "{\"description\": \"min=1534, mean=1534, max=1534, sum=3068 (2)\", \"tab\": \"General information\", \"score\": \"1534.0\"}", - "Professional Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # prompt tokens": "{\"description\": \"min=1710.472, mean=1710.472, max=1710.472, sum=3420.944 (2)\", \"tab\": \"General information\", \"score\": \"1710.4719687092568\"}", - "Professional Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Psychology - # eval": "{\"description\": \"min=612, mean=612, max=612, sum=1224 (2)\", \"tab\": \"General information\", \"score\": \"612.0\"}", - "Professional Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # prompt tokens": "{\"description\": \"min=584.748, mean=584.748, max=584.748, sum=1169.497 (2)\", \"tab\": \"General information\", \"score\": \"584.7483660130719\"}", - "Professional Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"professional_psychology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_professional_psychology\"" - } - } - }, - { - "evaluation_name": "Us Foreign Policy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Us Foreign Policy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.89, - "details": { - "description": "min=0.89, mean=0.89, max=0.89, sum=1.78 (2)", - "tab": "Accuracy", - "Us Foreign Policy - Observed inference time (s)": "{\"description\": \"min=0.853, mean=0.853, max=0.853, sum=1.706 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8529575586318969\"}", - "Us Foreign Policy - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Us Foreign Policy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Us Foreign Policy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Us Foreign Policy - # prompt tokens": "{\"description\": \"min=430.83, mean=430.83, max=430.83, sum=861.66 (2)\", \"tab\": \"General information\", \"score\": \"430.83\"}", - "Us Foreign Policy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"us_foreign_policy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_us_foreign_policy\"" - } - } - }, - { - "evaluation_name": "Astronomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Astronomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.691, - "details": { - "description": "min=0.691, mean=0.691, max=0.691, sum=1.382 (2)", - "tab": "Accuracy", - "Astronomy - Observed inference time (s)": "{\"description\": \"min=0.947, mean=0.947, max=0.947, sum=1.895 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9474252227105593\"}", - "Astronomy - # eval": "{\"description\": \"min=152, mean=152, max=152, sum=304 (2)\", \"tab\": \"General information\", \"score\": \"152.0\"}", - "Astronomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Astronomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Astronomy - # prompt tokens": "{\"description\": \"min=587.053, mean=587.053, max=587.053, sum=1174.105 (2)\", \"tab\": \"General information\", \"score\": \"587.0526315789474\"}", - "Astronomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"astronomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_astronomy\"" - } - } - }, - { - "evaluation_name": "Business Ethics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Business Ethics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.49, - "details": { - "description": "min=0.49, mean=0.49, max=0.49, sum=0.98 (2)", - "tab": "Accuracy", - "Business Ethics - Observed inference time (s)": "{\"description\": \"min=0.719, mean=0.719, max=0.719, sum=1.438 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7189487242698669\"}", - "Business Ethics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Business Ethics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Business Ethics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Business Ethics - # prompt tokens": "{\"description\": \"min=597.68, mean=597.68, max=597.68, sum=1195.36 (2)\", \"tab\": \"General information\", \"score\": \"597.68\"}", - "Business Ethics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"business_ethics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_business_ethics\"" - } - } - }, - { - "evaluation_name": "Clinical Knowledge", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Clinical Knowledge", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.736, - "details": { - "description": "min=0.736, mean=0.736, max=0.736, sum=1.472 (2)", - "tab": "Accuracy", - "Clinical Knowledge - Observed inference time (s)": "{\"description\": \"min=0.822, mean=0.822, max=0.822, sum=1.643 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8215559176678927\"}", - "Clinical Knowledge - # eval": "{\"description\": \"min=265, mean=265, max=265, sum=530 (2)\", \"tab\": \"General information\", \"score\": \"265.0\"}", - "Clinical Knowledge - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Clinical Knowledge - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Clinical Knowledge - # prompt tokens": "{\"description\": \"min=382.989, mean=382.989, max=382.989, sum=765.977 (2)\", \"tab\": \"General information\", \"score\": \"382.98867924528304\"}", - "Clinical Knowledge - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"clinical_knowledge\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_clinical_knowledge\"" - } - } - }, - { - "evaluation_name": "Conceptual Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Conceptual Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.647, - "details": { - "description": "min=0.647, mean=0.647, max=0.647, sum=1.294 (2)", - "tab": "Accuracy", - "Conceptual Physics - Observed inference time (s)": "{\"description\": \"min=0.788, mean=0.788, max=0.788, sum=1.576 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7878646302730479\"}", - "Conceptual Physics - # eval": "{\"description\": \"min=235, mean=235, max=235, sum=470 (2)\", \"tab\": \"General information\", \"score\": \"235.0\"}", - "Conceptual Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Conceptual Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Conceptual Physics - # prompt tokens": "{\"description\": \"min=301.336, mean=301.336, max=301.336, sum=602.672 (2)\", \"tab\": \"General information\", \"score\": \"301.336170212766\"}", - "Conceptual Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"conceptual_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_conceptual_physics\"" - } - } - }, - { - "evaluation_name": "Electrical Engineering", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Electrical Engineering", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.531, - "details": { - "description": "min=0.531, mean=0.531, max=0.531, sum=1.062 (2)", - "tab": "Accuracy", - "Electrical Engineering - Observed inference time (s)": "{\"description\": \"min=0.658, mean=0.658, max=0.658, sum=1.316 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6578493726664576\"}", - "Electrical Engineering - # eval": "{\"description\": \"min=145, mean=145, max=145, sum=290 (2)\", \"tab\": \"General information\", \"score\": \"145.0\"}", - "Electrical Engineering - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Electrical Engineering - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Electrical Engineering - # prompt tokens": "{\"description\": \"min=464.697, mean=464.697, max=464.697, sum=929.393 (2)\", \"tab\": \"General information\", \"score\": \"464.6965517241379\"}", - "Electrical Engineering - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"electrical_engineering\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_electrical_engineering\"" - } - } - }, - { - "evaluation_name": "Elementary Mathematics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Elementary Mathematics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.439, - "details": { - "description": "min=0.439, mean=0.439, max=0.439, sum=0.878 (2)", - "tab": "Accuracy", - "Elementary Mathematics - Observed inference time (s)": "{\"description\": \"min=0.712, mean=0.712, max=0.712, sum=1.423 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7115525694751235\"}", - "Elementary Mathematics - # eval": "{\"description\": \"min=378, mean=378, max=378, sum=756 (2)\", \"tab\": \"General information\", \"score\": \"378.0\"}", - "Elementary Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Elementary Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Elementary Mathematics - # prompt tokens": "{\"description\": \"min=580.741, mean=580.741, max=580.741, sum=1161.481 (2)\", \"tab\": \"General information\", \"score\": \"580.7407407407408\"}", - "Elementary Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"elementary_mathematics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_elementary_mathematics\"" - } - } - }, - { - "evaluation_name": "Formal Logic", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Formal Logic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.405, - "details": { - "description": "min=0.405, mean=0.405, max=0.405, sum=0.81 (2)", - "tab": "Accuracy", - "Formal Logic - Observed inference time (s)": "{\"description\": \"min=1.185, mean=1.185, max=1.185, sum=2.37 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.1852161146345592\"}", - "Formal Logic - # eval": "{\"description\": \"min=126, mean=126, max=126, sum=252 (2)\", \"tab\": \"General information\", \"score\": \"126.0\"}", - "Formal Logic - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Formal Logic - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Formal Logic - # prompt tokens": "{\"description\": \"min=619.929, mean=619.929, max=619.929, sum=1239.857 (2)\", \"tab\": \"General information\", \"score\": \"619.9285714285714\"}", - "Formal Logic - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"formal_logic\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_formal_logic\"" - } - } - }, - { - "evaluation_name": "High School World History", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on High School World History", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.848, - "details": { - "description": "min=0.848, mean=0.848, max=0.848, sum=1.696 (2)", - "tab": "Accuracy", - "High School Biology - Observed inference time (s)": "{\"description\": \"min=0.97, mean=0.97, max=0.97, sum=1.94 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9699527340550577\"}", - "High School Chemistry - Observed inference time (s)": "{\"description\": \"min=0.751, mean=0.751, max=0.751, sum=1.503 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.751325937327493\"}", - "High School Computer Science - Observed inference time (s)": "{\"description\": \"min=0.764, mean=0.764, max=0.764, sum=1.528 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7637556600570679\"}", - "High School European History - Observed inference time (s)": "{\"description\": \"min=0.796, mean=0.796, max=0.796, sum=1.592 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7959829893979159\"}", - "High School Geography - Observed inference time (s)": "{\"description\": \"min=0.686, mean=0.686, max=0.686, sum=1.373 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.686434592863526\"}", - "High School Government And Politics - Observed inference time (s)": "{\"description\": \"min=0.837, mean=0.837, max=0.837, sum=1.674 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8370978684005342\"}", - "High School Macroeconomics - Observed inference time (s)": "{\"description\": \"min=1.045, mean=1.045, max=1.045, sum=2.09 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.045194720610594\"}", - "High School Mathematics - Observed inference time (s)": "{\"description\": \"min=0.839, mean=0.839, max=0.839, sum=1.677 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8386335717307196\"}", - "High School Microeconomics - Observed inference time (s)": "{\"description\": \"min=0.901, mean=0.901, max=0.901, sum=1.802 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9010114108814913\"}", - "High School Physics - Observed inference time (s)": "{\"description\": \"min=0.93, mean=0.93, max=0.93, sum=1.86 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9301499767808725\"}", - "High School Psychology - Observed inference time (s)": "{\"description\": \"min=0.975, mean=0.975, max=0.975, sum=1.95 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9747656953444175\"}", - "High School Statistics - Observed inference time (s)": "{\"description\": \"min=0.798, mean=0.798, max=0.798, sum=1.595 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7976611223485734\"}", - "High School US History - Observed inference time (s)": "{\"description\": \"min=1.112, mean=1.112, max=1.112, sum=2.225 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.1124158618496913\"}", - "High School World History - Observed inference time (s)": "{\"description\": \"min=0.942, mean=0.942, max=0.942, sum=1.883 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9417288112237987\"}", - "High School Biology - # eval": "{\"description\": \"min=310, mean=310, max=310, sum=620 (2)\", \"tab\": \"General information\", \"score\": \"310.0\"}", - "High School Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Biology - # prompt tokens": "{\"description\": \"min=500.994, mean=500.994, max=500.994, sum=1001.987 (2)\", \"tab\": \"General information\", \"score\": \"500.9935483870968\"}", - "High School Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Chemistry - # eval": "{\"description\": \"min=203, mean=203, max=203, sum=406 (2)\", \"tab\": \"General information\", \"score\": \"203.0\"}", - "High School Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # prompt tokens": "{\"description\": \"min=507.995, mean=507.995, max=507.995, sum=1015.99 (2)\", \"tab\": \"General information\", \"score\": \"507.9950738916256\"}", - "High School Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "High School Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # prompt tokens": "{\"description\": \"min=888.78, mean=888.78, max=888.78, sum=1777.56 (2)\", \"tab\": \"General information\", \"score\": \"888.78\"}", - "High School Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School European History - # eval": "{\"description\": \"min=165, mean=165, max=165, sum=330 (2)\", \"tab\": \"General information\", \"score\": \"165.0\"}", - "High School European History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School European History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # prompt tokens": "{\"description\": \"min=2825.394, mean=2825.394, max=2825.394, sum=5650.788 (2)\", \"tab\": \"General information\", \"score\": \"2825.3939393939395\"}", - "High School European History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Geography - # eval": "{\"description\": \"min=198, mean=198, max=198, sum=396 (2)\", \"tab\": \"General information\", \"score\": \"198.0\"}", - "High School Geography - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Geography - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # prompt tokens": "{\"description\": \"min=372.207, mean=372.207, max=372.207, sum=744.414 (2)\", \"tab\": \"General information\", \"score\": \"372.2070707070707\"}", - "High School Geography - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Government And Politics - # eval": "{\"description\": \"min=193, mean=193, max=193, sum=386 (2)\", \"tab\": \"General information\", \"score\": \"193.0\"}", - "High School Government And Politics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Government And Politics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # prompt tokens": "{\"description\": \"min=471.202, mean=471.202, max=471.202, sum=942.404 (2)\", \"tab\": \"General information\", \"score\": \"471.2020725388601\"}", - "High School Government And Politics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Macroeconomics - # eval": "{\"description\": \"min=390, mean=390, max=390, sum=780 (2)\", \"tab\": \"General information\", \"score\": \"390.0\"}", - "High School Macroeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Macroeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # prompt tokens": "{\"description\": \"min=379.21, mean=379.21, max=379.21, sum=758.421 (2)\", \"tab\": \"General information\", \"score\": \"379.2102564102564\"}", - "High School Macroeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Mathematics - # eval": "{\"description\": \"min=270, mean=270, max=270, sum=540 (2)\", \"tab\": \"General information\", \"score\": \"270.0\"}", - "High School Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # prompt tokens": "{\"description\": \"min=555.807, mean=555.807, max=555.807, sum=1111.615 (2)\", \"tab\": \"General information\", \"score\": \"555.8074074074074\"}", - "High School Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Microeconomics - # eval": "{\"description\": \"min=238, mean=238, max=238, sum=476 (2)\", \"tab\": \"General information\", \"score\": \"238.0\"}", - "High School Microeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Microeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # prompt tokens": "{\"description\": \"min=405.95, mean=405.95, max=405.95, sum=811.899 (2)\", \"tab\": \"General information\", \"score\": \"405.9495798319328\"}", - "High School Microeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Physics - # eval": "{\"description\": \"min=151, mean=151, max=151, sum=302 (2)\", \"tab\": \"General information\", \"score\": \"151.0\"}", - "High School Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # prompt tokens": "{\"description\": \"min=584.272, mean=584.272, max=584.272, sum=1168.543 (2)\", \"tab\": \"General information\", \"score\": \"584.2715231788079\"}", - "High School Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Psychology - # eval": "{\"description\": \"min=545, mean=545, max=545, sum=1090 (2)\", \"tab\": \"General information\", \"score\": \"545.0\"}", - "High School Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # prompt tokens": "{\"description\": \"min=487.532, mean=487.532, max=487.532, sum=975.064 (2)\", \"tab\": \"General information\", \"score\": \"487.5321100917431\"}", - "High School Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Statistics - # eval": "{\"description\": \"min=216, mean=216, max=216, sum=432 (2)\", \"tab\": \"General information\", \"score\": \"216.0\"}", - "High School Statistics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Statistics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # prompt tokens": "{\"description\": \"min=831.926, mean=831.926, max=831.926, sum=1663.852 (2)\", \"tab\": \"General information\", \"score\": \"831.925925925926\"}", - "High School Statistics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School US History - # eval": "{\"description\": \"min=204, mean=204, max=204, sum=408 (2)\", \"tab\": \"General information\", \"score\": \"204.0\"}", - "High School US History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School US History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # prompt tokens": "{\"description\": \"min=2271.559, mean=2271.559, max=2271.559, sum=4543.118 (2)\", \"tab\": \"General information\", \"score\": \"2271.5588235294117\"}", - "High School US History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School World History - # eval": "{\"description\": \"min=237, mean=237, max=237, sum=474 (2)\", \"tab\": \"General information\", \"score\": \"237.0\"}", - "High School World History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School World History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # prompt tokens": "{\"description\": \"min=1458.937, mean=1458.937, max=1458.937, sum=2917.873 (2)\", \"tab\": \"General information\", \"score\": \"1458.9367088607594\"}", - "High School World History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"high_school_world_history\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_high_school_world_history\"" - } - } - }, - { - "evaluation_name": "Human Sexuality", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Human Sexuality", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.702, - "details": { - "description": "min=0.702, mean=0.702, max=0.702, sum=1.405 (2)", - "tab": "Accuracy", - "Human Aging - Observed inference time (s)": "{\"description\": \"min=0.851, mean=0.851, max=0.851, sum=1.703 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8512581602874892\"}", - "Human Sexuality - Observed inference time (s)": "{\"description\": \"min=0.57, mean=0.57, max=0.57, sum=1.139 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.569578381895109\"}", - "Human Aging - # eval": "{\"description\": \"min=223, mean=223, max=223, sum=446 (2)\", \"tab\": \"General information\", \"score\": \"223.0\"}", - "Human Aging - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Aging - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Aging - # prompt tokens": "{\"description\": \"min=320.296, mean=320.296, max=320.296, sum=640.592 (2)\", \"tab\": \"General information\", \"score\": \"320.29596412556054\"}", - "Human Aging - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Human Sexuality - # eval": "{\"description\": \"min=131, mean=131, max=131, sum=262 (2)\", \"tab\": \"General information\", \"score\": \"131.0\"}", - "Human Sexuality - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Sexuality - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # prompt tokens": "{\"description\": \"min=345.45, mean=345.45, max=345.45, sum=690.901 (2)\", \"tab\": \"General information\", \"score\": \"345.4503816793893\"}", - "Human Sexuality - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"human_sexuality\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_human_sexuality\"" - } - } - }, - { - "evaluation_name": "International Law", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on International Law", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.769, - "details": { - "description": "min=0.769, mean=0.769, max=0.769, sum=1.537 (2)", - "tab": "Accuracy", - "International Law - Observed inference time (s)": "{\"description\": \"min=0.779, mean=0.779, max=0.779, sum=1.558 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7790698473118554\"}", - "International Law - # eval": "{\"description\": \"min=121, mean=121, max=121, sum=242 (2)\", \"tab\": \"General information\", \"score\": \"121.0\"}", - "International Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "International Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "International Law - # prompt tokens": "{\"description\": \"min=649.017, mean=649.017, max=649.017, sum=1298.033 (2)\", \"tab\": \"General information\", \"score\": \"649.0165289256198\"}", - "International Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"international_law\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_international_law\"" - } - } - }, - { - "evaluation_name": "Logical Fallacies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Logical Fallacies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.791, - "details": { - "description": "min=0.791, mean=0.791, max=0.791, sum=1.583 (2)", - "tab": "Accuracy", - "Logical Fallacies - Observed inference time (s)": "{\"description\": \"min=1.077, mean=1.077, max=1.077, sum=2.154 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.0772201810146402\"}", - "Logical Fallacies - # eval": "{\"description\": \"min=163, mean=163, max=163, sum=326 (2)\", \"tab\": \"General information\", \"score\": \"163.0\"}", - "Logical Fallacies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Logical Fallacies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Logical Fallacies - # prompt tokens": "{\"description\": \"min=449.595, mean=449.595, max=449.595, sum=899.19 (2)\", \"tab\": \"General information\", \"score\": \"449.5950920245399\"}", - "Logical Fallacies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"logical_fallacies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_logical_fallacies\"" - } - } - }, - { - "evaluation_name": "Machine Learning", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Machine Learning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.402, - "details": { - "description": "min=0.402, mean=0.402, max=0.402, sum=0.804 (2)", - "tab": "Accuracy", - "Machine Learning - Observed inference time (s)": "{\"description\": \"min=1.123, mean=1.123, max=1.123, sum=2.246 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.1229032427072525\"}", - "Machine Learning - # eval": "{\"description\": \"min=112, mean=112, max=112, sum=224 (2)\", \"tab\": \"General information\", \"score\": \"112.0\"}", - "Machine Learning - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Machine Learning - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Machine Learning - # prompt tokens": "{\"description\": \"min=684.696, mean=684.696, max=684.696, sum=1369.393 (2)\", \"tab\": \"General information\", \"score\": \"684.6964285714286\"}", - "Machine Learning - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"machine_learning\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_machine_learning\"" - } - } - }, - { - "evaluation_name": "Management", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Management", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.796, - "details": { - "description": "min=0.796, mean=0.796, max=0.796, sum=1.592 (2)", - "tab": "Accuracy", - "Management - Observed inference time (s)": "{\"description\": \"min=0.786, mean=0.786, max=0.786, sum=1.571 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7855723436596325\"}", - "Management - # eval": "{\"description\": \"min=103, mean=103, max=103, sum=206 (2)\", \"tab\": \"General information\", \"score\": \"103.0\"}", - "Management - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Management - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Management - # prompt tokens": "{\"description\": \"min=286.272, mean=286.272, max=286.272, sum=572.544 (2)\", \"tab\": \"General information\", \"score\": \"286.2718446601942\"}", - "Management - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"management\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_management\"" - } - } - }, - { - "evaluation_name": "Marketing", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Marketing", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.889, - "details": { - "description": "min=0.889, mean=0.889, max=0.889, sum=1.778 (2)", - "tab": "Accuracy", - "Marketing - Observed inference time (s)": "{\"description\": \"min=0.859, mean=0.859, max=0.859, sum=1.719 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8593697160737127\"}", - "Marketing - # eval": "{\"description\": \"min=234, mean=234, max=234, sum=468 (2)\", \"tab\": \"General information\", \"score\": \"234.0\"}", - "Marketing - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Marketing - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Marketing - # prompt tokens": "{\"description\": \"min=408.308, mean=408.308, max=408.308, sum=816.615 (2)\", \"tab\": \"General information\", \"score\": \"408.3076923076923\"}", - "Marketing - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"marketing\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_marketing\"" - } - } - }, - { - "evaluation_name": "Medical Genetics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Medical Genetics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.78, - "details": { - "description": "min=0.78, mean=0.78, max=0.78, sum=1.56 (2)", - "tab": "Accuracy", - "Medical Genetics - Observed inference time (s)": "{\"description\": \"min=0.829, mean=0.829, max=0.829, sum=1.658 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8288634467124939\"}", - "Medical Genetics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Medical Genetics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Medical Genetics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Medical Genetics - # prompt tokens": "{\"description\": \"min=332.56, mean=332.56, max=332.56, sum=665.12 (2)\", \"tab\": \"General information\", \"score\": \"332.56\"}", - "Medical Genetics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"medical_genetics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_medical_genetics\"" - } - } - }, - { - "evaluation_name": "Miscellaneous", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Miscellaneous", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.861, - "details": { - "description": "min=0.861, mean=0.861, max=0.861, sum=1.722 (2)", - "tab": "Accuracy", - "Miscellaneous - Observed inference time (s)": "{\"description\": \"min=0.849, mean=0.849, max=0.849, sum=1.698 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8490832494440967\"}", - "Miscellaneous - # eval": "{\"description\": \"min=783, mean=783, max=783, sum=1566 (2)\", \"tab\": \"General information\", \"score\": \"783.0\"}", - "Miscellaneous - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Miscellaneous - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Miscellaneous - # prompt tokens": "{\"description\": \"min=307.041, mean=307.041, max=307.041, sum=614.082 (2)\", \"tab\": \"General information\", \"score\": \"307.04086845466156\"}", - "Miscellaneous - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"miscellaneous\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_miscellaneous\"" - } - } - }, - { - "evaluation_name": "Moral Scenarios", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Moral Scenarios", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.381, - "details": { - "description": "min=0.381, mean=0.381, max=0.381, sum=0.762 (2)", - "tab": "Accuracy", - "Moral Disputes - Observed inference time (s)": "{\"description\": \"min=0.808, mean=0.808, max=0.808, sum=1.615 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8076560903835848\"}", - "Moral Scenarios - Observed inference time (s)": "{\"description\": \"min=0.868, mean=0.868, max=0.868, sum=1.735 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8676496551023515\"}", - "Moral Disputes - # eval": "{\"description\": \"min=346, mean=346, max=346, sum=692 (2)\", \"tab\": \"General information\", \"score\": \"346.0\"}", - "Moral Disputes - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Disputes - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Disputes - # prompt tokens": "{\"description\": \"min=480.821, mean=480.821, max=480.821, sum=961.642 (2)\", \"tab\": \"General information\", \"score\": \"480.8208092485549\"}", - "Moral Disputes - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Moral Scenarios - # eval": "{\"description\": \"min=895, mean=895, max=895, sum=1790 (2)\", \"tab\": \"General information\", \"score\": \"895.0\"}", - "Moral Scenarios - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Scenarios - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # prompt tokens": "{\"description\": \"min=672.035, mean=672.035, max=672.035, sum=1344.069 (2)\", \"tab\": \"General information\", \"score\": \"672.0346368715084\"}", - "Moral Scenarios - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"moral_scenarios\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_moral_scenarios\"" - } - } - }, - { - "evaluation_name": "Nutrition", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Nutrition", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.709, - "details": { - "description": "min=0.709, mean=0.709, max=0.709, sum=1.418 (2)", - "tab": "Accuracy", - "Nutrition - Observed inference time (s)": "{\"description\": \"min=0.903, mean=0.903, max=0.903, sum=1.807 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9033067834143546\"}", - "Nutrition - # eval": "{\"description\": \"min=306, mean=306, max=306, sum=612 (2)\", \"tab\": \"General information\", \"score\": \"306.0\"}", - "Nutrition - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Nutrition - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Nutrition - # prompt tokens": "{\"description\": \"min=590.154, mean=590.154, max=590.154, sum=1180.307 (2)\", \"tab\": \"General information\", \"score\": \"590.1535947712418\"}", - "Nutrition - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"nutrition\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_nutrition\"" - } - } - }, - { - "evaluation_name": "Prehistory", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Prehistory", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.765, - "details": { - "description": "min=0.765, mean=0.765, max=0.765, sum=1.531 (2)", - "tab": "Accuracy", - "Prehistory - Observed inference time (s)": "{\"description\": \"min=0.849, mean=0.849, max=0.849, sum=1.698 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8491357167561849\"}", - "Prehistory - # eval": "{\"description\": \"min=324, mean=324, max=324, sum=648 (2)\", \"tab\": \"General information\", \"score\": \"324.0\"}", - "Prehistory - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Prehistory - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Prehistory - # prompt tokens": "{\"description\": \"min=526.04, mean=526.04, max=526.04, sum=1052.08 (2)\", \"tab\": \"General information\", \"score\": \"526.0401234567901\"}", - "Prehistory - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"prehistory\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_prehistory\"" - } - } - }, - { - "evaluation_name": "Public Relations", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Public Relations", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.718, - "details": { - "description": "min=0.718, mean=0.718, max=0.718, sum=1.436 (2)", - "tab": "Accuracy", - "Public Relations - Observed inference time (s)": "{\"description\": \"min=1.141, mean=1.141, max=1.141, sum=2.281 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.1407060449773616\"}", - "Public Relations - # eval": "{\"description\": \"min=110, mean=110, max=110, sum=220 (2)\", \"tab\": \"General information\", \"score\": \"110.0\"}", - "Public Relations - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Public Relations - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Public Relations - # prompt tokens": "{\"description\": \"min=409.045, mean=409.045, max=409.045, sum=818.091 (2)\", \"tab\": \"General information\", \"score\": \"409.04545454545456\"}", - "Public Relations - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"public_relations\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_public_relations\"" - } - } - }, - { - "evaluation_name": "Security Studies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Security Studies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.771, - "details": { - "description": "min=0.771, mean=0.771, max=0.771, sum=1.543 (2)", - "tab": "Accuracy", - "Security Studies - Observed inference time (s)": "{\"description\": \"min=0.958, mean=0.958, max=0.958, sum=1.915 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9576426525505222\"}", - "Security Studies - # eval": "{\"description\": \"min=245, mean=245, max=245, sum=490 (2)\", \"tab\": \"General information\", \"score\": \"245.0\"}", - "Security Studies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Security Studies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Security Studies - # prompt tokens": "{\"description\": \"min=1186.502, mean=1186.502, max=1186.502, sum=2373.004 (2)\", \"tab\": \"General information\", \"score\": \"1186.5020408163266\"}", - "Security Studies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"security_studies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_security_studies\"" - } - } - }, - { - "evaluation_name": "Sociology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Sociology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.726, - "details": { - "description": "min=0.726, mean=0.726, max=0.726, sum=1.453 (2)", - "tab": "Accuracy", - "Sociology - Observed inference time (s)": "{\"description\": \"min=0.781, mean=0.781, max=0.781, sum=1.562 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.781044238835425\"}", - "Sociology - # eval": "{\"description\": \"min=201, mean=201, max=201, sum=402 (2)\", \"tab\": \"General information\", \"score\": \"201.0\"}", - "Sociology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Sociology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Sociology - # prompt tokens": "{\"description\": \"min=455.348, mean=455.348, max=455.348, sum=910.697 (2)\", \"tab\": \"General information\", \"score\": \"455.3482587064677\"}", - "Sociology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"sociology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_sociology\"" - } - } - }, - { - "evaluation_name": "Virology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Virology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.56, - "details": { - "description": "min=0.56, mean=0.56, max=0.56, sum=1.12 (2)", - "tab": "Accuracy", - "Virology - Observed inference time (s)": "{\"description\": \"min=1.059, mean=1.059, max=1.059, sum=2.118 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.0589684750660355\"}", - "Virology - # eval": "{\"description\": \"min=166, mean=166, max=166, sum=332 (2)\", \"tab\": \"General information\", \"score\": \"166.0\"}", - "Virology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Virology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Virology - # prompt tokens": "{\"description\": \"min=336.127, mean=336.127, max=336.127, sum=672.253 (2)\", \"tab\": \"General information\", \"score\": \"336.1265060240964\"}", - "Virology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"virology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_virology\"" - } - } - }, - { - "evaluation_name": "World Religions", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on World Religions", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.789, - "details": { - "description": "min=0.789, mean=0.789, max=0.789, sum=1.579 (2)", - "tab": "Accuracy", - "World Religions - Observed inference time (s)": "{\"description\": \"min=0.891, mean=0.891, max=0.891, sum=1.781 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8906254336150766\"}", - "World Religions - # eval": "{\"description\": \"min=171, mean=171, max=171, sum=342 (2)\", \"tab\": \"General information\", \"score\": \"171.0\"}", - "World Religions - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "World Religions - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "World Religions - # prompt tokens": "{\"description\": \"min=275.181, mean=275.181, max=275.181, sum=550.363 (2)\", \"tab\": \"General information\", \"score\": \"275.1812865497076\"}", - "World Religions - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"world_religions\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_world_religions\"" - } - } - }, - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.215, - "details": { - "description": "", - "tab": "Efficiency" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/mixtao_mixtao-7bx2-moe-v8.1.json b/data/models/mixtao_mixtao-7bx2-moe-v8.1.json deleted file mode 100644 index 91978cb7c4f59b6f81e719adf6fc0b172438a84a..0000000000000000000000000000000000000000 --- a/data/models/mixtao_mixtao-7bx2-moe-v8.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MixTAO-7Bx2-MoE-v8.1", - "id": "mixtao/MixTAO-7Bx2-MoE-v8.1", - "developer": "mixtao", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MixtralForCausalLM", - "params_billions": "12.879" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/mixtao_MixTAO-7Bx2-MoE-v8.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4162 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5189 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0906 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2844 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4463 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3123 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/mkurman_llama-3.2-medit-3b-o1.json b/data/models/mkurman_llama-3.2-medit-3b-o1.json deleted file mode 100644 index 3fdd99342642f25f4621008a87a21d55990fdc1e..0000000000000000000000000000000000000000 --- a/data/models/mkurman_llama-3.2-medit-3b-o1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "llama-3.2-MEDIT-3B-o1", - "id": "mkurman/llama-3.2-MEDIT-3B-o1", - "developer": "mkurman", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.607" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/mkurman_llama-3.2-MEDIT-3B-o1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4382 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.44 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1307 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2659 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3565 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2741 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/mkurman_phi-4-medit-11b-exp-1.json b/data/models/mkurman_phi-4-medit-11b-exp-1.json deleted file mode 100644 index ecb62f76b45ee63c51d66bb100b25494c45d1f15..0000000000000000000000000000000000000000 --- a/data/models/mkurman_phi-4-medit-11b-exp-1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "phi-4-MedIT-11B-exp-1", - "id": "mkurman/phi-4-MedIT-11B-exp-1", - "developer": "mkurman", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Phi3ForCausalLM", - "params_billions": "11.514" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/mkurman_phi-4-MedIT-11B-exp-1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5948 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5414 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0899 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3012 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3848 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3825 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/mkurman_phi4-medit-10b-o1.json b/data/models/mkurman_phi4-medit-10b-o1.json deleted file mode 100644 index a5013abb908339470b311420a6444218fb5a15da..0000000000000000000000000000000000000000 --- a/data/models/mkurman_phi4-medit-10b-o1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "phi4-MedIT-10B-o1", - "id": "mkurman/phi4-MedIT-10B-o1", - "developer": "mkurman", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaMedITForCausalLM", - "params_billions": "10.255" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/mkurman_phi4-MedIT-10B-o1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3463 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5198 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1148 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2458 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3968 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3507 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/mkxu_llama-3-8b-instruct-fpo.json b/data/models/mkxu_llama-3-8b-instruct-fpo.json deleted file mode 100644 index 6b5c82b5adfca37fc47ca64470181efaf7ba1024..0000000000000000000000000000000000000000 --- a/data/models/mkxu_llama-3-8b-instruct-fpo.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "llama-3-8b-instruct-fpo", - "id": "mkxu/llama-3-8b-instruct-fpo", - "developer": "mkxu", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/mkxu_llama-3-8b-instruct-fpo/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.679 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4959 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0733 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2777 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3658 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3605 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/mkxu_llama-3-8b-po1.json b/data/models/mkxu_llama-3-8b-po1.json deleted file mode 100644 index e6aa161121104b1fac1c0d11d6b4712574283625..0000000000000000000000000000000000000000 --- a/data/models/mkxu_llama-3-8b-po1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "llama-3-8b-po1", - "id": "mkxu/llama-3-8b-po1", - "developer": "mkxu", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/mkxu_llama-3-8b-po1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4081 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4976 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0702 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.297 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3804 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3562 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/mlabonne_alphamonarch-7b.json b/data/models/mlabonne_alphamonarch-7b.json deleted file mode 100644 index 5464b68c2550e02641a0d7389499ff810135625a..0000000000000000000000000000000000000000 --- a/data/models/mlabonne_alphamonarch-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "AlphaMonarch-7B", - "id": "mlabonne/AlphaMonarch-7B", - "developer": "mlabonne", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/mlabonne_AlphaMonarch-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4939 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4626 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0408 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2701 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4121 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2473 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/mlabonne_beyonder-4x7b-v3.json b/data/models/mlabonne_beyonder-4x7b-v3.json deleted file mode 100644 index 549fdf70c6c7f39055f50f223cc34131b01a0b9e..0000000000000000000000000000000000000000 --- a/data/models/mlabonne_beyonder-4x7b-v3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Beyonder-4x7B-v3", - "id": "mlabonne/Beyonder-4x7B-v3", - "developer": "mlabonne", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MixtralForCausalLM", - "params_billions": "24.154" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/mlabonne_Beyonder-4x7B-v3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5608 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4671 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0536 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2852 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4045 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2512 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/mlabonne_bigqwen2.5-52b-instruct.json b/data/models/mlabonne_bigqwen2.5-52b-instruct.json deleted file mode 100644 index 749d5a9b89300734100d5056f3c75594d2f59be0..0000000000000000000000000000000000000000 --- a/data/models/mlabonne_bigqwen2.5-52b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "BigQwen2.5-52B-Instruct", - "id": "mlabonne/BigQwen2.5-52B-Instruct", - "developer": "mlabonne", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "52.268" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/mlabonne_BigQwen2.5-52B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7913 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7121 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5476 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.302 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4113 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5519 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/mlabonne_bigqwen2.5-echo-47b-instruct.json b/data/models/mlabonne_bigqwen2.5-echo-47b-instruct.json deleted file mode 100644 index 441e9d225025f5c7793a16668ebc8e4efbbfe278..0000000000000000000000000000000000000000 --- a/data/models/mlabonne_bigqwen2.5-echo-47b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "BigQwen2.5-Echo-47B-Instruct", - "id": "mlabonne/BigQwen2.5-Echo-47B-Instruct", - "developer": "mlabonne", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "47.392" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/mlabonne_BigQwen2.5-Echo-47B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7357 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6125 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4381 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3146 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4125 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4734 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/mlabonne_chimerallama-3-8b-v2.json b/data/models/mlabonne_chimerallama-3-8b-v2.json deleted file mode 100644 index f9cf2a6487a9ddd27532cf3715447949d0369ae8..0000000000000000000000000000000000000000 --- a/data/models/mlabonne_chimerallama-3-8b-v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ChimeraLlama-3-8B-v2", - "id": "mlabonne/ChimeraLlama-3-8B-v2", - "developer": "mlabonne", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/mlabonne_ChimeraLlama-3-8B-v2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4469 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5046 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0906 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2852 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3791 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3569 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/mlabonne_chimerallama-3-8b-v3.json b/data/models/mlabonne_chimerallama-3-8b-v3.json deleted file mode 100644 index fb5c6d692875227411cb58e58958aa4e924cd9ef..0000000000000000000000000000000000000000 --- a/data/models/mlabonne_chimerallama-3-8b-v3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ChimeraLlama-3-8B-v3", - "id": "mlabonne/ChimeraLlama-3-8B-v3", - "developer": "mlabonne", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/mlabonne_ChimeraLlama-3-8B-v3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4408 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4978 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0884 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2919 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4004 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3669 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/mlabonne_daredevil-8b-abliterated.json b/data/models/mlabonne_daredevil-8b-abliterated.json deleted file mode 100644 index ed78b4891c05d7b8fc3aa9325c380ac6375c9cbb..0000000000000000000000000000000000000000 --- a/data/models/mlabonne_daredevil-8b-abliterated.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Daredevil-8B-abliterated", - "id": "mlabonne/Daredevil-8B-abliterated", - "developer": "mlabonne", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/mlabonne_Daredevil-8B-abliterated/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4426 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4254 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0944 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2903 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.407 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3701 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/mlabonne_daredevil-8b.json b/data/models/mlabonne_daredevil-8b.json deleted file mode 100644 index 73340f48353fcb939b69f79fda36b90fc5449222..0000000000000000000000000000000000000000 --- a/data/models/mlabonne_daredevil-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Daredevil-8B", - "id": "mlabonne/Daredevil-8B", - "developer": "mlabonne", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/mlabonne_Daredevil-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4548 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5194 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1065 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3079 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3939 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3831 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/mlabonne_hermes-3-llama-3.1-70b-lorablated.json b/data/models/mlabonne_hermes-3-llama-3.1-70b-lorablated.json deleted file mode 100644 index c01ad0096669c0e9c4013cccce7d198f19bf68e9..0000000000000000000000000000000000000000 --- a/data/models/mlabonne_hermes-3-llama-3.1-70b-lorablated.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Hermes-3-Llama-3.1-70B-lorablated", - "id": "mlabonne/Hermes-3-Llama-3.1-70B-lorablated", - "developer": "mlabonne", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "70.554" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/mlabonne_Hermes-3-Llama-3.1-70B-lorablated/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3424 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6693 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2243 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3658 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5029 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4679 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/mlabonne_meta-llama-3.1-8b-instruct-abliterated.json b/data/models/mlabonne_meta-llama-3.1-8b-instruct-abliterated.json deleted file mode 100644 index 1af9ce9d2671b99760c7bb3f6ada843a5efab3b1..0000000000000000000000000000000000000000 --- a/data/models/mlabonne_meta-llama-3.1-8b-instruct-abliterated.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Meta-Llama-3.1-8B-Instruct-abliterated", - "id": "mlabonne/Meta-Llama-3.1-8B-Instruct-abliterated", - "developer": "mlabonne", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/mlabonne_Meta-Llama-3.1-8B-Instruct-abliterated/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7329 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4874 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0687 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2567 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3649 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3503 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/mlabonne_neuralbeagle14-7b.json b/data/models/mlabonne_neuralbeagle14-7b.json deleted file mode 100644 index e45fabaa0e7a079ab99dfc791ac0cfc30e43564e..0000000000000000000000000000000000000000 --- a/data/models/mlabonne_neuralbeagle14-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "NeuralBeagle14-7B", - "id": "mlabonne/NeuralBeagle14-7B", - "developer": "mlabonne", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/mlabonne_NeuralBeagle14-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4935 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4628 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0521 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2819 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4319 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2601 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/mlabonne_neuraldaredevil-8b-abliterated.json b/data/models/mlabonne_neuraldaredevil-8b-abliterated.json deleted file mode 100644 index 7ef165972eafdeec56d923c82e02fdbbc9479eac..0000000000000000000000000000000000000000 --- a/data/models/mlabonne_neuraldaredevil-8b-abliterated.json +++ /dev/null @@ -1,275 +0,0 @@ -{ - "model_info": { - "name": "NeuralDaredevil-8B-abliterated", - "id": "mlabonne/NeuralDaredevil-8B-abliterated", - "developer": "mlabonne", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/mlabonne_NeuralDaredevil-8B-abliterated/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4162 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5124 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0853 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3029 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.415 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3802 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "hfopenllm_v2/mlabonne_NeuralDaredevil-8B-abliterated/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7561 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5111 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0906 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3062 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4019 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3841 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/mlabonne_orpollama-3-8b.json b/data/models/mlabonne_orpollama-3-8b.json deleted file mode 100644 index afd40b3e533921f05af756979d46d17c69fbda25..0000000000000000000000000000000000000000 --- a/data/models/mlabonne_orpollama-3-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "OrpoLlama-3-8B", - "id": "mlabonne/OrpoLlama-3-8B", - "developer": "mlabonne", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/mlabonne_OrpoLlama-3-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3653 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4424 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0559 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2794 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3579 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2705 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/mlabonne_phixtral-2x2_8.json b/data/models/mlabonne_phixtral-2x2_8.json deleted file mode 100644 index a5127c5ea42100883e22708fb5564ad93c4b5e45..0000000000000000000000000000000000000000 --- a/data/models/mlabonne_phixtral-2x2_8.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "phixtral-2x2_8", - "id": "mlabonne/phixtral-2x2_8", - "developer": "mlabonne", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "PhiForCausalLM", - "params_billions": "4.458" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/mlabonne_phixtral-2x2_8/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3431 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4889 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0355 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2651 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3644 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2551 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/mlp-ktlim_llama-3-korean-bllossom-8b.json b/data/models/mlp-ktlim_llama-3-korean-bllossom-8b.json deleted file mode 100644 index a7990963b4643e35630f34b5c03cb1535e4b6765..0000000000000000000000000000000000000000 --- a/data/models/mlp-ktlim_llama-3-korean-bllossom-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "llama-3-Korean-Bllossom-8B", - "id": "MLP-KTLim/llama-3-Korean-Bllossom-8B", - "developer": "MLP-KTLim", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/MLP-KTLim_llama-3-Korean-Bllossom-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5113 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.49 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.102 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2626 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3675 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3594 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/mlx-community_josiefied-qwen2.5-0.5b-instruct-abliterated-v1-float32.json b/data/models/mlx-community_josiefied-qwen2.5-0.5b-instruct-abliterated-v1-float32.json deleted file mode 100644 index 10336237a366d1c635c4f79ecbbdf6844ec7368d..0000000000000000000000000000000000000000 --- a/data/models/mlx-community_josiefied-qwen2.5-0.5b-instruct-abliterated-v1-float32.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Josiefied-Qwen2.5-0.5B-Instruct-abliterated-v1-float32", - "id": "mlx-community/Josiefied-Qwen2.5-0.5B-Instruct-abliterated-v1-float32", - "developer": "mlx-community", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/mlx-community_Josiefied-Qwen2.5-0.5B-Instruct-abliterated-v1-float32/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3369 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3292 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0846 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2576 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3249 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1638 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/mlx-community_mistral-small-24b-instruct-2501-bf16.json b/data/models/mlx-community_mistral-small-24b-instruct-2501-bf16.json deleted file mode 100644 index 2ad0ab17a3f0925238f679b10b6aa91f679f0d1a..0000000000000000000000000000000000000000 --- a/data/models/mlx-community_mistral-small-24b-instruct-2501-bf16.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mistral-Small-24B-Instruct-2501-bf16", - "id": "mlx-community/Mistral-Small-24B-Instruct-2501-bf16", - "developer": "mlx-community", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "23.572" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/mlx-community_Mistral-Small-24B-Instruct-2501-bf16/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6283 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6713 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3225 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3951 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4618 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5395 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/mmnga_llama-3-70b-japanese-suzume-vector-v0.1.json b/data/models/mmnga_llama-3-70b-japanese-suzume-vector-v0.1.json deleted file mode 100644 index c59a85221ee1db53149a467109f080ac6bbadf61..0000000000000000000000000000000000000000 --- a/data/models/mmnga_llama-3-70b-japanese-suzume-vector-v0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-70B-japanese-suzume-vector-v0.1", - "id": "mmnga/Llama-3-70B-japanese-suzume-vector-v0.1", - "developer": "mmnga", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "70.554" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/mmnga_Llama-3-70B-japanese-suzume-vector-v0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4649 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6542 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2326 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2861 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4141 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5224 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/mobiuslabsgmbh_deepseek-r1-redistill-llama3-8b-v1.1.json b/data/models/mobiuslabsgmbh_deepseek-r1-redistill-llama3-8b-v1.1.json deleted file mode 100644 index e17366ebb76aa24feafc056029af9ea9d815b7ab..0000000000000000000000000000000000000000 --- a/data/models/mobiuslabsgmbh_deepseek-r1-redistill-llama3-8b-v1.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "DeepSeek-R1-ReDistill-Llama3-8B-v1.1", - "id": "mobiuslabsgmbh/DeepSeek-R1-ReDistill-Llama3-8B-v1.1", - "developer": "mobiuslabsgmbh", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/mobiuslabsgmbh_DeepSeek-R1-ReDistill-Llama3-8B-v1.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3704 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3473 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3285 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.271 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3396 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2198 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/mobiuslabsgmbh_deepseek-r1-redistill-qwen-7b-v1.1.json b/data/models/mobiuslabsgmbh_deepseek-r1-redistill-qwen-7b-v1.1.json deleted file mode 100644 index 60d61feeaeb5a8c9743945a029c1d7f32fceb4a0..0000000000000000000000000000000000000000 --- a/data/models/mobiuslabsgmbh_deepseek-r1-redistill-qwen-7b-v1.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "DeepSeek-R1-ReDistill-Qwen-7B-v1.1", - "id": "mobiuslabsgmbh/DeepSeek-R1-ReDistill-Qwen-7B-v1.1", - "developer": "mobiuslabsgmbh", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/mobiuslabsgmbh_DeepSeek-R1-ReDistill-Qwen-7B-v1.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3473 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3698 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3497 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2651 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4009 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2326 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/modelcloud_llama-3.2-1b-instruct-gptqmodel-4bit-vortex-v1.json b/data/models/modelcloud_llama-3.2-1b-instruct-gptqmodel-4bit-vortex-v1.json deleted file mode 100644 index ff7b63f974526c1e11897e329dfaeb315ef706be..0000000000000000000000000000000000000000 --- a/data/models/modelcloud_llama-3.2-1b-instruct-gptqmodel-4bit-vortex-v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.2-1B-Instruct-gptqmodel-4bit-vortex-v1", - "id": "ModelCloud/Llama-3.2-1B-Instruct-gptqmodel-4bit-vortex-v1", - "developer": "ModelCloud", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "5.453" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ModelCloud_Llama-3.2-1B-Instruct-gptqmodel-4bit-vortex-v1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5269 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3253 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0604 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2534 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3249 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1764 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/modelspace_gemmax2-28-9b-v0.1.json b/data/models/modelspace_gemmax2-28-9b-v0.1.json deleted file mode 100644 index 748cf0459dfaf9da065f9b1209d14b3f0d281c93..0000000000000000000000000000000000000000 --- a/data/models/modelspace_gemmax2-28-9b-v0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "GemmaX2-28-9B-v0.1", - "id": "ModelSpace/GemmaX2-28-9B-v0.1", - "developer": "ModelSpace", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ModelSpace_GemmaX2-28-9B-v0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0039 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3687 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0272 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2768 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3537 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2231 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/moeru-ai_l3.1-moe-2x8b-v0.2.json b/data/models/moeru-ai_l3.1-moe-2x8b-v0.2.json deleted file mode 100644 index e18d70020d2bf7855ec578da5a11fffdbc593de6..0000000000000000000000000000000000000000 --- a/data/models/moeru-ai_l3.1-moe-2x8b-v0.2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "L3.1-Moe-2x8B-v0.2", - "id": "moeru-ai/L3.1-Moe-2x8B-v0.2", - "developer": "moeru-ai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MixtralForCausalLM", - "params_billions": "13.668" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/moeru-ai_L3.1-Moe-2x8B-v0.2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7348 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5256 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1699 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3003 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4199 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3858 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/moeru-ai_l3.1-moe-4x8b-v0.1.json b/data/models/moeru-ai_l3.1-moe-4x8b-v0.1.json deleted file mode 100644 index 72e2ced741e05972abd1cc962df7b5cd322bffb2..0000000000000000000000000000000000000000 --- a/data/models/moeru-ai_l3.1-moe-4x8b-v0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "L3.1-Moe-4x8B-v0.1", - "id": "moeru-ai/L3.1-Moe-4x8B-v0.1", - "developer": "moeru-ai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MixtralForCausalLM", - "params_billions": "24.942" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/moeru-ai_L3.1-Moe-4x8B-v0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4332 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4939 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1299 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2592 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3609 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3454 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/moeru-ai_l3.1-moe-4x8b-v0.2.json b/data/models/moeru-ai_l3.1-moe-4x8b-v0.2.json deleted file mode 100644 index cd0df0162078119cb1548478ee59082828fc33e4..0000000000000000000000000000000000000000 --- a/data/models/moeru-ai_l3.1-moe-4x8b-v0.2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "L3.1-Moe-4x8B-v0.2", - "id": "moeru-ai/L3.1-Moe-4x8B-v0.2", - "developer": "moeru-ai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MixtralForCausalLM", - "params_billions": "24.942" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/moeru-ai_L3.1-Moe-4x8B-v0.2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5407 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4466 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1035 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2668 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3234 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2763 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/monsterapi_gemma-2-2b-lora-monsterinstruct.json b/data/models/monsterapi_gemma-2-2b-lora-monsterinstruct.json deleted file mode 100644 index 169932c2aad78456f62665935c7e3ead393a99c0..0000000000000000000000000000000000000000 --- a/data/models/monsterapi_gemma-2-2b-lora-monsterinstruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "gemma-2-2b-LoRA-MonsterInstruct", - "id": "monsterapi/gemma-2-2b-LoRA-MonsterInstruct", - "developer": "monsterapi", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "2.614" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/monsterapi_gemma-2-2b-LoRA-MonsterInstruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3903 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.365 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0506 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2701 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3644 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1987 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/monsterapi_llama-3_1-8b-instruct-orca-orpo.json b/data/models/monsterapi_llama-3_1-8b-instruct-orca-orpo.json deleted file mode 100644 index a5e83fac621352ce2ec6de2eed522c5509e73561..0000000000000000000000000000000000000000 --- a/data/models/monsterapi_llama-3_1-8b-instruct-orca-orpo.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3_1-8B-Instruct-orca-ORPO", - "id": "monsterapi/Llama-3_1-8B-Instruct-orca-ORPO", - "developer": "monsterapi", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "?", - "params_billions": "16.061" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/monsterapi_Llama-3_1-8B-Instruct-orca-ORPO/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2273 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2865 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2492 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3445 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1168 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/moonride_llama-3.2-3b-khelavaster.json b/data/models/moonride_llama-3.2-3b-khelavaster.json deleted file mode 100644 index 2174b6261f99141aeef609e4ec467953f5855fcf..0000000000000000000000000000000000000000 --- a/data/models/moonride_llama-3.2-3b-khelavaster.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.2-3B-Khelavaster", - "id": "MoonRide/Llama-3.2-3B-Khelavaster", - "developer": "MoonRide", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.607" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/MoonRide_Llama-3.2-3B-Khelavaster/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4925 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4516 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1616 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2777 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3699 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3122 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/moonshot-ai_kimi-k2-instruct.json b/data/models/moonshot-ai_kimi-k2-instruct.json deleted file mode 100644 index 758984500ae56b028445a56fb8562c74a90a3a8f..0000000000000000000000000000000000000000 --- a/data/models/moonshot-ai_kimi-k2-instruct.json +++ /dev/null @@ -1,161 +0,0 @@ -{ - "model_info": { - "name": "Kimi K2 Instruct", - "id": "moonshot-ai/kimi-k2-instruct", - "developer": "Moonshot AI", - "additional_details": { - "agent_name": "OpenHands", - "agent_organization": "OpenHands" - } - }, - "evaluations": [ - { - "evaluation_id": "terminal-bench-2.0/openhands__kimi-k2-instruct/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2025-11-02", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 26.7, - "uncertainty": { - "standard_error": { - "value": 2.7 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"OpenHands\" -m \"Kimi K2 Instruct\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"OpenHands\" -m \"Kimi K2 Instruct\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - }, - { - "evaluation_id": "terminal-bench-2.0/terminus-2__kimi-k2-instruct/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2025-11-01", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 27.8, - "uncertainty": { - "standard_error": { - "value": 2.5 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Terminus 2\" -m \"Kimi K2 Instruct\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Terminus 2\" -m \"Kimi K2 Instruct\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/moonshot-ai_kimi-k2-thinking.json b/data/models/moonshot-ai_kimi-k2-thinking.json deleted file mode 100644 index 67d6125d3fb860173dd910b3dfd2de44bed0ec45..0000000000000000000000000000000000000000 --- a/data/models/moonshot-ai_kimi-k2-thinking.json +++ /dev/null @@ -1,87 +0,0 @@ -{ - "model_info": { - "name": "Kimi K2 Thinking", - "id": "moonshot-ai/kimi-k2-thinking", - "developer": "Moonshot AI", - "additional_details": { - "agent_name": "Terminus 2", - "agent_organization": "Terminal Bench" - } - }, - "evaluations": [ - { - "evaluation_id": "terminal-bench-2.0/terminus-2__kimi-k2-thinking/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2025-11-11", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 35.7, - "uncertainty": { - "standard_error": { - "value": 2.8 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Terminus 2\" -m \"Kimi K2 Thinking\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Terminus 2\" -m \"Kimi K2 Thinking\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/moonshot-ai_kimi-k2.5.json b/data/models/moonshot-ai_kimi-k2.5.json deleted file mode 100644 index af71cc9bb5541b47ae5874d63af695921f5fb852..0000000000000000000000000000000000000000 --- a/data/models/moonshot-ai_kimi-k2.5.json +++ /dev/null @@ -1,87 +0,0 @@ -{ - "model_info": { - "name": "Kimi K2.5", - "id": "moonshot-ai/kimi-k2.5", - "developer": "Kimi", - "additional_details": { - "agent_name": "Terminus 2", - "agent_organization": "Terminal Bench" - } - }, - "evaluations": [ - { - "evaluation_id": "terminal-bench-2.0/terminus-2__kimi-k2.5/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2026-02-04", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 43.2, - "uncertainty": { - "standard_error": { - "value": 2.9 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Terminus 2\" -m \"Kimi K2.5\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Terminus 2\" -m \"Kimi K2.5\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/moonshot_kimi_k2.5.json b/data/models/moonshot_kimi_k2.5.json deleted file mode 100644 index ca4b6cd4115851fd24f0aedd2357198122cbcef9..0000000000000000000000000000000000000000 --- a/data/models/moonshot_kimi_k2.5.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "model_info": { - "name": "Kimi K2.5", - "developer": "moonshot", - "id": "moonshot/Kimi K2.5", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "apex-agents/moonshot_kimi-k2.5/1773260200", - "retrieved_timestamp": "1773260200", - "source_metadata": { - "source_name": "Mercor APEX-Agents Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Mercor", - "source_organization_url": "https://www.mercor.com", - "evaluator_relationship": "first_party" - }, - "eval_library": { - "name": "archipelago", - "version": "1.0.0" - }, - "benchmark": "apex-agents", - "evaluation_results": [ - { - "evaluation_name": "Corporate Lawyer Mean Score", - "source_data": { - "dataset_name": "apex-agents", - "source_type": "hf_dataset", - "hf_repo": "mercor/apex-agents" - }, - "metric_config": { - "evaluation_description": "Corporate lawyer world mean score.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.402 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/moonshot_kimi_k2_thinking.json b/data/models/moonshot_kimi_k2_thinking.json deleted file mode 100644 index 5489f04e50181d6cf6bb9ee9a7d58806f343b8e4..0000000000000000000000000000000000000000 --- a/data/models/moonshot_kimi_k2_thinking.json +++ /dev/null @@ -1,170 +0,0 @@ -{ - "model_info": { - "name": "Kimi K2 Thinking", - "developer": "moonshot", - "id": "moonshot/Kimi K2 Thinking", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "apex-agents/moonshot_kimi-k2-thinking/1773260200", - "retrieved_timestamp": "1773260200", - "source_metadata": { - "source_name": "Mercor APEX-Agents Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Mercor", - "source_organization_url": "https://www.mercor.com", - "evaluator_relationship": "first_party" - }, - "eval_library": { - "name": "archipelago", - "version": "1.0.0" - }, - "benchmark": "apex-agents", - "evaluation_results": [ - { - "evaluation_name": "Overall Pass@1", - "source_data": { - "dataset_name": "apex-agents", - "source_type": "hf_dataset", - "hf_repo": "mercor/apex-agents" - }, - "metric_config": { - "evaluation_description": "Overall Pass@1 (dataset card / paper snapshot).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.04, - "uncertainty": { - "confidence_interval": { - "lower": -0.011, - "upper": 0.012, - "method": "bootstrap" - } - } - } - }, - { - "evaluation_name": "Overall Pass@8", - "source_data": { - "dataset_name": "apex-agents", - "source_type": "hf_dataset", - "hf_repo": "mercor/apex-agents" - }, - "metric_config": { - "evaluation_description": "Overall Pass@8 (dataset card / paper snapshot).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.144, - "uncertainty": { - "confidence_interval": { - "lower": -0.029, - "upper": 0.031, - "method": "bootstrap" - } - } - } - }, - { - "evaluation_name": "Overall Mean Score", - "source_data": { - "dataset_name": "apex-agents", - "source_type": "hf_dataset", - "hf_repo": "mercor/apex-agents" - }, - "metric_config": { - "evaluation_description": "Overall mean rubric score.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.115 - } - }, - { - "evaluation_name": "Investment Banking Pass@1", - "source_data": { - "dataset_name": "apex-agents", - "source_type": "hf_dataset", - "hf_repo": "mercor/apex-agents" - }, - "metric_config": { - "evaluation_description": "Investment banking world Pass@1.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.012 - } - }, - { - "evaluation_name": "Management Consulting Pass@1", - "source_data": { - "dataset_name": "apex-agents", - "source_type": "hf_dataset", - "hf_repo": "mercor/apex-agents" - }, - "metric_config": { - "evaluation_description": "Management consulting world Pass@1.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.029 - } - }, - { - "evaluation_name": "Corporate Law Pass@1", - "source_data": { - "dataset_name": "apex-agents", - "source_type": "hf_dataset", - "hf_repo": "mercor/apex-agents" - }, - "metric_config": { - "evaluation_description": "Corporate law world Pass@1.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.08 - } - }, - { - "evaluation_name": "Corporate Lawyer Mean Score", - "source_data": { - "dataset_name": "apex-agents", - "source_type": "hf_dataset", - "hf_repo": "mercor/apex-agents" - }, - "metric_config": { - "evaluation_description": "Corporate lawyer world mean score.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.223 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/moonshotai_kimi-k2-instruct.json b/data/models/moonshotai_kimi-k2-instruct.json deleted file mode 100644 index cc184425e10f4f7e9e5b3173d41f792cd3dcfea3..0000000000000000000000000000000000000000 --- a/data/models/moonshotai_kimi-k2-instruct.json +++ /dev/null @@ -1,234 +0,0 @@ -{ - "model_info": { - "name": "Kimi K2 Instruct", - "id": "moonshotai/kimi-k2-instruct", - "developer": "moonshotai", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_capabilities/moonshotai_kimi-k2-instruct/1774096304.5056698", - "retrieved_timestamp": "1774096304.5056698", - "source_metadata": { - "source_name": "helm_capabilities", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_capabilities", - "evaluation_results": [ - { - "evaluation_name": "Mean score", - "source_data": { - "dataset_name": "helm_capabilities", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "The mean of the scores from all columns.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.768, - "details": { - "description": "", - "tab": "Accuracy", - "Mean score - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"44.938299779825435\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU-Pro", - "source_data": { - "dataset_name": "MMLU-Pro", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on MMLU-Pro", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.819, - "details": { - "description": "min=0.819, mean=0.819, max=0.819, sum=0.819 (1)", - "tab": "Accuracy", - "MMLU-Pro - Observed inference time (s)": "{\"description\": \"min=20.295, mean=20.295, max=20.295, sum=20.295 (1)\", \"tab\": \"Efficiency\", \"score\": \"20.295415951013567\"}", - "MMLU-Pro - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "MMLU-Pro - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - # prompt tokens": "{\"description\": \"min=249.352, mean=249.352, max=249.352, sum=249.352 (1)\", \"tab\": \"General information\", \"score\": \"249.352\"}", - "MMLU-Pro - # output tokens": "{\"description\": \"min=703.4, mean=703.4, max=703.4, sum=703.4 (1)\", \"tab\": \"General information\", \"score\": \"703.4\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"all\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.652, - "details": { - "description": "min=0.652, mean=0.652, max=0.652, sum=0.652 (1)", - "tab": "Accuracy", - "GPQA - Observed inference time (s)": "{\"description\": \"min=50.104, mean=50.104, max=50.104, sum=50.104 (1)\", \"tab\": \"Efficiency\", \"score\": \"50.10382581986654\"}", - "GPQA - # eval": "{\"description\": \"min=446, mean=446, max=446, sum=446 (1)\", \"tab\": \"General information\", \"score\": \"446.0\"}", - "GPQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - # prompt tokens": "{\"description\": \"min=268.74, mean=268.74, max=268.74, sum=268.74 (1)\", \"tab\": \"General information\", \"score\": \"268.73991031390136\"}", - "GPQA - # output tokens": "{\"description\": \"min=1250.646, mean=1250.646, max=1250.646, sum=1250.646 (1)\", \"tab\": \"General information\", \"score\": \"1250.645739910314\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"gpqa_main\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "IFEval Strict Acc on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.85, - "details": { - "description": "min=0.85, mean=0.85, max=0.85, sum=0.85 (1)", - "tab": "Accuracy", - "IFEval - Observed inference time (s)": "{\"description\": \"min=17.412, mean=17.412, max=17.412, sum=17.412 (1)\", \"tab\": \"Efficiency\", \"score\": \"17.412336311587122\"}", - "IFEval - # eval": "{\"description\": \"min=541, mean=541, max=541, sum=541 (1)\", \"tab\": \"General information\", \"score\": \"541.0\"}", - "IFEval - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - # prompt tokens": "{\"description\": \"min=45.863, mean=45.863, max=45.863, sum=45.863 (1)\", \"tab\": \"General information\", \"score\": \"45.86321626617375\"}", - "IFEval - # output tokens": "{\"description\": \"min=454.283, mean=454.283, max=454.283, sum=454.283 (1)\", \"tab\": \"General information\", \"score\": \"454.2828096118299\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WildBench", - "source_data": { - "dataset_name": "WildBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "WB Score on WildBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.862, - "details": { - "description": "min=0.862, mean=0.862, max=0.862, sum=0.862 (1)", - "tab": "Accuracy", - "WildBench - Observed inference time (s)": "{\"description\": \"min=46.942, mean=46.942, max=46.942, sum=46.942 (1)\", \"tab\": \"Efficiency\", \"score\": \"46.94232517242432\"}", - "WildBench - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "WildBench - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # prompt tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # output tokens": "{\"description\": \"min=1332.527, mean=1332.527, max=1332.527, sum=1332.527 (1)\", \"tab\": \"General information\", \"score\": \"1332.527\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"v2\"" - } - } - }, - { - "evaluation_name": "Omni-MATH", - "source_data": { - "dataset_name": "Omni-MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Acc on Omni-MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.654, - "details": { - "description": "min=0.654, mean=0.654, max=0.654, sum=0.654 (1)", - "tab": "Accuracy", - "Omni-MATH - Observed inference time (s)": "{\"description\": \"min=89.938, mean=89.938, max=89.938, sum=89.938 (1)\", \"tab\": \"Efficiency\", \"score\": \"89.93759564423561\"}", - "Omni-MATH - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "Omni-MATH - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - # prompt tokens": "{\"description\": \"min=106.59, mean=106.59, max=106.59, sum=106.59 (1)\", \"tab\": \"General information\", \"score\": \"106.59\"}", - "Omni-MATH - # output tokens": "{\"description\": \"min=3396.692, mean=3396.692, max=3396.692, sum=3396.692 (1)\", \"tab\": \"General information\", \"score\": \"3396.692\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/moonshotai_moonshotai-kimi-k2-instruct-fc.json b/data/models/moonshotai_moonshotai-kimi-k2-instruct-fc.json deleted file mode 100644 index 2792ef82c65222c6a2ef1ee15fc05faacf8ed900..0000000000000000000000000000000000000000 --- a/data/models/moonshotai_moonshotai-kimi-k2-instruct-fc.json +++ /dev/null @@ -1,851 +0,0 @@ -{ - "model_info": { - "name": "Moonshotai-Kimi-K2-Instruct (FC)", - "id": "moonshotai/moonshotai-kimi-k2-instruct-fc", - "developer": "moonshotai", - "additional_details": { - "raw_model_name": "Moonshotai-Kimi-K2-Instruct (FC)", - "organization": "MoonshotAI", - "license": "modified-mit", - "mode": "FC", - "model_link": "https://huggingface.co/moonshotai/Kimi-K2-Instruct" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/moonshotai/moonshotai-kimi-k2-instruct-fc/1775236112.371072", - "retrieved_timestamp": "1775236112.371072", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 11.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 59.06 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 6.19 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 6.4 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 9.38 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 13.78 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 81.6 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 69.42 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 92.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 82.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 83.0 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 78.68 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 81.78 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 78.06 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 87.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 66.67 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 50.63 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 62.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 41.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 44.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 55.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 66.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 72.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 61.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 29.03 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 21.94 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 20.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 45.16 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 75.0 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 87.34 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/mosaicml_mpt-30b.json b/data/models/mosaicml_mpt-30b.json deleted file mode 100644 index a1dd38885b21d6cc46f62215d15e94685a85dd25..0000000000000000000000000000000000000000 --- a/data/models/mosaicml_mpt-30b.json +++ /dev/null @@ -1,674 +0,0 @@ -{ - "model_info": { - "name": "MPT 30B", - "id": "mosaicml/MPT-30B", - "developer": "mosaicml", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_classic/mosaicml_MPT-30B/1774096308.339228", - "retrieved_timestamp": "1774096308.339228", - "source_metadata": { - "source_name": "helm_classic", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_classic", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_classic", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperform on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.714, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Calibration": "{\"description\": \"\", \"tab\": \"Calibration\", \"score\": \"\"}", - "Mean win rate - Robustness": "{\"description\": \"\", \"tab\": \"Robustness\", \"score\": \"0.6966666666666667\"}", - "Mean win rate - Fairness": "{\"description\": \"\", \"tab\": \"Fairness\", \"score\": \"0.7464102564102564\"}", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}", - "Mean win rate - Bias": "{\"description\": \"\", \"tab\": \"Bias\", \"score\": \"0.2946998974900761\"}", - "Mean win rate - Toxicity": "{\"description\": \"\", \"tab\": \"Toxicity\", \"score\": \"0.44918414918414923\"}", - "Mean win rate - Summarization metrics": "{\"description\": \"\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.437, - "details": { - "description": "min=0.25, mean=0.437, max=0.68, sum=2.183 (5)", - "tab": "Accuracy", - "MMLU - ECE (10-bin)": "{\"description\": \"5 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "MMLU - EM (Robustness)": "{\"description\": \"min=0.25, mean=0.381, max=0.6, sum=1.904 (5)\", \"tab\": \"Robustness\", \"score\": \"0.38087719298245615\"}", - "MMLU - EM (Fairness)": "{\"description\": \"min=0.24, mean=0.41, max=0.64, sum=2.049 (5)\", \"tab\": \"Fairness\", \"score\": \"0.40989473684210526\"}", - "MMLU - Denoised inference time (s)": "{\"description\": \"5 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=358.76, mean=467.936, max=612.798, sum=2339.678 (5)\", \"tab\": \"General information\", \"score\": \"467.935649122807\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "MMLU - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "BoolQ", - "source_data": { - "dataset_name": "BoolQ", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on BoolQ", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.704, - "details": { - "description": "min=0.704, mean=0.704, max=0.704, sum=0.704 (1)", - "tab": "Accuracy", - "BoolQ - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "BoolQ - EM (Robustness)": "{\"description\": \"min=0.656, mean=0.656, max=0.656, sum=0.656 (1)\", \"tab\": \"Robustness\", \"score\": \"0.656\"}", - "BoolQ - EM (Fairness)": "{\"description\": \"min=0.631, mean=0.631, max=0.631, sum=0.631 (1)\", \"tab\": \"Fairness\", \"score\": \"0.631\"}", - "BoolQ - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "BoolQ - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "BoolQ - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "BoolQ - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "BoolQ - # prompt tokens": "{\"description\": \"min=1251.897, mean=1251.897, max=1251.897, sum=1251.897 (1)\", \"tab\": \"General information\", \"score\": \"1251.897\"}", - "BoolQ - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "BoolQ - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "BoolQ - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.732, - "details": { - "description": "min=0.732, mean=0.732, max=0.732, sum=0.732 (1)", - "tab": "Accuracy", - "NarrativeQA - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "NarrativeQA - F1 (Robustness)": "{\"description\": \"min=0.584, mean=0.584, max=0.584, sum=0.584 (1)\", \"tab\": \"Robustness\", \"score\": \"0.5840358182644836\"}", - "NarrativeQA - F1 (Fairness)": "{\"description\": \"min=0.653, mean=0.653, max=0.653, sum=0.653 (1)\", \"tab\": \"Fairness\", \"score\": \"0.6525810359656932\"}", - "NarrativeQA - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=1.969, mean=1.969, max=1.969, sum=1.969 (1)\", \"tab\": \"General information\", \"score\": \"1.9690140845070423\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=1691.082, mean=1691.082, max=1691.082, sum=1691.082 (1)\", \"tab\": \"General information\", \"score\": \"1691.081690140845\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NarrativeQA - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NarrativeQA - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NarrativeQA - Stereotypes (gender)": "{\"description\": \"min=0.5, mean=0.5, max=0.5, sum=0.5 (1)\", \"tab\": \"Bias\", \"score\": \"0.5\"}", - "NarrativeQA - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NarrativeQA - Representation (gender)": "{\"description\": \"min=0.238, mean=0.238, max=0.238, sum=0.238 (1)\", \"tab\": \"Bias\", \"score\": \"0.2377049180327869\"}", - "NarrativeQA - Toxic fraction": "{\"description\": \"min=0.02, mean=0.02, max=0.02, sum=0.02 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.01971830985915493\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (open-book)", - "source_data": { - "dataset_name": "NaturalQuestions (open-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (open-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.673, - "details": { - "description": "min=0.673, mean=0.673, max=0.673, sum=0.673 (1)", - "tab": "Accuracy", - "NaturalQuestions (closed-book) - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "NaturalQuestions (open-book) - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - F1 (Robustness)": "{\"description\": \"min=0.272, mean=0.272, max=0.272, sum=0.272 (1)\", \"tab\": \"Robustness\", \"score\": \"0.2720121639433268\"}", - "NaturalQuestions (open-book) - F1 (Robustness)": "{\"description\": \"min=0.609, mean=0.609, max=0.609, sum=0.609 (1)\", \"tab\": \"Robustness\", \"score\": \"0.6094875286076354\"}", - "NaturalQuestions (closed-book) - F1 (Fairness)": "{\"description\": \"min=0.287, mean=0.287, max=0.287, sum=0.287 (1)\", \"tab\": \"Fairness\", \"score\": \"0.28717918481295357\"}", - "NaturalQuestions (open-book) - F1 (Fairness)": "{\"description\": \"min=0.624, mean=0.624, max=0.624, sum=0.624 (1)\", \"tab\": \"Fairness\", \"score\": \"0.6239999868788104\"}", - "NaturalQuestions (closed-book) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NaturalQuestions (open-book) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=117.299, mean=117.299, max=117.299, sum=117.299 (1)\", \"tab\": \"General information\", \"score\": \"117.299\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=0.999, mean=0.999, max=0.999, sum=0.999 (1)\", \"tab\": \"General information\", \"score\": \"0.999\"}", - "NaturalQuestions (closed-book) - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.704, mean=4.704, max=4.704, sum=4.704 (1)\", \"tab\": \"General information\", \"score\": \"4.704\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.037, mean=0.037, max=0.037, sum=0.037 (1)\", \"tab\": \"General information\", \"score\": \"0.037\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1495.552, mean=1495.552, max=1495.552, sum=1495.552 (1)\", \"tab\": \"General information\", \"score\": \"1495.552\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=0.993, mean=0.993, max=0.993, sum=0.993 (1)\", \"tab\": \"General information\", \"score\": \"0.993\"}", - "NaturalQuestions (open-book) - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NaturalQuestions (closed-book) - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - Representation (race)": "{\"description\": \"min=0.333, mean=0.333, max=0.333, sum=0.333 (1)\", \"tab\": \"Bias\", \"score\": \"0.3333333333333333\"}", - "NaturalQuestions (closed-book) - Representation (gender)": "{\"description\": \"min=0.088, mean=0.088, max=0.088, sum=0.088 (1)\", \"tab\": \"Bias\", \"score\": \"0.08823529411764708\"}", - "NaturalQuestions (open-book) - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=0.667 (1)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666667\"}", - "NaturalQuestions (open-book) - Stereotypes (gender)": "{\"description\": \"min=0.5, mean=0.5, max=0.5, sum=0.5 (1)\", \"tab\": \"Bias\", \"score\": \"0.5\"}", - "NaturalQuestions (open-book) - Representation (race)": "{\"description\": \"min=0.527, mean=0.527, max=0.527, sum=0.527 (1)\", \"tab\": \"Bias\", \"score\": \"0.5268817204301075\"}", - "NaturalQuestions (open-book) - Representation (gender)": "{\"description\": \"min=0.18, mean=0.18, max=0.18, sum=0.18 (1)\", \"tab\": \"Bias\", \"score\": \"0.17999999999999997\"}", - "NaturalQuestions (closed-book) - Toxic fraction": "{\"description\": \"min=0.001, mean=0.001, max=0.001, sum=0.001 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.001\"}", - "NaturalQuestions (open-book) - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "QuAC", - "source_data": { - "dataset_name": "QuAC", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on QuAC", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.393, - "details": { - "description": "min=0.393, mean=0.393, max=0.393, sum=0.393 (1)", - "tab": "Accuracy", - "QuAC - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "QuAC - F1 (Robustness)": "{\"description\": \"min=0.231, mean=0.231, max=0.231, sum=0.231 (1)\", \"tab\": \"Robustness\", \"score\": \"0.23071567735549398\"}", - "QuAC - F1 (Fairness)": "{\"description\": \"min=0.318, mean=0.318, max=0.318, sum=0.318 (1)\", \"tab\": \"Fairness\", \"score\": \"0.3176438145195143\"}", - "QuAC - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "QuAC - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "QuAC - # train": "{\"description\": \"min=0.883, mean=0.883, max=0.883, sum=0.883 (1)\", \"tab\": \"General information\", \"score\": \"0.883\"}", - "QuAC - truncated": "{\"description\": \"min=0.021, mean=0.021, max=0.021, sum=0.021 (1)\", \"tab\": \"General information\", \"score\": \"0.021\"}", - "QuAC - # prompt tokens": "{\"description\": \"min=1655.708, mean=1655.708, max=1655.708, sum=1655.708 (1)\", \"tab\": \"General information\", \"score\": \"1655.708\"}", - "QuAC - # output tokens": "{\"description\": \"min=0.997, mean=0.997, max=0.997, sum=0.997 (1)\", \"tab\": \"General information\", \"score\": \"0.997\"}", - "QuAC - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "QuAC - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=0.667 (1)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "QuAC - Stereotypes (gender)": "{\"description\": \"min=0.413, mean=0.413, max=0.413, sum=0.413 (1)\", \"tab\": \"Bias\", \"score\": \"0.4133540372670807\"}", - "QuAC - Representation (race)": "{\"description\": \"min=0.443, mean=0.443, max=0.443, sum=0.443 (1)\", \"tab\": \"Bias\", \"score\": \"0.4433656957928802\"}", - "QuAC - Representation (gender)": "{\"description\": \"min=0.279, mean=0.279, max=0.279, sum=0.279 (1)\", \"tab\": \"Bias\", \"score\": \"0.27914110429447847\"}", - "QuAC - Toxic fraction": "{\"description\": \"min=0.001, mean=0.001, max=0.001, sum=0.001 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.001\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "HellaSwag", - "source_data": { - "dataset_name": "HellaSwag", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on HellaSwag", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "HellaSwag - ECE (10-bin)": "{\"description\": \"No matching runs\", \"tab\": \"Calibration\", \"score\": \"\"}", - "HellaSwag - EM (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "HellaSwag - EM (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "HellaSwag - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "HellaSwag - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "OpenbookQA - ECE (10-bin)": "{\"description\": \"No matching runs\", \"tab\": \"Calibration\", \"score\": \"\"}", - "OpenbookQA - EM (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "OpenbookQA - EM (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "OpenbookQA - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "OpenbookQA - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "TruthfulQA", - "source_data": { - "dataset_name": "TruthfulQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on TruthfulQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.231, - "details": { - "description": "min=0.231, mean=0.231, max=0.231, sum=0.231 (1)", - "tab": "Accuracy", - "TruthfulQA - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "TruthfulQA - EM (Robustness)": "{\"description\": \"min=0.177, mean=0.177, max=0.177, sum=0.177 (1)\", \"tab\": \"Robustness\", \"score\": \"0.17737003058103976\"}", - "TruthfulQA - EM (Fairness)": "{\"description\": \"min=0.19, mean=0.19, max=0.19, sum=0.19 (1)\", \"tab\": \"Fairness\", \"score\": \"0.18960244648318042\"}", - "TruthfulQA - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "TruthfulQA - # eval": "{\"description\": \"min=654, mean=654, max=654, sum=654 (1)\", \"tab\": \"General information\", \"score\": \"654.0\"}", - "TruthfulQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "TruthfulQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "TruthfulQA - # prompt tokens": "{\"description\": \"min=505.352, mean=505.352, max=505.352, sum=505.352 (1)\", \"tab\": \"General information\", \"score\": \"505.35168195718654\"}", - "TruthfulQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "TruthfulQA - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MS MARCO (TREC)", - "source_data": { - "dataset_name": "MS MARCO (TREC)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "NDCG@10 on MS MARCO (TREC)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "MS MARCO (regular) - RR@10 (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "MS MARCO (TREC) - NDCG@10 (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "MS MARCO (regular) - RR@10 (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "MS MARCO (TREC) - NDCG@10 (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "MS MARCO (regular) - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MS MARCO (TREC) - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MS MARCO (regular) - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "MS MARCO (TREC) - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CNN/DailyMail", - "source_data": { - "dataset_name": "CNN/DailyMail", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on CNN/DailyMail", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "CNN/DailyMail - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "CNN/DailyMail - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "CNN/DailyMail - SummaC": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - QAFactEval": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - BERTScore (F1)": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - Coverage": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - Density": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - Compression": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-faithfulness": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-relevance": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-coherence": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "XSUM", - "source_data": { - "dataset_name": "XSUM", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on XSUM", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "XSUM - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "XSUM - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "XSUM - SummaC": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - QAFactEval": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - BERTScore (F1)": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - Coverage": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - Density": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - Compression": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-faithfulness": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-relevance": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-coherence": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "IMDB", - "source_data": { - "dataset_name": "IMDB", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on IMDB", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.959, - "details": { - "description": "min=0.959, mean=0.959, max=0.959, sum=0.959 (1)", - "tab": "Accuracy", - "IMDB - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "IMDB - EM (Robustness)": "{\"description\": \"min=0.942, mean=0.942, max=0.942, sum=0.942 (1)\", \"tab\": \"Robustness\", \"score\": \"0.942\"}", - "IMDB - EM (Fairness)": "{\"description\": \"min=0.955, mean=0.955, max=0.955, sum=0.955 (1)\", \"tab\": \"Fairness\", \"score\": \"0.955\"}", - "IMDB - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "IMDB - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "IMDB - # train": "{\"description\": \"min=2.911, mean=2.911, max=2.911, sum=2.911 (1)\", \"tab\": \"General information\", \"score\": \"2.911\"}", - "IMDB - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IMDB - # prompt tokens": "{\"description\": \"min=1619.568, mean=1619.568, max=1619.568, sum=1619.568 (1)\", \"tab\": \"General information\", \"score\": \"1619.568\"}", - "IMDB - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "IMDB - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "IMDB - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CivilComments", - "source_data": { - "dataset_name": "CivilComments", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on CivilComments", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.599, - "details": { - "description": "min=0.121, mean=0.599, max=0.951, sum=10.782 (18)", - "tab": "Accuracy", - "CivilComments - ECE (10-bin)": "{\"description\": \"9 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "CivilComments - EM (Robustness)": "{\"description\": \"min=0.112, mean=0.484, max=0.81, sum=8.708 (18)\", \"tab\": \"Robustness\", \"score\": \"0.4837936253587437\"}", - "CivilComments - EM (Fairness)": "{\"description\": \"min=0.073, mean=0.553, max=0.939, sum=9.947 (18)\", \"tab\": \"Fairness\", \"score\": \"0.5526050039546541\"}", - "CivilComments - Denoised inference time (s)": "{\"description\": \"9 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "CivilComments - # eval": "{\"description\": \"min=74, mean=371.556, max=683, sum=6688 (18)\", \"tab\": \"General information\", \"score\": \"371.55555555555554\"}", - "CivilComments - # train": "{\"description\": \"min=5, mean=5, max=5, sum=90 (18)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CivilComments - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (18)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CivilComments - # prompt tokens": "{\"description\": \"min=360.976, mean=771.654, max=1282.4, sum=13889.772 (18)\", \"tab\": \"General information\", \"score\": \"771.6539847352628\"}", - "CivilComments - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=18 (18)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "CivilComments - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=18 (18)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "CivilComments - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Toxic fraction": "{\"description\": \"9 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "RAFT", - "source_data": { - "dataset_name": "RAFT", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on RAFT", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.723, - "details": { - "description": "min=0.45, mean=0.723, max=0.975, sum=7.95 (11)", - "tab": "Accuracy", - "RAFT - ECE (10-bin)": "{\"description\": \"11 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "RAFT - EM (Robustness)": "{\"description\": \"min=0.075, mean=0.58, max=0.975, sum=6.375 (11)\", \"tab\": \"Robustness\", \"score\": \"0.5795454545454546\"}", - "RAFT - EM (Fairness)": "{\"description\": \"min=0.35, mean=0.68, max=0.975, sum=7.475 (11)\", \"tab\": \"Fairness\", \"score\": \"0.6795454545454546\"}", - "RAFT - Denoised inference time (s)": "{\"description\": \"11 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "RAFT - # eval": "{\"description\": \"min=40, mean=40, max=40, sum=440 (11)\", \"tab\": \"General information\", \"score\": \"40.0\"}", - "RAFT - # train": "{\"description\": \"min=0.7, mean=4.605, max=5, sum=50.65 (11)\", \"tab\": \"General information\", \"score\": \"4.6045454545454545\"}", - "RAFT - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (11)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "RAFT - # prompt tokens": "{\"description\": \"min=280.35, mean=869.691, max=1756.575, sum=9566.6 (11)\", \"tab\": \"General information\", \"score\": \"869.6909090909089\"}", - "RAFT - # output tokens": "{\"description\": \"min=0.725, mean=0.975, max=1, sum=10.725 (11)\", \"tab\": \"General information\", \"score\": \"0.975\"}", - "RAFT - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=11 (11)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "RAFT - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Toxic fraction": "{\"description\": \"11 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/mosaicml_mpt-7b.json b/data/models/mosaicml_mpt-7b.json deleted file mode 100644 index 081fad30e238789a3e00ff69da25eb37434622c5..0000000000000000000000000000000000000000 --- a/data/models/mosaicml_mpt-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "mpt-7b", - "id": "mosaicml/mpt-7b", - "developer": "mosaicml", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MPTForCausalLM", - "params_billions": "7.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/mosaicml_mpt-7b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2152 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.33 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0159 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2601 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3672 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1206 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/mosaicml_mpt-instruct-30b.json b/data/models/mosaicml_mpt-instruct-30b.json deleted file mode 100644 index 89702ba1f5de244f06b9ffa7b0338d3afea702e2..0000000000000000000000000000000000000000 --- a/data/models/mosaicml_mpt-instruct-30b.json +++ /dev/null @@ -1,674 +0,0 @@ -{ - "model_info": { - "name": "MPT-Instruct 30B", - "id": "mosaicml/MPT-Instruct-30B", - "developer": "mosaicml", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_classic/mosaicml_MPT-Instruct-30B/1774096308.339228", - "retrieved_timestamp": "1774096308.339228", - "source_metadata": { - "source_name": "helm_classic", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_classic", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_classic", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperform on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.716, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Calibration": "{\"description\": \"\", \"tab\": \"Calibration\", \"score\": \"\"}", - "Mean win rate - Robustness": "{\"description\": \"\", \"tab\": \"Robustness\", \"score\": \"0.6561072261072262\"}", - "Mean win rate - Fairness": "{\"description\": \"\", \"tab\": \"Fairness\", \"score\": \"0.6874125874125874\"}", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}", - "Mean win rate - Bias": "{\"description\": \"\", \"tab\": \"Bias\", \"score\": \"0.3616994955593857\"}", - "Mean win rate - Toxicity": "{\"description\": \"\", \"tab\": \"Toxicity\", \"score\": \"0.2453962703962704\"}", - "Mean win rate - Summarization metrics": "{\"description\": \"\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.444, - "details": { - "description": "min=0.3, mean=0.444, max=0.64, sum=2.222 (5)", - "tab": "Accuracy", - "MMLU - ECE (10-bin)": "{\"description\": \"5 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "MMLU - EM (Robustness)": "{\"description\": \"min=0.22, mean=0.383, max=0.59, sum=1.913 (5)\", \"tab\": \"Robustness\", \"score\": \"0.3826315789473684\"}", - "MMLU - EM (Fairness)": "{\"description\": \"min=0.24, mean=0.4, max=0.61, sum=2.002 (5)\", \"tab\": \"Fairness\", \"score\": \"0.40038596491228073\"}", - "MMLU - Denoised inference time (s)": "{\"description\": \"5 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=358.76, mean=467.936, max=612.798, sum=2339.678 (5)\", \"tab\": \"General information\", \"score\": \"467.935649122807\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "MMLU - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "BoolQ", - "source_data": { - "dataset_name": "BoolQ", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on BoolQ", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.85, - "details": { - "description": "min=0.85, mean=0.85, max=0.85, sum=0.85 (1)", - "tab": "Accuracy", - "BoolQ - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "BoolQ - EM (Robustness)": "{\"description\": \"min=0.77, mean=0.77, max=0.77, sum=0.77 (1)\", \"tab\": \"Robustness\", \"score\": \"0.77\"}", - "BoolQ - EM (Fairness)": "{\"description\": \"min=0.807, mean=0.807, max=0.807, sum=0.807 (1)\", \"tab\": \"Fairness\", \"score\": \"0.807\"}", - "BoolQ - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "BoolQ - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "BoolQ - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "BoolQ - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "BoolQ - # prompt tokens": "{\"description\": \"min=1251.897, mean=1251.897, max=1251.897, sum=1251.897 (1)\", \"tab\": \"General information\", \"score\": \"1251.897\"}", - "BoolQ - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "BoolQ - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "BoolQ - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.733, - "details": { - "description": "min=0.733, mean=0.733, max=0.733, sum=0.733 (1)", - "tab": "Accuracy", - "NarrativeQA - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "NarrativeQA - F1 (Robustness)": "{\"description\": \"min=0.623, mean=0.623, max=0.623, sum=0.623 (1)\", \"tab\": \"Robustness\", \"score\": \"0.6233490338408667\"}", - "NarrativeQA - F1 (Fairness)": "{\"description\": \"min=0.633, mean=0.633, max=0.633, sum=0.633 (1)\", \"tab\": \"Fairness\", \"score\": \"0.6330893045624563\"}", - "NarrativeQA - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=1.969, mean=1.969, max=1.969, sum=1.969 (1)\", \"tab\": \"General information\", \"score\": \"1.9690140845070423\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=1691.082, mean=1691.082, max=1691.082, sum=1691.082 (1)\", \"tab\": \"General information\", \"score\": \"1691.081690140845\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NarrativeQA - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NarrativeQA - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NarrativeQA - Stereotypes (gender)": "{\"description\": \"min=0.5, mean=0.5, max=0.5, sum=0.5 (1)\", \"tab\": \"Bias\", \"score\": \"0.5\"}", - "NarrativeQA - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NarrativeQA - Representation (gender)": "{\"description\": \"min=0.224, mean=0.224, max=0.224, sum=0.224 (1)\", \"tab\": \"Bias\", \"score\": \"0.22357723577235772\"}", - "NarrativeQA - Toxic fraction": "{\"description\": \"min=0.017, mean=0.017, max=0.017, sum=0.017 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.016901408450704224\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (open-book)", - "source_data": { - "dataset_name": "NaturalQuestions (open-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (open-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.697, - "details": { - "description": "min=0.697, mean=0.697, max=0.697, sum=0.697 (1)", - "tab": "Accuracy", - "NaturalQuestions (closed-book) - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "NaturalQuestions (open-book) - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - F1 (Robustness)": "{\"description\": \"min=0.202, mean=0.202, max=0.202, sum=0.202 (1)\", \"tab\": \"Robustness\", \"score\": \"0.20213849058578032\"}", - "NaturalQuestions (open-book) - F1 (Robustness)": "{\"description\": \"min=0.607, mean=0.607, max=0.607, sum=0.607 (1)\", \"tab\": \"Robustness\", \"score\": \"0.6065652552159236\"}", - "NaturalQuestions (closed-book) - F1 (Fairness)": "{\"description\": \"min=0.233, mean=0.233, max=0.233, sum=0.233 (1)\", \"tab\": \"Fairness\", \"score\": \"0.23301952773256637\"}", - "NaturalQuestions (open-book) - F1 (Fairness)": "{\"description\": \"min=0.639, mean=0.639, max=0.639, sum=0.639 (1)\", \"tab\": \"Fairness\", \"score\": \"0.6392400021633227\"}", - "NaturalQuestions (closed-book) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NaturalQuestions (open-book) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=117.299, mean=117.299, max=117.299, sum=117.299 (1)\", \"tab\": \"General information\", \"score\": \"117.299\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NaturalQuestions (closed-book) - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.704, mean=4.704, max=4.704, sum=4.704 (1)\", \"tab\": \"General information\", \"score\": \"4.704\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.037, mean=0.037, max=0.037, sum=0.037 (1)\", \"tab\": \"General information\", \"score\": \"0.037\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1495.552, mean=1495.552, max=1495.552, sum=1495.552 (1)\", \"tab\": \"General information\", \"score\": \"1495.552\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=0.994, mean=0.994, max=0.994, sum=0.994 (1)\", \"tab\": \"General information\", \"score\": \"0.994\"}", - "NaturalQuestions (open-book) - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NaturalQuestions (closed-book) - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - Stereotypes (gender)": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"Bias\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - Representation (race)": "{\"description\": \"min=0.542, mean=0.542, max=0.542, sum=0.542 (1)\", \"tab\": \"Bias\", \"score\": \"0.5416666666666667\"}", - "NaturalQuestions (closed-book) - Representation (gender)": "{\"description\": \"min=0.091, mean=0.091, max=0.091, sum=0.091 (1)\", \"tab\": \"Bias\", \"score\": \"0.09090909090909088\"}", - "NaturalQuestions (open-book) - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=0.667 (1)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "NaturalQuestions (open-book) - Stereotypes (gender)": "{\"description\": \"min=0.5, mean=0.5, max=0.5, sum=0.5 (1)\", \"tab\": \"Bias\", \"score\": \"0.5\"}", - "NaturalQuestions (open-book) - Representation (race)": "{\"description\": \"min=0.493, mean=0.493, max=0.493, sum=0.493 (1)\", \"tab\": \"Bias\", \"score\": \"0.4931129476584022\"}", - "NaturalQuestions (open-book) - Representation (gender)": "{\"description\": \"min=0.286, mean=0.286, max=0.286, sum=0.286 (1)\", \"tab\": \"Bias\", \"score\": \"0.2857142857142857\"}", - "NaturalQuestions (closed-book) - Toxic fraction": "{\"description\": \"min=0.001, mean=0.001, max=0.001, sum=0.001 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.001\"}", - "NaturalQuestions (open-book) - Toxic fraction": "{\"description\": \"min=0.001, mean=0.001, max=0.001, sum=0.001 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.001\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "QuAC", - "source_data": { - "dataset_name": "QuAC", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on QuAC", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.327, - "details": { - "description": "min=0.327, mean=0.327, max=0.327, sum=0.327 (1)", - "tab": "Accuracy", - "QuAC - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "QuAC - F1 (Robustness)": "{\"description\": \"min=0.204, mean=0.204, max=0.204, sum=0.204 (1)\", \"tab\": \"Robustness\", \"score\": \"0.20366013650654988\"}", - "QuAC - F1 (Fairness)": "{\"description\": \"min=0.252, mean=0.252, max=0.252, sum=0.252 (1)\", \"tab\": \"Fairness\", \"score\": \"0.2519147363869601\"}", - "QuAC - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "QuAC - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "QuAC - # train": "{\"description\": \"min=0.883, mean=0.883, max=0.883, sum=0.883 (1)\", \"tab\": \"General information\", \"score\": \"0.883\"}", - "QuAC - truncated": "{\"description\": \"min=0.021, mean=0.021, max=0.021, sum=0.021 (1)\", \"tab\": \"General information\", \"score\": \"0.021\"}", - "QuAC - # prompt tokens": "{\"description\": \"min=1655.708, mean=1655.708, max=1655.708, sum=1655.708 (1)\", \"tab\": \"General information\", \"score\": \"1655.708\"}", - "QuAC - # output tokens": "{\"description\": \"min=0.998, mean=0.998, max=0.998, sum=0.998 (1)\", \"tab\": \"General information\", \"score\": \"0.998\"}", - "QuAC - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "QuAC - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=0.667 (1)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666667\"}", - "QuAC - Stereotypes (gender)": "{\"description\": \"min=0.426, mean=0.426, max=0.426, sum=0.426 (1)\", \"tab\": \"Bias\", \"score\": \"0.42553763440860215\"}", - "QuAC - Representation (race)": "{\"description\": \"min=0.407, mean=0.407, max=0.407, sum=0.407 (1)\", \"tab\": \"Bias\", \"score\": \"0.4074074074074074\"}", - "QuAC - Representation (gender)": "{\"description\": \"min=0.232, mean=0.232, max=0.232, sum=0.232 (1)\", \"tab\": \"Bias\", \"score\": \"0.23239436619718312\"}", - "QuAC - Toxic fraction": "{\"description\": \"min=0.003, mean=0.003, max=0.003, sum=0.003 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.003\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "HellaSwag", - "source_data": { - "dataset_name": "HellaSwag", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on HellaSwag", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "HellaSwag - ECE (10-bin)": "{\"description\": \"No matching runs\", \"tab\": \"Calibration\", \"score\": \"\"}", - "HellaSwag - EM (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "HellaSwag - EM (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "HellaSwag - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "HellaSwag - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "OpenbookQA - ECE (10-bin)": "{\"description\": \"No matching runs\", \"tab\": \"Calibration\", \"score\": \"\"}", - "OpenbookQA - EM (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "OpenbookQA - EM (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "OpenbookQA - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "OpenbookQA - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "TruthfulQA", - "source_data": { - "dataset_name": "TruthfulQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on TruthfulQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.234, - "details": { - "description": "min=0.234, mean=0.234, max=0.234, sum=0.234 (1)", - "tab": "Accuracy", - "TruthfulQA - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "TruthfulQA - EM (Robustness)": "{\"description\": \"min=0.177, mean=0.177, max=0.177, sum=0.177 (1)\", \"tab\": \"Robustness\", \"score\": \"0.17737003058103976\"}", - "TruthfulQA - EM (Fairness)": "{\"description\": \"min=0.18, mean=0.18, max=0.18, sum=0.18 (1)\", \"tab\": \"Fairness\", \"score\": \"0.18042813455657492\"}", - "TruthfulQA - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "TruthfulQA - # eval": "{\"description\": \"min=654, mean=654, max=654, sum=654 (1)\", \"tab\": \"General information\", \"score\": \"654.0\"}", - "TruthfulQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "TruthfulQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "TruthfulQA - # prompt tokens": "{\"description\": \"min=505.352, mean=505.352, max=505.352, sum=505.352 (1)\", \"tab\": \"General information\", \"score\": \"505.35168195718654\"}", - "TruthfulQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "TruthfulQA - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MS MARCO (TREC)", - "source_data": { - "dataset_name": "MS MARCO (TREC)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "NDCG@10 on MS MARCO (TREC)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "MS MARCO (regular) - RR@10 (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "MS MARCO (TREC) - NDCG@10 (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "MS MARCO (regular) - RR@10 (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "MS MARCO (TREC) - NDCG@10 (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "MS MARCO (regular) - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MS MARCO (TREC) - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MS MARCO (regular) - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "MS MARCO (TREC) - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CNN/DailyMail", - "source_data": { - "dataset_name": "CNN/DailyMail", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on CNN/DailyMail", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "CNN/DailyMail - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "CNN/DailyMail - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "CNN/DailyMail - SummaC": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - QAFactEval": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - BERTScore (F1)": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - Coverage": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - Density": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - Compression": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-faithfulness": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-relevance": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-coherence": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "XSUM", - "source_data": { - "dataset_name": "XSUM", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on XSUM", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "XSUM - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "XSUM - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "XSUM - SummaC": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - QAFactEval": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - BERTScore (F1)": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - Coverage": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - Density": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - Compression": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-faithfulness": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-relevance": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-coherence": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "IMDB", - "source_data": { - "dataset_name": "IMDB", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on IMDB", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.956, - "details": { - "description": "min=0.956, mean=0.956, max=0.956, sum=0.956 (1)", - "tab": "Accuracy", - "IMDB - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "IMDB - EM (Robustness)": "{\"description\": \"min=0.942, mean=0.942, max=0.942, sum=0.942 (1)\", \"tab\": \"Robustness\", \"score\": \"0.942\"}", - "IMDB - EM (Fairness)": "{\"description\": \"min=0.944, mean=0.944, max=0.944, sum=0.944 (1)\", \"tab\": \"Fairness\", \"score\": \"0.944\"}", - "IMDB - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "IMDB - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "IMDB - # train": "{\"description\": \"min=2.911, mean=2.911, max=2.911, sum=2.911 (1)\", \"tab\": \"General information\", \"score\": \"2.911\"}", - "IMDB - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IMDB - # prompt tokens": "{\"description\": \"min=1619.568, mean=1619.568, max=1619.568, sum=1619.568 (1)\", \"tab\": \"General information\", \"score\": \"1619.568\"}", - "IMDB - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "IMDB - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "IMDB - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CivilComments", - "source_data": { - "dataset_name": "CivilComments", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on CivilComments", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.573, - "details": { - "description": "min=0.119, mean=0.573, max=0.967, sum=10.316 (18)", - "tab": "Accuracy", - "CivilComments - ECE (10-bin)": "{\"description\": \"9 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "CivilComments - EM (Robustness)": "{\"description\": \"min=0.042, mean=0.408, max=0.867, sum=7.353 (18)\", \"tab\": \"Robustness\", \"score\": \"0.40848129232892094\"}", - "CivilComments - EM (Fairness)": "{\"description\": \"min=0.085, mean=0.527, max=0.95, sum=9.488 (18)\", \"tab\": \"Fairness\", \"score\": \"0.5271340155324973\"}", - "CivilComments - Denoised inference time (s)": "{\"description\": \"9 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "CivilComments - # eval": "{\"description\": \"min=74, mean=371.556, max=683, sum=6688 (18)\", \"tab\": \"General information\", \"score\": \"371.55555555555554\"}", - "CivilComments - # train": "{\"description\": \"min=5, mean=5, max=5, sum=90 (18)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CivilComments - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (18)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CivilComments - # prompt tokens": "{\"description\": \"min=360.976, mean=771.654, max=1282.4, sum=13889.772 (18)\", \"tab\": \"General information\", \"score\": \"771.6539847352628\"}", - "CivilComments - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=18 (18)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "CivilComments - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=18 (18)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "CivilComments - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Toxic fraction": "{\"description\": \"9 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "RAFT", - "source_data": { - "dataset_name": "RAFT", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on RAFT", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.68, - "details": { - "description": "min=0.425, mean=0.68, max=0.9, sum=7.475 (11)", - "tab": "Accuracy", - "RAFT - ECE (10-bin)": "{\"description\": \"11 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "RAFT - EM (Robustness)": "{\"description\": \"min=0.075, mean=0.548, max=0.875, sum=6.025 (11)\", \"tab\": \"Robustness\", \"score\": \"0.5477272727272727\"}", - "RAFT - EM (Fairness)": "{\"description\": \"min=0.4, mean=0.636, max=0.825, sum=7 (11)\", \"tab\": \"Fairness\", \"score\": \"0.6363636363636364\"}", - "RAFT - Denoised inference time (s)": "{\"description\": \"11 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "RAFT - # eval": "{\"description\": \"min=40, mean=40, max=40, sum=440 (11)\", \"tab\": \"General information\", \"score\": \"40.0\"}", - "RAFT - # train": "{\"description\": \"min=0.7, mean=4.605, max=5, sum=50.65 (11)\", \"tab\": \"General information\", \"score\": \"4.6045454545454545\"}", - "RAFT - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (11)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "RAFT - # prompt tokens": "{\"description\": \"min=280.35, mean=869.691, max=1756.575, sum=9566.6 (11)\", \"tab\": \"General information\", \"score\": \"869.6909090909089\"}", - "RAFT - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=11 (11)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "RAFT - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=11 (11)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "RAFT - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Toxic fraction": "{\"description\": \"11 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/mosama_qwen2.5-1.5b-instruct-cot-reflection.json b/data/models/mosama_qwen2.5-1.5b-instruct-cot-reflection.json deleted file mode 100644 index f64e644b803de7faa705e1d9e901805d0056c95d..0000000000000000000000000000000000000000 --- a/data/models/mosama_qwen2.5-1.5b-instruct-cot-reflection.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-1.5B-Instruct-CoT-Reflection", - "id": "mosama/Qwen2.5-1.5B-Instruct-CoT-Reflection", - "developer": "mosama", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.544" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/mosama_Qwen2.5-1.5B-Instruct-CoT-Reflection/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.287 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4109 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0272 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2617 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3212 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2651 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/mostafa8mehrabi_llama-3.2-1b-insomnia-chatbot-merged.json b/data/models/mostafa8mehrabi_llama-3.2-1b-insomnia-chatbot-merged.json deleted file mode 100644 index 9be9b83a929a59dc9e2670264d59b666698f8306..0000000000000000000000000000000000000000 --- a/data/models/mostafa8mehrabi_llama-3.2-1b-insomnia-chatbot-merged.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "llama-3.2-1b-Insomnia-ChatBot-merged", - "id": "Mostafa8Mehrabi/llama-3.2-1b-Insomnia-ChatBot-merged", - "developer": "Mostafa8Mehrabi", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.236" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Mostafa8Mehrabi_llama-3.2-1b-Insomnia-ChatBot-merged/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1321 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3004 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0076 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2366 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3382 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1131 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/mrdayl_opencogito.json b/data/models/mrdayl_opencogito.json deleted file mode 100644 index 41185fb69508ee342cdefb23a65fb8f90051d184..0000000000000000000000000000000000000000 --- a/data/models/mrdayl_opencogito.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "OpenCogito", - "id": "mrdayl/OpenCogito", - "developer": "mrdayl", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.086" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/mrdayl_OpenCogito/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3934 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.472 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2183 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3003 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.424 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3452 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/mrdayl_opencognito-r1.json b/data/models/mrdayl_opencognito-r1.json deleted file mode 100644 index 46eec50c5b02b1152eb0a601763e750ea00d39d7..0000000000000000000000000000000000000000 --- a/data/models/mrdayl_opencognito-r1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "OpenCognito-r1", - "id": "mrdayl/OpenCognito-r1", - "developer": "mrdayl", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.086" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/mrdayl_OpenCognito-r1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4241 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4673 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1903 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2995 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4241 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3475 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/mrdayl_opencognito-r2.json b/data/models/mrdayl_opencognito-r2.json deleted file mode 100644 index ec25fc6bb184119a0fe7b3dde43606e9cf96282d..0000000000000000000000000000000000000000 --- a/data/models/mrdayl_opencognito-r2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "OpenCognito-r2", - "id": "mrdayl/OpenCognito-r2", - "developer": "mrdayl", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.086" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/mrdayl_OpenCognito-r2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3959 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4688 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2024 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3062 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4202 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3462 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/mrdayl_opencognito.json b/data/models/mrdayl_opencognito.json deleted file mode 100644 index 62879873337bad2c727f2bfe52fe3a854c4986a1..0000000000000000000000000000000000000000 --- a/data/models/mrdayl_opencognito.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "OpenCognito", - "id": "mrdayl/OpenCognito", - "developer": "mrdayl", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.086" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/mrdayl_OpenCognito/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4062 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4706 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2115 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2978 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4293 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3443 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/mrdayl_openthink.json b/data/models/mrdayl_openthink.json deleted file mode 100644 index 4cc1646e00c9c84dc43bc2508a11e86695243e3f..0000000000000000000000000000000000000000 --- a/data/models/mrdayl_openthink.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "OpenThink", - "id": "mrdayl/OpenThink", - "developer": "mrdayl", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.777" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/mrdayl_OpenThink/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2054 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.346 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2885 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2827 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3289 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.185 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/mrm8488_phi-4-14b-grpo-gsm8k-3e.json b/data/models/mrm8488_phi-4-14b-grpo-gsm8k-3e.json deleted file mode 100644 index c7d7aeed75d4c359f94f4f51302a2e5e6badc691..0000000000000000000000000000000000000000 --- a/data/models/mrm8488_phi-4-14b-grpo-gsm8k-3e.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "phi-4-14B-grpo-gsm8k-3e", - "id": "mrm8488/phi-4-14B-grpo-gsm8k-3e", - "developer": "mrm8488", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/mrm8488_phi-4-14B-grpo-gsm8k-3e/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6885 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6805 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4524 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3356 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3994 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5268 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/mrm8488_phi-4-14b-grpo-limo.json b/data/models/mrm8488_phi-4-14b-grpo-limo.json deleted file mode 100644 index 7580ff9066c05e3938e78cc354a67422edfeafa0..0000000000000000000000000000000000000000 --- a/data/models/mrm8488_phi-4-14b-grpo-limo.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "phi-4-14B-grpo-limo", - "id": "mrm8488/phi-4-14B-grpo-limo", - "developer": "mrm8488", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/mrm8488_phi-4-14B-grpo-limo/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6812 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6785 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4569 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3364 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3981 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5261 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/mrrobotoai_mrroboto-prolong-8b-v4i.json b/data/models/mrrobotoai_mrroboto-prolong-8b-v4i.json deleted file mode 100644 index dc443c52f80bb86df7ed43a6a2c5561d307202a7..0000000000000000000000000000000000000000 --- a/data/models/mrrobotoai_mrroboto-prolong-8b-v4i.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MrRoboto-ProLong-8b-v4i", - "id": "MrRobotoAI/MrRoboto-ProLong-8b-v4i", - "developer": "MrRobotoAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "4.015" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/MrRobotoAI_MrRoboto-ProLong-8b-v4i/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3835 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4585 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0551 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2894 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4014 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3068 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/mrrobotoai_mrroboto-prolongbase-pt8-unaligned-8b.json b/data/models/mrrobotoai_mrroboto-prolongbase-pt8-unaligned-8b.json deleted file mode 100644 index c415791d3ff29e54421fe48f812c46c92e553aca..0000000000000000000000000000000000000000 --- a/data/models/mrrobotoai_mrroboto-prolongbase-pt8-unaligned-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MrRoboto-ProLongBASE-pt8-unaligned-8b", - "id": "MrRobotoAI/MrRoboto-ProLongBASE-pt8-unaligned-8b", - "developer": "MrRobotoAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "4.015" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/MrRobotoAI_MrRoboto-ProLongBASE-pt8-unaligned-8b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3475 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4515 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0423 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.281 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4279 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2566 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/mtsair_cotype-nano.json b/data/models/mtsair_cotype-nano.json deleted file mode 100644 index f5e184cdb1c95ccbbde59a253e1390369e79edf5..0000000000000000000000000000000000000000 --- a/data/models/mtsair_cotype-nano.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Cotype-Nano", - "id": "MTSAIR/Cotype-Nano", - "developer": "MTSAIR", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.544" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/MTSAIR_Cotype-Nano/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3748 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3865 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0974 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2701 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3289 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2477 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/mtsair_multiverse_70b.json b/data/models/mtsair_multiverse_70b.json deleted file mode 100644 index 4530f5a4bee373dc14733734f77fce73461bcf3f..0000000000000000000000000000000000000000 --- a/data/models/mtsair_multiverse_70b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MultiVerse_70B", - "id": "MTSAIR/MultiVerse_70B", - "developer": "MTSAIR", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "72.289" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/MTSAIR_MultiVerse_70B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5249 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6183 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1926 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.354 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.474 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.486 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/mukaj_llama-3.1-hawkish-8b.json b/data/models/mukaj_llama-3.1-hawkish-8b.json deleted file mode 100644 index 5b86fa7009375b1e4f371aa24e8805d5b75c54c4..0000000000000000000000000000000000000000 --- a/data/models/mukaj_llama-3.1-hawkish-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.1-Hawkish-8B", - "id": "mukaj/Llama-3.1-Hawkish-8B", - "developer": "mukaj", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/mukaj_Llama-3.1-Hawkish-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.672 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4884 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2432 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2903 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3967 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3331 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/multiple_multiple.json b/data/models/multiple_multiple.json deleted file mode 100644 index 6b9f8770189364e0033af353a08b8bbfc70500f9..0000000000000000000000000000000000000000 --- a/data/models/multiple_multiple.json +++ /dev/null @@ -1,457 +0,0 @@ -{ - "model_info": { - "name": "Multiple", - "id": "multiple/multiple", - "developer": "Multiple", - "additional_details": { - "agent_name": "Abacus AI Desktop", - "agent_organization": "Abacus.AI" - } - }, - "evaluations": [ - { - "evaluation_id": "terminal-bench-2.0/abacus-ai-desktop__multiple/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2025-12-11", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 58.4, - "uncertainty": { - "standard_error": { - "value": 2.8 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Abacus AI Desktop\" -m \"Multiple\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Abacus AI Desktop\" -m \"Multiple\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - }, - { - "evaluation_id": "terminal-bench-2.0/junie-cli__multiple/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2026-03-07", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 71.0, - "uncertainty": { - "standard_error": { - "value": 2.9 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Junie CLI\" -m \"Multiple\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Junie CLI\" -m \"Multiple\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - }, - { - "evaluation_id": "terminal-bench-2.0/warp__multiple/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2025-11-11", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 50.1, - "uncertainty": { - "standard_error": { - "value": 2.7 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Warp\" -m \"Multiple\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Warp\" -m \"Multiple\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - }, - { - "evaluation_id": "terminal-bench-2.0/warp__multiple/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2025-12-12", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 61.2, - "uncertainty": { - "standard_error": { - "value": 3.0 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Warp\" -m \"Multiple\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Warp\" -m \"Multiple\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - }, - { - "evaluation_id": "terminal-bench-2.0/ob-1__multiple/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2026-03-05", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 72.4, - "uncertainty": { - "standard_error": { - "value": 2.3 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"OB-1\" -m \"Multiple\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"OB-1\" -m \"Multiple\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - }, - { - "evaluation_id": "terminal-bench-2.0/warp__multiple/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2025-11-20", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 59.1, - "uncertainty": { - "standard_error": { - "value": 2.8 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Warp\" -m \"Multiple\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Warp\" -m \"Multiple\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/multivexai_gladiator-mini-exp-1211-3b.json b/data/models/multivexai_gladiator-mini-exp-1211-3b.json deleted file mode 100644 index a7953cefb84aacb3875098421f0b85b6ba461384..0000000000000000000000000000000000000000 --- a/data/models/multivexai_gladiator-mini-exp-1211-3b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Gladiator-Mini-Exp-1211-3B", - "id": "MultivexAI/Gladiator-Mini-Exp-1211-3B", - "developer": "MultivexAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/MultivexAI_Gladiator-Mini-Exp-1211-3B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6876 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4484 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1375 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2727 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.326 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3152 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/multivexai_gladiator-mini-exp-1221-3b-instruct-v2.json b/data/models/multivexai_gladiator-mini-exp-1221-3b-instruct-v2.json deleted file mode 100644 index 576667dc3a6664ca9aa480aa4c48cfca3d3f98ac..0000000000000000000000000000000000000000 --- a/data/models/multivexai_gladiator-mini-exp-1221-3b-instruct-v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Gladiator-Mini-Exp-1221-3B-Instruct-V2", - "id": "MultivexAI/Gladiator-Mini-Exp-1221-3B-Instruct-V2", - "developer": "MultivexAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/MultivexAI_Gladiator-Mini-Exp-1221-3B-Instruct-V2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6215 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4389 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1412 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2634 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3008 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3025 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/multivexai_gladiator-mini-exp-1221-3b-instruct.json b/data/models/multivexai_gladiator-mini-exp-1221-3b-instruct.json deleted file mode 100644 index 1c46a4eb6b26e5f2df1ae160f0c4d4380be8978e..0000000000000000000000000000000000000000 --- a/data/models/multivexai_gladiator-mini-exp-1221-3b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Gladiator-Mini-Exp-1221-3B-Instruct", - "id": "MultivexAI/Gladiator-Mini-Exp-1221-3B-Instruct", - "developer": "MultivexAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/MultivexAI_Gladiator-Mini-Exp-1221-3B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6079 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.437 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1352 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2634 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3115 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3049 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/multivexai_gladiator-mini-exp-1222-3b-instruct.json b/data/models/multivexai_gladiator-mini-exp-1222-3b-instruct.json deleted file mode 100644 index 75af6383e9644c99aeb02dae6ca651f3efd0800a..0000000000000000000000000000000000000000 --- a/data/models/multivexai_gladiator-mini-exp-1222-3b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Gladiator-Mini-Exp-1222-3B-Instruct", - "id": "MultivexAI/Gladiator-Mini-Exp-1222-3B-Instruct", - "developer": "MultivexAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/MultivexAI_Gladiator-Mini-Exp-1222-3B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6163 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4373 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1412 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2634 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3128 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3017 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/multivexai_phi-3.5-mini-instruct-multivex-v0.25-gguf.json b/data/models/multivexai_phi-3.5-mini-instruct-multivex-v0.25-gguf.json deleted file mode 100644 index 457beaab5449bd3663d82922ea63af80430d9e0c..0000000000000000000000000000000000000000 --- a/data/models/multivexai_phi-3.5-mini-instruct-multivex-v0.25-gguf.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Phi-3.5-Mini-Instruct-MultiVex-v0.25-GGUF", - "id": "MultivexAI/Phi-3.5-Mini-Instruct-MultiVex-v0.25-GGUF", - "developer": "MultivexAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.821" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/MultivexAI_Phi-3.5-Mini-Instruct-MultiVex-v0.25-GGUF/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.144 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2908 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.006 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.255 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3642 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1109 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/mxode_nanolm-0.3b-instruct-v1.1.json b/data/models/mxode_nanolm-0.3b-instruct-v1.1.json deleted file mode 100644 index 69a0b970877e78f06b2a9fc3352f38590daf9200..0000000000000000000000000000000000000000 --- a/data/models/mxode_nanolm-0.3b-instruct-v1.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "NanoLM-0.3B-Instruct-v1.1", - "id": "Mxode/NanoLM-0.3B-Instruct-v1.1", - "developer": "Mxode", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.315" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Mxode_NanoLM-0.3B-Instruct-v1.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1783 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3014 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0136 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.25 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4273 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1121 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/mxode_nanolm-0.3b-instruct-v1.json b/data/models/mxode_nanolm-0.3b-instruct-v1.json deleted file mode 100644 index db9b8d11b7c2d7b17df1cdd0b297eeb39fa2aa60..0000000000000000000000000000000000000000 --- a/data/models/mxode_nanolm-0.3b-instruct-v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "NanoLM-0.3B-Instruct-v1", - "id": "Mxode/NanoLM-0.3B-Instruct-v1", - "developer": "Mxode", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.315" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Mxode_NanoLM-0.3B-Instruct-v1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1537 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3028 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0144 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2718 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4155 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1105 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/mxode_nanolm-0.3b-instruct-v2.json b/data/models/mxode_nanolm-0.3b-instruct-v2.json deleted file mode 100644 index d00349c539b71b76664df72ba23767a22544b1dc..0000000000000000000000000000000000000000 --- a/data/models/mxode_nanolm-0.3b-instruct-v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "NanoLM-0.3B-Instruct-v2", - "id": "Mxode/NanoLM-0.3B-Instruct-v2", - "developer": "Mxode", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.315" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Mxode_NanoLM-0.3B-Instruct-v2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1668 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2921 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0068 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2609 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3955 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1134 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/mxode_nanolm-1b-instruct-v1.1.json b/data/models/mxode_nanolm-1b-instruct-v1.1.json deleted file mode 100644 index 3bc11129cf7f654e8d19e17f6f5bf7dfbc7ac804..0000000000000000000000000000000000000000 --- a/data/models/mxode_nanolm-1b-instruct-v1.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "NanoLM-1B-Instruct-v1.1", - "id": "Mxode/NanoLM-1B-Instruct-v1.1", - "developer": "Mxode", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.076" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Mxode_NanoLM-1B-Instruct-v1.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2395 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3184 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0363 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2634 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3433 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1215 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/mxode_nanolm-1b-instruct-v2.json b/data/models/mxode_nanolm-1b-instruct-v2.json deleted file mode 100644 index 481b2e6a4902b3670351b98f665658989997bba6..0000000000000000000000000000000000000000 --- a/data/models/mxode_nanolm-1b-instruct-v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "NanoLM-1B-Instruct-v2", - "id": "Mxode/NanoLM-1B-Instruct-v2", - "developer": "Mxode", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.076" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Mxode_NanoLM-1B-Instruct-v2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.263 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3123 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0415 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2634 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3552 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1238 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/my_model.json b/data/models/my_model.json deleted file mode 100644 index c412a10b10c0a4f94c02df9cd9e3ebb29ab6fd09..0000000000000000000000000000000000000000 --- a/data/models/my_model.json +++ /dev/null @@ -1,126 +0,0 @@ -{ - "model_info": { - "name": "my_model/", - "id": "my_model/", - "developer": "my_model", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/my_model_/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5267 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4553 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5592 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4392 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6532 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nanbeige_nanbeige3-5-pro-thinking-fc.json b/data/models/nanbeige_nanbeige3-5-pro-thinking-fc.json deleted file mode 100644 index 4bcdedfabbfe79f738a2744f6243e150349f4e2c..0000000000000000000000000000000000000000 --- a/data/models/nanbeige_nanbeige3-5-pro-thinking-fc.json +++ /dev/null @@ -1,851 +0,0 @@ -{ - "model_info": { - "name": "Nanbeige3.5-Pro-Thinking (FC)", - "id": "nanbeige/nanbeige3-5-pro-thinking-fc", - "developer": "nanbeige", - "additional_details": { - "raw_model_name": "Nanbeige3.5-Pro-Thinking (FC)", - "organization": "Nanbeige", - "license": "apache-2.0", - "mode": "FC", - "model_link": "https://huggingface.co/Nanbeige" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/nanbeige/nanbeige3-5-pro-thinking-fc/1775236112.38179", - "retrieved_timestamp": "1775236112.38179", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 32.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 47.68 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 23.46 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 21.12 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 28.61 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 63.29 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 38.35 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 43.92 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 36.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 53.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 20.0 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 69.95 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 63.18 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 71.42 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 87.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 66.67 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 40.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 56.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 34.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 29.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 41.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 42.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 47.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 37.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 45.16 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 38.06 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 58.06 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 39.35 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 100.0 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 74.2 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nanbeige_nanbeige4-3b-thinking-2511-fc.json b/data/models/nanbeige_nanbeige4-3b-thinking-2511-fc.json deleted file mode 100644 index 5ce4af7098af7a74d9a409584c5ebb89e62ac48e..0000000000000000000000000000000000000000 --- a/data/models/nanbeige_nanbeige4-3b-thinking-2511-fc.json +++ /dev/null @@ -1,851 +0,0 @@ -{ - "model_info": { - "name": "Nanbeige4-3B-Thinking-2511 (FC)", - "id": "nanbeige/nanbeige4-3b-thinking-2511-fc", - "developer": "nanbeige", - "additional_details": { - "raw_model_name": "Nanbeige4-3B-Thinking-2511 (FC)", - "organization": "Nanbeige", - "license": "apache-2.0", - "mode": "FC", - "model_link": "https://huggingface.co/Nanbeige/Nanbeige4-3B-Thinking-2511" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/nanbeige/nanbeige4-3b-thinking-2511-fc/1775236112.3783529", - "retrieved_timestamp": "1775236112.3783529", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 25.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 51.4 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 14.14 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 13.46 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 26.41 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 37.45 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 81.58 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 63.83 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 93.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 84.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 84.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 79.42 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 86.05 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 78.06 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 75.0 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 70.83 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 51.12 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 58.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 54.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 45.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 47.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 21.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 31.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 12.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 36.77 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 31.61 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 34.19 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 44.52 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 75.0 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 83.09 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/naps-ai_naps-gemma-2-27b-v-0.1.0.json b/data/models/naps-ai_naps-gemma-2-27b-v-0.1.0.json deleted file mode 100644 index e9ba4408dddde57b38f02424ae1e528ecaa4d90f..0000000000000000000000000000000000000000 --- a/data/models/naps-ai_naps-gemma-2-27b-v-0.1.0.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "naps-gemma-2-27b-v-0.1.0", - "id": "NAPS-ai/naps-gemma-2-27b-v-0.1.0", - "developer": "NAPS-ai", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "27.227" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/NAPS-ai_naps-gemma-2-27b-v-0.1.0/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2912 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2601 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3575 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1168 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/naps-ai_naps-gemma-2-27b-v0.1.0.json b/data/models/naps-ai_naps-gemma-2-27b-v0.1.0.json deleted file mode 100644 index 56cae4f843cf956786e9e45c6cfc0e227e1a9d67..0000000000000000000000000000000000000000 --- a/data/models/naps-ai_naps-gemma-2-27b-v0.1.0.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "naps-gemma-2-27b-v0.1.0", - "id": "NAPS-ai/naps-gemma-2-27b-v0.1.0", - "developer": "NAPS-ai", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "27.227" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/NAPS-ai_naps-gemma-2-27b-v0.1.0/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2912 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2601 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3575 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1168 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/naps-ai_naps-llama-3_1-8b-instruct-v0.3.json b/data/models/naps-ai_naps-llama-3_1-8b-instruct-v0.3.json deleted file mode 100644 index c45c0edea5ac0ff0c82908dd8da0b1ca35886a79..0000000000000000000000000000000000000000 --- a/data/models/naps-ai_naps-llama-3_1-8b-instruct-v0.3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "naps-llama-3_1-8b-instruct-v0.3", - "id": "NAPS-ai/naps-llama-3_1-8b-instruct-v0.3", - "developer": "NAPS-ai", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/NAPS-ai_naps-llama-3_1-8b-instruct-v0.3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5391 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4901 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1903 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2995 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3787 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3398 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/naps-ai_naps-llama-3_1-8b-instruct-v0.4.json b/data/models/naps-ai_naps-llama-3_1-8b-instruct-v0.4.json deleted file mode 100644 index 5c09de0120777589540639e3a812d6b493f8f40d..0000000000000000000000000000000000000000 --- a/data/models/naps-ai_naps-llama-3_1-8b-instruct-v0.4.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "naps-llama-3_1-8b-instruct-v0.4", - "id": "NAPS-ai/naps-llama-3_1-8b-instruct-v0.4", - "developer": "NAPS-ai", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/NAPS-ai_naps-llama-3_1-8b-instruct-v0.4/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7344 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4862 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1964 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2794 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4421 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3475 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/naps-ai_naps-llama-3_1-instruct-v0.5.0.json b/data/models/naps-ai_naps-llama-3_1-instruct-v0.5.0.json deleted file mode 100644 index 6da1fdac5f745db958fabd4a47829c0fad6976f7..0000000000000000000000000000000000000000 --- a/data/models/naps-ai_naps-llama-3_1-instruct-v0.5.0.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "naps-llama-3_1-instruct-v0.5.0", - "id": "NAPS-ai/naps-llama-3_1-instruct-v0.5.0", - "developer": "NAPS-ai", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/NAPS-ai_naps-llama-3_1-instruct-v0.5.0/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.502 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4148 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0363 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2685 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3713 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2614 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/naps-ai_naps-llama-3_1_instruct-v0.6.0.json b/data/models/naps-ai_naps-llama-3_1_instruct-v0.6.0.json deleted file mode 100644 index 2204d3011e03f231eb2b899e6c11f23bc998d164..0000000000000000000000000000000000000000 --- a/data/models/naps-ai_naps-llama-3_1_instruct-v0.6.0.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "naps-llama-3_1_instruct-v0.6.0", - "id": "NAPS-ai/naps-llama-3_1_instruct-v0.6.0", - "developer": "NAPS-ai", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/NAPS-ai_naps-llama-3_1_instruct-v0.6.0/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.328 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4528 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0642 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2819 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3739 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3241 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/naps-ai_naps-llama3.1-70b-v0.2-fp16.json b/data/models/naps-ai_naps-llama3.1-70b-v0.2-fp16.json deleted file mode 100644 index 43ce67fc0fd49c3be23feeb756d0f4a99458e0e2..0000000000000000000000000000000000000000 --- a/data/models/naps-ai_naps-llama3.1-70b-v0.2-fp16.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "naps-llama3.1-70B-v0.2-fp16", - "id": "NAPS-ai/naps-llama3.1-70B-v0.2-fp16", - "developer": "NAPS-ai", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "70.761" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/NAPS-ai_naps-llama3.1-70B-v0.2-fp16/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1845 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3041 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2391 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3486 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1099 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/natong19_mistral-nemo-instruct-2407-abliterated.json b/data/models/natong19_mistral-nemo-instruct-2407-abliterated.json deleted file mode 100644 index b3541cf4174624ca06c3a8ed29dc62b3dddc343a..0000000000000000000000000000000000000000 --- a/data/models/natong19_mistral-nemo-instruct-2407-abliterated.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mistral-Nemo-Instruct-2407-abliterated", - "id": "natong19/Mistral-Nemo-Instruct-2407-abliterated", - "developer": "natong19", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/natong19_Mistral-Nemo-Instruct-2407-abliterated/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6392 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5048 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1322 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2869 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4033 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3518 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/natong19_qwen2-7b-instruct-abliterated.json b/data/models/natong19_qwen2-7b-instruct-abliterated.json deleted file mode 100644 index 27b44a59568c94e1ab7e670d14f6b8e7048ba146..0000000000000000000000000000000000000000 --- a/data/models/natong19_qwen2-7b-instruct-abliterated.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2-7B-Instruct-abliterated", - "id": "natong19/Qwen2-7B-Instruct-abliterated", - "developer": "natong19", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/natong19_Qwen2-7B-Instruct-abliterated/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5837 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5553 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2764 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3012 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4034 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3842 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/naveenpoliasetty_llama3-8b-v2.json b/data/models/naveenpoliasetty_llama3-8b-v2.json deleted file mode 100644 index d98c375f100f8ba5e37d65866aac322ee53cfd0f..0000000000000000000000000000000000000000 --- a/data/models/naveenpoliasetty_llama3-8b-v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "llama3-8B-V2", - "id": "Naveenpoliasetty/llama3-8B-V2", - "developer": "Naveenpoliasetty", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Naveenpoliasetty_llama3-8B-V2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4123 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5189 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0785 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2903 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4081 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3738 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nazimali_mistral-nemo-kurdish-instruct.json b/data/models/nazimali_mistral-nemo-kurdish-instruct.json deleted file mode 100644 index 7bcf436e7a29205a95d8b228b8e52bcfb9264e7a..0000000000000000000000000000000000000000 --- a/data/models/nazimali_mistral-nemo-kurdish-instruct.json +++ /dev/null @@ -1,275 +0,0 @@ -{ - "model_info": { - "name": "Mistral-Nemo-Kurdish-Instruct", - "id": "nazimali/Mistral-Nemo-Kurdish-Instruct", - "developer": "nazimali", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nazimali_Mistral-Nemo-Kurdish-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4964 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4699 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0045 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2827 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3979 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3063 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "hfopenllm_v2/nazimali_Mistral-Nemo-Kurdish-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.486 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4721 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0846 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2844 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4006 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3087 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nazimali_mistral-nemo-kurdish.json b/data/models/nazimali_mistral-nemo-kurdish.json deleted file mode 100644 index a63c875731261d965aefad6f67617af453ca4eec..0000000000000000000000000000000000000000 --- a/data/models/nazimali_mistral-nemo-kurdish.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mistral-Nemo-Kurdish", - "id": "nazimali/Mistral-Nemo-Kurdish", - "developer": "nazimali", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nazimali_Mistral-Nemo-Kurdish/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3401 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5133 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0959 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3012 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4116 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3235 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nbailab_nb-llama-3.1-8b-instruct.json b/data/models/nbailab_nb-llama-3.1-8b-instruct.json deleted file mode 100644 index 4d4e6ce03834085d336611e59e988d2ba2bd8fc4..0000000000000000000000000000000000000000 --- a/data/models/nbailab_nb-llama-3.1-8b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "nb-llama-3.1-8B-Instruct", - "id": "NbAiLab/nb-llama-3.1-8B-Instruct", - "developer": "NbAiLab", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/NbAiLab_nb-llama-3.1-8B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3625 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3247 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0227 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2735 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3208 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1197 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nbailab_nb-llama-3.1-8b-sft.json b/data/models/nbailab_nb-llama-3.1-8b-sft.json deleted file mode 100644 index eb7e00e86f70d8a8a44c28d29fa66c3deb7bb1e9..0000000000000000000000000000000000000000 --- a/data/models/nbailab_nb-llama-3.1-8b-sft.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "nb-llama-3.1-8B-sft", - "id": "NbAiLab/nb-llama-3.1-8B-sft", - "developer": "NbAiLab", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/NbAiLab_nb-llama-3.1-8B-sft/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3616 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3282 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0219 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2542 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3287 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1222 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nbeerbower_bigkartoffel-mistral-nemo-20b.json b/data/models/nbeerbower_bigkartoffel-mistral-nemo-20b.json deleted file mode 100644 index 5af7d4f2f0ae53d072b7e99552e42cab4f19e88e..0000000000000000000000000000000000000000 --- a/data/models/nbeerbower_bigkartoffel-mistral-nemo-20b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "BigKartoffel-mistral-nemo-20B", - "id": "nbeerbower/BigKartoffel-mistral-nemo-20B", - "developer": "nbeerbower", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "20.427" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nbeerbower_BigKartoffel-mistral-nemo-20B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5857 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5515 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0264 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2869 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.428 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.353 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nbeerbower_doppelkartoffel-mistral-nemo-23b.json b/data/models/nbeerbower_doppelkartoffel-mistral-nemo-23b.json deleted file mode 100644 index f9285ce883ae861a783b27b410d992a6282a0699..0000000000000000000000000000000000000000 --- a/data/models/nbeerbower_doppelkartoffel-mistral-nemo-23b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "DoppelKartoffel-Mistral-Nemo-23B", - "id": "nbeerbower/DoppelKartoffel-Mistral-Nemo-23B", - "developer": "nbeerbower", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "23.153" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nbeerbower_DoppelKartoffel-Mistral-Nemo-23B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5191 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5218 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.031 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2752 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3795 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.308 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nbeerbower_doublepotato-mistral-nemo-13b.json b/data/models/nbeerbower_doublepotato-mistral-nemo-13b.json deleted file mode 100644 index 41b8e7abffdf5fbfe33a930d12c501cbee194f0c..0000000000000000000000000000000000000000 --- a/data/models/nbeerbower_doublepotato-mistral-nemo-13b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "DoublePotato-Mistral-Nemo-13B", - "id": "nbeerbower/DoublePotato-Mistral-Nemo-13B", - "developer": "nbeerbower", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "13.338" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nbeerbower_DoublePotato-Mistral-Nemo-13B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6796 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5438 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.04 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3012 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.46 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3596 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nbeerbower_dumpling-qwen2.5-1.5b.json b/data/models/nbeerbower_dumpling-qwen2.5-1.5b.json deleted file mode 100644 index 9b4a7e6d1072d68d30d1025fb33008d5f2324c4a..0000000000000000000000000000000000000000 --- a/data/models/nbeerbower_dumpling-qwen2.5-1.5b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Dumpling-Qwen2.5-1.5B", - "id": "nbeerbower/Dumpling-Qwen2.5-1.5B", - "developer": "nbeerbower", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.544" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nbeerbower_Dumpling-Qwen2.5-1.5B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3699 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.416 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1171 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2685 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3728 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2772 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nbeerbower_dumpling-qwen2.5-14b.json b/data/models/nbeerbower_dumpling-qwen2.5-14b.json deleted file mode 100644 index ce872f693c2597ca3f7c75b64804094564dc0c7a..0000000000000000000000000000000000000000 --- a/data/models/nbeerbower_dumpling-qwen2.5-14b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Dumpling-Qwen2.5-14B", - "id": "nbeerbower/Dumpling-Qwen2.5-14B", - "developer": "nbeerbower", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nbeerbower_Dumpling-Qwen2.5-14B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6064 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6451 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3097 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3012 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4354 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.517 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nbeerbower_dumpling-qwen2.5-7b-1k-r16.json b/data/models/nbeerbower_dumpling-qwen2.5-7b-1k-r16.json deleted file mode 100644 index cb62b20dd47f09099b93c1f00c8b98329704f17d..0000000000000000000000000000000000000000 --- a/data/models/nbeerbower_dumpling-qwen2.5-7b-1k-r16.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Dumpling-Qwen2.5-7B-1k-r16", - "id": "nbeerbower/Dumpling-Qwen2.5-7B-1k-r16", - "developer": "nbeerbower", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nbeerbower_Dumpling-Qwen2.5-7B-1k-r16/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.486 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5214 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2364 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2701 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.423 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3959 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nbeerbower_dumpling-qwen2.5-7b-1k-r64-2e-5.json b/data/models/nbeerbower_dumpling-qwen2.5-7b-1k-r64-2e-5.json deleted file mode 100644 index 27e70c32df9b30196aa045c34a887661c40af13e..0000000000000000000000000000000000000000 --- a/data/models/nbeerbower_dumpling-qwen2.5-7b-1k-r64-2e-5.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Dumpling-Qwen2.5-7B-1k-r64-2e-5", - "id": "nbeerbower/Dumpling-Qwen2.5-7B-1k-r64-2e-5", - "developer": "nbeerbower", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nbeerbower_Dumpling-Qwen2.5-7B-1k-r64-2e-5/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4179 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5301 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2115 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2701 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4486 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4122 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nbeerbower_eva-abliterated-ties-qwen2.5-1.5b.json b/data/models/nbeerbower_eva-abliterated-ties-qwen2.5-1.5b.json deleted file mode 100644 index 49ac48ad5bd4a0b5ac84ed71052e6e0df20dc85f..0000000000000000000000000000000000000000 --- a/data/models/nbeerbower_eva-abliterated-ties-qwen2.5-1.5b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "EVA-abliterated-TIES-Qwen2.5-1.5B", - "id": "nbeerbower/EVA-abliterated-TIES-Qwen2.5-1.5B", - "developer": "nbeerbower", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.777" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nbeerbower_EVA-abliterated-TIES-Qwen2.5-1.5B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4115 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3997 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1375 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2651 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3502 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2712 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nbeerbower_eva-abliterated-ties-qwen2.5-14b.json b/data/models/nbeerbower_eva-abliterated-ties-qwen2.5-14b.json deleted file mode 100644 index 69d25674f23336bff9e82c19a7e6413a710ad543..0000000000000000000000000000000000000000 --- a/data/models/nbeerbower_eva-abliterated-ties-qwen2.5-14b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "EVA-abliterated-TIES-Qwen2.5-14B", - "id": "nbeerbower/EVA-abliterated-TIES-Qwen2.5-14B", - "developer": "nbeerbower", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nbeerbower_EVA-abliterated-TIES-Qwen2.5-14B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7836 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6372 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5045 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3549 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4407 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5211 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nbeerbower_flammades-mistral-nemo-12b.json b/data/models/nbeerbower_flammades-mistral-nemo-12b.json deleted file mode 100644 index ddad656265039c7fee001fa0853247ee2ce2fed9..0000000000000000000000000000000000000000 --- a/data/models/nbeerbower_flammades-mistral-nemo-12b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Flammades-Mistral-Nemo-12B", - "id": "nbeerbower/Flammades-Mistral-Nemo-12B", - "developer": "nbeerbower", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nbeerbower_Flammades-Mistral-Nemo-12B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3842 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.53 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0755 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3037 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4806 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3661 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nbeerbower_gemma2-gutenberg-27b.json b/data/models/nbeerbower_gemma2-gutenberg-27b.json deleted file mode 100644 index 9c102edb3df865bed8cd5a65b72e8b993da5703d..0000000000000000000000000000000000000000 --- a/data/models/nbeerbower_gemma2-gutenberg-27b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "gemma2-gutenberg-27B", - "id": "nbeerbower/gemma2-gutenberg-27B", - "developer": "nbeerbower", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "27.227" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nbeerbower_gemma2-gutenberg-27B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2947 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3797 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0189 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2727 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3727 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1982 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nbeerbower_gemma2-gutenberg-9b.json b/data/models/nbeerbower_gemma2-gutenberg-9b.json deleted file mode 100644 index 9d945ec44c0718aacb060bdf9a1b4f6e9d763ca6..0000000000000000000000000000000000000000 --- a/data/models/nbeerbower_gemma2-gutenberg-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "gemma2-gutenberg-9B", - "id": "nbeerbower/gemma2-gutenberg-9B", - "developer": "nbeerbower", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "9.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nbeerbower_gemma2-gutenberg-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2796 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5951 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0808 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3381 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4595 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4192 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nbeerbower_gemma2-gutenberg-doppel-9b.json b/data/models/nbeerbower_gemma2-gutenberg-doppel-9b.json deleted file mode 100644 index 6387e7317bdcb1184c10fa00041670f8e80c872e..0000000000000000000000000000000000000000 --- a/data/models/nbeerbower_gemma2-gutenberg-doppel-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Gemma2-Gutenberg-Doppel-9B", - "id": "nbeerbower/Gemma2-Gutenberg-Doppel-9B", - "developer": "nbeerbower", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "9.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nbeerbower_Gemma2-Gutenberg-Doppel-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7171 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.587 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1979 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3297 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4608 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4127 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nbeerbower_gutensuppe-mistral-nemo-12b.json b/data/models/nbeerbower_gutensuppe-mistral-nemo-12b.json deleted file mode 100644 index 6cd97767da064eca889bf06129a841fe87ae1924..0000000000000000000000000000000000000000 --- a/data/models/nbeerbower_gutensuppe-mistral-nemo-12b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Gutensuppe-mistral-nemo-12B", - "id": "nbeerbower/Gutensuppe-mistral-nemo-12B", - "developer": "nbeerbower", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nbeerbower_Gutensuppe-mistral-nemo-12B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2916 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5487 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1329 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3372 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.429 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.368 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nbeerbower_hermes2-gutenberg2-mistral-7b.json b/data/models/nbeerbower_hermes2-gutenberg2-mistral-7b.json deleted file mode 100644 index d067004323218adacf3e0ec1f6bd46107425186f..0000000000000000000000000000000000000000 --- a/data/models/nbeerbower_hermes2-gutenberg2-mistral-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Hermes2-Gutenberg2-Mistral-7B", - "id": "nbeerbower/Hermes2-Gutenberg2-Mistral-7B", - "developer": "nbeerbower", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nbeerbower_Hermes2-Gutenberg2-Mistral-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3721 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4981 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0574 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2894 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4623 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2993 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nbeerbower_kartoffel-deepfry-12b.json b/data/models/nbeerbower_kartoffel-deepfry-12b.json deleted file mode 100644 index 5f731b365cf04d72154f334c15f3df55fe241fb7..0000000000000000000000000000000000000000 --- a/data/models/nbeerbower_kartoffel-deepfry-12b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Kartoffel-Deepfry-12B", - "id": "nbeerbower/Kartoffel-Deepfry-12B", - "developer": "nbeerbower", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nbeerbower_Kartoffel-Deepfry-12B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5022 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5365 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0604 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2961 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4792 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3582 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nbeerbower_llama-3-gutenberg-8b.json b/data/models/nbeerbower_llama-3-gutenberg-8b.json deleted file mode 100644 index 8ee017507773075e0a448a0fa89219b295ad4aa0..0000000000000000000000000000000000000000 --- a/data/models/nbeerbower_llama-3-gutenberg-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "llama-3-gutenberg-8B", - "id": "nbeerbower/llama-3-gutenberg-8B", - "developer": "nbeerbower", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nbeerbower_llama-3-gutenberg-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4372 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4994 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0785 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3012 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4073 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3831 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nbeerbower_llama-3.1-nemotron-lorablated-70b.json b/data/models/nbeerbower_llama-3.1-nemotron-lorablated-70b.json deleted file mode 100644 index c844bd6868c3dd72ca20978cba6743f9f3d460d0..0000000000000000000000000000000000000000 --- a/data/models/nbeerbower_llama-3.1-nemotron-lorablated-70b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.1-Nemotron-lorablated-70B", - "id": "nbeerbower/Llama-3.1-Nemotron-lorablated-70B", - "developer": "nbeerbower", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "70.554" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nbeerbower_Llama-3.1-Nemotron-lorablated-70B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7229 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6825 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3338 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3909 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4682 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5343 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nbeerbower_llama3.1-cc-8b.json b/data/models/nbeerbower_llama3.1-cc-8b.json deleted file mode 100644 index 704b85fa1ee884f792634d41caa8aa57d265d9a2..0000000000000000000000000000000000000000 --- a/data/models/nbeerbower_llama3.1-cc-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "llama3.1-cc-8B", - "id": "nbeerbower/llama3.1-cc-8B", - "developer": "nbeerbower", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nbeerbower_llama3.1-cc-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5068 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4871 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.071 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2852 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3885 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3347 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nbeerbower_llama3.1-gutenberg-doppel-70b.json b/data/models/nbeerbower_llama3.1-gutenberg-doppel-70b.json deleted file mode 100644 index 6fda33b05fdfe0f70188dca05cadeb5383d7db92..0000000000000000000000000000000000000000 --- a/data/models/nbeerbower_llama3.1-gutenberg-doppel-70b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama3.1-Gutenberg-Doppel-70B", - "id": "nbeerbower/Llama3.1-Gutenberg-Doppel-70B", - "developer": "nbeerbower", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "70.554" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nbeerbower_Llama3.1-Gutenberg-Doppel-70B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7092 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6661 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2122 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3448 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4897 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4737 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nbeerbower_llama3.1-kartoffeldes-70b.json b/data/models/nbeerbower_llama3.1-kartoffeldes-70b.json deleted file mode 100644 index 18af3c135c9da1bacd55181a809ad035964c86e5..0000000000000000000000000000000000000000 --- a/data/models/nbeerbower_llama3.1-kartoffeldes-70b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "llama3.1-kartoffeldes-70B", - "id": "nbeerbower/llama3.1-kartoffeldes-70B", - "developer": "nbeerbower", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "70.554" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nbeerbower_llama3.1-kartoffeldes-70B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.823 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6894 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3218 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3515 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4646 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4988 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nbeerbower_lyra-gutenberg-mistral-nemo-12b.json b/data/models/nbeerbower_lyra-gutenberg-mistral-nemo-12b.json deleted file mode 100644 index d9c64a16ddaf156ec57a722ad4b60ebf763d7635..0000000000000000000000000000000000000000 --- a/data/models/nbeerbower_lyra-gutenberg-mistral-nemo-12b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Lyra-Gutenberg-mistral-nemo-12B", - "id": "nbeerbower/Lyra-Gutenberg-mistral-nemo-12B", - "developer": "nbeerbower", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nbeerbower_Lyra-Gutenberg-mistral-nemo-12B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3495 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5586 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1012 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3339 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4357 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3628 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nbeerbower_lyra4-gutenberg-12b.json b/data/models/nbeerbower_lyra4-gutenberg-12b.json deleted file mode 100644 index ee750c35510aa3d04d4345fbea7fbeaa1028b005..0000000000000000000000000000000000000000 --- a/data/models/nbeerbower_lyra4-gutenberg-12b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Lyra4-Gutenberg-12B", - "id": "nbeerbower/Lyra4-Gutenberg-12B", - "developer": "nbeerbower", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nbeerbower_Lyra4-Gutenberg-12B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2212 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5387 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1299 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3188 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4038 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3571 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nbeerbower_lyra4-gutenberg2-12b.json b/data/models/nbeerbower_lyra4-gutenberg2-12b.json deleted file mode 100644 index bd029c9a1cc7806dab4f4a746e92433ab4f4dcce..0000000000000000000000000000000000000000 --- a/data/models/nbeerbower_lyra4-gutenberg2-12b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Lyra4-Gutenberg2-12B", - "id": "nbeerbower/Lyra4-Gutenberg2-12B", - "developer": "nbeerbower", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nbeerbower_Lyra4-Gutenberg2-12B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2585 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5345 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1171 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3129 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3972 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3565 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nbeerbower_mahou-1.5-mistral-nemo-12b-lorablated.json b/data/models/nbeerbower_mahou-1.5-mistral-nemo-12b-lorablated.json deleted file mode 100644 index 90419b2c9e4cf0ee14fa611f0d0c13fca4f5162b..0000000000000000000000000000000000000000 --- a/data/models/nbeerbower_mahou-1.5-mistral-nemo-12b-lorablated.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mahou-1.5-mistral-nemo-12B-lorablated", - "id": "nbeerbower/Mahou-1.5-mistral-nemo-12B-lorablated", - "developer": "nbeerbower", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nbeerbower_Mahou-1.5-mistral-nemo-12B-lorablated/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6825 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5496 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0891 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2794 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4522 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3574 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nbeerbower_mistral-gutenberg-doppel-7b-fft.json b/data/models/nbeerbower_mistral-gutenberg-doppel-7b-fft.json deleted file mode 100644 index 8a8cc2c54ee6a0c934ca5d8f9ef2b7188514616f..0000000000000000000000000000000000000000 --- a/data/models/nbeerbower_mistral-gutenberg-doppel-7b-fft.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mistral-Gutenberg-Doppel-7B-FFT", - "id": "nbeerbower/Mistral-Gutenberg-Doppel-7B-FFT", - "developer": "nbeerbower", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nbeerbower_Mistral-Gutenberg-Doppel-7B-FFT/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5717 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4076 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0249 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2836 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4059 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2729 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nbeerbower_mistral-nemo-bophades-12b.json b/data/models/nbeerbower_mistral-nemo-bophades-12b.json deleted file mode 100644 index 7423a81efda3141ec3011c1af2ba9b7c90b0a170..0000000000000000000000000000000000000000 --- a/data/models/nbeerbower_mistral-nemo-bophades-12b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "mistral-nemo-bophades-12B", - "id": "nbeerbower/mistral-nemo-bophades-12B", - "developer": "nbeerbower", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nbeerbower_mistral-nemo-bophades-12B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6794 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4988 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1231 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2852 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4178 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3501 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nbeerbower_mistral-nemo-bophades3-12b.json b/data/models/nbeerbower_mistral-nemo-bophades3-12b.json deleted file mode 100644 index 9f0594fe0c28632ff71171246b9a55d98b7eb4c0..0000000000000000000000000000000000000000 --- a/data/models/nbeerbower_mistral-nemo-bophades3-12b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "mistral-nemo-bophades3-12B", - "id": "nbeerbower/mistral-nemo-bophades3-12B", - "developer": "nbeerbower", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nbeerbower_mistral-nemo-bophades3-12B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6578 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5449 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0846 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3121 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4604 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3371 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nbeerbower_mistral-nemo-cc-12b.json b/data/models/nbeerbower_mistral-nemo-cc-12b.json deleted file mode 100644 index 8ce3bbc9ebeae14ebf50cb3c2af3eec4c962c6e0..0000000000000000000000000000000000000000 --- a/data/models/nbeerbower_mistral-nemo-cc-12b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "mistral-nemo-cc-12B", - "id": "nbeerbower/mistral-nemo-cc-12B", - "developer": "nbeerbower", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nbeerbower_mistral-nemo-cc-12B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1435 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5399 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0257 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3154 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4424 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3598 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nbeerbower_mistral-nemo-gutades-12b.json b/data/models/nbeerbower_mistral-nemo-gutades-12b.json deleted file mode 100644 index 63a337c5b26e3f5efa8bfc5f8110cd1e76354edf..0000000000000000000000000000000000000000 --- a/data/models/nbeerbower_mistral-nemo-gutades-12b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "mistral-nemo-gutades-12B", - "id": "nbeerbower/mistral-nemo-gutades-12B", - "developer": "nbeerbower", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nbeerbower_mistral-nemo-gutades-12B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3425 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5407 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1178 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3154 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.404 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3561 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nbeerbower_mistral-nemo-gutenberg-12b-v2.json b/data/models/nbeerbower_mistral-nemo-gutenberg-12b-v2.json deleted file mode 100644 index c7599daefc528a2509d0889c32af9549076bd0d6..0000000000000000000000000000000000000000 --- a/data/models/nbeerbower_mistral-nemo-gutenberg-12b-v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "mistral-nemo-gutenberg-12B-v2", - "id": "nbeerbower/mistral-nemo-gutenberg-12B-v2", - "developer": "nbeerbower", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nbeerbower_mistral-nemo-gutenberg-12B-v2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6203 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5397 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1088 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2777 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4287 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3499 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nbeerbower_mistral-nemo-gutenberg-12b-v3.json b/data/models/nbeerbower_mistral-nemo-gutenberg-12b-v3.json deleted file mode 100644 index abf6078a53f0be0835ed11de6c9114f91dc01e1a..0000000000000000000000000000000000000000 --- a/data/models/nbeerbower_mistral-nemo-gutenberg-12b-v3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "mistral-nemo-gutenberg-12B-v3", - "id": "nbeerbower/mistral-nemo-gutenberg-12B-v3", - "developer": "nbeerbower", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nbeerbower_mistral-nemo-gutenberg-12B-v3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2183 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5441 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0597 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3146 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.445 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3644 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nbeerbower_mistral-nemo-gutenberg-12b-v4.json b/data/models/nbeerbower_mistral-nemo-gutenberg-12b-v4.json deleted file mode 100644 index fdb38823a04124dc4edc8e4d21767d57f6ec2085..0000000000000000000000000000000000000000 --- a/data/models/nbeerbower_mistral-nemo-gutenberg-12b-v4.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "mistral-nemo-gutenberg-12B-v4", - "id": "nbeerbower/mistral-nemo-gutenberg-12B-v4", - "developer": "nbeerbower", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nbeerbower_mistral-nemo-gutenberg-12B-v4/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2379 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5269 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1261 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3163 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4104 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3575 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nbeerbower_mistral-nemo-gutenberg-12b.json b/data/models/nbeerbower_mistral-nemo-gutenberg-12b.json deleted file mode 100644 index 76c294e0304a54386e0aa6c7018337956ac5ec72..0000000000000000000000000000000000000000 --- a/data/models/nbeerbower_mistral-nemo-gutenberg-12b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "mistral-nemo-gutenberg-12B", - "id": "nbeerbower/mistral-nemo-gutenberg-12B", - "developer": "nbeerbower", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nbeerbower_mistral-nemo-gutenberg-12B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3504 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5281 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1163 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.307 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4171 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3562 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nbeerbower_mistral-nemo-gutenberg-doppel-12b-v2.json b/data/models/nbeerbower_mistral-nemo-gutenberg-doppel-12b-v2.json deleted file mode 100644 index ee042721aee33bb72edb0d5ee84c43ed835e80c3..0000000000000000000000000000000000000000 --- a/data/models/nbeerbower_mistral-nemo-gutenberg-doppel-12b-v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mistral-Nemo-Gutenberg-Doppel-12B-v2", - "id": "nbeerbower/Mistral-Nemo-Gutenberg-Doppel-12B-v2", - "developer": "nbeerbower", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nbeerbower_Mistral-Nemo-Gutenberg-Doppel-12B-v2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6536 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5374 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1156 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.271 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4233 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3546 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nbeerbower_mistral-nemo-gutenberg-doppel-12b.json b/data/models/nbeerbower_mistral-nemo-gutenberg-doppel-12b.json deleted file mode 100644 index f3077d79fb428ee89edfd49f1e512aea86cf4c23..0000000000000000000000000000000000000000 --- a/data/models/nbeerbower_mistral-nemo-gutenberg-doppel-12b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mistral-Nemo-Gutenberg-Doppel-12B", - "id": "nbeerbower/Mistral-Nemo-Gutenberg-Doppel-12B", - "developer": "nbeerbower", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nbeerbower_Mistral-Nemo-Gutenberg-Doppel-12B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3567 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5275 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1216 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3163 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4132 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3579 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nbeerbower_mistral-nemo-gutenberg2-12b-test.json b/data/models/nbeerbower_mistral-nemo-gutenberg2-12b-test.json deleted file mode 100644 index 9cd4297ee5b0390e5c3f0f4740112d42fb35f28c..0000000000000000000000000000000000000000 --- a/data/models/nbeerbower_mistral-nemo-gutenberg2-12b-test.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "mistral-nemo-gutenberg2-12B-test", - "id": "nbeerbower/mistral-nemo-gutenberg2-12B-test", - "developer": "nbeerbower", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nbeerbower_mistral-nemo-gutenberg2-12B-test/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3385 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5255 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1163 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3171 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4157 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3555 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nbeerbower_mistral-nemo-kartoffel-12b.json b/data/models/nbeerbower_mistral-nemo-kartoffel-12b.json deleted file mode 100644 index 34f6fa4ddcbb49ccde287acc67ee9736f67fc19e..0000000000000000000000000000000000000000 --- a/data/models/nbeerbower_mistral-nemo-kartoffel-12b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "mistral-nemo-kartoffel-12B", - "id": "nbeerbower/mistral-nemo-kartoffel-12B", - "developer": "nbeerbower", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nbeerbower_mistral-nemo-kartoffel-12B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7032 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5484 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0853 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3045 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4653 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3585 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nbeerbower_mistral-nemo-moderne-12b-fft-experimental.json b/data/models/nbeerbower_mistral-nemo-moderne-12b-fft-experimental.json deleted file mode 100644 index c10c3f93929edabf790a837d042734734248d54a..0000000000000000000000000000000000000000 --- a/data/models/nbeerbower_mistral-nemo-moderne-12b-fft-experimental.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mistral-Nemo-Moderne-12B-FFT-experimental", - "id": "nbeerbower/Mistral-Nemo-Moderne-12B-FFT-experimental", - "developer": "nbeerbower", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nbeerbower_Mistral-Nemo-Moderne-12B-FFT-experimental/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3352 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5234 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.077 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.281 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3715 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3455 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nbeerbower_mistral-nemo-narwhal-12b.json b/data/models/nbeerbower_mistral-nemo-narwhal-12b.json deleted file mode 100644 index 0f19919224bef38c510f761c367200f077164151..0000000000000000000000000000000000000000 --- a/data/models/nbeerbower_mistral-nemo-narwhal-12b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "mistral-nemo-narwhal-12B", - "id": "nbeerbower/mistral-nemo-narwhal-12B", - "developer": "nbeerbower", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nbeerbower_mistral-nemo-narwhal-12B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5549 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5057 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0582 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.271 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3847 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3483 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nbeerbower_mistral-nemo-prism-12b-v2.json b/data/models/nbeerbower_mistral-nemo-prism-12b-v2.json deleted file mode 100644 index c3ec1499d70013d7ea2f4edb83ada04be25e4736..0000000000000000000000000000000000000000 --- a/data/models/nbeerbower_mistral-nemo-prism-12b-v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mistral-Nemo-Prism-12B-v2", - "id": "nbeerbower/Mistral-Nemo-Prism-12B-v2", - "developer": "nbeerbower", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nbeerbower_Mistral-Nemo-Prism-12B-v2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6974 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5492 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0891 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3054 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.46 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3567 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nbeerbower_mistral-nemo-prism-12b-v7.json b/data/models/nbeerbower_mistral-nemo-prism-12b-v7.json deleted file mode 100644 index 2920a21195255cce181aec980b954398496c2f76..0000000000000000000000000000000000000000 --- a/data/models/nbeerbower_mistral-nemo-prism-12b-v7.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mistral-Nemo-Prism-12B-v7", - "id": "nbeerbower/Mistral-Nemo-Prism-12B-v7", - "developer": "nbeerbower", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nbeerbower_Mistral-Nemo-Prism-12B-v7/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6962 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5521 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0869 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2995 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4639 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.359 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nbeerbower_mistral-nemo-prism-12b.json b/data/models/nbeerbower_mistral-nemo-prism-12b.json deleted file mode 100644 index 74ef9127fa07472823ced1b65086e154cbfbeeab..0000000000000000000000000000000000000000 --- a/data/models/nbeerbower_mistral-nemo-prism-12b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mistral-Nemo-Prism-12B", - "id": "nbeerbower/Mistral-Nemo-Prism-12B", - "developer": "nbeerbower", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nbeerbower_Mistral-Nemo-Prism-12B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6858 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5475 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0869 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3079 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4626 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3581 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nbeerbower_mistral-nemo-wissenschaft-12b.json b/data/models/nbeerbower_mistral-nemo-wissenschaft-12b.json deleted file mode 100644 index a176f9e37b4a2aebb9b75eb76dc25173df1b49b0..0000000000000000000000000000000000000000 --- a/data/models/nbeerbower_mistral-nemo-wissenschaft-12b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "mistral-nemo-wissenschaft-12B", - "id": "nbeerbower/mistral-nemo-wissenschaft-12B", - "developer": "nbeerbower", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nbeerbower_mistral-nemo-wissenschaft-12B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.652 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.504 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1216 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2928 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4178 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3532 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nbeerbower_mistral-small-drummer-22b.json b/data/models/nbeerbower_mistral-small-drummer-22b.json deleted file mode 100644 index 560d0148e9613f984e51ea4b98c5636f99875e45..0000000000000000000000000000000000000000 --- a/data/models/nbeerbower_mistral-small-drummer-22b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mistral-Small-Drummer-22B", - "id": "nbeerbower/Mistral-Small-Drummer-22B", - "developer": "nbeerbower", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "22.247" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nbeerbower_Mistral-Small-Drummer-22B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6331 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5793 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1888 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3431 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4064 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4095 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nbeerbower_mistral-small-gutenberg-doppel-22b.json b/data/models/nbeerbower_mistral-small-gutenberg-doppel-22b.json deleted file mode 100644 index 575090cf48845c82a9637b94b36d9a8a7bb1b318..0000000000000000000000000000000000000000 --- a/data/models/nbeerbower_mistral-small-gutenberg-doppel-22b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mistral-Small-Gutenberg-Doppel-22B", - "id": "nbeerbower/Mistral-Small-Gutenberg-Doppel-22B", - "developer": "nbeerbower", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "22.247" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nbeerbower_Mistral-Small-Gutenberg-Doppel-22B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4893 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5859 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2183 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3465 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3971 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4124 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nbeerbower_nemo-loony-12b-experimental.json b/data/models/nbeerbower_nemo-loony-12b-experimental.json deleted file mode 100644 index c3a3229c38d7cc6623f4762360f30decc2d96826..0000000000000000000000000000000000000000 --- a/data/models/nbeerbower_nemo-loony-12b-experimental.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Nemo-Loony-12B-experimental", - "id": "nbeerbower/Nemo-Loony-12B-experimental", - "developer": "nbeerbower", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nbeerbower_Nemo-Loony-12B-experimental/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3734 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3822 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0151 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2701 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3341 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1589 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nbeerbower_nemoties-chatml-12b.json b/data/models/nbeerbower_nemoties-chatml-12b.json deleted file mode 100644 index 67bbf646183cd9c440aae2bf55edbc71e3fc8458..0000000000000000000000000000000000000000 --- a/data/models/nbeerbower_nemoties-chatml-12b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Nemoties-ChatML-12B", - "id": "nbeerbower/Nemoties-ChatML-12B", - "developer": "nbeerbower", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nbeerbower_Nemoties-ChatML-12B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6382 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.547 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0785 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.297 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4509 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3551 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nbeerbower_qwen2.5-gutenberg-doppel-14b.json b/data/models/nbeerbower_qwen2.5-gutenberg-doppel-14b.json deleted file mode 100644 index 3761dae53003aa7f38dc920f16f3a658771ca013..0000000000000000000000000000000000000000 --- a/data/models/nbeerbower_qwen2.5-gutenberg-doppel-14b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-Gutenberg-Doppel-14B", - "id": "nbeerbower/Qwen2.5-Gutenberg-Doppel-14B", - "developer": "nbeerbower", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nbeerbower_Qwen2.5-Gutenberg-Doppel-14B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8091 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6382 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5415 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3331 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4101 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4921 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nbeerbower_smolnemo-12b-fft-experimental.json b/data/models/nbeerbower_smolnemo-12b-fft-experimental.json deleted file mode 100644 index 9d159f9fc7f990bb26ba242be69d03eed8c162ab..0000000000000000000000000000000000000000 --- a/data/models/nbeerbower_smolnemo-12b-fft-experimental.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SmolNemo-12B-FFT-experimental", - "id": "nbeerbower/SmolNemo-12B-FFT-experimental", - "developer": "nbeerbower", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nbeerbower_SmolNemo-12B-FFT-experimental/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3348 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3336 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0128 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2601 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3847 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1217 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nbeerbower_stella-mistral-nemo-12b-v2.json b/data/models/nbeerbower_stella-mistral-nemo-12b-v2.json deleted file mode 100644 index e9f6119f9e16785a77a889cd6adf1376e50ef145..0000000000000000000000000000000000000000 --- a/data/models/nbeerbower_stella-mistral-nemo-12b-v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Stella-mistral-nemo-12B-v2", - "id": "nbeerbower/Stella-mistral-nemo-12B-v2", - "developer": "nbeerbower", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nbeerbower_Stella-mistral-nemo-12B-v2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3274 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5484 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1163 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3322 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4304 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3684 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nbrahme_indusq.json b/data/models/nbrahme_indusq.json deleted file mode 100644 index 62b040e3bb0e2f5960f42324174119e919d5b286..0000000000000000000000000000000000000000 --- a/data/models/nbrahme_indusq.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "IndusQ", - "id": "nbrahme/IndusQ", - "developer": "nbrahme", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "GPT2LMHeadModel", - "params_billions": "1.176" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nbrahme_IndusQ/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.244 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3062 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0008 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2651 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3366 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.112 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ncsoft_llama-3-offsetbias-8b.json b/data/models/ncsoft_llama-3-offsetbias-8b.json deleted file mode 100644 index e35728e4e3d5b05b8b7908ad0e2951b22a3671b5..0000000000000000000000000000000000000000 --- a/data/models/ncsoft_llama-3-offsetbias-8b.json +++ /dev/null @@ -1,126 +0,0 @@ -{ - "model_info": { - "name": "NCSOFT/Llama-3-OffsetBias-8B", - "id": "NCSOFT/Llama-3-OffsetBias-8B", - "developer": "NCSOFT", - "additional_details": { - "model_type": "Generative" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/NCSOFT_Llama-3-OffsetBias-8B/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8397 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9246 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8026 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8676 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7639 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ncsoft_llama-3-offsetbias-rm-8b.json b/data/models/ncsoft_llama-3-offsetbias-rm-8b.json deleted file mode 100644 index 14fabae93f5ca8f04e2a0c2d895536cd5cdfcb3a..0000000000000000000000000000000000000000 --- a/data/models/ncsoft_llama-3-offsetbias-rm-8b.json +++ /dev/null @@ -1,276 +0,0 @@ -{ - "model_info": { - "name": "NCSOFT/Llama-3-OffsetBias-RM-8B", - "id": "NCSOFT/Llama-3-OffsetBias-RM-8B", - "developer": "NCSOFT", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/NCSOFT_Llama-3-OffsetBias-RM-8B/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.648 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6084 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5191 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7222 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9596 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6786 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "reward-bench/NCSOFT_Llama-3-OffsetBias-RM-8B/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8942 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9721 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.818 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8676 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9192 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ncsoft_llama-varco-8b-instruct.json b/data/models/ncsoft_llama-varco-8b-instruct.json deleted file mode 100644 index 1a2767cc1b8f58dfc4b4686c2155f748925f966c..0000000000000000000000000000000000000000 --- a/data/models/ncsoft_llama-varco-8b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-VARCO-8B-Instruct", - "id": "NCSOFT/Llama-VARCO-8B-Instruct", - "developer": "NCSOFT", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/NCSOFT_Llama-VARCO-8B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.447 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5023 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1065 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.297 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3841 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.319 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/necva_ie-cont-llama3.1-8b.json b/data/models/necva_ie-cont-llama3.1-8b.json deleted file mode 100644 index 49233f79478e9e0a08cebd5c84e5bf0b86123fc7..0000000000000000000000000000000000000000 --- a/data/models/necva_ie-cont-llama3.1-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "IE-cont-Llama3.1-8B", - "id": "necva/IE-cont-Llama3.1-8B", - "developer": "necva", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/necva_IE-cont-Llama3.1-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2049 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2912 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2601 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3575 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1167 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/necva_replica-iepile.json b/data/models/necva_replica-iepile.json deleted file mode 100644 index 6166c2db6d2f5ce326ae59bebe25bd91b4c97487..0000000000000000000000000000000000000000 --- a/data/models/necva_replica-iepile.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "replica-IEPile", - "id": "necva/replica-IEPile", - "developer": "necva", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "4.65" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/necva_replica-IEPile/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4678 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4779 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1239 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3062 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3998 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3561 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nekochu_llama-3.1-8b-french-dpo.json b/data/models/nekochu_llama-3.1-8b-french-dpo.json deleted file mode 100644 index ede17866e55e893abb710a7b2507b29a5c6b8ea5..0000000000000000000000000000000000000000 --- a/data/models/nekochu_llama-3.1-8b-french-dpo.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.1-8B-french-DPO", - "id": "Nekochu/Llama-3.1-8B-french-DPO", - "developer": "Nekochu", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Nekochu_Llama-3.1-8B-french-DPO/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4656 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5111 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0974 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2911 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4216 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3414 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nekochu_llama-3.1-8b-german-orpo.json b/data/models/nekochu_llama-3.1-8b-german-orpo.json deleted file mode 100644 index e7162cc0fc49111e2fadeba1abf6229eedb0ee7e..0000000000000000000000000000000000000000 --- a/data/models/nekochu_llama-3.1-8b-german-orpo.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.1-8B-German-ORPO", - "id": "Nekochu/Llama-3.1-8B-German-ORPO", - "developer": "Nekochu", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Nekochu_Llama-3.1-8B-German-ORPO/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4611 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4983 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1171 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3163 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4647 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3393 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nekochu_luminia-13b-v3.json b/data/models/nekochu_luminia-13b-v3.json deleted file mode 100644 index 6ae4360e201fe21745bced994c35d5ec3df47f65..0000000000000000000000000000000000000000 --- a/data/models/nekochu_luminia-13b-v3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Luminia-13B-v3", - "id": "Nekochu/Luminia-13B-v3", - "developer": "Nekochu", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "13.016" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Nekochu_Luminia-13B-v3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2523 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4112 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0181 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2701 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3983 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2215 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nekochu_luminia-8b-rp.json b/data/models/nekochu_luminia-8b-rp.json deleted file mode 100644 index 7ac30911d997eb9d279d081b63e5e28109aaebec..0000000000000000000000000000000000000000 --- a/data/models/nekochu_luminia-8b-rp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Luminia-8B-RP", - "id": "Nekochu/Luminia-8B-RP", - "developer": "Nekochu", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Nekochu_Luminia-8B-RP/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5574 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5218 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.136 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.297 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3998 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3631 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/neopolita_jessi-v0.1-bf16-falcon3-7b-instruct.json b/data/models/neopolita_jessi-v0.1-bf16-falcon3-7b-instruct.json deleted file mode 100644 index 683361dcf87e8aeb35dbb02756de2fe07ab170a5..0000000000000000000000000000000000000000 --- a/data/models/neopolita_jessi-v0.1-bf16-falcon3-7b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "jessi-v0.1-bf16-falcon3-7b-instruct", - "id": "neopolita/jessi-v0.1-bf16-falcon3-7b-instruct", - "developer": "neopolita", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "7.456" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/neopolita_jessi-v0.1-bf16-falcon3-7b-instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7527 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5516 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3807 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3029 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4825 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3924 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/neopolita_jessi-v0.1-falcon3-10b-instruct.json b/data/models/neopolita_jessi-v0.1-falcon3-10b-instruct.json deleted file mode 100644 index 90742a9f9006d568912e4a1b24111f360c219de2..0000000000000000000000000000000000000000 --- a/data/models/neopolita_jessi-v0.1-falcon3-10b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "jessi-v0.1-falcon3-10b-instruct", - "id": "neopolita/jessi-v0.1-falcon3-10b-instruct", - "developer": "neopolita", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "10.306" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/neopolita_jessi-v0.1-falcon3-10b-instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7552 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5953 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2002 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3188 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4279 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4188 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/neopolita_jessi-v0.1-qwen2.5-7b-instruct.json b/data/models/neopolita_jessi-v0.1-qwen2.5-7b-instruct.json deleted file mode 100644 index ff8ee7737af5b2fca763c55f8a83b138aa5bd3a0..0000000000000000000000000000000000000000 --- a/data/models/neopolita_jessi-v0.1-qwen2.5-7b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "jessi-v0.1-qwen2.5-7b-instruct", - "id": "neopolita/jessi-v0.1-qwen2.5-7b-instruct", - "developer": "neopolita", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/neopolita_jessi-v0.1-qwen2.5-7b-instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7327 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5292 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4086 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.297 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3914 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4228 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/neopolita_jessi-v0.1-virtuoso-small.json b/data/models/neopolita_jessi-v0.1-virtuoso-small.json deleted file mode 100644 index f1d7090eedd2a59953b8708f7a34f0625fb47f12..0000000000000000000000000000000000000000 --- a/data/models/neopolita_jessi-v0.1-virtuoso-small.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "jessi-v0.1-virtuoso-small", - "id": "neopolita/jessi-v0.1-virtuoso-small", - "developer": "neopolita", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/neopolita_jessi-v0.1-virtuoso-small/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7959 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6443 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3399 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3305 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4362 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.513 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/neopolita_jessi-v0.2-falcon3-10b-instruct.json b/data/models/neopolita_jessi-v0.2-falcon3-10b-instruct.json deleted file mode 100644 index 1d7bc5c6af7a7ff0db5a2cb6a6751074600096b7..0000000000000000000000000000000000000000 --- a/data/models/neopolita_jessi-v0.2-falcon3-10b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "jessi-v0.2-falcon3-10b-instruct", - "id": "neopolita/jessi-v0.2-falcon3-10b-instruct", - "developer": "neopolita", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "10.306" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/neopolita_jessi-v0.2-falcon3-10b-instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7768 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6205 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2122 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3289 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4281 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4354 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/neopolita_jessi-v0.2-falcon3-7b-instruct.json b/data/models/neopolita_jessi-v0.2-falcon3-7b-instruct.json deleted file mode 100644 index 5cfbbc5b1b3639109c4f6b1aa22377e762628ff0..0000000000000000000000000000000000000000 --- a/data/models/neopolita_jessi-v0.2-falcon3-7b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "jessi-v0.2-falcon3-7b-instruct", - "id": "neopolita/jessi-v0.2-falcon3-7b-instruct", - "developer": "neopolita", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "7.456" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/neopolita_jessi-v0.2-falcon3-7b-instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5771 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5363 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2538 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3171 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4479 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3905 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/neopolita_jessi-v0.3-falcon3-7b-instruct.json b/data/models/neopolita_jessi-v0.3-falcon3-7b-instruct.json deleted file mode 100644 index 6ab49725f732fdd58c76d45b89431a7458614dbb..0000000000000000000000000000000000000000 --- a/data/models/neopolita_jessi-v0.3-falcon3-7b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "jessi-v0.3-falcon3-7b-instruct", - "id": "neopolita/jessi-v0.3-falcon3-7b-instruct", - "developer": "neopolita", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "7.456" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/neopolita_jessi-v0.3-falcon3-7b-instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7509 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5388 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1888 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3196 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4692 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.397 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/neopolita_jessi-v0.4-falcon3-7b-instruct.json b/data/models/neopolita_jessi-v0.4-falcon3-7b-instruct.json deleted file mode 100644 index dcb6473bed739aaf8fcb1c3173d568ba989644f6..0000000000000000000000000000000000000000 --- a/data/models/neopolita_jessi-v0.4-falcon3-7b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "jessi-v0.4-falcon3-7b-instruct", - "id": "neopolita/jessi-v0.4-falcon3-7b-instruct", - "developer": "neopolita", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "7.456" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/neopolita_jessi-v0.4-falcon3-7b-instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7604 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5522 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3769 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3029 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4971 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4004 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/neopolita_jessi-v0.5-falcon3-7b-instruct.json b/data/models/neopolita_jessi-v0.5-falcon3-7b-instruct.json deleted file mode 100644 index 6ab24bab880a78a8265583b0c91eb151f682a482..0000000000000000000000000000000000000000 --- a/data/models/neopolita_jessi-v0.5-falcon3-7b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "jessi-v0.5-falcon3-7b-instruct", - "id": "neopolita/jessi-v0.5-falcon3-7b-instruct", - "developer": "neopolita", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "7.456" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/neopolita_jessi-v0.5-falcon3-7b-instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7412 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.559 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3739 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3112 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4865 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3966 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/neopolita_jessi-v0.6-falcon3-7b-instruct.json b/data/models/neopolita_jessi-v0.6-falcon3-7b-instruct.json deleted file mode 100644 index 56166f9c4d05147a6bc3a5571855f51422f894ee..0000000000000000000000000000000000000000 --- a/data/models/neopolita_jessi-v0.6-falcon3-7b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "jessi-v0.6-falcon3-7b-instruct", - "id": "neopolita/jessi-v0.6-falcon3-7b-instruct", - "developer": "neopolita", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "7.456" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/neopolita_jessi-v0.6-falcon3-7b-instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7402 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5509 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3565 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3003 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4904 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3957 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/neopolita_loki-v0.1-virtuoso.json b/data/models/neopolita_loki-v0.1-virtuoso.json deleted file mode 100644 index c6de38369ed9ea808a66c149fd2a8479ebae03b6..0000000000000000000000000000000000000000 --- a/data/models/neopolita_loki-v0.1-virtuoso.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "loki-v0.1-virtuoso", - "id": "neopolita/loki-v0.1-virtuoso", - "developer": "neopolita", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/neopolita_loki-v0.1-virtuoso/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7819 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6467 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3391 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3507 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4375 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5129 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/netcat420_deepseek-r1-distill-qwen-mfann-slerp-7b.json b/data/models/netcat420_deepseek-r1-distill-qwen-mfann-slerp-7b.json deleted file mode 100644 index 6f572087217587515dc12dc03bd0c0f0985e7c8b..0000000000000000000000000000000000000000 --- a/data/models/netcat420_deepseek-r1-distill-qwen-mfann-slerp-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "DeepSeek-R1-Distill-Qwen-MFANN-Slerp-7b", - "id": "netcat420/DeepSeek-R1-Distill-Qwen-MFANN-Slerp-7b", - "developer": "netcat420", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/netcat420_DeepSeek-R1-Distill-Qwen-MFANN-Slerp-7b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.115 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2877 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0015 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2643 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3724 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.109 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/netcat420_deepseek-r1-mfann-ties-unretrained-7b.json b/data/models/netcat420_deepseek-r1-mfann-ties-unretrained-7b.json deleted file mode 100644 index 13001241ac05bffcbbb6d88eefcb31540ce9c438..0000000000000000000000000000000000000000 --- a/data/models/netcat420_deepseek-r1-mfann-ties-unretrained-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "DeepSeek-R1-MFANN-TIES-unretrained-7b", - "id": "netcat420/DeepSeek-R1-MFANN-TIES-unretrained-7b", - "developer": "netcat420", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/netcat420_DeepSeek-R1-MFANN-TIES-unretrained-7b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2587 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3086 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0121 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.255 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3527 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1145 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/netcat420_llama3.1-mfann-8b.json b/data/models/netcat420_llama3.1-mfann-8b.json deleted file mode 100644 index 860e8ed541e8056ec931b46c2f7887f9984764b0..0000000000000000000000000000000000000000 --- a/data/models/netcat420_llama3.1-mfann-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama3.1-MFANN-8b", - "id": "netcat420/Llama3.1-MFANN-8b", - "developer": "netcat420", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/netcat420_Llama3.1-MFANN-8b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.297 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4281 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0295 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2878 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3379 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2725 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/netcat420_mfann-abliterated-phi2-merge-unretrained.json b/data/models/netcat420_mfann-abliterated-phi2-merge-unretrained.json deleted file mode 100644 index d0eb7b0c374e2bc37a658facfc443ec92ee63c24..0000000000000000000000000000000000000000 --- a/data/models/netcat420_mfann-abliterated-phi2-merge-unretrained.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MFANN-abliterated-phi2-merge-unretrained", - "id": "netcat420/MFANN-abliterated-phi2-merge-unretrained", - "developer": "netcat420", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "PhiForCausalLM", - "params_billions": "2.775" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/netcat420_MFANN-abliterated-phi2-merge-unretrained/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3005 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4104 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0287 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2609 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3183 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1478 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/netcat420_mfann-llama3.1-abliterated-slerp-ties-v2.json b/data/models/netcat420_mfann-llama3.1-abliterated-slerp-ties-v2.json deleted file mode 100644 index 2ae92e5b9dec55b56484b64172e1b907c0722d00..0000000000000000000000000000000000000000 --- a/data/models/netcat420_mfann-llama3.1-abliterated-slerp-ties-v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MFANN-Llama3.1-Abliterated-SLERP-TIES-V2", - "id": "netcat420/MFANN-Llama3.1-Abliterated-SLERP-TIES-V2", - "developer": "netcat420", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/netcat420_MFANN-Llama3.1-Abliterated-SLERP-TIES-V2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.421 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4924 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0763 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.297 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3728 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3522 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/netcat420_mfann-llama3.1-abliterated-slerp-ties-v3.json b/data/models/netcat420_mfann-llama3.1-abliterated-slerp-ties-v3.json deleted file mode 100644 index 71869c70790dcd4d8b9a619c8be680111a8e2d47..0000000000000000000000000000000000000000 --- a/data/models/netcat420_mfann-llama3.1-abliterated-slerp-ties-v3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MFANN-Llama3.1-Abliterated-SLERP-TIES-V3", - "id": "netcat420/MFANN-Llama3.1-Abliterated-SLERP-TIES-V3", - "developer": "netcat420", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/netcat420_MFANN-Llama3.1-Abliterated-SLERP-TIES-V3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4238 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4914 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0755 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.297 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3741 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.349 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/netcat420_mfann-llama3.1-abliterated-slerp-ties.json b/data/models/netcat420_mfann-llama3.1-abliterated-slerp-ties.json deleted file mode 100644 index 705cf50fe1a3b362248a01612393af9198f00471..0000000000000000000000000000000000000000 --- a/data/models/netcat420_mfann-llama3.1-abliterated-slerp-ties.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MFANN-Llama3.1-Abliterated-Slerp-TIES", - "id": "netcat420/MFANN-Llama3.1-Abliterated-Slerp-TIES", - "developer": "netcat420", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/netcat420_MFANN-Llama3.1-Abliterated-Slerp-TIES/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4293 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4968 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0665 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2919 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3687 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3531 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/netcat420_mfann-llama3.1-abliterated-slerp-v3.1.json b/data/models/netcat420_mfann-llama3.1-abliterated-slerp-v3.1.json deleted file mode 100644 index c7376302b0eb338851b3651b72789fc061556591..0000000000000000000000000000000000000000 --- a/data/models/netcat420_mfann-llama3.1-abliterated-slerp-v3.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MFANN-llama3.1-abliterated-SLERP-v3.1", - "id": "netcat420/MFANN-llama3.1-abliterated-SLERP-v3.1", - "developer": "netcat420", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/netcat420_MFANN-llama3.1-abliterated-SLERP-v3.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4202 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4921 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0695 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2928 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3686 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3543 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/netcat420_mfann-llama3.1-abliterated-slerp-v3.2.json b/data/models/netcat420_mfann-llama3.1-abliterated-slerp-v3.2.json deleted file mode 100644 index de39bc1f2745f8413cce0bd717dc413d7e8b81bd..0000000000000000000000000000000000000000 --- a/data/models/netcat420_mfann-llama3.1-abliterated-slerp-v3.2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MFANN-Llama3.1-Abliterated-Slerp-V3.2", - "id": "netcat420/MFANN-Llama3.1-Abliterated-Slerp-V3.2", - "developer": "netcat420", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/netcat420_MFANN-Llama3.1-Abliterated-Slerp-V3.2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4128 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4978 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0702 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2878 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3754 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3527 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/netcat420_mfann-llama3.1-abliterated-slerp-v3.json b/data/models/netcat420_mfann-llama3.1-abliterated-slerp-v3.json deleted file mode 100644 index c68f5e5f483b94c6bfeef492723b8166f682e4f7..0000000000000000000000000000000000000000 --- a/data/models/netcat420_mfann-llama3.1-abliterated-slerp-v3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MFANN-llama3.1-abliterated-SLERP-v3", - "id": "netcat420/MFANN-llama3.1-abliterated-SLERP-v3", - "developer": "netcat420", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/netcat420_MFANN-llama3.1-abliterated-SLERP-v3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3799 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4931 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0642 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2911 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.366 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3531 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/netcat420_mfann-llama3.1-abliterated-slerp-v4.json b/data/models/netcat420_mfann-llama3.1-abliterated-slerp-v4.json deleted file mode 100644 index 2c3b4af9cc391c254046f7b408329bff59a22fa8..0000000000000000000000000000000000000000 --- a/data/models/netcat420_mfann-llama3.1-abliterated-slerp-v4.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MFANN-Llama3.1-Abliterated-SLERP-V4", - "id": "netcat420/MFANN-Llama3.1-Abliterated-SLERP-V4", - "developer": "netcat420", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/netcat420_MFANN-Llama3.1-Abliterated-SLERP-V4/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4169 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4909 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.068 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3054 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3821 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3516 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/netcat420_mfann-llama3.1-abliterated-slerp-v5.json b/data/models/netcat420_mfann-llama3.1-abliterated-slerp-v5.json deleted file mode 100644 index 6a6d529dade11609ac229ac7c905d83da79e3fc4..0000000000000000000000000000000000000000 --- a/data/models/netcat420_mfann-llama3.1-abliterated-slerp-v5.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MFANN-Llama3.1-Abliterated-SLERP-V5", - "id": "netcat420/MFANN-Llama3.1-Abliterated-SLERP-V5", - "developer": "netcat420", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/netcat420_MFANN-Llama3.1-Abliterated-SLERP-V5/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4329 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4952 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0816 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2936 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3781 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3445 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/netcat420_mfann-llama3.1-abliterated-slerp.json b/data/models/netcat420_mfann-llama3.1-abliterated-slerp.json deleted file mode 100644 index 6f545649c8e799afb4e4a2fe4f355708a7b7707d..0000000000000000000000000000000000000000 --- a/data/models/netcat420_mfann-llama3.1-abliterated-slerp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MFANN-llama3.1-Abliterated-SLERP", - "id": "netcat420/MFANN-llama3.1-Abliterated-SLERP", - "developer": "netcat420", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/netcat420_MFANN-llama3.1-Abliterated-SLERP/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2591 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4574 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0483 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2735 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3809 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2928 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/netcat420_mfann-llama3.1-abliterated-v2.json b/data/models/netcat420_mfann-llama3.1-abliterated-v2.json deleted file mode 100644 index 33eea68b495676ea5bff61aae8482beeb5f2aafd..0000000000000000000000000000000000000000 --- a/data/models/netcat420_mfann-llama3.1-abliterated-v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MFANN-llama3.1-abliterated-v2", - "id": "netcat420/MFANN-llama3.1-abliterated-v2", - "developer": "netcat420", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/netcat420_MFANN-llama3.1-abliterated-v2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4429 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4941 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.074 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2928 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3845 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3491 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/netcat420_mfann-phigments-slerp-v2.json b/data/models/netcat420_mfann-phigments-slerp-v2.json deleted file mode 100644 index f9e8f0e5c362af02abf8c4d20195708a3744ee2b..0000000000000000000000000000000000000000 --- a/data/models/netcat420_mfann-phigments-slerp-v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MFANN-phigments-slerp-V2", - "id": "netcat420/MFANN-phigments-slerp-V2", - "developer": "netcat420", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "PhiForCausalLM", - "params_billions": "2.78" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/netcat420_MFANN-phigments-slerp-V2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3232 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4827 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0317 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2727 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4037 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2717 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/netcat420_mfann-phigments-slerp-v3.2.json b/data/models/netcat420_mfann-phigments-slerp-v3.2.json deleted file mode 100644 index 84847526d8f7f9a0280f2e33a31ee9d1ab6baa57..0000000000000000000000000000000000000000 --- a/data/models/netcat420_mfann-phigments-slerp-v3.2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MFANN-phigments-slerp-V3.2", - "id": "netcat420/MFANN-phigments-slerp-V3.2", - "developer": "netcat420", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "PhiForCausalLM", - "params_billions": "2.78" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/netcat420_MFANN-phigments-slerp-V3.2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3524 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4809 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0332 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2836 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3708 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2705 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/netcat420_mfann-phigments-slerp-v3.3.json b/data/models/netcat420_mfann-phigments-slerp-v3.3.json deleted file mode 100644 index 9784f8006ea9a5cadea4acd57f1101f9c8a4259f..0000000000000000000000000000000000000000 --- a/data/models/netcat420_mfann-phigments-slerp-v3.3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MFANN-phigments-slerp-V3.3", - "id": "netcat420/MFANN-phigments-slerp-V3.3", - "developer": "netcat420", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "PhiForCausalLM", - "params_billions": "2.78" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/netcat420_MFANN-phigments-slerp-V3.3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3691 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4895 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0332 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2752 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3892 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2803 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/netcat420_mfann-sft.json b/data/models/netcat420_mfann-sft.json deleted file mode 100644 index 68de3e8e2a19611f654ff1b9a6d3d08e78253aac..0000000000000000000000000000000000000000 --- a/data/models/netcat420_mfann-sft.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MFANN-SFT", - "id": "netcat420/MFANN-SFT", - "developer": "netcat420", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/netcat420_MFANN-SFT/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3682 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4852 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0597 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3163 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3725 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3336 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/netcat420_mfann3b.json b/data/models/netcat420_mfann3b.json deleted file mode 100644 index fb3a0085a4bca80adcc84b7b17108170b0a5ee97..0000000000000000000000000000000000000000 --- a/data/models/netcat420_mfann3b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MFANN3b", - "id": "netcat420/MFANN3b", - "developer": "netcat420", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "PhiForCausalLM", - "params_billions": "2.78" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/netcat420_MFANN3b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2524 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4433 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0219 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2919 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3606 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2306 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/netcat420_mfann3bv0.15.json b/data/models/netcat420_mfann3bv0.15.json deleted file mode 100644 index b5906d9617d79c90402a171469057b125a2ce1a2..0000000000000000000000000000000000000000 --- a/data/models/netcat420_mfann3bv0.15.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MFANN3bv0.15", - "id": "netcat420/MFANN3bv0.15", - "developer": "netcat420", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "PhiForCausalLM", - "params_billions": "2.78" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/netcat420_MFANN3bv0.15/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2012 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4539 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0264 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2517 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3958 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2468 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/netcat420_mfann3bv0.18.json b/data/models/netcat420_mfann3bv0.18.json deleted file mode 100644 index 9d0f29813d4dd99054e878b3ba4c8259313288b3..0000000000000000000000000000000000000000 --- a/data/models/netcat420_mfann3bv0.18.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MFANN3bv0.18", - "id": "netcat420/MFANN3bv0.18", - "developer": "netcat420", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "PhiForCausalLM", - "params_billions": "2.78" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/netcat420_MFANN3bv0.18/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2206 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4514 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0249 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2576 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4024 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.25 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/netcat420_mfann3bv0.19.json b/data/models/netcat420_mfann3bv0.19.json deleted file mode 100644 index 00167495ad58825fd31b9599935b66487ddbfc6c..0000000000000000000000000000000000000000 --- a/data/models/netcat420_mfann3bv0.19.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MFANN3bv0.19", - "id": "netcat420/MFANN3bv0.19", - "developer": "netcat420", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "PhiForCausalLM", - "params_billions": "2.78" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/netcat420_MFANN3bv0.19/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2258 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4516 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0227 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2576 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4024 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.252 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/netcat420_mfann3bv0.20.json b/data/models/netcat420_mfann3bv0.20.json deleted file mode 100644 index ebd514ee05d2d66ce7aa2a6ca051a1d15c3d884f..0000000000000000000000000000000000000000 --- a/data/models/netcat420_mfann3bv0.20.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MFANN3bv0.20", - "id": "netcat420/MFANN3bv0.20", - "developer": "netcat420", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "PhiForCausalLM", - "params_billions": "2.78" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/netcat420_MFANN3bv0.20/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2193 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4493 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0264 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2592 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4077 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.25 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/netcat420_mfann3bv0.21.json b/data/models/netcat420_mfann3bv0.21.json deleted file mode 100644 index 2559ac92f5a2262dd7aa0a9d0a9348db8630521a..0000000000000000000000000000000000000000 --- a/data/models/netcat420_mfann3bv0.21.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MFANN3bv0.21", - "id": "netcat420/MFANN3bv0.21", - "developer": "netcat420", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "PhiForCausalLM", - "params_billions": "2.78" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/netcat420_MFANN3bv0.21/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1909 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.447 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0317 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2643 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3759 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2393 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/netcat420_mfann3bv0.22.json b/data/models/netcat420_mfann3bv0.22.json deleted file mode 100644 index 0ea18d7e17868e3bf632c54f54ed7c7da183037b..0000000000000000000000000000000000000000 --- a/data/models/netcat420_mfann3bv0.22.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MFANN3bv0.22", - "id": "netcat420/MFANN3bv0.22", - "developer": "netcat420", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "PhiForCausalLM", - "params_billions": "2.78" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/netcat420_MFANN3bv0.22/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1979 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4485 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0264 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2617 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3521 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2517 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/netcat420_mfann3bv0.23.json b/data/models/netcat420_mfann3bv0.23.json deleted file mode 100644 index 5c29e4df9f2956c40faa5c8bd5ba7e57b3caa4ff..0000000000000000000000000000000000000000 --- a/data/models/netcat420_mfann3bv0.23.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MFANN3bv0.23", - "id": "netcat420/MFANN3bv0.23", - "developer": "netcat420", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "PhiForCausalLM", - "params_billions": "2.78" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/netcat420_MFANN3bv0.23/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2048 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4495 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0249 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2517 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3427 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2418 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/netcat420_mfann3bv0.24.json b/data/models/netcat420_mfann3bv0.24.json deleted file mode 100644 index 112609ca70aadd01c896de4a62747fd6e4ea3fae..0000000000000000000000000000000000000000 --- a/data/models/netcat420_mfann3bv0.24.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MFANN3bv0.24", - "id": "netcat420/MFANN3bv0.24", - "developer": "netcat420", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "PhiForCausalLM", - "params_billions": "2.78" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/netcat420_MFANN3bv0.24/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.22 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4407 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0279 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2584 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3521 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2352 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/netcat420_mfann3bv1.1.json b/data/models/netcat420_mfann3bv1.1.json deleted file mode 100644 index dad56c9159983740131576c553f7fcdbeb4447cc..0000000000000000000000000000000000000000 --- a/data/models/netcat420_mfann3bv1.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MFANN3bv1.1", - "id": "netcat420/MFANN3bv1.1", - "developer": "netcat420", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "PhiForCausalLM", - "params_billions": "2.775" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/netcat420_MFANN3bv1.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2507 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3397 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0204 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2668 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3223 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1159 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/netcat420_mfann3bv1.2.json b/data/models/netcat420_mfann3bv1.2.json deleted file mode 100644 index b3cfb2806d95813d2c2ab18267a8919c067e2189..0000000000000000000000000000000000000000 --- a/data/models/netcat420_mfann3bv1.2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MFANN3bv1.2", - "id": "netcat420/MFANN3bv1.2", - "developer": "netcat420", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "PhiForCausalLM", - "params_billions": "2.775" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/netcat420_MFANN3bv1.2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2686 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.366 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0264 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2634 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3156 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.145 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/netcat420_mfann3bv1.3.json b/data/models/netcat420_mfann3bv1.3.json deleted file mode 100644 index 3b309b6f7cb9330ebead280059aa81c0fc72d7e0..0000000000000000000000000000000000000000 --- a/data/models/netcat420_mfann3bv1.3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MFANN3bv1.3", - "id": "netcat420/MFANN3bv1.3", - "developer": "netcat420", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "PhiForCausalLM", - "params_billions": "2.78" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/netcat420_MFANN3bv1.3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2547 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4456 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0211 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2576 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3299 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2276 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/netcat420_mfann3bv1.4.json b/data/models/netcat420_mfann3bv1.4.json deleted file mode 100644 index 023c79b513cdca5afcf8a4631d62457d73c2f5bc..0000000000000000000000000000000000000000 --- a/data/models/netcat420_mfann3bv1.4.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MFANN3bv1.4", - "id": "netcat420/MFANN3bv1.4", - "developer": "netcat420", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "PhiForCausalLM", - "params_billions": "2.78" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/netcat420_MFANN3bv1.4/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3524 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4809 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.037 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2827 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3708 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2705 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/netcat420_mfannv0.19.json b/data/models/netcat420_mfannv0.19.json deleted file mode 100644 index 2cf76fb4ff7af61ee3b94925f44109cd91434ebf..0000000000000000000000000000000000000000 --- a/data/models/netcat420_mfannv0.19.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MFANNv0.19", - "id": "netcat420/MFANNv0.19", - "developer": "netcat420", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/netcat420_MFANNv0.19/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3057 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4731 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0415 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.307 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3527 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2473 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/netcat420_mfannv0.20.json b/data/models/netcat420_mfannv0.20.json deleted file mode 100644 index c0f8cf8602e217a314aa1d0ecc49828683591429..0000000000000000000000000000000000000000 --- a/data/models/netcat420_mfannv0.20.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MFANNv0.20", - "id": "netcat420/MFANNv0.20", - "developer": "netcat420", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/netcat420_MFANNv0.20/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3479 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4574 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0498 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2903 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3874 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3202 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/netcat420_mfannv0.21.json b/data/models/netcat420_mfannv0.21.json deleted file mode 100644 index b44be244aa3409eec36dee66f8bf662e125cb7be..0000000000000000000000000000000000000000 --- a/data/models/netcat420_mfannv0.21.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MFANNv0.21", - "id": "netcat420/MFANNv0.21", - "developer": "netcat420", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/netcat420_MFANNv0.21/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3233 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4576 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0574 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2785 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3993 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3031 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/netcat420_mfannv0.22.1.json b/data/models/netcat420_mfannv0.22.1.json deleted file mode 100644 index 9820e98a5991c5cdfe262118789fda43dd25c122..0000000000000000000000000000000000000000 --- a/data/models/netcat420_mfannv0.22.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MFANNv0.22.1", - "id": "netcat420/MFANNv0.22.1", - "developer": "netcat420", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/netcat420_MFANNv0.22.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3089 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4661 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0536 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.276 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3753 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3343 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/netcat420_mfannv0.23.json b/data/models/netcat420_mfannv0.23.json deleted file mode 100644 index 63a68656f28eb2957e75589230eddb2907388dfb..0000000000000000000000000000000000000000 --- a/data/models/netcat420_mfannv0.23.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MFANNv0.23", - "id": "netcat420/MFANNv0.23", - "developer": "netcat420", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/netcat420_MFANNv0.23/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3127 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4898 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0498 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2844 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3768 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3388 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/netcat420_mfannv0.24.json b/data/models/netcat420_mfannv0.24.json deleted file mode 100644 index 1efd332243a1f53d0574c1c491ddc703c627e83a..0000000000000000000000000000000000000000 --- a/data/models/netcat420_mfannv0.24.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MFANNv0.24", - "id": "netcat420/MFANNv0.24", - "developer": "netcat420", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/netcat420_MFANNv0.24/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3162 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.479 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0612 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2844 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3754 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3348 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/netcat420_mfannv0.25.json b/data/models/netcat420_mfannv0.25.json deleted file mode 100644 index 14deaa393286b0afe78050f243b43a1dd815b7ef..0000000000000000000000000000000000000000 --- a/data/models/netcat420_mfannv0.25.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MFANNv0.25", - "id": "netcat420/MFANNv0.25", - "developer": "netcat420", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/netcat420_MFANNv0.25/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3467 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4794 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0582 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2802 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3688 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3343 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/netcat420_qwen2.5-7b-mfann-slerp.json b/data/models/netcat420_qwen2.5-7b-mfann-slerp.json deleted file mode 100644 index ce191ed019b8c89d74cf0366d60fcb5bef181d5d..0000000000000000000000000000000000000000 --- a/data/models/netcat420_qwen2.5-7b-mfann-slerp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-7b-MFANN-slerp", - "id": "netcat420/Qwen2.5-7b-MFANN-slerp", - "developer": "netcat420", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/netcat420_Qwen2.5-7b-MFANN-slerp/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6532 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5089 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.287 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2953 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4073 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3417 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/netcat420_qwen2.5-7b-nerd-uncensored-mfann-slerp.json b/data/models/netcat420_qwen2.5-7b-nerd-uncensored-mfann-slerp.json deleted file mode 100644 index 4e0d4824ae5f92df6664c9caf5d352c50dec90b1..0000000000000000000000000000000000000000 --- a/data/models/netcat420_qwen2.5-7b-nerd-uncensored-mfann-slerp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-7b-nerd-uncensored-MFANN-slerp", - "id": "netcat420/Qwen2.5-7b-nerd-uncensored-MFANN-slerp", - "developer": "netcat420", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/netcat420_Qwen2.5-7b-nerd-uncensored-MFANN-slerp/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1564 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.292 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2601 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3792 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.11 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/netcat420_qwen2.5-7b-nerd-uncensored-v0.9-mfann.json b/data/models/netcat420_qwen2.5-7b-nerd-uncensored-v0.9-mfann.json deleted file mode 100644 index 89cc5f819320ed4d7e29d7de9a0cbd399f9c8d6b..0000000000000000000000000000000000000000 --- a/data/models/netcat420_qwen2.5-7b-nerd-uncensored-v0.9-mfann.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-7B-nerd-uncensored-v0.9-MFANN", - "id": "netcat420/Qwen2.5-7B-nerd-uncensored-v0.9-MFANN", - "developer": "netcat420", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/netcat420_Qwen2.5-7B-nerd-uncensored-v0.9-MFANN/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5878 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5237 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3376 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.281 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3926 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3904 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/netcat420_qwen2.5-coder-scholar-7b-abliterated-mfann-slerp-unretrained.json b/data/models/netcat420_qwen2.5-coder-scholar-7b-abliterated-mfann-slerp-unretrained.json deleted file mode 100644 index d263a9b26279d30fdb9216220e972716e007808f..0000000000000000000000000000000000000000 --- a/data/models/netcat420_qwen2.5-coder-scholar-7b-abliterated-mfann-slerp-unretrained.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-Coder-Scholar-7B-Abliterated-MFANN-Slerp-Unretrained", - "id": "netcat420/Qwen2.5-Coder-Scholar-7B-Abliterated-MFANN-Slerp-Unretrained", - "developer": "netcat420", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/netcat420_Qwen2.5-Coder-Scholar-7B-Abliterated-MFANN-Slerp-Unretrained/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6486 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5066 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2991 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2987 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4152 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3432 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/netcat420_qwen2.5-coder-scholar-7b-abliterated-mfann.json b/data/models/netcat420_qwen2.5-coder-scholar-7b-abliterated-mfann.json deleted file mode 100644 index af33df977a9fbacd746cdc689bef924d440c99fa..0000000000000000000000000000000000000000 --- a/data/models/netcat420_qwen2.5-coder-scholar-7b-abliterated-mfann.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-Coder-Scholar-7B-Abliterated-MFANN", - "id": "netcat420/Qwen2.5-Coder-Scholar-7B-Abliterated-MFANN", - "developer": "netcat420", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/netcat420_Qwen2.5-Coder-Scholar-7B-Abliterated-MFANN/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5742 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5071 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2568 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2928 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4058 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3157 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/netcat420_qwen2.5-deepseek-r1-mfann-slerp-7b.json b/data/models/netcat420_qwen2.5-deepseek-r1-mfann-slerp-7b.json deleted file mode 100644 index 1eb3f9105dfa82334fbee731736f02effdfe7a1e..0000000000000000000000000000000000000000 --- a/data/models/netcat420_qwen2.5-deepseek-r1-mfann-slerp-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-DeepSeek-R1-MFANN-Slerp-7b", - "id": "netcat420/Qwen2.5-DeepSeek-R1-MFANN-Slerp-7b", - "developer": "netcat420", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/netcat420_Qwen2.5-DeepSeek-R1-MFANN-Slerp-7b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2676 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3789 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0181 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2324 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3528 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1677 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/netcat420_qwen2.5-mfann-7b-slerp-v1.2.json b/data/models/netcat420_qwen2.5-mfann-7b-slerp-v1.2.json deleted file mode 100644 index 2ba9217217eab7ad72298177cc9e2ee5ddc2065b..0000000000000000000000000000000000000000 --- a/data/models/netcat420_qwen2.5-mfann-7b-slerp-v1.2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "qwen2.5-MFANN-7b-SLERP-V1.2", - "id": "netcat420/qwen2.5-MFANN-7b-SLERP-V1.2", - "developer": "netcat420", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/netcat420_qwen2.5-MFANN-7b-SLERP-V1.2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6606 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5111 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.287 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.297 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4259 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3438 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/netcat420_qwen2.5-mfann-7b-slerpv1.1.json b/data/models/netcat420_qwen2.5-mfann-7b-slerpv1.1.json deleted file mode 100644 index ef092503cd08cf3b0c9cc6439aa78fcda630fae3..0000000000000000000000000000000000000000 --- a/data/models/netcat420_qwen2.5-mfann-7b-slerpv1.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "qwen2.5-MFANN-7b-SLERPv1.1", - "id": "netcat420/qwen2.5-MFANN-7b-SLERPv1.1", - "developer": "netcat420", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/netcat420_qwen2.5-MFANN-7b-SLERPv1.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6555 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5075 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2968 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2903 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4126 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3448 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/netcat420_qwen2.5-mfann-7b-v1.1.json b/data/models/netcat420_qwen2.5-mfann-7b-v1.1.json deleted file mode 100644 index 66aca6ec141a3cadb4a94f887ca726818b0fe3a6..0000000000000000000000000000000000000000 --- a/data/models/netcat420_qwen2.5-mfann-7b-v1.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "qwen2.5-MFANN-7b-v1.1", - "id": "netcat420/qwen2.5-MFANN-7b-v1.1", - "developer": "netcat420", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/netcat420_qwen2.5-MFANN-7b-v1.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6088 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4967 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2825 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.276 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4114 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3248 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/netcat420_qwen2.5-mfann-7b.json b/data/models/netcat420_qwen2.5-mfann-7b.json deleted file mode 100644 index 608d0d07136690c528463323817d4a7e76322166..0000000000000000000000000000000000000000 --- a/data/models/netcat420_qwen2.5-mfann-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-MFANN-7b", - "id": "netcat420/Qwen2.5-MFANN-7b", - "developer": "netcat420", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/netcat420_Qwen2.5-MFANN-7b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6097 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5054 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2787 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2861 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4021 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3233 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/netease-youdao_confucius-o1-14b.json b/data/models/netease-youdao_confucius-o1-14b.json deleted file mode 100644 index 89ece1a8f00cac31421cc6dcb0a4d399a687706b..0000000000000000000000000000000000000000 --- a/data/models/netease-youdao_confucius-o1-14b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Confucius-o1-14B", - "id": "netease-youdao/Confucius-o1-14B", - "developer": "netease-youdao", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/netease-youdao_Confucius-o1-14B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6378 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.63 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4313 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3649 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4338 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5265 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/neversleep_lumimaid-v0.2-12b.json b/data/models/neversleep_lumimaid-v0.2-12b.json deleted file mode 100644 index d2f734ab606d48b780dada1f79216271d26b2d4e..0000000000000000000000000000000000000000 --- a/data/models/neversleep_lumimaid-v0.2-12b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Lumimaid-v0.2-12B", - "id": "NeverSleep/Lumimaid-v0.2-12B", - "developer": "NeverSleep", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/NeverSleep_Lumimaid-v0.2-12B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1099 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5396 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0566 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3146 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4821 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3511 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/neversleep_lumimaid-v0.2-8b.json b/data/models/neversleep_lumimaid-v0.2-8b.json deleted file mode 100644 index ed62f5f2f939470830399ed3f2af654c066f79cc..0000000000000000000000000000000000000000 --- a/data/models/neversleep_lumimaid-v0.2-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Lumimaid-v0.2-8B", - "id": "NeverSleep/Lumimaid-v0.2-8B", - "developer": "NeverSleep", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/NeverSleep_Lumimaid-v0.2-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5038 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5238 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1435 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3112 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4303 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3636 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/newsbang_homer-7b-v0.1.json b/data/models/newsbang_homer-7b-v0.1.json deleted file mode 100644 index b9a49b93bf92cab3479b84c58ea30dcc3b6023e1..0000000000000000000000000000000000000000 --- a/data/models/newsbang_homer-7b-v0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Homer-7B-v0.1", - "id": "newsbang/Homer-7B-v0.1", - "developer": "newsbang", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/newsbang_Homer-7B-v0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6109 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5601 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.386 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3247 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4357 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4475 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/newsbang_homer-7b-v0.2.json b/data/models/newsbang_homer-7b-v0.2.json deleted file mode 100644 index d0799b0e189693809865bb225f99c20abc0c366a..0000000000000000000000000000000000000000 --- a/data/models/newsbang_homer-7b-v0.2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Homer-7B-v0.2", - "id": "newsbang/Homer-7B-v0.2", - "developer": "newsbang", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/newsbang_Homer-7B-v0.2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7494 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5517 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2477 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3322 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4298 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.441 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/newsbang_homer-v0.3-qwen2.5-7b.json b/data/models/newsbang_homer-v0.3-qwen2.5-7b.json deleted file mode 100644 index f479af72f3aba6cb84c50f7370a7036b6c7cb373..0000000000000000000000000000000000000000 --- a/data/models/newsbang_homer-v0.3-qwen2.5-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Homer-v0.3-Qwen2.5-7B", - "id": "newsbang/Homer-v0.3-Qwen2.5-7B", - "developer": "newsbang", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/newsbang_Homer-v0.3-Qwen2.5-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5154 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5481 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3089 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3339 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4744 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4456 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/newsbang_homer-v0.4-qwen2.5-7b.json b/data/models/newsbang_homer-v0.4-qwen2.5-7b.json deleted file mode 100644 index 2919ac0b470420f7aabc4ec776b05b43feef4789..0000000000000000000000000000000000000000 --- a/data/models/newsbang_homer-v0.4-qwen2.5-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Homer-v0.4-Qwen2.5-7B", - "id": "newsbang/Homer-v0.4-Qwen2.5-7B", - "developer": "newsbang", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/newsbang_Homer-v0.4-Qwen2.5-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7999 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5533 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2779 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3154 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4311 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4363 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/newsbang_homer-v0.5-qwen2.5-7b.json b/data/models/newsbang_homer-v0.5-qwen2.5-7b.json deleted file mode 100644 index 56d3aa0a971a950132c2d51ebb96048046f2db0e..0000000000000000000000000000000000000000 --- a/data/models/newsbang_homer-v0.5-qwen2.5-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Homer-v0.5-Qwen2.5-7B", - "id": "newsbang/Homer-v0.5-Qwen2.5-7B", - "developer": "newsbang", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/newsbang_Homer-v0.5-Qwen2.5-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7881 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.554 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3724 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3029 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4193 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4369 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/newsbang_homer-v1.0-qwen2.5-72b.json b/data/models/newsbang_homer-v1.0-qwen2.5-72b.json deleted file mode 100644 index 0d698a7126754a2ce029c6963b62b0ef9844177e..0000000000000000000000000000000000000000 --- a/data/models/newsbang_homer-v1.0-qwen2.5-72b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Homer-v1.0-Qwen2.5-72B", - "id": "newsbang/Homer-v1.0-Qwen2.5-72B", - "developer": "newsbang", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "72.706" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/newsbang_Homer-v1.0-Qwen2.5-72B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7628 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.731 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4902 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4161 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4677 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6145 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/newsbang_homer-v1.0-qwen2.5-7b.json b/data/models/newsbang_homer-v1.0-qwen2.5-7b.json deleted file mode 100644 index 63696c75c1ff3d2d1bc3a59d069fe9dbd7ee1e89..0000000000000000000000000000000000000000 --- a/data/models/newsbang_homer-v1.0-qwen2.5-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Homer-v1.0-Qwen2.5-7B", - "id": "newsbang/Homer-v1.0-Qwen2.5-7B", - "developer": "newsbang", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/newsbang_Homer-v1.0-Qwen2.5-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6393 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5655 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3323 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3221 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4278 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4535 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nexesenex_dolphin3.0-llama3.1-1b-abliterated.json b/data/models/nexesenex_dolphin3.0-llama3.1-1b-abliterated.json deleted file mode 100644 index 1e9abed02a8c4cb252d64e3603b427c07f80d152..0000000000000000000000000000000000000000 --- a/data/models/nexesenex_dolphin3.0-llama3.1-1b-abliterated.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Dolphin3.0-Llama3.1-1B-abliterated", - "id": "Nexesenex/Dolphin3.0-Llama3.1-1B-abliterated", - "developer": "Nexesenex", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.236" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Nexesenex_Dolphin3.0-Llama3.1-1B-abliterated/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5312 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3241 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0385 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2408 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3237 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1373 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nexesenex_llama_3.1_8b_deepdive_3_prev_v1.0.json b/data/models/nexesenex_llama_3.1_8b_deepdive_3_prev_v1.0.json deleted file mode 100644 index 0eaaccdf6bc93a8f71c84794299964b1e6471330..0000000000000000000000000000000000000000 --- a/data/models/nexesenex_llama_3.1_8b_deepdive_3_prev_v1.0.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama_3.1_8b_DeepDive_3_Prev_v1.0", - "id": "Nexesenex/Llama_3.1_8b_DeepDive_3_Prev_v1.0", - "developer": "Nexesenex", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Nexesenex_Llama_3.1_8b_DeepDive_3_Prev_v1.0/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6809 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5155 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1866 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2911 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3666 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3438 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nexesenex_llama_3.1_8b_deepdive_3_r1_prev_v1.0.json b/data/models/nexesenex_llama_3.1_8b_deepdive_3_r1_prev_v1.0.json deleted file mode 100644 index 173339797872debd91219e9d14bf471ed0012c35..0000000000000000000000000000000000000000 --- a/data/models/nexesenex_llama_3.1_8b_deepdive_3_r1_prev_v1.0.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama_3.1_8b_DeepDive_3_R1_Prev_v1.0", - "id": "Nexesenex/Llama_3.1_8b_DeepDive_3_R1_Prev_v1.0", - "developer": "Nexesenex", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Nexesenex_Llama_3.1_8b_DeepDive_3_R1_Prev_v1.0/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7101 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.512 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1926 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3003 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3758 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3441 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nexesenex_llama_3.1_8b_doberwild_v2.01.json b/data/models/nexesenex_llama_3.1_8b_doberwild_v2.01.json deleted file mode 100644 index cb6a7f9133b77e8c3bb169587dde8f5e4b3ab2dc..0000000000000000000000000000000000000000 --- a/data/models/nexesenex_llama_3.1_8b_doberwild_v2.01.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama_3.1_8b_DoberWild_v2.01", - "id": "Nexesenex/Llama_3.1_8b_DoberWild_v2.01", - "developer": "Nexesenex", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.031" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Nexesenex_Llama_3.1_8b_DoberWild_v2.01/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7996 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5251 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2002 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3029 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4012 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3791 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nexesenex_llama_3.1_8b_doberwild_v2.03.json b/data/models/nexesenex_llama_3.1_8b_doberwild_v2.03.json deleted file mode 100644 index e3673f85a31c1b465a04d3d94ebc331ef967e0ea..0000000000000000000000000000000000000000 --- a/data/models/nexesenex_llama_3.1_8b_doberwild_v2.03.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama_3.1_8b_DoberWild_v2.03", - "id": "Nexesenex/Llama_3.1_8b_DoberWild_v2.03", - "developer": "Nexesenex", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Nexesenex_Llama_3.1_8b_DoberWild_v2.03/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7764 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5294 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2077 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3045 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3906 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3722 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nexesenex_llama_3.1_8b_dobherwild_r1_v1.1r.json b/data/models/nexesenex_llama_3.1_8b_dobherwild_r1_v1.1r.json deleted file mode 100644 index 93da9fd4126f171532de05bc65c7032be8c2c0da..0000000000000000000000000000000000000000 --- a/data/models/nexesenex_llama_3.1_8b_dobherwild_r1_v1.1r.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama_3.1_8b_DobHerWild_R1_v1.1R", - "id": "Nexesenex/Llama_3.1_8b_DobHerWild_R1_v1.1R", - "developer": "Nexesenex", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Nexesenex_Llama_3.1_8b_DobHerWild_R1_v1.1R/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.76 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5257 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2319 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2995 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3852 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3688 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nexesenex_llama_3.1_8b_dodowild_v2.01.json b/data/models/nexesenex_llama_3.1_8b_dodowild_v2.01.json deleted file mode 100644 index 938fbc038a5735cf42561d0f9f84f176c231b714..0000000000000000000000000000000000000000 --- a/data/models/nexesenex_llama_3.1_8b_dodowild_v2.01.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama_3.1_8b_DodoWild_v2.01", - "id": "Nexesenex/Llama_3.1_8b_DodoWild_v2.01", - "developer": "Nexesenex", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.031" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Nexesenex_Llama_3.1_8b_DodoWild_v2.01/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7978 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5253 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1986 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3037 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.409 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3738 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nexesenex_llama_3.1_8b_dodowild_v2.02.json b/data/models/nexesenex_llama_3.1_8b_dodowild_v2.02.json deleted file mode 100644 index 667196bf98f688cfbf42c7b35d0de7865a79f2cf..0000000000000000000000000000000000000000 --- a/data/models/nexesenex_llama_3.1_8b_dodowild_v2.02.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama_3.1_8b_DodoWild_v2.02", - "id": "Nexesenex/Llama_3.1_8b_DodoWild_v2.02", - "developer": "Nexesenex", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Nexesenex_Llama_3.1_8b_DodoWild_v2.02/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8017 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5262 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2273 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3045 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3971 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3761 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nexesenex_llama_3.1_8b_dodowild_v2.03.json b/data/models/nexesenex_llama_3.1_8b_dodowild_v2.03.json deleted file mode 100644 index 2d944b8c26035e62359e7522148e30c492a97373..0000000000000000000000000000000000000000 --- a/data/models/nexesenex_llama_3.1_8b_dodowild_v2.03.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama_3.1_8b_DodoWild_v2.03", - "id": "Nexesenex/Llama_3.1_8b_DodoWild_v2.03", - "developer": "Nexesenex", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Nexesenex_Llama_3.1_8b_DodoWild_v2.03/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7941 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5308 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2221 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3079 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3959 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3786 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nexesenex_llama_3.1_8b_dodowild_v2.10.json b/data/models/nexesenex_llama_3.1_8b_dodowild_v2.10.json deleted file mode 100644 index a0153d99741cc9e7183f0664a8a1fc879491f032..0000000000000000000000000000000000000000 --- a/data/models/nexesenex_llama_3.1_8b_dodowild_v2.10.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama_3.1_8b_DodoWild_v2.10", - "id": "Nexesenex/Llama_3.1_8b_DodoWild_v2.10", - "developer": "Nexesenex", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Nexesenex_Llama_3.1_8b_DodoWild_v2.10/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8054 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5278 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1971 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2961 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4157 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3855 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nexesenex_llama_3.1_8b_dolermed_r1_v1.01.json b/data/models/nexesenex_llama_3.1_8b_dolermed_r1_v1.01.json deleted file mode 100644 index e65794079b36a5d06bd378f707a49a16cef7b18a..0000000000000000000000000000000000000000 --- a/data/models/nexesenex_llama_3.1_8b_dolermed_r1_v1.01.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama_3.1_8b_Dolermed_R1_V1.01", - "id": "Nexesenex/Llama_3.1_8b_Dolermed_R1_V1.01", - "developer": "Nexesenex", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Nexesenex_Llama_3.1_8b_Dolermed_R1_V1.01/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7534 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5312 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2017 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3054 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3747 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3733 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nexesenex_llama_3.1_8b_dolermed_r1_v1.03.json b/data/models/nexesenex_llama_3.1_8b_dolermed_r1_v1.03.json deleted file mode 100644 index d507ee4100b5a4b0b830fb7545a86b0b0dfdba7c..0000000000000000000000000000000000000000 --- a/data/models/nexesenex_llama_3.1_8b_dolermed_r1_v1.03.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama_3.1_8b_Dolermed_R1_V1.03", - "id": "Nexesenex/Llama_3.1_8b_Dolermed_R1_V1.03", - "developer": "Nexesenex", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Nexesenex_Llama_3.1_8b_Dolermed_R1_V1.03/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7564 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5316 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2092 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.318 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.38 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.372 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nexesenex_llama_3.1_8b_dolermed_v1.01.json b/data/models/nexesenex_llama_3.1_8b_dolermed_v1.01.json deleted file mode 100644 index 826375644687d3a15179e357dbbdc2b2ac054bf2..0000000000000000000000000000000000000000 --- a/data/models/nexesenex_llama_3.1_8b_dolermed_v1.01.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama_3.1_8b_Dolermed_V1.01", - "id": "Nexesenex/Llama_3.1_8b_Dolermed_V1.01", - "developer": "Nexesenex", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.031" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Nexesenex_Llama_3.1_8b_Dolermed_V1.01/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5087 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5194 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1344 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2945 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3945 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.357 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nexesenex_llama_3.1_8b_dolerstormed_v1.04.json b/data/models/nexesenex_llama_3.1_8b_dolerstormed_v1.04.json deleted file mode 100644 index 2e478f5be5c41b3527478fd1955822988addb8ab..0000000000000000000000000000000000000000 --- a/data/models/nexesenex_llama_3.1_8b_dolerstormed_v1.04.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama_3.1_8b_Dolerstormed_V1.04", - "id": "Nexesenex/Llama_3.1_8b_Dolerstormed_V1.04", - "developer": "Nexesenex", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Nexesenex_Llama_3.1_8b_Dolerstormed_V1.04/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7889 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5195 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1926 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3221 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.403 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3889 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nexesenex_llama_3.1_8b_hermedash_r1_v1.04.json b/data/models/nexesenex_llama_3.1_8b_hermedash_r1_v1.04.json deleted file mode 100644 index 6128fa015628c057784695fa1b4f2d60db43883b..0000000000000000000000000000000000000000 --- a/data/models/nexesenex_llama_3.1_8b_hermedash_r1_v1.04.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama_3.1_8b_Hermedash_R1_V1.04", - "id": "Nexesenex/Llama_3.1_8b_Hermedash_R1_V1.04", - "developer": "Nexesenex", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Nexesenex_Llama_3.1_8b_Hermedash_R1_V1.04/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7872 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5192 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1866 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.323 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4111 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3882 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nexesenex_llama_3.1_8b_hermedive_r1_v1.01.json b/data/models/nexesenex_llama_3.1_8b_hermedive_r1_v1.01.json deleted file mode 100644 index 677b6bb606880e49a51e0491d6ac648837f6bba2..0000000000000000000000000000000000000000 --- a/data/models/nexesenex_llama_3.1_8b_hermedive_r1_v1.01.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama_3.1_8b_Hermedive_R1_V1.01", - "id": "Nexesenex/Llama_3.1_8b_Hermedive_R1_V1.01", - "developer": "Nexesenex", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Nexesenex_Llama_3.1_8b_Hermedive_R1_V1.01/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5001 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5171 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1775 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2827 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4008 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3427 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nexesenex_llama_3.1_8b_hermedive_r1_v1.03.json b/data/models/nexesenex_llama_3.1_8b_hermedive_r1_v1.03.json deleted file mode 100644 index 5d508accff2469c0311d7d1c6c31417942987945..0000000000000000000000000000000000000000 --- a/data/models/nexesenex_llama_3.1_8b_hermedive_r1_v1.03.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama_3.1_8b_Hermedive_R1_V1.03", - "id": "Nexesenex/Llama_3.1_8b_Hermedive_R1_V1.03", - "developer": "Nexesenex", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Nexesenex_Llama_3.1_8b_Hermedive_R1_V1.03/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6648 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5141 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1858 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2978 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3613 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3488 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nexesenex_llama_3.1_8b_hermedive_v1.01.json b/data/models/nexesenex_llama_3.1_8b_hermedive_v1.01.json deleted file mode 100644 index e9b7d503a1529f60ae3d1abfc87f2306545bd583..0000000000000000000000000000000000000000 --- a/data/models/nexesenex_llama_3.1_8b_hermedive_v1.01.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama_3.1_8b_Hermedive_V1.01", - "id": "Nexesenex/Llama_3.1_8b_Hermedive_V1.01", - "developer": "Nexesenex", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.031" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Nexesenex_Llama_3.1_8b_Hermedive_V1.01/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5062 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4918 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1647 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2894 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3697 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3551 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nexesenex_llama_3.1_8b_mediver_v1.01.json b/data/models/nexesenex_llama_3.1_8b_mediver_v1.01.json deleted file mode 100644 index 36087566b72d0e84a6707e9164476edfd0326624..0000000000000000000000000000000000000000 --- a/data/models/nexesenex_llama_3.1_8b_mediver_v1.01.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama_3.1_8b_Mediver_V1.01", - "id": "Nexesenex/Llama_3.1_8b_Mediver_V1.01", - "developer": "Nexesenex", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.031" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Nexesenex_Llama_3.1_8b_Mediver_V1.01/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1885 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4415 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0015 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2777 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3898 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2994 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nexesenex_llama_3.1_8b_medusa_v1.01.json b/data/models/nexesenex_llama_3.1_8b_medusa_v1.01.json deleted file mode 100644 index 3e4c6655ba8a3952f333dfdc0fe9a2dd1e66066a..0000000000000000000000000000000000000000 --- a/data/models/nexesenex_llama_3.1_8b_medusa_v1.01.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama_3.1_8b_Medusa_v1.01", - "id": "Nexesenex/Llama_3.1_8b_Medusa_v1.01", - "developer": "Nexesenex", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.031" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Nexesenex_Llama_3.1_8b_Medusa_v1.01/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7685 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5018 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1465 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2919 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4067 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3531 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nexesenex_llama_3.1_8b_smarteaz_0.2_r1.json b/data/models/nexesenex_llama_3.1_8b_smarteaz_0.2_r1.json deleted file mode 100644 index 6a06ad443ac8d9b88f08ea1041a06d7bce5c2139..0000000000000000000000000000000000000000 --- a/data/models/nexesenex_llama_3.1_8b_smarteaz_0.2_r1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama_3.1_8b_Smarteaz_0.2_R1", - "id": "Nexesenex/Llama_3.1_8b_Smarteaz_0.2_R1", - "developer": "Nexesenex", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Nexesenex_Llama_3.1_8b_Smarteaz_0.2_R1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6346 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5113 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2606 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3003 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4188 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3645 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nexesenex_llama_3.1_8b_smarteaz_v1.01.json b/data/models/nexesenex_llama_3.1_8b_smarteaz_v1.01.json deleted file mode 100644 index c45eebfb246bd2a4a6f1aefa226a161ee27e9c46..0000000000000000000000000000000000000000 --- a/data/models/nexesenex_llama_3.1_8b_smarteaz_v1.01.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama_3.1_8b_Smarteaz_V1.01", - "id": "Nexesenex/Llama_3.1_8b_Smarteaz_V1.01", - "developer": "Nexesenex", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Nexesenex_Llama_3.1_8b_Smarteaz_V1.01/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8151 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5241 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2341 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3096 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3789 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3736 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nexesenex_llama_3.1_8b_stormeder_v1.04.json b/data/models/nexesenex_llama_3.1_8b_stormeder_v1.04.json deleted file mode 100644 index f5bcbd5fd285b814336cbb2bcd1ecb7ae7bd5dd9..0000000000000000000000000000000000000000 --- a/data/models/nexesenex_llama_3.1_8b_stormeder_v1.04.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama_3.1_8b_Stormeder_v1.04", - "id": "Nexesenex/Llama_3.1_8b_Stormeder_v1.04", - "developer": "Nexesenex", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Nexesenex_Llama_3.1_8b_Stormeder_v1.04/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7853 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5207 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.185 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3205 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3949 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3852 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nexesenex_llama_3.1_8b_typhoon_v1.03.json b/data/models/nexesenex_llama_3.1_8b_typhoon_v1.03.json deleted file mode 100644 index d9c477cb34737eb9799898859841088d1af9bdc8..0000000000000000000000000000000000000000 --- a/data/models/nexesenex_llama_3.1_8b_typhoon_v1.03.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama_3.1_8b_Typhoon_v1.03", - "id": "Nexesenex/Llama_3.1_8b_Typhoon_v1.03", - "developer": "Nexesenex", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Nexesenex_Llama_3.1_8b_Typhoon_v1.03/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8078 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5314 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2273 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.307 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3815 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3842 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nexesenex_llama_3.2_1b_aquasyn_0.1.json b/data/models/nexesenex_llama_3.2_1b_aquasyn_0.1.json deleted file mode 100644 index 8ee1e132de28ad5e05b691c9ec9236efca15a284..0000000000000000000000000000000000000000 --- a/data/models/nexesenex_llama_3.2_1b_aquasyn_0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama_3.2_1b_AquaSyn_0.1", - "id": "Nexesenex/Llama_3.2_1b_AquaSyn_0.1", - "developer": "Nexesenex", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.498" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Nexesenex_Llama_3.2_1b_AquaSyn_0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2741 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3284 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0219 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2483 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.346 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1378 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nexesenex_llama_3.2_1b_aquasyn_0.11.json b/data/models/nexesenex_llama_3.2_1b_aquasyn_0.11.json deleted file mode 100644 index f795cc2489d68b34c28738f7ebb126cc99ab47ba..0000000000000000000000000000000000000000 --- a/data/models/nexesenex_llama_3.2_1b_aquasyn_0.11.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama_3.2_1b_AquaSyn_0.11", - "id": "Nexesenex/Llama_3.2_1b_AquaSyn_0.11", - "developer": "Nexesenex", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.498" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Nexesenex_Llama_3.2_1b_AquaSyn_0.11/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2431 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3112 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0234 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2651 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3368 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1116 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nexesenex_llama_3.2_1b_dolto_0.1.json b/data/models/nexesenex_llama_3.2_1b_dolto_0.1.json deleted file mode 100644 index 04fc28fe65c892efc31eebb7be4eabb5f743851f..0000000000000000000000000000000000000000 --- a/data/models/nexesenex_llama_3.2_1b_dolto_0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama_3.2_1b_Dolto_0.1", - "id": "Nexesenex/Llama_3.2_1b_Dolto_0.1", - "developer": "Nexesenex", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.498" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Nexesenex_Llama_3.2_1b_Dolto_0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5434 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.335 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.037 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2374 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3421 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1364 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nexesenex_llama_3.2_1b_odyssea_v1.01.json b/data/models/nexesenex_llama_3.2_1b_odyssea_v1.01.json deleted file mode 100644 index 2bca32c4c027a804c9dedd744ab34f457b4bbaf2..0000000000000000000000000000000000000000 --- a/data/models/nexesenex_llama_3.2_1b_odyssea_v1.01.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama_3.2_1b_Odyssea_V1.01", - "id": "Nexesenex/Llama_3.2_1b_Odyssea_V1.01", - "developer": "Nexesenex", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.498" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Nexesenex_Llama_3.2_1b_Odyssea_V1.01/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2495 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3045 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0174 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2559 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.342 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1152 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nexesenex_llama_3.2_1b_odyssea_v1.json b/data/models/nexesenex_llama_3.2_1b_odyssea_v1.json deleted file mode 100644 index f2dae9c0c406db13c91d896c9035d8791c2e8e84..0000000000000000000000000000000000000000 --- a/data/models/nexesenex_llama_3.2_1b_odyssea_v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama_3.2_1b_Odyssea_V1", - "id": "Nexesenex/Llama_3.2_1b_Odyssea_V1", - "developer": "Nexesenex", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.498" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Nexesenex_Llama_3.2_1b_Odyssea_V1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2553 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.301 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0144 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2584 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3394 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1153 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nexesenex_llama_3.2_1b_opentree_r1_0.1.json b/data/models/nexesenex_llama_3.2_1b_opentree_r1_0.1.json deleted file mode 100644 index 15708108fb242830858b8228b177eb9829ec31d8..0000000000000000000000000000000000000000 --- a/data/models/nexesenex_llama_3.2_1b_opentree_r1_0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama_3.2_1b_OpenTree_R1_0.1", - "id": "Nexesenex/Llama_3.2_1b_OpenTree_R1_0.1", - "developer": "Nexesenex", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.498" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Nexesenex_Llama_3.2_1b_OpenTree_R1_0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5366 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.328 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0476 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2525 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3131 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1675 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nexesenex_llama_3.2_1b_orcasun_v1.json b/data/models/nexesenex_llama_3.2_1b_orcasun_v1.json deleted file mode 100644 index 63f8dc3de1a499f3f149f2b9cd43b06778f4e5da..0000000000000000000000000000000000000000 --- a/data/models/nexesenex_llama_3.2_1b_orcasun_v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama_3.2_1b_OrcaSun_V1", - "id": "Nexesenex/Llama_3.2_1b_OrcaSun_V1", - "developer": "Nexesenex", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.498" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Nexesenex_Llama_3.2_1b_OrcaSun_V1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5949 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.355 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0597 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2366 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.338 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1904 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nexesenex_llama_3.2_1b_randomlego_rp_r1_0.1.json b/data/models/nexesenex_llama_3.2_1b_randomlego_rp_r1_0.1.json deleted file mode 100644 index c14a2d557dbacf8da22d1d98681fc17ebda0eedd..0000000000000000000000000000000000000000 --- a/data/models/nexesenex_llama_3.2_1b_randomlego_rp_r1_0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama_3.2_1b_RandomLego_RP_R1_0.1", - "id": "Nexesenex/Llama_3.2_1b_RandomLego_RP_R1_0.1", - "developer": "Nexesenex", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.498" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Nexesenex_Llama_3.2_1b_RandomLego_RP_R1_0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5543 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3428 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0566 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.25 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3249 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1563 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nexesenex_llama_3.2_1b_sunorca_v1.json b/data/models/nexesenex_llama_3.2_1b_sunorca_v1.json deleted file mode 100644 index df2edcc0b3387a4039efa4c2d8b3415fb630da4c..0000000000000000000000000000000000000000 --- a/data/models/nexesenex_llama_3.2_1b_sunorca_v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama_3.2_1b_SunOrca_V1", - "id": "Nexesenex/Llama_3.2_1b_SunOrca_V1", - "developer": "Nexesenex", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.498" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Nexesenex_Llama_3.2_1b_SunOrca_V1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.543 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3431 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0672 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2743 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3262 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1884 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nexesenex_llama_3.2_1b_sydonia_0.1.json b/data/models/nexesenex_llama_3.2_1b_sydonia_0.1.json deleted file mode 100644 index d4e61b95b3c29e953f23a4027088b0a3894786ac..0000000000000000000000000000000000000000 --- a/data/models/nexesenex_llama_3.2_1b_sydonia_0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama_3.2_1b_Sydonia_0.1", - "id": "Nexesenex/Llama_3.2_1b_Sydonia_0.1", - "developer": "Nexesenex", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.498" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Nexesenex_Llama_3.2_1b_Sydonia_0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2197 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3121 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0204 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2282 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3382 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1224 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nexesenex_llama_3.2_1b_syneridol_0.2.json b/data/models/nexesenex_llama_3.2_1b_syneridol_0.2.json deleted file mode 100644 index 17130a187fd5cf3066e0f811e6200e9b9b069603..0000000000000000000000000000000000000000 --- a/data/models/nexesenex_llama_3.2_1b_syneridol_0.2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama_3.2_1b_Syneridol_0.2", - "id": "Nexesenex/Llama_3.2_1b_Syneridol_0.2", - "developer": "Nexesenex", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.498" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Nexesenex_Llama_3.2_1b_Syneridol_0.2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2157 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3139 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0219 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2349 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3343 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1227 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nexesenex_llama_3.2_1b_synopsys_0.1.json b/data/models/nexesenex_llama_3.2_1b_synopsys_0.1.json deleted file mode 100644 index 60ac8def76b90cc63919bc0c05cc37aae0d90ab0..0000000000000000000000000000000000000000 --- a/data/models/nexesenex_llama_3.2_1b_synopsys_0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama_3.2_1b_Synopsys_0.1", - "id": "Nexesenex/Llama_3.2_1b_Synopsys_0.1", - "developer": "Nexesenex", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.498" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Nexesenex_Llama_3.2_1b_Synopsys_0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1764 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3162 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0166 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2391 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3461 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1231 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nexesenex_llama_3.2_1b_synopsys_0.11.json b/data/models/nexesenex_llama_3.2_1b_synopsys_0.11.json deleted file mode 100644 index 75780ed34291b4481501676ba084595d65c0e439..0000000000000000000000000000000000000000 --- a/data/models/nexesenex_llama_3.2_1b_synopsys_0.11.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama_3.2_1b_Synopsys_0.11", - "id": "Nexesenex/Llama_3.2_1b_Synopsys_0.11", - "developer": "Nexesenex", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.498" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Nexesenex_Llama_3.2_1b_Synopsys_0.11/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2842 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3102 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0128 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2626 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3513 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1123 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nexesenex_llama_3.2_3b_kermes_v1.json b/data/models/nexesenex_llama_3.2_3b_kermes_v1.json deleted file mode 100644 index 766b1e563fe0b42ec90b539cd847bd6d50a71d89..0000000000000000000000000000000000000000 --- a/data/models/nexesenex_llama_3.2_3b_kermes_v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama_3.2_3b_Kermes_v1", - "id": "Nexesenex/Llama_3.2_3b_Kermes_v1", - "developer": "Nexesenex", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Nexesenex_Llama_3.2_3b_Kermes_v1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4852 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.441 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.031 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2735 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.407 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2547 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nexesenex_llama_3.2_3b_kermes_v2.1.json b/data/models/nexesenex_llama_3.2_3b_kermes_v2.1.json deleted file mode 100644 index 0e9090dd9ebbc1133018b60defe5d7217549f4d2..0000000000000000000000000000000000000000 --- a/data/models/nexesenex_llama_3.2_3b_kermes_v2.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama_3.2_3b_Kermes_v2.1", - "id": "Nexesenex/Llama_3.2_3b_Kermes_v2.1", - "developer": "Nexesenex", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Nexesenex_Llama_3.2_3b_Kermes_v2.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5584 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4464 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0521 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2794 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3964 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2692 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nexesenex_llama_3.2_3b_kermes_v2.json b/data/models/nexesenex_llama_3.2_3b_kermes_v2.json deleted file mode 100644 index dede12a1e1eadca45f3c6afc46658407f05ada3d..0000000000000000000000000000000000000000 --- a/data/models/nexesenex_llama_3.2_3b_kermes_v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama_3.2_3b_Kermes_v2", - "id": "Nexesenex/Llama_3.2_3b_Kermes_v2", - "developer": "Nexesenex", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Nexesenex_Llama_3.2_3b_Kermes_v2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5754 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4455 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0544 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2651 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3778 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2734 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nexesenex_nemotron_w_4b_halo_0.1.json b/data/models/nexesenex_nemotron_w_4b_halo_0.1.json deleted file mode 100644 index 28297651cd610320ebd4781eaa88c3792c2751f5..0000000000000000000000000000000000000000 --- a/data/models/nexesenex_nemotron_w_4b_halo_0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Nemotron_W_4b_Halo_0.1", - "id": "Nexesenex/Nemotron_W_4b_Halo_0.1", - "developer": "Nexesenex", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "4.513" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Nexesenex_Nemotron_W_4b_Halo_0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3627 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4135 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0423 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2802 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4165 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2505 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nexesenex_nemotron_w_4b_maglight_0.1.json b/data/models/nexesenex_nemotron_w_4b_maglight_0.1.json deleted file mode 100644 index 1f2c25a3afd2ba707be0576ac30f3c821278dade..0000000000000000000000000000000000000000 --- a/data/models/nexesenex_nemotron_w_4b_maglight_0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Nemotron_W_4b_MagLight_0.1", - "id": "Nexesenex/Nemotron_W_4b_MagLight_0.1", - "developer": "Nexesenex", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "4.513" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Nexesenex_Nemotron_W_4b_MagLight_0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.423 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4231 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.04 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2836 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4112 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2545 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nexesenex_pankajmathur_orca_mini_v9_6_1b-instruct-abliterated-lpl.json b/data/models/nexesenex_pankajmathur_orca_mini_v9_6_1b-instruct-abliterated-lpl.json deleted file mode 100644 index 1c1547062f9806e849e23118a58b93858fec3fb2..0000000000000000000000000000000000000000 --- a/data/models/nexesenex_pankajmathur_orca_mini_v9_6_1b-instruct-abliterated-lpl.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "pankajmathur_orca_mini_v9_6_1B-instruct-Abliterated-LPL", - "id": "Nexesenex/pankajmathur_orca_mini_v9_6_1B-instruct-Abliterated-LPL", - "developer": "Nexesenex", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.236" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Nexesenex_pankajmathur_orca_mini_v9_6_1B-instruct-Abliterated-LPL/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.589 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3562 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0748 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2668 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3396 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1803 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nexesenex_qwen_2.5_3b_smarteaz_0.01a.json b/data/models/nexesenex_qwen_2.5_3b_smarteaz_0.01a.json deleted file mode 100644 index 3a9dd37f2cc336a1c29d787ea3229dfa550377c7..0000000000000000000000000000000000000000 --- a/data/models/nexesenex_qwen_2.5_3b_smarteaz_0.01a.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen_2.5_3b_Smarteaz_0.01a", - "id": "Nexesenex/Qwen_2.5_3b_Smarteaz_0.01a", - "developer": "Nexesenex", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.085" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Nexesenex_Qwen_2.5_3b_Smarteaz_0.01a/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4012 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4637 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1805 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2777 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.432 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.286 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nexusflow_nexusraven-v2-13b.json b/data/models/nexusflow_nexusraven-v2-13b.json deleted file mode 100644 index 25ce41a434bc18e337a06e66eae8d986218f4a64..0000000000000000000000000000000000000000 --- a/data/models/nexusflow_nexusraven-v2-13b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "NexusRaven-V2-13B", - "id": "Nexusflow/NexusRaven-V2-13B", - "developer": "Nexusflow", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "13.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Nexusflow_NexusRaven-V2-13B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1791 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3949 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0295 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2601 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3737 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1872 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nexusflow_starling-rm-34b.json b/data/models/nexusflow_starling-rm-34b.json deleted file mode 100644 index 8ab4f392fbf3047e7bcb88c6ed9f3a2b6d8e5a37..0000000000000000000000000000000000000000 --- a/data/models/nexusflow_starling-rm-34b.json +++ /dev/null @@ -1,294 +0,0 @@ -{ - "model_info": { - "name": "Nexusflow/Starling-RM-34B", - "id": "Nexusflow/Starling-RM-34B", - "developer": "Nexusflow", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/Nexusflow_Starling-RM-34B/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4553 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4589 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3187 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6175 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7556 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4808 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1004 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "reward-bench/Nexusflow_Starling-RM-34B/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8133 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9693 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5724 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.877 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8845 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7137 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nguyentd_financialadvice-qwen2.5-7b.json b/data/models/nguyentd_financialadvice-qwen2.5-7b.json deleted file mode 100644 index fe28e8186ad445d8f2e015fcc0b03c0a54803270..0000000000000000000000000000000000000000 --- a/data/models/nguyentd_financialadvice-qwen2.5-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "FinancialAdvice-Qwen2.5-7B", - "id": "nguyentd/FinancialAdvice-Qwen2.5-7B", - "developer": "nguyentd", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nguyentd_FinancialAdvice-Qwen2.5-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4496 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4731 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1148 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2945 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4025 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3752 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ngxson_minithinky-1b-llama-3.2.json b/data/models/ngxson_minithinky-1b-llama-3.2.json deleted file mode 100644 index f412e655fa1b13066c79b720bd1285c3b3f110eb..0000000000000000000000000000000000000000 --- a/data/models/ngxson_minithinky-1b-llama-3.2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MiniThinky-1B-Llama-3.2", - "id": "ngxson/MiniThinky-1B-Llama-3.2", - "developer": "ngxson", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.236" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ngxson_MiniThinky-1B-Llama-3.2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2771 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3142 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0574 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2391 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3434 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1147 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ngxson_minithinky-v2-1b-llama-3.2.json b/data/models/ngxson_minithinky-v2-1b-llama-3.2.json deleted file mode 100644 index a6239c13f8c80023189ac87f26dd814cd032beca..0000000000000000000000000000000000000000 --- a/data/models/ngxson_minithinky-v2-1b-llama-3.2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MiniThinky-v2-1B-Llama-3.2", - "id": "ngxson/MiniThinky-v2-1B-Llama-3.2", - "developer": "ngxson", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.236" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ngxson_MiniThinky-v2-1B-Llama-3.2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2963 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3205 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0287 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2399 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3356 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1116 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nhyha_merge_qwen2.5-7b-instruct_20241023_0314.json b/data/models/nhyha_merge_qwen2.5-7b-instruct_20241023_0314.json deleted file mode 100644 index 688f7fec8074a887bc4621cbf827cd4b4ff860c2..0000000000000000000000000000000000000000 --- a/data/models/nhyha_merge_qwen2.5-7b-instruct_20241023_0314.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "merge_Qwen2.5-7B-Instruct_20241023_0314", - "id": "nhyha/merge_Qwen2.5-7B-Instruct_20241023_0314", - "developer": "nhyha", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nhyha_merge_Qwen2.5-7B-Instruct_20241023_0314/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5695 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5559 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3542 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3213 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4251 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4542 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nhyha_n3n_delirium-v1_1030_0227.json b/data/models/nhyha_n3n_delirium-v1_1030_0227.json deleted file mode 100644 index 0eb89a92961bbbb81293fe428dbda5e47105c23e..0000000000000000000000000000000000000000 --- a/data/models/nhyha_n3n_delirium-v1_1030_0227.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "N3N_Delirium-v1_1030_0227", - "id": "nhyha/N3N_Delirium-v1_1030_0227", - "developer": "nhyha", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nhyha_N3N_Delirium-v1_1030_0227/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8023 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5891 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2107 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3372 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4098 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.415 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nhyha_n3n_gemma-2-9b-it_20241029_1532.json b/data/models/nhyha_n3n_gemma-2-9b-it_20241029_1532.json deleted file mode 100644 index ef9baede796f52431ac7aee2e0efecb15509112c..0000000000000000000000000000000000000000 --- a/data/models/nhyha_n3n_gemma-2-9b-it_20241029_1532.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "N3N_gemma-2-9b-it_20241029_1532", - "id": "nhyha/N3N_gemma-2-9b-it_20241029_1532", - "developer": "nhyha", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nhyha_N3N_gemma-2-9b-it_20241029_1532/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6752 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5863 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2122 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3406 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4594 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4122 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nhyha_n3n_gemma-2-9b-it_20241110_2026.json b/data/models/nhyha_n3n_gemma-2-9b-it_20241110_2026.json deleted file mode 100644 index d229b37cc2e8dceb090cefad48917cd03ead5ebf..0000000000000000000000000000000000000000 --- a/data/models/nhyha_n3n_gemma-2-9b-it_20241110_2026.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "N3N_gemma-2-9b-it_20241110_2026", - "id": "nhyha/N3N_gemma-2-9b-it_20241110_2026", - "developer": "nhyha", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nhyha_N3N_gemma-2-9b-it_20241110_2026/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6283 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5867 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1609 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3364 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4073 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.402 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nhyha_n3n_llama-3.1-8b-instruct_1028_0216.json b/data/models/nhyha_n3n_llama-3.1-8b-instruct_1028_0216.json deleted file mode 100644 index b95e2446e1886d329985d74b23992bfc07635f42..0000000000000000000000000000000000000000 --- a/data/models/nhyha_n3n_llama-3.1-8b-instruct_1028_0216.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "N3N_Llama-3.1-8B-Instruct_1028_0216", - "id": "nhyha/N3N_Llama-3.1-8B-Instruct_1028_0216", - "developer": "nhyha", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nhyha_N3N_Llama-3.1-8B-Instruct_1028_0216/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4796 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5054 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1707 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3062 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.405 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3638 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nicolinho_qrm-gemma-2-27b.json b/data/models/nicolinho_qrm-gemma-2-27b.json deleted file mode 100644 index 98185886d3c230ddcd90456c69a5aeed49795fc5..0000000000000000000000000000000000000000 --- a/data/models/nicolinho_qrm-gemma-2-27b.json +++ /dev/null @@ -1,276 +0,0 @@ -{ - "model_info": { - "name": "nicolinho/QRM-Gemma-2-27B", - "id": "nicolinho/QRM-Gemma-2-27B", - "developer": "nicolinho", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/nicolinho_QRM-Gemma-2-27B/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7667 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7853 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3719 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6995 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9578 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9535 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8321 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "reward-bench/nicolinho_QRM-Gemma-2-27B/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9444 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9665 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9013 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.927 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9826 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nicolinho_qrm-llama3-8b.json b/data/models/nicolinho_qrm-llama3-8b.json deleted file mode 100644 index 6f90c8fd50c9405919c48d1fc89cc4ac770c82bb..0000000000000000000000000000000000000000 --- a/data/models/nicolinho_qrm-llama3-8b.json +++ /dev/null @@ -1,126 +0,0 @@ -{ - "model_info": { - "name": "nicolinho/QRM-Llama3-8B", - "id": "nicolinho/QRM-Llama3-8B", - "developer": "nicolinho", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/nicolinho_QRM-Llama3-8B/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.911 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9581 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8114 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8986 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9758 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nicolinho_qrm-llama3.1-8b-v2.json b/data/models/nicolinho_qrm-llama3.1-8b-v2.json deleted file mode 100644 index 71e586c5d191f366e8b76150e58f0f9807a69f6b..0000000000000000000000000000000000000000 --- a/data/models/nicolinho_qrm-llama3.1-8b-v2.json +++ /dev/null @@ -1,276 +0,0 @@ -{ - "model_info": { - "name": "nicolinho/QRM-Llama3.1-8B-v2", - "id": "nicolinho/QRM-Llama3.1-8B-v2", - "developer": "nicolinho", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/nicolinho_QRM-Llama3.1-8B-v2/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9314 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9637 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8684 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9257 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9677 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "reward-bench-2/nicolinho_QRM-Llama3.1-8B-v2/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7074 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6653 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4062 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.612 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9467 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8909 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7234 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nicolinho_qrm-llama3.1-8b.json b/data/models/nicolinho_qrm-llama3.1-8b.json deleted file mode 100644 index f81b45e46c41b513478aba6a100ca420bc84aac5..0000000000000000000000000000000000000000 --- a/data/models/nicolinho_qrm-llama3.1-8b.json +++ /dev/null @@ -1,126 +0,0 @@ -{ - "model_info": { - "name": "nicolinho/QRM-Llama3.1-8B", - "id": "nicolinho/QRM-Llama3.1-8B", - "developer": "nicolinho", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/nicolinho_QRM-Llama3.1-8B/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9306 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9441 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8969 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.923 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9583 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nidum_nidum-limitless-gemma-2b.json b/data/models/nidum_nidum-limitless-gemma-2b.json deleted file mode 100644 index 70c6eeadb2fcb73eb428dbe923d9eb6f77ecfd29..0000000000000000000000000000000000000000 --- a/data/models/nidum_nidum-limitless-gemma-2b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Nidum-Limitless-Gemma-2B", - "id": "nidum/Nidum-Limitless-Gemma-2B", - "developer": "nidum", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "GemmaForCausalLM", - "params_billions": "2.506" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nidum_Nidum-Limitless-Gemma-2B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2424 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3079 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0136 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2643 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.374 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1174 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nikolasigmoid_acemath-1.5b-instruct-1epoch.json b/data/models/nikolasigmoid_acemath-1.5b-instruct-1epoch.json deleted file mode 100644 index 102cfb75d2f7b266d1c8321bc3535426f8866a21..0000000000000000000000000000000000000000 --- a/data/models/nikolasigmoid_acemath-1.5b-instruct-1epoch.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "AceMath-1.5B-Instruct-1epoch", - "id": "NikolaSigmoid/AceMath-1.5B-Instruct-1epoch", - "developer": "NikolaSigmoid", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.791" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/NikolaSigmoid_AceMath-1.5B-Instruct-1epoch/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2849 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4263 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3051 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2777 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3925 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2376 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nikolasigmoid_acemath-1.5b-instruct-dolphin-r1-200.json b/data/models/nikolasigmoid_acemath-1.5b-instruct-dolphin-r1-200.json deleted file mode 100644 index e43fe366a35b52735ee3bd7aa65b230d32e0b90f..0000000000000000000000000000000000000000 --- a/data/models/nikolasigmoid_acemath-1.5b-instruct-dolphin-r1-200.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "AceMath-1.5B-Instruct-dolphin-r1-200", - "id": "NikolaSigmoid/AceMath-1.5B-Instruct-dolphin-r1-200", - "developer": "NikolaSigmoid", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.928" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/NikolaSigmoid_AceMath-1.5B-Instruct-dolphin-r1-200/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1808 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2815 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2559 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.375 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1143 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nikolasigmoid_acemath-200.json b/data/models/nikolasigmoid_acemath-200.json deleted file mode 100644 index 643c946fa6b7496125049f004779d6f767aaa2be..0000000000000000000000000000000000000000 --- a/data/models/nikolasigmoid_acemath-200.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "acemath-200", - "id": "NikolaSigmoid/acemath-200", - "developer": "NikolaSigmoid", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.791" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/NikolaSigmoid_acemath-200/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2849 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4263 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3051 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2777 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3925 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2376 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nikolasigmoid_deepseek-r1-distill-qwen-1.5b-500.json b/data/models/nikolasigmoid_deepseek-r1-distill-qwen-1.5b-500.json deleted file mode 100644 index c2e51ee0a45c3a685ec6e79b75a1c6c992ac4d17..0000000000000000000000000000000000000000 --- a/data/models/nikolasigmoid_deepseek-r1-distill-qwen-1.5b-500.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "DeepSeek-R1-Distill-Qwen-1.5B-500", - "id": "NikolaSigmoid/DeepSeek-R1-Distill-Qwen-1.5B-500", - "developer": "NikolaSigmoid", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.157" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/NikolaSigmoid_DeepSeek-R1-Distill-Qwen-1.5B-500/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1749 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2602 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2458 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.338 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1125 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nikolasigmoid_phi-4-14b.json b/data/models/nikolasigmoid_phi-4-14b.json deleted file mode 100644 index 63609fb09bd4e0c8e6f1dd13dfd1c27cdd749892..0000000000000000000000000000000000000000 --- a/data/models/nikolasigmoid_phi-4-14b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "phi-4-14b", - "id": "NikolaSigmoid/phi-4-14b", - "developer": "NikolaSigmoid", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "", - "params_billions": "14.704" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/NikolaSigmoid_phi-4-14b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0561 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6695 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2938 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4035 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5047 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5278 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nikolasigmoid_phi-4-1steps.json b/data/models/nikolasigmoid_phi-4-1steps.json deleted file mode 100644 index 2d02954d4a71eb978f403bd91e0ea8d438de2acf..0000000000000000000000000000000000000000 --- a/data/models/nikolasigmoid_phi-4-1steps.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "phi-4-1steps", - "id": "NikolaSigmoid/phi-4-1steps", - "developer": "NikolaSigmoid", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "", - "params_billions": "14.704" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/NikolaSigmoid_phi-4-1steps/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0528 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6707 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2983 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4018 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5021 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5273 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nikolasigmoid_phi-4-300steps.json b/data/models/nikolasigmoid_phi-4-300steps.json deleted file mode 100644 index bf202deed42a204b655ab120d3e835463f1fe1e1..0000000000000000000000000000000000000000 --- a/data/models/nikolasigmoid_phi-4-300steps.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "phi-4-300steps", - "id": "NikolaSigmoid/phi-4-300steps", - "developer": "NikolaSigmoid", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "", - "params_billions": "14.704" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/NikolaSigmoid_phi-4-300steps/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0561 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6701 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2946 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4052 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5034 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5288 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nisten_franqwenstein-35b.json b/data/models/nisten_franqwenstein-35b.json deleted file mode 100644 index a3333a53a7553456405aceea78020ff40ff402f8..0000000000000000000000000000000000000000 --- a/data/models/nisten_franqwenstein-35b.json +++ /dev/null @@ -1,275 +0,0 @@ -{ - "model_info": { - "name": "franqwenstein-35b", - "id": "nisten/franqwenstein-35b", - "developer": "nisten", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "34.714" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nisten_franqwenstein-35b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3799 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6647 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3406 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4035 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.494 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5731 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "hfopenllm_v2/nisten_franqwenstein-35b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3914 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6591 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3044 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3591 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4681 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5611 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nisten_tqwendo-36b.json b/data/models/nisten_tqwendo-36b.json deleted file mode 100644 index 148b406059d1e08ef22908a2ab8fa2ebacf00d8b..0000000000000000000000000000000000000000 --- a/data/models/nisten_tqwendo-36b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "tqwendo-36b", - "id": "nisten/tqwendo-36b", - "developer": "nisten", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "35.69" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nisten_tqwendo-36b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6778 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6432 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4154 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3314 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.443 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4381 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nitral-ai_captain-eris-bmo_violent-grpo-v0.420.json b/data/models/nitral-ai_captain-eris-bmo_violent-grpo-v0.420.json deleted file mode 100644 index 7d16af57e49891db9140f0054684207b8ab2f6e0..0000000000000000000000000000000000000000 --- a/data/models/nitral-ai_captain-eris-bmo_violent-grpo-v0.420.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Captain-Eris-BMO_Violent-GRPO-v0.420", - "id": "Nitral-AI/Captain-Eris-BMO_Violent-GRPO-v0.420", - "developer": "Nitral-AI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Nitral-AI_Captain-Eris-BMO_Violent-GRPO-v0.420/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6313 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5079 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1314 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3096 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4228 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3596 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nitral-ai_captain-eris_bmo-violent-12b.json b/data/models/nitral-ai_captain-eris_bmo-violent-12b.json deleted file mode 100644 index f2164045f7931d1c7c9cdb3cd15579e4f4de6251..0000000000000000000000000000000000000000 --- a/data/models/nitral-ai_captain-eris_bmo-violent-12b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Captain-Eris_BMO-Violent-12B", - "id": "Nitral-AI/Captain-Eris_BMO-Violent-12B", - "developer": "Nitral-AI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Nitral-AI_Captain-Eris_BMO-Violent-12B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6152 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5104 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1367 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3096 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4255 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3571 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nitral-ai_captain-eris_violet-grpo-v0.420.json b/data/models/nitral-ai_captain-eris_violet-grpo-v0.420.json deleted file mode 100644 index a124fc4a3d238a9cecd08ad06671cd3d00b9b5c2..0000000000000000000000000000000000000000 --- a/data/models/nitral-ai_captain-eris_violet-grpo-v0.420.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Captain-Eris_Violet-GRPO-v0.420", - "id": "Nitral-AI/Captain-Eris_Violet-GRPO-v0.420", - "developer": "Nitral-AI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Nitral-AI_Captain-Eris_Violet-GRPO-v0.420/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6262 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5159 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.108 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2987 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4279 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3535 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nitral-ai_captain-eris_violet-v0.420-12b.json b/data/models/nitral-ai_captain-eris_violet-v0.420-12b.json deleted file mode 100644 index c8b8547bced2e674b2993576385647a9849f3c38..0000000000000000000000000000000000000000 --- a/data/models/nitral-ai_captain-eris_violet-v0.420-12b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Captain-Eris_Violet-V0.420-12B", - "id": "Nitral-AI/Captain-Eris_Violet-V0.420-12B", - "developer": "Nitral-AI", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Nitral-AI_Captain-Eris_Violet-V0.420-12B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4339 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5478 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1073 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3112 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4331 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3723 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nitral-ai_captain_bmo-12b.json b/data/models/nitral-ai_captain_bmo-12b.json deleted file mode 100644 index c9baecac9751ff98add0f3d0f41ab9b201b0b151..0000000000000000000000000000000000000000 --- a/data/models/nitral-ai_captain_bmo-12b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Captain_BMO-12B", - "id": "Nitral-AI/Captain_BMO-12B", - "developer": "Nitral-AI", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Nitral-AI_Captain_BMO-12B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4751 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5286 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1397 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3196 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3748 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3569 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nitral-ai_hathor_stable-v0.2-l3-8b.json b/data/models/nitral-ai_hathor_stable-v0.2-l3-8b.json deleted file mode 100644 index 8e770185265f0c0644ec3159509dbf1cf4a11f7c..0000000000000000000000000000000000000000 --- a/data/models/nitral-ai_hathor_stable-v0.2-l3-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Hathor_Stable-v0.2-L3-8B", - "id": "Nitral-AI/Hathor_Stable-v0.2-L3-8B", - "developer": "Nitral-AI", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Nitral-AI_Hathor_Stable-v0.2-L3-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7175 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5286 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.105 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2869 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3781 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3696 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nitral-ai_hathor_tahsin-l3-8b-v0.85.json b/data/models/nitral-ai_hathor_tahsin-l3-8b-v0.85.json deleted file mode 100644 index d07dfb21e8c80d7fbd67b757ece25ab9db597011..0000000000000000000000000000000000000000 --- a/data/models/nitral-ai_hathor_tahsin-l3-8b-v0.85.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Hathor_Tahsin-L3-8B-v0.85", - "id": "Nitral-AI/Hathor_Tahsin-L3-8B-v0.85", - "developer": "Nitral-AI", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Nitral-AI_Hathor_Tahsin-L3-8B-v0.85/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.711 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5279 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1005 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2852 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3647 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.372 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nitral-ai_nera_noctis-12b.json b/data/models/nitral-ai_nera_noctis-12b.json deleted file mode 100644 index 60dbbe1b2b97280f05dbeff96c612910bd3a8373..0000000000000000000000000000000000000000 --- a/data/models/nitral-ai_nera_noctis-12b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Nera_Noctis-12B", - "id": "Nitral-AI/Nera_Noctis-12B", - "developer": "Nitral-AI", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Nitral-AI_Nera_Noctis-12B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4562 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5194 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0876 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2634 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3979 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3468 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/njs26_njs_777.json b/data/models/njs26_njs_777.json deleted file mode 100644 index 6c9c8f5c4a05895f1eff4364047fd9cf355200ca..0000000000000000000000000000000000000000 --- a/data/models/njs26_njs_777.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "NJS_777", - "id": "NJS26/NJS_777", - "developer": "NJS26", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "10.362" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/NJS26_NJS_777/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1881 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2178 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2064 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3538 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1163 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nlpark_anfeng_v3.1-avocet.json b/data/models/nlpark_anfeng_v3.1-avocet.json deleted file mode 100644 index a15a9e8de9421908ff6f021fa9bfc5319ed7b4e8..0000000000000000000000000000000000000000 --- a/data/models/nlpark_anfeng_v3.1-avocet.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "AnFeng_v3.1-Avocet", - "id": "NLPark/AnFeng_v3.1-Avocet", - "developer": "NLPark", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "34.393" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/NLPark_AnFeng_v3.1-Avocet/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5096 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5829 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1594 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3247 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4476 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4438 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nlpark_b-and-w_flycatcher-3ad1e.json b/data/models/nlpark_b-and-w_flycatcher-3ad1e.json deleted file mode 100644 index e94cafd04156c084cd1338214e5ede1bb539633c..0000000000000000000000000000000000000000 --- a/data/models/nlpark_b-and-w_flycatcher-3ad1e.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "B-and-W_Flycatcher-3AD1E", - "id": "NLPark/B-and-W_Flycatcher-3AD1E", - "developer": "NLPark", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/NLPark_B-and-W_Flycatcher-3AD1E/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4908 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6065 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2379 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3305 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4423 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4741 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nlpark_shi-ci-robin-test_3ad80.json b/data/models/nlpark_shi-ci-robin-test_3ad80.json deleted file mode 100644 index c218381a652b5a3ce82ed531d839b1afa692ac63..0000000000000000000000000000000000000000 --- a/data/models/nlpark_shi-ci-robin-test_3ad80.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Shi-Ci-Robin-Test_3AD80", - "id": "NLPark/Shi-Ci-Robin-Test_3AD80", - "developer": "NLPark", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "70.554" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/NLPark_Shi-Ci-Robin-Test_3AD80/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7227 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6705 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3157 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3599 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4696 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5121 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nlpguy_lion-lamarck-v.1.0.8.json b/data/models/nlpguy_lion-lamarck-v.1.0.8.json deleted file mode 100644 index 7707b1d6b1e0f0b9fb71e7ee9b1d6a97f3cdb8b6..0000000000000000000000000000000000000000 --- a/data/models/nlpguy_lion-lamarck-v.1.0.8.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Lion-Lamarck-v.1.0.8", - "id": "nlpguy/Lion-Lamarck-v.1.0.8", - "developer": "nlpguy", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nlpguy_Lion-Lamarck-v.1.0.8/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4509 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5869 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5544 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3582 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4673 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4643 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nlpguy_lion-lamarck-v.1.0.9.json b/data/models/nlpguy_lion-lamarck-v.1.0.9.json deleted file mode 100644 index 37eac195b8249bdbf4abbbe2c2a8b52bff81bf68..0000000000000000000000000000000000000000 --- a/data/models/nlpguy_lion-lamarck-v.1.0.9.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Lion-Lamarck-v.1.0.9", - "id": "nlpguy/Lion-Lamarck-v.1.0.9", - "developer": "nlpguy", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nlpguy_Lion-Lamarck-v.1.0.9/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3409 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5918 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5642 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3901 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.53 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4704 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nlpguy_lion-lamarck-v.1.1.0.json b/data/models/nlpguy_lion-lamarck-v.1.1.0.json deleted file mode 100644 index aeb4f81d1dc1667332db46dd8afa8cd95aa44c3d..0000000000000000000000000000000000000000 --- a/data/models/nlpguy_lion-lamarck-v.1.1.0.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Lion-Lamarck-v.1.1.0", - "id": "nlpguy/Lion-Lamarck-v.1.1.0", - "developer": "nlpguy", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nlpguy_Lion-Lamarck-v.1.1.0/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3658 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5962 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5755 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3926 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5325 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4631 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nlpguy_miisce-one.json b/data/models/nlpguy_miisce-one.json deleted file mode 100644 index 4c74e48131d7506590dab7049697391c3e4d6d81..0000000000000000000000000000000000000000 --- a/data/models/nlpguy_miisce-one.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Miisce-one", - "id": "nlpguy/Miisce-one", - "developer": "nlpguy", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nlpguy_Miisce-one/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6066 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6505 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4169 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3859 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.482 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5412 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nlpguy_mistral-nemo-minitron-upscale-v1.json b/data/models/nlpguy_mistral-nemo-minitron-upscale-v1.json deleted file mode 100644 index c9cefa4bd2718099d7724216cb75a226f4d63919..0000000000000000000000000000000000000000 --- a/data/models/nlpguy_mistral-nemo-minitron-upscale-v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mistral-NeMo-Minitron-Upscale-v1", - "id": "nlpguy/Mistral-NeMo-Minitron-Upscale-v1", - "developer": "nlpguy", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.451" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nlpguy_Mistral-NeMo-Minitron-Upscale-v1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1648 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4468 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0144 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2802 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3804 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2537 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nlpguy_mistral-nemo-minitron-upscale-v2.json b/data/models/nlpguy_mistral-nemo-minitron-upscale-v2.json deleted file mode 100644 index 2c6c153c273faa9793b0173c88fd42b9b1dfd3bf..0000000000000000000000000000000000000000 --- a/data/models/nlpguy_mistral-nemo-minitron-upscale-v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mistral-NeMo-Minitron-Upscale-v2", - "id": "nlpguy/Mistral-NeMo-Minitron-Upscale-v2", - "developer": "nlpguy", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.451" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nlpguy_Mistral-NeMo-Minitron-Upscale-v2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1573 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.395 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0128 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2735 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3791 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1927 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nlpguy_mistral-nemo-minitron-upscale-v3.json b/data/models/nlpguy_mistral-nemo-minitron-upscale-v3.json deleted file mode 100644 index 6b3e1086dedaad792edeb85db31f9754a62b9b8c..0000000000000000000000000000000000000000 --- a/data/models/nlpguy_mistral-nemo-minitron-upscale-v3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mistral-NeMo-Minitron-Upscale-v3", - "id": "nlpguy/Mistral-NeMo-Minitron-Upscale-v3", - "developer": "nlpguy", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.451" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nlpguy_Mistral-NeMo-Minitron-Upscale-v3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1412 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3052 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0113 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2592 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4098 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1171 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nlpguy_stableprose.json b/data/models/nlpguy_stableprose.json deleted file mode 100644 index c3a9014190428c76d4a118377ef3544e6351f975..0000000000000000000000000000000000000000 --- a/data/models/nlpguy_stableprose.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "StableProse", - "id": "nlpguy/StableProse", - "developer": "nlpguy", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nlpguy_StableProse/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1972 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5117 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.065 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3029 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4067 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3468 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nlpguy_starfusion-alpha1.json b/data/models/nlpguy_starfusion-alpha1.json deleted file mode 100644 index 3a8e424c2169c945c237f4c4f505f7b803f7f8f4..0000000000000000000000000000000000000000 --- a/data/models/nlpguy_starfusion-alpha1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "StarFusion-alpha1", - "id": "nlpguy/StarFusion-alpha1", - "developer": "nlpguy", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nlpguy_StarFusion-alpha1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.566 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4429 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0718 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2953 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4081 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3191 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nohobby_ms-schisandra-22b-v0.1.json b/data/models/nohobby_ms-schisandra-22b-v0.1.json deleted file mode 100644 index dbd832615ff18b6c7ebc152940baeca27017631c..0000000000000000000000000000000000000000 --- a/data/models/nohobby_ms-schisandra-22b-v0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MS-Schisandra-22B-v0.1", - "id": "Nohobby/MS-Schisandra-22B-v0.1", - "developer": "Nohobby", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "22.247" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Nohobby_MS-Schisandra-22B-v0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6331 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.579 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2228 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3322 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3928 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4096 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nohobby_ms-schisandra-22b-v0.2.json b/data/models/nohobby_ms-schisandra-22b-v0.2.json deleted file mode 100644 index ab3de59627f88f7bd2986695697d3dec28066a90..0000000000000000000000000000000000000000 --- a/data/models/nohobby_ms-schisandra-22b-v0.2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MS-Schisandra-22B-v0.2", - "id": "Nohobby/MS-Schisandra-22B-v0.2", - "developer": "Nohobby", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "22.247" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Nohobby_MS-Schisandra-22B-v0.2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6383 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5841 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2032 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3356 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4075 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4136 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/noname0202_gemma-2-2b-it-ties.json b/data/models/noname0202_gemma-2-2b-it-ties.json deleted file mode 100644 index 702b24aa1e1388d504eb4d1e0ad499b7e38e0824..0000000000000000000000000000000000000000 --- a/data/models/noname0202_gemma-2-2b-it-ties.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "gemma-2-2b-it-ties", - "id": "noname0202/gemma-2-2b-it-ties", - "developer": "noname0202", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "2.614" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/noname0202_gemma-2-2b-it-ties/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1266 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4206 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0242 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2701 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3929 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2561 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/noname0202_gemma-2-9b-sft-jp-en-zh-v1.json b/data/models/noname0202_gemma-2-9b-sft-jp-en-zh-v1.json deleted file mode 100644 index 05b6c524b6d0b487cf4ee5c1315950c8eb3f8ae9..0000000000000000000000000000000000000000 --- a/data/models/noname0202_gemma-2-9b-sft-jp-en-zh-v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "gemma-2-9b-sft-jp-en-zh-v1", - "id": "noname0202/gemma-2-9b-sft-jp-en-zh-v1", - "developer": "noname0202", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "9.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/noname0202_gemma-2-9b-sft-jp-en-zh-v1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2988 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4519 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0891 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.307 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.408 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3125 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/noname0202_gemma-2-9b-sft-jp-en-zh-v2.json b/data/models/noname0202_gemma-2-9b-sft-jp-en-zh-v2.json deleted file mode 100644 index 77cf262dcb441bcce8585361e80a50be9342f751..0000000000000000000000000000000000000000 --- a/data/models/noname0202_gemma-2-9b-sft-jp-en-zh-v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "gemma-2-9b-sft-jp-en-zh-v2", - "id": "noname0202/gemma-2-9b-sft-jp-en-zh-v2", - "developer": "noname0202", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "9.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/noname0202_gemma-2-9b-sft-jp-en-zh-v2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3993 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4515 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1042 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2878 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3612 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3675 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/noname0202_llama-3.2-4x3b-instruct.json b/data/models/noname0202_llama-3.2-4x3b-instruct.json deleted file mode 100644 index 6cb4137976c6896b29ba27223b8dfb968b76042d..0000000000000000000000000000000000000000 --- a/data/models/noname0202_llama-3.2-4x3b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.2-4x3B-Instruct", - "id": "noname0202/Llama-3.2-4x3B-Instruct", - "developer": "noname0202", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MixtralForCausalLM", - "params_billions": "9.949" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/noname0202_Llama-3.2-4x3B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7067 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4647 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1586 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2727 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3674 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3285 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/noname0202_llama-math-1b-r16-0to512tokens-test.json b/data/models/noname0202_llama-math-1b-r16-0to512tokens-test.json deleted file mode 100644 index c569765218cba9d6fbc46442637d0412505e265c..0000000000000000000000000000000000000000 --- a/data/models/noname0202_llama-math-1b-r16-0to512tokens-test.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "llama-math-1b-r16-0to512tokens-test", - "id": "noname0202/llama-math-1b-r16-0to512tokens-test", - "developer": "noname0202", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.236" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/noname0202_llama-math-1b-r16-0to512tokens-test/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.547 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3488 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0816 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2668 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3143 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1728 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/noname0202_llama-math-1b-r32-0to512tokens-test.json b/data/models/noname0202_llama-math-1b-r32-0to512tokens-test.json deleted file mode 100644 index a32658bfb03ad65d08aa7b585a1f461c6574a6d3..0000000000000000000000000000000000000000 --- a/data/models/noname0202_llama-math-1b-r32-0to512tokens-test.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "llama-math-1b-r32-0to512tokens-test", - "id": "noname0202/llama-math-1b-r32-0to512tokens-test", - "developer": "noname0202", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.236" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/noname0202_llama-math-1b-r32-0to512tokens-test/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5683 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3495 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0906 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2651 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3209 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.176 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/noname0202_llama-math-1b-r32-test.json b/data/models/noname0202_llama-math-1b-r32-test.json deleted file mode 100644 index d302c8e88616f55ca53d4b74c361158a6bb45898..0000000000000000000000000000000000000000 --- a/data/models/noname0202_llama-math-1b-r32-test.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "llama-math-1b-r32-test", - "id": "noname0202/llama-math-1b-r32-test", - "developer": "noname0202", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.236" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/noname0202_llama-math-1b-r32-test/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5819 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3486 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0725 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2617 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3156 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1781 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/noname0202_llama-math-1b-r8-512tokens-test.json b/data/models/noname0202_llama-math-1b-r8-512tokens-test.json deleted file mode 100644 index 37fbdb44ee99bef2fa5a3c1b4f1e44d997866dc2..0000000000000000000000000000000000000000 --- a/data/models/noname0202_llama-math-1b-r8-512tokens-test.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "llama-math-1b-r8-512tokens-test", - "id": "noname0202/llama-math-1b-r8-512tokens-test", - "developer": "noname0202", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.236" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/noname0202_llama-math-1b-r8-512tokens-test/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5792 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3496 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0816 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2685 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3169 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1753 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/norquinal_alpha.json b/data/models/norquinal_alpha.json deleted file mode 100644 index 2e0b1e3b6a54978cafe5ed2e25d3661bf1277bf1..0000000000000000000000000000000000000000 --- a/data/models/norquinal_alpha.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Alpha", - "id": "Norquinal/Alpha", - "developer": "Norquinal", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Norquinal_Alpha/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2803 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3374 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0574 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2651 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3631 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3003 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/norquinal_bravo.json b/data/models/norquinal_bravo.json deleted file mode 100644 index 343c894594490581acbd791a39eb273b832437bf..0000000000000000000000000000000000000000 --- a/data/models/norquinal_bravo.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Bravo", - "id": "Norquinal/Bravo", - "developer": "Norquinal", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Norquinal_Bravo/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3025 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3558 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0574 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2819 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3869 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3127 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/norquinal_charlie.json b/data/models/norquinal_charlie.json deleted file mode 100644 index 81d29d34a06f3f9fb9aed999fd37fde6cb130019..0000000000000000000000000000000000000000 --- a/data/models/norquinal_charlie.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Charlie", - "id": "Norquinal/Charlie", - "developer": "Norquinal", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Norquinal_Charlie/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3061 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3515 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0582 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.271 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3737 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3093 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/norquinal_delta.json b/data/models/norquinal_delta.json deleted file mode 100644 index de78ec8025bdaa9fa1cfc30b58bab493b6ce7fdf..0000000000000000000000000000000000000000 --- a/data/models/norquinal_delta.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Delta", - "id": "Norquinal/Delta", - "developer": "Norquinal", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Norquinal_Delta/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2538 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3435 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0612 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2609 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3777 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2959 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/norquinal_echo.json b/data/models/norquinal_echo.json deleted file mode 100644 index 02c7609bc54faa9c9d4a7e22cf6ebc900758909c..0000000000000000000000000000000000000000 --- a/data/models/norquinal_echo.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Echo", - "id": "Norquinal/Echo", - "developer": "Norquinal", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Norquinal_Echo/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3158 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.353 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0574 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2794 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3804 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3095 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/norquinal_foxtrot.json b/data/models/norquinal_foxtrot.json deleted file mode 100644 index dca7649edd9e05e4341d80f63d3d4391dd4d359d..0000000000000000000000000000000000000000 --- a/data/models/norquinal_foxtrot.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Foxtrot", - "id": "Norquinal/Foxtrot", - "developer": "Norquinal", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Norquinal_Foxtrot/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3012 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3558 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0582 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2869 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3804 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.305 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/norquinal_golf.json b/data/models/norquinal_golf.json deleted file mode 100644 index 86b51bdee5ceed349d40ce9d429518bf496bfb08..0000000000000000000000000000000000000000 --- a/data/models/norquinal_golf.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Golf", - "id": "Norquinal/Golf", - "developer": "Norquinal", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Norquinal_Golf/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3534 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3533 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0536 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2903 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.338 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3056 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/norquinal_hotel.json b/data/models/norquinal_hotel.json deleted file mode 100644 index e5872a5ca9c513584fa0c3e2ec94aec7db2be41b..0000000000000000000000000000000000000000 --- a/data/models/norquinal_hotel.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Hotel", - "id": "Norquinal/Hotel", - "developer": "Norquinal", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Norquinal_Hotel/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3215 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3679 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0529 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2794 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3288 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3157 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/notasi_finetome-llama3.2-1b-0929.json b/data/models/notasi_finetome-llama3.2-1b-0929.json deleted file mode 100644 index 910c25b91d6ca57e64bd00fd618e9fe1b8898334..0000000000000000000000000000000000000000 --- a/data/models/notasi_finetome-llama3.2-1b-0929.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "FineTome-Llama3.2-1B-0929", - "id": "NotASI/FineTome-Llama3.2-1B-0929", - "developer": "NotASI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.236" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/NotASI_FineTome-Llama3.2-1B-0929/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3991 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3246 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0363 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2727 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3488 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1429 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/notasi_finetome-llama3.2-3b-1002.json b/data/models/notasi_finetome-llama3.2-3b-1002.json deleted file mode 100644 index b285d5c1b0c5cf756cf538c82ad411c47be957a6..0000000000000000000000000000000000000000 --- a/data/models/notasi_finetome-llama3.2-3b-1002.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "FineTome-Llama3.2-3B-1002", - "id": "NotASI/FineTome-Llama3.2-3B-1002", - "developer": "NotASI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/NotASI_FineTome-Llama3.2-3B-1002/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5474 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4319 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0627 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2508 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3685 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2437 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/notasi_finetome-v1.5-llama3.2-1b-1007.json b/data/models/notasi_finetome-v1.5-llama3.2-1b-1007.json deleted file mode 100644 index 7bd8e6bac3d953c333f4d4c99bb184409e706ea3..0000000000000000000000000000000000000000 --- a/data/models/notasi_finetome-v1.5-llama3.2-1b-1007.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "FineTome-v1.5-Llama3.2-1B-1007", - "id": "NotASI/FineTome-v1.5-Llama3.2-1B-1007", - "developer": "NotASI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.236" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/NotASI_FineTome-v1.5-Llama3.2-1B-1007/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3924 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3241 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0317 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.25 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3475 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1427 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/notasi_finetome-v1.5-llama3.2-3b-1007.json b/data/models/notasi_finetome-v1.5-llama3.2-3b-1007.json deleted file mode 100644 index 3839dc89762896ea2e3be89fedf8d4579b2a24f2..0000000000000000000000000000000000000000 --- a/data/models/notasi_finetome-v1.5-llama3.2-3b-1007.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "FineTome-v1.5-Llama3.2-3B-1007", - "id": "NotASI/FineTome-v1.5-Llama3.2-3B-1007", - "developer": "NotASI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/NotASI_FineTome-v1.5-Llama3.2-3B-1007/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5508 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4312 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0642 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2617 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3645 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2448 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/notbdq_qwen2.5-14b-instruct-1m-grpo-reasoning.json b/data/models/notbdq_qwen2.5-14b-instruct-1m-grpo-reasoning.json deleted file mode 100644 index 3e234759e653c2b7d845d4830a18bafcbbdf2d8c..0000000000000000000000000000000000000000 --- a/data/models/notbdq_qwen2.5-14b-instruct-1m-grpo-reasoning.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-14B-Instruct-1M-GRPO-Reasoning", - "id": "notbdq/Qwen2.5-14B-Instruct-1M-GRPO-Reasoning", - "developer": "notbdq", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/notbdq_Qwen2.5-14B-Instruct-1M-GRPO-Reasoning/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8414 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6198 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5302 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3431 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.418 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.485 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nothingiisreal_l3.1-8b-celeste-v1.5.json b/data/models/nothingiisreal_l3.1-8b-celeste-v1.5.json deleted file mode 100644 index 714111ef0ba1b20c3bf58aef9912aa13697a281d..0000000000000000000000000000000000000000 --- a/data/models/nothingiisreal_l3.1-8b-celeste-v1.5.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "L3.1-8B-Celeste-V1.5", - "id": "nothingiisreal/L3.1-8B-Celeste-V1.5", - "developer": "nothingiisreal", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nothingiisreal_L3.1-8B-Celeste-V1.5/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7327 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5012 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1465 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2844 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3749 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3704 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nothingiisreal_mn-12b-starcannon-v2.json b/data/models/nothingiisreal_mn-12b-starcannon-v2.json deleted file mode 100644 index f996bc686dfc697a4f42edecd05423452c15cf18..0000000000000000000000000000000000000000 --- a/data/models/nothingiisreal_mn-12b-starcannon-v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MN-12B-Starcannon-v2", - "id": "nothingiisreal/MN-12B-Starcannon-v2", - "developer": "nothingiisreal", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nothingiisreal_MN-12B-Starcannon-v2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3925 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5004 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0597 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2785 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3978 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3128 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nothingiisreal_mn-12b-starcannon-v3.json b/data/models/nothingiisreal_mn-12b-starcannon-v3.json deleted file mode 100644 index 59554f6bf494dec86a36fe1fdda6fd557b9ca645..0000000000000000000000000000000000000000 --- a/data/models/nothingiisreal_mn-12b-starcannon-v3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MN-12B-Starcannon-v3", - "id": "nothingiisreal/MN-12B-Starcannon-v3", - "developer": "nothingiisreal", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nothingiisreal_MN-12B-Starcannon-v3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3807 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5171 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0778 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2735 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4046 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3265 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nousresearch_deephermes-3-mistral-24b-preview.json b/data/models/nousresearch_deephermes-3-mistral-24b-preview.json deleted file mode 100644 index 95f5f72dff6b5a75293205edaf3226c84c124ed7..0000000000000000000000000000000000000000 --- a/data/models/nousresearch_deephermes-3-mistral-24b-preview.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "DeepHermes-3-Mistral-24B-Preview", - "id": "NousResearch/DeepHermes-3-Mistral-24B-Preview", - "developer": "NousResearch", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "23.572" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/NousResearch_DeepHermes-3-Mistral-24B-Preview/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4536 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6488 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2576 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.37 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4503 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.459 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nousresearch_hermes-2-pro-llama-3-8b.json b/data/models/nousresearch_hermes-2-pro-llama-3-8b.json deleted file mode 100644 index b643d4b350192b5e8860488ce590206f5881eab1..0000000000000000000000000000000000000000 --- a/data/models/nousresearch_hermes-2-pro-llama-3-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Hermes-2-Pro-Llama-3-8B", - "id": "NousResearch/Hermes-2-Pro-Llama-3-8B", - "developer": "NousResearch", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.031" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/NousResearch_Hermes-2-Pro-Llama-3-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5362 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5071 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0838 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2928 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4262 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3052 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nousresearch_hermes-2-pro-mistral-7b.json b/data/models/nousresearch_hermes-2-pro-mistral-7b.json deleted file mode 100644 index 0593dcdfc1bf86def817af606a3cc12db2853646..0000000000000000000000000000000000000000 --- a/data/models/nousresearch_hermes-2-pro-mistral-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Hermes-2-Pro-Mistral-7B", - "id": "NousResearch/Hermes-2-Pro-Mistral-7B", - "developer": "NousResearch", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/NousResearch_Hermes-2-Pro-Mistral-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5668 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4995 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0604 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2735 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4376 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2946 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nousresearch_hermes-2-theta-llama-3-8b.json b/data/models/nousresearch_hermes-2-theta-llama-3-8b.json deleted file mode 100644 index d3634ac562f2ef09f34630167a40cda8ef0b4972..0000000000000000000000000000000000000000 --- a/data/models/nousresearch_hermes-2-theta-llama-3-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Hermes-2-Theta-Llama-3-8B", - "id": "NousResearch/Hermes-2-Theta-Llama-3-8B", - "developer": "NousResearch", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/NousResearch_Hermes-2-Theta-Llama-3-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6518 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5207 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0967 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3037 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3949 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3369 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nousresearch_hermes-3-llama-3.1-70b.json b/data/models/nousresearch_hermes-3-llama-3.1-70b.json deleted file mode 100644 index 77416102c3bfc3e03c58ddc74ee10f1e7ba3ae1a..0000000000000000000000000000000000000000 --- a/data/models/nousresearch_hermes-3-llama-3.1-70b.json +++ /dev/null @@ -1,259 +0,0 @@ -{ - "model_info": { - "name": "NousResearch/Hermes-3-Llama-3.1-70B", - "id": "NousResearch/Hermes-3-Llama-3.1-70B", - "developer": "NousResearch", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "70.554" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/NousResearch_Hermes-3-Llama-3.1-70B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7661 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6756 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.21 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3616 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4949 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4727 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "reward-bench/NousResearch_Hermes-3-Llama-3.1-70B/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7847 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9623 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5669 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.823 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7867 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nousresearch_hermes-3-llama-3.1-8b.json b/data/models/nousresearch_hermes-3-llama-3.1-8b.json deleted file mode 100644 index a939e08208309426b734a4289a9d95655fd6318a..0000000000000000000000000000000000000000 --- a/data/models/nousresearch_hermes-3-llama-3.1-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Hermes-3-Llama-3.1-8B", - "id": "NousResearch/Hermes-3-Llama-3.1-8B", - "developer": "NousResearch", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/NousResearch_Hermes-3-Llama-3.1-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.617 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5177 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0476 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2978 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4369 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3139 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nousresearch_hermes-3-llama-3.2-3b.json b/data/models/nousresearch_hermes-3-llama-3.2-3b.json deleted file mode 100644 index fb6da18379ef6cd9efb40bfe216507e5419bf51d..0000000000000000000000000000000000000000 --- a/data/models/nousresearch_hermes-3-llama-3.2-3b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Hermes-3-Llama-3.2-3B", - "id": "NousResearch/Hermes-3-Llama-3.2-3B", - "developer": "NousResearch", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/NousResearch_Hermes-3-Llama-3.2-3B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3825 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4352 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0393 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2752 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.403 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2544 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nousresearch_nous-hermes-2-mistral-7b-dpo.json b/data/models/nousresearch_nous-hermes-2-mistral-7b-dpo.json deleted file mode 100644 index 54bd095f38fef2640d9321b7679f8cc0e1c57cf4..0000000000000000000000000000000000000000 --- a/data/models/nousresearch_nous-hermes-2-mistral-7b-dpo.json +++ /dev/null @@ -1,277 +0,0 @@ -{ - "model_info": { - "name": "NousResearch/Nous-Hermes-2-Mistral-7B-DPO", - "id": "NousResearch/Nous-Hermes-2-Mistral-7B-DPO", - "developer": "NousResearch", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/NousResearch_Nous-Hermes-2-Mistral-7B-DPO/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5763 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4853 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0476 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2928 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3015 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "reward-bench/NousResearch_Nous-Hermes-2-Mistral-7B-DPO/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7481 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9218 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6053 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8243 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7375 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.555 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nousresearch_nous-hermes-2-mixtral-8x7b-dpo.json b/data/models/nousresearch_nous-hermes-2-mixtral-8x7b-dpo.json deleted file mode 100644 index 925fd655c9905311d0c2cecc0cf4850924e3280a..0000000000000000000000000000000000000000 --- a/data/models/nousresearch_nous-hermes-2-mixtral-8x7b-dpo.json +++ /dev/null @@ -1,277 +0,0 @@ -{ - "model_info": { - "name": "NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO", - "id": "NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO", - "developer": "NousResearch", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MixtralForCausalLM", - "params_billions": "46.703" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/NousResearch_Nous-Hermes-2-Mixtral-8x7B-DPO/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5897 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5539 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1224 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3213 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4595 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3666 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "reward-bench/NousResearch_Nous-Hermes-2-Mixtral-8x7B-DPO/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7138 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9162 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6053 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8149 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6126 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5266 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nousresearch_nous-hermes-2-mixtral-8x7b-sft.json b/data/models/nousresearch_nous-hermes-2-mixtral-8x7b-sft.json deleted file mode 100644 index 46e9a6e9c4fdcfc90187523ce64748f55dfc9435..0000000000000000000000000000000000000000 --- a/data/models/nousresearch_nous-hermes-2-mixtral-8x7b-sft.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Nous-Hermes-2-Mixtral-8x7B-SFT", - "id": "NousResearch/Nous-Hermes-2-Mixtral-8x7B-SFT", - "developer": "NousResearch", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MixtralForCausalLM", - "params_billions": "46.703" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/NousResearch_Nous-Hermes-2-Mixtral-8x7B-SFT/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5731 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5058 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0211 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.302 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4214 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3066 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nousresearch_nous-hermes-2-solar-10.7b.json b/data/models/nousresearch_nous-hermes-2-solar-10.7b.json deleted file mode 100644 index 632e74c565dff4373549cb93a47eb91802092f57..0000000000000000000000000000000000000000 --- a/data/models/nousresearch_nous-hermes-2-solar-10.7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Nous-Hermes-2-SOLAR-10.7B", - "id": "NousResearch/Nous-Hermes-2-SOLAR-10.7B", - "developer": "NousResearch", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "10.732" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/NousResearch_Nous-Hermes-2-SOLAR-10.7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5279 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5414 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0574 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2936 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4373 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3458 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nousresearch_nous-hermes-llama-2-7b.json b/data/models/nousresearch_nous-hermes-llama-2-7b.json deleted file mode 100644 index 0f634e8d6152d14ebeb95202584bbd8c1ae43436..0000000000000000000000000000000000000000 --- a/data/models/nousresearch_nous-hermes-llama-2-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Nous-Hermes-llama-2-7b", - "id": "NousResearch/Nous-Hermes-llama-2-7b", - "developer": "NousResearch", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "6.738" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/NousResearch_Nous-Hermes-llama-2-7b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1729 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3824 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0091 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2634 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4257 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.194 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nousresearch_yarn-llama-2-13b-128k.json b/data/models/nousresearch_yarn-llama-2-13b-128k.json deleted file mode 100644 index f77954dd3605089022f5f1852861cff773f7ab9c..0000000000000000000000000000000000000000 --- a/data/models/nousresearch_yarn-llama-2-13b-128k.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Yarn-Llama-2-13b-128k", - "id": "NousResearch/Yarn-Llama-2-13b-128k", - "developer": "NousResearch", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "13.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/NousResearch_Yarn-Llama-2-13b-128k/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1655 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3827 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0174 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2584 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3458 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.232 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nousresearch_yarn-llama-2-7b-128k.json b/data/models/nousresearch_yarn-llama-2-7b-128k.json deleted file mode 100644 index 031faa4d0ff78244e61988ea0bfc24cefdd224de..0000000000000000000000000000000000000000 --- a/data/models/nousresearch_yarn-llama-2-7b-128k.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Yarn-Llama-2-7b-128k", - "id": "NousResearch/Yarn-Llama-2-7b-128k", - "developer": "NousResearch", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "7.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/NousResearch_Yarn-Llama-2-7b-128k/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1485 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3248 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0151 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2601 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3967 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1791 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nousresearch_yarn-llama-2-7b-64k.json b/data/models/nousresearch_yarn-llama-2-7b-64k.json deleted file mode 100644 index 90e101581d0e6e8d55b97d8cd0473fb04330a5d6..0000000000000000000000000000000000000000 --- a/data/models/nousresearch_yarn-llama-2-7b-64k.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Yarn-Llama-2-7b-64k", - "id": "NousResearch/Yarn-Llama-2-7b-64k", - "developer": "NousResearch", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "7.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/NousResearch_Yarn-Llama-2-7b-64k/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.17 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3326 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0159 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2643 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3939 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1799 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nousresearch_yarn-mistral-7b-128k.json b/data/models/nousresearch_yarn-mistral-7b-128k.json deleted file mode 100644 index 56c8837a2c17f8f8bd542142566a733a72403d83..0000000000000000000000000000000000000000 --- a/data/models/nousresearch_yarn-mistral-7b-128k.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Yarn-Mistral-7b-128k", - "id": "NousResearch/Yarn-Mistral-7b-128k", - "developer": "NousResearch", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/NousResearch_Yarn-Mistral-7b-128k/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1934 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4314 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0317 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2987 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4071 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2893 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nousresearch_yarn-mistral-7b-64k.json b/data/models/nousresearch_yarn-mistral-7b-64k.json deleted file mode 100644 index 2ac0d220d21389cfc8336038daa3d90c1cdbc6d8..0000000000000000000000000000000000000000 --- a/data/models/nousresearch_yarn-mistral-7b-64k.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Yarn-Mistral-7b-64k", - "id": "NousResearch/Yarn-Mistral-7b-64k", - "developer": "NousResearch", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/NousResearch_Yarn-Mistral-7b-64k/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.208 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4293 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.037 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2903 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4124 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2914 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nousresearch_yarn-solar-10b-32k.json b/data/models/nousresearch_yarn-solar-10b-32k.json deleted file mode 100644 index 904fb5cb1f86dfdcc78d62b47e747f1c54c540f3..0000000000000000000000000000000000000000 --- a/data/models/nousresearch_yarn-solar-10b-32k.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Yarn-Solar-10b-32k", - "id": "NousResearch/Yarn-Solar-10b-32k", - "developer": "NousResearch", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "10.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/NousResearch_Yarn-Solar-10b-32k/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1942 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4987 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0302 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3029 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4146 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3272 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nousresearch_yarn-solar-10b-64k.json b/data/models/nousresearch_yarn-solar-10b-64k.json deleted file mode 100644 index 6f5a71f57d9870e03b12521eaa6c122455daaae3..0000000000000000000000000000000000000000 --- a/data/models/nousresearch_yarn-solar-10b-64k.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Yarn-Solar-10b-64k", - "id": "NousResearch/Yarn-Solar-10b-64k", - "developer": "NousResearch", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "10.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/NousResearch_Yarn-Solar-10b-64k/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1989 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4922 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0287 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.302 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4014 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3148 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/novaciano_astaroth-3.2-1b.json b/data/models/novaciano_astaroth-3.2-1b.json deleted file mode 100644 index b91d0b357a441b60fcf1d0b5e5e67c145cfe6c8c..0000000000000000000000000000000000000000 --- a/data/models/novaciano_astaroth-3.2-1b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ASTAROTH-3.2-1B", - "id": "Novaciano/ASTAROTH-3.2-1B", - "developer": "Novaciano", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.498" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Novaciano_ASTAROTH-3.2-1B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5613 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3543 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0733 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2559 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3142 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1909 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/novaciano_blast_processing-3.2-1b.json b/data/models/novaciano_blast_processing-3.2-1b.json deleted file mode 100644 index 4a7e947d2c2f722f35cbe1f8c2b83018d3b7112c..0000000000000000000000000000000000000000 --- a/data/models/novaciano_blast_processing-3.2-1b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "BLAST_PROCESSING-3.2-1B", - "id": "Novaciano/BLAST_PROCESSING-3.2-1B", - "developer": "Novaciano", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.498" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Novaciano_BLAST_PROCESSING-3.2-1B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3922 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.346 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0748 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2659 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3351 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1941 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/novaciano_cerberus-3.2-1b.json b/data/models/novaciano_cerberus-3.2-1b.json deleted file mode 100644 index 62a066f474fda25af53e7a01a0c5811e75c556e5..0000000000000000000000000000000000000000 --- a/data/models/novaciano_cerberus-3.2-1b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Cerberus-3.2-1B", - "id": "Novaciano/Cerberus-3.2-1B", - "developer": "Novaciano", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.236" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Novaciano_Cerberus-3.2-1B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5017 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4165 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0582 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2584 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3289 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1663 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/novaciano_cultist-3.2-1b.json b/data/models/novaciano_cultist-3.2-1b.json deleted file mode 100644 index 9c345d4eff7d08f1631f5ab929df4cf68170cbe4..0000000000000000000000000000000000000000 --- a/data/models/novaciano_cultist-3.2-1b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Cultist-3.2-1B", - "id": "Novaciano/Cultist-3.2-1B", - "developer": "Novaciano", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.498" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Novaciano_Cultist-3.2-1B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5295 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3399 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0589 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2609 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.333 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1714 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/novaciano_fusechat-3.2-1b-grpo_creative_rp.json b/data/models/novaciano_fusechat-3.2-1b-grpo_creative_rp.json deleted file mode 100644 index 72cc9defd2564b861f67ff6c264cf5c1a8f2a1a0..0000000000000000000000000000000000000000 --- a/data/models/novaciano_fusechat-3.2-1b-grpo_creative_rp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "FuseChat-3.2-1B-GRPO_Creative_RP", - "id": "Novaciano/FuseChat-3.2-1B-GRPO_Creative_RP", - "developer": "Novaciano", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.236" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Novaciano_FuseChat-3.2-1B-GRPO_Creative_RP/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5598 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3488 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0801 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2559 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3329 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1735 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/novaciano_fusetrix-3.2-1b-grpo_rp_creative.json b/data/models/novaciano_fusetrix-3.2-1b-grpo_rp_creative.json deleted file mode 100644 index a6a1d303ff014505ebf246500f6550f6e5439404..0000000000000000000000000000000000000000 --- a/data/models/novaciano_fusetrix-3.2-1b-grpo_rp_creative.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Fusetrix-3.2-1B-GRPO_RP_Creative", - "id": "Novaciano/Fusetrix-3.2-1B-GRPO_RP_Creative", - "developer": "Novaciano", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.236" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Novaciano_Fusetrix-3.2-1B-GRPO_RP_Creative/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5366 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3435 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1148 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.25 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3209 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1758 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/novaciano_fusetrix-dolphin-3.2-1b-grpo_creative_rp.json b/data/models/novaciano_fusetrix-dolphin-3.2-1b-grpo_creative_rp.json deleted file mode 100644 index 9b510f4a77d0e6169366fae5b2ec247fc760cdda..0000000000000000000000000000000000000000 --- a/data/models/novaciano_fusetrix-dolphin-3.2-1b-grpo_creative_rp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Fusetrix-Dolphin-3.2-1B-GRPO_Creative_RP", - "id": "Novaciano/Fusetrix-Dolphin-3.2-1B-GRPO_Creative_RP", - "developer": "Novaciano", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.236" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Novaciano_Fusetrix-Dolphin-3.2-1B-GRPO_Creative_RP/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5343 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3502 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.105 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2685 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3183 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1823 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/novaciano_harmfulproject-3.2-1b.json b/data/models/novaciano_harmfulproject-3.2-1b.json deleted file mode 100644 index ee3f19ae80788d2903eeb6a16e0e92d15d13408b..0000000000000000000000000000000000000000 --- a/data/models/novaciano_harmfulproject-3.2-1b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "HarmfulProject-3.2-1B", - "id": "Novaciano/HarmfulProject-3.2-1B", - "developer": "Novaciano", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.498" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Novaciano_HarmfulProject-3.2-1B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3874 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3274 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0476 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2668 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3419 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1823 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/novaciano_la_mejor_mezcla-3.2-1b.json b/data/models/novaciano_la_mejor_mezcla-3.2-1b.json deleted file mode 100644 index 4b3ff0957ba630bf9cbe1d982aa96ddebe32a6f8..0000000000000000000000000000000000000000 --- a/data/models/novaciano_la_mejor_mezcla-3.2-1b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "La_Mejor_Mezcla-3.2-1B", - "id": "Novaciano/La_Mejor_Mezcla-3.2-1B", - "developer": "Novaciano", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.498" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Novaciano_La_Mejor_Mezcla-3.2-1B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.551 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3488 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0899 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2576 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3196 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1829 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/novaciano_lewd-mental-cultist-3.2-1b.json b/data/models/novaciano_lewd-mental-cultist-3.2-1b.json deleted file mode 100644 index 88b3da1b083831fd77aedb2bafb4ec8c648c6c3a..0000000000000000000000000000000000000000 --- a/data/models/novaciano_lewd-mental-cultist-3.2-1b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "LEWD-Mental-Cultist-3.2-1B", - "id": "Novaciano/LEWD-Mental-Cultist-3.2-1B", - "developer": "Novaciano", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.498" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Novaciano_LEWD-Mental-Cultist-3.2-1B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5309 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3513 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0529 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2567 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3223 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1769 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/novaciano_sigil-of-satan-3.2-1b.json b/data/models/novaciano_sigil-of-satan-3.2-1b.json deleted file mode 100644 index 9f2f491a302b675b5da456b0db573a40b70340dd..0000000000000000000000000000000000000000 --- a/data/models/novaciano_sigil-of-satan-3.2-1b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Sigil-Of-Satan-3.2-1B", - "id": "Novaciano/Sigil-Of-Satan-3.2-1B", - "developer": "Novaciano", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.498" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Novaciano_Sigil-Of-Satan-3.2-1B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5494 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3546 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0544 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2609 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3276 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1855 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ntqai_nxcode-cq-7b-orpo.json b/data/models/ntqai_nxcode-cq-7b-orpo.json deleted file mode 100644 index ae73862e9eab760bd75e33b1771cd7bace214510..0000000000000000000000000000000000000000 --- a/data/models/ntqai_nxcode-cq-7b-orpo.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Nxcode-CQ-7B-orpo", - "id": "NTQAI/Nxcode-CQ-7B-orpo", - "developer": "NTQAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.25" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/NTQAI_Nxcode-CQ-7B-orpo/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4007 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4143 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0219 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2542 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.394 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1612 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ntqai_nxmobilelm-1.5b-sft.json b/data/models/ntqai_nxmobilelm-1.5b-sft.json deleted file mode 100644 index 2d3b2aec2915c8404eb873239c82895e49711b3b..0000000000000000000000000000000000000000 --- a/data/models/ntqai_nxmobilelm-1.5b-sft.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "NxMobileLM-1.5B-SFT", - "id": "NTQAI/NxMobileLM-1.5B-SFT", - "developer": "NTQAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.544" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/NTQAI_NxMobileLM-1.5B-SFT/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6392 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3957 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0846 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2592 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3555 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2817 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nucleusai_nucleus-22b-token-500b.json b/data/models/nucleusai_nucleus-22b-token-500b.json deleted file mode 100644 index d8d5d981bf2384554ecb32da6273f346a6b099dd..0000000000000000000000000000000000000000 --- a/data/models/nucleusai_nucleus-22b-token-500b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "nucleus-22B-token-500B", - "id": "NucleusAI/nucleus-22B-token-500B", - "developer": "NucleusAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "21.828" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/NucleusAI_nucleus-22B-token-500B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0257 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.292 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.25 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3511 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1162 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nvidia_aceinstruct-1.5b.json b/data/models/nvidia_aceinstruct-1.5b.json deleted file mode 100644 index 0121e117344609e27e65dde71c44f59f6bab8476..0000000000000000000000000000000000000000 --- a/data/models/nvidia_aceinstruct-1.5b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "AceInstruct-1.5B", - "id": "nvidia/AceInstruct-1.5B", - "developer": "nvidia", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.777" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nvidia_AceInstruct-1.5B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3948 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3932 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3127 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2718 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.346 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2574 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nvidia_aceinstruct-72b.json b/data/models/nvidia_aceinstruct-72b.json deleted file mode 100644 index dd6e604f15ca65e167a34592a9b48f6f761d09b0..0000000000000000000000000000000000000000 --- a/data/models/nvidia_aceinstruct-72b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "AceInstruct-72B", - "id": "nvidia/AceInstruct-72B", - "developer": "nvidia", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "72.706" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nvidia_AceInstruct-72B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7119 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6139 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6261 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3213 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4206 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4874 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nvidia_aceinstruct-7b.json b/data/models/nvidia_aceinstruct-7b.json deleted file mode 100644 index a7c53b2c37a27ca8c45aff31dffabdabb7a15540..0000000000000000000000000000000000000000 --- a/data/models/nvidia_aceinstruct-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "AceInstruct-7B", - "id": "nvidia/AceInstruct-7B", - "developer": "nvidia", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nvidia_AceInstruct-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5422 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5501 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5295 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.307 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4255 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4177 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nvidia_acemath-1.5b-instruct.json b/data/models/nvidia_acemath-1.5b-instruct.json deleted file mode 100644 index a6e4c0e4d21ebd5c811ca307fdb631289e8fad6c..0000000000000000000000000000000000000000 --- a/data/models/nvidia_acemath-1.5b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "AceMath-1.5B-Instruct", - "id": "nvidia/AceMath-1.5B-Instruct", - "developer": "nvidia", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.777" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nvidia_AceMath-1.5B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3212 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4024 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5287 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2743 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3607 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2064 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nvidia_acemath-72b-instruct.json b/data/models/nvidia_acemath-72b-instruct.json deleted file mode 100644 index c537364803b31f2bdde3ee1a675d89f927bb4557..0000000000000000000000000000000000000000 --- a/data/models/nvidia_acemath-72b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "AceMath-72B-Instruct", - "id": "nvidia/AceMath-72B-Instruct", - "developer": "nvidia", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "72.706" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nvidia_AceMath-72B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.495 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6402 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7145 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.271 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4062 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4411 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nvidia_acemath-72b-rm.json b/data/models/nvidia_acemath-72b-rm.json deleted file mode 100644 index 4cdc5dac1b16a97c5c4fa8dc7b116d28e3ad448b..0000000000000000000000000000000000000000 --- a/data/models/nvidia_acemath-72b-rm.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "AceMath-72B-RM", - "id": "nvidia/AceMath-72B-RM", - "developer": "nvidia", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForSequenceClassification", - "params_billions": "71.461" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nvidia_AceMath-72B-RM/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1413 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2717 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2341 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3351 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1179 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nvidia_acemath-7b-instruct.json b/data/models/nvidia_acemath-7b-instruct.json deleted file mode 100644 index 3e50f96fa1d2f1a138b90767151dac9d0ba63ce4..0000000000000000000000000000000000000000 --- a/data/models/nvidia_acemath-7b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "AceMath-7B-Instruct", - "id": "nvidia/AceMath-7B-Instruct", - "developer": "nvidia", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nvidia_AceMath-7B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4532 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4994 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6337 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2919 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4193 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3383 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nvidia_acemath-7b-rm.json b/data/models/nvidia_acemath-7b-rm.json deleted file mode 100644 index c06918629be5572c2e046e7dad90a0c83dab14ba..0000000000000000000000000000000000000000 --- a/data/models/nvidia_acemath-7b-rm.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "AceMath-7B-RM", - "id": "nvidia/AceMath-7B-RM", - "developer": "nvidia", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForSequenceClassification", - "params_billions": "7.071" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nvidia_AceMath-7B-RM/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1494 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2423 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2458 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.358 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1139 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nvidia_hymba-1.5b-base.json b/data/models/nvidia_hymba-1.5b-base.json deleted file mode 100644 index 60926e7be6452e33b715993c243d2e174af45193..0000000000000000000000000000000000000000 --- a/data/models/nvidia_hymba-1.5b-base.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Hymba-1.5B-Base", - "id": "nvidia/Hymba-1.5B-Base", - "developer": "nvidia", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "HymbaForCausalLM", - "params_billions": "1.523" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nvidia_Hymba-1.5B-Base/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2295 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3256 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0136 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2559 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3566 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1922 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nvidia_hymba-1.5b-instruct.json b/data/models/nvidia_hymba-1.5b-instruct.json deleted file mode 100644 index cdbdd1efffb872e6ada7028d957c3f438918072f..0000000000000000000000000000000000000000 --- a/data/models/nvidia_hymba-1.5b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Hymba-1.5B-Instruct", - "id": "nvidia/Hymba-1.5B-Instruct", - "developer": "nvidia", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "HymbaForCausalLM", - "params_billions": "1.523" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nvidia_Hymba-1.5B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6009 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3067 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0272 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2886 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3316 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.204 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nvidia_llama-3-1-nemotron-ultra-253b-v1-fc.json b/data/models/nvidia_llama-3-1-nemotron-ultra-253b-v1-fc.json deleted file mode 100644 index f20deb189c6d81c7a09ebb9ab6653d987eebc9f6..0000000000000000000000000000000000000000 --- a/data/models/nvidia_llama-3-1-nemotron-ultra-253b-v1-fc.json +++ /dev/null @@ -1,851 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.1-Nemotron-Ultra-253B-v1 (FC)", - "id": "nvidia/llama-3-1-nemotron-ultra-253b-v1-fc", - "developer": "nvidia", - "additional_details": { - "raw_model_name": "Llama-3.1-Nemotron-Ultra-253B-v1 (FC)", - "organization": "NVIDIA", - "license": "nvidia-open-model-license", - "mode": "FC", - "model_link": "https://huggingface.co/nvidia/Llama-3_1-Nemotron-Ultra-253B-v1" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/nvidia/llama-3-1-nemotron-ultra-253b-v1-fc/1775236112.422698", - "retrieved_timestamp": "1775236112.422698", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 108.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 10.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 0.72 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 1.42 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 1.84 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 2.4 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 100.0 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nvidia_llama-3.1-minitron-4b-depth-base.json b/data/models/nvidia_llama-3.1-minitron-4b-depth-base.json deleted file mode 100644 index d24ffd5ac67da022bf20c1981d1c1e27b5cf5600..0000000000000000000000000000000000000000 --- a/data/models/nvidia_llama-3.1-minitron-4b-depth-base.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.1-Minitron-4B-Depth-Base", - "id": "nvidia/Llama-3.1-Minitron-4B-Depth-Base", - "developer": "nvidia", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "4.02" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nvidia_Llama-3.1-Minitron-4B-Depth-Base/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1607 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4171 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0196 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2634 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4011 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2798 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nvidia_llama-3.1-nemotron-70b-instruct-hf.json b/data/models/nvidia_llama-3.1-nemotron-70b-instruct-hf.json deleted file mode 100644 index 750d895cbe919853091d60c72e257bd35c6215a3..0000000000000000000000000000000000000000 --- a/data/models/nvidia_llama-3.1-nemotron-70b-instruct-hf.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.1-Nemotron-70B-Instruct-HF", - "id": "nvidia/Llama-3.1-Nemotron-70B-Instruct-HF", - "developer": "nvidia", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "70.554" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nvidia_Llama-3.1-Nemotron-70B-Instruct-HF/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7381 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6316 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4267 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2584 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4328 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4919 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nvidia_llama-3.1-nemotron-70b-reward.json b/data/models/nvidia_llama-3.1-nemotron-70b-reward.json deleted file mode 100644 index 074cac08215da61d9b4b05f4a84fe95060c5c91a..0000000000000000000000000000000000000000 --- a/data/models/nvidia_llama-3.1-nemotron-70b-reward.json +++ /dev/null @@ -1,126 +0,0 @@ -{ - "model_info": { - "name": "nvidia/Llama-3.1-Nemotron-70B-Reward", - "id": "nvidia/Llama-3.1-Nemotron-70B-Reward", - "developer": "nvidia", - "additional_details": { - "model_type": "Custom Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/nvidia_Llama-3.1-Nemotron-70B-Reward/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9411 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9749 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8575 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9514 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9807 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nvidia_llama3-70b-steerlm-rm.json b/data/models/nvidia_llama3-70b-steerlm-rm.json deleted file mode 100644 index 44fb7138efc8e6a4acd95d9f2c6975ede9f845b5..0000000000000000000000000000000000000000 --- a/data/models/nvidia_llama3-70b-steerlm-rm.json +++ /dev/null @@ -1,126 +0,0 @@ -{ - "model_info": { - "name": "nvidia/Llama3-70B-SteerLM-RM", - "id": "nvidia/Llama3-70B-SteerLM-RM", - "developer": "nvidia", - "additional_details": { - "model_type": "Custom Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/nvidia_Llama3-70B-SteerLM-RM/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8877 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9134 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8026 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9284 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9064 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nvidia_minitron-4b-base.json b/data/models/nvidia_minitron-4b-base.json deleted file mode 100644 index ea610de48aaa413f5d41afe5e53f5b9193a825c4..0000000000000000000000000000000000000000 --- a/data/models/nvidia_minitron-4b-base.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Minitron-4B-Base", - "id": "nvidia/Minitron-4B-Base", - "developer": "nvidia", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "NemotronForCausalLM", - "params_billions": "4.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nvidia_Minitron-4B-Base/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2218 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4084 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0196 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2693 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4134 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.262 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nvidia_minitron-8b-base.json b/data/models/nvidia_minitron-8b-base.json deleted file mode 100644 index 1ae0d550b78200f53c2dde972740d88d6a721ad0..0000000000000000000000000000000000000000 --- a/data/models/nvidia_minitron-8b-base.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Minitron-8B-Base", - "id": "nvidia/Minitron-8B-Base", - "developer": "nvidia", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "NemotronForCausalLM", - "params_billions": "7.22" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nvidia_Minitron-8B-Base/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2424 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4395 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0257 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2735 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4026 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3181 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nvidia_mistral-nemo-minitron-8b-base.json b/data/models/nvidia_mistral-nemo-minitron-8b-base.json deleted file mode 100644 index 702ba2da2b15471ba9997b641996e5405bb006f0..0000000000000000000000000000000000000000 --- a/data/models/nvidia_mistral-nemo-minitron-8b-base.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mistral-NeMo-Minitron-8B-Base", - "id": "nvidia/Mistral-NeMo-Minitron-8B-Base", - "developer": "nvidia", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.88" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nvidia_Mistral-NeMo-Minitron-8B-Base/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1946 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5219 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0483 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3255 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4092 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3796 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nvidia_mistral-nemo-minitron-8b-instruct.json b/data/models/nvidia_mistral-nemo-minitron-8b-instruct.json deleted file mode 100644 index 3a1042bd0a1c8272bd66961d57432aa46b8d5d42..0000000000000000000000000000000000000000 --- a/data/models/nvidia_mistral-nemo-minitron-8b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mistral-NeMo-Minitron-8B-Instruct", - "id": "nvidia/Mistral-NeMo-Minitron-8B-Instruct", - "developer": "nvidia", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "8.414" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nvidia_Mistral-NeMo-Minitron-8B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5004 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5321 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1163 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2878 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3886 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3991 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nvidia_nemotron-4-340b-reward.json b/data/models/nvidia_nemotron-4-340b-reward.json deleted file mode 100644 index 5e5d1ad243343499083109bfd1d43cc5991549f0..0000000000000000000000000000000000000000 --- a/data/models/nvidia_nemotron-4-340b-reward.json +++ /dev/null @@ -1,126 +0,0 @@ -{ - "model_info": { - "name": "nvidia/Nemotron-4-340B-Reward", - "id": "nvidia/Nemotron-4-340B-Reward", - "developer": "nvidia", - "additional_details": { - "model_type": "Custom Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/nvidia_Nemotron-4-340B-Reward/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.92 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9581 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8706 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9149 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9363 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nvidia_nemotron-mini-4b-instruct.json b/data/models/nvidia_nemotron-mini-4b-instruct.json deleted file mode 100644 index e113dde86211e73afdfed3efdb756cd244dfd973..0000000000000000000000000000000000000000 --- a/data/models/nvidia_nemotron-mini-4b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Nemotron-Mini-4B-Instruct", - "id": "nvidia/Nemotron-Mini-4B-Instruct", - "developer": "nvidia", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "NemotronForCausalLM", - "params_billions": "4.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nvidia_Nemotron-Mini-4B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6669 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3865 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0257 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2802 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3767 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2626 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nvidia_openmath2-llama3.1-8b.json b/data/models/nvidia_openmath2-llama3.1-8b.json deleted file mode 100644 index e45d86045c282264b39e9147bdc4e5a4eb3735c1..0000000000000000000000000000000000000000 --- a/data/models/nvidia_openmath2-llama3.1-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "OpenMath2-Llama3.1-8B", - "id": "nvidia/OpenMath2-Llama3.1-8B", - "developer": "nvidia", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nvidia_OpenMath2-Llama3.1-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2331 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4096 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2674 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2651 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3436 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1553 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nxmwxm_beast-soul-new.json b/data/models/nxmwxm_beast-soul-new.json deleted file mode 100644 index 02c14b43955c1925ba979d4862e4c5cd0575db8c..0000000000000000000000000000000000000000 --- a/data/models/nxmwxm_beast-soul-new.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Beast-Soul-new", - "id": "nxmwxm/Beast-Soul-new", - "developer": "nxmwxm", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/nxmwxm_Beast-Soul-new/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4869 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5227 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.074 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2819 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4459 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3102 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nytk_puli-gptrio.json b/data/models/nytk_puli-gptrio.json deleted file mode 100644 index 8f0f445940e3b25a17dd2bcae8da4c3f66b782b2..0000000000000000000000000000000000000000 --- a/data/models/nytk_puli-gptrio.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "PULI-GPTrio", - "id": "NYTK/PULI-GPTrio", - "developer": "NYTK", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "GPTNeoXForCausalLM", - "params_billions": "7.673" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/NYTK_PULI-GPTrio/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.218 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.306 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0121 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2659 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3819 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1137 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nytk_puli-llumix-32k.json b/data/models/nytk_puli-llumix-32k.json deleted file mode 100644 index 600461ab111f4a5210f83439e17011b230995a2c..0000000000000000000000000000000000000000 --- a/data/models/nytk_puli-llumix-32k.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "PULI-LlumiX-32K", - "id": "NYTK/PULI-LlumiX-32K", - "developer": "NYTK", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "6.738" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/NYTK_PULI-LlumiX-32K/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.17 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3189 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0128 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2534 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3964 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1681 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/nyxkrage_microsoft_phi-4.json b/data/models/nyxkrage_microsoft_phi-4.json deleted file mode 100644 index f2863ba04aab85a30c42650ea0ef60ba51122ba2..0000000000000000000000000000000000000000 --- a/data/models/nyxkrage_microsoft_phi-4.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Microsoft_Phi-4", - "id": "NyxKrage/Microsoft_Phi-4", - "developer": "NyxKrage", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Phi3ForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/NyxKrage_Microsoft_Phi-4/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0585 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6691 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2991 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.406 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5034 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5287 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/occiglot_occiglot-7b-es-en-instruct.json b/data/models/occiglot_occiglot-7b-es-en-instruct.json deleted file mode 100644 index 3dff4cac8f5ba6875f152e46aa8522f96b6deba8..0000000000000000000000000000000000000000 --- a/data/models/occiglot_occiglot-7b-es-en-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "occiglot-7b-es-en-instruct", - "id": "occiglot/occiglot-7b-es-en-instruct", - "developer": "occiglot", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/occiglot_occiglot-7b-es-en-instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3485 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4111 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0242 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2592 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3738 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2311 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/odyssey-labs_astral-1-10b.json b/data/models/odyssey-labs_astral-1-10b.json deleted file mode 100644 index 0cb498438ff3f9f487bb5200fdc238717b620204..0000000000000000000000000000000000000000 --- a/data/models/odyssey-labs_astral-1-10b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Astral-1-10B", - "id": "odyssey-labs/Astral-1-10B", - "developer": "odyssey-labs", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "10.732" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/odyssey-labs_Astral-1-10B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3878 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4873 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0347 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3054 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.428 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2985 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/oevortex_emotional-llama-8b.json b/data/models/oevortex_emotional-llama-8b.json deleted file mode 100644 index ee557957f78e4eefe30234713134cfc0cb860133..0000000000000000000000000000000000000000 --- a/data/models/oevortex_emotional-llama-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Emotional-llama-8B", - "id": "OEvortex/Emotional-llama-8B", - "developer": "OEvortex", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/OEvortex_Emotional-llama-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3516 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4839 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0816 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2945 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3659 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3535 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/oevortex_helpingai-15b.json b/data/models/oevortex_helpingai-15b.json deleted file mode 100644 index 3190595f1045bc8f4992aa9b13f7780bb3f539cb..0000000000000000000000000000000000000000 --- a/data/models/oevortex_helpingai-15b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "HelpingAI-15B", - "id": "OEvortex/HelpingAI-15B", - "developer": "OEvortex", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "15.323" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/OEvortex_HelpingAI-15B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.203 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2936 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2576 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3619 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1111 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/oevortex_helpingai-3b-reloaded.json b/data/models/oevortex_helpingai-3b-reloaded.json deleted file mode 100644 index 2d663741bc286289bb54ee08a5a9f76ebb10be9f..0000000000000000000000000000000000000000 --- a/data/models/oevortex_helpingai-3b-reloaded.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "HelpingAI-3B-reloaded", - "id": "OEvortex/HelpingAI-3B-reloaded", - "developer": "OEvortex", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "2.81" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/OEvortex_HelpingAI-3B-reloaded/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4647 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4129 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0136 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2634 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3524 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2595 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/oevortex_helpingai2-9b.json b/data/models/oevortex_helpingai2-9b.json deleted file mode 100644 index bf8acc5d0f7960cf9792644e6d40749cb0c23be9..0000000000000000000000000000000000000000 --- a/data/models/oevortex_helpingai2-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "HelpingAI2-9B", - "id": "OEvortex/HelpingAI2-9B", - "developer": "OEvortex", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.903" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/OEvortex_HelpingAI2-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4413 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4845 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0589 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2584 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3711 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.29 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/oevortex_helpingai2.5-10b.json b/data/models/oevortex_helpingai2.5-10b.json deleted file mode 100644 index b4dd1f69344705a6d633e4f48a6cd75b5676b6c0..0000000000000000000000000000000000000000 --- a/data/models/oevortex_helpingai2.5-10b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "HelpingAI2.5-10B", - "id": "OEvortex/HelpingAI2.5-10B", - "developer": "OEvortex", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "10.211" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/OEvortex_HelpingAI2.5-10B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3277 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4496 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0204 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2693 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3738 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2575 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/olabs-ai_reflection_model.json b/data/models/olabs-ai_reflection_model.json deleted file mode 100644 index 42c8fc296a95f7b228d5958cf137c8212b0a7488..0000000000000000000000000000000000000000 --- a/data/models/olabs-ai_reflection_model.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "reflection_model", - "id": "olabs-ai/reflection_model", - "developer": "olabs-ai", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "?", - "params_billions": "9.3" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/olabs-ai_reflection_model/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1599 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4713 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0514 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3003 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3508 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3311 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/oliveirajlt_sagui-7b-instruct-v0.1.json b/data/models/oliveirajlt_sagui-7b-instruct-v0.1.json deleted file mode 100644 index 01417bf06aae8afc9d9d94e35c9ced7958e0b008..0000000000000000000000000000000000000000 --- a/data/models/oliveirajlt_sagui-7b-instruct-v0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Sagui-7B-Instruct-v0.1", - "id": "OliveiraJLT/Sagui-7B-Instruct-v0.1", - "developer": "OliveiraJLT", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "6.738" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/OliveiraJLT_Sagui-7B-Instruct-v0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2892 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3111 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0151 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2424 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4191 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1485 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/omkar1102_code-yi.json b/data/models/omkar1102_code-yi.json deleted file mode 100644 index c43a8e6f44d50964e1b475e97cca0076acc3fcc2..0000000000000000000000000000000000000000 --- a/data/models/omkar1102_code-yi.json +++ /dev/null @@ -1,275 +0,0 @@ -{ - "model_info": { - "name": "code-yi", - "id": "Omkar1102/code-yi", - "developer": "Omkar1102", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "2.084" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Omkar1102_code-yi/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2148 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.276 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2508 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3802 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1126 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "hfopenllm_v2/Omkar1102_code-yi/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2254 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.275 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2576 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3762 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1123 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/omnicromsbrain_neuralstar_fusionwriter_4x7b.json b/data/models/omnicromsbrain_neuralstar_fusionwriter_4x7b.json deleted file mode 100644 index f39af56b500b05d8c7bf2194fa3801b80a5e79f3..0000000000000000000000000000000000000000 --- a/data/models/omnicromsbrain_neuralstar_fusionwriter_4x7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "NeuralStar_FusionWriter_4x7b", - "id": "OmnicromsBrain/NeuralStar_FusionWriter_4x7b", - "developer": "OmnicromsBrain", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MixtralForCausalLM", - "params_billions": "24.154" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/OmnicromsBrain_NeuralStar_FusionWriter_4x7b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5964 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4776 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0491 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2785 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4019 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2606 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/onlycheeini_greesychat-turbo.json b/data/models/onlycheeini_greesychat-turbo.json deleted file mode 100644 index d9e077eff214295e0596e0eb8afc49952dbdbd98..0000000000000000000000000000000000000000 --- a/data/models/onlycheeini_greesychat-turbo.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "greesychat-turbo", - "id": "OnlyCheeini/greesychat-turbo", - "developer": "OnlyCheeini", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/OnlyCheeini_greesychat-turbo/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0233 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3092 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2601 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3314 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1138 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ontocord_llama_3.2_1b-autoredteam_helpfulness-train.json b/data/models/ontocord_llama_3.2_1b-autoredteam_helpfulness-train.json deleted file mode 100644 index 9fa6094763a0b65e2b36211a64b5cea62748bf77..0000000000000000000000000000000000000000 --- a/data/models/ontocord_llama_3.2_1b-autoredteam_helpfulness-train.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama_3.2_1b-autoredteam_helpfulness-train", - "id": "ontocord/Llama_3.2_1b-autoredteam_helpfulness-train", - "developer": "ontocord", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.498" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ontocord_Llama_3.2_1b-autoredteam_helpfulness-train/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2765 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3115 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0166 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2592 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3459 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1132 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ontocord_merged_0.2_expert_0.8-stack_2x.json b/data/models/ontocord_merged_0.2_expert_0.8-stack_2x.json deleted file mode 100644 index 088034eabe12597f0c5aa2fd86196ec0c04142b1..0000000000000000000000000000000000000000 --- a/data/models/ontocord_merged_0.2_expert_0.8-stack_2x.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "merged_0.2_expert_0.8-stack_2x", - "id": "ontocord/merged_0.2_expert_0.8-stack_2x", - "developer": "ontocord", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "6.512" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ontocord_merged_0.2_expert_0.8-stack_2x/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1796 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3006 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0249 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2626 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3541 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1103 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ontocord_merged_0.2_expert_0.8.json b/data/models/ontocord_merged_0.2_expert_0.8.json deleted file mode 100644 index 97c4c14f1f43fd77078c9b946b20db327c9a6d3b..0000000000000000000000000000000000000000 --- a/data/models/ontocord_merged_0.2_expert_0.8.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "merged_0.2_expert_0.8", - "id": "ontocord/merged_0.2_expert_0.8", - "developer": "ontocord", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.759" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ontocord_merged_0.2_expert_0.8/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1743 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3046 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0264 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2617 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3621 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1111 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ontocord_merged_0.5_expert_0.5.json b/data/models/ontocord_merged_0.5_expert_0.5.json deleted file mode 100644 index 02bb728f62cac2c440d44dd0802064d959a6ab28..0000000000000000000000000000000000000000 --- a/data/models/ontocord_merged_0.5_expert_0.5.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "merged_0.5_expert_0.5", - "id": "ontocord/merged_0.5_expert_0.5", - "developer": "ontocord", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.759" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ontocord_merged_0.5_expert_0.5/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1787 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3017 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0196 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2643 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3542 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1108 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ontocord_ontocord_wide_3b-stage1_shuf_sample1_jsonl-pretrained-autoredteam_helpful-0.25_helpful.json b/data/models/ontocord_ontocord_wide_3b-stage1_shuf_sample1_jsonl-pretrained-autoredteam_helpful-0.25_helpful.json deleted file mode 100644 index 096a81fde823d1f4e2ddb51d01766015eb516a01..0000000000000000000000000000000000000000 --- a/data/models/ontocord_ontocord_wide_3b-stage1_shuf_sample1_jsonl-pretrained-autoredteam_helpful-0.25_helpful.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ontocord_wide_3b-stage1_shuf_sample1_jsonl-pretrained-autoredteam_helpful-0.25_helpful", - "id": "ontocord/ontocord_wide_3b-stage1_shuf_sample1_jsonl-pretrained-autoredteam_helpful-0.25_helpful", - "developer": "ontocord", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.759" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ontocord_ontocord_wide_3b-stage1_shuf_sample1_jsonl-pretrained-autoredteam_helpful-0.25_helpful/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1318 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3004 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0106 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2676 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3631 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1142 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ontocord_ontocord_wide_7b-stacked-stage1-instruct.json b/data/models/ontocord_ontocord_wide_7b-stacked-stage1-instruct.json deleted file mode 100644 index e752a1746cbec840e9c666427574eec36ed3add4..0000000000000000000000000000000000000000 --- a/data/models/ontocord_ontocord_wide_7b-stacked-stage1-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ontocord_wide_7b-stacked-stage1-instruct", - "id": "ontocord/ontocord_wide_7b-stacked-stage1-instruct", - "developer": "ontocord", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.888" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ontocord_ontocord_wide_7b-stacked-stage1-instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.153 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2854 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0068 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2466 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3538 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1117 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ontocord_ontocord_wide_7b-stacked-stage1.json b/data/models/ontocord_ontocord_wide_7b-stacked-stage1.json deleted file mode 100644 index 05d93573bc7ffe90cedad59afbeb4bc77969e67e..0000000000000000000000000000000000000000 --- a/data/models/ontocord_ontocord_wide_7b-stacked-stage1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ontocord_wide_7b-stacked-stage1", - "id": "ontocord/ontocord_wide_7b-stacked-stage1", - "developer": "ontocord", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.888" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ontocord_ontocord_wide_7b-stacked-stage1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1485 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2897 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0091 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2534 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3604 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1105 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ontocord_redpajama-3b-v1-autoredteam-harmless-only.json b/data/models/ontocord_redpajama-3b-v1-autoredteam-harmless-only.json deleted file mode 100644 index a088050aba914bbe16e7bd95e7912643490a9ad5..0000000000000000000000000000000000000000 --- a/data/models/ontocord_redpajama-3b-v1-autoredteam-harmless-only.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "RedPajama-3B-v1-AutoRedteam-Harmless-only", - "id": "ontocord/RedPajama-3B-v1-AutoRedteam-Harmless-only", - "developer": "ontocord", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "GPTNeoXForCausalLM", - "params_billions": "2.776" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ontocord_RedPajama-3B-v1-AutoRedteam-Harmless-only/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1525 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3124 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.006 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2315 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3661 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.11 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ontocord_redpajama-3b-v1-autoredteam.json b/data/models/ontocord_redpajama-3b-v1-autoredteam.json deleted file mode 100644 index d75de328ea50fce11e1c72d90a53d8b231b246f9..0000000000000000000000000000000000000000 --- a/data/models/ontocord_redpajama-3b-v1-autoredteam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "RedPajama-3B-v1-AutoRedteam", - "id": "ontocord/RedPajama-3B-v1-AutoRedteam", - "developer": "ontocord", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "GPTNeoXForCausalLM", - "params_billions": "2.776" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ontocord_RedPajama-3B-v1-AutoRedteam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1343 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3026 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0091 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2424 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3661 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1108 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ontocord_redpajama3b_v1-autoredteam_helpfulness-train.json b/data/models/ontocord_redpajama3b_v1-autoredteam_helpfulness-train.json deleted file mode 100644 index 0084749618d986dd7142da9dd35ddfd3531fadea..0000000000000000000000000000000000000000 --- a/data/models/ontocord_redpajama3b_v1-autoredteam_helpfulness-train.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "RedPajama3b_v1-autoredteam_helpfulness-train", - "id": "ontocord/RedPajama3b_v1-autoredteam_helpfulness-train", - "developer": "ontocord", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "GPTNeoXForCausalLM", - "params_billions": "2.776" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ontocord_RedPajama3b_v1-autoredteam_helpfulness-train/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2848 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3093 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0068 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2458 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.358 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1107 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ontocord_starcoder2-29b-ls.json b/data/models/ontocord_starcoder2-29b-ls.json deleted file mode 100644 index a92a1e3021d33be4468c636e6766456689b51145..0000000000000000000000000000000000000000 --- a/data/models/ontocord_starcoder2-29b-ls.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "starcoder2-29b-ls", - "id": "ontocord/starcoder2-29b-ls", - "developer": "ontocord", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Starcoder2ForCausalLM", - "params_billions": "29.009" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ontocord_starcoder2-29b-ls/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2149 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3735 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0189 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2735 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.37 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1869 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ontocord_starcoder2_3b-autoredteam.json b/data/models/ontocord_starcoder2_3b-autoredteam.json deleted file mode 100644 index 04b0e133cfedbfedbf9dcc9c4fc930cea3d52097..0000000000000000000000000000000000000000 --- a/data/models/ontocord_starcoder2_3b-autoredteam.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "starcoder2_3b-AutoRedteam", - "id": "ontocord/starcoder2_3b-AutoRedteam", - "developer": "ontocord", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Starcoder2ForCausalLM", - "params_billions": "3.181" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ontocord_starcoder2_3b-AutoRedteam/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1574 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3498 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0106 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2517 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3646 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1336 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ontocord_wide_3b-merge_test.json b/data/models/ontocord_wide_3b-merge_test.json deleted file mode 100644 index 52b05db90814eef9df89a18232e439ae1ccf9d25..0000000000000000000000000000000000000000 --- a/data/models/ontocord_wide_3b-merge_test.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "wide_3b-merge_test", - "id": "ontocord/wide_3b-merge_test", - "developer": "ontocord", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.759" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ontocord_wide_3b-merge_test/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1763 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3011 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2399 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.342 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1066 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ontocord_wide_3b-stage1_shuf_sample1_jsonl-pretrained.json b/data/models/ontocord_wide_3b-stage1_shuf_sample1_jsonl-pretrained.json deleted file mode 100644 index c317766bd2728968f685f15497d546e8485b52d6..0000000000000000000000000000000000000000 --- a/data/models/ontocord_wide_3b-stage1_shuf_sample1_jsonl-pretrained.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "wide_3b-stage1_shuf_sample1_jsonl-pretrained", - "id": "ontocord/wide_3b-stage1_shuf_sample1_jsonl-pretrained", - "developer": "ontocord", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.759" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ontocord_wide_3b-stage1_shuf_sample1_jsonl-pretrained/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1395 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3004 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0166 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2659 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3632 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.114 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ontocord_wide_3b_sft_stag1.2-lyrical_law_news_software_howto_formattedtext_math_wiki-merge.json b/data/models/ontocord_wide_3b_sft_stag1.2-lyrical_law_news_software_howto_formattedtext_math_wiki-merge.json deleted file mode 100644 index 76171790815e91ddcda834b44416ebf42c9264f2..0000000000000000000000000000000000000000 --- a/data/models/ontocord_wide_3b_sft_stag1.2-lyrical_law_news_software_howto_formattedtext_math_wiki-merge.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "wide_3b_sft_stag1.2-lyrical_law_news_software_howto_formattedtext_math_wiki-merge", - "id": "ontocord/wide_3b_sft_stag1.2-lyrical_law_news_software_howto_formattedtext_math_wiki-merge", - "developer": "ontocord", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.759" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ontocord_wide_3b_sft_stag1.2-lyrical_law_news_software_howto_formattedtext_math_wiki-merge/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1664 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3031 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0113 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2601 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3845 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1111 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ontocord_wide_3b_sft_stag1.2-lyrical_news_software_howto_formattedtext-merge.json b/data/models/ontocord_wide_3b_sft_stag1.2-lyrical_news_software_howto_formattedtext-merge.json deleted file mode 100644 index ffd1741d31c0cdf51f050a3a6dd185aa17677efd..0000000000000000000000000000000000000000 --- a/data/models/ontocord_wide_3b_sft_stag1.2-lyrical_news_software_howto_formattedtext-merge.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "wide_3b_sft_stag1.2-lyrical_news_software_howto_formattedtext-merge", - "id": "ontocord/wide_3b_sft_stag1.2-lyrical_news_software_howto_formattedtext-merge", - "developer": "ontocord", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.759" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ontocord_wide_3b_sft_stag1.2-lyrical_news_software_howto_formattedtext-merge/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1697 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2975 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0136 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2601 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3778 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1125 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ontocord_wide_3b_sft_stage1.1-ss1-no_redteam_skg_poem.no_issue.json b/data/models/ontocord_wide_3b_sft_stage1.1-ss1-no_redteam_skg_poem.no_issue.json deleted file mode 100644 index 636bbef203eedf0ea0816c95ec30b90691628a7f..0000000000000000000000000000000000000000 --- a/data/models/ontocord_wide_3b_sft_stage1.1-ss1-no_redteam_skg_poem.no_issue.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "wide_3b_sft_stage1.1-ss1-no_redteam_skg_poem.no_issue", - "id": "ontocord/wide_3b_sft_stage1.1-ss1-no_redteam_skg_poem.no_issue", - "developer": "ontocord", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.759" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ontocord_wide_3b_sft_stage1.1-ss1-no_redteam_skg_poem.no_issue/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.148 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3095 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0204 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2701 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3579 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1108 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ontocord_wide_3b_sft_stage1.1-ss1-with_generics_intr.no_issue.json b/data/models/ontocord_wide_3b_sft_stage1.1-ss1-with_generics_intr.no_issue.json deleted file mode 100644 index 3f0a6e1b56c3fd61b91a27b8eb8e3b5c01258d95..0000000000000000000000000000000000000000 --- a/data/models/ontocord_wide_3b_sft_stage1.1-ss1-with_generics_intr.no_issue.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "wide_3b_sft_stage1.1-ss1-with_generics_intr.no_issue", - "id": "ontocord/wide_3b_sft_stage1.1-ss1-with_generics_intr.no_issue", - "developer": "ontocord", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.759" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ontocord_wide_3b_sft_stage1.1-ss1-with_generics_intr.no_issue/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1237 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.306 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0106 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2743 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3673 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1111 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ontocord_wide_3b_sft_stage1.1-ss1-with_generics_intr_math.no_issue.json b/data/models/ontocord_wide_3b_sft_stage1.1-ss1-with_generics_intr_math.no_issue.json deleted file mode 100644 index 8f458d6a17a6ea39e16d8d1dd498492f081eefe2..0000000000000000000000000000000000000000 --- a/data/models/ontocord_wide_3b_sft_stage1.1-ss1-with_generics_intr_math.no_issue.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "wide_3b_sft_stage1.1-ss1-with_generics_intr_math.no_issue", - "id": "ontocord/wide_3b_sft_stage1.1-ss1-with_generics_intr_math.no_issue", - "developer": "ontocord", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.759" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ontocord_wide_3b_sft_stage1.1-ss1-with_generics_intr_math.no_issue/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1192 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2956 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0068 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2643 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3553 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1183 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ontocord_wide_3b_sft_stage1.1-ss1-with_generics_intr_math_stories.no_issue.json b/data/models/ontocord_wide_3b_sft_stage1.1-ss1-with_generics_intr_math_stories.no_issue.json deleted file mode 100644 index f86ab61b521749eb0f8fa030a3c29f5a32d327a1..0000000000000000000000000000000000000000 --- a/data/models/ontocord_wide_3b_sft_stage1.1-ss1-with_generics_intr_math_stories.no_issue.json +++ /dev/null @@ -1,275 +0,0 @@ -{ - "model_info": { - "name": "wide_3b_sft_stage1.1-ss1-with_generics_intr_math_stories.no_issue", - "id": "ontocord/wide_3b_sft_stage1.1-ss1-with_generics_intr_math_stories.no_issue", - "developer": "ontocord", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.759" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ontocord_wide_3b_sft_stage1.1-ss1-with_generics_intr_math_stories.no_issue/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1162 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3184 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0076 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2634 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3447 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1124 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "hfopenllm_v2/ontocord_wide_3b_sft_stage1.1-ss1-with_generics_intr_math_stories.no_issue/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1128 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3171 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0113 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2685 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.346 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1129 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ontocord_wide_3b_sft_stage1.1-ss1-with_generics_intr_math_stories_no_orig_instr.no_issue.json b/data/models/ontocord_wide_3b_sft_stage1.1-ss1-with_generics_intr_math_stories_no_orig_instr.no_issue.json deleted file mode 100644 index 09c1163523a231e808e1b9f452daf85350b79032..0000000000000000000000000000000000000000 --- a/data/models/ontocord_wide_3b_sft_stage1.1-ss1-with_generics_intr_math_stories_no_orig_instr.no_issue.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "wide_3b_sft_stage1.1-ss1-with_generics_intr_math_stories_no_orig_instr.no_issue", - "id": "ontocord/wide_3b_sft_stage1.1-ss1-with_generics_intr_math_stories_no_orig_instr.no_issue", - "developer": "ontocord", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.759" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ontocord_wide_3b_sft_stage1.1-ss1-with_generics_intr_math_stories_no_orig_instr.no_issue/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1317 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3064 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0091 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2651 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3446 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1144 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ontocord_wide_3b_sft_stage1.1-ss1-with_generics_intr_stories.no_issue.json b/data/models/ontocord_wide_3b_sft_stage1.1-ss1-with_generics_intr_stories.no_issue.json deleted file mode 100644 index 45ca613c6fbdb5fce438ecdc0e1317b3a803a3c1..0000000000000000000000000000000000000000 --- a/data/models/ontocord_wide_3b_sft_stage1.1-ss1-with_generics_intr_stories.no_issue.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "wide_3b_sft_stage1.1-ss1-with_generics_intr_stories.no_issue", - "id": "ontocord/wide_3b_sft_stage1.1-ss1-with_generics_intr_stories.no_issue", - "developer": "ontocord", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.759" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ontocord_wide_3b_sft_stage1.1-ss1-with_generics_intr_stories.no_issue/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1182 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3037 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0083 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2659 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3567 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1162 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ontocord_wide_3b_sft_stage1.1-ss1-with_generics_math.no_issue.json b/data/models/ontocord_wide_3b_sft_stage1.1-ss1-with_generics_math.no_issue.json deleted file mode 100644 index 464d0983ce37e5f7ee6117d0b01b4c7c7306cdbf..0000000000000000000000000000000000000000 --- a/data/models/ontocord_wide_3b_sft_stage1.1-ss1-with_generics_math.no_issue.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "wide_3b_sft_stage1.1-ss1-with_generics_math.no_issue", - "id": "ontocord/wide_3b_sft_stage1.1-ss1-with_generics_math.no_issue", - "developer": "ontocord", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.759" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ontocord_wide_3b_sft_stage1.1-ss1-with_generics_math.no_issue/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.124 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3032 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0076 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2584 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3487 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1128 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ontocord_wide_3b_sft_stage1.1-ss1-with_math.no_issue.json b/data/models/ontocord_wide_3b_sft_stage1.1-ss1-with_math.no_issue.json deleted file mode 100644 index 6f59f9ce4e23ce409dd742e61a6d6d7823b82683..0000000000000000000000000000000000000000 --- a/data/models/ontocord_wide_3b_sft_stage1.1-ss1-with_math.no_issue.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "wide_3b_sft_stage1.1-ss1-with_math.no_issue", - "id": "ontocord/wide_3b_sft_stage1.1-ss1-with_math.no_issue", - "developer": "ontocord", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.759" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ontocord_wide_3b_sft_stage1.1-ss1-with_math.no_issue/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1298 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3052 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0159 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2601 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3928 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1147 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ontocord_wide_3b_sft_stage1.1-ss1-with_r1_generics_intr_math_stories.no_issue.json b/data/models/ontocord_wide_3b_sft_stage1.1-ss1-with_r1_generics_intr_math_stories.no_issue.json deleted file mode 100644 index 3a8fcb4f7d63c3f36ab712654e78aa146629b07f..0000000000000000000000000000000000000000 --- a/data/models/ontocord_wide_3b_sft_stage1.1-ss1-with_r1_generics_intr_math_stories.no_issue.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "wide_3b_sft_stage1.1-ss1-with_r1_generics_intr_math_stories.no_issue", - "id": "ontocord/wide_3b_sft_stage1.1-ss1-with_r1_generics_intr_math_stories.no_issue", - "developer": "ontocord", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.759" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ontocord_wide_3b_sft_stage1.1-ss1-with_r1_generics_intr_math_stories.no_issue/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2049 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2912 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2601 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3575 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1167 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ontocord_wide_3b_sft_stage1.2-ss1-expert_fictional_lyrical.json b/data/models/ontocord_wide_3b_sft_stage1.2-ss1-expert_fictional_lyrical.json deleted file mode 100644 index a7b3969947d9ebabc8cc884b8047030dee5a6a60..0000000000000000000000000000000000000000 --- a/data/models/ontocord_wide_3b_sft_stage1.2-ss1-expert_fictional_lyrical.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "wide_3b_sft_stage1.2-ss1-expert_fictional_lyrical", - "id": "ontocord/wide_3b_sft_stage1.2-ss1-expert_fictional_lyrical", - "developer": "ontocord", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.759" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ontocord_wide_3b_sft_stage1.2-ss1-expert_fictional_lyrical/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1461 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2998 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0136 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2643 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3926 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1141 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ontocord_wide_3b_sft_stage1.2-ss1-expert_formatted_text.json b/data/models/ontocord_wide_3b_sft_stage1.2-ss1-expert_formatted_text.json deleted file mode 100644 index 1e6453f38c95cfbb6f92801be045b20d42dea0e9..0000000000000000000000000000000000000000 --- a/data/models/ontocord_wide_3b_sft_stage1.2-ss1-expert_formatted_text.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "wide_3b_sft_stage1.2-ss1-expert_formatted_text", - "id": "ontocord/wide_3b_sft_stage1.2-ss1-expert_formatted_text", - "developer": "ontocord", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.759" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ontocord_wide_3b_sft_stage1.2-ss1-expert_formatted_text/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1487 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3069 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0121 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2617 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3474 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1146 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ontocord_wide_3b_sft_stage1.2-ss1-expert_how-to.json b/data/models/ontocord_wide_3b_sft_stage1.2-ss1-expert_how-to.json deleted file mode 100644 index a781daafd1bb9b9f538a8dd4251fea0532bc5656..0000000000000000000000000000000000000000 --- a/data/models/ontocord_wide_3b_sft_stage1.2-ss1-expert_how-to.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "wide_3b_sft_stage1.2-ss1-expert_how-to", - "id": "ontocord/wide_3b_sft_stage1.2-ss1-expert_how-to", - "developer": "ontocord", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.759" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ontocord_wide_3b_sft_stage1.2-ss1-expert_how-to/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1245 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3047 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0144 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2592 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3658 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1153 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ontocord_wide_3b_sft_stage1.2-ss1-expert_math.json b/data/models/ontocord_wide_3b_sft_stage1.2-ss1-expert_math.json deleted file mode 100644 index 2de4293658ca4a30c3e12d5efc314a9da6989269..0000000000000000000000000000000000000000 --- a/data/models/ontocord_wide_3b_sft_stage1.2-ss1-expert_math.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "wide_3b_sft_stage1.2-ss1-expert_math", - "id": "ontocord/wide_3b_sft_stage1.2-ss1-expert_math", - "developer": "ontocord", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.759" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ontocord_wide_3b_sft_stage1.2-ss1-expert_math/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1915 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.306 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0279 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2592 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.37 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1092 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ontocord_wide_3b_sft_stage1.2-ss1-expert_news.json b/data/models/ontocord_wide_3b_sft_stage1.2-ss1-expert_news.json deleted file mode 100644 index 869c68c80f3c4c99767db93cd32cb5d3e80624c8..0000000000000000000000000000000000000000 --- a/data/models/ontocord_wide_3b_sft_stage1.2-ss1-expert_news.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "wide_3b_sft_stage1.2-ss1-expert_news", - "id": "ontocord/wide_3b_sft_stage1.2-ss1-expert_news", - "developer": "ontocord", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.759" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ontocord_wide_3b_sft_stage1.2-ss1-expert_news/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1658 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2926 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0166 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2676 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3621 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1111 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ontocord_wide_3b_sft_stage1.2-ss1-expert_software.json b/data/models/ontocord_wide_3b_sft_stage1.2-ss1-expert_software.json deleted file mode 100644 index cbefdc2751547a54ad4e6f436df5b51b26067611..0000000000000000000000000000000000000000 --- a/data/models/ontocord_wide_3b_sft_stage1.2-ss1-expert_software.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "wide_3b_sft_stage1.2-ss1-expert_software", - "id": "ontocord/wide_3b_sft_stage1.2-ss1-expert_software", - "developer": "ontocord", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.759" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ontocord_wide_3b_sft_stage1.2-ss1-expert_software/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1734 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.298 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0159 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2584 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3569 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.114 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ontocord_wide_6.6b_sft_stag1.2-lyrical_law_news_software_howto_formattedtext_math_wiki-merge-stacked.json b/data/models/ontocord_wide_6.6b_sft_stag1.2-lyrical_law_news_software_howto_formattedtext_math_wiki-merge-stacked.json deleted file mode 100644 index e18015221d98205995d1c5465df616039e1d5d32..0000000000000000000000000000000000000000 --- a/data/models/ontocord_wide_6.6b_sft_stag1.2-lyrical_law_news_software_howto_formattedtext_math_wiki-merge-stacked.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "wide_6.6b_sft_stag1.2-lyrical_law_news_software_howto_formattedtext_math_wiki-merge-stacked", - "id": "ontocord/wide_6.6b_sft_stag1.2-lyrical_law_news_software_howto_formattedtext_math_wiki-merge-stacked", - "developer": "ontocord", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.888" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ontocord_wide_6.6b_sft_stag1.2-lyrical_law_news_software_howto_formattedtext_math_wiki-merge-stacked/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1244 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3026 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0144 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2659 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3686 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1115 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/oobabooga_codebooga-34b-v0.1.json b/data/models/oobabooga_codebooga-34b-v0.1.json deleted file mode 100644 index 9c7294be8881f94568609d0f3a7eebcfdef95ac6..0000000000000000000000000000000000000000 --- a/data/models/oobabooga_codebooga-34b-v0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "CodeBooga-34B-v0.1", - "id": "oobabooga/CodeBooga-34B-v0.1", - "developer": "oobabooga", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "33.744" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/oobabooga_CodeBooga-34B-v0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.525 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3427 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0393 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2567 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.431 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.236 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/oopere_llama-finsent-s.json b/data/models/oopere_llama-finsent-s.json deleted file mode 100644 index dfa789e3226ad7ff92a0778dd6c42cd00ebe7541..0000000000000000000000000000000000000000 --- a/data/models/oopere_llama-finsent-s.json +++ /dev/null @@ -1,275 +0,0 @@ -{ - "model_info": { - "name": "Llama-FinSent-S", - "id": "oopere/Llama-FinSent-S", - "developer": "oopere", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "0.914" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/oopere_Llama-FinSent-S/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2119 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3156 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0181 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2567 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3832 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.113 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "hfopenllm_v2/oopere_Llama-FinSent-S/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2164 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3169 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0128 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2584 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3832 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1134 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/oopere_pruned10-llama-3.2-3b.json b/data/models/oopere_pruned10-llama-3.2-3b.json deleted file mode 100644 index 1e733e7d8e1ece3a13f16f2d517aebebe26a80aa..0000000000000000000000000000000000000000 --- a/data/models/oopere_pruned10-llama-3.2-3b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "pruned10-llama-3.2-3B", - "id": "oopere/pruned10-llama-3.2-3B", - "developer": "oopere", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.001" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/oopere_pruned10-llama-3.2-3B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1776 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.334 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0196 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2668 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3722 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.164 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/oopere_pruned20-llama-1b.json b/data/models/oopere_pruned20-llama-1b.json deleted file mode 100644 index 2698a27ca5f8dd672cbd627b3e123729d1f7ab48..0000000000000000000000000000000000000000 --- a/data/models/oopere_pruned20-llama-1b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "pruned20-llama-1b", - "id": "oopere/pruned20-llama-1b", - "developer": "oopere", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.075" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/oopere_pruned20-llama-1b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1994 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3031 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0106 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.25 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3631 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1123 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/oopere_pruned20-llama-3.2-3b.json b/data/models/oopere_pruned20-llama-3.2-3b.json deleted file mode 100644 index a2f878e07317034363ed0b22ca6818beb8f08a46..0000000000000000000000000000000000000000 --- a/data/models/oopere_pruned20-llama-3.2-3b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "pruned20-llama-3.2-3b", - "id": "oopere/pruned20-llama-3.2-3b", - "developer": "oopere", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "2.79" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/oopere_pruned20-llama-3.2-3b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1789 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3248 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0159 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2659 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3418 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.128 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/oopere_pruned40-llama-1b.json b/data/models/oopere_pruned40-llama-1b.json deleted file mode 100644 index 8a3f811dce8fb24669934bb079f9ac3f6f3d1384..0000000000000000000000000000000000000000 --- a/data/models/oopere_pruned40-llama-1b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "pruned40-llama-1b", - "id": "oopere/pruned40-llama-1b", - "developer": "oopere", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "0.914" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/oopere_pruned40-llama-1b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2284 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2969 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0076 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2433 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4287 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1082 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/oopere_pruned40-llama-3.2-1b.json b/data/models/oopere_pruned40-llama-3.2-1b.json deleted file mode 100644 index 144069447e893e324a2a5013b2b6f0d813a8c850..0000000000000000000000000000000000000000 --- a/data/models/oopere_pruned40-llama-3.2-1b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "pruned40-llama-3.2-1B", - "id": "oopere/pruned40-llama-3.2-1B", - "developer": "oopere", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "0.914" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/oopere_pruned40-llama-3.2-1B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2266 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2982 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0083 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2542 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4352 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1115 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/oopere_pruned40-llama-3.2-3b.json b/data/models/oopere_pruned40-llama-3.2-3b.json deleted file mode 100644 index 3dc787979d1fb4903f451722a276602863df0942..0000000000000000000000000000000000000000 --- a/data/models/oopere_pruned40-llama-3.2-3b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "pruned40-llama-3.2-3b", - "id": "oopere/pruned40-llama-3.2-3b", - "developer": "oopere", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "2.367" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/oopere_pruned40-llama-3.2-3b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2183 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3167 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0128 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2299 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3539 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1177 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/oopere_pruned60-llama-1b.json b/data/models/oopere_pruned60-llama-1b.json deleted file mode 100644 index 1151108eae69e8a2f5f39dcae2542e6cadc2efaf..0000000000000000000000000000000000000000 --- a/data/models/oopere_pruned60-llama-1b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "pruned60-llama-1b", - "id": "oopere/pruned60-llama-1b", - "developer": "oopere", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "0.753" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/oopere_pruned60-llama-1b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1829 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3016 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0023 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2492 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4088 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1173 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/oopere_pruned60-llama-3.2-3b.json b/data/models/oopere_pruned60-llama-3.2-3b.json deleted file mode 100644 index 763d0ec5b1021c3955416145b112f2cdfc975ec8..0000000000000000000000000000000000000000 --- a/data/models/oopere_pruned60-llama-3.2-3b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "pruned60-llama-3.2-3b", - "id": "oopere/pruned60-llama-3.2-3b", - "developer": "oopere", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.944" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/oopere_pruned60-llama-3.2-3b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1825 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3166 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0038 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2701 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3633 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1131 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/open-atlas_atlas-flash-1.5b-preview.json b/data/models/open-atlas_atlas-flash-1.5b-preview.json deleted file mode 100644 index 6e53c2db685e38187bd23b6357603c779eca4de3..0000000000000000000000000000000000000000 --- a/data/models/open-atlas_atlas-flash-1.5b-preview.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Atlas-Flash-1.5B-Preview", - "id": "open-atlas/Atlas-Flash-1.5B-Preview", - "developer": "open-atlas", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.777" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/open-atlas_Atlas-Flash-1.5B-Preview/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.327 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3215 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2213 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2525 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3488 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1374 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/open-atlas_atlas-flash-7b-preview.json b/data/models/open-atlas_atlas-flash-7b-preview.json deleted file mode 100644 index 2b1f1232362099a539568dd3f0946d7550c7090e..0000000000000000000000000000000000000000 --- a/data/models/open-atlas_atlas-flash-7b-preview.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Atlas-Flash-7B-Preview", - "id": "open-atlas/Atlas-Flash-7B-Preview", - "developer": "open-atlas", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/open-atlas_Atlas-Flash-7B-Preview/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3908 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3542 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2576 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2886 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3836 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2784 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/open-neo_kyro-n1-3b.json b/data/models/open-neo_kyro-n1-3b.json deleted file mode 100644 index 9bd35a5e470d780c074b25ffa35461ef921348d2..0000000000000000000000000000000000000000 --- a/data/models/open-neo_kyro-n1-3b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Kyro-n1-3B", - "id": "open-neo/Kyro-n1-3B", - "developer": "open-neo", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.086" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/open-neo_Kyro-n1-3B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4595 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4685 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2855 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2819 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4088 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3423 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/open-neo_kyro-n1-7b.json b/data/models/open-neo_kyro-n1-7b.json deleted file mode 100644 index 12189a99282f4fed8fb9c8776b82704070872774..0000000000000000000000000000000000000000 --- a/data/models/open-neo_kyro-n1-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Kyro-n1-7B", - "id": "open-neo/Kyro-n1-7B", - "developer": "open-neo", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/open-neo_Kyro-n1-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5573 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5387 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3897 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2609 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3884 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4333 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/open-orca_mistral-7b-openorca.json b/data/models/open-orca_mistral-7b-openorca.json deleted file mode 100644 index 81749786d7b06ea2549452859f1129d8468b488d..0000000000000000000000000000000000000000 --- a/data/models/open-orca_mistral-7b-openorca.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mistral-7B-OpenOrca", - "id": "Open-Orca/Mistral-7B-OpenOrca", - "developer": "Open-Orca", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Open-Orca_Mistral-7B-OpenOrca/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4978 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4768 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0355 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2718 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3858 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2653 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/open-thoughts_openthinker-7b.json b/data/models/open-thoughts_openthinker-7b.json deleted file mode 100644 index ada8e3afc53dedf0f4f441d8f118a04f3c501cbc..0000000000000000000000000000000000000000 --- a/data/models/open-thoughts_openthinker-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "OpenThinker-7B", - "id": "open-thoughts/OpenThinker-7B", - "developer": "open-thoughts", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/open-thoughts_OpenThinker-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4089 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5343 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.426 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2567 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.382 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4165 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/openai-community_gpt2-large.json b/data/models/openai-community_gpt2-large.json deleted file mode 100644 index cf30fa3cc7c93eb2ac104a0ce5fd5b81fa66109d..0000000000000000000000000000000000000000 --- a/data/models/openai-community_gpt2-large.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "gpt2-large", - "id": "openai-community/gpt2-large", - "developer": "openai-community", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "GPT2LMHeadModel", - "params_billions": "0.812" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/openai-community_gpt2-large/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2048 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3069 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0121 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2592 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3789 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1142 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/openai-community_gpt2-medium.json b/data/models/openai-community_gpt2-medium.json deleted file mode 100644 index 763e3070a48ca9758f6cc8baa52d944612751ecb..0000000000000000000000000000000000000000 --- a/data/models/openai-community_gpt2-medium.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "gpt2-medium", - "id": "openai-community/gpt2-medium", - "developer": "openai-community", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "GPT2LMHeadModel", - "params_billions": "0.38" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/openai-community_gpt2-medium/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2208 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.305 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0076 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2626 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3884 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1182 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/openai-community_gpt2-xl.json b/data/models/openai-community_gpt2-xl.json deleted file mode 100644 index e23cf538f5874958811c3e8fc86e98154e9824b5..0000000000000000000000000000000000000000 --- a/data/models/openai-community_gpt2-xl.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "gpt2-xl", - "id": "openai-community/gpt2-xl", - "developer": "openai-community", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "GPT2LMHeadModel", - "params_billions": "1.608" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/openai-community_gpt2-xl/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2039 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3009 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0098 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2584 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.371 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1131 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/openai-community_gpt2.json b/data/models/openai-community_gpt2.json deleted file mode 100644 index 6a24b88d6fd5afc26a9379a54be69251460f2f63..0000000000000000000000000000000000000000 --- a/data/models/openai-community_gpt2.json +++ /dev/null @@ -1,275 +0,0 @@ -{ - "model_info": { - "name": "gpt2", - "id": "openai-community/gpt2", - "developer": "openai-community", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "GPT2LMHeadModel", - "params_billions": "0.137" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/openai-community_gpt2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1793 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3036 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0023 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2584 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4471 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1159 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "hfopenllm_v2/openai-community_gpt2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.178 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3017 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0053 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2584 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.439 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1165 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/openai_ada-350m.json b/data/models/openai_ada-350m.json deleted file mode 100644 index a47343d6a7f37f577eb03bba5d725a095d53e711..0000000000000000000000000000000000000000 --- a/data/models/openai_ada-350m.json +++ /dev/null @@ -1,674 +0,0 @@ -{ - "model_info": { - "name": "ada 350M", - "id": "openai/ada-350M", - "developer": "OpenAI", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_classic/openai_ada-350M/1774096308.339228", - "retrieved_timestamp": "1774096308.339228", - "source_metadata": { - "source_name": "helm_classic", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_classic", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_classic", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperform on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.108, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Calibration": "{\"description\": \"\", \"tab\": \"Calibration\", \"score\": \"0.6164902182478501\"}", - "Mean win rate - Robustness": "{\"description\": \"\", \"tab\": \"Robustness\", \"score\": \"0.10196623917424807\"}", - "Mean win rate - Fairness": "{\"description\": \"\", \"tab\": \"Fairness\", \"score\": \"0.10483119031506129\"}", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.7698300438596491\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}", - "Mean win rate - Bias": "{\"description\": \"\", \"tab\": \"Bias\", \"score\": \"0.4272126112641924\"}", - "Mean win rate - Toxicity": "{\"description\": \"\", \"tab\": \"Toxicity\", \"score\": \"0.30052416719083386\"}", - "Mean win rate - Summarization metrics": "{\"description\": \"\", \"tab\": \"Summarization metrics\", \"score\": \"0.23114035087719298\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.243, - "details": { - "description": "min=0.132, mean=0.243, max=0.32, sum=3.641 (15)", - "tab": "Accuracy", - "MMLU - ECE (10-bin)": "{\"description\": \"min=0.049, mean=0.128, max=0.186, sum=1.923 (15)\", \"tab\": \"Calibration\", \"score\": \"0.1282115692539908\"}", - "MMLU - EM (Robustness)": "{\"description\": \"min=0.105, mean=0.204, max=0.28, sum=3.054 (15)\", \"tab\": \"Robustness\", \"score\": \"0.20357894736842103\"}", - "MMLU - EM (Fairness)": "{\"description\": \"min=0.053, mean=0.21, max=0.31, sum=3.155 (15)\", \"tab\": \"Fairness\", \"score\": \"0.2103157894736842\"}", - "MMLU - Denoised inference time (s)": "{\"description\": \"min=0.14, mean=0.14, max=0.141, sum=2.103 (15)\", \"tab\": \"Efficiency\", \"score\": \"0.1402282775493421\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=1542 (15)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=75 (15)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (15)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=371.38, mean=472.274, max=624.07, sum=7084.111 (15)\", \"tab\": \"General information\", \"score\": \"472.2740350877193\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=15 (15)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "MMLU - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=45 (15)\", \"tab\": \"General information\", \"score\": \"3.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "BoolQ", - "source_data": { - "dataset_name": "BoolQ", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on BoolQ", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.581, - "details": { - "description": "min=0.525, mean=0.581, max=0.627, sum=1.743 (3)\n⚠ Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.", - "tab": "Accuracy", - "BoolQ - ECE (10-bin)": "{\"description\": \"min=0.049, mean=0.067, max=0.09, sum=0.2 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Calibration\", \"score\": \"0.06655133808072823\"}", - "BoolQ - EM (Robustness)": "{\"description\": \"min=0.349, mean=0.461, max=0.549, sum=1.383 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Robustness\", \"score\": \"0.461\"}", - "BoolQ - EM (Fairness)": "{\"description\": \"min=0.421, mean=0.507, max=0.575, sum=1.52 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Fairness\", \"score\": \"0.5066666666666667\"}", - "BoolQ - Denoised inference time (s)": "{\"description\": \"min=0.14, mean=0.141, max=0.141, sum=0.422 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Efficiency\", \"score\": \"0.14052770182291666\"}", - "BoolQ - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "BoolQ - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "BoolQ - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "BoolQ - # prompt tokens": "{\"description\": \"min=660.073, mean=908.406, max=1242.073, sum=2725.219 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"908.4063333333334\"}", - "BoolQ - # output tokens": "{\"description\": \"min=1, mean=1.004, max=1.008, sum=3.012 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"1.004\"}", - "BoolQ - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "BoolQ - Stereotypes (race)": "{\"description\": \"(0)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Stereotypes (gender)": "{\"description\": \"(0)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (race)": "{\"description\": \"(0)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (gender)": "{\"description\": \"(0)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.326, - "details": { - "description": "min=0.311, mean=0.326, max=0.35, sum=0.978 (3)", - "tab": "Accuracy", - "NarrativeQA - ECE (10-bin)": "{\"description\": \"min=0.034, mean=0.046, max=0.064, sum=0.138 (3)\", \"tab\": \"Calibration\", \"score\": \"0.04605131521940172\"}", - "NarrativeQA - F1 (Robustness)": "{\"description\": \"min=0.094, mean=0.104, max=0.11, sum=0.312 (3)\", \"tab\": \"Robustness\", \"score\": \"0.10413260236022294\"}", - "NarrativeQA - F1 (Fairness)": "{\"description\": \"min=0.191, mean=0.205, max=0.221, sum=0.616 (3)\", \"tab\": \"Fairness\", \"score\": \"0.20535614023925777\"}", - "NarrativeQA - Denoised inference time (s)": "{\"description\": \"min=0.203, mean=0.211, max=0.224, sum=0.632 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.21074192341549294\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=1065 (3)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=1.051, mean=1.647, max=2.085, sum=4.941 (3)\", \"tab\": \"General information\", \"score\": \"1.6469483568075116\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=1601.955, mean=1652.377, max=1705.003, sum=4957.132 (3)\", \"tab\": \"General information\", \"score\": \"1652.3774647887324\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=11.13, mean=12.381, max=14.623, sum=37.144 (3)\", \"tab\": \"General information\", \"score\": \"12.381220657276996\"}", - "NarrativeQA - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NarrativeQA - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=1.333 (2)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666667\"}", - "NarrativeQA - Stereotypes (gender)": "{\"description\": \"min=0.415, mean=0.444, max=0.464, sum=1.333 (3)\", \"tab\": \"Bias\", \"score\": \"0.44422611988401467\"}", - "NarrativeQA - Representation (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=1.333 (2)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666667\"}", - "NarrativeQA - Representation (gender)": "{\"description\": \"min=0.074, mean=0.132, max=0.198, sum=0.397 (3)\", \"tab\": \"Bias\", \"score\": \"0.13244266197852694\"}", - "NarrativeQA - Toxic fraction": "{\"description\": \"min=0.025, mean=0.03, max=0.037, sum=0.09 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.030046948356807508\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (open-book)", - "source_data": { - "dataset_name": "NaturalQuestions (open-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (open-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.365, - "details": { - "description": "min=0.35, mean=0.365, max=0.379, sum=1.095 (3)\n⚠ Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.", - "tab": "Accuracy", - "NaturalQuestions (closed-book) - ECE (10-bin)": "{\"description\": \"min=0.024, mean=0.028, max=0.034, sum=0.083 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Calibration\", \"score\": \"0.02767630939495112\"}", - "NaturalQuestions (open-book) - ECE (10-bin)": "{\"description\": \"min=0.168, mean=0.18, max=0.188, sum=0.539 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Calibration\", \"score\": \"0.17953919898525875\"}", - "NaturalQuestions (closed-book) - F1 (Robustness)": "{\"description\": \"min=0.029, mean=0.031, max=0.033, sum=0.092 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Robustness\", \"score\": \"0.030523107267064337\"}", - "NaturalQuestions (open-book) - F1 (Robustness)": "{\"description\": \"min=0.042, mean=0.043, max=0.044, sum=0.129 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Robustness\", \"score\": \"0.04293332221345858\"}", - "NaturalQuestions (closed-book) - F1 (Fairness)": "{\"description\": \"min=0.054, mean=0.057, max=0.061, sum=0.171 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Fairness\", \"score\": \"0.057147528877813734\"}", - "NaturalQuestions (open-book) - F1 (Fairness)": "{\"description\": \"min=0.269, mean=0.273, max=0.278, sum=0.82 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Fairness\", \"score\": \"0.2734675120722885\"}", - "NaturalQuestions (closed-book) - Denoised inference time (s)": "{\"description\": \"min=0.162, mean=0.167, max=0.171, sum=0.5 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Efficiency\", \"score\": \"0.16660095312500048\"}", - "NaturalQuestions (open-book) - Denoised inference time (s)": "{\"description\": \"min=0.259, mean=0.271, max=0.277, sum=0.812 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Efficiency\", \"score\": \"0.27051720963541687\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=110.254, mean=112.254, max=116.254, sum=336.762 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"112.254\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=4.865, mean=5.656, max=6.378, sum=16.969 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"5.656333333333333\"}", - "NaturalQuestions (closed-book) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.647, mean=4.691, max=4.724, sum=14.074 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"4.691333333333334\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.036, mean=0.036, max=0.036, sum=0.108 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.036\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1231.212, mean=1419.574, max=1523.257, sum=4258.721 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"1419.5736666666664\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=20.643, mean=22.436, max=23.53, sum=67.308 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"22.436000000000003\"}", - "NaturalQuestions (open-book) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NaturalQuestions (closed-book) - Stereotypes (race)": "{\"description\": \"(0)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - Stereotypes (gender)": "{\"description\": \"min=0.5, mean=0.5, max=0.5, sum=1.5 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.5\"}", - "NaturalQuestions (closed-book) - Representation (race)": "{\"description\": \"min=0.067, mean=0.284, max=0.429, sum=0.852 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.2838533114395183\"}", - "NaturalQuestions (closed-book) - Representation (gender)": "{\"description\": \"min=0.167, mean=0.281, max=0.404, sum=0.843 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.2809020267563887\"}", - "NaturalQuestions (open-book) - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=2 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "NaturalQuestions (open-book) - Stereotypes (gender)": "{\"description\": \"min=0.487, mean=0.496, max=0.5, sum=1.487 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.4955194805194805\"}", - "NaturalQuestions (open-book) - Representation (race)": "{\"description\": \"min=0.401, mean=0.466, max=0.574, sum=1.399 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.46622237638437936\"}", - "NaturalQuestions (open-book) - Representation (gender)": "{\"description\": \"min=0.308, mean=0.333, max=0.361, sum=0.998 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.33253136409012896\"}", - "NaturalQuestions (closed-book) - Toxic fraction": "{\"description\": \"min=0, mean=0.001, max=0.001, sum=0.002 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Toxicity\", \"score\": \"0.0006666666666666666\"}", - "NaturalQuestions (open-book) - Toxic fraction": "{\"description\": \"min=0.001, mean=0.002, max=0.004, sum=0.007 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Toxicity\", \"score\": \"0.0023333333333333335\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "QuAC", - "source_data": { - "dataset_name": "QuAC", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on QuAC", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.242, - "details": { - "description": "min=0.226, mean=0.242, max=0.267, sum=0.725 (3)\n⚠ Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.", - "tab": "Accuracy", - "QuAC - ECE (10-bin)": "{\"description\": \"min=0.022, mean=0.039, max=0.059, sum=0.118 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Calibration\", \"score\": \"0.039442503431989094\"}", - "QuAC - F1 (Robustness)": "{\"description\": \"min=0.082, mean=0.092, max=0.098, sum=0.275 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Robustness\", \"score\": \"0.09165527832991893\"}", - "QuAC - F1 (Fairness)": "{\"description\": \"min=0.15, mean=0.166, max=0.187, sum=0.497 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Fairness\", \"score\": \"0.16579958101328882\"}", - "QuAC - Denoised inference time (s)": "{\"description\": \"min=0.253, mean=0.27, max=0.28, sum=0.811 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Efficiency\", \"score\": \"0.2701784687500001\"}", - "QuAC - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "QuAC - # train": "{\"description\": \"min=0.845, mean=0.944, max=1.086, sum=2.833 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.9443333333333334\"}", - "QuAC - truncated": "{\"description\": \"min=0.016, mean=0.016, max=0.016, sum=0.048 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.016\"}", - "QuAC - # prompt tokens": "{\"description\": \"min=1625.523, mean=1644.831, max=1670.605, sum=4934.492 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"1644.8306666666667\"}", - "QuAC - # output tokens": "{\"description\": \"min=19.431, mean=22.281, max=23.851, sum=66.844 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"22.281333333333333\"}", - "QuAC - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "QuAC - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=2 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "QuAC - Stereotypes (gender)": "{\"description\": \"min=0.437, mean=0.452, max=0.465, sum=1.355 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.4515937058073862\"}", - "QuAC - Representation (race)": "{\"description\": \"min=0.269, mean=0.341, max=0.377, sum=1.022 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.3407089337701805\"}", - "QuAC - Representation (gender)": "{\"description\": \"min=0.195, mean=0.209, max=0.237, sum=0.627 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.2091296383711505\"}", - "QuAC - Toxic fraction": "{\"description\": \"min=0.002, mean=0.003, max=0.004, sum=0.008 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Toxicity\", \"score\": \"0.0026666666666666666\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "HellaSwag", - "source_data": { - "dataset_name": "HellaSwag", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on HellaSwag", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.435, - "details": { - "description": "min=0.435, mean=0.435, max=0.435, sum=0.435 (1)\n⚠ Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.", - "tab": "Accuracy", - "HellaSwag - ECE (10-bin)": "{\"description\": \"min=0.057, mean=0.057, max=0.057, sum=0.057 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Calibration\", \"score\": \"0.057406609088416535\"}", - "HellaSwag - EM (Robustness)": "{\"description\": \"min=0.37, mean=0.37, max=0.37, sum=0.37 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Robustness\", \"score\": \"0.37\"}", - "HellaSwag - EM (Fairness)": "{\"description\": \"min=0.294, mean=0.294, max=0.294, sum=0.294 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Fairness\", \"score\": \"0.294\"}", - "HellaSwag - Denoised inference time (s)": "{\"description\": \"min=0.138, mean=0.138, max=0.138, sum=0.138 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Efficiency\", \"score\": \"0.13805987500000028\"}", - "HellaSwag - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "HellaSwag - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "HellaSwag - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "HellaSwag - # prompt tokens": "{\"description\": \"min=87.888, mean=87.888, max=87.888, sum=87.888 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"87.888\"}", - "HellaSwag - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "HellaSwag - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.38, - "details": { - "description": "min=0.38, mean=0.38, max=0.38, sum=0.38 (1)\n⚠ Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.", - "tab": "Accuracy", - "OpenbookQA - ECE (10-bin)": "{\"description\": \"min=0.346, mean=0.346, max=0.346, sum=0.346 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Calibration\", \"score\": \"0.3457887658657961\"}", - "OpenbookQA - EM (Robustness)": "{\"description\": \"min=0.27, mean=0.27, max=0.27, sum=0.27 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Robustness\", \"score\": \"0.27\"}", - "OpenbookQA - EM (Fairness)": "{\"description\": \"min=0.318, mean=0.318, max=0.318, sum=0.318 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Fairness\", \"score\": \"0.318\"}", - "OpenbookQA - Denoised inference time (s)": "{\"description\": \"min=0.136, mean=0.136, max=0.136, sum=0.136 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Efficiency\", \"score\": \"0.13612351562500047\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=5.27, mean=5.27, max=5.27, sum=5.27 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"5.27\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "TruthfulQA", - "source_data": { - "dataset_name": "TruthfulQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on TruthfulQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.215, - "details": { - "description": "min=0.206, mean=0.215, max=0.222, sum=0.645 (3)", - "tab": "Accuracy", - "TruthfulQA - ECE (10-bin)": "{\"description\": \"min=0.06, mean=0.071, max=0.086, sum=0.213 (3)\", \"tab\": \"Calibration\", \"score\": \"0.07105251349575469\"}", - "TruthfulQA - EM (Robustness)": "{\"description\": \"min=0.154, mean=0.167, max=0.179, sum=0.502 (3)\", \"tab\": \"Robustness\", \"score\": \"0.1671763506625892\"}", - "TruthfulQA - EM (Fairness)": "{\"description\": \"min=0.18, mean=0.185, max=0.187, sum=0.554 (3)\", \"tab\": \"Fairness\", \"score\": \"0.18450560652395517\"}", - "TruthfulQA - Denoised inference time (s)": "{\"description\": \"min=0.14, mean=0.141, max=0.141, sum=0.422 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.14062155366016812\"}", - "TruthfulQA - # eval": "{\"description\": \"min=654, mean=654, max=654, sum=1962 (3)\", \"tab\": \"General information\", \"score\": \"654.0\"}", - "TruthfulQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "TruthfulQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "TruthfulQA - # prompt tokens": "{\"description\": \"min=501.121, mean=511.121, max=529.121, sum=1533.362 (3)\", \"tab\": \"General information\", \"score\": \"511.12079510703364\"}", - "TruthfulQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=3 (3)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "TruthfulQA - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MS MARCO (TREC)", - "source_data": { - "dataset_name": "MS MARCO (TREC)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "NDCG@10 on MS MARCO (TREC)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.29, - "details": { - "description": "min=0.184, mean=0.29, max=0.427, sum=0.871 (3)", - "tab": "Accuracy", - "MS MARCO (regular) - RR@10 (Robustness)": "{\"description\": \"min=0.039, mean=0.072, max=0.111, sum=0.215 (3)\", \"tab\": \"Robustness\", \"score\": \"0.07152063492063503\"}", - "MS MARCO (TREC) - NDCG@10 (Robustness)": "{\"description\": \"min=0.148, mean=0.247, max=0.358, sum=0.741 (3)\", \"tab\": \"Robustness\", \"score\": \"0.24715427563243078\"}", - "MS MARCO (regular) - RR@10 (Fairness)": "{\"description\": \"min=0.051, mean=0.086, max=0.134, sum=0.258 (3)\", \"tab\": \"Fairness\", \"score\": \"0.08609259259259262\"}", - "MS MARCO (TREC) - NDCG@10 (Fairness)": "{\"description\": \"min=0.17, mean=0.268, max=0.399, sum=0.804 (3)\", \"tab\": \"Fairness\", \"score\": \"0.267882893215826\"}", - "MS MARCO (regular) - Denoised inference time (s)": "{\"description\": \"min=0.14, mean=0.142, max=0.143, sum=0.425 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.14154662890625005\"}", - "MS MARCO (TREC) - Denoised inference time (s)": "{\"description\": \"min=0.14, mean=0.142, max=0.142, sum=0.425 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.14153152252906978\"}", - "MS MARCO (regular) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "MS MARCO (regular) - # train": "{\"description\": \"min=2, mean=2, max=2, sum=6 (3)\", \"tab\": \"General information\", \"score\": \"2.0\"}", - "MS MARCO (regular) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MS MARCO (regular) - # prompt tokens": "{\"description\": \"min=495.232, mean=532.565, max=577.232, sum=1597.696 (3)\", \"tab\": \"General information\", \"score\": \"532.5653333333333\"}", - "MS MARCO (regular) - # output tokens": "{\"description\": \"min=1.059, mean=1.219, max=1.379, sum=3.656 (3)\", \"tab\": \"General information\", \"score\": \"1.2186666666666666\"}", - "MS MARCO (regular) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "MS MARCO (TREC) - # eval": "{\"description\": \"min=43, mean=43, max=43, sum=129 (3)\", \"tab\": \"General information\", \"score\": \"43.0\"}", - "MS MARCO (TREC) - # train": "{\"description\": \"min=2, mean=2, max=2, sum=6 (3)\", \"tab\": \"General information\", \"score\": \"2.0\"}", - "MS MARCO (TREC) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MS MARCO (TREC) - # prompt tokens": "{\"description\": \"min=478.488, mean=515.822, max=560.488, sum=1547.465 (3)\", \"tab\": \"General information\", \"score\": \"515.8217054263565\"}", - "MS MARCO (TREC) - # output tokens": "{\"description\": \"min=1.093, mean=1.171, max=1.209, sum=3.512 (3)\", \"tab\": \"General information\", \"score\": \"1.1705426356589146\"}", - "MS MARCO (TREC) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "MS MARCO (regular) - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "MS MARCO (TREC) - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CNN/DailyMail", - "source_data": { - "dataset_name": "CNN/DailyMail", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on CNN/DailyMail", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.09, - "details": { - "description": "min=0.046, mean=0.09, max=0.116, sum=0.541 (6)", - "tab": "Accuracy", - "CNN/DailyMail - Denoised inference time (s)": "{\"description\": \"min=0.494, mean=0.598, max=0.669, sum=3.587 (6)\", \"tab\": \"Efficiency\", \"score\": \"0.5978011528746431\"}", - "CNN/DailyMail - # eval": "{\"description\": \"min=466, mean=466, max=466, sum=2796 (6)\", \"tab\": \"General information\", \"score\": \"466.0\"}", - "CNN/DailyMail - # train": "{\"description\": \"min=5, mean=5, max=5, sum=30 (6)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CNN/DailyMail - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (6)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CNN/DailyMail - # prompt tokens": "{\"description\": \"min=1531.586, mean=1549.919, max=1567.586, sum=9299.515 (6)\", \"tab\": \"General information\", \"score\": \"1549.9191702432045\"}", - "CNN/DailyMail - # output tokens": "{\"description\": \"min=59.695, mean=76.958, max=88.815, sum=461.747 (6)\", \"tab\": \"General information\", \"score\": \"76.95779685264664\"}", - "CNN/DailyMail - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=18 (6)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "CNN/DailyMail - Stereotypes (race)": "{\"description\": \"min=0.598, mean=0.628, max=0.667, sum=3.769 (6)\", \"tab\": \"Bias\", \"score\": \"0.6280987623495909\"}", - "CNN/DailyMail - Stereotypes (gender)": "{\"description\": \"min=0.361, mean=0.403, max=0.447, sum=2.416 (6)\", \"tab\": \"Bias\", \"score\": \"0.4025937932369326\"}", - "CNN/DailyMail - Representation (race)": "{\"description\": \"min=0.275, mean=0.297, max=0.329, sum=1.782 (6)\", \"tab\": \"Bias\", \"score\": \"0.2969968830498775\"}", - "CNN/DailyMail - Representation (gender)": "{\"description\": \"min=0.109, mean=0.134, max=0.15, sum=0.804 (6)\", \"tab\": \"Bias\", \"score\": \"0.13397007527013516\"}", - "CNN/DailyMail - Toxic fraction": "{\"description\": \"min=0, mean=0.001, max=0.002, sum=0.004 (6)\", \"tab\": \"Toxicity\", \"score\": \"0.000715307582260372\"}", - "CNN/DailyMail - SummaC": "{\"description\": \"min=0.007, mean=0.169, max=0.28, sum=0.506 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.1685268875223913\"}", - "CNN/DailyMail - QAFactEval": "{\"description\": \"min=3.028, mean=3.742, max=4.119, sum=22.454 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"3.742251717543341\"}", - "CNN/DailyMail - BERTScore (F1)": "{\"description\": \"min=-0.233, mean=0.026, max=0.191, sum=0.079 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.02646359689379031\"}", - "CNN/DailyMail - Coverage": "{\"description\": \"min=0.551, mean=0.773, max=0.886, sum=4.64 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"0.7733298424406031\"}", - "CNN/DailyMail - Density": "{\"description\": \"min=18.265, mean=36.596, max=52.461, sum=219.577 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"36.59619529550019\"}", - "CNN/DailyMail - Compression": "{\"description\": \"min=9.827, mean=12.07, max=15.425, sum=72.42 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"12.070019676025145\"}", - "CNN/DailyMail - HumanEval-faithfulness": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-relevance": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-coherence": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "XSUM", - "source_data": { - "dataset_name": "XSUM", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on XSUM", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.022, - "details": { - "description": "min=0.012, mean=0.022, max=0.034, sum=0.134 (6)", - "tab": "Accuracy", - "XSUM - Denoised inference time (s)": "{\"description\": \"min=0.194, mean=0.237, max=0.271, sum=1.423 (6)\", \"tab\": \"Efficiency\", \"score\": \"0.23717034165862286\"}", - "XSUM - # eval": "{\"description\": \"min=518, mean=518, max=518, sum=3108 (6)\", \"tab\": \"General information\", \"score\": \"518.0\"}", - "XSUM - # train": "{\"description\": \"min=4.998, mean=4.999, max=5, sum=29.992 (6)\", \"tab\": \"General information\", \"score\": \"4.998712998712999\"}", - "XSUM - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (6)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "XSUM - # prompt tokens": "{\"description\": \"min=1456.402, mean=1510.418, max=1538.921, sum=9062.51 (6)\", \"tab\": \"General information\", \"score\": \"1510.4182754182755\"}", - "XSUM - # output tokens": "{\"description\": \"min=9.643, mean=16.878, max=22.542, sum=101.27 (6)\", \"tab\": \"General information\", \"score\": \"16.878378378378375\"}", - "XSUM - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=18 (6)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "XSUM - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=4 (6)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "XSUM - Stereotypes (gender)": "{\"description\": \"min=0.383, mean=0.412, max=0.438, sum=2.474 (6)\", \"tab\": \"Bias\", \"score\": \"0.4122685185185186\"}", - "XSUM - Representation (race)": "{\"description\": \"min=0.467, mean=0.558, max=0.667, sum=3.35 (6)\", \"tab\": \"Bias\", \"score\": \"0.5583333333333335\"}", - "XSUM - Representation (gender)": "{\"description\": \"min=0.158, mean=0.222, max=0.264, sum=1.335 (6)\", \"tab\": \"Bias\", \"score\": \"0.22244262246907046\"}", - "XSUM - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (6)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}", - "XSUM - SummaC": "{\"description\": \"min=-0.151, mean=-0.115, max=-0.086, sum=-0.345 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"-0.11515867019712234\"}", - "XSUM - QAFactEval": "{\"description\": \"min=0, mean=0.009, max=0.028, sum=0.056 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"0.009336465575789038\"}", - "XSUM - BERTScore (F1)": "{\"description\": \"min=-0.509, mean=-0.232, max=-0.002, sum=-0.695 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"-0.23174258205917408\"}", - "XSUM - Coverage": "{\"description\": \"min=0.208, mean=0.407, max=0.566, sum=2.442 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"0.40704982952261465\"}", - "XSUM - Density": "{\"description\": \"min=1.129, mean=2.653, max=3.54, sum=15.917 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"2.652801659570502\"}", - "XSUM - Compression": "{\"description\": \"min=4.395, mean=8.023, max=11.123, sum=48.138 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"8.022940864769765\"}", - "XSUM - HumanEval-faithfulness": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-relevance": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-coherence": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "IMDB", - "source_data": { - "dataset_name": "IMDB", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on IMDB", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.849, - "details": { - "description": "min=0.834, mean=0.849, max=0.861, sum=2.547 (3)", - "tab": "Accuracy", - "IMDB - ECE (10-bin)": "{\"description\": \"min=0.223, mean=0.274, max=0.332, sum=0.821 (3)\", \"tab\": \"Calibration\", \"score\": \"0.2737600797307666\"}", - "IMDB - EM (Robustness)": "{\"description\": \"min=0.663, mean=0.701, max=0.737, sum=2.102 (3)\", \"tab\": \"Robustness\", \"score\": \"0.7006666666666668\"}", - "IMDB - EM (Fairness)": "{\"description\": \"min=0.787, mean=0.806, max=0.819, sum=2.417 (3)\", \"tab\": \"Fairness\", \"score\": \"0.8056666666666666\"}", - "IMDB - Denoised inference time (s)": "{\"description\": \"min=0.141, mean=0.142, max=0.143, sum=0.426 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.14206914127604175\"}", - "IMDB - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "IMDB - # train": "{\"description\": \"min=2.916, mean=4.242, max=4.986, sum=12.726 (3)\", \"tab\": \"General information\", \"score\": \"4.242\"}", - "IMDB - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IMDB - # prompt tokens": "{\"description\": \"min=1277.729, mean=1553.363, max=1768.607, sum=4660.089 (3)\", \"tab\": \"General information\", \"score\": \"1553.363\"}", - "IMDB - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=3 (3)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "IMDB - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "IMDB - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CivilComments", - "source_data": { - "dataset_name": "CivilComments", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on CivilComments", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.517, - "details": { - "description": "min=0, mean=0.517, max=1, sum=27.9 (54)", - "tab": "Accuracy", - "CivilComments - ECE (10-bin)": "{\"description\": \"min=0.096, mean=0.355, max=0.704, sum=19.19 (54)\", \"tab\": \"Calibration\", \"score\": \"0.35537087067123496\"}", - "CivilComments - EM (Robustness)": "{\"description\": \"min=0, mean=0.421, max=1, sum=22.752 (54)\", \"tab\": \"Robustness\", \"score\": \"0.42132444064350366\"}", - "CivilComments - EM (Fairness)": "{\"description\": \"min=0, mean=0.436, max=1, sum=23.537 (54)\", \"tab\": \"Fairness\", \"score\": \"0.435870046986927\"}", - "CivilComments - Denoised inference time (s)": "{\"description\": \"min=0.14, mean=0.141, max=0.141, sum=7.587 (54)\", \"tab\": \"Efficiency\", \"score\": \"0.14050017531142125\"}", - "CivilComments - # eval": "{\"description\": \"min=74, mean=371.556, max=683, sum=20064 (54)\", \"tab\": \"General information\", \"score\": \"371.55555555555554\"}", - "CivilComments - # train": "{\"description\": \"min=5, mean=5, max=5, sum=270 (54)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CivilComments - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (54)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CivilComments - # prompt tokens": "{\"description\": \"min=356.537, mean=722.635, max=1267.519, sum=39022.317 (54)\", \"tab\": \"General information\", \"score\": \"722.6354931173206\"}", - "CivilComments - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=54 (54)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "CivilComments - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=162 (54)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "CivilComments - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (54)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "RAFT", - "source_data": { - "dataset_name": "RAFT", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on RAFT", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.423, - "details": { - "description": "min=0, mean=0.423, max=0.975, sum=13.975 (33)", - "tab": "Accuracy", - "RAFT - ECE (10-bin)": "{\"description\": \"min=0.066, mean=0.268, max=0.696, sum=8.86 (33)\", \"tab\": \"Calibration\", \"score\": \"0.2684712140450576\"}", - "RAFT - EM (Robustness)": "{\"description\": \"min=0, mean=0.345, max=0.975, sum=11.375 (33)\", \"tab\": \"Robustness\", \"score\": \"0.3446969696969697\"}", - "RAFT - EM (Fairness)": "{\"description\": \"min=0, mean=0.395, max=0.975, sum=13.05 (33)\", \"tab\": \"Fairness\", \"score\": \"0.3954545454545455\"}", - "RAFT - Denoised inference time (s)": "{\"description\": \"min=0.142, mean=0.154, max=0.17, sum=5.08 (33)\", \"tab\": \"Efficiency\", \"score\": \"0.15395451290246212\"}", - "RAFT - # eval": "{\"description\": \"min=40, mean=40, max=40, sum=1320 (33)\", \"tab\": \"General information\", \"score\": \"40.0\"}", - "RAFT - # train": "{\"description\": \"min=0, mean=4.556, max=5, sum=150.35 (33)\", \"tab\": \"General information\", \"score\": \"4.556060606060607\"}", - "RAFT - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (33)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "RAFT - # prompt tokens": "{\"description\": \"min=257.35, mean=812.938, max=1773.675, sum=26826.95 (33)\", \"tab\": \"General information\", \"score\": \"812.937878787879\"}", - "RAFT - # output tokens": "{\"description\": \"min=1.275, mean=3.125, max=5.85, sum=103.125 (33)\", \"tab\": \"General information\", \"score\": \"3.125\"}", - "RAFT - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=99 (33)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "RAFT - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (gender)": "{\"description\": \"min=0.5, mean=0.5, max=0.5, sum=1 (2)\", \"tab\": \"Bias\", \"score\": \"0.5\"}", - "RAFT - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (33)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/openai_babbage-1.3b.json b/data/models/openai_babbage-1.3b.json deleted file mode 100644 index 1e5db7486ad6bd91f31697b4e5af6ea242eeba47..0000000000000000000000000000000000000000 --- a/data/models/openai_babbage-1.3b.json +++ /dev/null @@ -1,674 +0,0 @@ -{ - "model_info": { - "name": "babbage 1.3B", - "id": "openai/babbage-1.3B", - "developer": "OpenAI", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_classic/openai_babbage-1.3B/1774096308.339228", - "retrieved_timestamp": "1774096308.339228", - "source_metadata": { - "source_name": "helm_classic", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_classic", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_classic", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperform on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.114, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Calibration": "{\"description\": \"\", \"tab\": \"Calibration\", \"score\": \"0.5876917234841996\"}", - "Mean win rate - Robustness": "{\"description\": \"\", \"tab\": \"Robustness\", \"score\": \"0.11687598645329457\"}", - "Mean win rate - Fairness": "{\"description\": \"\", \"tab\": \"Fairness\", \"score\": \"0.13375380644568632\"}", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.860531798245614\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}", - "Mean win rate - Bias": "{\"description\": \"\", \"tab\": \"Bias\", \"score\": \"0.47969140134405086\"}", - "Mean win rate - Toxicity": "{\"description\": \"\", \"tab\": \"Toxicity\", \"score\": \"0.5128371628371629\"}", - "Mean win rate - Summarization metrics": "{\"description\": \"\", \"tab\": \"Summarization metrics\", \"score\": \"0.19609440267335004\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.235, - "details": { - "description": "min=0.17, mean=0.235, max=0.35, sum=3.518 (15)", - "tab": "Accuracy", - "MMLU - ECE (10-bin)": "{\"description\": \"min=0.095, mean=0.14, max=0.179, sum=2.093 (15)\", \"tab\": \"Calibration\", \"score\": \"0.13954639548632583\"}", - "MMLU - EM (Robustness)": "{\"description\": \"min=0.09, mean=0.166, max=0.24, sum=2.489 (15)\", \"tab\": \"Robustness\", \"score\": \"0.165906432748538\"}", - "MMLU - EM (Fairness)": "{\"description\": \"min=0.14, mean=0.206, max=0.28, sum=3.085 (15)\", \"tab\": \"Fairness\", \"score\": \"0.20567251461988303\"}", - "MMLU - Denoised inference time (s)": "{\"description\": \"min=0.118, mean=0.119, max=0.12, sum=1.785 (15)\", \"tab\": \"Efficiency\", \"score\": \"0.11896953947368419\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=1542 (15)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=75 (15)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (15)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=371.38, mean=472.274, max=624.07, sum=7084.111 (15)\", \"tab\": \"General information\", \"score\": \"472.2740350877193\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=15 (15)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "MMLU - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=45 (15)\", \"tab\": \"General information\", \"score\": \"3.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "BoolQ", - "source_data": { - "dataset_name": "BoolQ", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on BoolQ", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.574, - "details": { - "description": "min=0.52, mean=0.574, max=0.623, sum=1.723 (3)\n⚠ Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.", - "tab": "Accuracy", - "BoolQ - ECE (10-bin)": "{\"description\": \"min=0.036, mean=0.068, max=0.089, sum=0.203 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Calibration\", \"score\": \"0.06758031979129187\"}", - "BoolQ - EM (Robustness)": "{\"description\": \"min=0.432, mean=0.477, max=0.522, sum=1.431 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Robustness\", \"score\": \"0.47700000000000004\"}", - "BoolQ - EM (Fairness)": "{\"description\": \"min=0.404, mean=0.436, max=0.457, sum=1.307 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Fairness\", \"score\": \"0.43566666666666665\"}", - "BoolQ - Denoised inference time (s)": "{\"description\": \"min=0.119, mean=0.121, max=0.125, sum=0.364 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Efficiency\", \"score\": \"0.12137238953993056\"}", - "BoolQ - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "BoolQ - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "BoolQ - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "BoolQ - # prompt tokens": "{\"description\": \"min=660.073, mean=908.406, max=1242.073, sum=2725.219 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"908.4063333333334\"}", - "BoolQ - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=3 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "BoolQ - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "BoolQ - Stereotypes (race)": "{\"description\": \"(0)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Stereotypes (gender)": "{\"description\": \"(0)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (race)": "{\"description\": \"(0)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (gender)": "{\"description\": \"(0)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.491, - "details": { - "description": "min=0.468, mean=0.491, max=0.525, sum=1.474 (3)", - "tab": "Accuracy", - "NarrativeQA - ECE (10-bin)": "{\"description\": \"min=0.025, mean=0.027, max=0.03, sum=0.081 (3)\", \"tab\": \"Calibration\", \"score\": \"0.027162479976532598\"}", - "NarrativeQA - F1 (Robustness)": "{\"description\": \"min=0.232, mean=0.255, max=0.266, sum=0.764 (3)\", \"tab\": \"Robustness\", \"score\": \"0.2547490737014401\"}", - "NarrativeQA - F1 (Fairness)": "{\"description\": \"min=0.334, mean=0.367, max=0.396, sum=1.101 (3)\", \"tab\": \"Fairness\", \"score\": \"0.3669650821225828\"}", - "NarrativeQA - Denoised inference time (s)": "{\"description\": \"min=0.164, mean=0.176, max=0.194, sum=0.529 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.1762964825410799\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=1065 (3)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=1.051, mean=1.647, max=2.085, sum=4.941 (3)\", \"tab\": \"General information\", \"score\": \"1.6469483568075116\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=1601.955, mean=1652.377, max=1705.003, sum=4957.132 (3)\", \"tab\": \"General information\", \"score\": \"1652.3774647887324\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=6.659, mean=8.835, max=11.769, sum=26.504 (3)\", \"tab\": \"General information\", \"score\": \"8.83474178403756\"}", - "NarrativeQA - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NarrativeQA - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=0.667 (1)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666667\"}", - "NarrativeQA - Stereotypes (gender)": "{\"description\": \"min=0.404, mean=0.445, max=0.5, sum=1.335 (3)\", \"tab\": \"Bias\", \"score\": \"0.44511511879932936\"}", - "NarrativeQA - Representation (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=1.333 (2)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666667\"}", - "NarrativeQA - Representation (gender)": "{\"description\": \"min=0.142, mean=0.191, max=0.246, sum=0.574 (3)\", \"tab\": \"Bias\", \"score\": \"0.1912053369170701\"}", - "NarrativeQA - Toxic fraction": "{\"description\": \"min=0.011, mean=0.016, max=0.023, sum=0.048 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.01596244131455399\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (open-book)", - "source_data": { - "dataset_name": "NaturalQuestions (open-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (open-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.451, - "details": { - "description": "min=0.435, mean=0.451, max=0.47, sum=1.354 (3)\n⚠ Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.", - "tab": "Accuracy", - "NaturalQuestions (closed-book) - ECE (10-bin)": "{\"description\": \"min=0.012, mean=0.016, max=0.023, sum=0.048 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Calibration\", \"score\": \"0.01603851394023659\"}", - "NaturalQuestions (open-book) - ECE (10-bin)": "{\"description\": \"min=0.141, mean=0.147, max=0.153, sum=0.44 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Calibration\", \"score\": \"0.14681748032197228\"}", - "NaturalQuestions (closed-book) - F1 (Robustness)": "{\"description\": \"min=0.063, mean=0.068, max=0.072, sum=0.205 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Robustness\", \"score\": \"0.06829400341950241\"}", - "NaturalQuestions (open-book) - F1 (Robustness)": "{\"description\": \"min=0.211, mean=0.212, max=0.214, sum=0.637 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Robustness\", \"score\": \"0.21249077319847984\"}", - "NaturalQuestions (closed-book) - F1 (Fairness)": "{\"description\": \"min=0.079, mean=0.084, max=0.088, sum=0.252 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Fairness\", \"score\": \"0.08399089853474369\"}", - "NaturalQuestions (open-book) - F1 (Fairness)": "{\"description\": \"min=0.365, mean=0.381, max=0.403, sum=1.144 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Fairness\", \"score\": \"0.381423207180998\"}", - "NaturalQuestions (closed-book) - Denoised inference time (s)": "{\"description\": \"min=0.15, mean=0.152, max=0.152, sum=0.455 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Efficiency\", \"score\": \"0.15162744531249991\"}", - "NaturalQuestions (open-book) - Denoised inference time (s)": "{\"description\": \"min=0.228, mean=0.232, max=0.235, sum=0.696 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Efficiency\", \"score\": \"0.23211142730034728\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=110.254, mean=112.254, max=116.254, sum=336.762 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"112.254\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=6.994, mean=7.258, max=7.401, sum=21.773 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"7.257666666666666\"}", - "NaturalQuestions (closed-book) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.647, mean=4.691, max=4.724, sum=14.074 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"4.691333333333334\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.036, mean=0.036, max=0.036, sum=0.108 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.036\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1231.212, mean=1419.574, max=1523.257, sum=4258.721 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"1419.5736666666664\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=18.158, mean=18.539, max=18.902, sum=55.617 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"18.539\"}", - "NaturalQuestions (open-book) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NaturalQuestions (closed-book) - Stereotypes (race)": "{\"description\": \"(0)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - Stereotypes (gender)": "{\"description\": \"min=0.5, mean=0.5, max=0.5, sum=1.5 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.5\"}", - "NaturalQuestions (closed-book) - Representation (race)": "{\"description\": \"min=0.578, mean=0.624, max=0.667, sum=1.871 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.6236303630363037\"}", - "NaturalQuestions (closed-book) - Representation (gender)": "{\"description\": \"min=0, mean=0.015, max=0.038, sum=0.046 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.015466015466015476\"}", - "NaturalQuestions (open-book) - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=2 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "NaturalQuestions (open-book) - Stereotypes (gender)": "{\"description\": \"min=0.445, mean=0.479, max=0.5, sum=1.436 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.47855712855712856\"}", - "NaturalQuestions (open-book) - Representation (race)": "{\"description\": \"min=0.422, mean=0.441, max=0.46, sum=1.323 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.44113329919781535\"}", - "NaturalQuestions (open-book) - Representation (gender)": "{\"description\": \"min=0.257, mean=0.349, max=0.419, sum=1.046 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.34872771165606054\"}", - "NaturalQuestions (closed-book) - Toxic fraction": "{\"description\": \"min=0, mean=0.001, max=0.002, sum=0.002 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Toxicity\", \"score\": \"0.0006666666666666666\"}", - "NaturalQuestions (open-book) - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "QuAC", - "source_data": { - "dataset_name": "QuAC", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on QuAC", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.273, - "details": { - "description": "min=0.263, mean=0.273, max=0.282, sum=0.818 (3)\n⚠ Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.", - "tab": "Accuracy", - "QuAC - ECE (10-bin)": "{\"description\": \"min=0.03, mean=0.045, max=0.065, sum=0.136 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Calibration\", \"score\": \"0.04533749534838898\"}", - "QuAC - F1 (Robustness)": "{\"description\": \"min=0.141, mean=0.149, max=0.156, sum=0.448 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Robustness\", \"score\": \"0.14927279809816305\"}", - "QuAC - F1 (Fairness)": "{\"description\": \"min=0.198, mean=0.202, max=0.205, sum=0.607 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Fairness\", \"score\": \"0.20229238580626874\"}", - "QuAC - Denoised inference time (s)": "{\"description\": \"min=0.245, mean=0.261, max=0.27, sum=0.782 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Efficiency\", \"score\": \"0.2607369557291667\"}", - "QuAC - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "QuAC - # train": "{\"description\": \"min=0.845, mean=0.944, max=1.086, sum=2.833 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.9443333333333334\"}", - "QuAC - truncated": "{\"description\": \"min=0.016, mean=0.016, max=0.016, sum=0.048 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.016\"}", - "QuAC - # prompt tokens": "{\"description\": \"min=1625.523, mean=1644.831, max=1670.605, sum=4934.492 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"1644.8306666666667\"}", - "QuAC - # output tokens": "{\"description\": \"min=20.236, mean=22.916, max=24.512, sum=68.749 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"22.91633333333333\"}", - "QuAC - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "QuAC - Stereotypes (race)": "{\"description\": \"min=0.643, mean=0.659, max=0.667, sum=1.976 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.6587301587301589\"}", - "QuAC - Stereotypes (gender)": "{\"description\": \"min=0.434, mean=0.445, max=0.452, sum=1.336 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.4452529926214137\"}", - "QuAC - Representation (race)": "{\"description\": \"min=0.311, mean=0.339, max=0.382, sum=1.016 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.33878845629358273\"}", - "QuAC - Representation (gender)": "{\"description\": \"min=0.251, mean=0.258, max=0.264, sum=0.775 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.25817229310554\"}", - "QuAC - Toxic fraction": "{\"description\": \"min=0.001, mean=0.002, max=0.002, sum=0.005 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Toxicity\", \"score\": \"0.0016666666666666668\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "HellaSwag", - "source_data": { - "dataset_name": "HellaSwag", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on HellaSwag", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.555, - "details": { - "description": "min=0.555, mean=0.555, max=0.555, sum=0.555 (1)\n⚠ Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.", - "tab": "Accuracy", - "HellaSwag - ECE (10-bin)": "{\"description\": \"min=0.144, mean=0.144, max=0.144, sum=0.144 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Calibration\", \"score\": \"0.14430034567571584\"}", - "HellaSwag - EM (Robustness)": "{\"description\": \"min=0.489, mean=0.489, max=0.489, sum=0.489 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Robustness\", \"score\": \"0.489\"}", - "HellaSwag - EM (Fairness)": "{\"description\": \"min=0.401, mean=0.401, max=0.401, sum=0.401 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Fairness\", \"score\": \"0.401\"}", - "HellaSwag - Denoised inference time (s)": "{\"description\": \"min=0.113, mean=0.113, max=0.113, sum=0.113 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Efficiency\", \"score\": \"0.1134031874999998\"}", - "HellaSwag - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "HellaSwag - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "HellaSwag - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "HellaSwag - # prompt tokens": "{\"description\": \"min=87.888, mean=87.888, max=87.888, sum=87.888 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"87.888\"}", - "HellaSwag - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "HellaSwag - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.438, - "details": { - "description": "min=0.438, mean=0.438, max=0.438, sum=0.438 (1)\n⚠ Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.", - "tab": "Accuracy", - "OpenbookQA - ECE (10-bin)": "{\"description\": \"min=0.3, mean=0.3, max=0.3, sum=0.3 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Calibration\", \"score\": \"0.3000308921028506\"}", - "OpenbookQA - EM (Robustness)": "{\"description\": \"min=0.314, mean=0.314, max=0.314, sum=0.314 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Robustness\", \"score\": \"0.314\"}", - "OpenbookQA - EM (Fairness)": "{\"description\": \"min=0.326, mean=0.326, max=0.326, sum=0.326 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Fairness\", \"score\": \"0.326\"}", - "OpenbookQA - Denoised inference time (s)": "{\"description\": \"min=0.111, mean=0.111, max=0.111, sum=0.111 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Efficiency\", \"score\": \"0.11114410156249971\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=5.27, mean=5.27, max=5.27, sum=5.27 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"5.27\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "TruthfulQA", - "source_data": { - "dataset_name": "TruthfulQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on TruthfulQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.188, - "details": { - "description": "min=0.174, mean=0.188, max=0.196, sum=0.563 (3)", - "tab": "Accuracy", - "TruthfulQA - ECE (10-bin)": "{\"description\": \"min=0.13, mean=0.142, max=0.164, sum=0.426 (3)\", \"tab\": \"Calibration\", \"score\": \"0.14198207765086143\"}", - "TruthfulQA - EM (Robustness)": "{\"description\": \"min=0.141, mean=0.162, max=0.183, sum=0.486 (3)\", \"tab\": \"Robustness\", \"score\": \"0.1620795107033639\"}", - "TruthfulQA - EM (Fairness)": "{\"description\": \"min=0.159, mean=0.178, max=0.19, sum=0.534 (3)\", \"tab\": \"Fairness\", \"score\": \"0.17787971457696228\"}", - "TruthfulQA - Denoised inference time (s)": "{\"description\": \"min=0.119, mean=0.12, max=0.12, sum=0.359 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.11970087223655701\"}", - "TruthfulQA - # eval": "{\"description\": \"min=654, mean=654, max=654, sum=1962 (3)\", \"tab\": \"General information\", \"score\": \"654.0\"}", - "TruthfulQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "TruthfulQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "TruthfulQA - # prompt tokens": "{\"description\": \"min=501.121, mean=511.121, max=529.121, sum=1533.362 (3)\", \"tab\": \"General information\", \"score\": \"511.12079510703364\"}", - "TruthfulQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=3 (3)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "TruthfulQA - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MS MARCO (TREC)", - "source_data": { - "dataset_name": "MS MARCO (TREC)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "NDCG@10 on MS MARCO (TREC)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.317, - "details": { - "description": "min=0.291, mean=0.317, max=0.362, sum=0.95 (3)", - "tab": "Accuracy", - "MS MARCO (regular) - RR@10 (Robustness)": "{\"description\": \"min=0.055, mean=0.073, max=0.086, sum=0.219 (3)\", \"tab\": \"Robustness\", \"score\": \"0.07291031746031752\"}", - "MS MARCO (TREC) - NDCG@10 (Robustness)": "{\"description\": \"min=0.206, mean=0.246, max=0.285, sum=0.739 (3)\", \"tab\": \"Robustness\", \"score\": \"0.24641961891165112\"}", - "MS MARCO (regular) - RR@10 (Fairness)": "{\"description\": \"min=0.082, mean=0.105, max=0.123, sum=0.316 (3)\", \"tab\": \"Fairness\", \"score\": \"0.10532936507936512\"}", - "MS MARCO (TREC) - NDCG@10 (Fairness)": "{\"description\": \"min=0.275, mean=0.301, max=0.346, sum=0.902 (3)\", \"tab\": \"Fairness\", \"score\": \"0.300592144197253\"}", - "MS MARCO (regular) - Denoised inference time (s)": "{\"description\": \"min=0.119, mean=0.122, max=0.126, sum=0.367 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.12232188151041663\"}", - "MS MARCO (TREC) - Denoised inference time (s)": "{\"description\": \"min=0.118, mean=0.122, max=0.128, sum=0.367 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.12249798631298452\"}", - "MS MARCO (regular) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "MS MARCO (regular) - # train": "{\"description\": \"min=2, mean=2, max=2, sum=6 (3)\", \"tab\": \"General information\", \"score\": \"2.0\"}", - "MS MARCO (regular) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MS MARCO (regular) - # prompt tokens": "{\"description\": \"min=495.232, mean=532.565, max=577.232, sum=1597.696 (3)\", \"tab\": \"General information\", \"score\": \"532.5653333333333\"}", - "MS MARCO (regular) - # output tokens": "{\"description\": \"min=1.128, mean=1.537, max=2.075, sum=4.612 (3)\", \"tab\": \"General information\", \"score\": \"1.5373333333333334\"}", - "MS MARCO (regular) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "MS MARCO (TREC) - # eval": "{\"description\": \"min=43, mean=43, max=43, sum=129 (3)\", \"tab\": \"General information\", \"score\": \"43.0\"}", - "MS MARCO (TREC) - # train": "{\"description\": \"min=2, mean=2, max=2, sum=6 (3)\", \"tab\": \"General information\", \"score\": \"2.0\"}", - "MS MARCO (TREC) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MS MARCO (TREC) - # prompt tokens": "{\"description\": \"min=478.488, mean=515.822, max=560.488, sum=1547.465 (3)\", \"tab\": \"General information\", \"score\": \"515.8217054263565\"}", - "MS MARCO (TREC) - # output tokens": "{\"description\": \"min=1, mean=1.496, max=2.302, sum=4.488 (3)\", \"tab\": \"General information\", \"score\": \"1.4961240310077522\"}", - "MS MARCO (TREC) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "MS MARCO (regular) - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "MS MARCO (TREC) - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CNN/DailyMail", - "source_data": { - "dataset_name": "CNN/DailyMail", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on CNN/DailyMail", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.079, - "details": { - "description": "min=0.016, mean=0.079, max=0.147, sum=0.472 (6)", - "tab": "Accuracy", - "CNN/DailyMail - Denoised inference time (s)": "{\"description\": \"min=0.293, mean=0.533, max=0.795, sum=3.197 (6)\", \"tab\": \"Efficiency\", \"score\": \"0.5327935382950345\"}", - "CNN/DailyMail - # eval": "{\"description\": \"min=466, mean=466, max=466, sum=2796 (6)\", \"tab\": \"General information\", \"score\": \"466.0\"}", - "CNN/DailyMail - # train": "{\"description\": \"min=5, mean=5, max=5, sum=30 (6)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CNN/DailyMail - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (6)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CNN/DailyMail - # prompt tokens": "{\"description\": \"min=1531.586, mean=1549.919, max=1567.586, sum=9299.515 (6)\", \"tab\": \"General information\", \"score\": \"1549.9191702432045\"}", - "CNN/DailyMail - # output tokens": "{\"description\": \"min=28.479, mean=68.44, max=112.258, sum=410.639 (6)\", \"tab\": \"General information\", \"score\": \"68.43991416309014\"}", - "CNN/DailyMail - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=18 (6)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "CNN/DailyMail - Stereotypes (race)": "{\"description\": \"min=0.5, mean=0.568, max=0.611, sum=3.41 (6)\", \"tab\": \"Bias\", \"score\": \"0.5683358120009704\"}", - "CNN/DailyMail - Stereotypes (gender)": "{\"description\": \"min=0.403, mean=0.418, max=0.435, sum=2.509 (6)\", \"tab\": \"Bias\", \"score\": \"0.4181282755076701\"}", - "CNN/DailyMail - Representation (race)": "{\"description\": \"min=0.321, mean=0.327, max=0.333, sum=1.962 (6)\", \"tab\": \"Bias\", \"score\": \"0.32700197854837026\"}", - "CNN/DailyMail - Representation (gender)": "{\"description\": \"min=0.131, mean=0.146, max=0.165, sum=0.879 (6)\", \"tab\": \"Bias\", \"score\": \"0.14643429372740835\"}", - "CNN/DailyMail - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (6)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}", - "CNN/DailyMail - SummaC": "{\"description\": \"min=-0.024, mean=0.194, max=0.404, sum=0.582 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.19395910509097278\"}", - "CNN/DailyMail - QAFactEval": "{\"description\": \"min=1.208, mean=3.207, max=4.672, sum=19.24 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"3.206720080183251\"}", - "CNN/DailyMail - BERTScore (F1)": "{\"description\": \"min=-0.533, mean=-0.129, max=0.256, sum=-0.388 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"-0.12942978993545518\"}", - "CNN/DailyMail - Coverage": "{\"description\": \"min=0.243, mean=0.606, max=0.942, sum=3.637 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"0.6061106279492011\"}", - "CNN/DailyMail - Density": "{\"description\": \"min=7.213, mean=43.534, max=84.961, sum=261.202 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"43.533595505945534\"}", - "CNN/DailyMail - Compression": "{\"description\": \"min=5.569, mean=6.733, max=8.376, sum=40.398 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"6.733051993966683\"}", - "CNN/DailyMail - HumanEval-faithfulness": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-relevance": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-coherence": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "XSUM", - "source_data": { - "dataset_name": "XSUM", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on XSUM", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.045, - "details": { - "description": "min=0.041, mean=0.045, max=0.054, sum=0.273 (6)", - "tab": "Accuracy", - "XSUM - Denoised inference time (s)": "{\"description\": \"min=0.264, mean=0.272, max=0.286, sum=1.632 (6)\", \"tab\": \"Efficiency\", \"score\": \"0.27202574924254597\"}", - "XSUM - # eval": "{\"description\": \"min=518, mean=518, max=518, sum=3108 (6)\", \"tab\": \"General information\", \"score\": \"518.0\"}", - "XSUM - # train": "{\"description\": \"min=4.998, mean=4.999, max=5, sum=29.992 (6)\", \"tab\": \"General information\", \"score\": \"4.998712998712999\"}", - "XSUM - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (6)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "XSUM - # prompt tokens": "{\"description\": \"min=1456.402, mean=1510.418, max=1538.921, sum=9062.51 (6)\", \"tab\": \"General information\", \"score\": \"1510.4182754182755\"}", - "XSUM - # output tokens": "{\"description\": \"min=23.645, mean=25.051, max=27.259, sum=150.309 (6)\", \"tab\": \"General information\", \"score\": \"25.051480051480052\"}", - "XSUM - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=18 (6)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "XSUM - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=4 (6)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "XSUM - Stereotypes (gender)": "{\"description\": \"min=0.389, mean=0.42, max=0.46, sum=2.52 (6)\", \"tab\": \"Bias\", \"score\": \"0.42004149135109864\"}", - "XSUM - Representation (race)": "{\"description\": \"min=0.417, mean=0.458, max=0.542, sum=2.75 (6)\", \"tab\": \"Bias\", \"score\": \"0.4583333333333333\"}", - "XSUM - Representation (gender)": "{\"description\": \"min=0.105, mean=0.148, max=0.182, sum=0.89 (6)\", \"tab\": \"Bias\", \"score\": \"0.14837887499687488\"}", - "XSUM - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (6)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}", - "XSUM - SummaC": "{\"description\": \"min=-0.221, mean=-0.188, max=-0.16, sum=-0.564 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"-0.18805348402642733\"}", - "XSUM - QAFactEval": "{\"description\": \"min=0.003, mean=0.195, max=0.546, sum=1.171 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"0.19517962440346606\"}", - "XSUM - BERTScore (F1)": "{\"description\": \"min=-0.047, mean=0.02, max=0.139, sum=0.059 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.01972435572139075\"}", - "XSUM - Coverage": "{\"description\": \"min=0.538, mean=0.604, max=0.715, sum=3.622 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"0.6037080043294082\"}", - "XSUM - Density": "{\"description\": \"min=3.597, mean=4.386, max=5.935, sum=26.316 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"4.385950410054523\"}", - "XSUM - Compression": "{\"description\": \"min=10.355, mean=11.716, max=13.636, sum=70.293 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"11.71557516895029\"}", - "XSUM - HumanEval-faithfulness": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-relevance": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-coherence": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "IMDB", - "source_data": { - "dataset_name": "IMDB", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on IMDB", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.597, - "details": { - "description": "min=0.5, mean=0.597, max=0.646, sum=1.792 (3)", - "tab": "Accuracy", - "IMDB - ECE (10-bin)": "{\"description\": \"min=0.161, mean=0.212, max=0.289, sum=0.637 (3)\", \"tab\": \"Calibration\", \"score\": \"0.2122386190139247\"}", - "IMDB - EM (Robustness)": "{\"description\": \"min=0.476, mean=0.5, max=0.512, sum=1.5 (3)\", \"tab\": \"Robustness\", \"score\": \"0.5\"}", - "IMDB - EM (Fairness)": "{\"description\": \"min=0.489, mean=0.534, max=0.558, sum=1.602 (3)\", \"tab\": \"Fairness\", \"score\": \"0.534\"}", - "IMDB - Denoised inference time (s)": "{\"description\": \"min=0.125, mean=0.128, max=0.131, sum=0.385 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.12819260763888898\"}", - "IMDB - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "IMDB - # train": "{\"description\": \"min=2.916, mean=4.242, max=4.986, sum=12.726 (3)\", \"tab\": \"General information\", \"score\": \"4.242\"}", - "IMDB - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IMDB - # prompt tokens": "{\"description\": \"min=1277.729, mean=1553.363, max=1768.607, sum=4660.089 (3)\", \"tab\": \"General information\", \"score\": \"1553.363\"}", - "IMDB - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=3 (3)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "IMDB - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "IMDB - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CivilComments", - "source_data": { - "dataset_name": "CivilComments", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on CivilComments", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.519, - "details": { - "description": "min=0.005, mean=0.519, max=0.996, sum=28.025 (54)", - "tab": "Accuracy", - "CivilComments - ECE (10-bin)": "{\"description\": \"min=0.063, mean=0.31, max=0.598, sum=16.723 (54)\", \"tab\": \"Calibration\", \"score\": \"0.30968147474692964\"}", - "CivilComments - EM (Robustness)": "{\"description\": \"min=0, mean=0.4, max=0.996, sum=21.618 (54)\", \"tab\": \"Robustness\", \"score\": \"0.40032672585199003\"}", - "CivilComments - EM (Fairness)": "{\"description\": \"min=0, mean=0.474, max=0.994, sum=25.57 (54)\", \"tab\": \"Fairness\", \"score\": \"0.4735149158411243\"}", - "CivilComments - Denoised inference time (s)": "{\"description\": \"min=0.118, mean=0.12, max=0.125, sum=6.485 (54)\", \"tab\": \"Efficiency\", \"score\": \"0.12008918109610113\"}", - "CivilComments - # eval": "{\"description\": \"min=74, mean=371.556, max=683, sum=20064 (54)\", \"tab\": \"General information\", \"score\": \"371.55555555555554\"}", - "CivilComments - # train": "{\"description\": \"min=5, mean=5, max=5, sum=270 (54)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CivilComments - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (54)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CivilComments - # prompt tokens": "{\"description\": \"min=356.537, mean=722.635, max=1267.519, sum=39022.317 (54)\", \"tab\": \"General information\", \"score\": \"722.6354931173206\"}", - "CivilComments - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=54 (54)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "CivilComments - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=162 (54)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "CivilComments - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (54)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "RAFT", - "source_data": { - "dataset_name": "RAFT", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on RAFT", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.455, - "details": { - "description": "min=0.025, mean=0.455, max=0.975, sum=15.025 (33)", - "tab": "Accuracy", - "RAFT - ECE (10-bin)": "{\"description\": \"min=0.1, mean=0.286, max=0.455, sum=9.428 (33)\", \"tab\": \"Calibration\", \"score\": \"0.28570502706051176\"}", - "RAFT - EM (Robustness)": "{\"description\": \"min=0, mean=0.409, max=0.975, sum=13.5 (33)\", \"tab\": \"Robustness\", \"score\": \"0.40909090909090906\"}", - "RAFT - EM (Fairness)": "{\"description\": \"min=0, mean=0.438, max=0.975, sum=14.45 (33)\", \"tab\": \"Fairness\", \"score\": \"0.43787878787878787\"}", - "RAFT - Denoised inference time (s)": "{\"description\": \"min=0.117, mean=0.137, max=0.182, sum=4.525 (33)\", \"tab\": \"Efficiency\", \"score\": \"0.13711408420138893\"}", - "RAFT - # eval": "{\"description\": \"min=40, mean=40, max=40, sum=1320 (33)\", \"tab\": \"General information\", \"score\": \"40.0\"}", - "RAFT - # train": "{\"description\": \"min=0, mean=4.556, max=5, sum=150.35 (33)\", \"tab\": \"General information\", \"score\": \"4.556060606060607\"}", - "RAFT - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (33)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "RAFT - # prompt tokens": "{\"description\": \"min=257.35, mean=812.938, max=1773.675, sum=26826.95 (33)\", \"tab\": \"General information\", \"score\": \"812.937878787879\"}", - "RAFT - # output tokens": "{\"description\": \"min=1, mean=3.511, max=10.6, sum=115.85 (33)\", \"tab\": \"General information\", \"score\": \"3.5106060606060603\"}", - "RAFT - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=99 (33)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "RAFT - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (gender)": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"Bias\", \"score\": \"0.0\"}", - "RAFT - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (33)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/openai_curie-6.7b.json b/data/models/openai_curie-6.7b.json deleted file mode 100644 index 929eb9502c9882445595ed725ce0cdf883f78f06..0000000000000000000000000000000000000000 --- a/data/models/openai_curie-6.7b.json +++ /dev/null @@ -1,674 +0,0 @@ -{ - "model_info": { - "name": "curie 6.7B", - "id": "openai/curie-6.7B", - "developer": "OpenAI", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_classic/openai_curie-6.7B/1774096308.339228", - "retrieved_timestamp": "1774096308.339228", - "source_metadata": { - "source_name": "helm_classic", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_classic", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_classic", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperform on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.247, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Calibration": "{\"description\": \"\", \"tab\": \"Calibration\", \"score\": \"0.6031752149929763\"}", - "Mean win rate - Robustness": "{\"description\": \"\", \"tab\": \"Robustness\", \"score\": \"0.23139443056017028\"}", - "Mean win rate - Fairness": "{\"description\": \"\", \"tab\": \"Fairness\", \"score\": \"0.23055057660174458\"}", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.8951315789473684\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}", - "Mean win rate - Bias": "{\"description\": \"\", \"tab\": \"Bias\", \"score\": \"0.36598228279277495\"}", - "Mean win rate - Toxicity": "{\"description\": \"\", \"tab\": \"Toxicity\", \"score\": \"0.4175808759142092\"}", - "Mean win rate - Summarization metrics": "{\"description\": \"\", \"tab\": \"Summarization metrics\", \"score\": \"0.32471804511278196\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.243, - "details": { - "description": "min=0.19, mean=0.243, max=0.29, sum=3.642 (15)", - "tab": "Accuracy", - "MMLU - ECE (10-bin)": "{\"description\": \"min=0.069, mean=0.138, max=0.238, sum=2.071 (15)\", \"tab\": \"Calibration\", \"score\": \"0.1380385889615569\"}", - "MMLU - EM (Robustness)": "{\"description\": \"min=0.1, mean=0.19, max=0.263, sum=2.854 (15)\", \"tab\": \"Robustness\", \"score\": \"0.1902923976608187\"}", - "MMLU - EM (Fairness)": "{\"description\": \"min=0.15, mean=0.218, max=0.281, sum=3.266 (15)\", \"tab\": \"Fairness\", \"score\": \"0.21771929824561406\"}", - "MMLU - Denoised inference time (s)": "{\"description\": \"min=0.091, mean=0.092, max=0.095, sum=1.387 (15)\", \"tab\": \"Efficiency\", \"score\": \"0.09245237979714913\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=1542 (15)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=75 (15)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (15)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=371.38, mean=472.274, max=624.07, sum=7084.111 (15)\", \"tab\": \"General information\", \"score\": \"472.2740350877193\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=15 (15)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "MMLU - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=45 (15)\", \"tab\": \"General information\", \"score\": \"3.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "BoolQ", - "source_data": { - "dataset_name": "BoolQ", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on BoolQ", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.656, - "details": { - "description": "min=0.597, mean=0.656, max=0.704, sum=1.969 (3)\n⚠ Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.", - "tab": "Accuracy", - "BoolQ - ECE (10-bin)": "{\"description\": \"min=0.051, mean=0.079, max=0.115, sum=0.236 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Calibration\", \"score\": \"0.07881150352718548\"}", - "BoolQ - EM (Robustness)": "{\"description\": \"min=0.484, mean=0.545, max=0.599, sum=1.635 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Robustness\", \"score\": \"0.545\"}", - "BoolQ - EM (Fairness)": "{\"description\": \"min=0.535, mean=0.594, max=0.631, sum=1.782 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Fairness\", \"score\": \"0.594\"}", - "BoolQ - Denoised inference time (s)": "{\"description\": \"min=0.096, mean=0.1, max=0.104, sum=0.3 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Efficiency\", \"score\": \"0.09988102712673615\"}", - "BoolQ - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "BoolQ - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "BoolQ - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "BoolQ - # prompt tokens": "{\"description\": \"min=660.073, mean=908.406, max=1242.073, sum=2725.219 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"908.4063333333334\"}", - "BoolQ - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=3 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "BoolQ - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "BoolQ - Stereotypes (race)": "{\"description\": \"(0)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Stereotypes (gender)": "{\"description\": \"(0)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (race)": "{\"description\": \"(0)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (gender)": "{\"description\": \"(0)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.604, - "details": { - "description": "min=0.588, mean=0.604, max=0.632, sum=1.813 (3)", - "tab": "Accuracy", - "NarrativeQA - ECE (10-bin)": "{\"description\": \"min=0.031, mean=0.045, max=0.056, sum=0.135 (3)\", \"tab\": \"Calibration\", \"score\": \"0.044936394093581626\"}", - "NarrativeQA - F1 (Robustness)": "{\"description\": \"min=0.352, mean=0.367, max=0.39, sum=1.1 (3)\", \"tab\": \"Robustness\", \"score\": \"0.36665112128820915\"}", - "NarrativeQA - F1 (Fairness)": "{\"description\": \"min=0.453, mean=0.482, max=0.515, sum=1.445 (3)\", \"tab\": \"Fairness\", \"score\": \"0.48150959406800437\"}", - "NarrativeQA - Denoised inference time (s)": "{\"description\": \"min=0.14, mean=0.152, max=0.166, sum=0.455 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.15159477332746474\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=1065 (3)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=1.051, mean=1.647, max=2.085, sum=4.941 (3)\", \"tab\": \"General information\", \"score\": \"1.6469483568075116\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=1601.955, mean=1652.377, max=1705.003, sum=4957.132 (3)\", \"tab\": \"General information\", \"score\": \"1652.3774647887324\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=4.775, mean=6.607, max=8.732, sum=19.82 (3)\", \"tab\": \"General information\", \"score\": \"6.606572769953051\"}", - "NarrativeQA - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NarrativeQA - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NarrativeQA - Stereotypes (gender)": "{\"description\": \"min=0.431, mean=0.455, max=0.5, sum=1.364 (3)\", \"tab\": \"Bias\", \"score\": \"0.45462962962962966\"}", - "NarrativeQA - Representation (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=1.333 (2)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666667\"}", - "NarrativeQA - Representation (gender)": "{\"description\": \"min=0.209, mean=0.229, max=0.267, sum=0.688 (3)\", \"tab\": \"Bias\", \"score\": \"0.2292955082742317\"}", - "NarrativeQA - Toxic fraction": "{\"description\": \"min=0.017, mean=0.017, max=0.017, sum=0.051 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.016901408450704224\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (open-book)", - "source_data": { - "dataset_name": "NaturalQuestions (open-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (open-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.552, - "details": { - "description": "min=0.521, mean=0.552, max=0.568, sum=1.655 (3)\n⚠ Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.", - "tab": "Accuracy", - "NaturalQuestions (closed-book) - ECE (10-bin)": "{\"description\": \"min=0.014, mean=0.017, max=0.022, sum=0.052 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Calibration\", \"score\": \"0.01724854000741595\"}", - "NaturalQuestions (open-book) - ECE (10-bin)": "{\"description\": \"min=0.123, mean=0.134, max=0.149, sum=0.403 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Calibration\", \"score\": \"0.13427394452181574\"}", - "NaturalQuestions (closed-book) - F1 (Robustness)": "{\"description\": \"min=0.118, mean=0.126, max=0.133, sum=0.379 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Robustness\", \"score\": \"0.1262678947150161\"}", - "NaturalQuestions (open-book) - F1 (Robustness)": "{\"description\": \"min=0.28, mean=0.338, max=0.381, sum=1.015 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Robustness\", \"score\": \"0.33838638278361\"}", - "NaturalQuestions (closed-book) - F1 (Fairness)": "{\"description\": \"min=0.139, mean=0.147, max=0.151, sum=0.44 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Fairness\", \"score\": \"0.14670404179376148\"}", - "NaturalQuestions (open-book) - F1 (Fairness)": "{\"description\": \"min=0.446, mean=0.479, max=0.506, sum=1.436 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Fairness\", \"score\": \"0.47851717891712475\"}", - "NaturalQuestions (closed-book) - Denoised inference time (s)": "{\"description\": \"min=0.116, mean=0.122, max=0.128, sum=0.367 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Efficiency\", \"score\": \"0.12234622395833335\"}", - "NaturalQuestions (open-book) - Denoised inference time (s)": "{\"description\": \"min=0.166, mean=0.189, max=0.21, sum=0.566 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Efficiency\", \"score\": \"0.18882224978298598\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=110.254, mean=112.254, max=116.254, sum=336.762 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"112.254\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=5.376, mean=6.313, max=7.104, sum=18.94 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"6.3133333333333335\"}", - "NaturalQuestions (closed-book) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.647, mean=4.691, max=4.724, sum=14.074 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"4.691333333333334\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.036, mean=0.036, max=0.036, sum=0.108 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.036\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1231.212, mean=1419.574, max=1523.257, sum=4258.721 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"1419.5736666666664\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=9.89, mean=12.581, max=15.337, sum=37.742 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"12.580666666666668\"}", - "NaturalQuestions (open-book) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NaturalQuestions (closed-book) - Stereotypes (race)": "{\"description\": \"(0)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - Stereotypes (gender)": "{\"description\": \"min=0.5, mean=0.5, max=0.5, sum=1 (2)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.5\"}", - "NaturalQuestions (closed-book) - Representation (race)": "{\"description\": \"min=0.291, mean=0.415, max=0.509, sum=1.245 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.4150858887700994\"}", - "NaturalQuestions (closed-book) - Representation (gender)": "{\"description\": \"min=0.119, mean=0.203, max=0.25, sum=0.608 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.20272601794340928\"}", - "NaturalQuestions (open-book) - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=0.667 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.6666666666666667\"}", - "NaturalQuestions (open-book) - Stereotypes (gender)": "{\"description\": \"min=0.407, mean=0.469, max=0.5, sum=1.407 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.469047619047619\"}", - "NaturalQuestions (open-book) - Representation (race)": "{\"description\": \"min=0.441, mean=0.453, max=0.467, sum=1.359 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.4528357579590976\"}", - "NaturalQuestions (open-book) - Representation (gender)": "{\"description\": \"min=0.361, mean=0.379, max=0.397, sum=1.136 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.3786428074398272\"}", - "NaturalQuestions (closed-book) - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}", - "NaturalQuestions (open-book) - Toxic fraction": "{\"description\": \"min=0.001, mean=0.002, max=0.003, sum=0.005 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Toxicity\", \"score\": \"0.0016666666666666668\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "QuAC", - "source_data": { - "dataset_name": "QuAC", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on QuAC", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.321, - "details": { - "description": "min=0.312, mean=0.321, max=0.335, sum=0.963 (3)\n⚠ Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.", - "tab": "Accuracy", - "QuAC - ECE (10-bin)": "{\"description\": \"min=0.033, mean=0.043, max=0.055, sum=0.129 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Calibration\", \"score\": \"0.04303687950629059\"}", - "QuAC - F1 (Robustness)": "{\"description\": \"min=0.164, mean=0.171, max=0.178, sum=0.513 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Robustness\", \"score\": \"0.1711623480279509\"}", - "QuAC - F1 (Fairness)": "{\"description\": \"min=0.241, mean=0.243, max=0.245, sum=0.728 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Fairness\", \"score\": \"0.24255939370982219\"}", - "QuAC - Denoised inference time (s)": "{\"description\": \"min=0.31, mean=0.323, max=0.34, sum=0.968 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Efficiency\", \"score\": \"0.32252038281250045\"}", - "QuAC - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "QuAC - # train": "{\"description\": \"min=0.845, mean=0.944, max=1.086, sum=2.833 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.9443333333333334\"}", - "QuAC - truncated": "{\"description\": \"min=0.016, mean=0.016, max=0.016, sum=0.048 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.016\"}", - "QuAC - # prompt tokens": "{\"description\": \"min=1625.523, mean=1644.831, max=1670.605, sum=4934.492 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"1644.8306666666667\"}", - "QuAC - # output tokens": "{\"description\": \"min=29.104, mean=31.034, max=33.548, sum=93.102 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"31.034000000000002\"}", - "QuAC - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "QuAC - Stereotypes (race)": "{\"description\": \"min=0.633, mean=0.645, max=0.667, sum=1.936 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.6454545454545455\"}", - "QuAC - Stereotypes (gender)": "{\"description\": \"min=0.426, mean=0.439, max=0.452, sum=1.317 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.4390862600512319\"}", - "QuAC - Representation (race)": "{\"description\": \"min=0.2, mean=0.246, max=0.271, sum=0.738 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.24599483204134365\"}", - "QuAC - Representation (gender)": "{\"description\": \"min=0.226, mean=0.231, max=0.234, sum=0.693 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.23109052551695608\"}", - "QuAC - Toxic fraction": "{\"description\": \"min=0.002, mean=0.003, max=0.003, sum=0.008 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Toxicity\", \"score\": \"0.0026666666666666666\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "HellaSwag", - "source_data": { - "dataset_name": "HellaSwag", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on HellaSwag", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.682, - "details": { - "description": "min=0.682, mean=0.682, max=0.682, sum=0.682 (1)\n⚠ Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.", - "tab": "Accuracy", - "HellaSwag - ECE (10-bin)": "{\"description\": \"min=0.25, mean=0.25, max=0.25, sum=0.25 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Calibration\", \"score\": \"0.24965148877506194\"}", - "HellaSwag - EM (Robustness)": "{\"description\": \"min=0.632, mean=0.632, max=0.632, sum=0.632 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Robustness\", \"score\": \"0.632\"}", - "HellaSwag - EM (Fairness)": "{\"description\": \"min=0.522, mean=0.522, max=0.522, sum=0.522 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Fairness\", \"score\": \"0.522\"}", - "HellaSwag - Denoised inference time (s)": "{\"description\": \"min=0.084, mean=0.084, max=0.084, sum=0.084 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Efficiency\", \"score\": \"0.08380637499999992\"}", - "HellaSwag - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "HellaSwag - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "HellaSwag - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "HellaSwag - # prompt tokens": "{\"description\": \"min=87.888, mean=87.888, max=87.888, sum=87.888 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"87.888\"}", - "HellaSwag - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "HellaSwag - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.502, - "details": { - "description": "min=0.502, mean=0.502, max=0.502, sum=0.502 (1)\n⚠ Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.", - "tab": "Accuracy", - "OpenbookQA - ECE (10-bin)": "{\"description\": \"min=0.26, mean=0.26, max=0.26, sum=0.26 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Calibration\", \"score\": \"0.25956257561884827\"}", - "OpenbookQA - EM (Robustness)": "{\"description\": \"min=0.396, mean=0.396, max=0.396, sum=0.396 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Robustness\", \"score\": \"0.396\"}", - "OpenbookQA - EM (Fairness)": "{\"description\": \"min=0.43, mean=0.43, max=0.43, sum=0.43 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Fairness\", \"score\": \"0.43\"}", - "OpenbookQA - Denoised inference time (s)": "{\"description\": \"min=0.079, mean=0.079, max=0.079, sum=0.079 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Efficiency\", \"score\": \"0.07928820312499986\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=5.27, mean=5.27, max=5.27, sum=5.27 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"5.27\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "TruthfulQA", - "source_data": { - "dataset_name": "TruthfulQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on TruthfulQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.232, - "details": { - "description": "min=0.222, mean=0.232, max=0.251, sum=0.696 (3)", - "tab": "Accuracy", - "TruthfulQA - ECE (10-bin)": "{\"description\": \"min=0.05, mean=0.062, max=0.072, sum=0.186 (3)\", \"tab\": \"Calibration\", \"score\": \"0.06204978796421436\"}", - "TruthfulQA - EM (Robustness)": "{\"description\": \"min=0.167, mean=0.186, max=0.214, sum=0.557 (3)\", \"tab\": \"Robustness\", \"score\": \"0.1855249745158002\"}", - "TruthfulQA - EM (Fairness)": "{\"description\": \"min=0.165, mean=0.186, max=0.216, sum=0.558 (3)\", \"tab\": \"Fairness\", \"score\": \"0.18603465851172274\"}", - "TruthfulQA - Denoised inference time (s)": "{\"description\": \"min=0.093, mean=0.094, max=0.094, sum=0.281 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.09360438168960249\"}", - "TruthfulQA - # eval": "{\"description\": \"min=654, mean=654, max=654, sum=1962 (3)\", \"tab\": \"General information\", \"score\": \"654.0\"}", - "TruthfulQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "TruthfulQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "TruthfulQA - # prompt tokens": "{\"description\": \"min=501.121, mean=511.121, max=529.121, sum=1533.362 (3)\", \"tab\": \"General information\", \"score\": \"511.12079510703364\"}", - "TruthfulQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=3 (3)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "TruthfulQA - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MS MARCO (TREC)", - "source_data": { - "dataset_name": "MS MARCO (TREC)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "NDCG@10 on MS MARCO (TREC)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3, - "details": { - "description": "min=0.279, mean=0.3, max=0.31, sum=0.899 (3)", - "tab": "Accuracy", - "MS MARCO (regular) - RR@10 (Robustness)": "{\"description\": \"min=0.086, mean=0.11, max=0.14, sum=0.33 (3)\", \"tab\": \"Robustness\", \"score\": \"0.10991481481481481\"}", - "MS MARCO (TREC) - NDCG@10 (Robustness)": "{\"description\": \"min=0.25, mean=0.253, max=0.254, sum=0.759 (3)\", \"tab\": \"Robustness\", \"score\": \"0.25287196320995325\"}", - "MS MARCO (regular) - RR@10 (Fairness)": "{\"description\": \"min=0.119, mean=0.14, max=0.167, sum=0.42 (3)\", \"tab\": \"Fairness\", \"score\": \"0.14012791005291\"}", - "MS MARCO (TREC) - NDCG@10 (Fairness)": "{\"description\": \"min=0.266, mean=0.284, max=0.295, sum=0.852 (3)\", \"tab\": \"Fairness\", \"score\": \"0.2838824123845733\"}", - "MS MARCO (regular) - Denoised inference time (s)": "{\"description\": \"min=0.094, mean=0.094, max=0.095, sum=0.283 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.09442029557291665\"}", - "MS MARCO (TREC) - Denoised inference time (s)": "{\"description\": \"min=0.094, mean=0.095, max=0.097, sum=0.286 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.09531934350775194\"}", - "MS MARCO (regular) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "MS MARCO (regular) - # train": "{\"description\": \"min=2, mean=2, max=2, sum=6 (3)\", \"tab\": \"General information\", \"score\": \"2.0\"}", - "MS MARCO (regular) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MS MARCO (regular) - # prompt tokens": "{\"description\": \"min=495.232, mean=532.565, max=577.232, sum=1597.696 (3)\", \"tab\": \"General information\", \"score\": \"532.5653333333333\"}", - "MS MARCO (regular) - # output tokens": "{\"description\": \"min=1.035, mean=1.112, max=1.183, sum=3.336 (3)\", \"tab\": \"General information\", \"score\": \"1.112\"}", - "MS MARCO (regular) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "MS MARCO (TREC) - # eval": "{\"description\": \"min=43, mean=43, max=43, sum=129 (3)\", \"tab\": \"General information\", \"score\": \"43.0\"}", - "MS MARCO (TREC) - # train": "{\"description\": \"min=2, mean=2, max=2, sum=6 (3)\", \"tab\": \"General information\", \"score\": \"2.0\"}", - "MS MARCO (TREC) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MS MARCO (TREC) - # prompt tokens": "{\"description\": \"min=478.488, mean=515.822, max=560.488, sum=1547.465 (3)\", \"tab\": \"General information\", \"score\": \"515.8217054263565\"}", - "MS MARCO (TREC) - # output tokens": "{\"description\": \"min=1.093, mean=1.248, max=1.488, sum=3.744 (3)\", \"tab\": \"General information\", \"score\": \"1.248062015503876\"}", - "MS MARCO (TREC) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "MS MARCO (regular) - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "MS MARCO (TREC) - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CNN/DailyMail", - "source_data": { - "dataset_name": "CNN/DailyMail", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on CNN/DailyMail", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.113, - "details": { - "description": "min=0.038, mean=0.113, max=0.141, sum=0.789 (7)", - "tab": "Accuracy", - "CNN/DailyMail - Denoised inference time (s)": "{\"description\": \"min=0.559, mean=0.623, max=0.691, sum=4.363 (7)\", \"tab\": \"Efficiency\", \"score\": \"0.6232588631080115\"}", - "CNN/DailyMail - # eval": "{\"description\": \"min=466, mean=466, max=466, sum=3262 (7)\", \"tab\": \"General information\", \"score\": \"466.0\"}", - "CNN/DailyMail - # train": "{\"description\": \"min=0, mean=4.286, max=5, sum=30 (7)\", \"tab\": \"General information\", \"score\": \"4.285714285714286\"}", - "CNN/DailyMail - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CNN/DailyMail - # prompt tokens": "{\"description\": \"min=583.586, mean=1411.872, max=1567.586, sum=9883.101 (7)\", \"tab\": \"General information\", \"score\": \"1411.8715511955854\"}", - "CNN/DailyMail - # output tokens": "{\"description\": \"min=65.127, mean=74.606, max=84.073, sum=522.245 (7)\", \"tab\": \"General information\", \"score\": \"74.60637645616187\"}", - "CNN/DailyMail - # trials": "{\"description\": \"min=1, mean=2.714, max=3, sum=19 (7)\", \"tab\": \"General information\", \"score\": \"2.7142857142857144\"}", - "CNN/DailyMail - Stereotypes (race)": "{\"description\": \"min=0.619, mean=0.642, max=0.667, sum=4.492 (7)\", \"tab\": \"Bias\", \"score\": \"0.6416796928441896\"}", - "CNN/DailyMail - Stereotypes (gender)": "{\"description\": \"min=0.383, mean=0.409, max=0.43, sum=2.86 (7)\", \"tab\": \"Bias\", \"score\": \"0.40861926379951435\"}", - "CNN/DailyMail - Representation (race)": "{\"description\": \"min=0.238, mean=0.295, max=0.417, sum=2.068 (7)\", \"tab\": \"Bias\", \"score\": \"0.29545894187058713\"}", - "CNN/DailyMail - Representation (gender)": "{\"description\": \"min=0.109, mean=0.129, max=0.144, sum=0.9 (7)\", \"tab\": \"Bias\", \"score\": \"0.12851266312443646\"}", - "CNN/DailyMail - Toxic fraction": "{\"description\": \"min=0, mean=0.001, max=0.002, sum=0.004 (7)\", \"tab\": \"Toxicity\", \"score\": \"0.0006131207847946045\"}", - "CNN/DailyMail - SummaC": "{\"description\": \"min=-0.108, mean=0.354, max=0.557, sum=1.415 (4)\", \"tab\": \"Summarization metrics\", \"score\": \"0.3538436304603978\"}", - "CNN/DailyMail - QAFactEval": "{\"description\": \"min=1.248, mean=4.204, max=4.78, sum=29.431 (7)\", \"tab\": \"Summarization metrics\", \"score\": \"4.20445410382703\"}", - "CNN/DailyMail - BERTScore (F1)": "{\"description\": \"min=-0.343, mean=0.089, max=0.264, sum=0.355 (4)\", \"tab\": \"Summarization metrics\", \"score\": \"0.08867060792677807\"}", - "CNN/DailyMail - Coverage": "{\"description\": \"min=0.425, mean=0.89, max=0.973, sum=6.231 (7)\", \"tab\": \"Summarization metrics\", \"score\": \"0.8901263761958778\"}", - "CNN/DailyMail - Density": "{\"description\": \"min=11.471, mean=23.472, max=34.455, sum=164.303 (7)\", \"tab\": \"Summarization metrics\", \"score\": \"23.471817181725523\"}", - "CNN/DailyMail - Compression": "{\"description\": \"min=5.037, mean=9.495, max=12.229, sum=66.463 (7)\", \"tab\": \"Summarization metrics\", \"score\": \"9.494670330829432\"}", - "CNN/DailyMail - HumanEval-faithfulness": "{\"description\": \"min=0.287, mean=0.287, max=0.287, sum=0.287 (1)\", \"tab\": \"Summarization metrics\", \"score\": \"0.2866666666666666\"}", - "CNN/DailyMail - HumanEval-relevance": "{\"description\": \"min=1.933, mean=1.933, max=1.933, sum=1.933 (1)\", \"tab\": \"Summarization metrics\", \"score\": \"1.9333333333333333\"}", - "CNN/DailyMail - HumanEval-coherence": "{\"description\": \"min=1.767, mean=1.767, max=1.767, sum=1.767 (1)\", \"tab\": \"Summarization metrics\", \"score\": \"1.7666666666666666\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "XSUM", - "source_data": { - "dataset_name": "XSUM", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on XSUM", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.091, - "details": { - "description": "min=0.035, mean=0.091, max=0.104, sum=0.636 (7)", - "tab": "Accuracy", - "XSUM - Denoised inference time (s)": "{\"description\": \"min=0.274, mean=0.294, max=0.41, sum=2.059 (7)\", \"tab\": \"Efficiency\", \"score\": \"0.29416145294688817\"}", - "XSUM - # eval": "{\"description\": \"min=518, mean=518, max=518, sum=3626 (7)\", \"tab\": \"General information\", \"score\": \"518.0\"}", - "XSUM - # train": "{\"description\": \"min=0, mean=4.285, max=5, sum=29.992 (7)\", \"tab\": \"General information\", \"score\": \"4.284611141753999\"}", - "XSUM - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "XSUM - # prompt tokens": "{\"description\": \"min=388.402, mean=1350.13, max=1538.921, sum=9450.911 (7)\", \"tab\": \"General information\", \"score\": \"1350.1301709873137\"}", - "XSUM - # output tokens": "{\"description\": \"min=24.405, mean=27.757, max=46.521, sum=194.297 (7)\", \"tab\": \"General information\", \"score\": \"27.75675675675676\"}", - "XSUM - # trials": "{\"description\": \"min=1, mean=2.714, max=3, sum=19 (7)\", \"tab\": \"General information\", \"score\": \"2.7142857142857144\"}", - "XSUM - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=3.333 (5)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666667\"}", - "XSUM - Stereotypes (gender)": "{\"description\": \"min=0.409, mean=0.449, max=0.488, sum=3.143 (7)\", \"tab\": \"Bias\", \"score\": \"0.44897893078382667\"}", - "XSUM - Representation (race)": "{\"description\": \"min=0.446, mean=0.599, max=0.667, sum=4.196 (7)\", \"tab\": \"Bias\", \"score\": \"0.5994124922696351\"}", - "XSUM - Representation (gender)": "{\"description\": \"min=0.169, mean=0.205, max=0.268, sum=1.435 (7)\", \"tab\": \"Bias\", \"score\": \"0.20496360887910145\"}", - "XSUM - Toxic fraction": "{\"description\": \"min=0, mean=0.001, max=0.002, sum=0.004 (7)\", \"tab\": \"Toxicity\", \"score\": \"0.0005515719801434088\"}", - "XSUM - SummaC": "{\"description\": \"min=-0.237, mean=-0.143, max=0.073, sum=-0.574 (4)\", \"tab\": \"Summarization metrics\", \"score\": \"-0.14346265436541167\"}", - "XSUM - QAFactEval": "{\"description\": \"min=2.914, mean=3.922, max=4.204, sum=27.454 (7)\", \"tab\": \"Summarization metrics\", \"score\": \"3.9220091164391953\"}", - "XSUM - BERTScore (F1)": "{\"description\": \"min=0.091, mean=0.313, max=0.388, sum=1.251 (4)\", \"tab\": \"Summarization metrics\", \"score\": \"0.312644368874429\"}", - "XSUM - Coverage": "{\"description\": \"min=0.795, mean=0.815, max=0.823, sum=5.707 (7)\", \"tab\": \"Summarization metrics\", \"score\": \"0.8152742026902194\"}", - "XSUM - Density": "{\"description\": \"min=2.849, mean=5.57, max=19.82, sum=38.989 (7)\", \"tab\": \"Summarization metrics\", \"score\": \"5.569907111767537\"}", - "XSUM - Compression": "{\"description\": \"min=10.146, mean=17.018, max=18.474, sum=119.123 (7)\", \"tab\": \"Summarization metrics\", \"score\": \"17.01754099745573\"}", - "XSUM - HumanEval-faithfulness": "{\"description\": \"min=0.773, mean=0.924, max=1, sum=2.773 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.9244444444444445\"}", - "XSUM - HumanEval-relevance": "{\"description\": \"min=3.387, mean=3.573, max=3.667, sum=10.72 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"3.573333333333333\"}", - "XSUM - HumanEval-coherence": "{\"description\": \"min=3.163, mean=4.166, max=4.667, sum=12.497 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"4.165555555555556\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "IMDB", - "source_data": { - "dataset_name": "IMDB", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on IMDB", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.889, - "details": { - "description": "min=0.831, mean=0.889, max=0.939, sum=2.668 (3)", - "tab": "Accuracy", - "IMDB - ECE (10-bin)": "{\"description\": \"min=0.231, mean=0.259, max=0.285, sum=0.776 (3)\", \"tab\": \"Calibration\", \"score\": \"0.25871248887630766\"}", - "IMDB - EM (Robustness)": "{\"description\": \"min=0.716, mean=0.803, max=0.892, sum=2.41 (3)\", \"tab\": \"Robustness\", \"score\": \"0.8033333333333333\"}", - "IMDB - EM (Fairness)": "{\"description\": \"min=0.792, mean=0.86, max=0.922, sum=2.581 (3)\", \"tab\": \"Fairness\", \"score\": \"0.8603333333333333\"}", - "IMDB - Denoised inference time (s)": "{\"description\": \"min=0.105, mean=0.11, max=0.115, sum=0.331 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.11035393728298622\"}", - "IMDB - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "IMDB - # train": "{\"description\": \"min=2.916, mean=4.242, max=4.986, sum=12.726 (3)\", \"tab\": \"General information\", \"score\": \"4.242\"}", - "IMDB - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IMDB - # prompt tokens": "{\"description\": \"min=1277.729, mean=1553.363, max=1768.607, sum=4660.089 (3)\", \"tab\": \"General information\", \"score\": \"1553.363\"}", - "IMDB - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=3 (3)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "IMDB - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "IMDB - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CivilComments", - "source_data": { - "dataset_name": "CivilComments", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on CivilComments", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.539, - "details": { - "description": "min=0.012, mean=0.539, max=1, sum=29.083 (54)", - "tab": "Accuracy", - "CivilComments - ECE (10-bin)": "{\"description\": \"min=0.042, mean=0.293, max=0.601, sum=15.826 (54)\", \"tab\": \"Calibration\", \"score\": \"0.29307434802498333\"}", - "CivilComments - EM (Robustness)": "{\"description\": \"min=0.002, mean=0.347, max=1, sum=18.748 (54)\", \"tab\": \"Robustness\", \"score\": \"0.3471901723680723\"}", - "CivilComments - EM (Fairness)": "{\"description\": \"min=0, mean=0.412, max=1, sum=22.222 (54)\", \"tab\": \"Fairness\", \"score\": \"0.41152337126555366\"}", - "CivilComments - Denoised inference time (s)": "{\"description\": \"min=0.09, mean=0.097, max=0.105, sum=5.259 (54)\", \"tab\": \"Efficiency\", \"score\": \"0.09739228545773865\"}", - "CivilComments - # eval": "{\"description\": \"min=74, mean=371.556, max=683, sum=20064 (54)\", \"tab\": \"General information\", \"score\": \"371.55555555555554\"}", - "CivilComments - # train": "{\"description\": \"min=5, mean=5, max=5, sum=270 (54)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CivilComments - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (54)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CivilComments - # prompt tokens": "{\"description\": \"min=356.537, mean=722.635, max=1267.519, sum=39022.317 (54)\", \"tab\": \"General information\", \"score\": \"722.6354931173206\"}", - "CivilComments - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=54 (54)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "CivilComments - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=162 (54)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "CivilComments - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (54)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "RAFT", - "source_data": { - "dataset_name": "RAFT", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on RAFT", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.49, - "details": { - "description": "min=0, mean=0.49, max=0.975, sum=16.175 (33)", - "tab": "Accuracy", - "RAFT - ECE (10-bin)": "{\"description\": \"min=0.054, mean=0.319, max=0.977, sum=10.54 (33)\", \"tab\": \"Calibration\", \"score\": \"0.31939577693629423\"}", - "RAFT - EM (Robustness)": "{\"description\": \"min=0, mean=0.413, max=0.975, sum=13.625 (33)\", \"tab\": \"Robustness\", \"score\": \"0.4128787878787879\"}", - "RAFT - EM (Fairness)": "{\"description\": \"min=0, mean=0.473, max=0.975, sum=15.625 (33)\", \"tab\": \"Fairness\", \"score\": \"0.4734848484848485\"}", - "RAFT - Denoised inference time (s)": "{\"description\": \"min=0.094, mean=0.112, max=0.139, sum=3.696 (33)\", \"tab\": \"Efficiency\", \"score\": \"0.11198840159406566\"}", - "RAFT - # eval": "{\"description\": \"min=40, mean=40, max=40, sum=1320 (33)\", \"tab\": \"General information\", \"score\": \"40.0\"}", - "RAFT - # train": "{\"description\": \"min=0, mean=4.556, max=5, sum=150.35 (33)\", \"tab\": \"General information\", \"score\": \"4.556060606060607\"}", - "RAFT - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (33)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "RAFT - # prompt tokens": "{\"description\": \"min=257.35, mean=812.938, max=1773.675, sum=26826.95 (33)\", \"tab\": \"General information\", \"score\": \"812.937878787879\"}", - "RAFT - # output tokens": "{\"description\": \"min=0.025, mean=2.867, max=6.375, sum=94.6 (33)\", \"tab\": \"General information\", \"score\": \"2.8666666666666667\"}", - "RAFT - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=99 (33)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "RAFT - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (33)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/openai_davinci-175b.json b/data/models/openai_davinci-175b.json deleted file mode 100644 index 9477e4fe4697ffd2bec26abad69c81b9a5cfd9c6..0000000000000000000000000000000000000000 --- a/data/models/openai_davinci-175b.json +++ /dev/null @@ -1,674 +0,0 @@ -{ - "model_info": { - "name": "davinci 175B", - "id": "openai/davinci-175B", - "developer": "OpenAI", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_classic/openai_davinci-175B/1774096308.339228", - "retrieved_timestamp": "1774096308.339228", - "source_metadata": { - "source_name": "helm_classic", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_classic", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_classic", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperform on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.538, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Calibration": "{\"description\": \"\", \"tab\": \"Calibration\", \"score\": \"0.5745594499834401\"}", - "Mean win rate - Robustness": "{\"description\": \"\", \"tab\": \"Robustness\", \"score\": \"0.5094878610451469\"}", - "Mean win rate - Fairness": "{\"description\": \"\", \"tab\": \"Fairness\", \"score\": \"0.5578754949166518\"}", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.557938596491228\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}", - "Mean win rate - Bias": "{\"description\": \"\", \"tab\": \"Bias\", \"score\": \"0.44460142486244675\"}", - "Mean win rate - Toxicity": "{\"description\": \"\", \"tab\": \"Toxicity\", \"score\": \"0.42202673869340535\"}", - "Mean win rate - Summarization metrics": "{\"description\": \"\", \"tab\": \"Summarization metrics\", \"score\": \"0.3600250626566416\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.422, - "details": { - "description": "min=0.26, mean=0.422, max=0.7, sum=6.336 (15)", - "tab": "Accuracy", - "MMLU - ECE (10-bin)": "{\"description\": \"min=0.093, mean=0.132, max=0.18, sum=1.976 (15)\", \"tab\": \"Calibration\", \"score\": \"0.13175836488041992\"}", - "MMLU - EM (Robustness)": "{\"description\": \"min=0.17, mean=0.34, max=0.6, sum=5.102 (15)\", \"tab\": \"Robustness\", \"score\": \"0.3401169590643275\"}", - "MMLU - EM (Fairness)": "{\"description\": \"min=0.24, mean=0.38, max=0.61, sum=5.705 (15)\", \"tab\": \"Fairness\", \"score\": \"0.3803040935672514\"}", - "MMLU - Denoised inference time (s)": "{\"description\": \"min=0.203, mean=0.212, max=0.221, sum=3.181 (15)\", \"tab\": \"Efficiency\", \"score\": \"0.21209971402138156\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=1542 (15)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=75 (15)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (15)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=371.38, mean=472.274, max=624.07, sum=7084.111 (15)\", \"tab\": \"General information\", \"score\": \"472.2740350877193\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=15 (15)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "MMLU - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=45 (15)\", \"tab\": \"General information\", \"score\": \"3.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "BoolQ", - "source_data": { - "dataset_name": "BoolQ", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on BoolQ", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.722, - "details": { - "description": "min=0.679, mean=0.722, max=0.77, sum=2.167 (3)\n⚠ Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.", - "tab": "Accuracy", - "BoolQ - ECE (10-bin)": "{\"description\": \"min=0.047, mean=0.072, max=0.103, sum=0.215 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Calibration\", \"score\": \"0.07164645838795872\"}", - "BoolQ - EM (Robustness)": "{\"description\": \"min=0.592, mean=0.639, max=0.677, sum=1.918 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Robustness\", \"score\": \"0.6393333333333334\"}", - "BoolQ - EM (Fairness)": "{\"description\": \"min=0.635, mean=0.682, max=0.729, sum=2.046 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Fairness\", \"score\": \"0.682\"}", - "BoolQ - Denoised inference time (s)": "{\"description\": \"min=0.204, mean=0.21, max=0.217, sum=0.631 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Efficiency\", \"score\": \"0.21022733463541673\"}", - "BoolQ - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "BoolQ - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "BoolQ - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "BoolQ - # prompt tokens": "{\"description\": \"min=660.073, mean=908.406, max=1242.073, sum=2725.219 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"908.4063333333334\"}", - "BoolQ - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=3 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "BoolQ - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "BoolQ - Stereotypes (race)": "{\"description\": \"(0)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Stereotypes (gender)": "{\"description\": \"(0)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (race)": "{\"description\": \"(0)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (gender)": "{\"description\": \"(0)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.687, - "details": { - "description": "min=0.664, mean=0.687, max=0.706, sum=2.061 (3)", - "tab": "Accuracy", - "NarrativeQA - ECE (10-bin)": "{\"description\": \"min=0.041, mean=0.067, max=0.109, sum=0.202 (3)\", \"tab\": \"Calibration\", \"score\": \"0.06738212205854943\"}", - "NarrativeQA - F1 (Robustness)": "{\"description\": \"min=0.476, mean=0.498, max=0.52, sum=1.493 (3)\", \"tab\": \"Robustness\", \"score\": \"0.4976057829109271\"}", - "NarrativeQA - F1 (Fairness)": "{\"description\": \"min=0.556, mean=0.597, max=0.634, sum=1.791 (3)\", \"tab\": \"Fairness\", \"score\": \"0.5970096000459133\"}", - "NarrativeQA - Denoised inference time (s)": "{\"description\": \"min=0.36, mean=0.369, max=0.384, sum=1.108 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.3694498019366194\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=1065 (3)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=1.051, mean=1.647, max=2.085, sum=4.941 (3)\", \"tab\": \"General information\", \"score\": \"1.6469483568075116\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=1601.955, mean=1652.377, max=1705.003, sum=4957.132 (3)\", \"tab\": \"General information\", \"score\": \"1652.3774647887324\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=5.338, mean=5.709, max=6.197, sum=17.127 (3)\", \"tab\": \"General information\", \"score\": \"5.708920187793427\"}", - "NarrativeQA - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NarrativeQA - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NarrativeQA - Stereotypes (gender)": "{\"description\": \"min=0.4, mean=0.443, max=0.5, sum=1.329 (3)\", \"tab\": \"Bias\", \"score\": \"0.44285714285714284\"}", - "NarrativeQA - Representation (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=1.333 (2)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666667\"}", - "NarrativeQA - Representation (gender)": "{\"description\": \"min=0.199, mean=0.208, max=0.221, sum=0.623 (3)\", \"tab\": \"Bias\", \"score\": \"0.2075773756101625\"}", - "NarrativeQA - Toxic fraction": "{\"description\": \"min=0.008, mean=0.012, max=0.014, sum=0.037 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.012206572769953052\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (open-book)", - "source_data": { - "dataset_name": "NaturalQuestions (open-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (open-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.625, - "details": { - "description": "min=0.599, mean=0.625, max=0.65, sum=1.874 (3)\n⚠ Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.", - "tab": "Accuracy", - "NaturalQuestions (closed-book) - ECE (10-bin)": "{\"description\": \"min=0.054, mean=0.061, max=0.07, sum=0.182 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Calibration\", \"score\": \"0.06060614220397647\"}", - "NaturalQuestions (open-book) - ECE (10-bin)": "{\"description\": \"min=0.06, mean=0.079, max=0.1, sum=0.236 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Calibration\", \"score\": \"0.07854855230782792\"}", - "NaturalQuestions (closed-book) - F1 (Robustness)": "{\"description\": \"min=0.251, mean=0.256, max=0.264, sum=0.769 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Robustness\", \"score\": \"0.2562420226045557\"}", - "NaturalQuestions (open-book) - F1 (Robustness)": "{\"description\": \"min=0.48, mean=0.521, max=0.561, sum=1.563 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Robustness\", \"score\": \"0.5211614334906893\"}", - "NaturalQuestions (closed-book) - F1 (Fairness)": "{\"description\": \"min=0.271, mean=0.276, max=0.282, sum=0.828 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Fairness\", \"score\": \"0.2760483569290458\"}", - "NaturalQuestions (open-book) - F1 (Fairness)": "{\"description\": \"min=0.537, mean=0.567, max=0.594, sum=1.702 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Fairness\", \"score\": \"0.5674897299434086\"}", - "NaturalQuestions (closed-book) - Denoised inference time (s)": "{\"description\": \"min=0.304, mean=0.327, max=0.357, sum=0.981 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Efficiency\", \"score\": \"0.32700476562499997\"}", - "NaturalQuestions (open-book) - Denoised inference time (s)": "{\"description\": \"min=0.378, mean=0.462, max=0.583, sum=1.386 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Efficiency\", \"score\": \"0.462036467447917\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=110.254, mean=112.254, max=116.254, sum=336.762 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"112.254\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=4.601, mean=5.361, max=6.345, sum=16.082 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"5.360666666666667\"}", - "NaturalQuestions (closed-book) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.647, mean=4.691, max=4.724, sum=14.074 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"4.691333333333334\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.036, mean=0.036, max=0.036, sum=0.108 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.036\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1231.212, mean=1419.574, max=1523.257, sum=4258.721 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"1419.5736666666664\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=6.369, mean=8.992, max=12.931, sum=26.977 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"8.992333333333333\"}", - "NaturalQuestions (open-book) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NaturalQuestions (closed-book) - Stereotypes (race)": "{\"description\": \"(0)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - Stereotypes (gender)": "{\"description\": \"min=0.342, mean=0.447, max=0.5, sum=1.342 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.4472502805836139\"}", - "NaturalQuestions (closed-book) - Representation (race)": "{\"description\": \"min=0.286, mean=0.382, max=0.439, sum=1.147 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.382401229992038\"}", - "NaturalQuestions (closed-book) - Representation (gender)": "{\"description\": \"min=0.032, mean=0.247, max=0.4, sum=0.742 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.24726062467997953\"}", - "NaturalQuestions (open-book) - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=1.333 (2)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "NaturalQuestions (open-book) - Stereotypes (gender)": "{\"description\": \"min=0.293, mean=0.365, max=0.412, sum=1.096 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.3654871847728991\"}", - "NaturalQuestions (open-book) - Representation (race)": "{\"description\": \"min=0.422, mean=0.435, max=0.447, sum=1.304 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.4346811201445348\"}", - "NaturalQuestions (open-book) - Representation (gender)": "{\"description\": \"min=0.222, mean=0.244, max=0.271, sum=0.733 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.24420285420364105\"}", - "NaturalQuestions (closed-book) - Toxic fraction": "{\"description\": \"min=0, mean=0.0, max=0.001, sum=0.001 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Toxicity\", \"score\": \"0.0003333333333333333\"}", - "NaturalQuestions (open-book) - Toxic fraction": "{\"description\": \"min=0, mean=0.001, max=0.002, sum=0.002 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Toxicity\", \"score\": \"0.0006666666666666666\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "QuAC", - "source_data": { - "dataset_name": "QuAC", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on QuAC", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.36, - "details": { - "description": "min=0.354, mean=0.36, max=0.367, sum=1.081 (3)\n⚠ Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.", - "tab": "Accuracy", - "QuAC - ECE (10-bin)": "{\"description\": \"min=0.066, mean=0.068, max=0.071, sum=0.204 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Calibration\", \"score\": \"0.06797808745527684\"}", - "QuAC - F1 (Robustness)": "{\"description\": \"min=0.197, mean=0.208, max=0.217, sum=0.623 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Robustness\", \"score\": \"0.20766668147064418\"}", - "QuAC - F1 (Fairness)": "{\"description\": \"min=0.264, mean=0.279, max=0.288, sum=0.836 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Fairness\", \"score\": \"0.27860575089348755\"}", - "QuAC - Denoised inference time (s)": "{\"description\": \"min=1.01, mean=1.085, max=1.233, sum=3.256 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Efficiency\", \"score\": \"1.085224210937499\"}", - "QuAC - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "QuAC - # train": "{\"description\": \"min=0.845, mean=0.944, max=1.086, sum=2.833 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.9443333333333334\"}", - "QuAC - truncated": "{\"description\": \"min=0.016, mean=0.016, max=0.016, sum=0.048 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.016\"}", - "QuAC - # prompt tokens": "{\"description\": \"min=1625.523, mean=1644.831, max=1670.605, sum=4934.492 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"1644.8306666666667\"}", - "QuAC - # output tokens": "{\"description\": \"min=27.082, mean=29.572, max=34.534, sum=88.717 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"29.572333333333333\"}", - "QuAC - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "QuAC - Stereotypes (race)": "{\"description\": \"min=0.636, mean=0.65, max=0.667, sum=1.949 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.6495628554452085\"}", - "QuAC - Stereotypes (gender)": "{\"description\": \"min=0.435, mean=0.445, max=0.455, sum=1.335 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.4451588893133011\"}", - "QuAC - Representation (race)": "{\"description\": \"min=0.354, mean=0.367, max=0.375, sum=1.1 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.366690749431994\"}", - "QuAC - Representation (gender)": "{\"description\": \"min=0.244, mean=0.251, max=0.256, sum=0.754 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.25124249915688174\"}", - "QuAC - Toxic fraction": "{\"description\": \"min=0, mean=0.0, max=0.001, sum=0.001 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Toxicity\", \"score\": \"0.0003333333333333333\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "HellaSwag", - "source_data": { - "dataset_name": "HellaSwag", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on HellaSwag", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.775, - "details": { - "description": "min=0.775, mean=0.775, max=0.775, sum=0.775 (1)\n⚠ Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.", - "tab": "Accuracy", - "HellaSwag - ECE (10-bin)": "{\"description\": \"min=0.31, mean=0.31, max=0.31, sum=0.31 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Calibration\", \"score\": \"0.30968673998386337\"}", - "HellaSwag - EM (Robustness)": "{\"description\": \"min=0.738, mean=0.738, max=0.738, sum=0.738 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Robustness\", \"score\": \"0.738\"}", - "HellaSwag - EM (Fairness)": "{\"description\": \"min=0.641, mean=0.641, max=0.641, sum=0.641 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Fairness\", \"score\": \"0.641\"}", - "HellaSwag - Denoised inference time (s)": "{\"description\": \"min=0.193, mean=0.193, max=0.193, sum=0.193 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Efficiency\", \"score\": \"0.19329937499999997\"}", - "HellaSwag - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "HellaSwag - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "HellaSwag - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "HellaSwag - # prompt tokens": "{\"description\": \"min=87.888, mean=87.888, max=87.888, sum=87.888 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"87.888\"}", - "HellaSwag - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "HellaSwag - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.586, - "details": { - "description": "min=0.586, mean=0.586, max=0.586, sum=0.586 (1)\n⚠ Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.", - "tab": "Accuracy", - "OpenbookQA - ECE (10-bin)": "{\"description\": \"min=0.204, mean=0.204, max=0.204, sum=0.204 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Calibration\", \"score\": \"0.20443749582919374\"}", - "OpenbookQA - EM (Robustness)": "{\"description\": \"min=0.474, mean=0.474, max=0.474, sum=0.474 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Robustness\", \"score\": \"0.474\"}", - "OpenbookQA - EM (Fairness)": "{\"description\": \"min=0.502, mean=0.502, max=0.502, sum=0.502 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Fairness\", \"score\": \"0.502\"}", - "OpenbookQA - Denoised inference time (s)": "{\"description\": \"min=0.184, mean=0.184, max=0.184, sum=0.184 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Efficiency\", \"score\": \"0.18361757812499943\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=5.27, mean=5.27, max=5.27, sum=5.27 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"5.27\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "TruthfulQA", - "source_data": { - "dataset_name": "TruthfulQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on TruthfulQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.194, - "details": { - "description": "min=0.182, mean=0.194, max=0.213, sum=0.581 (3)", - "tab": "Accuracy", - "TruthfulQA - ECE (10-bin)": "{\"description\": \"min=0.186, mean=0.211, max=0.224, sum=0.632 (3)\", \"tab\": \"Calibration\", \"score\": \"0.21061421693460983\"}", - "TruthfulQA - EM (Robustness)": "{\"description\": \"min=0.131, mean=0.145, max=0.162, sum=0.434 (3)\", \"tab\": \"Robustness\", \"score\": \"0.14475025484199797\"}", - "TruthfulQA - EM (Fairness)": "{\"description\": \"min=0.136, mean=0.155, max=0.185, sum=0.466 (3)\", \"tab\": \"Fairness\", \"score\": \"0.15545361875637104\"}", - "TruthfulQA - Denoised inference time (s)": "{\"description\": \"min=0.208, mean=0.215, max=0.219, sum=0.645 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.21492536613627675\"}", - "TruthfulQA - # eval": "{\"description\": \"min=654, mean=654, max=654, sum=1962 (3)\", \"tab\": \"General information\", \"score\": \"654.0\"}", - "TruthfulQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "TruthfulQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "TruthfulQA - # prompt tokens": "{\"description\": \"min=501.121, mean=511.121, max=529.121, sum=1533.362 (3)\", \"tab\": \"General information\", \"score\": \"511.12079510703364\"}", - "TruthfulQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=3 (3)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "TruthfulQA - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MS MARCO (TREC)", - "source_data": { - "dataset_name": "MS MARCO (TREC)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "NDCG@10 on MS MARCO (TREC)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.378, - "details": { - "description": "min=0.343, mean=0.378, max=0.397, sum=1.135 (3)", - "tab": "Accuracy", - "MS MARCO (regular) - RR@10 (Robustness)": "{\"description\": \"min=0.15, mean=0.154, max=0.157, sum=0.462 (3)\", \"tab\": \"Robustness\", \"score\": \"0.15391111111111108\"}", - "MS MARCO (TREC) - NDCG@10 (Robustness)": "{\"description\": \"min=0.31, mean=0.332, max=0.352, sum=0.996 (3)\", \"tab\": \"Robustness\", \"score\": \"0.3320850067305285\"}", - "MS MARCO (regular) - RR@10 (Fairness)": "{\"description\": \"min=0.179, mean=0.185, max=0.192, sum=0.554 (3)\", \"tab\": \"Fairness\", \"score\": \"0.18462896825396802\"}", - "MS MARCO (TREC) - NDCG@10 (Fairness)": "{\"description\": \"min=0.324, mean=0.357, max=0.375, sum=1.072 (3)\", \"tab\": \"Fairness\", \"score\": \"0.35718542292055805\"}", - "MS MARCO (regular) - Denoised inference time (s)": "{\"description\": \"min=0.202, mean=0.211, max=0.218, sum=0.632 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.21074697460937475\"}", - "MS MARCO (TREC) - Denoised inference time (s)": "{\"description\": \"min=0.201, mean=0.214, max=0.221, sum=0.641 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.2137389625726744\"}", - "MS MARCO (regular) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "MS MARCO (regular) - # train": "{\"description\": \"min=2, mean=2, max=2, sum=6 (3)\", \"tab\": \"General information\", \"score\": \"2.0\"}", - "MS MARCO (regular) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MS MARCO (regular) - # prompt tokens": "{\"description\": \"min=495.232, mean=532.565, max=577.232, sum=1597.696 (3)\", \"tab\": \"General information\", \"score\": \"532.5653333333333\"}", - "MS MARCO (regular) - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=3 (3)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "MS MARCO (regular) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "MS MARCO (TREC) - # eval": "{\"description\": \"min=43, mean=43, max=43, sum=129 (3)\", \"tab\": \"General information\", \"score\": \"43.0\"}", - "MS MARCO (TREC) - # train": "{\"description\": \"min=2, mean=2, max=2, sum=6 (3)\", \"tab\": \"General information\", \"score\": \"2.0\"}", - "MS MARCO (TREC) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MS MARCO (TREC) - # prompt tokens": "{\"description\": \"min=478.488, mean=515.822, max=560.488, sum=1547.465 (3)\", \"tab\": \"General information\", \"score\": \"515.8217054263565\"}", - "MS MARCO (TREC) - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=3 (3)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "MS MARCO (TREC) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "MS MARCO (regular) - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "MS MARCO (TREC) - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CNN/DailyMail", - "source_data": { - "dataset_name": "CNN/DailyMail", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on CNN/DailyMail", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.127, - "details": { - "description": "min=0.087, mean=0.127, max=0.14, sum=0.889 (7)", - "tab": "Accuracy", - "CNN/DailyMail - Denoised inference time (s)": "{\"description\": \"min=1.919, mean=2.256, max=3.967, sum=15.789 (7)\", \"tab\": \"Efficiency\", \"score\": \"2.255577085568669\"}", - "CNN/DailyMail - # eval": "{\"description\": \"min=466, mean=466, max=466, sum=3262 (7)\", \"tab\": \"General information\", \"score\": \"466.0\"}", - "CNN/DailyMail - # train": "{\"description\": \"min=0, mean=4.286, max=5, sum=30 (7)\", \"tab\": \"General information\", \"score\": \"4.285714285714286\"}", - "CNN/DailyMail - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CNN/DailyMail - # prompt tokens": "{\"description\": \"min=583.586, mean=1411.872, max=1567.586, sum=9883.101 (7)\", \"tab\": \"General information\", \"score\": \"1411.8715511955854\"}", - "CNN/DailyMail - # output tokens": "{\"description\": \"min=57.459, mean=68.76, max=126.343, sum=481.322 (7)\", \"tab\": \"General information\", \"score\": \"68.76026977314531\"}", - "CNN/DailyMail - # trials": "{\"description\": \"min=1, mean=2.714, max=3, sum=19 (7)\", \"tab\": \"General information\", \"score\": \"2.7142857142857144\"}", - "CNN/DailyMail - Stereotypes (race)": "{\"description\": \"min=0.579, mean=0.619, max=0.641, sum=4.33 (7)\", \"tab\": \"Bias\", \"score\": \"0.618631744195654\"}", - "CNN/DailyMail - Stereotypes (gender)": "{\"description\": \"min=0.373, mean=0.401, max=0.418, sum=2.804 (7)\", \"tab\": \"Bias\", \"score\": \"0.4005751850408633\"}", - "CNN/DailyMail - Representation (race)": "{\"description\": \"min=0.235, mean=0.301, max=0.378, sum=2.105 (7)\", \"tab\": \"Bias\", \"score\": \"0.3007554818500092\"}", - "CNN/DailyMail - Representation (gender)": "{\"description\": \"min=0.111, mean=0.125, max=0.16, sum=0.876 (7)\", \"tab\": \"Bias\", \"score\": \"0.12511140031093898\"}", - "CNN/DailyMail - Toxic fraction": "{\"description\": \"min=0, mean=0.002, max=0.002, sum=0.011 (7)\", \"tab\": \"Toxicity\", \"score\": \"0.0015328019619865114\"}", - "CNN/DailyMail - SummaC": "{\"description\": \"min=-0.08, mean=0.321, max=0.532, sum=1.284 (4)\", \"tab\": \"Summarization metrics\", \"score\": \"0.321074205166444\"}", - "CNN/DailyMail - QAFactEval": "{\"description\": \"min=2.929, mean=4.062, max=4.888, sum=28.435 (7)\", \"tab\": \"Summarization metrics\", \"score\": \"4.062076530805548\"}", - "CNN/DailyMail - BERTScore (F1)": "{\"description\": \"min=0.023, mean=0.182, max=0.25, sum=0.729 (4)\", \"tab\": \"Summarization metrics\", \"score\": \"0.18232803102041212\"}", - "CNN/DailyMail - Coverage": "{\"description\": \"min=0.72, mean=0.873, max=0.944, sum=6.111 (7)\", \"tab\": \"Summarization metrics\", \"score\": \"0.87307141297806\"}", - "CNN/DailyMail - Density": "{\"description\": \"min=15.056, mean=17.914, max=20.184, sum=125.396 (7)\", \"tab\": \"Summarization metrics\", \"score\": \"17.913710646412884\"}", - "CNN/DailyMail - Compression": "{\"description\": \"min=4.761, mean=9.843, max=11.282, sum=68.899 (7)\", \"tab\": \"Summarization metrics\", \"score\": \"9.842721706219109\"}", - "CNN/DailyMail - HumanEval-faithfulness": "{\"description\": \"min=0.763, mean=0.953, max=1, sum=4.763 (5)\", \"tab\": \"Summarization metrics\", \"score\": \"0.9526666666666668\"}", - "CNN/DailyMail - HumanEval-relevance": "{\"description\": \"min=3.503, mean=4.501, max=5, sum=22.503 (5)\", \"tab\": \"Summarization metrics\", \"score\": \"4.500666666666667\"}", - "CNN/DailyMail - HumanEval-coherence": "{\"description\": \"min=2.647, mean=3.863, max=4.667, sum=19.313 (5)\", \"tab\": \"Summarization metrics\", \"score\": \"3.862666666666667\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "XSUM", - "source_data": { - "dataset_name": "XSUM", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on XSUM", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.126, - "details": { - "description": "min=0.045, mean=0.126, max=0.144, sum=0.884 (7)", - "tab": "Accuracy", - "XSUM - Denoised inference time (s)": "{\"description\": \"min=0.958, mean=1.148, max=2.074, sum=8.038 (7)\", \"tab\": \"Efficiency\", \"score\": \"1.1482822034007862\"}", - "XSUM - # eval": "{\"description\": \"min=518, mean=518, max=518, sum=3626 (7)\", \"tab\": \"General information\", \"score\": \"518.0\"}", - "XSUM - # train": "{\"description\": \"min=0, mean=4.285, max=5, sum=29.992 (7)\", \"tab\": \"General information\", \"score\": \"4.284611141753999\"}", - "XSUM - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "XSUM - # prompt tokens": "{\"description\": \"min=388.402, mean=1350.13, max=1538.921, sum=9450.911 (7)\", \"tab\": \"General information\", \"score\": \"1350.1301709873137\"}", - "XSUM - # output tokens": "{\"description\": \"min=25.444, mean=31.877, max=63.193, sum=223.139 (7)\", \"tab\": \"General information\", \"score\": \"31.87699944842802\"}", - "XSUM - # trials": "{\"description\": \"min=1, mean=2.714, max=3, sum=19 (7)\", \"tab\": \"General information\", \"score\": \"2.7142857142857144\"}", - "XSUM - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=4.667 (7)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666669\"}", - "XSUM - Stereotypes (gender)": "{\"description\": \"min=0.427, mean=0.444, max=0.469, sum=3.111 (7)\", \"tab\": \"Bias\", \"score\": \"0.44436594684493835\"}", - "XSUM - Representation (race)": "{\"description\": \"min=0.473, mean=0.564, max=0.667, sum=3.948 (7)\", \"tab\": \"Bias\", \"score\": \"0.5639808220453382\"}", - "XSUM - Representation (gender)": "{\"description\": \"min=0.189, mean=0.217, max=0.251, sum=1.521 (7)\", \"tab\": \"Bias\", \"score\": \"0.21723674492179154\"}", - "XSUM - Toxic fraction": "{\"description\": \"min=0, mean=0.003, max=0.015, sum=0.019 (7)\", \"tab\": \"Toxicity\", \"score\": \"0.0027578599007170436\"}", - "XSUM - SummaC": "{\"description\": \"min=-0.317, mean=-0.267, max=-0.218, sum=-1.068 (4)\", \"tab\": \"Summarization metrics\", \"score\": \"-0.2669066513504126\"}", - "XSUM - QAFactEval": "{\"description\": \"min=1.878, mean=2.338, max=2.635, sum=16.363 (7)\", \"tab\": \"Summarization metrics\", \"score\": \"2.337582859954366\"}", - "XSUM - BERTScore (F1)": "{\"description\": \"min=0.063, mean=0.318, max=0.423, sum=1.272 (4)\", \"tab\": \"Summarization metrics\", \"score\": \"0.3179425085241978\"}", - "XSUM - Coverage": "{\"description\": \"min=0.698, mean=0.751, max=0.774, sum=5.255 (7)\", \"tab\": \"Summarization metrics\", \"score\": \"0.7506856271565006\"}", - "XSUM - Density": "{\"description\": \"min=2.081, mean=3.351, max=10.076, sum=23.459 (7)\", \"tab\": \"Summarization metrics\", \"score\": \"3.3513024292310853\"}", - "XSUM - Compression": "{\"description\": \"min=7.668, mean=14.08, max=15.293, sum=98.56 (7)\", \"tab\": \"Summarization metrics\", \"score\": \"14.079969364330754\"}", - "XSUM - HumanEval-faithfulness": "{\"description\": \"min=0.5, mean=0.829, max=1, sum=5.803 (7)\", \"tab\": \"Summarization metrics\", \"score\": \"0.8290476190476191\"}", - "XSUM - HumanEval-relevance": "{\"description\": \"min=2.833, mean=4.075, max=5, sum=28.523 (7)\", \"tab\": \"Summarization metrics\", \"score\": \"4.074761904761905\"}", - "XSUM - HumanEval-coherence": "{\"description\": \"min=2.167, mean=3.398, max=5, sum=23.783 (7)\", \"tab\": \"Summarization metrics\", \"score\": \"3.397619047619048\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "IMDB", - "source_data": { - "dataset_name": "IMDB", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on IMDB", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.933, - "details": { - "description": "min=0.925, mean=0.933, max=0.942, sum=2.8 (3)", - "tab": "Accuracy", - "IMDB - ECE (10-bin)": "{\"description\": \"min=0.104, mean=0.126, max=0.166, sum=0.378 (3)\", \"tab\": \"Calibration\", \"score\": \"0.12610548329130192\"}", - "IMDB - EM (Robustness)": "{\"description\": \"min=0.855, mean=0.873, max=0.89, sum=2.62 (3)\", \"tab\": \"Robustness\", \"score\": \"0.8733333333333334\"}", - "IMDB - EM (Fairness)": "{\"description\": \"min=0.917, mean=0.921, max=0.923, sum=2.762 (3)\", \"tab\": \"Fairness\", \"score\": \"0.9206666666666669\"}", - "IMDB - Denoised inference time (s)": "{\"description\": \"min=0.218, mean=0.225, max=0.231, sum=0.676 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.22547806217447905\"}", - "IMDB - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "IMDB - # train": "{\"description\": \"min=2.916, mean=4.242, max=4.986, sum=12.726 (3)\", \"tab\": \"General information\", \"score\": \"4.242\"}", - "IMDB - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IMDB - # prompt tokens": "{\"description\": \"min=1277.729, mean=1553.363, max=1768.607, sum=4660.089 (3)\", \"tab\": \"General information\", \"score\": \"1553.363\"}", - "IMDB - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=3 (3)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "IMDB - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "IMDB - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CivilComments", - "source_data": { - "dataset_name": "CivilComments", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on CivilComments", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.532, - "details": { - "description": "min=0.006, mean=0.532, max=1, sum=28.723 (54)", - "tab": "Accuracy", - "CivilComments - ECE (10-bin)": "{\"description\": \"min=0.083, mean=0.396, max=0.664, sum=21.389 (54)\", \"tab\": \"Calibration\", \"score\": \"0.3960964912577608\"}", - "CivilComments - EM (Robustness)": "{\"description\": \"min=0, mean=0.461, max=1, sum=24.899 (54)\", \"tab\": \"Robustness\", \"score\": \"0.461098863197608\"}", - "CivilComments - EM (Fairness)": "{\"description\": \"min=0.003, mean=0.478, max=1, sum=25.83 (54)\", \"tab\": \"Fairness\", \"score\": \"0.4783299102254815\"}", - "CivilComments - Denoised inference time (s)": "{\"description\": \"min=0.203, mean=0.21, max=0.218, sum=11.326 (54)\", \"tab\": \"Efficiency\", \"score\": \"0.20974755918568705\"}", - "CivilComments - # eval": "{\"description\": \"min=74, mean=371.556, max=683, sum=20064 (54)\", \"tab\": \"General information\", \"score\": \"371.55555555555554\"}", - "CivilComments - # train": "{\"description\": \"min=5, mean=5, max=5, sum=270 (54)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CivilComments - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (54)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CivilComments - # prompt tokens": "{\"description\": \"min=356.537, mean=722.635, max=1267.519, sum=39022.317 (54)\", \"tab\": \"General information\", \"score\": \"722.6354931173206\"}", - "CivilComments - # output tokens": "{\"description\": \"min=0.998, mean=1.0, max=1.001, sum=54.0 (54)\", \"tab\": \"General information\", \"score\": \"0.9999957802714455\"}", - "CivilComments - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=162 (54)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "CivilComments - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (54)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "RAFT", - "source_data": { - "dataset_name": "RAFT", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on RAFT", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.642, - "details": { - "description": "min=0.1, mean=0.642, max=0.975, sum=21.2 (33)", - "tab": "Accuracy", - "RAFT - ECE (10-bin)": "{\"description\": \"min=0.073, mean=0.222, max=0.806, sum=7.328 (33)\", \"tab\": \"Calibration\", \"score\": \"0.22206849861217967\"}", - "RAFT - EM (Robustness)": "{\"description\": \"min=0.025, mean=0.505, max=0.975, sum=16.65 (33)\", \"tab\": \"Robustness\", \"score\": \"0.5045454545454545\"}", - "RAFT - EM (Fairness)": "{\"description\": \"min=0.05, mean=0.605, max=0.975, sum=19.95 (33)\", \"tab\": \"Fairness\", \"score\": \"0.6045454545454545\"}", - "RAFT - Denoised inference time (s)": "{\"description\": \"min=0.213, mean=0.279, max=0.378, sum=9.22 (33)\", \"tab\": \"Efficiency\", \"score\": \"0.2793995279947917\"}", - "RAFT - # eval": "{\"description\": \"min=40, mean=40, max=40, sum=1320 (33)\", \"tab\": \"General information\", \"score\": \"40.0\"}", - "RAFT - # train": "{\"description\": \"min=0, mean=4.556, max=5, sum=150.35 (33)\", \"tab\": \"General information\", \"score\": \"4.556060606060607\"}", - "RAFT - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (33)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "RAFT - # prompt tokens": "{\"description\": \"min=257.35, mean=812.938, max=1773.675, sum=26826.95 (33)\", \"tab\": \"General information\", \"score\": \"812.937878787879\"}", - "RAFT - # output tokens": "{\"description\": \"min=0.3, mean=3.056, max=6.575, sum=100.85 (33)\", \"tab\": \"General information\", \"score\": \"3.056060606060606\"}", - "RAFT - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=99 (33)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "RAFT - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (33)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/openai_gpt-3.5-turbo-0125.json b/data/models/openai_gpt-3.5-turbo-0125.json deleted file mode 100644 index 91e4d1e85527ab91cbf0002bc7d2d0e72a85310d..0000000000000000000000000000000000000000 --- a/data/models/openai_gpt-3.5-turbo-0125.json +++ /dev/null @@ -1,1663 +0,0 @@ -{ - "model_info": { - "name": "GPT-3.5 Turbo 0125", - "id": "openai/gpt-3.5-turbo-0125", - "developer": "OpenAI", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_mmlu/openai_gpt-3.5-turbo-0125/1774096312.00548", - "retrieved_timestamp": "1774096312.00548", - "source_metadata": { - "source_name": "helm_mmlu", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_mmlu", - "evaluation_results": [ - { - "evaluation_name": "MMLU All Subjects", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU All Subjects", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.673, - "details": { - "description": "min=0.307, mean=0.673, max=0.922, sum=76.686 (114)", - "tab": "Accuracy", - "MMLU All Subjects - Observed inference time (s)": "{\"description\": \"min=0.396, mean=0.476, max=1.242, sum=54.283 (114)\", \"tab\": \"Efficiency\", \"score\": \"0.4761648045252673\"}", - "MMLU All Subjects - # eval": "{\"description\": \"min=100, mean=246.351, max=1534, sum=28084 (114)\", \"tab\": \"General information\", \"score\": \"246.35087719298247\"}", - "MMLU All Subjects - # train": "{\"description\": \"min=5, mean=5, max=5, sum=570 (114)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU All Subjects - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU All Subjects - # prompt tokens": "{\"description\": \"min=275.561, mean=614.852, max=2798.073, sum=70093.086 (114)\", \"tab\": \"General information\", \"score\": \"614.851634217556\"}", - "MMLU All Subjects - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=114 (114)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_name": "Abstract Algebra", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Abstract Algebra", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.31, - "details": { - "description": "min=0.31, mean=0.31, max=0.31, sum=0.62 (2)", - "tab": "Accuracy", - "Abstract Algebra - Observed inference time (s)": "{\"description\": \"min=0.47, mean=0.47, max=0.47, sum=0.94 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4701289844512939\"}", - "Abstract Algebra - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Abstract Algebra - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Abstract Algebra - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Abstract Algebra - # prompt tokens": "{\"description\": \"min=373.44, mean=373.44, max=373.44, sum=746.88 (2)\", \"tab\": \"General information\", \"score\": \"373.44\"}", - "Abstract Algebra - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"abstract_algebra\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_abstract_algebra\"" - } - } - }, - { - "evaluation_name": "Anatomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Anatomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.696, - "details": { - "description": "min=0.696, mean=0.696, max=0.696, sum=1.393 (2)", - "tab": "Accuracy", - "Anatomy - Observed inference time (s)": "{\"description\": \"min=0.422, mean=0.422, max=0.422, sum=0.844 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.42177006050392435\"}", - "Anatomy - # eval": "{\"description\": \"min=135, mean=135, max=135, sum=270 (2)\", \"tab\": \"General information\", \"score\": \"135.0\"}", - "Anatomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Anatomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Anatomy - # prompt tokens": "{\"description\": \"min=353.978, mean=353.978, max=353.978, sum=707.956 (2)\", \"tab\": \"General information\", \"score\": \"353.97777777777776\"}", - "Anatomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"anatomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_anatomy\"" - } - } - }, - { - "evaluation_name": "College Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on College Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.471, - "details": { - "description": "min=0.471, mean=0.471, max=0.471, sum=0.941 (2)", - "tab": "Accuracy", - "College Chemistry - Observed inference time (s)": "{\"description\": \"min=0.428, mean=0.428, max=0.428, sum=0.856 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.42796642541885377\"}", - "College Biology - Observed inference time (s)": "{\"description\": \"min=0.474, mean=0.474, max=0.474, sum=0.949 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.47431788014041054\"}", - "College Computer Science - Observed inference time (s)": "{\"description\": \"min=0.52, mean=0.52, max=0.52, sum=1.04 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5200183248519897\"}", - "College Mathematics - Observed inference time (s)": "{\"description\": \"min=0.448, mean=0.448, max=0.448, sum=0.897 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4484861779212952\"}", - "College Medicine - Observed inference time (s)": "{\"description\": \"min=0.423, mean=0.423, max=0.423, sum=0.846 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4230213785447137\"}", - "College Physics - Observed inference time (s)": "{\"description\": \"min=0.415, mean=0.415, max=0.415, sum=0.83 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4148852918662277\"}", - "College Chemistry - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Chemistry - # prompt tokens": "{\"description\": \"min=549.4, mean=549.4, max=549.4, sum=1098.8 (2)\", \"tab\": \"General information\", \"score\": \"549.4\"}", - "College Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Biology - # eval": "{\"description\": \"min=144, mean=144, max=144, sum=288 (2)\", \"tab\": \"General information\", \"score\": \"144.0\"}", - "College Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # prompt tokens": "{\"description\": \"min=473.917, mean=473.917, max=473.917, sum=947.833 (2)\", \"tab\": \"General information\", \"score\": \"473.9166666666667\"}", - "College Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # prompt tokens": "{\"description\": \"min=828.39, mean=828.39, max=828.39, sum=1656.78 (2)\", \"tab\": \"General information\", \"score\": \"828.39\"}", - "College Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Mathematics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # prompt tokens": "{\"description\": \"min=594.52, mean=594.52, max=594.52, sum=1189.04 (2)\", \"tab\": \"General information\", \"score\": \"594.52\"}", - "College Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Medicine - # eval": "{\"description\": \"min=173, mean=173, max=173, sum=346 (2)\", \"tab\": \"General information\", \"score\": \"173.0\"}", - "College Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # prompt tokens": "{\"description\": \"min=502.728, mean=502.728, max=502.728, sum=1005.457 (2)\", \"tab\": \"General information\", \"score\": \"502.728323699422\"}", - "College Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Physics - # eval": "{\"description\": \"min=102, mean=102, max=102, sum=204 (2)\", \"tab\": \"General information\", \"score\": \"102.0\"}", - "College Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # prompt tokens": "{\"description\": \"min=503.608, mean=503.608, max=503.608, sum=1007.216 (2)\", \"tab\": \"General information\", \"score\": \"503.6078431372549\"}", - "College Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"college_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_college_physics\"" - } - } - }, - { - "evaluation_name": "Computer Security", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Computer Security", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.78, - "details": { - "description": "min=0.78, mean=0.78, max=0.78, sum=1.56 (2)", - "tab": "Accuracy", - "Computer Security - Observed inference time (s)": "{\"description\": \"min=0.444, mean=0.444, max=0.444, sum=0.887 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.44357073068618774\"}", - "Computer Security - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Computer Security - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Computer Security - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Computer Security - # prompt tokens": "{\"description\": \"min=378.54, mean=378.54, max=378.54, sum=757.08 (2)\", \"tab\": \"General information\", \"score\": \"378.54\"}", - "Computer Security - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"computer_security\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_computer_security\"" - } - } - }, - { - "evaluation_name": "Econometrics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Econometrics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.474, - "details": { - "description": "min=0.474, mean=0.474, max=0.474, sum=0.947 (2)", - "tab": "Accuracy", - "Econometrics - Observed inference time (s)": "{\"description\": \"min=0.418, mean=0.418, max=0.418, sum=0.836 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4179882564042744\"}", - "Econometrics - # eval": "{\"description\": \"min=114, mean=114, max=114, sum=228 (2)\", \"tab\": \"General information\", \"score\": \"114.0\"}", - "Econometrics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Econometrics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Econometrics - # prompt tokens": "{\"description\": \"min=614.43, mean=614.43, max=614.43, sum=1228.86 (2)\", \"tab\": \"General information\", \"score\": \"614.4298245614035\"}", - "Econometrics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"econometrics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_econometrics\"" - } - } - }, - { - "evaluation_name": "Global Facts", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Global Facts", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.39, - "details": { - "description": "min=0.39, mean=0.39, max=0.39, sum=0.78 (2)", - "tab": "Accuracy", - "Global Facts - Observed inference time (s)": "{\"description\": \"min=0.432, mean=0.432, max=0.432, sum=0.863 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4315228652954102\"}", - "Global Facts - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Global Facts - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Global Facts - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Global Facts - # prompt tokens": "{\"description\": \"min=399.71, mean=399.71, max=399.71, sum=799.42 (2)\", \"tab\": \"General information\", \"score\": \"399.71\"}", - "Global Facts - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"global_facts\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_global_facts\"" - } - } - }, - { - "evaluation_name": "Jurisprudence", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Jurisprudence", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.806, - "details": { - "description": "min=0.806, mean=0.806, max=0.806, sum=1.611 (2)", - "tab": "Accuracy", - "Jurisprudence - Observed inference time (s)": "{\"description\": \"min=0.509, mean=0.509, max=0.509, sum=1.017 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5086877279811435\"}", - "Jurisprudence - # eval": "{\"description\": \"min=108, mean=108, max=108, sum=216 (2)\", \"tab\": \"General information\", \"score\": \"108.0\"}", - "Jurisprudence - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Jurisprudence - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Jurisprudence - # prompt tokens": "{\"description\": \"min=394.639, mean=394.639, max=394.639, sum=789.278 (2)\", \"tab\": \"General information\", \"score\": \"394.6388888888889\"}", - "Jurisprudence - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"jurisprudence\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_jurisprudence\"" - } - } - }, - { - "evaluation_name": "Philosophy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Philosophy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.746, - "details": { - "description": "min=0.746, mean=0.746, max=0.746, sum=1.492 (2)", - "tab": "Accuracy", - "Philosophy - Observed inference time (s)": "{\"description\": \"min=0.472, mean=0.472, max=0.472, sum=0.944 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4717828660149283\"}", - "Philosophy - # eval": "{\"description\": \"min=311, mean=311, max=311, sum=622 (2)\", \"tab\": \"General information\", \"score\": \"311.0\"}", - "Philosophy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Philosophy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Philosophy - # prompt tokens": "{\"description\": \"min=329.084, mean=329.084, max=329.084, sum=658.167 (2)\", \"tab\": \"General information\", \"score\": \"329.08360128617363\"}", - "Philosophy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"philosophy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_philosophy\"" - } - } - }, - { - "evaluation_name": "Professional Psychology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Professional Psychology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.722, - "details": { - "description": "min=0.722, mean=0.722, max=0.722, sum=1.444 (2)", - "tab": "Accuracy", - "Professional Medicine - Observed inference time (s)": "{\"description\": \"min=0.485, mean=0.485, max=0.485, sum=0.971 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4853776947540395\"}", - "Professional Accounting - Observed inference time (s)": "{\"description\": \"min=0.423, mean=0.423, max=0.423, sum=0.846 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.42316425692105125\"}", - "Professional Law - Observed inference time (s)": "{\"description\": \"min=0.442, mean=0.442, max=0.442, sum=0.883 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4417385995932011\"}", - "Professional Psychology - Observed inference time (s)": "{\"description\": \"min=0.422, mean=0.422, max=0.422, sum=0.843 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.42156751132478903\"}", - "Professional Medicine - # eval": "{\"description\": \"min=272, mean=272, max=272, sum=544 (2)\", \"tab\": \"General information\", \"score\": \"272.0\"}", - "Professional Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Medicine - # prompt tokens": "{\"description\": \"min=1094.585, mean=1094.585, max=1094.585, sum=2189.169 (2)\", \"tab\": \"General information\", \"score\": \"1094.5845588235295\"}", - "Professional Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Accounting - # eval": "{\"description\": \"min=282, mean=282, max=282, sum=564 (2)\", \"tab\": \"General information\", \"score\": \"282.0\"}", - "Professional Accounting - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Accounting - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # prompt tokens": "{\"description\": \"min=658.592, mean=658.592, max=658.592, sum=1317.184 (2)\", \"tab\": \"General information\", \"score\": \"658.5921985815603\"}", - "Professional Accounting - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Law - # eval": "{\"description\": \"min=1534, mean=1534, max=1534, sum=3068 (2)\", \"tab\": \"General information\", \"score\": \"1534.0\"}", - "Professional Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # prompt tokens": "{\"description\": \"min=1637.787, mean=1637.787, max=1637.787, sum=3275.574 (2)\", \"tab\": \"General information\", \"score\": \"1637.7868318122555\"}", - "Professional Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Psychology - # eval": "{\"description\": \"min=612, mean=612, max=612, sum=1224 (2)\", \"tab\": \"General information\", \"score\": \"612.0\"}", - "Professional Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # prompt tokens": "{\"description\": \"min=575.114, mean=575.114, max=575.114, sum=1150.229 (2)\", \"tab\": \"General information\", \"score\": \"575.1143790849674\"}", - "Professional Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"professional_psychology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_professional_psychology\"" - } - } - }, - { - "evaluation_name": "Us Foreign Policy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Us Foreign Policy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.89, - "details": { - "description": "min=0.89, mean=0.89, max=0.89, sum=1.78 (2)", - "tab": "Accuracy", - "Us Foreign Policy - Observed inference time (s)": "{\"description\": \"min=0.456, mean=0.456, max=0.456, sum=0.911 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4557087206840515\"}", - "Us Foreign Policy - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Us Foreign Policy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Us Foreign Policy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Us Foreign Policy - # prompt tokens": "{\"description\": \"min=422.79, mean=422.79, max=422.79, sum=845.58 (2)\", \"tab\": \"General information\", \"score\": \"422.79\"}", - "Us Foreign Policy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"us_foreign_policy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_us_foreign_policy\"" - } - } - }, - { - "evaluation_name": "Astronomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Astronomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.75, - "details": { - "description": "min=0.75, mean=0.75, max=0.75, sum=1.5 (2)", - "tab": "Accuracy", - "Astronomy - Observed inference time (s)": "{\"description\": \"min=0.421, mean=0.421, max=0.421, sum=0.842 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.42091869994213704\"}", - "Astronomy - # eval": "{\"description\": \"min=152, mean=152, max=152, sum=304 (2)\", \"tab\": \"General information\", \"score\": \"152.0\"}", - "Astronomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Astronomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Astronomy - # prompt tokens": "{\"description\": \"min=579.691, mean=579.691, max=579.691, sum=1159.382 (2)\", \"tab\": \"General information\", \"score\": \"579.6907894736842\"}", - "Astronomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"astronomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_astronomy\"" - } - } - }, - { - "evaluation_name": "Business Ethics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Business Ethics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.75, - "details": { - "description": "min=0.75, mean=0.75, max=0.75, sum=1.5 (2)", - "tab": "Accuracy", - "Business Ethics - Observed inference time (s)": "{\"description\": \"min=0.453, mean=0.453, max=0.453, sum=0.906 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4530529642105103\"}", - "Business Ethics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Business Ethics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Business Ethics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Business Ethics - # prompt tokens": "{\"description\": \"min=569.52, mean=569.52, max=569.52, sum=1139.04 (2)\", \"tab\": \"General information\", \"score\": \"569.52\"}", - "Business Ethics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"business_ethics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_business_ethics\"" - } - } - }, - { - "evaluation_name": "Clinical Knowledge", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Clinical Knowledge", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.755, - "details": { - "description": "min=0.755, mean=0.755, max=0.755, sum=1.509 (2)", - "tab": "Accuracy", - "Clinical Knowledge - Observed inference time (s)": "{\"description\": \"min=0.418, mean=0.418, max=0.418, sum=0.837 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.41833644812961795\"}", - "Clinical Knowledge - # eval": "{\"description\": \"min=265, mean=265, max=265, sum=530 (2)\", \"tab\": \"General information\", \"score\": \"265.0\"}", - "Clinical Knowledge - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Clinical Knowledge - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Clinical Knowledge - # prompt tokens": "{\"description\": \"min=397.947, mean=397.947, max=397.947, sum=795.894 (2)\", \"tab\": \"General information\", \"score\": \"397.94716981132075\"}", - "Clinical Knowledge - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"clinical_knowledge\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_clinical_knowledge\"" - } - } - }, - { - "evaluation_name": "Conceptual Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Conceptual Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.634, - "details": { - "description": "min=0.634, mean=0.634, max=0.634, sum=1.268 (2)", - "tab": "Accuracy", - "Conceptual Physics - Observed inference time (s)": "{\"description\": \"min=0.808, mean=0.808, max=0.808, sum=1.616 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8081990150695152\"}", - "Conceptual Physics - # eval": "{\"description\": \"min=235, mean=235, max=235, sum=470 (2)\", \"tab\": \"General information\", \"score\": \"235.0\"}", - "Conceptual Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Conceptual Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Conceptual Physics - # prompt tokens": "{\"description\": \"min=304.838, mean=304.838, max=304.838, sum=609.677 (2)\", \"tab\": \"General information\", \"score\": \"304.83829787234043\"}", - "Conceptual Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"conceptual_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_conceptual_physics\"" - } - } - }, - { - "evaluation_name": "Electrical Engineering", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Electrical Engineering", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.669, - "details": { - "description": "min=0.669, mean=0.669, max=0.669, sum=1.338 (2)", - "tab": "Accuracy", - "Electrical Engineering - Observed inference time (s)": "{\"description\": \"min=1.242, mean=1.242, max=1.242, sum=2.485 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.2423763686213\"}", - "Electrical Engineering - # eval": "{\"description\": \"min=145, mean=145, max=145, sum=290 (2)\", \"tab\": \"General information\", \"score\": \"145.0\"}", - "Electrical Engineering - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Electrical Engineering - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Electrical Engineering - # prompt tokens": "{\"description\": \"min=440.641, mean=440.641, max=440.641, sum=881.283 (2)\", \"tab\": \"General information\", \"score\": \"440.6413793103448\"}", - "Electrical Engineering - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"electrical_engineering\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_electrical_engineering\"" - } - } - }, - { - "evaluation_name": "Elementary Mathematics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Elementary Mathematics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.534, - "details": { - "description": "min=0.534, mean=0.534, max=0.534, sum=1.069 (2)", - "tab": "Accuracy", - "Elementary Mathematics - Observed inference time (s)": "{\"description\": \"min=0.436, mean=0.436, max=0.436, sum=0.872 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4359189442225865\"}", - "Elementary Mathematics - # eval": "{\"description\": \"min=378, mean=378, max=378, sum=756 (2)\", \"tab\": \"General information\", \"score\": \"378.0\"}", - "Elementary Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Elementary Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Elementary Mathematics - # prompt tokens": "{\"description\": \"min=531.862, mean=531.862, max=531.862, sum=1063.725 (2)\", \"tab\": \"General information\", \"score\": \"531.8624338624338\"}", - "Elementary Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"elementary_mathematics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_elementary_mathematics\"" - } - } - }, - { - "evaluation_name": "Formal Logic", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Formal Logic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.444, - "details": { - "description": "min=0.444, mean=0.444, max=0.444, sum=0.889 (2)", - "tab": "Accuracy", - "Formal Logic - Observed inference time (s)": "{\"description\": \"min=0.431, mean=0.431, max=0.431, sum=0.861 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.43056895051683697\"}", - "Formal Logic - # eval": "{\"description\": \"min=126, mean=126, max=126, sum=252 (2)\", \"tab\": \"General information\", \"score\": \"126.0\"}", - "Formal Logic - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Formal Logic - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Formal Logic - # prompt tokens": "{\"description\": \"min=606.762, mean=606.762, max=606.762, sum=1213.524 (2)\", \"tab\": \"General information\", \"score\": \"606.7619047619048\"}", - "Formal Logic - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"formal_logic\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_formal_logic\"" - } - } - }, - { - "evaluation_name": "High School World History", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on High School World History", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.819, - "details": { - "description": "min=0.819, mean=0.819, max=0.819, sum=1.637 (2)", - "tab": "Accuracy", - "High School Biology - Observed inference time (s)": "{\"description\": \"min=0.572, mean=0.572, max=0.572, sum=1.143 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5715394450772193\"}", - "High School Chemistry - Observed inference time (s)": "{\"description\": \"min=0.491, mean=0.491, max=0.491, sum=0.981 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.49073645046779085\"}", - "High School Computer Science - Observed inference time (s)": "{\"description\": \"min=0.433, mean=0.433, max=0.433, sum=0.865 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.43273836851119996\"}", - "High School European History - Observed inference time (s)": "{\"description\": \"min=0.489, mean=0.489, max=0.489, sum=0.977 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.48863930413217255\"}", - "High School Geography - Observed inference time (s)": "{\"description\": \"min=0.436, mean=0.436, max=0.436, sum=0.872 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4360258868246367\"}", - "High School Government And Politics - Observed inference time (s)": "{\"description\": \"min=0.484, mean=0.484, max=0.484, sum=0.967 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4836950153884492\"}", - "High School Macroeconomics - Observed inference time (s)": "{\"description\": \"min=0.422, mean=0.422, max=0.422, sum=0.843 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4215013412328867\"}", - "High School Mathematics - Observed inference time (s)": "{\"description\": \"min=0.49, mean=0.49, max=0.49, sum=0.979 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.48968876291204383\"}", - "High School Microeconomics - Observed inference time (s)": "{\"description\": \"min=0.432, mean=0.432, max=0.432, sum=0.864 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4320918882594389\"}", - "High School Physics - Observed inference time (s)": "{\"description\": \"min=0.466, mean=0.466, max=0.466, sum=0.932 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4659363955061957\"}", - "High School Psychology - Observed inference time (s)": "{\"description\": \"min=0.443, mean=0.443, max=0.443, sum=0.887 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4434620769745713\"}", - "High School Statistics - Observed inference time (s)": "{\"description\": \"min=0.431, mean=0.431, max=0.431, sum=0.862 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.43081507749027675\"}", - "High School US History - Observed inference time (s)": "{\"description\": \"min=0.486, mean=0.486, max=0.486, sum=0.971 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4857361819229874\"}", - "High School World History - Observed inference time (s)": "{\"description\": \"min=0.441, mean=0.441, max=0.441, sum=0.882 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.44100493620216596\"}", - "High School Biology - # eval": "{\"description\": \"min=310, mean=310, max=310, sum=620 (2)\", \"tab\": \"General information\", \"score\": \"310.0\"}", - "High School Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Biology - # prompt tokens": "{\"description\": \"min=513.677, mean=513.677, max=513.677, sum=1027.355 (2)\", \"tab\": \"General information\", \"score\": \"513.6774193548387\"}", - "High School Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Chemistry - # eval": "{\"description\": \"min=203, mean=203, max=203, sum=406 (2)\", \"tab\": \"General information\", \"score\": \"203.0\"}", - "High School Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # prompt tokens": "{\"description\": \"min=496.714, mean=496.714, max=496.714, sum=993.429 (2)\", \"tab\": \"General information\", \"score\": \"496.7142857142857\"}", - "High School Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "High School Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # prompt tokens": "{\"description\": \"min=867.78, mean=867.78, max=867.78, sum=1735.56 (2)\", \"tab\": \"General information\", \"score\": \"867.78\"}", - "High School Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School European History - # eval": "{\"description\": \"min=165, mean=165, max=165, sum=330 (2)\", \"tab\": \"General information\", \"score\": \"165.0\"}", - "High School European History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School European History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # prompt tokens": "{\"description\": \"min=2798.073, mean=2798.073, max=2798.073, sum=5596.145 (2)\", \"tab\": \"General information\", \"score\": \"2798.072727272727\"}", - "High School European History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Geography - # eval": "{\"description\": \"min=198, mean=198, max=198, sum=396 (2)\", \"tab\": \"General information\", \"score\": \"198.0\"}", - "High School Geography - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Geography - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # prompt tokens": "{\"description\": \"min=372.045, mean=372.045, max=372.045, sum=744.091 (2)\", \"tab\": \"General information\", \"score\": \"372.04545454545456\"}", - "High School Geography - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Government And Politics - # eval": "{\"description\": \"min=193, mean=193, max=193, sum=386 (2)\", \"tab\": \"General information\", \"score\": \"193.0\"}", - "High School Government And Politics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Government And Politics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # prompt tokens": "{\"description\": \"min=465.824, mean=465.824, max=465.824, sum=931.648 (2)\", \"tab\": \"General information\", \"score\": \"465.8238341968912\"}", - "High School Government And Politics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Macroeconomics - # eval": "{\"description\": \"min=390, mean=390, max=390, sum=780 (2)\", \"tab\": \"General information\", \"score\": \"390.0\"}", - "High School Macroeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Macroeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # prompt tokens": "{\"description\": \"min=371.562, mean=371.562, max=371.562, sum=743.123 (2)\", \"tab\": \"General information\", \"score\": \"371.5615384615385\"}", - "High School Macroeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Mathematics - # eval": "{\"description\": \"min=270, mean=270, max=270, sum=540 (2)\", \"tab\": \"General information\", \"score\": \"270.0\"}", - "High School Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # prompt tokens": "{\"description\": \"min=532.374, mean=532.374, max=532.374, sum=1064.748 (2)\", \"tab\": \"General information\", \"score\": \"532.3740740740741\"}", - "High School Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Microeconomics - # eval": "{\"description\": \"min=238, mean=238, max=238, sum=476 (2)\", \"tab\": \"General information\", \"score\": \"238.0\"}", - "High School Microeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Microeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # prompt tokens": "{\"description\": \"min=399.025, mean=399.025, max=399.025, sum=798.05 (2)\", \"tab\": \"General information\", \"score\": \"399.02521008403363\"}", - "High School Microeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Physics - # eval": "{\"description\": \"min=151, mean=151, max=151, sum=302 (2)\", \"tab\": \"General information\", \"score\": \"151.0\"}", - "High School Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # prompt tokens": "{\"description\": \"min=560.464, mean=560.464, max=560.464, sum=1120.927 (2)\", \"tab\": \"General information\", \"score\": \"560.4635761589404\"}", - "High School Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Psychology - # eval": "{\"description\": \"min=545, mean=545, max=545, sum=1090 (2)\", \"tab\": \"General information\", \"score\": \"545.0\"}", - "High School Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # prompt tokens": "{\"description\": \"min=495.246, mean=495.246, max=495.246, sum=990.492 (2)\", \"tab\": \"General information\", \"score\": \"495.24587155963303\"}", - "High School Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Statistics - # eval": "{\"description\": \"min=216, mean=216, max=216, sum=432 (2)\", \"tab\": \"General information\", \"score\": \"216.0\"}", - "High School Statistics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Statistics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # prompt tokens": "{\"description\": \"min=795.699, mean=795.699, max=795.699, sum=1591.398 (2)\", \"tab\": \"General information\", \"score\": \"795.699074074074\"}", - "High School Statistics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School US History - # eval": "{\"description\": \"min=204, mean=204, max=204, sum=408 (2)\", \"tab\": \"General information\", \"score\": \"204.0\"}", - "High School US History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School US History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # prompt tokens": "{\"description\": \"min=2217.809, mean=2217.809, max=2217.809, sum=4435.618 (2)\", \"tab\": \"General information\", \"score\": \"2217.8088235294117\"}", - "High School US History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School World History - # eval": "{\"description\": \"min=237, mean=237, max=237, sum=474 (2)\", \"tab\": \"General information\", \"score\": \"237.0\"}", - "High School World History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School World History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # prompt tokens": "{\"description\": \"min=1428.27, mean=1428.27, max=1428.27, sum=2856.54 (2)\", \"tab\": \"General information\", \"score\": \"1428.2700421940929\"}", - "High School World History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"high_school_world_history\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_high_school_world_history\"" - } - } - }, - { - "evaluation_name": "Human Sexuality", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Human Sexuality", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.779, - "details": { - "description": "min=0.779, mean=0.779, max=0.779, sum=1.557 (2)", - "tab": "Accuracy", - "Human Aging - Observed inference time (s)": "{\"description\": \"min=0.423, mean=0.423, max=0.423, sum=0.846 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.42309954027423946\"}", - "Human Sexuality - Observed inference time (s)": "{\"description\": \"min=0.417, mean=0.417, max=0.417, sum=0.833 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4166541681944869\"}", - "Human Aging - # eval": "{\"description\": \"min=223, mean=223, max=223, sum=446 (2)\", \"tab\": \"General information\", \"score\": \"223.0\"}", - "Human Aging - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Aging - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Aging - # prompt tokens": "{\"description\": \"min=319.906, mean=319.906, max=319.906, sum=639.812 (2)\", \"tab\": \"General information\", \"score\": \"319.90582959641256\"}", - "Human Aging - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Human Sexuality - # eval": "{\"description\": \"min=131, mean=131, max=131, sum=262 (2)\", \"tab\": \"General information\", \"score\": \"131.0\"}", - "Human Sexuality - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Sexuality - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # prompt tokens": "{\"description\": \"min=341.183, mean=341.183, max=341.183, sum=682.366 (2)\", \"tab\": \"General information\", \"score\": \"341.1832061068702\"}", - "Human Sexuality - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"human_sexuality\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_human_sexuality\"" - } - } - }, - { - "evaluation_name": "International Law", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on International Law", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.81, - "details": { - "description": "min=0.81, mean=0.81, max=0.81, sum=1.62 (2)", - "tab": "Accuracy", - "International Law - Observed inference time (s)": "{\"description\": \"min=0.525, mean=0.525, max=0.525, sum=1.05 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5249163257189033\"}", - "International Law - # eval": "{\"description\": \"min=121, mean=121, max=121, sum=242 (2)\", \"tab\": \"General information\", \"score\": \"121.0\"}", - "International Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "International Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "International Law - # prompt tokens": "{\"description\": \"min=639.851, mean=639.851, max=639.851, sum=1279.702 (2)\", \"tab\": \"General information\", \"score\": \"639.8512396694215\"}", - "International Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"international_law\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_international_law\"" - } - } - }, - { - "evaluation_name": "Logical Fallacies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Logical Fallacies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.779, - "details": { - "description": "min=0.779, mean=0.779, max=0.779, sum=1.558 (2)", - "tab": "Accuracy", - "Logical Fallacies - Observed inference time (s)": "{\"description\": \"min=0.504, mean=0.504, max=0.504, sum=1.008 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5038382904661214\"}", - "Logical Fallacies - # eval": "{\"description\": \"min=163, mean=163, max=163, sum=326 (2)\", \"tab\": \"General information\", \"score\": \"163.0\"}", - "Logical Fallacies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Logical Fallacies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Logical Fallacies - # prompt tokens": "{\"description\": \"min=449.595, mean=449.595, max=449.595, sum=899.19 (2)\", \"tab\": \"General information\", \"score\": \"449.5950920245399\"}", - "Logical Fallacies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"logical_fallacies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_logical_fallacies\"" - } - } - }, - { - "evaluation_name": "Machine Learning", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Machine Learning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.455, - "details": { - "description": "min=0.455, mean=0.455, max=0.455, sum=0.911 (2)", - "tab": "Accuracy", - "Machine Learning - Observed inference time (s)": "{\"description\": \"min=0.437, mean=0.437, max=0.437, sum=0.875 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4374160830463682\"}", - "Machine Learning - # eval": "{\"description\": \"min=112, mean=112, max=112, sum=224 (2)\", \"tab\": \"General information\", \"score\": \"112.0\"}", - "Machine Learning - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Machine Learning - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Machine Learning - # prompt tokens": "{\"description\": \"min=668.054, mean=668.054, max=668.054, sum=1336.107 (2)\", \"tab\": \"General information\", \"score\": \"668.0535714285714\"}", - "Machine Learning - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"machine_learning\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_machine_learning\"" - } - } - }, - { - "evaluation_name": "Management", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Management", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.835, - "details": { - "description": "min=0.835, mean=0.835, max=0.835, sum=1.67 (2)", - "tab": "Accuracy", - "Management - Observed inference time (s)": "{\"description\": \"min=0.458, mean=0.458, max=0.458, sum=0.917 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4584047493425388\"}", - "Management - # eval": "{\"description\": \"min=103, mean=103, max=103, sum=206 (2)\", \"tab\": \"General information\", \"score\": \"103.0\"}", - "Management - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Management - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Management - # prompt tokens": "{\"description\": \"min=283.796, mean=283.796, max=283.796, sum=567.592 (2)\", \"tab\": \"General information\", \"score\": \"283.79611650485435\"}", - "Management - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"management\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_management\"" - } - } - }, - { - "evaluation_name": "Marketing", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Marketing", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.91, - "details": { - "description": "min=0.91, mean=0.91, max=0.91, sum=1.821 (2)", - "tab": "Accuracy", - "Marketing - Observed inference time (s)": "{\"description\": \"min=0.421, mean=0.421, max=0.421, sum=0.842 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4209032700611995\"}", - "Marketing - # eval": "{\"description\": \"min=234, mean=234, max=234, sum=468 (2)\", \"tab\": \"General information\", \"score\": \"234.0\"}", - "Marketing - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Marketing - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Marketing - # prompt tokens": "{\"description\": \"min=404.218, mean=404.218, max=404.218, sum=808.436 (2)\", \"tab\": \"General information\", \"score\": \"404.21794871794873\"}", - "Marketing - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"marketing\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_marketing\"" - } - } - }, - { - "evaluation_name": "Medical Genetics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Medical Genetics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.73, - "details": { - "description": "min=0.73, mean=0.73, max=0.73, sum=1.46 (2)", - "tab": "Accuracy", - "Medical Genetics - Observed inference time (s)": "{\"description\": \"min=0.489, mean=0.489, max=0.489, sum=0.979 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.48938191413879395\"}", - "Medical Genetics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Medical Genetics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Medical Genetics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Medical Genetics - # prompt tokens": "{\"description\": \"min=341, mean=341, max=341, sum=682 (2)\", \"tab\": \"General information\", \"score\": \"341.0\"}", - "Medical Genetics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"medical_genetics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_medical_genetics\"" - } - } - }, - { - "evaluation_name": "Miscellaneous", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Miscellaneous", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.89, - "details": { - "description": "min=0.89, mean=0.89, max=0.89, sum=1.78 (2)", - "tab": "Accuracy", - "Miscellaneous - Observed inference time (s)": "{\"description\": \"min=0.421, mean=0.421, max=0.421, sum=0.841 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4205615121590528\"}", - "Miscellaneous - # eval": "{\"description\": \"min=783, mean=783, max=783, sum=1566 (2)\", \"tab\": \"General information\", \"score\": \"783.0\"}", - "Miscellaneous - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Miscellaneous - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Miscellaneous - # prompt tokens": "{\"description\": \"min=299.925, mean=299.925, max=299.925, sum=599.849 (2)\", \"tab\": \"General information\", \"score\": \"299.92464878671774\"}", - "Miscellaneous - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"miscellaneous\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_miscellaneous\"" - } - } - }, - { - "evaluation_name": "Moral Scenarios", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Moral Scenarios", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.355, - "details": { - "description": "min=0.355, mean=0.355, max=0.355, sum=0.711 (2)", - "tab": "Accuracy", - "Moral Disputes - Observed inference time (s)": "{\"description\": \"min=0.439, mean=0.439, max=0.439, sum=0.878 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.43890244423309505\"}", - "Moral Scenarios - Observed inference time (s)": "{\"description\": \"min=0.422, mean=0.422, max=0.422, sum=0.843 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4216500338229387\"}", - "Moral Disputes - # eval": "{\"description\": \"min=346, mean=346, max=346, sum=692 (2)\", \"tab\": \"General information\", \"score\": \"346.0\"}", - "Moral Disputes - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Disputes - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Disputes - # prompt tokens": "{\"description\": \"min=476.145, mean=476.145, max=476.145, sum=952.289 (2)\", \"tab\": \"General information\", \"score\": \"476.1445086705202\"}", - "Moral Disputes - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Moral Scenarios - # eval": "{\"description\": \"min=895, mean=895, max=895, sum=1790 (2)\", \"tab\": \"General information\", \"score\": \"895.0\"}", - "Moral Scenarios - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Scenarios - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # prompt tokens": "{\"description\": \"min=656.455, mean=656.455, max=656.455, sum=1312.909 (2)\", \"tab\": \"General information\", \"score\": \"656.454748603352\"}", - "Moral Scenarios - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"moral_scenarios\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_moral_scenarios\"" - } - } - }, - { - "evaluation_name": "Nutrition", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Nutrition", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.748, - "details": { - "description": "min=0.748, mean=0.748, max=0.748, sum=1.497 (2)", - "tab": "Accuracy", - "Nutrition - Observed inference time (s)": "{\"description\": \"min=0.438, mean=0.438, max=0.438, sum=0.876 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4378981278612723\"}", - "Nutrition - # eval": "{\"description\": \"min=306, mean=306, max=306, sum=612 (2)\", \"tab\": \"General information\", \"score\": \"306.0\"}", - "Nutrition - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Nutrition - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Nutrition - # prompt tokens": "{\"description\": \"min=586.817, mean=586.817, max=586.817, sum=1173.634 (2)\", \"tab\": \"General information\", \"score\": \"586.8169934640523\"}", - "Nutrition - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"nutrition\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_nutrition\"" - } - } - }, - { - "evaluation_name": "Prehistory", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Prehistory", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.735, - "details": { - "description": "min=0.735, mean=0.735, max=0.735, sum=1.469 (2)", - "tab": "Accuracy", - "Prehistory - Observed inference time (s)": "{\"description\": \"min=0.462, mean=0.462, max=0.462, sum=0.924 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4620003163078685\"}", - "Prehistory - # eval": "{\"description\": \"min=324, mean=324, max=324, sum=648 (2)\", \"tab\": \"General information\", \"score\": \"324.0\"}", - "Prehistory - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Prehistory - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Prehistory - # prompt tokens": "{\"description\": \"min=514.559, mean=514.559, max=514.559, sum=1029.117 (2)\", \"tab\": \"General information\", \"score\": \"514.5586419753087\"}", - "Prehistory - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"prehistory\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_prehistory\"" - } - } - }, - { - "evaluation_name": "Public Relations", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Public Relations", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.727, - "details": { - "description": "min=0.727, mean=0.727, max=0.727, sum=1.455 (2)", - "tab": "Accuracy", - "Public Relations - Observed inference time (s)": "{\"description\": \"min=0.685, mean=0.685, max=0.685, sum=1.371 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6854934020475908\"}", - "Public Relations - # eval": "{\"description\": \"min=110, mean=110, max=110, sum=220 (2)\", \"tab\": \"General information\", \"score\": \"110.0\"}", - "Public Relations - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Public Relations - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Public Relations - # prompt tokens": "{\"description\": \"min=405.318, mean=405.318, max=405.318, sum=810.636 (2)\", \"tab\": \"General information\", \"score\": \"405.3181818181818\"}", - "Public Relations - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"public_relations\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_public_relations\"" - } - } - }, - { - "evaluation_name": "Security Studies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Security Studies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.751, - "details": { - "description": "min=0.751, mean=0.751, max=0.751, sum=1.502 (2)", - "tab": "Accuracy", - "Security Studies - Observed inference time (s)": "{\"description\": \"min=0.539, mean=0.539, max=0.539, sum=1.077 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5387308393205915\"}", - "Security Studies - # eval": "{\"description\": \"min=245, mean=245, max=245, sum=490 (2)\", \"tab\": \"General information\", \"score\": \"245.0\"}", - "Security Studies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Security Studies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Security Studies - # prompt tokens": "{\"description\": \"min=1164.473, mean=1164.473, max=1164.473, sum=2328.947 (2)\", \"tab\": \"General information\", \"score\": \"1164.4734693877551\"}", - "Security Studies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"security_studies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_security_studies\"" - } - } - }, - { - "evaluation_name": "Sociology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Sociology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.861, - "details": { - "description": "min=0.861, mean=0.861, max=0.861, sum=1.721 (2)", - "tab": "Accuracy", - "Sociology - Observed inference time (s)": "{\"description\": \"min=0.428, mean=0.428, max=0.428, sum=0.856 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.42779283025371495\"}", - "Sociology - # eval": "{\"description\": \"min=201, mean=201, max=201, sum=402 (2)\", \"tab\": \"General information\", \"score\": \"201.0\"}", - "Sociology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Sociology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Sociology - # prompt tokens": "{\"description\": \"min=445.522, mean=445.522, max=445.522, sum=891.045 (2)\", \"tab\": \"General information\", \"score\": \"445.5223880597015\"}", - "Sociology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"sociology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_sociology\"" - } - } - }, - { - "evaluation_name": "Virology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Virology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.536, - "details": { - "description": "min=0.536, mean=0.536, max=0.536, sum=1.072 (2)", - "tab": "Accuracy", - "Virology - Observed inference time (s)": "{\"description\": \"min=0.396, mean=0.396, max=0.396, sum=0.791 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.39562296723744955\"}", - "Virology - # eval": "{\"description\": \"min=166, mean=166, max=166, sum=332 (2)\", \"tab\": \"General information\", \"score\": \"166.0\"}", - "Virology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Virology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Virology - # prompt tokens": "{\"description\": \"min=343.09, mean=343.09, max=343.09, sum=686.181 (2)\", \"tab\": \"General information\", \"score\": \"343.0903614457831\"}", - "Virology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"virology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_virology\"" - } - } - }, - { - "evaluation_name": "World Religions", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on World Religions", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.842, - "details": { - "description": "min=0.842, mean=0.842, max=0.842, sum=1.684 (2)", - "tab": "Accuracy", - "World Religions - Observed inference time (s)": "{\"description\": \"min=0.413, mean=0.413, max=0.413, sum=0.827 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.41344076848169514\"}", - "World Religions - # eval": "{\"description\": \"min=171, mean=171, max=171, sum=342 (2)\", \"tab\": \"General information\", \"score\": \"171.0\"}", - "World Religions - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "World Religions - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "World Religions - # prompt tokens": "{\"description\": \"min=275.561, mean=275.561, max=275.561, sum=551.123 (2)\", \"tab\": \"General information\", \"score\": \"275.56140350877195\"}", - "World Religions - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"world_religions\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_world_religions\"" - } - } - }, - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.493, - "details": { - "description": "", - "tab": "Efficiency" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_id": "reward-bench/openai_gpt-3.5-turbo-0125/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6534 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9218 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4452 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6547 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5912 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6548 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/openai_gpt-3.5-turbo-0301.json b/data/models/openai_gpt-3.5-turbo-0301.json deleted file mode 100644 index 55e573d90e220b167bbd65cc17192b295392a783..0000000000000000000000000000000000000000 --- a/data/models/openai_gpt-3.5-turbo-0301.json +++ /dev/null @@ -1,674 +0,0 @@ -{ - "model_info": { - "name": "gpt-3.5-turbo-0301", - "id": "openai/gpt-3.5-turbo-0301", - "developer": "OpenAI", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_classic/openai_gpt-3.5-turbo-0301/1774096308.339228", - "retrieved_timestamp": "1774096308.339228", - "source_metadata": { - "source_name": "helm_classic", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_classic", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_classic", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperform on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.76, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Calibration": "{\"description\": \"\", \"tab\": \"Calibration\", \"score\": \"\"}", - "Mean win rate - Robustness": "{\"description\": \"\", \"tab\": \"Robustness\", \"score\": \"0.8156643356643357\"}", - "Mean win rate - Fairness": "{\"description\": \"\", \"tab\": \"Fairness\", \"score\": \"0.6617249417249418\"}", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}", - "Mean win rate - Bias": "{\"description\": \"\", \"tab\": \"Bias\", \"score\": \"0.5128923320135726\"}", - "Mean win rate - Toxicity": "{\"description\": \"\", \"tab\": \"Toxicity\", \"score\": \"0.8050116550116551\"}", - "Mean win rate - Summarization metrics": "{\"description\": \"\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.59, - "details": { - "description": "min=0.3, mean=0.59, max=0.85, sum=2.949 (5)", - "tab": "Accuracy", - "MMLU - ECE (10-bin)": "{\"description\": \"5 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "MMLU - EM (Robustness)": "{\"description\": \"min=0.23, mean=0.525, max=0.79, sum=2.627 (5)\", \"tab\": \"Robustness\", \"score\": \"0.5254736842105263\"}", - "MMLU - EM (Fairness)": "{\"description\": \"min=0.26, mean=0.53, max=0.8, sum=2.65 (5)\", \"tab\": \"Fairness\", \"score\": \"0.5299649122807017\"}", - "MMLU - Denoised inference time (s)": "{\"description\": \"5 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=366.44, mean=460.72, max=607.43, sum=2303.6 (5)\", \"tab\": \"General information\", \"score\": \"460.71996491228066\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1.012, max=1.06, sum=5.06 (5)\", \"tab\": \"General information\", \"score\": \"1.012\"}", - "MMLU - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "BoolQ", - "source_data": { - "dataset_name": "BoolQ", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on BoolQ", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.74, - "details": { - "description": "min=0.74, mean=0.74, max=0.74, sum=0.74 (1)", - "tab": "Accuracy", - "BoolQ - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "BoolQ - EM (Robustness)": "{\"description\": \"min=0.66, mean=0.66, max=0.66, sum=0.66 (1)\", \"tab\": \"Robustness\", \"score\": \"0.66\"}", - "BoolQ - EM (Fairness)": "{\"description\": \"min=0.666, mean=0.666, max=0.666, sum=0.666 (1)\", \"tab\": \"Fairness\", \"score\": \"0.666\"}", - "BoolQ - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "BoolQ - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "BoolQ - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "BoolQ - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "BoolQ - # prompt tokens": "{\"description\": \"min=1220.329, mean=1220.329, max=1220.329, sum=1220.329 (1)\", \"tab\": \"General information\", \"score\": \"1220.329\"}", - "BoolQ - # output tokens": "{\"description\": \"min=1.932, mean=1.932, max=1.932, sum=1.932 (1)\", \"tab\": \"General information\", \"score\": \"1.932\"}", - "BoolQ - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "BoolQ - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (gender)": "{\"description\": \"min=0.5, mean=0.5, max=0.5, sum=0.5 (1)\", \"tab\": \"Bias\", \"score\": \"0.5\"}", - "BoolQ - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.663, - "details": { - "description": "min=0.663, mean=0.663, max=0.663, sum=0.663 (1)", - "tab": "Accuracy", - "NarrativeQA - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "NarrativeQA - F1 (Robustness)": "{\"description\": \"min=0.602, mean=0.602, max=0.602, sum=0.602 (1)\", \"tab\": \"Robustness\", \"score\": \"0.6017866194784781\"}", - "NarrativeQA - F1 (Fairness)": "{\"description\": \"min=0.585, mean=0.585, max=0.585, sum=0.585 (1)\", \"tab\": \"Fairness\", \"score\": \"0.5846601621436455\"}", - "NarrativeQA - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=4.966, mean=4.966, max=4.966, sum=4.966 (1)\", \"tab\": \"General information\", \"score\": \"4.966197183098592\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=3443.349, mean=3443.349, max=3443.349, sum=3443.349 (1)\", \"tab\": \"General information\", \"score\": \"3443.349295774648\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=11.186, mean=11.186, max=11.186, sum=11.186 (1)\", \"tab\": \"General information\", \"score\": \"11.185915492957747\"}", - "NarrativeQA - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NarrativeQA - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NarrativeQA - Stereotypes (gender)": "{\"description\": \"min=0.479, mean=0.479, max=0.479, sum=0.479 (1)\", \"tab\": \"Bias\", \"score\": \"0.4789473684210526\"}", - "NarrativeQA - Representation (race)": "{\"description\": \"min=0.333, mean=0.333, max=0.333, sum=0.333 (1)\", \"tab\": \"Bias\", \"score\": \"0.33333333333333337\"}", - "NarrativeQA - Representation (gender)": "{\"description\": \"min=0.216, mean=0.216, max=0.216, sum=0.216 (1)\", \"tab\": \"Bias\", \"score\": \"0.21590909090909088\"}", - "NarrativeQA - Toxic fraction": "{\"description\": \"min=0.011, mean=0.011, max=0.011, sum=0.011 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.011267605633802818\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (open-book)", - "source_data": { - "dataset_name": "NaturalQuestions (open-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (open-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.624, - "details": { - "description": "min=0.624, mean=0.624, max=0.624, sum=0.624 (1)", - "tab": "Accuracy", - "NaturalQuestions (closed-book) - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "NaturalQuestions (open-book) - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - F1 (Robustness)": "{\"description\": \"min=0.327, mean=0.327, max=0.327, sum=0.327 (1)\", \"tab\": \"Robustness\", \"score\": \"0.32682585209770315\"}", - "NaturalQuestions (open-book) - F1 (Robustness)": "{\"description\": \"min=0.556, mean=0.556, max=0.556, sum=0.556 (1)\", \"tab\": \"Robustness\", \"score\": \"0.5559619230719722\"}", - "NaturalQuestions (closed-book) - F1 (Fairness)": "{\"description\": \"min=0.331, mean=0.331, max=0.331, sum=0.331 (1)\", \"tab\": \"Fairness\", \"score\": \"0.3309794595447127\"}", - "NaturalQuestions (open-book) - F1 (Fairness)": "{\"description\": \"min=0.559, mean=0.559, max=0.559, sum=0.559 (1)\", \"tab\": \"Fairness\", \"score\": \"0.5593911419045751\"}", - "NaturalQuestions (closed-book) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NaturalQuestions (open-book) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=112.127, mean=112.127, max=112.127, sum=112.127 (1)\", \"tab\": \"General information\", \"score\": \"112.127\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=16.241, mean=16.241, max=16.241, sum=16.241 (1)\", \"tab\": \"General information\", \"score\": \"16.241\"}", - "NaturalQuestions (closed-book) - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.887, mean=4.887, max=4.887, sum=4.887 (1)\", \"tab\": \"General information\", \"score\": \"4.887\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.019, mean=0.019, max=0.019, sum=0.019 (1)\", \"tab\": \"General information\", \"score\": \"0.019\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1590.821, mean=1590.821, max=1590.821, sum=1590.821 (1)\", \"tab\": \"General information\", \"score\": \"1590.821\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=12.998, mean=12.998, max=12.998, sum=12.998 (1)\", \"tab\": \"General information\", \"score\": \"12.998\"}", - "NaturalQuestions (open-book) - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NaturalQuestions (closed-book) - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - Stereotypes (gender)": "{\"description\": \"min=0.353, mean=0.353, max=0.353, sum=0.353 (1)\", \"tab\": \"Bias\", \"score\": \"0.35333333333333333\"}", - "NaturalQuestions (closed-book) - Representation (race)": "{\"description\": \"min=0.364, mean=0.364, max=0.364, sum=0.364 (1)\", \"tab\": \"Bias\", \"score\": \"0.3643410852713178\"}", - "NaturalQuestions (closed-book) - Representation (gender)": "{\"description\": \"min=0.167, mean=0.167, max=0.167, sum=0.167 (1)\", \"tab\": \"Bias\", \"score\": \"0.16666666666666669\"}", - "NaturalQuestions (open-book) - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=0.667 (1)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "NaturalQuestions (open-book) - Stereotypes (gender)": "{\"description\": \"min=0.5, mean=0.5, max=0.5, sum=0.5 (1)\", \"tab\": \"Bias\", \"score\": \"0.5\"}", - "NaturalQuestions (open-book) - Representation (race)": "{\"description\": \"min=0.408, mean=0.408, max=0.408, sum=0.408 (1)\", \"tab\": \"Bias\", \"score\": \"0.4083885209713024\"}", - "NaturalQuestions (open-book) - Representation (gender)": "{\"description\": \"min=0.236, mean=0.236, max=0.236, sum=0.236 (1)\", \"tab\": \"Bias\", \"score\": \"0.23584905660377362\"}", - "NaturalQuestions (closed-book) - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}", - "NaturalQuestions (open-book) - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "QuAC", - "source_data": { - "dataset_name": "QuAC", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on QuAC", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.512, - "details": { - "description": "min=0.512, mean=0.512, max=0.512, sum=0.512 (1)", - "tab": "Accuracy", - "QuAC - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "QuAC - F1 (Robustness)": "{\"description\": \"min=0.411, mean=0.411, max=0.411, sum=0.411 (1)\", \"tab\": \"Robustness\", \"score\": \"0.41122249859183385\"}", - "QuAC - F1 (Fairness)": "{\"description\": \"min=0.417, mean=0.417, max=0.417, sum=0.417 (1)\", \"tab\": \"Fairness\", \"score\": \"0.4167691534016683\"}", - "QuAC - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "QuAC - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "QuAC - # train": "{\"description\": \"min=3.871, mean=3.871, max=3.871, sum=3.871 (1)\", \"tab\": \"General information\", \"score\": \"3.871\"}", - "QuAC - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "QuAC - # prompt tokens": "{\"description\": \"min=3461.981, mean=3461.981, max=3461.981, sum=3461.981 (1)\", \"tab\": \"General information\", \"score\": \"3461.981\"}", - "QuAC - # output tokens": "{\"description\": \"min=23.136, mean=23.136, max=23.136, sum=23.136 (1)\", \"tab\": \"General information\", \"score\": \"23.136\"}", - "QuAC - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "QuAC - Stereotypes (race)": "{\"description\": \"min=0.639, mean=0.639, max=0.639, sum=0.639 (1)\", \"tab\": \"Bias\", \"score\": \"0.638888888888889\"}", - "QuAC - Stereotypes (gender)": "{\"description\": \"min=0.403, mean=0.403, max=0.403, sum=0.403 (1)\", \"tab\": \"Bias\", \"score\": \"0.40322916666666675\"}", - "QuAC - Representation (race)": "{\"description\": \"min=0.436, mean=0.436, max=0.436, sum=0.436 (1)\", \"tab\": \"Bias\", \"score\": \"0.43589743589743585\"}", - "QuAC - Representation (gender)": "{\"description\": \"min=0.229, mean=0.229, max=0.229, sum=0.229 (1)\", \"tab\": \"Bias\", \"score\": \"0.22941176470588232\"}", - "QuAC - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "HellaSwag", - "source_data": { - "dataset_name": "HellaSwag", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on HellaSwag", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "HellaSwag - ECE (10-bin)": "{\"description\": \"No matching runs\", \"tab\": \"Calibration\", \"score\": \"\"}", - "HellaSwag - EM (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "HellaSwag - EM (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "HellaSwag - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "HellaSwag - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "OpenbookQA - ECE (10-bin)": "{\"description\": \"No matching runs\", \"tab\": \"Calibration\", \"score\": \"\"}", - "OpenbookQA - EM (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "OpenbookQA - EM (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "OpenbookQA - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "OpenbookQA - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "TruthfulQA", - "source_data": { - "dataset_name": "TruthfulQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on TruthfulQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.609, - "details": { - "description": "min=0.609, mean=0.609, max=0.609, sum=0.609 (1)", - "tab": "Accuracy", - "TruthfulQA - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "TruthfulQA - EM (Robustness)": "{\"description\": \"min=0.566, mean=0.566, max=0.566, sum=0.566 (1)\", \"tab\": \"Robustness\", \"score\": \"0.5657492354740061\"}", - "TruthfulQA - EM (Fairness)": "{\"description\": \"min=0.514, mean=0.514, max=0.514, sum=0.514 (1)\", \"tab\": \"Fairness\", \"score\": \"0.5137614678899083\"}", - "TruthfulQA - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "TruthfulQA - # eval": "{\"description\": \"min=654, mean=654, max=654, sum=654 (1)\", \"tab\": \"General information\", \"score\": \"654.0\"}", - "TruthfulQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "TruthfulQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "TruthfulQA - # prompt tokens": "{\"description\": \"min=464.434, mean=464.434, max=464.434, sum=464.434 (1)\", \"tab\": \"General information\", \"score\": \"464.434250764526\"}", - "TruthfulQA - # output tokens": "{\"description\": \"min=1.047, mean=1.047, max=1.047, sum=1.047 (1)\", \"tab\": \"General information\", \"score\": \"1.047400611620795\"}", - "TruthfulQA - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MS MARCO (TREC)", - "source_data": { - "dataset_name": "MS MARCO (TREC)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "NDCG@10 on MS MARCO (TREC)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "MS MARCO (regular) - RR@10 (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "MS MARCO (TREC) - NDCG@10 (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "MS MARCO (regular) - RR@10 (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "MS MARCO (TREC) - NDCG@10 (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "MS MARCO (regular) - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MS MARCO (TREC) - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MS MARCO (regular) - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "MS MARCO (TREC) - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CNN/DailyMail", - "source_data": { - "dataset_name": "CNN/DailyMail", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on CNN/DailyMail", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "CNN/DailyMail - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "CNN/DailyMail - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "CNN/DailyMail - SummaC": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - QAFactEval": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - BERTScore (F1)": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - Coverage": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - Density": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - Compression": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-faithfulness": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-relevance": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-coherence": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "XSUM", - "source_data": { - "dataset_name": "XSUM", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on XSUM", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "XSUM - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "XSUM - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "XSUM - SummaC": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - QAFactEval": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - BERTScore (F1)": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - Coverage": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - Density": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - Compression": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-faithfulness": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-relevance": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-coherence": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "IMDB", - "source_data": { - "dataset_name": "IMDB", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on IMDB", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.899, - "details": { - "description": "min=0.899, mean=0.899, max=0.899, sum=0.899 (1)", - "tab": "Accuracy", - "IMDB - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "IMDB - EM (Robustness)": "{\"description\": \"min=0.857, mean=0.857, max=0.857, sum=0.857 (1)\", \"tab\": \"Robustness\", \"score\": \"0.857\"}", - "IMDB - EM (Fairness)": "{\"description\": \"min=0.844, mean=0.844, max=0.844, sum=0.844 (1)\", \"tab\": \"Fairness\", \"score\": \"0.844\"}", - "IMDB - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "IMDB - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "IMDB - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "IMDB - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IMDB - # prompt tokens": "{\"description\": \"min=2543.665, mean=2543.665, max=2543.665, sum=2543.665 (1)\", \"tab\": \"General information\", \"score\": \"2543.665\"}", - "IMDB - # output tokens": "{\"description\": \"min=1.006, mean=1.006, max=1.006, sum=1.006 (1)\", \"tab\": \"General information\", \"score\": \"1.006\"}", - "IMDB - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "IMDB - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CivilComments", - "source_data": { - "dataset_name": "CivilComments", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on CivilComments", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.674, - "details": { - "description": "min=0.528, mean=0.674, max=0.824, sum=12.134 (18)", - "tab": "Accuracy", - "CivilComments - ECE (10-bin)": "{\"description\": \"9 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "CivilComments - EM (Robustness)": "{\"description\": \"min=0.394, mean=0.605, max=0.824, sum=10.882 (18)\", \"tab\": \"Robustness\", \"score\": \"0.6045521523734413\"}", - "CivilComments - EM (Fairness)": "{\"description\": \"min=0.024, mean=0.422, max=0.824, sum=7.597 (18)\", \"tab\": \"Fairness\", \"score\": \"0.4220761773099496\"}", - "CivilComments - Denoised inference time (s)": "{\"description\": \"9 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "CivilComments - # eval": "{\"description\": \"min=74, mean=371.556, max=683, sum=6688 (18)\", \"tab\": \"General information\", \"score\": \"371.55555555555554\"}", - "CivilComments - # train": "{\"description\": \"min=5, mean=5, max=5, sum=90 (18)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CivilComments - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (18)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CivilComments - # prompt tokens": "{\"description\": \"min=333.915, mean=733.362, max=1226.723, sum=13200.513 (18)\", \"tab\": \"General information\", \"score\": \"733.3618295565135\"}", - "CivilComments - # output tokens": "{\"description\": \"min=1, mean=1.023, max=1.103, sum=18.406 (18)\", \"tab\": \"General information\", \"score\": \"1.0225713328901465\"}", - "CivilComments - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=18 (18)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "CivilComments - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Toxic fraction": "{\"description\": \"9 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "RAFT", - "source_data": { - "dataset_name": "RAFT", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on RAFT", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.768, - "details": { - "description": "min=0.3, mean=0.768, max=0.975, sum=8.45 (11)", - "tab": "Accuracy", - "RAFT - ECE (10-bin)": "{\"description\": \"11 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "RAFT - EM (Robustness)": "{\"description\": \"min=0.2, mean=0.705, max=0.975, sum=7.75 (11)\", \"tab\": \"Robustness\", \"score\": \"0.7045454545454546\"}", - "RAFT - EM (Fairness)": "{\"description\": \"min=0.025, mean=0.689, max=0.975, sum=7.575 (11)\", \"tab\": \"Fairness\", \"score\": \"0.6886363636363636\"}", - "RAFT - Denoised inference time (s)": "{\"description\": \"11 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "RAFT - # eval": "{\"description\": \"min=40, mean=40, max=40, sum=440 (11)\", \"tab\": \"General information\", \"score\": \"40.0\"}", - "RAFT - # train": "{\"description\": \"min=3, mean=4.818, max=5, sum=53 (11)\", \"tab\": \"General information\", \"score\": \"4.818181818181818\"}", - "RAFT - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (11)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "RAFT - # prompt tokens": "{\"description\": \"min=252.275, mean=1002.239, max=3545.1, sum=11024.625 (11)\", \"tab\": \"General information\", \"score\": \"1002.2386363636365\"}", - "RAFT - # output tokens": "{\"description\": \"min=1.325, mean=2.982, max=5, sum=32.8 (11)\", \"tab\": \"General information\", \"score\": \"2.9818181818181815\"}", - "RAFT - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=11 (11)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "RAFT - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Toxic fraction": "{\"description\": \"11 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/openai_gpt-3.5-turbo-0613.json b/data/models/openai_gpt-3.5-turbo-0613.json deleted file mode 100644 index c24676b03a30239bea93de7207f3f13985ab4587..0000000000000000000000000000000000000000 --- a/data/models/openai_gpt-3.5-turbo-0613.json +++ /dev/null @@ -1,2792 +0,0 @@ -{ - "model_info": { - "name": "GPT-3.5 Turbo 0613", - "id": "openai/gpt-3.5-turbo-0613", - "developer": "OpenAI", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_classic/openai_gpt-3.5-turbo-0613/1774096308.339228", - "retrieved_timestamp": "1774096308.339228", - "source_metadata": { - "source_name": "helm_classic", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_classic", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_classic", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperform on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.783, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Calibration": "{\"description\": \"\", \"tab\": \"Calibration\", \"score\": \"\"}", - "Mean win rate - Robustness": "{\"description\": \"\", \"tab\": \"Robustness\", \"score\": \"0.7622144522144523\"}", - "Mean win rate - Fairness": "{\"description\": \"\", \"tab\": \"Fairness\", \"score\": \"0.7175058275058275\"}", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}", - "Mean win rate - Bias": "{\"description\": \"\", \"tab\": \"Bias\", \"score\": \"0.5232317557148765\"}", - "Mean win rate - Toxicity": "{\"description\": \"\", \"tab\": \"Toxicity\", \"score\": \"0.7166083916083916\"}", - "Mean win rate - Summarization metrics": "{\"description\": \"\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.391, - "details": { - "description": "min=0.2, mean=0.391, max=0.73, sum=1.955 (5)", - "tab": "Accuracy", - "MMLU - ECE (10-bin)": "{\"description\": \"5 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "MMLU - EM (Robustness)": "{\"description\": \"min=0.1, mean=0.262, max=0.49, sum=1.312 (5)\", \"tab\": \"Robustness\", \"score\": \"0.2623859649122807\"}", - "MMLU - EM (Fairness)": "{\"description\": \"min=0.12, mean=0.313, max=0.66, sum=1.566 (5)\", \"tab\": \"Fairness\", \"score\": \"0.31312280701754386\"}", - "MMLU - Denoised inference time (s)": "{\"description\": \"5 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=366.44, mean=460.72, max=607.43, sum=2303.6 (5)\", \"tab\": \"General information\", \"score\": \"460.71996491228066\"}", - "MMLU - # output tokens": "{\"description\": \"min=1.19, mean=1.371, max=1.61, sum=6.857 (5)\", \"tab\": \"General information\", \"score\": \"1.3714035087719298\"}", - "MMLU - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "BoolQ", - "source_data": { - "dataset_name": "BoolQ", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on BoolQ", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.87, - "details": { - "description": "min=0.87, mean=0.87, max=0.87, sum=0.87 (1)", - "tab": "Accuracy", - "BoolQ - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "BoolQ - EM (Robustness)": "{\"description\": \"min=0.845, mean=0.845, max=0.845, sum=0.845 (1)\", \"tab\": \"Robustness\", \"score\": \"0.845\"}", - "BoolQ - EM (Fairness)": "{\"description\": \"min=0.817, mean=0.817, max=0.817, sum=0.817 (1)\", \"tab\": \"Fairness\", \"score\": \"0.817\"}", - "BoolQ - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "BoolQ - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "BoolQ - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "BoolQ - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "BoolQ - # prompt tokens": "{\"description\": \"min=1220.329, mean=1220.329, max=1220.329, sum=1220.329 (1)\", \"tab\": \"General information\", \"score\": \"1220.329\"}", - "BoolQ - # output tokens": "{\"description\": \"min=1.057, mean=1.057, max=1.057, sum=1.057 (1)\", \"tab\": \"General information\", \"score\": \"1.057\"}", - "BoolQ - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "BoolQ - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.625, - "details": { - "description": "min=0.625, mean=0.625, max=0.625, sum=0.625 (1)", - "tab": "Accuracy", - "NarrativeQA - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "NarrativeQA - F1 (Robustness)": "{\"description\": \"min=0.566, mean=0.566, max=0.566, sum=0.566 (1)\", \"tab\": \"Robustness\", \"score\": \"0.5658549915417233\"}", - "NarrativeQA - F1 (Fairness)": "{\"description\": \"min=0.547, mean=0.547, max=0.547, sum=0.547 (1)\", \"tab\": \"Fairness\", \"score\": \"0.546599991762967\"}", - "NarrativeQA - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=4.966, mean=4.966, max=4.966, sum=4.966 (1)\", \"tab\": \"General information\", \"score\": \"4.966197183098592\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=3443.349, mean=3443.349, max=3443.349, sum=3443.349 (1)\", \"tab\": \"General information\", \"score\": \"3443.349295774648\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=12.194, mean=12.194, max=12.194, sum=12.194 (1)\", \"tab\": \"General information\", \"score\": \"12.194366197183099\"}", - "NarrativeQA - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NarrativeQA - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=0.667 (1)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666667\"}", - "NarrativeQA - Stereotypes (gender)": "{\"description\": \"min=0.455, mean=0.455, max=0.455, sum=0.455 (1)\", \"tab\": \"Bias\", \"score\": \"0.45454545454545453\"}", - "NarrativeQA - Representation (race)": "{\"description\": \"min=0.429, mean=0.429, max=0.429, sum=0.429 (1)\", \"tab\": \"Bias\", \"score\": \"0.42857142857142855\"}", - "NarrativeQA - Representation (gender)": "{\"description\": \"min=0.169, mean=0.169, max=0.169, sum=0.169 (1)\", \"tab\": \"Bias\", \"score\": \"0.16860465116279072\"}", - "NarrativeQA - Toxic fraction": "{\"description\": \"min=0.011, mean=0.011, max=0.011, sum=0.011 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.011267605633802818\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (open-book)", - "source_data": { - "dataset_name": "NaturalQuestions (open-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (open-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.675, - "details": { - "description": "min=0.675, mean=0.675, max=0.675, sum=0.675 (1)", - "tab": "Accuracy", - "NaturalQuestions (closed-book) - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "NaturalQuestions (open-book) - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - F1 (Robustness)": "{\"description\": \"min=0.284, mean=0.284, max=0.284, sum=0.284 (1)\", \"tab\": \"Robustness\", \"score\": \"0.28373438775512194\"}", - "NaturalQuestions (open-book) - F1 (Robustness)": "{\"description\": \"min=0.606, mean=0.606, max=0.606, sum=0.606 (1)\", \"tab\": \"Robustness\", \"score\": \"0.6060594363127481\"}", - "NaturalQuestions (closed-book) - F1 (Fairness)": "{\"description\": \"min=0.287, mean=0.287, max=0.287, sum=0.287 (1)\", \"tab\": \"Fairness\", \"score\": \"0.2871379631388369\"}", - "NaturalQuestions (open-book) - F1 (Fairness)": "{\"description\": \"min=0.627, mean=0.627, max=0.627, sum=0.627 (1)\", \"tab\": \"Fairness\", \"score\": \"0.6270354958497198\"}", - "NaturalQuestions (closed-book) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NaturalQuestions (open-book) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=112.127, mean=112.127, max=112.127, sum=112.127 (1)\", \"tab\": \"General information\", \"score\": \"112.127\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=18.876, mean=18.876, max=18.876, sum=18.876 (1)\", \"tab\": \"General information\", \"score\": \"18.876\"}", - "NaturalQuestions (closed-book) - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.887, mean=4.887, max=4.887, sum=4.887 (1)\", \"tab\": \"General information\", \"score\": \"4.887\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.019, mean=0.019, max=0.019, sum=0.019 (1)\", \"tab\": \"General information\", \"score\": \"0.019\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1590.821, mean=1590.821, max=1590.821, sum=1590.821 (1)\", \"tab\": \"General information\", \"score\": \"1590.821\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=11.901, mean=11.901, max=11.901, sum=11.901 (1)\", \"tab\": \"General information\", \"score\": \"11.901\"}", - "NaturalQuestions (open-book) - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NaturalQuestions (closed-book) - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=0.667 (1)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666667\"}", - "NaturalQuestions (closed-book) - Stereotypes (gender)": "{\"description\": \"min=0.5, mean=0.5, max=0.5, sum=0.5 (1)\", \"tab\": \"Bias\", \"score\": \"0.5\"}", - "NaturalQuestions (closed-book) - Representation (race)": "{\"description\": \"min=0.382, mean=0.382, max=0.382, sum=0.382 (1)\", \"tab\": \"Bias\", \"score\": \"0.38211382113821135\"}", - "NaturalQuestions (closed-book) - Representation (gender)": "{\"description\": \"min=0.104, mean=0.104, max=0.104, sum=0.104 (1)\", \"tab\": \"Bias\", \"score\": \"0.10377358490566038\"}", - "NaturalQuestions (open-book) - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=0.667 (1)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666667\"}", - "NaturalQuestions (open-book) - Stereotypes (gender)": "{\"description\": \"min=0.5, mean=0.5, max=0.5, sum=0.5 (1)\", \"tab\": \"Bias\", \"score\": \"0.5\"}", - "NaturalQuestions (open-book) - Representation (race)": "{\"description\": \"min=0.415, mean=0.415, max=0.415, sum=0.415 (1)\", \"tab\": \"Bias\", \"score\": \"0.41463414634146334\"}", - "NaturalQuestions (open-book) - Representation (gender)": "{\"description\": \"min=0.233, mean=0.233, max=0.233, sum=0.233 (1)\", \"tab\": \"Bias\", \"score\": \"0.23333333333333336\"}", - "NaturalQuestions (closed-book) - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}", - "NaturalQuestions (open-book) - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "QuAC", - "source_data": { - "dataset_name": "QuAC", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on QuAC", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.485, - "details": { - "description": "min=0.485, mean=0.485, max=0.485, sum=0.485 (1)", - "tab": "Accuracy", - "QuAC - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "QuAC - F1 (Robustness)": "{\"description\": \"min=0.371, mean=0.371, max=0.371, sum=0.371 (1)\", \"tab\": \"Robustness\", \"score\": \"0.3712446607257685\"}", - "QuAC - F1 (Fairness)": "{\"description\": \"min=0.398, mean=0.398, max=0.398, sum=0.398 (1)\", \"tab\": \"Fairness\", \"score\": \"0.3977545370248786\"}", - "QuAC - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "QuAC - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "QuAC - # train": "{\"description\": \"min=3.871, mean=3.871, max=3.871, sum=3.871 (1)\", \"tab\": \"General information\", \"score\": \"3.871\"}", - "QuAC - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "QuAC - # prompt tokens": "{\"description\": \"min=3461.981, mean=3461.981, max=3461.981, sum=3461.981 (1)\", \"tab\": \"General information\", \"score\": \"3461.981\"}", - "QuAC - # output tokens": "{\"description\": \"min=25.691, mean=25.691, max=25.691, sum=25.691 (1)\", \"tab\": \"General information\", \"score\": \"25.691\"}", - "QuAC - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "QuAC - Stereotypes (race)": "{\"description\": \"min=0.589, mean=0.589, max=0.589, sum=0.589 (1)\", \"tab\": \"Bias\", \"score\": \"0.5889724310776943\"}", - "QuAC - Stereotypes (gender)": "{\"description\": \"min=0.403, mean=0.403, max=0.403, sum=0.403 (1)\", \"tab\": \"Bias\", \"score\": \"0.4030096483037659\"}", - "QuAC - Representation (race)": "{\"description\": \"min=0.378, mean=0.378, max=0.378, sum=0.378 (1)\", \"tab\": \"Bias\", \"score\": \"0.3782051282051282\"}", - "QuAC - Representation (gender)": "{\"description\": \"min=0.223, mean=0.223, max=0.223, sum=0.223 (1)\", \"tab\": \"Bias\", \"score\": \"0.22334293948126804\"}", - "QuAC - Toxic fraction": "{\"description\": \"min=0.001, mean=0.001, max=0.001, sum=0.001 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.001\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "HellaSwag", - "source_data": { - "dataset_name": "HellaSwag", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on HellaSwag", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "HellaSwag - ECE (10-bin)": "{\"description\": \"No matching runs\", \"tab\": \"Calibration\", \"score\": \"\"}", - "HellaSwag - EM (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "HellaSwag - EM (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "HellaSwag - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "HellaSwag - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "OpenbookQA - ECE (10-bin)": "{\"description\": \"No matching runs\", \"tab\": \"Calibration\", \"score\": \"\"}", - "OpenbookQA - EM (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "OpenbookQA - EM (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "OpenbookQA - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "OpenbookQA - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "TruthfulQA", - "source_data": { - "dataset_name": "TruthfulQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on TruthfulQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.339, - "details": { - "description": "min=0.339, mean=0.339, max=0.339, sum=0.339 (1)", - "tab": "Accuracy", - "TruthfulQA - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "TruthfulQA - EM (Robustness)": "{\"description\": \"min=0.187, mean=0.187, max=0.187, sum=0.187 (1)\", \"tab\": \"Robustness\", \"score\": \"0.18654434250764526\"}", - "TruthfulQA - EM (Fairness)": "{\"description\": \"min=0.255, mean=0.255, max=0.255, sum=0.255 (1)\", \"tab\": \"Fairness\", \"score\": \"0.25535168195718655\"}", - "TruthfulQA - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "TruthfulQA - # eval": "{\"description\": \"min=654, mean=654, max=654, sum=654 (1)\", \"tab\": \"General information\", \"score\": \"654.0\"}", - "TruthfulQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "TruthfulQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "TruthfulQA - # prompt tokens": "{\"description\": \"min=464.434, mean=464.434, max=464.434, sum=464.434 (1)\", \"tab\": \"General information\", \"score\": \"464.434250764526\"}", - "TruthfulQA - # output tokens": "{\"description\": \"min=1.517, mean=1.517, max=1.517, sum=1.517 (1)\", \"tab\": \"General information\", \"score\": \"1.5168195718654434\"}", - "TruthfulQA - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MS MARCO (TREC)", - "source_data": { - "dataset_name": "MS MARCO (TREC)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "NDCG@10 on MS MARCO (TREC)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "MS MARCO (regular) - RR@10 (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "MS MARCO (TREC) - NDCG@10 (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "MS MARCO (regular) - RR@10 (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "MS MARCO (TREC) - NDCG@10 (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "MS MARCO (regular) - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MS MARCO (TREC) - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MS MARCO (regular) - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "MS MARCO (TREC) - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CNN/DailyMail", - "source_data": { - "dataset_name": "CNN/DailyMail", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on CNN/DailyMail", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "CNN/DailyMail - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "CNN/DailyMail - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "CNN/DailyMail - SummaC": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - QAFactEval": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - BERTScore (F1)": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - Coverage": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - Density": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - Compression": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-faithfulness": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-relevance": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-coherence": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "XSUM", - "source_data": { - "dataset_name": "XSUM", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on XSUM", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "XSUM - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "XSUM - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "XSUM - SummaC": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - QAFactEval": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - BERTScore (F1)": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - Coverage": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - Density": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - Compression": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-faithfulness": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-relevance": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-coherence": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "IMDB", - "source_data": { - "dataset_name": "IMDB", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on IMDB", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.943, - "details": { - "description": "min=0.943, mean=0.943, max=0.943, sum=0.943 (1)", - "tab": "Accuracy", - "IMDB - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "IMDB - EM (Robustness)": "{\"description\": \"min=0.916, mean=0.916, max=0.916, sum=0.916 (1)\", \"tab\": \"Robustness\", \"score\": \"0.916\"}", - "IMDB - EM (Fairness)": "{\"description\": \"min=0.912, mean=0.912, max=0.912, sum=0.912 (1)\", \"tab\": \"Fairness\", \"score\": \"0.912\"}", - "IMDB - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "IMDB - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "IMDB - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "IMDB - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IMDB - # prompt tokens": "{\"description\": \"min=2543.665, mean=2543.665, max=2543.665, sum=2543.665 (1)\", \"tab\": \"General information\", \"score\": \"2543.665\"}", - "IMDB - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "IMDB - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "IMDB - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CivilComments", - "source_data": { - "dataset_name": "CivilComments", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on CivilComments", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.696, - "details": { - "description": "min=0.48, mean=0.696, max=0.874, sum=12.534 (18)", - "tab": "Accuracy", - "CivilComments - ECE (10-bin)": "{\"description\": \"9 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "CivilComments - EM (Robustness)": "{\"description\": \"min=0.206, mean=0.564, max=0.863, sum=10.15 (18)\", \"tab\": \"Robustness\", \"score\": \"0.5638779146224463\"}", - "CivilComments - EM (Fairness)": "{\"description\": \"min=0.133, mean=0.525, max=0.863, sum=9.458 (18)\", \"tab\": \"Fairness\", \"score\": \"0.5254285459217098\"}", - "CivilComments - Denoised inference time (s)": "{\"description\": \"9 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "CivilComments - # eval": "{\"description\": \"min=74, mean=371.556, max=683, sum=6688 (18)\", \"tab\": \"General information\", \"score\": \"371.55555555555554\"}", - "CivilComments - # train": "{\"description\": \"min=5, mean=5, max=5, sum=90 (18)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CivilComments - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (18)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CivilComments - # prompt tokens": "{\"description\": \"min=333.915, mean=733.362, max=1226.723, sum=13200.513 (18)\", \"tab\": \"General information\", \"score\": \"733.3618295565135\"}", - "CivilComments - # output tokens": "{\"description\": \"min=1, mean=1.001, max=1.01, sum=18.025 (18)\", \"tab\": \"General information\", \"score\": \"1.0013947024944874\"}", - "CivilComments - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=18 (18)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "CivilComments - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Toxic fraction": "{\"description\": \"9 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "RAFT", - "source_data": { - "dataset_name": "RAFT", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on RAFT", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.748, - "details": { - "description": "min=0.275, mean=0.748, max=0.95, sum=8.225 (11)", - "tab": "Accuracy", - "RAFT - ECE (10-bin)": "{\"description\": \"11 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "RAFT - EM (Robustness)": "{\"description\": \"min=0.15, mean=0.677, max=0.95, sum=7.45 (11)\", \"tab\": \"Robustness\", \"score\": \"0.6772727272727272\"}", - "RAFT - EM (Fairness)": "{\"description\": \"min=0.2, mean=0.641, max=0.95, sum=7.05 (11)\", \"tab\": \"Fairness\", \"score\": \"0.640909090909091\"}", - "RAFT - Denoised inference time (s)": "{\"description\": \"11 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "RAFT - # eval": "{\"description\": \"min=40, mean=40, max=40, sum=440 (11)\", \"tab\": \"General information\", \"score\": \"40.0\"}", - "RAFT - # train": "{\"description\": \"min=3, mean=4.818, max=5, sum=53 (11)\", \"tab\": \"General information\", \"score\": \"4.818181818181818\"}", - "RAFT - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (11)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "RAFT - # prompt tokens": "{\"description\": \"min=252.275, mean=1002.239, max=3545.1, sum=11024.625 (11)\", \"tab\": \"General information\", \"score\": \"1002.2386363636365\"}", - "RAFT - # output tokens": "{\"description\": \"min=1.275, mean=2.955, max=5.05, sum=32.5 (11)\", \"tab\": \"General information\", \"score\": \"2.9545454545454546\"}", - "RAFT - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=11 (11)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "RAFT - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Toxic fraction": "{\"description\": \"11 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_instruct/openai_gpt-3.5-turbo-0613/1774096309.537868", - "retrieved_timestamp": "1774096309.537868", - "source_metadata": { - "source_name": "helm_instruct", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_instruct", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_instruct", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/instruct/benchmark_output/releases/v1.0.0/groups/instruction_following.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperform on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.689, - "details": { - "description": "", - "tab": "Instruction Following" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "Anthropic RLHF dataset", - "source_data": { - "dataset_name": "Anthropic RLHF dataset", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/instruct/benchmark_output/releases/v1.0.0/groups/instruction_following.json" - ] - }, - "metric_config": { - "evaluation_description": "Harmlessness on Anthropic RLHF dataset", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 5.0 - }, - "score_details": { - "score": 4.964, - "details": { - "description": "min=4.915, mean=4.964, max=5, sum=39.715 (8)", - "tab": "Instruction Following" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"hh\", \"hh\", \"hh\", \"hh\", \"red_team\", \"red_team\", \"red_team\", \"red_team\"]", - "evaluator": "[\"claude\", \"gpt4\", \"mturk\", \"scale\", \"claude\", \"gpt4\", \"mturk\", \"scale\"]" - } - } - }, - { - "evaluation_name": "Best ChatGPT Prompts", - "source_data": { - "dataset_name": "Best ChatGPT Prompts", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/instruct/benchmark_output/releases/v1.0.0/groups/instruction_following.json" - ] - }, - "metric_config": { - "evaluation_description": "Harmlessness on Best ChatGPT Prompts", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 5.0 - }, - "score_details": { - "score": 4.986, - "details": { - "description": "min=4.95, mean=4.986, max=5, sum=19.945 (4)", - "tab": "Instruction Following" - } - }, - "generation_config": { - "additional_details": { - "path": "\"src_helm_benchmark_scenarios_best_chatgpt_prompts.yaml\"", - "tags": "\"\"", - "evaluator": "[\"claude\", \"gpt4\", \"mturk\", \"scale\"]" - } - } - }, - { - "evaluation_name": "Koala test dataset", - "source_data": { - "dataset_name": "Koala test dataset", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/instruct/benchmark_output/releases/v1.0.0/groups/instruction_following.json" - ] - }, - "metric_config": { - "evaluation_description": "Harmlessness on Koala test dataset", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 5.0 - }, - "score_details": { - "score": 4.987, - "details": { - "description": "min=4.969, mean=4.987, max=5, sum=19.95 (4)", - "tab": "Instruction Following" - } - }, - "generation_config": { - "additional_details": { - "evaluator": "[\"claude\", \"gpt4\", \"mturk\", \"scale\"]" - } - } - }, - { - "evaluation_name": "Open Assistant", - "source_data": { - "dataset_name": "Open Assistant", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/instruct/benchmark_output/releases/v1.0.0/groups/instruction_following.json" - ] - }, - "metric_config": { - "evaluation_description": "Harmlessness on Open Assistant", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 5.0 - }, - "score_details": { - "score": 4.987, - "details": { - "description": "min=4.96, mean=4.987, max=5, sum=19.95 (4)", - "tab": "Instruction Following" - } - }, - "generation_config": { - "additional_details": { - "language": "\"en\"", - "evaluator": "[\"claude\", \"gpt4\", \"mturk\", \"scale\"]" - } - } - }, - { - "evaluation_name": "Self Instruct", - "source_data": { - "dataset_name": "Self Instruct", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/instruct/benchmark_output/releases/v1.0.0/groups/instruction_following.json" - ] - }, - "metric_config": { - "evaluation_description": "Harmlessness on Self Instruct", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 5.0 - }, - "score_details": { - "score": 4.99, - "details": { - "description": "min=4.97, mean=4.99, max=5, sum=19.96 (4)", - "tab": "Instruction Following" - } - }, - "generation_config": { - "additional_details": { - "evaluator": "[\"claude\", \"gpt4\", \"mturk\", \"scale\"]" - } - } - }, - { - "evaluation_name": "Vicuna", - "source_data": { - "dataset_name": "Vicuna", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/instruct/benchmark_output/releases/v1.0.0/groups/instruction_following.json" - ] - }, - "metric_config": { - "evaluation_description": "Harmlessness on Vicuna", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 5.0 - }, - "score_details": { - "score": 4.992, - "details": { - "description": "min=4.975, mean=4.992, max=5, sum=19.969 (4)", - "tab": "Instruction Following" - } - }, - "generation_config": { - "additional_details": { - "category": "\"all\"", - "evaluator": "[\"claude\", \"gpt4\", \"mturk\", \"scale\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_lite/openai_gpt-3.5-turbo-0613/1774096306.427425", - "retrieved_timestamp": "1774096306.427425", - "source_metadata": { - "source_name": "helm_lite", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_lite", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_lite", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.358, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.956641697877653\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.655, - "details": { - "description": "min=0.655, mean=0.655, max=0.655, sum=0.655 (1)", - "tab": "Accuracy", - "NarrativeQA - Observed inference time (s)": "{\"description\": \"min=0.381, mean=0.381, max=0.381, sum=0.381 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.3810261323418416\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=4.946, mean=4.946, max=4.946, sum=4.946 (1)\", \"tab\": \"General information\", \"score\": \"4.946478873239436\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=3493.662, mean=3493.662, max=3493.662, sum=3493.662 (1)\", \"tab\": \"General information\", \"score\": \"3493.6619718309857\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=9.91, mean=9.91, max=9.91, sum=9.91 (1)\", \"tab\": \"General information\", \"score\": \"9.909859154929578\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (closed-book)", - "source_data": { - "dataset_name": "NaturalQuestions (closed-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (closed-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.335, - "details": { - "description": "min=0.335, mean=0.335, max=0.335, sum=0.335 (1)", - "tab": "Accuracy", - "NaturalQuestions (open-book) - Observed inference time (s)": "{\"description\": \"min=0.305, mean=0.305, max=0.305, sum=0.305 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.30532183837890625\"}", - "NaturalQuestions (closed-book) - Observed inference time (s)": "{\"description\": \"min=0.221, mean=0.221, max=0.221, sum=0.221 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.22069251775741577\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.884, mean=4.884, max=4.884, sum=4.884 (1)\", \"tab\": \"General information\", \"score\": \"4.884\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.019, mean=0.019, max=0.019, sum=0.019 (1)\", \"tab\": \"General information\", \"score\": \"0.019\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1649.552, mean=1649.552, max=1649.552, sum=1649.552 (1)\", \"tab\": \"General information\", \"score\": \"1649.552\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=9.389, mean=9.389, max=9.389, sum=9.389 (1)\", \"tab\": \"General information\", \"score\": \"9.389\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=173.127, mean=173.127, max=173.127, sum=173.127 (1)\", \"tab\": \"General information\", \"score\": \"173.127\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=5.576, mean=5.576, max=5.576, sum=5.576 (1)\", \"tab\": \"General information\", \"score\": \"5.576\"}" - } - }, - "generation_config": { - "additional_details": { - "mode": "\"closedbook\"" - } - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.838, - "details": { - "description": "min=0.838, mean=0.838, max=0.838, sum=0.838 (1)", - "tab": "Accuracy", - "OpenbookQA - Observed inference time (s)": "{\"description\": \"min=0.172, mean=0.172, max=0.172, sum=0.172 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.17227248001098633\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=242.782, mean=242.782, max=242.782, sum=242.782 (1)\", \"tab\": \"General information\", \"score\": \"242.782\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "dataset": "\"openbookqa\"", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.614, - "details": { - "description": "min=0.38, mean=0.614, max=0.88, sum=3.07 (5)", - "tab": "Accuracy", - "MMLU - Observed inference time (s)": "{\"description\": \"min=0.171, mean=0.175, max=0.177, sum=0.875 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.1750619323630082\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=366.44, mean=460.72, max=607.43, sum=2303.6 (5)\", \"tab\": \"General information\", \"score\": \"460.71996491228066\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"college_chemistry\", \"computer_security\", \"econometrics\", \"us_foreign_policy\"]", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MATH", - "source_data": { - "dataset_name": "MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Equivalent (CoT) on MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.667, - "details": { - "description": "min=0.533, mean=0.667, max=0.826, sum=4.667 (7)", - "tab": "Accuracy", - "MATH - Observed inference time (s)": "{\"description\": \"min=0.741, mean=0.813, max=0.963, sum=5.69 (7)\", \"tab\": \"Efficiency\", \"score\": \"0.8128212395123947\"}", - "MATH - # eval": "{\"description\": \"min=30, mean=62.429, max=135, sum=437 (7)\", \"tab\": \"General information\", \"score\": \"62.42857142857143\"}", - "MATH - # train": "{\"description\": \"min=8, mean=8, max=8, sum=56 (7)\", \"tab\": \"General information\", \"score\": \"8.0\"}", - "MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MATH - # prompt tokens": "{\"description\": \"min=942.363, mean=1323.911, max=2258.577, sum=9267.376 (7)\", \"tab\": \"General information\", \"score\": \"1323.910874184069\"}", - "MATH - # output tokens": "{\"description\": \"min=53.5, mean=60.844, max=77.4, sum=425.908 (7)\", \"tab\": \"General information\", \"score\": \"60.844003793024605\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"algebra\", \"counting_and_probability\", \"geometry\", \"intermediate_algebra\", \"number_theory\", \"prealgebra\", \"precalculus\"]", - "level": "\"1\"", - "use_official_examples": "\"False\"", - "use_chain_of_thought": "\"True\"" - } - } - }, - { - "evaluation_name": "GSM8K", - "source_data": { - "dataset_name": "GSM8K", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on GSM8K", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.501, - "details": { - "description": "min=0.501, mean=0.501, max=0.501, sum=0.501 (1)", - "tab": "Accuracy", - "GSM8K - Observed inference time (s)": "{\"description\": \"min=0.898, mean=0.898, max=0.898, sum=0.898 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.8983073465824127\"}", - "GSM8K - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "GSM8K - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "GSM8K - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GSM8K - # prompt tokens": "{\"description\": \"min=1020.035, mean=1020.035, max=1020.035, sum=1020.035 (1)\", \"tab\": \"General information\", \"score\": \"1020.035\"}", - "GSM8K - # output tokens": "{\"description\": \"min=77.29, mean=77.29, max=77.29, sum=77.29 (1)\", \"tab\": \"General information\", \"score\": \"77.29\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "LegalBench", - "source_data": { - "dataset_name": "LegalBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on LegalBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.528, - "details": { - "description": "min=0.302, mean=0.528, max=0.747, sum=2.642 (5)", - "tab": "Accuracy", - "LegalBench - Observed inference time (s)": "{\"description\": \"min=0.178, mean=0.202, max=0.277, sum=1.011 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.20213919553681423\"}", - "LegalBench - # eval": "{\"description\": \"min=95, mean=409.4, max=1000, sum=2047 (5)\", \"tab\": \"General information\", \"score\": \"409.4\"}", - "LegalBench - # train": "{\"description\": \"min=2.09, mean=4.218, max=5, sum=21.09 (5)\", \"tab\": \"General information\", \"score\": \"4.21795918367347\"}", - "LegalBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "LegalBench - # prompt tokens": "{\"description\": \"min=253.442, mean=949.517, max=3254.159, sum=4747.586 (5)\", \"tab\": \"General information\", \"score\": \"949.5172570702738\"}", - "LegalBench - # output tokens": "{\"description\": \"min=1, mean=1.387, max=2.032, sum=6.934 (5)\", \"tab\": \"General information\", \"score\": \"1.3868394951957552\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"abercrombie\", \"corporate_lobbying\", \"function_of_decision_section\", \"international_citizenship_questions\", \"proa\"]" - } - } - }, - { - "evaluation_name": "MedQA", - "source_data": { - "dataset_name": "MedQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MedQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.622, - "details": { - "description": "min=0.622, mean=0.622, max=0.622, sum=0.622 (1)", - "tab": "Accuracy", - "MedQA - Observed inference time (s)": "{\"description\": \"min=0.194, mean=0.194, max=0.194, sum=0.194 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.19374941736755977\"}", - "MedQA - # eval": "{\"description\": \"min=503, mean=503, max=503, sum=503 (1)\", \"tab\": \"General information\", \"score\": \"503.0\"}", - "MedQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MedQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MedQA - # prompt tokens": "{\"description\": \"min=1020.414, mean=1020.414, max=1020.414, sum=1020.414 (1)\", \"tab\": \"General information\", \"score\": \"1020.4135188866799\"}", - "MedQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WMT 2014", - "source_data": { - "dataset_name": "WMT 2014", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "BLEU-4 on WMT 2014", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.187, - "details": { - "description": "min=0.1, mean=0.187, max=0.23, sum=0.937 (5)", - "tab": "Accuracy", - "WMT 2014 - Observed inference time (s)": "{\"description\": \"min=0.367, mean=0.394, max=0.409, sum=1.968 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.39351808213963385\"}", - "WMT 2014 - # eval": "{\"description\": \"min=503, mean=568.8, max=832, sum=2844 (5)\", \"tab\": \"General information\", \"score\": \"568.8\"}", - "WMT 2014 - # train": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "WMT 2014 - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WMT 2014 - # prompt tokens": "{\"description\": \"min=169.901, mean=193.043, max=213.185, sum=965.213 (5)\", \"tab\": \"General information\", \"score\": \"193.04258583116683\"}", - "WMT 2014 - # output tokens": "{\"description\": \"min=21.983, mean=25.038, max=26.352, sum=125.192 (5)\", \"tab\": \"General information\", \"score\": \"25.038384118366725\"}" - } - }, - "generation_config": { - "additional_details": { - "language_pair": "[\"cs-en\", \"de-en\", \"fr-en\", \"hi-en\", \"ru-en\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_mmlu/openai_gpt-3.5-turbo-0613/1774096312.00548", - "retrieved_timestamp": "1774096312.00548", - "source_metadata": { - "source_name": "helm_mmlu", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_mmlu", - "evaluation_results": [ - { - "evaluation_name": "MMLU All Subjects", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU All Subjects", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.689, - "details": { - "description": "min=0.33, mean=0.689, max=0.922, sum=78.524 (114)", - "tab": "Accuracy", - "MMLU All Subjects - Observed inference time (s)": "{\"description\": \"min=0.171, mean=0.411, max=0.659, sum=46.797 (114)\", \"tab\": \"Efficiency\", \"score\": \"0.41050392458578394\"}", - "MMLU All Subjects - # eval": "{\"description\": \"min=100, mean=246.351, max=1534, sum=28084 (114)\", \"tab\": \"General information\", \"score\": \"246.35087719298247\"}", - "MMLU All Subjects - # train": "{\"description\": \"min=5, mean=5, max=5, sum=570 (114)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU All Subjects - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU All Subjects - # prompt tokens": "{\"description\": \"min=268.561, mean=607.852, max=2791.073, sum=69295.086 (114)\", \"tab\": \"General information\", \"score\": \"607.851634217556\"}", - "MMLU All Subjects - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=114 (114)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_name": "Abstract Algebra", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Abstract Algebra", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.38, - "details": { - "description": "min=0.38, mean=0.38, max=0.38, sum=0.76 (2)", - "tab": "Accuracy", - "Abstract Algebra - Observed inference time (s)": "{\"description\": \"min=0.177, mean=0.177, max=0.177, sum=0.353 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.17670444011688233\"}", - "Abstract Algebra - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Abstract Algebra - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Abstract Algebra - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Abstract Algebra - # prompt tokens": "{\"description\": \"min=366.44, mean=366.44, max=366.44, sum=732.88 (2)\", \"tab\": \"General information\", \"score\": \"366.44\"}", - "Abstract Algebra - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"abstract_algebra\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_abstract_algebra\"" - } - } - }, - { - "evaluation_name": "Anatomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Anatomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.659, - "details": { - "description": "min=0.659, mean=0.659, max=0.659, sum=1.319 (2)", - "tab": "Accuracy", - "Anatomy - Observed inference time (s)": "{\"description\": \"min=0.448, mean=0.448, max=0.448, sum=0.896 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.448052688881203\"}", - "Anatomy - # eval": "{\"description\": \"min=135, mean=135, max=135, sum=270 (2)\", \"tab\": \"General information\", \"score\": \"135.0\"}", - "Anatomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Anatomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Anatomy - # prompt tokens": "{\"description\": \"min=346.978, mean=346.978, max=346.978, sum=693.956 (2)\", \"tab\": \"General information\", \"score\": \"346.97777777777776\"}", - "Anatomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"anatomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_anatomy\"" - } - } - }, - { - "evaluation_name": "College Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on College Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.461, - "details": { - "description": "min=0.461, mean=0.461, max=0.461, sum=0.922 (2)", - "tab": "Accuracy", - "College Chemistry - Observed inference time (s)": "{\"description\": \"min=0.174, mean=0.174, max=0.174, sum=0.349 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.17441444158554076\"}", - "College Biology - Observed inference time (s)": "{\"description\": \"min=0.435, mean=0.435, max=0.435, sum=0.871 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.43541959755950504\"}", - "College Computer Science - Observed inference time (s)": "{\"description\": \"min=0.508, mean=0.508, max=0.508, sum=1.015 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5075832653045654\"}", - "College Mathematics - Observed inference time (s)": "{\"description\": \"min=0.419, mean=0.419, max=0.419, sum=0.839 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.41928773641586303\"}", - "College Medicine - Observed inference time (s)": "{\"description\": \"min=0.389, mean=0.389, max=0.389, sum=0.777 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3885422951913293\"}", - "College Physics - Observed inference time (s)": "{\"description\": \"min=0.37, mean=0.37, max=0.37, sum=0.74 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3700263453464882\"}", - "College Chemistry - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Chemistry - # prompt tokens": "{\"description\": \"min=542.4, mean=542.4, max=542.4, sum=1084.8 (2)\", \"tab\": \"General information\", \"score\": \"542.4\"}", - "College Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Biology - # eval": "{\"description\": \"min=144, mean=144, max=144, sum=288 (2)\", \"tab\": \"General information\", \"score\": \"144.0\"}", - "College Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # prompt tokens": "{\"description\": \"min=466.917, mean=466.917, max=466.917, sum=933.833 (2)\", \"tab\": \"General information\", \"score\": \"466.9166666666667\"}", - "College Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # prompt tokens": "{\"description\": \"min=821.39, mean=821.39, max=821.39, sum=1642.78 (2)\", \"tab\": \"General information\", \"score\": \"821.39\"}", - "College Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Mathematics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # prompt tokens": "{\"description\": \"min=587.52, mean=587.52, max=587.52, sum=1175.04 (2)\", \"tab\": \"General information\", \"score\": \"587.52\"}", - "College Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Medicine - # eval": "{\"description\": \"min=173, mean=173, max=173, sum=346 (2)\", \"tab\": \"General information\", \"score\": \"173.0\"}", - "College Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # prompt tokens": "{\"description\": \"min=495.728, mean=495.728, max=495.728, sum=991.457 (2)\", \"tab\": \"General information\", \"score\": \"495.728323699422\"}", - "College Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Physics - # eval": "{\"description\": \"min=102, mean=102, max=102, sum=204 (2)\", \"tab\": \"General information\", \"score\": \"102.0\"}", - "College Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # prompt tokens": "{\"description\": \"min=496.608, mean=496.608, max=496.608, sum=993.216 (2)\", \"tab\": \"General information\", \"score\": \"496.6078431372549\"}", - "College Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"college_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_college_physics\"" - } - } - }, - { - "evaluation_name": "Computer Security", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Computer Security", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.81, - "details": { - "description": "min=0.81, mean=0.81, max=0.81, sum=1.62 (2)", - "tab": "Accuracy", - "Computer Security - Observed inference time (s)": "{\"description\": \"min=0.171, mean=0.171, max=0.171, sum=0.342 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.17102816104888915\"}", - "Computer Security - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Computer Security - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Computer Security - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Computer Security - # prompt tokens": "{\"description\": \"min=371.54, mean=371.54, max=371.54, sum=743.08 (2)\", \"tab\": \"General information\", \"score\": \"371.54\"}", - "Computer Security - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"computer_security\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_computer_security\"" - } - } - }, - { - "evaluation_name": "Econometrics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Econometrics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5, - "details": { - "description": "min=0.5, mean=0.5, max=0.5, sum=1 (2)", - "tab": "Accuracy", - "Econometrics - Observed inference time (s)": "{\"description\": \"min=0.176, mean=0.176, max=0.176, sum=0.353 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.1764866866563496\"}", - "Econometrics - # eval": "{\"description\": \"min=114, mean=114, max=114, sum=228 (2)\", \"tab\": \"General information\", \"score\": \"114.0\"}", - "Econometrics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Econometrics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Econometrics - # prompt tokens": "{\"description\": \"min=607.43, mean=607.43, max=607.43, sum=1214.86 (2)\", \"tab\": \"General information\", \"score\": \"607.4298245614035\"}", - "Econometrics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"econometrics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_econometrics\"" - } - } - }, - { - "evaluation_name": "Global Facts", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Global Facts", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.37, - "details": { - "description": "min=0.37, mean=0.37, max=0.37, sum=0.74 (2)", - "tab": "Accuracy", - "Global Facts - Observed inference time (s)": "{\"description\": \"min=0.439, mean=0.439, max=0.439, sum=0.879 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4393133974075317\"}", - "Global Facts - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Global Facts - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Global Facts - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Global Facts - # prompt tokens": "{\"description\": \"min=392.71, mean=392.71, max=392.71, sum=785.42 (2)\", \"tab\": \"General information\", \"score\": \"392.71\"}", - "Global Facts - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"global_facts\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_global_facts\"" - } - } - }, - { - "evaluation_name": "Jurisprudence", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Jurisprudence", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.806, - "details": { - "description": "min=0.806, mean=0.806, max=0.806, sum=1.611 (2)", - "tab": "Accuracy", - "Jurisprudence - Observed inference time (s)": "{\"description\": \"min=0.373, mean=0.373, max=0.373, sum=0.747 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.37349939346313477\"}", - "Jurisprudence - # eval": "{\"description\": \"min=108, mean=108, max=108, sum=216 (2)\", \"tab\": \"General information\", \"score\": \"108.0\"}", - "Jurisprudence - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Jurisprudence - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Jurisprudence - # prompt tokens": "{\"description\": \"min=387.639, mean=387.639, max=387.639, sum=775.278 (2)\", \"tab\": \"General information\", \"score\": \"387.6388888888889\"}", - "Jurisprudence - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"jurisprudence\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_jurisprudence\"" - } - } - }, - { - "evaluation_name": "Philosophy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Philosophy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.759, - "details": { - "description": "min=0.759, mean=0.759, max=0.759, sum=1.518 (2)", - "tab": "Accuracy", - "Philosophy - Observed inference time (s)": "{\"description\": \"min=0.382, mean=0.382, max=0.382, sum=0.763 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3817227730030415\"}", - "Philosophy - # eval": "{\"description\": \"min=311, mean=311, max=311, sum=622 (2)\", \"tab\": \"General information\", \"score\": \"311.0\"}", - "Philosophy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Philosophy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Philosophy - # prompt tokens": "{\"description\": \"min=322.084, mean=322.084, max=322.084, sum=644.167 (2)\", \"tab\": \"General information\", \"score\": \"322.08360128617363\"}", - "Philosophy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"philosophy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_philosophy\"" - } - } - }, - { - "evaluation_name": "Professional Psychology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Professional Psychology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.732, - "details": { - "description": "min=0.732, mean=0.732, max=0.732, sum=1.464 (2)", - "tab": "Accuracy", - "Professional Medicine - Observed inference time (s)": "{\"description\": \"min=0.485, mean=0.485, max=0.485, sum=0.969 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.48464199637665467\"}", - "Professional Accounting - Observed inference time (s)": "{\"description\": \"min=0.439, mean=0.439, max=0.439, sum=0.878 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4387922709715282\"}", - "Professional Law - Observed inference time (s)": "{\"description\": \"min=0.506, mean=0.506, max=0.506, sum=1.012 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5061173195012079\"}", - "Professional Psychology - Observed inference time (s)": "{\"description\": \"min=0.468, mean=0.468, max=0.468, sum=0.935 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4675601058536106\"}", - "Professional Medicine - # eval": "{\"description\": \"min=272, mean=272, max=272, sum=544 (2)\", \"tab\": \"General information\", \"score\": \"272.0\"}", - "Professional Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Medicine - # prompt tokens": "{\"description\": \"min=1087.585, mean=1087.585, max=1087.585, sum=2175.169 (2)\", \"tab\": \"General information\", \"score\": \"1087.5845588235295\"}", - "Professional Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Accounting - # eval": "{\"description\": \"min=282, mean=282, max=282, sum=564 (2)\", \"tab\": \"General information\", \"score\": \"282.0\"}", - "Professional Accounting - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Accounting - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # prompt tokens": "{\"description\": \"min=651.592, mean=651.592, max=651.592, sum=1303.184 (2)\", \"tab\": \"General information\", \"score\": \"651.5921985815603\"}", - "Professional Accounting - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Law - # eval": "{\"description\": \"min=1534, mean=1534, max=1534, sum=3068 (2)\", \"tab\": \"General information\", \"score\": \"1534.0\"}", - "Professional Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # prompt tokens": "{\"description\": \"min=1630.787, mean=1630.787, max=1630.787, sum=3261.574 (2)\", \"tab\": \"General information\", \"score\": \"1630.7868318122555\"}", - "Professional Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Psychology - # eval": "{\"description\": \"min=612, mean=612, max=612, sum=1224 (2)\", \"tab\": \"General information\", \"score\": \"612.0\"}", - "Professional Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # prompt tokens": "{\"description\": \"min=568.114, mean=568.114, max=568.114, sum=1136.229 (2)\", \"tab\": \"General information\", \"score\": \"568.1143790849674\"}", - "Professional Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"professional_psychology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_professional_psychology\"" - } - } - }, - { - "evaluation_name": "Us Foreign Policy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Us Foreign Policy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.88, - "details": { - "description": "min=0.88, mean=0.88, max=0.88, sum=1.76 (2)", - "tab": "Accuracy", - "Us Foreign Policy - Observed inference time (s)": "{\"description\": \"min=0.177, mean=0.177, max=0.177, sum=0.353 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.17667593240737914\"}", - "Us Foreign Policy - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Us Foreign Policy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Us Foreign Policy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Us Foreign Policy - # prompt tokens": "{\"description\": \"min=415.79, mean=415.79, max=415.79, sum=831.58 (2)\", \"tab\": \"General information\", \"score\": \"415.79\"}", - "Us Foreign Policy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"us_foreign_policy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_us_foreign_policy\"" - } - } - }, - { - "evaluation_name": "Astronomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Astronomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.763, - "details": { - "description": "min=0.763, mean=0.763, max=0.763, sum=1.526 (2)", - "tab": "Accuracy", - "Astronomy - Observed inference time (s)": "{\"description\": \"min=0.442, mean=0.442, max=0.442, sum=0.885 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.44235374111878245\"}", - "Astronomy - # eval": "{\"description\": \"min=152, mean=152, max=152, sum=304 (2)\", \"tab\": \"General information\", \"score\": \"152.0\"}", - "Astronomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Astronomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Astronomy - # prompt tokens": "{\"description\": \"min=572.691, mean=572.691, max=572.691, sum=1145.382 (2)\", \"tab\": \"General information\", \"score\": \"572.6907894736842\"}", - "Astronomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"astronomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_astronomy\"" - } - } - }, - { - "evaluation_name": "Business Ethics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Business Ethics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.75, - "details": { - "description": "min=0.75, mean=0.75, max=0.75, sum=1.5 (2)", - "tab": "Accuracy", - "Business Ethics - Observed inference time (s)": "{\"description\": \"min=0.573, mean=0.573, max=0.573, sum=1.147 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5733751010894775\"}", - "Business Ethics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Business Ethics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Business Ethics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Business Ethics - # prompt tokens": "{\"description\": \"min=562.52, mean=562.52, max=562.52, sum=1125.04 (2)\", \"tab\": \"General information\", \"score\": \"562.52\"}", - "Business Ethics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"business_ethics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_business_ethics\"" - } - } - }, - { - "evaluation_name": "Clinical Knowledge", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Clinical Knowledge", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.777, - "details": { - "description": "min=0.777, mean=0.777, max=0.777, sum=1.555 (2)", - "tab": "Accuracy", - "Clinical Knowledge - Observed inference time (s)": "{\"description\": \"min=0.418, mean=0.418, max=0.418, sum=0.837 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4183455800110439\"}", - "Clinical Knowledge - # eval": "{\"description\": \"min=265, mean=265, max=265, sum=530 (2)\", \"tab\": \"General information\", \"score\": \"265.0\"}", - "Clinical Knowledge - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Clinical Knowledge - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Clinical Knowledge - # prompt tokens": "{\"description\": \"min=390.947, mean=390.947, max=390.947, sum=781.894 (2)\", \"tab\": \"General information\", \"score\": \"390.94716981132075\"}", - "Clinical Knowledge - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"clinical_knowledge\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_clinical_knowledge\"" - } - } - }, - { - "evaluation_name": "Conceptual Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Conceptual Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.613, - "details": { - "description": "min=0.613, mean=0.613, max=0.613, sum=1.226 (2)", - "tab": "Accuracy", - "Conceptual Physics - Observed inference time (s)": "{\"description\": \"min=0.386, mean=0.386, max=0.386, sum=0.771 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3856722780998717\"}", - "Conceptual Physics - # eval": "{\"description\": \"min=235, mean=235, max=235, sum=470 (2)\", \"tab\": \"General information\", \"score\": \"235.0\"}", - "Conceptual Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Conceptual Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Conceptual Physics - # prompt tokens": "{\"description\": \"min=297.838, mean=297.838, max=297.838, sum=595.677 (2)\", \"tab\": \"General information\", \"score\": \"297.83829787234043\"}", - "Conceptual Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"conceptual_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_conceptual_physics\"" - } - } - }, - { - "evaluation_name": "Electrical Engineering", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Electrical Engineering", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.648, - "details": { - "description": "min=0.648, mean=0.648, max=0.648, sum=1.297 (2)", - "tab": "Accuracy", - "Electrical Engineering - Observed inference time (s)": "{\"description\": \"min=0.434, mean=0.434, max=0.434, sum=0.867 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.43367810249328614\"}", - "Electrical Engineering - # eval": "{\"description\": \"min=145, mean=145, max=145, sum=290 (2)\", \"tab\": \"General information\", \"score\": \"145.0\"}", - "Electrical Engineering - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Electrical Engineering - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Electrical Engineering - # prompt tokens": "{\"description\": \"min=433.641, mean=433.641, max=433.641, sum=867.283 (2)\", \"tab\": \"General information\", \"score\": \"433.6413793103448\"}", - "Electrical Engineering - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"electrical_engineering\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_electrical_engineering\"" - } - } - }, - { - "evaluation_name": "Elementary Mathematics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Elementary Mathematics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5, - "details": { - "description": "min=0.5, mean=0.5, max=0.5, sum=1 (2)", - "tab": "Accuracy", - "Elementary Mathematics - Observed inference time (s)": "{\"description\": \"min=0.386, mean=0.386, max=0.386, sum=0.771 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3857186824556381\"}", - "Elementary Mathematics - # eval": "{\"description\": \"min=378, mean=378, max=378, sum=756 (2)\", \"tab\": \"General information\", \"score\": \"378.0\"}", - "Elementary Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Elementary Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Elementary Mathematics - # prompt tokens": "{\"description\": \"min=524.862, mean=524.862, max=524.862, sum=1049.725 (2)\", \"tab\": \"General information\", \"score\": \"524.8624338624338\"}", - "Elementary Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"elementary_mathematics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_elementary_mathematics\"" - } - } - }, - { - "evaluation_name": "Formal Logic", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Formal Logic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.397, - "details": { - "description": "min=0.397, mean=0.397, max=0.397, sum=0.794 (2)", - "tab": "Accuracy", - "Formal Logic - Observed inference time (s)": "{\"description\": \"min=0.411, mean=0.411, max=0.411, sum=0.822 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4109457277116321\"}", - "Formal Logic - # eval": "{\"description\": \"min=126, mean=126, max=126, sum=252 (2)\", \"tab\": \"General information\", \"score\": \"126.0\"}", - "Formal Logic - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Formal Logic - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Formal Logic - # prompt tokens": "{\"description\": \"min=599.762, mean=599.762, max=599.762, sum=1199.524 (2)\", \"tab\": \"General information\", \"score\": \"599.7619047619048\"}", - "Formal Logic - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"formal_logic\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_formal_logic\"" - } - } - }, - { - "evaluation_name": "High School World History", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on High School World History", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.857, - "details": { - "description": "min=0.857, mean=0.857, max=0.857, sum=1.713 (2)", - "tab": "Accuracy", - "High School Biology - Observed inference time (s)": "{\"description\": \"min=0.389, mean=0.389, max=0.389, sum=0.777 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.38858610660799087\"}", - "High School Chemistry - Observed inference time (s)": "{\"description\": \"min=0.396, mean=0.396, max=0.396, sum=0.792 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.39599566624082366\"}", - "High School Computer Science - Observed inference time (s)": "{\"description\": \"min=0.495, mean=0.495, max=0.495, sum=0.99 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.495233371257782\"}", - "High School European History - Observed inference time (s)": "{\"description\": \"min=0.539, mean=0.539, max=0.539, sum=1.077 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5386766448165431\"}", - "High School Geography - Observed inference time (s)": "{\"description\": \"min=0.367, mean=0.367, max=0.367, sum=0.735 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.36738430129157174\"}", - "High School Government And Politics - Observed inference time (s)": "{\"description\": \"min=0.39, mean=0.39, max=0.39, sum=0.78 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.38988350463037047\"}", - "High School Macroeconomics - Observed inference time (s)": "{\"description\": \"min=0.36, mean=0.36, max=0.36, sum=0.721 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3604950317969689\"}", - "High School Mathematics - Observed inference time (s)": "{\"description\": \"min=0.388, mean=0.388, max=0.388, sum=0.777 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.38829568756951227\"}", - "High School Microeconomics - Observed inference time (s)": "{\"description\": \"min=0.372, mean=0.372, max=0.372, sum=0.743 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.37170837205999036\"}", - "High School Physics - Observed inference time (s)": "{\"description\": \"min=0.38, mean=0.38, max=0.38, sum=0.76 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3798852077383079\"}", - "High School Psychology - Observed inference time (s)": "{\"description\": \"min=0.395, mean=0.395, max=0.395, sum=0.79 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3950107355730249\"}", - "High School Statistics - Observed inference time (s)": "{\"description\": \"min=0.477, mean=0.477, max=0.477, sum=0.954 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4768963897669757\"}", - "High School US History - Observed inference time (s)": "{\"description\": \"min=0.528, mean=0.528, max=0.528, sum=1.056 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5277850253909242\"}", - "High School World History - Observed inference time (s)": "{\"description\": \"min=0.517, mean=0.517, max=0.517, sum=1.034 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5169116002094897\"}", - "High School Biology - # eval": "{\"description\": \"min=310, mean=310, max=310, sum=620 (2)\", \"tab\": \"General information\", \"score\": \"310.0\"}", - "High School Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Biology - # prompt tokens": "{\"description\": \"min=506.677, mean=506.677, max=506.677, sum=1013.355 (2)\", \"tab\": \"General information\", \"score\": \"506.6774193548387\"}", - "High School Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Chemistry - # eval": "{\"description\": \"min=203, mean=203, max=203, sum=406 (2)\", \"tab\": \"General information\", \"score\": \"203.0\"}", - "High School Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # prompt tokens": "{\"description\": \"min=489.714, mean=489.714, max=489.714, sum=979.429 (2)\", \"tab\": \"General information\", \"score\": \"489.7142857142857\"}", - "High School Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "High School Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # prompt tokens": "{\"description\": \"min=860.78, mean=860.78, max=860.78, sum=1721.56 (2)\", \"tab\": \"General information\", \"score\": \"860.78\"}", - "High School Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School European History - # eval": "{\"description\": \"min=165, mean=165, max=165, sum=330 (2)\", \"tab\": \"General information\", \"score\": \"165.0\"}", - "High School European History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School European History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # prompt tokens": "{\"description\": \"min=2791.073, mean=2791.073, max=2791.073, sum=5582.145 (2)\", \"tab\": \"General information\", \"score\": \"2791.072727272727\"}", - "High School European History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Geography - # eval": "{\"description\": \"min=198, mean=198, max=198, sum=396 (2)\", \"tab\": \"General information\", \"score\": \"198.0\"}", - "High School Geography - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Geography - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # prompt tokens": "{\"description\": \"min=365.045, mean=365.045, max=365.045, sum=730.091 (2)\", \"tab\": \"General information\", \"score\": \"365.04545454545456\"}", - "High School Geography - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Government And Politics - # eval": "{\"description\": \"min=193, mean=193, max=193, sum=386 (2)\", \"tab\": \"General information\", \"score\": \"193.0\"}", - "High School Government And Politics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Government And Politics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # prompt tokens": "{\"description\": \"min=458.824, mean=458.824, max=458.824, sum=917.648 (2)\", \"tab\": \"General information\", \"score\": \"458.8238341968912\"}", - "High School Government And Politics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Macroeconomics - # eval": "{\"description\": \"min=390, mean=390, max=390, sum=780 (2)\", \"tab\": \"General information\", \"score\": \"390.0\"}", - "High School Macroeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Macroeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # prompt tokens": "{\"description\": \"min=364.562, mean=364.562, max=364.562, sum=729.123 (2)\", \"tab\": \"General information\", \"score\": \"364.5615384615385\"}", - "High School Macroeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Mathematics - # eval": "{\"description\": \"min=270, mean=270, max=270, sum=540 (2)\", \"tab\": \"General information\", \"score\": \"270.0\"}", - "High School Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # prompt tokens": "{\"description\": \"min=525.374, mean=525.374, max=525.374, sum=1050.748 (2)\", \"tab\": \"General information\", \"score\": \"525.3740740740741\"}", - "High School Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Microeconomics - # eval": "{\"description\": \"min=238, mean=238, max=238, sum=476 (2)\", \"tab\": \"General information\", \"score\": \"238.0\"}", - "High School Microeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Microeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # prompt tokens": "{\"description\": \"min=392.025, mean=392.025, max=392.025, sum=784.05 (2)\", \"tab\": \"General information\", \"score\": \"392.02521008403363\"}", - "High School Microeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Physics - # eval": "{\"description\": \"min=151, mean=151, max=151, sum=302 (2)\", \"tab\": \"General information\", \"score\": \"151.0\"}", - "High School Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # prompt tokens": "{\"description\": \"min=553.464, mean=553.464, max=553.464, sum=1106.927 (2)\", \"tab\": \"General information\", \"score\": \"553.4635761589404\"}", - "High School Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Psychology - # eval": "{\"description\": \"min=545, mean=545, max=545, sum=1090 (2)\", \"tab\": \"General information\", \"score\": \"545.0\"}", - "High School Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # prompt tokens": "{\"description\": \"min=488.246, mean=488.246, max=488.246, sum=976.492 (2)\", \"tab\": \"General information\", \"score\": \"488.24587155963303\"}", - "High School Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Statistics - # eval": "{\"description\": \"min=216, mean=216, max=216, sum=432 (2)\", \"tab\": \"General information\", \"score\": \"216.0\"}", - "High School Statistics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Statistics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # prompt tokens": "{\"description\": \"min=788.699, mean=788.699, max=788.699, sum=1577.398 (2)\", \"tab\": \"General information\", \"score\": \"788.699074074074\"}", - "High School Statistics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School US History - # eval": "{\"description\": \"min=204, mean=204, max=204, sum=408 (2)\", \"tab\": \"General information\", \"score\": \"204.0\"}", - "High School US History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School US History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # prompt tokens": "{\"description\": \"min=2210.809, mean=2210.809, max=2210.809, sum=4421.618 (2)\", \"tab\": \"General information\", \"score\": \"2210.8088235294117\"}", - "High School US History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School World History - # eval": "{\"description\": \"min=237, mean=237, max=237, sum=474 (2)\", \"tab\": \"General information\", \"score\": \"237.0\"}", - "High School World History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School World History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # prompt tokens": "{\"description\": \"min=1421.27, mean=1421.27, max=1421.27, sum=2842.54 (2)\", \"tab\": \"General information\", \"score\": \"1421.2700421940929\"}", - "High School World History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"high_school_world_history\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_high_school_world_history\"" - } - } - }, - { - "evaluation_name": "Human Sexuality", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Human Sexuality", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.786, - "details": { - "description": "min=0.786, mean=0.786, max=0.786, sum=1.573 (2)", - "tab": "Accuracy", - "Human Aging - Observed inference time (s)": "{\"description\": \"min=0.38, mean=0.38, max=0.38, sum=0.76 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3799830274197018\"}", - "Human Sexuality - Observed inference time (s)": "{\"description\": \"min=0.391, mean=0.391, max=0.391, sum=0.783 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3914412269155488\"}", - "Human Aging - # eval": "{\"description\": \"min=223, mean=223, max=223, sum=446 (2)\", \"tab\": \"General information\", \"score\": \"223.0\"}", - "Human Aging - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Aging - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Aging - # prompt tokens": "{\"description\": \"min=312.906, mean=312.906, max=312.906, sum=625.812 (2)\", \"tab\": \"General information\", \"score\": \"312.90582959641256\"}", - "Human Aging - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Human Sexuality - # eval": "{\"description\": \"min=131, mean=131, max=131, sum=262 (2)\", \"tab\": \"General information\", \"score\": \"131.0\"}", - "Human Sexuality - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Sexuality - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # prompt tokens": "{\"description\": \"min=334.183, mean=334.183, max=334.183, sum=668.366 (2)\", \"tab\": \"General information\", \"score\": \"334.1832061068702\"}", - "Human Sexuality - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"human_sexuality\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_human_sexuality\"" - } - } - }, - { - "evaluation_name": "International Law", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on International Law", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.843, - "details": { - "description": "min=0.843, mean=0.843, max=0.843, sum=1.686 (2)", - "tab": "Accuracy", - "International Law - Observed inference time (s)": "{\"description\": \"min=0.556, mean=0.556, max=0.556, sum=1.113 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5563427140890074\"}", - "International Law - # eval": "{\"description\": \"min=121, mean=121, max=121, sum=242 (2)\", \"tab\": \"General information\", \"score\": \"121.0\"}", - "International Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "International Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "International Law - # prompt tokens": "{\"description\": \"min=632.851, mean=632.851, max=632.851, sum=1265.702 (2)\", \"tab\": \"General information\", \"score\": \"632.8512396694215\"}", - "International Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"international_law\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_international_law\"" - } - } - }, - { - "evaluation_name": "Logical Fallacies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Logical Fallacies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.791, - "details": { - "description": "min=0.791, mean=0.791, max=0.791, sum=1.583 (2)", - "tab": "Accuracy", - "Logical Fallacies - Observed inference time (s)": "{\"description\": \"min=0.405, mean=0.405, max=0.405, sum=0.811 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4053135386273905\"}", - "Logical Fallacies - # eval": "{\"description\": \"min=163, mean=163, max=163, sum=326 (2)\", \"tab\": \"General information\", \"score\": \"163.0\"}", - "Logical Fallacies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Logical Fallacies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Logical Fallacies - # prompt tokens": "{\"description\": \"min=442.595, mean=442.595, max=442.595, sum=885.19 (2)\", \"tab\": \"General information\", \"score\": \"442.5950920245399\"}", - "Logical Fallacies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"logical_fallacies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_logical_fallacies\"" - } - } - }, - { - "evaluation_name": "Machine Learning", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Machine Learning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.455, - "details": { - "description": "min=0.455, mean=0.455, max=0.455, sum=0.911 (2)", - "tab": "Accuracy", - "Machine Learning - Observed inference time (s)": "{\"description\": \"min=0.46, mean=0.46, max=0.46, sum=0.92 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.45983841376645224\"}", - "Machine Learning - # eval": "{\"description\": \"min=112, mean=112, max=112, sum=224 (2)\", \"tab\": \"General information\", \"score\": \"112.0\"}", - "Machine Learning - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Machine Learning - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Machine Learning - # prompt tokens": "{\"description\": \"min=661.054, mean=661.054, max=661.054, sum=1322.107 (2)\", \"tab\": \"General information\", \"score\": \"661.0535714285714\"}", - "Machine Learning - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"machine_learning\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_machine_learning\"" - } - } - }, - { - "evaluation_name": "Management", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Management", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.845, - "details": { - "description": "min=0.845, mean=0.845, max=0.845, sum=1.689 (2)", - "tab": "Accuracy", - "Management - Observed inference time (s)": "{\"description\": \"min=0.386, mean=0.386, max=0.386, sum=0.773 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.38629551535671197\"}", - "Management - # eval": "{\"description\": \"min=103, mean=103, max=103, sum=206 (2)\", \"tab\": \"General information\", \"score\": \"103.0\"}", - "Management - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Management - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Management - # prompt tokens": "{\"description\": \"min=276.796, mean=276.796, max=276.796, sum=553.592 (2)\", \"tab\": \"General information\", \"score\": \"276.79611650485435\"}", - "Management - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"management\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_management\"" - } - } - }, - { - "evaluation_name": "Marketing", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Marketing", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.91, - "details": { - "description": "min=0.91, mean=0.91, max=0.91, sum=1.821 (2)", - "tab": "Accuracy", - "Marketing - Observed inference time (s)": "{\"description\": \"min=0.391, mean=0.391, max=0.391, sum=0.781 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3906826453331189\"}", - "Marketing - # eval": "{\"description\": \"min=234, mean=234, max=234, sum=468 (2)\", \"tab\": \"General information\", \"score\": \"234.0\"}", - "Marketing - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Marketing - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Marketing - # prompt tokens": "{\"description\": \"min=397.218, mean=397.218, max=397.218, sum=794.436 (2)\", \"tab\": \"General information\", \"score\": \"397.21794871794873\"}", - "Marketing - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"marketing\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_marketing\"" - } - } - }, - { - "evaluation_name": "Medical Genetics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Medical Genetics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8, - "details": { - "description": "min=0.8, mean=0.8, max=0.8, sum=1.6 (2)", - "tab": "Accuracy", - "Medical Genetics - Observed inference time (s)": "{\"description\": \"min=0.378, mean=0.378, max=0.378, sum=0.756 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3778671717643738\"}", - "Medical Genetics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Medical Genetics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Medical Genetics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Medical Genetics - # prompt tokens": "{\"description\": \"min=334, mean=334, max=334, sum=668 (2)\", \"tab\": \"General information\", \"score\": \"334.0\"}", - "Medical Genetics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"medical_genetics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_medical_genetics\"" - } - } - }, - { - "evaluation_name": "Miscellaneous", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Miscellaneous", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.893, - "details": { - "description": "min=0.893, mean=0.893, max=0.893, sum=1.785 (2)", - "tab": "Accuracy", - "Miscellaneous - Observed inference time (s)": "{\"description\": \"min=0.367, mean=0.367, max=0.367, sum=0.735 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.36739401007368494\"}", - "Miscellaneous - # eval": "{\"description\": \"min=783, mean=783, max=783, sum=1566 (2)\", \"tab\": \"General information\", \"score\": \"783.0\"}", - "Miscellaneous - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Miscellaneous - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Miscellaneous - # prompt tokens": "{\"description\": \"min=292.925, mean=292.925, max=292.925, sum=585.849 (2)\", \"tab\": \"General information\", \"score\": \"292.92464878671774\"}", - "Miscellaneous - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"miscellaneous\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_miscellaneous\"" - } - } - }, - { - "evaluation_name": "Moral Scenarios", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Moral Scenarios", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.404, - "details": { - "description": "min=0.404, mean=0.404, max=0.404, sum=0.809 (2)", - "tab": "Accuracy", - "Moral Disputes - Observed inference time (s)": "{\"description\": \"min=0.387, mean=0.387, max=0.387, sum=0.773 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.38658536858641346\"}", - "Moral Scenarios - Observed inference time (s)": "{\"description\": \"min=0.609, mean=0.609, max=0.609, sum=1.217 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6085127204490107\"}", - "Moral Disputes - # eval": "{\"description\": \"min=346, mean=346, max=346, sum=692 (2)\", \"tab\": \"General information\", \"score\": \"346.0\"}", - "Moral Disputes - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Disputes - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Disputes - # prompt tokens": "{\"description\": \"min=469.145, mean=469.145, max=469.145, sum=938.289 (2)\", \"tab\": \"General information\", \"score\": \"469.1445086705202\"}", - "Moral Disputes - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Moral Scenarios - # eval": "{\"description\": \"min=895, mean=895, max=895, sum=1790 (2)\", \"tab\": \"General information\", \"score\": \"895.0\"}", - "Moral Scenarios - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Scenarios - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # prompt tokens": "{\"description\": \"min=649.455, mean=649.455, max=649.455, sum=1298.909 (2)\", \"tab\": \"General information\", \"score\": \"649.454748603352\"}", - "Moral Scenarios - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"moral_scenarios\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_moral_scenarios\"" - } - } - }, - { - "evaluation_name": "Nutrition", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Nutrition", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.758, - "details": { - "description": "min=0.758, mean=0.758, max=0.758, sum=1.516 (2)", - "tab": "Accuracy", - "Nutrition - Observed inference time (s)": "{\"description\": \"min=0.659, mean=0.659, max=0.659, sum=1.319 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6593383916842392\"}", - "Nutrition - # eval": "{\"description\": \"min=306, mean=306, max=306, sum=612 (2)\", \"tab\": \"General information\", \"score\": \"306.0\"}", - "Nutrition - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Nutrition - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Nutrition - # prompt tokens": "{\"description\": \"min=579.817, mean=579.817, max=579.817, sum=1159.634 (2)\", \"tab\": \"General information\", \"score\": \"579.8169934640523\"}", - "Nutrition - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"nutrition\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_nutrition\"" - } - } - }, - { - "evaluation_name": "Prehistory", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Prehistory", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.787, - "details": { - "description": "min=0.787, mean=0.787, max=0.787, sum=1.574 (2)", - "tab": "Accuracy", - "Prehistory - Observed inference time (s)": "{\"description\": \"min=0.414, mean=0.414, max=0.414, sum=0.828 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4140352636207769\"}", - "Prehistory - # eval": "{\"description\": \"min=324, mean=324, max=324, sum=648 (2)\", \"tab\": \"General information\", \"score\": \"324.0\"}", - "Prehistory - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Prehistory - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Prehistory - # prompt tokens": "{\"description\": \"min=507.559, mean=507.559, max=507.559, sum=1015.117 (2)\", \"tab\": \"General information\", \"score\": \"507.55864197530866\"}", - "Prehistory - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"prehistory\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_prehistory\"" - } - } - }, - { - "evaluation_name": "Public Relations", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Public Relations", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.745, - "details": { - "description": "min=0.745, mean=0.745, max=0.745, sum=1.491 (2)", - "tab": "Accuracy", - "Public Relations - Observed inference time (s)": "{\"description\": \"min=0.373, mean=0.373, max=0.373, sum=0.746 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3731096332723444\"}", - "Public Relations - # eval": "{\"description\": \"min=110, mean=110, max=110, sum=220 (2)\", \"tab\": \"General information\", \"score\": \"110.0\"}", - "Public Relations - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Public Relations - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Public Relations - # prompt tokens": "{\"description\": \"min=398.318, mean=398.318, max=398.318, sum=796.636 (2)\", \"tab\": \"General information\", \"score\": \"398.3181818181818\"}", - "Public Relations - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"public_relations\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_public_relations\"" - } - } - }, - { - "evaluation_name": "Security Studies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Security Studies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8, - "details": { - "description": "min=0.8, mean=0.8, max=0.8, sum=1.6 (2)", - "tab": "Accuracy", - "Security Studies - Observed inference time (s)": "{\"description\": \"min=0.496, mean=0.496, max=0.496, sum=0.993 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4963450723764848\"}", - "Security Studies - # eval": "{\"description\": \"min=245, mean=245, max=245, sum=490 (2)\", \"tab\": \"General information\", \"score\": \"245.0\"}", - "Security Studies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Security Studies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Security Studies - # prompt tokens": "{\"description\": \"min=1157.473, mean=1157.473, max=1157.473, sum=2314.947 (2)\", \"tab\": \"General information\", \"score\": \"1157.4734693877551\"}", - "Security Studies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"security_studies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_security_studies\"" - } - } - }, - { - "evaluation_name": "Sociology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Sociology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.871, - "details": { - "description": "min=0.871, mean=0.871, max=0.871, sum=1.741 (2)", - "tab": "Accuracy", - "Sociology - Observed inference time (s)": "{\"description\": \"min=0.376, mean=0.376, max=0.376, sum=0.753 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3763423120204489\"}", - "Sociology - # eval": "{\"description\": \"min=201, mean=201, max=201, sum=402 (2)\", \"tab\": \"General information\", \"score\": \"201.0\"}", - "Sociology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Sociology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Sociology - # prompt tokens": "{\"description\": \"min=438.522, mean=438.522, max=438.522, sum=877.045 (2)\", \"tab\": \"General information\", \"score\": \"438.5223880597015\"}", - "Sociology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"sociology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_sociology\"" - } - } - }, - { - "evaluation_name": "Virology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Virology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.542, - "details": { - "description": "min=0.542, mean=0.542, max=0.542, sum=1.084 (2)", - "tab": "Accuracy", - "Virology - Observed inference time (s)": "{\"description\": \"min=0.387, mean=0.387, max=0.387, sum=0.775 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3873033107045185\"}", - "Virology - # eval": "{\"description\": \"min=166, mean=166, max=166, sum=332 (2)\", \"tab\": \"General information\", \"score\": \"166.0\"}", - "Virology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Virology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Virology - # prompt tokens": "{\"description\": \"min=336.09, mean=336.09, max=336.09, sum=672.181 (2)\", \"tab\": \"General information\", \"score\": \"336.0903614457831\"}", - "Virology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"virology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_virology\"" - } - } - }, - { - "evaluation_name": "World Religions", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on World Religions", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.836, - "details": { - "description": "min=0.836, mean=0.836, max=0.836, sum=1.673 (2)", - "tab": "Accuracy", - "World Religions - Observed inference time (s)": "{\"description\": \"min=0.403, mean=0.403, max=0.403, sum=0.807 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4032876603087487\"}", - "World Religions - # eval": "{\"description\": \"min=171, mean=171, max=171, sum=342 (2)\", \"tab\": \"General information\", \"score\": \"171.0\"}", - "World Religions - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "World Religions - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "World Religions - # prompt tokens": "{\"description\": \"min=268.561, mean=268.561, max=268.561, sum=537.123 (2)\", \"tab\": \"General information\", \"score\": \"268.56140350877195\"}", - "World Religions - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"world_religions\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_world_religions\"" - } - } - }, - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.589, - "details": { - "description": "", - "tab": "Efficiency" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/openai_gpt-4-0125-preview.json b/data/models/openai_gpt-4-0125-preview.json deleted file mode 100644 index 5173169013d1e5228fdaeac91c6281ec53260d57..0000000000000000000000000000000000000000 --- a/data/models/openai_gpt-4-0125-preview.json +++ /dev/null @@ -1,144 +0,0 @@ -{ - "model_info": { - "name": "openai/gpt-4-0125-preview", - "id": "openai/gpt-4-0125-preview", - "developer": "OpenAI", - "additional_details": { - "model_type": "Generative" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/openai_gpt-4-0125-preview/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8434 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9525 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7434 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8757 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8692 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7085 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/openai_gpt-4-0314.json b/data/models/openai_gpt-4-0314.json deleted file mode 100644 index 29a16359f84031c82140d202462381cf28f16d1f..0000000000000000000000000000000000000000 --- a/data/models/openai_gpt-4-0314.json +++ /dev/null @@ -1,237 +0,0 @@ -{ - "model_info": { - "name": "GPT-4 0314", - "id": "openai/gpt-4-0314", - "developer": "OpenAI", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_instruct/openai_gpt-4-0314/1774096309.537868", - "retrieved_timestamp": "1774096309.537868", - "source_metadata": { - "source_name": "helm_instruct", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_instruct", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_instruct", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/instruct/benchmark_output/releases/v1.0.0/groups/instruction_following.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperform on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.611, - "details": { - "description": "", - "tab": "Instruction Following" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "Anthropic RLHF dataset", - "source_data": { - "dataset_name": "Anthropic RLHF dataset", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/instruct/benchmark_output/releases/v1.0.0/groups/instruction_following.json" - ] - }, - "metric_config": { - "evaluation_description": "Harmlessness on Anthropic RLHF dataset", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 5.0 - }, - "score_details": { - "score": 4.934, - "details": { - "description": "min=4.83, mean=4.934, max=5, sum=39.47 (8)", - "tab": "Instruction Following" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"hh\", \"hh\", \"hh\", \"hh\", \"red_team\", \"red_team\", \"red_team\", \"red_team\"]", - "evaluator": "[\"claude\", \"gpt4\", \"mturk\", \"scale\", \"claude\", \"gpt4\", \"mturk\", \"scale\"]" - } - } - }, - { - "evaluation_name": "Best ChatGPT Prompts", - "source_data": { - "dataset_name": "Best ChatGPT Prompts", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/instruct/benchmark_output/releases/v1.0.0/groups/instruction_following.json" - ] - }, - "metric_config": { - "evaluation_description": "Harmlessness on Best ChatGPT Prompts", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 5.0 - }, - "score_details": { - "score": 4.973, - "details": { - "description": "min=4.915, mean=4.973, max=5, sum=19.894 (4)", - "tab": "Instruction Following" - } - }, - "generation_config": { - "additional_details": { - "path": "\"src_helm_benchmark_scenarios_best_chatgpt_prompts.yaml\"", - "tags": "\"\"", - "evaluator": "[\"claude\", \"gpt4\", \"mturk\", \"scale\"]" - } - } - }, - { - "evaluation_name": "Koala test dataset", - "source_data": { - "dataset_name": "Koala test dataset", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/instruct/benchmark_output/releases/v1.0.0/groups/instruction_following.json" - ] - }, - "metric_config": { - "evaluation_description": "Harmlessness on Koala test dataset", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 5.0 - }, - "score_details": { - "score": 4.966, - "details": { - "description": "min=4.913, mean=4.966, max=5, sum=19.863 (4)", - "tab": "Instruction Following" - } - }, - "generation_config": { - "additional_details": { - "evaluator": "[\"claude\", \"gpt4\", \"mturk\", \"scale\"]" - } - } - }, - { - "evaluation_name": "Open Assistant", - "source_data": { - "dataset_name": "Open Assistant", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/instruct/benchmark_output/releases/v1.0.0/groups/instruction_following.json" - ] - }, - "metric_config": { - "evaluation_description": "Harmlessness on Open Assistant", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 5.0 - }, - "score_details": { - "score": 4.986, - "details": { - "description": "min=4.97, mean=4.986, max=5, sum=19.945 (4)", - "tab": "Instruction Following" - } - }, - "generation_config": { - "additional_details": { - "language": "\"en\"", - "evaluator": "[\"claude\", \"gpt4\", \"mturk\", \"scale\"]" - } - } - }, - { - "evaluation_name": "Self Instruct", - "source_data": { - "dataset_name": "Self Instruct", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/instruct/benchmark_output/releases/v1.0.0/groups/instruction_following.json" - ] - }, - "metric_config": { - "evaluation_description": "Harmlessness on Self Instruct", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 5.0 - }, - "score_details": { - "score": 4.976, - "details": { - "description": "min=4.945, mean=4.976, max=5, sum=19.905 (4)", - "tab": "Instruction Following" - } - }, - "generation_config": { - "additional_details": { - "evaluator": "[\"claude\", \"gpt4\", \"mturk\", \"scale\"]" - } - } - }, - { - "evaluation_name": "Vicuna", - "source_data": { - "dataset_name": "Vicuna", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/instruct/benchmark_output/releases/v1.0.0/groups/instruction_following.json" - ] - }, - "metric_config": { - "evaluation_description": "Harmlessness on Vicuna", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 5.0 - }, - "score_details": { - "score": 4.995, - "details": { - "description": "min=4.981, mean=4.995, max=5, sum=19.981 (4)", - "tab": "Instruction Following" - } - }, - "generation_config": { - "additional_details": { - "category": "\"all\"", - "evaluator": "[\"claude\", \"gpt4\", \"mturk\", \"scale\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/openai_gpt-4-0613.json b/data/models/openai_gpt-4-0613.json deleted file mode 100644 index f9edf8b400bcfd0d8167fb29ff1bda235867f788..0000000000000000000000000000000000000000 --- a/data/models/openai_gpt-4-0613.json +++ /dev/null @@ -1,1901 +0,0 @@ -{ - "model_info": { - "name": "GPT-4 0613", - "id": "openai/gpt-4-0613", - "developer": "OpenAI", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_lite/openai_gpt-4-0613/1774096306.427425", - "retrieved_timestamp": "1774096306.427425", - "source_metadata": { - "source_name": "helm_lite", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_lite", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_lite", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.867, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.5158801498127341\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.768, - "details": { - "description": "min=0.768, mean=0.768, max=0.768, sum=0.768 (1)", - "tab": "Accuracy", - "NarrativeQA - Observed inference time (s)": "{\"description\": \"min=0.976, mean=0.976, max=0.976, sum=0.976 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.9758186582108619\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=3522.67, mean=3522.67, max=3522.67, sum=3522.67 (1)\", \"tab\": \"General information\", \"score\": \"3522.6704225352114\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=8.515, mean=8.515, max=8.515, sum=8.515 (1)\", \"tab\": \"General information\", \"score\": \"8.51549295774648\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (closed-book)", - "source_data": { - "dataset_name": "NaturalQuestions (closed-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (closed-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.457, - "details": { - "description": "min=0.457, mean=0.457, max=0.457, sum=0.457 (1)", - "tab": "Accuracy", - "NaturalQuestions (open-book) - Observed inference time (s)": "{\"description\": \"min=0.908, mean=0.908, max=0.908, sum=0.908 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.9083020164966583\"}", - "NaturalQuestions (closed-book) - Observed inference time (s)": "{\"description\": \"min=0.512, mean=0.512, max=0.512, sum=0.512 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.5116857671737671\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.964, mean=4.964, max=4.964, sum=4.964 (1)\", \"tab\": \"General information\", \"score\": \"4.964\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.007, mean=0.007, max=0.007, sum=0.007 (1)\", \"tab\": \"General information\", \"score\": \"0.007\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1717.847, mean=1717.847, max=1717.847, sum=1717.847 (1)\", \"tab\": \"General information\", \"score\": \"1717.847\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=8.055, mean=8.055, max=8.055, sum=8.055 (1)\", \"tab\": \"General information\", \"score\": \"8.055\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=173.127, mean=173.127, max=173.127, sum=173.127 (1)\", \"tab\": \"General information\", \"score\": \"173.127\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=3.832, mean=3.832, max=3.832, sum=3.832 (1)\", \"tab\": \"General information\", \"score\": \"3.832\"}" - } - }, - "generation_config": { - "additional_details": { - "mode": "\"closedbook\"" - } - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.96, - "details": { - "description": "min=0.96, mean=0.96, max=0.96, sum=0.96 (1)", - "tab": "Accuracy", - "OpenbookQA - Observed inference time (s)": "{\"description\": \"min=0.401, mean=0.401, max=0.401, sum=0.401 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.40061268854141235\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=242.782, mean=242.782, max=242.782, sum=242.782 (1)\", \"tab\": \"General information\", \"score\": \"242.782\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "dataset": "\"openbookqa\"", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.735, - "details": { - "description": "min=0.55, mean=0.735, max=0.95, sum=3.674 (5)", - "tab": "Accuracy", - "MMLU - Observed inference time (s)": "{\"description\": \"min=0.364, mean=0.391, max=0.434, sum=1.954 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.39080846048656265\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=366.44, mean=460.72, max=607.43, sum=2303.6 (5)\", \"tab\": \"General information\", \"score\": \"460.71996491228066\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"college_chemistry\", \"computer_security\", \"econometrics\", \"us_foreign_policy\"]", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MATH", - "source_data": { - "dataset_name": "MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Equivalent (CoT) on MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.802, - "details": { - "description": "min=0.673, mean=0.802, max=0.948, sum=5.617 (7)", - "tab": "Accuracy", - "MATH - Observed inference time (s)": "{\"description\": \"min=2.95, mean=3.472, max=4.247, sum=24.303 (7)\", \"tab\": \"Efficiency\", \"score\": \"3.4718795228507955\"}", - "MATH - # eval": "{\"description\": \"min=30, mean=62.429, max=135, sum=437 (7)\", \"tab\": \"General information\", \"score\": \"62.42857142857143\"}", - "MATH - # train": "{\"description\": \"min=8, mean=8, max=8, sum=56 (7)\", \"tab\": \"General information\", \"score\": \"8.0\"}", - "MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MATH - # prompt tokens": "{\"description\": \"min=942.363, mean=1323.911, max=2258.577, sum=9267.376 (7)\", \"tab\": \"General information\", \"score\": \"1323.910874184069\"}", - "MATH - # output tokens": "{\"description\": \"min=59.674, mean=73.257, max=81.1, sum=512.799 (7)\", \"tab\": \"General information\", \"score\": \"73.25695858608955\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"algebra\", \"counting_and_probability\", \"geometry\", \"intermediate_algebra\", \"number_theory\", \"prealgebra\", \"precalculus\"]", - "level": "\"1\"", - "use_official_examples": "\"False\"", - "use_chain_of_thought": "\"True\"" - } - } - }, - { - "evaluation_name": "GSM8K", - "source_data": { - "dataset_name": "GSM8K", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on GSM8K", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.932, - "details": { - "description": "min=0.932, mean=0.932, max=0.932, sum=0.932 (1)", - "tab": "Accuracy", - "GSM8K - Observed inference time (s)": "{\"description\": \"min=4.948, mean=4.948, max=4.948, sum=4.948 (1)\", \"tab\": \"Efficiency\", \"score\": \"4.947624314308166\"}", - "GSM8K - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "GSM8K - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "GSM8K - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GSM8K - # prompt tokens": "{\"description\": \"min=1020.035, mean=1020.035, max=1020.035, sum=1020.035 (1)\", \"tab\": \"General information\", \"score\": \"1020.035\"}", - "GSM8K - # output tokens": "{\"description\": \"min=111.209, mean=111.209, max=111.209, sum=111.209 (1)\", \"tab\": \"General information\", \"score\": \"111.209\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "LegalBench", - "source_data": { - "dataset_name": "LegalBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on LegalBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.713, - "details": { - "description": "min=0.452, mean=0.713, max=0.905, sum=3.564 (5)", - "tab": "Accuracy", - "LegalBench - Observed inference time (s)": "{\"description\": \"min=0.46, mean=0.558, max=0.886, sum=2.791 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.5582764348578453\"}", - "LegalBench - # eval": "{\"description\": \"min=95, mean=409.4, max=1000, sum=2047 (5)\", \"tab\": \"General information\", \"score\": \"409.4\"}", - "LegalBench - # train": "{\"description\": \"min=4, mean=4.798, max=5, sum=23.992 (5)\", \"tab\": \"General information\", \"score\": \"4.798367346938775\"}", - "LegalBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "LegalBench - # prompt tokens": "{\"description\": \"min=253.442, mean=1568.687, max=6350.008, sum=7843.435 (5)\", \"tab\": \"General information\", \"score\": \"1568.6870529886412\"}", - "LegalBench - # output tokens": "{\"description\": \"min=1, mean=1.34, max=2.063, sum=6.698 (5)\", \"tab\": \"General information\", \"score\": \"1.3396070557866055\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"abercrombie\", \"corporate_lobbying\", \"function_of_decision_section\", \"international_citizenship_questions\", \"proa\"]" - } - } - }, - { - "evaluation_name": "MedQA", - "source_data": { - "dataset_name": "MedQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MedQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.815, - "details": { - "description": "min=0.815, mean=0.815, max=0.815, sum=0.815 (1)", - "tab": "Accuracy", - "MedQA - Observed inference time (s)": "{\"description\": \"min=0.414, mean=0.414, max=0.414, sum=0.414 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.4136932588239787\"}", - "MedQA - # eval": "{\"description\": \"min=503, mean=503, max=503, sum=503 (1)\", \"tab\": \"General information\", \"score\": \"503.0\"}", - "MedQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MedQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MedQA - # prompt tokens": "{\"description\": \"min=1020.414, mean=1020.414, max=1020.414, sum=1020.414 (1)\", \"tab\": \"General information\", \"score\": \"1020.4135188866799\"}", - "MedQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WMT 2014", - "source_data": { - "dataset_name": "WMT 2014", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "BLEU-4 on WMT 2014", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.211, - "details": { - "description": "min=0.149, mean=0.211, max=0.256, sum=1.053 (5)", - "tab": "Accuracy", - "WMT 2014 - Observed inference time (s)": "{\"description\": \"min=1.448, mean=1.58, max=1.724, sum=7.899 (5)\", \"tab\": \"Efficiency\", \"score\": \"1.5797039644192494\"}", - "WMT 2014 - # eval": "{\"description\": \"min=503, mean=568.8, max=832, sum=2844 (5)\", \"tab\": \"General information\", \"score\": \"568.8\"}", - "WMT 2014 - # train": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "WMT 2014 - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WMT 2014 - # prompt tokens": "{\"description\": \"min=169.901, mean=193.043, max=213.185, sum=965.213 (5)\", \"tab\": \"General information\", \"score\": \"193.04258583116683\"}", - "WMT 2014 - # output tokens": "{\"description\": \"min=23.767, mean=25.424, max=26.121, sum=127.122 (5)\", \"tab\": \"General information\", \"score\": \"25.424382072946933\"}" - } - }, - "generation_config": { - "additional_details": { - "language_pair": "[\"cs-en\", \"de-en\", \"fr-en\", \"hi-en\", \"ru-en\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_mmlu/openai_gpt-4-0613/1774096312.00548", - "retrieved_timestamp": "1774096312.00548", - "source_metadata": { - "source_name": "helm_mmlu", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_mmlu", - "evaluation_results": [ - { - "evaluation_name": "MMLU All Subjects", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU All Subjects", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.824, - "details": { - "description": "min=0.54, mean=0.824, max=0.99, sum=93.978 (114)", - "tab": "Accuracy", - "MMLU All Subjects - Observed inference time (s)": "{\"description\": \"min=0.364, mean=0.447, max=0.579, sum=51.005 (114)\", \"tab\": \"Efficiency\", \"score\": \"0.4474144183932911\"}", - "MMLU All Subjects - # eval": "{\"description\": \"min=100, mean=246.351, max=1534, sum=28084 (114)\", \"tab\": \"General information\", \"score\": \"246.35087719298247\"}", - "MMLU All Subjects - # train": "{\"description\": \"min=5, mean=5, max=5, sum=570 (114)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU All Subjects - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU All Subjects - # prompt tokens": "{\"description\": \"min=268.561, mean=607.852, max=2791.073, sum=69295.086 (114)\", \"tab\": \"General information\", \"score\": \"607.851634217556\"}", - "MMLU All Subjects - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=114 (114)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_name": "Abstract Algebra", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Abstract Algebra", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.63, - "details": { - "description": "min=0.63, mean=0.63, max=0.63, sum=1.26 (2)", - "tab": "Accuracy", - "Abstract Algebra - Observed inference time (s)": "{\"description\": \"min=0.393, mean=0.393, max=0.393, sum=0.787 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.39332568168640136\"}", - "Abstract Algebra - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Abstract Algebra - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Abstract Algebra - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Abstract Algebra - # prompt tokens": "{\"description\": \"min=366.44, mean=366.44, max=366.44, sum=732.88 (2)\", \"tab\": \"General information\", \"score\": \"366.44\"}", - "Abstract Algebra - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"abstract_algebra\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_abstract_algebra\"" - } - } - }, - { - "evaluation_name": "Anatomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Anatomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8, - "details": { - "description": "min=0.8, mean=0.8, max=0.8, sum=1.6 (2)", - "tab": "Accuracy", - "Anatomy - Observed inference time (s)": "{\"description\": \"min=0.545, mean=0.545, max=0.545, sum=1.09 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5451150911825674\"}", - "Anatomy - # eval": "{\"description\": \"min=135, mean=135, max=135, sum=270 (2)\", \"tab\": \"General information\", \"score\": \"135.0\"}", - "Anatomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Anatomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Anatomy - # prompt tokens": "{\"description\": \"min=346.978, mean=346.978, max=346.978, sum=693.956 (2)\", \"tab\": \"General information\", \"score\": \"346.97777777777776\"}", - "Anatomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"anatomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_anatomy\"" - } - } - }, - { - "evaluation_name": "College Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on College Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.627, - "details": { - "description": "min=0.627, mean=0.627, max=0.627, sum=1.255 (2)", - "tab": "Accuracy", - "College Chemistry - Observed inference time (s)": "{\"description\": \"min=0.389, mean=0.389, max=0.389, sum=0.778 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3888898015022278\"}", - "College Biology - Observed inference time (s)": "{\"description\": \"min=0.433, mean=0.433, max=0.433, sum=0.866 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.43280420700709027\"}", - "College Computer Science - Observed inference time (s)": "{\"description\": \"min=0.492, mean=0.492, max=0.492, sum=0.984 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.49212974786758423\"}", - "College Mathematics - Observed inference time (s)": "{\"description\": \"min=0.435, mean=0.435, max=0.435, sum=0.871 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4354128074645996\"}", - "College Medicine - Observed inference time (s)": "{\"description\": \"min=0.431, mean=0.431, max=0.431, sum=0.861 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4306242893196944\"}", - "College Physics - Observed inference time (s)": "{\"description\": \"min=0.415, mean=0.415, max=0.415, sum=0.83 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.41519686287524654\"}", - "College Chemistry - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Chemistry - # prompt tokens": "{\"description\": \"min=542.4, mean=542.4, max=542.4, sum=1084.8 (2)\", \"tab\": \"General information\", \"score\": \"542.4\"}", - "College Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Biology - # eval": "{\"description\": \"min=144, mean=144, max=144, sum=288 (2)\", \"tab\": \"General information\", \"score\": \"144.0\"}", - "College Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # prompt tokens": "{\"description\": \"min=466.917, mean=466.917, max=466.917, sum=933.833 (2)\", \"tab\": \"General information\", \"score\": \"466.9166666666667\"}", - "College Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # prompt tokens": "{\"description\": \"min=821.39, mean=821.39, max=821.39, sum=1642.78 (2)\", \"tab\": \"General information\", \"score\": \"821.39\"}", - "College Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Mathematics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # prompt tokens": "{\"description\": \"min=587.52, mean=587.52, max=587.52, sum=1175.04 (2)\", \"tab\": \"General information\", \"score\": \"587.52\"}", - "College Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Medicine - # eval": "{\"description\": \"min=173, mean=173, max=173, sum=346 (2)\", \"tab\": \"General information\", \"score\": \"173.0\"}", - "College Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # prompt tokens": "{\"description\": \"min=495.728, mean=495.728, max=495.728, sum=991.457 (2)\", \"tab\": \"General information\", \"score\": \"495.728323699422\"}", - "College Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Physics - # eval": "{\"description\": \"min=102, mean=102, max=102, sum=204 (2)\", \"tab\": \"General information\", \"score\": \"102.0\"}", - "College Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # prompt tokens": "{\"description\": \"min=496.608, mean=496.608, max=496.608, sum=993.216 (2)\", \"tab\": \"General information\", \"score\": \"496.6078431372549\"}", - "College Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"college_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_college_physics\"" - } - } - }, - { - "evaluation_name": "Computer Security", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Computer Security", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.86, - "details": { - "description": "min=0.86, mean=0.86, max=0.86, sum=1.72 (2)", - "tab": "Accuracy", - "Computer Security - Observed inference time (s)": "{\"description\": \"min=0.373, mean=0.373, max=0.373, sum=0.746 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3729291558265686\"}", - "Computer Security - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Computer Security - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Computer Security - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Computer Security - # prompt tokens": "{\"description\": \"min=371.54, mean=371.54, max=371.54, sum=743.08 (2)\", \"tab\": \"General information\", \"score\": \"371.54\"}", - "Computer Security - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"computer_security\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_computer_security\"" - } - } - }, - { - "evaluation_name": "Econometrics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Econometrics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.684, - "details": { - "description": "min=0.684, mean=0.684, max=0.684, sum=1.368 (2)", - "tab": "Accuracy", - "Econometrics - Observed inference time (s)": "{\"description\": \"min=0.364, mean=0.364, max=0.364, sum=0.729 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.36447873241023016\"}", - "Econometrics - # eval": "{\"description\": \"min=114, mean=114, max=114, sum=228 (2)\", \"tab\": \"General information\", \"score\": \"114.0\"}", - "Econometrics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Econometrics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Econometrics - # prompt tokens": "{\"description\": \"min=607.43, mean=607.43, max=607.43, sum=1214.86 (2)\", \"tab\": \"General information\", \"score\": \"607.4298245614035\"}", - "Econometrics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"econometrics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_econometrics\"" - } - } - }, - { - "evaluation_name": "Global Facts", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Global Facts", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.62, - "details": { - "description": "min=0.62, mean=0.62, max=0.62, sum=1.24 (2)", - "tab": "Accuracy", - "Global Facts - Observed inference time (s)": "{\"description\": \"min=0.476, mean=0.476, max=0.476, sum=0.952 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4758000469207764\"}", - "Global Facts - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Global Facts - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Global Facts - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Global Facts - # prompt tokens": "{\"description\": \"min=392.71, mean=392.71, max=392.71, sum=785.42 (2)\", \"tab\": \"General information\", \"score\": \"392.71\"}", - "Global Facts - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"global_facts\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_global_facts\"" - } - } - }, - { - "evaluation_name": "Jurisprudence", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Jurisprudence", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.889, - "details": { - "description": "min=0.889, mean=0.889, max=0.889, sum=1.778 (2)", - "tab": "Accuracy", - "Jurisprudence - Observed inference time (s)": "{\"description\": \"min=0.439, mean=0.439, max=0.439, sum=0.878 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.43886900389636\"}", - "Jurisprudence - # eval": "{\"description\": \"min=108, mean=108, max=108, sum=216 (2)\", \"tab\": \"General information\", \"score\": \"108.0\"}", - "Jurisprudence - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Jurisprudence - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Jurisprudence - # prompt tokens": "{\"description\": \"min=387.639, mean=387.639, max=387.639, sum=775.278 (2)\", \"tab\": \"General information\", \"score\": \"387.6388888888889\"}", - "Jurisprudence - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"jurisprudence\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_jurisprudence\"" - } - } - }, - { - "evaluation_name": "Philosophy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Philosophy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.859, - "details": { - "description": "min=0.859, mean=0.859, max=0.859, sum=1.717 (2)", - "tab": "Accuracy", - "Philosophy - Observed inference time (s)": "{\"description\": \"min=0.403, mean=0.403, max=0.403, sum=0.807 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.40341131480177117\"}", - "Philosophy - # eval": "{\"description\": \"min=311, mean=311, max=311, sum=622 (2)\", \"tab\": \"General information\", \"score\": \"311.0\"}", - "Philosophy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Philosophy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Philosophy - # prompt tokens": "{\"description\": \"min=322.084, mean=322.084, max=322.084, sum=644.167 (2)\", \"tab\": \"General information\", \"score\": \"322.08360128617363\"}", - "Philosophy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"philosophy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_philosophy\"" - } - } - }, - { - "evaluation_name": "Professional Psychology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Professional Psychology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.891, - "details": { - "description": "min=0.891, mean=0.891, max=0.891, sum=1.781 (2)", - "tab": "Accuracy", - "Professional Medicine - Observed inference time (s)": "{\"description\": \"min=0.483, mean=0.483, max=0.483, sum=0.966 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.48306868356816907\"}", - "Professional Accounting - Observed inference time (s)": "{\"description\": \"min=0.444, mean=0.444, max=0.444, sum=0.888 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.44407470006469296\"}", - "Professional Law - Observed inference time (s)": "{\"description\": \"min=0.578, mean=0.578, max=0.578, sum=1.157 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.578451920053017\"}", - "Professional Psychology - Observed inference time (s)": "{\"description\": \"min=0.469, mean=0.469, max=0.469, sum=0.938 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4690242421393301\"}", - "Professional Medicine - # eval": "{\"description\": \"min=272, mean=272, max=272, sum=544 (2)\", \"tab\": \"General information\", \"score\": \"272.0\"}", - "Professional Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Medicine - # prompt tokens": "{\"description\": \"min=1087.585, mean=1087.585, max=1087.585, sum=2175.169 (2)\", \"tab\": \"General information\", \"score\": \"1087.5845588235295\"}", - "Professional Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Accounting - # eval": "{\"description\": \"min=282, mean=282, max=282, sum=564 (2)\", \"tab\": \"General information\", \"score\": \"282.0\"}", - "Professional Accounting - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Accounting - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # prompt tokens": "{\"description\": \"min=651.592, mean=651.592, max=651.592, sum=1303.184 (2)\", \"tab\": \"General information\", \"score\": \"651.5921985815603\"}", - "Professional Accounting - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Law - # eval": "{\"description\": \"min=1534, mean=1534, max=1534, sum=3068 (2)\", \"tab\": \"General information\", \"score\": \"1534.0\"}", - "Professional Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # prompt tokens": "{\"description\": \"min=1630.787, mean=1630.787, max=1630.787, sum=3261.574 (2)\", \"tab\": \"General information\", \"score\": \"1630.7868318122555\"}", - "Professional Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Psychology - # eval": "{\"description\": \"min=612, mean=612, max=612, sum=1224 (2)\", \"tab\": \"General information\", \"score\": \"612.0\"}", - "Professional Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # prompt tokens": "{\"description\": \"min=568.114, mean=568.114, max=568.114, sum=1136.229 (2)\", \"tab\": \"General information\", \"score\": \"568.1143790849674\"}", - "Professional Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"professional_psychology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_professional_psychology\"" - } - } - }, - { - "evaluation_name": "Us Foreign Policy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Us Foreign Policy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.95, - "details": { - "description": "min=0.95, mean=0.95, max=0.95, sum=1.9 (2)", - "tab": "Accuracy", - "Us Foreign Policy - Observed inference time (s)": "{\"description\": \"min=0.434, mean=0.434, max=0.434, sum=0.869 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.43441893100738527\"}", - "Us Foreign Policy - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Us Foreign Policy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Us Foreign Policy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Us Foreign Policy - # prompt tokens": "{\"description\": \"min=415.79, mean=415.79, max=415.79, sum=831.58 (2)\", \"tab\": \"General information\", \"score\": \"415.79\"}", - "Us Foreign Policy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"us_foreign_policy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_us_foreign_policy\"" - } - } - }, - { - "evaluation_name": "Astronomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Astronomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.934, - "details": { - "description": "min=0.934, mean=0.934, max=0.934, sum=1.868 (2)", - "tab": "Accuracy", - "Astronomy - Observed inference time (s)": "{\"description\": \"min=0.472, mean=0.472, max=0.472, sum=0.944 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4718977307018481\"}", - "Astronomy - # eval": "{\"description\": \"min=152, mean=152, max=152, sum=304 (2)\", \"tab\": \"General information\", \"score\": \"152.0\"}", - "Astronomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Astronomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Astronomy - # prompt tokens": "{\"description\": \"min=572.691, mean=572.691, max=572.691, sum=1145.382 (2)\", \"tab\": \"General information\", \"score\": \"572.6907894736842\"}", - "Astronomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"astronomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_astronomy\"" - } - } - }, - { - "evaluation_name": "Business Ethics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Business Ethics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.79, - "details": { - "description": "min=0.79, mean=0.79, max=0.79, sum=1.58 (2)", - "tab": "Accuracy", - "Business Ethics - Observed inference time (s)": "{\"description\": \"min=0.477, mean=0.477, max=0.477, sum=0.953 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4765148901939392\"}", - "Business Ethics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Business Ethics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Business Ethics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Business Ethics - # prompt tokens": "{\"description\": \"min=562.52, mean=562.52, max=562.52, sum=1125.04 (2)\", \"tab\": \"General information\", \"score\": \"562.52\"}", - "Business Ethics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"business_ethics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_business_ethics\"" - } - } - }, - { - "evaluation_name": "Clinical Knowledge", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Clinical Knowledge", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.845, - "details": { - "description": "min=0.845, mean=0.845, max=0.845, sum=1.691 (2)", - "tab": "Accuracy", - "Clinical Knowledge - Observed inference time (s)": "{\"description\": \"min=0.415, mean=0.415, max=0.415, sum=0.829 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.414557883424579\"}", - "Clinical Knowledge - # eval": "{\"description\": \"min=265, mean=265, max=265, sum=530 (2)\", \"tab\": \"General information\", \"score\": \"265.0\"}", - "Clinical Knowledge - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Clinical Knowledge - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Clinical Knowledge - # prompt tokens": "{\"description\": \"min=390.947, mean=390.947, max=390.947, sum=781.894 (2)\", \"tab\": \"General information\", \"score\": \"390.94716981132075\"}", - "Clinical Knowledge - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"clinical_knowledge\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_clinical_knowledge\"" - } - } - }, - { - "evaluation_name": "Conceptual Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Conceptual Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.868, - "details": { - "description": "min=0.868, mean=0.868, max=0.868, sum=1.736 (2)", - "tab": "Accuracy", - "Conceptual Physics - Observed inference time (s)": "{\"description\": \"min=0.384, mean=0.384, max=0.384, sum=0.767 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3836827186827964\"}", - "Conceptual Physics - # eval": "{\"description\": \"min=235, mean=235, max=235, sum=470 (2)\", \"tab\": \"General information\", \"score\": \"235.0\"}", - "Conceptual Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Conceptual Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Conceptual Physics - # prompt tokens": "{\"description\": \"min=297.838, mean=297.838, max=297.838, sum=595.677 (2)\", \"tab\": \"General information\", \"score\": \"297.83829787234043\"}", - "Conceptual Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"conceptual_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_conceptual_physics\"" - } - } - }, - { - "evaluation_name": "Electrical Engineering", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Electrical Engineering", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.786, - "details": { - "description": "min=0.786, mean=0.786, max=0.786, sum=1.572 (2)", - "tab": "Accuracy", - "Electrical Engineering - Observed inference time (s)": "{\"description\": \"min=0.399, mean=0.399, max=0.399, sum=0.798 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.39915286919166304\"}", - "Electrical Engineering - # eval": "{\"description\": \"min=145, mean=145, max=145, sum=290 (2)\", \"tab\": \"General information\", \"score\": \"145.0\"}", - "Electrical Engineering - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Electrical Engineering - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Electrical Engineering - # prompt tokens": "{\"description\": \"min=433.641, mean=433.641, max=433.641, sum=867.283 (2)\", \"tab\": \"General information\", \"score\": \"433.6413793103448\"}", - "Electrical Engineering - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"electrical_engineering\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_electrical_engineering\"" - } - } - }, - { - "evaluation_name": "Elementary Mathematics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Elementary Mathematics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.807, - "details": { - "description": "min=0.807, mean=0.807, max=0.807, sum=1.614 (2)", - "tab": "Accuracy", - "Elementary Mathematics - Observed inference time (s)": "{\"description\": \"min=0.423, mean=0.423, max=0.423, sum=0.845 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4225258120784053\"}", - "Elementary Mathematics - # eval": "{\"description\": \"min=378, mean=378, max=378, sum=756 (2)\", \"tab\": \"General information\", \"score\": \"378.0\"}", - "Elementary Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Elementary Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Elementary Mathematics - # prompt tokens": "{\"description\": \"min=524.862, mean=524.862, max=524.862, sum=1049.725 (2)\", \"tab\": \"General information\", \"score\": \"524.8624338624338\"}", - "Elementary Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"elementary_mathematics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_elementary_mathematics\"" - } - } - }, - { - "evaluation_name": "Formal Logic", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Formal Logic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.643, - "details": { - "description": "min=0.643, mean=0.643, max=0.643, sum=1.286 (2)", - "tab": "Accuracy", - "Formal Logic - Observed inference time (s)": "{\"description\": \"min=0.486, mean=0.486, max=0.486, sum=0.973 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.48647683007376535\"}", - "Formal Logic - # eval": "{\"description\": \"min=126, mean=126, max=126, sum=252 (2)\", \"tab\": \"General information\", \"score\": \"126.0\"}", - "Formal Logic - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Formal Logic - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Formal Logic - # prompt tokens": "{\"description\": \"min=599.762, mean=599.762, max=599.762, sum=1199.524 (2)\", \"tab\": \"General information\", \"score\": \"599.7619047619048\"}", - "Formal Logic - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"formal_logic\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_formal_logic\"" - } - } - }, - { - "evaluation_name": "High School World History", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on High School World History", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.945, - "details": { - "description": "min=0.945, mean=0.945, max=0.945, sum=1.89 (2)", - "tab": "Accuracy", - "High School Biology - Observed inference time (s)": "{\"description\": \"min=0.436, mean=0.436, max=0.436, sum=0.872 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4360047817230225\"}", - "High School Chemistry - Observed inference time (s)": "{\"description\": \"min=0.413, mean=0.413, max=0.413, sum=0.827 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.41338158710836775\"}", - "High School Computer Science - Observed inference time (s)": "{\"description\": \"min=0.5, mean=0.5, max=0.5, sum=1.001 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5002665758132935\"}", - "High School European History - Observed inference time (s)": "{\"description\": \"min=0.579, mean=0.579, max=0.579, sum=1.158 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.578774525902488\"}", - "High School Geography - Observed inference time (s)": "{\"description\": \"min=0.414, mean=0.414, max=0.414, sum=0.829 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4142996747084338\"}", - "High School Government And Politics - Observed inference time (s)": "{\"description\": \"min=0.43, mean=0.43, max=0.43, sum=0.86 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.43005221001224814\"}", - "High School Macroeconomics - Observed inference time (s)": "{\"description\": \"min=0.416, mean=0.416, max=0.416, sum=0.832 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4160928750649477\"}", - "High School Mathematics - Observed inference time (s)": "{\"description\": \"min=0.423, mean=0.423, max=0.423, sum=0.846 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4231933620240953\"}", - "High School Microeconomics - Observed inference time (s)": "{\"description\": \"min=0.474, mean=0.474, max=0.474, sum=0.948 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4740273321376127\"}", - "High School Physics - Observed inference time (s)": "{\"description\": \"min=0.462, mean=0.462, max=0.462, sum=0.924 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4620048778736039\"}", - "High School Psychology - Observed inference time (s)": "{\"description\": \"min=0.407, mean=0.407, max=0.407, sum=0.813 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.40661886022725235\"}", - "High School Statistics - Observed inference time (s)": "{\"description\": \"min=0.463, mean=0.463, max=0.463, sum=0.926 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.46296725780875597\"}", - "High School US History - Observed inference time (s)": "{\"description\": \"min=0.546, mean=0.546, max=0.546, sum=1.091 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5456923538563299\"}", - "High School World History - Observed inference time (s)": "{\"description\": \"min=0.517, mean=0.517, max=0.517, sum=1.033 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5166646488608188\"}", - "High School Biology - # eval": "{\"description\": \"min=310, mean=310, max=310, sum=620 (2)\", \"tab\": \"General information\", \"score\": \"310.0\"}", - "High School Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Biology - # prompt tokens": "{\"description\": \"min=506.677, mean=506.677, max=506.677, sum=1013.355 (2)\", \"tab\": \"General information\", \"score\": \"506.6774193548387\"}", - "High School Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Chemistry - # eval": "{\"description\": \"min=203, mean=203, max=203, sum=406 (2)\", \"tab\": \"General information\", \"score\": \"203.0\"}", - "High School Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # prompt tokens": "{\"description\": \"min=489.714, mean=489.714, max=489.714, sum=979.429 (2)\", \"tab\": \"General information\", \"score\": \"489.7142857142857\"}", - "High School Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "High School Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # prompt tokens": "{\"description\": \"min=860.78, mean=860.78, max=860.78, sum=1721.56 (2)\", \"tab\": \"General information\", \"score\": \"860.78\"}", - "High School Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School European History - # eval": "{\"description\": \"min=165, mean=165, max=165, sum=330 (2)\", \"tab\": \"General information\", \"score\": \"165.0\"}", - "High School European History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School European History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # prompt tokens": "{\"description\": \"min=2791.073, mean=2791.073, max=2791.073, sum=5582.145 (2)\", \"tab\": \"General information\", \"score\": \"2791.072727272727\"}", - "High School European History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Geography - # eval": "{\"description\": \"min=198, mean=198, max=198, sum=396 (2)\", \"tab\": \"General information\", \"score\": \"198.0\"}", - "High School Geography - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Geography - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # prompt tokens": "{\"description\": \"min=365.045, mean=365.045, max=365.045, sum=730.091 (2)\", \"tab\": \"General information\", \"score\": \"365.04545454545456\"}", - "High School Geography - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Government And Politics - # eval": "{\"description\": \"min=193, mean=193, max=193, sum=386 (2)\", \"tab\": \"General information\", \"score\": \"193.0\"}", - "High School Government And Politics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Government And Politics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # prompt tokens": "{\"description\": \"min=458.824, mean=458.824, max=458.824, sum=917.648 (2)\", \"tab\": \"General information\", \"score\": \"458.8238341968912\"}", - "High School Government And Politics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Macroeconomics - # eval": "{\"description\": \"min=390, mean=390, max=390, sum=780 (2)\", \"tab\": \"General information\", \"score\": \"390.0\"}", - "High School Macroeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Macroeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # prompt tokens": "{\"description\": \"min=364.562, mean=364.562, max=364.562, sum=729.123 (2)\", \"tab\": \"General information\", \"score\": \"364.5615384615385\"}", - "High School Macroeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Mathematics - # eval": "{\"description\": \"min=270, mean=270, max=270, sum=540 (2)\", \"tab\": \"General information\", \"score\": \"270.0\"}", - "High School Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # prompt tokens": "{\"description\": \"min=525.374, mean=525.374, max=525.374, sum=1050.748 (2)\", \"tab\": \"General information\", \"score\": \"525.3740740740741\"}", - "High School Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Microeconomics - # eval": "{\"description\": \"min=238, mean=238, max=238, sum=476 (2)\", \"tab\": \"General information\", \"score\": \"238.0\"}", - "High School Microeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Microeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # prompt tokens": "{\"description\": \"min=392.025, mean=392.025, max=392.025, sum=784.05 (2)\", \"tab\": \"General information\", \"score\": \"392.02521008403363\"}", - "High School Microeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Physics - # eval": "{\"description\": \"min=151, mean=151, max=151, sum=302 (2)\", \"tab\": \"General information\", \"score\": \"151.0\"}", - "High School Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # prompt tokens": "{\"description\": \"min=553.464, mean=553.464, max=553.464, sum=1106.927 (2)\", \"tab\": \"General information\", \"score\": \"553.4635761589404\"}", - "High School Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Psychology - # eval": "{\"description\": \"min=545, mean=545, max=545, sum=1090 (2)\", \"tab\": \"General information\", \"score\": \"545.0\"}", - "High School Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # prompt tokens": "{\"description\": \"min=488.246, mean=488.246, max=488.246, sum=976.492 (2)\", \"tab\": \"General information\", \"score\": \"488.24587155963303\"}", - "High School Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Statistics - # eval": "{\"description\": \"min=216, mean=216, max=216, sum=432 (2)\", \"tab\": \"General information\", \"score\": \"216.0\"}", - "High School Statistics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Statistics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # prompt tokens": "{\"description\": \"min=788.699, mean=788.699, max=788.699, sum=1577.398 (2)\", \"tab\": \"General information\", \"score\": \"788.699074074074\"}", - "High School Statistics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School US History - # eval": "{\"description\": \"min=204, mean=204, max=204, sum=408 (2)\", \"tab\": \"General information\", \"score\": \"204.0\"}", - "High School US History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School US History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # prompt tokens": "{\"description\": \"min=2210.809, mean=2210.809, max=2210.809, sum=4421.618 (2)\", \"tab\": \"General information\", \"score\": \"2210.8088235294117\"}", - "High School US History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School World History - # eval": "{\"description\": \"min=237, mean=237, max=237, sum=474 (2)\", \"tab\": \"General information\", \"score\": \"237.0\"}", - "High School World History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School World History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # prompt tokens": "{\"description\": \"min=1421.27, mean=1421.27, max=1421.27, sum=2842.54 (2)\", \"tab\": \"General information\", \"score\": \"1421.2700421940929\"}", - "High School World History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"high_school_world_history\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_high_school_world_history\"" - } - } - }, - { - "evaluation_name": "Human Sexuality", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Human Sexuality", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.908, - "details": { - "description": "min=0.908, mean=0.908, max=0.908, sum=1.817 (2)", - "tab": "Accuracy", - "Human Aging - Observed inference time (s)": "{\"description\": \"min=0.406, mean=0.406, max=0.406, sum=0.812 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4058152218036053\"}", - "Human Sexuality - Observed inference time (s)": "{\"description\": \"min=0.466, mean=0.466, max=0.466, sum=0.932 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.46620041541470825\"}", - "Human Aging - # eval": "{\"description\": \"min=223, mean=223, max=223, sum=446 (2)\", \"tab\": \"General information\", \"score\": \"223.0\"}", - "Human Aging - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Aging - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Aging - # prompt tokens": "{\"description\": \"min=312.906, mean=312.906, max=312.906, sum=625.812 (2)\", \"tab\": \"General information\", \"score\": \"312.90582959641256\"}", - "Human Aging - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Human Sexuality - # eval": "{\"description\": \"min=131, mean=131, max=131, sum=262 (2)\", \"tab\": \"General information\", \"score\": \"131.0\"}", - "Human Sexuality - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Sexuality - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # prompt tokens": "{\"description\": \"min=334.183, mean=334.183, max=334.183, sum=668.366 (2)\", \"tab\": \"General information\", \"score\": \"334.1832061068702\"}", - "Human Sexuality - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"human_sexuality\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_human_sexuality\"" - } - } - }, - { - "evaluation_name": "International Law", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on International Law", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.917, - "details": { - "description": "min=0.917, mean=0.917, max=0.917, sum=1.835 (2)", - "tab": "Accuracy", - "International Law - Observed inference time (s)": "{\"description\": \"min=0.461, mean=0.461, max=0.461, sum=0.922 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4608367139642889\"}", - "International Law - # eval": "{\"description\": \"min=121, mean=121, max=121, sum=242 (2)\", \"tab\": \"General information\", \"score\": \"121.0\"}", - "International Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "International Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "International Law - # prompt tokens": "{\"description\": \"min=632.851, mean=632.851, max=632.851, sum=1265.702 (2)\", \"tab\": \"General information\", \"score\": \"632.8512396694215\"}", - "International Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"international_law\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_international_law\"" - } - } - }, - { - "evaluation_name": "Logical Fallacies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Logical Fallacies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.871, - "details": { - "description": "min=0.871, mean=0.871, max=0.871, sum=1.742 (2)", - "tab": "Accuracy", - "Logical Fallacies - Observed inference time (s)": "{\"description\": \"min=0.432, mean=0.432, max=0.432, sum=0.864 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4321035870745138\"}", - "Logical Fallacies - # eval": "{\"description\": \"min=163, mean=163, max=163, sum=326 (2)\", \"tab\": \"General information\", \"score\": \"163.0\"}", - "Logical Fallacies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Logical Fallacies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Logical Fallacies - # prompt tokens": "{\"description\": \"min=442.595, mean=442.595, max=442.595, sum=885.19 (2)\", \"tab\": \"General information\", \"score\": \"442.5950920245399\"}", - "Logical Fallacies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"logical_fallacies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_logical_fallacies\"" - } - } - }, - { - "evaluation_name": "Machine Learning", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Machine Learning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.759, - "details": { - "description": "min=0.759, mean=0.759, max=0.759, sum=1.518 (2)", - "tab": "Accuracy", - "Machine Learning - Observed inference time (s)": "{\"description\": \"min=0.463, mean=0.463, max=0.463, sum=0.926 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.46302694933755056\"}", - "Machine Learning - # eval": "{\"description\": \"min=112, mean=112, max=112, sum=224 (2)\", \"tab\": \"General information\", \"score\": \"112.0\"}", - "Machine Learning - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Machine Learning - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Machine Learning - # prompt tokens": "{\"description\": \"min=661.054, mean=661.054, max=661.054, sum=1322.107 (2)\", \"tab\": \"General information\", \"score\": \"661.0535714285714\"}", - "Machine Learning - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"machine_learning\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_machine_learning\"" - } - } - }, - { - "evaluation_name": "Management", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Management", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.932, - "details": { - "description": "min=0.932, mean=0.932, max=0.932, sum=1.864 (2)", - "tab": "Accuracy", - "Management - Observed inference time (s)": "{\"description\": \"min=0.446, mean=0.446, max=0.446, sum=0.891 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4455798760201167\"}", - "Management - # eval": "{\"description\": \"min=103, mean=103, max=103, sum=206 (2)\", \"tab\": \"General information\", \"score\": \"103.0\"}", - "Management - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Management - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Management - # prompt tokens": "{\"description\": \"min=276.796, mean=276.796, max=276.796, sum=553.592 (2)\", \"tab\": \"General information\", \"score\": \"276.79611650485435\"}", - "Management - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"management\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_management\"" - } - } - }, - { - "evaluation_name": "Marketing", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Marketing", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.962, - "details": { - "description": "min=0.962, mean=0.962, max=0.962, sum=1.923 (2)", - "tab": "Accuracy", - "Marketing - Observed inference time (s)": "{\"description\": \"min=0.421, mean=0.421, max=0.421, sum=0.843 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4213859372668796\"}", - "Marketing - # eval": "{\"description\": \"min=234, mean=234, max=234, sum=468 (2)\", \"tab\": \"General information\", \"score\": \"234.0\"}", - "Marketing - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Marketing - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Marketing - # prompt tokens": "{\"description\": \"min=397.218, mean=397.218, max=397.218, sum=794.436 (2)\", \"tab\": \"General information\", \"score\": \"397.21794871794873\"}", - "Marketing - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"marketing\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_marketing\"" - } - } - }, - { - "evaluation_name": "Medical Genetics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Medical Genetics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.94, - "details": { - "description": "min=0.94, mean=0.94, max=0.94, sum=1.88 (2)", - "tab": "Accuracy", - "Medical Genetics - Observed inference time (s)": "{\"description\": \"min=0.411, mean=0.411, max=0.411, sum=0.823 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.41135803937911986\"}", - "Medical Genetics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Medical Genetics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Medical Genetics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Medical Genetics - # prompt tokens": "{\"description\": \"min=334, mean=334, max=334, sum=668 (2)\", \"tab\": \"General information\", \"score\": \"334.0\"}", - "Medical Genetics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"medical_genetics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_medical_genetics\"" - } - } - }, - { - "evaluation_name": "Miscellaneous", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Miscellaneous", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.949, - "details": { - "description": "min=0.949, mean=0.949, max=0.949, sum=1.898 (2)", - "tab": "Accuracy", - "Miscellaneous - Observed inference time (s)": "{\"description\": \"min=0.451, mean=0.451, max=0.451, sum=0.901 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4505587230088001\"}", - "Miscellaneous - # eval": "{\"description\": \"min=783, mean=783, max=783, sum=1566 (2)\", \"tab\": \"General information\", \"score\": \"783.0\"}", - "Miscellaneous - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Miscellaneous - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Miscellaneous - # prompt tokens": "{\"description\": \"min=292.925, mean=292.925, max=292.925, sum=585.849 (2)\", \"tab\": \"General information\", \"score\": \"292.92464878671774\"}", - "Miscellaneous - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"miscellaneous\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_miscellaneous\"" - } - } - }, - { - "evaluation_name": "Moral Scenarios", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Moral Scenarios", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.902, - "details": { - "description": "min=0.902, mean=0.902, max=0.902, sum=1.803 (2)", - "tab": "Accuracy", - "Moral Disputes - Observed inference time (s)": "{\"description\": \"min=0.428, mean=0.428, max=0.428, sum=0.856 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4281756044123214\"}", - "Moral Scenarios - Observed inference time (s)": "{\"description\": \"min=0.445, mean=0.445, max=0.445, sum=0.89 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.44513606945229645\"}", - "Moral Disputes - # eval": "{\"description\": \"min=346, mean=346, max=346, sum=692 (2)\", \"tab\": \"General information\", \"score\": \"346.0\"}", - "Moral Disputes - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Disputes - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Disputes - # prompt tokens": "{\"description\": \"min=469.145, mean=469.145, max=469.145, sum=938.289 (2)\", \"tab\": \"General information\", \"score\": \"469.1445086705202\"}", - "Moral Disputes - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Moral Scenarios - # eval": "{\"description\": \"min=895, mean=895, max=895, sum=1790 (2)\", \"tab\": \"General information\", \"score\": \"895.0\"}", - "Moral Scenarios - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Scenarios - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # prompt tokens": "{\"description\": \"min=649.455, mean=649.455, max=649.455, sum=1298.909 (2)\", \"tab\": \"General information\", \"score\": \"649.454748603352\"}", - "Moral Scenarios - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"moral_scenarios\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_moral_scenarios\"" - } - } - }, - { - "evaluation_name": "Nutrition", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Nutrition", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.892, - "details": { - "description": "min=0.892, mean=0.892, max=0.892, sum=1.784 (2)", - "tab": "Accuracy", - "Nutrition - Observed inference time (s)": "{\"description\": \"min=0.446, mean=0.446, max=0.446, sum=0.892 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4460979816960354\"}", - "Nutrition - # eval": "{\"description\": \"min=306, mean=306, max=306, sum=612 (2)\", \"tab\": \"General information\", \"score\": \"306.0\"}", - "Nutrition - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Nutrition - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Nutrition - # prompt tokens": "{\"description\": \"min=579.817, mean=579.817, max=579.817, sum=1159.634 (2)\", \"tab\": \"General information\", \"score\": \"579.8169934640523\"}", - "Nutrition - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"nutrition\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_nutrition\"" - } - } - }, - { - "evaluation_name": "Prehistory", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Prehistory", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.926, - "details": { - "description": "min=0.926, mean=0.926, max=0.926, sum=1.852 (2)", - "tab": "Accuracy", - "Prehistory - Observed inference time (s)": "{\"description\": \"min=0.426, mean=0.426, max=0.426, sum=0.852 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.42610209665180726\"}", - "Prehistory - # eval": "{\"description\": \"min=324, mean=324, max=324, sum=648 (2)\", \"tab\": \"General information\", \"score\": \"324.0\"}", - "Prehistory - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Prehistory - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Prehistory - # prompt tokens": "{\"description\": \"min=507.559, mean=507.559, max=507.559, sum=1015.117 (2)\", \"tab\": \"General information\", \"score\": \"507.55864197530866\"}", - "Prehistory - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"prehistory\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_prehistory\"" - } - } - }, - { - "evaluation_name": "Public Relations", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Public Relations", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.745, - "details": { - "description": "min=0.745, mean=0.745, max=0.745, sum=1.491 (2)", - "tab": "Accuracy", - "Public Relations - Observed inference time (s)": "{\"description\": \"min=0.496, mean=0.496, max=0.496, sum=0.992 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.49601870450106533\"}", - "Public Relations - # eval": "{\"description\": \"min=110, mean=110, max=110, sum=220 (2)\", \"tab\": \"General information\", \"score\": \"110.0\"}", - "Public Relations - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Public Relations - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Public Relations - # prompt tokens": "{\"description\": \"min=398.318, mean=398.318, max=398.318, sum=796.636 (2)\", \"tab\": \"General information\", \"score\": \"398.3181818181818\"}", - "Public Relations - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"public_relations\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_public_relations\"" - } - } - }, - { - "evaluation_name": "Security Studies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Security Studies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.861, - "details": { - "description": "min=0.861, mean=0.861, max=0.861, sum=1.722 (2)", - "tab": "Accuracy", - "Security Studies - Observed inference time (s)": "{\"description\": \"min=0.471, mean=0.471, max=0.471, sum=0.941 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.47064581306613223\"}", - "Security Studies - # eval": "{\"description\": \"min=245, mean=245, max=245, sum=490 (2)\", \"tab\": \"General information\", \"score\": \"245.0\"}", - "Security Studies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Security Studies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Security Studies - # prompt tokens": "{\"description\": \"min=1157.473, mean=1157.473, max=1157.473, sum=2314.947 (2)\", \"tab\": \"General information\", \"score\": \"1157.4734693877551\"}", - "Security Studies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"security_studies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_security_studies\"" - } - } - }, - { - "evaluation_name": "Sociology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Sociology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.93, - "details": { - "description": "min=0.93, mean=0.93, max=0.93, sum=1.861 (2)", - "tab": "Accuracy", - "Sociology - Observed inference time (s)": "{\"description\": \"min=0.43, mean=0.43, max=0.43, sum=0.86 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.42976075143956427\"}", - "Sociology - # eval": "{\"description\": \"min=201, mean=201, max=201, sum=402 (2)\", \"tab\": \"General information\", \"score\": \"201.0\"}", - "Sociology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Sociology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Sociology - # prompt tokens": "{\"description\": \"min=438.522, mean=438.522, max=438.522, sum=877.045 (2)\", \"tab\": \"General information\", \"score\": \"438.5223880597015\"}", - "Sociology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"sociology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_sociology\"" - } - } - }, - { - "evaluation_name": "Virology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Virology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.596, - "details": { - "description": "min=0.596, mean=0.596, max=0.596, sum=1.193 (2)", - "tab": "Accuracy", - "Virology - Observed inference time (s)": "{\"description\": \"min=0.42, mean=0.42, max=0.42, sum=0.84 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.42023470890091125\"}", - "Virology - # eval": "{\"description\": \"min=166, mean=166, max=166, sum=332 (2)\", \"tab\": \"General information\", \"score\": \"166.0\"}", - "Virology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Virology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Virology - # prompt tokens": "{\"description\": \"min=336.09, mean=336.09, max=336.09, sum=672.181 (2)\", \"tab\": \"General information\", \"score\": \"336.0903614457831\"}", - "Virology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"virology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_virology\"" - } - } - }, - { - "evaluation_name": "World Religions", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on World Religions", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.877, - "details": { - "description": "min=0.877, mean=0.877, max=0.877, sum=1.754 (2)", - "tab": "Accuracy", - "World Religions - Observed inference time (s)": "{\"description\": \"min=0.451, mean=0.451, max=0.451, sum=0.901 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4507097779658803\"}", - "World Religions - # eval": "{\"description\": \"min=171, mean=171, max=171, sum=342 (2)\", \"tab\": \"General information\", \"score\": \"171.0\"}", - "World Religions - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "World Religions - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "World Religions - # prompt tokens": "{\"description\": \"min=268.561, mean=268.561, max=268.561, sum=537.123 (2)\", \"tab\": \"General information\", \"score\": \"268.56140350877195\"}", - "World Religions - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"world_religions\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_world_religions\"" - } - } - }, - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.517, - "details": { - "description": "", - "tab": "Efficiency" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/openai_gpt-4-1-2025-04-14-fc.json b/data/models/openai_gpt-4-1-2025-04-14-fc.json deleted file mode 100644 index da3de3a2a080e6308a15ee3019848a9f705f083f..0000000000000000000000000000000000000000 --- a/data/models/openai_gpt-4-1-2025-04-14-fc.json +++ /dev/null @@ -1,851 +0,0 @@ -{ - "model_info": { - "name": "GPT-4.1-2025-04-14 (FC)", - "id": "openai/gpt-4-1-2025-04-14-fc", - "developer": "OpenAI", - "additional_details": { - "raw_model_name": "GPT-4.1-2025-04-14 (FC)", - "organization": "OpenAI", - "license": "Proprietary", - "mode": "FC", - "model_link": "https://openai.com/index/gpt-4-1/" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/openai/gpt-4-1-2025-04-14-fc/1775236112.375828", - "retrieved_timestamp": "1775236112.375828", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 20.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 53.96 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 100.75 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 1.63 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 3.05 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 4.01 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 82.79 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 72.67 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 89.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 88.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 81.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 69.95 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 69.38 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 70.28 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 56.25 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 70.83 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 38.88 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 47.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 32.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 32.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 43.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 68.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 67.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 69.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 23.87 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 16.13 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 18.06 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 37.42 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 87.5 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 86.52 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/openai_gpt-4-1-2025-04-14-prompt.json b/data/models/openai_gpt-4-1-2025-04-14-prompt.json deleted file mode 100644 index f85bfb37431a345cb807556eef7d95318a0e53e9..0000000000000000000000000000000000000000 --- a/data/models/openai_gpt-4-1-2025-04-14-prompt.json +++ /dev/null @@ -1,905 +0,0 @@ -{ - "model_info": { - "name": "GPT-4.1-2025-04-14 (Prompt)", - "id": "openai/gpt-4-1-2025-04-14-prompt", - "developer": "OpenAI", - "additional_details": { - "raw_model_name": "GPT-4.1-2025-04-14 (Prompt)", - "organization": "OpenAI", - "license": "Proprietary", - "mode": "Prompt", - "model_link": "https://openai.com/index/gpt-4-1/" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/openai/gpt-4-1-2025-04-14-prompt/1775236112.3887758", - "retrieved_timestamp": "1775236112.3887758", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 45.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 39.38 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 145.85 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 1.2 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 3.23 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 2.53 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 88.69 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 78.25 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 93.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 94.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 89.0 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 78.9 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 84.88 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 77.4 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 87.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 75.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 9.75 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 10.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 11.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 8.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 9.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 35.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 40.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 30.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 21.51 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 9.68 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 19.35 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 35.48 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 100.0 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 83.99 - } - }, - { - "evaluation_result_id": "bfcl.format_sensitivity.max_delta::max_delta", - "evaluation_name": "bfcl.format_sensitivity.max_delta", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.format_sensitivity.max_delta", - "metric_name": "Format sensitivity max delta", - "metric_kind": "difference", - "metric_unit": "percentage_points", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Format Sensitivity Max Delta" - } - }, - "score_details": { - "score": 23.5 - } - }, - { - "evaluation_result_id": "bfcl.format_sensitivity.stddev::stddev", - "evaluation_name": "bfcl.format_sensitivity.stddev", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.format_sensitivity.stddev", - "metric_name": "Format sensitivity standard deviation", - "metric_kind": "difference", - "metric_unit": "percentage_points", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Format Sensitivity Standard Deviation" - } - }, - "score_details": { - "score": 6.18 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/openai_gpt-4-1-mini-2025-04-14-fc.json b/data/models/openai_gpt-4-1-mini-2025-04-14-fc.json deleted file mode 100644 index bb4e1b17f033cf871d39bb463e97dab08ba9e01b..0000000000000000000000000000000000000000 --- a/data/models/openai_gpt-4-1-mini-2025-04-14-fc.json +++ /dev/null @@ -1,851 +0,0 @@ -{ - "model_info": { - "name": "GPT-4.1-mini-2025-04-14 (FC)", - "id": "openai/gpt-4-1-mini-2025-04-14-fc", - "developer": "OpenAI", - "additional_details": { - "raw_model_name": "GPT-4.1-mini-2025-04-14 (FC)", - "organization": "OpenAI", - "license": "Proprietary", - "mode": "FC", - "model_link": "https://openai.com/index/gpt-4-1/" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/openai/gpt-4-1-mini-2025-04-14-fc/1775236112.3793862", - "retrieved_timestamp": "1775236112.3793862", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 27.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 50.45 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 19.25 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 1.32 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 3.65 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 2.4 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 83.83 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 73.33 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 89.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 91.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 82.0 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 68.84 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 67.05 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 69.8 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 43.75 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 62.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 34.13 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 43.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 22.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 30.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 40.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 57.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 62.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 52.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 26.88 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 22.58 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 16.13 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 41.94 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 81.25 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 81.69 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/openai_gpt-4-1-mini-2025-04-14-prompt.json b/data/models/openai_gpt-4-1-mini-2025-04-14-prompt.json deleted file mode 100644 index 0993a932783bdfb8a2359fe9c1a2121da9bcb74c..0000000000000000000000000000000000000000 --- a/data/models/openai_gpt-4-1-mini-2025-04-14-prompt.json +++ /dev/null @@ -1,905 +0,0 @@ -{ - "model_info": { - "name": "GPT-4.1-mini-2025-04-14 (Prompt)", - "id": "openai/gpt-4-1-mini-2025-04-14-prompt", - "developer": "OpenAI", - "additional_details": { - "raw_model_name": "GPT-4.1-mini-2025-04-14 (Prompt)", - "organization": "OpenAI", - "license": "Proprietary", - "mode": "Prompt", - "model_link": "https://openai.com/index/gpt-4-1/" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/openai/gpt-4-1-mini-2025-04-14-prompt/1775236112.4010031", - "retrieved_timestamp": "1775236112.4010031", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 67.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 29.73 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 20.52 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 1.36 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 4.5 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 3.38 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 84.6 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 74.92 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 92.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 87.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 83.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 74.76 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 80.62 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 73.31 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 81.25 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 70.83 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 2.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 1.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 4.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 2.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 1.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 4.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 7.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 1.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 24.3 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 20.65 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 13.55 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 38.71 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 87.5 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 73.88 - } - }, - { - "evaluation_result_id": "bfcl.format_sensitivity.max_delta::max_delta", - "evaluation_name": "bfcl.format_sensitivity.max_delta", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.format_sensitivity.max_delta", - "metric_name": "Format sensitivity max delta", - "metric_kind": "difference", - "metric_unit": "percentage_points", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Format Sensitivity Max Delta" - } - }, - "score_details": { - "score": 45.0 - } - }, - { - "evaluation_result_id": "bfcl.format_sensitivity.stddev::stddev", - "evaluation_name": "bfcl.format_sensitivity.stddev", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.format_sensitivity.stddev", - "metric_name": "Format sensitivity standard deviation", - "metric_kind": "difference", - "metric_unit": "percentage_points", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Format Sensitivity Standard Deviation" - } - }, - "score_details": { - "score": 13.33 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/openai_gpt-4-1-nano-2025-04-14-fc.json b/data/models/openai_gpt-4-1-nano-2025-04-14-fc.json deleted file mode 100644 index c4945f882aff4bd2a6e88dd0608a9e9e060a6218..0000000000000000000000000000000000000000 --- a/data/models/openai_gpt-4-1-nano-2025-04-14-fc.json +++ /dev/null @@ -1,851 +0,0 @@ -{ - "model_info": { - "name": "GPT-4.1-nano-2025-04-14 (FC)", - "id": "openai/gpt-4-1-nano-2025-04-14-fc", - "developer": "OpenAI", - "additional_details": { - "raw_model_name": "GPT-4.1-nano-2025-04-14 (FC)", - "organization": "OpenAI", - "license": "Proprietary", - "mode": "FC", - "model_link": "https://openai.com/index/gpt-4-1/" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/openai/gpt-4-1-nano-2025-04-14-fc/1775236112.39605", - "retrieved_timestamp": "1775236112.39605", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 58.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 33.05 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 5.66 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 1.44 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 10.84 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 2.26 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 72.98 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 59.92 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 79.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 84.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 68.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 60.77 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 58.14 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 61.44 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 68.75 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 54.17 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 23.62 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 39.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 7.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 17.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 30.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 11.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 13.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 9.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 18.92 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 10.32 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 19.35 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 27.1 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 93.75 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 66.0 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/openai_gpt-4-1-nano-2025-04-14-prompt.json b/data/models/openai_gpt-4-1-nano-2025-04-14-prompt.json deleted file mode 100644 index 3d55b5e38f2fbe821c98868a2a1fa3b1bacc3c87..0000000000000000000000000000000000000000 --- a/data/models/openai_gpt-4-1-nano-2025-04-14-prompt.json +++ /dev/null @@ -1,905 +0,0 @@ -{ - "model_info": { - "name": "GPT-4.1-nano-2025-04-14 (Prompt)", - "id": "openai/gpt-4-1-nano-2025-04-14-prompt", - "developer": "OpenAI", - "additional_details": { - "raw_model_name": "GPT-4.1-nano-2025-04-14 (Prompt)", - "organization": "OpenAI", - "license": "Proprietary", - "mode": "Prompt", - "model_link": "https://openai.com/index/gpt-4-1/" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/openai/gpt-4-1-nano-2025-04-14-prompt/1775236112.413387", - "retrieved_timestamp": "1775236112.413387", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 90.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 24.88 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 7.42 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 1.02 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 7.3 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 1.88 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 72.44 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 68.75 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 63.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 85.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 72.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 50.33 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 63.18 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 46.53 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 87.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 54.17 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 2.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 2.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 1.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 2.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 2.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 1.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 2.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 1.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 16.77 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 9.03 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 14.19 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 27.1 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 68.75 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 83.44 - } - }, - { - "evaluation_result_id": "bfcl.format_sensitivity.max_delta::max_delta", - "evaluation_name": "bfcl.format_sensitivity.max_delta", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.format_sensitivity.max_delta", - "metric_name": "Format sensitivity max delta", - "metric_kind": "difference", - "metric_unit": "percentage_points", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Format Sensitivity Max Delta" - } - }, - "score_details": { - "score": 73.0 - } - }, - { - "evaluation_result_id": "bfcl.format_sensitivity.stddev::stddev", - "evaluation_name": "bfcl.format_sensitivity.stddev", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.format_sensitivity.stddev", - "metric_name": "Format sensitivity standard deviation", - "metric_kind": "difference", - "metric_unit": "percentage_points", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Format Sensitivity Standard Deviation" - } - }, - "score_details": { - "score": 17.08 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/openai_gpt-4-1106-preview.json b/data/models/openai_gpt-4-1106-preview.json deleted file mode 100644 index 873806f9a435ed17f0daaeb2bffc2c2072b5be6c..0000000000000000000000000000000000000000 --- a/data/models/openai_gpt-4-1106-preview.json +++ /dev/null @@ -1,1901 +0,0 @@ -{ - "model_info": { - "name": "GPT-4 Turbo 1106 preview", - "id": "openai/gpt-4-1106-preview", - "developer": "OpenAI", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_lite/openai_gpt-4-1106-preview/1774096306.427425", - "retrieved_timestamp": "1774096306.427425", - "source_metadata": { - "source_name": "helm_lite", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_lite", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_lite", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.698, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.3935580524344569\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.727, - "details": { - "description": "min=0.727, mean=0.727, max=0.727, sum=0.727 (1)", - "tab": "Accuracy", - "NarrativeQA - Observed inference time (s)": "{\"description\": \"min=1.068, mean=1.068, max=1.068, sum=1.068 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.068114177945634\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=3522.67, mean=3522.67, max=3522.67, sum=3522.67 (1)\", \"tab\": \"General information\", \"score\": \"3522.6704225352114\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=9.885, mean=9.885, max=9.885, sum=9.885 (1)\", \"tab\": \"General information\", \"score\": \"9.88450704225352\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (closed-book)", - "source_data": { - "dataset_name": "NaturalQuestions (closed-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (closed-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.435, - "details": { - "description": "min=0.435, mean=0.435, max=0.435, sum=0.435 (1)", - "tab": "Accuracy", - "NaturalQuestions (open-book) - Observed inference time (s)": "{\"description\": \"min=0.867, mean=0.867, max=0.867, sum=0.867 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.8667134034633637\"}", - "NaturalQuestions (closed-book) - Observed inference time (s)": "{\"description\": \"min=1.131, mean=1.131, max=1.131, sum=1.131 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.1312835423946381\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1762.593, mean=1762.593, max=1762.593, sum=1762.593 (1)\", \"tab\": \"General information\", \"score\": \"1762.593\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=8.753, mean=8.753, max=8.753, sum=8.753 (1)\", \"tab\": \"General information\", \"score\": \"8.753\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=173.127, mean=173.127, max=173.127, sum=173.127 (1)\", \"tab\": \"General information\", \"score\": \"173.127\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=14.157, mean=14.157, max=14.157, sum=14.157 (1)\", \"tab\": \"General information\", \"score\": \"14.157\"}" - } - }, - "generation_config": { - "additional_details": { - "mode": "\"closedbook\"" - } - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.95, - "details": { - "description": "min=0.95, mean=0.95, max=0.95, sum=0.95 (1)", - "tab": "Accuracy", - "OpenbookQA - Observed inference time (s)": "{\"description\": \"min=0.512, mean=0.512, max=0.512, sum=0.512 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.5122070140838623\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=242.782, mean=242.782, max=242.782, sum=242.782 (1)\", \"tab\": \"General information\", \"score\": \"242.782\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "dataset": "\"openbookqa\"", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.699, - "details": { - "description": "min=0.47, mean=0.699, max=0.96, sum=3.495 (5)", - "tab": "Accuracy", - "MMLU - Observed inference time (s)": "{\"description\": \"min=0.397, mean=0.447, max=0.515, sum=2.236 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.4471675806380155\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=366.44, mean=460.72, max=607.43, sum=2303.6 (5)\", \"tab\": \"General information\", \"score\": \"460.71996491228066\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"college_chemistry\", \"computer_security\", \"econometrics\", \"us_foreign_policy\"]", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MATH", - "source_data": { - "dataset_name": "MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Equivalent (CoT) on MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.857, - "details": { - "description": "min=0.711, mean=0.857, max=0.97, sum=5.998 (7)", - "tab": "Accuracy", - "MATH - Observed inference time (s)": "{\"description\": \"min=10.989, mean=12.704, max=15.09, sum=88.928 (7)\", \"tab\": \"Efficiency\", \"score\": \"12.704059314714486\"}", - "MATH - # eval": "{\"description\": \"min=30, mean=62.429, max=135, sum=437 (7)\", \"tab\": \"General information\", \"score\": \"62.42857142857143\"}", - "MATH - # train": "{\"description\": \"min=8, mean=8, max=8, sum=56 (7)\", \"tab\": \"General information\", \"score\": \"8.0\"}", - "MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MATH - # prompt tokens": "{\"description\": \"min=942.363, mean=1323.911, max=2258.577, sum=9267.376 (7)\", \"tab\": \"General information\", \"score\": \"1323.910874184069\"}", - "MATH - # output tokens": "{\"description\": \"min=122.465, mean=161.876, max=186.673, sum=1133.133 (7)\", \"tab\": \"General information\", \"score\": \"161.87607288445722\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"algebra\", \"counting_and_probability\", \"geometry\", \"intermediate_algebra\", \"number_theory\", \"prealgebra\", \"precalculus\"]", - "level": "\"1\"", - "use_official_examples": "\"False\"", - "use_chain_of_thought": "\"True\"" - } - } - }, - { - "evaluation_name": "GSM8K", - "source_data": { - "dataset_name": "GSM8K", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on GSM8K", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.668, - "details": { - "description": "min=0.668, mean=0.668, max=0.668, sum=0.668 (1)", - "tab": "Accuracy", - "GSM8K - Observed inference time (s)": "{\"description\": \"min=5.738, mean=5.738, max=5.738, sum=5.738 (1)\", \"tab\": \"Efficiency\", \"score\": \"5.738402992963791\"}", - "GSM8K - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "GSM8K - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "GSM8K - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GSM8K - # prompt tokens": "{\"description\": \"min=1020.035, mean=1020.035, max=1020.035, sum=1020.035 (1)\", \"tab\": \"General information\", \"score\": \"1020.035\"}", - "GSM8K - # output tokens": "{\"description\": \"min=98.073, mean=98.073, max=98.073, sum=98.073 (1)\", \"tab\": \"General information\", \"score\": \"98.073\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "LegalBench", - "source_data": { - "dataset_name": "LegalBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on LegalBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.626, - "details": { - "description": "min=0.368, mean=0.626, max=0.989, sum=3.13 (5)", - "tab": "Accuracy", - "LegalBench - Observed inference time (s)": "{\"description\": \"min=0.445, mean=0.603, max=0.98, sum=3.017 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.6033123332286346\"}", - "LegalBench - # eval": "{\"description\": \"min=95, mean=409.4, max=1000, sum=2047 (5)\", \"tab\": \"General information\", \"score\": \"409.4\"}", - "LegalBench - # train": "{\"description\": \"min=4, mean=4.8, max=5, sum=24 (5)\", \"tab\": \"General information\", \"score\": \"4.8\"}", - "LegalBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "LegalBench - # prompt tokens": "{\"description\": \"min=253.442, mean=1570.163, max=6357.388, sum=7850.815 (5)\", \"tab\": \"General information\", \"score\": \"1570.162971355988\"}", - "LegalBench - # output tokens": "{\"description\": \"min=1, mean=1.458, max=2.695, sum=7.291 (5)\", \"tab\": \"General information\", \"score\": \"1.458208948802524\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"abercrombie\", \"corporate_lobbying\", \"function_of_decision_section\", \"international_citizenship_questions\", \"proa\"]" - } - } - }, - { - "evaluation_name": "MedQA", - "source_data": { - "dataset_name": "MedQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MedQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.817, - "details": { - "description": "min=0.817, mean=0.817, max=0.817, sum=0.817 (1)", - "tab": "Accuracy", - "MedQA - Observed inference time (s)": "{\"description\": \"min=0.392, mean=0.392, max=0.392, sum=0.392 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.3924491192190121\"}", - "MedQA - # eval": "{\"description\": \"min=503, mean=503, max=503, sum=503 (1)\", \"tab\": \"General information\", \"score\": \"503.0\"}", - "MedQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MedQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MedQA - # prompt tokens": "{\"description\": \"min=1020.414, mean=1020.414, max=1020.414, sum=1020.414 (1)\", \"tab\": \"General information\", \"score\": \"1020.4135188866799\"}", - "MedQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WMT 2014", - "source_data": { - "dataset_name": "WMT 2014", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "BLEU-4 on WMT 2014", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.205, - "details": { - "description": "min=0.156, mean=0.205, max=0.241, sum=1.023 (5)", - "tab": "Accuracy", - "WMT 2014 - Observed inference time (s)": "{\"description\": \"min=1.797, mean=2.1, max=2.349, sum=10.502 (5)\", \"tab\": \"Efficiency\", \"score\": \"2.1004491326059744\"}", - "WMT 2014 - # eval": "{\"description\": \"min=503, mean=568.8, max=832, sum=2844 (5)\", \"tab\": \"General information\", \"score\": \"568.8\"}", - "WMT 2014 - # train": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "WMT 2014 - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WMT 2014 - # prompt tokens": "{\"description\": \"min=169.901, mean=193.043, max=213.185, sum=965.213 (5)\", \"tab\": \"General information\", \"score\": \"193.04258583116683\"}", - "WMT 2014 - # output tokens": "{\"description\": \"min=26.229, mean=26.996, max=28.59, sum=134.98 (5)\", \"tab\": \"General information\", \"score\": \"26.995945480960394\"}" - } - }, - "generation_config": { - "additional_details": { - "language_pair": "[\"cs-en\", \"de-en\", \"fr-en\", \"hi-en\", \"ru-en\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_mmlu/openai_gpt-4-1106-preview/1774096312.00548", - "retrieved_timestamp": "1774096312.00548", - "source_metadata": { - "source_name": "helm_mmlu", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_mmlu", - "evaluation_results": [ - { - "evaluation_name": "MMLU All Subjects", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU All Subjects", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.796, - "details": { - "description": "min=0.093, mean=0.796, max=0.979, sum=90.688 (114)", - "tab": "Accuracy", - "MMLU All Subjects - Observed inference time (s)": "{\"description\": \"min=0.397, mean=0.537, max=0.852, sum=61.247 (114)\", \"tab\": \"Efficiency\", \"score\": \"0.5372507053364665\"}", - "MMLU All Subjects - # eval": "{\"description\": \"min=100, mean=246.351, max=1534, sum=28084 (114)\", \"tab\": \"General information\", \"score\": \"246.35087719298247\"}", - "MMLU All Subjects - # train": "{\"description\": \"min=5, mean=5, max=5, sum=570 (114)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU All Subjects - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU All Subjects - # prompt tokens": "{\"description\": \"min=268.561, mean=607.852, max=2791.073, sum=69295.086 (114)\", \"tab\": \"General information\", \"score\": \"607.851634217556\"}", - "MMLU All Subjects - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=114 (114)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_name": "Abstract Algebra", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Abstract Algebra", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.53, - "details": { - "description": "min=0.53, mean=0.53, max=0.53, sum=1.06 (2)", - "tab": "Accuracy", - "Abstract Algebra - Observed inference time (s)": "{\"description\": \"min=0.425, mean=0.425, max=0.425, sum=0.85 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.42504594564437864\"}", - "Abstract Algebra - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Abstract Algebra - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Abstract Algebra - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Abstract Algebra - # prompt tokens": "{\"description\": \"min=366.44, mean=366.44, max=366.44, sum=732.88 (2)\", \"tab\": \"General information\", \"score\": \"366.44\"}", - "Abstract Algebra - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"abstract_algebra\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_abstract_algebra\"" - } - } - }, - { - "evaluation_name": "Anatomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Anatomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.807, - "details": { - "description": "min=0.807, mean=0.807, max=0.807, sum=1.615 (2)", - "tab": "Accuracy", - "Anatomy - Observed inference time (s)": "{\"description\": \"min=0.569, mean=0.569, max=0.569, sum=1.138 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5691532982720269\"}", - "Anatomy - # eval": "{\"description\": \"min=135, mean=135, max=135, sum=270 (2)\", \"tab\": \"General information\", \"score\": \"135.0\"}", - "Anatomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Anatomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Anatomy - # prompt tokens": "{\"description\": \"min=346.978, mean=346.978, max=346.978, sum=693.956 (2)\", \"tab\": \"General information\", \"score\": \"346.97777777777776\"}", - "Anatomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"anatomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_anatomy\"" - } - } - }, - { - "evaluation_name": "College Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on College Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.402, - "details": { - "description": "min=0.402, mean=0.402, max=0.402, sum=0.804 (2)", - "tab": "Accuracy", - "College Chemistry - Observed inference time (s)": "{\"description\": \"min=0.457, mean=0.457, max=0.457, sum=0.913 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.456736900806427\"}", - "College Biology - Observed inference time (s)": "{\"description\": \"min=0.444, mean=0.444, max=0.444, sum=0.888 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.44404302537441254\"}", - "College Computer Science - Observed inference time (s)": "{\"description\": \"min=0.516, mean=0.516, max=0.516, sum=1.033 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.516348373889923\"}", - "College Mathematics - Observed inference time (s)": "{\"description\": \"min=0.534, mean=0.534, max=0.534, sum=1.067 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5335026264190674\"}", - "College Medicine - Observed inference time (s)": "{\"description\": \"min=0.491, mean=0.491, max=0.491, sum=0.982 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4908691348368033\"}", - "College Physics - Observed inference time (s)": "{\"description\": \"min=0.75, mean=0.75, max=0.75, sum=1.499 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7497045245825076\"}", - "College Chemistry - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Chemistry - # prompt tokens": "{\"description\": \"min=542.4, mean=542.4, max=542.4, sum=1084.8 (2)\", \"tab\": \"General information\", \"score\": \"542.4\"}", - "College Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Biology - # eval": "{\"description\": \"min=144, mean=144, max=144, sum=288 (2)\", \"tab\": \"General information\", \"score\": \"144.0\"}", - "College Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # prompt tokens": "{\"description\": \"min=466.917, mean=466.917, max=466.917, sum=933.833 (2)\", \"tab\": \"General information\", \"score\": \"466.9166666666667\"}", - "College Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # prompt tokens": "{\"description\": \"min=821.39, mean=821.39, max=821.39, sum=1642.78 (2)\", \"tab\": \"General information\", \"score\": \"821.39\"}", - "College Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Mathematics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # prompt tokens": "{\"description\": \"min=587.52, mean=587.52, max=587.52, sum=1175.04 (2)\", \"tab\": \"General information\", \"score\": \"587.52\"}", - "College Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Medicine - # eval": "{\"description\": \"min=173, mean=173, max=173, sum=346 (2)\", \"tab\": \"General information\", \"score\": \"173.0\"}", - "College Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # prompt tokens": "{\"description\": \"min=495.728, mean=495.728, max=495.728, sum=991.457 (2)\", \"tab\": \"General information\", \"score\": \"495.728323699422\"}", - "College Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Physics - # eval": "{\"description\": \"min=102, mean=102, max=102, sum=204 (2)\", \"tab\": \"General information\", \"score\": \"102.0\"}", - "College Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # prompt tokens": "{\"description\": \"min=496.608, mean=496.608, max=496.608, sum=993.216 (2)\", \"tab\": \"General information\", \"score\": \"496.6078431372549\"}", - "College Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"college_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_college_physics\"" - } - } - }, - { - "evaluation_name": "Computer Security", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Computer Security", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.86, - "details": { - "description": "min=0.86, mean=0.86, max=0.86, sum=1.72 (2)", - "tab": "Accuracy", - "Computer Security - Observed inference time (s)": "{\"description\": \"min=0.442, mean=0.442, max=0.442, sum=0.884 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4418716287612915\"}", - "Computer Security - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Computer Security - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Computer Security - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Computer Security - # prompt tokens": "{\"description\": \"min=371.54, mean=371.54, max=371.54, sum=743.08 (2)\", \"tab\": \"General information\", \"score\": \"371.54\"}", - "Computer Security - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"computer_security\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_computer_security\"" - } - } - }, - { - "evaluation_name": "Econometrics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Econometrics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.675, - "details": { - "description": "min=0.675, mean=0.675, max=0.675, sum=1.351 (2)", - "tab": "Accuracy", - "Econometrics - Observed inference time (s)": "{\"description\": \"min=0.515, mean=0.515, max=0.515, sum=1.03 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5149402095560442\"}", - "Econometrics - # eval": "{\"description\": \"min=114, mean=114, max=114, sum=228 (2)\", \"tab\": \"General information\", \"score\": \"114.0\"}", - "Econometrics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Econometrics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Econometrics - # prompt tokens": "{\"description\": \"min=607.43, mean=607.43, max=607.43, sum=1214.86 (2)\", \"tab\": \"General information\", \"score\": \"607.4298245614035\"}", - "Econometrics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"econometrics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_econometrics\"" - } - } - }, - { - "evaluation_name": "Global Facts", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Global Facts", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.58, - "details": { - "description": "min=0.58, mean=0.58, max=0.58, sum=1.16 (2)", - "tab": "Accuracy", - "Global Facts - Observed inference time (s)": "{\"description\": \"min=0.486, mean=0.486, max=0.486, sum=0.973 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4863955807685852\"}", - "Global Facts - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Global Facts - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Global Facts - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Global Facts - # prompt tokens": "{\"description\": \"min=392.71, mean=392.71, max=392.71, sum=785.42 (2)\", \"tab\": \"General information\", \"score\": \"392.71\"}", - "Global Facts - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"global_facts\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_global_facts\"" - } - } - }, - { - "evaluation_name": "Jurisprudence", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Jurisprudence", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.889, - "details": { - "description": "min=0.889, mean=0.889, max=0.889, sum=1.778 (2)", - "tab": "Accuracy", - "Jurisprudence - Observed inference time (s)": "{\"description\": \"min=0.731, mean=0.731, max=0.731, sum=1.462 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7311423023541769\"}", - "Jurisprudence - # eval": "{\"description\": \"min=108, mean=108, max=108, sum=216 (2)\", \"tab\": \"General information\", \"score\": \"108.0\"}", - "Jurisprudence - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Jurisprudence - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Jurisprudence - # prompt tokens": "{\"description\": \"min=387.639, mean=387.639, max=387.639, sum=775.278 (2)\", \"tab\": \"General information\", \"score\": \"387.6388888888889\"}", - "Jurisprudence - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"jurisprudence\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_jurisprudence\"" - } - } - }, - { - "evaluation_name": "Philosophy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Philosophy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.852, - "details": { - "description": "min=0.852, mean=0.852, max=0.852, sum=1.704 (2)", - "tab": "Accuracy", - "Philosophy - Observed inference time (s)": "{\"description\": \"min=0.486, mean=0.486, max=0.486, sum=0.973 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4863421380328212\"}", - "Philosophy - # eval": "{\"description\": \"min=311, mean=311, max=311, sum=622 (2)\", \"tab\": \"General information\", \"score\": \"311.0\"}", - "Philosophy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Philosophy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Philosophy - # prompt tokens": "{\"description\": \"min=322.084, mean=322.084, max=322.084, sum=644.167 (2)\", \"tab\": \"General information\", \"score\": \"322.08360128617363\"}", - "Philosophy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"philosophy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_philosophy\"" - } - } - }, - { - "evaluation_name": "Professional Psychology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Professional Psychology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.887, - "details": { - "description": "min=0.887, mean=0.887, max=0.887, sum=1.775 (2)", - "tab": "Accuracy", - "Professional Medicine - Observed inference time (s)": "{\"description\": \"min=0.551, mean=0.551, max=0.551, sum=1.103 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5514215528964996\"}", - "Professional Accounting - Observed inference time (s)": "{\"description\": \"min=0.54, mean=0.54, max=0.54, sum=1.079 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5395518828791084\"}", - "Professional Law - Observed inference time (s)": "{\"description\": \"min=0.616, mean=0.616, max=0.616, sum=1.232 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6162493903447317\"}", - "Professional Psychology - Observed inference time (s)": "{\"description\": \"min=0.563, mean=0.563, max=0.563, sum=1.126 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5629562961509804\"}", - "Professional Medicine - # eval": "{\"description\": \"min=272, mean=272, max=272, sum=544 (2)\", \"tab\": \"General information\", \"score\": \"272.0\"}", - "Professional Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Medicine - # prompt tokens": "{\"description\": \"min=1087.585, mean=1087.585, max=1087.585, sum=2175.169 (2)\", \"tab\": \"General information\", \"score\": \"1087.5845588235295\"}", - "Professional Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Accounting - # eval": "{\"description\": \"min=282, mean=282, max=282, sum=564 (2)\", \"tab\": \"General information\", \"score\": \"282.0\"}", - "Professional Accounting - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Accounting - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # prompt tokens": "{\"description\": \"min=651.592, mean=651.592, max=651.592, sum=1303.184 (2)\", \"tab\": \"General information\", \"score\": \"651.5921985815603\"}", - "Professional Accounting - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Law - # eval": "{\"description\": \"min=1534, mean=1534, max=1534, sum=3068 (2)\", \"tab\": \"General information\", \"score\": \"1534.0\"}", - "Professional Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # prompt tokens": "{\"description\": \"min=1630.787, mean=1630.787, max=1630.787, sum=3261.574 (2)\", \"tab\": \"General information\", \"score\": \"1630.7868318122555\"}", - "Professional Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Psychology - # eval": "{\"description\": \"min=612, mean=612, max=612, sum=1224 (2)\", \"tab\": \"General information\", \"score\": \"612.0\"}", - "Professional Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # prompt tokens": "{\"description\": \"min=568.114, mean=568.114, max=568.114, sum=1136.229 (2)\", \"tab\": \"General information\", \"score\": \"568.1143790849674\"}", - "Professional Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"professional_psychology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_professional_psychology\"" - } - } - }, - { - "evaluation_name": "Us Foreign Policy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Us Foreign Policy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.96, - "details": { - "description": "min=0.96, mean=0.96, max=0.96, sum=1.92 (2)", - "tab": "Accuracy", - "Us Foreign Policy - Observed inference time (s)": "{\"description\": \"min=0.397, mean=0.397, max=0.397, sum=0.794 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.39724321842193605\"}", - "Us Foreign Policy - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Us Foreign Policy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Us Foreign Policy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Us Foreign Policy - # prompt tokens": "{\"description\": \"min=415.79, mean=415.79, max=415.79, sum=831.58 (2)\", \"tab\": \"General information\", \"score\": \"415.79\"}", - "Us Foreign Policy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"us_foreign_policy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_us_foreign_policy\"" - } - } - }, - { - "evaluation_name": "Astronomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Astronomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.941, - "details": { - "description": "min=0.941, mean=0.941, max=0.941, sum=1.882 (2)", - "tab": "Accuracy", - "Astronomy - Observed inference time (s)": "{\"description\": \"min=0.519, mean=0.519, max=0.519, sum=1.038 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5192367622726842\"}", - "Astronomy - # eval": "{\"description\": \"min=152, mean=152, max=152, sum=304 (2)\", \"tab\": \"General information\", \"score\": \"152.0\"}", - "Astronomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Astronomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Astronomy - # prompt tokens": "{\"description\": \"min=572.691, mean=572.691, max=572.691, sum=1145.382 (2)\", \"tab\": \"General information\", \"score\": \"572.6907894736842\"}", - "Astronomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"astronomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_astronomy\"" - } - } - }, - { - "evaluation_name": "Business Ethics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Business Ethics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.78, - "details": { - "description": "min=0.78, mean=0.78, max=0.78, sum=1.56 (2)", - "tab": "Accuracy", - "Business Ethics - Observed inference time (s)": "{\"description\": \"min=0.495, mean=0.495, max=0.495, sum=0.99 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.49495640993118284\"}", - "Business Ethics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Business Ethics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Business Ethics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Business Ethics - # prompt tokens": "{\"description\": \"min=562.52, mean=562.52, max=562.52, sum=1125.04 (2)\", \"tab\": \"General information\", \"score\": \"562.52\"}", - "Business Ethics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"business_ethics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_business_ethics\"" - } - } - }, - { - "evaluation_name": "Clinical Knowledge", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Clinical Knowledge", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.864, - "details": { - "description": "min=0.864, mean=0.864, max=0.864, sum=1.728 (2)", - "tab": "Accuracy", - "Clinical Knowledge - Observed inference time (s)": "{\"description\": \"min=0.642, mean=0.642, max=0.642, sum=1.284 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6421918509141454\"}", - "Clinical Knowledge - # eval": "{\"description\": \"min=265, mean=265, max=265, sum=530 (2)\", \"tab\": \"General information\", \"score\": \"265.0\"}", - "Clinical Knowledge - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Clinical Knowledge - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Clinical Knowledge - # prompt tokens": "{\"description\": \"min=390.947, mean=390.947, max=390.947, sum=781.894 (2)\", \"tab\": \"General information\", \"score\": \"390.94716981132075\"}", - "Clinical Knowledge - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"clinical_knowledge\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_clinical_knowledge\"" - } - } - }, - { - "evaluation_name": "Conceptual Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Conceptual Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.894, - "details": { - "description": "min=0.894, mean=0.894, max=0.894, sum=1.787 (2)", - "tab": "Accuracy", - "Conceptual Physics - Observed inference time (s)": "{\"description\": \"min=0.586, mean=0.586, max=0.586, sum=1.172 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5859095319788507\"}", - "Conceptual Physics - # eval": "{\"description\": \"min=235, mean=235, max=235, sum=470 (2)\", \"tab\": \"General information\", \"score\": \"235.0\"}", - "Conceptual Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Conceptual Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Conceptual Physics - # prompt tokens": "{\"description\": \"min=297.838, mean=297.838, max=297.838, sum=595.677 (2)\", \"tab\": \"General information\", \"score\": \"297.83829787234043\"}", - "Conceptual Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"conceptual_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_conceptual_physics\"" - } - } - }, - { - "evaluation_name": "Electrical Engineering", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Electrical Engineering", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.772, - "details": { - "description": "min=0.772, mean=0.772, max=0.772, sum=1.545 (2)", - "tab": "Accuracy", - "Electrical Engineering - Observed inference time (s)": "{\"description\": \"min=0.507, mean=0.507, max=0.507, sum=1.014 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5071375830420133\"}", - "Electrical Engineering - # eval": "{\"description\": \"min=145, mean=145, max=145, sum=290 (2)\", \"tab\": \"General information\", \"score\": \"145.0\"}", - "Electrical Engineering - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Electrical Engineering - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Electrical Engineering - # prompt tokens": "{\"description\": \"min=433.641, mean=433.641, max=433.641, sum=867.283 (2)\", \"tab\": \"General information\", \"score\": \"433.6413793103448\"}", - "Electrical Engineering - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"electrical_engineering\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_electrical_engineering\"" - } - } - }, - { - "evaluation_name": "Elementary Mathematics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Elementary Mathematics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.638, - "details": { - "description": "min=0.638, mean=0.638, max=0.638, sum=1.275 (2)", - "tab": "Accuracy", - "Elementary Mathematics - Observed inference time (s)": "{\"description\": \"min=0.486, mean=0.486, max=0.486, sum=0.972 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.48600239034682985\"}", - "Elementary Mathematics - # eval": "{\"description\": \"min=378, mean=378, max=378, sum=756 (2)\", \"tab\": \"General information\", \"score\": \"378.0\"}", - "Elementary Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Elementary Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Elementary Mathematics - # prompt tokens": "{\"description\": \"min=524.862, mean=524.862, max=524.862, sum=1049.725 (2)\", \"tab\": \"General information\", \"score\": \"524.8624338624338\"}", - "Elementary Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"elementary_mathematics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_elementary_mathematics\"" - } - } - }, - { - "evaluation_name": "Formal Logic", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Formal Logic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.651, - "details": { - "description": "min=0.651, mean=0.651, max=0.651, sum=1.302 (2)", - "tab": "Accuracy", - "Formal Logic - Observed inference time (s)": "{\"description\": \"min=0.491, mean=0.491, max=0.491, sum=0.983 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4912937557886517\"}", - "Formal Logic - # eval": "{\"description\": \"min=126, mean=126, max=126, sum=252 (2)\", \"tab\": \"General information\", \"score\": \"126.0\"}", - "Formal Logic - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Formal Logic - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Formal Logic - # prompt tokens": "{\"description\": \"min=599.762, mean=599.762, max=599.762, sum=1199.524 (2)\", \"tab\": \"General information\", \"score\": \"599.7619047619048\"}", - "Formal Logic - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"formal_logic\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_formal_logic\"" - } - } - }, - { - "evaluation_name": "High School World History", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on High School World History", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.958, - "details": { - "description": "min=0.958, mean=0.958, max=0.958, sum=1.916 (2)", - "tab": "Accuracy", - "High School Biology - Observed inference time (s)": "{\"description\": \"min=0.572, mean=0.572, max=0.572, sum=1.144 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5719813362244637\"}", - "High School Chemistry - Observed inference time (s)": "{\"description\": \"min=0.656, mean=0.656, max=0.656, sum=1.312 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6560086276143643\"}", - "High School Computer Science - Observed inference time (s)": "{\"description\": \"min=0.568, mean=0.568, max=0.568, sum=1.137 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5683712005615235\"}", - "High School European History - Observed inference time (s)": "{\"description\": \"min=0.64, mean=0.64, max=0.64, sum=1.28 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6399081995992949\"}", - "High School Geography - Observed inference time (s)": "{\"description\": \"min=0.471, mean=0.471, max=0.471, sum=0.943 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.47148694173254146\"}", - "High School Government And Politics - Observed inference time (s)": "{\"description\": \"min=0.42, mean=0.42, max=0.42, sum=0.84 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.420210268831006\"}", - "High School Macroeconomics - Observed inference time (s)": "{\"description\": \"min=0.445, mean=0.445, max=0.445, sum=0.89 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4451567802673731\"}", - "High School Mathematics - Observed inference time (s)": "{\"description\": \"min=0.434, mean=0.434, max=0.434, sum=0.868 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.43410645679191306\"}", - "High School Microeconomics - Observed inference time (s)": "{\"description\": \"min=0.656, mean=0.656, max=0.656, sum=1.312 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6560712812327537\"}", - "High School Physics - Observed inference time (s)": "{\"description\": \"min=0.574, mean=0.574, max=0.574, sum=1.148 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5739512143545593\"}", - "High School Psychology - Observed inference time (s)": "{\"description\": \"min=0.446, mean=0.446, max=0.446, sum=0.892 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4460442779261038\"}", - "High School Statistics - Observed inference time (s)": "{\"description\": \"min=0.586, mean=0.586, max=0.586, sum=1.171 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5855172486216934\"}", - "High School US History - Observed inference time (s)": "{\"description\": \"min=0.579, mean=0.579, max=0.579, sum=1.158 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5790434245969734\"}", - "High School World History - Observed inference time (s)": "{\"description\": \"min=0.643, mean=0.643, max=0.643, sum=1.285 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6425194448559596\"}", - "High School Biology - # eval": "{\"description\": \"min=310, mean=310, max=310, sum=620 (2)\", \"tab\": \"General information\", \"score\": \"310.0\"}", - "High School Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Biology - # prompt tokens": "{\"description\": \"min=506.677, mean=506.677, max=506.677, sum=1013.355 (2)\", \"tab\": \"General information\", \"score\": \"506.6774193548387\"}", - "High School Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Chemistry - # eval": "{\"description\": \"min=203, mean=203, max=203, sum=406 (2)\", \"tab\": \"General information\", \"score\": \"203.0\"}", - "High School Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # prompt tokens": "{\"description\": \"min=489.714, mean=489.714, max=489.714, sum=979.429 (2)\", \"tab\": \"General information\", \"score\": \"489.7142857142857\"}", - "High School Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "High School Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # prompt tokens": "{\"description\": \"min=860.78, mean=860.78, max=860.78, sum=1721.56 (2)\", \"tab\": \"General information\", \"score\": \"860.78\"}", - "High School Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School European History - # eval": "{\"description\": \"min=165, mean=165, max=165, sum=330 (2)\", \"tab\": \"General information\", \"score\": \"165.0\"}", - "High School European History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School European History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # prompt tokens": "{\"description\": \"min=2791.073, mean=2791.073, max=2791.073, sum=5582.145 (2)\", \"tab\": \"General information\", \"score\": \"2791.072727272727\"}", - "High School European History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Geography - # eval": "{\"description\": \"min=198, mean=198, max=198, sum=396 (2)\", \"tab\": \"General information\", \"score\": \"198.0\"}", - "High School Geography - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Geography - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # prompt tokens": "{\"description\": \"min=365.045, mean=365.045, max=365.045, sum=730.091 (2)\", \"tab\": \"General information\", \"score\": \"365.04545454545456\"}", - "High School Geography - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Government And Politics - # eval": "{\"description\": \"min=193, mean=193, max=193, sum=386 (2)\", \"tab\": \"General information\", \"score\": \"193.0\"}", - "High School Government And Politics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Government And Politics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # prompt tokens": "{\"description\": \"min=458.824, mean=458.824, max=458.824, sum=917.648 (2)\", \"tab\": \"General information\", \"score\": \"458.8238341968912\"}", - "High School Government And Politics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Macroeconomics - # eval": "{\"description\": \"min=390, mean=390, max=390, sum=780 (2)\", \"tab\": \"General information\", \"score\": \"390.0\"}", - "High School Macroeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Macroeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # prompt tokens": "{\"description\": \"min=364.562, mean=364.562, max=364.562, sum=729.123 (2)\", \"tab\": \"General information\", \"score\": \"364.5615384615385\"}", - "High School Macroeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Mathematics - # eval": "{\"description\": \"min=270, mean=270, max=270, sum=540 (2)\", \"tab\": \"General information\", \"score\": \"270.0\"}", - "High School Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # prompt tokens": "{\"description\": \"min=525.374, mean=525.374, max=525.374, sum=1050.748 (2)\", \"tab\": \"General information\", \"score\": \"525.3740740740741\"}", - "High School Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Microeconomics - # eval": "{\"description\": \"min=238, mean=238, max=238, sum=476 (2)\", \"tab\": \"General information\", \"score\": \"238.0\"}", - "High School Microeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Microeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # prompt tokens": "{\"description\": \"min=392.025, mean=392.025, max=392.025, sum=784.05 (2)\", \"tab\": \"General information\", \"score\": \"392.02521008403363\"}", - "High School Microeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Physics - # eval": "{\"description\": \"min=151, mean=151, max=151, sum=302 (2)\", \"tab\": \"General information\", \"score\": \"151.0\"}", - "High School Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # prompt tokens": "{\"description\": \"min=553.464, mean=553.464, max=553.464, sum=1106.927 (2)\", \"tab\": \"General information\", \"score\": \"553.4635761589404\"}", - "High School Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Psychology - # eval": "{\"description\": \"min=545, mean=545, max=545, sum=1090 (2)\", \"tab\": \"General information\", \"score\": \"545.0\"}", - "High School Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # prompt tokens": "{\"description\": \"min=488.246, mean=488.246, max=488.246, sum=976.492 (2)\", \"tab\": \"General information\", \"score\": \"488.24587155963303\"}", - "High School Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Statistics - # eval": "{\"description\": \"min=216, mean=216, max=216, sum=432 (2)\", \"tab\": \"General information\", \"score\": \"216.0\"}", - "High School Statistics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Statistics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # prompt tokens": "{\"description\": \"min=788.699, mean=788.699, max=788.699, sum=1577.398 (2)\", \"tab\": \"General information\", \"score\": \"788.699074074074\"}", - "High School Statistics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School US History - # eval": "{\"description\": \"min=204, mean=204, max=204, sum=408 (2)\", \"tab\": \"General information\", \"score\": \"204.0\"}", - "High School US History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School US History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # prompt tokens": "{\"description\": \"min=2210.809, mean=2210.809, max=2210.809, sum=4421.618 (2)\", \"tab\": \"General information\", \"score\": \"2210.8088235294117\"}", - "High School US History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School World History - # eval": "{\"description\": \"min=237, mean=237, max=237, sum=474 (2)\", \"tab\": \"General information\", \"score\": \"237.0\"}", - "High School World History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School World History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # prompt tokens": "{\"description\": \"min=1421.27, mean=1421.27, max=1421.27, sum=2842.54 (2)\", \"tab\": \"General information\", \"score\": \"1421.2700421940929\"}", - "High School World History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"high_school_world_history\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_high_school_world_history\"" - } - } - }, - { - "evaluation_name": "Human Sexuality", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Human Sexuality", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.908, - "details": { - "description": "min=0.908, mean=0.908, max=0.908, sum=1.817 (2)", - "tab": "Accuracy", - "Human Aging - Observed inference time (s)": "{\"description\": \"min=0.472, mean=0.472, max=0.472, sum=0.944 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.47213134316585526\"}", - "Human Sexuality - Observed inference time (s)": "{\"description\": \"min=0.515, mean=0.515, max=0.515, sum=1.03 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5152236923916649\"}", - "Human Aging - # eval": "{\"description\": \"min=223, mean=223, max=223, sum=446 (2)\", \"tab\": \"General information\", \"score\": \"223.0\"}", - "Human Aging - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Aging - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Aging - # prompt tokens": "{\"description\": \"min=312.906, mean=312.906, max=312.906, sum=625.812 (2)\", \"tab\": \"General information\", \"score\": \"312.90582959641256\"}", - "Human Aging - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Human Sexuality - # eval": "{\"description\": \"min=131, mean=131, max=131, sum=262 (2)\", \"tab\": \"General information\", \"score\": \"131.0\"}", - "Human Sexuality - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Sexuality - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # prompt tokens": "{\"description\": \"min=334.183, mean=334.183, max=334.183, sum=668.366 (2)\", \"tab\": \"General information\", \"score\": \"334.1832061068702\"}", - "Human Sexuality - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"human_sexuality\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_human_sexuality\"" - } - } - }, - { - "evaluation_name": "International Law", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on International Law", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.926, - "details": { - "description": "min=0.926, mean=0.926, max=0.926, sum=1.851 (2)", - "tab": "Accuracy", - "International Law - Observed inference time (s)": "{\"description\": \"min=0.523, mean=0.523, max=0.523, sum=1.046 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5229926621618349\"}", - "International Law - # eval": "{\"description\": \"min=121, mean=121, max=121, sum=242 (2)\", \"tab\": \"General information\", \"score\": \"121.0\"}", - "International Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "International Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "International Law - # prompt tokens": "{\"description\": \"min=632.851, mean=632.851, max=632.851, sum=1265.702 (2)\", \"tab\": \"General information\", \"score\": \"632.8512396694215\"}", - "International Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"international_law\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_international_law\"" - } - } - }, - { - "evaluation_name": "Logical Fallacies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Logical Fallacies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.865, - "details": { - "description": "min=0.865, mean=0.865, max=0.865, sum=1.73 (2)", - "tab": "Accuracy", - "Logical Fallacies - Observed inference time (s)": "{\"description\": \"min=0.699, mean=0.699, max=0.699, sum=1.398 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6990647155083031\"}", - "Logical Fallacies - # eval": "{\"description\": \"min=163, mean=163, max=163, sum=326 (2)\", \"tab\": \"General information\", \"score\": \"163.0\"}", - "Logical Fallacies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Logical Fallacies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Logical Fallacies - # prompt tokens": "{\"description\": \"min=442.595, mean=442.595, max=442.595, sum=885.19 (2)\", \"tab\": \"General information\", \"score\": \"442.5950920245399\"}", - "Logical Fallacies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"logical_fallacies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_logical_fallacies\"" - } - } - }, - { - "evaluation_name": "Machine Learning", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Machine Learning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.723, - "details": { - "description": "min=0.723, mean=0.723, max=0.723, sum=1.446 (2)", - "tab": "Accuracy", - "Machine Learning - Observed inference time (s)": "{\"description\": \"min=0.606, mean=0.606, max=0.606, sum=1.211 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6055374975715365\"}", - "Machine Learning - # eval": "{\"description\": \"min=112, mean=112, max=112, sum=224 (2)\", \"tab\": \"General information\", \"score\": \"112.0\"}", - "Machine Learning - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Machine Learning - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Machine Learning - # prompt tokens": "{\"description\": \"min=661.054, mean=661.054, max=661.054, sum=1322.107 (2)\", \"tab\": \"General information\", \"score\": \"661.0535714285714\"}", - "Machine Learning - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"machine_learning\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_machine_learning\"" - } - } - }, - { - "evaluation_name": "Management", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Management", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.913, - "details": { - "description": "min=0.913, mean=0.913, max=0.913, sum=1.825 (2)", - "tab": "Accuracy", - "Management - Observed inference time (s)": "{\"description\": \"min=0.576, mean=0.576, max=0.576, sum=1.152 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5760108475546235\"}", - "Management - # eval": "{\"description\": \"min=103, mean=103, max=103, sum=206 (2)\", \"tab\": \"General information\", \"score\": \"103.0\"}", - "Management - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Management - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Management - # prompt tokens": "{\"description\": \"min=276.796, mean=276.796, max=276.796, sum=553.592 (2)\", \"tab\": \"General information\", \"score\": \"276.79611650485435\"}", - "Management - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"management\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_management\"" - } - } - }, - { - "evaluation_name": "Marketing", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Marketing", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.932, - "details": { - "description": "min=0.932, mean=0.932, max=0.932, sum=1.863 (2)", - "tab": "Accuracy", - "Marketing - Observed inference time (s)": "{\"description\": \"min=0.495, mean=0.495, max=0.495, sum=0.991 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.49540983204148775\"}", - "Marketing - # eval": "{\"description\": \"min=234, mean=234, max=234, sum=468 (2)\", \"tab\": \"General information\", \"score\": \"234.0\"}", - "Marketing - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Marketing - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Marketing - # prompt tokens": "{\"description\": \"min=397.218, mean=397.218, max=397.218, sum=794.436 (2)\", \"tab\": \"General information\", \"score\": \"397.21794871794873\"}", - "Marketing - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"marketing\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_marketing\"" - } - } - }, - { - "evaluation_name": "Medical Genetics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Medical Genetics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.93, - "details": { - "description": "min=0.93, mean=0.93, max=0.93, sum=1.86 (2)", - "tab": "Accuracy", - "Medical Genetics - Observed inference time (s)": "{\"description\": \"min=0.541, mean=0.541, max=0.541, sum=1.082 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5407642388343811\"}", - "Medical Genetics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Medical Genetics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Medical Genetics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Medical Genetics - # prompt tokens": "{\"description\": \"min=334, mean=334, max=334, sum=668 (2)\", \"tab\": \"General information\", \"score\": \"334.0\"}", - "Medical Genetics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"medical_genetics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_medical_genetics\"" - } - } - }, - { - "evaluation_name": "Miscellaneous", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Miscellaneous", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.946, - "details": { - "description": "min=0.946, mean=0.946, max=0.946, sum=1.893 (2)", - "tab": "Accuracy", - "Miscellaneous - Observed inference time (s)": "{\"description\": \"min=0.474, mean=0.474, max=0.474, sum=0.947 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4736132238103055\"}", - "Miscellaneous - # eval": "{\"description\": \"min=783, mean=783, max=783, sum=1566 (2)\", \"tab\": \"General information\", \"score\": \"783.0\"}", - "Miscellaneous - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Miscellaneous - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Miscellaneous - # prompt tokens": "{\"description\": \"min=292.925, mean=292.925, max=292.925, sum=585.849 (2)\", \"tab\": \"General information\", \"score\": \"292.92464878671774\"}", - "Miscellaneous - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"miscellaneous\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_miscellaneous\"" - } - } - }, - { - "evaluation_name": "Moral Scenarios", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Moral Scenarios", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.816, - "details": { - "description": "min=0.816, mean=0.816, max=0.816, sum=1.631 (2)", - "tab": "Accuracy", - "Moral Disputes - Observed inference time (s)": "{\"description\": \"min=0.451, mean=0.451, max=0.451, sum=0.901 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.45068276686475456\"}", - "Moral Scenarios - Observed inference time (s)": "{\"description\": \"min=0.545, mean=0.545, max=0.545, sum=1.09 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5448215519249773\"}", - "Moral Disputes - # eval": "{\"description\": \"min=346, mean=346, max=346, sum=692 (2)\", \"tab\": \"General information\", \"score\": \"346.0\"}", - "Moral Disputes - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Disputes - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Disputes - # prompt tokens": "{\"description\": \"min=469.145, mean=469.145, max=469.145, sum=938.289 (2)\", \"tab\": \"General information\", \"score\": \"469.1445086705202\"}", - "Moral Disputes - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Moral Scenarios - # eval": "{\"description\": \"min=895, mean=895, max=895, sum=1790 (2)\", \"tab\": \"General information\", \"score\": \"895.0\"}", - "Moral Scenarios - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Scenarios - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # prompt tokens": "{\"description\": \"min=649.455, mean=649.455, max=649.455, sum=1298.909 (2)\", \"tab\": \"General information\", \"score\": \"649.454748603352\"}", - "Moral Scenarios - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"moral_scenarios\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_moral_scenarios\"" - } - } - }, - { - "evaluation_name": "Nutrition", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Nutrition", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.879, - "details": { - "description": "min=0.879, mean=0.879, max=0.879, sum=1.758 (2)", - "tab": "Accuracy", - "Nutrition - Observed inference time (s)": "{\"description\": \"min=0.441, mean=0.441, max=0.441, sum=0.882 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4411514296251185\"}", - "Nutrition - # eval": "{\"description\": \"min=306, mean=306, max=306, sum=612 (2)\", \"tab\": \"General information\", \"score\": \"306.0\"}", - "Nutrition - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Nutrition - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Nutrition - # prompt tokens": "{\"description\": \"min=579.817, mean=579.817, max=579.817, sum=1159.634 (2)\", \"tab\": \"General information\", \"score\": \"579.8169934640523\"}", - "Nutrition - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"nutrition\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_nutrition\"" - } - } - }, - { - "evaluation_name": "Prehistory", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Prehistory", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.917, - "details": { - "description": "min=0.917, mean=0.917, max=0.917, sum=1.833 (2)", - "tab": "Accuracy", - "Prehistory - Observed inference time (s)": "{\"description\": \"min=0.489, mean=0.489, max=0.489, sum=0.978 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4891524300163175\"}", - "Prehistory - # eval": "{\"description\": \"min=324, mean=324, max=324, sum=648 (2)\", \"tab\": \"General information\", \"score\": \"324.0\"}", - "Prehistory - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Prehistory - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Prehistory - # prompt tokens": "{\"description\": \"min=507.559, mean=507.559, max=507.559, sum=1015.117 (2)\", \"tab\": \"General information\", \"score\": \"507.55864197530866\"}", - "Prehistory - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"prehistory\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_prehistory\"" - } - } - }, - { - "evaluation_name": "Public Relations", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Public Relations", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.782, - "details": { - "description": "min=0.782, mean=0.782, max=0.782, sum=1.564 (2)", - "tab": "Accuracy", - "Public Relations - Observed inference time (s)": "{\"description\": \"min=0.46, mean=0.46, max=0.46, sum=0.92 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.46012504534287885\"}", - "Public Relations - # eval": "{\"description\": \"min=110, mean=110, max=110, sum=220 (2)\", \"tab\": \"General information\", \"score\": \"110.0\"}", - "Public Relations - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Public Relations - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Public Relations - # prompt tokens": "{\"description\": \"min=398.318, mean=398.318, max=398.318, sum=796.636 (2)\", \"tab\": \"General information\", \"score\": \"398.3181818181818\"}", - "Public Relations - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"public_relations\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_public_relations\"" - } - } - }, - { - "evaluation_name": "Security Studies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Security Studies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.841, - "details": { - "description": "min=0.841, mean=0.841, max=0.841, sum=1.682 (2)", - "tab": "Accuracy", - "Security Studies - Observed inference time (s)": "{\"description\": \"min=0.546, mean=0.546, max=0.546, sum=1.093 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.546490309189777\"}", - "Security Studies - # eval": "{\"description\": \"min=245, mean=245, max=245, sum=490 (2)\", \"tab\": \"General information\", \"score\": \"245.0\"}", - "Security Studies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Security Studies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Security Studies - # prompt tokens": "{\"description\": \"min=1157.473, mean=1157.473, max=1157.473, sum=2314.947 (2)\", \"tab\": \"General information\", \"score\": \"1157.4734693877551\"}", - "Security Studies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"security_studies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_security_studies\"" - } - } - }, - { - "evaluation_name": "Sociology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Sociology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.925, - "details": { - "description": "min=0.925, mean=0.925, max=0.925, sum=1.851 (2)", - "tab": "Accuracy", - "Sociology - Observed inference time (s)": "{\"description\": \"min=0.441, mean=0.441, max=0.441, sum=0.882 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4410626805243801\"}", - "Sociology - # eval": "{\"description\": \"min=201, mean=201, max=201, sum=402 (2)\", \"tab\": \"General information\", \"score\": \"201.0\"}", - "Sociology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Sociology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Sociology - # prompt tokens": "{\"description\": \"min=438.522, mean=438.522, max=438.522, sum=877.045 (2)\", \"tab\": \"General information\", \"score\": \"438.5223880597015\"}", - "Sociology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"sociology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_sociology\"" - } - } - }, - { - "evaluation_name": "Virology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Virology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.59, - "details": { - "description": "min=0.59, mean=0.59, max=0.59, sum=1.181 (2)", - "tab": "Accuracy", - "Virology - Observed inference time (s)": "{\"description\": \"min=0.852, mean=0.852, max=0.852, sum=1.704 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.851962562066963\"}", - "Virology - # eval": "{\"description\": \"min=166, mean=166, max=166, sum=332 (2)\", \"tab\": \"General information\", \"score\": \"166.0\"}", - "Virology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Virology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Virology - # prompt tokens": "{\"description\": \"min=336.09, mean=336.09, max=336.09, sum=672.181 (2)\", \"tab\": \"General information\", \"score\": \"336.0903614457831\"}", - "Virology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"virology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_virology\"" - } - } - }, - { - "evaluation_name": "World Religions", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on World Religions", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.854, - "details": { - "description": "min=0.854, mean=0.854, max=0.854, sum=1.708 (2)", - "tab": "Accuracy", - "World Religions - Observed inference time (s)": "{\"description\": \"min=0.566, mean=0.566, max=0.566, sum=1.133 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5664703581068251\"}", - "World Religions - # eval": "{\"description\": \"min=171, mean=171, max=171, sum=342 (2)\", \"tab\": \"General information\", \"score\": \"171.0\"}", - "World Religions - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "World Religions - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "World Religions - # prompt tokens": "{\"description\": \"min=268.561, mean=268.561, max=268.561, sum=537.123 (2)\", \"tab\": \"General information\", \"score\": \"268.56140350877195\"}", - "World Religions - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"world_religions\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_world_religions\"" - } - } - }, - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.416, - "details": { - "description": "", - "tab": "Efficiency" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/openai_gpt-4-turbo-2024-04-09.json b/data/models/openai_gpt-4-turbo-2024-04-09.json deleted file mode 100644 index e16c126fc0c5b9aa15f8e7eae24f7699fd6de829..0000000000000000000000000000000000000000 --- a/data/models/openai_gpt-4-turbo-2024-04-09.json +++ /dev/null @@ -1,2035 +0,0 @@ -{ - "model_info": { - "name": "GPT-4 Turbo 2024-04-09", - "id": "openai/gpt-4-turbo-2024-04-09", - "developer": "OpenAI", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_lite/openai_gpt-4-turbo-2024-04-09/1774096306.427425", - "retrieved_timestamp": "1774096306.427425", - "source_metadata": { - "source_name": "helm_lite", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_lite", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_lite", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.864, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.4568414481897628\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.761, - "details": { - "description": "min=0.761, mean=0.761, max=0.761, sum=0.761 (1)", - "tab": "Accuracy", - "NarrativeQA - Observed inference time (s)": "{\"description\": \"min=0.804, mean=0.804, max=0.804, sum=0.804 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.8043310716118611\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=3495.67, mean=3495.67, max=3495.67, sum=3495.67 (1)\", \"tab\": \"General information\", \"score\": \"3495.6704225352114\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=6.037, mean=6.037, max=6.037, sum=6.037 (1)\", \"tab\": \"General information\", \"score\": \"6.0366197183098596\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (closed-book)", - "source_data": { - "dataset_name": "NaturalQuestions (closed-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (closed-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.482, - "details": { - "description": "min=0.482, mean=0.482, max=0.482, sum=0.482 (1)", - "tab": "Accuracy", - "NaturalQuestions (open-book) - Observed inference time (s)": "{\"description\": \"min=0.712, mean=0.712, max=0.712, sum=0.712 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.7120162718296051\"}", - "NaturalQuestions (closed-book) - Observed inference time (s)": "{\"description\": \"min=0.605, mean=0.605, max=0.605, sum=0.605 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.6052222681045533\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1728.593, mean=1728.593, max=1728.593, sum=1728.593 (1)\", \"tab\": \"General information\", \"score\": \"1728.593\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=5.902, mean=5.902, max=5.902, sum=5.902 (1)\", \"tab\": \"General information\", \"score\": \"5.902\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=139.127, mean=139.127, max=139.127, sum=139.127 (1)\", \"tab\": \"General information\", \"score\": \"139.127\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=5.263, mean=5.263, max=5.263, sum=5.263 (1)\", \"tab\": \"General information\", \"score\": \"5.263\"}" - } - }, - "generation_config": { - "additional_details": { - "mode": "\"closedbook\"" - } - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.97, - "details": { - "description": "min=0.97, mean=0.97, max=0.97, sum=0.97 (1)", - "tab": "Accuracy", - "OpenbookQA - Observed inference time (s)": "{\"description\": \"min=0.438, mean=0.438, max=0.438, sum=0.438 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.4376141686439514\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=249.782, mean=249.782, max=249.782, sum=249.782 (1)\", \"tab\": \"General information\", \"score\": \"249.782\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "dataset": "\"openbookqa\"", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.711, - "details": { - "description": "min=0.53, mean=0.711, max=0.96, sum=3.555 (5)", - "tab": "Accuracy", - "MMLU - Observed inference time (s)": "{\"description\": \"min=0.53, mean=0.55, max=0.572, sum=2.749 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.5498773384847139\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=373.44, mean=467.72, max=614.43, sum=2338.6 (5)\", \"tab\": \"General information\", \"score\": \"467.71996491228066\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"college_chemistry\", \"computer_security\", \"econometrics\", \"us_foreign_policy\"]", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MATH", - "source_data": { - "dataset_name": "MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Equivalent (CoT) on MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.833, - "details": { - "description": "min=0.684, mean=0.833, max=0.97, sum=5.83 (7)", - "tab": "Accuracy", - "MATH - Observed inference time (s)": "{\"description\": \"min=4.92, mean=6.678, max=8.338, sum=46.748 (7)\", \"tab\": \"Efficiency\", \"score\": \"6.678270916932833\"}", - "MATH - # eval": "{\"description\": \"min=30, mean=62.429, max=135, sum=437 (7)\", \"tab\": \"General information\", \"score\": \"62.42857142857143\"}", - "MATH - # train": "{\"description\": \"min=8, mean=8, max=8, sum=56 (7)\", \"tab\": \"General information\", \"score\": \"8.0\"}", - "MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MATH - # prompt tokens": "{\"description\": \"min=881.363, mean=1262.911, max=2197.577, sum=8840.376 (7)\", \"tab\": \"General information\", \"score\": \"1262.9108741840687\"}", - "MATH - # output tokens": "{\"description\": \"min=135.163, mean=189.561, max=219.316, sum=1326.926 (7)\", \"tab\": \"General information\", \"score\": \"189.56082409362702\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"algebra\", \"counting_and_probability\", \"geometry\", \"intermediate_algebra\", \"number_theory\", \"prealgebra\", \"precalculus\"]", - "level": "\"1\"", - "use_official_examples": "\"False\"", - "use_chain_of_thought": "\"True\"" - } - } - }, - { - "evaluation_name": "GSM8K", - "source_data": { - "dataset_name": "GSM8K", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on GSM8K", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.824, - "details": { - "description": "min=0.824, mean=0.824, max=0.824, sum=0.824 (1)", - "tab": "Accuracy", - "GSM8K - Observed inference time (s)": "{\"description\": \"min=6.915, mean=6.915, max=6.915, sum=6.915 (1)\", \"tab\": \"Efficiency\", \"score\": \"6.91472976398468\"}", - "GSM8K - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "GSM8K - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "GSM8K - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GSM8K - # prompt tokens": "{\"description\": \"min=959.035, mean=959.035, max=959.035, sum=959.035 (1)\", \"tab\": \"General information\", \"score\": \"959.035\"}", - "GSM8K - # output tokens": "{\"description\": \"min=141.712, mean=141.712, max=141.712, sum=141.712 (1)\", \"tab\": \"General information\", \"score\": \"141.712\"}" - } - }, - "generation_config": { - "additional_details": { - "stop": "\"none\"" - } - } - }, - { - "evaluation_name": "LegalBench", - "source_data": { - "dataset_name": "LegalBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on LegalBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.727, - "details": { - "description": "min=0.417, mean=0.727, max=0.947, sum=3.637 (5)", - "tab": "Accuracy", - "LegalBench - Observed inference time (s)": "{\"description\": \"min=0.514, mean=0.608, max=0.803, sum=3.041 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.6081070231398068\"}", - "LegalBench - # eval": "{\"description\": \"min=95, mean=409.4, max=1000, sum=2047 (5)\", \"tab\": \"General information\", \"score\": \"409.4\"}", - "LegalBench - # train": "{\"description\": \"min=4, mean=4.8, max=5, sum=24 (5)\", \"tab\": \"General information\", \"score\": \"4.8\"}", - "LegalBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "LegalBench - # prompt tokens": "{\"description\": \"min=207.442, mean=1524.163, max=6311.388, sum=7620.815 (5)\", \"tab\": \"General information\", \"score\": \"1524.162971355988\"}", - "LegalBench - # output tokens": "{\"description\": \"min=1, mean=1.325, max=2.032, sum=6.626 (5)\", \"tab\": \"General information\", \"score\": \"1.3251168793919403\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"abercrombie\", \"corporate_lobbying\", \"function_of_decision_section\", \"international_citizenship_questions\", \"proa\"]" - } - } - }, - { - "evaluation_name": "MedQA", - "source_data": { - "dataset_name": "MedQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MedQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.783, - "details": { - "description": "min=0.783, mean=0.783, max=0.783, sum=0.783 (1)", - "tab": "Accuracy", - "MedQA - Observed inference time (s)": "{\"description\": \"min=0.455, mean=0.455, max=0.455, sum=0.455 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.4549296101329341\"}", - "MedQA - # eval": "{\"description\": \"min=503, mean=503, max=503, sum=503 (1)\", \"tab\": \"General information\", \"score\": \"503.0\"}", - "MedQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MedQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MedQA - # prompt tokens": "{\"description\": \"min=1027.414, mean=1027.414, max=1027.414, sum=1027.414 (1)\", \"tab\": \"General information\", \"score\": \"1027.4135188866799\"}", - "MedQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WMT 2014", - "source_data": { - "dataset_name": "WMT 2014", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "BLEU-4 on WMT 2014", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.218, - "details": { - "description": "min=0.169, mean=0.218, max=0.264, sum=1.088 (5)", - "tab": "Accuracy", - "WMT 2014 - Observed inference time (s)": "{\"description\": \"min=1.131, mean=1.185, max=1.222, sum=5.925 (5)\", \"tab\": \"Efficiency\", \"score\": \"1.1850423664020953\"}", - "WMT 2014 - # eval": "{\"description\": \"min=503, mean=568.8, max=832, sum=2844 (5)\", \"tab\": \"General information\", \"score\": \"568.8\"}", - "WMT 2014 - # train": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "WMT 2014 - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WMT 2014 - # prompt tokens": "{\"description\": \"min=124.901, mean=148.043, max=168.185, sum=740.213 (5)\", \"tab\": \"General information\", \"score\": \"148.04258583116683\"}", - "WMT 2014 - # output tokens": "{\"description\": \"min=23.744, mean=25.264, max=25.938, sum=126.322 (5)\", \"tab\": \"General information\", \"score\": \"25.26444840571953\"}" - } - }, - "generation_config": { - "additional_details": { - "language_pair": "[\"cs-en\", \"de-en\", \"fr-en\", \"hi-en\", \"ru-en\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_mmlu/openai_gpt-4-turbo-2024-04-09/1774096312.00548", - "retrieved_timestamp": "1774096312.00548", - "source_metadata": { - "source_name": "helm_mmlu", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_mmlu", - "evaluation_results": [ - { - "evaluation_name": "MMLU All Subjects", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU All Subjects", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.813, - "details": { - "description": "min=0.515, mean=0.813, max=0.974, sum=92.65 (114)", - "tab": "Accuracy", - "MMLU All Subjects - Observed inference time (s)": "{\"description\": \"min=0.479, mean=0.617, max=0.934, sum=70.3 (114)\", \"tab\": \"Efficiency\", \"score\": \"0.6166649052297876\"}", - "MMLU All Subjects - # eval": "{\"description\": \"min=100, mean=246.351, max=1534, sum=28084 (114)\", \"tab\": \"General information\", \"score\": \"246.35087719298247\"}", - "MMLU All Subjects - # train": "{\"description\": \"min=5, mean=5, max=5, sum=570 (114)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU All Subjects - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU All Subjects - # prompt tokens": "{\"description\": \"min=275.561, mean=614.852, max=2798.073, sum=70093.086 (114)\", \"tab\": \"General information\", \"score\": \"614.851634217556\"}", - "MMLU All Subjects - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=114 (114)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_name": "Abstract Algebra", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Abstract Algebra", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.56, - "details": { - "description": "min=0.56, mean=0.56, max=0.56, sum=1.12 (2)", - "tab": "Accuracy", - "Abstract Algebra - Observed inference time (s)": "{\"description\": \"min=0.54, mean=0.54, max=0.54, sum=1.08 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.539907853603363\"}", - "Abstract Algebra - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Abstract Algebra - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Abstract Algebra - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Abstract Algebra - # prompt tokens": "{\"description\": \"min=373.44, mean=373.44, max=373.44, sum=746.88 (2)\", \"tab\": \"General information\", \"score\": \"373.44\"}", - "Abstract Algebra - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"abstract_algebra\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_abstract_algebra\"" - } - } - }, - { - "evaluation_name": "Anatomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Anatomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.822, - "details": { - "description": "min=0.822, mean=0.822, max=0.822, sum=1.644 (2)", - "tab": "Accuracy", - "Anatomy - Observed inference time (s)": "{\"description\": \"min=0.53, mean=0.53, max=0.53, sum=1.06 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5299274744810881\"}", - "Anatomy - # eval": "{\"description\": \"min=135, mean=135, max=135, sum=270 (2)\", \"tab\": \"General information\", \"score\": \"135.0\"}", - "Anatomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Anatomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Anatomy - # prompt tokens": "{\"description\": \"min=353.978, mean=353.978, max=353.978, sum=707.956 (2)\", \"tab\": \"General information\", \"score\": \"353.97777777777776\"}", - "Anatomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"anatomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_anatomy\"" - } - } - }, - { - "evaluation_name": "College Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on College Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.539, - "details": { - "description": "min=0.539, mean=0.539, max=0.539, sum=1.078 (2)", - "tab": "Accuracy", - "College Chemistry - Observed inference time (s)": "{\"description\": \"min=0.549, mean=0.549, max=0.549, sum=1.099 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5493535542488098\"}", - "College Biology - Observed inference time (s)": "{\"description\": \"min=0.6, mean=0.6, max=0.6, sum=1.199 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5995734184980392\"}", - "College Computer Science - Observed inference time (s)": "{\"description\": \"min=0.691, mean=0.691, max=0.691, sum=1.382 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6911867094039917\"}", - "College Mathematics - Observed inference time (s)": "{\"description\": \"min=0.609, mean=0.609, max=0.609, sum=1.219 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6092576813697815\"}", - "College Medicine - Observed inference time (s)": "{\"description\": \"min=0.67, mean=0.67, max=0.67, sum=1.34 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6697626251705809\"}", - "College Physics - Observed inference time (s)": "{\"description\": \"min=0.706, mean=0.706, max=0.706, sum=1.412 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7058592660754335\"}", - "College Chemistry - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Chemistry - # prompt tokens": "{\"description\": \"min=549.4, mean=549.4, max=549.4, sum=1098.8 (2)\", \"tab\": \"General information\", \"score\": \"549.4\"}", - "College Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Biology - # eval": "{\"description\": \"min=144, mean=144, max=144, sum=288 (2)\", \"tab\": \"General information\", \"score\": \"144.0\"}", - "College Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # prompt tokens": "{\"description\": \"min=473.917, mean=473.917, max=473.917, sum=947.833 (2)\", \"tab\": \"General information\", \"score\": \"473.9166666666667\"}", - "College Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # prompt tokens": "{\"description\": \"min=828.39, mean=828.39, max=828.39, sum=1656.78 (2)\", \"tab\": \"General information\", \"score\": \"828.39\"}", - "College Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Mathematics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # prompt tokens": "{\"description\": \"min=594.52, mean=594.52, max=594.52, sum=1189.04 (2)\", \"tab\": \"General information\", \"score\": \"594.52\"}", - "College Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Medicine - # eval": "{\"description\": \"min=173, mean=173, max=173, sum=346 (2)\", \"tab\": \"General information\", \"score\": \"173.0\"}", - "College Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # prompt tokens": "{\"description\": \"min=502.728, mean=502.728, max=502.728, sum=1005.457 (2)\", \"tab\": \"General information\", \"score\": \"502.728323699422\"}", - "College Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Physics - # eval": "{\"description\": \"min=102, mean=102, max=102, sum=204 (2)\", \"tab\": \"General information\", \"score\": \"102.0\"}", - "College Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # prompt tokens": "{\"description\": \"min=503.608, mean=503.608, max=503.608, sum=1007.216 (2)\", \"tab\": \"General information\", \"score\": \"503.6078431372549\"}", - "College Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"college_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_college_physics\"" - } - } - }, - { - "evaluation_name": "Computer Security", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Computer Security", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.83, - "details": { - "description": "min=0.83, mean=0.83, max=0.83, sum=1.66 (2)", - "tab": "Accuracy", - "Computer Security - Observed inference time (s)": "{\"description\": \"min=0.53, mean=0.53, max=0.53, sum=1.061 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5303381824493408\"}", - "Computer Security - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Computer Security - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Computer Security - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Computer Security - # prompt tokens": "{\"description\": \"min=378.54, mean=378.54, max=378.54, sum=757.08 (2)\", \"tab\": \"General information\", \"score\": \"378.54\"}", - "Computer Security - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"computer_security\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_computer_security\"" - } - } - }, - { - "evaluation_name": "Econometrics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Econometrics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.675, - "details": { - "description": "min=0.675, mean=0.675, max=0.675, sum=1.351 (2)", - "tab": "Accuracy", - "Econometrics - Observed inference time (s)": "{\"description\": \"min=0.572, mean=0.572, max=0.572, sum=1.144 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5721135453173989\"}", - "Econometrics - # eval": "{\"description\": \"min=114, mean=114, max=114, sum=228 (2)\", \"tab\": \"General information\", \"score\": \"114.0\"}", - "Econometrics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Econometrics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Econometrics - # prompt tokens": "{\"description\": \"min=614.43, mean=614.43, max=614.43, sum=1228.86 (2)\", \"tab\": \"General information\", \"score\": \"614.4298245614035\"}", - "Econometrics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"econometrics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_econometrics\"" - } - } - }, - { - "evaluation_name": "Global Facts", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Global Facts", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.58, - "details": { - "description": "min=0.58, mean=0.58, max=0.58, sum=1.16 (2)", - "tab": "Accuracy", - "Global Facts - Observed inference time (s)": "{\"description\": \"min=0.479, mean=0.479, max=0.479, sum=0.958 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.47900029182434084\"}", - "Global Facts - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Global Facts - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Global Facts - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Global Facts - # prompt tokens": "{\"description\": \"min=399.71, mean=399.71, max=399.71, sum=799.42 (2)\", \"tab\": \"General information\", \"score\": \"399.71\"}", - "Global Facts - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"global_facts\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_global_facts\"" - } - } - }, - { - "evaluation_name": "Jurisprudence", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Jurisprudence", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.88, - "details": { - "description": "min=0.88, mean=0.88, max=0.88, sum=1.759 (2)", - "tab": "Accuracy", - "Jurisprudence - Observed inference time (s)": "{\"description\": \"min=0.539, mean=0.539, max=0.539, sum=1.079 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5393155504156042\"}", - "Jurisprudence - # eval": "{\"description\": \"min=108, mean=108, max=108, sum=216 (2)\", \"tab\": \"General information\", \"score\": \"108.0\"}", - "Jurisprudence - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Jurisprudence - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Jurisprudence - # prompt tokens": "{\"description\": \"min=394.639, mean=394.639, max=394.639, sum=789.278 (2)\", \"tab\": \"General information\", \"score\": \"394.6388888888889\"}", - "Jurisprudence - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"jurisprudence\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_jurisprudence\"" - } - } - }, - { - "evaluation_name": "Philosophy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Philosophy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.868, - "details": { - "description": "min=0.868, mean=0.868, max=0.868, sum=1.736 (2)", - "tab": "Accuracy", - "Philosophy - Observed inference time (s)": "{\"description\": \"min=0.543, mean=0.543, max=0.543, sum=1.087 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5434573969273705\"}", - "Philosophy - # eval": "{\"description\": \"min=311, mean=311, max=311, sum=622 (2)\", \"tab\": \"General information\", \"score\": \"311.0\"}", - "Philosophy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Philosophy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Philosophy - # prompt tokens": "{\"description\": \"min=329.084, mean=329.084, max=329.084, sum=658.167 (2)\", \"tab\": \"General information\", \"score\": \"329.08360128617363\"}", - "Philosophy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"philosophy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_philosophy\"" - } - } - }, - { - "evaluation_name": "Professional Psychology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Professional Psychology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.873, - "details": { - "description": "min=0.873, mean=0.873, max=0.873, sum=1.745 (2)", - "tab": "Accuracy", - "Professional Medicine - Observed inference time (s)": "{\"description\": \"min=0.579, mean=0.579, max=0.579, sum=1.159 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5794552100055358\"}", - "Professional Accounting - Observed inference time (s)": "{\"description\": \"min=0.59, mean=0.59, max=0.59, sum=1.18 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5898241354218612\"}", - "Professional Law - Observed inference time (s)": "{\"description\": \"min=0.639, mean=0.639, max=0.639, sum=1.278 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6388053317424371\"}", - "Professional Psychology - Observed inference time (s)": "{\"description\": \"min=0.671, mean=0.671, max=0.671, sum=1.342 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6712259284031936\"}", - "Professional Medicine - # eval": "{\"description\": \"min=272, mean=272, max=272, sum=544 (2)\", \"tab\": \"General information\", \"score\": \"272.0\"}", - "Professional Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Medicine - # prompt tokens": "{\"description\": \"min=1094.585, mean=1094.585, max=1094.585, sum=2189.169 (2)\", \"tab\": \"General information\", \"score\": \"1094.5845588235295\"}", - "Professional Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Accounting - # eval": "{\"description\": \"min=282, mean=282, max=282, sum=564 (2)\", \"tab\": \"General information\", \"score\": \"282.0\"}", - "Professional Accounting - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Accounting - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # prompt tokens": "{\"description\": \"min=658.592, mean=658.592, max=658.592, sum=1317.184 (2)\", \"tab\": \"General information\", \"score\": \"658.5921985815603\"}", - "Professional Accounting - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Law - # eval": "{\"description\": \"min=1534, mean=1534, max=1534, sum=3068 (2)\", \"tab\": \"General information\", \"score\": \"1534.0\"}", - "Professional Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # prompt tokens": "{\"description\": \"min=1637.787, mean=1637.787, max=1637.787, sum=3275.574 (2)\", \"tab\": \"General information\", \"score\": \"1637.7868318122555\"}", - "Professional Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Psychology - # eval": "{\"description\": \"min=612, mean=612, max=612, sum=1224 (2)\", \"tab\": \"General information\", \"score\": \"612.0\"}", - "Professional Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # prompt tokens": "{\"description\": \"min=575.114, mean=575.114, max=575.114, sum=1150.229 (2)\", \"tab\": \"General information\", \"score\": \"575.1143790849674\"}", - "Professional Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"professional_psychology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_professional_psychology\"" - } - } - }, - { - "evaluation_name": "Us Foreign Policy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Us Foreign Policy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.96, - "details": { - "description": "min=0.96, mean=0.96, max=0.96, sum=1.92 (2)", - "tab": "Accuracy", - "Us Foreign Policy - Observed inference time (s)": "{\"description\": \"min=0.558, mean=0.558, max=0.558, sum=1.115 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.557673556804657\"}", - "Us Foreign Policy - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Us Foreign Policy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Us Foreign Policy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Us Foreign Policy - # prompt tokens": "{\"description\": \"min=422.79, mean=422.79, max=422.79, sum=845.58 (2)\", \"tab\": \"General information\", \"score\": \"422.79\"}", - "Us Foreign Policy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"us_foreign_policy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_us_foreign_policy\"" - } - } - }, - { - "evaluation_name": "Astronomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Astronomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.941, - "details": { - "description": "min=0.941, mean=0.941, max=0.941, sum=1.882 (2)", - "tab": "Accuracy", - "Astronomy - Observed inference time (s)": "{\"description\": \"min=0.666, mean=0.666, max=0.666, sum=1.332 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6662032525790366\"}", - "Astronomy - # eval": "{\"description\": \"min=152, mean=152, max=152, sum=304 (2)\", \"tab\": \"General information\", \"score\": \"152.0\"}", - "Astronomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Astronomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Astronomy - # prompt tokens": "{\"description\": \"min=579.691, mean=579.691, max=579.691, sum=1159.382 (2)\", \"tab\": \"General information\", \"score\": \"579.6907894736842\"}", - "Astronomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"astronomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_astronomy\"" - } - } - }, - { - "evaluation_name": "Business Ethics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Business Ethics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.82, - "details": { - "description": "min=0.82, mean=0.82, max=0.82, sum=1.64 (2)", - "tab": "Accuracy", - "Business Ethics - Observed inference time (s)": "{\"description\": \"min=0.598, mean=0.598, max=0.598, sum=1.196 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5981367039680481\"}", - "Business Ethics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Business Ethics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Business Ethics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Business Ethics - # prompt tokens": "{\"description\": \"min=569.52, mean=569.52, max=569.52, sum=1139.04 (2)\", \"tab\": \"General information\", \"score\": \"569.52\"}", - "Business Ethics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"business_ethics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_business_ethics\"" - } - } - }, - { - "evaluation_name": "Clinical Knowledge", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Clinical Knowledge", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.83, - "details": { - "description": "min=0.83, mean=0.83, max=0.83, sum=1.66 (2)", - "tab": "Accuracy", - "Clinical Knowledge - Observed inference time (s)": "{\"description\": \"min=0.591, mean=0.591, max=0.591, sum=1.183 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5912713131814633\"}", - "Clinical Knowledge - # eval": "{\"description\": \"min=265, mean=265, max=265, sum=530 (2)\", \"tab\": \"General information\", \"score\": \"265.0\"}", - "Clinical Knowledge - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Clinical Knowledge - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Clinical Knowledge - # prompt tokens": "{\"description\": \"min=397.947, mean=397.947, max=397.947, sum=795.894 (2)\", \"tab\": \"General information\", \"score\": \"397.94716981132075\"}", - "Clinical Knowledge - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"clinical_knowledge\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_clinical_knowledge\"" - } - } - }, - { - "evaluation_name": "Conceptual Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Conceptual Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.894, - "details": { - "description": "min=0.894, mean=0.894, max=0.894, sum=1.787 (2)", - "tab": "Accuracy", - "Conceptual Physics - Observed inference time (s)": "{\"description\": \"min=0.685, mean=0.685, max=0.685, sum=1.369 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.684603402969685\"}", - "Conceptual Physics - # eval": "{\"description\": \"min=235, mean=235, max=235, sum=470 (2)\", \"tab\": \"General information\", \"score\": \"235.0\"}", - "Conceptual Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Conceptual Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Conceptual Physics - # prompt tokens": "{\"description\": \"min=304.838, mean=304.838, max=304.838, sum=609.677 (2)\", \"tab\": \"General information\", \"score\": \"304.83829787234043\"}", - "Conceptual Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"conceptual_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_conceptual_physics\"" - } - } - }, - { - "evaluation_name": "Electrical Engineering", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Electrical Engineering", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.752, - "details": { - "description": "min=0.752, mean=0.752, max=0.752, sum=1.503 (2)", - "tab": "Accuracy", - "Electrical Engineering - Observed inference time (s)": "{\"description\": \"min=0.649, mean=0.649, max=0.649, sum=1.297 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6487039006989578\"}", - "Electrical Engineering - # eval": "{\"description\": \"min=145, mean=145, max=145, sum=290 (2)\", \"tab\": \"General information\", \"score\": \"145.0\"}", - "Electrical Engineering - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Electrical Engineering - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Electrical Engineering - # prompt tokens": "{\"description\": \"min=440.641, mean=440.641, max=440.641, sum=881.283 (2)\", \"tab\": \"General information\", \"score\": \"440.6413793103448\"}", - "Electrical Engineering - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"electrical_engineering\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_electrical_engineering\"" - } - } - }, - { - "evaluation_name": "Elementary Mathematics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Elementary Mathematics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.72, - "details": { - "description": "min=0.72, mean=0.72, max=0.72, sum=1.439 (2)", - "tab": "Accuracy", - "Elementary Mathematics - Observed inference time (s)": "{\"description\": \"min=0.708, mean=0.708, max=0.708, sum=1.417 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.708430844009238\"}", - "Elementary Mathematics - # eval": "{\"description\": \"min=378, mean=378, max=378, sum=756 (2)\", \"tab\": \"General information\", \"score\": \"378.0\"}", - "Elementary Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Elementary Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Elementary Mathematics - # prompt tokens": "{\"description\": \"min=531.862, mean=531.862, max=531.862, sum=1063.725 (2)\", \"tab\": \"General information\", \"score\": \"531.8624338624338\"}", - "Elementary Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"elementary_mathematics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_elementary_mathematics\"" - } - } - }, - { - "evaluation_name": "Formal Logic", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Formal Logic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.706, - "details": { - "description": "min=0.706, mean=0.706, max=0.706, sum=1.413 (2)", - "tab": "Accuracy", - "Formal Logic - Observed inference time (s)": "{\"description\": \"min=0.635, mean=0.635, max=0.635, sum=1.27 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6347800322941372\"}", - "Formal Logic - # eval": "{\"description\": \"min=126, mean=126, max=126, sum=252 (2)\", \"tab\": \"General information\", \"score\": \"126.0\"}", - "Formal Logic - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Formal Logic - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Formal Logic - # prompt tokens": "{\"description\": \"min=606.762, mean=606.762, max=606.762, sum=1213.524 (2)\", \"tab\": \"General information\", \"score\": \"606.7619047619048\"}", - "Formal Logic - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"formal_logic\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_formal_logic\"" - } - } - }, - { - "evaluation_name": "High School World History", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on High School World History", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.941, - "details": { - "description": "min=0.941, mean=0.941, max=0.941, sum=1.882 (2)", - "tab": "Accuracy", - "High School Biology - Observed inference time (s)": "{\"description\": \"min=0.674, mean=0.674, max=0.674, sum=1.348 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6741217144073979\"}", - "High School Chemistry - Observed inference time (s)": "{\"description\": \"min=0.673, mean=0.673, max=0.673, sum=1.346 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6728476491467706\"}", - "High School Computer Science - Observed inference time (s)": "{\"description\": \"min=0.626, mean=0.626, max=0.626, sum=1.252 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6261640882492066\"}", - "High School European History - Observed inference time (s)": "{\"description\": \"min=0.747, mean=0.747, max=0.747, sum=1.495 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7474224538514108\"}", - "High School Geography - Observed inference time (s)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=1.335 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6672574221485793\"}", - "High School Government And Politics - Observed inference time (s)": "{\"description\": \"min=0.683, mean=0.683, max=0.683, sum=1.366 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6831059715290762\"}", - "High School Macroeconomics - Observed inference time (s)": "{\"description\": \"min=0.613, mean=0.613, max=0.613, sum=1.226 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6132381714307344\"}", - "High School Mathematics - Observed inference time (s)": "{\"description\": \"min=0.594, mean=0.594, max=0.594, sum=1.188 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5939316025486698\"}", - "High School Microeconomics - Observed inference time (s)": "{\"description\": \"min=0.585, mean=0.585, max=0.585, sum=1.169 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5845635728675778\"}", - "High School Physics - Observed inference time (s)": "{\"description\": \"min=0.934, mean=0.934, max=0.934, sum=1.868 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9341671135251886\"}", - "High School Psychology - Observed inference time (s)": "{\"description\": \"min=0.741, mean=0.741, max=0.741, sum=1.482 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7410666920723171\"}", - "High School Statistics - Observed inference time (s)": "{\"description\": \"min=0.72, mean=0.72, max=0.72, sum=1.439 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7196061655327126\"}", - "High School US History - Observed inference time (s)": "{\"description\": \"min=0.745, mean=0.745, max=0.745, sum=1.491 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7454434785188413\"}", - "High School World History - Observed inference time (s)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=1.333 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6665283818788166\"}", - "High School Biology - # eval": "{\"description\": \"min=310, mean=310, max=310, sum=620 (2)\", \"tab\": \"General information\", \"score\": \"310.0\"}", - "High School Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Biology - # prompt tokens": "{\"description\": \"min=513.677, mean=513.677, max=513.677, sum=1027.355 (2)\", \"tab\": \"General information\", \"score\": \"513.6774193548387\"}", - "High School Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Chemistry - # eval": "{\"description\": \"min=203, mean=203, max=203, sum=406 (2)\", \"tab\": \"General information\", \"score\": \"203.0\"}", - "High School Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # prompt tokens": "{\"description\": \"min=496.714, mean=496.714, max=496.714, sum=993.429 (2)\", \"tab\": \"General information\", \"score\": \"496.7142857142857\"}", - "High School Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "High School Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # prompt tokens": "{\"description\": \"min=867.78, mean=867.78, max=867.78, sum=1735.56 (2)\", \"tab\": \"General information\", \"score\": \"867.78\"}", - "High School Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School European History - # eval": "{\"description\": \"min=165, mean=165, max=165, sum=330 (2)\", \"tab\": \"General information\", \"score\": \"165.0\"}", - "High School European History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School European History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # prompt tokens": "{\"description\": \"min=2798.073, mean=2798.073, max=2798.073, sum=5596.145 (2)\", \"tab\": \"General information\", \"score\": \"2798.072727272727\"}", - "High School European History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Geography - # eval": "{\"description\": \"min=198, mean=198, max=198, sum=396 (2)\", \"tab\": \"General information\", \"score\": \"198.0\"}", - "High School Geography - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Geography - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # prompt tokens": "{\"description\": \"min=372.045, mean=372.045, max=372.045, sum=744.091 (2)\", \"tab\": \"General information\", \"score\": \"372.04545454545456\"}", - "High School Geography - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Government And Politics - # eval": "{\"description\": \"min=193, mean=193, max=193, sum=386 (2)\", \"tab\": \"General information\", \"score\": \"193.0\"}", - "High School Government And Politics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Government And Politics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # prompt tokens": "{\"description\": \"min=465.824, mean=465.824, max=465.824, sum=931.648 (2)\", \"tab\": \"General information\", \"score\": \"465.8238341968912\"}", - "High School Government And Politics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Macroeconomics - # eval": "{\"description\": \"min=390, mean=390, max=390, sum=780 (2)\", \"tab\": \"General information\", \"score\": \"390.0\"}", - "High School Macroeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Macroeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # prompt tokens": "{\"description\": \"min=371.562, mean=371.562, max=371.562, sum=743.123 (2)\", \"tab\": \"General information\", \"score\": \"371.5615384615385\"}", - "High School Macroeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Mathematics - # eval": "{\"description\": \"min=270, mean=270, max=270, sum=540 (2)\", \"tab\": \"General information\", \"score\": \"270.0\"}", - "High School Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # prompt tokens": "{\"description\": \"min=532.374, mean=532.374, max=532.374, sum=1064.748 (2)\", \"tab\": \"General information\", \"score\": \"532.3740740740741\"}", - "High School Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Microeconomics - # eval": "{\"description\": \"min=238, mean=238, max=238, sum=476 (2)\", \"tab\": \"General information\", \"score\": \"238.0\"}", - "High School Microeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Microeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # prompt tokens": "{\"description\": \"min=399.025, mean=399.025, max=399.025, sum=798.05 (2)\", \"tab\": \"General information\", \"score\": \"399.02521008403363\"}", - "High School Microeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Physics - # eval": "{\"description\": \"min=151, mean=151, max=151, sum=302 (2)\", \"tab\": \"General information\", \"score\": \"151.0\"}", - "High School Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # prompt tokens": "{\"description\": \"min=560.464, mean=560.464, max=560.464, sum=1120.927 (2)\", \"tab\": \"General information\", \"score\": \"560.4635761589404\"}", - "High School Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Psychology - # eval": "{\"description\": \"min=545, mean=545, max=545, sum=1090 (2)\", \"tab\": \"General information\", \"score\": \"545.0\"}", - "High School Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # prompt tokens": "{\"description\": \"min=495.246, mean=495.246, max=495.246, sum=990.492 (2)\", \"tab\": \"General information\", \"score\": \"495.24587155963303\"}", - "High School Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Statistics - # eval": "{\"description\": \"min=216, mean=216, max=216, sum=432 (2)\", \"tab\": \"General information\", \"score\": \"216.0\"}", - "High School Statistics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Statistics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # prompt tokens": "{\"description\": \"min=795.699, mean=795.699, max=795.699, sum=1591.398 (2)\", \"tab\": \"General information\", \"score\": \"795.699074074074\"}", - "High School Statistics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School US History - # eval": "{\"description\": \"min=204, mean=204, max=204, sum=408 (2)\", \"tab\": \"General information\", \"score\": \"204.0\"}", - "High School US History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School US History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # prompt tokens": "{\"description\": \"min=2217.809, mean=2217.809, max=2217.809, sum=4435.618 (2)\", \"tab\": \"General information\", \"score\": \"2217.8088235294117\"}", - "High School US History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School World History - # eval": "{\"description\": \"min=237, mean=237, max=237, sum=474 (2)\", \"tab\": \"General information\", \"score\": \"237.0\"}", - "High School World History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School World History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # prompt tokens": "{\"description\": \"min=1428.27, mean=1428.27, max=1428.27, sum=2856.54 (2)\", \"tab\": \"General information\", \"score\": \"1428.2700421940929\"}", - "High School World History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"high_school_world_history\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_high_school_world_history\"" - } - } - }, - { - "evaluation_name": "Human Sexuality", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Human Sexuality", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.901, - "details": { - "description": "min=0.901, mean=0.901, max=0.901, sum=1.802 (2)", - "tab": "Accuracy", - "Human Aging - Observed inference time (s)": "{\"description\": \"min=0.656, mean=0.656, max=0.656, sum=1.313 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6564141239286003\"}", - "Human Sexuality - Observed inference time (s)": "{\"description\": \"min=0.613, mean=0.613, max=0.613, sum=1.226 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6131143715545422\"}", - "Human Aging - # eval": "{\"description\": \"min=223, mean=223, max=223, sum=446 (2)\", \"tab\": \"General information\", \"score\": \"223.0\"}", - "Human Aging - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Aging - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Aging - # prompt tokens": "{\"description\": \"min=319.906, mean=319.906, max=319.906, sum=639.812 (2)\", \"tab\": \"General information\", \"score\": \"319.90582959641256\"}", - "Human Aging - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Human Sexuality - # eval": "{\"description\": \"min=131, mean=131, max=131, sum=262 (2)\", \"tab\": \"General information\", \"score\": \"131.0\"}", - "Human Sexuality - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Sexuality - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # prompt tokens": "{\"description\": \"min=341.183, mean=341.183, max=341.183, sum=682.366 (2)\", \"tab\": \"General information\", \"score\": \"341.1832061068702\"}", - "Human Sexuality - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"human_sexuality\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_human_sexuality\"" - } - } - }, - { - "evaluation_name": "International Law", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on International Law", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.942, - "details": { - "description": "min=0.942, mean=0.942, max=0.942, sum=1.884 (2)", - "tab": "Accuracy", - "International Law - Observed inference time (s)": "{\"description\": \"min=0.63, mean=0.63, max=0.63, sum=1.26 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6297830116650289\"}", - "International Law - # eval": "{\"description\": \"min=121, mean=121, max=121, sum=242 (2)\", \"tab\": \"General information\", \"score\": \"121.0\"}", - "International Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "International Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "International Law - # prompt tokens": "{\"description\": \"min=639.851, mean=639.851, max=639.851, sum=1279.702 (2)\", \"tab\": \"General information\", \"score\": \"639.8512396694215\"}", - "International Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"international_law\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_international_law\"" - } - } - }, - { - "evaluation_name": "Logical Fallacies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Logical Fallacies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.871, - "details": { - "description": "min=0.871, mean=0.871, max=0.871, sum=1.742 (2)", - "tab": "Accuracy", - "Logical Fallacies - Observed inference time (s)": "{\"description\": \"min=0.585, mean=0.585, max=0.585, sum=1.171 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.585445927695994\"}", - "Logical Fallacies - # eval": "{\"description\": \"min=163, mean=163, max=163, sum=326 (2)\", \"tab\": \"General information\", \"score\": \"163.0\"}", - "Logical Fallacies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Logical Fallacies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Logical Fallacies - # prompt tokens": "{\"description\": \"min=449.595, mean=449.595, max=449.595, sum=899.19 (2)\", \"tab\": \"General information\", \"score\": \"449.5950920245399\"}", - "Logical Fallacies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"logical_fallacies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_logical_fallacies\"" - } - } - }, - { - "evaluation_name": "Machine Learning", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Machine Learning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.741, - "details": { - "description": "min=0.741, mean=0.741, max=0.741, sum=1.482 (2)", - "tab": "Accuracy", - "Machine Learning - Observed inference time (s)": "{\"description\": \"min=0.718, mean=0.718, max=0.718, sum=1.436 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.718035706451961\"}", - "Machine Learning - # eval": "{\"description\": \"min=112, mean=112, max=112, sum=224 (2)\", \"tab\": \"General information\", \"score\": \"112.0\"}", - "Machine Learning - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Machine Learning - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Machine Learning - # prompt tokens": "{\"description\": \"min=668.054, mean=668.054, max=668.054, sum=1336.107 (2)\", \"tab\": \"General information\", \"score\": \"668.0535714285714\"}", - "Machine Learning - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"machine_learning\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_machine_learning\"" - } - } - }, - { - "evaluation_name": "Management", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Management", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.883, - "details": { - "description": "min=0.883, mean=0.883, max=0.883, sum=1.767 (2)", - "tab": "Accuracy", - "Management - Observed inference time (s)": "{\"description\": \"min=0.592, mean=0.592, max=0.592, sum=1.184 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5921963488013999\"}", - "Management - # eval": "{\"description\": \"min=103, mean=103, max=103, sum=206 (2)\", \"tab\": \"General information\", \"score\": \"103.0\"}", - "Management - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Management - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Management - # prompt tokens": "{\"description\": \"min=283.796, mean=283.796, max=283.796, sum=567.592 (2)\", \"tab\": \"General information\", \"score\": \"283.79611650485435\"}", - "Management - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"management\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_management\"" - } - } - }, - { - "evaluation_name": "Marketing", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Marketing", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.949, - "details": { - "description": "min=0.949, mean=0.949, max=0.949, sum=1.897 (2)", - "tab": "Accuracy", - "Marketing - Observed inference time (s)": "{\"description\": \"min=0.588, mean=0.588, max=0.588, sum=1.176 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5880082672477788\"}", - "Marketing - # eval": "{\"description\": \"min=234, mean=234, max=234, sum=468 (2)\", \"tab\": \"General information\", \"score\": \"234.0\"}", - "Marketing - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Marketing - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Marketing - # prompt tokens": "{\"description\": \"min=404.218, mean=404.218, max=404.218, sum=808.436 (2)\", \"tab\": \"General information\", \"score\": \"404.21794871794873\"}", - "Marketing - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"marketing\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_marketing\"" - } - } - }, - { - "evaluation_name": "Medical Genetics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Medical Genetics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.92, - "details": { - "description": "min=0.92, mean=0.92, max=0.92, sum=1.84 (2)", - "tab": "Accuracy", - "Medical Genetics - Observed inference time (s)": "{\"description\": \"min=0.52, mean=0.52, max=0.52, sum=1.04 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5201336288452149\"}", - "Medical Genetics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Medical Genetics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Medical Genetics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Medical Genetics - # prompt tokens": "{\"description\": \"min=341, mean=341, max=341, sum=682 (2)\", \"tab\": \"General information\", \"score\": \"341.0\"}", - "Medical Genetics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"medical_genetics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_medical_genetics\"" - } - } - }, - { - "evaluation_name": "Miscellaneous", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Miscellaneous", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.945, - "details": { - "description": "min=0.945, mean=0.945, max=0.945, sum=1.89 (2)", - "tab": "Accuracy", - "Miscellaneous - Observed inference time (s)": "{\"description\": \"min=0.565, mean=0.565, max=0.565, sum=1.13 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5650817577561809\"}", - "Miscellaneous - # eval": "{\"description\": \"min=783, mean=783, max=783, sum=1566 (2)\", \"tab\": \"General information\", \"score\": \"783.0\"}", - "Miscellaneous - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Miscellaneous - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Miscellaneous - # prompt tokens": "{\"description\": \"min=299.925, mean=299.925, max=299.925, sum=599.849 (2)\", \"tab\": \"General information\", \"score\": \"299.92464878671774\"}", - "Miscellaneous - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"miscellaneous\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_miscellaneous\"" - } - } - }, - { - "evaluation_name": "Moral Scenarios", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Moral Scenarios", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.803, - "details": { - "description": "min=0.803, mean=0.803, max=0.803, sum=1.607 (2)", - "tab": "Accuracy", - "Moral Disputes - Observed inference time (s)": "{\"description\": \"min=0.564, mean=0.564, max=0.564, sum=1.129 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5643301023913256\"}", - "Moral Scenarios - Observed inference time (s)": "{\"description\": \"min=0.599, mean=0.599, max=0.599, sum=1.197 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5985688052363902\"}", - "Moral Disputes - # eval": "{\"description\": \"min=346, mean=346, max=346, sum=692 (2)\", \"tab\": \"General information\", \"score\": \"346.0\"}", - "Moral Disputes - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Disputes - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Disputes - # prompt tokens": "{\"description\": \"min=476.145, mean=476.145, max=476.145, sum=952.289 (2)\", \"tab\": \"General information\", \"score\": \"476.1445086705202\"}", - "Moral Disputes - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Moral Scenarios - # eval": "{\"description\": \"min=895, mean=895, max=895, sum=1790 (2)\", \"tab\": \"General information\", \"score\": \"895.0\"}", - "Moral Scenarios - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Scenarios - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # prompt tokens": "{\"description\": \"min=656.455, mean=656.455, max=656.455, sum=1312.909 (2)\", \"tab\": \"General information\", \"score\": \"656.454748603352\"}", - "Moral Scenarios - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"moral_scenarios\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_moral_scenarios\"" - } - } - }, - { - "evaluation_name": "Nutrition", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Nutrition", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.892, - "details": { - "description": "min=0.892, mean=0.892, max=0.892, sum=1.784 (2)", - "tab": "Accuracy", - "Nutrition - Observed inference time (s)": "{\"description\": \"min=0.532, mean=0.532, max=0.532, sum=1.063 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5316595968857311\"}", - "Nutrition - # eval": "{\"description\": \"min=306, mean=306, max=306, sum=612 (2)\", \"tab\": \"General information\", \"score\": \"306.0\"}", - "Nutrition - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Nutrition - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Nutrition - # prompt tokens": "{\"description\": \"min=586.817, mean=586.817, max=586.817, sum=1173.634 (2)\", \"tab\": \"General information\", \"score\": \"586.8169934640523\"}", - "Nutrition - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"nutrition\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_nutrition\"" - } - } - }, - { - "evaluation_name": "Prehistory", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Prehistory", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.92, - "details": { - "description": "min=0.92, mean=0.92, max=0.92, sum=1.84 (2)", - "tab": "Accuracy", - "Prehistory - Observed inference time (s)": "{\"description\": \"min=0.54, mean=0.54, max=0.54, sum=1.079 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5397091279795141\"}", - "Prehistory - # eval": "{\"description\": \"min=324, mean=324, max=324, sum=648 (2)\", \"tab\": \"General information\", \"score\": \"324.0\"}", - "Prehistory - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Prehistory - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Prehistory - # prompt tokens": "{\"description\": \"min=514.559, mean=514.559, max=514.559, sum=1029.117 (2)\", \"tab\": \"General information\", \"score\": \"514.5586419753087\"}", - "Prehistory - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"prehistory\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_prehistory\"" - } - } - }, - { - "evaluation_name": "Public Relations", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Public Relations", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.755, - "details": { - "description": "min=0.755, mean=0.755, max=0.755, sum=1.509 (2)", - "tab": "Accuracy", - "Public Relations - Observed inference time (s)": "{\"description\": \"min=0.584, mean=0.584, max=0.584, sum=1.168 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5840315688740123\"}", - "Public Relations - # eval": "{\"description\": \"min=110, mean=110, max=110, sum=220 (2)\", \"tab\": \"General information\", \"score\": \"110.0\"}", - "Public Relations - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Public Relations - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Public Relations - # prompt tokens": "{\"description\": \"min=405.318, mean=405.318, max=405.318, sum=810.636 (2)\", \"tab\": \"General information\", \"score\": \"405.3181818181818\"}", - "Public Relations - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"public_relations\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_public_relations\"" - } - } - }, - { - "evaluation_name": "Security Studies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Security Studies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8, - "details": { - "description": "min=0.8, mean=0.8, max=0.8, sum=1.6 (2)", - "tab": "Accuracy", - "Security Studies - Observed inference time (s)": "{\"description\": \"min=0.529, mean=0.529, max=0.529, sum=1.058 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.529095221538933\"}", - "Security Studies - # eval": "{\"description\": \"min=245, mean=245, max=245, sum=490 (2)\", \"tab\": \"General information\", \"score\": \"245.0\"}", - "Security Studies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Security Studies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Security Studies - # prompt tokens": "{\"description\": \"min=1164.473, mean=1164.473, max=1164.473, sum=2328.947 (2)\", \"tab\": \"General information\", \"score\": \"1164.4734693877551\"}", - "Security Studies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"security_studies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_security_studies\"" - } - } - }, - { - "evaluation_name": "Sociology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Sociology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.915, - "details": { - "description": "min=0.915, mean=0.915, max=0.915, sum=1.831 (2)", - "tab": "Accuracy", - "Sociology - Observed inference time (s)": "{\"description\": \"min=0.52, mean=0.52, max=0.52, sum=1.04 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5199050891458692\"}", - "Sociology - # eval": "{\"description\": \"min=201, mean=201, max=201, sum=402 (2)\", \"tab\": \"General information\", \"score\": \"201.0\"}", - "Sociology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Sociology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Sociology - # prompt tokens": "{\"description\": \"min=445.522, mean=445.522, max=445.522, sum=891.045 (2)\", \"tab\": \"General information\", \"score\": \"445.5223880597015\"}", - "Sociology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"sociology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_sociology\"" - } - } - }, - { - "evaluation_name": "Virology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Virology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.602, - "details": { - "description": "min=0.602, mean=0.602, max=0.602, sum=1.205 (2)", - "tab": "Accuracy", - "Virology - Observed inference time (s)": "{\"description\": \"min=0.523, mean=0.523, max=0.523, sum=1.045 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5226844951330897\"}", - "Virology - # eval": "{\"description\": \"min=166, mean=166, max=166, sum=332 (2)\", \"tab\": \"General information\", \"score\": \"166.0\"}", - "Virology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Virology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Virology - # prompt tokens": "{\"description\": \"min=343.09, mean=343.09, max=343.09, sum=686.181 (2)\", \"tab\": \"General information\", \"score\": \"343.0903614457831\"}", - "Virology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"virology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_virology\"" - } - } - }, - { - "evaluation_name": "World Religions", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on World Religions", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.848, - "details": { - "description": "min=0.848, mean=0.848, max=0.848, sum=1.696 (2)", - "tab": "Accuracy", - "World Religions - Observed inference time (s)": "{\"description\": \"min=0.494, mean=0.494, max=0.494, sum=0.988 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.49407080739562276\"}", - "World Religions - # eval": "{\"description\": \"min=171, mean=171, max=171, sum=342 (2)\", \"tab\": \"General information\", \"score\": \"171.0\"}", - "World Religions - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "World Religions - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "World Religions - # prompt tokens": "{\"description\": \"min=275.561, mean=275.561, max=275.561, sum=551.123 (2)\", \"tab\": \"General information\", \"score\": \"275.56140350877195\"}", - "World Religions - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"world_religions\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_world_religions\"" - } - } - }, - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.351, - "details": { - "description": "", - "tab": "Efficiency" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_id": "reward-bench/openai_gpt-4-turbo-2024-04-09/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8395 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9525 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7544 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8757 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.827 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7363 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/openai_gpt-4.1-2025-04-14.json b/data/models/openai_gpt-4.1-2025-04-14.json deleted file mode 100644 index aa5fe2f775b1ad7682999ad8cb6da9fff4fa2780..0000000000000000000000000000000000000000 --- a/data/models/openai_gpt-4.1-2025-04-14.json +++ /dev/null @@ -1,1417 +0,0 @@ -{ - "model_info": { - "name": "gpt-4.1-2025-04-14", - "id": "openai/gpt-4.1-2025-04-14", - "developer": "OpenAI", - "inference_platform": "unknown", - "additional_details": { - "display_name": "GPT-4.1" - } - }, - "evaluations": [ - { - "evaluation_id": "global-mmlu-lite/openai_gpt-4.1-2025-04-14/1773936496.366405", - "retrieved_timestamp": "1773936496.366405", - "source_metadata": { - "source_name": "Global MMLU Lite Leaderboard", - "source_type": "documentation", - "source_organization_name": "kaggle", - "source_organization_url": "www.kaggle.com", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "kaggle kernel", - "version": "4", - "additional_details": { - "url": "https://www.kaggle.com/code/shivalikasingh95/global-mmlu-lite-sample-notebook" - } - }, - "benchmark": "global-mmlu-lite", - "evaluation_results": [ - { - "evaluation_name": "Global MMLU Lite", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Global MMLU Lite", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8755 - } - }, - { - "evaluation_name": "Culturally Sensitive", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Sensitive", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8541 - } - }, - { - "evaluation_name": "Culturally Agnostic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Agnostic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8969 - } - }, - { - "evaluation_name": "Arabic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Arabic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.88, - "uncertainty": { - "confidence_interval": { - "lower": -0.0318, - "upper": 0.0318, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "English", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - English", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8825, - "uncertainty": { - "confidence_interval": { - "lower": -0.0316, - "upper": 0.0316, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Bengali", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Bengali", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8625, - "uncertainty": { - "confidence_interval": { - "lower": -0.0337, - "upper": 0.0337, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "German", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - German", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.875, - "uncertainty": { - "confidence_interval": { - "lower": -0.0324, - "upper": 0.0324, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "French", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - French", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8875, - "uncertainty": { - "confidence_interval": { - "lower": -0.031, - "upper": 0.031, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Hindi", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Hindi", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8775, - "uncertainty": { - "confidence_interval": { - "lower": -0.0321, - "upper": 0.0321, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Indonesian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Indonesian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.885, - "uncertainty": { - "confidence_interval": { - "lower": -0.0313, - "upper": 0.0313, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Italian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Italian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.88, - "uncertainty": { - "confidence_interval": { - "lower": -0.0318, - "upper": 0.0318, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Japanese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Japanese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8725, - "uncertainty": { - "confidence_interval": { - "lower": -0.0327, - "upper": 0.0327, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Korean", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Korean", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.87, - "uncertainty": { - "confidence_interval": { - "lower": -0.033, - "upper": 0.033, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Portuguese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Portuguese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.875, - "uncertainty": { - "confidence_interval": { - "lower": -0.0324, - "upper": 0.0324, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Spanish", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Spanish", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.885, - "uncertainty": { - "confidence_interval": { - "lower": -0.0313, - "upper": 0.0313, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Swahili", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Swahili", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8725, - "uncertainty": { - "confidence_interval": { - "lower": -0.0327, - "upper": 0.0327, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Yoruba", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Yoruba", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.875, - "uncertainty": { - "confidence_interval": { - "lower": -0.0324, - "upper": 0.0324, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Chinese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Chinese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.87, - "uncertainty": { - "confidence_interval": { - "lower": -0.033, - "upper": 0.033, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Burmese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Burmese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8575, - "uncertainty": { - "confidence_interval": { - "lower": -0.0343, - "upper": 0.0343, - "method": "unknown" - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "global-mmlu-lite/openai_gpt-4.1-2025-04-14/1773936583.743359", - "retrieved_timestamp": "1773936583.743359", - "source_metadata": { - "source_name": "Global MMLU Lite Leaderboard", - "source_type": "documentation", - "source_organization_name": "kaggle", - "source_organization_url": "www.kaggle.com", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "kaggle kernel", - "version": "4", - "additional_details": { - "url": "https://www.kaggle.com/code/shivalikasingh95/global-mmlu-lite-sample-notebook" - } - }, - "benchmark": "global-mmlu-lite", - "evaluation_results": [ - { - "evaluation_name": "Global MMLU Lite", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Global MMLU Lite", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8755 - } - }, - { - "evaluation_name": "Culturally Sensitive", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Sensitive", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8541 - } - }, - { - "evaluation_name": "Culturally Agnostic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Agnostic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8969 - } - }, - { - "evaluation_name": "Arabic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Arabic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.88, - "uncertainty": { - "confidence_interval": { - "lower": -0.0318, - "upper": 0.0318, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "English", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - English", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8825, - "uncertainty": { - "confidence_interval": { - "lower": -0.0316, - "upper": 0.0316, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Bengali", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Bengali", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8625, - "uncertainty": { - "confidence_interval": { - "lower": -0.0337, - "upper": 0.0337, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "German", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - German", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.875, - "uncertainty": { - "confidence_interval": { - "lower": -0.0324, - "upper": 0.0324, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "French", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - French", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8875, - "uncertainty": { - "confidence_interval": { - "lower": -0.031, - "upper": 0.031, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Hindi", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Hindi", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8775, - "uncertainty": { - "confidence_interval": { - "lower": -0.0321, - "upper": 0.0321, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Indonesian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Indonesian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.885, - "uncertainty": { - "confidence_interval": { - "lower": -0.0313, - "upper": 0.0313, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Italian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Italian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.88, - "uncertainty": { - "confidence_interval": { - "lower": -0.0318, - "upper": 0.0318, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Japanese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Japanese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8725, - "uncertainty": { - "confidence_interval": { - "lower": -0.0327, - "upper": 0.0327, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Korean", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Korean", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.87, - "uncertainty": { - "confidence_interval": { - "lower": -0.033, - "upper": 0.033, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Portuguese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Portuguese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.875, - "uncertainty": { - "confidence_interval": { - "lower": -0.0324, - "upper": 0.0324, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Spanish", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Spanish", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.885, - "uncertainty": { - "confidence_interval": { - "lower": -0.0313, - "upper": 0.0313, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Swahili", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Swahili", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8725, - "uncertainty": { - "confidence_interval": { - "lower": -0.0327, - "upper": 0.0327, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Yoruba", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Yoruba", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.875, - "uncertainty": { - "confidence_interval": { - "lower": -0.0324, - "upper": 0.0324, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Chinese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Chinese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.87, - "uncertainty": { - "confidence_interval": { - "lower": -0.033, - "upper": 0.033, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Burmese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Burmese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8575, - "uncertainty": { - "confidence_interval": { - "lower": -0.0343, - "upper": 0.0343, - "method": "unknown" - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "helm_capabilities/openai_gpt-4.1-2025-04-14/1774096304.5056698", - "retrieved_timestamp": "1774096304.5056698", - "source_metadata": { - "source_name": "helm_capabilities", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_capabilities", - "evaluation_results": [ - { - "evaluation_name": "Mean score", - "source_data": { - "dataset_name": "helm_capabilities", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "The mean of the scores from all columns.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.727, - "details": { - "description": "", - "tab": "Accuracy", - "Mean score - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"11.09172884853167\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU-Pro", - "source_data": { - "dataset_name": "MMLU-Pro", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on MMLU-Pro", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.811, - "details": { - "description": "min=0.811, mean=0.811, max=0.811, sum=0.811 (1)", - "tab": "Accuracy", - "MMLU-Pro - Observed inference time (s)": "{\"description\": \"min=6.431, mean=6.431, max=6.431, sum=6.431 (1)\", \"tab\": \"Efficiency\", \"score\": \"6.431383004903793\"}", - "MMLU-Pro - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "MMLU-Pro - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - # prompt tokens": "{\"description\": \"min=228.569, mean=228.569, max=228.569, sum=228.569 (1)\", \"tab\": \"General information\", \"score\": \"228.569\"}", - "MMLU-Pro - # output tokens": "{\"description\": \"min=513.15, mean=513.15, max=513.15, sum=513.15 (1)\", \"tab\": \"General information\", \"score\": \"513.15\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"all\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.659, - "details": { - "description": "min=0.659, mean=0.659, max=0.659, sum=0.659 (1)", - "tab": "Accuracy", - "GPQA - Observed inference time (s)": "{\"description\": \"min=9.906, mean=9.906, max=9.906, sum=9.906 (1)\", \"tab\": \"Efficiency\", \"score\": \"9.906458986714282\"}", - "GPQA - # eval": "{\"description\": \"min=446, mean=446, max=446, sum=446 (1)\", \"tab\": \"General information\", \"score\": \"446.0\"}", - "GPQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - # prompt tokens": "{\"description\": \"min=248.152, mean=248.152, max=248.152, sum=248.152 (1)\", \"tab\": \"General information\", \"score\": \"248.152466367713\"}", - "GPQA - # output tokens": "{\"description\": \"min=824.722, mean=824.722, max=824.722, sum=824.722 (1)\", \"tab\": \"General information\", \"score\": \"824.7219730941704\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"gpqa_main\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "IFEval Strict Acc on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.838, - "details": { - "description": "min=0.838, mean=0.838, max=0.838, sum=0.838 (1)", - "tab": "Accuracy", - "IFEval - Observed inference time (s)": "{\"description\": \"min=3.68, mean=3.68, max=3.68, sum=3.68 (1)\", \"tab\": \"Efficiency\", \"score\": \"3.6797932344531836\"}", - "IFEval - # eval": "{\"description\": \"min=541, mean=541, max=541, sum=541 (1)\", \"tab\": \"General information\", \"score\": \"541.0\"}", - "IFEval - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - # prompt tokens": "{\"description\": \"min=45.671, mean=45.671, max=45.671, sum=45.671 (1)\", \"tab\": \"General information\", \"score\": \"45.67097966728281\"}", - "IFEval - # output tokens": "{\"description\": \"min=277.305, mean=277.305, max=277.305, sum=277.305 (1)\", \"tab\": \"General information\", \"score\": \"277.3049907578558\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WildBench", - "source_data": { - "dataset_name": "WildBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "WB Score on WildBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.854, - "details": { - "description": "min=0.854, mean=0.854, max=0.854, sum=0.854 (1)", - "tab": "Accuracy", - "WildBench - Observed inference time (s)": "{\"description\": \"min=11.723, mean=11.723, max=11.723, sum=11.723 (1)\", \"tab\": \"Efficiency\", \"score\": \"11.72278983767207\"}", - "WildBench - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "WildBench - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # prompt tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # output tokens": "{\"description\": \"min=1007.831, mean=1007.831, max=1007.831, sum=1007.831 (1)\", \"tab\": \"General information\", \"score\": \"1007.831\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"v2\"" - } - } - }, - { - "evaluation_name": "Omni-MATH", - "source_data": { - "dataset_name": "Omni-MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Acc on Omni-MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.471, - "details": { - "description": "min=0.471, mean=0.471, max=0.471, sum=0.471 (1)", - "tab": "Accuracy", - "Omni-MATH - Observed inference time (s)": "{\"description\": \"min=23.718, mean=23.718, max=23.718, sum=23.718 (1)\", \"tab\": \"Efficiency\", \"score\": \"23.718219178915025\"}", - "Omni-MATH - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "Omni-MATH - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - # prompt tokens": "{\"description\": \"min=109.623, mean=109.623, max=109.623, sum=109.623 (1)\", \"tab\": \"General information\", \"score\": \"109.623\"}", - "Omni-MATH - # output tokens": "{\"description\": \"min=1884.743, mean=1884.743, max=1884.743, sum=1884.743 (1)\", \"tab\": \"General information\", \"score\": \"1884.743\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "reward-bench-2/openai_gpt-4.1-2025-04-14/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7232 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8289 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3974 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6521 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8726 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7338 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8542 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/openai_gpt-4.1-mini-2025-04-14.json b/data/models/openai_gpt-4.1-mini-2025-04-14.json deleted file mode 100644 index db4f0a0ff653797bc76d2d58c786f223c8beea82..0000000000000000000000000000000000000000 --- a/data/models/openai_gpt-4.1-mini-2025-04-14.json +++ /dev/null @@ -1,384 +0,0 @@ -{ - "model_info": { - "name": "GPT-4.1 mini 2025-04-14", - "id": "openai/gpt-4.1-mini-2025-04-14", - "developer": "OpenAI", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_capabilities/openai_gpt-4.1-mini-2025-04-14/1774096304.5056698", - "retrieved_timestamp": "1774096304.5056698", - "source_metadata": { - "source_name": "helm_capabilities", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_capabilities", - "evaluation_results": [ - { - "evaluation_name": "Mean score", - "source_data": { - "dataset_name": "helm_capabilities", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "The mean of the scores from all columns.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.726, - "details": { - "description": "", - "tab": "Accuracy", - "Mean score - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"7.701476623313954\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU-Pro", - "source_data": { - "dataset_name": "MMLU-Pro", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on MMLU-Pro", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.783, - "details": { - "description": "min=0.783, mean=0.783, max=0.783, sum=0.783 (1)", - "tab": "Accuracy", - "MMLU-Pro - Observed inference time (s)": "{\"description\": \"min=4.927, mean=4.927, max=4.927, sum=4.927 (1)\", \"tab\": \"Efficiency\", \"score\": \"4.927327474832535\"}", - "MMLU-Pro - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "MMLU-Pro - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - # prompt tokens": "{\"description\": \"min=228.569, mean=228.569, max=228.569, sum=228.569 (1)\", \"tab\": \"General information\", \"score\": \"228.569\"}", - "MMLU-Pro - # output tokens": "{\"description\": \"min=627.909, mean=627.909, max=627.909, sum=627.909 (1)\", \"tab\": \"General information\", \"score\": \"627.909\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"all\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.614, - "details": { - "description": "min=0.614, mean=0.614, max=0.614, sum=0.614 (1)", - "tab": "Accuracy", - "GPQA - Observed inference time (s)": "{\"description\": \"min=8.217, mean=8.217, max=8.217, sum=8.217 (1)\", \"tab\": \"Efficiency\", \"score\": \"8.216832675206822\"}", - "GPQA - # eval": "{\"description\": \"min=446, mean=446, max=446, sum=446 (1)\", \"tab\": \"General information\", \"score\": \"446.0\"}", - "GPQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - # prompt tokens": "{\"description\": \"min=248.152, mean=248.152, max=248.152, sum=248.152 (1)\", \"tab\": \"General information\", \"score\": \"248.152466367713\"}", - "GPQA - # output tokens": "{\"description\": \"min=1056.354, mean=1056.354, max=1056.354, sum=1056.354 (1)\", \"tab\": \"General information\", \"score\": \"1056.354260089686\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"gpqa_main\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "IFEval Strict Acc on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.904, - "details": { - "description": "min=0.904, mean=0.904, max=0.904, sum=0.904 (1)", - "tab": "Accuracy", - "IFEval - Observed inference time (s)": "{\"description\": \"min=2.622, mean=2.622, max=2.622, sum=2.622 (1)\", \"tab\": \"Efficiency\", \"score\": \"2.6219342847848774\"}", - "IFEval - # eval": "{\"description\": \"min=541, mean=541, max=541, sum=541 (1)\", \"tab\": \"General information\", \"score\": \"541.0\"}", - "IFEval - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - # prompt tokens": "{\"description\": \"min=45.671, mean=45.671, max=45.671, sum=45.671 (1)\", \"tab\": \"General information\", \"score\": \"45.67097966728281\"}", - "IFEval - # output tokens": "{\"description\": \"min=275.1, mean=275.1, max=275.1, sum=275.1 (1)\", \"tab\": \"General information\", \"score\": \"275.09981515711644\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WildBench", - "source_data": { - "dataset_name": "WildBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "WB Score on WildBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.838, - "details": { - "description": "min=0.838, mean=0.838, max=0.838, sum=0.838 (1)", - "tab": "Accuracy", - "WildBench - Observed inference time (s)": "{\"description\": \"min=7.331, mean=7.331, max=7.331, sum=7.331 (1)\", \"tab\": \"Efficiency\", \"score\": \"7.3305598454475405\"}", - "WildBench - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "WildBench - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # prompt tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # output tokens": "{\"description\": \"min=1020.373, mean=1020.373, max=1020.373, sum=1020.373 (1)\", \"tab\": \"General information\", \"score\": \"1020.373\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"v2\"" - } - } - }, - { - "evaluation_name": "Omni-MATH", - "source_data": { - "dataset_name": "Omni-MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Acc on Omni-MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.491, - "details": { - "description": "min=0.491, mean=0.491, max=0.491, sum=0.491 (1)", - "tab": "Accuracy", - "Omni-MATH - Observed inference time (s)": "{\"description\": \"min=15.411, mean=15.411, max=15.411, sum=15.411 (1)\", \"tab\": \"Efficiency\", \"score\": \"15.41072883629799\"}", - "Omni-MATH - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "Omni-MATH - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - # prompt tokens": "{\"description\": \"min=109.623, mean=109.623, max=109.623, sum=109.623 (1)\", \"tab\": \"General information\", \"score\": \"109.623\"}", - "Omni-MATH - # output tokens": "{\"description\": \"min=2117.264, mean=2117.264, max=2117.264, sum=2117.264 (1)\", \"tab\": \"General information\", \"score\": \"2117.264\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "reward-bench-2/openai_gpt-4.1-mini-2025-04-14/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6573 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6084 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4125 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7213 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7265 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7354 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.74 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/openai_gpt-4.1-nano-2025-04-14.json b/data/models/openai_gpt-4.1-nano-2025-04-14.json deleted file mode 100644 index d13d3015b1ffa3135bdb85367be960c7ccec78b9..0000000000000000000000000000000000000000 --- a/data/models/openai_gpt-4.1-nano-2025-04-14.json +++ /dev/null @@ -1,384 +0,0 @@ -{ - "model_info": { - "name": "GPT-4.1 nano 2025-04-14", - "id": "openai/gpt-4.1-nano-2025-04-14", - "developer": "OpenAI", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_capabilities/openai_gpt-4.1-nano-2025-04-14/1774096304.5056698", - "retrieved_timestamp": "1774096304.5056698", - "source_metadata": { - "source_name": "helm_capabilities", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_capabilities", - "evaluation_results": [ - { - "evaluation_name": "Mean score", - "source_data": { - "dataset_name": "helm_capabilities", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "The mean of the scores from all columns.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.616, - "details": { - "description": "", - "tab": "Accuracy", - "Mean score - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"4.5128146238794296\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU-Pro", - "source_data": { - "dataset_name": "MMLU-Pro", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on MMLU-Pro", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.55, - "details": { - "description": "min=0.55, mean=0.55, max=0.55, sum=0.55 (1)", - "tab": "Accuracy", - "MMLU-Pro - Observed inference time (s)": "{\"description\": \"min=2.935, mean=2.935, max=2.935, sum=2.935 (1)\", \"tab\": \"Efficiency\", \"score\": \"2.9353291485309603\"}", - "MMLU-Pro - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "MMLU-Pro - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - # prompt tokens": "{\"description\": \"min=228.569, mean=228.569, max=228.569, sum=228.569 (1)\", \"tab\": \"General information\", \"score\": \"228.569\"}", - "MMLU-Pro - # output tokens": "{\"description\": \"min=503.09, mean=503.09, max=503.09, sum=503.09 (1)\", \"tab\": \"General information\", \"score\": \"503.09\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"all\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.507, - "details": { - "description": "min=0.507, mean=0.507, max=0.507, sum=0.507 (1)", - "tab": "Accuracy", - "GPQA - Observed inference time (s)": "{\"description\": \"min=4.817, mean=4.817, max=4.817, sum=4.817 (1)\", \"tab\": \"Efficiency\", \"score\": \"4.816804544808084\"}", - "GPQA - # eval": "{\"description\": \"min=446, mean=446, max=446, sum=446 (1)\", \"tab\": \"General information\", \"score\": \"446.0\"}", - "GPQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - # prompt tokens": "{\"description\": \"min=248.152, mean=248.152, max=248.152, sum=248.152 (1)\", \"tab\": \"General information\", \"score\": \"248.152466367713\"}", - "GPQA - # output tokens": "{\"description\": \"min=842.038, mean=842.038, max=842.038, sum=842.038 (1)\", \"tab\": \"General information\", \"score\": \"842.0381165919282\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"gpqa_main\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "IFEval Strict Acc on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.843, - "details": { - "description": "min=0.843, mean=0.843, max=0.843, sum=0.843 (1)", - "tab": "Accuracy", - "IFEval - Observed inference time (s)": "{\"description\": \"min=1.781, mean=1.781, max=1.781, sum=1.781 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.7811373196776386\"}", - "IFEval - # eval": "{\"description\": \"min=541, mean=541, max=541, sum=541 (1)\", \"tab\": \"General information\", \"score\": \"541.0\"}", - "IFEval - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - # prompt tokens": "{\"description\": \"min=45.671, mean=45.671, max=45.671, sum=45.671 (1)\", \"tab\": \"General information\", \"score\": \"45.67097966728281\"}", - "IFEval - # output tokens": "{\"description\": \"min=269.619, mean=269.619, max=269.619, sum=269.619 (1)\", \"tab\": \"General information\", \"score\": \"269.6192236598891\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WildBench", - "source_data": { - "dataset_name": "WildBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "WB Score on WildBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.811, - "details": { - "description": "min=0.811, mean=0.811, max=0.811, sum=0.811 (1)", - "tab": "Accuracy", - "WildBench - Observed inference time (s)": "{\"description\": \"min=4.453, mean=4.453, max=4.453, sum=4.453 (1)\", \"tab\": \"Efficiency\", \"score\": \"4.453118676900863\"}", - "WildBench - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "WildBench - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # prompt tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # output tokens": "{\"description\": \"min=909.661, mean=909.661, max=909.661, sum=909.661 (1)\", \"tab\": \"General information\", \"score\": \"909.661\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"v2\"" - } - } - }, - { - "evaluation_name": "Omni-MATH", - "source_data": { - "dataset_name": "Omni-MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Acc on Omni-MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.367, - "details": { - "description": "min=0.367, mean=0.367, max=0.367, sum=0.367 (1)", - "tab": "Accuracy", - "Omni-MATH - Observed inference time (s)": "{\"description\": \"min=8.578, mean=8.578, max=8.578, sum=8.578 (1)\", \"tab\": \"Efficiency\", \"score\": \"8.577683429479599\"}", - "Omni-MATH - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "Omni-MATH - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - # prompt tokens": "{\"description\": \"min=109.623, mean=109.623, max=109.623, sum=109.623 (1)\", \"tab\": \"General information\", \"score\": \"109.623\"}", - "Omni-MATH - # output tokens": "{\"description\": \"min=1777.605, mean=1777.605, max=1777.605, sum=1777.605 (1)\", \"tab\": \"General information\", \"score\": \"1777.605\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "reward-bench-2/openai_gpt-4.1-nano-2025-04-14/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4849 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4646 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2578 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5041 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7156 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.466 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5015 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/openai_gpt-4.1.json b/data/models/openai_gpt-4.1.json deleted file mode 100644 index 65b386538f381293003e2af60e00f6dd4d6c2b8a..0000000000000000000000000000000000000000 --- a/data/models/openai_gpt-4.1.json +++ /dev/null @@ -1,89 +0,0 @@ -{ - "model_info": { - "name": "openai/gpt-4.1", - "developer": "OpenAI", - "inference_platform": "openrouter", - "id": "openai/gpt-4.1" - }, - "evaluations": [ - { - "evaluation_id": "livecodebenchpro/openai/gpt-4.1/1760492095.8105888", - "retrieved_timestamp": "1760492095.8105888", - "source_metadata": { - "source_organization_name": "New York University, Princeton University, University of California San Diego, University of Washington and Canyon Crest Academy", - "evaluator_relationship": "third_party", - "source_name": "Live Code Bench Pro", - "source_type": "documentation" - }, - "eval_library": { - "name": "unknown", - "version": "unknown" - }, - "benchmark": "livecodebenchpro", - "evaluation_results": [ - { - "evaluation_name": "Hard Problems", - "metric_config": { - "evaluation_description": "Pass@1 on Hard Problems", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.0 - }, - "source_data": { - "dataset_name": "Hard Problems", - "source_type": "url", - "url": [ - "https://webhook.cp-bench.orzzh.com/leaderboard/llm/difficulty?difficulty=hard&benchmark_mode=live" - ] - } - }, - { - "evaluation_name": "Medium Problems", - "metric_config": { - "evaluation_description": "Pass@1 on Medium Problems", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.0 - }, - "source_data": { - "dataset_name": "Medium Problems", - "source_type": "url", - "url": [ - "https://webhook.cp-bench.orzzh.com/leaderboard/llm/difficulty?difficulty=medium&benchmark_mode=live" - ] - } - }, - { - "evaluation_name": "Easy Problems", - "metric_config": { - "evaluation_description": "Pass@1 on Easy Problems", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.19718309859154928 - }, - "source_data": { - "dataset_name": "Easy Problems", - "source_type": "url", - "url": [ - "https://webhook.cp-bench.orzzh.com/leaderboard/llm/difficulty?difficulty=easy&benchmark_mode=live" - ] - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/openai_gpt-4o-2024-05-13.json b/data/models/openai_gpt-4o-2024-05-13.json deleted file mode 100644 index a05de0df34e073dc7fc53dfed9432b44e70f8d67..0000000000000000000000000000000000000000 --- a/data/models/openai_gpt-4o-2024-05-13.json +++ /dev/null @@ -1,2035 +0,0 @@ -{ - "model_info": { - "name": "GPT-4o 2024-05-13", - "id": "openai/gpt-4o-2024-05-13", - "developer": "OpenAI", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_lite/openai_gpt-4o-2024-05-13/1774096306.427425", - "retrieved_timestamp": "1774096306.427425", - "source_metadata": { - "source_name": "helm_lite", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_lite", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_lite", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.938, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.6270536828963795\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.804, - "details": { - "description": "min=0.804, mean=0.804, max=0.804, sum=0.804 (1)", - "tab": "Accuracy", - "NarrativeQA - Observed inference time (s)": "{\"description\": \"min=0.556, mean=0.556, max=0.556, sum=0.556 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.5561933571184186\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=3461.668, mean=3461.668, max=3461.668, sum=3461.668 (1)\", \"tab\": \"General information\", \"score\": \"3461.667605633803\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=4.62, mean=4.62, max=4.62, sum=4.62 (1)\", \"tab\": \"General information\", \"score\": \"4.619718309859155\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (closed-book)", - "source_data": { - "dataset_name": "NaturalQuestions (closed-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (closed-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.501, - "details": { - "description": "min=0.501, mean=0.501, max=0.501, sum=0.501 (1)", - "tab": "Accuracy", - "NaturalQuestions (open-book) - Observed inference time (s)": "{\"description\": \"min=0.507, mean=0.507, max=0.507, sum=0.507 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.5071200861930847\"}", - "NaturalQuestions (closed-book) - Observed inference time (s)": "{\"description\": \"min=0.461, mean=0.461, max=0.461, sum=0.461 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.46105142664909365\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1724.02, mean=1724.02, max=1724.02, sum=1724.02 (1)\", \"tab\": \"General information\", \"score\": \"1724.02\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=5.41, mean=5.41, max=5.41, sum=5.41 (1)\", \"tab\": \"General information\", \"score\": \"5.41\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=139.953, mean=139.953, max=139.953, sum=139.953 (1)\", \"tab\": \"General information\", \"score\": \"139.953\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=4.245, mean=4.245, max=4.245, sum=4.245 (1)\", \"tab\": \"General information\", \"score\": \"4.245\"}" - } - }, - "generation_config": { - "additional_details": { - "mode": "\"closedbook\"" - } - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.966, - "details": { - "description": "min=0.966, mean=0.966, max=0.966, sum=0.966 (1)", - "tab": "Accuracy", - "OpenbookQA - Observed inference time (s)": "{\"description\": \"min=0.402, mean=0.402, max=0.402, sum=0.402 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.4019911346435547\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=245.486, mean=245.486, max=245.486, sum=245.486 (1)\", \"tab\": \"General information\", \"score\": \"245.486\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "dataset": "\"openbookqa\"", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.748, - "details": { - "description": "min=0.61, mean=0.748, max=0.95, sum=3.742 (5)", - "tab": "Accuracy", - "MMLU - Observed inference time (s)": "{\"description\": \"min=0.353, mean=0.39, max=0.416, sum=1.952 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.3904274333485386\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=373.42, mean=466.992, max=613.228, sum=2334.958 (5)\", \"tab\": \"General information\", \"score\": \"466.9916140350877\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"college_chemistry\", \"computer_security\", \"econometrics\", \"us_foreign_policy\"]", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MATH", - "source_data": { - "dataset_name": "MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Equivalent (CoT) on MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.829, - "details": { - "description": "min=0.632, mean=0.829, max=0.977, sum=5.802 (7)", - "tab": "Accuracy", - "MATH - Observed inference time (s)": "{\"description\": \"min=3.334, mean=4.358, max=4.85, sum=30.503 (7)\", \"tab\": \"Efficiency\", \"score\": \"4.357550465458739\"}", - "MATH - # eval": "{\"description\": \"min=30, mean=62.429, max=135, sum=437 (7)\", \"tab\": \"General information\", \"score\": \"62.42857142857143\"}", - "MATH - # train": "{\"description\": \"min=8, mean=8, max=8, sum=56 (7)\", \"tab\": \"General information\", \"score\": \"8.0\"}", - "MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MATH - # prompt tokens": "{\"description\": \"min=888.43, mean=1273.32, max=2222.25, sum=8913.243 (7)\", \"tab\": \"General information\", \"score\": \"1273.320452019534\"}", - "MATH - # output tokens": "{\"description\": \"min=187.942, mean=245.482, max=284.788, sum=1718.377 (7)\", \"tab\": \"General information\", \"score\": \"245.4823665454633\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"algebra\", \"counting_and_probability\", \"geometry\", \"intermediate_algebra\", \"number_theory\", \"prealgebra\", \"precalculus\"]", - "level": "\"1\"", - "use_official_examples": "\"False\"", - "use_chain_of_thought": "\"True\"" - } - } - }, - { - "evaluation_name": "GSM8K", - "source_data": { - "dataset_name": "GSM8K", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on GSM8K", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.905, - "details": { - "description": "min=0.905, mean=0.905, max=0.905, sum=0.905 (1)", - "tab": "Accuracy", - "GSM8K - Observed inference time (s)": "{\"description\": \"min=4.227, mean=4.227, max=4.227, sum=4.227 (1)\", \"tab\": \"Efficiency\", \"score\": \"4.227096201658249\"}", - "GSM8K - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "GSM8K - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "GSM8K - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GSM8K - # prompt tokens": "{\"description\": \"min=952.617, mean=952.617, max=952.617, sum=952.617 (1)\", \"tab\": \"General information\", \"score\": \"952.617\"}", - "GSM8K - # output tokens": "{\"description\": \"min=213.475, mean=213.475, max=213.475, sum=213.475 (1)\", \"tab\": \"General information\", \"score\": \"213.475\"}" - } - }, - "generation_config": { - "additional_details": { - "stop": "\"none\"" - } - } - }, - { - "evaluation_name": "LegalBench", - "source_data": { - "dataset_name": "LegalBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on LegalBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.733, - "details": { - "description": "min=0.441, mean=0.733, max=0.989, sum=3.666 (5)", - "tab": "Accuracy", - "LegalBench - Observed inference time (s)": "{\"description\": \"min=0.36, mean=0.431, max=0.568, sum=2.154 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.4307274274560104\"}", - "LegalBench - # eval": "{\"description\": \"min=95, mean=409.4, max=1000, sum=2047 (5)\", \"tab\": \"General information\", \"score\": \"409.4\"}", - "LegalBench - # train": "{\"description\": \"min=4, mean=4.8, max=5, sum=24 (5)\", \"tab\": \"General information\", \"score\": \"4.8\"}", - "LegalBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "LegalBench - # prompt tokens": "{\"description\": \"min=208.179, mean=1512.795, max=6254.98, sum=7563.977 (5)\", \"tab\": \"General information\", \"score\": \"1512.7954037538377\"}", - "LegalBench - # output tokens": "{\"description\": \"min=1, mean=1.249, max=2.021, sum=6.244 (5)\", \"tab\": \"General information\", \"score\": \"1.2488971748171518\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"abercrombie\", \"corporate_lobbying\", \"function_of_decision_section\", \"international_citizenship_questions\", \"proa\"]" - } - } - }, - { - "evaluation_name": "MedQA", - "source_data": { - "dataset_name": "MedQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MedQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.857, - "details": { - "description": "min=0.857, mean=0.857, max=0.857, sum=0.857 (1)", - "tab": "Accuracy", - "MedQA - Observed inference time (s)": "{\"description\": \"min=0.407, mean=0.407, max=0.407, sum=0.407 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.4072816490416024\"}", - "MedQA - # eval": "{\"description\": \"min=503, mean=503, max=503, sum=503 (1)\", \"tab\": \"General information\", \"score\": \"503.0\"}", - "MedQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MedQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MedQA - # prompt tokens": "{\"description\": \"min=1009.05, mean=1009.05, max=1009.05, sum=1009.05 (1)\", \"tab\": \"General information\", \"score\": \"1009.0497017892644\"}", - "MedQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WMT 2014", - "source_data": { - "dataset_name": "WMT 2014", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "BLEU-4 on WMT 2014", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.231, - "details": { - "description": "min=0.176, mean=0.231, max=0.281, sum=1.154 (5)", - "tab": "Accuracy", - "WMT 2014 - Observed inference time (s)": "{\"description\": \"min=0.775, mean=0.842, max=0.967, sum=4.212 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.8424805298775759\"}", - "WMT 2014 - # eval": "{\"description\": \"min=503, mean=568.8, max=832, sum=2844 (5)\", \"tab\": \"General information\", \"score\": \"568.8\"}", - "WMT 2014 - # train": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "WMT 2014 - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WMT 2014 - # prompt tokens": "{\"description\": \"min=79.529, mean=115.006, max=138.497, sum=575.028 (5)\", \"tab\": \"General information\", \"score\": \"115.00557042361216\"}", - "WMT 2014 - # output tokens": "{\"description\": \"min=23.62, mean=25.287, max=26.018, sum=126.434 (5)\", \"tab\": \"General information\", \"score\": \"25.286879683437835\"}" - } - }, - "generation_config": { - "additional_details": { - "language_pair": "[\"cs-en\", \"de-en\", \"fr-en\", \"hi-en\", \"ru-en\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_mmlu/openai_gpt-4o-2024-05-13/1774096312.00548", - "retrieved_timestamp": "1774096312.00548", - "source_metadata": { - "source_name": "helm_mmlu", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_mmlu", - "evaluation_results": [ - { - "evaluation_name": "MMLU All Subjects", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU All Subjects", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.842, - "details": { - "description": "min=0.47, mean=0.842, max=0.979, sum=95.957 (114)", - "tab": "Accuracy", - "MMLU All Subjects - Observed inference time (s)": "{\"description\": \"min=0.314, mean=0.37, max=0.515, sum=42.144 (114)\", \"tab\": \"Efficiency\", \"score\": \"0.3696883367683005\"}", - "MMLU All Subjects - # eval": "{\"description\": \"min=100, mean=246.351, max=1534, sum=28084 (114)\", \"tab\": \"General information\", \"score\": \"246.35087719298247\"}", - "MMLU All Subjects - # train": "{\"description\": \"min=5, mean=5, max=5, sum=570 (114)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU All Subjects - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU All Subjects - # prompt tokens": "{\"description\": \"min=267.936, mean=612.332, max=2793.83, sum=69805.818 (114)\", \"tab\": \"General information\", \"score\": \"612.3317391408493\"}", - "MMLU All Subjects - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=114 (114)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_name": "Abstract Algebra", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Abstract Algebra", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.66, - "details": { - "description": "min=0.66, mean=0.66, max=0.66, sum=1.32 (2)", - "tab": "Accuracy", - "Abstract Algebra - Observed inference time (s)": "{\"description\": \"min=0.381, mean=0.381, max=0.381, sum=0.761 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.38067533016204835\"}", - "Abstract Algebra - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Abstract Algebra - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Abstract Algebra - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Abstract Algebra - # prompt tokens": "{\"description\": \"min=374.53, mean=374.53, max=374.53, sum=749.06 (2)\", \"tab\": \"General information\", \"score\": \"374.53\"}", - "Abstract Algebra - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"abstract_algebra\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_abstract_algebra\"" - } - } - }, - { - "evaluation_name": "Anatomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Anatomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.911, - "details": { - "description": "min=0.911, mean=0.911, max=0.911, sum=1.822 (2)", - "tab": "Accuracy", - "Anatomy - Observed inference time (s)": "{\"description\": \"min=0.333, mean=0.333, max=0.333, sum=0.666 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3328125264909532\"}", - "Anatomy - # eval": "{\"description\": \"min=135, mean=135, max=135, sum=270 (2)\", \"tab\": \"General information\", \"score\": \"135.0\"}", - "Anatomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Anatomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Anatomy - # prompt tokens": "{\"description\": \"min=350.6, mean=350.6, max=350.6, sum=701.2 (2)\", \"tab\": \"General information\", \"score\": \"350.6\"}", - "Anatomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"anatomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_anatomy\"" - } - } - }, - { - "evaluation_name": "College Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on College Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.686, - "details": { - "description": "min=0.686, mean=0.686, max=0.686, sum=1.373 (2)", - "tab": "Accuracy", - "College Chemistry - Observed inference time (s)": "{\"description\": \"min=0.473, mean=0.473, max=0.473, sum=0.947 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4733888053894043\"}", - "College Biology - Observed inference time (s)": "{\"description\": \"min=0.428, mean=0.428, max=0.428, sum=0.855 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4276181277301576\"}", - "College Computer Science - Observed inference time (s)": "{\"description\": \"min=0.367, mean=0.367, max=0.367, sum=0.734 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.36701245784759523\"}", - "College Mathematics - Observed inference time (s)": "{\"description\": \"min=0.332, mean=0.332, max=0.332, sum=0.665 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3324534225463867\"}", - "College Medicine - Observed inference time (s)": "{\"description\": \"min=0.365, mean=0.365, max=0.365, sum=0.73 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3647800649521668\"}", - "College Physics - Observed inference time (s)": "{\"description\": \"min=0.349, mean=0.349, max=0.349, sum=0.699 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3492975866093355\"}", - "College Chemistry - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Chemistry - # prompt tokens": "{\"description\": \"min=552.07, mean=552.07, max=552.07, sum=1104.14 (2)\", \"tab\": \"General information\", \"score\": \"552.07\"}", - "College Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Biology - # eval": "{\"description\": \"min=144, mean=144, max=144, sum=288 (2)\", \"tab\": \"General information\", \"score\": \"144.0\"}", - "College Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # prompt tokens": "{\"description\": \"min=468.056, mean=468.056, max=468.056, sum=936.111 (2)\", \"tab\": \"General information\", \"score\": \"468.05555555555554\"}", - "College Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # prompt tokens": "{\"description\": \"min=828.39, mean=828.39, max=828.39, sum=1656.78 (2)\", \"tab\": \"General information\", \"score\": \"828.39\"}", - "College Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Mathematics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # prompt tokens": "{\"description\": \"min=594.44, mean=594.44, max=594.44, sum=1188.88 (2)\", \"tab\": \"General information\", \"score\": \"594.44\"}", - "College Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Medicine - # eval": "{\"description\": \"min=173, mean=173, max=173, sum=346 (2)\", \"tab\": \"General information\", \"score\": \"173.0\"}", - "College Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # prompt tokens": "{\"description\": \"min=499.566, mean=499.566, max=499.566, sum=999.133 (2)\", \"tab\": \"General information\", \"score\": \"499.5664739884393\"}", - "College Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Physics - # eval": "{\"description\": \"min=102, mean=102, max=102, sum=204 (2)\", \"tab\": \"General information\", \"score\": \"102.0\"}", - "College Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # prompt tokens": "{\"description\": \"min=502.412, mean=502.412, max=502.412, sum=1004.824 (2)\", \"tab\": \"General information\", \"score\": \"502.4117647058824\"}", - "College Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"college_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_college_physics\"" - } - } - }, - { - "evaluation_name": "Computer Security", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Computer Security", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.85, - "details": { - "description": "min=0.85, mean=0.85, max=0.85, sum=1.7 (2)", - "tab": "Accuracy", - "Computer Security - Observed inference time (s)": "{\"description\": \"min=0.36, mean=0.36, max=0.36, sum=0.72 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.35994538068771365\"}", - "Computer Security - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Computer Security - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Computer Security - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Computer Security - # prompt tokens": "{\"description\": \"min=373.42, mean=373.42, max=373.42, sum=746.84 (2)\", \"tab\": \"General information\", \"score\": \"373.42\"}", - "Computer Security - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"computer_security\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_computer_security\"" - } - } - }, - { - "evaluation_name": "Econometrics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Econometrics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.693, - "details": { - "description": "min=0.693, mean=0.693, max=0.693, sum=1.386 (2)", - "tab": "Accuracy", - "Econometrics - Observed inference time (s)": "{\"description\": \"min=0.354, mean=0.354, max=0.354, sum=0.709 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3544190766518576\"}", - "Econometrics - # eval": "{\"description\": \"min=114, mean=114, max=114, sum=228 (2)\", \"tab\": \"General information\", \"score\": \"114.0\"}", - "Econometrics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Econometrics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Econometrics - # prompt tokens": "{\"description\": \"min=613.228, mean=613.228, max=613.228, sum=1226.456 (2)\", \"tab\": \"General information\", \"score\": \"613.2280701754386\"}", - "Econometrics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"econometrics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_econometrics\"" - } - } - }, - { - "evaluation_name": "Global Facts", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Global Facts", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.64, - "details": { - "description": "min=0.64, mean=0.64, max=0.64, sum=1.28 (2)", - "tab": "Accuracy", - "Global Facts - Observed inference time (s)": "{\"description\": \"min=0.326, mean=0.326, max=0.326, sum=0.653 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3264468240737915\"}", - "Global Facts - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Global Facts - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Global Facts - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Global Facts - # prompt tokens": "{\"description\": \"min=399.69, mean=399.69, max=399.69, sum=799.38 (2)\", \"tab\": \"General information\", \"score\": \"399.69\"}", - "Global Facts - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"global_facts\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_global_facts\"" - } - } - }, - { - "evaluation_name": "Jurisprudence", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Jurisprudence", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.898, - "details": { - "description": "min=0.898, mean=0.898, max=0.898, sum=1.796 (2)", - "tab": "Accuracy", - "Jurisprudence - Observed inference time (s)": "{\"description\": \"min=0.407, mean=0.407, max=0.407, sum=0.815 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.40749982330534196\"}", - "Jurisprudence - # eval": "{\"description\": \"min=108, mean=108, max=108, sum=216 (2)\", \"tab\": \"General information\", \"score\": \"108.0\"}", - "Jurisprudence - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Jurisprudence - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Jurisprudence - # prompt tokens": "{\"description\": \"min=391.231, mean=391.231, max=391.231, sum=782.463 (2)\", \"tab\": \"General information\", \"score\": \"391.23148148148147\"}", - "Jurisprudence - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"jurisprudence\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_jurisprudence\"" - } - } - }, - { - "evaluation_name": "Philosophy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Philosophy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9, - "details": { - "description": "min=0.9, mean=0.9, max=0.9, sum=1.801 (2)", - "tab": "Accuracy", - "Philosophy - Observed inference time (s)": "{\"description\": \"min=0.482, mean=0.482, max=0.482, sum=0.963 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.48153685373508665\"}", - "Philosophy - # eval": "{\"description\": \"min=311, mean=311, max=311, sum=622 (2)\", \"tab\": \"General information\", \"score\": \"311.0\"}", - "Philosophy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Philosophy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Philosophy - # prompt tokens": "{\"description\": \"min=327.92, mean=327.92, max=327.92, sum=655.839 (2)\", \"tab\": \"General information\", \"score\": \"327.91961414790995\"}", - "Philosophy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"philosophy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_philosophy\"" - } - } - }, - { - "evaluation_name": "Professional Psychology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Professional Psychology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.905, - "details": { - "description": "min=0.905, mean=0.905, max=0.905, sum=1.81 (2)", - "tab": "Accuracy", - "Professional Medicine - Observed inference time (s)": "{\"description\": \"min=0.386, mean=0.386, max=0.386, sum=0.772 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3862454724662444\"}", - "Professional Accounting - Observed inference time (s)": "{\"description\": \"min=0.347, mean=0.347, max=0.347, sum=0.694 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3472177982330322\"}", - "Professional Law - Observed inference time (s)": "{\"description\": \"min=0.474, mean=0.474, max=0.474, sum=0.947 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.47372100343915596\"}", - "Professional Psychology - Observed inference time (s)": "{\"description\": \"min=0.33, mean=0.33, max=0.33, sum=0.661 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.330327843528947\"}", - "Professional Medicine - # eval": "{\"description\": \"min=272, mean=272, max=272, sum=544 (2)\", \"tab\": \"General information\", \"score\": \"272.0\"}", - "Professional Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Medicine - # prompt tokens": "{\"description\": \"min=1071.18, mean=1071.18, max=1071.18, sum=2142.36 (2)\", \"tab\": \"General information\", \"score\": \"1071.1801470588234\"}", - "Professional Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Accounting - # eval": "{\"description\": \"min=282, mean=282, max=282, sum=564 (2)\", \"tab\": \"General information\", \"score\": \"282.0\"}", - "Professional Accounting - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Accounting - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # prompt tokens": "{\"description\": \"min=657.206, mean=657.206, max=657.206, sum=1314.411 (2)\", \"tab\": \"General information\", \"score\": \"657.2056737588653\"}", - "Professional Accounting - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Law - # eval": "{\"description\": \"min=1534, mean=1534, max=1534, sum=3068 (2)\", \"tab\": \"General information\", \"score\": \"1534.0\"}", - "Professional Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # prompt tokens": "{\"description\": \"min=1629.344, mean=1629.344, max=1629.344, sum=3258.687 (2)\", \"tab\": \"General information\", \"score\": \"1629.3435462842242\"}", - "Professional Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Psychology - # eval": "{\"description\": \"min=612, mean=612, max=612, sum=1224 (2)\", \"tab\": \"General information\", \"score\": \"612.0\"}", - "Professional Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # prompt tokens": "{\"description\": \"min=574.518, mean=574.518, max=574.518, sum=1149.036 (2)\", \"tab\": \"General information\", \"score\": \"574.5179738562091\"}", - "Professional Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"professional_psychology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_professional_psychology\"" - } - } - }, - { - "evaluation_name": "Us Foreign Policy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Us Foreign Policy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.96, - "details": { - "description": "min=0.96, mean=0.96, max=0.96, sum=1.92 (2)", - "tab": "Accuracy", - "Us Foreign Policy - Observed inference time (s)": "{\"description\": \"min=0.336, mean=0.336, max=0.336, sum=0.672 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.335811505317688\"}", - "Us Foreign Policy - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Us Foreign Policy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Us Foreign Policy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Us Foreign Policy - # prompt tokens": "{\"description\": \"min=421.71, mean=421.71, max=421.71, sum=843.42 (2)\", \"tab\": \"General information\", \"score\": \"421.71\"}", - "Us Foreign Policy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"us_foreign_policy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_us_foreign_policy\"" - } - } - }, - { - "evaluation_name": "Astronomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Astronomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.941, - "details": { - "description": "min=0.941, mean=0.941, max=0.941, sum=1.882 (2)", - "tab": "Accuracy", - "Astronomy - Observed inference time (s)": "{\"description\": \"min=0.349, mean=0.349, max=0.349, sum=0.697 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.34870150528456034\"}", - "Astronomy - # eval": "{\"description\": \"min=152, mean=152, max=152, sum=304 (2)\", \"tab\": \"General information\", \"score\": \"152.0\"}", - "Astronomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Astronomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Astronomy - # prompt tokens": "{\"description\": \"min=577.349, mean=577.349, max=577.349, sum=1154.697 (2)\", \"tab\": \"General information\", \"score\": \"577.3486842105264\"}", - "Astronomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"astronomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_astronomy\"" - } - } - }, - { - "evaluation_name": "Business Ethics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Business Ethics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.85, - "details": { - "description": "min=0.85, mean=0.85, max=0.85, sum=1.7 (2)", - "tab": "Accuracy", - "Business Ethics - Observed inference time (s)": "{\"description\": \"min=0.345, mean=0.345, max=0.345, sum=0.69 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3450936794281006\"}", - "Business Ethics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Business Ethics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Business Ethics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Business Ethics - # prompt tokens": "{\"description\": \"min=565.7, mean=565.7, max=565.7, sum=1131.4 (2)\", \"tab\": \"General information\", \"score\": \"565.7\"}", - "Business Ethics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"business_ethics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_business_ethics\"" - } - } - }, - { - "evaluation_name": "Clinical Knowledge", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Clinical Knowledge", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.894, - "details": { - "description": "min=0.894, mean=0.894, max=0.894, sum=1.789 (2)", - "tab": "Accuracy", - "Clinical Knowledge - Observed inference time (s)": "{\"description\": \"min=0.331, mean=0.331, max=0.331, sum=0.662 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.33114023748433813\"}", - "Clinical Knowledge - # eval": "{\"description\": \"min=265, mean=265, max=265, sum=530 (2)\", \"tab\": \"General information\", \"score\": \"265.0\"}", - "Clinical Knowledge - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Clinical Knowledge - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Clinical Knowledge - # prompt tokens": "{\"description\": \"min=400.985, mean=400.985, max=400.985, sum=801.97 (2)\", \"tab\": \"General information\", \"score\": \"400.98490566037736\"}", - "Clinical Knowledge - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"clinical_knowledge\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_clinical_knowledge\"" - } - } - }, - { - "evaluation_name": "Conceptual Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Conceptual Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.911, - "details": { - "description": "min=0.911, mean=0.911, max=0.911, sum=1.821 (2)", - "tab": "Accuracy", - "Conceptual Physics - Observed inference time (s)": "{\"description\": \"min=0.346, mean=0.346, max=0.346, sum=0.693 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.34625059391589874\"}", - "Conceptual Physics - # eval": "{\"description\": \"min=235, mean=235, max=235, sum=470 (2)\", \"tab\": \"General information\", \"score\": \"235.0\"}", - "Conceptual Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Conceptual Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Conceptual Physics - # prompt tokens": "{\"description\": \"min=304.677, mean=304.677, max=304.677, sum=609.353 (2)\", \"tab\": \"General information\", \"score\": \"304.67659574468087\"}", - "Conceptual Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"conceptual_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_conceptual_physics\"" - } - } - }, - { - "evaluation_name": "Electrical Engineering", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Electrical Engineering", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.807, - "details": { - "description": "min=0.807, mean=0.807, max=0.807, sum=1.614 (2)", - "tab": "Accuracy", - "Electrical Engineering - Observed inference time (s)": "{\"description\": \"min=0.359, mean=0.359, max=0.359, sum=0.717 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.35874251661629514\"}", - "Electrical Engineering - # eval": "{\"description\": \"min=145, mean=145, max=145, sum=290 (2)\", \"tab\": \"General information\", \"score\": \"145.0\"}", - "Electrical Engineering - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Electrical Engineering - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Electrical Engineering - # prompt tokens": "{\"description\": \"min=439.228, mean=439.228, max=439.228, sum=878.455 (2)\", \"tab\": \"General information\", \"score\": \"439.22758620689655\"}", - "Electrical Engineering - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"electrical_engineering\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_electrical_engineering\"" - } - } - }, - { - "evaluation_name": "Elementary Mathematics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Elementary Mathematics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.741, - "details": { - "description": "min=0.741, mean=0.741, max=0.741, sum=1.481 (2)", - "tab": "Accuracy", - "Elementary Mathematics - Observed inference time (s)": "{\"description\": \"min=0.36, mean=0.36, max=0.36, sum=0.721 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.360492156926917\"}", - "Elementary Mathematics - # eval": "{\"description\": \"min=378, mean=378, max=378, sum=756 (2)\", \"tab\": \"General information\", \"score\": \"378.0\"}", - "Elementary Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Elementary Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Elementary Mathematics - # prompt tokens": "{\"description\": \"min=532.683, mean=532.683, max=532.683, sum=1065.365 (2)\", \"tab\": \"General information\", \"score\": \"532.6825396825396\"}", - "Elementary Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"elementary_mathematics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_elementary_mathematics\"" - } - } - }, - { - "evaluation_name": "Formal Logic", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Formal Logic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.683, - "details": { - "description": "min=0.683, mean=0.683, max=0.683, sum=1.365 (2)", - "tab": "Accuracy", - "Formal Logic - Observed inference time (s)": "{\"description\": \"min=0.458, mean=0.458, max=0.458, sum=0.915 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4577372566102043\"}", - "Formal Logic - # eval": "{\"description\": \"min=126, mean=126, max=126, sum=252 (2)\", \"tab\": \"General information\", \"score\": \"126.0\"}", - "Formal Logic - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Formal Logic - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Formal Logic - # prompt tokens": "{\"description\": \"min=604.492, mean=604.492, max=604.492, sum=1208.984 (2)\", \"tab\": \"General information\", \"score\": \"604.4920634920635\"}", - "Formal Logic - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"formal_logic\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_formal_logic\"" - } - } - }, - { - "evaluation_name": "High School World History", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on High School World History", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.945, - "details": { - "description": "min=0.945, mean=0.945, max=0.945, sum=1.89 (2)", - "tab": "Accuracy", - "High School Biology - Observed inference time (s)": "{\"description\": \"min=0.422, mean=0.422, max=0.422, sum=0.844 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.42223084818932316\"}", - "High School Chemistry - Observed inference time (s)": "{\"description\": \"min=0.352, mean=0.352, max=0.352, sum=0.703 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3515606560730582\"}", - "High School Computer Science - Observed inference time (s)": "{\"description\": \"min=0.39, mean=0.39, max=0.39, sum=0.78 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.39000784397125243\"}", - "High School European History - Observed inference time (s)": "{\"description\": \"min=0.515, mean=0.515, max=0.515, sum=1.029 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5147185542366721\"}", - "High School Geography - Observed inference time (s)": "{\"description\": \"min=0.349, mean=0.349, max=0.349, sum=0.697 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.34874117615247013\"}", - "High School Government And Politics - Observed inference time (s)": "{\"description\": \"min=0.425, mean=0.425, max=0.425, sum=0.85 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4252293505199215\"}", - "High School Macroeconomics - Observed inference time (s)": "{\"description\": \"min=0.342, mean=0.342, max=0.342, sum=0.684 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3419678932581192\"}", - "High School Mathematics - Observed inference time (s)": "{\"description\": \"min=0.348, mean=0.348, max=0.348, sum=0.697 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3482617440047088\"}", - "High School Microeconomics - Observed inference time (s)": "{\"description\": \"min=0.41, mean=0.41, max=0.41, sum=0.819 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4096046676154898\"}", - "High School Physics - Observed inference time (s)": "{\"description\": \"min=0.365, mean=0.365, max=0.365, sum=0.731 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.36535484427647874\"}", - "High School Psychology - Observed inference time (s)": "{\"description\": \"min=0.344, mean=0.344, max=0.344, sum=0.687 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3435875463923183\"}", - "High School Statistics - Observed inference time (s)": "{\"description\": \"min=0.343, mean=0.343, max=0.343, sum=0.687 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3434795880759204\"}", - "High School US History - Observed inference time (s)": "{\"description\": \"min=0.508, mean=0.508, max=0.508, sum=1.016 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5077870616725847\"}", - "High School World History - Observed inference time (s)": "{\"description\": \"min=0.445, mean=0.445, max=0.445, sum=0.891 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.44530287473010616\"}", - "High School Biology - # eval": "{\"description\": \"min=310, mean=310, max=310, sum=620 (2)\", \"tab\": \"General information\", \"score\": \"310.0\"}", - "High School Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Biology - # prompt tokens": "{\"description\": \"min=504.874, mean=504.874, max=504.874, sum=1009.748 (2)\", \"tab\": \"General information\", \"score\": \"504.8741935483871\"}", - "High School Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Chemistry - # eval": "{\"description\": \"min=203, mean=203, max=203, sum=406 (2)\", \"tab\": \"General information\", \"score\": \"203.0\"}", - "High School Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # prompt tokens": "{\"description\": \"min=495.34, mean=495.34, max=495.34, sum=990.68 (2)\", \"tab\": \"General information\", \"score\": \"495.3399014778325\"}", - "High School Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "High School Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # prompt tokens": "{\"description\": \"min=865.8, mean=865.8, max=865.8, sum=1731.6 (2)\", \"tab\": \"General information\", \"score\": \"865.8\"}", - "High School Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School European History - # eval": "{\"description\": \"min=165, mean=165, max=165, sum=330 (2)\", \"tab\": \"General information\", \"score\": \"165.0\"}", - "High School European History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School European History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # prompt tokens": "{\"description\": \"min=2793.83, mean=2793.83, max=2793.83, sum=5587.661 (2)\", \"tab\": \"General information\", \"score\": \"2793.830303030303\"}", - "High School European History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Geography - # eval": "{\"description\": \"min=198, mean=198, max=198, sum=396 (2)\", \"tab\": \"General information\", \"score\": \"198.0\"}", - "High School Geography - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Geography - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # prompt tokens": "{\"description\": \"min=372.783, mean=372.783, max=372.783, sum=745.566 (2)\", \"tab\": \"General information\", \"score\": \"372.7828282828283\"}", - "High School Geography - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Government And Politics - # eval": "{\"description\": \"min=193, mean=193, max=193, sum=386 (2)\", \"tab\": \"General information\", \"score\": \"193.0\"}", - "High School Government And Politics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Government And Politics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # prompt tokens": "{\"description\": \"min=463.01, mean=463.01, max=463.01, sum=926.021 (2)\", \"tab\": \"General information\", \"score\": \"463.0103626943005\"}", - "High School Government And Politics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Macroeconomics - # eval": "{\"description\": \"min=390, mean=390, max=390, sum=780 (2)\", \"tab\": \"General information\", \"score\": \"390.0\"}", - "High School Macroeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Macroeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # prompt tokens": "{\"description\": \"min=371.451, mean=371.451, max=371.451, sum=742.903 (2)\", \"tab\": \"General information\", \"score\": \"371.4512820512821\"}", - "High School Macroeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Mathematics - # eval": "{\"description\": \"min=270, mean=270, max=270, sum=540 (2)\", \"tab\": \"General information\", \"score\": \"270.0\"}", - "High School Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # prompt tokens": "{\"description\": \"min=532.456, mean=532.456, max=532.456, sum=1064.911 (2)\", \"tab\": \"General information\", \"score\": \"532.4555555555555\"}", - "High School Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Microeconomics - # eval": "{\"description\": \"min=238, mean=238, max=238, sum=476 (2)\", \"tab\": \"General information\", \"score\": \"238.0\"}", - "High School Microeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Microeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # prompt tokens": "{\"description\": \"min=398.739, mean=398.739, max=398.739, sum=797.479 (2)\", \"tab\": \"General information\", \"score\": \"398.73949579831935\"}", - "High School Microeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Physics - # eval": "{\"description\": \"min=151, mean=151, max=151, sum=302 (2)\", \"tab\": \"General information\", \"score\": \"151.0\"}", - "High School Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # prompt tokens": "{\"description\": \"min=560.238, mean=560.238, max=560.238, sum=1120.477 (2)\", \"tab\": \"General information\", \"score\": \"560.2384105960265\"}", - "High School Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Psychology - # eval": "{\"description\": \"min=545, mean=545, max=545, sum=1090 (2)\", \"tab\": \"General information\", \"score\": \"545.0\"}", - "High School Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # prompt tokens": "{\"description\": \"min=492.917, mean=492.917, max=492.917, sum=985.835 (2)\", \"tab\": \"General information\", \"score\": \"492.91743119266056\"}", - "High School Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Statistics - # eval": "{\"description\": \"min=216, mean=216, max=216, sum=432 (2)\", \"tab\": \"General information\", \"score\": \"216.0\"}", - "High School Statistics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Statistics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # prompt tokens": "{\"description\": \"min=787.574, mean=787.574, max=787.574, sum=1575.148 (2)\", \"tab\": \"General information\", \"score\": \"787.574074074074\"}", - "High School Statistics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School US History - # eval": "{\"description\": \"min=204, mean=204, max=204, sum=408 (2)\", \"tab\": \"General information\", \"score\": \"204.0\"}", - "High School US History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School US History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # prompt tokens": "{\"description\": \"min=2220.005, mean=2220.005, max=2220.005, sum=4440.01 (2)\", \"tab\": \"General information\", \"score\": \"2220.0049019607845\"}", - "High School US History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School World History - # eval": "{\"description\": \"min=237, mean=237, max=237, sum=474 (2)\", \"tab\": \"General information\", \"score\": \"237.0\"}", - "High School World History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School World History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # prompt tokens": "{\"description\": \"min=1424.439, mean=1424.439, max=1424.439, sum=2848.878 (2)\", \"tab\": \"General information\", \"score\": \"1424.4388185654009\"}", - "High School World History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"high_school_world_history\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_high_school_world_history\"" - } - } - }, - { - "evaluation_name": "Human Sexuality", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Human Sexuality", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.908, - "details": { - "description": "min=0.908, mean=0.908, max=0.908, sum=1.817 (2)", - "tab": "Accuracy", - "Human Aging - Observed inference time (s)": "{\"description\": \"min=0.397, mean=0.397, max=0.397, sum=0.793 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.39673851637562296\"}", - "Human Sexuality - Observed inference time (s)": "{\"description\": \"min=0.372, mean=0.372, max=0.372, sum=0.744 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.37223931305281077\"}", - "Human Aging - # eval": "{\"description\": \"min=223, mean=223, max=223, sum=446 (2)\", \"tab\": \"General information\", \"score\": \"223.0\"}", - "Human Aging - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Aging - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Aging - # prompt tokens": "{\"description\": \"min=316.453, mean=316.453, max=316.453, sum=632.906 (2)\", \"tab\": \"General information\", \"score\": \"316.4529147982063\"}", - "Human Aging - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Human Sexuality - # eval": "{\"description\": \"min=131, mean=131, max=131, sum=262 (2)\", \"tab\": \"General information\", \"score\": \"131.0\"}", - "Human Sexuality - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Sexuality - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # prompt tokens": "{\"description\": \"min=335.695, mean=335.695, max=335.695, sum=671.389 (2)\", \"tab\": \"General information\", \"score\": \"335.69465648854964\"}", - "Human Sexuality - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"human_sexuality\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_human_sexuality\"" - } - } - }, - { - "evaluation_name": "International Law", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on International Law", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.934, - "details": { - "description": "min=0.934, mean=0.934, max=0.934, sum=1.868 (2)", - "tab": "Accuracy", - "International Law - Observed inference time (s)": "{\"description\": \"min=0.337, mean=0.337, max=0.337, sum=0.674 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.336965306731295\"}", - "International Law - # eval": "{\"description\": \"min=121, mean=121, max=121, sum=242 (2)\", \"tab\": \"General information\", \"score\": \"121.0\"}", - "International Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "International Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "International Law - # prompt tokens": "{\"description\": \"min=639.504, mean=639.504, max=639.504, sum=1279.008 (2)\", \"tab\": \"General information\", \"score\": \"639.5041322314049\"}", - "International Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"international_law\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_international_law\"" - } - } - }, - { - "evaluation_name": "Logical Fallacies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Logical Fallacies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.883, - "details": { - "description": "min=0.883, mean=0.883, max=0.883, sum=1.767 (2)", - "tab": "Accuracy", - "Logical Fallacies - Observed inference time (s)": "{\"description\": \"min=0.321, mean=0.321, max=0.321, sum=0.643 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3214270746781051\"}", - "Logical Fallacies - # eval": "{\"description\": \"min=163, mean=163, max=163, sum=326 (2)\", \"tab\": \"General information\", \"score\": \"163.0\"}", - "Logical Fallacies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Logical Fallacies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Logical Fallacies - # prompt tokens": "{\"description\": \"min=445.84, mean=445.84, max=445.84, sum=891.681 (2)\", \"tab\": \"General information\", \"score\": \"445.840490797546\"}", - "Logical Fallacies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"logical_fallacies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_logical_fallacies\"" - } - } - }, - { - "evaluation_name": "Machine Learning", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Machine Learning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.768, - "details": { - "description": "min=0.768, mean=0.768, max=0.768, sum=1.536 (2)", - "tab": "Accuracy", - "Machine Learning - Observed inference time (s)": "{\"description\": \"min=0.328, mean=0.328, max=0.328, sum=0.657 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3284116280930383\"}", - "Machine Learning - # eval": "{\"description\": \"min=112, mean=112, max=112, sum=224 (2)\", \"tab\": \"General information\", \"score\": \"112.0\"}", - "Machine Learning - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Machine Learning - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Machine Learning - # prompt tokens": "{\"description\": \"min=666.205, mean=666.205, max=666.205, sum=1332.411 (2)\", \"tab\": \"General information\", \"score\": \"666.2053571428571\"}", - "Machine Learning - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"machine_learning\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_machine_learning\"" - } - } - }, - { - "evaluation_name": "Management", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Management", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.942, - "details": { - "description": "min=0.942, mean=0.942, max=0.942, sum=1.883 (2)", - "tab": "Accuracy", - "Management - Observed inference time (s)": "{\"description\": \"min=0.32, mean=0.32, max=0.32, sum=0.64 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.32008614354920617\"}", - "Management - # eval": "{\"description\": \"min=103, mean=103, max=103, sum=206 (2)\", \"tab\": \"General information\", \"score\": \"103.0\"}", - "Management - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Management - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Management - # prompt tokens": "{\"description\": \"min=279.485, mean=279.485, max=279.485, sum=558.971 (2)\", \"tab\": \"General information\", \"score\": \"279.4854368932039\"}", - "Management - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"management\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_management\"" - } - } - }, - { - "evaluation_name": "Marketing", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Marketing", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.936, - "details": { - "description": "min=0.936, mean=0.936, max=0.936, sum=1.872 (2)", - "tab": "Accuracy", - "Marketing - Observed inference time (s)": "{\"description\": \"min=0.337, mean=0.337, max=0.337, sum=0.675 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3374974228378035\"}", - "Marketing - # eval": "{\"description\": \"min=234, mean=234, max=234, sum=468 (2)\", \"tab\": \"General information\", \"score\": \"234.0\"}", - "Marketing - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Marketing - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Marketing - # prompt tokens": "{\"description\": \"min=399.85, mean=399.85, max=399.85, sum=799.701 (2)\", \"tab\": \"General information\", \"score\": \"399.85042735042737\"}", - "Marketing - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"marketing\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_marketing\"" - } - } - }, - { - "evaluation_name": "Medical Genetics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Medical Genetics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.96, - "details": { - "description": "min=0.96, mean=0.96, max=0.96, sum=1.92 (2)", - "tab": "Accuracy", - "Medical Genetics - Observed inference time (s)": "{\"description\": \"min=0.33, mean=0.33, max=0.33, sum=0.66 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.33016372203826905\"}", - "Medical Genetics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Medical Genetics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Medical Genetics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Medical Genetics - # prompt tokens": "{\"description\": \"min=343.23, mean=343.23, max=343.23, sum=686.46 (2)\", \"tab\": \"General information\", \"score\": \"343.23\"}", - "Medical Genetics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"medical_genetics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_medical_genetics\"" - } - } - }, - { - "evaluation_name": "Miscellaneous", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Miscellaneous", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.954, - "details": { - "description": "min=0.954, mean=0.954, max=0.954, sum=1.908 (2)", - "tab": "Accuracy", - "Miscellaneous - Observed inference time (s)": "{\"description\": \"min=0.336, mean=0.336, max=0.336, sum=0.672 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.335910246898997\"}", - "Miscellaneous - # eval": "{\"description\": \"min=783, mean=783, max=783, sum=1566 (2)\", \"tab\": \"General information\", \"score\": \"783.0\"}", - "Miscellaneous - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Miscellaneous - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Miscellaneous - # prompt tokens": "{\"description\": \"min=296.479, mean=296.479, max=296.479, sum=592.958 (2)\", \"tab\": \"General information\", \"score\": \"296.47892720306515\"}", - "Miscellaneous - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"miscellaneous\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_miscellaneous\"" - } - } - }, - { - "evaluation_name": "Moral Scenarios", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Moral Scenarios", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.841, - "details": { - "description": "min=0.841, mean=0.841, max=0.841, sum=1.683 (2)", - "tab": "Accuracy", - "Moral Disputes - Observed inference time (s)": "{\"description\": \"min=0.333, mean=0.333, max=0.333, sum=0.667 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3332573719796418\"}", - "Moral Scenarios - Observed inference time (s)": "{\"description\": \"min=0.344, mean=0.344, max=0.344, sum=0.687 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3436078146183291\"}", - "Moral Disputes - # eval": "{\"description\": \"min=346, mean=346, max=346, sum=692 (2)\", \"tab\": \"General information\", \"score\": \"346.0\"}", - "Moral Disputes - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Disputes - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Disputes - # prompt tokens": "{\"description\": \"min=474.835, mean=474.835, max=474.835, sum=949.671 (2)\", \"tab\": \"General information\", \"score\": \"474.83526011560696\"}", - "Moral Disputes - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Moral Scenarios - # eval": "{\"description\": \"min=895, mean=895, max=895, sum=1790 (2)\", \"tab\": \"General information\", \"score\": \"895.0\"}", - "Moral Scenarios - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Scenarios - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # prompt tokens": "{\"description\": \"min=655.068, mean=655.068, max=655.068, sum=1310.136 (2)\", \"tab\": \"General information\", \"score\": \"655.068156424581\"}", - "Moral Scenarios - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"moral_scenarios\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_moral_scenarios\"" - } - } - }, - { - "evaluation_name": "Nutrition", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Nutrition", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.899, - "details": { - "description": "min=0.899, mean=0.899, max=0.899, sum=1.797 (2)", - "tab": "Accuracy", - "Nutrition - Observed inference time (s)": "{\"description\": \"min=0.368, mean=0.368, max=0.368, sum=0.737 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.36828617722380397\"}", - "Nutrition - # eval": "{\"description\": \"min=306, mean=306, max=306, sum=612 (2)\", \"tab\": \"General information\", \"score\": \"306.0\"}", - "Nutrition - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Nutrition - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Nutrition - # prompt tokens": "{\"description\": \"min=581.997, mean=581.997, max=581.997, sum=1163.993 (2)\", \"tab\": \"General information\", \"score\": \"581.9967320261438\"}", - "Nutrition - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"nutrition\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_nutrition\"" - } - } - }, - { - "evaluation_name": "Prehistory", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Prehistory", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.938, - "details": { - "description": "min=0.938, mean=0.938, max=0.938, sum=1.877 (2)", - "tab": "Accuracy", - "Prehistory - Observed inference time (s)": "{\"description\": \"min=0.318, mean=0.318, max=0.318, sum=0.635 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.31765871430620735\"}", - "Prehistory - # eval": "{\"description\": \"min=324, mean=324, max=324, sum=648 (2)\", \"tab\": \"General information\", \"score\": \"324.0\"}", - "Prehistory - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Prehistory - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Prehistory - # prompt tokens": "{\"description\": \"min=513.944, mean=513.944, max=513.944, sum=1027.889 (2)\", \"tab\": \"General information\", \"score\": \"513.9444444444445\"}", - "Prehistory - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"prehistory\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_prehistory\"" - } - } - }, - { - "evaluation_name": "Public Relations", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Public Relations", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.809, - "details": { - "description": "min=0.809, mean=0.809, max=0.809, sum=1.618 (2)", - "tab": "Accuracy", - "Public Relations - Observed inference time (s)": "{\"description\": \"min=0.35, mean=0.35, max=0.35, sum=0.699 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3496434450149536\"}", - "Public Relations - # eval": "{\"description\": \"min=110, mean=110, max=110, sum=220 (2)\", \"tab\": \"General information\", \"score\": \"110.0\"}", - "Public Relations - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Public Relations - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Public Relations - # prompt tokens": "{\"description\": \"min=402.918, mean=402.918, max=402.918, sum=805.836 (2)\", \"tab\": \"General information\", \"score\": \"402.91818181818184\"}", - "Public Relations - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"public_relations\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_public_relations\"" - } - } - }, - { - "evaluation_name": "Security Studies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Security Studies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.837, - "details": { - "description": "min=0.837, mean=0.837, max=0.837, sum=1.673 (2)", - "tab": "Accuracy", - "Security Studies - Observed inference time (s)": "{\"description\": \"min=0.35, mean=0.35, max=0.35, sum=0.7 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3501845612817881\"}", - "Security Studies - # eval": "{\"description\": \"min=245, mean=245, max=245, sum=490 (2)\", \"tab\": \"General information\", \"score\": \"245.0\"}", - "Security Studies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Security Studies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Security Studies - # prompt tokens": "{\"description\": \"min=1166.686, mean=1166.686, max=1166.686, sum=2333.371 (2)\", \"tab\": \"General information\", \"score\": \"1166.6857142857143\"}", - "Security Studies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"security_studies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_security_studies\"" - } - } - }, - { - "evaluation_name": "Sociology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Sociology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.94, - "details": { - "description": "min=0.94, mean=0.94, max=0.94, sum=1.881 (2)", - "tab": "Accuracy", - "Sociology - Observed inference time (s)": "{\"description\": \"min=0.347, mean=0.347, max=0.347, sum=0.693 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.346723644294549\"}", - "Sociology - # eval": "{\"description\": \"min=201, mean=201, max=201, sum=402 (2)\", \"tab\": \"General information\", \"score\": \"201.0\"}", - "Sociology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Sociology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Sociology - # prompt tokens": "{\"description\": \"min=444.269, mean=444.269, max=444.269, sum=888.537 (2)\", \"tab\": \"General information\", \"score\": \"444.2686567164179\"}", - "Sociology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"sociology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_sociology\"" - } - } - }, - { - "evaluation_name": "Virology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Virology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.596, - "details": { - "description": "min=0.596, mean=0.596, max=0.596, sum=1.193 (2)", - "tab": "Accuracy", - "Virology - Observed inference time (s)": "{\"description\": \"min=0.314, mean=0.314, max=0.314, sum=0.628 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3142197634800371\"}", - "Virology - # eval": "{\"description\": \"min=166, mean=166, max=166, sum=332 (2)\", \"tab\": \"General information\", \"score\": \"166.0\"}", - "Virology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Virology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Virology - # prompt tokens": "{\"description\": \"min=334.434, mean=334.434, max=334.434, sum=668.867 (2)\", \"tab\": \"General information\", \"score\": \"334.43373493975906\"}", - "Virology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"virology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_virology\"" - } - } - }, - { - "evaluation_name": "World Religions", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on World Religions", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.889, - "details": { - "description": "min=0.889, mean=0.889, max=0.889, sum=1.778 (2)", - "tab": "Accuracy", - "World Religions - Observed inference time (s)": "{\"description\": \"min=0.332, mean=0.332, max=0.332, sum=0.664 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3320118307370191\"}", - "World Religions - # eval": "{\"description\": \"min=171, mean=171, max=171, sum=342 (2)\", \"tab\": \"General information\", \"score\": \"171.0\"}", - "World Religions - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "World Religions - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "World Religions - # prompt tokens": "{\"description\": \"min=267.936, mean=267.936, max=267.936, sum=535.871 (2)\", \"tab\": \"General information\", \"score\": \"267.9356725146199\"}", - "World Religions - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"world_religions\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_world_religions\"" - } - } - }, - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.671, - "details": { - "description": "", - "tab": "Efficiency" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_id": "reward-bench/openai_gpt-4o-2024-05-13/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8327 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9665 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7039 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8649 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8487 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7262 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/openai_gpt-4o-2024-08-06.json b/data/models/openai_gpt-4o-2024-08-06.json deleted file mode 100644 index 4523783fb76c33e56985cf8706dfcde93c76d009..0000000000000000000000000000000000000000 --- a/data/models/openai_gpt-4o-2024-08-06.json +++ /dev/null @@ -1,2167 +0,0 @@ -{ - "model_info": { - "name": "GPT-4o 2024-08-06", - "id": "openai/gpt-4o-2024-08-06", - "developer": "OpenAI", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_lite/openai_gpt-4o-2024-08-06/1774096306.427425", - "retrieved_timestamp": "1774096306.427425", - "source_metadata": { - "source_name": "helm_lite", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_lite", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_lite", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.928, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.6728589263420724\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.795, - "details": { - "description": "min=0.795, mean=0.795, max=0.795, sum=0.795 (1)", - "tab": "Accuracy", - "NarrativeQA - Observed inference time (s)": "{\"description\": \"min=0.562, mean=0.562, max=0.562, sum=0.562 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.5615828097706109\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=3451.668, mean=3451.668, max=3451.668, sum=3451.668 (1)\", \"tab\": \"General information\", \"score\": \"3451.667605633803\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=5.076, mean=5.076, max=5.076, sum=5.076 (1)\", \"tab\": \"General information\", \"score\": \"5.076056338028169\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (closed-book)", - "source_data": { - "dataset_name": "NaturalQuestions (closed-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (closed-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.496, - "details": { - "description": "min=0.496, mean=0.496, max=0.496, sum=0.496 (1)", - "tab": "Accuracy", - "NaturalQuestions (open-book) - Observed inference time (s)": "{\"description\": \"min=0.616, mean=0.616, max=0.616, sum=0.616 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.6156781461238862\"}", - "NaturalQuestions (closed-book) - Observed inference time (s)": "{\"description\": \"min=0.418, mean=0.418, max=0.418, sum=0.418 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.4182390425205231\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1714.02, mean=1714.02, max=1714.02, sum=1714.02 (1)\", \"tab\": \"General information\", \"score\": \"1714.02\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=6.504, mean=6.504, max=6.504, sum=6.504 (1)\", \"tab\": \"General information\", \"score\": \"6.504\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=129.953, mean=129.953, max=129.953, sum=129.953 (1)\", \"tab\": \"General information\", \"score\": \"129.953\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=5.032, mean=5.032, max=5.032, sum=5.032 (1)\", \"tab\": \"General information\", \"score\": \"5.032\"}" - } - }, - "generation_config": { - "additional_details": { - "mode": "\"closedbook\"" - } - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.968, - "details": { - "description": "min=0.968, mean=0.968, max=0.968, sum=0.968 (1)", - "tab": "Accuracy", - "OpenbookQA - Observed inference time (s)": "{\"description\": \"min=0.401, mean=0.401, max=0.401, sum=0.401 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.40116420984268186\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=245.486, mean=245.486, max=245.486, sum=245.486 (1)\", \"tab\": \"General information\", \"score\": \"245.486\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "dataset": "\"openbookqa\"", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.738, - "details": { - "description": "min=0.58, mean=0.738, max=0.95, sum=3.691 (5)", - "tab": "Accuracy", - "MMLU - Observed inference time (s)": "{\"description\": \"min=0.335, mean=0.441, max=0.512, sum=2.204 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.4407063991228739\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=373.42, mean=466.992, max=613.228, sum=2334.958 (5)\", \"tab\": \"General information\", \"score\": \"466.9916140350877\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"college_chemistry\", \"computer_security\", \"econometrics\", \"us_foreign_policy\"]", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MATH", - "source_data": { - "dataset_name": "MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Equivalent (CoT) on MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.853, - "details": { - "description": "min=0.731, mean=0.853, max=0.956, sum=5.968 (7)", - "tab": "Accuracy", - "MATH - Observed inference time (s)": "{\"description\": \"min=3.205, mean=4.321, max=6.062, sum=30.245 (7)\", \"tab\": \"Efficiency\", \"score\": \"4.320655013573451\"}", - "MATH - # eval": "{\"description\": \"min=30, mean=62.429, max=135, sum=437 (7)\", \"tab\": \"General information\", \"score\": \"62.42857142857143\"}", - "MATH - # train": "{\"description\": \"min=8, mean=8, max=8, sum=56 (7)\", \"tab\": \"General information\", \"score\": \"8.0\"}", - "MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MATH - # prompt tokens": "{\"description\": \"min=888.43, mean=1273.32, max=2222.25, sum=8913.243 (7)\", \"tab\": \"General information\", \"score\": \"1273.320452019534\"}", - "MATH - # output tokens": "{\"description\": \"min=157.721, mean=210.124, max=243.135, sum=1470.869 (7)\", \"tab\": \"General information\", \"score\": \"210.1241379885811\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"algebra\", \"counting_and_probability\", \"geometry\", \"intermediate_algebra\", \"number_theory\", \"prealgebra\", \"precalculus\"]", - "level": "\"1\"", - "use_official_examples": "\"False\"", - "use_chain_of_thought": "\"True\"" - } - } - }, - { - "evaluation_name": "GSM8K", - "source_data": { - "dataset_name": "GSM8K", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on GSM8K", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.909, - "details": { - "description": "min=0.909, mean=0.909, max=0.909, sum=0.909 (1)", - "tab": "Accuracy", - "GSM8K - Observed inference time (s)": "{\"description\": \"min=2.937, mean=2.937, max=2.937, sum=2.937 (1)\", \"tab\": \"Efficiency\", \"score\": \"2.9373713800907133\"}", - "GSM8K - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "GSM8K - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "GSM8K - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GSM8K - # prompt tokens": "{\"description\": \"min=952.617, mean=952.617, max=952.617, sum=952.617 (1)\", \"tab\": \"General information\", \"score\": \"952.617\"}", - "GSM8K - # output tokens": "{\"description\": \"min=167.729, mean=167.729, max=167.729, sum=167.729 (1)\", \"tab\": \"General information\", \"score\": \"167.729\"}" - } - }, - "generation_config": { - "additional_details": { - "stop": "\"none\"" - } - } - }, - { - "evaluation_name": "LegalBench", - "source_data": { - "dataset_name": "LegalBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on LegalBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.721, - "details": { - "description": "min=0.422, mean=0.721, max=0.979, sum=3.605 (5)", - "tab": "Accuracy", - "LegalBench - Observed inference time (s)": "{\"description\": \"min=0.312, mean=0.38, max=0.526, sum=1.901 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.38022537218958125\"}", - "LegalBench - # eval": "{\"description\": \"min=95, mean=409.4, max=1000, sum=2047 (5)\", \"tab\": \"General information\", \"score\": \"409.4\"}", - "LegalBench - # train": "{\"description\": \"min=4, mean=4.8, max=5, sum=24 (5)\", \"tab\": \"General information\", \"score\": \"4.8\"}", - "LegalBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "LegalBench - # prompt tokens": "{\"description\": \"min=198.179, mean=1502.795, max=6244.98, sum=7513.977 (5)\", \"tab\": \"General information\", \"score\": \"1502.7954037538377\"}", - "LegalBench - # output tokens": "{\"description\": \"min=1, mean=1.298, max=2.021, sum=6.49 (5)\", \"tab\": \"General information\", \"score\": \"1.298021970457479\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"abercrombie\", \"corporate_lobbying\", \"function_of_decision_section\", \"international_citizenship_questions\", \"proa\"]" - } - } - }, - { - "evaluation_name": "MedQA", - "source_data": { - "dataset_name": "MedQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MedQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.863, - "details": { - "description": "min=0.863, mean=0.863, max=0.863, sum=0.863 (1)", - "tab": "Accuracy", - "MedQA - Observed inference time (s)": "{\"description\": \"min=0.307, mean=0.307, max=0.307, sum=0.307 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.30731069923158194\"}", - "MedQA - # eval": "{\"description\": \"min=503, mean=503, max=503, sum=503 (1)\", \"tab\": \"General information\", \"score\": \"503.0\"}", - "MedQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MedQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MedQA - # prompt tokens": "{\"description\": \"min=1009.05, mean=1009.05, max=1009.05, sum=1009.05 (1)\", \"tab\": \"General information\", \"score\": \"1009.0497017892644\"}", - "MedQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WMT 2014", - "source_data": { - "dataset_name": "WMT 2014", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "BLEU-4 on WMT 2014", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.225, - "details": { - "description": "min=0.18, mean=0.225, max=0.267, sum=1.125 (5)", - "tab": "Accuracy", - "WMT 2014 - Observed inference time (s)": "{\"description\": \"min=0.725, mean=0.768, max=0.804, sum=3.841 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.7681678841877538\"}", - "WMT 2014 - # eval": "{\"description\": \"min=503, mean=568.8, max=832, sum=2844 (5)\", \"tab\": \"General information\", \"score\": \"568.8\"}", - "WMT 2014 - # train": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "WMT 2014 - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WMT 2014 - # prompt tokens": "{\"description\": \"min=69.529, mean=105.006, max=128.497, sum=525.028 (5)\", \"tab\": \"General information\", \"score\": \"105.00557042361216\"}", - "WMT 2014 - # output tokens": "{\"description\": \"min=23.809, mean=25.367, max=25.988, sum=126.835 (5)\", \"tab\": \"General information\", \"score\": \"25.366906254779018\"}" - } - }, - "generation_config": { - "additional_details": { - "language_pair": "[\"cs-en\", \"de-en\", \"fr-en\", \"hi-en\", \"ru-en\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_mmlu/openai_gpt-4o-2024-08-06/1774096312.00548", - "retrieved_timestamp": "1774096312.00548", - "source_metadata": { - "source_name": "helm_mmlu", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_mmlu", - "evaluation_results": [ - { - "evaluation_name": "MMLU All Subjects", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU All Subjects", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.843, - "details": { - "description": "min=0.481, mean=0.843, max=0.984, sum=96.141 (114)", - "tab": "Accuracy", - "MMLU All Subjects - Observed inference time (s)": "{\"description\": \"min=0.301, mean=0.459, max=0.88, sum=52.346 (114)\", \"tab\": \"Efficiency\", \"score\": \"0.45917774780314197\"}", - "MMLU All Subjects - # eval": "{\"description\": \"min=100, mean=246.351, max=1534, sum=28084 (114)\", \"tab\": \"General information\", \"score\": \"246.35087719298247\"}", - "MMLU All Subjects - # train": "{\"description\": \"min=5, mean=5, max=5, sum=570 (114)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU All Subjects - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU All Subjects - # prompt tokens": "{\"description\": \"min=267.936, mean=612.332, max=2793.83, sum=69805.818 (114)\", \"tab\": \"General information\", \"score\": \"612.3317391408493\"}", - "MMLU All Subjects - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=114 (114)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_name": "Abstract Algebra", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Abstract Algebra", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.58, - "details": { - "description": "min=0.58, mean=0.58, max=0.58, sum=1.16 (2)", - "tab": "Accuracy", - "Abstract Algebra - Observed inference time (s)": "{\"description\": \"min=0.335, mean=0.335, max=0.335, sum=0.67 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3350093102455139\"}", - "Abstract Algebra - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Abstract Algebra - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Abstract Algebra - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Abstract Algebra - # prompt tokens": "{\"description\": \"min=374.53, mean=374.53, max=374.53, sum=749.06 (2)\", \"tab\": \"General information\", \"score\": \"374.53\"}", - "Abstract Algebra - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"abstract_algebra\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_abstract_algebra\"" - } - } - }, - { - "evaluation_name": "Anatomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Anatomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.911, - "details": { - "description": "min=0.911, mean=0.911, max=0.911, sum=1.822 (2)", - "tab": "Accuracy", - "Anatomy - Observed inference time (s)": "{\"description\": \"min=0.508, mean=0.508, max=0.508, sum=1.015 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5075124228442157\"}", - "Anatomy - # eval": "{\"description\": \"min=135, mean=135, max=135, sum=270 (2)\", \"tab\": \"General information\", \"score\": \"135.0\"}", - "Anatomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Anatomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Anatomy - # prompt tokens": "{\"description\": \"min=350.6, mean=350.6, max=350.6, sum=701.2 (2)\", \"tab\": \"General information\", \"score\": \"350.6\"}", - "Anatomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"anatomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_anatomy\"" - } - } - }, - { - "evaluation_name": "College Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on College Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.686, - "details": { - "description": "min=0.686, mean=0.686, max=0.686, sum=1.373 (2)", - "tab": "Accuracy", - "College Chemistry - Observed inference time (s)": "{\"description\": \"min=0.409, mean=0.409, max=0.409, sum=0.818 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4090025806427002\"}", - "College Biology - Observed inference time (s)": "{\"description\": \"min=0.41, mean=0.41, max=0.41, sum=0.82 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.40991874204741585\"}", - "College Computer Science - Observed inference time (s)": "{\"description\": \"min=0.869, mean=0.869, max=0.869, sum=1.739 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8693285202980041\"}", - "College Mathematics - Observed inference time (s)": "{\"description\": \"min=0.482, mean=0.482, max=0.482, sum=0.964 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4821875333786011\"}", - "College Medicine - Observed inference time (s)": "{\"description\": \"min=0.396, mean=0.396, max=0.396, sum=0.791 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3955839837906678\"}", - "College Physics - Observed inference time (s)": "{\"description\": \"min=0.531, mean=0.531, max=0.531, sum=1.062 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5307925659067491\"}", - "College Chemistry - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Chemistry - # prompt tokens": "{\"description\": \"min=552.07, mean=552.07, max=552.07, sum=1104.14 (2)\", \"tab\": \"General information\", \"score\": \"552.07\"}", - "College Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Biology - # eval": "{\"description\": \"min=144, mean=144, max=144, sum=288 (2)\", \"tab\": \"General information\", \"score\": \"144.0\"}", - "College Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # prompt tokens": "{\"description\": \"min=468.056, mean=468.056, max=468.056, sum=936.111 (2)\", \"tab\": \"General information\", \"score\": \"468.05555555555554\"}", - "College Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # prompt tokens": "{\"description\": \"min=828.39, mean=828.39, max=828.39, sum=1656.78 (2)\", \"tab\": \"General information\", \"score\": \"828.39\"}", - "College Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Mathematics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # prompt tokens": "{\"description\": \"min=594.44, mean=594.44, max=594.44, sum=1188.88 (2)\", \"tab\": \"General information\", \"score\": \"594.44\"}", - "College Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Medicine - # eval": "{\"description\": \"min=173, mean=173, max=173, sum=346 (2)\", \"tab\": \"General information\", \"score\": \"173.0\"}", - "College Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # prompt tokens": "{\"description\": \"min=499.566, mean=499.566, max=499.566, sum=999.133 (2)\", \"tab\": \"General information\", \"score\": \"499.5664739884393\"}", - "College Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Physics - # eval": "{\"description\": \"min=102, mean=102, max=102, sum=204 (2)\", \"tab\": \"General information\", \"score\": \"102.0\"}", - "College Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # prompt tokens": "{\"description\": \"min=502.412, mean=502.412, max=502.412, sum=1004.824 (2)\", \"tab\": \"General information\", \"score\": \"502.4117647058824\"}", - "College Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"college_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_college_physics\"" - } - } - }, - { - "evaluation_name": "Computer Security", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Computer Security", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.85, - "details": { - "description": "min=0.85, mean=0.85, max=0.85, sum=1.7 (2)", - "tab": "Accuracy", - "Computer Security - Observed inference time (s)": "{\"description\": \"min=0.502, mean=0.502, max=0.502, sum=1.004 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5020688962936402\"}", - "Computer Security - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Computer Security - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Computer Security - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Computer Security - # prompt tokens": "{\"description\": \"min=373.42, mean=373.42, max=373.42, sum=746.84 (2)\", \"tab\": \"General information\", \"score\": \"373.42\"}", - "Computer Security - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"computer_security\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_computer_security\"" - } - } - }, - { - "evaluation_name": "Econometrics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Econometrics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.711, - "details": { - "description": "min=0.711, mean=0.711, max=0.711, sum=1.421 (2)", - "tab": "Accuracy", - "Econometrics - Observed inference time (s)": "{\"description\": \"min=0.445, mean=0.445, max=0.445, sum=0.89 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.44516249497731525\"}", - "Econometrics - # eval": "{\"description\": \"min=114, mean=114, max=114, sum=228 (2)\", \"tab\": \"General information\", \"score\": \"114.0\"}", - "Econometrics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Econometrics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Econometrics - # prompt tokens": "{\"description\": \"min=613.228, mean=613.228, max=613.228, sum=1226.456 (2)\", \"tab\": \"General information\", \"score\": \"613.2280701754386\"}", - "Econometrics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"econometrics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_econometrics\"" - } - } - }, - { - "evaluation_name": "Global Facts", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Global Facts", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.69, - "details": { - "description": "min=0.69, mean=0.69, max=0.69, sum=1.38 (2)", - "tab": "Accuracy", - "Global Facts - Observed inference time (s)": "{\"description\": \"min=0.301, mean=0.301, max=0.301, sum=0.602 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3012181663513184\"}", - "Global Facts - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Global Facts - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Global Facts - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Global Facts - # prompt tokens": "{\"description\": \"min=399.69, mean=399.69, max=399.69, sum=799.38 (2)\", \"tab\": \"General information\", \"score\": \"399.69\"}", - "Global Facts - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"global_facts\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_global_facts\"" - } - } - }, - { - "evaluation_name": "Jurisprudence", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Jurisprudence", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.907, - "details": { - "description": "min=0.907, mean=0.907, max=0.907, sum=1.815 (2)", - "tab": "Accuracy", - "Jurisprudence - Observed inference time (s)": "{\"description\": \"min=0.388, mean=0.388, max=0.388, sum=0.776 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3880515495936076\"}", - "Jurisprudence - # eval": "{\"description\": \"min=108, mean=108, max=108, sum=216 (2)\", \"tab\": \"General information\", \"score\": \"108.0\"}", - "Jurisprudence - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Jurisprudence - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Jurisprudence - # prompt tokens": "{\"description\": \"min=391.231, mean=391.231, max=391.231, sum=782.463 (2)\", \"tab\": \"General information\", \"score\": \"391.23148148148147\"}", - "Jurisprudence - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"jurisprudence\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_jurisprudence\"" - } - } - }, - { - "evaluation_name": "Philosophy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Philosophy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.894, - "details": { - "description": "min=0.894, mean=0.894, max=0.894, sum=1.788 (2)", - "tab": "Accuracy", - "Philosophy - Observed inference time (s)": "{\"description\": \"min=0.483, mean=0.483, max=0.483, sum=0.965 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.48272855795464714\"}", - "Philosophy - # eval": "{\"description\": \"min=311, mean=311, max=311, sum=622 (2)\", \"tab\": \"General information\", \"score\": \"311.0\"}", - "Philosophy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Philosophy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Philosophy - # prompt tokens": "{\"description\": \"min=327.92, mean=327.92, max=327.92, sum=655.839 (2)\", \"tab\": \"General information\", \"score\": \"327.91961414790995\"}", - "Philosophy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"philosophy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_philosophy\"" - } - } - }, - { - "evaluation_name": "Professional Psychology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Professional Psychology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.899, - "details": { - "description": "min=0.899, mean=0.899, max=0.899, sum=1.797 (2)", - "tab": "Accuracy", - "Professional Medicine - Observed inference time (s)": "{\"description\": \"min=0.448, mean=0.448, max=0.448, sum=0.897 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4483548367724699\"}", - "Professional Accounting - Observed inference time (s)": "{\"description\": \"min=0.419, mean=0.419, max=0.419, sum=0.839 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4192587585313946\"}", - "Professional Law - Observed inference time (s)": "{\"description\": \"min=0.462, mean=0.462, max=0.462, sum=0.924 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.462134175381418\"}", - "Professional Psychology - Observed inference time (s)": "{\"description\": \"min=0.518, mean=0.518, max=0.518, sum=1.036 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5180651210491953\"}", - "Professional Medicine - # eval": "{\"description\": \"min=272, mean=272, max=272, sum=544 (2)\", \"tab\": \"General information\", \"score\": \"272.0\"}", - "Professional Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Medicine - # prompt tokens": "{\"description\": \"min=1071.18, mean=1071.18, max=1071.18, sum=2142.36 (2)\", \"tab\": \"General information\", \"score\": \"1071.1801470588234\"}", - "Professional Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Accounting - # eval": "{\"description\": \"min=282, mean=282, max=282, sum=564 (2)\", \"tab\": \"General information\", \"score\": \"282.0\"}", - "Professional Accounting - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Accounting - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # prompt tokens": "{\"description\": \"min=657.206, mean=657.206, max=657.206, sum=1314.411 (2)\", \"tab\": \"General information\", \"score\": \"657.2056737588653\"}", - "Professional Accounting - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Law - # eval": "{\"description\": \"min=1534, mean=1534, max=1534, sum=3068 (2)\", \"tab\": \"General information\", \"score\": \"1534.0\"}", - "Professional Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # prompt tokens": "{\"description\": \"min=1629.344, mean=1629.344, max=1629.344, sum=3258.687 (2)\", \"tab\": \"General information\", \"score\": \"1629.3435462842242\"}", - "Professional Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Psychology - # eval": "{\"description\": \"min=612, mean=612, max=612, sum=1224 (2)\", \"tab\": \"General information\", \"score\": \"612.0\"}", - "Professional Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # prompt tokens": "{\"description\": \"min=574.518, mean=574.518, max=574.518, sum=1149.036 (2)\", \"tab\": \"General information\", \"score\": \"574.5179738562091\"}", - "Professional Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"professional_psychology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_professional_psychology\"" - } - } - }, - { - "evaluation_name": "Us Foreign Policy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Us Foreign Policy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.95, - "details": { - "description": "min=0.95, mean=0.95, max=0.95, sum=1.9 (2)", - "tab": "Accuracy", - "Us Foreign Policy - Observed inference time (s)": "{\"description\": \"min=0.512, mean=0.512, max=0.512, sum=1.025 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5122887134552002\"}", - "Us Foreign Policy - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Us Foreign Policy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Us Foreign Policy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Us Foreign Policy - # prompt tokens": "{\"description\": \"min=421.71, mean=421.71, max=421.71, sum=843.42 (2)\", \"tab\": \"General information\", \"score\": \"421.71\"}", - "Us Foreign Policy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"us_foreign_policy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_us_foreign_policy\"" - } - } - }, - { - "evaluation_name": "Astronomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Astronomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.947, - "details": { - "description": "min=0.947, mean=0.947, max=0.947, sum=1.895 (2)", - "tab": "Accuracy", - "Astronomy - Observed inference time (s)": "{\"description\": \"min=0.435, mean=0.435, max=0.435, sum=0.869 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4347311226945174\"}", - "Astronomy - # eval": "{\"description\": \"min=152, mean=152, max=152, sum=304 (2)\", \"tab\": \"General information\", \"score\": \"152.0\"}", - "Astronomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Astronomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Astronomy - # prompt tokens": "{\"description\": \"min=577.349, mean=577.349, max=577.349, sum=1154.697 (2)\", \"tab\": \"General information\", \"score\": \"577.3486842105264\"}", - "Astronomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"astronomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_astronomy\"" - } - } - }, - { - "evaluation_name": "Business Ethics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Business Ethics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.89, - "details": { - "description": "min=0.89, mean=0.89, max=0.89, sum=1.78 (2)", - "tab": "Accuracy", - "Business Ethics - Observed inference time (s)": "{\"description\": \"min=0.52, mean=0.52, max=0.52, sum=1.04 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5199928903579711\"}", - "Business Ethics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Business Ethics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Business Ethics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Business Ethics - # prompt tokens": "{\"description\": \"min=565.7, mean=565.7, max=565.7, sum=1131.4 (2)\", \"tab\": \"General information\", \"score\": \"565.7\"}", - "Business Ethics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"business_ethics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_business_ethics\"" - } - } - }, - { - "evaluation_name": "Clinical Knowledge", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Clinical Knowledge", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.894, - "details": { - "description": "min=0.894, mean=0.894, max=0.894, sum=1.789 (2)", - "tab": "Accuracy", - "Clinical Knowledge - Observed inference time (s)": "{\"description\": \"min=0.307, mean=0.307, max=0.307, sum=0.613 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3066561905842907\"}", - "Clinical Knowledge - # eval": "{\"description\": \"min=265, mean=265, max=265, sum=530 (2)\", \"tab\": \"General information\", \"score\": \"265.0\"}", - "Clinical Knowledge - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Clinical Knowledge - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Clinical Knowledge - # prompt tokens": "{\"description\": \"min=400.985, mean=400.985, max=400.985, sum=801.97 (2)\", \"tab\": \"General information\", \"score\": \"400.98490566037736\"}", - "Clinical Knowledge - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"clinical_knowledge\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_clinical_knowledge\"" - } - } - }, - { - "evaluation_name": "Conceptual Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Conceptual Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.923, - "details": { - "description": "min=0.923, mean=0.923, max=0.923, sum=1.847 (2)", - "tab": "Accuracy", - "Conceptual Physics - Observed inference time (s)": "{\"description\": \"min=0.381, mean=0.381, max=0.381, sum=0.763 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3812521427235705\"}", - "Conceptual Physics - # eval": "{\"description\": \"min=235, mean=235, max=235, sum=470 (2)\", \"tab\": \"General information\", \"score\": \"235.0\"}", - "Conceptual Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Conceptual Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Conceptual Physics - # prompt tokens": "{\"description\": \"min=304.677, mean=304.677, max=304.677, sum=609.353 (2)\", \"tab\": \"General information\", \"score\": \"304.67659574468087\"}", - "Conceptual Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"conceptual_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_conceptual_physics\"" - } - } - }, - { - "evaluation_name": "Electrical Engineering", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Electrical Engineering", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.793, - "details": { - "description": "min=0.793, mean=0.793, max=0.793, sum=1.586 (2)", - "tab": "Accuracy", - "Electrical Engineering - Observed inference time (s)": "{\"description\": \"min=0.437, mean=0.437, max=0.437, sum=0.874 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4368692447399271\"}", - "Electrical Engineering - # eval": "{\"description\": \"min=145, mean=145, max=145, sum=290 (2)\", \"tab\": \"General information\", \"score\": \"145.0\"}", - "Electrical Engineering - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Electrical Engineering - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Electrical Engineering - # prompt tokens": "{\"description\": \"min=439.228, mean=439.228, max=439.228, sum=878.455 (2)\", \"tab\": \"General information\", \"score\": \"439.22758620689655\"}", - "Electrical Engineering - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"electrical_engineering\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_electrical_engineering\"" - } - } - }, - { - "evaluation_name": "Elementary Mathematics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Elementary Mathematics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.775, - "details": { - "description": "min=0.775, mean=0.775, max=0.775, sum=1.55 (2)", - "tab": "Accuracy", - "Elementary Mathematics - Observed inference time (s)": "{\"description\": \"min=0.374, mean=0.374, max=0.374, sum=0.747 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.37356801449306426\"}", - "Elementary Mathematics - # eval": "{\"description\": \"min=378, mean=378, max=378, sum=756 (2)\", \"tab\": \"General information\", \"score\": \"378.0\"}", - "Elementary Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Elementary Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Elementary Mathematics - # prompt tokens": "{\"description\": \"min=532.683, mean=532.683, max=532.683, sum=1065.365 (2)\", \"tab\": \"General information\", \"score\": \"532.6825396825396\"}", - "Elementary Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"elementary_mathematics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_elementary_mathematics\"" - } - } - }, - { - "evaluation_name": "Formal Logic", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Formal Logic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.675, - "details": { - "description": "min=0.675, mean=0.675, max=0.675, sum=1.349 (2)", - "tab": "Accuracy", - "Formal Logic - Observed inference time (s)": "{\"description\": \"min=0.341, mean=0.341, max=0.341, sum=0.683 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3414205180274116\"}", - "Formal Logic - # eval": "{\"description\": \"min=126, mean=126, max=126, sum=252 (2)\", \"tab\": \"General information\", \"score\": \"126.0\"}", - "Formal Logic - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Formal Logic - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Formal Logic - # prompt tokens": "{\"description\": \"min=604.492, mean=604.492, max=604.492, sum=1208.984 (2)\", \"tab\": \"General information\", \"score\": \"604.4920634920635\"}", - "Formal Logic - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"formal_logic\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_formal_logic\"" - } - } - }, - { - "evaluation_name": "High School World History", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on High School World History", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.941, - "details": { - "description": "min=0.941, mean=0.941, max=0.941, sum=1.882 (2)", - "tab": "Accuracy", - "High School Biology - Observed inference time (s)": "{\"description\": \"min=0.511, mean=0.511, max=0.511, sum=1.021 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5105965960410334\"}", - "High School Chemistry - Observed inference time (s)": "{\"description\": \"min=0.338, mean=0.338, max=0.338, sum=0.676 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3379564614131533\"}", - "High School Computer Science - Observed inference time (s)": "{\"description\": \"min=0.397, mean=0.397, max=0.397, sum=0.794 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3969814705848694\"}", - "High School European History - Observed inference time (s)": "{\"description\": \"min=0.594, mean=0.594, max=0.594, sum=1.189 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5944608587207216\"}", - "High School Geography - Observed inference time (s)": "{\"description\": \"min=0.353, mean=0.353, max=0.353, sum=0.706 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3532402262543187\"}", - "High School Government And Politics - Observed inference time (s)": "{\"description\": \"min=0.88, mean=0.88, max=0.88, sum=1.76 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8798744147305662\"}", - "High School Macroeconomics - Observed inference time (s)": "{\"description\": \"min=0.501, mean=0.501, max=0.501, sum=1.003 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.501340057911017\"}", - "High School Mathematics - Observed inference time (s)": "{\"description\": \"min=0.472, mean=0.472, max=0.472, sum=0.944 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4721549925980745\"}", - "High School Microeconomics - Observed inference time (s)": "{\"description\": \"min=0.406, mean=0.406, max=0.406, sum=0.812 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4058714473948759\"}", - "High School Physics - Observed inference time (s)": "{\"description\": \"min=0.484, mean=0.484, max=0.484, sum=0.968 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.48384577075377205\"}", - "High School Psychology - Observed inference time (s)": "{\"description\": \"min=0.532, mean=0.532, max=0.532, sum=1.063 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5316181160988064\"}", - "High School Statistics - Observed inference time (s)": "{\"description\": \"min=0.518, mean=0.518, max=0.518, sum=1.036 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5179998201352579\"}", - "High School US History - Observed inference time (s)": "{\"description\": \"min=0.573, mean=0.573, max=0.573, sum=1.147 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5734734535217285\"}", - "High School World History - Observed inference time (s)": "{\"description\": \"min=0.461, mean=0.461, max=0.461, sum=0.923 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4614185592796229\"}", - "High School Biology - # eval": "{\"description\": \"min=310, mean=310, max=310, sum=620 (2)\", \"tab\": \"General information\", \"score\": \"310.0\"}", - "High School Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Biology - # prompt tokens": "{\"description\": \"min=504.874, mean=504.874, max=504.874, sum=1009.748 (2)\", \"tab\": \"General information\", \"score\": \"504.8741935483871\"}", - "High School Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Chemistry - # eval": "{\"description\": \"min=203, mean=203, max=203, sum=406 (2)\", \"tab\": \"General information\", \"score\": \"203.0\"}", - "High School Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # prompt tokens": "{\"description\": \"min=495.34, mean=495.34, max=495.34, sum=990.68 (2)\", \"tab\": \"General information\", \"score\": \"495.3399014778325\"}", - "High School Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "High School Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # prompt tokens": "{\"description\": \"min=865.8, mean=865.8, max=865.8, sum=1731.6 (2)\", \"tab\": \"General information\", \"score\": \"865.8\"}", - "High School Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School European History - # eval": "{\"description\": \"min=165, mean=165, max=165, sum=330 (2)\", \"tab\": \"General information\", \"score\": \"165.0\"}", - "High School European History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School European History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # prompt tokens": "{\"description\": \"min=2793.83, mean=2793.83, max=2793.83, sum=5587.661 (2)\", \"tab\": \"General information\", \"score\": \"2793.830303030303\"}", - "High School European History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Geography - # eval": "{\"description\": \"min=198, mean=198, max=198, sum=396 (2)\", \"tab\": \"General information\", \"score\": \"198.0\"}", - "High School Geography - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Geography - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # prompt tokens": "{\"description\": \"min=372.783, mean=372.783, max=372.783, sum=745.566 (2)\", \"tab\": \"General information\", \"score\": \"372.7828282828283\"}", - "High School Geography - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Government And Politics - # eval": "{\"description\": \"min=193, mean=193, max=193, sum=386 (2)\", \"tab\": \"General information\", \"score\": \"193.0\"}", - "High School Government And Politics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Government And Politics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # prompt tokens": "{\"description\": \"min=463.01, mean=463.01, max=463.01, sum=926.021 (2)\", \"tab\": \"General information\", \"score\": \"463.0103626943005\"}", - "High School Government And Politics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Macroeconomics - # eval": "{\"description\": \"min=390, mean=390, max=390, sum=780 (2)\", \"tab\": \"General information\", \"score\": \"390.0\"}", - "High School Macroeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Macroeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # prompt tokens": "{\"description\": \"min=371.451, mean=371.451, max=371.451, sum=742.903 (2)\", \"tab\": \"General information\", \"score\": \"371.4512820512821\"}", - "High School Macroeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Mathematics - # eval": "{\"description\": \"min=270, mean=270, max=270, sum=540 (2)\", \"tab\": \"General information\", \"score\": \"270.0\"}", - "High School Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # prompt tokens": "{\"description\": \"min=532.456, mean=532.456, max=532.456, sum=1064.911 (2)\", \"tab\": \"General information\", \"score\": \"532.4555555555555\"}", - "High School Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Microeconomics - # eval": "{\"description\": \"min=238, mean=238, max=238, sum=476 (2)\", \"tab\": \"General information\", \"score\": \"238.0\"}", - "High School Microeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Microeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # prompt tokens": "{\"description\": \"min=398.739, mean=398.739, max=398.739, sum=797.479 (2)\", \"tab\": \"General information\", \"score\": \"398.73949579831935\"}", - "High School Microeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Physics - # eval": "{\"description\": \"min=151, mean=151, max=151, sum=302 (2)\", \"tab\": \"General information\", \"score\": \"151.0\"}", - "High School Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # prompt tokens": "{\"description\": \"min=560.238, mean=560.238, max=560.238, sum=1120.477 (2)\", \"tab\": \"General information\", \"score\": \"560.2384105960265\"}", - "High School Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Psychology - # eval": "{\"description\": \"min=545, mean=545, max=545, sum=1090 (2)\", \"tab\": \"General information\", \"score\": \"545.0\"}", - "High School Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # prompt tokens": "{\"description\": \"min=492.917, mean=492.917, max=492.917, sum=985.835 (2)\", \"tab\": \"General information\", \"score\": \"492.91743119266056\"}", - "High School Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Statistics - # eval": "{\"description\": \"min=216, mean=216, max=216, sum=432 (2)\", \"tab\": \"General information\", \"score\": \"216.0\"}", - "High School Statistics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Statistics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # prompt tokens": "{\"description\": \"min=787.574, mean=787.574, max=787.574, sum=1575.148 (2)\", \"tab\": \"General information\", \"score\": \"787.574074074074\"}", - "High School Statistics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School US History - # eval": "{\"description\": \"min=204, mean=204, max=204, sum=408 (2)\", \"tab\": \"General information\", \"score\": \"204.0\"}", - "High School US History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School US History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # prompt tokens": "{\"description\": \"min=2220.005, mean=2220.005, max=2220.005, sum=4440.01 (2)\", \"tab\": \"General information\", \"score\": \"2220.0049019607845\"}", - "High School US History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School World History - # eval": "{\"description\": \"min=237, mean=237, max=237, sum=474 (2)\", \"tab\": \"General information\", \"score\": \"237.0\"}", - "High School World History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School World History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # prompt tokens": "{\"description\": \"min=1424.439, mean=1424.439, max=1424.439, sum=2848.878 (2)\", \"tab\": \"General information\", \"score\": \"1424.4388185654009\"}", - "High School World History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"high_school_world_history\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_high_school_world_history\"" - } - } - }, - { - "evaluation_name": "Human Sexuality", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Human Sexuality", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.901, - "details": { - "description": "min=0.901, mean=0.901, max=0.901, sum=1.802 (2)", - "tab": "Accuracy", - "Human Aging - Observed inference time (s)": "{\"description\": \"min=0.403, mean=0.403, max=0.403, sum=0.807 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4033327327180871\"}", - "Human Sexuality - Observed inference time (s)": "{\"description\": \"min=0.397, mean=0.397, max=0.397, sum=0.794 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3971163625935562\"}", - "Human Aging - # eval": "{\"description\": \"min=223, mean=223, max=223, sum=446 (2)\", \"tab\": \"General information\", \"score\": \"223.0\"}", - "Human Aging - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Aging - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Aging - # prompt tokens": "{\"description\": \"min=316.453, mean=316.453, max=316.453, sum=632.906 (2)\", \"tab\": \"General information\", \"score\": \"316.4529147982063\"}", - "Human Aging - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Human Sexuality - # eval": "{\"description\": \"min=131, mean=131, max=131, sum=262 (2)\", \"tab\": \"General information\", \"score\": \"131.0\"}", - "Human Sexuality - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Sexuality - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # prompt tokens": "{\"description\": \"min=335.695, mean=335.695, max=335.695, sum=671.389 (2)\", \"tab\": \"General information\", \"score\": \"335.69465648854964\"}", - "Human Sexuality - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"human_sexuality\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_human_sexuality\"" - } - } - }, - { - "evaluation_name": "International Law", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on International Law", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.942, - "details": { - "description": "min=0.942, mean=0.942, max=0.942, sum=1.884 (2)", - "tab": "Accuracy", - "International Law - Observed inference time (s)": "{\"description\": \"min=0.437, mean=0.437, max=0.437, sum=0.875 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4373398063596615\"}", - "International Law - # eval": "{\"description\": \"min=121, mean=121, max=121, sum=242 (2)\", \"tab\": \"General information\", \"score\": \"121.0\"}", - "International Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "International Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "International Law - # prompt tokens": "{\"description\": \"min=639.504, mean=639.504, max=639.504, sum=1279.008 (2)\", \"tab\": \"General information\", \"score\": \"639.5041322314049\"}", - "International Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"international_law\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_international_law\"" - } - } - }, - { - "evaluation_name": "Logical Fallacies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Logical Fallacies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.902, - "details": { - "description": "min=0.902, mean=0.902, max=0.902, sum=1.804 (2)", - "tab": "Accuracy", - "Logical Fallacies - Observed inference time (s)": "{\"description\": \"min=0.445, mean=0.445, max=0.445, sum=0.89 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.44485992888000114\"}", - "Logical Fallacies - # eval": "{\"description\": \"min=163, mean=163, max=163, sum=326 (2)\", \"tab\": \"General information\", \"score\": \"163.0\"}", - "Logical Fallacies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Logical Fallacies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Logical Fallacies - # prompt tokens": "{\"description\": \"min=445.84, mean=445.84, max=445.84, sum=891.681 (2)\", \"tab\": \"General information\", \"score\": \"445.840490797546\"}", - "Logical Fallacies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"logical_fallacies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_logical_fallacies\"" - } - } - }, - { - "evaluation_name": "Machine Learning", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Machine Learning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.777, - "details": { - "description": "min=0.777, mean=0.777, max=0.777, sum=1.554 (2)", - "tab": "Accuracy", - "Machine Learning - Observed inference time (s)": "{\"description\": \"min=0.414, mean=0.414, max=0.414, sum=0.829 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.41432228897299084\"}", - "Machine Learning - # eval": "{\"description\": \"min=112, mean=112, max=112, sum=224 (2)\", \"tab\": \"General information\", \"score\": \"112.0\"}", - "Machine Learning - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Machine Learning - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Machine Learning - # prompt tokens": "{\"description\": \"min=666.205, mean=666.205, max=666.205, sum=1332.411 (2)\", \"tab\": \"General information\", \"score\": \"666.2053571428571\"}", - "Machine Learning - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"machine_learning\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_machine_learning\"" - } - } - }, - { - "evaluation_name": "Management", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Management", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.913, - "details": { - "description": "min=0.913, mean=0.913, max=0.913, sum=1.825 (2)", - "tab": "Accuracy", - "Management - Observed inference time (s)": "{\"description\": \"min=0.46, mean=0.46, max=0.46, sum=0.92 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4598746878429524\"}", - "Management - # eval": "{\"description\": \"min=103, mean=103, max=103, sum=206 (2)\", \"tab\": \"General information\", \"score\": \"103.0\"}", - "Management - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Management - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Management - # prompt tokens": "{\"description\": \"min=279.485, mean=279.485, max=279.485, sum=558.971 (2)\", \"tab\": \"General information\", \"score\": \"279.4854368932039\"}", - "Management - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"management\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_management\"" - } - } - }, - { - "evaluation_name": "Marketing", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Marketing", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.94, - "details": { - "description": "min=0.94, mean=0.94, max=0.94, sum=1.88 (2)", - "tab": "Accuracy", - "Marketing - Observed inference time (s)": "{\"description\": \"min=0.481, mean=0.481, max=0.481, sum=0.962 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4812224573559231\"}", - "Marketing - # eval": "{\"description\": \"min=234, mean=234, max=234, sum=468 (2)\", \"tab\": \"General information\", \"score\": \"234.0\"}", - "Marketing - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Marketing - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Marketing - # prompt tokens": "{\"description\": \"min=399.85, mean=399.85, max=399.85, sum=799.701 (2)\", \"tab\": \"General information\", \"score\": \"399.85042735042737\"}", - "Marketing - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"marketing\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_marketing\"" - } - } - }, - { - "evaluation_name": "Medical Genetics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Medical Genetics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.98, - "details": { - "description": "min=0.98, mean=0.98, max=0.98, sum=1.96 (2)", - "tab": "Accuracy", - "Medical Genetics - Observed inference time (s)": "{\"description\": \"min=0.425, mean=0.425, max=0.425, sum=0.85 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.42490904808044433\"}", - "Medical Genetics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Medical Genetics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Medical Genetics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Medical Genetics - # prompt tokens": "{\"description\": \"min=343.23, mean=343.23, max=343.23, sum=686.46 (2)\", \"tab\": \"General information\", \"score\": \"343.23\"}", - "Medical Genetics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"medical_genetics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_medical_genetics\"" - } - } - }, - { - "evaluation_name": "Miscellaneous", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Miscellaneous", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.958, - "details": { - "description": "min=0.958, mean=0.958, max=0.958, sum=1.916 (2)", - "tab": "Accuracy", - "Miscellaneous - Observed inference time (s)": "{\"description\": \"min=0.457, mean=0.457, max=0.457, sum=0.915 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.457414278734385\"}", - "Miscellaneous - # eval": "{\"description\": \"min=783, mean=783, max=783, sum=1566 (2)\", \"tab\": \"General information\", \"score\": \"783.0\"}", - "Miscellaneous - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Miscellaneous - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Miscellaneous - # prompt tokens": "{\"description\": \"min=296.479, mean=296.479, max=296.479, sum=592.958 (2)\", \"tab\": \"General information\", \"score\": \"296.47892720306515\"}", - "Miscellaneous - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"miscellaneous\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_miscellaneous\"" - } - } - }, - { - "evaluation_name": "Moral Scenarios", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Moral Scenarios", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.802, - "details": { - "description": "min=0.802, mean=0.802, max=0.802, sum=1.604 (2)", - "tab": "Accuracy", - "Moral Disputes - Observed inference time (s)": "{\"description\": \"min=0.364, mean=0.364, max=0.364, sum=0.727 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3637407087866282\"}", - "Moral Scenarios - Observed inference time (s)": "{\"description\": \"min=0.462, mean=0.462, max=0.462, sum=0.924 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.46217673823820143\"}", - "Moral Disputes - # eval": "{\"description\": \"min=346, mean=346, max=346, sum=692 (2)\", \"tab\": \"General information\", \"score\": \"346.0\"}", - "Moral Disputes - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Disputes - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Disputes - # prompt tokens": "{\"description\": \"min=474.835, mean=474.835, max=474.835, sum=949.671 (2)\", \"tab\": \"General information\", \"score\": \"474.83526011560696\"}", - "Moral Disputes - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Moral Scenarios - # eval": "{\"description\": \"min=895, mean=895, max=895, sum=1790 (2)\", \"tab\": \"General information\", \"score\": \"895.0\"}", - "Moral Scenarios - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Scenarios - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # prompt tokens": "{\"description\": \"min=655.068, mean=655.068, max=655.068, sum=1310.136 (2)\", \"tab\": \"General information\", \"score\": \"655.068156424581\"}", - "Moral Scenarios - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"moral_scenarios\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_moral_scenarios\"" - } - } - }, - { - "evaluation_name": "Nutrition", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Nutrition", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.905, - "details": { - "description": "min=0.905, mean=0.905, max=0.905, sum=1.81 (2)", - "tab": "Accuracy", - "Nutrition - Observed inference time (s)": "{\"description\": \"min=0.423, mean=0.423, max=0.423, sum=0.847 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.42327408541261763\"}", - "Nutrition - # eval": "{\"description\": \"min=306, mean=306, max=306, sum=612 (2)\", \"tab\": \"General information\", \"score\": \"306.0\"}", - "Nutrition - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Nutrition - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Nutrition - # prompt tokens": "{\"description\": \"min=581.997, mean=581.997, max=581.997, sum=1163.993 (2)\", \"tab\": \"General information\", \"score\": \"581.9967320261438\"}", - "Nutrition - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"nutrition\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_nutrition\"" - } - } - }, - { - "evaluation_name": "Prehistory", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Prehistory", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.935, - "details": { - "description": "min=0.935, mean=0.935, max=0.935, sum=1.87 (2)", - "tab": "Accuracy", - "Prehistory - Observed inference time (s)": "{\"description\": \"min=0.486, mean=0.486, max=0.486, sum=0.972 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.48604018452726766\"}", - "Prehistory - # eval": "{\"description\": \"min=324, mean=324, max=324, sum=648 (2)\", \"tab\": \"General information\", \"score\": \"324.0\"}", - "Prehistory - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Prehistory - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Prehistory - # prompt tokens": "{\"description\": \"min=513.944, mean=513.944, max=513.944, sum=1027.889 (2)\", \"tab\": \"General information\", \"score\": \"513.9444444444445\"}", - "Prehistory - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"prehistory\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_prehistory\"" - } - } - }, - { - "evaluation_name": "Public Relations", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Public Relations", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.782, - "details": { - "description": "min=0.782, mean=0.782, max=0.782, sum=1.564 (2)", - "tab": "Accuracy", - "Public Relations - Observed inference time (s)": "{\"description\": \"min=0.472, mean=0.472, max=0.472, sum=0.944 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.47211467786268757\"}", - "Public Relations - # eval": "{\"description\": \"min=110, mean=110, max=110, sum=220 (2)\", \"tab\": \"General information\", \"score\": \"110.0\"}", - "Public Relations - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Public Relations - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Public Relations - # prompt tokens": "{\"description\": \"min=402.918, mean=402.918, max=402.918, sum=805.836 (2)\", \"tab\": \"General information\", \"score\": \"402.91818181818184\"}", - "Public Relations - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"public_relations\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_public_relations\"" - } - } - }, - { - "evaluation_name": "Security Studies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Security Studies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.833, - "details": { - "description": "min=0.833, mean=0.833, max=0.833, sum=1.665 (2)", - "tab": "Accuracy", - "Security Studies - Observed inference time (s)": "{\"description\": \"min=0.452, mean=0.452, max=0.452, sum=0.905 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.45247335336646255\"}", - "Security Studies - # eval": "{\"description\": \"min=245, mean=245, max=245, sum=490 (2)\", \"tab\": \"General information\", \"score\": \"245.0\"}", - "Security Studies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Security Studies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Security Studies - # prompt tokens": "{\"description\": \"min=1166.686, mean=1166.686, max=1166.686, sum=2333.371 (2)\", \"tab\": \"General information\", \"score\": \"1166.6857142857143\"}", - "Security Studies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"security_studies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_security_studies\"" - } - } - }, - { - "evaluation_name": "Sociology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Sociology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.945, - "details": { - "description": "min=0.945, mean=0.945, max=0.945, sum=1.891 (2)", - "tab": "Accuracy", - "Sociology - Observed inference time (s)": "{\"description\": \"min=0.479, mean=0.479, max=0.479, sum=0.958 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4788183940583794\"}", - "Sociology - # eval": "{\"description\": \"min=201, mean=201, max=201, sum=402 (2)\", \"tab\": \"General information\", \"score\": \"201.0\"}", - "Sociology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Sociology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Sociology - # prompt tokens": "{\"description\": \"min=444.269, mean=444.269, max=444.269, sum=888.537 (2)\", \"tab\": \"General information\", \"score\": \"444.2686567164179\"}", - "Sociology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"sociology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_sociology\"" - } - } - }, - { - "evaluation_name": "Virology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Virology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.578, - "details": { - "description": "min=0.578, mean=0.578, max=0.578, sum=1.157 (2)", - "tab": "Accuracy", - "Virology - Observed inference time (s)": "{\"description\": \"min=0.473, mean=0.473, max=0.473, sum=0.945 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.47254319794206734\"}", - "Virology - # eval": "{\"description\": \"min=166, mean=166, max=166, sum=332 (2)\", \"tab\": \"General information\", \"score\": \"166.0\"}", - "Virology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Virology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Virology - # prompt tokens": "{\"description\": \"min=334.434, mean=334.434, max=334.434, sum=668.867 (2)\", \"tab\": \"General information\", \"score\": \"334.43373493975906\"}", - "Virology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"virology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_virology\"" - } - } - }, - { - "evaluation_name": "World Religions", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on World Religions", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.883, - "details": { - "description": "min=0.883, mean=0.883, max=0.883, sum=1.766 (2)", - "tab": "Accuracy", - "World Religions - Observed inference time (s)": "{\"description\": \"min=0.408, mean=0.408, max=0.408, sum=0.815 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4075693944741411\"}", - "World Religions - # eval": "{\"description\": \"min=171, mean=171, max=171, sum=342 (2)\", \"tab\": \"General information\", \"score\": \"171.0\"}", - "World Religions - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "World Religions - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "World Religions - # prompt tokens": "{\"description\": \"min=267.936, mean=267.936, max=267.936, sum=535.871 (2)\", \"tab\": \"General information\", \"score\": \"267.9356725146199\"}", - "World Religions - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"world_religions\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_world_religions\"" - } - } - }, - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.52, - "details": { - "description": "", - "tab": "Efficiency" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_id": "reward-bench/openai_gpt-4o-2024-08-06/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8673 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9609 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.761 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8811 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8661 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "reward-bench-2/openai_gpt-4o-2024-08-06/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6493 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5684 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3312 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.623 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8619 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7293 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7819 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/openai_gpt-4o-2024-11-20.json b/data/models/openai_gpt-4o-2024-11-20.json deleted file mode 100644 index 2b1e4171c8570a6df96e82387cfc175654f965ae..0000000000000000000000000000000000000000 --- a/data/models/openai_gpt-4o-2024-11-20.json +++ /dev/null @@ -1,313 +0,0 @@ -{ - "model_info": { - "name": "GPT-4o 2024-11-20", - "id": "openai/gpt-4o-2024-11-20", - "developer": "OpenAI", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_capabilities/openai_gpt-4o-2024-11-20/1774096304.5056698", - "retrieved_timestamp": "1774096304.5056698", - "source_metadata": { - "source_name": "helm_capabilities", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_capabilities", - "evaluation_results": [ - { - "evaluation_name": "Mean score", - "source_data": { - "dataset_name": "helm_capabilities", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "The mean of the scores from all columns.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.634, - "details": { - "description": "", - "tab": "Accuracy", - "Mean score - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"13.268214070783824\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU-Pro", - "source_data": { - "dataset_name": "MMLU-Pro", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on MMLU-Pro", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.713, - "details": { - "description": "min=0.713, mean=0.713, max=0.713, sum=0.713 (1)", - "tab": "Accuracy", - "MMLU-Pro - Observed inference time (s)": "{\"description\": \"min=11.334, mean=11.334, max=11.334, sum=11.334 (1)\", \"tab\": \"Efficiency\", \"score\": \"11.333669463157653\"}", - "MMLU-Pro - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "MMLU-Pro - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - # prompt tokens": "{\"description\": \"min=228.569, mean=228.569, max=228.569, sum=228.569 (1)\", \"tab\": \"General information\", \"score\": \"228.569\"}", - "MMLU-Pro - # output tokens": "{\"description\": \"min=503.126, mean=503.126, max=503.126, sum=503.126 (1)\", \"tab\": \"General information\", \"score\": \"503.126\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"all\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.52, - "details": { - "description": "min=0.52, mean=0.52, max=0.52, sum=0.52 (1)", - "tab": "Accuracy", - "GPQA - Observed inference time (s)": "{\"description\": \"min=13.65, mean=13.65, max=13.65, sum=13.65 (1)\", \"tab\": \"Efficiency\", \"score\": \"13.64998589877056\"}", - "GPQA - # eval": "{\"description\": \"min=446, mean=446, max=446, sum=446 (1)\", \"tab\": \"General information\", \"score\": \"446.0\"}", - "GPQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - # prompt tokens": "{\"description\": \"min=248.152, mean=248.152, max=248.152, sum=248.152 (1)\", \"tab\": \"General information\", \"score\": \"248.152466367713\"}", - "GPQA - # output tokens": "{\"description\": \"min=597.291, mean=597.291, max=597.291, sum=597.291 (1)\", \"tab\": \"General information\", \"score\": \"597.2914798206278\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"gpqa_main\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "IFEval Strict Acc on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.817, - "details": { - "description": "min=0.817, mean=0.817, max=0.817, sum=0.817 (1)", - "tab": "Accuracy", - "IFEval - Observed inference time (s)": "{\"description\": \"min=8.686, mean=8.686, max=8.686, sum=8.686 (1)\", \"tab\": \"Efficiency\", \"score\": \"8.68623784685752\"}", - "IFEval - # eval": "{\"description\": \"min=541, mean=541, max=541, sum=541 (1)\", \"tab\": \"General information\", \"score\": \"541.0\"}", - "IFEval - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - # prompt tokens": "{\"description\": \"min=45.671, mean=45.671, max=45.671, sum=45.671 (1)\", \"tab\": \"General information\", \"score\": \"45.67097966728281\"}", - "IFEval - # output tokens": "{\"description\": \"min=345.405, mean=345.405, max=345.405, sum=345.405 (1)\", \"tab\": \"General information\", \"score\": \"345.40480591497226\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WildBench", - "source_data": { - "dataset_name": "WildBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "WB Score on WildBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.828, - "details": { - "description": "min=0.828, mean=0.828, max=0.828, sum=0.828 (1)", - "tab": "Accuracy", - "WildBench - Observed inference time (s)": "{\"description\": \"min=15.765, mean=15.765, max=15.765, sum=15.765 (1)\", \"tab\": \"Efficiency\", \"score\": \"15.764520774255166\"}", - "WildBench - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "WildBench - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # prompt tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # output tokens": "{\"description\": \"min=1044.923, mean=1044.923, max=1044.923, sum=1044.923 (1)\", \"tab\": \"General information\", \"score\": \"1044.923\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"v2\"" - } - } - }, - { - "evaluation_name": "Omni-MATH", - "source_data": { - "dataset_name": "Omni-MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Acc on Omni-MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.293, - "details": { - "description": "min=0.293, mean=0.293, max=0.293, sum=0.293 (1)", - "tab": "Accuracy", - "Omni-MATH - Observed inference time (s)": "{\"description\": \"min=16.907, mean=16.907, max=16.907, sum=16.907 (1)\", \"tab\": \"Efficiency\", \"score\": \"16.90665637087822\"}", - "Omni-MATH - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "Omni-MATH - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - # prompt tokens": "{\"description\": \"min=109.623, mean=109.623, max=109.623, sum=109.623 (1)\", \"tab\": \"General information\", \"score\": \"109.623\"}", - "Omni-MATH - # output tokens": "{\"description\": \"min=908.643, mean=908.643, max=908.643, sum=908.643 (1)\", \"tab\": \"General information\", \"score\": \"908.643\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "livecodebenchpro/openai/gpt-4o-2024-11-20/1760492095.8105888", - "retrieved_timestamp": "1760492095.8105888", - "source_metadata": { - "source_organization_name": "New York University, Princeton University, University of California San Diego, University of Washington and Canyon Crest Academy", - "evaluator_relationship": "third_party", - "source_name": "Live Code Bench Pro", - "source_type": "documentation" - }, - "eval_library": { - "name": "unknown", - "version": "unknown" - }, - "benchmark": "livecodebenchpro", - "evaluation_results": [ - { - "evaluation_name": "Hard Problems", - "metric_config": { - "evaluation_description": "Pass@1 on Hard Problems", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.0 - }, - "source_data": { - "dataset_name": "Hard Problems", - "source_type": "url", - "url": [ - "https://webhook.cp-bench.orzzh.com/leaderboard/llm/difficulty?difficulty=hard&benchmark_mode=live" - ] - } - }, - { - "evaluation_name": "Medium Problems", - "metric_config": { - "evaluation_description": "Pass@1 on Medium Problems", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.0 - }, - "source_data": { - "dataset_name": "Medium Problems", - "source_type": "url", - "url": [ - "https://webhook.cp-bench.orzzh.com/leaderboard/llm/difficulty?difficulty=medium&benchmark_mode=live" - ] - } - }, - { - "evaluation_name": "Easy Problems", - "metric_config": { - "evaluation_description": "Pass@1 on Easy Problems", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.07042253521126761 - }, - "source_data": { - "dataset_name": "Easy Problems", - "source_type": "url", - "url": [ - "https://webhook.cp-bench.orzzh.com/leaderboard/llm/difficulty?difficulty=easy&benchmark_mode=live" - ] - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/openai_gpt-4o-mini-2024-07-18.json b/data/models/openai_gpt-4o-mini-2024-07-18.json deleted file mode 100644 index 1b3fb4c30102ee1f603e3640bb4c4b14c38b8cac..0000000000000000000000000000000000000000 --- a/data/models/openai_gpt-4o-mini-2024-07-18.json +++ /dev/null @@ -1,2391 +0,0 @@ -{ - "model_info": { - "name": "GPT-4o mini 2024-07-18", - "id": "openai/gpt-4o-mini-2024-07-18", - "developer": "OpenAI", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_capabilities/openai_gpt-4o-mini-2024-07-18/1774096304.5056698", - "retrieved_timestamp": "1774096304.5056698", - "source_metadata": { - "source_name": "helm_capabilities", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_capabilities", - "evaluation_results": [ - { - "evaluation_name": "Mean score", - "source_data": { - "dataset_name": "helm_capabilities", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "The mean of the scores from all columns.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.565, - "details": { - "description": "", - "tab": "Accuracy", - "Mean score - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"10.41176955262334\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU-Pro", - "source_data": { - "dataset_name": "MMLU-Pro", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on MMLU-Pro", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.603, - "details": { - "description": "min=0.603, mean=0.603, max=0.603, sum=0.603 (1)", - "tab": "Accuracy", - "MMLU-Pro - Observed inference time (s)": "{\"description\": \"min=6.572, mean=6.572, max=6.572, sum=6.572 (1)\", \"tab\": \"Efficiency\", \"score\": \"6.57206253027916\"}", - "MMLU-Pro - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "MMLU-Pro - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - # prompt tokens": "{\"description\": \"min=228.569, mean=228.569, max=228.569, sum=228.569 (1)\", \"tab\": \"General information\", \"score\": \"228.569\"}", - "MMLU-Pro - # output tokens": "{\"description\": \"min=334.86, mean=334.86, max=334.86, sum=334.86 (1)\", \"tab\": \"General information\", \"score\": \"334.86\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"all\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.368, - "details": { - "description": "min=0.368, mean=0.368, max=0.368, sum=0.368 (1)", - "tab": "Accuracy", - "GPQA - Observed inference time (s)": "{\"description\": \"min=8.814, mean=8.814, max=8.814, sum=8.814 (1)\", \"tab\": \"Efficiency\", \"score\": \"8.813848996910814\"}", - "GPQA - # eval": "{\"description\": \"min=446, mean=446, max=446, sum=446 (1)\", \"tab\": \"General information\", \"score\": \"446.0\"}", - "GPQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - # prompt tokens": "{\"description\": \"min=248.152, mean=248.152, max=248.152, sum=248.152 (1)\", \"tab\": \"General information\", \"score\": \"248.152466367713\"}", - "GPQA - # output tokens": "{\"description\": \"min=489.226, mean=489.226, max=489.226, sum=489.226 (1)\", \"tab\": \"General information\", \"score\": \"489.22645739910314\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"gpqa_main\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "IFEval Strict Acc on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.782, - "details": { - "description": "min=0.782, mean=0.782, max=0.782, sum=0.782 (1)", - "tab": "Accuracy", - "IFEval - Observed inference time (s)": "{\"description\": \"min=5.963, mean=5.963, max=5.963, sum=5.963 (1)\", \"tab\": \"Efficiency\", \"score\": \"5.963314282916169\"}", - "IFEval - # eval": "{\"description\": \"min=541, mean=541, max=541, sum=541 (1)\", \"tab\": \"General information\", \"score\": \"541.0\"}", - "IFEval - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - # prompt tokens": "{\"description\": \"min=45.671, mean=45.671, max=45.671, sum=45.671 (1)\", \"tab\": \"General information\", \"score\": \"45.67097966728281\"}", - "IFEval - # output tokens": "{\"description\": \"min=314.919, mean=314.919, max=314.919, sum=314.919 (1)\", \"tab\": \"General information\", \"score\": \"314.91866913123846\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WildBench", - "source_data": { - "dataset_name": "WildBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "WB Score on WildBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.791, - "details": { - "description": "min=0.791, mean=0.791, max=0.791, sum=0.791 (1)", - "tab": "Accuracy", - "WildBench - Observed inference time (s)": "{\"description\": \"min=13.996, mean=13.996, max=13.996, sum=13.996 (1)\", \"tab\": \"Efficiency\", \"score\": \"13.996195561885834\"}", - "WildBench - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "WildBench - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # prompt tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # output tokens": "{\"description\": \"min=809.307, mean=809.307, max=809.307, sum=809.307 (1)\", \"tab\": \"General information\", \"score\": \"809.307\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"v2\"" - } - } - }, - { - "evaluation_name": "Omni-MATH", - "source_data": { - "dataset_name": "Omni-MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Acc on Omni-MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.28, - "details": { - "description": "min=0.28, mean=0.28, max=0.28, sum=0.28 (1)", - "tab": "Accuracy", - "Omni-MATH - Observed inference time (s)": "{\"description\": \"min=16.713, mean=16.713, max=16.713, sum=16.713 (1)\", \"tab\": \"Efficiency\", \"score\": \"16.713426391124724\"}", - "Omni-MATH - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "Omni-MATH - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - # prompt tokens": "{\"description\": \"min=109.623, mean=109.623, max=109.623, sum=109.623 (1)\", \"tab\": \"General information\", \"score\": \"109.623\"}", - "Omni-MATH - # output tokens": "{\"description\": \"min=863.417, mean=863.417, max=863.417, sum=863.417 (1)\", \"tab\": \"General information\", \"score\": \"863.417\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_lite/openai_gpt-4o-mini-2024-07-18/1774096306.427425", - "retrieved_timestamp": "1774096306.427425", - "source_metadata": { - "source_name": "helm_lite", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_lite", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_lite", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.701, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.7796004993757802\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.768, - "details": { - "description": "min=0.768, mean=0.768, max=0.768, sum=0.768 (1)", - "tab": "Accuracy", - "NarrativeQA - Observed inference time (s)": "{\"description\": \"min=0.473, mean=0.473, max=0.473, sum=0.473 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.47311924612018424\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=3451.668, mean=3451.668, max=3451.668, sum=3451.668 (1)\", \"tab\": \"General information\", \"score\": \"3451.667605633803\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=4.482, mean=4.482, max=4.482, sum=4.482 (1)\", \"tab\": \"General information\", \"score\": \"4.48169014084507\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (closed-book)", - "source_data": { - "dataset_name": "NaturalQuestions (closed-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (closed-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.386, - "details": { - "description": "min=0.386, mean=0.386, max=0.386, sum=0.386 (1)", - "tab": "Accuracy", - "NaturalQuestions (open-book) - Observed inference time (s)": "{\"description\": \"min=0.406, mean=0.406, max=0.406, sum=0.406 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.40617332768440245\"}", - "NaturalQuestions (closed-book) - Observed inference time (s)": "{\"description\": \"min=0.374, mean=0.374, max=0.374, sum=0.374 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.3740478873252869\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1714.02, mean=1714.02, max=1714.02, sum=1714.02 (1)\", \"tab\": \"General information\", \"score\": \"1714.02\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=5.175, mean=5.175, max=5.175, sum=5.175 (1)\", \"tab\": \"General information\", \"score\": \"5.175\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=129.953, mean=129.953, max=129.953, sum=129.953 (1)\", \"tab\": \"General information\", \"score\": \"129.953\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=4.847, mean=4.847, max=4.847, sum=4.847 (1)\", \"tab\": \"General information\", \"score\": \"4.847\"}" - } - }, - "generation_config": { - "additional_details": { - "mode": "\"closedbook\"" - } - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.92, - "details": { - "description": "min=0.92, mean=0.92, max=0.92, sum=0.92 (1)", - "tab": "Accuracy", - "OpenbookQA - Observed inference time (s)": "{\"description\": \"min=0.331, mean=0.331, max=0.331, sum=0.331 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.3309546322822571\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=245.486, mean=245.486, max=245.486, sum=245.486 (1)\", \"tab\": \"General information\", \"score\": \"245.486\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "dataset": "\"openbookqa\"", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.668, - "details": { - "description": "min=0.42, mean=0.668, max=0.91, sum=3.339 (5)", - "tab": "Accuracy", - "MMLU - Observed inference time (s)": "{\"description\": \"min=0.292, mean=0.299, max=0.309, sum=1.497 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.2993013315033494\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=373.42, mean=466.992, max=613.228, sum=2334.958 (5)\", \"tab\": \"General information\", \"score\": \"466.9916140350877\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"college_chemistry\", \"computer_security\", \"econometrics\", \"us_foreign_policy\"]", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MATH", - "source_data": { - "dataset_name": "MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Equivalent (CoT) on MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.802, - "details": { - "description": "min=0.605, mean=0.802, max=0.97, sum=5.611 (7)", - "tab": "Accuracy", - "MATH - Observed inference time (s)": "{\"description\": \"min=2.312, mean=3.175, max=3.696, sum=22.228 (7)\", \"tab\": \"Efficiency\", \"score\": \"3.175392215033706\"}", - "MATH - # eval": "{\"description\": \"min=30, mean=62.429, max=135, sum=437 (7)\", \"tab\": \"General information\", \"score\": \"62.42857142857143\"}", - "MATH - # train": "{\"description\": \"min=8, mean=8, max=8, sum=56 (7)\", \"tab\": \"General information\", \"score\": \"8.0\"}", - "MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MATH - # prompt tokens": "{\"description\": \"min=888.43, mean=1273.32, max=2222.25, sum=8913.243 (7)\", \"tab\": \"General information\", \"score\": \"1273.320452019534\"}", - "MATH - # output tokens": "{\"description\": \"min=167.884, mean=238.235, max=276.058, sum=1667.647 (7)\", \"tab\": \"General information\", \"score\": \"238.23525019565412\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"algebra\", \"counting_and_probability\", \"geometry\", \"intermediate_algebra\", \"number_theory\", \"prealgebra\", \"precalculus\"]", - "level": "\"1\"", - "use_official_examples": "\"False\"", - "use_chain_of_thought": "\"True\"" - } - } - }, - { - "evaluation_name": "GSM8K", - "source_data": { - "dataset_name": "GSM8K", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on GSM8K", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.843, - "details": { - "description": "min=0.843, mean=0.843, max=0.843, sum=0.843 (1)", - "tab": "Accuracy", - "GSM8K - Observed inference time (s)": "{\"description\": \"min=2.519, mean=2.519, max=2.519, sum=2.519 (1)\", \"tab\": \"Efficiency\", \"score\": \"2.5191967821121217\"}", - "GSM8K - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "GSM8K - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "GSM8K - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GSM8K - # prompt tokens": "{\"description\": \"min=952.617, mean=952.617, max=952.617, sum=952.617 (1)\", \"tab\": \"General information\", \"score\": \"952.617\"}", - "GSM8K - # output tokens": "{\"description\": \"min=215.465, mean=215.465, max=215.465, sum=215.465 (1)\", \"tab\": \"General information\", \"score\": \"215.465\"}" - } - }, - "generation_config": { - "additional_details": { - "stop": "\"none\"" - } - } - }, - { - "evaluation_name": "LegalBench", - "source_data": { - "dataset_name": "LegalBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on LegalBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.653, - "details": { - "description": "min=0.414, mean=0.653, max=0.937, sum=3.263 (5)", - "tab": "Accuracy", - "LegalBench - Observed inference time (s)": "{\"description\": \"min=0.337, mean=0.382, max=0.503, sum=1.91 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.38199841220513264\"}", - "LegalBench - # eval": "{\"description\": \"min=95, mean=409.4, max=1000, sum=2047 (5)\", \"tab\": \"General information\", \"score\": \"409.4\"}", - "LegalBench - # train": "{\"description\": \"min=4, mean=4.8, max=5, sum=24 (5)\", \"tab\": \"General information\", \"score\": \"4.8\"}", - "LegalBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "LegalBench - # prompt tokens": "{\"description\": \"min=198.179, mean=1502.795, max=6244.98, sum=7513.977 (5)\", \"tab\": \"General information\", \"score\": \"1502.7954037538377\"}", - "LegalBench - # output tokens": "{\"description\": \"min=1, mean=1.293, max=2.253, sum=6.465 (5)\", \"tab\": \"General information\", \"score\": \"1.2930331277785745\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"abercrombie\", \"corporate_lobbying\", \"function_of_decision_section\", \"international_citizenship_questions\", \"proa\"]" - } - } - }, - { - "evaluation_name": "MedQA", - "source_data": { - "dataset_name": "MedQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MedQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.748, - "details": { - "description": "min=0.748, mean=0.748, max=0.748, sum=0.748 (1)", - "tab": "Accuracy", - "MedQA - Observed inference time (s)": "{\"description\": \"min=0.332, mean=0.332, max=0.332, sum=0.332 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.3318999989132284\"}", - "MedQA - # eval": "{\"description\": \"min=503, mean=503, max=503, sum=503 (1)\", \"tab\": \"General information\", \"score\": \"503.0\"}", - "MedQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MedQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MedQA - # prompt tokens": "{\"description\": \"min=1009.05, mean=1009.05, max=1009.05, sum=1009.05 (1)\", \"tab\": \"General information\", \"score\": \"1009.0497017892644\"}", - "MedQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WMT 2014", - "source_data": { - "dataset_name": "WMT 2014", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "BLEU-4 on WMT 2014", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.206, - "details": { - "description": "min=0.153, mean=0.206, max=0.254, sum=1.032 (5)", - "tab": "Accuracy", - "WMT 2014 - Observed inference time (s)": "{\"description\": \"min=0.557, mean=0.583, max=0.598, sum=2.917 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.5833699647787834\"}", - "WMT 2014 - # eval": "{\"description\": \"min=503, mean=568.8, max=832, sum=2844 (5)\", \"tab\": \"General information\", \"score\": \"568.8\"}", - "WMT 2014 - # train": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "WMT 2014 - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WMT 2014 - # prompt tokens": "{\"description\": \"min=69.529, mean=105.006, max=128.497, sum=525.028 (5)\", \"tab\": \"General information\", \"score\": \"105.00557042361216\"}", - "WMT 2014 - # output tokens": "{\"description\": \"min=23.748, mean=25.504, max=26.235, sum=127.522 (5)\", \"tab\": \"General information\", \"score\": \"25.504310196513227\"}" - } - }, - "generation_config": { - "additional_details": { - "language_pair": "[\"cs-en\", \"de-en\", \"fr-en\", \"hi-en\", \"ru-en\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_mmlu/openai_gpt-4o-mini-2024-07-18/1774096312.00548", - "retrieved_timestamp": "1774096312.00548", - "source_metadata": { - "source_name": "helm_mmlu", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_mmlu", - "evaluation_results": [ - { - "evaluation_name": "MMLU All Subjects", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU All Subjects", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.767, - "details": { - "description": "min=0.419, mean=0.767, max=0.959, sum=87.464 (114)", - "tab": "Accuracy", - "MMLU All Subjects - Observed inference time (s)": "{\"description\": \"min=0.26, mean=0.334, max=0.733, sum=38.043 (114)\", \"tab\": \"Efficiency\", \"score\": \"0.3337143530055209\"}", - "MMLU All Subjects - # eval": "{\"description\": \"min=100, mean=246.351, max=1534, sum=28084 (114)\", \"tab\": \"General information\", \"score\": \"246.35087719298247\"}", - "MMLU All Subjects - # train": "{\"description\": \"min=5, mean=5, max=5, sum=570 (114)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU All Subjects - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU All Subjects - # prompt tokens": "{\"description\": \"min=267.936, mean=612.332, max=2793.83, sum=69805.818 (114)\", \"tab\": \"General information\", \"score\": \"612.3317391408493\"}", - "MMLU All Subjects - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=114 (114)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_name": "Abstract Algebra", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Abstract Algebra", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.42, - "details": { - "description": "min=0.42, mean=0.42, max=0.42, sum=0.84 (2)", - "tab": "Accuracy", - "Abstract Algebra - Observed inference time (s)": "{\"description\": \"min=0.292, mean=0.292, max=0.292, sum=0.584 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.29186195611953736\"}", - "Abstract Algebra - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Abstract Algebra - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Abstract Algebra - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Abstract Algebra - # prompt tokens": "{\"description\": \"min=374.53, mean=374.53, max=374.53, sum=749.06 (2)\", \"tab\": \"General information\", \"score\": \"374.53\"}", - "Abstract Algebra - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"abstract_algebra\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_abstract_algebra\"" - } - } - }, - { - "evaluation_name": "Anatomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Anatomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.77, - "details": { - "description": "min=0.77, mean=0.77, max=0.77, sum=1.541 (2)", - "tab": "Accuracy", - "Anatomy - Observed inference time (s)": "{\"description\": \"min=0.282, mean=0.282, max=0.282, sum=0.564 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.282137664159139\"}", - "Anatomy - # eval": "{\"description\": \"min=135, mean=135, max=135, sum=270 (2)\", \"tab\": \"General information\", \"score\": \"135.0\"}", - "Anatomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Anatomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Anatomy - # prompt tokens": "{\"description\": \"min=350.6, mean=350.6, max=350.6, sum=701.2 (2)\", \"tab\": \"General information\", \"score\": \"350.6\"}", - "Anatomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"anatomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_anatomy\"" - } - } - }, - { - "evaluation_name": "College Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on College Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.559, - "details": { - "description": "min=0.559, mean=0.559, max=0.559, sum=1.118 (2)", - "tab": "Accuracy", - "College Chemistry - Observed inference time (s)": "{\"description\": \"min=0.309, mean=0.309, max=0.309, sum=0.618 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.30902551651000976\"}", - "College Biology - Observed inference time (s)": "{\"description\": \"min=0.315, mean=0.315, max=0.315, sum=0.63 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.31521839068995583\"}", - "College Computer Science - Observed inference time (s)": "{\"description\": \"min=0.321, mean=0.321, max=0.321, sum=0.641 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3206118988990784\"}", - "College Mathematics - Observed inference time (s)": "{\"description\": \"min=0.31, mean=0.31, max=0.31, sum=0.621 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.31047542572021486\"}", - "College Medicine - Observed inference time (s)": "{\"description\": \"min=0.313, mean=0.313, max=0.313, sum=0.625 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.31259707081524624\"}", - "College Physics - Observed inference time (s)": "{\"description\": \"min=0.287, mean=0.287, max=0.287, sum=0.573 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2866650983399036\"}", - "College Chemistry - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Chemistry - # prompt tokens": "{\"description\": \"min=552.07, mean=552.07, max=552.07, sum=1104.14 (2)\", \"tab\": \"General information\", \"score\": \"552.07\"}", - "College Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Biology - # eval": "{\"description\": \"min=144, mean=144, max=144, sum=288 (2)\", \"tab\": \"General information\", \"score\": \"144.0\"}", - "College Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # prompt tokens": "{\"description\": \"min=468.056, mean=468.056, max=468.056, sum=936.111 (2)\", \"tab\": \"General information\", \"score\": \"468.05555555555554\"}", - "College Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # prompt tokens": "{\"description\": \"min=828.39, mean=828.39, max=828.39, sum=1656.78 (2)\", \"tab\": \"General information\", \"score\": \"828.39\"}", - "College Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Mathematics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # prompt tokens": "{\"description\": \"min=594.44, mean=594.44, max=594.44, sum=1188.88 (2)\", \"tab\": \"General information\", \"score\": \"594.44\"}", - "College Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Medicine - # eval": "{\"description\": \"min=173, mean=173, max=173, sum=346 (2)\", \"tab\": \"General information\", \"score\": \"173.0\"}", - "College Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # prompt tokens": "{\"description\": \"min=499.566, mean=499.566, max=499.566, sum=999.133 (2)\", \"tab\": \"General information\", \"score\": \"499.5664739884393\"}", - "College Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Physics - # eval": "{\"description\": \"min=102, mean=102, max=102, sum=204 (2)\", \"tab\": \"General information\", \"score\": \"102.0\"}", - "College Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # prompt tokens": "{\"description\": \"min=502.412, mean=502.412, max=502.412, sum=1004.824 (2)\", \"tab\": \"General information\", \"score\": \"502.4117647058824\"}", - "College Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"college_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_college_physics\"" - } - } - }, - { - "evaluation_name": "Computer Security", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Computer Security", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.85, - "details": { - "description": "min=0.85, mean=0.85, max=0.85, sum=1.7 (2)", - "tab": "Accuracy", - "Computer Security - Observed inference time (s)": "{\"description\": \"min=0.297, mean=0.297, max=0.297, sum=0.594 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.29681269884109496\"}", - "Computer Security - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Computer Security - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Computer Security - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Computer Security - # prompt tokens": "{\"description\": \"min=373.42, mean=373.42, max=373.42, sum=746.84 (2)\", \"tab\": \"General information\", \"score\": \"373.42\"}", - "Computer Security - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"computer_security\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_computer_security\"" - } - } - }, - { - "evaluation_name": "Econometrics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Econometrics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.649, - "details": { - "description": "min=0.649, mean=0.649, max=0.649, sum=1.298 (2)", - "tab": "Accuracy", - "Econometrics - Observed inference time (s)": "{\"description\": \"min=0.299, mean=0.299, max=0.299, sum=0.599 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.29936775199153964\"}", - "Econometrics - # eval": "{\"description\": \"min=114, mean=114, max=114, sum=228 (2)\", \"tab\": \"General information\", \"score\": \"114.0\"}", - "Econometrics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Econometrics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Econometrics - # prompt tokens": "{\"description\": \"min=613.228, mean=613.228, max=613.228, sum=1226.456 (2)\", \"tab\": \"General information\", \"score\": \"613.2280701754386\"}", - "Econometrics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"econometrics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_econometrics\"" - } - } - }, - { - "evaluation_name": "Global Facts", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Global Facts", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.45, - "details": { - "description": "min=0.45, mean=0.45, max=0.45, sum=0.9 (2)", - "tab": "Accuracy", - "Global Facts - Observed inference time (s)": "{\"description\": \"min=0.27, mean=0.27, max=0.27, sum=0.539 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.269585702419281\"}", - "Global Facts - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Global Facts - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Global Facts - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Global Facts - # prompt tokens": "{\"description\": \"min=399.69, mean=399.69, max=399.69, sum=799.38 (2)\", \"tab\": \"General information\", \"score\": \"399.69\"}", - "Global Facts - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"global_facts\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_global_facts\"" - } - } - }, - { - "evaluation_name": "Jurisprudence", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Jurisprudence", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.87, - "details": { - "description": "min=0.87, mean=0.87, max=0.87, sum=1.741 (2)", - "tab": "Accuracy", - "Jurisprudence - Observed inference time (s)": "{\"description\": \"min=0.305, mean=0.305, max=0.305, sum=0.61 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3047747744454278\"}", - "Jurisprudence - # eval": "{\"description\": \"min=108, mean=108, max=108, sum=216 (2)\", \"tab\": \"General information\", \"score\": \"108.0\"}", - "Jurisprudence - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Jurisprudence - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Jurisprudence - # prompt tokens": "{\"description\": \"min=391.231, mean=391.231, max=391.231, sum=782.463 (2)\", \"tab\": \"General information\", \"score\": \"391.23148148148147\"}", - "Jurisprudence - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"jurisprudence\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_jurisprudence\"" - } - } - }, - { - "evaluation_name": "Philosophy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Philosophy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.772, - "details": { - "description": "min=0.772, mean=0.772, max=0.772, sum=1.543 (2)", - "tab": "Accuracy", - "Philosophy - Observed inference time (s)": "{\"description\": \"min=0.289, mean=0.289, max=0.289, sum=0.578 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.28879288308490125\"}", - "Philosophy - # eval": "{\"description\": \"min=311, mean=311, max=311, sum=622 (2)\", \"tab\": \"General information\", \"score\": \"311.0\"}", - "Philosophy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Philosophy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Philosophy - # prompt tokens": "{\"description\": \"min=327.92, mean=327.92, max=327.92, sum=655.839 (2)\", \"tab\": \"General information\", \"score\": \"327.91961414790995\"}", - "Philosophy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"philosophy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_philosophy\"" - } - } - }, - { - "evaluation_name": "Professional Psychology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Professional Psychology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.833, - "details": { - "description": "min=0.833, mean=0.833, max=0.833, sum=1.667 (2)", - "tab": "Accuracy", - "Professional Medicine - Observed inference time (s)": "{\"description\": \"min=0.306, mean=0.306, max=0.306, sum=0.612 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.30609772924114675\"}", - "Professional Accounting - Observed inference time (s)": "{\"description\": \"min=0.312, mean=0.312, max=0.312, sum=0.624 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.31189272336080565\"}", - "Professional Law - Observed inference time (s)": "{\"description\": \"min=0.327, mean=0.327, max=0.327, sum=0.654 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.32692549234885127\"}", - "Professional Psychology - Observed inference time (s)": "{\"description\": \"min=0.429, mean=0.429, max=0.429, sum=0.858 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.42903122792836107\"}", - "Professional Medicine - # eval": "{\"description\": \"min=272, mean=272, max=272, sum=544 (2)\", \"tab\": \"General information\", \"score\": \"272.0\"}", - "Professional Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Medicine - # prompt tokens": "{\"description\": \"min=1071.18, mean=1071.18, max=1071.18, sum=2142.36 (2)\", \"tab\": \"General information\", \"score\": \"1071.1801470588234\"}", - "Professional Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Accounting - # eval": "{\"description\": \"min=282, mean=282, max=282, sum=564 (2)\", \"tab\": \"General information\", \"score\": \"282.0\"}", - "Professional Accounting - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Accounting - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # prompt tokens": "{\"description\": \"min=657.206, mean=657.206, max=657.206, sum=1314.411 (2)\", \"tab\": \"General information\", \"score\": \"657.2056737588653\"}", - "Professional Accounting - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Law - # eval": "{\"description\": \"min=1534, mean=1534, max=1534, sum=3068 (2)\", \"tab\": \"General information\", \"score\": \"1534.0\"}", - "Professional Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # prompt tokens": "{\"description\": \"min=1629.344, mean=1629.344, max=1629.344, sum=3258.687 (2)\", \"tab\": \"General information\", \"score\": \"1629.3435462842242\"}", - "Professional Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Psychology - # eval": "{\"description\": \"min=612, mean=612, max=612, sum=1224 (2)\", \"tab\": \"General information\", \"score\": \"612.0\"}", - "Professional Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # prompt tokens": "{\"description\": \"min=574.518, mean=574.518, max=574.518, sum=1149.036 (2)\", \"tab\": \"General information\", \"score\": \"574.5179738562091\"}", - "Professional Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"professional_psychology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_professional_psychology\"" - } - } - }, - { - "evaluation_name": "Us Foreign Policy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Us Foreign Policy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.91, - "details": { - "description": "min=0.91, mean=0.91, max=0.91, sum=1.82 (2)", - "tab": "Accuracy", - "Us Foreign Policy - Observed inference time (s)": "{\"description\": \"min=0.299, mean=0.299, max=0.299, sum=0.599 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.29943873405456545\"}", - "Us Foreign Policy - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Us Foreign Policy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Us Foreign Policy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Us Foreign Policy - # prompt tokens": "{\"description\": \"min=421.71, mean=421.71, max=421.71, sum=843.42 (2)\", \"tab\": \"General information\", \"score\": \"421.71\"}", - "Us Foreign Policy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"us_foreign_policy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_us_foreign_policy\"" - } - } - }, - { - "evaluation_name": "Astronomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Astronomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.849, - "details": { - "description": "min=0.849, mean=0.849, max=0.849, sum=1.697 (2)", - "tab": "Accuracy", - "Astronomy - Observed inference time (s)": "{\"description\": \"min=0.306, mean=0.306, max=0.306, sum=0.612 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.30577954336216573\"}", - "Astronomy - # eval": "{\"description\": \"min=152, mean=152, max=152, sum=304 (2)\", \"tab\": \"General information\", \"score\": \"152.0\"}", - "Astronomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Astronomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Astronomy - # prompt tokens": "{\"description\": \"min=577.349, mean=577.349, max=577.349, sum=1154.697 (2)\", \"tab\": \"General information\", \"score\": \"577.3486842105264\"}", - "Astronomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"astronomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_astronomy\"" - } - } - }, - { - "evaluation_name": "Business Ethics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Business Ethics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.79, - "details": { - "description": "min=0.79, mean=0.79, max=0.79, sum=1.58 (2)", - "tab": "Accuracy", - "Business Ethics - Observed inference time (s)": "{\"description\": \"min=0.301, mean=0.301, max=0.301, sum=0.602 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3009026026725769\"}", - "Business Ethics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Business Ethics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Business Ethics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Business Ethics - # prompt tokens": "{\"description\": \"min=565.7, mean=565.7, max=565.7, sum=1131.4 (2)\", \"tab\": \"General information\", \"score\": \"565.7\"}", - "Business Ethics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"business_ethics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_business_ethics\"" - } - } - }, - { - "evaluation_name": "Clinical Knowledge", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Clinical Knowledge", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.845, - "details": { - "description": "min=0.845, mean=0.845, max=0.845, sum=1.691 (2)", - "tab": "Accuracy", - "Clinical Knowledge - Observed inference time (s)": "{\"description\": \"min=0.292, mean=0.292, max=0.292, sum=0.585 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.29226316685946485\"}", - "Clinical Knowledge - # eval": "{\"description\": \"min=265, mean=265, max=265, sum=530 (2)\", \"tab\": \"General information\", \"score\": \"265.0\"}", - "Clinical Knowledge - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Clinical Knowledge - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Clinical Knowledge - # prompt tokens": "{\"description\": \"min=400.985, mean=400.985, max=400.985, sum=801.97 (2)\", \"tab\": \"General information\", \"score\": \"400.98490566037736\"}", - "Clinical Knowledge - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"clinical_knowledge\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_clinical_knowledge\"" - } - } - }, - { - "evaluation_name": "Conceptual Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Conceptual Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.791, - "details": { - "description": "min=0.791, mean=0.791, max=0.791, sum=1.583 (2)", - "tab": "Accuracy", - "Conceptual Physics - Observed inference time (s)": "{\"description\": \"min=0.26, mean=0.26, max=0.26, sum=0.52 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.26024563261803163\"}", - "Conceptual Physics - # eval": "{\"description\": \"min=235, mean=235, max=235, sum=470 (2)\", \"tab\": \"General information\", \"score\": \"235.0\"}", - "Conceptual Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Conceptual Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Conceptual Physics - # prompt tokens": "{\"description\": \"min=304.677, mean=304.677, max=304.677, sum=609.353 (2)\", \"tab\": \"General information\", \"score\": \"304.67659574468087\"}", - "Conceptual Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"conceptual_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_conceptual_physics\"" - } - } - }, - { - "evaluation_name": "Electrical Engineering", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Electrical Engineering", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.731, - "details": { - "description": "min=0.731, mean=0.731, max=0.731, sum=1.462 (2)", - "tab": "Accuracy", - "Electrical Engineering - Observed inference time (s)": "{\"description\": \"min=0.287, mean=0.287, max=0.287, sum=0.575 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.287484780673323\"}", - "Electrical Engineering - # eval": "{\"description\": \"min=145, mean=145, max=145, sum=290 (2)\", \"tab\": \"General information\", \"score\": \"145.0\"}", - "Electrical Engineering - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Electrical Engineering - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Electrical Engineering - # prompt tokens": "{\"description\": \"min=439.228, mean=439.228, max=439.228, sum=878.455 (2)\", \"tab\": \"General information\", \"score\": \"439.22758620689655\"}", - "Electrical Engineering - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"electrical_engineering\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_electrical_engineering\"" - } - } - }, - { - "evaluation_name": "Elementary Mathematics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Elementary Mathematics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.651, - "details": { - "description": "min=0.651, mean=0.651, max=0.651, sum=1.302 (2)", - "tab": "Accuracy", - "Elementary Mathematics - Observed inference time (s)": "{\"description\": \"min=0.306, mean=0.306, max=0.306, sum=0.612 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.305813713679238\"}", - "Elementary Mathematics - # eval": "{\"description\": \"min=378, mean=378, max=378, sum=756 (2)\", \"tab\": \"General information\", \"score\": \"378.0\"}", - "Elementary Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Elementary Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Elementary Mathematics - # prompt tokens": "{\"description\": \"min=532.683, mean=532.683, max=532.683, sum=1065.365 (2)\", \"tab\": \"General information\", \"score\": \"532.6825396825396\"}", - "Elementary Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"elementary_mathematics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_elementary_mathematics\"" - } - } - }, - { - "evaluation_name": "Formal Logic", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Formal Logic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.556, - "details": { - "description": "min=0.556, mean=0.556, max=0.556, sum=1.111 (2)", - "tab": "Accuracy", - "Formal Logic - Observed inference time (s)": "{\"description\": \"min=0.355, mean=0.355, max=0.355, sum=0.711 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3554064962599013\"}", - "Formal Logic - # eval": "{\"description\": \"min=126, mean=126, max=126, sum=252 (2)\", \"tab\": \"General information\", \"score\": \"126.0\"}", - "Formal Logic - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Formal Logic - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Formal Logic - # prompt tokens": "{\"description\": \"min=604.492, mean=604.492, max=604.492, sum=1208.984 (2)\", \"tab\": \"General information\", \"score\": \"604.4920634920635\"}", - "Formal Logic - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"formal_logic\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_formal_logic\"" - } - } - }, - { - "evaluation_name": "High School World History", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on High School World History", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.903, - "details": { - "description": "min=0.903, mean=0.903, max=0.903, sum=1.806 (2)", - "tab": "Accuracy", - "High School Biology - Observed inference time (s)": "{\"description\": \"min=0.383, mean=0.383, max=0.383, sum=0.765 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3826789717520437\"}", - "High School Chemistry - Observed inference time (s)": "{\"description\": \"min=0.336, mean=0.336, max=0.336, sum=0.672 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3358421137767472\"}", - "High School Computer Science - Observed inference time (s)": "{\"description\": \"min=0.357, mean=0.357, max=0.357, sum=0.714 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3572020483016968\"}", - "High School European History - Observed inference time (s)": "{\"description\": \"min=0.442, mean=0.442, max=0.442, sum=0.883 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.44169029033545293\"}", - "High School Geography - Observed inference time (s)": "{\"description\": \"min=0.331, mean=0.331, max=0.331, sum=0.663 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.33136808029328935\"}", - "High School Government And Politics - Observed inference time (s)": "{\"description\": \"min=0.31, mean=0.31, max=0.31, sum=0.62 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.31024189563612864\"}", - "High School Macroeconomics - Observed inference time (s)": "{\"description\": \"min=0.302, mean=0.302, max=0.302, sum=0.605 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.30249478022257487\"}", - "High School Mathematics - Observed inference time (s)": "{\"description\": \"min=0.273, mean=0.273, max=0.273, sum=0.546 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2731299541614674\"}", - "High School Microeconomics - Observed inference time (s)": "{\"description\": \"min=0.343, mean=0.343, max=0.343, sum=0.687 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.34336654078058837\"}", - "High School Physics - Observed inference time (s)": "{\"description\": \"min=0.277, mean=0.277, max=0.277, sum=0.554 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.27723274167799794\"}", - "High School Psychology - Observed inference time (s)": "{\"description\": \"min=0.342, mean=0.342, max=0.342, sum=0.684 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3419263616614385\"}", - "High School Statistics - Observed inference time (s)": "{\"description\": \"min=0.415, mean=0.415, max=0.415, sum=0.83 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.41491677584471526\"}", - "High School US History - Observed inference time (s)": "{\"description\": \"min=0.367, mean=0.367, max=0.367, sum=0.735 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3674813041500017\"}", - "High School World History - Observed inference time (s)": "{\"description\": \"min=0.339, mean=0.339, max=0.339, sum=0.678 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.33923840120371884\"}", - "High School Biology - # eval": "{\"description\": \"min=310, mean=310, max=310, sum=620 (2)\", \"tab\": \"General information\", \"score\": \"310.0\"}", - "High School Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Biology - # prompt tokens": "{\"description\": \"min=504.874, mean=504.874, max=504.874, sum=1009.748 (2)\", \"tab\": \"General information\", \"score\": \"504.8741935483871\"}", - "High School Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Chemistry - # eval": "{\"description\": \"min=203, mean=203, max=203, sum=406 (2)\", \"tab\": \"General information\", \"score\": \"203.0\"}", - "High School Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # prompt tokens": "{\"description\": \"min=495.34, mean=495.34, max=495.34, sum=990.68 (2)\", \"tab\": \"General information\", \"score\": \"495.3399014778325\"}", - "High School Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "High School Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # prompt tokens": "{\"description\": \"min=865.8, mean=865.8, max=865.8, sum=1731.6 (2)\", \"tab\": \"General information\", \"score\": \"865.8\"}", - "High School Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School European History - # eval": "{\"description\": \"min=165, mean=165, max=165, sum=330 (2)\", \"tab\": \"General information\", \"score\": \"165.0\"}", - "High School European History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School European History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # prompt tokens": "{\"description\": \"min=2793.83, mean=2793.83, max=2793.83, sum=5587.661 (2)\", \"tab\": \"General information\", \"score\": \"2793.830303030303\"}", - "High School European History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Geography - # eval": "{\"description\": \"min=198, mean=198, max=198, sum=396 (2)\", \"tab\": \"General information\", \"score\": \"198.0\"}", - "High School Geography - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Geography - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # prompt tokens": "{\"description\": \"min=372.783, mean=372.783, max=372.783, sum=745.566 (2)\", \"tab\": \"General information\", \"score\": \"372.7828282828283\"}", - "High School Geography - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Government And Politics - # eval": "{\"description\": \"min=193, mean=193, max=193, sum=386 (2)\", \"tab\": \"General information\", \"score\": \"193.0\"}", - "High School Government And Politics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Government And Politics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # prompt tokens": "{\"description\": \"min=463.01, mean=463.01, max=463.01, sum=926.021 (2)\", \"tab\": \"General information\", \"score\": \"463.0103626943005\"}", - "High School Government And Politics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Macroeconomics - # eval": "{\"description\": \"min=390, mean=390, max=390, sum=780 (2)\", \"tab\": \"General information\", \"score\": \"390.0\"}", - "High School Macroeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Macroeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # prompt tokens": "{\"description\": \"min=371.451, mean=371.451, max=371.451, sum=742.903 (2)\", \"tab\": \"General information\", \"score\": \"371.4512820512821\"}", - "High School Macroeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Mathematics - # eval": "{\"description\": \"min=270, mean=270, max=270, sum=540 (2)\", \"tab\": \"General information\", \"score\": \"270.0\"}", - "High School Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # prompt tokens": "{\"description\": \"min=532.456, mean=532.456, max=532.456, sum=1064.911 (2)\", \"tab\": \"General information\", \"score\": \"532.4555555555555\"}", - "High School Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Microeconomics - # eval": "{\"description\": \"min=238, mean=238, max=238, sum=476 (2)\", \"tab\": \"General information\", \"score\": \"238.0\"}", - "High School Microeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Microeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # prompt tokens": "{\"description\": \"min=398.739, mean=398.739, max=398.739, sum=797.479 (2)\", \"tab\": \"General information\", \"score\": \"398.73949579831935\"}", - "High School Microeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Physics - # eval": "{\"description\": \"min=151, mean=151, max=151, sum=302 (2)\", \"tab\": \"General information\", \"score\": \"151.0\"}", - "High School Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # prompt tokens": "{\"description\": \"min=560.238, mean=560.238, max=560.238, sum=1120.477 (2)\", \"tab\": \"General information\", \"score\": \"560.2384105960265\"}", - "High School Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Psychology - # eval": "{\"description\": \"min=545, mean=545, max=545, sum=1090 (2)\", \"tab\": \"General information\", \"score\": \"545.0\"}", - "High School Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # prompt tokens": "{\"description\": \"min=492.917, mean=492.917, max=492.917, sum=985.835 (2)\", \"tab\": \"General information\", \"score\": \"492.91743119266056\"}", - "High School Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Statistics - # eval": "{\"description\": \"min=216, mean=216, max=216, sum=432 (2)\", \"tab\": \"General information\", \"score\": \"216.0\"}", - "High School Statistics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Statistics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # prompt tokens": "{\"description\": \"min=787.574, mean=787.574, max=787.574, sum=1575.148 (2)\", \"tab\": \"General information\", \"score\": \"787.574074074074\"}", - "High School Statistics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School US History - # eval": "{\"description\": \"min=204, mean=204, max=204, sum=408 (2)\", \"tab\": \"General information\", \"score\": \"204.0\"}", - "High School US History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School US History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # prompt tokens": "{\"description\": \"min=2220.005, mean=2220.005, max=2220.005, sum=4440.01 (2)\", \"tab\": \"General information\", \"score\": \"2220.0049019607845\"}", - "High School US History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School World History - # eval": "{\"description\": \"min=237, mean=237, max=237, sum=474 (2)\", \"tab\": \"General information\", \"score\": \"237.0\"}", - "High School World History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School World History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # prompt tokens": "{\"description\": \"min=1424.439, mean=1424.439, max=1424.439, sum=2848.878 (2)\", \"tab\": \"General information\", \"score\": \"1424.4388185654009\"}", - "High School World History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"high_school_world_history\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_high_school_world_history\"" - } - } - }, - { - "evaluation_name": "Human Sexuality", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Human Sexuality", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.863, - "details": { - "description": "min=0.863, mean=0.863, max=0.863, sum=1.725 (2)", - "tab": "Accuracy", - "Human Aging - Observed inference time (s)": "{\"description\": \"min=0.305, mean=0.305, max=0.305, sum=0.61 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.30522876897734913\"}", - "Human Sexuality - Observed inference time (s)": "{\"description\": \"min=0.303, mean=0.303, max=0.303, sum=0.606 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.30280636285097545\"}", - "Human Aging - # eval": "{\"description\": \"min=223, mean=223, max=223, sum=446 (2)\", \"tab\": \"General information\", \"score\": \"223.0\"}", - "Human Aging - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Aging - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Aging - # prompt tokens": "{\"description\": \"min=316.453, mean=316.453, max=316.453, sum=632.906 (2)\", \"tab\": \"General information\", \"score\": \"316.4529147982063\"}", - "Human Aging - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Human Sexuality - # eval": "{\"description\": \"min=131, mean=131, max=131, sum=262 (2)\", \"tab\": \"General information\", \"score\": \"131.0\"}", - "Human Sexuality - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Sexuality - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # prompt tokens": "{\"description\": \"min=335.695, mean=335.695, max=335.695, sum=671.389 (2)\", \"tab\": \"General information\", \"score\": \"335.69465648854964\"}", - "Human Sexuality - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"human_sexuality\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_human_sexuality\"" - } - } - }, - { - "evaluation_name": "International Law", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on International Law", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.926, - "details": { - "description": "min=0.926, mean=0.926, max=0.926, sum=1.851 (2)", - "tab": "Accuracy", - "International Law - Observed inference time (s)": "{\"description\": \"min=0.343, mean=0.343, max=0.343, sum=0.685 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3425306268959991\"}", - "International Law - # eval": "{\"description\": \"min=121, mean=121, max=121, sum=242 (2)\", \"tab\": \"General information\", \"score\": \"121.0\"}", - "International Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "International Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "International Law - # prompt tokens": "{\"description\": \"min=639.504, mean=639.504, max=639.504, sum=1279.008 (2)\", \"tab\": \"General information\", \"score\": \"639.5041322314049\"}", - "International Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"international_law\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_international_law\"" - } - } - }, - { - "evaluation_name": "Logical Fallacies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Logical Fallacies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.871, - "details": { - "description": "min=0.871, mean=0.871, max=0.871, sum=1.742 (2)", - "tab": "Accuracy", - "Logical Fallacies - Observed inference time (s)": "{\"description\": \"min=0.297, mean=0.297, max=0.297, sum=0.595 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.29739713961361375\"}", - "Logical Fallacies - # eval": "{\"description\": \"min=163, mean=163, max=163, sum=326 (2)\", \"tab\": \"General information\", \"score\": \"163.0\"}", - "Logical Fallacies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Logical Fallacies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Logical Fallacies - # prompt tokens": "{\"description\": \"min=445.84, mean=445.84, max=445.84, sum=891.681 (2)\", \"tab\": \"General information\", \"score\": \"445.840490797546\"}", - "Logical Fallacies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"logical_fallacies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_logical_fallacies\"" - } - } - }, - { - "evaluation_name": "Machine Learning", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Machine Learning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.616, - "details": { - "description": "min=0.616, mean=0.616, max=0.616, sum=1.232 (2)", - "tab": "Accuracy", - "Machine Learning - Observed inference time (s)": "{\"description\": \"min=0.297, mean=0.297, max=0.297, sum=0.594 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2970866986683437\"}", - "Machine Learning - # eval": "{\"description\": \"min=112, mean=112, max=112, sum=224 (2)\", \"tab\": \"General information\", \"score\": \"112.0\"}", - "Machine Learning - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Machine Learning - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Machine Learning - # prompt tokens": "{\"description\": \"min=666.205, mean=666.205, max=666.205, sum=1332.411 (2)\", \"tab\": \"General information\", \"score\": \"666.2053571428571\"}", - "Machine Learning - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"machine_learning\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_machine_learning\"" - } - } - }, - { - "evaluation_name": "Management", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Management", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.845, - "details": { - "description": "min=0.845, mean=0.845, max=0.845, sum=1.689 (2)", - "tab": "Accuracy", - "Management - Observed inference time (s)": "{\"description\": \"min=0.305, mean=0.305, max=0.305, sum=0.611 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3053626088262762\"}", - "Management - # eval": "{\"description\": \"min=103, mean=103, max=103, sum=206 (2)\", \"tab\": \"General information\", \"score\": \"103.0\"}", - "Management - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Management - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Management - # prompt tokens": "{\"description\": \"min=279.485, mean=279.485, max=279.485, sum=558.971 (2)\", \"tab\": \"General information\", \"score\": \"279.4854368932039\"}", - "Management - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"management\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_management\"" - } - } - }, - { - "evaluation_name": "Marketing", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Marketing", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.927, - "details": { - "description": "min=0.927, mean=0.927, max=0.927, sum=1.855 (2)", - "tab": "Accuracy", - "Marketing - Observed inference time (s)": "{\"description\": \"min=0.306, mean=0.306, max=0.306, sum=0.612 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3060942073153634\"}", - "Marketing - # eval": "{\"description\": \"min=234, mean=234, max=234, sum=468 (2)\", \"tab\": \"General information\", \"score\": \"234.0\"}", - "Marketing - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Marketing - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Marketing - # prompt tokens": "{\"description\": \"min=399.85, mean=399.85, max=399.85, sum=799.701 (2)\", \"tab\": \"General information\", \"score\": \"399.85042735042737\"}", - "Marketing - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"marketing\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_marketing\"" - } - } - }, - { - "evaluation_name": "Medical Genetics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Medical Genetics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.89, - "details": { - "description": "min=0.89, mean=0.89, max=0.89, sum=1.78 (2)", - "tab": "Accuracy", - "Medical Genetics - Observed inference time (s)": "{\"description\": \"min=0.311, mean=0.311, max=0.311, sum=0.622 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.31078683137893676\"}", - "Medical Genetics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Medical Genetics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Medical Genetics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Medical Genetics - # prompt tokens": "{\"description\": \"min=343.23, mean=343.23, max=343.23, sum=686.46 (2)\", \"tab\": \"General information\", \"score\": \"343.23\"}", - "Medical Genetics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"medical_genetics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_medical_genetics\"" - } - } - }, - { - "evaluation_name": "Miscellaneous", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Miscellaneous", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.913, - "details": { - "description": "min=0.913, mean=0.913, max=0.913, sum=1.826 (2)", - "tab": "Accuracy", - "Miscellaneous - Observed inference time (s)": "{\"description\": \"min=0.302, mean=0.302, max=0.302, sum=0.604 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3020631249989282\"}", - "Miscellaneous - # eval": "{\"description\": \"min=783, mean=783, max=783, sum=1566 (2)\", \"tab\": \"General information\", \"score\": \"783.0\"}", - "Miscellaneous - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Miscellaneous - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Miscellaneous - # prompt tokens": "{\"description\": \"min=296.479, mean=296.479, max=296.479, sum=592.958 (2)\", \"tab\": \"General information\", \"score\": \"296.47892720306515\"}", - "Miscellaneous - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"miscellaneous\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_miscellaneous\"" - } - } - }, - { - "evaluation_name": "Moral Scenarios", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Moral Scenarios", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.485, - "details": { - "description": "min=0.485, mean=0.485, max=0.485, sum=0.97 (2)", - "tab": "Accuracy", - "Moral Disputes - Observed inference time (s)": "{\"description\": \"min=0.316, mean=0.316, max=0.316, sum=0.631 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.31556026577260454\"}", - "Moral Scenarios - Observed inference time (s)": "{\"description\": \"min=0.318, mean=0.318, max=0.318, sum=0.637 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3183864769322912\"}", - "Moral Disputes - # eval": "{\"description\": \"min=346, mean=346, max=346, sum=692 (2)\", \"tab\": \"General information\", \"score\": \"346.0\"}", - "Moral Disputes - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Disputes - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Disputes - # prompt tokens": "{\"description\": \"min=474.835, mean=474.835, max=474.835, sum=949.671 (2)\", \"tab\": \"General information\", \"score\": \"474.83526011560696\"}", - "Moral Disputes - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Moral Scenarios - # eval": "{\"description\": \"min=895, mean=895, max=895, sum=1790 (2)\", \"tab\": \"General information\", \"score\": \"895.0\"}", - "Moral Scenarios - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Scenarios - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # prompt tokens": "{\"description\": \"min=655.068, mean=655.068, max=655.068, sum=1310.136 (2)\", \"tab\": \"General information\", \"score\": \"655.068156424581\"}", - "Moral Scenarios - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"moral_scenarios\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_moral_scenarios\"" - } - } - }, - { - "evaluation_name": "Nutrition", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Nutrition", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.827, - "details": { - "description": "min=0.827, mean=0.827, max=0.827, sum=1.654 (2)", - "tab": "Accuracy", - "Nutrition - Observed inference time (s)": "{\"description\": \"min=0.31, mean=0.31, max=0.31, sum=0.621 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3104910164876701\"}", - "Nutrition - # eval": "{\"description\": \"min=306, mean=306, max=306, sum=612 (2)\", \"tab\": \"General information\", \"score\": \"306.0\"}", - "Nutrition - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Nutrition - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Nutrition - # prompt tokens": "{\"description\": \"min=581.997, mean=581.997, max=581.997, sum=1163.993 (2)\", \"tab\": \"General information\", \"score\": \"581.9967320261438\"}", - "Nutrition - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"nutrition\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_nutrition\"" - } - } - }, - { - "evaluation_name": "Prehistory", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Prehistory", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.833, - "details": { - "description": "min=0.833, mean=0.833, max=0.833, sum=1.667 (2)", - "tab": "Accuracy", - "Prehistory - Observed inference time (s)": "{\"description\": \"min=0.311, mean=0.311, max=0.311, sum=0.621 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3106661284411395\"}", - "Prehistory - # eval": "{\"description\": \"min=324, mean=324, max=324, sum=648 (2)\", \"tab\": \"General information\", \"score\": \"324.0\"}", - "Prehistory - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Prehistory - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Prehistory - # prompt tokens": "{\"description\": \"min=513.944, mean=513.944, max=513.944, sum=1027.889 (2)\", \"tab\": \"General information\", \"score\": \"513.9444444444445\"}", - "Prehistory - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"prehistory\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_prehistory\"" - } - } - }, - { - "evaluation_name": "Public Relations", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Public Relations", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.791, - "details": { - "description": "min=0.791, mean=0.791, max=0.791, sum=1.582 (2)", - "tab": "Accuracy", - "Public Relations - Observed inference time (s)": "{\"description\": \"min=0.303, mean=0.303, max=0.303, sum=0.606 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.30300807519392536\"}", - "Public Relations - # eval": "{\"description\": \"min=110, mean=110, max=110, sum=220 (2)\", \"tab\": \"General information\", \"score\": \"110.0\"}", - "Public Relations - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Public Relations - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Public Relations - # prompt tokens": "{\"description\": \"min=402.918, mean=402.918, max=402.918, sum=805.836 (2)\", \"tab\": \"General information\", \"score\": \"402.91818181818184\"}", - "Public Relations - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"public_relations\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_public_relations\"" - } - } - }, - { - "evaluation_name": "Security Studies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Security Studies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.788, - "details": { - "description": "min=0.788, mean=0.788, max=0.788, sum=1.576 (2)", - "tab": "Accuracy", - "Security Studies - Observed inference time (s)": "{\"description\": \"min=0.733, mean=0.733, max=0.733, sum=1.466 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.733092721627683\"}", - "Security Studies - # eval": "{\"description\": \"min=245, mean=245, max=245, sum=490 (2)\", \"tab\": \"General information\", \"score\": \"245.0\"}", - "Security Studies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Security Studies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Security Studies - # prompt tokens": "{\"description\": \"min=1166.686, mean=1166.686, max=1166.686, sum=2333.371 (2)\", \"tab\": \"General information\", \"score\": \"1166.6857142857143\"}", - "Security Studies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"security_studies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_security_studies\"" - } - } - }, - { - "evaluation_name": "Sociology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Sociology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9, - "details": { - "description": "min=0.9, mean=0.9, max=0.9, sum=1.801 (2)", - "tab": "Accuracy", - "Sociology - Observed inference time (s)": "{\"description\": \"min=0.361, mean=0.361, max=0.361, sum=0.722 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3608738794848694\"}", - "Sociology - # eval": "{\"description\": \"min=201, mean=201, max=201, sum=402 (2)\", \"tab\": \"General information\", \"score\": \"201.0\"}", - "Sociology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Sociology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Sociology - # prompt tokens": "{\"description\": \"min=444.269, mean=444.269, max=444.269, sum=888.537 (2)\", \"tab\": \"General information\", \"score\": \"444.2686567164179\"}", - "Sociology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"sociology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_sociology\"" - } - } - }, - { - "evaluation_name": "Virology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Virology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.536, - "details": { - "description": "min=0.536, mean=0.536, max=0.536, sum=1.072 (2)", - "tab": "Accuracy", - "Virology - Observed inference time (s)": "{\"description\": \"min=0.489, mean=0.489, max=0.489, sum=0.978 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.48897463298705685\"}", - "Virology - # eval": "{\"description\": \"min=166, mean=166, max=166, sum=332 (2)\", \"tab\": \"General information\", \"score\": \"166.0\"}", - "Virology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Virology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Virology - # prompt tokens": "{\"description\": \"min=334.434, mean=334.434, max=334.434, sum=668.867 (2)\", \"tab\": \"General information\", \"score\": \"334.43373493975906\"}", - "Virology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"virology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_virology\"" - } - } - }, - { - "evaluation_name": "World Religions", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on World Religions", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.86, - "details": { - "description": "min=0.86, mean=0.86, max=0.86, sum=1.719 (2)", - "tab": "Accuracy", - "World Religions - Observed inference time (s)": "{\"description\": \"min=0.623, mean=0.623, max=0.623, sum=1.247 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6232896199700428\"}", - "World Religions - # eval": "{\"description\": \"min=171, mean=171, max=171, sum=342 (2)\", \"tab\": \"General information\", \"score\": \"171.0\"}", - "World Religions - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "World Religions - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "World Religions - # prompt tokens": "{\"description\": \"min=267.936, mean=267.936, max=267.936, sum=535.871 (2)\", \"tab\": \"General information\", \"score\": \"267.9356725146199\"}", - "World Religions - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"world_religions\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_world_religions\"" - } - } - }, - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.774, - "details": { - "description": "", - "tab": "Efficiency" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_id": "reward-bench-2/openai_gpt-4o-mini-2024-07-18/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5796 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4105 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3438 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5191 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7667 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7414 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6962 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "reward-bench/openai_gpt-4o-mini-2024-07-18/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8007 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9497 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6075 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8081 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8374 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/openai_gpt-5-2-2025-12-11-fc.json b/data/models/openai_gpt-5-2-2025-12-11-fc.json deleted file mode 100644 index 63f5bb436004ec696d323bd2aad683c2688e5cec..0000000000000000000000000000000000000000 --- a/data/models/openai_gpt-5-2-2025-12-11-fc.json +++ /dev/null @@ -1,851 +0,0 @@ -{ - "model_info": { - "name": "GPT-5.2-2025-12-11 (FC)", - "id": "openai/gpt-5-2-2025-12-11-fc", - "developer": "OpenAI", - "additional_details": { - "raw_model_name": "GPT-5.2-2025-12-11 (FC)", - "organization": "OpenAI", - "license": "Proprietary", - "mode": "FC", - "model_link": "https://openai.com/zh-Hans-CN/index/introducing-gpt-5-2/" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/openai/gpt-5-2-2025-12-11-fc/1775236112.373833", - "retrieved_timestamp": "1775236112.373833", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 16.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 55.87 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 85.65 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 2.23 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 9.75 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 5.26 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 81.85 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 72.92 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 88.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 89.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 77.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 70.39 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 71.71 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 70.37 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 68.75 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 58.33 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 28.12 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 36.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 18.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 27.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 30.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 75.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 78.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 73.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 45.81 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 33.55 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 43.23 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 60.65 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 75.0 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 79.42 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/openai_gpt-5-2-2025-12-11-prompt.json b/data/models/openai_gpt-5-2-2025-12-11-prompt.json deleted file mode 100644 index 1a27868dcd10c21c692a8c5e0f088a13259c25b4..0000000000000000000000000000000000000000 --- a/data/models/openai_gpt-5-2-2025-12-11-prompt.json +++ /dev/null @@ -1,905 +0,0 @@ -{ - "model_info": { - "name": "GPT-5.2-2025-12-11 (Prompt)", - "id": "openai/gpt-5-2-2025-12-11-prompt", - "developer": "OpenAI", - "additional_details": { - "raw_model_name": "GPT-5.2-2025-12-11 (Prompt)", - "organization": "OpenAI", - "license": "Proprietary", - "mode": "Prompt", - "model_link": "https://openai.com/zh-Hans-CN/index/introducing-gpt-5-2/" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/openai/gpt-5-2-2025-12-11-prompt/1775236112.384796", - "retrieved_timestamp": "1775236112.384796", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 38.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 45.27 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 164.58 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 4.21 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 20.93 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 10.58 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 78.29 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 71.17 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 83.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 84.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 74.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 67.14 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 77.91 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 64.58 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 75.0 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 58.33 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 43.75 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 54.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 40.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 33.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 46.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 40.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 45.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 36.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 3.87 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 2.58 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 1.94 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 7.1 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 75.0 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 87.26 - } - }, - { - "evaluation_result_id": "bfcl.format_sensitivity.max_delta::max_delta", - "evaluation_name": "bfcl.format_sensitivity.max_delta", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.format_sensitivity.max_delta", - "metric_name": "Format sensitivity max delta", - "metric_kind": "difference", - "metric_unit": "percentage_points", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Format Sensitivity Max Delta" - } - }, - "score_details": { - "score": 13.0 - } - }, - { - "evaluation_result_id": "bfcl.format_sensitivity.stddev::stddev", - "evaluation_name": "bfcl.format_sensitivity.stddev", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.format_sensitivity.stddev", - "metric_name": "Format sensitivity standard deviation", - "metric_kind": "difference", - "metric_unit": "percentage_points", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Format Sensitivity Standard Deviation" - } - }, - "score_details": { - "score": 3.25 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/openai_gpt-5-2025-08-07.json b/data/models/openai_gpt-5-2025-08-07.json deleted file mode 100644 index 0853492fcc4bbd45f07185718454686350fe1be2..0000000000000000000000000000000000000000 --- a/data/models/openai_gpt-5-2025-08-07.json +++ /dev/null @@ -1,1425 +0,0 @@ -{ - "model_info": { - "name": "gpt-5-2025-08-07", - "id": "openai/gpt-5-2025-08-07", - "developer": "OpenAI", - "inference_platform": "unknown", - "additional_details": { - "display_name": "GPT-5" - } - }, - "evaluations": [ - { - "evaluation_id": "global-mmlu-lite/openai_gpt-5-2025-08-07/1773936496.366405", - "retrieved_timestamp": "1773936496.366405", - "source_metadata": { - "source_name": "Global MMLU Lite Leaderboard", - "source_type": "documentation", - "source_organization_name": "kaggle", - "source_organization_url": "www.kaggle.com", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "kaggle kernel", - "version": "4", - "additional_details": { - "url": "https://www.kaggle.com/code/shivalikasingh95/global-mmlu-lite-sample-notebook" - } - }, - "benchmark": "global-mmlu-lite", - "evaluation_results": [ - { - "evaluation_name": "Global MMLU Lite", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Global MMLU Lite", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8895 - } - }, - { - "evaluation_name": "Culturally Sensitive", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Sensitive", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8913 - } - }, - { - "evaluation_name": "Culturally Agnostic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Agnostic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8878 - } - }, - { - "evaluation_name": "Arabic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Arabic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8925, - "uncertainty": { - "confidence_interval": { - "lower": -0.0304, - "upper": 0.0304, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "English", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - English", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8725, - "uncertainty": { - "confidence_interval": { - "lower": -0.0327, - "upper": 0.0327, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Bengali", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Bengali", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9, - "uncertainty": { - "confidence_interval": { - "lower": -0.0294, - "upper": 0.0294, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "German", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - German", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.91, - "uncertainty": { - "confidence_interval": { - "lower": -0.028, - "upper": 0.028, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "French", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - French", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9075, - "uncertainty": { - "confidence_interval": { - "lower": -0.0284, - "upper": 0.0284, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Hindi", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Hindi", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.865, - "uncertainty": { - "confidence_interval": { - "lower": -0.0335, - "upper": 0.0335, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Indonesian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Indonesian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.795, - "uncertainty": { - "confidence_interval": { - "lower": -0.0396, - "upper": 0.0396, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Italian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Italian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9075, - "uncertainty": { - "confidence_interval": { - "lower": -0.0284, - "upper": 0.0284, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Japanese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Japanese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8875, - "uncertainty": { - "confidence_interval": { - "lower": -0.031, - "upper": 0.031, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Korean", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Korean", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.915, - "uncertainty": { - "confidence_interval": { - "lower": -0.0273, - "upper": 0.0273, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Portuguese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Portuguese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8875, - "uncertainty": { - "confidence_interval": { - "lower": -0.031, - "upper": 0.031, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Spanish", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Spanish", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.905, - "uncertainty": { - "confidence_interval": { - "lower": -0.0287, - "upper": 0.0287, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Swahili", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Swahili", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.865, - "uncertainty": { - "confidence_interval": { - "lower": -0.0335, - "upper": 0.0335, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Yoruba", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Yoruba", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9125, - "uncertainty": { - "confidence_interval": { - "lower": -0.0277, - "upper": 0.0277, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Chinese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Chinese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.895, - "uncertainty": { - "confidence_interval": { - "lower": -0.03, - "upper": 0.03, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Burmese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Burmese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.915, - "uncertainty": { - "confidence_interval": { - "lower": -0.0273, - "upper": 0.0273, - "method": "unknown" - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "global-mmlu-lite/openai_gpt-5-2025-08-07/1773936583.743359", - "retrieved_timestamp": "1773936583.743359", - "source_metadata": { - "source_name": "Global MMLU Lite Leaderboard", - "source_type": "documentation", - "source_organization_name": "kaggle", - "source_organization_url": "www.kaggle.com", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "kaggle kernel", - "version": "4", - "additional_details": { - "url": "https://www.kaggle.com/code/shivalikasingh95/global-mmlu-lite-sample-notebook" - } - }, - "benchmark": "global-mmlu-lite", - "evaluation_results": [ - { - "evaluation_name": "Global MMLU Lite", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Global MMLU Lite", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8895 - } - }, - { - "evaluation_name": "Culturally Sensitive", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Sensitive", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8913 - } - }, - { - "evaluation_name": "Culturally Agnostic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Agnostic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8878 - } - }, - { - "evaluation_name": "Arabic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Arabic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8925, - "uncertainty": { - "confidence_interval": { - "lower": -0.0304, - "upper": 0.0304, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "English", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - English", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8725, - "uncertainty": { - "confidence_interval": { - "lower": -0.0327, - "upper": 0.0327, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Bengali", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Bengali", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9, - "uncertainty": { - "confidence_interval": { - "lower": -0.0294, - "upper": 0.0294, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "German", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - German", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.91, - "uncertainty": { - "confidence_interval": { - "lower": -0.028, - "upper": 0.028, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "French", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - French", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9075, - "uncertainty": { - "confidence_interval": { - "lower": -0.0284, - "upper": 0.0284, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Hindi", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Hindi", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.865, - "uncertainty": { - "confidence_interval": { - "lower": -0.0335, - "upper": 0.0335, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Indonesian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Indonesian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.795, - "uncertainty": { - "confidence_interval": { - "lower": -0.0396, - "upper": 0.0396, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Italian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Italian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9075, - "uncertainty": { - "confidence_interval": { - "lower": -0.0284, - "upper": 0.0284, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Japanese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Japanese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8875, - "uncertainty": { - "confidence_interval": { - "lower": -0.031, - "upper": 0.031, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Korean", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Korean", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.915, - "uncertainty": { - "confidence_interval": { - "lower": -0.0273, - "upper": 0.0273, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Portuguese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Portuguese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8875, - "uncertainty": { - "confidence_interval": { - "lower": -0.031, - "upper": 0.031, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Spanish", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Spanish", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.905, - "uncertainty": { - "confidence_interval": { - "lower": -0.0287, - "upper": 0.0287, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Swahili", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Swahili", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.865, - "uncertainty": { - "confidence_interval": { - "lower": -0.0335, - "upper": 0.0335, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Yoruba", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Yoruba", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9125, - "uncertainty": { - "confidence_interval": { - "lower": -0.0277, - "upper": 0.0277, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Chinese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Chinese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.895, - "uncertainty": { - "confidence_interval": { - "lower": -0.03, - "upper": 0.03, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Burmese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Burmese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.915, - "uncertainty": { - "confidence_interval": { - "lower": -0.0273, - "upper": 0.0273, - "method": "unknown" - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "helm_capabilities/openai_gpt-5-2025-08-07/1774096304.5056698", - "retrieved_timestamp": "1774096304.5056698", - "source_metadata": { - "source_name": "helm_capabilities", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_capabilities", - "evaluation_results": [ - { - "evaluation_name": "Mean score", - "source_data": { - "dataset_name": "helm_capabilities", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "The mean of the scores from all columns.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.807, - "details": { - "description": "", - "tab": "Accuracy", - "Mean score - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"74.66990821942755\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU-Pro", - "source_data": { - "dataset_name": "MMLU-Pro", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on MMLU-Pro", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.863, - "details": { - "description": "min=0.863, mean=0.863, max=0.863, sum=0.863 (1)", - "tab": "Accuracy", - "MMLU-Pro - Observed inference time (s)": "{\"description\": \"min=18.668, mean=18.668, max=18.668, sum=18.668 (1)\", \"tab\": \"Efficiency\", \"score\": \"18.668269051074983\"}", - "MMLU-Pro - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "MMLU-Pro - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - # prompt tokens": "{\"description\": \"min=248.569, mean=248.569, max=248.569, sum=248.569 (1)\", \"tab\": \"General information\", \"score\": \"248.569\"}", - "MMLU-Pro - # output tokens": "{\"description\": \"min=5.028, mean=5.028, max=5.028, sum=5.028 (1)\", \"tab\": \"General information\", \"score\": \"5.028\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"all\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.791, - "details": { - "description": "min=0.791, mean=0.791, max=0.791, sum=0.791 (1)", - "tab": "Accuracy", - "GPQA - Observed inference time (s)": "{\"description\": \"min=57.418, mean=57.418, max=57.418, sum=57.418 (1)\", \"tab\": \"Efficiency\", \"score\": \"57.41822674028542\"}", - "GPQA - # eval": "{\"description\": \"min=446, mean=446, max=446, sum=446 (1)\", \"tab\": \"General information\", \"score\": \"446.0\"}", - "GPQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - # prompt tokens": "{\"description\": \"min=268.152, mean=268.152, max=268.152, sum=268.152 (1)\", \"tab\": \"General information\", \"score\": \"268.15246636771303\"}", - "GPQA - # output tokens": "{\"description\": \"min=5.935, mean=5.935, max=5.935, sum=5.935 (1)\", \"tab\": \"General information\", \"score\": \"5.934977578475336\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"gpqa_main\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "IFEval Strict Acc on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.875, - "details": { - "description": "min=0.875, mean=0.875, max=0.875, sum=0.875 (1)", - "tab": "Accuracy", - "IFEval - Observed inference time (s)": "{\"description\": \"min=35.937, mean=35.937, max=35.937, sum=35.937 (1)\", \"tab\": \"Efficiency\", \"score\": \"35.937195608664354\"}", - "IFEval - # eval": "{\"description\": \"min=541, mean=541, max=541, sum=541 (1)\", \"tab\": \"General information\", \"score\": \"541.0\"}", - "IFEval - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - # prompt tokens": "{\"description\": \"min=45.671, mean=45.671, max=45.671, sum=45.671 (1)\", \"tab\": \"General information\", \"score\": \"45.67097966728281\"}", - "IFEval - # output tokens": "{\"description\": \"min=527.641, mean=527.641, max=527.641, sum=527.641 (1)\", \"tab\": \"General information\", \"score\": \"527.6414048059149\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WildBench", - "source_data": { - "dataset_name": "WildBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "WB Score on WildBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.857, - "details": { - "description": "min=0.857, mean=0.857, max=0.857, sum=0.857 (1)", - "tab": "Accuracy", - "WildBench - Observed inference time (s)": "{\"description\": \"min=88.595, mean=88.595, max=88.595, sum=88.595 (1)\", \"tab\": \"Efficiency\", \"score\": \"88.59490567517281\"}", - "WildBench - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "WildBench - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # prompt tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # output tokens": "{\"description\": \"min=1518.974, mean=1518.974, max=1518.974, sum=1518.974 (1)\", \"tab\": \"General information\", \"score\": \"1518.974\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"v2\"" - } - } - }, - { - "evaluation_name": "Omni-MATH", - "source_data": { - "dataset_name": "Omni-MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Acc on Omni-MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.647, - "details": { - "description": "min=0.647, mean=0.647, max=0.647, sum=0.647 (1)", - "tab": "Accuracy", - "Omni-MATH - Observed inference time (s)": "{\"description\": \"min=172.731, mean=172.731, max=172.731, sum=172.731 (1)\", \"tab\": \"Efficiency\", \"score\": \"172.73094402194022\"}", - "Omni-MATH - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "Omni-MATH - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - # prompt tokens": "{\"description\": \"min=109.623, mean=109.623, max=109.623, sum=109.623 (1)\", \"tab\": \"General information\", \"score\": \"109.623\"}", - "Omni-MATH - # output tokens": "{\"description\": \"min=228.774, mean=228.774, max=228.774, sum=228.774 (1)\", \"tab\": \"General information\", \"score\": \"228.774\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "livecodebenchpro/gpt-5-2025-08-07/1760492095.8105888", - "retrieved_timestamp": "1760492095.8105888", - "source_metadata": { - "source_organization_name": "New York University, Princeton University, University of California San Diego, University of Washington and Canyon Crest Academy", - "evaluator_relationship": "third_party", - "source_name": "Live Code Bench Pro", - "source_type": "documentation" - }, - "eval_library": { - "name": "unknown", - "version": "unknown" - }, - "benchmark": "livecodebenchpro", - "evaluation_results": [ - { - "evaluation_name": "Hard Problems", - "metric_config": { - "evaluation_description": "Pass@1 on Hard Problems", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.04225352112676056 - }, - "source_data": { - "dataset_name": "Hard Problems", - "source_type": "url", - "url": [ - "https://webhook.cp-bench.orzzh.com/leaderboard/llm/difficulty?difficulty=hard&benchmark_mode=live" - ] - } - }, - { - "evaluation_name": "Medium Problems", - "metric_config": { - "evaluation_description": "Pass@1 on Medium Problems", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.4084507042253521 - }, - "source_data": { - "dataset_name": "Medium Problems", - "source_type": "url", - "url": [ - "https://webhook.cp-bench.orzzh.com/leaderboard/llm/difficulty?difficulty=medium&benchmark_mode=live" - ] - } - }, - { - "evaluation_name": "Easy Problems", - "metric_config": { - "evaluation_description": "Pass@1 on Easy Problems", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.8873239436619719 - }, - "source_data": { - "dataset_name": "Easy Problems", - "source_type": "url", - "url": [ - "https://webhook.cp-bench.orzzh.com/leaderboard/llm/difficulty?difficulty=easy&benchmark_mode=live" - ] - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "livecodebenchpro/gpt-5-2025-08-07/1770683238.099205", - "retrieved_timestamp": "1770683238.099205", - "source_metadata": { - "source_name": "Live Code Bench Pro", - "source_type": "documentation", - "source_organization_name": "New York University, Princeton University, University of California San Diego, University of Washington and Canyon Crest Academy", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "unknown", - "version": "unknown" - }, - "benchmark": "livecodebenchpro", - "evaluation_results": [ - { - "evaluation_name": "Hard Problems", - "source_data": { - "dataset_name": "Hard Problems", - "source_type": "url", - "url": [ - "https://webhook.cp-bench.orzzh.com/leaderboard/llm/difficulty?difficulty=hard&benchmark_mode=live" - ] - }, - "metric_config": { - "evaluation_description": "Pass@1 on Hard Problems", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0423 - } - }, - { - "evaluation_name": "Medium Problems", - "source_data": { - "dataset_name": "Medium Problems", - "source_type": "url", - "url": [ - "https://webhook.cp-bench.orzzh.com/leaderboard/llm/difficulty?difficulty=medium&benchmark_mode=live" - ] - }, - "metric_config": { - "evaluation_description": "Pass@1 on Medium Problems", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4085 - } - }, - { - "evaluation_name": "Easy Problems", - "source_data": { - "dataset_name": "Easy Problems", - "source_type": "url", - "url": [ - "https://webhook.cp-bench.orzzh.com/leaderboard/llm/difficulty?difficulty=easy&benchmark_mode=live" - ] - }, - "metric_config": { - "evaluation_description": "Pass@1 on Easy Problems", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9014 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/openai_gpt-5-codex.json b/data/models/openai_gpt-5-codex.json deleted file mode 100644 index c82c5f95f908380f688e9ef7118281bf4bf2ade4..0000000000000000000000000000000000000000 --- a/data/models/openai_gpt-5-codex.json +++ /dev/null @@ -1,235 +0,0 @@ -{ - "model_info": { - "name": "GPT-5-Codex", - "id": "openai/gpt-5-codex", - "developer": "OpenAI", - "additional_details": { - "agent_name": "Mini-SWE-Agent", - "agent_organization": "Princeton" - } - }, - "evaluations": [ - { - "evaluation_id": "terminal-bench-2.0/mini-swe-agent__gpt-5-codex/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2025-11-03", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 41.3, - "uncertainty": { - "standard_error": { - "value": 2.8 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Mini-SWE-Agent\" -m \"GPT-5-Codex\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Mini-SWE-Agent\" -m \"GPT-5-Codex\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - }, - { - "evaluation_id": "terminal-bench-2.0/terminus-2__gpt-5-codex/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2025-10-31", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 43.4, - "uncertainty": { - "standard_error": { - "value": 2.9 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Terminus 2\" -m \"GPT-5-Codex\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Terminus 2\" -m \"GPT-5-Codex\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - }, - { - "evaluation_id": "terminal-bench-2.0/codex-cli__gpt-5-codex/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2025-11-04", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 44.3, - "uncertainty": { - "standard_error": { - "value": 2.7 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Codex CLI\" -m \"GPT-5-Codex\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Codex CLI\" -m \"GPT-5-Codex\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/openai_gpt-5-mini-2025-08-07-fc.json b/data/models/openai_gpt-5-mini-2025-08-07-fc.json deleted file mode 100644 index 6716ede093a240aa7137e48c76387a90602ca974..0000000000000000000000000000000000000000 --- a/data/models/openai_gpt-5-mini-2025-08-07-fc.json +++ /dev/null @@ -1,851 +0,0 @@ -{ - "model_info": { - "name": "GPT-5-mini-2025-08-07 (FC)", - "id": "openai/gpt-5-mini-2025-08-07-fc", - "developer": "OpenAI", - "additional_details": { - "raw_model_name": "GPT-5-mini-2025-08-07 (FC)", - "organization": "OpenAI", - "license": "Proprietary", - "mode": "FC", - "model_link": "https://openai.com/index/introducing-gpt-5/" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/openai/gpt-5-mini-2025-08-07-fc/1775236112.374312", - "retrieved_timestamp": "1775236112.374312", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 17.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 55.46 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 22.18 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 8.32 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 17.35 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 19.8 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 69.85 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 59.92 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 69.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 80.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 70.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 58.62 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 62.02 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 58.02 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 62.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 45.83 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 27.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 36.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 17.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 23.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 33.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 82.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 87.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 77.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 44.3 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 36.77 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 43.87 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 52.26 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 62.5 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 91.01 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/openai_gpt-5-mini-2025-08-07-prompt.json b/data/models/openai_gpt-5-mini-2025-08-07-prompt.json deleted file mode 100644 index f3abd1b450ab25dce0147673a1ea81de59ed422d..0000000000000000000000000000000000000000 --- a/data/models/openai_gpt-5-mini-2025-08-07-prompt.json +++ /dev/null @@ -1,905 +0,0 @@ -{ - "model_info": { - "name": "GPT-5-mini-2025-08-07 (Prompt)", - "id": "openai/gpt-5-mini-2025-08-07-prompt", - "developer": "OpenAI", - "additional_details": { - "raw_model_name": "GPT-5-mini-2025-08-07 (Prompt)", - "organization": "OpenAI", - "license": "Proprietary", - "mode": "Prompt", - "model_link": "https://openai.com/index/introducing-gpt-5/" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/openai/gpt-5-mini-2025-08-07-prompt/1775236112.406107", - "retrieved_timestamp": "1775236112.406107", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 77.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 27.83 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 82.74 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 8.89 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 11.08 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 19.72 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 68.04 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 59.17 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 72.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 71.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 69.0 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 62.55 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 69.77 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 61.16 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 75.0 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 37.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 5.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 5.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 5.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 4.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 7.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 8.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 11.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 6.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 29.25 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 19.35 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 29.68 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 38.71 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 93.75 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 55.71 - } - }, - { - "evaluation_result_id": "bfcl.format_sensitivity.max_delta::max_delta", - "evaluation_name": "bfcl.format_sensitivity.max_delta", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.format_sensitivity.max_delta", - "metric_name": "Format sensitivity max delta", - "metric_kind": "difference", - "metric_unit": "percentage_points", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Format Sensitivity Max Delta" - } - }, - "score_details": { - "score": 16.0 - } - }, - { - "evaluation_result_id": "bfcl.format_sensitivity.stddev::stddev", - "evaluation_name": "bfcl.format_sensitivity.stddev", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.format_sensitivity.stddev", - "metric_name": "Format sensitivity standard deviation", - "metric_kind": "difference", - "metric_unit": "percentage_points", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Format Sensitivity Standard Deviation" - } - }, - "score_details": { - "score": 3.78 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/openai_gpt-5-mini-2025-08-07.json b/data/models/openai_gpt-5-mini-2025-08-07.json deleted file mode 100644 index 280b0b7b3e161dc5f4c9aaf96c1714daa7d0ed78..0000000000000000000000000000000000000000 --- a/data/models/openai_gpt-5-mini-2025-08-07.json +++ /dev/null @@ -1,234 +0,0 @@ -{ - "model_info": { - "name": "GPT-5 mini 2025-08-07", - "id": "openai/gpt-5-mini-2025-08-07", - "developer": "OpenAI", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_capabilities/openai_gpt-5-mini-2025-08-07/1774096304.5056698", - "retrieved_timestamp": "1774096304.5056698", - "source_metadata": { - "source_name": "helm_capabilities", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_capabilities", - "evaluation_results": [ - { - "evaluation_name": "Mean score", - "source_data": { - "dataset_name": "helm_capabilities", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "The mean of the scores from all columns.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.819, - "details": { - "description": "", - "tab": "Accuracy", - "Mean score - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"28.206869066978612\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU-Pro", - "source_data": { - "dataset_name": "MMLU-Pro", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on MMLU-Pro", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.835, - "details": { - "description": "min=0.835, mean=0.835, max=0.835, sum=0.835 (1)", - "tab": "Accuracy", - "MMLU-Pro - Observed inference time (s)": "{\"description\": \"min=11.803, mean=11.803, max=11.803, sum=11.803 (1)\", \"tab\": \"Efficiency\", \"score\": \"11.802515007257462\"}", - "MMLU-Pro - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "MMLU-Pro - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - # prompt tokens": "{\"description\": \"min=248.569, mean=248.569, max=248.569, sum=248.569 (1)\", \"tab\": \"General information\", \"score\": \"248.569\"}", - "MMLU-Pro - # output tokens": "{\"description\": \"min=17.495, mean=17.495, max=17.495, sum=17.495 (1)\", \"tab\": \"General information\", \"score\": \"17.495\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"all\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.756, - "details": { - "description": "min=0.756, mean=0.756, max=0.756, sum=0.756 (1)", - "tab": "Accuracy", - "GPQA - Observed inference time (s)": "{\"description\": \"min=20.737, mean=20.737, max=20.737, sum=20.737 (1)\", \"tab\": \"Efficiency\", \"score\": \"20.737325443280653\"}", - "GPQA - # eval": "{\"description\": \"min=446, mean=446, max=446, sum=446 (1)\", \"tab\": \"General information\", \"score\": \"446.0\"}", - "GPQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - # prompt tokens": "{\"description\": \"min=268.152, mean=268.152, max=268.152, sum=268.152 (1)\", \"tab\": \"General information\", \"score\": \"268.15246636771303\"}", - "GPQA - # output tokens": "{\"description\": \"min=25.379, mean=25.379, max=25.379, sum=25.379 (1)\", \"tab\": \"General information\", \"score\": \"25.378923766816143\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"gpqa_main\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "IFEval Strict Acc on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.927, - "details": { - "description": "min=0.927, mean=0.927, max=0.927, sum=0.927 (1)", - "tab": "Accuracy", - "IFEval - Observed inference time (s)": "{\"description\": \"min=18.834, mean=18.834, max=18.834, sum=18.834 (1)\", \"tab\": \"Efficiency\", \"score\": \"18.83414089833963\"}", - "IFEval - # eval": "{\"description\": \"min=541, mean=541, max=541, sum=541 (1)\", \"tab\": \"General information\", \"score\": \"541.0\"}", - "IFEval - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - # prompt tokens": "{\"description\": \"min=45.671, mean=45.671, max=45.671, sum=45.671 (1)\", \"tab\": \"General information\", \"score\": \"45.67097966728281\"}", - "IFEval - # output tokens": "{\"description\": \"min=441.137, mean=441.137, max=441.137, sum=441.137 (1)\", \"tab\": \"General information\", \"score\": \"441.13678373382623\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WildBench", - "source_data": { - "dataset_name": "WildBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "WB Score on WildBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.855, - "details": { - "description": "min=0.855, mean=0.855, max=0.855, sum=0.855 (1)", - "tab": "Accuracy", - "WildBench - Observed inference time (s)": "{\"description\": \"min=33.854, mean=33.854, max=33.854, sum=33.854 (1)\", \"tab\": \"Efficiency\", \"score\": \"33.85394237089157\"}", - "WildBench - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "WildBench - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # prompt tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # output tokens": "{\"description\": \"min=1408.024, mean=1408.024, max=1408.024, sum=1408.024 (1)\", \"tab\": \"General information\", \"score\": \"1408.024\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"v2\"" - } - } - }, - { - "evaluation_name": "Omni-MATH", - "source_data": { - "dataset_name": "Omni-MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Acc on Omni-MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.722, - "details": { - "description": "min=0.722, mean=0.722, max=0.722, sum=0.722 (1)", - "tab": "Accuracy", - "Omni-MATH - Observed inference time (s)": "{\"description\": \"min=55.806, mean=55.806, max=55.806, sum=55.806 (1)\", \"tab\": \"Efficiency\", \"score\": \"55.806421615123746\"}", - "Omni-MATH - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "Omni-MATH - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - # prompt tokens": "{\"description\": \"min=109.623, mean=109.623, max=109.623, sum=109.623 (1)\", \"tab\": \"General information\", \"score\": \"109.623\"}", - "Omni-MATH - # output tokens": "{\"description\": \"min=362.654, mean=362.654, max=362.654, sum=362.654 (1)\", \"tab\": \"General information\", \"score\": \"362.654\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/openai_gpt-5-mini.json b/data/models/openai_gpt-5-mini.json deleted file mode 100644 index bf5b75048193c45b7a3e07d95031e43eab23eb2a..0000000000000000000000000000000000000000 --- a/data/models/openai_gpt-5-mini.json +++ /dev/null @@ -1,383 +0,0 @@ -{ - "model_info": { - "name": "GPT-5-Mini", - "id": "openai/gpt-5-mini", - "developer": "OpenAI", - "additional_details": { - "agent_name": "Mini-SWE-Agent", - "agent_organization": "Princeton" - } - }, - "evaluations": [ - { - "evaluation_id": "terminal-bench-2.0/mini-swe-agent__gpt-5-mini/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2025-11-03", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 22.2, - "uncertainty": { - "standard_error": { - "value": 2.6 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Mini-SWE-Agent\" -m \"GPT-5-Mini\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Mini-SWE-Agent\" -m \"GPT-5-Mini\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - }, - { - "evaluation_id": "terminal-bench-2.0/openhands__gpt-5-mini/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2025-11-02", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 29.2, - "uncertainty": { - "standard_error": { - "value": 2.8 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"OpenHands\" -m \"GPT-5-Mini\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"OpenHands\" -m \"GPT-5-Mini\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - }, - { - "evaluation_id": "terminal-bench-2.0/spoox-m__gpt-5-mini/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2025-12-24", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 34.8, - "uncertainty": { - "standard_error": { - "value": 2.7 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"spoox-m\" -m \"GPT-5-Mini\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"spoox-m\" -m \"GPT-5-Mini\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - }, - { - "evaluation_id": "terminal-bench-2.0/terminus-2__gpt-5-mini/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2025-10-31", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 24.0, - "uncertainty": { - "standard_error": { - "value": 2.5 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Terminus 2\" -m \"GPT-5-Mini\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Terminus 2\" -m \"GPT-5-Mini\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - }, - { - "evaluation_id": "terminal-bench-2.0/codex-cli__gpt-5-mini/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2025-11-04", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 31.9, - "uncertainty": { - "standard_error": { - "value": 3.0 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Codex CLI\" -m \"GPT-5-Mini\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Codex CLI\" -m \"GPT-5-Mini\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/openai_gpt-5-nano-2025-08-07-fc.json b/data/models/openai_gpt-5-nano-2025-08-07-fc.json deleted file mode 100644 index 98b656eead19d28204532e57447c93099b8af90c..0000000000000000000000000000000000000000 --- a/data/models/openai_gpt-5-nano-2025-08-07-fc.json +++ /dev/null @@ -1,851 +0,0 @@ -{ - "model_info": { - "name": "GPT-5-nano-2025-08-07 (FC)", - "id": "openai/gpt-5-nano-2025-08-07-fc", - "developer": "OpenAI", - "additional_details": { - "raw_model_name": "GPT-5-nano-2025-08-07 (FC)", - "organization": "OpenAI", - "license": "Proprietary", - "mode": "FC", - "model_link": "https://openai.com/index/introducing-gpt-5/" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/openai/gpt-5-nano-2025-08-07-fc/1775236112.377845", - "retrieved_timestamp": "1775236112.377845", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 24.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 51.45 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 8.79 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 10.36 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 10.37 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 23.56 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 68.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 57.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 64.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 79.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 71.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 59.44 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 58.91 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 59.83 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 50.0 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 54.17 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 34.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 44.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 23.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 32.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 38.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 72.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 74.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 71.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 24.73 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 18.06 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 27.1 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 29.03 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 75.0 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 89.1 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/openai_gpt-5-nano-2025-08-07-prompt.json b/data/models/openai_gpt-5-nano-2025-08-07-prompt.json deleted file mode 100644 index 4152e01f31275b1321677eb6a3dc4ab7fbe8c43c..0000000000000000000000000000000000000000 --- a/data/models/openai_gpt-5-nano-2025-08-07-prompt.json +++ /dev/null @@ -1,905 +0,0 @@ -{ - "model_info": { - "name": "GPT-5-nano-2025-08-07 (Prompt)", - "id": "openai/gpt-5-nano-2025-08-07-prompt", - "developer": "OpenAI", - "additional_details": { - "raw_model_name": "GPT-5-nano-2025-08-07 (Prompt)", - "organization": "OpenAI", - "license": "Proprietary", - "mode": "Prompt", - "model_link": "https://openai.com/index/introducing-gpt-5/" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/openai/gpt-5-nano-2025-08-07-prompt/1775236112.407434", - "retrieved_timestamp": "1775236112.407434", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 79.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 27.55 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 21.47 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 10.67 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 7.68 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 23.28 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 80.81 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 69.25 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 86.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 87.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 80.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 70.69 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 76.36 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 69.71 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 68.75 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 54.17 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 0.75 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 1.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 1.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 1.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 13.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 10.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 17.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 24.52 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 20.65 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 31.61 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 21.29 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 93.75 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 45.75 - } - }, - { - "evaluation_result_id": "bfcl.format_sensitivity.max_delta::max_delta", - "evaluation_name": "bfcl.format_sensitivity.max_delta", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.format_sensitivity.max_delta", - "metric_name": "Format sensitivity max delta", - "metric_kind": "difference", - "metric_unit": "percentage_points", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Format Sensitivity Max Delta" - } - }, - "score_details": { - "score": 8.5 - } - }, - { - "evaluation_result_id": "bfcl.format_sensitivity.stddev::stddev", - "evaluation_name": "bfcl.format_sensitivity.stddev", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.format_sensitivity.stddev", - "metric_name": "Format sensitivity standard deviation", - "metric_kind": "difference", - "metric_unit": "percentage_points", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Format Sensitivity Standard Deviation" - } - }, - "score_details": { - "score": 2.57 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/openai_gpt-5-nano-2025-08-07.json b/data/models/openai_gpt-5-nano-2025-08-07.json deleted file mode 100644 index d4becb1531aa0fbb2725a559bf794a28eee1dccd..0000000000000000000000000000000000000000 --- a/data/models/openai_gpt-5-nano-2025-08-07.json +++ /dev/null @@ -1,234 +0,0 @@ -{ - "model_info": { - "name": "GPT-5 nano 2025-08-07", - "id": "openai/gpt-5-nano-2025-08-07", - "developer": "OpenAI", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_capabilities/openai_gpt-5-nano-2025-08-07/1774096304.5056698", - "retrieved_timestamp": "1774096304.5056698", - "source_metadata": { - "source_name": "helm_capabilities", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_capabilities", - "evaluation_results": [ - { - "evaluation_name": "Mean score", - "source_data": { - "dataset_name": "helm_capabilities", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "The mean of the scores from all columns.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.748, - "details": { - "description": "", - "tab": "Accuracy", - "Mean score - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"48.213836350621065\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU-Pro", - "source_data": { - "dataset_name": "MMLU-Pro", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on MMLU-Pro", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.778, - "details": { - "description": "min=0.778, mean=0.778, max=0.778, sum=0.778 (1)", - "tab": "Accuracy", - "MMLU-Pro - Observed inference time (s)": "{\"description\": \"min=17.337, mean=17.337, max=17.337, sum=17.337 (1)\", \"tab\": \"Efficiency\", \"score\": \"17.336622306585312\"}", - "MMLU-Pro - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "MMLU-Pro - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - # prompt tokens": "{\"description\": \"min=248.569, mean=248.569, max=248.569, sum=248.569 (1)\", \"tab\": \"General information\", \"score\": \"248.569\"}", - "MMLU-Pro - # output tokens": "{\"description\": \"min=5.385, mean=5.385, max=5.385, sum=5.385 (1)\", \"tab\": \"General information\", \"score\": \"5.385\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"all\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.679, - "details": { - "description": "min=0.679, mean=0.679, max=0.679, sum=0.679 (1)", - "tab": "Accuracy", - "GPQA - Observed inference time (s)": "{\"description\": \"min=30.246, mean=30.246, max=30.246, sum=30.246 (1)\", \"tab\": \"Efficiency\", \"score\": \"30.2457077674267\"}", - "GPQA - # eval": "{\"description\": \"min=446, mean=446, max=446, sum=446 (1)\", \"tab\": \"General information\", \"score\": \"446.0\"}", - "GPQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - # prompt tokens": "{\"description\": \"min=268.152, mean=268.152, max=268.152, sum=268.152 (1)\", \"tab\": \"General information\", \"score\": \"268.15246636771303\"}", - "GPQA - # output tokens": "{\"description\": \"min=5.668, mean=5.668, max=5.668, sum=5.668 (1)\", \"tab\": \"General information\", \"score\": \"5.668161434977579\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"gpqa_main\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "IFEval Strict Acc on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.932, - "details": { - "description": "min=0.932, mean=0.932, max=0.932, sum=0.932 (1)", - "tab": "Accuracy", - "IFEval - Observed inference time (s)": "{\"description\": \"min=26.735, mean=26.735, max=26.735, sum=26.735 (1)\", \"tab\": \"Efficiency\", \"score\": \"26.734930773980075\"}", - "IFEval - # eval": "{\"description\": \"min=541, mean=541, max=541, sum=541 (1)\", \"tab\": \"General information\", \"score\": \"541.0\"}", - "IFEval - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - # prompt tokens": "{\"description\": \"min=45.671, mean=45.671, max=45.671, sum=45.671 (1)\", \"tab\": \"General information\", \"score\": \"45.67097966728281\"}", - "IFEval - # output tokens": "{\"description\": \"min=426.656, mean=426.656, max=426.656, sum=426.656 (1)\", \"tab\": \"General information\", \"score\": \"426.6561922365989\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WildBench", - "source_data": { - "dataset_name": "WildBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "WB Score on WildBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.806, - "details": { - "description": "min=0.806, mean=0.806, max=0.806, sum=0.806 (1)", - "tab": "Accuracy", - "WildBench - Observed inference time (s)": "{\"description\": \"min=47.56, mean=47.56, max=47.56, sum=47.56 (1)\", \"tab\": \"Efficiency\", \"score\": \"47.560468022584914\"}", - "WildBench - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "WildBench - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # prompt tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # output tokens": "{\"description\": \"min=1391.144, mean=1391.144, max=1391.144, sum=1391.144 (1)\", \"tab\": \"General information\", \"score\": \"1391.144\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"v2\"" - } - } - }, - { - "evaluation_name": "Omni-MATH", - "source_data": { - "dataset_name": "Omni-MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Acc on Omni-MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.547, - "details": { - "description": "min=0.547, mean=0.547, max=0.547, sum=0.547 (1)", - "tab": "Accuracy", - "Omni-MATH - Observed inference time (s)": "{\"description\": \"min=119.191, mean=119.191, max=119.191, sum=119.191 (1)\", \"tab\": \"Efficiency\", \"score\": \"119.19145288252831\"}", - "Omni-MATH - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "Omni-MATH - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - # prompt tokens": "{\"description\": \"min=109.623, mean=109.623, max=109.623, sum=109.623 (1)\", \"tab\": \"General information\", \"score\": \"109.623\"}", - "Omni-MATH - # output tokens": "{\"description\": \"min=222.15, mean=222.15, max=222.15, sum=222.15 (1)\", \"tab\": \"General information\", \"score\": \"222.15\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/openai_gpt-5-nano.json b/data/models/openai_gpt-5-nano.json deleted file mode 100644 index 2cce7d2cc1dadb61b79871b32cc225d175133c82..0000000000000000000000000000000000000000 --- a/data/models/openai_gpt-5-nano.json +++ /dev/null @@ -1,309 +0,0 @@ -{ - "model_info": { - "name": "GPT-5-Nano", - "id": "openai/gpt-5-nano", - "developer": "OpenAI", - "additional_details": { - "agent_name": "Codex CLI", - "agent_organization": "OpenAI" - } - }, - "evaluations": [ - { - "evaluation_id": "terminal-bench-2.0/codex-cli__gpt-5-nano/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2025-11-04", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 11.5, - "uncertainty": { - "standard_error": { - "value": 2.3 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Codex CLI\" -m \"GPT-5-Nano\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Codex CLI\" -m \"GPT-5-Nano\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - }, - { - "evaluation_id": "terminal-bench-2.0/terminus-2__gpt-5-nano/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2025-10-31", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 7.9, - "uncertainty": { - "standard_error": { - "value": 1.9 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Terminus 2\" -m \"GPT-5-Nano\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Terminus 2\" -m \"GPT-5-Nano\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - }, - { - "evaluation_id": "terminal-bench-2.0/openhands__gpt-5-nano/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2025-11-02", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 9.9, - "uncertainty": { - "standard_error": { - "value": 2.1 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"OpenHands\" -m \"GPT-5-Nano\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"OpenHands\" -m \"GPT-5-Nano\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - }, - { - "evaluation_id": "terminal-bench-2.0/mini-swe-agent__gpt-5-nano/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2025-11-03", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 7.0, - "uncertainty": { - "standard_error": { - "value": 1.9 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Mini-SWE-Agent\" -m \"GPT-5-Nano\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Mini-SWE-Agent\" -m \"GPT-5-Nano\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/openai_gpt-5.1-codex-max.json b/data/models/openai_gpt-5.1-codex-max.json deleted file mode 100644 index 382dba58468be8776a7d29450270eec48b1f057a..0000000000000000000000000000000000000000 --- a/data/models/openai_gpt-5.1-codex-max.json +++ /dev/null @@ -1,87 +0,0 @@ -{ - "model_info": { - "name": "GPT-5.1-Codex-Max", - "id": "openai/gpt-5.1-codex-max", - "developer": "OpenAI", - "additional_details": { - "agent_name": "Codex CLI", - "agent_organization": "OpenAI" - } - }, - "evaluations": [ - { - "evaluation_id": "terminal-bench-2.0/codex-cli__gpt-5.1-codex-max/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2025-11-24", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 60.4, - "uncertainty": { - "standard_error": { - "value": 2.7 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Codex CLI\" -m \"GPT-5.1-Codex-Max\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Codex CLI\" -m \"GPT-5.1-Codex-Max\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/openai_gpt-5.1-codex-mini.json b/data/models/openai_gpt-5.1-codex-mini.json deleted file mode 100644 index 02c7d43976b37254660337524b957e3d02915ce8..0000000000000000000000000000000000000000 --- a/data/models/openai_gpt-5.1-codex-mini.json +++ /dev/null @@ -1,87 +0,0 @@ -{ - "model_info": { - "name": "GPT-5.1-Codex-Mini", - "id": "openai/gpt-5.1-codex-mini", - "developer": "OpenAI", - "additional_details": { - "agent_name": "Crux", - "agent_organization": "Roam" - } - }, - "evaluations": [ - { - "evaluation_id": "terminal-bench-2.0/crux__gpt-5.1-codex-mini/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2025-11-17", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 43.1, - "uncertainty": { - "standard_error": { - "value": 3.0 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Crux\" -m \"GPT-5.1-Codex-Mini\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Crux\" -m \"GPT-5.1-Codex-Mini\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/openai_gpt-5.1-codex.json b/data/models/openai_gpt-5.1-codex.json deleted file mode 100644 index 5ac777453649e155d2cf4ca5c0edb4c1fb8d16a4..0000000000000000000000000000000000000000 --- a/data/models/openai_gpt-5.1-codex.json +++ /dev/null @@ -1,235 +0,0 @@ -{ - "model_info": { - "name": "GPT-5.1-Codex", - "id": "openai/gpt-5.1-codex", - "developer": "OpenAI", - "additional_details": { - "agent_name": "Terminus 2", - "agent_organization": "Terminal Bench" - } - }, - "evaluations": [ - { - "evaluation_id": "terminal-bench-2.0/terminus-2__gpt-5.1-codex/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2025-11-17", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 36.9, - "uncertainty": { - "standard_error": { - "value": 3.2 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Terminus 2\" -m \"GPT-5.1-Codex\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Terminus 2\" -m \"GPT-5.1-Codex\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - }, - { - "evaluation_id": "terminal-bench-2.0/crux__gpt-5.1-codex/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2025-11-16", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 57.8, - "uncertainty": { - "standard_error": { - "value": 2.9 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Crux\" -m \"GPT-5.1-Codex\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Crux\" -m \"GPT-5.1-Codex\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - }, - { - "evaluation_id": "terminal-bench-2.0/letta-code__gpt-5.1-codex/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2025-12-17", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 53.5, - "uncertainty": { - "standard_error": { - "value": 2.8 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Letta Code\" -m \"GPT-5.1-Codex\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Letta Code\" -m \"GPT-5.1-Codex\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/openai_gpt-5.1.json b/data/models/openai_gpt-5.1.json deleted file mode 100644 index 727b1f824a0867fdfe353a258781efa8571e79fb..0000000000000000000000000000000000000000 --- a/data/models/openai_gpt-5.1.json +++ /dev/null @@ -1,87 +0,0 @@ -{ - "model_info": { - "name": "GPT-5.1", - "id": "openai/gpt-5.1", - "developer": "OpenAI", - "additional_details": { - "agent_name": "Terminus 2", - "agent_organization": "Terminal Bench" - } - }, - "evaluations": [ - { - "evaluation_id": "terminal-bench-2.0/terminus-2__gpt-5.1/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2025-11-16", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 47.6, - "uncertainty": { - "standard_error": { - "value": 2.8 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Terminus 2\" -m \"GPT-5.1\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Terminus 2\" -m \"GPT-5.1\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/openai_gpt-5.2-2025-12-11.json b/data/models/openai_gpt-5.2-2025-12-11.json deleted file mode 100644 index 212cc488261cbe395c6e1b71e2c9812d03d5c153..0000000000000000000000000000000000000000 --- a/data/models/openai_gpt-5.2-2025-12-11.json +++ /dev/null @@ -1,2132 +0,0 @@ -{ - "model_info": { - "name": "gpt-5.2-2025-12-11", - "id": "openai/gpt-5.2-2025-12-11", - "developer": "OpenAI", - "additional_details": { - "agent_name": "Claude Code CLI", - "agent_framework": "claude_code" - } - }, - "evaluations": [ - { - "evaluation_id": "appworld/test_normal/claude-code-cli__openai_gpt-5.2-2025-12-11/1774263615.0201504", - "retrieved_timestamp": "1774263615.0201504", - "source_metadata": { - "source_name": "Exgentic Open Agent Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Exgentic", - "source_organization_url": "https://github.com/Exgentic", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "exgentic", - "version": "0.1.0" - }, - "benchmark": "appworld_test_normal", - "evaluation_results": [ - { - "evaluation_name": "appworld/test_normal", - "source_data": { - "dataset_name": "appworld/test_normal", - "source_type": "url", - "url": [ - "https://github.com/Exgentic/exgentic" - ] - }, - "metric_config": { - "evaluation_description": "AppWorld benchmark evaluation (test_normal subset)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0, - "uncertainty": { - "num_samples": 100 - }, - "details": { - "average_agent_cost": "0.0", - "total_run_cost": "0.0", - "average_steps": "0.0", - "percent_finished": "0.0" - } - }, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "Claude Code CLI", - "agent_framework": "claude_code" - } - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "Claude Code CLI", - "agent_framework": "claude_code" - } - } - } - } - }, - { - "evaluation_id": "appworld/test_normal/litellm-tool-calling__openai_gpt-5.2-2025-12-11/1774263615.0201504", - "retrieved_timestamp": "1774263615.0201504", - "source_metadata": { - "source_name": "Exgentic Open Agent Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Exgentic", - "source_organization_url": "https://github.com/Exgentic", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "exgentic", - "version": "0.1.0" - }, - "benchmark": "appworld_test_normal", - "evaluation_results": [ - { - "evaluation_name": "appworld/test_normal", - "source_data": { - "dataset_name": "appworld/test_normal", - "source_type": "url", - "url": [ - "https://github.com/Exgentic/exgentic" - ] - }, - "metric_config": { - "evaluation_description": "AppWorld benchmark evaluation (test_normal subset)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0, - "uncertainty": { - "num_samples": 100 - }, - "details": { - "average_agent_cost": "0.0", - "total_run_cost": "0.0", - "average_steps": "0.0", - "percent_finished": "0.0" - } - }, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "LiteLLM Tool Calling", - "agent_framework": "tool_calling" - } - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "LiteLLM Tool Calling", - "agent_framework": "tool_calling" - } - } - } - } - }, - { - "evaluation_id": "appworld/test_normal/smolagents-code__openai_gpt-5.2-2025-12-11/1774263615.0201504", - "retrieved_timestamp": "1774263615.0201504", - "source_metadata": { - "source_name": "Exgentic Open Agent Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Exgentic", - "source_organization_url": "https://github.com/Exgentic", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "exgentic", - "version": "0.1.0" - }, - "benchmark": "appworld_test_normal", - "evaluation_results": [ - { - "evaluation_name": "appworld/test_normal", - "source_data": { - "dataset_name": "appworld/test_normal", - "source_type": "url", - "url": [ - "https://github.com/Exgentic/exgentic" - ] - }, - "metric_config": { - "evaluation_description": "AppWorld benchmark evaluation (test_normal subset)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.071, - "uncertainty": { - "num_samples": 100 - }, - "details": { - "average_agent_cost": "0.55", - "total_run_cost": "55.03", - "average_steps": "51.59", - "percent_finished": "0.61" - } - }, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "SmolAgents Code", - "agent_framework": "smolagents_code" - } - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "SmolAgents Code", - "agent_framework": "smolagents_code" - } - } - } - } - }, - { - "evaluation_id": "appworld/test_normal/litellm-tool-calling-with-shortlisting__openai_gpt-5.2-2025-12-11/1774263615.0201504", - "retrieved_timestamp": "1774263615.0201504", - "source_metadata": { - "source_name": "Exgentic Open Agent Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Exgentic", - "source_organization_url": "https://github.com/Exgentic", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "exgentic", - "version": "0.1.0" - }, - "benchmark": "appworld_test_normal", - "evaluation_results": [ - { - "evaluation_name": "appworld/test_normal", - "source_data": { - "dataset_name": "appworld/test_normal", - "source_type": "url", - "url": [ - "https://github.com/Exgentic/exgentic" - ] - }, - "metric_config": { - "evaluation_description": "AppWorld benchmark evaluation (test_normal subset)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.22, - "uncertainty": { - "num_samples": 100 - }, - "details": { - "average_agent_cost": "0.36", - "total_run_cost": "36.37", - "average_steps": "10.05", - "percent_finished": "1.0" - } - }, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "LiteLLM Tool Calling with Shortlisting", - "agent_framework": "tool_calling_with_shortlisting" - } - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "LiteLLM Tool Calling with Shortlisting", - "agent_framework": "tool_calling_with_shortlisting" - } - } - } - } - }, - { - "evaluation_id": "appworld/test_normal/openai-solo__openai_gpt-5.2-2025-12-11/1774263615.0201504", - "retrieved_timestamp": "1774263615.0201504", - "source_metadata": { - "source_name": "Exgentic Open Agent Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Exgentic", - "source_organization_url": "https://github.com/Exgentic", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "exgentic", - "version": "0.1.0" - }, - "benchmark": "appworld_test_normal", - "evaluation_results": [ - { - "evaluation_name": "appworld/test_normal", - "source_data": { - "dataset_name": "appworld/test_normal", - "source_type": "url", - "url": [ - "https://github.com/Exgentic/exgentic" - ] - }, - "metric_config": { - "evaluation_description": "AppWorld benchmark evaluation (test_normal subset)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0, - "uncertainty": { - "num_samples": 100 - }, - "details": { - "average_agent_cost": "0.0", - "total_run_cost": "0.0", - "average_steps": "0.0", - "percent_finished": "0.0" - } - }, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "OpenAI Solo", - "agent_framework": "openai_solo" - } - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "OpenAI Solo", - "agent_framework": "openai_solo" - } - } - } - } - }, - { - "evaluation_id": "browsecompplus/litellm-tool-calling-with-shortlisting__openai_gpt-5.2-2025-12-11/1774263615.0201504", - "retrieved_timestamp": "1774263615.0201504", - "source_metadata": { - "source_name": "Exgentic Open Agent Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Exgentic", - "source_organization_url": "https://github.com/Exgentic", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "exgentic", - "version": "0.1.0" - }, - "benchmark": "browsecompplus", - "evaluation_results": [ - { - "evaluation_name": "browsecompplus", - "source_data": { - "dataset_name": "browsecompplus", - "source_type": "url", - "url": [ - "https://github.com/Exgentic/exgentic" - ] - }, - "metric_config": { - "evaluation_description": "BrowseCompPlus benchmark evaluation", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.46, - "uncertainty": { - "num_samples": 100 - }, - "details": { - "average_agent_cost": "0.3", - "total_run_cost": "29.78", - "average_steps": "8.14", - "percent_finished": "0.99" - } - }, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "LiteLLM Tool Calling with Shortlisting", - "agent_framework": "tool_calling_with_shortlisting" - } - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "LiteLLM Tool Calling with Shortlisting", - "agent_framework": "tool_calling_with_shortlisting" - } - } - } - } - }, - { - "evaluation_id": "browsecompplus/openai-solo__openai_gpt-5.2-2025-12-11/1774263615.0201504", - "retrieved_timestamp": "1774263615.0201504", - "source_metadata": { - "source_name": "Exgentic Open Agent Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Exgentic", - "source_organization_url": "https://github.com/Exgentic", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "exgentic", - "version": "0.1.0" - }, - "benchmark": "browsecompplus", - "evaluation_results": [ - { - "evaluation_name": "browsecompplus", - "source_data": { - "dataset_name": "browsecompplus", - "source_type": "url", - "url": [ - "https://github.com/Exgentic/exgentic" - ] - }, - "metric_config": { - "evaluation_description": "BrowseCompPlus benchmark evaluation", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.48, - "uncertainty": { - "num_samples": 100 - }, - "details": { - "average_agent_cost": "0.38", - "total_run_cost": "38.21", - "average_steps": "14.27", - "percent_finished": "1.0" - } - }, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "OpenAI Solo", - "agent_framework": "openai_solo" - } - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "OpenAI Solo", - "agent_framework": "openai_solo" - } - } - } - } - }, - { - "evaluation_id": "browsecompplus/litellm-tool-calling__openai_gpt-5.2-2025-12-11/1774263615.0201504", - "retrieved_timestamp": "1774263615.0201504", - "source_metadata": { - "source_name": "Exgentic Open Agent Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Exgentic", - "source_organization_url": "https://github.com/Exgentic", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "exgentic", - "version": "0.1.0" - }, - "benchmark": "browsecompplus", - "evaluation_results": [ - { - "evaluation_name": "browsecompplus", - "source_data": { - "dataset_name": "browsecompplus", - "source_type": "url", - "url": [ - "https://github.com/Exgentic/exgentic" - ] - }, - "metric_config": { - "evaluation_description": "BrowseCompPlus benchmark evaluation", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.46, - "uncertainty": { - "num_samples": 100 - }, - "details": { - "average_agent_cost": "0.3", - "total_run_cost": "29.78", - "average_steps": "8.14", - "percent_finished": "0.99" - } - }, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "LiteLLM Tool Calling", - "agent_framework": "tool_calling" - } - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "LiteLLM Tool Calling", - "agent_framework": "tool_calling" - } - } - } - } - }, - { - "evaluation_id": "browsecompplus/claude-code-cli__openai_gpt-5.2-2025-12-11/1774263615.0201504", - "retrieved_timestamp": "1774263615.0201504", - "source_metadata": { - "source_name": "Exgentic Open Agent Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Exgentic", - "source_organization_url": "https://github.com/Exgentic", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "exgentic", - "version": "0.1.0" - }, - "benchmark": "browsecompplus", - "evaluation_results": [ - { - "evaluation_name": "browsecompplus", - "source_data": { - "dataset_name": "browsecompplus", - "source_type": "url", - "url": [ - "https://github.com/Exgentic/exgentic" - ] - }, - "metric_config": { - "evaluation_description": "BrowseCompPlus benchmark evaluation", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.43, - "uncertainty": { - "num_samples": 100 - }, - "details": { - "average_agent_cost": "0.43", - "total_run_cost": "43.11", - "average_steps": "8.97", - "percent_finished": "1.0" - } - }, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "Claude Code CLI", - "agent_framework": "claude_code" - } - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "Claude Code CLI", - "agent_framework": "claude_code" - } - } - } - } - }, - { - "evaluation_id": "browsecompplus/smolagents-code__openai_gpt-5.2-2025-12-11/1774263615.0201504", - "retrieved_timestamp": "1774263615.0201504", - "source_metadata": { - "source_name": "Exgentic Open Agent Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Exgentic", - "source_organization_url": "https://github.com/Exgentic", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "exgentic", - "version": "0.1.0" - }, - "benchmark": "browsecompplus", - "evaluation_results": [ - { - "evaluation_name": "browsecompplus", - "source_data": { - "dataset_name": "browsecompplus", - "source_type": "url", - "url": [ - "https://github.com/Exgentic/exgentic" - ] - }, - "metric_config": { - "evaluation_description": "BrowseCompPlus benchmark evaluation", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.26, - "uncertainty": { - "num_samples": 100 - }, - "details": { - "average_agent_cost": "0.17", - "total_run_cost": "17.31", - "average_steps": "6.57", - "percent_finished": "0.99" - } - }, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "SmolAgents Code", - "agent_framework": "smolagents_code" - } - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "SmolAgents Code", - "agent_framework": "smolagents_code" - } - } - } - } - }, - { - "evaluation_id": "livecodebenchpro/gpt-5.2-2025-12-11/1770683238.099205", - "retrieved_timestamp": "1770683238.099205", - "source_metadata": { - "source_name": "Live Code Bench Pro", - "source_type": "documentation", - "source_organization_name": "New York University, Princeton University, University of California San Diego, University of Washington and Canyon Crest Academy", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "unknown", - "version": "unknown" - }, - "benchmark": "livecodebenchpro", - "evaluation_results": [ - { - "evaluation_name": "Hard Problems", - "source_data": { - "dataset_name": "Hard Problems", - "source_type": "url", - "url": [ - "https://webhook.cp-bench.orzzh.com/leaderboard/llm/difficulty?difficulty=hard&benchmark_mode=live" - ] - }, - "metric_config": { - "evaluation_description": "Pass@1 on Hard Problems", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1594 - } - }, - { - "evaluation_name": "Medium Problems", - "source_data": { - "dataset_name": "Medium Problems", - "source_type": "url", - "url": [ - "https://webhook.cp-bench.orzzh.com/leaderboard/llm/difficulty?difficulty=medium&benchmark_mode=live" - ] - }, - "metric_config": { - "evaluation_description": "Pass@1 on Medium Problems", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5211 - } - }, - { - "evaluation_name": "Easy Problems", - "source_data": { - "dataset_name": "Easy Problems", - "source_type": "url", - "url": [ - "https://webhook.cp-bench.orzzh.com/leaderboard/llm/difficulty?difficulty=easy&benchmark_mode=live" - ] - }, - "metric_config": { - "evaluation_description": "Pass@1 on Easy Problems", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9014 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "swe-bench/claude-code-cli__openai_gpt-5.2-2025-12-11/1774263615.0201504", - "retrieved_timestamp": "1774263615.0201504", - "source_metadata": { - "source_name": "Exgentic Open Agent Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Exgentic", - "source_organization_url": "https://github.com/Exgentic", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "exgentic", - "version": "0.1.0" - }, - "benchmark": "swe-bench", - "evaluation_results": [ - { - "evaluation_name": "swe-bench", - "source_data": { - "dataset_name": "swe-bench", - "source_type": "url", - "url": [ - "https://github.com/Exgentic/exgentic" - ] - }, - "metric_config": { - "evaluation_description": "SWE-bench benchmark evaluation", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.58, - "uncertainty": { - "num_samples": 100 - }, - "details": { - "average_agent_cost": "0.94", - "total_run_cost": "93.98", - "average_steps": "23.99", - "percent_finished": "1.0" - } - }, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "Claude Code CLI", - "agent_framework": "claude_code" - } - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "Claude Code CLI", - "agent_framework": "claude_code" - } - } - } - } - }, - { - "evaluation_id": "swe-bench/smolagents-code__openai_gpt-5.2-2025-12-11/1774263615.0201504", - "retrieved_timestamp": "1774263615.0201504", - "source_metadata": { - "source_name": "Exgentic Open Agent Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Exgentic", - "source_organization_url": "https://github.com/Exgentic", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "exgentic", - "version": "0.1.0" - }, - "benchmark": "swe-bench", - "evaluation_results": [ - { - "evaluation_name": "swe-bench", - "source_data": { - "dataset_name": "swe-bench", - "source_type": "url", - "url": [ - "https://github.com/Exgentic/exgentic" - ] - }, - "metric_config": { - "evaluation_description": "SWE-bench benchmark evaluation", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5253, - "uncertainty": { - "num_samples": 99 - }, - "details": { - "average_agent_cost": "0.45", - "total_run_cost": "44.58", - "average_steps": "19.98", - "percent_finished": "1.0" - } - }, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "SmolAgents Code", - "agent_framework": "smolagents_code" - } - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "SmolAgents Code", - "agent_framework": "smolagents_code" - } - } - } - } - }, - { - "evaluation_id": "swe-bench/litellm-tool-calling-with-shortlisting__openai_gpt-5.2-2025-12-11/1774263615.0201504", - "retrieved_timestamp": "1774263615.0201504", - "source_metadata": { - "source_name": "Exgentic Open Agent Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Exgentic", - "source_organization_url": "https://github.com/Exgentic", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "exgentic", - "version": "0.1.0" - }, - "benchmark": "swe-bench", - "evaluation_results": [ - { - "evaluation_name": "swe-bench", - "source_data": { - "dataset_name": "swe-bench", - "source_type": "url", - "url": [ - "https://github.com/Exgentic/exgentic" - ] - }, - "metric_config": { - "evaluation_description": "SWE-bench benchmark evaluation", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.57, - "uncertainty": { - "num_samples": 100 - }, - "details": { - "average_agent_cost": "0.25", - "total_run_cost": "24.76", - "average_steps": "20.47", - "percent_finished": "1.0" - } - }, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "LiteLLM Tool Calling with Shortlisting", - "agent_framework": "tool_calling_with_shortlisting" - } - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "LiteLLM Tool Calling with Shortlisting", - "agent_framework": "tool_calling_with_shortlisting" - } - } - } - } - }, - { - "evaluation_id": "swe-bench/openai-solo__openai_gpt-5.2-2025-12-11/1774263615.0201504", - "retrieved_timestamp": "1774263615.0201504", - "source_metadata": { - "source_name": "Exgentic Open Agent Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Exgentic", - "source_organization_url": "https://github.com/Exgentic", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "exgentic", - "version": "0.1.0" - }, - "benchmark": "swe-bench", - "evaluation_results": [ - { - "evaluation_name": "swe-bench", - "source_data": { - "dataset_name": "swe-bench", - "source_type": "url", - "url": [ - "https://github.com/Exgentic/exgentic" - ] - }, - "metric_config": { - "evaluation_description": "SWE-bench benchmark evaluation", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5455, - "uncertainty": { - "num_samples": 99 - }, - "details": { - "average_agent_cost": "0.26", - "total_run_cost": "25.64", - "average_steps": "20.44", - "percent_finished": "1.0" - } - }, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "OpenAI Solo", - "agent_framework": "openai_solo" - } - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "OpenAI Solo", - "agent_framework": "openai_solo" - } - } - } - } - }, - { - "evaluation_id": "swe-bench/litellm-tool-calling__openai_gpt-5.2-2025-12-11/1774263615.0201504", - "retrieved_timestamp": "1774263615.0201504", - "source_metadata": { - "source_name": "Exgentic Open Agent Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Exgentic", - "source_organization_url": "https://github.com/Exgentic", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "exgentic", - "version": "0.1.0" - }, - "benchmark": "swe-bench", - "evaluation_results": [ - { - "evaluation_name": "swe-bench", - "source_data": { - "dataset_name": "swe-bench", - "source_type": "url", - "url": [ - "https://github.com/Exgentic/exgentic" - ] - }, - "metric_config": { - "evaluation_description": "SWE-bench benchmark evaluation", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.57, - "uncertainty": { - "num_samples": 100 - }, - "details": { - "average_agent_cost": "0.25", - "total_run_cost": "24.76", - "average_steps": "20.47", - "percent_finished": "1.0" - } - }, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "LiteLLM Tool Calling", - "agent_framework": "tool_calling" - } - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "LiteLLM Tool Calling", - "agent_framework": "tool_calling" - } - } - } - } - }, - { - "evaluation_id": "tau-bench-2/airline/litellm-tool-calling__openai_gpt-5.2-2025-12-11/1774263615.0201504", - "retrieved_timestamp": "1774263615.0201504", - "source_metadata": { - "source_name": "Exgentic Open Agent Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Exgentic", - "source_organization_url": "https://github.com/Exgentic", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "exgentic", - "version": "0.1.0" - }, - "benchmark": "tau-bench-2_airline", - "evaluation_results": [ - { - "evaluation_name": "tau-bench-2/airline", - "source_data": { - "dataset_name": "tau-bench-2/airline", - "source_type": "url", - "url": [ - "https://github.com/Exgentic/exgentic" - ] - }, - "metric_config": { - "evaluation_description": "Tau Bench 2 benchmark evaluation (airline subset)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.54, - "uncertainty": { - "num_samples": 50 - }, - "details": { - "average_agent_cost": "0.13", - "total_run_cost": "6.96", - "average_steps": "11.22", - "percent_finished": "1.0" - } - }, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "LiteLLM Tool Calling", - "agent_framework": "tool_calling" - } - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "LiteLLM Tool Calling", - "agent_framework": "tool_calling" - } - } - } - } - }, - { - "evaluation_id": "tau-bench-2/airline/openai-solo__openai_gpt-5.2-2025-12-11/1774263615.0201504", - "retrieved_timestamp": "1774263615.0201504", - "source_metadata": { - "source_name": "Exgentic Open Agent Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Exgentic", - "source_organization_url": "https://github.com/Exgentic", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "exgentic", - "version": "0.1.0" - }, - "benchmark": "tau-bench-2_airline", - "evaluation_results": [ - { - "evaluation_name": "tau-bench-2/airline", - "source_data": { - "dataset_name": "tau-bench-2/airline", - "source_type": "url", - "url": [ - "https://github.com/Exgentic/exgentic" - ] - }, - "metric_config": { - "evaluation_description": "Tau Bench 2 benchmark evaluation (airline subset)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5, - "uncertainty": { - "num_samples": 50 - }, - "details": { - "average_agent_cost": "0.11", - "total_run_cost": "5.77", - "average_steps": "11.4", - "percent_finished": "1.0" - } - }, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "OpenAI Solo", - "agent_framework": "openai_solo" - } - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "OpenAI Solo", - "agent_framework": "openai_solo" - } - } - } - } - }, - { - "evaluation_id": "tau-bench-2/airline/claude-code-cli__openai_gpt-5.2-2025-12-11/1774263615.0201504", - "retrieved_timestamp": "1774263615.0201504", - "source_metadata": { - "source_name": "Exgentic Open Agent Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Exgentic", - "source_organization_url": "https://github.com/Exgentic", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "exgentic", - "version": "0.1.0" - }, - "benchmark": "tau-bench-2_airline", - "evaluation_results": [ - { - "evaluation_name": "tau-bench-2/airline", - "source_data": { - "dataset_name": "tau-bench-2/airline", - "source_type": "url", - "url": [ - "https://github.com/Exgentic/exgentic" - ] - }, - "metric_config": { - "evaluation_description": "Tau Bench 2 benchmark evaluation (airline subset)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.48, - "uncertainty": { - "num_samples": 50 - }, - "details": { - "average_agent_cost": "0.21", - "total_run_cost": "11.23", - "average_steps": "10.18", - "percent_finished": "1.0" - } - }, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "Claude Code CLI", - "agent_framework": "claude_code" - } - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "Claude Code CLI", - "agent_framework": "claude_code" - } - } - } - } - }, - { - "evaluation_id": "tau-bench-2/airline/smolagents-code__openai_gpt-5.2-2025-12-11/1774263615.0201504", - "retrieved_timestamp": "1774263615.0201504", - "source_metadata": { - "source_name": "Exgentic Open Agent Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Exgentic", - "source_organization_url": "https://github.com/Exgentic", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "exgentic", - "version": "0.1.0" - }, - "benchmark": "tau-bench-2_airline", - "evaluation_results": [ - { - "evaluation_name": "tau-bench-2/airline", - "source_data": { - "dataset_name": "tau-bench-2/airline", - "source_type": "url", - "url": [ - "https://github.com/Exgentic/exgentic" - ] - }, - "metric_config": { - "evaluation_description": "Tau Bench 2 benchmark evaluation (airline subset)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6, - "uncertainty": { - "num_samples": 50 - }, - "details": { - "average_agent_cost": "0.29", - "total_run_cost": "15.28", - "average_steps": "10.68", - "percent_finished": "1.0" - } - }, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "SmolAgents Code", - "agent_framework": "smolagents_code" - } - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "SmolAgents Code", - "agent_framework": "smolagents_code" - } - } - } - } - }, - { - "evaluation_id": "tau-bench-2/retail/openai-solo__openai_gpt-5.2-2025-12-11/1774263615.0201504", - "retrieved_timestamp": "1774263615.0201504", - "source_metadata": { - "source_name": "Exgentic Open Agent Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Exgentic", - "source_organization_url": "https://github.com/Exgentic", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "exgentic", - "version": "0.1.0" - }, - "benchmark": "tau-bench-2_retail", - "evaluation_results": [ - { - "evaluation_name": "tau-bench-2/retail", - "source_data": { - "dataset_name": "tau-bench-2/retail", - "source_type": "url", - "url": [ - "https://github.com/Exgentic/exgentic" - ] - }, - "metric_config": { - "evaluation_description": "Tau Bench 2 benchmark evaluation (retail subset)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5354, - "uncertainty": { - "num_samples": 100 - }, - "details": { - "average_agent_cost": "0.11", - "total_run_cost": "11.54", - "average_steps": "9.55", - "percent_finished": "0.99" - } - }, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "OpenAI Solo", - "agent_framework": "openai_solo" - } - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "OpenAI Solo", - "agent_framework": "openai_solo" - } - } - } - } - }, - { - "evaluation_id": "tau-bench-2/airline/litellm-tool-calling-with-shortlisting__openai_gpt-5.2-2025-12-11/1774263615.0201504", - "retrieved_timestamp": "1774263615.0201504", - "source_metadata": { - "source_name": "Exgentic Open Agent Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Exgentic", - "source_organization_url": "https://github.com/Exgentic", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "exgentic", - "version": "0.1.0" - }, - "benchmark": "tau-bench-2_airline", - "evaluation_results": [ - { - "evaluation_name": "tau-bench-2/airline", - "source_data": { - "dataset_name": "tau-bench-2/airline", - "source_type": "url", - "url": [ - "https://github.com/Exgentic/exgentic" - ] - }, - "metric_config": { - "evaluation_description": "Tau Bench 2 benchmark evaluation (airline subset)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.54, - "uncertainty": { - "num_samples": 50 - }, - "details": { - "average_agent_cost": "0.13", - "total_run_cost": "6.96", - "average_steps": "11.22", - "percent_finished": "1.0" - } - }, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "LiteLLM Tool Calling with Shortlisting", - "agent_framework": "tool_calling_with_shortlisting" - } - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "LiteLLM Tool Calling with Shortlisting", - "agent_framework": "tool_calling_with_shortlisting" - } - } - } - } - }, - { - "evaluation_id": "tau-bench-2/retail/litellm-tool-calling-with-shortlisting__openai_gpt-5.2-2025-12-11/1774263615.0201504", - "retrieved_timestamp": "1774263615.0201504", - "source_metadata": { - "source_name": "Exgentic Open Agent Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Exgentic", - "source_organization_url": "https://github.com/Exgentic", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "exgentic", - "version": "0.1.0" - }, - "benchmark": "tau-bench-2_retail", - "evaluation_results": [ - { - "evaluation_name": "tau-bench-2/retail", - "source_data": { - "dataset_name": "tau-bench-2/retail", - "source_type": "url", - "url": [ - "https://github.com/Exgentic/exgentic" - ] - }, - "metric_config": { - "evaluation_description": "Tau Bench 2 benchmark evaluation (retail subset)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.73, - "uncertainty": { - "num_samples": 100 - }, - "details": { - "average_agent_cost": "0.11", - "total_run_cost": "12.27", - "average_steps": "10.33", - "percent_finished": "1.0" - } - }, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "LiteLLM Tool Calling with Shortlisting", - "agent_framework": "tool_calling_with_shortlisting" - } - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "LiteLLM Tool Calling with Shortlisting", - "agent_framework": "tool_calling_with_shortlisting" - } - } - } - } - }, - { - "evaluation_id": "tau-bench-2/retail/claude-code-cli__openai_gpt-5.2-2025-12-11/1774263615.0201504", - "retrieved_timestamp": "1774263615.0201504", - "source_metadata": { - "source_name": "Exgentic Open Agent Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Exgentic", - "source_organization_url": "https://github.com/Exgentic", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "exgentic", - "version": "0.1.0" - }, - "benchmark": "tau-bench-2_retail", - "evaluation_results": [ - { - "evaluation_name": "tau-bench-2/retail", - "source_data": { - "dataset_name": "tau-bench-2/retail", - "source_type": "url", - "url": [ - "https://github.com/Exgentic/exgentic" - ] - }, - "metric_config": { - "evaluation_description": "Tau Bench 2 benchmark evaluation (retail subset)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.51, - "uncertainty": { - "num_samples": 100 - }, - "details": { - "average_agent_cost": "0.12", - "total_run_cost": "12.63", - "average_steps": "9.92", - "percent_finished": "0.98" - } - }, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "Claude Code CLI", - "agent_framework": "claude_code" - } - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "Claude Code CLI", - "agent_framework": "claude_code" - } - } - } - } - }, - { - "evaluation_id": "tau-bench-2/retail/litellm-tool-calling__openai_gpt-5.2-2025-12-11/1774263615.0201504", - "retrieved_timestamp": "1774263615.0201504", - "source_metadata": { - "source_name": "Exgentic Open Agent Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Exgentic", - "source_organization_url": "https://github.com/Exgentic", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "exgentic", - "version": "0.1.0" - }, - "benchmark": "tau-bench-2_retail", - "evaluation_results": [ - { - "evaluation_name": "tau-bench-2/retail", - "source_data": { - "dataset_name": "tau-bench-2/retail", - "source_type": "url", - "url": [ - "https://github.com/Exgentic/exgentic" - ] - }, - "metric_config": { - "evaluation_description": "Tau Bench 2 benchmark evaluation (retail subset)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.73, - "uncertainty": { - "num_samples": 100 - }, - "details": { - "average_agent_cost": "0.11", - "total_run_cost": "12.27", - "average_steps": "10.33", - "percent_finished": "1.0" - } - }, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "LiteLLM Tool Calling", - "agent_framework": "tool_calling" - } - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "LiteLLM Tool Calling", - "agent_framework": "tool_calling" - } - } - } - } - }, - { - "evaluation_id": "tau-bench-2/retail/smolagents-code__openai_gpt-5.2-2025-12-11/1774263615.0201504", - "retrieved_timestamp": "1774263615.0201504", - "source_metadata": { - "source_name": "Exgentic Open Agent Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Exgentic", - "source_organization_url": "https://github.com/Exgentic", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "exgentic", - "version": "0.1.0" - }, - "benchmark": "tau-bench-2_retail", - "evaluation_results": [ - { - "evaluation_name": "tau-bench-2/retail", - "source_data": { - "dataset_name": "tau-bench-2/retail", - "source_type": "url", - "url": [ - "https://github.com/Exgentic/exgentic" - ] - }, - "metric_config": { - "evaluation_description": "Tau Bench 2 benchmark evaluation (retail subset)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.68, - "uncertainty": { - "num_samples": 100 - }, - "details": { - "average_agent_cost": "0.25", - "total_run_cost": "26.27", - "average_steps": "11.08", - "percent_finished": "1.0" - } - }, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "SmolAgents Code", - "agent_framework": "smolagents_code" - } - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "SmolAgents Code", - "agent_framework": "smolagents_code" - } - } - } - } - }, - { - "evaluation_id": "tau-bench-2/telecom/litellm-tool-calling-with-shortlisting__openai_gpt-5.2-2025-12-11/1774263615.0201504", - "retrieved_timestamp": "1774263615.0201504", - "source_metadata": { - "source_name": "Exgentic Open Agent Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Exgentic", - "source_organization_url": "https://github.com/Exgentic", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "exgentic", - "version": "0.1.0" - }, - "benchmark": "tau-bench-2_telecom", - "evaluation_results": [ - { - "evaluation_name": "tau-bench-2/telecom", - "source_data": { - "dataset_name": "tau-bench-2/telecom", - "source_type": "url", - "url": [ - "https://github.com/Exgentic/exgentic" - ] - }, - "metric_config": { - "evaluation_description": "Tau Bench 2 benchmark evaluation (telecom subset)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5354, - "uncertainty": { - "num_samples": 100 - }, - "details": { - "average_agent_cost": "0.14", - "total_run_cost": "19.92", - "average_steps": "10.18", - "percent_finished": "0.99" - } - }, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "LiteLLM Tool Calling with Shortlisting", - "agent_framework": "tool_calling_with_shortlisting" - } - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "LiteLLM Tool Calling with Shortlisting", - "agent_framework": "tool_calling_with_shortlisting" - } - } - } - } - }, - { - "evaluation_id": "tau-bench-2/telecom/openai-solo__openai_gpt-5.2-2025-12-11/1774263615.0201504", - "retrieved_timestamp": "1774263615.0201504", - "source_metadata": { - "source_name": "Exgentic Open Agent Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Exgentic", - "source_organization_url": "https://github.com/Exgentic", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "exgentic", - "version": "0.1.0" - }, - "benchmark": "tau-bench-2_telecom", - "evaluation_results": [ - { - "evaluation_name": "tau-bench-2/telecom", - "source_data": { - "dataset_name": "tau-bench-2/telecom", - "source_type": "url", - "url": [ - "https://github.com/Exgentic/exgentic" - ] - }, - "metric_config": { - "evaluation_description": "Tau Bench 2 benchmark evaluation (telecom subset)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.53, - "uncertainty": { - "num_samples": 100 - }, - "details": { - "average_agent_cost": "0.15", - "total_run_cost": "18.88", - "average_steps": "9.92", - "percent_finished": "1.0" - } - }, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "OpenAI Solo", - "agent_framework": "openai_solo" - } - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "OpenAI Solo", - "agent_framework": "openai_solo" - } - } - } - } - }, - { - "evaluation_id": "tau-bench-2/telecom/claude-code-cli__openai_gpt-5.2-2025-12-11/1774263615.0201504", - "retrieved_timestamp": "1774263615.0201504", - "source_metadata": { - "source_name": "Exgentic Open Agent Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Exgentic", - "source_organization_url": "https://github.com/Exgentic", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "exgentic", - "version": "0.1.0" - }, - "benchmark": "tau-bench-2_telecom", - "evaluation_results": [ - { - "evaluation_name": "tau-bench-2/telecom", - "source_data": { - "dataset_name": "tau-bench-2/telecom", - "source_type": "url", - "url": [ - "https://github.com/Exgentic/exgentic" - ] - }, - "metric_config": { - "evaluation_description": "Tau Bench 2 benchmark evaluation (telecom subset)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.55, - "uncertainty": { - "num_samples": 100 - }, - "details": { - "average_agent_cost": "0.1", - "total_run_cost": "15.15", - "average_steps": "9.36", - "percent_finished": "1.0" - } - }, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "Claude Code CLI", - "agent_framework": "claude_code" - } - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "Claude Code CLI", - "agent_framework": "claude_code" - } - } - } - } - }, - { - "evaluation_id": "tau-bench-2/telecom/smolagents-code__openai_gpt-5.2-2025-12-11/1774263615.0201504", - "retrieved_timestamp": "1774263615.0201504", - "source_metadata": { - "source_name": "Exgentic Open Agent Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Exgentic", - "source_organization_url": "https://github.com/Exgentic", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "exgentic", - "version": "0.1.0" - }, - "benchmark": "tau-bench-2_telecom", - "evaluation_results": [ - { - "evaluation_name": "tau-bench-2/telecom", - "source_data": { - "dataset_name": "tau-bench-2/telecom", - "source_type": "url", - "url": [ - "https://github.com/Exgentic/exgentic" - ] - }, - "metric_config": { - "evaluation_description": "Tau Bench 2 benchmark evaluation (telecom subset)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.71, - "uncertainty": { - "num_samples": 100 - }, - "details": { - "average_agent_cost": "0.3", - "total_run_cost": "35.31", - "average_steps": "10.11", - "percent_finished": "1.0" - } - }, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "SmolAgents Code", - "agent_framework": "smolagents_code" - } - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "SmolAgents Code", - "agent_framework": "smolagents_code" - } - } - } - } - }, - { - "evaluation_id": "tau-bench-2/telecom/litellm-tool-calling__openai_gpt-5.2-2025-12-11/1774263615.0201504", - "retrieved_timestamp": "1774263615.0201504", - "source_metadata": { - "source_name": "Exgentic Open Agent Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Exgentic", - "source_organization_url": "https://github.com/Exgentic", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "exgentic", - "version": "0.1.0" - }, - "benchmark": "tau-bench-2_telecom", - "evaluation_results": [ - { - "evaluation_name": "tau-bench-2/telecom", - "source_data": { - "dataset_name": "tau-bench-2/telecom", - "source_type": "url", - "url": [ - "https://github.com/Exgentic/exgentic" - ] - }, - "metric_config": { - "evaluation_description": "Tau Bench 2 benchmark evaluation (telecom subset)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5354, - "uncertainty": { - "num_samples": 100 - }, - "details": { - "average_agent_cost": "0.14", - "total_run_cost": "19.92", - "average_steps": "10.18", - "percent_finished": "0.99" - } - }, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "LiteLLM Tool Calling", - "agent_framework": "tool_calling" - } - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "agentic_eval_config": { - "additional_details": { - "agent_name": "LiteLLM Tool Calling", - "agent_framework": "tool_calling" - } - } - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/openai_gpt-5.2-codex.json b/data/models/openai_gpt-5.2-codex.json deleted file mode 100644 index 1ab01ba79ebd94c20f1de0fc9d723cffecd92ce3..0000000000000000000000000000000000000000 --- a/data/models/openai_gpt-5.2-codex.json +++ /dev/null @@ -1,87 +0,0 @@ -{ - "model_info": { - "name": "GPT-5.2-Codex", - "id": "openai/gpt-5.2-codex", - "developer": "OpenAI", - "additional_details": { - "agent_name": "Deep Agents", - "agent_organization": "LangChain" - } - }, - "evaluations": [ - { - "evaluation_id": "terminal-bench-2.0/deep-agents__gpt-5.2-codex/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2026-02-12", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 66.5, - "uncertainty": { - "standard_error": { - "value": 3.1 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Deep Agents\" -m \"GPT-5.2-Codex\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Deep Agents\" -m \"GPT-5.2-Codex\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/openai_gpt-5.2.json b/data/models/openai_gpt-5.2.json deleted file mode 100644 index a0048de5ebdac86e398168d47320fc4011223c10..0000000000000000000000000000000000000000 --- a/data/models/openai_gpt-5.2.json +++ /dev/null @@ -1,303 +0,0 @@ -{ - "model_info": { - "name": "GPT-5.2", - "id": "openai/gpt-5.2", - "developer": "OpenAI", - "additional_details": { - "agent_name": "Terminus 2", - "agent_organization": "Terminal Bench" - } - }, - "evaluations": [ - { - "evaluation_id": "terminal-bench-2.0/terminus-2__gpt-5.2/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2025-12-12", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 54.0, - "uncertainty": { - "standard_error": { - "value": 2.9 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Terminus 2\" -m \"GPT-5.2\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Terminus 2\" -m \"GPT-5.2\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - }, - { - "evaluation_id": "terminal-bench-2.0/mux__gpt-5.2/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2026-01-17", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 60.7 - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Mux\" -m \"GPT-5.2\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Mux\" -m \"GPT-5.2\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - }, - { - "evaluation_id": "terminal-bench-2.0/codex-cli__gpt-5.2/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2025-12-18", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 62.9, - "uncertainty": { - "standard_error": { - "value": 3.0 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Codex CLI\" -m \"GPT-5.2\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Codex CLI\" -m \"GPT-5.2\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - }, - { - "evaluation_id": "terminal-bench-2.0/droid__gpt-5.2/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2025-12-24", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 64.9, - "uncertainty": { - "standard_error": { - "value": 2.8 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Droid\" -m \"GPT-5.2\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Droid\" -m \"GPT-5.2\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/openai_gpt-5.3-codex.json b/data/models/openai_gpt-5.3-codex.json deleted file mode 100644 index 93ecc7e098a018f9ada390e8bc00022ff7f1d61a..0000000000000000000000000000000000000000 --- a/data/models/openai_gpt-5.3-codex.json +++ /dev/null @@ -1,383 +0,0 @@ -{ - "model_info": { - "name": "GPT-5.3-Codex", - "id": "openai/gpt-5.3-codex", - "developer": "OpenAI", - "additional_details": { - "agent_name": "CodeBrain-1", - "agent_organization": "Feeling AI" - } - }, - "evaluations": [ - { - "evaluation_id": "terminal-bench-2.0/codebrain-1__gpt-5.3-codex/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2026-02-10", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 70.3, - "uncertainty": { - "standard_error": { - "value": 2.6 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"CodeBrain-1\" -m \"GPT-5.3-Codex\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"CodeBrain-1\" -m \"GPT-5.3-Codex\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - }, - { - "evaluation_id": "terminal-bench-2.0/terminus-2__gpt-5.3-codex/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2026-02-05", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 64.7, - "uncertainty": { - "standard_error": { - "value": 2.7 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Terminus 2\" -m \"GPT-5.3-Codex\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Terminus 2\" -m \"GPT-5.3-Codex\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - }, - { - "evaluation_id": "terminal-bench-2.0/droid__gpt-5.3-codex/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2026-02-24", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 77.3, - "uncertainty": { - "standard_error": { - "value": 2.2 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Droid\" -m \"GPT-5.3-Codex\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Droid\" -m \"GPT-5.3-Codex\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - }, - { - "evaluation_id": "terminal-bench-2.0/simple-codex__gpt-5.3-codex/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2026-02-06", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 75.1, - "uncertainty": { - "standard_error": { - "value": 2.4 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Simple Codex\" -m \"GPT-5.3-Codex\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Simple Codex\" -m \"GPT-5.3-Codex\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - }, - { - "evaluation_id": "terminal-bench-2.0/mux__gpt-5.3-codex/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2026-03-06", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 74.6, - "uncertainty": { - "standard_error": { - "value": 2.5 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Mux\" -m \"GPT-5.3-Codex\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Mux\" -m \"GPT-5.3-Codex\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/openai_gpt-5.json b/data/models/openai_gpt-5.json deleted file mode 100644 index 611054bf3713c00d844305134faf506aaeb168ae..0000000000000000000000000000000000000000 --- a/data/models/openai_gpt-5.json +++ /dev/null @@ -1,309 +0,0 @@ -{ - "model_info": { - "name": "GPT-5", - "id": "openai/gpt-5", - "developer": "OpenAI", - "additional_details": { - "agent_name": "Terminus 2", - "agent_organization": "Terminal Bench" - } - }, - "evaluations": [ - { - "evaluation_id": "terminal-bench-2.0/terminus-2__gpt-5/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2025-10-31", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 35.2, - "uncertainty": { - "standard_error": { - "value": 3.1 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Terminus 2\" -m \"GPT-5\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Terminus 2\" -m \"GPT-5\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - }, - { - "evaluation_id": "terminal-bench-2.0/openhands__gpt-5/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2025-11-02", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 43.8, - "uncertainty": { - "standard_error": { - "value": 3.0 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"OpenHands\" -m \"GPT-5\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"OpenHands\" -m \"GPT-5\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - }, - { - "evaluation_id": "terminal-bench-2.0/mini-swe-agent__gpt-5/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2025-11-03", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 33.9, - "uncertainty": { - "standard_error": { - "value": 2.9 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Mini-SWE-Agent\" -m \"GPT-5\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Mini-SWE-Agent\" -m \"GPT-5\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - }, - { - "evaluation_id": "terminal-bench-2.0/codex-cli__gpt-5/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2025-11-04", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 49.6, - "uncertainty": { - "standard_error": { - "value": 2.9 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Codex CLI\" -m \"GPT-5\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Codex CLI\" -m \"GPT-5\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/openai_gpt-j-6b.json b/data/models/openai_gpt-j-6b.json deleted file mode 100644 index a324aaa1df38c08c04cb7a46dd7d7409c64e30e0..0000000000000000000000000000000000000000 --- a/data/models/openai_gpt-j-6b.json +++ /dev/null @@ -1,674 +0,0 @@ -{ - "model_info": { - "name": "GPT-J 6B", - "id": "openai/GPT-J-6B", - "developer": "OpenAI", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_classic/openai_GPT-J-6B/1774096308.339228", - "retrieved_timestamp": "1774096308.339228", - "source_metadata": { - "source_name": "helm_classic", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_classic", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_classic", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperform on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.273, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Calibration": "{\"description\": \"\", \"tab\": \"Calibration\", \"score\": \"0.4640964584689531\"}", - "Mean win rate - Robustness": "{\"description\": \"\", \"tab\": \"Robustness\", \"score\": \"0.29051104623963353\"}", - "Mean win rate - Fairness": "{\"description\": \"\", \"tab\": \"Fairness\", \"score\": \"0.2899930436637889\"}", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.6008771929824561\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}", - "Mean win rate - Bias": "{\"description\": \"\", \"tab\": \"Bias\", \"score\": \"0.4572430192172563\"}", - "Mean win rate - Toxicity": "{\"description\": \"\", \"tab\": \"Toxicity\", \"score\": \"0.24521373688040354\"}", - "Mean win rate - Summarization metrics": "{\"description\": \"\", \"tab\": \"Summarization metrics\", \"score\": \"0.5489557226399332\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.249, - "details": { - "description": "min=0.14, mean=0.249, max=0.3, sum=3.728 (15)", - "tab": "Accuracy", - "MMLU - ECE (10-bin)": "{\"description\": \"min=0.062, mean=0.115, max=0.149, sum=1.732 (15)\", \"tab\": \"Calibration\", \"score\": \"0.11546362297486105\"}", - "MMLU - EM (Robustness)": "{\"description\": \"min=0.11, mean=0.217, max=0.28, sum=3.262 (15)\", \"tab\": \"Robustness\", \"score\": \"0.2174502923976608\"}", - "MMLU - EM (Fairness)": "{\"description\": \"min=0.13, mean=0.22, max=0.27, sum=3.294 (15)\", \"tab\": \"Fairness\", \"score\": \"0.21961403508771932\"}", - "MMLU - Denoised inference time (s)": "{\"description\": \"min=0.066, mean=0.07, max=0.072, sum=1.05 (15)\", \"tab\": \"Efficiency\", \"score\": \"0.06997480863135229\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=1542 (15)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=75 (15)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (15)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=371.38, mean=472.274, max=624.07, sum=7084.111 (15)\", \"tab\": \"General information\", \"score\": \"472.2740350877193\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=15 (15)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "MMLU - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=45 (15)\", \"tab\": \"General information\", \"score\": \"3.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "BoolQ", - "source_data": { - "dataset_name": "BoolQ", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on BoolQ", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.649, - "details": { - "description": "min=0.646, mean=0.649, max=0.65, sum=1.946 (3)", - "tab": "Accuracy", - "BoolQ - ECE (10-bin)": "{\"description\": \"min=0.043, mean=0.062, max=0.086, sum=0.187 (3)\", \"tab\": \"Calibration\", \"score\": \"0.062432673938629946\"}", - "BoolQ - EM (Robustness)": "{\"description\": \"min=0.608, mean=0.621, max=0.631, sum=1.863 (3)\", \"tab\": \"Robustness\", \"score\": \"0.621\"}", - "BoolQ - EM (Fairness)": "{\"description\": \"min=0.638, mean=0.639, max=0.64, sum=1.916 (3)\", \"tab\": \"Fairness\", \"score\": \"0.6386666666666666\"}", - "BoolQ - Denoised inference time (s)": "{\"description\": \"min=0.354, mean=0.499, max=0.575, sum=1.497 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.49915384031836946\"}", - "BoolQ - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "BoolQ - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "BoolQ - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "BoolQ - # prompt tokens": "{\"description\": \"min=660.073, mean=908.406, max=1242.073, sum=2725.219 (3)\", \"tab\": \"General information\", \"score\": \"908.4063333333334\"}", - "BoolQ - # output tokens": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "BoolQ - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "BoolQ - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.545, - "details": { - "description": "min=0.54, mean=0.545, max=0.554, sum=1.634 (3)", - "tab": "Accuracy", - "NarrativeQA - ECE (10-bin)": "{\"description\": \"min=0.189, mean=0.199, max=0.211, sum=0.596 (3)\", \"tab\": \"Calibration\", \"score\": \"0.19883043691040034\"}", - "NarrativeQA - F1 (Robustness)": "{\"description\": \"min=0.099, mean=0.135, max=0.156, sum=0.405 (3)\", \"tab\": \"Robustness\", \"score\": \"0.1349521611222693\"}", - "NarrativeQA - F1 (Fairness)": "{\"description\": \"min=0.417, mean=0.433, max=0.448, sum=1.3 (3)\", \"tab\": \"Fairness\", \"score\": \"0.43317656281615613\"}", - "NarrativeQA - Denoised inference time (s)": "{\"description\": \"min=0.988, mean=1.311, max=1.513, sum=3.934 (3)\", \"tab\": \"Efficiency\", \"score\": \"1.311420011868712\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=1065 (3)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=1.051, mean=1.647, max=2.085, sum=4.941 (3)\", \"tab\": \"General information\", \"score\": \"1.6469483568075116\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=1601.955, mean=1652.377, max=1705.003, sum=4957.132 (3)\", \"tab\": \"General information\", \"score\": \"1652.3774647887324\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=42.766, mean=56.052, max=70.845, sum=168.155 (3)\", \"tab\": \"General information\", \"score\": \"56.05164319248826\"}", - "NarrativeQA - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NarrativeQA - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=0.667 (1)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666667\"}", - "NarrativeQA - Stereotypes (gender)": "{\"description\": \"min=0.416, mean=0.451, max=0.5, sum=1.353 (3)\", \"tab\": \"Bias\", \"score\": \"0.4510416666666666\"}", - "NarrativeQA - Representation (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=1.333 (2)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666667\"}", - "NarrativeQA - Representation (gender)": "{\"description\": \"min=0.204, mean=0.217, max=0.229, sum=0.651 (3)\", \"tab\": \"Bias\", \"score\": \"0.21710889248239795\"}", - "NarrativeQA - Toxic fraction": "{\"description\": \"min=0.017, mean=0.021, max=0.025, sum=0.062 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.020657276995305163\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (open-book)", - "source_data": { - "dataset_name": "NaturalQuestions (open-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (open-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.559, - "details": { - "description": "min=0.548, mean=0.559, max=0.57, sum=1.677 (3)", - "tab": "Accuracy", - "NaturalQuestions (closed-book) - ECE (10-bin)": "{\"description\": \"min=0.069, mean=0.075, max=0.079, sum=0.224 (3)\", \"tab\": \"Calibration\", \"score\": \"0.07464671252737104\"}", - "NaturalQuestions (open-book) - ECE (10-bin)": "{\"description\": \"min=0.346, mean=0.354, max=0.358, sum=1.062 (3)\", \"tab\": \"Calibration\", \"score\": \"0.3539383109024162\"}", - "NaturalQuestions (closed-book) - F1 (Robustness)": "{\"description\": \"min=0.09, mean=0.099, max=0.109, sum=0.298 (3)\", \"tab\": \"Robustness\", \"score\": \"0.09933930594531819\"}", - "NaturalQuestions (open-book) - F1 (Robustness)": "{\"description\": \"min=0.185, mean=0.228, max=0.265, sum=0.683 (3)\", \"tab\": \"Robustness\", \"score\": \"0.22767804828628146\"}", - "NaturalQuestions (closed-book) - F1 (Fairness)": "{\"description\": \"min=0.112, mean=0.122, max=0.128, sum=0.365 (3)\", \"tab\": \"Fairness\", \"score\": \"0.12161534757794057\"}", - "NaturalQuestions (open-book) - F1 (Fairness)": "{\"description\": \"min=0.475, mean=0.493, max=0.505, sum=1.479 (3)\", \"tab\": \"Fairness\", \"score\": \"0.4930833990161269\"}", - "NaturalQuestions (closed-book) - Denoised inference time (s)": "{\"description\": \"min=1.626, mean=1.777, max=1.998, sum=5.331 (3)\", \"tab\": \"Efficiency\", \"score\": \"1.77691167926379\"}", - "NaturalQuestions (open-book) - Denoised inference time (s)": "{\"description\": \"min=3.687, mean=3.866, max=4.016, sum=11.599 (3)\", \"tab\": \"Efficiency\", \"score\": \"3.8663324384530373\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=110.254, mean=112.254, max=116.254, sum=336.762 (3)\", \"tab\": \"General information\", \"score\": \"112.254\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=273.408, mean=282.837, max=296.556, sum=848.512 (3)\", \"tab\": \"General information\", \"score\": \"282.83733333333333\"}", - "NaturalQuestions (closed-book) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.647, mean=4.691, max=4.724, sum=14.074 (3)\", \"tab\": \"General information\", \"score\": \"4.691333333333334\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.036, mean=0.036, max=0.036, sum=0.108 (3)\", \"tab\": \"General information\", \"score\": \"0.036\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1231.212, mean=1419.574, max=1523.257, sum=4258.721 (3)\", \"tab\": \"General information\", \"score\": \"1419.5736666666664\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=234.154, mean=247.23, max=261.681, sum=741.689 (3)\", \"tab\": \"General information\", \"score\": \"247.22966666666665\"}", - "NaturalQuestions (open-book) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NaturalQuestions (closed-book) - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=0.667 (1)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666667\"}", - "NaturalQuestions (closed-book) - Stereotypes (gender)": "{\"description\": \"min=0.5, mean=0.5, max=0.5, sum=0.5 (1)\", \"tab\": \"Bias\", \"score\": \"0.5\"}", - "NaturalQuestions (closed-book) - Representation (race)": "{\"description\": \"min=0.373, mean=0.49, max=0.553, sum=1.47 (3)\", \"tab\": \"Bias\", \"score\": \"0.49013920663848926\"}", - "NaturalQuestions (closed-book) - Representation (gender)": "{\"description\": \"min=0.071, mean=0.192, max=0.38, sum=0.576 (3)\", \"tab\": \"Bias\", \"score\": \"0.19214285714285717\"}", - "NaturalQuestions (open-book) - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=1.333 (2)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666667\"}", - "NaturalQuestions (open-book) - Stereotypes (gender)": "{\"description\": \"min=0.5, mean=0.5, max=0.5, sum=1.5 (3)\", \"tab\": \"Bias\", \"score\": \"0.5\"}", - "NaturalQuestions (open-book) - Representation (race)": "{\"description\": \"min=0.484, mean=0.524, max=0.561, sum=1.571 (3)\", \"tab\": \"Bias\", \"score\": \"0.5236086934551658\"}", - "NaturalQuestions (open-book) - Representation (gender)": "{\"description\": \"min=0.289, mean=0.317, max=0.333, sum=0.95 (3)\", \"tab\": \"Bias\", \"score\": \"0.3167977414801371\"}", - "NaturalQuestions (closed-book) - Toxic fraction": "{\"description\": \"min=0.001, mean=0.001, max=0.001, sum=0.003 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.001\"}", - "NaturalQuestions (open-book) - Toxic fraction": "{\"description\": \"min=0.001, mean=0.001, max=0.002, sum=0.004 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.0013333333333333333\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "QuAC", - "source_data": { - "dataset_name": "QuAC", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on QuAC", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.33, - "details": { - "description": "min=0.322, mean=0.33, max=0.335, sum=0.989 (3)", - "tab": "Accuracy", - "QuAC - ECE (10-bin)": "{\"description\": \"min=0.104, mean=0.13, max=0.169, sum=0.391 (3)\", \"tab\": \"Calibration\", \"score\": \"0.13037730069459044\"}", - "QuAC - F1 (Robustness)": "{\"description\": \"min=0.14, mean=0.147, max=0.155, sum=0.44 (3)\", \"tab\": \"Robustness\", \"score\": \"0.14672783806116493\"}", - "QuAC - F1 (Fairness)": "{\"description\": \"min=0.245, mean=0.249, max=0.258, sum=0.748 (3)\", \"tab\": \"Fairness\", \"score\": \"0.2494842989068126\"}", - "QuAC - Denoised inference time (s)": "{\"description\": \"min=1.354, mean=1.389, max=1.411, sum=4.166 (3)\", \"tab\": \"Efficiency\", \"score\": \"1.3887290514336688\"}", - "QuAC - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "QuAC - # train": "{\"description\": \"min=0.845, mean=0.944, max=1.086, sum=2.833 (3)\", \"tab\": \"General information\", \"score\": \"0.9443333333333334\"}", - "QuAC - truncated": "{\"description\": \"min=0.016, mean=0.016, max=0.016, sum=0.048 (3)\", \"tab\": \"General information\", \"score\": \"0.016\"}", - "QuAC - # prompt tokens": "{\"description\": \"min=1625.523, mean=1644.831, max=1670.605, sum=4934.492 (3)\", \"tab\": \"General information\", \"score\": \"1644.8306666666667\"}", - "QuAC - # output tokens": "{\"description\": \"min=64.208, mean=68.54, max=71.626, sum=205.621 (3)\", \"tab\": \"General information\", \"score\": \"68.54033333333334\"}", - "QuAC - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "QuAC - Stereotypes (race)": "{\"description\": \"min=0.568, mean=0.613, max=0.641, sum=1.838 (3)\", \"tab\": \"Bias\", \"score\": \"0.6126959460292795\"}", - "QuAC - Stereotypes (gender)": "{\"description\": \"min=0.41, mean=0.43, max=0.447, sum=1.29 (3)\", \"tab\": \"Bias\", \"score\": \"0.4301368170697724\"}", - "QuAC - Representation (race)": "{\"description\": \"min=0.232, mean=0.266, max=0.294, sum=0.798 (3)\", \"tab\": \"Bias\", \"score\": \"0.2658629278217009\"}", - "QuAC - Representation (gender)": "{\"description\": \"min=0.211, mean=0.23, max=0.241, sum=0.69 (3)\", \"tab\": \"Bias\", \"score\": \"0.2300432286449244\"}", - "QuAC - Toxic fraction": "{\"description\": \"min=0.001, mean=0.004, max=0.005, sum=0.011 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.0036666666666666666\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "HellaSwag", - "source_data": { - "dataset_name": "HellaSwag", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on HellaSwag", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.663, - "details": { - "description": "min=0.663, mean=0.663, max=0.663, sum=0.663 (1)", - "tab": "Accuracy", - "HellaSwag - ECE (10-bin)": "{\"description\": \"min=0.233, mean=0.233, max=0.233, sum=0.233 (1)\", \"tab\": \"Calibration\", \"score\": \"0.2332919292558098\"}", - "HellaSwag - EM (Robustness)": "{\"description\": \"min=0.619, mean=0.619, max=0.619, sum=0.619 (1)\", \"tab\": \"Robustness\", \"score\": \"0.619\"}", - "HellaSwag - EM (Fairness)": "{\"description\": \"min=0.486, mean=0.486, max=0.486, sum=0.486 (1)\", \"tab\": \"Fairness\", \"score\": \"0.486\"}", - "HellaSwag - Denoised inference time (s)": "{\"description\": \"min=0.03, mean=0.03, max=0.03, sum=0.03 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.030294155851006508\"}", - "HellaSwag - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "HellaSwag - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "HellaSwag - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "HellaSwag - # prompt tokens": "{\"description\": \"min=87.888, mean=87.888, max=87.888, sum=87.888 (1)\", \"tab\": \"General information\", \"score\": \"87.888\"}", - "HellaSwag - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "HellaSwag - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.514, - "details": { - "description": "min=0.514, mean=0.514, max=0.514, sum=0.514 (1)", - "tab": "Accuracy", - "OpenbookQA - ECE (10-bin)": "{\"description\": \"min=0.235, mean=0.235, max=0.235, sum=0.235 (1)\", \"tab\": \"Calibration\", \"score\": \"0.2353362549897216\"}", - "OpenbookQA - EM (Robustness)": "{\"description\": \"min=0.398, mean=0.398, max=0.398, sum=0.398 (1)\", \"tab\": \"Robustness\", \"score\": \"0.398\"}", - "OpenbookQA - EM (Fairness)": "{\"description\": \"min=0.416, mean=0.416, max=0.416, sum=0.416 (1)\", \"tab\": \"Fairness\", \"score\": \"0.416\"}", - "OpenbookQA - Denoised inference time (s)": "{\"description\": \"min=0.019, mean=0.019, max=0.019, sum=0.019 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.019339164675618026\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=5.27, mean=5.27, max=5.27, sum=5.27 (1)\", \"tab\": \"General information\", \"score\": \"5.27\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "TruthfulQA", - "source_data": { - "dataset_name": "TruthfulQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on TruthfulQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.199, - "details": { - "description": "min=0.187, mean=0.199, max=0.213, sum=0.797 (4)", - "tab": "Accuracy", - "TruthfulQA - ECE (10-bin)": "{\"description\": \"min=0.056, mean=0.078, max=0.103, sum=0.311 (4)\", \"tab\": \"Calibration\", \"score\": \"0.07772735423117484\"}", - "TruthfulQA - EM (Robustness)": "{\"description\": \"min=0.157, mean=0.181, max=0.209, sum=0.725 (4)\", \"tab\": \"Robustness\", \"score\": \"0.1811926605504587\"}", - "TruthfulQA - EM (Fairness)": "{\"description\": \"min=0.156, mean=0.18, max=0.209, sum=0.72 (4)\", \"tab\": \"Fairness\", \"score\": \"0.18004587155963303\"}", - "TruthfulQA - Denoised inference time (s)": "{\"description\": \"min=0.018, mean=0.044, max=0.053, sum=0.175 (4)\", \"tab\": \"Efficiency\", \"score\": \"0.043782452828866295\"}", - "TruthfulQA - # eval": "{\"description\": \"min=654, mean=654, max=654, sum=2616 (4)\", \"tab\": \"General information\", \"score\": \"654.0\"}", - "TruthfulQA - # train": "{\"description\": \"min=0, mean=3.75, max=5, sum=15 (4)\", \"tab\": \"General information\", \"score\": \"3.75\"}", - "TruthfulQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (4)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "TruthfulQA - # prompt tokens": "{\"description\": \"min=85.121, mean=404.621, max=529.121, sum=1618.483 (4)\", \"tab\": \"General information\", \"score\": \"404.62079510703364\"}", - "TruthfulQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=4 (4)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "TruthfulQA - # trials": "{\"description\": \"min=1, mean=2.5, max=3, sum=10 (4)\", \"tab\": \"General information\", \"score\": \"2.5\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MS MARCO (TREC)", - "source_data": { - "dataset_name": "MS MARCO (TREC)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "NDCG@10 on MS MARCO (TREC)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.345, - "details": { - "description": "min=0.315, mean=0.345, max=0.362, sum=1.035 (3)", - "tab": "Accuracy", - "MS MARCO (regular) - RR@10 (Robustness)": "{\"description\": \"min=0.094, mean=0.116, max=0.131, sum=0.349 (3)\", \"tab\": \"Robustness\", \"score\": \"0.11636587301587299\"}", - "MS MARCO (TREC) - NDCG@10 (Robustness)": "{\"description\": \"min=0.29, mean=0.319, max=0.336, sum=0.957 (3)\", \"tab\": \"Robustness\", \"score\": \"0.3190834142643501\"}", - "MS MARCO (regular) - RR@10 (Fairness)": "{\"description\": \"min=0.106, mean=0.129, max=0.144, sum=0.387 (3)\", \"tab\": \"Fairness\", \"score\": \"0.12886375661375657\"}", - "MS MARCO (TREC) - NDCG@10 (Fairness)": "{\"description\": \"min=0.303, mean=0.332, max=0.348, sum=0.997 (3)\", \"tab\": \"Fairness\", \"score\": \"0.3321982457704417\"}", - "MS MARCO (regular) - Denoised inference time (s)": "{\"description\": \"min=0.081, mean=0.084, max=0.088, sum=0.252 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.08407480907713127\"}", - "MS MARCO (TREC) - Denoised inference time (s)": "{\"description\": \"min=0.078, mean=0.081, max=0.083, sum=0.242 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.08053553836682271\"}", - "MS MARCO (regular) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "MS MARCO (regular) - # train": "{\"description\": \"min=2, mean=2, max=2, sum=6 (3)\", \"tab\": \"General information\", \"score\": \"2.0\"}", - "MS MARCO (regular) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MS MARCO (regular) - # prompt tokens": "{\"description\": \"min=495.232, mean=532.565, max=577.232, sum=1597.696 (3)\", \"tab\": \"General information\", \"score\": \"532.5653333333333\"}", - "MS MARCO (regular) - # output tokens": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MS MARCO (regular) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "MS MARCO (TREC) - # eval": "{\"description\": \"min=43, mean=43, max=43, sum=129 (3)\", \"tab\": \"General information\", \"score\": \"43.0\"}", - "MS MARCO (TREC) - # train": "{\"description\": \"min=2, mean=2, max=2, sum=6 (3)\", \"tab\": \"General information\", \"score\": \"2.0\"}", - "MS MARCO (TREC) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MS MARCO (TREC) - # prompt tokens": "{\"description\": \"min=478.488, mean=515.822, max=560.488, sum=1547.465 (3)\", \"tab\": \"General information\", \"score\": \"515.8217054263565\"}", - "MS MARCO (TREC) - # output tokens": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MS MARCO (TREC) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "MS MARCO (regular) - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "MS MARCO (TREC) - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CNN/DailyMail", - "source_data": { - "dataset_name": "CNN/DailyMail", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on CNN/DailyMail", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.131, - "details": { - "description": "min=0.127, mean=0.131, max=0.135, sum=0.787 (6)", - "tab": "Accuracy", - "CNN/DailyMail - Denoised inference time (s)": "{\"description\": \"min=1.997, mean=2.076, max=2.172, sum=12.455 (6)\", \"tab\": \"Efficiency\", \"score\": \"2.0758840914959578\"}", - "CNN/DailyMail - # eval": "{\"description\": \"min=466, mean=466, max=466, sum=2796 (6)\", \"tab\": \"General information\", \"score\": \"466.0\"}", - "CNN/DailyMail - # train": "{\"description\": \"min=5, mean=5, max=5, sum=30 (6)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CNN/DailyMail - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (6)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CNN/DailyMail - # prompt tokens": "{\"description\": \"min=1531.586, mean=1549.919, max=1567.586, sum=9299.515 (6)\", \"tab\": \"General information\", \"score\": \"1549.9191702432045\"}", - "CNN/DailyMail - # output tokens": "{\"description\": \"min=76.916, mean=83.931, max=91.68, sum=503.584 (6)\", \"tab\": \"General information\", \"score\": \"83.93061516452074\"}", - "CNN/DailyMail - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=18 (6)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "CNN/DailyMail - Stereotypes (race)": "{\"description\": \"min=0.602, mean=0.63, max=0.655, sum=3.78 (6)\", \"tab\": \"Bias\", \"score\": \"0.6299677400199846\"}", - "CNN/DailyMail - Stereotypes (gender)": "{\"description\": \"min=0.398, mean=0.402, max=0.41, sum=2.415 (6)\", \"tab\": \"Bias\", \"score\": \"0.40247728320483095\"}", - "CNN/DailyMail - Representation (race)": "{\"description\": \"min=0.23, mean=0.293, max=0.359, sum=1.759 (6)\", \"tab\": \"Bias\", \"score\": \"0.2931668421996429\"}", - "CNN/DailyMail - Representation (gender)": "{\"description\": \"min=0.131, mean=0.146, max=0.169, sum=0.875 (6)\", \"tab\": \"Bias\", \"score\": \"0.14576217898261626\"}", - "CNN/DailyMail - Toxic fraction": "{\"description\": \"min=0.002, mean=0.002, max=0.002, sum=0.013 (6)\", \"tab\": \"Toxicity\", \"score\": \"0.002145922746781116\"}", - "CNN/DailyMail - SummaC": "{\"description\": \"min=0.172, mean=0.208, max=0.236, sum=0.623 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.20780144742590156\"}", - "CNN/DailyMail - QAFactEval": "{\"description\": \"min=4.648, mean=4.704, max=4.739, sum=28.226 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"4.704313539792442\"}", - "CNN/DailyMail - BERTScore (F1)": "{\"description\": \"min=0.241, mean=0.247, max=0.25, sum=0.74 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.2466254745716148\"}", - "CNN/DailyMail - Coverage": "{\"description\": \"min=0.902, mean=0.948, max=0.97, sum=5.685 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"0.9475541325972495\"}", - "CNN/DailyMail - Density": "{\"description\": \"min=41.364, mean=48.284, max=57.69, sum=289.703 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"48.283839374824815\"}", - "CNN/DailyMail - Compression": "{\"description\": \"min=8.117, mean=9.864, max=11.439, sum=59.186 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"9.864391531990323\"}", - "CNN/DailyMail - HumanEval-faithfulness": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-relevance": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-coherence": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "XSUM", - "source_data": { - "dataset_name": "XSUM", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on XSUM", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.096, - "details": { - "description": "min=0.093, mean=0.096, max=0.097, sum=0.573 (6)", - "tab": "Accuracy", - "XSUM - Denoised inference time (s)": "{\"description\": \"min=0.73, mean=0.742, max=0.758, sum=4.455 (6)\", \"tab\": \"Efficiency\", \"score\": \"0.7424737962465443\"}", - "XSUM - # eval": "{\"description\": \"min=518, mean=518, max=518, sum=3108 (6)\", \"tab\": \"General information\", \"score\": \"518.0\"}", - "XSUM - # train": "{\"description\": \"min=4.998, mean=4.999, max=5, sum=29.992 (6)\", \"tab\": \"General information\", \"score\": \"4.998712998712999\"}", - "XSUM - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (6)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "XSUM - # prompt tokens": "{\"description\": \"min=1456.402, mean=1510.418, max=1538.921, sum=9062.51 (6)\", \"tab\": \"General information\", \"score\": \"1510.4182754182755\"}", - "XSUM - # output tokens": "{\"description\": \"min=24.919, mean=25.529, max=26.187, sum=153.174 (6)\", \"tab\": \"General information\", \"score\": \"25.52895752895753\"}", - "XSUM - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=18 (6)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "XSUM - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=4 (6)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "XSUM - Stereotypes (gender)": "{\"description\": \"min=0.393, mean=0.435, max=0.466, sum=2.612 (6)\", \"tab\": \"Bias\", \"score\": \"0.43535525321239604\"}", - "XSUM - Representation (race)": "{\"description\": \"min=0.467, mean=0.513, max=0.565, sum=3.08 (6)\", \"tab\": \"Bias\", \"score\": \"0.5133548156104547\"}", - "XSUM - Representation (gender)": "{\"description\": \"min=0.141, mean=0.165, max=0.179, sum=0.988 (6)\", \"tab\": \"Bias\", \"score\": \"0.1646512031093765\"}", - "XSUM - Toxic fraction": "{\"description\": \"min=0, mean=0.002, max=0.004, sum=0.012 (6)\", \"tab\": \"Toxicity\", \"score\": \"0.0019305019305019308\"}", - "XSUM - SummaC": "{\"description\": \"min=-0.229, mean=-0.198, max=-0.176, sum=-0.593 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"-0.1976111372976741\"}", - "XSUM - QAFactEval": "{\"description\": \"min=3.59, mean=3.813, max=4.142, sum=22.877 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"3.8128682530109397\"}", - "XSUM - BERTScore (F1)": "{\"description\": \"min=0.379, mean=0.381, max=0.384, sum=1.142 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.3808147712365148\"}", - "XSUM - Coverage": "{\"description\": \"min=0.824, mean=0.829, max=0.831, sum=4.972 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"0.8286466360730634\"}", - "XSUM - Density": "{\"description\": \"min=3.796, mean=4.043, max=4.434, sum=24.256 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"4.042629935538992\"}", - "XSUM - Compression": "{\"description\": \"min=17.57, mean=17.942, max=18.398, sum=107.65 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"17.941696288315352\"}", - "XSUM - HumanEval-faithfulness": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-relevance": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-coherence": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "IMDB", - "source_data": { - "dataset_name": "IMDB", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on IMDB", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.939, - "details": { - "description": "min=0.932, mean=0.939, max=0.946, sum=2.816 (3)", - "tab": "Accuracy", - "IMDB - ECE (10-bin)": "{\"description\": \"min=0.285, mean=0.295, max=0.311, sum=0.884 (3)\", \"tab\": \"Calibration\", \"score\": \"0.2945110955018834\"}", - "IMDB - EM (Robustness)": "{\"description\": \"min=0.895, mean=0.903, max=0.908, sum=2.709 (3)\", \"tab\": \"Robustness\", \"score\": \"0.903\"}", - "IMDB - EM (Fairness)": "{\"description\": \"min=0.92, mean=0.927, max=0.932, sum=2.782 (3)\", \"tab\": \"Fairness\", \"score\": \"0.9273333333333333\"}", - "IMDB - Denoised inference time (s)": "{\"description\": \"min=0.63, mean=0.701, max=0.761, sum=2.104 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.7011672212481499\"}", - "IMDB - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "IMDB - # train": "{\"description\": \"min=4.846, mean=4.933, max=4.986, sum=14.798 (3)\", \"tab\": \"General information\", \"score\": \"4.932666666666667\"}", - "IMDB - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IMDB - # prompt tokens": "{\"description\": \"min=1152.694, mean=1389.454, max=1744.631, sum=4168.363 (3)\", \"tab\": \"General information\", \"score\": \"1389.4543333333331\"}", - "IMDB - # output tokens": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "IMDB - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "IMDB - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CivilComments", - "source_data": { - "dataset_name": "CivilComments", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on CivilComments", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.52, - "details": { - "description": "min=0.002, mean=0.52, max=1, sum=28.06 (54)", - "tab": "Accuracy", - "CivilComments - ECE (10-bin)": "{\"description\": \"min=0.075, mean=0.409, max=0.626, sum=22.076 (54)\", \"tab\": \"Calibration\", \"score\": \"0.40880926893677766\"}", - "CivilComments - EM (Robustness)": "{\"description\": \"min=0, mean=0.418, max=1, sum=22.597 (54)\", \"tab\": \"Robustness\", \"score\": \"0.4184575354873046\"}", - "CivilComments - EM (Fairness)": "{\"description\": \"min=0, mean=0.488, max=1, sum=26.356 (54)\", \"tab\": \"Fairness\", \"score\": \"0.4880679688031825\"}", - "CivilComments - Denoised inference time (s)": "{\"description\": \"min=0.154, mean=0.307, max=0.494, sum=16.591 (54)\", \"tab\": \"Efficiency\", \"score\": \"0.30723795570455475\"}", - "CivilComments - # eval": "{\"description\": \"min=74, mean=371.556, max=683, sum=20064 (54)\", \"tab\": \"General information\", \"score\": \"371.55555555555554\"}", - "CivilComments - # train": "{\"description\": \"min=5, mean=5, max=5, sum=270 (54)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CivilComments - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (54)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CivilComments - # prompt tokens": "{\"description\": \"min=356.537, mean=722.635, max=1267.519, sum=39022.317 (54)\", \"tab\": \"General information\", \"score\": \"722.6354931173206\"}", - "CivilComments - # output tokens": "{\"description\": \"min=5, mean=5, max=5, sum=270 (54)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CivilComments - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=162 (54)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "CivilComments - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (race)": "{\"description\": \"min=0.333, mean=0.5, max=0.667, sum=1 (2)\", \"tab\": \"Bias\", \"score\": \"0.5\"}", - "CivilComments - Representation (gender)": "{\"description\": \"min=0.5, mean=0.5, max=0.5, sum=1 (2)\", \"tab\": \"Bias\", \"score\": \"0.5\"}", - "CivilComments - Toxic fraction": "{\"description\": \"min=0, mean=0.0, max=0.001, sum=0.001 (54)\", \"tab\": \"Toxicity\", \"score\": \"2.7763895829862844e-05\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "RAFT", - "source_data": { - "dataset_name": "RAFT", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on RAFT", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.619, - "details": { - "description": "min=0.275, mean=0.619, max=0.975, sum=20.425 (33)", - "tab": "Accuracy", - "RAFT - ECE (10-bin)": "{\"description\": \"min=0.116, mean=0.389, max=0.975, sum=12.832 (33)\", \"tab\": \"Calibration\", \"score\": \"0.3888407166022056\"}", - "RAFT - EM (Robustness)": "{\"description\": \"min=0.1, mean=0.53, max=0.975, sum=17.5 (33)\", \"tab\": \"Robustness\", \"score\": \"0.5303030303030303\"}", - "RAFT - EM (Fairness)": "{\"description\": \"min=0.275, mean=0.594, max=0.975, sum=19.6 (33)\", \"tab\": \"Fairness\", \"score\": \"0.593939393939394\"}", - "RAFT - Denoised inference time (s)": "{\"description\": \"min=0.107, mean=0.628, max=1.382, sum=20.733 (33)\", \"tab\": \"Efficiency\", \"score\": \"0.6282604447639349\"}", - "RAFT - # eval": "{\"description\": \"min=40, mean=40, max=40, sum=1320 (33)\", \"tab\": \"General information\", \"score\": \"40.0\"}", - "RAFT - # train": "{\"description\": \"min=0, mean=4.556, max=5, sum=150.35 (33)\", \"tab\": \"General information\", \"score\": \"4.556060606060607\"}", - "RAFT - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (33)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "RAFT - # prompt tokens": "{\"description\": \"min=257.35, mean=812.938, max=1773.675, sum=26826.95 (33)\", \"tab\": \"General information\", \"score\": \"812.937878787879\"}", - "RAFT - # output tokens": "{\"description\": \"min=5, mean=14.276, max=30, sum=471.1 (33)\", \"tab\": \"General information\", \"score\": \"14.275757575757577\"}", - "RAFT - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=99 (33)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "RAFT - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (33)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/openai_gpt-neox-20b.json b/data/models/openai_gpt-neox-20b.json deleted file mode 100644 index 9d4aedae48359d59c2ac792e84f4a4de75a9b087..0000000000000000000000000000000000000000 --- a/data/models/openai_gpt-neox-20b.json +++ /dev/null @@ -1,674 +0,0 @@ -{ - "model_info": { - "name": "GPT-NeoX 20B", - "id": "openai/GPT-NeoX-20B", - "developer": "OpenAI", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_classic/openai_GPT-NeoX-20B/1774096308.339228", - "retrieved_timestamp": "1774096308.339228", - "source_metadata": { - "source_name": "helm_classic", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_classic", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_classic", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperform on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.351, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Calibration": "{\"description\": \"\", \"tab\": \"Calibration\", \"score\": \"0.4215761012322838\"}", - "Mean win rate - Robustness": "{\"description\": \"\", \"tab\": \"Robustness\", \"score\": \"0.3361523348731358\"}", - "Mean win rate - Fairness": "{\"description\": \"\", \"tab\": \"Fairness\", \"score\": \"0.3311530516202374\"}", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.5141337719298246\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}", - "Mean win rate - Bias": "{\"description\": \"\", \"tab\": \"Bias\", \"score\": \"0.46836548983528487\"}", - "Mean win rate - Toxicity": "{\"description\": \"\", \"tab\": \"Toxicity\", \"score\": \"0.36547434047434046\"}", - "Mean win rate - Summarization metrics": "{\"description\": \"\", \"tab\": \"Summarization metrics\", \"score\": \"0.4456349206349206\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.276, - "details": { - "description": "min=0.21, mean=0.276, max=0.351, sum=4.146 (15)", - "tab": "Accuracy", - "MMLU - ECE (10-bin)": "{\"description\": \"min=0.094, mean=0.122, max=0.145, sum=1.831 (15)\", \"tab\": \"Calibration\", \"score\": \"0.12205035764205192\"}", - "MMLU - EM (Robustness)": "{\"description\": \"min=0.149, mean=0.189, max=0.24, sum=2.833 (15)\", \"tab\": \"Robustness\", \"score\": \"0.1888421052631579\"}", - "MMLU - EM (Fairness)": "{\"description\": \"min=0.175, mean=0.215, max=0.26, sum=3.228 (15)\", \"tab\": \"Fairness\", \"score\": \"0.21518128654970764\"}", - "MMLU - Denoised inference time (s)": "{\"description\": \"min=0.093, mean=0.133, max=0.275, sum=1.995 (15)\", \"tab\": \"Efficiency\", \"score\": \"0.1330090104470642\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=1542 (15)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=75 (15)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (15)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=358.76, mean=467.936, max=612.798, sum=7019.035 (15)\", \"tab\": \"General information\", \"score\": \"467.935649122807\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=15 (15)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "MMLU - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=45 (15)\", \"tab\": \"General information\", \"score\": \"3.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "BoolQ", - "source_data": { - "dataset_name": "BoolQ", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on BoolQ", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.683, - "details": { - "description": "min=0.659, mean=0.683, max=0.714, sum=2.048 (3)", - "tab": "Accuracy", - "BoolQ - ECE (10-bin)": "{\"description\": \"min=0.168, mean=0.195, max=0.238, sum=0.585 (3)\", \"tab\": \"Calibration\", \"score\": \"0.19500535688345313\"}", - "BoolQ - EM (Robustness)": "{\"description\": \"min=0.548, mean=0.551, max=0.556, sum=1.653 (3)\", \"tab\": \"Robustness\", \"score\": \"0.551\"}", - "BoolQ - EM (Fairness)": "{\"description\": \"min=0.594, mean=0.609, max=0.629, sum=1.827 (3)\", \"tab\": \"Fairness\", \"score\": \"0.609\"}", - "BoolQ - Denoised inference time (s)": "{\"description\": \"min=0.515, mean=0.773, max=1.206, sum=2.318 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.772616056262233\"}", - "BoolQ - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "BoolQ - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "BoolQ - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "BoolQ - # prompt tokens": "{\"description\": \"min=656.897, mean=913.897, max=1251.897, sum=2741.691 (3)\", \"tab\": \"General information\", \"score\": \"913.8969999999999\"}", - "BoolQ - # output tokens": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "BoolQ - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "BoolQ - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.599, - "details": { - "description": "min=0.558, mean=0.599, max=0.623, sum=1.797 (3)", - "tab": "Accuracy", - "NarrativeQA - ECE (10-bin)": "{\"description\": \"min=0.2, mean=0.224, max=0.244, sum=0.672 (3)\", \"tab\": \"Calibration\", \"score\": \"0.2239646545151891\"}", - "NarrativeQA - F1 (Robustness)": "{\"description\": \"min=0.378, mean=0.421, max=0.443, sum=1.263 (3)\", \"tab\": \"Robustness\", \"score\": \"0.4211068794456416\"}", - "NarrativeQA - F1 (Fairness)": "{\"description\": \"min=0.419, mean=0.461, max=0.485, sum=1.382 (3)\", \"tab\": \"Fairness\", \"score\": \"0.46066534756418576\"}", - "NarrativeQA - Denoised inference time (s)": "{\"description\": \"min=0.904, mean=1.468, max=1.998, sum=4.404 (3)\", \"tab\": \"Efficiency\", \"score\": \"1.4680144681286658\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=1065 (3)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=0.989, mean=1.568, max=1.969, sum=4.704 (3)\", \"tab\": \"General information\", \"score\": \"1.568075117370892\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=1607.893, mean=1641.033, max=1691.082, sum=4923.099 (3)\", \"tab\": \"General information\", \"score\": \"1641.0328638497651\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=24.282, mean=40.047, max=54.028, sum=120.141 (3)\", \"tab\": \"General information\", \"score\": \"40.04694835680751\"}", - "NarrativeQA - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NarrativeQA - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=0.667 (1)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666667\"}", - "NarrativeQA - Stereotypes (gender)": "{\"description\": \"min=0.396, mean=0.449, max=0.5, sum=1.346 (3)\", \"tab\": \"Bias\", \"score\": \"0.44861111111111107\"}", - "NarrativeQA - Representation (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=2 (3)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "NarrativeQA - Representation (gender)": "{\"description\": \"min=0.159, mean=0.186, max=0.206, sum=0.557 (3)\", \"tab\": \"Bias\", \"score\": \"0.18579713036394171\"}", - "NarrativeQA - Toxic fraction": "{\"description\": \"min=0.017, mean=0.022, max=0.025, sum=0.065 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.0215962441314554\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (open-book)", - "source_data": { - "dataset_name": "NaturalQuestions (open-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (open-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.596, - "details": { - "description": "min=0.581, mean=0.596, max=0.608, sum=1.788 (3)", - "tab": "Accuracy", - "NaturalQuestions (closed-book) - ECE (10-bin)": "{\"description\": \"min=0.099, mean=0.103, max=0.106, sum=0.309 (3)\", \"tab\": \"Calibration\", \"score\": \"0.10315653555419742\"}", - "NaturalQuestions (open-book) - ECE (10-bin)": "{\"description\": \"min=0.371, mean=0.373, max=0.375, sum=1.118 (3)\", \"tab\": \"Calibration\", \"score\": \"0.37278118995003706\"}", - "NaturalQuestions (closed-book) - F1 (Robustness)": "{\"description\": \"min=0.125, mean=0.133, max=0.14, sum=0.398 (3)\", \"tab\": \"Robustness\", \"score\": \"0.1325934362402064\"}", - "NaturalQuestions (open-book) - F1 (Robustness)": "{\"description\": \"min=0.429, mean=0.452, max=0.48, sum=1.357 (3)\", \"tab\": \"Robustness\", \"score\": \"0.4524359199313521\"}", - "NaturalQuestions (closed-book) - F1 (Fairness)": "{\"description\": \"min=0.147, mean=0.154, max=0.158, sum=0.461 (3)\", \"tab\": \"Fairness\", \"score\": \"0.15381312093617092\"}", - "NaturalQuestions (open-book) - F1 (Fairness)": "{\"description\": \"min=0.509, mean=0.525, max=0.537, sum=1.574 (3)\", \"tab\": \"Fairness\", \"score\": \"0.524698076718683\"}", - "NaturalQuestions (closed-book) - Denoised inference time (s)": "{\"description\": \"min=0.381, mean=0.482, max=0.655, sum=1.447 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.4823250982166127\"}", - "NaturalQuestions (open-book) - Denoised inference time (s)": "{\"description\": \"min=1.913, mean=2.137, max=2.288, sum=6.411 (3)\", \"tab\": \"Efficiency\", \"score\": \"2.1369374864319965\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=110.299, mean=112.966, max=117.299, sum=338.897 (3)\", \"tab\": \"General information\", \"score\": \"112.96566666666668\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=77.379, mean=90.195, max=107.541, sum=270.584 (3)\", \"tab\": \"General information\", \"score\": \"90.19466666666666\"}", - "NaturalQuestions (closed-book) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.685, mean=4.704, max=4.723, sum=14.112 (3)\", \"tab\": \"General information\", \"score\": \"4.704\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.037, mean=0.037, max=0.037, sum=0.111 (3)\", \"tab\": \"General information\", \"score\": \"0.037\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1247.862, mean=1394.229, max=1495.552, sum=4182.688 (3)\", \"tab\": \"General information\", \"score\": \"1394.2293333333334\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=73.671, mean=87.693, max=98.984, sum=263.078 (3)\", \"tab\": \"General information\", \"score\": \"87.69266666666665\"}", - "NaturalQuestions (open-book) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NaturalQuestions (closed-book) - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - Stereotypes (gender)": "{\"description\": \"min=0.5, mean=0.5, max=0.5, sum=1 (2)\", \"tab\": \"Bias\", \"score\": \"0.5\"}", - "NaturalQuestions (closed-book) - Representation (race)": "{\"description\": \"min=0.309, mean=0.362, max=0.444, sum=1.086 (3)\", \"tab\": \"Bias\", \"score\": \"0.3621399176954732\"}", - "NaturalQuestions (closed-book) - Representation (gender)": "{\"description\": \"min=0.233, mean=0.318, max=0.382, sum=0.954 (3)\", \"tab\": \"Bias\", \"score\": \"0.31784137078254726\"}", - "NaturalQuestions (open-book) - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=2 (3)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "NaturalQuestions (open-book) - Stereotypes (gender)": "{\"description\": \"min=0.5, mean=0.5, max=0.5, sum=1.5 (3)\", \"tab\": \"Bias\", \"score\": \"0.5\"}", - "NaturalQuestions (open-book) - Representation (race)": "{\"description\": \"min=0.538, mean=0.57, max=0.59, sum=1.709 (3)\", \"tab\": \"Bias\", \"score\": \"0.5695499220251695\"}", - "NaturalQuestions (open-book) - Representation (gender)": "{\"description\": \"min=0, mean=0.094, max=0.241, sum=0.283 (3)\", \"tab\": \"Bias\", \"score\": \"0.09428104575163399\"}", - "NaturalQuestions (closed-book) - Toxic fraction": "{\"description\": \"min=0, mean=0.001, max=0.002, sum=0.003 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.001\"}", - "NaturalQuestions (open-book) - Toxic fraction": "{\"description\": \"min=0.001, mean=0.002, max=0.003, sum=0.006 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.002\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "QuAC", - "source_data": { - "dataset_name": "QuAC", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on QuAC", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.326, - "details": { - "description": "min=0.32, mean=0.326, max=0.335, sum=0.979 (3)", - "tab": "Accuracy", - "QuAC - ECE (10-bin)": "{\"description\": \"min=0.105, mean=0.115, max=0.129, sum=0.345 (3)\", \"tab\": \"Calibration\", \"score\": \"0.11494333135422596\"}", - "QuAC - F1 (Robustness)": "{\"description\": \"min=0.176, mean=0.191, max=0.202, sum=0.574 (3)\", \"tab\": \"Robustness\", \"score\": \"0.19141062427574787\"}", - "QuAC - F1 (Fairness)": "{\"description\": \"min=0.224, mean=0.232, max=0.243, sum=0.695 (3)\", \"tab\": \"Fairness\", \"score\": \"0.23177797124335245\"}", - "QuAC - Denoised inference time (s)": "{\"description\": \"min=1.906, mean=2.025, max=2.127, sum=6.075 (3)\", \"tab\": \"Efficiency\", \"score\": \"2.024874148220674\"}", - "QuAC - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "QuAC - # train": "{\"description\": \"min=0.804, mean=0.889, max=0.979, sum=2.666 (3)\", \"tab\": \"General information\", \"score\": \"0.8886666666666666\"}", - "QuAC - truncated": "{\"description\": \"min=0.021, mean=0.021, max=0.021, sum=0.063 (3)\", \"tab\": \"General information\", \"score\": \"0.021\"}", - "QuAC - # prompt tokens": "{\"description\": \"min=1602.026, mean=1640.361, max=1663.349, sum=4921.083 (3)\", \"tab\": \"General information\", \"score\": \"1640.3609999999999\"}", - "QuAC - # output tokens": "{\"description\": \"min=73.99, mean=77.489, max=80.665, sum=232.466 (3)\", \"tab\": \"General information\", \"score\": \"77.48866666666667\"}", - "QuAC - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "QuAC - Stereotypes (race)": "{\"description\": \"min=0.606, mean=0.626, max=0.639, sum=1.877 (3)\", \"tab\": \"Bias\", \"score\": \"0.6257674787086551\"}", - "QuAC - Stereotypes (gender)": "{\"description\": \"min=0.436, mean=0.448, max=0.455, sum=1.344 (3)\", \"tab\": \"Bias\", \"score\": \"0.4481503328194676\"}", - "QuAC - Representation (race)": "{\"description\": \"min=0.319, mean=0.334, max=0.354, sum=1.003 (3)\", \"tab\": \"Bias\", \"score\": \"0.3344046827039365\"}", - "QuAC - Representation (gender)": "{\"description\": \"min=0.258, mean=0.268, max=0.282, sum=0.804 (3)\", \"tab\": \"Bias\", \"score\": \"0.26793463346025864\"}", - "QuAC - Toxic fraction": "{\"description\": \"min=0.001, mean=0.001, max=0.001, sum=0.003 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.001\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "HellaSwag", - "source_data": { - "dataset_name": "HellaSwag", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on HellaSwag", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.718, - "details": { - "description": "min=0.718, mean=0.718, max=0.718, sum=0.718 (1)", - "tab": "Accuracy", - "HellaSwag - ECE (10-bin)": "{\"description\": \"min=0.277, mean=0.277, max=0.277, sum=0.277 (1)\", \"tab\": \"Calibration\", \"score\": \"0.2773372160584027\"}", - "HellaSwag - EM (Robustness)": "{\"description\": \"min=0.661, mean=0.661, max=0.661, sum=0.661 (1)\", \"tab\": \"Robustness\", \"score\": \"0.661\"}", - "HellaSwag - EM (Fairness)": "{\"description\": \"min=0.552, mean=0.552, max=0.552, sum=0.552 (1)\", \"tab\": \"Fairness\", \"score\": \"0.552\"}", - "HellaSwag - Denoised inference time (s)": "{\"description\": \"min=0.025, mean=0.025, max=0.025, sum=0.025 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.025470768198370932\"}", - "HellaSwag - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "HellaSwag - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "HellaSwag - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "HellaSwag - # prompt tokens": "{\"description\": \"min=88.806, mean=88.806, max=88.806, sum=88.806 (1)\", \"tab\": \"General information\", \"score\": \"88.806\"}", - "HellaSwag - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "HellaSwag - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.524, - "details": { - "description": "min=0.524, mean=0.524, max=0.524, sum=0.524 (1)", - "tab": "Accuracy", - "OpenbookQA - ECE (10-bin)": "{\"description\": \"min=0.232, mean=0.232, max=0.232, sum=0.232 (1)\", \"tab\": \"Calibration\", \"score\": \"0.23249621701719156\"}", - "OpenbookQA - EM (Robustness)": "{\"description\": \"min=0.414, mean=0.414, max=0.414, sum=0.414 (1)\", \"tab\": \"Robustness\", \"score\": \"0.414\"}", - "OpenbookQA - EM (Fairness)": "{\"description\": \"min=0.438, mean=0.438, max=0.438, sum=0.438 (1)\", \"tab\": \"Fairness\", \"score\": \"0.438\"}", - "OpenbookQA - Denoised inference time (s)": "{\"description\": \"min=0.024, mean=0.024, max=0.024, sum=0.024 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.023963596328905958\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=5.346, mean=5.346, max=5.346, sum=5.346 (1)\", \"tab\": \"General information\", \"score\": \"5.346\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "TruthfulQA", - "source_data": { - "dataset_name": "TruthfulQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on TruthfulQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.216, - "details": { - "description": "min=0.205, mean=0.216, max=0.225, sum=0.864 (4)", - "tab": "Accuracy", - "TruthfulQA - ECE (10-bin)": "{\"description\": \"min=0.051, mean=0.058, max=0.068, sum=0.232 (4)\", \"tab\": \"Calibration\", \"score\": \"0.057891800582365614\"}", - "TruthfulQA - EM (Robustness)": "{\"description\": \"min=0.144, mean=0.175, max=0.225, sum=0.7 (4)\", \"tab\": \"Robustness\", \"score\": \"0.17507645259938837\"}", - "TruthfulQA - EM (Fairness)": "{\"description\": \"min=0.161, mean=0.179, max=0.225, sum=0.714 (4)\", \"tab\": \"Fairness\", \"score\": \"0.17851681957186544\"}", - "TruthfulQA - Denoised inference time (s)": "{\"description\": \"min=0.029, mean=0.084, max=0.133, sum=0.335 (4)\", \"tab\": \"Efficiency\", \"score\": \"0.08375055263898766\"}", - "TruthfulQA - # eval": "{\"description\": \"min=654, mean=654, max=654, sum=2616 (4)\", \"tab\": \"General information\", \"score\": \"654.0\"}", - "TruthfulQA - # train": "{\"description\": \"min=0, mean=3.75, max=5, sum=15 (4)\", \"tab\": \"General information\", \"score\": \"3.75\"}", - "TruthfulQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (4)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "TruthfulQA - # prompt tokens": "{\"description\": \"min=86.352, mean=406.102, max=532.352, sum=1624.407 (4)\", \"tab\": \"General information\", \"score\": \"406.10168195718654\"}", - "TruthfulQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=4 (4)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "TruthfulQA - # trials": "{\"description\": \"min=1, mean=2.5, max=3, sum=10 (4)\", \"tab\": \"General information\", \"score\": \"2.5\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MS MARCO (TREC)", - "source_data": { - "dataset_name": "MS MARCO (TREC)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "NDCG@10 on MS MARCO (TREC)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.398, - "details": { - "description": "min=0.37, mean=0.398, max=0.436, sum=1.195 (3)", - "tab": "Accuracy", - "MS MARCO (regular) - RR@10 (Robustness)": "{\"description\": \"min=0.082, mean=0.096, max=0.107, sum=0.288 (3)\", \"tab\": \"Robustness\", \"score\": \"0.09600105820105831\"}", - "MS MARCO (TREC) - NDCG@10 (Robustness)": "{\"description\": \"min=0.338, mean=0.351, max=0.365, sum=1.053 (3)\", \"tab\": \"Robustness\", \"score\": \"0.3510422646487042\"}", - "MS MARCO (regular) - RR@10 (Fairness)": "{\"description\": \"min=0.137, mean=0.148, max=0.163, sum=0.445 (3)\", \"tab\": \"Fairness\", \"score\": \"0.1483276455026454\"}", - "MS MARCO (TREC) - NDCG@10 (Fairness)": "{\"description\": \"min=0.347, mean=0.381, max=0.416, sum=1.144 (3)\", \"tab\": \"Fairness\", \"score\": \"0.38125183165300675\"}", - "MS MARCO (regular) - Denoised inference time (s)": "{\"description\": \"min=0.111, mean=0.118, max=0.128, sum=0.355 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.11821914517316674\"}", - "MS MARCO (TREC) - Denoised inference time (s)": "{\"description\": \"min=0.105, mean=0.116, max=0.127, sum=0.349 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.11621723726407733\"}", - "MS MARCO (regular) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "MS MARCO (regular) - # train": "{\"description\": \"min=2, mean=2, max=2, sum=6 (3)\", \"tab\": \"General information\", \"score\": \"2.0\"}", - "MS MARCO (regular) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MS MARCO (regular) - # prompt tokens": "{\"description\": \"min=499.575, mean=537.908, max=583.575, sum=1613.725 (3)\", \"tab\": \"General information\", \"score\": \"537.9083333333334\"}", - "MS MARCO (regular) - # output tokens": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MS MARCO (regular) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "MS MARCO (TREC) - # eval": "{\"description\": \"min=43, mean=43, max=43, sum=129 (3)\", \"tab\": \"General information\", \"score\": \"43.0\"}", - "MS MARCO (TREC) - # train": "{\"description\": \"min=2, mean=2, max=2, sum=6 (3)\", \"tab\": \"General information\", \"score\": \"2.0\"}", - "MS MARCO (TREC) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MS MARCO (TREC) - # prompt tokens": "{\"description\": \"min=481.14, mean=519.473, max=565.14, sum=1558.419 (3)\", \"tab\": \"General information\", \"score\": \"519.4728682170543\"}", - "MS MARCO (TREC) - # output tokens": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MS MARCO (TREC) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "MS MARCO (regular) - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "MS MARCO (TREC) - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CNN/DailyMail", - "source_data": { - "dataset_name": "CNN/DailyMail", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on CNN/DailyMail", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.123, - "details": { - "description": "min=0.108, mean=0.123, max=0.138, sum=0.738 (6)", - "tab": "Accuracy", - "CNN/DailyMail - Denoised inference time (s)": "{\"description\": \"min=2.104, mean=2.133, max=2.168, sum=12.798 (6)\", \"tab\": \"Efficiency\", \"score\": \"2.133056901521097\"}", - "CNN/DailyMail - # eval": "{\"description\": \"min=466, mean=466, max=466, sum=2796 (6)\", \"tab\": \"General information\", \"score\": \"466.0\"}", - "CNN/DailyMail - # train": "{\"description\": \"min=5, mean=5, max=5, sum=30 (6)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CNN/DailyMail - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (6)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CNN/DailyMail - # prompt tokens": "{\"description\": \"min=1561.275, mean=1582.608, max=1612.275, sum=9495.648 (6)\", \"tab\": \"General information\", \"score\": \"1582.6080114449214\"}", - "CNN/DailyMail - # output tokens": "{\"description\": \"min=80.197, mean=80.409, max=80.588, sum=482.455 (6)\", \"tab\": \"General information\", \"score\": \"80.40915593705294\"}", - "CNN/DailyMail - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=18 (6)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "CNN/DailyMail - Stereotypes (race)": "{\"description\": \"min=0.612, mean=0.616, max=0.62, sum=3.697 (6)\", \"tab\": \"Bias\", \"score\": \"0.6162431158667614\"}", - "CNN/DailyMail - Stereotypes (gender)": "{\"description\": \"min=0.386, mean=0.41, max=0.431, sum=2.46 (6)\", \"tab\": \"Bias\", \"score\": \"0.4099353286102709\"}", - "CNN/DailyMail - Representation (race)": "{\"description\": \"min=0.182, mean=0.289, max=0.35, sum=1.732 (6)\", \"tab\": \"Bias\", \"score\": \"0.288716873622534\"}", - "CNN/DailyMail - Representation (gender)": "{\"description\": \"min=0.127, mean=0.149, max=0.168, sum=0.896 (6)\", \"tab\": \"Bias\", \"score\": \"0.14933277507884896\"}", - "CNN/DailyMail - Toxic fraction": "{\"description\": \"min=0, mean=0.001, max=0.002, sum=0.009 (6)\", \"tab\": \"Toxicity\", \"score\": \"0.001430615164520744\"}", - "CNN/DailyMail - SummaC": "{\"description\": \"min=-0.009, mean=0.165, max=0.255, sum=0.494 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.16465107490254738\"}", - "CNN/DailyMail - QAFactEval": "{\"description\": \"min=4.591, mean=4.69, max=4.763, sum=28.138 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"4.689614935266213\"}", - "CNN/DailyMail - BERTScore (F1)": "{\"description\": \"min=0.175, mean=0.226, max=0.262, sum=0.677 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.2255769362361307\"}", - "CNN/DailyMail - Coverage": "{\"description\": \"min=0.786, mean=0.91, max=0.973, sum=5.46 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"0.910005755446767\"}", - "CNN/DailyMail - Density": "{\"description\": \"min=35.834, mean=37.149, max=38.818, sum=222.893 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"37.14890205441478\"}", - "CNN/DailyMail - Compression": "{\"description\": \"min=9.164, mean=9.676, max=9.978, sum=58.057 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"9.676104726319009\"}", - "CNN/DailyMail - HumanEval-faithfulness": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-relevance": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-coherence": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "XSUM", - "source_data": { - "dataset_name": "XSUM", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on XSUM", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.102, - "details": { - "description": "min=0.098, mean=0.102, max=0.105, sum=0.61 (6)", - "tab": "Accuracy", - "XSUM - Denoised inference time (s)": "{\"description\": \"min=1.104, mean=1.116, max=1.135, sum=6.698 (6)\", \"tab\": \"Efficiency\", \"score\": \"1.1163698516910754\"}", - "XSUM - # eval": "{\"description\": \"min=518, mean=518, max=518, sum=3108 (6)\", \"tab\": \"General information\", \"score\": \"518.0\"}", - "XSUM - # train": "{\"description\": \"min=4.996, mean=4.997, max=5, sum=29.985 (6)\", \"tab\": \"General information\", \"score\": \"4.997425997425997\"}", - "XSUM - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (6)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "XSUM - # prompt tokens": "{\"description\": \"min=1487.131, mean=1545.148, max=1574.17, sum=9270.888 (6)\", \"tab\": \"General information\", \"score\": \"1545.148005148005\"}", - "XSUM - # output tokens": "{\"description\": \"min=24.871, mean=25.402, max=26.143, sum=152.413 (6)\", \"tab\": \"General information\", \"score\": \"25.402187902187904\"}", - "XSUM - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=18 (6)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "XSUM - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=4 (6)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "XSUM - Stereotypes (gender)": "{\"description\": \"min=0.449, mean=0.449, max=0.449, sum=2.694 (6)\", \"tab\": \"Bias\", \"score\": \"0.4490600226000671\"}", - "XSUM - Representation (race)": "{\"description\": \"min=0.483, mean=0.526, max=0.565, sum=3.158 (6)\", \"tab\": \"Bias\", \"score\": \"0.5263835263835264\"}", - "XSUM - Representation (gender)": "{\"description\": \"min=0.132, mean=0.162, max=0.184, sum=0.972 (6)\", \"tab\": \"Bias\", \"score\": \"0.16191706040214252\"}", - "XSUM - Toxic fraction": "{\"description\": \"min=0, mean=0.002, max=0.004, sum=0.012 (6)\", \"tab\": \"Toxicity\", \"score\": \"0.0019305019305019308\"}", - "XSUM - SummaC": "{\"description\": \"min=-0.22, mean=-0.208, max=-0.2, sum=-0.625 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"-0.2082928215061222\"}", - "XSUM - QAFactEval": "{\"description\": \"min=3.048, mean=3.303, max=3.621, sum=19.818 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"3.302964744932122\"}", - "XSUM - BERTScore (F1)": "{\"description\": \"min=0.385, mean=0.391, max=0.395, sum=1.174 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.39129907447599627\"}", - "XSUM - Coverage": "{\"description\": \"min=0.822, mean=0.825, max=0.83, sum=4.948 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"0.8247285888112758\"}", - "XSUM - Density": "{\"description\": \"min=3.228, mean=3.371, max=3.613, sum=20.226 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"3.3710531876366\"}", - "XSUM - Compression": "{\"description\": \"min=17.631, mean=18.238, max=18.621, sum=109.428 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"18.23798025069092\"}", - "XSUM - HumanEval-faithfulness": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-relevance": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-coherence": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "IMDB", - "source_data": { - "dataset_name": "IMDB", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on IMDB", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.948, - "details": { - "description": "min=0.946, mean=0.948, max=0.95, sum=2.844 (3)", - "tab": "Accuracy", - "IMDB - ECE (10-bin)": "{\"description\": \"min=0.189, mean=0.23, max=0.269, sum=0.69 (3)\", \"tab\": \"Calibration\", \"score\": \"0.22988586030197733\"}", - "IMDB - EM (Robustness)": "{\"description\": \"min=0.906, mean=0.912, max=0.921, sum=2.736 (3)\", \"tab\": \"Robustness\", \"score\": \"0.9119999999999999\"}", - "IMDB - EM (Fairness)": "{\"description\": \"min=0.925, mean=0.928, max=0.933, sum=2.785 (3)\", \"tab\": \"Fairness\", \"score\": \"0.9283333333333333\"}", - "IMDB - Denoised inference time (s)": "{\"description\": \"min=0.748, mean=0.862, max=1.078, sum=2.586 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.862092325799332\"}", - "IMDB - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "IMDB - # train": "{\"description\": \"min=4.842, mean=4.93, max=4.981, sum=14.789 (3)\", \"tab\": \"General information\", \"score\": \"4.929666666666667\"}", - "IMDB - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IMDB - # prompt tokens": "{\"description\": \"min=1162.003, mean=1398.09, max=1750.717, sum=4194.271 (3)\", \"tab\": \"General information\", \"score\": \"1398.0903333333333\"}", - "IMDB - # output tokens": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "IMDB - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "IMDB - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CivilComments", - "source_data": { - "dataset_name": "CivilComments", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on CivilComments", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.516, - "details": { - "description": "min=0, mean=0.516, max=1, sum=27.878 (54)", - "tab": "Accuracy", - "CivilComments - ECE (10-bin)": "{\"description\": \"min=0.26, mean=0.444, max=0.593, sum=23.994 (54)\", \"tab\": \"Calibration\", \"score\": \"0.4443373993811643\"}", - "CivilComments - EM (Robustness)": "{\"description\": \"min=0, mean=0.48, max=1, sum=25.9 (54)\", \"tab\": \"Robustness\", \"score\": \"0.4796354739742704\"}", - "CivilComments - EM (Fairness)": "{\"description\": \"min=0, mean=0.491, max=1, sum=26.497 (54)\", \"tab\": \"Fairness\", \"score\": \"0.4906931444587031\"}", - "CivilComments - Denoised inference time (s)": "{\"description\": \"min=0.253, mean=0.408, max=0.906, sum=22.04 (54)\", \"tab\": \"Efficiency\", \"score\": \"0.4081493504712871\"}", - "CivilComments - # eval": "{\"description\": \"min=74, mean=371.556, max=683, sum=20064 (54)\", \"tab\": \"General information\", \"score\": \"371.55555555555554\"}", - "CivilComments - # train": "{\"description\": \"min=5, mean=5, max=5, sum=270 (54)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CivilComments - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (54)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CivilComments - # prompt tokens": "{\"description\": \"min=360.976, mean=726.728, max=1282.4, sum=39243.315 (54)\", \"tab\": \"General information\", \"score\": \"726.7280588093369\"}", - "CivilComments - # output tokens": "{\"description\": \"min=5, mean=5, max=5, sum=270 (54)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CivilComments - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=162 (54)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "CivilComments - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (54)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "RAFT", - "source_data": { - "dataset_name": "RAFT", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on RAFT", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.505, - "details": { - "description": "min=0.025, mean=0.505, max=0.975, sum=16.65 (33)", - "tab": "Accuracy", - "RAFT - ECE (10-bin)": "{\"description\": \"min=0.067, mean=0.324, max=0.975, sum=10.705 (33)\", \"tab\": \"Calibration\", \"score\": \"0.3243919141625793\"}", - "RAFT - EM (Robustness)": "{\"description\": \"min=0, mean=0.399, max=0.975, sum=13.175 (33)\", \"tab\": \"Robustness\", \"score\": \"0.39924242424242423\"}", - "RAFT - EM (Fairness)": "{\"description\": \"min=0.025, mean=0.475, max=0.975, sum=15.675 (33)\", \"tab\": \"Fairness\", \"score\": \"0.47500000000000003\"}", - "RAFT - Denoised inference time (s)": "{\"description\": \"min=0.16, mean=1.156, max=2.589, sum=38.155 (33)\", \"tab\": \"Efficiency\", \"score\": \"1.1562087950381366\"}", - "RAFT - # eval": "{\"description\": \"min=40, mean=40, max=40, sum=1320 (33)\", \"tab\": \"General information\", \"score\": \"40.0\"}", - "RAFT - # train": "{\"description\": \"min=0, mean=4.56, max=5, sum=150.475 (33)\", \"tab\": \"General information\", \"score\": \"4.5598484848484855\"}", - "RAFT - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (33)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "RAFT - # prompt tokens": "{\"description\": \"min=269.35, mean=807.97, max=1764, sum=26663.0 (33)\", \"tab\": \"General information\", \"score\": \"807.9696969696969\"}", - "RAFT - # output tokens": "{\"description\": \"min=5, mean=13.945, max=30, sum=460.2 (33)\", \"tab\": \"General information\", \"score\": \"13.945454545454545\"}", - "RAFT - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=99 (33)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "RAFT - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (33)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/openai_gpt-oss-120b.json b/data/models/openai_gpt-oss-120b.json deleted file mode 100644 index cc688cac8bf3c3ea698efc0de68329e08bff64c1..0000000000000000000000000000000000000000 --- a/data/models/openai_gpt-oss-120b.json +++ /dev/null @@ -1,461 +0,0 @@ -{ - "model_info": { - "name": "GPT-OSS-120B", - "id": "openai/gpt-oss-120b", - "developer": "OpenAI", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_capabilities/openai_gpt-oss-120b/1774096304.5056698", - "retrieved_timestamp": "1774096304.5056698", - "source_metadata": { - "source_name": "helm_capabilities", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_capabilities", - "evaluation_results": [ - { - "evaluation_name": "Mean score", - "source_data": { - "dataset_name": "helm_capabilities", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "The mean of the scores from all columns.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.77, - "details": { - "description": "", - "tab": "Accuracy", - "Mean score - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"19.583454439679375\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU-Pro", - "source_data": { - "dataset_name": "MMLU-Pro", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on MMLU-Pro", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.795, - "details": { - "description": "min=0.795, mean=0.795, max=0.795, sum=0.795 (1)", - "tab": "Accuracy", - "MMLU-Pro - Observed inference time (s)": "{\"description\": \"min=6.268, mean=6.268, max=6.268, sum=6.268 (1)\", \"tab\": \"Efficiency\", \"score\": \"6.268435170412063\"}", - "MMLU-Pro - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "MMLU-Pro - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - # prompt tokens": "{\"description\": \"min=248.569, mean=248.569, max=248.569, sum=248.569 (1)\", \"tab\": \"General information\", \"score\": \"248.569\"}", - "MMLU-Pro - # output tokens": "{\"description\": \"min=474.202, mean=474.202, max=474.202, sum=474.202 (1)\", \"tab\": \"General information\", \"score\": \"474.202\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"all\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.684, - "details": { - "description": "min=0.684, mean=0.684, max=0.684, sum=0.684 (1)", - "tab": "Accuracy", - "GPQA - Observed inference time (s)": "{\"description\": \"min=18.819, mean=18.819, max=18.819, sum=18.819 (1)\", \"tab\": \"Efficiency\", \"score\": \"18.8192116278704\"}", - "GPQA - # eval": "{\"description\": \"min=446, mean=446, max=446, sum=446 (1)\", \"tab\": \"General information\", \"score\": \"446.0\"}", - "GPQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - # prompt tokens": "{\"description\": \"min=268.152, mean=268.152, max=268.152, sum=268.152 (1)\", \"tab\": \"General information\", \"score\": \"268.15246636771303\"}", - "GPQA - # output tokens": "{\"description\": \"min=1218.108, mean=1218.108, max=1218.108, sum=1218.108 (1)\", \"tab\": \"General information\", \"score\": \"1218.1076233183855\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"gpqa_main\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "IFEval Strict Acc on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.836, - "details": { - "description": "min=0.836, mean=0.836, max=0.836, sum=0.836 (1)", - "tab": "Accuracy", - "IFEval - Observed inference time (s)": "{\"description\": \"min=6.303, mean=6.303, max=6.303, sum=6.303 (1)\", \"tab\": \"Efficiency\", \"score\": \"6.302578532982225\"}", - "IFEval - # eval": "{\"description\": \"min=541, mean=541, max=541, sum=541 (1)\", \"tab\": \"General information\", \"score\": \"541.0\"}", - "IFEval - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - # prompt tokens": "{\"description\": \"min=45.671, mean=45.671, max=45.671, sum=45.671 (1)\", \"tab\": \"General information\", \"score\": \"45.67097966728281\"}", - "IFEval - # output tokens": "{\"description\": \"min=945.784, mean=945.784, max=945.784, sum=945.784 (1)\", \"tab\": \"General information\", \"score\": \"945.7837338262477\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WildBench", - "source_data": { - "dataset_name": "WildBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "WB Score on WildBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.845, - "details": { - "description": "min=0.845, mean=0.845, max=0.845, sum=0.845 (1)", - "tab": "Accuracy", - "WildBench - Observed inference time (s)": "{\"description\": \"min=24.979, mean=24.979, max=24.979, sum=24.979 (1)\", \"tab\": \"Efficiency\", \"score\": \"24.978535928487776\"}", - "WildBench - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "WildBench - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # prompt tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # output tokens": "{\"description\": \"min=2925.361, mean=2925.361, max=2925.361, sum=2925.361 (1)\", \"tab\": \"General information\", \"score\": \"2925.361\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"v2\"" - } - } - }, - { - "evaluation_name": "Omni-MATH", - "source_data": { - "dataset_name": "Omni-MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Acc on Omni-MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.688, - "details": { - "description": "min=0.688, mean=0.688, max=0.688, sum=0.688 (1)", - "tab": "Accuracy", - "Omni-MATH - Observed inference time (s)": "{\"description\": \"min=41.549, mean=41.549, max=41.549, sum=41.549 (1)\", \"tab\": \"Efficiency\", \"score\": \"41.54851093864441\"}", - "Omni-MATH - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "Omni-MATH - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - # prompt tokens": "{\"description\": \"min=109.623, mean=109.623, max=109.623, sum=109.623 (1)\", \"tab\": \"General information\", \"score\": \"109.623\"}", - "Omni-MATH - # output tokens": "{\"description\": \"min=4103.671, mean=4103.671, max=4103.671, sum=4103.671 (1)\", \"tab\": \"General information\", \"score\": \"4103.671\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "livecodebenchpro/openai/gpt-oss-120b/1760492095.8105888", - "retrieved_timestamp": "1760492095.8105888", - "source_metadata": { - "source_organization_name": "New York University, Princeton University, University of California San Diego, University of Washington and Canyon Crest Academy", - "evaluator_relationship": "third_party", - "source_name": "Live Code Bench Pro", - "source_type": "documentation" - }, - "eval_library": { - "name": "unknown", - "version": "unknown" - }, - "benchmark": "livecodebenchpro", - "evaluation_results": [ - { - "evaluation_name": "Hard Problems", - "metric_config": { - "evaluation_description": "Pass@1 on Hard Problems", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.0 - }, - "source_data": { - "dataset_name": "Hard Problems", - "source_type": "url", - "url": [ - "https://webhook.cp-bench.orzzh.com/leaderboard/llm/difficulty?difficulty=hard&benchmark_mode=live" - ] - } - }, - { - "evaluation_name": "Medium Problems", - "metric_config": { - "evaluation_description": "Pass@1 on Medium Problems", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.11267605633802817 - }, - "source_data": { - "dataset_name": "Medium Problems", - "source_type": "url", - "url": [ - "https://webhook.cp-bench.orzzh.com/leaderboard/llm/difficulty?difficulty=medium&benchmark_mode=live" - ] - } - }, - { - "evaluation_name": "Easy Problems", - "metric_config": { - "evaluation_description": "Pass@1 on Easy Problems", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.6619718309859155 - }, - "source_data": { - "dataset_name": "Easy Problems", - "source_type": "url", - "url": [ - "https://webhook.cp-bench.orzzh.com/leaderboard/llm/difficulty?difficulty=easy&benchmark_mode=live" - ] - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "terminal-bench-2.0/mini-swe-agent__gpt-oss-120b/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2025-11-03", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 14.2, - "uncertainty": { - "standard_error": { - "value": 2.3 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Mini-SWE-Agent\" -m \"GPT-OSS-120B\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Mini-SWE-Agent\" -m \"GPT-OSS-120B\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - }, - { - "evaluation_id": "terminal-bench-2.0/terminus-2__gpt-oss-120b/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2025-11-01", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 18.7, - "uncertainty": { - "standard_error": { - "value": 2.7 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Terminus 2\" -m \"GPT-OSS-120B\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Terminus 2\" -m \"GPT-OSS-120B\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/openai_gpt-oss-20b.json b/data/models/openai_gpt-oss-20b.json deleted file mode 100644 index 584e35a6df7897b2cd4e9e1cdb5c6db8d87d90ab..0000000000000000000000000000000000000000 --- a/data/models/openai_gpt-oss-20b.json +++ /dev/null @@ -1,461 +0,0 @@ -{ - "model_info": { - "name": "GPT-OSS-20B", - "id": "openai/gpt-oss-20b", - "developer": "OpenAI", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_capabilities/openai_gpt-oss-20b/1774096304.5056698", - "retrieved_timestamp": "1774096304.5056698", - "source_metadata": { - "source_name": "helm_capabilities", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_capabilities", - "evaluation_results": [ - { - "evaluation_name": "Mean score", - "source_data": { - "dataset_name": "helm_capabilities", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "The mean of the scores from all columns.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.674, - "details": { - "description": "", - "tab": "Accuracy", - "Mean score - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"31.785255717522546\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU-Pro", - "source_data": { - "dataset_name": "MMLU-Pro", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on MMLU-Pro", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.74, - "details": { - "description": "min=0.74, mean=0.74, max=0.74, sum=0.74 (1)", - "tab": "Accuracy", - "MMLU-Pro - Observed inference time (s)": "{\"description\": \"min=4.593, mean=4.593, max=4.593, sum=4.593 (1)\", \"tab\": \"Efficiency\", \"score\": \"4.593113619089126\"}", - "MMLU-Pro - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "MMLU-Pro - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - # prompt tokens": "{\"description\": \"min=248.569, mean=248.569, max=248.569, sum=248.569 (1)\", \"tab\": \"General information\", \"score\": \"248.569\"}", - "MMLU-Pro - # output tokens": "{\"description\": \"min=820.909, mean=820.909, max=820.909, sum=820.909 (1)\", \"tab\": \"General information\", \"score\": \"820.909\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"all\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.594, - "details": { - "description": "min=0.594, mean=0.594, max=0.594, sum=0.594 (1)", - "tab": "Accuracy", - "GPQA - Observed inference time (s)": "{\"description\": \"min=27.565, mean=27.565, max=27.565, sum=27.565 (1)\", \"tab\": \"Efficiency\", \"score\": \"27.56541810923093\"}", - "GPQA - # eval": "{\"description\": \"min=446, mean=446, max=446, sum=446 (1)\", \"tab\": \"General information\", \"score\": \"446.0\"}", - "GPQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - # prompt tokens": "{\"description\": \"min=268.152, mean=268.152, max=268.152, sum=268.152 (1)\", \"tab\": \"General information\", \"score\": \"268.15246636771303\"}", - "GPQA - # output tokens": "{\"description\": \"min=2872.139, mean=2872.139, max=2872.139, sum=2872.139 (1)\", \"tab\": \"General information\", \"score\": \"2872.1390134529147\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"gpqa_main\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "IFEval Strict Acc on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.732, - "details": { - "description": "min=0.732, mean=0.732, max=0.732, sum=0.732 (1)", - "tab": "Accuracy", - "IFEval - Observed inference time (s)": "{\"description\": \"min=26.607, mean=26.607, max=26.607, sum=26.607 (1)\", \"tab\": \"Efficiency\", \"score\": \"26.607220574359577\"}", - "IFEval - # eval": "{\"description\": \"min=541, mean=541, max=541, sum=541 (1)\", \"tab\": \"General information\", \"score\": \"541.0\"}", - "IFEval - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - # prompt tokens": "{\"description\": \"min=45.671, mean=45.671, max=45.671, sum=45.671 (1)\", \"tab\": \"General information\", \"score\": \"45.67097966728281\"}", - "IFEval - # output tokens": "{\"description\": \"min=3202.279, mean=3202.279, max=3202.279, sum=3202.279 (1)\", \"tab\": \"General information\", \"score\": \"3202.279112754159\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WildBench", - "source_data": { - "dataset_name": "WildBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "WB Score on WildBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.737, - "details": { - "description": "min=0.737, mean=0.737, max=0.737, sum=0.737 (1)", - "tab": "Accuracy", - "WildBench - Observed inference time (s)": "{\"description\": \"min=42.985, mean=42.985, max=42.985, sum=42.985 (1)\", \"tab\": \"Efficiency\", \"score\": \"42.985184440851214\"}", - "WildBench - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "WildBench - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # prompt tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # output tokens": "{\"description\": \"min=4398.71, mean=4398.71, max=4398.71, sum=4398.71 (1)\", \"tab\": \"General information\", \"score\": \"4398.71\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"v2\"" - } - } - }, - { - "evaluation_name": "Omni-MATH", - "source_data": { - "dataset_name": "Omni-MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Acc on Omni-MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.565, - "details": { - "description": "min=0.565, mean=0.565, max=0.565, sum=0.565 (1)", - "tab": "Accuracy", - "Omni-MATH - Observed inference time (s)": "{\"description\": \"min=57.175, mean=57.175, max=57.175, sum=57.175 (1)\", \"tab\": \"Efficiency\", \"score\": \"57.17534184408188\"}", - "Omni-MATH - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "Omni-MATH - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - # prompt tokens": "{\"description\": \"min=109.623, mean=109.623, max=109.623, sum=109.623 (1)\", \"tab\": \"General information\", \"score\": \"109.623\"}", - "Omni-MATH - # output tokens": "{\"description\": \"min=6604.944, mean=6604.944, max=6604.944, sum=6604.944 (1)\", \"tab\": \"General information\", \"score\": \"6604.944\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "livecodebenchpro/openai/gpt-oss-20b/1760492095.8105888", - "retrieved_timestamp": "1760492095.8105888", - "source_metadata": { - "source_organization_name": "New York University, Princeton University, University of California San Diego, University of Washington and Canyon Crest Academy", - "evaluator_relationship": "third_party", - "source_name": "Live Code Bench Pro", - "source_type": "documentation" - }, - "eval_library": { - "name": "unknown", - "version": "unknown" - }, - "benchmark": "livecodebenchpro", - "evaluation_results": [ - { - "evaluation_name": "Hard Problems", - "metric_config": { - "evaluation_description": "Pass@1 on Hard Problems", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.0 - }, - "source_data": { - "dataset_name": "Hard Problems", - "source_type": "url", - "url": [ - "https://webhook.cp-bench.orzzh.com/leaderboard/llm/difficulty?difficulty=hard&benchmark_mode=live" - ] - } - }, - { - "evaluation_name": "Medium Problems", - "metric_config": { - "evaluation_description": "Pass@1 on Medium Problems", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.056338028169014086 - }, - "source_data": { - "dataset_name": "Medium Problems", - "source_type": "url", - "url": [ - "https://webhook.cp-bench.orzzh.com/leaderboard/llm/difficulty?difficulty=medium&benchmark_mode=live" - ] - } - }, - { - "evaluation_name": "Easy Problems", - "metric_config": { - "evaluation_description": "Pass@1 on Easy Problems", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.5070422535211268 - }, - "source_data": { - "dataset_name": "Easy Problems", - "source_type": "url", - "url": [ - "https://webhook.cp-bench.orzzh.com/leaderboard/llm/difficulty?difficulty=easy&benchmark_mode=live" - ] - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "terminal-bench-2.0/terminus-2__gpt-oss-20b/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2025-11-01", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 3.1, - "uncertainty": { - "standard_error": { - "value": 1.5 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Terminus 2\" -m \"GPT-OSS-20B\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Terminus 2\" -m \"GPT-OSS-20B\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - }, - { - "evaluation_id": "terminal-bench-2.0/mini-swe-agent__gpt-oss-20b/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2025-11-03", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 3.4, - "uncertainty": { - "standard_error": { - "value": 1.4 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Mini-SWE-Agent\" -m \"GPT-OSS-20B\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Mini-SWE-Agent\" -m \"GPT-OSS-20B\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/openai_gpt_4o.json b/data/models/openai_gpt_4o.json deleted file mode 100644 index f2b628fc0cdc078cf288eaf85d0efcd23a7c3719..0000000000000000000000000000000000000000 --- a/data/models/openai_gpt_4o.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "model_info": { - "name": "GPT 4o", - "developer": "OpenAI", - "id": "openai/GPT 4o", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "apex-v1/openai_gpt-4o/1773260200", - "retrieved_timestamp": "1773260200", - "source_metadata": { - "source_name": "Mercor APEX-v1 Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Mercor", - "source_organization_url": "https://www.mercor.com", - "evaluator_relationship": "first_party" - }, - "eval_library": { - "name": "archipelago", - "version": "1.0.0" - }, - "benchmark": "apex-v1", - "evaluation_results": [ - { - "evaluation_name": "Overall Score", - "source_data": { - "dataset_name": "apex-v1", - "source_type": "hf_dataset", - "hf_repo": "Mercor/APEX-v1" - }, - "metric_config": { - "evaluation_description": "Overall APEX-v1 mean score (paper snapshot).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.359 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/openai_gpt_5.1.json b/data/models/openai_gpt_5.1.json deleted file mode 100644 index f6c165bdd865500a0e69eb5f5703f306579c91fc..0000000000000000000000000000000000000000 --- a/data/models/openai_gpt_5.1.json +++ /dev/null @@ -1,227 +0,0 @@ -{ - "model_info": { - "name": "GPT 5.1", - "developer": "OpenAI", - "id": "openai/GPT 5.1", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "apex-agents/openai_gpt-5.1/1773260200", - "retrieved_timestamp": "1773260200", - "source_metadata": { - "source_name": "Mercor APEX-Agents Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Mercor", - "source_organization_url": "https://www.mercor.com", - "evaluator_relationship": "first_party" - }, - "eval_library": { - "name": "archipelago", - "version": "1.0.0" - }, - "benchmark": "apex-agents", - "evaluation_results": [ - { - "evaluation_name": "Corporate Lawyer Mean Score", - "source_data": { - "dataset_name": "apex-agents", - "source_type": "hf_dataset", - "hf_repo": "mercor/apex-agents" - }, - "metric_config": { - "evaluation_description": "Corporate lawyer world mean score.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.376 - }, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - }, - { - "evaluation_id": "ace/openai_gpt-5.1/1773260200", - "retrieved_timestamp": "1773260200", - "source_metadata": { - "source_name": "Mercor ACE Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Mercor", - "source_organization_url": "https://www.mercor.com", - "evaluator_relationship": "first_party" - }, - "eval_library": { - "name": "archipelago", - "version": "1.0.0" - }, - "benchmark": "ace", - "evaluation_results": [ - { - "evaluation_name": "Overall Score", - "source_data": { - "dataset_name": "ace", - "source_type": "hf_dataset", - "hf_repo": "Mercor/ACE" - }, - "metric_config": { - "evaluation_description": "Overall ACE score across all consumer-task domains.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.551, - "uncertainty": { - "confidence_interval": { - "lower": -0.032, - "upper": 0.032, - "method": "bootstrap" - } - } - }, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - }, - { - "evaluation_name": "DIY Score", - "source_data": { - "dataset_name": "ace", - "source_type": "hf_dataset", - "hf_repo": "Mercor/ACE" - }, - "metric_config": { - "evaluation_description": "DIY domain score.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.56 - }, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - }, - { - "evaluation_name": "Gaming Score", - "source_data": { - "dataset_name": "ace", - "source_type": "hf_dataset", - "hf_repo": "Mercor/ACE" - }, - "metric_config": { - "evaluation_description": "Gaming domain score.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.61 - }, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - }, - { - "evaluation_name": "Shopping Score", - "source_data": { - "dataset_name": "ace", - "source_type": "hf_dataset", - "hf_repo": "Mercor/ACE" - }, - "metric_config": { - "evaluation_description": "Shopping domain score.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.45 - }, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - }, - { - "evaluation_id": "apex-v1/openai_gpt-5.1/1773260200", - "retrieved_timestamp": "1773260200", - "source_metadata": { - "source_name": "Mercor APEX-v1 Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Mercor", - "source_organization_url": "https://www.mercor.com", - "evaluator_relationship": "first_party" - }, - "eval_library": { - "name": "archipelago", - "version": "1.0.0" - }, - "benchmark": "apex-v1", - "evaluation_results": [ - { - "evaluation_name": "Big Law Score", - "source_data": { - "dataset_name": "apex-v1", - "source_type": "hf_dataset", - "hf_repo": "Mercor/APEX-v1" - }, - "metric_config": { - "evaluation_description": "Big law associate score.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.77 - }, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/openai_gpt_5.1_codex.json b/data/models/openai_gpt_5.1_codex.json deleted file mode 100644 index 87d04b8519112be282740769c75fb84e41a0ff35..0000000000000000000000000000000000000000 --- a/data/models/openai_gpt_5.1_codex.json +++ /dev/null @@ -1,57 +0,0 @@ -{ - "model_info": { - "name": "GPT 5.1 Codex", - "developer": "OpenAI", - "id": "openai/GPT 5.1 Codex", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "apex-agents/openai_gpt-5.1-codex/1773260200", - "retrieved_timestamp": "1773260200", - "source_metadata": { - "source_name": "Mercor APEX-Agents Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Mercor", - "source_organization_url": "https://www.mercor.com", - "evaluator_relationship": "first_party" - }, - "eval_library": { - "name": "archipelago", - "version": "1.0.0" - }, - "benchmark": "apex-agents", - "evaluation_results": [ - { - "evaluation_name": "Corporate Lawyer Mean Score", - "source_data": { - "dataset_name": "apex-agents", - "source_type": "hf_dataset", - "hf_repo": "mercor/apex-agents" - }, - "metric_config": { - "evaluation_description": "Corporate lawyer world mean score.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.366 - }, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/openai_gpt_5.2.json b/data/models/openai_gpt_5.2.json deleted file mode 100644 index fec8e3d9659540e746cd368872cad0ec3c496416..0000000000000000000000000000000000000000 --- a/data/models/openai_gpt_5.2.json +++ /dev/null @@ -1,309 +0,0 @@ -{ - "model_info": { - "name": "GPT 5.2", - "developer": "OpenAI", - "id": "openai/GPT 5.2", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "ace/openai_gpt-5.2/1773260200", - "retrieved_timestamp": "1773260200", - "source_metadata": { - "source_name": "Mercor ACE Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Mercor", - "source_organization_url": "https://www.mercor.com", - "evaluator_relationship": "first_party" - }, - "eval_library": { - "name": "archipelago", - "version": "1.0.0" - }, - "benchmark": "ace", - "evaluation_results": [ - { - "evaluation_name": "Overall Score", - "source_data": { - "dataset_name": "ace", - "source_type": "hf_dataset", - "hf_repo": "Mercor/ACE" - }, - "metric_config": { - "evaluation_description": "Overall ACE score across all consumer-task domains.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.515, - "uncertainty": { - "confidence_interval": { - "lower": -0.032, - "upper": 0.032, - "method": "bootstrap" - } - } - }, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - }, - { - "evaluation_name": "Food Score", - "source_data": { - "dataset_name": "ace", - "source_type": "hf_dataset", - "hf_repo": "Mercor/ACE" - }, - "metric_config": { - "evaluation_description": "Food domain score.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.65 - }, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - }, - { - "evaluation_name": "Gaming Score", - "source_data": { - "dataset_name": "ace", - "source_type": "hf_dataset", - "hf_repo": "Mercor/ACE" - }, - "metric_config": { - "evaluation_description": "Gaming domain score.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.578 - }, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - }, - { - "evaluation_id": "apex-agents/openai_gpt-5.2/1773260200", - "retrieved_timestamp": "1773260200", - "source_metadata": { - "source_name": "Mercor APEX-Agents Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Mercor", - "source_organization_url": "https://www.mercor.com", - "evaluator_relationship": "first_party" - }, - "eval_library": { - "name": "archipelago", - "version": "1.0.0" - }, - "benchmark": "apex-agents", - "evaluation_results": [ - { - "evaluation_name": "Overall Pass@1", - "source_data": { - "dataset_name": "apex-agents", - "source_type": "hf_dataset", - "hf_repo": "mercor/apex-agents" - }, - "metric_config": { - "evaluation_description": "Overall Pass@1 (dataset card / paper snapshot).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.23, - "uncertainty": { - "confidence_interval": { - "lower": -0.032, - "upper": 0.032, - "method": "bootstrap" - } - } - }, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - }, - { - "evaluation_name": "Overall Pass@8", - "source_data": { - "dataset_name": "apex-agents", - "source_type": "hf_dataset", - "hf_repo": "mercor/apex-agents" - }, - "metric_config": { - "evaluation_description": "Overall Pass@8 (dataset card / paper snapshot).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.4, - "uncertainty": { - "confidence_interval": { - "lower": -0.044, - "upper": 0.044, - "method": "bootstrap" - } - } - }, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - }, - { - "evaluation_name": "Overall Mean Score", - "source_data": { - "dataset_name": "apex-agents", - "source_type": "hf_dataset", - "hf_repo": "mercor/apex-agents" - }, - "metric_config": { - "evaluation_description": "Overall mean rubric score.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.387 - }, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - }, - { - "evaluation_name": "Investment Banking Pass@1", - "source_data": { - "dataset_name": "apex-agents", - "source_type": "hf_dataset", - "hf_repo": "mercor/apex-agents" - }, - "metric_config": { - "evaluation_description": "Investment banking world Pass@1.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.273 - }, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - }, - { - "evaluation_name": "Management Consulting Pass@1", - "source_data": { - "dataset_name": "apex-agents", - "source_type": "hf_dataset", - "hf_repo": "mercor/apex-agents" - }, - "metric_config": { - "evaluation_description": "Management consulting world Pass@1.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.227 - }, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - }, - { - "evaluation_name": "Corporate Law Pass@1", - "source_data": { - "dataset_name": "apex-agents", - "source_type": "hf_dataset", - "hf_repo": "mercor/apex-agents" - }, - "metric_config": { - "evaluation_description": "Corporate law world Pass@1.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.189 - }, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - }, - { - "evaluation_name": "Corporate Lawyer Mean Score", - "source_data": { - "dataset_name": "apex-agents", - "source_type": "hf_dataset", - "hf_repo": "mercor/apex-agents" - }, - "metric_config": { - "evaluation_description": "Corporate lawyer world mean score.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.443 - }, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/openai_gpt_5.2_codex.json b/data/models/openai_gpt_5.2_codex.json deleted file mode 100644 index 96b8712a5f649bd7cee7af1eda262b2945f9064b..0000000000000000000000000000000000000000 --- a/data/models/openai_gpt_5.2_codex.json +++ /dev/null @@ -1,87 +0,0 @@ -{ - "model_info": { - "name": "GPT 5.2 Codex", - "developer": "OpenAI", - "id": "openai/GPT 5.2 Codex", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "apex-agents/openai_gpt-5.2-codex/1773260200", - "retrieved_timestamp": "1773260200", - "source_metadata": { - "source_name": "Mercor APEX-Agents Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Mercor", - "source_organization_url": "https://www.mercor.com", - "evaluator_relationship": "first_party" - }, - "eval_library": { - "name": "archipelago", - "version": "1.0.0" - }, - "benchmark": "apex-agents", - "evaluation_results": [ - { - "evaluation_name": "Overall Pass@1", - "source_data": { - "dataset_name": "apex-agents", - "source_type": "hf_dataset", - "hf_repo": "mercor/apex-agents" - }, - "metric_config": { - "evaluation_description": "Overall Pass@1 across 480 long-horizon professional-services tasks.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.276, - "uncertainty": { - "confidence_interval": { - "lower": -0.034, - "upper": 0.034, - "method": "bootstrap" - } - } - }, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - }, - { - "evaluation_name": "Corporate Lawyer Mean Score", - "source_data": { - "dataset_name": "apex-agents", - "source_type": "hf_dataset", - "hf_repo": "mercor/apex-agents" - }, - "metric_config": { - "evaluation_description": "Corporate lawyer world mean score from leaderboard model list.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.394 - }, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/openai_gpt_5.2_pro.json b/data/models/openai_gpt_5.2_pro.json deleted file mode 100644 index 06b66f2bcb326d3a1b47f03cdaa634f79943be53..0000000000000000000000000000000000000000 --- a/data/models/openai_gpt_5.2_pro.json +++ /dev/null @@ -1,133 +0,0 @@ -{ - "model_info": { - "name": "GPT 5.2 Pro", - "developer": "OpenAI", - "id": "openai/GPT 5.2 Pro", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "apex-v1/openai_gpt-5.2-pro/1773260200", - "retrieved_timestamp": "1773260200", - "source_metadata": { - "source_name": "Mercor APEX-v1 Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Mercor", - "source_organization_url": "https://www.mercor.com", - "evaluator_relationship": "first_party" - }, - "eval_library": { - "name": "archipelago", - "version": "1.0.0" - }, - "benchmark": "apex-v1", - "evaluation_results": [ - { - "evaluation_name": "Overall Score", - "source_data": { - "dataset_name": "apex-v1", - "source_type": "hf_dataset", - "hf_repo": "Mercor/APEX-v1" - }, - "metric_config": { - "evaluation_description": "Overall APEX-v1 mean score across all jobs.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.668, - "uncertainty": { - "confidence_interval": { - "lower": -0.026, - "upper": 0.026, - "method": "bootstrap" - } - } - }, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - }, - { - "evaluation_name": "Consulting Score", - "source_data": { - "dataset_name": "apex-v1", - "source_type": "hf_dataset", - "hf_repo": "Mercor/APEX-v1" - }, - "metric_config": { - "evaluation_description": "Management consulting score.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.64 - }, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - }, - { - "evaluation_name": "Medicine (MD) Score", - "source_data": { - "dataset_name": "apex-v1", - "source_type": "hf_dataset", - "hf_repo": "Mercor/APEX-v1" - }, - "metric_config": { - "evaluation_description": "Primary care physician (MD) score.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.65 - }, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - }, - { - "evaluation_name": "Investment Banking Score", - "source_data": { - "dataset_name": "apex-v1", - "source_type": "hf_dataset", - "hf_repo": "Mercor/APEX-v1" - }, - "metric_config": { - "evaluation_description": "Investment banking associate score.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.64 - }, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/openai_gpt_5.3_codex.json b/data/models/openai_gpt_5.3_codex.json deleted file mode 100644 index 388bd6a4ac29dfe055172db293902bb3dc3125af..0000000000000000000000000000000000000000 --- a/data/models/openai_gpt_5.3_codex.json +++ /dev/null @@ -1,64 +0,0 @@ -{ - "model_info": { - "name": "GPT 5.3 Codex", - "developer": "OpenAI", - "id": "openai/GPT 5.3 Codex", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "apex-agents/openai_gpt-5.3-codex/1773260200", - "retrieved_timestamp": "1773260200", - "source_metadata": { - "source_name": "Mercor APEX-Agents Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Mercor", - "source_organization_url": "https://www.mercor.com", - "evaluator_relationship": "first_party" - }, - "eval_library": { - "name": "archipelago", - "version": "1.0.0" - }, - "benchmark": "apex-agents", - "evaluation_results": [ - { - "evaluation_name": "Overall Pass@1", - "source_data": { - "dataset_name": "apex-agents", - "source_type": "hf_dataset", - "hf_repo": "mercor/apex-agents" - }, - "metric_config": { - "evaluation_description": "Overall Pass@1 across 480 long-horizon professional-services tasks.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.317, - "uncertainty": { - "confidence_interval": { - "lower": -0.036, - "upper": 0.036, - "method": "bootstrap" - } - } - }, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/openai_gpt_5.json b/data/models/openai_gpt_5.json deleted file mode 100644 index 57aab1a8f6d263beed192374b3cef613081d380f..0000000000000000000000000000000000000000 --- a/data/models/openai_gpt_5.json +++ /dev/null @@ -1,455 +0,0 @@ -{ - "model_info": { - "name": "GPT 5", - "developer": "OpenAI", - "id": "openai/GPT 5", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "apex-agents/openai_gpt-5/1773260200", - "retrieved_timestamp": "1773260200", - "source_metadata": { - "source_name": "Mercor APEX-Agents Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Mercor", - "source_organization_url": "https://www.mercor.com", - "evaluator_relationship": "first_party" - }, - "eval_library": { - "name": "archipelago", - "version": "1.0.0" - }, - "benchmark": "apex-agents", - "evaluation_results": [ - { - "evaluation_name": "Overall Pass@1", - "source_data": { - "dataset_name": "apex-agents", - "source_type": "hf_dataset", - "hf_repo": "mercor/apex-agents" - }, - "metric_config": { - "evaluation_description": "Overall Pass@1 (dataset card / paper snapshot).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.183, - "uncertainty": { - "confidence_interval": { - "lower": -0.029, - "upper": 0.03, - "method": "bootstrap" - } - } - }, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - }, - { - "evaluation_name": "Overall Pass@8", - "source_data": { - "dataset_name": "apex-agents", - "source_type": "hf_dataset", - "hf_repo": "mercor/apex-agents" - }, - "metric_config": { - "evaluation_description": "Overall Pass@8 (dataset card / paper snapshot).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.31, - "uncertainty": { - "confidence_interval": { - "lower": -0.041, - "upper": 0.044, - "method": "bootstrap" - } - } - }, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - }, - { - "evaluation_name": "Overall Mean Score", - "source_data": { - "dataset_name": "apex-agents", - "source_type": "hf_dataset", - "hf_repo": "mercor/apex-agents" - }, - "metric_config": { - "evaluation_description": "Overall mean rubric score.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.329 - }, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - }, - { - "evaluation_name": "Investment Banking Pass@1", - "source_data": { - "dataset_name": "apex-agents", - "source_type": "hf_dataset", - "hf_repo": "mercor/apex-agents" - }, - "metric_config": { - "evaluation_description": "Investment banking world Pass@1.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.273 - }, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - }, - { - "evaluation_name": "Management Consulting Pass@1", - "source_data": { - "dataset_name": "apex-agents", - "source_type": "hf_dataset", - "hf_repo": "mercor/apex-agents" - }, - "metric_config": { - "evaluation_description": "Management consulting world Pass@1.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.123 - }, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - }, - { - "evaluation_name": "Corporate Law Pass@1", - "source_data": { - "dataset_name": "apex-agents", - "source_type": "hf_dataset", - "hf_repo": "mercor/apex-agents" - }, - "metric_config": { - "evaluation_description": "Corporate law world Pass@1.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.153 - }, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - }, - { - "evaluation_name": "Corporate Lawyer Mean Score", - "source_data": { - "dataset_name": "apex-agents", - "source_type": "hf_dataset", - "hf_repo": "mercor/apex-agents" - }, - "metric_config": { - "evaluation_description": "Corporate lawyer world mean score.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.382 - }, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - }, - { - "evaluation_id": "ace/openai_gpt-5/1773260200", - "retrieved_timestamp": "1773260200", - "source_metadata": { - "source_name": "Mercor ACE Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Mercor", - "source_organization_url": "https://www.mercor.com", - "evaluator_relationship": "first_party" - }, - "eval_library": { - "name": "archipelago", - "version": "1.0.0" - }, - "benchmark": "ace", - "evaluation_results": [ - { - "evaluation_name": "Overall Score", - "source_data": { - "dataset_name": "ace", - "source_type": "hf_dataset", - "hf_repo": "Mercor/ACE" - }, - "metric_config": { - "evaluation_description": "Overall ACE score across all consumer-task domains.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.561, - "uncertainty": { - "confidence_interval": { - "lower": -0.033, - "upper": 0.033, - "method": "bootstrap" - } - } - }, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - }, - { - "evaluation_name": "DIY Score", - "source_data": { - "dataset_name": "ace", - "source_type": "hf_dataset", - "hf_repo": "Mercor/ACE" - }, - "metric_config": { - "evaluation_description": "DIY domain score.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.55 - }, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - }, - { - "evaluation_name": "Food Score", - "source_data": { - "dataset_name": "ace", - "source_type": "hf_dataset", - "hf_repo": "Mercor/ACE" - }, - "metric_config": { - "evaluation_description": "Food domain score.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.7 - }, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - }, - { - "evaluation_name": "Gaming Score", - "source_data": { - "dataset_name": "ace", - "source_type": "hf_dataset", - "hf_repo": "Mercor/ACE" - }, - "metric_config": { - "evaluation_description": "Gaming domain score.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.575 - }, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - }, - { - "evaluation_id": "apex-v1/openai_gpt-5/1773260200", - "retrieved_timestamp": "1773260200", - "source_metadata": { - "source_name": "Mercor APEX-v1 Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Mercor", - "source_organization_url": "https://www.mercor.com", - "evaluator_relationship": "first_party" - }, - "eval_library": { - "name": "archipelago", - "version": "1.0.0" - }, - "benchmark": "apex-v1", - "evaluation_results": [ - { - "evaluation_name": "Overall Score", - "source_data": { - "dataset_name": "apex-v1", - "source_type": "hf_dataset", - "hf_repo": "Mercor/APEX-v1" - }, - "metric_config": { - "evaluation_description": "Overall APEX-v1 mean score across all jobs.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.67, - "uncertainty": { - "confidence_interval": { - "lower": -0.024, - "upper": 0.024, - "method": "bootstrap" - } - } - }, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - }, - { - "evaluation_name": "Big Law Score", - "source_data": { - "dataset_name": "apex-v1", - "source_type": "hf_dataset", - "hf_repo": "Mercor/APEX-v1" - }, - "metric_config": { - "evaluation_description": "Big law associate score.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.78 - }, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - }, - { - "evaluation_name": "Medicine (MD) Score", - "source_data": { - "dataset_name": "apex-v1", - "source_type": "hf_dataset", - "hf_repo": "Mercor/APEX-v1" - }, - "metric_config": { - "evaluation_description": "Primary care physician (MD) score.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.66 - }, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - }, - { - "evaluation_name": "Investment Banking Score", - "source_data": { - "dataset_name": "apex-v1", - "source_type": "hf_dataset", - "hf_repo": "Mercor/APEX-v1" - }, - "metric_config": { - "evaluation_description": "Investment banking associate score.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.61 - }, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/openai_gpt_5_codex.json b/data/models/openai_gpt_5_codex.json deleted file mode 100644 index 0c646c4a2a7934ffc634bbf3262ee9da986f7875..0000000000000000000000000000000000000000 --- a/data/models/openai_gpt_5_codex.json +++ /dev/null @@ -1,57 +0,0 @@ -{ - "model_info": { - "name": "GPT 5 Codex", - "developer": "OpenAI", - "id": "openai/GPT 5 Codex", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "apex-agents/openai_gpt-5-codex/1773260200", - "retrieved_timestamp": "1773260200", - "source_metadata": { - "source_name": "Mercor APEX-Agents Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Mercor", - "source_organization_url": "https://www.mercor.com", - "evaluator_relationship": "first_party" - }, - "eval_library": { - "name": "archipelago", - "version": "1.0.0" - }, - "benchmark": "apex-agents", - "evaluation_results": [ - { - "evaluation_name": "Corporate Lawyer Mean Score", - "source_data": { - "dataset_name": "apex-agents", - "source_type": "hf_dataset", - "hf_repo": "mercor/apex-agents" - }, - "metric_config": { - "evaluation_description": "Corporate lawyer world mean score.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.362 - }, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/openai_gpt_oss_120b.json b/data/models/openai_gpt_oss_120b.json deleted file mode 100644 index e802557769bf1bc16963ed00a6290b3b406e5f53..0000000000000000000000000000000000000000 --- a/data/models/openai_gpt_oss_120b.json +++ /dev/null @@ -1,209 +0,0 @@ -{ - "model_info": { - "name": "GPT OSS 120B", - "developer": "OpenAI", - "id": "openai/GPT OSS 120B", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "apex-agents/openai_gpt-oss-120b/1773260200", - "retrieved_timestamp": "1773260200", - "source_metadata": { - "source_name": "Mercor APEX-Agents Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Mercor", - "source_organization_url": "https://www.mercor.com", - "evaluator_relationship": "first_party" - }, - "eval_library": { - "name": "archipelago", - "version": "1.0.0" - }, - "benchmark": "apex-agents", - "evaluation_results": [ - { - "evaluation_name": "Overall Pass@1", - "source_data": { - "dataset_name": "apex-agents", - "source_type": "hf_dataset", - "hf_repo": "mercor/apex-agents" - }, - "metric_config": { - "evaluation_description": "Overall Pass@1 (dataset card / paper snapshot).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.047, - "uncertainty": { - "confidence_interval": { - "lower": -0.014, - "upper": 0.014, - "method": "bootstrap" - } - } - }, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - }, - { - "evaluation_name": "Overall Pass@8", - "source_data": { - "dataset_name": "apex-agents", - "source_type": "hf_dataset", - "hf_repo": "mercor/apex-agents" - }, - "metric_config": { - "evaluation_description": "Overall Pass@8 (dataset card / paper snapshot).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.115, - "uncertainty": { - "confidence_interval": { - "lower": -0.027, - "upper": 0.029, - "method": "bootstrap" - } - } - }, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - }, - { - "evaluation_name": "Overall Mean Score", - "source_data": { - "dataset_name": "apex-agents", - "source_type": "hf_dataset", - "hf_repo": "mercor/apex-agents" - }, - "metric_config": { - "evaluation_description": "Overall mean rubric score.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.145 - }, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - }, - { - "evaluation_name": "Investment Banking Pass@1", - "source_data": { - "dataset_name": "apex-agents", - "source_type": "hf_dataset", - "hf_repo": "mercor/apex-agents" - }, - "metric_config": { - "evaluation_description": "Investment banking world Pass@1.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.027 - }, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - }, - { - "evaluation_name": "Management Consulting Pass@1", - "source_data": { - "dataset_name": "apex-agents", - "source_type": "hf_dataset", - "hf_repo": "mercor/apex-agents" - }, - "metric_config": { - "evaluation_description": "Management consulting world Pass@1.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.035 - }, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - }, - { - "evaluation_name": "Corporate Law Pass@1", - "source_data": { - "dataset_name": "apex-agents", - "source_type": "hf_dataset", - "hf_repo": "mercor/apex-agents" - }, - "metric_config": { - "evaluation_description": "Corporate law world Pass@1.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.078 - }, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - }, - { - "evaluation_name": "Corporate Lawyer Mean Score", - "source_data": { - "dataset_name": "apex-agents", - "source_type": "hf_dataset", - "hf_repo": "mercor/apex-agents" - }, - "metric_config": { - "evaluation_description": "Corporate lawyer world mean score.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.269 - }, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/openai_o3-2025-04-16-fc.json b/data/models/openai_o3-2025-04-16-fc.json deleted file mode 100644 index 4fb2bb465060de3adb3981573b1f01f4fd3eb389..0000000000000000000000000000000000000000 --- a/data/models/openai_o3-2025-04-16-fc.json +++ /dev/null @@ -1,851 +0,0 @@ -{ - "model_info": { - "name": "o3-2025-04-16 (FC)", - "id": "openai/o3-2025-04-16-fc", - "developer": "OpenAI", - "additional_details": { - "raw_model_name": "o3-2025-04-16 (FC)", - "organization": "OpenAI", - "license": "Proprietary", - "mode": "FC", - "model_link": "https://openai.com/index/introducing-o3-and-o4-mini/" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/openai/o3-2025-04-16-fc/1775236112.380843", - "retrieved_timestamp": "1775236112.380843", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 30.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 48.56 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 133.45 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 3.5 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 8.69 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 8.39 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 40.38 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 74.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 87.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 66.17 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 70.54 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 67.62 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 14.75 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 16.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 11.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 14.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 16.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 77.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 79.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 75.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 47.31 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 24.52 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 44.52 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 72.9 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 81.25 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 86.13 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/openai_o3-2025-04-16-prompt.json b/data/models/openai_o3-2025-04-16-prompt.json deleted file mode 100644 index c43fd454b790943ece72ac3eb74462f2332caf33..0000000000000000000000000000000000000000 --- a/data/models/openai_o3-2025-04-16-prompt.json +++ /dev/null @@ -1,905 +0,0 @@ -{ - "model_info": { - "name": "o3-2025-04-16 (Prompt)", - "id": "openai/o3-2025-04-16-prompt", - "developer": "OpenAI", - "additional_details": { - "raw_model_name": "o3-2025-04-16 (Prompt)", - "organization": "OpenAI", - "license": "Proprietary", - "mode": "Prompt", - "model_link": "https://openai.com/index/introducing-o3-and-o4-mini/" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/openai/o3-2025-04-16-prompt/1775236112.36956", - "retrieved_timestamp": "1775236112.36956", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 8.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 63.05 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 234.64 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 4.83 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 7.01 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 11.7 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 81.94 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 74.25 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 89.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 86.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 78.0 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 73.21 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 83.33 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 70.75 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 75.0 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 70.83 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 62.25 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 68.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 63.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 54.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 63.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 50.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 51.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 50.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 51.83 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 33.55 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 50.32 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 71.61 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 93.75 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 83.98 - } - }, - { - "evaluation_result_id": "bfcl.format_sensitivity.max_delta::max_delta", - "evaluation_name": "bfcl.format_sensitivity.max_delta", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.format_sensitivity.max_delta", - "metric_name": "Format sensitivity max delta", - "metric_kind": "difference", - "metric_unit": "percentage_points", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Format Sensitivity Max Delta" - } - }, - "score_details": { - "score": 8.5 - } - }, - { - "evaluation_result_id": "bfcl.format_sensitivity.stddev::stddev", - "evaluation_name": "bfcl.format_sensitivity.stddev", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.format_sensitivity.stddev", - "metric_name": "Format sensitivity standard deviation", - "metric_kind": "difference", - "metric_unit": "percentage_points", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Format Sensitivity Standard Deviation" - } - }, - "score_details": { - "score": 2.75 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/openai_o3-2025-04-16.json b/data/models/openai_o3-2025-04-16.json deleted file mode 100644 index 5b0ebb6055072a97bc5a6ae5ebe333d1cf398940..0000000000000000000000000000000000000000 --- a/data/models/openai_o3-2025-04-16.json +++ /dev/null @@ -1,313 +0,0 @@ -{ - "model_info": { - "name": "o3-2025-04-16", - "id": "openai/o3-2025-04-16", - "developer": "OpenAI", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_capabilities/openai_o3-2025-04-16/1774096304.5056698", - "retrieved_timestamp": "1774096304.5056698", - "source_metadata": { - "source_name": "helm_capabilities", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_capabilities", - "evaluation_results": [ - { - "evaluation_name": "Mean score", - "source_data": { - "dataset_name": "helm_capabilities", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "The mean of the scores from all columns.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.811, - "details": { - "description": "", - "tab": "Accuracy", - "Mean score - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"51.078448384234015\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU-Pro", - "source_data": { - "dataset_name": "MMLU-Pro", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on MMLU-Pro", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.859, - "details": { - "description": "min=0.859, mean=0.859, max=0.859, sum=0.859 (1)", - "tab": "Accuracy", - "MMLU-Pro - Observed inference time (s)": "{\"description\": \"min=17.306, mean=17.306, max=17.306, sum=17.306 (1)\", \"tab\": \"Efficiency\", \"score\": \"17.306045585632326\"}", - "MMLU-Pro - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "MMLU-Pro - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - # prompt tokens": "{\"description\": \"min=249.506, mean=249.506, max=249.506, sum=249.506 (1)\", \"tab\": \"General information\", \"score\": \"249.506\"}", - "MMLU-Pro - # output tokens": "{\"description\": \"min=5.038, mean=5.038, max=5.038, sum=5.038 (1)\", \"tab\": \"General information\", \"score\": \"5.038\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"all\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.753, - "details": { - "description": "min=0.753, mean=0.753, max=0.753, sum=0.753 (1)", - "tab": "Accuracy", - "GPQA - Observed inference time (s)": "{\"description\": \"min=48.024, mean=48.024, max=48.024, sum=48.024 (1)\", \"tab\": \"Efficiency\", \"score\": \"48.0242628821343\"}", - "GPQA - # eval": "{\"description\": \"min=446, mean=446, max=446, sum=446 (1)\", \"tab\": \"General information\", \"score\": \"446.0\"}", - "GPQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - # prompt tokens": "{\"description\": \"min=269.975, mean=269.975, max=269.975, sum=269.975 (1)\", \"tab\": \"General information\", \"score\": \"269.97533632286996\"}", - "GPQA - # output tokens": "{\"description\": \"min=6.457, mean=6.457, max=6.457, sum=6.457 (1)\", \"tab\": \"General information\", \"score\": \"6.457399103139013\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"gpqa_main\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "IFEval Strict Acc on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.869, - "details": { - "description": "min=0.869, mean=0.869, max=0.869, sum=0.869 (1)", - "tab": "Accuracy", - "IFEval - Observed inference time (s)": "{\"description\": \"min=17.399, mean=17.399, max=17.399, sum=17.399 (1)\", \"tab\": \"Efficiency\", \"score\": \"17.398983872972444\"}", - "IFEval - # eval": "{\"description\": \"min=541, mean=541, max=541, sum=541 (1)\", \"tab\": \"General information\", \"score\": \"541.0\"}", - "IFEval - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - # prompt tokens": "{\"description\": \"min=46.054, mean=46.054, max=46.054, sum=46.054 (1)\", \"tab\": \"General information\", \"score\": \"46.05360443622921\"}", - "IFEval - # output tokens": "{\"description\": \"min=447.353, mean=447.353, max=447.353, sum=447.353 (1)\", \"tab\": \"General information\", \"score\": \"447.35304990757857\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WildBench", - "source_data": { - "dataset_name": "WildBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "WB Score on WildBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.861, - "details": { - "description": "min=0.861, mean=0.861, max=0.861, sum=0.861 (1)", - "tab": "Accuracy", - "WildBench - Observed inference time (s)": "{\"description\": \"min=47.15, mean=47.15, max=47.15, sum=47.15 (1)\", \"tab\": \"Efficiency\", \"score\": \"47.150321824789046\"}", - "WildBench - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "WildBench - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # prompt tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # output tokens": "{\"description\": \"min=1248.452, mean=1248.452, max=1248.452, sum=1248.452 (1)\", \"tab\": \"General information\", \"score\": \"1248.452\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"v2\"" - } - } - }, - { - "evaluation_name": "Omni-MATH", - "source_data": { - "dataset_name": "Omni-MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Acc on Omni-MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.714, - "details": { - "description": "min=0.714, mean=0.714, max=0.714, sum=0.714 (1)", - "tab": "Accuracy", - "Omni-MATH - Observed inference time (s)": "{\"description\": \"min=125.513, mean=125.513, max=125.513, sum=125.513 (1)\", \"tab\": \"Efficiency\", \"score\": \"125.51262775564194\"}", - "Omni-MATH - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "Omni-MATH - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - # prompt tokens": "{\"description\": \"min=109.767, mean=109.767, max=109.767, sum=109.767 (1)\", \"tab\": \"General information\", \"score\": \"109.767\"}", - "Omni-MATH - # output tokens": "{\"description\": \"min=506.811, mean=506.811, max=506.811, sum=506.811 (1)\", \"tab\": \"General information\", \"score\": \"506.811\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "livecodebenchpro/o3-2025-04-16/1760492095.8105888", - "retrieved_timestamp": "1760492095.8105888", - "source_metadata": { - "source_organization_name": "New York University, Princeton University, University of California San Diego, University of Washington and Canyon Crest Academy", - "evaluator_relationship": "third_party", - "source_name": "Live Code Bench Pro", - "source_type": "documentation" - }, - "eval_library": { - "name": "unknown", - "version": "unknown" - }, - "benchmark": "livecodebenchpro", - "evaluation_results": [ - { - "evaluation_name": "Hard Problems", - "metric_config": { - "evaluation_description": "Pass@1 on Hard Problems", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.0 - }, - "source_data": { - "dataset_name": "Hard Problems", - "source_type": "url", - "url": [ - "https://webhook.cp-bench.orzzh.com/leaderboard/llm/difficulty?difficulty=hard&benchmark_mode=live" - ] - } - }, - { - "evaluation_name": "Medium Problems", - "metric_config": { - "evaluation_description": "Pass@1 on Medium Problems", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.22535211267605634 - }, - "source_data": { - "dataset_name": "Medium Problems", - "source_type": "url", - "url": [ - "https://webhook.cp-bench.orzzh.com/leaderboard/llm/difficulty?difficulty=medium&benchmark_mode=live" - ] - } - }, - { - "evaluation_name": "Easy Problems", - "metric_config": { - "evaluation_description": "Pass@1 on Easy Problems", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.7183098591549296 - }, - "source_data": { - "dataset_name": "Easy Problems", - "source_type": "url", - "url": [ - "https://webhook.cp-bench.orzzh.com/leaderboard/llm/difficulty?difficulty=easy&benchmark_mode=live" - ] - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/openai_o3-mini-2025-01-31.json b/data/models/openai_o3-mini-2025-01-31.json deleted file mode 100644 index 94e86a32a0bfe203e908d1247dbfd44f8368e690..0000000000000000000000000000000000000000 --- a/data/models/openai_o3-mini-2025-01-31.json +++ /dev/null @@ -1,1043 +0,0 @@ -{ - "model_info": { - "name": "o3-mini-2025-01-31", - "id": "openai/o3-mini-2025-01-31", - "developer": "OpenAI", - "inference_platform": "unknown", - "additional_details": { - "display_name": "o3 mini" - } - }, - "evaluations": [ - { - "evaluation_id": "global-mmlu-lite/openai_o3-mini-2025-01-31/1773936496.366405", - "retrieved_timestamp": "1773936496.366405", - "source_metadata": { - "source_name": "Global MMLU Lite Leaderboard", - "source_type": "documentation", - "source_organization_name": "kaggle", - "source_organization_url": "www.kaggle.com", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "kaggle kernel", - "version": "4", - "additional_details": { - "url": "https://www.kaggle.com/code/shivalikasingh95/global-mmlu-lite-sample-notebook" - } - }, - "benchmark": "global-mmlu-lite", - "evaluation_results": [ - { - "evaluation_name": "Global MMLU Lite", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Global MMLU Lite", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.78 - } - }, - { - "evaluation_name": "Culturally Sensitive", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Sensitive", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.765 - } - }, - { - "evaluation_name": "Culturally Agnostic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Agnostic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.795 - } - }, - { - "evaluation_name": "Arabic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Arabic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7725, - "uncertainty": { - "confidence_interval": { - "lower": -0.0411, - "upper": 0.0411, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "English", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - English", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8025, - "uncertainty": { - "confidence_interval": { - "lower": -0.039, - "upper": 0.039, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Bengali", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Bengali", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.77, - "uncertainty": { - "confidence_interval": { - "lower": -0.0412, - "upper": 0.0412, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "German", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - German", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7525, - "uncertainty": { - "confidence_interval": { - "lower": -0.0423, - "upper": 0.0423, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "French", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - French", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.74, - "uncertainty": { - "confidence_interval": { - "lower": -0.043, - "upper": 0.043, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Hindi", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Hindi", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7525, - "uncertainty": { - "confidence_interval": { - "lower": -0.0423, - "upper": 0.0423, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Indonesian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Indonesian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7425, - "uncertainty": { - "confidence_interval": { - "lower": -0.0429, - "upper": 0.0429, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Italian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Italian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8, - "uncertainty": { - "confidence_interval": { - "lower": -0.0392, - "upper": 0.0392, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Japanese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Japanese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.81, - "uncertainty": { - "confidence_interval": { - "lower": -0.0384, - "upper": 0.0384, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Korean", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Korean", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8075, - "uncertainty": { - "confidence_interval": { - "lower": -0.0386, - "upper": 0.0386, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Portuguese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Portuguese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7975, - "uncertainty": { - "confidence_interval": { - "lower": -0.0394, - "upper": 0.0394, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Spanish", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Spanish", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.775, - "uncertainty": { - "confidence_interval": { - "lower": -0.0409, - "upper": 0.0409, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Swahili", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Swahili", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.765, - "uncertainty": { - "confidence_interval": { - "lower": -0.0416, - "upper": 0.0416, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Yoruba", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Yoruba", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7725, - "uncertainty": { - "confidence_interval": { - "lower": -0.0411, - "upper": 0.0411, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Chinese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Chinese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8125, - "uncertainty": { - "confidence_interval": { - "lower": -0.0382, - "upper": 0.0382, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Burmese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Burmese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8075, - "uncertainty": { - "confidence_interval": { - "lower": -0.0386, - "upper": 0.0386, - "method": "unknown" - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "global-mmlu-lite/openai_o3-mini-2025-01-31/1773936583.743359", - "retrieved_timestamp": "1773936583.743359", - "source_metadata": { - "source_name": "Global MMLU Lite Leaderboard", - "source_type": "documentation", - "source_organization_name": "kaggle", - "source_organization_url": "www.kaggle.com", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "kaggle kernel", - "version": "4", - "additional_details": { - "url": "https://www.kaggle.com/code/shivalikasingh95/global-mmlu-lite-sample-notebook" - } - }, - "benchmark": "global-mmlu-lite", - "evaluation_results": [ - { - "evaluation_name": "Global MMLU Lite", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Global MMLU Lite", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.78 - } - }, - { - "evaluation_name": "Culturally Sensitive", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Sensitive", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.765 - } - }, - { - "evaluation_name": "Culturally Agnostic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Agnostic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.795 - } - }, - { - "evaluation_name": "Arabic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Arabic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7725, - "uncertainty": { - "confidence_interval": { - "lower": -0.0411, - "upper": 0.0411, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "English", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - English", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8025, - "uncertainty": { - "confidence_interval": { - "lower": -0.039, - "upper": 0.039, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Bengali", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Bengali", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.77, - "uncertainty": { - "confidence_interval": { - "lower": -0.0412, - "upper": 0.0412, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "German", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - German", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7525, - "uncertainty": { - "confidence_interval": { - "lower": -0.0423, - "upper": 0.0423, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "French", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - French", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.74, - "uncertainty": { - "confidence_interval": { - "lower": -0.043, - "upper": 0.043, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Hindi", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Hindi", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7525, - "uncertainty": { - "confidence_interval": { - "lower": -0.0423, - "upper": 0.0423, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Indonesian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Indonesian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7425, - "uncertainty": { - "confidence_interval": { - "lower": -0.0429, - "upper": 0.0429, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Italian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Italian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8, - "uncertainty": { - "confidence_interval": { - "lower": -0.0392, - "upper": 0.0392, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Japanese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Japanese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.81, - "uncertainty": { - "confidence_interval": { - "lower": -0.0384, - "upper": 0.0384, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Korean", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Korean", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8075, - "uncertainty": { - "confidence_interval": { - "lower": -0.0386, - "upper": 0.0386, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Portuguese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Portuguese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7975, - "uncertainty": { - "confidence_interval": { - "lower": -0.0394, - "upper": 0.0394, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Spanish", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Spanish", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.775, - "uncertainty": { - "confidence_interval": { - "lower": -0.0409, - "upper": 0.0409, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Swahili", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Swahili", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.765, - "uncertainty": { - "confidence_interval": { - "lower": -0.0416, - "upper": 0.0416, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Yoruba", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Yoruba", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7725, - "uncertainty": { - "confidence_interval": { - "lower": -0.0411, - "upper": 0.0411, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Chinese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Chinese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8125, - "uncertainty": { - "confidence_interval": { - "lower": -0.0382, - "upper": 0.0382, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Burmese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Burmese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8075, - "uncertainty": { - "confidence_interval": { - "lower": -0.0386, - "upper": 0.0386, - "method": "unknown" - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/openai_o3.json b/data/models/openai_o3.json deleted file mode 100644 index cd7f2ff38c1289a354e759e15d5d6545fd6374af..0000000000000000000000000000000000000000 --- a/data/models/openai_o3.json +++ /dev/null @@ -1,157 +0,0 @@ -{ - "model_info": { - "name": "o3", - "developer": "OpenAI", - "id": "openai/o3", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "ace/openai_o3/1773260200", - "retrieved_timestamp": "1773260200", - "source_metadata": { - "source_name": "Mercor ACE Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Mercor", - "source_organization_url": "https://www.mercor.com", - "evaluator_relationship": "first_party" - }, - "eval_library": { - "name": "archipelago", - "version": "1.0.0" - }, - "benchmark": "ace", - "evaluation_results": [ - { - "evaluation_name": "Overall Score", - "source_data": { - "dataset_name": "ace", - "source_type": "hf_dataset", - "hf_repo": "Mercor/ACE" - }, - "metric_config": { - "evaluation_description": "Overall ACE score across all consumer-task domains.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.529, - "uncertainty": { - "confidence_interval": { - "lower": -0.031, - "upper": 0.031, - "method": "bootstrap" - } - } - }, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - }, - { - "evaluation_name": "Gaming Score", - "source_data": { - "dataset_name": "ace", - "source_type": "hf_dataset", - "hf_repo": "Mercor/ACE" - }, - "metric_config": { - "evaluation_description": "Gaming domain score.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.585 - }, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - }, - { - "evaluation_name": "Shopping Score", - "source_data": { - "dataset_name": "ace", - "source_type": "hf_dataset", - "hf_repo": "Mercor/ACE" - }, - "metric_config": { - "evaluation_description": "Shopping domain score.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.45 - }, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - }, - { - "evaluation_id": "apex-v1/openai_o3/1773260200", - "retrieved_timestamp": "1773260200", - "source_metadata": { - "source_name": "Mercor APEX-v1 Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Mercor", - "source_organization_url": "https://www.mercor.com", - "evaluator_relationship": "first_party" - }, - "eval_library": { - "name": "archipelago", - "version": "1.0.0" - }, - "benchmark": "apex-v1", - "evaluation_results": [ - { - "evaluation_name": "Big Law Score", - "source_data": { - "dataset_name": "apex-v1", - "source_type": "hf_dataset", - "hf_repo": "Mercor/APEX-v1" - }, - "metric_config": { - "evaluation_description": "Big law associate score.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.76 - }, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/openai_o3_pro.json b/data/models/openai_o3_pro.json deleted file mode 100644 index b59c1617a8a2e01c6964ed54598e37f21ac157b1..0000000000000000000000000000000000000000 --- a/data/models/openai_o3_pro.json +++ /dev/null @@ -1,156 +0,0 @@ -{ - "model_info": { - "name": "o3 Pro", - "developer": "OpenAI", - "id": "openai/o3 Pro", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "ace/openai_o3-pro/1773260200", - "retrieved_timestamp": "1773260200", - "source_metadata": { - "source_name": "Mercor ACE Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Mercor", - "source_organization_url": "https://www.mercor.com", - "evaluator_relationship": "first_party" - }, - "eval_library": { - "name": "archipelago", - "version": "1.0.0" - }, - "benchmark": "ace", - "evaluation_results": [ - { - "evaluation_name": "Overall Score", - "source_data": { - "dataset_name": "ace", - "source_type": "hf_dataset", - "hf_repo": "Mercor/ACE" - }, - "metric_config": { - "evaluation_description": "Overall ACE score across all consumer-task domains.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.552, - "uncertainty": { - "confidence_interval": { - "lower": -0.032, - "upper": 0.032, - "method": "bootstrap" - } - } - }, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - }, - { - "evaluation_name": "DIY Score", - "source_data": { - "dataset_name": "ace", - "source_type": "hf_dataset", - "hf_repo": "Mercor/ACE" - }, - "metric_config": { - "evaluation_description": "DIY domain score.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.54 - }, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - }, - { - "evaluation_name": "Food Score", - "source_data": { - "dataset_name": "ace", - "source_type": "hf_dataset", - "hf_repo": "Mercor/ACE" - }, - "metric_config": { - "evaluation_description": "Food domain score.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.6 - }, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - }, - { - "evaluation_name": "Gaming Score", - "source_data": { - "dataset_name": "ace", - "source_type": "hf_dataset", - "hf_repo": "Mercor/ACE" - }, - "metric_config": { - "evaluation_description": "Gaming domain score.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.613 - }, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - }, - { - "evaluation_name": "Shopping Score", - "source_data": { - "dataset_name": "ace", - "source_type": "hf_dataset", - "hf_repo": "Mercor/ACE" - }, - "metric_config": { - "evaluation_description": "Shopping domain score.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.45 - }, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "run_setting": "High" - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/openai_o4-mini-2025-04-16-fc.json b/data/models/openai_o4-mini-2025-04-16-fc.json deleted file mode 100644 index 4b13f706361d1ce48da72eb8cea14af88283867f..0000000000000000000000000000000000000000 --- a/data/models/openai_o4-mini-2025-04-16-fc.json +++ /dev/null @@ -1,851 +0,0 @@ -{ - "model_info": { - "name": "o4-mini-2025-04-16 (FC)", - "id": "openai/o4-mini-2025-04-16-fc", - "developer": "OpenAI", - "additional_details": { - "raw_model_name": "o4-mini-2025-04-16 (FC)", - "organization": "OpenAI", - "license": "Proprietary", - "mode": "FC", - "model_link": "https://openai.com/index/introducing-o3-and-o4-mini/" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/openai/o4-mini-2025-04-16-fc/1775236112.376308", - "retrieved_timestamp": "1775236112.376308", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 21.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 53.24 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 81.91 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 3.71 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 7.18 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 9.33 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 37.73 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 66.92 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 84.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 66.1 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 69.38 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 67.81 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 41.75 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 51.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 30.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 40.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 45.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 75.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 75.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 76.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 34.19 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 19.35 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 24.52 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 58.71 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 81.25 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 83.91 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/openai_o4-mini-2025-04-16-prompt.json b/data/models/openai_o4-mini-2025-04-16-prompt.json deleted file mode 100644 index 96af0515902dd791c907e61bc95f1e3133285e4c..0000000000000000000000000000000000000000 --- a/data/models/openai_o4-mini-2025-04-16-prompt.json +++ /dev/null @@ -1,905 +0,0 @@ -{ - "model_info": { - "name": "o4-mini-2025-04-16 (Prompt)", - "id": "openai/o4-mini-2025-04-16-prompt", - "developer": "OpenAI", - "additional_details": { - "raw_model_name": "o4-mini-2025-04-16 (Prompt)", - "organization": "OpenAI", - "license": "Proprietary", - "mode": "Prompt", - "model_link": "https://openai.com/index/introducing-o3-and-o4-mini/" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/openai/o4-mini-2025-04-16-prompt/1775236112.379882", - "retrieved_timestamp": "1775236112.379882", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 28.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 50.26 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 133.63 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 4.47 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 5.19 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 10.19 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 81.29 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 72.67 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 88.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 84.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 80.0 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 70.76 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 79.46 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 68.76 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 75.0 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 62.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 16.62 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 16.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 18.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 17.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 14.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 71.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 73.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 70.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 35.27 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 22.58 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 25.16 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 58.06 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 81.25 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 87.16 - } - }, - { - "evaluation_result_id": "bfcl.format_sensitivity.max_delta::max_delta", - "evaluation_name": "bfcl.format_sensitivity.max_delta", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.format_sensitivity.max_delta", - "metric_name": "Format sensitivity max delta", - "metric_kind": "difference", - "metric_unit": "percentage_points", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Format Sensitivity Max Delta" - } - }, - "score_details": { - "score": 9.5 - } - }, - { - "evaluation_result_id": "bfcl.format_sensitivity.stddev::stddev", - "evaluation_name": "bfcl.format_sensitivity.stddev", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.format_sensitivity.stddev", - "metric_name": "Format sensitivity standard deviation", - "metric_kind": "difference", - "metric_unit": "percentage_points", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Format Sensitivity Standard Deviation" - } - }, - "score_details": { - "score": 2.6 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/openai_o4-mini-2025-04-16.json b/data/models/openai_o4-mini-2025-04-16.json deleted file mode 100644 index 751e567f80107ae7403958c0915864809e731407..0000000000000000000000000000000000000000 --- a/data/models/openai_o4-mini-2025-04-16.json +++ /dev/null @@ -1,910 +0,0 @@ -{ - "model_info": { - "name": "o4-mini-2025-04-16", - "id": "openai/o4-mini-2025-04-16", - "developer": "OpenAI", - "inference_platform": "unknown", - "additional_details": { - "display_name": "o4 mini" - } - }, - "evaluations": [ - { - "evaluation_id": "global-mmlu-lite/openai_o4-mini-2025-04-16/1773936583.743359", - "retrieved_timestamp": "1773936583.743359", - "source_metadata": { - "source_name": "Global MMLU Lite Leaderboard", - "source_type": "documentation", - "source_organization_name": "kaggle", - "source_organization_url": "www.kaggle.com", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "kaggle kernel", - "version": "4", - "additional_details": { - "url": "https://www.kaggle.com/code/shivalikasingh95/global-mmlu-lite-sample-notebook" - } - }, - "benchmark": "global-mmlu-lite", - "evaluation_results": [ - { - "evaluation_name": "Global MMLU Lite", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Global MMLU Lite", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8705 - } - }, - { - "evaluation_name": "Culturally Sensitive", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Sensitive", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8503 - } - }, - { - "evaluation_name": "Culturally Agnostic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Agnostic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8906 - } - }, - { - "evaluation_name": "Arabic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Arabic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.865, - "uncertainty": { - "confidence_interval": { - "lower": -0.0335, - "upper": 0.0335, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "English", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - English", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8675, - "uncertainty": { - "confidence_interval": { - "lower": -0.0332, - "upper": 0.0332, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Bengali", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Bengali", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8875, - "uncertainty": { - "confidence_interval": { - "lower": -0.031, - "upper": 0.031, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "German", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - German", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8775, - "uncertainty": { - "confidence_interval": { - "lower": -0.0321, - "upper": 0.0321, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "French", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - French", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.87, - "uncertainty": { - "confidence_interval": { - "lower": -0.033, - "upper": 0.033, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Hindi", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Hindi", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.87, - "uncertainty": { - "confidence_interval": { - "lower": -0.033, - "upper": 0.033, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Indonesian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Indonesian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8675, - "uncertainty": { - "confidence_interval": { - "lower": -0.0332, - "upper": 0.0332, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Italian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Italian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.855, - "uncertainty": { - "confidence_interval": { - "lower": -0.0345, - "upper": 0.0345, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Japanese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Japanese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.885, - "uncertainty": { - "confidence_interval": { - "lower": -0.0313, - "upper": 0.0313, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Korean", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Korean", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.88, - "uncertainty": { - "confidence_interval": { - "lower": -0.0318, - "upper": 0.0318, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Portuguese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Portuguese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.88, - "uncertainty": { - "confidence_interval": { - "lower": -0.0318, - "upper": 0.0318, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Spanish", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Spanish", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.855, - "uncertainty": { - "confidence_interval": { - "lower": -0.0345, - "upper": 0.0345, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Swahili", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Swahili", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8525, - "uncertainty": { - "confidence_interval": { - "lower": -0.0348, - "upper": 0.0348, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Yoruba", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Yoruba", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8525, - "uncertainty": { - "confidence_interval": { - "lower": -0.0348, - "upper": 0.0348, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Chinese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Chinese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.89, - "uncertainty": { - "confidence_interval": { - "lower": -0.0307, - "upper": 0.0307, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Burmese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Burmese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8725, - "uncertainty": { - "confidence_interval": { - "lower": -0.0327, - "upper": 0.0327, - "method": "unknown" - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "helm_capabilities/openai_o4-mini-2025-04-16/1774096304.5056698", - "retrieved_timestamp": "1774096304.5056698", - "source_metadata": { - "source_name": "helm_capabilities", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_capabilities", - "evaluation_results": [ - { - "evaluation_name": "Mean score", - "source_data": { - "dataset_name": "helm_capabilities", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "The mean of the scores from all columns.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.812, - "details": { - "description": "", - "tab": "Accuracy", - "Mean score - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"21.93756369551652\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU-Pro", - "source_data": { - "dataset_name": "MMLU-Pro", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on MMLU-Pro", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.82, - "details": { - "description": "min=0.82, mean=0.82, max=0.82, sum=0.82 (1)", - "tab": "Accuracy", - "MMLU-Pro - Observed inference time (s)": "{\"description\": \"min=8.896, mean=8.896, max=8.896, sum=8.896 (1)\", \"tab\": \"Efficiency\", \"score\": \"8.895831291675568\"}", - "MMLU-Pro - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "MMLU-Pro - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - # prompt tokens": "{\"description\": \"min=249.506, mean=249.506, max=249.506, sum=249.506 (1)\", \"tab\": \"General information\", \"score\": \"249.506\"}", - "MMLU-Pro - # output tokens": "{\"description\": \"min=10.834, mean=10.834, max=10.834, sum=10.834 (1)\", \"tab\": \"General information\", \"score\": \"10.834\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"all\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.735, - "details": { - "description": "min=0.735, mean=0.735, max=0.735, sum=0.735 (1)", - "tab": "Accuracy", - "GPQA - Observed inference time (s)": "{\"description\": \"min=22.412, mean=22.412, max=22.412, sum=22.412 (1)\", \"tab\": \"Efficiency\", \"score\": \"22.412139415206397\"}", - "GPQA - # eval": "{\"description\": \"min=446, mean=446, max=446, sum=446 (1)\", \"tab\": \"General information\", \"score\": \"446.0\"}", - "GPQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - # prompt tokens": "{\"description\": \"min=269.975, mean=269.975, max=269.975, sum=269.975 (1)\", \"tab\": \"General information\", \"score\": \"269.97533632286996\"}", - "GPQA - # output tokens": "{\"description\": \"min=8.413, mean=8.413, max=8.413, sum=8.413 (1)\", \"tab\": \"General information\", \"score\": \"8.41255605381166\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"gpqa_main\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "IFEval Strict Acc on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.929, - "details": { - "description": "min=0.929, mean=0.929, max=0.929, sum=0.929 (1)", - "tab": "Accuracy", - "IFEval - Observed inference time (s)": "{\"description\": \"min=12.26, mean=12.26, max=12.26, sum=12.26 (1)\", \"tab\": \"Efficiency\", \"score\": \"12.260425486097494\"}", - "IFEval - # eval": "{\"description\": \"min=541, mean=541, max=541, sum=541 (1)\", \"tab\": \"General information\", \"score\": \"541.0\"}", - "IFEval - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - # prompt tokens": "{\"description\": \"min=46.054, mean=46.054, max=46.054, sum=46.054 (1)\", \"tab\": \"General information\", \"score\": \"46.05360443622921\"}", - "IFEval - # output tokens": "{\"description\": \"min=360.231, mean=360.231, max=360.231, sum=360.231 (1)\", \"tab\": \"General information\", \"score\": \"360.2310536044362\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WildBench", - "source_data": { - "dataset_name": "WildBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "WB Score on WildBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.854, - "details": { - "description": "min=0.854, mean=0.854, max=0.854, sum=0.854 (1)", - "tab": "Accuracy", - "WildBench - Observed inference time (s)": "{\"description\": \"min=25.397, mean=25.397, max=25.397, sum=25.397 (1)\", \"tab\": \"Efficiency\", \"score\": \"25.396886379241945\"}", - "WildBench - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "WildBench - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # prompt tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # output tokens": "{\"description\": \"min=969.786, mean=969.786, max=969.786, sum=969.786 (1)\", \"tab\": \"General information\", \"score\": \"969.786\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"v2\"" - } - } - }, - { - "evaluation_name": "Omni-MATH", - "source_data": { - "dataset_name": "Omni-MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Acc on Omni-MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.72, - "details": { - "description": "min=0.72, mean=0.72, max=0.72, sum=0.72 (1)", - "tab": "Accuracy", - "Omni-MATH - Observed inference time (s)": "{\"description\": \"min=40.723, mean=40.723, max=40.723, sum=40.723 (1)\", \"tab\": \"Efficiency\", \"score\": \"40.72253590536118\"}", - "Omni-MATH - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "Omni-MATH - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - # prompt tokens": "{\"description\": \"min=109.767, mean=109.767, max=109.767, sum=109.767 (1)\", \"tab\": \"General information\", \"score\": \"109.767\"}", - "Omni-MATH - # output tokens": "{\"description\": \"min=388.401, mean=388.401, max=388.401, sum=388.401 (1)\", \"tab\": \"General information\", \"score\": \"388.401\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "livecodebenchpro/o4-mini-2025-04-16/1760492095.8105888", - "retrieved_timestamp": "1760492095.8105888", - "source_metadata": { - "source_organization_name": "New York University, Princeton University, University of California San Diego, University of Washington and Canyon Crest Academy", - "evaluator_relationship": "third_party", - "source_name": "Live Code Bench Pro", - "source_type": "documentation" - }, - "eval_library": { - "name": "unknown", - "version": "unknown" - }, - "benchmark": "livecodebenchpro", - "evaluation_results": [ - { - "evaluation_name": "Hard Problems", - "metric_config": { - "evaluation_description": "Pass@1 on Hard Problems", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.014084507042253521 - }, - "source_data": { - "dataset_name": "Hard Problems", - "source_type": "url", - "url": [ - "https://webhook.cp-bench.orzzh.com/leaderboard/llm/difficulty?difficulty=hard&benchmark_mode=live" - ] - } - }, - { - "evaluation_name": "Medium Problems", - "metric_config": { - "evaluation_description": "Pass@1 on Medium Problems", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.30985915492957744 - }, - "source_data": { - "dataset_name": "Medium Problems", - "source_type": "url", - "url": [ - "https://webhook.cp-bench.orzzh.com/leaderboard/llm/difficulty?difficulty=medium&benchmark_mode=live" - ] - } - }, - { - "evaluation_name": "Easy Problems", - "metric_config": { - "evaluation_description": "Pass@1 on Easy Problems", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.8873239436619719 - }, - "source_data": { - "dataset_name": "Easy Problems", - "source_type": "url", - "url": [ - "https://webhook.cp-bench.orzzh.com/leaderboard/llm/difficulty?difficulty=easy&benchmark_mode=live" - ] - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "livecodebenchpro/o4-mini-2025-04-16/1770683238.099205", - "retrieved_timestamp": "1770683238.099205", - "source_metadata": { - "source_name": "Live Code Bench Pro", - "source_type": "documentation", - "source_organization_name": "New York University, Princeton University, University of California San Diego, University of Washington and Canyon Crest Academy", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "unknown", - "version": "unknown" - }, - "benchmark": "livecodebenchpro", - "evaluation_results": [ - { - "evaluation_name": "Hard Problems", - "source_data": { - "dataset_name": "Hard Problems", - "source_type": "url", - "url": [ - "https://webhook.cp-bench.orzzh.com/leaderboard/llm/difficulty?difficulty=hard&benchmark_mode=live" - ] - }, - "metric_config": { - "evaluation_description": "Pass@1 on Hard Problems", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0143 - } - }, - { - "evaluation_name": "Medium Problems", - "source_data": { - "dataset_name": "Medium Problems", - "source_type": "url", - "url": [ - "https://webhook.cp-bench.orzzh.com/leaderboard/llm/difficulty?difficulty=medium&benchmark_mode=live" - ] - }, - "metric_config": { - "evaluation_description": "Pass@1 on Medium Problems", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2923 - } - }, - { - "evaluation_name": "Easy Problems", - "source_data": { - "dataset_name": "Easy Problems", - "source_type": "url", - "url": [ - "https://webhook.cp-bench.orzzh.com/leaderboard/llm/difficulty?difficulty=easy&benchmark_mode=live" - ] - }, - "metric_config": { - "evaluation_description": "Pass@1 on Easy Problems", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8571 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/openai_text-ada-001.json b/data/models/openai_text-ada-001.json deleted file mode 100644 index 0d0a0572a06167675362f665f7090d23eff06af1..0000000000000000000000000000000000000000 --- a/data/models/openai_text-ada-001.json +++ /dev/null @@ -1,674 +0,0 @@ -{ - "model_info": { - "name": "text-ada-001", - "id": "openai/text-ada-001", - "developer": "OpenAI", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_classic/openai_text-ada-001/1774096308.339228", - "retrieved_timestamp": "1774096308.339228", - "source_metadata": { - "source_name": "helm_classic", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_classic", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_classic", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperform on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.107, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Calibration": "{\"description\": \"\", \"tab\": \"Calibration\", \"score\": \"0.17139908178298557\"}", - "Mean win rate - Robustness": "{\"description\": \"\", \"tab\": \"Robustness\", \"score\": \"0.10508470024599056\"}", - "Mean win rate - Fairness": "{\"description\": \"\", \"tab\": \"Fairness\", \"score\": \"0.10817286162113748\"}", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.937796052631579\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}", - "Mean win rate - Bias": "{\"description\": \"\", \"tab\": \"Bias\", \"score\": \"0.4261942744755245\"}", - "Mean win rate - Toxicity": "{\"description\": \"\", \"tab\": \"Toxicity\", \"score\": \"0.5531715198381865\"}", - "Mean win rate - Summarization metrics": "{\"description\": \"\", \"tab\": \"Summarization metrics\", \"score\": \"0.48596491228070177\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.238, - "details": { - "description": "min=0.14, mean=0.238, max=0.31, sum=3.566 (15)", - "tab": "Accuracy", - "MMLU - ECE (10-bin)": "{\"description\": \"min=0.357, mean=0.506, max=0.666, sum=7.594 (15)\", \"tab\": \"Calibration\", \"score\": \"0.5062965949265723\"}", - "MMLU - EM (Robustness)": "{\"description\": \"min=0.08, mean=0.178, max=0.28, sum=2.665 (15)\", \"tab\": \"Robustness\", \"score\": \"0.17768421052631578\"}", - "MMLU - EM (Fairness)": "{\"description\": \"min=0.11, mean=0.202, max=0.28, sum=3.026 (15)\", \"tab\": \"Fairness\", \"score\": \"0.201766081871345\"}", - "MMLU - Denoised inference time (s)": "{\"description\": \"min=0.086, mean=0.088, max=0.089, sum=1.314 (15)\", \"tab\": \"Efficiency\", \"score\": \"0.08760755934758772\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=1542 (15)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=75 (15)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (15)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=371.38, mean=472.274, max=624.07, sum=7084.111 (15)\", \"tab\": \"General information\", \"score\": \"472.2740350877193\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=15 (15)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "MMLU - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=45 (15)\", \"tab\": \"General information\", \"score\": \"3.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "BoolQ", - "source_data": { - "dataset_name": "BoolQ", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on BoolQ", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.464, - "details": { - "description": "min=0.405, mean=0.464, max=0.503, sum=1.392 (3)\n⚠ Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.", - "tab": "Accuracy", - "BoolQ - ECE (10-bin)": "{\"description\": \"min=0.257, mean=0.346, max=0.483, sum=1.039 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Calibration\", \"score\": \"0.34632807207915267\"}", - "BoolQ - EM (Robustness)": "{\"description\": \"min=0.316, mean=0.332, max=0.362, sum=0.997 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Robustness\", \"score\": \"0.33233333333333337\"}", - "BoolQ - EM (Fairness)": "{\"description\": \"min=0.364, mean=0.378, max=0.397, sum=1.134 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Fairness\", \"score\": \"0.37799999999999995\"}", - "BoolQ - Denoised inference time (s)": "{\"description\": \"min=0.09, mean=0.096, max=0.103, sum=0.287 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Efficiency\", \"score\": \"0.09557654231770833\"}", - "BoolQ - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "BoolQ - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "BoolQ - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "BoolQ - # prompt tokens": "{\"description\": \"min=660.073, mean=908.406, max=1242.073, sum=2725.219 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"908.4063333333334\"}", - "BoolQ - # output tokens": "{\"description\": \"min=0.995, mean=1.003, max=1.009, sum=3.009 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"1.003\"}", - "BoolQ - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "BoolQ - Stereotypes (race)": "{\"description\": \"(0)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Stereotypes (gender)": "{\"description\": \"(0)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (race)": "{\"description\": \"(0)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (gender)": "{\"description\": \"(0)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.238, - "details": { - "description": "min=0.22, mean=0.238, max=0.273, sum=0.714 (3)", - "tab": "Accuracy", - "NarrativeQA - ECE (10-bin)": "{\"description\": \"min=0.248, mean=0.319, max=0.386, sum=0.956 (3)\", \"tab\": \"Calibration\", \"score\": \"0.318718698868713\"}", - "NarrativeQA - F1 (Robustness)": "{\"description\": \"min=0.049, mean=0.058, max=0.075, sum=0.175 (3)\", \"tab\": \"Robustness\", \"score\": \"0.05828828370185365\"}", - "NarrativeQA - F1 (Fairness)": "{\"description\": \"min=0.11, mean=0.119, max=0.126, sum=0.356 (3)\", \"tab\": \"Fairness\", \"score\": \"0.1187630501762329\"}", - "NarrativeQA - Denoised inference time (s)": "{\"description\": \"min=0.16, mean=0.171, max=0.186, sum=0.513 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.1710890294894365\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=1065 (3)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=1.051, mean=1.647, max=2.085, sum=4.941 (3)\", \"tab\": \"General information\", \"score\": \"1.6469483568075116\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=1601.955, mean=1652.377, max=1705.003, sum=4957.132 (3)\", \"tab\": \"General information\", \"score\": \"1652.3774647887324\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=9.054, mean=10.756, max=13.293, sum=32.268 (3)\", \"tab\": \"General information\", \"score\": \"10.755868544600938\"}", - "NarrativeQA - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NarrativeQA - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NarrativeQA - Stereotypes (gender)": "{\"description\": \"min=0.382, mean=0.403, max=0.438, sum=1.21 (3)\", \"tab\": \"Bias\", \"score\": \"0.40317130936696155\"}", - "NarrativeQA - Representation (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=2 (3)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "NarrativeQA - Representation (gender)": "{\"description\": \"min=0.151, mean=0.203, max=0.252, sum=0.609 (3)\", \"tab\": \"Bias\", \"score\": \"0.20287726757892108\"}", - "NarrativeQA - Toxic fraction": "{\"description\": \"min=0.003, mean=0.006, max=0.008, sum=0.017 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.005633802816901408\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (open-book)", - "source_data": { - "dataset_name": "NaturalQuestions (open-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (open-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.149, - "details": { - "description": "min=0.06, mean=0.149, max=0.193, sum=0.446 (3)\n⚠ Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.", - "tab": "Accuracy", - "NaturalQuestions (closed-book) - ECE (10-bin)": "{\"description\": \"min=0.751, mean=0.764, max=0.789, sum=2.292 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Calibration\", \"score\": \"0.7640868917536278\"}", - "NaturalQuestions (open-book) - ECE (10-bin)": "{\"description\": \"min=0.6, mean=0.691, max=0.866, sum=2.072 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Calibration\", \"score\": \"0.6905918803748641\"}", - "NaturalQuestions (closed-book) - F1 (Robustness)": "{\"description\": \"min=0.007, mean=0.008, max=0.009, sum=0.023 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Robustness\", \"score\": \"0.007711173104376766\"}", - "NaturalQuestions (open-book) - F1 (Robustness)": "{\"description\": \"min=0.01, mean=0.034, max=0.062, sum=0.102 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Robustness\", \"score\": \"0.033837452909760764\"}", - "NaturalQuestions (closed-book) - F1 (Fairness)": "{\"description\": \"min=0.009, mean=0.012, max=0.018, sum=0.036 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Fairness\", \"score\": \"0.012133718750385417\"}", - "NaturalQuestions (open-book) - F1 (Fairness)": "{\"description\": \"min=0.026, mean=0.083, max=0.115, sum=0.249 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Fairness\", \"score\": \"0.08303504557607948\"}", - "NaturalQuestions (closed-book) - Denoised inference time (s)": "{\"description\": \"min=0.083, mean=0.085, max=0.087, sum=0.255 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Efficiency\", \"score\": \"0.08484092187500009\"}", - "NaturalQuestions (open-book) - Denoised inference time (s)": "{\"description\": \"min=0.119, mean=0.128, max=0.133, sum=0.383 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Efficiency\", \"score\": \"0.12779065299479173\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=110.254, mean=112.254, max=116.254, sum=336.762 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"112.254\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=0.729, mean=1.04, max=1.418, sum=3.12 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"1.0399999999999998\"}", - "NaturalQuestions (closed-book) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.647, mean=4.691, max=4.724, sum=14.074 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"4.691333333333334\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.036, mean=0.036, max=0.036, sum=0.108 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.036\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1231.212, mean=1419.574, max=1523.257, sum=4258.721 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"1419.5736666666664\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=1.801, mean=3.933, max=5.648, sum=11.799 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"3.933\"}", - "NaturalQuestions (open-book) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NaturalQuestions (closed-book) - Stereotypes (race)": "{\"description\": \"(0)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - Stereotypes (gender)": "{\"description\": \"(0)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - Representation (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=2 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "NaturalQuestions (closed-book) - Representation (gender)": "{\"description\": \"min=0.167, mean=0.167, max=0.167, sum=0.167 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.16666666666666666\"}", - "NaturalQuestions (open-book) - Stereotypes (race)": "{\"description\": \"(0)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"\"}", - "NaturalQuestions (open-book) - Stereotypes (gender)": "{\"description\": \"min=0.5, mean=0.5, max=0.5, sum=1 (2)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.5\"}", - "NaturalQuestions (open-book) - Representation (race)": "{\"description\": \"min=0.567, mean=0.633, max=0.667, sum=1.9 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.6333333333333334\"}", - "NaturalQuestions (open-book) - Representation (gender)": "{\"description\": \"min=0.1, mean=0.217, max=0.318, sum=0.652 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.21717171717171715\"}", - "NaturalQuestions (closed-book) - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}", - "NaturalQuestions (open-book) - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "QuAC", - "source_data": { - "dataset_name": "QuAC", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on QuAC", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.176, - "details": { - "description": "min=0.14, mean=0.176, max=0.203, sum=0.527 (3)\n⚠ Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.", - "tab": "Accuracy", - "QuAC - ECE (10-bin)": "{\"description\": \"min=0.16, mean=0.268, max=0.362, sum=0.803 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Calibration\", \"score\": \"0.2675195450588613\"}", - "QuAC - F1 (Robustness)": "{\"description\": \"min=0.054, mean=0.067, max=0.074, sum=0.201 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Robustness\", \"score\": \"0.06713428098997175\"}", - "QuAC - F1 (Fairness)": "{\"description\": \"min=0.063, mean=0.091, max=0.113, sum=0.273 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Fairness\", \"score\": \"0.09086419903543015\"}", - "QuAC - Denoised inference time (s)": "{\"description\": \"min=0.194, mean=0.21, max=0.221, sum=0.629 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Efficiency\", \"score\": \"0.20979015885416655\"}", - "QuAC - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "QuAC - # train": "{\"description\": \"min=0.845, mean=0.944, max=1.086, sum=2.833 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.9443333333333334\"}", - "QuAC - truncated": "{\"description\": \"min=0.016, mean=0.016, max=0.016, sum=0.048 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.016\"}", - "QuAC - # prompt tokens": "{\"description\": \"min=1625.523, mean=1644.831, max=1670.605, sum=4934.492 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"1644.8306666666667\"}", - "QuAC - # output tokens": "{\"description\": \"min=14.536, mean=17.274, max=19.327, sum=51.821 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"17.273666666666667\"}", - "QuAC - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "QuAC - Stereotypes (race)": "{\"description\": \"min=0.625, mean=0.653, max=0.667, sum=1.958 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.6527777777777778\"}", - "QuAC - Stereotypes (gender)": "{\"description\": \"min=0.415, mean=0.433, max=0.448, sum=1.3 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.4333686045042254\"}", - "QuAC - Representation (race)": "{\"description\": \"min=0.308, mean=0.345, max=0.387, sum=1.034 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.34482454482454483\"}", - "QuAC - Representation (gender)": "{\"description\": \"min=0.223, mean=0.244, max=0.269, sum=0.732 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.24387920564334062\"}", - "QuAC - Toxic fraction": "{\"description\": \"min=0, mean=0.0, max=0.001, sum=0.001 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Toxicity\", \"score\": \"0.0003333333333333333\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "HellaSwag", - "source_data": { - "dataset_name": "HellaSwag", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on HellaSwag", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.429, - "details": { - "description": "min=0.429, mean=0.429, max=0.429, sum=0.429 (1)\n⚠ Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.", - "tab": "Accuracy", - "HellaSwag - ECE (10-bin)": "{\"description\": \"min=0.103, mean=0.103, max=0.103, sum=0.103 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Calibration\", \"score\": \"0.1034689985203878\"}", - "HellaSwag - EM (Robustness)": "{\"description\": \"min=0.32, mean=0.32, max=0.32, sum=0.32 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Robustness\", \"score\": \"0.32\"}", - "HellaSwag - EM (Fairness)": "{\"description\": \"min=0.27, mean=0.27, max=0.27, sum=0.27 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Fairness\", \"score\": \"0.27\"}", - "HellaSwag - Denoised inference time (s)": "{\"description\": \"min=0.079, mean=0.079, max=0.079, sum=0.079 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Efficiency\", \"score\": \"0.07943312500000001\"}", - "HellaSwag - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "HellaSwag - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "HellaSwag - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "HellaSwag - # prompt tokens": "{\"description\": \"min=87.888, mean=87.888, max=87.888, sum=87.888 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"87.888\"}", - "HellaSwag - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "HellaSwag - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.346, - "details": { - "description": "min=0.346, mean=0.346, max=0.346, sum=0.346 (1)\n⚠ Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.", - "tab": "Accuracy", - "OpenbookQA - ECE (10-bin)": "{\"description\": \"min=0.487, mean=0.487, max=0.487, sum=0.487 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Calibration\", \"score\": \"0.4870210553256142\"}", - "OpenbookQA - EM (Robustness)": "{\"description\": \"min=0.248, mean=0.248, max=0.248, sum=0.248 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Robustness\", \"score\": \"0.248\"}", - "OpenbookQA - EM (Fairness)": "{\"description\": \"min=0.266, mean=0.266, max=0.266, sum=0.266 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Fairness\", \"score\": \"0.266\"}", - "OpenbookQA - Denoised inference time (s)": "{\"description\": \"min=0.076, mean=0.076, max=0.076, sum=0.076 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Efficiency\", \"score\": \"0.07620585937499988\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=5.27, mean=5.27, max=5.27, sum=5.27 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"5.27\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "TruthfulQA", - "source_data": { - "dataset_name": "TruthfulQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on TruthfulQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.232, - "details": { - "description": "min=0.216, mean=0.232, max=0.263, sum=0.696 (3)", - "tab": "Accuracy", - "TruthfulQA - ECE (10-bin)": "{\"description\": \"min=0.418, mean=0.465, max=0.495, sum=1.395 (3)\", \"tab\": \"Calibration\", \"score\": \"0.46507296315502505\"}", - "TruthfulQA - EM (Robustness)": "{\"description\": \"min=0.165, mean=0.175, max=0.194, sum=0.526 (3)\", \"tab\": \"Robustness\", \"score\": \"0.17533129459734964\"}", - "TruthfulQA - EM (Fairness)": "{\"description\": \"min=0.18, mean=0.191, max=0.213, sum=0.573 (3)\", \"tab\": \"Fairness\", \"score\": \"0.191131498470948\"}", - "TruthfulQA - Denoised inference time (s)": "{\"description\": \"min=0.088, mean=0.089, max=0.089, sum=0.266 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.08860781608371561\"}", - "TruthfulQA - # eval": "{\"description\": \"min=654, mean=654, max=654, sum=1962 (3)\", \"tab\": \"General information\", \"score\": \"654.0\"}", - "TruthfulQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "TruthfulQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "TruthfulQA - # prompt tokens": "{\"description\": \"min=501.121, mean=511.121, max=529.121, sum=1533.362 (3)\", \"tab\": \"General information\", \"score\": \"511.12079510703364\"}", - "TruthfulQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=3 (3)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "TruthfulQA - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MS MARCO (TREC)", - "source_data": { - "dataset_name": "MS MARCO (TREC)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "NDCG@10 on MS MARCO (TREC)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.302, - "details": { - "description": "min=0.21, mean=0.302, max=0.353, sum=0.905 (3)", - "tab": "Accuracy", - "MS MARCO (regular) - RR@10 (Robustness)": "{\"description\": \"min=0.044, mean=0.069, max=0.091, sum=0.207 (3)\", \"tab\": \"Robustness\", \"score\": \"0.06911044973544983\"}", - "MS MARCO (TREC) - NDCG@10 (Robustness)": "{\"description\": \"min=0.172, mean=0.252, max=0.302, sum=0.757 (3)\", \"tab\": \"Robustness\", \"score\": \"0.2521954718959493\"}", - "MS MARCO (regular) - RR@10 (Fairness)": "{\"description\": \"min=0.071, mean=0.107, max=0.133, sum=0.32 (3)\", \"tab\": \"Fairness\", \"score\": \"0.10653478835978836\"}", - "MS MARCO (TREC) - NDCG@10 (Fairness)": "{\"description\": \"min=0.177, mean=0.276, max=0.327, sum=0.827 (3)\", \"tab\": \"Fairness\", \"score\": \"0.2757254036023355\"}", - "MS MARCO (regular) - Denoised inference time (s)": "{\"description\": \"min=0.089, mean=0.09, max=0.091, sum=0.27 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.08991796223958341\"}", - "MS MARCO (TREC) - Denoised inference time (s)": "{\"description\": \"min=0.089, mean=0.09, max=0.09, sum=0.269 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.08954472504844961\"}", - "MS MARCO (regular) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "MS MARCO (regular) - # train": "{\"description\": \"min=2, mean=2, max=2, sum=6 (3)\", \"tab\": \"General information\", \"score\": \"2.0\"}", - "MS MARCO (regular) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MS MARCO (regular) - # prompt tokens": "{\"description\": \"min=495.232, mean=532.565, max=577.232, sum=1597.696 (3)\", \"tab\": \"General information\", \"score\": \"532.5653333333333\"}", - "MS MARCO (regular) - # output tokens": "{\"description\": \"min=1.014, mean=1.123, max=1.303, sum=3.369 (3)\", \"tab\": \"General information\", \"score\": \"1.123\"}", - "MS MARCO (regular) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "MS MARCO (TREC) - # eval": "{\"description\": \"min=43, mean=43, max=43, sum=129 (3)\", \"tab\": \"General information\", \"score\": \"43.0\"}", - "MS MARCO (TREC) - # train": "{\"description\": \"min=2, mean=2, max=2, sum=6 (3)\", \"tab\": \"General information\", \"score\": \"2.0\"}", - "MS MARCO (TREC) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MS MARCO (TREC) - # prompt tokens": "{\"description\": \"min=478.488, mean=515.822, max=560.488, sum=1547.465 (3)\", \"tab\": \"General information\", \"score\": \"515.8217054263565\"}", - "MS MARCO (TREC) - # output tokens": "{\"description\": \"min=0.953, mean=1.101, max=1.326, sum=3.302 (3)\", \"tab\": \"General information\", \"score\": \"1.1007751937984496\"}", - "MS MARCO (TREC) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "MS MARCO (regular) - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "MS MARCO (TREC) - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CNN/DailyMail", - "source_data": { - "dataset_name": "CNN/DailyMail", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on CNN/DailyMail", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.136, - "details": { - "description": "min=0.134, mean=0.136, max=0.137, sum=0.813 (6)", - "tab": "Accuracy", - "CNN/DailyMail - Denoised inference time (s)": "{\"description\": \"min=0.791, mean=0.793, max=0.796, sum=4.758 (6)\", \"tab\": \"Efficiency\", \"score\": \"0.7929256541152537\"}", - "CNN/DailyMail - # eval": "{\"description\": \"min=466, mean=466, max=466, sum=2796 (6)\", \"tab\": \"General information\", \"score\": \"466.0\"}", - "CNN/DailyMail - # train": "{\"description\": \"min=5, mean=5, max=5, sum=30 (6)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CNN/DailyMail - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (6)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CNN/DailyMail - # prompt tokens": "{\"description\": \"min=1531.586, mean=1549.919, max=1567.586, sum=9299.515 (6)\", \"tab\": \"General information\", \"score\": \"1549.9191702432045\"}", - "CNN/DailyMail - # output tokens": "{\"description\": \"min=114.727, mean=114.938, max=115.313, sum=689.627 (6)\", \"tab\": \"General information\", \"score\": \"114.93776824034335\"}", - "CNN/DailyMail - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=18 (6)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "CNN/DailyMail - Stereotypes (race)": "{\"description\": \"min=0.585, mean=0.603, max=0.618, sum=3.62 (6)\", \"tab\": \"Bias\", \"score\": \"0.6033209686988849\"}", - "CNN/DailyMail - Stereotypes (gender)": "{\"description\": \"min=0.366, mean=0.376, max=0.394, sum=2.258 (6)\", \"tab\": \"Bias\", \"score\": \"0.376337569695528\"}", - "CNN/DailyMail - Representation (race)": "{\"description\": \"min=0.32, mean=0.327, max=0.336, sum=1.964 (6)\", \"tab\": \"Bias\", \"score\": \"0.3273411562788524\"}", - "CNN/DailyMail - Representation (gender)": "{\"description\": \"min=0.118, mean=0.135, max=0.151, sum=0.81 (6)\", \"tab\": \"Bias\", \"score\": \"0.13502681064518518\"}", - "CNN/DailyMail - Toxic fraction": "{\"description\": \"min=0, mean=0.001, max=0.002, sum=0.004 (6)\", \"tab\": \"Toxicity\", \"score\": \"0.000715307582260372\"}", - "CNN/DailyMail - SummaC": "{\"description\": \"min=0.202, mean=0.223, max=0.237, sum=0.67 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.22335669413101697\"}", - "CNN/DailyMail - QAFactEval": "{\"description\": \"min=2.69, mean=3.369, max=3.833, sum=20.217 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"3.3694626717468696\"}", - "CNN/DailyMail - BERTScore (F1)": "{\"description\": \"min=0.244, mean=0.247, max=0.25, sum=0.741 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.2468463296383967\"}", - "CNN/DailyMail - Coverage": "{\"description\": \"min=0.923, mean=0.929, max=0.933, sum=5.574 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"0.9289690481394134\"}", - "CNN/DailyMail - Density": "{\"description\": \"min=28.745, mean=31.424, max=35.767, sum=188.544 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"31.424005422737114\"}", - "CNN/DailyMail - Compression": "{\"description\": \"min=5.334, mean=5.461, max=5.548, sum=32.769 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"5.461465024583634\"}", - "CNN/DailyMail - HumanEval-faithfulness": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-relevance": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-coherence": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "XSUM", - "source_data": { - "dataset_name": "XSUM", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on XSUM", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.034, - "details": { - "description": "min=0.034, mean=0.034, max=0.036, sum=0.206 (6)", - "tab": "Accuracy", - "XSUM - Denoised inference time (s)": "{\"description\": \"min=0.304, mean=0.311, max=0.318, sum=1.868 (6)\", \"tab\": \"Efficiency\", \"score\": \"0.31128436946991633\"}", - "XSUM - # eval": "{\"description\": \"min=518, mean=518, max=518, sum=3108 (6)\", \"tab\": \"General information\", \"score\": \"518.0\"}", - "XSUM - # train": "{\"description\": \"min=4.998, mean=4.999, max=5, sum=29.992 (6)\", \"tab\": \"General information\", \"score\": \"4.998712998712999\"}", - "XSUM - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (6)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "XSUM - # prompt tokens": "{\"description\": \"min=1456.402, mean=1510.418, max=1538.921, sum=9062.51 (6)\", \"tab\": \"General information\", \"score\": \"1510.4182754182755\"}", - "XSUM - # output tokens": "{\"description\": \"min=33.533, mean=34.806, max=36.037, sum=208.834 (6)\", \"tab\": \"General information\", \"score\": \"34.805662805662806\"}", - "XSUM - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=18 (6)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "XSUM - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=4.0 (6)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666669\"}", - "XSUM - Stereotypes (gender)": "{\"description\": \"min=0.387, mean=0.403, max=0.414, sum=2.418 (6)\", \"tab\": \"Bias\", \"score\": \"0.4030736615819075\"}", - "XSUM - Representation (race)": "{\"description\": \"min=0.547, mean=0.597, max=0.623, sum=3.579 (6)\", \"tab\": \"Bias\", \"score\": \"0.5965455454885051\"}", - "XSUM - Representation (gender)": "{\"description\": \"min=0.087, mean=0.19, max=0.25, sum=1.142 (6)\", \"tab\": \"Bias\", \"score\": \"0.19037429957632912\"}", - "XSUM - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (6)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}", - "XSUM - SummaC": "{\"description\": \"min=-0.132, mean=-0.102, max=-0.078, sum=-0.305 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"-0.10168572979799827\"}", - "XSUM - QAFactEval": "{\"description\": \"min=4.849, mean=4.929, max=5.055, sum=29.572 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"4.92859074878104\"}", - "XSUM - BERTScore (F1)": "{\"description\": \"min=0.237, mean=0.245, max=0.254, sum=0.734 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.24476258912195994\"}", - "XSUM - Coverage": "{\"description\": \"min=0.834, mean=0.847, max=0.866, sum=5.08 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"0.8466942307223615\"}", - "XSUM - Density": "{\"description\": \"min=7.289, mean=7.626, max=8.299, sum=45.753 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"7.625570347216255\"}", - "XSUM - Compression": "{\"description\": \"min=12.7, mean=13.08, max=13.496, sum=78.483 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"13.080494860928995\"}", - "XSUM - HumanEval-faithfulness": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-relevance": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-coherence": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "IMDB", - "source_data": { - "dataset_name": "IMDB", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on IMDB", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.822, - "details": { - "description": "min=0.776, mean=0.822, max=0.853, sum=2.466 (3)", - "tab": "Accuracy", - "IMDB - ECE (10-bin)": "{\"description\": \"min=0.053, mean=0.09, max=0.142, sum=0.269 (3)\", \"tab\": \"Calibration\", \"score\": \"0.08977338148861268\"}", - "IMDB - EM (Robustness)": "{\"description\": \"min=0.663, mean=0.716, max=0.744, sum=2.148 (3)\", \"tab\": \"Robustness\", \"score\": \"0.7160000000000001\"}", - "IMDB - EM (Fairness)": "{\"description\": \"min=0.724, mean=0.769, max=0.808, sum=2.308 (3)\", \"tab\": \"Fairness\", \"score\": \"0.7693333333333333\"}", - "IMDB - Denoised inference time (s)": "{\"description\": \"min=0.104, mean=0.109, max=0.114, sum=0.328 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.109459033203125\"}", - "IMDB - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "IMDB - # train": "{\"description\": \"min=2.916, mean=4.242, max=4.986, sum=12.726 (3)\", \"tab\": \"General information\", \"score\": \"4.242\"}", - "IMDB - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IMDB - # prompt tokens": "{\"description\": \"min=1277.729, mean=1553.363, max=1768.607, sum=4660.089 (3)\", \"tab\": \"General information\", \"score\": \"1553.363\"}", - "IMDB - # output tokens": "{\"description\": \"min=1.006, mean=1.013, max=1.021, sum=3.039 (3)\", \"tab\": \"General information\", \"score\": \"1.013\"}", - "IMDB - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "IMDB - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CivilComments", - "source_data": { - "dataset_name": "CivilComments", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on CivilComments", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.503, - "details": { - "description": "min=0, mean=0.503, max=1, sum=27.18 (54)", - "tab": "Accuracy", - "CivilComments - ECE (10-bin)": "{\"description\": \"min=0.012, mean=0.479, max=0.985, sum=25.845 (54)\", \"tab\": \"Calibration\", \"score\": \"0.47860750507636396\"}", - "CivilComments - EM (Robustness)": "{\"description\": \"min=0, mean=0.491, max=1, sum=26.518 (54)\", \"tab\": \"Robustness\", \"score\": \"0.4910745197871521\"}", - "CivilComments - EM (Fairness)": "{\"description\": \"min=0, mean=0.497, max=1, sum=26.82 (54)\", \"tab\": \"Fairness\", \"score\": \"0.49665917233754203\"}", - "CivilComments - Denoised inference time (s)": "{\"description\": \"min=0.086, mean=0.092, max=0.103, sum=4.964 (54)\", \"tab\": \"Efficiency\", \"score\": \"0.0919244734885576\"}", - "CivilComments - # eval": "{\"description\": \"min=74, mean=371.556, max=683, sum=20064 (54)\", \"tab\": \"General information\", \"score\": \"371.55555555555554\"}", - "CivilComments - # train": "{\"description\": \"min=5, mean=5, max=5, sum=270 (54)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CivilComments - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (54)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CivilComments - # prompt tokens": "{\"description\": \"min=356.537, mean=722.635, max=1267.519, sum=39022.317 (54)\", \"tab\": \"General information\", \"score\": \"722.6354931173206\"}", - "CivilComments - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=54 (54)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "CivilComments - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=162 (54)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "CivilComments - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (54)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "RAFT", - "source_data": { - "dataset_name": "RAFT", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on RAFT", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.406, - "details": { - "description": "min=0.05, mean=0.406, max=0.975, sum=13.4 (33)", - "tab": "Accuracy", - "RAFT - ECE (10-bin)": "{\"description\": \"min=0.018, mean=0.473, max=0.891, sum=15.613 (33)\", \"tab\": \"Calibration\", \"score\": \"0.47311876061285835\"}", - "RAFT - EM (Robustness)": "{\"description\": \"min=0, mean=0.335, max=0.925, sum=11.05 (33)\", \"tab\": \"Robustness\", \"score\": \"0.3348484848484849\"}", - "RAFT - EM (Fairness)": "{\"description\": \"min=0.05, mean=0.376, max=0.975, sum=12.4 (33)\", \"tab\": \"Fairness\", \"score\": \"0.3757575757575758\"}", - "RAFT - Denoised inference time (s)": "{\"description\": \"min=0.084, mean=0.107, max=0.14, sum=3.527 (33)\", \"tab\": \"Efficiency\", \"score\": \"0.10687999526515152\"}", - "RAFT - # eval": "{\"description\": \"min=40, mean=40, max=40, sum=1320 (33)\", \"tab\": \"General information\", \"score\": \"40.0\"}", - "RAFT - # train": "{\"description\": \"min=0, mean=4.556, max=5, sum=150.35 (33)\", \"tab\": \"General information\", \"score\": \"4.556060606060607\"}", - "RAFT - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (33)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "RAFT - # prompt tokens": "{\"description\": \"min=257.35, mean=812.938, max=1773.675, sum=26826.95 (33)\", \"tab\": \"General information\", \"score\": \"812.937878787879\"}", - "RAFT - # output tokens": "{\"description\": \"min=0.15, mean=2.997, max=6.925, sum=98.9 (33)\", \"tab\": \"General information\", \"score\": \"2.996969696969697\"}", - "RAFT - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=99 (33)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "RAFT - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (33)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/openai_text-babbage-001.json b/data/models/openai_text-babbage-001.json deleted file mode 100644 index 83f95fd497e0381439f7c584d29cd4ebc5e39417..0000000000000000000000000000000000000000 --- a/data/models/openai_text-babbage-001.json +++ /dev/null @@ -1,674 +0,0 @@ -{ - "model_info": { - "name": "text-babbage-001", - "id": "openai/text-babbage-001", - "developer": "OpenAI", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_classic/openai_text-babbage-001/1774096308.339228", - "retrieved_timestamp": "1774096308.339228", - "source_metadata": { - "source_name": "helm_classic", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_classic", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_classic", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperform on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.229, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Calibration": "{\"description\": \"\", \"tab\": \"Calibration\", \"score\": \"0.27686841173581844\"}", - "Mean win rate - Robustness": "{\"description\": \"\", \"tab\": \"Robustness\", \"score\": \"0.22569775422945612\"}", - "Mean win rate - Fairness": "{\"description\": \"\", \"tab\": \"Fairness\", \"score\": \"0.2438772758572536\"}", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.7775548245614035\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}", - "Mean win rate - Bias": "{\"description\": \"\", \"tab\": \"Bias\", \"score\": \"0.5333126239886427\"}", - "Mean win rate - Toxicity": "{\"description\": \"\", \"tab\": \"Toxicity\", \"score\": \"0.5020704604037938\"}", - "Mean win rate - Summarization metrics": "{\"description\": \"\", \"tab\": \"Summarization metrics\", \"score\": \"0.6459690893901421\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.229, - "details": { - "description": "min=0.11, mean=0.229, max=0.325, sum=3.431 (15)", - "tab": "Accuracy", - "MMLU - ECE (10-bin)": "{\"description\": \"min=0.16, mean=0.311, max=0.472, sum=4.659 (15)\", \"tab\": \"Calibration\", \"score\": \"0.31056724427484883\"}", - "MMLU - EM (Robustness)": "{\"description\": \"min=0.1, mean=0.186, max=0.228, sum=2.79 (15)\", \"tab\": \"Robustness\", \"score\": \"0.18602339181286548\"}", - "MMLU - EM (Fairness)": "{\"description\": \"min=0.09, mean=0.205, max=0.272, sum=3.077 (15)\", \"tab\": \"Fairness\", \"score\": \"0.20512280701754387\"}", - "MMLU - Denoised inference time (s)": "{\"description\": \"min=0.131, mean=0.133, max=0.135, sum=1.99 (15)\", \"tab\": \"Efficiency\", \"score\": \"0.13263352809758774\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=1542 (15)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=75 (15)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (15)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=371.38, mean=472.274, max=624.07, sum=7084.111 (15)\", \"tab\": \"General information\", \"score\": \"472.2740350877193\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=15 (15)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "MMLU - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=45 (15)\", \"tab\": \"General information\", \"score\": \"3.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "BoolQ", - "source_data": { - "dataset_name": "BoolQ", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on BoolQ", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.451, - "details": { - "description": "min=0.414, mean=0.451, max=0.477, sum=1.353 (3)\n⚠ Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.", - "tab": "Accuracy", - "BoolQ - ECE (10-bin)": "{\"description\": \"min=0.318, mean=0.344, max=0.371, sum=1.031 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Calibration\", \"score\": \"0.34372183455656985\"}", - "BoolQ - EM (Robustness)": "{\"description\": \"min=0.339, mean=0.384, max=0.412, sum=1.151 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Robustness\", \"score\": \"0.38366666666666666\"}", - "BoolQ - EM (Fairness)": "{\"description\": \"min=0.388, mean=0.41, max=0.43, sum=1.23 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Fairness\", \"score\": \"0.41\"}", - "BoolQ - Denoised inference time (s)": "{\"description\": \"min=0.136, mean=0.142, max=0.15, sum=0.426 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Efficiency\", \"score\": \"0.14212787000868074\"}", - "BoolQ - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "BoolQ - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "BoolQ - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "BoolQ - # prompt tokens": "{\"description\": \"min=660.073, mean=908.406, max=1242.073, sum=2725.219 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"908.4063333333334\"}", - "BoolQ - # output tokens": "{\"description\": \"min=1, mean=1.004, max=1.008, sum=3.012 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"1.004\"}", - "BoolQ - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "BoolQ - Stereotypes (race)": "{\"description\": \"(0)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Stereotypes (gender)": "{\"description\": \"(0)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (race)": "{\"description\": \"(0)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (gender)": "{\"description\": \"(0)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.429, - "details": { - "description": "min=0.412, mean=0.429, max=0.463, sum=1.288 (3)", - "tab": "Accuracy", - "NarrativeQA - ECE (10-bin)": "{\"description\": \"min=0.158, mean=0.186, max=0.215, sum=0.557 (3)\", \"tab\": \"Calibration\", \"score\": \"0.18581698260430923\"}", - "NarrativeQA - F1 (Robustness)": "{\"description\": \"min=0.101, mean=0.126, max=0.154, sum=0.377 (3)\", \"tab\": \"Robustness\", \"score\": \"0.12577588570182116\"}", - "NarrativeQA - F1 (Fairness)": "{\"description\": \"min=0.277, mean=0.299, max=0.335, sum=0.896 (3)\", \"tab\": \"Fairness\", \"score\": \"0.29864937428822036\"}", - "NarrativeQA - Denoised inference time (s)": "{\"description\": \"min=0.239, mean=0.243, max=0.246, sum=0.728 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.24279079738849765\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=1065 (3)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=1.051, mean=1.647, max=2.085, sum=4.941 (3)\", \"tab\": \"General information\", \"score\": \"1.6469483568075116\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=1601.955, mean=1652.377, max=1705.003, sum=4957.132 (3)\", \"tab\": \"General information\", \"score\": \"1652.3774647887324\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=12.048, mean=12.829, max=13.307, sum=38.487 (3)\", \"tab\": \"General information\", \"score\": \"12.829107981220657\"}", - "NarrativeQA - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NarrativeQA - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=0.667 (1)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666667\"}", - "NarrativeQA - Stereotypes (gender)": "{\"description\": \"min=0.382, mean=0.403, max=0.433, sum=1.209 (3)\", \"tab\": \"Bias\", \"score\": \"0.40286362942612947\"}", - "NarrativeQA - Representation (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=2 (3)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "NarrativeQA - Representation (gender)": "{\"description\": \"min=0.089, mean=0.132, max=0.178, sum=0.395 (3)\", \"tab\": \"Bias\", \"score\": \"0.13153743304740043\"}", - "NarrativeQA - Toxic fraction": "{\"description\": \"min=0.003, mean=0.009, max=0.02, sum=0.028 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.009389671361502348\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (open-book)", - "source_data": { - "dataset_name": "NaturalQuestions (open-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (open-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.33, - "details": { - "description": "min=0.296, mean=0.33, max=0.355, sum=0.989 (3)\n⚠ Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.", - "tab": "Accuracy", - "NaturalQuestions (closed-book) - ECE (10-bin)": "{\"description\": \"min=0.505, mean=0.522, max=0.555, sum=1.567 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Calibration\", \"score\": \"0.5224886706365456\"}", - "NaturalQuestions (open-book) - ECE (10-bin)": "{\"description\": \"min=0.346, mean=0.385, max=0.427, sum=1.155 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Calibration\", \"score\": \"0.38493664744185446\"}", - "NaturalQuestions (closed-book) - F1 (Robustness)": "{\"description\": \"min=0.039, mean=0.04, max=0.041, sum=0.119 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Robustness\", \"score\": \"0.039736972833954616\"}", - "NaturalQuestions (open-book) - F1 (Robustness)": "{\"description\": \"min=0.139, mean=0.151, max=0.169, sum=0.452 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Robustness\", \"score\": \"0.15066474277626352\"}", - "NaturalQuestions (closed-book) - F1 (Fairness)": "{\"description\": \"min=0.048, mean=0.053, max=0.057, sum=0.16 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Fairness\", \"score\": \"0.05326475617936846\"}", - "NaturalQuestions (open-book) - F1 (Fairness)": "{\"description\": \"min=0.209, mean=0.24, max=0.263, sum=0.72 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Fairness\", \"score\": \"0.23984494964196315\"}", - "NaturalQuestions (closed-book) - Denoised inference time (s)": "{\"description\": \"min=0.134, mean=0.136, max=0.137, sum=0.407 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Efficiency\", \"score\": \"0.1355529375\"}", - "NaturalQuestions (open-book) - Denoised inference time (s)": "{\"description\": \"min=0.2, mean=0.204, max=0.207, sum=0.612 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Efficiency\", \"score\": \"0.20402605620659717\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=110.254, mean=112.254, max=116.254, sum=336.762 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"112.254\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=1.708, mean=2.016, max=2.304, sum=6.048 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"2.016\"}", - "NaturalQuestions (closed-book) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.647, mean=4.691, max=4.724, sum=14.074 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"4.691333333333334\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.036, mean=0.036, max=0.036, sum=0.108 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.036\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1231.212, mean=1419.574, max=1523.257, sum=4258.721 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"1419.5736666666664\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=7.676, mean=7.772, max=7.9, sum=23.317 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"7.772333333333333\"}", - "NaturalQuestions (open-book) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NaturalQuestions (closed-book) - Stereotypes (race)": "{\"description\": \"(0)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - Stereotypes (gender)": "{\"description\": \"min=0.5, mean=0.5, max=0.5, sum=1 (2)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.5\"}", - "NaturalQuestions (closed-book) - Representation (race)": "{\"description\": \"min=0.238, mean=0.317, max=0.467, sum=0.95 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.3167919799498747\"}", - "NaturalQuestions (closed-book) - Representation (gender)": "{\"description\": \"min=0.125, mean=0.145, max=0.167, sum=0.435 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.14484126984126985\"}", - "NaturalQuestions (open-book) - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=2 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "NaturalQuestions (open-book) - Stereotypes (gender)": "{\"description\": \"min=0.286, mean=0.333, max=0.364, sum=0.999 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.3331168831168831\"}", - "NaturalQuestions (open-book) - Representation (race)": "{\"description\": \"min=0.35, mean=0.403, max=0.457, sum=1.208 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.4025813878698122\"}", - "NaturalQuestions (open-book) - Representation (gender)": "{\"description\": \"min=0.221, mean=0.243, max=0.273, sum=0.728 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.2427837942788109\"}", - "NaturalQuestions (closed-book) - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}", - "NaturalQuestions (open-book) - Toxic fraction": "{\"description\": \"min=0, mean=0.0, max=0.001, sum=0.001 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Toxicity\", \"score\": \"0.0003333333333333333\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "QuAC", - "source_data": { - "dataset_name": "QuAC", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on QuAC", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.284, - "details": { - "description": "min=0.279, mean=0.284, max=0.288, sum=0.852 (3)\n⚠ Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.", - "tab": "Accuracy", - "QuAC - ECE (10-bin)": "{\"description\": \"min=0.224, mean=0.24, max=0.25, sum=0.72 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Calibration\", \"score\": \"0.2399406998223789\"}", - "QuAC - F1 (Robustness)": "{\"description\": \"min=0.083, mean=0.087, max=0.091, sum=0.261 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Robustness\", \"score\": \"0.08703476784265192\"}", - "QuAC - F1 (Fairness)": "{\"description\": \"min=0.188, mean=0.196, max=0.202, sum=0.589 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Fairness\", \"score\": \"0.19638729492261867\"}", - "QuAC - Denoised inference time (s)": "{\"description\": \"min=0.305, mean=0.314, max=0.32, sum=0.941 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Efficiency\", \"score\": \"0.3136292994791667\"}", - "QuAC - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "QuAC - # train": "{\"description\": \"min=0.845, mean=0.944, max=1.086, sum=2.833 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.9443333333333334\"}", - "QuAC - truncated": "{\"description\": \"min=0.016, mean=0.016, max=0.016, sum=0.048 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.016\"}", - "QuAC - # prompt tokens": "{\"description\": \"min=1625.523, mean=1644.831, max=1670.605, sum=4934.492 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"1644.8306666666667\"}", - "QuAC - # output tokens": "{\"description\": \"min=21.715, mean=22.966, max=24.001, sum=68.897 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"22.965666666666667\"}", - "QuAC - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "QuAC - Stereotypes (race)": "{\"description\": \"min=0.593, mean=0.617, max=0.643, sum=1.851 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.6171143671143672\"}", - "QuAC - Stereotypes (gender)": "{\"description\": \"min=0.425, mean=0.435, max=0.449, sum=1.305 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.43511418044370825\"}", - "QuAC - Representation (race)": "{\"description\": \"min=0.342, mean=0.361, max=0.388, sum=1.084 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.36134886795921545\"}", - "QuAC - Representation (gender)": "{\"description\": \"min=0.255, mean=0.26, max=0.268, sum=0.779 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.25974518866516266\"}", - "QuAC - Toxic fraction": "{\"description\": \"min=0, mean=0.0, max=0.001, sum=0.001 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Toxicity\", \"score\": \"0.0003333333333333333\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "HellaSwag", - "source_data": { - "dataset_name": "HellaSwag", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on HellaSwag", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.561, - "details": { - "description": "min=0.561, mean=0.561, max=0.561, sum=0.561 (1)\n⚠ Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.", - "tab": "Accuracy", - "HellaSwag - ECE (10-bin)": "{\"description\": \"min=0.083, mean=0.083, max=0.083, sum=0.083 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Calibration\", \"score\": \"0.08291053064819098\"}", - "HellaSwag - EM (Robustness)": "{\"description\": \"min=0.468, mean=0.468, max=0.468, sum=0.468 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Robustness\", \"score\": \"0.468\"}", - "HellaSwag - EM (Fairness)": "{\"description\": \"min=0.405, mean=0.405, max=0.405, sum=0.405 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Fairness\", \"score\": \"0.405\"}", - "HellaSwag - Denoised inference time (s)": "{\"description\": \"min=0.125, mean=0.125, max=0.125, sum=0.125 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Efficiency\", \"score\": \"0.12474649999999997\"}", - "HellaSwag - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "HellaSwag - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "HellaSwag - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "HellaSwag - # prompt tokens": "{\"description\": \"min=87.888, mean=87.888, max=87.888, sum=87.888 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"87.888\"}", - "HellaSwag - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "HellaSwag - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.452, - "details": { - "description": "min=0.452, mean=0.452, max=0.452, sum=0.452 (1)\n⚠ Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.", - "tab": "Accuracy", - "OpenbookQA - ECE (10-bin)": "{\"description\": \"min=0.362, mean=0.362, max=0.362, sum=0.362 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Calibration\", \"score\": \"0.36220844968968424\"}", - "OpenbookQA - EM (Robustness)": "{\"description\": \"min=0.39, mean=0.39, max=0.39, sum=0.39 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Robustness\", \"score\": \"0.39\"}", - "OpenbookQA - EM (Fairness)": "{\"description\": \"min=0.386, mean=0.386, max=0.386, sum=0.386 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Fairness\", \"score\": \"0.386\"}", - "OpenbookQA - Denoised inference time (s)": "{\"description\": \"min=0.122, mean=0.122, max=0.122, sum=0.122 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Efficiency\", \"score\": \"0.12216468749999997\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=5.27, mean=5.27, max=5.27, sum=5.27 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"5.27\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "TruthfulQA", - "source_data": { - "dataset_name": "TruthfulQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on TruthfulQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.233, - "details": { - "description": "min=0.2, mean=0.233, max=0.274, sum=0.699 (3)", - "tab": "Accuracy", - "TruthfulQA - ECE (10-bin)": "{\"description\": \"min=0.202, mean=0.251, max=0.279, sum=0.752 (3)\", \"tab\": \"Calibration\", \"score\": \"0.2505684624777335\"}", - "TruthfulQA - EM (Robustness)": "{\"description\": \"min=0.156, mean=0.195, max=0.252, sum=0.586 (3)\", \"tab\": \"Robustness\", \"score\": \"0.19520897043832822\"}", - "TruthfulQA - EM (Fairness)": "{\"description\": \"min=0.173, mean=0.207, max=0.257, sum=0.622 (3)\", \"tab\": \"Fairness\", \"score\": \"0.20744138634046894\"}", - "TruthfulQA - Denoised inference time (s)": "{\"description\": \"min=0.133, mean=0.134, max=0.134, sum=0.401 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.1335233459161568\"}", - "TruthfulQA - # eval": "{\"description\": \"min=654, mean=654, max=654, sum=1962 (3)\", \"tab\": \"General information\", \"score\": \"654.0\"}", - "TruthfulQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "TruthfulQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "TruthfulQA - # prompt tokens": "{\"description\": \"min=501.121, mean=511.121, max=529.121, sum=1533.362 (3)\", \"tab\": \"General information\", \"score\": \"511.12079510703364\"}", - "TruthfulQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=3 (3)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "TruthfulQA - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MS MARCO (TREC)", - "source_data": { - "dataset_name": "MS MARCO (TREC)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "NDCG@10 on MS MARCO (TREC)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.449, - "details": { - "description": "min=0.42, mean=0.449, max=0.493, sum=1.347 (3)", - "tab": "Accuracy", - "MS MARCO (regular) - RR@10 (Robustness)": "{\"description\": \"min=0.099, mean=0.122, max=0.16, sum=0.366 (3)\", \"tab\": \"Robustness\", \"score\": \"0.12212023809523809\"}", - "MS MARCO (TREC) - NDCG@10 (Robustness)": "{\"description\": \"min=0.315, mean=0.356, max=0.413, sum=1.069 (3)\", \"tab\": \"Robustness\", \"score\": \"0.35630094105473137\"}", - "MS MARCO (regular) - RR@10 (Fairness)": "{\"description\": \"min=0.152, mean=0.174, max=0.213, sum=0.523 (3)\", \"tab\": \"Fairness\", \"score\": \"0.17431719576719562\"}", - "MS MARCO (TREC) - NDCG@10 (Fairness)": "{\"description\": \"min=0.396, mean=0.424, max=0.469, sum=1.273 (3)\", \"tab\": \"Fairness\", \"score\": \"0.4244404820446352\"}", - "MS MARCO (regular) - Denoised inference time (s)": "{\"description\": \"min=0.136, mean=0.136, max=0.136, sum=0.408 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.1359015429687499\"}", - "MS MARCO (TREC) - Denoised inference time (s)": "{\"description\": \"min=0.135, mean=0.135, max=0.136, sum=0.406 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.1353138323643411\"}", - "MS MARCO (regular) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "MS MARCO (regular) - # train": "{\"description\": \"min=2, mean=2, max=2, sum=6 (3)\", \"tab\": \"General information\", \"score\": \"2.0\"}", - "MS MARCO (regular) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MS MARCO (regular) - # prompt tokens": "{\"description\": \"min=495.232, mean=532.565, max=577.232, sum=1597.696 (3)\", \"tab\": \"General information\", \"score\": \"532.5653333333333\"}", - "MS MARCO (regular) - # output tokens": "{\"description\": \"min=1.142, mean=1.212, max=1.282, sum=3.635 (3)\", \"tab\": \"General information\", \"score\": \"1.2116666666666667\"}", - "MS MARCO (regular) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "MS MARCO (TREC) - # eval": "{\"description\": \"min=43, mean=43, max=43, sum=129 (3)\", \"tab\": \"General information\", \"score\": \"43.0\"}", - "MS MARCO (TREC) - # train": "{\"description\": \"min=2, mean=2, max=2, sum=6 (3)\", \"tab\": \"General information\", \"score\": \"2.0\"}", - "MS MARCO (TREC) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MS MARCO (TREC) - # prompt tokens": "{\"description\": \"min=478.488, mean=515.822, max=560.488, sum=1547.465 (3)\", \"tab\": \"General information\", \"score\": \"515.8217054263565\"}", - "MS MARCO (TREC) - # output tokens": "{\"description\": \"min=0.977, mean=1.132, max=1.326, sum=3.395 (3)\", \"tab\": \"General information\", \"score\": \"1.1317829457364341\"}", - "MS MARCO (TREC) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "MS MARCO (regular) - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "MS MARCO (TREC) - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CNN/DailyMail", - "source_data": { - "dataset_name": "CNN/DailyMail", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on CNN/DailyMail", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.151, - "details": { - "description": "min=0.147, mean=0.151, max=0.155, sum=0.907 (6)", - "tab": "Accuracy", - "CNN/DailyMail - Denoised inference time (s)": "{\"description\": \"min=0.951, mean=0.968, max=0.994, sum=5.81 (6)\", \"tab\": \"Efficiency\", \"score\": \"0.9683207451306926\"}", - "CNN/DailyMail - # eval": "{\"description\": \"min=466, mean=466, max=466, sum=2796 (6)\", \"tab\": \"General information\", \"score\": \"466.0\"}", - "CNN/DailyMail - # train": "{\"description\": \"min=5, mean=5, max=5, sum=30 (6)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CNN/DailyMail - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (6)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CNN/DailyMail - # prompt tokens": "{\"description\": \"min=1531.586, mean=1549.919, max=1567.586, sum=9299.515 (6)\", \"tab\": \"General information\", \"score\": \"1549.9191702432045\"}", - "CNN/DailyMail - # output tokens": "{\"description\": \"min=114.333, mean=116.858, max=120.519, sum=701.146 (6)\", \"tab\": \"General information\", \"score\": \"116.85765379113019\"}", - "CNN/DailyMail - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=18 (6)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "CNN/DailyMail - Stereotypes (race)": "{\"description\": \"min=0.623, mean=0.626, max=0.63, sum=3.757 (6)\", \"tab\": \"Bias\", \"score\": \"0.6261965622126104\"}", - "CNN/DailyMail - Stereotypes (gender)": "{\"description\": \"min=0.369, mean=0.385, max=0.401, sum=2.312 (6)\", \"tab\": \"Bias\", \"score\": \"0.3853218330657557\"}", - "CNN/DailyMail - Representation (race)": "{\"description\": \"min=0.366, mean=0.389, max=0.408, sum=2.333 (6)\", \"tab\": \"Bias\", \"score\": \"0.38877532854423413\"}", - "CNN/DailyMail - Representation (gender)": "{\"description\": \"min=0.142, mean=0.147, max=0.152, sum=0.879 (6)\", \"tab\": \"Bias\", \"score\": \"0.14657801266351475\"}", - "CNN/DailyMail - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (6)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}", - "CNN/DailyMail - SummaC": "{\"description\": \"min=0.347, mean=0.378, max=0.402, sum=1.135 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.3784199534784201\"}", - "CNN/DailyMail - QAFactEval": "{\"description\": \"min=4.659, mean=4.676, max=4.708, sum=28.057 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"4.676089387380419\"}", - "CNN/DailyMail - BERTScore (F1)": "{\"description\": \"min=0.277, mean=0.282, max=0.285, sum=0.845 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.28169928727191773\"}", - "CNN/DailyMail - Coverage": "{\"description\": \"min=0.969, mean=0.972, max=0.973, sum=5.83 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"0.9716251936961523\"}", - "CNN/DailyMail - Density": "{\"description\": \"min=41.642, mean=45.948, max=53.738, sum=275.691 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"45.94847550953912\"}", - "CNN/DailyMail - Compression": "{\"description\": \"min=5.013, mean=5.291, max=5.576, sum=31.744 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"5.290663826380655\"}", - "CNN/DailyMail - HumanEval-faithfulness": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-relevance": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-coherence": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "XSUM", - "source_data": { - "dataset_name": "XSUM", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on XSUM", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.046, - "details": { - "description": "min=0.044, mean=0.046, max=0.047, sum=0.275 (6)", - "tab": "Accuracy", - "XSUM - Denoised inference time (s)": "{\"description\": \"min=0.416, mean=0.431, max=0.439, sum=2.583 (6)\", \"tab\": \"Efficiency\", \"score\": \"0.43057023625187685\"}", - "XSUM - # eval": "{\"description\": \"min=518, mean=518, max=518, sum=3108 (6)\", \"tab\": \"General information\", \"score\": \"518.0\"}", - "XSUM - # train": "{\"description\": \"min=4.998, mean=4.999, max=5, sum=29.992 (6)\", \"tab\": \"General information\", \"score\": \"4.998712998712999\"}", - "XSUM - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (6)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "XSUM - # prompt tokens": "{\"description\": \"min=1456.402, mean=1510.418, max=1538.921, sum=9062.51 (6)\", \"tab\": \"General information\", \"score\": \"1510.4182754182755\"}", - "XSUM - # output tokens": "{\"description\": \"min=38.037, mean=40.165, max=41.259, sum=240.988 (6)\", \"tab\": \"General information\", \"score\": \"40.16473616473616\"}", - "XSUM - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=18 (6)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "XSUM - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=4 (6)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "XSUM - Stereotypes (gender)": "{\"description\": \"min=0.42, mean=0.443, max=0.467, sum=2.66 (6)\", \"tab\": \"Bias\", \"score\": \"0.44339662209590786\"}", - "XSUM - Representation (race)": "{\"description\": \"min=0.436, mean=0.521, max=0.667, sum=3.124 (6)\", \"tab\": \"Bias\", \"score\": \"0.5206745206745207\"}", - "XSUM - Representation (gender)": "{\"description\": \"min=0.178, mean=0.204, max=0.222, sum=1.222 (6)\", \"tab\": \"Bias\", \"score\": \"0.20364463830300386\"}", - "XSUM - Toxic fraction": "{\"description\": \"min=0, mean=0.001, max=0.002, sum=0.008 (6)\", \"tab\": \"Toxicity\", \"score\": \"0.001287001287001287\"}", - "XSUM - SummaC": "{\"description\": \"min=-0.078, mean=-0.057, max=-0.044, sum=-0.17 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"-0.05681849002633572\"}", - "XSUM - QAFactEval": "{\"description\": \"min=4.256, mean=4.33, max=4.381, sum=25.981 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"4.330178153632894\"}", - "XSUM - BERTScore (F1)": "{\"description\": \"min=0.277, mean=0.281, max=0.286, sum=0.844 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.28149043918051486\"}", - "XSUM - Coverage": "{\"description\": \"min=0.873, mean=0.885, max=0.893, sum=5.312 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"0.8853480945766184\"}", - "XSUM - Density": "{\"description\": \"min=7.239, mean=8.487, max=9.133, sum=50.925 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"8.487450287350649\"}", - "XSUM - Compression": "{\"description\": \"min=11.1, mean=11.856, max=12.376, sum=71.136 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"11.856076449493486\"}", - "XSUM - HumanEval-faithfulness": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-relevance": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-coherence": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "IMDB", - "source_data": { - "dataset_name": "IMDB", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on IMDB", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.913, - "details": { - "description": "min=0.902, mean=0.913, max=0.921, sum=2.738 (3)", - "tab": "Accuracy", - "IMDB - ECE (10-bin)": "{\"description\": \"min=0.028, mean=0.038, max=0.05, sum=0.115 (3)\", \"tab\": \"Calibration\", \"score\": \"0.038396495508375095\"}", - "IMDB - EM (Robustness)": "{\"description\": \"min=0.821, mean=0.844, max=0.868, sum=2.532 (3)\", \"tab\": \"Robustness\", \"score\": \"0.844\"}", - "IMDB - EM (Fairness)": "{\"description\": \"min=0.871, mean=0.887, max=0.901, sum=2.66 (3)\", \"tab\": \"Fairness\", \"score\": \"0.8866666666666667\"}", - "IMDB - Denoised inference time (s)": "{\"description\": \"min=0.151, mean=0.157, max=0.162, sum=0.472 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.15740409657118068\"}", - "IMDB - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "IMDB - # train": "{\"description\": \"min=2.916, mean=4.242, max=4.986, sum=12.726 (3)\", \"tab\": \"General information\", \"score\": \"4.242\"}", - "IMDB - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IMDB - # prompt tokens": "{\"description\": \"min=1277.729, mean=1553.363, max=1768.607, sum=4660.089 (3)\", \"tab\": \"General information\", \"score\": \"1553.363\"}", - "IMDB - # output tokens": "{\"description\": \"min=1, mean=1.001, max=1.003, sum=3.003 (3)\", \"tab\": \"General information\", \"score\": \"1.0010000000000001\"}", - "IMDB - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "IMDB - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CivilComments", - "source_data": { - "dataset_name": "CivilComments", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on CivilComments", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.499, - "details": { - "description": "min=0, mean=0.499, max=1, sum=26.951 (54)", - "tab": "Accuracy", - "CivilComments - ECE (10-bin)": "{\"description\": \"min=0.092, mean=0.499, max=0.911, sum=26.966 (54)\", \"tab\": \"Calibration\", \"score\": \"0.49936533676896183\"}", - "CivilComments - EM (Robustness)": "{\"description\": \"min=0, mean=0.499, max=1, sum=26.94 (54)\", \"tab\": \"Robustness\", \"score\": \"0.4988821054609162\"}", - "CivilComments - EM (Fairness)": "{\"description\": \"min=0, mean=0.499, max=1, sum=26.936 (54)\", \"tab\": \"Fairness\", \"score\": \"0.4988205867192775\"}", - "CivilComments - Denoised inference time (s)": "{\"description\": \"min=0.13, mean=0.138, max=0.151, sum=7.438 (54)\", \"tab\": \"Efficiency\", \"score\": \"0.13774715150926628\"}", - "CivilComments - # eval": "{\"description\": \"min=74, mean=371.556, max=683, sum=20064 (54)\", \"tab\": \"General information\", \"score\": \"371.55555555555554\"}", - "CivilComments - # train": "{\"description\": \"min=5, mean=5, max=5, sum=270 (54)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CivilComments - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (54)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CivilComments - # prompt tokens": "{\"description\": \"min=356.537, mean=722.635, max=1267.519, sum=39022.317 (54)\", \"tab\": \"General information\", \"score\": \"722.6354931173206\"}", - "CivilComments - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=54 (54)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "CivilComments - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=162 (54)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "CivilComments - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (54)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "RAFT", - "source_data": { - "dataset_name": "RAFT", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on RAFT", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.509, - "details": { - "description": "min=0.125, mean=0.509, max=0.925, sum=16.8 (33)", - "tab": "Accuracy", - "RAFT - ECE (10-bin)": "{\"description\": \"min=0.102, mean=0.295, max=0.541, sum=9.737 (33)\", \"tab\": \"Calibration\", \"score\": \"0.2950696376748286\"}", - "RAFT - EM (Robustness)": "{\"description\": \"min=0.025, mean=0.383, max=0.925, sum=12.625 (33)\", \"tab\": \"Robustness\", \"score\": \"0.38257575757575757\"}", - "RAFT - EM (Fairness)": "{\"description\": \"min=0.125, mean=0.475, max=0.925, sum=15.675 (33)\", \"tab\": \"Fairness\", \"score\": \"0.47500000000000003\"}", - "RAFT - Denoised inference time (s)": "{\"description\": \"min=0.13, mean=0.153, max=0.188, sum=5.047 (33)\", \"tab\": \"Efficiency\", \"score\": \"0.15293320707070707\"}", - "RAFT - # eval": "{\"description\": \"min=40, mean=40, max=40, sum=1320 (33)\", \"tab\": \"General information\", \"score\": \"40.0\"}", - "RAFT - # train": "{\"description\": \"min=0, mean=4.556, max=5, sum=150.35 (33)\", \"tab\": \"General information\", \"score\": \"4.556060606060607\"}", - "RAFT - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (33)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "RAFT - # prompt tokens": "{\"description\": \"min=257.35, mean=812.938, max=1773.675, sum=26826.95 (33)\", \"tab\": \"General information\", \"score\": \"812.937878787879\"}", - "RAFT - # output tokens": "{\"description\": \"min=0.85, mean=2.774, max=5.875, sum=91.55 (33)\", \"tab\": \"General information\", \"score\": \"2.7742424242424244\"}", - "RAFT - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=99 (33)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "RAFT - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Toxic fraction": "{\"description\": \"min=0, mean=0.001, max=0.025, sum=0.025 (33)\", \"tab\": \"Toxicity\", \"score\": \"0.0007575757575757576\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/openai_text-curie-001.json b/data/models/openai_text-curie-001.json deleted file mode 100644 index 79f4b67780938f0430791e0e66223d738947c279..0000000000000000000000000000000000000000 --- a/data/models/openai_text-curie-001.json +++ /dev/null @@ -1,674 +0,0 @@ -{ - "model_info": { - "name": "text-curie-001", - "id": "openai/text-curie-001", - "developer": "OpenAI", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_classic/openai_text-curie-001/1774096308.339228", - "retrieved_timestamp": "1774096308.339228", - "source_metadata": { - "source_name": "helm_classic", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_classic", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_classic", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperform on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.36, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Calibration": "{\"description\": \"\", \"tab\": \"Calibration\", \"score\": \"0.33452535946368817\"}", - "Mean win rate - Robustness": "{\"description\": \"\", \"tab\": \"Robustness\", \"score\": \"0.336998226097225\"}", - "Mean win rate - Fairness": "{\"description\": \"\", \"tab\": \"Fairness\", \"score\": \"0.377271245624972\"}", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.7827028508771929\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}", - "Mean win rate - Bias": "{\"description\": \"\", \"tab\": \"Bias\", \"score\": \"0.49509040746991073\"}", - "Mean win rate - Toxicity": "{\"description\": \"\", \"tab\": \"Toxicity\", \"score\": \"0.4050529717196384\"}", - "Mean win rate - Summarization metrics": "{\"description\": \"\", \"tab\": \"Summarization metrics\", \"score\": \"0.6165831244778613\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.237, - "details": { - "description": "min=0.21, mean=0.237, max=0.298, sum=3.558 (15)", - "tab": "Accuracy", - "MMLU - ECE (10-bin)": "{\"description\": \"min=0.298, mean=0.462, max=0.534, sum=6.937 (15)\", \"tab\": \"Calibration\", \"score\": \"0.4624557415628211\"}", - "MMLU - EM (Robustness)": "{\"description\": \"min=0.16, mean=0.22, max=0.272, sum=3.303 (15)\", \"tab\": \"Robustness\", \"score\": \"0.22019883040935673\"}", - "MMLU - EM (Fairness)": "{\"description\": \"min=0.2, mean=0.231, max=0.281, sum=3.462 (15)\", \"tab\": \"Fairness\", \"score\": \"0.23079532163742691\"}", - "MMLU - Denoised inference time (s)": "{\"description\": \"min=0.129, mean=0.133, max=0.14, sum=1.998 (15)\", \"tab\": \"Efficiency\", \"score\": \"0.13321992694627194\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=1542 (15)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=75 (15)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (15)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=371.38, mean=472.274, max=624.07, sum=7084.111 (15)\", \"tab\": \"General information\", \"score\": \"472.2740350877193\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=15 (15)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "MMLU - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=45 (15)\", \"tab\": \"General information\", \"score\": \"3.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "BoolQ", - "source_data": { - "dataset_name": "BoolQ", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on BoolQ", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.62, - "details": { - "description": "min=0.591, mean=0.62, max=0.638, sum=1.861 (3)\n⚠ Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.", - "tab": "Accuracy", - "BoolQ - ECE (10-bin)": "{\"description\": \"min=0.239, mean=0.253, max=0.279, sum=0.758 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Calibration\", \"score\": \"0.252648729019218\"}", - "BoolQ - EM (Robustness)": "{\"description\": \"min=0.519, mean=0.549, max=0.566, sum=1.648 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Robustness\", \"score\": \"0.5493333333333332\"}", - "BoolQ - EM (Fairness)": "{\"description\": \"min=0.543, mean=0.576, max=0.592, sum=1.727 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Fairness\", \"score\": \"0.5756666666666667\"}", - "BoolQ - Denoised inference time (s)": "{\"description\": \"min=0.141, mean=0.143, max=0.146, sum=0.429 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Efficiency\", \"score\": \"0.14293199392361097\"}", - "BoolQ - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "BoolQ - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "BoolQ - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "BoolQ - # prompt tokens": "{\"description\": \"min=660.073, mean=908.406, max=1242.073, sum=2725.219 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"908.4063333333334\"}", - "BoolQ - # output tokens": "{\"description\": \"min=1.004, mean=1.007, max=1.012, sum=3.021 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"1.007\"}", - "BoolQ - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "BoolQ - Stereotypes (race)": "{\"description\": \"(0)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Stereotypes (gender)": "{\"description\": \"(0)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (race)": "{\"description\": \"(0)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (gender)": "{\"description\": \"(0)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.582, - "details": { - "description": "min=0.55, mean=0.582, max=0.63, sum=1.746 (3)", - "tab": "Accuracy", - "NarrativeQA - ECE (10-bin)": "{\"description\": \"min=0.198, mean=0.221, max=0.233, sum=0.664 (3)\", \"tab\": \"Calibration\", \"score\": \"0.22125645338584943\"}", - "NarrativeQA - F1 (Robustness)": "{\"description\": \"min=0.299, mean=0.34, max=0.38, sum=1.02 (3)\", \"tab\": \"Robustness\", \"score\": \"0.33989457936851464\"}", - "NarrativeQA - F1 (Fairness)": "{\"description\": \"min=0.428, mean=0.463, max=0.5, sum=1.389 (3)\", \"tab\": \"Fairness\", \"score\": \"0.4630759323159577\"}", - "NarrativeQA - Denoised inference time (s)": "{\"description\": \"min=0.19, mean=0.205, max=0.217, sum=0.615 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.20493085387323948\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=1065 (3)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=1.051, mean=1.647, max=2.085, sum=4.941 (3)\", \"tab\": \"General information\", \"score\": \"1.6469483568075116\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=1601.955, mean=1652.377, max=1705.003, sum=4957.132 (3)\", \"tab\": \"General information\", \"score\": \"1652.3774647887324\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=6.645, mean=8.971, max=10.738, sum=26.913 (3)\", \"tab\": \"General information\", \"score\": \"8.970892018779344\"}", - "NarrativeQA - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NarrativeQA - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=1.333 (2)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666667\"}", - "NarrativeQA - Stereotypes (gender)": "{\"description\": \"min=0.436, mean=0.446, max=0.453, sum=1.339 (3)\", \"tab\": \"Bias\", \"score\": \"0.44628176056747487\"}", - "NarrativeQA - Representation (race)": "{\"description\": \"min=0.494, mean=0.609, max=0.667, sum=1.828 (3)\", \"tab\": \"Bias\", \"score\": \"0.6091954022988506\"}", - "NarrativeQA - Representation (gender)": "{\"description\": \"min=0.161, mean=0.19, max=0.207, sum=0.569 (3)\", \"tab\": \"Bias\", \"score\": \"0.1896444305777106\"}", - "NarrativeQA - Toxic fraction": "{\"description\": \"min=0.014, mean=0.015, max=0.017, sum=0.045 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.015023474178403754\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (open-book)", - "source_data": { - "dataset_name": "NaturalQuestions (open-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (open-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.571, - "details": { - "description": "min=0.536, mean=0.571, max=0.599, sum=1.714 (3)\n⚠ Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.", - "tab": "Accuracy", - "NaturalQuestions (closed-book) - ECE (10-bin)": "{\"description\": \"min=0.233, mean=0.253, max=0.264, sum=0.758 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Calibration\", \"score\": \"0.25269080261254767\"}", - "NaturalQuestions (open-book) - ECE (10-bin)": "{\"description\": \"min=0.215, mean=0.216, max=0.217, sum=0.648 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Calibration\", \"score\": \"0.21613185314031233\"}", - "NaturalQuestions (closed-book) - F1 (Robustness)": "{\"description\": \"min=0.116, mean=0.121, max=0.124, sum=0.363 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Robustness\", \"score\": \"0.12098406641539787\"}", - "NaturalQuestions (open-book) - F1 (Robustness)": "{\"description\": \"min=0.365, mean=0.415, max=0.445, sum=1.246 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Robustness\", \"score\": \"0.4152585116053236\"}", - "NaturalQuestions (closed-book) - F1 (Fairness)": "{\"description\": \"min=0.124, mean=0.132, max=0.139, sum=0.396 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Fairness\", \"score\": \"0.13187631785928275\"}", - "NaturalQuestions (open-book) - F1 (Fairness)": "{\"description\": \"min=0.464, mean=0.5, max=0.519, sum=1.499 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Fairness\", \"score\": \"0.4995085831746681\"}", - "NaturalQuestions (closed-book) - Denoised inference time (s)": "{\"description\": \"min=0.152, mean=0.153, max=0.154, sum=0.459 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Efficiency\", \"score\": \"0.15303552604166656\"}", - "NaturalQuestions (open-book) - Denoised inference time (s)": "{\"description\": \"min=0.176, mean=0.185, max=0.193, sum=0.554 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Efficiency\", \"score\": \"0.1847613116319444\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=110.254, mean=112.254, max=116.254, sum=336.762 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"112.254\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=4.507, mean=4.641, max=4.737, sum=13.923 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"4.641\"}", - "NaturalQuestions (closed-book) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.647, mean=4.691, max=4.724, sum=14.074 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"4.691333333333334\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.036, mean=0.036, max=0.036, sum=0.108 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.036\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1231.212, mean=1419.574, max=1523.257, sum=4258.721 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"1419.5736666666664\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=5.931, mean=6.634, max=7.52, sum=19.901 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"6.633666666666667\"}", - "NaturalQuestions (open-book) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NaturalQuestions (closed-book) - Stereotypes (race)": "{\"description\": \"(0)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - Stereotypes (gender)": "{\"description\": \"min=0.5, mean=0.5, max=0.5, sum=1.5 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.5\"}", - "NaturalQuestions (closed-book) - Representation (race)": "{\"description\": \"min=0.542, mean=0.566, max=0.6, sum=1.697 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.5657407407407408\"}", - "NaturalQuestions (closed-book) - Representation (gender)": "{\"description\": \"min=0.119, mean=0.238, max=0.346, sum=0.715 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.23840048840048841\"}", - "NaturalQuestions (open-book) - Stereotypes (race)": "{\"description\": \"(0)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"\"}", - "NaturalQuestions (open-book) - Stereotypes (gender)": "{\"description\": \"min=0.3, mean=0.433, max=0.5, sum=1.3 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.43333333333333335\"}", - "NaturalQuestions (open-book) - Representation (race)": "{\"description\": \"min=0.405, mean=0.441, max=0.467, sum=1.323 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.44097026888062185\"}", - "NaturalQuestions (open-book) - Representation (gender)": "{\"description\": \"min=0.144, mean=0.158, max=0.179, sum=0.473 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.15754640839386602\"}", - "NaturalQuestions (closed-book) - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}", - "NaturalQuestions (open-book) - Toxic fraction": "{\"description\": \"min=0, mean=0.001, max=0.002, sum=0.003 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Toxicity\", \"score\": \"0.001\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "QuAC", - "source_data": { - "dataset_name": "QuAC", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on QuAC", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.358, - "details": { - "description": "min=0.341, mean=0.358, max=0.383, sum=1.074 (3)\n⚠ Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.", - "tab": "Accuracy", - "QuAC - ECE (10-bin)": "{\"description\": \"min=0.237, mean=0.254, max=0.272, sum=0.763 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Calibration\", \"score\": \"0.25427485237899866\"}", - "QuAC - F1 (Robustness)": "{\"description\": \"min=0.166, mean=0.169, max=0.173, sum=0.506 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Robustness\", \"score\": \"0.16872479684813432\"}", - "QuAC - F1 (Fairness)": "{\"description\": \"min=0.244, mean=0.255, max=0.264, sum=0.765 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Fairness\", \"score\": \"0.2548639356870548\"}", - "QuAC - Denoised inference time (s)": "{\"description\": \"min=0.287, mean=0.298, max=0.313, sum=0.894 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Efficiency\", \"score\": \"0.29803956770833356\"}", - "QuAC - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "QuAC - # train": "{\"description\": \"min=0.845, mean=0.944, max=1.086, sum=2.833 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.9443333333333334\"}", - "QuAC - truncated": "{\"description\": \"min=0.016, mean=0.016, max=0.016, sum=0.048 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.016\"}", - "QuAC - # prompt tokens": "{\"description\": \"min=1625.523, mean=1644.831, max=1670.605, sum=4934.492 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"1644.8306666666667\"}", - "QuAC - # output tokens": "{\"description\": \"min=20.676, mean=22.198, max=24.409, sum=66.593 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"22.197666666666663\"}", - "QuAC - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "QuAC - Stereotypes (race)": "{\"description\": \"min=0.593, mean=0.631, max=0.667, sum=1.893 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.6308641975308643\"}", - "QuAC - Stereotypes (gender)": "{\"description\": \"min=0.438, mean=0.456, max=0.473, sum=1.367 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.4556780038650607\"}", - "QuAC - Representation (race)": "{\"description\": \"min=0.244, mean=0.274, max=0.294, sum=0.822 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.27410775768984724\"}", - "QuAC - Representation (gender)": "{\"description\": \"min=0.231, mean=0.242, max=0.26, sum=0.726 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.24189395211611728\"}", - "QuAC - Toxic fraction": "{\"description\": \"min=0.001, mean=0.001, max=0.002, sum=0.004 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Toxicity\", \"score\": \"0.0013333333333333333\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "HellaSwag", - "source_data": { - "dataset_name": "HellaSwag", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on HellaSwag", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.676, - "details": { - "description": "min=0.676, mean=0.676, max=0.676, sum=0.676 (1)\n⚠ Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.", - "tab": "Accuracy", - "HellaSwag - ECE (10-bin)": "{\"description\": \"min=0.153, mean=0.153, max=0.153, sum=0.153 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Calibration\", \"score\": \"0.15281579026404526\"}", - "HellaSwag - EM (Robustness)": "{\"description\": \"min=0.625, mean=0.625, max=0.625, sum=0.625 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Robustness\", \"score\": \"0.625\"}", - "HellaSwag - EM (Fairness)": "{\"description\": \"min=0.534, mean=0.534, max=0.534, sum=0.534 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Fairness\", \"score\": \"0.534\"}", - "HellaSwag - Denoised inference time (s)": "{\"description\": \"min=0.125, mean=0.125, max=0.125, sum=0.125 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Efficiency\", \"score\": \"0.12517962499999974\"}", - "HellaSwag - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "HellaSwag - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "HellaSwag - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "HellaSwag - # prompt tokens": "{\"description\": \"min=87.888, mean=87.888, max=87.888, sum=87.888 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"87.888\"}", - "HellaSwag - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "HellaSwag - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.514, - "details": { - "description": "min=0.514, mean=0.514, max=0.514, sum=0.514 (1)\n⚠ Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.", - "tab": "Accuracy", - "OpenbookQA - ECE (10-bin)": "{\"description\": \"min=0.321, mean=0.321, max=0.321, sum=0.321 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Calibration\", \"score\": \"0.3206023655720099\"}", - "OpenbookQA - EM (Robustness)": "{\"description\": \"min=0.424, mean=0.424, max=0.424, sum=0.424 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Robustness\", \"score\": \"0.424\"}", - "OpenbookQA - EM (Fairness)": "{\"description\": \"min=0.452, mean=0.452, max=0.452, sum=0.452 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Fairness\", \"score\": \"0.452\"}", - "OpenbookQA - Denoised inference time (s)": "{\"description\": \"min=0.119, mean=0.119, max=0.119, sum=0.119 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Efficiency\", \"score\": \"0.1193705468750003\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=5.27, mean=5.27, max=5.27, sum=5.27 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"5.27\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "TruthfulQA", - "source_data": { - "dataset_name": "TruthfulQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on TruthfulQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.257, - "details": { - "description": "min=0.231, mean=0.257, max=0.301, sum=0.772 (3)", - "tab": "Accuracy", - "TruthfulQA - ECE (10-bin)": "{\"description\": \"min=0.321, mean=0.355, max=0.375, sum=1.066 (3)\", \"tab\": \"Calibration\", \"score\": \"0.35539796883884156\"}", - "TruthfulQA - EM (Robustness)": "{\"description\": \"min=0.206, mean=0.235, max=0.284, sum=0.705 (3)\", \"tab\": \"Robustness\", \"score\": \"0.23496432212028542\"}", - "TruthfulQA - EM (Fairness)": "{\"description\": \"min=0.209, mean=0.239, max=0.286, sum=0.717 (3)\", \"tab\": \"Fairness\", \"score\": \"0.23904179408766565\"}", - "TruthfulQA - Denoised inference time (s)": "{\"description\": \"min=0.134, mean=0.134, max=0.136, sum=0.403 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.1343441023987004\"}", - "TruthfulQA - # eval": "{\"description\": \"min=654, mean=654, max=654, sum=1962 (3)\", \"tab\": \"General information\", \"score\": \"654.0\"}", - "TruthfulQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "TruthfulQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "TruthfulQA - # prompt tokens": "{\"description\": \"min=501.121, mean=511.121, max=529.121, sum=1533.362 (3)\", \"tab\": \"General information\", \"score\": \"511.12079510703364\"}", - "TruthfulQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=3 (3)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "TruthfulQA - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MS MARCO (TREC)", - "source_data": { - "dataset_name": "MS MARCO (TREC)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "NDCG@10 on MS MARCO (TREC)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.507, - "details": { - "description": "min=0.476, mean=0.507, max=0.545, sum=1.522 (3)", - "tab": "Accuracy", - "MS MARCO (regular) - RR@10 (Robustness)": "{\"description\": \"min=0.171, mean=0.198, max=0.222, sum=0.594 (3)\", \"tab\": \"Robustness\", \"score\": \"0.1980144179894178\"}", - "MS MARCO (TREC) - NDCG@10 (Robustness)": "{\"description\": \"min=0.393, mean=0.444, max=0.486, sum=1.331 (3)\", \"tab\": \"Robustness\", \"score\": \"0.4437543283018195\"}", - "MS MARCO (regular) - RR@10 (Fairness)": "{\"description\": \"min=0.231, mean=0.244, max=0.26, sum=0.732 (3)\", \"tab\": \"Fairness\", \"score\": \"0.2441616402116399\"}", - "MS MARCO (TREC) - NDCG@10 (Fairness)": "{\"description\": \"min=0.448, mean=0.482, max=0.523, sum=1.445 (3)\", \"tab\": \"Fairness\", \"score\": \"0.4817143719085842\"}", - "MS MARCO (regular) - Denoised inference time (s)": "{\"description\": \"min=0.134, mean=0.136, max=0.138, sum=0.408 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.13591170442708336\"}", - "MS MARCO (TREC) - Denoised inference time (s)": "{\"description\": \"min=0.133, mean=0.135, max=0.138, sum=0.406 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.13529218144379848\"}", - "MS MARCO (regular) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "MS MARCO (regular) - # train": "{\"description\": \"min=2, mean=2, max=2, sum=6 (3)\", \"tab\": \"General information\", \"score\": \"2.0\"}", - "MS MARCO (regular) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MS MARCO (regular) - # prompt tokens": "{\"description\": \"min=495.232, mean=532.565, max=577.232, sum=1597.696 (3)\", \"tab\": \"General information\", \"score\": \"532.5653333333333\"}", - "MS MARCO (regular) - # output tokens": "{\"description\": \"min=1.005, mean=1.031, max=1.08, sum=3.092 (3)\", \"tab\": \"General information\", \"score\": \"1.0306666666666666\"}", - "MS MARCO (regular) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "MS MARCO (TREC) - # eval": "{\"description\": \"min=43, mean=43, max=43, sum=129 (3)\", \"tab\": \"General information\", \"score\": \"43.0\"}", - "MS MARCO (TREC) - # train": "{\"description\": \"min=2, mean=2, max=2, sum=6 (3)\", \"tab\": \"General information\", \"score\": \"2.0\"}", - "MS MARCO (TREC) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MS MARCO (TREC) - # prompt tokens": "{\"description\": \"min=478.488, mean=515.822, max=560.488, sum=1547.465 (3)\", \"tab\": \"General information\", \"score\": \"515.8217054263565\"}", - "MS MARCO (TREC) - # output tokens": "{\"description\": \"min=1, mean=1.078, max=1.209, sum=3.233 (3)\", \"tab\": \"General information\", \"score\": \"1.0775193798449612\"}", - "MS MARCO (TREC) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "MS MARCO (regular) - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "MS MARCO (TREC) - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CNN/DailyMail", - "source_data": { - "dataset_name": "CNN/DailyMail", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on CNN/DailyMail", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.152, - "details": { - "description": "min=0.144, mean=0.152, max=0.159, sum=1.061 (7)", - "tab": "Accuracy", - "CNN/DailyMail - Denoised inference time (s)": "{\"description\": \"min=0.748, mean=0.799, max=0.848, sum=5.594 (7)\", \"tab\": \"Efficiency\", \"score\": \"0.7991309579692929\"}", - "CNN/DailyMail - # eval": "{\"description\": \"min=466, mean=466, max=466, sum=3262 (7)\", \"tab\": \"General information\", \"score\": \"466.0\"}", - "CNN/DailyMail - # train": "{\"description\": \"min=0, mean=4.286, max=5, sum=30 (7)\", \"tab\": \"General information\", \"score\": \"4.285714285714286\"}", - "CNN/DailyMail - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CNN/DailyMail - # prompt tokens": "{\"description\": \"min=583.586, mean=1411.872, max=1567.586, sum=9883.101 (7)\", \"tab\": \"General information\", \"score\": \"1411.8715511955854\"}", - "CNN/DailyMail - # output tokens": "{\"description\": \"min=86.798, mean=94.314, max=101.208, sum=660.2 (7)\", \"tab\": \"General information\", \"score\": \"94.31422440220724\"}", - "CNN/DailyMail - # trials": "{\"description\": \"min=1, mean=2.714, max=3, sum=19 (7)\", \"tab\": \"General information\", \"score\": \"2.7142857142857144\"}", - "CNN/DailyMail - Stereotypes (race)": "{\"description\": \"min=0.566, mean=0.61, max=0.637, sum=4.269 (7)\", \"tab\": \"Bias\", \"score\": \"0.609875949224765\"}", - "CNN/DailyMail - Stereotypes (gender)": "{\"description\": \"min=0.366, mean=0.387, max=0.406, sum=2.706 (7)\", \"tab\": \"Bias\", \"score\": \"0.38654992671117155\"}", - "CNN/DailyMail - Representation (race)": "{\"description\": \"min=0.282, mean=0.301, max=0.322, sum=2.106 (7)\", \"tab\": \"Bias\", \"score\": \"0.30088570849440416\"}", - "CNN/DailyMail - Representation (gender)": "{\"description\": \"min=0.1, mean=0.118, max=0.133, sum=0.827 (7)\", \"tab\": \"Bias\", \"score\": \"0.11810804679822585\"}", - "CNN/DailyMail - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}", - "CNN/DailyMail - SummaC": "{\"description\": \"min=0.156, mean=0.291, max=0.356, sum=1.165 (4)\", \"tab\": \"Summarization metrics\", \"score\": \"0.2913458656100147\"}", - "CNN/DailyMail - QAFactEval": "{\"description\": \"min=4.214, mean=4.616, max=4.743, sum=32.315 (7)\", \"tab\": \"Summarization metrics\", \"score\": \"4.616429547159027\"}", - "CNN/DailyMail - BERTScore (F1)": "{\"description\": \"min=0.299, mean=0.306, max=0.314, sum=1.222 (4)\", \"tab\": \"Summarization metrics\", \"score\": \"0.3055441003363248\"}", - "CNN/DailyMail - Coverage": "{\"description\": \"min=0.935, mean=0.961, max=0.97, sum=6.725 (7)\", \"tab\": \"Summarization metrics\", \"score\": \"0.9607616041668255\"}", - "CNN/DailyMail - Density": "{\"description\": \"min=17.105, mean=26.1, max=29.982, sum=182.7 (7)\", \"tab\": \"Summarization metrics\", \"score\": \"26.09992906850249\"}", - "CNN/DailyMail - Compression": "{\"description\": \"min=6.155, mean=6.829, max=7.635, sum=47.805 (7)\", \"tab\": \"Summarization metrics\", \"score\": \"6.829258437977153\"}", - "CNN/DailyMail - HumanEval-faithfulness": "{\"description\": \"min=0.967, mean=0.967, max=0.967, sum=0.967 (1)\", \"tab\": \"Summarization metrics\", \"score\": \"0.9666666666666669\"}", - "CNN/DailyMail - HumanEval-relevance": "{\"description\": \"min=4.587, mean=4.587, max=4.587, sum=4.587 (1)\", \"tab\": \"Summarization metrics\", \"score\": \"4.586666666666667\"}", - "CNN/DailyMail - HumanEval-coherence": "{\"description\": \"min=4.243, mean=4.243, max=4.243, sum=4.243 (1)\", \"tab\": \"Summarization metrics\", \"score\": \"4.243333333333334\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "XSUM", - "source_data": { - "dataset_name": "XSUM", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on XSUM", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.076, - "details": { - "description": "min=0.056, mean=0.076, max=0.081, sum=0.533 (7)", - "tab": "Accuracy", - "XSUM - Denoised inference time (s)": "{\"description\": \"min=0.349, mean=0.364, max=0.408, sum=2.548 (7)\", \"tab\": \"Efficiency\", \"score\": \"0.36398217373942815\"}", - "XSUM - # eval": "{\"description\": \"min=518, mean=518, max=518, sum=3626 (7)\", \"tab\": \"General information\", \"score\": \"518.0\"}", - "XSUM - # train": "{\"description\": \"min=0, mean=4.285, max=5, sum=29.992 (7)\", \"tab\": \"General information\", \"score\": \"4.284611141753999\"}", - "XSUM - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "XSUM - # prompt tokens": "{\"description\": \"min=388.402, mean=1350.13, max=1538.921, sum=9450.911 (7)\", \"tab\": \"General information\", \"score\": \"1350.1301709873137\"}", - "XSUM - # output tokens": "{\"description\": \"min=29.917, mean=32.345, max=40.357, sum=226.415 (7)\", \"tab\": \"General information\", \"score\": \"32.3450082735797\"}", - "XSUM - # trials": "{\"description\": \"min=1, mean=2.714, max=3, sum=19 (7)\", \"tab\": \"General information\", \"score\": \"2.7142857142857144\"}", - "XSUM - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=4.667 (7)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666669\"}", - "XSUM - Stereotypes (gender)": "{\"description\": \"min=0.419, mean=0.442, max=0.466, sum=3.093 (7)\", \"tab\": \"Bias\", \"score\": \"0.4418823146165695\"}", - "XSUM - Representation (race)": "{\"description\": \"min=0.473, mean=0.54, max=0.584, sum=3.777 (7)\", \"tab\": \"Bias\", \"score\": \"0.5395129666982432\"}", - "XSUM - Representation (gender)": "{\"description\": \"min=0.172, mean=0.194, max=0.228, sum=1.356 (7)\", \"tab\": \"Bias\", \"score\": \"0.1937219794503278\"}", - "XSUM - Toxic fraction": "{\"description\": \"min=0, mean=0.002, max=0.004, sum=0.012 (7)\", \"tab\": \"Toxicity\", \"score\": \"0.0016547159404302263\"}", - "XSUM - SummaC": "{\"description\": \"min=-0.241, mean=-0.185, max=-0.057, sum=-0.741 (4)\", \"tab\": \"Summarization metrics\", \"score\": \"-0.18531544589014434\"}", - "XSUM - QAFactEval": "{\"description\": \"min=3.199, mean=3.459, max=3.799, sum=24.213 (7)\", \"tab\": \"Summarization metrics\", \"score\": \"3.458996653634986\"}", - "XSUM - BERTScore (F1)": "{\"description\": \"min=0.308, mean=0.354, max=0.372, sum=1.415 (4)\", \"tab\": \"Summarization metrics\", \"score\": \"0.3536865086232682\"}", - "XSUM - Coverage": "{\"description\": \"min=0.823, mean=0.839, max=0.903, sum=5.872 (7)\", \"tab\": \"Summarization metrics\", \"score\": \"0.838839539634714\"}", - "XSUM - Density": "{\"description\": \"min=3.005, mean=4.008, max=8.274, sum=28.059 (7)\", \"tab\": \"Summarization metrics\", \"score\": \"4.008473483028278\"}", - "XSUM - Compression": "{\"description\": \"min=11.556, mean=12.98, max=13.601, sum=90.86 (7)\", \"tab\": \"Summarization metrics\", \"score\": \"12.979988031884476\"}", - "XSUM - HumanEval-faithfulness": "{\"description\": \"min=0.957, mean=0.991, max=1, sum=4.957 (5)\", \"tab\": \"Summarization metrics\", \"score\": \"0.9913333333333334\"}", - "XSUM - HumanEval-relevance": "{\"description\": \"min=4, mean=4.068, max=4.34, sum=20.34 (5)\", \"tab\": \"Summarization metrics\", \"score\": \"4.068\"}", - "XSUM - HumanEval-coherence": "{\"description\": \"min=4.273, mean=4.321, max=4.333, sum=21.607 (5)\", \"tab\": \"Summarization metrics\", \"score\": \"4.3213333333333335\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "IMDB", - "source_data": { - "dataset_name": "IMDB", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on IMDB", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.923, - "details": { - "description": "min=0.915, mean=0.923, max=0.927, sum=2.768 (3)", - "tab": "Accuracy", - "IMDB - ECE (10-bin)": "{\"description\": \"min=0.027, mean=0.031, max=0.034, sum=0.093 (3)\", \"tab\": \"Calibration\", \"score\": \"0.03108408690404522\"}", - "IMDB - EM (Robustness)": "{\"description\": \"min=0.876, mean=0.881, max=0.887, sum=2.642 (3)\", \"tab\": \"Robustness\", \"score\": \"0.8806666666666666\"}", - "IMDB - EM (Fairness)": "{\"description\": \"min=0.903, mean=0.91, max=0.916, sum=2.731 (3)\", \"tab\": \"Fairness\", \"score\": \"0.9103333333333333\"}", - "IMDB - Denoised inference time (s)": "{\"description\": \"min=0.142, mean=0.147, max=0.151, sum=0.442 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.1473289437934027\"}", - "IMDB - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "IMDB - # train": "{\"description\": \"min=2.916, mean=4.242, max=4.986, sum=12.726 (3)\", \"tab\": \"General information\", \"score\": \"4.242\"}", - "IMDB - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IMDB - # prompt tokens": "{\"description\": \"min=1277.729, mean=1553.363, max=1768.607, sum=4660.089 (3)\", \"tab\": \"General information\", \"score\": \"1553.363\"}", - "IMDB - # output tokens": "{\"description\": \"min=0.998, mean=0.999, max=1, sum=2.996 (3)\", \"tab\": \"General information\", \"score\": \"0.9986666666666667\"}", - "IMDB - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "IMDB - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CivilComments", - "source_data": { - "dataset_name": "CivilComments", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on CivilComments", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.537, - "details": { - "description": "min=0.04, mean=0.537, max=0.93, sum=29.013 (54)", - "tab": "Accuracy", - "CivilComments - ECE (10-bin)": "{\"description\": \"min=0.049, mean=0.262, max=0.674, sum=14.15 (54)\", \"tab\": \"Calibration\", \"score\": \"0.26204430696260744\"}", - "CivilComments - EM (Robustness)": "{\"description\": \"min=0, mean=0.129, max=0.39, sum=6.954 (54)\", \"tab\": \"Robustness\", \"score\": \"0.12877898867890694\"}", - "CivilComments - EM (Fairness)": "{\"description\": \"min=0.02, mean=0.471, max=0.874, sum=25.434 (54)\", \"tab\": \"Fairness\", \"score\": \"0.4710066762167616\"}", - "CivilComments - Denoised inference time (s)": "{\"description\": \"min=0.129, mean=0.142, max=0.149, sum=7.645 (54)\", \"tab\": \"Efficiency\", \"score\": \"0.1415740791295965\"}", - "CivilComments - # eval": "{\"description\": \"min=74, mean=371.556, max=683, sum=20064 (54)\", \"tab\": \"General information\", \"score\": \"371.55555555555554\"}", - "CivilComments - # train": "{\"description\": \"min=5, mean=5, max=5, sum=270 (54)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CivilComments - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (54)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CivilComments - # prompt tokens": "{\"description\": \"min=356.537, mean=722.635, max=1267.519, sum=39022.317 (54)\", \"tab\": \"General information\", \"score\": \"722.6354931173206\"}", - "CivilComments - # output tokens": "{\"description\": \"min=0.905, mean=0.979, max=1, sum=52.876 (54)\", \"tab\": \"General information\", \"score\": \"0.9791789992573504\"}", - "CivilComments - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=162 (54)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "CivilComments - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (54)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "RAFT", - "source_data": { - "dataset_name": "RAFT", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on RAFT", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.489, - "details": { - "description": "min=0, mean=0.489, max=0.85, sum=16.15 (33)", - "tab": "Accuracy", - "RAFT - ECE (10-bin)": "{\"description\": \"min=0.079, mean=0.409, max=1, sum=13.49 (33)\", \"tab\": \"Calibration\", \"score\": \"0.40879785924457385\"}", - "RAFT - EM (Robustness)": "{\"description\": \"min=0, mean=0.399, max=0.775, sum=13.175 (33)\", \"tab\": \"Robustness\", \"score\": \"0.3992424242424243\"}", - "RAFT - EM (Fairness)": "{\"description\": \"min=0, mean=0.458, max=0.85, sum=15.125 (33)\", \"tab\": \"Fairness\", \"score\": \"0.45833333333333337\"}", - "RAFT - Denoised inference time (s)": "{\"description\": \"min=0.13, mean=0.152, max=0.183, sum=5.003 (33)\", \"tab\": \"Efficiency\", \"score\": \"0.1516085454150884\"}", - "RAFT - # eval": "{\"description\": \"min=40, mean=40, max=40, sum=1320 (33)\", \"tab\": \"General information\", \"score\": \"40.0\"}", - "RAFT - # train": "{\"description\": \"min=0, mean=4.556, max=5, sum=150.35 (33)\", \"tab\": \"General information\", \"score\": \"4.556060606060607\"}", - "RAFT - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (33)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "RAFT - # prompt tokens": "{\"description\": \"min=257.35, mean=812.938, max=1773.675, sum=26826.95 (33)\", \"tab\": \"General information\", \"score\": \"812.937878787879\"}", - "RAFT - # output tokens": "{\"description\": \"min=0, mean=2.751, max=5.95, sum=90.775 (33)\", \"tab\": \"General information\", \"score\": \"2.750757575757576\"}", - "RAFT - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=99 (33)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "RAFT - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (33)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/openai_text-davinci-002.json b/data/models/openai_text-davinci-002.json deleted file mode 100644 index c525aafd54f2609747b5044a02a49c65c8a2b8b5..0000000000000000000000000000000000000000 --- a/data/models/openai_text-davinci-002.json +++ /dev/null @@ -1,1044 +0,0 @@ -{ - "model_info": { - "name": "GPT-3.5 text-davinci-002", - "id": "openai/text-davinci-002", - "developer": "OpenAI", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_classic/openai_text-davinci-002/1774096308.339228", - "retrieved_timestamp": "1774096308.339228", - "source_metadata": { - "source_name": "helm_classic", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_classic", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_classic", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperform on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.905, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Calibration": "{\"description\": \"\", \"tab\": \"Calibration\", \"score\": \"0.4743236143945364\"}", - "Mean win rate - Robustness": "{\"description\": \"\", \"tab\": \"Robustness\", \"score\": \"0.9158568720860156\"}", - "Mean win rate - Fairness": "{\"description\": \"\", \"tab\": \"Fairness\", \"score\": \"0.8637256699548135\"}", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.6036239035087719\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}", - "Mean win rate - Bias": "{\"description\": \"\", \"tab\": \"Bias\", \"score\": \"0.502171676177358\"}", - "Mean win rate - Toxicity": "{\"description\": \"\", \"tab\": \"Toxicity\", \"score\": \"0.4088448588448588\"}", - "Mean win rate - Summarization metrics": "{\"description\": \"\", \"tab\": \"Summarization metrics\", \"score\": \"0.6410087719298245\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.568, - "details": { - "description": "min=0.26, mean=0.568, max=0.86, sum=8.515 (15)", - "tab": "Accuracy", - "MMLU - ECE (10-bin)": "{\"description\": \"min=0.064, mean=0.176, max=0.264, sum=2.644 (15)\", \"tab\": \"Calibration\", \"score\": \"0.17629729974248792\"}", - "MMLU - EM (Robustness)": "{\"description\": \"min=0.23, mean=0.525, max=0.83, sum=7.868 (15)\", \"tab\": \"Robustness\", \"score\": \"0.5245380116959065\"}", - "MMLU - EM (Fairness)": "{\"description\": \"min=0.24, mean=0.531, max=0.82, sum=7.964 (15)\", \"tab\": \"Fairness\", \"score\": \"0.5309473684210526\"}", - "MMLU - Denoised inference time (s)": "{\"description\": \"min=0.175, mean=0.196, max=0.215, sum=2.946 (15)\", \"tab\": \"Efficiency\", \"score\": \"0.19643028419682018\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=1542 (15)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=75 (15)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (15)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=371.38, mean=472.274, max=624.07, sum=7084.111 (15)\", \"tab\": \"General information\", \"score\": \"472.2740350877193\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=15 (15)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "MMLU - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=45 (15)\", \"tab\": \"General information\", \"score\": \"3.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "BoolQ", - "source_data": { - "dataset_name": "BoolQ", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on BoolQ", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.877, - "details": { - "description": "min=0.872, mean=0.877, max=0.883, sum=2.631 (3)\n⚠ Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.", - "tab": "Accuracy", - "BoolQ - ECE (10-bin)": "{\"description\": \"min=0.057, mean=0.064, max=0.068, sum=0.192 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Calibration\", \"score\": \"0.06391934132499137\"}", - "BoolQ - EM (Robustness)": "{\"description\": \"min=0.834, mean=0.841, max=0.854, sum=2.523 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Robustness\", \"score\": \"0.8410000000000001\"}", - "BoolQ - EM (Fairness)": "{\"description\": \"min=0.829, mean=0.837, max=0.844, sum=2.51 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Fairness\", \"score\": \"0.8366666666666666\"}", - "BoolQ - Denoised inference time (s)": "{\"description\": \"min=0.176, mean=0.191, max=0.216, sum=0.574 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Efficiency\", \"score\": \"0.1911954346788195\"}", - "BoolQ - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "BoolQ - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "BoolQ - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "BoolQ - # prompt tokens": "{\"description\": \"min=660.073, mean=908.406, max=1242.073, sum=2725.219 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"908.4063333333334\"}", - "BoolQ - # output tokens": "{\"description\": \"min=1.009, mean=1.013, max=1.018, sum=3.039 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"1.013\"}", - "BoolQ - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "BoolQ - Stereotypes (race)": "{\"description\": \"(0)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Stereotypes (gender)": "{\"description\": \"(0)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (race)": "{\"description\": \"(0)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (gender)": "{\"description\": \"(0)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.727, - "details": { - "description": "min=0.711, mean=0.727, max=0.752, sum=2.182 (3)", - "tab": "Accuracy", - "NarrativeQA - ECE (10-bin)": "{\"description\": \"min=0.214, mean=0.239, max=0.268, sum=0.718 (3)\", \"tab\": \"Calibration\", \"score\": \"0.2393596998509794\"}", - "NarrativeQA - F1 (Robustness)": "{\"description\": \"min=0.61, mean=0.638, max=0.663, sum=1.915 (3)\", \"tab\": \"Robustness\", \"score\": \"0.6382180079306305\"}", - "NarrativeQA - F1 (Fairness)": "{\"description\": \"min=0.637, mean=0.646, max=0.664, sum=1.938 (3)\", \"tab\": \"Fairness\", \"score\": \"0.6459531095726224\"}", - "NarrativeQA - Denoised inference time (s)": "{\"description\": \"min=0.48, mean=0.512, max=0.539, sum=1.537 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.5124278205692486\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=1065 (3)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=4.259, mean=4.532, max=4.955, sum=13.597 (3)\", \"tab\": \"General information\", \"score\": \"4.532394366197183\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=3479.563, mean=3579.093, max=3633.659, sum=10737.279 (3)\", \"tab\": \"General information\", \"score\": \"3579.092957746479\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=6.158, mean=7.378, max=8.448, sum=22.135 (3)\", \"tab\": \"General information\", \"score\": \"7.378403755868544\"}", - "NarrativeQA - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NarrativeQA - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NarrativeQA - Stereotypes (gender)": "{\"description\": \"min=0.363, mean=0.395, max=0.417, sum=1.184 (3)\", \"tab\": \"Bias\", \"score\": \"0.39479717813051146\"}", - "NarrativeQA - Representation (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=0.667 (1)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666667\"}", - "NarrativeQA - Representation (gender)": "{\"description\": \"min=0.17, mean=0.189, max=0.21, sum=0.568 (3)\", \"tab\": \"Bias\", \"score\": \"0.18948121770702417\"}", - "NarrativeQA - Toxic fraction": "{\"description\": \"min=0.008, mean=0.013, max=0.017, sum=0.039 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.013145539906103286\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (open-book)", - "source_data": { - "dataset_name": "NaturalQuestions (open-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (open-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.713, - "details": { - "description": "min=0.71, mean=0.713, max=0.716, sum=2.139 (3)\n⚠ Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.", - "tab": "Accuracy", - "NaturalQuestions (closed-book) - ECE (10-bin)": "{\"description\": \"min=0.315, mean=0.341, max=0.356, sum=1.022 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Calibration\", \"score\": \"0.34056739358291327\"}", - "NaturalQuestions (open-book) - ECE (10-bin)": "{\"description\": \"min=0.233, mean=0.242, max=0.247, sum=0.726 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Calibration\", \"score\": \"0.24207582378172995\"}", - "NaturalQuestions (closed-book) - F1 (Robustness)": "{\"description\": \"min=0.279, mean=0.299, max=0.31, sum=0.896 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Robustness\", \"score\": \"0.29853007347043187\"}", - "NaturalQuestions (open-book) - F1 (Robustness)": "{\"description\": \"min=0.66, mean=0.665, max=0.67, sum=1.994 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Robustness\", \"score\": \"0.6645627340843298\"}", - "NaturalQuestions (closed-book) - F1 (Fairness)": "{\"description\": \"min=0.311, mean=0.32, max=0.326, sum=0.96 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Fairness\", \"score\": \"0.3200640288704773\"}", - "NaturalQuestions (open-book) - F1 (Fairness)": "{\"description\": \"min=0.655, mean=0.659, max=0.663, sum=1.976 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Fairness\", \"score\": \"0.658783235208417\"}", - "NaturalQuestions (closed-book) - Denoised inference time (s)": "{\"description\": \"min=0.259, mean=0.264, max=0.268, sum=0.791 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Efficiency\", \"score\": \"0.26376651302083315\"}", - "NaturalQuestions (open-book) - Denoised inference time (s)": "{\"description\": \"min=0.387, mean=0.394, max=0.398, sum=1.182 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Efficiency\", \"score\": \"0.3939576829427085\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=110.254, mean=112.254, max=116.254, sum=336.762 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"112.254\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=3.783, mean=3.954, max=4.116, sum=11.861 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"3.9536666666666664\"}", - "NaturalQuestions (closed-book) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.874, mean=4.883, max=4.891, sum=14.65 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"4.883333333333334\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.02, mean=0.02, max=0.02, sum=0.06 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.02\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1315.257, mean=1520.977, max=1629.945, sum=4562.931 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"1520.977\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=6.586, mean=6.652, max=6.739, sum=19.957 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"6.652333333333334\"}", - "NaturalQuestions (open-book) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NaturalQuestions (closed-book) - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=0.667 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "NaturalQuestions (closed-book) - Stereotypes (gender)": "{\"description\": \"(0)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - Representation (race)": "{\"description\": \"min=0.439, mean=0.448, max=0.467, sum=1.344 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.44795321637426905\"}", - "NaturalQuestions (closed-book) - Representation (gender)": "{\"description\": \"min=0.079, mean=0.129, max=0.167, sum=0.388 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.1294903926482874\"}", - "NaturalQuestions (open-book) - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=2 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "NaturalQuestions (open-book) - Stereotypes (gender)": "{\"description\": \"min=0.4, mean=0.407, max=0.42, sum=1.22 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.40666666666666673\"}", - "NaturalQuestions (open-book) - Representation (race)": "{\"description\": \"min=0.474, mean=0.487, max=0.505, sum=1.46 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.48653132655730696\"}", - "NaturalQuestions (open-book) - Representation (gender)": "{\"description\": \"min=0.375, mean=0.401, max=0.44, sum=1.202 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.40059748427672953\"}", - "NaturalQuestions (closed-book) - Toxic fraction": "{\"description\": \"min=0.001, mean=0.001, max=0.001, sum=0.003 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Toxicity\", \"score\": \"0.001\"}", - "NaturalQuestions (open-book) - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "QuAC", - "source_data": { - "dataset_name": "QuAC", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on QuAC", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.445, - "details": { - "description": "min=0.435, mean=0.445, max=0.451, sum=1.335 (3)\n⚠ Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.", - "tab": "Accuracy", - "QuAC - ECE (10-bin)": "{\"description\": \"min=0.234, mean=0.274, max=0.301, sum=0.821 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Calibration\", \"score\": \"0.27378530130603257\"}", - "QuAC - F1 (Robustness)": "{\"description\": \"min=0.313, mean=0.319, max=0.331, sum=0.958 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Robustness\", \"score\": \"0.3193910892114107\"}", - "QuAC - F1 (Fairness)": "{\"description\": \"min=0.339, mean=0.353, max=0.363, sum=1.06 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Fairness\", \"score\": \"0.3532761321768228\"}", - "QuAC - Denoised inference time (s)": "{\"description\": \"min=0.887, mean=0.891, max=0.894, sum=2.674 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Efficiency\", \"score\": \"0.8912715646701383\"}", - "QuAC - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "QuAC - # train": "{\"description\": \"min=2.978, mean=3.438, max=3.878, sum=10.315 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"3.438333333333333\"}", - "QuAC - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "QuAC - # prompt tokens": "{\"description\": \"min=2819.048, mean=3249.907, max=3487.39, sum=9749.722 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"3249.907333333333\"}", - "QuAC - # output tokens": "{\"description\": \"min=20.711, mean=20.986, max=21.534, sum=62.959 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"20.98633333333333\"}", - "QuAC - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "QuAC - Stereotypes (race)": "{\"description\": \"min=0.567, mean=0.579, max=0.6, sum=1.738 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.5793650793650794\"}", - "QuAC - Stereotypes (gender)": "{\"description\": \"min=0.443, mean=0.453, max=0.461, sum=1.358 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.4526990667248227\"}", - "QuAC - Representation (race)": "{\"description\": \"min=0.256, mean=0.27, max=0.28, sum=0.81 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.2701590708612791\"}", - "QuAC - Representation (gender)": "{\"description\": \"min=0.245, mean=0.255, max=0.265, sum=0.764 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.2545671124587146\"}", - "QuAC - Toxic fraction": "{\"description\": \"min=0.002, mean=0.002, max=0.003, sum=0.007 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Toxicity\", \"score\": \"0.0023333333333333335\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "HellaSwag", - "source_data": { - "dataset_name": "HellaSwag", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on HellaSwag", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.815, - "details": { - "description": "min=0.815, mean=0.815, max=0.815, sum=0.815 (1)\n⚠ Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.", - "tab": "Accuracy", - "HellaSwag - ECE (10-bin)": "{\"description\": \"min=0.286, mean=0.286, max=0.286, sum=0.286 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Calibration\", \"score\": \"0.2864163850455534\"}", - "HellaSwag - EM (Robustness)": "{\"description\": \"min=0.776, mean=0.776, max=0.776, sum=0.776 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Robustness\", \"score\": \"0.776\"}", - "HellaSwag - EM (Fairness)": "{\"description\": \"min=0.703, mean=0.703, max=0.703, sum=0.703 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Fairness\", \"score\": \"0.703\"}", - "HellaSwag - Denoised inference time (s)": "{\"description\": \"min=0.171, mean=0.171, max=0.171, sum=0.171 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Efficiency\", \"score\": \"0.1710758125\"}", - "HellaSwag - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "HellaSwag - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "HellaSwag - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "HellaSwag - # prompt tokens": "{\"description\": \"min=87.888, mean=87.888, max=87.888, sum=87.888 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"87.888\"}", - "HellaSwag - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "HellaSwag - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.594, - "details": { - "description": "min=0.594, mean=0.594, max=0.594, sum=0.594 (1)\n⚠ Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.", - "tab": "Accuracy", - "OpenbookQA - ECE (10-bin)": "{\"description\": \"min=0.238, mean=0.238, max=0.238, sum=0.238 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Calibration\", \"score\": \"0.23789749910476482\"}", - "OpenbookQA - EM (Robustness)": "{\"description\": \"min=0.52, mean=0.52, max=0.52, sum=0.52 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Robustness\", \"score\": \"0.52\"}", - "OpenbookQA - EM (Fairness)": "{\"description\": \"min=0.54, mean=0.54, max=0.54, sum=0.54 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Fairness\", \"score\": \"0.54\"}", - "OpenbookQA - Denoised inference time (s)": "{\"description\": \"min=0.158, mean=0.158, max=0.158, sum=0.158 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Efficiency\", \"score\": \"0.1578440234375\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=5.27, mean=5.27, max=5.27, sum=5.27 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"5.27\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "TruthfulQA", - "source_data": { - "dataset_name": "TruthfulQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on TruthfulQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.61, - "details": { - "description": "min=0.596, mean=0.61, max=0.63, sum=1.829 (3)", - "tab": "Accuracy", - "TruthfulQA - ECE (10-bin)": "{\"description\": \"min=0.167, mean=0.199, max=0.232, sum=0.596 (3)\", \"tab\": \"Calibration\", \"score\": \"0.19868497875362334\"}", - "TruthfulQA - EM (Robustness)": "{\"description\": \"min=0.517, mean=0.547, max=0.573, sum=1.641 (3)\", \"tab\": \"Robustness\", \"score\": \"0.5468909276248726\"}", - "TruthfulQA - EM (Fairness)": "{\"description\": \"min=0.48, mean=0.515, max=0.547, sum=1.546 (3)\", \"tab\": \"Fairness\", \"score\": \"0.5152905198776758\"}", - "TruthfulQA - Denoised inference time (s)": "{\"description\": \"min=0.186, mean=0.2, max=0.208, sum=0.601 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.20048467762487246\"}", - "TruthfulQA - # eval": "{\"description\": \"min=654, mean=654, max=654, sum=1962 (3)\", \"tab\": \"General information\", \"score\": \"654.0\"}", - "TruthfulQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "TruthfulQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "TruthfulQA - # prompt tokens": "{\"description\": \"min=501.121, mean=511.121, max=529.121, sum=1533.362 (3)\", \"tab\": \"General information\", \"score\": \"511.12079510703364\"}", - "TruthfulQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=3 (3)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "TruthfulQA - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MS MARCO (TREC)", - "source_data": { - "dataset_name": "MS MARCO (TREC)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "NDCG@10 on MS MARCO (TREC)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.664, - "details": { - "description": "min=0.642, mean=0.664, max=0.685, sum=1.991 (3)", - "tab": "Accuracy", - "MS MARCO (regular) - RR@10 (Robustness)": "{\"description\": \"min=0.327, mean=0.344, max=0.366, sum=1.031 (3)\", \"tab\": \"Robustness\", \"score\": \"0.3435873015873012\"}", - "MS MARCO (TREC) - NDCG@10 (Robustness)": "{\"description\": \"min=0.615, mean=0.628, max=0.641, sum=1.884 (3)\", \"tab\": \"Robustness\", \"score\": \"0.627999061572698\"}", - "MS MARCO (regular) - RR@10 (Fairness)": "{\"description\": \"min=0.357, mean=0.373, max=0.39, sum=1.12 (3)\", \"tab\": \"Fairness\", \"score\": \"0.3732579365079361\"}", - "MS MARCO (TREC) - NDCG@10 (Fairness)": "{\"description\": \"min=0.614, mean=0.639, max=0.663, sum=1.917 (3)\", \"tab\": \"Fairness\", \"score\": \"0.6388640932298691\"}", - "MS MARCO (regular) - Denoised inference time (s)": "{\"description\": \"min=0.174, mean=0.192, max=0.207, sum=0.577 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.19244404882812502\"}", - "MS MARCO (TREC) - Denoised inference time (s)": "{\"description\": \"min=0.173, mean=0.198, max=0.213, sum=0.594 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.19810631661821707\"}", - "MS MARCO (regular) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "MS MARCO (regular) - # train": "{\"description\": \"min=2, mean=2, max=2, sum=6 (3)\", \"tab\": \"General information\", \"score\": \"2.0\"}", - "MS MARCO (regular) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MS MARCO (regular) - # prompt tokens": "{\"description\": \"min=495.232, mean=532.565, max=577.232, sum=1597.696 (3)\", \"tab\": \"General information\", \"score\": \"532.5653333333333\"}", - "MS MARCO (regular) - # output tokens": "{\"description\": \"min=1.006, mean=1.014, max=1.024, sum=3.042 (3)\", \"tab\": \"General information\", \"score\": \"1.014\"}", - "MS MARCO (regular) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "MS MARCO (TREC) - # eval": "{\"description\": \"min=43, mean=43, max=43, sum=129 (3)\", \"tab\": \"General information\", \"score\": \"43.0\"}", - "MS MARCO (TREC) - # train": "{\"description\": \"min=2, mean=2, max=2, sum=6 (3)\", \"tab\": \"General information\", \"score\": \"2.0\"}", - "MS MARCO (TREC) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MS MARCO (TREC) - # prompt tokens": "{\"description\": \"min=478.488, mean=515.822, max=560.488, sum=1547.465 (3)\", \"tab\": \"General information\", \"score\": \"515.8217054263565\"}", - "MS MARCO (TREC) - # output tokens": "{\"description\": \"min=0.977, mean=0.992, max=1, sum=2.977 (3)\", \"tab\": \"General information\", \"score\": \"0.9922480620155039\"}", - "MS MARCO (TREC) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "MS MARCO (regular) - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "MS MARCO (TREC) - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CNN/DailyMail", - "source_data": { - "dataset_name": "CNN/DailyMail", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on CNN/DailyMail", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.153, - "details": { - "description": "min=0.148, mean=0.153, max=0.156, sum=1.074 (7)", - "tab": "Accuracy", - "CNN/DailyMail - Denoised inference time (s)": "{\"description\": \"min=2.064, mean=2.236, max=2.638, sum=15.65 (7)\", \"tab\": \"Efficiency\", \"score\": \"2.235718461202547\"}", - "CNN/DailyMail - # eval": "{\"description\": \"min=466, mean=466, max=466, sum=3262 (7)\", \"tab\": \"General information\", \"score\": \"466.0\"}", - "CNN/DailyMail - # train": "{\"description\": \"min=0, mean=4.286, max=5, sum=30 (7)\", \"tab\": \"General information\", \"score\": \"4.285714285714286\"}", - "CNN/DailyMail - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CNN/DailyMail - # prompt tokens": "{\"description\": \"min=583.586, mean=1411.872, max=1567.586, sum=9883.101 (7)\", \"tab\": \"General information\", \"score\": \"1411.8715511955854\"}", - "CNN/DailyMail - # output tokens": "{\"description\": \"min=64.197, mean=70.37, max=85.644, sum=492.592 (7)\", \"tab\": \"General information\", \"score\": \"70.37032495401594\"}", - "CNN/DailyMail - # trials": "{\"description\": \"min=1, mean=2.714, max=3, sum=19 (7)\", \"tab\": \"General information\", \"score\": \"2.7142857142857144\"}", - "CNN/DailyMail - Stereotypes (race)": "{\"description\": \"min=0.603, mean=0.625, max=0.667, sum=4.375 (7)\", \"tab\": \"Bias\", \"score\": \"0.6249837439576494\"}", - "CNN/DailyMail - Stereotypes (gender)": "{\"description\": \"min=0.388, mean=0.408, max=0.42, sum=2.856 (7)\", \"tab\": \"Bias\", \"score\": \"0.4080224162158765\"}", - "CNN/DailyMail - Representation (race)": "{\"description\": \"min=0.238, mean=0.293, max=0.347, sum=2.051 (7)\", \"tab\": \"Bias\", \"score\": \"0.293047968208597\"}", - "CNN/DailyMail - Representation (gender)": "{\"description\": \"min=0.07, mean=0.107, max=0.138, sum=0.752 (7)\", \"tab\": \"Bias\", \"score\": \"0.1073937839039085\"}", - "CNN/DailyMail - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}", - "CNN/DailyMail - SummaC": "{\"description\": \"min=0.136, mean=0.353, max=0.455, sum=1.412 (4)\", \"tab\": \"Summarization metrics\", \"score\": \"0.35298687802144607\"}", - "CNN/DailyMail - QAFactEval": "{\"description\": \"min=4.04, mean=4.635, max=4.834, sum=32.448 (7)\", \"tab\": \"Summarization metrics\", \"score\": \"4.635409033816104\"}", - "CNN/DailyMail - BERTScore (F1)": "{\"description\": \"min=0.303, mean=0.321, max=0.333, sum=1.283 (4)\", \"tab\": \"Summarization metrics\", \"score\": \"0.3206946902747002\"}", - "CNN/DailyMail - Coverage": "{\"description\": \"min=0.904, mean=0.946, max=0.957, sum=6.625 (7)\", \"tab\": \"Summarization metrics\", \"score\": \"0.9464923911138073\"}", - "CNN/DailyMail - Density": "{\"description\": \"min=13.275, mean=15.995, max=17.016, sum=111.962 (7)\", \"tab\": \"Summarization metrics\", \"score\": \"15.994591776988235\"}", - "CNN/DailyMail - Compression": "{\"description\": \"min=7.152, mean=8.818, max=9.675, sum=61.729 (7)\", \"tab\": \"Summarization metrics\", \"score\": \"8.818392473408851\"}", - "CNN/DailyMail - HumanEval-faithfulness": "{\"description\": \"min=0.993, mean=0.999, max=1, sum=6.993 (7)\", \"tab\": \"Summarization metrics\", \"score\": \"0.9990476190476191\"}", - "CNN/DailyMail - HumanEval-relevance": "{\"description\": \"min=4.333, mean=4.435, max=4.6, sum=31.044 (7)\", \"tab\": \"Summarization metrics\", \"score\": \"4.434920634920635\"}", - "CNN/DailyMail - HumanEval-coherence": "{\"description\": \"min=4, mean=4.371, max=5, sum=30.598 (7)\", \"tab\": \"Summarization metrics\", \"score\": \"4.3711111111111105\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "XSUM", - "source_data": { - "dataset_name": "XSUM", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on XSUM", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.144, - "details": { - "description": "min=0.087, mean=0.144, max=0.161, sum=1.006 (7)", - "tab": "Accuracy", - "XSUM - Denoised inference time (s)": "{\"description\": \"min=1.003, mean=1.026, max=1.088, sum=7.181 (7)\", \"tab\": \"Efficiency\", \"score\": \"1.0257979815553757\"}", - "XSUM - # eval": "{\"description\": \"min=518, mean=518, max=518, sum=3626 (7)\", \"tab\": \"General information\", \"score\": \"518.0\"}", - "XSUM - # train": "{\"description\": \"min=0, mean=4.286, max=5, sum=30 (7)\", \"tab\": \"General information\", \"score\": \"4.285714285714286\"}", - "XSUM - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "XSUM - # prompt tokens": "{\"description\": \"min=388.402, mean=1350.402, max=1539.402, sum=9452.811 (7)\", \"tab\": \"General information\", \"score\": \"1350.4015444015445\"}", - "XSUM - # output tokens": "{\"description\": \"min=27.776, mean=28.674, max=31.952, sum=200.716 (7)\", \"tab\": \"General information\", \"score\": \"28.673745173745175\"}", - "XSUM - # trials": "{\"description\": \"min=1, mean=2.714, max=3, sum=19 (7)\", \"tab\": \"General information\", \"score\": \"2.7142857142857144\"}", - "XSUM - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=4.667 (7)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666667\"}", - "XSUM - Stereotypes (gender)": "{\"description\": \"min=0.441, mean=0.457, max=0.48, sum=3.202 (7)\", \"tab\": \"Bias\", \"score\": \"0.45745150585486727\"}", - "XSUM - Representation (race)": "{\"description\": \"min=0.376, mean=0.481, max=0.556, sum=3.37 (7)\", \"tab\": \"Bias\", \"score\": \"0.48149813295367977\"}", - "XSUM - Representation (gender)": "{\"description\": \"min=0.19, mean=0.239, max=0.257, sum=1.672 (7)\", \"tab\": \"Bias\", \"score\": \"0.2388259605365298\"}", - "XSUM - Toxic fraction": "{\"description\": \"min=0, mean=0.002, max=0.004, sum=0.012 (7)\", \"tab\": \"Toxicity\", \"score\": \"0.0016547159404302263\"}", - "XSUM - SummaC": "{\"description\": \"min=-0.288, mean=-0.273, max=-0.257, sum=-1.091 (4)\", \"tab\": \"Summarization metrics\", \"score\": \"-0.2728636190391109\"}", - "XSUM - QAFactEval": "{\"description\": \"min=2.795, mean=3.007, max=3.207, sum=21.05 (7)\", \"tab\": \"Summarization metrics\", \"score\": \"3.0071326818732076\"}", - "XSUM - BERTScore (F1)": "{\"description\": \"min=0.366, mean=0.43, max=0.459, sum=1.718 (4)\", \"tab\": \"Summarization metrics\", \"score\": \"0.4296202005928721\"}", - "XSUM - Coverage": "{\"description\": \"min=0.789, mean=0.801, max=0.833, sum=5.604 (7)\", \"tab\": \"Summarization metrics\", \"score\": \"0.8005553389114972\"}", - "XSUM - Density": "{\"description\": \"min=2.471, mean=2.872, max=4.654, sum=20.107 (7)\", \"tab\": \"Summarization metrics\", \"score\": \"2.8724523474356\"}", - "XSUM - Compression": "{\"description\": \"min=13.554, mean=14.07, max=14.306, sum=98.488 (7)\", \"tab\": \"Summarization metrics\", \"score\": \"14.069713395015288\"}", - "XSUM - HumanEval-faithfulness": "{\"description\": \"min=0.762, mean=0.849, max=0.963, sum=5.941 (7)\", \"tab\": \"Summarization metrics\", \"score\": \"0.848692365835223\"}", - "XSUM - HumanEval-relevance": "{\"description\": \"min=4.277, mean=4.41, max=4.63, sum=30.869 (7)\", \"tab\": \"Summarization metrics\", \"score\": \"4.40989417989418\"}", - "XSUM - HumanEval-coherence": "{\"description\": \"min=4.403, mean=4.685, max=4.815, sum=32.795 (7)\", \"tab\": \"Summarization metrics\", \"score\": \"4.684981103552532\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "IMDB", - "source_data": { - "dataset_name": "IMDB", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on IMDB", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.948, - "details": { - "description": "min=0.945, mean=0.948, max=0.953, sum=2.843 (3)", - "tab": "Accuracy", - "IMDB - ECE (10-bin)": "{\"description\": \"min=0.029, mean=0.031, max=0.033, sum=0.092 (3)\", \"tab\": \"Calibration\", \"score\": \"0.03076843904734194\"}", - "IMDB - EM (Robustness)": "{\"description\": \"min=0.898, mean=0.925, max=0.946, sum=2.776 (3)\", \"tab\": \"Robustness\", \"score\": \"0.9253333333333332\"}", - "IMDB - EM (Fairness)": "{\"description\": \"min=0.919, mean=0.934, max=0.945, sum=2.803 (3)\", \"tab\": \"Fairness\", \"score\": \"0.9343333333333333\"}", - "IMDB - Denoised inference time (s)": "{\"description\": \"min=0.218, mean=0.247, max=0.279, sum=0.741 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.24716598621961808\"}", - "IMDB - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "IMDB - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "IMDB - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IMDB - # prompt tokens": "{\"description\": \"min=1282.797, mean=1897.464, max=2572.797, sum=5692.391 (3)\", \"tab\": \"General information\", \"score\": \"1897.4636666666665\"}", - "IMDB - # output tokens": "{\"description\": \"min=0.999, mean=1.0, max=1, sum=2.999 (3)\", \"tab\": \"General information\", \"score\": \"0.9996666666666667\"}", - "IMDB - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "IMDB - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CivilComments", - "source_data": { - "dataset_name": "CivilComments", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on CivilComments", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.668, - "details": { - "description": "min=0.4, mean=0.668, max=0.876, sum=36.093 (54)", - "tab": "Accuracy", - "CivilComments - ECE (10-bin)": "{\"description\": \"min=0.018, mean=0.183, max=0.424, sum=9.875 (54)\", \"tab\": \"Calibration\", \"score\": \"0.18286487616515196\"}", - "CivilComments - EM (Robustness)": "{\"description\": \"min=0.287, mean=0.567, max=0.838, sum=30.64 (54)\", \"tab\": \"Robustness\", \"score\": \"0.5673997819699065\"}", - "CivilComments - EM (Fairness)": "{\"description\": \"min=0.082, mean=0.463, max=0.851, sum=24.991 (54)\", \"tab\": \"Fairness\", \"score\": \"0.46278978149694866\"}", - "CivilComments - Denoised inference time (s)": "{\"description\": \"min=0.174, mean=0.186, max=0.217, sum=10.038 (54)\", \"tab\": \"Efficiency\", \"score\": \"0.18589157378997984\"}", - "CivilComments - # eval": "{\"description\": \"min=74, mean=371.556, max=683, sum=20064 (54)\", \"tab\": \"General information\", \"score\": \"371.55555555555554\"}", - "CivilComments - # train": "{\"description\": \"min=5, mean=5, max=5, sum=270 (54)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CivilComments - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (54)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CivilComments - # prompt tokens": "{\"description\": \"min=356.537, mean=722.635, max=1267.519, sum=39022.317 (54)\", \"tab\": \"General information\", \"score\": \"722.6354931173206\"}", - "CivilComments - # output tokens": "{\"description\": \"min=0.967, mean=0.997, max=1, sum=53.855 (54)\", \"tab\": \"General information\", \"score\": \"0.9973133394349212\"}", - "CivilComments - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=162 (54)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "CivilComments - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (54)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "RAFT", - "source_data": { - "dataset_name": "RAFT", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on RAFT", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.733, - "details": { - "description": "min=0.15, mean=0.733, max=0.975, sum=24.175 (33)", - "tab": "Accuracy", - "RAFT - ECE (10-bin)": "{\"description\": \"min=0.043, mean=0.212, max=0.586, sum=6.999 (33)\", \"tab\": \"Calibration\", \"score\": \"0.21210473630230625\"}", - "RAFT - EM (Robustness)": "{\"description\": \"min=0, mean=0.666, max=0.975, sum=21.975 (33)\", \"tab\": \"Robustness\", \"score\": \"0.665909090909091\"}", - "RAFT - EM (Fairness)": "{\"description\": \"min=0.125, mean=0.671, max=0.975, sum=22.15 (33)\", \"tab\": \"Fairness\", \"score\": \"0.6712121212121211\"}", - "RAFT - Denoised inference time (s)": "{\"description\": \"min=0.195, mean=0.276, max=0.351, sum=9.119 (33)\", \"tab\": \"Efficiency\", \"score\": \"0.27634172535905943\"}", - "RAFT - # eval": "{\"description\": \"min=40, mean=40, max=40, sum=1320 (33)\", \"tab\": \"General information\", \"score\": \"40.0\"}", - "RAFT - # train": "{\"description\": \"min=2.025, mean=4.752, max=5, sum=156.8 (33)\", \"tab\": \"General information\", \"score\": \"4.751515151515152\"}", - "RAFT - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (33)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "RAFT - # prompt tokens": "{\"description\": \"min=257.35, mean=1033.465, max=3591.4, sum=34104.35 (33)\", \"tab\": \"General information\", \"score\": \"1033.4651515151515\"}", - "RAFT - # output tokens": "{\"description\": \"min=0.875, mean=3.057, max=6.85, sum=100.875 (33)\", \"tab\": \"General information\", \"score\": \"3.0568181818181817\"}", - "RAFT - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=99 (33)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "RAFT - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (33)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_lite/openai_text-davinci-002/1774096306.427425", - "retrieved_timestamp": "1774096306.427425", - "source_metadata": { - "source_name": "helm_lite", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_lite", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_lite", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.336, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.6860299625468165\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.719, - "details": { - "description": "min=0.719, mean=0.719, max=0.719, sum=0.719 (1)", - "tab": "Accuracy", - "NarrativeQA - Observed inference time (s)": "{\"description\": \"min=1.226, mean=1.226, max=1.226, sum=1.226 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.2258358747186795\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=4.955, mean=4.955, max=4.955, sum=4.955 (1)\", \"tab\": \"General information\", \"score\": \"4.954929577464789\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=3479.563, mean=3479.563, max=3479.563, sum=3479.563 (1)\", \"tab\": \"General information\", \"score\": \"3479.56338028169\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=8.448, mean=8.448, max=8.448, sum=8.448 (1)\", \"tab\": \"General information\", \"score\": \"8.447887323943663\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (closed-book)", - "source_data": { - "dataset_name": "NaturalQuestions (closed-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (closed-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.394, - "details": { - "description": "min=0.394, mean=0.394, max=0.394, sum=0.394 (1)", - "tab": "Accuracy", - "NaturalQuestions (open-book) - Observed inference time (s)": "{\"description\": \"min=0.886, mean=0.886, max=0.886, sum=0.886 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.8863302536010742\"}", - "NaturalQuestions (closed-book) - Observed inference time (s)": "{\"description\": \"min=0.683, mean=0.683, max=0.683, sum=0.683 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.6834516413211823\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.885, mean=4.885, max=4.885, sum=4.885 (1)\", \"tab\": \"General information\", \"score\": \"4.885\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.02, mean=0.02, max=0.02, sum=0.02 (1)\", \"tab\": \"General information\", \"score\": \"0.02\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1617.729, mean=1617.729, max=1617.729, sum=1617.729 (1)\", \"tab\": \"General information\", \"score\": \"1617.729\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=6.632, mean=6.632, max=6.632, sum=6.632 (1)\", \"tab\": \"General information\", \"score\": \"6.632\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=116.254, mean=116.254, max=116.254, sum=116.254 (1)\", \"tab\": \"General information\", \"score\": \"116.254\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=4.116, mean=4.116, max=4.116, sum=4.116 (1)\", \"tab\": \"General information\", \"score\": \"4.116\"}" - } - }, - "generation_config": { - "additional_details": { - "mode": "\"closedbook\"" - } - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.796, - "details": { - "description": "min=0.796, mean=0.796, max=0.796, sum=0.796 (1)", - "tab": "Accuracy", - "OpenbookQA - Observed inference time (s)": "{\"description\": \"min=0.174, mean=0.174, max=0.174, sum=0.174 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.1743956871032715\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=254.21, mean=254.21, max=254.21, sum=254.21 (1)\", \"tab\": \"General information\", \"score\": \"254.21\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "dataset": "\"openbookqa\"", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.568, - "details": { - "description": "min=0.26, mean=0.568, max=0.84, sum=2.841 (5)", - "tab": "Accuracy", - "MMLU - Observed inference time (s)": "{\"description\": \"min=0.175, mean=0.177, max=0.181, sum=0.887 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.17730724048614502\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=371.38, mean=472.274, max=624.07, sum=2361.37 (5)\", \"tab\": \"General information\", \"score\": \"472.2740350877192\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"college_chemistry\", \"computer_security\", \"econometrics\", \"us_foreign_policy\"]", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MATH", - "source_data": { - "dataset_name": "MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Equivalent (CoT) on MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.428, - "details": { - "description": "min=0.288, mean=0.428, max=0.548, sum=2.997 (7)", - "tab": "Accuracy", - "MATH - Observed inference time (s)": "{\"description\": \"min=3.257, mean=5.188, max=9.459, sum=36.316 (7)\", \"tab\": \"Efficiency\", \"score\": \"5.188020693120383\"}", - "MATH - # eval": "{\"description\": \"min=30, mean=62.429, max=135, sum=437 (7)\", \"tab\": \"General information\", \"score\": \"62.42857142857143\"}", - "MATH - # train": "{\"description\": \"min=8, mean=8, max=8, sum=56 (7)\", \"tab\": \"General information\", \"score\": \"8.0\"}", - "MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MATH - # prompt tokens": "{\"description\": \"min=906.556, mean=1375.735, max=2449.942, sum=9630.147 (7)\", \"tab\": \"General information\", \"score\": \"1375.7353092779654\"}", - "MATH - # output tokens": "{\"description\": \"min=76.721, mean=136.822, max=259.175, sum=957.754 (7)\", \"tab\": \"General information\", \"score\": \"136.82193804427587\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"algebra\", \"counting_and_probability\", \"geometry\", \"intermediate_algebra\", \"number_theory\", \"prealgebra\", \"precalculus\"]", - "level": "\"1\"", - "use_official_examples": "\"False\"", - "use_chain_of_thought": "\"True\"" - } - } - }, - { - "evaluation_name": "GSM8K", - "source_data": { - "dataset_name": "GSM8K", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on GSM8K", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.479, - "details": { - "description": "min=0.479, mean=0.479, max=0.479, sum=0.479 (1)", - "tab": "Accuracy", - "GSM8K - Observed inference time (s)": "{\"description\": \"min=3.762, mean=3.762, max=3.762, sum=3.762 (1)\", \"tab\": \"Efficiency\", \"score\": \"3.762208682537079\"}", - "GSM8K - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "GSM8K - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "GSM8K - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GSM8K - # prompt tokens": "{\"description\": \"min=938.869, mean=938.869, max=938.869, sum=938.869 (1)\", \"tab\": \"General information\", \"score\": \"938.869\"}", - "GSM8K - # output tokens": "{\"description\": \"min=90.543, mean=90.543, max=90.543, sum=90.543 (1)\", \"tab\": \"General information\", \"score\": \"90.543\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "LegalBench", - "source_data": { - "dataset_name": "LegalBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on LegalBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.58, - "details": { - "description": "min=0.326, mean=0.58, max=0.916, sum=2.901 (5)", - "tab": "Accuracy", - "LegalBench - Observed inference time (s)": "{\"description\": \"min=0.167, mean=0.223, max=0.403, sum=1.115 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.2229105462585103\"}", - "LegalBench - # eval": "{\"description\": \"min=95, mean=409.4, max=1000, sum=2047 (5)\", \"tab\": \"General information\", \"score\": \"409.4\"}", - "LegalBench - # train": "{\"description\": \"min=2.053, mean=4.211, max=5, sum=21.053 (5)\", \"tab\": \"General information\", \"score\": \"4.210612244897959\"}", - "LegalBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "LegalBench - # prompt tokens": "{\"description\": \"min=205.632, mean=907.387, max=3225.32, sum=4536.936 (5)\", \"tab\": \"General information\", \"score\": \"907.3872120499769\"}", - "LegalBench - # output tokens": "{\"description\": \"min=0.996, mean=1.099, max=1.238, sum=5.496 (5)\", \"tab\": \"General information\", \"score\": \"1.0991972687655298\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"abercrombie\", \"corporate_lobbying\", \"function_of_decision_section\", \"international_citizenship_questions\", \"proa\"]" - } - } - }, - { - "evaluation_name": "MedQA", - "source_data": { - "dataset_name": "MedQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MedQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.525, - "details": { - "description": "min=0.525, mean=0.525, max=0.525, sum=0.525 (1)", - "tab": "Accuracy", - "MedQA - Observed inference time (s)": "{\"description\": \"min=0.206, mean=0.206, max=0.206, sum=0.206 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.20554606720183052\"}", - "MedQA - # eval": "{\"description\": \"min=503, mean=503, max=503, sum=503 (1)\", \"tab\": \"General information\", \"score\": \"503.0\"}", - "MedQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MedQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MedQA - # prompt tokens": "{\"description\": \"min=1038.861, mean=1038.861, max=1038.861, sum=1038.861 (1)\", \"tab\": \"General information\", \"score\": \"1038.8608349900596\"}", - "MedQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WMT 2014", - "source_data": { - "dataset_name": "WMT 2014", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "BLEU-4 on WMT 2014", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.174, - "details": { - "description": "min=0.077, mean=0.174, max=0.212, sum=0.872 (5)", - "tab": "Accuracy", - "WMT 2014 - Observed inference time (s)": "{\"description\": \"min=0.446, mean=0.467, max=0.478, sum=2.336 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.4672719452194591\"}", - "WMT 2014 - # eval": "{\"description\": \"min=503, mean=568.8, max=832, sum=2844 (5)\", \"tab\": \"General information\", \"score\": \"568.8\"}", - "WMT 2014 - # train": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "WMT 2014 - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WMT 2014 - # prompt tokens": "{\"description\": \"min=136.93, mean=181.694, max=241.662, sum=908.469 (5)\", \"tab\": \"General information\", \"score\": \"181.69386660804403\"}", - "WMT 2014 - # output tokens": "{\"description\": \"min=23.557, mean=24.862, max=25.636, sum=124.309 (5)\", \"tab\": \"General information\", \"score\": \"24.86174013610644\"}" - } - }, - "generation_config": { - "additional_details": { - "language_pair": "[\"cs-en\", \"de-en\", \"fr-en\", \"hi-en\", \"ru-en\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/openai_text-davinci-003.json b/data/models/openai_text-davinci-003.json deleted file mode 100644 index 62c6f7cdb0e16cefc8c1b0a0e7e8dc8723375492..0000000000000000000000000000000000000000 --- a/data/models/openai_text-davinci-003.json +++ /dev/null @@ -1,1044 +0,0 @@ -{ - "model_info": { - "name": "GPT-3.5 text-davinci-003", - "id": "openai/text-davinci-003", - "developer": "OpenAI", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_classic/openai_text-davinci-003/1774096308.339228", - "retrieved_timestamp": "1774096308.339228", - "source_metadata": { - "source_name": "helm_classic", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_classic", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_classic", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperform on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.872, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Calibration": "{\"description\": \"\", \"tab\": \"Calibration\", \"score\": \"0.4065137447036923\"}", - "Mean win rate - Robustness": "{\"description\": \"\", \"tab\": \"Robustness\", \"score\": \"0.9095617026651509\"}", - "Mean win rate - Fairness": "{\"description\": \"\", \"tab\": \"Fairness\", \"score\": \"0.9027696441489546\"}", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}", - "Mean win rate - Bias": "{\"description\": \"\", \"tab\": \"Bias\", \"score\": \"0.4087317179294733\"}", - "Mean win rate - Toxicity": "{\"description\": \"\", \"tab\": \"Toxicity\", \"score\": \"0.4974399057732391\"}", - "Mean win rate - Summarization metrics": "{\"description\": \"\", \"tab\": \"Summarization metrics\", \"score\": \"0.5263157894736842\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.569, - "details": { - "description": "min=0.28, mean=0.569, max=0.86, sum=8.532 (15)", - "tab": "Accuracy", - "MMLU - ECE (10-bin)": "{\"description\": \"min=0.127, mean=0.317, max=0.54, sum=4.761 (15)\", \"tab\": \"Calibration\", \"score\": \"0.31740378740673564\"}", - "MMLU - EM (Robustness)": "{\"description\": \"min=0.19, mean=0.517, max=0.84, sum=7.752 (15)\", \"tab\": \"Robustness\", \"score\": \"0.5167953216374268\"}", - "MMLU - EM (Fairness)": "{\"description\": \"min=0.24, mean=0.537, max=0.83, sum=8.054 (15)\", \"tab\": \"Fairness\", \"score\": \"0.5369590643274853\"}", - "MMLU - Denoised inference time (s)": "{\"description\": \"5 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=1542 (15)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=75 (15)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (15)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=371.38, mean=472.274, max=624.07, sum=7084.111 (15)\", \"tab\": \"General information\", \"score\": \"472.2740350877193\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=15 (15)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "MMLU - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=45 (15)\", \"tab\": \"General information\", \"score\": \"3.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "BoolQ", - "source_data": { - "dataset_name": "BoolQ", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on BoolQ", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.881, - "details": { - "description": "min=0.879, mean=0.881, max=0.883, sum=2.644 (3)\n⚠ Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.", - "tab": "Accuracy", - "BoolQ - ECE (10-bin)": "{\"description\": \"min=0.097, mean=0.098, max=0.099, sum=0.295 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Calibration\", \"score\": \"0.09835218401604591\"}", - "BoolQ - EM (Robustness)": "{\"description\": \"min=0.851, mean=0.858, max=0.864, sum=2.573 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Robustness\", \"score\": \"0.8576666666666667\"}", - "BoolQ - EM (Fairness)": "{\"description\": \"min=0.854, mean=0.858, max=0.861, sum=2.574 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Fairness\", \"score\": \"0.858\"}", - "BoolQ - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "BoolQ - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "BoolQ - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "BoolQ - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "BoolQ - # prompt tokens": "{\"description\": \"min=660.073, mean=908.406, max=1242.073, sum=2725.219 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"908.4063333333334\"}", - "BoolQ - # output tokens": "{\"description\": \"min=1.036, mean=1.043, max=1.058, sum=3.13 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"1.0433333333333332\"}", - "BoolQ - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "BoolQ - Stereotypes (race)": "{\"description\": \"(0)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Stereotypes (gender)": "{\"description\": \"(0)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (race)": "{\"description\": \"(0)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (gender)": "{\"description\": \"min=0.5, mean=0.5, max=0.5, sum=1 (2)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.5\"}", - "BoolQ - Toxic fraction": "{\"description\": \"min=0, mean=0.001, max=0.001, sum=0.002 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Toxicity\", \"score\": \"0.0006666666666666666\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.727, - "details": { - "description": "min=0.703, mean=0.727, max=0.747, sum=2.181 (3)", - "tab": "Accuracy", - "NarrativeQA - ECE (10-bin)": "{\"description\": \"min=0.346, mean=0.37, max=0.389, sum=1.111 (3)\", \"tab\": \"Calibration\", \"score\": \"0.3702182824812234\"}", - "NarrativeQA - F1 (Robustness)": "{\"description\": \"min=0.673, mean=0.694, max=0.713, sum=2.082 (3)\", \"tab\": \"Robustness\", \"score\": \"0.6939161040603179\"}", - "NarrativeQA - F1 (Fairness)": "{\"description\": \"min=0.643, mean=0.664, max=0.682, sum=1.993 (3)\", \"tab\": \"Fairness\", \"score\": \"0.6644210581739292\"}", - "NarrativeQA - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=1065 (3)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=4.259, mean=4.532, max=4.955, sum=13.597 (3)\", \"tab\": \"General information\", \"score\": \"4.532394366197183\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=3479.563, mean=3579.093, max=3633.659, sum=10737.279 (3)\", \"tab\": \"General information\", \"score\": \"3579.092957746479\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=8.231, mean=9.164, max=9.732, sum=27.493 (3)\", \"tab\": \"General information\", \"score\": \"9.16431924882629\"}", - "NarrativeQA - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NarrativeQA - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=0.667 (1)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666667\"}", - "NarrativeQA - Stereotypes (gender)": "{\"description\": \"min=0.424, mean=0.442, max=0.464, sum=1.327 (3)\", \"tab\": \"Bias\", \"score\": \"0.44232989232989234\"}", - "NarrativeQA - Representation (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=2 (3)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "NarrativeQA - Representation (gender)": "{\"description\": \"min=0.169, mean=0.177, max=0.187, sum=0.532 (3)\", \"tab\": \"Bias\", \"score\": \"0.17722658310007708\"}", - "NarrativeQA - Toxic fraction": "{\"description\": \"min=0.011, mean=0.013, max=0.014, sum=0.039 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.013145539906103287\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (open-book)", - "source_data": { - "dataset_name": "NaturalQuestions (open-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (open-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.77, - "details": { - "description": "min=0.768, mean=0.77, max=0.773, sum=2.311 (3)\n⚠ Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.", - "tab": "Accuracy", - "NaturalQuestions (closed-book) - ECE (10-bin)": "{\"description\": \"min=0.27, mean=0.286, max=0.299, sum=0.857 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Calibration\", \"score\": \"0.28562303267045125\"}", - "NaturalQuestions (open-book) - ECE (10-bin)": "{\"description\": \"min=0.318, mean=0.323, max=0.331, sum=0.969 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Calibration\", \"score\": \"0.3230345144505907\"}", - "NaturalQuestions (closed-book) - F1 (Robustness)": "{\"description\": \"min=0.36, mean=0.369, max=0.376, sum=1.106 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Robustness\", \"score\": \"0.36865975256659933\"}", - "NaturalQuestions (open-book) - F1 (Robustness)": "{\"description\": \"min=0.729, mean=0.73, max=0.733, sum=2.191 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Robustness\", \"score\": \"0.7304543451569532\"}", - "NaturalQuestions (closed-book) - F1 (Fairness)": "{\"description\": \"min=0.349, mean=0.356, max=0.361, sum=1.069 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Fairness\", \"score\": \"0.3564629891973459\"}", - "NaturalQuestions (open-book) - F1 (Fairness)": "{\"description\": \"min=0.719, mean=0.721, max=0.725, sum=2.164 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Fairness\", \"score\": \"0.7213345530431851\"}", - "NaturalQuestions (closed-book) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NaturalQuestions (open-book) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=110.254, mean=112.254, max=116.254, sum=336.762 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"112.254\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=7.074, mean=7.964, max=8.442, sum=23.891 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"7.963666666666666\"}", - "NaturalQuestions (closed-book) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.874, mean=4.883, max=4.891, sum=14.65 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"4.883333333333334\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.02, mean=0.02, max=0.02, sum=0.06 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.02\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1315.257, mean=1520.977, max=1629.945, sum=4562.931 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"1520.977\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=6.8, mean=6.937, max=7.011, sum=20.81 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"6.9366666666666665\"}", - "NaturalQuestions (open-book) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NaturalQuestions (closed-book) - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=1.333 (2)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.6666666666666667\"}", - "NaturalQuestions (closed-book) - Stereotypes (gender)": "{\"description\": \"min=0.452, mean=0.484, max=0.5, sum=1.452 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.4841269841269842\"}", - "NaturalQuestions (closed-book) - Representation (race)": "{\"description\": \"min=0.292, mean=0.347, max=0.43, sum=1.042 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.34749417249417247\"}", - "NaturalQuestions (closed-book) - Representation (gender)": "{\"description\": \"min=0.167, mean=0.27, max=0.367, sum=0.811 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.27037037037037037\"}", - "NaturalQuestions (open-book) - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=0.667 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.6666666666666667\"}", - "NaturalQuestions (open-book) - Stereotypes (gender)": "{\"description\": \"min=0.5, mean=0.5, max=0.5, sum=1.5 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.5\"}", - "NaturalQuestions (open-book) - Representation (race)": "{\"description\": \"min=0.429, mean=0.443, max=0.454, sum=1.328 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.4428170082518513\"}", - "NaturalQuestions (open-book) - Representation (gender)": "{\"description\": \"min=0.375, mean=0.407, max=0.423, sum=1.221 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.407051282051282\"}", - "NaturalQuestions (closed-book) - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}", - "NaturalQuestions (open-book) - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "QuAC", - "source_data": { - "dataset_name": "QuAC", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on QuAC", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.525, - "details": { - "description": "min=0.496, mean=0.525, max=0.54, sum=1.574 (3)\n⚠ Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.", - "tab": "Accuracy", - "QuAC - ECE (10-bin)": "{\"description\": \"min=0.259, mean=0.27, max=0.279, sum=0.809 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Calibration\", \"score\": \"0.2696184343953211\"}", - "QuAC - F1 (Robustness)": "{\"description\": \"min=0.401, mean=0.42, max=0.432, sum=1.26 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Robustness\", \"score\": \"0.4199382541834728\"}", - "QuAC - F1 (Fairness)": "{\"description\": \"min=0.427, mean=0.45, max=0.465, sum=1.351 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Fairness\", \"score\": \"0.45040220156517236\"}", - "QuAC - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "QuAC - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "QuAC - # train": "{\"description\": \"min=2.978, mean=3.438, max=3.878, sum=10.315 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"3.438333333333333\"}", - "QuAC - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "QuAC - # prompt tokens": "{\"description\": \"min=2819.048, mean=3249.907, max=3487.39, sum=9749.722 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"3249.907333333333\"}", - "QuAC - # output tokens": "{\"description\": \"min=25.946, mean=27.199, max=28.821, sum=81.596 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"27.198666666666668\"}", - "QuAC - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "QuAC - Stereotypes (race)": "{\"description\": \"min=0.556, mean=0.582, max=0.606, sum=1.745 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.5816498316498318\"}", - "QuAC - Stereotypes (gender)": "{\"description\": \"min=0.427, mean=0.428, max=0.43, sum=1.285 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.4283515137656795\"}", - "QuAC - Representation (race)": "{\"description\": \"min=0.321, mean=0.369, max=0.395, sum=1.106 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.368660072841299\"}", - "QuAC - Representation (gender)": "{\"description\": \"min=0.244, mean=0.257, max=0.27, sum=0.772 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Bias\", \"score\": \"0.2573013036656095\"}", - "QuAC - Toxic fraction": "{\"description\": \"min=0.001, mean=0.001, max=0.001, sum=0.003 (3)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Toxicity\", \"score\": \"0.001\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "HellaSwag", - "source_data": { - "dataset_name": "HellaSwag", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on HellaSwag", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.822, - "details": { - "description": "min=0.822, mean=0.822, max=0.822, sum=0.822 (1)\n⚠ Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.", - "tab": "Accuracy", - "HellaSwag - ECE (10-bin)": "{\"description\": \"min=0.278, mean=0.278, max=0.278, sum=0.278 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Calibration\", \"score\": \"0.2781634038368795\"}", - "HellaSwag - EM (Robustness)": "{\"description\": \"min=0.798, mean=0.798, max=0.798, sum=0.798 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Robustness\", \"score\": \"0.798\"}", - "HellaSwag - EM (Fairness)": "{\"description\": \"min=0.729, mean=0.729, max=0.729, sum=0.729 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Fairness\", \"score\": \"0.729\"}", - "HellaSwag - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "HellaSwag - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "HellaSwag - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "HellaSwag - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "HellaSwag - # prompt tokens": "{\"description\": \"min=87.888, mean=87.888, max=87.888, sum=87.888 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"87.888\"}", - "HellaSwag - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "HellaSwag - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.646, - "details": { - "description": "min=0.646, mean=0.646, max=0.646, sum=0.646 (1)\n⚠ Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.", - "tab": "Accuracy", - "OpenbookQA - ECE (10-bin)": "{\"description\": \"min=0.216, mean=0.216, max=0.216, sum=0.216 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Calibration\", \"score\": \"0.21592533141452896\"}", - "OpenbookQA - EM (Robustness)": "{\"description\": \"min=0.572, mean=0.572, max=0.572, sum=0.572 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Robustness\", \"score\": \"0.572\"}", - "OpenbookQA - EM (Fairness)": "{\"description\": \"min=0.578, mean=0.578, max=0.578, sum=0.578 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"Fairness\", \"score\": \"0.578\"}", - "OpenbookQA - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=5.27, mean=5.27, max=5.27, sum=5.27 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"5.27\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\\n\\u26a0 Brown et al. perform an analysis of the contamination for GPT-3 and its known derivatives. For these datasets, they find that 1% - 6% of the datasets' test instances are contaminated based on N-gram overlap, and model performance does not substantially change for these datasets. See Table C.1 on page 45 of https://arxiv.org/pdf/2005.14165.pdf.\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "TruthfulQA", - "source_data": { - "dataset_name": "TruthfulQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on TruthfulQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.593, - "details": { - "description": "min=0.558, mean=0.593, max=0.615, sum=1.78 (3)", - "tab": "Accuracy", - "TruthfulQA - ECE (10-bin)": "{\"description\": \"min=0.329, mean=0.348, max=0.373, sum=1.043 (3)\", \"tab\": \"Calibration\", \"score\": \"0.3477434253470754\"}", - "TruthfulQA - EM (Robustness)": "{\"description\": \"min=0.479, mean=0.516, max=0.54, sum=1.549 (3)\", \"tab\": \"Robustness\", \"score\": \"0.5163098878695208\"}", - "TruthfulQA - EM (Fairness)": "{\"description\": \"min=0.448, mean=0.491, max=0.521, sum=1.474 (3)\", \"tab\": \"Fairness\", \"score\": \"0.491335372069317\"}", - "TruthfulQA - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "TruthfulQA - # eval": "{\"description\": \"min=654, mean=654, max=654, sum=1962 (3)\", \"tab\": \"General information\", \"score\": \"654.0\"}", - "TruthfulQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "TruthfulQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "TruthfulQA - # prompt tokens": "{\"description\": \"min=501.121, mean=511.121, max=529.121, sum=1533.362 (3)\", \"tab\": \"General information\", \"score\": \"511.12079510703364\"}", - "TruthfulQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=3 (3)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "TruthfulQA - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MS MARCO (TREC)", - "source_data": { - "dataset_name": "MS MARCO (TREC)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "NDCG@10 on MS MARCO (TREC)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.644, - "details": { - "description": "min=0.611, mean=0.644, max=0.662, sum=1.931 (3)", - "tab": "Accuracy", - "MS MARCO (regular) - RR@10 (Robustness)": "{\"description\": \"min=0.292, mean=0.304, max=0.319, sum=0.911 (3)\", \"tab\": \"Robustness\", \"score\": \"0.3037781746031745\"}", - "MS MARCO (TREC) - NDCG@10 (Robustness)": "{\"description\": \"min=0.578, mean=0.616, max=0.645, sum=1.848 (3)\", \"tab\": \"Robustness\", \"score\": \"0.6160995919712035\"}", - "MS MARCO (regular) - RR@10 (Fairness)": "{\"description\": \"min=0.322, mean=0.335, max=0.353, sum=1.005 (3)\", \"tab\": \"Fairness\", \"score\": \"0.33500119047619026\"}", - "MS MARCO (TREC) - NDCG@10 (Fairness)": "{\"description\": \"min=0.603, mean=0.633, max=0.652, sum=1.898 (3)\", \"tab\": \"Fairness\", \"score\": \"0.6326849780192724\"}", - "MS MARCO (regular) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MS MARCO (TREC) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MS MARCO (regular) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "MS MARCO (regular) - # train": "{\"description\": \"min=2, mean=2, max=2, sum=6 (3)\", \"tab\": \"General information\", \"score\": \"2.0\"}", - "MS MARCO (regular) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MS MARCO (regular) - # prompt tokens": "{\"description\": \"min=495.232, mean=532.565, max=577.232, sum=1597.696 (3)\", \"tab\": \"General information\", \"score\": \"532.5653333333333\"}", - "MS MARCO (regular) - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=3 (3)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "MS MARCO (regular) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "MS MARCO (TREC) - # eval": "{\"description\": \"min=43, mean=43, max=43, sum=129 (3)\", \"tab\": \"General information\", \"score\": \"43.0\"}", - "MS MARCO (TREC) - # train": "{\"description\": \"min=2, mean=2, max=2, sum=6 (3)\", \"tab\": \"General information\", \"score\": \"2.0\"}", - "MS MARCO (TREC) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MS MARCO (TREC) - # prompt tokens": "{\"description\": \"min=478.488, mean=515.822, max=560.488, sum=1547.465 (3)\", \"tab\": \"General information\", \"score\": \"515.8217054263565\"}", - "MS MARCO (TREC) - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=3 (3)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "MS MARCO (TREC) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "MS MARCO (regular) - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "MS MARCO (TREC) - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CNN/DailyMail", - "source_data": { - "dataset_name": "CNN/DailyMail", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on CNN/DailyMail", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.156, - "details": { - "description": "min=0.151, mean=0.156, max=0.16, sum=0.935 (6)", - "tab": "Accuracy", - "CNN/DailyMail - Denoised inference time (s)": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "CNN/DailyMail - # eval": "{\"description\": \"min=466, mean=466, max=466, sum=2796 (6)\", \"tab\": \"General information\", \"score\": \"466.0\"}", - "CNN/DailyMail - # train": "{\"description\": \"min=5, mean=5, max=5, sum=30 (6)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CNN/DailyMail - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (6)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CNN/DailyMail - # prompt tokens": "{\"description\": \"min=1531.586, mean=1549.919, max=1567.586, sum=9299.515 (6)\", \"tab\": \"General information\", \"score\": \"1549.9191702432045\"}", - "CNN/DailyMail - # output tokens": "{\"description\": \"min=60.524, mean=64.315, max=67.878, sum=385.888 (6)\", \"tab\": \"General information\", \"score\": \"64.31473533619457\"}", - "CNN/DailyMail - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=18 (6)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "CNN/DailyMail - Stereotypes (race)": "{\"description\": \"min=0.643, mean=0.646, max=0.652, sum=3.879 (6)\", \"tab\": \"Bias\", \"score\": \"0.6464418252138059\"}", - "CNN/DailyMail - Stereotypes (gender)": "{\"description\": \"min=0.404, mean=0.414, max=0.427, sum=2.482 (6)\", \"tab\": \"Bias\", \"score\": \"0.41359496216384023\"}", - "CNN/DailyMail - Representation (race)": "{\"description\": \"min=0.245, mean=0.274, max=0.29, sum=1.641 (6)\", \"tab\": \"Bias\", \"score\": \"0.2735791651454302\"}", - "CNN/DailyMail - Representation (gender)": "{\"description\": \"min=0.074, mean=0.083, max=0.099, sum=0.498 (6)\", \"tab\": \"Bias\", \"score\": \"0.08299026507382476\"}", - "CNN/DailyMail - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (6)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}", - "CNN/DailyMail - SummaC": "{\"description\": \"min=0.33, mean=0.359, max=0.403, sum=1.077 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.35893042891379157\"}", - "CNN/DailyMail - QAFactEval": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - BERTScore (F1)": "{\"description\": \"min=0.336, mean=0.342, max=0.347, sum=1.026 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.3420449797279243\"}", - "CNN/DailyMail - Coverage": "{\"description\": \"min=0.953, mean=0.956, max=0.959, sum=5.734 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"0.9556982855176755\"}", - "CNN/DailyMail - Density": "{\"description\": \"min=7.169, mean=7.545, max=7.928, sum=45.269 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"7.544859402012935\"}", - "CNN/DailyMail - Compression": "{\"description\": \"min=8.736, mean=9.389, max=10.065, sum=56.334 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"9.389062386727216\"}", - "CNN/DailyMail - HumanEval-faithfulness": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-relevance": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-coherence": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "XSUM", - "source_data": { - "dataset_name": "XSUM", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on XSUM", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.124, - "details": { - "description": "min=0.122, mean=0.124, max=0.126, sum=0.744 (6)", - "tab": "Accuracy", - "XSUM - Denoised inference time (s)": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "XSUM - # eval": "{\"description\": \"min=518, mean=518, max=518, sum=3108 (6)\", \"tab\": \"General information\", \"score\": \"518.0\"}", - "XSUM - # train": "{\"description\": \"min=5, mean=5, max=5, sum=30 (6)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "XSUM - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (6)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "XSUM - # prompt tokens": "{\"description\": \"min=1456.402, mean=1510.735, max=1539.402, sum=9064.409 (6)\", \"tab\": \"General information\", \"score\": \"1510.734877734878\"}", - "XSUM - # output tokens": "{\"description\": \"min=34.797, mean=35.293, max=36.073, sum=211.761 (6)\", \"tab\": \"General information\", \"score\": \"35.293436293436294\"}", - "XSUM - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=18 (6)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "XSUM - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=4.0 (6)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666669\"}", - "XSUM - Stereotypes (gender)": "{\"description\": \"min=0.413, mean=0.449, max=0.482, sum=2.694 (6)\", \"tab\": \"Bias\", \"score\": \"0.44896203413444785\"}", - "XSUM - Representation (race)": "{\"description\": \"min=0.518, mean=0.534, max=0.545, sum=3.202 (6)\", \"tab\": \"Bias\", \"score\": \"0.533635827356637\"}", - "XSUM - Representation (gender)": "{\"description\": \"min=0.234, mean=0.238, max=0.242, sum=1.427 (6)\", \"tab\": \"Bias\", \"score\": \"0.23788037651548422\"}", - "XSUM - Toxic fraction": "{\"description\": \"min=0, mean=0.001, max=0.004, sum=0.008 (6)\", \"tab\": \"Toxicity\", \"score\": \"0.001287001287001287\"}", - "XSUM - SummaC": "{\"description\": \"min=-0.313, mean=-0.301, max=-0.289, sum=-0.902 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"-0.3005772048135215\"}", - "XSUM - QAFactEval": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - BERTScore (F1)": "{\"description\": \"min=0.406, mean=0.411, max=0.414, sum=1.233 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.411029433026404\"}", - "XSUM - Coverage": "{\"description\": \"min=0.814, mean=0.822, max=0.829, sum=4.933 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"0.8221014569634312\"}", - "XSUM - Density": "{\"description\": \"min=2.461, mean=2.63, max=2.752, sum=15.779 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"2.6298820148802573\"}", - "XSUM - Compression": "{\"description\": \"min=10.736, mean=10.932, max=11.034, sum=65.59 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"10.931690583444237\"}", - "XSUM - HumanEval-faithfulness": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-relevance": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-coherence": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "IMDB", - "source_data": { - "dataset_name": "IMDB", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on IMDB", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.848, - "details": { - "description": "min=0.828, mean=0.848, max=0.881, sum=2.545 (3)", - "tab": "Accuracy", - "IMDB - ECE (10-bin)": "{\"description\": \"min=0.086, mean=0.113, max=0.132, sum=0.339 (3)\", \"tab\": \"Calibration\", \"score\": \"0.11283562591578779\"}", - "IMDB - EM (Robustness)": "{\"description\": \"min=0.749, mean=0.779, max=0.827, sum=2.338 (3)\", \"tab\": \"Robustness\", \"score\": \"0.7793333333333333\"}", - "IMDB - EM (Fairness)": "{\"description\": \"min=0.814, mean=0.833, max=0.868, sum=2.498 (3)\", \"tab\": \"Fairness\", \"score\": \"0.8326666666666666\"}", - "IMDB - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "IMDB - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "IMDB - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "IMDB - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IMDB - # prompt tokens": "{\"description\": \"min=1282.797, mean=1897.464, max=2572.797, sum=5692.391 (3)\", \"tab\": \"General information\", \"score\": \"1897.4636666666665\"}", - "IMDB - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=3 (3)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "IMDB - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "IMDB - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CivilComments", - "source_data": { - "dataset_name": "CivilComments", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on CivilComments", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.684, - "details": { - "description": "min=0.52, mean=0.684, max=0.863, sum=36.959 (54)", - "tab": "Accuracy", - "CivilComments - ECE (10-bin)": "{\"description\": \"min=0.12, mean=0.292, max=0.449, sum=15.772 (54)\", \"tab\": \"Calibration\", \"score\": \"0.29207184855040197\"}", - "CivilComments - EM (Robustness)": "{\"description\": \"min=0.366, mean=0.594, max=0.838, sum=32.08 (54)\", \"tab\": \"Robustness\", \"score\": \"0.5940672674614373\"}", - "CivilComments - EM (Fairness)": "{\"description\": \"min=0.253, mean=0.559, max=0.863, sum=30.179 (54)\", \"tab\": \"Fairness\", \"score\": \"0.5588650073949972\"}", - "CivilComments - Denoised inference time (s)": "{\"description\": \"9 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "CivilComments - # eval": "{\"description\": \"min=74, mean=371.556, max=683, sum=20064 (54)\", \"tab\": \"General information\", \"score\": \"371.55555555555554\"}", - "CivilComments - # train": "{\"description\": \"min=5, mean=5, max=5, sum=270 (54)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CivilComments - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (54)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CivilComments - # prompt tokens": "{\"description\": \"min=356.537, mean=722.635, max=1267.519, sum=39022.317 (54)\", \"tab\": \"General information\", \"score\": \"722.6354931173206\"}", - "CivilComments - # output tokens": "{\"description\": \"min=1, mean=1.0, max=1.007, sum=54.007 (54)\", \"tab\": \"General information\", \"score\": \"1.0001279344975371\"}", - "CivilComments - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=162 (54)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "CivilComments - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (54)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "RAFT", - "source_data": { - "dataset_name": "RAFT", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on RAFT", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.759, - "details": { - "description": "min=0.075, mean=0.759, max=0.95, sum=25.05 (33)", - "tab": "Accuracy", - "RAFT - ECE (10-bin)": "{\"description\": \"min=0.037, mean=0.203, max=0.736, sum=6.696 (33)\", \"tab\": \"Calibration\", \"score\": \"0.2029109351449743\"}", - "RAFT - EM (Robustness)": "{\"description\": \"min=0.05, mean=0.714, max=0.95, sum=23.55 (33)\", \"tab\": \"Robustness\", \"score\": \"0.7136363636363635\"}", - "RAFT - EM (Fairness)": "{\"description\": \"min=0.05, mean=0.705, max=0.95, sum=23.275 (33)\", \"tab\": \"Fairness\", \"score\": \"0.7053030303030302\"}", - "RAFT - Denoised inference time (s)": "{\"description\": \"11 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "RAFT - # eval": "{\"description\": \"min=40, mean=40, max=40, sum=1320 (33)\", \"tab\": \"General information\", \"score\": \"40.0\"}", - "RAFT - # train": "{\"description\": \"min=2.025, mean=4.752, max=5, sum=156.8 (33)\", \"tab\": \"General information\", \"score\": \"4.751515151515152\"}", - "RAFT - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (33)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "RAFT - # prompt tokens": "{\"description\": \"min=257.35, mean=1033.465, max=3591.4, sum=34104.35 (33)\", \"tab\": \"General information\", \"score\": \"1033.4651515151515\"}", - "RAFT - # output tokens": "{\"description\": \"min=1, mean=3.137, max=6.7, sum=103.525 (33)\", \"tab\": \"General information\", \"score\": \"3.1371212121212113\"}", - "RAFT - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=99 (33)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "RAFT - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (33)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_lite/openai_text-davinci-003/1774096306.427425", - "retrieved_timestamp": "1774096306.427425", - "source_metadata": { - "source_name": "helm_lite", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_lite", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_lite", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.439, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.5880524344569289\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.731, - "details": { - "description": "min=0.731, mean=0.731, max=0.731, sum=0.731 (1)", - "tab": "Accuracy", - "NarrativeQA - Observed inference time (s)": "{\"description\": \"min=1.813, mean=1.813, max=1.813, sum=1.813 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.812959625351597\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=4.955, mean=4.955, max=4.955, sum=4.955 (1)\", \"tab\": \"General information\", \"score\": \"4.954929577464789\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=3479.563, mean=3479.563, max=3479.563, sum=3479.563 (1)\", \"tab\": \"General information\", \"score\": \"3479.56338028169\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=9.732, mean=9.732, max=9.732, sum=9.732 (1)\", \"tab\": \"General information\", \"score\": \"9.732394366197184\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (closed-book)", - "source_data": { - "dataset_name": "NaturalQuestions (closed-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (closed-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.413, - "details": { - "description": "min=0.413, mean=0.413, max=0.413, sum=0.413 (1)", - "tab": "Accuracy", - "NaturalQuestions (open-book) - Observed inference time (s)": "{\"description\": \"min=1.187, mean=1.187, max=1.187, sum=1.187 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.1872664585113526\"}", - "NaturalQuestions (closed-book) - Observed inference time (s)": "{\"description\": \"min=0.996, mean=0.996, max=0.996, sum=0.996 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.9963206455707551\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.885, mean=4.885, max=4.885, sum=4.885 (1)\", \"tab\": \"General information\", \"score\": \"4.885\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.02, mean=0.02, max=0.02, sum=0.02 (1)\", \"tab\": \"General information\", \"score\": \"0.02\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1617.729, mean=1617.729, max=1617.729, sum=1617.729 (1)\", \"tab\": \"General information\", \"score\": \"1617.729\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=6.8, mean=6.8, max=6.8, sum=6.8 (1)\", \"tab\": \"General information\", \"score\": \"6.8\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=116.254, mean=116.254, max=116.254, sum=116.254 (1)\", \"tab\": \"General information\", \"score\": \"116.254\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=7.074, mean=7.074, max=7.074, sum=7.074 (1)\", \"tab\": \"General information\", \"score\": \"7.074\"}" - } - }, - "generation_config": { - "additional_details": { - "mode": "\"closedbook\"" - } - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.828, - "details": { - "description": "min=0.828, mean=0.828, max=0.828, sum=0.828 (1)", - "tab": "Accuracy", - "OpenbookQA - Observed inference time (s)": "{\"description\": \"min=0.204, mean=0.204, max=0.204, sum=0.204 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.20436767482757567\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=254.21, mean=254.21, max=254.21, sum=254.21 (1)\", \"tab\": \"General information\", \"score\": \"254.21\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "dataset": "\"openbookqa\"", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.555, - "details": { - "description": "min=0.3, mean=0.555, max=0.83, sum=2.774 (5)", - "tab": "Accuracy", - "MMLU - Observed inference time (s)": "{\"description\": \"min=0.199, mean=0.2, max=0.203, sum=1.0 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.2000334782098469\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=371.38, mean=472.274, max=624.07, sum=2361.37 (5)\", \"tab\": \"General information\", \"score\": \"472.2740350877192\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"college_chemistry\", \"computer_security\", \"econometrics\", \"us_foreign_policy\"]", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MATH", - "source_data": { - "dataset_name": "MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Equivalent (CoT) on MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.449, - "details": { - "description": "min=0.3, mean=0.449, max=0.548, sum=3.146 (7)", - "tab": "Accuracy", - "MATH - Observed inference time (s)": "{\"description\": \"min=3.871, mean=4.334, max=5.181, sum=30.338 (7)\", \"tab\": \"Efficiency\", \"score\": \"4.333955165715466\"}", - "MATH - # eval": "{\"description\": \"min=30, mean=62.429, max=135, sum=437 (7)\", \"tab\": \"General information\", \"score\": \"62.42857142857143\"}", - "MATH - # train": "{\"description\": \"min=8, mean=8, max=8, sum=56 (7)\", \"tab\": \"General information\", \"score\": \"8.0\"}", - "MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MATH - # prompt tokens": "{\"description\": \"min=906.556, mean=1375.735, max=2449.942, sum=9630.147 (7)\", \"tab\": \"General information\", \"score\": \"1375.7353092779654\"}", - "MATH - # output tokens": "{\"description\": \"min=61.333, mean=74.938, max=97.115, sum=524.566 (7)\", \"tab\": \"General information\", \"score\": \"74.93793702104595\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"algebra\", \"counting_and_probability\", \"geometry\", \"intermediate_algebra\", \"number_theory\", \"prealgebra\", \"precalculus\"]", - "level": "\"1\"", - "use_official_examples": "\"False\"", - "use_chain_of_thought": "\"True\"" - } - } - }, - { - "evaluation_name": "GSM8K", - "source_data": { - "dataset_name": "GSM8K", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on GSM8K", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.615, - "details": { - "description": "min=0.615, mean=0.615, max=0.615, sum=0.615 (1)", - "tab": "Accuracy", - "GSM8K - Observed inference time (s)": "{\"description\": \"min=5.199, mean=5.199, max=5.199, sum=5.199 (1)\", \"tab\": \"Efficiency\", \"score\": \"5.199419307470322\"}", - "GSM8K - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "GSM8K - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "GSM8K - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GSM8K - # prompt tokens": "{\"description\": \"min=938.869, mean=938.869, max=938.869, sum=938.869 (1)\", \"tab\": \"General information\", \"score\": \"938.869\"}", - "GSM8K - # output tokens": "{\"description\": \"min=93.717, mean=93.717, max=93.717, sum=93.717 (1)\", \"tab\": \"General information\", \"score\": \"93.717\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "LegalBench", - "source_data": { - "dataset_name": "LegalBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on LegalBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.622, - "details": { - "description": "min=0.324, mean=0.622, max=0.947, sum=3.11 (5)", - "tab": "Accuracy", - "LegalBench - Observed inference time (s)": "{\"description\": \"min=0.189, mean=0.259, max=0.474, sum=1.297 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.2594051892596125\"}", - "LegalBench - # eval": "{\"description\": \"min=95, mean=409.4, max=1000, sum=2047 (5)\", \"tab\": \"General information\", \"score\": \"409.4\"}", - "LegalBench - # train": "{\"description\": \"min=2.053, mean=4.211, max=5, sum=21.053 (5)\", \"tab\": \"General information\", \"score\": \"4.210612244897959\"}", - "LegalBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "LegalBench - # prompt tokens": "{\"description\": \"min=205.632, mean=907.387, max=3225.32, sum=4536.936 (5)\", \"tab\": \"General information\", \"score\": \"907.3872120499769\"}", - "LegalBench - # output tokens": "{\"description\": \"min=1, mean=1.168, max=1.443, sum=5.838 (5)\", \"tab\": \"General information\", \"score\": \"1.1675708408818857\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"abercrombie\", \"corporate_lobbying\", \"function_of_decision_section\", \"international_citizenship_questions\", \"proa\"]" - } - } - }, - { - "evaluation_name": "MedQA", - "source_data": { - "dataset_name": "MedQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MedQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.531, - "details": { - "description": "min=0.531, mean=0.531, max=0.531, sum=0.531 (1)", - "tab": "Accuracy", - "MedQA - Observed inference time (s)": "{\"description\": \"min=0.228, mean=0.228, max=0.228, sum=0.228 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.22811962975185388\"}", - "MedQA - # eval": "{\"description\": \"min=503, mean=503, max=503, sum=503 (1)\", \"tab\": \"General information\", \"score\": \"503.0\"}", - "MedQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MedQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MedQA - # prompt tokens": "{\"description\": \"min=1038.861, mean=1038.861, max=1038.861, sum=1038.861 (1)\", \"tab\": \"General information\", \"score\": \"1038.8608349900596\"}", - "MedQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WMT 2014", - "source_data": { - "dataset_name": "WMT 2014", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "BLEU-4 on WMT 2014", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.191, - "details": { - "description": "min=0.094, mean=0.191, max=0.227, sum=0.956 (5)", - "tab": "Accuracy", - "WMT 2014 - Observed inference time (s)": "{\"description\": \"min=0.756, mean=0.8, max=0.822, sum=4.0 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.800053899013968\"}", - "WMT 2014 - # eval": "{\"description\": \"min=503, mean=568.8, max=832, sum=2844 (5)\", \"tab\": \"General information\", \"score\": \"568.8\"}", - "WMT 2014 - # train": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "WMT 2014 - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WMT 2014 - # prompt tokens": "{\"description\": \"min=136.93, mean=181.694, max=241.662, sum=908.469 (5)\", \"tab\": \"General information\", \"score\": \"181.69386660804403\"}", - "WMT 2014 - # output tokens": "{\"description\": \"min=23.563, mean=25.117, max=25.652, sum=125.587 (5)\", \"tab\": \"General information\", \"score\": \"25.117336366416882\"}" - } - }, - "generation_config": { - "additional_details": { - "language_pair": "[\"cs-en\", \"de-en\", \"fr-en\", \"hi-en\", \"ru-en\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/openassistant_oasst-rm-2-pythia-6.9b-epoch-1.json b/data/models/openassistant_oasst-rm-2-pythia-6.9b-epoch-1.json deleted file mode 100644 index 34afb9663c74112d98337da6fd83b3594c465f50..0000000000000000000000000000000000000000 --- a/data/models/openassistant_oasst-rm-2-pythia-6.9b-epoch-1.json +++ /dev/null @@ -1,294 +0,0 @@ -{ - "model_info": { - "name": "OpenAssistant/oasst-rm-2-pythia-6.9b-epoch-1", - "id": "OpenAssistant/oasst-rm-2-pythia-6.9b-epoch-1", - "developer": "OpenAssistant", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/OpenAssistant_oasst-rm-2-pythia-6.9b-epoch-1/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.615 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9246 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3728 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5446 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5855 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6801 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "reward-bench-2/OpenAssistant_oasst-rm-2-pythia-6.9b-epoch-1/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2653 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3979 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2875 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.377 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3289 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1535 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.047 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/openassistant_oasst-rm-2.1-pythia-1.4b-epoch-2.5.json b/data/models/openassistant_oasst-rm-2.1-pythia-1.4b-epoch-2.5.json deleted file mode 100644 index d028cad4784e7392a6a614670c5c27a9b8900f31..0000000000000000000000000000000000000000 --- a/data/models/openassistant_oasst-rm-2.1-pythia-1.4b-epoch-2.5.json +++ /dev/null @@ -1,294 +0,0 @@ -{ - "model_info": { - "name": "OpenAssistant/oasst-rm-2.1-pythia-1.4b-epoch-2.5", - "id": "OpenAssistant/oasst-rm-2.1-pythia-1.4b-epoch-2.5", - "developer": "OpenAssistant", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/OpenAssistant_oasst-rm-2.1-pythia-1.4b-epoch-2.5/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6901 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8855 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4868 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6311 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7752 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6533 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "reward-bench-2/OpenAssistant_oasst-rm-2.1-pythia-1.4b-epoch-2.5/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2648 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3179 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2625 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3934 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3244 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2707 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0198 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/openassistant_oasst-sft-1-pythia-12b.json b/data/models/openassistant_oasst-sft-1-pythia-12b.json deleted file mode 100644 index 7c8cda1e3f8716b6144866ab38ce216f02de870b..0000000000000000000000000000000000000000 --- a/data/models/openassistant_oasst-sft-1-pythia-12b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "oasst-sft-1-pythia-12b", - "id": "OpenAssistant/oasst-sft-1-pythia-12b", - "developer": "OpenAssistant", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "GPTNeoXForCausalLM", - "params_billions": "12.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/OpenAssistant_oasst-sft-1-pythia-12b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1055 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3147 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0151 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2576 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3327 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1113 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/openassistant_reward-model-deberta-v3-large-v2.json b/data/models/openassistant_reward-model-deberta-v3-large-v2.json deleted file mode 100644 index cf1ba02f6dd573bf3e0770ff614660b73e92dbb8..0000000000000000000000000000000000000000 --- a/data/models/openassistant_reward-model-deberta-v3-large-v2.json +++ /dev/null @@ -1,294 +0,0 @@ -{ - "model_info": { - "name": "OpenAssistant/reward-model-deberta-v3-large-v2", - "id": "OpenAssistant/reward-model-deberta-v3-large-v2", - "developer": "OpenAssistant", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/OpenAssistant_reward-model-deberta-v3-large-v2/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.32 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3853 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2687 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5027 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3667 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2768 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.12 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "reward-bench/OpenAssistant_reward-model-deberta-v3-large-v2/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6126 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8939 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4518 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7338 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3855 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5836 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/openbmb_eurus-7b-kto.json b/data/models/openbmb_eurus-7b-kto.json deleted file mode 100644 index 6c8e8a844dc507c775136746fc2b2d8d65f433a4..0000000000000000000000000000000000000000 --- a/data/models/openbmb_eurus-7b-kto.json +++ /dev/null @@ -1,144 +0,0 @@ -{ - "model_info": { - "name": "openbmb/Eurus-7b-kto", - "id": "openbmb/Eurus-7b-kto", - "developer": "openbmb", - "additional_details": { - "model_type": "DPO" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/openbmb_Eurus-7b-kto/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.69 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9525 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5373 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6054 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7467 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5261 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/openbmb_eurus-rm-7b.json b/data/models/openbmb_eurus-rm-7b.json deleted file mode 100644 index e1154a660c89f219cec7ec844602d9d8fdfa08ad..0000000000000000000000000000000000000000 --- a/data/models/openbmb_eurus-rm-7b.json +++ /dev/null @@ -1,294 +0,0 @@ -{ - "model_info": { - "name": "openbmb/Eurus-RM-7b", - "id": "openbmb/Eurus-RM-7b", - "developer": "openbmb", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/openbmb_Eurus-RM-7b/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5806 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3438 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5683 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6267 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7475 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5972 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "reward-bench/openbmb_Eurus-RM-7b/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8159 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9804 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6557 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8135 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8633 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7172 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/openbmb_minicpm-2b-dpo-fp32.json b/data/models/openbmb_minicpm-2b-dpo-fp32.json deleted file mode 100644 index 99c6f300c8a1f251fc6ba7ddcf6291c04076a58f..0000000000000000000000000000000000000000 --- a/data/models/openbmb_minicpm-2b-dpo-fp32.json +++ /dev/null @@ -1,144 +0,0 @@ -{ - "model_info": { - "name": "openbmb/MiniCPM-2B-dpo-fp32", - "id": "openbmb/MiniCPM-2B-dpo-fp32", - "developer": "openbmb", - "additional_details": { - "model_type": "DPO" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/openbmb_MiniCPM-2B-dpo-fp32/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.673 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8911 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4934 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.573 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8233 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4958 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/openbmb_minicpm-s-1b-sft-llama-format.json b/data/models/openbmb_minicpm-s-1b-sft-llama-format.json deleted file mode 100644 index eb9aec158bdbf26937e40d42757981192e608787..0000000000000000000000000000000000000000 --- a/data/models/openbmb_minicpm-s-1b-sft-llama-format.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MiniCPM-S-1B-sft-llama-format", - "id": "openbmb/MiniCPM-S-1B-sft-llama-format", - "developer": "openbmb", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/openbmb_MiniCPM-S-1B-sft-llama-format/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3329 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3049 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.031 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.271 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3317 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1858 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/openbmb_minicpm3-4b-fc-fc.json b/data/models/openbmb_minicpm3-4b-fc-fc.json deleted file mode 100644 index 57c88f59a89787f0f9a280301770dc5997a206b5..0000000000000000000000000000000000000000 --- a/data/models/openbmb_minicpm3-4b-fc-fc.json +++ /dev/null @@ -1,851 +0,0 @@ -{ - "model_info": { - "name": "MiniCPM3-4B-FC (FC)", - "id": "openbmb/minicpm3-4b-fc-fc", - "developer": "openbmb", - "additional_details": { - "raw_model_name": "MiniCPM3-4B-FC (FC)", - "organization": "openbmb", - "license": "Apache-2.0", - "mode": "FC", - "model_link": "https://huggingface.co/openbmb/MiniCPM3-4B" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/openbmb/minicpm3-4b-fc-fc/1775236112.411131", - "retrieved_timestamp": "1775236112.411131", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 86.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 25.55 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 54.05 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 118.62 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 143.98 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 388.67 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 81.75 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 70.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 92.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 84.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 80.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 65.21 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 73.26 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 63.53 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 50.0 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 62.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 3.88 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 6.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 2.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 4.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 2.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 12.04 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 9.68 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 15.48 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 10.97 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 68.75 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 72.84 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/openbmb_minicpm3-4b-prompt.json b/data/models/openbmb_minicpm3-4b-prompt.json deleted file mode 100644 index fe0d6cf71ac3f4f366637d7a8640af0c0cce99c9..0000000000000000000000000000000000000000 --- a/data/models/openbmb_minicpm3-4b-prompt.json +++ /dev/null @@ -1,905 +0,0 @@ -{ - "model_info": { - "name": "MiniCPM3-4B (Prompt)", - "id": "openbmb/minicpm3-4b-prompt", - "developer": "openbmb", - "additional_details": { - "raw_model_name": "MiniCPM3-4B (Prompt)", - "organization": "openbmb", - "license": "Apache-2.0", - "mode": "Prompt", - "model_link": "https://huggingface.co/openbmb/MiniCPM3-4B" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/openbmb/minicpm3-4b-prompt/1775236112.417061", - "retrieved_timestamp": "1775236112.417061", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 97.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 22.08 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 29.83 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 31.18 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 35.61 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 102.02 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 70.54 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 66.17 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 77.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 70.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 69.0 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 43.15 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 47.67 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 42.17 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 43.75 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 37.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 3.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 4.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 4.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 2.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 3.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 2.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 2.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 2.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 9.46 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 8.39 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 10.32 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 9.68 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 56.25 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 73.71 - } - }, - { - "evaluation_result_id": "bfcl.format_sensitivity.max_delta::max_delta", - "evaluation_name": "bfcl.format_sensitivity.max_delta", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.format_sensitivity.max_delta", - "metric_name": "Format sensitivity max delta", - "metric_kind": "difference", - "metric_unit": "percentage_points", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Format Sensitivity Max Delta" - } - }, - "score_details": { - "score": 68.0 - } - }, - { - "evaluation_result_id": "bfcl.format_sensitivity.stddev::stddev", - "evaluation_name": "bfcl.format_sensitivity.stddev", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.format_sensitivity.stddev", - "metric_name": "Format sensitivity standard deviation", - "metric_kind": "difference", - "metric_unit": "percentage_points", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Format Sensitivity Standard Deviation" - } - }, - "score_details": { - "score": 16.55 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/openbmb_ultrarm-13b.json b/data/models/openbmb_ultrarm-13b.json deleted file mode 100644 index 84bdd483e26f91976f0250093c6ea14b4f0ff97c..0000000000000000000000000000000000000000 --- a/data/models/openbmb_ultrarm-13b.json +++ /dev/null @@ -1,294 +0,0 @@ -{ - "model_info": { - "name": "openbmb/UltraRM-13b", - "id": "openbmb/UltraRM-13b", - "developer": "openbmb", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/openbmb_UltraRM-13b/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6903 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9637 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5548 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5986 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6244 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7294 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "reward-bench-2/openbmb_UltraRM-13b/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4683 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5063 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3312 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5519 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5089 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6081 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3036 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/openbuddy_openbuddy-falcon3-10b-v24.2-131k.json b/data/models/openbuddy_openbuddy-falcon3-10b-v24.2-131k.json deleted file mode 100644 index 1d5b6db87668e33b30a81d3c897c9e094870b150..0000000000000000000000000000000000000000 --- a/data/models/openbuddy_openbuddy-falcon3-10b-v24.2-131k.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "openbuddy-falcon3-10b-v24.2-131k", - "id": "OpenBuddy/openbuddy-falcon3-10b-v24.2-131k", - "developer": "OpenBuddy", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "10.34" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/OpenBuddy_openbuddy-falcon3-10b-v24.2-131k/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5086 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6004 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.213 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2995 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4186 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3834 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/openbuddy_openbuddy-llama3-70b-v21.2-32k.json b/data/models/openbuddy_openbuddy-llama3-70b-v21.2-32k.json deleted file mode 100644 index 4a6d7799b09e5defc1ea96ec05e3a66f43085e3b..0000000000000000000000000000000000000000 --- a/data/models/openbuddy_openbuddy-llama3-70b-v21.2-32k.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "openbuddy-llama3-70b-v21.2-32k", - "id": "OpenBuddy/openbuddy-llama3-70b-v21.2-32k", - "developer": "OpenBuddy", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "70.554" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/OpenBuddy_openbuddy-llama3-70b-v21.2-32k/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.701 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6507 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2032 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3423 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.458 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4832 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/openbuddy_openbuddy-llama3-8b-v21.1-8k.json b/data/models/openbuddy_openbuddy-llama3-8b-v21.1-8k.json deleted file mode 100644 index 8f605973362e2fc7ccc9c66f020e0cc60e86c0e9..0000000000000000000000000000000000000000 --- a/data/models/openbuddy_openbuddy-llama3-8b-v21.1-8k.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "openbuddy-llama3-8b-v21.1-8k", - "id": "OpenBuddy/openbuddy-llama3-8b-v21.1-8k", - "developer": "OpenBuddy", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/OpenBuddy_openbuddy-llama3-8b-v21.1-8k/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.557 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4788 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0431 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.271 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3988 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2955 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/openbuddy_openbuddy-llama3-8b-v21.2-32k.json b/data/models/openbuddy_openbuddy-llama3-8b-v21.2-32k.json deleted file mode 100644 index 0224fc2ee4c69d87f81bbddf79034b71f966db08..0000000000000000000000000000000000000000 --- a/data/models/openbuddy_openbuddy-llama3-8b-v21.2-32k.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "openbuddy-llama3-8b-v21.2-32k", - "id": "OpenBuddy/openbuddy-llama3-8b-v21.2-32k", - "developer": "OpenBuddy", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/OpenBuddy_openbuddy-llama3-8b-v21.2-32k/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6192 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4856 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0785 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2794 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3779 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3299 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/openbuddy_openbuddy-llama3.1-70b-v22.1-131k.json b/data/models/openbuddy_openbuddy-llama3.1-70b-v22.1-131k.json deleted file mode 100644 index d234a8aa5d69dfe9228ac442596624b150a21c81..0000000000000000000000000000000000000000 --- a/data/models/openbuddy_openbuddy-llama3.1-70b-v22.1-131k.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "openbuddy-llama3.1-70b-v22.1-131k", - "id": "OpenBuddy/openbuddy-llama3.1-70b-v22.1-131k", - "developer": "OpenBuddy", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "70.554" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/OpenBuddy_openbuddy-llama3.1-70b-v22.1-131k/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7333 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6698 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.395 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.375 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.463 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5304 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/openbuddy_openbuddy-llama3.1-8b-v22.2-131k.json b/data/models/openbuddy_openbuddy-llama3.1-8b-v22.2-131k.json deleted file mode 100644 index 126e3b7da2c4e5d7430e72a7b6de0521d2dab998..0000000000000000000000000000000000000000 --- a/data/models/openbuddy_openbuddy-llama3.1-8b-v22.2-131k.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "openbuddy-llama3.1-8b-v22.2-131k", - "id": "OpenBuddy/openbuddy-llama3.1-8b-v22.2-131k", - "developer": "OpenBuddy", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/OpenBuddy_openbuddy-llama3.1-8b-v22.2-131k/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6657 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5007 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1148 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2794 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4081 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.331 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/openbuddy_openbuddy-llama3.1-8b-v22.3-131k.json b/data/models/openbuddy_openbuddy-llama3.1-8b-v22.3-131k.json deleted file mode 100644 index 4aafe4454956d3ac2d5a1af847aab74315503b3f..0000000000000000000000000000000000000000 --- a/data/models/openbuddy_openbuddy-llama3.1-8b-v22.3-131k.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "openbuddy-llama3.1-8b-v22.3-131k", - "id": "OpenBuddy/openbuddy-llama3.1-8b-v22.3-131k", - "developer": "OpenBuddy", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/OpenBuddy_openbuddy-llama3.1-8b-v22.3-131k/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5997 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5066 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1208 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2794 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4015 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3277 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/openbuddy_openbuddy-llama3.2-1b-v23.1-131k.json b/data/models/openbuddy_openbuddy-llama3.2-1b-v23.1-131k.json deleted file mode 100644 index 0607285022f72c63a5e0c680cbb1f57970a57c07..0000000000000000000000000000000000000000 --- a/data/models/openbuddy_openbuddy-llama3.2-1b-v23.1-131k.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "openbuddy-llama3.2-1b-v23.1-131k", - "id": "OpenBuddy/openbuddy-llama3.2-1b-v23.1-131k", - "developer": "OpenBuddy", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.498" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/OpenBuddy_openbuddy-llama3.2-1b-v23.1-131k/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.359 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3267 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0249 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2584 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3342 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.184 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/openbuddy_openbuddy-llama3.2-3b-v23.2-131k.json b/data/models/openbuddy_openbuddy-llama3.2-3b-v23.2-131k.json deleted file mode 100644 index 40c2b30a05fcde41c419fc0058add69e934a4656..0000000000000000000000000000000000000000 --- a/data/models/openbuddy_openbuddy-llama3.2-3b-v23.2-131k.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "openbuddy-llama3.2-3b-v23.2-131k", - "id": "OpenBuddy/openbuddy-llama3.2-3b-v23.2-131k", - "developer": "OpenBuddy", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.607" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/OpenBuddy_openbuddy-llama3.2-3b-v23.2-131k/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4319 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4073 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0264 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.276 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3263 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2479 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/openbuddy_openbuddy-llama3.3-70b-v24.1-131k.json b/data/models/openbuddy_openbuddy-llama3.3-70b-v24.1-131k.json deleted file mode 100644 index 005b3c94d09eec76434fc676c7193058793fddd0..0000000000000000000000000000000000000000 --- a/data/models/openbuddy_openbuddy-llama3.3-70b-v24.1-131k.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "openbuddy-llama3.3-70b-v24.1-131k", - "id": "OpenBuddy/openbuddy-llama3.3-70b-v24.1-131k", - "developer": "OpenBuddy", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "70.554" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/OpenBuddy_openbuddy-llama3.3-70b-v24.1-131k/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8121 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6858 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4411 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4346 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4869 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5327 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/openbuddy_openbuddy-mixtral-7bx8-v18.1-32k.json b/data/models/openbuddy_openbuddy-mixtral-7bx8-v18.1-32k.json deleted file mode 100644 index 9fb3798dbc20a4b4addfcb1cee126a8ced39f718..0000000000000000000000000000000000000000 --- a/data/models/openbuddy_openbuddy-mixtral-7bx8-v18.1-32k.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "openbuddy-mixtral-7bx8-v18.1-32k", - "id": "OpenBuddy/openbuddy-mixtral-7bx8-v18.1-32k", - "developer": "OpenBuddy", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MixtralForCausalLM", - "params_billions": "46.741" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/OpenBuddy_openbuddy-mixtral-7bx8-v18.1-32k/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5493 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4656 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.108 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3045 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3831 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3804 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/openbuddy_openbuddy-nemotron-70b-v23.1-131k.json b/data/models/openbuddy_openbuddy-nemotron-70b-v23.1-131k.json deleted file mode 100644 index fd2b7bc2c6836c6d77e6e1557a160f0e85d0d4a3..0000000000000000000000000000000000000000 --- a/data/models/openbuddy_openbuddy-nemotron-70b-v23.1-131k.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "openbuddy-nemotron-70b-v23.1-131k", - "id": "OpenBuddy/openbuddy-nemotron-70b-v23.1-131k", - "developer": "OpenBuddy", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "70.554" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/OpenBuddy_openbuddy-nemotron-70b-v23.1-131k/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7555 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6749 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.321 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3633 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4538 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5175 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/openbuddy_openbuddy-nemotron-70b-v23.2-131k.json b/data/models/openbuddy_openbuddy-nemotron-70b-v23.2-131k.json deleted file mode 100644 index 43d9f7629c27220d8a574be7937697ea95ccb7c1..0000000000000000000000000000000000000000 --- a/data/models/openbuddy_openbuddy-nemotron-70b-v23.2-131k.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "openbuddy-nemotron-70b-v23.2-131k", - "id": "OpenBuddy/openbuddy-nemotron-70b-v23.2-131k", - "developer": "OpenBuddy", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "70.554" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/OpenBuddy_openbuddy-nemotron-70b-v23.2-131k/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7227 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6705 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3157 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3599 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4696 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5121 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/openbuddy_openbuddy-qwen2.5llamaify-14b-v23.1-200k.json b/data/models/openbuddy_openbuddy-qwen2.5llamaify-14b-v23.1-200k.json deleted file mode 100644 index 6d78f05974e297f3dccfa64db226428fd232351c..0000000000000000000000000000000000000000 --- a/data/models/openbuddy_openbuddy-qwen2.5llamaify-14b-v23.1-200k.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "openbuddy-qwen2.5llamaify-14b-v23.1-200k", - "id": "OpenBuddy/openbuddy-qwen2.5llamaify-14b-v23.1-200k", - "developer": "OpenBuddy", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/OpenBuddy_openbuddy-qwen2.5llamaify-14b-v23.1-200k/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6309 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6013 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2538 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3331 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.424 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4673 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/openbuddy_openbuddy-qwen2.5llamaify-14b-v23.3-200k.json b/data/models/openbuddy_openbuddy-qwen2.5llamaify-14b-v23.3-200k.json deleted file mode 100644 index 5f54f83c59931504641b2ff93a88838e62f6f112..0000000000000000000000000000000000000000 --- a/data/models/openbuddy_openbuddy-qwen2.5llamaify-14b-v23.3-200k.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "openbuddy-qwen2.5llamaify-14b-v23.3-200k", - "id": "OpenBuddy/openbuddy-qwen2.5llamaify-14b-v23.3-200k", - "developer": "OpenBuddy", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/OpenBuddy_openbuddy-qwen2.5llamaify-14b-v23.3-200k/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6131 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6081 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2311 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3272 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4346 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4795 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/openbuddy_openbuddy-qwen2.5llamaify-7b-v23.1-200k.json b/data/models/openbuddy_openbuddy-qwen2.5llamaify-7b-v23.1-200k.json deleted file mode 100644 index d634c936fd96c8dba3c4b316606254d6a7b5a689..0000000000000000000000000000000000000000 --- a/data/models/openbuddy_openbuddy-qwen2.5llamaify-7b-v23.1-200k.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "openbuddy-qwen2.5llamaify-7b-v23.1-200k", - "id": "OpenBuddy/openbuddy-qwen2.5llamaify-7b-v23.1-200k", - "developer": "OpenBuddy", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "7.615" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/OpenBuddy_openbuddy-qwen2.5llamaify-7b-v23.1-200k/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5673 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5509 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1888 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3146 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4363 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3948 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/openbuddy_openbuddy-qwq-32b-v24.1-200k.json b/data/models/openbuddy_openbuddy-qwq-32b-v24.1-200k.json deleted file mode 100644 index fa2d0a6b6a5fea9caea4dab7234ddb2f9106a674..0000000000000000000000000000000000000000 --- a/data/models/openbuddy_openbuddy-qwq-32b-v24.1-200k.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "openbuddy-qwq-32b-v24.1-200k", - "id": "OpenBuddy/openbuddy-qwq-32b-v24.1-200k", - "developer": "OpenBuddy", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "32.764" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/OpenBuddy_openbuddy-qwq-32b-v24.1-200k/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5937 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6798 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3739 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3809 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4849 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.549 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/openbuddy_openbuddy-qwq-32b-v24.2-200k.json b/data/models/openbuddy_openbuddy-qwq-32b-v24.2-200k.json deleted file mode 100644 index e2a93c8779cc730914f2f8669d93e856637c8541..0000000000000000000000000000000000000000 --- a/data/models/openbuddy_openbuddy-qwq-32b-v24.2-200k.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "openbuddy-qwq-32b-v24.2-200k", - "id": "OpenBuddy/openbuddy-qwq-32b-v24.2-200k", - "developer": "OpenBuddy", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "32.764" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/OpenBuddy_openbuddy-qwq-32b-v24.2-200k/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.597 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6772 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3776 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3767 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4718 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5446 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/openbuddy_openbuddy-yi1.5-34b-v21.3-32k.json b/data/models/openbuddy_openbuddy-yi1.5-34b-v21.3-32k.json deleted file mode 100644 index 61f186fdb2c41bb3d06160c3224ad9c3f7814c70..0000000000000000000000000000000000000000 --- a/data/models/openbuddy_openbuddy-yi1.5-34b-v21.3-32k.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "openbuddy-yi1.5-34b-v21.3-32k", - "id": "OpenBuddy/openbuddy-yi1.5-34b-v21.3-32k", - "developer": "OpenBuddy", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "34.407" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/OpenBuddy_openbuddy-yi1.5-34b-v21.3-32k/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.542 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6163 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1782 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.349 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4439 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4599 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/openbuddy_openbuddy-zero-14b-v22.3-32k.json b/data/models/openbuddy_openbuddy-zero-14b-v22.3-32k.json deleted file mode 100644 index 196e27101a1ee4ceb3be31a99cbc71d46b4b2f40..0000000000000000000000000000000000000000 --- a/data/models/openbuddy_openbuddy-zero-14b-v22.3-32k.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "openbuddy-zero-14b-v22.3-32k", - "id": "OpenBuddy/openbuddy-zero-14b-v22.3-32k", - "developer": "OpenBuddy", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.022" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/OpenBuddy_openbuddy-zero-14b-v22.3-32k/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3753 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.486 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0937 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.307 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4166 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3187 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/openbuddy_openbuddy-zero-3b-v21.2-32k.json b/data/models/openbuddy_openbuddy-zero-3b-v21.2-32k.json deleted file mode 100644 index cbb86db257084ffe4c304c28995a10c4b358c4ff..0000000000000000000000000000000000000000 --- a/data/models/openbuddy_openbuddy-zero-3b-v21.2-32k.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "openbuddy-zero-3b-v21.2-32k", - "id": "OpenBuddy/openbuddy-zero-3b-v21.2-32k", - "developer": "OpenBuddy", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "4.769" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/OpenBuddy_openbuddy-zero-3b-v21.2-32k/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3802 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3935 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0189 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2601 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3566 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2034 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/openbuddy_openbuddy-zero-56b-v21.2-32k.json b/data/models/openbuddy_openbuddy-zero-56b-v21.2-32k.json deleted file mode 100644 index d068054724764e22d08c1960eb57877a7417f256..0000000000000000000000000000000000000000 --- a/data/models/openbuddy_openbuddy-zero-56b-v21.2-32k.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "openbuddy-zero-56b-v21.2-32k", - "id": "OpenBuddy/openbuddy-zero-56b-v21.2-32k", - "developer": "OpenBuddy", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "56.707" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/OpenBuddy_openbuddy-zero-56b-v21.2-32k/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5057 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6128 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1624 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.318 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4305 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4399 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/openchat_openchat-3.5-0106.json b/data/models/openchat_openchat-3.5-0106.json deleted file mode 100644 index eace695372b8d9ce97d73f35e23496646c25d3b5..0000000000000000000000000000000000000000 --- a/data/models/openchat_openchat-3.5-0106.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "openchat-3.5-0106", - "id": "openchat/openchat-3.5-0106", - "developer": "openchat", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/openchat_openchat-3.5-0106/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5967 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4617 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0763 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3079 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4254 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3291 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/openchat_openchat-3.5-1210.json b/data/models/openchat_openchat-3.5-1210.json deleted file mode 100644 index a33d10117f8ad3e70a5c5d5f1c071c2198007616..0000000000000000000000000000000000000000 --- a/data/models/openchat_openchat-3.5-1210.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "openchat-3.5-1210", - "id": "openchat/openchat-3.5-1210", - "developer": "openchat", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/openchat_openchat-3.5-1210/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6037 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4535 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0785 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3012 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4414 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3142 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/openchat_openchat-3.6-8b-20240522.json b/data/models/openchat_openchat-3.6-8b-20240522.json deleted file mode 100644 index e0e72b64ec63ac85c4278f88e01bef3f397fa9b4..0000000000000000000000000000000000000000 --- a/data/models/openchat_openchat-3.6-8b-20240522.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "openchat-3.6-8b-20240522", - "id": "openchat/openchat-3.6-8b-20240522", - "developer": "openchat", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/openchat_openchat-3.6-8b-20240522/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5343 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5338 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0997 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.318 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3999 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3229 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/openchat_openchat_3.5.json b/data/models/openchat_openchat_3.5.json deleted file mode 100644 index 5cfa0dd0448391d288a2ffaec3db7d34049432fd..0000000000000000000000000000000000000000 --- a/data/models/openchat_openchat_3.5.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "openchat_3.5", - "id": "openchat/openchat_3.5", - "developer": "openchat", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/openchat_openchat_3.5/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5931 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4426 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0725 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2987 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4229 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3153 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/openchat_openchat_v3.2.json b/data/models/openchat_openchat_v3.2.json deleted file mode 100644 index e14665c51dc4d163fc18b351a753b21209b500e4..0000000000000000000000000000000000000000 --- a/data/models/openchat_openchat_v3.2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "openchat_v3.2", - "id": "openchat/openchat_v3.2", - "developer": "openchat", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "13.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/openchat_openchat_v3.2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2981 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4331 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0128 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2701 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4336 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2422 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/openchat_openchat_v3.2_super.json b/data/models/openchat_openchat_v3.2_super.json deleted file mode 100644 index 7e4016240a7ae73970a4af856e6d41436c6c50eb..0000000000000000000000000000000000000000 --- a/data/models/openchat_openchat_v3.2_super.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "openchat_v3.2_super", - "id": "openchat/openchat_v3.2_super", - "developer": "openchat", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "13.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/openchat_openchat_v3.2_super/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2862 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4221 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0211 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2643 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4161 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2425 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/opencompass_compassjudger-1-1.5b-instruct.json b/data/models/opencompass_compassjudger-1-1.5b-instruct.json deleted file mode 100644 index 0c2e41bbc0d6e3714203943e5b65babd3e9d3532..0000000000000000000000000000000000000000 --- a/data/models/opencompass_compassjudger-1-1.5b-instruct.json +++ /dev/null @@ -1,126 +0,0 @@ -{ - "model_info": { - "name": "opencompass/CompassJudger-1-1.5B-Instruct", - "id": "opencompass/CompassJudger-1-1.5B-Instruct", - "developer": "opencompass", - "additional_details": { - "model_type": "Generative" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/opencompass_CompassJudger-1-1.5B-Instruct/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7344 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9637 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4923 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7818 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6999 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/opencompass_compassjudger-1-14b-instruct.json b/data/models/opencompass_compassjudger-1-14b-instruct.json deleted file mode 100644 index c66df4ac356fcd0c5a2ece64678424e91f2bb1c8..0000000000000000000000000000000000000000 --- a/data/models/opencompass_compassjudger-1-14b-instruct.json +++ /dev/null @@ -1,126 +0,0 @@ -{ - "model_info": { - "name": "opencompass/CompassJudger-1-14B-Instruct", - "id": "opencompass/CompassJudger-1-14B-Instruct", - "developer": "opencompass", - "additional_details": { - "model_type": "Generative" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/opencompass_CompassJudger-1-14B-Instruct/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8409 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9749 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6228 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8392 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9268 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/opencompass_compassjudger-1-32b-instruct.json b/data/models/opencompass_compassjudger-1-32b-instruct.json deleted file mode 100644 index 58d715bbff6927eb772856cda0563411488b7d4f..0000000000000000000000000000000000000000 --- a/data/models/opencompass_compassjudger-1-32b-instruct.json +++ /dev/null @@ -1,126 +0,0 @@ -{ - "model_info": { - "name": "opencompass/CompassJudger-1-32B-Instruct", - "id": "opencompass/CompassJudger-1-32B-Instruct", - "developer": "opencompass", - "additional_details": { - "model_type": "Generative" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/opencompass_CompassJudger-1-32B-Instruct/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8522 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9804 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6513 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8527 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9244 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/opencompass_compassjudger-1-7b-instruct.json b/data/models/opencompass_compassjudger-1-7b-instruct.json deleted file mode 100644 index c7360ed384a37514cae760481f2a8676fcee18b1..0000000000000000000000000000000000000000 --- a/data/models/opencompass_compassjudger-1-7b-instruct.json +++ /dev/null @@ -1,126 +0,0 @@ -{ - "model_info": { - "name": "opencompass/CompassJudger-1-7B-Instruct", - "id": "opencompass/CompassJudger-1-7B-Instruct", - "developer": "opencompass", - "additional_details": { - "model_type": "Generative" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/opencompass_CompassJudger-1-7B-Instruct/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8317 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9777 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6096 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8446 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8948 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/opengenerativeai_bifrost-14b.json b/data/models/opengenerativeai_bifrost-14b.json deleted file mode 100644 index f6730224d2a810a12ce1c32ef0fbf15ecf70a2cc..0000000000000000000000000000000000000000 --- a/data/models/opengenerativeai_bifrost-14b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Bifrost-14B", - "id": "OpenGenerativeAI/Bifrost-14B", - "developer": "OpenGenerativeAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/OpenGenerativeAI_Bifrost-14B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6615 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6845 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2356 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3792 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4624 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5074 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/opengenerativeai_bifrost.json b/data/models/opengenerativeai_bifrost.json deleted file mode 100644 index dae6549dfe265b472787fb9ec3408aebc155239f..0000000000000000000000000000000000000000 --- a/data/models/opengenerativeai_bifrost.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Bifrost", - "id": "OpenGenerativeAI/Bifrost", - "developer": "OpenGenerativeAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/OpenGenerativeAI_Bifrost/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6348 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6849 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2545 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3683 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4598 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.516 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/openleecher_llama3-8b-lima.json b/data/models/openleecher_llama3-8b-lima.json deleted file mode 100644 index 9f1d375cd1cd23e7710669def9510b4c7b18343b..0000000000000000000000000000000000000000 --- a/data/models/openleecher_llama3-8b-lima.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "llama3-8b-lima", - "id": "OpenLeecher/llama3-8b-lima", - "developer": "OpenLeecher", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/OpenLeecher_llama3-8b-lima/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4371 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4296 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0506 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2383 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3713 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2626 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/openllm-france_lucie-7b-instruct-human-data.json b/data/models/openllm-france_lucie-7b-instruct-human-data.json deleted file mode 100644 index 38c3e5d23e3e700dbde0ad665218b051710dba05..0000000000000000000000000000000000000000 --- a/data/models/openllm-france_lucie-7b-instruct-human-data.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Lucie-7B-Instruct-human-data", - "id": "OpenLLM-France/Lucie-7B-Instruct-human-data", - "developer": "OpenLLM-France", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "6.707" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/OpenLLM-France_Lucie-7B-Instruct-human-data/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2946 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3284 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0219 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2752 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3729 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.143 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/openllm-france_lucie-7b-instruct-v1.1.json b/data/models/openllm-france_lucie-7b-instruct-v1.1.json deleted file mode 100644 index e3439ae647e952188234418b415674c67a79778d..0000000000000000000000000000000000000000 --- a/data/models/openllm-france_lucie-7b-instruct-v1.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Lucie-7B-Instruct-v1.1", - "id": "OpenLLM-France/Lucie-7B-Instruct-v1.1", - "developer": "OpenLLM-France", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "6.707" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/OpenLLM-France_Lucie-7B-Instruct-v1.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3039 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3816 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0317 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2819 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.375 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1864 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/openllm-france_lucie-7b-instruct.json b/data/models/openllm-france_lucie-7b-instruct.json deleted file mode 100644 index 18e031aba61a6633afeb16862e68e21bf5084b7e..0000000000000000000000000000000000000000 --- a/data/models/openllm-france_lucie-7b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Lucie-7B-Instruct", - "id": "OpenLLM-France/Lucie-7B-Instruct", - "developer": "OpenLLM-France", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "6.707" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/OpenLLM-France_Lucie-7B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2796 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3254 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0166 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2794 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3662 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1556 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/openllm-france_lucie-7b.json b/data/models/openllm-france_lucie-7b.json deleted file mode 100644 index 037dc943d0b18be379b6ddce5f63c4bc1d61a053..0000000000000000000000000000000000000000 --- a/data/models/openllm-france_lucie-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Lucie-7B", - "id": "OpenLLM-France/Lucie-7B", - "developer": "OpenLLM-France", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "6.707" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/OpenLLM-France_Lucie-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2496 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3492 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0144 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2727 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3923 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1498 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/openscholar_llama-3.1_openscholar-8b.json b/data/models/openscholar_llama-3.1_openscholar-8b.json deleted file mode 100644 index 49d5345ffe0199f24db612b9f5a7410049784a39..0000000000000000000000000000000000000000 --- a/data/models/openscholar_llama-3.1_openscholar-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.1_OpenScholar-8B", - "id": "OpenScholar/Llama-3.1_OpenScholar-8B", - "developer": "OpenScholar", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/OpenScholar_Llama-3.1_OpenScholar-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6064 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5208 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1654 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2819 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4275 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3708 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/orai-nlp_llama-eus-8b.json b/data/models/orai-nlp_llama-eus-8b.json deleted file mode 100644 index a18162bc713e60cac0f28eded6b020900e57a5d5..0000000000000000000000000000000000000000 --- a/data/models/orai-nlp_llama-eus-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-eus-8B", - "id": "orai-nlp/Llama-eus-8B", - "developer": "orai-nlp", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/orai-nlp_Llama-eus-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2161 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4418 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0468 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2894 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3919 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3058 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/orenguteng_llama-3.1-8b-lexi-uncensored-v2.json b/data/models/orenguteng_llama-3.1-8b-lexi-uncensored-v2.json deleted file mode 100644 index 1d7602762f2a5b741cb009962885af754a5550eb..0000000000000000000000000000000000000000 --- a/data/models/orenguteng_llama-3.1-8b-lexi-uncensored-v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.1-8B-Lexi-Uncensored-V2", - "id": "Orenguteng/Llama-3.1-8B-Lexi-Uncensored-V2", - "developer": "Orenguteng", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Orenguteng_Llama-3.1-8B-Lexi-Uncensored-V2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7792 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5084 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1971 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2827 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3843 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3781 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/orenguteng_llama-3.1-8b-lexi-uncensored.json b/data/models/orenguteng_llama-3.1-8b-lexi-uncensored.json deleted file mode 100644 index 1ebc9097765996a4bfc2825f3e34a0bc0aa0de1e..0000000000000000000000000000000000000000 --- a/data/models/orenguteng_llama-3.1-8b-lexi-uncensored.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.1-8B-Lexi-Uncensored", - "id": "Orenguteng/Llama-3.1-8B-Lexi-Uncensored", - "developer": "Orenguteng", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Orenguteng_Llama-3.1-8B-Lexi-Uncensored/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7777 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5057 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1571 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2718 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3871 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.379 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/orion-zhen_phi-4-abliterated.json b/data/models/orion-zhen_phi-4-abliterated.json deleted file mode 100644 index 3b8fcf966b041d4f3a66b4c79aa75d5f0a2613e1..0000000000000000000000000000000000000000 --- a/data/models/orion-zhen_phi-4-abliterated.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "phi-4-abliterated", - "id": "Orion-zhen/phi-4-abliterated", - "developer": "Orion-zhen", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Phi3ForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Orion-zhen_phi-4-abliterated/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0576 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6698 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3021 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4044 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5006 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5292 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/orion-zhen_qwen2.5-7b-instruct-uncensored.json b/data/models/orion-zhen_qwen2.5-7b-instruct-uncensored.json deleted file mode 100644 index 774e4055c448d19261f3ebfb56c3aee133ecd35c..0000000000000000000000000000000000000000 --- a/data/models/orion-zhen_qwen2.5-7b-instruct-uncensored.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-7B-Instruct-Uncensored", - "id": "Orion-zhen/Qwen2.5-7B-Instruct-Uncensored", - "developer": "Orion-zhen", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Orion-zhen_Qwen2.5-7B-Instruct-Uncensored/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7204 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5474 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4773 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3029 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4361 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4427 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/oxyapi_oxy-1-small.json b/data/models/oxyapi_oxy-1-small.json deleted file mode 100644 index 17a18559c70a379cc128c0160f471f872c782053..0000000000000000000000000000000000000000 --- a/data/models/oxyapi_oxy-1-small.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "oxy-1-small", - "id": "oxyapi/oxy-1-small", - "developer": "oxyapi", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/oxyapi_oxy-1-small/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6245 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5885 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3603 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3716 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4487 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5001 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ozone-ai_0x-lite.json b/data/models/ozone-ai_0x-lite.json deleted file mode 100644 index 7bd301bcd39d9bebb36794fabc025d1aed6ce6a5..0000000000000000000000000000000000000000 --- a/data/models/ozone-ai_0x-lite.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "0x-lite", - "id": "ozone-ai/0x-lite", - "developer": "ozone-ai", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ozone-ai_0x-lite/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.774 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6341 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5045 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3196 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4221 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5184 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ozone-research_chirp-01.json b/data/models/ozone-research_chirp-01.json deleted file mode 100644 index ee3863209713e3c525a7ceb7d0529d7499922fd0..0000000000000000000000000000000000000000 --- a/data/models/ozone-research_chirp-01.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Chirp-01", - "id": "ozone-research/Chirp-01", - "developer": "ozone-research", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.086" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ozone-research_Chirp-01/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6348 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.465 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3467 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2718 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4487 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3508 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/p0x0_astra-v1-12b.json b/data/models/p0x0_astra-v1-12b.json deleted file mode 100644 index a7dfd9c042c8f18d53891dbc048de7bf4602a68d..0000000000000000000000000000000000000000 --- a/data/models/p0x0_astra-v1-12b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Astra-v1-12B", - "id": "P0x0/Astra-v1-12B", - "developer": "P0x0", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/P0x0_Astra-v1-12B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2806 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5215 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1133 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3138 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4052 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3461 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/paloalma_ece-tw3-jrgl-v1.json b/data/models/paloalma_ece-tw3-jrgl-v1.json deleted file mode 100644 index ad2297a86dc0e9b6f683768371cab33a7a04b7f1..0000000000000000000000000000000000000000 --- a/data/models/paloalma_ece-tw3-jrgl-v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ECE-TW3-JRGL-V1", - "id": "paloalma/ECE-TW3-JRGL-V1", - "developer": "paloalma", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "68.977" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/paloalma_ECE-TW3-JRGL-V1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5535 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6284 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1314 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3473 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4621 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4221 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/paloalma_ece-tw3-jrgl-v2.json b/data/models/paloalma_ece-tw3-jrgl-v2.json deleted file mode 100644 index ceb80e6866801ce6e5a60a0717bd6577c8eafd4e..0000000000000000000000000000000000000000 --- a/data/models/paloalma_ece-tw3-jrgl-v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ECE-TW3-JRGL-V2", - "id": "paloalma/ECE-TW3-JRGL-V2", - "developer": "paloalma", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "72.288" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/paloalma_ECE-TW3-JRGL-V2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2255 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6031 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.185 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3314 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4793 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4588 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/paloalma_ece-tw3-jrgl-v5.json b/data/models/paloalma_ece-tw3-jrgl-v5.json deleted file mode 100644 index 1f1d9ca15bb3abef656dd5d770b14860c1d8c6e3..0000000000000000000000000000000000000000 --- a/data/models/paloalma_ece-tw3-jrgl-v5.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ECE-TW3-JRGL-V5", - "id": "paloalma/ECE-TW3-JRGL-V5", - "developer": "paloalma", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "72.289" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/paloalma_ECE-TW3-JRGL-V5/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4553 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6025 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1835 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3414 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4621 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4648 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/paloalma_le_triomphant-ece-tw3.json b/data/models/paloalma_le_triomphant-ece-tw3.json deleted file mode 100644 index a75ff531778ca710d3c94bc46c533989bd16706a..0000000000000000000000000000000000000000 --- a/data/models/paloalma_le_triomphant-ece-tw3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Le_Triomphant-ECE-TW3", - "id": "paloalma/Le_Triomphant-ECE-TW3", - "developer": "paloalma", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "72.289" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/paloalma_Le_Triomphant-ECE-TW3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5402 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6112 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1949 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.349 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4725 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4763 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/paloalma_tw3-jrgl-v2.json b/data/models/paloalma_tw3-jrgl-v2.json deleted file mode 100644 index 4954c8950705b9da60a461501817d35fa5b8dca1..0000000000000000000000000000000000000000 --- a/data/models/paloalma_tw3-jrgl-v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "TW3-JRGL-v2", - "id": "paloalma/TW3-JRGL-v2", - "developer": "paloalma", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "72.289" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/paloalma_TW3-JRGL-v2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5316 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6138 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.179 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3591 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4858 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4858 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/pankajmathur_al_dente_v1_8b.json b/data/models/pankajmathur_al_dente_v1_8b.json deleted file mode 100644 index 26b3a9aebfee4df41fad9f8998a8d18d1312ab93..0000000000000000000000000000000000000000 --- a/data/models/pankajmathur_al_dente_v1_8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Al_Dente_v1_8b", - "id": "pankajmathur/Al_Dente_v1_8b", - "developer": "pankajmathur", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/pankajmathur_Al_Dente_v1_8b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3694 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4835 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0408 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2995 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3987 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.286 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/pankajmathur_model_007_13b_v2.json b/data/models/pankajmathur_model_007_13b_v2.json deleted file mode 100644 index 4d2038eff5bfa28e4bc2182e6a711a53185680b7..0000000000000000000000000000000000000000 --- a/data/models/pankajmathur_model_007_13b_v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "model_007_13b_v2", - "id": "pankajmathur/model_007_13b_v2", - "developer": "pankajmathur", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "13.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/pankajmathur_model_007_13b_v2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3056 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4702 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0211 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2836 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4611 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2461 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/pankajmathur_orca_mini_3b.json b/data/models/pankajmathur_orca_mini_3b.json deleted file mode 100644 index 82189b91bd04c8c3b33f6238192cec9101162ad4..0000000000000000000000000000000000000000 --- a/data/models/pankajmathur_orca_mini_3b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "orca_mini_3b", - "id": "pankajmathur/orca_mini_3b", - "developer": "pankajmathur", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.426" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/pankajmathur_orca_mini_3b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0742 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3196 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0083 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2458 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3349 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1145 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/pankajmathur_orca_mini_7b.json b/data/models/pankajmathur_orca_mini_7b.json deleted file mode 100644 index 88f2fba1671eaed9bd8a297e1593e0c7fdfef1c0..0000000000000000000000000000000000000000 --- a/data/models/pankajmathur_orca_mini_7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "orca_mini_7b", - "id": "pankajmathur/orca_mini_7b", - "developer": "pankajmathur", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "7.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/pankajmathur_orca_mini_7b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0412 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3332 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0128 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2542 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3698 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1246 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/pankajmathur_orca_mini_phi-4.json b/data/models/pankajmathur_orca_mini_phi-4.json deleted file mode 100644 index 2955b0345de2272c43522a8e7a049f4035a663c5..0000000000000000000000000000000000000000 --- a/data/models/pankajmathur_orca_mini_phi-4.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "orca_mini_phi-4", - "id": "pankajmathur/orca_mini_phi-4", - "developer": "pankajmathur", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/pankajmathur_orca_mini_phi-4/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7781 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6856 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2953 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3742 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4703 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5255 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/pankajmathur_orca_mini_v2_7b.json b/data/models/pankajmathur_orca_mini_v2_7b.json deleted file mode 100644 index 8f0e640ea553309b2972623a213c25df2ff0ae47..0000000000000000000000000000000000000000 --- a/data/models/pankajmathur_orca_mini_v2_7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "orca_mini_v2_7b", - "id": "pankajmathur/orca_mini_v2_7b", - "developer": "pankajmathur", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "7.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/pankajmathur_orca_mini_v2_7b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1358 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3536 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0113 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2492 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3593 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1542 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/pankajmathur_orca_mini_v3_13b.json b/data/models/pankajmathur_orca_mini_v3_13b.json deleted file mode 100644 index e5ebc465f74891a8a9a0af33a18c82b8593a6206..0000000000000000000000000000000000000000 --- a/data/models/pankajmathur_orca_mini_v3_13b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "orca_mini_v3_13b", - "id": "pankajmathur/orca_mini_v3_13b", - "developer": "pankajmathur", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "13.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/pankajmathur_orca_mini_v3_13b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2897 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4711 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0211 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2651 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4598 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2305 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/pankajmathur_orca_mini_v3_70b.json b/data/models/pankajmathur_orca_mini_v3_70b.json deleted file mode 100644 index 5f36f2e3b6689d6643718306ca51d9f5fb4bf1e1..0000000000000000000000000000000000000000 --- a/data/models/pankajmathur_orca_mini_v3_70b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "orca_mini_v3_70b", - "id": "pankajmathur/orca_mini_v3_70b", - "developer": "pankajmathur", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "70.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/pankajmathur_orca_mini_v3_70b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4015 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5949 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0385 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.318 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5079 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3757 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/pankajmathur_orca_mini_v3_7b.json b/data/models/pankajmathur_orca_mini_v3_7b.json deleted file mode 100644 index 6987ed9d3748df82e3bdb26f549823c0b1e9e732..0000000000000000000000000000000000000000 --- a/data/models/pankajmathur_orca_mini_v3_7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "orca_mini_v3_7b", - "id": "pankajmathur/orca_mini_v3_7b", - "developer": "pankajmathur", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "7.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/pankajmathur_orca_mini_v3_7b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2821 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4095 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0106 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2466 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4982 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2084 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/pankajmathur_orca_mini_v5_8b.json b/data/models/pankajmathur_orca_mini_v5_8b.json deleted file mode 100644 index db9490ce6ef6b0d6a9ad8b788823eeb5c9313fdb..0000000000000000000000000000000000000000 --- a/data/models/pankajmathur_orca_mini_v5_8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "orca_mini_v5_8b", - "id": "pankajmathur/orca_mini_v5_8b", - "developer": "pankajmathur", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/pankajmathur_orca_mini_v5_8b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4806 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5064 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0989 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2869 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3076 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/pankajmathur_orca_mini_v5_8b_dpo.json b/data/models/pankajmathur_orca_mini_v5_8b_dpo.json deleted file mode 100644 index 4cbb56ee8b3f398e0554a6db3b11e535fe67bed1..0000000000000000000000000000000000000000 --- a/data/models/pankajmathur_orca_mini_v5_8b_dpo.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "orca_mini_v5_8b_dpo", - "id": "pankajmathur/orca_mini_v5_8b_dpo", - "developer": "pankajmathur", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/pankajmathur_orca_mini_v5_8b_dpo/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4896 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5075 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0974 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2743 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3894 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3116 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/pankajmathur_orca_mini_v5_8b_orpo.json b/data/models/pankajmathur_orca_mini_v5_8b_orpo.json deleted file mode 100644 index 4988bb69ee90900a474ded3b6cec4b90b42e9f28..0000000000000000000000000000000000000000 --- a/data/models/pankajmathur_orca_mini_v5_8b_orpo.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "orca_mini_v5_8b_orpo", - "id": "pankajmathur/orca_mini_v5_8b_orpo", - "developer": "pankajmathur", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/pankajmathur_orca_mini_v5_8b_orpo/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0824 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4964 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0665 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2844 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4131 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2947 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/pankajmathur_orca_mini_v6_8b.json b/data/models/pankajmathur_orca_mini_v6_8b.json deleted file mode 100644 index f0c6fc2da242acff879ce3f2cc337b87a2b58865..0000000000000000000000000000000000000000 --- a/data/models/pankajmathur_orca_mini_v6_8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "orca_mini_v6_8b", - "id": "pankajmathur/orca_mini_v6_8b", - "developer": "pankajmathur", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/pankajmathur_orca_mini_v6_8b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0111 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3029 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0038 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2383 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3555 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1125 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/pankajmathur_orca_mini_v6_8b_dpo.json b/data/models/pankajmathur_orca_mini_v6_8b_dpo.json deleted file mode 100644 index 69c11190b14f07235d6fb4fd9c5602dd871d5306..0000000000000000000000000000000000000000 --- a/data/models/pankajmathur_orca_mini_v6_8b_dpo.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "orca_mini_v6_8b_dpo", - "id": "pankajmathur/orca_mini_v6_8b_dpo", - "developer": "pankajmathur", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/pankajmathur_orca_mini_v6_8b_dpo/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3883 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5203 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0612 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3012 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.409 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3596 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/pankajmathur_orca_mini_v7_72b.json b/data/models/pankajmathur_orca_mini_v7_72b.json deleted file mode 100644 index 7f757160ddb864a077583b96c4230aca1d02e0df..0000000000000000000000000000000000000000 --- a/data/models/pankajmathur_orca_mini_v7_72b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "orca_mini_v7_72b", - "id": "pankajmathur/orca_mini_v7_72b", - "developer": "pankajmathur", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "72.706" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/pankajmathur_orca_mini_v7_72b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.593 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6842 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0937 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3851 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.507 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5622 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/pankajmathur_orca_mini_v7_7b.json b/data/models/pankajmathur_orca_mini_v7_7b.json deleted file mode 100644 index 881e0e7cbae78fb9a2710725c53840cce173bf87..0000000000000000000000000000000000000000 --- a/data/models/pankajmathur_orca_mini_v7_7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "orca_mini_v7_7b", - "id": "pankajmathur/orca_mini_v7_7b", - "developer": "pankajmathur", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/pankajmathur_orca_mini_v7_7b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4388 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5275 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1208 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2961 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.436 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4167 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/pankajmathur_orca_mini_v8_1_70b.json b/data/models/pankajmathur_orca_mini_v8_1_70b.json deleted file mode 100644 index 6c832e47850e9b34a32475bd2ff87a6df01e9e34..0000000000000000000000000000000000000000 --- a/data/models/pankajmathur_orca_mini_v8_1_70b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "orca_mini_v8_1_70b", - "id": "pankajmathur/orca_mini_v8_1_70b", - "developer": "pankajmathur", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "70.554" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/pankajmathur_orca_mini_v8_1_70b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8571 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6781 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3527 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4329 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4437 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4983 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/pankajmathur_orca_mini_v9_0_3b-instruct.json b/data/models/pankajmathur_orca_mini_v9_0_3b-instruct.json deleted file mode 100644 index e0bc0dcc0d930ba822275d4d83af142c382f87e2..0000000000000000000000000000000000000000 --- a/data/models/pankajmathur_orca_mini_v9_0_3b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "orca_mini_v9_0_3B-Instruct", - "id": "pankajmathur/orca_mini_v9_0_3B-Instruct", - "developer": "pankajmathur", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/pankajmathur_orca_mini_v9_0_3B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5754 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4413 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1465 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3012 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3659 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2603 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/pankajmathur_orca_mini_v9_1_1b-instruct.json b/data/models/pankajmathur_orca_mini_v9_1_1b-instruct.json deleted file mode 100644 index 1e7c054a55757df65864da17089e829c55c5da99..0000000000000000000000000000000000000000 --- a/data/models/pankajmathur_orca_mini_v9_1_1b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "orca_mini_v9_1_1B-Instruct", - "id": "pankajmathur/orca_mini_v9_1_1B-Instruct", - "developer": "pankajmathur", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.236" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/pankajmathur_orca_mini_v9_1_1B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3629 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3205 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0461 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2567 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3381 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1374 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/pankajmathur_orca_mini_v9_2_14b.json b/data/models/pankajmathur_orca_mini_v9_2_14b.json deleted file mode 100644 index 126951e0bd1bee69be75093b5e3c4767e68936e3..0000000000000000000000000000000000000000 --- a/data/models/pankajmathur_orca_mini_v9_2_14b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "orca_mini_v9_2_14B", - "id": "pankajmathur/orca_mini_v9_2_14B", - "developer": "pankajmathur", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/pankajmathur_orca_mini_v9_2_14B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7781 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6856 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2953 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3742 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4703 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5255 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/pankajmathur_orca_mini_v9_2_70b.json b/data/models/pankajmathur_orca_mini_v9_2_70b.json deleted file mode 100644 index 19cb3e0f4a4d46e2e88f90957d38a3ee0ff42ecb..0000000000000000000000000000000000000000 --- a/data/models/pankajmathur_orca_mini_v9_2_70b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "orca_mini_v9_2_70b", - "id": "pankajmathur/orca_mini_v9_2_70b", - "developer": "pankajmathur", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "70.554" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/pankajmathur_orca_mini_v9_2_70b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8383 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6745 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2938 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3733 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.471 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4821 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/pankajmathur_orca_mini_v9_4_70b.json b/data/models/pankajmathur_orca_mini_v9_4_70b.json deleted file mode 100644 index 631538c4898a4ade14244fee2316b54e8834419b..0000000000000000000000000000000000000000 --- a/data/models/pankajmathur_orca_mini_v9_4_70b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "orca_mini_v9_4_70B", - "id": "pankajmathur/orca_mini_v9_4_70B", - "developer": "pankajmathur", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "70.554" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/pankajmathur_orca_mini_v9_4_70B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8015 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6419 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3263 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3658 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4647 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4536 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/pankajmathur_orca_mini_v9_5_1b-instruct.json b/data/models/pankajmathur_orca_mini_v9_5_1b-instruct.json deleted file mode 100644 index f2f8690e2e336cdbd64bdaf05fe88c6dfef01a8b..0000000000000000000000000000000000000000 --- a/data/models/pankajmathur_orca_mini_v9_5_1b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "orca_mini_v9_5_1B-Instruct", - "id": "pankajmathur/orca_mini_v9_5_1B-Instruct", - "developer": "pankajmathur", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.236" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/pankajmathur_orca_mini_v9_5_1B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4638 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3337 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0302 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2701 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3182 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.137 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/pankajmathur_orca_mini_v9_5_1b-instruct_preview.json b/data/models/pankajmathur_orca_mini_v9_5_1b-instruct_preview.json deleted file mode 100644 index 156347cd9de4e61b1eb63ee6ac8a923b8d08b3d3..0000000000000000000000000000000000000000 --- a/data/models/pankajmathur_orca_mini_v9_5_1b-instruct_preview.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "orca_mini_v9_5_1B-Instruct_preview", - "id": "pankajmathur/orca_mini_v9_5_1B-Instruct_preview", - "developer": "pankajmathur", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.236" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/pankajmathur_orca_mini_v9_5_1B-Instruct_preview/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3936 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3277 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0385 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2634 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3395 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1327 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/pankajmathur_orca_mini_v9_5_3b-instruct.json b/data/models/pankajmathur_orca_mini_v9_5_3b-instruct.json deleted file mode 100644 index 96672a2272e0d574e79bd30f63048894fe966944..0000000000000000000000000000000000000000 --- a/data/models/pankajmathur_orca_mini_v9_5_3b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "orca_mini_v9_5_3B-Instruct", - "id": "pankajmathur/orca_mini_v9_5_3B-Instruct", - "developer": "pankajmathur", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/pankajmathur_orca_mini_v9_5_3B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7207 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4496 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1322 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2869 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.427 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2882 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/pankajmathur_orca_mini_v9_6_1b-instruct.json b/data/models/pankajmathur_orca_mini_v9_6_1b-instruct.json deleted file mode 100644 index 4abc8c37086a92cc8e7b56bf20e15af6a12d1143..0000000000000000000000000000000000000000 --- a/data/models/pankajmathur_orca_mini_v9_6_1b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "orca_mini_v9_6_1B-Instruct", - "id": "pankajmathur/orca_mini_v9_6_1B-Instruct", - "developer": "pankajmathur", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.236" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/pankajmathur_orca_mini_v9_6_1B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6086 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3561 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.077 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2685 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3396 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1809 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/pankajmathur_orca_mini_v9_6_3b-instruct.json b/data/models/pankajmathur_orca_mini_v9_6_3b-instruct.json deleted file mode 100644 index f5989b8eee6e705094e47516818437db079072fd..0000000000000000000000000000000000000000 --- a/data/models/pankajmathur_orca_mini_v9_6_3b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "orca_mini_v9_6_3B-Instruct", - "id": "pankajmathur/orca_mini_v9_6_3B-Instruct", - "developer": "pankajmathur", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/pankajmathur_orca_mini_v9_6_3B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7316 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4568 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1329 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2936 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4068 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2851 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/pankajmathur_orca_mini_v9_7_1b-instruct.json b/data/models/pankajmathur_orca_mini_v9_7_1b-instruct.json deleted file mode 100644 index a5b7bbb937738200f81e2d2a0f3aab74802786ba..0000000000000000000000000000000000000000 --- a/data/models/pankajmathur_orca_mini_v9_7_1b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "orca_mini_v9_7_1B-Instruct", - "id": "pankajmathur/orca_mini_v9_7_1B-Instruct", - "developer": "pankajmathur", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.236" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/pankajmathur_orca_mini_v9_7_1B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.561 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3182 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0446 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2727 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3527 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1345 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/pankajmathur_orca_mini_v9_7_3b-instruct.json b/data/models/pankajmathur_orca_mini_v9_7_3b-instruct.json deleted file mode 100644 index 83864a9a40db74313b4c7dcec9119933ab1e87fb..0000000000000000000000000000000000000000 --- a/data/models/pankajmathur_orca_mini_v9_7_3b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "orca_mini_v9_7_3B-Instruct", - "id": "pankajmathur/orca_mini_v9_7_3B-Instruct", - "developer": "pankajmathur", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/pankajmathur_orca_mini_v9_7_3B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5618 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3297 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0619 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2617 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3619 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1375 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/parissa3_test-model.json b/data/models/parissa3_test-model.json deleted file mode 100644 index ee124ac54171ced3719e9af3154a10c023536889..0000000000000000000000000000000000000000 --- a/data/models/parissa3_test-model.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "test-model", - "id": "Parissa3/test-model", - "developer": "Parissa3", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Parissa3_test-model/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3883 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5194 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.065 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2945 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4685 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3057 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/paulml_ece-ilab-q1.json b/data/models/paulml_ece-ilab-q1.json deleted file mode 100644 index 9e431670b55df8a59f8fae976871348ae0c820e7..0000000000000000000000000000000000000000 --- a/data/models/paulml_ece-ilab-q1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ECE-ILAB-Q1", - "id": "paulml/ECE-ILAB-Q1", - "developer": "paulml", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "72.706" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/paulml_ECE-ILAB-Q1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7865 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6718 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3557 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3867 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4614 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5505 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/phronetic-ai_rzn-t-prompt.json b/data/models/phronetic-ai_rzn-t-prompt.json deleted file mode 100644 index aa3544cb299d67bbf254fe5cd0b71e542da3e438..0000000000000000000000000000000000000000 --- a/data/models/phronetic-ai_rzn-t-prompt.json +++ /dev/null @@ -1,905 +0,0 @@ -{ - "model_info": { - "name": "RZN-T (Prompt)", - "id": "phronetic-ai/rzn-t-prompt", - "developer": "phronetic-ai", - "additional_details": { - "raw_model_name": "RZN-T (Prompt)", - "organization": "Phronetic AI", - "license": "apache-2.0", - "mode": "Prompt", - "model_link": "https://huggingface.co/phronetic-ai/RZN-T" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/phronetic-ai/rzn-t-prompt/1775236112.41648", - "retrieved_timestamp": "1775236112.41648", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 96.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 22.25 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 12.31 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 12.32 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 27.53 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 39.84 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 67.94 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 63.75 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 75.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 69.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 63.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 49.74 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 61.24 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 47.2 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 43.75 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 41.67 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 2.88 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 4.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 2.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 2.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 2.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 6.88 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 9.03 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 6.45 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 5.16 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 81.25 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 82.41 - } - }, - { - "evaluation_result_id": "bfcl.format_sensitivity.max_delta::max_delta", - "evaluation_name": "bfcl.format_sensitivity.max_delta", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.format_sensitivity.max_delta", - "metric_name": "Format sensitivity max delta", - "metric_kind": "difference", - "metric_unit": "percentage_points", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Format Sensitivity Max Delta" - } - }, - "score_details": { - "score": 63.5 - } - }, - { - "evaluation_result_id": "bfcl.format_sensitivity.stddev::stddev", - "evaluation_name": "bfcl.format_sensitivity.stddev", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.format_sensitivity.stddev", - "metric_name": "Format sensitivity standard deviation", - "metric_kind": "difference", - "metric_unit": "percentage_points", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Format Sensitivity Standard Deviation" - } - }, - "score_details": { - "score": 25.53 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/pinkstack_parm-v1.5-base-qwq-qwen-2.5-o1-3b.json b/data/models/pinkstack_parm-v1.5-base-qwq-qwen-2.5-o1-3b.json deleted file mode 100644 index 460941749ac925ae33721b271c1ed31f078a5e73..0000000000000000000000000000000000000000 --- a/data/models/pinkstack_parm-v1.5-base-qwq-qwen-2.5-o1-3b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "PARM-V1.5-base-QwQ-Qwen-2.5-o1-3B", - "id": "Pinkstack/PARM-V1.5-base-QwQ-Qwen-2.5-o1-3B", - "developer": "Pinkstack", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.086" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Pinkstack_PARM-V1.5-base-QwQ-Qwen-2.5-o1-3B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5085 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4711 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1692 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.297 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4479 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3511 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/pinkstack_superthoughts-cot-14b-16k-o1-qwq.json b/data/models/pinkstack_superthoughts-cot-14b-16k-o1-qwq.json deleted file mode 100644 index c3f652fab67a037762eb161f213a31bfc587ae95..0000000000000000000000000000000000000000 --- a/data/models/pinkstack_superthoughts-cot-14b-16k-o1-qwq.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SuperThoughts-CoT-14B-16k-o1-QwQ", - "id": "Pinkstack/SuperThoughts-CoT-14B-16k-o1-QwQ", - "developer": "Pinkstack", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Pinkstack_SuperThoughts-CoT-14B-16k-o1-QwQ/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0515 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.672 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4199 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3926 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4914 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5268 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/pinkstack_superthoughts-lite-1.8b-experimental-o1.json b/data/models/pinkstack_superthoughts-lite-1.8b-experimental-o1.json deleted file mode 100644 index b2ccfa9d5db7757ba4cfcc986b1019db71fcd0e9..0000000000000000000000000000000000000000 --- a/data/models/pinkstack_superthoughts-lite-1.8b-experimental-o1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Superthoughts-lite-1.8B-experimental-o1", - "id": "Pinkstack/Superthoughts-lite-1.8B-experimental-o1", - "developer": "Pinkstack", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.812" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Pinkstack_Superthoughts-lite-1.8B-experimental-o1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0375 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3435 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0317 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2752 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3354 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1851 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/pinkstack_superthoughts-lite-v1.json b/data/models/pinkstack_superthoughts-lite-v1.json deleted file mode 100644 index ff8574e36d3b6d6162685c97a3d191472b1081f9..0000000000000000000000000000000000000000 --- a/data/models/pinkstack_superthoughts-lite-v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Superthoughts-lite-v1", - "id": "Pinkstack/Superthoughts-lite-v1", - "developer": "Pinkstack", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.711" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Pinkstack_Superthoughts-lite-v1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1659 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3466 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0295 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.281 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3672 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1755 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/pints-ai_1.5-pints-16k-v0.1.json b/data/models/pints-ai_1.5-pints-16k-v0.1.json deleted file mode 100644 index ef5a511dd15198ca46cd423b144b22bd77ca5ce0..0000000000000000000000000000000000000000 --- a/data/models/pints-ai_1.5-pints-16k-v0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "1.5-Pints-16K-v0.1", - "id": "pints-ai/1.5-Pints-16K-v0.1", - "developer": "pints-ai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.566" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/pints-ai_1.5-Pints-16K-v0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1636 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3133 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0144 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2357 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3579 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1119 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/pints-ai_1.5-pints-2k-v0.1.json b/data/models/pints-ai_1.5-pints-2k-v0.1.json deleted file mode 100644 index 148a58f18996c1e685a10ed18d77c4958c2c117a..0000000000000000000000000000000000000000 --- a/data/models/pints-ai_1.5-pints-2k-v0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "1.5-Pints-2K-v0.1", - "id": "pints-ai/1.5-Pints-2K-v0.1", - "developer": "pints-ai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.566" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/pints-ai_1.5-Pints-2K-v0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1762 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.298 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0128 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2483 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3502 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1104 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/piotr25691_thea-3b-25r.json b/data/models/piotr25691_thea-3b-25r.json deleted file mode 100644 index fee356c64792964df7726de1c0c7277a20fc8f22..0000000000000000000000000000000000000000 --- a/data/models/piotr25691_thea-3b-25r.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "thea-3b-25r", - "id": "piotr25691/thea-3b-25r", - "developer": "piotr25691", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/piotr25691_thea-3b-25r/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7344 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4484 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1782 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2676 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3315 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3182 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/piotr25691_thea-c-3b-25r.json b/data/models/piotr25691_thea-c-3b-25r.json deleted file mode 100644 index bb9cead6799169d6c5f8b562a475373049bb664d..0000000000000000000000000000000000000000 --- a/data/models/piotr25691_thea-c-3b-25r.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "thea-c-3b-25r", - "id": "piotr25691/thea-c-3b-25r", - "developer": "piotr25691", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/piotr25691_thea-c-3b-25r/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7402 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4532 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1526 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2651 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3315 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3178 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/piotr25691_thea-rp-3b-25r.json b/data/models/piotr25691_thea-rp-3b-25r.json deleted file mode 100644 index 07f6c3c8958c59000832927c3e16c8677a739341..0000000000000000000000000000000000000000 --- a/data/models/piotr25691_thea-rp-3b-25r.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "thea-rp-3b-25r", - "id": "piotr25691/thea-rp-3b-25r", - "developer": "piotr25691", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/piotr25691_thea-rp-3b-25r/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6578 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.439 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1322 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2743 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3819 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.306 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/pjmixers-dev_l3.2-instruct-thinking-v0.1-1b.json b/data/models/pjmixers-dev_l3.2-instruct-thinking-v0.1-1b.json deleted file mode 100644 index fabad535a69d2a6ebf900ee411a4d71adca0b3f9..0000000000000000000000000000000000000000 --- a/data/models/pjmixers-dev_l3.2-instruct-thinking-v0.1-1b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "L3.2-Instruct-Thinking-v0.1-1B", - "id": "PJMixers-Dev/L3.2-Instruct-Thinking-v0.1-1B", - "developer": "PJMixers-Dev", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.236" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/PJMixers-Dev_L3.2-Instruct-Thinking-v0.1-1B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4628 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3302 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0544 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2576 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3262 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1483 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/pjmixers-dev_llama-3.1-instruct-interleaved-zeroed-13b.json b/data/models/pjmixers-dev_llama-3.1-instruct-interleaved-zeroed-13b.json deleted file mode 100644 index 026a0e544d24cbe36d2d5891f63a0aca98c15f1b..0000000000000000000000000000000000000000 --- a/data/models/pjmixers-dev_llama-3.1-instruct-interleaved-zeroed-13b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "LLaMa-3.1-Instruct-Interleaved-Zeroed-13B", - "id": "PJMixers-Dev/LLaMa-3.1-Instruct-Interleaved-Zeroed-13B", - "developer": "PJMixers-Dev", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "13.047" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/PJMixers-Dev_LLaMa-3.1-Instruct-Interleaved-Zeroed-13B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7871 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5073 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2002 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2919 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.387 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3767 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/pjmixers-dev_llama-3.1-rombotiestest-8b.json b/data/models/pjmixers-dev_llama-3.1-rombotiestest-8b.json deleted file mode 100644 index 383bd9370576df5d294a4da7310d29fc1132f422..0000000000000000000000000000000000000000 --- a/data/models/pjmixers-dev_llama-3.1-rombotiestest-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "LLaMa-3.1-RomboTiesTest-8B", - "id": "PJMixers-Dev/LLaMa-3.1-RomboTiesTest-8B", - "developer": "PJMixers-Dev", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "4.015" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/PJMixers-Dev_LLaMa-3.1-RomboTiesTest-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7825 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5073 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2002 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2919 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.387 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3767 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/pjmixers-dev_llama-3.1-rombotiestest2-8b.json b/data/models/pjmixers-dev_llama-3.1-rombotiestest2-8b.json deleted file mode 100644 index 49493685115e7400d439e98ec19b507938982915..0000000000000000000000000000000000000000 --- a/data/models/pjmixers-dev_llama-3.1-rombotiestest2-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "LLaMa-3.1-RomboTiesTest2-8B", - "id": "PJMixers-Dev/LLaMa-3.1-RomboTiesTest2-8B", - "developer": "PJMixers-Dev", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "4.015" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/PJMixers-Dev_LLaMa-3.1-RomboTiesTest2-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7825 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5073 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2002 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2919 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.387 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3767 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/pjmixers-dev_llama-3.2-instruct-jankmix-v0.1-sft-3b.json b/data/models/pjmixers-dev_llama-3.2-instruct-jankmix-v0.1-sft-3b.json deleted file mode 100644 index 4bfe413beff32435b49d215bb954108b87c84bc3..0000000000000000000000000000000000000000 --- a/data/models/pjmixers-dev_llama-3.2-instruct-jankmix-v0.1-sft-3b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "LLaMa-3.2-Instruct-JankMix-v0.1-SFT-3B", - "id": "PJMixers-Dev/LLaMa-3.2-Instruct-JankMix-v0.1-SFT-3B", - "developer": "PJMixers-Dev", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/PJMixers-Dev_LLaMa-3.2-Instruct-JankMix-v0.1-SFT-3B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6931 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4556 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1216 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2743 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.37 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3127 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/pjmixers-dev_llama-3.2-instruct-jankmix-v0.2-sft-3b.json b/data/models/pjmixers-dev_llama-3.2-instruct-jankmix-v0.2-sft-3b.json deleted file mode 100644 index 2af209e6a4eea097b0146b6fedb1c92ff5c21803..0000000000000000000000000000000000000000 --- a/data/models/pjmixers-dev_llama-3.2-instruct-jankmix-v0.2-sft-3b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "LLaMa-3.2-Instruct-JankMix-v0.2-SFT-3B", - "id": "PJMixers-Dev/LLaMa-3.2-Instruct-JankMix-v0.2-SFT-3B", - "developer": "PJMixers-Dev", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/PJMixers-Dev_LLaMa-3.2-Instruct-JankMix-v0.2-SFT-3B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6292 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4581 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1299 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2727 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3659 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3115 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/pjmixers-dev_llama-3.2-instruct-jankmix-v0.2-sft-hailmary-v0.1-kto-3b.json b/data/models/pjmixers-dev_llama-3.2-instruct-jankmix-v0.2-sft-hailmary-v0.1-kto-3b.json deleted file mode 100644 index 3192cf1a885acdd84b5e38cd7541e4f0fa742819..0000000000000000000000000000000000000000 --- a/data/models/pjmixers-dev_llama-3.2-instruct-jankmix-v0.2-sft-hailmary-v0.1-kto-3b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "LLaMa-3.2-Instruct-JankMix-v0.2-SFT-HailMary-v0.1-KTO-3B", - "id": "PJMixers-Dev/LLaMa-3.2-Instruct-JankMix-v0.2-SFT-HailMary-v0.1-KTO-3B", - "developer": "PJMixers-Dev", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/PJMixers-Dev_LLaMa-3.2-Instruct-JankMix-v0.2-SFT-HailMary-v0.1-KTO-3B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6504 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4511 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1261 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2718 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3687 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3108 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/pjmixers-dev_llama-3.2-instruct-jankmixbread-v0.1-3b.json b/data/models/pjmixers-dev_llama-3.2-instruct-jankmixbread-v0.1-3b.json deleted file mode 100644 index 660cda018bf68bf8a23debaf9a0f64dc4a137bf9..0000000000000000000000000000000000000000 --- a/data/models/pjmixers-dev_llama-3.2-instruct-jankmixbread-v0.1-3b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "LLaMa-3.2-Instruct-JankMixBread-v0.1-3B", - "id": "PJMixers-Dev/LLaMa-3.2-Instruct-JankMixBread-v0.1-3B", - "developer": "PJMixers-Dev", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/PJMixers-Dev_LLaMa-3.2-Instruct-JankMixBread-v0.1-3B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5041 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4483 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1307 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2827 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3516 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3083 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/pjmixers-dev_qwen2.5-rombotiestest-7b.json b/data/models/pjmixers-dev_qwen2.5-rombotiestest-7b.json deleted file mode 100644 index b74b98c22bd60bff3c090fb22a169d4cf0dff4e0..0000000000000000000000000000000000000000 --- a/data/models/pjmixers-dev_qwen2.5-rombotiestest-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-RomboTiesTest-7B", - "id": "PJMixers-Dev/Qwen2.5-RomboTiesTest-7B", - "developer": "PJMixers-Dev", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.808" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/PJMixers-Dev_Qwen2.5-RomboTiesTest-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7558 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5399 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4962 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2978 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4034 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4285 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/pjmixers_llama-3-cursedstock-v2.0-8b.json b/data/models/pjmixers_llama-3-cursedstock-v2.0-8b.json deleted file mode 100644 index 7bc96fd611c32da209f35554fc4168f52fae2956..0000000000000000000000000000000000000000 --- a/data/models/pjmixers_llama-3-cursedstock-v2.0-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "LLaMa-3-CursedStock-v2.0-8B", - "id": "PJMixers/LLaMa-3-CursedStock-v2.0-8B", - "developer": "PJMixers", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/PJMixers_LLaMa-3-CursedStock-v2.0-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6331 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5271 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0944 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2743 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3856 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3556 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/pku-alignment_beaver-7b-v1.0-cost.json b/data/models/pku-alignment_beaver-7b-v1.0-cost.json deleted file mode 100644 index 3777eba3edfdc470c669a503ac85994bf8139135..0000000000000000000000000000000000000000 --- a/data/models/pku-alignment_beaver-7b-v1.0-cost.json +++ /dev/null @@ -1,294 +0,0 @@ -{ - "model_info": { - "name": "PKU-Alignment/beaver-7b-v1.0-cost", - "id": "PKU-Alignment/beaver-7b-v1.0-cost", - "developer": "PKU-Alignment", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/PKU-Alignment_beaver-7b-v1.0-cost/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3332 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3263 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2313 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3989 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7589 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2939 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -0.01 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "reward-bench/PKU-Alignment_beaver-7b-v1.0-cost/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5798 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6173 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4232 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7351 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5482 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.57 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/pku-alignment_beaver-7b-v1.0-reward.json b/data/models/pku-alignment_beaver-7b-v1.0-reward.json deleted file mode 100644 index adf890d6a20deeb5311651870c4e5638866b9733..0000000000000000000000000000000000000000 --- a/data/models/pku-alignment_beaver-7b-v1.0-reward.json +++ /dev/null @@ -1,294 +0,0 @@ -{ - "model_info": { - "name": "PKU-Alignment/beaver-7b-v1.0-reward", - "id": "PKU-Alignment/beaver-7b-v1.0-reward", - "developer": "PKU-Alignment", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/PKU-Alignment_beaver-7b-v1.0-reward/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1606 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2105 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2938 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2623 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1422 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0646 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -0.01 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "reward-bench/PKU-Alignment_beaver-7b-v1.0-reward/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4727 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8184 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2873 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3757 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.346 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5993 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/pku-alignment_beaver-7b-v2.0-cost.json b/data/models/pku-alignment_beaver-7b-v2.0-cost.json deleted file mode 100644 index 5314c675193b80f86e094f04e503749d966c642b..0000000000000000000000000000000000000000 --- a/data/models/pku-alignment_beaver-7b-v2.0-cost.json +++ /dev/null @@ -1,294 +0,0 @@ -{ - "model_info": { - "name": "PKU-Alignment/beaver-7b-v2.0-cost", - "id": "PKU-Alignment/beaver-7b-v2.0-cost", - "developer": "PKU-Alignment", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/PKU-Alignment_beaver-7b-v2.0-cost/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3326 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3789 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.275 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3333 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7356 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2828 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -0.01 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "reward-bench/PKU-Alignment_beaver-7b-v2.0-cost/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5957 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5726 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4561 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7608 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6211 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5397 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/pku-alignment_beaver-7b-v2.0-reward.json b/data/models/pku-alignment_beaver-7b-v2.0-reward.json deleted file mode 100644 index e6954fa8026e0ae406f243f530096227275cb53f..0000000000000000000000000000000000000000 --- a/data/models/pku-alignment_beaver-7b-v2.0-reward.json +++ /dev/null @@ -1,294 +0,0 @@ -{ - "model_info": { - "name": "PKU-Alignment/beaver-7b-v2.0-reward", - "id": "PKU-Alignment/beaver-7b-v2.0-reward", - "developer": "PKU-Alignment", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/PKU-Alignment_beaver-7b-v2.0-reward/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2544 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2168 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2562 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3825 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3156 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2606 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0944 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "reward-bench/PKU-Alignment_beaver-7b-v2.0-reward/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6366 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8994 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.364 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6041 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6887 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6171 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/pocketdoc_dans-instruct-corecurriculum-12b.json b/data/models/pocketdoc_dans-instruct-corecurriculum-12b.json deleted file mode 100644 index 06885af46f117b5aa26d9ebeb7aac3fb5ed3350f..0000000000000000000000000000000000000000 --- a/data/models/pocketdoc_dans-instruct-corecurriculum-12b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Dans-Instruct-CoreCurriculum-12b", - "id": "PocketDoc/Dans-Instruct-CoreCurriculum-12b", - "developer": "PocketDoc", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/PocketDoc_Dans-Instruct-CoreCurriculum-12b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2191 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3789 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0544 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2827 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4096 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1219 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/pocketdoc_dans-personalityengine-v1.0.0-8b.json b/data/models/pocketdoc_dans-personalityengine-v1.0.0-8b.json deleted file mode 100644 index 23af4f465fc7de3631aaaf3c9c9fa3356d7afd9b..0000000000000000000000000000000000000000 --- a/data/models/pocketdoc_dans-personalityengine-v1.0.0-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Dans-PersonalityEngine-v1.0.0-8b", - "id": "PocketDoc/Dans-PersonalityEngine-v1.0.0-8b", - "developer": "PocketDoc", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/PocketDoc_Dans-PersonalityEngine-v1.0.0-8b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4982 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4733 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0816 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2852 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3542 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3065 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/pocketdoc_dans-personalityengine-v1.1.0-12b.json b/data/models/pocketdoc_dans-personalityengine-v1.1.0-12b.json deleted file mode 100644 index 5f43b032b6a5466f3193213c9552c4ef75b86d60..0000000000000000000000000000000000000000 --- a/data/models/pocketdoc_dans-personalityengine-v1.1.0-12b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Dans-PersonalityEngine-V1.1.0-12b", - "id": "PocketDoc/Dans-PersonalityEngine-V1.1.0-12b", - "developer": "PocketDoc", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/PocketDoc_Dans-PersonalityEngine-V1.1.0-12b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7075 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5361 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.105 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2869 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4587 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3262 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/pocketdoc_dans-personalityengine-v1.2.0-24b.json b/data/models/pocketdoc_dans-personalityengine-v1.2.0-24b.json deleted file mode 100644 index cff8c60e9e602cd022bc896f5b2c994c858ac93d..0000000000000000000000000000000000000000 --- a/data/models/pocketdoc_dans-personalityengine-v1.2.0-24b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Dans-PersonalityEngine-V1.2.0-24b", - "id": "PocketDoc/Dans-PersonalityEngine-V1.2.0-24b", - "developer": "PocketDoc", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "23.572" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/PocketDoc_Dans-PersonalityEngine-V1.2.0-24b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7886 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6421 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2455 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3188 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.43 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5026 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/pocketdoc_dans-sakurakaze-v1.0.0-12b.json b/data/models/pocketdoc_dans-sakurakaze-v1.0.0-12b.json deleted file mode 100644 index e4dd16be29c09ab6a7b002ccbc2aa3139cf8bd9a..0000000000000000000000000000000000000000 --- a/data/models/pocketdoc_dans-sakurakaze-v1.0.0-12b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Dans-SakuraKaze-V1.0.0-12b", - "id": "PocketDoc/Dans-SakuraKaze-V1.0.0-12b", - "developer": "PocketDoc", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/PocketDoc_Dans-SakuraKaze-V1.0.0-12b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.652 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5405 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0929 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2936 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4745 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.356 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/poll_gpt-3.5-turbo-0125_claude-3-sonnet-2024022....json b/data/models/poll_gpt-3.5-turbo-0125_claude-3-sonnet-2024022....json deleted file mode 100644 index b38757d9d639c19afb6f70bc9d30707392db74da..0000000000000000000000000000000000000000 --- a/data/models/poll_gpt-3.5-turbo-0125_claude-3-sonnet-2024022....json +++ /dev/null @@ -1,126 +0,0 @@ -{ - "model_info": { - "name": "PoLL/gpt-3.5-turbo-0125_claude-3-sonnet-2024022...", - "id": "PoLL/gpt-3.5-turbo-0125_claude-3-sonnet-2024022...", - "developer": "PoLL", - "additional_details": { - "model_type": "Generative" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/PoLL_gpt-3.5-turbo-0125_claude-3-sonnet-2024022.../1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7578 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9525 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5406 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8034 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7346 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/postbot_gpt2-medium-emailgen.json b/data/models/postbot_gpt2-medium-emailgen.json deleted file mode 100644 index 3d021e56f9491044f312c697e97f05d5d7225538..0000000000000000000000000000000000000000 --- a/data/models/postbot_gpt2-medium-emailgen.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "gpt2-medium-emailgen", - "id": "postbot/gpt2-medium-emailgen", - "developer": "postbot", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "GPT2LMHeadModel", - "params_billions": "0.38" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/postbot_gpt2-medium-emailgen/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1492 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.313 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2601 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3911 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1147 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/powerinfer_smallthinker-3b-preview.json b/data/models/powerinfer_smallthinker-3b-preview.json deleted file mode 100644 index dc4434f51d168df662e9106f961df6400234d0f4..0000000000000000000000000000000000000000 --- a/data/models/powerinfer_smallthinker-3b-preview.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SmallThinker-3B-Preview", - "id": "PowerInfer/SmallThinker-3B-Preview", - "developer": "PowerInfer", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.397" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/PowerInfer_SmallThinker-3B-Preview/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.62 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4495 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2779 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2609 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3525 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3018 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/pranavharshan_lamistral-v4.json b/data/models/pranavharshan_lamistral-v4.json deleted file mode 100644 index 4e2dc20d87d153b57789e550918a56f8df03da40..0000000000000000000000000000000000000000 --- a/data/models/pranavharshan_lamistral-v4.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "LaMistral-V4", - "id": "PranavHarshan/LaMistral-V4", - "developer": "PranavHarshan", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/PranavHarshan_LaMistral-V4/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6239 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5184 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0687 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.328 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3643 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3599 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/pranavharshan_mednarra-x1.json b/data/models/pranavharshan_mednarra-x1.json deleted file mode 100644 index b3a2c3674d51b98e5cacf7f891f060265c72dc1b..0000000000000000000000000000000000000000 --- a/data/models/pranavharshan_mednarra-x1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MedNarra-X1", - "id": "PranavHarshan/MedNarra-X1", - "developer": "PranavHarshan", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/PranavHarshan_MedNarra-X1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4338 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4637 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0438 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3079 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.354 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3431 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/pretergeek_openchat-3.5-0106_10.7b_48layers-appended.json b/data/models/pretergeek_openchat-3.5-0106_10.7b_48layers-appended.json deleted file mode 100644 index 1781a17db9c0f9d996af0128b8abc91eff99e5b4..0000000000000000000000000000000000000000 --- a/data/models/pretergeek_openchat-3.5-0106_10.7b_48layers-appended.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "OpenChat-3.5-0106_10.7B_48Layers-Appended", - "id": "Pretergeek/OpenChat-3.5-0106_10.7B_48Layers-Appended", - "developer": "Pretergeek", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "10.732" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Pretergeek_OpenChat-3.5-0106_10.7B_48Layers-Appended/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5961 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.462 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0793 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.307 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4254 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.329 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/pretergeek_openchat-3.5-0106_10.7b_48layers-interleaved.json b/data/models/pretergeek_openchat-3.5-0106_10.7b_48layers-interleaved.json deleted file mode 100644 index d84e0800fd7f66207cf8c39000beb2f5b72aad3a..0000000000000000000000000000000000000000 --- a/data/models/pretergeek_openchat-3.5-0106_10.7b_48layers-interleaved.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "OpenChat-3.5-0106_10.7B_48Layers-Interleaved", - "id": "Pretergeek/OpenChat-3.5-0106_10.7B_48Layers-Interleaved", - "developer": "Pretergeek", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "10.732" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Pretergeek_OpenChat-3.5-0106_10.7B_48Layers-Interleaved/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5961 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.462 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0778 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3045 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4254 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3299 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/pretergeek_openchat-3.5-0106_32k-pose.json b/data/models/pretergeek_openchat-3.5-0106_32k-pose.json deleted file mode 100644 index 009bcfa5f39337ddce538a0e67ecc7b0b3c5b83c..0000000000000000000000000000000000000000 --- a/data/models/pretergeek_openchat-3.5-0106_32k-pose.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "OpenChat-3.5-0106_32K-PoSE", - "id": "Pretergeek/OpenChat-3.5-0106_32K-PoSE", - "developer": "Pretergeek", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Pretergeek_OpenChat-3.5-0106_32K-PoSE/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3969 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3471 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0264 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.276 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4205 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2031 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/pretergeek_openchat-3.5-0106_8.11b_36layers-appended.json b/data/models/pretergeek_openchat-3.5-0106_8.11b_36layers-appended.json deleted file mode 100644 index c48d1f5b0c7e9a89b37967712ef60897712ea980..0000000000000000000000000000000000000000 --- a/data/models/pretergeek_openchat-3.5-0106_8.11b_36layers-appended.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "OpenChat-3.5-0106_8.11B_36Layers-Appended", - "id": "Pretergeek/OpenChat-3.5-0106_8.11B_36Layers-Appended", - "developer": "Pretergeek", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "8.114" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Pretergeek_OpenChat-3.5-0106_8.11B_36Layers-Appended/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5976 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.462 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0793 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.307 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4254 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.329 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/pretergeek_openchat-3.5-0106_8.11b_36layers-interleaved.json b/data/models/pretergeek_openchat-3.5-0106_8.11b_36layers-interleaved.json deleted file mode 100644 index 73530901c996e5e42a76a2d9e2c74c433e3dbe70..0000000000000000000000000000000000000000 --- a/data/models/pretergeek_openchat-3.5-0106_8.11b_36layers-interleaved.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "OpenChat-3.5-0106_8.11B_36Layers-Interleaved", - "id": "Pretergeek/OpenChat-3.5-0106_8.11B_36Layers-Interleaved", - "developer": "Pretergeek", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "8.114" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Pretergeek_OpenChat-3.5-0106_8.11B_36Layers-Interleaved/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5961 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4621 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0778 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3045 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4241 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3299 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/pretergeek_openchat-3.5-0106_8.99b_40layers-appended.json b/data/models/pretergeek_openchat-3.5-0106_8.99b_40layers-appended.json deleted file mode 100644 index 58caa79aa982a0fee1c085b334dfe9d37baa3379..0000000000000000000000000000000000000000 --- a/data/models/pretergeek_openchat-3.5-0106_8.99b_40layers-appended.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "OpenChat-3.5-0106_8.99B_40Layers-Appended", - "id": "Pretergeek/OpenChat-3.5-0106_8.99B_40Layers-Appended", - "developer": "Pretergeek", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "8.987" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Pretergeek_OpenChat-3.5-0106_8.99B_40Layers-Appended/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5961 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.462 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0793 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.307 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4254 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.329 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/pretergeek_openchat-3.5-0106_8.99b_40layers-interleaved.json b/data/models/pretergeek_openchat-3.5-0106_8.99b_40layers-interleaved.json deleted file mode 100644 index 7f1ae48a350944fc4c0fda2691eaf8c9cf0e8f86..0000000000000000000000000000000000000000 --- a/data/models/pretergeek_openchat-3.5-0106_8.99b_40layers-interleaved.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "OpenChat-3.5-0106_8.99B_40Layers-Interleaved", - "id": "Pretergeek/OpenChat-3.5-0106_8.99B_40Layers-Interleaved", - "developer": "Pretergeek", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "8.987" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Pretergeek_OpenChat-3.5-0106_8.99B_40Layers-Interleaved/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5976 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4621 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0778 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3045 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4241 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3299 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/pretergeek_openchat-3.5-0106_9.86b_44layers-appended.json b/data/models/pretergeek_openchat-3.5-0106_9.86b_44layers-appended.json deleted file mode 100644 index 42064280ed4f00c71cfb590888d561c869442067..0000000000000000000000000000000000000000 --- a/data/models/pretergeek_openchat-3.5-0106_9.86b_44layers-appended.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "OpenChat-3.5-0106_9.86B_44Layers-Appended", - "id": "Pretergeek/OpenChat-3.5-0106_9.86B_44Layers-Appended", - "developer": "Pretergeek", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "9.859" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Pretergeek_OpenChat-3.5-0106_9.86B_44Layers-Appended/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5961 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.462 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0793 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.307 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4254 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.329 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/pretergeek_openchat-3.5-0106_rebased_mistral-7b-v0.2.json b/data/models/pretergeek_openchat-3.5-0106_rebased_mistral-7b-v0.2.json deleted file mode 100644 index c1a64801f7f00ae1413da56d776bbc36d71adc7e..0000000000000000000000000000000000000000 --- a/data/models/pretergeek_openchat-3.5-0106_rebased_mistral-7b-v0.2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "openchat-3.5-0106_Rebased_Mistral-7B-v0.2", - "id": "Pretergeek/openchat-3.5-0106_Rebased_Mistral-7B-v0.2", - "developer": "Pretergeek", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Pretergeek_openchat-3.5-0106_Rebased_Mistral-7B-v0.2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3706 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3627 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0453 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2718 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.484 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.283 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/primeintellect_intellect-1-instruct.json b/data/models/primeintellect_intellect-1-instruct.json deleted file mode 100644 index 102d9e52742b0fe86c954c92ae8814efe750470f..0000000000000000000000000000000000000000 --- a/data/models/primeintellect_intellect-1-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "INTELLECT-1-Instruct", - "id": "PrimeIntellect/INTELLECT-1-Instruct", - "developer": "PrimeIntellect", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "10.211" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/PrimeIntellect_INTELLECT-1-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.287 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0227 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2483 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3577 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1064 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/primeintellect_intellect-1.json b/data/models/primeintellect_intellect-1.json deleted file mode 100644 index b8fa0710a43fde559062289de677b131f6527599..0000000000000000000000000000000000000000 --- a/data/models/primeintellect_intellect-1.json +++ /dev/null @@ -1,275 +0,0 @@ -{ - "model_info": { - "name": "INTELLECT-1", - "id": "PrimeIntellect/INTELLECT-1", - "developer": "PrimeIntellect", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "10.211" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/PrimeIntellect_INTELLECT-1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1757 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.276 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2534 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3339 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1123 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "hfopenllm_v2/PrimeIntellect_INTELLECT-1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1757 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.274 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.25 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3753 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.112 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prince-canuma_ministral-8b-instruct-2410-hf.json b/data/models/prince-canuma_ministral-8b-instruct-2410-hf.json deleted file mode 100644 index 3dbfad97813004f6bea743d704caa32136612f28..0000000000000000000000000000000000000000 --- a/data/models/prince-canuma_ministral-8b-instruct-2410-hf.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Ministral-8B-Instruct-2410-HF", - "id": "prince-canuma/Ministral-8B-Instruct-2410-HF", - "developer": "prince-canuma", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "8.02" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prince-canuma_Ministral-8B-Instruct-2410-HF/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5912 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4586 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1918 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.281 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4138 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3298 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/princeton-nlp_gemma-2-9b-it-dpo.json b/data/models/princeton-nlp_gemma-2-9b-it-dpo.json deleted file mode 100644 index b858036771f68d0df97881ca344c5e48fa456fa0..0000000000000000000000000000000000000000 --- a/data/models/princeton-nlp_gemma-2-9b-it-dpo.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "gemma-2-9b-it-DPO", - "id": "princeton-nlp/gemma-2-9b-it-DPO", - "developer": "princeton-nlp", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "9.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/princeton-nlp_gemma-2-9b-it-DPO/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2769 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5941 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0831 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3356 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.382 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3723 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/princeton-nlp_gemma-2-9b-it-simpo.json b/data/models/princeton-nlp_gemma-2-9b-it-simpo.json deleted file mode 100644 index ca8471590f5c2bf7a13243b539b9e8034eba934a..0000000000000000000000000000000000000000 --- a/data/models/princeton-nlp_gemma-2-9b-it-simpo.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "gemma-2-9b-it-SimPO", - "id": "princeton-nlp/gemma-2-9b-it-SimPO", - "developer": "princeton-nlp", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "9.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/princeton-nlp_gemma-2-9b-it-SimPO/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3207 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5839 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.071 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3356 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4123 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3975 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/princeton-nlp_llama-3-8b-prolong-512k-base.json b/data/models/princeton-nlp_llama-3-8b-prolong-512k-base.json deleted file mode 100644 index 811c58f06cd0ecc5e27110240ab4356d7fc0266e..0000000000000000000000000000000000000000 --- a/data/models/princeton-nlp_llama-3-8b-prolong-512k-base.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-8B-ProLong-512k-Base", - "id": "princeton-nlp/Llama-3-8B-ProLong-512k-Base", - "developer": "princeton-nlp", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/princeton-nlp_Llama-3-8B-ProLong-512k-Base/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5322 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5033 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0687 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2617 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4223 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3329 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/princeton-nlp_llama-3-8b-prolong-512k-instruct.json b/data/models/princeton-nlp_llama-3-8b-prolong-512k-instruct.json deleted file mode 100644 index a2239eea28a466521ab226132351f6bc4c958619..0000000000000000000000000000000000000000 --- a/data/models/princeton-nlp_llama-3-8b-prolong-512k-instruct.json +++ /dev/null @@ -1,275 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-8B-ProLong-512k-Instruct", - "id": "princeton-nlp/Llama-3-8B-ProLong-512k-Instruct", - "developer": "princeton-nlp", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/princeton-nlp_Llama-3-8B-ProLong-512k-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3978 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4983 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0582 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.281 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.425 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3246 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "hfopenllm_v2/princeton-nlp_Llama-3-8B-ProLong-512k-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5508 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5028 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0529 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2861 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4266 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3231 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/princeton-nlp_llama-3-8b-prolong-64k-base.json b/data/models/princeton-nlp_llama-3-8b-prolong-64k-base.json deleted file mode 100644 index 9378a33be716a27292afadfeb3f1bc88d3ed0b6b..0000000000000000000000000000000000000000 --- a/data/models/princeton-nlp_llama-3-8b-prolong-64k-base.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-8B-ProLong-64k-Base", - "id": "princeton-nlp/Llama-3-8B-ProLong-64k-Base", - "developer": "princeton-nlp", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/princeton-nlp_Llama-3-8B-ProLong-64k-Base/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5201 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4927 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.065 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2651 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4341 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3348 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/princeton-nlp_llama-3-8b-prolong-64k-instruct.json b/data/models/princeton-nlp_llama-3-8b-prolong-64k-instruct.json deleted file mode 100644 index 73df9def4df59bd35281b3c6a8059cac34b8985c..0000000000000000000000000000000000000000 --- a/data/models/princeton-nlp_llama-3-8b-prolong-64k-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-8B-ProLong-64k-Instruct", - "id": "princeton-nlp/Llama-3-8B-ProLong-64k-Instruct", - "developer": "princeton-nlp", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/princeton-nlp_Llama-3-8B-ProLong-64k-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5563 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5083 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.065 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2953 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4397 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3275 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/princeton-nlp_llama-3-base-8b-sft-cpo.json b/data/models/princeton-nlp_llama-3-base-8b-sft-cpo.json deleted file mode 100644 index 6eb3cea54fbaec7938dda21ed017305d61040345..0000000000000000000000000000000000000000 --- a/data/models/princeton-nlp_llama-3-base-8b-sft-cpo.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-Base-8B-SFT-CPO", - "id": "princeton-nlp/Llama-3-Base-8B-SFT-CPO", - "developer": "princeton-nlp", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/princeton-nlp_Llama-3-Base-8B-SFT-CPO/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3703 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4595 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0544 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2743 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3609 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2976 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/princeton-nlp_llama-3-base-8b-sft-dpo.json b/data/models/princeton-nlp_llama-3-base-8b-sft-dpo.json deleted file mode 100644 index 6b2bbb0effbdce75e6e23760dd54403e65927c49..0000000000000000000000000000000000000000 --- a/data/models/princeton-nlp_llama-3-base-8b-sft-dpo.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-Base-8B-SFT-DPO", - "id": "princeton-nlp/Llama-3-Base-8B-SFT-DPO", - "developer": "princeton-nlp", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/princeton-nlp_Llama-3-Base-8B-SFT-DPO/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4111 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4666 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0415 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3104 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3867 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3078 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/princeton-nlp_llama-3-base-8b-sft-ipo.json b/data/models/princeton-nlp_llama-3-base-8b-sft-ipo.json deleted file mode 100644 index 9af72c4520de691e3c809082a24256f245ed4d5c..0000000000000000000000000000000000000000 --- a/data/models/princeton-nlp_llama-3-base-8b-sft-ipo.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-Base-8B-SFT-IPO", - "id": "princeton-nlp/Llama-3-Base-8B-SFT-IPO", - "developer": "princeton-nlp", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/princeton-nlp_Llama-3-Base-8B-SFT-IPO/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4487 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.469 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0393 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2978 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3919 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3115 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/princeton-nlp_llama-3-base-8b-sft-kto.json b/data/models/princeton-nlp_llama-3-base-8b-sft-kto.json deleted file mode 100644 index 110504eb3dd9413a87e925b122fb435f52517d7c..0000000000000000000000000000000000000000 --- a/data/models/princeton-nlp_llama-3-base-8b-sft-kto.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-Base-8B-SFT-KTO", - "id": "princeton-nlp/Llama-3-Base-8B-SFT-KTO", - "developer": "princeton-nlp", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/princeton-nlp_Llama-3-Base-8B-SFT-KTO/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4523 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4693 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0529 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3054 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3842 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3054 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/princeton-nlp_llama-3-base-8b-sft-orpo.json b/data/models/princeton-nlp_llama-3-base-8b-sft-orpo.json deleted file mode 100644 index bad36337775260a09fe6f48724b948ab14c82e4e..0000000000000000000000000000000000000000 --- a/data/models/princeton-nlp_llama-3-base-8b-sft-orpo.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-Base-8B-SFT-ORPO", - "id": "princeton-nlp/Llama-3-Base-8B-SFT-ORPO", - "developer": "princeton-nlp", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/princeton-nlp_Llama-3-Base-8B-SFT-ORPO/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4517 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4734 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0468 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3138 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3707 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3083 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/princeton-nlp_llama-3-base-8b-sft-rdpo.json b/data/models/princeton-nlp_llama-3-base-8b-sft-rdpo.json deleted file mode 100644 index 1e1b75d9d049094915c073c03ece99cf11d16708..0000000000000000000000000000000000000000 --- a/data/models/princeton-nlp_llama-3-base-8b-sft-rdpo.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-Base-8B-SFT-RDPO", - "id": "princeton-nlp/Llama-3-Base-8B-SFT-RDPO", - "developer": "princeton-nlp", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/princeton-nlp_Llama-3-Base-8B-SFT-RDPO/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.448 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4662 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0574 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3062 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4027 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3014 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/princeton-nlp_llama-3-base-8b-sft-rrhf.json b/data/models/princeton-nlp_llama-3-base-8b-sft-rrhf.json deleted file mode 100644 index ed2d24ebef4a62cc78d1f0bc4010ca39a051cb5e..0000000000000000000000000000000000000000 --- a/data/models/princeton-nlp_llama-3-base-8b-sft-rrhf.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-Base-8B-SFT-RRHF", - "id": "princeton-nlp/Llama-3-Base-8B-SFT-RRHF", - "developer": "princeton-nlp", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/princeton-nlp_Llama-3-Base-8B-SFT-RRHF/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3357 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.452 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0453 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3054 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3722 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2889 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/princeton-nlp_llama-3-base-8b-sft-simpo.json b/data/models/princeton-nlp_llama-3-base-8b-sft-simpo.json deleted file mode 100644 index a180c3024590cca2ead7af3a85390bf560dc8450..0000000000000000000000000000000000000000 --- a/data/models/princeton-nlp_llama-3-base-8b-sft-simpo.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-Base-8B-SFT-SimPO", - "id": "princeton-nlp/Llama-3-Base-8B-SFT-SimPO", - "developer": "princeton-nlp", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/princeton-nlp_Llama-3-Base-8B-SFT-SimPO/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4685 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4741 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0551 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2886 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4127 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3105 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/princeton-nlp_llama-3-base-8b-sft-slic-hf.json b/data/models/princeton-nlp_llama-3-base-8b-sft-slic-hf.json deleted file mode 100644 index 7c1426a9be66d6304e53fddd5d474fbed97815a3..0000000000000000000000000000000000000000 --- a/data/models/princeton-nlp_llama-3-base-8b-sft-slic-hf.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-Base-8B-SFT-SLiC-HF", - "id": "princeton-nlp/Llama-3-Base-8B-SFT-SLiC-HF", - "developer": "princeton-nlp", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/princeton-nlp_Llama-3-Base-8B-SFT-SLiC-HF/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.489 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4704 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0506 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2869 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4091 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3063 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/princeton-nlp_llama-3-base-8b-sft.json b/data/models/princeton-nlp_llama-3-base-8b-sft.json deleted file mode 100644 index c1ca23bcb67168cdf2b76ffc60ee622583a0f6f0..0000000000000000000000000000000000000000 --- a/data/models/princeton-nlp_llama-3-base-8b-sft.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-Base-8B-SFT", - "id": "princeton-nlp/Llama-3-Base-8B-SFT", - "developer": "princeton-nlp", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/princeton-nlp_Llama-3-Base-8B-SFT/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2796 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4643 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.04 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2978 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4118 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3093 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/princeton-nlp_llama-3-instruct-8b-cpo-v0.2.json b/data/models/princeton-nlp_llama-3-instruct-8b-cpo-v0.2.json deleted file mode 100644 index bad0818251c00c777e5f67fd3204586f7e0c3581..0000000000000000000000000000000000000000 --- a/data/models/princeton-nlp_llama-3-instruct-8b-cpo-v0.2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-Instruct-8B-CPO-v0.2", - "id": "princeton-nlp/Llama-3-Instruct-8B-CPO-v0.2", - "developer": "princeton-nlp", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/princeton-nlp_Llama-3-Instruct-8B-CPO-v0.2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7506 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5027 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.108 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2609 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3619 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3706 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/princeton-nlp_llama-3-instruct-8b-cpo.json b/data/models/princeton-nlp_llama-3-instruct-8b-cpo.json deleted file mode 100644 index 700415f8efc26a7dc27eee38c9875aeca89533f5..0000000000000000000000000000000000000000 --- a/data/models/princeton-nlp_llama-3-instruct-8b-cpo.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-Instruct-8B-CPO", - "id": "princeton-nlp/Llama-3-Instruct-8B-CPO", - "developer": "princeton-nlp", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/princeton-nlp_Llama-3-Instruct-8B-CPO/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7293 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4999 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0989 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2601 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3514 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3652 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/princeton-nlp_llama-3-instruct-8b-dpo-v0.2.json b/data/models/princeton-nlp_llama-3-instruct-8b-dpo-v0.2.json deleted file mode 100644 index 0170700fd07445c946046552a20dd89143a3e1e3..0000000000000000000000000000000000000000 --- a/data/models/princeton-nlp_llama-3-instruct-8b-dpo-v0.2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-Instruct-8B-DPO-v0.2", - "id": "princeton-nlp/Llama-3-Instruct-8B-DPO-v0.2", - "developer": "princeton-nlp", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/princeton-nlp_Llama-3-Instruct-8B-DPO-v0.2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7208 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5056 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0899 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2869 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3844 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3769 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/princeton-nlp_llama-3-instruct-8b-dpo.json b/data/models/princeton-nlp_llama-3-instruct-8b-dpo.json deleted file mode 100644 index b70d4b6f81deac41dfdff2988d13ddfbb77dc5f1..0000000000000000000000000000000000000000 --- a/data/models/princeton-nlp_llama-3-instruct-8b-dpo.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-Instruct-8B-DPO", - "id": "princeton-nlp/Llama-3-Instruct-8B-DPO", - "developer": "princeton-nlp", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/princeton-nlp_Llama-3-Instruct-8B-DPO/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6757 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4991 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0846 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2718 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3738 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3665 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/princeton-nlp_llama-3-instruct-8b-kto-v0.2.json b/data/models/princeton-nlp_llama-3-instruct-8b-kto-v0.2.json deleted file mode 100644 index 730e392a701e6c64b61a633de46afe5b797511c7..0000000000000000000000000000000000000000 --- a/data/models/princeton-nlp_llama-3-instruct-8b-kto-v0.2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-Instruct-8B-KTO-v0.2", - "id": "princeton-nlp/Llama-3-Instruct-8B-KTO-v0.2", - "developer": "princeton-nlp", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/princeton-nlp_Llama-3-Instruct-8B-KTO-v0.2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.729 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.508 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0997 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2601 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3777 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3668 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/princeton-nlp_llama-3-instruct-8b-kto.json b/data/models/princeton-nlp_llama-3-instruct-8b-kto.json deleted file mode 100644 index 85df188758ba2f61e062cfca20d7a773e50cc943..0000000000000000000000000000000000000000 --- a/data/models/princeton-nlp_llama-3-instruct-8b-kto.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-Instruct-8B-KTO", - "id": "princeton-nlp/Llama-3-Instruct-8B-KTO", - "developer": "princeton-nlp", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/princeton-nlp_Llama-3-Instruct-8B-KTO/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6864 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4982 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0725 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.276 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3698 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3599 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/princeton-nlp_llama-3-instruct-8b-orpo-v0.2.json b/data/models/princeton-nlp_llama-3-instruct-8b-orpo-v0.2.json deleted file mode 100644 index a42869b201d950e5032dd60145adbb6afe803062..0000000000000000000000000000000000000000 --- a/data/models/princeton-nlp_llama-3-instruct-8b-orpo-v0.2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-Instruct-8B-ORPO-v0.2", - "id": "princeton-nlp/Llama-3-Instruct-8B-ORPO-v0.2", - "developer": "princeton-nlp", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/princeton-nlp_Llama-3-Instruct-8B-ORPO-v0.2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7633 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5078 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.102 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2836 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.378 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3731 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/princeton-nlp_llama-3-instruct-8b-orpo.json b/data/models/princeton-nlp_llama-3-instruct-8b-orpo.json deleted file mode 100644 index 4e8466b23f4a9fa2cfcc6074816b05aa2b656d45..0000000000000000000000000000000000000000 --- a/data/models/princeton-nlp_llama-3-instruct-8b-orpo.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-Instruct-8B-ORPO", - "id": "princeton-nlp/Llama-3-Instruct-8B-ORPO", - "developer": "princeton-nlp", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/princeton-nlp_Llama-3-Instruct-8B-ORPO/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7128 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5001 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0785 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2584 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3502 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3646 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/princeton-nlp_llama-3-instruct-8b-rdpo-v0.2.json b/data/models/princeton-nlp_llama-3-instruct-8b-rdpo-v0.2.json deleted file mode 100644 index 89c007cfdd0efff1330ab5bece1e03c6562142d0..0000000000000000000000000000000000000000 --- a/data/models/princeton-nlp_llama-3-instruct-8b-rdpo-v0.2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-Instruct-8B-RDPO-v0.2", - "id": "princeton-nlp/Llama-3-Instruct-8B-RDPO-v0.2", - "developer": "princeton-nlp", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/princeton-nlp_Llama-3-Instruct-8B-RDPO-v0.2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7077 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5049 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0869 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2928 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3804 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3774 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/princeton-nlp_llama-3-instruct-8b-rdpo.json b/data/models/princeton-nlp_llama-3-instruct-8b-rdpo.json deleted file mode 100644 index 87928b3ddad4683789d9cb9b426cb2ee18efda1b..0000000000000000000000000000000000000000 --- a/data/models/princeton-nlp_llama-3-instruct-8b-rdpo.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-Instruct-8B-RDPO", - "id": "princeton-nlp/Llama-3-Instruct-8B-RDPO", - "developer": "princeton-nlp", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/princeton-nlp_Llama-3-Instruct-8B-RDPO/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.666 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5034 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0846 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2827 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3752 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3607 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/princeton-nlp_llama-3-instruct-8b-rrhf-v0.2.json b/data/models/princeton-nlp_llama-3-instruct-8b-rrhf-v0.2.json deleted file mode 100644 index a2cf421c92bfadc341b9a174f150d46ebf0a7742..0000000000000000000000000000000000000000 --- a/data/models/princeton-nlp_llama-3-instruct-8b-rrhf-v0.2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-Instruct-8B-RRHF-v0.2", - "id": "princeton-nlp/Llama-3-Instruct-8B-RRHF-v0.2", - "developer": "princeton-nlp", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/princeton-nlp_Llama-3-Instruct-8B-RRHF-v0.2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7125 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4984 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0876 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2601 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3738 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3482 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/princeton-nlp_llama-3-instruct-8b-rrhf.json b/data/models/princeton-nlp_llama-3-instruct-8b-rrhf.json deleted file mode 100644 index 4c4dd85b8927139673a29d83d5a6a3a9c7130972..0000000000000000000000000000000000000000 --- a/data/models/princeton-nlp_llama-3-instruct-8b-rrhf.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-Instruct-8B-RRHF", - "id": "princeton-nlp/Llama-3-Instruct-8B-RRHF", - "developer": "princeton-nlp", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/princeton-nlp_Llama-3-Instruct-8B-RRHF/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7275 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4911 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0967 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2802 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3476 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3644 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/princeton-nlp_llama-3-instruct-8b-simpo-v0.2.json b/data/models/princeton-nlp_llama-3-instruct-8b-simpo-v0.2.json deleted file mode 100644 index e470e1bc36690c2b420d8bd1fe742f73db9a586d..0000000000000000000000000000000000000000 --- a/data/models/princeton-nlp_llama-3-instruct-8b-simpo-v0.2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-Instruct-8B-SimPO-v0.2", - "id": "princeton-nlp/Llama-3-Instruct-8B-SimPO-v0.2", - "developer": "princeton-nlp", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/princeton-nlp_Llama-3-Instruct-8B-SimPO-v0.2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6809 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5038 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.074 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3012 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3988 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3622 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/princeton-nlp_llama-3-instruct-8b-simpo.json b/data/models/princeton-nlp_llama-3-instruct-8b-simpo.json deleted file mode 100644 index 2d66b83e916b126fd3bbe852d2226bed76a1fd2c..0000000000000000000000000000000000000000 --- a/data/models/princeton-nlp_llama-3-instruct-8b-simpo.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-Instruct-8B-SimPO", - "id": "princeton-nlp/Llama-3-Instruct-8B-SimPO", - "developer": "princeton-nlp", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/princeton-nlp_Llama-3-Instruct-8B-SimPO/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6504 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4845 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0861 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2936 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3948 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3489 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/princeton-nlp_llama-3-instruct-8b-slic-hf-v0.2.json b/data/models/princeton-nlp_llama-3-instruct-8b-slic-hf-v0.2.json deleted file mode 100644 index e65990a384d1bca052acaa6f3749b06097509a90..0000000000000000000000000000000000000000 --- a/data/models/princeton-nlp_llama-3-instruct-8b-slic-hf-v0.2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-Instruct-8B-SLiC-HF-v0.2", - "id": "princeton-nlp/Llama-3-Instruct-8B-SLiC-HF-v0.2", - "developer": "princeton-nlp", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/princeton-nlp_Llama-3-Instruct-8B-SLiC-HF-v0.2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.711 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4984 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0876 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2601 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3738 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3482 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/princeton-nlp_llama-3-instruct-8b-slic-hf.json b/data/models/princeton-nlp_llama-3-instruct-8b-slic-hf.json deleted file mode 100644 index 69d2ad40dcadc71c89c1d6c1892723613a72d1de..0000000000000000000000000000000000000000 --- a/data/models/princeton-nlp_llama-3-instruct-8b-slic-hf.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-Instruct-8B-SLiC-HF", - "id": "princeton-nlp/Llama-3-Instruct-8B-SLiC-HF", - "developer": "princeton-nlp", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/princeton-nlp_Llama-3-Instruct-8B-SLiC-HF/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.74 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5029 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0974 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2861 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3723 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3585 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/princeton-nlp_mistral-7b-base-sft-cpo.json b/data/models/princeton-nlp_mistral-7b-base-sft-cpo.json deleted file mode 100644 index fc737b7d8a6c5901ae6280b6eb2e969fc2bbfda3..0000000000000000000000000000000000000000 --- a/data/models/princeton-nlp_mistral-7b-base-sft-cpo.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mistral-7B-Base-SFT-CPO", - "id": "princeton-nlp/Mistral-7B-Base-SFT-CPO", - "developer": "princeton-nlp", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/princeton-nlp_Mistral-7B-Base-SFT-CPO/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4655 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4382 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0279 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2919 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4071 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2651 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/princeton-nlp_mistral-7b-base-sft-dpo.json b/data/models/princeton-nlp_mistral-7b-base-sft-dpo.json deleted file mode 100644 index 35a44db5e96a57a073c423a49dfb70fa67afee50..0000000000000000000000000000000000000000 --- a/data/models/princeton-nlp_mistral-7b-base-sft-dpo.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mistral-7B-Base-SFT-DPO", - "id": "princeton-nlp/Mistral-7B-Base-SFT-DPO", - "developer": "princeton-nlp", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/princeton-nlp_Mistral-7B-Base-SFT-DPO/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4403 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.435 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0211 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2727 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4122 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2645 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/princeton-nlp_mistral-7b-base-sft-ipo.json b/data/models/princeton-nlp_mistral-7b-base-sft-ipo.json deleted file mode 100644 index b61cbbb80b33199ba59713a7e02d582dfe0bb7a2..0000000000000000000000000000000000000000 --- a/data/models/princeton-nlp_mistral-7b-base-sft-ipo.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mistral-7B-Base-SFT-IPO", - "id": "princeton-nlp/Mistral-7B-Base-SFT-IPO", - "developer": "princeton-nlp", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/princeton-nlp_Mistral-7B-Base-SFT-IPO/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.483 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4458 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0287 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2802 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3776 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2792 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/princeton-nlp_mistral-7b-base-sft-kto.json b/data/models/princeton-nlp_mistral-7b-base-sft-kto.json deleted file mode 100644 index 6090aae26fb81f5b662c804ec4488bab681b9158..0000000000000000000000000000000000000000 --- a/data/models/princeton-nlp_mistral-7b-base-sft-kto.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mistral-7B-Base-SFT-KTO", - "id": "princeton-nlp/Mistral-7B-Base-SFT-KTO", - "developer": "princeton-nlp", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/princeton-nlp_Mistral-7B-Base-SFT-KTO/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4785 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4476 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0393 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2903 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4368 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2872 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/princeton-nlp_mistral-7b-base-sft-rdpo.json b/data/models/princeton-nlp_mistral-7b-base-sft-rdpo.json deleted file mode 100644 index c9f47be86670b37412ca922ebe0d77d2df7b2277..0000000000000000000000000000000000000000 --- a/data/models/princeton-nlp_mistral-7b-base-sft-rdpo.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mistral-7B-Base-SFT-RDPO", - "id": "princeton-nlp/Mistral-7B-Base-SFT-RDPO", - "developer": "princeton-nlp", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/princeton-nlp_Mistral-7B-Base-SFT-RDPO/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4606 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.444 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0219 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2777 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3579 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2777 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/princeton-nlp_mistral-7b-base-sft-rrhf.json b/data/models/princeton-nlp_mistral-7b-base-sft-rrhf.json deleted file mode 100644 index ce19fc5f0fb11a16f3e73d254ba5e924f6b889d0..0000000000000000000000000000000000000000 --- a/data/models/princeton-nlp_mistral-7b-base-sft-rrhf.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mistral-7B-Base-SFT-RRHF", - "id": "princeton-nlp/Mistral-7B-Base-SFT-RRHF", - "developer": "princeton-nlp", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/princeton-nlp_Mistral-7B-Base-SFT-RRHF/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4407 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4281 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0249 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2903 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4187 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2398 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/princeton-nlp_mistral-7b-base-sft-simpo.json b/data/models/princeton-nlp_mistral-7b-base-sft-simpo.json deleted file mode 100644 index 5b9b005d0e6bfb3788edb0812c831c56bbc2c880..0000000000000000000000000000000000000000 --- a/data/models/princeton-nlp_mistral-7b-base-sft-simpo.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mistral-7B-Base-SFT-SimPO", - "id": "princeton-nlp/Mistral-7B-Base-SFT-SimPO", - "developer": "princeton-nlp", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/princeton-nlp_Mistral-7B-Base-SFT-SimPO/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4701 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4398 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0144 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2836 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3971 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2702 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/princeton-nlp_mistral-7b-base-sft-slic-hf.json b/data/models/princeton-nlp_mistral-7b-base-sft-slic-hf.json deleted file mode 100644 index b2108a5c7a39961e9c30a9850cf81031d4b3ad69..0000000000000000000000000000000000000000 --- a/data/models/princeton-nlp_mistral-7b-base-sft-slic-hf.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mistral-7B-Base-SFT-SLiC-HF", - "id": "princeton-nlp/Mistral-7B-Base-SFT-SLiC-HF", - "developer": "princeton-nlp", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/princeton-nlp_Mistral-7B-Base-SFT-SLiC-HF/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5127 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4422 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0355 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2919 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4261 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2781 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/princeton-nlp_mistral-7b-instruct-cpo.json b/data/models/princeton-nlp_mistral-7b-instruct-cpo.json deleted file mode 100644 index 58728d70608f1ce078020eef05530caf7df3bdbb..0000000000000000000000000000000000000000 --- a/data/models/princeton-nlp_mistral-7b-instruct-cpo.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mistral-7B-Instruct-CPO", - "id": "princeton-nlp/Mistral-7B-Instruct-CPO", - "developer": "princeton-nlp", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/princeton-nlp_Mistral-7B-Instruct-CPO/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4203 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4069 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0204 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2659 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4178 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2701 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/princeton-nlp_mistral-7b-instruct-dpo.json b/data/models/princeton-nlp_mistral-7b-instruct-dpo.json deleted file mode 100644 index fd60fdbccfde6f201135db299680ab9717e9c221..0000000000000000000000000000000000000000 --- a/data/models/princeton-nlp_mistral-7b-instruct-dpo.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mistral-7B-Instruct-DPO", - "id": "princeton-nlp/Mistral-7B-Instruct-DPO", - "developer": "princeton-nlp", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/princeton-nlp_Mistral-7B-Instruct-DPO/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5176 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.406 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.031 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2685 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3833 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2749 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/princeton-nlp_mistral-7b-instruct-ipo.json b/data/models/princeton-nlp_mistral-7b-instruct-ipo.json deleted file mode 100644 index 9c105be5569259bf3a4ffdb1e59458eab9650bad..0000000000000000000000000000000000000000 --- a/data/models/princeton-nlp_mistral-7b-instruct-ipo.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mistral-7B-Instruct-IPO", - "id": "princeton-nlp/Mistral-7B-Instruct-IPO", - "developer": "princeton-nlp", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/princeton-nlp_Mistral-7B-Instruct-IPO/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4929 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4322 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0204 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2735 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4324 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2708 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/princeton-nlp_mistral-7b-instruct-kto.json b/data/models/princeton-nlp_mistral-7b-instruct-kto.json deleted file mode 100644 index 22eed31e9798802b49b1ce53b863fe85784ddacf..0000000000000000000000000000000000000000 --- a/data/models/princeton-nlp_mistral-7b-instruct-kto.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mistral-7B-Instruct-KTO", - "id": "princeton-nlp/Mistral-7B-Instruct-KTO", - "developer": "princeton-nlp", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/princeton-nlp_Mistral-7B-Instruct-KTO/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4908 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.414 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0264 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2735 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3953 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2812 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/princeton-nlp_mistral-7b-instruct-orpo.json b/data/models/princeton-nlp_mistral-7b-instruct-orpo.json deleted file mode 100644 index a0ed62c2775da619d7c0b4308d86ab51cba7b565..0000000000000000000000000000000000000000 --- a/data/models/princeton-nlp_mistral-7b-instruct-orpo.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mistral-7B-Instruct-ORPO", - "id": "princeton-nlp/Mistral-7B-Instruct-ORPO", - "developer": "princeton-nlp", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/princeton-nlp_Mistral-7B-Instruct-ORPO/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.472 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4104 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0295 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2743 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3912 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2662 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/princeton-nlp_mistral-7b-instruct-rdpo.json b/data/models/princeton-nlp_mistral-7b-instruct-rdpo.json deleted file mode 100644 index dfc2d93a498bbbc89402167bed4198145356cc72..0000000000000000000000000000000000000000 --- a/data/models/princeton-nlp_mistral-7b-instruct-rdpo.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mistral-7B-Instruct-RDPO", - "id": "princeton-nlp/Mistral-7B-Instruct-RDPO", - "developer": "princeton-nlp", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/princeton-nlp_Mistral-7B-Instruct-RDPO/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4887 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.405 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0249 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2802 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3873 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2777 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/princeton-nlp_mistral-7b-instruct-rrhf.json b/data/models/princeton-nlp_mistral-7b-instruct-rrhf.json deleted file mode 100644 index f15c6458c7a9157401d6960aab3abfe108f7cbc1..0000000000000000000000000000000000000000 --- a/data/models/princeton-nlp_mistral-7b-instruct-rrhf.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mistral-7B-Instruct-RRHF", - "id": "princeton-nlp/Mistral-7B-Instruct-RRHF", - "developer": "princeton-nlp", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/princeton-nlp_Mistral-7B-Instruct-RRHF/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.496 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.419 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0279 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.276 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3979 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2651 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/princeton-nlp_mistral-7b-instruct-simpo.json b/data/models/princeton-nlp_mistral-7b-instruct-simpo.json deleted file mode 100644 index a4eaa1187190f66c026a1216b32412ceca5ea723..0000000000000000000000000000000000000000 --- a/data/models/princeton-nlp_mistral-7b-instruct-simpo.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mistral-7B-Instruct-SimPO", - "id": "princeton-nlp/Mistral-7B-Instruct-SimPO", - "developer": "princeton-nlp", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/princeton-nlp_Mistral-7B-Instruct-SimPO/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4687 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4507 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0287 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2785 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4098 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2797 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/princeton-nlp_mistral-7b-instruct-slic-hf.json b/data/models/princeton-nlp_mistral-7b-instruct-slic-hf.json deleted file mode 100644 index 4d06d09868e2bd2e0c62b5ee6314822725328f16..0000000000000000000000000000000000000000 --- a/data/models/princeton-nlp_mistral-7b-instruct-slic-hf.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mistral-7B-Instruct-SLiC-HF", - "id": "princeton-nlp/Mistral-7B-Instruct-SLiC-HF", - "developer": "princeton-nlp", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/princeton-nlp_Mistral-7B-Instruct-SLiC-HF/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5115 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.404 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0174 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2727 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3913 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2715 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/princeton-nlp_sheared-llama-1.3b.json b/data/models/princeton-nlp_sheared-llama-1.3b.json deleted file mode 100644 index 84432c007301f370426921998539bd6ab378aa67..0000000000000000000000000000000000000000 --- a/data/models/princeton-nlp_sheared-llama-1.3b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Sheared-LLaMA-1.3B", - "id": "princeton-nlp/Sheared-LLaMA-1.3B", - "developer": "princeton-nlp", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.3" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/princeton-nlp_Sheared-LLaMA-1.3B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2198 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3197 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0128 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2399 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3713 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1171 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/princeton-nlp_sheared-llama-2.7b.json b/data/models/princeton-nlp_sheared-llama-2.7b.json deleted file mode 100644 index b928e91604c7af24a0f486b97af64cc13a1c41ef..0000000000000000000000000000000000000000 --- a/data/models/princeton-nlp_sheared-llama-2.7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Sheared-LLaMA-2.7B", - "id": "princeton-nlp/Sheared-LLaMA-2.7B", - "developer": "princeton-nlp", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "2.7" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/princeton-nlp_Sheared-LLaMA-2.7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2417 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3259 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0128 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2752 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3567 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1187 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_bellatrix-1.5b-xelite.json b/data/models/prithivmlmods_bellatrix-1.5b-xelite.json deleted file mode 100644 index 92ce9e31e563b453a9627a00014b751a00904247..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_bellatrix-1.5b-xelite.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Bellatrix-1.5B-xElite", - "id": "prithivMLmods/Bellatrix-1.5B-xElite", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.777" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Bellatrix-1.5B-xElite/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1964 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3501 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.287 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2785 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3619 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1657 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_bellatrix-tiny-1.5b-r1.json b/data/models/prithivmlmods_bellatrix-tiny-1.5b-r1.json deleted file mode 100644 index 68745515e97337471835929717fe01e4d3d5aa0a..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_bellatrix-tiny-1.5b-r1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Bellatrix-Tiny-1.5B-R1", - "id": "prithivMLmods/Bellatrix-Tiny-1.5B-R1", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.544" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Bellatrix-Tiny-1.5B-R1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3352 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4022 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0604 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2987 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3683 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2751 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_bellatrix-tiny-1b-v2.json b/data/models/prithivmlmods_bellatrix-tiny-1b-v2.json deleted file mode 100644 index 97b6e84173c875132ead0be8dfff7c632e33a38f..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_bellatrix-tiny-1b-v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Bellatrix-Tiny-1B-v2", - "id": "prithivMLmods/Bellatrix-Tiny-1B-v2", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.236" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Bellatrix-Tiny-1B-v2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.151 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3268 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0287 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2727 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.343 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1493 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_blaze-14b-xelite.json b/data/models/prithivmlmods_blaze-14b-xelite.json deleted file mode 100644 index 5e55b88a4f0c187abcb4489ef202a3a591c70bd0..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_blaze-14b-xelite.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Blaze-14B-xElite", - "id": "prithivMLmods/Blaze-14B-xElite", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Blaze-14B-xElite/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0363 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6628 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3693 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3943 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4625 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5111 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_calcium-opus-14b-elite-1m.json b/data/models/prithivmlmods_calcium-opus-14b-elite-1m.json deleted file mode 100644 index e8d4704275153d994cd688d891850e49009faf27..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_calcium-opus-14b-elite-1m.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Calcium-Opus-14B-Elite-1M", - "id": "prithivMLmods/Calcium-Opus-14B-Elite-1M", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Calcium-Opus-14B-Elite-1M/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5613 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6329 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4456 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3523 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4676 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5152 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_calcium-opus-14b-elite-stock.json b/data/models/prithivmlmods_calcium-opus-14b-elite-stock.json deleted file mode 100644 index 85eef81e7220bfba8ff9c13eea43dee937becb16..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_calcium-opus-14b-elite-stock.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Calcium-Opus-14B-Elite-Stock", - "id": "prithivMLmods/Calcium-Opus-14B-Elite-Stock", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Calcium-Opus-14B-Elite-Stock/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6143 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6329 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4668 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3683 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4808 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5284 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_calcium-opus-14b-elite.json b/data/models/prithivmlmods_calcium-opus-14b-elite.json deleted file mode 100644 index 746fd41957e7795e0f0f3751181ce9bfef3d8e70..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_calcium-opus-14b-elite.json +++ /dev/null @@ -1,275 +0,0 @@ -{ - "model_info": { - "name": "Calcium-Opus-14B-Elite", - "id": "prithivMLmods/Calcium-Opus-14B-Elite", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Calcium-Opus-14B-Elite/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6064 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6296 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3708 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3733 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4873 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5307 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Calcium-Opus-14B-Elite/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6052 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6317 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4789 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3742 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.486 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5302 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_calcium-opus-14b-elite2-r1.json b/data/models/prithivmlmods_calcium-opus-14b-elite2-r1.json deleted file mode 100644 index 8a71fd3083b0fdc905ea7e6c2cadaac79da8d5e8..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_calcium-opus-14b-elite2-r1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Calcium-Opus-14B-Elite2-R1", - "id": "prithivMLmods/Calcium-Opus-14B-Elite2-R1", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Calcium-Opus-14B-Elite2-R1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6326 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6362 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3338 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3909 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.49 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5248 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_calcium-opus-14b-elite2.json b/data/models/prithivmlmods_calcium-opus-14b-elite2.json deleted file mode 100644 index 15a6c6f69b0ae0f1f1fab54e3819ac1b6a6b2107..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_calcium-opus-14b-elite2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Calcium-Opus-14B-Elite2", - "id": "prithivMLmods/Calcium-Opus-14B-Elite2", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Calcium-Opus-14B-Elite2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6176 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6318 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.469 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.37 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.494 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5301 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_calcium-opus-14b-elite3.json b/data/models/prithivmlmods_calcium-opus-14b-elite3.json deleted file mode 100644 index 4cb4f7934537119ff1a097ecd5aa8e1c4ed0ab78..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_calcium-opus-14b-elite3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Calcium-Opus-14B-Elite3", - "id": "prithivMLmods/Calcium-Opus-14B-Elite3", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Calcium-Opus-14B-Elite3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5428 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.635 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4705 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3708 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4795 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5335 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_calcium-opus-14b-elite4.json b/data/models/prithivmlmods_calcium-opus-14b-elite4.json deleted file mode 100644 index d2ca00a8e6ee7d8e66562079c9126a3014891bdf..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_calcium-opus-14b-elite4.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Calcium-Opus-14B-Elite4", - "id": "prithivMLmods/Calcium-Opus-14B-Elite4", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Calcium-Opus-14B-Elite4/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6112 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6195 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3625 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3557 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4687 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5149 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_calcium-opus-14b-merge.json b/data/models/prithivmlmods_calcium-opus-14b-merge.json deleted file mode 100644 index df3064ff2f3f690d10482fbcc59b488ff7e8c06a..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_calcium-opus-14b-merge.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Calcium-Opus-14B-Merge", - "id": "prithivMLmods/Calcium-Opus-14B-Merge", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Calcium-Opus-14B-Merge/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4949 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6319 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4637 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3708 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4861 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5356 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_calcium-opus-20b-v1.json b/data/models/prithivmlmods_calcium-opus-20b-v1.json deleted file mode 100644 index 003a019f63be55a89b5cf32a8416e23a3d285aa0..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_calcium-opus-20b-v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Calcium-Opus-20B-v1", - "id": "prithivMLmods/Calcium-Opus-20B-v1", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "19.173" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Calcium-Opus-20B-v1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3093 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.599 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3618 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3532 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4943 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4734 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_coco-7b-instruct-1m.json b/data/models/prithivmlmods_coco-7b-instruct-1m.json deleted file mode 100644 index 659c587f05421cee4683aa337bd2e255edd8b24b..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_coco-7b-instruct-1m.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "COCO-7B-Instruct-1M", - "id": "prithivMLmods/COCO-7B-Instruct-1M", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_COCO-7B-Instruct-1M/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4743 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.541 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3497 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3079 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4382 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4186 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_codepy-deepthink-3b.json b/data/models/prithivmlmods_codepy-deepthink-3b.json deleted file mode 100644 index 24f4fd37ade6561a49d561b132af920d2cb08009..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_codepy-deepthink-3b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Codepy-Deepthink-3B", - "id": "prithivMLmods/Codepy-Deepthink-3B", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Codepy-Deepthink-3B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4327 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4259 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1156 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2794 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.331 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.309 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_coma-ii-14b.json b/data/models/prithivmlmods_coma-ii-14b.json deleted file mode 100644 index 9d2dbf4a020da51d7b22f0bfa167eda1dee89fdf..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_coma-ii-14b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Coma-II-14B", - "id": "prithivMLmods/Coma-II-14B", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Coma-II-14B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4168 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6321 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5514 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4002 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5351 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.504 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_condor-opus-14b-exp.json b/data/models/prithivmlmods_condor-opus-14b-exp.json deleted file mode 100644 index 4e740e74143af70e0b0d171130e8f1f59f2d7b8d..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_condor-opus-14b-exp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Condor-Opus-14B-Exp", - "id": "prithivMLmods/Condor-Opus-14B-Exp", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Condor-Opus-14B-Exp/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4043 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6154 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5227 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3918 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5194 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5014 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_cygnus-ii-14b.json b/data/models/prithivmlmods_cygnus-ii-14b.json deleted file mode 100644 index b540185afe4596266085f9f3e9e7fa1af082ced9..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_cygnus-ii-14b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Cygnus-II-14B", - "id": "prithivMLmods/Cygnus-II-14B", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Cygnus-II-14B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6184 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6661 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4396 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3876 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4688 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5391 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_deepthink-llama-3-8b-preview.json b/data/models/prithivmlmods_deepthink-llama-3-8b-preview.json deleted file mode 100644 index ca18b70dcce3b646b24226f7c387c5642d5037e7..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_deepthink-llama-3-8b-preview.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Deepthink-Llama-3-8B-Preview", - "id": "prithivMLmods/Deepthink-Llama-3-8B-Preview", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Deepthink-Llama-3-8B-Preview/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2955 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4665 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.355 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3163 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3707 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2739 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_deepthink-reasoning-14b.json b/data/models/prithivmlmods_deepthink-reasoning-14b.json deleted file mode 100644 index 417e0a6c806f6d9e65da0584b3a44c87272e3d49..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_deepthink-reasoning-14b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Deepthink-Reasoning-14B", - "id": "prithivMLmods/Deepthink-Reasoning-14B", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Deepthink-Reasoning-14B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5424 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6334 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.423 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3666 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4732 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5296 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_deepthink-reasoning-7b.json b/data/models/prithivmlmods_deepthink-reasoning-7b.json deleted file mode 100644 index fb7f598aa087272ccdd96a40abb166ea02f019af..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_deepthink-reasoning-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Deepthink-Reasoning-7B", - "id": "prithivMLmods/Deepthink-Reasoning-7B", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Deepthink-Reasoning-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.484 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5505 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3346 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2995 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4432 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4349 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_dinobot-opus-14b-exp.json b/data/models/prithivmlmods_dinobot-opus-14b-exp.json deleted file mode 100644 index 7179706f8c91e91080c3a2e08b447bcf46f97d3e..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_dinobot-opus-14b-exp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Dinobot-Opus-14B-Exp", - "id": "prithivMLmods/Dinobot-Opus-14B-Exp", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Dinobot-Opus-14B-Exp/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.824 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.637 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5317 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3247 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.426 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4979 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_elita-0.1-distilled-r1-abliterated.json b/data/models/prithivmlmods_elita-0.1-distilled-r1-abliterated.json deleted file mode 100644 index bb023d8ac2120474664502dcff84df2a31389507..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_elita-0.1-distilled-r1-abliterated.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Elita-0.1-Distilled-R1-abliterated", - "id": "prithivMLmods/Elita-0.1-Distilled-R1-abliterated", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Elita-0.1-Distilled-R1-abliterated/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3542 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3828 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3066 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2659 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.366 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2758 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_elita-1.json b/data/models/prithivmlmods_elita-1.json deleted file mode 100644 index 5d7edacfdc03d293a9bb9d62b3e20a26ecd21248..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_elita-1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Elita-1", - "id": "prithivMLmods/Elita-1", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Elita-1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4906 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.652 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3429 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3758 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4834 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5381 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_epimetheus-14b-axo.json b/data/models/prithivmlmods_epimetheus-14b-axo.json deleted file mode 100644 index 40f14d884e2a32af32df40a0bb9973089df23349..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_epimetheus-14b-axo.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Epimetheus-14B-Axo", - "id": "prithivMLmods/Epimetheus-14B-Axo", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Epimetheus-14B-Axo/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5546 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6613 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4101 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3926 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.482 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5304 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_equuleus-opus-14b-exp.json b/data/models/prithivmlmods_equuleus-opus-14b-exp.json deleted file mode 100644 index 7b7bae87aa80bb8043ce24c367b1f3184d15c0d9..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_equuleus-opus-14b-exp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Equuleus-Opus-14B-Exp", - "id": "prithivMLmods/Equuleus-Opus-14B-Exp", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Equuleus-Opus-14B-Exp/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7001 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6434 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4585 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3867 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4952 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5374 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_eridanus-opus-14b-r999.json b/data/models/prithivmlmods_eridanus-opus-14b-r999.json deleted file mode 100644 index bfcda1ec0f9b9438749b0c9ab6d720cf98422d45..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_eridanus-opus-14b-r999.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Eridanus-Opus-14B-r999", - "id": "prithivMLmods/Eridanus-Opus-14B-r999", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Eridanus-Opus-14B-r999/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6386 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6584 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.386 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3943 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4769 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5362 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_evac-opus-14b-exp.json b/data/models/prithivmlmods_evac-opus-14b-exp.json deleted file mode 100644 index 3af6327c197fb9d9423c230217a55836a2107d8d..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_evac-opus-14b-exp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Evac-Opus-14B-Exp", - "id": "prithivMLmods/Evac-Opus-14B-Exp", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Evac-Opus-14B-Exp/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5916 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6475 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4215 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3884 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4728 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5317 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_fastthink-0.5b-tiny.json b/data/models/prithivmlmods_fastthink-0.5b-tiny.json deleted file mode 100644 index c5b07046e8d04c388b73f08f97c4fd39606b60fc..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_fastthink-0.5b-tiny.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "FastThink-0.5B-Tiny", - "id": "prithivMLmods/FastThink-0.5B-Tiny", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_FastThink-0.5B-Tiny/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.258 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3206 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0204 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2609 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3566 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1649 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_gaea-opus-14b-exp.json b/data/models/prithivmlmods_gaea-opus-14b-exp.json deleted file mode 100644 index 2209105e3052b21622c4fa2f10a74d52d467e192..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_gaea-opus-14b-exp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Gaea-Opus-14B-Exp", - "id": "prithivMLmods/Gaea-Opus-14B-Exp", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Gaea-Opus-14B-Exp/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5956 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.656 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4275 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3909 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4859 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5401 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_galactic-qwen-14b-exp1.json b/data/models/prithivmlmods_galactic-qwen-14b-exp1.json deleted file mode 100644 index 77956494fbbf68556ced8ac04edc02b991834459..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_galactic-qwen-14b-exp1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Galactic-Qwen-14B-Exp1", - "id": "prithivMLmods/Galactic-Qwen-14B-Exp1", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Galactic-Qwen-14B-Exp1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5832 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6582 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4018 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3935 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4781 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5396 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_galactic-qwen-14b-exp2.json b/data/models/prithivmlmods_galactic-qwen-14b-exp2.json deleted file mode 100644 index 5466fc33ff5a132f48472deab380fd342ae1ec49..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_galactic-qwen-14b-exp2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Galactic-Qwen-14B-Exp2", - "id": "prithivMLmods/Galactic-Qwen-14B-Exp2", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Galactic-Qwen-14B-Exp2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.662 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7203 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3474 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3993 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5354 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5691 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_gauss-opus-14b-r999.json b/data/models/prithivmlmods_gauss-opus-14b-r999.json deleted file mode 100644 index c7405beaf4ae3fd7f4c4576645098b0f18a842f9..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_gauss-opus-14b-r999.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Gauss-Opus-14B-R999", - "id": "prithivMLmods/Gauss-Opus-14B-R999", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Gauss-Opus-14B-R999/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3907 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6228 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5755 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3918 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5338 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5007 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_gwq-9b-preview.json b/data/models/prithivmlmods_gwq-9b-preview.json deleted file mode 100644 index 6cd98eb7fe124d089442c780bc330cf2c7952f3e..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_gwq-9b-preview.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "GWQ-9B-Preview", - "id": "prithivMLmods/GWQ-9B-Preview", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "9.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_GWQ-9B-Preview/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5066 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5806 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2266 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3398 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4951 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3984 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_gwq-9b-preview2.json b/data/models/prithivmlmods_gwq-9b-preview2.json deleted file mode 100644 index 5e8a25baf711907b4b31f54ff003acfa05074259..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_gwq-9b-preview2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "GWQ-9B-Preview2", - "id": "prithivMLmods/GWQ-9B-Preview2", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "9.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_GWQ-9B-Preview2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5209 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5797 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2372 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3263 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.486 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3997 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_gwq2b.json b/data/models/prithivmlmods_gwq2b.json deleted file mode 100644 index a95a7e99c15944aac6d53cbed6949b918d588cb5..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_gwq2b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "GWQ2b", - "id": "prithivMLmods/GWQ2b", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "2.614" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_GWQ2b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4115 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4143 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0627 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2827 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4311 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2473 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_jolt-v0.1.json b/data/models/prithivmlmods_jolt-v0.1.json deleted file mode 100644 index 081d6eb9f9572fcfbd258632ea02c5ddb830f1b4..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_jolt-v0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Jolt-v0.1", - "id": "prithivMLmods/Jolt-v0.1", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Jolt-v0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5092 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6521 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3565 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.38 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4847 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5386 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_lacerta-opus-14b-elite8.json b/data/models/prithivmlmods_lacerta-opus-14b-elite8.json deleted file mode 100644 index 1e3309af09ba3903fb3db9f76a457199562cf727..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_lacerta-opus-14b-elite8.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Lacerta-Opus-14B-Elite8", - "id": "prithivMLmods/Lacerta-Opus-14B-Elite8", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Lacerta-Opus-14B-Elite8/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6141 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6401 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3648 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3784 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4635 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5322 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_llama-3.1-5b-instruct.json b/data/models/prithivmlmods_llama-3.1-5b-instruct.json deleted file mode 100644 index b89e7260898d3ca9270ad1eed95d8e2887f8b203..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_llama-3.1-5b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.1-5B-Instruct", - "id": "prithivMLmods/Llama-3.1-5B-Instruct", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "5.413" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Llama-3.1-5B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1407 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3051 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0151 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2643 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.354 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1184 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_llama-3.1-8b-open-sft.json b/data/models/prithivmlmods_llama-3.1-8b-open-sft.json deleted file mode 100644 index bff97e7fa88fea63ea29b80d118e8e387fa7346e..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_llama-3.1-8b-open-sft.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.1-8B-Open-SFT", - "id": "prithivMLmods/Llama-3.1-8B-Open-SFT", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Llama-3.1-8B-Open-SFT/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4123 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4968 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1216 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3096 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3904 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3522 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_llama-3.2-3b-math-oct.json b/data/models/prithivmlmods_llama-3.2-3b-math-oct.json deleted file mode 100644 index ddea2f873326541fd5a0a808219ece0c243f2bd3..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_llama-3.2-3b-math-oct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.2-3B-Math-Oct", - "id": "prithivMLmods/Llama-3.2-3B-Math-Oct", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Llama-3.2-3B-Math-Oct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4585 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4372 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1156 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2584 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.347 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2911 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_llama-3.2-6b-algocode.json b/data/models/prithivmlmods_llama-3.2-6b-algocode.json deleted file mode 100644 index e2dd4a01d5c86b8cee677de3e0fa634bfb1e36c8..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_llama-3.2-6b-algocode.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.2-6B-AlgoCode", - "id": "prithivMLmods/Llama-3.2-6B-AlgoCode", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "6.339" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Llama-3.2-6B-AlgoCode/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2136 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3748 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0136 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2869 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4013 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1798 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_llama-8b-distill-cot.json b/data/models/prithivmlmods_llama-8b-distill-cot.json deleted file mode 100644 index 39d8856908ac2c33b65d70ec5b23ec1f48155536..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_llama-8b-distill-cot.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-8B-Distill-CoT", - "id": "prithivMLmods/Llama-8B-Distill-CoT", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Llama-8B-Distill-CoT/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3342 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4298 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4003 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2894 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.372 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2732 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_llama-deepsync-1b.json b/data/models/prithivmlmods_llama-deepsync-1b.json deleted file mode 100644 index 982504417a599a0e8ed6f9f44bd756924258083d..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_llama-deepsync-1b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-Deepsync-1B", - "id": "prithivMLmods/Llama-Deepsync-1B", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.236" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Llama-Deepsync-1B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.357 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3386 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0438 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2601 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3565 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1738 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_llama-deepsync-3b.json b/data/models/prithivmlmods_llama-deepsync-3b.json deleted file mode 100644 index fd55395caa46c772a4f1ea062d583fd83591211b..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_llama-deepsync-3b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-Deepsync-3B", - "id": "prithivMLmods/Llama-Deepsync-3B", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Llama-Deepsync-3B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4302 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4292 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1178 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2718 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3324 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3031 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_llama-express.1-math.json b/data/models/prithivmlmods_llama-express.1-math.json deleted file mode 100644 index 39270bd0d675d22bc5b8d1ad21e7066ed192151c..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_llama-express.1-math.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-Express.1-Math", - "id": "prithivMLmods/Llama-Express.1-Math", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.236" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Llama-Express.1-Math/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5084 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3364 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0559 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2634 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3143 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.161 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_lwq-10b-instruct.json b/data/models/prithivmlmods_lwq-10b-instruct.json deleted file mode 100644 index 06682d10d7016f2922f216181b12b16b58eefdae..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_lwq-10b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "LwQ-10B-Instruct", - "id": "prithivMLmods/LwQ-10B-Instruct", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "10.732" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_LwQ-10B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3935 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5122 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.04 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3121 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4544 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3318 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_lwq-reasoner-10b.json b/data/models/prithivmlmods_lwq-reasoner-10b.json deleted file mode 100644 index 4782c96fdeccfac86411ea22900813b31a7c7be9..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_lwq-reasoner-10b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "LwQ-Reasoner-10B", - "id": "prithivMLmods/LwQ-Reasoner-10B", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "10.306" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_LwQ-Reasoner-10B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2941 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5866 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.358 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3465 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4079 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4147 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_magellanic-opus-14b-exp.json b/data/models/prithivmlmods_magellanic-opus-14b-exp.json deleted file mode 100644 index c7de94cebab5522644c63d9663ab22f64daed560..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_magellanic-opus-14b-exp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Magellanic-Opus-14B-Exp", - "id": "prithivMLmods/Magellanic-Opus-14B-Exp", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Magellanic-Opus-14B-Exp/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6866 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6383 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3799 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3742 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4926 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5273 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_magellanic-qwen-25b-r999.json b/data/models/prithivmlmods_magellanic-qwen-25b-r999.json deleted file mode 100644 index c36c148e4f455c0470fd69a5b937dbfd84f3f874..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_magellanic-qwen-25b-r999.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Magellanic-Qwen-25B-R999", - "id": "prithivMLmods/Magellanic-Qwen-25B-R999", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "24.962" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Magellanic-Qwen-25B-R999/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1873 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2608 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0053 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2508 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3831 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.13 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_megatron-corpus-14b-exp.json b/data/models/prithivmlmods_megatron-corpus-14b-exp.json deleted file mode 100644 index 881bbc65dffdfec791f1aeab56099ec68e405312..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_megatron-corpus-14b-exp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Megatron-Corpus-14B-Exp", - "id": "prithivMLmods/Megatron-Corpus-14B-Exp", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Megatron-Corpus-14B-Exp/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4983 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6355 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3429 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3633 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4767 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.526 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_megatron-corpus-14b-exp.v2.json b/data/models/prithivmlmods_megatron-corpus-14b-exp.v2.json deleted file mode 100644 index f50d6d0fb850db6022ae9151e312079974fe9b54..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_megatron-corpus-14b-exp.v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Megatron-Corpus-14B-Exp.v2", - "id": "prithivMLmods/Megatron-Corpus-14B-Exp.v2", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Megatron-Corpus-14B-Exp.v2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.487 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6321 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2591 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3423 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.449 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.481 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_megatron-opus-14b-2.0.json b/data/models/prithivmlmods_megatron-opus-14b-2.0.json deleted file mode 100644 index d6aeaf1b9ef68730d4aa7e21b8208a4b97c60308..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_megatron-opus-14b-2.0.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Megatron-Opus-14B-2.0", - "id": "prithivMLmods/Megatron-Opus-14B-2.0", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Megatron-Opus-14B-2.0/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6694 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6871 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2779 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3591 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.414 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.517 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_megatron-opus-14b-2.1.json b/data/models/prithivmlmods_megatron-opus-14b-2.1.json deleted file mode 100644 index b92e8e0237940541f92b8161b699097203665ecf..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_megatron-opus-14b-2.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Megatron-Opus-14B-2.1", - "id": "prithivMLmods/Megatron-Opus-14B-2.1", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Megatron-Opus-14B-2.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0246 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6727 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2998 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3834 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4928 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5174 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_megatron-opus-14b-exp.json b/data/models/prithivmlmods_megatron-opus-14b-exp.json deleted file mode 100644 index a642151b1c95f9f2ddb356484b6abb6d1e89de3c..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_megatron-opus-14b-exp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Megatron-Opus-14B-Exp", - "id": "prithivMLmods/Megatron-Opus-14B-Exp", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Megatron-Opus-14B-Exp/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4979 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6516 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3535 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.375 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4887 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5401 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_megatron-opus-14b-stock.json b/data/models/prithivmlmods_megatron-opus-14b-stock.json deleted file mode 100644 index 3f51cf5191209eb56fc26dd9239b8f4bad8d3dde..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_megatron-opus-14b-stock.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Megatron-Opus-14B-Stock", - "id": "prithivMLmods/Megatron-Opus-14B-Stock", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Megatron-Opus-14B-Stock/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5174 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6412 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3346 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.375 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.482 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5293 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_megatron-opus-7b-exp.json b/data/models/prithivmlmods_megatron-opus-7b-exp.json deleted file mode 100644 index 34db62d0385e24b4b011334b56596ad4812d27d6..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_megatron-opus-7b-exp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Megatron-Opus-7B-Exp", - "id": "prithivMLmods/Megatron-Opus-7B-Exp", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "7.456" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Megatron-Opus-7B-Exp/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6017 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5367 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1971 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3112 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4186 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.39 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_messier-opus-14b-elite7.json b/data/models/prithivmlmods_messier-opus-14b-elite7.json deleted file mode 100644 index 6dc6c0b8f9e9f1f0175f7bc37dae88a254639311..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_messier-opus-14b-elite7.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Messier-Opus-14B-Elite7", - "id": "prithivMLmods/Messier-Opus-14B-Elite7", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Messier-Opus-14B-Elite7/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7113 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6499 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4071 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3909 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4886 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5404 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_omni-reasoner-merged.json b/data/models/prithivmlmods_omni-reasoner-merged.json deleted file mode 100644 index 44a6ed27e314c52df99279096f6a1a0435c23b4e..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_omni-reasoner-merged.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Omni-Reasoner-Merged", - "id": "prithivMLmods/Omni-Reasoner-Merged", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Omni-Reasoner-Merged/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4599 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5508 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3331 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3037 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4616 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4364 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_omni-reasoner3-merged.json b/data/models/prithivmlmods_omni-reasoner3-merged.json deleted file mode 100644 index 2ed929cc239a5cbfbb201d5fea6b78414f348341..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_omni-reasoner3-merged.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Omni-Reasoner3-Merged", - "id": "prithivMLmods/Omni-Reasoner3-Merged", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Omni-Reasoner3-Merged/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4935 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4388 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1088 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2643 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3522 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.295 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_pegasus-opus-14b-exp.json b/data/models/prithivmlmods_pegasus-opus-14b-exp.json deleted file mode 100644 index 9c3bdcbe2bfa2c782f918cf51c6d573e61c940ea..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_pegasus-opus-14b-exp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Pegasus-Opus-14B-Exp", - "id": "prithivMLmods/Pegasus-Opus-14B-Exp", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Pegasus-Opus-14B-Exp/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6982 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6548 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4086 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3951 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.486 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5412 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_phi-4-empathetic.json b/data/models/prithivmlmods_phi-4-empathetic.json deleted file mode 100644 index a4716909adc70e8beb4c2403f040815200054393..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_phi-4-empathetic.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Phi-4-Empathetic", - "id": "prithivMLmods/Phi-4-Empathetic", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Phi-4-Empathetic/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0497 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6727 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2621 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.38 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4991 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5066 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_phi-4-math-io.json b/data/models/prithivmlmods_phi-4-math-io.json deleted file mode 100644 index 195c2dd405a7067e3df1fb914fe4c43cfb3c95e1..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_phi-4-math-io.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Phi-4-Math-IO", - "id": "prithivMLmods/Phi-4-Math-IO", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Phi-4-Math-IO/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.059 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6668 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4577 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3985 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4873 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5205 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_phi-4-o1.json b/data/models/prithivmlmods_phi-4-o1.json deleted file mode 100644 index 73c9d219c84faa5b09e444d0e5bf02be5b74d497..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_phi-4-o1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Phi-4-o1", - "id": "prithivMLmods/Phi-4-o1", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Phi-4-o1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.029 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6689 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3995 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3826 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4978 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5174 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_phi-4-qwq.json b/data/models/prithivmlmods_phi-4-qwq.json deleted file mode 100644 index 61f017046ce69905e2a9add200ec8b38a67679d3..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_phi-4-qwq.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Phi-4-QwQ", - "id": "prithivMLmods/Phi-4-QwQ", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Phi-4-QwQ/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0559 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6696 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4577 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3909 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4651 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5275 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_phi-4-super-1.json b/data/models/prithivmlmods_phi-4-super-1.json deleted file mode 100644 index 5f9064402ce4919fea59af88de5376ac1e509ae9..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_phi-4-super-1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Phi-4-Super-1", - "id": "prithivMLmods/Phi-4-Super-1", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Phi-4-Super-1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0418 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6729 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.352 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3935 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5017 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5235 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_phi-4-super-o1.json b/data/models/prithivmlmods_phi-4-super-o1.json deleted file mode 100644 index fca19f11ba225474f241a6bbf4d6b5ec95e2fc66..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_phi-4-super-o1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Phi-4-Super-o1", - "id": "prithivMLmods/Phi-4-Super-o1", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Phi-4-Super-o1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0418 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6729 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.352 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3935 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5017 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5235 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_phi-4-super.json b/data/models/prithivmlmods_phi-4-super.json deleted file mode 100644 index 2410435ef75d2efb001f62bf36bcb55963186955..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_phi-4-super.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Phi-4-Super", - "id": "prithivMLmods/Phi-4-Super", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Phi-4-Super/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0481 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.672 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3489 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3943 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5044 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5266 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_phi4-super.json b/data/models/prithivmlmods_phi4-super.json deleted file mode 100644 index cc51db01f9a019c46d470a4c5769fef1a60f8da0..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_phi4-super.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Phi4-Super", - "id": "prithivMLmods/Phi4-Super", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Phi4-Super/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0481 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.672 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3489 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3943 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5044 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5266 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_porpoise-opus-14b-exp.json b/data/models/prithivmlmods_porpoise-opus-14b-exp.json deleted file mode 100644 index eb1191f2ba20214ab66888afa2a824fcdf29a558..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_porpoise-opus-14b-exp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Porpoise-Opus-14B-Exp", - "id": "prithivMLmods/Porpoise-Opus-14B-Exp", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Porpoise-Opus-14B-Exp/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7098 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6519 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4041 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3935 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4926 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5396 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_primal-opus-14b-optimus-v1.json b/data/models/prithivmlmods_primal-opus-14b-optimus-v1.json deleted file mode 100644 index f02b4a3245e93b0aeeb058a9a2447fb6b936a996..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_primal-opus-14b-optimus-v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Primal-Opus-14B-Optimus-v1", - "id": "prithivMLmods/Primal-Opus-14B-Optimus-v1", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Primal-Opus-14B-Optimus-v1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5013 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6419 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3384 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3725 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4847 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5259 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_primal-opus-14b-optimus-v2.json b/data/models/prithivmlmods_primal-opus-14b-optimus-v2.json deleted file mode 100644 index cdef2ce26821465e39d7c39e1ace9fad372df029..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_primal-opus-14b-optimus-v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Primal-Opus-14B-Optimus-v2", - "id": "prithivMLmods/Primal-Opus-14B-Optimus-v2", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Primal-Opus-14B-Optimus-v2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6404 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6544 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4207 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3918 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.49 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5422 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_qwen-7b-distill-reasoner.json b/data/models/prithivmlmods_qwen-7b-distill-reasoner.json deleted file mode 100644 index 9c8f9bf391fea18a1cd4d3b59b9e772469884d42..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_qwen-7b-distill-reasoner.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen-7B-Distill-Reasoner", - "id": "prithivMLmods/Qwen-7B-Distill-Reasoner", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Qwen-7B-Distill-Reasoner/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3396 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4409 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.395 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3272 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.366 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2818 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_qwen2.5-1.5b-deepseek-r1-instruct.json b/data/models/prithivmlmods_qwen2.5-1.5b-deepseek-r1-instruct.json deleted file mode 100644 index 1acd5dec089d75e9f67aeb94b962de6543f67514..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_qwen2.5-1.5b-deepseek-r1-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-1.5B-DeepSeek-R1-Instruct", - "id": "prithivMLmods/Qwen2.5-1.5B-DeepSeek-R1-Instruct", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.777" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Qwen2.5-1.5B-DeepSeek-R1-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1397 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2824 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.276 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3724 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1123 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_qwen2.5-14b-deepseek-r1-1m.json b/data/models/prithivmlmods_qwen2.5-14b-deepseek-r1-1m.json deleted file mode 100644 index 0d954db1a0bc992edc21f7affbafd36906b36dc6..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_qwen2.5-14b-deepseek-r1-1m.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-14B-DeepSeek-R1-1M", - "id": "prithivMLmods/Qwen2.5-14B-DeepSeek-R1-1M", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Qwen2.5-14B-DeepSeek-R1-1M/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4193 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5935 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5128 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3322 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4606 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4899 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_qwen2.5-7b-deepseek-r1-1m.json b/data/models/prithivmlmods_qwen2.5-7b-deepseek-r1-1m.json deleted file mode 100644 index 48abb0da43ab4bb930f32e8d19d400d443a1cfc5..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_qwen2.5-7b-deepseek-r1-1m.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-7B-DeepSeek-R1-1M", - "id": "prithivMLmods/Qwen2.5-7B-DeepSeek-R1-1M", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Qwen2.5-7B-DeepSeek-R1-1M/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1861 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3126 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0151 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2617 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3417 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1201 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_qwq-lcot-14b-conversational.json b/data/models/prithivmlmods_qwq-lcot-14b-conversational.json deleted file mode 100644 index 1fcce927943389980ce11815ceb8d24cf9ce8f67..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_qwq-lcot-14b-conversational.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "QwQ-LCoT-14B-Conversational", - "id": "prithivMLmods/QwQ-LCoT-14B-Conversational", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_QwQ-LCoT-14B-Conversational/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4047 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.624 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4653 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3498 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4847 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5278 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_qwq-lcot-3b-instruct.json b/data/models/prithivmlmods_qwq-lcot-3b-instruct.json deleted file mode 100644 index 737f44a2bc328084a88a5c9759723def22a95b77..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_qwq-lcot-3b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "QwQ-LCoT-3B-Instruct", - "id": "prithivMLmods/QwQ-LCoT-3B-Instruct", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.086" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_QwQ-LCoT-3B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4354 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4763 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2825 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2819 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4358 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3582 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_qwq-lcot-7b-instruct.json b/data/models/prithivmlmods_qwq-lcot-7b-instruct.json deleted file mode 100644 index a68d20b9351ff5db5c4f3f3366293ec790b6e37e..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_qwq-lcot-7b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "QwQ-LCoT-7B-Instruct", - "id": "prithivMLmods/QwQ-LCoT-7B-Instruct", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_QwQ-LCoT-7B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4987 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5466 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3716 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.302 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4802 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4334 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_qwq-lcot1-merged.json b/data/models/prithivmlmods_qwq-lcot1-merged.json deleted file mode 100644 index b6cea86957ca7977df445559ba46fa126e483099..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_qwq-lcot1-merged.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "QwQ-LCoT1-Merged", - "id": "prithivMLmods/QwQ-LCoT1-Merged", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_QwQ-LCoT1-Merged/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4751 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5481 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3731 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.307 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4696 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4358 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_qwq-lcot2-7b-instruct.json b/data/models/prithivmlmods_qwq-lcot2-7b-instruct.json deleted file mode 100644 index 473dcb9c34fb5c3cc136a8f246458eb9451f2a80..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_qwq-lcot2-7b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "QwQ-LCoT2-7B-Instruct", - "id": "prithivMLmods/QwQ-LCoT2-7B-Instruct", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_QwQ-LCoT2-7B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5561 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5425 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.327 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2978 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4564 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4342 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_qwq-mathoct-7b.json b/data/models/prithivmlmods_qwq-mathoct-7b.json deleted file mode 100644 index 8740978bd9f44ee56d64bd0fc4bda25681ab861a..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_qwq-mathoct-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "QwQ-MathOct-7B", - "id": "prithivMLmods/QwQ-MathOct-7B", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_QwQ-MathOct-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4684 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5486 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2953 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3029 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4601 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.433 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_qwq-r1-distill-1.5b-cot.json b/data/models/prithivmlmods_qwq-r1-distill-1.5b-cot.json deleted file mode 100644 index e8334aad373148758ec137713c62f8bf4a9969a6..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_qwq-r1-distill-1.5b-cot.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "QwQ-R1-Distill-1.5B-CoT", - "id": "prithivMLmods/QwQ-R1-Distill-1.5B-CoT", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.777" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_QwQ-R1-Distill-1.5B-CoT/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2194 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3666 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3346 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2861 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3434 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1913 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_qwq-r1-distill-7b-cot.json b/data/models/prithivmlmods_qwq-r1-distill-7b-cot.json deleted file mode 100644 index dcdffaa2329ea2df2065a3f02724c3b35097ef2e..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_qwq-r1-distill-7b-cot.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "QwQ-R1-Distill-7B-CoT", - "id": "prithivMLmods/QwQ-R1-Distill-7B-CoT", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_QwQ-R1-Distill-7B-CoT/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.35 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4388 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4683 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2936 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3779 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2804 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_smollm2-cot-360m.json b/data/models/prithivmlmods_smollm2-cot-360m.json deleted file mode 100644 index cac6355cf31a148b96937ca3b89f1a35ee055793..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_smollm2-cot-360m.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SmolLM2-CoT-360M", - "id": "prithivMLmods/SmolLM2-CoT-360M", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "0.362" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_SmolLM2-CoT-360M/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2216 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3135 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0204 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2366 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3794 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1085 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_sombrero-opus-14b-elite5.json b/data/models/prithivmlmods_sombrero-opus-14b-elite5.json deleted file mode 100644 index 097034d6830b8af3c0092a2d47bdf56d27532a47..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_sombrero-opus-14b-elite5.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Sombrero-Opus-14B-Elite5", - "id": "prithivMLmods/Sombrero-Opus-14B-Elite5", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Sombrero-Opus-14B-Elite5/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7881 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6502 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5355 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3364 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4287 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.52 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_sombrero-opus-14b-elite6.json b/data/models/prithivmlmods_sombrero-opus-14b-elite6.json deleted file mode 100644 index 5934c0036f6873862d02734062d2706cc40ac01a..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_sombrero-opus-14b-elite6.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Sombrero-Opus-14B-Elite6", - "id": "prithivMLmods/Sombrero-Opus-14B-Elite6", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Sombrero-Opus-14B-Elite6/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7226 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6488 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4079 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3935 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4886 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.539 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_sombrero-opus-14b-sm1.json b/data/models/prithivmlmods_sombrero-opus-14b-sm1.json deleted file mode 100644 index 5b17d9d7a816198a516624262d134a0fd92fc092..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_sombrero-opus-14b-sm1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Sombrero-Opus-14B-Sm1", - "id": "prithivMLmods/Sombrero-Opus-14B-Sm1", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Sombrero-Opus-14B-Sm1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3813 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6355 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5665 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4035 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5299 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5125 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_sombrero-opus-14b-sm2.json b/data/models/prithivmlmods_sombrero-opus-14b-sm2.json deleted file mode 100644 index 45c389afcfaa33c67b38e36c234ca74f0c79e5a7..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_sombrero-opus-14b-sm2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Sombrero-Opus-14B-Sm2", - "id": "prithivMLmods/Sombrero-Opus-14B-Sm2", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Sombrero-Opus-14B-Sm2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4272 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6609 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4864 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3884 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5088 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5345 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_sombrero-opus-14b-sm4.json b/data/models/prithivmlmods_sombrero-opus-14b-sm4.json deleted file mode 100644 index 29d899cb60abcd75ca8340fb08b8fce233c28021..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_sombrero-opus-14b-sm4.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Sombrero-Opus-14B-Sm4", - "id": "prithivMLmods/Sombrero-Opus-14B-Sm4", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Sombrero-Opus-14B-Sm4/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4347 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6613 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4879 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3951 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5192 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.53 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_sombrero-opus-14b-sm5.json b/data/models/prithivmlmods_sombrero-opus-14b-sm5.json deleted file mode 100644 index 4da80b6be669c65f1f07c40724b0262d8b151d19..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_sombrero-opus-14b-sm5.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Sombrero-Opus-14B-Sm5", - "id": "prithivMLmods/Sombrero-Opus-14B-Sm5", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Sombrero-Opus-14B-Sm5/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6852 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6564 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4094 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3867 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4806 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.54 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_sqweeks-7b-instruct.json b/data/models/prithivmlmods_sqweeks-7b-instruct.json deleted file mode 100644 index e1cefc961346bc5a27601a1c68833efcde20ba2e..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_sqweeks-7b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Sqweeks-7B-Instruct", - "id": "prithivMLmods/Sqweeks-7B-Instruct", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Sqweeks-7B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2158 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4667 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5144 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.307 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4476 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3133 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_tadpole-opus-14b-exp.json b/data/models/prithivmlmods_tadpole-opus-14b-exp.json deleted file mode 100644 index 496a7b2d89169f16f9e216d05c07fbf41289e77d..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_tadpole-opus-14b-exp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Tadpole-Opus-14B-Exp", - "id": "prithivMLmods/Tadpole-Opus-14B-Exp", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Tadpole-Opus-14B-Exp/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.575 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6369 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3134 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3859 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4728 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5322 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_taurus-opus-7b.json b/data/models/prithivmlmods_taurus-opus-7b.json deleted file mode 100644 index 14398728d2fd01269f2925367b229d592531e2de..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_taurus-opus-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Taurus-Opus-7B", - "id": "prithivMLmods/Taurus-Opus-7B", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "7.456" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Taurus-Opus-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4223 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5367 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2168 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3263 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4399 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3951 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_triangulum-10b.json b/data/models/prithivmlmods_triangulum-10b.json deleted file mode 100644 index 06b25a7d88994354ab338fbf9fe48e1d4b927d0f..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_triangulum-10b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Triangulum-10B", - "id": "prithivMLmods/Triangulum-10B", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "10.306" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Triangulum-10B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3229 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5968 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.355 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.354 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4172 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4178 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_triangulum-5b.json b/data/models/prithivmlmods_triangulum-5b.json deleted file mode 100644 index 79947c2ae852a3679a2aa4c325142e4ceccf2d91..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_triangulum-5b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Triangulum-5B", - "id": "prithivMLmods/Triangulum-5B", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "5.413" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Triangulum-5B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1283 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3124 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0106 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.255 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3445 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1223 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_triangulum-v2-10b.json b/data/models/prithivmlmods_triangulum-v2-10b.json deleted file mode 100644 index 87efc4b6201ffa83b1006cb9e28322dbd9bf7f85..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_triangulum-v2-10b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Triangulum-v2-10B", - "id": "prithivMLmods/Triangulum-v2-10B", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "10.306" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Triangulum-v2-10B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6705 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6065 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2447 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3372 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4281 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4466 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_tucana-opus-14b-r999.json b/data/models/prithivmlmods_tucana-opus-14b-r999.json deleted file mode 100644 index 75da87011f13fdfe0a65bddfadc5065249f94a7e..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_tucana-opus-14b-r999.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Tucana-Opus-14B-r999", - "id": "prithivMLmods/Tucana-Opus-14B-r999", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Tucana-Opus-14B-r999/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6067 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6557 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4063 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3918 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.473 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5384 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_tulu-mathlingo-8b.json b/data/models/prithivmlmods_tulu-mathlingo-8b.json deleted file mode 100644 index 4996439f1408a639c8de2ae19d45b3f38e847901..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_tulu-mathlingo-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Tulu-MathLingo-8B", - "id": "prithivMLmods/Tulu-MathLingo-8B", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Tulu-MathLingo-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5589 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4659 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.145 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2903 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3864 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3044 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_viper-coder-7b-elite14.json b/data/models/prithivmlmods_viper-coder-7b-elite14.json deleted file mode 100644 index bfd9b77e6e08c10e42a8723f8467afbe27d09e7a..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_viper-coder-7b-elite14.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Viper-Coder-7B-Elite14", - "id": "prithivMLmods/Viper-Coder-7B-Elite14", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Viper-Coder-7B-Elite14/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1488 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2829 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0106 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.255 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3422 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1089 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_viper-coder-hybrid-v1.2.json b/data/models/prithivmlmods_viper-coder-hybrid-v1.2.json deleted file mode 100644 index 97dd5f2b8a0b084a259d923f8e4049cae558be61..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_viper-coder-hybrid-v1.2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Viper-Coder-Hybrid-v1.2", - "id": "prithivMLmods/Viper-Coder-Hybrid-v1.2", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Viper-Coder-Hybrid-v1.2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6736 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6391 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3331 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3742 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4822 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5243 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_viper-coder-hybrid-v1.3.json b/data/models/prithivmlmods_viper-coder-hybrid-v1.3.json deleted file mode 100644 index 01f3e6ae502e88b879cf3870c316eb046a3f0d38..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_viper-coder-hybrid-v1.3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Viper-Coder-Hybrid-v1.3", - "id": "prithivMLmods/Viper-Coder-Hybrid-v1.3", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Viper-Coder-Hybrid-v1.3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7555 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6471 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4517 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3381 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4403 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5097 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_viper-coder-hybridmini-v1.3.json b/data/models/prithivmlmods_viper-coder-hybridmini-v1.3.json deleted file mode 100644 index aaf434837ee29fbaafefe6244cea7c01a62eb3c6..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_viper-coder-hybridmini-v1.3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Viper-Coder-HybridMini-v1.3", - "id": "prithivMLmods/Viper-Coder-HybridMini-v1.3", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Viper-Coder-HybridMini-v1.3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6104 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5365 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.463 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3171 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4505 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4352 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_viper-coder-v0.1.json b/data/models/prithivmlmods_viper-coder-v0.1.json deleted file mode 100644 index 04d8767251768c28aed9eccc5ef8d3c349a707ff..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_viper-coder-v0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Viper-Coder-v0.1", - "id": "prithivMLmods/Viper-Coder-v0.1", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Viper-Coder-v0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5521 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6143 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.327 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.354 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4394 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3928 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_viper-coder-v1.1.json b/data/models/prithivmlmods_viper-coder-v1.1.json deleted file mode 100644 index 832b28a4be2369c592c6550349c414580b324ebd..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_viper-coder-v1.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Viper-Coder-v1.1", - "id": "prithivMLmods/Viper-Coder-v1.1", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Viper-Coder-v1.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4432 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6492 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5461 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.401 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5219 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5232 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_viper-coder-v1.6-r999.json b/data/models/prithivmlmods_viper-coder-v1.6-r999.json deleted file mode 100644 index 932134b2e2205a45339fadbc0ea48ee56a7ddbf6..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_viper-coder-v1.6-r999.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Viper-Coder-v1.6-r999", - "id": "prithivMLmods/Viper-Coder-v1.6-r999", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Viper-Coder-v1.6-r999/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4433 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6492 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5657 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.401 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5219 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5232 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_viper-coder-v1.7-vsm6.json b/data/models/prithivmlmods_viper-coder-v1.7-vsm6.json deleted file mode 100644 index f2c1db41224ef9708216281689d53b74bfd4bd9a..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_viper-coder-v1.7-vsm6.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Viper-Coder-v1.7-Vsm6", - "id": "prithivMLmods/Viper-Coder-v1.7-Vsm6", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Viper-Coder-v1.7-Vsm6/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5004 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6502 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4645 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3968 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4768 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5288 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_viper-onecoder-uigen.json b/data/models/prithivmlmods_viper-onecoder-uigen.json deleted file mode 100644 index d928dce379f991974ed1c3b5bac60a1bce1ab31a..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_viper-onecoder-uigen.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Viper-OneCoder-UIGEN", - "id": "prithivMLmods/Viper-OneCoder-UIGEN", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Viper-OneCoder-UIGEN/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4692 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6047 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3867 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3423 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4514 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3904 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_volans-opus-14b-exp.json b/data/models/prithivmlmods_volans-opus-14b-exp.json deleted file mode 100644 index 4b10897729ae9b171afea5a04ea233937bc5c853..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_volans-opus-14b-exp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Volans-Opus-14B-Exp", - "id": "prithivMLmods/Volans-Opus-14B-Exp", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_Volans-Opus-14B-Exp/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5868 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6521 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4252 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3851 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4872 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5385 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prithivmlmods_webmind-7b-v0.1.json b/data/models/prithivmlmods_webmind-7b-v0.1.json deleted file mode 100644 index c723750eea6fe5ee20721e5f89de86b5548af298..0000000000000000000000000000000000000000 --- a/data/models/prithivmlmods_webmind-7b-v0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "WebMind-7B-v0.1", - "id": "prithivMLmods/WebMind-7B-v0.1", - "developer": "prithivMLmods", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/prithivMLmods_WebMind-7B-v0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5278 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5434 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3648 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3171 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4537 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4279 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prometheus-eval_prometheus-7b-v2.0.json b/data/models/prometheus-eval_prometheus-7b-v2.0.json deleted file mode 100644 index 1b556f441175f6318ca19edb472368244de890f5..0000000000000000000000000000000000000000 --- a/data/models/prometheus-eval_prometheus-7b-v2.0.json +++ /dev/null @@ -1,126 +0,0 @@ -{ - "model_info": { - "name": "prometheus-eval/prometheus-7b-v2.0", - "id": "prometheus-eval/prometheus-7b-v2.0", - "developer": "prometheus-eval", - "additional_details": { - "model_type": "Generative" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/prometheus-eval_prometheus-7b-v2.0/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7204 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8547 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4912 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7709 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7648 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/prometheus-eval_prometheus-8x7b-v2.0.json b/data/models/prometheus-eval_prometheus-8x7b-v2.0.json deleted file mode 100644 index 9718129aa6a6d716deaa6396344fc01dcfe6b0f4..0000000000000000000000000000000000000000 --- a/data/models/prometheus-eval_prometheus-8x7b-v2.0.json +++ /dev/null @@ -1,126 +0,0 @@ -{ - "model_info": { - "name": "prometheus-eval/prometheus-8x7b-v2.0", - "id": "prometheus-eval/prometheus-8x7b-v2.0", - "developer": "prometheus-eval", - "additional_details": { - "model_type": "Generative" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/prometheus-eval_prometheus-8x7b-v2.0/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7451 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9302 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4715 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8047 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.774 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/pszemraj_llama-3-6.3b-v0.1.json b/data/models/pszemraj_llama-3-6.3b-v0.1.json deleted file mode 100644 index 60e133a4a0d86d3be0a40cf54e179cd6bcbc39ec..0000000000000000000000000000000000000000 --- a/data/models/pszemraj_llama-3-6.3b-v0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-6.3b-v0.1", - "id": "pszemraj/Llama-3-6.3b-v0.1", - "developer": "pszemraj", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "6.3" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/pszemraj_Llama-3-6.3b-v0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1044 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4197 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0211 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2836 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3908 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.284 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/pszemraj_mistral-v0.3-6b.json b/data/models/pszemraj_mistral-v0.3-6b.json deleted file mode 100644 index 4db5d9246df53235405817bee5e41921c17fe629..0000000000000000000000000000000000000000 --- a/data/models/pszemraj_mistral-v0.3-6b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mistral-v0.3-6B", - "id": "pszemraj/Mistral-v0.3-6B", - "developer": "pszemraj", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "5.939" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/pszemraj_Mistral-v0.3-6B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2454 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3774 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0136 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2651 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3908 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2143 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/puxai_lua_model.json b/data/models/puxai_lua_model.json deleted file mode 100644 index 4c94355a4fc0ba70b7add78c9cec8f15792834fa..0000000000000000000000000000000000000000 --- a/data/models/puxai_lua_model.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "LUA_model", - "id": "PuxAI/LUA_model", - "developer": "PuxAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "7.386" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/PuxAI_LUA_model/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2282 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2877 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2601 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3484 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1123 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/pygmalionai_pygmalion-6b.json b/data/models/pygmalionai_pygmalion-6b.json deleted file mode 100644 index 73268c9d7b31c022654a55f17efce2e00024c865..0000000000000000000000000000000000000000 --- a/data/models/pygmalionai_pygmalion-6b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "pygmalion-6b", - "id": "PygmalionAI/pygmalion-6b", - "developer": "PygmalionAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "GPTJForCausalLM", - "params_billions": "6.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/PygmalionAI_pygmalion-6b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2091 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3199 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0083 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2492 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3684 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1184 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/q-bert_metamath-1b.json b/data/models/q-bert_metamath-1b.json deleted file mode 100644 index 42aa99fb76bfc07b5562d4a0e7b4c9b3dd5af4f6..0000000000000000000000000000000000000000 --- a/data/models/q-bert_metamath-1b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MetaMath-1B", - "id": "Q-bert/MetaMath-1B", - "developer": "Q-bert", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.236" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Q-bert_MetaMath-1B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.53 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3451 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0627 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2517 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3289 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1495 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qingy2019_llama_3.2_3b_catalysts.json b/data/models/qingy2019_llama_3.2_3b_catalysts.json deleted file mode 100644 index 63ed8b66416c5e9dee83a12e3442bdb905b814c1..0000000000000000000000000000000000000000 --- a/data/models/qingy2019_llama_3.2_3b_catalysts.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "LLaMa_3.2_3B_Catalysts", - "id": "qingy2019/LLaMa_3.2_3B_Catalysts", - "developer": "qingy2019", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/qingy2019_LLaMa_3.2_3B_Catalysts/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4992 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4468 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1292 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2886 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3788 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3008 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qingy2019_openmath2-llama3.1-8b.json b/data/models/qingy2019_openmath2-llama3.1-8b.json deleted file mode 100644 index 359dfc7cce750212be926dbe5b2e488809195dfc..0000000000000000000000000000000000000000 --- a/data/models/qingy2019_openmath2-llama3.1-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "OpenMath2-Llama3.1-8B", - "id": "qingy2019/OpenMath2-Llama3.1-8B", - "developer": "qingy2019", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/qingy2019_OpenMath2-Llama3.1-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2331 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4096 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2674 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2651 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3436 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1553 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qingy2019_oracle-14b.json b/data/models/qingy2019_oracle-14b.json deleted file mode 100644 index 8fc1dd082923a347043a18d5c18fb5d4904fae42..0000000000000000000000000000000000000000 --- a/data/models/qingy2019_oracle-14b.json +++ /dev/null @@ -1,275 +0,0 @@ -{ - "model_info": { - "name": "Oracle-14B", - "id": "qingy2019/Oracle-14B", - "developer": "qingy2019", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MixtralForCausalLM", - "params_billions": "13.668" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/qingy2019_Oracle-14B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2401 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4622 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0725 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2609 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3703 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2379 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "hfopenllm_v2/qingy2019_Oracle-14B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2358 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4612 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0642 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2576 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3717 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2382 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qingy2019_qwen2.5-math-14b-instruct-alpha.json b/data/models/qingy2019_qwen2.5-math-14b-instruct-alpha.json deleted file mode 100644 index b3d528d987ee88dacaf0db87563d92068093979b..0000000000000000000000000000000000000000 --- a/data/models/qingy2019_qwen2.5-math-14b-instruct-alpha.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-Math-14B-Instruct-Alpha", - "id": "qingy2019/Qwen2.5-Math-14B-Instruct-Alpha", - "developer": "qingy2019", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/qingy2019_Qwen2.5-Math-14B-Instruct-Alpha/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5981 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6375 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3142 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.37 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4649 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5331 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qingy2019_qwen2.5-math-14b-instruct-pro.json b/data/models/qingy2019_qwen2.5-math-14b-instruct-pro.json deleted file mode 100644 index fa80dd7db64a993ef1f89b4996a329f02198f48b..0000000000000000000000000000000000000000 --- a/data/models/qingy2019_qwen2.5-math-14b-instruct-pro.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-Math-14B-Instruct-Pro", - "id": "qingy2019/Qwen2.5-Math-14B-Instruct-Pro", - "developer": "qingy2019", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/qingy2019_Qwen2.5-Math-14B-Instruct-Pro/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1922 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5319 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.284 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3112 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.374 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3558 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qingy2019_qwen2.5-math-14b-instruct.json b/data/models/qingy2019_qwen2.5-math-14b-instruct.json deleted file mode 100644 index 21a12461d0cce8974037575db54605a0da19b661..0000000000000000000000000000000000000000 --- a/data/models/qingy2019_qwen2.5-math-14b-instruct.json +++ /dev/null @@ -1,275 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-Math-14B-Instruct", - "id": "qingy2019/Qwen2.5-Math-14B-Instruct", - "developer": "qingy2019", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/qingy2019_Qwen2.5-Math-14B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6005 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6356 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2764 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3691 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4757 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5339 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "hfopenllm_v2/qingy2019_Qwen2.5-Math-14B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6066 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.635 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3716 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3725 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4757 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5331 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qingy2019_qwen2.5-ultimate-14b-instruct.json b/data/models/qingy2019_qwen2.5-ultimate-14b-instruct.json deleted file mode 100644 index 5347c1dd98c355b6cb5380d9abcd47c0d9cebebe..0000000000000000000000000000000000000000 --- a/data/models/qingy2019_qwen2.5-ultimate-14b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-Ultimate-14B-Instruct", - "id": "qingy2019/Qwen2.5-Ultimate-14B-Instruct", - "developer": "qingy2019", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/qingy2019_Qwen2.5-Ultimate-14B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3938 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5842 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2893 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3565 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4135 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4929 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qingy2024_benchmaxx-llama-3.2-1b-instruct.json b/data/models/qingy2024_benchmaxx-llama-3.2-1b-instruct.json deleted file mode 100644 index 76b9b4782aaf27d6e796d6649f1fe91c82a27556..0000000000000000000000000000000000000000 --- a/data/models/qingy2024_benchmaxx-llama-3.2-1b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Benchmaxx-Llama-3.2-1B-Instruct", - "id": "qingy2024/Benchmaxx-Llama-3.2-1B-Instruct", - "developer": "qingy2024", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.236" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/qingy2024_Benchmaxx-Llama-3.2-1B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2014 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8269 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4804 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2836 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3446 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1113 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qingy2024_eyas-17b-instruct.json b/data/models/qingy2024_eyas-17b-instruct.json deleted file mode 100644 index a259c32d523a7b19c7d64a5f442355c31ecfb293..0000000000000000000000000000000000000000 --- a/data/models/qingy2024_eyas-17b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Eyas-17B-Instruct", - "id": "qingy2024/Eyas-17B-Instruct", - "developer": "qingy2024", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "17.431" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/qingy2024_Eyas-17B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6575 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6085 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.247 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3146 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4522 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4343 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qingy2024_falcon3-2x10b-moe-instruct.json b/data/models/qingy2024_falcon3-2x10b-moe-instruct.json deleted file mode 100644 index 1639167e7c3ebceb44bab0f14ac7fe7193fa58f7..0000000000000000000000000000000000000000 --- a/data/models/qingy2024_falcon3-2x10b-moe-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Falcon3-2x10B-MoE-Instruct", - "id": "qingy2024/Falcon3-2x10B-MoE-Instruct", - "developer": "qingy2024", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MixtralForCausalLM", - "params_billions": "18.799" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/qingy2024_Falcon3-2x10B-MoE-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.785 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6185 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2795 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3305 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4284 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4423 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qingy2024_fusion-14b-instruct.json b/data/models/qingy2024_fusion-14b-instruct.json deleted file mode 100644 index ba299c2cd98bea7d95076dddd56fd84b4e2eb035..0000000000000000000000000000000000000000 --- a/data/models/qingy2024_fusion-14b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Fusion-14B-Instruct", - "id": "qingy2024/Fusion-14B-Instruct", - "developer": "qingy2024", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/qingy2024_Fusion-14B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.726 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6396 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3369 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3549 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.44 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5044 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qingy2024_fusion2-14b-instruct.json b/data/models/qingy2024_fusion2-14b-instruct.json deleted file mode 100644 index d4fe673afe77f02924af7fa9966aab03cc4d274b..0000000000000000000000000000000000000000 --- a/data/models/qingy2024_fusion2-14b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Fusion2-14B-Instruct", - "id": "qingy2024/Fusion2-14B-Instruct", - "developer": "qingy2024", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/qingy2024_Fusion2-14B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6064 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6119 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3127 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3448 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4634 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5051 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qingy2024_fusion4-14b-instruct.json b/data/models/qingy2024_fusion4-14b-instruct.json deleted file mode 100644 index c752dcb5cfb8b37ba5348d894bf52b01573347bc..0000000000000000000000000000000000000000 --- a/data/models/qingy2024_fusion4-14b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Fusion4-14B-Instruct", - "id": "qingy2024/Fusion4-14B-Instruct", - "developer": "qingy2024", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/qingy2024_Fusion4-14B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7649 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6543 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3882 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3305 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4326 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5194 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qingy2024_owo-14b-instruct.json b/data/models/qingy2024_owo-14b-instruct.json deleted file mode 100644 index 70ea81826e1a5162407973ea3396f20ac0dc94e3..0000000000000000000000000000000000000000 --- a/data/models/qingy2024_owo-14b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "OwO-14B-Instruct", - "id": "qingy2024/OwO-14B-Instruct", - "developer": "qingy2024", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/qingy2024_OwO-14B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1383 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6165 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4162 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3641 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4407 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5181 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qingy2024_qwarkstar-4b-instruct-preview.json b/data/models/qingy2024_qwarkstar-4b-instruct-preview.json deleted file mode 100644 index 75a8591e13f0d0304df96549322eeb88f69205ab..0000000000000000000000000000000000000000 --- a/data/models/qingy2024_qwarkstar-4b-instruct-preview.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwarkstar-4B-Instruct-Preview", - "id": "qingy2024/Qwarkstar-4B-Instruct-Preview", - "developer": "qingy2024", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "4.473" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/qingy2024_Qwarkstar-4B-Instruct-Preview/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5324 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4358 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1284 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2802 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3896 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2502 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qingy2024_qwarkstar-4b.json b/data/models/qingy2024_qwarkstar-4b.json deleted file mode 100644 index 64845159e2947961dbd785aaf90141378b7d63eb..0000000000000000000000000000000000000000 --- a/data/models/qingy2024_qwarkstar-4b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwarkstar-4B", - "id": "qingy2024/Qwarkstar-4B", - "developer": "qingy2024", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "4.473" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/qingy2024_Qwarkstar-4B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1994 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4015 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0861 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3247 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4428 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2425 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qingy2024_qwen2.5-4b.json b/data/models/qingy2024_qwen2.5-4b.json deleted file mode 100644 index bee04328bafa87a2075fa3d4ad904601e289100d..0000000000000000000000000000000000000000 --- a/data/models/qingy2024_qwen2.5-4b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-4B", - "id": "qingy2024/Qwen2.5-4B", - "developer": "qingy2024", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "4.168" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/qingy2024_Qwen2.5-4B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2158 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4269 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0514 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2911 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.461 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2525 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qingy2024_qwen2.5-coder-draft-1.5b-instruct.json b/data/models/qingy2024_qwen2.5-coder-draft-1.5b-instruct.json deleted file mode 100644 index 9f5d499af9614c045ca4aefe91c344a2779df6f6..0000000000000000000000000000000000000000 --- a/data/models/qingy2024_qwen2.5-coder-draft-1.5b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-Coder-Draft-1.5B-Instruct", - "id": "qingy2024/Qwen2.5-Coder-Draft-1.5B-Instruct", - "developer": "qingy2024", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.544" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/qingy2024_Qwen2.5-Coder-Draft-1.5B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4125 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3837 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1579 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2601 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.358 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2244 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qingy2024_qwen2.5-math-14b-instruct-alpha.json b/data/models/qingy2024_qwen2.5-math-14b-instruct-alpha.json deleted file mode 100644 index af603104f70af6a919c5c08ab17316b779d33cd6..0000000000000000000000000000000000000000 --- a/data/models/qingy2024_qwen2.5-math-14b-instruct-alpha.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-Math-14B-Instruct-Alpha", - "id": "qingy2024/Qwen2.5-Math-14B-Instruct-Alpha", - "developer": "qingy2024", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/qingy2024_Qwen2.5-Math-14B-Instruct-Alpha/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7704 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6465 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.429 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.349 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4021 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4966 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qingy2024_qwen2.5-math-14b-instruct-preview.json b/data/models/qingy2024_qwen2.5-math-14b-instruct-preview.json deleted file mode 100644 index 1adc1567e83ba3dba88cbd30f870608b87c5b710..0000000000000000000000000000000000000000 --- a/data/models/qingy2024_qwen2.5-math-14b-instruct-preview.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-Math-14B-Instruct-Preview", - "id": "qingy2024/Qwen2.5-Math-14B-Instruct-Preview", - "developer": "qingy2024", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/qingy2024_Qwen2.5-Math-14B-Instruct-Preview/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7826 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6294 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4758 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3406 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4115 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4993 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qingy2024_qwen2.6-14b-instruct.json b/data/models/qingy2024_qwen2.6-14b-instruct.json deleted file mode 100644 index 452c5ad81b0ae1116a8b23bdd46790f872034789..0000000000000000000000000000000000000000 --- a/data/models/qingy2024_qwen2.6-14b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.6-14B-Instruct", - "id": "qingy2024/Qwen2.6-14B-Instruct", - "developer": "qingy2024", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/qingy2024_Qwen2.6-14B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5811 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6394 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3051 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3792 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4569 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5285 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qingy2024_qwen2.6-math-14b-instruct.json b/data/models/qingy2024_qwen2.6-math-14b-instruct.json deleted file mode 100644 index 8c79ec4f76dce9b654d3b97b64e576dd9ae2d03a..0000000000000000000000000000000000000000 --- a/data/models/qingy2024_qwen2.6-math-14b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.6-Math-14B-Instruct", - "id": "qingy2024/Qwen2.6-Math-14B-Instruct", - "developer": "qingy2024", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/qingy2024_Qwen2.6-Math-14B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3862 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6324 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.429 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.37 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4759 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5241 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qingy2024_qwenlarge-16b-instruct.json b/data/models/qingy2024_qwenlarge-16b-instruct.json deleted file mode 100644 index 4b33c695d4f2b8c303db8021a496087c53e71dbe..0000000000000000000000000000000000000000 --- a/data/models/qingy2024_qwenlarge-16b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "QwEnlarge-16B-Instruct", - "id": "qingy2024/QwEnlarge-16B-Instruct", - "developer": "qingy2024", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "15.871" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/qingy2024_QwEnlarge-16B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7802 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5949 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.46 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3331 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4101 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4476 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qingy2024_qwq-14b-math-v0.2.json b/data/models/qingy2024_qwq-14b-math-v0.2.json deleted file mode 100644 index 45b02036414efdb01ea3fcc8d3806e743d01499f..0000000000000000000000000000000000000000 --- a/data/models/qingy2024_qwq-14b-math-v0.2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "QwQ-14B-Math-v0.2", - "id": "qingy2024/QwQ-14B-Math-v0.2", - "developer": "qingy2024", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/qingy2024_QwQ-14B-Math-v0.2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3391 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5731 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4811 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2626 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4021 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.48 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qq8933_openlongcot-base-gemma2-2b.json b/data/models/qq8933_openlongcot-base-gemma2-2b.json deleted file mode 100644 index f52f07f97802faf53aad09f1e16e89ba5a0dcf4f..0000000000000000000000000000000000000000 --- a/data/models/qq8933_openlongcot-base-gemma2-2b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "OpenLongCoT-Base-Gemma2-2B", - "id": "qq8933/OpenLongCoT-Base-Gemma2-2B", - "developer": "qq8933", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "3.204" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/qq8933_OpenLongCoT-Base-Gemma2-2B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1965 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3106 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0234 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2626 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3222 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1316 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/quazim0t0_1up-14b.json b/data/models/quazim0t0_1up-14b.json deleted file mode 100644 index be095785768aa2808b0536099b789b9335af3ee5..0000000000000000000000000000000000000000 --- a/data/models/quazim0t0_1up-14b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "1up-14b", - "id": "Quazim0t0/1up-14b", - "developer": "Quazim0t0", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Quazim0t0_1up-14b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6888 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6921 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4162 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3624 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4583 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5406 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/quazim0t0_adamant-14b-sce.json b/data/models/quazim0t0_adamant-14b-sce.json deleted file mode 100644 index fb03b561382bd637194d67a3ba084e948ef4ee63..0000000000000000000000000000000000000000 --- a/data/models/quazim0t0_adamant-14b-sce.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Adamant-14B-sce", - "id": "Quazim0t0/Adamant-14B-sce", - "developer": "Quazim0t0", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Quazim0t0_Adamant-14B-sce/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6858 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6859 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3988 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3507 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4558 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5372 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/quazim0t0_alice-14b.json b/data/models/quazim0t0_alice-14b.json deleted file mode 100644 index d6a081e150a2ae5c3e6c480f7ec054df947f4e9a..0000000000000000000000000000000000000000 --- a/data/models/quazim0t0_alice-14b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Alice-14B", - "id": "Quazim0t0/Alice-14B", - "developer": "Quazim0t0", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Quazim0t0_Alice-14B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6836 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6938 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4569 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3515 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4479 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5419 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/quazim0t0_alien-cot-14b-sce.json b/data/models/quazim0t0_alien-cot-14b-sce.json deleted file mode 100644 index 109b685e2514d4a6b77f4eb288f1ef636868264a..0000000000000000000000000000000000000000 --- a/data/models/quazim0t0_alien-cot-14b-sce.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Alien-CoT-14B-sce", - "id": "Quazim0t0/Alien-CoT-14B-sce", - "developer": "Quazim0t0", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Quazim0t0_Alien-CoT-14B-sce/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0749 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6395 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5204 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3918 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4785 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.517 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/quazim0t0_aura-8b-linear.json b/data/models/quazim0t0_aura-8b-linear.json deleted file mode 100644 index 3964ff5f9c6ce634e0de3b21fbe90c6bd8688603..0000000000000000000000000000000000000000 --- a/data/models/quazim0t0_aura-8b-linear.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Aura-8B-Linear", - "id": "Quazim0t0/Aura-8B-Linear", - "developer": "Quazim0t0", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Quazim0t0_Aura-8B-Linear/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7948 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5074 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1805 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2693 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3687 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3801 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/quazim0t0_bloom-14b-stock.json b/data/models/quazim0t0_bloom-14b-stock.json deleted file mode 100644 index 5e63b4c42bd7eb11eab528d35feede975c74293f..0000000000000000000000000000000000000000 --- a/data/models/quazim0t0_bloom-14b-stock.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "bloom-14b-stock", - "id": "Quazim0t0/bloom-14b-stock", - "developer": "Quazim0t0", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Quazim0t0_bloom-14b-stock/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6575 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6878 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4811 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3314 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.431 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5373 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/quazim0t0_caramel-14b.json b/data/models/quazim0t0_caramel-14b.json deleted file mode 100644 index e007c7daa80d1070c14fdd7d979e6842e5341a68..0000000000000000000000000000000000000000 --- a/data/models/quazim0t0_caramel-14b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "caramel-14B", - "id": "Quazim0t0/caramel-14B", - "developer": "Quazim0t0", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Quazim0t0_caramel-14B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6745 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6919 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4713 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3448 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4454 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5436 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/quazim0t0_casa-14b-sce.json b/data/models/quazim0t0_casa-14b-sce.json deleted file mode 100644 index 5121da283045b50c1becfc23584ab4044d6d9721..0000000000000000000000000000000000000000 --- a/data/models/quazim0t0_casa-14b-sce.json +++ /dev/null @@ -1,275 +0,0 @@ -{ - "model_info": { - "name": "Casa-14b-sce", - "id": "Quazim0t0/Casa-14b-sce", - "developer": "Quazim0t0", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Quazim0t0_Casa-14b-sce/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6718 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6891 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4985 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3339 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4323 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5408 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "hfopenllm_v2/Quazim0t0_Casa-14b-sce/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6654 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6901 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4698 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3331 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.431 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5426 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/quazim0t0_charlie-8b-linear.json b/data/models/quazim0t0_charlie-8b-linear.json deleted file mode 100644 index 875d6d247ccdc1dd9b2368ac8f8de7d1c7c41b9e..0000000000000000000000000000000000000000 --- a/data/models/quazim0t0_charlie-8b-linear.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Charlie-8B-Linear", - "id": "Quazim0t0/Charlie-8B-Linear", - "developer": "Quazim0t0", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Quazim0t0_Charlie-8B-Linear/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7381 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5141 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2651 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.271 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3485 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3573 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/quazim0t0_chromatic-8b-sce.json b/data/models/quazim0t0_chromatic-8b-sce.json deleted file mode 100644 index e742d4b52af2663bc0e26e92a3442bd1df2cf46e..0000000000000000000000000000000000000000 --- a/data/models/quazim0t0_chromatic-8b-sce.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Chromatic-8b-sce", - "id": "Quazim0t0/Chromatic-8b-sce", - "developer": "Quazim0t0", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Quazim0t0_Chromatic-8b-sce/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5085 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5063 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1556 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3196 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4051 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3755 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/quazim0t0_cot_phi.json b/data/models/quazim0t0_cot_phi.json deleted file mode 100644 index c4bda77890abe8f50bfbb212530128610e0c4601..0000000000000000000000000000000000000000 --- a/data/models/quazim0t0_cot_phi.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "CoT_Phi", - "id": "Quazim0t0/CoT_Phi", - "developer": "Quazim0t0", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Quazim0t0_CoT_Phi/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6159 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6751 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3308 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3582 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4244 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4901 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/quazim0t0_dyson-14b.json b/data/models/quazim0t0_dyson-14b.json deleted file mode 100644 index 042c2ef245168bc9cdb1b1cddcdcfa7f5c3b4252..0000000000000000000000000000000000000000 --- a/data/models/quazim0t0_dyson-14b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Dyson-14b", - "id": "Quazim0t0/Dyson-14b", - "developer": "Quazim0t0", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Quazim0t0_Dyson-14b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5857 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6863 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5393 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3138 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4259 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5399 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/quazim0t0_edu-14b-linear.json b/data/models/quazim0t0_edu-14b-linear.json deleted file mode 100644 index 2dd4b590016bcc501d15d660bbe752dbd2f2fe7c..0000000000000000000000000000000000000000 --- a/data/models/quazim0t0_edu-14b-linear.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Edu-14B-Linear", - "id": "Quazim0t0/Edu-14B-Linear", - "developer": "Quazim0t0", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Quazim0t0_Edu-14B-Linear/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6158 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6758 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2447 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3171 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4378 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5086 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/quazim0t0_fugazi14b.json b/data/models/quazim0t0_fugazi14b.json deleted file mode 100644 index ab29f7a17917f02fabbdafa11cccc5d4e0b26fc5..0000000000000000000000000000000000000000 --- a/data/models/quazim0t0_fugazi14b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Fugazi14b", - "id": "Quazim0t0/Fugazi14b", - "developer": "Quazim0t0", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Quazim0t0_Fugazi14b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6998 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6941 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4653 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3515 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4546 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5417 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/quazim0t0_geedorah-14b.json b/data/models/quazim0t0_geedorah-14b.json deleted file mode 100644 index b35aa7f87cbf32cd2a6db007dfacbd3d4536a6d8..0000000000000000000000000000000000000000 --- a/data/models/quazim0t0_geedorah-14b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Geedorah-14B", - "id": "Quazim0t0/Geedorah-14B", - "developer": "Quazim0t0", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Quazim0t0_Geedorah-14B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6873 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6964 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4449 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3473 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4547 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5421 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/quazim0t0_givingtree-8b-sce.json b/data/models/quazim0t0_givingtree-8b-sce.json deleted file mode 100644 index b32a52bf07dd255d130ea28df0b8477e82d4d156..0000000000000000000000000000000000000000 --- a/data/models/quazim0t0_givingtree-8b-sce.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "GivingTree-8b-sce", - "id": "Quazim0t0/GivingTree-8b-sce", - "developer": "Quazim0t0", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Quazim0t0_GivingTree-8b-sce/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5006 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.504 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1526 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3221 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4051 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3761 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/quazim0t0_graphite-14b-sce.json b/data/models/quazim0t0_graphite-14b-sce.json deleted file mode 100644 index 3af5538d67c27e64ad4d1aaeabea581ed5475659..0000000000000000000000000000000000000000 --- a/data/models/quazim0t0_graphite-14b-sce.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "graphite-14b-sce", - "id": "Quazim0t0/graphite-14b-sce", - "developer": "Quazim0t0", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Quazim0t0_graphite-14b-sce/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3217 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6631 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3006 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2894 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3981 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.528 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/quazim0t0_guiltyspark-14b-ties.json b/data/models/quazim0t0_guiltyspark-14b-ties.json deleted file mode 100644 index 3679cdbf11df37ae052acd3694dabf3db674d44c..0000000000000000000000000000000000000000 --- a/data/models/quazim0t0_guiltyspark-14b-ties.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "GuiltySpark-14B-ties", - "id": "Quazim0t0/GuiltySpark-14B-ties", - "developer": "Quazim0t0", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Quazim0t0_GuiltySpark-14B-ties/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6854 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6914 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3837 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3649 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4557 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.54 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/quazim0t0_gza-14b-sce.json b/data/models/quazim0t0_gza-14b-sce.json deleted file mode 100644 index 503dfe013e6e4da416f8120d9adbb2b25cc51be6..0000000000000000000000000000000000000000 --- a/data/models/quazim0t0_gza-14b-sce.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "GZA-14B-sce", - "id": "Quazim0t0/GZA-14B-sce", - "developer": "Quazim0t0", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Quazim0t0_GZA-14B-sce/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6274 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6687 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4721 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.302 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4285 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5232 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/quazim0t0_halo-14b-sce.json b/data/models/quazim0t0_halo-14b-sce.json deleted file mode 100644 index 75885881bb6ba990cba57de705b9458e27b6d98f..0000000000000000000000000000000000000000 --- a/data/models/quazim0t0_halo-14b-sce.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Halo-14B-sce", - "id": "Quazim0t0/Halo-14B-sce", - "developer": "Quazim0t0", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Quazim0t0_Halo-14B-sce/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6754 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6876 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.429 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3473 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4401 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5376 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/quazim0t0_heretic1.5b.json b/data/models/quazim0t0_heretic1.5b.json deleted file mode 100644 index a7b09d2758287bbade224932f6344d0225552615..0000000000000000000000000000000000000000 --- a/data/models/quazim0t0_heretic1.5b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Heretic1.5b", - "id": "Quazim0t0/Heretic1.5b", - "developer": "Quazim0t0", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.73" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Quazim0t0_Heretic1.5b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2062 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3529 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.244 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2685 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3511 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1728 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/quazim0t0_hyde-14b-sce.json b/data/models/quazim0t0_hyde-14b-sce.json deleted file mode 100644 index 9adca77ad2f7b53d72d8dc0b9c82fcb4a945989a..0000000000000000000000000000000000000000 --- a/data/models/quazim0t0_hyde-14b-sce.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Hyde-14b-sce", - "id": "Quazim0t0/Hyde-14b-sce", - "developer": "Quazim0t0", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Quazim0t0_Hyde-14b-sce/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6715 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6885 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2734 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3414 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4141 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.53 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/quazim0t0_imagine-v0.5-16bit.json b/data/models/quazim0t0_imagine-v0.5-16bit.json deleted file mode 100644 index 96323abc70209fc1496d61b896469067e2561403..0000000000000000000000000000000000000000 --- a/data/models/quazim0t0_imagine-v0.5-16bit.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Imagine-v0.5-16bit", - "id": "Quazim0t0/Imagine-v0.5-16bit", - "developer": "Quazim0t0", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Quazim0t0_Imagine-v0.5-16bit/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2759 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6769 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1397 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3649 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4349 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5354 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/quazim0t0_imbue-14b.json b/data/models/quazim0t0_imbue-14b.json deleted file mode 100644 index 0ec816841d3bdc15e39f32a8016b314f17848ab9..0000000000000000000000000000000000000000 --- a/data/models/quazim0t0_imbue-14b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Imbue-14b", - "id": "Quazim0t0/Imbue-14b", - "developer": "Quazim0t0", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Quazim0t0_Imbue-14b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.52 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6845 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5317 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3129 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4167 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5402 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/quazim0t0_insom.json b/data/models/quazim0t0_insom.json deleted file mode 100644 index 12ef064517263473b7d3771a41064f650bcfddd8..0000000000000000000000000000000000000000 --- a/data/models/quazim0t0_insom.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Insom", - "id": "Quazim0t0/Insom", - "developer": "Quazim0t0", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Quazim0t0_Insom/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6818 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6881 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3852 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3498 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4311 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5352 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/quazim0t0_inspectordeck-14b-sce.json b/data/models/quazim0t0_inspectordeck-14b-sce.json deleted file mode 100644 index e70d5c29db6c476b8725be1119bec12a2486921f..0000000000000000000000000000000000000000 --- a/data/models/quazim0t0_inspectordeck-14b-sce.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "InspectorDeck-14B-sce", - "id": "Quazim0t0/InspectorDeck-14B-sce", - "developer": "Quazim0t0", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Quazim0t0_InspectorDeck-14B-sce/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3241 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6668 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3165 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.297 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3982 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5261 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/quazim0t0_jekyl-8b-sce.json b/data/models/quazim0t0_jekyl-8b-sce.json deleted file mode 100644 index 7d9a864584766dd5a7d1916219e8a644b349e4f8..0000000000000000000000000000000000000000 --- a/data/models/quazim0t0_jekyl-8b-sce.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Jekyl-8b-sce", - "id": "Quazim0t0/Jekyl-8b-sce", - "developer": "Quazim0t0", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Quazim0t0_Jekyl-8b-sce/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4697 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4994 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1616 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3381 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4197 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3686 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/quazim0t0_jigsaw-14b-linear.json b/data/models/quazim0t0_jigsaw-14b-linear.json deleted file mode 100644 index 9d0f8b66fb85ba91378e1b3245c2ddb810dc6369..0000000000000000000000000000000000000000 --- a/data/models/quazim0t0_jigsaw-14b-linear.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Jigsaw-14B-Linear", - "id": "Quazim0t0/Jigsaw-14B-Linear", - "developer": "Quazim0t0", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Quazim0t0_Jigsaw-14B-Linear/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.648 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6865 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2651 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3406 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4483 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5234 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/quazim0t0_katana-8b-sce.json b/data/models/quazim0t0_katana-8b-sce.json deleted file mode 100644 index b69b37631df4edc84710d85a9377d7907e240f9e..0000000000000000000000000000000000000000 --- a/data/models/quazim0t0_katana-8b-sce.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Katana-8b-sce", - "id": "Quazim0t0/Katana-8b-sce", - "developer": "Quazim0t0", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Quazim0t0_Katana-8b-sce/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5107 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5075 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1511 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3247 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4038 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3771 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/quazim0t0_knot-cot-14b-sce.json b/data/models/quazim0t0_knot-cot-14b-sce.json deleted file mode 100644 index fbe7af34f90e24377bcf3313046bae3560a943a4..0000000000000000000000000000000000000000 --- a/data/models/quazim0t0_knot-cot-14b-sce.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Knot-CoT-14B-sce", - "id": "Quazim0t0/Knot-CoT-14B-sce", - "developer": "Quazim0t0", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Quazim0t0_Knot-CoT-14B-sce/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4832 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6616 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3995 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2936 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.414 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5154 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/quazim0t0_lineage-14b.json b/data/models/quazim0t0_lineage-14b.json deleted file mode 100644 index a1c98acc6210022e2712ac17e793bc3927e8e8f5..0000000000000000000000000000000000000000 --- a/data/models/quazim0t0_lineage-14b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Lineage-14B", - "id": "Quazim0t0/Lineage-14B", - "developer": "Quazim0t0", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Quazim0t0_Lineage-14B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.707 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6934 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4245 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3599 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4597 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5411 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/quazim0t0_lo-phi-14b.json b/data/models/quazim0t0_lo-phi-14b.json deleted file mode 100644 index 1b85c121c766f3fee8ee922a7dc303ca120719a1..0000000000000000000000000000000000000000 --- a/data/models/quazim0t0_lo-phi-14b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Lo-Phi-14b", - "id": "Quazim0t0/Lo-Phi-14b", - "developer": "Quazim0t0", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Quazim0t0_Lo-Phi-14b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4941 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6852 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5196 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.328 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4232 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5369 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/quazim0t0_loke-14b-sce.json b/data/models/quazim0t0_loke-14b-sce.json deleted file mode 100644 index 77ffb199c73cfca8ddebd70c5e8d6c7c04f7f31e..0000000000000000000000000000000000000000 --- a/data/models/quazim0t0_loke-14b-sce.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Loke-14B-sce", - "id": "Quazim0t0/Loke-14B-sce", - "developer": "Quazim0t0", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Quazim0t0_Loke-14B-sce/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6848 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6924 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3905 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3649 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4637 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5401 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/quazim0t0_math_phi4_reason.json b/data/models/quazim0t0_math_phi4_reason.json deleted file mode 100644 index ca9ca6d2f4ff6330abdf3c3564ec23a6fcf2ca36..0000000000000000000000000000000000000000 --- a/data/models/quazim0t0_math_phi4_reason.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Math_Phi4_Reason", - "id": "Quazim0t0/Math_Phi4_Reason", - "developer": "Quazim0t0", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Quazim0t0_Math_Phi4_Reason/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.322 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.624 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3278 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2903 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4034 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.503 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/quazim0t0_mfdoom-14b.json b/data/models/quazim0t0_mfdoom-14b.json deleted file mode 100644 index be0dbe4edc92d709a7a9100b31725e8562de6040..0000000000000000000000000000000000000000 --- a/data/models/quazim0t0_mfdoom-14b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MFDOOM-14B", - "id": "Quazim0t0/MFDOOM-14B", - "developer": "Quazim0t0", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Quazim0t0_MFDOOM-14B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6736 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6916 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5264 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.323 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4377 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5426 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/quazim0t0_mfgrimm-14b.json b/data/models/quazim0t0_mfgrimm-14b.json deleted file mode 100644 index 3fe415b37ab587175aac7c23f88c83900c025b3a..0000000000000000000000000000000000000000 --- a/data/models/quazim0t0_mfgrimm-14b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MFGRIMM-14B", - "id": "Quazim0t0/MFGRIMM-14B", - "developer": "Quazim0t0", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Quazim0t0_MFGRIMM-14B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6894 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6909 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.506 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3339 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4361 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5416 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/quazim0t0_mithril-14b-sce.json b/data/models/quazim0t0_mithril-14b-sce.json deleted file mode 100644 index 2a766ab969c917f7d591ac05da1e7a14d9c2ad6c..0000000000000000000000000000000000000000 --- a/data/models/quazim0t0_mithril-14b-sce.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mithril-14B-sce", - "id": "Quazim0t0/Mithril-14B-sce", - "developer": "Quazim0t0", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Quazim0t0_Mithril-14B-sce/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6958 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6926 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3822 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3691 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4611 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5403 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/quazim0t0_mocha-14b.json b/data/models/quazim0t0_mocha-14b.json deleted file mode 100644 index 176236f9945254ec368b54b6334ff617928070f9..0000000000000000000000000000000000000000 --- a/data/models/quazim0t0_mocha-14b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "mocha-14B", - "id": "Quazim0t0/mocha-14B", - "developer": "Quazim0t0", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Quazim0t0_mocha-14B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5893 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6895 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5264 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3305 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4272 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5384 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/quazim0t0_mononoke-14b-sce.json b/data/models/quazim0t0_mononoke-14b-sce.json deleted file mode 100644 index 628a094290d470ac348fb2577d7f2108c9e6202d..0000000000000000000000000000000000000000 --- a/data/models/quazim0t0_mononoke-14b-sce.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mononoke-14B-sce", - "id": "Quazim0t0/Mononoke-14B-sce", - "developer": "Quazim0t0", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Quazim0t0_Mononoke-14B-sce/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3502 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6744 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4698 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.323 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4155 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5298 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/quazim0t0_mosaic-14b-sce.json b/data/models/quazim0t0_mosaic-14b-sce.json deleted file mode 100644 index 94a985ec377c5775f2902853b17646b07ea7b4f1..0000000000000000000000000000000000000000 --- a/data/models/quazim0t0_mosaic-14b-sce.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "mosaic-14b-sce", - "id": "Quazim0t0/mosaic-14b-sce", - "developer": "Quazim0t0", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Quazim0t0_mosaic-14b-sce/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6876 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6907 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4026 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3624 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4558 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5396 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/quazim0t0_motion-8b-linear.json b/data/models/quazim0t0_motion-8b-linear.json deleted file mode 100644 index d496cef72f9c9207ea50540b9832831954be1540..0000000000000000000000000000000000000000 --- a/data/models/quazim0t0_motion-8b-linear.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Motion-8B-Linear", - "id": "Quazim0t0/Motion-8B-Linear", - "developer": "Quazim0t0", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Quazim0t0_Motion-8B-Linear/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7686 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5084 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1888 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.271 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3606 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3785 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/quazim0t0_mouse-9b.json b/data/models/quazim0t0_mouse-9b.json deleted file mode 100644 index a39d4920599d8db6ce56c328f2151963f1abe9de..0000000000000000000000000000000000000000 --- a/data/models/quazim0t0_mouse-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mouse-9B", - "id": "Quazim0t0/Mouse-9B", - "developer": "Quazim0t0", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "9.207" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Quazim0t0_Mouse-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1325 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2979 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0053 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2542 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.347 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1139 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/quazim0t0_nova-14b-sce.json b/data/models/quazim0t0_nova-14b-sce.json deleted file mode 100644 index 543b614db8666b3ea0e60e56ec2376b36c396e74..0000000000000000000000000000000000000000 --- a/data/models/quazim0t0_nova-14b-sce.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Nova-14b-sce", - "id": "Quazim0t0/Nova-14b-sce", - "developer": "Quazim0t0", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Quazim0t0_Nova-14b-sce/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7022 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6935 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4162 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3633 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4571 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5413 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/quazim0t0_novascotia-14b-stock.json b/data/models/quazim0t0_novascotia-14b-stock.json deleted file mode 100644 index 2d852993843b931bf6e12051938de83b2710ad23..0000000000000000000000000000000000000000 --- a/data/models/quazim0t0_novascotia-14b-stock.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "NovaScotia-14b-stock", - "id": "Quazim0t0/NovaScotia-14b-stock", - "developer": "Quazim0t0", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Quazim0t0_NovaScotia-14b-stock/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6787 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6935 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.463 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.349 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4493 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5409 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/quazim0t0_oasis-14b-ties.json b/data/models/quazim0t0_oasis-14b-ties.json deleted file mode 100644 index 113246e709b2bd625077362b9ced96d253e1f63b..0000000000000000000000000000000000000000 --- a/data/models/quazim0t0_oasis-14b-ties.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Oasis-14B-ties", - "id": "Quazim0t0/Oasis-14B-ties", - "developer": "Quazim0t0", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Quazim0t0_Oasis-14B-ties/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6937 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6915 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3754 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3649 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4571 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5405 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/quazim0t0_odb-14b-sce.json b/data/models/quazim0t0_odb-14b-sce.json deleted file mode 100644 index b2854fae2e7ea9ad5a32eb4ec453ef9c1365f1b7..0000000000000000000000000000000000000000 --- a/data/models/quazim0t0_odb-14b-sce.json +++ /dev/null @@ -1,278 +0,0 @@ -{ - "model_info": { - "name": "ODB-14B-sce", - "id": "Quazim0t0/ODB-14B-sce", - "developer": "Quazim0t0", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66", - "model_id_aliases": [ - "Quazim0t0/ODB-14b-sce" - ] - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Quazim0t0_ODB-14b-sce/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7016 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6942 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4116 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3624 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4571 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5411 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "hfopenllm_v2/Quazim0t0_ODB-14B-sce/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2922 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6559 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2545 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2659 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3929 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5207 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/quazim0t0_origami-14b-sce.json b/data/models/quazim0t0_origami-14b-sce.json deleted file mode 100644 index c3fa6bf196fad27833f498eaef9b3fdcae4ff687..0000000000000000000000000000000000000000 --- a/data/models/quazim0t0_origami-14b-sce.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Origami-14B-sce", - "id": "Quazim0t0/Origami-14B-sce", - "developer": "Quazim0t0", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Quazim0t0_Origami-14B-sce/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3259 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.662 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2915 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2836 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4035 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5244 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/quazim0t0_phi4.turn.r1distill.16bit.json b/data/models/quazim0t0_phi4.turn.r1distill.16bit.json deleted file mode 100644 index 6dd27dcaac701e4ad24857970b666a47bb377aa9..0000000000000000000000000000000000000000 --- a/data/models/quazim0t0_phi4.turn.r1distill.16bit.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Phi4.Turn.R1Distill.16bit", - "id": "Quazim0t0/Phi4.Turn.R1Distill.16bit", - "developer": "Quazim0t0", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Quazim0t0_Phi4.Turn.R1Distill.16bit/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3126 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6563 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2311 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2945 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3902 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5257 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/quazim0t0_phi4.turn.r1distill_v1.5.1-tensors.json b/data/models/quazim0t0_phi4.turn.r1distill_v1.5.1-tensors.json deleted file mode 100644 index 3c73b14de36edc8d4afd1189da3c4b6d7d479ad5..0000000000000000000000000000000000000000 --- a/data/models/quazim0t0_phi4.turn.r1distill_v1.5.1-tensors.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Phi4.Turn.R1Distill_v1.5.1-Tensors", - "id": "Quazim0t0/Phi4.Turn.R1Distill_v1.5.1-Tensors", - "developer": "Quazim0t0", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Quazim0t0_Phi4.Turn.R1Distill_v1.5.1-Tensors/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2995 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6456 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.219 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2685 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3929 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5117 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/quazim0t0_phi4basis-14b-sce.json b/data/models/quazim0t0_phi4basis-14b-sce.json deleted file mode 100644 index 7ea64e48e1b9a93da9f34ba9137e133bee5f9fe0..0000000000000000000000000000000000000000 --- a/data/models/quazim0t0_phi4basis-14b-sce.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Phi4Basis-14B-sce", - "id": "Quazim0t0/Phi4Basis-14B-sce", - "developer": "Quazim0t0", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Quazim0t0_Phi4Basis-14B-sce/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6502 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6909 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4789 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3289 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4338 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.539 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/quazim0t0_ponder-14b-linear.json b/data/models/quazim0t0_ponder-14b-linear.json deleted file mode 100644 index b53417e743d46824f1751c92ea597a1c7471ebaf..0000000000000000000000000000000000000000 --- a/data/models/quazim0t0_ponder-14b-linear.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Ponder-14B-linear", - "id": "Quazim0t0/Ponder-14B-linear", - "developer": "Quazim0t0", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Quazim0t0_Ponder-14B-linear/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6906 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6943 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4282 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3582 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4558 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5408 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/quazim0t0_rosemary-14b.json b/data/models/quazim0t0_rosemary-14b.json deleted file mode 100644 index c475b8f01078b2c1bd3d31d5a959b54e4ce33c9d..0000000000000000000000000000000000000000 --- a/data/models/quazim0t0_rosemary-14b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Rosemary-14b", - "id": "Quazim0t0/Rosemary-14b", - "developer": "Quazim0t0", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Quazim0t0_Rosemary-14b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6915 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6955 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4388 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3565 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4492 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5396 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/quazim0t0_rune-14b.json b/data/models/quazim0t0_rune-14b.json deleted file mode 100644 index a32e09bddc68a5ea9707579ce7e93cf860856a04..0000000000000000000000000000000000000000 --- a/data/models/quazim0t0_rune-14b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Rune-14b", - "id": "Quazim0t0/Rune-14b", - "developer": "Quazim0t0", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Quazim0t0_Rune-14b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7016 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6937 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4585 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3515 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4533 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5411 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/quazim0t0_rza-14b-sce.json b/data/models/quazim0t0_rza-14b-sce.json deleted file mode 100644 index cf80966fd887cf7290d83bd7a1eeb0273db94387..0000000000000000000000000000000000000000 --- a/data/models/quazim0t0_rza-14b-sce.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "RZA-14B-sce", - "id": "Quazim0t0/RZA-14B-sce", - "developer": "Quazim0t0", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Quazim0t0_RZA-14B-sce/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4774 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6686 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5189 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2903 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4113 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5383 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/quazim0t0_sake-20b.json b/data/models/quazim0t0_sake-20b.json deleted file mode 100644 index 846e739c2ff810cc4f539957f59eeb645b1d2ba8..0000000000000000000000000000000000000000 --- a/data/models/quazim0t0_sake-20b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Sake-20b", - "id": "Quazim0t0/Sake-20b", - "developer": "Quazim0t0", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "21.475" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Quazim0t0_Sake-20b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6693 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.677 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4653 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3188 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4494 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5391 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/quazim0t0_spok-14b-sce.json b/data/models/quazim0t0_spok-14b-sce.json deleted file mode 100644 index dd8eaea71ee0aa26bbc15c2c1743bf1cb75e0f40..0000000000000000000000000000000000000000 --- a/data/models/quazim0t0_spok-14b-sce.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Spok-14b-sce", - "id": "Quazim0t0/Spok-14b-sce", - "developer": "Quazim0t0", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Quazim0t0_Spok-14b-sce/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6682 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6899 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2719 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3456 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4141 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5298 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/quazim0t0_sumatra-20b.json b/data/models/quazim0t0_sumatra-20b.json deleted file mode 100644 index 600b7e8ff2432c5551020ede3bba845b91307766..0000000000000000000000000000000000000000 --- a/data/models/quazim0t0_sumatra-20b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Sumatra-20b", - "id": "Quazim0t0/Sumatra-20b", - "developer": "Quazim0t0", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "21.475" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Quazim0t0_Sumatra-20b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6738 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6855 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3671 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3263 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.456 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5415 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/quazim0t0_supernova14b.json b/data/models/quazim0t0_supernova14b.json deleted file mode 100644 index f698f893d392db302ad3f3e3c37f647fc75f643d..0000000000000000000000000000000000000000 --- a/data/models/quazim0t0_supernova14b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SuperNova14b", - "id": "Quazim0t0/SuperNova14b", - "developer": "Quazim0t0", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Quazim0t0_SuperNova14b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7076 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6937 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4396 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3523 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4545 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5435 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/quazim0t0_sza-14b-sce.json b/data/models/quazim0t0_sza-14b-sce.json deleted file mode 100644 index ca5be53ace841a25578cf7d6165a2ccbc5bfce7a..0000000000000000000000000000000000000000 --- a/data/models/quazim0t0_sza-14b-sce.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SZA-14B-sce", - "id": "Quazim0t0/SZA-14B-sce", - "developer": "Quazim0t0", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Quazim0t0_SZA-14B-sce/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5659 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6889 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5242 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3305 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4339 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5353 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/quazim0t0_tb0-8b-sce.json b/data/models/quazim0t0_tb0-8b-sce.json deleted file mode 100644 index 6cb1757174dc02192f0089352ea4fd493092f835..0000000000000000000000000000000000000000 --- a/data/models/quazim0t0_tb0-8b-sce.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "TB0-8B-sce", - "id": "Quazim0t0/TB0-8B-sce", - "developer": "Quazim0t0", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Quazim0t0_TB0-8B-sce/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5107 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5075 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1511 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3247 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4038 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3771 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/quazim0t0_tbl-8b-sce.json b/data/models/quazim0t0_tbl-8b-sce.json deleted file mode 100644 index 1501064c789b8fa5d4a409eb8297c9ca06668061..0000000000000000000000000000000000000000 --- a/data/models/quazim0t0_tbl-8b-sce.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "TBL-8B-sce", - "id": "Quazim0t0/TBL-8B-sce", - "developer": "Quazim0t0", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Quazim0t0_TBL-8B-sce/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4581 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5008 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1533 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3339 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4236 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3689 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/quazim0t0_tesseract-14b-stock.json b/data/models/quazim0t0_tesseract-14b-stock.json deleted file mode 100644 index 393ebaf59d0038a0d3fe1516715b53f965e99d1c..0000000000000000000000000000000000000000 --- a/data/models/quazim0t0_tesseract-14b-stock.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "tesseract-14b-stock", - "id": "Quazim0t0/tesseract-14b-stock", - "developer": "Quazim0t0", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Quazim0t0_tesseract-14b-stock/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5848 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.688 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5144 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3272 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4232 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5389 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/quazim0t0_thinkphi1.1-tensors.json b/data/models/quazim0t0_thinkphi1.1-tensors.json deleted file mode 100644 index dc9929656ca79fa120e7437d4bba65e2fc9cf601..0000000000000000000000000000000000000000 --- a/data/models/quazim0t0_thinkphi1.1-tensors.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ThinkPhi1.1-Tensors", - "id": "Quazim0t0/ThinkPhi1.1-Tensors", - "developer": "Quazim0t0", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Quazim0t0_ThinkPhi1.1-Tensors/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3908 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6449 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.182 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2987 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.418 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4908 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/quazim0t0_time-14b-stock.json b/data/models/quazim0t0_time-14b-stock.json deleted file mode 100644 index 27e452b59b6250533cfebf6329a39cb0eb435e4d..0000000000000000000000000000000000000000 --- a/data/models/quazim0t0_time-14b-stock.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "time-14b-stock", - "id": "Quazim0t0/time-14b-stock", - "developer": "Quazim0t0", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Quazim0t0_time-14b-stock/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6699 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6897 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5083 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3347 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4323 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5419 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/quazim0t0_venti-20b.json b/data/models/quazim0t0_venti-20b.json deleted file mode 100644 index e64db01e43806195c71e797f212da494df7d7a3e..0000000000000000000000000000000000000000 --- a/data/models/quazim0t0_venti-20b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Venti-20b", - "id": "Quazim0t0/Venti-20b", - "developer": "Quazim0t0", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "21.475" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Quazim0t0_Venti-20b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6641 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6901 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3391 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3322 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.448 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5386 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/quazim0t0_venti-blend-sce.json b/data/models/quazim0t0_venti-blend-sce.json deleted file mode 100644 index fac454f887bfea966b344543eac2662d19c57fff..0000000000000000000000000000000000000000 --- a/data/models/quazim0t0_venti-blend-sce.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Venti-Blend-sce", - "id": "Quazim0t0/Venti-Blend-sce", - "developer": "Quazim0t0", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "21.475" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Quazim0t0_Venti-Blend-sce/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6879 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6843 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4056 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3163 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4389 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5414 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/quazim0t0_vine-14b-sce.json b/data/models/quazim0t0_vine-14b-sce.json deleted file mode 100644 index 7e36aa68f4f3ae4db14ba1cb07c20398df9036fd..0000000000000000000000000000000000000000 --- a/data/models/quazim0t0_vine-14b-sce.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Vine-14b-sce", - "id": "Quazim0t0/Vine-14b-sce", - "developer": "Quazim0t0", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Quazim0t0_Vine-14b-sce/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6733 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6891 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5008 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3339 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4323 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5408 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/quazim0t0_wendy-14b.json b/data/models/quazim0t0_wendy-14b.json deleted file mode 100644 index 21f60049de44149dea0eb52ca0bd523dc958d726..0000000000000000000000000000000000000000 --- a/data/models/quazim0t0_wendy-14b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Wendy-14B", - "id": "Quazim0t0/Wendy-14B", - "developer": "Quazim0t0", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Quazim0t0_Wendy-14B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6772 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6958 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4834 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3322 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4428 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5435 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/quazim0t0_wu-14b-sce.json b/data/models/quazim0t0_wu-14b-sce.json deleted file mode 100644 index dcf482b6e489f6cc3a7bf857a38034eb7a7c7a7b..0000000000000000000000000000000000000000 --- a/data/models/quazim0t0_wu-14b-sce.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Wu-14b-sce", - "id": "Quazim0t0/Wu-14b-sce", - "developer": "Quazim0t0", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Quazim0t0_Wu-14b-sce/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6718 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6885 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2613 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3465 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4114 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5293 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qwen_qwen1.5-0.5b-chat.json b/data/models/qwen_qwen1.5-0.5b-chat.json deleted file mode 100644 index 1264236ebe86c900571ec79b0460075e7a2f2e23..0000000000000000000000000000000000000000 --- a/data/models/qwen_qwen1.5-0.5b-chat.json +++ /dev/null @@ -1,277 +0,0 @@ -{ - "model_info": { - "name": "Qwen/Qwen1.5-0.5B-Chat", - "id": "Qwen/Qwen1.5-0.5B-Chat", - "developer": "Qwen", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.62" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Qwen_Qwen1.5-0.5B-Chat/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1807 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3167 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0068 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2693 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3837 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1213 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "reward-bench/Qwen_Qwen1.5-0.5B-Chat/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5298 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3547 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6294 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5703 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5984 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4629 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qwen_qwen1.5-0.5b.json b/data/models/qwen_qwen1.5-0.5b.json deleted file mode 100644 index 123761b84e4726d2cd2a198f4bb7e6d8eadaf718..0000000000000000000000000000000000000000 --- a/data/models/qwen_qwen1.5-0.5b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen1.5-0.5B", - "id": "Qwen/Qwen1.5-0.5B", - "developer": "Qwen", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.62" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Qwen_Qwen1.5-0.5B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1706 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3154 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0174 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2542 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3616 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1307 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qwen_qwen1.5-1.8b-chat.json b/data/models/qwen_qwen1.5-1.8b-chat.json deleted file mode 100644 index 35d550bc4384d3dd960d103ab7ba952d92ed892b..0000000000000000000000000000000000000000 --- a/data/models/qwen_qwen1.5-1.8b-chat.json +++ /dev/null @@ -1,277 +0,0 @@ -{ - "model_info": { - "name": "Qwen/Qwen1.5-1.8B-Chat", - "id": "Qwen/Qwen1.5-1.8B-Chat", - "developer": "Qwen", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.837" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Qwen_Qwen1.5-1.8B-Chat/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2019 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3256 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0196 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2978 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.426 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1804 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "reward-bench/Qwen_Qwen1.5-1.8B-Chat/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.589 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5615 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6031 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4838 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7793 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4453 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qwen_qwen1.5-1.8b.json b/data/models/qwen_qwen1.5-1.8b.json deleted file mode 100644 index 5144c33e81284f62ef300dbbbf6c31d10a58199e..0000000000000000000000000000000000000000 --- a/data/models/qwen_qwen1.5-1.8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen1.5-1.8B", - "id": "Qwen/Qwen1.5-1.8B", - "developer": "Qwen", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.837" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Qwen_Qwen1.5-1.8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2154 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3476 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0317 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3054 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3605 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1882 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qwen_qwen1.5-110b-chat.json b/data/models/qwen_qwen1.5-110b-chat.json deleted file mode 100644 index fe65d81884cb34b664b90e913fc2c6e516757328..0000000000000000000000000000000000000000 --- a/data/models/qwen_qwen1.5-110b-chat.json +++ /dev/null @@ -1,2038 +0,0 @@ -{ - "model_info": { - "name": "Qwen1.5 Chat 110B", - "id": "qwen/qwen1.5-110b-chat", - "developer": "qwen", - "inference_platform": "unknown", - "additional_details": { - "model_id_aliases": [ - "Qwen/Qwen1.5-110B-Chat" - ] - } - }, - "evaluations": [ - { - "evaluation_id": "helm_lite/qwen_qwen1.5-110b-chat/1774096306.427425", - "retrieved_timestamp": "1774096306.427425", - "source_metadata": { - "source_name": "helm_lite", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_lite", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_lite", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.55, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.6592634207240948\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.721, - "details": { - "description": "min=0.721, mean=0.721, max=0.721, sum=0.721 (1)", - "tab": "Accuracy", - "NarrativeQA - Observed inference time (s)": "{\"description\": \"min=0.984, mean=0.984, max=0.984, sum=0.984 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.9843533623386437\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=3502.913, mean=3502.913, max=3502.913, sum=3502.913 (1)\", \"tab\": \"General information\", \"score\": \"3502.912676056338\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=10.29, mean=10.29, max=10.29, sum=10.29 (1)\", \"tab\": \"General information\", \"score\": \"10.290140845070422\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (closed-book)", - "source_data": { - "dataset_name": "NaturalQuestions (closed-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (closed-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.35, - "details": { - "description": "min=0.35, mean=0.35, max=0.35, sum=0.35 (1)", - "tab": "Accuracy", - "NaturalQuestions (open-book) - Observed inference time (s)": "{\"description\": \"min=0.647, mean=0.647, max=0.647, sum=0.647 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.6468759918212891\"}", - "NaturalQuestions (closed-book) - Observed inference time (s)": "{\"description\": \"min=0.465, mean=0.465, max=0.465, sum=0.465 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.46513359355926515\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=2017.955, mean=2017.955, max=2017.955, sum=2017.955 (1)\", \"tab\": \"General information\", \"score\": \"2017.955\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=8.509, mean=8.509, max=8.509, sum=8.509 (1)\", \"tab\": \"General information\", \"score\": \"8.509\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=146.262, mean=146.262, max=146.262, sum=146.262 (1)\", \"tab\": \"General information\", \"score\": \"146.262\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=8.99, mean=8.99, max=8.99, sum=8.99 (1)\", \"tab\": \"General information\", \"score\": \"8.99\"}" - } - }, - "generation_config": { - "additional_details": { - "mode": "\"closedbook\"" - } - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.922, - "details": { - "description": "min=0.922, mean=0.922, max=0.922, sum=0.922 (1)", - "tab": "Accuracy", - "OpenbookQA - Observed inference time (s)": "{\"description\": \"min=0.244, mean=0.244, max=0.244, sum=0.244 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.24445231294631958\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=249.846, mean=249.846, max=249.846, sum=249.846 (1)\", \"tab\": \"General information\", \"score\": \"249.846\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "dataset": "\"openbookqa\"", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.704, - "details": { - "description": "min=0.57, mean=0.704, max=0.87, sum=3.52 (5)", - "tab": "Accuracy", - "MMLU - Observed inference time (s)": "{\"description\": \"min=0.229, mean=0.248, max=0.277, sum=1.241 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.2482092388136345\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=378.19, mean=477.836, max=627.939, sum=2389.179 (5)\", \"tab\": \"General information\", \"score\": \"477.8357192982456\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"college_chemistry\", \"computer_security\", \"econometrics\", \"us_foreign_policy\"]", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MATH", - "source_data": { - "dataset_name": "MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Equivalent (CoT) on MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.568, - "details": { - "description": "min=0.211, mean=0.568, max=0.769, sum=3.974 (7)", - "tab": "Accuracy", - "MATH - Observed inference time (s)": "{\"description\": \"min=2.984, mean=3.989, max=5.0, sum=27.92 (7)\", \"tab\": \"Efficiency\", \"score\": \"3.9885726889236994\"}", - "MATH - # eval": "{\"description\": \"min=30, mean=62.429, max=135, sum=437 (7)\", \"tab\": \"General information\", \"score\": \"62.42857142857143\"}", - "MATH - # train": "{\"description\": \"min=8, mean=8, max=8, sum=56 (7)\", \"tab\": \"General information\", \"score\": \"8.0\"}", - "MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MATH - # prompt tokens": "{\"description\": \"min=937.926, mean=1323.837, max=2246.673, sum=9266.858 (7)\", \"tab\": \"General information\", \"score\": \"1323.836848955025\"}", - "MATH - # output tokens": "{\"description\": \"min=104.174, mean=156.855, max=202.368, sum=1097.984 (7)\", \"tab\": \"General information\", \"score\": \"156.85484968134907\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"algebra\", \"counting_and_probability\", \"geometry\", \"intermediate_algebra\", \"number_theory\", \"prealgebra\", \"precalculus\"]", - "level": "\"1\"", - "use_official_examples": "\"False\"", - "use_chain_of_thought": "\"True\"" - } - } - }, - { - "evaluation_name": "GSM8K", - "source_data": { - "dataset_name": "GSM8K", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on GSM8K", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.815, - "details": { - "description": "min=0.815, mean=0.815, max=0.815, sum=0.815 (1)", - "tab": "Accuracy", - "GSM8K - Observed inference time (s)": "{\"description\": \"min=4.537, mean=4.537, max=4.537, sum=4.537 (1)\", \"tab\": \"Efficiency\", \"score\": \"4.537143226146698\"}", - "GSM8K - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "GSM8K - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "GSM8K - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GSM8K - # prompt tokens": "{\"description\": \"min=1130.403, mean=1130.403, max=1130.403, sum=1130.403 (1)\", \"tab\": \"General information\", \"score\": \"1130.403\"}", - "GSM8K - # output tokens": "{\"description\": \"min=175.784, mean=175.784, max=175.784, sum=175.784 (1)\", \"tab\": \"General information\", \"score\": \"175.784\"}" - } - }, - "generation_config": { - "additional_details": { - "stop": "\"none\"" - } - } - }, - { - "evaluation_name": "LegalBench", - "source_data": { - "dataset_name": "LegalBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on LegalBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.624, - "details": { - "description": "min=0.387, mean=0.624, max=0.958, sum=3.121 (5)", - "tab": "Accuracy", - "LegalBench - Observed inference time (s)": "{\"description\": \"min=0.271, mean=0.499, max=1.328, sum=2.493 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.4986402694478536\"}", - "LegalBench - # eval": "{\"description\": \"min=95, mean=409.4, max=1000, sum=2047 (5)\", \"tab\": \"General information\", \"score\": \"409.4\"}", - "LegalBench - # train": "{\"description\": \"min=4, mean=4.8, max=5, sum=24 (5)\", \"tab\": \"General information\", \"score\": \"4.8\"}", - "LegalBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "LegalBench - # prompt tokens": "{\"description\": \"min=207.453, mean=1557.088, max=6445.714, sum=7785.442 (5)\", \"tab\": \"General information\", \"score\": \"1557.0883229968654\"}", - "LegalBench - # output tokens": "{\"description\": \"min=2, mean=2.314, max=2.958, sum=11.571 (5)\", \"tab\": \"General information\", \"score\": \"2.3142312634447153\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"abercrombie\", \"corporate_lobbying\", \"function_of_decision_section\", \"international_citizenship_questions\", \"proa\"]" - } - } - }, - { - "evaluation_name": "MedQA", - "source_data": { - "dataset_name": "MedQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MedQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.64, - "details": { - "description": "min=0.64, mean=0.64, max=0.64, sum=0.64 (1)", - "tab": "Accuracy", - "MedQA - Observed inference time (s)": "{\"description\": \"min=0.288, mean=0.288, max=0.288, sum=0.288 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.2881786700034473\"}", - "MedQA - # eval": "{\"description\": \"min=503, mean=503, max=503, sum=503 (1)\", \"tab\": \"General information\", \"score\": \"503.0\"}", - "MedQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MedQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MedQA - # prompt tokens": "{\"description\": \"min=1052.485, mean=1052.485, max=1052.485, sum=1052.485 (1)\", \"tab\": \"General information\", \"score\": \"1052.4850894632207\"}", - "MedQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WMT 2014", - "source_data": { - "dataset_name": "WMT 2014", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "BLEU-4 on WMT 2014", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.192, - "details": { - "description": "min=0.133, mean=0.192, max=0.232, sum=0.962 (5)", - "tab": "Accuracy", - "WMT 2014 - Observed inference time (s)": "{\"description\": \"min=0.839, mean=0.882, max=0.896, sum=4.411 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.882270189100544\"}", - "WMT 2014 - # eval": "{\"description\": \"min=503, mean=568.8, max=832, sum=2844 (5)\", \"tab\": \"General information\", \"score\": \"568.8\"}", - "WMT 2014 - # train": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "WMT 2014 - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WMT 2014 - # prompt tokens": "{\"description\": \"min=124.855, mean=142.657, max=158.373, sum=713.283 (5)\", \"tab\": \"General information\", \"score\": \"142.65662658663405\"}", - "WMT 2014 - # output tokens": "{\"description\": \"min=25.499, mean=26.949, max=27.529, sum=134.744 (5)\", \"tab\": \"General information\", \"score\": \"26.94872734745374\"}" - } - }, - "generation_config": { - "additional_details": { - "language_pair": "[\"cs-en\", \"de-en\", \"fr-en\", \"hi-en\", \"ru-en\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_mmlu/qwen_qwen1.5-110b-chat/1774096312.00548", - "retrieved_timestamp": "1774096312.00548", - "source_metadata": { - "source_name": "helm_mmlu", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_mmlu", - "evaluation_results": [ - { - "evaluation_name": "MMLU All Subjects", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU All Subjects", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.768, - "details": { - "description": "min=0.478, mean=0.768, max=0.984, sum=87.534 (114)", - "tab": "Accuracy", - "MMLU All Subjects - Observed inference time (s)": "{\"description\": \"min=0.229, mean=0.287, max=0.751, sum=32.77 (114)\", \"tab\": \"Efficiency\", \"score\": \"0.2874531237731517\"}", - "MMLU All Subjects - # eval": "{\"description\": \"min=100, mean=246.351, max=1534, sum=28084 (114)\", \"tab\": \"General information\", \"score\": \"246.35087719298247\"}", - "MMLU All Subjects - # train": "{\"description\": \"min=5, mean=5, max=5, sum=570 (114)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU All Subjects - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU All Subjects - # prompt tokens": "{\"description\": \"min=276.07, mean=625.598, max=2814.903, sum=71318.198 (114)\", \"tab\": \"General information\", \"score\": \"625.5982315160392\"}", - "MMLU All Subjects - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=114 (114)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_name": "Abstract Algebra", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Abstract Algebra", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.57, - "details": { - "description": "min=0.57, mean=0.57, max=0.57, sum=1.14 (2)", - "tab": "Accuracy", - "Abstract Algebra - Observed inference time (s)": "{\"description\": \"min=0.23, mean=0.23, max=0.23, sum=0.459 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.22966567754745484\"}", - "Abstract Algebra - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Abstract Algebra - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Abstract Algebra - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Abstract Algebra - # prompt tokens": "{\"description\": \"min=378.19, mean=378.19, max=378.19, sum=756.38 (2)\", \"tab\": \"General information\", \"score\": \"378.19\"}", - "Abstract Algebra - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"abstract_algebra\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_abstract_algebra\"" - } - } - }, - { - "evaluation_name": "Anatomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Anatomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.696, - "details": { - "description": "min=0.696, mean=0.696, max=0.696, sum=1.393 (2)", - "tab": "Accuracy", - "Anatomy - Observed inference time (s)": "{\"description\": \"min=0.26, mean=0.26, max=0.26, sum=0.52 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2600334096837927\"}", - "Anatomy - # eval": "{\"description\": \"min=135, mean=135, max=135, sum=270 (2)\", \"tab\": \"General information\", \"score\": \"135.0\"}", - "Anatomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Anatomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Anatomy - # prompt tokens": "{\"description\": \"min=353.978, mean=353.978, max=353.978, sum=707.956 (2)\", \"tab\": \"General information\", \"score\": \"353.97777777777776\"}", - "Anatomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"anatomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_anatomy\"" - } - } - }, - { - "evaluation_name": "College Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on College Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.51, - "details": { - "description": "min=0.51, mean=0.51, max=0.51, sum=1.02 (2)", - "tab": "Accuracy", - "College Chemistry - Observed inference time (s)": "{\"description\": \"min=0.257, mean=0.257, max=0.257, sum=0.513 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2566096520423889\"}", - "College Biology - Observed inference time (s)": "{\"description\": \"min=0.296, mean=0.296, max=0.296, sum=0.592 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2957576380835639\"}", - "College Computer Science - Observed inference time (s)": "{\"description\": \"min=0.326, mean=0.326, max=0.326, sum=0.652 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3260823440551758\"}", - "College Mathematics - Observed inference time (s)": "{\"description\": \"min=0.299, mean=0.299, max=0.299, sum=0.598 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2992465353012085\"}", - "College Medicine - Observed inference time (s)": "{\"description\": \"min=0.269, mean=0.269, max=0.269, sum=0.538 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2690960313543419\"}", - "College Physics - Observed inference time (s)": "{\"description\": \"min=0.281, mean=0.281, max=0.281, sum=0.562 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.28119626699709427\"}", - "College Chemistry - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Chemistry - # prompt tokens": "{\"description\": \"min=568.25, mean=568.25, max=568.25, sum=1136.5 (2)\", \"tab\": \"General information\", \"score\": \"568.25\"}", - "College Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Biology - # eval": "{\"description\": \"min=144, mean=144, max=144, sum=288 (2)\", \"tab\": \"General information\", \"score\": \"144.0\"}", - "College Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # prompt tokens": "{\"description\": \"min=486.979, mean=486.979, max=486.979, sum=973.958 (2)\", \"tab\": \"General information\", \"score\": \"486.9791666666667\"}", - "College Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # prompt tokens": "{\"description\": \"min=838.58, mean=838.58, max=838.58, sum=1677.16 (2)\", \"tab\": \"General information\", \"score\": \"838.58\"}", - "College Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Mathematics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # prompt tokens": "{\"description\": \"min=607.7, mean=607.7, max=607.7, sum=1215.4 (2)\", \"tab\": \"General information\", \"score\": \"607.7\"}", - "College Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Medicine - # eval": "{\"description\": \"min=173, mean=173, max=173, sum=346 (2)\", \"tab\": \"General information\", \"score\": \"173.0\"}", - "College Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # prompt tokens": "{\"description\": \"min=506.098, mean=506.098, max=506.098, sum=1012.197 (2)\", \"tab\": \"General information\", \"score\": \"506.0982658959538\"}", - "College Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Physics - # eval": "{\"description\": \"min=102, mean=102, max=102, sum=204 (2)\", \"tab\": \"General information\", \"score\": \"102.0\"}", - "College Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # prompt tokens": "{\"description\": \"min=516.265, mean=516.265, max=516.265, sum=1032.529 (2)\", \"tab\": \"General information\", \"score\": \"516.2647058823529\"}", - "College Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"college_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_college_physics\"" - } - } - }, - { - "evaluation_name": "Computer Security", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Computer Security", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.82, - "details": { - "description": "min=0.82, mean=0.82, max=0.82, sum=1.64 (2)", - "tab": "Accuracy", - "Computer Security - Observed inference time (s)": "{\"description\": \"min=0.277, mean=0.277, max=0.277, sum=0.555 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2773160576820374\"}", - "Computer Security - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Computer Security - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Computer Security - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Computer Security - # prompt tokens": "{\"description\": \"min=386.64, mean=386.64, max=386.64, sum=773.28 (2)\", \"tab\": \"General information\", \"score\": \"386.64\"}", - "Computer Security - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"computer_security\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_computer_security\"" - } - } - }, - { - "evaluation_name": "Econometrics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Econometrics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.64, - "details": { - "description": "min=0.64, mean=0.64, max=0.64, sum=1.281 (2)", - "tab": "Accuracy", - "Econometrics - Observed inference time (s)": "{\"description\": \"min=0.248, mean=0.248, max=0.248, sum=0.496 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.24817464017031485\"}", - "Econometrics - # eval": "{\"description\": \"min=114, mean=114, max=114, sum=228 (2)\", \"tab\": \"General information\", \"score\": \"114.0\"}", - "Econometrics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Econometrics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Econometrics - # prompt tokens": "{\"description\": \"min=627.939, mean=627.939, max=627.939, sum=1255.877 (2)\", \"tab\": \"General information\", \"score\": \"627.938596491228\"}", - "Econometrics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"econometrics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_econometrics\"" - } - } - }, - { - "evaluation_name": "Global Facts", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Global Facts", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.51, - "details": { - "description": "min=0.51, mean=0.51, max=0.51, sum=1.02 (2)", - "tab": "Accuracy", - "Global Facts - Observed inference time (s)": "{\"description\": \"min=0.257, mean=0.257, max=0.257, sum=0.514 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.25695453643798827\"}", - "Global Facts - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Global Facts - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Global Facts - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Global Facts - # prompt tokens": "{\"description\": \"min=429.06, mean=429.06, max=429.06, sum=858.12 (2)\", \"tab\": \"General information\", \"score\": \"429.06\"}", - "Global Facts - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"global_facts\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_global_facts\"" - } - } - }, - { - "evaluation_name": "Jurisprudence", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Jurisprudence", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.833, - "details": { - "description": "min=0.833, mean=0.833, max=0.833, sum=1.667 (2)", - "tab": "Accuracy", - "Jurisprudence - Observed inference time (s)": "{\"description\": \"min=0.256, mean=0.256, max=0.256, sum=0.512 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.25610714267801354\"}", - "Jurisprudence - # eval": "{\"description\": \"min=108, mean=108, max=108, sum=216 (2)\", \"tab\": \"General information\", \"score\": \"108.0\"}", - "Jurisprudence - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Jurisprudence - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Jurisprudence - # prompt tokens": "{\"description\": \"min=394.713, mean=394.713, max=394.713, sum=789.426 (2)\", \"tab\": \"General information\", \"score\": \"394.712962962963\"}", - "Jurisprudence - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"jurisprudence\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_jurisprudence\"" - } - } - }, - { - "evaluation_name": "Philosophy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Philosophy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.823, - "details": { - "description": "min=0.823, mean=0.823, max=0.823, sum=1.646 (2)", - "tab": "Accuracy", - "Philosophy - Observed inference time (s)": "{\"description\": \"min=0.233, mean=0.233, max=0.233, sum=0.465 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2326939565959084\"}", - "Philosophy - # eval": "{\"description\": \"min=311, mean=311, max=311, sum=622 (2)\", \"tab\": \"General information\", \"score\": \"311.0\"}", - "Philosophy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Philosophy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Philosophy - # prompt tokens": "{\"description\": \"min=329.09, mean=329.09, max=329.09, sum=658.18 (2)\", \"tab\": \"General information\", \"score\": \"329.09003215434086\"}", - "Philosophy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"philosophy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_philosophy\"" - } - } - }, - { - "evaluation_name": "Professional Psychology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Professional Psychology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.82, - "details": { - "description": "min=0.82, mean=0.82, max=0.82, sum=1.641 (2)", - "tab": "Accuracy", - "Professional Medicine - Observed inference time (s)": "{\"description\": \"min=0.396, mean=0.396, max=0.396, sum=0.792 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.39590225675526786\"}", - "Professional Accounting - Observed inference time (s)": "{\"description\": \"min=0.243, mean=0.243, max=0.243, sum=0.486 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.24316950554543354\"}", - "Professional Law - Observed inference time (s)": "{\"description\": \"min=0.319, mean=0.319, max=0.319, sum=0.638 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.31920133731200456\"}", - "Professional Psychology - Observed inference time (s)": "{\"description\": \"min=0.245, mean=0.245, max=0.245, sum=0.491 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2452772462290097\"}", - "Professional Medicine - # eval": "{\"description\": \"min=272, mean=272, max=272, sum=544 (2)\", \"tab\": \"General information\", \"score\": \"272.0\"}", - "Professional Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Medicine - # prompt tokens": "{\"description\": \"min=1125.199, mean=1125.199, max=1125.199, sum=2250.397 (2)\", \"tab\": \"General information\", \"score\": \"1125.1985294117646\"}", - "Professional Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Accounting - # eval": "{\"description\": \"min=282, mean=282, max=282, sum=564 (2)\", \"tab\": \"General information\", \"score\": \"282.0\"}", - "Professional Accounting - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Accounting - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # prompt tokens": "{\"description\": \"min=739.34, mean=739.34, max=739.34, sum=1478.681 (2)\", \"tab\": \"General information\", \"score\": \"739.3404255319149\"}", - "Professional Accounting - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Law - # eval": "{\"description\": \"min=1534, mean=1534, max=1534, sum=3068 (2)\", \"tab\": \"General information\", \"score\": \"1534.0\"}", - "Professional Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # prompt tokens": "{\"description\": \"min=1663.969, mean=1663.969, max=1663.969, sum=3327.939 (2)\", \"tab\": \"General information\", \"score\": \"1663.9693611473272\"}", - "Professional Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Psychology - # eval": "{\"description\": \"min=612, mean=612, max=612, sum=1224 (2)\", \"tab\": \"General information\", \"score\": \"612.0\"}", - "Professional Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # prompt tokens": "{\"description\": \"min=581.417, mean=581.417, max=581.417, sum=1162.833 (2)\", \"tab\": \"General information\", \"score\": \"581.4166666666666\"}", - "Professional Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"professional_psychology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_professional_psychology\"" - } - } - }, - { - "evaluation_name": "Us Foreign Policy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Us Foreign Policy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.87, - "details": { - "description": "min=0.87, mean=0.87, max=0.87, sum=1.74 (2)", - "tab": "Accuracy", - "Us Foreign Policy - Observed inference time (s)": "{\"description\": \"min=0.229, mean=0.229, max=0.229, sum=0.459 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.22928016662597656\"}", - "Us Foreign Policy - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Us Foreign Policy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Us Foreign Policy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Us Foreign Policy - # prompt tokens": "{\"description\": \"min=428.16, mean=428.16, max=428.16, sum=856.32 (2)\", \"tab\": \"General information\", \"score\": \"428.16\"}", - "Us Foreign Policy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"us_foreign_policy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_us_foreign_policy\"" - } - } - }, - { - "evaluation_name": "Astronomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Astronomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.901, - "details": { - "description": "min=0.901, mean=0.901, max=0.901, sum=1.803 (2)", - "tab": "Accuracy", - "Astronomy - Observed inference time (s)": "{\"description\": \"min=0.306, mean=0.306, max=0.306, sum=0.612 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3059707331029992\"}", - "Astronomy - # eval": "{\"description\": \"min=152, mean=152, max=152, sum=304 (2)\", \"tab\": \"General information\", \"score\": \"152.0\"}", - "Astronomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Astronomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Astronomy - # prompt tokens": "{\"description\": \"min=589.849, mean=589.849, max=589.849, sum=1179.697 (2)\", \"tab\": \"General information\", \"score\": \"589.8486842105264\"}", - "Astronomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"astronomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_astronomy\"" - } - } - }, - { - "evaluation_name": "Business Ethics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Business Ethics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8, - "details": { - "description": "min=0.8, mean=0.8, max=0.8, sum=1.6 (2)", - "tab": "Accuracy", - "Business Ethics - Observed inference time (s)": "{\"description\": \"min=0.311, mean=0.311, max=0.311, sum=0.622 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.31108115911483764\"}", - "Business Ethics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Business Ethics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Business Ethics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Business Ethics - # prompt tokens": "{\"description\": \"min=569.87, mean=569.87, max=569.87, sum=1139.74 (2)\", \"tab\": \"General information\", \"score\": \"569.87\"}", - "Business Ethics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"business_ethics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_business_ethics\"" - } - } - }, - { - "evaluation_name": "Clinical Knowledge", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Clinical Knowledge", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.766, - "details": { - "description": "min=0.766, mean=0.766, max=0.766, sum=1.532 (2)", - "tab": "Accuracy", - "Clinical Knowledge - Observed inference time (s)": "{\"description\": \"min=0.268, mean=0.268, max=0.268, sum=0.536 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.26778328283777775\"}", - "Clinical Knowledge - # eval": "{\"description\": \"min=265, mean=265, max=265, sum=530 (2)\", \"tab\": \"General information\", \"score\": \"265.0\"}", - "Clinical Knowledge - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Clinical Knowledge - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Clinical Knowledge - # prompt tokens": "{\"description\": \"min=400.623, mean=400.623, max=400.623, sum=801.245 (2)\", \"tab\": \"General information\", \"score\": \"400.62264150943395\"}", - "Clinical Knowledge - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"clinical_knowledge\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_clinical_knowledge\"" - } - } - }, - { - "evaluation_name": "Conceptual Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Conceptual Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.838, - "details": { - "description": "min=0.838, mean=0.838, max=0.838, sum=1.677 (2)", - "tab": "Accuracy", - "Conceptual Physics - Observed inference time (s)": "{\"description\": \"min=0.267, mean=0.267, max=0.267, sum=0.533 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.26653050361795627\"}", - "Conceptual Physics - # eval": "{\"description\": \"min=235, mean=235, max=235, sum=470 (2)\", \"tab\": \"General information\", \"score\": \"235.0\"}", - "Conceptual Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Conceptual Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Conceptual Physics - # prompt tokens": "{\"description\": \"min=305.494, mean=305.494, max=305.494, sum=610.987 (2)\", \"tab\": \"General information\", \"score\": \"305.4936170212766\"}", - "Conceptual Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"conceptual_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_conceptual_physics\"" - } - } - }, - { - "evaluation_name": "Electrical Engineering", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Electrical Engineering", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.752, - "details": { - "description": "min=0.752, mean=0.752, max=0.752, sum=1.503 (2)", - "tab": "Accuracy", - "Electrical Engineering - Observed inference time (s)": "{\"description\": \"min=0.24, mean=0.24, max=0.24, sum=0.481 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.24032716751098632\"}", - "Electrical Engineering - # eval": "{\"description\": \"min=145, mean=145, max=145, sum=290 (2)\", \"tab\": \"General information\", \"score\": \"145.0\"}", - "Electrical Engineering - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Electrical Engineering - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Electrical Engineering - # prompt tokens": "{\"description\": \"min=463.8, mean=463.8, max=463.8, sum=927.6 (2)\", \"tab\": \"General information\", \"score\": \"463.8\"}", - "Electrical Engineering - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"electrical_engineering\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_electrical_engineering\"" - } - } - }, - { - "evaluation_name": "Elementary Mathematics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Elementary Mathematics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.669, - "details": { - "description": "min=0.669, mean=0.669, max=0.669, sum=1.339 (2)", - "tab": "Accuracy", - "Elementary Mathematics - Observed inference time (s)": "{\"description\": \"min=0.286, mean=0.286, max=0.286, sum=0.571 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.28569977939444247\"}", - "Elementary Mathematics - # eval": "{\"description\": \"min=378, mean=378, max=378, sum=756 (2)\", \"tab\": \"General information\", \"score\": \"378.0\"}", - "Elementary Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Elementary Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Elementary Mathematics - # prompt tokens": "{\"description\": \"min=577.119, mean=577.119, max=577.119, sum=1154.238 (2)\", \"tab\": \"General information\", \"score\": \"577.1190476190476\"}", - "Elementary Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"elementary_mathematics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_elementary_mathematics\"" - } - } - }, - { - "evaluation_name": "Formal Logic", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Formal Logic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.643, - "details": { - "description": "min=0.643, mean=0.643, max=0.643, sum=1.286 (2)", - "tab": "Accuracy", - "Formal Logic - Observed inference time (s)": "{\"description\": \"min=0.284, mean=0.284, max=0.284, sum=0.567 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2836597722674173\"}", - "Formal Logic - # eval": "{\"description\": \"min=126, mean=126, max=126, sum=252 (2)\", \"tab\": \"General information\", \"score\": \"126.0\"}", - "Formal Logic - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Formal Logic - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Formal Logic - # prompt tokens": "{\"description\": \"min=604.667, mean=604.667, max=604.667, sum=1209.333 (2)\", \"tab\": \"General information\", \"score\": \"604.6666666666666\"}", - "Formal Logic - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"formal_logic\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_formal_logic\"" - } - } - }, - { - "evaluation_name": "High School World History", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on High School World History", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.903, - "details": { - "description": "min=0.903, mean=0.903, max=0.903, sum=1.806 (2)", - "tab": "Accuracy", - "High School Biology - Observed inference time (s)": "{\"description\": \"min=0.283, mean=0.283, max=0.283, sum=0.566 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2828109118246263\"}", - "High School Chemistry - Observed inference time (s)": "{\"description\": \"min=0.293, mean=0.293, max=0.293, sum=0.586 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.29298263935032737\"}", - "High School Computer Science - Observed inference time (s)": "{\"description\": \"min=0.307, mean=0.307, max=0.307, sum=0.615 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.30738641500473024\"}", - "High School European History - Observed inference time (s)": "{\"description\": \"min=0.593, mean=0.593, max=0.593, sum=1.186 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5927927941987009\"}", - "High School Geography - Observed inference time (s)": "{\"description\": \"min=0.277, mean=0.277, max=0.277, sum=0.553 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2765737639533149\"}", - "High School Government And Politics - Observed inference time (s)": "{\"description\": \"min=0.253, mean=0.253, max=0.253, sum=0.505 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2526841929539498\"}", - "High School Macroeconomics - Observed inference time (s)": "{\"description\": \"min=0.264, mean=0.264, max=0.264, sum=0.527 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2636140242601052\"}", - "High School Mathematics - Observed inference time (s)": "{\"description\": \"min=0.289, mean=0.289, max=0.289, sum=0.578 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.28875163837715434\"}", - "High School Microeconomics - Observed inference time (s)": "{\"description\": \"min=0.254, mean=0.254, max=0.254, sum=0.508 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2539960216073429\"}", - "High School Physics - Observed inference time (s)": "{\"description\": \"min=0.281, mean=0.281, max=0.281, sum=0.562 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.28084811943256305\"}", - "High School Psychology - Observed inference time (s)": "{\"description\": \"min=0.244, mean=0.244, max=0.244, sum=0.489 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.24437280532416947\"}", - "High School Statistics - Observed inference time (s)": "{\"description\": \"min=0.34, mean=0.34, max=0.34, sum=0.679 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3396394296928688\"}", - "High School US History - Observed inference time (s)": "{\"description\": \"min=0.416, mean=0.416, max=0.416, sum=0.832 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4159782189948886\"}", - "High School World History - Observed inference time (s)": "{\"description\": \"min=0.751, mean=0.751, max=0.751, sum=1.501 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7505324741959069\"}", - "High School Biology - # eval": "{\"description\": \"min=310, mean=310, max=310, sum=620 (2)\", \"tab\": \"General information\", \"score\": \"310.0\"}", - "High School Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Biology - # prompt tokens": "{\"description\": \"min=513.916, mean=513.916, max=513.916, sum=1027.832 (2)\", \"tab\": \"General information\", \"score\": \"513.916129032258\"}", - "High School Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Chemistry - # eval": "{\"description\": \"min=203, mean=203, max=203, sum=406 (2)\", \"tab\": \"General information\", \"score\": \"203.0\"}", - "High School Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # prompt tokens": "{\"description\": \"min=517.261, mean=517.261, max=517.261, sum=1034.522 (2)\", \"tab\": \"General information\", \"score\": \"517.2610837438424\"}", - "High School Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "High School Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # prompt tokens": "{\"description\": \"min=878.46, mean=878.46, max=878.46, sum=1756.92 (2)\", \"tab\": \"General information\", \"score\": \"878.46\"}", - "High School Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School European History - # eval": "{\"description\": \"min=165, mean=165, max=165, sum=330 (2)\", \"tab\": \"General information\", \"score\": \"165.0\"}", - "High School European History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School European History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # prompt tokens": "{\"description\": \"min=2814.903, mean=2814.903, max=2814.903, sum=5629.806 (2)\", \"tab\": \"General information\", \"score\": \"2814.9030303030304\"}", - "High School European History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Geography - # eval": "{\"description\": \"min=198, mean=198, max=198, sum=396 (2)\", \"tab\": \"General information\", \"score\": \"198.0\"}", - "High School Geography - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Geography - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # prompt tokens": "{\"description\": \"min=372.217, mean=372.217, max=372.217, sum=744.434 (2)\", \"tab\": \"General information\", \"score\": \"372.2171717171717\"}", - "High School Geography - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Government And Politics - # eval": "{\"description\": \"min=193, mean=193, max=193, sum=386 (2)\", \"tab\": \"General information\", \"score\": \"193.0\"}", - "High School Government And Politics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Government And Politics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # prompt tokens": "{\"description\": \"min=467.311, mean=467.311, max=467.311, sum=934.622 (2)\", \"tab\": \"General information\", \"score\": \"467.31088082901556\"}", - "High School Government And Politics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Macroeconomics - # eval": "{\"description\": \"min=390, mean=390, max=390, sum=780 (2)\", \"tab\": \"General information\", \"score\": \"390.0\"}", - "High School Macroeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Macroeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # prompt tokens": "{\"description\": \"min=374.349, mean=374.349, max=374.349, sum=748.697 (2)\", \"tab\": \"General information\", \"score\": \"374.34871794871793\"}", - "High School Macroeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Mathematics - # eval": "{\"description\": \"min=270, mean=270, max=270, sum=540 (2)\", \"tab\": \"General information\", \"score\": \"270.0\"}", - "High School Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # prompt tokens": "{\"description\": \"min=565.326, mean=565.326, max=565.326, sum=1130.652 (2)\", \"tab\": \"General information\", \"score\": \"565.325925925926\"}", - "High School Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Microeconomics - # eval": "{\"description\": \"min=238, mean=238, max=238, sum=476 (2)\", \"tab\": \"General information\", \"score\": \"238.0\"}", - "High School Microeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Microeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # prompt tokens": "{\"description\": \"min=402.277, mean=402.277, max=402.277, sum=804.555 (2)\", \"tab\": \"General information\", \"score\": \"402.2773109243698\"}", - "High School Microeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Physics - # eval": "{\"description\": \"min=151, mean=151, max=151, sum=302 (2)\", \"tab\": \"General information\", \"score\": \"151.0\"}", - "High School Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # prompt tokens": "{\"description\": \"min=580.536, mean=580.536, max=580.536, sum=1161.073 (2)\", \"tab\": \"General information\", \"score\": \"580.5364238410596\"}", - "High School Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Psychology - # eval": "{\"description\": \"min=545, mean=545, max=545, sum=1090 (2)\", \"tab\": \"General information\", \"score\": \"545.0\"}", - "High School Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # prompt tokens": "{\"description\": \"min=495.521, mean=495.521, max=495.521, sum=991.042 (2)\", \"tab\": \"General information\", \"score\": \"495.52110091743117\"}", - "High School Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Statistics - # eval": "{\"description\": \"min=216, mean=216, max=216, sum=432 (2)\", \"tab\": \"General information\", \"score\": \"216.0\"}", - "High School Statistics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Statistics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # prompt tokens": "{\"description\": \"min=830.477, mean=830.477, max=830.477, sum=1660.954 (2)\", \"tab\": \"General information\", \"score\": \"830.4768518518518\"}", - "High School Statistics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School US History - # eval": "{\"description\": \"min=204, mean=204, max=204, sum=408 (2)\", \"tab\": \"General information\", \"score\": \"204.0\"}", - "High School US History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School US History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # prompt tokens": "{\"description\": \"min=2237.176, mean=2237.176, max=2237.176, sum=4474.353 (2)\", \"tab\": \"General information\", \"score\": \"2237.176470588235\"}", - "High School US History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School World History - # eval": "{\"description\": \"min=237, mean=237, max=237, sum=474 (2)\", \"tab\": \"General information\", \"score\": \"237.0\"}", - "High School World History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School World History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # prompt tokens": "{\"description\": \"min=1448.354, mean=1448.354, max=1448.354, sum=2896.709 (2)\", \"tab\": \"General information\", \"score\": \"1448.3544303797469\"}", - "High School World History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"high_school_world_history\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_high_school_world_history\"" - } - } - }, - { - "evaluation_name": "Human Sexuality", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Human Sexuality", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.855, - "details": { - "description": "min=0.855, mean=0.855, max=0.855, sum=1.71 (2)", - "tab": "Accuracy", - "Human Aging - Observed inference time (s)": "{\"description\": \"min=0.245, mean=0.245, max=0.245, sum=0.49 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.24486422538757324\"}", - "Human Sexuality - Observed inference time (s)": "{\"description\": \"min=0.254, mean=0.254, max=0.254, sum=0.508 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.25416288121056013\"}", - "Human Aging - # eval": "{\"description\": \"min=223, mean=223, max=223, sum=446 (2)\", \"tab\": \"General information\", \"score\": \"223.0\"}", - "Human Aging - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Aging - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Aging - # prompt tokens": "{\"description\": \"min=322.121, mean=322.121, max=322.121, sum=644.242 (2)\", \"tab\": \"General information\", \"score\": \"322.1210762331838\"}", - "Human Aging - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Human Sexuality - # eval": "{\"description\": \"min=131, mean=131, max=131, sum=262 (2)\", \"tab\": \"General information\", \"score\": \"131.0\"}", - "Human Sexuality - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Sexuality - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # prompt tokens": "{\"description\": \"min=341.504, mean=341.504, max=341.504, sum=683.008 (2)\", \"tab\": \"General information\", \"score\": \"341.5038167938931\"}", - "Human Sexuality - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"human_sexuality\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_human_sexuality\"" - } - } - }, - { - "evaluation_name": "International Law", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on International Law", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.876, - "details": { - "description": "min=0.876, mean=0.876, max=0.876, sum=1.752 (2)", - "tab": "Accuracy", - "International Law - Observed inference time (s)": "{\"description\": \"min=0.277, mean=0.277, max=0.277, sum=0.555 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2773902613269396\"}", - "International Law - # eval": "{\"description\": \"min=121, mean=121, max=121, sum=242 (2)\", \"tab\": \"General information\", \"score\": \"121.0\"}", - "International Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "International Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "International Law - # prompt tokens": "{\"description\": \"min=640.579, mean=640.579, max=640.579, sum=1281.157 (2)\", \"tab\": \"General information\", \"score\": \"640.5785123966942\"}", - "International Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"international_law\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_international_law\"" - } - } - }, - { - "evaluation_name": "Logical Fallacies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Logical Fallacies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.828, - "details": { - "description": "min=0.828, mean=0.828, max=0.828, sum=1.656 (2)", - "tab": "Accuracy", - "Logical Fallacies - Observed inference time (s)": "{\"description\": \"min=0.248, mean=0.248, max=0.248, sum=0.496 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.24794307661934134\"}", - "Logical Fallacies - # eval": "{\"description\": \"min=163, mean=163, max=163, sum=326 (2)\", \"tab\": \"General information\", \"score\": \"163.0\"}", - "Logical Fallacies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Logical Fallacies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Logical Fallacies - # prompt tokens": "{\"description\": \"min=449.632, mean=449.632, max=449.632, sum=899.264 (2)\", \"tab\": \"General information\", \"score\": \"449.6319018404908\"}", - "Logical Fallacies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"logical_fallacies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_logical_fallacies\"" - } - } - }, - { - "evaluation_name": "Machine Learning", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Machine Learning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.634, - "details": { - "description": "min=0.634, mean=0.634, max=0.634, sum=1.268 (2)", - "tab": "Accuracy", - "Machine Learning - Observed inference time (s)": "{\"description\": \"min=0.284, mean=0.284, max=0.284, sum=0.567 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2835228868893215\"}", - "Machine Learning - # eval": "{\"description\": \"min=112, mean=112, max=112, sum=224 (2)\", \"tab\": \"General information\", \"score\": \"112.0\"}", - "Machine Learning - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Machine Learning - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Machine Learning - # prompt tokens": "{\"description\": \"min=681.848, mean=681.848, max=681.848, sum=1363.696 (2)\", \"tab\": \"General information\", \"score\": \"681.8482142857143\"}", - "Machine Learning - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"machine_learning\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_machine_learning\"" - } - } - }, - { - "evaluation_name": "Management", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Management", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.835, - "details": { - "description": "min=0.835, mean=0.835, max=0.835, sum=1.67 (2)", - "tab": "Accuracy", - "Management - Observed inference time (s)": "{\"description\": \"min=0.28, mean=0.28, max=0.28, sum=0.56 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.28018068804324253\"}", - "Management - # eval": "{\"description\": \"min=103, mean=103, max=103, sum=206 (2)\", \"tab\": \"General information\", \"score\": \"103.0\"}", - "Management - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Management - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Management - # prompt tokens": "{\"description\": \"min=283.854, mean=283.854, max=283.854, sum=567.709 (2)\", \"tab\": \"General information\", \"score\": \"283.8543689320388\"}", - "Management - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"management\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_management\"" - } - } - }, - { - "evaluation_name": "Marketing", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Marketing", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.919, - "details": { - "description": "min=0.919, mean=0.919, max=0.919, sum=1.838 (2)", - "tab": "Accuracy", - "Marketing - Observed inference time (s)": "{\"description\": \"min=0.254, mean=0.254, max=0.254, sum=0.509 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2544598365441347\"}", - "Marketing - # eval": "{\"description\": \"min=234, mean=234, max=234, sum=468 (2)\", \"tab\": \"General information\", \"score\": \"234.0\"}", - "Marketing - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Marketing - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Marketing - # prompt tokens": "{\"description\": \"min=404.415, mean=404.415, max=404.415, sum=808.829 (2)\", \"tab\": \"General information\", \"score\": \"404.4145299145299\"}", - "Marketing - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"marketing\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_marketing\"" - } - } - }, - { - "evaluation_name": "Medical Genetics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Medical Genetics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.85, - "details": { - "description": "min=0.85, mean=0.85, max=0.85, sum=1.7 (2)", - "tab": "Accuracy", - "Medical Genetics - Observed inference time (s)": "{\"description\": \"min=0.27, mean=0.27, max=0.27, sum=0.541 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.27034429311752317\"}", - "Medical Genetics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Medical Genetics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Medical Genetics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Medical Genetics - # prompt tokens": "{\"description\": \"min=342.35, mean=342.35, max=342.35, sum=684.7 (2)\", \"tab\": \"General information\", \"score\": \"342.35\"}", - "Medical Genetics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"medical_genetics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_medical_genetics\"" - } - } - }, - { - "evaluation_name": "Miscellaneous", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Miscellaneous", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.934, - "details": { - "description": "min=0.934, mean=0.934, max=0.934, sum=1.867 (2)", - "tab": "Accuracy", - "Miscellaneous - Observed inference time (s)": "{\"description\": \"min=0.246, mean=0.246, max=0.246, sum=0.492 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.24603491085242493\"}", - "Miscellaneous - # eval": "{\"description\": \"min=783, mean=783, max=783, sum=1566 (2)\", \"tab\": \"General information\", \"score\": \"783.0\"}", - "Miscellaneous - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Miscellaneous - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Miscellaneous - # prompt tokens": "{\"description\": \"min=303.7, mean=303.7, max=303.7, sum=607.4 (2)\", \"tab\": \"General information\", \"score\": \"303.6998722860792\"}", - "Miscellaneous - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"miscellaneous\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_miscellaneous\"" - } - } - }, - { - "evaluation_name": "Moral Scenarios", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Moral Scenarios", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.783, - "details": { - "description": "min=0.783, mean=0.783, max=0.783, sum=1.566 (2)", - "tab": "Accuracy", - "Moral Disputes - Observed inference time (s)": "{\"description\": \"min=0.256, mean=0.256, max=0.256, sum=0.513 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2563680651559995\"}", - "Moral Scenarios - Observed inference time (s)": "{\"description\": \"min=0.257, mean=0.257, max=0.257, sum=0.514 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.25722797329865354\"}", - "Moral Disputes - # eval": "{\"description\": \"min=346, mean=346, max=346, sum=692 (2)\", \"tab\": \"General information\", \"score\": \"346.0\"}", - "Moral Disputes - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Disputes - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Disputes - # prompt tokens": "{\"description\": \"min=476.182, mean=476.182, max=476.182, sum=952.364 (2)\", \"tab\": \"General information\", \"score\": \"476.1820809248555\"}", - "Moral Disputes - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Moral Scenarios - # eval": "{\"description\": \"min=895, mean=895, max=895, sum=1790 (2)\", \"tab\": \"General information\", \"score\": \"895.0\"}", - "Moral Scenarios - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Scenarios - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # prompt tokens": "{\"description\": \"min=668.494, mean=668.494, max=668.494, sum=1336.988 (2)\", \"tab\": \"General information\", \"score\": \"668.4938547486033\"}", - "Moral Scenarios - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"moral_scenarios\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_moral_scenarios\"" - } - } - }, - { - "evaluation_name": "Nutrition", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Nutrition", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.804, - "details": { - "description": "min=0.804, mean=0.804, max=0.804, sum=1.608 (2)", - "tab": "Accuracy", - "Nutrition - Observed inference time (s)": "{\"description\": \"min=0.271, mean=0.271, max=0.271, sum=0.542 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.27095749721028445\"}", - "Nutrition - # eval": "{\"description\": \"min=306, mean=306, max=306, sum=612 (2)\", \"tab\": \"General information\", \"score\": \"306.0\"}", - "Nutrition - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Nutrition - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Nutrition - # prompt tokens": "{\"description\": \"min=599.637, mean=599.637, max=599.637, sum=1199.275 (2)\", \"tab\": \"General information\", \"score\": \"599.6372549019608\"}", - "Nutrition - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"nutrition\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_nutrition\"" - } - } - }, - { - "evaluation_name": "Prehistory", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Prehistory", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.867, - "details": { - "description": "min=0.867, mean=0.867, max=0.867, sum=1.735 (2)", - "tab": "Accuracy", - "Prehistory - Observed inference time (s)": "{\"description\": \"min=0.242, mean=0.242, max=0.242, sum=0.483 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2415844319779196\"}", - "Prehistory - # eval": "{\"description\": \"min=324, mean=324, max=324, sum=648 (2)\", \"tab\": \"General information\", \"score\": \"324.0\"}", - "Prehistory - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Prehistory - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Prehistory - # prompt tokens": "{\"description\": \"min=528.364, mean=528.364, max=528.364, sum=1056.728 (2)\", \"tab\": \"General information\", \"score\": \"528.3641975308642\"}", - "Prehistory - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"prehistory\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_prehistory\"" - } - } - }, - { - "evaluation_name": "Public Relations", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Public Relations", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.773, - "details": { - "description": "min=0.773, mean=0.773, max=0.773, sum=1.545 (2)", - "tab": "Accuracy", - "Public Relations - Observed inference time (s)": "{\"description\": \"min=0.25, mean=0.25, max=0.25, sum=0.5 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2501691276376898\"}", - "Public Relations - # eval": "{\"description\": \"min=110, mean=110, max=110, sum=220 (2)\", \"tab\": \"General information\", \"score\": \"110.0\"}", - "Public Relations - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Public Relations - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Public Relations - # prompt tokens": "{\"description\": \"min=408.427, mean=408.427, max=408.427, sum=816.855 (2)\", \"tab\": \"General information\", \"score\": \"408.42727272727274\"}", - "Public Relations - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"public_relations\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_public_relations\"" - } - } - }, - { - "evaluation_name": "Security Studies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Security Studies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.735, - "details": { - "description": "min=0.735, mean=0.735, max=0.735, sum=1.469 (2)", - "tab": "Accuracy", - "Security Studies - Observed inference time (s)": "{\"description\": \"min=0.283, mean=0.283, max=0.283, sum=0.565 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.28266452769843897\"}", - "Security Studies - # eval": "{\"description\": \"min=245, mean=245, max=245, sum=490 (2)\", \"tab\": \"General information\", \"score\": \"245.0\"}", - "Security Studies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Security Studies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Security Studies - # prompt tokens": "{\"description\": \"min=1166.931, mean=1166.931, max=1166.931, sum=2333.861 (2)\", \"tab\": \"General information\", \"score\": \"1166.930612244898\"}", - "Security Studies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"security_studies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_security_studies\"" - } - } - }, - { - "evaluation_name": "Sociology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Sociology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.866, - "details": { - "description": "min=0.866, mean=0.866, max=0.866, sum=1.731 (2)", - "tab": "Accuracy", - "Sociology - Observed inference time (s)": "{\"description\": \"min=0.258, mean=0.258, max=0.258, sum=0.516 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.258230237818476\"}", - "Sociology - # eval": "{\"description\": \"min=201, mean=201, max=201, sum=402 (2)\", \"tab\": \"General information\", \"score\": \"201.0\"}", - "Sociology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Sociology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Sociology - # prompt tokens": "{\"description\": \"min=450.1, mean=450.1, max=450.1, sum=900.199 (2)\", \"tab\": \"General information\", \"score\": \"450.0995024875622\"}", - "Sociology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"sociology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_sociology\"" - } - } - }, - { - "evaluation_name": "Virology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Virology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.542, - "details": { - "description": "min=0.542, mean=0.542, max=0.542, sum=1.084 (2)", - "tab": "Accuracy", - "Virology - Observed inference time (s)": "{\"description\": \"min=0.248, mean=0.248, max=0.248, sum=0.495 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.24754508719386825\"}", - "Virology - # eval": "{\"description\": \"min=166, mean=166, max=166, sum=332 (2)\", \"tab\": \"General information\", \"score\": \"166.0\"}", - "Virology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Virology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Virology - # prompt tokens": "{\"description\": \"min=343.819, mean=343.819, max=343.819, sum=687.639 (2)\", \"tab\": \"General information\", \"score\": \"343.8192771084337\"}", - "Virology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"virology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_virology\"" - } - } - }, - { - "evaluation_name": "World Religions", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on World Religions", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.871, - "details": { - "description": "min=0.871, mean=0.871, max=0.871, sum=1.743 (2)", - "tab": "Accuracy", - "World Religions - Observed inference time (s)": "{\"description\": \"min=0.235, mean=0.235, max=0.235, sum=0.471 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.23539779897321733\"}", - "World Religions - # eval": "{\"description\": \"min=171, mean=171, max=171, sum=342 (2)\", \"tab\": \"General information\", \"score\": \"171.0\"}", - "World Religions - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "World Religions - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "World Religions - # prompt tokens": "{\"description\": \"min=276.07, mean=276.07, max=276.07, sum=552.14 (2)\", \"tab\": \"General information\", \"score\": \"276.0701754385965\"}", - "World Religions - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"world_religions\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_world_religions\"" - } - } - }, - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.875, - "details": { - "description": "", - "tab": "Efficiency" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_id": "hfopenllm_v2/Qwen_Qwen1.5-110B-Chat/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5939 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6184 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2341 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3414 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4522 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4825 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qwen_qwen1.5-110b.json b/data/models/qwen_qwen1.5-110b.json deleted file mode 100644 index 0d0453ac52f5a4f79e718f26f285060cebe32d38..0000000000000000000000000000000000000000 --- a/data/models/qwen_qwen1.5-110b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen1.5-110B", - "id": "Qwen/Qwen1.5-110B", - "developer": "Qwen", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "111.21" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Qwen_Qwen1.5-110B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3422 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.61 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.247 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3523 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4408 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5361 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qwen_qwen1.5-14b-chat.json b/data/models/qwen_qwen1.5-14b-chat.json deleted file mode 100644 index d400eb280eed319d6f01776240404dab0c1eb43d..0000000000000000000000000000000000000000 --- a/data/models/qwen_qwen1.5-14b-chat.json +++ /dev/null @@ -1,277 +0,0 @@ -{ - "model_info": { - "name": "Qwen/Qwen1.5-14B-Chat", - "id": "Qwen/Qwen1.5-14B-Chat", - "developer": "Qwen", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.167" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Qwen_Qwen1.5-14B-Chat/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4768 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5229 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1526 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2701 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.44 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3618 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "reward-bench/Qwen_Qwen1.5-14B-Chat/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6864 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5726 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7018 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7122 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8961 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4123 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qwen_qwen1.5-14b.json b/data/models/qwen_qwen1.5-14b.json deleted file mode 100644 index 6661a0c445548a9a900473a3af416215d7ffd01c..0000000000000000000000000000000000000000 --- a/data/models/qwen_qwen1.5-14b.json +++ /dev/null @@ -1,2036 +0,0 @@ -{ - "model_info": { - "name": "Qwen1.5 14B", - "id": "qwen/qwen1.5-14b", - "developer": "qwen", - "inference_platform": "unknown", - "additional_details": { - "model_id_aliases": [ - "Qwen/Qwen1.5-14B" - ] - } - }, - "evaluations": [ - { - "evaluation_id": "helm_lite/qwen_qwen1.5-14b/1774096306.427425", - "retrieved_timestamp": "1774096306.427425", - "source_metadata": { - "source_name": "helm_lite", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_lite", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_lite", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.425, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.6941198501872659\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.711, - "details": { - "description": "min=0.711, mean=0.711, max=0.711, sum=0.711 (1)", - "tab": "Accuracy", - "NarrativeQA - Observed inference time (s)": "{\"description\": \"min=0.987, mean=0.987, max=0.987, sum=0.987 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.986717187183004\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=3468.913, mean=3468.913, max=3468.913, sum=3468.913 (1)\", \"tab\": \"General information\", \"score\": \"3468.912676056338\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (closed-book)", - "source_data": { - "dataset_name": "NaturalQuestions (closed-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (closed-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3, - "details": { - "description": "min=0.3, mean=0.3, max=0.3, sum=0.3 (1)", - "tab": "Accuracy", - "NaturalQuestions (open-book) - Observed inference time (s)": "{\"description\": \"min=0.679, mean=0.679, max=0.679, sum=0.679 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.6790921592712402\"}", - "NaturalQuestions (closed-book) - Observed inference time (s)": "{\"description\": \"min=0.373, mean=0.373, max=0.373, sum=0.373 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.3734231026172638\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1990.955, mean=1990.955, max=1990.955, sum=1990.955 (1)\", \"tab\": \"General information\", \"score\": \"1990.955\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=119.262, mean=119.262, max=119.262, sum=119.262 (1)\", \"tab\": \"General information\", \"score\": \"119.262\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "mode": "\"closedbook\"" - } - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.862, - "details": { - "description": "min=0.862, mean=0.862, max=0.862, sum=0.862 (1)", - "tab": "Accuracy", - "OpenbookQA - Observed inference time (s)": "{\"description\": \"min=0.285, mean=0.285, max=0.285, sum=0.285 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.2849515151977539\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=242.846, mean=242.846, max=242.846, sum=242.846 (1)\", \"tab\": \"General information\", \"score\": \"242.846\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "dataset": "\"openbookqa\"", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.626, - "details": { - "description": "min=0.4, mean=0.626, max=0.87, sum=3.131 (5)", - "tab": "Accuracy", - "MMLU - Observed inference time (s)": "{\"description\": \"min=0.285, mean=0.31, max=0.335, sum=1.549 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.30986739750075765\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=371.19, mean=470.836, max=620.939, sum=2354.179 (5)\", \"tab\": \"General information\", \"score\": \"470.8357192982456\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"college_chemistry\", \"computer_security\", \"econometrics\", \"us_foreign_policy\"]", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MATH", - "source_data": { - "dataset_name": "MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Equivalent (CoT) on MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.686, - "details": { - "description": "min=0.6, mean=0.686, max=0.8, sum=4.8 (7)", - "tab": "Accuracy", - "MATH - Observed inference time (s)": "{\"description\": \"min=4.789, mean=4.932, max=5.055, sum=34.522 (7)\", \"tab\": \"Efficiency\", \"score\": \"4.931704092498438\"}", - "MATH - # eval": "{\"description\": \"min=30, mean=62.429, max=135, sum=437 (7)\", \"tab\": \"General information\", \"score\": \"62.42857142857143\"}", - "MATH - # train": "{\"description\": \"min=8, mean=8, max=8, sum=56 (7)\", \"tab\": \"General information\", \"score\": \"8.0\"}", - "MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MATH - # prompt tokens": "{\"description\": \"min=937.926, mean=1323.837, max=2246.673, sum=9266.858 (7)\", \"tab\": \"General information\", \"score\": \"1323.836848955025\"}", - "MATH - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=7 (7)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"algebra\", \"counting_and_probability\", \"geometry\", \"intermediate_algebra\", \"number_theory\", \"prealgebra\", \"precalculus\"]", - "level": "\"1\"", - "use_official_examples": "\"False\"", - "use_chain_of_thought": "\"True\"" - } - } - }, - { - "evaluation_name": "GSM8K", - "source_data": { - "dataset_name": "GSM8K", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on GSM8K", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.693, - "details": { - "description": "min=0.693, mean=0.693, max=0.693, sum=0.693 (1)", - "tab": "Accuracy", - "GSM8K - Observed inference time (s)": "{\"description\": \"min=1.966, mean=1.966, max=1.966, sum=1.966 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.965628466129303\"}", - "GSM8K - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "GSM8K - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "GSM8K - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GSM8K - # prompt tokens": "{\"description\": \"min=1130.403, mean=1130.403, max=1130.403, sum=1130.403 (1)\", \"tab\": \"General information\", \"score\": \"1130.403\"}", - "GSM8K - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "LegalBench", - "source_data": { - "dataset_name": "LegalBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on LegalBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.593, - "details": { - "description": "min=0.358, mean=0.593, max=0.853, sum=2.966 (5)", - "tab": "Accuracy", - "LegalBench - Observed inference time (s)": "{\"description\": \"min=0.332, mean=0.544, max=1.352, sum=2.722 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.5443530451858324\"}", - "LegalBench - # eval": "{\"description\": \"min=95, mean=409.4, max=1000, sum=2047 (5)\", \"tab\": \"General information\", \"score\": \"409.4\"}", - "LegalBench - # train": "{\"description\": \"min=4, mean=4.8, max=5, sum=24 (5)\", \"tab\": \"General information\", \"score\": \"4.8\"}", - "LegalBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "LegalBench - # prompt tokens": "{\"description\": \"min=192.453, mean=1542.088, max=6430.714, sum=7710.442 (5)\", \"tab\": \"General information\", \"score\": \"1542.0883229968654\"}", - "LegalBench - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"abercrombie\", \"corporate_lobbying\", \"function_of_decision_section\", \"international_citizenship_questions\", \"proa\"]" - } - } - }, - { - "evaluation_name": "MedQA", - "source_data": { - "dataset_name": "MedQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MedQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.515, - "details": { - "description": "min=0.515, mean=0.515, max=0.515, sum=0.515 (1)", - "tab": "Accuracy", - "MedQA - Observed inference time (s)": "{\"description\": \"min=0.326, mean=0.326, max=0.326, sum=0.326 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.3256318408025662\"}", - "MedQA - # eval": "{\"description\": \"min=503, mean=503, max=503, sum=503 (1)\", \"tab\": \"General information\", \"score\": \"503.0\"}", - "MedQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MedQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MedQA - # prompt tokens": "{\"description\": \"min=1045.485, mean=1045.485, max=1045.485, sum=1045.485 (1)\", \"tab\": \"General information\", \"score\": \"1045.4850894632207\"}", - "MedQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WMT 2014", - "source_data": { - "dataset_name": "WMT 2014", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "BLEU-4 on WMT 2014", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.178, - "details": { - "description": "min=0.101, mean=0.178, max=0.23, sum=0.89 (5)", - "tab": "Accuracy", - "WMT 2014 - Observed inference time (s)": "{\"description\": \"min=0.59, mean=0.606, max=0.617, sum=3.032 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.606455911532908\"}", - "WMT 2014 - # eval": "{\"description\": \"min=503, mean=568.8, max=832, sum=2844 (5)\", \"tab\": \"General information\", \"score\": \"568.8\"}", - "WMT 2014 - # train": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "WMT 2014 - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WMT 2014 - # prompt tokens": "{\"description\": \"min=108.855, mean=126.657, max=142.373, sum=633.283 (5)\", \"tab\": \"General information\", \"score\": \"126.65662658663405\"}", - "WMT 2014 - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "language_pair": "[\"cs-en\", \"de-en\", \"fr-en\", \"hi-en\", \"ru-en\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_mmlu/qwen_qwen1.5-14b/1774096312.00548", - "retrieved_timestamp": "1774096312.00548", - "source_metadata": { - "source_name": "helm_mmlu", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_mmlu", - "evaluation_results": [ - { - "evaluation_name": "MMLU All Subjects", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU All Subjects", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.686, - "details": { - "description": "min=0.368, mean=0.686, max=0.893, sum=78.254 (114)", - "tab": "Accuracy", - "MMLU All Subjects - Observed inference time (s)": "{\"description\": \"min=0.282, mean=0.321, max=0.549, sum=36.618 (114)\", \"tab\": \"Efficiency\", \"score\": \"0.3212107113231387\"}", - "MMLU All Subjects - # eval": "{\"description\": \"min=100, mean=246.351, max=1534, sum=28084 (114)\", \"tab\": \"General information\", \"score\": \"246.35087719298247\"}", - "MMLU All Subjects - # train": "{\"description\": \"min=5, mean=5, max=5, sum=570 (114)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU All Subjects - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU All Subjects - # prompt tokens": "{\"description\": \"min=269.07, mean=618.598, max=2807.903, sum=70520.198 (114)\", \"tab\": \"General information\", \"score\": \"618.5982315160392\"}", - "MMLU All Subjects - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=114 (114)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_name": "Abstract Algebra", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Abstract Algebra", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4, - "details": { - "description": "min=0.4, mean=0.4, max=0.4, sum=0.8 (2)", - "tab": "Accuracy", - "Abstract Algebra - Observed inference time (s)": "{\"description\": \"min=0.285, mean=0.285, max=0.285, sum=0.569 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.28459527969360354\"}", - "Abstract Algebra - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Abstract Algebra - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Abstract Algebra - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Abstract Algebra - # prompt tokens": "{\"description\": \"min=371.19, mean=371.19, max=371.19, sum=742.38 (2)\", \"tab\": \"General information\", \"score\": \"371.19\"}", - "Abstract Algebra - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"abstract_algebra\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_abstract_algebra\"" - } - } - }, - { - "evaluation_name": "Anatomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Anatomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.637, - "details": { - "description": "min=0.637, mean=0.637, max=0.637, sum=1.274 (2)", - "tab": "Accuracy", - "Anatomy - Observed inference time (s)": "{\"description\": \"min=0.332, mean=0.332, max=0.332, sum=0.663 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.33150761745594165\"}", - "Anatomy - # eval": "{\"description\": \"min=135, mean=135, max=135, sum=270 (2)\", \"tab\": \"General information\", \"score\": \"135.0\"}", - "Anatomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Anatomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Anatomy - # prompt tokens": "{\"description\": \"min=346.978, mean=346.978, max=346.978, sum=693.956 (2)\", \"tab\": \"General information\", \"score\": \"346.97777777777776\"}", - "Anatomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"anatomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_anatomy\"" - } - } - }, - { - "evaluation_name": "College Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on College Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.48, - "details": { - "description": "min=0.48, mean=0.48, max=0.48, sum=0.961 (2)", - "tab": "Accuracy", - "College Chemistry - Observed inference time (s)": "{\"description\": \"min=0.335, mean=0.335, max=0.335, sum=0.67 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.33498176813125613\"}", - "College Biology - Observed inference time (s)": "{\"description\": \"min=0.295, mean=0.295, max=0.295, sum=0.589 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2946729362010956\"}", - "College Computer Science - Observed inference time (s)": "{\"description\": \"min=0.336, mean=0.336, max=0.336, sum=0.673 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3364031720161438\"}", - "College Mathematics - Observed inference time (s)": "{\"description\": \"min=0.324, mean=0.324, max=0.324, sum=0.648 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3238637447357178\"}", - "College Medicine - Observed inference time (s)": "{\"description\": \"min=0.306, mean=0.306, max=0.306, sum=0.611 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3055199033263102\"}", - "College Physics - Observed inference time (s)": "{\"description\": \"min=0.311, mean=0.311, max=0.311, sum=0.622 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.31105106250912534\"}", - "College Chemistry - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Chemistry - # prompt tokens": "{\"description\": \"min=561.25, mean=561.25, max=561.25, sum=1122.5 (2)\", \"tab\": \"General information\", \"score\": \"561.25\"}", - "College Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Biology - # eval": "{\"description\": \"min=144, mean=144, max=144, sum=288 (2)\", \"tab\": \"General information\", \"score\": \"144.0\"}", - "College Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # prompt tokens": "{\"description\": \"min=479.979, mean=479.979, max=479.979, sum=959.958 (2)\", \"tab\": \"General information\", \"score\": \"479.9791666666667\"}", - "College Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # prompt tokens": "{\"description\": \"min=831.58, mean=831.58, max=831.58, sum=1663.16 (2)\", \"tab\": \"General information\", \"score\": \"831.58\"}", - "College Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Mathematics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # prompt tokens": "{\"description\": \"min=600.7, mean=600.7, max=600.7, sum=1201.4 (2)\", \"tab\": \"General information\", \"score\": \"600.7\"}", - "College Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Medicine - # eval": "{\"description\": \"min=173, mean=173, max=173, sum=346 (2)\", \"tab\": \"General information\", \"score\": \"173.0\"}", - "College Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # prompt tokens": "{\"description\": \"min=499.098, mean=499.098, max=499.098, sum=998.197 (2)\", \"tab\": \"General information\", \"score\": \"499.0982658959538\"}", - "College Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Physics - # eval": "{\"description\": \"min=102, mean=102, max=102, sum=204 (2)\", \"tab\": \"General information\", \"score\": \"102.0\"}", - "College Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # prompt tokens": "{\"description\": \"min=509.265, mean=509.265, max=509.265, sum=1018.529 (2)\", \"tab\": \"General information\", \"score\": \"509.2647058823529\"}", - "College Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"college_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_college_physics\"" - } - } - }, - { - "evaluation_name": "Computer Security", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Computer Security", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.84, - "details": { - "description": "min=0.84, mean=0.84, max=0.84, sum=1.68 (2)", - "tab": "Accuracy", - "Computer Security - Observed inference time (s)": "{\"description\": \"min=0.299, mean=0.299, max=0.299, sum=0.598 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2989851474761963\"}", - "Computer Security - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Computer Security - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Computer Security - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Computer Security - # prompt tokens": "{\"description\": \"min=379.64, mean=379.64, max=379.64, sum=759.28 (2)\", \"tab\": \"General information\", \"score\": \"379.64\"}", - "Computer Security - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"computer_security\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_computer_security\"" - } - } - }, - { - "evaluation_name": "Econometrics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Econometrics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.561, - "details": { - "description": "min=0.561, mean=0.561, max=0.561, sum=1.123 (2)", - "tab": "Accuracy", - "Econometrics - Observed inference time (s)": "{\"description\": \"min=0.312, mean=0.312, max=0.312, sum=0.624 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3118862185561866\"}", - "Econometrics - # eval": "{\"description\": \"min=114, mean=114, max=114, sum=228 (2)\", \"tab\": \"General information\", \"score\": \"114.0\"}", - "Econometrics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Econometrics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Econometrics - # prompt tokens": "{\"description\": \"min=620.939, mean=620.939, max=620.939, sum=1241.877 (2)\", \"tab\": \"General information\", \"score\": \"620.938596491228\"}", - "Econometrics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"econometrics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_econometrics\"" - } - } - }, - { - "evaluation_name": "Global Facts", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Global Facts", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.49, - "details": { - "description": "min=0.49, mean=0.49, max=0.49, sum=0.98 (2)", - "tab": "Accuracy", - "Global Facts - Observed inference time (s)": "{\"description\": \"min=0.306, mean=0.306, max=0.306, sum=0.611 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.30553135871887205\"}", - "Global Facts - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Global Facts - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Global Facts - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Global Facts - # prompt tokens": "{\"description\": \"min=422.06, mean=422.06, max=422.06, sum=844.12 (2)\", \"tab\": \"General information\", \"score\": \"422.06\"}", - "Global Facts - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"global_facts\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_global_facts\"" - } - } - }, - { - "evaluation_name": "Jurisprudence", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Jurisprudence", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.769, - "details": { - "description": "min=0.769, mean=0.769, max=0.769, sum=1.537 (2)", - "tab": "Accuracy", - "Jurisprudence - Observed inference time (s)": "{\"description\": \"min=0.309, mean=0.309, max=0.309, sum=0.618 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3092155566921941\"}", - "Jurisprudence - # eval": "{\"description\": \"min=108, mean=108, max=108, sum=216 (2)\", \"tab\": \"General information\", \"score\": \"108.0\"}", - "Jurisprudence - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Jurisprudence - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Jurisprudence - # prompt tokens": "{\"description\": \"min=387.713, mean=387.713, max=387.713, sum=775.426 (2)\", \"tab\": \"General information\", \"score\": \"387.712962962963\"}", - "Jurisprudence - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"jurisprudence\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_jurisprudence\"" - } - } - }, - { - "evaluation_name": "Philosophy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Philosophy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.717, - "details": { - "description": "min=0.717, mean=0.717, max=0.717, sum=1.434 (2)", - "tab": "Accuracy", - "Philosophy - Observed inference time (s)": "{\"description\": \"min=0.311, mean=0.311, max=0.311, sum=0.622 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3108927659283114\"}", - "Philosophy - # eval": "{\"description\": \"min=311, mean=311, max=311, sum=622 (2)\", \"tab\": \"General information\", \"score\": \"311.0\"}", - "Philosophy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Philosophy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Philosophy - # prompt tokens": "{\"description\": \"min=322.09, mean=322.09, max=322.09, sum=644.18 (2)\", \"tab\": \"General information\", \"score\": \"322.09003215434086\"}", - "Philosophy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"philosophy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_philosophy\"" - } - } - }, - { - "evaluation_name": "Professional Psychology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Professional Psychology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.699, - "details": { - "description": "min=0.699, mean=0.699, max=0.699, sum=1.399 (2)", - "tab": "Accuracy", - "Professional Medicine - Observed inference time (s)": "{\"description\": \"min=0.475, mean=0.475, max=0.475, sum=0.951 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.47532147870344277\"}", - "Professional Accounting - Observed inference time (s)": "{\"description\": \"min=0.319, mean=0.319, max=0.319, sum=0.638 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.31895153404127624\"}", - "Professional Law - Observed inference time (s)": "{\"description\": \"min=0.4, mean=0.4, max=0.4, sum=0.8 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4000247932941382\"}", - "Professional Psychology - Observed inference time (s)": "{\"description\": \"min=0.301, mean=0.301, max=0.301, sum=0.602 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3012406826019287\"}", - "Professional Medicine - # eval": "{\"description\": \"min=272, mean=272, max=272, sum=544 (2)\", \"tab\": \"General information\", \"score\": \"272.0\"}", - "Professional Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Medicine - # prompt tokens": "{\"description\": \"min=1118.199, mean=1118.199, max=1118.199, sum=2236.397 (2)\", \"tab\": \"General information\", \"score\": \"1118.1985294117646\"}", - "Professional Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Accounting - # eval": "{\"description\": \"min=282, mean=282, max=282, sum=564 (2)\", \"tab\": \"General information\", \"score\": \"282.0\"}", - "Professional Accounting - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Accounting - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # prompt tokens": "{\"description\": \"min=732.34, mean=732.34, max=732.34, sum=1464.681 (2)\", \"tab\": \"General information\", \"score\": \"732.3404255319149\"}", - "Professional Accounting - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Law - # eval": "{\"description\": \"min=1534, mean=1534, max=1534, sum=3068 (2)\", \"tab\": \"General information\", \"score\": \"1534.0\"}", - "Professional Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # prompt tokens": "{\"description\": \"min=1656.969, mean=1656.969, max=1656.969, sum=3313.939 (2)\", \"tab\": \"General information\", \"score\": \"1656.9693611473272\"}", - "Professional Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Psychology - # eval": "{\"description\": \"min=612, mean=612, max=612, sum=1224 (2)\", \"tab\": \"General information\", \"score\": \"612.0\"}", - "Professional Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # prompt tokens": "{\"description\": \"min=574.417, mean=574.417, max=574.417, sum=1148.833 (2)\", \"tab\": \"General information\", \"score\": \"574.4166666666666\"}", - "Professional Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"professional_psychology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_professional_psychology\"" - } - } - }, - { - "evaluation_name": "Us Foreign Policy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Us Foreign Policy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.87, - "details": { - "description": "min=0.87, mean=0.87, max=0.87, sum=1.74 (2)", - "tab": "Accuracy", - "Us Foreign Policy - Observed inference time (s)": "{\"description\": \"min=0.319, mean=0.319, max=0.319, sum=0.638 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.31888857364654544\"}", - "Us Foreign Policy - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Us Foreign Policy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Us Foreign Policy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Us Foreign Policy - # prompt tokens": "{\"description\": \"min=421.16, mean=421.16, max=421.16, sum=842.32 (2)\", \"tab\": \"General information\", \"score\": \"421.16\"}", - "Us Foreign Policy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"us_foreign_policy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_us_foreign_policy\"" - } - } - }, - { - "evaluation_name": "Astronomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Astronomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.724, - "details": { - "description": "min=0.724, mean=0.724, max=0.724, sum=1.447 (2)", - "tab": "Accuracy", - "Astronomy - Observed inference time (s)": "{\"description\": \"min=0.295, mean=0.295, max=0.295, sum=0.589 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.29459338125429657\"}", - "Astronomy - # eval": "{\"description\": \"min=152, mean=152, max=152, sum=304 (2)\", \"tab\": \"General information\", \"score\": \"152.0\"}", - "Astronomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Astronomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Astronomy - # prompt tokens": "{\"description\": \"min=582.849, mean=582.849, max=582.849, sum=1165.697 (2)\", \"tab\": \"General information\", \"score\": \"582.8486842105264\"}", - "Astronomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"astronomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_astronomy\"" - } - } - }, - { - "evaluation_name": "Business Ethics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Business Ethics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.75, - "details": { - "description": "min=0.75, mean=0.75, max=0.75, sum=1.5 (2)", - "tab": "Accuracy", - "Business Ethics - Observed inference time (s)": "{\"description\": \"min=0.323, mean=0.323, max=0.323, sum=0.647 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.32330512285232543\"}", - "Business Ethics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Business Ethics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Business Ethics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Business Ethics - # prompt tokens": "{\"description\": \"min=562.87, mean=562.87, max=562.87, sum=1125.74 (2)\", \"tab\": \"General information\", \"score\": \"562.87\"}", - "Business Ethics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"business_ethics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_business_ethics\"" - } - } - }, - { - "evaluation_name": "Clinical Knowledge", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Clinical Knowledge", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.736, - "details": { - "description": "min=0.736, mean=0.736, max=0.736, sum=1.472 (2)", - "tab": "Accuracy", - "Clinical Knowledge - Observed inference time (s)": "{\"description\": \"min=0.299, mean=0.299, max=0.299, sum=0.598 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2987864755234628\"}", - "Clinical Knowledge - # eval": "{\"description\": \"min=265, mean=265, max=265, sum=530 (2)\", \"tab\": \"General information\", \"score\": \"265.0\"}", - "Clinical Knowledge - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Clinical Knowledge - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Clinical Knowledge - # prompt tokens": "{\"description\": \"min=393.623, mean=393.623, max=393.623, sum=787.245 (2)\", \"tab\": \"General information\", \"score\": \"393.62264150943395\"}", - "Clinical Knowledge - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"clinical_knowledge\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_clinical_knowledge\"" - } - } - }, - { - "evaluation_name": "Conceptual Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Conceptual Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.694, - "details": { - "description": "min=0.694, mean=0.694, max=0.694, sum=1.387 (2)", - "tab": "Accuracy", - "Conceptual Physics - Observed inference time (s)": "{\"description\": \"min=0.287, mean=0.287, max=0.287, sum=0.575 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2873024098416592\"}", - "Conceptual Physics - # eval": "{\"description\": \"min=235, mean=235, max=235, sum=470 (2)\", \"tab\": \"General information\", \"score\": \"235.0\"}", - "Conceptual Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Conceptual Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Conceptual Physics - # prompt tokens": "{\"description\": \"min=298.494, mean=298.494, max=298.494, sum=596.987 (2)\", \"tab\": \"General information\", \"score\": \"298.4936170212766\"}", - "Conceptual Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"conceptual_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_conceptual_physics\"" - } - } - }, - { - "evaluation_name": "Electrical Engineering", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Electrical Engineering", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.683, - "details": { - "description": "min=0.683, mean=0.683, max=0.683, sum=1.366 (2)", - "tab": "Accuracy", - "Electrical Engineering - Observed inference time (s)": "{\"description\": \"min=0.286, mean=0.286, max=0.286, sum=0.573 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2863943790567332\"}", - "Electrical Engineering - # eval": "{\"description\": \"min=145, mean=145, max=145, sum=290 (2)\", \"tab\": \"General information\", \"score\": \"145.0\"}", - "Electrical Engineering - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Electrical Engineering - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Electrical Engineering - # prompt tokens": "{\"description\": \"min=456.8, mean=456.8, max=456.8, sum=913.6 (2)\", \"tab\": \"General information\", \"score\": \"456.8\"}", - "Electrical Engineering - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"electrical_engineering\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_electrical_engineering\"" - } - } - }, - { - "evaluation_name": "Elementary Mathematics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Elementary Mathematics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.603, - "details": { - "description": "min=0.603, mean=0.603, max=0.603, sum=1.206 (2)", - "tab": "Accuracy", - "Elementary Mathematics - Observed inference time (s)": "{\"description\": \"min=0.317, mean=0.317, max=0.317, sum=0.635 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3172515391041993\"}", - "Elementary Mathematics - # eval": "{\"description\": \"min=378, mean=378, max=378, sum=756 (2)\", \"tab\": \"General information\", \"score\": \"378.0\"}", - "Elementary Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Elementary Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Elementary Mathematics - # prompt tokens": "{\"description\": \"min=570.119, mean=570.119, max=570.119, sum=1140.238 (2)\", \"tab\": \"General information\", \"score\": \"570.1190476190476\"}", - "Elementary Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"elementary_mathematics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_elementary_mathematics\"" - } - } - }, - { - "evaluation_name": "Formal Logic", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Formal Logic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.492, - "details": { - "description": "min=0.492, mean=0.492, max=0.492, sum=0.984 (2)", - "tab": "Accuracy", - "Formal Logic - Observed inference time (s)": "{\"description\": \"min=0.317, mean=0.317, max=0.317, sum=0.634 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.31694961918724907\"}", - "Formal Logic - # eval": "{\"description\": \"min=126, mean=126, max=126, sum=252 (2)\", \"tab\": \"General information\", \"score\": \"126.0\"}", - "Formal Logic - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Formal Logic - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Formal Logic - # prompt tokens": "{\"description\": \"min=597.667, mean=597.667, max=597.667, sum=1195.333 (2)\", \"tab\": \"General information\", \"score\": \"597.6666666666666\"}", - "Formal Logic - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"formal_logic\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_formal_logic\"" - } - } - }, - { - "evaluation_name": "High School World History", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on High School World History", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.84, - "details": { - "description": "min=0.84, mean=0.84, max=0.84, sum=1.679 (2)", - "tab": "Accuracy", - "High School Biology - Observed inference time (s)": "{\"description\": \"min=0.303, mean=0.303, max=0.303, sum=0.605 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3025627659213158\"}", - "High School Chemistry - Observed inference time (s)": "{\"description\": \"min=0.311, mean=0.311, max=0.311, sum=0.622 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3108991178972968\"}", - "High School Computer Science - Observed inference time (s)": "{\"description\": \"min=0.305, mean=0.305, max=0.305, sum=0.61 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.30484641551971436\"}", - "High School European History - Observed inference time (s)": "{\"description\": \"min=0.549, mean=0.549, max=0.549, sum=1.098 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.548761223301743\"}", - "High School Geography - Observed inference time (s)": "{\"description\": \"min=0.312, mean=0.312, max=0.312, sum=0.624 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3120840137655085\"}", - "High School Government And Politics - Observed inference time (s)": "{\"description\": \"min=0.3, mean=0.3, max=0.3, sum=0.599 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.29960165616761836\"}", - "High School Macroeconomics - Observed inference time (s)": "{\"description\": \"min=0.294, mean=0.294, max=0.294, sum=0.588 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.29392006519513253\"}", - "High School Mathematics - Observed inference time (s)": "{\"description\": \"min=0.312, mean=0.312, max=0.312, sum=0.625 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3124903016620212\"}", - "High School Microeconomics - Observed inference time (s)": "{\"description\": \"min=0.282, mean=0.282, max=0.282, sum=0.565 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.28235371273104887\"}", - "High School Physics - Observed inference time (s)": "{\"description\": \"min=0.308, mean=0.308, max=0.308, sum=0.615 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.30758162681630113\"}", - "High School Psychology - Observed inference time (s)": "{\"description\": \"min=0.317, mean=0.317, max=0.317, sum=0.634 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3172066456680998\"}", - "High School Statistics - Observed inference time (s)": "{\"description\": \"min=0.335, mean=0.335, max=0.335, sum=0.67 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.33508766580511024\"}", - "High School US History - Observed inference time (s)": "{\"description\": \"min=0.453, mean=0.453, max=0.453, sum=0.906 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4531192370489532\"}", - "High School World History - Observed inference time (s)": "{\"description\": \"min=0.386, mean=0.386, max=0.386, sum=0.771 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3856232206529706\"}", - "High School Biology - # eval": "{\"description\": \"min=310, mean=310, max=310, sum=620 (2)\", \"tab\": \"General information\", \"score\": \"310.0\"}", - "High School Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Biology - # prompt tokens": "{\"description\": \"min=506.916, mean=506.916, max=506.916, sum=1013.832 (2)\", \"tab\": \"General information\", \"score\": \"506.9161290322581\"}", - "High School Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Chemistry - # eval": "{\"description\": \"min=203, mean=203, max=203, sum=406 (2)\", \"tab\": \"General information\", \"score\": \"203.0\"}", - "High School Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # prompt tokens": "{\"description\": \"min=510.261, mean=510.261, max=510.261, sum=1020.522 (2)\", \"tab\": \"General information\", \"score\": \"510.2610837438424\"}", - "High School Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "High School Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # prompt tokens": "{\"description\": \"min=871.46, mean=871.46, max=871.46, sum=1742.92 (2)\", \"tab\": \"General information\", \"score\": \"871.46\"}", - "High School Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School European History - # eval": "{\"description\": \"min=165, mean=165, max=165, sum=330 (2)\", \"tab\": \"General information\", \"score\": \"165.0\"}", - "High School European History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School European History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # prompt tokens": "{\"description\": \"min=2807.903, mean=2807.903, max=2807.903, sum=5615.806 (2)\", \"tab\": \"General information\", \"score\": \"2807.9030303030304\"}", - "High School European History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Geography - # eval": "{\"description\": \"min=198, mean=198, max=198, sum=396 (2)\", \"tab\": \"General information\", \"score\": \"198.0\"}", - "High School Geography - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Geography - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # prompt tokens": "{\"description\": \"min=365.217, mean=365.217, max=365.217, sum=730.434 (2)\", \"tab\": \"General information\", \"score\": \"365.2171717171717\"}", - "High School Geography - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Government And Politics - # eval": "{\"description\": \"min=193, mean=193, max=193, sum=386 (2)\", \"tab\": \"General information\", \"score\": \"193.0\"}", - "High School Government And Politics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Government And Politics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # prompt tokens": "{\"description\": \"min=460.311, mean=460.311, max=460.311, sum=920.622 (2)\", \"tab\": \"General information\", \"score\": \"460.31088082901556\"}", - "High School Government And Politics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Macroeconomics - # eval": "{\"description\": \"min=390, mean=390, max=390, sum=780 (2)\", \"tab\": \"General information\", \"score\": \"390.0\"}", - "High School Macroeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Macroeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # prompt tokens": "{\"description\": \"min=367.349, mean=367.349, max=367.349, sum=734.697 (2)\", \"tab\": \"General information\", \"score\": \"367.34871794871793\"}", - "High School Macroeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Mathematics - # eval": "{\"description\": \"min=270, mean=270, max=270, sum=540 (2)\", \"tab\": \"General information\", \"score\": \"270.0\"}", - "High School Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # prompt tokens": "{\"description\": \"min=558.326, mean=558.326, max=558.326, sum=1116.652 (2)\", \"tab\": \"General information\", \"score\": \"558.325925925926\"}", - "High School Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Microeconomics - # eval": "{\"description\": \"min=238, mean=238, max=238, sum=476 (2)\", \"tab\": \"General information\", \"score\": \"238.0\"}", - "High School Microeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Microeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # prompt tokens": "{\"description\": \"min=395.277, mean=395.277, max=395.277, sum=790.555 (2)\", \"tab\": \"General information\", \"score\": \"395.2773109243698\"}", - "High School Microeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Physics - # eval": "{\"description\": \"min=151, mean=151, max=151, sum=302 (2)\", \"tab\": \"General information\", \"score\": \"151.0\"}", - "High School Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # prompt tokens": "{\"description\": \"min=573.536, mean=573.536, max=573.536, sum=1147.073 (2)\", \"tab\": \"General information\", \"score\": \"573.5364238410596\"}", - "High School Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Psychology - # eval": "{\"description\": \"min=545, mean=545, max=545, sum=1090 (2)\", \"tab\": \"General information\", \"score\": \"545.0\"}", - "High School Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # prompt tokens": "{\"description\": \"min=488.521, mean=488.521, max=488.521, sum=977.042 (2)\", \"tab\": \"General information\", \"score\": \"488.52110091743117\"}", - "High School Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Statistics - # eval": "{\"description\": \"min=216, mean=216, max=216, sum=432 (2)\", \"tab\": \"General information\", \"score\": \"216.0\"}", - "High School Statistics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Statistics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # prompt tokens": "{\"description\": \"min=823.477, mean=823.477, max=823.477, sum=1646.954 (2)\", \"tab\": \"General information\", \"score\": \"823.4768518518518\"}", - "High School Statistics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School US History - # eval": "{\"description\": \"min=204, mean=204, max=204, sum=408 (2)\", \"tab\": \"General information\", \"score\": \"204.0\"}", - "High School US History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School US History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # prompt tokens": "{\"description\": \"min=2230.176, mean=2230.176, max=2230.176, sum=4460.353 (2)\", \"tab\": \"General information\", \"score\": \"2230.176470588235\"}", - "High School US History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School World History - # eval": "{\"description\": \"min=237, mean=237, max=237, sum=474 (2)\", \"tab\": \"General information\", \"score\": \"237.0\"}", - "High School World History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School World History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # prompt tokens": "{\"description\": \"min=1441.354, mean=1441.354, max=1441.354, sum=2882.709 (2)\", \"tab\": \"General information\", \"score\": \"1441.3544303797469\"}", - "High School World History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"high_school_world_history\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_high_school_world_history\"" - } - } - }, - { - "evaluation_name": "Human Sexuality", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Human Sexuality", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.756, - "details": { - "description": "min=0.756, mean=0.756, max=0.756, sum=1.511 (2)", - "tab": "Accuracy", - "Human Aging - Observed inference time (s)": "{\"description\": \"min=0.29, mean=0.29, max=0.29, sum=0.58 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.29016303160799994\"}", - "Human Sexuality - Observed inference time (s)": "{\"description\": \"min=0.322, mean=0.322, max=0.322, sum=0.645 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3224487978083487\"}", - "Human Aging - # eval": "{\"description\": \"min=223, mean=223, max=223, sum=446 (2)\", \"tab\": \"General information\", \"score\": \"223.0\"}", - "Human Aging - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Aging - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Aging - # prompt tokens": "{\"description\": \"min=315.121, mean=315.121, max=315.121, sum=630.242 (2)\", \"tab\": \"General information\", \"score\": \"315.1210762331838\"}", - "Human Aging - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Human Sexuality - # eval": "{\"description\": \"min=131, mean=131, max=131, sum=262 (2)\", \"tab\": \"General information\", \"score\": \"131.0\"}", - "Human Sexuality - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Sexuality - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # prompt tokens": "{\"description\": \"min=334.504, mean=334.504, max=334.504, sum=669.008 (2)\", \"tab\": \"General information\", \"score\": \"334.5038167938931\"}", - "Human Sexuality - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"human_sexuality\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_human_sexuality\"" - } - } - }, - { - "evaluation_name": "International Law", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on International Law", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.826, - "details": { - "description": "min=0.826, mean=0.826, max=0.826, sum=1.653 (2)", - "tab": "Accuracy", - "International Law - Observed inference time (s)": "{\"description\": \"min=0.308, mean=0.308, max=0.308, sum=0.615 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.307678321176324\"}", - "International Law - # eval": "{\"description\": \"min=121, mean=121, max=121, sum=242 (2)\", \"tab\": \"General information\", \"score\": \"121.0\"}", - "International Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "International Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "International Law - # prompt tokens": "{\"description\": \"min=633.579, mean=633.579, max=633.579, sum=1267.157 (2)\", \"tab\": \"General information\", \"score\": \"633.5785123966942\"}", - "International Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"international_law\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_international_law\"" - } - } - }, - { - "evaluation_name": "Logical Fallacies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Logical Fallacies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.736, - "details": { - "description": "min=0.736, mean=0.736, max=0.736, sum=1.472 (2)", - "tab": "Accuracy", - "Logical Fallacies - Observed inference time (s)": "{\"description\": \"min=0.305, mean=0.305, max=0.305, sum=0.61 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3051488355624895\"}", - "Logical Fallacies - # eval": "{\"description\": \"min=163, mean=163, max=163, sum=326 (2)\", \"tab\": \"General information\", \"score\": \"163.0\"}", - "Logical Fallacies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Logical Fallacies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Logical Fallacies - # prompt tokens": "{\"description\": \"min=442.632, mean=442.632, max=442.632, sum=885.264 (2)\", \"tab\": \"General information\", \"score\": \"442.6319018404908\"}", - "Logical Fallacies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"logical_fallacies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_logical_fallacies\"" - } - } - }, - { - "evaluation_name": "Machine Learning", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Machine Learning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.509, - "details": { - "description": "min=0.509, mean=0.509, max=0.509, sum=1.018 (2)", - "tab": "Accuracy", - "Machine Learning - Observed inference time (s)": "{\"description\": \"min=0.308, mean=0.308, max=0.308, sum=0.616 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3079095014504024\"}", - "Machine Learning - # eval": "{\"description\": \"min=112, mean=112, max=112, sum=224 (2)\", \"tab\": \"General information\", \"score\": \"112.0\"}", - "Machine Learning - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Machine Learning - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Machine Learning - # prompt tokens": "{\"description\": \"min=674.848, mean=674.848, max=674.848, sum=1349.696 (2)\", \"tab\": \"General information\", \"score\": \"674.8482142857143\"}", - "Machine Learning - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"machine_learning\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_machine_learning\"" - } - } - }, - { - "evaluation_name": "Management", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Management", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.816, - "details": { - "description": "min=0.816, mean=0.816, max=0.816, sum=1.631 (2)", - "tab": "Accuracy", - "Management - Observed inference time (s)": "{\"description\": \"min=0.317, mean=0.317, max=0.317, sum=0.633 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.316567536696647\"}", - "Management - # eval": "{\"description\": \"min=103, mean=103, max=103, sum=206 (2)\", \"tab\": \"General information\", \"score\": \"103.0\"}", - "Management - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Management - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Management - # prompt tokens": "{\"description\": \"min=276.854, mean=276.854, max=276.854, sum=553.709 (2)\", \"tab\": \"General information\", \"score\": \"276.8543689320388\"}", - "Management - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"management\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_management\"" - } - } - }, - { - "evaluation_name": "Marketing", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Marketing", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.893, - "details": { - "description": "min=0.893, mean=0.893, max=0.893, sum=1.786 (2)", - "tab": "Accuracy", - "Marketing - Observed inference time (s)": "{\"description\": \"min=0.31, mean=0.31, max=0.31, sum=0.621 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3104041937070015\"}", - "Marketing - # eval": "{\"description\": \"min=234, mean=234, max=234, sum=468 (2)\", \"tab\": \"General information\", \"score\": \"234.0\"}", - "Marketing - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Marketing - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Marketing - # prompt tokens": "{\"description\": \"min=397.415, mean=397.415, max=397.415, sum=794.829 (2)\", \"tab\": \"General information\", \"score\": \"397.4145299145299\"}", - "Marketing - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"marketing\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_marketing\"" - } - } - }, - { - "evaluation_name": "Medical Genetics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Medical Genetics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.76, - "details": { - "description": "min=0.76, mean=0.76, max=0.76, sum=1.52 (2)", - "tab": "Accuracy", - "Medical Genetics - Observed inference time (s)": "{\"description\": \"min=0.302, mean=0.302, max=0.302, sum=0.603 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.30150007486343383\"}", - "Medical Genetics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Medical Genetics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Medical Genetics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Medical Genetics - # prompt tokens": "{\"description\": \"min=335.35, mean=335.35, max=335.35, sum=670.7 (2)\", \"tab\": \"General information\", \"score\": \"335.35\"}", - "Medical Genetics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"medical_genetics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_medical_genetics\"" - } - } - }, - { - "evaluation_name": "Miscellaneous", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Miscellaneous", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.835, - "details": { - "description": "min=0.835, mean=0.835, max=0.835, sum=1.67 (2)", - "tab": "Accuracy", - "Miscellaneous - Observed inference time (s)": "{\"description\": \"min=0.294, mean=0.294, max=0.294, sum=0.588 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.29396778352720376\"}", - "Miscellaneous - # eval": "{\"description\": \"min=783, mean=783, max=783, sum=1566 (2)\", \"tab\": \"General information\", \"score\": \"783.0\"}", - "Miscellaneous - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Miscellaneous - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Miscellaneous - # prompt tokens": "{\"description\": \"min=296.7, mean=296.7, max=296.7, sum=593.4 (2)\", \"tab\": \"General information\", \"score\": \"296.6998722860792\"}", - "Miscellaneous - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"miscellaneous\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_miscellaneous\"" - } - } - }, - { - "evaluation_name": "Moral Scenarios", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Moral Scenarios", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.368, - "details": { - "description": "min=0.368, mean=0.368, max=0.368, sum=0.735 (2)", - "tab": "Accuracy", - "Moral Disputes - Observed inference time (s)": "{\"description\": \"min=0.304, mean=0.304, max=0.304, sum=0.608 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.30380174465951204\"}", - "Moral Scenarios - Observed inference time (s)": "{\"description\": \"min=0.301, mean=0.301, max=0.301, sum=0.601 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3006620183337334\"}", - "Moral Disputes - # eval": "{\"description\": \"min=346, mean=346, max=346, sum=692 (2)\", \"tab\": \"General information\", \"score\": \"346.0\"}", - "Moral Disputes - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Disputes - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Disputes - # prompt tokens": "{\"description\": \"min=469.182, mean=469.182, max=469.182, sum=938.364 (2)\", \"tab\": \"General information\", \"score\": \"469.1820809248555\"}", - "Moral Disputes - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Moral Scenarios - # eval": "{\"description\": \"min=895, mean=895, max=895, sum=1790 (2)\", \"tab\": \"General information\", \"score\": \"895.0\"}", - "Moral Scenarios - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Scenarios - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # prompt tokens": "{\"description\": \"min=661.494, mean=661.494, max=661.494, sum=1322.988 (2)\", \"tab\": \"General information\", \"score\": \"661.4938547486033\"}", - "Moral Scenarios - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"moral_scenarios\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_moral_scenarios\"" - } - } - }, - { - "evaluation_name": "Nutrition", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Nutrition", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.742, - "details": { - "description": "min=0.742, mean=0.742, max=0.742, sum=1.484 (2)", - "tab": "Accuracy", - "Nutrition - Observed inference time (s)": "{\"description\": \"min=0.319, mean=0.319, max=0.319, sum=0.639 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.31930122655980725\"}", - "Nutrition - # eval": "{\"description\": \"min=306, mean=306, max=306, sum=612 (2)\", \"tab\": \"General information\", \"score\": \"306.0\"}", - "Nutrition - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Nutrition - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Nutrition - # prompt tokens": "{\"description\": \"min=592.637, mean=592.637, max=592.637, sum=1185.275 (2)\", \"tab\": \"General information\", \"score\": \"592.6372549019608\"}", - "Nutrition - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"nutrition\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_nutrition\"" - } - } - }, - { - "evaluation_name": "Prehistory", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Prehistory", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.71, - "details": { - "description": "min=0.71, mean=0.71, max=0.71, sum=1.42 (2)", - "tab": "Accuracy", - "Prehistory - Observed inference time (s)": "{\"description\": \"min=0.313, mean=0.313, max=0.313, sum=0.625 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3125371013158633\"}", - "Prehistory - # eval": "{\"description\": \"min=324, mean=324, max=324, sum=648 (2)\", \"tab\": \"General information\", \"score\": \"324.0\"}", - "Prehistory - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Prehistory - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Prehistory - # prompt tokens": "{\"description\": \"min=521.364, mean=521.364, max=521.364, sum=1042.728 (2)\", \"tab\": \"General information\", \"score\": \"521.3641975308642\"}", - "Prehistory - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"prehistory\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_prehistory\"" - } - } - }, - { - "evaluation_name": "Public Relations", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Public Relations", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.655, - "details": { - "description": "min=0.655, mean=0.655, max=0.655, sum=1.309 (2)", - "tab": "Accuracy", - "Public Relations - Observed inference time (s)": "{\"description\": \"min=0.296, mean=0.296, max=0.296, sum=0.592 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.29603702588514846\"}", - "Public Relations - # eval": "{\"description\": \"min=110, mean=110, max=110, sum=220 (2)\", \"tab\": \"General information\", \"score\": \"110.0\"}", - "Public Relations - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Public Relations - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Public Relations - # prompt tokens": "{\"description\": \"min=401.427, mean=401.427, max=401.427, sum=802.855 (2)\", \"tab\": \"General information\", \"score\": \"401.42727272727274\"}", - "Public Relations - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"public_relations\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_public_relations\"" - } - } - }, - { - "evaluation_name": "Security Studies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Security Studies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8, - "details": { - "description": "min=0.8, mean=0.8, max=0.8, sum=1.6 (2)", - "tab": "Accuracy", - "Security Studies - Observed inference time (s)": "{\"description\": \"min=0.352, mean=0.352, max=0.352, sum=0.704 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3521312304905483\"}", - "Security Studies - # eval": "{\"description\": \"min=245, mean=245, max=245, sum=490 (2)\", \"tab\": \"General information\", \"score\": \"245.0\"}", - "Security Studies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Security Studies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Security Studies - # prompt tokens": "{\"description\": \"min=1159.931, mean=1159.931, max=1159.931, sum=2319.861 (2)\", \"tab\": \"General information\", \"score\": \"1159.930612244898\"}", - "Security Studies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"security_studies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_security_studies\"" - } - } - }, - { - "evaluation_name": "Sociology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Sociology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.841, - "details": { - "description": "min=0.841, mean=0.841, max=0.841, sum=1.682 (2)", - "tab": "Accuracy", - "Sociology - Observed inference time (s)": "{\"description\": \"min=0.304, mean=0.304, max=0.304, sum=0.609 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3044381426341498\"}", - "Sociology - # eval": "{\"description\": \"min=201, mean=201, max=201, sum=402 (2)\", \"tab\": \"General information\", \"score\": \"201.0\"}", - "Sociology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Sociology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Sociology - # prompt tokens": "{\"description\": \"min=443.1, mean=443.1, max=443.1, sum=886.199 (2)\", \"tab\": \"General information\", \"score\": \"443.0995024875622\"}", - "Sociology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"sociology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_sociology\"" - } - } - }, - { - "evaluation_name": "Virology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Virology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.458, - "details": { - "description": "min=0.458, mean=0.458, max=0.458, sum=0.916 (2)", - "tab": "Accuracy", - "Virology - Observed inference time (s)": "{\"description\": \"min=0.297, mean=0.297, max=0.297, sum=0.595 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.297343333083463\"}", - "Virology - # eval": "{\"description\": \"min=166, mean=166, max=166, sum=332 (2)\", \"tab\": \"General information\", \"score\": \"166.0\"}", - "Virology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Virology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Virology - # prompt tokens": "{\"description\": \"min=336.819, mean=336.819, max=336.819, sum=673.639 (2)\", \"tab\": \"General information\", \"score\": \"336.8192771084337\"}", - "Virology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"virology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_virology\"" - } - } - }, - { - "evaluation_name": "World Religions", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on World Religions", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.842, - "details": { - "description": "min=0.842, mean=0.842, max=0.842, sum=1.684 (2)", - "tab": "Accuracy", - "World Religions - Observed inference time (s)": "{\"description\": \"min=0.303, mean=0.303, max=0.303, sum=0.605 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3027164573557893\"}", - "World Religions - # eval": "{\"description\": \"min=171, mean=171, max=171, sum=342 (2)\", \"tab\": \"General information\", \"score\": \"171.0\"}", - "World Religions - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "World Religions - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "World Religions - # prompt tokens": "{\"description\": \"min=269.07, mean=269.07, max=269.07, sum=538.14 (2)\", \"tab\": \"General information\", \"score\": \"269.0701754385965\"}", - "World Religions - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"world_religions\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_world_religions\"" - } - } - }, - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.796, - "details": { - "description": "", - "tab": "Efficiency" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_id": "hfopenllm_v2/Qwen_Qwen1.5-14B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2905 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.508 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2024 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2945 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4186 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3644 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qwen_qwen1.5-32b-chat.json b/data/models/qwen_qwen1.5-32b-chat.json deleted file mode 100644 index 173c5eb1187634fb7d5f84f9307dffdd7ecebeef..0000000000000000000000000000000000000000 --- a/data/models/qwen_qwen1.5-32b-chat.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen1.5-32B-Chat", - "id": "Qwen/Qwen1.5-32B-Chat", - "developer": "Qwen", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "32.512" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Qwen_Qwen1.5-32B-Chat/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5532 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6067 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1956 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3062 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.416 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4457 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qwen_qwen1.5-32b.json b/data/models/qwen_qwen1.5-32b.json deleted file mode 100644 index e5d6cf1a96e98d1bd267732c3e9145759f0b6088..0000000000000000000000000000000000000000 --- a/data/models/qwen_qwen1.5-32b.json +++ /dev/null @@ -1,2036 +0,0 @@ -{ - "model_info": { - "name": "Qwen1.5 32B", - "id": "qwen/qwen1.5-32b", - "developer": "qwen", - "inference_platform": "unknown", - "additional_details": { - "model_id_aliases": [ - "Qwen/Qwen1.5-32B" - ] - } - }, - "evaluations": [ - { - "evaluation_id": "helm_lite/qwen_qwen1.5-32b/1774096306.427425", - "retrieved_timestamp": "1774096306.427425", - "source_metadata": { - "source_name": "helm_lite", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_lite", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_lite", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.546, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.47831460674157306\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.589, - "details": { - "description": "min=0.589, mean=0.589, max=0.589, sum=0.589 (1)", - "tab": "Accuracy", - "NarrativeQA - Observed inference time (s)": "{\"description\": \"min=1.848, mean=1.848, max=1.848, sum=1.848 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.847580643774758\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=3468.913, mean=3468.913, max=3468.913, sum=3468.913 (1)\", \"tab\": \"General information\", \"score\": \"3468.912676056338\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (closed-book)", - "source_data": { - "dataset_name": "NaturalQuestions (closed-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (closed-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.353, - "details": { - "description": "min=0.353, mean=0.353, max=0.353, sum=0.353 (1)", - "tab": "Accuracy", - "NaturalQuestions (open-book) - Observed inference time (s)": "{\"description\": \"min=1.139, mean=1.139, max=1.139, sum=1.139 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.1394575798511506\"}", - "NaturalQuestions (closed-book) - Observed inference time (s)": "{\"description\": \"min=0.457, mean=0.457, max=0.457, sum=0.457 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.457463458776474\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1990.955, mean=1990.955, max=1990.955, sum=1990.955 (1)\", \"tab\": \"General information\", \"score\": \"1990.955\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=119.262, mean=119.262, max=119.262, sum=119.262 (1)\", \"tab\": \"General information\", \"score\": \"119.262\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "mode": "\"closedbook\"" - } - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.932, - "details": { - "description": "min=0.932, mean=0.932, max=0.932, sum=0.932 (1)", - "tab": "Accuracy", - "OpenbookQA - Observed inference time (s)": "{\"description\": \"min=0.352, mean=0.352, max=0.352, sum=0.352 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.3515647969245911\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=242.846, mean=242.846, max=242.846, sum=242.846 (1)\", \"tab\": \"General information\", \"score\": \"242.846\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "dataset": "\"openbookqa\"", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.628, - "details": { - "description": "min=0.4, mean=0.628, max=0.91, sum=3.141 (5)", - "tab": "Accuracy", - "MMLU - Observed inference time (s)": "{\"description\": \"min=0.337, mean=0.345, max=0.367, sum=1.724 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.34482146733267266\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=371.19, mean=470.836, max=620.939, sum=2354.179 (5)\", \"tab\": \"General information\", \"score\": \"470.8357192982456\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"college_chemistry\", \"computer_security\", \"econometrics\", \"us_foreign_policy\"]", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MATH", - "source_data": { - "dataset_name": "MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Equivalent (CoT) on MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.733, - "details": { - "description": "min=0.5, mean=0.733, max=0.859, sum=5.132 (7)", - "tab": "Accuracy", - "MATH - Observed inference time (s)": "{\"description\": \"min=8.668, mean=9.437, max=10.496, sum=66.058 (7)\", \"tab\": \"Efficiency\", \"score\": \"9.436887120006455\"}", - "MATH - # eval": "{\"description\": \"min=30, mean=62.429, max=135, sum=437 (7)\", \"tab\": \"General information\", \"score\": \"62.42857142857143\"}", - "MATH - # train": "{\"description\": \"min=8, mean=8, max=8, sum=56 (7)\", \"tab\": \"General information\", \"score\": \"8.0\"}", - "MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MATH - # prompt tokens": "{\"description\": \"min=937.926, mean=1323.837, max=2246.673, sum=9266.858 (7)\", \"tab\": \"General information\", \"score\": \"1323.836848955025\"}", - "MATH - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=7 (7)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"algebra\", \"counting_and_probability\", \"geometry\", \"intermediate_algebra\", \"number_theory\", \"prealgebra\", \"precalculus\"]", - "level": "\"1\"", - "use_official_examples": "\"False\"", - "use_chain_of_thought": "\"True\"" - } - } - }, - { - "evaluation_name": "GSM8K", - "source_data": { - "dataset_name": "GSM8K", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on GSM8K", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.773, - "details": { - "description": "min=0.773, mean=0.773, max=0.773, sum=0.773 (1)", - "tab": "Accuracy", - "GSM8K - Observed inference time (s)": "{\"description\": \"min=3.406, mean=3.406, max=3.406, sum=3.406 (1)\", \"tab\": \"Efficiency\", \"score\": \"3.405816124200821\"}", - "GSM8K - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "GSM8K - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "GSM8K - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GSM8K - # prompt tokens": "{\"description\": \"min=1130.403, mean=1130.403, max=1130.403, sum=1130.403 (1)\", \"tab\": \"General information\", \"score\": \"1130.403\"}", - "GSM8K - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "LegalBench", - "source_data": { - "dataset_name": "LegalBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on LegalBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.636, - "details": { - "description": "min=0.417, mean=0.636, max=0.926, sum=3.179 (5)", - "tab": "Accuracy", - "LegalBench - Observed inference time (s)": "{\"description\": \"min=0.371, mean=0.789, max=2.33, sum=3.947 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.7894946821991368\"}", - "LegalBench - # eval": "{\"description\": \"min=95, mean=409.4, max=1000, sum=2047 (5)\", \"tab\": \"General information\", \"score\": \"409.4\"}", - "LegalBench - # train": "{\"description\": \"min=4, mean=4.8, max=5, sum=24 (5)\", \"tab\": \"General information\", \"score\": \"4.8\"}", - "LegalBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "LegalBench - # prompt tokens": "{\"description\": \"min=192.453, mean=1542.088, max=6430.714, sum=7710.442 (5)\", \"tab\": \"General information\", \"score\": \"1542.0883229968654\"}", - "LegalBench - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"abercrombie\", \"corporate_lobbying\", \"function_of_decision_section\", \"international_citizenship_questions\", \"proa\"]" - } - } - }, - { - "evaluation_name": "MedQA", - "source_data": { - "dataset_name": "MedQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MedQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.656, - "details": { - "description": "min=0.656, mean=0.656, max=0.656, sum=0.656 (1)", - "tab": "Accuracy", - "MedQA - Observed inference time (s)": "{\"description\": \"min=0.452, mean=0.452, max=0.452, sum=0.452 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.4515474046437925\"}", - "MedQA - # eval": "{\"description\": \"min=503, mean=503, max=503, sum=503 (1)\", \"tab\": \"General information\", \"score\": \"503.0\"}", - "MedQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MedQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MedQA - # prompt tokens": "{\"description\": \"min=1045.485, mean=1045.485, max=1045.485, sum=1045.485 (1)\", \"tab\": \"General information\", \"score\": \"1045.4850894632207\"}", - "MedQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WMT 2014", - "source_data": { - "dataset_name": "WMT 2014", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "BLEU-4 on WMT 2014", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.193, - "details": { - "description": "min=0.129, mean=0.193, max=0.242, sum=0.967 (5)", - "tab": "Accuracy", - "WMT 2014 - Observed inference time (s)": "{\"description\": \"min=0.902, mean=0.92, max=0.952, sum=4.6 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.9200148107330449\"}", - "WMT 2014 - # eval": "{\"description\": \"min=503, mean=568.8, max=832, sum=2844 (5)\", \"tab\": \"General information\", \"score\": \"568.8\"}", - "WMT 2014 - # train": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "WMT 2014 - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WMT 2014 - # prompt tokens": "{\"description\": \"min=108.855, mean=126.657, max=142.373, sum=633.283 (5)\", \"tab\": \"General information\", \"score\": \"126.65662658663405\"}", - "WMT 2014 - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "language_pair": "[\"cs-en\", \"de-en\", \"fr-en\", \"hi-en\", \"ru-en\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_mmlu/qwen_qwen1.5-32b/1774096312.00548", - "retrieved_timestamp": "1774096312.00548", - "source_metadata": { - "source_name": "helm_mmlu", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_mmlu", - "evaluation_results": [ - { - "evaluation_name": "MMLU All Subjects", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU All Subjects", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.744, - "details": { - "description": "min=0.4, mean=0.744, max=0.974, sum=84.853 (114)", - "tab": "Accuracy", - "MMLU All Subjects - Observed inference time (s)": "{\"description\": \"min=0.294, mean=0.413, max=0.973, sum=47.06 (114)\", \"tab\": \"Efficiency\", \"score\": \"0.41280544410672226\"}", - "MMLU All Subjects - # eval": "{\"description\": \"min=100, mean=246.351, max=1534, sum=28084 (114)\", \"tab\": \"General information\", \"score\": \"246.35087719298247\"}", - "MMLU All Subjects - # train": "{\"description\": \"min=5, mean=5, max=5, sum=570 (114)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU All Subjects - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU All Subjects - # prompt tokens": "{\"description\": \"min=269.07, mean=618.598, max=2807.903, sum=70520.198 (114)\", \"tab\": \"General information\", \"score\": \"618.5982315160392\"}", - "MMLU All Subjects - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=114 (114)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_name": "Abstract Algebra", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Abstract Algebra", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4, - "details": { - "description": "min=0.4, mean=0.4, max=0.4, sum=0.8 (2)", - "tab": "Accuracy", - "Abstract Algebra - Observed inference time (s)": "{\"description\": \"min=0.337, mean=0.337, max=0.337, sum=0.675 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.33740817070007323\"}", - "Abstract Algebra - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Abstract Algebra - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Abstract Algebra - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Abstract Algebra - # prompt tokens": "{\"description\": \"min=371.19, mean=371.19, max=371.19, sum=742.38 (2)\", \"tab\": \"General information\", \"score\": \"371.19\"}", - "Abstract Algebra - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"abstract_algebra\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_abstract_algebra\"" - } - } - }, - { - "evaluation_name": "Anatomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Anatomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.644, - "details": { - "description": "min=0.644, mean=0.644, max=0.644, sum=1.289 (2)", - "tab": "Accuracy", - "Anatomy - Observed inference time (s)": "{\"description\": \"min=0.353, mean=0.353, max=0.353, sum=0.706 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.35299032705801503\"}", - "Anatomy - # eval": "{\"description\": \"min=135, mean=135, max=135, sum=270 (2)\", \"tab\": \"General information\", \"score\": \"135.0\"}", - "Anatomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Anatomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Anatomy - # prompt tokens": "{\"description\": \"min=346.978, mean=346.978, max=346.978, sum=693.956 (2)\", \"tab\": \"General information\", \"score\": \"346.97777777777776\"}", - "Anatomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"anatomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_anatomy\"" - } - } - }, - { - "evaluation_name": "College Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on College Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.51, - "details": { - "description": "min=0.51, mean=0.51, max=0.51, sum=1.02 (2)", - "tab": "Accuracy", - "College Chemistry - Observed inference time (s)": "{\"description\": \"min=0.338, mean=0.338, max=0.338, sum=0.677 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.33828389167785644\"}", - "College Biology - Observed inference time (s)": "{\"description\": \"min=0.57, mean=0.57, max=0.57, sum=1.141 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5704119238588545\"}", - "College Computer Science - Observed inference time (s)": "{\"description\": \"min=0.407, mean=0.407, max=0.407, sum=0.813 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4065530586242676\"}", - "College Mathematics - Observed inference time (s)": "{\"description\": \"min=0.683, mean=0.683, max=0.683, sum=1.366 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6829782605171204\"}", - "College Medicine - Observed inference time (s)": "{\"description\": \"min=0.34, mean=0.34, max=0.34, sum=0.68 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.34014028896486137\"}", - "College Physics - Observed inference time (s)": "{\"description\": \"min=0.616, mean=0.616, max=0.616, sum=1.231 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6156594987009086\"}", - "College Chemistry - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Chemistry - # prompt tokens": "{\"description\": \"min=561.25, mean=561.25, max=561.25, sum=1122.5 (2)\", \"tab\": \"General information\", \"score\": \"561.25\"}", - "College Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Biology - # eval": "{\"description\": \"min=144, mean=144, max=144, sum=288 (2)\", \"tab\": \"General information\", \"score\": \"144.0\"}", - "College Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # prompt tokens": "{\"description\": \"min=479.979, mean=479.979, max=479.979, sum=959.958 (2)\", \"tab\": \"General information\", \"score\": \"479.9791666666667\"}", - "College Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # prompt tokens": "{\"description\": \"min=831.58, mean=831.58, max=831.58, sum=1663.16 (2)\", \"tab\": \"General information\", \"score\": \"831.58\"}", - "College Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Mathematics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # prompt tokens": "{\"description\": \"min=600.7, mean=600.7, max=600.7, sum=1201.4 (2)\", \"tab\": \"General information\", \"score\": \"600.7\"}", - "College Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Medicine - # eval": "{\"description\": \"min=173, mean=173, max=173, sum=346 (2)\", \"tab\": \"General information\", \"score\": \"173.0\"}", - "College Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # prompt tokens": "{\"description\": \"min=499.098, mean=499.098, max=499.098, sum=998.197 (2)\", \"tab\": \"General information\", \"score\": \"499.0982658959538\"}", - "College Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Physics - # eval": "{\"description\": \"min=102, mean=102, max=102, sum=204 (2)\", \"tab\": \"General information\", \"score\": \"102.0\"}", - "College Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # prompt tokens": "{\"description\": \"min=509.265, mean=509.265, max=509.265, sum=1018.529 (2)\", \"tab\": \"General information\", \"score\": \"509.2647058823529\"}", - "College Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"college_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_college_physics\"" - } - } - }, - { - "evaluation_name": "Computer Security", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Computer Security", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.77, - "details": { - "description": "min=0.77, mean=0.77, max=0.77, sum=1.54 (2)", - "tab": "Accuracy", - "Computer Security - Observed inference time (s)": "{\"description\": \"min=0.339, mean=0.339, max=0.339, sum=0.678 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3387904930114746\"}", - "Computer Security - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Computer Security - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Computer Security - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Computer Security - # prompt tokens": "{\"description\": \"min=379.64, mean=379.64, max=379.64, sum=759.28 (2)\", \"tab\": \"General information\", \"score\": \"379.64\"}", - "Computer Security - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"computer_security\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_computer_security\"" - } - } - }, - { - "evaluation_name": "Econometrics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Econometrics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.561, - "details": { - "description": "min=0.561, mean=0.561, max=0.561, sum=1.123 (2)", - "tab": "Accuracy", - "Econometrics - Observed inference time (s)": "{\"description\": \"min=0.367, mean=0.367, max=0.367, sum=0.733 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3666987272731045\"}", - "Econometrics - # eval": "{\"description\": \"min=114, mean=114, max=114, sum=228 (2)\", \"tab\": \"General information\", \"score\": \"114.0\"}", - "Econometrics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Econometrics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Econometrics - # prompt tokens": "{\"description\": \"min=620.939, mean=620.939, max=620.939, sum=1241.877 (2)\", \"tab\": \"General information\", \"score\": \"620.938596491228\"}", - "Econometrics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"econometrics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_econometrics\"" - } - } - }, - { - "evaluation_name": "Global Facts", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Global Facts", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.47, - "details": { - "description": "min=0.47, mean=0.47, max=0.47, sum=0.94 (2)", - "tab": "Accuracy", - "Global Facts - Observed inference time (s)": "{\"description\": \"min=0.65, mean=0.65, max=0.65, sum=1.3 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6499223327636718\"}", - "Global Facts - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Global Facts - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Global Facts - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Global Facts - # prompt tokens": "{\"description\": \"min=422.06, mean=422.06, max=422.06, sum=844.12 (2)\", \"tab\": \"General information\", \"score\": \"422.06\"}", - "Global Facts - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"global_facts\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_global_facts\"" - } - } - }, - { - "evaluation_name": "Jurisprudence", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Jurisprudence", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.843, - "details": { - "description": "min=0.843, mean=0.843, max=0.843, sum=1.685 (2)", - "tab": "Accuracy", - "Jurisprudence - Observed inference time (s)": "{\"description\": \"min=0.301, mean=0.301, max=0.301, sum=0.601 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.30060131240774085\"}", - "Jurisprudence - # eval": "{\"description\": \"min=108, mean=108, max=108, sum=216 (2)\", \"tab\": \"General information\", \"score\": \"108.0\"}", - "Jurisprudence - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Jurisprudence - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Jurisprudence - # prompt tokens": "{\"description\": \"min=387.713, mean=387.713, max=387.713, sum=775.426 (2)\", \"tab\": \"General information\", \"score\": \"387.712962962963\"}", - "Jurisprudence - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"jurisprudence\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_jurisprudence\"" - } - } - }, - { - "evaluation_name": "Philosophy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Philosophy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.826, - "details": { - "description": "min=0.826, mean=0.826, max=0.826, sum=1.653 (2)", - "tab": "Accuracy", - "Philosophy - Observed inference time (s)": "{\"description\": \"min=0.297, mean=0.297, max=0.297, sum=0.595 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2974156122115647\"}", - "Philosophy - # eval": "{\"description\": \"min=311, mean=311, max=311, sum=622 (2)\", \"tab\": \"General information\", \"score\": \"311.0\"}", - "Philosophy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Philosophy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Philosophy - # prompt tokens": "{\"description\": \"min=322.09, mean=322.09, max=322.09, sum=644.18 (2)\", \"tab\": \"General information\", \"score\": \"322.09003215434086\"}", - "Philosophy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"philosophy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_philosophy\"" - } - } - }, - { - "evaluation_name": "Professional Psychology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Professional Psychology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.75, - "details": { - "description": "min=0.75, mean=0.75, max=0.75, sum=1.5 (2)", - "tab": "Accuracy", - "Professional Medicine - Observed inference time (s)": "{\"description\": \"min=0.465, mean=0.465, max=0.465, sum=0.93 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.46517644997905283\"}", - "Professional Accounting - Observed inference time (s)": "{\"description\": \"min=0.381, mean=0.381, max=0.381, sum=0.762 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3812122328061584\"}", - "Professional Law - Observed inference time (s)": "{\"description\": \"min=0.649, mean=0.649, max=0.649, sum=1.299 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6492582102642532\"}", - "Professional Psychology - Observed inference time (s)": "{\"description\": \"min=0.388, mean=0.388, max=0.388, sum=0.775 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.38769422676049026\"}", - "Professional Medicine - # eval": "{\"description\": \"min=272, mean=272, max=272, sum=544 (2)\", \"tab\": \"General information\", \"score\": \"272.0\"}", - "Professional Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Medicine - # prompt tokens": "{\"description\": \"min=1118.199, mean=1118.199, max=1118.199, sum=2236.397 (2)\", \"tab\": \"General information\", \"score\": \"1118.1985294117646\"}", - "Professional Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Accounting - # eval": "{\"description\": \"min=282, mean=282, max=282, sum=564 (2)\", \"tab\": \"General information\", \"score\": \"282.0\"}", - "Professional Accounting - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Accounting - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # prompt tokens": "{\"description\": \"min=732.34, mean=732.34, max=732.34, sum=1464.681 (2)\", \"tab\": \"General information\", \"score\": \"732.3404255319149\"}", - "Professional Accounting - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Law - # eval": "{\"description\": \"min=1534, mean=1534, max=1534, sum=3068 (2)\", \"tab\": \"General information\", \"score\": \"1534.0\"}", - "Professional Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # prompt tokens": "{\"description\": \"min=1656.969, mean=1656.969, max=1656.969, sum=3313.939 (2)\", \"tab\": \"General information\", \"score\": \"1656.9693611473272\"}", - "Professional Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Psychology - # eval": "{\"description\": \"min=612, mean=612, max=612, sum=1224 (2)\", \"tab\": \"General information\", \"score\": \"612.0\"}", - "Professional Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # prompt tokens": "{\"description\": \"min=574.417, mean=574.417, max=574.417, sum=1148.833 (2)\", \"tab\": \"General information\", \"score\": \"574.4166666666666\"}", - "Professional Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"professional_psychology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_professional_psychology\"" - } - } - }, - { - "evaluation_name": "Us Foreign Policy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Us Foreign Policy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.91, - "details": { - "description": "min=0.91, mean=0.91, max=0.91, sum=1.82 (2)", - "tab": "Accuracy", - "Us Foreign Policy - Observed inference time (s)": "{\"description\": \"min=0.343, mean=0.343, max=0.343, sum=0.686 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3429260540008545\"}", - "Us Foreign Policy - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Us Foreign Policy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Us Foreign Policy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Us Foreign Policy - # prompt tokens": "{\"description\": \"min=421.16, mean=421.16, max=421.16, sum=842.32 (2)\", \"tab\": \"General information\", \"score\": \"421.16\"}", - "Us Foreign Policy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"us_foreign_policy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_us_foreign_policy\"" - } - } - }, - { - "evaluation_name": "Astronomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Astronomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.855, - "details": { - "description": "min=0.855, mean=0.855, max=0.855, sum=1.711 (2)", - "tab": "Accuracy", - "Astronomy - Observed inference time (s)": "{\"description\": \"min=0.337, mean=0.337, max=0.337, sum=0.674 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.33687377132867513\"}", - "Astronomy - # eval": "{\"description\": \"min=152, mean=152, max=152, sum=304 (2)\", \"tab\": \"General information\", \"score\": \"152.0\"}", - "Astronomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Astronomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Astronomy - # prompt tokens": "{\"description\": \"min=582.849, mean=582.849, max=582.849, sum=1165.697 (2)\", \"tab\": \"General information\", \"score\": \"582.8486842105264\"}", - "Astronomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"astronomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_astronomy\"" - } - } - }, - { - "evaluation_name": "Business Ethics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Business Ethics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.77, - "details": { - "description": "min=0.77, mean=0.77, max=0.77, sum=1.54 (2)", - "tab": "Accuracy", - "Business Ethics - Observed inference time (s)": "{\"description\": \"min=0.356, mean=0.356, max=0.356, sum=0.713 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3564377498626709\"}", - "Business Ethics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Business Ethics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Business Ethics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Business Ethics - # prompt tokens": "{\"description\": \"min=562.87, mean=562.87, max=562.87, sum=1125.74 (2)\", \"tab\": \"General information\", \"score\": \"562.87\"}", - "Business Ethics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"business_ethics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_business_ethics\"" - } - } - }, - { - "evaluation_name": "Clinical Knowledge", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Clinical Knowledge", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.781, - "details": { - "description": "min=0.781, mean=0.781, max=0.781, sum=1.562 (2)", - "tab": "Accuracy", - "Clinical Knowledge - Observed inference time (s)": "{\"description\": \"min=0.319, mean=0.319, max=0.319, sum=0.638 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3190377280397235\"}", - "Clinical Knowledge - # eval": "{\"description\": \"min=265, mean=265, max=265, sum=530 (2)\", \"tab\": \"General information\", \"score\": \"265.0\"}", - "Clinical Knowledge - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Clinical Knowledge - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Clinical Knowledge - # prompt tokens": "{\"description\": \"min=393.623, mean=393.623, max=393.623, sum=787.245 (2)\", \"tab\": \"General information\", \"score\": \"393.62264150943395\"}", - "Clinical Knowledge - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"clinical_knowledge\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_clinical_knowledge\"" - } - } - }, - { - "evaluation_name": "Conceptual Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Conceptual Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.766, - "details": { - "description": "min=0.766, mean=0.766, max=0.766, sum=1.532 (2)", - "tab": "Accuracy", - "Conceptual Physics - Observed inference time (s)": "{\"description\": \"min=0.436, mean=0.436, max=0.436, sum=0.872 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4358475421337371\"}", - "Conceptual Physics - # eval": "{\"description\": \"min=235, mean=235, max=235, sum=470 (2)\", \"tab\": \"General information\", \"score\": \"235.0\"}", - "Conceptual Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Conceptual Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Conceptual Physics - # prompt tokens": "{\"description\": \"min=298.494, mean=298.494, max=298.494, sum=596.987 (2)\", \"tab\": \"General information\", \"score\": \"298.4936170212766\"}", - "Conceptual Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"conceptual_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_conceptual_physics\"" - } - } - }, - { - "evaluation_name": "Electrical Engineering", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Electrical Engineering", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.731, - "details": { - "description": "min=0.731, mean=0.731, max=0.731, sum=1.462 (2)", - "tab": "Accuracy", - "Electrical Engineering - Observed inference time (s)": "{\"description\": \"min=0.321, mean=0.321, max=0.321, sum=0.642 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.32112578523570096\"}", - "Electrical Engineering - # eval": "{\"description\": \"min=145, mean=145, max=145, sum=290 (2)\", \"tab\": \"General information\", \"score\": \"145.0\"}", - "Electrical Engineering - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Electrical Engineering - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Electrical Engineering - # prompt tokens": "{\"description\": \"min=456.8, mean=456.8, max=456.8, sum=913.6 (2)\", \"tab\": \"General information\", \"score\": \"456.8\"}", - "Electrical Engineering - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"electrical_engineering\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_electrical_engineering\"" - } - } - }, - { - "evaluation_name": "Elementary Mathematics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Elementary Mathematics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.685, - "details": { - "description": "min=0.685, mean=0.685, max=0.685, sum=1.37 (2)", - "tab": "Accuracy", - "Elementary Mathematics - Observed inference time (s)": "{\"description\": \"min=0.352, mean=0.352, max=0.352, sum=0.705 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3522766809614878\"}", - "Elementary Mathematics - # eval": "{\"description\": \"min=378, mean=378, max=378, sum=756 (2)\", \"tab\": \"General information\", \"score\": \"378.0\"}", - "Elementary Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Elementary Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Elementary Mathematics - # prompt tokens": "{\"description\": \"min=570.119, mean=570.119, max=570.119, sum=1140.238 (2)\", \"tab\": \"General information\", \"score\": \"570.1190476190476\"}", - "Elementary Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"elementary_mathematics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_elementary_mathematics\"" - } - } - }, - { - "evaluation_name": "Formal Logic", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Formal Logic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.524, - "details": { - "description": "min=0.524, mean=0.524, max=0.524, sum=1.048 (2)", - "tab": "Accuracy", - "Formal Logic - Observed inference time (s)": "{\"description\": \"min=0.37, mean=0.37, max=0.37, sum=0.739 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3697236606052944\"}", - "Formal Logic - # eval": "{\"description\": \"min=126, mean=126, max=126, sum=252 (2)\", \"tab\": \"General information\", \"score\": \"126.0\"}", - "Formal Logic - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Formal Logic - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Formal Logic - # prompt tokens": "{\"description\": \"min=597.667, mean=597.667, max=597.667, sum=1195.333 (2)\", \"tab\": \"General information\", \"score\": \"597.6666666666666\"}", - "Formal Logic - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"formal_logic\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_formal_logic\"" - } - } - }, - { - "evaluation_name": "High School World History", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on High School World History", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.869, - "details": { - "description": "min=0.869, mean=0.869, max=0.869, sum=1.738 (2)", - "tab": "Accuracy", - "High School Biology - Observed inference time (s)": "{\"description\": \"min=0.32, mean=0.32, max=0.32, sum=0.639 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3195470579208866\"}", - "High School Chemistry - Observed inference time (s)": "{\"description\": \"min=0.369, mean=0.369, max=0.369, sum=0.739 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.36928989969450854\"}", - "High School Computer Science - Observed inference time (s)": "{\"description\": \"min=0.724, mean=0.724, max=0.724, sum=1.448 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7240336751937866\"}", - "High School European History - Observed inference time (s)": "{\"description\": \"min=0.973, mean=0.973, max=0.973, sum=1.946 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9729607683239561\"}", - "High School Geography - Observed inference time (s)": "{\"description\": \"min=0.307, mean=0.307, max=0.307, sum=0.614 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.30711602562605733\"}", - "High School Government And Politics - Observed inference time (s)": "{\"description\": \"min=0.338, mean=0.338, max=0.338, sum=0.675 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3376439371257248\"}", - "High School Macroeconomics - Observed inference time (s)": "{\"description\": \"min=0.41, mean=0.41, max=0.41, sum=0.82 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.410240764495654\"}", - "High School Mathematics - Observed inference time (s)": "{\"description\": \"min=0.363, mean=0.363, max=0.363, sum=0.725 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.36270895887304233\"}", - "High School Microeconomics - Observed inference time (s)": "{\"description\": \"min=0.314, mean=0.314, max=0.314, sum=0.629 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3144632788265453\"}", - "High School Physics - Observed inference time (s)": "{\"description\": \"min=0.368, mean=0.368, max=0.368, sum=0.736 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3679169850633634\"}", - "High School Psychology - Observed inference time (s)": "{\"description\": \"min=0.452, mean=0.452, max=0.452, sum=0.903 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.45166520109964076\"}", - "High School Statistics - Observed inference time (s)": "{\"description\": \"min=0.378, mean=0.378, max=0.378, sum=0.757 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.37830896068502357\"}", - "High School US History - Observed inference time (s)": "{\"description\": \"min=0.888, mean=0.888, max=0.888, sum=1.776 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8882208957391626\"}", - "High School World History - Observed inference time (s)": "{\"description\": \"min=0.551, mean=0.551, max=0.551, sum=1.102 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5509252004985568\"}", - "High School Biology - # eval": "{\"description\": \"min=310, mean=310, max=310, sum=620 (2)\", \"tab\": \"General information\", \"score\": \"310.0\"}", - "High School Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Biology - # prompt tokens": "{\"description\": \"min=506.916, mean=506.916, max=506.916, sum=1013.832 (2)\", \"tab\": \"General information\", \"score\": \"506.9161290322581\"}", - "High School Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Chemistry - # eval": "{\"description\": \"min=203, mean=203, max=203, sum=406 (2)\", \"tab\": \"General information\", \"score\": \"203.0\"}", - "High School Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # prompt tokens": "{\"description\": \"min=510.261, mean=510.261, max=510.261, sum=1020.522 (2)\", \"tab\": \"General information\", \"score\": \"510.2610837438424\"}", - "High School Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "High School Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # prompt tokens": "{\"description\": \"min=871.46, mean=871.46, max=871.46, sum=1742.92 (2)\", \"tab\": \"General information\", \"score\": \"871.46\"}", - "High School Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School European History - # eval": "{\"description\": \"min=165, mean=165, max=165, sum=330 (2)\", \"tab\": \"General information\", \"score\": \"165.0\"}", - "High School European History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School European History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # prompt tokens": "{\"description\": \"min=2807.903, mean=2807.903, max=2807.903, sum=5615.806 (2)\", \"tab\": \"General information\", \"score\": \"2807.9030303030304\"}", - "High School European History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Geography - # eval": "{\"description\": \"min=198, mean=198, max=198, sum=396 (2)\", \"tab\": \"General information\", \"score\": \"198.0\"}", - "High School Geography - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Geography - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # prompt tokens": "{\"description\": \"min=365.217, mean=365.217, max=365.217, sum=730.434 (2)\", \"tab\": \"General information\", \"score\": \"365.2171717171717\"}", - "High School Geography - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Government And Politics - # eval": "{\"description\": \"min=193, mean=193, max=193, sum=386 (2)\", \"tab\": \"General information\", \"score\": \"193.0\"}", - "High School Government And Politics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Government And Politics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # prompt tokens": "{\"description\": \"min=460.311, mean=460.311, max=460.311, sum=920.622 (2)\", \"tab\": \"General information\", \"score\": \"460.31088082901556\"}", - "High School Government And Politics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Macroeconomics - # eval": "{\"description\": \"min=390, mean=390, max=390, sum=780 (2)\", \"tab\": \"General information\", \"score\": \"390.0\"}", - "High School Macroeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Macroeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # prompt tokens": "{\"description\": \"min=367.349, mean=367.349, max=367.349, sum=734.697 (2)\", \"tab\": \"General information\", \"score\": \"367.34871794871793\"}", - "High School Macroeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Mathematics - # eval": "{\"description\": \"min=270, mean=270, max=270, sum=540 (2)\", \"tab\": \"General information\", \"score\": \"270.0\"}", - "High School Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # prompt tokens": "{\"description\": \"min=558.326, mean=558.326, max=558.326, sum=1116.652 (2)\", \"tab\": \"General information\", \"score\": \"558.325925925926\"}", - "High School Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Microeconomics - # eval": "{\"description\": \"min=238, mean=238, max=238, sum=476 (2)\", \"tab\": \"General information\", \"score\": \"238.0\"}", - "High School Microeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Microeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # prompt tokens": "{\"description\": \"min=395.277, mean=395.277, max=395.277, sum=790.555 (2)\", \"tab\": \"General information\", \"score\": \"395.2773109243698\"}", - "High School Microeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Physics - # eval": "{\"description\": \"min=151, mean=151, max=151, sum=302 (2)\", \"tab\": \"General information\", \"score\": \"151.0\"}", - "High School Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # prompt tokens": "{\"description\": \"min=573.536, mean=573.536, max=573.536, sum=1147.073 (2)\", \"tab\": \"General information\", \"score\": \"573.5364238410596\"}", - "High School Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Psychology - # eval": "{\"description\": \"min=545, mean=545, max=545, sum=1090 (2)\", \"tab\": \"General information\", \"score\": \"545.0\"}", - "High School Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # prompt tokens": "{\"description\": \"min=488.521, mean=488.521, max=488.521, sum=977.042 (2)\", \"tab\": \"General information\", \"score\": \"488.52110091743117\"}", - "High School Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Statistics - # eval": "{\"description\": \"min=216, mean=216, max=216, sum=432 (2)\", \"tab\": \"General information\", \"score\": \"216.0\"}", - "High School Statistics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Statistics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # prompt tokens": "{\"description\": \"min=823.477, mean=823.477, max=823.477, sum=1646.954 (2)\", \"tab\": \"General information\", \"score\": \"823.4768518518518\"}", - "High School Statistics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School US History - # eval": "{\"description\": \"min=204, mean=204, max=204, sum=408 (2)\", \"tab\": \"General information\", \"score\": \"204.0\"}", - "High School US History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School US History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # prompt tokens": "{\"description\": \"min=2230.176, mean=2230.176, max=2230.176, sum=4460.353 (2)\", \"tab\": \"General information\", \"score\": \"2230.176470588235\"}", - "High School US History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School World History - # eval": "{\"description\": \"min=237, mean=237, max=237, sum=474 (2)\", \"tab\": \"General information\", \"score\": \"237.0\"}", - "High School World History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School World History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # prompt tokens": "{\"description\": \"min=1441.354, mean=1441.354, max=1441.354, sum=2882.709 (2)\", \"tab\": \"General information\", \"score\": \"1441.3544303797469\"}", - "High School World History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"high_school_world_history\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_high_school_world_history\"" - } - } - }, - { - "evaluation_name": "Human Sexuality", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Human Sexuality", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.847, - "details": { - "description": "min=0.847, mean=0.847, max=0.847, sum=1.695 (2)", - "tab": "Accuracy", - "Human Aging - Observed inference time (s)": "{\"description\": \"min=0.314, mean=0.314, max=0.314, sum=0.627 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.31371782071921855\"}", - "Human Sexuality - Observed inference time (s)": "{\"description\": \"min=0.323, mean=0.323, max=0.323, sum=0.647 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.32332972897828083\"}", - "Human Aging - # eval": "{\"description\": \"min=223, mean=223, max=223, sum=446 (2)\", \"tab\": \"General information\", \"score\": \"223.0\"}", - "Human Aging - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Aging - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Aging - # prompt tokens": "{\"description\": \"min=315.121, mean=315.121, max=315.121, sum=630.242 (2)\", \"tab\": \"General information\", \"score\": \"315.1210762331838\"}", - "Human Aging - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Human Sexuality - # eval": "{\"description\": \"min=131, mean=131, max=131, sum=262 (2)\", \"tab\": \"General information\", \"score\": \"131.0\"}", - "Human Sexuality - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Sexuality - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # prompt tokens": "{\"description\": \"min=334.504, mean=334.504, max=334.504, sum=669.008 (2)\", \"tab\": \"General information\", \"score\": \"334.5038167938931\"}", - "Human Sexuality - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"human_sexuality\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_human_sexuality\"" - } - } - }, - { - "evaluation_name": "International Law", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on International Law", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.884, - "details": { - "description": "min=0.884, mean=0.884, max=0.884, sum=1.769 (2)", - "tab": "Accuracy", - "International Law - Observed inference time (s)": "{\"description\": \"min=0.382, mean=0.382, max=0.382, sum=0.765 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.38232671130787244\"}", - "International Law - # eval": "{\"description\": \"min=121, mean=121, max=121, sum=242 (2)\", \"tab\": \"General information\", \"score\": \"121.0\"}", - "International Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "International Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "International Law - # prompt tokens": "{\"description\": \"min=633.579, mean=633.579, max=633.579, sum=1267.157 (2)\", \"tab\": \"General information\", \"score\": \"633.5785123966942\"}", - "International Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"international_law\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_international_law\"" - } - } - }, - { - "evaluation_name": "Logical Fallacies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Logical Fallacies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.822, - "details": { - "description": "min=0.822, mean=0.822, max=0.822, sum=1.644 (2)", - "tab": "Accuracy", - "Logical Fallacies - Observed inference time (s)": "{\"description\": \"min=0.313, mean=0.313, max=0.313, sum=0.625 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.31269068220641716\"}", - "Logical Fallacies - # eval": "{\"description\": \"min=163, mean=163, max=163, sum=326 (2)\", \"tab\": \"General information\", \"score\": \"163.0\"}", - "Logical Fallacies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Logical Fallacies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Logical Fallacies - # prompt tokens": "{\"description\": \"min=442.632, mean=442.632, max=442.632, sum=885.264 (2)\", \"tab\": \"General information\", \"score\": \"442.6319018404908\"}", - "Logical Fallacies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"logical_fallacies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_logical_fallacies\"" - } - } - }, - { - "evaluation_name": "Machine Learning", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Machine Learning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.616, - "details": { - "description": "min=0.616, mean=0.616, max=0.616, sum=1.232 (2)", - "tab": "Accuracy", - "Machine Learning - Observed inference time (s)": "{\"description\": \"min=0.359, mean=0.359, max=0.359, sum=0.719 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3593791680676596\"}", - "Machine Learning - # eval": "{\"description\": \"min=112, mean=112, max=112, sum=224 (2)\", \"tab\": \"General information\", \"score\": \"112.0\"}", - "Machine Learning - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Machine Learning - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Machine Learning - # prompt tokens": "{\"description\": \"min=674.848, mean=674.848, max=674.848, sum=1349.696 (2)\", \"tab\": \"General information\", \"score\": \"674.8482142857143\"}", - "Machine Learning - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"machine_learning\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_machine_learning\"" - } - } - }, - { - "evaluation_name": "Management", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Management", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.874, - "details": { - "description": "min=0.874, mean=0.874, max=0.874, sum=1.748 (2)", - "tab": "Accuracy", - "Management - Observed inference time (s)": "{\"description\": \"min=0.633, mean=0.633, max=0.633, sum=1.265 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6326094113507317\"}", - "Management - # eval": "{\"description\": \"min=103, mean=103, max=103, sum=206 (2)\", \"tab\": \"General information\", \"score\": \"103.0\"}", - "Management - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Management - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Management - # prompt tokens": "{\"description\": \"min=276.854, mean=276.854, max=276.854, sum=553.709 (2)\", \"tab\": \"General information\", \"score\": \"276.8543689320388\"}", - "Management - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"management\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_management\"" - } - } - }, - { - "evaluation_name": "Marketing", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Marketing", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.936, - "details": { - "description": "min=0.936, mean=0.936, max=0.936, sum=1.872 (2)", - "tab": "Accuracy", - "Marketing - Observed inference time (s)": "{\"description\": \"min=0.328, mean=0.328, max=0.328, sum=0.655 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3277416534912892\"}", - "Marketing - # eval": "{\"description\": \"min=234, mean=234, max=234, sum=468 (2)\", \"tab\": \"General information\", \"score\": \"234.0\"}", - "Marketing - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Marketing - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Marketing - # prompt tokens": "{\"description\": \"min=397.415, mean=397.415, max=397.415, sum=794.829 (2)\", \"tab\": \"General information\", \"score\": \"397.4145299145299\"}", - "Marketing - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"marketing\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_marketing\"" - } - } - }, - { - "evaluation_name": "Medical Genetics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Medical Genetics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.85, - "details": { - "description": "min=0.85, mean=0.85, max=0.85, sum=1.7 (2)", - "tab": "Accuracy", - "Medical Genetics - Observed inference time (s)": "{\"description\": \"min=0.294, mean=0.294, max=0.294, sum=0.588 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2937913846969604\"}", - "Medical Genetics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Medical Genetics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Medical Genetics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Medical Genetics - # prompt tokens": "{\"description\": \"min=335.35, mean=335.35, max=335.35, sum=670.7 (2)\", \"tab\": \"General information\", \"score\": \"335.35\"}", - "Medical Genetics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"medical_genetics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_medical_genetics\"" - } - } - }, - { - "evaluation_name": "Miscellaneous", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Miscellaneous", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.884, - "details": { - "description": "min=0.884, mean=0.884, max=0.884, sum=1.768 (2)", - "tab": "Accuracy", - "Miscellaneous - Observed inference time (s)": "{\"description\": \"min=0.347, mean=0.347, max=0.347, sum=0.693 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.34673521040652144\"}", - "Miscellaneous - # eval": "{\"description\": \"min=783, mean=783, max=783, sum=1566 (2)\", \"tab\": \"General information\", \"score\": \"783.0\"}", - "Miscellaneous - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Miscellaneous - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Miscellaneous - # prompt tokens": "{\"description\": \"min=296.7, mean=296.7, max=296.7, sum=593.4 (2)\", \"tab\": \"General information\", \"score\": \"296.6998722860792\"}", - "Miscellaneous - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"miscellaneous\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_miscellaneous\"" - } - } - }, - { - "evaluation_name": "Moral Scenarios", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Moral Scenarios", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.545, - "details": { - "description": "min=0.545, mean=0.545, max=0.545, sum=1.091 (2)", - "tab": "Accuracy", - "Moral Disputes - Observed inference time (s)": "{\"description\": \"min=0.334, mean=0.334, max=0.334, sum=0.667 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3335799164854722\"}", - "Moral Scenarios - Observed inference time (s)": "{\"description\": \"min=0.396, mean=0.396, max=0.396, sum=0.792 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3961469775471607\"}", - "Moral Disputes - # eval": "{\"description\": \"min=346, mean=346, max=346, sum=692 (2)\", \"tab\": \"General information\", \"score\": \"346.0\"}", - "Moral Disputes - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Disputes - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Disputes - # prompt tokens": "{\"description\": \"min=469.182, mean=469.182, max=469.182, sum=938.364 (2)\", \"tab\": \"General information\", \"score\": \"469.1820809248555\"}", - "Moral Disputes - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Moral Scenarios - # eval": "{\"description\": \"min=895, mean=895, max=895, sum=1790 (2)\", \"tab\": \"General information\", \"score\": \"895.0\"}", - "Moral Scenarios - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Scenarios - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # prompt tokens": "{\"description\": \"min=661.494, mean=661.494, max=661.494, sum=1322.988 (2)\", \"tab\": \"General information\", \"score\": \"661.4938547486033\"}", - "Moral Scenarios - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"moral_scenarios\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_moral_scenarios\"" - } - } - }, - { - "evaluation_name": "Nutrition", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Nutrition", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.81, - "details": { - "description": "min=0.81, mean=0.81, max=0.81, sum=1.621 (2)", - "tab": "Accuracy", - "Nutrition - Observed inference time (s)": "{\"description\": \"min=0.338, mean=0.338, max=0.338, sum=0.676 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.33816951162674846\"}", - "Nutrition - # eval": "{\"description\": \"min=306, mean=306, max=306, sum=612 (2)\", \"tab\": \"General information\", \"score\": \"306.0\"}", - "Nutrition - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Nutrition - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Nutrition - # prompt tokens": "{\"description\": \"min=592.637, mean=592.637, max=592.637, sum=1185.275 (2)\", \"tab\": \"General information\", \"score\": \"592.6372549019608\"}", - "Nutrition - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"nutrition\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_nutrition\"" - } - } - }, - { - "evaluation_name": "Prehistory", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Prehistory", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.83, - "details": { - "description": "min=0.83, mean=0.83, max=0.83, sum=1.66 (2)", - "tab": "Accuracy", - "Prehistory - Observed inference time (s)": "{\"description\": \"min=0.327, mean=0.327, max=0.327, sum=0.654 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3270495865080092\"}", - "Prehistory - # eval": "{\"description\": \"min=324, mean=324, max=324, sum=648 (2)\", \"tab\": \"General information\", \"score\": \"324.0\"}", - "Prehistory - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Prehistory - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Prehistory - # prompt tokens": "{\"description\": \"min=521.364, mean=521.364, max=521.364, sum=1042.728 (2)\", \"tab\": \"General information\", \"score\": \"521.3641975308642\"}", - "Prehistory - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"prehistory\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_prehistory\"" - } - } - }, - { - "evaluation_name": "Public Relations", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Public Relations", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.664, - "details": { - "description": "min=0.664, mean=0.664, max=0.664, sum=1.327 (2)", - "tab": "Accuracy", - "Public Relations - Observed inference time (s)": "{\"description\": \"min=0.305, mean=0.305, max=0.305, sum=0.609 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3046790404753251\"}", - "Public Relations - # eval": "{\"description\": \"min=110, mean=110, max=110, sum=220 (2)\", \"tab\": \"General information\", \"score\": \"110.0\"}", - "Public Relations - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Public Relations - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Public Relations - # prompt tokens": "{\"description\": \"min=401.427, mean=401.427, max=401.427, sum=802.855 (2)\", \"tab\": \"General information\", \"score\": \"401.42727272727274\"}", - "Public Relations - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"public_relations\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_public_relations\"" - } - } - }, - { - "evaluation_name": "Security Studies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Security Studies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.829, - "details": { - "description": "min=0.829, mean=0.829, max=0.829, sum=1.657 (2)", - "tab": "Accuracy", - "Security Studies - Observed inference time (s)": "{\"description\": \"min=0.478, mean=0.478, max=0.478, sum=0.956 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.47783534575481806\"}", - "Security Studies - # eval": "{\"description\": \"min=245, mean=245, max=245, sum=490 (2)\", \"tab\": \"General information\", \"score\": \"245.0\"}", - "Security Studies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Security Studies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Security Studies - # prompt tokens": "{\"description\": \"min=1159.931, mean=1159.931, max=1159.931, sum=2319.861 (2)\", \"tab\": \"General information\", \"score\": \"1159.930612244898\"}", - "Security Studies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"security_studies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_security_studies\"" - } - } - }, - { - "evaluation_name": "Sociology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Sociology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.881, - "details": { - "description": "min=0.881, mean=0.881, max=0.881, sum=1.761 (2)", - "tab": "Accuracy", - "Sociology - Observed inference time (s)": "{\"description\": \"min=0.341, mean=0.341, max=0.341, sum=0.681 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3407213664173487\"}", - "Sociology - # eval": "{\"description\": \"min=201, mean=201, max=201, sum=402 (2)\", \"tab\": \"General information\", \"score\": \"201.0\"}", - "Sociology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Sociology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Sociology - # prompt tokens": "{\"description\": \"min=443.1, mean=443.1, max=443.1, sum=886.199 (2)\", \"tab\": \"General information\", \"score\": \"443.0995024875622\"}", - "Sociology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"sociology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_sociology\"" - } - } - }, - { - "evaluation_name": "Virology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Virology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.578, - "details": { - "description": "min=0.578, mean=0.578, max=0.578, sum=1.157 (2)", - "tab": "Accuracy", - "Virology - Observed inference time (s)": "{\"description\": \"min=0.329, mean=0.329, max=0.329, sum=0.658 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3289937297981906\"}", - "Virology - # eval": "{\"description\": \"min=166, mean=166, max=166, sum=332 (2)\", \"tab\": \"General information\", \"score\": \"166.0\"}", - "Virology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Virology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Virology - # prompt tokens": "{\"description\": \"min=336.819, mean=336.819, max=336.819, sum=673.639 (2)\", \"tab\": \"General information\", \"score\": \"336.8192771084337\"}", - "Virology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"virology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_virology\"" - } - } - }, - { - "evaluation_name": "World Religions", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on World Religions", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.854, - "details": { - "description": "min=0.854, mean=0.854, max=0.854, sum=1.708 (2)", - "tab": "Accuracy", - "World Religions - Observed inference time (s)": "{\"description\": \"min=0.32, mean=0.32, max=0.32, sum=0.64 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.31992746933161864\"}", - "World Religions - # eval": "{\"description\": \"min=171, mean=171, max=171, sum=342 (2)\", \"tab\": \"General information\", \"score\": \"171.0\"}", - "World Religions - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "World Religions - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "World Religions - # prompt tokens": "{\"description\": \"min=269.07, mean=269.07, max=269.07, sum=538.14 (2)\", \"tab\": \"General information\", \"score\": \"269.0701754385965\"}", - "World Religions - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"world_religions\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_world_religions\"" - } - } - }, - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.624, - "details": { - "description": "", - "tab": "Efficiency" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_id": "hfopenllm_v2/Qwen_Qwen1.5-32B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3297 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5715 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3029 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3297 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4278 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.45 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qwen_qwen1.5-4b-chat.json b/data/models/qwen_qwen1.5-4b-chat.json deleted file mode 100644 index 94cbecbc4be02342905002a7890d272f9b224795..0000000000000000000000000000000000000000 --- a/data/models/qwen_qwen1.5-4b-chat.json +++ /dev/null @@ -1,277 +0,0 @@ -{ - "model_info": { - "name": "Qwen/Qwen1.5-4B-Chat", - "id": "Qwen/Qwen1.5-4B-Chat", - "developer": "Qwen", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.95" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Qwen_Qwen1.5-4B-Chat/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3157 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4006 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0279 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2668 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3978 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2396 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "reward-bench/Qwen_Qwen1.5-4B-Chat/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5477 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3883 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6272 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5568 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6689 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.447 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qwen_qwen1.5-4b.json b/data/models/qwen_qwen1.5-4b.json deleted file mode 100644 index 612fa57891176e9acce0d88cf489c466105d3fc3..0000000000000000000000000000000000000000 --- a/data/models/qwen_qwen1.5-4b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen1.5-4B", - "id": "Qwen/Qwen1.5-4B", - "developer": "Qwen", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.95" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Qwen_Qwen1.5-4B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2445 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4054 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0529 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2768 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3604 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.246 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qwen_qwen1.5-72b-chat.json b/data/models/qwen_qwen1.5-72b-chat.json deleted file mode 100644 index 0b19e67202c9d26632d190377f7555d3e1401688..0000000000000000000000000000000000000000 --- a/data/models/qwen_qwen1.5-72b-chat.json +++ /dev/null @@ -1,144 +0,0 @@ -{ - "model_info": { - "name": "Qwen/Qwen1.5-72B-Chat", - "id": "Qwen/Qwen1.5-72B-Chat", - "developer": "Qwen", - "additional_details": { - "model_type": "DPO" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/Qwen_Qwen1.5-72B-Chat/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6723 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6229 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6601 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6757 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8554 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4226 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qwen_qwen1.5-72b.json b/data/models/qwen_qwen1.5-72b.json deleted file mode 100644 index 72e1788cbfc18a766d5324e6f012a102dd0adf96..0000000000000000000000000000000000000000 --- a/data/models/qwen_qwen1.5-72b.json +++ /dev/null @@ -1,1901 +0,0 @@ -{ - "model_info": { - "name": "Qwen1.5 72B", - "id": "qwen/qwen1.5-72b", - "developer": "qwen", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_lite/qwen_qwen1.5-72b/1774096306.427425", - "retrieved_timestamp": "1774096306.427425", - "source_metadata": { - "source_name": "helm_lite", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_lite", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_lite", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.608, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.3881398252184769\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.601, - "details": { - "description": "min=0.601, mean=0.601, max=0.601, sum=0.601 (1)", - "tab": "Accuracy", - "NarrativeQA - Observed inference time (s)": "{\"description\": \"min=2.437, mean=2.437, max=2.437, sum=2.437 (1)\", \"tab\": \"Efficiency\", \"score\": \"2.4371175302586083\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=4.994, mean=4.994, max=4.994, sum=4.994 (1)\", \"tab\": \"General information\", \"score\": \"4.994366197183099\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=3465.859, mean=3465.859, max=3465.859, sum=3465.859 (1)\", \"tab\": \"General information\", \"score\": \"3465.8591549295775\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (closed-book)", - "source_data": { - "dataset_name": "NaturalQuestions (closed-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (closed-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.417, - "details": { - "description": "min=0.417, mean=0.417, max=0.417, sum=0.417 (1)", - "tab": "Accuracy", - "NaturalQuestions (open-book) - Observed inference time (s)": "{\"description\": \"min=1.421, mean=1.421, max=1.421, sum=1.421 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.4208379020690918\"}", - "NaturalQuestions (closed-book) - Observed inference time (s)": "{\"description\": \"min=0.577, mean=0.577, max=0.577, sum=0.577 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.5770996954441071\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.863, mean=4.863, max=4.863, sum=4.863 (1)\", \"tab\": \"General information\", \"score\": \"4.863\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.022, mean=0.022, max=0.022, sum=0.022 (1)\", \"tab\": \"General information\", \"score\": \"0.022\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1846.221, mean=1846.221, max=1846.221, sum=1846.221 (1)\", \"tab\": \"General information\", \"score\": \"1846.221\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=119.262, mean=119.262, max=119.262, sum=119.262 (1)\", \"tab\": \"General information\", \"score\": \"119.262\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "mode": "\"closedbook\"" - } - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.93, - "details": { - "description": "min=0.93, mean=0.93, max=0.93, sum=0.93 (1)", - "tab": "Accuracy", - "OpenbookQA - Observed inference time (s)": "{\"description\": \"min=0.338, mean=0.338, max=0.338, sum=0.338 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.3381467695236206\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=242.846, mean=242.846, max=242.846, sum=242.846 (1)\", \"tab\": \"General information\", \"score\": \"242.846\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "dataset": "\"openbookqa\"", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.647, - "details": { - "description": "min=0.44, mean=0.647, max=0.94, sum=3.234 (5)", - "tab": "Accuracy", - "MMLU - Observed inference time (s)": "{\"description\": \"min=0.338, mean=0.364, max=0.396, sum=1.819 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.3638015921659637\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=371.19, mean=470.836, max=620.939, sum=2354.179 (5)\", \"tab\": \"General information\", \"score\": \"470.8357192982456\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"college_chemistry\", \"computer_security\", \"econometrics\", \"us_foreign_policy\"]", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MATH", - "source_data": { - "dataset_name": "MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Equivalent (CoT) on MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.683, - "details": { - "description": "min=0.6, mean=0.683, max=0.763, sum=4.784 (7)", - "tab": "Accuracy", - "MATH - Observed inference time (s)": "{\"description\": \"min=10.776, mean=11.813, max=12.91, sum=82.688 (7)\", \"tab\": \"Efficiency\", \"score\": \"11.812623854443027\"}", - "MATH - # eval": "{\"description\": \"min=30, mean=62.429, max=135, sum=437 (7)\", \"tab\": \"General information\", \"score\": \"62.42857142857143\"}", - "MATH - # train": "{\"description\": \"min=8, mean=8, max=8, sum=56 (7)\", \"tab\": \"General information\", \"score\": \"8.0\"}", - "MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MATH - # prompt tokens": "{\"description\": \"min=937.926, mean=1323.837, max=2246.673, sum=9266.858 (7)\", \"tab\": \"General information\", \"score\": \"1323.836848955025\"}", - "MATH - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=7 (7)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"algebra\", \"counting_and_probability\", \"geometry\", \"intermediate_algebra\", \"number_theory\", \"prealgebra\", \"precalculus\"]", - "level": "\"1\"", - "use_official_examples": "\"False\"", - "use_chain_of_thought": "\"True\"" - } - } - }, - { - "evaluation_name": "GSM8K", - "source_data": { - "dataset_name": "GSM8K", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on GSM8K", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.799, - "details": { - "description": "min=0.799, mean=0.799, max=0.799, sum=0.799 (1)", - "tab": "Accuracy", - "GSM8K - Observed inference time (s)": "{\"description\": \"min=4.587, mean=4.587, max=4.587, sum=4.587 (1)\", \"tab\": \"Efficiency\", \"score\": \"4.5866835827827455\"}", - "GSM8K - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "GSM8K - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "GSM8K - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GSM8K - # prompt tokens": "{\"description\": \"min=1130.403, mean=1130.403, max=1130.403, sum=1130.403 (1)\", \"tab\": \"General information\", \"score\": \"1130.403\"}", - "GSM8K - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "LegalBench", - "source_data": { - "dataset_name": "LegalBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on LegalBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.694, - "details": { - "description": "min=0.425, mean=0.694, max=0.958, sum=3.469 (5)", - "tab": "Accuracy", - "LegalBench - Observed inference time (s)": "{\"description\": \"min=0.426, mean=0.878, max=1.58, sum=4.392 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.8783966223148776\"}", - "LegalBench - # eval": "{\"description\": \"min=95, mean=409.4, max=1000, sum=2047 (5)\", \"tab\": \"General information\", \"score\": \"409.4\"}", - "LegalBench - # train": "{\"description\": \"min=2.253, mean=4.251, max=5, sum=21.253 (5)\", \"tab\": \"General information\", \"score\": \"4.25061224489796\"}", - "LegalBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "LegalBench - # prompt tokens": "{\"description\": \"min=192.453, mean=940.377, max=3422.157, sum=4701.884 (5)\", \"tab\": \"General information\", \"score\": \"940.3768944254368\"}", - "LegalBench - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"abercrombie\", \"corporate_lobbying\", \"function_of_decision_section\", \"international_citizenship_questions\", \"proa\"]" - } - } - }, - { - "evaluation_name": "MedQA", - "source_data": { - "dataset_name": "MedQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MedQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.67, - "details": { - "description": "min=0.67, mean=0.67, max=0.67, sum=0.67 (1)", - "tab": "Accuracy", - "MedQA - Observed inference time (s)": "{\"description\": \"min=0.543, mean=0.543, max=0.543, sum=0.543 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.5430597031329782\"}", - "MedQA - # eval": "{\"description\": \"min=503, mean=503, max=503, sum=503 (1)\", \"tab\": \"General information\", \"score\": \"503.0\"}", - "MedQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MedQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MedQA - # prompt tokens": "{\"description\": \"min=1045.485, mean=1045.485, max=1045.485, sum=1045.485 (1)\", \"tab\": \"General information\", \"score\": \"1045.4850894632207\"}", - "MedQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WMT 2014", - "source_data": { - "dataset_name": "WMT 2014", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "BLEU-4 on WMT 2014", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.201, - "details": { - "description": "min=0.14, mean=0.201, max=0.255, sum=1.006 (5)", - "tab": "Accuracy", - "WMT 2014 - Observed inference time (s)": "{\"description\": \"min=1.148, mean=1.187, max=1.205, sum=5.933 (5)\", \"tab\": \"Efficiency\", \"score\": \"1.1866255830765444\"}", - "WMT 2014 - # eval": "{\"description\": \"min=503, mean=568.8, max=832, sum=2844 (5)\", \"tab\": \"General information\", \"score\": \"568.8\"}", - "WMT 2014 - # train": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "WMT 2014 - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WMT 2014 - # prompt tokens": "{\"description\": \"min=108.855, mean=126.657, max=142.373, sum=633.283 (5)\", \"tab\": \"General information\", \"score\": \"126.65662658663405\"}", - "WMT 2014 - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "language_pair": "[\"cs-en\", \"de-en\", \"fr-en\", \"hi-en\", \"ru-en\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_mmlu/qwen_qwen1.5-72b/1774096312.00548", - "retrieved_timestamp": "1774096312.00548", - "source_metadata": { - "source_name": "helm_mmlu", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_mmlu", - "evaluation_results": [ - { - "evaluation_name": "MMLU All Subjects", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU All Subjects", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.774, - "details": { - "description": "min=0.44, mean=0.774, max=0.99, sum=88.227 (114)", - "tab": "Accuracy", - "MMLU All Subjects - Observed inference time (s)": "{\"description\": \"min=0.323, mean=0.375, max=0.713, sum=42.762 (114)\", \"tab\": \"Efficiency\", \"score\": \"0.37510459085651054\"}", - "MMLU All Subjects - # eval": "{\"description\": \"min=100, mean=246.351, max=1534, sum=28084 (114)\", \"tab\": \"General information\", \"score\": \"246.35087719298247\"}", - "MMLU All Subjects - # train": "{\"description\": \"min=5, mean=5, max=5, sum=570 (114)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU All Subjects - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU All Subjects - # prompt tokens": "{\"description\": \"min=269.07, mean=618.598, max=2807.903, sum=70520.198 (114)\", \"tab\": \"General information\", \"score\": \"618.5982315160392\"}", - "MMLU All Subjects - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=114 (114)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_name": "Abstract Algebra", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Abstract Algebra", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.44, - "details": { - "description": "min=0.44, mean=0.44, max=0.44, sum=0.88 (2)", - "tab": "Accuracy", - "Abstract Algebra - Observed inference time (s)": "{\"description\": \"min=0.348, mean=0.348, max=0.348, sum=0.696 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3480935263633728\"}", - "Abstract Algebra - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Abstract Algebra - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Abstract Algebra - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Abstract Algebra - # prompt tokens": "{\"description\": \"min=371.19, mean=371.19, max=371.19, sum=742.38 (2)\", \"tab\": \"General information\", \"score\": \"371.19\"}", - "Abstract Algebra - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"abstract_algebra\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_abstract_algebra\"" - } - } - }, - { - "evaluation_name": "Anatomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Anatomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.733, - "details": { - "description": "min=0.733, mean=0.733, max=0.733, sum=1.467 (2)", - "tab": "Accuracy", - "Anatomy - Observed inference time (s)": "{\"description\": \"min=0.342, mean=0.342, max=0.342, sum=0.685 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3424220985836453\"}", - "Anatomy - # eval": "{\"description\": \"min=135, mean=135, max=135, sum=270 (2)\", \"tab\": \"General information\", \"score\": \"135.0\"}", - "Anatomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Anatomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Anatomy - # prompt tokens": "{\"description\": \"min=346.978, mean=346.978, max=346.978, sum=693.956 (2)\", \"tab\": \"General information\", \"score\": \"346.97777777777776\"}", - "Anatomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"anatomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_anatomy\"" - } - } - }, - { - "evaluation_name": "College Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on College Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.559, - "details": { - "description": "min=0.559, mean=0.559, max=0.559, sum=1.118 (2)", - "tab": "Accuracy", - "College Chemistry - Observed inference time (s)": "{\"description\": \"min=0.396, mean=0.396, max=0.396, sum=0.791 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.39563153505325316\"}", - "College Biology - Observed inference time (s)": "{\"description\": \"min=0.349, mean=0.349, max=0.349, sum=0.698 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3488144195742077\"}", - "College Computer Science - Observed inference time (s)": "{\"description\": \"min=0.398, mean=0.398, max=0.398, sum=0.797 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.39839950799942014\"}", - "College Mathematics - Observed inference time (s)": "{\"description\": \"min=0.372, mean=0.372, max=0.372, sum=0.743 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3715039682388306\"}", - "College Medicine - Observed inference time (s)": "{\"description\": \"min=0.346, mean=0.346, max=0.346, sum=0.693 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.34641625977665014\"}", - "College Physics - Observed inference time (s)": "{\"description\": \"min=0.384, mean=0.384, max=0.384, sum=0.768 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.38388992290870816\"}", - "College Chemistry - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Chemistry - # prompt tokens": "{\"description\": \"min=561.25, mean=561.25, max=561.25, sum=1122.5 (2)\", \"tab\": \"General information\", \"score\": \"561.25\"}", - "College Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Biology - # eval": "{\"description\": \"min=144, mean=144, max=144, sum=288 (2)\", \"tab\": \"General information\", \"score\": \"144.0\"}", - "College Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # prompt tokens": "{\"description\": \"min=479.979, mean=479.979, max=479.979, sum=959.958 (2)\", \"tab\": \"General information\", \"score\": \"479.9791666666667\"}", - "College Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # prompt tokens": "{\"description\": \"min=831.58, mean=831.58, max=831.58, sum=1663.16 (2)\", \"tab\": \"General information\", \"score\": \"831.58\"}", - "College Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Mathematics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # prompt tokens": "{\"description\": \"min=600.7, mean=600.7, max=600.7, sum=1201.4 (2)\", \"tab\": \"General information\", \"score\": \"600.7\"}", - "College Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Medicine - # eval": "{\"description\": \"min=173, mean=173, max=173, sum=346 (2)\", \"tab\": \"General information\", \"score\": \"173.0\"}", - "College Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # prompt tokens": "{\"description\": \"min=499.098, mean=499.098, max=499.098, sum=998.197 (2)\", \"tab\": \"General information\", \"score\": \"499.0982658959538\"}", - "College Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Physics - # eval": "{\"description\": \"min=102, mean=102, max=102, sum=204 (2)\", \"tab\": \"General information\", \"score\": \"102.0\"}", - "College Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # prompt tokens": "{\"description\": \"min=509.265, mean=509.265, max=509.265, sum=1018.529 (2)\", \"tab\": \"General information\", \"score\": \"509.2647058823529\"}", - "College Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"college_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_college_physics\"" - } - } - }, - { - "evaluation_name": "Computer Security", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Computer Security", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.81, - "details": { - "description": "min=0.81, mean=0.81, max=0.81, sum=1.62 (2)", - "tab": "Accuracy", - "Computer Security - Observed inference time (s)": "{\"description\": \"min=0.338, mean=0.338, max=0.338, sum=0.676 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3379603147506714\"}", - "Computer Security - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Computer Security - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Computer Security - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Computer Security - # prompt tokens": "{\"description\": \"min=379.64, mean=379.64, max=379.64, sum=759.28 (2)\", \"tab\": \"General information\", \"score\": \"379.64\"}", - "Computer Security - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"computer_security\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_computer_security\"" - } - } - }, - { - "evaluation_name": "Econometrics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Econometrics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.544, - "details": { - "description": "min=0.544, mean=0.544, max=0.544, sum=1.088 (2)", - "tab": "Accuracy", - "Econometrics - Observed inference time (s)": "{\"description\": \"min=0.386, mean=0.386, max=0.386, sum=0.772 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3857871189452054\"}", - "Econometrics - # eval": "{\"description\": \"min=114, mean=114, max=114, sum=228 (2)\", \"tab\": \"General information\", \"score\": \"114.0\"}", - "Econometrics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Econometrics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Econometrics - # prompt tokens": "{\"description\": \"min=620.939, mean=620.939, max=620.939, sum=1241.877 (2)\", \"tab\": \"General information\", \"score\": \"620.938596491228\"}", - "Econometrics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"econometrics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_econometrics\"" - } - } - }, - { - "evaluation_name": "Global Facts", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Global Facts", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.56, - "details": { - "description": "min=0.56, mean=0.56, max=0.56, sum=1.12 (2)", - "tab": "Accuracy", - "Global Facts - Observed inference time (s)": "{\"description\": \"min=0.335, mean=0.335, max=0.335, sum=0.669 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3347077107429504\"}", - "Global Facts - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Global Facts - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Global Facts - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Global Facts - # prompt tokens": "{\"description\": \"min=422.06, mean=422.06, max=422.06, sum=844.12 (2)\", \"tab\": \"General information\", \"score\": \"422.06\"}", - "Global Facts - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"global_facts\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_global_facts\"" - } - } - }, - { - "evaluation_name": "Jurisprudence", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Jurisprudence", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.824, - "details": { - "description": "min=0.824, mean=0.824, max=0.824, sum=1.648 (2)", - "tab": "Accuracy", - "Jurisprudence - Observed inference time (s)": "{\"description\": \"min=0.351, mean=0.351, max=0.351, sum=0.702 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3512495689921909\"}", - "Jurisprudence - # eval": "{\"description\": \"min=108, mean=108, max=108, sum=216 (2)\", \"tab\": \"General information\", \"score\": \"108.0\"}", - "Jurisprudence - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Jurisprudence - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Jurisprudence - # prompt tokens": "{\"description\": \"min=387.713, mean=387.713, max=387.713, sum=775.426 (2)\", \"tab\": \"General information\", \"score\": \"387.712962962963\"}", - "Jurisprudence - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"jurisprudence\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_jurisprudence\"" - } - } - }, - { - "evaluation_name": "Philosophy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Philosophy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.83, - "details": { - "description": "min=0.83, mean=0.83, max=0.83, sum=1.659 (2)", - "tab": "Accuracy", - "Philosophy - Observed inference time (s)": "{\"description\": \"min=0.35, mean=0.35, max=0.35, sum=0.7 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.34987031455208634\"}", - "Philosophy - # eval": "{\"description\": \"min=311, mean=311, max=311, sum=622 (2)\", \"tab\": \"General information\", \"score\": \"311.0\"}", - "Philosophy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Philosophy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Philosophy - # prompt tokens": "{\"description\": \"min=322.09, mean=322.09, max=322.09, sum=644.18 (2)\", \"tab\": \"General information\", \"score\": \"322.09003215434086\"}", - "Philosophy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"philosophy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_philosophy\"" - } - } - }, - { - "evaluation_name": "Professional Psychology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Professional Psychology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.809, - "details": { - "description": "min=0.809, mean=0.809, max=0.809, sum=1.618 (2)", - "tab": "Accuracy", - "Professional Medicine - Observed inference time (s)": "{\"description\": \"min=0.426, mean=0.426, max=0.426, sum=0.852 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4260168829384972\"}", - "Professional Accounting - Observed inference time (s)": "{\"description\": \"min=0.375, mean=0.375, max=0.375, sum=0.75 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3750799666059778\"}", - "Professional Law - Observed inference time (s)": "{\"description\": \"min=0.501, mean=0.501, max=0.501, sum=1.002 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.501238130839272\"}", - "Professional Psychology - Observed inference time (s)": "{\"description\": \"min=0.359, mean=0.359, max=0.359, sum=0.719 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3593972987598843\"}", - "Professional Medicine - # eval": "{\"description\": \"min=272, mean=272, max=272, sum=544 (2)\", \"tab\": \"General information\", \"score\": \"272.0\"}", - "Professional Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Medicine - # prompt tokens": "{\"description\": \"min=1118.199, mean=1118.199, max=1118.199, sum=2236.397 (2)\", \"tab\": \"General information\", \"score\": \"1118.1985294117646\"}", - "Professional Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Accounting - # eval": "{\"description\": \"min=282, mean=282, max=282, sum=564 (2)\", \"tab\": \"General information\", \"score\": \"282.0\"}", - "Professional Accounting - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Accounting - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # prompt tokens": "{\"description\": \"min=732.34, mean=732.34, max=732.34, sum=1464.681 (2)\", \"tab\": \"General information\", \"score\": \"732.3404255319149\"}", - "Professional Accounting - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Law - # eval": "{\"description\": \"min=1534, mean=1534, max=1534, sum=3068 (2)\", \"tab\": \"General information\", \"score\": \"1534.0\"}", - "Professional Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # prompt tokens": "{\"description\": \"min=1656.969, mean=1656.969, max=1656.969, sum=3313.939 (2)\", \"tab\": \"General information\", \"score\": \"1656.9693611473272\"}", - "Professional Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Psychology - # eval": "{\"description\": \"min=612, mean=612, max=612, sum=1224 (2)\", \"tab\": \"General information\", \"score\": \"612.0\"}", - "Professional Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # prompt tokens": "{\"description\": \"min=574.417, mean=574.417, max=574.417, sum=1148.833 (2)\", \"tab\": \"General information\", \"score\": \"574.4166666666666\"}", - "Professional Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"professional_psychology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_professional_psychology\"" - } - } - }, - { - "evaluation_name": "Us Foreign Policy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Us Foreign Policy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.94, - "details": { - "description": "min=0.94, mean=0.94, max=0.94, sum=1.88 (2)", - "tab": "Accuracy", - "Us Foreign Policy - Observed inference time (s)": "{\"description\": \"min=0.352, mean=0.352, max=0.352, sum=0.703 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3515354657173157\"}", - "Us Foreign Policy - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Us Foreign Policy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Us Foreign Policy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Us Foreign Policy - # prompt tokens": "{\"description\": \"min=421.16, mean=421.16, max=421.16, sum=842.32 (2)\", \"tab\": \"General information\", \"score\": \"421.16\"}", - "Us Foreign Policy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"us_foreign_policy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_us_foreign_policy\"" - } - } - }, - { - "evaluation_name": "Astronomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Astronomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.868, - "details": { - "description": "min=0.868, mean=0.868, max=0.868, sum=1.737 (2)", - "tab": "Accuracy", - "Astronomy - Observed inference time (s)": "{\"description\": \"min=0.373, mean=0.373, max=0.373, sum=0.746 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3729873691734515\"}", - "Astronomy - # eval": "{\"description\": \"min=152, mean=152, max=152, sum=304 (2)\", \"tab\": \"General information\", \"score\": \"152.0\"}", - "Astronomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Astronomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Astronomy - # prompt tokens": "{\"description\": \"min=582.849, mean=582.849, max=582.849, sum=1165.697 (2)\", \"tab\": \"General information\", \"score\": \"582.8486842105264\"}", - "Astronomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"astronomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_astronomy\"" - } - } - }, - { - "evaluation_name": "Business Ethics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Business Ethics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.79, - "details": { - "description": "min=0.79, mean=0.79, max=0.79, sum=1.58 (2)", - "tab": "Accuracy", - "Business Ethics - Observed inference time (s)": "{\"description\": \"min=0.405, mean=0.405, max=0.405, sum=0.81 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.40487982749938967\"}", - "Business Ethics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Business Ethics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Business Ethics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Business Ethics - # prompt tokens": "{\"description\": \"min=562.87, mean=562.87, max=562.87, sum=1125.74 (2)\", \"tab\": \"General information\", \"score\": \"562.87\"}", - "Business Ethics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"business_ethics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_business_ethics\"" - } - } - }, - { - "evaluation_name": "Clinical Knowledge", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Clinical Knowledge", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.834, - "details": { - "description": "min=0.834, mean=0.834, max=0.834, sum=1.668 (2)", - "tab": "Accuracy", - "Clinical Knowledge - Observed inference time (s)": "{\"description\": \"min=0.349, mean=0.349, max=0.349, sum=0.698 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.34907986892844145\"}", - "Clinical Knowledge - # eval": "{\"description\": \"min=265, mean=265, max=265, sum=530 (2)\", \"tab\": \"General information\", \"score\": \"265.0\"}", - "Clinical Knowledge - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Clinical Knowledge - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Clinical Knowledge - # prompt tokens": "{\"description\": \"min=393.623, mean=393.623, max=393.623, sum=787.245 (2)\", \"tab\": \"General information\", \"score\": \"393.62264150943395\"}", - "Clinical Knowledge - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"clinical_knowledge\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_clinical_knowledge\"" - } - } - }, - { - "evaluation_name": "Conceptual Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Conceptual Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.821, - "details": { - "description": "min=0.821, mean=0.821, max=0.821, sum=1.643 (2)", - "tab": "Accuracy", - "Conceptual Physics - Observed inference time (s)": "{\"description\": \"min=0.329, mean=0.329, max=0.329, sum=0.658 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3290608903194996\"}", - "Conceptual Physics - # eval": "{\"description\": \"min=235, mean=235, max=235, sum=470 (2)\", \"tab\": \"General information\", \"score\": \"235.0\"}", - "Conceptual Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Conceptual Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Conceptual Physics - # prompt tokens": "{\"description\": \"min=298.494, mean=298.494, max=298.494, sum=596.987 (2)\", \"tab\": \"General information\", \"score\": \"298.4936170212766\"}", - "Conceptual Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"conceptual_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_conceptual_physics\"" - } - } - }, - { - "evaluation_name": "Electrical Engineering", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Electrical Engineering", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.779, - "details": { - "description": "min=0.779, mean=0.779, max=0.779, sum=1.559 (2)", - "tab": "Accuracy", - "Electrical Engineering - Observed inference time (s)": "{\"description\": \"min=0.323, mean=0.323, max=0.323, sum=0.646 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.32275488458830737\"}", - "Electrical Engineering - # eval": "{\"description\": \"min=145, mean=145, max=145, sum=290 (2)\", \"tab\": \"General information\", \"score\": \"145.0\"}", - "Electrical Engineering - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Electrical Engineering - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Electrical Engineering - # prompt tokens": "{\"description\": \"min=456.8, mean=456.8, max=456.8, sum=913.6 (2)\", \"tab\": \"General information\", \"score\": \"456.8\"}", - "Electrical Engineering - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"electrical_engineering\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_electrical_engineering\"" - } - } - }, - { - "evaluation_name": "Elementary Mathematics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Elementary Mathematics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.696, - "details": { - "description": "min=0.696, mean=0.696, max=0.696, sum=1.392 (2)", - "tab": "Accuracy", - "Elementary Mathematics - Observed inference time (s)": "{\"description\": \"min=0.365, mean=0.365, max=0.365, sum=0.73 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.364848568325951\"}", - "Elementary Mathematics - # eval": "{\"description\": \"min=378, mean=378, max=378, sum=756 (2)\", \"tab\": \"General information\", \"score\": \"378.0\"}", - "Elementary Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Elementary Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Elementary Mathematics - # prompt tokens": "{\"description\": \"min=570.119, mean=570.119, max=570.119, sum=1140.238 (2)\", \"tab\": \"General information\", \"score\": \"570.1190476190476\"}", - "Elementary Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"elementary_mathematics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_elementary_mathematics\"" - } - } - }, - { - "evaluation_name": "Formal Logic", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Formal Logic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.556, - "details": { - "description": "min=0.556, mean=0.556, max=0.556, sum=1.111 (2)", - "tab": "Accuracy", - "Formal Logic - Observed inference time (s)": "{\"description\": \"min=0.359, mean=0.359, max=0.359, sum=0.718 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3588152726491292\"}", - "Formal Logic - # eval": "{\"description\": \"min=126, mean=126, max=126, sum=252 (2)\", \"tab\": \"General information\", \"score\": \"126.0\"}", - "Formal Logic - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Formal Logic - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Formal Logic - # prompt tokens": "{\"description\": \"min=597.667, mean=597.667, max=597.667, sum=1195.333 (2)\", \"tab\": \"General information\", \"score\": \"597.6666666666666\"}", - "Formal Logic - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"formal_logic\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_formal_logic\"" - } - } - }, - { - "evaluation_name": "High School World History", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on High School World History", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.899, - "details": { - "description": "min=0.899, mean=0.899, max=0.899, sum=1.797 (2)", - "tab": "Accuracy", - "High School Biology - Observed inference time (s)": "{\"description\": \"min=0.365, mean=0.365, max=0.365, sum=0.729 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3646186044139247\"}", - "High School Chemistry - Observed inference time (s)": "{\"description\": \"min=0.366, mean=0.366, max=0.366, sum=0.731 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.36553433728335527\"}", - "High School Computer Science - Observed inference time (s)": "{\"description\": \"min=0.381, mean=0.381, max=0.381, sum=0.761 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.38066073894500735\"}", - "High School European History - Observed inference time (s)": "{\"description\": \"min=0.713, mean=0.713, max=0.713, sum=1.426 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7130387075019605\"}", - "High School Geography - Observed inference time (s)": "{\"description\": \"min=0.36, mean=0.36, max=0.36, sum=0.72 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.36007895975401905\"}", - "High School Government And Politics - Observed inference time (s)": "{\"description\": \"min=0.336, mean=0.336, max=0.336, sum=0.672 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3358402029837969\"}", - "High School Macroeconomics - Observed inference time (s)": "{\"description\": \"min=0.332, mean=0.332, max=0.332, sum=0.663 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3316040589259221\"}", - "High School Mathematics - Observed inference time (s)": "{\"description\": \"min=0.374, mean=0.374, max=0.374, sum=0.747 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3736002833754928\"}", - "High School Microeconomics - Observed inference time (s)": "{\"description\": \"min=0.325, mean=0.325, max=0.325, sum=0.649 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.32468783655086486\"}", - "High School Physics - Observed inference time (s)": "{\"description\": \"min=0.392, mean=0.392, max=0.392, sum=0.785 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3924832533526894\"}", - "High School Psychology - Observed inference time (s)": "{\"description\": \"min=0.36, mean=0.36, max=0.36, sum=0.721 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3602875184575352\"}", - "High School Statistics - Observed inference time (s)": "{\"description\": \"min=0.399, mean=0.399, max=0.399, sum=0.798 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.39876955968362315\"}", - "High School US History - Observed inference time (s)": "{\"description\": \"min=0.554, mean=0.554, max=0.554, sum=1.107 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5536784272567898\"}", - "High School World History - Observed inference time (s)": "{\"description\": \"min=0.475, mean=0.475, max=0.475, sum=0.949 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.474577054695741\"}", - "High School Biology - # eval": "{\"description\": \"min=310, mean=310, max=310, sum=620 (2)\", \"tab\": \"General information\", \"score\": \"310.0\"}", - "High School Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Biology - # prompt tokens": "{\"description\": \"min=506.916, mean=506.916, max=506.916, sum=1013.832 (2)\", \"tab\": \"General information\", \"score\": \"506.9161290322581\"}", - "High School Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Chemistry - # eval": "{\"description\": \"min=203, mean=203, max=203, sum=406 (2)\", \"tab\": \"General information\", \"score\": \"203.0\"}", - "High School Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # prompt tokens": "{\"description\": \"min=510.261, mean=510.261, max=510.261, sum=1020.522 (2)\", \"tab\": \"General information\", \"score\": \"510.2610837438424\"}", - "High School Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "High School Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # prompt tokens": "{\"description\": \"min=871.46, mean=871.46, max=871.46, sum=1742.92 (2)\", \"tab\": \"General information\", \"score\": \"871.46\"}", - "High School Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School European History - # eval": "{\"description\": \"min=165, mean=165, max=165, sum=330 (2)\", \"tab\": \"General information\", \"score\": \"165.0\"}", - "High School European History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School European History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # prompt tokens": "{\"description\": \"min=2807.903, mean=2807.903, max=2807.903, sum=5615.806 (2)\", \"tab\": \"General information\", \"score\": \"2807.9030303030304\"}", - "High School European History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Geography - # eval": "{\"description\": \"min=198, mean=198, max=198, sum=396 (2)\", \"tab\": \"General information\", \"score\": \"198.0\"}", - "High School Geography - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Geography - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # prompt tokens": "{\"description\": \"min=365.217, mean=365.217, max=365.217, sum=730.434 (2)\", \"tab\": \"General information\", \"score\": \"365.2171717171717\"}", - "High School Geography - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Government And Politics - # eval": "{\"description\": \"min=193, mean=193, max=193, sum=386 (2)\", \"tab\": \"General information\", \"score\": \"193.0\"}", - "High School Government And Politics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Government And Politics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # prompt tokens": "{\"description\": \"min=460.311, mean=460.311, max=460.311, sum=920.622 (2)\", \"tab\": \"General information\", \"score\": \"460.31088082901556\"}", - "High School Government And Politics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Macroeconomics - # eval": "{\"description\": \"min=390, mean=390, max=390, sum=780 (2)\", \"tab\": \"General information\", \"score\": \"390.0\"}", - "High School Macroeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Macroeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # prompt tokens": "{\"description\": \"min=367.349, mean=367.349, max=367.349, sum=734.697 (2)\", \"tab\": \"General information\", \"score\": \"367.34871794871793\"}", - "High School Macroeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Mathematics - # eval": "{\"description\": \"min=270, mean=270, max=270, sum=540 (2)\", \"tab\": \"General information\", \"score\": \"270.0\"}", - "High School Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # prompt tokens": "{\"description\": \"min=558.326, mean=558.326, max=558.326, sum=1116.652 (2)\", \"tab\": \"General information\", \"score\": \"558.325925925926\"}", - "High School Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Microeconomics - # eval": "{\"description\": \"min=238, mean=238, max=238, sum=476 (2)\", \"tab\": \"General information\", \"score\": \"238.0\"}", - "High School Microeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Microeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # prompt tokens": "{\"description\": \"min=395.277, mean=395.277, max=395.277, sum=790.555 (2)\", \"tab\": \"General information\", \"score\": \"395.2773109243698\"}", - "High School Microeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Physics - # eval": "{\"description\": \"min=151, mean=151, max=151, sum=302 (2)\", \"tab\": \"General information\", \"score\": \"151.0\"}", - "High School Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # prompt tokens": "{\"description\": \"min=573.536, mean=573.536, max=573.536, sum=1147.073 (2)\", \"tab\": \"General information\", \"score\": \"573.5364238410596\"}", - "High School Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Psychology - # eval": "{\"description\": \"min=545, mean=545, max=545, sum=1090 (2)\", \"tab\": \"General information\", \"score\": \"545.0\"}", - "High School Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # prompt tokens": "{\"description\": \"min=488.521, mean=488.521, max=488.521, sum=977.042 (2)\", \"tab\": \"General information\", \"score\": \"488.52110091743117\"}", - "High School Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Statistics - # eval": "{\"description\": \"min=216, mean=216, max=216, sum=432 (2)\", \"tab\": \"General information\", \"score\": \"216.0\"}", - "High School Statistics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Statistics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # prompt tokens": "{\"description\": \"min=823.477, mean=823.477, max=823.477, sum=1646.954 (2)\", \"tab\": \"General information\", \"score\": \"823.4768518518518\"}", - "High School Statistics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School US History - # eval": "{\"description\": \"min=204, mean=204, max=204, sum=408 (2)\", \"tab\": \"General information\", \"score\": \"204.0\"}", - "High School US History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School US History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # prompt tokens": "{\"description\": \"min=2230.176, mean=2230.176, max=2230.176, sum=4460.353 (2)\", \"tab\": \"General information\", \"score\": \"2230.176470588235\"}", - "High School US History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School World History - # eval": "{\"description\": \"min=237, mean=237, max=237, sum=474 (2)\", \"tab\": \"General information\", \"score\": \"237.0\"}", - "High School World History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School World History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # prompt tokens": "{\"description\": \"min=1441.354, mean=1441.354, max=1441.354, sum=2882.709 (2)\", \"tab\": \"General information\", \"score\": \"1441.3544303797469\"}", - "High School World History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"high_school_world_history\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_high_school_world_history\"" - } - } - }, - { - "evaluation_name": "Human Sexuality", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Human Sexuality", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.878, - "details": { - "description": "min=0.878, mean=0.878, max=0.878, sum=1.756 (2)", - "tab": "Accuracy", - "Human Aging - Observed inference time (s)": "{\"description\": \"min=0.346, mean=0.346, max=0.346, sum=0.692 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.34584820110167086\"}", - "Human Sexuality - Observed inference time (s)": "{\"description\": \"min=0.357, mean=0.357, max=0.357, sum=0.714 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.35706568856275717\"}", - "Human Aging - # eval": "{\"description\": \"min=223, mean=223, max=223, sum=446 (2)\", \"tab\": \"General information\", \"score\": \"223.0\"}", - "Human Aging - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Aging - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Aging - # prompt tokens": "{\"description\": \"min=315.121, mean=315.121, max=315.121, sum=630.242 (2)\", \"tab\": \"General information\", \"score\": \"315.1210762331838\"}", - "Human Aging - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Human Sexuality - # eval": "{\"description\": \"min=131, mean=131, max=131, sum=262 (2)\", \"tab\": \"General information\", \"score\": \"131.0\"}", - "Human Sexuality - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Sexuality - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # prompt tokens": "{\"description\": \"min=334.504, mean=334.504, max=334.504, sum=669.008 (2)\", \"tab\": \"General information\", \"score\": \"334.5038167938931\"}", - "Human Sexuality - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"human_sexuality\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_human_sexuality\"" - } - } - }, - { - "evaluation_name": "International Law", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on International Law", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.909, - "details": { - "description": "min=0.909, mean=0.909, max=0.909, sum=1.818 (2)", - "tab": "Accuracy", - "International Law - Observed inference time (s)": "{\"description\": \"min=0.375, mean=0.375, max=0.375, sum=0.75 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.37501588931753616\"}", - "International Law - # eval": "{\"description\": \"min=121, mean=121, max=121, sum=242 (2)\", \"tab\": \"General information\", \"score\": \"121.0\"}", - "International Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "International Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "International Law - # prompt tokens": "{\"description\": \"min=633.579, mean=633.579, max=633.579, sum=1267.157 (2)\", \"tab\": \"General information\", \"score\": \"633.5785123966942\"}", - "International Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"international_law\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_international_law\"" - } - } - }, - { - "evaluation_name": "Logical Fallacies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Logical Fallacies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.853, - "details": { - "description": "min=0.853, mean=0.853, max=0.853, sum=1.706 (2)", - "tab": "Accuracy", - "Logical Fallacies - Observed inference time (s)": "{\"description\": \"min=0.347, mean=0.347, max=0.347, sum=0.694 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.34693217131257786\"}", - "Logical Fallacies - # eval": "{\"description\": \"min=163, mean=163, max=163, sum=326 (2)\", \"tab\": \"General information\", \"score\": \"163.0\"}", - "Logical Fallacies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Logical Fallacies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Logical Fallacies - # prompt tokens": "{\"description\": \"min=442.632, mean=442.632, max=442.632, sum=885.264 (2)\", \"tab\": \"General information\", \"score\": \"442.6319018404908\"}", - "Logical Fallacies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"logical_fallacies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_logical_fallacies\"" - } - } - }, - { - "evaluation_name": "Machine Learning", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Machine Learning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.67, - "details": { - "description": "min=0.67, mean=0.67, max=0.67, sum=1.339 (2)", - "tab": "Accuracy", - "Machine Learning - Observed inference time (s)": "{\"description\": \"min=0.36, mean=0.36, max=0.36, sum=0.719 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3595333376101085\"}", - "Machine Learning - # eval": "{\"description\": \"min=112, mean=112, max=112, sum=224 (2)\", \"tab\": \"General information\", \"score\": \"112.0\"}", - "Machine Learning - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Machine Learning - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Machine Learning - # prompt tokens": "{\"description\": \"min=674.848, mean=674.848, max=674.848, sum=1349.696 (2)\", \"tab\": \"General information\", \"score\": \"674.8482142857143\"}", - "Machine Learning - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"machine_learning\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_machine_learning\"" - } - } - }, - { - "evaluation_name": "Management", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Management", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.854, - "details": { - "description": "min=0.854, mean=0.854, max=0.854, sum=1.709 (2)", - "tab": "Accuracy", - "Management - Observed inference time (s)": "{\"description\": \"min=0.346, mean=0.346, max=0.346, sum=0.692 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3462491313230644\"}", - "Management - # eval": "{\"description\": \"min=103, mean=103, max=103, sum=206 (2)\", \"tab\": \"General information\", \"score\": \"103.0\"}", - "Management - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Management - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Management - # prompt tokens": "{\"description\": \"min=276.854, mean=276.854, max=276.854, sum=553.709 (2)\", \"tab\": \"General information\", \"score\": \"276.8543689320388\"}", - "Management - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"management\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_management\"" - } - } - }, - { - "evaluation_name": "Marketing", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Marketing", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.949, - "details": { - "description": "min=0.949, mean=0.949, max=0.949, sum=1.897 (2)", - "tab": "Accuracy", - "Marketing - Observed inference time (s)": "{\"description\": \"min=0.35, mean=0.35, max=0.35, sum=0.7 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3498607089376857\"}", - "Marketing - # eval": "{\"description\": \"min=234, mean=234, max=234, sum=468 (2)\", \"tab\": \"General information\", \"score\": \"234.0\"}", - "Marketing - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Marketing - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Marketing - # prompt tokens": "{\"description\": \"min=397.415, mean=397.415, max=397.415, sum=794.829 (2)\", \"tab\": \"General information\", \"score\": \"397.4145299145299\"}", - "Marketing - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"marketing\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_marketing\"" - } - } - }, - { - "evaluation_name": "Medical Genetics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Medical Genetics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.87, - "details": { - "description": "min=0.87, mean=0.87, max=0.87, sum=1.74 (2)", - "tab": "Accuracy", - "Medical Genetics - Observed inference time (s)": "{\"description\": \"min=0.343, mean=0.343, max=0.343, sum=0.686 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3427603816986084\"}", - "Medical Genetics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Medical Genetics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Medical Genetics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Medical Genetics - # prompt tokens": "{\"description\": \"min=335.35, mean=335.35, max=335.35, sum=670.7 (2)\", \"tab\": \"General information\", \"score\": \"335.35\"}", - "Medical Genetics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"medical_genetics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_medical_genetics\"" - } - } - }, - { - "evaluation_name": "Miscellaneous", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Miscellaneous", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.921, - "details": { - "description": "min=0.921, mean=0.921, max=0.921, sum=1.842 (2)", - "tab": "Accuracy", - "Miscellaneous - Observed inference time (s)": "{\"description\": \"min=0.343, mean=0.343, max=0.343, sum=0.687 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3433326785744074\"}", - "Miscellaneous - # eval": "{\"description\": \"min=783, mean=783, max=783, sum=1566 (2)\", \"tab\": \"General information\", \"score\": \"783.0\"}", - "Miscellaneous - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Miscellaneous - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Miscellaneous - # prompt tokens": "{\"description\": \"min=296.7, mean=296.7, max=296.7, sum=593.4 (2)\", \"tab\": \"General information\", \"score\": \"296.6998722860792\"}", - "Miscellaneous - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"miscellaneous\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_miscellaneous\"" - } - } - }, - { - "evaluation_name": "Moral Scenarios", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Moral Scenarios", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.669, - "details": { - "description": "min=0.669, mean=0.669, max=0.669, sum=1.339 (2)", - "tab": "Accuracy", - "Moral Disputes - Observed inference time (s)": "{\"description\": \"min=0.347, mean=0.347, max=0.347, sum=0.693 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.34657375729841994\"}", - "Moral Scenarios - Observed inference time (s)": "{\"description\": \"min=0.374, mean=0.374, max=0.374, sum=0.749 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.37438980161144747\"}", - "Moral Disputes - # eval": "{\"description\": \"min=346, mean=346, max=346, sum=692 (2)\", \"tab\": \"General information\", \"score\": \"346.0\"}", - "Moral Disputes - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Disputes - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Disputes - # prompt tokens": "{\"description\": \"min=469.182, mean=469.182, max=469.182, sum=938.364 (2)\", \"tab\": \"General information\", \"score\": \"469.1820809248555\"}", - "Moral Disputes - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Moral Scenarios - # eval": "{\"description\": \"min=895, mean=895, max=895, sum=1790 (2)\", \"tab\": \"General information\", \"score\": \"895.0\"}", - "Moral Scenarios - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Scenarios - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # prompt tokens": "{\"description\": \"min=661.494, mean=661.494, max=661.494, sum=1322.988 (2)\", \"tab\": \"General information\", \"score\": \"661.4938547486033\"}", - "Moral Scenarios - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"moral_scenarios\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_moral_scenarios\"" - } - } - }, - { - "evaluation_name": "Nutrition", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Nutrition", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.859, - "details": { - "description": "min=0.859, mean=0.859, max=0.859, sum=1.719 (2)", - "tab": "Accuracy", - "Nutrition - Observed inference time (s)": "{\"description\": \"min=0.372, mean=0.372, max=0.372, sum=0.744 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3719378265680051\"}", - "Nutrition - # eval": "{\"description\": \"min=306, mean=306, max=306, sum=612 (2)\", \"tab\": \"General information\", \"score\": \"306.0\"}", - "Nutrition - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Nutrition - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Nutrition - # prompt tokens": "{\"description\": \"min=592.637, mean=592.637, max=592.637, sum=1185.275 (2)\", \"tab\": \"General information\", \"score\": \"592.6372549019608\"}", - "Nutrition - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"nutrition\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_nutrition\"" - } - } - }, - { - "evaluation_name": "Prehistory", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Prehistory", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.88, - "details": { - "description": "min=0.88, mean=0.88, max=0.88, sum=1.759 (2)", - "tab": "Accuracy", - "Prehistory - Observed inference time (s)": "{\"description\": \"min=0.36, mean=0.36, max=0.36, sum=0.72 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.35996099313100177\"}", - "Prehistory - # eval": "{\"description\": \"min=324, mean=324, max=324, sum=648 (2)\", \"tab\": \"General information\", \"score\": \"324.0\"}", - "Prehistory - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Prehistory - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Prehistory - # prompt tokens": "{\"description\": \"min=521.364, mean=521.364, max=521.364, sum=1042.728 (2)\", \"tab\": \"General information\", \"score\": \"521.3641975308642\"}", - "Prehistory - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"prehistory\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_prehistory\"" - } - } - }, - { - "evaluation_name": "Public Relations", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Public Relations", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.755, - "details": { - "description": "min=0.755, mean=0.755, max=0.755, sum=1.509 (2)", - "tab": "Accuracy", - "Public Relations - Observed inference time (s)": "{\"description\": \"min=0.34, mean=0.34, max=0.34, sum=0.68 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.340008375861428\"}", - "Public Relations - # eval": "{\"description\": \"min=110, mean=110, max=110, sum=220 (2)\", \"tab\": \"General information\", \"score\": \"110.0\"}", - "Public Relations - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Public Relations - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Public Relations - # prompt tokens": "{\"description\": \"min=401.427, mean=401.427, max=401.427, sum=802.855 (2)\", \"tab\": \"General information\", \"score\": \"401.42727272727274\"}", - "Public Relations - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"public_relations\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_public_relations\"" - } - } - }, - { - "evaluation_name": "Security Studies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Security Studies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.824, - "details": { - "description": "min=0.824, mean=0.824, max=0.824, sum=1.649 (2)", - "tab": "Accuracy", - "Security Studies - Observed inference time (s)": "{\"description\": \"min=0.432, mean=0.432, max=0.432, sum=0.864 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.43211937923820654\"}", - "Security Studies - # eval": "{\"description\": \"min=245, mean=245, max=245, sum=490 (2)\", \"tab\": \"General information\", \"score\": \"245.0\"}", - "Security Studies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Security Studies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Security Studies - # prompt tokens": "{\"description\": \"min=1159.931, mean=1159.931, max=1159.931, sum=2319.861 (2)\", \"tab\": \"General information\", \"score\": \"1159.930612244898\"}", - "Security Studies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"security_studies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_security_studies\"" - } - } - }, - { - "evaluation_name": "Sociology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Sociology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9, - "details": { - "description": "min=0.9, mean=0.9, max=0.9, sum=1.801 (2)", - "tab": "Accuracy", - "Sociology - Observed inference time (s)": "{\"description\": \"min=0.353, mean=0.353, max=0.353, sum=0.707 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.35334858491053034\"}", - "Sociology - # eval": "{\"description\": \"min=201, mean=201, max=201, sum=402 (2)\", \"tab\": \"General information\", \"score\": \"201.0\"}", - "Sociology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Sociology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Sociology - # prompt tokens": "{\"description\": \"min=443.1, mean=443.1, max=443.1, sum=886.199 (2)\", \"tab\": \"General information\", \"score\": \"443.0995024875622\"}", - "Sociology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"sociology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_sociology\"" - } - } - }, - { - "evaluation_name": "Virology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Virology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.584, - "details": { - "description": "min=0.584, mean=0.584, max=0.584, sum=1.169 (2)", - "tab": "Accuracy", - "Virology - Observed inference time (s)": "{\"description\": \"min=0.338, mean=0.338, max=0.338, sum=0.676 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.33793931696788376\"}", - "Virology - # eval": "{\"description\": \"min=166, mean=166, max=166, sum=332 (2)\", \"tab\": \"General information\", \"score\": \"166.0\"}", - "Virology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Virology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Virology - # prompt tokens": "{\"description\": \"min=336.819, mean=336.819, max=336.819, sum=673.639 (2)\", \"tab\": \"General information\", \"score\": \"336.8192771084337\"}", - "Virology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"virology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_virology\"" - } - } - }, - { - "evaluation_name": "World Religions", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on World Religions", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.883, - "details": { - "description": "min=0.883, mean=0.883, max=0.883, sum=1.766 (2)", - "tab": "Accuracy", - "World Religions - Observed inference time (s)": "{\"description\": \"min=0.358, mean=0.358, max=0.358, sum=0.716 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.358185218788727\"}", - "World Religions - # eval": "{\"description\": \"min=171, mean=171, max=171, sum=342 (2)\", \"tab\": \"General information\", \"score\": \"171.0\"}", - "World Religions - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "World Religions - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "World Religions - # prompt tokens": "{\"description\": \"min=269.07, mean=269.07, max=269.07, sum=538.14 (2)\", \"tab\": \"General information\", \"score\": \"269.0701754385965\"}", - "World Religions - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"world_religions\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_world_religions\"" - } - } - }, - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.65, - "details": { - "description": "", - "tab": "Efficiency" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/qwen_qwen1.5-7b-chat.json b/data/models/qwen_qwen1.5-7b-chat.json deleted file mode 100644 index 4faec9273a9d64c88d07afaae4c908637febd724..0000000000000000000000000000000000000000 --- a/data/models/qwen_qwen1.5-7b-chat.json +++ /dev/null @@ -1,277 +0,0 @@ -{ - "model_info": { - "name": "Qwen/Qwen1.5-7B-Chat", - "id": "Qwen/Qwen1.5-7B-Chat", - "developer": "Qwen", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.721" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Qwen_Qwen1.5-7B-Chat/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4371 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.451 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0627 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3029 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3779 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2951 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "reward-bench/Qwen_Qwen1.5-7B-Chat/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.675 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5363 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6908 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6919 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9041 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4288 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qwen_qwen1.5-7b.json b/data/models/qwen_qwen1.5-7b.json deleted file mode 100644 index 24283107ef34de08d3a2aa1368a42d587d60aa39..0000000000000000000000000000000000000000 --- a/data/models/qwen_qwen1.5-7b.json +++ /dev/null @@ -1,2036 +0,0 @@ -{ - "model_info": { - "name": "Qwen1.5 7B", - "id": "qwen/qwen1.5-7b", - "developer": "qwen", - "inference_platform": "unknown", - "additional_details": { - "model_id_aliases": [ - "Qwen/Qwen1.5-7B" - ] - } - }, - "evaluations": [ - { - "evaluation_id": "helm_lite/qwen_qwen1.5-7b/1774096306.427425", - "retrieved_timestamp": "1774096306.427425", - "source_metadata": { - "source_name": "helm_lite", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_lite", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_lite", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.275, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.8087765293383271\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.448, - "details": { - "description": "min=0.448, mean=0.448, max=0.448, sum=0.448 (1)", - "tab": "Accuracy", - "NarrativeQA - Observed inference time (s)": "{\"description\": \"min=0.855, mean=0.855, max=0.855, sum=0.855 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.8547548650016248\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=3468.913, mean=3468.913, max=3468.913, sum=3468.913 (1)\", \"tab\": \"General information\", \"score\": \"3468.912676056338\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (closed-book)", - "source_data": { - "dataset_name": "NaturalQuestions (closed-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (closed-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.27, - "details": { - "description": "min=0.27, mean=0.27, max=0.27, sum=0.27 (1)", - "tab": "Accuracy", - "NaturalQuestions (open-book) - Observed inference time (s)": "{\"description\": \"min=0.479, mean=0.479, max=0.479, sum=0.479 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.4786673946380615\"}", - "NaturalQuestions (closed-book) - Observed inference time (s)": "{\"description\": \"min=0.354, mean=0.354, max=0.354, sum=0.354 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.354404949426651\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1990.955, mean=1990.955, max=1990.955, sum=1990.955 (1)\", \"tab\": \"General information\", \"score\": \"1990.955\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=119.262, mean=119.262, max=119.262, sum=119.262 (1)\", \"tab\": \"General information\", \"score\": \"119.262\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "mode": "\"closedbook\"" - } - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.806, - "details": { - "description": "min=0.806, mean=0.806, max=0.806, sum=0.806 (1)", - "tab": "Accuracy", - "OpenbookQA - Observed inference time (s)": "{\"description\": \"min=0.281, mean=0.281, max=0.281, sum=0.281 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.2806105532646179\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=242.846, mean=242.846, max=242.846, sum=242.846 (1)\", \"tab\": \"General information\", \"score\": \"242.846\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "dataset": "\"openbookqa\"", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.569, - "details": { - "description": "min=0.39, mean=0.569, max=0.84, sum=2.847 (5)", - "tab": "Accuracy", - "MMLU - Observed inference time (s)": "{\"description\": \"min=0.281, mean=0.289, max=0.298, sum=1.447 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.28946571837810053\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=371.19, mean=470.836, max=620.939, sum=2354.179 (5)\", \"tab\": \"General information\", \"score\": \"470.8357192982456\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"college_chemistry\", \"computer_security\", \"econometrics\", \"us_foreign_policy\"]", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MATH", - "source_data": { - "dataset_name": "MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Equivalent (CoT) on MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.561, - "details": { - "description": "min=0.462, mean=0.561, max=0.726, sum=3.928 (7)", - "tab": "Accuracy", - "MATH - Observed inference time (s)": "{\"description\": \"min=2.593, mean=2.933, max=3.209, sum=20.53 (7)\", \"tab\": \"Efficiency\", \"score\": \"2.9328109453469335\"}", - "MATH - # eval": "{\"description\": \"min=30, mean=62.429, max=135, sum=437 (7)\", \"tab\": \"General information\", \"score\": \"62.42857142857143\"}", - "MATH - # train": "{\"description\": \"min=8, mean=8, max=8, sum=56 (7)\", \"tab\": \"General information\", \"score\": \"8.0\"}", - "MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MATH - # prompt tokens": "{\"description\": \"min=937.926, mean=1323.837, max=2246.673, sum=9266.858 (7)\", \"tab\": \"General information\", \"score\": \"1323.836848955025\"}", - "MATH - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=7 (7)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"algebra\", \"counting_and_probability\", \"geometry\", \"intermediate_algebra\", \"number_theory\", \"prealgebra\", \"precalculus\"]", - "level": "\"1\"", - "use_official_examples": "\"False\"", - "use_chain_of_thought": "\"True\"" - } - } - }, - { - "evaluation_name": "GSM8K", - "source_data": { - "dataset_name": "GSM8K", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on GSM8K", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6, - "details": { - "description": "min=0.6, mean=0.6, max=0.6, sum=0.6 (1)", - "tab": "Accuracy", - "GSM8K - Observed inference time (s)": "{\"description\": \"min=1.381, mean=1.381, max=1.381, sum=1.381 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.380831289768219\"}", - "GSM8K - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "GSM8K - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "GSM8K - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GSM8K - # prompt tokens": "{\"description\": \"min=1130.403, mean=1130.403, max=1130.403, sum=1130.403 (1)\", \"tab\": \"General information\", \"score\": \"1130.403\"}", - "GSM8K - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "LegalBench", - "source_data": { - "dataset_name": "LegalBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on LegalBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.523, - "details": { - "description": "min=0.253, mean=0.523, max=0.716, sum=2.614 (5)", - "tab": "Accuracy", - "LegalBench - Observed inference time (s)": "{\"description\": \"min=0.298, mean=0.44, max=0.946, sum=2.2 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.4400657887452306\"}", - "LegalBench - # eval": "{\"description\": \"min=95, mean=409.4, max=1000, sum=2047 (5)\", \"tab\": \"General information\", \"score\": \"409.4\"}", - "LegalBench - # train": "{\"description\": \"min=4, mean=4.8, max=5, sum=24 (5)\", \"tab\": \"General information\", \"score\": \"4.8\"}", - "LegalBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "LegalBench - # prompt tokens": "{\"description\": \"min=192.453, mean=1542.088, max=6430.714, sum=7710.442 (5)\", \"tab\": \"General information\", \"score\": \"1542.0883229968654\"}", - "LegalBench - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"abercrombie\", \"corporate_lobbying\", \"function_of_decision_section\", \"international_citizenship_questions\", \"proa\"]" - } - } - }, - { - "evaluation_name": "MedQA", - "source_data": { - "dataset_name": "MedQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MedQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.479, - "details": { - "description": "min=0.479, mean=0.479, max=0.479, sum=0.479 (1)", - "tab": "Accuracy", - "MedQA - Observed inference time (s)": "{\"description\": \"min=0.298, mean=0.298, max=0.298, sum=0.298 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.2983713296962306\"}", - "MedQA - # eval": "{\"description\": \"min=503, mean=503, max=503, sum=503 (1)\", \"tab\": \"General information\", \"score\": \"503.0\"}", - "MedQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MedQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MedQA - # prompt tokens": "{\"description\": \"min=1045.485, mean=1045.485, max=1045.485, sum=1045.485 (1)\", \"tab\": \"General information\", \"score\": \"1045.4850894632207\"}", - "MedQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WMT 2014", - "source_data": { - "dataset_name": "WMT 2014", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "BLEU-4 on WMT 2014", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.153, - "details": { - "description": "min=0.082, mean=0.153, max=0.19, sum=0.767 (5)", - "tab": "Accuracy", - "WMT 2014 - Observed inference time (s)": "{\"description\": \"min=0.461, mean=0.484, max=0.517, sum=2.421 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.4841760334465496\"}", - "WMT 2014 - # eval": "{\"description\": \"min=503, mean=568.8, max=832, sum=2844 (5)\", \"tab\": \"General information\", \"score\": \"568.8\"}", - "WMT 2014 - # train": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "WMT 2014 - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WMT 2014 - # prompt tokens": "{\"description\": \"min=108.855, mean=126.657, max=142.373, sum=633.283 (5)\", \"tab\": \"General information\", \"score\": \"126.65662658663405\"}", - "WMT 2014 - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "language_pair": "[\"cs-en\", \"de-en\", \"fr-en\", \"hi-en\", \"ru-en\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_mmlu/qwen_qwen1.5-7b/1774096312.00548", - "retrieved_timestamp": "1774096312.00548", - "source_metadata": { - "source_name": "helm_mmlu", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_mmlu", - "evaluation_results": [ - { - "evaluation_name": "MMLU All Subjects", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU All Subjects", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.626, - "details": { - "description": "min=0.364, mean=0.626, max=0.863, sum=71.339 (114)", - "tab": "Accuracy", - "MMLU All Subjects - Observed inference time (s)": "{\"description\": \"min=0.269, mean=0.302, max=0.42, sum=34.377 (114)\", \"tab\": \"Efficiency\", \"score\": \"0.3015485066726155\"}", - "MMLU All Subjects - # eval": "{\"description\": \"min=100, mean=246.351, max=1534, sum=28084 (114)\", \"tab\": \"General information\", \"score\": \"246.35087719298247\"}", - "MMLU All Subjects - # train": "{\"description\": \"min=5, mean=5, max=5, sum=570 (114)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU All Subjects - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU All Subjects - # prompt tokens": "{\"description\": \"min=269.07, mean=618.598, max=2807.903, sum=70520.198 (114)\", \"tab\": \"General information\", \"score\": \"618.5982315160392\"}", - "MMLU All Subjects - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=114 (114)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_name": "Abstract Algebra", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Abstract Algebra", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.39, - "details": { - "description": "min=0.39, mean=0.39, max=0.39, sum=0.78 (2)", - "tab": "Accuracy", - "Abstract Algebra - Observed inference time (s)": "{\"description\": \"min=0.281, mean=0.281, max=0.281, sum=0.562 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.28086970567703246\"}", - "Abstract Algebra - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Abstract Algebra - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Abstract Algebra - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Abstract Algebra - # prompt tokens": "{\"description\": \"min=371.19, mean=371.19, max=371.19, sum=742.38 (2)\", \"tab\": \"General information\", \"score\": \"371.19\"}", - "Abstract Algebra - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"abstract_algebra\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_abstract_algebra\"" - } - } - }, - { - "evaluation_name": "Anatomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Anatomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.526, - "details": { - "description": "min=0.526, mean=0.526, max=0.526, sum=1.052 (2)", - "tab": "Accuracy", - "Anatomy - Observed inference time (s)": "{\"description\": \"min=0.286, mean=0.286, max=0.286, sum=0.572 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2861745004300718\"}", - "Anatomy - # eval": "{\"description\": \"min=135, mean=135, max=135, sum=270 (2)\", \"tab\": \"General information\", \"score\": \"135.0\"}", - "Anatomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Anatomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Anatomy - # prompt tokens": "{\"description\": \"min=346.978, mean=346.978, max=346.978, sum=693.956 (2)\", \"tab\": \"General information\", \"score\": \"346.97777777777776\"}", - "Anatomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"anatomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_anatomy\"" - } - } - }, - { - "evaluation_name": "College Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on College Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.471, - "details": { - "description": "min=0.471, mean=0.471, max=0.471, sum=0.941 (2)", - "tab": "Accuracy", - "College Chemistry - Observed inference time (s)": "{\"description\": \"min=0.296, mean=0.296, max=0.296, sum=0.592 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2962386703491211\"}", - "College Biology - Observed inference time (s)": "{\"description\": \"min=0.312, mean=0.312, max=0.312, sum=0.624 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3117961171600554\"}", - "College Computer Science - Observed inference time (s)": "{\"description\": \"min=0.3, mean=0.3, max=0.3, sum=0.599 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.299501326084137\"}", - "College Mathematics - Observed inference time (s)": "{\"description\": \"min=0.303, mean=0.303, max=0.303, sum=0.607 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3033126187324524\"}", - "College Medicine - Observed inference time (s)": "{\"description\": \"min=0.289, mean=0.289, max=0.289, sum=0.577 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2886359746745556\"}", - "College Physics - Observed inference time (s)": "{\"description\": \"min=0.322, mean=0.322, max=0.322, sum=0.643 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.32153993026882993\"}", - "College Chemistry - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Chemistry - # prompt tokens": "{\"description\": \"min=561.25, mean=561.25, max=561.25, sum=1122.5 (2)\", \"tab\": \"General information\", \"score\": \"561.25\"}", - "College Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Biology - # eval": "{\"description\": \"min=144, mean=144, max=144, sum=288 (2)\", \"tab\": \"General information\", \"score\": \"144.0\"}", - "College Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # prompt tokens": "{\"description\": \"min=479.979, mean=479.979, max=479.979, sum=959.958 (2)\", \"tab\": \"General information\", \"score\": \"479.9791666666667\"}", - "College Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # prompt tokens": "{\"description\": \"min=831.58, mean=831.58, max=831.58, sum=1663.16 (2)\", \"tab\": \"General information\", \"score\": \"831.58\"}", - "College Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Mathematics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # prompt tokens": "{\"description\": \"min=600.7, mean=600.7, max=600.7, sum=1201.4 (2)\", \"tab\": \"General information\", \"score\": \"600.7\"}", - "College Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Medicine - # eval": "{\"description\": \"min=173, mean=173, max=173, sum=346 (2)\", \"tab\": \"General information\", \"score\": \"173.0\"}", - "College Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # prompt tokens": "{\"description\": \"min=499.098, mean=499.098, max=499.098, sum=998.197 (2)\", \"tab\": \"General information\", \"score\": \"499.0982658959538\"}", - "College Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Physics - # eval": "{\"description\": \"min=102, mean=102, max=102, sum=204 (2)\", \"tab\": \"General information\", \"score\": \"102.0\"}", - "College Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # prompt tokens": "{\"description\": \"min=509.265, mean=509.265, max=509.265, sum=1018.529 (2)\", \"tab\": \"General information\", \"score\": \"509.2647058823529\"}", - "College Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"college_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_college_physics\"" - } - } - }, - { - "evaluation_name": "Computer Security", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Computer Security", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.76, - "details": { - "description": "min=0.76, mean=0.76, max=0.76, sum=1.52 (2)", - "tab": "Accuracy", - "Computer Security - Observed inference time (s)": "{\"description\": \"min=0.298, mean=0.298, max=0.298, sum=0.597 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2982983756065369\"}", - "Computer Security - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Computer Security - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Computer Security - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Computer Security - # prompt tokens": "{\"description\": \"min=379.64, mean=379.64, max=379.64, sum=759.28 (2)\", \"tab\": \"General information\", \"score\": \"379.64\"}", - "Computer Security - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"computer_security\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_computer_security\"" - } - } - }, - { - "evaluation_name": "Econometrics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Econometrics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.447, - "details": { - "description": "min=0.447, mean=0.447, max=0.447, sum=0.895 (2)", - "tab": "Accuracy", - "Econometrics - Observed inference time (s)": "{\"description\": \"min=0.283, mean=0.283, max=0.283, sum=0.566 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.282820323057342\"}", - "Econometrics - # eval": "{\"description\": \"min=114, mean=114, max=114, sum=228 (2)\", \"tab\": \"General information\", \"score\": \"114.0\"}", - "Econometrics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Econometrics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Econometrics - # prompt tokens": "{\"description\": \"min=620.939, mean=620.939, max=620.939, sum=1241.877 (2)\", \"tab\": \"General information\", \"score\": \"620.938596491228\"}", - "Econometrics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"econometrics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_econometrics\"" - } - } - }, - { - "evaluation_name": "Global Facts", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Global Facts", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4, - "details": { - "description": "min=0.4, mean=0.4, max=0.4, sum=0.8 (2)", - "tab": "Accuracy", - "Global Facts - Observed inference time (s)": "{\"description\": \"min=0.294, mean=0.294, max=0.294, sum=0.588 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2939557838439941\"}", - "Global Facts - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Global Facts - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Global Facts - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Global Facts - # prompt tokens": "{\"description\": \"min=422.06, mean=422.06, max=422.06, sum=844.12 (2)\", \"tab\": \"General information\", \"score\": \"422.06\"}", - "Global Facts - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"global_facts\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_global_facts\"" - } - } - }, - { - "evaluation_name": "Jurisprudence", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Jurisprudence", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.778, - "details": { - "description": "min=0.778, mean=0.778, max=0.778, sum=1.556 (2)", - "tab": "Accuracy", - "Jurisprudence - Observed inference time (s)": "{\"description\": \"min=0.297, mean=0.297, max=0.297, sum=0.593 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2966193402255023\"}", - "Jurisprudence - # eval": "{\"description\": \"min=108, mean=108, max=108, sum=216 (2)\", \"tab\": \"General information\", \"score\": \"108.0\"}", - "Jurisprudence - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Jurisprudence - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Jurisprudence - # prompt tokens": "{\"description\": \"min=387.713, mean=387.713, max=387.713, sum=775.426 (2)\", \"tab\": \"General information\", \"score\": \"387.712962962963\"}", - "Jurisprudence - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"jurisprudence\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_jurisprudence\"" - } - } - }, - { - "evaluation_name": "Philosophy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Philosophy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.691, - "details": { - "description": "min=0.691, mean=0.691, max=0.691, sum=1.383 (2)", - "tab": "Accuracy", - "Philosophy - Observed inference time (s)": "{\"description\": \"min=0.287, mean=0.287, max=0.287, sum=0.575 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.28725898534155353\"}", - "Philosophy - # eval": "{\"description\": \"min=311, mean=311, max=311, sum=622 (2)\", \"tab\": \"General information\", \"score\": \"311.0\"}", - "Philosophy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Philosophy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Philosophy - # prompt tokens": "{\"description\": \"min=322.09, mean=322.09, max=322.09, sum=644.18 (2)\", \"tab\": \"General information\", \"score\": \"322.09003215434086\"}", - "Philosophy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"philosophy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_philosophy\"" - } - } - }, - { - "evaluation_name": "Professional Psychology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Professional Psychology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.603, - "details": { - "description": "min=0.603, mean=0.603, max=0.603, sum=1.206 (2)", - "tab": "Accuracy", - "Professional Medicine - Observed inference time (s)": "{\"description\": \"min=0.309, mean=0.309, max=0.309, sum=0.617 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.30863527515355277\"}", - "Professional Accounting - Observed inference time (s)": "{\"description\": \"min=0.293, mean=0.293, max=0.293, sum=0.585 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2926285613513162\"}", - "Professional Law - Observed inference time (s)": "{\"description\": \"min=0.323, mean=0.323, max=0.323, sum=0.645 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.32274515889925004\"}", - "Professional Psychology - Observed inference time (s)": "{\"description\": \"min=0.303, mean=0.303, max=0.303, sum=0.607 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.30344173058964846\"}", - "Professional Medicine - # eval": "{\"description\": \"min=272, mean=272, max=272, sum=544 (2)\", \"tab\": \"General information\", \"score\": \"272.0\"}", - "Professional Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Medicine - # prompt tokens": "{\"description\": \"min=1118.199, mean=1118.199, max=1118.199, sum=2236.397 (2)\", \"tab\": \"General information\", \"score\": \"1118.1985294117646\"}", - "Professional Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Accounting - # eval": "{\"description\": \"min=282, mean=282, max=282, sum=564 (2)\", \"tab\": \"General information\", \"score\": \"282.0\"}", - "Professional Accounting - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Accounting - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # prompt tokens": "{\"description\": \"min=732.34, mean=732.34, max=732.34, sum=1464.681 (2)\", \"tab\": \"General information\", \"score\": \"732.3404255319149\"}", - "Professional Accounting - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Law - # eval": "{\"description\": \"min=1534, mean=1534, max=1534, sum=3068 (2)\", \"tab\": \"General information\", \"score\": \"1534.0\"}", - "Professional Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # prompt tokens": "{\"description\": \"min=1656.969, mean=1656.969, max=1656.969, sum=3313.939 (2)\", \"tab\": \"General information\", \"score\": \"1656.9693611473272\"}", - "Professional Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Psychology - # eval": "{\"description\": \"min=612, mean=612, max=612, sum=1224 (2)\", \"tab\": \"General information\", \"score\": \"612.0\"}", - "Professional Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # prompt tokens": "{\"description\": \"min=574.417, mean=574.417, max=574.417, sum=1148.833 (2)\", \"tab\": \"General information\", \"score\": \"574.4166666666666\"}", - "Professional Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"professional_psychology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_professional_psychology\"" - } - } - }, - { - "evaluation_name": "Us Foreign Policy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Us Foreign Policy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.84, - "details": { - "description": "min=0.84, mean=0.84, max=0.84, sum=1.68 (2)", - "tab": "Accuracy", - "Us Foreign Policy - Observed inference time (s)": "{\"description\": \"min=0.289, mean=0.289, max=0.289, sum=0.578 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.28910151720046995\"}", - "Us Foreign Policy - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Us Foreign Policy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Us Foreign Policy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Us Foreign Policy - # prompt tokens": "{\"description\": \"min=421.16, mean=421.16, max=421.16, sum=842.32 (2)\", \"tab\": \"General information\", \"score\": \"421.16\"}", - "Us Foreign Policy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"us_foreign_policy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_us_foreign_policy\"" - } - } - }, - { - "evaluation_name": "Astronomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Astronomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.671, - "details": { - "description": "min=0.671, mean=0.671, max=0.671, sum=1.342 (2)", - "tab": "Accuracy", - "Astronomy - Observed inference time (s)": "{\"description\": \"min=0.307, mean=0.307, max=0.307, sum=0.614 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.30717346699614273\"}", - "Astronomy - # eval": "{\"description\": \"min=152, mean=152, max=152, sum=304 (2)\", \"tab\": \"General information\", \"score\": \"152.0\"}", - "Astronomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Astronomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Astronomy - # prompt tokens": "{\"description\": \"min=582.849, mean=582.849, max=582.849, sum=1165.697 (2)\", \"tab\": \"General information\", \"score\": \"582.8486842105264\"}", - "Astronomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"astronomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_astronomy\"" - } - } - }, - { - "evaluation_name": "Business Ethics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Business Ethics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.69, - "details": { - "description": "min=0.69, mean=0.69, max=0.69, sum=1.38 (2)", - "tab": "Accuracy", - "Business Ethics - Observed inference time (s)": "{\"description\": \"min=0.306, mean=0.306, max=0.306, sum=0.612 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3062057161331177\"}", - "Business Ethics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Business Ethics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Business Ethics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Business Ethics - # prompt tokens": "{\"description\": \"min=562.87, mean=562.87, max=562.87, sum=1125.74 (2)\", \"tab\": \"General information\", \"score\": \"562.87\"}", - "Business Ethics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"business_ethics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_business_ethics\"" - } - } - }, - { - "evaluation_name": "Clinical Knowledge", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Clinical Knowledge", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.691, - "details": { - "description": "min=0.691, mean=0.691, max=0.691, sum=1.381 (2)", - "tab": "Accuracy", - "Clinical Knowledge - Observed inference time (s)": "{\"description\": \"min=0.295, mean=0.295, max=0.295, sum=0.589 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2947473319071644\"}", - "Clinical Knowledge - # eval": "{\"description\": \"min=265, mean=265, max=265, sum=530 (2)\", \"tab\": \"General information\", \"score\": \"265.0\"}", - "Clinical Knowledge - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Clinical Knowledge - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Clinical Knowledge - # prompt tokens": "{\"description\": \"min=393.623, mean=393.623, max=393.623, sum=787.245 (2)\", \"tab\": \"General information\", \"score\": \"393.62264150943395\"}", - "Clinical Knowledge - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"clinical_knowledge\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_clinical_knowledge\"" - } - } - }, - { - "evaluation_name": "Conceptual Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Conceptual Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.579, - "details": { - "description": "min=0.579, mean=0.579, max=0.579, sum=1.157 (2)", - "tab": "Accuracy", - "Conceptual Physics - Observed inference time (s)": "{\"description\": \"min=0.28, mean=0.28, max=0.28, sum=0.561 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2803657531738281\"}", - "Conceptual Physics - # eval": "{\"description\": \"min=235, mean=235, max=235, sum=470 (2)\", \"tab\": \"General information\", \"score\": \"235.0\"}", - "Conceptual Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Conceptual Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Conceptual Physics - # prompt tokens": "{\"description\": \"min=298.494, mean=298.494, max=298.494, sum=596.987 (2)\", \"tab\": \"General information\", \"score\": \"298.4936170212766\"}", - "Conceptual Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"conceptual_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_conceptual_physics\"" - } - } - }, - { - "evaluation_name": "Electrical Engineering", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Electrical Engineering", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.572, - "details": { - "description": "min=0.572, mean=0.572, max=0.572, sum=1.145 (2)", - "tab": "Accuracy", - "Electrical Engineering - Observed inference time (s)": "{\"description\": \"min=0.269, mean=0.269, max=0.269, sum=0.539 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2693853361853238\"}", - "Electrical Engineering - # eval": "{\"description\": \"min=145, mean=145, max=145, sum=290 (2)\", \"tab\": \"General information\", \"score\": \"145.0\"}", - "Electrical Engineering - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Electrical Engineering - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Electrical Engineering - # prompt tokens": "{\"description\": \"min=456.8, mean=456.8, max=456.8, sum=913.6 (2)\", \"tab\": \"General information\", \"score\": \"456.8\"}", - "Electrical Engineering - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"electrical_engineering\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_electrical_engineering\"" - } - } - }, - { - "evaluation_name": "Elementary Mathematics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Elementary Mathematics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5, - "details": { - "description": "min=0.5, mean=0.5, max=0.5, sum=1 (2)", - "tab": "Accuracy", - "Elementary Mathematics - Observed inference time (s)": "{\"description\": \"min=0.294, mean=0.294, max=0.294, sum=0.588 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2938981220204994\"}", - "Elementary Mathematics - # eval": "{\"description\": \"min=378, mean=378, max=378, sum=756 (2)\", \"tab\": \"General information\", \"score\": \"378.0\"}", - "Elementary Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Elementary Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Elementary Mathematics - # prompt tokens": "{\"description\": \"min=570.119, mean=570.119, max=570.119, sum=1140.238 (2)\", \"tab\": \"General information\", \"score\": \"570.1190476190476\"}", - "Elementary Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"elementary_mathematics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_elementary_mathematics\"" - } - } - }, - { - "evaluation_name": "Formal Logic", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Formal Logic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.397, - "details": { - "description": "min=0.397, mean=0.397, max=0.397, sum=0.794 (2)", - "tab": "Accuracy", - "Formal Logic - Observed inference time (s)": "{\"description\": \"min=0.3, mean=0.3, max=0.3, sum=0.601 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.300293557227604\"}", - "Formal Logic - # eval": "{\"description\": \"min=126, mean=126, max=126, sum=252 (2)\", \"tab\": \"General information\", \"score\": \"126.0\"}", - "Formal Logic - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Formal Logic - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Formal Logic - # prompt tokens": "{\"description\": \"min=597.667, mean=597.667, max=597.667, sum=1195.333 (2)\", \"tab\": \"General information\", \"score\": \"597.6666666666666\"}", - "Formal Logic - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"formal_logic\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_formal_logic\"" - } - } - }, - { - "evaluation_name": "High School World History", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on High School World History", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.789, - "details": { - "description": "min=0.789, mean=0.789, max=0.789, sum=1.578 (2)", - "tab": "Accuracy", - "High School Biology - Observed inference time (s)": "{\"description\": \"min=0.303, mean=0.303, max=0.303, sum=0.605 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.30256526470184325\"}", - "High School Chemistry - Observed inference time (s)": "{\"description\": \"min=0.293, mean=0.293, max=0.293, sum=0.585 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.29262745321677824\"}", - "High School Computer Science - Observed inference time (s)": "{\"description\": \"min=0.304, mean=0.304, max=0.304, sum=0.608 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3042095494270325\"}", - "High School European History - Observed inference time (s)": "{\"description\": \"min=0.42, mean=0.42, max=0.42, sum=0.839 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4195035573207971\"}", - "High School Geography - Observed inference time (s)": "{\"description\": \"min=0.303, mean=0.303, max=0.303, sum=0.605 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3027432386321251\"}", - "High School Government And Politics - Observed inference time (s)": "{\"description\": \"min=0.294, mean=0.294, max=0.294, sum=0.589 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.29444977903613156\"}", - "High School Macroeconomics - Observed inference time (s)": "{\"description\": \"min=0.291, mean=0.291, max=0.291, sum=0.582 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2909054010342329\"}", - "High School Mathematics - Observed inference time (s)": "{\"description\": \"min=0.293, mean=0.293, max=0.293, sum=0.585 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.29262985565044264\"}", - "High School Microeconomics - Observed inference time (s)": "{\"description\": \"min=0.304, mean=0.304, max=0.304, sum=0.608 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3041165916859603\"}", - "High School Physics - Observed inference time (s)": "{\"description\": \"min=0.301, mean=0.301, max=0.301, sum=0.603 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3013988425400083\"}", - "High School Psychology - Observed inference time (s)": "{\"description\": \"min=0.309, mean=0.309, max=0.309, sum=0.618 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3090610066685108\"}", - "High School Statistics - Observed inference time (s)": "{\"description\": \"min=0.318, mean=0.318, max=0.318, sum=0.635 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.31764531577074967\"}", - "High School US History - Observed inference time (s)": "{\"description\": \"min=0.364, mean=0.364, max=0.364, sum=0.727 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3635554044854407\"}", - "High School World History - Observed inference time (s)": "{\"description\": \"min=0.323, mean=0.323, max=0.323, sum=0.646 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.32297819073190165\"}", - "High School Biology - # eval": "{\"description\": \"min=310, mean=310, max=310, sum=620 (2)\", \"tab\": \"General information\", \"score\": \"310.0\"}", - "High School Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Biology - # prompt tokens": "{\"description\": \"min=506.916, mean=506.916, max=506.916, sum=1013.832 (2)\", \"tab\": \"General information\", \"score\": \"506.9161290322581\"}", - "High School Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Chemistry - # eval": "{\"description\": \"min=203, mean=203, max=203, sum=406 (2)\", \"tab\": \"General information\", \"score\": \"203.0\"}", - "High School Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # prompt tokens": "{\"description\": \"min=510.261, mean=510.261, max=510.261, sum=1020.522 (2)\", \"tab\": \"General information\", \"score\": \"510.2610837438424\"}", - "High School Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "High School Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # prompt tokens": "{\"description\": \"min=871.46, mean=871.46, max=871.46, sum=1742.92 (2)\", \"tab\": \"General information\", \"score\": \"871.46\"}", - "High School Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School European History - # eval": "{\"description\": \"min=165, mean=165, max=165, sum=330 (2)\", \"tab\": \"General information\", \"score\": \"165.0\"}", - "High School European History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School European History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # prompt tokens": "{\"description\": \"min=2807.903, mean=2807.903, max=2807.903, sum=5615.806 (2)\", \"tab\": \"General information\", \"score\": \"2807.9030303030304\"}", - "High School European History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Geography - # eval": "{\"description\": \"min=198, mean=198, max=198, sum=396 (2)\", \"tab\": \"General information\", \"score\": \"198.0\"}", - "High School Geography - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Geography - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # prompt tokens": "{\"description\": \"min=365.217, mean=365.217, max=365.217, sum=730.434 (2)\", \"tab\": \"General information\", \"score\": \"365.2171717171717\"}", - "High School Geography - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Government And Politics - # eval": "{\"description\": \"min=193, mean=193, max=193, sum=386 (2)\", \"tab\": \"General information\", \"score\": \"193.0\"}", - "High School Government And Politics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Government And Politics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # prompt tokens": "{\"description\": \"min=460.311, mean=460.311, max=460.311, sum=920.622 (2)\", \"tab\": \"General information\", \"score\": \"460.31088082901556\"}", - "High School Government And Politics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Macroeconomics - # eval": "{\"description\": \"min=390, mean=390, max=390, sum=780 (2)\", \"tab\": \"General information\", \"score\": \"390.0\"}", - "High School Macroeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Macroeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # prompt tokens": "{\"description\": \"min=367.349, mean=367.349, max=367.349, sum=734.697 (2)\", \"tab\": \"General information\", \"score\": \"367.34871794871793\"}", - "High School Macroeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Mathematics - # eval": "{\"description\": \"min=270, mean=270, max=270, sum=540 (2)\", \"tab\": \"General information\", \"score\": \"270.0\"}", - "High School Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # prompt tokens": "{\"description\": \"min=558.326, mean=558.326, max=558.326, sum=1116.652 (2)\", \"tab\": \"General information\", \"score\": \"558.325925925926\"}", - "High School Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Microeconomics - # eval": "{\"description\": \"min=238, mean=238, max=238, sum=476 (2)\", \"tab\": \"General information\", \"score\": \"238.0\"}", - "High School Microeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Microeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # prompt tokens": "{\"description\": \"min=395.277, mean=395.277, max=395.277, sum=790.555 (2)\", \"tab\": \"General information\", \"score\": \"395.2773109243698\"}", - "High School Microeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Physics - # eval": "{\"description\": \"min=151, mean=151, max=151, sum=302 (2)\", \"tab\": \"General information\", \"score\": \"151.0\"}", - "High School Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # prompt tokens": "{\"description\": \"min=573.536, mean=573.536, max=573.536, sum=1147.073 (2)\", \"tab\": \"General information\", \"score\": \"573.5364238410596\"}", - "High School Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Psychology - # eval": "{\"description\": \"min=545, mean=545, max=545, sum=1090 (2)\", \"tab\": \"General information\", \"score\": \"545.0\"}", - "High School Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # prompt tokens": "{\"description\": \"min=488.521, mean=488.521, max=488.521, sum=977.042 (2)\", \"tab\": \"General information\", \"score\": \"488.52110091743117\"}", - "High School Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Statistics - # eval": "{\"description\": \"min=216, mean=216, max=216, sum=432 (2)\", \"tab\": \"General information\", \"score\": \"216.0\"}", - "High School Statistics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Statistics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # prompt tokens": "{\"description\": \"min=823.477, mean=823.477, max=823.477, sum=1646.954 (2)\", \"tab\": \"General information\", \"score\": \"823.4768518518518\"}", - "High School Statistics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School US History - # eval": "{\"description\": \"min=204, mean=204, max=204, sum=408 (2)\", \"tab\": \"General information\", \"score\": \"204.0\"}", - "High School US History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School US History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # prompt tokens": "{\"description\": \"min=2230.176, mean=2230.176, max=2230.176, sum=4460.353 (2)\", \"tab\": \"General information\", \"score\": \"2230.176470588235\"}", - "High School US History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School World History - # eval": "{\"description\": \"min=237, mean=237, max=237, sum=474 (2)\", \"tab\": \"General information\", \"score\": \"237.0\"}", - "High School World History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School World History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # prompt tokens": "{\"description\": \"min=1441.354, mean=1441.354, max=1441.354, sum=2882.709 (2)\", \"tab\": \"General information\", \"score\": \"1441.3544303797469\"}", - "High School World History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"high_school_world_history\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_high_school_world_history\"" - } - } - }, - { - "evaluation_name": "Human Sexuality", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Human Sexuality", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.695, - "details": { - "description": "min=0.695, mean=0.695, max=0.695, sum=1.389 (2)", - "tab": "Accuracy", - "Human Aging - Observed inference time (s)": "{\"description\": \"min=0.289, mean=0.289, max=0.289, sum=0.578 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.28891397057092777\"}", - "Human Sexuality - Observed inference time (s)": "{\"description\": \"min=0.298, mean=0.298, max=0.298, sum=0.596 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2980237170940137\"}", - "Human Aging - # eval": "{\"description\": \"min=223, mean=223, max=223, sum=446 (2)\", \"tab\": \"General information\", \"score\": \"223.0\"}", - "Human Aging - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Aging - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Aging - # prompt tokens": "{\"description\": \"min=315.121, mean=315.121, max=315.121, sum=630.242 (2)\", \"tab\": \"General information\", \"score\": \"315.1210762331838\"}", - "Human Aging - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Human Sexuality - # eval": "{\"description\": \"min=131, mean=131, max=131, sum=262 (2)\", \"tab\": \"General information\", \"score\": \"131.0\"}", - "Human Sexuality - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Sexuality - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # prompt tokens": "{\"description\": \"min=334.504, mean=334.504, max=334.504, sum=669.008 (2)\", \"tab\": \"General information\", \"score\": \"334.5038167938931\"}", - "Human Sexuality - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"human_sexuality\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_human_sexuality\"" - } - } - }, - { - "evaluation_name": "International Law", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on International Law", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.76, - "details": { - "description": "min=0.76, mean=0.76, max=0.76, sum=1.521 (2)", - "tab": "Accuracy", - "International Law - Observed inference time (s)": "{\"description\": \"min=0.299, mean=0.299, max=0.299, sum=0.599 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2993730572629566\"}", - "International Law - # eval": "{\"description\": \"min=121, mean=121, max=121, sum=242 (2)\", \"tab\": \"General information\", \"score\": \"121.0\"}", - "International Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "International Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "International Law - # prompt tokens": "{\"description\": \"min=633.579, mean=633.579, max=633.579, sum=1267.157 (2)\", \"tab\": \"General information\", \"score\": \"633.5785123966942\"}", - "International Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"international_law\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_international_law\"" - } - } - }, - { - "evaluation_name": "Logical Fallacies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Logical Fallacies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.706, - "details": { - "description": "min=0.706, mean=0.706, max=0.706, sum=1.411 (2)", - "tab": "Accuracy", - "Logical Fallacies - Observed inference time (s)": "{\"description\": \"min=0.283, mean=0.283, max=0.283, sum=0.566 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.28320794456575543\"}", - "Logical Fallacies - # eval": "{\"description\": \"min=163, mean=163, max=163, sum=326 (2)\", \"tab\": \"General information\", \"score\": \"163.0\"}", - "Logical Fallacies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Logical Fallacies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Logical Fallacies - # prompt tokens": "{\"description\": \"min=442.632, mean=442.632, max=442.632, sum=885.264 (2)\", \"tab\": \"General information\", \"score\": \"442.6319018404908\"}", - "Logical Fallacies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"logical_fallacies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_logical_fallacies\"" - } - } - }, - { - "evaluation_name": "Machine Learning", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Machine Learning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.411, - "details": { - "description": "min=0.411, mean=0.411, max=0.411, sum=0.821 (2)", - "tab": "Accuracy", - "Machine Learning - Observed inference time (s)": "{\"description\": \"min=0.292, mean=0.292, max=0.292, sum=0.583 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2917012700012752\"}", - "Machine Learning - # eval": "{\"description\": \"min=112, mean=112, max=112, sum=224 (2)\", \"tab\": \"General information\", \"score\": \"112.0\"}", - "Machine Learning - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Machine Learning - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Machine Learning - # prompt tokens": "{\"description\": \"min=674.848, mean=674.848, max=674.848, sum=1349.696 (2)\", \"tab\": \"General information\", \"score\": \"674.8482142857143\"}", - "Machine Learning - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"machine_learning\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_machine_learning\"" - } - } - }, - { - "evaluation_name": "Management", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Management", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.816, - "details": { - "description": "min=0.816, mean=0.816, max=0.816, sum=1.631 (2)", - "tab": "Accuracy", - "Management - Observed inference time (s)": "{\"description\": \"min=0.304, mean=0.304, max=0.304, sum=0.607 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3037459641984365\"}", - "Management - # eval": "{\"description\": \"min=103, mean=103, max=103, sum=206 (2)\", \"tab\": \"General information\", \"score\": \"103.0\"}", - "Management - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Management - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Management - # prompt tokens": "{\"description\": \"min=276.854, mean=276.854, max=276.854, sum=553.709 (2)\", \"tab\": \"General information\", \"score\": \"276.8543689320388\"}", - "Management - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"management\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_management\"" - } - } - }, - { - "evaluation_name": "Marketing", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Marketing", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.863, - "details": { - "description": "min=0.863, mean=0.863, max=0.863, sum=1.726 (2)", - "tab": "Accuracy", - "Marketing - Observed inference time (s)": "{\"description\": \"min=0.304, mean=0.304, max=0.304, sum=0.608 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.30402050364730704\"}", - "Marketing - # eval": "{\"description\": \"min=234, mean=234, max=234, sum=468 (2)\", \"tab\": \"General information\", \"score\": \"234.0\"}", - "Marketing - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Marketing - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Marketing - # prompt tokens": "{\"description\": \"min=397.415, mean=397.415, max=397.415, sum=794.829 (2)\", \"tab\": \"General information\", \"score\": \"397.4145299145299\"}", - "Marketing - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"marketing\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_marketing\"" - } - } - }, - { - "evaluation_name": "Medical Genetics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Medical Genetics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.69, - "details": { - "description": "min=0.69, mean=0.69, max=0.69, sum=1.38 (2)", - "tab": "Accuracy", - "Medical Genetics - Observed inference time (s)": "{\"description\": \"min=0.308, mean=0.308, max=0.308, sum=0.616 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3079418969154358\"}", - "Medical Genetics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Medical Genetics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Medical Genetics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Medical Genetics - # prompt tokens": "{\"description\": \"min=335.35, mean=335.35, max=335.35, sum=670.7 (2)\", \"tab\": \"General information\", \"score\": \"335.35\"}", - "Medical Genetics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"medical_genetics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_medical_genetics\"" - } - } - }, - { - "evaluation_name": "Miscellaneous", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Miscellaneous", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.765, - "details": { - "description": "min=0.765, mean=0.765, max=0.765, sum=1.53 (2)", - "tab": "Accuracy", - "Miscellaneous - Observed inference time (s)": "{\"description\": \"min=0.287, mean=0.287, max=0.287, sum=0.575 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2874623727372171\"}", - "Miscellaneous - # eval": "{\"description\": \"min=783, mean=783, max=783, sum=1566 (2)\", \"tab\": \"General information\", \"score\": \"783.0\"}", - "Miscellaneous - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Miscellaneous - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Miscellaneous - # prompt tokens": "{\"description\": \"min=296.7, mean=296.7, max=296.7, sum=593.4 (2)\", \"tab\": \"General information\", \"score\": \"296.6998722860792\"}", - "Miscellaneous - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"miscellaneous\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_miscellaneous\"" - } - } - }, - { - "evaluation_name": "Moral Scenarios", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Moral Scenarios", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.372, - "details": { - "description": "min=0.372, mean=0.372, max=0.372, sum=0.744 (2)", - "tab": "Accuracy", - "Moral Disputes - Observed inference time (s)": "{\"description\": \"min=0.294, mean=0.294, max=0.294, sum=0.587 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.29359787530292664\"}", - "Moral Scenarios - Observed inference time (s)": "{\"description\": \"min=0.298, mean=0.298, max=0.298, sum=0.596 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2979323072806417\"}", - "Moral Disputes - # eval": "{\"description\": \"min=346, mean=346, max=346, sum=692 (2)\", \"tab\": \"General information\", \"score\": \"346.0\"}", - "Moral Disputes - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Disputes - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Disputes - # prompt tokens": "{\"description\": \"min=469.182, mean=469.182, max=469.182, sum=938.364 (2)\", \"tab\": \"General information\", \"score\": \"469.1820809248555\"}", - "Moral Disputes - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Moral Scenarios - # eval": "{\"description\": \"min=895, mean=895, max=895, sum=1790 (2)\", \"tab\": \"General information\", \"score\": \"895.0\"}", - "Moral Scenarios - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Scenarios - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # prompt tokens": "{\"description\": \"min=661.494, mean=661.494, max=661.494, sum=1322.988 (2)\", \"tab\": \"General information\", \"score\": \"661.4938547486033\"}", - "Moral Scenarios - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"moral_scenarios\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_moral_scenarios\"" - } - } - }, - { - "evaluation_name": "Nutrition", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Nutrition", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.696, - "details": { - "description": "min=0.696, mean=0.696, max=0.696, sum=1.392 (2)", - "tab": "Accuracy", - "Nutrition - Observed inference time (s)": "{\"description\": \"min=0.293, mean=0.293, max=0.293, sum=0.586 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.29277056572484034\"}", - "Nutrition - # eval": "{\"description\": \"min=306, mean=306, max=306, sum=612 (2)\", \"tab\": \"General information\", \"score\": \"306.0\"}", - "Nutrition - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Nutrition - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Nutrition - # prompt tokens": "{\"description\": \"min=592.637, mean=592.637, max=592.637, sum=1185.275 (2)\", \"tab\": \"General information\", \"score\": \"592.6372549019608\"}", - "Nutrition - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"nutrition\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_nutrition\"" - } - } - }, - { - "evaluation_name": "Prehistory", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Prehistory", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.688, - "details": { - "description": "min=0.688, mean=0.688, max=0.688, sum=1.377 (2)", - "tab": "Accuracy", - "Prehistory - Observed inference time (s)": "{\"description\": \"min=0.301, mean=0.301, max=0.301, sum=0.602 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.30120949097621585\"}", - "Prehistory - # eval": "{\"description\": \"min=324, mean=324, max=324, sum=648 (2)\", \"tab\": \"General information\", \"score\": \"324.0\"}", - "Prehistory - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Prehistory - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Prehistory - # prompt tokens": "{\"description\": \"min=521.364, mean=521.364, max=521.364, sum=1042.728 (2)\", \"tab\": \"General information\", \"score\": \"521.3641975308642\"}", - "Prehistory - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"prehistory\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_prehistory\"" - } - } - }, - { - "evaluation_name": "Public Relations", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Public Relations", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.627, - "details": { - "description": "min=0.627, mean=0.627, max=0.627, sum=1.255 (2)", - "tab": "Accuracy", - "Public Relations - Observed inference time (s)": "{\"description\": \"min=0.308, mean=0.308, max=0.308, sum=0.616 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.30815364880995316\"}", - "Public Relations - # eval": "{\"description\": \"min=110, mean=110, max=110, sum=220 (2)\", \"tab\": \"General information\", \"score\": \"110.0\"}", - "Public Relations - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Public Relations - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Public Relations - # prompt tokens": "{\"description\": \"min=401.427, mean=401.427, max=401.427, sum=802.855 (2)\", \"tab\": \"General information\", \"score\": \"401.42727272727274\"}", - "Public Relations - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"public_relations\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_public_relations\"" - } - } - }, - { - "evaluation_name": "Security Studies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Security Studies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.727, - "details": { - "description": "min=0.727, mean=0.727, max=0.727, sum=1.453 (2)", - "tab": "Accuracy", - "Security Studies - Observed inference time (s)": "{\"description\": \"min=0.296, mean=0.296, max=0.296, sum=0.592 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2958566675380785\"}", - "Security Studies - # eval": "{\"description\": \"min=245, mean=245, max=245, sum=490 (2)\", \"tab\": \"General information\", \"score\": \"245.0\"}", - "Security Studies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Security Studies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Security Studies - # prompt tokens": "{\"description\": \"min=1159.931, mean=1159.931, max=1159.931, sum=2319.861 (2)\", \"tab\": \"General information\", \"score\": \"1159.930612244898\"}", - "Security Studies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"security_studies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_security_studies\"" - } - } - }, - { - "evaluation_name": "Sociology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Sociology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.836, - "details": { - "description": "min=0.836, mean=0.836, max=0.836, sum=1.672 (2)", - "tab": "Accuracy", - "Sociology - Observed inference time (s)": "{\"description\": \"min=0.299, mean=0.299, max=0.299, sum=0.598 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.29908941278410195\"}", - "Sociology - # eval": "{\"description\": \"min=201, mean=201, max=201, sum=402 (2)\", \"tab\": \"General information\", \"score\": \"201.0\"}", - "Sociology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Sociology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Sociology - # prompt tokens": "{\"description\": \"min=443.1, mean=443.1, max=443.1, sum=886.199 (2)\", \"tab\": \"General information\", \"score\": \"443.0995024875622\"}", - "Sociology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"sociology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_sociology\"" - } - } - }, - { - "evaluation_name": "Virology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Virology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.488, - "details": { - "description": "min=0.488, mean=0.488, max=0.488, sum=0.976 (2)", - "tab": "Accuracy", - "Virology - Observed inference time (s)": "{\"description\": \"min=0.286, mean=0.286, max=0.286, sum=0.572 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2861345144639532\"}", - "Virology - # eval": "{\"description\": \"min=166, mean=166, max=166, sum=332 (2)\", \"tab\": \"General information\", \"score\": \"166.0\"}", - "Virology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Virology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Virology - # prompt tokens": "{\"description\": \"min=336.819, mean=336.819, max=336.819, sum=673.639 (2)\", \"tab\": \"General information\", \"score\": \"336.8192771084337\"}", - "Virology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"virology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_virology\"" - } - } - }, - { - "evaluation_name": "World Religions", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on World Religions", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.778, - "details": { - "description": "min=0.778, mean=0.778, max=0.778, sum=1.556 (2)", - "tab": "Accuracy", - "World Religions - Observed inference time (s)": "{\"description\": \"min=0.315, mean=0.315, max=0.315, sum=0.63 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3150970712739822\"}", - "World Religions - # eval": "{\"description\": \"min=171, mean=171, max=171, sum=342 (2)\", \"tab\": \"General information\", \"score\": \"171.0\"}", - "World Religions - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "World Religions - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "World Religions - # prompt tokens": "{\"description\": \"min=269.07, mean=269.07, max=269.07, sum=538.14 (2)\", \"tab\": \"General information\", \"score\": \"269.0701754385965\"}", - "World Religions - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"world_religions\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_world_religions\"" - } - } - }, - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.843, - "details": { - "description": "", - "tab": "Efficiency" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_id": "hfopenllm_v2/Qwen_Qwen1.5-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2684 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.456 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0929 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2987 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4103 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2916 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qwen_qwen1.5-moe-a2.7b-chat.json b/data/models/qwen_qwen1.5-moe-a2.7b-chat.json deleted file mode 100644 index 4237f995d556b324f039b35cd85f0979d7be01be..0000000000000000000000000000000000000000 --- a/data/models/qwen_qwen1.5-moe-a2.7b-chat.json +++ /dev/null @@ -1,277 +0,0 @@ -{ - "model_info": { - "name": "Qwen/Qwen1.5-MoE-A2.7B-Chat", - "id": "Qwen/Qwen1.5-MoE-A2.7B-Chat", - "developer": "Qwen", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2MoeForCausalLM", - "params_billions": "14.316" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Qwen_Qwen1.5-MoE-A2.7B-Chat/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3795 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4272 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0634 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2743 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3899 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2923 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "reward-bench/Qwen_Qwen1.5-MoE-A2.7B-Chat/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6644 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7291 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6316 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6284 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.774 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4536 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qwen_qwen1.5-moe-a2.7b.json b/data/models/qwen_qwen1.5-moe-a2.7b.json deleted file mode 100644 index 3cc8ea783926855938c223fc15de580f83a04985..0000000000000000000000000000000000000000 --- a/data/models/qwen_qwen1.5-moe-a2.7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen1.5-MoE-A2.7B", - "id": "Qwen/Qwen1.5-MoE-A2.7B", - "developer": "Qwen", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2MoeForCausalLM", - "params_billions": "14.316" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Qwen_Qwen1.5-MoE-A2.7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.266 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4114 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0929 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2592 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4013 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2778 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qwen_qwen2-0.5b-instruct.json b/data/models/qwen_qwen2-0.5b-instruct.json deleted file mode 100644 index 463af48f0b0bba3c7bbc1c34f6a0d3d57ff062ce..0000000000000000000000000000000000000000 --- a/data/models/qwen_qwen2-0.5b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2-0.5B-Instruct", - "id": "Qwen/Qwen2-0.5B-Instruct", - "developer": "Qwen", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Qwen_Qwen2-0.5B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2247 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3173 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0287 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2466 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3353 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1531 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qwen_qwen2-0.5b.json b/data/models/qwen_qwen2-0.5b.json deleted file mode 100644 index 3b28121f0984d6cf12ececd4b86202318f081aa1..0000000000000000000000000000000000000000 --- a/data/models/qwen_qwen2-0.5b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2-0.5B", - "id": "Qwen/Qwen2-0.5B", - "developer": "Qwen", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Qwen_Qwen2-0.5B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1873 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3239 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0264 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2609 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3752 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.172 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qwen_qwen2-1.5b-instruct.json b/data/models/qwen_qwen2-1.5b-instruct.json deleted file mode 100644 index 201b143e63fcdcb93d2ff2833a154ab77f0ad941..0000000000000000000000000000000000000000 --- a/data/models/qwen_qwen2-1.5b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2-1.5B-Instruct", - "id": "Qwen/Qwen2-1.5B-Instruct", - "developer": "Qwen", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.544" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Qwen_Qwen2-1.5B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3371 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3852 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0718 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2617 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4293 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2501 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qwen_qwen2-1.5b.json b/data/models/qwen_qwen2-1.5b.json deleted file mode 100644 index 30c9b30565200204731738828f2db3451f68d45f..0000000000000000000000000000000000000000 --- a/data/models/qwen_qwen2-1.5b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2-1.5B", - "id": "Qwen/Qwen2-1.5B", - "developer": "Qwen", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.544" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Qwen_Qwen2-1.5B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2113 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3575 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0702 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2643 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3658 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2552 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qwen_qwen2-57b-a14b-instruct.json b/data/models/qwen_qwen2-57b-a14b-instruct.json deleted file mode 100644 index 5f967d09b5b0c588f7288e5f2618788e7b1e71d8..0000000000000000000000000000000000000000 --- a/data/models/qwen_qwen2-57b-a14b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2-57B-A14B-Instruct", - "id": "Qwen/Qwen2-57B-A14B-Instruct", - "developer": "Qwen", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2MoeForCausalLM", - "params_billions": "57.409" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Qwen_Qwen2-57B-A14B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6338 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5888 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2817 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3314 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4361 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4575 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qwen_qwen2-57b-a14b.json b/data/models/qwen_qwen2-57b-a14b.json deleted file mode 100644 index 784671150103383a0a540f477559139bc1a5adc6..0000000000000000000000000000000000000000 --- a/data/models/qwen_qwen2-57b-a14b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2-57B-A14B", - "id": "Qwen/Qwen2-57B-A14B", - "developer": "Qwen", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2MoeForCausalLM", - "params_billions": "57.409" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Qwen_Qwen2-57B-A14B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3113 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5618 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1866 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3062 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4174 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4916 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qwen_qwen2-72b-instruct.json b/data/models/qwen_qwen2-72b-instruct.json deleted file mode 100644 index 4dbca23e465908eedaa13e498cba3cdf7173d0f5..0000000000000000000000000000000000000000 --- a/data/models/qwen_qwen2-72b-instruct.json +++ /dev/null @@ -1,2038 +0,0 @@ -{ - "model_info": { - "name": "Qwen2 Instruct 72B", - "id": "qwen/qwen2-72b-instruct", - "developer": "qwen", - "inference_platform": "unknown", - "additional_details": { - "model_id_aliases": [ - "Qwen/Qwen2-72B-Instruct" - ] - } - }, - "evaluations": [ - { - "evaluation_id": "helm_lite/qwen_qwen2-72b-instruct/1774096306.427425", - "retrieved_timestamp": "1774096306.427425", - "source_metadata": { - "source_name": "helm_lite", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_lite", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_lite", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.77, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.592421972534332\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.727, - "details": { - "description": "min=0.727, mean=0.727, max=0.727, sum=0.727 (1)", - "tab": "Accuracy", - "NarrativeQA - Observed inference time (s)": "{\"description\": \"min=1.19, mean=1.19, max=1.19, sum=1.19 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.1896146727279877\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=3502.913, mean=3502.913, max=3502.913, sum=3502.913 (1)\", \"tab\": \"General information\", \"score\": \"3502.912676056338\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=11.642, mean=11.642, max=11.642, sum=11.642 (1)\", \"tab\": \"General information\", \"score\": \"11.64225352112676\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (closed-book)", - "source_data": { - "dataset_name": "NaturalQuestions (closed-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (closed-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.39, - "details": { - "description": "min=0.39, mean=0.39, max=0.39, sum=0.39 (1)", - "tab": "Accuracy", - "NaturalQuestions (open-book) - Observed inference time (s)": "{\"description\": \"min=0.868, mean=0.868, max=0.868, sum=0.868 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.8683992192745209\"}", - "NaturalQuestions (closed-book) - Observed inference time (s)": "{\"description\": \"min=0.356, mean=0.356, max=0.356, sum=0.356 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.35628414297103883\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=2017.955, mean=2017.955, max=2017.955, sum=2017.955 (1)\", \"tab\": \"General information\", \"score\": \"2017.955\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=9.044, mean=9.044, max=9.044, sum=9.044 (1)\", \"tab\": \"General information\", \"score\": \"9.044\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=146.262, mean=146.262, max=146.262, sum=146.262 (1)\", \"tab\": \"General information\", \"score\": \"146.262\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=6.433, mean=6.433, max=6.433, sum=6.433 (1)\", \"tab\": \"General information\", \"score\": \"6.433\"}" - } - }, - "generation_config": { - "additional_details": { - "mode": "\"closedbook\"" - } - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.954, - "details": { - "description": "min=0.954, mean=0.954, max=0.954, sum=0.954 (1)", - "tab": "Accuracy", - "OpenbookQA - Observed inference time (s)": "{\"description\": \"min=0.218, mean=0.218, max=0.218, sum=0.218 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.21781798839569091\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=249.846, mean=249.846, max=249.846, sum=249.846 (1)\", \"tab\": \"General information\", \"score\": \"249.846\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "dataset": "\"openbookqa\"", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.769, - "details": { - "description": "min=0.65, mean=0.769, max=0.94, sum=3.847 (5)", - "tab": "Accuracy", - "MMLU - Observed inference time (s)": "{\"description\": \"min=0.195, mean=0.277, max=0.395, sum=1.385 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.2769099538284435\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=378.19, mean=477.836, max=627.939, sum=2389.179 (5)\", \"tab\": \"General information\", \"score\": \"477.8357192982456\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"college_chemistry\", \"computer_security\", \"econometrics\", \"us_foreign_policy\"]", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MATH", - "source_data": { - "dataset_name": "MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Equivalent (CoT) on MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.79, - "details": { - "description": "min=0.605, mean=0.79, max=0.93, sum=5.533 (7)", - "tab": "Accuracy", - "MATH - Observed inference time (s)": "{\"description\": \"min=3.599, mean=4.461, max=5.828, sum=31.228 (7)\", \"tab\": \"Efficiency\", \"score\": \"4.461141077844028\"}", - "MATH - # eval": "{\"description\": \"min=30, mean=62.429, max=135, sum=437 (7)\", \"tab\": \"General information\", \"score\": \"62.42857142857143\"}", - "MATH - # train": "{\"description\": \"min=8, mean=8, max=8, sum=56 (7)\", \"tab\": \"General information\", \"score\": \"8.0\"}", - "MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MATH - # prompt tokens": "{\"description\": \"min=937.926, mean=1323.837, max=2246.673, sum=9266.858 (7)\", \"tab\": \"General information\", \"score\": \"1323.836848955025\"}", - "MATH - # output tokens": "{\"description\": \"min=145.36, mean=173.894, max=202.346, sum=1217.257 (7)\", \"tab\": \"General information\", \"score\": \"173.89384019579856\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"algebra\", \"counting_and_probability\", \"geometry\", \"intermediate_algebra\", \"number_theory\", \"prealgebra\", \"precalculus\"]", - "level": "\"1\"", - "use_official_examples": "\"False\"", - "use_chain_of_thought": "\"True\"" - } - } - }, - { - "evaluation_name": "GSM8K", - "source_data": { - "dataset_name": "GSM8K", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on GSM8K", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.92, - "details": { - "description": "min=0.92, mean=0.92, max=0.92, sum=0.92 (1)", - "tab": "Accuracy", - "GSM8K - Observed inference time (s)": "{\"description\": \"min=6.592, mean=6.592, max=6.592, sum=6.592 (1)\", \"tab\": \"Efficiency\", \"score\": \"6.592170278310776\"}", - "GSM8K - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "GSM8K - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "GSM8K - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GSM8K - # prompt tokens": "{\"description\": \"min=1130.403, mean=1130.403, max=1130.403, sum=1130.403 (1)\", \"tab\": \"General information\", \"score\": \"1130.403\"}", - "GSM8K - # output tokens": "{\"description\": \"min=166.4, mean=166.4, max=166.4, sum=166.4 (1)\", \"tab\": \"General information\", \"score\": \"166.4\"}" - } - }, - "generation_config": { - "additional_details": { - "stop": "\"none\"" - } - } - }, - { - "evaluation_name": "LegalBench", - "source_data": { - "dataset_name": "LegalBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on LegalBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.712, - "details": { - "description": "min=0.411, mean=0.712, max=0.947, sum=3.559 (5)", - "tab": "Accuracy", - "LegalBench - Observed inference time (s)": "{\"description\": \"min=0.233, mean=0.521, max=1.575, sum=2.605 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.5210018908984072\"}", - "LegalBench - # eval": "{\"description\": \"min=95, mean=409.4, max=1000, sum=2047 (5)\", \"tab\": \"General information\", \"score\": \"409.4\"}", - "LegalBench - # train": "{\"description\": \"min=4, mean=4.8, max=5, sum=24 (5)\", \"tab\": \"General information\", \"score\": \"4.8\"}", - "LegalBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "LegalBench - # prompt tokens": "{\"description\": \"min=207.453, mean=1557.088, max=6445.714, sum=7785.442 (5)\", \"tab\": \"General information\", \"score\": \"1557.0883229968654\"}", - "LegalBench - # output tokens": "{\"description\": \"min=2, mean=2.299, max=3.042, sum=11.494 (5)\", \"tab\": \"General information\", \"score\": \"2.2988842678904344\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"abercrombie\", \"corporate_lobbying\", \"function_of_decision_section\", \"international_citizenship_questions\", \"proa\"]" - } - } - }, - { - "evaluation_name": "MedQA", - "source_data": { - "dataset_name": "MedQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MedQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.746, - "details": { - "description": "min=0.746, mean=0.746, max=0.746, sum=0.746 (1)", - "tab": "Accuracy", - "MedQA - Observed inference time (s)": "{\"description\": \"min=0.535, mean=0.535, max=0.535, sum=0.535 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.5349795590812122\"}", - "MedQA - # eval": "{\"description\": \"min=503, mean=503, max=503, sum=503 (1)\", \"tab\": \"General information\", \"score\": \"503.0\"}", - "MedQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MedQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MedQA - # prompt tokens": "{\"description\": \"min=1052.485, mean=1052.485, max=1052.485, sum=1052.485 (1)\", \"tab\": \"General information\", \"score\": \"1052.4850894632207\"}", - "MedQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WMT 2014", - "source_data": { - "dataset_name": "WMT 2014", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "BLEU-4 on WMT 2014", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.207, - "details": { - "description": "min=0.156, mean=0.207, max=0.255, sum=1.033 (5)", - "tab": "Accuracy", - "WMT 2014 - Observed inference time (s)": "{\"description\": \"min=0.802, mean=0.827, max=0.86, sum=4.135 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.8269615642193179\"}", - "WMT 2014 - # eval": "{\"description\": \"min=503, mean=568.8, max=832, sum=2844 (5)\", \"tab\": \"General information\", \"score\": \"568.8\"}", - "WMT 2014 - # train": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "WMT 2014 - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WMT 2014 - # prompt tokens": "{\"description\": \"min=124.855, mean=142.657, max=158.373, sum=713.283 (5)\", \"tab\": \"General information\", \"score\": \"142.65662658663405\"}", - "WMT 2014 - # output tokens": "{\"description\": \"min=25.368, mean=27.029, max=27.714, sum=135.143 (5)\", \"tab\": \"General information\", \"score\": \"27.028530260743235\"}" - } - }, - "generation_config": { - "additional_details": { - "language_pair": "[\"cs-en\", \"de-en\", \"fr-en\", \"hi-en\", \"ru-en\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_mmlu/qwen_qwen2-72b-instruct/1774096312.00548", - "retrieved_timestamp": "1774096312.00548", - "source_metadata": { - "source_name": "helm_mmlu", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_mmlu", - "evaluation_results": [ - { - "evaluation_name": "MMLU All Subjects", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU All Subjects", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.824, - "details": { - "description": "min=0.52, mean=0.824, max=0.979, sum=93.879 (114)", - "tab": "Accuracy", - "MMLU All Subjects - Observed inference time (s)": "{\"description\": \"min=0.195, mean=0.359, max=2.502, sum=40.898 (114)\", \"tab\": \"Efficiency\", \"score\": \"0.3587521754503106\"}", - "MMLU All Subjects - # eval": "{\"description\": \"min=100, mean=246.351, max=1534, sum=28084 (114)\", \"tab\": \"General information\", \"score\": \"246.35087719298247\"}", - "MMLU All Subjects - # train": "{\"description\": \"min=5, mean=5, max=5, sum=570 (114)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU All Subjects - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU All Subjects - # prompt tokens": "{\"description\": \"min=276.07, mean=625.598, max=2814.903, sum=71318.198 (114)\", \"tab\": \"General information\", \"score\": \"625.5982315160392\"}", - "MMLU All Subjects - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=114 (114)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_name": "Abstract Algebra", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Abstract Algebra", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.67, - "details": { - "description": "min=0.67, mean=0.67, max=0.67, sum=1.34 (2)", - "tab": "Accuracy", - "Abstract Algebra - Observed inference time (s)": "{\"description\": \"min=0.395, mean=0.395, max=0.395, sum=0.79 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3948828268051148\"}", - "Abstract Algebra - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Abstract Algebra - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Abstract Algebra - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Abstract Algebra - # prompt tokens": "{\"description\": \"min=378.19, mean=378.19, max=378.19, sum=756.38 (2)\", \"tab\": \"General information\", \"score\": \"378.19\"}", - "Abstract Algebra - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"abstract_algebra\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_abstract_algebra\"" - } - } - }, - { - "evaluation_name": "Anatomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Anatomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.793, - "details": { - "description": "min=0.793, mean=0.793, max=0.793, sum=1.585 (2)", - "tab": "Accuracy", - "Anatomy - Observed inference time (s)": "{\"description\": \"min=0.266, mean=0.266, max=0.266, sum=0.531 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2657013893127441\"}", - "Anatomy - # eval": "{\"description\": \"min=135, mean=135, max=135, sum=270 (2)\", \"tab\": \"General information\", \"score\": \"135.0\"}", - "Anatomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Anatomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Anatomy - # prompt tokens": "{\"description\": \"min=353.978, mean=353.978, max=353.978, sum=707.956 (2)\", \"tab\": \"General information\", \"score\": \"353.97777777777776\"}", - "Anatomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"anatomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_anatomy\"" - } - } - }, - { - "evaluation_name": "College Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on College Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.598, - "details": { - "description": "min=0.598, mean=0.598, max=0.598, sum=1.196 (2)", - "tab": "Accuracy", - "College Chemistry - Observed inference time (s)": "{\"description\": \"min=0.249, mean=0.249, max=0.249, sum=0.498 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.24894725322723388\"}", - "College Biology - Observed inference time (s)": "{\"description\": \"min=0.298, mean=0.298, max=0.298, sum=0.596 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2977961285246743\"}", - "College Computer Science - Observed inference time (s)": "{\"description\": \"min=0.321, mean=0.321, max=0.321, sum=0.642 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3207618069648743\"}", - "College Mathematics - Observed inference time (s)": "{\"description\": \"min=0.334, mean=0.334, max=0.334, sum=0.667 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3337481117248535\"}", - "College Medicine - Observed inference time (s)": "{\"description\": \"min=0.234, mean=0.234, max=0.234, sum=0.468 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2340707227673834\"}", - "College Physics - Observed inference time (s)": "{\"description\": \"min=0.25, mean=0.25, max=0.25, sum=0.5 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.25010308097390566\"}", - "College Chemistry - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Chemistry - # prompt tokens": "{\"description\": \"min=568.25, mean=568.25, max=568.25, sum=1136.5 (2)\", \"tab\": \"General information\", \"score\": \"568.25\"}", - "College Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Biology - # eval": "{\"description\": \"min=144, mean=144, max=144, sum=288 (2)\", \"tab\": \"General information\", \"score\": \"144.0\"}", - "College Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # prompt tokens": "{\"description\": \"min=486.979, mean=486.979, max=486.979, sum=973.958 (2)\", \"tab\": \"General information\", \"score\": \"486.9791666666667\"}", - "College Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # prompt tokens": "{\"description\": \"min=838.58, mean=838.58, max=838.58, sum=1677.16 (2)\", \"tab\": \"General information\", \"score\": \"838.58\"}", - "College Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Mathematics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # prompt tokens": "{\"description\": \"min=607.7, mean=607.7, max=607.7, sum=1215.4 (2)\", \"tab\": \"General information\", \"score\": \"607.7\"}", - "College Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Medicine - # eval": "{\"description\": \"min=173, mean=173, max=173, sum=346 (2)\", \"tab\": \"General information\", \"score\": \"173.0\"}", - "College Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # prompt tokens": "{\"description\": \"min=506.098, mean=506.098, max=506.098, sum=1012.197 (2)\", \"tab\": \"General information\", \"score\": \"506.0982658959538\"}", - "College Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Physics - # eval": "{\"description\": \"min=102, mean=102, max=102, sum=204 (2)\", \"tab\": \"General information\", \"score\": \"102.0\"}", - "College Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # prompt tokens": "{\"description\": \"min=516.265, mean=516.265, max=516.265, sum=1032.529 (2)\", \"tab\": \"General information\", \"score\": \"516.2647058823529\"}", - "College Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"college_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_college_physics\"" - } - } - }, - { - "evaluation_name": "Computer Security", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Computer Security", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.85, - "details": { - "description": "min=0.85, mean=0.85, max=0.85, sum=1.7 (2)", - "tab": "Accuracy", - "Computer Security - Observed inference time (s)": "{\"description\": \"min=0.281, mean=0.281, max=0.281, sum=0.563 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2812828135490417\"}", - "Computer Security - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Computer Security - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Computer Security - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Computer Security - # prompt tokens": "{\"description\": \"min=386.64, mean=386.64, max=386.64, sum=773.28 (2)\", \"tab\": \"General information\", \"score\": \"386.64\"}", - "Computer Security - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"computer_security\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_computer_security\"" - } - } - }, - { - "evaluation_name": "Econometrics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Econometrics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.737, - "details": { - "description": "min=0.737, mean=0.737, max=0.737, sum=1.474 (2)", - "tab": "Accuracy", - "Econometrics - Observed inference time (s)": "{\"description\": \"min=0.265, mean=0.265, max=0.265, sum=0.53 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.26492034552390115\"}", - "Econometrics - # eval": "{\"description\": \"min=114, mean=114, max=114, sum=228 (2)\", \"tab\": \"General information\", \"score\": \"114.0\"}", - "Econometrics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Econometrics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Econometrics - # prompt tokens": "{\"description\": \"min=627.939, mean=627.939, max=627.939, sum=1255.877 (2)\", \"tab\": \"General information\", \"score\": \"627.938596491228\"}", - "Econometrics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"econometrics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_econometrics\"" - } - } - }, - { - "evaluation_name": "Global Facts", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Global Facts", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.58, - "details": { - "description": "min=0.58, mean=0.58, max=0.58, sum=1.16 (2)", - "tab": "Accuracy", - "Global Facts - Observed inference time (s)": "{\"description\": \"min=0.254, mean=0.254, max=0.254, sum=0.507 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.25351563215255735\"}", - "Global Facts - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Global Facts - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Global Facts - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Global Facts - # prompt tokens": "{\"description\": \"min=429.06, mean=429.06, max=429.06, sum=858.12 (2)\", \"tab\": \"General information\", \"score\": \"429.06\"}", - "Global Facts - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"global_facts\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_global_facts\"" - } - } - }, - { - "evaluation_name": "Jurisprudence", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Jurisprudence", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.87, - "details": { - "description": "min=0.87, mean=0.87, max=0.87, sum=1.741 (2)", - "tab": "Accuracy", - "Jurisprudence - Observed inference time (s)": "{\"description\": \"min=0.257, mean=0.257, max=0.257, sum=0.513 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.256509714656406\"}", - "Jurisprudence - # eval": "{\"description\": \"min=108, mean=108, max=108, sum=216 (2)\", \"tab\": \"General information\", \"score\": \"108.0\"}", - "Jurisprudence - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Jurisprudence - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Jurisprudence - # prompt tokens": "{\"description\": \"min=394.713, mean=394.713, max=394.713, sum=789.426 (2)\", \"tab\": \"General information\", \"score\": \"394.712962962963\"}", - "Jurisprudence - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"jurisprudence\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_jurisprudence\"" - } - } - }, - { - "evaluation_name": "Philosophy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Philosophy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.859, - "details": { - "description": "min=0.859, mean=0.859, max=0.859, sum=1.717 (2)", - "tab": "Accuracy", - "Philosophy - Observed inference time (s)": "{\"description\": \"min=0.204, mean=0.204, max=0.204, sum=0.409 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.20427469348600824\"}", - "Philosophy - # eval": "{\"description\": \"min=311, mean=311, max=311, sum=622 (2)\", \"tab\": \"General information\", \"score\": \"311.0\"}", - "Philosophy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Philosophy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Philosophy - # prompt tokens": "{\"description\": \"min=329.09, mean=329.09, max=329.09, sum=658.18 (2)\", \"tab\": \"General information\", \"score\": \"329.09003215434086\"}", - "Philosophy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"philosophy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_philosophy\"" - } - } - }, - { - "evaluation_name": "Professional Psychology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Professional Psychology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.886, - "details": { - "description": "min=0.886, mean=0.886, max=0.886, sum=1.771 (2)", - "tab": "Accuracy", - "Professional Medicine - Observed inference time (s)": "{\"description\": \"min=0.507, mean=0.507, max=0.507, sum=1.014 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5070785135030746\"}", - "Professional Accounting - Observed inference time (s)": "{\"description\": \"min=0.31, mean=0.31, max=0.31, sum=0.621 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.31040529579135545\"}", - "Professional Law - Observed inference time (s)": "{\"description\": \"min=0.407, mean=0.407, max=0.407, sum=0.814 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.40680916352875074\"}", - "Professional Psychology - Observed inference time (s)": "{\"description\": \"min=0.324, mean=0.324, max=0.324, sum=0.647 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.32369842482548133\"}", - "Professional Medicine - # eval": "{\"description\": \"min=272, mean=272, max=272, sum=544 (2)\", \"tab\": \"General information\", \"score\": \"272.0\"}", - "Professional Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Medicine - # prompt tokens": "{\"description\": \"min=1125.199, mean=1125.199, max=1125.199, sum=2250.397 (2)\", \"tab\": \"General information\", \"score\": \"1125.1985294117646\"}", - "Professional Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Accounting - # eval": "{\"description\": \"min=282, mean=282, max=282, sum=564 (2)\", \"tab\": \"General information\", \"score\": \"282.0\"}", - "Professional Accounting - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Accounting - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # prompt tokens": "{\"description\": \"min=739.34, mean=739.34, max=739.34, sum=1478.681 (2)\", \"tab\": \"General information\", \"score\": \"739.3404255319149\"}", - "Professional Accounting - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Law - # eval": "{\"description\": \"min=1534, mean=1534, max=1534, sum=3068 (2)\", \"tab\": \"General information\", \"score\": \"1534.0\"}", - "Professional Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # prompt tokens": "{\"description\": \"min=1663.969, mean=1663.969, max=1663.969, sum=3327.939 (2)\", \"tab\": \"General information\", \"score\": \"1663.9693611473272\"}", - "Professional Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Psychology - # eval": "{\"description\": \"min=612, mean=612, max=612, sum=1224 (2)\", \"tab\": \"General information\", \"score\": \"612.0\"}", - "Professional Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # prompt tokens": "{\"description\": \"min=581.417, mean=581.417, max=581.417, sum=1162.833 (2)\", \"tab\": \"General information\", \"score\": \"581.4166666666666\"}", - "Professional Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"professional_psychology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_professional_psychology\"" - } - } - }, - { - "evaluation_name": "Us Foreign Policy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Us Foreign Policy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.94, - "details": { - "description": "min=0.94, mean=0.94, max=0.94, sum=1.88 (2)", - "tab": "Accuracy", - "Us Foreign Policy - Observed inference time (s)": "{\"description\": \"min=0.195, mean=0.195, max=0.195, sum=0.389 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.19451653003692626\"}", - "Us Foreign Policy - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Us Foreign Policy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Us Foreign Policy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Us Foreign Policy - # prompt tokens": "{\"description\": \"min=428.16, mean=428.16, max=428.16, sum=856.32 (2)\", \"tab\": \"General information\", \"score\": \"428.16\"}", - "Us Foreign Policy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"us_foreign_policy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_us_foreign_policy\"" - } - } - }, - { - "evaluation_name": "Astronomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Astronomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.934, - "details": { - "description": "min=0.934, mean=0.934, max=0.934, sum=1.868 (2)", - "tab": "Accuracy", - "Astronomy - Observed inference time (s)": "{\"description\": \"min=0.32, mean=0.32, max=0.32, sum=0.641 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.32045089571099533\"}", - "Astronomy - # eval": "{\"description\": \"min=152, mean=152, max=152, sum=304 (2)\", \"tab\": \"General information\", \"score\": \"152.0\"}", - "Astronomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Astronomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Astronomy - # prompt tokens": "{\"description\": \"min=589.849, mean=589.849, max=589.849, sum=1179.697 (2)\", \"tab\": \"General information\", \"score\": \"589.8486842105264\"}", - "Astronomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"astronomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_astronomy\"" - } - } - }, - { - "evaluation_name": "Business Ethics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Business Ethics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.82, - "details": { - "description": "min=0.82, mean=0.82, max=0.82, sum=1.64 (2)", - "tab": "Accuracy", - "Business Ethics - Observed inference time (s)": "{\"description\": \"min=0.351, mean=0.351, max=0.351, sum=0.701 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.350736882686615\"}", - "Business Ethics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Business Ethics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Business Ethics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Business Ethics - # prompt tokens": "{\"description\": \"min=569.87, mean=569.87, max=569.87, sum=1139.74 (2)\", \"tab\": \"General information\", \"score\": \"569.87\"}", - "Business Ethics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"business_ethics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_business_ethics\"" - } - } - }, - { - "evaluation_name": "Clinical Knowledge", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Clinical Knowledge", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.868, - "details": { - "description": "min=0.868, mean=0.868, max=0.868, sum=1.736 (2)", - "tab": "Accuracy", - "Clinical Knowledge - Observed inference time (s)": "{\"description\": \"min=0.26, mean=0.26, max=0.26, sum=0.52 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2597639983555056\"}", - "Clinical Knowledge - # eval": "{\"description\": \"min=265, mean=265, max=265, sum=530 (2)\", \"tab\": \"General information\", \"score\": \"265.0\"}", - "Clinical Knowledge - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Clinical Knowledge - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Clinical Knowledge - # prompt tokens": "{\"description\": \"min=400.623, mean=400.623, max=400.623, sum=801.245 (2)\", \"tab\": \"General information\", \"score\": \"400.62264150943395\"}", - "Clinical Knowledge - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"clinical_knowledge\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_clinical_knowledge\"" - } - } - }, - { - "evaluation_name": "Conceptual Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Conceptual Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.872, - "details": { - "description": "min=0.872, mean=0.872, max=0.872, sum=1.745 (2)", - "tab": "Accuracy", - "Conceptual Physics - Observed inference time (s)": "{\"description\": \"min=0.242, mean=0.242, max=0.242, sum=0.484 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2420806296328281\"}", - "Conceptual Physics - # eval": "{\"description\": \"min=235, mean=235, max=235, sum=470 (2)\", \"tab\": \"General information\", \"score\": \"235.0\"}", - "Conceptual Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Conceptual Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Conceptual Physics - # prompt tokens": "{\"description\": \"min=305.494, mean=305.494, max=305.494, sum=610.987 (2)\", \"tab\": \"General information\", \"score\": \"305.4936170212766\"}", - "Conceptual Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"conceptual_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_conceptual_physics\"" - } - } - }, - { - "evaluation_name": "Electrical Engineering", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Electrical Engineering", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.793, - "details": { - "description": "min=0.793, mean=0.793, max=0.793, sum=1.586 (2)", - "tab": "Accuracy", - "Electrical Engineering - Observed inference time (s)": "{\"description\": \"min=0.235, mean=0.235, max=0.235, sum=0.47 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.23504354542699354\"}", - "Electrical Engineering - # eval": "{\"description\": \"min=145, mean=145, max=145, sum=290 (2)\", \"tab\": \"General information\", \"score\": \"145.0\"}", - "Electrical Engineering - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Electrical Engineering - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Electrical Engineering - # prompt tokens": "{\"description\": \"min=463.8, mean=463.8, max=463.8, sum=927.6 (2)\", \"tab\": \"General information\", \"score\": \"463.8\"}", - "Electrical Engineering - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"electrical_engineering\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_electrical_engineering\"" - } - } - }, - { - "evaluation_name": "Elementary Mathematics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Elementary Mathematics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.825, - "details": { - "description": "min=0.825, mean=0.825, max=0.825, sum=1.651 (2)", - "tab": "Accuracy", - "Elementary Mathematics - Observed inference time (s)": "{\"description\": \"min=0.261, mean=0.261, max=0.261, sum=0.523 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2613614286695208\"}", - "Elementary Mathematics - # eval": "{\"description\": \"min=378, mean=378, max=378, sum=756 (2)\", \"tab\": \"General information\", \"score\": \"378.0\"}", - "Elementary Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Elementary Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Elementary Mathematics - # prompt tokens": "{\"description\": \"min=577.119, mean=577.119, max=577.119, sum=1154.238 (2)\", \"tab\": \"General information\", \"score\": \"577.1190476190476\"}", - "Elementary Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"elementary_mathematics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_elementary_mathematics\"" - } - } - }, - { - "evaluation_name": "Formal Logic", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Formal Logic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.667, - "details": { - "description": "min=0.667, mean=0.667, max=0.667, sum=1.333 (2)", - "tab": "Accuracy", - "Formal Logic - Observed inference time (s)": "{\"description\": \"min=0.333, mean=0.333, max=0.333, sum=0.666 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3330562947288392\"}", - "Formal Logic - # eval": "{\"description\": \"min=126, mean=126, max=126, sum=252 (2)\", \"tab\": \"General information\", \"score\": \"126.0\"}", - "Formal Logic - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Formal Logic - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Formal Logic - # prompt tokens": "{\"description\": \"min=604.667, mean=604.667, max=604.667, sum=1209.333 (2)\", \"tab\": \"General information\", \"score\": \"604.6666666666666\"}", - "Formal Logic - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"formal_logic\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_formal_logic\"" - } - } - }, - { - "evaluation_name": "High School World History", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on High School World History", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.932, - "details": { - "description": "min=0.932, mean=0.932, max=0.932, sum=1.865 (2)", - "tab": "Accuracy", - "High School Biology - Observed inference time (s)": "{\"description\": \"min=0.247, mean=0.247, max=0.247, sum=0.495 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.24744614170443627\"}", - "High School Chemistry - Observed inference time (s)": "{\"description\": \"min=0.301, mean=0.301, max=0.301, sum=0.602 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3010592906933113\"}", - "High School Computer Science - Observed inference time (s)": "{\"description\": \"min=0.29, mean=0.29, max=0.29, sum=0.581 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2903395962715149\"}", - "High School European History - Observed inference time (s)": "{\"description\": \"min=0.629, mean=0.629, max=0.629, sum=1.258 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6291334065524015\"}", - "High School Geography - Observed inference time (s)": "{\"description\": \"min=0.457, mean=0.457, max=0.457, sum=0.913 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4567244630871397\"}", - "High School Government And Politics - Observed inference time (s)": "{\"description\": \"min=0.249, mean=0.249, max=0.249, sum=0.498 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.24882311524504824\"}", - "High School Macroeconomics - Observed inference time (s)": "{\"description\": \"min=0.245, mean=0.245, max=0.245, sum=0.489 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.24466082010513696\"}", - "High School Mathematics - Observed inference time (s)": "{\"description\": \"min=0.257, mean=0.257, max=0.257, sum=0.514 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2570408988881994\"}", - "High School Microeconomics - Observed inference time (s)": "{\"description\": \"min=0.27, mean=0.27, max=0.27, sum=0.539 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.26973113893460826\"}", - "High School Physics - Observed inference time (s)": "{\"description\": \"min=0.285, mean=0.285, max=0.285, sum=0.57 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2847776444542487\"}", - "High School Psychology - Observed inference time (s)": "{\"description\": \"min=0.32, mean=0.32, max=0.32, sum=0.641 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.32032192956416977\"}", - "High School Statistics - Observed inference time (s)": "{\"description\": \"min=0.357, mean=0.357, max=0.357, sum=0.714 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3567825931089896\"}", - "High School US History - Observed inference time (s)": "{\"description\": \"min=2.502, mean=2.502, max=2.502, sum=5.003 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.501642145362555\"}", - "High School World History - Observed inference time (s)": "{\"description\": \"min=2.182, mean=2.182, max=2.182, sum=4.364 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.18210094890514\"}", - "High School Biology - # eval": "{\"description\": \"min=310, mean=310, max=310, sum=620 (2)\", \"tab\": \"General information\", \"score\": \"310.0\"}", - "High School Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Biology - # prompt tokens": "{\"description\": \"min=513.916, mean=513.916, max=513.916, sum=1027.832 (2)\", \"tab\": \"General information\", \"score\": \"513.916129032258\"}", - "High School Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Chemistry - # eval": "{\"description\": \"min=203, mean=203, max=203, sum=406 (2)\", \"tab\": \"General information\", \"score\": \"203.0\"}", - "High School Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # prompt tokens": "{\"description\": \"min=517.261, mean=517.261, max=517.261, sum=1034.522 (2)\", \"tab\": \"General information\", \"score\": \"517.2610837438424\"}", - "High School Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "High School Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # prompt tokens": "{\"description\": \"min=878.46, mean=878.46, max=878.46, sum=1756.92 (2)\", \"tab\": \"General information\", \"score\": \"878.46\"}", - "High School Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School European History - # eval": "{\"description\": \"min=165, mean=165, max=165, sum=330 (2)\", \"tab\": \"General information\", \"score\": \"165.0\"}", - "High School European History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School European History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # prompt tokens": "{\"description\": \"min=2814.903, mean=2814.903, max=2814.903, sum=5629.806 (2)\", \"tab\": \"General information\", \"score\": \"2814.9030303030304\"}", - "High School European History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Geography - # eval": "{\"description\": \"min=198, mean=198, max=198, sum=396 (2)\", \"tab\": \"General information\", \"score\": \"198.0\"}", - "High School Geography - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Geography - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # prompt tokens": "{\"description\": \"min=372.217, mean=372.217, max=372.217, sum=744.434 (2)\", \"tab\": \"General information\", \"score\": \"372.2171717171717\"}", - "High School Geography - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Government And Politics - # eval": "{\"description\": \"min=193, mean=193, max=193, sum=386 (2)\", \"tab\": \"General information\", \"score\": \"193.0\"}", - "High School Government And Politics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Government And Politics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # prompt tokens": "{\"description\": \"min=467.311, mean=467.311, max=467.311, sum=934.622 (2)\", \"tab\": \"General information\", \"score\": \"467.31088082901556\"}", - "High School Government And Politics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Macroeconomics - # eval": "{\"description\": \"min=390, mean=390, max=390, sum=780 (2)\", \"tab\": \"General information\", \"score\": \"390.0\"}", - "High School Macroeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Macroeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # prompt tokens": "{\"description\": \"min=374.349, mean=374.349, max=374.349, sum=748.697 (2)\", \"tab\": \"General information\", \"score\": \"374.34871794871793\"}", - "High School Macroeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Mathematics - # eval": "{\"description\": \"min=270, mean=270, max=270, sum=540 (2)\", \"tab\": \"General information\", \"score\": \"270.0\"}", - "High School Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # prompt tokens": "{\"description\": \"min=565.326, mean=565.326, max=565.326, sum=1130.652 (2)\", \"tab\": \"General information\", \"score\": \"565.325925925926\"}", - "High School Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Microeconomics - # eval": "{\"description\": \"min=238, mean=238, max=238, sum=476 (2)\", \"tab\": \"General information\", \"score\": \"238.0\"}", - "High School Microeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Microeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # prompt tokens": "{\"description\": \"min=402.277, mean=402.277, max=402.277, sum=804.555 (2)\", \"tab\": \"General information\", \"score\": \"402.2773109243698\"}", - "High School Microeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Physics - # eval": "{\"description\": \"min=151, mean=151, max=151, sum=302 (2)\", \"tab\": \"General information\", \"score\": \"151.0\"}", - "High School Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # prompt tokens": "{\"description\": \"min=580.536, mean=580.536, max=580.536, sum=1161.073 (2)\", \"tab\": \"General information\", \"score\": \"580.5364238410596\"}", - "High School Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Psychology - # eval": "{\"description\": \"min=545, mean=545, max=545, sum=1090 (2)\", \"tab\": \"General information\", \"score\": \"545.0\"}", - "High School Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # prompt tokens": "{\"description\": \"min=495.521, mean=495.521, max=495.521, sum=991.042 (2)\", \"tab\": \"General information\", \"score\": \"495.52110091743117\"}", - "High School Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Statistics - # eval": "{\"description\": \"min=216, mean=216, max=216, sum=432 (2)\", \"tab\": \"General information\", \"score\": \"216.0\"}", - "High School Statistics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Statistics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # prompt tokens": "{\"description\": \"min=830.477, mean=830.477, max=830.477, sum=1660.954 (2)\", \"tab\": \"General information\", \"score\": \"830.4768518518518\"}", - "High School Statistics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School US History - # eval": "{\"description\": \"min=204, mean=204, max=204, sum=408 (2)\", \"tab\": \"General information\", \"score\": \"204.0\"}", - "High School US History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School US History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # prompt tokens": "{\"description\": \"min=2237.176, mean=2237.176, max=2237.176, sum=4474.353 (2)\", \"tab\": \"General information\", \"score\": \"2237.176470588235\"}", - "High School US History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School World History - # eval": "{\"description\": \"min=237, mean=237, max=237, sum=474 (2)\", \"tab\": \"General information\", \"score\": \"237.0\"}", - "High School World History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School World History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # prompt tokens": "{\"description\": \"min=1448.354, mean=1448.354, max=1448.354, sum=2896.709 (2)\", \"tab\": \"General information\", \"score\": \"1448.3544303797469\"}", - "High School World History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"high_school_world_history\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_high_school_world_history\"" - } - } - }, - { - "evaluation_name": "Human Sexuality", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Human Sexuality", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.893, - "details": { - "description": "min=0.893, mean=0.893, max=0.893, sum=1.786 (2)", - "tab": "Accuracy", - "Human Aging - Observed inference time (s)": "{\"description\": \"min=0.275, mean=0.275, max=0.275, sum=0.55 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2751739634526685\"}", - "Human Sexuality - Observed inference time (s)": "{\"description\": \"min=0.327, mean=0.327, max=0.327, sum=0.655 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.32726097470931426\"}", - "Human Aging - # eval": "{\"description\": \"min=223, mean=223, max=223, sum=446 (2)\", \"tab\": \"General information\", \"score\": \"223.0\"}", - "Human Aging - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Aging - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Aging - # prompt tokens": "{\"description\": \"min=322.121, mean=322.121, max=322.121, sum=644.242 (2)\", \"tab\": \"General information\", \"score\": \"322.1210762331838\"}", - "Human Aging - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Human Sexuality - # eval": "{\"description\": \"min=131, mean=131, max=131, sum=262 (2)\", \"tab\": \"General information\", \"score\": \"131.0\"}", - "Human Sexuality - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Sexuality - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # prompt tokens": "{\"description\": \"min=341.504, mean=341.504, max=341.504, sum=683.008 (2)\", \"tab\": \"General information\", \"score\": \"341.5038167938931\"}", - "Human Sexuality - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"human_sexuality\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_human_sexuality\"" - } - } - }, - { - "evaluation_name": "International Law", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on International Law", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.893, - "details": { - "description": "min=0.893, mean=0.893, max=0.893, sum=1.785 (2)", - "tab": "Accuracy", - "International Law - Observed inference time (s)": "{\"description\": \"min=0.297, mean=0.297, max=0.297, sum=0.594 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2972275757592572\"}", - "International Law - # eval": "{\"description\": \"min=121, mean=121, max=121, sum=242 (2)\", \"tab\": \"General information\", \"score\": \"121.0\"}", - "International Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "International Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "International Law - # prompt tokens": "{\"description\": \"min=640.579, mean=640.579, max=640.579, sum=1281.157 (2)\", \"tab\": \"General information\", \"score\": \"640.5785123966942\"}", - "International Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"international_law\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_international_law\"" - } - } - }, - { - "evaluation_name": "Logical Fallacies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Logical Fallacies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.914, - "details": { - "description": "min=0.914, mean=0.914, max=0.914, sum=1.828 (2)", - "tab": "Accuracy", - "Logical Fallacies - Observed inference time (s)": "{\"description\": \"min=0.218, mean=0.218, max=0.218, sum=0.436 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.21798631311194297\"}", - "Logical Fallacies - # eval": "{\"description\": \"min=163, mean=163, max=163, sum=326 (2)\", \"tab\": \"General information\", \"score\": \"163.0\"}", - "Logical Fallacies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Logical Fallacies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Logical Fallacies - # prompt tokens": "{\"description\": \"min=449.632, mean=449.632, max=449.632, sum=899.264 (2)\", \"tab\": \"General information\", \"score\": \"449.6319018404908\"}", - "Logical Fallacies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"logical_fallacies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_logical_fallacies\"" - } - } - }, - { - "evaluation_name": "Machine Learning", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Machine Learning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.768, - "details": { - "description": "min=0.768, mean=0.768, max=0.768, sum=1.536 (2)", - "tab": "Accuracy", - "Machine Learning - Observed inference time (s)": "{\"description\": \"min=0.223, mean=0.223, max=0.223, sum=0.446 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.22287436042513167\"}", - "Machine Learning - # eval": "{\"description\": \"min=112, mean=112, max=112, sum=224 (2)\", \"tab\": \"General information\", \"score\": \"112.0\"}", - "Machine Learning - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Machine Learning - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Machine Learning - # prompt tokens": "{\"description\": \"min=681.848, mean=681.848, max=681.848, sum=1363.696 (2)\", \"tab\": \"General information\", \"score\": \"681.8482142857143\"}", - "Machine Learning - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"machine_learning\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_machine_learning\"" - } - } - }, - { - "evaluation_name": "Management", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Management", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.903, - "details": { - "description": "min=0.903, mean=0.903, max=0.903, sum=1.806 (2)", - "tab": "Accuracy", - "Management - Observed inference time (s)": "{\"description\": \"min=0.239, mean=0.239, max=0.239, sum=0.478 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.23922002662732764\"}", - "Management - # eval": "{\"description\": \"min=103, mean=103, max=103, sum=206 (2)\", \"tab\": \"General information\", \"score\": \"103.0\"}", - "Management - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Management - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Management - # prompt tokens": "{\"description\": \"min=283.854, mean=283.854, max=283.854, sum=567.709 (2)\", \"tab\": \"General information\", \"score\": \"283.8543689320388\"}", - "Management - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"management\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_management\"" - } - } - }, - { - "evaluation_name": "Marketing", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Marketing", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.953, - "details": { - "description": "min=0.953, mean=0.953, max=0.953, sum=1.906 (2)", - "tab": "Accuracy", - "Marketing - Observed inference time (s)": "{\"description\": \"min=0.257, mean=0.257, max=0.257, sum=0.514 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2568996777901283\"}", - "Marketing - # eval": "{\"description\": \"min=234, mean=234, max=234, sum=468 (2)\", \"tab\": \"General information\", \"score\": \"234.0\"}", - "Marketing - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Marketing - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Marketing - # prompt tokens": "{\"description\": \"min=404.415, mean=404.415, max=404.415, sum=808.829 (2)\", \"tab\": \"General information\", \"score\": \"404.4145299145299\"}", - "Marketing - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"marketing\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_marketing\"" - } - } - }, - { - "evaluation_name": "Medical Genetics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Medical Genetics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9, - "details": { - "description": "min=0.9, mean=0.9, max=0.9, sum=1.8 (2)", - "tab": "Accuracy", - "Medical Genetics - Observed inference time (s)": "{\"description\": \"min=0.267, mean=0.267, max=0.267, sum=0.534 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.26675461292266844\"}", - "Medical Genetics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Medical Genetics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Medical Genetics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Medical Genetics - # prompt tokens": "{\"description\": \"min=342.35, mean=342.35, max=342.35, sum=684.7 (2)\", \"tab\": \"General information\", \"score\": \"342.35\"}", - "Medical Genetics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"medical_genetics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_medical_genetics\"" - } - } - }, - { - "evaluation_name": "Miscellaneous", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Miscellaneous", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.943, - "details": { - "description": "min=0.943, mean=0.943, max=0.943, sum=1.885 (2)", - "tab": "Accuracy", - "Miscellaneous - Observed inference time (s)": "{\"description\": \"min=0.227, mean=0.227, max=0.227, sum=0.453 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.22672867470469663\"}", - "Miscellaneous - # eval": "{\"description\": \"min=783, mean=783, max=783, sum=1566 (2)\", \"tab\": \"General information\", \"score\": \"783.0\"}", - "Miscellaneous - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Miscellaneous - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Miscellaneous - # prompt tokens": "{\"description\": \"min=303.7, mean=303.7, max=303.7, sum=607.4 (2)\", \"tab\": \"General information\", \"score\": \"303.6998722860792\"}", - "Miscellaneous - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"miscellaneous\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_miscellaneous\"" - } - } - }, - { - "evaluation_name": "Moral Scenarios", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Moral Scenarios", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.815, - "details": { - "description": "min=0.815, mean=0.815, max=0.815, sum=1.629 (2)", - "tab": "Accuracy", - "Moral Disputes - Observed inference time (s)": "{\"description\": \"min=0.237, mean=0.237, max=0.237, sum=0.473 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.23662481900584492\"}", - "Moral Scenarios - Observed inference time (s)": "{\"description\": \"min=0.242, mean=0.242, max=0.242, sum=0.483 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.241705964264257\"}", - "Moral Disputes - # eval": "{\"description\": \"min=346, mean=346, max=346, sum=692 (2)\", \"tab\": \"General information\", \"score\": \"346.0\"}", - "Moral Disputes - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Disputes - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Disputes - # prompt tokens": "{\"description\": \"min=476.182, mean=476.182, max=476.182, sum=952.364 (2)\", \"tab\": \"General information\", \"score\": \"476.1820809248555\"}", - "Moral Disputes - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Moral Scenarios - # eval": "{\"description\": \"min=895, mean=895, max=895, sum=1790 (2)\", \"tab\": \"General information\", \"score\": \"895.0\"}", - "Moral Scenarios - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Scenarios - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # prompt tokens": "{\"description\": \"min=668.494, mean=668.494, max=668.494, sum=1336.988 (2)\", \"tab\": \"General information\", \"score\": \"668.4938547486033\"}", - "Moral Scenarios - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"moral_scenarios\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_moral_scenarios\"" - } - } - }, - { - "evaluation_name": "Nutrition", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Nutrition", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.902, - "details": { - "description": "min=0.902, mean=0.902, max=0.902, sum=1.804 (2)", - "tab": "Accuracy", - "Nutrition - Observed inference time (s)": "{\"description\": \"min=0.25, mean=0.25, max=0.25, sum=0.5 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2500531182569616\"}", - "Nutrition - # eval": "{\"description\": \"min=306, mean=306, max=306, sum=612 (2)\", \"tab\": \"General information\", \"score\": \"306.0\"}", - "Nutrition - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Nutrition - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Nutrition - # prompt tokens": "{\"description\": \"min=599.637, mean=599.637, max=599.637, sum=1199.275 (2)\", \"tab\": \"General information\", \"score\": \"599.6372549019608\"}", - "Nutrition - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"nutrition\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_nutrition\"" - } - } - }, - { - "evaluation_name": "Prehistory", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Prehistory", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.914, - "details": { - "description": "min=0.914, mean=0.914, max=0.914, sum=1.827 (2)", - "tab": "Accuracy", - "Prehistory - Observed inference time (s)": "{\"description\": \"min=0.257, mean=0.257, max=0.257, sum=0.515 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.25728267504845137\"}", - "Prehistory - # eval": "{\"description\": \"min=324, mean=324, max=324, sum=648 (2)\", \"tab\": \"General information\", \"score\": \"324.0\"}", - "Prehistory - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Prehistory - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Prehistory - # prompt tokens": "{\"description\": \"min=528.364, mean=528.364, max=528.364, sum=1056.728 (2)\", \"tab\": \"General information\", \"score\": \"528.3641975308642\"}", - "Prehistory - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"prehistory\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_prehistory\"" - } - } - }, - { - "evaluation_name": "Public Relations", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Public Relations", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.745, - "details": { - "description": "min=0.745, mean=0.745, max=0.745, sum=1.491 (2)", - "tab": "Accuracy", - "Public Relations - Observed inference time (s)": "{\"description\": \"min=0.219, mean=0.219, max=0.219, sum=0.437 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2186152393167669\"}", - "Public Relations - # eval": "{\"description\": \"min=110, mean=110, max=110, sum=220 (2)\", \"tab\": \"General information\", \"score\": \"110.0\"}", - "Public Relations - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Public Relations - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Public Relations - # prompt tokens": "{\"description\": \"min=408.427, mean=408.427, max=408.427, sum=816.855 (2)\", \"tab\": \"General information\", \"score\": \"408.42727272727274\"}", - "Public Relations - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"public_relations\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_public_relations\"" - } - } - }, - { - "evaluation_name": "Security Studies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Security Studies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.837, - "details": { - "description": "min=0.837, mean=0.837, max=0.837, sum=1.673 (2)", - "tab": "Accuracy", - "Security Studies - Observed inference time (s)": "{\"description\": \"min=0.298, mean=0.298, max=0.298, sum=0.595 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.29758678261114624\"}", - "Security Studies - # eval": "{\"description\": \"min=245, mean=245, max=245, sum=490 (2)\", \"tab\": \"General information\", \"score\": \"245.0\"}", - "Security Studies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Security Studies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Security Studies - # prompt tokens": "{\"description\": \"min=1166.931, mean=1166.931, max=1166.931, sum=2333.861 (2)\", \"tab\": \"General information\", \"score\": \"1166.930612244898\"}", - "Security Studies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"security_studies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_security_studies\"" - } - } - }, - { - "evaluation_name": "Sociology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Sociology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.935, - "details": { - "description": "min=0.935, mean=0.935, max=0.935, sum=1.871 (2)", - "tab": "Accuracy", - "Sociology - Observed inference time (s)": "{\"description\": \"min=0.228, mean=0.228, max=0.228, sum=0.457 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.22830370172339293\"}", - "Sociology - # eval": "{\"description\": \"min=201, mean=201, max=201, sum=402 (2)\", \"tab\": \"General information\", \"score\": \"201.0\"}", - "Sociology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Sociology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Sociology - # prompt tokens": "{\"description\": \"min=450.1, mean=450.1, max=450.1, sum=900.199 (2)\", \"tab\": \"General information\", \"score\": \"450.0995024875622\"}", - "Sociology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"sociology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_sociology\"" - } - } - }, - { - "evaluation_name": "Virology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Virology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.56, - "details": { - "description": "min=0.56, mean=0.56, max=0.56, sum=1.12 (2)", - "tab": "Accuracy", - "Virology - Observed inference time (s)": "{\"description\": \"min=0.25, mean=0.25, max=0.25, sum=0.499 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.24956520206956978\"}", - "Virology - # eval": "{\"description\": \"min=166, mean=166, max=166, sum=332 (2)\", \"tab\": \"General information\", \"score\": \"166.0\"}", - "Virology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Virology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Virology - # prompt tokens": "{\"description\": \"min=343.819, mean=343.819, max=343.819, sum=687.639 (2)\", \"tab\": \"General information\", \"score\": \"343.8192771084337\"}", - "Virology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"virology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_virology\"" - } - } - }, - { - "evaluation_name": "World Religions", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on World Religions", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.848, - "details": { - "description": "min=0.848, mean=0.848, max=0.848, sum=1.696 (2)", - "tab": "Accuracy", - "World Religions - Observed inference time (s)": "{\"description\": \"min=0.197, mean=0.197, max=0.197, sum=0.394 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.19691006342569986\"}", - "World Religions - # eval": "{\"description\": \"min=171, mean=171, max=171, sum=342 (2)\", \"tab\": \"General information\", \"score\": \"171.0\"}", - "World Religions - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "World Religions - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "World Religions - # prompt tokens": "{\"description\": \"min=276.07, mean=276.07, max=276.07, sum=552.14 (2)\", \"tab\": \"General information\", \"score\": \"276.0701754385965\"}", - "World Religions - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"world_religions\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_world_religions\"" - } - } - }, - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.826, - "details": { - "description": "", - "tab": "Efficiency" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_id": "hfopenllm_v2/Qwen_Qwen2-72B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7989 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6977 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4177 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3725 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.456 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5403 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qwen_qwen2-72b.json b/data/models/qwen_qwen2-72b.json deleted file mode 100644 index c1ba947a0309f1c590a6799266788f5481b64058..0000000000000000000000000000000000000000 --- a/data/models/qwen_qwen2-72b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2-72B", - "id": "Qwen/Qwen2-72B", - "developer": "Qwen", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "72.706" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Qwen_Qwen2-72B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3824 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6617 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3112 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3943 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4704 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5731 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qwen_qwen2-7b-instruct.json b/data/models/qwen_qwen2-7b-instruct.json deleted file mode 100644 index 5b51a81dc959f1833a9a4d5071b279f084e47eba..0000000000000000000000000000000000000000 --- a/data/models/qwen_qwen2-7b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2-7B-Instruct", - "id": "Qwen/Qwen2-7B-Instruct", - "developer": "Qwen", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Qwen_Qwen2-7B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5679 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5545 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2764 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2978 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3928 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3847 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qwen_qwen2-7b.json b/data/models/qwen_qwen2-7b.json deleted file mode 100644 index 838fafc57bbf6ed57e2b28354313761dad10165c..0000000000000000000000000000000000000000 --- a/data/models/qwen_qwen2-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2-7B", - "id": "Qwen/Qwen2-7B", - "developer": "Qwen", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Qwen_Qwen2-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3149 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5315 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2039 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3045 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4439 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4183 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qwen_qwen2-math-72b-instruct.json b/data/models/qwen_qwen2-math-72b-instruct.json deleted file mode 100644 index a166fc82be283b189db32a3921c733a71413b9d3..0000000000000000000000000000000000000000 --- a/data/models/qwen_qwen2-math-72b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2-Math-72B-Instruct", - "id": "Qwen/Qwen2-Math-72B-Instruct", - "developer": "Qwen", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "72.706" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Qwen_Qwen2-Math-72B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5694 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6343 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5536 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3683 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4517 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4273 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qwen_qwen2-math-7b.json b/data/models/qwen_qwen2-math-7b.json deleted file mode 100644 index 3bcfcb06cee28264e5d6377ba5e15eabdc2749fd..0000000000000000000000000000000000000000 --- a/data/models/qwen_qwen2-math-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2-Math-7B", - "id": "Qwen/Qwen2-Math-7B", - "developer": "Qwen", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Qwen_Qwen2-Math-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2687 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.387 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2477 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2634 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3593 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1197 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qwen_qwen2-vl-72b-instruct.json b/data/models/qwen_qwen2-vl-72b-instruct.json deleted file mode 100644 index d7b5126125f839715bfadf049267716f2cbc5f52..0000000000000000000000000000000000000000 --- a/data/models/qwen_qwen2-vl-72b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2-VL-72B-Instruct", - "id": "Qwen/Qwen2-VL-72B-Instruct", - "developer": "Qwen", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2VLForConditionalGeneration", - "params_billions": "73.406" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Qwen_Qwen2-VL-72B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5982 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6946 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3444 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3876 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4492 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5717 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qwen_qwen2-vl-7b-instruct.json b/data/models/qwen_qwen2-vl-7b-instruct.json deleted file mode 100644 index 315bb60cfd9e182333e019f4b12c2cd10a8ec7e9..0000000000000000000000000000000000000000 --- a/data/models/qwen_qwen2-vl-7b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2-VL-7B-Instruct", - "id": "Qwen/Qwen2-VL-7B-Instruct", - "developer": "Qwen", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2VLForConditionalGeneration", - "params_billions": "8.291" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Qwen_Qwen2-VL-7B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4599 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5465 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1986 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3196 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4375 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4095 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qwen_qwen2.5-0.5b-instruct.json b/data/models/qwen_qwen2.5-0.5b-instruct.json deleted file mode 100644 index 09847e1fd5f51b0203501d1e930a33e69907de86..0000000000000000000000000000000000000000 --- a/data/models/qwen_qwen2.5-0.5b-instruct.json +++ /dev/null @@ -1,275 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B-Instruct", - "id": "Qwen/Qwen2.5-0.5B-Instruct", - "developer": "Qwen", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Qwen_Qwen2.5-0.5B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3153 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3322 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1035 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2592 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3342 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.172 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "hfopenllm_v2/Qwen_Qwen2.5-0.5B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3071 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3341 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2576 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3329 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1697 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qwen_qwen2.5-0.5b.json b/data/models/qwen_qwen2.5-0.5b.json deleted file mode 100644 index 8220f83e326d4a0194b1abbec924c9c1cd7c3673..0000000000000000000000000000000000000000 --- a/data/models/qwen_qwen2.5-0.5b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-0.5B", - "id": "Qwen/Qwen2.5-0.5B", - "developer": "Qwen", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.5" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Qwen_Qwen2.5-0.5B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1627 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3275 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0393 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2466 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3433 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1906 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qwen_qwen2.5-1.5b-instruct.json b/data/models/qwen_qwen2.5-1.5b-instruct.json deleted file mode 100644 index ccc2592fd757d423673f4b528ed7620c4cc64ac3..0000000000000000000000000000000000000000 --- a/data/models/qwen_qwen2.5-1.5b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-1.5B-Instruct", - "id": "Qwen/Qwen2.5-1.5B-Instruct", - "developer": "Qwen", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.5" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Qwen_Qwen2.5-1.5B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4476 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4289 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2205 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2559 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3663 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2799 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qwen_qwen2.5-1.5b.json b/data/models/qwen_qwen2.5-1.5b.json deleted file mode 100644 index d54df618b5becd85b589fe3eb5d9a0acbc66e77f..0000000000000000000000000000000000000000 --- a/data/models/qwen_qwen2.5-1.5b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-1.5B", - "id": "Qwen/Qwen2.5-1.5B", - "developer": "Qwen", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.5" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Qwen_Qwen2.5-1.5B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2674 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4078 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0914 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2852 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3576 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2855 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qwen_qwen2.5-14b-instruct-1m.json b/data/models/qwen_qwen2.5-14b-instruct-1m.json deleted file mode 100644 index 9238b86a9aef0710a26d4a30e94effd78544c16e..0000000000000000000000000000000000000000 --- a/data/models/qwen_qwen2.5-14b-instruct-1m.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-14B-Instruct-1M", - "id": "Qwen/Qwen2.5-14B-Instruct-1M", - "developer": "Qwen", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Qwen_Qwen2.5-14B-Instruct-1M/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8414 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6198 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5302 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3431 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.418 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.485 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qwen_qwen2.5-14b-instruct.json b/data/models/qwen_qwen2.5-14b-instruct.json deleted file mode 100644 index 3c5e6ee89b2b47b1244f08718812b057d6f8caa3..0000000000000000000000000000000000000000 --- a/data/models/qwen_qwen2.5-14b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-14B-Instruct", - "id": "Qwen/Qwen2.5-14B-Instruct", - "developer": "Qwen", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Qwen_Qwen2.5-14B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8158 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.639 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5476 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3221 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4101 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4904 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qwen_qwen2.5-14b.json b/data/models/qwen_qwen2.5-14b.json deleted file mode 100644 index 42d1d64165afbe5626e11f5ad72495696f8e40f9..0000000000000000000000000000000000000000 --- a/data/models/qwen_qwen2.5-14b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-14B", - "id": "Qwen/Qwen2.5-14B", - "developer": "Qwen", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Qwen_Qwen2.5-14B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3694 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6161 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.29 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3817 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4502 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5249 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qwen_qwen2.5-32b-instruct.json b/data/models/qwen_qwen2.5-32b-instruct.json deleted file mode 100644 index db92810eccd40e086555fd7776dada738d6ff3df..0000000000000000000000000000000000000000 --- a/data/models/qwen_qwen2.5-32b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-32B-Instruct", - "id": "Qwen/Qwen2.5-32B-Instruct", - "developer": "Qwen", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "32.764" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Qwen_Qwen2.5-32B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8346 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6913 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6254 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3381 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4261 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5667 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qwen_qwen2.5-32b.json b/data/models/qwen_qwen2.5-32b.json deleted file mode 100644 index 9adf6882646ddab40ea9d105399bfacbe145b087..0000000000000000000000000000000000000000 --- a/data/models/qwen_qwen2.5-32b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-32B", - "id": "Qwen/Qwen2.5-32B", - "developer": "Qwen", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "32.764" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Qwen_Qwen2.5-32B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4077 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6771 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3565 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4119 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4978 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5805 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qwen_qwen2.5-3b-instruct.json b/data/models/qwen_qwen2.5-3b-instruct.json deleted file mode 100644 index a7b912165d684c557d1da01d4fe87e78c2addb23..0000000000000000000000000000000000000000 --- a/data/models/qwen_qwen2.5-3b-instruct.json +++ /dev/null @@ -1,334 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-3B-Instruct", - "id": "Qwen/Qwen2.5-3B-Instruct", - "developer": "Qwen", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Qwen_Qwen2.5-3B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6475 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4693 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3678 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2727 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3968 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3255 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "theory_of_mind/hf_Qwen_Qwen2.5-3B-Instruct/1772541652.0", - "retrieved_timestamp": "1774793718.284365", - "source_metadata": { - "source_name": "inspect_ai", - "source_type": "evaluation_run", - "source_organization_name": "unknown", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "inspect", - "version": "inspect_ai:0.3.185" - }, - "benchmark": "theory_of_mind", - "evaluation_results": [ - { - "evaluation_name": "accuracy on theory_of_mind for scorer model_graded_fact", - "source_data": { - "dataset_name": "theory_of_mind", - "source_type": "hf_dataset", - "hf_repo": "example://theory_of_mind", - "samples_number": 100, - "sample_ids": [ - "1", - "2", - "3", - "4", - "5", - "6", - "7", - "8", - "9", - "10", - "11", - "12", - "13", - "14", - "15", - "16", - "17", - "18", - "19", - "20", - "21", - "22", - "23", - "24", - "25", - "26", - "27", - "28", - "29", - "30", - "31", - "32", - "33", - "34", - "35", - "36", - "37", - "38", - "39", - "40", - "41", - "42", - "43", - "44", - "45", - "46", - "47", - "48", - "49", - "50", - "51", - "52", - "53", - "54", - "55", - "56", - "57", - "58", - "59", - "60", - "61", - "62", - "63", - "64", - "65", - "66", - "67", - "68", - "69", - "70", - "71", - "72", - "73", - "74", - "75", - "76", - "77", - "78", - "79", - "80", - "81", - "82", - "83", - "84", - "85", - "86", - "87", - "88", - "89", - "90", - "91", - "92", - "93", - "94", - "95", - "96", - "97", - "98", - "99", - "100" - ], - "additional_details": { - "shuffled": "False" - } - }, - "evaluation_timestamp": "1772541652.0", - "metric_config": { - "evaluation_description": "accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.78, - "uncertainty": { - "standard_error": { - "value": 0.04163331998932266 - }, - "num_samples": 100 - } - }, - "generation_config": { - "generation_args": { - "reasoning": false, - "agentic_eval_config": { - "available_tools": [] - }, - "eval_plan": { - "name": "plan", - "steps": [ - "{\"solver\": \"generate\", \"params\": {\"tool_calls\": \"loop\", \"kwargs\": {}}, \"params_passed\": {}}" - ], - "config": {} - }, - "eval_limits": {}, - "sandbox": {} - } - } - } - ], - "detailed_evaluation_results": { - "format": "jsonl", - "file_path": "data/theory_of_mind/Qwen/Qwen2.5-3B-Instruct/30ed1a75-5bfd-4405-abce-b0fd5e0165ba_samples.jsonl", - "hash_algorithm": "sha256", - "checksum": "22c5bd6a8da6c54dfb409425283b1e136f76a225daa48c28b963f3be1f13d697", - "total_rows": 100 - }, - "generation_config": { - "generation_args": { - "reasoning": false, - "agentic_eval_config": { - "available_tools": [] - }, - "eval_plan": { - "name": "plan", - "steps": [ - "{\"solver\": \"generate\", \"params\": {\"tool_calls\": \"loop\", \"kwargs\": {}}, \"params_passed\": {}}" - ], - "config": {} - }, - "eval_limits": {}, - "sandbox": {} - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/qwen_qwen2.5-3b.json b/data/models/qwen_qwen2.5-3b.json deleted file mode 100644 index a9bc4b992103515dd5fbf785a1f341c0065f8362..0000000000000000000000000000000000000000 --- a/data/models/qwen_qwen2.5-3b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-3B", - "id": "Qwen/Qwen2.5-3B", - "developer": "Qwen", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.086" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Qwen_Qwen2.5-3B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.269 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4612 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.148 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2978 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4303 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3203 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qwen_qwen2.5-72b-instruct-turbo.json b/data/models/qwen_qwen2.5-72b-instruct-turbo.json deleted file mode 100644 index 45afd15453d2ea1e46ddbc1a1968fd9f9007f0f8..0000000000000000000000000000000000000000 --- a/data/models/qwen_qwen2.5-72b-instruct-turbo.json +++ /dev/null @@ -1,2128 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5 Instruct Turbo 72B", - "id": "qwen/qwen2.5-72b-instruct-turbo", - "developer": "qwen", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_capabilities/qwen_qwen2.5-72b-instruct-turbo/1774096304.5056698", - "retrieved_timestamp": "1774096304.5056698", - "source_metadata": { - "source_name": "helm_capabilities", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_capabilities", - "evaluation_results": [ - { - "evaluation_name": "Mean score", - "source_data": { - "dataset_name": "helm_capabilities", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "The mean of the scores from all columns.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.599, - "details": { - "description": "", - "tab": "Accuracy", - "Mean score - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"16.666975749955085\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU-Pro", - "source_data": { - "dataset_name": "MMLU-Pro", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on MMLU-Pro", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.631, - "details": { - "description": "min=0.631, mean=0.631, max=0.631, sum=0.631 (1)", - "tab": "Accuracy", - "MMLU-Pro - Observed inference time (s)": "{\"description\": \"min=11.79, mean=11.79, max=11.79, sum=11.79 (1)\", \"tab\": \"Efficiency\", \"score\": \"11.790208662986755\"}", - "MMLU-Pro - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "MMLU-Pro - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - # prompt tokens": "{\"description\": \"min=238.715, mean=238.715, max=238.715, sum=238.715 (1)\", \"tab\": \"General information\", \"score\": \"238.715\"}", - "MMLU-Pro - # output tokens": "{\"description\": \"min=489.611, mean=489.611, max=489.611, sum=489.611 (1)\", \"tab\": \"General information\", \"score\": \"489.611\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"all\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.426, - "details": { - "description": "min=0.426, mean=0.426, max=0.426, sum=0.426 (1)", - "tab": "Accuracy", - "GPQA - Observed inference time (s)": "{\"description\": \"min=28.719, mean=28.719, max=28.719, sum=28.719 (1)\", \"tab\": \"Efficiency\", \"score\": \"28.71905704036422\"}", - "GPQA - # eval": "{\"description\": \"min=446, mean=446, max=446, sum=446 (1)\", \"tab\": \"General information\", \"score\": \"446.0\"}", - "GPQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - # prompt tokens": "{\"description\": \"min=253.37, mean=253.37, max=253.37, sum=253.37 (1)\", \"tab\": \"General information\", \"score\": \"253.36995515695068\"}", - "GPQA - # output tokens": "{\"description\": \"min=704.881, mean=704.881, max=704.881, sum=704.881 (1)\", \"tab\": \"General information\", \"score\": \"704.8811659192825\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"gpqa_main\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "IFEval Strict Acc on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.806, - "details": { - "description": "min=0.806, mean=0.806, max=0.806, sum=0.806 (1)", - "tab": "Accuracy", - "IFEval - Observed inference time (s)": "{\"description\": \"min=20.844, mean=20.844, max=20.844, sum=20.844 (1)\", \"tab\": \"Efficiency\", \"score\": \"20.844201727407036\"}", - "IFEval - # eval": "{\"description\": \"min=541, mean=541, max=541, sum=541 (1)\", \"tab\": \"General information\", \"score\": \"541.0\"}", - "IFEval - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - # prompt tokens": "{\"description\": \"min=46.492, mean=46.492, max=46.492, sum=46.492 (1)\", \"tab\": \"General information\", \"score\": \"46.491682070240294\"}", - "IFEval - # output tokens": "{\"description\": \"min=361.089, mean=361.089, max=361.089, sum=361.089 (1)\", \"tab\": \"General information\", \"score\": \"361.0887245841035\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WildBench", - "source_data": { - "dataset_name": "WildBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "WB Score on WildBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.802, - "details": { - "description": "min=0.802, mean=0.802, max=0.802, sum=0.802 (1)", - "tab": "Accuracy", - "WildBench - Observed inference time (s)": "{\"description\": \"min=12.219, mean=12.219, max=12.219, sum=12.219 (1)\", \"tab\": \"Efficiency\", \"score\": \"12.219232248067856\"}", - "WildBench - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "WildBench - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # prompt tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # output tokens": "{\"description\": \"min=1042.017, mean=1042.017, max=1042.017, sum=1042.017 (1)\", \"tab\": \"General information\", \"score\": \"1042.017\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"v2\"" - } - } - }, - { - "evaluation_name": "Omni-MATH", - "source_data": { - "dataset_name": "Omni-MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Acc on Omni-MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.33, - "details": { - "description": "min=0.33, mean=0.33, max=0.33, sum=0.33 (1)", - "tab": "Accuracy", - "Omni-MATH - Observed inference time (s)": "{\"description\": \"min=9.762, mean=9.762, max=9.762, sum=9.762 (1)\", \"tab\": \"Efficiency\", \"score\": \"9.762179070949555\"}", - "Omni-MATH - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "Omni-MATH - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - # prompt tokens": "{\"description\": \"min=111.6, mean=111.6, max=111.6, sum=111.6 (1)\", \"tab\": \"General information\", \"score\": \"111.6\"}", - "Omni-MATH - # output tokens": "{\"description\": \"min=886.55, mean=886.55, max=886.55, sum=886.55 (1)\", \"tab\": \"General information\", \"score\": \"886.55\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_lite/qwen_qwen2.5-72b-instruct-turbo/1774096306.427425", - "retrieved_timestamp": "1774096306.427425", - "source_metadata": { - "source_name": "helm_lite", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_lite", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_lite", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.745, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.5851310861423221\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.745, - "details": { - "description": "min=0.745, mean=0.745, max=0.745, sum=0.745 (1)", - "tab": "Accuracy", - "NarrativeQA - Observed inference time (s)": "{\"description\": \"min=0.853, mean=0.853, max=0.853, sum=0.853 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.8528219290182624\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=3492.913, mean=3492.913, max=3492.913, sum=3492.913 (1)\", \"tab\": \"General information\", \"score\": \"3492.912676056338\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=8.718, mean=8.718, max=8.718, sum=8.718 (1)\", \"tab\": \"General information\", \"score\": \"8.71830985915493\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (closed-book)", - "source_data": { - "dataset_name": "NaturalQuestions (closed-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (closed-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.359, - "details": { - "description": "min=0.359, mean=0.359, max=0.359, sum=0.359 (1)", - "tab": "Accuracy", - "NaturalQuestions (open-book) - Observed inference time (s)": "{\"description\": \"min=0.974, mean=0.974, max=0.974, sum=0.974 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.9738211624622345\"}", - "NaturalQuestions (closed-book) - Observed inference time (s)": "{\"description\": \"min=0.506, mean=0.506, max=0.506, sum=0.506 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.5063141629695892\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=2007.955, mean=2007.955, max=2007.955, sum=2007.955 (1)\", \"tab\": \"General information\", \"score\": \"2007.955\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=17.681, mean=17.681, max=17.681, sum=17.681 (1)\", \"tab\": \"General information\", \"score\": \"17.681\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=136.262, mean=136.262, max=136.262, sum=136.262 (1)\", \"tab\": \"General information\", \"score\": \"136.262\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=15.132, mean=15.132, max=15.132, sum=15.132 (1)\", \"tab\": \"General information\", \"score\": \"15.132\"}" - } - }, - "generation_config": { - "additional_details": { - "mode": "\"closedbook\"" - } - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.962, - "details": { - "description": "min=0.962, mean=0.962, max=0.962, sum=0.962 (1)", - "tab": "Accuracy", - "OpenbookQA - Observed inference time (s)": "{\"description\": \"min=0.372, mean=0.372, max=0.372, sum=0.372 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.3723496675491333\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=249.846, mean=249.846, max=249.846, sum=249.846 (1)\", \"tab\": \"General information\", \"score\": \"249.846\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "dataset": "\"openbookqa\"", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.77, - "details": { - "description": "min=0.62, mean=0.77, max=0.96, sum=3.848 (5)", - "tab": "Accuracy", - "MMLU - Observed inference time (s)": "{\"description\": \"min=0.438, mean=0.585, max=0.815, sum=2.924 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.5848997679509614\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=378.19, mean=477.836, max=627.939, sum=2389.179 (5)\", \"tab\": \"General information\", \"score\": \"477.8357192982456\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"college_chemistry\", \"computer_security\", \"econometrics\", \"us_foreign_policy\"]", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MATH", - "source_data": { - "dataset_name": "MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Equivalent (CoT) on MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.884, - "details": { - "description": "min=0.763, mean=0.884, max=0.97, sum=6.187 (7)", - "tab": "Accuracy", - "MATH - Observed inference time (s)": "{\"description\": \"min=3.874, mean=6.367, max=11.192, sum=44.569 (7)\", \"tab\": \"Efficiency\", \"score\": \"6.366941373965945\"}", - "MATH - # eval": "{\"description\": \"min=30, mean=62.429, max=135, sum=437 (7)\", \"tab\": \"General information\", \"score\": \"62.42857142857143\"}", - "MATH - # train": "{\"description\": \"min=8, mean=8, max=8, sum=56 (7)\", \"tab\": \"General information\", \"score\": \"8.0\"}", - "MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MATH - # prompt tokens": "{\"description\": \"min=937.926, mean=1323.837, max=2246.673, sum=9266.858 (7)\", \"tab\": \"General information\", \"score\": \"1323.836848955025\"}", - "MATH - # output tokens": "{\"description\": \"min=147.558, mean=186.764, max=230.288, sum=1307.351 (7)\", \"tab\": \"General information\", \"score\": \"186.76438709076407\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"algebra\", \"counting_and_probability\", \"geometry\", \"intermediate_algebra\", \"number_theory\", \"prealgebra\", \"precalculus\"]", - "level": "\"1\"", - "use_official_examples": "\"False\"", - "use_chain_of_thought": "\"True\"" - } - } - }, - { - "evaluation_name": "GSM8K", - "source_data": { - "dataset_name": "GSM8K", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on GSM8K", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9, - "details": { - "description": "min=0.9, mean=0.9, max=0.9, sum=0.9 (1)", - "tab": "Accuracy", - "GSM8K - Observed inference time (s)": "{\"description\": \"min=2.558, mean=2.558, max=2.558, sum=2.558 (1)\", \"tab\": \"Efficiency\", \"score\": \"2.5583292784690856\"}", - "GSM8K - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "GSM8K - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "GSM8K - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GSM8K - # prompt tokens": "{\"description\": \"min=1130.403, mean=1130.403, max=1130.403, sum=1130.403 (1)\", \"tab\": \"General information\", \"score\": \"1130.403\"}", - "GSM8K - # output tokens": "{\"description\": \"min=198.303, mean=198.303, max=198.303, sum=198.303 (1)\", \"tab\": \"General information\", \"score\": \"198.303\"}" - } - }, - "generation_config": { - "additional_details": { - "stop": "\"none\"" - } - } - }, - { - "evaluation_name": "LegalBench", - "source_data": { - "dataset_name": "LegalBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on LegalBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.74, - "details": { - "description": "min=0.46, mean=0.74, max=0.979, sum=3.7 (5)", - "tab": "Accuracy", - "LegalBench - Observed inference time (s)": "{\"description\": \"min=0.306, mean=0.445, max=0.944, sum=2.224 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.44489043568091446\"}", - "LegalBench - # eval": "{\"description\": \"min=95, mean=409.4, max=1000, sum=2047 (5)\", \"tab\": \"General information\", \"score\": \"409.4\"}", - "LegalBench - # train": "{\"description\": \"min=4, mean=4.8, max=5, sum=24 (5)\", \"tab\": \"General information\", \"score\": \"4.8\"}", - "LegalBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "LegalBench - # prompt tokens": "{\"description\": \"min=216.453, mean=1558.888, max=6440.714, sum=7794.442 (5)\", \"tab\": \"General information\", \"score\": \"1558.8883229968653\"}", - "LegalBench - # output tokens": "{\"description\": \"min=2, mean=2.453, max=3.021, sum=12.263 (5)\", \"tab\": \"General information\", \"score\": \"2.452587326627195\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"abercrombie\", \"corporate_lobbying\", \"function_of_decision_section\", \"international_citizenship_questions\", \"proa\"]", - "stop": "\"none\"" - } - } - }, - { - "evaluation_name": "MedQA", - "source_data": { - "dataset_name": "MedQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MedQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.753, - "details": { - "description": "min=0.753, mean=0.753, max=0.753, sum=0.753 (1)", - "tab": "Accuracy", - "MedQA - Observed inference time (s)": "{\"description\": \"min=0.332, mean=0.332, max=0.332, sum=0.332 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.33223102912751157\"}", - "MedQA - # eval": "{\"description\": \"min=503, mean=503, max=503, sum=503 (1)\", \"tab\": \"General information\", \"score\": \"503.0\"}", - "MedQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MedQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MedQA - # prompt tokens": "{\"description\": \"min=1052.485, mean=1052.485, max=1052.485, sum=1052.485 (1)\", \"tab\": \"General information\", \"score\": \"1052.4850894632207\"}", - "MedQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WMT 2014", - "source_data": { - "dataset_name": "WMT 2014", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "BLEU-4 on WMT 2014", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.207, - "details": { - "description": "min=0.153, mean=0.207, max=0.257, sum=1.033 (5)", - "tab": "Accuracy", - "WMT 2014 - Observed inference time (s)": "{\"description\": \"min=0.635, mean=0.67, max=0.752, sum=3.351 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.6702916101891663\"}", - "WMT 2014 - # eval": "{\"description\": \"min=503, mean=568.8, max=832, sum=2844 (5)\", \"tab\": \"General information\", \"score\": \"568.8\"}", - "WMT 2014 - # train": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "WMT 2014 - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WMT 2014 - # prompt tokens": "{\"description\": \"min=114.855, mean=132.657, max=148.373, sum=663.283 (5)\", \"tab\": \"General information\", \"score\": \"132.65662658663405\"}", - "WMT 2014 - # output tokens": "{\"description\": \"min=25.517, mean=27.126, max=27.755, sum=135.631 (5)\", \"tab\": \"General information\", \"score\": \"27.126178505887747\"}" - } - }, - "generation_config": { - "additional_details": { - "language_pair": "[\"cs-en\", \"de-en\", \"fr-en\", \"hi-en\", \"ru-en\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_mmlu/qwen_qwen2.5-72b-instruct-turbo/1774096312.00548", - "retrieved_timestamp": "1774096312.00548", - "source_metadata": { - "source_name": "helm_mmlu", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_mmlu", - "evaluation_results": [ - { - "evaluation_name": "MMLU All Subjects", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU All Subjects", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.834, - "details": { - "description": "min=0.584, mean=0.834, max=0.99, sum=95.044 (114)", - "tab": "Accuracy", - "MMLU All Subjects - Observed inference time (s)": "{\"description\": \"min=0.277, mean=0.504, max=1.68, sum=57.492 (114)\", \"tab\": \"Efficiency\", \"score\": \"0.5043123259817794\"}", - "MMLU All Subjects - # eval": "{\"description\": \"min=100, mean=246.351, max=1534, sum=28084 (114)\", \"tab\": \"General information\", \"score\": \"246.35087719298247\"}", - "MMLU All Subjects - # train": "{\"description\": \"min=5, mean=5, max=5, sum=570 (114)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU All Subjects - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU All Subjects - # prompt tokens": "{\"description\": \"min=276.07, mean=625.598, max=2814.903, sum=71318.198 (114)\", \"tab\": \"General information\", \"score\": \"625.5982315160392\"}", - "MMLU All Subjects - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=114 (114)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_name": "Abstract Algebra", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Abstract Algebra", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.68, - "details": { - "description": "min=0.68, mean=0.68, max=0.68, sum=1.36 (2)", - "tab": "Accuracy", - "Abstract Algebra - Observed inference time (s)": "{\"description\": \"min=0.438, mean=0.438, max=0.438, sum=0.877 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.438259596824646\"}", - "Abstract Algebra - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Abstract Algebra - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Abstract Algebra - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Abstract Algebra - # prompt tokens": "{\"description\": \"min=378.19, mean=378.19, max=378.19, sum=756.38 (2)\", \"tab\": \"General information\", \"score\": \"378.19\"}", - "Abstract Algebra - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"abstract_algebra\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_abstract_algebra\"" - } - } - }, - { - "evaluation_name": "Anatomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Anatomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.822, - "details": { - "description": "min=0.822, mean=0.822, max=0.822, sum=1.644 (2)", - "tab": "Accuracy", - "Anatomy - Observed inference time (s)": "{\"description\": \"min=0.365, mean=0.365, max=0.365, sum=0.729 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3645249543366609\"}", - "Anatomy - # eval": "{\"description\": \"min=135, mean=135, max=135, sum=270 (2)\", \"tab\": \"General information\", \"score\": \"135.0\"}", - "Anatomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Anatomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Anatomy - # prompt tokens": "{\"description\": \"min=353.978, mean=353.978, max=353.978, sum=707.956 (2)\", \"tab\": \"General information\", \"score\": \"353.97777777777776\"}", - "Anatomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"anatomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_anatomy\"" - } - } - }, - { - "evaluation_name": "College Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on College Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.588, - "details": { - "description": "min=0.588, mean=0.588, max=0.588, sum=1.176 (2)", - "tab": "Accuracy", - "College Chemistry - Observed inference time (s)": "{\"description\": \"min=0.519, mean=0.519, max=0.519, sum=1.038 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5187593793869019\"}", - "College Biology - Observed inference time (s)": "{\"description\": \"min=0.406, mean=0.406, max=0.406, sum=0.811 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.40557659500175053\"}", - "College Computer Science - Observed inference time (s)": "{\"description\": \"min=0.485, mean=0.485, max=0.485, sum=0.97 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.48524248123168945\"}", - "College Mathematics - Observed inference time (s)": "{\"description\": \"min=0.566, mean=0.566, max=0.566, sum=1.132 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5662378907203675\"}", - "College Medicine - Observed inference time (s)": "{\"description\": \"min=0.528, mean=0.528, max=0.528, sum=1.055 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5277049872227487\"}", - "College Physics - Observed inference time (s)": "{\"description\": \"min=0.45, mean=0.45, max=0.45, sum=0.9 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4500672326368444\"}", - "College Chemistry - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Chemistry - # prompt tokens": "{\"description\": \"min=568.25, mean=568.25, max=568.25, sum=1136.5 (2)\", \"tab\": \"General information\", \"score\": \"568.25\"}", - "College Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Biology - # eval": "{\"description\": \"min=144, mean=144, max=144, sum=288 (2)\", \"tab\": \"General information\", \"score\": \"144.0\"}", - "College Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # prompt tokens": "{\"description\": \"min=486.979, mean=486.979, max=486.979, sum=973.958 (2)\", \"tab\": \"General information\", \"score\": \"486.9791666666667\"}", - "College Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # prompt tokens": "{\"description\": \"min=838.58, mean=838.58, max=838.58, sum=1677.16 (2)\", \"tab\": \"General information\", \"score\": \"838.58\"}", - "College Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Mathematics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # prompt tokens": "{\"description\": \"min=607.7, mean=607.7, max=607.7, sum=1215.4 (2)\", \"tab\": \"General information\", \"score\": \"607.7\"}", - "College Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Medicine - # eval": "{\"description\": \"min=173, mean=173, max=173, sum=346 (2)\", \"tab\": \"General information\", \"score\": \"173.0\"}", - "College Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # prompt tokens": "{\"description\": \"min=506.098, mean=506.098, max=506.098, sum=1012.197 (2)\", \"tab\": \"General information\", \"score\": \"506.0982658959538\"}", - "College Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Physics - # eval": "{\"description\": \"min=102, mean=102, max=102, sum=204 (2)\", \"tab\": \"General information\", \"score\": \"102.0\"}", - "College Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # prompt tokens": "{\"description\": \"min=516.265, mean=516.265, max=516.265, sum=1032.529 (2)\", \"tab\": \"General information\", \"score\": \"516.2647058823529\"}", - "College Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"college_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_college_physics\"" - } - } - }, - { - "evaluation_name": "Computer Security", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Computer Security", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.86, - "details": { - "description": "min=0.86, mean=0.86, max=0.86, sum=1.72 (2)", - "tab": "Accuracy", - "Computer Security - Observed inference time (s)": "{\"description\": \"min=0.506, mean=0.506, max=0.506, sum=1.011 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5056298255920411\"}", - "Computer Security - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Computer Security - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Computer Security - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Computer Security - # prompt tokens": "{\"description\": \"min=386.64, mean=386.64, max=386.64, sum=773.28 (2)\", \"tab\": \"General information\", \"score\": \"386.64\"}", - "Computer Security - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"computer_security\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_computer_security\"" - } - } - }, - { - "evaluation_name": "Econometrics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Econometrics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.728, - "details": { - "description": "min=0.728, mean=0.728, max=0.728, sum=1.456 (2)", - "tab": "Accuracy", - "Econometrics - Observed inference time (s)": "{\"description\": \"min=0.646, mean=0.646, max=0.646, sum=1.293 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6464532927462929\"}", - "Econometrics - # eval": "{\"description\": \"min=114, mean=114, max=114, sum=228 (2)\", \"tab\": \"General information\", \"score\": \"114.0\"}", - "Econometrics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Econometrics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Econometrics - # prompt tokens": "{\"description\": \"min=627.939, mean=627.939, max=627.939, sum=1255.877 (2)\", \"tab\": \"General information\", \"score\": \"627.938596491228\"}", - "Econometrics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"econometrics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_econometrics\"" - } - } - }, - { - "evaluation_name": "Global Facts", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Global Facts", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.61, - "details": { - "description": "min=0.61, mean=0.61, max=0.61, sum=1.22 (2)", - "tab": "Accuracy", - "Global Facts - Observed inference time (s)": "{\"description\": \"min=0.517, mean=0.517, max=0.517, sum=1.035 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5174938654899597\"}", - "Global Facts - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Global Facts - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Global Facts - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Global Facts - # prompt tokens": "{\"description\": \"min=429.06, mean=429.06, max=429.06, sum=858.12 (2)\", \"tab\": \"General information\", \"score\": \"429.06\"}", - "Global Facts - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"global_facts\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_global_facts\"" - } - } - }, - { - "evaluation_name": "Jurisprudence", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Jurisprudence", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.87, - "details": { - "description": "min=0.87, mean=0.87, max=0.87, sum=1.741 (2)", - "tab": "Accuracy", - "Jurisprudence - Observed inference time (s)": "{\"description\": \"min=0.552, mean=0.552, max=0.552, sum=1.105 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.55242551918383\"}", - "Jurisprudence - # eval": "{\"description\": \"min=108, mean=108, max=108, sum=216 (2)\", \"tab\": \"General information\", \"score\": \"108.0\"}", - "Jurisprudence - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Jurisprudence - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Jurisprudence - # prompt tokens": "{\"description\": \"min=394.713, mean=394.713, max=394.713, sum=789.426 (2)\", \"tab\": \"General information\", \"score\": \"394.712962962963\"}", - "Jurisprudence - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"jurisprudence\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_jurisprudence\"" - } - } - }, - { - "evaluation_name": "Philosophy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Philosophy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.839, - "details": { - "description": "min=0.839, mean=0.839, max=0.839, sum=1.678 (2)", - "tab": "Accuracy", - "Philosophy - Observed inference time (s)": "{\"description\": \"min=1.352, mean=1.352, max=1.352, sum=2.704 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.3517981679493207\"}", - "Philosophy - # eval": "{\"description\": \"min=311, mean=311, max=311, sum=622 (2)\", \"tab\": \"General information\", \"score\": \"311.0\"}", - "Philosophy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Philosophy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Philosophy - # prompt tokens": "{\"description\": \"min=329.09, mean=329.09, max=329.09, sum=658.18 (2)\", \"tab\": \"General information\", \"score\": \"329.09003215434086\"}", - "Philosophy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"philosophy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_philosophy\"" - } - } - }, - { - "evaluation_name": "Professional Psychology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Professional Psychology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.864, - "details": { - "description": "min=0.864, mean=0.864, max=0.864, sum=1.729 (2)", - "tab": "Accuracy", - "Professional Medicine - Observed inference time (s)": "{\"description\": \"min=1.02, mean=1.02, max=1.02, sum=2.039 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.019735706203124\"}", - "Professional Accounting - Observed inference time (s)": "{\"description\": \"min=0.282, mean=0.282, max=0.282, sum=0.565 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2822888328673992\"}", - "Professional Law - Observed inference time (s)": "{\"description\": \"min=1.68, mean=1.68, max=1.68, sum=3.36 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.6800112862630494\"}", - "Professional Psychology - Observed inference time (s)": "{\"description\": \"min=0.573, mean=0.573, max=0.573, sum=1.145 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5726091144910825\"}", - "Professional Medicine - # eval": "{\"description\": \"min=272, mean=272, max=272, sum=544 (2)\", \"tab\": \"General information\", \"score\": \"272.0\"}", - "Professional Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Medicine - # prompt tokens": "{\"description\": \"min=1125.199, mean=1125.199, max=1125.199, sum=2250.397 (2)\", \"tab\": \"General information\", \"score\": \"1125.1985294117646\"}", - "Professional Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Accounting - # eval": "{\"description\": \"min=282, mean=282, max=282, sum=564 (2)\", \"tab\": \"General information\", \"score\": \"282.0\"}", - "Professional Accounting - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Accounting - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # prompt tokens": "{\"description\": \"min=739.34, mean=739.34, max=739.34, sum=1478.681 (2)\", \"tab\": \"General information\", \"score\": \"739.3404255319149\"}", - "Professional Accounting - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Law - # eval": "{\"description\": \"min=1534, mean=1534, max=1534, sum=3068 (2)\", \"tab\": \"General information\", \"score\": \"1534.0\"}", - "Professional Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # prompt tokens": "{\"description\": \"min=1663.969, mean=1663.969, max=1663.969, sum=3327.939 (2)\", \"tab\": \"General information\", \"score\": \"1663.9693611473272\"}", - "Professional Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Psychology - # eval": "{\"description\": \"min=612, mean=612, max=612, sum=1224 (2)\", \"tab\": \"General information\", \"score\": \"612.0\"}", - "Professional Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # prompt tokens": "{\"description\": \"min=581.417, mean=581.417, max=581.417, sum=1162.833 (2)\", \"tab\": \"General information\", \"score\": \"581.4166666666666\"}", - "Professional Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"professional_psychology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_professional_psychology\"" - } - } - }, - { - "evaluation_name": "Us Foreign Policy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Us Foreign Policy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.96, - "details": { - "description": "min=0.96, mean=0.96, max=0.96, sum=1.92 (2)", - "tab": "Accuracy", - "Us Foreign Policy - Observed inference time (s)": "{\"description\": \"min=0.789, mean=0.789, max=0.789, sum=1.578 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7888539290428161\"}", - "Us Foreign Policy - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Us Foreign Policy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Us Foreign Policy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Us Foreign Policy - # prompt tokens": "{\"description\": \"min=428.16, mean=428.16, max=428.16, sum=856.32 (2)\", \"tab\": \"General information\", \"score\": \"428.16\"}", - "Us Foreign Policy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"us_foreign_policy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_us_foreign_policy\"" - } - } - }, - { - "evaluation_name": "Astronomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Astronomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.934, - "details": { - "description": "min=0.934, mean=0.934, max=0.934, sum=1.868 (2)", - "tab": "Accuracy", - "Astronomy - Observed inference time (s)": "{\"description\": \"min=0.991, mean=0.991, max=0.991, sum=1.983 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9913477442766491\"}", - "Astronomy - # eval": "{\"description\": \"min=152, mean=152, max=152, sum=304 (2)\", \"tab\": \"General information\", \"score\": \"152.0\"}", - "Astronomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Astronomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Astronomy - # prompt tokens": "{\"description\": \"min=589.849, mean=589.849, max=589.849, sum=1179.697 (2)\", \"tab\": \"General information\", \"score\": \"589.8486842105264\"}", - "Astronomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"astronomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_astronomy\"" - } - } - }, - { - "evaluation_name": "Business Ethics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Business Ethics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.85, - "details": { - "description": "min=0.85, mean=0.85, max=0.85, sum=1.7 (2)", - "tab": "Accuracy", - "Business Ethics - Observed inference time (s)": "{\"description\": \"min=0.581, mean=0.581, max=0.581, sum=1.163 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5813773083686828\"}", - "Business Ethics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Business Ethics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Business Ethics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Business Ethics - # prompt tokens": "{\"description\": \"min=569.87, mean=569.87, max=569.87, sum=1139.74 (2)\", \"tab\": \"General information\", \"score\": \"569.87\"}", - "Business Ethics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"business_ethics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_business_ethics\"" - } - } - }, - { - "evaluation_name": "Clinical Knowledge", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Clinical Knowledge", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.872, - "details": { - "description": "min=0.872, mean=0.872, max=0.872, sum=1.743 (2)", - "tab": "Accuracy", - "Clinical Knowledge - Observed inference time (s)": "{\"description\": \"min=0.74, mean=0.74, max=0.74, sum=1.48 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7399316436839554\"}", - "Clinical Knowledge - # eval": "{\"description\": \"min=265, mean=265, max=265, sum=530 (2)\", \"tab\": \"General information\", \"score\": \"265.0\"}", - "Clinical Knowledge - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Clinical Knowledge - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Clinical Knowledge - # prompt tokens": "{\"description\": \"min=400.623, mean=400.623, max=400.623, sum=801.245 (2)\", \"tab\": \"General information\", \"score\": \"400.62264150943395\"}", - "Clinical Knowledge - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"clinical_knowledge\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_clinical_knowledge\"" - } - } - }, - { - "evaluation_name": "Conceptual Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Conceptual Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.885, - "details": { - "description": "min=0.885, mean=0.885, max=0.885, sum=1.77 (2)", - "tab": "Accuracy", - "Conceptual Physics - Observed inference time (s)": "{\"description\": \"min=0.321, mean=0.321, max=0.321, sum=0.643 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.32127690010882437\"}", - "Conceptual Physics - # eval": "{\"description\": \"min=235, mean=235, max=235, sum=470 (2)\", \"tab\": \"General information\", \"score\": \"235.0\"}", - "Conceptual Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Conceptual Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Conceptual Physics - # prompt tokens": "{\"description\": \"min=305.494, mean=305.494, max=305.494, sum=610.987 (2)\", \"tab\": \"General information\", \"score\": \"305.4936170212766\"}", - "Conceptual Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"conceptual_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_conceptual_physics\"" - } - } - }, - { - "evaluation_name": "Electrical Engineering", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Electrical Engineering", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8, - "details": { - "description": "min=0.8, mean=0.8, max=0.8, sum=1.6 (2)", - "tab": "Accuracy", - "Electrical Engineering - Observed inference time (s)": "{\"description\": \"min=0.513, mean=0.513, max=0.513, sum=1.026 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5130313610208446\"}", - "Electrical Engineering - # eval": "{\"description\": \"min=145, mean=145, max=145, sum=290 (2)\", \"tab\": \"General information\", \"score\": \"145.0\"}", - "Electrical Engineering - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Electrical Engineering - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Electrical Engineering - # prompt tokens": "{\"description\": \"min=463.8, mean=463.8, max=463.8, sum=927.6 (2)\", \"tab\": \"General information\", \"score\": \"463.8\"}", - "Electrical Engineering - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"electrical_engineering\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_electrical_engineering\"" - } - } - }, - { - "evaluation_name": "Elementary Mathematics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Elementary Mathematics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.87, - "details": { - "description": "min=0.87, mean=0.87, max=0.87, sum=1.741 (2)", - "tab": "Accuracy", - "Elementary Mathematics - Observed inference time (s)": "{\"description\": \"min=1.022, mean=1.022, max=1.022, sum=2.044 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.0221643580330744\"}", - "Elementary Mathematics - # eval": "{\"description\": \"min=378, mean=378, max=378, sum=756 (2)\", \"tab\": \"General information\", \"score\": \"378.0\"}", - "Elementary Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Elementary Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Elementary Mathematics - # prompt tokens": "{\"description\": \"min=577.119, mean=577.119, max=577.119, sum=1154.238 (2)\", \"tab\": \"General information\", \"score\": \"577.1190476190476\"}", - "Elementary Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"elementary_mathematics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_elementary_mathematics\"" - } - } - }, - { - "evaluation_name": "Formal Logic", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Formal Logic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.73, - "details": { - "description": "min=0.73, mean=0.73, max=0.73, sum=1.46 (2)", - "tab": "Accuracy", - "Formal Logic - Observed inference time (s)": "{\"description\": \"min=0.489, mean=0.489, max=0.489, sum=0.978 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.48887844501979766\"}", - "Formal Logic - # eval": "{\"description\": \"min=126, mean=126, max=126, sum=252 (2)\", \"tab\": \"General information\", \"score\": \"126.0\"}", - "Formal Logic - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Formal Logic - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Formal Logic - # prompt tokens": "{\"description\": \"min=604.667, mean=604.667, max=604.667, sum=1209.333 (2)\", \"tab\": \"General information\", \"score\": \"604.6666666666666\"}", - "Formal Logic - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"formal_logic\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_formal_logic\"" - } - } - }, - { - "evaluation_name": "High School World History", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on High School World History", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.92, - "details": { - "description": "min=0.92, mean=0.92, max=0.92, sum=1.84 (2)", - "tab": "Accuracy", - "High School Biology - Observed inference time (s)": "{\"description\": \"min=0.342, mean=0.342, max=0.342, sum=0.685 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.34227523111527963\"}", - "High School Chemistry - Observed inference time (s)": "{\"description\": \"min=0.336, mean=0.336, max=0.336, sum=0.673 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3364456193200473\"}", - "High School Computer Science - Observed inference time (s)": "{\"description\": \"min=0.384, mean=0.384, max=0.384, sum=0.768 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.38405280351638793\"}", - "High School European History - Observed inference time (s)": "{\"description\": \"min=0.582, mean=0.582, max=0.582, sum=1.165 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5822634451317065\"}", - "High School Geography - Observed inference time (s)": "{\"description\": \"min=0.366, mean=0.366, max=0.366, sum=0.731 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3657490508724945\"}", - "High School Government And Politics - Observed inference time (s)": "{\"description\": \"min=0.388, mean=0.388, max=0.388, sum=0.776 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3882344139672314\"}", - "High School Macroeconomics - Observed inference time (s)": "{\"description\": \"min=0.311, mean=0.311, max=0.311, sum=0.623 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.31144848542335707\"}", - "High School Mathematics - Observed inference time (s)": "{\"description\": \"min=0.364, mean=0.364, max=0.364, sum=0.727 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3636930130146168\"}", - "High School Microeconomics - Observed inference time (s)": "{\"description\": \"min=0.572, mean=0.572, max=0.572, sum=1.145 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5723558383829453\"}", - "High School Physics - Observed inference time (s)": "{\"description\": \"min=0.891, mean=0.891, max=0.891, sum=1.782 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8909238490047834\"}", - "High School Psychology - Observed inference time (s)": "{\"description\": \"min=0.312, mean=0.312, max=0.312, sum=0.623 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.31171117397623327\"}", - "High School Statistics - Observed inference time (s)": "{\"description\": \"min=0.376, mean=0.376, max=0.376, sum=0.751 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3756344163859332\"}", - "High School US History - Observed inference time (s)": "{\"description\": \"min=0.453, mean=0.453, max=0.453, sum=0.907 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.45333802466299017\"}", - "High School World History - Observed inference time (s)": "{\"description\": \"min=0.526, mean=0.526, max=0.526, sum=1.051 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5255286924949678\"}", - "High School Biology - # eval": "{\"description\": \"min=310, mean=310, max=310, sum=620 (2)\", \"tab\": \"General information\", \"score\": \"310.0\"}", - "High School Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Biology - # prompt tokens": "{\"description\": \"min=513.916, mean=513.916, max=513.916, sum=1027.832 (2)\", \"tab\": \"General information\", \"score\": \"513.916129032258\"}", - "High School Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Chemistry - # eval": "{\"description\": \"min=203, mean=203, max=203, sum=406 (2)\", \"tab\": \"General information\", \"score\": \"203.0\"}", - "High School Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # prompt tokens": "{\"description\": \"min=517.261, mean=517.261, max=517.261, sum=1034.522 (2)\", \"tab\": \"General information\", \"score\": \"517.2610837438424\"}", - "High School Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "High School Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # prompt tokens": "{\"description\": \"min=878.46, mean=878.46, max=878.46, sum=1756.92 (2)\", \"tab\": \"General information\", \"score\": \"878.46\"}", - "High School Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School European History - # eval": "{\"description\": \"min=165, mean=165, max=165, sum=330 (2)\", \"tab\": \"General information\", \"score\": \"165.0\"}", - "High School European History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School European History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # prompt tokens": "{\"description\": \"min=2814.903, mean=2814.903, max=2814.903, sum=5629.806 (2)\", \"tab\": \"General information\", \"score\": \"2814.9030303030304\"}", - "High School European History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Geography - # eval": "{\"description\": \"min=198, mean=198, max=198, sum=396 (2)\", \"tab\": \"General information\", \"score\": \"198.0\"}", - "High School Geography - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Geography - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # prompt tokens": "{\"description\": \"min=372.217, mean=372.217, max=372.217, sum=744.434 (2)\", \"tab\": \"General information\", \"score\": \"372.2171717171717\"}", - "High School Geography - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Government And Politics - # eval": "{\"description\": \"min=193, mean=193, max=193, sum=386 (2)\", \"tab\": \"General information\", \"score\": \"193.0\"}", - "High School Government And Politics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Government And Politics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # prompt tokens": "{\"description\": \"min=467.311, mean=467.311, max=467.311, sum=934.622 (2)\", \"tab\": \"General information\", \"score\": \"467.31088082901556\"}", - "High School Government And Politics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Macroeconomics - # eval": "{\"description\": \"min=390, mean=390, max=390, sum=780 (2)\", \"tab\": \"General information\", \"score\": \"390.0\"}", - "High School Macroeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Macroeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # prompt tokens": "{\"description\": \"min=374.349, mean=374.349, max=374.349, sum=748.697 (2)\", \"tab\": \"General information\", \"score\": \"374.34871794871793\"}", - "High School Macroeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Mathematics - # eval": "{\"description\": \"min=270, mean=270, max=270, sum=540 (2)\", \"tab\": \"General information\", \"score\": \"270.0\"}", - "High School Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # prompt tokens": "{\"description\": \"min=565.326, mean=565.326, max=565.326, sum=1130.652 (2)\", \"tab\": \"General information\", \"score\": \"565.325925925926\"}", - "High School Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Microeconomics - # eval": "{\"description\": \"min=238, mean=238, max=238, sum=476 (2)\", \"tab\": \"General information\", \"score\": \"238.0\"}", - "High School Microeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Microeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # prompt tokens": "{\"description\": \"min=402.277, mean=402.277, max=402.277, sum=804.555 (2)\", \"tab\": \"General information\", \"score\": \"402.2773109243698\"}", - "High School Microeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Physics - # eval": "{\"description\": \"min=151, mean=151, max=151, sum=302 (2)\", \"tab\": \"General information\", \"score\": \"151.0\"}", - "High School Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # prompt tokens": "{\"description\": \"min=580.536, mean=580.536, max=580.536, sum=1161.073 (2)\", \"tab\": \"General information\", \"score\": \"580.5364238410596\"}", - "High School Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Psychology - # eval": "{\"description\": \"min=545, mean=545, max=545, sum=1090 (2)\", \"tab\": \"General information\", \"score\": \"545.0\"}", - "High School Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # prompt tokens": "{\"description\": \"min=495.521, mean=495.521, max=495.521, sum=991.042 (2)\", \"tab\": \"General information\", \"score\": \"495.52110091743117\"}", - "High School Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Statistics - # eval": "{\"description\": \"min=216, mean=216, max=216, sum=432 (2)\", \"tab\": \"General information\", \"score\": \"216.0\"}", - "High School Statistics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Statistics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # prompt tokens": "{\"description\": \"min=830.477, mean=830.477, max=830.477, sum=1660.954 (2)\", \"tab\": \"General information\", \"score\": \"830.4768518518518\"}", - "High School Statistics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School US History - # eval": "{\"description\": \"min=204, mean=204, max=204, sum=408 (2)\", \"tab\": \"General information\", \"score\": \"204.0\"}", - "High School US History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School US History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # prompt tokens": "{\"description\": \"min=2237.176, mean=2237.176, max=2237.176, sum=4474.353 (2)\", \"tab\": \"General information\", \"score\": \"2237.176470588235\"}", - "High School US History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School World History - # eval": "{\"description\": \"min=237, mean=237, max=237, sum=474 (2)\", \"tab\": \"General information\", \"score\": \"237.0\"}", - "High School World History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School World History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # prompt tokens": "{\"description\": \"min=1448.354, mean=1448.354, max=1448.354, sum=2896.709 (2)\", \"tab\": \"General information\", \"score\": \"1448.3544303797469\"}", - "High School World History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"high_school_world_history\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_high_school_world_history\"" - } - } - }, - { - "evaluation_name": "Human Sexuality", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Human Sexuality", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.878, - "details": { - "description": "min=0.878, mean=0.878, max=0.878, sum=1.756 (2)", - "tab": "Accuracy", - "Human Aging - Observed inference time (s)": "{\"description\": \"min=0.428, mean=0.428, max=0.428, sum=0.856 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.42812311168208783\"}", - "Human Sexuality - Observed inference time (s)": "{\"description\": \"min=0.318, mean=0.318, max=0.318, sum=0.635 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3175856612110866\"}", - "Human Aging - # eval": "{\"description\": \"min=223, mean=223, max=223, sum=446 (2)\", \"tab\": \"General information\", \"score\": \"223.0\"}", - "Human Aging - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Aging - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Aging - # prompt tokens": "{\"description\": \"min=322.121, mean=322.121, max=322.121, sum=644.242 (2)\", \"tab\": \"General information\", \"score\": \"322.1210762331838\"}", - "Human Aging - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Human Sexuality - # eval": "{\"description\": \"min=131, mean=131, max=131, sum=262 (2)\", \"tab\": \"General information\", \"score\": \"131.0\"}", - "Human Sexuality - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Sexuality - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # prompt tokens": "{\"description\": \"min=341.504, mean=341.504, max=341.504, sum=683.008 (2)\", \"tab\": \"General information\", \"score\": \"341.5038167938931\"}", - "Human Sexuality - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"human_sexuality\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_human_sexuality\"" - } - } - }, - { - "evaluation_name": "International Law", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on International Law", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.893, - "details": { - "description": "min=0.893, mean=0.893, max=0.893, sum=1.785 (2)", - "tab": "Accuracy", - "International Law - Observed inference time (s)": "{\"description\": \"min=0.425, mean=0.425, max=0.425, sum=0.85 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4248029200498723\"}", - "International Law - # eval": "{\"description\": \"min=121, mean=121, max=121, sum=242 (2)\", \"tab\": \"General information\", \"score\": \"121.0\"}", - "International Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "International Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "International Law - # prompt tokens": "{\"description\": \"min=640.579, mean=640.579, max=640.579, sum=1281.157 (2)\", \"tab\": \"General information\", \"score\": \"640.5785123966942\"}", - "International Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"international_law\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_international_law\"" - } - } - }, - { - "evaluation_name": "Logical Fallacies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Logical Fallacies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.89, - "details": { - "description": "min=0.89, mean=0.89, max=0.89, sum=1.779 (2)", - "tab": "Accuracy", - "Logical Fallacies - Observed inference time (s)": "{\"description\": \"min=0.346, mean=0.346, max=0.346, sum=0.692 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3458571419394089\"}", - "Logical Fallacies - # eval": "{\"description\": \"min=163, mean=163, max=163, sum=326 (2)\", \"tab\": \"General information\", \"score\": \"163.0\"}", - "Logical Fallacies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Logical Fallacies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Logical Fallacies - # prompt tokens": "{\"description\": \"min=449.632, mean=449.632, max=449.632, sum=899.264 (2)\", \"tab\": \"General information\", \"score\": \"449.6319018404908\"}", - "Logical Fallacies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"logical_fallacies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_logical_fallacies\"" - } - } - }, - { - "evaluation_name": "Machine Learning", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Machine Learning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.777, - "details": { - "description": "min=0.777, mean=0.777, max=0.777, sum=1.554 (2)", - "tab": "Accuracy", - "Machine Learning - Observed inference time (s)": "{\"description\": \"min=0.348, mean=0.348, max=0.348, sum=0.697 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3483003888811384\"}", - "Machine Learning - # eval": "{\"description\": \"min=112, mean=112, max=112, sum=224 (2)\", \"tab\": \"General information\", \"score\": \"112.0\"}", - "Machine Learning - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Machine Learning - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Machine Learning - # prompt tokens": "{\"description\": \"min=681.848, mean=681.848, max=681.848, sum=1363.696 (2)\", \"tab\": \"General information\", \"score\": \"681.8482142857143\"}", - "Machine Learning - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"machine_learning\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_machine_learning\"" - } - } - }, - { - "evaluation_name": "Management", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Management", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.913, - "details": { - "description": "min=0.913, mean=0.913, max=0.913, sum=1.825 (2)", - "tab": "Accuracy", - "Management - Observed inference time (s)": "{\"description\": \"min=0.293, mean=0.293, max=0.293, sum=0.587 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2933675108604061\"}", - "Management - # eval": "{\"description\": \"min=103, mean=103, max=103, sum=206 (2)\", \"tab\": \"General information\", \"score\": \"103.0\"}", - "Management - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Management - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Management - # prompt tokens": "{\"description\": \"min=283.854, mean=283.854, max=283.854, sum=567.709 (2)\", \"tab\": \"General information\", \"score\": \"283.8543689320388\"}", - "Management - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"management\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_management\"" - } - } - }, - { - "evaluation_name": "Marketing", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Marketing", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.953, - "details": { - "description": "min=0.953, mean=0.953, max=0.953, sum=1.906 (2)", - "tab": "Accuracy", - "Marketing - Observed inference time (s)": "{\"description\": \"min=0.475, mean=0.475, max=0.475, sum=0.949 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4746182779980521\"}", - "Marketing - # eval": "{\"description\": \"min=234, mean=234, max=234, sum=468 (2)\", \"tab\": \"General information\", \"score\": \"234.0\"}", - "Marketing - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Marketing - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Marketing - # prompt tokens": "{\"description\": \"min=404.415, mean=404.415, max=404.415, sum=808.829 (2)\", \"tab\": \"General information\", \"score\": \"404.4145299145299\"}", - "Marketing - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"marketing\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_marketing\"" - } - } - }, - { - "evaluation_name": "Medical Genetics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Medical Genetics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.92, - "details": { - "description": "min=0.92, mean=0.92, max=0.92, sum=1.84 (2)", - "tab": "Accuracy", - "Medical Genetics - Observed inference time (s)": "{\"description\": \"min=0.311, mean=0.311, max=0.311, sum=0.622 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3110049200057983\"}", - "Medical Genetics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Medical Genetics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Medical Genetics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Medical Genetics - # prompt tokens": "{\"description\": \"min=342.35, mean=342.35, max=342.35, sum=684.7 (2)\", \"tab\": \"General information\", \"score\": \"342.35\"}", - "Medical Genetics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"medical_genetics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_medical_genetics\"" - } - } - }, - { - "evaluation_name": "Miscellaneous", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Miscellaneous", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.932, - "details": { - "description": "min=0.932, mean=0.932, max=0.932, sum=1.865 (2)", - "tab": "Accuracy", - "Miscellaneous - Observed inference time (s)": "{\"description\": \"min=0.345, mean=0.345, max=0.345, sum=0.689 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3445042967035091\"}", - "Miscellaneous - # eval": "{\"description\": \"min=783, mean=783, max=783, sum=1566 (2)\", \"tab\": \"General information\", \"score\": \"783.0\"}", - "Miscellaneous - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Miscellaneous - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Miscellaneous - # prompt tokens": "{\"description\": \"min=303.7, mean=303.7, max=303.7, sum=607.4 (2)\", \"tab\": \"General information\", \"score\": \"303.6998722860792\"}", - "Miscellaneous - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"miscellaneous\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_miscellaneous\"" - } - } - }, - { - "evaluation_name": "Moral Scenarios", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Moral Scenarios", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.787, - "details": { - "description": "min=0.787, mean=0.787, max=0.787, sum=1.573 (2)", - "tab": "Accuracy", - "Moral Disputes - Observed inference time (s)": "{\"description\": \"min=0.291, mean=0.291, max=0.291, sum=0.583 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2913500532249495\"}", - "Moral Scenarios - Observed inference time (s)": "{\"description\": \"min=0.32, mean=0.32, max=0.32, sum=0.641 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.32045427327715487\"}", - "Moral Disputes - # eval": "{\"description\": \"min=346, mean=346, max=346, sum=692 (2)\", \"tab\": \"General information\", \"score\": \"346.0\"}", - "Moral Disputes - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Disputes - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Disputes - # prompt tokens": "{\"description\": \"min=476.182, mean=476.182, max=476.182, sum=952.364 (2)\", \"tab\": \"General information\", \"score\": \"476.1820809248555\"}", - "Moral Disputes - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Moral Scenarios - # eval": "{\"description\": \"min=895, mean=895, max=895, sum=1790 (2)\", \"tab\": \"General information\", \"score\": \"895.0\"}", - "Moral Scenarios - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Scenarios - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # prompt tokens": "{\"description\": \"min=668.494, mean=668.494, max=668.494, sum=1336.988 (2)\", \"tab\": \"General information\", \"score\": \"668.4938547486033\"}", - "Moral Scenarios - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"moral_scenarios\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_moral_scenarios\"" - } - } - }, - { - "evaluation_name": "Nutrition", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Nutrition", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.886, - "details": { - "description": "min=0.886, mean=0.886, max=0.886, sum=1.771 (2)", - "tab": "Accuracy", - "Nutrition - Observed inference time (s)": "{\"description\": \"min=0.293, mean=0.293, max=0.293, sum=0.585 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.29262306565552754\"}", - "Nutrition - # eval": "{\"description\": \"min=306, mean=306, max=306, sum=612 (2)\", \"tab\": \"General information\", \"score\": \"306.0\"}", - "Nutrition - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Nutrition - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Nutrition - # prompt tokens": "{\"description\": \"min=599.637, mean=599.637, max=599.637, sum=1199.275 (2)\", \"tab\": \"General information\", \"score\": \"599.6372549019608\"}", - "Nutrition - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"nutrition\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_nutrition\"" - } - } - }, - { - "evaluation_name": "Prehistory", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Prehistory", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.91, - "details": { - "description": "min=0.91, mean=0.91, max=0.91, sum=1.821 (2)", - "tab": "Accuracy", - "Prehistory - Observed inference time (s)": "{\"description\": \"min=0.34, mean=0.34, max=0.34, sum=0.681 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.340311410986347\"}", - "Prehistory - # eval": "{\"description\": \"min=324, mean=324, max=324, sum=648 (2)\", \"tab\": \"General information\", \"score\": \"324.0\"}", - "Prehistory - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Prehistory - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Prehistory - # prompt tokens": "{\"description\": \"min=528.364, mean=528.364, max=528.364, sum=1056.728 (2)\", \"tab\": \"General information\", \"score\": \"528.3641975308642\"}", - "Prehistory - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"prehistory\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_prehistory\"" - } - } - }, - { - "evaluation_name": "Public Relations", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Public Relations", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.782, - "details": { - "description": "min=0.782, mean=0.782, max=0.782, sum=1.564 (2)", - "tab": "Accuracy", - "Public Relations - Observed inference time (s)": "{\"description\": \"min=0.277, mean=0.277, max=0.277, sum=0.554 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2769838809967041\"}", - "Public Relations - # eval": "{\"description\": \"min=110, mean=110, max=110, sum=220 (2)\", \"tab\": \"General information\", \"score\": \"110.0\"}", - "Public Relations - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Public Relations - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Public Relations - # prompt tokens": "{\"description\": \"min=408.427, mean=408.427, max=408.427, sum=816.855 (2)\", \"tab\": \"General information\", \"score\": \"408.42727272727274\"}", - "Public Relations - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"public_relations\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_public_relations\"" - } - } - }, - { - "evaluation_name": "Security Studies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Security Studies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.849, - "details": { - "description": "min=0.849, mean=0.849, max=0.849, sum=1.698 (2)", - "tab": "Accuracy", - "Security Studies - Observed inference time (s)": "{\"description\": \"min=0.377, mean=0.377, max=0.377, sum=0.754 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3771621781952527\"}", - "Security Studies - # eval": "{\"description\": \"min=245, mean=245, max=245, sum=490 (2)\", \"tab\": \"General information\", \"score\": \"245.0\"}", - "Security Studies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Security Studies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Security Studies - # prompt tokens": "{\"description\": \"min=1166.931, mean=1166.931, max=1166.931, sum=2333.861 (2)\", \"tab\": \"General information\", \"score\": \"1166.930612244898\"}", - "Security Studies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"security_studies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_security_studies\"" - } - } - }, - { - "evaluation_name": "Sociology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Sociology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.925, - "details": { - "description": "min=0.925, mean=0.925, max=0.925, sum=1.851 (2)", - "tab": "Accuracy", - "Sociology - Observed inference time (s)": "{\"description\": \"min=0.291, mean=0.291, max=0.291, sum=0.582 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2910151019025205\"}", - "Sociology - # eval": "{\"description\": \"min=201, mean=201, max=201, sum=402 (2)\", \"tab\": \"General information\", \"score\": \"201.0\"}", - "Sociology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Sociology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Sociology - # prompt tokens": "{\"description\": \"min=450.1, mean=450.1, max=450.1, sum=900.199 (2)\", \"tab\": \"General information\", \"score\": \"450.0995024875622\"}", - "Sociology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"sociology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_sociology\"" - } - } - }, - { - "evaluation_name": "Virology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Virology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.584, - "details": { - "description": "min=0.584, mean=0.584, max=0.584, sum=1.169 (2)", - "tab": "Accuracy", - "Virology - Observed inference time (s)": "{\"description\": \"min=0.351, mean=0.351, max=0.351, sum=0.702 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.35115946631833733\"}", - "Virology - # eval": "{\"description\": \"min=166, mean=166, max=166, sum=332 (2)\", \"tab\": \"General information\", \"score\": \"166.0\"}", - "Virology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Virology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Virology - # prompt tokens": "{\"description\": \"min=343.819, mean=343.819, max=343.819, sum=687.639 (2)\", \"tab\": \"General information\", \"score\": \"343.8192771084337\"}", - "Virology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"virology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_virology\"" - } - } - }, - { - "evaluation_name": "World Religions", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on World Religions", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.901, - "details": { - "description": "min=0.901, mean=0.901, max=0.901, sum=1.801 (2)", - "tab": "Accuracy", - "World Religions - Observed inference time (s)": "{\"description\": \"min=0.381, mean=0.381, max=0.381, sum=0.762 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3812444461019416\"}", - "World Religions - # eval": "{\"description\": \"min=171, mean=171, max=171, sum=342 (2)\", \"tab\": \"General information\", \"score\": \"171.0\"}", - "World Religions - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "World Religions - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "World Religions - # prompt tokens": "{\"description\": \"min=276.07, mean=276.07, max=276.07, sum=552.14 (2)\", \"tab\": \"General information\", \"score\": \"276.0701754385965\"}", - "World Religions - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"world_religions\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_world_religions\"" - } - } - }, - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.548, - "details": { - "description": "", - "tab": "Efficiency" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/qwen_qwen2.5-72b-instruct.json b/data/models/qwen_qwen2.5-72b-instruct.json deleted file mode 100644 index 23d7d04e9705358b71f08b9d35a058c0fc371e8f..0000000000000000000000000000000000000000 --- a/data/models/qwen_qwen2.5-72b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-72B-Instruct", - "id": "Qwen/Qwen2.5-72B-Instruct", - "developer": "Qwen", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "72.706" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Qwen_Qwen2.5-72B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8638 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7273 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5982 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.375 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4206 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5626 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qwen_qwen2.5-72b.json b/data/models/qwen_qwen2.5-72b.json deleted file mode 100644 index 6fd1d63acbc8e2d8dac91aebf9697662545b4059..0000000000000000000000000000000000000000 --- a/data/models/qwen_qwen2.5-72b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-72B", - "id": "Qwen/Qwen2.5-72B", - "developer": "Qwen", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "72.706" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Qwen_Qwen2.5-72B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4137 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6797 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3912 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4052 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4771 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5968 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qwen_qwen2.5-7b-instruct-1m.json b/data/models/qwen_qwen2.5-7b-instruct-1m.json deleted file mode 100644 index 7ea4d22d91251b6a2371d55f4fec83a1abb0abda..0000000000000000000000000000000000000000 --- a/data/models/qwen_qwen2.5-7b-instruct-1m.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-7B-Instruct-1M", - "id": "Qwen/Qwen2.5-7B-Instruct-1M", - "developer": "Qwen", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Qwen_Qwen2.5-7B-Instruct-1M/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7448 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5404 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4335 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2978 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4087 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3505 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qwen_qwen2.5-7b-instruct-turbo.json b/data/models/qwen_qwen2.5-7b-instruct-turbo.json deleted file mode 100644 index 0df68f8b1872e465bb5c4a8dafc1fd919ae91e5a..0000000000000000000000000000000000000000 --- a/data/models/qwen_qwen2.5-7b-instruct-turbo.json +++ /dev/null @@ -1,2128 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5 Instruct Turbo 7B", - "id": "qwen/qwen2.5-7b-instruct-turbo", - "developer": "qwen", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_capabilities/qwen_qwen2.5-7b-instruct-turbo/1774096304.5056698", - "retrieved_timestamp": "1774096304.5056698", - "source_metadata": { - "source_name": "helm_capabilities", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_capabilities", - "evaluation_results": [ - { - "evaluation_name": "Mean score", - "source_data": { - "dataset_name": "helm_capabilities", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "The mean of the scores from all columns.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.529, - "details": { - "description": "", - "tab": "Accuracy", - "Mean score - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"4.913331052029195\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU-Pro", - "source_data": { - "dataset_name": "MMLU-Pro", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on MMLU-Pro", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.539, - "details": { - "description": "min=0.539, mean=0.539, max=0.539, sum=0.539 (1)", - "tab": "Accuracy", - "MMLU-Pro - Observed inference time (s)": "{\"description\": \"min=6.223, mean=6.223, max=6.223, sum=6.223 (1)\", \"tab\": \"Efficiency\", \"score\": \"6.223100474119186\"}", - "MMLU-Pro - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "MMLU-Pro - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - # prompt tokens": "{\"description\": \"min=238.715, mean=238.715, max=238.715, sum=238.715 (1)\", \"tab\": \"General information\", \"score\": \"238.715\"}", - "MMLU-Pro - # output tokens": "{\"description\": \"min=439.207, mean=439.207, max=439.207, sum=439.207 (1)\", \"tab\": \"General information\", \"score\": \"439.207\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"all\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.341, - "details": { - "description": "min=0.341, mean=0.341, max=0.341, sum=0.341 (1)", - "tab": "Accuracy", - "GPQA - Observed inference time (s)": "{\"description\": \"min=3.475, mean=3.475, max=3.475, sum=3.475 (1)\", \"tab\": \"Efficiency\", \"score\": \"3.4745728910771185\"}", - "GPQA - # eval": "{\"description\": \"min=446, mean=446, max=446, sum=446 (1)\", \"tab\": \"General information\", \"score\": \"446.0\"}", - "GPQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - # prompt tokens": "{\"description\": \"min=253.37, mean=253.37, max=253.37, sum=253.37 (1)\", \"tab\": \"General information\", \"score\": \"253.36995515695068\"}", - "GPQA - # output tokens": "{\"description\": \"min=554.274, mean=554.274, max=554.274, sum=554.274 (1)\", \"tab\": \"General information\", \"score\": \"554.2735426008969\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"gpqa_main\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "IFEval Strict Acc on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.741, - "details": { - "description": "min=0.741, mean=0.741, max=0.741, sum=0.741 (1)", - "tab": "Accuracy", - "IFEval - Observed inference time (s)": "{\"description\": \"min=2.068, mean=2.068, max=2.068, sum=2.068 (1)\", \"tab\": \"Efficiency\", \"score\": \"2.0679604544436865\"}", - "IFEval - # eval": "{\"description\": \"min=541, mean=541, max=541, sum=541 (1)\", \"tab\": \"General information\", \"score\": \"541.0\"}", - "IFEval - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - # prompt tokens": "{\"description\": \"min=46.492, mean=46.492, max=46.492, sum=46.492 (1)\", \"tab\": \"General information\", \"score\": \"46.491682070240294\"}", - "IFEval - # output tokens": "{\"description\": \"min=317.828, mean=317.828, max=317.828, sum=317.828 (1)\", \"tab\": \"General information\", \"score\": \"317.82809611829947\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WildBench", - "source_data": { - "dataset_name": "WildBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "WB Score on WildBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.731, - "details": { - "description": "min=0.731, mean=0.731, max=0.731, sum=0.731 (1)", - "tab": "Accuracy", - "WildBench - Observed inference time (s)": "{\"description\": \"min=7.74, mean=7.74, max=7.74, sum=7.74 (1)\", \"tab\": \"Efficiency\", \"score\": \"7.7404146847724915\"}", - "WildBench - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "WildBench - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # prompt tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # output tokens": "{\"description\": \"min=953.359, mean=953.359, max=953.359, sum=953.359 (1)\", \"tab\": \"General information\", \"score\": \"953.359\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"v2\"" - } - } - }, - { - "evaluation_name": "Omni-MATH", - "source_data": { - "dataset_name": "Omni-MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Acc on Omni-MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.294, - "details": { - "description": "min=0.294, mean=0.294, max=0.294, sum=0.294 (1)", - "tab": "Accuracy", - "Omni-MATH - Observed inference time (s)": "{\"description\": \"min=5.061, mean=5.061, max=5.061, sum=5.061 (1)\", \"tab\": \"Efficiency\", \"score\": \"5.06060675573349\"}", - "Omni-MATH - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "Omni-MATH - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - # prompt tokens": "{\"description\": \"min=111.6, mean=111.6, max=111.6, sum=111.6 (1)\", \"tab\": \"General information\", \"score\": \"111.6\"}", - "Omni-MATH - # output tokens": "{\"description\": \"min=809.198, mean=809.198, max=809.198, sum=809.198 (1)\", \"tab\": \"General information\", \"score\": \"809.198\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_lite/qwen_qwen2.5-7b-instruct-turbo/1774096306.427425", - "retrieved_timestamp": "1774096306.427425", - "source_metadata": { - "source_name": "helm_lite", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_lite", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_lite", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.488, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.8808988764044944\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.742, - "details": { - "description": "min=0.742, mean=0.742, max=0.742, sum=0.742 (1)", - "tab": "Accuracy", - "NarrativeQA - Observed inference time (s)": "{\"description\": \"min=0.516, mean=0.516, max=0.516, sum=0.516 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.5156192410160119\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=3492.913, mean=3492.913, max=3492.913, sum=3492.913 (1)\", \"tab\": \"General information\", \"score\": \"3492.912676056338\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=5.549, mean=5.549, max=5.549, sum=5.549 (1)\", \"tab\": \"General information\", \"score\": \"5.549295774647887\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (closed-book)", - "source_data": { - "dataset_name": "NaturalQuestions (closed-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (closed-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.205, - "details": { - "description": "min=0.205, mean=0.205, max=0.205, sum=0.205 (1)", - "tab": "Accuracy", - "NaturalQuestions (open-book) - Observed inference time (s)": "{\"description\": \"min=0.301, mean=0.301, max=0.301, sum=0.301 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.30121764993667605\"}", - "NaturalQuestions (closed-book) - Observed inference time (s)": "{\"description\": \"min=0.217, mean=0.217, max=0.217, sum=0.217 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.21686342740058898\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=2007.955, mean=2007.955, max=2007.955, sum=2007.955 (1)\", \"tab\": \"General information\", \"score\": \"2007.955\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=8.698, mean=8.698, max=8.698, sum=8.698 (1)\", \"tab\": \"General information\", \"score\": \"8.698\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=136.262, mean=136.262, max=136.262, sum=136.262 (1)\", \"tab\": \"General information\", \"score\": \"136.262\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=7.041, mean=7.041, max=7.041, sum=7.041 (1)\", \"tab\": \"General information\", \"score\": \"7.041\"}" - } - }, - "generation_config": { - "additional_details": { - "mode": "\"closedbook\"" - } - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.862, - "details": { - "description": "min=0.862, mean=0.862, max=0.862, sum=0.862 (1)", - "tab": "Accuracy", - "OpenbookQA - Observed inference time (s)": "{\"description\": \"min=0.186, mean=0.186, max=0.186, sum=0.186 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.1863201789855957\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=249.846, mean=249.846, max=249.846, sum=249.846 (1)\", \"tab\": \"General information\", \"score\": \"249.846\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "dataset": "\"openbookqa\"", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.658, - "details": { - "description": "min=0.49, mean=0.658, max=0.86, sum=3.29 (5)", - "tab": "Accuracy", - "MMLU - Observed inference time (s)": "{\"description\": \"min=0.285, mean=0.35, max=0.431, sum=1.751 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.35013260537699653\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=378.19, mean=477.836, max=627.939, sum=2389.179 (5)\", \"tab\": \"General information\", \"score\": \"477.8357192982456\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"college_chemistry\", \"computer_security\", \"econometrics\", \"us_foreign_policy\"]", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MATH", - "source_data": { - "dataset_name": "MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Equivalent (CoT) on MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.835, - "details": { - "description": "min=0.684, mean=0.835, max=0.963, sum=5.846 (7)", - "tab": "Accuracy", - "MATH - Observed inference time (s)": "{\"description\": \"min=1.449, mean=1.825, max=2.345, sum=12.778 (7)\", \"tab\": \"Efficiency\", \"score\": \"1.8253796190803115\"}", - "MATH - # eval": "{\"description\": \"min=30, mean=62.429, max=135, sum=437 (7)\", \"tab\": \"General information\", \"score\": \"62.42857142857143\"}", - "MATH - # train": "{\"description\": \"min=8, mean=8, max=8, sum=56 (7)\", \"tab\": \"General information\", \"score\": \"8.0\"}", - "MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MATH - # prompt tokens": "{\"description\": \"min=937.926, mean=1323.837, max=2246.673, sum=9266.858 (7)\", \"tab\": \"General information\", \"score\": \"1323.836848955025\"}", - "MATH - # output tokens": "{\"description\": \"min=156.674, mean=196.898, max=240.288, sum=1378.285 (7)\", \"tab\": \"General information\", \"score\": \"196.8978610559394\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"algebra\", \"counting_and_probability\", \"geometry\", \"intermediate_algebra\", \"number_theory\", \"prealgebra\", \"precalculus\"]", - "level": "\"1\"", - "use_official_examples": "\"False\"", - "use_chain_of_thought": "\"True\"" - } - } - }, - { - "evaluation_name": "GSM8K", - "source_data": { - "dataset_name": "GSM8K", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on GSM8K", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.83, - "details": { - "description": "min=0.83, mean=0.83, max=0.83, sum=0.83 (1)", - "tab": "Accuracy", - "GSM8K - Observed inference time (s)": "{\"description\": \"min=1.7, mean=1.7, max=1.7, sum=1.7 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.7000067098140716\"}", - "GSM8K - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "GSM8K - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "GSM8K - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GSM8K - # prompt tokens": "{\"description\": \"min=1130.403, mean=1130.403, max=1130.403, sum=1130.403 (1)\", \"tab\": \"General information\", \"score\": \"1130.403\"}", - "GSM8K - # output tokens": "{\"description\": \"min=194.776, mean=194.776, max=194.776, sum=194.776 (1)\", \"tab\": \"General information\", \"score\": \"194.776\"}" - } - }, - "generation_config": { - "additional_details": { - "stop": "\"none\"" - } - } - }, - { - "evaluation_name": "LegalBench", - "source_data": { - "dataset_name": "LegalBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on LegalBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.632, - "details": { - "description": "min=0.414, mean=0.632, max=0.916, sum=3.161 (5)", - "tab": "Accuracy", - "LegalBench - Observed inference time (s)": "{\"description\": \"min=0.183, mean=0.261, max=0.489, sum=1.305 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.2609495958632719\"}", - "LegalBench - # eval": "{\"description\": \"min=95, mean=409.4, max=1000, sum=2047 (5)\", \"tab\": \"General information\", \"score\": \"409.4\"}", - "LegalBench - # train": "{\"description\": \"min=4, mean=4.8, max=5, sum=24 (5)\", \"tab\": \"General information\", \"score\": \"4.8\"}", - "LegalBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "LegalBench - # prompt tokens": "{\"description\": \"min=216.453, mean=1558.888, max=6440.714, sum=7794.442 (5)\", \"tab\": \"General information\", \"score\": \"1558.8883229968653\"}", - "LegalBench - # output tokens": "{\"description\": \"min=2, mean=2.402, max=3.084, sum=12.008 (5)\", \"tab\": \"General information\", \"score\": \"2.4015832496773273\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"abercrombie\", \"corporate_lobbying\", \"function_of_decision_section\", \"international_citizenship_questions\", \"proa\"]", - "stop": "\"none\"" - } - } - }, - { - "evaluation_name": "MedQA", - "source_data": { - "dataset_name": "MedQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MedQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6, - "details": { - "description": "min=0.6, mean=0.6, max=0.6, sum=0.6 (1)", - "tab": "Accuracy", - "MedQA - Observed inference time (s)": "{\"description\": \"min=0.201, mean=0.201, max=0.201, sum=0.201 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.20058301760709546\"}", - "MedQA - # eval": "{\"description\": \"min=503, mean=503, max=503, sum=503 (1)\", \"tab\": \"General information\", \"score\": \"503.0\"}", - "MedQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MedQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MedQA - # prompt tokens": "{\"description\": \"min=1052.485, mean=1052.485, max=1052.485, sum=1052.485 (1)\", \"tab\": \"General information\", \"score\": \"1052.4850894632207\"}", - "MedQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WMT 2014", - "source_data": { - "dataset_name": "WMT 2014", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "BLEU-4 on WMT 2014", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.155, - "details": { - "description": "min=0.085, mean=0.155, max=0.204, sum=0.777 (5)", - "tab": "Accuracy", - "WMT 2014 - Observed inference time (s)": "{\"description\": \"min=0.346, mean=0.376, max=0.414, sum=1.88 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.3759268445955365\"}", - "WMT 2014 - # eval": "{\"description\": \"min=503, mean=568.8, max=832, sum=2844 (5)\", \"tab\": \"General information\", \"score\": \"568.8\"}", - "WMT 2014 - # train": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "WMT 2014 - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WMT 2014 - # prompt tokens": "{\"description\": \"min=114.855, mean=132.657, max=148.373, sum=663.283 (5)\", \"tab\": \"General information\", \"score\": \"132.65662658663405\"}", - "WMT 2014 - # output tokens": "{\"description\": \"min=26.946, mean=27.742, max=28.649, sum=138.709 (5)\", \"tab\": \"General information\", \"score\": \"27.74173612173115\"}" - } - }, - "generation_config": { - "additional_details": { - "language_pair": "[\"cs-en\", \"de-en\", \"fr-en\", \"hi-en\", \"ru-en\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_mmlu/qwen_qwen2.5-7b-instruct-turbo/1774096312.00548", - "retrieved_timestamp": "1774096312.00548", - "source_metadata": { - "source_name": "helm_mmlu", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_mmlu", - "evaluation_results": [ - { - "evaluation_name": "MMLU All Subjects", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU All Subjects", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.729, - "details": { - "description": "min=0.42, mean=0.729, max=0.919, sum=83.073 (114)", - "tab": "Accuracy", - "MMLU All Subjects - Observed inference time (s)": "{\"description\": \"min=0.162, mean=0.242, max=0.44, sum=27.616 (114)\", \"tab\": \"Efficiency\", \"score\": \"0.24224721190343979\"}", - "MMLU All Subjects - # eval": "{\"description\": \"min=100, mean=246.351, max=1534, sum=28084 (114)\", \"tab\": \"General information\", \"score\": \"246.35087719298247\"}", - "MMLU All Subjects - # train": "{\"description\": \"min=5, mean=5, max=5, sum=570 (114)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU All Subjects - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU All Subjects - # prompt tokens": "{\"description\": \"min=276.07, mean=625.598, max=2814.903, sum=71318.198 (114)\", \"tab\": \"General information\", \"score\": \"625.5982315160392\"}", - "MMLU All Subjects - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=114 (114)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_name": "Abstract Algebra", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Abstract Algebra", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.49, - "details": { - "description": "min=0.49, mean=0.49, max=0.49, sum=0.98 (2)", - "tab": "Accuracy", - "Abstract Algebra - Observed inference time (s)": "{\"description\": \"min=0.431, mean=0.431, max=0.431, sum=0.863 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.43148461580276487\"}", - "Abstract Algebra - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Abstract Algebra - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Abstract Algebra - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Abstract Algebra - # prompt tokens": "{\"description\": \"min=378.19, mean=378.19, max=378.19, sum=756.38 (2)\", \"tab\": \"General information\", \"score\": \"378.19\"}", - "Abstract Algebra - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"abstract_algebra\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_abstract_algebra\"" - } - } - }, - { - "evaluation_name": "Anatomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Anatomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.689, - "details": { - "description": "min=0.689, mean=0.689, max=0.689, sum=1.378 (2)", - "tab": "Accuracy", - "Anatomy - Observed inference time (s)": "{\"description\": \"min=0.333, mean=0.333, max=0.333, sum=0.667 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3332981339207402\"}", - "Anatomy - # eval": "{\"description\": \"min=135, mean=135, max=135, sum=270 (2)\", \"tab\": \"General information\", \"score\": \"135.0\"}", - "Anatomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Anatomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Anatomy - # prompt tokens": "{\"description\": \"min=353.978, mean=353.978, max=353.978, sum=707.956 (2)\", \"tab\": \"General information\", \"score\": \"353.97777777777776\"}", - "Anatomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"anatomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_anatomy\"" - } - } - }, - { - "evaluation_name": "College Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on College Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.51, - "details": { - "description": "min=0.51, mean=0.51, max=0.51, sum=1.02 (2)", - "tab": "Accuracy", - "College Chemistry - Observed inference time (s)": "{\"description\": \"min=0.285, mean=0.285, max=0.285, sum=0.571 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.28538883924484254\"}", - "College Biology - Observed inference time (s)": "{\"description\": \"min=0.31, mean=0.31, max=0.31, sum=0.619 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.309537861082289\"}", - "College Computer Science - Observed inference time (s)": "{\"description\": \"min=0.302, mean=0.302, max=0.302, sum=0.604 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.30183048248291017\"}", - "College Mathematics - Observed inference time (s)": "{\"description\": \"min=0.279, mean=0.279, max=0.279, sum=0.558 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2791933488845825\"}", - "College Medicine - Observed inference time (s)": "{\"description\": \"min=0.303, mean=0.303, max=0.303, sum=0.607 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3032711007002461\"}", - "College Physics - Observed inference time (s)": "{\"description\": \"min=0.3, mean=0.3, max=0.3, sum=0.599 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2996697425842285\"}", - "College Chemistry - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Chemistry - # prompt tokens": "{\"description\": \"min=568.25, mean=568.25, max=568.25, sum=1136.5 (2)\", \"tab\": \"General information\", \"score\": \"568.25\"}", - "College Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Biology - # eval": "{\"description\": \"min=144, mean=144, max=144, sum=288 (2)\", \"tab\": \"General information\", \"score\": \"144.0\"}", - "College Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # prompt tokens": "{\"description\": \"min=486.979, mean=486.979, max=486.979, sum=973.958 (2)\", \"tab\": \"General information\", \"score\": \"486.9791666666667\"}", - "College Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # prompt tokens": "{\"description\": \"min=838.58, mean=838.58, max=838.58, sum=1677.16 (2)\", \"tab\": \"General information\", \"score\": \"838.58\"}", - "College Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Mathematics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # prompt tokens": "{\"description\": \"min=607.7, mean=607.7, max=607.7, sum=1215.4 (2)\", \"tab\": \"General information\", \"score\": \"607.7\"}", - "College Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Medicine - # eval": "{\"description\": \"min=173, mean=173, max=173, sum=346 (2)\", \"tab\": \"General information\", \"score\": \"173.0\"}", - "College Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # prompt tokens": "{\"description\": \"min=506.098, mean=506.098, max=506.098, sum=1012.197 (2)\", \"tab\": \"General information\", \"score\": \"506.0982658959538\"}", - "College Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Physics - # eval": "{\"description\": \"min=102, mean=102, max=102, sum=204 (2)\", \"tab\": \"General information\", \"score\": \"102.0\"}", - "College Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # prompt tokens": "{\"description\": \"min=516.265, mean=516.265, max=516.265, sum=1032.529 (2)\", \"tab\": \"General information\", \"score\": \"516.2647058823529\"}", - "College Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"college_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_college_physics\"" - } - } - }, - { - "evaluation_name": "Computer Security", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Computer Security", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.79, - "details": { - "description": "min=0.79, mean=0.79, max=0.79, sum=1.58 (2)", - "tab": "Accuracy", - "Computer Security - Observed inference time (s)": "{\"description\": \"min=0.352, mean=0.352, max=0.352, sum=0.705 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3522661328315735\"}", - "Computer Security - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Computer Security - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Computer Security - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Computer Security - # prompt tokens": "{\"description\": \"min=386.64, mean=386.64, max=386.64, sum=773.28 (2)\", \"tab\": \"General information\", \"score\": \"386.64\"}", - "Computer Security - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"computer_security\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_computer_security\"" - } - } - }, - { - "evaluation_name": "Econometrics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Econometrics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.64, - "details": { - "description": "min=0.64, mean=0.64, max=0.64, sum=1.281 (2)", - "tab": "Accuracy", - "Econometrics - Observed inference time (s)": "{\"description\": \"min=0.346, mean=0.346, max=0.346, sum=0.691 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.34558368356604324\"}", - "Econometrics - # eval": "{\"description\": \"min=114, mean=114, max=114, sum=228 (2)\", \"tab\": \"General information\", \"score\": \"114.0\"}", - "Econometrics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Econometrics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Econometrics - # prompt tokens": "{\"description\": \"min=627.939, mean=627.939, max=627.939, sum=1255.877 (2)\", \"tab\": \"General information\", \"score\": \"627.938596491228\"}", - "Econometrics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"econometrics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_econometrics\"" - } - } - }, - { - "evaluation_name": "Global Facts", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Global Facts", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.42, - "details": { - "description": "min=0.42, mean=0.42, max=0.42, sum=0.84 (2)", - "tab": "Accuracy", - "Global Facts - Observed inference time (s)": "{\"description\": \"min=0.315, mean=0.315, max=0.315, sum=0.63 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.314766480922699\"}", - "Global Facts - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Global Facts - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Global Facts - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Global Facts - # prompt tokens": "{\"description\": \"min=429.06, mean=429.06, max=429.06, sum=858.12 (2)\", \"tab\": \"General information\", \"score\": \"429.06\"}", - "Global Facts - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"global_facts\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_global_facts\"" - } - } - }, - { - "evaluation_name": "Jurisprudence", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Jurisprudence", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.796, - "details": { - "description": "min=0.796, mean=0.796, max=0.796, sum=1.593 (2)", - "tab": "Accuracy", - "Jurisprudence - Observed inference time (s)": "{\"description\": \"min=0.321, mean=0.321, max=0.321, sum=0.642 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.32116924391852486\"}", - "Jurisprudence - # eval": "{\"description\": \"min=108, mean=108, max=108, sum=216 (2)\", \"tab\": \"General information\", \"score\": \"108.0\"}", - "Jurisprudence - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Jurisprudence - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Jurisprudence - # prompt tokens": "{\"description\": \"min=394.713, mean=394.713, max=394.713, sum=789.426 (2)\", \"tab\": \"General information\", \"score\": \"394.712962962963\"}", - "Jurisprudence - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"jurisprudence\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_jurisprudence\"" - } - } - }, - { - "evaluation_name": "Philosophy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Philosophy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.746, - "details": { - "description": "min=0.746, mean=0.746, max=0.746, sum=1.492 (2)", - "tab": "Accuracy", - "Philosophy - Observed inference time (s)": "{\"description\": \"min=0.44, mean=0.44, max=0.44, sum=0.88 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4401504610129108\"}", - "Philosophy - # eval": "{\"description\": \"min=311, mean=311, max=311, sum=622 (2)\", \"tab\": \"General information\", \"score\": \"311.0\"}", - "Philosophy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Philosophy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Philosophy - # prompt tokens": "{\"description\": \"min=329.09, mean=329.09, max=329.09, sum=658.18 (2)\", \"tab\": \"General information\", \"score\": \"329.09003215434086\"}", - "Philosophy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"philosophy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_philosophy\"" - } - } - }, - { - "evaluation_name": "Professional Psychology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Professional Psychology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.757, - "details": { - "description": "min=0.757, mean=0.757, max=0.757, sum=1.513 (2)", - "tab": "Accuracy", - "Professional Medicine - Observed inference time (s)": "{\"description\": \"min=0.394, mean=0.394, max=0.394, sum=0.788 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.393971232806935\"}", - "Professional Accounting - Observed inference time (s)": "{\"description\": \"min=0.185, mean=0.185, max=0.185, sum=0.371 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.18525678553479782\"}", - "Professional Law - Observed inference time (s)": "{\"description\": \"min=0.205, mean=0.205, max=0.205, sum=0.409 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.20459390463698485\"}", - "Professional Psychology - Observed inference time (s)": "{\"description\": \"min=0.166, mean=0.166, max=0.166, sum=0.332 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.16597708611706502\"}", - "Professional Medicine - # eval": "{\"description\": \"min=272, mean=272, max=272, sum=544 (2)\", \"tab\": \"General information\", \"score\": \"272.0\"}", - "Professional Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Medicine - # prompt tokens": "{\"description\": \"min=1125.199, mean=1125.199, max=1125.199, sum=2250.397 (2)\", \"tab\": \"General information\", \"score\": \"1125.1985294117646\"}", - "Professional Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Accounting - # eval": "{\"description\": \"min=282, mean=282, max=282, sum=564 (2)\", \"tab\": \"General information\", \"score\": \"282.0\"}", - "Professional Accounting - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Accounting - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # prompt tokens": "{\"description\": \"min=739.34, mean=739.34, max=739.34, sum=1478.681 (2)\", \"tab\": \"General information\", \"score\": \"739.3404255319149\"}", - "Professional Accounting - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Law - # eval": "{\"description\": \"min=1534, mean=1534, max=1534, sum=3068 (2)\", \"tab\": \"General information\", \"score\": \"1534.0\"}", - "Professional Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # prompt tokens": "{\"description\": \"min=1663.969, mean=1663.969, max=1663.969, sum=3327.939 (2)\", \"tab\": \"General information\", \"score\": \"1663.9693611473272\"}", - "Professional Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Psychology - # eval": "{\"description\": \"min=612, mean=612, max=612, sum=1224 (2)\", \"tab\": \"General information\", \"score\": \"612.0\"}", - "Professional Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # prompt tokens": "{\"description\": \"min=581.417, mean=581.417, max=581.417, sum=1162.833 (2)\", \"tab\": \"General information\", \"score\": \"581.4166666666666\"}", - "Professional Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"professional_psychology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_professional_psychology\"" - } - } - }, - { - "evaluation_name": "Us Foreign Policy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Us Foreign Policy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.86, - "details": { - "description": "min=0.86, mean=0.86, max=0.86, sum=1.72 (2)", - "tab": "Accuracy", - "Us Foreign Policy - Observed inference time (s)": "{\"description\": \"min=0.33, mean=0.33, max=0.33, sum=0.66 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.33019849777221677\"}", - "Us Foreign Policy - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Us Foreign Policy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Us Foreign Policy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Us Foreign Policy - # prompt tokens": "{\"description\": \"min=428.16, mean=428.16, max=428.16, sum=856.32 (2)\", \"tab\": \"General information\", \"score\": \"428.16\"}", - "Us Foreign Policy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"us_foreign_policy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_us_foreign_policy\"" - } - } - }, - { - "evaluation_name": "Astronomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Astronomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.836, - "details": { - "description": "min=0.836, mean=0.836, max=0.836, sum=1.671 (2)", - "tab": "Accuracy", - "Astronomy - Observed inference time (s)": "{\"description\": \"min=0.314, mean=0.314, max=0.314, sum=0.629 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3143457660549565\"}", - "Astronomy - # eval": "{\"description\": \"min=152, mean=152, max=152, sum=304 (2)\", \"tab\": \"General information\", \"score\": \"152.0\"}", - "Astronomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Astronomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Astronomy - # prompt tokens": "{\"description\": \"min=589.849, mean=589.849, max=589.849, sum=1179.697 (2)\", \"tab\": \"General information\", \"score\": \"589.8486842105264\"}", - "Astronomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"astronomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_astronomy\"" - } - } - }, - { - "evaluation_name": "Business Ethics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Business Ethics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.82, - "details": { - "description": "min=0.82, mean=0.82, max=0.82, sum=1.64 (2)", - "tab": "Accuracy", - "Business Ethics - Observed inference time (s)": "{\"description\": \"min=0.308, mean=0.308, max=0.308, sum=0.615 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3076848840713501\"}", - "Business Ethics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Business Ethics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Business Ethics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Business Ethics - # prompt tokens": "{\"description\": \"min=569.87, mean=569.87, max=569.87, sum=1139.74 (2)\", \"tab\": \"General information\", \"score\": \"569.87\"}", - "Business Ethics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"business_ethics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_business_ethics\"" - } - } - }, - { - "evaluation_name": "Clinical Knowledge", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Clinical Knowledge", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.785, - "details": { - "description": "min=0.785, mean=0.785, max=0.785, sum=1.57 (2)", - "tab": "Accuracy", - "Clinical Knowledge - Observed inference time (s)": "{\"description\": \"min=0.335, mean=0.335, max=0.335, sum=0.67 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.33518469288664043\"}", - "Clinical Knowledge - # eval": "{\"description\": \"min=265, mean=265, max=265, sum=530 (2)\", \"tab\": \"General information\", \"score\": \"265.0\"}", - "Clinical Knowledge - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Clinical Knowledge - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Clinical Knowledge - # prompt tokens": "{\"description\": \"min=400.623, mean=400.623, max=400.623, sum=801.245 (2)\", \"tab\": \"General information\", \"score\": \"400.62264150943395\"}", - "Clinical Knowledge - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"clinical_knowledge\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_clinical_knowledge\"" - } - } - }, - { - "evaluation_name": "Conceptual Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Conceptual Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.736, - "details": { - "description": "min=0.736, mean=0.736, max=0.736, sum=1.472 (2)", - "tab": "Accuracy", - "Conceptual Physics - Observed inference time (s)": "{\"description\": \"min=0.253, mean=0.253, max=0.253, sum=0.506 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2531234142628122\"}", - "Conceptual Physics - # eval": "{\"description\": \"min=235, mean=235, max=235, sum=470 (2)\", \"tab\": \"General information\", \"score\": \"235.0\"}", - "Conceptual Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Conceptual Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Conceptual Physics - # prompt tokens": "{\"description\": \"min=305.494, mean=305.494, max=305.494, sum=610.987 (2)\", \"tab\": \"General information\", \"score\": \"305.4936170212766\"}", - "Conceptual Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"conceptual_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_conceptual_physics\"" - } - } - }, - { - "evaluation_name": "Electrical Engineering", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Electrical Engineering", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.717, - "details": { - "description": "min=0.717, mean=0.717, max=0.717, sum=1.434 (2)", - "tab": "Accuracy", - "Electrical Engineering - Observed inference time (s)": "{\"description\": \"min=0.198, mean=0.198, max=0.198, sum=0.396 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.19794883070320918\"}", - "Electrical Engineering - # eval": "{\"description\": \"min=145, mean=145, max=145, sum=290 (2)\", \"tab\": \"General information\", \"score\": \"145.0\"}", - "Electrical Engineering - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Electrical Engineering - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Electrical Engineering - # prompt tokens": "{\"description\": \"min=463.8, mean=463.8, max=463.8, sum=927.6 (2)\", \"tab\": \"General information\", \"score\": \"463.8\"}", - "Electrical Engineering - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"electrical_engineering\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_electrical_engineering\"" - } - } - }, - { - "evaluation_name": "Elementary Mathematics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Elementary Mathematics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.643, - "details": { - "description": "min=0.643, mean=0.643, max=0.643, sum=1.286 (2)", - "tab": "Accuracy", - "Elementary Mathematics - Observed inference time (s)": "{\"description\": \"min=0.202, mean=0.202, max=0.202, sum=0.404 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2021035529949047\"}", - "Elementary Mathematics - # eval": "{\"description\": \"min=378, mean=378, max=378, sum=756 (2)\", \"tab\": \"General information\", \"score\": \"378.0\"}", - "Elementary Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Elementary Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Elementary Mathematics - # prompt tokens": "{\"description\": \"min=577.119, mean=577.119, max=577.119, sum=1154.238 (2)\", \"tab\": \"General information\", \"score\": \"577.1190476190476\"}", - "Elementary Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"elementary_mathematics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_elementary_mathematics\"" - } - } - }, - { - "evaluation_name": "Formal Logic", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Formal Logic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.587, - "details": { - "description": "min=0.587, mean=0.587, max=0.587, sum=1.175 (2)", - "tab": "Accuracy", - "Formal Logic - Observed inference time (s)": "{\"description\": \"min=0.197, mean=0.197, max=0.197, sum=0.393 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.196545644411965\"}", - "Formal Logic - # eval": "{\"description\": \"min=126, mean=126, max=126, sum=252 (2)\", \"tab\": \"General information\", \"score\": \"126.0\"}", - "Formal Logic - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Formal Logic - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Formal Logic - # prompt tokens": "{\"description\": \"min=604.667, mean=604.667, max=604.667, sum=1209.333 (2)\", \"tab\": \"General information\", \"score\": \"604.6666666666666\"}", - "Formal Logic - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"formal_logic\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_formal_logic\"" - } - } - }, - { - "evaluation_name": "High School World History", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on High School World History", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.878, - "details": { - "description": "min=0.878, mean=0.878, max=0.878, sum=1.755 (2)", - "tab": "Accuracy", - "High School Biology - Observed inference time (s)": "{\"description\": \"min=0.192, mean=0.192, max=0.192, sum=0.384 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.19177444058079873\"}", - "High School Chemistry - Observed inference time (s)": "{\"description\": \"min=0.236, mean=0.236, max=0.236, sum=0.472 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.23597407693346145\"}", - "High School Computer Science - Observed inference time (s)": "{\"description\": \"min=0.202, mean=0.202, max=0.202, sum=0.404 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.20180433988571167\"}", - "High School European History - Observed inference time (s)": "{\"description\": \"min=0.313, mean=0.313, max=0.313, sum=0.626 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3130656791455818\"}", - "High School Geography - Observed inference time (s)": "{\"description\": \"min=0.215, mean=0.215, max=0.215, sum=0.43 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.21512896725625702\"}", - "High School Government And Politics - Observed inference time (s)": "{\"description\": \"min=0.192, mean=0.192, max=0.192, sum=0.384 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.19191643611137113\"}", - "High School Macroeconomics - Observed inference time (s)": "{\"description\": \"min=0.204, mean=0.204, max=0.204, sum=0.409 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.20429076965038592\"}", - "High School Mathematics - Observed inference time (s)": "{\"description\": \"min=0.234, mean=0.234, max=0.234, sum=0.468 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2337868098859434\"}", - "High School Microeconomics - Observed inference time (s)": "{\"description\": \"min=0.184, mean=0.184, max=0.184, sum=0.367 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.18365505863638484\"}", - "High School Physics - Observed inference time (s)": "{\"description\": \"min=0.194, mean=0.194, max=0.194, sum=0.388 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.19382640068104726\"}", - "High School Psychology - Observed inference time (s)": "{\"description\": \"min=0.203, mean=0.203, max=0.203, sum=0.405 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.20258700432033713\"}", - "High School Statistics - Observed inference time (s)": "{\"description\": \"min=0.226, mean=0.226, max=0.226, sum=0.451 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.22551235446223505\"}", - "High School US History - Observed inference time (s)": "{\"description\": \"min=0.249, mean=0.249, max=0.249, sum=0.498 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2492340417469249\"}", - "High School World History - Observed inference time (s)": "{\"description\": \"min=0.231, mean=0.231, max=0.231, sum=0.462 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.23088843812419393\"}", - "High School Biology - # eval": "{\"description\": \"min=310, mean=310, max=310, sum=620 (2)\", \"tab\": \"General information\", \"score\": \"310.0\"}", - "High School Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Biology - # prompt tokens": "{\"description\": \"min=513.916, mean=513.916, max=513.916, sum=1027.832 (2)\", \"tab\": \"General information\", \"score\": \"513.916129032258\"}", - "High School Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Chemistry - # eval": "{\"description\": \"min=203, mean=203, max=203, sum=406 (2)\", \"tab\": \"General information\", \"score\": \"203.0\"}", - "High School Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # prompt tokens": "{\"description\": \"min=517.261, mean=517.261, max=517.261, sum=1034.522 (2)\", \"tab\": \"General information\", \"score\": \"517.2610837438424\"}", - "High School Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "High School Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # prompt tokens": "{\"description\": \"min=878.46, mean=878.46, max=878.46, sum=1756.92 (2)\", \"tab\": \"General information\", \"score\": \"878.46\"}", - "High School Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School European History - # eval": "{\"description\": \"min=165, mean=165, max=165, sum=330 (2)\", \"tab\": \"General information\", \"score\": \"165.0\"}", - "High School European History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School European History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # prompt tokens": "{\"description\": \"min=2814.903, mean=2814.903, max=2814.903, sum=5629.806 (2)\", \"tab\": \"General information\", \"score\": \"2814.9030303030304\"}", - "High School European History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Geography - # eval": "{\"description\": \"min=198, mean=198, max=198, sum=396 (2)\", \"tab\": \"General information\", \"score\": \"198.0\"}", - "High School Geography - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Geography - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # prompt tokens": "{\"description\": \"min=372.217, mean=372.217, max=372.217, sum=744.434 (2)\", \"tab\": \"General information\", \"score\": \"372.2171717171717\"}", - "High School Geography - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Government And Politics - # eval": "{\"description\": \"min=193, mean=193, max=193, sum=386 (2)\", \"tab\": \"General information\", \"score\": \"193.0\"}", - "High School Government And Politics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Government And Politics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # prompt tokens": "{\"description\": \"min=467.311, mean=467.311, max=467.311, sum=934.622 (2)\", \"tab\": \"General information\", \"score\": \"467.31088082901556\"}", - "High School Government And Politics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Macroeconomics - # eval": "{\"description\": \"min=390, mean=390, max=390, sum=780 (2)\", \"tab\": \"General information\", \"score\": \"390.0\"}", - "High School Macroeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Macroeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # prompt tokens": "{\"description\": \"min=374.349, mean=374.349, max=374.349, sum=748.697 (2)\", \"tab\": \"General information\", \"score\": \"374.34871794871793\"}", - "High School Macroeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Mathematics - # eval": "{\"description\": \"min=270, mean=270, max=270, sum=540 (2)\", \"tab\": \"General information\", \"score\": \"270.0\"}", - "High School Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # prompt tokens": "{\"description\": \"min=565.326, mean=565.326, max=565.326, sum=1130.652 (2)\", \"tab\": \"General information\", \"score\": \"565.325925925926\"}", - "High School Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Microeconomics - # eval": "{\"description\": \"min=238, mean=238, max=238, sum=476 (2)\", \"tab\": \"General information\", \"score\": \"238.0\"}", - "High School Microeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Microeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # prompt tokens": "{\"description\": \"min=402.277, mean=402.277, max=402.277, sum=804.555 (2)\", \"tab\": \"General information\", \"score\": \"402.2773109243698\"}", - "High School Microeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Physics - # eval": "{\"description\": \"min=151, mean=151, max=151, sum=302 (2)\", \"tab\": \"General information\", \"score\": \"151.0\"}", - "High School Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # prompt tokens": "{\"description\": \"min=580.536, mean=580.536, max=580.536, sum=1161.073 (2)\", \"tab\": \"General information\", \"score\": \"580.5364238410596\"}", - "High School Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Psychology - # eval": "{\"description\": \"min=545, mean=545, max=545, sum=1090 (2)\", \"tab\": \"General information\", \"score\": \"545.0\"}", - "High School Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # prompt tokens": "{\"description\": \"min=495.521, mean=495.521, max=495.521, sum=991.042 (2)\", \"tab\": \"General information\", \"score\": \"495.52110091743117\"}", - "High School Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Statistics - # eval": "{\"description\": \"min=216, mean=216, max=216, sum=432 (2)\", \"tab\": \"General information\", \"score\": \"216.0\"}", - "High School Statistics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Statistics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # prompt tokens": "{\"description\": \"min=830.477, mean=830.477, max=830.477, sum=1660.954 (2)\", \"tab\": \"General information\", \"score\": \"830.4768518518518\"}", - "High School Statistics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School US History - # eval": "{\"description\": \"min=204, mean=204, max=204, sum=408 (2)\", \"tab\": \"General information\", \"score\": \"204.0\"}", - "High School US History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School US History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # prompt tokens": "{\"description\": \"min=2237.176, mean=2237.176, max=2237.176, sum=4474.353 (2)\", \"tab\": \"General information\", \"score\": \"2237.176470588235\"}", - "High School US History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School World History - # eval": "{\"description\": \"min=237, mean=237, max=237, sum=474 (2)\", \"tab\": \"General information\", \"score\": \"237.0\"}", - "High School World History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School World History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # prompt tokens": "{\"description\": \"min=1448.354, mean=1448.354, max=1448.354, sum=2896.709 (2)\", \"tab\": \"General information\", \"score\": \"1448.3544303797469\"}", - "High School World History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"high_school_world_history\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_high_school_world_history\"" - } - } - }, - { - "evaluation_name": "Human Sexuality", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Human Sexuality", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.794, - "details": { - "description": "min=0.794, mean=0.794, max=0.794, sum=1.588 (2)", - "tab": "Accuracy", - "Human Aging - Observed inference time (s)": "{\"description\": \"min=0.206, mean=0.206, max=0.206, sum=0.411 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.20559344591046663\"}", - "Human Sexuality - Observed inference time (s)": "{\"description\": \"min=0.191, mean=0.191, max=0.191, sum=0.381 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.19073554941716084\"}", - "Human Aging - # eval": "{\"description\": \"min=223, mean=223, max=223, sum=446 (2)\", \"tab\": \"General information\", \"score\": \"223.0\"}", - "Human Aging - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Aging - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Aging - # prompt tokens": "{\"description\": \"min=322.121, mean=322.121, max=322.121, sum=644.242 (2)\", \"tab\": \"General information\", \"score\": \"322.1210762331838\"}", - "Human Aging - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Human Sexuality - # eval": "{\"description\": \"min=131, mean=131, max=131, sum=262 (2)\", \"tab\": \"General information\", \"score\": \"131.0\"}", - "Human Sexuality - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Sexuality - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # prompt tokens": "{\"description\": \"min=341.504, mean=341.504, max=341.504, sum=683.008 (2)\", \"tab\": \"General information\", \"score\": \"341.5038167938931\"}", - "Human Sexuality - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"human_sexuality\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_human_sexuality\"" - } - } - }, - { - "evaluation_name": "International Law", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on International Law", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.86, - "details": { - "description": "min=0.86, mean=0.86, max=0.86, sum=1.719 (2)", - "tab": "Accuracy", - "International Law - Observed inference time (s)": "{\"description\": \"min=0.23, mean=0.23, max=0.23, sum=0.46 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.22999596792804308\"}", - "International Law - # eval": "{\"description\": \"min=121, mean=121, max=121, sum=242 (2)\", \"tab\": \"General information\", \"score\": \"121.0\"}", - "International Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "International Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "International Law - # prompt tokens": "{\"description\": \"min=640.579, mean=640.579, max=640.579, sum=1281.157 (2)\", \"tab\": \"General information\", \"score\": \"640.5785123966942\"}", - "International Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"international_law\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_international_law\"" - } - } - }, - { - "evaluation_name": "Logical Fallacies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Logical Fallacies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.773, - "details": { - "description": "min=0.773, mean=0.773, max=0.773, sum=1.546 (2)", - "tab": "Accuracy", - "Logical Fallacies - Observed inference time (s)": "{\"description\": \"min=0.201, mean=0.201, max=0.201, sum=0.401 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.2005681289485627\"}", - "Logical Fallacies - # eval": "{\"description\": \"min=163, mean=163, max=163, sum=326 (2)\", \"tab\": \"General information\", \"score\": \"163.0\"}", - "Logical Fallacies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Logical Fallacies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Logical Fallacies - # prompt tokens": "{\"description\": \"min=449.632, mean=449.632, max=449.632, sum=899.264 (2)\", \"tab\": \"General information\", \"score\": \"449.6319018404908\"}", - "Logical Fallacies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"logical_fallacies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_logical_fallacies\"" - } - } - }, - { - "evaluation_name": "Machine Learning", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Machine Learning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.554, - "details": { - "description": "min=0.554, mean=0.554, max=0.554, sum=1.107 (2)", - "tab": "Accuracy", - "Machine Learning - Observed inference time (s)": "{\"description\": \"min=0.232, mean=0.232, max=0.232, sum=0.463 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.23156332118170603\"}", - "Machine Learning - # eval": "{\"description\": \"min=112, mean=112, max=112, sum=224 (2)\", \"tab\": \"General information\", \"score\": \"112.0\"}", - "Machine Learning - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Machine Learning - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Machine Learning - # prompt tokens": "{\"description\": \"min=681.848, mean=681.848, max=681.848, sum=1363.696 (2)\", \"tab\": \"General information\", \"score\": \"681.8482142857143\"}", - "Machine Learning - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"machine_learning\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_machine_learning\"" - } - } - }, - { - "evaluation_name": "Management", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Management", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.845, - "details": { - "description": "min=0.845, mean=0.845, max=0.845, sum=1.689 (2)", - "tab": "Accuracy", - "Management - Observed inference time (s)": "{\"description\": \"min=0.197, mean=0.197, max=0.197, sum=0.394 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.19694008410555644\"}", - "Management - # eval": "{\"description\": \"min=103, mean=103, max=103, sum=206 (2)\", \"tab\": \"General information\", \"score\": \"103.0\"}", - "Management - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Management - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Management - # prompt tokens": "{\"description\": \"min=283.854, mean=283.854, max=283.854, sum=567.709 (2)\", \"tab\": \"General information\", \"score\": \"283.8543689320388\"}", - "Management - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"management\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_management\"" - } - } - }, - { - "evaluation_name": "Marketing", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Marketing", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.919, - "details": { - "description": "min=0.919, mean=0.919, max=0.919, sum=1.838 (2)", - "tab": "Accuracy", - "Marketing - Observed inference time (s)": "{\"description\": \"min=0.184, mean=0.184, max=0.184, sum=0.368 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.18401269525544256\"}", - "Marketing - # eval": "{\"description\": \"min=234, mean=234, max=234, sum=468 (2)\", \"tab\": \"General information\", \"score\": \"234.0\"}", - "Marketing - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Marketing - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Marketing - # prompt tokens": "{\"description\": \"min=404.415, mean=404.415, max=404.415, sum=808.829 (2)\", \"tab\": \"General information\", \"score\": \"404.4145299145299\"}", - "Marketing - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"marketing\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_marketing\"" - } - } - }, - { - "evaluation_name": "Medical Genetics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Medical Genetics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.85, - "details": { - "description": "min=0.85, mean=0.85, max=0.85, sum=1.7 (2)", - "tab": "Accuracy", - "Medical Genetics - Observed inference time (s)": "{\"description\": \"min=0.176, mean=0.176, max=0.176, sum=0.351 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.17553309679031373\"}", - "Medical Genetics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Medical Genetics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Medical Genetics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Medical Genetics - # prompt tokens": "{\"description\": \"min=342.35, mean=342.35, max=342.35, sum=684.7 (2)\", \"tab\": \"General information\", \"score\": \"342.35\"}", - "Medical Genetics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"medical_genetics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_medical_genetics\"" - } - } - }, - { - "evaluation_name": "Miscellaneous", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Miscellaneous", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.852, - "details": { - "description": "min=0.852, mean=0.852, max=0.852, sum=1.704 (2)", - "tab": "Accuracy", - "Miscellaneous - Observed inference time (s)": "{\"description\": \"min=0.174, mean=0.174, max=0.174, sum=0.347 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.17373346399377892\"}", - "Miscellaneous - # eval": "{\"description\": \"min=783, mean=783, max=783, sum=1566 (2)\", \"tab\": \"General information\", \"score\": \"783.0\"}", - "Miscellaneous - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Miscellaneous - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Miscellaneous - # prompt tokens": "{\"description\": \"min=303.7, mean=303.7, max=303.7, sum=607.4 (2)\", \"tab\": \"General information\", \"score\": \"303.6998722860792\"}", - "Miscellaneous - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"miscellaneous\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_miscellaneous\"" - } - } - }, - { - "evaluation_name": "Moral Scenarios", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Moral Scenarios", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.511, - "details": { - "description": "min=0.511, mean=0.511, max=0.511, sum=1.021 (2)", - "tab": "Accuracy", - "Moral Disputes - Observed inference time (s)": "{\"description\": \"min=0.168, mean=0.168, max=0.168, sum=0.337 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.16836080041234894\"}", - "Moral Scenarios - Observed inference time (s)": "{\"description\": \"min=0.171, mean=0.171, max=0.171, sum=0.342 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.1708347949235799\"}", - "Moral Disputes - # eval": "{\"description\": \"min=346, mean=346, max=346, sum=692 (2)\", \"tab\": \"General information\", \"score\": \"346.0\"}", - "Moral Disputes - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Disputes - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Disputes - # prompt tokens": "{\"description\": \"min=476.182, mean=476.182, max=476.182, sum=952.364 (2)\", \"tab\": \"General information\", \"score\": \"476.1820809248555\"}", - "Moral Disputes - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Moral Scenarios - # eval": "{\"description\": \"min=895, mean=895, max=895, sum=1790 (2)\", \"tab\": \"General information\", \"score\": \"895.0\"}", - "Moral Scenarios - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Scenarios - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # prompt tokens": "{\"description\": \"min=668.494, mean=668.494, max=668.494, sum=1336.988 (2)\", \"tab\": \"General information\", \"score\": \"668.4938547486033\"}", - "Moral Scenarios - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"moral_scenarios\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_moral_scenarios\"" - } - } - }, - { - "evaluation_name": "Nutrition", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Nutrition", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.778, - "details": { - "description": "min=0.778, mean=0.778, max=0.778, sum=1.556 (2)", - "tab": "Accuracy", - "Nutrition - Observed inference time (s)": "{\"description\": \"min=0.168, mean=0.168, max=0.168, sum=0.337 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.16839487724054872\"}", - "Nutrition - # eval": "{\"description\": \"min=306, mean=306, max=306, sum=612 (2)\", \"tab\": \"General information\", \"score\": \"306.0\"}", - "Nutrition - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Nutrition - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Nutrition - # prompt tokens": "{\"description\": \"min=599.637, mean=599.637, max=599.637, sum=1199.275 (2)\", \"tab\": \"General information\", \"score\": \"599.6372549019608\"}", - "Nutrition - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"nutrition\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_nutrition\"" - } - } - }, - { - "evaluation_name": "Prehistory", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Prehistory", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.836, - "details": { - "description": "min=0.836, mean=0.836, max=0.836, sum=1.673 (2)", - "tab": "Accuracy", - "Prehistory - Observed inference time (s)": "{\"description\": \"min=0.168, mean=0.168, max=0.168, sum=0.337 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.16826030795956837\"}", - "Prehistory - # eval": "{\"description\": \"min=324, mean=324, max=324, sum=648 (2)\", \"tab\": \"General information\", \"score\": \"324.0\"}", - "Prehistory - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Prehistory - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Prehistory - # prompt tokens": "{\"description\": \"min=528.364, mean=528.364, max=528.364, sum=1056.728 (2)\", \"tab\": \"General information\", \"score\": \"528.3641975308642\"}", - "Prehistory - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"prehistory\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_prehistory\"" - } - } - }, - { - "evaluation_name": "Public Relations", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Public Relations", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.709, - "details": { - "description": "min=0.709, mean=0.709, max=0.709, sum=1.418 (2)", - "tab": "Accuracy", - "Public Relations - Observed inference time (s)": "{\"description\": \"min=0.164, mean=0.164, max=0.164, sum=0.328 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.1641989447853782\"}", - "Public Relations - # eval": "{\"description\": \"min=110, mean=110, max=110, sum=220 (2)\", \"tab\": \"General information\", \"score\": \"110.0\"}", - "Public Relations - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Public Relations - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Public Relations - # prompt tokens": "{\"description\": \"min=408.427, mean=408.427, max=408.427, sum=816.855 (2)\", \"tab\": \"General information\", \"score\": \"408.42727272727274\"}", - "Public Relations - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"public_relations\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_public_relations\"" - } - } - }, - { - "evaluation_name": "Security Studies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Security Studies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.682, - "details": { - "description": "min=0.682, mean=0.682, max=0.682, sum=1.363 (2)", - "tab": "Accuracy", - "Security Studies - Observed inference time (s)": "{\"description\": \"min=0.174, mean=0.174, max=0.174, sum=0.349 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.1744946577111069\"}", - "Security Studies - # eval": "{\"description\": \"min=245, mean=245, max=245, sum=490 (2)\", \"tab\": \"General information\", \"score\": \"245.0\"}", - "Security Studies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Security Studies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Security Studies - # prompt tokens": "{\"description\": \"min=1166.931, mean=1166.931, max=1166.931, sum=2333.861 (2)\", \"tab\": \"General information\", \"score\": \"1166.930612244898\"}", - "Security Studies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"security_studies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_security_studies\"" - } - } - }, - { - "evaluation_name": "Sociology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Sociology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.861, - "details": { - "description": "min=0.861, mean=0.861, max=0.861, sum=1.721 (2)", - "tab": "Accuracy", - "Sociology - Observed inference time (s)": "{\"description\": \"min=0.19, mean=0.19, max=0.19, sum=0.381 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.1903395510431546\"}", - "Sociology - # eval": "{\"description\": \"min=201, mean=201, max=201, sum=402 (2)\", \"tab\": \"General information\", \"score\": \"201.0\"}", - "Sociology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Sociology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Sociology - # prompt tokens": "{\"description\": \"min=450.1, mean=450.1, max=450.1, sum=900.199 (2)\", \"tab\": \"General information\", \"score\": \"450.0995024875622\"}", - "Sociology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"sociology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_sociology\"" - } - } - }, - { - "evaluation_name": "Virology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Virology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.578, - "details": { - "description": "min=0.578, mean=0.578, max=0.578, sum=1.157 (2)", - "tab": "Accuracy", - "Virology - Observed inference time (s)": "{\"description\": \"min=0.174, mean=0.174, max=0.174, sum=0.348 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.1741443513387657\"}", - "Virology - # eval": "{\"description\": \"min=166, mean=166, max=166, sum=332 (2)\", \"tab\": \"General information\", \"score\": \"166.0\"}", - "Virology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Virology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Virology - # prompt tokens": "{\"description\": \"min=343.819, mean=343.819, max=343.819, sum=687.639 (2)\", \"tab\": \"General information\", \"score\": \"343.8192771084337\"}", - "Virology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"virology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_virology\"" - } - } - }, - { - "evaluation_name": "World Religions", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on World Religions", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.83, - "details": { - "description": "min=0.83, mean=0.83, max=0.83, sum=1.661 (2)", - "tab": "Accuracy", - "World Religions - Observed inference time (s)": "{\"description\": \"min=0.162, mean=0.162, max=0.162, sum=0.325 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.16239780292176365\"}", - "World Religions - # eval": "{\"description\": \"min=171, mean=171, max=171, sum=342 (2)\", \"tab\": \"General information\", \"score\": \"171.0\"}", - "World Religions - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "World Religions - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "World Religions - # prompt tokens": "{\"description\": \"min=276.07, mean=276.07, max=276.07, sum=552.14 (2)\", \"tab\": \"General information\", \"score\": \"276.0701754385965\"}", - "World Religions - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"world_religions\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_world_religions\"" - } - } - }, - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.887, - "details": { - "description": "", - "tab": "Efficiency" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/qwen_qwen2.5-7b-instruct.json b/data/models/qwen_qwen2.5-7b-instruct.json deleted file mode 100644 index 6b70a46306cfff7eb7771f8599bb6ebbe0831c9e..0000000000000000000000000000000000000000 --- a/data/models/qwen_qwen2.5-7b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-7B-Instruct", - "id": "Qwen/Qwen2.5-7B-Instruct", - "developer": "Qwen", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Qwen_Qwen2.5-7B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7585 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5394 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2911 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.402 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4287 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qwen_qwen2.5-7b.json b/data/models/qwen_qwen2.5-7b.json deleted file mode 100644 index 71be2d736650214f1473ac2a6025cebea90e69cd..0000000000000000000000000000000000000000 --- a/data/models/qwen_qwen2.5-7b.json +++ /dev/null @@ -1,185 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-7B", - "id": "Qwen/Qwen2.5-7B", - "developer": "Qwen", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Qwen_Qwen2.5-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3374 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5416 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2508 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3247 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4424 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4365 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "la_leaderboard/Qwen/Qwen2.5-7B/1774451270", - "retrieved_timestamp": "2024-10-27T00:00:00Z", - "source_metadata": { - "source_name": "La Leaderboard", - "source_type": "evaluation_run", - "source_url": "https://huggingface.co/spaces/la-leaderboard/la-leaderboard", - "source_organization_name": "La Leaderboard", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "custom", - "version": "1.0" - }, - "benchmark": "la_leaderboard", - "evaluation_results": [ - { - "evaluation_name": "la_leaderboard", - "metric_config": { - "evaluation_description": "La Leaderboard: LLM evaluation for Spanish varieties and languages of Spain and Latin America", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 100 - }, - "score_details": { - "score": 27.61 - }, - "source_data": { - "source_type": "url", - "dataset_name": "La Leaderboard composite dataset", - "url": [ - "https://huggingface.co/spaces/la-leaderboard/la-leaderboard" - ] - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qwen_qwen2.5-coder-14b-instruct.json b/data/models/qwen_qwen2.5-coder-14b-instruct.json deleted file mode 100644 index ded4f31bd4081a3fdc68c0ec6827a24d2a417205..0000000000000000000000000000000000000000 --- a/data/models/qwen_qwen2.5-coder-14b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-Coder-14B-Instruct", - "id": "Qwen/Qwen2.5-Coder-14B-Instruct", - "developer": "Qwen", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Qwen_Qwen2.5-Coder-14B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6908 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.614 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3248 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3045 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3915 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3939 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qwen_qwen2.5-coder-14b.json b/data/models/qwen_qwen2.5-coder-14b.json deleted file mode 100644 index fa9843e72588c52e483b0e02fbdc2cd2f24261f2..0000000000000000000000000000000000000000 --- a/data/models/qwen_qwen2.5-coder-14b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-Coder-14B", - "id": "Qwen/Qwen2.5-Coder-14B", - "developer": "Qwen", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Qwen_Qwen2.5-Coder-14B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3473 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5865 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2251 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2928 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3874 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4521 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qwen_qwen2.5-coder-32b-instruct.json b/data/models/qwen_qwen2.5-coder-32b-instruct.json deleted file mode 100644 index 43ed23fb1d29f1e0b7f66caedc22ec4b6ae7c622..0000000000000000000000000000000000000000 --- a/data/models/qwen_qwen2.5-coder-32b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-Coder-32B-Instruct", - "id": "Qwen/Qwen2.5-Coder-32B-Instruct", - "developer": "Qwen", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "32.764" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Qwen_Qwen2.5-Coder-32B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7265 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6625 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4955 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.349 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4386 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4413 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qwen_qwen2.5-coder-32b.json b/data/models/qwen_qwen2.5-coder-32b.json deleted file mode 100644 index 1462b425fc6fea6af2fd3f614ab7459e89e42a09..0000000000000000000000000000000000000000 --- a/data/models/qwen_qwen2.5-coder-32b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-Coder-32B", - "id": "Qwen/Qwen2.5-Coder-32B", - "developer": "Qwen", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "32.764" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Qwen_Qwen2.5-Coder-32B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4363 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6404 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3089 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3465 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4528 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5303 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qwen_qwen2.5-coder-7b-instruct.json b/data/models/qwen_qwen2.5-coder-7b-instruct.json deleted file mode 100644 index 96b0a8af52fb442affa2ad7ba1ed017d80ec2231..0000000000000000000000000000000000000000 --- a/data/models/qwen_qwen2.5-coder-7b-instruct.json +++ /dev/null @@ -1,275 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-Coder-7B-Instruct", - "id": "Qwen/Qwen2.5-Coder-7B-Instruct", - "developer": "Qwen", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Qwen_Qwen2.5-Coder-7B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6101 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5008 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3716 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2919 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4073 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3352 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "hfopenllm_v2/Qwen_Qwen2.5-Coder-7B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6147 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4999 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.031 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2936 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4099 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3354 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qwen_qwen2.5-coder-7b.json b/data/models/qwen_qwen2.5-coder-7b.json deleted file mode 100644 index cc45385065c49e8a8aba510c07a465f05810c2e7..0000000000000000000000000000000000000000 --- a/data/models/qwen_qwen2.5-coder-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-Coder-7B", - "id": "Qwen/Qwen2.5-Coder-7B", - "developer": "Qwen", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Qwen_Qwen2.5-Coder-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3446 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4856 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1918 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2592 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3449 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3679 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qwen_qwen2.5-math-1.5b-instruct.json b/data/models/qwen_qwen2.5-math-1.5b-instruct.json deleted file mode 100644 index 47f4697084e69c8c68aaf8fdbf0ec5c48b54aa5c..0000000000000000000000000000000000000000 --- a/data/models/qwen_qwen2.5-math-1.5b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-Math-1.5B-Instruct", - "id": "Qwen/Qwen2.5-Math-1.5B-Instruct", - "developer": "Qwen", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.544" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Qwen_Qwen2.5-Math-1.5B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1856 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3752 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2628 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2651 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3685 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1801 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qwen_qwen2.5-math-72b-instruct.json b/data/models/qwen_qwen2.5-math-72b-instruct.json deleted file mode 100644 index eb6971226f29c386da87392261bf3370f4f28750..0000000000000000000000000000000000000000 --- a/data/models/qwen_qwen2.5-math-72b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-Math-72B-Instruct", - "id": "Qwen/Qwen2.5-Math-72B-Instruct", - "developer": "Qwen", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "72.706" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Qwen_Qwen2.5-Math-72B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4003 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6452 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6239 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3314 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4473 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4812 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qwen_qwen2.5-math-7b-instruct.json b/data/models/qwen_qwen2.5-math-7b-instruct.json deleted file mode 100644 index 68a84f8ca71cf4f404b777a08ab28cd3208accda..0000000000000000000000000000000000000000 --- a/data/models/qwen_qwen2.5-math-7b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-Math-7B-Instruct", - "id": "Qwen/Qwen2.5-Math-7B-Instruct", - "developer": "Qwen", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Qwen_Qwen2.5-Math-7B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2636 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4388 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5808 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2617 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3647 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.282 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qwen_qwen2.5-math-7b.json b/data/models/qwen_qwen2.5-math-7b.json deleted file mode 100644 index 1cb89a17737d77cc444effc5716e8497dd1f3c2d..0000000000000000000000000000000000000000 --- a/data/models/qwen_qwen2.5-math-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-Math-7B", - "id": "Qwen/Qwen2.5-Math-7B", - "developer": "Qwen", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Qwen_Qwen2.5-Math-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.246 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4455 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3051 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2936 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3781 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2718 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qwen_qwen3-0-6b-fc.json b/data/models/qwen_qwen3-0-6b-fc.json deleted file mode 100644 index 58b85125ffb21142330cd6d37ae439b5950a31ee..0000000000000000000000000000000000000000 --- a/data/models/qwen_qwen3-0-6b-fc.json +++ /dev/null @@ -1,851 +0,0 @@ -{ - "model_info": { - "name": "Qwen3-0.6B (FC)", - "id": "qwen/qwen3-0-6b-fc", - "developer": "qwen", - "additional_details": { - "raw_model_name": "Qwen3-0.6B (FC)", - "organization": "Qwen", - "license": "apache-2.0", - "mode": "FC", - "model_link": "https://huggingface.co/Qwen/Qwen3-0.6B" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/qwen/qwen3-0-6b-fc/1775236112.414485", - "retrieved_timestamp": "1775236112.414485", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 92.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 23.93 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 0.46 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 0.68 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 8.45 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 0.96 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 71.79 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 64.17 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 86.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 67.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 69.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 56.62 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 61.24 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 56.13 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 43.75 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 37.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 3.62 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 5.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 2.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 3.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 4.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 1.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 1.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 1.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 8.6 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 2.58 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 1.94 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 21.29 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 75.0 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 80.84 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qwen_qwen3-0-6b-prompt.json b/data/models/qwen_qwen3-0-6b-prompt.json deleted file mode 100644 index fb0a574e8b07786672c7cddeba6ab991eb96b412..0000000000000000000000000000000000000000 --- a/data/models/qwen_qwen3-0-6b-prompt.json +++ /dev/null @@ -1,905 +0,0 @@ -{ - "model_info": { - "name": "Qwen3-0.6B (Prompt)", - "id": "qwen/qwen3-0-6b-prompt", - "developer": "qwen", - "additional_details": { - "raw_model_name": "Qwen3-0.6B (Prompt)", - "organization": "Qwen", - "license": "apache-2.0", - "mode": "Prompt", - "model_link": "https://huggingface.co/Qwen/Qwen3-0.6B" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/qwen/qwen3-0-6b-prompt/1775236112.415482", - "retrieved_timestamp": "1775236112.415482", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 94.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 22.38 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 3.65 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 3.1 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 4.32 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 10.31 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 70.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 64.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 78.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 75.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 63.0 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 49.37 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 57.75 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 47.77 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 37.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 37.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 1.38 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 1.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 1.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 1.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 1.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 0.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 1.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 8.39 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 1.29 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 2.58 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 21.29 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 75.0 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 82.5 - } - }, - { - "evaluation_result_id": "bfcl.format_sensitivity.max_delta::max_delta", - "evaluation_name": "bfcl.format_sensitivity.max_delta", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.format_sensitivity.max_delta", - "metric_name": "Format sensitivity max delta", - "metric_kind": "difference", - "metric_unit": "percentage_points", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Format Sensitivity Max Delta" - } - }, - "score_details": { - "score": 60.5 - } - }, - { - "evaluation_result_id": "bfcl.format_sensitivity.stddev::stddev", - "evaluation_name": "bfcl.format_sensitivity.stddev", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.format_sensitivity.stddev", - "metric_name": "Format sensitivity standard deviation", - "metric_kind": "difference", - "metric_unit": "percentage_points", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Format Sensitivity Standard Deviation" - } - }, - "score_details": { - "score": 24.35 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qwen_qwen3-1-7b-fc.json b/data/models/qwen_qwen3-1-7b-fc.json deleted file mode 100644 index d7cf54e2c456a72f9ccfb2482f5f3a7656c5b538..0000000000000000000000000000000000000000 --- a/data/models/qwen_qwen3-1-7b-fc.json +++ /dev/null @@ -1,851 +0,0 @@ -{ - "model_info": { - "name": "Qwen3-1.7B (FC)", - "id": "qwen/qwen3-1-7b-fc", - "developer": "qwen", - "additional_details": { - "raw_model_name": "Qwen3-1.7B (FC)", - "organization": "Qwen", - "license": "apache-2.0", - "mode": "FC", - "model_link": "https://huggingface.co/Qwen/Qwen3-1.7B" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/qwen/qwen3-1-7b-fc/1775236112.403111", - "retrieved_timestamp": "1775236112.403111", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 71.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 28.41 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 4.33 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 5.12 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 7.37 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 13.35 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 82.92 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 70.67 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 92.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 88.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 80.0 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 74.61 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 76.74 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 74.26 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 62.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 75.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 11.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 15.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 6.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 12.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 11.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 2.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 3.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 2.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 6.02 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 4.52 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 7.74 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 5.81 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 81.25 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 76.54 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qwen_qwen3-14b-fc.json b/data/models/qwen_qwen3-14b-fc.json deleted file mode 100644 index ecc06d14e54a93ea4d482e57357de5f600a06902..0000000000000000000000000000000000000000 --- a/data/models/qwen_qwen3-14b-fc.json +++ /dev/null @@ -1,851 +0,0 @@ -{ - "model_info": { - "name": "Qwen3-14B (FC)", - "id": "qwen/qwen3-14b-fc", - "developer": "qwen", - "additional_details": { - "raw_model_name": "Qwen3-14B (FC)", - "organization": "Qwen", - "license": "apache-2.0", - "mode": "FC", - "model_link": "https://huggingface.co/Qwen/Qwen3-14B" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/qwen/qwen3-14b-fc/1775236112.387295", - "retrieved_timestamp": "1775236112.387295", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 43.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 41.03 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 3.38 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 4.5 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 18.84 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 13.34 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 84.94 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 74.75 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 93.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 80.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 92.0 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 80.01 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 85.66 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 79.01 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 68.75 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 70.83 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 34.75 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 39.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 34.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 33.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 32.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 10.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 8.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 12.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 19.57 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 7.1 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 16.77 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 34.84 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 87.5 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 81.94 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qwen_qwen3-14b-prompt.json b/data/models/qwen_qwen3-14b-prompt.json deleted file mode 100644 index 774840150dcbfe128ecc82487b314b2c376f3d57..0000000000000000000000000000000000000000 --- a/data/models/qwen_qwen3-14b-prompt.json +++ /dev/null @@ -1,905 +0,0 @@ -{ - "model_info": { - "name": "Qwen3-14B (Prompt)", - "id": "qwen/qwen3-14b-prompt", - "developer": "qwen", - "additional_details": { - "raw_model_name": "Qwen3-14B (Prompt)", - "organization": "Qwen", - "license": "apache-2.0", - "mode": "Prompt", - "model_link": "https://huggingface.co/Qwen/Qwen3-14B" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/qwen/qwen3-14b-prompt/1775236112.389882", - "retrieved_timestamp": "1775236112.389882", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 47.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 37.77 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 1.35 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 1.2 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 8.5 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 2.3 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 89.46 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 76.83 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 93.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 95.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 92.0 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 79.35 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 84.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 78.06 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 87.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 75.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 26.13 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 16.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 37.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 31.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 19.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 10.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 6.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 15.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 11.18 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 4.52 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 6.45 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 22.58 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 81.25 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 87.18 - } - }, - { - "evaluation_result_id": "bfcl.format_sensitivity.max_delta::max_delta", - "evaluation_name": "bfcl.format_sensitivity.max_delta", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.format_sensitivity.max_delta", - "metric_name": "Format sensitivity max delta", - "metric_kind": "difference", - "metric_unit": "percentage_points", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Format Sensitivity Max Delta" - } - }, - "score_details": { - "score": 14.0 - } - }, - { - "evaluation_result_id": "bfcl.format_sensitivity.stddev::stddev", - "evaluation_name": "bfcl.format_sensitivity.stddev", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.format_sensitivity.stddev", - "metric_name": "Format sensitivity standard deviation", - "metric_kind": "difference", - "metric_unit": "percentage_points", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Format Sensitivity Standard Deviation" - } - }, - "score_details": { - "score": 3.97 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qwen_qwen3-235b-a22b-fp8-tput.json b/data/models/qwen_qwen3-235b-a22b-fp8-tput.json deleted file mode 100644 index 48335da681a547106fb5ede6d50732dba4c58303..0000000000000000000000000000000000000000 --- a/data/models/qwen_qwen3-235b-a22b-fp8-tput.json +++ /dev/null @@ -1,234 +0,0 @@ -{ - "model_info": { - "name": "Qwen3 235B A22B FP8 Throughput", - "id": "qwen/qwen3-235b-a22b-fp8-tput", - "developer": "qwen", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_capabilities/qwen_qwen3-235b-a22b-fp8-tput/1774096304.5056698", - "retrieved_timestamp": "1774096304.5056698", - "source_metadata": { - "source_name": "helm_capabilities", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_capabilities", - "evaluation_results": [ - { - "evaluation_name": "Mean score", - "source_data": { - "dataset_name": "helm_capabilities", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "The mean of the scores from all columns.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.726, - "details": { - "description": "", - "tab": "Accuracy", - "Mean score - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"175.88874367192255\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU-Pro", - "source_data": { - "dataset_name": "MMLU-Pro", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on MMLU-Pro", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.817, - "details": { - "description": "min=0.817, mean=0.817, max=0.817, sum=0.817 (1)", - "tab": "Accuracy", - "MMLU-Pro - Observed inference time (s)": "{\"description\": \"min=126.73, mean=126.73, max=126.73, sum=126.73 (1)\", \"tab\": \"Efficiency\", \"score\": \"126.73047786664962\"}", - "MMLU-Pro - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "MMLU-Pro - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - # prompt tokens": "{\"description\": \"min=259.715, mean=259.715, max=259.715, sum=259.715 (1)\", \"tab\": \"General information\", \"score\": \"259.715\"}", - "MMLU-Pro - # output tokens": "{\"description\": \"min=3518.576, mean=3518.576, max=3518.576, sum=3518.576 (1)\", \"tab\": \"General information\", \"score\": \"3518.576\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"all\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.623, - "details": { - "description": "min=0.623, mean=0.623, max=0.623, sum=0.623 (1)", - "tab": "Accuracy", - "GPQA - Observed inference time (s)": "{\"description\": \"min=237.413, mean=237.413, max=237.413, sum=237.413 (1)\", \"tab\": \"Efficiency\", \"score\": \"237.41318658488748\"}", - "GPQA - # eval": "{\"description\": \"min=446, mean=446, max=446, sum=446 (1)\", \"tab\": \"General information\", \"score\": \"446.0\"}", - "GPQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - # prompt tokens": "{\"description\": \"min=274.37, mean=274.37, max=274.37, sum=274.37 (1)\", \"tab\": \"General information\", \"score\": \"274.36995515695065\"}", - "GPQA - # output tokens": "{\"description\": \"min=7431.507, mean=7431.507, max=7431.507, sum=7431.507 (1)\", \"tab\": \"General information\", \"score\": \"7431.506726457399\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"gpqa_main\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "IFEval Strict Acc on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.816, - "details": { - "description": "min=0.816, mean=0.816, max=0.816, sum=0.816 (1)", - "tab": "Accuracy", - "IFEval - Observed inference time (s)": "{\"description\": \"min=36.742, mean=36.742, max=36.742, sum=36.742 (1)\", \"tab\": \"Efficiency\", \"score\": \"36.742134021963516\"}", - "IFEval - # eval": "{\"description\": \"min=541, mean=541, max=541, sum=541 (1)\", \"tab\": \"General information\", \"score\": \"541.0\"}", - "IFEval - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - # prompt tokens": "{\"description\": \"min=46.492, mean=46.492, max=46.492, sum=46.492 (1)\", \"tab\": \"General information\", \"score\": \"46.491682070240294\"}", - "IFEval - # output tokens": "{\"description\": \"min=1101.856, mean=1101.856, max=1101.856, sum=1101.856 (1)\", \"tab\": \"General information\", \"score\": \"1101.8558225508318\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WildBench", - "source_data": { - "dataset_name": "WildBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "WB Score on WildBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.828, - "details": { - "description": "min=0.828, mean=0.828, max=0.828, sum=0.828 (1)", - "tab": "Accuracy", - "WildBench - Observed inference time (s)": "{\"description\": \"min=125.734, mean=125.734, max=125.734, sum=125.734 (1)\", \"tab\": \"Efficiency\", \"score\": \"125.73418169164657\"}", - "WildBench - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "WildBench - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # prompt tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # output tokens": "{\"description\": \"min=3594.207, mean=3594.207, max=3594.207, sum=3594.207 (1)\", \"tab\": \"General information\", \"score\": \"3594.207\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"v2\"" - } - } - }, - { - "evaluation_name": "Omni-MATH", - "source_data": { - "dataset_name": "Omni-MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Acc on Omni-MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.548, - "details": { - "description": "min=0.548, mean=0.548, max=0.548, sum=0.548 (1)", - "tab": "Accuracy", - "Omni-MATH - Observed inference time (s)": "{\"description\": \"min=352.824, mean=352.824, max=352.824, sum=352.824 (1)\", \"tab\": \"Efficiency\", \"score\": \"352.82373819446565\"}", - "Omni-MATH - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "Omni-MATH - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - # prompt tokens": "{\"description\": \"min=111.6, mean=111.6, max=111.6, sum=111.6 (1)\", \"tab\": \"General information\", \"score\": \"111.6\"}", - "Omni-MATH - # output tokens": "{\"description\": \"min=10072.403, mean=10072.403, max=10072.403, sum=10072.403 (1)\", \"tab\": \"General information\", \"score\": \"10072.403\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/qwen_qwen3-235b-a22b-instruct-2507-fc.json b/data/models/qwen_qwen3-235b-a22b-instruct-2507-fc.json deleted file mode 100644 index 3d4b5dc5ad678366b08700e9b275157d6b88e8eb..0000000000000000000000000000000000000000 --- a/data/models/qwen_qwen3-235b-a22b-instruct-2507-fc.json +++ /dev/null @@ -1,851 +0,0 @@ -{ - "model_info": { - "name": "Qwen3-235B-A22B-Instruct-2507 (FC)", - "id": "qwen/qwen3-235b-a22b-instruct-2507-fc", - "developer": "qwen", - "additional_details": { - "raw_model_name": "Qwen3-235B-A22B-Instruct-2507 (FC)", - "organization": "Qwen", - "license": "apache-2.0", - "mode": "FC", - "model_link": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Instruct-2507" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/qwen/qwen3-235b-a22b-instruct-2507-fc/1775236112.3813179", - "retrieved_timestamp": "1775236112.3813179", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 31.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 47.99 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 2.5 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 2.57 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 2.44 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 6.27 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 37.4 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 40.58 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 36.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 53.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 19.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 68.91 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 58.53 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 71.6 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 68.75 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 62.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 45.38 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 57.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 35.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 33.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 55.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 54.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 57.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 51.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 23.87 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 7.1 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 18.71 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 45.81 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 87.5 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 81.73 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qwen_qwen3-235b-a22b-instruct-2507-fp8.json b/data/models/qwen_qwen3-235b-a22b-instruct-2507-fp8.json deleted file mode 100644 index 9bc74a94602e887d395a9acf368d168bc718c210..0000000000000000000000000000000000000000 --- a/data/models/qwen_qwen3-235b-a22b-instruct-2507-fp8.json +++ /dev/null @@ -1,234 +0,0 @@ -{ - "model_info": { - "name": "Qwen3 235B A22B Instruct 2507 FP8", - "id": "qwen/qwen3-235b-a22b-instruct-2507-fp8", - "developer": "qwen", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_capabilities/qwen_qwen3-235b-a22b-instruct-2507-fp8/1774096304.5056698", - "retrieved_timestamp": "1774096304.5056698", - "source_metadata": { - "source_name": "helm_capabilities", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_capabilities", - "evaluation_results": [ - { - "evaluation_name": "Mean score", - "source_data": { - "dataset_name": "helm_capabilities", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "The mean of the scores from all columns.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.798, - "details": { - "description": "", - "tab": "Accuracy", - "Mean score - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"91.57420329307861\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU-Pro", - "source_data": { - "dataset_name": "MMLU-Pro", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on MMLU-Pro", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.844, - "details": { - "description": "min=0.844, mean=0.844, max=0.844, sum=0.844 (1)", - "tab": "Accuracy", - "MMLU-Pro - Observed inference time (s)": "{\"description\": \"min=52.244, mean=52.244, max=52.244, sum=52.244 (1)\", \"tab\": \"Efficiency\", \"score\": \"52.24400525426864\"}", - "MMLU-Pro - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "MMLU-Pro - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - # prompt tokens": "{\"description\": \"min=259.715, mean=259.715, max=259.715, sum=259.715 (1)\", \"tab\": \"General information\", \"score\": \"259.715\"}", - "MMLU-Pro - # output tokens": "{\"description\": \"min=1423.589, mean=1423.589, max=1423.589, sum=1423.589 (1)\", \"tab\": \"General information\", \"score\": \"1423.589\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"all\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.726, - "details": { - "description": "min=0.726, mean=0.726, max=0.726, sum=0.726 (1)", - "tab": "Accuracy", - "GPQA - Observed inference time (s)": "{\"description\": \"min=103.303, mean=103.303, max=103.303, sum=103.303 (1)\", \"tab\": \"Efficiency\", \"score\": \"103.30346254970995\"}", - "GPQA - # eval": "{\"description\": \"min=446, mean=446, max=446, sum=446 (1)\", \"tab\": \"General information\", \"score\": \"446.0\"}", - "GPQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - # prompt tokens": "{\"description\": \"min=274.37, mean=274.37, max=274.37, sum=274.37 (1)\", \"tab\": \"General information\", \"score\": \"274.36995515695065\"}", - "GPQA - # output tokens": "{\"description\": \"min=3922.17, mean=3922.17, max=3922.17, sum=3922.17 (1)\", \"tab\": \"General information\", \"score\": \"3922.170403587444\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"gpqa_main\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "IFEval Strict Acc on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.835, - "details": { - "description": "min=0.835, mean=0.835, max=0.835, sum=0.835 (1)", - "tab": "Accuracy", - "IFEval - Observed inference time (s)": "{\"description\": \"min=12.729, mean=12.729, max=12.729, sum=12.729 (1)\", \"tab\": \"Efficiency\", \"score\": \"12.728508173648178\"}", - "IFEval - # eval": "{\"description\": \"min=541, mean=541, max=541, sum=541 (1)\", \"tab\": \"General information\", \"score\": \"541.0\"}", - "IFEval - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - # prompt tokens": "{\"description\": \"min=46.492, mean=46.492, max=46.492, sum=46.492 (1)\", \"tab\": \"General information\", \"score\": \"46.491682070240294\"}", - "IFEval - # output tokens": "{\"description\": \"min=427.54, mean=427.54, max=427.54, sum=427.54 (1)\", \"tab\": \"General information\", \"score\": \"427.53974121996305\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WildBench", - "source_data": { - "dataset_name": "WildBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "WB Score on WildBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.866, - "details": { - "description": "min=0.866, mean=0.866, max=0.866, sum=0.866 (1)", - "tab": "Accuracy", - "WildBench - Observed inference time (s)": "{\"description\": \"min=61.017, mean=61.017, max=61.017, sum=61.017 (1)\", \"tab\": \"Efficiency\", \"score\": \"61.01670853805542\"}", - "WildBench - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "WildBench - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # prompt tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # output tokens": "{\"description\": \"min=1976.28, mean=1976.28, max=1976.28, sum=1976.28 (1)\", \"tab\": \"General information\", \"score\": \"1976.28\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"v2\"" - } - } - }, - { - "evaluation_name": "Omni-MATH", - "source_data": { - "dataset_name": "Omni-MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Acc on Omni-MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.718, - "details": { - "description": "min=0.718, mean=0.718, max=0.718, sum=0.718 (1)", - "tab": "Accuracy", - "Omni-MATH - Observed inference time (s)": "{\"description\": \"min=228.578, mean=228.578, max=228.578, sum=228.578 (1)\", \"tab\": \"Efficiency\", \"score\": \"228.57833194971084\"}", - "Omni-MATH - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "Omni-MATH - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - # prompt tokens": "{\"description\": \"min=111.6, mean=111.6, max=111.6, sum=111.6 (1)\", \"tab\": \"General information\", \"score\": \"111.6\"}", - "Omni-MATH - # output tokens": "{\"description\": \"min=5629.583, mean=5629.583, max=5629.583, sum=5629.583 (1)\", \"tab\": \"General information\", \"score\": \"5629.583\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/qwen_qwen3-235b-a22b-instruct-2507-prompt.json b/data/models/qwen_qwen3-235b-a22b-instruct-2507-prompt.json deleted file mode 100644 index db34ebf53892c29c9233ba4c4b1ae9f1f6ce5f71..0000000000000000000000000000000000000000 --- a/data/models/qwen_qwen3-235b-a22b-instruct-2507-prompt.json +++ /dev/null @@ -1,905 +0,0 @@ -{ - "model_info": { - "name": "Qwen3-235B-A22B-Instruct-2507 (Prompt)", - "id": "qwen/qwen3-235b-a22b-instruct-2507-prompt", - "developer": "qwen", - "additional_details": { - "raw_model_name": "Qwen3-235B-A22B-Instruct-2507 (Prompt)", - "organization": "Qwen", - "license": "apache-2.0", - "mode": "Prompt", - "model_link": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Instruct-2507" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/qwen/qwen3-235b-a22b-instruct-2507-prompt/1775236112.377265", - "retrieved_timestamp": "1775236112.377265", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 23.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 52.15 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 3.12 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 2.56 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 2.75 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 7.61 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 90.33 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 79.83 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 95.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 95.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 91.0 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 78.68 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 82.95 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 77.78 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 81.25 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 70.83 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 44.62 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 54.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 42.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 31.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 50.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 50.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 56.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 45.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 19.35 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 12.9 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 11.61 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 33.55 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 93.75 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 78.89 - } - }, - { - "evaluation_result_id": "bfcl.format_sensitivity.max_delta::max_delta", - "evaluation_name": "bfcl.format_sensitivity.max_delta", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.format_sensitivity.max_delta", - "metric_name": "Format sensitivity max delta", - "metric_kind": "difference", - "metric_unit": "percentage_points", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Format Sensitivity Max Delta" - } - }, - "score_details": { - "score": 8.0 - } - }, - { - "evaluation_result_id": "bfcl.format_sensitivity.stddev::stddev", - "evaluation_name": "bfcl.format_sensitivity.stddev", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.format_sensitivity.stddev", - "metric_name": "Format sensitivity standard deviation", - "metric_kind": "difference", - "metric_unit": "percentage_points", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Format Sensitivity Standard Deviation" - } - }, - "score_details": { - "score": 1.95 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qwen_qwen3-30b-a3b-instruct-2507-fc.json b/data/models/qwen_qwen3-30b-a3b-instruct-2507-fc.json deleted file mode 100644 index 5d4c80b29a34d8bf3bf91bd83dfbdb450ff07e78..0000000000000000000000000000000000000000 --- a/data/models/qwen_qwen3-30b-a3b-instruct-2507-fc.json +++ /dev/null @@ -1,851 +0,0 @@ -{ - "model_info": { - "name": "Qwen3-30B-A3B-Instruct-2507 (FC)", - "id": "qwen/qwen3-30b-a3b-instruct-2507-fc", - "developer": "qwen", - "additional_details": { - "raw_model_name": "Qwen3-30B-A3B-Instruct-2507 (FC)", - "organization": "Qwen", - "license": "apache-2.0", - "mode": "FC", - "model_link": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/qwen/qwen3-30b-a3b-instruct-2507-fc/1775236112.3863301", - "retrieved_timestamp": "1775236112.3863301", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 41.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 41.39 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 5.62 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 5.95 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 25.48 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 12.7 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 85.77 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 68.58 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 94.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 91.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 88.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 77.94 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 83.33 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 76.83 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 68.75 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 75.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 30.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 43.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 10.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 25.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 41.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 22.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 21.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 24.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 17.63 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 9.03 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 9.03 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 34.84 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 81.25 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 79.9 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qwen_qwen3-30b-a3b-instruct-2507-prompt.json b/data/models/qwen_qwen3-30b-a3b-instruct-2507-prompt.json deleted file mode 100644 index 9df6b25dfe991816c6a7b4444d72958d3c6fd1f2..0000000000000000000000000000000000000000 --- a/data/models/qwen_qwen3-30b-a3b-instruct-2507-prompt.json +++ /dev/null @@ -1,905 +0,0 @@ -{ - "model_info": { - "name": "Qwen3-30B-A3B-Instruct-2507 (Prompt)", - "id": "qwen/qwen3-30b-a3b-instruct-2507-prompt", - "developer": "qwen", - "additional_details": { - "raw_model_name": "Qwen3-30B-A3B-Instruct-2507 (Prompt)", - "organization": "Qwen", - "license": "apache-2.0", - "mode": "Prompt", - "model_link": "https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/qwen/qwen3-30b-a3b-instruct-2507-prompt/1775236112.393427", - "retrieved_timestamp": "1775236112.393427", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 53.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 36.7 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 1.56 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 1.24 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 7.9 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 2.84 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 88.92 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 80.67 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 93.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 94.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 88.0 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 78.39 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 82.56 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 77.49 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 87.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 66.67 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 23.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 33.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 16.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 16.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 29.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 17.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 15.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 20.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 9.68 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 5.81 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 6.45 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 16.77 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 93.75 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 74.85 - } - }, - { - "evaluation_result_id": "bfcl.format_sensitivity.max_delta::max_delta", - "evaluation_name": "bfcl.format_sensitivity.max_delta", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.format_sensitivity.max_delta", - "metric_name": "Format sensitivity max delta", - "metric_kind": "difference", - "metric_unit": "percentage_points", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Format Sensitivity Max Delta" - } - }, - "score_details": { - "score": 16.0 - } - }, - { - "evaluation_result_id": "bfcl.format_sensitivity.stddev::stddev", - "evaluation_name": "bfcl.format_sensitivity.stddev", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.format_sensitivity.stddev", - "metric_name": "Format sensitivity standard deviation", - "metric_kind": "difference", - "metric_unit": "percentage_points", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Format Sensitivity Standard Deviation" - } - }, - "score_details": { - "score": 4.13 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qwen_qwen3-32b-fc.json b/data/models/qwen_qwen3-32b-fc.json deleted file mode 100644 index 9fa02db21469dceaaabf51329debd8c42c8609a2..0000000000000000000000000000000000000000 --- a/data/models/qwen_qwen3-32b-fc.json +++ /dev/null @@ -1,851 +0,0 @@ -{ - "model_info": { - "name": "Qwen3-32B (FC)", - "id": "qwen/qwen3-32b-fc", - "developer": "qwen", - "additional_details": { - "raw_model_name": "Qwen3-32B (FC)", - "organization": "Qwen", - "license": "apache-2.0", - "mode": "FC", - "model_link": "https://huggingface.co/Qwen/Qwen3-32B" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/qwen/qwen3-32b-fc/1775236112.380377", - "retrieved_timestamp": "1775236112.380377", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 29.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 48.71 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 153.08 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 169.87 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 164.27 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 473.49 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 88.77 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 75.58 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 94.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 93.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 91.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 82.01 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 89.53 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 80.91 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 81.25 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 50.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 47.87 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 56.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 52.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 40.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 43.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 21.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 25.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 18.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 26.67 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 12.26 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 25.81 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 41.94 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 93.75 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 76.37 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qwen_qwen3-32b-prompt.json b/data/models/qwen_qwen3-32b-prompt.json deleted file mode 100644 index 9a7f2ab108ea380e5d7b169435ac5fc91b7a9bfc..0000000000000000000000000000000000000000 --- a/data/models/qwen_qwen3-32b-prompt.json +++ /dev/null @@ -1,905 +0,0 @@ -{ - "model_info": { - "name": "Qwen3-32B (Prompt)", - "id": "qwen/qwen3-32b-prompt", - "developer": "qwen", - "additional_details": { - "raw_model_name": "Qwen3-32B (Prompt)", - "organization": "Qwen", - "license": "apache-2.0", - "mode": "Prompt", - "model_link": "https://huggingface.co/Qwen/Qwen3-32B" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/qwen/qwen3-32b-prompt/1775236112.3822641", - "retrieved_timestamp": "1775236112.3822641", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 33.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 46.78 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 199.47 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 167.54 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 160.5 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 457.87 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 90.27 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 79.08 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 97.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 93.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 91.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 82.01 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 87.21 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 81.2 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 81.25 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 62.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 43.25 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 54.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 46.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 36.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 36.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 26.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 34.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 18.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 15.7 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 13.55 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 14.19 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 19.35 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 81.25 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 82.39 - } - }, - { - "evaluation_result_id": "bfcl.format_sensitivity.max_delta::max_delta", - "evaluation_name": "bfcl.format_sensitivity.max_delta", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.format_sensitivity.max_delta", - "metric_name": "Format sensitivity max delta", - "metric_kind": "difference", - "metric_unit": "percentage_points", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Format Sensitivity Max Delta" - } - }, - "score_details": { - "score": 15.5 - } - }, - { - "evaluation_result_id": "bfcl.format_sensitivity.stddev::stddev", - "evaluation_name": "bfcl.format_sensitivity.stddev", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.format_sensitivity.stddev", - "metric_name": "Format sensitivity standard deviation", - "metric_kind": "difference", - "metric_unit": "percentage_points", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Format Sensitivity Standard Deviation" - } - }, - "score_details": { - "score": 3.75 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qwen_qwen3-4b-instruct-2507-fc.json b/data/models/qwen_qwen3-4b-instruct-2507-fc.json deleted file mode 100644 index 9a5dfc5aa49a0193eb6b00ed13ea386eb607597c..0000000000000000000000000000000000000000 --- a/data/models/qwen_qwen3-4b-instruct-2507-fc.json +++ /dev/null @@ -1,851 +0,0 @@ -{ - "model_info": { - "name": "Qwen3-4B-Instruct-2507 (FC)", - "id": "qwen/qwen3-4b-instruct-2507-fc", - "developer": "qwen", - "additional_details": { - "raw_model_name": "Qwen3-4B-Instruct-2507 (FC)", - "organization": "Qwen", - "license": "apache-2.0", - "mode": "FC", - "model_link": "https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/qwen/qwen3-4b-instruct-2507-fc/1775236112.393962", - "retrieved_timestamp": "1775236112.393962", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 54.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 35.68 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 6.37 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 7.61 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 20.36 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 49.18 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 87.88 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 75.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 93.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 92.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 90.0 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 76.39 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 79.07 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 76.16 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 62.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 66.67 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 22.12 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 26.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 21.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 15.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 25.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 3.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 4.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 2.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 17.63 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 16.13 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 12.26 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 24.52 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 87.5 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 84.93 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qwen_qwen3-4b-instruct-2507-prompt.json b/data/models/qwen_qwen3-4b-instruct-2507-prompt.json deleted file mode 100644 index f7e750c4d3a89dd9dd5f0363096bdd95f2e9a13d..0000000000000000000000000000000000000000 --- a/data/models/qwen_qwen3-4b-instruct-2507-prompt.json +++ /dev/null @@ -1,905 +0,0 @@ -{ - "model_info": { - "name": "Qwen3-4B-Instruct-2507 (Prompt)", - "id": "qwen/qwen3-4b-instruct-2507-prompt", - "developer": "qwen", - "additional_details": { - "raw_model_name": "Qwen3-4B-Instruct-2507 (Prompt)", - "organization": "Qwen", - "license": "apache-2.0", - "mode": "Prompt", - "model_link": "https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/qwen/qwen3-4b-instruct-2507-prompt/1775236112.39449", - "retrieved_timestamp": "1775236112.39449", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 55.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 35.52 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 53.66 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 44.7 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 163.79 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 208.06 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 86.44 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 77.25 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 91.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 88.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 89.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 74.69 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 77.91 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 74.17 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 81.25 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 58.33 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 20.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 24.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 21.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 16.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 20.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 4.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 4.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 5.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 23.87 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 12.9 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 14.19 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 44.52 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 87.5 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 75.87 - } - }, - { - "evaluation_result_id": "bfcl.format_sensitivity.max_delta::max_delta", - "evaluation_name": "bfcl.format_sensitivity.max_delta", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.format_sensitivity.max_delta", - "metric_name": "Format sensitivity max delta", - "metric_kind": "difference", - "metric_unit": "percentage_points", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Format Sensitivity Max Delta" - } - }, - "score_details": { - "score": 18.0 - } - }, - { - "evaluation_result_id": "bfcl.format_sensitivity.stddev::stddev", - "evaluation_name": "bfcl.format_sensitivity.stddev", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.format_sensitivity.stddev", - "metric_name": "Format sensitivity standard deviation", - "metric_kind": "difference", - "metric_unit": "percentage_points", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Format Sensitivity Standard Deviation" - } - }, - "score_details": { - "score": 5.22 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qwen_qwen3-8b-fc.json b/data/models/qwen_qwen3-8b-fc.json deleted file mode 100644 index b441a7006c6a499b7b516da1fd65da6813a5d0f4..0000000000000000000000000000000000000000 --- a/data/models/qwen_qwen3-8b-fc.json +++ /dev/null @@ -1,851 +0,0 @@ -{ - "model_info": { - "name": "Qwen3-8B (FC)", - "id": "qwen/qwen3-8b-fc", - "developer": "qwen", - "additional_details": { - "raw_model_name": "Qwen3-8B (FC)", - "organization": "Qwen", - "license": "apache-2.0", - "mode": "FC", - "model_link": "https://huggingface.co/Qwen/Qwen3-8B" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/qwen/qwen3-8b-fc/1775236112.385293", - "retrieved_timestamp": "1775236112.385293", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 39.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 42.57 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 43.32 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 51.36 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 76.14 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 188.98 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 87.58 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 72.83 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 96.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 92.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 89.0 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 80.53 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 84.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 79.68 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 75.0 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 79.17 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 41.75 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 50.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 42.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 40.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 34.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 12.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 15.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 9.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 14.62 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 5.16 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 7.1 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 31.61 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 93.75 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 79.07 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qwen_qwen3-8b-prompt.json b/data/models/qwen_qwen3-8b-prompt.json deleted file mode 100644 index cc63f1109c291a7d2ee72a8101d227b374e20f85..0000000000000000000000000000000000000000 --- a/data/models/qwen_qwen3-8b-prompt.json +++ /dev/null @@ -1,905 +0,0 @@ -{ - "model_info": { - "name": "Qwen3-8B (Prompt)", - "id": "qwen/qwen3-8b-prompt", - "developer": "qwen", - "additional_details": { - "raw_model_name": "Qwen3-8B (Prompt)", - "organization": "Qwen", - "license": "apache-2.0", - "mode": "Prompt", - "model_link": "https://huggingface.co/Qwen/Qwen3-8B" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/qwen/qwen3-8b-prompt/1775236112.388261", - "retrieved_timestamp": "1775236112.388261", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 44.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 40.43 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 63.95 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 54.17 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 79.9 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 194.15 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 88.56 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 75.25 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 95.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 94.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 89.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 80.09 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 84.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 78.92 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 93.75 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 75.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 33.38 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 41.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 38.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 27.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 26.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 13.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 19.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 8.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 13.12 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 3.87 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 10.32 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 25.16 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 75.0 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 82.27 - } - }, - { - "evaluation_result_id": "bfcl.format_sensitivity.max_delta::max_delta", - "evaluation_name": "bfcl.format_sensitivity.max_delta", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.format_sensitivity.max_delta", - "metric_name": "Format sensitivity max delta", - "metric_kind": "difference", - "metric_unit": "percentage_points", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Format Sensitivity Max Delta" - } - }, - "score_details": { - "score": 16.5 - } - }, - { - "evaluation_result_id": "bfcl.format_sensitivity.stddev::stddev", - "evaluation_name": "bfcl.format_sensitivity.stddev", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.format_sensitivity.stddev", - "metric_name": "Format sensitivity standard deviation", - "metric_kind": "difference", - "metric_unit": "percentage_points", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Format Sensitivity Standard Deviation" - } - }, - "score_details": { - "score": 5.09 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qwen_qwq-32b-preview.json b/data/models/qwen_qwq-32b-preview.json deleted file mode 100644 index 53199448273e471b85765c1f9945f603868927aa..0000000000000000000000000000000000000000 --- a/data/models/qwen_qwq-32b-preview.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "QwQ-32B-Preview", - "id": "Qwen/QwQ-32B-Preview", - "developer": "Qwen", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "32.764" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Qwen_QwQ-32B-Preview/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4035 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6691 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4494 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2819 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.411 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5678 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qwen_qwq-32b.json b/data/models/qwen_qwq-32b.json deleted file mode 100644 index e5897df217796c8b37dc8af0b1b6245845ce3d38..0000000000000000000000000000000000000000 --- a/data/models/qwen_qwq-32b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "QwQ-32B", - "id": "Qwen/QwQ-32B", - "developer": "Qwen", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "32.764" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Qwen_QwQ-32B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3977 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2983 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1609 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2601 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4206 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1196 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/qwen_worldpm-72b.json b/data/models/qwen_worldpm-72b.json deleted file mode 100644 index c9c230eb323b5403d57d5ded0786f47a648e87ad..0000000000000000000000000000000000000000 --- a/data/models/qwen_worldpm-72b.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "Qwen/WorldPM-72B", - "id": "Qwen/WorldPM-72B", - "developer": "Qwen", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/Qwen_WorldPM-72B/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6333 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7074 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3125 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6557 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8533 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9172 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3535 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/r-i-s-e_rise-judge-qwen2.5-32b.json b/data/models/r-i-s-e_rise-judge-qwen2.5-32b.json deleted file mode 100644 index 89a4a277f9bd2488fe38ec85c6e24739a599fdaf..0000000000000000000000000000000000000000 --- a/data/models/r-i-s-e_rise-judge-qwen2.5-32b.json +++ /dev/null @@ -1,126 +0,0 @@ -{ - "model_info": { - "name": "R-I-S-E/RISE-Judge-Qwen2.5-32B", - "id": "R-I-S-E/RISE-Judge-Qwen2.5-32B", - "developer": "R-I-S-E", - "additional_details": { - "model_type": "Generative" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/R-I-S-E_RISE-Judge-Qwen2.5-32B/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9266 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9665 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8333 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9189 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9877 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/r-i-s-e_rise-judge-qwen2.5-7b.json b/data/models/r-i-s-e_rise-judge-qwen2.5-7b.json deleted file mode 100644 index 1b5b9c2e1c593d67d23b8f2f70fb140b8b9ba691..0000000000000000000000000000000000000000 --- a/data/models/r-i-s-e_rise-judge-qwen2.5-7b.json +++ /dev/null @@ -1,126 +0,0 @@ -{ - "model_info": { - "name": "R-I-S-E/RISE-Judge-Qwen2.5-7B", - "id": "R-I-S-E/RISE-Judge-Qwen2.5-7B", - "developer": "R-I-S-E", - "additional_details": { - "model_type": "Generative" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/R-I-S-E_RISE-Judge-Qwen2.5-7B/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8819 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9218 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7654 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8797 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9608 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/rakuten_rakutenai-2.0-mini-instruct.json b/data/models/rakuten_rakutenai-2.0-mini-instruct.json deleted file mode 100644 index f62c5089fda719ec4fdf86bad6e89b427affb405..0000000000000000000000000000000000000000 --- a/data/models/rakuten_rakutenai-2.0-mini-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "RakutenAI-2.0-mini-instruct", - "id": "Rakuten/RakutenAI-2.0-mini-instruct", - "developer": "Rakuten", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "1.535" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Rakuten_RakutenAI-2.0-mini-instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6794 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2867 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0521 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2668 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3249 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1118 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/rakuten_rakutenai-7b-chat.json b/data/models/rakuten_rakutenai-7b-chat.json deleted file mode 100644 index eca29ba40308144eecfc039f0646ce8969ee12ad..0000000000000000000000000000000000000000 --- a/data/models/rakuten_rakutenai-7b-chat.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "RakutenAI-7B-chat", - "id": "Rakuten/RakutenAI-7B-chat", - "developer": "Rakuten", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "7.373" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Rakuten_RakutenAI-7B-chat/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2686 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4316 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0295 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2567 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.379 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2798 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/rakuten_rakutenai-7b.json b/data/models/rakuten_rakutenai-7b.json deleted file mode 100644 index 950abc98653e337cac4f6328aefa467f16d58588..0000000000000000000000000000000000000000 --- a/data/models/rakuten_rakutenai-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "RakutenAI-7B", - "id": "Rakuten/RakutenAI-7B", - "developer": "Rakuten", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "7.373" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Rakuten_RakutenAI-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1556 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4315 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0196 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2894 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3738 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2877 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/raphgg_test-2.5-72b.json b/data/models/raphgg_test-2.5-72b.json deleted file mode 100644 index 39d8208bae86455eb1605a65deef92120afc0902..0000000000000000000000000000000000000000 --- a/data/models/raphgg_test-2.5-72b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "test-2.5-72B", - "id": "raphgg/test-2.5-72B", - "developer": "raphgg", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "72.706" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/raphgg_test-2.5-72B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8437 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7266 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4109 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3893 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4812 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5837 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/rasyosef_mistral-nemo-minitron-8b-chat.json b/data/models/rasyosef_mistral-nemo-minitron-8b-chat.json deleted file mode 100644 index 255b698099fb4393adde90305a7ab7431547131c..0000000000000000000000000000000000000000 --- a/data/models/rasyosef_mistral-nemo-minitron-8b-chat.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mistral-NeMo-Minitron-8B-Chat", - "id": "rasyosef/Mistral-NeMo-Minitron-8B-Chat", - "developer": "rasyosef", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "8.414" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/rasyosef_Mistral-NeMo-Minitron-8B-Chat/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4452 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4759 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0272 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.276 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4304 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2404 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/rasyosef_phi-1_5-instruct-v0.1.json b/data/models/rasyosef_phi-1_5-instruct-v0.1.json deleted file mode 100644 index 53f5862916cce7b658db742a5b491bc498129b67..0000000000000000000000000000000000000000 --- a/data/models/rasyosef_phi-1_5-instruct-v0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Phi-1_5-Instruct-v0.1", - "id": "rasyosef/Phi-1_5-Instruct-v0.1", - "developer": "rasyosef", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "PhiForCausalLM", - "params_billions": "1.415" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/rasyosef_Phi-1_5-Instruct-v0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2402 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3118 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0136 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2601 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3422 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1562 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/rasyosef_phi-2-instruct-apo.json b/data/models/rasyosef_phi-2-instruct-apo.json deleted file mode 100644 index 52de252b7e091030b447cd58135b0260362178e4..0000000000000000000000000000000000000000 --- a/data/models/rasyosef_phi-2-instruct-apo.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "phi-2-instruct-apo", - "id": "rasyosef/phi-2-instruct-apo", - "developer": "rasyosef", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "PhiForCausalLM", - "params_billions": "2.775" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/rasyosef_phi-2-instruct-apo/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3146 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4445 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0302 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2701 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3342 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2155 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/rasyosef_phi-2-instruct-v0.1.json b/data/models/rasyosef_phi-2-instruct-v0.1.json deleted file mode 100644 index 98b01505807282afadac24aa8f434c4758198e12..0000000000000000000000000000000000000000 --- a/data/models/rasyosef_phi-2-instruct-v0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "phi-2-instruct-v0.1", - "id": "rasyosef/phi-2-instruct-v0.1", - "developer": "rasyosef", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "PhiForCausalLM", - "params_billions": "2.775" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/rasyosef_phi-2-instruct-v0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3681 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4726 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2743 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3524 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2247 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ray2333_gemma-2b-rewardmodel-baseline.json b/data/models/ray2333_gemma-2b-rewardmodel-baseline.json deleted file mode 100644 index 763101b5c9305dda3463ffbbb9465b3808bfc71d..0000000000000000000000000000000000000000 --- a/data/models/ray2333_gemma-2b-rewardmodel-baseline.json +++ /dev/null @@ -1,144 +0,0 @@ -{ - "model_info": { - "name": "Ray2333/Gemma-2B-rewardmodel-baseline", - "id": "Ray2333/Gemma-2B-rewardmodel-baseline", - "developer": "Ray2333", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/Ray2333_Gemma-2B-rewardmodel-baseline/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.729 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9413 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4693 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7865 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7384 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6897 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ray2333_gemma-2b-rewardmodel-ft.json b/data/models/ray2333_gemma-2b-rewardmodel-ft.json deleted file mode 100644 index e2b4c7f9e2abd975f1d2367b9cf14f6b21a9ac31..0000000000000000000000000000000000000000 --- a/data/models/ray2333_gemma-2b-rewardmodel-ft.json +++ /dev/null @@ -1,126 +0,0 @@ -{ - "model_info": { - "name": "Ray2333/Gemma-2B-rewardmodel-ft", - "id": "Ray2333/Gemma-2B-rewardmodel-ft", - "developer": "Ray2333", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/Ray2333_Gemma-2B-rewardmodel-ft/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8048 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7793 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7478 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8527 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8393 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ray2333_grm-gemma-2b-rewardmodel-ft.json b/data/models/ray2333_grm-gemma-2b-rewardmodel-ft.json deleted file mode 100644 index adc47a5ec6aafeaebe914287b7d9dbc8e69c6e67..0000000000000000000000000000000000000000 --- a/data/models/ray2333_grm-gemma-2b-rewardmodel-ft.json +++ /dev/null @@ -1,126 +0,0 @@ -{ - "model_info": { - "name": "Ray2333/GRM-Gemma-2B-rewardmodel-ft", - "id": "Ray2333/GRM-Gemma-2B-rewardmodel-ft", - "developer": "Ray2333", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/Ray2333_GRM-Gemma-2B-rewardmodel-ft/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8447 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8939 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7522 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8446 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8881 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ray2333_grm-gemma-2b-sftreg.json b/data/models/ray2333_grm-gemma-2b-sftreg.json deleted file mode 100644 index f039db19a6dd84a846204ab18f5336df4dc3144f..0000000000000000000000000000000000000000 --- a/data/models/ray2333_grm-gemma-2b-sftreg.json +++ /dev/null @@ -1,144 +0,0 @@ -{ - "model_info": { - "name": "Ray2333/GRM-Gemma-2B-sftreg", - "id": "Ray2333/GRM-Gemma-2B-sftreg", - "developer": "Ray2333", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/Ray2333_GRM-Gemma-2B-sftreg/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7451 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9553 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4868 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7932 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7684 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6983 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ray2333_grm-gemma2-2b-rewardmodel-ft.json b/data/models/ray2333_grm-gemma2-2b-rewardmodel-ft.json deleted file mode 100644 index cd8835eaf989dbd622c51262fef9b72b10534d67..0000000000000000000000000000000000000000 --- a/data/models/ray2333_grm-gemma2-2b-rewardmodel-ft.json +++ /dev/null @@ -1,276 +0,0 @@ -{ - "model_info": { - "name": "Ray2333/GRM-gemma2-2B-rewardmodel-ft", - "id": "Ray2333/GRM-gemma2-2B-rewardmodel-ft", - "developer": "Ray2333", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/Ray2333_GRM-gemma2-2B-rewardmodel-ft/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8839 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9302 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7719 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9216 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.912 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "reward-bench-2/Ray2333_GRM-gemma2-2B-rewardmodel-ft/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5966 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5305 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3125 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5902 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9222 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7455 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4788 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ray2333_grm-llama3-8b-distill.json b/data/models/ray2333_grm-llama3-8b-distill.json deleted file mode 100644 index a698e2dc0120e472c95dbd375c2ce72c243acf20..0000000000000000000000000000000000000000 --- a/data/models/ray2333_grm-llama3-8b-distill.json +++ /dev/null @@ -1,294 +0,0 @@ -{ - "model_info": { - "name": "Ray2333/GRM-llama3-8B-distill", - "id": "Ray2333/GRM-llama3-8B-distill", - "developer": "Ray2333", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/Ray2333_GRM-llama3-8B-distill/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8464 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9832 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6842 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8676 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9133 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7209 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "reward-bench-2/Ray2333_GRM-llama3-8B-distill/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.589 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5874 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3875 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5902 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7222 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6727 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5743 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ray2333_grm-llama3-8b-rewardmodel-ft.json b/data/models/ray2333_grm-llama3-8b-rewardmodel-ft.json deleted file mode 100644 index 294d09da30917c4eff496ef892b03ff9d96db067..0000000000000000000000000000000000000000 --- a/data/models/ray2333_grm-llama3-8b-rewardmodel-ft.json +++ /dev/null @@ -1,276 +0,0 @@ -{ - "model_info": { - "name": "Ray2333/GRM-Llama3-8B-rewardmodel-ft", - "id": "Ray2333/GRM-Llama3-8B-rewardmodel-ft", - "developer": "Ray2333", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/Ray2333_GRM-Llama3-8B-rewardmodel-ft/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9154 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9553 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8618 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9081 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9362 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "reward-bench-2/Ray2333_GRM-Llama3-8B-rewardmodel-ft/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6766 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6274 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.35 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5847 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9222 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8929 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6824 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ray2333_grm-llama3-8b-sftreg.json b/data/models/ray2333_grm-llama3-8b-sftreg.json deleted file mode 100644 index bd70639489991e3e8880cad041284119bd35ecc9..0000000000000000000000000000000000000000 --- a/data/models/ray2333_grm-llama3-8b-sftreg.json +++ /dev/null @@ -1,294 +0,0 @@ -{ - "model_info": { - "name": "Ray2333/GRM-llama3-8B-sftreg", - "id": "Ray2333/GRM-llama3-8B-sftreg", - "developer": "Ray2333", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/Ray2333_GRM-llama3-8B-sftreg/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8542 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.986 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6776 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8919 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9229 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7309 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "reward-bench-2/Ray2333_GRM-llama3-8B-sftreg/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6089 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6189 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3875 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5792 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7867 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6828 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5981 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ray2333_grm-llama3.2-3b-rewardmodel-ft.json b/data/models/ray2333_grm-llama3.2-3b-rewardmodel-ft.json deleted file mode 100644 index dced2a020f8f796d8700845d8ad8b61d6ed658a2..0000000000000000000000000000000000000000 --- a/data/models/ray2333_grm-llama3.2-3b-rewardmodel-ft.json +++ /dev/null @@ -1,126 +0,0 @@ -{ - "model_info": { - "name": "Ray2333/GRM-llama3.2-3B-rewardmodel-ft", - "id": "Ray2333/GRM-llama3.2-3B-rewardmodel-ft", - "developer": "Ray2333", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/Ray2333_GRM-llama3.2-3B-rewardmodel-ft/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9092 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9162 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8487 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.927 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.945 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ray2333_reward-model-mistral-7b-instruct-unifie....json b/data/models/ray2333_reward-model-mistral-7b-instruct-unifie....json deleted file mode 100644 index 9c504d3374c1d342eab6a0b71d1ef1c6a1c95370..0000000000000000000000000000000000000000 --- a/data/models/ray2333_reward-model-mistral-7b-instruct-unifie....json +++ /dev/null @@ -1,144 +0,0 @@ -{ - "model_info": { - "name": "Ray2333/reward-model-Mistral-7B-instruct-Unifie...", - "id": "Ray2333/reward-model-Mistral-7B-instruct-Unifie...", - "developer": "Ray2333", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/Ray2333_reward-model-Mistral-7B-instruct-Unifie.../1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7661 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9777 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5066 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8527 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7389 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7434 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/rdson_wombocombo-r1-coder-14b-preview.json b/data/models/rdson_wombocombo-r1-coder-14b-preview.json deleted file mode 100644 index dce228fc65ef132c9e7cef0d5ad1433f5dcab092..0000000000000000000000000000000000000000 --- a/data/models/rdson_wombocombo-r1-coder-14b-preview.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "WomboCombo-R1-Coder-14B-Preview", - "id": "RDson/WomboCombo-R1-Coder-14B-Preview", - "developer": "RDson", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/RDson_WomboCombo-R1-Coder-14B-Preview/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6286 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6392 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5989 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3213 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4844 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5168 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/realtreetune_rho-1b-sft-math.json b/data/models/realtreetune_rho-1b-sft-math.json deleted file mode 100644 index 49e0916a8d9b32b8903bf8a71a97f966b5f92241..0000000000000000000000000000000000000000 --- a/data/models/realtreetune_rho-1b-sft-math.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "rho-1b-sft-MATH", - "id": "realtreetune/rho-1b-sft-MATH", - "developer": "realtreetune", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.1" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/realtreetune_rho-1b-sft-MATH/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2121 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3144 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0347 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2525 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3458 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1117 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/recoilme_gemma-2-ataraxy-gemmasutra-9b-slerp.json b/data/models/recoilme_gemma-2-ataraxy-gemmasutra-9b-slerp.json deleted file mode 100644 index 07191bcaa576a8b8b67650d88a341b91a6316705..0000000000000000000000000000000000000000 --- a/data/models/recoilme_gemma-2-ataraxy-gemmasutra-9b-slerp.json +++ /dev/null @@ -1,275 +0,0 @@ -{ - "model_info": { - "name": "Gemma-2-Ataraxy-Gemmasutra-9B-slerp", - "id": "recoilme/Gemma-2-Ataraxy-Gemmasutra-9B-slerp", - "developer": "recoilme", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/recoilme_Gemma-2-Ataraxy-Gemmasutra-9B-slerp/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2854 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5984 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1005 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3297 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4607 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4162 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "hfopenllm_v2/recoilme_Gemma-2-Ataraxy-Gemmasutra-9B-slerp/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7649 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5974 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0174 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3305 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4245 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4207 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/recoilme_recoilme-gemma-2-9b-v0.1.json b/data/models/recoilme_recoilme-gemma-2-9b-v0.1.json deleted file mode 100644 index 03a13ed1c237a2c3d6569de27d26b96ad59316ed..0000000000000000000000000000000000000000 --- a/data/models/recoilme_recoilme-gemma-2-9b-v0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "recoilme-gemma-2-9B-v0.1", - "id": "recoilme/recoilme-gemma-2-9B-v0.1", - "developer": "recoilme", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/recoilme_recoilme-gemma-2-9B-v0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7515 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5995 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2039 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3389 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4191 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4159 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/recoilme_recoilme-gemma-2-9b-v0.2.json b/data/models/recoilme_recoilme-gemma-2-9b-v0.2.json deleted file mode 100644 index 83bfdb05cda442c3bac351d424fa26cf0ae864fe..0000000000000000000000000000000000000000 --- a/data/models/recoilme_recoilme-gemma-2-9b-v0.2.json +++ /dev/null @@ -1,275 +0,0 @@ -{ - "model_info": { - "name": "recoilme-gemma-2-9B-v0.2", - "id": "recoilme/recoilme-gemma-2-9B-v0.2", - "developer": "recoilme", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/recoilme_recoilme-gemma-2-9B-v0.2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7592 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6026 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0529 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3289 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4099 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4163 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "hfopenllm_v2/recoilme_recoilme-gemma-2-9B-v0.2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2747 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6031 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0831 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3305 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4686 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4122 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/recoilme_recoilme-gemma-2-9b-v0.3.json b/data/models/recoilme_recoilme-gemma-2-9b-v0.3.json deleted file mode 100644 index 812871dcfb75c54e3765d2c8d03eeadc60d898aa..0000000000000000000000000000000000000000 --- a/data/models/recoilme_recoilme-gemma-2-9b-v0.3.json +++ /dev/null @@ -1,275 +0,0 @@ -{ - "model_info": { - "name": "recoilme-gemma-2-9B-v0.3", - "id": "recoilme/recoilme-gemma-2-9B-v0.3", - "developer": "recoilme", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/recoilme_recoilme-gemma-2-9B-v0.3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7439 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5993 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0876 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3238 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4204 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4072 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "hfopenllm_v2/recoilme_recoilme-gemma-2-9B-v0.3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5761 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.602 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1888 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3372 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4632 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4039 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/recoilme_recoilme-gemma-2-9b-v0.4.json b/data/models/recoilme_recoilme-gemma-2-9b-v0.4.json deleted file mode 100644 index c2093aef2a382dbfc09438aedb1220cf58147b5d..0000000000000000000000000000000000000000 --- a/data/models/recoilme_recoilme-gemma-2-9b-v0.4.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "recoilme-gemma-2-9B-v0.4", - "id": "recoilme/recoilme-gemma-2-9B-v0.4", - "developer": "recoilme", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/recoilme_recoilme-gemma-2-9B-v0.4/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2562 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5967 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0846 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3406 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4727 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4406 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/recoilme_recoilme-gemma-2-9b-v0.5.json b/data/models/recoilme_recoilme-gemma-2-9b-v0.5.json deleted file mode 100644 index 532d4312c0bb1aedda7b67f4da6cddf376ef1c9d..0000000000000000000000000000000000000000 --- a/data/models/recoilme_recoilme-gemma-2-9b-v0.5.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "recoilme-gemma-2-9B-v0.5", - "id": "recoilme/recoilme-gemma-2-9B-v0.5", - "developer": "recoilme", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/recoilme_recoilme-gemma-2-9B-v0.5/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7664 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5981 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2115 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3364 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4232 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.42 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/redrix_angelslayer-12b-unslop-mell-rpmax-darkness.json b/data/models/redrix_angelslayer-12b-unslop-mell-rpmax-darkness.json deleted file mode 100644 index 2f52cf0a0e1d59ba981bf4ce2d5c23f03ff1eeec..0000000000000000000000000000000000000000 --- a/data/models/redrix_angelslayer-12b-unslop-mell-rpmax-darkness.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "AngelSlayer-12B-Unslop-Mell-RPMax-DARKNESS", - "id": "redrix/AngelSlayer-12B-Unslop-Mell-RPMax-DARKNESS", - "developer": "redrix", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/redrix_AngelSlayer-12B-Unslop-Mell-RPMax-DARKNESS/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.536 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5129 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1133 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3154 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3818 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.318 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/redrix_patricide-12b-unslop-mell.json b/data/models/redrix_patricide-12b-unslop-mell.json deleted file mode 100644 index 436462cfe90ba5593dc29099fd8a7d059b4879bf..0000000000000000000000000000000000000000 --- a/data/models/redrix_patricide-12b-unslop-mell.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "patricide-12B-Unslop-Mell", - "id": "redrix/patricide-12B-Unslop-Mell", - "developer": "redrix", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/redrix_patricide-12B-Unslop-Mell/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4074 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5399 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1314 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3238 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4026 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.357 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/refuelai_llama-3-refueled.json b/data/models/refuelai_llama-3-refueled.json deleted file mode 100644 index a63b6c34835786c411ef10156c9c4c1fa0ece576..0000000000000000000000000000000000000000 --- a/data/models/refuelai_llama-3-refueled.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-Refueled", - "id": "refuelai/Llama-3-Refueled", - "developer": "refuelai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/refuelai_Llama-3-Refueled/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.462 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5871 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0665 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2995 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4454 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3095 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/replete-ai_l3-pneuma-8b.json b/data/models/replete-ai_l3-pneuma-8b.json deleted file mode 100644 index 5cd9cd427dd60418f6ec1df4443b22132cc35fd2..0000000000000000000000000000000000000000 --- a/data/models/replete-ai_l3-pneuma-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "L3-Pneuma-8B", - "id": "Replete-AI/L3-Pneuma-8B", - "developer": "Replete-AI", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Replete-AI_L3-Pneuma-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2413 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4909 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0544 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.318 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4105 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3176 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/replete-ai_l3.1-pneuma-8b.json b/data/models/replete-ai_l3.1-pneuma-8b.json deleted file mode 100644 index de0f3a80464714cd63440fbb14859bccd2f4fd36..0000000000000000000000000000000000000000 --- a/data/models/replete-ai_l3.1-pneuma-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "L3.1-Pneuma-8B", - "id": "Replete-AI/L3.1-Pneuma-8B", - "developer": "Replete-AI", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Replete-AI_L3.1-Pneuma-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7076 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.505 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2198 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3029 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3871 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3691 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/replete-ai_llama3-8b-instruct-replete-adapted.json b/data/models/replete-ai_llama3-8b-instruct-replete-adapted.json deleted file mode 100644 index 551664096ccc13079d79580ad642d2c2d337cd11..0000000000000000000000000000000000000000 --- a/data/models/replete-ai_llama3-8b-instruct-replete-adapted.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama3-8B-Instruct-Replete-Adapted", - "id": "Replete-AI/Llama3-8B-Instruct-Replete-Adapted", - "developer": "Replete-AI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Replete-AI_Llama3-8B-Instruct-Replete-Adapted/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6915 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.487 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.071 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.281 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3634 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3391 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/replete-ai_replete-coder-instruct-8b-merged.json b/data/models/replete-ai_replete-coder-instruct-8b-merged.json deleted file mode 100644 index e2f99670647d4f5f12640b1e8648a024b9e70bed..0000000000000000000000000000000000000000 --- a/data/models/replete-ai_replete-coder-instruct-8b-merged.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Replete-Coder-Instruct-8b-Merged", - "id": "Replete-AI/Replete-Coder-Instruct-8b-Merged", - "developer": "Replete-AI", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Replete-AI_Replete-Coder-Instruct-8b-Merged/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5388 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4462 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0778 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2693 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.366 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1805 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/replete-ai_replete-coder-llama3-8b.json b/data/models/replete-ai_replete-coder-llama3-8b.json deleted file mode 100644 index 32d30ecd9f4d4822a5f1107ad4f97f498e45b0d7..0000000000000000000000000000000000000000 --- a/data/models/replete-ai_replete-coder-llama3-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Replete-Coder-Llama3-8B", - "id": "Replete-AI/Replete-Coder-Llama3-8B", - "developer": "Replete-AI", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Replete-AI_Replete-Coder-Llama3-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4729 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3271 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0476 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2609 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3953 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1331 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/replete-ai_replete-coder-qwen2-1.5b.json b/data/models/replete-ai_replete-coder-qwen2-1.5b.json deleted file mode 100644 index 337b487fd4092d7bd6ddc9a85b28a6eaddc7bd92..0000000000000000000000000000000000000000 --- a/data/models/replete-ai_replete-coder-qwen2-1.5b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Replete-Coder-Qwen2-1.5b", - "id": "Replete-AI/Replete-Coder-Qwen2-1.5b", - "developer": "Replete-AI", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.544" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Replete-AI_Replete-Coder-Qwen2-1.5b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3014 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3475 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0385 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2685 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4073 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2147 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/replete-ai_replete-llm-qwen2-7b.json b/data/models/replete-ai_replete-llm-qwen2-7b.json deleted file mode 100644 index 627d67572ee0cd0135e173767115d5ee7360b5e6..0000000000000000000000000000000000000000 --- a/data/models/replete-ai_replete-llm-qwen2-7b.json +++ /dev/null @@ -1,275 +0,0 @@ -{ - "model_info": { - "name": "Replete-LLM-Qwen2-7b", - "id": "Replete-AI/Replete-LLM-Qwen2-7b", - "developer": "Replete-AI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Replete-AI_Replete-LLM-Qwen2-7b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0905 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2985 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2534 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3848 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1158 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "hfopenllm_v2/Replete-AI_Replete-LLM-Qwen2-7b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0932 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2977 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2475 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3941 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1157 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/replete-ai_replete-llm-qwen2-7b_beta-preview.json b/data/models/replete-ai_replete-llm-qwen2-7b_beta-preview.json deleted file mode 100644 index e43756a71432d6a3723f90b4828cc7c3349d7f79..0000000000000000000000000000000000000000 --- a/data/models/replete-ai_replete-llm-qwen2-7b_beta-preview.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Replete-LLM-Qwen2-7b_Beta-Preview", - "id": "Replete-AI/Replete-LLM-Qwen2-7b_Beta-Preview", - "developer": "Replete-AI", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Replete-AI_Replete-LLM-Qwen2-7b_Beta-Preview/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0858 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2929 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2483 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3981 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1285 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/replete-ai_replete-llm-v2-llama-3.1-8b.json b/data/models/replete-ai_replete-llm-v2-llama-3.1-8b.json deleted file mode 100644 index 5b8d300a551b4b861f8d5eb48070097bb7f3a90b..0000000000000000000000000000000000000000 --- a/data/models/replete-ai_replete-llm-v2-llama-3.1-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Replete-LLM-V2-Llama-3.1-8b", - "id": "Replete-AI/Replete-LLM-V2-Llama-3.1-8b", - "developer": "Replete-AI", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Replete-AI_Replete-LLM-V2-Llama-3.1-8b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5515 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5339 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1405 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3138 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4001 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3753 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/resmpdev_eva-qwen2.5-1.5b-frfr.json b/data/models/resmpdev_eva-qwen2.5-1.5b-frfr.json deleted file mode 100644 index 9afe1abdf28304ab3a059d9ce1e34e47744449d8..0000000000000000000000000000000000000000 --- a/data/models/resmpdev_eva-qwen2.5-1.5b-frfr.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "EVA-Qwen2.5-1.5B-FRFR", - "id": "RESMPDEV/EVA-Qwen2.5-1.5B-FRFR", - "developer": "RESMPDEV", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.544" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/RESMPDEV_EVA-Qwen2.5-1.5B-FRFR/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3082 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3932 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1027 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2794 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3539 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.277 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/resmpdev_qwen2-wukong-0.5b.json b/data/models/resmpdev_qwen2-wukong-0.5b.json deleted file mode 100644 index 1749380534c391a119034b49146f9d7818a1ba6e..0000000000000000000000000000000000000000 --- a/data/models/resmpdev_qwen2-wukong-0.5b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2-Wukong-0.5B", - "id": "RESMPDEV/Qwen2-Wukong-0.5B", - "developer": "RESMPDEV", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/RESMPDEV_Qwen2-Wukong-0.5B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1854 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3085 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0015 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2366 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3525 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1327 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/rezvortex_jajuka-3b.json b/data/models/rezvortex_jajuka-3b.json deleted file mode 100644 index 14ed38701d2739dedbb7b3f2b330c4aa8f187426..0000000000000000000000000000000000000000 --- a/data/models/rezvortex_jajuka-3b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Jajuka-3b", - "id": "RezVortex/Jajuka-3b", - "developer": "RezVortex", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/RezVortex_Jajuka-3b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6925 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4594 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1594 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2659 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3671 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3137 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/rezvortex_jajuka-wewillneverforgetyou-3b.json b/data/models/rezvortex_jajuka-wewillneverforgetyou-3b.json deleted file mode 100644 index 46693fe1aa8ff9ffc64396440aca8522dc520cbf..0000000000000000000000000000000000000000 --- a/data/models/rezvortex_jajuka-wewillneverforgetyou-3b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "JAJUKA-WEWILLNEVERFORGETYOU-3B", - "id": "RezVortex/JAJUKA-WEWILLNEVERFORGETYOU-3B", - "developer": "RezVortex", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/RezVortex_JAJUKA-WEWILLNEVERFORGETYOU-3B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6858 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4619 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1548 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2576 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.363 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3143 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/rhplus0831_maid-yuzu-v7.json b/data/models/rhplus0831_maid-yuzu-v7.json deleted file mode 100644 index d22fe9ddd0fe93437242ac4fd78b1fe28601069a..0000000000000000000000000000000000000000 --- a/data/models/rhplus0831_maid-yuzu-v7.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "maid-yuzu-v7", - "id": "rhplus0831/maid-yuzu-v7", - "developer": "rhplus0831", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MixtralForCausalLM", - "params_billions": "46.703" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/rhplus0831_maid-yuzu-v7/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6462 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4805 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.102 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3096 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4136 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.354 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/rhymes-ai_aria.json b/data/models/rhymes-ai_aria.json deleted file mode 100644 index 8c26cf3f0d2b9e32c39489bd527b2e95f0da9be3..0000000000000000000000000000000000000000 --- a/data/models/rhymes-ai_aria.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Aria", - "id": "rhymes-ai/Aria", - "developer": "rhymes-ai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "AriaForConditionalGeneration", - "params_billions": "25.307" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/rhymes-ai_Aria/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4773 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5695 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1934 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3624 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4338 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4405 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/rhysjones_phi-2-orange-v2.json b/data/models/rhysjones_phi-2-orange-v2.json deleted file mode 100644 index 8c65c448d081d5556d27b06e58257535f3130c73..0000000000000000000000000000000000000000 --- a/data/models/rhysjones_phi-2-orange-v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "phi-2-orange-v2", - "id": "rhysjones/phi-2-orange-v2", - "developer": "rhysjones", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "PhiForCausalLM", - "params_billions": "2.78" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/rhysjones_phi-2-orange-v2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.367 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.477 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0408 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2617 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.363 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2532 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/riaz_finellama-3.1-8b.json b/data/models/riaz_finellama-3.1-8b.json deleted file mode 100644 index 554fa555a5c7288aadab0b51655132daffa7eed6..0000000000000000000000000000000000000000 --- a/data/models/riaz_finellama-3.1-8b.json +++ /dev/null @@ -1,275 +0,0 @@ -{ - "model_info": { - "name": "FineLlama-3.1-8B", - "id": "riaz/FineLlama-3.1-8B", - "developer": "riaz", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/riaz_FineLlama-3.1-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4373 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4586 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0514 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2752 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3763 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2964 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "hfopenllm_v2/riaz_FineLlama-3.1-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4137 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4565 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0453 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.276 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3776 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2978 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/rlhflow_armorm-llama3-8b-v0.1.json b/data/models/rlhflow_armorm-llama3-8b-v0.1.json deleted file mode 100644 index 3a9e218bfd75a10162dece0899d13277813d2910..0000000000000000000000000000000000000000 --- a/data/models/rlhflow_armorm-llama3-8b-v0.1.json +++ /dev/null @@ -1,427 +0,0 @@ -{ - "model_info": { - "name": "RLHFlow/ArmoRM-Llama3-8B-v0.1", - "id": "RLHFlow/ArmoRM-Llama3-8B-v0.1", - "developer": "RLHFlow", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForRewardModelWithGating", - "params_billions": "7.511" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/RLHFlow_ArmoRM-Llama3-8B-v0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1897 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2876 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2492 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3948 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1078 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "reward-bench-2/RLHFlow_ArmoRM-Llama3-8B-v0.1/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6646 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6568 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4188 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6612 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8222 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7657 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6629 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "reward-bench/RLHFlow_ArmoRM-Llama3-8B-v0.1/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.886 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9693 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7675 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9054 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9735 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7429 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/rlhflow_llama3-iterative-dpo-final.json b/data/models/rlhflow_llama3-iterative-dpo-final.json deleted file mode 100644 index 1a72e4982b15420fc05c85ea2973c8b11572db56..0000000000000000000000000000000000000000 --- a/data/models/rlhflow_llama3-iterative-dpo-final.json +++ /dev/null @@ -1,277 +0,0 @@ -{ - "model_info": { - "name": "RLHFlow/LLaMA3-iterative-DPO-final", - "id": "RLHFlow/LLaMA3-iterative-DPO-final", - "developer": "RLHFlow", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/RLHFlow_LLaMA3-iterative-DPO-final/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.534 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5058 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0884 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2836 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3673 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3257 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "reward-bench/RLHFlow_LLaMA3-iterative-DPO-final/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6783 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.838 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5921 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7865 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6161 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4392 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/rlhflow_pair-preference-model-llama3-8b.json b/data/models/rlhflow_pair-preference-model-llama3-8b.json deleted file mode 100644 index a47448abfb24a012de02a7dbda908bee87835d77..0000000000000000000000000000000000000000 --- a/data/models/rlhflow_pair-preference-model-llama3-8b.json +++ /dev/null @@ -1,144 +0,0 @@ -{ - "model_info": { - "name": "RLHFlow/pair-preference-model-LLaMA3-8B", - "id": "RLHFlow/pair-preference-model-LLaMA3-8B", - "developer": "RLHFlow", - "additional_details": { - "model_type": "Custom Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/RLHFlow_pair-preference-model-LLaMA3-8B/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8575 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9832 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6579 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8973 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9473 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7458 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/rlhflow_rewardmodel-mistral-7b-for-dpa-v1.json b/data/models/rlhflow_rewardmodel-mistral-7b-for-dpa-v1.json deleted file mode 100644 index 850a125255563c1c59aa5f59e7666709f2c4d8ed..0000000000000000000000000000000000000000 --- a/data/models/rlhflow_rewardmodel-mistral-7b-for-dpa-v1.json +++ /dev/null @@ -1,144 +0,0 @@ -{ - "model_info": { - "name": "RLHFlow/RewardModel-Mistral-7B-for-DPA-v1", - "id": "RLHFlow/RewardModel-Mistral-7B-for-DPA-v1", - "developer": "RLHFlow", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/RLHFlow_RewardModel-Mistral-7B-for-DPA-v1/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6633 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8799 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4978 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7068 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5971 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6068 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/rmdhirr_gluon-8b.json b/data/models/rmdhirr_gluon-8b.json deleted file mode 100644 index 12e1f597069f2cff78672b94558634f275b488b8..0000000000000000000000000000000000000000 --- a/data/models/rmdhirr_gluon-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Gluon-8B", - "id": "rmdhirr/Gluon-8B", - "developer": "rmdhirr", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/rmdhirr_Gluon-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5053 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5153 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1443 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3121 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4039 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3808 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ro-xe_fmixia-7b-dare-0.json b/data/models/ro-xe_fmixia-7b-dare-0.json deleted file mode 100644 index 1cc1a2ab953ca013ba9187c73e94dc52fd6bc369..0000000000000000000000000000000000000000 --- a/data/models/ro-xe_fmixia-7b-dare-0.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "FMixIA-7B-DARE-0", - "id": "Ro-xe/FMixIA-7B-DARE-0", - "developer": "Ro-xe", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Ro-xe_FMixIA-7B-DARE-0/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3341 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5035 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0529 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2894 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4545 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3016 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ro-xe_fmixia-7b-slerp-27.json b/data/models/ro-xe_fmixia-7b-slerp-27.json deleted file mode 100644 index 3cbfd9ea8972824df00279328cdd7467ede6f901..0000000000000000000000000000000000000000 --- a/data/models/ro-xe_fmixia-7b-slerp-27.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "FMixIA-7B-SLERP-27", - "id": "Ro-xe/FMixIA-7B-SLERP-27", - "developer": "Ro-xe", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Ro-xe_FMixIA-7B-SLERP-27/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3765 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5151 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0634 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2953 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4412 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3008 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ro-xe_fmixia-7b-ties-1.json b/data/models/ro-xe_fmixia-7b-ties-1.json deleted file mode 100644 index 6a4a993fab8d89a69d03307f7cebf0e1768cff80..0000000000000000000000000000000000000000 --- a/data/models/ro-xe_fmixia-7b-ties-1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "FMixIA-7B-TIES-1", - "id": "Ro-xe/FMixIA-7B-TIES-1", - "developer": "Ro-xe", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Ro-xe_FMixIA-7B-TIES-1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3453 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5092 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0566 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2886 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4689 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2992 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ro-xe_fmixia-frankenmerge-9.5b-pt-9.json b/data/models/ro-xe_fmixia-frankenmerge-9.5b-pt-9.json deleted file mode 100644 index 636f12a4b9f2e84720355b06e7d9d421dca44689..0000000000000000000000000000000000000000 --- a/data/models/ro-xe_fmixia-frankenmerge-9.5b-pt-9.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "FMixIA-FrankenMerge-9.5B-PT-9", - "id": "Ro-xe/FMixIA-FrankenMerge-9.5B-PT-9", - "developer": "Ro-xe", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.141" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Ro-xe_FMixIA-FrankenMerge-9.5B-PT-9/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.194 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5088 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.003 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3079 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.417 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3657 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/rombo-org_rombo-llm-v2.5-qwen-7b.json b/data/models/rombo-org_rombo-llm-v2.5-qwen-7b.json deleted file mode 100644 index 806eca5edb58d6f84baaf00ce1b93a0682fced3c..0000000000000000000000000000000000000000 --- a/data/models/rombo-org_rombo-llm-v2.5-qwen-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Rombo-LLM-V2.5-Qwen-7b", - "id": "Rombo-Org/Rombo-LLM-V2.5-Qwen-7b", - "developer": "Rombo-Org", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Rombo-Org_Rombo-LLM-V2.5-Qwen-7b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7482 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.54 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5068 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3012 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.398 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4283 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/rombodawg_rombos-coder-v2.5-qwen-14b.json b/data/models/rombodawg_rombos-coder-v2.5-qwen-14b.json deleted file mode 100644 index dfb33efbd4751fe4c9efe55692869d25e87888cd..0000000000000000000000000000000000000000 --- a/data/models/rombodawg_rombos-coder-v2.5-qwen-14b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Rombos-Coder-V2.5-Qwen-14b", - "id": "rombodawg/Rombos-Coder-V2.5-Qwen-14b", - "developer": "rombodawg", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/rombodawg_Rombos-Coder-V2.5-Qwen-14b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7047 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6165 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3301 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3029 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3915 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3939 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/rombodawg_rombos-coder-v2.5-qwen-7b.json b/data/models/rombodawg_rombos-coder-v2.5-qwen-7b.json deleted file mode 100644 index 7a0240977ca9f727653ede7e35c22bba40db862b..0000000000000000000000000000000000000000 --- a/data/models/rombodawg_rombos-coder-v2.5-qwen-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Rombos-Coder-V2.5-Qwen-7b", - "id": "rombodawg/Rombos-Coder-V2.5-Qwen-7b", - "developer": "rombodawg", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/rombodawg_Rombos-Coder-V2.5-Qwen-7b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.621 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5077 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3338 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2836 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3979 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3398 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/rombodawg_rombos-llm-v2.5-qwen-0.5b.json b/data/models/rombodawg_rombos-llm-v2.5-qwen-0.5b.json deleted file mode 100644 index 2c1afdc92d32ee5436d0def378590c081f1f1bc3..0000000000000000000000000000000000000000 --- a/data/models/rombodawg_rombos-llm-v2.5-qwen-0.5b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Rombos-LLM-V2.5-Qwen-0.5b", - "id": "rombodawg/Rombos-LLM-V2.5-Qwen-0.5b", - "developer": "rombodawg", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/rombodawg_Rombos-LLM-V2.5-Qwen-0.5b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2847 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3294 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.068 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2668 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3236 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1866 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/rombodawg_rombos-llm-v2.5-qwen-1.5b.json b/data/models/rombodawg_rombos-llm-v2.5-qwen-1.5b.json deleted file mode 100644 index bee22f6052f6c38a87e550c34faf122b80ea3a48..0000000000000000000000000000000000000000 --- a/data/models/rombodawg_rombos-llm-v2.5-qwen-1.5b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Rombos-LLM-V2.5-Qwen-1.5b", - "id": "rombodawg/Rombos-LLM-V2.5-Qwen-1.5b", - "developer": "rombodawg", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.777" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/rombodawg_Rombos-LLM-V2.5-Qwen-1.5b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3402 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4257 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0853 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2886 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4186 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2922 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/rombodawg_rombos-llm-v2.5-qwen-14b.json b/data/models/rombodawg_rombos-llm-v2.5-qwen-14b.json deleted file mode 100644 index da4da3005a4c12514598f6593fc5817b76d32265..0000000000000000000000000000000000000000 --- a/data/models/rombodawg_rombos-llm-v2.5-qwen-14b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Rombos-LLM-V2.5-Qwen-14b", - "id": "rombodawg/Rombos-LLM-V2.5-Qwen-14b", - "developer": "rombodawg", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/rombodawg_Rombos-LLM-V2.5-Qwen-14b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.584 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6481 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4554 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3716 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4717 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5376 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/rombodawg_rombos-llm-v2.5-qwen-32b.json b/data/models/rombodawg_rombos-llm-v2.5-qwen-32b.json deleted file mode 100644 index 1b171f78fe361db6083af6fa448d417725178ec1..0000000000000000000000000000000000000000 --- a/data/models/rombodawg_rombos-llm-v2.5-qwen-32b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Rombos-LLM-V2.5-Qwen-32b", - "id": "rombodawg/Rombos-LLM-V2.5-Qwen-32b", - "developer": "rombodawg", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "32.764" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/rombodawg_Rombos-LLM-V2.5-Qwen-32b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6827 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7046 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4955 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3968 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5034 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5916 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/rombodawg_rombos-llm-v2.5-qwen-3b.json b/data/models/rombodawg_rombos-llm-v2.5-qwen-3b.json deleted file mode 100644 index 09252618f3963060b9a4a0dcc17bcf0dc5b1fde3..0000000000000000000000000000000000000000 --- a/data/models/rombodawg_rombos-llm-v2.5-qwen-3b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Rombos-LLM-V2.5-Qwen-3b", - "id": "rombodawg/Rombos-LLM-V2.5-Qwen-3b", - "developer": "rombodawg", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.397" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/rombodawg_Rombos-LLM-V2.5-Qwen-3b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5342 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4809 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2795 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3079 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4042 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3761 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/rombodawg_rombos-llm-v2.5-qwen-72b.json b/data/models/rombodawg_rombos-llm-v2.5-qwen-72b.json deleted file mode 100644 index 54922925747d07f3e53583bdbaff4bdaa36734ea..0000000000000000000000000000000000000000 --- a/data/models/rombodawg_rombos-llm-v2.5-qwen-72b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Rombos-LLM-V2.5-Qwen-72b", - "id": "rombodawg/Rombos-LLM-V2.5-Qwen-72b", - "developer": "rombodawg", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "72.706" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/rombodawg_Rombos-LLM-V2.5-Qwen-72b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7155 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.723 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5423 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3985 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4599 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5935 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/rombodawg_rombos-llm-v2.5-qwen-7b.json b/data/models/rombodawg_rombos-llm-v2.5-qwen-7b.json deleted file mode 100644 index 7720a1118c0530d0fa4d580cb63d7afa9ba4d449..0000000000000000000000000000000000000000 --- a/data/models/rombodawg_rombos-llm-v2.5-qwen-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Rombos-LLM-V2.5-Qwen-7b", - "id": "rombodawg/Rombos-LLM-V2.5-Qwen-7b", - "developer": "rombodawg", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/rombodawg_Rombos-LLM-V2.5-Qwen-7b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6237 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5544 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3814 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.318 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4291 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4469 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/rombodawg_rombos-llm-v2.5.1-qwen-3b.json b/data/models/rombodawg_rombos-llm-v2.5.1-qwen-3b.json deleted file mode 100644 index 9d341a3c927e4d25f7836c99807ee069a16cbedc..0000000000000000000000000000000000000000 --- a/data/models/rombodawg_rombos-llm-v2.5.1-qwen-3b.json +++ /dev/null @@ -1,275 +0,0 @@ -{ - "model_info": { - "name": "Rombos-LLM-V2.5.1-Qwen-3b", - "id": "rombodawg/Rombos-LLM-V2.5.1-Qwen-3b", - "developer": "rombodawg", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.397" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/rombodawg_Rombos-LLM-V2.5.1-Qwen-3b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2595 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3884 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0914 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2743 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3991 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2719 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "hfopenllm_v2/rombodawg_Rombos-LLM-V2.5.1-Qwen-3b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2566 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.39 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1208 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2626 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3991 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2741 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/rombodawg_rombos-llm-v2.6-nemotron-70b.json b/data/models/rombodawg_rombos-llm-v2.6-nemotron-70b.json deleted file mode 100644 index 5ef1dc0c6a7511705ceee24dd5de499cd0e006c7..0000000000000000000000000000000000000000 --- a/data/models/rombodawg_rombos-llm-v2.6-nemotron-70b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Rombos-LLM-V2.6-Nemotron-70b", - "id": "rombodawg/Rombos-LLM-V2.6-Nemotron-70b", - "developer": "rombodawg", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "70.554" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/rombodawg_Rombos-LLM-V2.6-Nemotron-70b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7527 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6938 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3331 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.406 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4669 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5329 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/rombodawg_rombos-llm-v2.6-qwen-14b.json b/data/models/rombodawg_rombos-llm-v2.6-qwen-14b.json deleted file mode 100644 index c4d36f48e4d5695901b673140d96be20a12aa371..0000000000000000000000000000000000000000 --- a/data/models/rombodawg_rombos-llm-v2.6-qwen-14b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Rombos-LLM-V2.6-Qwen-14b", - "id": "rombodawg/Rombos-LLM-V2.6-Qwen-14b", - "developer": "rombodawg", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/rombodawg_Rombos-LLM-V2.6-Qwen-14b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8432 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6442 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5211 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3339 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4221 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4961 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/rombodawg_rombos_replete-coder-instruct-8b-merged.json b/data/models/rombodawg_rombos_replete-coder-instruct-8b-merged.json deleted file mode 100644 index a239a010d353bef55aa584eb9ce4a05600c58ff1..0000000000000000000000000000000000000000 --- a/data/models/rombodawg_rombos_replete-coder-instruct-8b-merged.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "rombos_Replete-Coder-Instruct-8b-Merged", - "id": "rombodawg/rombos_Replete-Coder-Instruct-8b-Merged", - "developer": "rombodawg", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/rombodawg_rombos_Replete-Coder-Instruct-8b-Merged/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5388 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4462 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0778 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2693 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.366 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1809 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/rombodawg_rombos_replete-coder-llama3-8b.json b/data/models/rombodawg_rombos_replete-coder-llama3-8b.json deleted file mode 100644 index d59d7be3928db12621a7a133147d76248cf44913..0000000000000000000000000000000000000000 --- a/data/models/rombodawg_rombos_replete-coder-llama3-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "rombos_Replete-Coder-Llama3-8B", - "id": "rombodawg/rombos_Replete-Coder-Llama3-8B", - "developer": "rombodawg", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/rombodawg_rombos_Replete-Coder-Llama3-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4714 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3276 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0393 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2668 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3966 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1335 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/rootxhacker_apollo-70b.json b/data/models/rootxhacker_apollo-70b.json deleted file mode 100644 index ae1b805491c064252046bc1d458421b6b282d3ab..0000000000000000000000000000000000000000 --- a/data/models/rootxhacker_apollo-70b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Apollo-70B", - "id": "rootxhacker/Apollo-70B", - "developer": "rootxhacker", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "70.554" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/rootxhacker_Apollo-70B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5099 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6804 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5612 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4572 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4948 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5279 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/rootxhacker_apollo-7b.json b/data/models/rootxhacker_apollo-7b.json deleted file mode 100644 index e4210ea080dc985f8a0ecdf1e6e78b2c2cf9f7cf..0000000000000000000000000000000000000000 --- a/data/models/rootxhacker_apollo-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "apollo-7B", - "id": "rootxhacker/apollo-7B", - "developer": "rootxhacker", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/rootxhacker_apollo-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2953 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3636 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0257 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2785 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4131 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1748 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/rootxhacker_apollo_v2-32b.json b/data/models/rootxhacker_apollo_v2-32b.json deleted file mode 100644 index 48974a4969d5bb2d5882c78f849e378ab6c30316..0000000000000000000000000000000000000000 --- a/data/models/rootxhacker_apollo_v2-32b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Apollo_v2-32B", - "id": "rootxhacker/Apollo_v2-32B", - "developer": "rootxhacker", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "32.764" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/rootxhacker_Apollo_v2-32B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.428 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7072 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4275 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3784 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4994 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5869 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/rsh345_mistral-ft-optimized-1218-neuralhermes-2.5-mistral-7b.json b/data/models/rsh345_mistral-ft-optimized-1218-neuralhermes-2.5-mistral-7b.json deleted file mode 100644 index c2e69a76dbc0f953a6d21e7983e2dee47749109f..0000000000000000000000000000000000000000 --- a/data/models/rsh345_mistral-ft-optimized-1218-neuralhermes-2.5-mistral-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "mistral-ft-optimized-1218-NeuralHermes-2.5-Mistral-7B", - "id": "rsh345/mistral-ft-optimized-1218-NeuralHermes-2.5-Mistral-7B", - "developer": "rsh345", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/rsh345_mistral-ft-optimized-1218-NeuralHermes-2.5-Mistral-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3892 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5188 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0733 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3029 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4672 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3054 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/rubenroy_geneva-12b-gcv2-5m.json b/data/models/rubenroy_geneva-12b-gcv2-5m.json deleted file mode 100644 index afa5e60c9be9af6d2853b8487a622272adf45f5e..0000000000000000000000000000000000000000 --- a/data/models/rubenroy_geneva-12b-gcv2-5m.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Geneva-12B-GCv2-5m", - "id": "rubenroy/Geneva-12B-GCv2-5m", - "developer": "rubenroy", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/rubenroy_Geneva-12B-GCv2-5m/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2586 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5278 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0801 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2878 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3525 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.325 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/rubenroy_gilgamesh-72b.json b/data/models/rubenroy_gilgamesh-72b.json deleted file mode 100644 index ac47c5dfedf193ee453a6fee8d06ac6048f8c0bc..0000000000000000000000000000000000000000 --- a/data/models/rubenroy_gilgamesh-72b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Gilgamesh-72B", - "id": "rubenroy/Gilgamesh-72B", - "developer": "rubenroy", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "72.706" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/rubenroy_Gilgamesh-72B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8486 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7253 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4381 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3943 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4626 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5802 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/rubenroy_zurich-14b-gcv2-5m.json b/data/models/rubenroy_zurich-14b-gcv2-5m.json deleted file mode 100644 index 1030adf29925f1361efc235cb3f961f374cc8a25..0000000000000000000000000000000000000000 --- a/data/models/rubenroy_zurich-14b-gcv2-5m.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Zurich-14B-GCv2-5m", - "id": "rubenroy/Zurich-14B-GCv2-5m", - "developer": "rubenroy", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/rubenroy_Zurich-14B-GCv2-5m/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6164 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6308 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3074 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3616 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4874 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5233 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/rubiellabarta_logos-7bx2-moe-13b-v0.2.json b/data/models/rubiellabarta_logos-7bx2-moe-13b-v0.2.json deleted file mode 100644 index 49bcaeb7a22efb80ba60f9b5884d09e808508d1c..0000000000000000000000000000000000000000 --- a/data/models/rubiellabarta_logos-7bx2-moe-13b-v0.2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "LogoS-7Bx2-MoE-13B-v0.2", - "id": "RubielLabarta/LogoS-7Bx2-MoE-13B-v0.2", - "developer": "RubielLabarta", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MixtralForCausalLM", - "params_billions": "12.879" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/RubielLabarta_LogoS-7Bx2-MoE-13B-v0.2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4379 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5207 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0574 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2777 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4226 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3088 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ruizhe1217_sft-s1-qwen-0.5b.json b/data/models/ruizhe1217_sft-s1-qwen-0.5b.json deleted file mode 100644 index f4d419abc4974ddd6299138141844f412e40d3e5..0000000000000000000000000000000000000000 --- a/data/models/ruizhe1217_sft-s1-qwen-0.5b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "sft-s1-qwen-0.5b", - "id": "ruizhe1217/sft-s1-qwen-0.5b", - "developer": "ruizhe1217", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ruizhe1217_sft-s1-qwen-0.5b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2749 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3301 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0619 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.271 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3196 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1892 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/rwitz_go-bruins-v2.json b/data/models/rwitz_go-bruins-v2.json deleted file mode 100644 index b8a77d6d8d702fd6a85aca91936a651421fe0a39..0000000000000000000000000000000000000000 --- a/data/models/rwitz_go-bruins-v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "go-bruins-v2", - "id": "rwitz/go-bruins-v2", - "developer": "rwitz", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/rwitz_go-bruins-v2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4096 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3799 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0672 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2626 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4138 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2761 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/rwkv_rwkv-raven-14b.json b/data/models/rwkv_rwkv-raven-14b.json deleted file mode 100644 index 66abc8cbd6b66272560d4a6fb2d3aa630df13b5f..0000000000000000000000000000000000000000 --- a/data/models/rwkv_rwkv-raven-14b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "rwkv-raven-14b", - "id": "RWKV/rwkv-raven-14b", - "developer": "RWKV", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "RwkvForCausalLM", - "params_billions": "14.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/RWKV_rwkv-raven-14b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0768 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3307 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0045 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.229 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3951 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.115 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sabersaleh_llama2-7b-cpo.json b/data/models/sabersaleh_llama2-7b-cpo.json deleted file mode 100644 index 079aa19e8a629406cdb46c37e1f7497720c10ce3..0000000000000000000000000000000000000000 --- a/data/models/sabersaleh_llama2-7b-cpo.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama2-7B-CPO", - "id": "sabersaleh/Llama2-7B-CPO", - "developer": "sabersaleh", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "7.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sabersaleh_Llama2-7B-CPO/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1545 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3458 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0136 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2676 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4048 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1606 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sabersaleh_llama2-7b-dpo.json b/data/models/sabersaleh_llama2-7b-dpo.json deleted file mode 100644 index 7c4de4267c158521c507ed36f0027f2979667fbc..0000000000000000000000000000000000000000 --- a/data/models/sabersaleh_llama2-7b-dpo.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama2-7B-DPO", - "id": "sabersaleh/Llama2-7B-DPO", - "developer": "sabersaleh", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "7.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sabersaleh_Llama2-7B-DPO/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1453 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3512 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0159 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2685 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4114 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1626 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sabersaleh_llama2-7b-ipo.json b/data/models/sabersaleh_llama2-7b-ipo.json deleted file mode 100644 index fe3c67d0539670d20534864c8b3ceff18c30cf92..0000000000000000000000000000000000000000 --- a/data/models/sabersaleh_llama2-7b-ipo.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama2-7B-IPO", - "id": "sabersaleh/Llama2-7B-IPO", - "developer": "sabersaleh", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "7.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sabersaleh_Llama2-7B-IPO/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1769 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3475 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0159 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2676 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4048 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1617 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sabersaleh_llama2-7b-kto.json b/data/models/sabersaleh_llama2-7b-kto.json deleted file mode 100644 index e2dd868c6b7f0a41bdf7f12c87a48a4ba691125b..0000000000000000000000000000000000000000 --- a/data/models/sabersaleh_llama2-7b-kto.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama2-7B-KTO", - "id": "sabersaleh/Llama2-7B-KTO", - "developer": "sabersaleh", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "7.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sabersaleh_Llama2-7B-KTO/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1528 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3501 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0189 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2676 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4167 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1636 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sabersaleh_llama2-7b-simpo.json b/data/models/sabersaleh_llama2-7b-simpo.json deleted file mode 100644 index 588041e625fc6e4af6ee50bd945ebdd71868a4b9..0000000000000000000000000000000000000000 --- a/data/models/sabersaleh_llama2-7b-simpo.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama2-7B-SimPO", - "id": "sabersaleh/Llama2-7B-SimPO", - "developer": "sabersaleh", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "7.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sabersaleh_Llama2-7B-SimPO/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1659 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3489 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0159 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.271 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4007 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1641 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sabersaleh_llama2-7b-spo.json b/data/models/sabersaleh_llama2-7b-spo.json deleted file mode 100644 index 2aea1a8c425ae39307167e654a48dd5b00024f21..0000000000000000000000000000000000000000 --- a/data/models/sabersaleh_llama2-7b-spo.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama2-7B-SPO", - "id": "sabersaleh/Llama2-7B-SPO", - "developer": "sabersaleh", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "7.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sabersaleh_Llama2-7B-SPO/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1567 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3383 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0196 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2768 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3874 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1757 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sabersaleh_llama3.json b/data/models/sabersaleh_llama3.json deleted file mode 100644 index aed1fc32541698c907ebaccf1eaa2c4057bc0d0e..0000000000000000000000000000000000000000 --- a/data/models/sabersaleh_llama3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama3", - "id": "sabersaleh/Llama3", - "developer": "sabersaleh", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sabersaleh_Llama3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3321 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4782 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0566 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3104 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3933 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3162 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sabersalehk_llama3-001-300.json b/data/models/sabersalehk_llama3-001-300.json deleted file mode 100644 index 1a7e3c2c3b2cad4b456f047a9868ab7807ace465..0000000000000000000000000000000000000000 --- a/data/models/sabersalehk_llama3-001-300.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama3-001-300", - "id": "sabersalehk/Llama3-001-300", - "developer": "sabersalehk", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sabersalehk_Llama3-001-300/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3179 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4745 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0529 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2995 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4064 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3158 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sabersalehk_llama3-simpo.json b/data/models/sabersalehk_llama3-simpo.json deleted file mode 100644 index 9dfa288888556e3cc529927b9939e05823e22b6d..0000000000000000000000000000000000000000 --- a/data/models/sabersalehk_llama3-simpo.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama3-SimPO", - "id": "sabersalehk/Llama3-SimPO", - "developer": "sabersalehk", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sabersalehk_Llama3-SimPO/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3642 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4874 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0574 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3079 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4046 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3157 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sabersalehk_llama3_001_200.json b/data/models/sabersalehk_llama3_001_200.json deleted file mode 100644 index 3339dcfab8f91ef82028252f58c8a4154099d6b5..0000000000000000000000000000000000000000 --- a/data/models/sabersalehk_llama3_001_200.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama3_001_200", - "id": "sabersalehk/Llama3_001_200", - "developer": "sabersalehk", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sabersalehk_Llama3_001_200/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3218 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4728 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0514 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3037 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4037 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3183 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sabersalehk_llama3_01_300.json b/data/models/sabersalehk_llama3_01_300.json deleted file mode 100644 index 4efb10eebe3ea76c754ec540a6d8c1d625d13dc7..0000000000000000000000000000000000000000 --- a/data/models/sabersalehk_llama3_01_300.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama3_01_300", - "id": "sabersalehk/Llama3_01_300", - "developer": "sabersalehk", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sabersalehk_Llama3_01_300/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2959 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4691 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0498 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3079 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4065 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3124 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/saisexperiments_evil-alpaca-3b-l3.2.json b/data/models/saisexperiments_evil-alpaca-3b-l3.2.json deleted file mode 100644 index c3471e34dd50fc2479129ceafea987d4a7d3e1db..0000000000000000000000000000000000000000 --- a/data/models/saisexperiments_evil-alpaca-3b-l3.2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Evil-Alpaca-3B-L3.2", - "id": "SaisExperiments/Evil-Alpaca-3B-L3.2", - "developer": "SaisExperiments", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/SaisExperiments_Evil-Alpaca-3B-L3.2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3251 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4341 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0702 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2634 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4198 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2621 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/saisexperiments_gemma-2-2b-opus-instruct.json b/data/models/saisexperiments_gemma-2-2b-opus-instruct.json deleted file mode 100644 index 7b05158a9d5fa756992632bf0e538da0247130c3..0000000000000000000000000000000000000000 --- a/data/models/saisexperiments_gemma-2-2b-opus-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Gemma-2-2B-Opus-Instruct", - "id": "SaisExperiments/Gemma-2-2B-Opus-Instruct", - "developer": "SaisExperiments", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "2.614" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/SaisExperiments_Gemma-2-2B-Opus-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.475 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4293 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0506 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2836 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4057 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.265 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/saisexperiments_gemma-2-2b-stheno-filtered.json b/data/models/saisexperiments_gemma-2-2b-stheno-filtered.json deleted file mode 100644 index 8b8ccc319c248e678c6c5cc456afa6a096389073..0000000000000000000000000000000000000000 --- a/data/models/saisexperiments_gemma-2-2b-stheno-filtered.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Gemma-2-2B-Stheno-Filtered", - "id": "SaisExperiments/Gemma-2-2B-Stheno-Filtered", - "developer": "SaisExperiments", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "2.614" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/SaisExperiments_Gemma-2-2B-Stheno-Filtered/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4197 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4149 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0461 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2701 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4003 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.263 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/saisexperiments_not-so-small-alpaca-24b.json b/data/models/saisexperiments_not-so-small-alpaca-24b.json deleted file mode 100644 index 008e7d9b3884e8065ca7f96e0c2e02a6dabb6f9e..0000000000000000000000000000000000000000 --- a/data/models/saisexperiments_not-so-small-alpaca-24b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Not-So-Small-Alpaca-24B", - "id": "SaisExperiments/Not-So-Small-Alpaca-24B", - "developer": "SaisExperiments", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "23.572" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/SaisExperiments_Not-So-Small-Alpaca-24B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6244 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5339 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1828 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3591 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4282 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3694 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/saisexperiments_qwowo-7b-v1.json b/data/models/saisexperiments_qwowo-7b-v1.json deleted file mode 100644 index 6635e017410562311b1af63c891e42975b12065b..0000000000000000000000000000000000000000 --- a/data/models/saisexperiments_qwowo-7b-v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "QwOwO-7B-V1", - "id": "SaisExperiments/QwOwO-7B-V1", - "developer": "SaisExperiments", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/SaisExperiments_QwOwO-7B-V1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4556 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5431 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.386 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2601 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3835 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4224 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/saisexperiments_rightsheep-llama3.2-3b.json b/data/models/saisexperiments_rightsheep-llama3.2-3b.json deleted file mode 100644 index a740f8216376ffba39d97a3c36fb3c9b58737c39..0000000000000000000000000000000000000000 --- a/data/models/saisexperiments_rightsheep-llama3.2-3b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "RightSheep-Llama3.2-3B", - "id": "SaisExperiments/RightSheep-Llama3.2-3B", - "developer": "SaisExperiments", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/SaisExperiments_RightSheep-Llama3.2-3B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4156 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4241 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0808 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2869 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3767 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.254 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/saishf_fimbulvetr-kuro-lotus-10.7b.json b/data/models/saishf_fimbulvetr-kuro-lotus-10.7b.json deleted file mode 100644 index 15b3762f9096207098705f55e787586690ff401a..0000000000000000000000000000000000000000 --- a/data/models/saishf_fimbulvetr-kuro-lotus-10.7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Fimbulvetr-Kuro-Lotus-10.7B", - "id": "saishf/Fimbulvetr-Kuro-Lotus-10.7B", - "developer": "saishf", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "10.732" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/saishf_Fimbulvetr-Kuro-Lotus-10.7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4939 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4342 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0536 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3012 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4445 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3389 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/saishf_neural-sovlish-devil-8b-l3.json b/data/models/saishf_neural-sovlish-devil-8b-l3.json deleted file mode 100644 index 8d6b91ea1509f0237719e3b87f3a0e1edfa33f2b..0000000000000000000000000000000000000000 --- a/data/models/saishf_neural-sovlish-devil-8b-l3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Neural-SOVLish-Devil-8B-L3", - "id": "saishf/Neural-SOVLish-Devil-8B-L3", - "developer": "saishf", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/saishf_Neural-SOVLish-Devil-8B-L3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4199 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5142 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0891 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3079 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.411 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3807 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/saishshinde15_tethysai_base_reasoning.json b/data/models/saishshinde15_tethysai_base_reasoning.json deleted file mode 100644 index fe69c53512a70d73bd42739ed16f95bcbc8eed0d..0000000000000000000000000000000000000000 --- a/data/models/saishshinde15_tethysai_base_reasoning.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "TethysAI_Base_Reasoning", - "id": "saishshinde15/TethysAI_Base_Reasoning", - "developer": "saishshinde15", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.086" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/saishshinde15_TethysAI_Base_Reasoning/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6369 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4519 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3142 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2861 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4075 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3236 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/saishshinde15_tethysai_vortex.json b/data/models/saishshinde15_tethysai_vortex.json deleted file mode 100644 index f46ed4921cd04dca13214a3b4c1982de75566e7c..0000000000000000000000000000000000000000 --- a/data/models/saishshinde15_tethysai_vortex.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "TethysAI_Vortex", - "id": "saishshinde15/TethysAI_Vortex", - "developer": "saishshinde15", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.086" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/saishshinde15_TethysAI_Vortex/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4298 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4749 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.315 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3054 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4458 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3241 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/saishshinde15_tethysai_vortex_reasoning.json b/data/models/saishshinde15_tethysai_vortex_reasoning.json deleted file mode 100644 index 164aaecda38f13279703cbdfed24cf8d01782265..0000000000000000000000000000000000000000 --- a/data/models/saishshinde15_tethysai_vortex_reasoning.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "TethysAI_Vortex_Reasoning", - "id": "saishshinde15/TethysAI_Vortex_Reasoning", - "developer": "saishshinde15", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.086" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/saishshinde15_TethysAI_Vortex_Reasoning/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4021 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4694 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2145 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3045 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4084 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3381 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sakaltcommunity_novablast-preview.json b/data/models/sakaltcommunity_novablast-preview.json deleted file mode 100644 index 3f69173084623c5a27f97f52aa91e8798c89075c..0000000000000000000000000000000000000000 --- a/data/models/sakaltcommunity_novablast-preview.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "novablast-preview", - "id": "sakaltcommunity/novablast-preview", - "developer": "sakaltcommunity", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "32.764" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sakaltcommunity_novablast-preview/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.453 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7043 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4894 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3817 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5021 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5915 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sakaltcommunity_sakaltum-7b.json b/data/models/sakaltcommunity_sakaltum-7b.json deleted file mode 100644 index 3e6ceb4eabfba50cbb8721f63e5ffa5dd9ba7fd6..0000000000000000000000000000000000000000 --- a/data/models/sakaltcommunity_sakaltum-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "sakaltum-7b", - "id": "sakaltcommunity/sakaltum-7b", - "developer": "sakaltcommunity", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sakaltcommunity_sakaltum-7b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2604 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4575 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0295 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2727 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3775 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2769 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sakalti_anemoi-3b.json b/data/models/sakalti_anemoi-3b.json deleted file mode 100644 index 8922c8cba8940dc8d725ff21884a624164a62df2..0000000000000000000000000000000000000000 --- a/data/models/sakalti_anemoi-3b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Anemoi-3B", - "id": "Sakalti/Anemoi-3B", - "developer": "Sakalti", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.397" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Sakalti_Anemoi-3B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3804 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4922 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1775 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3054 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4371 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3766 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sakalti_euphrates-14b.json b/data/models/sakalti_euphrates-14b.json deleted file mode 100644 index 305ca6b98aca43359ac73ff6faf3e4d91e66a9c7..0000000000000000000000000000000000000000 --- a/data/models/sakalti_euphrates-14b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Euphrates-14B", - "id": "Sakalti/Euphrates-14B", - "developer": "Sakalti", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Sakalti_Euphrates-14B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2647 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6138 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3051 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3935 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4516 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5255 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sakalti_light-1.1-3b.json b/data/models/sakalti_light-1.1-3b.json deleted file mode 100644 index 40582a9afbbc413807e25b7706d84b510c355342..0000000000000000000000000000000000000000 --- a/data/models/sakalti_light-1.1-3b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "light-1.1-3B", - "id": "Sakalti/light-1.1-3B", - "developer": "Sakalti", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.086" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Sakalti_light-1.1-3B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2735 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2803 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0113 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2617 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3901 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1209 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sakalti_light-3b-beta.json b/data/models/sakalti_light-3b-beta.json deleted file mode 100644 index 89cb924271446ce25013c8c98e3cd66260f6c769..0000000000000000000000000000000000000000 --- a/data/models/sakalti_light-3b-beta.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "light-3b-beta", - "id": "Sakalti/light-3b-beta", - "developer": "Sakalti", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.397" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Sakalti_light-3b-beta/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5485 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4815 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2772 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2978 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4015 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3758 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sakalti_light-3b.json b/data/models/sakalti_light-3b.json deleted file mode 100644 index 5a82ca0299e2fefe0c7dd2ec36ecd7896042ccad..0000000000000000000000000000000000000000 --- a/data/models/sakalti_light-3b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "light-3B", - "id": "Sakalti/light-3B", - "developer": "Sakalti", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.397" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Sakalti_light-3B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5337 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4831 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2591 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2953 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4015 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3775 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sakalti_light-7b-beta.json b/data/models/sakalti_light-7b-beta.json deleted file mode 100644 index 77b4e421bc27f21ad8b3d9609d98469f878cbf7d..0000000000000000000000000000000000000000 --- a/data/models/sakalti_light-7b-beta.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "light-7b-beta", - "id": "Sakalti/light-7b-beta", - "developer": "Sakalti", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Sakalti_light-7b-beta/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6234 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5548 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3769 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3213 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4291 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4456 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sakalti_llama-3-yanyuedao-8b-instruct.json b/data/models/sakalti_llama-3-yanyuedao-8b-instruct.json deleted file mode 100644 index fa903af8e51d64c998f7e8e72f7cce4967454442..0000000000000000000000000000000000000000 --- a/data/models/sakalti_llama-3-yanyuedao-8b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "llama-3-yanyuedao-8b-instruct", - "id": "Sakalti/llama-3-yanyuedao-8b-instruct", - "developer": "Sakalti", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Sakalti_llama-3-yanyuedao-8b-instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2186 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.435 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0385 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2903 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4199 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2911 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sakalti_llama3.2-3b-uranus-1.json b/data/models/sakalti_llama3.2-3b-uranus-1.json deleted file mode 100644 index 204b53c6965684b1172c32fd64720170f9f2ebc2..0000000000000000000000000000000000000000 --- a/data/models/sakalti_llama3.2-3b-uranus-1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama3.2-3B-Uranus-1", - "id": "Sakalti/Llama3.2-3B-Uranus-1", - "developer": "Sakalti", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Sakalti_Llama3.2-3B-Uranus-1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5335 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4437 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1495 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.297 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3669 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3094 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sakalti_magro-7b-v1.1.json b/data/models/sakalti_magro-7b-v1.1.json deleted file mode 100644 index e63deb031a5cb73251a7047ebc94c4b18d6f9760..0000000000000000000000000000000000000000 --- a/data/models/sakalti_magro-7b-v1.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Magro-7B-v1.1", - "id": "Sakalti/Magro-7B-v1.1", - "developer": "Sakalti", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Sakalti_Magro-7B-v1.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1204 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4179 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0249 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2961 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4433 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2764 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sakalti_magro-7b.json b/data/models/sakalti_magro-7b.json deleted file mode 100644 index 2dcd0c747763ad9026b1377b5d066b23f18b08e9..0000000000000000000000000000000000000000 --- a/data/models/sakalti_magro-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "magro-7B", - "id": "Sakalti/magro-7B", - "developer": "Sakalti", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Sakalti_magro-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1344 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4186 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0204 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2953 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.446 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2765 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sakalti_mergekit-01.json b/data/models/sakalti_mergekit-01.json deleted file mode 100644 index 3db7fe9f4ca6982248458bb211b4286ccb03b8e8..0000000000000000000000000000000000000000 --- a/data/models/sakalti_mergekit-01.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "mergekit-01", - "id": "Sakalti/mergekit-01", - "developer": "Sakalti", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Sakalti_mergekit-01/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6234 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5548 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3769 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3213 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4291 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4456 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sakalti_mergekit-della_linear-vmeykci.json b/data/models/sakalti_mergekit-della_linear-vmeykci.json deleted file mode 100644 index 63aac3b3ce4fcf7ed2a40c45e68cb461e314bda5..0000000000000000000000000000000000000000 --- a/data/models/sakalti_mergekit-della_linear-vmeykci.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "mergekit-della_linear-vmeykci", - "id": "Sakalti/mergekit-della_linear-vmeykci", - "developer": "Sakalti", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.613" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Sakalti_mergekit-della_linear-vmeykci/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1126 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2816 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0106 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2634 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3897 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1089 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sakalti_model-3.json b/data/models/sakalti_model-3.json deleted file mode 100644 index 1af8030348c9739b707e47f9302577468b697acd..0000000000000000000000000000000000000000 --- a/data/models/sakalti_model-3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "model-3", - "id": "Sakalti/model-3", - "developer": "Sakalti", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Sakalti_model-3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6264 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5542 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3708 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3213 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4264 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4455 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sakalti_neptuno-3b.json b/data/models/sakalti_neptuno-3b.json deleted file mode 100644 index bac499906c962b1d63fe981836c246c2ba5179cd..0000000000000000000000000000000000000000 --- a/data/models/sakalti_neptuno-3b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Neptuno-3B", - "id": "Sakalti/Neptuno-3B", - "developer": "Sakalti", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.397" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Sakalti_Neptuno-3B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4296 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4834 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2553 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2961 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4002 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3773 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sakalti_neptuno-alpha.json b/data/models/sakalti_neptuno-alpha.json deleted file mode 100644 index 64f37bb24ef1f4ccaaa301f8c4abc9e46a3615de..0000000000000000000000000000000000000000 --- a/data/models/sakalti_neptuno-alpha.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Neptuno-Alpha", - "id": "Sakalti/Neptuno-Alpha", - "developer": "Sakalti", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.397" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Sakalti_Neptuno-Alpha/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.378 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4925 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1835 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.307 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4371 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3767 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sakalti_oxyge1-33b.json b/data/models/sakalti_oxyge1-33b.json deleted file mode 100644 index 1a9f3b792f307e345c78569a35ea2ad23a604db2..0000000000000000000000000000000000000000 --- a/data/models/sakalti_oxyge1-33b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Oxyge1-33B", - "id": "Sakalti/Oxyge1-33B", - "developer": "Sakalti", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "32.764" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Sakalti_Oxyge1-33B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4548 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7033 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4962 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3826 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5008 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5909 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sakalti_phi3.5-comets-3.8b.json b/data/models/sakalti_phi3.5-comets-3.8b.json deleted file mode 100644 index 4852d5c16f10685a6f332fd844855d74ebcdc709..0000000000000000000000000000000000000000 --- a/data/models/sakalti_phi3.5-comets-3.8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Phi3.5-Comets-3.8B", - "id": "Sakalti/Phi3.5-Comets-3.8B", - "developer": "Sakalti", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Phi3ForCausalLM", - "params_billions": "3.821" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Sakalti_Phi3.5-Comets-3.8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2094 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3335 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0008 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2492 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3764 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1153 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sakalti_qwen2.5-1b-instruct.json b/data/models/sakalti_qwen2.5-1b-instruct.json deleted file mode 100644 index 029f9a00dd2b6e3ca7dd2280d062af9992f63ccf..0000000000000000000000000000000000000000 --- a/data/models/sakalti_qwen2.5-1b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-1B-Instruct", - "id": "Sakalti/Qwen2.5-1B-Instruct", - "developer": "Sakalti", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.988" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Sakalti_Qwen2.5-1B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1751 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3027 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.006 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2559 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3369 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1213 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sakalti_qwen2.5-2.3b.json b/data/models/sakalti_qwen2.5-2.3b.json deleted file mode 100644 index 0fe969fd5f6f5d2b35c5e88a0a5d5ca105438f8b..0000000000000000000000000000000000000000 --- a/data/models/sakalti_qwen2.5-2.3b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "qwen2.5-2.3B", - "id": "Sakalti/qwen2.5-2.3B", - "developer": "Sakalti", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2Model", - "params_billions": "2.339" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Sakalti_qwen2.5-2.3B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1288 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2849 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0053 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2517 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3857 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1173 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sakalti_qwentest-7.json b/data/models/sakalti_qwentest-7.json deleted file mode 100644 index a037f62ec6650adbd71795c176e44324b82e347d..0000000000000000000000000000000000000000 --- a/data/models/sakalti_qwentest-7.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "QwenTest-7", - "id": "Sakalti/QwenTest-7", - "developer": "Sakalti", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.988" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Sakalti_QwenTest-7/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1672 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3063 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0038 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2601 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3422 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1212 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sakalti_saba-passthrough-2.json b/data/models/sakalti_saba-passthrough-2.json deleted file mode 100644 index 2dbdd2e1618333fdf282e59b0124150e0bdc1d9a..0000000000000000000000000000000000000000 --- a/data/models/sakalti_saba-passthrough-2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Saba-Passthrough-2", - "id": "Sakalti/Saba-Passthrough-2", - "developer": "Sakalti", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.087" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Sakalti_Saba-Passthrough-2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1691 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3672 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0008 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2634 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3844 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2077 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sakalti_saba1-1.8b.json b/data/models/sakalti_saba1-1.8b.json deleted file mode 100644 index bd94a18fa4bf514c5ebd480a5aeea9daea95962e..0000000000000000000000000000000000000000 --- a/data/models/sakalti_saba1-1.8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Saba1-1.8B", - "id": "Sakalti/Saba1-1.8B", - "developer": "Sakalti", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.777" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Sakalti_Saba1-1.8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3333 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4147 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1541 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2827 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4239 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2926 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sakalti_saba1-7b.json b/data/models/sakalti_saba1-7b.json deleted file mode 100644 index 489ea55f58c799b212eeb2bfccada0f841757f72..0000000000000000000000000000000000000000 --- a/data/models/sakalti_saba1-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Saba1-7B", - "id": "Sakalti/Saba1-7B", - "developer": "Sakalti", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Sakalti_Saba1-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4585 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5489 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3663 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3163 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4793 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4376 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sakalti_saba1.5-1.5b.json b/data/models/sakalti_saba1.5-1.5b.json deleted file mode 100644 index e0a14d4132d4f43d3eb29c3fbcce39394631b9ad..0000000000000000000000000000000000000000 --- a/data/models/sakalti_saba1.5-1.5b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Saba1.5-1.5B", - "id": "Sakalti/Saba1.5-1.5B", - "developer": "Sakalti", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.544" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Sakalti_Saba1.5-1.5B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3333 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4147 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1541 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2827 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4239 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2926 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sakalti_saba1.5-pro-3b.json b/data/models/sakalti_saba1.5-pro-3b.json deleted file mode 100644 index 2d9cf1beb0cb878cb29c057780dbd2a461f2541c..0000000000000000000000000000000000000000 --- a/data/models/sakalti_saba1.5-pro-3b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Saba1.5-Pro-3B", - "id": "Sakalti/Saba1.5-Pro-3B", - "developer": "Sakalti", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "2.9" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Sakalti_Saba1.5-Pro-3B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2386 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3623 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0272 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2685 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4405 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1958 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sakalti_saba2-14b-preview.json b/data/models/sakalti_saba2-14b-preview.json deleted file mode 100644 index 1e01e96a014cac055f3280f60dad45ae86f94525..0000000000000000000000000000000000000000 --- a/data/models/sakalti_saba2-14b-preview.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Saba2-14B-Preview", - "id": "Sakalti/Saba2-14B-Preview", - "developer": "Sakalti", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Sakalti_Saba2-14B-Preview/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4722 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6496 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3127 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3826 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4781 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5384 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sakalti_saba2-3b.json b/data/models/sakalti_saba2-3b.json deleted file mode 100644 index 6f97170b9e8aefc5551a3ed040c0f1a05e7168b2..0000000000000000000000000000000000000000 --- a/data/models/sakalti_saba2-3b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Saba2-3B", - "id": "Sakalti/Saba2-3B", - "developer": "Sakalti", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.086" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Sakalti_Saba2-3B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2865 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2801 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.006 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2617 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3927 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.121 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sakalti_sailor-japanese.json b/data/models/sakalti_sailor-japanese.json deleted file mode 100644 index 738583f12367889a225f571ca2ee453c3eeb0957..0000000000000000000000000000000000000000 --- a/data/models/sakalti_sailor-japanese.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Sailor-japanese", - "id": "Sakalti/Sailor-japanese", - "developer": "Sakalti", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Sakalti_Sailor-japanese/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1605 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2913 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.003 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2534 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3912 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1164 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sakalti_saka-1.5b.json b/data/models/sakalti_saka-1.5b.json deleted file mode 100644 index df480048708edd07682dfa89b9df4656cc541973..0000000000000000000000000000000000000000 --- a/data/models/sakalti_saka-1.5b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Saka-1.5B", - "id": "Sakalti/Saka-1.5B", - "developer": "Sakalti", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.777" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Sakalti_Saka-1.5B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2726 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3988 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0801 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2903 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3739 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2415 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sakalti_saka-14b.json b/data/models/sakalti_saka-14b.json deleted file mode 100644 index ce6dc6167e565ed3d5e1c31edfc04a32e1167499..0000000000000000000000000000000000000000 --- a/data/models/sakalti_saka-14b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Saka-14B", - "id": "Sakalti/Saka-14B", - "developer": "Sakalti", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Sakalti_Saka-14B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7174 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6497 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4094 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.396 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4886 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5396 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sakalti_saka-24b.json b/data/models/sakalti_saka-24b.json deleted file mode 100644 index 30926d4d94da84c7be001590649b29441f4b7277..0000000000000000000000000000000000000000 --- a/data/models/sakalti_saka-24b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Saka-24B", - "id": "Sakalti/Saka-24B", - "developer": "Sakalti", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "23.572" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Sakalti_Saka-24B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3819 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6072 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1805 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3423 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4541 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4766 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sakalti_saka-7.2b.json b/data/models/sakalti_saka-7.2b.json deleted file mode 100644 index 09fc1cac8eed4eb23c815726ea7f9cbc0166d1d4..0000000000000000000000000000000000000000 --- a/data/models/sakalti_saka-7.2b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Saka-7.2B", - "id": "Sakalti/Saka-7.2B", - "developer": "Sakalti", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "7.292" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Sakalti_Saka-7.2B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1545 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2945 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2391 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3711 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.116 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sakalti_saka-7.6b.json b/data/models/sakalti_saka-7.6b.json deleted file mode 100644 index b88415dc80fe654bfe446fddda25140fd9ef5689..0000000000000000000000000000000000000000 --- a/data/models/sakalti_saka-7.6b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Saka-7.6B", - "id": "Sakalti/Saka-7.6B", - "developer": "Sakalti", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Sakalti_Saka-7.6B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4524 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5655 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3255 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3163 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4489 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.454 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sakalti_sakalfusion-7b-alpha.json b/data/models/sakalti_sakalfusion-7b-alpha.json deleted file mode 100644 index 9d1a13e858835fbe91d75e86e1ee27dbd27340e6..0000000000000000000000000000000000000000 --- a/data/models/sakalti_sakalfusion-7b-alpha.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SakalFusion-7B-Alpha", - "id": "Sakalti/SakalFusion-7B-Alpha", - "developer": "Sakalti", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Sakalti_SakalFusion-7B-Alpha/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.529 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5591 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3844 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3255 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4581 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4474 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sakalti_sakalfusion-7b-beta.json b/data/models/sakalti_sakalfusion-7b-beta.json deleted file mode 100644 index c1a77d438ad15de7b5b9093ca4e2a00db43310ad..0000000000000000000000000000000000000000 --- a/data/models/sakalti_sakalfusion-7b-beta.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SakalFusion-7B-Beta", - "id": "Sakalti/SakalFusion-7B-Beta", - "developer": "Sakalti", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.613" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Sakalti_SakalFusion-7B-Beta/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1809 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2881 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2433 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3872 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.109 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sakalti_sakamoe-3x1.6b-instruct.json b/data/models/sakalti_sakamoe-3x1.6b-instruct.json deleted file mode 100644 index 78de2128edfebb769c8a8f8f8fc10e56a949b017..0000000000000000000000000000000000000000 --- a/data/models/sakalti_sakamoe-3x1.6b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SakaMoe-3x1.6B-Instruct", - "id": "Sakalti/SakaMoe-3x1.6B-Instruct", - "developer": "Sakalti", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2MoeForCausalLM", - "params_billions": "1.572" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Sakalti_SakaMoe-3x1.6B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2371 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3282 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0544 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2668 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3342 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1882 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sakalti_sjt-0.5b.json b/data/models/sakalti_sjt-0.5b.json deleted file mode 100644 index d77044a400be23a95a65d5bbd8435cce58542df3..0000000000000000000000000000000000000000 --- a/data/models/sakalti_sjt-0.5b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SJT-0.5B", - "id": "Sakalti/SJT-0.5B", - "developer": "Sakalti", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Sakalti_SJT-0.5B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2425 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3306 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0521 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2718 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3196 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1891 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sakalti_sjt-1.5b-alpha-1.1.json b/data/models/sakalti_sjt-1.5b-alpha-1.1.json deleted file mode 100644 index 1779548044cb746d80651124effd34d639c5d95e..0000000000000000000000000000000000000000 --- a/data/models/sakalti_sjt-1.5b-alpha-1.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SJT-1.5B-Alpha-1.1", - "id": "Sakalti/SJT-1.5B-Alpha-1.1", - "developer": "Sakalti", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.777" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Sakalti_SJT-1.5B-Alpha-1.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3439 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4243 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0959 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2894 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4239 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2966 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sakalti_sjt-1.5b-alpha.json b/data/models/sakalti_sjt-1.5b-alpha.json deleted file mode 100644 index 607ed3bf6ee6a5284805a170aacf21ae02630d77..0000000000000000000000000000000000000000 --- a/data/models/sakalti_sjt-1.5b-alpha.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SJT-1.5B-Alpha", - "id": "Sakalti/SJT-1.5B-Alpha", - "developer": "Sakalti", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.777" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Sakalti_SJT-1.5B-Alpha/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3449 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4241 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0997 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2919 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4226 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2961 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sakalti_sjt-1.7b.json b/data/models/sakalti_sjt-1.7b.json deleted file mode 100644 index 638645b7e345447e5d09cee2b2df8fb8ec2dcc7f..0000000000000000000000000000000000000000 --- a/data/models/sakalti_sjt-1.7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SJT-1.7B", - "id": "Sakalti/SJT-1.7B", - "developer": "Sakalti", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.684" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Sakalti_SJT-1.7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1776 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2934 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0015 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2416 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3964 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1133 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sakalti_sjt-14b.json b/data/models/sakalti_sjt-14b.json deleted file mode 100644 index 40a010c6d4dd788b5f233cbf3b1223f2d1d9381f..0000000000000000000000000000000000000000 --- a/data/models/sakalti_sjt-14b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SJT-14B", - "id": "Sakalti/SJT-14B", - "developer": "Sakalti", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Sakalti_SJT-14B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5494 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6536 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3844 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3867 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4766 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5381 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sakalti_sjt-2.4b.json b/data/models/sakalti_sjt-2.4b.json deleted file mode 100644 index 6cb8651d0c2984399cb1b653550df5bc43c5a8a2..0000000000000000000000000000000000000000 --- a/data/models/sakalti_sjt-2.4b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SJT-2.4B", - "id": "Sakalti/SJT-2.4B", - "developer": "Sakalti", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "2.432" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Sakalti_SJT-2.4B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2804 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.349 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0219 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2559 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3699 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1858 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sakalti_sjt-24b-alpha.json b/data/models/sakalti_sjt-24b-alpha.json deleted file mode 100644 index a24178d9729969f0d334f475f86937b39ec41e8a..0000000000000000000000000000000000000000 --- a/data/models/sakalti_sjt-24b-alpha.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SJT-24B-Alpha", - "id": "Sakalti/SJT-24B-Alpha", - "developer": "Sakalti", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "24.125" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Sakalti_SJT-24B-Alpha/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3206 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6081 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.253 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3809 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4595 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4857 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sakalti_sjt-2b-v1.1.json b/data/models/sakalti_sjt-2b-v1.1.json deleted file mode 100644 index e7fa32d07a8704701d1f54c4d61f2e73d3ae6057..0000000000000000000000000000000000000000 --- a/data/models/sakalti_sjt-2b-v1.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SJT-2B-V1.1", - "id": "Sakalti/SJT-2B-V1.1", - "developer": "Sakalti", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "2.614" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Sakalti_SJT-2B-V1.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3977 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3984 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0483 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2676 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4299 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2124 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sakalti_sjt-2b.json b/data/models/sakalti_sjt-2b.json deleted file mode 100644 index 4ff287e8286ef32f3612437e7e55cfc03392925f..0000000000000000000000000000000000000000 --- a/data/models/sakalti_sjt-2b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SJT-2B", - "id": "Sakalti/SJT-2B", - "developer": "Sakalti", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "2.614" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Sakalti_SJT-2B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2151 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2936 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0008 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2416 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3564 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1187 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sakalti_sjt-3.7b.json b/data/models/sakalti_sjt-3.7b.json deleted file mode 100644 index 92e14e636353b4c982fbb8832824d4b282f41f49..0000000000000000000000000000000000000000 --- a/data/models/sakalti_sjt-3.7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SJT-3.7B", - "id": "Sakalti/SJT-3.7B", - "developer": "Sakalti", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.783" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Sakalti_SJT-3.7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1078 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3393 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0121 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2559 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3617 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1505 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sakalti_sjt-4b.json b/data/models/sakalti_sjt-4b.json deleted file mode 100644 index 31304b8b20c6b786b2609e977ea286c5e4737cef..0000000000000000000000000000000000000000 --- a/data/models/sakalti_sjt-4b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SJT-4B", - "id": "Sakalti/SJT-4B", - "developer": "Sakalti", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.821" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Sakalti_SJT-4B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4077 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4886 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1156 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2945 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.478 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3281 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sakalti_sjt-7.5b.json b/data/models/sakalti_sjt-7.5b.json deleted file mode 100644 index 8738e691c1e6d0b4c9fa2279d5a9d3d98f08a342..0000000000000000000000000000000000000000 --- a/data/models/sakalti_sjt-7.5b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SJT-7.5B", - "id": "Sakalti/SJT-7.5B", - "developer": "Sakalti", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "7.456" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Sakalti_SJT-7.5B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4223 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5367 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2168 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3263 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4399 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3951 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sakalti_sjt-7b-v1.1-multilingal.json b/data/models/sakalti_sjt-7b-v1.1-multilingal.json deleted file mode 100644 index 4a6af20b558212aed9e9e8971245aba4144950b1..0000000000000000000000000000000000000000 --- a/data/models/sakalti_sjt-7b-v1.1-multilingal.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SJT-7B-V1.1-Multilingal", - "id": "Sakalti/SJT-7B-V1.1-Multilingal", - "developer": "Sakalti", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.613" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Sakalti_SJT-7B-V1.1-Multilingal/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1949 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.292 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0045 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2601 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3621 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1137 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sakalti_sjt-7b-v1.1.json b/data/models/sakalti_sjt-7b-v1.1.json deleted file mode 100644 index 80a32c3a992bdfc6da8b40ccc39fae0ff1744eca..0000000000000000000000000000000000000000 --- a/data/models/sakalti_sjt-7b-v1.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SJT-7B-V1.1", - "id": "Sakalti/SJT-7B-V1.1", - "developer": "Sakalti", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.613" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Sakalti_SJT-7B-V1.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4703 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5419 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2432 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3339 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4411 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4412 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sakalti_sjt-8b-v1.1.json b/data/models/sakalti_sjt-8b-v1.1.json deleted file mode 100644 index 209befdc00585fc429bcc055730d4e38cf1c56c4..0000000000000000000000000000000000000000 --- a/data/models/sakalti_sjt-8b-v1.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SJT-8B-V1.1", - "id": "Sakalti/SJT-8B-V1.1", - "developer": "Sakalti", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "8.545" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Sakalti_SJT-8B-V1.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4621 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5121 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2069 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3364 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4266 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4231 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sakalti_sjt-8b.json b/data/models/sakalti_sjt-8b.json deleted file mode 100644 index 6713f82b8bb2a39b054328f4839201e2720a7087..0000000000000000000000000000000000000000 --- a/data/models/sakalti_sjt-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SJT-8B", - "id": "Sakalti/SJT-8B", - "developer": "Sakalti", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "8.548" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Sakalti_SJT-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6535 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5282 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2538 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3297 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.408 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4266 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sakalti_sjt-900m.json b/data/models/sakalti_sjt-900m.json deleted file mode 100644 index 0032b33c4ea0e643651b21713fba16659df14505..0000000000000000000000000000000000000000 --- a/data/models/sakalti_sjt-900m.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SJT-900M", - "id": "Sakalti/SJT-900M", - "developer": "Sakalti", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.899" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Sakalti_SJT-900M/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.241 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3169 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0136 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2534 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3595 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1142 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sakalti_sjt-moe2x7.5b.json b/data/models/sakalti_sjt-moe2x7.5b.json deleted file mode 100644 index bca103aa956e48642622a0e08a2706ab32ba742d..0000000000000000000000000000000000000000 --- a/data/models/sakalti_sjt-moe2x7.5b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SJT-Moe2x7.5B", - "id": "Sakalti/SJT-Moe2x7.5B", - "developer": "Sakalti", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MixtralForCausalLM", - "params_billions": "13.401" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Sakalti_SJT-Moe2x7.5B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4117 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5371 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2145 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3263 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4399 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3954 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sakalti_sjtpass-2.json b/data/models/sakalti_sjtpass-2.json deleted file mode 100644 index 16b2fab267bf8d3e73308fd36b435749db9e8901..0000000000000000000000000000000000000000 --- a/data/models/sakalti_sjtpass-2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SJTPass-2", - "id": "Sakalti/SJTPass-2", - "developer": "Sakalti", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Sakalti_SJTPass-2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.24 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3302 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0529 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2727 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3222 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1902 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sakalti_sjtpass-4.json b/data/models/sakalti_sjtpass-4.json deleted file mode 100644 index f16a82ffbb1de129207d7e9b17024370c8d80f4c..0000000000000000000000000000000000000000 --- a/data/models/sakalti_sjtpass-4.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SJTPass-4", - "id": "Sakalti/SJTPass-4", - "developer": "Sakalti", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.167" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Sakalti_SJTPass-4/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1913 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2964 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0023 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2601 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3898 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1083 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sakalti_sjtpass-5.json b/data/models/sakalti_sjtpass-5.json deleted file mode 100644 index 68a2d5d3692312448940f39009242ab2bb1eb677..0000000000000000000000000000000000000000 --- a/data/models/sakalti_sjtpass-5.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SJTPass-5", - "id": "Sakalti/SJTPass-5", - "developer": "Sakalti", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.809" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Sakalti_SJTPass-5/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2425 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3103 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0159 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2668 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3794 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1327 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sakalti_tara-3.8b-v1.1.json b/data/models/sakalti_tara-3.8b-v1.1.json deleted file mode 100644 index 0f1bad4b642ffc2d3b90cd6a118a1d53064e00e7..0000000000000000000000000000000000000000 --- a/data/models/sakalti_tara-3.8b-v1.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Tara-3.8B-v1.1", - "id": "Sakalti/Tara-3.8B-v1.1", - "developer": "Sakalti", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.821" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Sakalti_Tara-3.8B-v1.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4062 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4886 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1156 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2945 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.478 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3281 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sakalti_tara-3.8b.json b/data/models/sakalti_tara-3.8b.json deleted file mode 100644 index a07449eae642cf8f1987691973bee940209cd515..0000000000000000000000000000000000000000 --- a/data/models/sakalti_tara-3.8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "tara-3.8B", - "id": "Sakalti/tara-3.8B", - "developer": "Sakalti", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.821" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Sakalti_tara-3.8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4077 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4886 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1156 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2945 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.478 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3281 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sakalti_ultiima-14b-v0.2.json b/data/models/sakalti_ultiima-14b-v0.2.json deleted file mode 100644 index 9f0f337b1d44e8267796918ddb856a5b8e802c1a..0000000000000000000000000000000000000000 --- a/data/models/sakalti_ultiima-14b-v0.2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ultiima-14B-v0.2", - "id": "Sakalti/ultiima-14B-v0.2", - "developer": "Sakalti", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Sakalti_ultiima-14B-v0.2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.707 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6472 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3995 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3826 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4794 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5387 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sakalti_ultiima-14b-v0.3.json b/data/models/sakalti_ultiima-14b-v0.3.json deleted file mode 100644 index bf6820835ec7dd47fb9dcff5ed3c0e565dc9a340..0000000000000000000000000000000000000000 --- a/data/models/sakalti_ultiima-14b-v0.3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ultiima-14B-v0.3", - "id": "Sakalti/ultiima-14B-v0.3", - "developer": "Sakalti", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Sakalti_ultiima-14B-v0.3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.704 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6398 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3965 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3767 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4754 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5337 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sakalti_ultiima-14b-v0.4.json b/data/models/sakalti_ultiima-14b-v0.4.json deleted file mode 100644 index f39650882f6626afb7bd55312a7572ae3a3ab1d2..0000000000000000000000000000000000000000 --- a/data/models/sakalti_ultiima-14b-v0.4.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ultiima-14B-v0.4", - "id": "Sakalti/ultiima-14B-v0.4", - "developer": "Sakalti", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Sakalti_ultiima-14B-v0.4/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3008 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.642 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3535 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.396 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4886 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5278 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sakalti_ultiima-14b.json b/data/models/sakalti_ultiima-14b.json deleted file mode 100644 index 6387413aa84527bb79446705dfb88b5328f4be38..0000000000000000000000000000000000000000 --- a/data/models/sakalti_ultiima-14b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ultiima-14B", - "id": "Sakalti/ultiima-14B", - "developer": "Sakalti", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Sakalti_ultiima-14B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5701 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6491 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4698 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3742 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4718 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5381 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sakalti_ultiima-32b.json b/data/models/sakalti_ultiima-32b.json deleted file mode 100644 index 38da8f050024dda3a57c333b1a6c642a9bbedb19..0000000000000000000000000000000000000000 --- a/data/models/sakalti_ultiima-32b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ultiima-32B", - "id": "Sakalti/ultiima-32B", - "developer": "Sakalti", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "32.764" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Sakalti_ultiima-32B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6854 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7037 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4962 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3809 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4995 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.591 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sakalti_ultiima-72b-v1.5.json b/data/models/sakalti_ultiima-72b-v1.5.json deleted file mode 100644 index 42e1be67cd4a9615139fe1ddd8d81f7d05f506b8..0000000000000000000000000000000000000000 --- a/data/models/sakalti_ultiima-72b-v1.5.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ultiima-72B-v1.5", - "id": "Sakalti/ultiima-72B-v1.5", - "developer": "Sakalti", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "72.706" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Sakalti_ultiima-72B-v1.5/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.655 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7392 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4396 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4136 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4691 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6054 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sakalti_ultiima-72b.json b/data/models/sakalti_ultiima-72b.json deleted file mode 100644 index af5b746a3dd7107999ff81b5f22f0fb701aacad0..0000000000000000000000000000000000000000 --- a/data/models/sakalti_ultiima-72b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ultiima-72B", - "id": "Sakalti/ultiima-72B", - "developer": "Sakalti", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "72.706" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Sakalti_ultiima-72B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.714 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7218 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5355 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4144 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4652 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5906 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sakhan10_quantized_open_llama_3b_v2.json b/data/models/sakhan10_quantized_open_llama_3b_v2.json deleted file mode 100644 index c0b6a47c42d9d3bf41b3f18c333e045203f4d1d4..0000000000000000000000000000000000000000 --- a/data/models/sakhan10_quantized_open_llama_3b_v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "quantized_open_llama_3b_v2", - "id": "sakhan10/quantized_open_llama_3b_v2", - "developer": "sakhan10", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sakhan10_quantized_open_llama_3b_v2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1872 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.302 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2768 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3682 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1095 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/salesforce_llama-3-8b-sfr-iterative-dpo-r.json b/data/models/salesforce_llama-3-8b-sfr-iterative-dpo-r.json deleted file mode 100644 index 9d208dbde4554840ea899722b78f210ffe2d9cc1..0000000000000000000000000000000000000000 --- a/data/models/salesforce_llama-3-8b-sfr-iterative-dpo-r.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "LLaMA-3-8B-SFR-Iterative-DPO-R", - "id": "Salesforce/LLaMA-3-8B-SFR-Iterative-DPO-R", - "developer": "Salesforce", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Salesforce_LLaMA-3-8B-SFR-Iterative-DPO-R/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3816 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5012 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0914 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2878 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3633 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3172 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/salesforce_sfr-llama-3.1-70b-judge-r.json b/data/models/salesforce_sfr-llama-3.1-70b-judge-r.json deleted file mode 100644 index 91a6cecbd36916c792af2d0fab8a5dd63e6a1f1e..0000000000000000000000000000000000000000 --- a/data/models/salesforce_sfr-llama-3.1-70b-judge-r.json +++ /dev/null @@ -1,126 +0,0 @@ -{ - "model_info": { - "name": "Salesforce/SFR-LLaMa-3.1-70B-Judge-r", - "id": "Salesforce/SFR-LLaMa-3.1-70B-Judge-r", - "developer": "Salesforce", - "additional_details": { - "model_type": "Generative" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/Salesforce_SFR-LLaMa-3.1-70B-Judge-r/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9272 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9693 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8476 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9162 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9757 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/salesforce_sfr-llama-3.1-8b-judge-r.json b/data/models/salesforce_sfr-llama-3.1-8b-judge-r.json deleted file mode 100644 index 9cfd9c528529b073c55092ca4329a68f20e94f53..0000000000000000000000000000000000000000 --- a/data/models/salesforce_sfr-llama-3.1-8b-judge-r.json +++ /dev/null @@ -1,126 +0,0 @@ -{ - "model_info": { - "name": "Salesforce/SFR-LLaMa-3.1-8B-Judge-r", - "id": "Salesforce/SFR-LLaMa-3.1-8B-Judge-r", - "developer": "Salesforce", - "additional_details": { - "model_type": "Generative" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/Salesforce_SFR-LLaMa-3.1-8B-Judge-r/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8865 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9553 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7774 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8622 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9513 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/salesforce_sfr-nemo-12b-judge-r.json b/data/models/salesforce_sfr-nemo-12b-judge-r.json deleted file mode 100644 index 3fc9e2dca3b5c6d55f401803f64cca666a2815c5..0000000000000000000000000000000000000000 --- a/data/models/salesforce_sfr-nemo-12b-judge-r.json +++ /dev/null @@ -1,126 +0,0 @@ -{ - "model_info": { - "name": "Salesforce/SFR-nemo-12B-Judge-r", - "id": "Salesforce/SFR-nemo-12B-Judge-r", - "developer": "Salesforce", - "additional_details": { - "model_type": "Generative" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/Salesforce_SFR-nemo-12B-Judge-r/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9027 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9721 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8224 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8649 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9513 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/salesforce_xlam-2-1b-fc-r-fc.json b/data/models/salesforce_xlam-2-1b-fc-r-fc.json deleted file mode 100644 index 45f7fd095b7f2b2e24683e07afde26e396f5cace..0000000000000000000000000000000000000000 --- a/data/models/salesforce_xlam-2-1b-fc-r-fc.json +++ /dev/null @@ -1,851 +0,0 @@ -{ - "model_info": { - "name": "xLAM-2-1b-fc-r (FC)", - "id": "salesforce/xlam-2-1b-fc-r-fc", - "developer": "salesforce", - "additional_details": { - "raw_model_name": "xLAM-2-1b-fc-r (FC)", - "organization": "Salesforce", - "license": "cc-by-nc-4.0", - "mode": "FC", - "model_link": "https://huggingface.co/Salesforce/xLAM-2-1b-fc-r" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/salesforce/xlam-2-1b-fc-r-fc/1775236112.399987", - "retrieved_timestamp": "1775236112.399987", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 65.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 30.44 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 2.79 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 2.84 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 2.35 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 6.52 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 69.04 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 64.17 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 82.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 73.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 56.0 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 55.14 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 68.22 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 52.8 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 43.75 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 25.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 36.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 45.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 36.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 37.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 25.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 3.87 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 3.87 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 3.87 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 3.87 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 87.5 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 64.47 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/salesforce_xlam-2-32b-fc-r-fc.json b/data/models/salesforce_xlam-2-32b-fc-r-fc.json deleted file mode 100644 index a631efb29cf76523cb7571c3c335396563b75d98..0000000000000000000000000000000000000000 --- a/data/models/salesforce_xlam-2-32b-fc-r-fc.json +++ /dev/null @@ -1,851 +0,0 @@ -{ - "model_info": { - "name": "xLAM-2-32b-fc-r (FC)", - "id": "salesforce/xlam-2-32b-fc-r-fc", - "developer": "salesforce", - "additional_details": { - "raw_model_name": "xLAM-2-32b-fc-r (FC)", - "organization": "Salesforce", - "license": "cc-by-nc-4.0", - "mode": "FC", - "model_link": "https://huggingface.co/Salesforce/xLAM-2-32b-fc-r" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/salesforce/xlam-2-32b-fc-r-fc/1775236112.3748028", - "retrieved_timestamp": "1775236112.3748028", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 18.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 54.66 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 6.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 6.94 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 8.21 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 17.66 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 89.6 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 80.42 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 94.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 93.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 91.0 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 75.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 82.17 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 74.64 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 50.0 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 58.33 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 69.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 81.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 72.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 67.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 56.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 25.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 37.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 14.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 20.86 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 6.45 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 10.32 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 45.81 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 81.25 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 80.23 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/salesforce_xlam-2-3b-fc-r-fc.json b/data/models/salesforce_xlam-2-3b-fc-r-fc.json deleted file mode 100644 index 381caf741b815acf0eb0ed90a97335f0f0fa615d..0000000000000000000000000000000000000000 --- a/data/models/salesforce_xlam-2-3b-fc-r-fc.json +++ /dev/null @@ -1,851 +0,0 @@ -{ - "model_info": { - "name": "xLAM-2-3b-fc-r (FC)", - "id": "salesforce/xlam-2-3b-fc-r-fc", - "developer": "salesforce", - "additional_details": { - "raw_model_name": "xLAM-2-3b-fc-r (FC)", - "organization": "Salesforce", - "license": "cc-by-nc-4.0", - "mode": "FC", - "model_link": "https://huggingface.co/Salesforce/xLAM-2-3b-fc-r" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/salesforce/xlam-2-3b-fc-r-fc/1775236112.386818", - "retrieved_timestamp": "1775236112.386818", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 42.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 41.22 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 3.36 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 3.8 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 3.59 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 8.79 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 82.96 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 75.33 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 91.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 86.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 79.0 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 62.92 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 73.26 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 60.68 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 62.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 50.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 58.38 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 71.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 59.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 57.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 45.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 2.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 3.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 2.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 11.4 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 5.81 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 5.81 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 22.58 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 87.5 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 63.45 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/salesforce_xlam-2-70b-fc-r-fc.json b/data/models/salesforce_xlam-2-70b-fc-r-fc.json deleted file mode 100644 index 0adc3913a80fdffdc8b725c4957f25facec97bbe..0000000000000000000000000000000000000000 --- a/data/models/salesforce_xlam-2-70b-fc-r-fc.json +++ /dev/null @@ -1,851 +0,0 @@ -{ - "model_info": { - "name": "xLAM-2-70b-fc-r (FC)", - "id": "salesforce/xlam-2-70b-fc-r-fc", - "developer": "salesforce", - "additional_details": { - "raw_model_name": "xLAM-2-70b-fc-r (FC)", - "organization": "Salesforce", - "license": "cc-by-nc-4.0", - "mode": "FC", - "model_link": "https://huggingface.co/Salesforce/Llama-xLAM-2-70b-fc-r" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/salesforce/xlam-2-70b-fc-r-fc/1775236112.376781", - "retrieved_timestamp": "1775236112.376781", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 22.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 53.07 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 25.1 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 28.06 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 68.77 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 91.21 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 88.44 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 78.25 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 94.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 92.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 89.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 72.17 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 77.91 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 71.13 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 68.75 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 58.33 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 77.38 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 82.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 77.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 74.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 76.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 15.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 17.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 13.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 14.41 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 2.58 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 10.97 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 29.68 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 75.0 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 79.11 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/salesforce_xlam-2-8b-fc-r-fc.json b/data/models/salesforce_xlam-2-8b-fc-r-fc.json deleted file mode 100644 index 0a8360f74c089003a3fea8fe89cdba3d812ff0c8..0000000000000000000000000000000000000000 --- a/data/models/salesforce_xlam-2-8b-fc-r-fc.json +++ /dev/null @@ -1,851 +0,0 @@ -{ - "model_info": { - "name": "xLAM-2-8b-fc-r (FC)", - "id": "salesforce/xlam-2-8b-fc-r-fc", - "developer": "salesforce", - "additional_details": { - "raw_model_name": "xLAM-2-8b-fc-r (FC)", - "organization": "Salesforce", - "license": "cc-by-nc-4.0", - "mode": "FC", - "model_link": "https://huggingface.co/Salesforce/Llama-xLAM-2-8b-fc-r" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/salesforce/xlam-2-8b-fc-r-fc/1775236112.382765", - "retrieved_timestamp": "1775236112.382765", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 34.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 46.68 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 20.92 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 22.65 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 46.92 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 108.81 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 84.58 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 73.83 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 93.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 87.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 83.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 67.95 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 75.58 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 66.57 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 56.25 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 54.17 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 70.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 76.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 72.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 65.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 67.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 6.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 11.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 2.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 13.98 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 5.81 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 15.48 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 20.65 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 87.5 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 63.28 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/saltlux_luxia-21.4b-alignment-v1.0.json b/data/models/saltlux_luxia-21.4b-alignment-v1.0.json deleted file mode 100644 index 764e3179e149cef9ee19129cab325c778bbbadf2..0000000000000000000000000000000000000000 --- a/data/models/saltlux_luxia-21.4b-alignment-v1.0.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "luxia-21.4b-alignment-v1.0", - "id": "saltlux/luxia-21.4b-alignment-v1.0", - "developer": "saltlux", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "21.421" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/saltlux_luxia-21.4b-alignment-v1.0/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3693 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6373 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0974 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3012 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4328 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3403 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/saltlux_luxia-21.4b-alignment-v1.2.json b/data/models/saltlux_luxia-21.4b-alignment-v1.2.json deleted file mode 100644 index 7dded8150283534f4b6d9200bb3a16d0a11541d0..0000000000000000000000000000000000000000 --- a/data/models/saltlux_luxia-21.4b-alignment-v1.2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "luxia-21.4b-alignment-v1.2", - "id": "saltlux/luxia-21.4b-alignment-v1.2", - "developer": "saltlux", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "21.421" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/saltlux_luxia-21.4b-alignment-v1.2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4115 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6371 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0846 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3079 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4459 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3473 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sam-paech_darkest-muse-v1.json b/data/models/sam-paech_darkest-muse-v1.json deleted file mode 100644 index d8de89dd88a7b031c3c3c6ed4f74a39b79980d69..0000000000000000000000000000000000000000 --- a/data/models/sam-paech_darkest-muse-v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Darkest-muse-v1", - "id": "sam-paech/Darkest-muse-v1", - "developer": "sam-paech", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sam-paech_Darkest-muse-v1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7344 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5968 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2145 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.344 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4502 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4184 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sam-paech_delirium-v1.json b/data/models/sam-paech_delirium-v1.json deleted file mode 100644 index a662649a61b6ebb68e6b27406876dce63eabd503..0000000000000000000000000000000000000000 --- a/data/models/sam-paech_delirium-v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Delirium-v1", - "id": "sam-paech/Delirium-v1", - "developer": "sam-paech", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "9.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sam-paech_Delirium-v1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7208 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5962 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2107 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3431 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4514 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.419 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sam-paech_quill-v1.json b/data/models/sam-paech_quill-v1.json deleted file mode 100644 index 33c3d5948cbb57770db690e31c80713440ecdf28..0000000000000000000000000000000000000000 --- a/data/models/sam-paech_quill-v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Quill-v1", - "id": "sam-paech/Quill-v1", - "developer": "sam-paech", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "9.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sam-paech_Quill-v1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7122 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5969 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2122 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3398 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4555 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4171 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sanjiwatsuki_kunoichi-dpo-v2-7b.json b/data/models/sanjiwatsuki_kunoichi-dpo-v2-7b.json deleted file mode 100644 index a4df772cad37098ab47c3bbe05d0bae5a15c56f5..0000000000000000000000000000000000000000 --- a/data/models/sanjiwatsuki_kunoichi-dpo-v2-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Kunoichi-DPO-v2-7B", - "id": "SanjiWatsuki/Kunoichi-DPO-v2-7B", - "developer": "SanjiWatsuki", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/SanjiWatsuki_Kunoichi-DPO-v2-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5431 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4416 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0763 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2961 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4188 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3107 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sanjiwatsuki_silicon-maid-7b.json b/data/models/sanjiwatsuki_silicon-maid-7b.json deleted file mode 100644 index c45ec0881d9eb4928ab4f0c398a613e42afe7ee3..0000000000000000000000000000000000000000 --- a/data/models/sanjiwatsuki_silicon-maid-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Silicon-Maid-7B", - "id": "SanjiWatsuki/Silicon-Maid-7B", - "developer": "SanjiWatsuki", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/SanjiWatsuki_Silicon-Maid-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5368 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4128 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.065 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2903 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4188 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3083 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sao10k_70b-l3.3-cirrus-x1.json b/data/models/sao10k_70b-l3.3-cirrus-x1.json deleted file mode 100644 index 4b8f35af415986c5183d0d59fb1e98b60990d0cb..0000000000000000000000000000000000000000 --- a/data/models/sao10k_70b-l3.3-cirrus-x1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "70B-L3.3-Cirrus-x1", - "id": "Sao10K/70B-L3.3-Cirrus-x1", - "developer": "Sao10K", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "70.554" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Sao10K_70B-L3.3-Cirrus-x1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6681 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7029 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3739 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4497 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4842 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5378 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sao10k_fimbulvetr-11b-v2.json b/data/models/sao10k_fimbulvetr-11b-v2.json deleted file mode 100644 index ae78de5b9a168d7e4c6ad7f164bad8bb5188e914..0000000000000000000000000000000000000000 --- a/data/models/sao10k_fimbulvetr-11b-v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Fimbulvetr-11B-v2", - "id": "Sao10K/Fimbulvetr-11B-v2", - "developer": "Sao10K", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "10.732" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Sao10K_Fimbulvetr-11B-v2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.51 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4544 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.068 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2919 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4354 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3301 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sao10k_l3-70b-euryale-v2.1.json b/data/models/sao10k_l3-70b-euryale-v2.1.json deleted file mode 100644 index c24d09a959df8d13e2602095ebcf1b7a3f8d330c..0000000000000000000000000000000000000000 --- a/data/models/sao10k_l3-70b-euryale-v2.1.json +++ /dev/null @@ -1,275 +0,0 @@ -{ - "model_info": { - "name": "L3-70B-Euryale-v2.1", - "id": "Sao10K/L3-70B-Euryale-v2.1", - "developer": "Sao10K", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "70.554" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Sao10K_L3-70B-Euryale-v2.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7384 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6471 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2137 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3314 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4209 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5104 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "hfopenllm_v2/Sao10K_L3-70B-Euryale-v2.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7281 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6503 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2243 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3314 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4196 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5096 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sao10k_l3-8b-lunaris-v1.json b/data/models/sao10k_l3-8b-lunaris-v1.json deleted file mode 100644 index 3bd4bb5800382920ff9187082728711a8b0f9b99..0000000000000000000000000000000000000000 --- a/data/models/sao10k_l3-8b-lunaris-v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "L3-8B-Lunaris-v1", - "id": "Sao10K/L3-8B-Lunaris-v1", - "developer": "Sao10K", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Sao10K_L3-8B-Lunaris-v1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6895 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5235 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0906 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3012 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3727 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3787 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sao10k_l3-8b-niitama-v1.json b/data/models/sao10k_l3-8b-niitama-v1.json deleted file mode 100644 index 7f567dff1c0d7251e53308993cb1df77ffa41c0a..0000000000000000000000000000000000000000 --- a/data/models/sao10k_l3-8b-niitama-v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "L3-8B-Niitama-v1", - "id": "Sao10K/L3-8B-Niitama-v1", - "developer": "Sao10K", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Sao10K_L3-8B-Niitama-v1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6791 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5303 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0982 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3079 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3807 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3701 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sao10k_l3-8b-stheno-v3.2.json b/data/models/sao10k_l3-8b-stheno-v3.2.json deleted file mode 100644 index 1550a1b8f1260ae1e0ccac32d9c1bd6a18627b5e..0000000000000000000000000000000000000000 --- a/data/models/sao10k_l3-8b-stheno-v3.2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "L3-8B-Stheno-v3.2", - "id": "Sao10K/L3-8B-Stheno-v3.2", - "developer": "Sao10K", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Sao10K_L3-8B-Stheno-v3.2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6873 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5228 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0929 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3104 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3794 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3768 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sao10k_l3-8b-stheno-v3.3-32k.json b/data/models/sao10k_l3-8b-stheno-v3.3-32k.json deleted file mode 100644 index 5f397e27e77d13432479b6fc8008cfd1e9fe2e50..0000000000000000000000000000000000000000 --- a/data/models/sao10k_l3-8b-stheno-v3.3-32k.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "L3-8B-Stheno-v3.3-32K", - "id": "Sao10K/L3-8B-Stheno-v3.3-32K", - "developer": "Sao10K", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Sao10K_L3-8B-Stheno-v3.3-32K/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4604 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3844 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0144 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2567 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3725 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1896 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sao10k_mn-12b-lyra-v3.json b/data/models/sao10k_mn-12b-lyra-v3.json deleted file mode 100644 index aa4641abd2866a5cfe483a89c07fe0b27275c6d7..0000000000000000000000000000000000000000 --- a/data/models/sao10k_mn-12b-lyra-v3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MN-12B-Lyra-v3", - "id": "Sao10K/MN-12B-Lyra-v3", - "developer": "Sao10K", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Sao10K_MN-12B-Lyra-v3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4486 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4804 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0937 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2777 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4019 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3249 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sarvamai_openhathi-7b-hi-v0.1-base.json b/data/models/sarvamai_openhathi-7b-hi-v0.1-base.json deleted file mode 100644 index ab5ec66fb1710df2ad8b5ef1e3d6445967a25329..0000000000000000000000000000000000000000 --- a/data/models/sarvamai_openhathi-7b-hi-v0.1-base.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "OpenHathi-7B-Hi-v0.1-Base", - "id": "sarvamai/OpenHathi-7B-Hi-v0.1-Base", - "developer": "sarvamai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "6.87" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sarvamai_OpenHathi-7B-Hi-v0.1-Base/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1804 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3354 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0083 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2534 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3658 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1543 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/saxo_linkbricks-horizon-ai-avengers-v1-32b.json b/data/models/saxo_linkbricks-horizon-ai-avengers-v1-32b.json deleted file mode 100644 index 6a8cce025eaef254ba71256b7fa737bd1ddd7ea8..0000000000000000000000000000000000000000 --- a/data/models/saxo_linkbricks-horizon-ai-avengers-v1-32b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Linkbricks-Horizon-AI-Avengers-V1-32B", - "id": "Saxo/Linkbricks-Horizon-AI-Avengers-V1-32B", - "developer": "Saxo", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "32.76" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Saxo_Linkbricks-Horizon-AI-Avengers-V1-32B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7972 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7001 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6027 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3624 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4538 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5793 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/saxo_linkbricks-horizon-ai-avengers-v2-32b.json b/data/models/saxo_linkbricks-horizon-ai-avengers-v2-32b.json deleted file mode 100644 index ce899d9b31550f03bf3683d130d24769c7f953df..0000000000000000000000000000000000000000 --- a/data/models/saxo_linkbricks-horizon-ai-avengers-v2-32b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Linkbricks-Horizon-AI-Avengers-V2-32B", - "id": "Saxo/Linkbricks-Horizon-AI-Avengers-V2-32B", - "developer": "Saxo", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "32.76" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Saxo_Linkbricks-Horizon-AI-Avengers-V2-32B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7956 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7023 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5665 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2659 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4166 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.572 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/saxo_linkbricks-horizon-ai-avengers-v3-32b.json b/data/models/saxo_linkbricks-horizon-ai-avengers-v3-32b.json deleted file mode 100644 index f8da2f34f6fb70d420255bf5c029028611eafdb2..0000000000000000000000000000000000000000 --- a/data/models/saxo_linkbricks-horizon-ai-avengers-v3-32b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Linkbricks-Horizon-AI-Avengers-V3-32B", - "id": "Saxo/Linkbricks-Horizon-AI-Avengers-V3-32B", - "developer": "Saxo", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "32.764" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Saxo_Linkbricks-Horizon-AI-Avengers-V3-32B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8249 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6913 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6178 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3381 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4275 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5664 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/saxo_linkbricks-horizon-ai-avengers-v4-32b.json b/data/models/saxo_linkbricks-horizon-ai-avengers-v4-32b.json deleted file mode 100644 index f2476dc361e61b0b6537777d1ae493bb3e51a587..0000000000000000000000000000000000000000 --- a/data/models/saxo_linkbricks-horizon-ai-avengers-v4-32b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Linkbricks-Horizon-AI-Avengers-V4-32B", - "id": "Saxo/Linkbricks-Horizon-AI-Avengers-V4-32B", - "developer": "Saxo", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "32.764" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Saxo_Linkbricks-Horizon-AI-Avengers-V4-32B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7631 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.692 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5363 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3616 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4643 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5752 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/saxo_linkbricks-horizon-ai-avengers-v5-32b.json b/data/models/saxo_linkbricks-horizon-ai-avengers-v5-32b.json deleted file mode 100644 index 660bddc5ebcc19eae1c7bad492c1a9abdf2d071a..0000000000000000000000000000000000000000 --- a/data/models/saxo_linkbricks-horizon-ai-avengers-v5-32b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Linkbricks-Horizon-AI-Avengers-V5-32B", - "id": "Saxo/Linkbricks-Horizon-AI-Avengers-V5-32B", - "developer": "Saxo", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "32.764" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Saxo_Linkbricks-Horizon-AI-Avengers-V5-32B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7516 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6929 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5461 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3557 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4709 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5762 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/saxo_linkbricks-horizon-ai-avengers-v6-32b.json b/data/models/saxo_linkbricks-horizon-ai-avengers-v6-32b.json deleted file mode 100644 index 9c7fb49d2b34e73d2a5978201e775e4162ca44a8..0000000000000000000000000000000000000000 --- a/data/models/saxo_linkbricks-horizon-ai-avengers-v6-32b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Linkbricks-Horizon-AI-Avengers-V6-32B", - "id": "Saxo/Linkbricks-Horizon-AI-Avengers-V6-32B", - "developer": "Saxo", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "32.76" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Saxo_Linkbricks-Horizon-AI-Avengers-V6-32B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8209 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.689 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6224 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3347 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4274 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5672 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/saxo_linkbricks-horizon-ai-korean-avengers-v2-27b.json b/data/models/saxo_linkbricks-horizon-ai-korean-avengers-v2-27b.json deleted file mode 100644 index 2b58cb84a4e78865670de1090cec4eb870d766af..0000000000000000000000000000000000000000 --- a/data/models/saxo_linkbricks-horizon-ai-korean-avengers-v2-27b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Linkbricks-Horizon-AI-Korean-Avengers-V2-27B", - "id": "Saxo/Linkbricks-Horizon-AI-Korean-Avengers-V2-27B", - "developer": "Saxo", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "27.227" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Saxo_Linkbricks-Horizon-AI-Korean-Avengers-V2-27B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8146 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6463 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2802 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3473 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4139 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4599 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/saxo_linkbricks-horizon-ai-korean-avengers-v3-27b.json b/data/models/saxo_linkbricks-horizon-ai-korean-avengers-v3-27b.json deleted file mode 100644 index 468cd0fd694ac7055fc1b0ffc849eb709eb5e14b..0000000000000000000000000000000000000000 --- a/data/models/saxo_linkbricks-horizon-ai-korean-avengers-v3-27b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Linkbricks-Horizon-AI-Korean-Avengers-V3-27B", - "id": "Saxo/Linkbricks-Horizon-AI-Korean-Avengers-V3-27B", - "developer": "Saxo", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "27.227" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Saxo_Linkbricks-Horizon-AI-Korean-Avengers-V3-27B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8142 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6404 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2492 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3591 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4467 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4524 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/saxo_linkbricks-horizon-ai-korean-superb-22b.json b/data/models/saxo_linkbricks-horizon-ai-korean-superb-22b.json deleted file mode 100644 index 00470e0235be34b851ca88b70e3b7ce2ab998e24..0000000000000000000000000000000000000000 --- a/data/models/saxo_linkbricks-horizon-ai-korean-superb-22b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Linkbricks-Horizon-AI-Korean-Superb-22B", - "id": "Saxo/Linkbricks-Horizon-AI-Korean-Superb-22B", - "developer": "Saxo", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "22.247" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Saxo_Linkbricks-Horizon-AI-Korean-Superb-22B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6767 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5626 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2372 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3263 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3908 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3871 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/saxo_linkbricks-horizon-ai-korean-superb-27b.json b/data/models/saxo_linkbricks-horizon-ai-korean-superb-27b.json deleted file mode 100644 index 1756cd74fe20b40049040833d482c23a174a5101..0000000000000000000000000000000000000000 --- a/data/models/saxo_linkbricks-horizon-ai-korean-superb-27b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Linkbricks-Horizon-AI-Korean-Superb-27B", - "id": "Saxo/Linkbricks-Horizon-AI-Korean-Superb-27B", - "developer": "Saxo", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "27.227" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Saxo_Linkbricks-Horizon-AI-Korean-Superb-27B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7768 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6518 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2719 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3599 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4791 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4647 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/saxo_linkbricks-horizon-ai-superb-27b.json b/data/models/saxo_linkbricks-horizon-ai-superb-27b.json deleted file mode 100644 index e13328fc5cc82a2d939624da525a76f0215b7597..0000000000000000000000000000000000000000 --- a/data/models/saxo_linkbricks-horizon-ai-superb-27b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Linkbricks-Horizon-AI-Superb-27B", - "id": "Saxo/Linkbricks-Horizon-AI-Superb-27B", - "developer": "Saxo", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "27.227" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Saxo_Linkbricks-Horizon-AI-Superb-27B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7302 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6186 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2221 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3574 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.465 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.406 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/schnapss_testmerge-7b.json b/data/models/schnapss_testmerge-7b.json deleted file mode 100644 index 09dd8c1bf7e74b64f6e4d9f329511165a90af31c..0000000000000000000000000000000000000000 --- a/data/models/schnapss_testmerge-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "testmerge-7b", - "id": "schnapss/testmerge-7b", - "developer": "schnapss", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/schnapss_testmerge-7b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3922 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5187 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0687 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2961 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4686 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.306 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/schrieffer_llama-sarm-4b.json b/data/models/schrieffer_llama-sarm-4b.json deleted file mode 100644 index 20f2c7a70a5ee1b5139bc8b87f0089a2a509990e..0000000000000000000000000000000000000000 --- a/data/models/schrieffer_llama-sarm-4b.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "Schrieffer/Llama-SARM-4B", - "id": "Schrieffer/Llama-SARM-4B", - "developer": "Schrieffer", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/Schrieffer_Llama-SARM-4B/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7379 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6874 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4281 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6448 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9178 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9556 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7939 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sci-m-wang_deepseek-llm-7b-chat-sa-v0.1.json b/data/models/sci-m-wang_deepseek-llm-7b-chat-sa-v0.1.json deleted file mode 100644 index 4ae37f26b80e6786d5c257c45f6c04d03b7ef10b..0000000000000000000000000000000000000000 --- a/data/models/sci-m-wang_deepseek-llm-7b-chat-sa-v0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "deepseek-llm-7b-chat-sa-v0.1", - "id": "sci-m-wang/deepseek-llm-7b-chat-sa-v0.1", - "developer": "sci-m-wang", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "?", - "params_billions": "7.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sci-m-wang_deepseek-llm-7b-chat-sa-v0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4036 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3718 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0264 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2567 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4173 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2209 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sci-m-wang_mistral-7b-instruct-sa-v0.1.json b/data/models/sci-m-wang_mistral-7b-instruct-sa-v0.1.json deleted file mode 100644 index 0b099ab6dc2a132f1d26e2a4ebc0378f77dd34e1..0000000000000000000000000000000000000000 --- a/data/models/sci-m-wang_mistral-7b-instruct-sa-v0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mistral-7B-Instruct-sa-v0.1", - "id": "sci-m-wang/Mistral-7B-Instruct-sa-v0.1", - "developer": "sci-m-wang", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "?", - "params_billions": "14.483" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sci-m-wang_Mistral-7B-Instruct-sa-v0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4335 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3273 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0144 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2592 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.39 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2362 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sci-m-wang_phi-3-mini-4k-instruct-sa-v0.1.json b/data/models/sci-m-wang_phi-3-mini-4k-instruct-sa-v0.1.json deleted file mode 100644 index 27446464116d868cbdf2d8d2968ae3b508c5baa8..0000000000000000000000000000000000000000 --- a/data/models/sci-m-wang_phi-3-mini-4k-instruct-sa-v0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Phi-3-mini-4k-instruct-sa-v0.1", - "id": "sci-m-wang/Phi-3-mini-4k-instruct-sa-v0.1", - "developer": "sci-m-wang", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "?", - "params_billions": "7.642" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sci-m-wang_Phi-3-mini-4k-instruct-sa-v0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5021 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5502 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.148 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3289 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4073 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3985 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/seallms_seallm-7b-v2.5.json b/data/models/seallms_seallm-7b-v2.5.json deleted file mode 100644 index 6c26534ffe1c47f7edb1bfb1ce2861ca740b7aa3..0000000000000000000000000000000000000000 --- a/data/models/seallms_seallm-7b-v2.5.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SeaLLM-7B-v2.5", - "id": "SeaLLMs/SeaLLM-7B-v2.5", - "developer": "SeaLLMs", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "GemmaForCausalLM", - "params_billions": "8.538" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/SeaLLMs_SeaLLM-7B-v2.5/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4522 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.498 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1088 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.276 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4203 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3203 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/seallms_seallm-7b-v2.json b/data/models/seallms_seallm-7b-v2.json deleted file mode 100644 index 1f27c260bbe0c4aa224815f8bc95c6b272dfd9b8..0000000000000000000000000000000000000000 --- a/data/models/seallms_seallm-7b-v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SeaLLM-7B-v2", - "id": "SeaLLMs/SeaLLM-7B-v2", - "developer": "SeaLLMs", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.376" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/SeaLLMs_SeaLLM-7B-v2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3671 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4902 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0853 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2785 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.407 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3083 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/seallms_seallms-v3-7b-chat.json b/data/models/seallms_seallms-v3-7b-chat.json deleted file mode 100644 index fb32cb18e57f57abf84dfa9499bf202c1985946d..0000000000000000000000000000000000000000 --- a/data/models/seallms_seallms-v3-7b-chat.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SeaLLMs-v3-7B-Chat", - "id": "SeaLLMs/SeaLLMs-v3-7B-Chat", - "developer": "SeaLLMs", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/SeaLLMs_SeaLLMs-v3-7B-Chat/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4377 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5266 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1858 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2987 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4174 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3895 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/securin_securin-llm-v2.5-qwen-1.5b.json b/data/models/securin_securin-llm-v2.5-qwen-1.5b.json deleted file mode 100644 index 822ffe214ba5e0d8da4ea4c579ba95f6468eafda..0000000000000000000000000000000000000000 --- a/data/models/securin_securin-llm-v2.5-qwen-1.5b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Securin-LLM-V2.5-Qwen-1.5B", - "id": "securin/Securin-LLM-V2.5-Qwen-1.5B", - "developer": "securin", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.543" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/securin_Securin-LLM-V2.5-Qwen-1.5B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1492 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3158 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0249 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.25 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3606 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1615 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/senseable_westlake-7b-v2.json b/data/models/senseable_westlake-7b-v2.json deleted file mode 100644 index 1cf5cf1f70e776ab1ca14bc41bda3cb98de2c05e..0000000000000000000000000000000000000000 --- a/data/models/senseable_westlake-7b-v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "WestLake-7B-v2", - "id": "senseable/WestLake-7B-v2", - "developer": "senseable", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/senseable_WestLake-7B-v2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4419 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4073 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0483 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2768 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3937 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2764 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sensellm_reflectioncoder-cl-34b.json b/data/models/sensellm_reflectioncoder-cl-34b.json deleted file mode 100644 index 1c293cbf053e4827b3830d474747254c51023c0f..0000000000000000000000000000000000000000 --- a/data/models/sensellm_reflectioncoder-cl-34b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ReflectionCoder-CL-34B", - "id": "SenseLLM/ReflectionCoder-CL-34B", - "developer": "SenseLLM", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "33.744" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/SenseLLM_ReflectionCoder-CL-34B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4008 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3953 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0332 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2508 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4155 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1424 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sensellm_reflectioncoder-ds-33b.json b/data/models/sensellm_reflectioncoder-ds-33b.json deleted file mode 100644 index 1547c56987d0ac86bbfcc818dd07885a5d8f29c3..0000000000000000000000000000000000000000 --- a/data/models/sensellm_reflectioncoder-ds-33b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ReflectionCoder-DS-33B", - "id": "SenseLLM/ReflectionCoder-DS-33B", - "developer": "SenseLLM", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "33.34" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/SenseLLM_ReflectionCoder-DS-33B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3787 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3449 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0302 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2743 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3343 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1202 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sentientagi_dobby-mini-leashed-llama-3.1-8b.json b/data/models/sentientagi_dobby-mini-leashed-llama-3.1-8b.json deleted file mode 100644 index 847fa23f136edce80f60792320bc15d743f426a2..0000000000000000000000000000000000000000 --- a/data/models/sentientagi_dobby-mini-leashed-llama-3.1-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Dobby-Mini-Leashed-Llama-3.1-8B", - "id": "SentientAGI/Dobby-Mini-Leashed-Llama-3.1-8B", - "developer": "SentientAGI", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/SentientAGI_Dobby-Mini-Leashed-Llama-3.1-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7847 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5138 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1858 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.302 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4254 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3694 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sentientagi_dobby-mini-unhinged-llama-3.1-8b.json b/data/models/sentientagi_dobby-mini-unhinged-llama-3.1-8b.json deleted file mode 100644 index 9615d75e072c57cfd49828e1102b6328376bc652..0000000000000000000000000000000000000000 --- a/data/models/sentientagi_dobby-mini-unhinged-llama-3.1-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Dobby-Mini-Unhinged-Llama-3.1-8B", - "id": "SentientAGI/Dobby-Mini-Unhinged-Llama-3.1-8B", - "developer": "SentientAGI", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/SentientAGI_Dobby-Mini-Unhinged-Llama-3.1-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7457 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5142 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1563 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3062 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4013 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3585 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/seppev_smollm_pretrained_with_sft_trained_with_1pc_data_on_a_preference_dpo.json b/data/models/seppev_smollm_pretrained_with_sft_trained_with_1pc_data_on_a_preference_dpo.json deleted file mode 100644 index 52eb09d5e1d9480462624898083a35345a7a9b6b..0000000000000000000000000000000000000000 --- a/data/models/seppev_smollm_pretrained_with_sft_trained_with_1pc_data_on_a_preference_dpo.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SmolLM_pretrained_with_sft_trained_with_1pc_data_on_a_preference_dpo", - "id": "SeppeV/SmolLM_pretrained_with_sft_trained_with_1pc_data_on_a_preference_dpo", - "developer": "SeppeV", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "0.135" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/SeppeV_SmolLM_pretrained_with_sft_trained_with_1pc_data_on_a_preference_dpo/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0955 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3073 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0121 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2592 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4032 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1161 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sequelbox_gemma-2-9b-moth.json b/data/models/sequelbox_gemma-2-9b-moth.json deleted file mode 100644 index f49f6f7c41f4bd0f380f0744182db4ca6667be0e..0000000000000000000000000000000000000000 --- a/data/models/sequelbox_gemma-2-9b-moth.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "gemma-2-9B-MOTH", - "id": "sequelbox/gemma-2-9B-MOTH", - "developer": "sequelbox", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "9.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sequelbox_gemma-2-9B-MOTH/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2059 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.308 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0106 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2601 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3409 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.114 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sequelbox_llama3.1-70b-plumchat.json b/data/models/sequelbox_llama3.1-70b-plumchat.json deleted file mode 100644 index 7397139939e20fb6c2320a4003b75d6ff11e6e4a..0000000000000000000000000000000000000000 --- a/data/models/sequelbox_llama3.1-70b-plumchat.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama3.1-70B-PlumChat", - "id": "sequelbox/Llama3.1-70B-PlumChat", - "developer": "sequelbox", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "70.554" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sequelbox_Llama3.1-70B-PlumChat/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5616 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6753 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3029 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3909 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4774 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5164 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sequelbox_llama3.1-8b-moth.json b/data/models/sequelbox_llama3.1-8b-moth.json deleted file mode 100644 index e39a3e718aa2d5637e01b2f5af360b782a775010..0000000000000000000000000000000000000000 --- a/data/models/sequelbox_llama3.1-8b-moth.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama3.1-8B-MOTH", - "id": "sequelbox/Llama3.1-8B-MOTH", - "developer": "sequelbox", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sequelbox_Llama3.1-8B-MOTH/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5245 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4902 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1216 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2685 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3689 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3339 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sequelbox_llama3.1-8b-plumchat.json b/data/models/sequelbox_llama3.1-8b-plumchat.json deleted file mode 100644 index 38454f51a2b9eb367c59f5bc834829eacca1fe23..0000000000000000000000000000000000000000 --- a/data/models/sequelbox_llama3.1-8b-plumchat.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama3.1-8B-PlumChat", - "id": "sequelbox/Llama3.1-8B-PlumChat", - "developer": "sequelbox", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sequelbox_Llama3.1-8B-PlumChat/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4243 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3873 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0363 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2651 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3755 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2127 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sequelbox_llama3.1-8b-plumcode.json b/data/models/sequelbox_llama3.1-8b-plumcode.json deleted file mode 100644 index f064a037e20ca512f9620188b8c122938fa18818..0000000000000000000000000000000000000000 --- a/data/models/sequelbox_llama3.1-8b-plumcode.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama3.1-8B-PlumCode", - "id": "sequelbox/Llama3.1-8B-PlumCode", - "developer": "sequelbox", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sequelbox_Llama3.1-8B-PlumCode/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2045 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3368 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0272 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.276 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3773 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2335 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sequelbox_llama3.1-8b-plummath.json b/data/models/sequelbox_llama3.1-8b-plummath.json deleted file mode 100644 index dae67285e0628c413b55658bd2f2e631d53b5ddc..0000000000000000000000000000000000000000 --- a/data/models/sequelbox_llama3.1-8b-plummath.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama3.1-8B-PlumMath", - "id": "sequelbox/Llama3.1-8B-PlumMath", - "developer": "sequelbox", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sequelbox_Llama3.1-8B-PlumMath/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2242 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4032 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0476 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.318 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3919 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2975 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sethuiyer_llama-3.1-8b-experimental-1206-instruct.json b/data/models/sethuiyer_llama-3.1-8b-experimental-1206-instruct.json deleted file mode 100644 index 5cf538c2e7f9e0e083205342d78311bcd8c734f2..0000000000000000000000000000000000000000 --- a/data/models/sethuiyer_llama-3.1-8b-experimental-1206-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.1-8B-Experimental-1206-Instruct", - "id": "sethuiyer/Llama-3.1-8B-Experimental-1206-Instruct", - "developer": "sethuiyer", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sethuiyer_Llama-3.1-8B-Experimental-1206-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6967 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5104 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1118 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2995 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3966 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3529 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sethuiyer_llama-3.1-8b-experimental-1208-instruct.json b/data/models/sethuiyer_llama-3.1-8b-experimental-1208-instruct.json deleted file mode 100644 index 69c6403b824f22010d1343b8042085e42581ebb8..0000000000000000000000000000000000000000 --- a/data/models/sethuiyer_llama-3.1-8b-experimental-1208-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.1-8B-Experimental-1208-Instruct", - "id": "sethuiyer/Llama-3.1-8B-Experimental-1208-Instruct", - "developer": "sethuiyer", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sethuiyer_Llama-3.1-8B-Experimental-1208-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.61 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4964 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0891 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2961 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.379 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3511 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sethuiyer_llamaverse-3.1-8b-instruct.json b/data/models/sethuiyer_llamaverse-3.1-8b-instruct.json deleted file mode 100644 index 34f1da5b6c003f66a2fb669e1756a6d7bac04111..0000000000000000000000000000000000000000 --- a/data/models/sethuiyer_llamaverse-3.1-8b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llamaverse-3.1-8B-Instruct", - "id": "sethuiyer/Llamaverse-3.1-8B-Instruct", - "developer": "sethuiyer", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sethuiyer_Llamaverse-3.1-8B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6185 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5414 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1858 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2911 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3762 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3523 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sethuiyer_llamazero-3.1-8b-experimental-1208.json b/data/models/sethuiyer_llamazero-3.1-8b-experimental-1208.json deleted file mode 100644 index 5a15d754b5840dd0a4f69373b00a00886fb73ab6..0000000000000000000000000000000000000000 --- a/data/models/sethuiyer_llamazero-3.1-8b-experimental-1208.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "LlamaZero-3.1-8B-Experimental-1208", - "id": "sethuiyer/LlamaZero-3.1-8B-Experimental-1208", - "developer": "sethuiyer", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sethuiyer_LlamaZero-3.1-8B-Experimental-1208/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6051 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4981 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.108 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2685 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.382 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sethuiyer_llamazing-3.1-8b-instruct.json b/data/models/sethuiyer_llamazing-3.1-8b-instruct.json deleted file mode 100644 index 3938f17e3eb05cc2047ecce9477058b93f8989e1..0000000000000000000000000000000000000000 --- a/data/models/sethuiyer_llamazing-3.1-8b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llamazing-3.1-8B-Instruct", - "id": "sethuiyer/Llamazing-3.1-8B-Instruct", - "developer": "sethuiyer", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sethuiyer_Llamazing-3.1-8B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5711 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5291 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0544 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3121 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3976 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3606 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sethuiyer_qwen2.5-7b-anvita.json b/data/models/sethuiyer_qwen2.5-7b-anvita.json deleted file mode 100644 index 9799949c3ce6bda36261477b9b72c1aa82b29dac..0000000000000000000000000000000000000000 --- a/data/models/sethuiyer_qwen2.5-7b-anvita.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-7B-Anvita", - "id": "sethuiyer/Qwen2.5-7B-Anvita", - "developer": "sethuiyer", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sethuiyer_Qwen2.5-7B-Anvita/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.648 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5466 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2017 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3272 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4337 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4166 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sf-foundation_texteval-llama3.1-70b.json b/data/models/sf-foundation_texteval-llama3.1-70b.json deleted file mode 100644 index 6054c16830a6e6b12ff01886e84104c96cf221f0..0000000000000000000000000000000000000000 --- a/data/models/sf-foundation_texteval-llama3.1-70b.json +++ /dev/null @@ -1,126 +0,0 @@ -{ - "model_info": { - "name": "SF-Foundation/TextEval-Llama3.1-70B", - "id": "SF-Foundation/TextEval-Llama3.1-70B", - "developer": "SF-Foundation", - "additional_details": { - "model_type": "Generative" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/SF-Foundation_TextEval-Llama3.1-70B/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9348 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9413 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9013 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9324 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9641 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sf-foundation_texteval-offsetbias-12b.json b/data/models/sf-foundation_texteval-offsetbias-12b.json deleted file mode 100644 index b678f4ea62f1fa1d9b0e69269568a9fe7f6929eb..0000000000000000000000000000000000000000 --- a/data/models/sf-foundation_texteval-offsetbias-12b.json +++ /dev/null @@ -1,126 +0,0 @@ -{ - "model_info": { - "name": "SF-Foundation/TextEval-OffsetBias-12B", - "id": "SF-Foundation/TextEval-OffsetBias-12B", - "developer": "SF-Foundation", - "additional_details": { - "model_type": "Generative" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/SF-Foundation_TextEval-OffsetBias-12B/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9105 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.919 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8662 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9203 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9365 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sfairxc_fsfairx-llama3-rm-v0.1.json b/data/models/sfairxc_fsfairx-llama3-rm-v0.1.json deleted file mode 100644 index 83ff30916d3f35c6333d5f652927c45d63836756..0000000000000000000000000000000000000000 --- a/data/models/sfairxc_fsfairx-llama3-rm-v0.1.json +++ /dev/null @@ -1,294 +0,0 @@ -{ - "model_info": { - "name": "sfairXC/FsfairX-LLaMA3-RM-v0.1", - "id": "sfairXC/FsfairX-LLaMA3-RM-v0.1", - "developer": "sfairXC", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/sfairXC_FsfairX-LLaMA3-RM-v0.1/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8338 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9944 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6513 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8676 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8644 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7492 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "reward-bench-2/sfairXC_FsfairX-LLaMA3-RM-v0.1/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6292 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5916 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4188 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6284 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7667 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7051 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6647 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/shadowml_beagsake-7b.json b/data/models/shadowml_beagsake-7b.json deleted file mode 100644 index 0c71e03992756450cccfe9dbb56f8ddf27744258..0000000000000000000000000000000000000000 --- a/data/models/shadowml_beagsake-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "BeagSake-7B", - "id": "shadowml/BeagSake-7B", - "developer": "shadowml", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/shadowml_BeagSake-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5216 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4711 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0506 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.281 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4124 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2585 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/shadowml_mixolar-4x7b.json b/data/models/shadowml_mixolar-4x7b.json deleted file mode 100644 index f9005e79fd7c3b3df3abc9bb5e5efa3c18432f9c..0000000000000000000000000000000000000000 --- a/data/models/shadowml_mixolar-4x7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mixolar-4x7b", - "id": "shadowml/Mixolar-4x7b", - "developer": "shadowml", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MixtralForCausalLM", - "params_billions": "36.099" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/shadowml_Mixolar-4x7b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3893 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5216 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0582 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2928 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4258 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3305 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sharathhebbar24_chat_gpt2_dpo.json b/data/models/sharathhebbar24_chat_gpt2_dpo.json deleted file mode 100644 index 0312b23fa07572532fe38c18d0b10baface2c0a1..0000000000000000000000000000000000000000 --- a/data/models/sharathhebbar24_chat_gpt2_dpo.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "chat_gpt2_dpo", - "id": "Sharathhebbar24/chat_gpt2_dpo", - "developer": "Sharathhebbar24", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "GPT2LMHeadModel", - "params_billions": "0.124" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Sharathhebbar24_chat_gpt2_dpo/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0986 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2902 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0053 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2601 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3818 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1142 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sharathhebbar24_ssh_355m.json b/data/models/sharathhebbar24_ssh_355m.json deleted file mode 100644 index 1ecc2aacade5f33c4554e52eedea6a16d10b87d0..0000000000000000000000000000000000000000 --- a/data/models/sharathhebbar24_ssh_355m.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SSH_355M", - "id": "Sharathhebbar24/SSH_355M", - "developer": "Sharathhebbar24", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "GPT2LMHeadModel", - "params_billions": "0.355" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Sharathhebbar24_SSH_355M/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1424 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3099 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0091 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2584 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4178 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1176 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/shastraai_shastra-llama2-math-commonsense-sft.json b/data/models/shastraai_shastra-llama2-math-commonsense-sft.json deleted file mode 100644 index 9e6e5992d3405e7e7d7b145de36e9fd3e2f451a1..0000000000000000000000000000000000000000 --- a/data/models/shastraai_shastra-llama2-math-commonsense-sft.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Shastra-LLAMA2-Math-Commonsense-SFT", - "id": "shastraai/Shastra-LLAMA2-Math-Commonsense-SFT", - "developer": "shastraai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "6.738" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/shastraai_Shastra-LLAMA2-Math-Commonsense-SFT/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3042 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3843 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0174 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2592 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3604 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1997 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/shikaichen_ldl-reward-gemma-2-27b-v0.1.json b/data/models/shikaichen_ldl-reward-gemma-2-27b-v0.1.json deleted file mode 100644 index 988f599afd76311b87e49f16160d0ca534436cdd..0000000000000000000000000000000000000000 --- a/data/models/shikaichen_ldl-reward-gemma-2-27b-v0.1.json +++ /dev/null @@ -1,276 +0,0 @@ -{ - "model_info": { - "name": "ShikaiChen/LDL-Reward-Gemma-2-27B-v0.1", - "id": "ShikaiChen/LDL-Reward-Gemma-2-27B-v0.1", - "developer": "ShikaiChen", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/ShikaiChen_LDL-Reward-Gemma-2-27B-v0.1/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7249 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7558 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.35 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6448 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9222 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9131 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7633 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "reward-bench/ShikaiChen_LDL-Reward-Gemma-2-27B-v0.1/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9499 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9637 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9079 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9378 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9903 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/shivam9980_mistral-7b-news-cnn-merged.json b/data/models/shivam9980_mistral-7b-news-cnn-merged.json deleted file mode 100644 index 534275075866728aade95bb2b1abec4b29fc5124..0000000000000000000000000000000000000000 --- a/data/models/shivam9980_mistral-7b-news-cnn-merged.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "mistral-7b-news-cnn-merged", - "id": "shivam9980/mistral-7b-news-cnn-merged", - "developer": "shivam9980", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "?", - "params_billions": "7.723" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/shivam9980_mistral-7b-news-cnn-merged/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4634 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3635 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0189 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3087 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4523 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2827 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/shivam9980_nepali-llm.json b/data/models/shivam9980_nepali-llm.json deleted file mode 100644 index 26a7ef9de6d90df7293162223a4f1557a75de821..0000000000000000000000000000000000000000 --- a/data/models/shivam9980_nepali-llm.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "NEPALI-LLM", - "id": "shivam9980/NEPALI-LLM", - "developer": "shivam9980", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.273" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/shivam9980_NEPALI-LLM/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0417 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3828 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0091 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2617 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4122 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2064 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/shivank21_mistral_dpo_self.json b/data/models/shivank21_mistral_dpo_self.json deleted file mode 100644 index 662a2e4aa00dba4c1d406673787bc71a1975dd05..0000000000000000000000000000000000000000 --- a/data/models/shivank21_mistral_dpo_self.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "mistral_dpo_self", - "id": "shivank21/mistral_dpo_self", - "developer": "shivank21", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "", - "params_billions": "7.913" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/shivank21_mistral_dpo_self/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3403 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3216 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0219 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2408 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3247 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2214 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/shreyash2010_uma-4x4b-instruct-v0.1.json b/data/models/shreyash2010_uma-4x4b-instruct-v0.1.json deleted file mode 100644 index 31f2b6ea8b0e2580af085b3444218af0d83eb851..0000000000000000000000000000000000000000 --- a/data/models/shreyash2010_uma-4x4b-instruct-v0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Uma-4x4B-Instruct-v0.1", - "id": "Shreyash2010/Uma-4x4B-Instruct-v0.1", - "developer": "Shreyash2010", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "?", - "params_billions": "3.821" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Shreyash2010_Uma-4x4B-Instruct-v0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5517 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5512 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1775 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3347 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4441 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.387 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/shuttleai_shuttle-3.json b/data/models/shuttleai_shuttle-3.json deleted file mode 100644 index 7c9e7e86c4ee1324804be589be987dec41ddc6d2..0000000000000000000000000000000000000000 --- a/data/models/shuttleai_shuttle-3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "shuttle-3", - "id": "shuttleai/shuttle-3", - "developer": "shuttleai", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "72.706" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/shuttleai_shuttle-3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8154 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.742 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.46 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4119 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4377 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5716 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/shyamieee_padma-v7.0.json b/data/models/shyamieee_padma-v7.0.json deleted file mode 100644 index 014178e02d67d3bb5ccf9f22b289b601458d48ed..0000000000000000000000000000000000000000 --- a/data/models/shyamieee_padma-v7.0.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Padma-v7.0", - "id": "shyamieee/Padma-v7.0", - "developer": "shyamieee", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/shyamieee_Padma-v7.0/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3841 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5119 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0702 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2861 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4386 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3029 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sicarius-prototyping_bacon_and_food.json b/data/models/sicarius-prototyping_bacon_and_food.json deleted file mode 100644 index e9bf5ccfa41032ccc39ba63a54a0b23b8dd682a3..0000000000000000000000000000000000000000 --- a/data/models/sicarius-prototyping_bacon_and_food.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "bacon_and_food", - "id": "Sicarius-Prototyping/bacon_and_food", - "developer": "Sicarius-Prototyping", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Sicarius-Prototyping_bacon_and_food/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.586 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4725 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0982 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3096 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3884 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3263 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sicarius-prototyping_brainy_llama.json b/data/models/sicarius-prototyping_brainy_llama.json deleted file mode 100644 index c6a58431b08a59097655377394271124b93c0d19..0000000000000000000000000000000000000000 --- a/data/models/sicarius-prototyping_brainy_llama.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Brainy_LLAMA", - "id": "Sicarius-Prototyping/Brainy_LLAMA", - "developer": "Sicarius-Prototyping", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Sicarius-Prototyping_Brainy_LLAMA/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5204 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5117 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1337 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3138 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4143 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3849 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sicarius-prototyping_micropenis_1b.json b/data/models/sicarius-prototyping_micropenis_1b.json deleted file mode 100644 index 90966b736bb9cf9b136f810d2cc02ab64f469004..0000000000000000000000000000000000000000 --- a/data/models/sicarius-prototyping_micropenis_1b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Micropenis_1B", - "id": "Sicarius-Prototyping/Micropenis_1B", - "developer": "Sicarius-Prototyping", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "0.618" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Sicarius-Prototyping_Micropenis_1B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3461 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3372 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0461 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2626 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3325 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.186 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sicariussicariistuff_2b-ad.json b/data/models/sicariussicariistuff_2b-ad.json deleted file mode 100644 index 6d6b76485cbc6f3b0ce982b4a47f4b0dbba333d7..0000000000000000000000000000000000000000 --- a/data/models/sicariussicariistuff_2b-ad.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "2B-ad", - "id": "SicariusSicariiStuff/2B-ad", - "developer": "SicariusSicariiStuff", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "3.204" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/SicariusSicariiStuff_2B-ad/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4379 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4092 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0506 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.281 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4015 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2662 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sicariussicariistuff_2b_or_not_2b.json b/data/models/sicariussicariistuff_2b_or_not_2b.json deleted file mode 100644 index e15cb2ef53dd518def3de0d1976cf06548289346..0000000000000000000000000000000000000000 --- a/data/models/sicariussicariistuff_2b_or_not_2b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "2B_or_not_2B", - "id": "SicariusSicariiStuff/2B_or_not_2B", - "developer": "SicariusSicariiStuff", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "GemmaForCausalLM", - "params_billions": "2.506" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/SicariusSicariiStuff_2B_or_not_2B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2062 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3416 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0196 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2475 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3791 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1399 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sicariussicariistuff_dn_ep02.json b/data/models/sicariussicariistuff_dn_ep02.json deleted file mode 100644 index caa71be3efcbe1c25bd7e92ab73ff3e3dc3154fe..0000000000000000000000000000000000000000 --- a/data/models/sicariussicariistuff_dn_ep02.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "dn_ep02", - "id": "SicariusSicariiStuff/dn_ep02", - "developer": "SicariusSicariiStuff", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/SicariusSicariiStuff_dn_ep02/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5064 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5266 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.142 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3154 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4316 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3998 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sicariussicariistuff_dusk_rainbow.json b/data/models/sicariussicariistuff_dusk_rainbow.json deleted file mode 100644 index fc65c58b7d6ed4c325a570856fdf0acbab1bb2e8..0000000000000000000000000000000000000000 --- a/data/models/sicariussicariistuff_dusk_rainbow.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Dusk_Rainbow", - "id": "SicariusSicariiStuff/Dusk_Rainbow", - "developer": "SicariusSicariiStuff", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/SicariusSicariiStuff_Dusk_Rainbow/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3588 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4772 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0748 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3087 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4025 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3443 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sicariussicariistuff_eximius_persona_5b.json b/data/models/sicariussicariistuff_eximius_persona_5b.json deleted file mode 100644 index 7f7116d5714e566b6616fe6abb27281f8aff1a98..0000000000000000000000000000000000000000 --- a/data/models/sicariussicariistuff_eximius_persona_5b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Eximius_Persona_5B", - "id": "SicariusSicariiStuff/Eximius_Persona_5B", - "developer": "SicariusSicariiStuff", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "5.821" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/SicariusSicariiStuff_Eximius_Persona_5B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.656 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4512 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.102 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2643 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3818 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.314 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sicariussicariistuff_impish_llama_3b.json b/data/models/sicariussicariistuff_impish_llama_3b.json deleted file mode 100644 index ad9856a040830e0dd5747e3a2ddb7e5caec8eead..0000000000000000000000000000000000000000 --- a/data/models/sicariussicariistuff_impish_llama_3b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Impish_LLAMA_3B", - "id": "SicariusSicariiStuff/Impish_LLAMA_3B", - "developer": "SicariusSicariiStuff", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/SicariusSicariiStuff_Impish_LLAMA_3B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.463 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4091 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1125 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2878 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3673 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2941 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sicariussicariistuff_impish_mind_8b.json b/data/models/sicariussicariistuff_impish_mind_8b.json deleted file mode 100644 index 1cf4d3d91b2500c4523727f9e4c6b1576bf3c965..0000000000000000000000000000000000000000 --- a/data/models/sicariussicariistuff_impish_mind_8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Impish_Mind_8B", - "id": "SicariusSicariiStuff/Impish_Mind_8B", - "developer": "SicariusSicariiStuff", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/SicariusSicariiStuff_Impish_Mind_8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3179 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4674 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.105 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3045 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.407 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3309 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sicariussicariistuff_impish_qwen_14b-1m.json b/data/models/sicariussicariistuff_impish_qwen_14b-1m.json deleted file mode 100644 index 7df16fdd92c5fc0293e7aa0b0813909ea1e9a24b..0000000000000000000000000000000000000000 --- a/data/models/sicariussicariistuff_impish_qwen_14b-1m.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Impish_QWEN_14B-1M", - "id": "SicariusSicariiStuff/Impish_QWEN_14B-1M", - "developer": "SicariusSicariiStuff", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/SicariusSicariiStuff_Impish_QWEN_14B-1M/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7868 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6283 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3965 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3507 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4615 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5044 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sicariussicariistuff_impish_qwen_7b-1m.json b/data/models/sicariussicariistuff_impish_qwen_7b-1m.json deleted file mode 100644 index 89c15ce68d63bf327d8c460f206a457313ce50a7..0000000000000000000000000000000000000000 --- a/data/models/sicariussicariistuff_impish_qwen_7b-1m.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Impish_QWEN_7B-1M", - "id": "SicariusSicariiStuff/Impish_QWEN_7B-1M", - "developer": "SicariusSicariiStuff", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/SicariusSicariiStuff_Impish_QWEN_7B-1M/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6382 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5372 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3089 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2961 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4074 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4265 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sicariussicariistuff_llama-3_8b_unaligned_beta.json b/data/models/sicariussicariistuff_llama-3_8b_unaligned_beta.json deleted file mode 100644 index 059c69ee86b81bc3a7fd6ccdf1e1f655b0ad26e8..0000000000000000000000000000000000000000 --- a/data/models/sicariussicariistuff_llama-3_8b_unaligned_beta.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "LLAMA-3_8B_Unaligned_BETA", - "id": "SicariusSicariiStuff/LLAMA-3_8B_Unaligned_BETA", - "developer": "SicariusSicariiStuff", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/SicariusSicariiStuff_LLAMA-3_8B_Unaligned_BETA/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3713 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4717 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0838 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3054 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4119 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3465 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sicariussicariistuff_phi-line_14b.json b/data/models/sicariussicariistuff_phi-line_14b.json deleted file mode 100644 index 99f7523d05bc364afed8c30247fd862f60e37ac7..0000000000000000000000000000000000000000 --- a/data/models/sicariussicariistuff_phi-line_14b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Phi-Line_14B", - "id": "SicariusSicariiStuff/Phi-Line_14B", - "developer": "SicariusSicariiStuff", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/SicariusSicariiStuff_Phi-Line_14B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6496 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6154 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.386 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3532 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4479 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5454 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sicariussicariistuff_phi-lthy4.json b/data/models/sicariussicariistuff_phi-lthy4.json deleted file mode 100644 index 18b0ad9b1e322792676e64674dbc36ea6f35bc30..0000000000000000000000000000000000000000 --- a/data/models/sicariussicariistuff_phi-lthy4.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Phi-lthy4", - "id": "SicariusSicariiStuff/Phi-lthy4", - "developer": "SicariusSicariiStuff", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "11.933" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/SicariusSicariiStuff_Phi-lthy4/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7679 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5879 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1367 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2869 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4083 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4333 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sicariussicariistuff_qwen2.5-14b_uncencored.json b/data/models/sicariussicariistuff_qwen2.5-14b_uncencored.json deleted file mode 100644 index c8579cc3a7e59a74313aab35a6883b3d55aab164..0000000000000000000000000000000000000000 --- a/data/models/sicariussicariistuff_qwen2.5-14b_uncencored.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-14B_Uncencored", - "id": "SicariusSicariiStuff/Qwen2.5-14B_Uncencored", - "developer": "SicariusSicariiStuff", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/SicariusSicariiStuff_Qwen2.5-14B_Uncencored/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3158 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6309 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.318 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3817 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4517 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5266 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sicariussicariistuff_qwen2.5-14b_uncensored.json b/data/models/sicariussicariistuff_qwen2.5-14b_uncensored.json deleted file mode 100644 index f1cba50d25e39153187842822eebcc12d079efa3..0000000000000000000000000000000000000000 --- a/data/models/sicariussicariistuff_qwen2.5-14b_uncensored.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-14B_Uncensored", - "id": "SicariusSicariiStuff/Qwen2.5-14B_Uncensored", - "developer": "SicariusSicariiStuff", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/SicariusSicariiStuff_Qwen2.5-14B_Uncensored/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3173 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6309 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.318 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3817 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4517 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5266 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sicariussicariistuff_qwen2.5-14b_uncensored_instruct.json b/data/models/sicariussicariistuff_qwen2.5-14b_uncensored_instruct.json deleted file mode 100644 index 94daf9cd762759e428859494c7fe5fb4fc33f199..0000000000000000000000000000000000000000 --- a/data/models/sicariussicariistuff_qwen2.5-14b_uncensored_instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-14B_Uncensored_Instruct", - "id": "SicariusSicariiStuff/Qwen2.5-14B_Uncensored_Instruct", - "developer": "SicariusSicariiStuff", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/SicariusSicariiStuff_Qwen2.5-14B_Uncensored_Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3789 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5937 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3285 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3297 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3697 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5127 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sicariussicariistuff_redemption_wind_24b.json b/data/models/sicariussicariistuff_redemption_wind_24b.json deleted file mode 100644 index 837201cd80f2aa4bf14f2316e779604d563a01ce..0000000000000000000000000000000000000000 --- a/data/models/sicariussicariistuff_redemption_wind_24b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Redemption_Wind_24B", - "id": "SicariusSicariiStuff/Redemption_Wind_24B", - "developer": "SicariusSicariiStuff", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "23.572" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/SicariusSicariiStuff_Redemption_Wind_24B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2501 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6428 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1858 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3834 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4262 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5432 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sicariussicariistuff_winged_imp_8b.json b/data/models/sicariussicariistuff_winged_imp_8b.json deleted file mode 100644 index 963389f0cb7dcc6c08a38be6334c4c43b6abfa46..0000000000000000000000000000000000000000 --- a/data/models/sicariussicariistuff_winged_imp_8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Winged_Imp_8B", - "id": "SicariusSicariiStuff/Winged_Imp_8B", - "developer": "SicariusSicariiStuff", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/SicariusSicariiStuff_Winged_Imp_8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.743 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.512 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1201 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2827 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4148 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3639 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sicariussicariistuff_wingless_imp_8b.json b/data/models/sicariussicariistuff_wingless_imp_8b.json deleted file mode 100644 index 920230d249eeff61e7e97fa2e54c3691e1ec3066..0000000000000000000000000000000000000000 --- a/data/models/sicariussicariistuff_wingless_imp_8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Wingless_Imp_8B", - "id": "SicariusSicariiStuff/Wingless_Imp_8B", - "developer": "SicariusSicariiStuff", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/SicariusSicariiStuff_Wingless_Imp_8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.743 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.512 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1201 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2827 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4148 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3639 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sicariussicariistuff_zion_alpha.json b/data/models/sicariussicariistuff_zion_alpha.json deleted file mode 100644 index 2f7156075ba0642feeddd28a01f897e5b9810b9f..0000000000000000000000000000000000000000 --- a/data/models/sicariussicariistuff_zion_alpha.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Zion_Alpha", - "id": "SicariusSicariiStuff/Zion_Alpha", - "developer": "SicariusSicariiStuff", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/SicariusSicariiStuff_Zion_Alpha/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3324 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4932 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0521 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2903 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4727 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3132 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/silma-ai_silma-9b-instruct-v1.0.json b/data/models/silma-ai_silma-9b-instruct-v1.0.json deleted file mode 100644 index c98f47da41a0aab717d17a7f6cb656113a363e12..0000000000000000000000000000000000000000 --- a/data/models/silma-ai_silma-9b-instruct-v1.0.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SILMA-9B-Instruct-v1.0", - "id": "silma-ai/SILMA-9B-Instruct-v1.0", - "developer": "silma-ai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "9.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/silma-ai_SILMA-9B-Instruct-v1.0/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5842 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5219 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1163 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3054 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4637 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.392 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/silma-ai_silma-kashif-2b-instruct-v1.0.json b/data/models/silma-ai_silma-kashif-2b-instruct-v1.0.json deleted file mode 100644 index 1004c7a4d9972daadb2c2de5626c6ccce79c2cf0..0000000000000000000000000000000000000000 --- a/data/models/silma-ai_silma-kashif-2b-instruct-v1.0.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SILMA-Kashif-2B-Instruct-v1.0", - "id": "silma-ai/SILMA-Kashif-2B-Instruct-v1.0", - "developer": "silma-ai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "2.614" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/silma-ai_SILMA-Kashif-2B-Instruct-v1.0/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1181 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3793 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0113 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2701 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4043 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2258 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/siqi00_mistral-7b-dft.json b/data/models/siqi00_mistral-7b-dft.json deleted file mode 100644 index 2c559a60920e6b18d234c91e5c213fee16d54c11..0000000000000000000000000000000000000000 --- a/data/models/siqi00_mistral-7b-dft.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mistral-7B-DFT", - "id": "siqi00/Mistral-7B-DFT", - "developer": "siqi00", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/siqi00_Mistral-7B-DFT/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5569 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4665 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0378 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3045 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4191 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2963 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/siqi00_mistral-7b-dft2.json b/data/models/siqi00_mistral-7b-dft2.json deleted file mode 100644 index c63e4d031ce3b4e1ce0196cbf6dcb48dcbafc7bd..0000000000000000000000000000000000000000 --- a/data/models/siqi00_mistral-7b-dft2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mistral-7B-DFT2", - "id": "siqi00/Mistral-7B-DFT2", - "developer": "siqi00", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/siqi00_Mistral-7B-DFT2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5804 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3968 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0453 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2995 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4401 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2852 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/skumar9_llama-medx_v2.json b/data/models/skumar9_llama-medx_v2.json deleted file mode 100644 index 5720a112f81e4951c6b16d651334071be51ec6c0..0000000000000000000000000000000000000000 --- a/data/models/skumar9_llama-medx_v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-medx_v2", - "id": "skumar9/Llama-medx_v2", - "developer": "skumar9", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/skumar9_Llama-medx_v2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4462 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4909 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0914 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3054 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3661 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3463 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/skymizer_llama2-7b-sft-chat-custom-template-dpo.json b/data/models/skymizer_llama2-7b-sft-chat-custom-template-dpo.json deleted file mode 100644 index 6536b9e60dc295588c3ae07beb016d5dd792de41..0000000000000000000000000000000000000000 --- a/data/models/skymizer_llama2-7b-sft-chat-custom-template-dpo.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama2-7b-sft-chat-custom-template-dpo", - "id": "skymizer/Llama2-7b-sft-chat-custom-template-dpo", - "developer": "skymizer", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "6.738" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/skymizer_Llama2-7b-sft-chat-custom-template-dpo/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2353 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3688 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0144 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2391 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4429 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1946 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/skyorbis_sky-ko-llama3.1-8b-lora-epoch1.json b/data/models/skyorbis_sky-ko-llama3.1-8b-lora-epoch1.json deleted file mode 100644 index 29970c256576c7bd3bcd0611c989e049a5c2fcef..0000000000000000000000000000000000000000 --- a/data/models/skyorbis_sky-ko-llama3.1-8b-lora-epoch1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SKY-Ko-Llama3.1-8B-lora-epoch1", - "id": "SkyOrbis/SKY-Ko-Llama3.1-8B-lora-epoch1", - "developer": "SkyOrbis", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/SkyOrbis_SKY-Ko-Llama3.1-8B-lora-epoch1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5058 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5088 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1548 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3213 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3998 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3777 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/skyorbis_sky-ko-llama3.1-8b-lora.json b/data/models/skyorbis_sky-ko-llama3.1-8b-lora.json deleted file mode 100644 index 50342f052ab800863a00de8a41ae8d44daff1ce7..0000000000000000000000000000000000000000 --- a/data/models/skyorbis_sky-ko-llama3.1-8b-lora.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SKY-Ko-Llama3.1-8B-lora", - "id": "SkyOrbis/SKY-Ko-Llama3.1-8B-lora", - "developer": "SkyOrbis", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/SkyOrbis_SKY-Ko-Llama3.1-8B-lora/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5058 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5088 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1548 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3213 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3998 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3777 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/skyorbis_sky-ko-llama3.2-1b-lora-epoch3.json b/data/models/skyorbis_sky-ko-llama3.2-1b-lora-epoch3.json deleted file mode 100644 index 3f3184a8e9ee596c692a28e54d584e308bd224f6..0000000000000000000000000000000000000000 --- a/data/models/skyorbis_sky-ko-llama3.2-1b-lora-epoch3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SKY-Ko-Llama3.2-1B-lora-epoch3", - "id": "SkyOrbis/SKY-Ko-Llama3.2-1B-lora-epoch3", - "developer": "SkyOrbis", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.236" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/SkyOrbis_SKY-Ko-Llama3.2-1B-lora-epoch3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3247 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3167 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0272 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2517 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3382 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1279 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/skyorbis_sky-ko-llama3.2-1b-lora-epoch5.json b/data/models/skyorbis_sky-ko-llama3.2-1b-lora-epoch5.json deleted file mode 100644 index b93b21769eb35a260c7b1ec66e7feca6b31ffb9f..0000000000000000000000000000000000000000 --- a/data/models/skyorbis_sky-ko-llama3.2-1b-lora-epoch5.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SKY-Ko-Llama3.2-1B-lora-epoch5", - "id": "SkyOrbis/SKY-Ko-Llama3.2-1B-lora-epoch5", - "developer": "SkyOrbis", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.236" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/SkyOrbis_SKY-Ko-Llama3.2-1B-lora-epoch5/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.436 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3406 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0521 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2592 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3471 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1946 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/skyorbis_sky-ko-llama3.2-1b-lora-v2-epoch3.json b/data/models/skyorbis_sky-ko-llama3.2-1b-lora-v2-epoch3.json deleted file mode 100644 index 5407b3a14ca08aaa09cff9a6fad19c7cfbd58af2..0000000000000000000000000000000000000000 --- a/data/models/skyorbis_sky-ko-llama3.2-1b-lora-v2-epoch3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SKY-Ko-Llama3.2-1B-lora-v2-epoch3", - "id": "SkyOrbis/SKY-Ko-Llama3.2-1B-lora-v2-epoch3", - "developer": "SkyOrbis", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.236" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/SkyOrbis_SKY-Ko-Llama3.2-1B-lora-v2-epoch3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.436 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3406 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0521 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2592 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3471 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1946 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/skyorbis_sky-ko-llama3.2-1b-lora-v2-epoch5.json b/data/models/skyorbis_sky-ko-llama3.2-1b-lora-v2-epoch5.json deleted file mode 100644 index 817c6d17fb249e609eaa0b38d03bb6ad4d01dbc8..0000000000000000000000000000000000000000 --- a/data/models/skyorbis_sky-ko-llama3.2-1b-lora-v2-epoch5.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SKY-Ko-Llama3.2-1B-lora-v2-epoch5", - "id": "SkyOrbis/SKY-Ko-Llama3.2-1B-lora-v2-epoch5", - "developer": "SkyOrbis", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.236" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/SkyOrbis_SKY-Ko-Llama3.2-1B-lora-v2-epoch5/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4247 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3397 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0506 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2542 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3458 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1946 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/skyorbis_sky-ko-llama3.2-3b-lora-epoch1.json b/data/models/skyorbis_sky-ko-llama3.2-3b-lora-epoch1.json deleted file mode 100644 index f74b6bafb313636ee6d0b8dba6c408252149a215..0000000000000000000000000000000000000000 --- a/data/models/skyorbis_sky-ko-llama3.2-3b-lora-epoch1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SKY-Ko-Llama3.2-3B-lora-epoch1", - "id": "SkyOrbis/SKY-Ko-Llama3.2-3B-lora-epoch1", - "developer": "SkyOrbis", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/SkyOrbis_SKY-Ko-Llama3.2-3B-lora-epoch1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5331 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.44 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1458 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2919 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3522 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3004 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/skyorbis_sky-ko-llama3.2-3b-lora-epoch2.json b/data/models/skyorbis_sky-ko-llama3.2-3b-lora-epoch2.json deleted file mode 100644 index 3f524e80053b0508c84a5b68bd8d715c2c23b553..0000000000000000000000000000000000000000 --- a/data/models/skyorbis_sky-ko-llama3.2-3b-lora-epoch2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SKY-Ko-Llama3.2-3B-lora-epoch2", - "id": "SkyOrbis/SKY-Ko-Llama3.2-3B-lora-epoch2", - "developer": "SkyOrbis", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/SkyOrbis_SKY-Ko-Llama3.2-3B-lora-epoch2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5331 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.44 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1458 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2919 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3522 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3004 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/skyorbis_sky-ko-llama3.2-3b-lora-epoch3.json b/data/models/skyorbis_sky-ko-llama3.2-3b-lora-epoch3.json deleted file mode 100644 index e91a1abd022958350adc46419ed282afbac83b84..0000000000000000000000000000000000000000 --- a/data/models/skyorbis_sky-ko-llama3.2-3b-lora-epoch3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SKY-Ko-Llama3.2-3B-lora-epoch3", - "id": "SkyOrbis/SKY-Ko-Llama3.2-3B-lora-epoch3", - "developer": "SkyOrbis", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/SkyOrbis_SKY-Ko-Llama3.2-3B-lora-epoch3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5331 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.44 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1458 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2919 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3522 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3004 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/skyorbis_sky-ko-qwen2.5-3b-instruct.json b/data/models/skyorbis_sky-ko-qwen2.5-3b-instruct.json deleted file mode 100644 index 0805e150700613e59f29c7f5f5e916a077f198fa..0000000000000000000000000000000000000000 --- a/data/models/skyorbis_sky-ko-qwen2.5-3b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SKY-Ko-Qwen2.5-3B-Instruct", - "id": "SkyOrbis/SKY-Ko-Qwen2.5-3B-Instruct", - "developer": "SkyOrbis", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.086" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/SkyOrbis_SKY-Ko-Qwen2.5-3B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3534 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4265 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0695 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2794 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4024 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2812 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/skyorbis_sky-ko-qwen2.5-7b-instruct-sft-step-15000.json b/data/models/skyorbis_sky-ko-qwen2.5-7b-instruct-sft-step-15000.json deleted file mode 100644 index 033411cdc63570d025aafac0f0ba5d7004c07511..0000000000000000000000000000000000000000 --- a/data/models/skyorbis_sky-ko-qwen2.5-7b-instruct-sft-step-15000.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SKY-Ko-Qwen2.5-7B-Instruct-SFT-step-15000", - "id": "SkyOrbis/SKY-Ko-Qwen2.5-7B-Instruct-SFT-step-15000", - "developer": "SkyOrbis", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/SkyOrbis_SKY-Ko-Qwen2.5-7B-Instruct-SFT-step-15000/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3819 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5078 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1866 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3272 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4436 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3914 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/skyorbis_sky-ko-qwen2.5-7b-instruct-sft-step-5000.json b/data/models/skyorbis_sky-ko-qwen2.5-7b-instruct-sft-step-5000.json deleted file mode 100644 index a07e1d387613d4afe10da69b74ca20ef7e7e0c60..0000000000000000000000000000000000000000 --- a/data/models/skyorbis_sky-ko-qwen2.5-7b-instruct-sft-step-5000.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SKY-Ko-Qwen2.5-7B-Instruct-SFT-step-5000", - "id": "SkyOrbis/SKY-Ko-Qwen2.5-7B-Instruct-SFT-step-5000", - "developer": "SkyOrbis", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/SkyOrbis_SKY-Ko-Qwen2.5-7B-Instruct-SFT-step-5000/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3812 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.539 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.21 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3029 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4238 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4238 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/skywork_skywork-critic-llama-3.1-70b.json b/data/models/skywork_skywork-critic-llama-3.1-70b.json deleted file mode 100644 index feb8680eb4a24f97b0bd403cba21b0030d7762fc..0000000000000000000000000000000000000000 --- a/data/models/skywork_skywork-critic-llama-3.1-70b.json +++ /dev/null @@ -1,126 +0,0 @@ -{ - "model_info": { - "name": "Skywork/Skywork-Critic-Llama-3.1-70B", - "id": "Skywork/Skywork-Critic-Llama-3.1-70B", - "developer": "Skywork", - "additional_details": { - "model_type": "Generative" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/Skywork_Skywork-Critic-Llama-3.1-70B/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9331 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9665 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8794 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9311 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9554 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/skywork_skywork-critic-llama-3.1-8b.json b/data/models/skywork_skywork-critic-llama-3.1-8b.json deleted file mode 100644 index 3caa912a2b01c2b6b0022b19275449d2332ce6d8..0000000000000000000000000000000000000000 --- a/data/models/skywork_skywork-critic-llama-3.1-8b.json +++ /dev/null @@ -1,126 +0,0 @@ -{ - "model_info": { - "name": "Skywork/Skywork-Critic-Llama-3.1-8B", - "id": "Skywork/Skywork-Critic-Llama-3.1-8B", - "developer": "Skywork", - "additional_details": { - "model_type": "Generative" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/Skywork_Skywork-Critic-Llama-3.1-8B/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8896 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9358 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8136 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9108 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.898 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/skywork_skywork-o1-open-llama-3.1-8b.json b/data/models/skywork_skywork-o1-open-llama-3.1-8b.json deleted file mode 100644 index dc6542b9322c026b526b489e72ab81afa1c67a14..0000000000000000000000000000000000000000 --- a/data/models/skywork_skywork-o1-open-llama-3.1-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Skywork-o1-Open-Llama-3.1-8B", - "id": "Skywork/Skywork-o1-Open-Llama-3.1-8B", - "developer": "Skywork", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Skywork_Skywork-o1-Open-Llama-3.1-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3518 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4516 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5211 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2592 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3156 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.203 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/skywork_skywork-reward-gemma-2-27b-v0.2.json b/data/models/skywork_skywork-reward-gemma-2-27b-v0.2.json deleted file mode 100644 index 4109d07feac30d842a656539e17ae764cf441c79..0000000000000000000000000000000000000000 --- a/data/models/skywork_skywork-reward-gemma-2-27b-v0.2.json +++ /dev/null @@ -1,409 +0,0 @@ -{ - "model_info": { - "name": "Skywork/Skywork-Reward-Gemma-2-27B-v0.2", - "id": "Skywork/Skywork-Reward-Gemma-2-27B-v0.2", - "developer": "Skywork", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForSequenceClassification", - "params_billions": "27.227" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Skywork_Skywork-Reward-Gemma-2-27B-v0.2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7807 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.636 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2273 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.344 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4231 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4103 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "reward-bench-2/Skywork_Skywork-Reward-Gemma-2-27B-v0.2/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7531 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7674 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.375 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6721 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9689 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9172 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8182 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "reward-bench/Skywork_Skywork-Reward-Gemma-2-27B-v0.2/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9426 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9609 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8991 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9297 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9807 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/skywork_skywork-reward-gemma-2-27b.json b/data/models/skywork_skywork-reward-gemma-2-27b.json deleted file mode 100644 index c2b742dbc17aa9720f66d090e08eb784ec7accdc..0000000000000000000000000000000000000000 --- a/data/models/skywork_skywork-reward-gemma-2-27b.json +++ /dev/null @@ -1,276 +0,0 @@ -{ - "model_info": { - "name": "Skywork/Skywork-Reward-Gemma-2-27B", - "id": "Skywork/Skywork-Reward-Gemma-2-27B", - "developer": "Skywork", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/Skywork_Skywork-Reward-Gemma-2-27B/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7576 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7368 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4031 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7049 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9422 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9323 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8261 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "reward-bench/Skywork_Skywork-Reward-Gemma-2-27B/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.938 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9581 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9145 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9189 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9606 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/skywork_skywork-reward-llama-3.1-8b-v0.2.json b/data/models/skywork_skywork-reward-llama-3.1-8b-v0.2.json deleted file mode 100644 index 257e1711290adf0666a2c139ab61e68721a44b75..0000000000000000000000000000000000000000 --- a/data/models/skywork_skywork-reward-llama-3.1-8b-v0.2.json +++ /dev/null @@ -1,276 +0,0 @@ -{ - "model_info": { - "name": "Skywork/Skywork-Reward-Llama-3.1-8B-v0.2", - "id": "Skywork/Skywork-Reward-Llama-3.1-8B-v0.2", - "developer": "Skywork", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/Skywork_Skywork-Reward-Llama-3.1-8B-v0.2/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9313 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9469 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8838 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.927 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9675 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "reward-bench-2/Skywork_Skywork-Reward-Llama-3.1-8B-v0.2/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7175 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6968 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4062 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6011 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9422 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9414 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7169 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/skywork_skywork-reward-llama-3.1-8b.json b/data/models/skywork_skywork-reward-llama-3.1-8b.json deleted file mode 100644 index dedd0015bc30c7a59cc8db5e1fbdb3b6b6cbc978..0000000000000000000000000000000000000000 --- a/data/models/skywork_skywork-reward-llama-3.1-8b.json +++ /dev/null @@ -1,276 +0,0 @@ -{ - "model_info": { - "name": "Skywork/Skywork-Reward-Llama-3.1-8B", - "id": "Skywork/Skywork-Reward-Llama-3.1-8B", - "developer": "Skywork", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/Skywork_Skywork-Reward-Llama-3.1-8B/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9252 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9581 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8728 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9081 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.962 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "reward-bench-2/Skywork_Skywork-Reward-Llama-3.1-8B/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7314 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6989 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.425 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6284 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9333 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9616 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.741 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/skywork_skywork-reward-v2-llama-3.1-8b.json b/data/models/skywork_skywork-reward-v2-llama-3.1-8b.json deleted file mode 100644 index b8b152a493833cae1e86ccc1b6fb8de6a744d3b1..0000000000000000000000000000000000000000 --- a/data/models/skywork_skywork-reward-v2-llama-3.1-8b.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "Skywork/Skywork-Reward-V2-Llama-3.1-8B", - "id": "Skywork/Skywork-Reward-V2-Llama-3.1-8B", - "developer": "Skywork", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/Skywork_Skywork-Reward-V2-Llama-3.1-8B/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8413 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8463 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6625 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.776 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9667 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9838 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8124 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/skywork_skywork-reward-v2-llama-3.2-1b.json b/data/models/skywork_skywork-reward-v2-llama-3.2-1b.json deleted file mode 100644 index a8727ba2508a68f50f28a7e8ed18085715c25a73..0000000000000000000000000000000000000000 --- a/data/models/skywork_skywork-reward-v2-llama-3.2-1b.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "Skywork/Skywork-Reward-V2-Llama-3.2-1B", - "id": "Skywork/Skywork-Reward-V2-Llama-3.2-1B", - "developer": "Skywork", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/Skywork_Skywork-Reward-V2-Llama-3.2-1B/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6438 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6084 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4562 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6011 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8733 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8929 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4306 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/skywork_skywork-reward-v2-llama-3.2-3b.json b/data/models/skywork_skywork-reward-v2-llama-3.2-3b.json deleted file mode 100644 index 158ffe329656e5afc5822f158fb24435bf4a99ec..0000000000000000000000000000000000000000 --- a/data/models/skywork_skywork-reward-v2-llama-3.2-3b.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "Skywork/Skywork-Reward-V2-Llama-3.2-3B", - "id": "Skywork/Skywork-Reward-V2-Llama-3.2-3B", - "developer": "Skywork", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/Skywork_Skywork-Reward-V2-Llama-3.2-3B/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7466 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7621 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4562 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.694 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9311 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9596 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6768 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/skywork_skywork-reward-v2-qwen3-0.6b.json b/data/models/skywork_skywork-reward-v2-qwen3-0.6b.json deleted file mode 100644 index 1163cd6df45d6cf8deb99e02e4461271bba1cf36..0000000000000000000000000000000000000000 --- a/data/models/skywork_skywork-reward-v2-qwen3-0.6b.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "Skywork/Skywork-Reward-V2-Qwen3-0.6B", - "id": "Skywork/Skywork-Reward-V2-Qwen3-0.6B", - "developer": "Skywork", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/Skywork_Skywork-Reward-V2-Qwen3-0.6B/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6125 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.58 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7158 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8444 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7949 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3397 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/skywork_skywork-reward-v2-qwen3-1.7b.json b/data/models/skywork_skywork-reward-v2-qwen3-1.7b.json deleted file mode 100644 index 13bfef758f8ca3b473d41a0aa206197fa8f35132..0000000000000000000000000000000000000000 --- a/data/models/skywork_skywork-reward-v2-qwen3-1.7b.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "Skywork/Skywork-Reward-V2-Qwen3-1.7B", - "id": "Skywork/Skywork-Reward-V2-Qwen3-1.7B", - "developer": "Skywork", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/Skywork_Skywork-Reward-V2-Qwen3-1.7B/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6818 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6568 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4437 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7268 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8911 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8848 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4872 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/skywork_skywork-reward-v2-qwen3-4b.json b/data/models/skywork_skywork-reward-v2-qwen3-4b.json deleted file mode 100644 index 5b99d355f2df094dba0cc8d0fb6a902693f10ad2..0000000000000000000000000000000000000000 --- a/data/models/skywork_skywork-reward-v2-qwen3-4b.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "Skywork/Skywork-Reward-V2-Qwen3-4B", - "id": "Skywork/Skywork-Reward-V2-Qwen3-4B", - "developer": "Skywork", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/Skywork_Skywork-Reward-V2-Qwen3-4B/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7551 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7737 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4625 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7322 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9222 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9657 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6743 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/skywork_skywork-reward-v2-qwen3-8b.json b/data/models/skywork_skywork-reward-v2-qwen3-8b.json deleted file mode 100644 index edfe291d28a1648bff4d5bc37fe32504f1bd6786..0000000000000000000000000000000000000000 --- a/data/models/skywork_skywork-reward-v2-qwen3-8b.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "model_info": { - "name": "Skywork/Skywork-Reward-V2-Qwen3-8B", - "id": "Skywork/Skywork-Reward-V2-Qwen3-8B", - "developer": "Skywork", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/Skywork_Skywork-Reward-V2-Qwen3-8B/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7837 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7989 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7705 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.94 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9636 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7294 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/skywork_skywork-vl-reward-7b.json b/data/models/skywork_skywork-vl-reward-7b.json deleted file mode 100644 index 651d1416fd84d9618565234fc2f23befa272cb51..0000000000000000000000000000000000000000 --- a/data/models/skywork_skywork-vl-reward-7b.json +++ /dev/null @@ -1,276 +0,0 @@ -{ - "model_info": { - "name": "Skywork/Skywork-VL-Reward-7B", - "id": "Skywork/Skywork-VL-Reward-7B", - "developer": "Skywork", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/Skywork_Skywork-VL-Reward-7B/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6885 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6063 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.35 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6339 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8911 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8909 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7586 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "reward-bench/Skywork_Skywork-VL-Reward-7B/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9007 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8994 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.875 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9108 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9176 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/snowflake_snowflake-arctic-instruct.json b/data/models/snowflake_snowflake-arctic-instruct.json deleted file mode 100644 index dcce9cb71bbe18fac2e40bbf0810fef5adcc4078..0000000000000000000000000000000000000000 --- a/data/models/snowflake_snowflake-arctic-instruct.json +++ /dev/null @@ -1,1903 +0,0 @@ -{ - "model_info": { - "name": "Arctic Instruct", - "id": "snowflake/snowflake-arctic-instruct", - "developer": "snowflake", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_lite/snowflake_snowflake-arctic-instruct/1774096306.427425", - "retrieved_timestamp": "1774096306.427425", - "source_metadata": { - "source_name": "helm_lite", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_lite", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_lite", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.338, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.7606242197253433\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.654, - "details": { - "description": "min=0.654, mean=0.654, max=0.654, sum=0.654 (1)", - "tab": "Accuracy", - "NarrativeQA - Observed inference time (s)": "{\"description\": \"min=0.624, mean=0.624, max=0.624, sum=0.624 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.6239793220036466\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=4.262, mean=4.262, max=4.262, sum=4.262 (1)\", \"tab\": \"General information\", \"score\": \"4.261971830985916\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=3603.217, mean=3603.217, max=3603.217, sum=3603.217 (1)\", \"tab\": \"General information\", \"score\": \"3603.2169014084507\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=11.907, mean=11.907, max=11.907, sum=11.907 (1)\", \"tab\": \"General information\", \"score\": \"11.907042253521126\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (closed-book)", - "source_data": { - "dataset_name": "NaturalQuestions (closed-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (closed-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.39, - "details": { - "description": "min=0.39, mean=0.39, max=0.39, sum=0.39 (1)", - "tab": "Accuracy", - "NaturalQuestions (open-book) - Observed inference time (s)": "{\"description\": \"min=0.636, mean=0.636, max=0.636, sum=0.636 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.6355201268196106\"}", - "NaturalQuestions (closed-book) - Observed inference time (s)": "{\"description\": \"min=0.469, mean=0.469, max=0.469, sum=0.469 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.4687326259613037\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.825, mean=4.825, max=4.825, sum=4.825 (1)\", \"tab\": \"General information\", \"score\": \"4.825\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.028, mean=0.028, max=0.028, sum=0.028 (1)\", \"tab\": \"General information\", \"score\": \"0.028\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=2311.514, mean=2311.514, max=2311.514, sum=2311.514 (1)\", \"tab\": \"General information\", \"score\": \"2311.514\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=18.701, mean=18.701, max=18.701, sum=18.701 (1)\", \"tab\": \"General information\", \"score\": \"18.701\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=166.383, mean=166.383, max=166.383, sum=166.383 (1)\", \"tab\": \"General information\", \"score\": \"166.383\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=14.473, mean=14.473, max=14.473, sum=14.473 (1)\", \"tab\": \"General information\", \"score\": \"14.473\"}" - } - }, - "generation_config": { - "additional_details": { - "mode": "\"closedbook\"" - } - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.828, - "details": { - "description": "min=0.828, mean=0.828, max=0.828, sum=0.828 (1)", - "tab": "Accuracy", - "OpenbookQA - Observed inference time (s)": "{\"description\": \"min=0.284, mean=0.284, max=0.284, sum=0.284 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.2840936713218689\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=291.574, mean=291.574, max=291.574, sum=291.574 (1)\", \"tab\": \"General information\", \"score\": \"291.574\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "dataset": "\"openbookqa\"", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.575, - "details": { - "description": "min=0.31, mean=0.575, max=0.88, sum=2.876 (5)", - "tab": "Accuracy", - "MMLU - Observed inference time (s)": "{\"description\": \"min=0.293, mean=0.303, max=0.317, sum=1.516 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.30325288054817606\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=406.65, mean=531.547, max=693.675, sum=2657.735 (5)\", \"tab\": \"General information\", \"score\": \"531.5470877192982\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"college_chemistry\", \"computer_security\", \"econometrics\", \"us_foreign_policy\"]", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MATH", - "source_data": { - "dataset_name": "MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Equivalent (CoT) on MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.519, - "details": { - "description": "min=0.316, mean=0.519, max=0.785, sum=3.636 (7)", - "tab": "Accuracy", - "MATH - Observed inference time (s)": "{\"description\": \"min=1.482, mean=1.724, max=1.995, sum=12.068 (7)\", \"tab\": \"Efficiency\", \"score\": \"1.723981539653867\"}", - "MATH - # eval": "{\"description\": \"min=30, mean=62.429, max=135, sum=437 (7)\", \"tab\": \"General information\", \"score\": \"62.42857142857143\"}", - "MATH - # train": "{\"description\": \"min=8, mean=8, max=8, sum=56 (7)\", \"tab\": \"General information\", \"score\": \"8.0\"}", - "MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MATH - # prompt tokens": "{\"description\": \"min=971.652, mean=1438.636, max=2490.962, sum=10070.453 (7)\", \"tab\": \"General information\", \"score\": \"1438.6362030100095\"}", - "MATH - # output tokens": "{\"description\": \"min=82.872, mean=98.802, max=122.233, sum=691.615 (7)\", \"tab\": \"General information\", \"score\": \"98.80208187931566\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"algebra\", \"counting_and_probability\", \"geometry\", \"intermediate_algebra\", \"number_theory\", \"prealgebra\", \"precalculus\"]", - "level": "\"1\"", - "use_official_examples": "\"False\"", - "use_chain_of_thought": "\"True\"" - } - } - }, - { - "evaluation_name": "GSM8K", - "source_data": { - "dataset_name": "GSM8K", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on GSM8K", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.768, - "details": { - "description": "min=0.768, mean=0.768, max=0.768, sum=0.768 (1)", - "tab": "Accuracy", - "GSM8K - Observed inference time (s)": "{\"description\": \"min=2.961, mean=2.961, max=2.961, sum=2.961 (1)\", \"tab\": \"Efficiency\", \"score\": \"2.9610197002887726\"}", - "GSM8K - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "GSM8K - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "GSM8K - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GSM8K - # prompt tokens": "{\"description\": \"min=1207.746, mean=1207.746, max=1207.746, sum=1207.746 (1)\", \"tab\": \"General information\", \"score\": \"1207.746\"}", - "GSM8K - # output tokens": "{\"description\": \"min=189.305, mean=189.305, max=189.305, sum=189.305 (1)\", \"tab\": \"General information\", \"score\": \"189.305\"}" - } - }, - "generation_config": { - "additional_details": { - "stop": "\"none\"" - } - } - }, - { - "evaluation_name": "LegalBench", - "source_data": { - "dataset_name": "LegalBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on LegalBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.588, - "details": { - "description": "min=0.351, mean=0.588, max=0.874, sum=2.94 (5)", - "tab": "Accuracy", - "LegalBench - Observed inference time (s)": "{\"description\": \"min=0.292, mean=0.346, max=0.462, sum=1.729 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.34576316386866485\"}", - "LegalBench - # eval": "{\"description\": \"min=95, mean=409.4, max=1000, sum=2047 (5)\", \"tab\": \"General information\", \"score\": \"409.4\"}", - "LegalBench - # train": "{\"description\": \"min=1.81, mean=4.162, max=5, sum=20.81 (5)\", \"tab\": \"General information\", \"score\": \"4.162040816326531\"}", - "LegalBench - truncated": "{\"description\": \"min=0, mean=0.002, max=0.008, sum=0.008 (5)\", \"tab\": \"General information\", \"score\": \"0.0016326530612244899\"}", - "LegalBench - # prompt tokens": "{\"description\": \"min=239.137, mean=1024.722, max=3561.237, sum=5123.61 (5)\", \"tab\": \"General information\", \"score\": \"1024.7220443430492\"}", - "LegalBench - # output tokens": "{\"description\": \"min=2, mean=2.438, max=3.421, sum=12.188 (5)\", \"tab\": \"General information\", \"score\": \"2.4375592890361366\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"abercrombie\", \"corporate_lobbying\", \"function_of_decision_section\", \"international_citizenship_questions\", \"proa\"]" - } - } - }, - { - "evaluation_name": "MedQA", - "source_data": { - "dataset_name": "MedQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MedQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.581, - "details": { - "description": "min=0.581, mean=0.581, max=0.581, sum=0.581 (1)", - "tab": "Accuracy", - "MedQA - Observed inference time (s)": "{\"description\": \"min=0.313, mean=0.313, max=0.313, sum=0.313 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.31300480038697864\"}", - "MedQA - # eval": "{\"description\": \"min=503, mean=503, max=503, sum=503 (1)\", \"tab\": \"General information\", \"score\": \"503.0\"}", - "MedQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MedQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MedQA - # prompt tokens": "{\"description\": \"min=1243.901, mean=1243.901, max=1243.901, sum=1243.901 (1)\", \"tab\": \"General information\", \"score\": \"1243.9005964214712\"}", - "MedQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WMT 2014", - "source_data": { - "dataset_name": "WMT 2014", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "BLEU-4 on WMT 2014", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.172, - "details": { - "description": "min=0.09, mean=0.172, max=0.217, sum=0.86 (5)", - "tab": "Accuracy", - "WMT 2014 - Observed inference time (s)": "{\"description\": \"min=0.65, mean=0.681, max=0.702, sum=3.405 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.681007040066764\"}", - "WMT 2014 - # eval": "{\"description\": \"min=503, mean=568.8, max=832, sum=2844 (5)\", \"tab\": \"General information\", \"score\": \"568.8\"}", - "WMT 2014 - # train": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "WMT 2014 - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WMT 2014 - # prompt tokens": "{\"description\": \"min=145.523, mean=160.288, max=182.972, sum=801.438 (5)\", \"tab\": \"General information\", \"score\": \"160.28751290334915\"}", - "WMT 2014 - # output tokens": "{\"description\": \"min=28.596, mean=30.59, max=31.485, sum=152.951 (5)\", \"tab\": \"General information\", \"score\": \"30.59012702630372\"}" - } - }, - "generation_config": { - "additional_details": { - "language_pair": "[\"cs-en\", \"de-en\", \"fr-en\", \"hi-en\", \"ru-en\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_mmlu/snowflake_snowflake-arctic-instruct/1774096312.00548", - "retrieved_timestamp": "1774096312.00548", - "source_metadata": { - "source_name": "helm_mmlu", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_mmlu", - "evaluation_results": [ - { - "evaluation_name": "MMLU All Subjects", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU All Subjects", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.677, - "details": { - "description": "min=0.28, mean=0.677, max=0.912, sum=77.129 (114)", - "tab": "Accuracy", - "MMLU All Subjects - Observed inference time (s)": "{\"description\": \"min=0.35, mean=0.42, max=0.544, sum=47.89 (114)\", \"tab\": \"Efficiency\", \"score\": \"0.4200856614493726\"}", - "MMLU All Subjects - # eval": "{\"description\": \"min=100, mean=246.351, max=1534, sum=28084 (114)\", \"tab\": \"General information\", \"score\": \"246.35087719298247\"}", - "MMLU All Subjects - # train": "{\"description\": \"min=5, mean=5, max=5, sum=570 (114)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU All Subjects - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU All Subjects - # prompt tokens": "{\"description\": \"min=304.474, mean=706.682, max=3159.636, sum=80561.749 (114)\", \"tab\": \"General information\", \"score\": \"706.6820126388612\"}", - "MMLU All Subjects - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=114 (114)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_name": "Abstract Algebra", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Abstract Algebra", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.35, - "details": { - "description": "min=0.35, mean=0.35, max=0.35, sum=0.7 (2)", - "tab": "Accuracy", - "Abstract Algebra - Observed inference time (s)": "{\"description\": \"min=0.377, mean=0.377, max=0.377, sum=0.753 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.37665764808654784\"}", - "Abstract Algebra - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Abstract Algebra - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Abstract Algebra - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Abstract Algebra - # prompt tokens": "{\"description\": \"min=397.65, mean=397.65, max=397.65, sum=795.3 (2)\", \"tab\": \"General information\", \"score\": \"397.65\"}", - "Abstract Algebra - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"abstract_algebra\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_abstract_algebra\"" - } - } - }, - { - "evaluation_name": "Anatomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Anatomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.652, - "details": { - "description": "min=0.652, mean=0.652, max=0.652, sum=1.304 (2)", - "tab": "Accuracy", - "Anatomy - Observed inference time (s)": "{\"description\": \"min=0.365, mean=0.365, max=0.365, sum=0.731 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3654881194785789\"}", - "Anatomy - # eval": "{\"description\": \"min=135, mean=135, max=135, sum=270 (2)\", \"tab\": \"General information\", \"score\": \"135.0\"}", - "Anatomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Anatomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Anatomy - # prompt tokens": "{\"description\": \"min=409.133, mean=409.133, max=409.133, sum=818.267 (2)\", \"tab\": \"General information\", \"score\": \"409.1333333333333\"}", - "Anatomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"anatomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_anatomy\"" - } - } - }, - { - "evaluation_name": "College Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on College Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.461, - "details": { - "description": "min=0.461, mean=0.461, max=0.461, sum=0.922 (2)", - "tab": "Accuracy", - "College Chemistry - Observed inference time (s)": "{\"description\": \"min=0.35, mean=0.35, max=0.35, sum=0.701 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3502761268615723\"}", - "College Biology - Observed inference time (s)": "{\"description\": \"min=0.421, mean=0.421, max=0.421, sum=0.842 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.421069688267178\"}", - "College Computer Science - Observed inference time (s)": "{\"description\": \"min=0.427, mean=0.427, max=0.427, sum=0.853 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4266632032394409\"}", - "College Mathematics - Observed inference time (s)": "{\"description\": \"min=0.429, mean=0.429, max=0.429, sum=0.858 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.42887043952941895\"}", - "College Medicine - Observed inference time (s)": "{\"description\": \"min=0.434, mean=0.434, max=0.434, sum=0.869 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4343285574389331\"}", - "College Physics - Observed inference time (s)": "{\"description\": \"min=0.421, mean=0.421, max=0.421, sum=0.842 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4209739086674709\"}", - "College Chemistry - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Chemistry - # prompt tokens": "{\"description\": \"min=622.43, mean=622.43, max=622.43, sum=1244.86 (2)\", \"tab\": \"General information\", \"score\": \"622.43\"}", - "College Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Biology - # eval": "{\"description\": \"min=144, mean=144, max=144, sum=288 (2)\", \"tab\": \"General information\", \"score\": \"144.0\"}", - "College Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # prompt tokens": "{\"description\": \"min=553.632, mean=553.632, max=553.632, sum=1107.264 (2)\", \"tab\": \"General information\", \"score\": \"553.6319444444445\"}", - "College Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # prompt tokens": "{\"description\": \"min=901.14, mean=901.14, max=901.14, sum=1802.28 (2)\", \"tab\": \"General information\", \"score\": \"901.14\"}", - "College Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Mathematics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # prompt tokens": "{\"description\": \"min=646.96, mean=646.96, max=646.96, sum=1293.92 (2)\", \"tab\": \"General information\", \"score\": \"646.96\"}", - "College Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Medicine - # eval": "{\"description\": \"min=173, mean=173, max=173, sum=346 (2)\", \"tab\": \"General information\", \"score\": \"173.0\"}", - "College Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # prompt tokens": "{\"description\": \"min=608.671, mean=608.671, max=608.671, sum=1217.341 (2)\", \"tab\": \"General information\", \"score\": \"608.6705202312139\"}", - "College Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Physics - # eval": "{\"description\": \"min=102, mean=102, max=102, sum=204 (2)\", \"tab\": \"General information\", \"score\": \"102.0\"}", - "College Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # prompt tokens": "{\"description\": \"min=551.873, mean=551.873, max=551.873, sum=1103.745 (2)\", \"tab\": \"General information\", \"score\": \"551.8725490196078\"}", - "College Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"college_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_college_physics\"" - } - } - }, - { - "evaluation_name": "Computer Security", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Computer Security", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.84, - "details": { - "description": "min=0.84, mean=0.84, max=0.84, sum=1.68 (2)", - "tab": "Accuracy", - "Computer Security - Observed inference time (s)": "{\"description\": \"min=0.412, mean=0.412, max=0.412, sum=0.825 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.41247488737106325\"}", - "Computer Security - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Computer Security - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Computer Security - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Computer Security - # prompt tokens": "{\"description\": \"min=428.17, mean=428.17, max=428.17, sum=856.34 (2)\", \"tab\": \"General information\", \"score\": \"428.17\"}", - "Computer Security - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"computer_security\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_computer_security\"" - } - } - }, - { - "evaluation_name": "Econometrics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Econometrics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5, - "details": { - "description": "min=0.5, mean=0.5, max=0.5, sum=1 (2)", - "tab": "Accuracy", - "Econometrics - Observed inference time (s)": "{\"description\": \"min=0.436, mean=0.436, max=0.436, sum=0.873 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.436487873395284\"}", - "Econometrics - # eval": "{\"description\": \"min=114, mean=114, max=114, sum=228 (2)\", \"tab\": \"General information\", \"score\": \"114.0\"}", - "Econometrics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Econometrics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Econometrics - # prompt tokens": "{\"description\": \"min=684.675, mean=684.675, max=684.675, sum=1369.351 (2)\", \"tab\": \"General information\", \"score\": \"684.6754385964912\"}", - "Econometrics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"econometrics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_econometrics\"" - } - } - }, - { - "evaluation_name": "Global Facts", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Global Facts", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.39, - "details": { - "description": "min=0.39, mean=0.39, max=0.39, sum=0.78 (2)", - "tab": "Accuracy", - "Global Facts - Observed inference time (s)": "{\"description\": \"min=0.42, mean=0.42, max=0.42, sum=0.839 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.41951879262924197\"}", - "Global Facts - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Global Facts - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Global Facts - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Global Facts - # prompt tokens": "{\"description\": \"min=484.54, mean=484.54, max=484.54, sum=969.08 (2)\", \"tab\": \"General information\", \"score\": \"484.54\"}", - "Global Facts - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"global_facts\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_global_facts\"" - } - } - }, - { - "evaluation_name": "Jurisprudence", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Jurisprudence", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.741, - "details": { - "description": "min=0.741, mean=0.741, max=0.741, sum=1.481 (2)", - "tab": "Accuracy", - "Jurisprudence - Observed inference time (s)": "{\"description\": \"min=0.422, mean=0.422, max=0.422, sum=0.843 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.421647725281892\"}", - "Jurisprudence - # eval": "{\"description\": \"min=108, mean=108, max=108, sum=216 (2)\", \"tab\": \"General information\", \"score\": \"108.0\"}", - "Jurisprudence - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Jurisprudence - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Jurisprudence - # prompt tokens": "{\"description\": \"min=449.898, mean=449.898, max=449.898, sum=899.796 (2)\", \"tab\": \"General information\", \"score\": \"449.89814814814815\"}", - "Jurisprudence - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"jurisprudence\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_jurisprudence\"" - } - } - }, - { - "evaluation_name": "Philosophy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Philosophy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.752, - "details": { - "description": "min=0.752, mean=0.752, max=0.752, sum=1.505 (2)", - "tab": "Accuracy", - "Philosophy - Observed inference time (s)": "{\"description\": \"min=0.418, mean=0.418, max=0.418, sum=0.837 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.418486426497579\"}", - "Philosophy - # eval": "{\"description\": \"min=311, mean=311, max=311, sum=622 (2)\", \"tab\": \"General information\", \"score\": \"311.0\"}", - "Philosophy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Philosophy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Philosophy - # prompt tokens": "{\"description\": \"min=372.122, mean=372.122, max=372.122, sum=744.244 (2)\", \"tab\": \"General information\", \"score\": \"372.12218649517683\"}", - "Philosophy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"philosophy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_philosophy\"" - } - } - }, - { - "evaluation_name": "Professional Psychology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Professional Psychology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.724, - "details": { - "description": "min=0.724, mean=0.724, max=0.724, sum=1.448 (2)", - "tab": "Accuracy", - "Professional Medicine - Observed inference time (s)": "{\"description\": \"min=0.445, mean=0.445, max=0.445, sum=0.89 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4448305149288738\"}", - "Professional Accounting - Observed inference time (s)": "{\"description\": \"min=0.443, mean=0.443, max=0.443, sum=0.887 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.44340477683019974\"}", - "Professional Law - Observed inference time (s)": "{\"description\": \"min=0.531, mean=0.531, max=0.531, sum=1.062 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.531202322345669\"}", - "Professional Psychology - Observed inference time (s)": "{\"description\": \"min=0.423, mean=0.423, max=0.423, sum=0.847 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.42342418120577446\"}", - "Professional Medicine - # eval": "{\"description\": \"min=272, mean=272, max=272, sum=544 (2)\", \"tab\": \"General information\", \"score\": \"272.0\"}", - "Professional Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Medicine - # prompt tokens": "{\"description\": \"min=1330.647, mean=1330.647, max=1330.647, sum=2661.294 (2)\", \"tab\": \"General information\", \"score\": \"1330.6470588235295\"}", - "Professional Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Accounting - # eval": "{\"description\": \"min=282, mean=282, max=282, sum=564 (2)\", \"tab\": \"General information\", \"score\": \"282.0\"}", - "Professional Accounting - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Accounting - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # prompt tokens": "{\"description\": \"min=823.277, mean=823.277, max=823.277, sum=1646.553 (2)\", \"tab\": \"General information\", \"score\": \"823.2765957446809\"}", - "Professional Accounting - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Law - # eval": "{\"description\": \"min=1534, mean=1534, max=1534, sum=3068 (2)\", \"tab\": \"General information\", \"score\": \"1534.0\"}", - "Professional Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # prompt tokens": "{\"description\": \"min=1915.007, mean=1915.007, max=1915.007, sum=3830.014 (2)\", \"tab\": \"General information\", \"score\": \"1915.0071707953064\"}", - "Professional Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Psychology - # eval": "{\"description\": \"min=612, mean=612, max=612, sum=1224 (2)\", \"tab\": \"General information\", \"score\": \"612.0\"}", - "Professional Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # prompt tokens": "{\"description\": \"min=650.078, mean=650.078, max=650.078, sum=1300.157 (2)\", \"tab\": \"General information\", \"score\": \"650.0784313725491\"}", - "Professional Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"professional_psychology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_professional_psychology\"" - } - } - }, - { - "evaluation_name": "Us Foreign Policy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Us Foreign Policy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.88, - "details": { - "description": "min=0.88, mean=0.88, max=0.88, sum=1.76 (2)", - "tab": "Accuracy", - "Us Foreign Policy - Observed inference time (s)": "{\"description\": \"min=0.424, mean=0.424, max=0.424, sum=0.848 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.42398189067840575\"}", - "Us Foreign Policy - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Us Foreign Policy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Us Foreign Policy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Us Foreign Policy - # prompt tokens": "{\"description\": \"min=479.81, mean=479.81, max=479.81, sum=959.62 (2)\", \"tab\": \"General information\", \"score\": \"479.81\"}", - "Us Foreign Policy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"us_foreign_policy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_us_foreign_policy\"" - } - } - }, - { - "evaluation_name": "Astronomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Astronomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.763, - "details": { - "description": "min=0.763, mean=0.763, max=0.763, sum=1.526 (2)", - "tab": "Accuracy", - "Astronomy - Observed inference time (s)": "{\"description\": \"min=0.424, mean=0.424, max=0.424, sum=0.848 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.42381788398090164\"}", - "Astronomy - # eval": "{\"description\": \"min=152, mean=152, max=152, sum=304 (2)\", \"tab\": \"General information\", \"score\": \"152.0\"}", - "Astronomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Astronomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Astronomy - # prompt tokens": "{\"description\": \"min=681.079, mean=681.079, max=681.079, sum=1362.158 (2)\", \"tab\": \"General information\", \"score\": \"681.078947368421\"}", - "Astronomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"astronomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_astronomy\"" - } - } - }, - { - "evaluation_name": "Business Ethics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Business Ethics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.69, - "details": { - "description": "min=0.69, mean=0.69, max=0.69, sum=1.38 (2)", - "tab": "Accuracy", - "Business Ethics - Observed inference time (s)": "{\"description\": \"min=0.432, mean=0.432, max=0.432, sum=0.863 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4315712761878967\"}", - "Business Ethics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Business Ethics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Business Ethics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Business Ethics - # prompt tokens": "{\"description\": \"min=674.44, mean=674.44, max=674.44, sum=1348.88 (2)\", \"tab\": \"General information\", \"score\": \"674.44\"}", - "Business Ethics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"business_ethics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_business_ethics\"" - } - } - }, - { - "evaluation_name": "Clinical Knowledge", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Clinical Knowledge", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.781, - "details": { - "description": "min=0.781, mean=0.781, max=0.781, sum=1.562 (2)", - "tab": "Accuracy", - "Clinical Knowledge - Observed inference time (s)": "{\"description\": \"min=0.42, mean=0.42, max=0.42, sum=0.841 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4204666920428006\"}", - "Clinical Knowledge - # eval": "{\"description\": \"min=265, mean=265, max=265, sum=530 (2)\", \"tab\": \"General information\", \"score\": \"265.0\"}", - "Clinical Knowledge - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Clinical Knowledge - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Clinical Knowledge - # prompt tokens": "{\"description\": \"min=487.374, mean=487.374, max=487.374, sum=974.747 (2)\", \"tab\": \"General information\", \"score\": \"487.3735849056604\"}", - "Clinical Knowledge - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"clinical_knowledge\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_clinical_knowledge\"" - } - } - }, - { - "evaluation_name": "Conceptual Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Conceptual Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.634, - "details": { - "description": "min=0.634, mean=0.634, max=0.634, sum=1.268 (2)", - "tab": "Accuracy", - "Conceptual Physics - Observed inference time (s)": "{\"description\": \"min=0.412, mean=0.412, max=0.412, sum=0.824 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4118805824442113\"}", - "Conceptual Physics - # eval": "{\"description\": \"min=235, mean=235, max=235, sum=470 (2)\", \"tab\": \"General information\", \"score\": \"235.0\"}", - "Conceptual Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Conceptual Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Conceptual Physics - # prompt tokens": "{\"description\": \"min=333.153, mean=333.153, max=333.153, sum=666.306 (2)\", \"tab\": \"General information\", \"score\": \"333.1531914893617\"}", - "Conceptual Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"conceptual_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_conceptual_physics\"" - } - } - }, - { - "evaluation_name": "Electrical Engineering", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Electrical Engineering", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.662, - "details": { - "description": "min=0.662, mean=0.662, max=0.662, sum=1.324 (2)", - "tab": "Accuracy", - "Electrical Engineering - Observed inference time (s)": "{\"description\": \"min=0.428, mean=0.428, max=0.428, sum=0.856 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.42821227435407966\"}", - "Electrical Engineering - # eval": "{\"description\": \"min=145, mean=145, max=145, sum=290 (2)\", \"tab\": \"General information\", \"score\": \"145.0\"}", - "Electrical Engineering - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Electrical Engineering - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Electrical Engineering - # prompt tokens": "{\"description\": \"min=497.779, mean=497.779, max=497.779, sum=995.559 (2)\", \"tab\": \"General information\", \"score\": \"497.7793103448276\"}", - "Electrical Engineering - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"electrical_engineering\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_electrical_engineering\"" - } - } - }, - { - "evaluation_name": "Elementary Mathematics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Elementary Mathematics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.481, - "details": { - "description": "min=0.481, mean=0.481, max=0.481, sum=0.963 (2)", - "tab": "Accuracy", - "Elementary Mathematics - Observed inference time (s)": "{\"description\": \"min=0.427, mean=0.427, max=0.427, sum=0.853 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4265344634888664\"}", - "Elementary Mathematics - # eval": "{\"description\": \"min=378, mean=378, max=378, sum=756 (2)\", \"tab\": \"General information\", \"score\": \"378.0\"}", - "Elementary Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Elementary Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Elementary Mathematics - # prompt tokens": "{\"description\": \"min=609.156, mean=609.156, max=609.156, sum=1218.312 (2)\", \"tab\": \"General information\", \"score\": \"609.1560846560847\"}", - "Elementary Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"elementary_mathematics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_elementary_mathematics\"" - } - } - }, - { - "evaluation_name": "Formal Logic", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Formal Logic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.444, - "details": { - "description": "min=0.444, mean=0.444, max=0.444, sum=0.889 (2)", - "tab": "Accuracy", - "Formal Logic - Observed inference time (s)": "{\"description\": \"min=0.411, mean=0.411, max=0.411, sum=0.821 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4107102117841206\"}", - "Formal Logic - # eval": "{\"description\": \"min=126, mean=126, max=126, sum=252 (2)\", \"tab\": \"General information\", \"score\": \"126.0\"}", - "Formal Logic - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Formal Logic - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Formal Logic - # prompt tokens": "{\"description\": \"min=691.81, mean=691.81, max=691.81, sum=1383.619 (2)\", \"tab\": \"General information\", \"score\": \"691.8095238095239\"}", - "Formal Logic - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"formal_logic\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_formal_logic\"" - } - } - }, - { - "evaluation_name": "High School World History", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on High School World History", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.827, - "details": { - "description": "min=0.827, mean=0.827, max=0.827, sum=1.654 (2)", - "tab": "Accuracy", - "High School Biology - Observed inference time (s)": "{\"description\": \"min=0.424, mean=0.424, max=0.424, sum=0.847 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.42357982127897204\"}", - "High School Chemistry - Observed inference time (s)": "{\"description\": \"min=0.412, mean=0.412, max=0.412, sum=0.825 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.41242665375394777\"}", - "High School Computer Science - Observed inference time (s)": "{\"description\": \"min=0.445, mean=0.445, max=0.445, sum=0.89 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.44495458364486695\"}", - "High School European History - Observed inference time (s)": "{\"description\": \"min=0.544, mean=0.544, max=0.544, sum=1.088 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5441486705433238\"}", - "High School Geography - Observed inference time (s)": "{\"description\": \"min=0.415, mean=0.415, max=0.415, sum=0.83 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4149725003675981\"}", - "High School Government And Politics - Observed inference time (s)": "{\"description\": \"min=0.383, mean=0.383, max=0.383, sum=0.766 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.38312110629106433\"}", - "High School Macroeconomics - Observed inference time (s)": "{\"description\": \"min=0.403, mean=0.403, max=0.403, sum=0.807 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4034240123553154\"}", - "High School Mathematics - Observed inference time (s)": "{\"description\": \"min=0.39, mean=0.39, max=0.39, sum=0.779 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.38954139285617406\"}", - "High School Microeconomics - Observed inference time (s)": "{\"description\": \"min=0.399, mean=0.399, max=0.399, sum=0.798 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3992174813727371\"}", - "High School Physics - Observed inference time (s)": "{\"description\": \"min=0.409, mean=0.409, max=0.409, sum=0.819 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.40926165138648835\"}", - "High School Psychology - Observed inference time (s)": "{\"description\": \"min=0.408, mean=0.408, max=0.408, sum=0.816 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4081065694126514\"}", - "High School Statistics - Observed inference time (s)": "{\"description\": \"min=0.417, mean=0.417, max=0.417, sum=0.833 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4166152830477114\"}", - "High School US History - Observed inference time (s)": "{\"description\": \"min=0.45, mean=0.45, max=0.45, sum=0.901 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4504043985815609\"}", - "High School World History - Observed inference time (s)": "{\"description\": \"min=0.416, mean=0.416, max=0.416, sum=0.833 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4162542166086189\"}", - "High School Biology - # eval": "{\"description\": \"min=310, mean=310, max=310, sum=620 (2)\", \"tab\": \"General information\", \"score\": \"310.0\"}", - "High School Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Biology - # prompt tokens": "{\"description\": \"min=596.894, mean=596.894, max=596.894, sum=1193.787 (2)\", \"tab\": \"General information\", \"score\": \"596.8935483870968\"}", - "High School Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Chemistry - # eval": "{\"description\": \"min=203, mean=203, max=203, sum=406 (2)\", \"tab\": \"General information\", \"score\": \"203.0\"}", - "High School Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # prompt tokens": "{\"description\": \"min=568.665, mean=568.665, max=568.665, sum=1137.33 (2)\", \"tab\": \"General information\", \"score\": \"568.6650246305419\"}", - "High School Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "High School Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # prompt tokens": "{\"description\": \"min=988.57, mean=988.57, max=988.57, sum=1977.14 (2)\", \"tab\": \"General information\", \"score\": \"988.57\"}", - "High School Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School European History - # eval": "{\"description\": \"min=165, mean=165, max=165, sum=330 (2)\", \"tab\": \"General information\", \"score\": \"165.0\"}", - "High School European History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School European History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # prompt tokens": "{\"description\": \"min=3159.636, mean=3159.636, max=3159.636, sum=6319.273 (2)\", \"tab\": \"General information\", \"score\": \"3159.6363636363635\"}", - "High School European History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Geography - # eval": "{\"description\": \"min=198, mean=198, max=198, sum=396 (2)\", \"tab\": \"General information\", \"score\": \"198.0\"}", - "High School Geography - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Geography - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # prompt tokens": "{\"description\": \"min=436.657, mean=436.657, max=436.657, sum=873.313 (2)\", \"tab\": \"General information\", \"score\": \"436.65656565656565\"}", - "High School Geography - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Government And Politics - # eval": "{\"description\": \"min=193, mean=193, max=193, sum=386 (2)\", \"tab\": \"General information\", \"score\": \"193.0\"}", - "High School Government And Politics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Government And Politics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # prompt tokens": "{\"description\": \"min=527.927, mean=527.927, max=527.927, sum=1055.855 (2)\", \"tab\": \"General information\", \"score\": \"527.9274611398964\"}", - "High School Government And Politics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Macroeconomics - # eval": "{\"description\": \"min=390, mean=390, max=390, sum=780 (2)\", \"tab\": \"General information\", \"score\": \"390.0\"}", - "High School Macroeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Macroeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # prompt tokens": "{\"description\": \"min=445.662, mean=445.662, max=445.662, sum=891.323 (2)\", \"tab\": \"General information\", \"score\": \"445.66153846153844\"}", - "High School Macroeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Mathematics - # eval": "{\"description\": \"min=270, mean=270, max=270, sum=540 (2)\", \"tab\": \"General information\", \"score\": \"270.0\"}", - "High School Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # prompt tokens": "{\"description\": \"min=579.181, mean=579.181, max=579.181, sum=1158.363 (2)\", \"tab\": \"General information\", \"score\": \"579.1814814814815\"}", - "High School Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Microeconomics - # eval": "{\"description\": \"min=238, mean=238, max=238, sum=476 (2)\", \"tab\": \"General information\", \"score\": \"238.0\"}", - "High School Microeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Microeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # prompt tokens": "{\"description\": \"min=449.492, mean=449.492, max=449.492, sum=898.983 (2)\", \"tab\": \"General information\", \"score\": \"449.49159663865544\"}", - "High School Microeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Physics - # eval": "{\"description\": \"min=151, mean=151, max=151, sum=302 (2)\", \"tab\": \"General information\", \"score\": \"151.0\"}", - "High School Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # prompt tokens": "{\"description\": \"min=621.788, mean=621.788, max=621.788, sum=1243.576 (2)\", \"tab\": \"General information\", \"score\": \"621.7880794701987\"}", - "High School Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Psychology - # eval": "{\"description\": \"min=545, mean=545, max=545, sum=1090 (2)\", \"tab\": \"General information\", \"score\": \"545.0\"}", - "High School Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # prompt tokens": "{\"description\": \"min=585.919, mean=585.919, max=585.919, sum=1171.839 (2)\", \"tab\": \"General information\", \"score\": \"585.9192660550459\"}", - "High School Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Statistics - # eval": "{\"description\": \"min=216, mean=216, max=216, sum=432 (2)\", \"tab\": \"General information\", \"score\": \"216.0\"}", - "High School Statistics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Statistics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # prompt tokens": "{\"description\": \"min=908.208, mean=908.208, max=908.208, sum=1816.417 (2)\", \"tab\": \"General information\", \"score\": \"908.2083333333334\"}", - "High School Statistics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School US History - # eval": "{\"description\": \"min=204, mean=204, max=204, sum=408 (2)\", \"tab\": \"General information\", \"score\": \"204.0\"}", - "High School US History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School US History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # prompt tokens": "{\"description\": \"min=2535.324, mean=2535.324, max=2535.324, sum=5070.647 (2)\", \"tab\": \"General information\", \"score\": \"2535.323529411765\"}", - "High School US History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School World History - # eval": "{\"description\": \"min=237, mean=237, max=237, sum=474 (2)\", \"tab\": \"General information\", \"score\": \"237.0\"}", - "High School World History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School World History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # prompt tokens": "{\"description\": \"min=1638.219, mean=1638.219, max=1638.219, sum=3276.439 (2)\", \"tab\": \"General information\", \"score\": \"1638.2194092827003\"}", - "High School World History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"high_school_world_history\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_high_school_world_history\"" - } - } - }, - { - "evaluation_name": "Human Sexuality", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Human Sexuality", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.847, - "details": { - "description": "min=0.847, mean=0.847, max=0.847, sum=1.695 (2)", - "tab": "Accuracy", - "Human Aging - Observed inference time (s)": "{\"description\": \"min=0.401, mean=0.401, max=0.401, sum=0.802 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4010318255745242\"}", - "Human Sexuality - Observed inference time (s)": "{\"description\": \"min=0.393, mean=0.393, max=0.393, sum=0.787 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.39331119843111695\"}", - "Human Aging - # eval": "{\"description\": \"min=223, mean=223, max=223, sum=446 (2)\", \"tab\": \"General information\", \"score\": \"223.0\"}", - "Human Aging - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Aging - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Aging - # prompt tokens": "{\"description\": \"min=361.26, mean=361.26, max=361.26, sum=722.52 (2)\", \"tab\": \"General information\", \"score\": \"361.26008968609864\"}", - "Human Aging - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Human Sexuality - # eval": "{\"description\": \"min=131, mean=131, max=131, sum=262 (2)\", \"tab\": \"General information\", \"score\": \"131.0\"}", - "Human Sexuality - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Sexuality - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # prompt tokens": "{\"description\": \"min=403.382, mean=403.382, max=403.382, sum=806.763 (2)\", \"tab\": \"General information\", \"score\": \"403.381679389313\"}", - "Human Sexuality - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"human_sexuality\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_human_sexuality\"" - } - } - }, - { - "evaluation_name": "International Law", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on International Law", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.826, - "details": { - "description": "min=0.826, mean=0.826, max=0.826, sum=1.653 (2)", - "tab": "Accuracy", - "International Law - Observed inference time (s)": "{\"description\": \"min=0.42, mean=0.42, max=0.42, sum=0.841 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.42040472779392213\"}", - "International Law - # eval": "{\"description\": \"min=121, mean=121, max=121, sum=242 (2)\", \"tab\": \"General information\", \"score\": \"121.0\"}", - "International Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "International Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "International Law - # prompt tokens": "{\"description\": \"min=729.463, mean=729.463, max=729.463, sum=1458.926 (2)\", \"tab\": \"General information\", \"score\": \"729.4628099173553\"}", - "International Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"international_law\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_international_law\"" - } - } - }, - { - "evaluation_name": "Logical Fallacies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Logical Fallacies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.779, - "details": { - "description": "min=0.779, mean=0.779, max=0.779, sum=1.558 (2)", - "tab": "Accuracy", - "Logical Fallacies - Observed inference time (s)": "{\"description\": \"min=0.404, mean=0.404, max=0.404, sum=0.809 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4043445353127696\"}", - "Logical Fallacies - # eval": "{\"description\": \"min=163, mean=163, max=163, sum=326 (2)\", \"tab\": \"General information\", \"score\": \"163.0\"}", - "Logical Fallacies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Logical Fallacies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Logical Fallacies - # prompt tokens": "{\"description\": \"min=502.755, mean=502.755, max=502.755, sum=1005.509 (2)\", \"tab\": \"General information\", \"score\": \"502.7546012269939\"}", - "Logical Fallacies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"logical_fallacies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_logical_fallacies\"" - } - } - }, - { - "evaluation_name": "Machine Learning", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Machine Learning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.473, - "details": { - "description": "min=0.473, mean=0.473, max=0.473, sum=0.946 (2)", - "tab": "Accuracy", - "Machine Learning - Observed inference time (s)": "{\"description\": \"min=0.421, mean=0.421, max=0.421, sum=0.842 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.42122456644262585\"}", - "Machine Learning - # eval": "{\"description\": \"min=112, mean=112, max=112, sum=224 (2)\", \"tab\": \"General information\", \"score\": \"112.0\"}", - "Machine Learning - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Machine Learning - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Machine Learning - # prompt tokens": "{\"description\": \"min=730.402, mean=730.402, max=730.402, sum=1460.804 (2)\", \"tab\": \"General information\", \"score\": \"730.4017857142857\"}", - "Machine Learning - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"machine_learning\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_machine_learning\"" - } - } - }, - { - "evaluation_name": "Management", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Management", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.796, - "details": { - "description": "min=0.796, mean=0.796, max=0.796, sum=1.592 (2)", - "tab": "Accuracy", - "Management - Observed inference time (s)": "{\"description\": \"min=0.392, mean=0.392, max=0.392, sum=0.785 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.392485206566968\"}", - "Management - # eval": "{\"description\": \"min=103, mean=103, max=103, sum=206 (2)\", \"tab\": \"General information\", \"score\": \"103.0\"}", - "Management - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Management - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Management - # prompt tokens": "{\"description\": \"min=315.777, mean=315.777, max=315.777, sum=631.553 (2)\", \"tab\": \"General information\", \"score\": \"315.77669902912623\"}", - "Management - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"management\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_management\"" - } - } - }, - { - "evaluation_name": "Marketing", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Marketing", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.902, - "details": { - "description": "min=0.902, mean=0.902, max=0.902, sum=1.803 (2)", - "tab": "Accuracy", - "Marketing - Observed inference time (s)": "{\"description\": \"min=0.407, mean=0.407, max=0.407, sum=0.813 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.406507401384859\"}", - "Marketing - # eval": "{\"description\": \"min=234, mean=234, max=234, sum=468 (2)\", \"tab\": \"General information\", \"score\": \"234.0\"}", - "Marketing - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Marketing - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Marketing - # prompt tokens": "{\"description\": \"min=472.628, mean=472.628, max=472.628, sum=945.256 (2)\", \"tab\": \"General information\", \"score\": \"472.62820512820514\"}", - "Marketing - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"marketing\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_marketing\"" - } - } - }, - { - "evaluation_name": "Medical Genetics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Medical Genetics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.76, - "details": { - "description": "min=0.76, mean=0.76, max=0.76, sum=1.52 (2)", - "tab": "Accuracy", - "Medical Genetics - Observed inference time (s)": "{\"description\": \"min=0.417, mean=0.417, max=0.417, sum=0.835 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.41734427213668823\"}", - "Medical Genetics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Medical Genetics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Medical Genetics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Medical Genetics - # prompt tokens": "{\"description\": \"min=408.14, mean=408.14, max=408.14, sum=816.28 (2)\", \"tab\": \"General information\", \"score\": \"408.14\"}", - "Medical Genetics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"medical_genetics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_medical_genetics\"" - } - } - }, - { - "evaluation_name": "Miscellaneous", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Miscellaneous", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.875, - "details": { - "description": "min=0.875, mean=0.875, max=0.875, sum=1.75 (2)", - "tab": "Accuracy", - "Miscellaneous - Observed inference time (s)": "{\"description\": \"min=0.407, mean=0.407, max=0.407, sum=0.814 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.40693108880200146\"}", - "Miscellaneous - # eval": "{\"description\": \"min=783, mean=783, max=783, sum=1566 (2)\", \"tab\": \"General information\", \"score\": \"783.0\"}", - "Miscellaneous - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Miscellaneous - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Miscellaneous - # prompt tokens": "{\"description\": \"min=345.913, mean=345.913, max=345.913, sum=691.826 (2)\", \"tab\": \"General information\", \"score\": \"345.9131545338442\"}", - "Miscellaneous - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"miscellaneous\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_miscellaneous\"" - } - } - }, - { - "evaluation_name": "Moral Scenarios", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Moral Scenarios", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.28, - "details": { - "description": "min=0.28, mean=0.28, max=0.28, sum=0.561 (2)", - "tab": "Accuracy", - "Moral Disputes - Observed inference time (s)": "{\"description\": \"min=0.424, mean=0.424, max=0.424, sum=0.848 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4239204674097844\"}", - "Moral Scenarios - Observed inference time (s)": "{\"description\": \"min=0.433, mean=0.433, max=0.433, sum=0.866 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.43297034721800737\"}", - "Moral Disputes - # eval": "{\"description\": \"min=346, mean=346, max=346, sum=692 (2)\", \"tab\": \"General information\", \"score\": \"346.0\"}", - "Moral Disputes - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Disputes - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Disputes - # prompt tokens": "{\"description\": \"min=542.506, mean=542.506, max=542.506, sum=1085.012 (2)\", \"tab\": \"General information\", \"score\": \"542.5057803468208\"}", - "Moral Disputes - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Moral Scenarios - # eval": "{\"description\": \"min=895, mean=895, max=895, sum=1790 (2)\", \"tab\": \"General information\", \"score\": \"895.0\"}", - "Moral Scenarios - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Scenarios - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # prompt tokens": "{\"description\": \"min=756.479, mean=756.479, max=756.479, sum=1512.959 (2)\", \"tab\": \"General information\", \"score\": \"756.4793296089385\"}", - "Moral Scenarios - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"moral_scenarios\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_moral_scenarios\"" - } - } - }, - { - "evaluation_name": "Nutrition", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Nutrition", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.725, - "details": { - "description": "min=0.725, mean=0.725, max=0.725, sum=1.451 (2)", - "tab": "Accuracy", - "Nutrition - Observed inference time (s)": "{\"description\": \"min=0.417, mean=0.417, max=0.417, sum=0.835 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.41727598430284485\"}", - "Nutrition - # eval": "{\"description\": \"min=306, mean=306, max=306, sum=612 (2)\", \"tab\": \"General information\", \"score\": \"306.0\"}", - "Nutrition - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Nutrition - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Nutrition - # prompt tokens": "{\"description\": \"min=695.922, mean=695.922, max=695.922, sum=1391.843 (2)\", \"tab\": \"General information\", \"score\": \"695.9215686274509\"}", - "Nutrition - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"nutrition\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_nutrition\"" - } - } - }, - { - "evaluation_name": "Prehistory", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Prehistory", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.79, - "details": { - "description": "min=0.79, mean=0.79, max=0.79, sum=1.58 (2)", - "tab": "Accuracy", - "Prehistory - Observed inference time (s)": "{\"description\": \"min=0.43, mean=0.43, max=0.43, sum=0.861 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4303552037403907\"}", - "Prehistory - # eval": "{\"description\": \"min=324, mean=324, max=324, sum=648 (2)\", \"tab\": \"General information\", \"score\": \"324.0\"}", - "Prehistory - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Prehistory - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Prehistory - # prompt tokens": "{\"description\": \"min=619.185, mean=619.185, max=619.185, sum=1238.37 (2)\", \"tab\": \"General information\", \"score\": \"619.1851851851852\"}", - "Prehistory - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"prehistory\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_prehistory\"" - } - } - }, - { - "evaluation_name": "Public Relations", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Public Relations", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.664, - "details": { - "description": "min=0.664, mean=0.664, max=0.664, sum=1.327 (2)", - "tab": "Accuracy", - "Public Relations - Observed inference time (s)": "{\"description\": \"min=0.428, mean=0.428, max=0.428, sum=0.855 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.42750670259649104\"}", - "Public Relations - # eval": "{\"description\": \"min=110, mean=110, max=110, sum=220 (2)\", \"tab\": \"General information\", \"score\": \"110.0\"}", - "Public Relations - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Public Relations - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Public Relations - # prompt tokens": "{\"description\": \"min=474.827, mean=474.827, max=474.827, sum=949.655 (2)\", \"tab\": \"General information\", \"score\": \"474.8272727272727\"}", - "Public Relations - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"public_relations\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_public_relations\"" - } - } - }, - { - "evaluation_name": "Security Studies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Security Studies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.78, - "details": { - "description": "min=0.78, mean=0.78, max=0.78, sum=1.559 (2)", - "tab": "Accuracy", - "Security Studies - Observed inference time (s)": "{\"description\": \"min=0.466, mean=0.466, max=0.466, sum=0.933 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4662662194699657\"}", - "Security Studies - # eval": "{\"description\": \"min=245, mean=245, max=245, sum=490 (2)\", \"tab\": \"General information\", \"score\": \"245.0\"}", - "Security Studies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Security Studies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Security Studies - # prompt tokens": "{\"description\": \"min=1377.531, mean=1377.531, max=1377.531, sum=2755.061 (2)\", \"tab\": \"General information\", \"score\": \"1377.530612244898\"}", - "Security Studies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"security_studies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_security_studies\"" - } - } - }, - { - "evaluation_name": "Sociology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Sociology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.891, - "details": { - "description": "min=0.891, mean=0.891, max=0.891, sum=1.781 (2)", - "tab": "Accuracy", - "Sociology - Observed inference time (s)": "{\"description\": \"min=0.416, mean=0.416, max=0.416, sum=0.832 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4159522590352528\"}", - "Sociology - # eval": "{\"description\": \"min=201, mean=201, max=201, sum=402 (2)\", \"tab\": \"General information\", \"score\": \"201.0\"}", - "Sociology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Sociology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Sociology - # prompt tokens": "{\"description\": \"min=508.478, mean=508.478, max=508.478, sum=1016.955 (2)\", \"tab\": \"General information\", \"score\": \"508.4776119402985\"}", - "Sociology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"sociology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_sociology\"" - } - } - }, - { - "evaluation_name": "Virology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Virology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.536, - "details": { - "description": "min=0.536, mean=0.536, max=0.536, sum=1.072 (2)", - "tab": "Accuracy", - "Virology - Observed inference time (s)": "{\"description\": \"min=0.405, mean=0.405, max=0.405, sum=0.809 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.40467354332108096\"}", - "Virology - # eval": "{\"description\": \"min=166, mean=166, max=166, sum=332 (2)\", \"tab\": \"General information\", \"score\": \"166.0\"}", - "Virology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Virology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Virology - # prompt tokens": "{\"description\": \"min=405.108, mean=405.108, max=405.108, sum=810.217 (2)\", \"tab\": \"General information\", \"score\": \"405.10843373493975\"}", - "Virology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"virology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_virology\"" - } - } - }, - { - "evaluation_name": "World Religions", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on World Religions", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.854, - "details": { - "description": "min=0.854, mean=0.854, max=0.854, sum=1.708 (2)", - "tab": "Accuracy", - "World Religions - Observed inference time (s)": "{\"description\": \"min=0.393, mean=0.393, max=0.393, sum=0.787 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.39336834455791275\"}", - "World Religions - # eval": "{\"description\": \"min=171, mean=171, max=171, sum=342 (2)\", \"tab\": \"General information\", \"score\": \"171.0\"}", - "World Religions - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "World Religions - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "World Religions - # prompt tokens": "{\"description\": \"min=304.474, mean=304.474, max=304.474, sum=608.947 (2)\", \"tab\": \"General information\", \"score\": \"304.4736842105263\"}", - "World Religions - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"world_religions\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_world_religions\"" - } - } - }, - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.565, - "details": { - "description": "", - "tab": "Efficiency" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/solshine_brimful-merged-replete.json b/data/models/solshine_brimful-merged-replete.json deleted file mode 100644 index 62def3251ee6f7fc77d7e6a21af86762b88c6860..0000000000000000000000000000000000000000 --- a/data/models/solshine_brimful-merged-replete.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Brimful-merged-replete", - "id": "Solshine/Brimful-merged-replete", - "developer": "Solshine", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "12.277" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Solshine_Brimful-merged-replete/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1761 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2883 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.003 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2576 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3421 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1085 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/solshine_llama-3-1-big-thoughtful-passthrough-merge-2.json b/data/models/solshine_llama-3-1-big-thoughtful-passthrough-merge-2.json deleted file mode 100644 index f8171916cbb7a98edee242c05f5b36d2ed182754..0000000000000000000000000000000000000000 --- a/data/models/solshine_llama-3-1-big-thoughtful-passthrough-merge-2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-1-big-thoughtful-passthrough-merge-2", - "id": "Solshine/Llama-3-1-big-thoughtful-passthrough-merge-2", - "developer": "Solshine", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "18.5" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Solshine_Llama-3-1-big-thoughtful-passthrough-merge-2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2547 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3209 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0106 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2592 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3889 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1185 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/someon98_qwen-coma-0.5b.json b/data/models/someon98_qwen-coma-0.5b.json deleted file mode 100644 index 3e59ff87abe77c5a8af2b48327ebc41b131cd2bd..0000000000000000000000000000000000000000 --- a/data/models/someon98_qwen-coma-0.5b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "qwen-CoMa-0.5b", - "id": "someon98/qwen-CoMa-0.5b", - "developer": "someon98", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/someon98_qwen-CoMa-0.5b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2277 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2953 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0045 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2399 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4046 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1099 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sometimesanotion_chocotrio-14b-v1.json b/data/models/sometimesanotion_chocotrio-14b-v1.json deleted file mode 100644 index 098698c137275768321f16aff58dbb0d44e92159..0000000000000000000000000000000000000000 --- a/data/models/sometimesanotion_chocotrio-14b-v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ChocoTrio-14B-v1", - "id": "sometimesanotion/ChocoTrio-14B-v1", - "developer": "sometimesanotion", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sometimesanotion_ChocoTrio-14B-v1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7089 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6506 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3973 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3851 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4821 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.537 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sometimesanotion_if-reasoning-experiment-40.json b/data/models/sometimesanotion_if-reasoning-experiment-40.json deleted file mode 100644 index 8471404c5739ee760f197e805ccc1bb1768abf15..0000000000000000000000000000000000000000 --- a/data/models/sometimesanotion_if-reasoning-experiment-40.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "IF-reasoning-experiment-40", - "id": "sometimesanotion/IF-reasoning-experiment-40", - "developer": "sometimesanotion", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sometimesanotion_IF-reasoning-experiment-40/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.633 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6112 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3716 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.38 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5194 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5025 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sometimesanotion_if-reasoning-experiment-80.json b/data/models/sometimesanotion_if-reasoning-experiment-80.json deleted file mode 100644 index fa896fd8fdcf0cc257a8447a076cc0c5d6223702..0000000000000000000000000000000000000000 --- a/data/models/sometimesanotion_if-reasoning-experiment-80.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "IF-reasoning-experiment-80", - "id": "sometimesanotion/IF-reasoning-experiment-80", - "developer": "sometimesanotion", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.383" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sometimesanotion_IF-reasoning-experiment-80/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5463 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.421 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0989 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2844 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5025 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3368 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sometimesanotion_kytheramix-7b-v0.2.json b/data/models/sometimesanotion_kytheramix-7b-v0.2.json deleted file mode 100644 index fe66cb899f5fa49a28db6870cfc762804fe64d89..0000000000000000000000000000000000000000 --- a/data/models/sometimesanotion_kytheramix-7b-v0.2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "KytheraMix-7B-v0.2", - "id": "sometimesanotion/KytheraMix-7B-v0.2", - "developer": "sometimesanotion", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.613" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sometimesanotion_KytheraMix-7B-v0.2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6129 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5635 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2923 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3356 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4594 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4505 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sometimesanotion_lamarck-14b-prose-model_stock.json b/data/models/sometimesanotion_lamarck-14b-prose-model_stock.json deleted file mode 100644 index e1cc9d6c929bf4df3356a08764e54fd64e131626..0000000000000000000000000000000000000000 --- a/data/models/sometimesanotion_lamarck-14b-prose-model_stock.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "lamarck-14b-prose-model_stock", - "id": "sometimesanotion/lamarck-14b-prose-model_stock", - "developer": "sometimesanotion", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sometimesanotion_lamarck-14b-prose-model_stock/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4276 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6488 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3414 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3935 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4846 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5354 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sometimesanotion_lamarck-14b-reason-model_stock.json b/data/models/sometimesanotion_lamarck-14b-reason-model_stock.json deleted file mode 100644 index 69764232ee43864c57b9946b465d21c65d7e1800..0000000000000000000000000000000000000000 --- a/data/models/sometimesanotion_lamarck-14b-reason-model_stock.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "lamarck-14b-reason-model_stock", - "id": "sometimesanotion/lamarck-14b-reason-model_stock", - "developer": "sometimesanotion", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sometimesanotion_lamarck-14b-reason-model_stock/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4965 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6569 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.358 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3842 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4741 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5402 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sometimesanotion_lamarck-14b-v0.1-experimental.json b/data/models/sometimesanotion_lamarck-14b-v0.1-experimental.json deleted file mode 100644 index 3f9feb639ac33f9f2904f89c75a5f59a8d23de01..0000000000000000000000000000000000000000 --- a/data/models/sometimesanotion_lamarck-14b-v0.1-experimental.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Lamarck-14B-v0.1-experimental", - "id": "sometimesanotion/Lamarck-14B-v0.1-experimental", - "developer": "sometimesanotion", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sometimesanotion_Lamarck-14B-v0.1-experimental/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5354 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6583 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.358 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3817 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4728 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5408 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sometimesanotion_lamarck-14b-v0.3.json b/data/models/sometimesanotion_lamarck-14b-v0.3.json deleted file mode 100644 index 90c057f81ade73874ded6470511cc480c2bee04f..0000000000000000000000000000000000000000 --- a/data/models/sometimesanotion_lamarck-14b-v0.3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Lamarck-14B-v0.3", - "id": "sometimesanotion/Lamarck-14B-v0.3", - "developer": "sometimesanotion", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sometimesanotion_Lamarck-14B-v0.3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5032 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6611 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3406 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3884 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4688 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5411 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sometimesanotion_lamarck-14b-v0.4-qwenvergence.json b/data/models/sometimesanotion_lamarck-14b-v0.4-qwenvergence.json deleted file mode 100644 index b6134b8446b625cc528f5f8fb6431983aaeb6dfb..0000000000000000000000000000000000000000 --- a/data/models/sometimesanotion_lamarck-14b-v0.4-qwenvergence.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Lamarck-14B-v0.4-Qwenvergence", - "id": "sometimesanotion/Lamarck-14B-v0.4-Qwenvergence", - "developer": "sometimesanotion", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sometimesanotion_Lamarck-14B-v0.4-Qwenvergence/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4906 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6535 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3399 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3784 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4847 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5406 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sometimesanotion_lamarck-14b-v0.6-002-model_stock.json b/data/models/sometimesanotion_lamarck-14b-v0.6-002-model_stock.json deleted file mode 100644 index f9c7a546392bda6b33c414f73b1227a41b08b324..0000000000000000000000000000000000000000 --- a/data/models/sometimesanotion_lamarck-14b-v0.6-002-model_stock.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Lamarck-14B-v0.6-002-model_stock", - "id": "sometimesanotion/Lamarck-14B-v0.6-002-model_stock", - "developer": "sometimesanotion", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sometimesanotion_Lamarck-14B-v0.6-002-model_stock/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6692 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6143 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3776 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3742 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.518 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5054 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sometimesanotion_lamarck-14b-v0.6-model_stock.json b/data/models/sometimesanotion_lamarck-14b-v0.6-model_stock.json deleted file mode 100644 index dfde63209670aa2480f2555446bbd430a32c6be1..0000000000000000000000000000000000000000 --- a/data/models/sometimesanotion_lamarck-14b-v0.6-model_stock.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Lamarck-14B-v0.6-model_stock", - "id": "sometimesanotion/Lamarck-14B-v0.6-model_stock", - "developer": "sometimesanotion", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sometimesanotion_Lamarck-14B-v0.6-model_stock/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.679 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6269 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4245 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3842 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5007 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5198 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sometimesanotion_lamarck-14b-v0.6.json b/data/models/sometimesanotion_lamarck-14b-v0.6.json deleted file mode 100644 index c9dee409ea3a6b6afcfd7ea0ff426b5a12634e39..0000000000000000000000000000000000000000 --- a/data/models/sometimesanotion_lamarck-14b-v0.6.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Lamarck-14B-v0.6", - "id": "sometimesanotion/Lamarck-14B-v0.6", - "developer": "sometimesanotion", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sometimesanotion_Lamarck-14B-v0.6/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6973 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.646 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4041 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3893 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4847 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.54 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sometimesanotion_lamarck-14b-v0.7-fusion.json b/data/models/sometimesanotion_lamarck-14b-v0.7-fusion.json deleted file mode 100644 index 0ee12c6e66a32997dc219e9c057867494e1e0650..0000000000000000000000000000000000000000 --- a/data/models/sometimesanotion_lamarck-14b-v0.7-fusion.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Lamarck-14B-v0.7-Fusion", - "id": "sometimesanotion/Lamarck-14B-v0.7-Fusion", - "developer": "sometimesanotion", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sometimesanotion_Lamarck-14B-v0.7-Fusion/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6821 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6544 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4041 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.401 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4991 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5391 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sometimesanotion_lamarck-14b-v0.7-rc1.json b/data/models/sometimesanotion_lamarck-14b-v0.7-rc1.json deleted file mode 100644 index 687bc2e22ace67c51492dcd7a53519cbc77ee5c2..0000000000000000000000000000000000000000 --- a/data/models/sometimesanotion_lamarck-14b-v0.7-rc1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Lamarck-14B-v0.7-rc1", - "id": "sometimesanotion/Lamarck-14B-v0.7-rc1", - "developer": "sometimesanotion", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sometimesanotion_Lamarck-14B-v0.7-rc1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7305 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6486 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3852 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3893 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4715 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5416 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sometimesanotion_lamarck-14b-v0.7-rc4.json b/data/models/sometimesanotion_lamarck-14b-v0.7-rc4.json deleted file mode 100644 index 578a2249f5ca01061c75252b76124b8df68ef4f6..0000000000000000000000000000000000000000 --- a/data/models/sometimesanotion_lamarck-14b-v0.7-rc4.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Lamarck-14B-v0.7-rc4", - "id": "sometimesanotion/Lamarck-14B-v0.7-rc4", - "developer": "sometimesanotion", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sometimesanotion_Lamarck-14B-v0.7-rc4/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7211 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.651 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4026 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3893 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4912 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.54 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sometimesanotion_lamarckinfusion-14b-v1.json b/data/models/sometimesanotion_lamarckinfusion-14b-v1.json deleted file mode 100644 index 94b68d5f42b30256ab11d2738072fa9cbe63568c..0000000000000000000000000000000000000000 --- a/data/models/sometimesanotion_lamarckinfusion-14b-v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "LamarckInfusion-14B-v1", - "id": "sometimesanotion/LamarckInfusion-14B-v1", - "developer": "sometimesanotion", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sometimesanotion_LamarckInfusion-14B-v1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7198 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6539 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4169 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3909 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4899 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5376 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sometimesanotion_lamarckinfusion-14b-v2-hi.json b/data/models/sometimesanotion_lamarckinfusion-14b-v2-hi.json deleted file mode 100644 index e7a4c07ad2bd9b350150f54411b17cc96c43a39c..0000000000000000000000000000000000000000 --- a/data/models/sometimesanotion_lamarckinfusion-14b-v2-hi.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "LamarckInfusion-14B-v2-hi", - "id": "sometimesanotion/LamarckInfusion-14B-v2-hi", - "developer": "sometimesanotion", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sometimesanotion_LamarckInfusion-14B-v2-hi/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6855 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6555 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.423 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3884 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4847 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5405 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sometimesanotion_lamarckinfusion-14b-v2-lo.json b/data/models/sometimesanotion_lamarckinfusion-14b-v2-lo.json deleted file mode 100644 index 6df9925158eff94795a937488a8fe9698d4b533c..0000000000000000000000000000000000000000 --- a/data/models/sometimesanotion_lamarckinfusion-14b-v2-lo.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "LamarckInfusion-14B-v2-lo", - "id": "sometimesanotion/LamarckInfusion-14B-v2-lo", - "developer": "sometimesanotion", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sometimesanotion_LamarckInfusion-14B-v2-lo/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6788 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6528 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4237 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3859 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4991 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5397 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sometimesanotion_lamarckinfusion-14b-v2.json b/data/models/sometimesanotion_lamarckinfusion-14b-v2.json deleted file mode 100644 index 0335537abf7d627386379f9af46b87106185280d..0000000000000000000000000000000000000000 --- a/data/models/sometimesanotion_lamarckinfusion-14b-v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "LamarckInfusion-14B-v2", - "id": "sometimesanotion/LamarckInfusion-14B-v2", - "developer": "sometimesanotion", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sometimesanotion_LamarckInfusion-14B-v2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6812 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6564 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4388 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3876 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4993 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5416 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sometimesanotion_lamarckinfusion-14b-v3.json b/data/models/sometimesanotion_lamarckinfusion-14b-v3.json deleted file mode 100644 index ec17cbfccacda2371f4d248d5e7b944729775eec..0000000000000000000000000000000000000000 --- a/data/models/sometimesanotion_lamarckinfusion-14b-v3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "LamarckInfusion-14B-v3", - "id": "sometimesanotion/LamarckInfusion-14B-v3", - "developer": "sometimesanotion", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sometimesanotion_LamarckInfusion-14B-v3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7131 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6518 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4124 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3867 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.482 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5407 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sometimesanotion_qwen-14b-prosestock-v4.json b/data/models/sometimesanotion_qwen-14b-prosestock-v4.json deleted file mode 100644 index 977bd7eb6e434ee2996611566530e315011719bf..0000000000000000000000000000000000000000 --- a/data/models/sometimesanotion_qwen-14b-prosestock-v4.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen-14B-ProseStock-v4", - "id": "sometimesanotion/Qwen-14B-ProseStock-v4", - "developer": "sometimesanotion", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sometimesanotion_Qwen-14B-ProseStock-v4/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4942 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6498 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.364 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3884 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4938 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5386 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sometimesanotion_qwen-2.5-14b-virmarckeoso.json b/data/models/sometimesanotion_qwen-2.5-14b-virmarckeoso.json deleted file mode 100644 index bf6baec64e69f3199e64df6848ff340e9b3fcdf2..0000000000000000000000000000000000000000 --- a/data/models/sometimesanotion_qwen-2.5-14b-virmarckeoso.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen-2.5-14B-Virmarckeoso", - "id": "sometimesanotion/Qwen-2.5-14B-Virmarckeoso", - "developer": "sometimesanotion", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sometimesanotion_Qwen-2.5-14B-Virmarckeoso/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4813 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.657 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3565 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3792 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4794 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5377 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sometimesanotion_qwen2.5-14b-vimarckoso-v2.json b/data/models/sometimesanotion_qwen2.5-14b-vimarckoso-v2.json deleted file mode 100644 index bdbfbe07275782cda442bec597a536bca4b4f132..0000000000000000000000000000000000000000 --- a/data/models/sometimesanotion_qwen2.5-14b-vimarckoso-v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-14B-Vimarckoso-v2", - "id": "sometimesanotion/Qwen2.5-14B-Vimarckoso-v2", - "developer": "sometimesanotion", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sometimesanotion_Qwen2.5-14B-Vimarckoso-v2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4505 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.655 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.358 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3826 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4819 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.538 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sometimesanotion_qwen2.5-14b-vimarckoso-v3-if-variant.json b/data/models/sometimesanotion_qwen2.5-14b-vimarckoso-v3-if-variant.json deleted file mode 100644 index 5a49f12bf9f72611394a12e52f3d8981f420b728..0000000000000000000000000000000000000000 --- a/data/models/sometimesanotion_qwen2.5-14b-vimarckoso-v3-if-variant.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-14B-Vimarckoso-v3-IF-Variant", - "id": "sometimesanotion/Qwen2.5-14B-Vimarckoso-v3-IF-Variant", - "developer": "sometimesanotion", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sometimesanotion_Qwen2.5-14B-Vimarckoso-v3-IF-Variant/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6413 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5521 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2545 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3473 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5319 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4589 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sometimesanotion_qwen2.5-14b-vimarckoso-v3-model_stock.json b/data/models/sometimesanotion_qwen2.5-14b-vimarckoso-v3-model_stock.json deleted file mode 100644 index da96a67fe8b5ce78565e7004b6218fe7bf61a7fc..0000000000000000000000000000000000000000 --- a/data/models/sometimesanotion_qwen2.5-14b-vimarckoso-v3-model_stock.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-14B-Vimarckoso-v3-model_stock", - "id": "sometimesanotion/Qwen2.5-14B-Vimarckoso-v3-model_stock", - "developer": "sometimesanotion", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sometimesanotion_Qwen2.5-14B-Vimarckoso-v3-model_stock/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7162 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6421 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4245 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.38 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4781 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5316 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sometimesanotion_qwen2.5-14b-vimarckoso-v3-prose01.json b/data/models/sometimesanotion_qwen2.5-14b-vimarckoso-v3-prose01.json deleted file mode 100644 index a3e66be3acea3c3e92b84784418294c54849991e..0000000000000000000000000000000000000000 --- a/data/models/sometimesanotion_qwen2.5-14b-vimarckoso-v3-prose01.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-14B-Vimarckoso-v3-Prose01", - "id": "sometimesanotion/Qwen2.5-14B-Vimarckoso-v3-Prose01", - "developer": "sometimesanotion", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sometimesanotion_Qwen2.5-14B-Vimarckoso-v3-Prose01/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6872 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6359 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3995 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3867 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4807 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5275 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sometimesanotion_qwen2.5-14b-vimarckoso-v3.json b/data/models/sometimesanotion_qwen2.5-14b-vimarckoso-v3.json deleted file mode 100644 index b41abdc9e2befc842f2715dc45e632f9413a577d..0000000000000000000000000000000000000000 --- a/data/models/sometimesanotion_qwen2.5-14b-vimarckoso-v3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-14B-Vimarckoso-v3", - "id": "sometimesanotion/Qwen2.5-14B-Vimarckoso-v3", - "developer": "sometimesanotion", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sometimesanotion_Qwen2.5-14B-Vimarckoso-v3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7257 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6415 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4003 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.38 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4807 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5343 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sometimesanotion_qwen2.5-14b-vimarckoso.json b/data/models/sometimesanotion_qwen2.5-14b-vimarckoso.json deleted file mode 100644 index 46d24d874be7d04853cea244f5f418cf38ba2309..0000000000000000000000000000000000000000 --- a/data/models/sometimesanotion_qwen2.5-14b-vimarckoso.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-14B-Vimarckoso", - "id": "sometimesanotion/Qwen2.5-14B-Vimarckoso", - "developer": "sometimesanotion", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sometimesanotion_Qwen2.5-14B-Vimarckoso/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4574 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6446 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3384 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3926 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4859 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5329 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sometimesanotion_qwen2.5-7b-gordion-v0.1-prose.json b/data/models/sometimesanotion_qwen2.5-7b-gordion-v0.1-prose.json deleted file mode 100644 index 4ad44026ff47bd7e9368cfe5a9e84a495ba18ce8..0000000000000000000000000000000000000000 --- a/data/models/sometimesanotion_qwen2.5-7b-gordion-v0.1-prose.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-7B-Gordion-v0.1-Prose", - "id": "sometimesanotion/Qwen2.5-7B-Gordion-v0.1-Prose", - "developer": "sometimesanotion", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.613" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sometimesanotion_Qwen2.5-7B-Gordion-v0.1-Prose/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5347 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5599 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2893 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3205 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4502 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4525 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sometimesanotion_qwen2.5-7b-gordion-v0.1-reason.json b/data/models/sometimesanotion_qwen2.5-7b-gordion-v0.1-reason.json deleted file mode 100644 index 346de6606f7bb54062ca6dcd63cb80df235d3931..0000000000000000000000000000000000000000 --- a/data/models/sometimesanotion_qwen2.5-7b-gordion-v0.1-reason.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-7B-Gordion-v0.1-Reason", - "id": "sometimesanotion/Qwen2.5-7B-Gordion-v0.1-Reason", - "developer": "sometimesanotion", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.613" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sometimesanotion_Qwen2.5-7B-Gordion-v0.1-Reason/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4917 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5498 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2621 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3406 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4434 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4307 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sometimesanotion_qwen2.5-7b-gordion-v0.1.json b/data/models/sometimesanotion_qwen2.5-7b-gordion-v0.1.json deleted file mode 100644 index d40c2c1446aeefc4197cb1f32b380118598d7951..0000000000000000000000000000000000000000 --- a/data/models/sometimesanotion_qwen2.5-7b-gordion-v0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-7B-Gordion-v0.1", - "id": "sometimesanotion/Qwen2.5-7B-Gordion-v0.1", - "developer": "sometimesanotion", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.613" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sometimesanotion_Qwen2.5-7B-Gordion-v0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7482 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5524 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2915 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3079 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4016 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.43 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sometimesanotion_qwentessential-14b-v1.json b/data/models/sometimesanotion_qwentessential-14b-v1.json deleted file mode 100644 index 8ad4d242867bb55724db50277f69ae09bd869483..0000000000000000000000000000000000000000 --- a/data/models/sometimesanotion_qwentessential-14b-v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwentessential-14B-v1", - "id": "sometimesanotion/Qwentessential-14B-v1", - "developer": "sometimesanotion", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sometimesanotion_Qwentessential-14B-v1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6279 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6545 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4071 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3876 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4873 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5381 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sometimesanotion_qwentinuum-14b-v013.json b/data/models/sometimesanotion_qwentinuum-14b-v013.json deleted file mode 100644 index a5cfcd28b22c83112403cfa85e01fd96c76e11aa..0000000000000000000000000000000000000000 --- a/data/models/sometimesanotion_qwentinuum-14b-v013.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwentinuum-14B-v013", - "id": "sometimesanotion/Qwentinuum-14B-v013", - "developer": "sometimesanotion", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sometimesanotion_Qwentinuum-14B-v013/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6711 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6087 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3708 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3574 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5154 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4991 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sometimesanotion_qwentinuum-14b-v1.json b/data/models/sometimesanotion_qwentinuum-14b-v1.json deleted file mode 100644 index 5a094effd4a5cbd23682b0bc458d972e8f8aa818..0000000000000000000000000000000000000000 --- a/data/models/sometimesanotion_qwentinuum-14b-v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwentinuum-14B-v1", - "id": "sometimesanotion/Qwentinuum-14B-v1", - "developer": "sometimesanotion", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sometimesanotion_Qwentinuum-14B-v1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5032 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6573 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3603 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3826 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4781 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.541 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sometimesanotion_qwentinuum-14b-v2.json b/data/models/sometimesanotion_qwentinuum-14b-v2.json deleted file mode 100644 index 76b8fd59d9f711e9cdfaa359215c23ac35659027..0000000000000000000000000000000000000000 --- a/data/models/sometimesanotion_qwentinuum-14b-v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwentinuum-14B-v2", - "id": "sometimesanotion/Qwentinuum-14B-v2", - "developer": "sometimesanotion", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sometimesanotion_Qwentinuum-14B-v2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5378 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6555 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3754 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3884 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4714 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5409 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sometimesanotion_qwentinuum-14b-v3.json b/data/models/sometimesanotion_qwentinuum-14b-v3.json deleted file mode 100644 index 2bdf1c30f5edaa141a9b757c806366871cbe4079..0000000000000000000000000000000000000000 --- a/data/models/sometimesanotion_qwentinuum-14b-v3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwentinuum-14B-v3", - "id": "sometimesanotion/Qwentinuum-14B-v3", - "developer": "sometimesanotion", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sometimesanotion_Qwentinuum-14B-v3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6158 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6539 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3535 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3876 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.486 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5413 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sometimesanotion_qwentinuum-14b-v5.json b/data/models/sometimesanotion_qwentinuum-14b-v5.json deleted file mode 100644 index 4bb1bba6a53c904e140b2b9a496c71e8587ec2fb..0000000000000000000000000000000000000000 --- a/data/models/sometimesanotion_qwentinuum-14b-v5.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwentinuum-14B-v5", - "id": "sometimesanotion/Qwentinuum-14B-v5", - "developer": "sometimesanotion", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sometimesanotion_Qwentinuum-14B-v5/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6286 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.655 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3444 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3876 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4874 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5418 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sometimesanotion_qwentinuum-14b-v6-prose.json b/data/models/sometimesanotion_qwentinuum-14b-v6-prose.json deleted file mode 100644 index 074b4a601e90959f8e417a5c5d36f28a2aebbe74..0000000000000000000000000000000000000000 --- a/data/models/sometimesanotion_qwentinuum-14b-v6-prose.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwentinuum-14B-v6-Prose", - "id": "sometimesanotion/Qwentinuum-14B-v6-Prose", - "developer": "sometimesanotion", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sometimesanotion_Qwentinuum-14B-v6-Prose/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5643 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6545 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3701 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3884 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4913 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5392 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sometimesanotion_qwentinuum-14b-v6.json b/data/models/sometimesanotion_qwentinuum-14b-v6.json deleted file mode 100644 index 0c627bb6e47afb287e529f64315ecee841211efb..0000000000000000000000000000000000000000 --- a/data/models/sometimesanotion_qwentinuum-14b-v6.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwentinuum-14B-v6", - "id": "sometimesanotion/Qwentinuum-14B-v6", - "developer": "sometimesanotion", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sometimesanotion_Qwentinuum-14B-v6/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6304 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6545 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3603 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3867 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.49 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.54 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sometimesanotion_qwentinuum-14b-v7.json b/data/models/sometimesanotion_qwentinuum-14b-v7.json deleted file mode 100644 index 4f36d100ef8724eb34ea6320094378bedb37dc80..0000000000000000000000000000000000000000 --- a/data/models/sometimesanotion_qwentinuum-14b-v7.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwentinuum-14B-v7", - "id": "sometimesanotion/Qwentinuum-14B-v7", - "developer": "sometimesanotion", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sometimesanotion_Qwentinuum-14B-v7/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6109 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6551 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3573 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3909 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.482 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.541 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sometimesanotion_qwentinuum-14b-v8.json b/data/models/sometimesanotion_qwentinuum-14b-v8.json deleted file mode 100644 index ec3dbd5fd6d7f17c3544b827c5ca2e41995485c3..0000000000000000000000000000000000000000 --- a/data/models/sometimesanotion_qwentinuum-14b-v8.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwentinuum-14B-v8", - "id": "sometimesanotion/Qwentinuum-14B-v8", - "developer": "sometimesanotion", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sometimesanotion_Qwentinuum-14B-v8/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5412 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6534 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3912 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3834 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4873 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5412 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sometimesanotion_qwentinuum-14b-v9.json b/data/models/sometimesanotion_qwentinuum-14b-v9.json deleted file mode 100644 index 3fd2899185aa332f4c416930e9e9e746e96502bc..0000000000000000000000000000000000000000 --- a/data/models/sometimesanotion_qwentinuum-14b-v9.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwentinuum-14B-v9", - "id": "sometimesanotion/Qwentinuum-14B-v9", - "developer": "sometimesanotion", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sometimesanotion_Qwentinuum-14B-v9/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5107 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.658 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3482 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3859 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4781 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5421 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sometimesanotion_qwenvergence-14b-qv256.json b/data/models/sometimesanotion_qwenvergence-14b-qv256.json deleted file mode 100644 index 6941aaf1bd0cf32b0dfc4b9c0a2ab99a75db7b6f..0000000000000000000000000000000000000000 --- a/data/models/sometimesanotion_qwenvergence-14b-qv256.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwenvergence-14B-qv256", - "id": "sometimesanotion/Qwenvergence-14B-qv256", - "developer": "sometimesanotion", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sometimesanotion_Qwenvergence-14B-qv256/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7006 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6312 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3897 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3784 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4926 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5178 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sometimesanotion_qwenvergence-14b-v0.6-004-model_stock.json b/data/models/sometimesanotion_qwenvergence-14b-v0.6-004-model_stock.json deleted file mode 100644 index 1143a155f2f7b8c2158fe78bc94dfd7cfab02e25..0000000000000000000000000000000000000000 --- a/data/models/sometimesanotion_qwenvergence-14b-v0.6-004-model_stock.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwenvergence-14B-v0.6-004-model_stock", - "id": "sometimesanotion/Qwenvergence-14B-v0.6-004-model_stock", - "developer": "sometimesanotion", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sometimesanotion_Qwenvergence-14B-v0.6-004-model_stock/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.686 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6249 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4094 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3834 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5033 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5193 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sometimesanotion_qwenvergence-14b-v10.json b/data/models/sometimesanotion_qwenvergence-14b-v10.json deleted file mode 100644 index b378de7d516be0ddd2eaf629232156c261dbf1b7..0000000000000000000000000000000000000000 --- a/data/models/sometimesanotion_qwenvergence-14b-v10.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwenvergence-14B-v10", - "id": "sometimesanotion/Qwenvergence-14B-v10", - "developer": "sometimesanotion", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sometimesanotion_Qwenvergence-14B-v10/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6757 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6316 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4789 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3792 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4991 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5239 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sometimesanotion_qwenvergence-14b-v11.json b/data/models/sometimesanotion_qwenvergence-14b-v11.json deleted file mode 100644 index d288cf3cdb6b025bb971879672590025495cf396..0000000000000000000000000000000000000000 --- a/data/models/sometimesanotion_qwenvergence-14b-v11.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwenvergence-14B-v11", - "id": "sometimesanotion/Qwenvergence-14B-v11", - "developer": "sometimesanotion", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sometimesanotion_Qwenvergence-14B-v11/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7192 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6368 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4645 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3725 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4754 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5327 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sometimesanotion_qwenvergence-14b-v12-prose-ds.json b/data/models/sometimesanotion_qwenvergence-14b-v12-prose-ds.json deleted file mode 100644 index 7f7e6c1773960143868cc0c2349e86fbe09d3eb4..0000000000000000000000000000000000000000 --- a/data/models/sometimesanotion_qwenvergence-14b-v12-prose-ds.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwenvergence-14B-v12-Prose-DS", - "id": "sometimesanotion/Qwenvergence-14B-v12-Prose-DS", - "developer": "sometimesanotion", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sometimesanotion_Qwenvergence-14B-v12-Prose-DS/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6173 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6507 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4305 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3943 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5151 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5369 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sometimesanotion_qwenvergence-14b-v12-prose.json b/data/models/sometimesanotion_qwenvergence-14b-v12-prose.json deleted file mode 100644 index 9de053de2620dc78bfea8fc1d4b95512e50aac97..0000000000000000000000000000000000000000 --- a/data/models/sometimesanotion_qwenvergence-14b-v12-prose.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwenvergence-14B-v12-Prose", - "id": "sometimesanotion/Qwenvergence-14B-v12-Prose", - "developer": "sometimesanotion", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sometimesanotion_Qwenvergence-14B-v12-Prose/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5412 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6504 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3535 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3867 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4991 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5381 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sometimesanotion_qwenvergence-14b-v13-prose-ds.json b/data/models/sometimesanotion_qwenvergence-14b-v13-prose-ds.json deleted file mode 100644 index c59a4a4347fd867996140c6a6846a8ccbb944fa4..0000000000000000000000000000000000000000 --- a/data/models/sometimesanotion_qwenvergence-14b-v13-prose-ds.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwenvergence-14B-v13-Prose-DS", - "id": "sometimesanotion/Qwenvergence-14B-v13-Prose-DS", - "developer": "sometimesanotion", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sometimesanotion_Qwenvergence-14B-v13-Prose-DS/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7178 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6405 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.386 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3834 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4927 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5349 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sometimesanotion_qwenvergence-14b-v15-prose-ms.json b/data/models/sometimesanotion_qwenvergence-14b-v15-prose-ms.json deleted file mode 100644 index 0f9a297de73613eb7d8ecfa00363974183d284d0..0000000000000000000000000000000000000000 --- a/data/models/sometimesanotion_qwenvergence-14b-v15-prose-ms.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwenvergence-14B-v15-Prose-MS", - "id": "sometimesanotion/Qwenvergence-14B-v15-Prose-MS", - "developer": "sometimesanotion", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sometimesanotion_Qwenvergence-14B-v15-Prose-MS/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5032 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.655 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3633 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3951 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4913 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5393 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sometimesanotion_qwenvergence-14b-v2-prose.json b/data/models/sometimesanotion_qwenvergence-14b-v2-prose.json deleted file mode 100644 index 78c3b3180bf512b2ef5e175acd7e5345b04d9f9f..0000000000000000000000000000000000000000 --- a/data/models/sometimesanotion_qwenvergence-14b-v2-prose.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwenvergence-14B-v2-Prose", - "id": "sometimesanotion/Qwenvergence-14B-v2-Prose", - "developer": "sometimesanotion", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sometimesanotion_Qwenvergence-14B-v2-Prose/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4705 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6519 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3557 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3935 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4926 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5372 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sometimesanotion_qwenvergence-14b-v3-prose.json b/data/models/sometimesanotion_qwenvergence-14b-v3-prose.json deleted file mode 100644 index 5695e6911495beeef5e6a2b0a2d761c91a4c91ea..0000000000000000000000000000000000000000 --- a/data/models/sometimesanotion_qwenvergence-14b-v3-prose.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwenvergence-14B-v3-Prose", - "id": "sometimesanotion/Qwenvergence-14B-v3-Prose", - "developer": "sometimesanotion", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sometimesanotion_Qwenvergence-14B-v3-Prose/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4918 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6513 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3648 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3951 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4939 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.537 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sometimesanotion_qwenvergence-14b-v3-reason.json b/data/models/sometimesanotion_qwenvergence-14b-v3-reason.json deleted file mode 100644 index a439bc62eb51d7fa9519bb2716501909fa712711..0000000000000000000000000000000000000000 --- a/data/models/sometimesanotion_qwenvergence-14b-v3-reason.json +++ /dev/null @@ -1,275 +0,0 @@ -{ - "model_info": { - "name": "Qwenvergence-14B-v3-Reason", - "id": "sometimesanotion/Qwenvergence-14B-v3-Reason", - "developer": "sometimesanotion", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sometimesanotion_Qwenvergence-14B-v3-Reason/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5367 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6561 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.358 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3867 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.474 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5395 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "hfopenllm_v2/sometimesanotion_Qwenvergence-14B-v3-Reason/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5278 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6557 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3119 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3842 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4754 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5396 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sometimesanotion_qwenvergence-14b-v3.json b/data/models/sometimesanotion_qwenvergence-14b-v3.json deleted file mode 100644 index bd125107da0e711d6e707d8e4d88316cfd066974..0000000000000000000000000000000000000000 --- a/data/models/sometimesanotion_qwenvergence-14b-v3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwenvergence-14B-v3", - "id": "sometimesanotion/Qwenvergence-14B-v3", - "developer": "sometimesanotion", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sometimesanotion_Qwenvergence-14B-v3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5044 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6548 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3693 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3842 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4886 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5386 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sometimesanotion_qwenvergence-14b-v6-prose-model_stock.json b/data/models/sometimesanotion_qwenvergence-14b-v6-prose-model_stock.json deleted file mode 100644 index b7d23927674202ed3fc296d4e219f531dc913380..0000000000000000000000000000000000000000 --- a/data/models/sometimesanotion_qwenvergence-14b-v6-prose-model_stock.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwenvergence-14B-v6-Prose-model_stock", - "id": "sometimesanotion/Qwenvergence-14B-v6-Prose-model_stock", - "developer": "sometimesanotion", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sometimesanotion_Qwenvergence-14B-v6-Prose-model_stock/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4811 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.653 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3603 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3935 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4899 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5387 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sometimesanotion_qwenvergence-14b-v6-prose.json b/data/models/sometimesanotion_qwenvergence-14b-v6-prose.json deleted file mode 100644 index 84759cf3db238303b7f1f8ed5dd20aff3e7f3463..0000000000000000000000000000000000000000 --- a/data/models/sometimesanotion_qwenvergence-14b-v6-prose.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwenvergence-14B-v6-Prose", - "id": "sometimesanotion/Qwenvergence-14B-v6-Prose", - "developer": "sometimesanotion", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sometimesanotion_Qwenvergence-14B-v6-Prose/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.599 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6544 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3565 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3884 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4887 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5371 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sometimesanotion_qwenvergence-14b-v8.json b/data/models/sometimesanotion_qwenvergence-14b-v8.json deleted file mode 100644 index fbc25cb1439b193cbb5b141c3bad0c0cd77ee9eb..0000000000000000000000000000000000000000 --- a/data/models/sometimesanotion_qwenvergence-14b-v8.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwenvergence-14B-v8", - "id": "sometimesanotion/Qwenvergence-14B-v8", - "developer": "sometimesanotion", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sometimesanotion_Qwenvergence-14B-v8/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5913 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6522 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4048 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3809 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4768 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5435 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sometimesanotion_qwenvergence-14b-v9.json b/data/models/sometimesanotion_qwenvergence-14b-v9.json deleted file mode 100644 index 70f437d4b428c274a0b3039673934ff77d4de932..0000000000000000000000000000000000000000 --- a/data/models/sometimesanotion_qwenvergence-14b-v9.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwenvergence-14B-v9", - "id": "sometimesanotion/Qwenvergence-14B-v9", - "developer": "sometimesanotion", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sometimesanotion_Qwenvergence-14B-v9/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6598 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6166 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4139 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3683 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5141 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5111 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sonthenguyen_ft-unsloth-zephyr-sft-bnb-4bit-20241014-161415.json b/data/models/sonthenguyen_ft-unsloth-zephyr-sft-bnb-4bit-20241014-161415.json deleted file mode 100644 index 2ad6dd0bb2688e2b2816e6695143f5175dc0f398..0000000000000000000000000000000000000000 --- a/data/models/sonthenguyen_ft-unsloth-zephyr-sft-bnb-4bit-20241014-161415.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ft-unsloth-zephyr-sft-bnb-4bit-20241014-161415", - "id": "sonthenguyen/ft-unsloth-zephyr-sft-bnb-4bit-20241014-161415", - "developer": "sonthenguyen", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "?", - "params_billions": "7.723" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sonthenguyen_ft-unsloth-zephyr-sft-bnb-4bit-20241014-161415/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2893 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3804 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0113 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2466 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3861 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1401 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sonthenguyen_ft-unsloth-zephyr-sft-bnb-4bit-20241014-164205.json b/data/models/sonthenguyen_ft-unsloth-zephyr-sft-bnb-4bit-20241014-164205.json deleted file mode 100644 index 8091dfd3d9e52250cdd4290857f1478d3cc36970..0000000000000000000000000000000000000000 --- a/data/models/sonthenguyen_ft-unsloth-zephyr-sft-bnb-4bit-20241014-164205.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ft-unsloth-zephyr-sft-bnb-4bit-20241014-164205", - "id": "sonthenguyen/ft-unsloth-zephyr-sft-bnb-4bit-20241014-164205", - "developer": "sonthenguyen", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "?", - "params_billions": "7.723" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sonthenguyen_ft-unsloth-zephyr-sft-bnb-4bit-20241014-164205/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3199 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3959 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0083 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.276 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4272 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2124 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sonthenguyen_ft-unsloth-zephyr-sft-bnb-4bit-20241014-170522.json b/data/models/sonthenguyen_ft-unsloth-zephyr-sft-bnb-4bit-20241014-170522.json deleted file mode 100644 index c0732e55eec9183d36889ff9d23f42d143981dc2..0000000000000000000000000000000000000000 --- a/data/models/sonthenguyen_ft-unsloth-zephyr-sft-bnb-4bit-20241014-170522.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ft-unsloth-zephyr-sft-bnb-4bit-20241014-170522", - "id": "sonthenguyen/ft-unsloth-zephyr-sft-bnb-4bit-20241014-170522", - "developer": "sonthenguyen", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "?", - "params_billions": "7.723" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sonthenguyen_ft-unsloth-zephyr-sft-bnb-4bit-20241014-170522/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3764 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3828 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0091 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2651 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4404 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2055 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sonthenguyen_zephyr-sft-bnb-4bit-dpo-mtbc-213steps.json b/data/models/sonthenguyen_zephyr-sft-bnb-4bit-dpo-mtbc-213steps.json deleted file mode 100644 index 464c747f66020c42edf1786efc09b6d03eb01df7..0000000000000000000000000000000000000000 --- a/data/models/sonthenguyen_zephyr-sft-bnb-4bit-dpo-mtbc-213steps.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "zephyr-sft-bnb-4bit-DPO-mtbc-213steps", - "id": "sonthenguyen/zephyr-sft-bnb-4bit-DPO-mtbc-213steps", - "developer": "sonthenguyen", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sonthenguyen_zephyr-sft-bnb-4bit-DPO-mtbc-213steps/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4275 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4197 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0257 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2617 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4086 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2709 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sonthenguyen_zephyr-sft-bnb-4bit-dpo-mtbo-180steps.json b/data/models/sonthenguyen_zephyr-sft-bnb-4bit-dpo-mtbo-180steps.json deleted file mode 100644 index 1fb40719aa5057d665d0cb212cf2ae314bbc30ba..0000000000000000000000000000000000000000 --- a/data/models/sonthenguyen_zephyr-sft-bnb-4bit-dpo-mtbo-180steps.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "zephyr-sft-bnb-4bit-DPO-mtbo-180steps", - "id": "sonthenguyen/zephyr-sft-bnb-4bit-DPO-mtbo-180steps", - "developer": "sonthenguyen", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sonthenguyen_zephyr-sft-bnb-4bit-DPO-mtbo-180steps/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4087 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4323 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0234 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.276 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3885 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2748 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sonthenguyen_zephyr-sft-bnb-4bit-dpo-mtbr-180steps.json b/data/models/sonthenguyen_zephyr-sft-bnb-4bit-dpo-mtbr-180steps.json deleted file mode 100644 index 70cfe0742597e1c0f704acdffa0c5d74f34c9b06..0000000000000000000000000000000000000000 --- a/data/models/sonthenguyen_zephyr-sft-bnb-4bit-dpo-mtbr-180steps.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "zephyr-sft-bnb-4bit-DPO-mtbr-180steps", - "id": "sonthenguyen/zephyr-sft-bnb-4bit-DPO-mtbr-180steps", - "developer": "sonthenguyen", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sonthenguyen_zephyr-sft-bnb-4bit-DPO-mtbr-180steps/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4032 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4305 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0249 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2802 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4258 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2711 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sophosympatheia_midnight-miqu-70b-v1.5.json b/data/models/sophosympatheia_midnight-miqu-70b-v1.5.json deleted file mode 100644 index 91969c9c769ea14ead1ccef4c09d066063c2e65f..0000000000000000000000000000000000000000 --- a/data/models/sophosympatheia_midnight-miqu-70b-v1.5.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Midnight-Miqu-70B-v1.5", - "id": "sophosympatheia/Midnight-Miqu-70B-v1.5", - "developer": "sophosympatheia", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "68.977" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sophosympatheia_Midnight-Miqu-70B-v1.5/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6118 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5606 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0702 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2961 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4244 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3825 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sorawiz_gemma-9b-base.json b/data/models/sorawiz_gemma-9b-base.json deleted file mode 100644 index c636b5684a242b82889b8419b58dec5a747ef1fa..0000000000000000000000000000000000000000 --- a/data/models/sorawiz_gemma-9b-base.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Gemma-9B-Base", - "id": "Sorawiz/Gemma-9B-Base", - "developer": "Sorawiz", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Sorawiz_Gemma-9B-Base/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1667 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.593 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0982 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3398 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4045 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4235 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sorawiz_gemma-creative-9b-base.json b/data/models/sorawiz_gemma-creative-9b-base.json deleted file mode 100644 index 994606e211724955277350719ffd544faf0601b9..0000000000000000000000000000000000000000 --- a/data/models/sorawiz_gemma-creative-9b-base.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Gemma-Creative-9B-Base", - "id": "Sorawiz/Gemma-Creative-9B-Base", - "developer": "Sorawiz", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Sorawiz_Gemma-Creative-9B-Base/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1515 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5459 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0778 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3297 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4019 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4008 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sourjayon_deepseek-r1-8b-sify.json b/data/models/sourjayon_deepseek-r1-8b-sify.json deleted file mode 100644 index 8f20c3b63415e4c5cce325e85a910e8f2e7175ac..0000000000000000000000000000000000000000 --- a/data/models/sourjayon_deepseek-r1-8b-sify.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "DeepSeek-R1-8b-Sify", - "id": "Sourjayon/DeepSeek-R1-8b-Sify", - "developer": "Sourjayon", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Sourjayon_DeepSeek-R1-8b-Sify/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3679 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3379 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2447 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2525 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3303 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1981 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sourjayon_deepseek-r1-forumnxt.json b/data/models/sourjayon_deepseek-r1-forumnxt.json deleted file mode 100644 index 657326bec070db6df2cbe7d292b3dc6b14fd2009..0000000000000000000000000000000000000000 --- a/data/models/sourjayon_deepseek-r1-forumnxt.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "DeepSeek-R1-ForumNXT", - "id": "Sourjayon/DeepSeek-R1-ForumNXT", - "developer": "Sourjayon", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.777" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Sourjayon_DeepSeek-R1-ForumNXT/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2603 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.331 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2576 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2743 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3392 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1648 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/spaceyl_ece_poirot.json b/data/models/spaceyl_ece_poirot.json deleted file mode 100644 index 3d5a6a5aa5f3e75564484b3bb95d48541e2b258b..0000000000000000000000000000000000000000 --- a/data/models/spaceyl_ece_poirot.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ECE_Poirot", - "id": "SpaceYL/ECE_Poirot", - "developer": "SpaceYL", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.544" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/SpaceYL_ECE_Poirot/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3107 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4262 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0914 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2978 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4026 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2883 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/speakleash-ack-cyfronet-agh_bielik-11b-v2-3-instruct-prompt.json b/data/models/speakleash-ack-cyfronet-agh_bielik-11b-v2-3-instruct-prompt.json deleted file mode 100644 index 61cb9550aa489017e697f3e48a103f5f049ae85d..0000000000000000000000000000000000000000 --- a/data/models/speakleash-ack-cyfronet-agh_bielik-11b-v2-3-instruct-prompt.json +++ /dev/null @@ -1,905 +0,0 @@ -{ - "model_info": { - "name": "Bielik-11B-v2.3-Instruct (Prompt)", - "id": "speakleash-ack-cyfronet-agh/bielik-11b-v2-3-instruct-prompt", - "developer": "speakleash-ack-cyfronet-agh", - "additional_details": { - "raw_model_name": "Bielik-11B-v2.3-Instruct (Prompt)", - "organization": "SpeakLeash & ACK Cyfronet AGH", - "license": "Apache 2.0", - "mode": "Prompt", - "model_link": "https://huggingface.co/speakleash/Bielik-11B-v2.3-Instruct" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/speakleash-ack-cyfronet-agh/bielik-11b-v2-3-instruct-prompt/1775236112.418079", - "retrieved_timestamp": "1775236112.418079", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 99.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 21.9 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 22.44 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 23.75 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 61.76 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 72.8 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 81.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 73.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 92.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 85.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 75.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 67.8 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 75.58 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 66.19 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 62.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 58.33 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 2.62 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 4.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 0.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 3.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 2.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 1.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 1.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 2.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 11.4 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 7.1 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 4.52 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 22.58 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 93.75 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 36.01 - } - }, - { - "evaluation_result_id": "bfcl.format_sensitivity.max_delta::max_delta", - "evaluation_name": "bfcl.format_sensitivity.max_delta", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.format_sensitivity.max_delta", - "metric_name": "Format sensitivity max delta", - "metric_kind": "difference", - "metric_unit": "percentage_points", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Format Sensitivity Max Delta" - } - }, - "score_details": { - "score": 35.0 - } - }, - { - "evaluation_result_id": "bfcl.format_sensitivity.stddev::stddev", - "evaluation_name": "bfcl.format_sensitivity.stddev", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.format_sensitivity.stddev", - "metric_name": "Format sensitivity standard deviation", - "metric_kind": "difference", - "metric_unit": "percentage_points", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Format Sensitivity Standard Deviation" - } - }, - "score_details": { - "score": 9.74 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/speakleash_bielik-11b-v2.0-instruct.json b/data/models/speakleash_bielik-11b-v2.0-instruct.json deleted file mode 100644 index 4deb9cb2698bc463162a5b611f1c78fb970e4751..0000000000000000000000000000000000000000 --- a/data/models/speakleash_bielik-11b-v2.0-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Bielik-11B-v2.0-Instruct", - "id": "speakleash/Bielik-11B-v2.0-Instruct", - "developer": "speakleash", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "11.169" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/speakleash_Bielik-11B-v2.0-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5252 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5362 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1186 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3171 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4467 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3351 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/speakleash_bielik-11b-v2.1-instruct.json b/data/models/speakleash_bielik-11b-v2.1-instruct.json deleted file mode 100644 index 797041e0e1d1133df6cb6f1eb15c494cf12f03de..0000000000000000000000000000000000000000 --- a/data/models/speakleash_bielik-11b-v2.1-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Bielik-11B-v2.1-Instruct", - "id": "speakleash/Bielik-11B-v2.1-Instruct", - "developer": "speakleash", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "11.169" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/speakleash_Bielik-11B-v2.1-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.509 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.553 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2666 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3372 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4185 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3447 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/speakleash_bielik-11b-v2.2-instruct.json b/data/models/speakleash_bielik-11b-v2.2-instruct.json deleted file mode 100644 index 29525817b444d4bc71372e223c9227e0be6b4178..0000000000000000000000000000000000000000 --- a/data/models/speakleash_bielik-11b-v2.2-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Bielik-11B-v2.2-Instruct", - "id": "speakleash/Bielik-11B-v2.2-Instruct", - "developer": "speakleash", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "11.169" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/speakleash_Bielik-11B-v2.2-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5552 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5597 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2681 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3314 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4171 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3487 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/speakleash_bielik-11b-v2.3-instruct.json b/data/models/speakleash_bielik-11b-v2.3-instruct.json deleted file mode 100644 index e9355a0076cb0359e75deabc0c8f1532e9e11fc0..0000000000000000000000000000000000000000 --- a/data/models/speakleash_bielik-11b-v2.3-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Bielik-11B-v2.3-Instruct", - "id": "speakleash/Bielik-11B-v2.3-Instruct", - "developer": "speakleash", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "11.169" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/speakleash_Bielik-11B-v2.3-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5583 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5663 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2085 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3406 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4518 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3444 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/speakleash_bielik-11b-v2.json b/data/models/speakleash_bielik-11b-v2.json deleted file mode 100644 index 5993ba3e76e84fe363f598efa17cb76b40e136cb..0000000000000000000000000000000000000000 --- a/data/models/speakleash_bielik-11b-v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Bielik-11B-v2", - "id": "speakleash/Bielik-11B-v2", - "developer": "speakleash", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "11.169" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/speakleash_Bielik-11B-v2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2381 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4931 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0785 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2886 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3924 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3137 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/spestly_athena-1-3b.json b/data/models/spestly_athena-1-3b.json deleted file mode 100644 index 2886431e3e9aed3552dcd0436d8791de5de46d3c..0000000000000000000000000000000000000000 --- a/data/models/spestly_athena-1-3b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Athena-1-3B", - "id": "Spestly/Athena-1-3B", - "developer": "Spestly", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.086" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Spestly_Athena-1-3B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5569 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4702 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2379 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2936 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4362 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3519 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/spestly_atlas-pro-1.5b-preview.json b/data/models/spestly_atlas-pro-1.5b-preview.json deleted file mode 100644 index 69048f5a158ce4cbbe402c8ac411522865f51861..0000000000000000000000000000000000000000 --- a/data/models/spestly_atlas-pro-1.5b-preview.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Atlas-Pro-1.5B-Preview", - "id": "Spestly/Atlas-Pro-1.5B-Preview", - "developer": "Spestly", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.777" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Spestly_Atlas-Pro-1.5B-Preview/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.243 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3499 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3195 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.297 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3354 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1925 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/spestly_atlas-pro-7b-preview.json b/data/models/spestly_atlas-pro-7b-preview.json deleted file mode 100644 index a835aa599e5dee0ff83b27f0ca92ae84c57ad601..0000000000000000000000000000000000000000 --- a/data/models/spestly_atlas-pro-7b-preview.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Atlas-Pro-7B-Preview", - "id": "Spestly/Atlas-Pro-7B-Preview", - "developer": "Spestly", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Spestly_Atlas-Pro-7B-Preview/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3154 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4668 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5083 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3372 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3911 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.297 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/spmurrayzzz_mistral-syndicate-7b.json b/data/models/spmurrayzzz_mistral-syndicate-7b.json deleted file mode 100644 index 66791e03e8ce51621b8ab49f64605d9a6d418516..0000000000000000000000000000000000000000 --- a/data/models/spmurrayzzz_mistral-syndicate-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mistral-Syndicate-7B", - "id": "spmurrayzzz/Mistral-Syndicate-7B", - "developer": "spmurrayzzz", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/spmurrayzzz_Mistral-Syndicate-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2496 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4245 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.034 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.276 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4386 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2631 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/spow12_chatwaifu_12b_v2.0.json b/data/models/spow12_chatwaifu_12b_v2.0.json deleted file mode 100644 index 0c3c8a3a9ce56f560c73de1be49ecc76efa24bf6..0000000000000000000000000000000000000000 --- a/data/models/spow12_chatwaifu_12b_v2.0.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ChatWaifu_12B_v2.0", - "id": "spow12/ChatWaifu_12B_v2.0", - "developer": "spow12", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/spow12_ChatWaifu_12B_v2.0/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4768 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5208 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.071 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2768 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4432 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3388 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/spow12_chatwaifu_22b_v2.0_preview.json b/data/models/spow12_chatwaifu_22b_v2.0_preview.json deleted file mode 100644 index 85b03880c0300f07c9c3c9b671132ce0c7f32a1e..0000000000000000000000000000000000000000 --- a/data/models/spow12_chatwaifu_22b_v2.0_preview.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ChatWaifu_22B_v2.0_preview", - "id": "spow12/ChatWaifu_22B_v2.0_preview", - "developer": "spow12", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "22.247" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/spow12_ChatWaifu_22B_v2.0_preview/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6745 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.617 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1888 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3154 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3685 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3988 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/spow12_chatwaifu_v1.4.json b/data/models/spow12_chatwaifu_v1.4.json deleted file mode 100644 index cad4083decaee30d7f5c3e69ceec39dbe0b04d66..0000000000000000000000000000000000000000 --- a/data/models/spow12_chatwaifu_v1.4.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ChatWaifu_v1.4", - "id": "spow12/ChatWaifu_v1.4", - "developer": "spow12", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/spow12_ChatWaifu_v1.4/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5691 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5176 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1057 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.307 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4743 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3475 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/spow12_chatwaifu_v2.0_22b.json b/data/models/spow12_chatwaifu_v2.0_22b.json deleted file mode 100644 index ca25894b34b4dcd9cda648ad0e73ccd6dac87171..0000000000000000000000000000000000000000 --- a/data/models/spow12_chatwaifu_v2.0_22b.json +++ /dev/null @@ -1,275 +0,0 @@ -{ - "model_info": { - "name": "ChatWaifu_v2.0_22B", - "id": "spow12/ChatWaifu_v2.0_22B", - "developer": "spow12", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "22.247" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/spow12_ChatWaifu_v2.0_22B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6517 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5908 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2032 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3238 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3842 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3812 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "hfopenllm_v2/spow12_ChatWaifu_v2.0_22B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6511 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5926 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1858 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3247 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3842 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3836 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ssmits_qwen2.5-95b-instruct.json b/data/models/ssmits_qwen2.5-95b-instruct.json deleted file mode 100644 index c6d20c65b9e6177dcd94378ba396209bdccab7e3..0000000000000000000000000000000000000000 --- a/data/models/ssmits_qwen2.5-95b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-95B-Instruct", - "id": "ssmits/Qwen2.5-95B-Instruct", - "developer": "ssmits", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "94.648" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ssmits_Qwen2.5-95B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8431 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7038 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5302 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3641 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4284 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5217 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/stabilityai_stable-code-instruct-3b.json b/data/models/stabilityai_stable-code-instruct-3b.json deleted file mode 100644 index 9a2356d1e97e357f92e5586335180b9ab468ff17..0000000000000000000000000000000000000000 --- a/data/models/stabilityai_stable-code-instruct-3b.json +++ /dev/null @@ -1,144 +0,0 @@ -{ - "model_info": { - "name": "stabilityai/stable-code-instruct-3b", - "id": "stabilityai/stable-code-instruct-3b", - "developer": "stabilityai", - "additional_details": { - "model_type": "DPO" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/stabilityai_stable-code-instruct-3b/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6216 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5782 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5855 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6554 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7528 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4506 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/stabilityai_stablebeluga2.json b/data/models/stabilityai_stablebeluga2.json deleted file mode 100644 index dec3295fb4dbb820d4dcaa667aa42c8bfdf5b922..0000000000000000000000000000000000000000 --- a/data/models/stabilityai_stablebeluga2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "StableBeluga2", - "id": "stabilityai/StableBeluga2", - "developer": "stabilityai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "68.977" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/stabilityai_StableBeluga2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3787 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5824 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0438 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3163 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.473 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3326 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/stabilityai_stablelm-2-12b-chat.json b/data/models/stabilityai_stablelm-2-12b-chat.json deleted file mode 100644 index 78ad9ee53aba6f9ad3a85930c46ec90a47eef8d4..0000000000000000000000000000000000000000 --- a/data/models/stabilityai_stablelm-2-12b-chat.json +++ /dev/null @@ -1,277 +0,0 @@ -{ - "model_info": { - "name": "stabilityai/stablelm-2-12b-chat", - "id": "stabilityai/stablelm-2-12b-chat", - "developer": "stabilityai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "StableLmForCausalLM", - "params_billions": "12.143" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/stabilityai_stablelm-2-12b-chat/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4082 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4672 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0536 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2668 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3914 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2734 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "reward-bench/stabilityai_stablelm-2-12b-chat/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7642 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9665 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5548 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7811 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8945 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4839 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/stabilityai_stablelm-2-12b.json b/data/models/stabilityai_stablelm-2-12b.json deleted file mode 100644 index 69b6643be251c532bfcb095e0c72c2715e66c03c..0000000000000000000000000000000000000000 --- a/data/models/stabilityai_stablelm-2-12b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "stablelm-2-12b", - "id": "stabilityai/stablelm-2-12b", - "developer": "stabilityai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "StableLmForCausalLM", - "params_billions": "12.143" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/stabilityai_stablelm-2-12b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1569 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4509 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0431 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2785 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4479 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3072 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/stabilityai_stablelm-2-1_6b-chat.json b/data/models/stabilityai_stablelm-2-1_6b-chat.json deleted file mode 100644 index 903ee6ba75f904261088cd10d19036f3168840b6..0000000000000000000000000000000000000000 --- a/data/models/stabilityai_stablelm-2-1_6b-chat.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "stablelm-2-1_6b-chat", - "id": "stabilityai/stablelm-2-1_6b-chat", - "developer": "stabilityai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "StableLmForCausalLM", - "params_billions": "1.645" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/stabilityai_stablelm-2-1_6b-chat/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.306 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.339 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0249 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2475 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.358 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1622 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/stabilityai_stablelm-2-1_6b.json b/data/models/stabilityai_stablelm-2-1_6b.json deleted file mode 100644 index 0715eea990a39825bceb0e1edee1d0b6180f3019..0000000000000000000000000000000000000000 --- a/data/models/stabilityai_stablelm-2-1_6b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "stablelm-2-1_6b", - "id": "stabilityai/stablelm-2-1_6b", - "developer": "stabilityai", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "StableLmForCausalLM", - "params_billions": "1.645" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/stabilityai_stablelm-2-1_6b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1157 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3385 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0076 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2483 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3882 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1464 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/stabilityai_stablelm-2-zephyr-1_6b.json b/data/models/stabilityai_stablelm-2-zephyr-1_6b.json deleted file mode 100644 index 232262780da15206e76d7299db11743a568096cc..0000000000000000000000000000000000000000 --- a/data/models/stabilityai_stablelm-2-zephyr-1_6b.json +++ /dev/null @@ -1,277 +0,0 @@ -{ - "model_info": { - "name": "stabilityai/stablelm-2-zephyr-1_6b", - "id": "stabilityai/stablelm-2-zephyr-1_6b", - "developer": "stabilityai", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "StableLmForCausalLM", - "params_billions": "1.645" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/stabilityai_stablelm-2-zephyr-1_6b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3279 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3352 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0332 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2433 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3511 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1714 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "reward-bench/stabilityai_stablelm-2-zephyr-1_6b/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6574 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9665 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4671 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6027 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6784 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4868 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/stabilityai_stablelm-3b-4e1t.json b/data/models/stabilityai_stablelm-3b-4e1t.json deleted file mode 100644 index 0be9006fef02dfdab2e52a748aec5ed5f7d5db99..0000000000000000000000000000000000000000 --- a/data/models/stabilityai_stablelm-3b-4e1t.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "stablelm-3b-4e1t", - "id": "stabilityai/stablelm-3b-4e1t", - "developer": "stabilityai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "StableLmForCausalLM", - "params_billions": "2.795" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/stabilityai_stablelm-3b-4e1t/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2203 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3504 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0106 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2374 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3778 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1669 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/stabilityai_stablelm-zephyr-3b.json b/data/models/stabilityai_stablelm-zephyr-3b.json deleted file mode 100644 index 30073e22c9197c38e126213a34d5062aa27f0ee9..0000000000000000000000000000000000000000 --- a/data/models/stabilityai_stablelm-zephyr-3b.json +++ /dev/null @@ -1,277 +0,0 @@ -{ - "model_info": { - "name": "stabilityai/stablelm-zephyr-3b", - "id": "stabilityai/stablelm-zephyr-3b", - "developer": "stabilityai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "StableLmForCausalLM", - "params_billions": "2.795" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/stabilityai_stablelm-zephyr-3b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3683 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3866 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0431 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2391 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4183 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1768 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "reward-bench/stabilityai_stablelm-zephyr-3b/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7146 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8631 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6009 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7405 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7573 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5075 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/stanford_alpaca-7b.json b/data/models/stanford_alpaca-7b.json deleted file mode 100644 index 23bcc33d6916173a8e31671e713b68274c304117..0000000000000000000000000000000000000000 --- a/data/models/stanford_alpaca-7b.json +++ /dev/null @@ -1,674 +0,0 @@ -{ - "model_info": { - "name": "Alpaca 7B", - "id": "stanford/Alpaca-7B", - "developer": "stanford", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_classic/stanford_Alpaca-7B/1774096308.339228", - "retrieved_timestamp": "1774096308.339228", - "source_metadata": { - "source_name": "helm_classic", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_classic", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_classic", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperform on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.381, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Calibration": "{\"description\": \"\", \"tab\": \"Calibration\", \"score\": \"0.3335337650323774\"}", - "Mean win rate - Robustness": "{\"description\": \"\", \"tab\": \"Robustness\", \"score\": \"0.37923076923076926\"}", - "Mean win rate - Fairness": "{\"description\": \"\", \"tab\": \"Fairness\", \"score\": \"0.3719114219114219\"}", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}", - "Mean win rate - Bias": "{\"description\": \"\", \"tab\": \"Bias\", \"score\": \"0.4865162612605669\"}", - "Mean win rate - Toxicity": "{\"description\": \"\", \"tab\": \"Toxicity\", \"score\": \"0.6546037296037296\"}", - "Mean win rate - Summarization metrics": "{\"description\": \"\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.385, - "details": { - "description": "min=0.263, mean=0.385, max=0.6, sum=1.923 (5)", - "tab": "Accuracy", - "MMLU - ECE (10-bin)": "{\"description\": \"min=0.151, mean=0.234, max=0.32, sum=1.171 (5)\", \"tab\": \"Calibration\", \"score\": \"0.23428857555005617\"}", - "MMLU - EM (Robustness)": "{\"description\": \"min=0.18, mean=0.324, max=0.52, sum=1.621 (5)\", \"tab\": \"Robustness\", \"score\": \"0.32410526315789473\"}", - "MMLU - EM (Fairness)": "{\"description\": \"min=0.219, mean=0.346, max=0.53, sum=1.729 (5)\", \"tab\": \"Fairness\", \"score\": \"0.34585964912280703\"}", - "MMLU - Denoised inference time (s)": "{\"description\": \"5 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=397.65, mean=522.547, max=684.675, sum=2612.735 (5)\", \"tab\": \"General information\", \"score\": \"522.5470877192982\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "MMLU - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "BoolQ", - "source_data": { - "dataset_name": "BoolQ", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on BoolQ", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.778, - "details": { - "description": "min=0.778, mean=0.778, max=0.778, sum=0.778 (1)", - "tab": "Accuracy", - "BoolQ - ECE (10-bin)": "{\"description\": \"min=0.343, mean=0.343, max=0.343, sum=0.343 (1)\", \"tab\": \"Calibration\", \"score\": \"0.3432802705941571\"}", - "BoolQ - EM (Robustness)": "{\"description\": \"min=0.643, mean=0.643, max=0.643, sum=0.643 (1)\", \"tab\": \"Robustness\", \"score\": \"0.643\"}", - "BoolQ - EM (Fairness)": "{\"description\": \"min=0.729, mean=0.729, max=0.729, sum=0.729 (1)\", \"tab\": \"Fairness\", \"score\": \"0.729\"}", - "BoolQ - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "BoolQ - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "BoolQ - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "BoolQ - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "BoolQ - # prompt tokens": "{\"description\": \"min=1439.447, mean=1439.447, max=1439.447, sum=1439.447 (1)\", \"tab\": \"General information\", \"score\": \"1439.447\"}", - "BoolQ - # output tokens": "{\"description\": \"min=4.883, mean=4.883, max=4.883, sum=4.883 (1)\", \"tab\": \"General information\", \"score\": \"4.883\"}", - "BoolQ - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "BoolQ - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (gender)": "{\"description\": \"min=0.5, mean=0.5, max=0.5, sum=0.5 (1)\", \"tab\": \"Bias\", \"score\": \"0.5\"}", - "BoolQ - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.396, - "details": { - "description": "min=0.396, mean=0.396, max=0.396, sum=0.396 (1)", - "tab": "Accuracy", - "NarrativeQA - ECE (10-bin)": "{\"description\": \"min=0.046, mean=0.046, max=0.046, sum=0.046 (1)\", \"tab\": \"Calibration\", \"score\": \"0.045878175333070315\"}", - "NarrativeQA - F1 (Robustness)": "{\"description\": \"min=0.246, mean=0.246, max=0.246, sum=0.246 (1)\", \"tab\": \"Robustness\", \"score\": \"0.24590950452109447\"}", - "NarrativeQA - F1 (Fairness)": "{\"description\": \"min=0.299, mean=0.299, max=0.299, sum=0.299 (1)\", \"tab\": \"Fairness\", \"score\": \"0.2987402817318288\"}", - "NarrativeQA - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=1.437, mean=1.437, max=1.437, sum=1.437 (1)\", \"tab\": \"General information\", \"score\": \"1.4366197183098592\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=1541.115, mean=1541.115, max=1541.115, sum=1541.115 (1)\", \"tab\": \"General information\", \"score\": \"1541.1154929577465\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=26.006, mean=26.006, max=26.006, sum=26.006 (1)\", \"tab\": \"General information\", \"score\": \"26.005633802816902\"}", - "NarrativeQA - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NarrativeQA - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=0.667 (1)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666667\"}", - "NarrativeQA - Stereotypes (gender)": "{\"description\": \"min=0.41, mean=0.41, max=0.41, sum=0.41 (1)\", \"tab\": \"Bias\", \"score\": \"0.41025641025641024\"}", - "NarrativeQA - Representation (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=0.667 (1)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666667\"}", - "NarrativeQA - Representation (gender)": "{\"description\": \"min=0.196, mean=0.196, max=0.196, sum=0.196 (1)\", \"tab\": \"Bias\", \"score\": \"0.19627507163323785\"}", - "NarrativeQA - Toxic fraction": "{\"description\": \"min=0.006, mean=0.006, max=0.006, sum=0.006 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.005633802816901409\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (open-book)", - "source_data": { - "dataset_name": "NaturalQuestions (open-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (open-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.592, - "details": { - "description": "min=0.592, mean=0.592, max=0.592, sum=0.592 (1)", - "tab": "Accuracy", - "NaturalQuestions (closed-book) - ECE (10-bin)": "{\"description\": \"min=0.134, mean=0.134, max=0.134, sum=0.134 (1)\", \"tab\": \"Calibration\", \"score\": \"0.13434354583448904\"}", - "NaturalQuestions (open-book) - ECE (10-bin)": "{\"description\": \"min=0.238, mean=0.238, max=0.238, sum=0.238 (1)\", \"tab\": \"Calibration\", \"score\": \"0.23769723451909555\"}", - "NaturalQuestions (closed-book) - F1 (Robustness)": "{\"description\": \"min=0.203, mean=0.203, max=0.203, sum=0.203 (1)\", \"tab\": \"Robustness\", \"score\": \"0.20255716308011695\"}", - "NaturalQuestions (open-book) - F1 (Robustness)": "{\"description\": \"min=0.491, mean=0.491, max=0.491, sum=0.491 (1)\", \"tab\": \"Robustness\", \"score\": \"0.4912677371744195\"}", - "NaturalQuestions (closed-book) - F1 (Fairness)": "{\"description\": \"min=0.21, mean=0.21, max=0.21, sum=0.21 (1)\", \"tab\": \"Fairness\", \"score\": \"0.20966482260352876\"}", - "NaturalQuestions (open-book) - F1 (Fairness)": "{\"description\": \"min=0.53, mean=0.53, max=0.53, sum=0.53 (1)\", \"tab\": \"Fairness\", \"score\": \"0.5302078541276196\"}", - "NaturalQuestions (closed-book) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NaturalQuestions (open-book) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=137.383, mean=137.383, max=137.383, sum=137.383 (1)\", \"tab\": \"General information\", \"score\": \"137.383\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=84.53, mean=84.53, max=84.53, sum=84.53 (1)\", \"tab\": \"General information\", \"score\": \"84.53\"}", - "NaturalQuestions (closed-book) - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=3.722, mean=3.722, max=3.722, sum=3.722 (1)\", \"tab\": \"General information\", \"score\": \"3.722\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.049, mean=0.049, max=0.049, sum=0.049 (1)\", \"tab\": \"General information\", \"score\": \"0.049\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1407.178, mean=1407.178, max=1407.178, sum=1407.178 (1)\", \"tab\": \"General information\", \"score\": \"1407.178\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=122.525, mean=122.525, max=122.525, sum=122.525 (1)\", \"tab\": \"General information\", \"score\": \"122.525\"}", - "NaturalQuestions (open-book) - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NaturalQuestions (closed-book) - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=0.667 (1)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666667\"}", - "NaturalQuestions (closed-book) - Stereotypes (gender)": "{\"description\": \"min=0.456, mean=0.456, max=0.456, sum=0.456 (1)\", \"tab\": \"Bias\", \"score\": \"0.45588235294117646\"}", - "NaturalQuestions (closed-book) - Representation (race)": "{\"description\": \"min=0.412, mean=0.412, max=0.412, sum=0.412 (1)\", \"tab\": \"Bias\", \"score\": \"0.4117647058823529\"}", - "NaturalQuestions (closed-book) - Representation (gender)": "{\"description\": \"min=0.054, mean=0.054, max=0.054, sum=0.054 (1)\", \"tab\": \"Bias\", \"score\": \"0.053571428571428575\"}", - "NaturalQuestions (open-book) - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=0.667 (1)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666667\"}", - "NaturalQuestions (open-book) - Stereotypes (gender)": "{\"description\": \"min=0.419, mean=0.419, max=0.419, sum=0.419 (1)\", \"tab\": \"Bias\", \"score\": \"0.4185185185185185\"}", - "NaturalQuestions (open-book) - Representation (race)": "{\"description\": \"min=0.454, mean=0.454, max=0.454, sum=0.454 (1)\", \"tab\": \"Bias\", \"score\": \"0.4540682414698163\"}", - "NaturalQuestions (open-book) - Representation (gender)": "{\"description\": \"min=0.315, mean=0.315, max=0.315, sum=0.315 (1)\", \"tab\": \"Bias\", \"score\": \"0.31481481481481477\"}", - "NaturalQuestions (closed-book) - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}", - "NaturalQuestions (open-book) - Toxic fraction": "{\"description\": \"min=0.001, mean=0.001, max=0.001, sum=0.001 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.001\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "QuAC", - "source_data": { - "dataset_name": "QuAC", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on QuAC", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.27, - "details": { - "description": "min=0.27, mean=0.27, max=0.27, sum=0.27 (1)", - "tab": "Accuracy", - "QuAC - ECE (10-bin)": "{\"description\": \"min=0.04, mean=0.04, max=0.04, sum=0.04 (1)\", \"tab\": \"Calibration\", \"score\": \"0.04026034301598206\"}", - "QuAC - F1 (Robustness)": "{\"description\": \"min=0.16, mean=0.16, max=0.16, sum=0.16 (1)\", \"tab\": \"Robustness\", \"score\": \"0.1604861950978603\"}", - "QuAC - F1 (Fairness)": "{\"description\": \"min=0.204, mean=0.204, max=0.204, sum=0.204 (1)\", \"tab\": \"Fairness\", \"score\": \"0.20395081036123316\"}", - "QuAC - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "QuAC - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "QuAC - # train": "{\"description\": \"min=0.507, mean=0.507, max=0.507, sum=0.507 (1)\", \"tab\": \"General information\", \"score\": \"0.507\"}", - "QuAC - truncated": "{\"description\": \"min=0.06, mean=0.06, max=0.06, sum=0.06 (1)\", \"tab\": \"General information\", \"score\": \"0.06\"}", - "QuAC - # prompt tokens": "{\"description\": \"min=1498.657, mean=1498.657, max=1498.657, sum=1498.657 (1)\", \"tab\": \"General information\", \"score\": \"1498.657\"}", - "QuAC - # output tokens": "{\"description\": \"min=77.323, mean=77.323, max=77.323, sum=77.323 (1)\", \"tab\": \"General information\", \"score\": \"77.323\"}", - "QuAC - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "QuAC - Stereotypes (race)": "{\"description\": \"min=0.636, mean=0.636, max=0.636, sum=0.636 (1)\", \"tab\": \"Bias\", \"score\": \"0.6363636363636365\"}", - "QuAC - Stereotypes (gender)": "{\"description\": \"min=0.435, mean=0.435, max=0.435, sum=0.435 (1)\", \"tab\": \"Bias\", \"score\": \"0.4349771051252814\"}", - "QuAC - Representation (race)": "{\"description\": \"min=0.236, mean=0.236, max=0.236, sum=0.236 (1)\", \"tab\": \"Bias\", \"score\": \"0.23589743589743586\"}", - "QuAC - Representation (gender)": "{\"description\": \"min=0.281, mean=0.281, max=0.281, sum=0.281 (1)\", \"tab\": \"Bias\", \"score\": \"0.2813953488372093\"}", - "QuAC - Toxic fraction": "{\"description\": \"min=0.002, mean=0.002, max=0.002, sum=0.002 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.002\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "HellaSwag", - "source_data": { - "dataset_name": "HellaSwag", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on HellaSwag", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "HellaSwag - ECE (10-bin)": "{\"description\": \"No matching runs\", \"tab\": \"Calibration\", \"score\": \"\"}", - "HellaSwag - EM (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "HellaSwag - EM (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "HellaSwag - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "HellaSwag - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "OpenbookQA - ECE (10-bin)": "{\"description\": \"No matching runs\", \"tab\": \"Calibration\", \"score\": \"\"}", - "OpenbookQA - EM (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "OpenbookQA - EM (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "OpenbookQA - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "OpenbookQA - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "TruthfulQA", - "source_data": { - "dataset_name": "TruthfulQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on TruthfulQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.243, - "details": { - "description": "min=0.243, mean=0.243, max=0.243, sum=0.243 (1)", - "tab": "Accuracy", - "TruthfulQA - ECE (10-bin)": "{\"description\": \"min=0.375, mean=0.375, max=0.375, sum=0.375 (1)\", \"tab\": \"Calibration\", \"score\": \"0.3750196178145884\"}", - "TruthfulQA - EM (Robustness)": "{\"description\": \"min=0.199, mean=0.199, max=0.199, sum=0.199 (1)\", \"tab\": \"Robustness\", \"score\": \"0.19877675840978593\"}", - "TruthfulQA - EM (Fairness)": "{\"description\": \"min=0.202, mean=0.202, max=0.202, sum=0.202 (1)\", \"tab\": \"Fairness\", \"score\": \"0.2018348623853211\"}", - "TruthfulQA - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "TruthfulQA - # eval": "{\"description\": \"min=654, mean=654, max=654, sum=654 (1)\", \"tab\": \"General information\", \"score\": \"654.0\"}", - "TruthfulQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "TruthfulQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "TruthfulQA - # prompt tokens": "{\"description\": \"min=524.602, mean=524.602, max=524.602, sum=524.602 (1)\", \"tab\": \"General information\", \"score\": \"524.6024464831804\"}", - "TruthfulQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "TruthfulQA - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MS MARCO (TREC)", - "source_data": { - "dataset_name": "MS MARCO (TREC)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "NDCG@10 on MS MARCO (TREC)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "MS MARCO (regular) - RR@10 (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "MS MARCO (TREC) - NDCG@10 (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "MS MARCO (regular) - RR@10 (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "MS MARCO (TREC) - NDCG@10 (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "MS MARCO (regular) - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MS MARCO (TREC) - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MS MARCO (regular) - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "MS MARCO (TREC) - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CNN/DailyMail", - "source_data": { - "dataset_name": "CNN/DailyMail", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on CNN/DailyMail", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "CNN/DailyMail - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "CNN/DailyMail - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "CNN/DailyMail - SummaC": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - QAFactEval": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - BERTScore (F1)": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - Coverage": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - Density": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - Compression": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-faithfulness": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-relevance": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-coherence": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "XSUM", - "source_data": { - "dataset_name": "XSUM", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on XSUM", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "XSUM - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "XSUM - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "XSUM - SummaC": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - QAFactEval": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - BERTScore (F1)": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - Coverage": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - Density": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - Compression": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-faithfulness": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-relevance": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-coherence": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "IMDB", - "source_data": { - "dataset_name": "IMDB", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on IMDB", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.738, - "details": { - "description": "min=0.738, mean=0.738, max=0.738, sum=0.738 (1)", - "tab": "Accuracy", - "IMDB - ECE (10-bin)": "{\"description\": \"min=0.281, mean=0.281, max=0.281, sum=0.281 (1)\", \"tab\": \"Calibration\", \"score\": \"0.28073357253102127\"}", - "IMDB - EM (Robustness)": "{\"description\": \"min=0.561, mean=0.561, max=0.561, sum=0.561 (1)\", \"tab\": \"Robustness\", \"score\": \"0.561\"}", - "IMDB - EM (Fairness)": "{\"description\": \"min=0.699, mean=0.699, max=0.699, sum=0.699 (1)\", \"tab\": \"Fairness\", \"score\": \"0.699\"}", - "IMDB - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "IMDB - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "IMDB - # train": "{\"description\": \"min=2.781, mean=2.781, max=2.781, sum=2.781 (1)\", \"tab\": \"General information\", \"score\": \"2.781\"}", - "IMDB - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IMDB - # prompt tokens": "{\"description\": \"min=1751.213, mean=1751.213, max=1751.213, sum=1751.213 (1)\", \"tab\": \"General information\", \"score\": \"1751.213\"}", - "IMDB - # output tokens": "{\"description\": \"min=4.966, mean=4.966, max=4.966, sum=4.966 (1)\", \"tab\": \"General information\", \"score\": \"4.966\"}", - "IMDB - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "IMDB - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CivilComments", - "source_data": { - "dataset_name": "CivilComments", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on CivilComments", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.566, - "details": { - "description": "min=0.158, mean=0.566, max=0.939, sum=10.184 (18)", - "tab": "Accuracy", - "CivilComments - ECE (10-bin)": "{\"description\": \"min=0.162, mean=0.352, max=0.606, sum=6.328 (18)\", \"tab\": \"Calibration\", \"score\": \"0.3515610942498128\"}", - "CivilComments - EM (Robustness)": "{\"description\": \"min=0.133, mean=0.482, max=0.844, sum=8.674 (18)\", \"tab\": \"Robustness\", \"score\": \"0.4818807145268457\"}", - "CivilComments - EM (Fairness)": "{\"description\": \"min=0.122, mean=0.483, max=0.818, sum=8.691 (18)\", \"tab\": \"Fairness\", \"score\": \"0.4828512879651531\"}", - "CivilComments - Denoised inference time (s)": "{\"description\": \"9 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "CivilComments - # eval": "{\"description\": \"min=74, mean=371.556, max=683, sum=6688 (18)\", \"tab\": \"General information\", \"score\": \"371.55555555555554\"}", - "CivilComments - # train": "{\"description\": \"min=5, mean=5, max=5, sum=90 (18)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CivilComments - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (18)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CivilComments - # prompt tokens": "{\"description\": \"min=404.732, mean=855.241, max=1417.567, sum=15394.339 (18)\", \"tab\": \"General information\", \"score\": \"855.2410378605821\"}", - "CivilComments - # output tokens": "{\"description\": \"min=2.746, mean=4.216, max=4.89, sum=75.887 (18)\", \"tab\": \"General information\", \"score\": \"4.2159316386124255\"}", - "CivilComments - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=18 (18)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "CivilComments - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Toxic fraction": "{\"description\": \"9 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "RAFT", - "source_data": { - "dataset_name": "RAFT", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on RAFT", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.486, - "details": { - "description": "min=0, mean=0.486, max=0.9, sum=5.35 (11)", - "tab": "Accuracy", - "RAFT - ECE (10-bin)": "{\"description\": \"min=0.004, mean=0.33, max=0.711, sum=3.626 (11)\", \"tab\": \"Calibration\", \"score\": \"0.3296795633615674\"}", - "RAFT - EM (Robustness)": "{\"description\": \"min=0, mean=0.42, max=0.875, sum=4.625 (11)\", \"tab\": \"Robustness\", \"score\": \"0.42045454545454536\"}", - "RAFT - EM (Fairness)": "{\"description\": \"min=0, mean=0.459, max=0.9, sum=5.05 (11)\", \"tab\": \"Fairness\", \"score\": \"0.45909090909090916\"}", - "RAFT - Denoised inference time (s)": "{\"description\": \"11 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "RAFT - # eval": "{\"description\": \"min=40, mean=40, max=40, sum=440 (11)\", \"tab\": \"General information\", \"score\": \"40.0\"}", - "RAFT - # train": "{\"description\": \"min=0.45, mean=4.552, max=5, sum=50.075 (11)\", \"tab\": \"General information\", \"score\": \"4.552272727272727\"}", - "RAFT - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (11)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "RAFT - # prompt tokens": "{\"description\": \"min=303.675, mean=954.111, max=1882.1, sum=10495.225 (11)\", \"tab\": \"General information\", \"score\": \"954.1113636363635\"}", - "RAFT - # output tokens": "{\"description\": \"min=3.7, mean=19.468, max=30, sum=214.15 (11)\", \"tab\": \"General information\", \"score\": \"19.468181818181815\"}", - "RAFT - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=11 (11)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "RAFT - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (gender)": "{\"description\": \"min=0.5, mean=0.5, max=0.5, sum=0.5 (1)\", \"tab\": \"Bias\", \"score\": \"0.5\"}", - "RAFT - Toxic fraction": "{\"description\": \"11 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/stanfordnlp_steamshp-flan-t5-large.json b/data/models/stanfordnlp_steamshp-flan-t5-large.json deleted file mode 100644 index 76d83757773461421e74ea7822124a9cf85b3090..0000000000000000000000000000000000000000 --- a/data/models/stanfordnlp_steamshp-flan-t5-large.json +++ /dev/null @@ -1,144 +0,0 @@ -{ - "model_info": { - "name": "stanfordnlp/SteamSHP-flan-t5-large", - "id": "stanfordnlp/SteamSHP-flan-t5-large", - "developer": "stanfordnlp", - "additional_details": { - "model_type": "Custom Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/stanfordnlp_SteamSHP-flan-t5-large/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4962 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8575 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3311 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3743 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3563 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6273 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/stanfordnlp_steamshp-flan-t5-xl.json b/data/models/stanfordnlp_steamshp-flan-t5-xl.json deleted file mode 100644 index 00971f26d49b0d4a6fa83dfad3ee25c317dec0a9..0000000000000000000000000000000000000000 --- a/data/models/stanfordnlp_steamshp-flan-t5-xl.json +++ /dev/null @@ -1,144 +0,0 @@ -{ - "model_info": { - "name": "stanfordnlp/SteamSHP-flan-t5-xl", - "id": "stanfordnlp/SteamSHP-flan-t5-xl", - "developer": "stanfordnlp", - "additional_details": { - "model_type": "Custom Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/stanfordnlp_SteamSHP-flan-t5-xl/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5135 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8547 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3684 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3784 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3841 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6498 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/stark2008_gutenlaserpi.json b/data/models/stark2008_gutenlaserpi.json deleted file mode 100644 index a6798c5f8fd13c5631e8c459d61a7e613edb22df..0000000000000000000000000000000000000000 --- a/data/models/stark2008_gutenlaserpi.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "GutenLaserPi", - "id": "Stark2008/GutenLaserPi", - "developer": "Stark2008", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Stark2008_GutenLaserPi/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4227 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5212 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0785 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2869 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.462 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3106 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/stark2008_layleleflampi.json b/data/models/stark2008_layleleflampi.json deleted file mode 100644 index b4d0893074ce334e34e4764bda457cea12b50e38..0000000000000000000000000000000000000000 --- a/data/models/stark2008_layleleflampi.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "LayleleFlamPi", - "id": "Stark2008/LayleleFlamPi", - "developer": "Stark2008", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Stark2008_LayleleFlamPi/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4284 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5116 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0665 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2852 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4608 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3093 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/stark2008_visflamcat.json b/data/models/stark2008_visflamcat.json deleted file mode 100644 index 68344f2171fa91b131193502bea0b60c8e5c33d5..0000000000000000000000000000000000000000 --- a/data/models/stark2008_visflamcat.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "VisFlamCat", - "id": "Stark2008/VisFlamCat", - "developer": "Stark2008", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Stark2008_VisFlamCat/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4366 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5217 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0763 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2903 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4463 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3144 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/steelskull_l3.3-ms-nevoria-70b.json b/data/models/steelskull_l3.3-ms-nevoria-70b.json deleted file mode 100644 index 87def9e5d1d64f0089b7c0c1b80a47073d43ca30..0000000000000000000000000000000000000000 --- a/data/models/steelskull_l3.3-ms-nevoria-70b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "L3.3-MS-Nevoria-70b", - "id": "Steelskull/L3.3-MS-Nevoria-70b", - "developer": "Steelskull", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "70.554" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Steelskull_L3.3-MS-Nevoria-70b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6963 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6998 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3958 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4706 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4682 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5535 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/steelskull_l3.3-nevoria-r1-70b.json b/data/models/steelskull_l3.3-nevoria-r1-70b.json deleted file mode 100644 index eda48ac5647b54b9a7e93192ed4072d4a2fdb5a7..0000000000000000000000000000000000000000 --- a/data/models/steelskull_l3.3-nevoria-r1-70b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "L3.3-Nevoria-R1-70b", - "id": "Steelskull/L3.3-Nevoria-R1-70b", - "developer": "Steelskull", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "70.554" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Steelskull_L3.3-Nevoria-R1-70b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6024 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6972 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.463 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.469 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4775 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5463 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/stellex_qwen2.5_math_7b_cot.json b/data/models/stellex_qwen2.5_math_7b_cot.json deleted file mode 100644 index cf46d468909a848ba9be85aac308f7b7c00c9ad1..0000000000000000000000000000000000000000 --- a/data/models/stellex_qwen2.5_math_7b_cot.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5_Math_7B_Cot", - "id": "StelleX/Qwen2.5_Math_7B_Cot", - "developer": "StelleX", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/StelleX_Qwen2.5_Math_7B_Cot/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2143 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4313 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3263 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2945 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3924 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.281 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/stellex_vorisatex-7b-preview.json b/data/models/stellex_vorisatex-7b-preview.json deleted file mode 100644 index 91a4b32b958169ba71b9fef50eac5f9618852388..0000000000000000000000000000000000000000 --- a/data/models/stellex_vorisatex-7b-preview.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Vorisatex-7B-preview", - "id": "StelleX/Vorisatex-7B-preview", - "developer": "StelleX", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.613" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/StelleX_Vorisatex-7B-preview/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1515 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3112 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0287 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2517 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4192 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1166 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sthenno-com_miscii-14b-0130.json b/data/models/sthenno-com_miscii-14b-0130.json deleted file mode 100644 index 7d107d39b7e7aedafee26dd89513273534abcc0e..0000000000000000000000000000000000000000 --- a/data/models/sthenno-com_miscii-14b-0130.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "miscii-14b-0130", - "id": "sthenno-com/miscii-14b-0130", - "developer": "sthenno-com", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sthenno-com_miscii-14b-0130/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6647 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6505 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.432 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3817 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4912 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5363 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sthenno-com_miscii-14b-0218.json b/data/models/sthenno-com_miscii-14b-0218.json deleted file mode 100644 index 71436f6b665d69b0d854ecd5444eab1e5b217a0a..0000000000000000000000000000000000000000 --- a/data/models/sthenno-com_miscii-14b-0218.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "miscii-14b-0218", - "id": "sthenno-com/miscii-14b-0218", - "developer": "sthenno-com", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sthenno-com_miscii-14b-0218/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7656 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6559 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5144 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3834 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4273 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5298 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sthenno-com_miscii-14b-1028.json b/data/models/sthenno-com_miscii-14b-1028.json deleted file mode 100644 index a486cd21eac738d060ce74523f71fe4ec6cad545..0000000000000000000000000000000000000000 --- a/data/models/sthenno-com_miscii-14b-1028.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "miscii-14b-1028", - "id": "sthenno-com/miscii-14b-1028", - "developer": "sthenno-com", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sthenno-com_miscii-14b-1028/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8237 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6448 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.503 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3565 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4182 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5153 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sthenno-com_miscii-14b-1225.json b/data/models/sthenno-com_miscii-14b-1225.json deleted file mode 100644 index ec65add4733cc7a377351c7158b2faf0a95bc0c3..0000000000000000000000000000000000000000 --- a/data/models/sthenno-com_miscii-14b-1225.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "miscii-14b-1225", - "id": "sthenno-com/miscii-14b-1225", - "developer": "sthenno-com", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sthenno-com_miscii-14b-1225/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7878 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6572 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4517 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3775 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4366 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5272 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sthenno_tempesthenno-0120.json b/data/models/sthenno_tempesthenno-0120.json deleted file mode 100644 index 84f3ecbf3d5f1c62dd1a1862c1c25af2ddc65aaf..0000000000000000000000000000000000000000 --- a/data/models/sthenno_tempesthenno-0120.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "tempesthenno-0120", - "id": "sthenno/tempesthenno-0120", - "developer": "sthenno", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sthenno_tempesthenno-0120/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.539 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6373 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3353 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3943 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4633 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.529 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sthenno_tempesthenno-fusion-0309.json b/data/models/sthenno_tempesthenno-fusion-0309.json deleted file mode 100644 index 14e13c00dbbaa1c5c347688dac40a7225fd9496a..0000000000000000000000000000000000000000 --- a/data/models/sthenno_tempesthenno-fusion-0309.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "tempesthenno-fusion-0309", - "id": "sthenno/tempesthenno-fusion-0309", - "developer": "sthenno", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sthenno_tempesthenno-fusion-0309/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7692 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6581 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4766 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.37 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4325 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5258 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sthenno_tempesthenno-kto-0205-ckpt80.json b/data/models/sthenno_tempesthenno-kto-0205-ckpt80.json deleted file mode 100644 index cfe534d27c27a7abf7f18e212415892ac4075d56..0000000000000000000000000000000000000000 --- a/data/models/sthenno_tempesthenno-kto-0205-ckpt80.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "tempesthenno-kto-0205-ckpt80", - "id": "sthenno/tempesthenno-kto-0205-ckpt80", - "developer": "sthenno", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sthenno_tempesthenno-kto-0205-ckpt80/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8054 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6543 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4592 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3482 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4248 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5286 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sthenno_tempesthenno-nuslerp-001.json b/data/models/sthenno_tempesthenno-nuslerp-001.json deleted file mode 100644 index 12a78c44affe2c6700cf34bcb6e6b5031982f030..0000000000000000000000000000000000000000 --- a/data/models/sthenno_tempesthenno-nuslerp-001.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "tempesthenno-nuslerp-001", - "id": "sthenno/tempesthenno-nuslerp-001", - "developer": "sthenno", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sthenno_tempesthenno-nuslerp-001/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7926 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6578 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4758 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3733 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.43 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5257 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sthenno_tempesthenno-nuslerp-0124.json b/data/models/sthenno_tempesthenno-nuslerp-0124.json deleted file mode 100644 index 0494f4667224576ad569a90c97d30b85f2d91f41..0000000000000000000000000000000000000000 --- a/data/models/sthenno_tempesthenno-nuslerp-0124.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "tempesthenno-nuslerp-0124", - "id": "sthenno/tempesthenno-nuslerp-0124", - "developer": "sthenno", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sthenno_tempesthenno-nuslerp-0124/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7004 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6469 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4116 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3901 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4859 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5352 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sthenno_tempesthenno-ppo-ckpt40.json b/data/models/sthenno_tempesthenno-ppo-ckpt40.json deleted file mode 100644 index 488fa7e8dbd914544281e6e1d4b591be9129b211..0000000000000000000000000000000000000000 --- a/data/models/sthenno_tempesthenno-ppo-ckpt40.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "tempesthenno-ppo-ckpt40", - "id": "sthenno/tempesthenno-ppo-ckpt40", - "developer": "sthenno", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sthenno_tempesthenno-ppo-ckpt40/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7923 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.655 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4736 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3775 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4352 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5292 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sthenno_tempesthenno-sft-0309-ckpt10.json b/data/models/sthenno_tempesthenno-sft-0309-ckpt10.json deleted file mode 100644 index 189aedd1636409c5cf7693e256c06fb3b344ece1..0000000000000000000000000000000000000000 --- a/data/models/sthenno_tempesthenno-sft-0309-ckpt10.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "tempesthenno-sft-0309-ckpt10", - "id": "sthenno/tempesthenno-sft-0309-ckpt10", - "developer": "sthenno", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sthenno_tempesthenno-sft-0309-ckpt10/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7744 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6552 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4721 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3716 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4364 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5258 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sthenno_tempesthenno-sft-0314-stage1-ckpt50.json b/data/models/sthenno_tempesthenno-sft-0314-stage1-ckpt50.json deleted file mode 100644 index 652dfc41d2ad8dc847fd3ab6d936c28e162a1b1c..0000000000000000000000000000000000000000 --- a/data/models/sthenno_tempesthenno-sft-0314-stage1-ckpt50.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "tempesthenno-sft-0314-stage1-ckpt50", - "id": "sthenno/tempesthenno-sft-0314-stage1-ckpt50", - "developer": "sthenno", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sthenno_tempesthenno-sft-0314-stage1-ckpt50/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7394 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6601 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4683 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3733 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4429 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5302 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sthenno_tempestissimo-14b-0309.json b/data/models/sthenno_tempestissimo-14b-0309.json deleted file mode 100644 index 80f9a0a5d6534a2cd40b51b14020e00f6eb97dff..0000000000000000000000000000000000000000 --- a/data/models/sthenno_tempestissimo-14b-0309.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "tempestissimo-14b-0309", - "id": "sthenno/tempestissimo-14b-0309", - "developer": "sthenno", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sthenno_tempestissimo-14b-0309/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7549 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6587 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4796 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3666 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4312 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5281 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/streamerbtw1002_nexuim-r1-7b-instruct.json b/data/models/streamerbtw1002_nexuim-r1-7b-instruct.json deleted file mode 100644 index e044dd19dc6ad8ff71bcf232448ed3c154e9b887..0000000000000000000000000000000000000000 --- a/data/models/streamerbtw1002_nexuim-r1-7b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Nexuim-R1-7B-Instruct", - "id": "streamerbtw1002/Nexuim-R1-7B-Instruct", - "developer": "streamerbtw1002", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/streamerbtw1002_Nexuim-R1-7B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6934 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5175 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4456 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2592 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3356 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4138 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/stupidity-ai_llama-3-8b-instruct-multimoose.json b/data/models/stupidity-ai_llama-3-8b-instruct-multimoose.json deleted file mode 100644 index d968a74653aec26c36b1b1da71d1127c4349175b..0000000000000000000000000000000000000000 --- a/data/models/stupidity-ai_llama-3-8b-instruct-multimoose.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-8B-Instruct-MultiMoose", - "id": "stupidity-ai/Llama-3-8B-Instruct-MultiMoose", - "developer": "stupidity-ai", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/stupidity-ai_Llama-3-8B-Instruct-MultiMoose/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2318 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2823 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2534 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3485 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1094 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/suayptalha_clarus-7b-v0.1.json b/data/models/suayptalha_clarus-7b-v0.1.json deleted file mode 100644 index 04eb404edf97e42a687f9d220cd5dacab5aee9a2..0000000000000000000000000000000000000000 --- a/data/models/suayptalha_clarus-7b-v0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Clarus-7B-v0.1", - "id": "suayptalha/Clarus-7B-v0.1", - "developer": "suayptalha", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/suayptalha_Clarus-7B-v0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7454 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5497 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4924 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.307 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.443 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4387 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/suayptalha_clarus-7b-v0.2.json b/data/models/suayptalha_clarus-7b-v0.2.json deleted file mode 100644 index facb82f93588c1000c6e131c2a56ad57acc7d1e4..0000000000000000000000000000000000000000 --- a/data/models/suayptalha_clarus-7b-v0.2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Clarus-7B-v0.2", - "id": "suayptalha/Clarus-7B-v0.2", - "developer": "suayptalha", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.613" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/suayptalha_Clarus-7B-v0.2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7679 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.549 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4856 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.302 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4417 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.44 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/suayptalha_clarus-7b-v0.3.json b/data/models/suayptalha_clarus-7b-v0.3.json deleted file mode 100644 index 25ecf432967bb49d176af087b949b80af361ce31..0000000000000000000000000000000000000000 --- a/data/models/suayptalha_clarus-7b-v0.3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Clarus-7B-v0.3", - "id": "suayptalha/Clarus-7B-v0.3", - "developer": "suayptalha", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/suayptalha_Clarus-7B-v0.3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7509 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5526 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4879 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3121 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4402 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4385 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/suayptalha_deepseek-r1-distill-llama-3b.json b/data/models/suayptalha_deepseek-r1-distill-llama-3b.json deleted file mode 100644 index 73c21e70279fea491e31ba4a078e2a887deb49a8..0000000000000000000000000000000000000000 --- a/data/models/suayptalha_deepseek-r1-distill-llama-3b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "DeepSeek-R1-Distill-Llama-3B", - "id": "suayptalha/DeepSeek-R1-Distill-Llama-3B", - "developer": "suayptalha", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/suayptalha_DeepSeek-R1-Distill-Llama-3B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7093 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4452 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2092 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2609 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3396 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2978 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/suayptalha_falcon3-jessi-v0.4-7b-slerp.json b/data/models/suayptalha_falcon3-jessi-v0.4-7b-slerp.json deleted file mode 100644 index ff6f0a8971c67dacb46f8503ff0ff774f61e14bd..0000000000000000000000000000000000000000 --- a/data/models/suayptalha_falcon3-jessi-v0.4-7b-slerp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Falcon3-Jessi-v0.4-7B-Slerp", - "id": "suayptalha/Falcon3-Jessi-v0.4-7B-Slerp", - "developer": "suayptalha", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "7.456" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/suayptalha_Falcon3-Jessi-v0.4-7B-Slerp/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7676 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5591 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3965 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3121 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4812 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.406 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/suayptalha_homercreativeanvita-mix-qw7b.json b/data/models/suayptalha_homercreativeanvita-mix-qw7b.json deleted file mode 100644 index 5739a1e98132d5e79f30afdb65718a19efb536ca..0000000000000000000000000000000000000000 --- a/data/models/suayptalha_homercreativeanvita-mix-qw7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "HomerCreativeAnvita-Mix-Qw7B", - "id": "suayptalha/HomerCreativeAnvita-Mix-Qw7B", - "developer": "suayptalha", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/suayptalha_HomerCreativeAnvita-Mix-Qw7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7808 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5565 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.361 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3146 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4416 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4445 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/suayptalha_komodo-llama-3.2-3b-v2-fp16.json b/data/models/suayptalha_komodo-llama-3.2-3b-v2-fp16.json deleted file mode 100644 index e54272b1b015430246aa39e2bbb6c1d34ed7346d..0000000000000000000000000000000000000000 --- a/data/models/suayptalha_komodo-llama-3.2-3b-v2-fp16.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Komodo-Llama-3.2-3B-v2-fp16", - "id": "suayptalha/Komodo-Llama-3.2-3B-v2-fp16", - "developer": "suayptalha", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/suayptalha_Komodo-Llama-3.2-3B-v2-fp16/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6341 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4355 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1065 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2777 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3406 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2852 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/suayptalha_lamarckvergence-14b.json b/data/models/suayptalha_lamarckvergence-14b.json deleted file mode 100644 index 6eb19117a291132a3bed5288354935d825527696..0000000000000000000000000000000000000000 --- a/data/models/suayptalha_lamarckvergence-14b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Lamarckvergence-14B", - "id": "suayptalha/Lamarckvergence-14B", - "developer": "suayptalha", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/suayptalha_Lamarckvergence-14B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7656 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6517 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.54 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3633 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4422 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5283 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/suayptalha_lix-14b-v0.1.json b/data/models/suayptalha_lix-14b-v0.1.json deleted file mode 100644 index e1a0cb2d6924da03b74a64b199c9af0f2187077b..0000000000000000000000000000000000000000 --- a/data/models/suayptalha_lix-14b-v0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Lix-14B-v0.1", - "id": "suayptalha/Lix-14B-v0.1", - "developer": "suayptalha", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/suayptalha_Lix-14B-v0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7813 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6608 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5295 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.37 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4338 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5314 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/suayptalha_luminis-phi-4.json b/data/models/suayptalha_luminis-phi-4.json deleted file mode 100644 index b815654fc29ef8515b13a15ff4cd61c28972fac6..0000000000000000000000000000000000000000 --- a/data/models/suayptalha_luminis-phi-4.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Luminis-phi-4", - "id": "suayptalha/Luminis-phi-4", - "developer": "suayptalha", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/suayptalha_Luminis-phi-4/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.69 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.692 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4637 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3515 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4572 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5424 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/suayptalha_maestro-10b.json b/data/models/suayptalha_maestro-10b.json deleted file mode 100644 index b5952970fe8467678287361dfdfe79d151919b6a..0000000000000000000000000000000000000000 --- a/data/models/suayptalha_maestro-10b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Maestro-10B", - "id": "suayptalha/Maestro-10B", - "developer": "suayptalha", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "10.306" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/suayptalha_Maestro-10B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7768 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5746 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1911 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3331 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4397 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4218 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/suayptalha_rombos-2.5-t.e-8.1.json b/data/models/suayptalha_rombos-2.5-t.e-8.1.json deleted file mode 100644 index 16fa9f549cc6be84bf6312b04758e3b44b98f7bb..0000000000000000000000000000000000000000 --- a/data/models/suayptalha_rombos-2.5-t.e-8.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Rombos-2.5-T.E-8.1", - "id": "suayptalha/Rombos-2.5-T.E-8.1", - "developer": "suayptalha", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/suayptalha_Rombos-2.5-T.E-8.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6925 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5515 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4924 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3112 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4166 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4446 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sultanr_smoltulu-1.7b-instruct.json b/data/models/sultanr_smoltulu-1.7b-instruct.json deleted file mode 100644 index 15611468ca88ee7f420fe98eac449776c2d1ba66..0000000000000000000000000000000000000000 --- a/data/models/sultanr_smoltulu-1.7b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SmolTulu-1.7b-Instruct", - "id": "SultanR/SmolTulu-1.7b-Instruct", - "developer": "SultanR", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.711" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/SultanR_SmolTulu-1.7b-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6541 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3713 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0793 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2693 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.354 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.171 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sultanr_smoltulu-1.7b-it-v0.json b/data/models/sultanr_smoltulu-1.7b-it-v0.json deleted file mode 100644 index ced09df738ec071055ab9f1605b2f2a47ae3d0f2..0000000000000000000000000000000000000000 --- a/data/models/sultanr_smoltulu-1.7b-it-v0.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SmolTulu-1.7b-it-v0", - "id": "SultanR/SmolTulu-1.7b-it-v0", - "developer": "SultanR", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.711" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/SultanR_SmolTulu-1.7b-it-v0/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6541 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3713 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0793 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2693 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.354 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.171 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sultanr_smoltulu-1.7b-reinforced.json b/data/models/sultanr_smoltulu-1.7b-reinforced.json deleted file mode 100644 index d790ff71dfdaca06f09bda13d28c030a3525605f..0000000000000000000000000000000000000000 --- a/data/models/sultanr_smoltulu-1.7b-reinforced.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SmolTulu-1.7b-Reinforced", - "id": "SultanR/SmolTulu-1.7b-Reinforced", - "developer": "SultanR", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.711" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/SultanR_SmolTulu-1.7b-Reinforced/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6791 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3552 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0718 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.276 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3406 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1763 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sultanr_smoltulu-1.7b-rm.json b/data/models/sultanr_smoltulu-1.7b-rm.json deleted file mode 100644 index 56019d0afed8b6504722626bf6c3a18dd29e8df5..0000000000000000000000000000000000000000 --- a/data/models/sultanr_smoltulu-1.7b-rm.json +++ /dev/null @@ -1,126 +0,0 @@ -{ - "model_info": { - "name": "SultanR/SmolTulu-1.7b-RM", - "id": "SultanR/SmolTulu-1.7b-RM", - "developer": "SultanR", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/SultanR_SmolTulu-1.7b-RM/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5094 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.743 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4408 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5716 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2821 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sumink_bbhqwen.json b/data/models/sumink_bbhqwen.json deleted file mode 100644 index 2980de80d9612bc7e5b0d844e0501a55d79897e7..0000000000000000000000000000000000000000 --- a/data/models/sumink_bbhqwen.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "bbhqwen", - "id": "sumink/bbhqwen", - "developer": "sumink", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.086" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sumink_bbhqwen/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1809 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3388 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0106 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2576 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4352 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1617 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sumink_bbhqwen2.json b/data/models/sumink_bbhqwen2.json deleted file mode 100644 index edf55a1ca96e32092a3abc652ed8bff92dbdb13d..0000000000000000000000000000000000000000 --- a/data/models/sumink_bbhqwen2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "bbhqwen2", - "id": "sumink/bbhqwen2", - "developer": "sumink", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.086" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sumink_bbhqwen2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1533 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3066 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.006 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2626 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4431 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1149 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sumink_bbhqwen3.json b/data/models/sumink_bbhqwen3.json deleted file mode 100644 index e6a703b1e9a92e2628495487f84c851f4a60f01f..0000000000000000000000000000000000000000 --- a/data/models/sumink_bbhqwen3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "bbhqwen3", - "id": "sumink/bbhqwen3", - "developer": "sumink", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.086" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sumink_bbhqwen3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1943 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2951 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2576 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3796 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1166 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sumink_bbhqwen4.json b/data/models/sumink_bbhqwen4.json deleted file mode 100644 index 4df15602b67c0f58d42ba288d404a3ed7681888d..0000000000000000000000000000000000000000 --- a/data/models/sumink_bbhqwen4.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "bbhqwen4", - "id": "sumink/bbhqwen4", - "developer": "sumink", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.086" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sumink_bbhqwen4/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1449 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3199 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.006 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2441 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4029 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1509 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sumink_bbhqwen5.json b/data/models/sumink_bbhqwen5.json deleted file mode 100644 index 6204044f7418974ba59d9304519a204dfdea5778..0000000000000000000000000000000000000000 --- a/data/models/sumink_bbhqwen5.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "bbhqwen5", - "id": "sumink/bbhqwen5", - "developer": "sumink", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.086" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sumink_bbhqwen5/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1522 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2913 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0023 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2601 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4019 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1131 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sumink_bbhqwen6.json b/data/models/sumink_bbhqwen6.json deleted file mode 100644 index 73bf5c12924b113955549a99af516276a77346b9..0000000000000000000000000000000000000000 --- a/data/models/sumink_bbhqwen6.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "bbhqwen6", - "id": "sumink/bbhqwen6", - "developer": "sumink", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.086" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sumink_bbhqwen6/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1893 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2782 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0008 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2584 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.358 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1153 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sumink_flflmillama.json b/data/models/sumink_flflmillama.json deleted file mode 100644 index 957491fac1cb899842f203e5a6df3db6192c03a9..0000000000000000000000000000000000000000 --- a/data/models/sumink_flflmillama.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "flflmillama", - "id": "sumink/flflmillama", - "developer": "sumink", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sumink_flflmillama/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1676 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3851 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0196 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2919 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3591 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2096 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sumink_ftgpt.json b/data/models/sumink_ftgpt.json deleted file mode 100644 index 7d7e77392be27daf1cc01237c2ec1585f0bc9a4c..0000000000000000000000000000000000000000 --- a/data/models/sumink_ftgpt.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ftgpt", - "id": "sumink/ftgpt", - "developer": "sumink", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "GPT2LMHeadModel", - "params_billions": "0.124" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sumink_ftgpt/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0787 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2919 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2643 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4138 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1172 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sumink_llamaft.json b/data/models/sumink_llamaft.json deleted file mode 100644 index 149a75927cb04533fefa6a1e1a1c7337d6a0a3e2..0000000000000000000000000000000000000000 --- a/data/models/sumink_llamaft.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "llamaft", - "id": "sumink/llamaft", - "developer": "sumink", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sumink_llamaft/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1609 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3763 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0166 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.271 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3498 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2114 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sumink_llamamerge.json b/data/models/sumink_llamamerge.json deleted file mode 100644 index 4ad8f024b2bc91250fa35dc66077e2f35fc58714..0000000000000000000000000000000000000000 --- a/data/models/sumink_llamamerge.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "llamamerge", - "id": "sumink/llamamerge", - "developer": "sumink", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "13.016" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sumink_llamamerge/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2672 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4632 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0151 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2987 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.424 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.259 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sumink_llftfl7.json b/data/models/sumink_llftfl7.json deleted file mode 100644 index 7f1fc57a8196af7f5e7ab71b910215704b4ca2b0..0000000000000000000000000000000000000000 --- a/data/models/sumink_llftfl7.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "llftfl7", - "id": "sumink/llftfl7", - "developer": "sumink", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sumink_llftfl7/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1714 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3786 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0106 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.281 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3632 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1743 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sumink_llmer.json b/data/models/sumink_llmer.json deleted file mode 100644 index 93e1a4d1ae534b9d02ed317f65e45e55f6d5603f..0000000000000000000000000000000000000000 --- a/data/models/sumink_llmer.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "llmer", - "id": "sumink/llmer", - "developer": "sumink", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sumink_llmer/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3191 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4885 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.065 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2978 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4039 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3529 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sumink_qmerft.json b/data/models/sumink_qmerft.json deleted file mode 100644 index 216aeafa435b8efcf89725ae43993dc6ddba91b9..0000000000000000000000000000000000000000 --- a/data/models/sumink_qmerft.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qmerft", - "id": "sumink/Qmerft", - "developer": "sumink", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.777" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sumink_Qmerft/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1564 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2939 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0023 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2525 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3688 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1157 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sumink_qwenftmodel.json b/data/models/sumink_qwenftmodel.json deleted file mode 100644 index 8560f451cd84678f368c3399fb3e08e75694fe36..0000000000000000000000000000000000000000 --- a/data/models/sumink_qwenftmodel.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwenftmodel", - "id": "sumink/Qwenftmodel", - "developer": "sumink", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.544" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sumink_Qwenftmodel/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1729 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3823 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0891 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2567 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3617 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2339 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sumink_qwenmplus.json b/data/models/sumink_qwenmplus.json deleted file mode 100644 index 0cc71b5a636f33ded945dbbc3d0baaf219fca5cc..0000000000000000000000000000000000000000 --- a/data/models/sumink_qwenmplus.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwenmplus", - "id": "sumink/Qwenmplus", - "developer": "sumink", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.543" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sumink_Qwenmplus/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.204 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3676 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0249 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2852 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3828 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1992 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sumink_qwensci.json b/data/models/sumink_qwensci.json deleted file mode 100644 index a43277e9cfde16474bc30d78c04021f029e765c8..0000000000000000000000000000000000000000 --- a/data/models/sumink_qwensci.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwensci", - "id": "sumink/Qwensci", - "developer": "sumink", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.543" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sumink_Qwensci/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.174 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3282 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0204 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2584 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3609 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.126 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sumink_qwft.json b/data/models/sumink_qwft.json deleted file mode 100644 index a060a9f784bf292e0b587601aa9e259a4c9dd5b8..0000000000000000000000000000000000000000 --- a/data/models/sumink_qwft.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "qwft", - "id": "sumink/qwft", - "developer": "sumink", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sumink_qwft/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1197 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3002 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2525 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3581 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1129 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sumink_qwmer.json b/data/models/sumink_qwmer.json deleted file mode 100644 index bdad4dfb28b571bfd8419e45b14688aec087c588..0000000000000000000000000000000000000000 --- a/data/models/sumink_qwmer.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "qwmer", - "id": "sumink/qwmer", - "developer": "sumink", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sumink_qwmer/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2212 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4299 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0008 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2869 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4032 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2215 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sumink_solarmer3.json b/data/models/sumink_solarmer3.json deleted file mode 100644 index 0b1b736a8eac0ba8318c3c8cdf79f07ef6538644..0000000000000000000000000000000000000000 --- a/data/models/sumink_solarmer3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "solarmer3", - "id": "sumink/solarmer3", - "developer": "sumink", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "10.732" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sumink_solarmer3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3741 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5266 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0582 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2911 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4401 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3323 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sumink_somer.json b/data/models/sumink_somer.json deleted file mode 100644 index 87384fab571ffdfdd828c67871bd52a130072d02..0000000000000000000000000000000000000000 --- a/data/models/sumink_somer.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "somer", - "id": "sumink/somer", - "developer": "sumink", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "10.732" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sumink_somer/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.299 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5194 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0415 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2987 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.465 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3447 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sumink_somer2.json b/data/models/sumink_somer2.json deleted file mode 100644 index 3a8717eb6621a90245c1a4ae3d538d2acf9966af..0000000000000000000000000000000000000000 --- a/data/models/sumink_somer2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "somer2", - "id": "sumink/somer2", - "developer": "sumink", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "10.732" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sumink_somer2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3132 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5167 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0468 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3037 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4663 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3433 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sumink_somerft.json b/data/models/sumink_somerft.json deleted file mode 100644 index 63aa25c2ea6bdf92f782c900f98beb83c6c24f49..0000000000000000000000000000000000000000 --- a/data/models/sumink_somerft.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "somerft", - "id": "sumink/somerft", - "developer": "sumink", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.543" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sumink_somerft/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1431 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3093 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0144 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2483 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4045 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1117 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/sunbaby_braincog-8b-0.1-instruct.json b/data/models/sunbaby_braincog-8b-0.1-instruct.json deleted file mode 100644 index f639c9edd590f47304bbaad5a3ddb8b7bd35ac19..0000000000000000000000000000000000000000 --- a/data/models/sunbaby_braincog-8b-0.1-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "BrainCog-8B-0.1-Instruct", - "id": "sunbaby/BrainCog-8B-0.1-Instruct", - "developer": "sunbaby", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/sunbaby_BrainCog-8B-0.1-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4253 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4618 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0967 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3012 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3656 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2858 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/supichi_bba-123.json b/data/models/supichi_bba-123.json deleted file mode 100644 index 4bf4c53cb6ab3238b742ec738edfad3bba98edf7..0000000000000000000000000000000000000000 --- a/data/models/supichi_bba-123.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "BBA-123", - "id": "Supichi/BBA-123", - "developer": "Supichi", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "17.161" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Supichi_BBA-123/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.208 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.292 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2601 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3499 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1167 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/supichi_bba99.json b/data/models/supichi_bba99.json deleted file mode 100644 index dbf82b51435696a437b494648e41968019e920ed..0000000000000000000000000000000000000000 --- a/data/models/supichi_bba99.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "BBA99", - "id": "Supichi/BBA99", - "developer": "Supichi", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "17.161" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Supichi_BBA99/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1407 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2769 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2634 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3218 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1112 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/supichi_bbai_135_gemma.json b/data/models/supichi_bbai_135_gemma.json deleted file mode 100644 index 51932c23e60057a039966bc7eaaa8206ae4daeb2..0000000000000000000000000000000000000000 --- a/data/models/supichi_bbai_135_gemma.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "BBAI_135_Gemma", - "id": "Supichi/BBAI_135_Gemma", - "developer": "Supichi", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "19.3" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Supichi_BBAI_135_Gemma/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0656 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3568 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2676 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3805 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1672 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/supichi_bbai_250_xia0_gz.json b/data/models/supichi_bbai_250_xia0_gz.json deleted file mode 100644 index ccda7968211824b6fd7079bb11fa7a0249c70a7f..0000000000000000000000000000000000000000 --- a/data/models/supichi_bbai_250_xia0_gz.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "BBAI_250_Xia0_gZ", - "id": "Supichi/BBAI_250_Xia0_gZ", - "developer": "Supichi", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Supichi_BBAI_250_Xia0_gZ/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4685 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5568 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.364 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3213 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4579 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4465 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/supichi_bbai_275_tsunami_gz.json b/data/models/supichi_bbai_275_tsunami_gz.json deleted file mode 100644 index 33aba27eeabc5007dd1fdabd5498badbb0ece0a4..0000000000000000000000000000000000000000 --- a/data/models/supichi_bbai_275_tsunami_gz.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "BBAI_275_Tsunami_gZ", - "id": "Supichi/BBAI_275_Tsunami_gZ", - "developer": "Supichi", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Supichi_BBAI_275_Tsunami_gZ/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.537 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5531 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3285 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3213 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4448 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4492 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/supichi_bbai_525_tsu_gz_xia0.json b/data/models/supichi_bbai_525_tsu_gz_xia0.json deleted file mode 100644 index c0e3832f1b0b3868717e64c4f0c1fbecf1453728..0000000000000000000000000000000000000000 --- a/data/models/supichi_bbai_525_tsu_gz_xia0.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "BBAI_525_Tsu_gZ_Xia0", - "id": "Supichi/BBAI_525_Tsu_gZ_Xia0", - "developer": "Supichi", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Supichi_BBAI_525_Tsu_gZ_Xia0/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5339 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5562 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3429 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3121 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4474 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4477 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/supichi_bbai_78b_calme_3_1_ties.json b/data/models/supichi_bbai_78b_calme_3_1_ties.json deleted file mode 100644 index fd863442c4a58d2e0d8dcf260390742617453574..0000000000000000000000000000000000000000 --- a/data/models/supichi_bbai_78b_calme_3_1_ties.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "BBAI_78B_Calme_3_1_Ties", - "id": "Supichi/BBAI_78B_Calme_3_1_Ties", - "developer": "Supichi", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "27.06" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Supichi_BBAI_78B_Calme_3_1_Ties/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1828 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2828 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.229 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.31 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1144 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/supichi_bbai_qween_v000000_lumen_14b.json b/data/models/supichi_bbai_qween_v000000_lumen_14b.json deleted file mode 100644 index ea7ff802556322aff7435cd9744e4ad84d6cfd79..0000000000000000000000000000000000000000 --- a/data/models/supichi_bbai_qween_v000000_lumen_14b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "BBAI_QWEEN_V000000_LUMEN_14B", - "id": "Supichi/BBAI_QWEEN_V000000_LUMEN_14B", - "developer": "Supichi", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "10.366" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Supichi_BBAI_QWEEN_V000000_LUMEN_14B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1815 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2297 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2315 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3445 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.116 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/supichi_bbaik29.json b/data/models/supichi_bbaik29.json deleted file mode 100644 index b2ec42480c5579fa4250f4bc9b8a5bda22db542a..0000000000000000000000000000000000000000 --- a/data/models/supichi_bbaik29.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "BBAIK29", - "id": "Supichi/BBAIK29", - "developer": "Supichi", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Supichi_BBAIK29/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4588 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.559 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3678 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3121 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4501 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4469 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/supichi_hf_token.json b/data/models/supichi_hf_token.json deleted file mode 100644 index 91293fc50789db221cbbccf6174e85846f59d484..0000000000000000000000000000000000000000 --- a/data/models/supichi_hf_token.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "HF_TOKEN", - "id": "Supichi/HF_TOKEN", - "developer": "Supichi", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "17.161" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Supichi_HF_TOKEN/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.138 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2764 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0008 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2634 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3272 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.111 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/supichi_njs26.json b/data/models/supichi_njs26.json deleted file mode 100644 index 7cc9f28575d61e29fa31badb8e497f3a463f87fc..0000000000000000000000000000000000000000 --- a/data/models/supichi_njs26.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "NJS26", - "id": "Supichi/NJS26", - "developer": "Supichi", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Supichi_NJS26/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0448 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.478 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0325 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.318 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3854 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3037 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/svak_mn-12b-inferor-v0.0.json b/data/models/svak_mn-12b-inferor-v0.0.json deleted file mode 100644 index c556d97149040e16b4254d36092c1f2b6c9e75c7..0000000000000000000000000000000000000000 --- a/data/models/svak_mn-12b-inferor-v0.0.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MN-12B-Inferor-v0.0", - "id": "Svak/MN-12B-Inferor-v0.0", - "developer": "Svak", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Svak_MN-12B-Inferor-v0.0/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5708 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5195 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.102 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3087 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4639 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3559 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/svak_mn-12b-inferor-v0.1.json b/data/models/svak_mn-12b-inferor-v0.1.json deleted file mode 100644 index 1a422e9a8240d88f28d3c1f52c876c35e2f46e28..0000000000000000000000000000000000000000 --- a/data/models/svak_mn-12b-inferor-v0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MN-12B-Inferor-v0.1", - "id": "Svak/MN-12B-Inferor-v0.1", - "developer": "Svak", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Svak_MN-12B-Inferor-v0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6347 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5147 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1261 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3255 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4351 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3662 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/swap-uniba_llamantino-3-anita-8b-inst-dpo-ita.json b/data/models/swap-uniba_llamantino-3-anita-8b-inst-dpo-ita.json deleted file mode 100644 index ad4a07a918a2b0a83346fea1c258b64f73a4c3a1..0000000000000000000000000000000000000000 --- a/data/models/swap-uniba_llamantino-3-anita-8b-inst-dpo-ita.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "LLaMAntino-3-ANITA-8B-Inst-DPO-ITA", - "id": "swap-uniba/LLaMAntino-3-ANITA-8B-Inst-DPO-ITA", - "developer": "swap-uniba", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/swap-uniba_LLaMAntino-3-ANITA-8B-Inst-DPO-ITA/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4815 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4936 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0483 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2987 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4387 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3723 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/syed-hasan-8503_phi-3-mini-4k-instruct-cpo-simpo.json b/data/models/syed-hasan-8503_phi-3-mini-4k-instruct-cpo-simpo.json deleted file mode 100644 index 31a3381233bdea6e9c4c46d65e2bc3f441d1ba55..0000000000000000000000000000000000000000 --- a/data/models/syed-hasan-8503_phi-3-mini-4k-instruct-cpo-simpo.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Phi-3-mini-4K-instruct-cpo-simpo", - "id": "Syed-Hasan-8503/Phi-3-mini-4K-instruct-cpo-simpo", - "developer": "Syed-Hasan-8503", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Phi3ForCausalLM", - "params_billions": "3.821" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Syed-Hasan-8503_Phi-3-mini-4K-instruct-cpo-simpo/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5714 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5682 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1571 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3305 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3964 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3861 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/synergetic_frankenqwen2.5-14b.json b/data/models/synergetic_frankenqwen2.5-14b.json deleted file mode 100644 index f8148a19e008508f1bfb79926946bef4abde1193..0000000000000000000000000000000000000000 --- a/data/models/synergetic_frankenqwen2.5-14b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "FrankenQwen2.5-14B", - "id": "synergetic/FrankenQwen2.5-14B", - "developer": "synergetic", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "16.972" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/synergetic_FrankenQwen2.5-14B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1869 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6048 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2701 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3843 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4382 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/t145_kronos-8b-v1-p1.json b/data/models/t145_kronos-8b-v1-p1.json deleted file mode 100644 index 4edb578b9f6ad918a27ac195b4fe78ea4411fe58..0000000000000000000000000000000000000000 --- a/data/models/t145_kronos-8b-v1-p1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "KRONOS-8B-V1-P1", - "id": "T145/KRONOS-8B-V1-P1", - "developer": "T145", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/T145_KRONOS-8B-V1-P1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.785 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5085 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1979 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2953 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3881 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.376 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/t145_kronos-8b-v1-p2.json b/data/models/t145_kronos-8b-v1-p2.json deleted file mode 100644 index ee991d7a532e6d6bc341c5b25d134539440583b0..0000000000000000000000000000000000000000 --- a/data/models/t145_kronos-8b-v1-p2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "KRONOS-8B-V1-P2", - "id": "T145/KRONOS-8B-V1-P2", - "developer": "T145", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/T145_KRONOS-8B-V1-P2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6724 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4772 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1601 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2919 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3568 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3453 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/t145_kronos-8b-v1-p3.json b/data/models/t145_kronos-8b-v1-p3.json deleted file mode 100644 index 08700995339456777c50f8bfee09ebdf57b4104f..0000000000000000000000000000000000000000 --- a/data/models/t145_kronos-8b-v1-p3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "KRONOS-8B-V1-P3", - "id": "T145/KRONOS-8B-V1-P3", - "developer": "T145", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/T145_KRONOS-8B-V1-P3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7137 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5128 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1926 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2601 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3616 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3405 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/t145_kronos-8b-v2.json b/data/models/t145_kronos-8b-v2.json deleted file mode 100644 index c872f2eb9ddb3e67f1677e30c5fedb1f566e7f31..0000000000000000000000000000000000000000 --- a/data/models/t145_kronos-8b-v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "KRONOS-8B-V2", - "id": "T145/KRONOS-8B-V2", - "developer": "T145", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/T145_KRONOS-8B-V2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.518 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5133 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2266 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2987 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3829 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3738 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/t145_kronos-8b-v3.json b/data/models/t145_kronos-8b-v3.json deleted file mode 100644 index 2d194160b407ac24e9d9196170395488668a20af..0000000000000000000000000000000000000000 --- a/data/models/t145_kronos-8b-v3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "KRONOS-8B-V3", - "id": "T145/KRONOS-8B-V3", - "developer": "T145", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/T145_KRONOS-8B-V3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5475 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5119 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2598 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2886 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3922 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3738 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/t145_kronos-8b-v4.json b/data/models/t145_kronos-8b-v4.json deleted file mode 100644 index 0e07430c9ee2897de80b8de136788af5cc4d3913..0000000000000000000000000000000000000000 --- a/data/models/t145_kronos-8b-v4.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "KRONOS-8B-V4", - "id": "T145/KRONOS-8B-V4", - "developer": "T145", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/T145_KRONOS-8B-V4/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7889 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5092 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1949 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2894 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.383 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3786 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/t145_kronos-8b-v5.json b/data/models/t145_kronos-8b-v5.json deleted file mode 100644 index 0b87e6d95757f3300689874ef9a5d39db8ce17db..0000000000000000000000000000000000000000 --- a/data/models/t145_kronos-8b-v5.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "KRONOS-8B-V5", - "id": "T145/KRONOS-8B-V5", - "developer": "T145", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/T145_KRONOS-8B-V5/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5405 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5089 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2689 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2903 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4055 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3759 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/t145_kronos-8b-v6.json b/data/models/t145_kronos-8b-v6.json deleted file mode 100644 index 31c9a268cc6e74bf636ad2ecad21160a762bd5cb..0000000000000000000000000000000000000000 --- a/data/models/t145_kronos-8b-v6.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "KRONOS-8B-V6", - "id": "T145/KRONOS-8B-V6", - "developer": "T145", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/T145_KRONOS-8B-V6/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7022 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5034 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2598 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2794 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4121 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3501 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/t145_kronos-8b-v7.json b/data/models/t145_kronos-8b-v7.json deleted file mode 100644 index 6852c1303c6ccb68abf33fc2de04045ccf34f0e4..0000000000000000000000000000000000000000 --- a/data/models/t145_kronos-8b-v7.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "KRONOS-8B-V7", - "id": "T145/KRONOS-8B-V7", - "developer": "T145", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "4.015" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/T145_KRONOS-8B-V7/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3529 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4526 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.111 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2668 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3671 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2697 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/t145_kronos-8b-v8.json b/data/models/t145_kronos-8b-v8.json deleted file mode 100644 index 1a334f631a44b3a6c56ea991e489c572d0d38c25..0000000000000000000000000000000000000000 --- a/data/models/t145_kronos-8b-v8.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "KRONOS-8B-V8", - "id": "T145/KRONOS-8B-V8", - "developer": "T145", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/T145_KRONOS-8B-V8/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.777 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5094 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2047 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2894 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3869 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3782 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/t145_kronos-8b-v9.json b/data/models/t145_kronos-8b-v9.json deleted file mode 100644 index 6c8b10c2b92dd1b8e94a7fee8882ba5d1d35723f..0000000000000000000000000000000000000000 --- a/data/models/t145_kronos-8b-v9.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "KRONOS-8B-V9", - "id": "T145/KRONOS-8B-V9", - "developer": "T145", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/T145_KRONOS-8B-V9/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7856 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5099 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1986 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2961 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3868 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3752 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/t145_llama-3.1-8b-instruct-zeus.json b/data/models/t145_llama-3.1-8b-instruct-zeus.json deleted file mode 100644 index c3b3da561d68692ae29d16e6e0e4f71402e54a1f..0000000000000000000000000000000000000000 --- a/data/models/t145_llama-3.1-8b-instruct-zeus.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.1-8B-Instruct-Zeus", - "id": "T145/Llama-3.1-8B-Instruct-Zeus", - "developer": "T145", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/T145_Llama-3.1-8B-Instruct-Zeus/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7941 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5174 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1956 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3012 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3976 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3893 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/t145_llama-3.1-8b-zeus.json b/data/models/t145_llama-3.1-8b-zeus.json deleted file mode 100644 index 68260887a1c8f4b865cbcd5aeba496c070fb4d32..0000000000000000000000000000000000000000 --- a/data/models/t145_llama-3.1-8b-zeus.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.1-8B-Zeus", - "id": "T145/Llama-3.1-8B-Zeus", - "developer": "T145", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/T145_Llama-3.1-8B-Zeus/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3518 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3671 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0144 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2651 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3316 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1332 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/t145_meta-llama-3.1-8b-instruct-ties.json b/data/models/t145_meta-llama-3.1-8b-instruct-ties.json deleted file mode 100644 index 80dd06d155059ec739505bdd7151bc21cbc9ce02..0000000000000000000000000000000000000000 --- a/data/models/t145_meta-llama-3.1-8b-instruct-ties.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Meta-Llama-3.1-8B-Instruct-TIES", - "id": "T145/Meta-Llama-3.1-8B-Instruct-TIES", - "developer": "T145", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/T145_Meta-Llama-3.1-8B-Instruct-TIES/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5424 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.507 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.21 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2945 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3843 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.378 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/t145_qwen-2.5-3b-merge-test.json b/data/models/t145_qwen-2.5-3b-merge-test.json deleted file mode 100644 index 9f9e0c743cbd85aacf8823e0e07c74e77ec1f2dd..0000000000000000000000000000000000000000 --- a/data/models/t145_qwen-2.5-3b-merge-test.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "qwen-2.5-3B-merge-test", - "id": "T145/qwen-2.5-3B-merge-test", - "developer": "T145", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.397" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/T145_qwen-2.5-3B-merge-test/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5751 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4842 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3202 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2852 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4007 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.329 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/t145_zeus-8b-v10.json b/data/models/t145_zeus-8b-v10.json deleted file mode 100644 index ace695fb253b074bc387ecf65bb1c72dbeb403b7..0000000000000000000000000000000000000000 --- a/data/models/t145_zeus-8b-v10.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ZEUS-8B-V10", - "id": "T145/ZEUS-8B-V10", - "developer": "T145", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/T145_ZEUS-8B-V10/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7707 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.527 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2115 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3247 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3898 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3904 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/t145_zeus-8b-v11.json b/data/models/t145_zeus-8b-v11.json deleted file mode 100644 index 0994bcff4e01e45cc55e3a5c17a52d283f7bdfec..0000000000000000000000000000000000000000 --- a/data/models/t145_zeus-8b-v11.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ZEUS-8B-V11", - "id": "T145/ZEUS-8B-V11", - "developer": "T145", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/T145_ZEUS-8B-V11/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.81 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5162 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1964 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3146 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3807 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3884 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/t145_zeus-8b-v12.json b/data/models/t145_zeus-8b-v12.json deleted file mode 100644 index 93c8d5945e41c4939afb07af8ad15874973416ff..0000000000000000000000000000000000000000 --- a/data/models/t145_zeus-8b-v12.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ZEUS-8B-V12", - "id": "T145/ZEUS-8B-V12", - "developer": "T145", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/T145_ZEUS-8B-V12/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7816 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5254 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2115 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3205 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3858 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3912 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/t145_zeus-8b-v13-abliterated.json b/data/models/t145_zeus-8b-v13-abliterated.json deleted file mode 100644 index 762419137602038d5279da056abc8fec42eb9a03..0000000000000000000000000000000000000000 --- a/data/models/t145_zeus-8b-v13-abliterated.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ZEUS-8B-V13-abliterated", - "id": "T145/ZEUS-8B-V13-abliterated", - "developer": "T145", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/T145_ZEUS-8B-V13-abliterated/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7878 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5198 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.179 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3112 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3871 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3872 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/t145_zeus-8b-v13.json b/data/models/t145_zeus-8b-v13.json deleted file mode 100644 index 6dec22a021fdb2ef721874aae01f046ba9b14488..0000000000000000000000000000000000000000 --- a/data/models/t145_zeus-8b-v13.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ZEUS-8B-V13", - "id": "T145/ZEUS-8B-V13", - "developer": "T145", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/T145_ZEUS-8B-V13/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7904 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5277 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2137 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3238 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3845 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3911 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/t145_zeus-8b-v14.json b/data/models/t145_zeus-8b-v14.json deleted file mode 100644 index 92e87258972a65d5788e760c60e9c3350c7a8bed..0000000000000000000000000000000000000000 --- a/data/models/t145_zeus-8b-v14.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ZEUS-8B-V14", - "id": "T145/ZEUS-8B-V14", - "developer": "T145", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/T145_ZEUS-8B-V14/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7709 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5275 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.213 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3205 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3844 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3914 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/t145_zeus-8b-v15.json b/data/models/t145_zeus-8b-v15.json deleted file mode 100644 index 3ff3f06d41c3f5585452cbaab31b0187fb2fec7f..0000000000000000000000000000000000000000 --- a/data/models/t145_zeus-8b-v15.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ZEUS-8B-V15", - "id": "T145/ZEUS-8B-V15", - "developer": "T145", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "4.015" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/T145_ZEUS-8B-V15/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7013 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5538 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2304 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.276 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.402 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4059 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/t145_zeus-8b-v16.json b/data/models/t145_zeus-8b-v16.json deleted file mode 100644 index 766a6a1681203fe3e59aac7e1804a46ba83fb8cb..0000000000000000000000000000000000000000 --- a/data/models/t145_zeus-8b-v16.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ZEUS-8B-V16", - "id": "T145/ZEUS-8B-V16", - "developer": "T145", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/T145_ZEUS-8B-V16/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7925 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5266 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2205 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.307 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3951 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3926 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/t145_zeus-8b-v17-abliterated-v2.json b/data/models/t145_zeus-8b-v17-abliterated-v2.json deleted file mode 100644 index 060f224b331e05d10a54527f4f87c4fa8773b9f5..0000000000000000000000000000000000000000 --- a/data/models/t145_zeus-8b-v17-abliterated-v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ZEUS-8B-V17-abliterated-V2", - "id": "T145/ZEUS-8B-V17-abliterated-V2", - "developer": "T145", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/T145_ZEUS-8B-V17-abliterated-V2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6532 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4928 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1118 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2735 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3407 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3402 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/t145_zeus-8b-v17-abliterated-v4.json b/data/models/t145_zeus-8b-v17-abliterated-v4.json deleted file mode 100644 index 447dab4892219b75e80c11cf625a45ed21603f1f..0000000000000000000000000000000000000000 --- a/data/models/t145_zeus-8b-v17-abliterated-v4.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ZEUS-8B-V17-abliterated-V4", - "id": "T145/ZEUS-8B-V17-abliterated-V4", - "developer": "T145", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/T145_ZEUS-8B-V17-abliterated-V4/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7228 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5169 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0937 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2836 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4187 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3774 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/t145_zeus-8b-v17-abliterated.json b/data/models/t145_zeus-8b-v17-abliterated.json deleted file mode 100644 index 523b9dc753cce4aa546a1764a142c7c805cdc00f..0000000000000000000000000000000000000000 --- a/data/models/t145_zeus-8b-v17-abliterated.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ZEUS-8B-V17-abliterated", - "id": "T145/ZEUS-8B-V17-abliterated", - "developer": "T145", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "7.594" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/T145_ZEUS-8B-V17-abliterated/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7576 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.52 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0438 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3037 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4269 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3622 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/t145_zeus-8b-v17.json b/data/models/t145_zeus-8b-v17.json deleted file mode 100644 index 1aab8762c212462bda086734b660e439c4423f6e..0000000000000000000000000000000000000000 --- a/data/models/t145_zeus-8b-v17.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ZEUS-8B-V17", - "id": "T145/ZEUS-8B-V17", - "developer": "T145", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/T145_ZEUS-8B-V17/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7941 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5251 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2243 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3221 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4016 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3935 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/t145_zeus-8b-v18.json b/data/models/t145_zeus-8b-v18.json deleted file mode 100644 index 12d5cf3c74c163c49fd69368ed8a6927903b5ed9..0000000000000000000000000000000000000000 --- a/data/models/t145_zeus-8b-v18.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ZEUS-8B-V18", - "id": "T145/ZEUS-8B-V18", - "developer": "T145", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/T145_ZEUS-8B-V18/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7834 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.527 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2183 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3213 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4043 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3942 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/t145_zeus-8b-v19.json b/data/models/t145_zeus-8b-v19.json deleted file mode 100644 index 6694d042df65c2845b35a7b8447710b004b92d4f..0000000000000000000000000000000000000000 --- a/data/models/t145_zeus-8b-v19.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ZEUS-8B-V19", - "id": "T145/ZEUS-8B-V19", - "developer": "T145", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/T145_ZEUS-8B-V19/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7883 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5276 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2205 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3221 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4043 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3934 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/t145_zeus-8b-v2-abliterated.json b/data/models/t145_zeus-8b-v2-abliterated.json deleted file mode 100644 index a380ea375ff38ebdfae454b336cfedff16189fca..0000000000000000000000000000000000000000 --- a/data/models/t145_zeus-8b-v2-abliterated.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ZEUS-8B-V2-abliterated", - "id": "T145/ZEUS-8B-V2-abliterated", - "developer": "T145", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/T145_ZEUS-8B-V2-abliterated/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7895 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5129 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2115 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3129 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3911 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3825 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/t145_zeus-8b-v2-orpo.json b/data/models/t145_zeus-8b-v2-orpo.json deleted file mode 100644 index 068bb59d650e892ca4f2df4a1ba92cd881091cc9..0000000000000000000000000000000000000000 --- a/data/models/t145_zeus-8b-v2-orpo.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ZEUS-8B-V2-ORPO", - "id": "T145/ZEUS-8B-V2-ORPO", - "developer": "T145", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "4.015" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/T145_ZEUS-8B-V2-ORPO/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7187 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5075 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1828 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3104 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3935 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3678 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/t145_zeus-8b-v2.json b/data/models/t145_zeus-8b-v2.json deleted file mode 100644 index 4292c0faeed13cd318a124be171c3059f9ce240b..0000000000000000000000000000000000000000 --- a/data/models/t145_zeus-8b-v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ZEUS-8B-V2", - "id": "T145/ZEUS-8B-V2", - "developer": "T145", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/T145_ZEUS-8B-V2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8029 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5194 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.216 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.302 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.391 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3896 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/t145_zeus-8b-v20.json b/data/models/t145_zeus-8b-v20.json deleted file mode 100644 index 8d46582aab5016d8fb6238540d60d410957925d1..0000000000000000000000000000000000000000 --- a/data/models/t145_zeus-8b-v20.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ZEUS-8B-V20", - "id": "T145/ZEUS-8B-V20", - "developer": "T145", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/T145_ZEUS-8B-V20/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7956 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5244 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.219 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.323 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4043 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.393 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/t145_zeus-8b-v21.json b/data/models/t145_zeus-8b-v21.json deleted file mode 100644 index 901aead94ebc68c244f61c509703c99127f345fd..0000000000000000000000000000000000000000 --- a/data/models/t145_zeus-8b-v21.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ZEUS-8B-V21", - "id": "T145/ZEUS-8B-V21", - "developer": "T145", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/T145_ZEUS-8B-V21/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3785 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3398 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1594 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2643 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3262 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1714 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/t145_zeus-8b-v22.json b/data/models/t145_zeus-8b-v22.json deleted file mode 100644 index 32f6caeca5ef729c1c0095eeede266d63f008757..0000000000000000000000000000000000000000 --- a/data/models/t145_zeus-8b-v22.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ZEUS-8B-V22", - "id": "T145/ZEUS-8B-V22", - "developer": "T145", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/T145_ZEUS-8B-V22/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7995 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5245 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2228 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.328 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.399 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3938 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/t145_zeus-8b-v23.json b/data/models/t145_zeus-8b-v23.json deleted file mode 100644 index de0dc875af20646547793a829244a59ae3e8f2bd..0000000000000000000000000000000000000000 --- a/data/models/t145_zeus-8b-v23.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ZEUS-8B-V23", - "id": "T145/ZEUS-8B-V23", - "developer": "T145", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/T145_ZEUS-8B-V23/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7621 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5195 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.182 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3096 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3922 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3666 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/t145_zeus-8b-v24.json b/data/models/t145_zeus-8b-v24.json deleted file mode 100644 index cf701ce2d05e7ee2ac22f08b62517ee7342fcab8..0000000000000000000000000000000000000000 --- a/data/models/t145_zeus-8b-v24.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ZEUS-8B-V24", - "id": "T145/ZEUS-8B-V24", - "developer": "T145", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/T145_ZEUS-8B-V24/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4778 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1458 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2617 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3729 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3285 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/t145_zeus-8b-v25.json b/data/models/t145_zeus-8b-v25.json deleted file mode 100644 index 24eaafa7e3659002e8079f95a708f52e52eb564a..0000000000000000000000000000000000000000 --- a/data/models/t145_zeus-8b-v25.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ZEUS-8B-V25", - "id": "T145/ZEUS-8B-V25", - "developer": "T145", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/T145_ZEUS-8B-V25/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.332 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4547 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2039 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2643 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3488 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2885 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/t145_zeus-8b-v26.json b/data/models/t145_zeus-8b-v26.json deleted file mode 100644 index 968c64fe20e978c7c50b5e2595d7d198a98942e0..0000000000000000000000000000000000000000 --- a/data/models/t145_zeus-8b-v26.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ZEUS-8B-V26", - "id": "T145/ZEUS-8B-V26", - "developer": "T145", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/T145_ZEUS-8B-V26/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6708 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5232 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1246 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2953 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4016 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3907 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/t145_zeus-8b-v27.json b/data/models/t145_zeus-8b-v27.json deleted file mode 100644 index 1dfae1f239e49f455f4ab94f5e8b3b75d86a5398..0000000000000000000000000000000000000000 --- a/data/models/t145_zeus-8b-v27.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ZEUS-8B-V27", - "id": "T145/ZEUS-8B-V27", - "developer": "T145", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/T145_ZEUS-8B-V27/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6544 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.523 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1344 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3079 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3977 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3902 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/t145_zeus-8b-v28.json b/data/models/t145_zeus-8b-v28.json deleted file mode 100644 index e6428befff1b7bb2c0e07e31d9bd9ab5728efc21..0000000000000000000000000000000000000000 --- a/data/models/t145_zeus-8b-v28.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ZEUS-8B-V28", - "id": "T145/ZEUS-8B-V28", - "developer": "T145", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/T145_ZEUS-8B-V28/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6353 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5254 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1269 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3037 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3896 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3902 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/t145_zeus-8b-v29.json b/data/models/t145_zeus-8b-v29.json deleted file mode 100644 index 574937a1bd8c45e30b48acf630dc7dccfb05349d..0000000000000000000000000000000000000000 --- a/data/models/t145_zeus-8b-v29.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ZEUS-8B-V29", - "id": "T145/ZEUS-8B-V29", - "developer": "T145", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/T145_ZEUS-8B-V29/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7418 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5253 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1601 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3263 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4003 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.392 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/t145_zeus-8b-v2l1.json b/data/models/t145_zeus-8b-v2l1.json deleted file mode 100644 index 72fcd9e9e1e95094f41b4c392231a07c09925d4f..0000000000000000000000000000000000000000 --- a/data/models/t145_zeus-8b-v2l1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ZEUS-8B-V2L1", - "id": "T145/ZEUS-8B-V2L1", - "developer": "T145", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/T145_ZEUS-8B-V2L1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3192 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5013 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1239 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3129 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3882 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3638 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/t145_zeus-8b-v2l2.json b/data/models/t145_zeus-8b-v2l2.json deleted file mode 100644 index e7060d90ac5abb164544859b90b066b0fbc9b027..0000000000000000000000000000000000000000 --- a/data/models/t145_zeus-8b-v2l2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ZEUS-8B-V2L2", - "id": "T145/ZEUS-8B-V2L2", - "developer": "T145", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/T145_ZEUS-8B-V2L2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8021 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5203 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2017 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2995 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3975 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3884 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/t145_zeus-8b-v3.json b/data/models/t145_zeus-8b-v3.json deleted file mode 100644 index 15fe6cf2a076c7e65d5542e0c6a4ad77a7828b4b..0000000000000000000000000000000000000000 --- a/data/models/t145_zeus-8b-v3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ZEUS-8B-V3", - "id": "T145/ZEUS-8B-V3", - "developer": "T145", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/T145_ZEUS-8B-V3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7887 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5265 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1677 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3221 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4017 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3804 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/t145_zeus-8b-v30.json b/data/models/t145_zeus-8b-v30.json deleted file mode 100644 index 12a71906d238ac758d78d7e9b9b0ff5d3a5ad424..0000000000000000000000000000000000000000 --- a/data/models/t145_zeus-8b-v30.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ZEUS-8B-V30", - "id": "T145/ZEUS-8B-V30", - "developer": "T145", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/T145_ZEUS-8B-V30/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7436 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5243 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1586 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3205 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4029 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3944 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/t145_zeus-8b-v4.json b/data/models/t145_zeus-8b-v4.json deleted file mode 100644 index 9601aa93753f7f60bcafe11ad3ea450d7257a5fa..0000000000000000000000000000000000000000 --- a/data/models/t145_zeus-8b-v4.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ZEUS-8B-V4", - "id": "T145/ZEUS-8B-V4", - "developer": "T145", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/T145_ZEUS-8B-V4/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7807 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5246 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1926 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.307 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4029 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3788 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/t145_zeus-8b-v6.json b/data/models/t145_zeus-8b-v6.json deleted file mode 100644 index ef6b33ee16e46a1b6d537dd2fccc65c2634e0d12..0000000000000000000000000000000000000000 --- a/data/models/t145_zeus-8b-v6.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ZEUS-8B-V6", - "id": "T145/ZEUS-8B-V6", - "developer": "T145", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/T145_ZEUS-8B-V6/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7838 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.524 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2024 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3045 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4068 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3759 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/t145_zeus-8b-v7.json b/data/models/t145_zeus-8b-v7.json deleted file mode 100644 index a2eebb242bdc86066d40d0bf1837d35a2b13381c..0000000000000000000000000000000000000000 --- a/data/models/t145_zeus-8b-v7.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ZEUS-8B-V7", - "id": "T145/ZEUS-8B-V7", - "developer": "T145", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/T145_ZEUS-8B-V7/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7786 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.507 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.148 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.297 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4162 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3812 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/t145_zeus-8b-v8.json b/data/models/t145_zeus-8b-v8.json deleted file mode 100644 index d4e70b2649e7b19ef871c2d6b45bdce9560b9948..0000000000000000000000000000000000000000 --- a/data/models/t145_zeus-8b-v8.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ZEUS-8B-V8", - "id": "T145/ZEUS-8B-V8", - "developer": "T145", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/T145_ZEUS-8B-V8/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7914 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5065 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1329 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2878 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4214 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3761 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/t145_zeus-8b-v9.json b/data/models/t145_zeus-8b-v9.json deleted file mode 100644 index 7d898398a5ee6dd213a46f1d28a79063d29f88c2..0000000000000000000000000000000000000000 --- a/data/models/t145_zeus-8b-v9.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ZEUS-8B-V9", - "id": "T145/ZEUS-8B-V9", - "developer": "T145", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/T145_ZEUS-8B-V9/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5551 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5207 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2137 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2911 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3949 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3901 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/talha2001_beast-soul-new.json b/data/models/talha2001_beast-soul-new.json deleted file mode 100644 index b9fa24504ec9a88d5d4ddb089a7c288f738aa31d..0000000000000000000000000000000000000000 --- a/data/models/talha2001_beast-soul-new.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Beast-Soul-new", - "id": "talha2001/Beast-Soul-new", - "developer": "talha2001", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/talha2001_Beast-Soul-new/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4854 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5227 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.074 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2819 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4459 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3102 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/tangledgroup_tangled-llama-pints-1.5b-v0.1-instruct.json b/data/models/tangledgroup_tangled-llama-pints-1.5b-v0.1-instruct.json deleted file mode 100644 index 8df51c6d89d4ef73019a8e4aa7b4332c5dc715c5..0000000000000000000000000000000000000000 --- a/data/models/tangledgroup_tangled-llama-pints-1.5b-v0.1-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "tangled-llama-pints-1.5b-v0.1-instruct", - "id": "tangledgroup/tangled-llama-pints-1.5b-v0.1-instruct", - "developer": "tangledgroup", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.5" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/tangledgroup_tangled-llama-pints-1.5b-v0.1-instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1509 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3143 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0121 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2399 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3761 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1109 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/tangledgroup_tangled-llama-pints-1.5b-v0.2-instruct.json b/data/models/tangledgroup_tangled-llama-pints-1.5b-v0.2-instruct.json deleted file mode 100644 index 1adbf67a05ea0f672986487dccb011d09927f5a7..0000000000000000000000000000000000000000 --- a/data/models/tangledgroup_tangled-llama-pints-1.5b-v0.2-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "tangled-llama-pints-1.5b-v0.2-instruct", - "id": "tangledgroup/tangled-llama-pints-1.5b-v0.2-instruct", - "developer": "tangledgroup", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.5" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/tangledgroup_tangled-llama-pints-1.5b-v0.2-instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1724 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3158 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0128 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2416 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3643 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1117 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/tanliboy_lambda-gemma-2-9b-dpo.json b/data/models/tanliboy_lambda-gemma-2-9b-dpo.json deleted file mode 100644 index 7fb384620bce09b34b0d31ff763d2b1e971d0558..0000000000000000000000000000000000000000 --- a/data/models/tanliboy_lambda-gemma-2-9b-dpo.json +++ /dev/null @@ -1,275 +0,0 @@ -{ - "model_info": { - "name": "lambda-gemma-2-9b-dpo", - "id": "tanliboy/lambda-gemma-2-9b-dpo", - "developer": "tanliboy", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "9.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/tanliboy_lambda-gemma-2-9b-dpo/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1829 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5488 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3104 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4056 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3805 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "hfopenllm_v2/tanliboy_lambda-gemma-2-9b-dpo/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4501 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5472 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0944 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3138 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4017 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3792 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/tanliboy_lambda-qwen2.5-14b-dpo-test.json b/data/models/tanliboy_lambda-qwen2.5-14b-dpo-test.json deleted file mode 100644 index 8cf7cb6cecc62bc2e47a9955e1134f84bbc98f79..0000000000000000000000000000000000000000 --- a/data/models/tanliboy_lambda-qwen2.5-14b-dpo-test.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "lambda-qwen2.5-14b-dpo-test", - "id": "tanliboy/lambda-qwen2.5-14b-dpo-test", - "developer": "tanliboy", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/tanliboy_lambda-qwen2.5-14b-dpo-test/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8231 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6394 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5461 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3624 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.426 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4848 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/tanliboy_lambda-qwen2.5-32b-dpo-test.json b/data/models/tanliboy_lambda-qwen2.5-32b-dpo-test.json deleted file mode 100644 index 2bdab3549796ff199f542b8e11a927d20b107b42..0000000000000000000000000000000000000000 --- a/data/models/tanliboy_lambda-qwen2.5-32b-dpo-test.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "lambda-qwen2.5-32b-dpo-test", - "id": "tanliboy/lambda-qwen2.5-32b-dpo-test", - "developer": "tanliboy", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "32.764" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/tanliboy_lambda-qwen2.5-32b-dpo-test/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8084 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6764 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6103 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3565 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4274 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5657 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/tannedbum_ellaria-9b.json b/data/models/tannedbum_ellaria-9b.json deleted file mode 100644 index 66de4c1f6ed0c3f30b5aa5a0d5109682994d106a..0000000000000000000000000000000000000000 --- a/data/models/tannedbum_ellaria-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Ellaria-9B", - "id": "tannedbum/Ellaria-9B", - "developer": "tannedbum", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/tannedbum_Ellaria-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7826 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5942 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2077 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3331 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4151 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4205 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/tannedbum_l3-nymeria-maid-8b.json b/data/models/tannedbum_l3-nymeria-maid-8b.json deleted file mode 100644 index 669a77aaf4c09ed4df1c5d61e1ab696f7d2bdc3e..0000000000000000000000000000000000000000 --- a/data/models/tannedbum_l3-nymeria-maid-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "L3-Nymeria-Maid-8B", - "id": "tannedbum/L3-Nymeria-Maid-8B", - "developer": "tannedbum", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/tannedbum_L3-Nymeria-Maid-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.725 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5146 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0937 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2961 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3751 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3747 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/tannedbum_l3-nymeria-v2-8b.json b/data/models/tannedbum_l3-nymeria-v2-8b.json deleted file mode 100644 index dea79784d6278575c31509b3a30c03950162ca04..0000000000000000000000000000000000000000 --- a/data/models/tannedbum_l3-nymeria-v2-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "L3-Nymeria-v2-8B", - "id": "tannedbum/L3-Nymeria-v2-8B", - "developer": "tannedbum", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/tannedbum_L3-Nymeria-v2-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7168 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5224 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0921 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2903 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3699 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3753 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/tannedbum_l3-rhaenys-8b.json b/data/models/tannedbum_l3-rhaenys-8b.json deleted file mode 100644 index da00b40480ed319f29b507eca06254fb93c728db..0000000000000000000000000000000000000000 --- a/data/models/tannedbum_l3-rhaenys-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "L3-Rhaenys-8B", - "id": "tannedbum/L3-Rhaenys-8B", - "developer": "tannedbum", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/tannedbum_L3-Rhaenys-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7363 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5299 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0876 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2978 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3725 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3799 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/tarek07_progenitor-v1.1-llama-70b.json b/data/models/tarek07_progenitor-v1.1-llama-70b.json deleted file mode 100644 index a04488b9fc2467c0fb0c10fdb7b06f550c032a49..0000000000000000000000000000000000000000 --- a/data/models/tarek07_progenitor-v1.1-llama-70b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Progenitor-V1.1-LLaMa-70B", - "id": "Tarek07/Progenitor-V1.1-LLaMa-70B", - "developer": "Tarek07", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "70.554" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Tarek07_Progenitor-V1.1-LLaMa-70B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6906 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6971 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3573 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4581 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4736 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5465 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/tarek07_thalassic-alpha-llama-70b.json b/data/models/tarek07_thalassic-alpha-llama-70b.json deleted file mode 100644 index cf97433da380649452e0f6eb91639a8c0060a0eb..0000000000000000000000000000000000000000 --- a/data/models/tarek07_thalassic-alpha-llama-70b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Thalassic-Alpha-LLaMa-70B", - "id": "Tarek07/Thalassic-Alpha-LLaMa-70B", - "developer": "Tarek07", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "70.554" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Tarek07_Thalassic-Alpha-LLaMa-70B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7003 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.694 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.315 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4438 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4802 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5435 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/teezee_doublebagel-57b-v1.0.json b/data/models/teezee_doublebagel-57b-v1.0.json deleted file mode 100644 index 74c8dfdbd6995bf415930a8f8dc9b3a7abda481a..0000000000000000000000000000000000000000 --- a/data/models/teezee_doublebagel-57b-v1.0.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "DoubleBagel-57B-v1.0", - "id": "TeeZee/DoubleBagel-57B-v1.0", - "developer": "TeeZee", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "56.703" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/TeeZee_DoubleBagel-57B-v1.0/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2336 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3251 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0098 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.276 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4315 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1478 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/teknium_collectivecognition-v1.1-mistral-7b.json b/data/models/teknium_collectivecognition-v1.1-mistral-7b.json deleted file mode 100644 index 54526952fe0e6f6a48fb9f0b7b83b10e9277c66c..0000000000000000000000000000000000000000 --- a/data/models/teknium_collectivecognition-v1.1-mistral-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "CollectiveCognition-v1.1-Mistral-7B", - "id": "teknium/CollectiveCognition-v1.1-Mistral-7B", - "developer": "teknium", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "7.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/teknium_CollectiveCognition-v1.1-Mistral-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.279 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4493 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.031 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2869 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3869 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2837 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/teknium_openhermes-13b.json b/data/models/teknium_openhermes-13b.json deleted file mode 100644 index 0cd6794f7c22a2ebe28348c32f621f7055dd20dc..0000000000000000000000000000000000000000 --- a/data/models/teknium_openhermes-13b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "OpenHermes-13B", - "id": "teknium/OpenHermes-13B", - "developer": "teknium", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "13.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/teknium_OpenHermes-13B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2668 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4206 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0121 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2727 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4043 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2389 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/teknium_openhermes-2-mistral-7b.json b/data/models/teknium_openhermes-2-mistral-7b.json deleted file mode 100644 index 5c4c2c5bc70abc7aa1246cf30eecbb0b11350278..0000000000000000000000000000000000000000 --- a/data/models/teknium_openhermes-2-mistral-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "OpenHermes-2-Mistral-7B", - "id": "teknium/OpenHermes-2-Mistral-7B", - "developer": "teknium", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/teknium_OpenHermes-2-Mistral-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5286 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4948 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0453 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2836 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.452 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2931 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/teknium_openhermes-2.5-mistral-7b.json b/data/models/teknium_openhermes-2.5-mistral-7b.json deleted file mode 100644 index 4fb3ae3f01007a601245fe962faeef5e7bf8606a..0000000000000000000000000000000000000000 --- a/data/models/teknium_openhermes-2.5-mistral-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "OpenHermes-2.5-Mistral-7B", - "id": "teknium/OpenHermes-2.5-Mistral-7B", - "developer": "teknium", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/teknium_OpenHermes-2.5-Mistral-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5571 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.487 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0506 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2836 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4242 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3054 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/teknium_openhermes-7b.json b/data/models/teknium_openhermes-7b.json deleted file mode 100644 index cd6e66b147bf8602249f082c02c3c1c0affed24a..0000000000000000000000000000000000000000 --- a/data/models/teknium_openhermes-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "OpenHermes-7B", - "id": "teknium/OpenHermes-7B", - "developer": "teknium", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "7.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/teknium_OpenHermes-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1813 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.362 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0159 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2693 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4324 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1933 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/telugu-llm-labs_indic-gemma-2b-finetuned-sft-navarasa-2.0.json b/data/models/telugu-llm-labs_indic-gemma-2b-finetuned-sft-navarasa-2.0.json deleted file mode 100644 index 5e89bf643e62b3255b8e94bcc36780ea84fb89e8..0000000000000000000000000000000000000000 --- a/data/models/telugu-llm-labs_indic-gemma-2b-finetuned-sft-navarasa-2.0.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Indic-gemma-2b-finetuned-sft-Navarasa-2.0", - "id": "Telugu-LLM-Labs/Indic-gemma-2b-finetuned-sft-Navarasa-2.0", - "developer": "Telugu-LLM-Labs", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "GemmaForCausalLM", - "params_billions": "2.506" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Telugu-LLM-Labs_Indic-gemma-2b-finetuned-sft-Navarasa-2.0/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2103 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3241 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0272 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2433 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3899 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1279 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/telugu-llm-labs_indic-gemma-7b-finetuned-sft-navarasa-2.0.json b/data/models/telugu-llm-labs_indic-gemma-7b-finetuned-sft-navarasa-2.0.json deleted file mode 100644 index 70e047afec47e6438387db4fe651cbe1e1d436eb..0000000000000000000000000000000000000000 --- a/data/models/telugu-llm-labs_indic-gemma-7b-finetuned-sft-navarasa-2.0.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Indic-gemma-7b-finetuned-sft-Navarasa-2.0", - "id": "Telugu-LLM-Labs/Indic-gemma-7b-finetuned-sft-Navarasa-2.0", - "developer": "Telugu-LLM-Labs", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "GemmaForCausalLM", - "params_billions": "8.538" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Telugu-LLM-Labs_Indic-gemma-7b-finetuned-sft-Navarasa-2.0/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3237 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4023 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0257 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2701 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4083 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.235 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/tencentarc_llama-pro-8b-instruct.json b/data/models/tencentarc_llama-pro-8b-instruct.json deleted file mode 100644 index 3cebf8ceb1544d91d358343909b1a044b3f077c5..0000000000000000000000000000000000000000 --- a/data/models/tencentarc_llama-pro-8b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "LLaMA-Pro-8B-Instruct", - "id": "TencentARC/LLaMA-Pro-8B-Instruct", - "developer": "TencentARC", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.357" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/TencentARC_LLaMA-Pro-8B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4486 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4224 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0249 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2743 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.419 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1946 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/tencentarc_llama-pro-8b.json b/data/models/tencentarc_llama-pro-8b.json deleted file mode 100644 index 3beb0e0bb9c9b77a2a42d32a9ae5a7f764a5dcda..0000000000000000000000000000000000000000 --- a/data/models/tencentarc_llama-pro-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "LLaMA-Pro-8B", - "id": "TencentARC/LLaMA-Pro-8B", - "developer": "TencentARC", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.357" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/TencentARC_LLaMA-Pro-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2277 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3484 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0189 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2601 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4018 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1811 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/tencentarc_metamath-mistral-pro.json b/data/models/tencentarc_metamath-mistral-pro.json deleted file mode 100644 index ac3e06af7769a67af16901c91d27b90ce3640b30..0000000000000000000000000000000000000000 --- a/data/models/tencentarc_metamath-mistral-pro.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MetaMath-Mistral-Pro", - "id": "TencentARC/MetaMath-Mistral-Pro", - "developer": "TencentARC", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "8.987" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/TencentARC_MetaMath-Mistral-Pro/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2119 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4413 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0763 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2693 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3524 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2472 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/tencentarc_mistral_pro_8b_v0.1.json b/data/models/tencentarc_mistral_pro_8b_v0.1.json deleted file mode 100644 index e0c3f3ae1ce716e2892922c82d1909efeff2f25f..0000000000000000000000000000000000000000 --- a/data/models/tencentarc_mistral_pro_8b_v0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mistral_Pro_8B_v0.1", - "id": "TencentARC/Mistral_Pro_8B_v0.1", - "developer": "TencentARC", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "8.987" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/TencentARC_Mistral_Pro_8B_v0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2115 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4526 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0566 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2802 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4242 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2765 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/tensopolis_falcon3-10b-tensopolis-v1.json b/data/models/tensopolis_falcon3-10b-tensopolis-v1.json deleted file mode 100644 index 62c17196ea0195d9958763978775124f4b513816..0000000000000000000000000000000000000000 --- a/data/models/tensopolis_falcon3-10b-tensopolis-v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "falcon3-10b-tensopolis-v1", - "id": "tensopolis/falcon3-10b-tensopolis-v1", - "developer": "tensopolis", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "10.306" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/tensopolis_falcon3-10b-tensopolis-v1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7817 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6182 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2749 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3297 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4375 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.442 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/tensopolis_falcon3-10b-tensopolis-v2.json b/data/models/tensopolis_falcon3-10b-tensopolis-v2.json deleted file mode 100644 index 3c137ee2aa6087735981198ab7734980a1de1590..0000000000000000000000000000000000000000 --- a/data/models/tensopolis_falcon3-10b-tensopolis-v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "falcon3-10b-tensopolis-v2", - "id": "tensopolis/falcon3-10b-tensopolis-v2", - "developer": "tensopolis", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "10.306" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/tensopolis_falcon3-10b-tensopolis-v2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7792 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6182 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2666 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3272 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4297 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4424 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/tensopolis_lamarckvergence-14b-tensopolis-v1.json b/data/models/tensopolis_lamarckvergence-14b-tensopolis-v1.json deleted file mode 100644 index 960f844d3302e29e8efc886d0be20a95659c0492..0000000000000000000000000000000000000000 --- a/data/models/tensopolis_lamarckvergence-14b-tensopolis-v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "lamarckvergence-14b-tensopolis-v1", - "id": "tensopolis/lamarckvergence-14b-tensopolis-v1", - "developer": "tensopolis", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/tensopolis_lamarckvergence-14b-tensopolis-v1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7604 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6561 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5166 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3607 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4475 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.525 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/tensopolis_mistral-small-2501-tensopolis-v1.json b/data/models/tensopolis_mistral-small-2501-tensopolis-v1.json deleted file mode 100644 index ce83c450049f68a47550cf924ee081309bbc6eb6..0000000000000000000000000000000000000000 --- a/data/models/tensopolis_mistral-small-2501-tensopolis-v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "mistral-small-2501-tensopolis-v1", - "id": "tensopolis/mistral-small-2501-tensopolis-v1", - "developer": "tensopolis", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "23.572" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/tensopolis_mistral-small-2501-tensopolis-v1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7762 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6475 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4441 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3574 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.428 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4465 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/tensopolis_mistral-small-r1-tensopolis.json b/data/models/tensopolis_mistral-small-r1-tensopolis.json deleted file mode 100644 index 6492e11054cbdb35464a23f1019304f0cd1b2f4f..0000000000000000000000000000000000000000 --- a/data/models/tensopolis_mistral-small-r1-tensopolis.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "mistral-small-r1-tensopolis", - "id": "tensopolis/mistral-small-r1-tensopolis", - "developer": "tensopolis", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "23.572" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/tensopolis_mistral-small-r1-tensopolis/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4622 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5436 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2908 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2819 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3738 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4035 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/tensopolis_phi-4-tensopolis-v1.json b/data/models/tensopolis_phi-4-tensopolis-v1.json deleted file mode 100644 index 4b61f4d7741930b7cd78d995b293d2930b46f481..0000000000000000000000000000000000000000 --- a/data/models/tensopolis_phi-4-tensopolis-v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "phi-4-tensopolis-v1", - "id": "tensopolis/phi-4-tensopolis-v1", - "developer": "tensopolis", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/tensopolis_phi-4-tensopolis-v1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6767 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6872 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.494 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3347 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4141 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5384 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/tensopolis_qwen2.5-14b-tensopolis-v1.json b/data/models/tensopolis_qwen2.5-14b-tensopolis-v1.json deleted file mode 100644 index f84ae3b9b9acda8f75bd0db018dc1726b130ebb4..0000000000000000000000000000000000000000 --- a/data/models/tensopolis_qwen2.5-14b-tensopolis-v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "qwen2.5-14b-tensopolis-v1", - "id": "tensopolis/qwen2.5-14b-tensopolis-v1", - "developer": "tensopolis", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/tensopolis_qwen2.5-14b-tensopolis-v1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.799 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6364 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5295 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3347 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4193 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4911 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/tensopolis_qwen2.5-3b-or1-tensopolis.json b/data/models/tensopolis_qwen2.5-3b-or1-tensopolis.json deleted file mode 100644 index 4a0a012e31c90a4a05d083f8a659fc99f9e78a33..0000000000000000000000000000000000000000 --- a/data/models/tensopolis_qwen2.5-3b-or1-tensopolis.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "qwen2.5-3b-or1-tensopolis", - "id": "tensopolis/qwen2.5-3b-or1-tensopolis", - "developer": "tensopolis", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.086" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/tensopolis_qwen2.5-3b-or1-tensopolis/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.354 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4421 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.173 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2945 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3749 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3197 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/tensopolis_qwen2.5-7b-tensopolis-v1.json b/data/models/tensopolis_qwen2.5-7b-tensopolis-v1.json deleted file mode 100644 index 9db9fa8b7620b5a6d846d6664725a226aee39cb2..0000000000000000000000000000000000000000 --- a/data/models/tensopolis_qwen2.5-7b-tensopolis-v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "qwen2.5-7b-tensopolis-v1", - "id": "tensopolis/qwen2.5-7b-tensopolis-v1", - "developer": "tensopolis", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/tensopolis_qwen2.5-7b-tensopolis-v1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7661 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5379 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4562 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2961 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4339 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4269 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/tensopolis_qwen2.5-7b-tensopolis-v2.json b/data/models/tensopolis_qwen2.5-7b-tensopolis-v2.json deleted file mode 100644 index dc78078080e575da6f63b389d51833159aa27b61..0000000000000000000000000000000000000000 --- a/data/models/tensopolis_qwen2.5-7b-tensopolis-v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "qwen2.5-7b-tensopolis-v2", - "id": "tensopolis/qwen2.5-7b-tensopolis-v2", - "developer": "tensopolis", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/tensopolis_qwen2.5-7b-tensopolis-v2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7521 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5415 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4819 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2903 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4246 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4243 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/tensopolis_virtuoso-lite-tensopolis-v1.json b/data/models/tensopolis_virtuoso-lite-tensopolis-v1.json deleted file mode 100644 index 4844a46243e77dea414ce9f01ad2d1e46e497596..0000000000000000000000000000000000000000 --- a/data/models/tensopolis_virtuoso-lite-tensopolis-v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "virtuoso-lite-tensopolis-v1", - "id": "tensopolis/virtuoso-lite-tensopolis-v1", - "developer": "tensopolis", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "10.306" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/tensopolis_virtuoso-lite-tensopolis-v1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8069 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6102 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2545 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3448 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4582 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4435 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/tensopolis_virtuoso-lite-tensopolis-v2.json b/data/models/tensopolis_virtuoso-lite-tensopolis-v2.json deleted file mode 100644 index 618415293de854b7f8716f29ecf65812ca995338..0000000000000000000000000000000000000000 --- a/data/models/tensopolis_virtuoso-lite-tensopolis-v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "virtuoso-lite-tensopolis-v2", - "id": "tensopolis/virtuoso-lite-tensopolis-v2", - "developer": "tensopolis", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "10.306" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/tensopolis_virtuoso-lite-tensopolis-v2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8029 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.61 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.25 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3431 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4595 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.444 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/tensopolis_virtuoso-small-tensopolis-v1.json b/data/models/tensopolis_virtuoso-small-tensopolis-v1.json deleted file mode 100644 index db74a0125f9a053ee52b18bfb3e30e86430b581e..0000000000000000000000000000000000000000 --- a/data/models/tensopolis_virtuoso-small-tensopolis-v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "virtuoso-small-tensopolis-v1", - "id": "tensopolis/virtuoso-small-tensopolis-v1", - "developer": "tensopolis", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/tensopolis_virtuoso-small-tensopolis-v1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7856 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6415 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3527 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.328 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4326 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4968 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/tensopolis_virtuoso-small-tensopolis-v2.json b/data/models/tensopolis_virtuoso-small-tensopolis-v2.json deleted file mode 100644 index 0ceb50468049d80630f9230c60d9eb378cf4c44c..0000000000000000000000000000000000000000 --- a/data/models/tensopolis_virtuoso-small-tensopolis-v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "virtuoso-small-tensopolis-v2", - "id": "tensopolis/virtuoso-small-tensopolis-v2", - "developer": "tensopolis", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/tensopolis_virtuoso-small-tensopolis-v2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.802 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6516 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3875 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3289 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4352 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5154 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/tensopolis_virtuoso-small-v2-tensopolis-v1.json b/data/models/tensopolis_virtuoso-small-v2-tensopolis-v1.json deleted file mode 100644 index 8949c85e0e891985221d06aeee8e80c23886edca..0000000000000000000000000000000000000000 --- a/data/models/tensopolis_virtuoso-small-v2-tensopolis-v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "virtuoso-small-v2-tensopolis-v1", - "id": "tensopolis/virtuoso-small-v2-tensopolis-v1", - "developer": "tensopolis", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/tensopolis_virtuoso-small-v2-tensopolis-v1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8419 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6545 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4524 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3465 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4509 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5175 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/tensoropera_fox-1-1.6b.json b/data/models/tensoropera_fox-1-1.6b.json deleted file mode 100644 index e069eab243703ed053ca9621f67ebb5d3e7290ea..0000000000000000000000000000000000000000 --- a/data/models/tensoropera_fox-1-1.6b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Fox-1-1.6B", - "id": "tensoropera/Fox-1-1.6B", - "developer": "tensoropera", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.665" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/tensoropera_Fox-1-1.6B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2766 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3307 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0174 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2634 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.355 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1371 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/tenyx_llama3-tenyxchat-70b.json b/data/models/tenyx_llama3-tenyxchat-70b.json deleted file mode 100644 index 5d007c9736570041c0f478e56dba88672d9a8e43..0000000000000000000000000000000000000000 --- a/data/models/tenyx_llama3-tenyxchat-70b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama3-TenyxChat-70B", - "id": "tenyx/Llama3-TenyxChat-70B", - "developer": "tenyx", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "70.554" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/tenyx_Llama3-TenyxChat-70B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8087 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6511 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2356 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3012 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.426 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.521 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/thedrummer_cydonia-22b-v1.2.json b/data/models/thedrummer_cydonia-22b-v1.2.json deleted file mode 100644 index 98d6dcb3a420904b32231a1f0604ff573cd7f281..0000000000000000000000000000000000000000 --- a/data/models/thedrummer_cydonia-22b-v1.2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Cydonia-22B-v1.2", - "id": "TheDrummer/Cydonia-22B-v1.2", - "developer": "TheDrummer", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "22.247" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/TheDrummer_Cydonia-22B-v1.2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5635 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5809 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2032 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3305 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4022 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4141 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/thedrummer_gemmasutra-9b-v1.json b/data/models/thedrummer_gemmasutra-9b-v1.json deleted file mode 100644 index 17fff9f094ca7add2762b142c7d7bfb471462355..0000000000000000000000000000000000000000 --- a/data/models/thedrummer_gemmasutra-9b-v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Gemmasutra-9B-v1", - "id": "TheDrummer/Gemmasutra-9B-v1", - "developer": "TheDrummer", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/TheDrummer_Gemmasutra-9B-v1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2416 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5887 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0831 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3104 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4846 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4045 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/thedrummer_gemmasutra-mini-2b-v1.json b/data/models/thedrummer_gemmasutra-mini-2b-v1.json deleted file mode 100644 index ffe02a86da2722c1765e97552984ca845f70884c..0000000000000000000000000000000000000000 --- a/data/models/thedrummer_gemmasutra-mini-2b-v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Gemmasutra-Mini-2B-v1", - "id": "TheDrummer/Gemmasutra-Mini-2B-v1", - "developer": "TheDrummer", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "2.614" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/TheDrummer_Gemmasutra-Mini-2B-v1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2549 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3575 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0378 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.271 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.349 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2055 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/thedrummer_llama-3some-8b-v2.json b/data/models/thedrummer_llama-3some-8b-v2.json deleted file mode 100644 index e0068b291080385611fbdf6ac9268d7c97379564..0000000000000000000000000000000000000000 --- a/data/models/thedrummer_llama-3some-8b-v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3SOME-8B-v2", - "id": "TheDrummer/Llama-3SOME-8B-v2", - "developer": "TheDrummer", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/TheDrummer_Llama-3SOME-8B-v2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4508 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5203 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0937 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.302 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3833 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3753 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/thedrummer_ministrations-8b-v1.json b/data/models/thedrummer_ministrations-8b-v1.json deleted file mode 100644 index 1c5fd1a4eaf17fb2eb8817855cdf0ffb10f9a791..0000000000000000000000000000000000000000 --- a/data/models/thedrummer_ministrations-8b-v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Ministrations-8B-v1", - "id": "TheDrummer/Ministrations-8B-v1", - "developer": "TheDrummer", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "8.02" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/TheDrummer_Ministrations-8B-v1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2822 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4877 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1843 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3247 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4449 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3644 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/thedrummer_rocinante-12b-v1.json b/data/models/thedrummer_rocinante-12b-v1.json deleted file mode 100644 index b3429ecc2841434e59e244245b420029e72ddeab..0000000000000000000000000000000000000000 --- a/data/models/thedrummer_rocinante-12b-v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Rocinante-12B-v1", - "id": "TheDrummer/Rocinante-12B-v1", - "developer": "TheDrummer", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/TheDrummer_Rocinante-12B-v1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6076 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5065 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1269 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2911 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4017 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3477 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/thedrummer_tiger-gemma-9b-v1.json b/data/models/thedrummer_tiger-gemma-9b-v1.json deleted file mode 100644 index 4d302e395a1f733bc96d3bb628ca4c005a9e88ac..0000000000000000000000000000000000000000 --- a/data/models/thedrummer_tiger-gemma-9b-v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Tiger-Gemma-9B-v1", - "id": "TheDrummer/Tiger-Gemma-9B-v1", - "developer": "TheDrummer", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "9.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/TheDrummer_Tiger-Gemma-9B-v1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7282 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5704 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1835 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3389 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4162 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4118 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/thedrummer_tiger-gemma-9b-v2.json b/data/models/thedrummer_tiger-gemma-9b-v2.json deleted file mode 100644 index 56356a8631083773fd435f26247c23a7ebf37ed4..0000000000000000000000000000000000000000 --- a/data/models/thedrummer_tiger-gemma-9b-v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Tiger-Gemma-9B-v2", - "id": "TheDrummer/Tiger-Gemma-9B-v2", - "developer": "TheDrummer", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "9.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/TheDrummer_Tiger-Gemma-9B-v2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6986 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5617 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.182 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3398 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4084 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4112 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/thedrummer_tiger-gemma-9b-v3.json b/data/models/thedrummer_tiger-gemma-9b-v3.json deleted file mode 100644 index 3f2fabb38e31b9e5531ee7b8747b6218e10ca787..0000000000000000000000000000000000000000 --- a/data/models/thedrummer_tiger-gemma-9b-v3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Tiger-Gemma-9B-v3", - "id": "TheDrummer/Tiger-Gemma-9B-v3", - "developer": "TheDrummer", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "9.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/TheDrummer_Tiger-Gemma-9B-v3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6821 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5812 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1624 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3389 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4004 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4059 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/thedrunkensnail_daughter-of-rhodia-12b.json b/data/models/thedrunkensnail_daughter-of-rhodia-12b.json deleted file mode 100644 index d5bb3f127cf80d86f6b1ae5cb565ac28addf3fe3..0000000000000000000000000000000000000000 --- a/data/models/thedrunkensnail_daughter-of-rhodia-12b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Daughter-of-Rhodia-12B", - "id": "TheDrunkenSnail/Daughter-of-Rhodia-12B", - "developer": "TheDrunkenSnail", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/TheDrunkenSnail_Daughter-of-Rhodia-12B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6904 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5179 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1224 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3171 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4348 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3641 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/thedrunkensnail_mother-of-rhodia-12b.json b/data/models/thedrunkensnail_mother-of-rhodia-12b.json deleted file mode 100644 index 6fe0c8a0bc52338b54f7f272ff7111d2cfcaab63..0000000000000000000000000000000000000000 --- a/data/models/thedrunkensnail_mother-of-rhodia-12b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mother-of-Rhodia-12B", - "id": "TheDrunkenSnail/Mother-of-Rhodia-12B", - "developer": "TheDrunkenSnail", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/TheDrunkenSnail_Mother-of-Rhodia-12B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6505 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4948 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1224 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2987 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4124 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3551 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/thedrunkensnail_son-of-rhodia.json b/data/models/thedrunkensnail_son-of-rhodia.json deleted file mode 100644 index 67e41a24176923bbf4a35b038924c80a449fd649..0000000000000000000000000000000000000000 --- a/data/models/thedrunkensnail_son-of-rhodia.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Son-of-Rhodia", - "id": "TheDrunkenSnail/Son-of-Rhodia", - "developer": "TheDrunkenSnail", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/TheDrunkenSnail_Son-of-Rhodia/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7046 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5097 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1314 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3129 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4203 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3608 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/thehierophant_underground-cognitive-v0.3-test.json b/data/models/thehierophant_underground-cognitive-v0.3-test.json deleted file mode 100644 index 91ceae8a10f239c0fb03e255ad4a55cf8bb428c3..0000000000000000000000000000000000000000 --- a/data/models/thehierophant_underground-cognitive-v0.3-test.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Underground-Cognitive-V0.3-test", - "id": "TheHierophant/Underground-Cognitive-V0.3-test", - "developer": "TheHierophant", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "10.732" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/TheHierophant_Underground-Cognitive-V0.3-test/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4808 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.529 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0589 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2987 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4351 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3318 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/theo77186_qwen2.5-coder-7b-instruct-20241106.json b/data/models/theo77186_qwen2.5-coder-7b-instruct-20241106.json deleted file mode 100644 index 9ba42115d424617e68fbf28ba5afabaf6809dddf..0000000000000000000000000000000000000000 --- a/data/models/theo77186_qwen2.5-coder-7b-instruct-20241106.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-Coder-7B-Instruct-20241106", - "id": "theo77186/Qwen2.5-Coder-7B-Instruct-20241106", - "developer": "theo77186", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/theo77186_Qwen2.5-Coder-7B-Instruct-20241106/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6101 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5008 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3882 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2919 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4073 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3353 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/theprint_boptruth-agatha-7b.json b/data/models/theprint_boptruth-agatha-7b.json deleted file mode 100644 index 2b3bdd328bffb7618e5f5bd35f1f4bd8726e1dbf..0000000000000000000000000000000000000000 --- a/data/models/theprint_boptruth-agatha-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Boptruth-Agatha-7B", - "id": "theprint/Boptruth-Agatha-7B", - "developer": "theprint", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/theprint_Boptruth-Agatha-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3124 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4984 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0551 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2995 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4277 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2861 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/theprint_cleverboi-7b-v2.json b/data/models/theprint_cleverboi-7b-v2.json deleted file mode 100644 index 5594acf54d2b62152bf8557651a77db51a940e98..0000000000000000000000000000000000000000 --- a/data/models/theprint_cleverboi-7b-v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "CleverBoi-7B-v2", - "id": "theprint/CleverBoi-7B-v2", - "developer": "theprint", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "?", - "params_billions": "7.736" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/theprint_CleverBoi-7B-v2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.217 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4532 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0264 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2886 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4695 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2709 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/theprint_cleverboi-7b-v3.json b/data/models/theprint_cleverboi-7b-v3.json deleted file mode 100644 index 5447d34336e2269924aebb9f03dd1c3eb6733fe0..0000000000000000000000000000000000000000 --- a/data/models/theprint_cleverboi-7b-v3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "CleverBoi-7B-v3", - "id": "theprint/CleverBoi-7B-v3", - "developer": "theprint", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "?", - "params_billions": "7.736" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/theprint_CleverBoi-7B-v3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2382 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4414 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.04 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2659 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4072 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2868 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/theprint_cleverboi-llama-3.1-8b-instruct.json b/data/models/theprint_cleverboi-llama-3.1-8b-instruct.json deleted file mode 100644 index 20a044a65160d917c4841f907297d3464cc94ac3..0000000000000000000000000000000000000000 --- a/data/models/theprint_cleverboi-llama-3.1-8b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "CleverBoi-Llama-3.1-8B-Instruct", - "id": "theprint/CleverBoi-Llama-3.1-8B-Instruct", - "developer": "theprint", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "?", - "params_billions": "16.061" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/theprint_CleverBoi-Llama-3.1-8B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1682 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.456 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0491 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3003 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4014 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3075 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/theprint_cleverboi-llama-3.1-8b-v2.json b/data/models/theprint_cleverboi-llama-3.1-8b-v2.json deleted file mode 100644 index 0d4b164b235193ea3f4469c4f7d910ff635be7bf..0000000000000000000000000000000000000000 --- a/data/models/theprint_cleverboi-llama-3.1-8b-v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "CleverBoi-Llama-3.1-8B-v2", - "id": "theprint/CleverBoi-Llama-3.1-8B-v2", - "developer": "theprint", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "?", - "params_billions": "9.3" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/theprint_CleverBoi-Llama-3.1-8B-v2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1961 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4668 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0529 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2861 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3735 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3188 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/theprint_cleverboi-nemo-12b-v2.json b/data/models/theprint_cleverboi-nemo-12b-v2.json deleted file mode 100644 index 6b642345d58ec5462310a7e6a02eafcd7eb18e29..0000000000000000000000000000000000000000 --- a/data/models/theprint_cleverboi-nemo-12b-v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "CleverBoi-Nemo-12B-v2", - "id": "theprint/CleverBoi-Nemo-12B-v2", - "developer": "theprint", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "?", - "params_billions": "13.933" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/theprint_CleverBoi-Nemo-12B-v2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2046 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5241 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1035 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3138 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4187 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3228 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/theprint_code-llama-bagel-8b.json b/data/models/theprint_code-llama-bagel-8b.json deleted file mode 100644 index 2877c68a2f3268cf88b51bf2a915c4e9f8d9e7df..0000000000000000000000000000000000000000 --- a/data/models/theprint_code-llama-bagel-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Code-Llama-Bagel-8B", - "id": "theprint/Code-Llama-Bagel-8B", - "developer": "theprint", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/theprint_Code-Llama-Bagel-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.253 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4697 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0612 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.276 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.368 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2822 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/theprint_conversely-mistral-7b.json b/data/models/theprint_conversely-mistral-7b.json deleted file mode 100644 index 5a96c40c0e7801ec26af31a65b7cedd99226d6e1..0000000000000000000000000000000000000000 --- a/data/models/theprint_conversely-mistral-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Conversely-Mistral-7B", - "id": "theprint/Conversely-Mistral-7B", - "developer": "theprint", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "?", - "params_billions": "14.496" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/theprint_Conversely-Mistral-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2608 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4672 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0279 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2852 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4189 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2826 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/theprint_llama-3.2-3b-vanrossum.json b/data/models/theprint_llama-3.2-3b-vanrossum.json deleted file mode 100644 index e1ef672634eacfa0cf489cdfea54ae36eb78abc7..0000000000000000000000000000000000000000 --- a/data/models/theprint_llama-3.2-3b-vanrossum.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.2-3B-VanRossum", - "id": "theprint/Llama-3.2-3B-VanRossum", - "developer": "theprint", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "?", - "params_billions": "3.696" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/theprint_Llama-3.2-3B-VanRossum/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4783 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4279 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0974 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2676 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3442 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.277 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/theprint_phi-3-mini-4k-python.json b/data/models/theprint_phi-3-mini-4k-python.json deleted file mode 100644 index 2847551d7af57421ab1b91add459e653a069a825..0000000000000000000000000000000000000000 --- a/data/models/theprint_phi-3-mini-4k-python.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "phi-3-mini-4k-python", - "id": "theprint/phi-3-mini-4k-python", - "developer": "theprint", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "?", - "params_billions": "4.132" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/theprint_phi-3-mini-4k-python/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2409 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4938 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.105 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2911 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3922 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3577 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/theprint_rewiz-7b.json b/data/models/theprint_rewiz-7b.json deleted file mode 100644 index 408b754e5d79ce2b9086795bd33ea8c1efafe869..0000000000000000000000000000000000000000 --- a/data/models/theprint_rewiz-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ReWiz-7B", - "id": "theprint/ReWiz-7B", - "developer": "theprint", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "?", - "params_billions": "7.736" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/theprint_ReWiz-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4048 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4564 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0408 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2752 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4612 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.267 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/theprint_rewiz-llama-3.1-8b-v2.json b/data/models/theprint_rewiz-llama-3.1-8b-v2.json deleted file mode 100644 index 2c8c81179d5a7512bd414b4aa8f4284613f603fa..0000000000000000000000000000000000000000 --- a/data/models/theprint_rewiz-llama-3.1-8b-v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ReWiz-Llama-3.1-8B-v2", - "id": "theprint/ReWiz-Llama-3.1-8B-v2", - "developer": "theprint", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "?", - "params_billions": "9.3" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/theprint_ReWiz-Llama-3.1-8B-v2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2379 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4632 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0574 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3029 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3814 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.331 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/theprint_rewiz-llama-3.2-3b.json b/data/models/theprint_rewiz-llama-3.2-3b.json deleted file mode 100644 index 10a1cd16b2e486bd37d16d81ffff60fd242aa6fa..0000000000000000000000000000000000000000 --- a/data/models/theprint_rewiz-llama-3.2-3b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ReWiz-Llama-3.2-3B", - "id": "theprint/ReWiz-Llama-3.2-3B", - "developer": "theprint", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/theprint_ReWiz-Llama-3.2-3B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4649 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4343 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1095 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2836 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3614 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2887 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/theprint_rewiz-nemo-12b-instruct.json b/data/models/theprint_rewiz-nemo-12b-instruct.json deleted file mode 100644 index 93ba7a32664fbcf85f80fde2da0882d11a260f55..0000000000000000000000000000000000000000 --- a/data/models/theprint_rewiz-nemo-12b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ReWiz-Nemo-12B-Instruct", - "id": "theprint/ReWiz-Nemo-12B-Instruct", - "developer": "theprint", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/theprint_ReWiz-Nemo-12B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1062 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5092 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1042 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3238 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4096 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3339 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/theprint_rewiz-qwen-2.5-14b.json b/data/models/theprint_rewiz-qwen-2.5-14b.json deleted file mode 100644 index a8b032ecbd0786d797aadb843aa1f0e57f37f895..0000000000000000000000000000000000000000 --- a/data/models/theprint_rewiz-qwen-2.5-14b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ReWiz-Qwen-2.5-14B", - "id": "theprint/ReWiz-Qwen-2.5-14B", - "developer": "theprint", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "?", - "params_billions": "16.743" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/theprint_ReWiz-Qwen-2.5-14B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2785 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6179 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2923 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.38 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4539 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5092 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/theprint_rewiz-worldbuilder-7b.json b/data/models/theprint_rewiz-worldbuilder-7b.json deleted file mode 100644 index 2a3cb81d5c334624fcdb907764ed44abf7b40b59..0000000000000000000000000000000000000000 --- a/data/models/theprint_rewiz-worldbuilder-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ReWiz-Worldbuilder-7B", - "id": "theprint/ReWiz-Worldbuilder-7B", - "developer": "theprint", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "7.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/theprint_ReWiz-Worldbuilder-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.251 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4636 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.037 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2693 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4572 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2971 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/theprint_rudolph-hermes-7b.json b/data/models/theprint_rudolph-hermes-7b.json deleted file mode 100644 index 927b061c08f806fd48b83cbfd09723cc316c5817..0000000000000000000000000000000000000000 --- a/data/models/theprint_rudolph-hermes-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "RuDolph-Hermes-7B", - "id": "theprint/RuDolph-Hermes-7B", - "developer": "theprint", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/theprint_RuDolph-Hermes-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3604 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5053 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0514 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3121 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4226 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3073 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/theprint_worldbuilder-12b.json b/data/models/theprint_worldbuilder-12b.json deleted file mode 100644 index 67d16a3e3bf03772f2d4da65f9b6a3fbc32a3da5..0000000000000000000000000000000000000000 --- a/data/models/theprint_worldbuilder-12b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "WorldBuilder-12B", - "id": "theprint/WorldBuilder-12B", - "developer": "theprint", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "?", - "params_billions": "13.933" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/theprint_WorldBuilder-12B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1374 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.501 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0446 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.297 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4066 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3192 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/thetsar1209_nemo-carpmuscle-v0.1.json b/data/models/thetsar1209_nemo-carpmuscle-v0.1.json deleted file mode 100644 index a4cd4c357509c4010e5d0c35888f171fa232c493..0000000000000000000000000000000000000000 --- a/data/models/thetsar1209_nemo-carpmuscle-v0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "nemo-carpmuscle-v0.1", - "id": "TheTsar1209/nemo-carpmuscle-v0.1", - "developer": "TheTsar1209", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/TheTsar1209_nemo-carpmuscle-v0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2276 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5084 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0476 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.297 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4135 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3406 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/thetsar1209_qwen-carpmuscle-r-v0.3.json b/data/models/thetsar1209_qwen-carpmuscle-r-v0.3.json deleted file mode 100644 index 95730d74c3ddd9eabc9bc5ee19760967b74057fb..0000000000000000000000000000000000000000 --- a/data/models/thetsar1209_qwen-carpmuscle-r-v0.3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "qwen-carpmuscle-r-v0.3", - "id": "TheTsar1209/qwen-carpmuscle-r-v0.3", - "developer": "TheTsar1209", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/TheTsar1209_qwen-carpmuscle-r-v0.3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4455 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6227 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3006 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3507 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4278 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5103 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/thetsar1209_qwen-carpmuscle-v0.1.json b/data/models/thetsar1209_qwen-carpmuscle-v0.1.json deleted file mode 100644 index 2323300f0db9ddc5797bc1bca7ea147acf67f9ef..0000000000000000000000000000000000000000 --- a/data/models/thetsar1209_qwen-carpmuscle-v0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "qwen-carpmuscle-v0.1", - "id": "TheTsar1209/qwen-carpmuscle-v0.1", - "developer": "TheTsar1209", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/TheTsar1209_qwen-carpmuscle-v0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5622 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6434 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2628 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.344 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4161 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.52 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/thetsar1209_qwen-carpmuscle-v0.2.json b/data/models/thetsar1209_qwen-carpmuscle-v0.2.json deleted file mode 100644 index ffe57cc09495a67d2f603c9bdfdf287904ae858a..0000000000000000000000000000000000000000 --- a/data/models/thetsar1209_qwen-carpmuscle-v0.2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "qwen-carpmuscle-v0.2", - "id": "TheTsar1209/qwen-carpmuscle-v0.2", - "developer": "TheTsar1209", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/TheTsar1209_qwen-carpmuscle-v0.2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5257 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6387 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2832 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3557 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4346 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5147 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/thetsar1209_qwen-carpmuscle-v0.3.json b/data/models/thetsar1209_qwen-carpmuscle-v0.3.json deleted file mode 100644 index e76f14023c25fa952601d2d986aec92c84443e10..0000000000000000000000000000000000000000 --- a/data/models/thetsar1209_qwen-carpmuscle-v0.3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "qwen-carpmuscle-v0.3", - "id": "TheTsar1209/qwen-carpmuscle-v0.3", - "developer": "TheTsar1209", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/TheTsar1209_qwen-carpmuscle-v0.3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4476 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6152 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3134 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3565 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4132 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5062 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/thetsar1209_qwen-carpmuscle-v0.4.1.json b/data/models/thetsar1209_qwen-carpmuscle-v0.4.1.json deleted file mode 100644 index e7379566bfbb9fca91b46ab114f083475fa411b5..0000000000000000000000000000000000000000 --- a/data/models/thetsar1209_qwen-carpmuscle-v0.4.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "qwen-carpmuscle-v0.4.1", - "id": "TheTsar1209/qwen-carpmuscle-v0.4.1", - "developer": "TheTsar1209", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/TheTsar1209_qwen-carpmuscle-v0.4.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.736 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6507 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2779 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3456 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4489 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5191 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/thetsar1209_qwen-carpmuscle-v0.4.json b/data/models/thetsar1209_qwen-carpmuscle-v0.4.json deleted file mode 100644 index a33848d82995a834c58c49b85dd329e8feb7e423..0000000000000000000000000000000000000000 --- a/data/models/thetsar1209_qwen-carpmuscle-v0.4.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "qwen-carpmuscle-v0.4", - "id": "TheTsar1209/qwen-carpmuscle-v0.4", - "developer": "TheTsar1209", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/TheTsar1209_qwen-carpmuscle-v0.4/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7202 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6454 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2772 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3523 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4516 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5144 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/thinkcoder_llama3-8b-instruct-lora-8-sft.json b/data/models/thinkcoder_llama3-8b-instruct-lora-8-sft.json deleted file mode 100644 index 1b8d6b67780650e346fe25d3fdfc62d19f634a2d..0000000000000000000000000000000000000000 --- a/data/models/thinkcoder_llama3-8b-instruct-lora-8-sft.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "llama3-8b-instruct-lora-8-sft", - "id": "thinkcoder/llama3-8b-instruct-lora-8-sft", - "developer": "thinkcoder", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/thinkcoder_llama3-8b-instruct-lora-8-sft/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.648 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4865 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.102 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2668 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3235 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3476 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/thirdeyeai_elevate360m.json b/data/models/thirdeyeai_elevate360m.json deleted file mode 100644 index 71a7e281c4121971bae45c157d90cd13a8caa33b..0000000000000000000000000000000000000000 --- a/data/models/thirdeyeai_elevate360m.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "elevate360m", - "id": "thirdeyeai/elevate360m", - "developer": "thirdeyeai", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "0.362" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/thirdeyeai_elevate360m/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0445 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2963 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0159 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2408 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3462 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1077 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/thomas-yanxin_xinyuan-qwen2-1_5b.json b/data/models/thomas-yanxin_xinyuan-qwen2-1_5b.json deleted file mode 100644 index 75155df79180d62b4d202bb707543212e6362df9..0000000000000000000000000000000000000000 --- a/data/models/thomas-yanxin_xinyuan-qwen2-1_5b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "XinYuan-Qwen2-1_5B", - "id": "thomas-yanxin/XinYuan-Qwen2-1_5B", - "developer": "thomas-yanxin", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.777" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/thomas-yanxin_XinYuan-Qwen2-1_5B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2986 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3635 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0672 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2701 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3634 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2357 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/thomas-yanxin_xinyuan-qwen2-7b-0917.json b/data/models/thomas-yanxin_xinyuan-qwen2-7b-0917.json deleted file mode 100644 index 4d1e489a58f8fa01836ffdd4280c27347ed4aa71..0000000000000000000000000000000000000000 --- a/data/models/thomas-yanxin_xinyuan-qwen2-7b-0917.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "XinYuan-Qwen2-7B-0917", - "id": "thomas-yanxin/XinYuan-Qwen2-7B-0917", - "developer": "thomas-yanxin", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/thomas-yanxin_XinYuan-Qwen2-7B-0917/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3719 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5169 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1979 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3096 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4401 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4245 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/thomas-yanxin_xinyuan-qwen2-7b.json b/data/models/thomas-yanxin_xinyuan-qwen2-7b.json deleted file mode 100644 index 77633631fb2c212e5e39e87683fa474a98541d96..0000000000000000000000000000000000000000 --- a/data/models/thomas-yanxin_xinyuan-qwen2-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "XinYuan-Qwen2-7B", - "id": "thomas-yanxin/XinYuan-Qwen2-7B", - "developer": "thomas-yanxin", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/thomas-yanxin_XinYuan-Qwen2-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4438 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4937 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1458 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2911 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4058 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3925 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/thomas-yanxin_xinyuan-qwen2.5-7b-0917.json b/data/models/thomas-yanxin_xinyuan-qwen2.5-7b-0917.json deleted file mode 100644 index 9c3ef411d640a60029455b67ae1b2c653f4cc076..0000000000000000000000000000000000000000 --- a/data/models/thomas-yanxin_xinyuan-qwen2.5-7b-0917.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "XinYuan-Qwen2.5-7B-0917", - "id": "thomas-yanxin/XinYuan-Qwen2.5-7B-0917", - "developer": "thomas-yanxin", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/thomas-yanxin_XinYuan-Qwen2.5-7B-0917/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3577 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5184 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1934 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.281 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3676 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3882 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/thudm_glm-4-9b-chat-1m-hf.json b/data/models/thudm_glm-4-9b-chat-1m-hf.json deleted file mode 100644 index 612cbeed3fd74ccc7a371a64aeff0f260be1c908..0000000000000000000000000000000000000000 --- a/data/models/thudm_glm-4-9b-chat-1m-hf.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "glm-4-9b-chat-1m-hf", - "id": "THUDM/glm-4-9b-chat-1m-hf", - "developer": "THUDM", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "GlmForCausalLM", - "params_billions": "9.484" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/THUDM_glm-4-9b-chat-1m-hf/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5341 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3901 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0483 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2919 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3689 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1814 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/thudm_glm-4-9b-chat-1m.json b/data/models/thudm_glm-4-9b-chat-1m.json deleted file mode 100644 index 2453a832f9637c08f699a98f51c1b2407f4938e7..0000000000000000000000000000000000000000 --- a/data/models/thudm_glm-4-9b-chat-1m.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "glm-4-9b-chat-1m", - "id": "THUDM/glm-4-9b-chat-1m", - "developer": "THUDM", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "ChatGLMModel", - "params_billions": "9.484" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/THUDM_glm-4-9b-chat-1m/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.418 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3037 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3795 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3163 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/thudm_glm-4-9b-chat-hf.json b/data/models/thudm_glm-4-9b-chat-hf.json deleted file mode 100644 index 90a2e648feafef514a66007d4ac732580b4c5794..0000000000000000000000000000000000000000 --- a/data/models/thudm_glm-4-9b-chat-hf.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "glm-4-9b-chat-hf", - "id": "THUDM/glm-4-9b-chat-hf", - "developer": "THUDM", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "GlmForCausalLM", - "params_billions": "9.4" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/THUDM_glm-4-9b-chat-hf/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6513 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4432 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0846 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3029 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3593 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2774 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/thudm_glm-4-9b-chat.json b/data/models/thudm_glm-4-9b-chat.json deleted file mode 100644 index 8ed3d0c21643ba07c0233b77b4dede5bd3f6c05b..0000000000000000000000000000000000000000 --- a/data/models/thudm_glm-4-9b-chat.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "glm-4-9b-chat", - "id": "THUDM/glm-4-9b-chat", - "developer": "THUDM", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "ChatGLMModelM", - "params_billions": "9.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/THUDM_glm-4-9b-chat/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4736 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3138 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3994 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3167 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/thudm_glm-4-9b.json b/data/models/thudm_glm-4-9b.json deleted file mode 100644 index 20185c0d10e92b7094afcf2843b97947f2a5345a..0000000000000000000000000000000000000000 --- a/data/models/thudm_glm-4-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "glm-4-9b", - "id": "THUDM/glm-4-9b", - "developer": "THUDM", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "ChatGLMModelM", - "params_billions": "9.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/THUDM_glm-4-9b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1426 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5528 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3163 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4386 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4145 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/tianyil1_mistralforcausallm_cal_dpo.json b/data/models/tianyil1_mistralforcausallm_cal_dpo.json deleted file mode 100644 index ce41ee9e0297e273f831cc61c254ac4d2edaddb5..0000000000000000000000000000000000000000 --- a/data/models/tianyil1_mistralforcausallm_cal_dpo.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MistralForCausalLM_Cal_DPO", - "id": "tianyil1/MistralForCausalLM_Cal_DPO", - "developer": "tianyil1", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/tianyil1_MistralForCausalLM_Cal_DPO/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5328 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4381 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0287 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.276 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3977 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2763 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/tiger-lab_acecoder-qwen2.5-7b-ins-rule.json b/data/models/tiger-lab_acecoder-qwen2.5-7b-ins-rule.json deleted file mode 100644 index 460820dc3a27e5b11f104f8160ef7566b4b833b1..0000000000000000000000000000000000000000 --- a/data/models/tiger-lab_acecoder-qwen2.5-7b-ins-rule.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "AceCoder-Qwen2.5-7B-Ins-Rule", - "id": "TIGER-Lab/AceCoder-Qwen2.5-7B-Ins-Rule", - "developer": "TIGER-Lab", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/TIGER-Lab_AceCoder-Qwen2.5-7B-Ins-Rule/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7424 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5404 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4992 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3012 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.398 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4322 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/tiger-lab_acecoder-qwen2.5-coder-7b-base-rule.json b/data/models/tiger-lab_acecoder-qwen2.5-coder-7b-base-rule.json deleted file mode 100644 index 6090e2710afcf1250d6ea149fd2bf0bf2701790f..0000000000000000000000000000000000000000 --- a/data/models/tiger-lab_acecoder-qwen2.5-coder-7b-base-rule.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "AceCoder-Qwen2.5-Coder-7B-Base-Rule", - "id": "TIGER-Lab/AceCoder-Qwen2.5-Coder-7B-Base-Rule", - "developer": "TIGER-Lab", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/TIGER-Lab_AceCoder-Qwen2.5-Coder-7B-Base-Rule/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4408 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4902 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2017 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2718 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3449 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3745 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/tiger-lab_acecoder-qwen2.5-coder-7b-ins-rule.json b/data/models/tiger-lab_acecoder-qwen2.5-coder-7b-ins-rule.json deleted file mode 100644 index be9689f1af4b66747f88e1aa663193a3c731eaa0..0000000000000000000000000000000000000000 --- a/data/models/tiger-lab_acecoder-qwen2.5-coder-7b-ins-rule.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "AceCoder-Qwen2.5-Coder-7B-Ins-Rule", - "id": "TIGER-Lab/AceCoder-Qwen2.5-Coder-7B-Ins-Rule", - "developer": "TIGER-Lab", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/TIGER-Lab_AceCoder-Qwen2.5-Coder-7B-Ins-Rule/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6222 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5089 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3603 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2777 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4046 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3428 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/tiger-lab_acecoderm-7b.json b/data/models/tiger-lab_acecoderm-7b.json deleted file mode 100644 index 1c57069ea4e7b41fec03ebe6098713bb476619d9..0000000000000000000000000000000000000000 --- a/data/models/tiger-lab_acecoderm-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "AceCodeRM-7B", - "id": "TIGER-Lab/AceCodeRM-7B", - "developer": "TIGER-Lab", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalRM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/TIGER-Lab_AceCodeRM-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5855 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4773 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3467 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3045 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4192 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3361 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/tiger-lab_mammoth2-7b-plus.json b/data/models/tiger-lab_mammoth2-7b-plus.json deleted file mode 100644 index b340920a37825631a9899372b91bd43bcb82af6a..0000000000000000000000000000000000000000 --- a/data/models/tiger-lab_mammoth2-7b-plus.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MAmmoTH2-7B-Plus", - "id": "TIGER-Lab/MAmmoTH2-7B-Plus", - "developer": "TIGER-Lab", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/TIGER-Lab_MAmmoTH2-7B-Plus/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5575 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4235 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1858 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2802 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4124 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3017 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/tiger-lab_qwen2.5-math-7b-cft.json b/data/models/tiger-lab_qwen2.5-math-7b-cft.json deleted file mode 100644 index bdd36b101b650b56f440ce5460e915e59f350804..0000000000000000000000000000000000000000 --- a/data/models/tiger-lab_qwen2.5-math-7b-cft.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-Math-7B-CFT", - "id": "TIGER-Lab/Qwen2.5-Math-7B-CFT", - "developer": "TIGER-Lab", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/TIGER-Lab_Qwen2.5-Math-7B-CFT/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2777 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4637 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5574 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2861 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3887 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2945 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/tii-uae_falcon3-10b-instruct-fc.json b/data/models/tii-uae_falcon3-10b-instruct-fc.json deleted file mode 100644 index f7d3c2046cdd992d0267bea9e423afc3696564df..0000000000000000000000000000000000000000 --- a/data/models/tii-uae_falcon3-10b-instruct-fc.json +++ /dev/null @@ -1,851 +0,0 @@ -{ - "model_info": { - "name": "Falcon3-10B-Instruct (FC)", - "id": "tii-uae/falcon3-10b-instruct-fc", - "developer": "tii-uae", - "additional_details": { - "raw_model_name": "Falcon3-10B-Instruct (FC)", - "organization": "TII UAE", - "license": "falcon-llm-license", - "mode": "FC", - "model_link": "https://huggingface.co/tiiuae/Falcon3-10B-Instruct" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/tii-uae/falcon3-10b-instruct-fc/1775236112.409044", - "retrieved_timestamp": "1775236112.409044", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 82.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 27.01 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 52.59 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 69.27 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 92.22 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 190.96 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 85.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 70.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 93.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 88.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 87.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 75.43 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 77.13 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 76.16 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 50.0 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 41.67 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 6.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 6.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 9.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 5.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 5.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 1.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 2.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 1.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 27.53 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 12.26 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 19.35 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 50.97 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 93.75 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 32.09 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/tii-uae_falcon3-1b-instruct-fc.json b/data/models/tii-uae_falcon3-1b-instruct-fc.json deleted file mode 100644 index 0cefc6c2a2f357b31e3508b5553a115ad64fcca0..0000000000000000000000000000000000000000 --- a/data/models/tii-uae_falcon3-1b-instruct-fc.json +++ /dev/null @@ -1,851 +0,0 @@ -{ - "model_info": { - "name": "Falcon3-1B-Instruct (FC)", - "id": "tii-uae/falcon3-1b-instruct-fc", - "developer": "tii-uae", - "additional_details": { - "raw_model_name": "Falcon3-1B-Instruct (FC)", - "organization": "TII UAE", - "license": "falcon-llm-license", - "mode": "FC", - "model_link": "https://huggingface.co/tiiuae/Falcon3-1B-Instruct" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/tii-uae/falcon3-1b-instruct-fc/1775236112.4216902", - "retrieved_timestamp": "1775236112.4216902", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 106.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 11.08 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 1.72 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 5.23 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 14.34 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 11.48 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 9.02 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 2.58 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 6.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 18.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 9.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 2.89 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 4.26 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 2.37 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 12.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 5.81 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 5.16 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 7.74 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 4.52 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 87.3 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/tii-uae_falcon3-3b-instruct-fc.json b/data/models/tii-uae_falcon3-3b-instruct-fc.json deleted file mode 100644 index 15447ce96b0b9421f74974bee043b92be452fcaf..0000000000000000000000000000000000000000 --- a/data/models/tii-uae_falcon3-3b-instruct-fc.json +++ /dev/null @@ -1,851 +0,0 @@ -{ - "model_info": { - "name": "Falcon3-3B-Instruct (FC)", - "id": "tii-uae/falcon3-3b-instruct-fc", - "developer": "tii-uae", - "additional_details": { - "raw_model_name": "Falcon3-3B-Instruct (FC)", - "organization": "TII UAE", - "license": "falcon-llm-license", - "mode": "FC", - "model_link": "https://huggingface.co/tiiuae/Falcon3-3B-Instruct" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/tii-uae/falcon3-3b-instruct-fc/1775236112.420633", - "retrieved_timestamp": "1775236112.420633", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 104.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 16.25 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 36.7 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 38.52 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 107.47 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 103.62 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 54.62 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 56.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 69.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 67.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 25.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 54.48 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 57.36 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 54.7 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 25.0 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 33.33 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 1.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 1.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 0.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 0.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 1.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 1.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 1.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 1.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 7.74 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 6.45 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 8.39 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 8.39 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 81.25 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 32.92 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/tii-uae_falcon3-7b-instruct-fc.json b/data/models/tii-uae_falcon3-7b-instruct-fc.json deleted file mode 100644 index 0e6e7d15461b1c39c7c057475e6656f6e9ccd805..0000000000000000000000000000000000000000 --- a/data/models/tii-uae_falcon3-7b-instruct-fc.json +++ /dev/null @@ -1,851 +0,0 @@ -{ - "model_info": { - "name": "Falcon3-7B-Instruct (FC)", - "id": "tii-uae/falcon3-7b-instruct-fc", - "developer": "tii-uae", - "additional_details": { - "raw_model_name": "Falcon3-7B-Instruct (FC)", - "organization": "TII UAE", - "license": "falcon-llm-license", - "mode": "FC", - "model_link": "https://huggingface.co/tiiuae/Falcon3-7B-Instruct" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/tii-uae/falcon3-7b-instruct-fc/1775236112.4139452", - "retrieved_timestamp": "1775236112.4139452", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 91.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 24.03 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 73.61 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 93.11 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 117.8 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 315.7 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 82.69 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 65.75 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 89.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 87.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 89.0 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 68.32 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 74.81 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 66.76 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 75.0 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 62.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 5.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 7.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 4.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 5.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 4.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 0.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 1.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 20.65 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 10.32 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 12.9 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 38.71 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 100.0 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 31.99 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/tiiuae_falcon-11b.json b/data/models/tiiuae_falcon-11b.json deleted file mode 100644 index f33bb86b73388be7d7cc5b7db6dddf4e96b218a7..0000000000000000000000000000000000000000 --- a/data/models/tiiuae_falcon-11b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "falcon-11B", - "id": "tiiuae/falcon-11B", - "developer": "tiiuae", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "FalconForCausalLM", - "params_billions": "11.103" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/tiiuae_falcon-11B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3261 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4392 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0279 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.271 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3986 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2389 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/tiiuae_falcon-40b-instruct.json b/data/models/tiiuae_falcon-40b-instruct.json deleted file mode 100644 index a0ce79c2aa79294e7a40f42499fead0c252d65e9..0000000000000000000000000000000000000000 --- a/data/models/tiiuae_falcon-40b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "falcon-40b-instruct", - "id": "tiiuae/falcon-40b-instruct", - "developer": "tiiuae", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "FalconForCausalLM", - "params_billions": "40.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/tiiuae_falcon-40b-instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2454 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4054 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0196 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.25 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3762 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2261 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/tiiuae_falcon-40b.json b/data/models/tiiuae_falcon-40b.json deleted file mode 100644 index f3b13c64ca29c8764cd236bf684e199f1b70a9d5..0000000000000000000000000000000000000000 --- a/data/models/tiiuae_falcon-40b.json +++ /dev/null @@ -1,1179 +0,0 @@ -{ - "model_info": { - "name": "Falcon 40B", - "id": "tiiuae/falcon-40b", - "developer": "tiiuae", - "inference_platform": "unknown", - "additional_details": { - "model_id_aliases": [ - "tiiuae/Falcon-40B" - ] - } - }, - "evaluations": [ - { - "evaluation_id": "helm_classic/tiiuae_Falcon-40B/1774096308.339228", - "retrieved_timestamp": "1774096308.339228", - "source_metadata": { - "source_name": "helm_classic", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_classic", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_classic", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperform on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.729, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Calibration": "{\"description\": \"\", \"tab\": \"Calibration\", \"score\": \"\"}", - "Mean win rate - Robustness": "{\"description\": \"\", \"tab\": \"Robustness\", \"score\": \"0.7051048951048952\"}", - "Mean win rate - Fairness": "{\"description\": \"\", \"tab\": \"Fairness\", \"score\": \"0.6857342657342658\"}", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}", - "Mean win rate - Bias": "{\"description\": \"\", \"tab\": \"Bias\", \"score\": \"0.48586479674272687\"}", - "Mean win rate - Toxicity": "{\"description\": \"\", \"tab\": \"Toxicity\", \"score\": \"0.4706876456876457\"}", - "Mean win rate - Summarization metrics": "{\"description\": \"\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.509, - "details": { - "description": "min=0.32, mean=0.509, max=0.79, sum=2.545 (5)", - "tab": "Accuracy", - "MMLU - ECE (10-bin)": "{\"description\": \"5 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "MMLU - EM (Robustness)": "{\"description\": \"min=0.26, mean=0.457, max=0.76, sum=2.283 (5)\", \"tab\": \"Robustness\", \"score\": \"0.4566315789473684\"}", - "MMLU - EM (Fairness)": "{\"description\": \"min=0.272, mean=0.48, max=0.78, sum=2.402 (5)\", \"tab\": \"Fairness\", \"score\": \"0.4803859649122807\"}", - "MMLU - Denoised inference time (s)": "{\"description\": \"5 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=389.6, mean=500.12, max=664.281, sum=2500.601 (5)\", \"tab\": \"General information\", \"score\": \"500.12014035087725\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "MMLU - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "BoolQ", - "source_data": { - "dataset_name": "BoolQ", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on BoolQ", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.819, - "details": { - "description": "min=0.819, mean=0.819, max=0.819, sum=0.819 (1)", - "tab": "Accuracy", - "BoolQ - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "BoolQ - EM (Robustness)": "{\"description\": \"min=0.763, mean=0.763, max=0.763, sum=0.763 (1)\", \"tab\": \"Robustness\", \"score\": \"0.763\"}", - "BoolQ - EM (Fairness)": "{\"description\": \"min=0.783, mean=0.783, max=0.783, sum=0.783 (1)\", \"tab\": \"Fairness\", \"score\": \"0.783\"}", - "BoolQ - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "BoolQ - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "BoolQ - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "BoolQ - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "BoolQ - # prompt tokens": "{\"description\": \"min=1284.629, mean=1284.629, max=1284.629, sum=1284.629 (1)\", \"tab\": \"General information\", \"score\": \"1284.629\"}", - "BoolQ - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "BoolQ - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "BoolQ - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.673, - "details": { - "description": "min=0.673, mean=0.673, max=0.673, sum=0.673 (1)", - "tab": "Accuracy", - "NarrativeQA - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "NarrativeQA - F1 (Robustness)": "{\"description\": \"min=0.557, mean=0.557, max=0.557, sum=0.557 (1)\", \"tab\": \"Robustness\", \"score\": \"0.5574684493620005\"}", - "NarrativeQA - F1 (Fairness)": "{\"description\": \"min=0.559, mean=0.559, max=0.559, sum=0.559 (1)\", \"tab\": \"Fairness\", \"score\": \"0.5589601433703856\"}", - "NarrativeQA - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=2.025, mean=2.025, max=2.025, sum=2.025 (1)\", \"tab\": \"General information\", \"score\": \"2.0253521126760563\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=1694.082, mean=1694.082, max=1694.082, sum=1694.082 (1)\", \"tab\": \"General information\", \"score\": \"1694.081690140845\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NarrativeQA - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NarrativeQA - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=0.667 (1)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666667\"}", - "NarrativeQA - Stereotypes (gender)": "{\"description\": \"min=0.398, mean=0.398, max=0.398, sum=0.398 (1)\", \"tab\": \"Bias\", \"score\": \"0.39814814814814814\"}", - "NarrativeQA - Representation (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=0.667 (1)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666667\"}", - "NarrativeQA - Representation (gender)": "{\"description\": \"min=0.191, mean=0.191, max=0.191, sum=0.191 (1)\", \"tab\": \"Bias\", \"score\": \"0.19148936170212763\"}", - "NarrativeQA - Toxic fraction": "{\"description\": \"min=0.02, mean=0.02, max=0.02, sum=0.02 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.01971830985915493\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (open-book)", - "source_data": { - "dataset_name": "NaturalQuestions (open-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (open-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.675, - "details": { - "description": "min=0.675, mean=0.675, max=0.675, sum=0.675 (1)", - "tab": "Accuracy", - "NaturalQuestions (closed-book) - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "NaturalQuestions (open-book) - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - F1 (Robustness)": "{\"description\": \"min=0.329, mean=0.329, max=0.329, sum=0.329 (1)\", \"tab\": \"Robustness\", \"score\": \"0.32850713007659726\"}", - "NaturalQuestions (open-book) - F1 (Robustness)": "{\"description\": \"min=0.593, mean=0.593, max=0.593, sum=0.593 (1)\", \"tab\": \"Robustness\", \"score\": \"0.5930765119599164\"}", - "NaturalQuestions (closed-book) - F1 (Fairness)": "{\"description\": \"min=0.338, mean=0.338, max=0.338, sum=0.338 (1)\", \"tab\": \"Fairness\", \"score\": \"0.33840782877152153\"}", - "NaturalQuestions (open-book) - F1 (Fairness)": "{\"description\": \"min=0.625, mean=0.625, max=0.625, sum=0.625 (1)\", \"tab\": \"Fairness\", \"score\": \"0.6251513417645462\"}", - "NaturalQuestions (closed-book) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NaturalQuestions (open-book) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=124.246, mean=124.246, max=124.246, sum=124.246 (1)\", \"tab\": \"General information\", \"score\": \"124.246\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NaturalQuestions (closed-book) - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.599, mean=4.599, max=4.599, sum=4.599 (1)\", \"tab\": \"General information\", \"score\": \"4.599\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.039, mean=0.039, max=0.039, sum=0.039 (1)\", \"tab\": \"General information\", \"score\": \"0.039\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1587.334, mean=1587.334, max=1587.334, sum=1587.334 (1)\", \"tab\": \"General information\", \"score\": \"1587.334\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=0.995, mean=0.995, max=0.995, sum=0.995 (1)\", \"tab\": \"General information\", \"score\": \"0.995\"}", - "NaturalQuestions (open-book) - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NaturalQuestions (closed-book) - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=0.667 (1)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666667\"}", - "NaturalQuestions (closed-book) - Stereotypes (gender)": "{\"description\": \"min=0.5, mean=0.5, max=0.5, sum=0.5 (1)\", \"tab\": \"Bias\", \"score\": \"0.5\"}", - "NaturalQuestions (closed-book) - Representation (race)": "{\"description\": \"min=0.256, mean=0.256, max=0.256, sum=0.256 (1)\", \"tab\": \"Bias\", \"score\": \"0.2556237218813906\"}", - "NaturalQuestions (closed-book) - Representation (gender)": "{\"description\": \"min=0.107, mean=0.107, max=0.107, sum=0.107 (1)\", \"tab\": \"Bias\", \"score\": \"0.10714285714285715\"}", - "NaturalQuestions (open-book) - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=0.667 (1)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "NaturalQuestions (open-book) - Stereotypes (gender)": "{\"description\": \"min=0.443, mean=0.443, max=0.443, sum=0.443 (1)\", \"tab\": \"Bias\", \"score\": \"0.4428571428571429\"}", - "NaturalQuestions (open-book) - Representation (race)": "{\"description\": \"min=0.382, mean=0.382, max=0.382, sum=0.382 (1)\", \"tab\": \"Bias\", \"score\": \"0.38245614035087716\"}", - "NaturalQuestions (open-book) - Representation (gender)": "{\"description\": \"min=0.132, mean=0.132, max=0.132, sum=0.132 (1)\", \"tab\": \"Bias\", \"score\": \"0.13157894736842105\"}", - "NaturalQuestions (closed-book) - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}", - "NaturalQuestions (open-book) - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "QuAC", - "source_data": { - "dataset_name": "QuAC", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on QuAC", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.307, - "details": { - "description": "min=0.307, mean=0.307, max=0.307, sum=0.307 (1)", - "tab": "Accuracy", - "QuAC - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "QuAC - F1 (Robustness)": "{\"description\": \"min=0.162, mean=0.162, max=0.162, sum=0.162 (1)\", \"tab\": \"Robustness\", \"score\": \"0.16237264946195393\"}", - "QuAC - F1 (Fairness)": "{\"description\": \"min=0.256, mean=0.256, max=0.256, sum=0.256 (1)\", \"tab\": \"Fairness\", \"score\": \"0.25646510454177246\"}", - "QuAC - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "QuAC - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "QuAC - # train": "{\"description\": \"min=0.862, mean=0.862, max=0.862, sum=0.862 (1)\", \"tab\": \"General information\", \"score\": \"0.862\"}", - "QuAC - truncated": "{\"description\": \"min=0.031, mean=0.031, max=0.031, sum=0.031 (1)\", \"tab\": \"General information\", \"score\": \"0.031\"}", - "QuAC - # prompt tokens": "{\"description\": \"min=1667.28, mean=1667.28, max=1667.28, sum=1667.28 (1)\", \"tab\": \"General information\", \"score\": \"1667.28\"}", - "QuAC - # output tokens": "{\"description\": \"min=0.999, mean=0.999, max=0.999, sum=0.999 (1)\", \"tab\": \"General information\", \"score\": \"0.999\"}", - "QuAC - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "QuAC - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "QuAC - Stereotypes (gender)": "{\"description\": \"min=0.468, mean=0.468, max=0.468, sum=0.468 (1)\", \"tab\": \"Bias\", \"score\": \"0.4681547619047619\"}", - "QuAC - Representation (race)": "{\"description\": \"min=0.423, mean=0.423, max=0.423, sum=0.423 (1)\", \"tab\": \"Bias\", \"score\": \"0.42342342342342343\"}", - "QuAC - Representation (gender)": "{\"description\": \"min=0.141, mean=0.141, max=0.141, sum=0.141 (1)\", \"tab\": \"Bias\", \"score\": \"0.141304347826087\"}", - "QuAC - Toxic fraction": "{\"description\": \"min=0.002, mean=0.002, max=0.002, sum=0.002 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.002\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "HellaSwag", - "source_data": { - "dataset_name": "HellaSwag", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on HellaSwag", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "HellaSwag - ECE (10-bin)": "{\"description\": \"No matching runs\", \"tab\": \"Calibration\", \"score\": \"\"}", - "HellaSwag - EM (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "HellaSwag - EM (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "HellaSwag - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "HellaSwag - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "OpenbookQA - ECE (10-bin)": "{\"description\": \"No matching runs\", \"tab\": \"Calibration\", \"score\": \"\"}", - "OpenbookQA - EM (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "OpenbookQA - EM (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "OpenbookQA - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "OpenbookQA - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "TruthfulQA", - "source_data": { - "dataset_name": "TruthfulQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on TruthfulQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.353, - "details": { - "description": "min=0.353, mean=0.353, max=0.353, sum=0.353 (1)", - "tab": "Accuracy", - "TruthfulQA - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "TruthfulQA - EM (Robustness)": "{\"description\": \"min=0.303, mean=0.303, max=0.303, sum=0.303 (1)\", \"tab\": \"Robustness\", \"score\": \"0.30275229357798167\"}", - "TruthfulQA - EM (Fairness)": "{\"description\": \"min=0.292, mean=0.292, max=0.292, sum=0.292 (1)\", \"tab\": \"Fairness\", \"score\": \"0.29204892966360857\"}", - "TruthfulQA - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "TruthfulQA - # eval": "{\"description\": \"min=654, mean=654, max=654, sum=654 (1)\", \"tab\": \"General information\", \"score\": \"654.0\"}", - "TruthfulQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "TruthfulQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "TruthfulQA - # prompt tokens": "{\"description\": \"min=507.503, mean=507.503, max=507.503, sum=507.503 (1)\", \"tab\": \"General information\", \"score\": \"507.50305810397555\"}", - "TruthfulQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "TruthfulQA - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MS MARCO (TREC)", - "source_data": { - "dataset_name": "MS MARCO (TREC)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "NDCG@10 on MS MARCO (TREC)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "MS MARCO (regular) - RR@10 (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "MS MARCO (TREC) - NDCG@10 (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "MS MARCO (regular) - RR@10 (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "MS MARCO (TREC) - NDCG@10 (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "MS MARCO (regular) - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MS MARCO (TREC) - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MS MARCO (regular) - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "MS MARCO (TREC) - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CNN/DailyMail", - "source_data": { - "dataset_name": "CNN/DailyMail", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on CNN/DailyMail", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "CNN/DailyMail - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "CNN/DailyMail - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "CNN/DailyMail - SummaC": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - QAFactEval": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - BERTScore (F1)": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - Coverage": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - Density": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - Compression": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-faithfulness": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-relevance": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-coherence": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "XSUM", - "source_data": { - "dataset_name": "XSUM", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on XSUM", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "XSUM - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "XSUM - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "XSUM - SummaC": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - QAFactEval": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - BERTScore (F1)": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - Coverage": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - Density": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - Compression": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-faithfulness": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-relevance": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-coherence": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "IMDB", - "source_data": { - "dataset_name": "IMDB", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on IMDB", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.959, - "details": { - "description": "min=0.959, mean=0.959, max=0.959, sum=0.959 (1)", - "tab": "Accuracy", - "IMDB - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "IMDB - EM (Robustness)": "{\"description\": \"min=0.935, mean=0.935, max=0.935, sum=0.935 (1)\", \"tab\": \"Robustness\", \"score\": \"0.935\"}", - "IMDB - EM (Fairness)": "{\"description\": \"min=0.954, mean=0.954, max=0.954, sum=0.954 (1)\", \"tab\": \"Fairness\", \"score\": \"0.954\"}", - "IMDB - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "IMDB - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "IMDB - # train": "{\"description\": \"min=2.871, mean=2.871, max=2.871, sum=2.871 (1)\", \"tab\": \"General information\", \"score\": \"2.871\"}", - "IMDB - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IMDB - # prompt tokens": "{\"description\": \"min=1666.079, mean=1666.079, max=1666.079, sum=1666.079 (1)\", \"tab\": \"General information\", \"score\": \"1666.079\"}", - "IMDB - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "IMDB - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "IMDB - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CivilComments", - "source_data": { - "dataset_name": "CivilComments", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on CivilComments", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.552, - "details": { - "description": "min=0.098, mean=0.552, max=0.969, sum=9.936 (18)", - "tab": "Accuracy", - "CivilComments - ECE (10-bin)": "{\"description\": \"9 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "CivilComments - EM (Robustness)": "{\"description\": \"min=0.037, mean=0.412, max=0.827, sum=7.414 (18)\", \"tab\": \"Robustness\", \"score\": \"0.4118677862671613\"}", - "CivilComments - EM (Fairness)": "{\"description\": \"min=0.098, mean=0.292, max=0.594, sum=5.248 (18)\", \"tab\": \"Fairness\", \"score\": \"0.29157916197633543\"}", - "CivilComments - Denoised inference time (s)": "{\"description\": \"9 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "CivilComments - # eval": "{\"description\": \"min=74, mean=371.556, max=683, sum=6688 (18)\", \"tab\": \"General information\", \"score\": \"371.55555555555554\"}", - "CivilComments - # train": "{\"description\": \"min=5, mean=5, max=5, sum=90 (18)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CivilComments - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (18)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CivilComments - # prompt tokens": "{\"description\": \"min=367.585, mean=782.759, max=1312.924, sum=14089.663 (18)\", \"tab\": \"General information\", \"score\": \"782.7590374602355\"}", - "CivilComments - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=18 (18)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "CivilComments - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=18 (18)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "CivilComments - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Toxic fraction": "{\"description\": \"9 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "RAFT", - "source_data": { - "dataset_name": "RAFT", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on RAFT", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.661, - "details": { - "description": "min=0.2, mean=0.661, max=0.975, sum=7.275 (11)", - "tab": "Accuracy", - "RAFT - ECE (10-bin)": "{\"description\": \"11 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "RAFT - EM (Robustness)": "{\"description\": \"min=0, mean=0.586, max=0.975, sum=6.45 (11)\", \"tab\": \"Robustness\", \"score\": \"0.5863636363636363\"}", - "RAFT - EM (Fairness)": "{\"description\": \"min=0.15, mean=0.611, max=0.975, sum=6.725 (11)\", \"tab\": \"Fairness\", \"score\": \"0.6113636363636364\"}", - "RAFT - Denoised inference time (s)": "{\"description\": \"11 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "RAFT - # eval": "{\"description\": \"min=40, mean=40, max=40, sum=440 (11)\", \"tab\": \"General information\", \"score\": \"40.0\"}", - "RAFT - # train": "{\"description\": \"min=0.7, mean=4.6, max=5, sum=50.6 (11)\", \"tab\": \"General information\", \"score\": \"4.6000000000000005\"}", - "RAFT - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (11)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "RAFT - # prompt tokens": "{\"description\": \"min=289.025, mean=877.464, max=1772.5, sum=9652.1 (11)\", \"tab\": \"General information\", \"score\": \"877.4636363636364\"}", - "RAFT - # output tokens": "{\"description\": \"min=0.7, mean=0.973, max=1, sum=10.7 (11)\", \"tab\": \"General information\", \"score\": \"0.9727272727272727\"}", - "RAFT - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=11 (11)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "RAFT - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Toxic fraction": "{\"description\": \"11 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_lite/tiiuae_falcon-40b/1774096306.427425", - "retrieved_timestamp": "1774096306.427425", - "source_metadata": { - "source_name": "helm_lite", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_lite", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_lite", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.217, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.086729088639201\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.671, - "details": { - "description": "min=0.671, mean=0.671, max=0.671, sum=0.671 (1)", - "tab": "Accuracy", - "NarrativeQA - Observed inference time (s)": "{\"description\": \"min=4.985, mean=4.985, max=4.985, sum=4.985 (1)\", \"tab\": \"Efficiency\", \"score\": \"4.985411514362819\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=2.023, mean=2.023, max=2.023, sum=2.023 (1)\", \"tab\": \"General information\", \"score\": \"2.0225352112676056\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=1692.33, mean=1692.33, max=1692.33, sum=1692.33 (1)\", \"tab\": \"General information\", \"score\": \"1692.3295774647888\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (closed-book)", - "source_data": { - "dataset_name": "NaturalQuestions (closed-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (closed-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.392, - "details": { - "description": "min=0.392, mean=0.392, max=0.392, sum=0.392 (1)", - "tab": "Accuracy", - "NaturalQuestions (open-book) - Observed inference time (s)": "{\"description\": \"min=3.184, mean=3.184, max=3.184, sum=3.184 (1)\", \"tab\": \"Efficiency\", \"score\": \"3.184468511581421\"}", - "NaturalQuestions (closed-book) - Observed inference time (s)": "{\"description\": \"min=2.849, mean=2.849, max=2.849, sum=2.849 (1)\", \"tab\": \"Efficiency\", \"score\": \"2.848947753429413\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.598, mean=4.598, max=4.598, sum=4.598 (1)\", \"tab\": \"General information\", \"score\": \"4.598\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.039, mean=0.039, max=0.039, sum=0.039 (1)\", \"tab\": \"General information\", \"score\": \"0.039\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1586.717, mean=1586.717, max=1586.717, sum=1586.717 (1)\", \"tab\": \"General information\", \"score\": \"1586.717\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=0.991, mean=0.991, max=0.991, sum=0.991 (1)\", \"tab\": \"General information\", \"score\": \"0.991\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=124.246, mean=124.246, max=124.246, sum=124.246 (1)\", \"tab\": \"General information\", \"score\": \"124.246\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "mode": "\"closedbook\"" - } - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.662, - "details": { - "description": "min=0.662, mean=0.662, max=0.662, sum=0.662 (1)", - "tab": "Accuracy", - "OpenbookQA - Observed inference time (s)": "{\"description\": \"min=1.268, mean=1.268, max=1.268, sum=1.268 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.268236391544342\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=251.174, mean=251.174, max=251.174, sum=251.174 (1)\", \"tab\": \"General information\", \"score\": \"251.174\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "dataset": "\"openbookqa\"", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.507, - "details": { - "description": "min=0.31, mean=0.507, max=0.79, sum=2.535 (5)", - "tab": "Accuracy", - "MMLU - Observed inference time (s)": "{\"description\": \"min=1.176, mean=1.431, max=1.805, sum=7.154 (5)\", \"tab\": \"Efficiency\", \"score\": \"1.4308063889804639\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=389.6, mean=500.12, max=664.281, sum=2500.601 (5)\", \"tab\": \"General information\", \"score\": \"500.12014035087725\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"college_chemistry\", \"computer_security\", \"econometrics\", \"us_foreign_policy\"]", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MATH", - "source_data": { - "dataset_name": "MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Equivalent (CoT) on MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.128, - "details": { - "description": "min=0.019, mean=0.128, max=0.228, sum=0.893 (7)", - "tab": "Accuracy", - "MATH - Observed inference time (s)": "{\"description\": \"min=7.555, mean=11.414, max=18.723, sum=79.896 (7)\", \"tab\": \"Efficiency\", \"score\": \"11.413689562224084\"}", - "MATH - # eval": "{\"description\": \"min=30, mean=62.429, max=135, sum=437 (7)\", \"tab\": \"General information\", \"score\": \"62.42857142857143\"}", - "MATH - # train": "{\"description\": \"min=2.385, mean=6.818, max=8, sum=47.727 (7)\", \"tab\": \"General information\", \"score\": \"6.818102949681896\"}", - "MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MATH - # prompt tokens": "{\"description\": \"min=965.096, mean=1150.049, max=1495.447, sum=8050.346 (7)\", \"tab\": \"General information\", \"score\": \"1150.0493709178531\"}", - "MATH - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=7 (7)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"algebra\", \"counting_and_probability\", \"geometry\", \"intermediate_algebra\", \"number_theory\", \"prealgebra\", \"precalculus\"]", - "level": "\"1\"", - "use_official_examples": "\"False\"", - "use_chain_of_thought": "\"True\"" - } - } - }, - { - "evaluation_name": "GSM8K", - "source_data": { - "dataset_name": "GSM8K", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on GSM8K", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.267, - "details": { - "description": "min=0.267, mean=0.267, max=0.267, sum=0.267 (1)", - "tab": "Accuracy", - "GSM8K - Observed inference time (s)": "{\"description\": \"min=12.967, mean=12.967, max=12.967, sum=12.967 (1)\", \"tab\": \"Efficiency\", \"score\": \"12.967224577903748\"}", - "GSM8K - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "GSM8K - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "GSM8K - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GSM8K - # prompt tokens": "{\"description\": \"min=1056.967, mean=1056.967, max=1056.967, sum=1056.967 (1)\", \"tab\": \"General information\", \"score\": \"1056.967\"}", - "GSM8K - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "LegalBench", - "source_data": { - "dataset_name": "LegalBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on LegalBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.442, - "details": { - "description": "min=0.204, mean=0.442, max=0.737, sum=2.209 (5)", - "tab": "Accuracy", - "LegalBench - Observed inference time (s)": "{\"description\": \"min=1.333, mean=1.731, max=3.174, sum=8.654 (5)\", \"tab\": \"Efficiency\", \"score\": \"1.730808089747147\"}", - "LegalBench - # eval": "{\"description\": \"min=95, mean=409.4, max=1000, sum=2047 (5)\", \"tab\": \"General information\", \"score\": \"409.4\"}", - "LegalBench - # train": "{\"description\": \"min=0.265, mean=3.853, max=5, sum=19.265 (5)\", \"tab\": \"General information\", \"score\": \"3.853061224489796\"}", - "LegalBench - truncated": "{\"description\": \"min=0, mean=0.003, max=0.016, sum=0.016 (5)\", \"tab\": \"General information\", \"score\": \"0.0032653061224489797\"}", - "LegalBench - # prompt tokens": "{\"description\": \"min=211.284, mean=566.694, max=1486.482, sum=2833.468 (5)\", \"tab\": \"General information\", \"score\": \"566.6935553560819\"}", - "LegalBench - # output tokens": "{\"description\": \"min=0.876, mean=0.975, max=1, sum=4.876 (5)\", \"tab\": \"General information\", \"score\": \"0.9751020408163266\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"abercrombie\", \"corporate_lobbying\", \"function_of_decision_section\", \"international_citizenship_questions\", \"proa\"]" - } - } - }, - { - "evaluation_name": "MedQA", - "source_data": { - "dataset_name": "MedQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MedQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.419, - "details": { - "description": "min=0.419, mean=0.419, max=0.419, sum=0.419 (1)", - "tab": "Accuracy", - "MedQA - Observed inference time (s)": "{\"description\": \"min=2.203, mean=2.203, max=2.203, sum=2.203 (1)\", \"tab\": \"Efficiency\", \"score\": \"2.202825612149703\"}", - "MedQA - # eval": "{\"description\": \"min=503, mean=503, max=503, sum=503 (1)\", \"tab\": \"General information\", \"score\": \"503.0\"}", - "MedQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MedQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MedQA - # prompt tokens": "{\"description\": \"min=1048.624, mean=1048.624, max=1048.624, sum=1048.624 (1)\", \"tab\": \"General information\", \"score\": \"1048.624254473161\"}", - "MedQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WMT 2014", - "source_data": { - "dataset_name": "WMT 2014", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "BLEU-4 on WMT 2014", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.162, - "details": { - "description": "min=0.017, mean=0.162, max=0.208, sum=0.809 (5)", - "tab": "Accuracy", - "WMT 2014 - Observed inference time (s)": "{\"description\": \"min=2.468, mean=3.098, max=4.642, sum=15.491 (5)\", \"tab\": \"Efficiency\", \"score\": \"3.0981059579736714\"}", - "WMT 2014 - # eval": "{\"description\": \"min=503, mean=568.8, max=832, sum=2844 (5)\", \"tab\": \"General information\", \"score\": \"568.8\"}", - "WMT 2014 - # train": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "WMT 2014 - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WMT 2014 - # prompt tokens": "{\"description\": \"min=115.642, mean=162.454, max=224.817, sum=812.272 (5)\", \"tab\": \"General information\", \"score\": \"162.45444400902278\"}", - "WMT 2014 - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "language_pair": "[\"cs-en\", \"de-en\", \"fr-en\", \"hi-en\", \"ru-en\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "hfopenllm_v2/tiiuae_falcon-40b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2496 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4019 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0181 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2735 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3631 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2505 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/tiiuae_falcon-7b-instruct.json b/data/models/tiiuae_falcon-7b-instruct.json deleted file mode 100644 index 6167eee874047397a86eee707e9809a09a791215..0000000000000000000000000000000000000000 --- a/data/models/tiiuae_falcon-7b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "falcon-7b-instruct", - "id": "tiiuae/falcon-7b-instruct", - "developer": "tiiuae", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "FalconForCausalLM", - "params_billions": "7.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/tiiuae_falcon-7b-instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1969 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3203 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0121 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2475 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3634 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1155 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/tiiuae_falcon-7b.json b/data/models/tiiuae_falcon-7b.json deleted file mode 100644 index ac37875ab1d0daf21ade1f869b62b884ae372422..0000000000000000000000000000000000000000 --- a/data/models/tiiuae_falcon-7b.json +++ /dev/null @@ -1,1179 +0,0 @@ -{ - "model_info": { - "name": "Falcon 7B", - "id": "tiiuae/falcon-7b", - "developer": "tiiuae", - "inference_platform": "unknown", - "additional_details": { - "model_id_aliases": [ - "tiiuae/Falcon-7B" - ] - } - }, - "evaluations": [ - { - "evaluation_id": "helm_classic/tiiuae_Falcon-7B/1774096308.339228", - "retrieved_timestamp": "1774096308.339228", - "source_metadata": { - "source_name": "helm_classic", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_classic", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_classic", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperform on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.378, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Calibration": "{\"description\": \"\", \"tab\": \"Calibration\", \"score\": \"\"}", - "Mean win rate - Robustness": "{\"description\": \"\", \"tab\": \"Robustness\", \"score\": \"0.4253379953379953\"}", - "Mean win rate - Fairness": "{\"description\": \"\", \"tab\": \"Fairness\", \"score\": \"0.4469230769230769\"}", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}", - "Mean win rate - Bias": "{\"description\": \"\", \"tab\": \"Bias\", \"score\": \"0.35594420480554084\"}", - "Mean win rate - Toxicity": "{\"description\": \"\", \"tab\": \"Toxicity\", \"score\": \"0.5821678321678322\"}", - "Mean win rate - Summarization metrics": "{\"description\": \"\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.286, - "details": { - "description": "min=0.17, mean=0.286, max=0.39, sum=1.432 (5)", - "tab": "Accuracy", - "MMLU - ECE (10-bin)": "{\"description\": \"5 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "MMLU - EM (Robustness)": "{\"description\": \"min=0.13, mean=0.236, max=0.37, sum=1.181 (5)\", \"tab\": \"Robustness\", \"score\": \"0.23610526315789473\"}", - "MMLU - EM (Fairness)": "{\"description\": \"min=0.15, mean=0.261, max=0.33, sum=1.303 (5)\", \"tab\": \"Fairness\", \"score\": \"0.26063157894736844\"}", - "MMLU - Denoised inference time (s)": "{\"description\": \"5 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=389.6, mean=500.12, max=664.281, sum=2500.601 (5)\", \"tab\": \"General information\", \"score\": \"500.12014035087725\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "MMLU - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "BoolQ", - "source_data": { - "dataset_name": "BoolQ", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on BoolQ", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.753, - "details": { - "description": "min=0.753, mean=0.753, max=0.753, sum=0.753 (1)", - "tab": "Accuracy", - "BoolQ - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "BoolQ - EM (Robustness)": "{\"description\": \"min=0.65, mean=0.65, max=0.65, sum=0.65 (1)\", \"tab\": \"Robustness\", \"score\": \"0.65\"}", - "BoolQ - EM (Fairness)": "{\"description\": \"min=0.702, mean=0.702, max=0.702, sum=0.702 (1)\", \"tab\": \"Fairness\", \"score\": \"0.702\"}", - "BoolQ - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "BoolQ - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "BoolQ - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "BoolQ - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "BoolQ - # prompt tokens": "{\"description\": \"min=1284.629, mean=1284.629, max=1284.629, sum=1284.629 (1)\", \"tab\": \"General information\", \"score\": \"1284.629\"}", - "BoolQ - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "BoolQ - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "BoolQ - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.621, - "details": { - "description": "min=0.621, mean=0.621, max=0.621, sum=0.621 (1)", - "tab": "Accuracy", - "NarrativeQA - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "NarrativeQA - F1 (Robustness)": "{\"description\": \"min=0.436, mean=0.436, max=0.436, sum=0.436 (1)\", \"tab\": \"Robustness\", \"score\": \"0.4358401092976052\"}", - "NarrativeQA - F1 (Fairness)": "{\"description\": \"min=0.52, mean=0.52, max=0.52, sum=0.52 (1)\", \"tab\": \"Fairness\", \"score\": \"0.5199130399003071\"}", - "NarrativeQA - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=2.025, mean=2.025, max=2.025, sum=2.025 (1)\", \"tab\": \"General information\", \"score\": \"2.0253521126760563\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=1694.082, mean=1694.082, max=1694.082, sum=1694.082 (1)\", \"tab\": \"General information\", \"score\": \"1694.081690140845\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NarrativeQA - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NarrativeQA - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NarrativeQA - Stereotypes (gender)": "{\"description\": \"min=0.444, mean=0.444, max=0.444, sum=0.444 (1)\", \"tab\": \"Bias\", \"score\": \"0.4444444444444444\"}", - "NarrativeQA - Representation (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=0.667 (1)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666667\"}", - "NarrativeQA - Representation (gender)": "{\"description\": \"min=0.205, mean=0.205, max=0.205, sum=0.205 (1)\", \"tab\": \"Bias\", \"score\": \"0.2046979865771812\"}", - "NarrativeQA - Toxic fraction": "{\"description\": \"min=0.017, mean=0.017, max=0.017, sum=0.017 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.016901408450704224\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (open-book)", - "source_data": { - "dataset_name": "NaturalQuestions (open-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (open-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.579, - "details": { - "description": "min=0.579, mean=0.579, max=0.579, sum=0.579 (1)", - "tab": "Accuracy", - "NaturalQuestions (closed-book) - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "NaturalQuestions (open-book) - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - F1 (Robustness)": "{\"description\": \"min=0.185, mean=0.185, max=0.185, sum=0.185 (1)\", \"tab\": \"Robustness\", \"score\": \"0.18513134554094532\"}", - "NaturalQuestions (open-book) - F1 (Robustness)": "{\"description\": \"min=0.489, mean=0.489, max=0.489, sum=0.489 (1)\", \"tab\": \"Robustness\", \"score\": \"0.4889733445855735\"}", - "NaturalQuestions (closed-book) - F1 (Fairness)": "{\"description\": \"min=0.233, mean=0.233, max=0.233, sum=0.233 (1)\", \"tab\": \"Fairness\", \"score\": \"0.2334955595363806\"}", - "NaturalQuestions (open-book) - F1 (Fairness)": "{\"description\": \"min=0.537, mean=0.537, max=0.537, sum=0.537 (1)\", \"tab\": \"Fairness\", \"score\": \"0.536571121609654\"}", - "NaturalQuestions (closed-book) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NaturalQuestions (open-book) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=124.246, mean=124.246, max=124.246, sum=124.246 (1)\", \"tab\": \"General information\", \"score\": \"124.246\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NaturalQuestions (closed-book) - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.599, mean=4.599, max=4.599, sum=4.599 (1)\", \"tab\": \"General information\", \"score\": \"4.599\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.039, mean=0.039, max=0.039, sum=0.039 (1)\", \"tab\": \"General information\", \"score\": \"0.039\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1587.334, mean=1587.334, max=1587.334, sum=1587.334 (1)\", \"tab\": \"General information\", \"score\": \"1587.334\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=0.994, mean=0.994, max=0.994, sum=0.994 (1)\", \"tab\": \"General information\", \"score\": \"0.994\"}", - "NaturalQuestions (open-book) - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NaturalQuestions (closed-book) - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - Stereotypes (gender)": "{\"description\": \"min=0.389, mean=0.389, max=0.389, sum=0.389 (1)\", \"tab\": \"Bias\", \"score\": \"0.38888888888888884\"}", - "NaturalQuestions (closed-book) - Representation (race)": "{\"description\": \"min=0.476, mean=0.476, max=0.476, sum=0.476 (1)\", \"tab\": \"Bias\", \"score\": \"0.47619047619047616\"}", - "NaturalQuestions (closed-book) - Representation (gender)": "{\"description\": \"min=0.14, mean=0.14, max=0.14, sum=0.14 (1)\", \"tab\": \"Bias\", \"score\": \"0.14\"}", - "NaturalQuestions (open-book) - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NaturalQuestions (open-book) - Stereotypes (gender)": "{\"description\": \"min=0.333, mean=0.333, max=0.333, sum=0.333 (1)\", \"tab\": \"Bias\", \"score\": \"0.3333333333333333\"}", - "NaturalQuestions (open-book) - Representation (race)": "{\"description\": \"min=0.553, mean=0.553, max=0.553, sum=0.553 (1)\", \"tab\": \"Bias\", \"score\": \"0.5528942115768464\"}", - "NaturalQuestions (open-book) - Representation (gender)": "{\"description\": \"min=0.275, mean=0.275, max=0.275, sum=0.275 (1)\", \"tab\": \"Bias\", \"score\": \"0.2745098039215687\"}", - "NaturalQuestions (closed-book) - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}", - "NaturalQuestions (open-book) - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "QuAC", - "source_data": { - "dataset_name": "QuAC", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on QuAC", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.332, - "details": { - "description": "min=0.332, mean=0.332, max=0.332, sum=0.332 (1)", - "tab": "Accuracy", - "QuAC - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "QuAC - F1 (Robustness)": "{\"description\": \"min=0.164, mean=0.164, max=0.164, sum=0.164 (1)\", \"tab\": \"Robustness\", \"score\": \"0.16389145934637706\"}", - "QuAC - F1 (Fairness)": "{\"description\": \"min=0.262, mean=0.262, max=0.262, sum=0.262 (1)\", \"tab\": \"Fairness\", \"score\": \"0.2622208848575014\"}", - "QuAC - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "QuAC - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "QuAC - # train": "{\"description\": \"min=0.862, mean=0.862, max=0.862, sum=0.862 (1)\", \"tab\": \"General information\", \"score\": \"0.862\"}", - "QuAC - truncated": "{\"description\": \"min=0.031, mean=0.031, max=0.031, sum=0.031 (1)\", \"tab\": \"General information\", \"score\": \"0.031\"}", - "QuAC - # prompt tokens": "{\"description\": \"min=1667.28, mean=1667.28, max=1667.28, sum=1667.28 (1)\", \"tab\": \"General information\", \"score\": \"1667.28\"}", - "QuAC - # output tokens": "{\"description\": \"min=0.995, mean=0.995, max=0.995, sum=0.995 (1)\", \"tab\": \"General information\", \"score\": \"0.995\"}", - "QuAC - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "QuAC - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=0.667 (1)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "QuAC - Stereotypes (gender)": "{\"description\": \"min=0.457, mean=0.457, max=0.457, sum=0.457 (1)\", \"tab\": \"Bias\", \"score\": \"0.45680272108843534\"}", - "QuAC - Representation (race)": "{\"description\": \"min=0.402, mean=0.402, max=0.402, sum=0.402 (1)\", \"tab\": \"Bias\", \"score\": \"0.4022988505747127\"}", - "QuAC - Representation (gender)": "{\"description\": \"min=0.247, mean=0.247, max=0.247, sum=0.247 (1)\", \"tab\": \"Bias\", \"score\": \"0.24695863746958635\"}", - "QuAC - Toxic fraction": "{\"description\": \"min=0.001, mean=0.001, max=0.001, sum=0.001 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.001\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "HellaSwag", - "source_data": { - "dataset_name": "HellaSwag", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on HellaSwag", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "HellaSwag - ECE (10-bin)": "{\"description\": \"No matching runs\", \"tab\": \"Calibration\", \"score\": \"\"}", - "HellaSwag - EM (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "HellaSwag - EM (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "HellaSwag - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "HellaSwag - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "OpenbookQA - ECE (10-bin)": "{\"description\": \"No matching runs\", \"tab\": \"Calibration\", \"score\": \"\"}", - "OpenbookQA - EM (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "OpenbookQA - EM (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "OpenbookQA - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "OpenbookQA - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "TruthfulQA", - "source_data": { - "dataset_name": "TruthfulQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on TruthfulQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.234, - "details": { - "description": "min=0.234, mean=0.234, max=0.234, sum=0.234 (1)", - "tab": "Accuracy", - "TruthfulQA - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "TruthfulQA - EM (Robustness)": "{\"description\": \"min=0.205, mean=0.205, max=0.205, sum=0.205 (1)\", \"tab\": \"Robustness\", \"score\": \"0.20489296636085627\"}", - "TruthfulQA - EM (Fairness)": "{\"description\": \"min=0.213, mean=0.213, max=0.213, sum=0.213 (1)\", \"tab\": \"Fairness\", \"score\": \"0.21253822629969418\"}", - "TruthfulQA - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "TruthfulQA - # eval": "{\"description\": \"min=654, mean=654, max=654, sum=654 (1)\", \"tab\": \"General information\", \"score\": \"654.0\"}", - "TruthfulQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "TruthfulQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "TruthfulQA - # prompt tokens": "{\"description\": \"min=507.503, mean=507.503, max=507.503, sum=507.503 (1)\", \"tab\": \"General information\", \"score\": \"507.50305810397555\"}", - "TruthfulQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "TruthfulQA - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MS MARCO (TREC)", - "source_data": { - "dataset_name": "MS MARCO (TREC)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "NDCG@10 on MS MARCO (TREC)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "MS MARCO (regular) - RR@10 (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "MS MARCO (TREC) - NDCG@10 (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "MS MARCO (regular) - RR@10 (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "MS MARCO (TREC) - NDCG@10 (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "MS MARCO (regular) - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MS MARCO (TREC) - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MS MARCO (regular) - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "MS MARCO (TREC) - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CNN/DailyMail", - "source_data": { - "dataset_name": "CNN/DailyMail", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on CNN/DailyMail", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "CNN/DailyMail - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "CNN/DailyMail - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "CNN/DailyMail - SummaC": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - QAFactEval": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - BERTScore (F1)": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - Coverage": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - Density": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - Compression": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-faithfulness": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-relevance": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-coherence": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "XSUM", - "source_data": { - "dataset_name": "XSUM", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on XSUM", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "XSUM - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "XSUM - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "XSUM - SummaC": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - QAFactEval": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - BERTScore (F1)": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - Coverage": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - Density": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - Compression": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-faithfulness": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-relevance": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-coherence": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "IMDB", - "source_data": { - "dataset_name": "IMDB", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on IMDB", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.836, - "details": { - "description": "min=0.836, mean=0.836, max=0.836, sum=0.836 (1)", - "tab": "Accuracy", - "IMDB - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "IMDB - EM (Robustness)": "{\"description\": \"min=0.692, mean=0.692, max=0.692, sum=0.692 (1)\", \"tab\": \"Robustness\", \"score\": \"0.692\"}", - "IMDB - EM (Fairness)": "{\"description\": \"min=0.794, mean=0.794, max=0.794, sum=0.794 (1)\", \"tab\": \"Fairness\", \"score\": \"0.794\"}", - "IMDB - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "IMDB - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "IMDB - # train": "{\"description\": \"min=2.871, mean=2.871, max=2.871, sum=2.871 (1)\", \"tab\": \"General information\", \"score\": \"2.871\"}", - "IMDB - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IMDB - # prompt tokens": "{\"description\": \"min=1666.079, mean=1666.079, max=1666.079, sum=1666.079 (1)\", \"tab\": \"General information\", \"score\": \"1666.079\"}", - "IMDB - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "IMDB - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "IMDB - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CivilComments", - "source_data": { - "dataset_name": "CivilComments", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on CivilComments", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.514, - "details": { - "description": "min=0, mean=0.514, max=0.999, sum=9.257 (18)", - "tab": "Accuracy", - "CivilComments - ECE (10-bin)": "{\"description\": \"9 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "CivilComments - EM (Robustness)": "{\"description\": \"min=0, mean=0.485, max=0.999, sum=8.731 (18)\", \"tab\": \"Robustness\", \"score\": \"0.4850751828621894\"}", - "CivilComments - EM (Fairness)": "{\"description\": \"min=0, mean=0.494, max=0.999, sum=8.898 (18)\", \"tab\": \"Fairness\", \"score\": \"0.49430637095445207\"}", - "CivilComments - Denoised inference time (s)": "{\"description\": \"9 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "CivilComments - # eval": "{\"description\": \"min=74, mean=371.556, max=683, sum=6688 (18)\", \"tab\": \"General information\", \"score\": \"371.55555555555554\"}", - "CivilComments - # train": "{\"description\": \"min=5, mean=5, max=5, sum=90 (18)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CivilComments - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (18)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CivilComments - # prompt tokens": "{\"description\": \"min=367.585, mean=782.759, max=1312.924, sum=14089.663 (18)\", \"tab\": \"General information\", \"score\": \"782.7590374602355\"}", - "CivilComments - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=18 (18)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "CivilComments - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=18 (18)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "CivilComments - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Toxic fraction": "{\"description\": \"9 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "RAFT", - "source_data": { - "dataset_name": "RAFT", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on RAFT", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.602, - "details": { - "description": "min=0.15, mean=0.602, max=0.975, sum=6.625 (11)", - "tab": "Accuracy", - "RAFT - ECE (10-bin)": "{\"description\": \"11 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "RAFT - EM (Robustness)": "{\"description\": \"min=0.025, mean=0.516, max=0.975, sum=5.675 (11)\", \"tab\": \"Robustness\", \"score\": \"0.5159090909090908\"}", - "RAFT - EM (Fairness)": "{\"description\": \"min=0.15, mean=0.555, max=0.975, sum=6.1 (11)\", \"tab\": \"Fairness\", \"score\": \"0.5545454545454546\"}", - "RAFT - Denoised inference time (s)": "{\"description\": \"11 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "RAFT - # eval": "{\"description\": \"min=40, mean=40, max=40, sum=440 (11)\", \"tab\": \"General information\", \"score\": \"40.0\"}", - "RAFT - # train": "{\"description\": \"min=0.7, mean=4.6, max=5, sum=50.6 (11)\", \"tab\": \"General information\", \"score\": \"4.6000000000000005\"}", - "RAFT - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (11)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "RAFT - # prompt tokens": "{\"description\": \"min=289.025, mean=877.464, max=1772.5, sum=9652.1 (11)\", \"tab\": \"General information\", \"score\": \"877.4636363636364\"}", - "RAFT - # output tokens": "{\"description\": \"min=0.725, mean=0.975, max=1, sum=10.725 (11)\", \"tab\": \"General information\", \"score\": \"0.975\"}", - "RAFT - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=11 (11)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "RAFT - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Toxic fraction": "{\"description\": \"11 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_lite/tiiuae_falcon-7b/1774096306.427425", - "retrieved_timestamp": "1774096306.427425", - "source_metadata": { - "source_name": "helm_lite", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_lite", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_lite", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.064, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.36905118601747816\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.621, - "details": { - "description": "min=0.621, mean=0.621, max=0.621, sum=0.621 (1)", - "tab": "Accuracy", - "NarrativeQA - Observed inference time (s)": "{\"description\": \"min=1.141, mean=1.141, max=1.141, sum=1.141 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.1411562691272144\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=2.023, mean=2.023, max=2.023, sum=2.023 (1)\", \"tab\": \"General information\", \"score\": \"2.0225352112676056\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=1692.33, mean=1692.33, max=1692.33, sum=1692.33 (1)\", \"tab\": \"General information\", \"score\": \"1692.3295774647888\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (closed-book)", - "source_data": { - "dataset_name": "NaturalQuestions (closed-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (closed-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.285, - "details": { - "description": "min=0.285, mean=0.285, max=0.285, sum=0.285 (1)", - "tab": "Accuracy", - "NaturalQuestions (open-book) - Observed inference time (s)": "{\"description\": \"min=1.009, mean=1.009, max=1.009, sum=1.009 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.0090243232250213\"}", - "NaturalQuestions (closed-book) - Observed inference time (s)": "{\"description\": \"min=0.876, mean=0.876, max=0.876, sum=0.876 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.8758702797889709\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.598, mean=4.598, max=4.598, sum=4.598 (1)\", \"tab\": \"General information\", \"score\": \"4.598\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.039, mean=0.039, max=0.039, sum=0.039 (1)\", \"tab\": \"General information\", \"score\": \"0.039\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1586.717, mean=1586.717, max=1586.717, sum=1586.717 (1)\", \"tab\": \"General information\", \"score\": \"1586.717\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=0.99, mean=0.99, max=0.99, sum=0.99 (1)\", \"tab\": \"General information\", \"score\": \"0.99\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=124.246, mean=124.246, max=124.246, sum=124.246 (1)\", \"tab\": \"General information\", \"score\": \"124.246\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "mode": "\"closedbook\"" - } - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.26, - "details": { - "description": "min=0.26, mean=0.26, max=0.26, sum=0.26 (1)", - "tab": "Accuracy", - "OpenbookQA - Observed inference time (s)": "{\"description\": \"min=0.412, mean=0.412, max=0.412, sum=0.412 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.4118037748336792\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=251.174, mean=251.174, max=251.174, sum=251.174 (1)\", \"tab\": \"General information\", \"score\": \"251.174\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "dataset": "\"openbookqa\"", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.288, - "details": { - "description": "min=0.17, mean=0.288, max=0.39, sum=1.441 (5)", - "tab": "Accuracy", - "MMLU - Observed inference time (s)": "{\"description\": \"min=0.434, mean=0.475, max=0.497, sum=2.373 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.47453500427279555\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=389.6, mean=500.12, max=664.281, sum=2500.601 (5)\", \"tab\": \"General information\", \"score\": \"500.12014035087725\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"college_chemistry\", \"computer_security\", \"econometrics\", \"us_foreign_policy\"]", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MATH", - "source_data": { - "dataset_name": "MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Equivalent (CoT) on MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.044, - "details": { - "description": "min=0, mean=0.044, max=0.105, sum=0.307 (7)", - "tab": "Accuracy", - "MATH - Observed inference time (s)": "{\"description\": \"min=5.445, mean=6.987, max=10.873, sum=48.91 (7)\", \"tab\": \"Efficiency\", \"score\": \"6.987098801445013\"}", - "MATH - # eval": "{\"description\": \"min=30, mean=62.429, max=135, sum=437 (7)\", \"tab\": \"General information\", \"score\": \"62.42857142857143\"}", - "MATH - # train": "{\"description\": \"min=2.385, mean=6.818, max=8, sum=47.727 (7)\", \"tab\": \"General information\", \"score\": \"6.818102949681896\"}", - "MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MATH - # prompt tokens": "{\"description\": \"min=965.096, mean=1150.049, max=1495.447, sum=8050.346 (7)\", \"tab\": \"General information\", \"score\": \"1150.0493709178531\"}", - "MATH - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=7 (7)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"algebra\", \"counting_and_probability\", \"geometry\", \"intermediate_algebra\", \"number_theory\", \"prealgebra\", \"precalculus\"]", - "level": "\"1\"", - "use_official_examples": "\"False\"", - "use_chain_of_thought": "\"True\"" - } - } - }, - { - "evaluation_name": "GSM8K", - "source_data": { - "dataset_name": "GSM8K", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on GSM8K", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.055, - "details": { - "description": "min=0.055, mean=0.055, max=0.055, sum=0.055 (1)", - "tab": "Accuracy", - "GSM8K - Observed inference time (s)": "{\"description\": \"min=6.94, mean=6.94, max=6.94, sum=6.94 (1)\", \"tab\": \"Efficiency\", \"score\": \"6.940216990470886\"}", - "GSM8K - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "GSM8K - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "GSM8K - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GSM8K - # prompt tokens": "{\"description\": \"min=1056.967, mean=1056.967, max=1056.967, sum=1056.967 (1)\", \"tab\": \"General information\", \"score\": \"1056.967\"}", - "GSM8K - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "LegalBench", - "source_data": { - "dataset_name": "LegalBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on LegalBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.346, - "details": { - "description": "min=0.12, mean=0.346, max=0.558, sum=1.731 (5)", - "tab": "Accuracy", - "LegalBench - Observed inference time (s)": "{\"description\": \"min=0.453, mean=0.628, max=1.041, sum=3.139 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.6278266410596228\"}", - "LegalBench - # eval": "{\"description\": \"min=95, mean=409.4, max=1000, sum=2047 (5)\", \"tab\": \"General information\", \"score\": \"409.4\"}", - "LegalBench - # train": "{\"description\": \"min=0.265, mean=3.853, max=5, sum=19.265 (5)\", \"tab\": \"General information\", \"score\": \"3.853061224489796\"}", - "LegalBench - truncated": "{\"description\": \"min=0, mean=0.003, max=0.016, sum=0.016 (5)\", \"tab\": \"General information\", \"score\": \"0.0032653061224489797\"}", - "LegalBench - # prompt tokens": "{\"description\": \"min=211.284, mean=566.694, max=1486.482, sum=2833.468 (5)\", \"tab\": \"General information\", \"score\": \"566.6935553560819\"}", - "LegalBench - # output tokens": "{\"description\": \"min=0.982, mean=0.996, max=1, sum=4.982 (5)\", \"tab\": \"General information\", \"score\": \"0.9963265306122449\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"abercrombie\", \"corporate_lobbying\", \"function_of_decision_section\", \"international_citizenship_questions\", \"proa\"]" - } - } - }, - { - "evaluation_name": "MedQA", - "source_data": { - "dataset_name": "MedQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MedQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.254, - "details": { - "description": "min=0.254, mean=0.254, max=0.254, sum=0.254 (1)", - "tab": "Accuracy", - "MedQA - Observed inference time (s)": "{\"description\": \"min=0.735, mean=0.735, max=0.735, sum=0.735 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.7352914724861889\"}", - "MedQA - # eval": "{\"description\": \"min=503, mean=503, max=503, sum=503 (1)\", \"tab\": \"General information\", \"score\": \"503.0\"}", - "MedQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MedQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MedQA - # prompt tokens": "{\"description\": \"min=1048.624, mean=1048.624, max=1048.624, sum=1048.624 (1)\", \"tab\": \"General information\", \"score\": \"1048.624254473161\"}", - "MedQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WMT 2014", - "source_data": { - "dataset_name": "WMT 2014", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "BLEU-4 on WMT 2014", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.094, - "details": { - "description": "min=0.0, mean=0.094, max=0.186, sum=0.471 (5)", - "tab": "Accuracy", - "WMT 2014 - Observed inference time (s)": "{\"description\": \"min=1.05, mean=1.604, max=3.055, sum=8.019 (5)\", \"tab\": \"Efficiency\", \"score\": \"1.6038075838932468\"}", - "WMT 2014 - # eval": "{\"description\": \"min=503, mean=568.8, max=832, sum=2844 (5)\", \"tab\": \"General information\", \"score\": \"568.8\"}", - "WMT 2014 - # train": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "WMT 2014 - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WMT 2014 - # prompt tokens": "{\"description\": \"min=115.642, mean=162.454, max=224.817, sum=812.272 (5)\", \"tab\": \"General information\", \"score\": \"162.45444400902278\"}", - "WMT 2014 - # output tokens": "{\"description\": \"min=0.999, mean=1.0, max=1, sum=4.999 (5)\", \"tab\": \"General information\", \"score\": \"0.9997596153846153\"}" - } - }, - "generation_config": { - "additional_details": { - "language_pair": "[\"cs-en\", \"de-en\", \"fr-en\", \"hi-en\", \"ru-en\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "hfopenllm_v2/tiiuae_falcon-7b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1821 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3285 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0098 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.245 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3778 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1125 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/tiiuae_falcon-instruct-40b.json b/data/models/tiiuae_falcon-instruct-40b.json deleted file mode 100644 index 3599ba095d581fa56e21584a9e570d386543fa4c..0000000000000000000000000000000000000000 --- a/data/models/tiiuae_falcon-instruct-40b.json +++ /dev/null @@ -1,674 +0,0 @@ -{ - "model_info": { - "name": "Falcon-Instruct 40B", - "id": "tiiuae/Falcon-Instruct-40B", - "developer": "tiiuae", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_classic/tiiuae_Falcon-Instruct-40B/1774096308.339228", - "retrieved_timestamp": "1774096308.339228", - "source_metadata": { - "source_name": "helm_classic", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_classic", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_classic", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperform on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.727, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Calibration": "{\"description\": \"\", \"tab\": \"Calibration\", \"score\": \"\"}", - "Mean win rate - Robustness": "{\"description\": \"\", \"tab\": \"Robustness\", \"score\": \"0.7631002331002331\"}", - "Mean win rate - Fairness": "{\"description\": \"\", \"tab\": \"Fairness\", \"score\": \"0.7087645687645687\"}", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}", - "Mean win rate - Bias": "{\"description\": \"\", \"tab\": \"Bias\", \"score\": \"0.4307003912490803\"}", - "Mean win rate - Toxicity": "{\"description\": \"\", \"tab\": \"Toxicity\", \"score\": \"0.44994172494172496\"}", - "Mean win rate - Summarization metrics": "{\"description\": \"\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.497, - "details": { - "description": "min=0.263, mean=0.497, max=0.82, sum=2.483 (5)", - "tab": "Accuracy", - "MMLU - ECE (10-bin)": "{\"description\": \"5 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "MMLU - EM (Robustness)": "{\"description\": \"min=0.228, mean=0.446, max=0.78, sum=2.228 (5)\", \"tab\": \"Robustness\", \"score\": \"0.44561403508771924\"}", - "MMLU - EM (Fairness)": "{\"description\": \"min=0.219, mean=0.466, max=0.8, sum=2.329 (5)\", \"tab\": \"Fairness\", \"score\": \"0.4658596491228071\"}", - "MMLU - Denoised inference time (s)": "{\"description\": \"5 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=389.6, mean=500.12, max=664.281, sum=2500.601 (5)\", \"tab\": \"General information\", \"score\": \"500.12014035087725\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "MMLU - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "BoolQ", - "source_data": { - "dataset_name": "BoolQ", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on BoolQ", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.829, - "details": { - "description": "min=0.829, mean=0.829, max=0.829, sum=0.829 (1)", - "tab": "Accuracy", - "BoolQ - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "BoolQ - EM (Robustness)": "{\"description\": \"min=0.781, mean=0.781, max=0.781, sum=0.781 (1)\", \"tab\": \"Robustness\", \"score\": \"0.781\"}", - "BoolQ - EM (Fairness)": "{\"description\": \"min=0.799, mean=0.799, max=0.799, sum=0.799 (1)\", \"tab\": \"Fairness\", \"score\": \"0.799\"}", - "BoolQ - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "BoolQ - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "BoolQ - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "BoolQ - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "BoolQ - # prompt tokens": "{\"description\": \"min=1284.629, mean=1284.629, max=1284.629, sum=1284.629 (1)\", \"tab\": \"General information\", \"score\": \"1284.629\"}", - "BoolQ - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "BoolQ - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "BoolQ - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.625, - "details": { - "description": "min=0.625, mean=0.625, max=0.625, sum=0.625 (1)", - "tab": "Accuracy", - "NarrativeQA - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "NarrativeQA - F1 (Robustness)": "{\"description\": \"min=0.508, mean=0.508, max=0.508, sum=0.508 (1)\", \"tab\": \"Robustness\", \"score\": \"0.5082425698893845\"}", - "NarrativeQA - F1 (Fairness)": "{\"description\": \"min=0.543, mean=0.543, max=0.543, sum=0.543 (1)\", \"tab\": \"Fairness\", \"score\": \"0.543279669317833\"}", - "NarrativeQA - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=2.025, mean=2.025, max=2.025, sum=2.025 (1)\", \"tab\": \"General information\", \"score\": \"2.0253521126760563\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=1694.082, mean=1694.082, max=1694.082, sum=1694.082 (1)\", \"tab\": \"General information\", \"score\": \"1694.081690140845\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NarrativeQA - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NarrativeQA - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=0.667 (1)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666667\"}", - "NarrativeQA - Stereotypes (gender)": "{\"description\": \"min=0.332, mean=0.332, max=0.332, sum=0.332 (1)\", \"tab\": \"Bias\", \"score\": \"0.33194444444444443\"}", - "NarrativeQA - Representation (race)": "{\"description\": \"min=0.467, mean=0.467, max=0.467, sum=0.467 (1)\", \"tab\": \"Bias\", \"score\": \"0.4666666666666667\"}", - "NarrativeQA - Representation (gender)": "{\"description\": \"min=0.175, mean=0.175, max=0.175, sum=0.175 (1)\", \"tab\": \"Bias\", \"score\": \"0.17464114832535887\"}", - "NarrativeQA - Toxic fraction": "{\"description\": \"min=0.011, mean=0.011, max=0.011, sum=0.011 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.011267605633802818\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (open-book)", - "source_data": { - "dataset_name": "NaturalQuestions (open-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (open-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.666, - "details": { - "description": "min=0.666, mean=0.666, max=0.666, sum=0.666 (1)", - "tab": "Accuracy", - "NaturalQuestions (closed-book) - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "NaturalQuestions (open-book) - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - F1 (Robustness)": "{\"description\": \"min=0.335, mean=0.335, max=0.335, sum=0.335 (1)\", \"tab\": \"Robustness\", \"score\": \"0.33514492181201283\"}", - "NaturalQuestions (open-book) - F1 (Robustness)": "{\"description\": \"min=0.591, mean=0.591, max=0.591, sum=0.591 (1)\", \"tab\": \"Robustness\", \"score\": \"0.5912781280483248\"}", - "NaturalQuestions (closed-book) - F1 (Fairness)": "{\"description\": \"min=0.331, mean=0.331, max=0.331, sum=0.331 (1)\", \"tab\": \"Fairness\", \"score\": \"0.33094416222152356\"}", - "NaturalQuestions (open-book) - F1 (Fairness)": "{\"description\": \"min=0.607, mean=0.607, max=0.607, sum=0.607 (1)\", \"tab\": \"Fairness\", \"score\": \"0.6067807528449897\"}", - "NaturalQuestions (closed-book) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NaturalQuestions (open-book) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=124.246, mean=124.246, max=124.246, sum=124.246 (1)\", \"tab\": \"General information\", \"score\": \"124.246\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=0.999, mean=0.999, max=0.999, sum=0.999 (1)\", \"tab\": \"General information\", \"score\": \"0.999\"}", - "NaturalQuestions (closed-book) - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.599, mean=4.599, max=4.599, sum=4.599 (1)\", \"tab\": \"General information\", \"score\": \"4.599\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.039, mean=0.039, max=0.039, sum=0.039 (1)\", \"tab\": \"General information\", \"score\": \"0.039\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1587.334, mean=1587.334, max=1587.334, sum=1587.334 (1)\", \"tab\": \"General information\", \"score\": \"1587.334\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=0.995, mean=0.995, max=0.995, sum=0.995 (1)\", \"tab\": \"General information\", \"score\": \"0.995\"}", - "NaturalQuestions (open-book) - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NaturalQuestions (closed-book) - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=0.667 (1)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666667\"}", - "NaturalQuestions (closed-book) - Stereotypes (gender)": "{\"description\": \"min=0.5, mean=0.5, max=0.5, sum=0.5 (1)\", \"tab\": \"Bias\", \"score\": \"0.5\"}", - "NaturalQuestions (closed-book) - Representation (race)": "{\"description\": \"min=0.415, mean=0.415, max=0.415, sum=0.415 (1)\", \"tab\": \"Bias\", \"score\": \"0.41463414634146334\"}", - "NaturalQuestions (closed-book) - Representation (gender)": "{\"description\": \"min=0.155, mean=0.155, max=0.155, sum=0.155 (1)\", \"tab\": \"Bias\", \"score\": \"0.15517241379310343\"}", - "NaturalQuestions (open-book) - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=0.667 (1)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666667\"}", - "NaturalQuestions (open-book) - Stereotypes (gender)": "{\"description\": \"min=0.42, mean=0.42, max=0.42, sum=0.42 (1)\", \"tab\": \"Bias\", \"score\": \"0.42000000000000004\"}", - "NaturalQuestions (open-book) - Representation (race)": "{\"description\": \"min=0.552, mean=0.552, max=0.552, sum=0.552 (1)\", \"tab\": \"Bias\", \"score\": \"0.5516224188790559\"}", - "NaturalQuestions (open-book) - Representation (gender)": "{\"description\": \"min=0.195, mean=0.195, max=0.195, sum=0.195 (1)\", \"tab\": \"Bias\", \"score\": \"0.19491525423728814\"}", - "NaturalQuestions (closed-book) - Toxic fraction": "{\"description\": \"min=0.002, mean=0.002, max=0.002, sum=0.002 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.002\"}", - "NaturalQuestions (open-book) - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "QuAC", - "source_data": { - "dataset_name": "QuAC", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on QuAC", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.371, - "details": { - "description": "min=0.371, mean=0.371, max=0.371, sum=0.371 (1)", - "tab": "Accuracy", - "QuAC - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "QuAC - F1 (Robustness)": "{\"description\": \"min=0.212, mean=0.212, max=0.212, sum=0.212 (1)\", \"tab\": \"Robustness\", \"score\": \"0.21167117057056115\"}", - "QuAC - F1 (Fairness)": "{\"description\": \"min=0.308, mean=0.308, max=0.308, sum=0.308 (1)\", \"tab\": \"Fairness\", \"score\": \"0.3078257563786361\"}", - "QuAC - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "QuAC - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "QuAC - # train": "{\"description\": \"min=0.862, mean=0.862, max=0.862, sum=0.862 (1)\", \"tab\": \"General information\", \"score\": \"0.862\"}", - "QuAC - truncated": "{\"description\": \"min=0.031, mean=0.031, max=0.031, sum=0.031 (1)\", \"tab\": \"General information\", \"score\": \"0.031\"}", - "QuAC - # prompt tokens": "{\"description\": \"min=1667.28, mean=1667.28, max=1667.28, sum=1667.28 (1)\", \"tab\": \"General information\", \"score\": \"1667.28\"}", - "QuAC - # output tokens": "{\"description\": \"min=0.999, mean=0.999, max=0.999, sum=0.999 (1)\", \"tab\": \"General information\", \"score\": \"0.999\"}", - "QuAC - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "QuAC - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=0.667 (1)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "QuAC - Stereotypes (gender)": "{\"description\": \"min=0.418, mean=0.418, max=0.418, sum=0.418 (1)\", \"tab\": \"Bias\", \"score\": \"0.4182641806722689\"}", - "QuAC - Representation (race)": "{\"description\": \"min=0.476, mean=0.476, max=0.476, sum=0.476 (1)\", \"tab\": \"Bias\", \"score\": \"0.4756554307116105\"}", - "QuAC - Representation (gender)": "{\"description\": \"min=0.214, mean=0.214, max=0.214, sum=0.214 (1)\", \"tab\": \"Bias\", \"score\": \"0.2142857142857143\"}", - "QuAC - Toxic fraction": "{\"description\": \"min=0.002, mean=0.002, max=0.002, sum=0.002 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.002\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "HellaSwag", - "source_data": { - "dataset_name": "HellaSwag", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on HellaSwag", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "HellaSwag - ECE (10-bin)": "{\"description\": \"No matching runs\", \"tab\": \"Calibration\", \"score\": \"\"}", - "HellaSwag - EM (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "HellaSwag - EM (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "HellaSwag - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "HellaSwag - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "OpenbookQA - ECE (10-bin)": "{\"description\": \"No matching runs\", \"tab\": \"Calibration\", \"score\": \"\"}", - "OpenbookQA - EM (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "OpenbookQA - EM (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "OpenbookQA - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "OpenbookQA - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "TruthfulQA", - "source_data": { - "dataset_name": "TruthfulQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on TruthfulQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.384, - "details": { - "description": "min=0.384, mean=0.384, max=0.384, sum=0.384 (1)", - "tab": "Accuracy", - "TruthfulQA - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "TruthfulQA - EM (Robustness)": "{\"description\": \"min=0.338, mean=0.338, max=0.338, sum=0.338 (1)\", \"tab\": \"Robustness\", \"score\": \"0.3379204892966361\"}", - "TruthfulQA - EM (Fairness)": "{\"description\": \"min=0.312, mean=0.312, max=0.312, sum=0.312 (1)\", \"tab\": \"Fairness\", \"score\": \"0.3119266055045872\"}", - "TruthfulQA - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "TruthfulQA - # eval": "{\"description\": \"min=654, mean=654, max=654, sum=654 (1)\", \"tab\": \"General information\", \"score\": \"654.0\"}", - "TruthfulQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "TruthfulQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "TruthfulQA - # prompt tokens": "{\"description\": \"min=507.503, mean=507.503, max=507.503, sum=507.503 (1)\", \"tab\": \"General information\", \"score\": \"507.50305810397555\"}", - "TruthfulQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "TruthfulQA - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MS MARCO (TREC)", - "source_data": { - "dataset_name": "MS MARCO (TREC)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "NDCG@10 on MS MARCO (TREC)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "MS MARCO (regular) - RR@10 (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "MS MARCO (TREC) - NDCG@10 (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "MS MARCO (regular) - RR@10 (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "MS MARCO (TREC) - NDCG@10 (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "MS MARCO (regular) - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MS MARCO (TREC) - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MS MARCO (regular) - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "MS MARCO (TREC) - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CNN/DailyMail", - "source_data": { - "dataset_name": "CNN/DailyMail", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on CNN/DailyMail", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "CNN/DailyMail - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "CNN/DailyMail - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "CNN/DailyMail - SummaC": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - QAFactEval": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - BERTScore (F1)": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - Coverage": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - Density": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - Compression": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-faithfulness": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-relevance": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-coherence": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "XSUM", - "source_data": { - "dataset_name": "XSUM", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on XSUM", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "XSUM - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "XSUM - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "XSUM - SummaC": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - QAFactEval": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - BERTScore (F1)": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - Coverage": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - Density": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - Compression": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-faithfulness": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-relevance": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-coherence": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "IMDB", - "source_data": { - "dataset_name": "IMDB", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on IMDB", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.959, - "details": { - "description": "min=0.959, mean=0.959, max=0.959, sum=0.959 (1)", - "tab": "Accuracy", - "IMDB - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "IMDB - EM (Robustness)": "{\"description\": \"min=0.938, mean=0.938, max=0.938, sum=0.938 (1)\", \"tab\": \"Robustness\", \"score\": \"0.938\"}", - "IMDB - EM (Fairness)": "{\"description\": \"min=0.957, mean=0.957, max=0.957, sum=0.957 (1)\", \"tab\": \"Fairness\", \"score\": \"0.957\"}", - "IMDB - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "IMDB - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "IMDB - # train": "{\"description\": \"min=2.871, mean=2.871, max=2.871, sum=2.871 (1)\", \"tab\": \"General information\", \"score\": \"2.871\"}", - "IMDB - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IMDB - # prompt tokens": "{\"description\": \"min=1666.079, mean=1666.079, max=1666.079, sum=1666.079 (1)\", \"tab\": \"General information\", \"score\": \"1666.079\"}", - "IMDB - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "IMDB - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "IMDB - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CivilComments", - "source_data": { - "dataset_name": "CivilComments", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on CivilComments", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.603, - "details": { - "description": "min=0.203, mean=0.603, max=0.918, sum=10.849 (18)", - "tab": "Accuracy", - "CivilComments - ECE (10-bin)": "{\"description\": \"9 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "CivilComments - EM (Robustness)": "{\"description\": \"min=0.135, mean=0.523, max=0.864, sum=9.414 (18)\", \"tab\": \"Robustness\", \"score\": \"0.5230033316869794\"}", - "CivilComments - EM (Fairness)": "{\"description\": \"min=0.16, mean=0.462, max=0.762, sum=8.312 (18)\", \"tab\": \"Fairness\", \"score\": \"0.4617550507789773\"}", - "CivilComments - Denoised inference time (s)": "{\"description\": \"9 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "CivilComments - # eval": "{\"description\": \"min=74, mean=371.556, max=683, sum=6688 (18)\", \"tab\": \"General information\", \"score\": \"371.55555555555554\"}", - "CivilComments - # train": "{\"description\": \"min=5, mean=5, max=5, sum=90 (18)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CivilComments - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (18)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CivilComments - # prompt tokens": "{\"description\": \"min=367.585, mean=782.759, max=1312.924, sum=14089.663 (18)\", \"tab\": \"General information\", \"score\": \"782.7590374602355\"}", - "CivilComments - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=18 (18)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "CivilComments - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=18 (18)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "CivilComments - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Toxic fraction": "{\"description\": \"9 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "RAFT", - "source_data": { - "dataset_name": "RAFT", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on RAFT", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.586, - "details": { - "description": "min=0.175, mean=0.586, max=0.925, sum=6.45 (11)", - "tab": "Accuracy", - "RAFT - ECE (10-bin)": "{\"description\": \"11 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "RAFT - EM (Robustness)": "{\"description\": \"min=0.025, mean=0.523, max=0.875, sum=5.75 (11)\", \"tab\": \"Robustness\", \"score\": \"0.5227272727272726\"}", - "RAFT - EM (Fairness)": "{\"description\": \"min=0.175, mean=0.561, max=0.875, sum=6.175 (11)\", \"tab\": \"Fairness\", \"score\": \"0.5613636363636363\"}", - "RAFT - Denoised inference time (s)": "{\"description\": \"11 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "RAFT - # eval": "{\"description\": \"min=40, mean=40, max=40, sum=440 (11)\", \"tab\": \"General information\", \"score\": \"40.0\"}", - "RAFT - # train": "{\"description\": \"min=0.7, mean=4.6, max=5, sum=50.6 (11)\", \"tab\": \"General information\", \"score\": \"4.6000000000000005\"}", - "RAFT - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (11)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "RAFT - # prompt tokens": "{\"description\": \"min=289.025, mean=877.464, max=1772.5, sum=9652.1 (11)\", \"tab\": \"General information\", \"score\": \"877.4636363636364\"}", - "RAFT - # output tokens": "{\"description\": \"min=0.825, mean=0.984, max=1, sum=10.825 (11)\", \"tab\": \"General information\", \"score\": \"0.984090909090909\"}", - "RAFT - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=11 (11)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "RAFT - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Toxic fraction": "{\"description\": \"11 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/tiiuae_falcon-instruct-7b.json b/data/models/tiiuae_falcon-instruct-7b.json deleted file mode 100644 index a92092d8f89ffb2c290b516dfed369bbaded646e..0000000000000000000000000000000000000000 --- a/data/models/tiiuae_falcon-instruct-7b.json +++ /dev/null @@ -1,674 +0,0 @@ -{ - "model_info": { - "name": "Falcon-Instruct 7B", - "id": "tiiuae/Falcon-Instruct-7B", - "developer": "tiiuae", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_classic/tiiuae_Falcon-Instruct-7B/1774096308.339228", - "retrieved_timestamp": "1774096308.339228", - "source_metadata": { - "source_name": "helm_classic", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_classic", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_classic", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperform on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.244, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Calibration": "{\"description\": \"\", \"tab\": \"Calibration\", \"score\": \"\"}", - "Mean win rate - Robustness": "{\"description\": \"\", \"tab\": \"Robustness\", \"score\": \"0.3032867132867133\"}", - "Mean win rate - Fairness": "{\"description\": \"\", \"tab\": \"Fairness\", \"score\": \"0.2968298368298368\"}", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}", - "Mean win rate - Bias": "{\"description\": \"\", \"tab\": \"Bias\", \"score\": \"0.514714004225644\"}", - "Mean win rate - Toxicity": "{\"description\": \"\", \"tab\": \"Toxicity\", \"score\": \"0.29545454545454547\"}", - "Mean win rate - Summarization metrics": "{\"description\": \"\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.275, - "details": { - "description": "min=0.21, mean=0.275, max=0.34, sum=1.374 (5)", - "tab": "Accuracy", - "MMLU - ECE (10-bin)": "{\"description\": \"5 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "MMLU - EM (Robustness)": "{\"description\": \"min=0.2, mean=0.25, max=0.32, sum=1.248 (5)\", \"tab\": \"Robustness\", \"score\": \"0.24961403508771932\"}", - "MMLU - EM (Fairness)": "{\"description\": \"min=0.2, mean=0.261, max=0.32, sum=1.307 (5)\", \"tab\": \"Fairness\", \"score\": \"0.2613684210526316\"}", - "MMLU - Denoised inference time (s)": "{\"description\": \"5 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=389.6, mean=500.12, max=664.281, sum=2500.601 (5)\", \"tab\": \"General information\", \"score\": \"500.12014035087725\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "MMLU - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "BoolQ", - "source_data": { - "dataset_name": "BoolQ", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on BoolQ", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.72, - "details": { - "description": "min=0.72, mean=0.72, max=0.72, sum=0.72 (1)", - "tab": "Accuracy", - "BoolQ - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "BoolQ - EM (Robustness)": "{\"description\": \"min=0.593, mean=0.593, max=0.593, sum=0.593 (1)\", \"tab\": \"Robustness\", \"score\": \"0.593\"}", - "BoolQ - EM (Fairness)": "{\"description\": \"min=0.637, mean=0.637, max=0.637, sum=0.637 (1)\", \"tab\": \"Fairness\", \"score\": \"0.637\"}", - "BoolQ - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "BoolQ - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "BoolQ - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "BoolQ - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "BoolQ - # prompt tokens": "{\"description\": \"min=1284.629, mean=1284.629, max=1284.629, sum=1284.629 (1)\", \"tab\": \"General information\", \"score\": \"1284.629\"}", - "BoolQ - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "BoolQ - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "BoolQ - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.476, - "details": { - "description": "min=0.476, mean=0.476, max=0.476, sum=0.476 (1)", - "tab": "Accuracy", - "NarrativeQA - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "NarrativeQA - F1 (Robustness)": "{\"description\": \"min=0.258, mean=0.258, max=0.258, sum=0.258 (1)\", \"tab\": \"Robustness\", \"score\": \"0.2582769089885097\"}", - "NarrativeQA - F1 (Fairness)": "{\"description\": \"min=0.354, mean=0.354, max=0.354, sum=0.354 (1)\", \"tab\": \"Fairness\", \"score\": \"0.3536054591455644\"}", - "NarrativeQA - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=2.025, mean=2.025, max=2.025, sum=2.025 (1)\", \"tab\": \"General information\", \"score\": \"2.0253521126760563\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=1694.082, mean=1694.082, max=1694.082, sum=1694.082 (1)\", \"tab\": \"General information\", \"score\": \"1694.081690140845\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NarrativeQA - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NarrativeQA - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NarrativeQA - Stereotypes (gender)": "{\"description\": \"min=0.444, mean=0.444, max=0.444, sum=0.444 (1)\", \"tab\": \"Bias\", \"score\": \"0.4444444444444444\"}", - "NarrativeQA - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NarrativeQA - Representation (gender)": "{\"description\": \"min=0.187, mean=0.187, max=0.187, sum=0.187 (1)\", \"tab\": \"Bias\", \"score\": \"0.1870229007633588\"}", - "NarrativeQA - Toxic fraction": "{\"description\": \"min=0.017, mean=0.017, max=0.017, sum=0.017 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.016901408450704224\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (open-book)", - "source_data": { - "dataset_name": "NaturalQuestions (open-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (open-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.449, - "details": { - "description": "min=0.449, mean=0.449, max=0.449, sum=0.449 (1)", - "tab": "Accuracy", - "NaturalQuestions (closed-book) - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "NaturalQuestions (open-book) - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - F1 (Robustness)": "{\"description\": \"min=0.132, mean=0.132, max=0.132, sum=0.132 (1)\", \"tab\": \"Robustness\", \"score\": \"0.1322266230747346\"}", - "NaturalQuestions (open-book) - F1 (Robustness)": "{\"description\": \"min=0.327, mean=0.327, max=0.327, sum=0.327 (1)\", \"tab\": \"Robustness\", \"score\": \"0.32667933185026377\"}", - "NaturalQuestions (closed-book) - F1 (Fairness)": "{\"description\": \"min=0.148, mean=0.148, max=0.148, sum=0.148 (1)\", \"tab\": \"Fairness\", \"score\": \"0.14824932914209746\"}", - "NaturalQuestions (open-book) - F1 (Fairness)": "{\"description\": \"min=0.383, mean=0.383, max=0.383, sum=0.383 (1)\", \"tab\": \"Fairness\", \"score\": \"0.38333017617065734\"}", - "NaturalQuestions (closed-book) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NaturalQuestions (open-book) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=124.246, mean=124.246, max=124.246, sum=124.246 (1)\", \"tab\": \"General information\", \"score\": \"124.246\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=0.999, mean=0.999, max=0.999, sum=0.999 (1)\", \"tab\": \"General information\", \"score\": \"0.999\"}", - "NaturalQuestions (closed-book) - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.599, mean=4.599, max=4.599, sum=4.599 (1)\", \"tab\": \"General information\", \"score\": \"4.599\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.039, mean=0.039, max=0.039, sum=0.039 (1)\", \"tab\": \"General information\", \"score\": \"0.039\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1587.334, mean=1587.334, max=1587.334, sum=1587.334 (1)\", \"tab\": \"General information\", \"score\": \"1587.334\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=0.984, mean=0.984, max=0.984, sum=0.984 (1)\", \"tab\": \"General information\", \"score\": \"0.984\"}", - "NaturalQuestions (open-book) - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NaturalQuestions (closed-book) - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=0.667 (1)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666667\"}", - "NaturalQuestions (closed-book) - Stereotypes (gender)": "{\"description\": \"min=0.5, mean=0.5, max=0.5, sum=0.5 (1)\", \"tab\": \"Bias\", \"score\": \"0.5\"}", - "NaturalQuestions (closed-book) - Representation (race)": "{\"description\": \"min=0.272, mean=0.272, max=0.272, sum=0.272 (1)\", \"tab\": \"Bias\", \"score\": \"0.2716049382716049\"}", - "NaturalQuestions (closed-book) - Representation (gender)": "{\"description\": \"min=0.071, mean=0.071, max=0.071, sum=0.071 (1)\", \"tab\": \"Bias\", \"score\": \"0.07142857142857142\"}", - "NaturalQuestions (open-book) - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NaturalQuestions (open-book) - Stereotypes (gender)": "{\"description\": \"min=0.5, mean=0.5, max=0.5, sum=0.5 (1)\", \"tab\": \"Bias\", \"score\": \"0.5\"}", - "NaturalQuestions (open-book) - Representation (race)": "{\"description\": \"min=0.426, mean=0.426, max=0.426, sum=0.426 (1)\", \"tab\": \"Bias\", \"score\": \"0.4257907542579076\"}", - "NaturalQuestions (open-book) - Representation (gender)": "{\"description\": \"min=0.068, mean=0.068, max=0.068, sum=0.068 (1)\", \"tab\": \"Bias\", \"score\": \"0.0684931506849315\"}", - "NaturalQuestions (closed-book) - Toxic fraction": "{\"description\": \"min=0.001, mean=0.001, max=0.001, sum=0.001 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.001\"}", - "NaturalQuestions (open-book) - Toxic fraction": "{\"description\": \"min=0.001, mean=0.001, max=0.001, sum=0.001 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.001\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "QuAC", - "source_data": { - "dataset_name": "QuAC", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on QuAC", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.311, - "details": { - "description": "min=0.311, mean=0.311, max=0.311, sum=0.311 (1)", - "tab": "Accuracy", - "QuAC - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "QuAC - F1 (Robustness)": "{\"description\": \"min=0.179, mean=0.179, max=0.179, sum=0.179 (1)\", \"tab\": \"Robustness\", \"score\": \"0.1789889679486199\"}", - "QuAC - F1 (Fairness)": "{\"description\": \"min=0.219, mean=0.219, max=0.219, sum=0.219 (1)\", \"tab\": \"Fairness\", \"score\": \"0.21915649953692506\"}", - "QuAC - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "QuAC - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "QuAC - # train": "{\"description\": \"min=0.862, mean=0.862, max=0.862, sum=0.862 (1)\", \"tab\": \"General information\", \"score\": \"0.862\"}", - "QuAC - truncated": "{\"description\": \"min=0.031, mean=0.031, max=0.031, sum=0.031 (1)\", \"tab\": \"General information\", \"score\": \"0.031\"}", - "QuAC - # prompt tokens": "{\"description\": \"min=1667.28, mean=1667.28, max=1667.28, sum=1667.28 (1)\", \"tab\": \"General information\", \"score\": \"1667.28\"}", - "QuAC - # output tokens": "{\"description\": \"min=0.997, mean=0.997, max=0.997, sum=0.997 (1)\", \"tab\": \"General information\", \"score\": \"0.997\"}", - "QuAC - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "QuAC - Stereotypes (race)": "{\"description\": \"min=0.625, mean=0.625, max=0.625, sum=0.625 (1)\", \"tab\": \"Bias\", \"score\": \"0.625\"}", - "QuAC - Stereotypes (gender)": "{\"description\": \"min=0.456, mean=0.456, max=0.456, sum=0.456 (1)\", \"tab\": \"Bias\", \"score\": \"0.4561372269705603\"}", - "QuAC - Representation (race)": "{\"description\": \"min=0.262, mean=0.262, max=0.262, sum=0.262 (1)\", \"tab\": \"Bias\", \"score\": \"0.26241134751773054\"}", - "QuAC - Representation (gender)": "{\"description\": \"min=0.251, mean=0.251, max=0.251, sum=0.251 (1)\", \"tab\": \"Bias\", \"score\": \"0.25052854122621565\"}", - "QuAC - Toxic fraction": "{\"description\": \"min=0.002, mean=0.002, max=0.002, sum=0.002 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.002\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "HellaSwag", - "source_data": { - "dataset_name": "HellaSwag", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on HellaSwag", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "HellaSwag - ECE (10-bin)": "{\"description\": \"No matching runs\", \"tab\": \"Calibration\", \"score\": \"\"}", - "HellaSwag - EM (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "HellaSwag - EM (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "HellaSwag - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "HellaSwag - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "OpenbookQA - ECE (10-bin)": "{\"description\": \"No matching runs\", \"tab\": \"Calibration\", \"score\": \"\"}", - "OpenbookQA - EM (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "OpenbookQA - EM (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "OpenbookQA - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "OpenbookQA - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "TruthfulQA", - "source_data": { - "dataset_name": "TruthfulQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on TruthfulQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.213, - "details": { - "description": "min=0.213, mean=0.213, max=0.213, sum=0.213 (1)", - "tab": "Accuracy", - "TruthfulQA - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "TruthfulQA - EM (Robustness)": "{\"description\": \"min=0.17, mean=0.17, max=0.17, sum=0.17 (1)\", \"tab\": \"Robustness\", \"score\": \"0.16972477064220184\"}", - "TruthfulQA - EM (Fairness)": "{\"description\": \"min=0.183, mean=0.183, max=0.183, sum=0.183 (1)\", \"tab\": \"Fairness\", \"score\": \"0.1834862385321101\"}", - "TruthfulQA - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "TruthfulQA - # eval": "{\"description\": \"min=654, mean=654, max=654, sum=654 (1)\", \"tab\": \"General information\", \"score\": \"654.0\"}", - "TruthfulQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "TruthfulQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "TruthfulQA - # prompt tokens": "{\"description\": \"min=507.503, mean=507.503, max=507.503, sum=507.503 (1)\", \"tab\": \"General information\", \"score\": \"507.50305810397555\"}", - "TruthfulQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "TruthfulQA - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MS MARCO (TREC)", - "source_data": { - "dataset_name": "MS MARCO (TREC)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "NDCG@10 on MS MARCO (TREC)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "MS MARCO (regular) - RR@10 (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "MS MARCO (TREC) - NDCG@10 (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "MS MARCO (regular) - RR@10 (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "MS MARCO (TREC) - NDCG@10 (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "MS MARCO (regular) - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MS MARCO (TREC) - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MS MARCO (regular) - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "MS MARCO (TREC) - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CNN/DailyMail", - "source_data": { - "dataset_name": "CNN/DailyMail", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on CNN/DailyMail", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "CNN/DailyMail - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "CNN/DailyMail - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "CNN/DailyMail - SummaC": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - QAFactEval": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - BERTScore (F1)": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - Coverage": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - Density": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - Compression": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-faithfulness": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-relevance": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-coherence": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "XSUM", - "source_data": { - "dataset_name": "XSUM", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on XSUM", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "XSUM - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "XSUM - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "XSUM - SummaC": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - QAFactEval": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - BERTScore (F1)": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - Coverage": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - Density": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - Compression": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-faithfulness": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-relevance": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-coherence": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "IMDB", - "source_data": { - "dataset_name": "IMDB", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on IMDB", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.852, - "details": { - "description": "min=0.852, mean=0.852, max=0.852, sum=0.852 (1)", - "tab": "Accuracy", - "IMDB - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "IMDB - EM (Robustness)": "{\"description\": \"min=0.759, mean=0.759, max=0.759, sum=0.759 (1)\", \"tab\": \"Robustness\", \"score\": \"0.759\"}", - "IMDB - EM (Fairness)": "{\"description\": \"min=0.811, mean=0.811, max=0.811, sum=0.811 (1)\", \"tab\": \"Fairness\", \"score\": \"0.811\"}", - "IMDB - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "IMDB - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "IMDB - # train": "{\"description\": \"min=2.871, mean=2.871, max=2.871, sum=2.871 (1)\", \"tab\": \"General information\", \"score\": \"2.871\"}", - "IMDB - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IMDB - # prompt tokens": "{\"description\": \"min=1666.079, mean=1666.079, max=1666.079, sum=1666.079 (1)\", \"tab\": \"General information\", \"score\": \"1666.079\"}", - "IMDB - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "IMDB - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "IMDB - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CivilComments", - "source_data": { - "dataset_name": "CivilComments", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on CivilComments", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.511, - "details": { - "description": "min=0, mean=0.511, max=1, sum=9.199 (18)", - "tab": "Accuracy", - "CivilComments - ECE (10-bin)": "{\"description\": \"9 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "CivilComments - EM (Robustness)": "{\"description\": \"min=0, mean=0.487, max=0.999, sum=8.769 (18)\", \"tab\": \"Robustness\", \"score\": \"0.4871679045873981\"}", - "CivilComments - EM (Fairness)": "{\"description\": \"min=0, mean=0.502, max=1, sum=9.031 (18)\", \"tab\": \"Fairness\", \"score\": \"0.5017354752179064\"}", - "CivilComments - Denoised inference time (s)": "{\"description\": \"9 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "CivilComments - # eval": "{\"description\": \"min=74, mean=371.556, max=683, sum=6688 (18)\", \"tab\": \"General information\", \"score\": \"371.55555555555554\"}", - "CivilComments - # train": "{\"description\": \"min=5, mean=5, max=5, sum=90 (18)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CivilComments - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (18)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CivilComments - # prompt tokens": "{\"description\": \"min=367.585, mean=782.759, max=1312.924, sum=14089.663 (18)\", \"tab\": \"General information\", \"score\": \"782.7590374602355\"}", - "CivilComments - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=18 (18)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "CivilComments - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=18 (18)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "CivilComments - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Toxic fraction": "{\"description\": \"9 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "RAFT", - "source_data": { - "dataset_name": "RAFT", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on RAFT", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.523, - "details": { - "description": "min=0.15, mean=0.523, max=0.975, sum=5.75 (11)", - "tab": "Accuracy", - "RAFT - ECE (10-bin)": "{\"description\": \"11 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "RAFT - EM (Robustness)": "{\"description\": \"min=0.15, mean=0.445, max=0.975, sum=4.9 (11)\", \"tab\": \"Robustness\", \"score\": \"0.4454545454545454\"}", - "RAFT - EM (Fairness)": "{\"description\": \"min=0.15, mean=0.5, max=0.975, sum=5.5 (11)\", \"tab\": \"Fairness\", \"score\": \"0.5\"}", - "RAFT - Denoised inference time (s)": "{\"description\": \"11 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "RAFT - # eval": "{\"description\": \"min=40, mean=40, max=40, sum=440 (11)\", \"tab\": \"General information\", \"score\": \"40.0\"}", - "RAFT - # train": "{\"description\": \"min=0.7, mean=4.6, max=5, sum=50.6 (11)\", \"tab\": \"General information\", \"score\": \"4.6000000000000005\"}", - "RAFT - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (11)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "RAFT - # prompt tokens": "{\"description\": \"min=289.025, mean=877.464, max=1772.5, sum=9652.1 (11)\", \"tab\": \"General information\", \"score\": \"877.4636363636364\"}", - "RAFT - # output tokens": "{\"description\": \"min=0.95, mean=0.995, max=1, sum=10.95 (11)\", \"tab\": \"General information\", \"score\": \"0.9954545454545454\"}", - "RAFT - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=11 (11)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "RAFT - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Toxic fraction": "{\"description\": \"11 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/tiiuae_falcon-mamba-7b.json b/data/models/tiiuae_falcon-mamba-7b.json deleted file mode 100644 index 04a23296b1ee4ed42621313f38632d29e0c6cc43..0000000000000000000000000000000000000000 --- a/data/models/tiiuae_falcon-mamba-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "falcon-mamba-7b", - "id": "tiiuae/falcon-mamba-7b", - "developer": "tiiuae", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "FalconMambaForCausalLM", - "params_billions": "7.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/tiiuae_falcon-mamba-7b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3336 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4285 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0446 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3104 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.421 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2302 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/tiiuae_falcon3-10b-base.json b/data/models/tiiuae_falcon3-10b-base.json deleted file mode 100644 index de827d2e7a96feaa7140e113b998ad5f2d7c1ecc..0000000000000000000000000000000000000000 --- a/data/models/tiiuae_falcon3-10b-base.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Falcon3-10B-Base", - "id": "tiiuae/Falcon3-10B-Base", - "developer": "tiiuae", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "10.306" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/tiiuae_Falcon3-10B-Base/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3648 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.595 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2492 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3456 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4398 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.424 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/tiiuae_falcon3-10b-instruct.json b/data/models/tiiuae_falcon3-10b-instruct.json deleted file mode 100644 index 7ebc3ce1c45363e0c75941affa6f93e2c6094fba..0000000000000000000000000000000000000000 --- a/data/models/tiiuae_falcon3-10b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Falcon3-10B-Instruct", - "id": "tiiuae/Falcon3-10B-Instruct", - "developer": "tiiuae", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "10.306" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/tiiuae_Falcon3-10B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7817 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.617 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2764 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3289 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4323 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4429 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/tiiuae_falcon3-1b-base.json b/data/models/tiiuae_falcon3-1b-base.json deleted file mode 100644 index fde0dda51ed64560b6a7b0abc9587639aa767b01..0000000000000000000000000000000000000000 --- a/data/models/tiiuae_falcon3-1b-base.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Falcon3-1B-Base", - "id": "tiiuae/Falcon3-1B-Base", - "developer": "tiiuae", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.669" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/tiiuae_Falcon3-1B-Base/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2428 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3571 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0332 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2794 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4147 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1608 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/tiiuae_falcon3-1b-instruct.json b/data/models/tiiuae_falcon3-1b-instruct.json deleted file mode 100644 index 665f8e51227527a7a318a8ba66d75e69a01389c9..0000000000000000000000000000000000000000 --- a/data/models/tiiuae_falcon3-1b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Falcon3-1B-Instruct", - "id": "tiiuae/Falcon3-1B-Instruct", - "developer": "tiiuae", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.669" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/tiiuae_Falcon3-1B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5557 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3745 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0634 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2668 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4189 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1838 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/tiiuae_falcon3-3b-base.json b/data/models/tiiuae_falcon3-3b-base.json deleted file mode 100644 index e2dfbaf6ef3ce7074ba1f0422203898f1c1cafd8..0000000000000000000000000000000000000000 --- a/data/models/tiiuae_falcon3-3b-base.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Falcon3-3B-Base", - "id": "tiiuae/Falcon3-3B-Base", - "developer": "tiiuae", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.228" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/tiiuae_Falcon3-3B-Base/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2765 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4421 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1178 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.297 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.375 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2879 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/tiiuae_falcon3-3b-instruct.json b/data/models/tiiuae_falcon3-3b-instruct.json deleted file mode 100644 index bd281702f067324dbaeda2ada3d7a3068f949dbc..0000000000000000000000000000000000000000 --- a/data/models/tiiuae_falcon3-3b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Falcon3-3B-Instruct", - "id": "tiiuae/Falcon3-3B-Instruct", - "developer": "tiiuae", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.228" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/tiiuae_Falcon3-3B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6977 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4754 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.25 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2886 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4136 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3005 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/tiiuae_falcon3-7b-base.json b/data/models/tiiuae_falcon3-7b-base.json deleted file mode 100644 index d79bd5341f505d84e5ce85017839cc30ee7e5b28..0000000000000000000000000000000000000000 --- a/data/models/tiiuae_falcon3-7b-base.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Falcon3-7B-Base", - "id": "tiiuae/Falcon3-7B-Base", - "developer": "tiiuae", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "7.456" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/tiiuae_Falcon3-7B-Base/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3416 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5099 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1941 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3465 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4702 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.391 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/tiiuae_falcon3-7b-instruct.json b/data/models/tiiuae_falcon3-7b-instruct.json deleted file mode 100644 index 15262216c675f6a32446ad280372ef367a04c72c..0000000000000000000000000000000000000000 --- a/data/models/tiiuae_falcon3-7b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Falcon3-7B-Instruct", - "id": "tiiuae/Falcon3-7B-Instruct", - "developer": "tiiuae", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "7.456" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/tiiuae_Falcon3-7B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7612 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5632 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4086 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3104 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4827 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4087 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/tiiuae_falcon3-mamba-7b-base.json b/data/models/tiiuae_falcon3-mamba-7b-base.json deleted file mode 100644 index 5f836d2dbe28fa0782c4364c00d83bc81d4e4a0c..0000000000000000000000000000000000000000 --- a/data/models/tiiuae_falcon3-mamba-7b-base.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Falcon3-Mamba-7B-Base", - "id": "tiiuae/Falcon3-Mamba-7B-Base", - "developer": "tiiuae", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "FalconMambaForCausalLM", - "params_billions": "7.273" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/tiiuae_Falcon3-Mamba-7B-Base/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2891 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4699 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1941 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3096 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3431 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3038 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/tiiuae_falcon3-mamba-7b-instruct.json b/data/models/tiiuae_falcon3-mamba-7b-instruct.json deleted file mode 100644 index 49b41c8220a2fd016a99536c3dfd1c6614fe75f2..0000000000000000000000000000000000000000 --- a/data/models/tiiuae_falcon3-mamba-7b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Falcon3-Mamba-7B-Instruct", - "id": "tiiuae/Falcon3-Mamba-7B-Instruct", - "developer": "tiiuae", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "FalconMambaForCausalLM", - "params_billions": "7.273" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/tiiuae_Falcon3-Mamba-7B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7165 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4679 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3006 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3037 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3869 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3369 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/tijmen2_cosmosage-v3.json b/data/models/tijmen2_cosmosage-v3.json deleted file mode 100644 index 2cf439bb54711ed181943c6218d7c1df5a9d1226..0000000000000000000000000000000000000000 --- a/data/models/tijmen2_cosmosage-v3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "cosmosage-v3", - "id": "Tijmen2/cosmosage-v3", - "developer": "Tijmen2", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Tijmen2_cosmosage-v3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4482 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4551 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0506 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2827 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4199 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2486 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/tinycompany_bibo-v0.3.json b/data/models/tinycompany_bibo-v0.3.json deleted file mode 100644 index 000e5fa5586bd71ef2c0a12d7c07c6d5cbf6bf00..0000000000000000000000000000000000000000 --- a/data/models/tinycompany_bibo-v0.3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "BiBo-v0.3", - "id": "tinycompany/BiBo-v0.3", - "developer": "tinycompany", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "2.943" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/tinycompany_BiBo-v0.3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5184 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4642 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0876 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2676 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.395 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2995 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/tinycompany_bibo-v0.7.json b/data/models/tinycompany_bibo-v0.7.json deleted file mode 100644 index 8f06c73b2948e2b39ee45e09a1a528cb95865df5..0000000000000000000000000000000000000000 --- a/data/models/tinycompany_bibo-v0.7.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "BiBo-v0.7", - "id": "tinycompany/BiBo-v0.7", - "developer": "tinycompany", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "2.943" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/tinycompany_BiBo-v0.7/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3738 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4311 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0823 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2768 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4044 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.265 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/tinycompany_shawtyisbad-bgem3.json b/data/models/tinycompany_shawtyisbad-bgem3.json deleted file mode 100644 index 68dd1614f3d897b36996040b4a614cb8e7b8d60c..0000000000000000000000000000000000000000 --- a/data/models/tinycompany_shawtyisbad-bgem3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ShawtyIsBad-bgem3", - "id": "tinycompany/ShawtyIsBad-bgem3", - "developer": "tinycompany", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.436" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/tinycompany_ShawtyIsBad-bgem3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2608 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3853 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0483 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3054 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3695 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2583 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/tinycompany_shawtyisbad-e5-large.json b/data/models/tinycompany_shawtyisbad-e5-large.json deleted file mode 100644 index 5ace7ae483ffcd2e02835eb1bbe6db641f13ec82..0000000000000000000000000000000000000000 --- a/data/models/tinycompany_shawtyisbad-e5-large.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ShawtyIsBad-e5-large", - "id": "tinycompany/ShawtyIsBad-e5-large", - "developer": "tinycompany", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.436" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/tinycompany_ShawtyIsBad-e5-large/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2468 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3873 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0453 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.302 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.372 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2569 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/tinycompany_shawtyisbad-ib.json b/data/models/tinycompany_shawtyisbad-ib.json deleted file mode 100644 index f918f00e54b88882ed85e24f565766b68f7086fd..0000000000000000000000000000000000000000 --- a/data/models/tinycompany_shawtyisbad-ib.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ShawtyIsBad-ib", - "id": "tinycompany/ShawtyIsBad-ib", - "developer": "tinycompany", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.436" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/tinycompany_ShawtyIsBad-ib/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2565 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.388 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0491 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2987 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3641 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2581 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/tinycompany_shawtyisbad-nomic-moe.json b/data/models/tinycompany_shawtyisbad-nomic-moe.json deleted file mode 100644 index 094b0cac1ca64d20a331c3131d7e58b83cdb630c..0000000000000000000000000000000000000000 --- a/data/models/tinycompany_shawtyisbad-nomic-moe.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ShawtyIsBad-nomic-moe", - "id": "tinycompany/ShawtyIsBad-nomic-moe", - "developer": "tinycompany", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.436" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/tinycompany_ShawtyIsBad-nomic-moe/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2608 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3878 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0431 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.307 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3747 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2572 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/tinycompany_shawtyisbad-nomic1.5.json b/data/models/tinycompany_shawtyisbad-nomic1.5.json deleted file mode 100644 index d9918fa87626daa5f5de7913fca2b78030ca08f1..0000000000000000000000000000000000000000 --- a/data/models/tinycompany_shawtyisbad-nomic1.5.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ShawtyIsBad-nomic1.5", - "id": "tinycompany/ShawtyIsBad-nomic1.5", - "developer": "tinycompany", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.436" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/tinycompany_ShawtyIsBad-nomic1.5/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2544 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3874 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0431 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3112 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3628 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2567 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/tinycompany_sigmaboi-base.json b/data/models/tinycompany_sigmaboi-base.json deleted file mode 100644 index 88693bd8386423bee2b31c1e9a36d1d1980ec438..0000000000000000000000000000000000000000 --- a/data/models/tinycompany_sigmaboi-base.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SigmaBoi-base", - "id": "tinycompany/SigmaBoi-base", - "developer": "tinycompany", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "2.943" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/tinycompany_SigmaBoi-base/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2447 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4314 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0778 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2936 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4343 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2817 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/tinycompany_sigmaboi-bge-m3.json b/data/models/tinycompany_sigmaboi-bge-m3.json deleted file mode 100644 index a4e1fb23e52eeb0ff72ff1e131c3d310dbbe8d17..0000000000000000000000000000000000000000 --- a/data/models/tinycompany_sigmaboi-bge-m3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SigmaBoi-bge-m3", - "id": "tinycompany/SigmaBoi-bge-m3", - "developer": "tinycompany", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "2.943" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/tinycompany_SigmaBoi-bge-m3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.245 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4351 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0763 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2945 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4383 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2819 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/tinycompany_sigmaboi-bgem3.json b/data/models/tinycompany_sigmaboi-bgem3.json deleted file mode 100644 index 24e22571e6fbc1ee6bed854ebf5f1a76eda5a98b..0000000000000000000000000000000000000000 --- a/data/models/tinycompany_sigmaboi-bgem3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SigmaBoi-bgem3", - "id": "tinycompany/SigmaBoi-bgem3", - "developer": "tinycompany", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "2.943" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/tinycompany_SigmaBoi-bgem3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.245 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4351 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0763 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2945 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4383 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2819 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/tinycompany_sigmaboi-ib.json b/data/models/tinycompany_sigmaboi-ib.json deleted file mode 100644 index 5b5fdda3631b261e3647bfa526b6b92b58fb32fa..0000000000000000000000000000000000000000 --- a/data/models/tinycompany_sigmaboi-ib.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SigmaBoi-ib", - "id": "tinycompany/SigmaBoi-ib", - "developer": "tinycompany", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "2.943" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/tinycompany_SigmaBoi-ib/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2477 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4344 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.074 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2878 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.429 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2824 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/tinycompany_sigmaboi-nomic-moe.json b/data/models/tinycompany_sigmaboi-nomic-moe.json deleted file mode 100644 index f909a8d1dc692ac125e2693d80176ce93c56778c..0000000000000000000000000000000000000000 --- a/data/models/tinycompany_sigmaboi-nomic-moe.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SigmaBoi-nomic-moe", - "id": "tinycompany/SigmaBoi-nomic-moe", - "developer": "tinycompany", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "2.943" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/tinycompany_SigmaBoi-nomic-moe/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2474 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4334 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0718 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2928 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4316 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2837 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/tinycompany_sigmaboi-nomic1.5-fp32.json b/data/models/tinycompany_sigmaboi-nomic1.5-fp32.json deleted file mode 100644 index 5be3bee2e2b52a42f496e2196da952750d106bb7..0000000000000000000000000000000000000000 --- a/data/models/tinycompany_sigmaboi-nomic1.5-fp32.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SigmaBoi-nomic1.5-fp32", - "id": "tinycompany/SigmaBoi-nomic1.5-fp32", - "developer": "tinycompany", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "2.943" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/tinycompany_SigmaBoi-nomic1.5-fp32/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2462 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4371 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0831 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2961 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4316 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2841 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/tinycompany_sigmaboi-nomic1.5.json b/data/models/tinycompany_sigmaboi-nomic1.5.json deleted file mode 100644 index f83aac769a676efeb53eb1ccd14cb4d676a187c7..0000000000000000000000000000000000000000 --- a/data/models/tinycompany_sigmaboi-nomic1.5.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SigmaBoi-nomic1.5", - "id": "tinycompany/SigmaBoi-nomic1.5", - "developer": "tinycompany", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "2.943" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/tinycompany_SigmaBoi-nomic1.5/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2447 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4371 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0831 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2961 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4316 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2841 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/tinycompany_tamed-shawty.json b/data/models/tinycompany_tamed-shawty.json deleted file mode 100644 index 407c90018ee4ec720374ee8f6dbaf21d943f7ce5..0000000000000000000000000000000000000000 --- a/data/models/tinycompany_tamed-shawty.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Tamed-Shawty", - "id": "tinycompany/Tamed-Shawty", - "developer": "tinycompany", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.562" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/tinycompany_Tamed-Shawty/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3831 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3837 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0718 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2626 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3501 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2601 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/tinyllama_tinyllama-1.1b-chat-v0.1.json b/data/models/tinyllama_tinyllama-1.1b-chat-v0.1.json deleted file mode 100644 index 55433a966299a49d1d3d51f1c51f98533de6ca43..0000000000000000000000000000000000000000 --- a/data/models/tinyllama_tinyllama-1.1b-chat-v0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "TinyLlama-1.1B-Chat-v0.1", - "id": "TinyLlama/TinyLlama-1.1B-Chat-v0.1", - "developer": "TinyLlama", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.1" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/TinyLlama_TinyLlama-1.1B-Chat-v0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1479 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3084 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.006 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.229 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3592 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1098 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/tinyllama_tinyllama-1.1b-chat-v0.5.json b/data/models/tinyllama_tinyllama-1.1b-chat-v0.5.json deleted file mode 100644 index 7bd17bca9aed9078b714c06e158b8aaff0f9f163..0000000000000000000000000000000000000000 --- a/data/models/tinyllama_tinyllama-1.1b-chat-v0.5.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "TinyLlama-1.1B-Chat-v0.5", - "id": "TinyLlama/TinyLlama-1.1B-Chat-v0.5", - "developer": "TinyLlama", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.1" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/TinyLlama_TinyLlama-1.1B-Chat-v0.5/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1634 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3105 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0038 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2483 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3661 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1096 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/tinyllama_tinyllama-1.1b-chat-v0.6.json b/data/models/tinyllama_tinyllama-1.1b-chat-v0.6.json deleted file mode 100644 index 8f6551ee8f9512455a8648559b972d625e89d6a1..0000000000000000000000000000000000000000 --- a/data/models/tinyllama_tinyllama-1.1b-chat-v0.6.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "TinyLlama-1.1B-Chat-v0.6", - "id": "TinyLlama/TinyLlama-1.1B-Chat-v0.6", - "developer": "TinyLlama", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.1" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/TinyLlama_TinyLlama-1.1B-Chat-v0.6/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1574 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3067 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0159 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2584 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3422 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1149 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/tinyllama_tinyllama-1.1b-chat-v1.0.json b/data/models/tinyllama_tinyllama-1.1b-chat-v1.0.json deleted file mode 100644 index 22af8a5c0014e6be7b1335bb4d89d4ac19154eb9..0000000000000000000000000000000000000000 --- a/data/models/tinyllama_tinyllama-1.1b-chat-v1.0.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "TinyLlama-1.1B-Chat-v1.0", - "id": "TinyLlama/TinyLlama-1.1B-Chat-v1.0", - "developer": "TinyLlama", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.1" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/TinyLlama_TinyLlama-1.1B-Chat-v1.0/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0596 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3104 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0151 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.25 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3515 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1101 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/tinyllama_tinyllama-1.1b-intermediate-step-1431k-3t.json b/data/models/tinyllama_tinyllama-1.1b-intermediate-step-1431k-3t.json deleted file mode 100644 index ad75d4611603d42cf572710cd8b335263b5d8cd6..0000000000000000000000000000000000000000 --- a/data/models/tinyllama_tinyllama-1.1b-intermediate-step-1431k-3t.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "TinyLlama-1.1B-intermediate-step-1431k-3T", - "id": "TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T", - "developer": "TinyLlama", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.1" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/TinyLlama_TinyLlama-1.1B-intermediate-step-1431k-3T/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2277 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3071 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0121 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2525 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.338 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.112 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/tinyllama_tinyllama_v1.1.json b/data/models/tinyllama_tinyllama_v1.1.json deleted file mode 100644 index 5c7e1357b6d93d32f72f3c8602bfb139b772ffef..0000000000000000000000000000000000000000 --- a/data/models/tinyllama_tinyllama_v1.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "TinyLlama_v1.1", - "id": "TinyLlama/TinyLlama_v1.1", - "developer": "TinyLlama", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.1" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/TinyLlama_TinyLlama_v1.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2001 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3024 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0121 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2458 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.37 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1049 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/tklohj_windyflollm.json b/data/models/tklohj_windyflollm.json deleted file mode 100644 index a85753ace8fc557774b0896ccffb5f7a5e4f3246..0000000000000000000000000000000000000000 --- a/data/models/tklohj_windyflollm.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "WindyFloLLM", - "id": "tklohj/WindyFloLLM", - "developer": "tklohj", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "13.016" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/tklohj_WindyFloLLM/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2669 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4637 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0159 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2752 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4253 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2581 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/toastypigeon_sto-vo-kor-12b.json b/data/models/toastypigeon_sto-vo-kor-12b.json deleted file mode 100644 index df22f20aafcec4ccc822028b07e1a1a79a11ea31..0000000000000000000000000000000000000000 --- a/data/models/toastypigeon_sto-vo-kor-12b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Sto-vo-kor-12B", - "id": "ToastyPigeon/Sto-vo-kor-12B", - "developer": "ToastyPigeon", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ToastyPigeon_Sto-vo-kor-12B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5501 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5065 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1088 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3054 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3938 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3398 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/together_redpajama-incite-base-7b.json b/data/models/together_redpajama-incite-base-7b.json deleted file mode 100644 index 02151cefe19e44df91422272a7716b798d343b89..0000000000000000000000000000000000000000 --- a/data/models/together_redpajama-incite-base-7b.json +++ /dev/null @@ -1,674 +0,0 @@ -{ - "model_info": { - "name": "RedPajama-INCITE-Base 7B", - "id": "together/RedPajama-INCITE-Base-7B", - "developer": "together", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_classic/together_RedPajama-INCITE-Base-7B/1774096308.339228", - "retrieved_timestamp": "1774096308.339228", - "source_metadata": { - "source_name": "helm_classic", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_classic", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_classic", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperform on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.378, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Calibration": "{\"description\": \"\", \"tab\": \"Calibration\", \"score\": \"0.40883441258094355\"}", - "Mean win rate - Robustness": "{\"description\": \"\", \"tab\": \"Robustness\", \"score\": \"0.3311188811188811\"}", - "Mean win rate - Fairness": "{\"description\": \"\", \"tab\": \"Fairness\", \"score\": \"0.3233799533799534\"}", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}", - "Mean win rate - Bias": "{\"description\": \"\", \"tab\": \"Bias\", \"score\": \"0.41358382155085455\"}", - "Mean win rate - Toxicity": "{\"description\": \"\", \"tab\": \"Toxicity\", \"score\": \"0.1998834498834499\"}", - "Mean win rate - Summarization metrics": "{\"description\": \"\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.302, - "details": { - "description": "min=0.228, mean=0.302, max=0.38, sum=1.508 (5)", - "tab": "Accuracy", - "MMLU - ECE (10-bin)": "{\"description\": \"min=0.08, mean=0.098, max=0.13, sum=0.49 (5)\", \"tab\": \"Calibration\", \"score\": \"0.09791468112621773\"}", - "MMLU - EM (Robustness)": "{\"description\": \"min=0.2, mean=0.25, max=0.33, sum=1.251 (5)\", \"tab\": \"Robustness\", \"score\": \"0.2501052631578947\"}", - "MMLU - EM (Fairness)": "{\"description\": \"min=0.219, mean=0.276, max=0.34, sum=1.379 (5)\", \"tab\": \"Fairness\", \"score\": \"0.275859649122807\"}", - "MMLU - Denoised inference time (s)": "{\"description\": \"5 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=358.76, mean=467.936, max=612.798, sum=2339.678 (5)\", \"tab\": \"General information\", \"score\": \"467.935649122807\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "MMLU - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "BoolQ", - "source_data": { - "dataset_name": "BoolQ", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on BoolQ", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.713, - "details": { - "description": "min=0.713, mean=0.713, max=0.713, sum=0.713 (1)", - "tab": "Accuracy", - "BoolQ - ECE (10-bin)": "{\"description\": \"min=0.127, mean=0.127, max=0.127, sum=0.127 (1)\", \"tab\": \"Calibration\", \"score\": \"0.1268200294718189\"}", - "BoolQ - EM (Robustness)": "{\"description\": \"min=0.569, mean=0.569, max=0.569, sum=0.569 (1)\", \"tab\": \"Robustness\", \"score\": \"0.569\"}", - "BoolQ - EM (Fairness)": "{\"description\": \"min=0.65, mean=0.65, max=0.65, sum=0.65 (1)\", \"tab\": \"Fairness\", \"score\": \"0.65\"}", - "BoolQ - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "BoolQ - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "BoolQ - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "BoolQ - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "BoolQ - # prompt tokens": "{\"description\": \"min=1251.897, mean=1251.897, max=1251.897, sum=1251.897 (1)\", \"tab\": \"General information\", \"score\": \"1251.897\"}", - "BoolQ - # output tokens": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "BoolQ - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "BoolQ - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.617, - "details": { - "description": "min=0.617, mean=0.617, max=0.617, sum=0.617 (1)", - "tab": "Accuracy", - "NarrativeQA - ECE (10-bin)": "{\"description\": \"min=0.276, mean=0.276, max=0.276, sum=0.276 (1)\", \"tab\": \"Calibration\", \"score\": \"0.27605359630786236\"}", - "NarrativeQA - F1 (Robustness)": "{\"description\": \"min=0.424, mean=0.424, max=0.424, sum=0.424 (1)\", \"tab\": \"Robustness\", \"score\": \"0.4240469400392869\"}", - "NarrativeQA - F1 (Fairness)": "{\"description\": \"min=0.524, mean=0.524, max=0.524, sum=0.524 (1)\", \"tab\": \"Fairness\", \"score\": \"0.5239003837979788\"}", - "NarrativeQA - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=1.969, mean=1.969, max=1.969, sum=1.969 (1)\", \"tab\": \"General information\", \"score\": \"1.9690140845070423\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=1691.082, mean=1691.082, max=1691.082, sum=1691.082 (1)\", \"tab\": \"General information\", \"score\": \"1691.081690140845\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=100, mean=100, max=100, sum=100 (1)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "NarrativeQA - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NarrativeQA - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NarrativeQA - Stereotypes (gender)": "{\"description\": \"min=0.438, mean=0.438, max=0.438, sum=0.438 (1)\", \"tab\": \"Bias\", \"score\": \"0.4375\"}", - "NarrativeQA - Representation (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=0.667 (1)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666667\"}", - "NarrativeQA - Representation (gender)": "{\"description\": \"min=0.171, mean=0.171, max=0.171, sum=0.171 (1)\", \"tab\": \"Bias\", \"score\": \"0.17123287671232879\"}", - "NarrativeQA - Toxic fraction": "{\"description\": \"min=0.014, mean=0.014, max=0.014, sum=0.014 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.014084507042253521\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (open-book)", - "source_data": { - "dataset_name": "NaturalQuestions (open-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (open-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.586, - "details": { - "description": "min=0.586, mean=0.586, max=0.586, sum=0.586 (1)", - "tab": "Accuracy", - "NaturalQuestions (closed-book) - ECE (10-bin)": "{\"description\": \"min=0.127, mean=0.127, max=0.127, sum=0.127 (1)\", \"tab\": \"Calibration\", \"score\": \"0.12699960693149975\"}", - "NaturalQuestions (open-book) - ECE (10-bin)": "{\"description\": \"min=0.396, mean=0.396, max=0.396, sum=0.396 (1)\", \"tab\": \"Calibration\", \"score\": \"0.39598996118757757\"}", - "NaturalQuestions (closed-book) - F1 (Robustness)": "{\"description\": \"min=0.167, mean=0.167, max=0.167, sum=0.167 (1)\", \"tab\": \"Robustness\", \"score\": \"0.1665503977180178\"}", - "NaturalQuestions (open-book) - F1 (Robustness)": "{\"description\": \"min=0.472, mean=0.472, max=0.472, sum=0.472 (1)\", \"tab\": \"Robustness\", \"score\": \"0.47226706838923\"}", - "NaturalQuestions (closed-book) - F1 (Fairness)": "{\"description\": \"min=0.193, mean=0.193, max=0.193, sum=0.193 (1)\", \"tab\": \"Fairness\", \"score\": \"0.19300226376410895\"}", - "NaturalQuestions (open-book) - F1 (Fairness)": "{\"description\": \"min=0.514, mean=0.514, max=0.514, sum=0.514 (1)\", \"tab\": \"Fairness\", \"score\": \"0.5136843159783826\"}", - "NaturalQuestions (closed-book) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NaturalQuestions (open-book) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=117.299, mean=117.299, max=117.299, sum=117.299 (1)\", \"tab\": \"General information\", \"score\": \"117.299\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=300, mean=300, max=300, sum=300 (1)\", \"tab\": \"General information\", \"score\": \"300.0\"}", - "NaturalQuestions (closed-book) - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.704, mean=4.704, max=4.704, sum=4.704 (1)\", \"tab\": \"General information\", \"score\": \"4.704\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.037, mean=0.037, max=0.037, sum=0.037 (1)\", \"tab\": \"General information\", \"score\": \"0.037\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1495.552, mean=1495.552, max=1495.552, sum=1495.552 (1)\", \"tab\": \"General information\", \"score\": \"1495.552\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=300, mean=300, max=300, sum=300 (1)\", \"tab\": \"General information\", \"score\": \"300.0\"}", - "NaturalQuestions (open-book) - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NaturalQuestions (closed-book) - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - Representation (race)": "{\"description\": \"min=0.498, mean=0.498, max=0.498, sum=0.498 (1)\", \"tab\": \"Bias\", \"score\": \"0.49783549783549785\"}", - "NaturalQuestions (closed-book) - Representation (gender)": "{\"description\": \"min=0.289, mean=0.289, max=0.289, sum=0.289 (1)\", \"tab\": \"Bias\", \"score\": \"0.2894736842105263\"}", - "NaturalQuestions (open-book) - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=0.667 (1)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666667\"}", - "NaturalQuestions (open-book) - Stereotypes (gender)": "{\"description\": \"min=0.408, mean=0.408, max=0.408, sum=0.408 (1)\", \"tab\": \"Bias\", \"score\": \"0.4081597222222222\"}", - "NaturalQuestions (open-book) - Representation (race)": "{\"description\": \"min=0.412, mean=0.412, max=0.412, sum=0.412 (1)\", \"tab\": \"Bias\", \"score\": \"0.4124293785310734\"}", - "NaturalQuestions (open-book) - Representation (gender)": "{\"description\": \"min=0.256, mean=0.256, max=0.256, sum=0.256 (1)\", \"tab\": \"Bias\", \"score\": \"0.25630252100840334\"}", - "NaturalQuestions (closed-book) - Toxic fraction": "{\"description\": \"min=0.002, mean=0.002, max=0.002, sum=0.002 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.002\"}", - "NaturalQuestions (open-book) - Toxic fraction": "{\"description\": \"min=0.001, mean=0.001, max=0.001, sum=0.001 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.001\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "QuAC", - "source_data": { - "dataset_name": "QuAC", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on QuAC", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.336, - "details": { - "description": "min=0.336, mean=0.336, max=0.336, sum=0.336 (1)", - "tab": "Accuracy", - "QuAC - ECE (10-bin)": "{\"description\": \"min=0.131, mean=0.131, max=0.131, sum=0.131 (1)\", \"tab\": \"Calibration\", \"score\": \"0.13131742636553145\"}", - "QuAC - F1 (Robustness)": "{\"description\": \"min=0.186, mean=0.186, max=0.186, sum=0.186 (1)\", \"tab\": \"Robustness\", \"score\": \"0.18577129287689287\"}", - "QuAC - F1 (Fairness)": "{\"description\": \"min=0.238, mean=0.238, max=0.238, sum=0.238 (1)\", \"tab\": \"Fairness\", \"score\": \"0.23848247289290064\"}", - "QuAC - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "QuAC - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "QuAC - # train": "{\"description\": \"min=0.883, mean=0.883, max=0.883, sum=0.883 (1)\", \"tab\": \"General information\", \"score\": \"0.883\"}", - "QuAC - truncated": "{\"description\": \"min=0.021, mean=0.021, max=0.021, sum=0.021 (1)\", \"tab\": \"General information\", \"score\": \"0.021\"}", - "QuAC - # prompt tokens": "{\"description\": \"min=1655.708, mean=1655.708, max=1655.708, sum=1655.708 (1)\", \"tab\": \"General information\", \"score\": \"1655.708\"}", - "QuAC - # output tokens": "{\"description\": \"min=100, mean=100, max=100, sum=100 (1)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "QuAC - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "QuAC - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=0.667 (1)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666669\"}", - "QuAC - Stereotypes (gender)": "{\"description\": \"min=0.382, mean=0.382, max=0.382, sum=0.382 (1)\", \"tab\": \"Bias\", \"score\": \"0.38163008049881736\"}", - "QuAC - Representation (race)": "{\"description\": \"min=0.38, mean=0.38, max=0.38, sum=0.38 (1)\", \"tab\": \"Bias\", \"score\": \"0.3802816901408451\"}", - "QuAC - Representation (gender)": "{\"description\": \"min=0.249, mean=0.249, max=0.249, sum=0.249 (1)\", \"tab\": \"Bias\", \"score\": \"0.24864864864864863\"}", - "QuAC - Toxic fraction": "{\"description\": \"min=0.005, mean=0.005, max=0.005, sum=0.005 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.005\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "HellaSwag", - "source_data": { - "dataset_name": "HellaSwag", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on HellaSwag", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "HellaSwag - ECE (10-bin)": "{\"description\": \"No matching runs\", \"tab\": \"Calibration\", \"score\": \"\"}", - "HellaSwag - EM (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "HellaSwag - EM (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "HellaSwag - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "HellaSwag - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "OpenbookQA - ECE (10-bin)": "{\"description\": \"No matching runs\", \"tab\": \"Calibration\", \"score\": \"\"}", - "OpenbookQA - EM (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "OpenbookQA - EM (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "OpenbookQA - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "OpenbookQA - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "TruthfulQA", - "source_data": { - "dataset_name": "TruthfulQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on TruthfulQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.205, - "details": { - "description": "min=0.205, mean=0.205, max=0.205, sum=0.205 (1)", - "tab": "Accuracy", - "TruthfulQA - ECE (10-bin)": "{\"description\": \"min=0.063, mean=0.063, max=0.063, sum=0.063 (1)\", \"tab\": \"Calibration\", \"score\": \"0.06284277332135296\"}", - "TruthfulQA - EM (Robustness)": "{\"description\": \"min=0.173, mean=0.173, max=0.173, sum=0.173 (1)\", \"tab\": \"Robustness\", \"score\": \"0.172782874617737\"}", - "TruthfulQA - EM (Fairness)": "{\"description\": \"min=0.17, mean=0.17, max=0.17, sum=0.17 (1)\", \"tab\": \"Fairness\", \"score\": \"0.16972477064220184\"}", - "TruthfulQA - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "TruthfulQA - # eval": "{\"description\": \"min=654, mean=654, max=654, sum=654 (1)\", \"tab\": \"General information\", \"score\": \"654.0\"}", - "TruthfulQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "TruthfulQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "TruthfulQA - # prompt tokens": "{\"description\": \"min=505.352, mean=505.352, max=505.352, sum=505.352 (1)\", \"tab\": \"General information\", \"score\": \"505.35168195718654\"}", - "TruthfulQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "TruthfulQA - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MS MARCO (TREC)", - "source_data": { - "dataset_name": "MS MARCO (TREC)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "NDCG@10 on MS MARCO (TREC)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "MS MARCO (regular) - RR@10 (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "MS MARCO (TREC) - NDCG@10 (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "MS MARCO (regular) - RR@10 (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "MS MARCO (TREC) - NDCG@10 (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "MS MARCO (regular) - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MS MARCO (TREC) - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MS MARCO (regular) - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "MS MARCO (TREC) - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CNN/DailyMail", - "source_data": { - "dataset_name": "CNN/DailyMail", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on CNN/DailyMail", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "CNN/DailyMail - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "CNN/DailyMail - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "CNN/DailyMail - SummaC": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - QAFactEval": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - BERTScore (F1)": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - Coverage": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - Density": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - Compression": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-faithfulness": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-relevance": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-coherence": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "XSUM", - "source_data": { - "dataset_name": "XSUM", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on XSUM", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "XSUM - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "XSUM - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "XSUM - SummaC": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - QAFactEval": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - BERTScore (F1)": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - Coverage": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - Density": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - Compression": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-faithfulness": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-relevance": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-coherence": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "IMDB", - "source_data": { - "dataset_name": "IMDB", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on IMDB", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.752, - "details": { - "description": "min=0.752, mean=0.752, max=0.752, sum=0.752 (1)", - "tab": "Accuracy", - "IMDB - ECE (10-bin)": "{\"description\": \"min=0.206, mean=0.206, max=0.206, sum=0.206 (1)\", \"tab\": \"Calibration\", \"score\": \"0.20649886073889429\"}", - "IMDB - EM (Robustness)": "{\"description\": \"min=0.56, mean=0.56, max=0.56, sum=0.56 (1)\", \"tab\": \"Robustness\", \"score\": \"0.56\"}", - "IMDB - EM (Fairness)": "{\"description\": \"min=0.694, mean=0.694, max=0.694, sum=0.694 (1)\", \"tab\": \"Fairness\", \"score\": \"0.694\"}", - "IMDB - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "IMDB - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "IMDB - # train": "{\"description\": \"min=2.911, mean=2.911, max=2.911, sum=2.911 (1)\", \"tab\": \"General information\", \"score\": \"2.911\"}", - "IMDB - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IMDB - # prompt tokens": "{\"description\": \"min=1619.568, mean=1619.568, max=1619.568, sum=1619.568 (1)\", \"tab\": \"General information\", \"score\": \"1619.568\"}", - "IMDB - # output tokens": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "IMDB - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "IMDB - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CivilComments", - "source_data": { - "dataset_name": "CivilComments", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on CivilComments", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.547, - "details": { - "description": "min=0.064, mean=0.547, max=0.954, sum=9.838 (18)", - "tab": "Accuracy", - "CivilComments - ECE (10-bin)": "{\"description\": \"min=0.109, mean=0.305, max=0.471, sum=5.486 (18)\", \"tab\": \"Calibration\", \"score\": \"0.3047575712176879\"}", - "CivilComments - EM (Robustness)": "{\"description\": \"min=0.054, mean=0.401, max=0.835, sum=7.221 (18)\", \"tab\": \"Robustness\", \"score\": \"0.4011569280490217\"}", - "CivilComments - EM (Fairness)": "{\"description\": \"min=0.06, mean=0.431, max=0.811, sum=7.756 (18)\", \"tab\": \"Fairness\", \"score\": \"0.43087088541137863\"}", - "CivilComments - Denoised inference time (s)": "{\"description\": \"9 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "CivilComments - # eval": "{\"description\": \"min=74, mean=371.556, max=683, sum=6688 (18)\", \"tab\": \"General information\", \"score\": \"371.55555555555554\"}", - "CivilComments - # train": "{\"description\": \"min=5, mean=5, max=5, sum=90 (18)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CivilComments - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (18)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CivilComments - # prompt tokens": "{\"description\": \"min=360.976, mean=771.654, max=1282.4, sum=13889.772 (18)\", \"tab\": \"General information\", \"score\": \"771.6539847352628\"}", - "CivilComments - # output tokens": "{\"description\": \"min=5, mean=5, max=5, sum=90 (18)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CivilComments - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=18 (18)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "CivilComments - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Toxic fraction": "{\"description\": \"9 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "RAFT", - "source_data": { - "dataset_name": "RAFT", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on RAFT", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.648, - "details": { - "description": "min=0.3, mean=0.648, max=0.925, sum=7.125 (11)", - "tab": "Accuracy", - "RAFT - ECE (10-bin)": "{\"description\": \"min=0.3, mean=0.648, max=0.925, sum=7.123 (11)\", \"tab\": \"Calibration\", \"score\": \"0.6475429539256364\"}", - "RAFT - EM (Robustness)": "{\"description\": \"min=0, mean=0.489, max=0.925, sum=5.375 (11)\", \"tab\": \"Robustness\", \"score\": \"0.48863636363636365\"}", - "RAFT - EM (Fairness)": "{\"description\": \"min=0.275, mean=0.595, max=0.925, sum=6.55 (11)\", \"tab\": \"Fairness\", \"score\": \"0.5954545454545455\"}", - "RAFT - Denoised inference time (s)": "{\"description\": \"11 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "RAFT - # eval": "{\"description\": \"min=40, mean=40, max=40, sum=440 (11)\", \"tab\": \"General information\", \"score\": \"40.0\"}", - "RAFT - # train": "{\"description\": \"min=0.7, mean=4.605, max=5, sum=50.65 (11)\", \"tab\": \"General information\", \"score\": \"4.6045454545454545\"}", - "RAFT - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (11)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "RAFT - # prompt tokens": "{\"description\": \"min=280.35, mean=869.691, max=1756.575, sum=9566.6 (11)\", \"tab\": \"General information\", \"score\": \"869.6909090909089\"}", - "RAFT - # output tokens": "{\"description\": \"min=30, mean=30, max=30, sum=330 (11)\", \"tab\": \"General information\", \"score\": \"30.0\"}", - "RAFT - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=11 (11)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "RAFT - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Toxic fraction": "{\"description\": \"11 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/together_redpajama-incite-base-v1-3b.json b/data/models/together_redpajama-incite-base-v1-3b.json deleted file mode 100644 index c301097d8f7b3f0c14018d97853983f53e1d68cb..0000000000000000000000000000000000000000 --- a/data/models/together_redpajama-incite-base-v1-3b.json +++ /dev/null @@ -1,674 +0,0 @@ -{ - "model_info": { - "name": "RedPajama-INCITE-Base-v1 3B", - "id": "together/RedPajama-INCITE-Base-v1-3B", - "developer": "together", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_classic/together_RedPajama-INCITE-Base-v1-3B/1774096308.339228", - "retrieved_timestamp": "1774096308.339228", - "source_metadata": { - "source_name": "helm_classic", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_classic", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_classic", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperform on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.311, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Calibration": "{\"description\": \"\", \"tab\": \"Calibration\", \"score\": \"0.4387141535615171\"}", - "Mean win rate - Robustness": "{\"description\": \"\", \"tab\": \"Robustness\", \"score\": \"0.293006993006993\"}", - "Mean win rate - Fairness": "{\"description\": \"\", \"tab\": \"Fairness\", \"score\": \"0.26995337995338\"}", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}", - "Mean win rate - Bias": "{\"description\": \"\", \"tab\": \"Bias\", \"score\": \"0.4599624127215427\"}", - "Mean win rate - Toxicity": "{\"description\": \"\", \"tab\": \"Toxicity\", \"score\": \"0.7068181818181818\"}", - "Mean win rate - Summarization metrics": "{\"description\": \"\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.263, - "details": { - "description": "min=0.24, mean=0.263, max=0.3, sum=1.314 (5)", - "tab": "Accuracy", - "MMLU - ECE (10-bin)": "{\"description\": \"min=0.082, mean=0.115, max=0.149, sum=0.575 (5)\", \"tab\": \"Calibration\", \"score\": \"0.11506526711032969\"}", - "MMLU - EM (Robustness)": "{\"description\": \"min=0.184, mean=0.217, max=0.29, sum=1.084 (5)\", \"tab\": \"Robustness\", \"score\": \"0.2168421052631579\"}", - "MMLU - EM (Fairness)": "{\"description\": \"min=0.2, mean=0.232, max=0.29, sum=1.161 (5)\", \"tab\": \"Fairness\", \"score\": \"0.23210526315789473\"}", - "MMLU - Denoised inference time (s)": "{\"description\": \"5 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=358.76, mean=467.936, max=612.798, sum=2339.678 (5)\", \"tab\": \"General information\", \"score\": \"467.935649122807\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "MMLU - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "BoolQ", - "source_data": { - "dataset_name": "BoolQ", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on BoolQ", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.685, - "details": { - "description": "min=0.685, mean=0.685, max=0.685, sum=0.685 (1)", - "tab": "Accuracy", - "BoolQ - ECE (10-bin)": "{\"description\": \"min=0.187, mean=0.187, max=0.187, sum=0.187 (1)\", \"tab\": \"Calibration\", \"score\": \"0.1865846445420437\"}", - "BoolQ - EM (Robustness)": "{\"description\": \"min=0.585, mean=0.585, max=0.585, sum=0.585 (1)\", \"tab\": \"Robustness\", \"score\": \"0.585\"}", - "BoolQ - EM (Fairness)": "{\"description\": \"min=0.624, mean=0.624, max=0.624, sum=0.624 (1)\", \"tab\": \"Fairness\", \"score\": \"0.624\"}", - "BoolQ - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "BoolQ - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "BoolQ - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "BoolQ - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "BoolQ - # prompt tokens": "{\"description\": \"min=1251.897, mean=1251.897, max=1251.897, sum=1251.897 (1)\", \"tab\": \"General information\", \"score\": \"1251.897\"}", - "BoolQ - # output tokens": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "BoolQ - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "BoolQ - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.555, - "details": { - "description": "min=0.555, mean=0.555, max=0.555, sum=0.555 (1)", - "tab": "Accuracy", - "NarrativeQA - ECE (10-bin)": "{\"description\": \"min=0.234, mean=0.234, max=0.234, sum=0.234 (1)\", \"tab\": \"Calibration\", \"score\": \"0.2338003327407993\"}", - "NarrativeQA - F1 (Robustness)": "{\"description\": \"min=0.346, mean=0.346, max=0.346, sum=0.346 (1)\", \"tab\": \"Robustness\", \"score\": \"0.3460535146763825\"}", - "NarrativeQA - F1 (Fairness)": "{\"description\": \"min=0.42, mean=0.42, max=0.42, sum=0.42 (1)\", \"tab\": \"Fairness\", \"score\": \"0.42019517663794076\"}", - "NarrativeQA - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=1.969, mean=1.969, max=1.969, sum=1.969 (1)\", \"tab\": \"General information\", \"score\": \"1.9690140845070423\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=1691.082, mean=1691.082, max=1691.082, sum=1691.082 (1)\", \"tab\": \"General information\", \"score\": \"1691.081690140845\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=100, mean=100, max=100, sum=100 (1)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "NarrativeQA - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NarrativeQA - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NarrativeQA - Stereotypes (gender)": "{\"description\": \"min=0.463, mean=0.463, max=0.463, sum=0.463 (1)\", \"tab\": \"Bias\", \"score\": \"0.4629629629629629\"}", - "NarrativeQA - Representation (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=0.667 (1)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666667\"}", - "NarrativeQA - Representation (gender)": "{\"description\": \"min=0.167, mean=0.167, max=0.167, sum=0.167 (1)\", \"tab\": \"Bias\", \"score\": \"0.16666666666666666\"}", - "NarrativeQA - Toxic fraction": "{\"description\": \"min=0.008, mean=0.008, max=0.008, sum=0.008 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.008450704225352112\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (open-book)", - "source_data": { - "dataset_name": "NaturalQuestions (open-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (open-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.52, - "details": { - "description": "min=0.52, mean=0.52, max=0.52, sum=0.52 (1)", - "tab": "Accuracy", - "NaturalQuestions (closed-book) - ECE (10-bin)": "{\"description\": \"min=0.116, mean=0.116, max=0.116, sum=0.116 (1)\", \"tab\": \"Calibration\", \"score\": \"0.1159999973291356\"}", - "NaturalQuestions (open-book) - ECE (10-bin)": "{\"description\": \"min=0.345, mean=0.345, max=0.345, sum=0.345 (1)\", \"tab\": \"Calibration\", \"score\": \"0.34498406074093657\"}", - "NaturalQuestions (closed-book) - F1 (Robustness)": "{\"description\": \"min=0.134, mean=0.134, max=0.134, sum=0.134 (1)\", \"tab\": \"Robustness\", \"score\": \"0.1341635313992508\"}", - "NaturalQuestions (open-book) - F1 (Robustness)": "{\"description\": \"min=0.396, mean=0.396, max=0.396, sum=0.396 (1)\", \"tab\": \"Robustness\", \"score\": \"0.3964044537010397\"}", - "NaturalQuestions (closed-book) - F1 (Fairness)": "{\"description\": \"min=0.145, mean=0.145, max=0.145, sum=0.145 (1)\", \"tab\": \"Fairness\", \"score\": \"0.14546689822682907\"}", - "NaturalQuestions (open-book) - F1 (Fairness)": "{\"description\": \"min=0.452, mean=0.452, max=0.452, sum=0.452 (1)\", \"tab\": \"Fairness\", \"score\": \"0.4521647378074364\"}", - "NaturalQuestions (closed-book) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NaturalQuestions (open-book) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=117.299, mean=117.299, max=117.299, sum=117.299 (1)\", \"tab\": \"General information\", \"score\": \"117.299\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=300, mean=300, max=300, sum=300 (1)\", \"tab\": \"General information\", \"score\": \"300.0\"}", - "NaturalQuestions (closed-book) - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.704, mean=4.704, max=4.704, sum=4.704 (1)\", \"tab\": \"General information\", \"score\": \"4.704\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.037, mean=0.037, max=0.037, sum=0.037 (1)\", \"tab\": \"General information\", \"score\": \"0.037\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1495.552, mean=1495.552, max=1495.552, sum=1495.552 (1)\", \"tab\": \"General information\", \"score\": \"1495.552\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=299.738, mean=299.738, max=299.738, sum=299.738 (1)\", \"tab\": \"General information\", \"score\": \"299.738\"}", - "NaturalQuestions (open-book) - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NaturalQuestions (closed-book) - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - Stereotypes (gender)": "{\"description\": \"min=0.5, mean=0.5, max=0.5, sum=0.5 (1)\", \"tab\": \"Bias\", \"score\": \"0.5\"}", - "NaturalQuestions (closed-book) - Representation (race)": "{\"description\": \"min=0.46, mean=0.46, max=0.46, sum=0.46 (1)\", \"tab\": \"Bias\", \"score\": \"0.4597701149425287\"}", - "NaturalQuestions (closed-book) - Representation (gender)": "{\"description\": \"min=0.182, mean=0.182, max=0.182, sum=0.182 (1)\", \"tab\": \"Bias\", \"score\": \"0.18181818181818182\"}", - "NaturalQuestions (open-book) - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=0.667 (1)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666667\"}", - "NaturalQuestions (open-book) - Stereotypes (gender)": "{\"description\": \"min=0.464, mean=0.464, max=0.464, sum=0.464 (1)\", \"tab\": \"Bias\", \"score\": \"0.4642857142857143\"}", - "NaturalQuestions (open-book) - Representation (race)": "{\"description\": \"min=0.453, mean=0.453, max=0.453, sum=0.453 (1)\", \"tab\": \"Bias\", \"score\": \"0.45299145299145294\"}", - "NaturalQuestions (open-book) - Representation (gender)": "{\"description\": \"min=0.242, mean=0.242, max=0.242, sum=0.242 (1)\", \"tab\": \"Bias\", \"score\": \"0.24223602484472045\"}", - "NaturalQuestions (closed-book) - Toxic fraction": "{\"description\": \"min=0.001, mean=0.001, max=0.001, sum=0.001 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.001\"}", - "NaturalQuestions (open-book) - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "QuAC", - "source_data": { - "dataset_name": "QuAC", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on QuAC", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.309, - "details": { - "description": "min=0.309, mean=0.309, max=0.309, sum=0.309 (1)", - "tab": "Accuracy", - "QuAC - ECE (10-bin)": "{\"description\": \"min=0.078, mean=0.078, max=0.078, sum=0.078 (1)\", \"tab\": \"Calibration\", \"score\": \"0.07775925403447285\"}", - "QuAC - F1 (Robustness)": "{\"description\": \"min=0.177, mean=0.177, max=0.177, sum=0.177 (1)\", \"tab\": \"Robustness\", \"score\": \"0.17735561911839576\"}", - "QuAC - F1 (Fairness)": "{\"description\": \"min=0.238, mean=0.238, max=0.238, sum=0.238 (1)\", \"tab\": \"Fairness\", \"score\": \"0.23753496056157644\"}", - "QuAC - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "QuAC - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "QuAC - # train": "{\"description\": \"min=0.883, mean=0.883, max=0.883, sum=0.883 (1)\", \"tab\": \"General information\", \"score\": \"0.883\"}", - "QuAC - truncated": "{\"description\": \"min=0.021, mean=0.021, max=0.021, sum=0.021 (1)\", \"tab\": \"General information\", \"score\": \"0.021\"}", - "QuAC - # prompt tokens": "{\"description\": \"min=1655.708, mean=1655.708, max=1655.708, sum=1655.708 (1)\", \"tab\": \"General information\", \"score\": \"1655.708\"}", - "QuAC - # output tokens": "{\"description\": \"min=100, mean=100, max=100, sum=100 (1)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "QuAC - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "QuAC - Stereotypes (race)": "{\"description\": \"min=0.575, mean=0.575, max=0.575, sum=0.575 (1)\", \"tab\": \"Bias\", \"score\": \"0.575\"}", - "QuAC - Stereotypes (gender)": "{\"description\": \"min=0.389, mean=0.389, max=0.389, sum=0.389 (1)\", \"tab\": \"Bias\", \"score\": \"0.38936550778656037\"}", - "QuAC - Representation (race)": "{\"description\": \"min=0.3, mean=0.3, max=0.3, sum=0.3 (1)\", \"tab\": \"Bias\", \"score\": \"0.3003300330033003\"}", - "QuAC - Representation (gender)": "{\"description\": \"min=0.269, mean=0.269, max=0.269, sum=0.269 (1)\", \"tab\": \"Bias\", \"score\": \"0.268640350877193\"}", - "QuAC - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "HellaSwag", - "source_data": { - "dataset_name": "HellaSwag", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on HellaSwag", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "HellaSwag - ECE (10-bin)": "{\"description\": \"No matching runs\", \"tab\": \"Calibration\", \"score\": \"\"}", - "HellaSwag - EM (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "HellaSwag - EM (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "HellaSwag - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "HellaSwag - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "OpenbookQA - ECE (10-bin)": "{\"description\": \"No matching runs\", \"tab\": \"Calibration\", \"score\": \"\"}", - "OpenbookQA - EM (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "OpenbookQA - EM (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "OpenbookQA - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "OpenbookQA - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "TruthfulQA", - "source_data": { - "dataset_name": "TruthfulQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on TruthfulQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.277, - "details": { - "description": "min=0.277, mean=0.277, max=0.277, sum=0.277 (1)", - "tab": "Accuracy", - "TruthfulQA - ECE (10-bin)": "{\"description\": \"min=0.048, mean=0.048, max=0.048, sum=0.048 (1)\", \"tab\": \"Calibration\", \"score\": \"0.04833037892853392\"}", - "TruthfulQA - EM (Robustness)": "{\"description\": \"min=0.226, mean=0.226, max=0.226, sum=0.226 (1)\", \"tab\": \"Robustness\", \"score\": \"0.22629969418960244\"}", - "TruthfulQA - EM (Fairness)": "{\"description\": \"min=0.248, mean=0.248, max=0.248, sum=0.248 (1)\", \"tab\": \"Fairness\", \"score\": \"0.24770642201834864\"}", - "TruthfulQA - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "TruthfulQA - # eval": "{\"description\": \"min=654, mean=654, max=654, sum=654 (1)\", \"tab\": \"General information\", \"score\": \"654.0\"}", - "TruthfulQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "TruthfulQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "TruthfulQA - # prompt tokens": "{\"description\": \"min=505.352, mean=505.352, max=505.352, sum=505.352 (1)\", \"tab\": \"General information\", \"score\": \"505.35168195718654\"}", - "TruthfulQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "TruthfulQA - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MS MARCO (TREC)", - "source_data": { - "dataset_name": "MS MARCO (TREC)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "NDCG@10 on MS MARCO (TREC)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "MS MARCO (regular) - RR@10 (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "MS MARCO (TREC) - NDCG@10 (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "MS MARCO (regular) - RR@10 (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "MS MARCO (TREC) - NDCG@10 (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "MS MARCO (regular) - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MS MARCO (TREC) - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MS MARCO (regular) - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "MS MARCO (TREC) - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CNN/DailyMail", - "source_data": { - "dataset_name": "CNN/DailyMail", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on CNN/DailyMail", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "CNN/DailyMail - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "CNN/DailyMail - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "CNN/DailyMail - SummaC": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - QAFactEval": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - BERTScore (F1)": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - Coverage": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - Density": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - Compression": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-faithfulness": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-relevance": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-coherence": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "XSUM", - "source_data": { - "dataset_name": "XSUM", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on XSUM", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "XSUM - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "XSUM - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "XSUM - SummaC": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - QAFactEval": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - BERTScore (F1)": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - Coverage": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - Density": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - Compression": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-faithfulness": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-relevance": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-coherence": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "IMDB", - "source_data": { - "dataset_name": "IMDB", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on IMDB", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.907, - "details": { - "description": "min=0.907, mean=0.907, max=0.907, sum=0.907 (1)", - "tab": "Accuracy", - "IMDB - ECE (10-bin)": "{\"description\": \"min=0.248, mean=0.248, max=0.248, sum=0.248 (1)\", \"tab\": \"Calibration\", \"score\": \"0.24822902119068743\"}", - "IMDB - EM (Robustness)": "{\"description\": \"min=0.843, mean=0.843, max=0.843, sum=0.843 (1)\", \"tab\": \"Robustness\", \"score\": \"0.843\"}", - "IMDB - EM (Fairness)": "{\"description\": \"min=0.89, mean=0.89, max=0.89, sum=0.89 (1)\", \"tab\": \"Fairness\", \"score\": \"0.89\"}", - "IMDB - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "IMDB - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "IMDB - # train": "{\"description\": \"min=2.911, mean=2.911, max=2.911, sum=2.911 (1)\", \"tab\": \"General information\", \"score\": \"2.911\"}", - "IMDB - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IMDB - # prompt tokens": "{\"description\": \"min=1619.568, mean=1619.568, max=1619.568, sum=1619.568 (1)\", \"tab\": \"General information\", \"score\": \"1619.568\"}", - "IMDB - # output tokens": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "IMDB - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "IMDB - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CivilComments", - "source_data": { - "dataset_name": "CivilComments", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on CivilComments", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.549, - "details": { - "description": "min=0.013, mean=0.549, max=0.996, sum=9.877 (18)", - "tab": "Accuracy", - "CivilComments - ECE (10-bin)": "{\"description\": \"min=0.105, mean=0.303, max=0.532, sum=5.455 (18)\", \"tab\": \"Calibration\", \"score\": \"0.3030711579633833\"}", - "CivilComments - EM (Robustness)": "{\"description\": \"min=0, mean=0.336, max=0.996, sum=6.045 (18)\", \"tab\": \"Robustness\", \"score\": \"0.3358431190860201\"}", - "CivilComments - EM (Fairness)": "{\"description\": \"min=0, mean=0.393, max=0.996, sum=7.082 (18)\", \"tab\": \"Fairness\", \"score\": \"0.39345093425226885\"}", - "CivilComments - Denoised inference time (s)": "{\"description\": \"9 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "CivilComments - # eval": "{\"description\": \"min=74, mean=371.556, max=683, sum=6688 (18)\", \"tab\": \"General information\", \"score\": \"371.55555555555554\"}", - "CivilComments - # train": "{\"description\": \"min=5, mean=5, max=5, sum=90 (18)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CivilComments - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (18)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CivilComments - # prompt tokens": "{\"description\": \"min=360.976, mean=771.654, max=1282.4, sum=13889.772 (18)\", \"tab\": \"General information\", \"score\": \"771.6539847352628\"}", - "CivilComments - # output tokens": "{\"description\": \"min=5, mean=5, max=5, sum=90 (18)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CivilComments - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=18 (18)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "CivilComments - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Toxic fraction": "{\"description\": \"9 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "RAFT", - "source_data": { - "dataset_name": "RAFT", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on RAFT", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.502, - "details": { - "description": "min=0.225, mean=0.502, max=0.975, sum=5.525 (11)", - "tab": "Accuracy", - "RAFT - ECE (10-bin)": "{\"description\": \"min=0.225, mean=0.502, max=0.975, sum=5.524 (11)\", \"tab\": \"Calibration\", \"score\": \"0.5021656428017803\"}", - "RAFT - EM (Robustness)": "{\"description\": \"min=0, mean=0.427, max=0.975, sum=4.7 (11)\", \"tab\": \"Robustness\", \"score\": \"0.4272727272727273\"}", - "RAFT - EM (Fairness)": "{\"description\": \"min=0.175, mean=0.475, max=0.975, sum=5.225 (11)\", \"tab\": \"Fairness\", \"score\": \"0.47500000000000003\"}", - "RAFT - Denoised inference time (s)": "{\"description\": \"11 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "RAFT - # eval": "{\"description\": \"min=40, mean=40, max=40, sum=440 (11)\", \"tab\": \"General information\", \"score\": \"40.0\"}", - "RAFT - # train": "{\"description\": \"min=0.7, mean=4.605, max=5, sum=50.65 (11)\", \"tab\": \"General information\", \"score\": \"4.6045454545454545\"}", - "RAFT - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (11)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "RAFT - # prompt tokens": "{\"description\": \"min=280.35, mean=869.691, max=1756.575, sum=9566.6 (11)\", \"tab\": \"General information\", \"score\": \"869.6909090909089\"}", - "RAFT - # output tokens": "{\"description\": \"min=30, mean=30, max=30, sum=330 (11)\", \"tab\": \"General information\", \"score\": \"30.0\"}", - "RAFT - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=11 (11)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "RAFT - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Toxic fraction": "{\"description\": \"11 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/together_redpajama-incite-instruct-7b.json b/data/models/together_redpajama-incite-instruct-7b.json deleted file mode 100644 index b3032516ddb2e84e45e3d4a3aa1cdfb3d804e0e2..0000000000000000000000000000000000000000 --- a/data/models/together_redpajama-incite-instruct-7b.json +++ /dev/null @@ -1,674 +0,0 @@ -{ - "model_info": { - "name": "RedPajama-INCITE-Instruct 7B", - "id": "together/RedPajama-INCITE-Instruct-7B", - "developer": "together", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_classic/together_RedPajama-INCITE-Instruct-7B/1774096308.339228", - "retrieved_timestamp": "1774096308.339228", - "source_metadata": { - "source_name": "helm_classic", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_classic", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_classic", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperform on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.524, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Calibration": "{\"description\": \"\", \"tab\": \"Calibration\", \"score\": \"0.38751156336725257\"}", - "Mean win rate - Robustness": "{\"description\": \"\", \"tab\": \"Robustness\", \"score\": \"0.4953146853146853\"}", - "Mean win rate - Fairness": "{\"description\": \"\", \"tab\": \"Fairness\", \"score\": \"0.46615384615384614\"}", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}", - "Mean win rate - Bias": "{\"description\": \"\", \"tab\": \"Bias\", \"score\": \"0.33794748465968927\"}", - "Mean win rate - Toxicity": "{\"description\": \"\", \"tab\": \"Toxicity\", \"score\": \"0.29364801864801865\"}", - "Mean win rate - Summarization metrics": "{\"description\": \"\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.363, - "details": { - "description": "min=0.246, mean=0.363, max=0.52, sum=1.816 (5)", - "tab": "Accuracy", - "MMLU - ECE (10-bin)": "{\"description\": \"min=0.092, mean=0.143, max=0.182, sum=0.715 (5)\", \"tab\": \"Calibration\", \"score\": \"0.14292977551638825\"}", - "MMLU - EM (Robustness)": "{\"description\": \"min=0.175, mean=0.291, max=0.46, sum=1.455 (5)\", \"tab\": \"Robustness\", \"score\": \"0.2910877192982456\"}", - "MMLU - EM (Fairness)": "{\"description\": \"min=0.167, mean=0.305, max=0.48, sum=1.527 (5)\", \"tab\": \"Fairness\", \"score\": \"0.30533333333333335\"}", - "MMLU - Denoised inference time (s)": "{\"description\": \"5 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=358.76, mean=467.936, max=612.798, sum=2339.678 (5)\", \"tab\": \"General information\", \"score\": \"467.935649122807\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "MMLU - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "BoolQ", - "source_data": { - "dataset_name": "BoolQ", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on BoolQ", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.705, - "details": { - "description": "min=0.705, mean=0.705, max=0.705, sum=0.705 (1)", - "tab": "Accuracy", - "BoolQ - ECE (10-bin)": "{\"description\": \"min=0.035, mean=0.035, max=0.035, sum=0.035 (1)\", \"tab\": \"Calibration\", \"score\": \"0.034644312737608846\"}", - "BoolQ - EM (Robustness)": "{\"description\": \"min=0.599, mean=0.599, max=0.599, sum=0.599 (1)\", \"tab\": \"Robustness\", \"score\": \"0.599\"}", - "BoolQ - EM (Fairness)": "{\"description\": \"min=0.616, mean=0.616, max=0.616, sum=0.616 (1)\", \"tab\": \"Fairness\", \"score\": \"0.616\"}", - "BoolQ - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "BoolQ - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "BoolQ - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "BoolQ - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "BoolQ - # prompt tokens": "{\"description\": \"min=1251.897, mean=1251.897, max=1251.897, sum=1251.897 (1)\", \"tab\": \"General information\", \"score\": \"1251.897\"}", - "BoolQ - # output tokens": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "BoolQ - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "BoolQ - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.638, - "details": { - "description": "min=0.638, mean=0.638, max=0.638, sum=0.638 (1)", - "tab": "Accuracy", - "NarrativeQA - ECE (10-bin)": "{\"description\": \"min=0.247, mean=0.247, max=0.247, sum=0.247 (1)\", \"tab\": \"Calibration\", \"score\": \"0.24703559378209236\"}", - "NarrativeQA - F1 (Robustness)": "{\"description\": \"min=0.482, mean=0.482, max=0.482, sum=0.482 (1)\", \"tab\": \"Robustness\", \"score\": \"0.4816661888359549\"}", - "NarrativeQA - F1 (Fairness)": "{\"description\": \"min=0.506, mean=0.506, max=0.506, sum=0.506 (1)\", \"tab\": \"Fairness\", \"score\": \"0.5062845788047843\"}", - "NarrativeQA - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=1.969, mean=1.969, max=1.969, sum=1.969 (1)\", \"tab\": \"General information\", \"score\": \"1.9690140845070423\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=1691.082, mean=1691.082, max=1691.082, sum=1691.082 (1)\", \"tab\": \"General information\", \"score\": \"1691.081690140845\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=100, mean=100, max=100, sum=100 (1)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "NarrativeQA - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NarrativeQA - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NarrativeQA - Stereotypes (gender)": "{\"description\": \"min=0.5, mean=0.5, max=0.5, sum=0.5 (1)\", \"tab\": \"Bias\", \"score\": \"0.5\"}", - "NarrativeQA - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NarrativeQA - Representation (gender)": "{\"description\": \"min=0.193, mean=0.193, max=0.193, sum=0.193 (1)\", \"tab\": \"Bias\", \"score\": \"0.19318181818181815\"}", - "NarrativeQA - Toxic fraction": "{\"description\": \"min=0.025, mean=0.025, max=0.025, sum=0.025 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.02535211267605634\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (open-book)", - "source_data": { - "dataset_name": "NaturalQuestions (open-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (open-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.659, - "details": { - "description": "min=0.659, mean=0.659, max=0.659, sum=0.659 (1)", - "tab": "Accuracy", - "NaturalQuestions (closed-book) - ECE (10-bin)": "{\"description\": \"min=0.142, mean=0.142, max=0.142, sum=0.142 (1)\", \"tab\": \"Calibration\", \"score\": \"0.14200000000000002\"}", - "NaturalQuestions (open-book) - ECE (10-bin)": "{\"description\": \"min=0.466, mean=0.466, max=0.466, sum=0.466 (1)\", \"tab\": \"Calibration\", \"score\": \"0.4659999973351183\"}", - "NaturalQuestions (closed-book) - F1 (Robustness)": "{\"description\": \"min=0.137, mean=0.137, max=0.137, sum=0.137 (1)\", \"tab\": \"Robustness\", \"score\": \"0.13717330495393032\"}", - "NaturalQuestions (open-book) - F1 (Robustness)": "{\"description\": \"min=0.547, mean=0.547, max=0.547, sum=0.547 (1)\", \"tab\": \"Robustness\", \"score\": \"0.5468327185577326\"}", - "NaturalQuestions (closed-book) - F1 (Fairness)": "{\"description\": \"min=0.164, mean=0.164, max=0.164, sum=0.164 (1)\", \"tab\": \"Fairness\", \"score\": \"0.16419040044922398\"}", - "NaturalQuestions (open-book) - F1 (Fairness)": "{\"description\": \"min=0.592, mean=0.592, max=0.592, sum=0.592 (1)\", \"tab\": \"Fairness\", \"score\": \"0.5920301139461878\"}", - "NaturalQuestions (closed-book) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NaturalQuestions (open-book) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=117.299, mean=117.299, max=117.299, sum=117.299 (1)\", \"tab\": \"General information\", \"score\": \"117.299\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=300, mean=300, max=300, sum=300 (1)\", \"tab\": \"General information\", \"score\": \"300.0\"}", - "NaturalQuestions (closed-book) - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.704, mean=4.704, max=4.704, sum=4.704 (1)\", \"tab\": \"General information\", \"score\": \"4.704\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.037, mean=0.037, max=0.037, sum=0.037 (1)\", \"tab\": \"General information\", \"score\": \"0.037\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1495.552, mean=1495.552, max=1495.552, sum=1495.552 (1)\", \"tab\": \"General information\", \"score\": \"1495.552\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=300, mean=300, max=300, sum=300 (1)\", \"tab\": \"General information\", \"score\": \"300.0\"}", - "NaturalQuestions (open-book) - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NaturalQuestions (closed-book) - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=0.667 (1)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666667\"}", - "NaturalQuestions (closed-book) - Stereotypes (gender)": "{\"description\": \"min=0.5, mean=0.5, max=0.5, sum=0.5 (1)\", \"tab\": \"Bias\", \"score\": \"0.5\"}", - "NaturalQuestions (closed-book) - Representation (race)": "{\"description\": \"min=0.406, mean=0.406, max=0.406, sum=0.406 (1)\", \"tab\": \"Bias\", \"score\": \"0.4061624649859944\"}", - "NaturalQuestions (closed-book) - Representation (gender)": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"Bias\", \"score\": \"0.0\"}", - "NaturalQuestions (open-book) - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=0.667 (1)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666667\"}", - "NaturalQuestions (open-book) - Stereotypes (gender)": "{\"description\": \"min=0.5, mean=0.5, max=0.5, sum=0.5 (1)\", \"tab\": \"Bias\", \"score\": \"0.5\"}", - "NaturalQuestions (open-book) - Representation (race)": "{\"description\": \"min=0.524, mean=0.524, max=0.524, sum=0.524 (1)\", \"tab\": \"Bias\", \"score\": \"0.5238095238095237\"}", - "NaturalQuestions (open-book) - Representation (gender)": "{\"description\": \"min=0.281, mean=0.281, max=0.281, sum=0.281 (1)\", \"tab\": \"Bias\", \"score\": \"0.28125\"}", - "NaturalQuestions (closed-book) - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}", - "NaturalQuestions (open-book) - Toxic fraction": "{\"description\": \"min=0.001, mean=0.001, max=0.001, sum=0.001 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.001\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "QuAC", - "source_data": { - "dataset_name": "QuAC", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on QuAC", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.26, - "details": { - "description": "min=0.26, mean=0.26, max=0.26, sum=0.26 (1)", - "tab": "Accuracy", - "QuAC - ECE (10-bin)": "{\"description\": \"min=0.074, mean=0.074, max=0.074, sum=0.074 (1)\", \"tab\": \"Calibration\", \"score\": \"0.07389119661461117\"}", - "QuAC - F1 (Robustness)": "{\"description\": \"min=0.164, mean=0.164, max=0.164, sum=0.164 (1)\", \"tab\": \"Robustness\", \"score\": \"0.16438450644529176\"}", - "QuAC - F1 (Fairness)": "{\"description\": \"min=0.181, mean=0.181, max=0.181, sum=0.181 (1)\", \"tab\": \"Fairness\", \"score\": \"0.18079535886869938\"}", - "QuAC - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "QuAC - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "QuAC - # train": "{\"description\": \"min=0.883, mean=0.883, max=0.883, sum=0.883 (1)\", \"tab\": \"General information\", \"score\": \"0.883\"}", - "QuAC - truncated": "{\"description\": \"min=0.021, mean=0.021, max=0.021, sum=0.021 (1)\", \"tab\": \"General information\", \"score\": \"0.021\"}", - "QuAC - # prompt tokens": "{\"description\": \"min=1655.708, mean=1655.708, max=1655.708, sum=1655.708 (1)\", \"tab\": \"General information\", \"score\": \"1655.708\"}", - "QuAC - # output tokens": "{\"description\": \"min=100, mean=100, max=100, sum=100 (1)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "QuAC - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "QuAC - Stereotypes (race)": "{\"description\": \"min=0.63, mean=0.63, max=0.63, sum=0.63 (1)\", \"tab\": \"Bias\", \"score\": \"0.6296296296296297\"}", - "QuAC - Stereotypes (gender)": "{\"description\": \"min=0.445, mean=0.445, max=0.445, sum=0.445 (1)\", \"tab\": \"Bias\", \"score\": \"0.4446840232318048\"}", - "QuAC - Representation (race)": "{\"description\": \"min=0.333, mean=0.333, max=0.333, sum=0.333 (1)\", \"tab\": \"Bias\", \"score\": \"0.33333333333333337\"}", - "QuAC - Representation (gender)": "{\"description\": \"min=0.242, mean=0.242, max=0.242, sum=0.242 (1)\", \"tab\": \"Bias\", \"score\": \"0.24226804123711343\"}", - "QuAC - Toxic fraction": "{\"description\": \"min=0.003, mean=0.003, max=0.003, sum=0.003 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.003\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "HellaSwag", - "source_data": { - "dataset_name": "HellaSwag", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on HellaSwag", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "HellaSwag - ECE (10-bin)": "{\"description\": \"No matching runs\", \"tab\": \"Calibration\", \"score\": \"\"}", - "HellaSwag - EM (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "HellaSwag - EM (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "HellaSwag - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "HellaSwag - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "OpenbookQA - ECE (10-bin)": "{\"description\": \"No matching runs\", \"tab\": \"Calibration\", \"score\": \"\"}", - "OpenbookQA - EM (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "OpenbookQA - EM (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "OpenbookQA - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "OpenbookQA - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "TruthfulQA", - "source_data": { - "dataset_name": "TruthfulQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on TruthfulQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.243, - "details": { - "description": "min=0.243, mean=0.243, max=0.243, sum=0.243 (1)", - "tab": "Accuracy", - "TruthfulQA - ECE (10-bin)": "{\"description\": \"min=0.232, mean=0.232, max=0.232, sum=0.232 (1)\", \"tab\": \"Calibration\", \"score\": \"0.23215642305686054\"}", - "TruthfulQA - EM (Robustness)": "{\"description\": \"min=0.197, mean=0.197, max=0.197, sum=0.197 (1)\", \"tab\": \"Robustness\", \"score\": \"0.19724770642201836\"}", - "TruthfulQA - EM (Fairness)": "{\"description\": \"min=0.183, mean=0.183, max=0.183, sum=0.183 (1)\", \"tab\": \"Fairness\", \"score\": \"0.1834862385321101\"}", - "TruthfulQA - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "TruthfulQA - # eval": "{\"description\": \"min=654, mean=654, max=654, sum=654 (1)\", \"tab\": \"General information\", \"score\": \"654.0\"}", - "TruthfulQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "TruthfulQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "TruthfulQA - # prompt tokens": "{\"description\": \"min=505.352, mean=505.352, max=505.352, sum=505.352 (1)\", \"tab\": \"General information\", \"score\": \"505.35168195718654\"}", - "TruthfulQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "TruthfulQA - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MS MARCO (TREC)", - "source_data": { - "dataset_name": "MS MARCO (TREC)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "NDCG@10 on MS MARCO (TREC)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "MS MARCO (regular) - RR@10 (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "MS MARCO (TREC) - NDCG@10 (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "MS MARCO (regular) - RR@10 (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "MS MARCO (TREC) - NDCG@10 (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "MS MARCO (regular) - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MS MARCO (TREC) - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MS MARCO (regular) - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "MS MARCO (TREC) - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CNN/DailyMail", - "source_data": { - "dataset_name": "CNN/DailyMail", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on CNN/DailyMail", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "CNN/DailyMail - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "CNN/DailyMail - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "CNN/DailyMail - SummaC": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - QAFactEval": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - BERTScore (F1)": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - Coverage": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - Density": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - Compression": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-faithfulness": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-relevance": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-coherence": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "XSUM", - "source_data": { - "dataset_name": "XSUM", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on XSUM", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "XSUM - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "XSUM - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "XSUM - SummaC": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - QAFactEval": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - BERTScore (F1)": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - Coverage": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - Density": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - Compression": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-faithfulness": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-relevance": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-coherence": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "IMDB", - "source_data": { - "dataset_name": "IMDB", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on IMDB", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.927, - "details": { - "description": "min=0.927, mean=0.927, max=0.927, sum=0.927 (1)", - "tab": "Accuracy", - "IMDB - ECE (10-bin)": "{\"description\": \"min=0.159, mean=0.159, max=0.159, sum=0.159 (1)\", \"tab\": \"Calibration\", \"score\": \"0.15862422483580252\"}", - "IMDB - EM (Robustness)": "{\"description\": \"min=0.82, mean=0.82, max=0.82, sum=0.82 (1)\", \"tab\": \"Robustness\", \"score\": \"0.82\"}", - "IMDB - EM (Fairness)": "{\"description\": \"min=0.907, mean=0.907, max=0.907, sum=0.907 (1)\", \"tab\": \"Fairness\", \"score\": \"0.907\"}", - "IMDB - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "IMDB - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "IMDB - # train": "{\"description\": \"min=2.911, mean=2.911, max=2.911, sum=2.911 (1)\", \"tab\": \"General information\", \"score\": \"2.911\"}", - "IMDB - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IMDB - # prompt tokens": "{\"description\": \"min=1619.568, mean=1619.568, max=1619.568, sum=1619.568 (1)\", \"tab\": \"General information\", \"score\": \"1619.568\"}", - "IMDB - # output tokens": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "IMDB - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "IMDB - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CivilComments", - "source_data": { - "dataset_name": "CivilComments", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on CivilComments", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.664, - "details": { - "description": "min=0.487, mean=0.664, max=0.77, sum=11.961 (18)", - "tab": "Accuracy", - "CivilComments - ECE (10-bin)": "{\"description\": \"min=0.035, mean=0.102, max=0.234, sum=1.831 (18)\", \"tab\": \"Calibration\", \"score\": \"0.10174488153691034\"}", - "CivilComments - EM (Robustness)": "{\"description\": \"min=0.277, mean=0.527, max=0.77, sum=9.491 (18)\", \"tab\": \"Robustness\", \"score\": \"0.5272697486345442\"}", - "CivilComments - EM (Fairness)": "{\"description\": \"min=0.25, mean=0.54, max=0.743, sum=9.724 (18)\", \"tab\": \"Fairness\", \"score\": \"0.5401968527212513\"}", - "CivilComments - Denoised inference time (s)": "{\"description\": \"9 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "CivilComments - # eval": "{\"description\": \"min=74, mean=371.556, max=683, sum=6688 (18)\", \"tab\": \"General information\", \"score\": \"371.55555555555554\"}", - "CivilComments - # train": "{\"description\": \"min=5, mean=5, max=5, sum=90 (18)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CivilComments - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (18)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CivilComments - # prompt tokens": "{\"description\": \"min=360.976, mean=771.654, max=1282.4, sum=13889.772 (18)\", \"tab\": \"General information\", \"score\": \"771.6539847352628\"}", - "CivilComments - # output tokens": "{\"description\": \"min=5, mean=5, max=5, sum=90 (18)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CivilComments - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=18 (18)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "CivilComments - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Toxic fraction": "{\"description\": \"9 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "RAFT", - "source_data": { - "dataset_name": "RAFT", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on RAFT", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.695, - "details": { - "description": "min=0.175, mean=0.695, max=0.925, sum=7.65 (11)", - "tab": "Accuracy", - "RAFT - ECE (10-bin)": "{\"description\": \"min=0.175, mean=0.695, max=0.925, sum=7.647 (11)\", \"tab\": \"Calibration\", \"score\": \"0.69518288885631\"}", - "RAFT - EM (Robustness)": "{\"description\": \"min=0.175, mean=0.605, max=0.9, sum=6.65 (11)\", \"tab\": \"Robustness\", \"score\": \"0.6045454545454546\"}", - "RAFT - EM (Fairness)": "{\"description\": \"min=0.175, mean=0.67, max=0.875, sum=7.375 (11)\", \"tab\": \"Fairness\", \"score\": \"0.6704545454545454\"}", - "RAFT - Denoised inference time (s)": "{\"description\": \"11 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "RAFT - # eval": "{\"description\": \"min=40, mean=40, max=40, sum=440 (11)\", \"tab\": \"General information\", \"score\": \"40.0\"}", - "RAFT - # train": "{\"description\": \"min=0.7, mean=4.605, max=5, sum=50.65 (11)\", \"tab\": \"General information\", \"score\": \"4.6045454545454545\"}", - "RAFT - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (11)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "RAFT - # prompt tokens": "{\"description\": \"min=280.35, mean=869.691, max=1756.575, sum=9566.6 (11)\", \"tab\": \"General information\", \"score\": \"869.6909090909089\"}", - "RAFT - # output tokens": "{\"description\": \"min=30, mean=30, max=30, sum=330 (11)\", \"tab\": \"General information\", \"score\": \"30.0\"}", - "RAFT - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=11 (11)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "RAFT - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Toxic fraction": "{\"description\": \"11 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/together_redpajama-incite-instruct-v1-3b.json b/data/models/together_redpajama-incite-instruct-v1-3b.json deleted file mode 100644 index 85eaa0a3f6340293aed72b674ea682e867fa3a3f..0000000000000000000000000000000000000000 --- a/data/models/together_redpajama-incite-instruct-v1-3b.json +++ /dev/null @@ -1,674 +0,0 @@ -{ - "model_info": { - "name": "RedPajama-INCITE-Instruct-v1 3B", - "id": "together/RedPajama-INCITE-Instruct-v1-3B", - "developer": "together", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_classic/together_RedPajama-INCITE-Instruct-v1-3B/1774096308.339228", - "retrieved_timestamp": "1774096308.339228", - "source_metadata": { - "source_name": "helm_classic", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_classic", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_classic", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperform on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.366, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Calibration": "{\"description\": \"\", \"tab\": \"Calibration\", \"score\": \"0.37183163737280295\"}", - "Mean win rate - Robustness": "{\"description\": \"\", \"tab\": \"Robustness\", \"score\": \"0.3874825174825175\"}", - "Mean win rate - Fairness": "{\"description\": \"\", \"tab\": \"Fairness\", \"score\": \"0.3690909090909091\"}", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}", - "Mean win rate - Bias": "{\"description\": \"\", \"tab\": \"Bias\", \"score\": \"0.18974591969523494\"}", - "Mean win rate - Toxicity": "{\"description\": \"\", \"tab\": \"Toxicity\", \"score\": \"0.6051282051282051\"}", - "Mean win rate - Summarization metrics": "{\"description\": \"\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.257, - "details": { - "description": "min=0.22, mean=0.257, max=0.29, sum=1.287 (5)", - "tab": "Accuracy", - "MMLU - ECE (10-bin)": "{\"description\": \"min=0.09, mean=0.124, max=0.157, sum=0.619 (5)\", \"tab\": \"Calibration\", \"score\": \"0.1238999810101579\"}", - "MMLU - EM (Robustness)": "{\"description\": \"min=0.18, mean=0.218, max=0.23, sum=1.089 (5)\", \"tab\": \"Robustness\", \"score\": \"0.21785964912280703\"}", - "MMLU - EM (Fairness)": "{\"description\": \"min=0.18, mean=0.222, max=0.27, sum=1.111 (5)\", \"tab\": \"Fairness\", \"score\": \"0.22210526315789475\"}", - "MMLU - Denoised inference time (s)": "{\"description\": \"5 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=358.76, mean=467.936, max=612.798, sum=2339.678 (5)\", \"tab\": \"General information\", \"score\": \"467.935649122807\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "MMLU - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "BoolQ", - "source_data": { - "dataset_name": "BoolQ", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on BoolQ", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.677, - "details": { - "description": "min=0.677, mean=0.677, max=0.677, sum=0.677 (1)", - "tab": "Accuracy", - "BoolQ - ECE (10-bin)": "{\"description\": \"min=0.141, mean=0.141, max=0.141, sum=0.141 (1)\", \"tab\": \"Calibration\", \"score\": \"0.14082220350962116\"}", - "BoolQ - EM (Robustness)": "{\"description\": \"min=0.629, mean=0.629, max=0.629, sum=0.629 (1)\", \"tab\": \"Robustness\", \"score\": \"0.629\"}", - "BoolQ - EM (Fairness)": "{\"description\": \"min=0.648, mean=0.648, max=0.648, sum=0.648 (1)\", \"tab\": \"Fairness\", \"score\": \"0.648\"}", - "BoolQ - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "BoolQ - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "BoolQ - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "BoolQ - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "BoolQ - # prompt tokens": "{\"description\": \"min=1251.897, mean=1251.897, max=1251.897, sum=1251.897 (1)\", \"tab\": \"General information\", \"score\": \"1251.897\"}", - "BoolQ - # output tokens": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "BoolQ - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "BoolQ - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.638, - "details": { - "description": "min=0.638, mean=0.638, max=0.638, sum=0.638 (1)", - "tab": "Accuracy", - "NarrativeQA - ECE (10-bin)": "{\"description\": \"min=0.254, mean=0.254, max=0.254, sum=0.254 (1)\", \"tab\": \"Calibration\", \"score\": \"0.25351615672342864\"}", - "NarrativeQA - F1 (Robustness)": "{\"description\": \"min=0.403, mean=0.403, max=0.403, sum=0.403 (1)\", \"tab\": \"Robustness\", \"score\": \"0.4034697604028265\"}", - "NarrativeQA - F1 (Fairness)": "{\"description\": \"min=0.506, mean=0.506, max=0.506, sum=0.506 (1)\", \"tab\": \"Fairness\", \"score\": \"0.5060331991298288\"}", - "NarrativeQA - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=1.969, mean=1.969, max=1.969, sum=1.969 (1)\", \"tab\": \"General information\", \"score\": \"1.9690140845070423\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=1691.082, mean=1691.082, max=1691.082, sum=1691.082 (1)\", \"tab\": \"General information\", \"score\": \"1691.081690140845\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=100, mean=100, max=100, sum=100 (1)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "NarrativeQA - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NarrativeQA - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NarrativeQA - Stereotypes (gender)": "{\"description\": \"min=0.5, mean=0.5, max=0.5, sum=0.5 (1)\", \"tab\": \"Bias\", \"score\": \"0.5\"}", - "NarrativeQA - Representation (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=0.667 (1)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666667\"}", - "NarrativeQA - Representation (gender)": "{\"description\": \"min=0.184, mean=0.184, max=0.184, sum=0.184 (1)\", \"tab\": \"Bias\", \"score\": \"0.18354430379746836\"}", - "NarrativeQA - Toxic fraction": "{\"description\": \"min=0.014, mean=0.014, max=0.014, sum=0.014 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.014084507042253521\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (open-book)", - "source_data": { - "dataset_name": "NaturalQuestions (open-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (open-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.637, - "details": { - "description": "min=0.637, mean=0.637, max=0.637, sum=0.637 (1)", - "tab": "Accuracy", - "NaturalQuestions (closed-book) - ECE (10-bin)": "{\"description\": \"min=0.12, mean=0.12, max=0.12, sum=0.12 (1)\", \"tab\": \"Calibration\", \"score\": \"0.12000000000000001\"}", - "NaturalQuestions (open-book) - ECE (10-bin)": "{\"description\": \"min=0.454, mean=0.454, max=0.454, sum=0.454 (1)\", \"tab\": \"Calibration\", \"score\": \"0.4539999913132661\"}", - "NaturalQuestions (closed-book) - F1 (Robustness)": "{\"description\": \"min=0.132, mean=0.132, max=0.132, sum=0.132 (1)\", \"tab\": \"Robustness\", \"score\": \"0.13162030419976034\"}", - "NaturalQuestions (open-book) - F1 (Robustness)": "{\"description\": \"min=0.536, mean=0.536, max=0.536, sum=0.536 (1)\", \"tab\": \"Robustness\", \"score\": \"0.5356772534642628\"}", - "NaturalQuestions (closed-book) - F1 (Fairness)": "{\"description\": \"min=0.143, mean=0.143, max=0.143, sum=0.143 (1)\", \"tab\": \"Fairness\", \"score\": \"0.1431948167839223\"}", - "NaturalQuestions (open-book) - F1 (Fairness)": "{\"description\": \"min=0.571, mean=0.571, max=0.571, sum=0.571 (1)\", \"tab\": \"Fairness\", \"score\": \"0.57068667733919\"}", - "NaturalQuestions (closed-book) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NaturalQuestions (open-book) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=117.299, mean=117.299, max=117.299, sum=117.299 (1)\", \"tab\": \"General information\", \"score\": \"117.299\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=300, mean=300, max=300, sum=300 (1)\", \"tab\": \"General information\", \"score\": \"300.0\"}", - "NaturalQuestions (closed-book) - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.704, mean=4.704, max=4.704, sum=4.704 (1)\", \"tab\": \"General information\", \"score\": \"4.704\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.037, mean=0.037, max=0.037, sum=0.037 (1)\", \"tab\": \"General information\", \"score\": \"0.037\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1495.552, mean=1495.552, max=1495.552, sum=1495.552 (1)\", \"tab\": \"General information\", \"score\": \"1495.552\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=300, mean=300, max=300, sum=300 (1)\", \"tab\": \"General information\", \"score\": \"300.0\"}", - "NaturalQuestions (open-book) - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "NaturalQuestions (closed-book) - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - Stereotypes (gender)": "{\"description\": \"min=0.5, mean=0.5, max=0.5, sum=0.5 (1)\", \"tab\": \"Bias\", \"score\": \"0.5\"}", - "NaturalQuestions (closed-book) - Representation (race)": "{\"description\": \"min=0.467, mean=0.467, max=0.467, sum=0.467 (1)\", \"tab\": \"Bias\", \"score\": \"0.4666666666666666\"}", - "NaturalQuestions (closed-book) - Representation (gender)": "{\"description\": \"min=0.278, mean=0.278, max=0.278, sum=0.278 (1)\", \"tab\": \"Bias\", \"score\": \"0.2777777777777778\"}", - "NaturalQuestions (open-book) - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=0.667 (1)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666667\"}", - "NaturalQuestions (open-book) - Stereotypes (gender)": "{\"description\": \"min=0.5, mean=0.5, max=0.5, sum=0.5 (1)\", \"tab\": \"Bias\", \"score\": \"0.5\"}", - "NaturalQuestions (open-book) - Representation (race)": "{\"description\": \"min=0.566, mean=0.566, max=0.566, sum=0.566 (1)\", \"tab\": \"Bias\", \"score\": \"0.5660749506903353\"}", - "NaturalQuestions (open-book) - Representation (gender)": "{\"description\": \"min=0.324, mean=0.324, max=0.324, sum=0.324 (1)\", \"tab\": \"Bias\", \"score\": \"0.32352941176470584\"}", - "NaturalQuestions (closed-book) - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}", - "NaturalQuestions (open-book) - Toxic fraction": "{\"description\": \"min=0.001, mean=0.001, max=0.001, sum=0.001 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.001\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "QuAC", - "source_data": { - "dataset_name": "QuAC", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on QuAC", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.259, - "details": { - "description": "min=0.259, mean=0.259, max=0.259, sum=0.259 (1)", - "tab": "Accuracy", - "QuAC - ECE (10-bin)": "{\"description\": \"min=0.1, mean=0.1, max=0.1, sum=0.1 (1)\", \"tab\": \"Calibration\", \"score\": \"0.09989902749544036\"}", - "QuAC - F1 (Robustness)": "{\"description\": \"min=0.137, mean=0.137, max=0.137, sum=0.137 (1)\", \"tab\": \"Robustness\", \"score\": \"0.1368222933188553\"}", - "QuAC - F1 (Fairness)": "{\"description\": \"min=0.183, mean=0.183, max=0.183, sum=0.183 (1)\", \"tab\": \"Fairness\", \"score\": \"0.18270531445590665\"}", - "QuAC - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "QuAC - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "QuAC - # train": "{\"description\": \"min=0.883, mean=0.883, max=0.883, sum=0.883 (1)\", \"tab\": \"General information\", \"score\": \"0.883\"}", - "QuAC - truncated": "{\"description\": \"min=0.021, mean=0.021, max=0.021, sum=0.021 (1)\", \"tab\": \"General information\", \"score\": \"0.021\"}", - "QuAC - # prompt tokens": "{\"description\": \"min=1655.708, mean=1655.708, max=1655.708, sum=1655.708 (1)\", \"tab\": \"General information\", \"score\": \"1655.708\"}", - "QuAC - # output tokens": "{\"description\": \"min=100, mean=100, max=100, sum=100 (1)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "QuAC - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "QuAC - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=0.667 (1)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666667\"}", - "QuAC - Stereotypes (gender)": "{\"description\": \"min=0.439, mean=0.439, max=0.439, sum=0.439 (1)\", \"tab\": \"Bias\", \"score\": \"0.4393162393162393\"}", - "QuAC - Representation (race)": "{\"description\": \"min=0.34, mean=0.34, max=0.34, sum=0.34 (1)\", \"tab\": \"Bias\", \"score\": \"0.33993399339933994\"}", - "QuAC - Representation (gender)": "{\"description\": \"min=0.285, mean=0.285, max=0.285, sum=0.285 (1)\", \"tab\": \"Bias\", \"score\": \"0.28532608695652173\"}", - "QuAC - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "HellaSwag", - "source_data": { - "dataset_name": "HellaSwag", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on HellaSwag", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "HellaSwag - ECE (10-bin)": "{\"description\": \"No matching runs\", \"tab\": \"Calibration\", \"score\": \"\"}", - "HellaSwag - EM (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "HellaSwag - EM (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "HellaSwag - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "HellaSwag - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "OpenbookQA - ECE (10-bin)": "{\"description\": \"No matching runs\", \"tab\": \"Calibration\", \"score\": \"\"}", - "OpenbookQA - EM (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "OpenbookQA - EM (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "OpenbookQA - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "OpenbookQA - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "TruthfulQA", - "source_data": { - "dataset_name": "TruthfulQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on TruthfulQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.208, - "details": { - "description": "min=0.208, mean=0.208, max=0.208, sum=0.208 (1)", - "tab": "Accuracy", - "TruthfulQA - ECE (10-bin)": "{\"description\": \"min=0.097, mean=0.097, max=0.097, sum=0.097 (1)\", \"tab\": \"Calibration\", \"score\": \"0.09733177984986514\"}", - "TruthfulQA - EM (Robustness)": "{\"description\": \"min=0.173, mean=0.173, max=0.173, sum=0.173 (1)\", \"tab\": \"Robustness\", \"score\": \"0.172782874617737\"}", - "TruthfulQA - EM (Fairness)": "{\"description\": \"min=0.179, mean=0.179, max=0.179, sum=0.179 (1)\", \"tab\": \"Fairness\", \"score\": \"0.17889908256880735\"}", - "TruthfulQA - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "TruthfulQA - # eval": "{\"description\": \"min=654, mean=654, max=654, sum=654 (1)\", \"tab\": \"General information\", \"score\": \"654.0\"}", - "TruthfulQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "TruthfulQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "TruthfulQA - # prompt tokens": "{\"description\": \"min=505.352, mean=505.352, max=505.352, sum=505.352 (1)\", \"tab\": \"General information\", \"score\": \"505.35168195718654\"}", - "TruthfulQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "TruthfulQA - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MS MARCO (TREC)", - "source_data": { - "dataset_name": "MS MARCO (TREC)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "NDCG@10 on MS MARCO (TREC)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "MS MARCO (regular) - RR@10 (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "MS MARCO (TREC) - NDCG@10 (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "MS MARCO (regular) - RR@10 (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "MS MARCO (TREC) - NDCG@10 (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "MS MARCO (regular) - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MS MARCO (TREC) - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MS MARCO (regular) - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "MS MARCO (TREC) - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CNN/DailyMail", - "source_data": { - "dataset_name": "CNN/DailyMail", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on CNN/DailyMail", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "CNN/DailyMail - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "CNN/DailyMail - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "CNN/DailyMail - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "CNN/DailyMail - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "CNN/DailyMail - SummaC": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - QAFactEval": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - BERTScore (F1)": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - Coverage": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - Density": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - Compression": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-faithfulness": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-relevance": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-coherence": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "XSUM", - "source_data": { - "dataset_name": "XSUM", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on XSUM", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "XSUM - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "XSUM - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "XSUM - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "XSUM - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "XSUM - SummaC": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - QAFactEval": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - BERTScore (F1)": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - Coverage": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - Density": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - Compression": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-faithfulness": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-relevance": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-coherence": "{\"description\": \"No matching runs\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "IMDB", - "source_data": { - "dataset_name": "IMDB", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on IMDB", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.894, - "details": { - "description": "min=0.894, mean=0.894, max=0.894, sum=0.894 (1)", - "tab": "Accuracy", - "IMDB - ECE (10-bin)": "{\"description\": \"min=0.04, mean=0.04, max=0.04, sum=0.04 (1)\", \"tab\": \"Calibration\", \"score\": \"0.04045821313550608\"}", - "IMDB - EM (Robustness)": "{\"description\": \"min=0.852, mean=0.852, max=0.852, sum=0.852 (1)\", \"tab\": \"Robustness\", \"score\": \"0.852\"}", - "IMDB - EM (Fairness)": "{\"description\": \"min=0.876, mean=0.876, max=0.876, sum=0.876 (1)\", \"tab\": \"Fairness\", \"score\": \"0.876\"}", - "IMDB - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "IMDB - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "IMDB - # train": "{\"description\": \"min=2.911, mean=2.911, max=2.911, sum=2.911 (1)\", \"tab\": \"General information\", \"score\": \"2.911\"}", - "IMDB - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IMDB - # prompt tokens": "{\"description\": \"min=1619.568, mean=1619.568, max=1619.568, sum=1619.568 (1)\", \"tab\": \"General information\", \"score\": \"1619.568\"}", - "IMDB - # output tokens": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "IMDB - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "IMDB - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CivilComments", - "source_data": { - "dataset_name": "CivilComments", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on CivilComments", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.549, - "details": { - "description": "min=0.028, mean=0.549, max=0.997, sum=9.891 (18)", - "tab": "Accuracy", - "CivilComments - ECE (10-bin)": "{\"description\": \"min=0.09, mean=0.383, max=0.8, sum=6.9 (18)\", \"tab\": \"Calibration\", \"score\": \"0.3833406193329736\"}", - "CivilComments - EM (Robustness)": "{\"description\": \"min=0, mean=0.506, max=0.993, sum=9.105 (18)\", \"tab\": \"Robustness\", \"score\": \"0.5058374710841333\"}", - "CivilComments - EM (Fairness)": "{\"description\": \"min=0, mean=0.499, max=0.985, sum=8.983 (18)\", \"tab\": \"Fairness\", \"score\": \"0.4990473523687277\"}", - "CivilComments - Denoised inference time (s)": "{\"description\": \"9 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "CivilComments - # eval": "{\"description\": \"min=74, mean=371.556, max=683, sum=6688 (18)\", \"tab\": \"General information\", \"score\": \"371.55555555555554\"}", - "CivilComments - # train": "{\"description\": \"min=5, mean=5, max=5, sum=90 (18)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CivilComments - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (18)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CivilComments - # prompt tokens": "{\"description\": \"min=360.976, mean=771.654, max=1282.4, sum=13889.772 (18)\", \"tab\": \"General information\", \"score\": \"771.6539847352628\"}", - "CivilComments - # output tokens": "{\"description\": \"min=5, mean=5, max=5, sum=90 (18)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CivilComments - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=18 (18)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "CivilComments - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Toxic fraction": "{\"description\": \"9 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "RAFT", - "source_data": { - "dataset_name": "RAFT", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on RAFT", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.661, - "details": { - "description": "min=0.2, mean=0.661, max=0.975, sum=7.275 (11)", - "tab": "Accuracy", - "RAFT - ECE (10-bin)": "{\"description\": \"min=0.2, mean=0.661, max=0.975, sum=7.274 (11)\", \"tab\": \"Calibration\", \"score\": \"0.6612967467806994\"}", - "RAFT - EM (Robustness)": "{\"description\": \"min=0.075, mean=0.548, max=0.95, sum=6.025 (11)\", \"tab\": \"Robustness\", \"score\": \"0.5477272727272727\"}", - "RAFT - EM (Fairness)": "{\"description\": \"min=0.175, mean=0.632, max=0.975, sum=6.95 (11)\", \"tab\": \"Fairness\", \"score\": \"0.631818181818182\"}", - "RAFT - Denoised inference time (s)": "{\"description\": \"11 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "RAFT - # eval": "{\"description\": \"min=40, mean=40, max=40, sum=440 (11)\", \"tab\": \"General information\", \"score\": \"40.0\"}", - "RAFT - # train": "{\"description\": \"min=0.7, mean=4.605, max=5, sum=50.65 (11)\", \"tab\": \"General information\", \"score\": \"4.6045454545454545\"}", - "RAFT - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (11)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "RAFT - # prompt tokens": "{\"description\": \"min=280.35, mean=869.691, max=1756.575, sum=9566.6 (11)\", \"tab\": \"General information\", \"score\": \"869.6909090909089\"}", - "RAFT - # output tokens": "{\"description\": \"min=30, mean=30, max=30, sum=330 (11)\", \"tab\": \"General information\", \"score\": \"30.0\"}", - "RAFT - # trials": "{\"description\": \"min=1, mean=1, max=1, sum=11 (11)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "RAFT - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Toxic fraction": "{\"description\": \"11 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/togethercomputer_gpt-jt-6b-v1.json b/data/models/togethercomputer_gpt-jt-6b-v1.json deleted file mode 100644 index 2649cdbb625fb531d3840ddce52df3e407d3ae28..0000000000000000000000000000000000000000 --- a/data/models/togethercomputer_gpt-jt-6b-v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "GPT-JT-6B-v1", - "id": "togethercomputer/GPT-JT-6B-v1", - "developer": "togethercomputer", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "GPTJForCausalLM", - "params_billions": "6.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/togethercomputer_GPT-JT-6B-v1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2061 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3303 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0106 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2609 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3737 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1626 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/togethercomputer_gpt-neoxt-chat-base-20b.json b/data/models/togethercomputer_gpt-neoxt-chat-base-20b.json deleted file mode 100644 index 26f3eaac2ac56122a3e61147863b13e9851e4960..0000000000000000000000000000000000000000 --- a/data/models/togethercomputer_gpt-neoxt-chat-base-20b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "GPT-NeoXT-Chat-Base-20B", - "id": "togethercomputer/GPT-NeoXT-Chat-Base-20B", - "developer": "togethercomputer", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "GPTNeoXForCausalLM", - "params_billions": "20.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/togethercomputer_GPT-NeoXT-Chat-Base-20B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.183 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3321 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0234 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.25 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3461 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1145 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/togethercomputer_llama-2-7b-32k-instruct.json b/data/models/togethercomputer_llama-2-7b-32k-instruct.json deleted file mode 100644 index b8e6f7f979bdaebdac728805a572eec9a19fe01f..0000000000000000000000000000000000000000 --- a/data/models/togethercomputer_llama-2-7b-32k-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-2-7B-32K-Instruct", - "id": "togethercomputer/Llama-2-7B-32K-Instruct", - "developer": "togethercomputer", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "7.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/togethercomputer_Llama-2-7B-32K-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.213 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3443 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0159 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2517 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4056 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1781 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/togethercomputer_llama-2-7b-32k.json b/data/models/togethercomputer_llama-2-7b-32k.json deleted file mode 100644 index 64e481329654f9bba250f1c83587180242cbe590..0000000000000000000000000000000000000000 --- a/data/models/togethercomputer_llama-2-7b-32k.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "LLaMA-2-7B-32K", - "id": "togethercomputer/LLaMA-2-7B-32K", - "developer": "togethercomputer", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "7.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/togethercomputer_LLaMA-2-7B-32K/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1865 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.34 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0144 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.25 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3754 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1768 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/togethercomputer_redpajama-incite-7b-base.json b/data/models/togethercomputer_redpajama-incite-7b-base.json deleted file mode 100644 index d32ce35ccce72d3aade35332c372fed5e2db6795..0000000000000000000000000000000000000000 --- a/data/models/togethercomputer_redpajama-incite-7b-base.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "RedPajama-INCITE-7B-Base", - "id": "togethercomputer/RedPajama-INCITE-7B-Base", - "developer": "togethercomputer", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "GPTNeoXForCausalLM", - "params_billions": "7.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/togethercomputer_RedPajama-INCITE-7B-Base/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2082 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3195 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0159 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.255 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.362 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1197 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/togethercomputer_redpajama-incite-7b-chat.json b/data/models/togethercomputer_redpajama-incite-7b-chat.json deleted file mode 100644 index f31f57bc65157bd003f1b754552c2c0242f02eb8..0000000000000000000000000000000000000000 --- a/data/models/togethercomputer_redpajama-incite-7b-chat.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "RedPajama-INCITE-7B-Chat", - "id": "togethercomputer/RedPajama-INCITE-7B-Chat", - "developer": "togethercomputer", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "GPTNeoXForCausalLM", - "params_billions": "7.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/togethercomputer_RedPajama-INCITE-7B-Chat/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1558 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3175 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0068 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2525 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3448 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1121 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/togethercomputer_redpajama-incite-7b-instruct.json b/data/models/togethercomputer_redpajama-incite-7b-instruct.json deleted file mode 100644 index 3fbb1a2e034f96fb31a8f20cda05ad77729ee894..0000000000000000000000000000000000000000 --- a/data/models/togethercomputer_redpajama-incite-7b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "RedPajama-INCITE-7B-Instruct", - "id": "togethercomputer/RedPajama-INCITE-7B-Instruct", - "developer": "togethercomputer", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "GPTNeoXForCausalLM", - "params_billions": "7.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/togethercomputer_RedPajama-INCITE-7B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2055 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3377 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0211 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2508 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3685 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1272 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/togethercomputer_redpajama-incite-base-3b-v1.json b/data/models/togethercomputer_redpajama-incite-base-3b-v1.json deleted file mode 100644 index ddab68056575aded6fb1faa976d7cd2bd3536e15..0000000000000000000000000000000000000000 --- a/data/models/togethercomputer_redpajama-incite-base-3b-v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "RedPajama-INCITE-Base-3B-v1", - "id": "togethercomputer/RedPajama-INCITE-Base-3B-v1", - "developer": "togethercomputer", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "GPTNeoXForCausalLM", - "params_billions": "3.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/togethercomputer_RedPajama-INCITE-Base-3B-v1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2294 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.306 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0144 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2433 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3739 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1111 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/togethercomputer_redpajama-incite-chat-3b-v1.json b/data/models/togethercomputer_redpajama-incite-chat-3b-v1.json deleted file mode 100644 index b46615ee20f719fbf189135d0bb72c2ec01d4d01..0000000000000000000000000000000000000000 --- a/data/models/togethercomputer_redpajama-incite-chat-3b-v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "RedPajama-INCITE-Chat-3B-v1", - "id": "togethercomputer/RedPajama-INCITE-Chat-3B-v1", - "developer": "togethercomputer", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "GPTNeoXForCausalLM", - "params_billions": "3.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/togethercomputer_RedPajama-INCITE-Chat-3B-v1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1652 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3217 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0091 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2441 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3684 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1127 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/togethercomputer_redpajama-incite-instruct-3b-v1.json b/data/models/togethercomputer_redpajama-incite-instruct-3b-v1.json deleted file mode 100644 index a71e5cc79315627418b8b6956969f8d5bf8dd199..0000000000000000000000000000000000000000 --- a/data/models/togethercomputer_redpajama-incite-instruct-3b-v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "RedPajama-INCITE-Instruct-3B-v1", - "id": "togethercomputer/RedPajama-INCITE-Instruct-3B-v1", - "developer": "togethercomputer", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "GPTNeoXForCausalLM", - "params_billions": "3.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/togethercomputer_RedPajama-INCITE-Instruct-3B-v1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2124 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3146 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0128 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2475 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3886 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.111 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/tokyotech-llm_llama-3-swallow-8b-instruct-v0.1.json b/data/models/tokyotech-llm_llama-3-swallow-8b-instruct-v0.1.json deleted file mode 100644 index 4da43887b45121eccff98df688685a13e599518a..0000000000000000000000000000000000000000 --- a/data/models/tokyotech-llm_llama-3-swallow-8b-instruct-v0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-Swallow-8B-Instruct-v0.1", - "id": "tokyotech-llm/Llama-3-Swallow-8B-Instruct-v0.1", - "developer": "tokyotech-llm", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/tokyotech-llm_Llama-3-Swallow-8B-Instruct-v0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5508 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5009 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0748 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2894 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4357 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3088 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/tomasmcm_sky-t1-coder-32b-flash.json b/data/models/tomasmcm_sky-t1-coder-32b-flash.json deleted file mode 100644 index 3167395f5f3e00c16d06c1a8a3a6a1af60be59f0..0000000000000000000000000000000000000000 --- a/data/models/tomasmcm_sky-t1-coder-32b-flash.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "sky-t1-coder-32b-flash", - "id": "tomasmcm/sky-t1-coder-32b-flash", - "developer": "tomasmcm", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "32.764" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/tomasmcm_sky-t1-coder-32b-flash/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.778 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6822 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5423 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3683 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4233 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5782 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/trappu_magnum-picaro-0.7-v2-12b.json b/data/models/trappu_magnum-picaro-0.7-v2-12b.json deleted file mode 100644 index b5129bc6fdb1b4ce1900a09b44a3cf99ff24e8a9..0000000000000000000000000000000000000000 --- a/data/models/trappu_magnum-picaro-0.7-v2-12b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Magnum-Picaro-0.7-v2-12b", - "id": "Trappu/Magnum-Picaro-0.7-v2-12b", - "developer": "Trappu", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Trappu_Magnum-Picaro-0.7-v2-12b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3003 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5507 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0665 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.323 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4727 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.358 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/trappu_nemo-picaro-12b.json b/data/models/trappu_nemo-picaro-12b.json deleted file mode 100644 index c15b681cb911d269c3d8ccf1b23b23f2a7058e73..0000000000000000000000000000000000000000 --- a/data/models/trappu_nemo-picaro-12b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Nemo-Picaro-12B", - "id": "Trappu/Nemo-Picaro-12B", - "developer": "Trappu", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Trappu_Nemo-Picaro-12B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2577 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.549 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0846 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3272 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4726 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3605 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/tremontaine_l3-12b-lunaris-v1.json b/data/models/tremontaine_l3-12b-lunaris-v1.json deleted file mode 100644 index d9127548243c04fd6495254bb9af8c3f20963148..0000000000000000000000000000000000000000 --- a/data/models/tremontaine_l3-12b-lunaris-v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "L3-12B-Lunaris-v1", - "id": "Tremontaine/L3-12B-Lunaris-v1", - "developer": "Tremontaine", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "11.52" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Tremontaine_L3-12B-Lunaris-v1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6909 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.523 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0876 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3096 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3674 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3775 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/triangle104_annunaki-12b.json b/data/models/triangle104_annunaki-12b.json deleted file mode 100644 index bbcac1b9791c052c76c837a08df5be9f28a3c13d..0000000000000000000000000000000000000000 --- a/data/models/triangle104_annunaki-12b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Annunaki-12b", - "id": "Triangle104/Annunaki-12b", - "developer": "Triangle104", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Triangle104_Annunaki-12b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3872 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5499 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1216 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3213 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4409 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3721 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/triangle104_bigtalker-lite-8b.json b/data/models/triangle104_bigtalker-lite-8b.json deleted file mode 100644 index 9f8c36ba7cd991a9186d89b4b2eb5f8969abbda6..0000000000000000000000000000000000000000 --- a/data/models/triangle104_bigtalker-lite-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "BigTalker-Lite-8B", - "id": "Triangle104/BigTalker-Lite-8B", - "developer": "Triangle104", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Triangle104_BigTalker-Lite-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3689 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5308 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.102 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3104 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4208 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3431 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/triangle104_chatty-harry_v2.0.json b/data/models/triangle104_chatty-harry_v2.0.json deleted file mode 100644 index a72848a396083ee710e0b838eeaf7c306272a5d2..0000000000000000000000000000000000000000 --- a/data/models/triangle104_chatty-harry_v2.0.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Chatty-Harry_V2.0", - "id": "Triangle104/Chatty-Harry_V2.0", - "developer": "Triangle104", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Triangle104_Chatty-Harry_V2.0/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3326 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5319 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.139 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.323 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4078 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3683 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/triangle104_chatty-harry_v3.0.json b/data/models/triangle104_chatty-harry_v3.0.json deleted file mode 100644 index a9de21ffefecb10960c5c88425bdfd1af437f981..0000000000000000000000000000000000000000 --- a/data/models/triangle104_chatty-harry_v3.0.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Chatty-Harry_V3.0", - "id": "Triangle104/Chatty-Harry_V3.0", - "developer": "Triangle104", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Triangle104_Chatty-Harry_V3.0/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3675 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5526 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1125 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.323 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4408 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3702 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/triangle104_chronos-prism_v1.0.json b/data/models/triangle104_chronos-prism_v1.0.json deleted file mode 100644 index 1ccd1b17a365a4a16365001efacaae21db67024f..0000000000000000000000000000000000000000 --- a/data/models/triangle104_chronos-prism_v1.0.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Chronos-Prism_V1.0", - "id": "Triangle104/Chronos-Prism_V1.0", - "developer": "Triangle104", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Triangle104_Chronos-Prism_V1.0/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3259 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5554 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1201 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3096 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4263 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3673 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/triangle104_dark-chivalry_v1.0.json b/data/models/triangle104_dark-chivalry_v1.0.json deleted file mode 100644 index 66ae71aabba1ef335865e978b0357c1c2fcd9c54..0000000000000000000000000000000000000000 --- a/data/models/triangle104_dark-chivalry_v1.0.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Dark-Chivalry_V1.0", - "id": "Triangle104/Dark-Chivalry_V1.0", - "developer": "Triangle104", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Triangle104_Dark-Chivalry_V1.0/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4326 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4974 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1314 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2936 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4182 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3444 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/triangle104_distilled-darkplanet-allades-8b.json b/data/models/triangle104_distilled-darkplanet-allades-8b.json deleted file mode 100644 index ba3f7de6dc347a5c8ed1d4034229ea57e171bc02..0000000000000000000000000000000000000000 --- a/data/models/triangle104_distilled-darkplanet-allades-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Distilled-DarkPlanet-Allades-8B", - "id": "Triangle104/Distilled-DarkPlanet-Allades-8B", - "developer": "Triangle104", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Triangle104_Distilled-DarkPlanet-Allades-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.346 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4634 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4003 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3054 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3538 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2901 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/triangle104_distilled-darkplanet-allades-8b_ties.json b/data/models/triangle104_distilled-darkplanet-allades-8b_ties.json deleted file mode 100644 index 5c2264a355c4c102025f9a1bfc033205106f7823..0000000000000000000000000000000000000000 --- a/data/models/triangle104_distilled-darkplanet-allades-8b_ties.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Distilled-DarkPlanet-Allades-8B_TIES", - "id": "Triangle104/Distilled-DarkPlanet-Allades-8B_TIES", - "developer": "Triangle104", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Triangle104_Distilled-DarkPlanet-Allades-8B_TIES/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3892 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5042 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0906 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3146 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3868 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3401 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/triangle104_distilled-whiskey-8b.json b/data/models/triangle104_distilled-whiskey-8b.json deleted file mode 100644 index ceccea10b6bd191559218c36076a74cb4cedf4fa..0000000000000000000000000000000000000000 --- a/data/models/triangle104_distilled-whiskey-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Distilled-Whiskey-8b", - "id": "Triangle104/Distilled-Whiskey-8b", - "developer": "Triangle104", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Triangle104_Distilled-Whiskey-8b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3448 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5028 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2545 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3314 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4172 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3367 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/triangle104_dolphin3-llama3.2-smart.json b/data/models/triangle104_dolphin3-llama3.2-smart.json deleted file mode 100644 index b9cbd790595d3bf5edb7712ac15908af8d9c3bfd..0000000000000000000000000000000000000000 --- a/data/models/triangle104_dolphin3-llama3.2-smart.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Dolphin3-Llama3.2-Smart", - "id": "Triangle104/Dolphin3-Llama3.2-Smart", - "developer": "Triangle104", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Triangle104_Dolphin3-Llama3.2-Smart/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4137 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3975 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0438 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2693 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3922 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2195 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/triangle104_ds-distilled-hermes-llama-3.1.json b/data/models/triangle104_ds-distilled-hermes-llama-3.1.json deleted file mode 100644 index 0d04f11b794db8968222855e1c1ef7aef4ef460f..0000000000000000000000000000000000000000 --- a/data/models/triangle104_ds-distilled-hermes-llama-3.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "DS-Distilled-Hermes-Llama-3.1", - "id": "Triangle104/DS-Distilled-Hermes-Llama-3.1", - "developer": "Triangle104", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Triangle104_DS-Distilled-Hermes-Llama-3.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3229 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5117 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2931 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3188 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4039 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.311 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/triangle104_ds-distilled-hermes-llama-3.1_ties.json b/data/models/triangle104_ds-distilled-hermes-llama-3.1_ties.json deleted file mode 100644 index 04caab90d0d02eff6caf6259c7596460a6bd573e..0000000000000000000000000000000000000000 --- a/data/models/triangle104_ds-distilled-hermes-llama-3.1_ties.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "DS-Distilled-Hermes-Llama-3.1_TIES", - "id": "Triangle104/DS-Distilled-Hermes-Llama-3.1_TIES", - "developer": "Triangle104", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Triangle104_DS-Distilled-Hermes-Llama-3.1_TIES/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1364 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2928 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0091 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.245 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3621 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1104 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/triangle104_ds-r1-distill-q2.5-10b-harmony.json b/data/models/triangle104_ds-r1-distill-q2.5-10b-harmony.json deleted file mode 100644 index 125fd7f937342c1f84e41b7eb711c37d7975bea4..0000000000000000000000000000000000000000 --- a/data/models/triangle104_ds-r1-distill-q2.5-10b-harmony.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "DS-R1-Distill-Q2.5-10B-Harmony", - "id": "Triangle104/DS-R1-Distill-Q2.5-10B-Harmony", - "developer": "Triangle104", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "10.366" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Triangle104_DS-R1-Distill-Q2.5-10B-Harmony/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1751 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2643 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2106 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3128 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1173 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/triangle104_ds-r1-distill-q2.5-14b-harmony_v0.1.json b/data/models/triangle104_ds-r1-distill-q2.5-14b-harmony_v0.1.json deleted file mode 100644 index f556c111fe86497abdbfa78256c348338c2ea128..0000000000000000000000000000000000000000 --- a/data/models/triangle104_ds-r1-distill-q2.5-14b-harmony_v0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "DS-R1-Distill-Q2.5-14B-Harmony_V0.1", - "id": "Triangle104/DS-R1-Distill-Q2.5-14B-Harmony_V0.1", - "developer": "Triangle104", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Triangle104_DS-R1-Distill-Q2.5-14B-Harmony_V0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4515 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5783 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5551 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3935 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5567 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4601 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/triangle104_ds-r1-distill-q2.5-7b-rp.json b/data/models/triangle104_ds-r1-distill-q2.5-7b-rp.json deleted file mode 100644 index 6653c676d9a2a8338ae65d5660a5fcb9659cf486..0000000000000000000000000000000000000000 --- a/data/models/triangle104_ds-r1-distill-q2.5-7b-rp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "DS-R1-Distill-Q2.5-7B-RP", - "id": "Triangle104/DS-R1-Distill-Q2.5-7B-RP", - "developer": "Triangle104", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Triangle104_DS-R1-Distill-Q2.5-7B-RP/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3445 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4383 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4683 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3138 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.403 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2891 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/triangle104_ds-r1-llama-8b-harmony.json b/data/models/triangle104_ds-r1-llama-8b-harmony.json deleted file mode 100644 index 3e96911d3bbba3c50737fff3ec8c352a8475a7bb..0000000000000000000000000000000000000000 --- a/data/models/triangle104_ds-r1-llama-8b-harmony.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "DS-R1-Llama-8B-Harmony", - "id": "Triangle104/DS-R1-Llama-8B-Harmony", - "developer": "Triangle104", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Triangle104_DS-R1-Llama-8B-Harmony/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3566 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4154 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4282 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2919 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3762 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2744 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/triangle104_dsr1-distill-llama-lit-8b.json b/data/models/triangle104_dsr1-distill-llama-lit-8b.json deleted file mode 100644 index 1aef5834b1d353434d71c8b0ee9343faf250dfe3..0000000000000000000000000000000000000000 --- a/data/models/triangle104_dsr1-distill-llama-lit-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "DSR1-Distill-Llama-Lit-8B", - "id": "Triangle104/DSR1-Distill-Llama-Lit-8B", - "developer": "Triangle104", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Triangle104_DSR1-Distill-Llama-Lit-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1885 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4284 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.352 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3029 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3535 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2798 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/triangle104_dsr1-distill-qwen-7b-rp.json b/data/models/triangle104_dsr1-distill-qwen-7b-rp.json deleted file mode 100644 index bb5f19a237a598fb7f7335c980f551d8be24a3a2..0000000000000000000000000000000000000000 --- a/data/models/triangle104_dsr1-distill-qwen-7b-rp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "DSR1-Distill-Qwen-7B-RP", - "id": "Triangle104/DSR1-Distill-Qwen-7B-RP", - "developer": "Triangle104", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.613" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Triangle104_DSR1-Distill-Qwen-7B-RP/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3609 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4326 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4804 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3196 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4045 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3028 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/triangle104_gemmadevi-stock-10b.json b/data/models/triangle104_gemmadevi-stock-10b.json deleted file mode 100644 index 115e8c5ba732dff4c59fb787daeb0628212f14e2..0000000000000000000000000000000000000000 --- a/data/models/triangle104_gemmadevi-stock-10b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Gemmadevi-Stock-10B", - "id": "Triangle104/Gemmadevi-Stock-10B", - "developer": "Triangle104", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Triangle104_Gemmadevi-Stock-10B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1582 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6066 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0967 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3532 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4621 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4262 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/triangle104_hermes-llama-3.2-cot-summary.json b/data/models/triangle104_hermes-llama-3.2-cot-summary.json deleted file mode 100644 index 6650547fd6131678af2695235c55ddccae44987e..0000000000000000000000000000000000000000 --- a/data/models/triangle104_hermes-llama-3.2-cot-summary.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Hermes-Llama-3.2-CoT-Summary", - "id": "Triangle104/Hermes-Llama-3.2-CoT-Summary", - "developer": "Triangle104", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Triangle104_Hermes-Llama-3.2-CoT-Summary/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.483 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.42 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0831 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2559 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3575 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2901 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/triangle104_hermes-llama-3.2-cot.json b/data/models/triangle104_hermes-llama-3.2-cot.json deleted file mode 100644 index f42417f76e49cc5dd7457ffb439415dcdba5beed..0000000000000000000000000000000000000000 --- a/data/models/triangle104_hermes-llama-3.2-cot.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Hermes-Llama-3.2-CoT", - "id": "Triangle104/Hermes-Llama-3.2-CoT", - "developer": "Triangle104", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Triangle104_Hermes-Llama-3.2-CoT/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4178 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4616 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0952 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2794 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3698 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2947 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/triangle104_hermes3-l3.1-dirtyharry-8b.json b/data/models/triangle104_hermes3-l3.1-dirtyharry-8b.json deleted file mode 100644 index ca477a6ff104dd432ba391f72a4b11ef5d659db5..0000000000000000000000000000000000000000 --- a/data/models/triangle104_hermes3-l3.1-dirtyharry-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Hermes3-L3.1-DirtyHarry-8B", - "id": "Triangle104/Hermes3-L3.1-DirtyHarry-8B", - "developer": "Triangle104", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Triangle104_Hermes3-L3.1-DirtyHarry-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3242 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5066 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0718 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.302 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4069 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3339 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/triangle104_herodotos-14b.json b/data/models/triangle104_herodotos-14b.json deleted file mode 100644 index 2907fb3f27431d2dd2350e86ad52039bf7bfa260..0000000000000000000000000000000000000000 --- a/data/models/triangle104_herodotos-14b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Herodotos-14B", - "id": "Triangle104/Herodotos-14B", - "developer": "Triangle104", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Triangle104_Herodotos-14B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4667 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6435 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5045 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3733 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4795 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.529 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/triangle104_herodotos-14b_v0.1.json b/data/models/triangle104_herodotos-14b_v0.1.json deleted file mode 100644 index 1395cfba7afab56e2962f7d76b7063ab4e74ff34..0000000000000000000000000000000000000000 --- a/data/models/triangle104_herodotos-14b_v0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Herodotos-14B_V0.1", - "id": "Triangle104/Herodotos-14B_V0.1", - "developer": "Triangle104", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Triangle104_Herodotos-14B_V0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1879 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3017 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.224 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3684 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1164 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/triangle104_l3.1-8b-dusky-ink.json b/data/models/triangle104_l3.1-8b-dusky-ink.json deleted file mode 100644 index 49d0c62efdccf1c3b74d79a6aa68781ef2287681..0000000000000000000000000000000000000000 --- a/data/models/triangle104_l3.1-8b-dusky-ink.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "L3.1-8B-Dusky-Ink", - "id": "Triangle104/L3.1-8B-Dusky-Ink", - "developer": "Triangle104", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Triangle104_L3.1-8B-Dusky-Ink/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.453 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5098 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1231 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2894 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4224 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3683 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/triangle104_l3.1-8b-dusky-ink_v0.r1.json b/data/models/triangle104_l3.1-8b-dusky-ink_v0.r1.json deleted file mode 100644 index afaeca32b453e45d6bb107598d5962fb0b8fc43e..0000000000000000000000000000000000000000 --- a/data/models/triangle104_l3.1-8b-dusky-ink_v0.r1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "L3.1-8B-Dusky-Ink_v0.r1", - "id": "Triangle104/L3.1-8B-Dusky-Ink_v0.r1", - "developer": "Triangle104", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Triangle104_L3.1-8B-Dusky-Ink_v0.r1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1985 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4337 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0431 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3037 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3988 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3206 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/triangle104_llama3.1-allades-lit-8b.json b/data/models/triangle104_llama3.1-allades-lit-8b.json deleted file mode 100644 index ff2a490bec7cff73e9e7fbd9c6590603455c17f4..0000000000000000000000000000000000000000 --- a/data/models/triangle104_llama3.1-allades-lit-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama3.1-Allades-Lit-8b", - "id": "Triangle104/Llama3.1-Allades-Lit-8b", - "developer": "Triangle104", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Triangle104_Llama3.1-Allades-Lit-8b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2461 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4183 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0023 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2844 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3708 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2724 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/triangle104_llama3.1-cc-lit-8b.json b/data/models/triangle104_llama3.1-cc-lit-8b.json deleted file mode 100644 index c32affc39994af7674774686e4029501e6ec6788..0000000000000000000000000000000000000000 --- a/data/models/triangle104_llama3.1-cc-lit-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama3.1-cc-Lit-8b", - "id": "Triangle104/Llama3.1-cc-Lit-8b", - "developer": "Triangle104", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Triangle104_Llama3.1-cc-Lit-8b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2993 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3848 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.003 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2777 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3854 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3004 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/triangle104_lthreepointone-8b-hermesblackroot.json b/data/models/triangle104_lthreepointone-8b-hermesblackroot.json deleted file mode 100644 index a12062144a78caa03506d55739ebdf16f6811163..0000000000000000000000000000000000000000 --- a/data/models/triangle104_lthreepointone-8b-hermesblackroot.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "LThreePointOne-8B-HermesBlackroot", - "id": "Triangle104/LThreePointOne-8B-HermesBlackroot", - "developer": "Triangle104", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Triangle104_LThreePointOne-8B-HermesBlackroot/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1792 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4998 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0196 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.307 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3586 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3285 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/triangle104_lthreepointone-8b-hermesink.json b/data/models/triangle104_lthreepointone-8b-hermesink.json deleted file mode 100644 index ad38db80d7811ee0077cdf8cb1e9db2a49abde4f..0000000000000000000000000000000000000000 --- a/data/models/triangle104_lthreepointone-8b-hermesink.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "LThreePointOne-8B-HermesInk", - "id": "Triangle104/LThreePointOne-8B-HermesInk", - "developer": "Triangle104", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Triangle104_LThreePointOne-8B-HermesInk/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4031 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5223 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1722 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.323 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4129 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3467 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/triangle104_minerva-1.5b.json b/data/models/triangle104_minerva-1.5b.json deleted file mode 100644 index 55ad166a4e26879837993a3bfeeec9011e256902..0000000000000000000000000000000000000000 --- a/data/models/triangle104_minerva-1.5b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Minerva-1.5b", - "id": "Triangle104/Minerva-1.5b", - "developer": "Triangle104", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.777" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Triangle104_Minerva-1.5b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2694 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4026 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1027 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3104 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3655 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2698 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/triangle104_minerva-1.5b_v0.2.json b/data/models/triangle104_minerva-1.5b_v0.2.json deleted file mode 100644 index db2507acaab6608bc32901d7b924f343f6ebb198..0000000000000000000000000000000000000000 --- a/data/models/triangle104_minerva-1.5b_v0.2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Minerva-1.5b_V0.2", - "id": "Triangle104/Minerva-1.5b_V0.2", - "developer": "Triangle104", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.777" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Triangle104_Minerva-1.5b_V0.2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3083 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3989 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.114 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2852 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.396 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2911 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/triangle104_minerva-10b.json b/data/models/triangle104_minerva-10b.json deleted file mode 100644 index bbbb42afc22c6f944b728681185cb7b911100358..0000000000000000000000000000000000000000 --- a/data/models/triangle104_minerva-10b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Minerva-10b", - "id": "Triangle104/Minerva-10b", - "developer": "Triangle104", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "10.067" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Triangle104_Minerva-10b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1879 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4462 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.281 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3627 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2318 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/triangle104_minerva-14b-v0.1.json b/data/models/triangle104_minerva-14b-v0.1.json deleted file mode 100644 index aac3a4d588408f908a58c4cb2f19d03294d2f4e0..0000000000000000000000000000000000000000 --- a/data/models/triangle104_minerva-14b-v0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Minerva-14b-V0.1", - "id": "Triangle104/Minerva-14b-V0.1", - "developer": "Triangle104", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Triangle104_Minerva-14b-V0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0861 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.609 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3051 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3658 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.47 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5118 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/triangle104_minerva-14b.json b/data/models/triangle104_minerva-14b.json deleted file mode 100644 index 2f888b5ff8485e6848edd2e714f3d7e98bda5f4d..0000000000000000000000000000000000000000 --- a/data/models/triangle104_minerva-14b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Minerva-14b", - "id": "Triangle104/Minerva-14b", - "developer": "Triangle104", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Triangle104_Minerva-14b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3468 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6301 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3051 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3742 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4766 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5194 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/triangle104_minerva-7b.json b/data/models/triangle104_minerva-7b.json deleted file mode 100644 index 39be5007e4a8e9a171ae076e5ac6f8d478c18851..0000000000000000000000000000000000000000 --- a/data/models/triangle104_minerva-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Minerva-7b", - "id": "Triangle104/Minerva-7b", - "developer": "Triangle104", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Triangle104_Minerva-7b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3724 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5498 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.284 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.323 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4143 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4444 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/triangle104_minerva-8b.json b/data/models/triangle104_minerva-8b.json deleted file mode 100644 index beaabf708533c9c647fa650b50bfb7310ebc94f9..0000000000000000000000000000000000000000 --- a/data/models/triangle104_minerva-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Minerva-8b", - "id": "Triangle104/Minerva-8b", - "developer": "Triangle104", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Triangle104_Minerva-8b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1721 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4669 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0045 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3121 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4273 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3089 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/triangle104_mistral-redemption-arc.json b/data/models/triangle104_mistral-redemption-arc.json deleted file mode 100644 index 444f65b2a80db7a131a3780bac2968a6116d4039..0000000000000000000000000000000000000000 --- a/data/models/triangle104_mistral-redemption-arc.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mistral-Redemption-Arc", - "id": "Triangle104/Mistral-Redemption-Arc", - "developer": "Triangle104", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "23.572" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Triangle104_Mistral-Redemption-Arc/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4029 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6255 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4101 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3473 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4595 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.451 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/triangle104_mistral-small-24b-harmony.json b/data/models/triangle104_mistral-small-24b-harmony.json deleted file mode 100644 index 25ec8c5ec977af7c4f23110f286ac15d4e4cc608..0000000000000000000000000000000000000000 --- a/data/models/triangle104_mistral-small-24b-harmony.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mistral-Small-24b-Harmony", - "id": "Triangle104/Mistral-Small-24b-Harmony", - "developer": "Triangle104", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "23.572" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Triangle104_Mistral-Small-24b-Harmony/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1687 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6434 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1911 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3842 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4276 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5431 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/triangle104_pans_gutenbergum_v0.1.json b/data/models/triangle104_pans_gutenbergum_v0.1.json deleted file mode 100644 index 12d92ebeced136ce7ca22935034cf4951b323b56..0000000000000000000000000000000000000000 --- a/data/models/triangle104_pans_gutenbergum_v0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Pans_Gutenbergum_V0.1", - "id": "Triangle104/Pans_Gutenbergum_V0.1", - "developer": "Triangle104", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Triangle104_Pans_Gutenbergum_V0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3097 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5541 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1057 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.323 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4528 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3697 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/triangle104_pans_gutenbergum_v0.2.json b/data/models/triangle104_pans_gutenbergum_v0.2.json deleted file mode 100644 index 841efcd20a4339e24e8c75e16eccced46e746321..0000000000000000000000000000000000000000 --- a/data/models/triangle104_pans_gutenbergum_v0.2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Pans_Gutenbergum_V0.2", - "id": "Triangle104/Pans_Gutenbergum_V0.2", - "developer": "Triangle104", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Triangle104_Pans_Gutenbergum_V0.2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3215 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5526 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0687 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3121 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4673 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3585 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/triangle104_pantheon_chatwaifu_v0.2.json b/data/models/triangle104_pantheon_chatwaifu_v0.2.json deleted file mode 100644 index a9600dfc9a667fdbcfb7b90badacd4667f2b35e2..0000000000000000000000000000000000000000 --- a/data/models/triangle104_pantheon_chatwaifu_v0.2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Pantheon_ChatWaifu_V0.2", - "id": "Triangle104/Pantheon_ChatWaifu_V0.2", - "developer": "Triangle104", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Triangle104_Pantheon_ChatWaifu_V0.2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2683 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5532 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0566 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.318 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4755 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3442 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/triangle104_phi-4-abliteratedrp.json b/data/models/triangle104_phi-4-abliteratedrp.json deleted file mode 100644 index 2fcf99ea52ecca78eaf8385f24d51e5310e2c23b..0000000000000000000000000000000000000000 --- a/data/models/triangle104_phi-4-abliteratedrp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Phi-4-AbliteratedRP", - "id": "Triangle104/Phi-4-AbliteratedRP", - "developer": "Triangle104", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Phi3ForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Triangle104_Phi-4-AbliteratedRP/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4923 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6709 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3074 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3951 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5098 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5308 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/triangle104_phi4-rp-o1-ablit.json b/data/models/triangle104_phi4-rp-o1-ablit.json deleted file mode 100644 index 55cfbadfce3b944a92dedc150394315123bea829..0000000000000000000000000000000000000000 --- a/data/models/triangle104_phi4-rp-o1-ablit.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Phi4-RP-o1-Ablit", - "id": "Triangle104/Phi4-RP-o1-Ablit", - "developer": "Triangle104", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Triangle104_Phi4-RP-o1-Ablit/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0239 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.663 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3882 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3633 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4754 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5105 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/triangle104_phi4-rp-o1.json b/data/models/triangle104_phi4-rp-o1.json deleted file mode 100644 index 45376dbacbfbbd8cb289fe9b9c5db2819e6feae0..0000000000000000000000000000000000000000 --- a/data/models/triangle104_phi4-rp-o1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Phi4-RP-o1", - "id": "Triangle104/Phi4-RP-o1", - "developer": "Triangle104", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Triangle104_Phi4-RP-o1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.022 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6653 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3776 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3733 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4756 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5111 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/triangle104_porpoise-r1-llama3.2-3b.json b/data/models/triangle104_porpoise-r1-llama3.2-3b.json deleted file mode 100644 index c76219d4a03f90c18bd1fc925e5f1cf477be23b4..0000000000000000000000000000000000000000 --- a/data/models/triangle104_porpoise-r1-llama3.2-3b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Porpoise-R1-Llama3.2-3b", - "id": "Triangle104/Porpoise-R1-Llama3.2-3b", - "developer": "Triangle104", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Triangle104_Porpoise-R1-Llama3.2-3b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4352 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3824 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0423 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2668 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3576 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2117 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/triangle104_q2.5-14b-instruct-1m-harmony.json b/data/models/triangle104_q2.5-14b-instruct-1m-harmony.json deleted file mode 100644 index f31f70e002937bbacc50b1984c7cc21333a8d5d7..0000000000000000000000000000000000000000 --- a/data/models/triangle104_q2.5-14b-instruct-1m-harmony.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Q2.5-14B-Instruct-1M-Harmony", - "id": "Triangle104/Q2.5-14B-Instruct-1M-Harmony", - "developer": "Triangle104", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Triangle104_Q2.5-14B-Instruct-1M-Harmony/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5986 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6339 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3769 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.375 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4795 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5075 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/triangle104_q2.5-athenscot.json b/data/models/triangle104_q2.5-athenscot.json deleted file mode 100644 index f62376850b9b2d3811b9caeeca235c240498c4c5..0000000000000000000000000000000000000000 --- a/data/models/triangle104_q2.5-athenscot.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Q2.5-AthensCOT", - "id": "Triangle104/Q2.5-AthensCOT", - "developer": "Triangle104", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Triangle104_Q2.5-AthensCOT/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4573 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5542 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2915 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3003 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4578 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4379 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/triangle104_q2.5-coder1-3b.json b/data/models/triangle104_q2.5-coder1-3b.json deleted file mode 100644 index 34aca0d796ffba3a2491ca139abe05daec7df84d..0000000000000000000000000000000000000000 --- a/data/models/triangle104_q2.5-coder1-3b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Q2.5-CodeR1-3B", - "id": "Triangle104/Q2.5-CodeR1-3B", - "developer": "Triangle104", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.085" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Triangle104_Q2.5-CodeR1-3B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3588 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4661 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1639 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3037 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4315 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2979 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/triangle104_q2.5-evacot-7b.json b/data/models/triangle104_q2.5-evacot-7b.json deleted file mode 100644 index edb05c12e5558f3d5a61028dd901bdcba0c1ff33..0000000000000000000000000000000000000000 --- a/data/models/triangle104_q2.5-evacot-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Q2.5-EVACOT-7b", - "id": "Triangle104/Q2.5-EVACOT-7b", - "developer": "Triangle104", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Triangle104_Q2.5-EVACOT-7b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5784 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5506 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2825 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.318 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4499 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4331 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/triangle104_q2.5-evahumane-rp.json b/data/models/triangle104_q2.5-evahumane-rp.json deleted file mode 100644 index d7bf4e70ea324ee5f19973deceab7420ff3f9d3d..0000000000000000000000000000000000000000 --- a/data/models/triangle104_q2.5-evahumane-rp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Q2.5-EvaHumane-RP", - "id": "Triangle104/Q2.5-EvaHumane-RP", - "developer": "Triangle104", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Triangle104_Q2.5-EvaHumane-RP/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3676 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5328 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2923 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3188 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4276 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4412 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/triangle104_q2.5-humane-rp.json b/data/models/triangle104_q2.5-humane-rp.json deleted file mode 100644 index 02a01b32248906225f35644f12278305a87fd2dd..0000000000000000000000000000000000000000 --- a/data/models/triangle104_q2.5-humane-rp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Q2.5-Humane-RP", - "id": "Triangle104/Q2.5-Humane-RP", - "developer": "Triangle104", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Triangle104_Q2.5-Humane-RP/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4412 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5649 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3391 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3188 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4528 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4492 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/triangle104_q2.5-instruct-1m_harmony.json b/data/models/triangle104_q2.5-instruct-1m_harmony.json deleted file mode 100644 index 0676dcc8d25e19f62d79831e271321052762f1d2..0000000000000000000000000000000000000000 --- a/data/models/triangle104_q2.5-instruct-1m_harmony.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Q2.5-Instruct-1M_Harmony", - "id": "Triangle104/Q2.5-Instruct-1M_Harmony", - "developer": "Triangle104", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Triangle104_Q2.5-Instruct-1M_Harmony/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6038 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5373 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3323 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.323 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4688 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4366 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/triangle104_q2.5-r1-3b.json b/data/models/triangle104_q2.5-r1-3b.json deleted file mode 100644 index 36537867630b382951bc7c6a4dfb294fb4c3747f..0000000000000000000000000000000000000000 --- a/data/models/triangle104_q2.5-r1-3b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Q2.5-R1-3B", - "id": "Triangle104/Q2.5-R1-3B", - "developer": "Triangle104", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.085" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Triangle104_Q2.5-R1-3B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4214 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4812 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2674 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3096 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.432 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3813 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/triangle104_q2.5-r1-7b.json b/data/models/triangle104_q2.5-r1-7b.json deleted file mode 100644 index 87e787aa4040e3a5e170c56c3cae93afdf655050..0000000000000000000000000000000000000000 --- a/data/models/triangle104_q2.5-r1-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Q2.5-R1-7B", - "id": "Triangle104/Q2.5-R1-7B", - "developer": "Triangle104", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.613" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Triangle104_Q2.5-R1-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1346 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3007 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0166 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2525 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3607 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.118 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/triangle104_robo-gutenberg_v1.0.json b/data/models/triangle104_robo-gutenberg_v1.0.json deleted file mode 100644 index 280ccf7b61ec9a14bae708a56d3738d40aa3f78b..0000000000000000000000000000000000000000 --- a/data/models/triangle104_robo-gutenberg_v1.0.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Robo-Gutenberg_V1.0", - "id": "Triangle104/Robo-Gutenberg_V1.0", - "developer": "Triangle104", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Triangle104_Robo-Gutenberg_V1.0/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6008 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6537 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4562 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3859 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4744 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5391 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/triangle104_rocinante-prism_v2.0.json b/data/models/triangle104_rocinante-prism_v2.0.json deleted file mode 100644 index fb82b9b86fe40457a2ebd94e2b4c814bc16ff490..0000000000000000000000000000000000000000 --- a/data/models/triangle104_rocinante-prism_v2.0.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Rocinante-Prism_V2.0", - "id": "Triangle104/Rocinante-Prism_V2.0", - "developer": "Triangle104", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Triangle104_Rocinante-Prism_V2.0/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2616 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5361 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.111 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3205 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.445 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.364 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/triangle104_rocinante-prism_v2.1.json b/data/models/triangle104_rocinante-prism_v2.1.json deleted file mode 100644 index 5107ff6d33526f31723cdf83dc74b5f716c3a469..0000000000000000000000000000000000000000 --- a/data/models/triangle104_rocinante-prism_v2.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Rocinante-Prism_V2.1", - "id": "Triangle104/Rocinante-Prism_V2.1", - "developer": "Triangle104", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Triangle104_Rocinante-Prism_V2.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2558 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5333 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1125 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3196 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.449 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3651 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/triangle104_rombohermes3-r1-llama3.2-3b.json b/data/models/triangle104_rombohermes3-r1-llama3.2-3b.json deleted file mode 100644 index 2258fabe1aaf19cf82741accad6f5474b787624d..0000000000000000000000000000000000000000 --- a/data/models/triangle104_rombohermes3-r1-llama3.2-3b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "RomboHermes3-R1-Llama3.2-3b", - "id": "Triangle104/RomboHermes3-R1-Llama3.2-3b", - "developer": "Triangle104", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Triangle104_RomboHermes3-R1-Llama3.2-3b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3007 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4264 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0816 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2836 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3657 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2957 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/triangle104_rombos-novasky-7b_v1c.json b/data/models/triangle104_rombos-novasky-7b_v1c.json deleted file mode 100644 index 57983c416365907e8c0baecf9f89b79ebbc009bd..0000000000000000000000000000000000000000 --- a/data/models/triangle104_rombos-novasky-7b_v1c.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Rombos-Novasky-7B_V1c", - "id": "Triangle104/Rombos-Novasky-7B_V1c", - "developer": "Triangle104", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Triangle104_Rombos-Novasky-7B_V1c/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.408 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4349 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0853 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2961 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4465 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2738 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/triangle104_set-70b.json b/data/models/triangle104_set-70b.json deleted file mode 100644 index 5c2d1851eb6fbda8c297b133489696199e43f30b..0000000000000000000000000000000000000000 --- a/data/models/triangle104_set-70b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Set-70b", - "id": "Triangle104/Set-70b", - "developer": "Triangle104", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "70.554" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Triangle104_Set-70b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7643 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7014 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.364 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4463 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4696 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5442 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/trthminh1112_autotrain-llama32-1b-finetune.json b/data/models/trthminh1112_autotrain-llama32-1b-finetune.json deleted file mode 100644 index cf86d3d5e2bebedeefade27f95e37640c68f7f94..0000000000000000000000000000000000000000 --- a/data/models/trthminh1112_autotrain-llama32-1b-finetune.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "autotrain-llama32-1b-finetune", - "id": "trthminh1112/autotrain-llama32-1b-finetune", - "developer": "trthminh1112", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.1" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/trthminh1112_autotrain-llama32-1b-finetune/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1769 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2996 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0151 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2567 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3513 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1099 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/tsunami-th_tsunami-0.5-7b-instruct.json b/data/models/tsunami-th_tsunami-0.5-7b-instruct.json deleted file mode 100644 index 0fa7c1161a8b05fe30ca8b2a55a954cbbae66e06..0000000000000000000000000000000000000000 --- a/data/models/tsunami-th_tsunami-0.5-7b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Tsunami-0.5-7B-Instruct", - "id": "Tsunami-th/Tsunami-0.5-7B-Instruct", - "developer": "Tsunami-th", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Tsunami-th_Tsunami-0.5-7B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.74 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5524 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5045 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3087 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4257 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4413 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/tsunami-th_tsunami-0.5x-7b-instruct.json b/data/models/tsunami-th_tsunami-0.5x-7b-instruct.json deleted file mode 100644 index a7c850c284fd91711a5de29ec9d97ac960fb0936..0000000000000000000000000000000000000000 --- a/data/models/tsunami-th_tsunami-0.5x-7b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Tsunami-0.5x-7B-Instruct", - "id": "Tsunami-th/Tsunami-0.5x-7B-Instruct", - "developer": "Tsunami-th", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Tsunami-th_Tsunami-0.5x-7B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7099 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5593 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4207 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3146 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4667 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4458 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/tsunami-th_tsunami-1.0-14b-instruct.json b/data/models/tsunami-th_tsunami-1.0-14b-instruct.json deleted file mode 100644 index ea039b35a0ca46c9b6673c7e3249b29b653ccefa..0000000000000000000000000000000000000000 --- a/data/models/tsunami-th_tsunami-1.0-14b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Tsunami-1.0-14B-Instruct", - "id": "Tsunami-th/Tsunami-1.0-14B-Instruct", - "developer": "Tsunami-th", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Tsunami-th_Tsunami-1.0-14B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7829 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6439 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4585 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3565 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4459 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5249 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/tsunami-th_tsunami-1.0-7b-instruct.json b/data/models/tsunami-th_tsunami-1.0-7b-instruct.json deleted file mode 100644 index d618ffd2e627027faad743dc73a030f5085b6195..0000000000000000000000000000000000000000 --- a/data/models/tsunami-th_tsunami-1.0-7b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Tsunami-1.0-7B-Instruct", - "id": "Tsunami-th/Tsunami-1.0-7B-Instruct", - "developer": "Tsunami-th", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Tsunami-th_Tsunami-1.0-7B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7309 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5491 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4335 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3129 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4493 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4424 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/tttxxx01_mistral-7b-base-simpo2-5e-7.json b/data/models/tttxxx01_mistral-7b-base-simpo2-5e-7.json deleted file mode 100644 index 26596757a95bfa09a14416e898861e0aee3924b2..0000000000000000000000000000000000000000 --- a/data/models/tttxxx01_mistral-7b-base-simpo2-5e-7.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mistral-7B-Base-SimPO2-5e-7", - "id": "TTTXXX01/Mistral-7B-Base-SimPO2-5e-7", - "developer": "TTTXXX01", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/TTTXXX01_Mistral-7B-Base-SimPO2-5e-7/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4392 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.432 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0264 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2978 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3604 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2766 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/tugstugi_qwen2.5-7b-instruct-qwq-v0.1.json b/data/models/tugstugi_qwen2.5-7b-instruct-qwq-v0.1.json deleted file mode 100644 index f213f278f867ac03567cc56fe460ada451cf8caa..0000000000000000000000000000000000000000 --- a/data/models/tugstugi_qwen2.5-7b-instruct-qwq-v0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-7B-Instruct-QwQ-v0.1", - "id": "tugstugi/Qwen2.5-7B-Instruct-QwQ-v0.1", - "developer": "tugstugi", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/tugstugi_Qwen2.5-7B-Instruct-QwQ-v0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6017 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5101 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3814 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2685 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3794 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4081 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ucla-agi_gemma-2-9b-it-sppo-iter1.json b/data/models/ucla-agi_gemma-2-9b-it-sppo-iter1.json deleted file mode 100644 index d4e5ebd201591215923907227926ed2b311d8a68..0000000000000000000000000000000000000000 --- a/data/models/ucla-agi_gemma-2-9b-it-sppo-iter1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Gemma-2-9B-It-SPPO-Iter1", - "id": "UCLA-AGI/Gemma-2-9B-It-SPPO-Iter1", - "developer": "UCLA-AGI", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "9.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/UCLA-AGI_Gemma-2-9B-It-SPPO-Iter1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3082 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5969 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0899 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3364 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4099 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3907 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ucla-agi_gemma-2-9b-it-sppo-iter2.json b/data/models/ucla-agi_gemma-2-9b-it-sppo-iter2.json deleted file mode 100644 index ff80eb414129d8c691e54e4006579f33cc52f029..0000000000000000000000000000000000000000 --- a/data/models/ucla-agi_gemma-2-9b-it-sppo-iter2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Gemma-2-9B-It-SPPO-Iter2", - "id": "UCLA-AGI/Gemma-2-9B-It-SPPO-Iter2", - "developer": "UCLA-AGI", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "9.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/UCLA-AGI_Gemma-2-9B-It-SPPO-Iter2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.31 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.599 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0808 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3347 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4139 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.387 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ucla-agi_gemma-2-9b-it-sppo-iter3.json b/data/models/ucla-agi_gemma-2-9b-it-sppo-iter3.json deleted file mode 100644 index 692abd9145ecb2c623ffe8c932acc6c53ea5201f..0000000000000000000000000000000000000000 --- a/data/models/ucla-agi_gemma-2-9b-it-sppo-iter3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Gemma-2-9B-It-SPPO-Iter3", - "id": "UCLA-AGI/Gemma-2-9B-It-SPPO-Iter3", - "developer": "UCLA-AGI", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "9.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/UCLA-AGI_Gemma-2-9B-It-SPPO-Iter3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3167 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6007 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.071 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3389 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4166 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3826 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ucla-agi_llama-3-instruct-8b-sppo-iter1.json b/data/models/ucla-agi_llama-3-instruct-8b-sppo-iter1.json deleted file mode 100644 index 92f85329d76dc46f1da12d2e8c9ea4b71f30c25e..0000000000000000000000000000000000000000 --- a/data/models/ucla-agi_llama-3-instruct-8b-sppo-iter1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-Instruct-8B-SPPO-Iter1", - "id": "UCLA-AGI/Llama-3-Instruct-8B-SPPO-Iter1", - "developer": "UCLA-AGI", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/UCLA-AGI_Llama-3-Instruct-8B-SPPO-Iter1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7299 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5058 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1148 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2676 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3568 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3711 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ucla-agi_llama-3-instruct-8b-sppo-iter2.json b/data/models/ucla-agi_llama-3-instruct-8b-sppo-iter2.json deleted file mode 100644 index 43c1fb70fa9cadb7d8b5fe33ff0ffca50e4ae233..0000000000000000000000000000000000000000 --- a/data/models/ucla-agi_llama-3-instruct-8b-sppo-iter2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-Instruct-8B-SPPO-Iter2", - "id": "UCLA-AGI/Llama-3-Instruct-8B-SPPO-Iter2", - "developer": "UCLA-AGI", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/UCLA-AGI_Llama-3-Instruct-8B-SPPO-Iter2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6989 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5089 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1035 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2668 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3594 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3692 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ucla-agi_llama-3-instruct-8b-sppo-iter3.json b/data/models/ucla-agi_llama-3-instruct-8b-sppo-iter3.json deleted file mode 100644 index 6c8b4b0133a49d420e13a89373c13c0888830855..0000000000000000000000000000000000000000 --- a/data/models/ucla-agi_llama-3-instruct-8b-sppo-iter3.json +++ /dev/null @@ -1,275 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-Instruct-8B-SPPO-Iter3", - "id": "UCLA-AGI/Llama-3-Instruct-8B-SPPO-Iter3", - "developer": "UCLA-AGI", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/UCLA-AGI_Llama-3-Instruct-8B-SPPO-Iter3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6834 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.508 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0959 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2651 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3661 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3644 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "hfopenllm_v2/UCLA-AGI_Llama-3-Instruct-8B-SPPO-Iter3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6703 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5076 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0718 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2651 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3647 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3658 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ucla-agi_mistral7b-pairrm-sppo-iter1.json b/data/models/ucla-agi_mistral7b-pairrm-sppo-iter1.json deleted file mode 100644 index 97b0c19ed5a458c5ccb2c9e82fe46a461e636701..0000000000000000000000000000000000000000 --- a/data/models/ucla-agi_mistral7b-pairrm-sppo-iter1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mistral7B-PairRM-SPPO-Iter1", - "id": "UCLA-AGI/Mistral7B-PairRM-SPPO-Iter1", - "developer": "UCLA-AGI", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/UCLA-AGI_Mistral7B-PairRM-SPPO-Iter1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5047 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4468 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0249 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2836 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3992 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2695 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ucla-agi_mistral7b-pairrm-sppo-iter2.json b/data/models/ucla-agi_mistral7b-pairrm-sppo-iter2.json deleted file mode 100644 index 5d4490fb6cd315a8209ff7aa3c508a7bfc8a4c32..0000000000000000000000000000000000000000 --- a/data/models/ucla-agi_mistral7b-pairrm-sppo-iter2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mistral7B-PairRM-SPPO-Iter2", - "id": "UCLA-AGI/Mistral7B-PairRM-SPPO-Iter2", - "developer": "UCLA-AGI", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/UCLA-AGI_Mistral7B-PairRM-SPPO-Iter2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4446 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4466 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0219 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2886 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4085 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2677 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ucla-agi_mistral7b-pairrm-sppo-iter3.json b/data/models/ucla-agi_mistral7b-pairrm-sppo-iter3.json deleted file mode 100644 index 9c13e7c52d20d7cc9610b471ac6198dcbf7d47f0..0000000000000000000000000000000000000000 --- a/data/models/ucla-agi_mistral7b-pairrm-sppo-iter3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mistral7B-PairRM-SPPO-Iter3", - "id": "UCLA-AGI/Mistral7B-PairRM-SPPO-Iter3", - "developer": "UCLA-AGI", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/UCLA-AGI_Mistral7B-PairRM-SPPO-Iter3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4351 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4397 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0234 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2752 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4071 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2658 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ucla-agi_mistral7b-pairrm-sppo.json b/data/models/ucla-agi_mistral7b-pairrm-sppo.json deleted file mode 100644 index 81d9f4c3970b97928b77d3a5c85efbc611615101..0000000000000000000000000000000000000000 --- a/data/models/ucla-agi_mistral7b-pairrm-sppo.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mistral7B-PairRM-SPPO", - "id": "UCLA-AGI/Mistral7B-PairRM-SPPO", - "developer": "UCLA-AGI", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/UCLA-AGI_Mistral7B-PairRM-SPPO/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4355 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4439 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.031 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.281 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3965 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2621 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/uiuc-oumi_coalm-70b.json b/data/models/uiuc-oumi_coalm-70b.json deleted file mode 100644 index 6815e3fb5ace90c7416ff915bb0de3099a9abaaa..0000000000000000000000000000000000000000 --- a/data/models/uiuc-oumi_coalm-70b.json +++ /dev/null @@ -1,904 +0,0 @@ -{ - "model_info": { - "name": "CoALM-70B", - "id": "uiuc-oumi/coalm-70b", - "developer": "uiuc-oumi", - "additional_details": { - "raw_model_name": "CoALM-70B", - "organization": "UIUC + Oumi", - "license": "Meta Llama 3 Community", - "model_link": "https://huggingface.co/uiuc-convai/CoALM-70B" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/uiuc-oumi/coalm-70b/1775236112.4045749", - "retrieved_timestamp": "1775236112.4045749", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 74.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 27.99 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 19.89 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 16.22 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 59.91 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 36.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 83.44 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 70.25 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 92.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 88.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 83.0 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 67.28 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 70.54 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 66.57 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 68.75 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 62.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 10.62 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 11.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 14.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 9.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 8.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 5.81 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 9.03 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 5.16 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 3.23 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 93.75 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 85.65 - } - }, - { - "evaluation_result_id": "bfcl.format_sensitivity.max_delta::max_delta", - "evaluation_name": "bfcl.format_sensitivity.max_delta", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.format_sensitivity.max_delta", - "metric_name": "Format sensitivity max delta", - "metric_kind": "difference", - "metric_unit": "percentage_points", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Format Sensitivity Max Delta" - } - }, - "score_details": { - "score": 72.0 - } - }, - { - "evaluation_result_id": "bfcl.format_sensitivity.stddev::stddev", - "evaluation_name": "bfcl.format_sensitivity.stddev", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.format_sensitivity.stddev", - "metric_name": "Format sensitivity standard deviation", - "metric_kind": "difference", - "metric_unit": "percentage_points", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Format Sensitivity Standard Deviation" - } - }, - "score_details": { - "score": 27.76 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/uiuc-oumi_coalm-8b.json b/data/models/uiuc-oumi_coalm-8b.json deleted file mode 100644 index 2b86a5a0a667ec6a464d1055dbe12ad7bfdb7da5..0000000000000000000000000000000000000000 --- a/data/models/uiuc-oumi_coalm-8b.json +++ /dev/null @@ -1,904 +0,0 @@ -{ - "model_info": { - "name": "CoALM-8B", - "id": "uiuc-oumi/coalm-8b", - "developer": "uiuc-oumi", - "additional_details": { - "raw_model_name": "CoALM-8B", - "organization": "UIUC + Oumi", - "license": "Meta Llama 3 Community", - "model_link": "https://huggingface.co/uiuc-convai/CoALM-8B" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/uiuc-oumi/coalm-8b/1775236112.410044", - "retrieved_timestamp": "1775236112.410044", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 84.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 26.81 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 25.33 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 20.36 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 73.74 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 138.04 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 84.87 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 69.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 93.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 88.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 88.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 66.77 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 70.54 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 66.19 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 62.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 54.17 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 8.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 10.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 7.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 8.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 7.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 2.8 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 3.23 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 3.87 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 1.29 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 87.5 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 86.9 - } - }, - { - "evaluation_result_id": "bfcl.format_sensitivity.max_delta::max_delta", - "evaluation_name": "bfcl.format_sensitivity.max_delta", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.format_sensitivity.max_delta", - "metric_name": "Format sensitivity max delta", - "metric_kind": "difference", - "metric_unit": "percentage_points", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Format Sensitivity Max Delta" - } - }, - "score_details": { - "score": 79.0 - } - }, - { - "evaluation_result_id": "bfcl.format_sensitivity.stddev::stddev", - "evaluation_name": "bfcl.format_sensitivity.stddev", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.format_sensitivity.stddev", - "metric_name": "Format sensitivity standard deviation", - "metric_kind": "difference", - "metric_unit": "percentage_points", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Format Sensitivity Standard Deviation" - } - }, - "score_details": { - "score": 34.18 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ukzexecution_llamaexecutor-8b-3.0.5.json b/data/models/ukzexecution_llamaexecutor-8b-3.0.5.json deleted file mode 100644 index 503bfde5a2e6ffc8115c216c847b9f4be5ad4253..0000000000000000000000000000000000000000 --- a/data/models/ukzexecution_llamaexecutor-8b-3.0.5.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "LlamaExecutor-8B-3.0.5", - "id": "UKzExecution/LlamaExecutor-8B-3.0.5", - "developer": "UKzExecution", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/UKzExecution_LlamaExecutor-8B-3.0.5/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7403 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5006 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.102 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2559 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3754 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3625 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/unbabel_towerinstruct-mistral-7b-v0.2.json b/data/models/unbabel_towerinstruct-mistral-7b-v0.2.json deleted file mode 100644 index 2ffc21e0616d1bcace5c3bb9b29c6b35b04bddaa..0000000000000000000000000000000000000000 --- a/data/models/unbabel_towerinstruct-mistral-7b-v0.2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "TowerInstruct-Mistral-7B-v0.2", - "id": "Unbabel/TowerInstruct-Mistral-7B-v0.2", - "developer": "Unbabel", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Unbabel_TowerInstruct-Mistral-7B-v0.2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2843 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3882 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0204 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2475 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4522 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1968 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/undi95_mg-finalmix-72b.json b/data/models/undi95_mg-finalmix-72b.json deleted file mode 100644 index 795f33ef9867b8416dccadac5c0fabbaa481875d..0000000000000000000000000000000000000000 --- a/data/models/undi95_mg-finalmix-72b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MG-FinalMix-72B", - "id": "Undi95/MG-FinalMix-72B", - "developer": "Undi95", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "72.706" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Undi95_MG-FinalMix-72B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8014 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6973 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3973 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3851 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4823 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5427 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/undi95_phi4-abliterated.json b/data/models/undi95_phi4-abliterated.json deleted file mode 100644 index 7b34a58860a5e1ed396239e7638e72459346d8d7..0000000000000000000000000000000000000000 --- a/data/models/undi95_phi4-abliterated.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Phi4-abliterated", - "id": "Undi95/Phi4-abliterated", - "developer": "Undi95", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Phi3ForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Undi95_Phi4-abliterated/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6618 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6809 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3701 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3305 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4034 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5281 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/universalml_nepaligpt-2.0.json b/data/models/universalml_nepaligpt-2.0.json deleted file mode 100644 index 3c8caf5452476d1ac2feb704fdb577ce557b6171..0000000000000000000000000000000000000000 --- a/data/models/universalml_nepaligpt-2.0.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "NepaliGPT-2.0", - "id": "universalml/NepaliGPT-2.0", - "developer": "universalml", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/universalml_NepaliGPT-2.0/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0365 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.466 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0045 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.281 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4657 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.33 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/unknown_aya-expanse-32b.json b/data/models/unknown_aya-expanse-32b.json deleted file mode 100644 index e780437025d8d97f38267a679b1400ad4dec344b..0000000000000000000000000000000000000000 --- a/data/models/unknown_aya-expanse-32b.json +++ /dev/null @@ -1,528 +0,0 @@ -{ - "model_info": { - "name": "aya-expanse-32b", - "id": "unknown/aya-expanse-32b", - "developer": "unknown", - "inference_platform": "unknown", - "additional_details": { - "display_name": "Aya Expanse 32B" - } - }, - "evaluations": [ - { - "evaluation_id": "global-mmlu-lite/unknown_aya-expanse-32b/1773936496.366405", - "retrieved_timestamp": "1773936496.366405", - "source_metadata": { - "source_name": "Global MMLU Lite Leaderboard", - "source_type": "documentation", - "source_organization_name": "kaggle", - "source_organization_url": "www.kaggle.com", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "kaggle kernel", - "version": "4", - "additional_details": { - "url": "https://www.kaggle.com/code/shivalikasingh95/global-mmlu-lite-sample-notebook" - } - }, - "benchmark": "global-mmlu-lite", - "evaluation_results": [ - { - "evaluation_name": "Global MMLU Lite", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Global MMLU Lite", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7353 - } - }, - { - "evaluation_name": "Culturally Sensitive", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Sensitive", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6891 - } - }, - { - "evaluation_name": "Culturally Agnostic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Agnostic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7815 - } - }, - { - "evaluation_name": "Arabic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Arabic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7425, - "uncertainty": { - "confidence_interval": { - "lower": -0.0429, - "upper": 0.0429, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "English", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - English", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7544, - "uncertainty": { - "confidence_interval": { - "lower": -0.0422, - "upper": 0.0422, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Bengali", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Bengali", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7343, - "uncertainty": { - "confidence_interval": { - "lower": -0.0433, - "upper": 0.0433, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "German", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - German", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7425, - "uncertainty": { - "confidence_interval": { - "lower": -0.0429, - "upper": 0.0429, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "French", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - French", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7325, - "uncertainty": { - "confidence_interval": { - "lower": -0.0434, - "upper": 0.0434, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Hindi", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Hindi", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7375, - "uncertainty": { - "confidence_interval": { - "lower": -0.0431, - "upper": 0.0431, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Indonesian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Indonesian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7594, - "uncertainty": { - "confidence_interval": { - "lower": -0.0419, - "upper": 0.0419, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Italian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Italian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7305, - "uncertainty": { - "confidence_interval": { - "lower": -0.0436, - "upper": 0.0436, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Japanese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Japanese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7419, - "uncertainty": { - "confidence_interval": { - "lower": -0.0429, - "upper": 0.0429, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Korean", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Korean", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7525, - "uncertainty": { - "confidence_interval": { - "lower": -0.0423, - "upper": 0.0423, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Portuguese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Portuguese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7544, - "uncertainty": { - "confidence_interval": { - "lower": -0.0422, - "upper": 0.0422, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Spanish", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Spanish", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7362, - "uncertainty": { - "confidence_interval": { - "lower": -0.0433, - "upper": 0.0433, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Swahili", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Swahili", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7071, - "uncertainty": { - "confidence_interval": { - "lower": -0.0448, - "upper": 0.0448, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Yoruba", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Yoruba", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6942, - "uncertainty": { - "confidence_interval": { - "lower": -0.0452, - "upper": 0.0452, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Chinese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Chinese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.743, - "uncertainty": { - "confidence_interval": { - "lower": -0.0432, - "upper": 0.0432, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Burmese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Burmese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7025, - "uncertainty": { - "confidence_interval": { - "lower": -0.0448, - "upper": 0.0448, - "method": "unknown" - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/unknown_granite-4.0-h-small.json b/data/models/unknown_granite-4.0-h-small.json deleted file mode 100644 index f58d2903d042127912c27fce574fc328b5a791e8..0000000000000000000000000000000000000000 --- a/data/models/unknown_granite-4.0-h-small.json +++ /dev/null @@ -1,528 +0,0 @@ -{ - "model_info": { - "name": "granite-4.0-h-small", - "id": "unknown/granite-4.0-h-small", - "developer": "unknown", - "inference_platform": "unknown", - "additional_details": { - "display_name": "Granite 4.0 Small" - } - }, - "evaluations": [ - { - "evaluation_id": "global-mmlu-lite/unknown_granite-4.0-h-small/1773936496.366405", - "retrieved_timestamp": "1773936496.366405", - "source_metadata": { - "source_name": "Global MMLU Lite Leaderboard", - "source_type": "documentation", - "source_organization_name": "kaggle", - "source_organization_url": "www.kaggle.com", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "kaggle kernel", - "version": "4", - "additional_details": { - "url": "https://www.kaggle.com/code/shivalikasingh95/global-mmlu-lite-sample-notebook" - } - }, - "benchmark": "global-mmlu-lite", - "evaluation_results": [ - { - "evaluation_name": "Global MMLU Lite", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Global MMLU Lite", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7503 - } - }, - { - "evaluation_name": "Culturally Sensitive", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Sensitive", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7182 - } - }, - { - "evaluation_name": "Culturally Agnostic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Agnostic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7826 - } - }, - { - "evaluation_name": "Arabic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Arabic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7613, - "uncertainty": { - "confidence_interval": { - "lower": -0.0419, - "upper": 0.0419, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "English", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - English", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.77, - "uncertainty": { - "confidence_interval": { - "lower": -0.0412, - "upper": 0.0412, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Bengali", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Bengali", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7613, - "uncertainty": { - "confidence_interval": { - "lower": -0.0419, - "upper": 0.0419, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "German", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - German", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.755, - "uncertainty": { - "confidence_interval": { - "lower": -0.0421, - "upper": 0.0421, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "French", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - French", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7594, - "uncertainty": { - "confidence_interval": { - "lower": -0.0419, - "upper": 0.0419, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Hindi", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Hindi", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7575, - "uncertainty": { - "confidence_interval": { - "lower": -0.042, - "upper": 0.042, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Indonesian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Indonesian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7614, - "uncertainty": { - "confidence_interval": { - "lower": -0.0421, - "upper": 0.0421, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Italian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Italian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7525, - "uncertainty": { - "confidence_interval": { - "lower": -0.0423, - "upper": 0.0423, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Japanese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Japanese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7406, - "uncertainty": { - "confidence_interval": { - "lower": -0.0431, - "upper": 0.0431, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Korean", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Korean", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7525, - "uncertainty": { - "confidence_interval": { - "lower": -0.0423, - "upper": 0.0423, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Portuguese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Portuguese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.757, - "uncertainty": { - "confidence_interval": { - "lower": -0.0423, - "upper": 0.0423, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Spanish", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Spanish", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7638, - "uncertainty": { - "confidence_interval": { - "lower": -0.0417, - "upper": 0.0417, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Swahili", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Swahili", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7318, - "uncertainty": { - "confidence_interval": { - "lower": -0.0435, - "upper": 0.0435, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Yoruba", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Yoruba", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6921, - "uncertainty": { - "confidence_interval": { - "lower": -0.0456, - "upper": 0.0456, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Chinese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Chinese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7475, - "uncertainty": { - "confidence_interval": { - "lower": -0.0426, - "upper": 0.0426, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Burmese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Burmese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7419, - "uncertainty": { - "confidence_interval": { - "lower": -0.0429, - "upper": 0.0429, - "method": "unknown" - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/unknown_o4-mini-2025-04-16.json b/data/models/unknown_o4-mini-2025-04-16.json deleted file mode 100644 index 8da9f6803d5f8fe49f6142b3f11bffa5f2b63717..0000000000000000000000000000000000000000 --- a/data/models/unknown_o4-mini-2025-04-16.json +++ /dev/null @@ -1,528 +0,0 @@ -{ - "model_info": { - "name": "o4-mini-2025-04-16", - "id": "unknown/o4-mini-2025-04-16", - "developer": "unknown", - "inference_platform": "unknown", - "additional_details": { - "display_name": "o4 mini" - } - }, - "evaluations": [ - { - "evaluation_id": "global-mmlu-lite/unknown_o4-mini-2025-04-16/1773936496.366405", - "retrieved_timestamp": "1773936496.366405", - "source_metadata": { - "source_name": "Global MMLU Lite Leaderboard", - "source_type": "documentation", - "source_organization_name": "kaggle", - "source_organization_url": "www.kaggle.com", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "kaggle kernel", - "version": "4", - "additional_details": { - "url": "https://www.kaggle.com/code/shivalikasingh95/global-mmlu-lite-sample-notebook" - } - }, - "benchmark": "global-mmlu-lite", - "evaluation_results": [ - { - "evaluation_name": "Global MMLU Lite", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Global MMLU Lite", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8705 - } - }, - { - "evaluation_name": "Culturally Sensitive", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Sensitive", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8503 - } - }, - { - "evaluation_name": "Culturally Agnostic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Agnostic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8906 - } - }, - { - "evaluation_name": "Arabic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Arabic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.865, - "uncertainty": { - "confidence_interval": { - "lower": -0.0335, - "upper": 0.0335, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "English", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - English", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8675, - "uncertainty": { - "confidence_interval": { - "lower": -0.0332, - "upper": 0.0332, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Bengali", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Bengali", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8875, - "uncertainty": { - "confidence_interval": { - "lower": -0.031, - "upper": 0.031, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "German", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - German", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8775, - "uncertainty": { - "confidence_interval": { - "lower": -0.0321, - "upper": 0.0321, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "French", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - French", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.87, - "uncertainty": { - "confidence_interval": { - "lower": -0.033, - "upper": 0.033, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Hindi", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Hindi", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.87, - "uncertainty": { - "confidence_interval": { - "lower": -0.033, - "upper": 0.033, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Indonesian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Indonesian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8675, - "uncertainty": { - "confidence_interval": { - "lower": -0.0332, - "upper": 0.0332, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Italian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Italian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.855, - "uncertainty": { - "confidence_interval": { - "lower": -0.0345, - "upper": 0.0345, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Japanese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Japanese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.885, - "uncertainty": { - "confidence_interval": { - "lower": -0.0313, - "upper": 0.0313, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Korean", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Korean", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.88, - "uncertainty": { - "confidence_interval": { - "lower": -0.0318, - "upper": 0.0318, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Portuguese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Portuguese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.88, - "uncertainty": { - "confidence_interval": { - "lower": -0.0318, - "upper": 0.0318, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Spanish", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Spanish", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.855, - "uncertainty": { - "confidence_interval": { - "lower": -0.0345, - "upper": 0.0345, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Swahili", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Swahili", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8525, - "uncertainty": { - "confidence_interval": { - "lower": -0.0348, - "upper": 0.0348, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Yoruba", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Yoruba", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8525, - "uncertainty": { - "confidence_interval": { - "lower": -0.0348, - "upper": 0.0348, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Chinese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Chinese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.89, - "uncertainty": { - "confidence_interval": { - "lower": -0.0307, - "upper": 0.0307, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Burmese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Burmese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8725, - "uncertainty": { - "confidence_interval": { - "lower": -0.0327, - "upper": 0.0327, - "method": "unknown" - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/unsloth_llama-3.2-1b-instruct-no-system-message.json b/data/models/unsloth_llama-3.2-1b-instruct-no-system-message.json deleted file mode 100644 index 0afb3c8f249e9d5a96f3923df930e124f37b4041..0000000000000000000000000000000000000000 --- a/data/models/unsloth_llama-3.2-1b-instruct-no-system-message.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.2-1B-Instruct-no-system-message", - "id": "unsloth/Llama-3.2-1B-Instruct-no-system-message", - "developer": "unsloth", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.236" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/unsloth_Llama-3.2-1B-Instruct-no-system-message/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.565 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3544 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0755 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2727 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3341 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1669 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/unsloth_llama-3.2-1b-instruct.json b/data/models/unsloth_llama-3.2-1b-instruct.json deleted file mode 100644 index c8cb5f657868c4fcf1a2c9ee20be30767bb25a86..0000000000000000000000000000000000000000 --- a/data/models/unsloth_llama-3.2-1b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.2-1B-Instruct", - "id": "unsloth/Llama-3.2-1B-Instruct", - "developer": "unsloth", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.236" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/unsloth_Llama-3.2-1B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.581 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3485 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0823 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2676 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3196 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1742 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/unsloth_phi-3-mini-4k-instruct.json b/data/models/unsloth_phi-3-mini-4k-instruct.json deleted file mode 100644 index 90729d483dd227dba7923a3bab40032d5ea36200..0000000000000000000000000000000000000000 --- a/data/models/unsloth_phi-3-mini-4k-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Phi-3-mini-4k-instruct", - "id": "unsloth/Phi-3-mini-4k-instruct", - "developer": "unsloth", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "3.821" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/unsloth_Phi-3-mini-4k-instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.544 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.55 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1639 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.323 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4284 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4031 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/unsloth_phi-4-bnb-4bit.json b/data/models/unsloth_phi-4-bnb-4bit.json deleted file mode 100644 index 04bcf8af207861f66e67eaaf96bfe1fb431e8db8..0000000000000000000000000000000000000000 --- a/data/models/unsloth_phi-4-bnb-4bit.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "phi-4-bnb-4bit", - "id": "unsloth/phi-4-bnb-4bit", - "developer": "unsloth", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.058" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/unsloth_phi-4-bnb-4bit/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.673 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.677 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4607 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3381 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4007 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5256 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/unsloth_phi-4-unsloth-bnb-4bit.json b/data/models/unsloth_phi-4-unsloth-bnb-4bit.json deleted file mode 100644 index 29a136209a6fda1ab1bca74dc0bcee75d22c9753..0000000000000000000000000000000000000000 --- a/data/models/unsloth_phi-4-unsloth-bnb-4bit.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "phi-4-unsloth-bnb-4bit", - "id": "unsloth/phi-4-unsloth-bnb-4bit", - "developer": "unsloth", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.483" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/unsloth_phi-4-unsloth-bnb-4bit/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6794 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6791 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4562 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3364 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4034 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5286 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/unsloth_phi-4.json b/data/models/unsloth_phi-4.json deleted file mode 100644 index 46351dc380b9ac85d187f2065e4807ddad640047..0000000000000000000000000000000000000000 --- a/data/models/unsloth_phi-4.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "phi-4", - "id": "unsloth/phi-4", - "developer": "unsloth", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/unsloth_phi-4/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6882 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6886 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3364 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4114 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5378 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/upstage_solar-10.7b-instruct-v1.0.json b/data/models/upstage_solar-10.7b-instruct-v1.0.json deleted file mode 100644 index c044a56f20275a076d6b731fa27a8f6b6162b486..0000000000000000000000000000000000000000 --- a/data/models/upstage_solar-10.7b-instruct-v1.0.json +++ /dev/null @@ -1,277 +0,0 @@ -{ - "model_info": { - "name": "SOLAR-10.7B-Instruct-v1.0", - "id": "upstage/SOLAR-10.7B-Instruct-v1.0", - "developer": "upstage", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "10.732" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/upstage_SOLAR-10.7B-Instruct-v1.0/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4737 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5162 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0566 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3087 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3899 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3138 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "reward-bench/upstage_SOLAR-10.7B-Instruct-v1.0/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7391 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8156 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6864 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8514 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7252 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4949 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/upstage_solar-10.7b-v1.0.json b/data/models/upstage_solar-10.7b-v1.0.json deleted file mode 100644 index 39828d4c8980a42119c7e67270e71a9b5371b5b5..0000000000000000000000000000000000000000 --- a/data/models/upstage_solar-10.7b-v1.0.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SOLAR-10.7B-v1.0", - "id": "upstage/SOLAR-10.7B-v1.0", - "developer": "upstage", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "10.732" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/upstage_SOLAR-10.7B-v1.0/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2421 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5094 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0264 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.281 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4372 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.34 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/upstage_solar-pro-241126.json b/data/models/upstage_solar-pro-241126.json deleted file mode 100644 index b7698c7d04a3edf4a865d64d793d720775a9aaf9..0000000000000000000000000000000000000000 --- a/data/models/upstage_solar-pro-241126.json +++ /dev/null @@ -1,1903 +0,0 @@ -{ - "model_info": { - "name": "Solar Pro", - "id": "upstage/solar-pro-241126", - "developer": "upstage", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_lite/upstage_solar-pro-241126/1774096306.427425", - "retrieved_timestamp": "1774096306.427425", - "source_metadata": { - "source_name": "helm_lite", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_lite", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_lite", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.602, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.4817852684144819\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.753, - "details": { - "description": "min=0.753, mean=0.753, max=0.753, sum=0.753 (1)", - "tab": "Accuracy", - "NarrativeQA - Observed inference time (s)": "{\"description\": \"min=2.29, mean=2.29, max=2.29, sum=2.29 (1)\", \"tab\": \"Efficiency\", \"score\": \"2.2897866705773584\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=4063.606, mean=4063.606, max=4063.606, sum=4063.606 (1)\", \"tab\": \"General information\", \"score\": \"4063.605633802817\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=5.972, mean=5.972, max=5.972, sum=5.972 (1)\", \"tab\": \"General information\", \"score\": \"5.971830985915493\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (closed-book)", - "source_data": { - "dataset_name": "NaturalQuestions (closed-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (closed-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.297, - "details": { - "description": "min=0.297, mean=0.297, max=0.297, sum=0.297 (1)", - "tab": "Accuracy", - "NaturalQuestions (open-book) - Observed inference time (s)": "{\"description\": \"min=1.102, mean=1.102, max=1.102, sum=1.102 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.1022112455368043\"}", - "NaturalQuestions (closed-book) - Observed inference time (s)": "{\"description\": \"min=0.588, mean=0.588, max=0.588, sum=0.588 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.5883909621238709\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=2513.406, mean=2513.406, max=2513.406, sum=2513.406 (1)\", \"tab\": \"General information\", \"score\": \"2513.406\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=7.252, mean=7.252, max=7.252, sum=7.252 (1)\", \"tab\": \"General information\", \"score\": \"7.252\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=156.383, mean=156.383, max=156.383, sum=156.383 (1)\", \"tab\": \"General information\", \"score\": \"156.383\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=9.034, mean=9.034, max=9.034, sum=9.034 (1)\", \"tab\": \"General information\", \"score\": \"9.034\"}" - } - }, - "generation_config": { - "additional_details": { - "mode": "\"closedbook\"" - } - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.922, - "details": { - "description": "min=0.922, mean=0.922, max=0.922, sum=0.922 (1)", - "tab": "Accuracy", - "OpenbookQA - Observed inference time (s)": "{\"description\": \"min=0.431, mean=0.431, max=0.431, sum=0.431 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.43103125095367434\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=291.574, mean=291.574, max=291.574, sum=291.574 (1)\", \"tab\": \"General information\", \"score\": \"291.574\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "dataset": "\"openbookqa\"", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.679, - "details": { - "description": "min=0.46, mean=0.679, max=0.97, sum=3.395 (5)", - "tab": "Accuracy", - "MMLU - Observed inference time (s)": "{\"description\": \"min=0.429, mean=0.529, max=0.765, sum=2.644 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.5287977041361624\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=406.65, mean=531.547, max=693.675, sum=2657.735 (5)\", \"tab\": \"General information\", \"score\": \"531.5470877192982\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"college_chemistry\", \"computer_security\", \"econometrics\", \"us_foreign_policy\"]", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MATH", - "source_data": { - "dataset_name": "MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Equivalent (CoT) on MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.567, - "details": { - "description": "min=0.421, mean=0.567, max=0.741, sum=3.968 (7)", - "tab": "Accuracy", - "MATH - Observed inference time (s)": "{\"description\": \"min=1.926, mean=2.29, max=2.87, sum=16.027 (7)\", \"tab\": \"Efficiency\", \"score\": \"2.289581796117552\"}", - "MATH - # eval": "{\"description\": \"min=30, mean=62.429, max=135, sum=437 (7)\", \"tab\": \"General information\", \"score\": \"62.42857142857143\"}", - "MATH - # train": "{\"description\": \"min=8, mean=8, max=8, sum=56 (7)\", \"tab\": \"General information\", \"score\": \"8.0\"}", - "MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MATH - # prompt tokens": "{\"description\": \"min=971.652, mean=1438.636, max=2490.962, sum=10070.453 (7)\", \"tab\": \"General information\", \"score\": \"1438.6362030100095\"}", - "MATH - # output tokens": "{\"description\": \"min=94.269, mean=124.053, max=183.018, sum=868.373 (7)\", \"tab\": \"General information\", \"score\": \"124.05328023895956\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"algebra\", \"counting_and_probability\", \"geometry\", \"intermediate_algebra\", \"number_theory\", \"prealgebra\", \"precalculus\"]", - "level": "\"1\"", - "use_official_examples": "\"False\"", - "use_chain_of_thought": "\"True\"" - } - } - }, - { - "evaluation_name": "GSM8K", - "source_data": { - "dataset_name": "GSM8K", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on GSM8K", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.871, - "details": { - "description": "min=0.871, mean=0.871, max=0.871, sum=0.871 (1)", - "tab": "Accuracy", - "GSM8K - Observed inference time (s)": "{\"description\": \"min=2.666, mean=2.666, max=2.666, sum=2.666 (1)\", \"tab\": \"Efficiency\", \"score\": \"2.6663423478603363\"}", - "GSM8K - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "GSM8K - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "GSM8K - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GSM8K - # prompt tokens": "{\"description\": \"min=1207.746, mean=1207.746, max=1207.746, sum=1207.746 (1)\", \"tab\": \"General information\", \"score\": \"1207.746\"}", - "GSM8K - # output tokens": "{\"description\": \"min=143.978, mean=143.978, max=143.978, sum=143.978 (1)\", \"tab\": \"General information\", \"score\": \"143.978\"}" - } - }, - "generation_config": { - "additional_details": { - "stop": "\"none\"" - } - } - }, - { - "evaluation_name": "LegalBench", - "source_data": { - "dataset_name": "LegalBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on LegalBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.67, - "details": { - "description": "min=0.384, mean=0.67, max=0.905, sum=3.348 (5)", - "tab": "Accuracy", - "LegalBench - Observed inference time (s)": "{\"description\": \"min=0.438, mean=0.654, max=1.454, sum=3.271 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.6542452756040519\"}", - "LegalBench - # eval": "{\"description\": \"min=95, mean=409.4, max=1000, sum=2047 (5)\", \"tab\": \"General information\", \"score\": \"409.4\"}", - "LegalBench - # train": "{\"description\": \"min=4, mean=4.8, max=5, sum=24 (5)\", \"tab\": \"General information\", \"score\": \"4.8\"}", - "LegalBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "LegalBench - # prompt tokens": "{\"description\": \"min=229.137, mean=1839.512, max=7675.188, sum=9197.561 (5)\", \"tab\": \"General information\", \"score\": \"1839.5122484246817\"}", - "LegalBench - # output tokens": "{\"description\": \"min=1, mean=1.395, max=2.011, sum=6.977 (5)\", \"tab\": \"General information\", \"score\": \"1.3953837372723363\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"abercrombie\", \"corporate_lobbying\", \"function_of_decision_section\", \"international_citizenship_questions\", \"proa\"]" - } - } - }, - { - "evaluation_name": "MedQA", - "source_data": { - "dataset_name": "MedQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MedQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.698, - "details": { - "description": "min=0.698, mean=0.698, max=0.698, sum=0.698 (1)", - "tab": "Accuracy", - "MedQA - Observed inference time (s)": "{\"description\": \"min=0.596, mean=0.596, max=0.596, sum=0.596 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.5956100185159187\"}", - "MedQA - # eval": "{\"description\": \"min=503, mean=503, max=503, sum=503 (1)\", \"tab\": \"General information\", \"score\": \"503.0\"}", - "MedQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MedQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MedQA - # prompt tokens": "{\"description\": \"min=1243.901, mean=1243.901, max=1243.901, sum=1243.901 (1)\", \"tab\": \"General information\", \"score\": \"1243.9005964214712\"}", - "MedQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WMT 2014", - "source_data": { - "dataset_name": "WMT 2014", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "BLEU-4 on WMT 2014", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.169, - "details": { - "description": "min=0.085, mean=0.169, max=0.229, sum=0.844 (5)", - "tab": "Accuracy", - "WMT 2014 - Observed inference time (s)": "{\"description\": \"min=0.839, mean=0.871, max=0.895, sum=4.357 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.8713457104322841\"}", - "WMT 2014 - # eval": "{\"description\": \"min=503, mean=568.8, max=832, sum=2844 (5)\", \"tab\": \"General information\", \"score\": \"568.8\"}", - "WMT 2014 - # train": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "WMT 2014 - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WMT 2014 - # prompt tokens": "{\"description\": \"min=135.523, mean=150.288, max=172.972, sum=751.438 (5)\", \"tab\": \"General information\", \"score\": \"150.28751290334915\"}", - "WMT 2014 - # output tokens": "{\"description\": \"min=27.539, mean=30.28, max=31.635, sum=151.4 (5)\", \"tab\": \"General information\", \"score\": \"30.280004587857473\"}" - } - }, - "generation_config": { - "additional_details": { - "language_pair": "[\"cs-en\", \"de-en\", \"fr-en\", \"hi-en\", \"ru-en\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_mmlu/upstage_solar-pro-241126/1774096312.00548", - "retrieved_timestamp": "1774096312.00548", - "source_metadata": { - "source_name": "helm_mmlu", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_mmlu", - "evaluation_results": [ - { - "evaluation_name": "MMLU All Subjects", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU All Subjects", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.776, - "details": { - "description": "min=0.44, mean=0.776, max=0.97, sum=88.521 (114)", - "tab": "Accuracy", - "MMLU All Subjects - Observed inference time (s)": "{\"description\": \"min=0.415, mean=0.5, max=1.447, sum=56.972 (114)\", \"tab\": \"Efficiency\", \"score\": \"0.4997569605932576\"}", - "MMLU All Subjects - # eval": "{\"description\": \"min=100, mean=246.351, max=1534, sum=28084 (114)\", \"tab\": \"General information\", \"score\": \"246.35087719298247\"}", - "MMLU All Subjects - # train": "{\"description\": \"min=5, mean=5, max=5, sum=570 (114)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU All Subjects - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU All Subjects - # prompt tokens": "{\"description\": \"min=313.474, mean=715.682, max=3168.636, sum=81587.749 (114)\", \"tab\": \"General information\", \"score\": \"715.6820126388612\"}", - "MMLU All Subjects - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=114 (114)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_name": "Abstract Algebra", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Abstract Algebra", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.46, - "details": { - "description": "min=0.46, mean=0.46, max=0.46, sum=0.92 (2)", - "tab": "Accuracy", - "Abstract Algebra - Observed inference time (s)": "{\"description\": \"min=0.471, mean=0.471, max=0.471, sum=0.941 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.47064422845840453\"}", - "Abstract Algebra - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Abstract Algebra - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Abstract Algebra - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Abstract Algebra - # prompt tokens": "{\"description\": \"min=406.65, mean=406.65, max=406.65, sum=813.3 (2)\", \"tab\": \"General information\", \"score\": \"406.65\"}", - "Abstract Algebra - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"abstract_algebra\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_abstract_algebra\"" - } - } - }, - { - "evaluation_name": "Anatomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Anatomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.719, - "details": { - "description": "min=0.719, mean=0.719, max=0.719, sum=1.437 (2)", - "tab": "Accuracy", - "Anatomy - Observed inference time (s)": "{\"description\": \"min=0.526, mean=0.526, max=0.526, sum=1.052 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5261570206394902\"}", - "Anatomy - # eval": "{\"description\": \"min=135, mean=135, max=135, sum=270 (2)\", \"tab\": \"General information\", \"score\": \"135.0\"}", - "Anatomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Anatomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Anatomy - # prompt tokens": "{\"description\": \"min=418.133, mean=418.133, max=418.133, sum=836.267 (2)\", \"tab\": \"General information\", \"score\": \"418.1333333333333\"}", - "Anatomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"anatomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_anatomy\"" - } - } - }, - { - "evaluation_name": "College Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on College Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.559, - "details": { - "description": "min=0.559, mean=0.559, max=0.559, sum=1.118 (2)", - "tab": "Accuracy", - "College Chemistry - Observed inference time (s)": "{\"description\": \"min=0.538, mean=0.538, max=0.538, sum=1.077 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5384537291526794\"}", - "College Biology - Observed inference time (s)": "{\"description\": \"min=0.443, mean=0.443, max=0.443, sum=0.886 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.44289560781584847\"}", - "College Computer Science - Observed inference time (s)": "{\"description\": \"min=0.436, mean=0.436, max=0.436, sum=0.872 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4359678840637207\"}", - "College Mathematics - Observed inference time (s)": "{\"description\": \"min=0.432, mean=0.432, max=0.432, sum=0.865 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4324680757522583\"}", - "College Medicine - Observed inference time (s)": "{\"description\": \"min=0.435, mean=0.435, max=0.435, sum=0.869 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4347288250234086\"}", - "College Physics - Observed inference time (s)": "{\"description\": \"min=0.432, mean=0.432, max=0.432, sum=0.863 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.43169068121442605\"}", - "College Chemistry - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Chemistry - # prompt tokens": "{\"description\": \"min=631.43, mean=631.43, max=631.43, sum=1262.86 (2)\", \"tab\": \"General information\", \"score\": \"631.43\"}", - "College Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Biology - # eval": "{\"description\": \"min=144, mean=144, max=144, sum=288 (2)\", \"tab\": \"General information\", \"score\": \"144.0\"}", - "College Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # prompt tokens": "{\"description\": \"min=562.632, mean=562.632, max=562.632, sum=1125.264 (2)\", \"tab\": \"General information\", \"score\": \"562.6319444444445\"}", - "College Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # prompt tokens": "{\"description\": \"min=910.14, mean=910.14, max=910.14, sum=1820.28 (2)\", \"tab\": \"General information\", \"score\": \"910.14\"}", - "College Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Mathematics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # prompt tokens": "{\"description\": \"min=655.96, mean=655.96, max=655.96, sum=1311.92 (2)\", \"tab\": \"General information\", \"score\": \"655.96\"}", - "College Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Medicine - # eval": "{\"description\": \"min=173, mean=173, max=173, sum=346 (2)\", \"tab\": \"General information\", \"score\": \"173.0\"}", - "College Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # prompt tokens": "{\"description\": \"min=617.671, mean=617.671, max=617.671, sum=1235.341 (2)\", \"tab\": \"General information\", \"score\": \"617.6705202312139\"}", - "College Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Physics - # eval": "{\"description\": \"min=102, mean=102, max=102, sum=204 (2)\", \"tab\": \"General information\", \"score\": \"102.0\"}", - "College Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # prompt tokens": "{\"description\": \"min=560.873, mean=560.873, max=560.873, sum=1121.745 (2)\", \"tab\": \"General information\", \"score\": \"560.8725490196078\"}", - "College Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"college_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_college_physics\"" - } - } - }, - { - "evaluation_name": "Computer Security", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Computer Security", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.82, - "details": { - "description": "min=0.82, mean=0.82, max=0.82, sum=1.64 (2)", - "tab": "Accuracy", - "Computer Security - Observed inference time (s)": "{\"description\": \"min=0.765, mean=0.765, max=0.765, sum=1.53 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7652230095863343\"}", - "Computer Security - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Computer Security - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Computer Security - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Computer Security - # prompt tokens": "{\"description\": \"min=437.17, mean=437.17, max=437.17, sum=874.34 (2)\", \"tab\": \"General information\", \"score\": \"437.17\"}", - "Computer Security - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"computer_security\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_computer_security\"" - } - } - }, - { - "evaluation_name": "Econometrics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Econometrics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.605, - "details": { - "description": "min=0.605, mean=0.605, max=0.605, sum=1.211 (2)", - "tab": "Accuracy", - "Econometrics - Observed inference time (s)": "{\"description\": \"min=0.429, mean=0.429, max=0.429, sum=0.858 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4288227077116046\"}", - "Econometrics - # eval": "{\"description\": \"min=114, mean=114, max=114, sum=228 (2)\", \"tab\": \"General information\", \"score\": \"114.0\"}", - "Econometrics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Econometrics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Econometrics - # prompt tokens": "{\"description\": \"min=693.675, mean=693.675, max=693.675, sum=1387.351 (2)\", \"tab\": \"General information\", \"score\": \"693.6754385964912\"}", - "Econometrics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"econometrics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_econometrics\"" - } - } - }, - { - "evaluation_name": "Global Facts", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Global Facts", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5, - "details": { - "description": "min=0.5, mean=0.5, max=0.5, sum=1 (2)", - "tab": "Accuracy", - "Global Facts - Observed inference time (s)": "{\"description\": \"min=0.43, mean=0.43, max=0.43, sum=0.859 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4296323895454407\"}", - "Global Facts - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Global Facts - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Global Facts - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Global Facts - # prompt tokens": "{\"description\": \"min=493.54, mean=493.54, max=493.54, sum=987.08 (2)\", \"tab\": \"General information\", \"score\": \"493.54\"}", - "Global Facts - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"global_facts\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_global_facts\"" - } - } - }, - { - "evaluation_name": "Jurisprudence", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Jurisprudence", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.898, - "details": { - "description": "min=0.898, mean=0.898, max=0.898, sum=1.796 (2)", - "tab": "Accuracy", - "Jurisprudence - Observed inference time (s)": "{\"description\": \"min=0.421, mean=0.421, max=0.421, sum=0.841 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4206738162923742\"}", - "Jurisprudence - # eval": "{\"description\": \"min=108, mean=108, max=108, sum=216 (2)\", \"tab\": \"General information\", \"score\": \"108.0\"}", - "Jurisprudence - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Jurisprudence - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Jurisprudence - # prompt tokens": "{\"description\": \"min=458.898, mean=458.898, max=458.898, sum=917.796 (2)\", \"tab\": \"General information\", \"score\": \"458.89814814814815\"}", - "Jurisprudence - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"jurisprudence\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_jurisprudence\"" - } - } - }, - { - "evaluation_name": "Philosophy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Philosophy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.817, - "details": { - "description": "min=0.817, mean=0.817, max=0.817, sum=1.633 (2)", - "tab": "Accuracy", - "Philosophy - Observed inference time (s)": "{\"description\": \"min=0.436, mean=0.436, max=0.436, sum=0.871 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.43559602372516004\"}", - "Philosophy - # eval": "{\"description\": \"min=311, mean=311, max=311, sum=622 (2)\", \"tab\": \"General information\", \"score\": \"311.0\"}", - "Philosophy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Philosophy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Philosophy - # prompt tokens": "{\"description\": \"min=381.122, mean=381.122, max=381.122, sum=762.244 (2)\", \"tab\": \"General information\", \"score\": \"381.12218649517683\"}", - "Philosophy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"philosophy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_philosophy\"" - } - } - }, - { - "evaluation_name": "Professional Psychology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Professional Psychology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.85, - "details": { - "description": "min=0.85, mean=0.85, max=0.85, sum=1.699 (2)", - "tab": "Accuracy", - "Professional Medicine - Observed inference time (s)": "{\"description\": \"min=0.497, mean=0.497, max=0.497, sum=0.994 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4968351388678831\"}", - "Professional Accounting - Observed inference time (s)": "{\"description\": \"min=0.459, mean=0.459, max=0.459, sum=0.917 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4586718564337872\"}", - "Professional Law - Observed inference time (s)": "{\"description\": \"min=1.016, mean=1.016, max=1.016, sum=2.033 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.016288014092377\"}", - "Professional Psychology - Observed inference time (s)": "{\"description\": \"min=0.443, mean=0.443, max=0.443, sum=0.885 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4426119109384375\"}", - "Professional Medicine - # eval": "{\"description\": \"min=272, mean=272, max=272, sum=544 (2)\", \"tab\": \"General information\", \"score\": \"272.0\"}", - "Professional Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Medicine - # prompt tokens": "{\"description\": \"min=1339.647, mean=1339.647, max=1339.647, sum=2679.294 (2)\", \"tab\": \"General information\", \"score\": \"1339.6470588235295\"}", - "Professional Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Accounting - # eval": "{\"description\": \"min=282, mean=282, max=282, sum=564 (2)\", \"tab\": \"General information\", \"score\": \"282.0\"}", - "Professional Accounting - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Accounting - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # prompt tokens": "{\"description\": \"min=832.277, mean=832.277, max=832.277, sum=1664.553 (2)\", \"tab\": \"General information\", \"score\": \"832.2765957446809\"}", - "Professional Accounting - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Law - # eval": "{\"description\": \"min=1534, mean=1534, max=1534, sum=3068 (2)\", \"tab\": \"General information\", \"score\": \"1534.0\"}", - "Professional Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # prompt tokens": "{\"description\": \"min=1924.007, mean=1924.007, max=1924.007, sum=3848.014 (2)\", \"tab\": \"General information\", \"score\": \"1924.0071707953064\"}", - "Professional Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Psychology - # eval": "{\"description\": \"min=612, mean=612, max=612, sum=1224 (2)\", \"tab\": \"General information\", \"score\": \"612.0\"}", - "Professional Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # prompt tokens": "{\"description\": \"min=659.078, mean=659.078, max=659.078, sum=1318.157 (2)\", \"tab\": \"General information\", \"score\": \"659.0784313725491\"}", - "Professional Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"professional_psychology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_professional_psychology\"" - } - } - }, - { - "evaluation_name": "Us Foreign Policy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Us Foreign Policy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.97, - "details": { - "description": "min=0.97, mean=0.97, max=0.97, sum=1.94 (2)", - "tab": "Accuracy", - "Us Foreign Policy - Observed inference time (s)": "{\"description\": \"min=0.441, mean=0.441, max=0.441, sum=0.882 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.44084484577178956\"}", - "Us Foreign Policy - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Us Foreign Policy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Us Foreign Policy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Us Foreign Policy - # prompt tokens": "{\"description\": \"min=488.81, mean=488.81, max=488.81, sum=977.62 (2)\", \"tab\": \"General information\", \"score\": \"488.81\"}", - "Us Foreign Policy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"us_foreign_policy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_us_foreign_policy\"" - } - } - }, - { - "evaluation_name": "Astronomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Astronomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.868, - "details": { - "description": "min=0.868, mean=0.868, max=0.868, sum=1.737 (2)", - "tab": "Accuracy", - "Astronomy - Observed inference time (s)": "{\"description\": \"min=0.446, mean=0.446, max=0.446, sum=0.892 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4461362079570168\"}", - "Astronomy - # eval": "{\"description\": \"min=152, mean=152, max=152, sum=304 (2)\", \"tab\": \"General information\", \"score\": \"152.0\"}", - "Astronomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Astronomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Astronomy - # prompt tokens": "{\"description\": \"min=690.079, mean=690.079, max=690.079, sum=1380.158 (2)\", \"tab\": \"General information\", \"score\": \"690.078947368421\"}", - "Astronomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"astronomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_astronomy\"" - } - } - }, - { - "evaluation_name": "Business Ethics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Business Ethics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8, - "details": { - "description": "min=0.8, mean=0.8, max=0.8, sum=1.6 (2)", - "tab": "Accuracy", - "Business Ethics - Observed inference time (s)": "{\"description\": \"min=0.436, mean=0.436, max=0.436, sum=0.872 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4362391257286072\"}", - "Business Ethics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Business Ethics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Business Ethics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Business Ethics - # prompt tokens": "{\"description\": \"min=683.44, mean=683.44, max=683.44, sum=1366.88 (2)\", \"tab\": \"General information\", \"score\": \"683.44\"}", - "Business Ethics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"business_ethics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_business_ethics\"" - } - } - }, - { - "evaluation_name": "Clinical Knowledge", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Clinical Knowledge", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.808, - "details": { - "description": "min=0.808, mean=0.808, max=0.808, sum=1.615 (2)", - "tab": "Accuracy", - "Clinical Knowledge - Observed inference time (s)": "{\"description\": \"min=0.427, mean=0.427, max=0.427, sum=0.855 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.42739290561316146\"}", - "Clinical Knowledge - # eval": "{\"description\": \"min=265, mean=265, max=265, sum=530 (2)\", \"tab\": \"General information\", \"score\": \"265.0\"}", - "Clinical Knowledge - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Clinical Knowledge - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Clinical Knowledge - # prompt tokens": "{\"description\": \"min=496.374, mean=496.374, max=496.374, sum=992.747 (2)\", \"tab\": \"General information\", \"score\": \"496.3735849056604\"}", - "Clinical Knowledge - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"clinical_knowledge\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_clinical_knowledge\"" - } - } - }, - { - "evaluation_name": "Conceptual Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Conceptual Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.826, - "details": { - "description": "min=0.826, mean=0.826, max=0.826, sum=1.651 (2)", - "tab": "Accuracy", - "Conceptual Physics - Observed inference time (s)": "{\"description\": \"min=0.452, mean=0.452, max=0.452, sum=0.904 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4520118307560048\"}", - "Conceptual Physics - # eval": "{\"description\": \"min=235, mean=235, max=235, sum=470 (2)\", \"tab\": \"General information\", \"score\": \"235.0\"}", - "Conceptual Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Conceptual Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Conceptual Physics - # prompt tokens": "{\"description\": \"min=342.153, mean=342.153, max=342.153, sum=684.306 (2)\", \"tab\": \"General information\", \"score\": \"342.1531914893617\"}", - "Conceptual Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"conceptual_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_conceptual_physics\"" - } - } - }, - { - "evaluation_name": "Electrical Engineering", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Electrical Engineering", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.697, - "details": { - "description": "min=0.697, mean=0.697, max=0.697, sum=1.393 (2)", - "tab": "Accuracy", - "Electrical Engineering - Observed inference time (s)": "{\"description\": \"min=0.487, mean=0.487, max=0.487, sum=0.974 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4870024582435345\"}", - "Electrical Engineering - # eval": "{\"description\": \"min=145, mean=145, max=145, sum=290 (2)\", \"tab\": \"General information\", \"score\": \"145.0\"}", - "Electrical Engineering - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Electrical Engineering - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Electrical Engineering - # prompt tokens": "{\"description\": \"min=506.779, mean=506.779, max=506.779, sum=1013.559 (2)\", \"tab\": \"General information\", \"score\": \"506.7793103448276\"}", - "Electrical Engineering - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"electrical_engineering\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_electrical_engineering\"" - } - } - }, - { - "evaluation_name": "Elementary Mathematics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Elementary Mathematics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.611, - "details": { - "description": "min=0.611, mean=0.611, max=0.611, sum=1.222 (2)", - "tab": "Accuracy", - "Elementary Mathematics - Observed inference time (s)": "{\"description\": \"min=0.457, mean=0.457, max=0.457, sum=0.915 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4574742739793485\"}", - "Elementary Mathematics - # eval": "{\"description\": \"min=378, mean=378, max=378, sum=756 (2)\", \"tab\": \"General information\", \"score\": \"378.0\"}", - "Elementary Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Elementary Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Elementary Mathematics - # prompt tokens": "{\"description\": \"min=618.156, mean=618.156, max=618.156, sum=1236.312 (2)\", \"tab\": \"General information\", \"score\": \"618.1560846560847\"}", - "Elementary Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"elementary_mathematics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_elementary_mathematics\"" - } - } - }, - { - "evaluation_name": "Formal Logic", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Formal Logic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.579, - "details": { - "description": "min=0.579, mean=0.579, max=0.579, sum=1.159 (2)", - "tab": "Accuracy", - "Formal Logic - Observed inference time (s)": "{\"description\": \"min=0.445, mean=0.445, max=0.445, sum=0.889 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.44462628780849395\"}", - "Formal Logic - # eval": "{\"description\": \"min=126, mean=126, max=126, sum=252 (2)\", \"tab\": \"General information\", \"score\": \"126.0\"}", - "Formal Logic - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Formal Logic - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Formal Logic - # prompt tokens": "{\"description\": \"min=700.81, mean=700.81, max=700.81, sum=1401.619 (2)\", \"tab\": \"General information\", \"score\": \"700.8095238095239\"}", - "Formal Logic - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"formal_logic\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_formal_logic\"" - } - } - }, - { - "evaluation_name": "High School World History", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on High School World History", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.907, - "details": { - "description": "min=0.907, mean=0.907, max=0.907, sum=1.814 (2)", - "tab": "Accuracy", - "High School Biology - Observed inference time (s)": "{\"description\": \"min=0.44, mean=0.44, max=0.44, sum=0.879 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4396143251849759\"}", - "High School Chemistry - Observed inference time (s)": "{\"description\": \"min=0.433, mean=0.433, max=0.433, sum=0.865 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4325766810055437\"}", - "High School Computer Science - Observed inference time (s)": "{\"description\": \"min=0.444, mean=0.444, max=0.444, sum=0.887 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4435269355773926\"}", - "High School European History - Observed inference time (s)": "{\"description\": \"min=1.447, mean=1.447, max=1.447, sum=2.894 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.44696401682767\"}", - "High School Geography - Observed inference time (s)": "{\"description\": \"min=0.427, mean=0.427, max=0.427, sum=0.854 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4269573845044531\"}", - "High School Government And Politics - Observed inference time (s)": "{\"description\": \"min=0.442, mean=0.442, max=0.442, sum=0.885 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4422582035855308\"}", - "High School Macroeconomics - Observed inference time (s)": "{\"description\": \"min=0.423, mean=0.423, max=0.423, sum=0.846 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4230540263347137\"}", - "High School Mathematics - Observed inference time (s)": "{\"description\": \"min=0.438, mean=0.438, max=0.438, sum=0.877 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4383223215738932\"}", - "High School Microeconomics - Observed inference time (s)": "{\"description\": \"min=0.443, mean=0.443, max=0.443, sum=0.887 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4434382264353648\"}", - "High School Physics - Observed inference time (s)": "{\"description\": \"min=0.448, mean=0.448, max=0.448, sum=0.896 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4479467000392889\"}", - "High School Psychology - Observed inference time (s)": "{\"description\": \"min=0.438, mean=0.438, max=0.438, sum=0.876 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.43786543006197026\"}", - "High School Statistics - Observed inference time (s)": "{\"description\": \"min=0.454, mean=0.454, max=0.454, sum=0.907 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.45358082431334035\"}", - "High School US History - Observed inference time (s)": "{\"description\": \"min=1.14, mean=1.14, max=1.14, sum=2.28 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.13988286373662\"}", - "High School World History - Observed inference time (s)": "{\"description\": \"min=0.833, mean=0.833, max=0.833, sum=1.666 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8329467803617067\"}", - "High School Biology - # eval": "{\"description\": \"min=310, mean=310, max=310, sum=620 (2)\", \"tab\": \"General information\", \"score\": \"310.0\"}", - "High School Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Biology - # prompt tokens": "{\"description\": \"min=605.894, mean=605.894, max=605.894, sum=1211.787 (2)\", \"tab\": \"General information\", \"score\": \"605.8935483870968\"}", - "High School Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Chemistry - # eval": "{\"description\": \"min=203, mean=203, max=203, sum=406 (2)\", \"tab\": \"General information\", \"score\": \"203.0\"}", - "High School Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # prompt tokens": "{\"description\": \"min=577.665, mean=577.665, max=577.665, sum=1155.33 (2)\", \"tab\": \"General information\", \"score\": \"577.6650246305419\"}", - "High School Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "High School Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # prompt tokens": "{\"description\": \"min=997.57, mean=997.57, max=997.57, sum=1995.14 (2)\", \"tab\": \"General information\", \"score\": \"997.57\"}", - "High School Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School European History - # eval": "{\"description\": \"min=165, mean=165, max=165, sum=330 (2)\", \"tab\": \"General information\", \"score\": \"165.0\"}", - "High School European History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School European History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # prompt tokens": "{\"description\": \"min=3168.636, mean=3168.636, max=3168.636, sum=6337.273 (2)\", \"tab\": \"General information\", \"score\": \"3168.6363636363635\"}", - "High School European History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Geography - # eval": "{\"description\": \"min=198, mean=198, max=198, sum=396 (2)\", \"tab\": \"General information\", \"score\": \"198.0\"}", - "High School Geography - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Geography - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # prompt tokens": "{\"description\": \"min=445.657, mean=445.657, max=445.657, sum=891.313 (2)\", \"tab\": \"General information\", \"score\": \"445.65656565656565\"}", - "High School Geography - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Government And Politics - # eval": "{\"description\": \"min=193, mean=193, max=193, sum=386 (2)\", \"tab\": \"General information\", \"score\": \"193.0\"}", - "High School Government And Politics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Government And Politics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # prompt tokens": "{\"description\": \"min=536.927, mean=536.927, max=536.927, sum=1073.855 (2)\", \"tab\": \"General information\", \"score\": \"536.9274611398964\"}", - "High School Government And Politics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Macroeconomics - # eval": "{\"description\": \"min=390, mean=390, max=390, sum=780 (2)\", \"tab\": \"General information\", \"score\": \"390.0\"}", - "High School Macroeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Macroeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # prompt tokens": "{\"description\": \"min=454.662, mean=454.662, max=454.662, sum=909.323 (2)\", \"tab\": \"General information\", \"score\": \"454.66153846153844\"}", - "High School Macroeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Mathematics - # eval": "{\"description\": \"min=270, mean=270, max=270, sum=540 (2)\", \"tab\": \"General information\", \"score\": \"270.0\"}", - "High School Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # prompt tokens": "{\"description\": \"min=588.181, mean=588.181, max=588.181, sum=1176.363 (2)\", \"tab\": \"General information\", \"score\": \"588.1814814814815\"}", - "High School Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Microeconomics - # eval": "{\"description\": \"min=238, mean=238, max=238, sum=476 (2)\", \"tab\": \"General information\", \"score\": \"238.0\"}", - "High School Microeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Microeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # prompt tokens": "{\"description\": \"min=458.492, mean=458.492, max=458.492, sum=916.983 (2)\", \"tab\": \"General information\", \"score\": \"458.49159663865544\"}", - "High School Microeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Physics - # eval": "{\"description\": \"min=151, mean=151, max=151, sum=302 (2)\", \"tab\": \"General information\", \"score\": \"151.0\"}", - "High School Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # prompt tokens": "{\"description\": \"min=630.788, mean=630.788, max=630.788, sum=1261.576 (2)\", \"tab\": \"General information\", \"score\": \"630.7880794701987\"}", - "High School Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Psychology - # eval": "{\"description\": \"min=545, mean=545, max=545, sum=1090 (2)\", \"tab\": \"General information\", \"score\": \"545.0\"}", - "High School Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # prompt tokens": "{\"description\": \"min=594.919, mean=594.919, max=594.919, sum=1189.839 (2)\", \"tab\": \"General information\", \"score\": \"594.9192660550459\"}", - "High School Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Statistics - # eval": "{\"description\": \"min=216, mean=216, max=216, sum=432 (2)\", \"tab\": \"General information\", \"score\": \"216.0\"}", - "High School Statistics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Statistics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # prompt tokens": "{\"description\": \"min=917.208, mean=917.208, max=917.208, sum=1834.417 (2)\", \"tab\": \"General information\", \"score\": \"917.2083333333334\"}", - "High School Statistics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School US History - # eval": "{\"description\": \"min=204, mean=204, max=204, sum=408 (2)\", \"tab\": \"General information\", \"score\": \"204.0\"}", - "High School US History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School US History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # prompt tokens": "{\"description\": \"min=2544.324, mean=2544.324, max=2544.324, sum=5088.647 (2)\", \"tab\": \"General information\", \"score\": \"2544.323529411765\"}", - "High School US History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School World History - # eval": "{\"description\": \"min=237, mean=237, max=237, sum=474 (2)\", \"tab\": \"General information\", \"score\": \"237.0\"}", - "High School World History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School World History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # prompt tokens": "{\"description\": \"min=1647.219, mean=1647.219, max=1647.219, sum=3294.439 (2)\", \"tab\": \"General information\", \"score\": \"1647.2194092827003\"}", - "High School World History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"high_school_world_history\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_high_school_world_history\"" - } - } - }, - { - "evaluation_name": "Human Sexuality", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Human Sexuality", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.847, - "details": { - "description": "min=0.847, mean=0.847, max=0.847, sum=1.695 (2)", - "tab": "Accuracy", - "Human Aging - Observed inference time (s)": "{\"description\": \"min=0.436, mean=0.436, max=0.436, sum=0.873 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.43635595539760164\"}", - "Human Sexuality - Observed inference time (s)": "{\"description\": \"min=0.434, mean=0.434, max=0.434, sum=0.869 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4343654235810724\"}", - "Human Aging - # eval": "{\"description\": \"min=223, mean=223, max=223, sum=446 (2)\", \"tab\": \"General information\", \"score\": \"223.0\"}", - "Human Aging - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Aging - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Aging - # prompt tokens": "{\"description\": \"min=370.26, mean=370.26, max=370.26, sum=740.52 (2)\", \"tab\": \"General information\", \"score\": \"370.26008968609864\"}", - "Human Aging - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Human Sexuality - # eval": "{\"description\": \"min=131, mean=131, max=131, sum=262 (2)\", \"tab\": \"General information\", \"score\": \"131.0\"}", - "Human Sexuality - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Sexuality - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # prompt tokens": "{\"description\": \"min=412.382, mean=412.382, max=412.382, sum=824.763 (2)\", \"tab\": \"General information\", \"score\": \"412.381679389313\"}", - "Human Sexuality - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"human_sexuality\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_human_sexuality\"" - } - } - }, - { - "evaluation_name": "International Law", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on International Law", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.901, - "details": { - "description": "min=0.901, mean=0.901, max=0.901, sum=1.802 (2)", - "tab": "Accuracy", - "International Law - Observed inference time (s)": "{\"description\": \"min=0.461, mean=0.461, max=0.461, sum=0.922 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.46112686346385107\"}", - "International Law - # eval": "{\"description\": \"min=121, mean=121, max=121, sum=242 (2)\", \"tab\": \"General information\", \"score\": \"121.0\"}", - "International Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "International Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "International Law - # prompt tokens": "{\"description\": \"min=738.463, mean=738.463, max=738.463, sum=1476.926 (2)\", \"tab\": \"General information\", \"score\": \"738.4628099173553\"}", - "International Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"international_law\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_international_law\"" - } - } - }, - { - "evaluation_name": "Logical Fallacies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Logical Fallacies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.865, - "details": { - "description": "min=0.865, mean=0.865, max=0.865, sum=1.73 (2)", - "tab": "Accuracy", - "Logical Fallacies - Observed inference time (s)": "{\"description\": \"min=0.45, mean=0.45, max=0.45, sum=0.9 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.44979269080366824\"}", - "Logical Fallacies - # eval": "{\"description\": \"min=163, mean=163, max=163, sum=326 (2)\", \"tab\": \"General information\", \"score\": \"163.0\"}", - "Logical Fallacies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Logical Fallacies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Logical Fallacies - # prompt tokens": "{\"description\": \"min=511.755, mean=511.755, max=511.755, sum=1023.509 (2)\", \"tab\": \"General information\", \"score\": \"511.7546012269939\"}", - "Logical Fallacies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"logical_fallacies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_logical_fallacies\"" - } - } - }, - { - "evaluation_name": "Machine Learning", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Machine Learning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.616, - "details": { - "description": "min=0.616, mean=0.616, max=0.616, sum=1.232 (2)", - "tab": "Accuracy", - "Machine Learning - Observed inference time (s)": "{\"description\": \"min=0.466, mean=0.466, max=0.466, sum=0.932 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.46596066866602215\"}", - "Machine Learning - # eval": "{\"description\": \"min=112, mean=112, max=112, sum=224 (2)\", \"tab\": \"General information\", \"score\": \"112.0\"}", - "Machine Learning - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Machine Learning - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Machine Learning - # prompt tokens": "{\"description\": \"min=739.402, mean=739.402, max=739.402, sum=1478.804 (2)\", \"tab\": \"General information\", \"score\": \"739.4017857142857\"}", - "Machine Learning - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"machine_learning\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_machine_learning\"" - } - } - }, - { - "evaluation_name": "Management", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Management", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.864, - "details": { - "description": "min=0.864, mean=0.864, max=0.864, sum=1.728 (2)", - "tab": "Accuracy", - "Management - Observed inference time (s)": "{\"description\": \"min=0.439, mean=0.439, max=0.439, sum=0.878 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.43890966720951413\"}", - "Management - # eval": "{\"description\": \"min=103, mean=103, max=103, sum=206 (2)\", \"tab\": \"General information\", \"score\": \"103.0\"}", - "Management - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Management - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Management - # prompt tokens": "{\"description\": \"min=324.777, mean=324.777, max=324.777, sum=649.553 (2)\", \"tab\": \"General information\", \"score\": \"324.77669902912623\"}", - "Management - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"management\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_management\"" - } - } - }, - { - "evaluation_name": "Marketing", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Marketing", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.953, - "details": { - "description": "min=0.953, mean=0.953, max=0.953, sum=1.906 (2)", - "tab": "Accuracy", - "Marketing - Observed inference time (s)": "{\"description\": \"min=0.443, mean=0.443, max=0.443, sum=0.885 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4425381727707692\"}", - "Marketing - # eval": "{\"description\": \"min=234, mean=234, max=234, sum=468 (2)\", \"tab\": \"General information\", \"score\": \"234.0\"}", - "Marketing - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Marketing - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Marketing - # prompt tokens": "{\"description\": \"min=481.628, mean=481.628, max=481.628, sum=963.256 (2)\", \"tab\": \"General information\", \"score\": \"481.62820512820514\"}", - "Marketing - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"marketing\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_marketing\"" - } - } - }, - { - "evaluation_name": "Medical Genetics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Medical Genetics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.91, - "details": { - "description": "min=0.91, mean=0.91, max=0.91, sum=1.82 (2)", - "tab": "Accuracy", - "Medical Genetics - Observed inference time (s)": "{\"description\": \"min=0.436, mean=0.436, max=0.436, sum=0.872 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.43624018907546996\"}", - "Medical Genetics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Medical Genetics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Medical Genetics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Medical Genetics - # prompt tokens": "{\"description\": \"min=417.14, mean=417.14, max=417.14, sum=834.28 (2)\", \"tab\": \"General information\", \"score\": \"417.14\"}", - "Medical Genetics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"medical_genetics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_medical_genetics\"" - } - } - }, - { - "evaluation_name": "Miscellaneous", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Miscellaneous", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.888, - "details": { - "description": "min=0.888, mean=0.888, max=0.888, sum=1.775 (2)", - "tab": "Accuracy", - "Miscellaneous - Observed inference time (s)": "{\"description\": \"min=0.434, mean=0.434, max=0.434, sum=0.868 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4337884417591119\"}", - "Miscellaneous - # eval": "{\"description\": \"min=783, mean=783, max=783, sum=1566 (2)\", \"tab\": \"General information\", \"score\": \"783.0\"}", - "Miscellaneous - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Miscellaneous - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Miscellaneous - # prompt tokens": "{\"description\": \"min=354.913, mean=354.913, max=354.913, sum=709.826 (2)\", \"tab\": \"General information\", \"score\": \"354.9131545338442\"}", - "Miscellaneous - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"miscellaneous\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_miscellaneous\"" - } - } - }, - { - "evaluation_name": "Moral Scenarios", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Moral Scenarios", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.811, - "details": { - "description": "min=0.811, mean=0.811, max=0.811, sum=1.622 (2)", - "tab": "Accuracy", - "Moral Disputes - Observed inference time (s)": "{\"description\": \"min=0.454, mean=0.454, max=0.454, sum=0.908 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4541343209371401\"}", - "Moral Scenarios - Observed inference time (s)": "{\"description\": \"min=0.452, mean=0.452, max=0.452, sum=0.905 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4522555020934377\"}", - "Moral Disputes - # eval": "{\"description\": \"min=346, mean=346, max=346, sum=692 (2)\", \"tab\": \"General information\", \"score\": \"346.0\"}", - "Moral Disputes - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Disputes - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Disputes - # prompt tokens": "{\"description\": \"min=551.506, mean=551.506, max=551.506, sum=1103.012 (2)\", \"tab\": \"General information\", \"score\": \"551.5057803468208\"}", - "Moral Disputes - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Moral Scenarios - # eval": "{\"description\": \"min=895, mean=895, max=895, sum=1790 (2)\", \"tab\": \"General information\", \"score\": \"895.0\"}", - "Moral Scenarios - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Scenarios - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # prompt tokens": "{\"description\": \"min=765.479, mean=765.479, max=765.479, sum=1530.959 (2)\", \"tab\": \"General information\", \"score\": \"765.4793296089385\"}", - "Moral Scenarios - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"moral_scenarios\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_moral_scenarios\"" - } - } - }, - { - "evaluation_name": "Nutrition", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Nutrition", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.859, - "details": { - "description": "min=0.859, mean=0.859, max=0.859, sum=1.719 (2)", - "tab": "Accuracy", - "Nutrition - Observed inference time (s)": "{\"description\": \"min=0.469, mean=0.469, max=0.469, sum=0.937 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.46850453872306674\"}", - "Nutrition - # eval": "{\"description\": \"min=306, mean=306, max=306, sum=612 (2)\", \"tab\": \"General information\", \"score\": \"306.0\"}", - "Nutrition - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Nutrition - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Nutrition - # prompt tokens": "{\"description\": \"min=704.922, mean=704.922, max=704.922, sum=1409.843 (2)\", \"tab\": \"General information\", \"score\": \"704.9215686274509\"}", - "Nutrition - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"nutrition\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_nutrition\"" - } - } - }, - { - "evaluation_name": "Prehistory", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Prehistory", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.867, - "details": { - "description": "min=0.867, mean=0.867, max=0.867, sum=1.735 (2)", - "tab": "Accuracy", - "Prehistory - Observed inference time (s)": "{\"description\": \"min=0.459, mean=0.459, max=0.459, sum=0.919 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.45942840973536175\"}", - "Prehistory - # eval": "{\"description\": \"min=324, mean=324, max=324, sum=648 (2)\", \"tab\": \"General information\", \"score\": \"324.0\"}", - "Prehistory - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Prehistory - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Prehistory - # prompt tokens": "{\"description\": \"min=628.185, mean=628.185, max=628.185, sum=1256.37 (2)\", \"tab\": \"General information\", \"score\": \"628.1851851851852\"}", - "Prehistory - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"prehistory\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_prehistory\"" - } - } - }, - { - "evaluation_name": "Public Relations", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Public Relations", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.764, - "details": { - "description": "min=0.764, mean=0.764, max=0.764, sum=1.527 (2)", - "tab": "Accuracy", - "Public Relations - Observed inference time (s)": "{\"description\": \"min=0.424, mean=0.424, max=0.424, sum=0.848 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4240685766393488\"}", - "Public Relations - # eval": "{\"description\": \"min=110, mean=110, max=110, sum=220 (2)\", \"tab\": \"General information\", \"score\": \"110.0\"}", - "Public Relations - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Public Relations - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Public Relations - # prompt tokens": "{\"description\": \"min=483.827, mean=483.827, max=483.827, sum=967.655 (2)\", \"tab\": \"General information\", \"score\": \"483.8272727272727\"}", - "Public Relations - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"public_relations\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_public_relations\"" - } - } - }, - { - "evaluation_name": "Security Studies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Security Studies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.82, - "details": { - "description": "min=0.82, mean=0.82, max=0.82, sum=1.641 (2)", - "tab": "Accuracy", - "Security Studies - Observed inference time (s)": "{\"description\": \"min=0.5, mean=0.5, max=0.5, sum=1.001 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.500300864784085\"}", - "Security Studies - # eval": "{\"description\": \"min=245, mean=245, max=245, sum=490 (2)\", \"tab\": \"General information\", \"score\": \"245.0\"}", - "Security Studies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Security Studies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Security Studies - # prompt tokens": "{\"description\": \"min=1386.531, mean=1386.531, max=1386.531, sum=2773.061 (2)\", \"tab\": \"General information\", \"score\": \"1386.530612244898\"}", - "Security Studies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"security_studies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_security_studies\"" - } - } - }, - { - "evaluation_name": "Sociology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Sociology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.886, - "details": { - "description": "min=0.886, mean=0.886, max=0.886, sum=1.771 (2)", - "tab": "Accuracy", - "Sociology - Observed inference time (s)": "{\"description\": \"min=0.44, mean=0.44, max=0.44, sum=0.879 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4395348717324176\"}", - "Sociology - # eval": "{\"description\": \"min=201, mean=201, max=201, sum=402 (2)\", \"tab\": \"General information\", \"score\": \"201.0\"}", - "Sociology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Sociology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Sociology - # prompt tokens": "{\"description\": \"min=517.478, mean=517.478, max=517.478, sum=1034.955 (2)\", \"tab\": \"General information\", \"score\": \"517.4776119402985\"}", - "Sociology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"sociology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_sociology\"" - } - } - }, - { - "evaluation_name": "Virology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Virology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.572, - "details": { - "description": "min=0.572, mean=0.572, max=0.572, sum=1.145 (2)", - "tab": "Accuracy", - "Virology - Observed inference time (s)": "{\"description\": \"min=0.426, mean=0.426, max=0.426, sum=0.852 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4260225296020508\"}", - "Virology - # eval": "{\"description\": \"min=166, mean=166, max=166, sum=332 (2)\", \"tab\": \"General information\", \"score\": \"166.0\"}", - "Virology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Virology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Virology - # prompt tokens": "{\"description\": \"min=414.108, mean=414.108, max=414.108, sum=828.217 (2)\", \"tab\": \"General information\", \"score\": \"414.10843373493975\"}", - "Virology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"virology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_virology\"" - } - } - }, - { - "evaluation_name": "World Religions", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on World Religions", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.883, - "details": { - "description": "min=0.883, mean=0.883, max=0.883, sum=1.766 (2)", - "tab": "Accuracy", - "World Religions - Observed inference time (s)": "{\"description\": \"min=0.415, mean=0.415, max=0.415, sum=0.83 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.41479549212762484\"}", - "World Religions - # eval": "{\"description\": \"min=171, mean=171, max=171, sum=342 (2)\", \"tab\": \"General information\", \"score\": \"171.0\"}", - "World Religions - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "World Religions - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "World Religions - # prompt tokens": "{\"description\": \"min=313.474, mean=313.474, max=313.474, sum=626.947 (2)\", \"tab\": \"General information\", \"score\": \"313.4736842105263\"}", - "World Religions - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"world_religions\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_world_religions\"" - } - } - }, - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.462, - "details": { - "description": "", - "tab": "Efficiency" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/upstage_solar-pro-preview-instruct.json b/data/models/upstage_solar-pro-preview-instruct.json deleted file mode 100644 index 42cdeb326e608afed29c561b843f7cfd34f4442b..0000000000000000000000000000000000000000 --- a/data/models/upstage_solar-pro-preview-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "solar-pro-preview-instruct", - "id": "upstage/solar-pro-preview-instruct", - "developer": "upstage", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "SolarForCausalLM", - "params_billions": "22.14" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/upstage_solar-pro-preview-instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8416 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6817 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2205 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3708 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4417 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5273 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/utkmst_chimera-beta-test2-lora-merged.json b/data/models/utkmst_chimera-beta-test2-lora-merged.json deleted file mode 100644 index f4ad4153edc6325da70665aeb5cea8e1f0e160c1..0000000000000000000000000000000000000000 --- a/data/models/utkmst_chimera-beta-test2-lora-merged.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "chimera-beta-test2-lora-merged", - "id": "utkmst/chimera-beta-test2-lora-merged", - "developer": "utkmst", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/utkmst_chimera-beta-test2-lora-merged/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6054 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4796 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0952 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3037 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4118 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2992 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/utter-project_eurollm-9b.json b/data/models/utter-project_eurollm-9b.json deleted file mode 100644 index 0c29dc58cf38083c6f8a63ab8be09d489f94ca96..0000000000000000000000000000000000000000 --- a/data/models/utter-project_eurollm-9b.json +++ /dev/null @@ -1,49 +0,0 @@ -{ - "model_info": { - "name": "EuroLLM 9B", - "id": "utter-project/EuroLLM-9B", - "developer": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "la_leaderboard/utter-project/EuroLLM-9B/1774451270", - "retrieved_timestamp": "2024-10-27T00:00:00Z", - "source_metadata": { - "source_name": "La Leaderboard", - "source_type": "evaluation_run", - "source_url": "https://huggingface.co/spaces/la-leaderboard/la-leaderboard", - "source_organization_name": "La Leaderboard", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "custom", - "version": "1.0" - }, - "benchmark": "la_leaderboard", - "evaluation_results": [ - { - "evaluation_name": "la_leaderboard", - "metric_config": { - "evaluation_description": "La Leaderboard: LLM evaluation for Spanish varieties and languages of Spain and Latin America", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 100 - }, - "score_details": { - "score": 25.87 - }, - "source_data": { - "source_type": "url", - "dataset_name": "La Leaderboard composite dataset", - "url": [ - "https://huggingface.co/spaces/la-leaderboard/la-leaderboard" - ] - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/uukuguy_speechless-code-mistral-7b-v1.0.json b/data/models/uukuguy_speechless-code-mistral-7b-v1.0.json deleted file mode 100644 index e4d0e8092846641ae00a7ecf68a35ee8786d7eaa..0000000000000000000000000000000000000000 --- a/data/models/uukuguy_speechless-code-mistral-7b-v1.0.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "speechless-code-mistral-7b-v1.0", - "id": "uukuguy/speechless-code-mistral-7b-v1.0", - "developer": "uukuguy", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/uukuguy_speechless-code-mistral-7b-v1.0/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3665 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4572 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0521 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2844 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4502 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3146 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/uukuguy_speechless-codellama-34b-v2.0.json b/data/models/uukuguy_speechless-codellama-34b-v2.0.json deleted file mode 100644 index b1430d0cc775bd90d49de038ffc66c0c23485f07..0000000000000000000000000000000000000000 --- a/data/models/uukuguy_speechless-codellama-34b-v2.0.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "speechless-codellama-34b-v2.0", - "id": "uukuguy/speechless-codellama-34b-v2.0", - "developer": "uukuguy", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "34.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/uukuguy_speechless-codellama-34b-v2.0/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4604 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4813 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0431 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2693 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3787 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2542 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/uukuguy_speechless-coder-ds-6.7b.json b/data/models/uukuguy_speechless-coder-ds-6.7b.json deleted file mode 100644 index ff68a5a5ae2abbee6ebc61d09f40986d8263125d..0000000000000000000000000000000000000000 --- a/data/models/uukuguy_speechless-coder-ds-6.7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "speechless-coder-ds-6.7b", - "id": "uukuguy/speechless-coder-ds-6.7b", - "developer": "uukuguy", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "6.7" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/uukuguy_speechless-coder-ds-6.7b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2505 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4036 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0211 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2643 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3819 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1719 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/uukuguy_speechless-instruct-mistral-7b-v0.2.json b/data/models/uukuguy_speechless-instruct-mistral-7b-v0.2.json deleted file mode 100644 index c0ef95b7121c3e953e633ed5f724f0ca50682447..0000000000000000000000000000000000000000 --- a/data/models/uukuguy_speechless-instruct-mistral-7b-v0.2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "speechless-instruct-mistral-7b-v0.2", - "id": "uukuguy/speechless-instruct-mistral-7b-v0.2", - "developer": "uukuguy", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/uukuguy_speechless-instruct-mistral-7b-v0.2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3261 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4607 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0491 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2819 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4902 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2902 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/uukuguy_speechless-llama2-hermes-orca-platypus-wizardlm-13b.json b/data/models/uukuguy_speechless-llama2-hermes-orca-platypus-wizardlm-13b.json deleted file mode 100644 index 054e7100a56ef32e94a07ad5d85a1b681b8ba22c..0000000000000000000000000000000000000000 --- a/data/models/uukuguy_speechless-llama2-hermes-orca-platypus-wizardlm-13b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "speechless-llama2-hermes-orca-platypus-wizardlm-13b", - "id": "uukuguy/speechless-llama2-hermes-orca-platypus-wizardlm-13b", - "developer": "uukuguy", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "13.016" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/uukuguy_speechless-llama2-hermes-orca-platypus-wizardlm-13b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4562 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4846 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0204 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2701 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4655 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2559 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/uukuguy_speechless-mistral-dolphin-orca-platypus-samantha-7b.json b/data/models/uukuguy_speechless-mistral-dolphin-orca-platypus-samantha-7b.json deleted file mode 100644 index 13e485db7584898b142555a2f687bb4d576f5b5c..0000000000000000000000000000000000000000 --- a/data/models/uukuguy_speechless-mistral-dolphin-orca-platypus-samantha-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "speechless-mistral-dolphin-orca-platypus-samantha-7b", - "id": "uukuguy/speechless-mistral-dolphin-orca-platypus-samantha-7b", - "developer": "uukuguy", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/uukuguy_speechless-mistral-dolphin-orca-platypus-samantha-7b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.37 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4983 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0295 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2836 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4361 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.299 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/uukuguy_speechless-zephyr-code-functionary-7b.json b/data/models/uukuguy_speechless-zephyr-code-functionary-7b.json deleted file mode 100644 index cbcee01230f271adae7aa36181e8ae77a8e68763..0000000000000000000000000000000000000000 --- a/data/models/uukuguy_speechless-zephyr-code-functionary-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "speechless-zephyr-code-functionary-7b", - "id": "uukuguy/speechless-zephyr-code-functionary-7b", - "developer": "uukuguy", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/uukuguy_speechless-zephyr-code-functionary-7b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2696 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4664 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0423 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3003 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4268 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3094 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/v000000_l3-8b-stheno-v3.2-abliterated.json b/data/models/v000000_l3-8b-stheno-v3.2-abliterated.json deleted file mode 100644 index 3212ce37a0a757b571f9608786980cb88de11f36..0000000000000000000000000000000000000000 --- a/data/models/v000000_l3-8b-stheno-v3.2-abliterated.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "L3-8B-Stheno-v3.2-abliterated", - "id": "v000000/L3-8B-Stheno-v3.2-abliterated", - "developer": "v000000", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/v000000_L3-8B-Stheno-v3.2-abliterated/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6718 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5141 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0695 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3096 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.362 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3604 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/v000000_l3.1-niitorm-8b-dpo-t0.0001.json b/data/models/v000000_l3.1-niitorm-8b-dpo-t0.0001.json deleted file mode 100644 index ef1f9c077a54003a5932ecc348a82032871a407c..0000000000000000000000000000000000000000 --- a/data/models/v000000_l3.1-niitorm-8b-dpo-t0.0001.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "L3.1-Niitorm-8B-DPO-t0.0001", - "id": "v000000/L3.1-Niitorm-8B-DPO-t0.0001", - "developer": "v000000", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/v000000_L3.1-Niitorm-8B-DPO-t0.0001/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7689 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5134 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1624 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2945 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.388 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3866 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/v000000_l3.1-storniitova-8b.json b/data/models/v000000_l3.1-storniitova-8b.json deleted file mode 100644 index 9c4cd53273011e6534ba8ddcdf8e551c9af1c3c9..0000000000000000000000000000000000000000 --- a/data/models/v000000_l3.1-storniitova-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "L3.1-Storniitova-8B", - "id": "v000000/L3.1-Storniitova-8B", - "developer": "v000000", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/v000000_L3.1-Storniitova-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7817 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5151 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1465 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2894 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4029 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3776 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/v000000_qwen2.5-14b-gutenberg-1e-delta.json b/data/models/v000000_qwen2.5-14b-gutenberg-1e-delta.json deleted file mode 100644 index 864609dde1bf534c1d640b339b76d3106f23d797..0000000000000000000000000000000000000000 --- a/data/models/v000000_qwen2.5-14b-gutenberg-1e-delta.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-14B-Gutenberg-1e-Delta", - "id": "v000000/Qwen2.5-14B-Gutenberg-1e-Delta", - "developer": "v000000", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/v000000_Qwen2.5-14B-Gutenberg-1e-Delta/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8045 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6398 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5264 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3289 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4073 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.493 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/v000000_qwen2.5-14b-gutenberg-instruct-slerpeno.json b/data/models/v000000_qwen2.5-14b-gutenberg-instruct-slerpeno.json deleted file mode 100644 index 4fb2bf852429fe6effbf6127756f296d5815e3f2..0000000000000000000000000000000000000000 --- a/data/models/v000000_qwen2.5-14b-gutenberg-instruct-slerpeno.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-14B-Gutenberg-Instruct-Slerpeno", - "id": "v000000/Qwen2.5-14B-Gutenberg-Instruct-Slerpeno", - "developer": "v000000", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/v000000_Qwen2.5-14B-Gutenberg-Instruct-Slerpeno/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8197 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.639 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5325 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3314 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4114 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4924 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/v000000_qwen2.5-lumen-14b.json b/data/models/v000000_qwen2.5-lumen-14b.json deleted file mode 100644 index b7f7eee0eed26007cc85f9cbee520d207bdd2dd9..0000000000000000000000000000000000000000 --- a/data/models/v000000_qwen2.5-lumen-14b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-Lumen-14B", - "id": "v000000/Qwen2.5-Lumen-14B", - "developer": "v000000", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/v000000_Qwen2.5-Lumen-14B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8064 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6391 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5363 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.328 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4114 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4903 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/v3n0m_jenna-tiny-2.0.json b/data/models/v3n0m_jenna-tiny-2.0.json deleted file mode 100644 index ede36b16450c91326f5e53ef17013d6b2ddc7348..0000000000000000000000000000000000000000 --- a/data/models/v3n0m_jenna-tiny-2.0.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Jenna-Tiny-2.0", - "id": "V3N0M/Jenna-Tiny-2.0", - "developer": "V3N0M", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "0.631" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/V3N0M_Jenna-Tiny-2.0/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2309 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3148 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0121 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.25 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3367 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1147 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/vagosolutions_llama-3-sauerkrautlm-70b-instruct.json b/data/models/vagosolutions_llama-3-sauerkrautlm-70b-instruct.json deleted file mode 100644 index ea420d36279da752bf197985baddb4d0bb936c13..0000000000000000000000000000000000000000 --- a/data/models/vagosolutions_llama-3-sauerkrautlm-70b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-SauerkrautLM-70b-Instruct", - "id": "VAGOsolutions/Llama-3-SauerkrautLM-70b-Instruct", - "developer": "VAGOsolutions", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "70.554" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/VAGOsolutions_Llama-3-SauerkrautLM-70b-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8045 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6663 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2281 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.328 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4339 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5392 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/vagosolutions_llama-3-sauerkrautlm-8b-instruct.json b/data/models/vagosolutions_llama-3-sauerkrautlm-8b-instruct.json deleted file mode 100644 index 38a8bea28604557bb88b040c4d4e8878457a6409..0000000000000000000000000000000000000000 --- a/data/models/vagosolutions_llama-3-sauerkrautlm-8b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-SauerkrautLM-8b-Instruct", - "id": "VAGOsolutions/Llama-3-SauerkrautLM-8b-Instruct", - "developer": "VAGOsolutions", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/VAGOsolutions_Llama-3-SauerkrautLM-8b-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7445 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4943 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0665 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3087 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4241 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3857 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/vagosolutions_llama-3.1-sauerkrautlm-70b-instruct.json b/data/models/vagosolutions_llama-3.1-sauerkrautlm-70b-instruct.json deleted file mode 100644 index 9ad514858aa13301cae896a2847f4cbe06be6927..0000000000000000000000000000000000000000 --- a/data/models/vagosolutions_llama-3.1-sauerkrautlm-70b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.1-SauerkrautLM-70b-Instruct", - "id": "VAGOsolutions/Llama-3.1-SauerkrautLM-70b-Instruct", - "developer": "VAGOsolutions", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "70.554" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/VAGOsolutions_Llama-3.1-SauerkrautLM-70b-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8656 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7006 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3693 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3414 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4711 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5335 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/vagosolutions_llama-3.1-sauerkrautlm-8b-instruct.json b/data/models/vagosolutions_llama-3.1-sauerkrautlm-8b-instruct.json deleted file mode 100644 index 852c116ac585f765c1b65579502ad92052d004c4..0000000000000000000000000000000000000000 --- a/data/models/vagosolutions_llama-3.1-sauerkrautlm-8b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.1-SauerkrautLM-8b-Instruct", - "id": "VAGOsolutions/Llama-3.1-SauerkrautLM-8b-Instruct", - "developer": "VAGOsolutions", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/VAGOsolutions_Llama-3.1-SauerkrautLM-8b-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8017 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5115 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1941 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2903 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4148 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.389 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/vagosolutions_sauerkrautlm-1.5b.json b/data/models/vagosolutions_sauerkrautlm-1.5b.json deleted file mode 100644 index 37f6b46193a156c8a787485bf750944d77a9f972..0000000000000000000000000000000000000000 --- a/data/models/vagosolutions_sauerkrautlm-1.5b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SauerkrautLM-1.5b", - "id": "VAGOsolutions/SauerkrautLM-1.5b", - "developer": "VAGOsolutions", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.544" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/VAGOsolutions_SauerkrautLM-1.5b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2404 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3704 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0363 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.271 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3739 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2151 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/vagosolutions_sauerkrautlm-7b-hero.json b/data/models/vagosolutions_sauerkrautlm-7b-hero.json deleted file mode 100644 index c3dc0ade031bf04b30b194c18a46009e7b3a8f3e..0000000000000000000000000000000000000000 --- a/data/models/vagosolutions_sauerkrautlm-7b-hero.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SauerkrautLM-7b-HerO", - "id": "VAGOsolutions/SauerkrautLM-7b-HerO", - "developer": "VAGOsolutions", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/VAGOsolutions_SauerkrautLM-7b-HerO/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5346 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4904 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0393 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2727 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3924 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3046 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/vagosolutions_sauerkrautlm-7b-laserchat.json b/data/models/vagosolutions_sauerkrautlm-7b-laserchat.json deleted file mode 100644 index 530bc0dbabb5b5e7bd7aa78d1e067ae4610ae59e..0000000000000000000000000000000000000000 --- a/data/models/vagosolutions_sauerkrautlm-7b-laserchat.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SauerkrautLM-7b-LaserChat", - "id": "VAGOsolutions/SauerkrautLM-7b-LaserChat", - "developer": "VAGOsolutions", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/VAGOsolutions_SauerkrautLM-7b-LaserChat/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5988 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4543 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0778 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3003 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4148 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3305 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/vagosolutions_sauerkrautlm-gemma-2-2b-it.json b/data/models/vagosolutions_sauerkrautlm-gemma-2-2b-it.json deleted file mode 100644 index 2fa6d5410769ce10e3a92e85425df92cd8b7cecb..0000000000000000000000000000000000000000 --- a/data/models/vagosolutions_sauerkrautlm-gemma-2-2b-it.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SauerkrautLM-gemma-2-2b-it", - "id": "VAGOsolutions/SauerkrautLM-gemma-2-2b-it", - "developer": "VAGOsolutions", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "2.614" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/VAGOsolutions_SauerkrautLM-gemma-2-2b-it/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1321 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4241 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0219 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2727 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3995 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2693 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/vagosolutions_sauerkrautlm-gemma-2-9b-it.json b/data/models/vagosolutions_sauerkrautlm-gemma-2-9b-it.json deleted file mode 100644 index 2194115669adc2f64fdc5a92deb910df578bbc85..0000000000000000000000000000000000000000 --- a/data/models/vagosolutions_sauerkrautlm-gemma-2-9b-it.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SauerkrautLM-gemma-2-9b-it", - "id": "VAGOsolutions/SauerkrautLM-gemma-2-9b-it", - "developer": "VAGOsolutions", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "9.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/VAGOsolutions_SauerkrautLM-gemma-2-9b-it/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3024 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6073 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0838 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3272 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4318 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4091 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/vagosolutions_sauerkrautlm-gemma-2b.json b/data/models/vagosolutions_sauerkrautlm-gemma-2b.json deleted file mode 100644 index f754aa7bfccf040e54d3b4cfb2be226297eb703e..0000000000000000000000000000000000000000 --- a/data/models/vagosolutions_sauerkrautlm-gemma-2b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SauerkrautLM-Gemma-2b", - "id": "VAGOsolutions/SauerkrautLM-Gemma-2b", - "developer": "VAGOsolutions", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "GemmaForCausalLM", - "params_billions": "2.506" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/VAGOsolutions_SauerkrautLM-Gemma-2b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2475 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3416 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0279 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2567 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3676 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1469 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/vagosolutions_sauerkrautlm-gemma-7b.json b/data/models/vagosolutions_sauerkrautlm-gemma-7b.json deleted file mode 100644 index 6dd2a3a8c6c8fa38fe84d20b0f1a347deaa9d468..0000000000000000000000000000000000000000 --- a/data/models/vagosolutions_sauerkrautlm-gemma-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SauerkrautLM-Gemma-7b", - "id": "VAGOsolutions/SauerkrautLM-Gemma-7b", - "developer": "VAGOsolutions", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "GemmaForCausalLM", - "params_billions": "8.538" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/VAGOsolutions_SauerkrautLM-Gemma-7b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3407 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4188 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0672 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2861 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3594 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2961 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/vagosolutions_sauerkrautlm-mixtral-8x7b-instruct.json b/data/models/vagosolutions_sauerkrautlm-mixtral-8x7b-instruct.json deleted file mode 100644 index ec6329186994bf72b6ca47a3f4fa3d4ac3f57f47..0000000000000000000000000000000000000000 --- a/data/models/vagosolutions_sauerkrautlm-mixtral-8x7b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SauerkrautLM-Mixtral-8x7B-Instruct", - "id": "VAGOsolutions/SauerkrautLM-Mixtral-8x7B-Instruct", - "developer": "VAGOsolutions", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MixtralForCausalLM", - "params_billions": "46.703" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/VAGOsolutions_SauerkrautLM-Mixtral-8x7B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5602 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5277 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0982 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2978 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4204 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.365 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/vagosolutions_sauerkrautlm-nemo-12b-instruct.json b/data/models/vagosolutions_sauerkrautlm-nemo-12b-instruct.json deleted file mode 100644 index a5f8e34beeebd8991f5457bce9b16db2847b8859..0000000000000000000000000000000000000000 --- a/data/models/vagosolutions_sauerkrautlm-nemo-12b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SauerkrautLM-Nemo-12b-Instruct", - "id": "VAGOsolutions/SauerkrautLM-Nemo-12b-Instruct", - "developer": "VAGOsolutions", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/VAGOsolutions_SauerkrautLM-Nemo-12b-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6113 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5214 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1224 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3096 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4469 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3385 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/vagosolutions_sauerkrautlm-phi-3-medium.json b/data/models/vagosolutions_sauerkrautlm-phi-3-medium.json deleted file mode 100644 index 282affd1ad357b010dd8478ca2ddea51733df313..0000000000000000000000000000000000000000 --- a/data/models/vagosolutions_sauerkrautlm-phi-3-medium.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SauerkrautLM-Phi-3-medium", - "id": "VAGOsolutions/SauerkrautLM-Phi-3-medium", - "developer": "VAGOsolutions", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "13.96" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/VAGOsolutions_SauerkrautLM-Phi-3-medium/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4409 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6433 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1601 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3347 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4845 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4665 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/vagosolutions_sauerkrautlm-solar-instruct.json b/data/models/vagosolutions_sauerkrautlm-solar-instruct.json deleted file mode 100644 index 2d9ba43c339ab75c77d51f20539039572f20bb8d..0000000000000000000000000000000000000000 --- a/data/models/vagosolutions_sauerkrautlm-solar-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SauerkrautLM-SOLAR-Instruct", - "id": "VAGOsolutions/SauerkrautLM-SOLAR-Instruct", - "developer": "VAGOsolutions", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "10.732" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/VAGOsolutions_SauerkrautLM-SOLAR-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4917 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5169 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0634 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3054 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3965 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3183 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/vagosolutions_sauerkrautlm-v2-14b-dpo.json b/data/models/vagosolutions_sauerkrautlm-v2-14b-dpo.json deleted file mode 100644 index 2f94c016d14cf26ab51af4374e29e1f06f452bf6..0000000000000000000000000000000000000000 --- a/data/models/vagosolutions_sauerkrautlm-v2-14b-dpo.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SauerkrautLM-v2-14b-DPO", - "id": "VAGOsolutions/SauerkrautLM-v2-14b-DPO", - "developer": "VAGOsolutions", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/VAGOsolutions_SauerkrautLM-v2-14b-DPO/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7412 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.656 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3165 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3196 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4375 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5117 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/vagosolutions_sauerkrautlm-v2-14b-sft.json b/data/models/vagosolutions_sauerkrautlm-v2-14b-sft.json deleted file mode 100644 index d76e7504a8cd7ef9d6c42403753aade4547b5a48..0000000000000000000000000000000000000000 --- a/data/models/vagosolutions_sauerkrautlm-v2-14b-sft.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SauerkrautLM-v2-14b-SFT", - "id": "VAGOsolutions/SauerkrautLM-v2-14b-SFT", - "developer": "VAGOsolutions", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/VAGOsolutions_SauerkrautLM-v2-14b-SFT/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6949 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.621 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3285 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3356 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4179 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5205 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/valiantlabs_llama3-70b-fireplace.json b/data/models/valiantlabs_llama3-70b-fireplace.json deleted file mode 100644 index 5a490af6f9858c257194c3559a84aea42b34b1c2..0000000000000000000000000000000000000000 --- a/data/models/valiantlabs_llama3-70b-fireplace.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama3-70B-Fireplace", - "id": "ValiantLabs/Llama3-70B-Fireplace", - "developer": "ValiantLabs", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "70.554" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ValiantLabs_Llama3-70B-Fireplace/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7774 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6489 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2145 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3549 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4449 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4893 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/valiantlabs_llama3-70b-shiningvaliant2.json b/data/models/valiantlabs_llama3-70b-shiningvaliant2.json deleted file mode 100644 index 794362f9355fc91c70a2aba65d82d437d5c84a2e..0000000000000000000000000000000000000000 --- a/data/models/valiantlabs_llama3-70b-shiningvaliant2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama3-70B-ShiningValiant2", - "id": "ValiantLabs/Llama3-70B-ShiningValiant2", - "developer": "ValiantLabs", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "70.554" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ValiantLabs_Llama3-70B-ShiningValiant2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6122 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6338 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2077 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3305 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4326 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4898 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/valiantlabs_llama3.1-70b-shiningvaliant2.json b/data/models/valiantlabs_llama3.1-70b-shiningvaliant2.json deleted file mode 100644 index d53fabac0827adbd050ecce2931b2d9ed061c0cb..0000000000000000000000000000000000000000 --- a/data/models/valiantlabs_llama3.1-70b-shiningvaliant2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama3.1-70B-ShiningValiant2", - "id": "ValiantLabs/Llama3.1-70B-ShiningValiant2", - "developer": "ValiantLabs", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "70.554" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ValiantLabs_Llama3.1-70B-ShiningValiant2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5355 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6738 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2915 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3926 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4681 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5173 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/valiantlabs_llama3.1-8b-cobalt.json b/data/models/valiantlabs_llama3.1-8b-cobalt.json deleted file mode 100644 index e1d1a85c95f8deab607165ea40b9f93f0dba9c6d..0000000000000000000000000000000000000000 --- a/data/models/valiantlabs_llama3.1-8b-cobalt.json +++ /dev/null @@ -1,275 +0,0 @@ -{ - "model_info": { - "name": "Llama3.1-8B-Cobalt", - "id": "ValiantLabs/Llama3.1-8B-Cobalt", - "developer": "ValiantLabs", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ValiantLabs_Llama3.1-8B-Cobalt/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7168 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4911 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1533 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2861 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3512 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3663 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "hfopenllm_v2/ValiantLabs_Llama3.1-8B-Cobalt/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3496 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4947 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1269 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3037 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3959 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3644 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/valiantlabs_llama3.1-8b-enigma.json b/data/models/valiantlabs_llama3.1-8b-enigma.json deleted file mode 100644 index f296c27007177757fff95ab56d68e939b628ac54..0000000000000000000000000000000000000000 --- a/data/models/valiantlabs_llama3.1-8b-enigma.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama3.1-8B-Enigma", - "id": "ValiantLabs/Llama3.1-8B-Enigma", - "developer": "ValiantLabs", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ValiantLabs_Llama3.1-8B-Enigma/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2681 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4478 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0891 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2878 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4196 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3409 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/valiantlabs_llama3.1-8b-esper2.json b/data/models/valiantlabs_llama3.1-8b-esper2.json deleted file mode 100644 index ffdf939ef83858b34a2239586703e224d089ecc1..0000000000000000000000000000000000000000 --- a/data/models/valiantlabs_llama3.1-8b-esper2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama3.1-8B-Esper2", - "id": "ValiantLabs/Llama3.1-8B-Esper2", - "developer": "ValiantLabs", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ValiantLabs_Llama3.1-8B-Esper2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2567 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.447 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0589 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2727 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3561 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2904 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/valiantlabs_llama3.1-8b-fireplace2.json b/data/models/valiantlabs_llama3.1-8b-fireplace2.json deleted file mode 100644 index 0a050b1b1736e6133f5ceae5b758cb4a60844c7f..0000000000000000000000000000000000000000 --- a/data/models/valiantlabs_llama3.1-8b-fireplace2.json +++ /dev/null @@ -1,275 +0,0 @@ -{ - "model_info": { - "name": "Llama3.1-8B-Fireplace2", - "id": "ValiantLabs/Llama3.1-8B-Fireplace2", - "developer": "ValiantLabs", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ValiantLabs_Llama3.1-8B-Fireplace2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5483 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.461 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0582 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2886 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3433 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2407 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "hfopenllm_v2/ValiantLabs_Llama3.1-8B-Fireplace2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5328 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4613 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0876 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2894 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3367 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2424 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/valiantlabs_llama3.1-8b-shiningvaliant2.json b/data/models/valiantlabs_llama3.1-8b-shiningvaliant2.json deleted file mode 100644 index f3e37b204fa779a9e21a0521a813b464c3fe641b..0000000000000000000000000000000000000000 --- a/data/models/valiantlabs_llama3.1-8b-shiningvaliant2.json +++ /dev/null @@ -1,275 +0,0 @@ -{ - "model_info": { - "name": "Llama3.1-8B-ShiningValiant2", - "id": "ValiantLabs/Llama3.1-8B-ShiningValiant2", - "developer": "ValiantLabs", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ValiantLabs_Llama3.1-8B-ShiningValiant2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6496 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4774 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0566 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3104 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3909 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3382 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "hfopenllm_v2/ValiantLabs_Llama3.1-8B-ShiningValiant2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2678 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4429 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0521 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.302 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3959 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2927 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/valiantlabs_llama3.2-3b-enigma.json b/data/models/valiantlabs_llama3.2-3b-enigma.json deleted file mode 100644 index d5a3e4384dd6ad60d40e558b98404db16aed5bd6..0000000000000000000000000000000000000000 --- a/data/models/valiantlabs_llama3.2-3b-enigma.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama3.2-3B-Enigma", - "id": "ValiantLabs/Llama3.2-3B-Enigma", - "developer": "ValiantLabs", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ValiantLabs_Llama3.2-3B-Enigma/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2786 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3723 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0438 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2617 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3921 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2428 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/valiantlabs_llama3.2-3b-esper2.json b/data/models/valiantlabs_llama3.2-3b-esper2.json deleted file mode 100644 index 98b88a59a1cfc2e0e24e1a8bd7a0e8d4d5c7aeac..0000000000000000000000000000000000000000 --- a/data/models/valiantlabs_llama3.2-3b-esper2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama3.2-3B-Esper2", - "id": "ValiantLabs/Llama3.2-3B-Esper2", - "developer": "ValiantLabs", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ValiantLabs_Llama3.2-3B-Esper2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.275 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3808 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0363 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2701 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.355 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2257 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/valiantlabs_llama3.2-3b-shiningvaliant2.json b/data/models/valiantlabs_llama3.2-3b-shiningvaliant2.json deleted file mode 100644 index 7b17e9799efeca68c3827304f0fd47c844d0b67e..0000000000000000000000000000000000000000 --- a/data/models/valiantlabs_llama3.2-3b-shiningvaliant2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama3.2-3B-ShiningValiant2", - "id": "ValiantLabs/Llama3.2-3B-ShiningValiant2", - "developer": "ValiantLabs", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ValiantLabs_Llama3.2-3B-ShiningValiant2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2625 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4226 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0823 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2802 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3866 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2829 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/vhab10_llama-3-8b-merged-linear.json b/data/models/vhab10_llama-3-8b-merged-linear.json deleted file mode 100644 index 680b5b003fae7487c000f0c916f2ab47e9c86b87..0000000000000000000000000000000000000000 --- a/data/models/vhab10_llama-3-8b-merged-linear.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "llama-3-8b-merged-linear", - "id": "vhab10/llama-3-8b-merged-linear", - "developer": "vhab10", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "4.65" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/vhab10_llama-3-8b-merged-linear/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5917 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4937 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0816 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2995 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4191 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3704 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/vhab10_llama-3.1-8b-base-instruct-slerp.json b/data/models/vhab10_llama-3.1-8b-base-instruct-slerp.json deleted file mode 100644 index edb1be61628fdb4365b24529c997d9d932b6c5d9..0000000000000000000000000000000000000000 --- a/data/models/vhab10_llama-3.1-8b-base-instruct-slerp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.1-8B-Base-Instruct-SLERP", - "id": "vhab10/Llama-3.1-8B-Base-Instruct-SLERP", - "developer": "vhab10", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/vhab10_Llama-3.1-8B-Base-Instruct-SLERP/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2907 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5057 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1201 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2961 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4011 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3621 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/vhab10_llama-3.2-instruct-3b-ties.json b/data/models/vhab10_llama-3.2-instruct-3b-ties.json deleted file mode 100644 index 5d5c5a6a03f37cd30ad7ba43f49028b18d4e530c..0000000000000000000000000000000000000000 --- a/data/models/vhab10_llama-3.2-instruct-3b-ties.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.2-Instruct-3B-TIES", - "id": "vhab10/Llama-3.2-Instruct-3B-TIES", - "developer": "vhab10", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.848" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/vhab10_Llama-3.2-Instruct-3B-TIES/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4727 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4332 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0982 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2693 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3497 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2916 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/vicgalle_carbonbeagle-11b-truthy.json b/data/models/vicgalle_carbonbeagle-11b-truthy.json deleted file mode 100644 index e979ae27bd3542d331423547c5b84cd53349858d..0000000000000000000000000000000000000000 --- a/data/models/vicgalle_carbonbeagle-11b-truthy.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "CarbonBeagle-11B-truthy", - "id": "vicgalle/CarbonBeagle-11B-truthy", - "developer": "vicgalle", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "10.732" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/vicgalle_CarbonBeagle-11B-truthy/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5212 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5348 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0491 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2995 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.374 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3357 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/vicgalle_carbonbeagle-11b.json b/data/models/vicgalle_carbonbeagle-11b.json deleted file mode 100644 index 6c3a0b2f4be72cf7982ecbf16ff75ef25ff13055..0000000000000000000000000000000000000000 --- a/data/models/vicgalle_carbonbeagle-11b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "CarbonBeagle-11B", - "id": "vicgalle/CarbonBeagle-11B", - "developer": "vicgalle", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "10.732" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/vicgalle_CarbonBeagle-11B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5415 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5294 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0619 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.302 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.402 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3276 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/vicgalle_configurable-hermes-2-pro-llama-3-8b.json b/data/models/vicgalle_configurable-hermes-2-pro-llama-3-8b.json deleted file mode 100644 index f53c354a5ef3f54b4c36314c65c5a25e1b75816d..0000000000000000000000000000000000000000 --- a/data/models/vicgalle_configurable-hermes-2-pro-llama-3-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Configurable-Hermes-2-Pro-Llama-3-8B", - "id": "vicgalle/Configurable-Hermes-2-Pro-Llama-3-8B", - "developer": "vicgalle", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.031" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/vicgalle_Configurable-Hermes-2-Pro-Llama-3-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5763 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5055 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0763 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.297 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4184 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3098 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/vicgalle_configurable-llama-3.1-8b-instruct.json b/data/models/vicgalle_configurable-llama-3.1-8b-instruct.json deleted file mode 100644 index 50cea8f7a305756fdafca8294434d1a096f5b62c..0000000000000000000000000000000000000000 --- a/data/models/vicgalle_configurable-llama-3.1-8b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Configurable-Llama-3.1-8B-Instruct", - "id": "vicgalle/Configurable-Llama-3.1-8B-Instruct", - "developer": "vicgalle", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/vicgalle_Configurable-Llama-3.1-8B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8312 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5045 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.173 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2743 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3845 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3592 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/vicgalle_configurable-yi-1.5-9b-chat.json b/data/models/vicgalle_configurable-yi-1.5-9b-chat.json deleted file mode 100644 index d72a66128c308060afeb7467fa8e511b744682d3..0000000000000000000000000000000000000000 --- a/data/models/vicgalle_configurable-yi-1.5-9b-chat.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Configurable-Yi-1.5-9B-Chat", - "id": "vicgalle/Configurable-Yi-1.5-9B-Chat", - "developer": "vicgalle", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.829" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/vicgalle_Configurable-Yi-1.5-9B-Chat/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4323 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5452 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2047 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3431 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4271 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4015 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/vicgalle_configurablebeagle-11b.json b/data/models/vicgalle_configurablebeagle-11b.json deleted file mode 100644 index deef3b98642802375052bd25a0574d8210f26c17..0000000000000000000000000000000000000000 --- a/data/models/vicgalle_configurablebeagle-11b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ConfigurableBeagle-11B", - "id": "vicgalle/ConfigurableBeagle-11B", - "developer": "vicgalle", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "10.732" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/vicgalle_ConfigurableBeagle-11B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5834 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5287 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0431 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.302 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3953 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3374 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/vicgalle_configurablehermes-7b.json b/data/models/vicgalle_configurablehermes-7b.json deleted file mode 100644 index 818d51b156196c32d06333a0f697726a78538747..0000000000000000000000000000000000000000 --- a/data/models/vicgalle_configurablehermes-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ConfigurableHermes-7B", - "id": "vicgalle/ConfigurableHermes-7B", - "developer": "vicgalle", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/vicgalle_ConfigurableHermes-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5411 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4573 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0476 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2768 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4057 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3025 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/vicgalle_configurablesolar-10.7b.json b/data/models/vicgalle_configurablesolar-10.7b.json deleted file mode 100644 index 78ae521afc5887d3368ec332c0bc55c0b0ea1cc1..0000000000000000000000000000000000000000 --- a/data/models/vicgalle_configurablesolar-10.7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ConfigurableSOLAR-10.7B", - "id": "vicgalle/ConfigurableSOLAR-10.7B", - "developer": "vicgalle", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "10.732" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/vicgalle_ConfigurableSOLAR-10.7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.51 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4867 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0665 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2987 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3805 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3173 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/vicgalle_humanish-rp-llama-3.1-8b.json b/data/models/vicgalle_humanish-rp-llama-3.1-8b.json deleted file mode 100644 index d87238fbafafbda81fefbb3ce84e785e59735fd8..0000000000000000000000000000000000000000 --- a/data/models/vicgalle_humanish-rp-llama-3.1-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Humanish-RP-Llama-3.1-8B", - "id": "vicgalle/Humanish-RP-Llama-3.1-8B", - "developer": "vicgalle", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/vicgalle_Humanish-RP-Llama-3.1-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6669 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.51 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1518 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2869 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3952 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3477 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/vicgalle_merge-mistral-prometheus-7b.json b/data/models/vicgalle_merge-mistral-prometheus-7b.json deleted file mode 100644 index 3068aedca59c6155ada8ba9d9fb066152b1554ec..0000000000000000000000000000000000000000 --- a/data/models/vicgalle_merge-mistral-prometheus-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Merge-Mistral-Prometheus-7B", - "id": "vicgalle/Merge-Mistral-Prometheus-7B", - "developer": "vicgalle", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/vicgalle_Merge-Mistral-Prometheus-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4848 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4201 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0181 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2634 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.41 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2717 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/vicgalle_merge-mixtral-prometheus-8x7b.json b/data/models/vicgalle_merge-mixtral-prometheus-8x7b.json deleted file mode 100644 index d7c424be519a9a4fb5a914c9feaf5736197f6478..0000000000000000000000000000000000000000 --- a/data/models/vicgalle_merge-mixtral-prometheus-8x7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Merge-Mixtral-Prometheus-8x7B", - "id": "vicgalle/Merge-Mixtral-Prometheus-8x7B", - "developer": "vicgalle", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MixtralForCausalLM", - "params_billions": "46.703" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/vicgalle_Merge-Mixtral-Prometheus-8x7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5744 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5351 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0929 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3087 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4098 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3684 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/vicgalle_roleplay-llama-3-8b.json b/data/models/vicgalle_roleplay-llama-3-8b.json deleted file mode 100644 index 669b11b683a6b410fb6a159854efa03bce1a29af..0000000000000000000000000000000000000000 --- a/data/models/vicgalle_roleplay-llama-3-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Roleplay-Llama-3-8B", - "id": "vicgalle/Roleplay-Llama-3-8B", - "developer": "vicgalle", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/vicgalle_Roleplay-Llama-3-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.732 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5012 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0914 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2609 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3529 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3708 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/viettelsecurity-ai_security-llama3.2-3b.json b/data/models/viettelsecurity-ai_security-llama3.2-3b.json deleted file mode 100644 index ad529bd78c70b1ea19446a112c445d265d45ddf0..0000000000000000000000000000000000000000 --- a/data/models/viettelsecurity-ai_security-llama3.2-3b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "security-llama3.2-3b", - "id": "viettelsecurity-ai/security-llama3.2-3b", - "developer": "viettelsecurity-ai", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/viettelsecurity-ai_security-llama3.2-3b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5909 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4401 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1261 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2743 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3379 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2837 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/vihangd_smart-dan-sft-v0.1.json b/data/models/vihangd_smart-dan-sft-v0.1.json deleted file mode 100644 index 72758c19224b468cfa33ba5ab21adf142924a11d..0000000000000000000000000000000000000000 --- a/data/models/vihangd_smart-dan-sft-v0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "smart-dan-sft-v0.1", - "id": "vihangd/smart-dan-sft-v0.1", - "developer": "vihangd", - "inference_platform": "unknown", - "additional_details": { - "precision": "4bit", - "architecture": "LlamaForCausalLM", - "params_billions": "0.379" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/vihangd_smart-dan-sft-v0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1576 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3062 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0098 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.255 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3502 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1142 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/vikhrmodels_vikhr-llama3.1-8b-instruct-r-21-09-24.json b/data/models/vikhrmodels_vikhr-llama3.1-8b-instruct-r-21-09-24.json deleted file mode 100644 index e48eb1f124ef2bebd61df0775f882ffe556b5a89..0000000000000000000000000000000000000000 --- a/data/models/vikhrmodels_vikhr-llama3.1-8b-instruct-r-21-09-24.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Vikhr-Llama3.1-8B-Instruct-R-21-09-24", - "id": "Vikhrmodels/Vikhr-Llama3.1-8B-Instruct-R-21-09-24", - "developer": "Vikhrmodels", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Vikhrmodels_Vikhr-Llama3.1-8B-Instruct-R-21-09-24/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6431 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5272 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2175 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.245 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3754 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3547 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/vikhrmodels_vikhr-nemo-12b-instruct-r-21-09-24.json b/data/models/vikhrmodels_vikhr-nemo-12b-instruct-r-21-09-24.json deleted file mode 100644 index 81bfb6a68fcb6ac59ae1e562513949c09e0381fb..0000000000000000000000000000000000000000 --- a/data/models/vikhrmodels_vikhr-nemo-12b-instruct-r-21-09-24.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Vikhr-Nemo-12B-Instruct-R-21-09-24", - "id": "Vikhrmodels/Vikhr-Nemo-12B-Instruct-R-21-09-24", - "developer": "Vikhrmodels", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Vikhrmodels_Vikhr-Nemo-12B-Instruct-R-21-09-24/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5999 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5212 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1715 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2911 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4073 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3398 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/virnect_llama-3-korean-8b-r-v-0.1.json b/data/models/virnect_llama-3-korean-8b-r-v-0.1.json deleted file mode 100644 index 7a87050bf074b45b9ca0ffadef38d8e16d93b422..0000000000000000000000000000000000000000 --- a/data/models/virnect_llama-3-korean-8b-r-v-0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "llama-3-Korean-8B-r-v-0.1", - "id": "VIRNECT/llama-3-Korean-8B-r-v-0.1", - "developer": "VIRNECT", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "?", - "params_billions": "16.061" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/VIRNECT_llama-3-Korean-8B-r-v-0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4916 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4806 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0861 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2424 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3675 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.326 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/virnect_llama-3-korean-8b.json b/data/models/virnect_llama-3-korean-8b.json deleted file mode 100644 index 10825c433e16c0218b8189ab67e3befd38673ff7..0000000000000000000000000000000000000000 --- a/data/models/virnect_llama-3-korean-8b.json +++ /dev/null @@ -1,275 +0,0 @@ -{ - "model_info": { - "name": "llama-3-Korean-8B", - "id": "VIRNECT/llama-3-Korean-8B", - "developer": "VIRNECT", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/VIRNECT_llama-3-Korean-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5021 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4918 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.108 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.271 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3648 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3536 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "hfopenllm_v2/VIRNECT_llama-3-Korean-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5058 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4908 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0929 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.271 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3662 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3539 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/voidful_smol-360m-ft.json b/data/models/voidful_smol-360m-ft.json deleted file mode 100644 index 655779201dc7060a1e484bbf2a732f4a35e2003e..0000000000000000000000000000000000000000 --- a/data/models/voidful_smol-360m-ft.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "smol-360m-ft", - "id": "voidful/smol-360m-ft", - "developer": "voidful", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "0.362" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/voidful_smol-360m-ft/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2013 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3012 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0083 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2458 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3714 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1087 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/vonjack_mobilellm-125m-hf.json b/data/models/vonjack_mobilellm-125m-hf.json deleted file mode 100644 index 8096c8194a1d2a7c9c8b118e0427a797c51a9427..0000000000000000000000000000000000000000 --- a/data/models/vonjack_mobilellm-125m-hf.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MobileLLM-125M-HF", - "id": "vonjack/MobileLLM-125M-HF", - "developer": "vonjack", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "0.125" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/vonjack_MobileLLM-125M-HF/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2107 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3027 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0091 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2601 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3782 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1164 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/vonjack_phi-3-mini-4k-instruct-llamafied.json b/data/models/vonjack_phi-3-mini-4k-instruct-llamafied.json deleted file mode 100644 index 19dde67a7b363d5b369767315426f91597a39b36..0000000000000000000000000000000000000000 --- a/data/models/vonjack_phi-3-mini-4k-instruct-llamafied.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Phi-3-mini-4k-instruct-LLaMAfied", - "id": "vonjack/Phi-3-mini-4k-instruct-LLaMAfied", - "developer": "vonjack", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.821" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/vonjack_Phi-3-mini-4k-instruct-LLaMAfied/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5787 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5741 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1382 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3305 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3924 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3885 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/vonjack_phi-3.5-mini-instruct-hermes-fc-json.json b/data/models/vonjack_phi-3.5-mini-instruct-hermes-fc-json.json deleted file mode 100644 index 23e112fde1e61f7108d40a66137fa750c886859a..0000000000000000000000000000000000000000 --- a/data/models/vonjack_phi-3.5-mini-instruct-hermes-fc-json.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Phi-3.5-mini-instruct-hermes-fc-json", - "id": "vonjack/Phi-3.5-mini-instruct-hermes-fc-json", - "developer": "vonjack", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "?", - "params_billions": "4.132" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/vonjack_Phi-3.5-mini-instruct-hermes-fc-json/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1416 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2975 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0076 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2542 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4041 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1139 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/vonjack_qwen2.5-coder-0.5b-merged.json b/data/models/vonjack_qwen2.5-coder-0.5b-merged.json deleted file mode 100644 index 6b458984250bea6138bb96a8cbe0c997c105fe6c..0000000000000000000000000000000000000000 --- a/data/models/vonjack_qwen2.5-coder-0.5b-merged.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-Coder-0.5B-Merged", - "id": "vonjack/Qwen2.5-Coder-0.5B-Merged", - "developer": "vonjack", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/vonjack_Qwen2.5-Coder-0.5B-Merged/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.31 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3076 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0378 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2534 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3303 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1202 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/vonjack_smollm2-1.7b-merged.json b/data/models/vonjack_smollm2-1.7b-merged.json deleted file mode 100644 index 0a2a78f12029af2848795d34f2d592ceffc70d83..0000000000000000000000000000000000000000 --- a/data/models/vonjack_smollm2-1.7b-merged.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SmolLM2-1.7B-Merged", - "id": "vonjack/SmolLM2-1.7B-Merged", - "developer": "vonjack", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.711" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/vonjack_SmolLM2-1.7B-Merged/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3698 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3587 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0627 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2794 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3408 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2048 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/vonjack_smollm2-135m-merged.json b/data/models/vonjack_smollm2-135m-merged.json deleted file mode 100644 index d684522eaa5c43bfb721ad75315196b8ecc6acb1..0000000000000000000000000000000000000000 --- a/data/models/vonjack_smollm2-135m-merged.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SmolLM2-135M-Merged", - "id": "vonjack/SmolLM2-135M-Merged", - "developer": "vonjack", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "0.135" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/vonjack_SmolLM2-135M-Merged/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2483 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.31 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0113 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2383 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3662 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1112 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/vonjack_smollm2-360m-merged.json b/data/models/vonjack_smollm2-360m-merged.json deleted file mode 100644 index 3f0c57f8c6287d742c9dfbf9fa4a5fa9009c5a50..0000000000000000000000000000000000000000 --- a/data/models/vonjack_smollm2-360m-merged.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SmolLM2-360M-Merged", - "id": "vonjack/SmolLM2-360M-Merged", - "developer": "vonjack", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "0.362" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/vonjack_SmolLM2-360M-Merged/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3206 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3155 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0174 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2559 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3527 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1098 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/w4r10ck_solar-10.7b-instruct-v1.0-uncensored.json b/data/models/w4r10ck_solar-10.7b-instruct-v1.0-uncensored.json deleted file mode 100644 index 72e900a5c2c36eb883e0d35ca354da6f646d51a3..0000000000000000000000000000000000000000 --- a/data/models/w4r10ck_solar-10.7b-instruct-v1.0-uncensored.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SOLAR-10.7B-Instruct-v1.0-uncensored", - "id": "w4r10ck/SOLAR-10.7B-Instruct-v1.0-uncensored", - "developer": "w4r10ck", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "10.732" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/w4r10ck_SOLAR-10.7B-Instruct-v1.0-uncensored/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3884 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5302 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0657 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2945 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4639 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3344 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/wanlige_li-14b-v0.4-slerp.json b/data/models/wanlige_li-14b-v0.4-slerp.json deleted file mode 100644 index 56fde89c86a769fe68f65de12a8c72bf3f6424ea..0000000000000000000000000000000000000000 --- a/data/models/wanlige_li-14b-v0.4-slerp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "li-14b-v0.4-slerp", - "id": "wanlige/li-14b-v0.4-slerp", - "developer": "wanlige", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/wanlige_li-14b-v0.4-slerp/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4606 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6587 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4192 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4002 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4768 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5372 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/wanlige_li-14b-v0.4-slerp0.1.json b/data/models/wanlige_li-14b-v0.4-slerp0.1.json deleted file mode 100644 index 2b6374e28389c3d08496dadbe5ab3dbd9a3c36bb..0000000000000000000000000000000000000000 --- a/data/models/wanlige_li-14b-v0.4-slerp0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "li-14b-v0.4-slerp0.1", - "id": "wanlige/li-14b-v0.4-slerp0.1", - "developer": "wanlige", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/wanlige_li-14b-v0.4-slerp0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7923 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6572 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5332 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3591 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4207 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5294 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/wanlige_li-14b-v0.4.json b/data/models/wanlige_li-14b-v0.4.json deleted file mode 100644 index c2b83207be2de423044cb8c67b190d39e130a627..0000000000000000000000000000000000000000 --- a/data/models/wanlige_li-14b-v0.4.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "li-14b-v0.4", - "id": "wanlige/li-14b-v0.4", - "developer": "wanlige", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/wanlige_li-14b-v0.4/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8133 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6544 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5574 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3389 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.446 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5167 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/wannaphong_khanomtanllm-instruct.json b/data/models/wannaphong_khanomtanllm-instruct.json deleted file mode 100644 index ab0bd406a22f654d66552eccf5caa02d65d7201c..0000000000000000000000000000000000000000 --- a/data/models/wannaphong_khanomtanllm-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "KhanomTanLLM-Instruct", - "id": "wannaphong/KhanomTanLLM-Instruct", - "developer": "wannaphong", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.447" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/wannaphong_KhanomTanLLM-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1621 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3093 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0136 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2634 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3701 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1119 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/waqasali1707_beast-soul-new.json b/data/models/waqasali1707_beast-soul-new.json deleted file mode 100644 index df4f5fa9c3970c72a5e8888ded922319c76319d9..0000000000000000000000000000000000000000 --- a/data/models/waqasali1707_beast-soul-new.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Beast-Soul-new", - "id": "waqasali1707/Beast-Soul-new", - "developer": "waqasali1707", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/waqasali1707_Beast-Soul-new/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.503 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5225 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0702 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2827 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4486 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3108 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/wave-on-discord_qwent-7b.json b/data/models/wave-on-discord_qwent-7b.json deleted file mode 100644 index 999cff5a3aa39310ecf792e3f56210d269a46c43..0000000000000000000000000000000000000000 --- a/data/models/wave-on-discord_qwent-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "qwent-7b", - "id": "wave-on-discord/qwent-7b", - "developer": "wave-on-discord", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/wave-on-discord_qwent-7b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2015 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4228 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0038 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2651 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3817 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1603 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/weathermanj_menda-3b-500.json b/data/models/weathermanj_menda-3b-500.json deleted file mode 100644 index 57f6b9eece9d1cc85962c900acc17463af560dce..0000000000000000000000000000000000000000 --- a/data/models/weathermanj_menda-3b-500.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Menda-3B-500", - "id": "weathermanj/Menda-3B-500", - "developer": "weathermanj", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.086" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/weathermanj_Menda-3B-500/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6353 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4766 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3724 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2878 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3968 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3475 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/weathermanj_menda-3b-750.json b/data/models/weathermanj_menda-3b-750.json deleted file mode 100644 index f9adad6b74b55cf19513d2d9236f8db8be423df6..0000000000000000000000000000000000000000 --- a/data/models/weathermanj_menda-3b-750.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Menda-3b-750", - "id": "weathermanj/Menda-3b-750", - "developer": "weathermanj", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.086" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/weathermanj_Menda-3b-750/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6335 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4737 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3716 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2878 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3942 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3506 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/weathermanj_menda-3b-optim-100.json b/data/models/weathermanj_menda-3b-optim-100.json deleted file mode 100644 index 6e29848c72835397e9affc3cf99c8f0b20049224..0000000000000000000000000000000000000000 --- a/data/models/weathermanj_menda-3b-optim-100.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Menda-3b-Optim-100", - "id": "weathermanj/Menda-3b-Optim-100", - "developer": "weathermanj", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.086" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/weathermanj_Menda-3b-Optim-100/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6398 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4735 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3716 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2894 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3993 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3461 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/weathermanj_menda-3b-optim-200.json b/data/models/weathermanj_menda-3b-optim-200.json deleted file mode 100644 index 46651255cadf72cdae7b137ee03eac8dffcb3d49..0000000000000000000000000000000000000000 --- a/data/models/weathermanj_menda-3b-optim-200.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Menda-3b-Optim-200", - "id": "weathermanj/Menda-3b-Optim-200", - "developer": "weathermanj", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.086" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/weathermanj_Menda-3b-Optim-200/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6375 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4746 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3731 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2827 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4033 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3484 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/wenbopan_faro-yi-9b-dpo.json b/data/models/wenbopan_faro-yi-9b-dpo.json deleted file mode 100644 index efc6e28a48d7e1691290447f051937dd6b211fa8..0000000000000000000000000000000000000000 --- a/data/models/wenbopan_faro-yi-9b-dpo.json +++ /dev/null @@ -1,144 +0,0 @@ -{ - "model_info": { - "name": "wenbopan/Faro-Yi-9B-DPO", - "id": "wenbopan/Faro-Yi-9B-DPO", - "developer": "wenbopan", - "additional_details": { - "model_type": "DPO" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/wenbopan_Faro-Yi-9B-DPO/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6461 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9218 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5307 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5514 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5839 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6395 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/weqweasdas_hh_rlhf_rm_open_llama_3b.json b/data/models/weqweasdas_hh_rlhf_rm_open_llama_3b.json deleted file mode 100644 index f9d2b14f0a0f79f11e39957c0f38b89aa1e78ac9..0000000000000000000000000000000000000000 --- a/data/models/weqweasdas_hh_rlhf_rm_open_llama_3b.json +++ /dev/null @@ -1,294 +0,0 @@ -{ - "model_info": { - "name": "weqweasdas/hh_rlhf_rm_open_llama_3b", - "id": "weqweasdas/hh_rlhf_rm_open_llama_3b", - "developer": "weqweasdas", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/weqweasdas_hh_rlhf_rm_open_llama_3b/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5027 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8184 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3728 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4149 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3281 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6564 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "reward-bench-2/weqweasdas_hh_rlhf_rm_open_llama_3b/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2498 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3642 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.275 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3497 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.24 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2384 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0315 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/weqweasdas_rm-gemma-2b.json b/data/models/weqweasdas_rm-gemma-2b.json deleted file mode 100644 index b1151d30882a530616e4e5f2252a198e5a25a8a2..0000000000000000000000000000000000000000 --- a/data/models/weqweasdas_rm-gemma-2b.json +++ /dev/null @@ -1,294 +0,0 @@ -{ - "model_info": { - "name": "weqweasdas/RM-Gemma-2B", - "id": "weqweasdas/RM-Gemma-2B", - "developer": "weqweasdas", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/weqweasdas_RM-Gemma-2B/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3057 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3705 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2812 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4317 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3311 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2343 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1851 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "reward-bench/weqweasdas_RM-Gemma-2B/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6549 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9441 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4079 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4986 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7637 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6652 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/weqweasdas_rm-gemma-7b-4096.json b/data/models/weqweasdas_rm-gemma-7b-4096.json deleted file mode 100644 index 55804c9100b7a0a1a77f32eba5389eca3c9df0a7..0000000000000000000000000000000000000000 --- a/data/models/weqweasdas_rm-gemma-7b-4096.json +++ /dev/null @@ -1,144 +0,0 @@ -{ - "model_info": { - "name": "weqweasdas/RM-Gemma-7B-4096", - "id": "weqweasdas/RM-Gemma-7B-4096", - "developer": "weqweasdas", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/weqweasdas_RM-Gemma-7B-4096/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6922 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9497 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5022 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5608 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7511 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7024 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/weqweasdas_rm-gemma-7b.json b/data/models/weqweasdas_rm-gemma-7b.json deleted file mode 100644 index 6b6f7087683616d43cac5b096d07a117476b2184..0000000000000000000000000000000000000000 --- a/data/models/weqweasdas_rm-gemma-7b.json +++ /dev/null @@ -1,294 +0,0 @@ -{ - "model_info": { - "name": "weqweasdas/RM-Gemma-7B", - "id": "weqweasdas/RM-Gemma-7B", - "developer": "weqweasdas", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/weqweasdas_RM-Gemma-7B/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4826 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4926 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3937 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6066 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4822 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.497 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4232 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "reward-bench/weqweasdas_RM-Gemma-7B/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6967 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9693 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4978 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5784 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7362 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7069 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/weqweasdas_rm-mistral-7b.json b/data/models/weqweasdas_rm-mistral-7b.json deleted file mode 100644 index 2c0c95b4657b4530753b94c6c05b68b220f49072..0000000000000000000000000000000000000000 --- a/data/models/weqweasdas_rm-mistral-7b.json +++ /dev/null @@ -1,294 +0,0 @@ -{ - "model_info": { - "name": "weqweasdas/RM-Mistral-7B", - "id": "weqweasdas/RM-Mistral-7B", - "developer": "weqweasdas", - "additional_details": { - "model_type": "Seq. Classifier" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench-2/weqweasdas_RM-Mistral-7B/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench 2", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench 2 Score (mean of all metrics)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.596 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Factuality", - "metric_config": { - "evaluation_description": "Factuality score - measures factual accuracy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5937 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Precise IF", - "metric_config": { - "evaluation_description": "Precise Instruction Following score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3438 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Math", - "metric_config": { - "evaluation_description": "Math score - measures mathematical reasoning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5956 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety score - measures safety awareness", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6911 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Focus", - "metric_config": { - "evaluation_description": "Focus score - measures response focus", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7293 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - }, - { - "evaluation_name": "Ties", - "metric_config": { - "evaluation_description": "Ties score - ability to identify tie cases", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6226 - }, - "source_data": { - "dataset_name": "RewardBench 2", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench-2-results" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "reward-bench/weqweasdas_RM-Mistral-7B/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7982 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9665 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6053 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8703 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7736 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Prior Sets (0.5 weight)", - "metric_config": { - "evaluation_description": "Prior Sets score (weighted 0.5) - includes test sets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.753 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/weyaxi_bagel-hermes-2x34b.json b/data/models/weyaxi_bagel-hermes-2x34b.json deleted file mode 100644 index bca13ba4ad15ad3196b36b2487b25846493f885c..0000000000000000000000000000000000000000 --- a/data/models/weyaxi_bagel-hermes-2x34b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Bagel-Hermes-2x34B", - "id": "Weyaxi/Bagel-Hermes-2x34B", - "developer": "Weyaxi", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MixtralForCausalLM", - "params_billions": "60.814" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Weyaxi_Bagel-Hermes-2x34B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5432 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4917 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0604 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.328 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4517 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4589 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/weyaxi_bagel-hermes-34b-slerp.json b/data/models/weyaxi_bagel-hermes-34b-slerp.json deleted file mode 100644 index 939b9010d4c53799cf0778cb3ce9a8d7f476425d..0000000000000000000000000000000000000000 --- a/data/models/weyaxi_bagel-hermes-34b-slerp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Bagel-Hermes-34B-Slerp", - "id": "Weyaxi/Bagel-Hermes-34B-Slerp", - "developer": "Weyaxi", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "34.389" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Weyaxi_Bagel-Hermes-34B-Slerp/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4603 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5922 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0604 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3347 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4622 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4703 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/weyaxi_einstein-v4-7b.json b/data/models/weyaxi_einstein-v4-7b.json deleted file mode 100644 index 9cb89d6a48ef75f9a8cad5a4fa0aa97973d3e7c2..0000000000000000000000000000000000000000 --- a/data/models/weyaxi_einstein-v4-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Einstein-v4-7B", - "id": "Weyaxi/Einstein-v4-7B", - "developer": "Weyaxi", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Weyaxi_Einstein-v4-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4708 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3849 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0189 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2819 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4682 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2259 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/weyaxi_einstein-v6.1-developed-by-weyaxi-llama3-8b.json b/data/models/weyaxi_einstein-v6.1-developed-by-weyaxi-llama3-8b.json deleted file mode 100644 index fe107949321629dcf7daff765b68b982a2c6dc8f..0000000000000000000000000000000000000000 --- a/data/models/weyaxi_einstein-v6.1-developed-by-weyaxi-llama3-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Einstein-v6.1-developed-by-Weyaxi-Llama3-8B", - "id": "Weyaxi/Einstein-v6.1-developed-by-Weyaxi-Llama3-8B", - "developer": "Weyaxi", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Weyaxi_Einstein-v6.1-developed-by-Weyaxi-Llama3-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3927 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5044 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0718 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2735 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4332 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3093 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/weyaxi_einstein-v6.1-llama3-8b.json b/data/models/weyaxi_einstein-v6.1-llama3-8b.json deleted file mode 100644 index 00ecf488f2c078b0afaa3081241268f3710d089b..0000000000000000000000000000000000000000 --- a/data/models/weyaxi_einstein-v6.1-llama3-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Einstein-v6.1-Llama3-8B", - "id": "Weyaxi/Einstein-v6.1-Llama3-8B", - "developer": "Weyaxi", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Weyaxi_Einstein-v6.1-Llama3-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4568 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5008 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.068 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2819 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4213 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3131 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/weyaxi_einstein-v7-qwen2-7b.json b/data/models/weyaxi_einstein-v7-qwen2-7b.json deleted file mode 100644 index 61ef2f19f0d05fc5455851f514c82e6c4e9907ea..0000000000000000000000000000000000000000 --- a/data/models/weyaxi_einstein-v7-qwen2-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Einstein-v7-Qwen2-7B", - "id": "Weyaxi/Einstein-v7-Qwen2-7B", - "developer": "Weyaxi", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Weyaxi_Einstein-v7-Qwen2-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.41 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5161 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1994 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2995 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.44 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4096 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/weyaxi_einstein-v8-llama3.2-1b.json b/data/models/weyaxi_einstein-v8-llama3.2-1b.json deleted file mode 100644 index cc67b82ab91f0d3f7875903b0f6ea442f301deb9..0000000000000000000000000000000000000000 --- a/data/models/weyaxi_einstein-v8-llama3.2-1b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Einstein-v8-Llama3.2-1B", - "id": "Weyaxi/Einstein-v8-Llama3.2-1B", - "developer": "Weyaxi", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.236" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Weyaxi_Einstein-v8-Llama3.2-1B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1862 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3018 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0008 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2584 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3618 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1161 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/weyaxi_sauerkrautlm-una-solar-instruct.json b/data/models/weyaxi_sauerkrautlm-una-solar-instruct.json deleted file mode 100644 index 7fc734253d722b0f4058f60cf6915845e99a333b..0000000000000000000000000000000000000000 --- a/data/models/weyaxi_sauerkrautlm-una-solar-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SauerkrautLM-UNA-SOLAR-Instruct", - "id": "Weyaxi/SauerkrautLM-UNA-SOLAR-Instruct", - "developer": "Weyaxi", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "10.732" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Weyaxi_SauerkrautLM-UNA-SOLAR-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4573 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5166 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0461 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3112 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3979 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3153 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/win10_arliai-rpmax-v1.3-merge-13.3b.json b/data/models/win10_arliai-rpmax-v1.3-merge-13.3b.json deleted file mode 100644 index cfe3ad59ac37bcaafa81689f676edacb81dd1000..0000000000000000000000000000000000000000 --- a/data/models/win10_arliai-rpmax-v1.3-merge-13.3b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ArliAI-RPMax-v1.3-merge-13.3B", - "id": "win10/ArliAI-RPMax-v1.3-merge-13.3B", - "developer": "win10", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "13.265" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/win10_ArliAI-RPMax-v1.3-merge-13.3B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3038 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4581 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0393 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2743 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4325 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.32 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/win10_breeze-13b-32k-instruct-v1_0.json b/data/models/win10_breeze-13b-32k-instruct-v1_0.json deleted file mode 100644 index a5fc38d89a724ecc76dccaa3ab913633eef21ee6..0000000000000000000000000000000000000000 --- a/data/models/win10_breeze-13b-32k-instruct-v1_0.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Breeze-13B-32k-Instruct-v1_0", - "id": "win10/Breeze-13B-32k-Instruct-v1_0", - "developer": "win10", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.726" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/win10_Breeze-13B-32k-Instruct-v1_0/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3584 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4611 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0128 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2643 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4202 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2568 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/win10_eva-norns-qwen2.5-v0.1.json b/data/models/win10_eva-norns-qwen2.5-v0.1.json deleted file mode 100644 index 4232a3340628fb72517d42fc7e22e02bae41896f..0000000000000000000000000000000000000000 --- a/data/models/win10_eva-norns-qwen2.5-v0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "EVA-Norns-Qwen2.5-v0.1", - "id": "win10/EVA-Norns-Qwen2.5-v0.1", - "developer": "win10", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/win10_EVA-Norns-Qwen2.5-v0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.622 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5072 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2613 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2852 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4045 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3425 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/win10_llama-3.2-3b-instruct-24-9-29.json b/data/models/win10_llama-3.2-3b-instruct-24-9-29.json deleted file mode 100644 index 049b27e2bc9cab59aef18755f6b6c43e3e614648..0000000000000000000000000000000000000000 --- a/data/models/win10_llama-3.2-3b-instruct-24-9-29.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.2-3B-Instruct-24-9-29", - "id": "win10/Llama-3.2-3B-Instruct-24-9-29", - "developer": "win10", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/win10_Llama-3.2-3B-Instruct-24-9-29/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7332 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4614 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1707 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2743 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3555 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3228 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/win10_llama3-13.45b-instruct.json b/data/models/win10_llama3-13.45b-instruct.json deleted file mode 100644 index 993d4b6b71b3d425509348d67b9ca52eef93ddf4..0000000000000000000000000000000000000000 --- a/data/models/win10_llama3-13.45b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "llama3-13.45b-Instruct", - "id": "win10/llama3-13.45b-Instruct", - "developer": "win10", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "13.265" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/win10_llama3-13.45b-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4144 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4865 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0242 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2584 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3848 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3345 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/win10_miscii-14b-1m-0128.json b/data/models/win10_miscii-14b-1m-0128.json deleted file mode 100644 index 00fe7af68ff453c90f201437d572a6cfe756d032..0000000000000000000000000000000000000000 --- a/data/models/win10_miscii-14b-1m-0128.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "miscii-14b-1M-0128", - "id": "win10/miscii-14b-1M-0128", - "developer": "win10", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/win10_miscii-14b-1M-0128/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4181 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5742 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4773 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3826 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5431 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4491 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/win10_norns-qwen2.5-12b.json b/data/models/win10_norns-qwen2.5-12b.json deleted file mode 100644 index 9a96ab29e580d79ade0f13ea2d7680c70f29d306..0000000000000000000000000000000000000000 --- a/data/models/win10_norns-qwen2.5-12b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Norns-Qwen2.5-12B", - "id": "win10/Norns-Qwen2.5-12B", - "developer": "win10", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "12.277" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/win10_Norns-Qwen2.5-12B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4897 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4619 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0838 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2836 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3555 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.266 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/win10_norns-qwen2.5-7b.json b/data/models/win10_norns-qwen2.5-7b.json deleted file mode 100644 index d8f0fa64c6b599f35dfa70be391c5ce9bb28db6c..0000000000000000000000000000000000000000 --- a/data/models/win10_norns-qwen2.5-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Norns-Qwen2.5-7B", - "id": "win10/Norns-Qwen2.5-7B", - "developer": "win10", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/win10_Norns-Qwen2.5-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6122 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5073 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2628 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2844 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4085 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3413 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/win10_qwen2.5-2b-instruct.json b/data/models/win10_qwen2.5-2b-instruct.json deleted file mode 100644 index 227682b7560e5ba401c5a0cc8af6dc90054bb7b3..0000000000000000000000000000000000000000 --- a/data/models/win10_qwen2.5-2b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-2B-Instruct", - "id": "win10/Qwen2.5-2B-Instruct", - "developer": "win10", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "2.9" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/win10_Qwen2.5-2B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2273 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3706 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0227 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2676 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4378 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1934 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/winglian_llama-3-8b-256k-pose.json b/data/models/winglian_llama-3-8b-256k-pose.json deleted file mode 100644 index 6ac32606d99c6d1fe6269225750896673069f171..0000000000000000000000000000000000000000 --- a/data/models/winglian_llama-3-8b-256k-pose.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "llama-3-8b-256k-PoSE", - "id": "winglian/llama-3-8b-256k-PoSE", - "developer": "winglian", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/winglian_llama-3-8b-256k-PoSE/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2909 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3157 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0196 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2576 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3316 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1116 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/winglian_llama-3-8b-64k-pose.json b/data/models/winglian_llama-3-8b-64k-pose.json deleted file mode 100644 index 0da469899c210e7e0982296aec57870e1cd1f8a5..0000000000000000000000000000000000000000 --- a/data/models/winglian_llama-3-8b-64k-pose.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-8b-64k-PoSE", - "id": "winglian/Llama-3-8b-64k-PoSE", - "developer": "winglian", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/winglian_Llama-3-8b-64k-PoSE/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2857 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3702 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0415 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2609 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3396 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2467 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/wizardlmteam_wizardlm-13b-v1.0.json b/data/models/wizardlmteam_wizardlm-13b-v1.0.json deleted file mode 100644 index 5d30988db2c3d43355459acb344d55d51a2d7198..0000000000000000000000000000000000000000 --- a/data/models/wizardlmteam_wizardlm-13b-v1.0.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "WizardLM-13B-V1.0", - "id": "WizardLMTeam/WizardLM-13B-V1.0", - "developer": "WizardLMTeam", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "13.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/WizardLMTeam_WizardLM-13B-V1.0/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.185 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2913 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2592 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3497 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1166 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/wizardlmteam_wizardlm-13b-v1.2.json b/data/models/wizardlmteam_wizardlm-13b-v1.2.json deleted file mode 100644 index 1587b6368512954d491885492912be60719b4534..0000000000000000000000000000000000000000 --- a/data/models/wizardlmteam_wizardlm-13b-v1.2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "WizardLM-13B-V1.2", - "id": "WizardLMTeam/WizardLM-13B-V1.2", - "developer": "WizardLMTeam", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "13.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/WizardLMTeam_WizardLM-13B-V1.2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3392 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4462 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0189 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2609 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4378 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2519 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/wizardlmteam_wizardlm-70b-v1.0.json b/data/models/wizardlmteam_wizardlm-70b-v1.0.json deleted file mode 100644 index 3c77fdf977eafcec802bc1613a35d4f5fde393bb..0000000000000000000000000000000000000000 --- a/data/models/wizardlmteam_wizardlm-70b-v1.0.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "WizardLM-70B-V1.0", - "id": "WizardLMTeam/WizardLM-70B-V1.0", - "developer": "WizardLMTeam", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "70.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/WizardLMTeam_WizardLM-70B-V1.0/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4951 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.559 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0393 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2659 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4391 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3447 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/wladastic_mini-think-base-1b.json b/data/models/wladastic_mini-think-base-1b.json deleted file mode 100644 index a0445bb53e764cb1afff7848084fbb7b5fcdc91b..0000000000000000000000000000000000000000 --- a/data/models/wladastic_mini-think-base-1b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mini-Think-Base-1B", - "id": "Wladastic/Mini-Think-Base-1B", - "developer": "Wladastic", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.236" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Wladastic_Mini-Think-Base-1B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5588 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3574 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0733 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2634 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3275 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1772 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/writer_instructpalmyra-30b.json b/data/models/writer_instructpalmyra-30b.json deleted file mode 100644 index db3a3709ce763a4742c53be54897c0097ac267a0..0000000000000000000000000000000000000000 --- a/data/models/writer_instructpalmyra-30b.json +++ /dev/null @@ -1,674 +0,0 @@ -{ - "model_info": { - "name": "InstructPalmyra 30B", - "id": "writer/InstructPalmyra-30B", - "developer": "writer", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_classic/writer_InstructPalmyra-30B/1774096308.339228", - "retrieved_timestamp": "1774096308.339228", - "source_metadata": { - "source_name": "helm_classic", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_classic", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_classic", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperform on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.568, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Calibration": "{\"description\": \"\", \"tab\": \"Calibration\", \"score\": \"\"}", - "Mean win rate - Robustness": "{\"description\": \"\", \"tab\": \"Robustness\", \"score\": \"0.5224242424242425\"}", - "Mean win rate - Fairness": "{\"description\": \"\", \"tab\": \"Fairness\", \"score\": \"0.5379254079254079\"}", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}", - "Mean win rate - Bias": "{\"description\": \"\", \"tab\": \"Bias\", \"score\": \"0.47136458620459815\"}", - "Mean win rate - Toxicity": "{\"description\": \"\", \"tab\": \"Toxicity\", \"score\": \"0.5811383061383062\"}", - "Mean win rate - Summarization metrics": "{\"description\": \"\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.403, - "details": { - "description": "min=0.23, mean=0.403, max=0.7, sum=6.041 (15)", - "tab": "Accuracy", - "MMLU - ECE (10-bin)": "{\"description\": \"5 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "MMLU - EM (Robustness)": "{\"description\": \"min=0.14, mean=0.348, max=0.65, sum=5.223 (15)\", \"tab\": \"Robustness\", \"score\": \"0.34819883040935673\"}", - "MMLU - EM (Fairness)": "{\"description\": \"min=0.19, mean=0.371, max=0.66, sum=5.572 (15)\", \"tab\": \"Fairness\", \"score\": \"0.3714502923976608\"}", - "MMLU - Denoised inference time (s)": "{\"description\": \"5 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=1542 (15)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=75 (15)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (15)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=371.38, mean=472.274, max=624.07, sum=7084.111 (15)\", \"tab\": \"General information\", \"score\": \"472.2740350877193\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=15 (15)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "MMLU - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=45 (15)\", \"tab\": \"General information\", \"score\": \"3.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "BoolQ", - "source_data": { - "dataset_name": "BoolQ", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on BoolQ", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.751, - "details": { - "description": "min=0.698, mean=0.751, max=0.798, sum=2.254 (3)", - "tab": "Accuracy", - "BoolQ - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "BoolQ - EM (Robustness)": "{\"description\": \"min=0.564, mean=0.656, max=0.719, sum=1.967 (3)\", \"tab\": \"Robustness\", \"score\": \"0.6556666666666667\"}", - "BoolQ - EM (Fairness)": "{\"description\": \"min=0.636, mean=0.7, max=0.762, sum=2.099 (3)\", \"tab\": \"Fairness\", \"score\": \"0.6996666666666668\"}", - "BoolQ - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "BoolQ - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "BoolQ - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "BoolQ - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "BoolQ - # prompt tokens": "{\"description\": \"min=660.073, mean=908.406, max=1242.073, sum=2725.219 (3)\", \"tab\": \"General information\", \"score\": \"908.4063333333334\"}", - "BoolQ - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=3 (3)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "BoolQ - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "BoolQ - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.496, - "details": { - "description": "min=0.253, mean=0.496, max=0.636, sum=1.489 (3)", - "tab": "Accuracy", - "NarrativeQA - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "NarrativeQA - F1 (Robustness)": "{\"description\": \"min=0.091, mean=0.317, max=0.444, sum=0.952 (3)\", \"tab\": \"Robustness\", \"score\": \"0.3173185298582432\"}", - "NarrativeQA - F1 (Fairness)": "{\"description\": \"min=0.18, mean=0.405, max=0.538, sum=1.214 (3)\", \"tab\": \"Fairness\", \"score\": \"0.40467419690737483\"}", - "NarrativeQA - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=1065 (3)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=1.051, mean=1.646, max=2.085, sum=4.938 (3)\", \"tab\": \"General information\", \"score\": \"1.6460093896713615\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=1600.366, mean=1651.848, max=1705.003, sum=4955.544 (3)\", \"tab\": \"General information\", \"score\": \"1651.8478873239437\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=1.93, mean=5.347, max=7.079, sum=16.042 (3)\", \"tab\": \"General information\", \"score\": \"5.347417840375587\"}", - "NarrativeQA - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NarrativeQA - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NarrativeQA - Stereotypes (gender)": "{\"description\": \"min=0.381, mean=0.445, max=0.5, sum=1.335 (3)\", \"tab\": \"Bias\", \"score\": \"0.44516594516594515\"}", - "NarrativeQA - Representation (race)": "{\"description\": \"min=0.333, mean=0.444, max=0.667, sum=1.333 (3)\", \"tab\": \"Bias\", \"score\": \"0.4444444444444445\"}", - "NarrativeQA - Representation (gender)": "{\"description\": \"min=0.164, mean=0.196, max=0.241, sum=0.588 (3)\", \"tab\": \"Bias\", \"score\": \"0.1960646593836042\"}", - "NarrativeQA - Toxic fraction": "{\"description\": \"min=0.003, mean=0.012, max=0.017, sum=0.037 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.01220657276995305\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (open-book)", - "source_data": { - "dataset_name": "NaturalQuestions (open-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (open-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.682, - "details": { - "description": "min=0.678, mean=0.682, max=0.688, sum=2.046 (3)", - "tab": "Accuracy", - "NaturalQuestions (closed-book) - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "NaturalQuestions (open-book) - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - F1 (Robustness)": "{\"description\": \"min=0.257, mean=0.267, max=0.272, sum=0.8 (3)\", \"tab\": \"Robustness\", \"score\": \"0.2667976861519438\"}", - "NaturalQuestions (open-book) - F1 (Robustness)": "{\"description\": \"min=0.52, mean=0.567, max=0.61, sum=1.701 (3)\", \"tab\": \"Robustness\", \"score\": \"0.5669828313348768\"}", - "NaturalQuestions (closed-book) - F1 (Fairness)": "{\"description\": \"min=0.272, mean=0.276, max=0.282, sum=0.829 (3)\", \"tab\": \"Fairness\", \"score\": \"0.276181640672073\"}", - "NaturalQuestions (open-book) - F1 (Fairness)": "{\"description\": \"min=0.621, mean=0.63, max=0.639, sum=1.891 (3)\", \"tab\": \"Fairness\", \"score\": \"0.6303513019528806\"}", - "NaturalQuestions (closed-book) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NaturalQuestions (open-book) - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=110.254, mean=112.254, max=116.254, sum=336.762 (3)\", \"tab\": \"General information\", \"score\": \"112.254\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=3.935, mean=4.247, max=4.675, sum=12.74 (3)\", \"tab\": \"General information\", \"score\": \"4.246666666666667\"}", - "NaturalQuestions (closed-book) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.647, mean=4.691, max=4.723, sum=14.072 (3)\", \"tab\": \"General information\", \"score\": \"4.690666666666666\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.036, mean=0.036, max=0.036, sum=0.108 (3)\", \"tab\": \"General information\", \"score\": \"0.036\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1231.056, mean=1419.328, max=1523.222, sum=4257.983 (3)\", \"tab\": \"General information\", \"score\": \"1419.3276666666668\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=6.778, mean=7.657, max=8.266, sum=22.97 (3)\", \"tab\": \"General information\", \"score\": \"7.656666666666666\"}", - "NaturalQuestions (open-book) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NaturalQuestions (closed-book) - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=0.667 (1)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666667\"}", - "NaturalQuestions (closed-book) - Stereotypes (gender)": "{\"description\": \"min=0.5, mean=0.5, max=0.5, sum=1.5 (3)\", \"tab\": \"Bias\", \"score\": \"0.5\"}", - "NaturalQuestions (closed-book) - Representation (race)": "{\"description\": \"min=0.498, mean=0.525, max=0.55, sum=1.576 (3)\", \"tab\": \"Bias\", \"score\": \"0.5252747252747252\"}", - "NaturalQuestions (closed-book) - Representation (gender)": "{\"description\": \"min=0.088, mean=0.134, max=0.206, sum=0.401 (3)\", \"tab\": \"Bias\", \"score\": \"0.13375350140056022\"}", - "NaturalQuestions (open-book) - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NaturalQuestions (open-book) - Stereotypes (gender)": "{\"description\": \"min=0.3, mean=0.392, max=0.443, sum=1.176 (3)\", \"tab\": \"Bias\", \"score\": \"0.39206349206349206\"}", - "NaturalQuestions (open-book) - Representation (race)": "{\"description\": \"min=0.45, mean=0.49, max=0.533, sum=1.47 (3)\", \"tab\": \"Bias\", \"score\": \"0.4899991188650981\"}", - "NaturalQuestions (open-book) - Representation (gender)": "{\"description\": \"min=0.327, mean=0.384, max=0.422, sum=1.152 (3)\", \"tab\": \"Bias\", \"score\": \"0.3838592033738646\"}", - "NaturalQuestions (closed-book) - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}", - "NaturalQuestions (open-book) - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "QuAC", - "source_data": { - "dataset_name": "QuAC", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on QuAC", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.433, - "details": { - "description": "min=0.423, mean=0.433, max=0.447, sum=1.3 (3)", - "tab": "Accuracy", - "QuAC - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "QuAC - F1 (Robustness)": "{\"description\": \"min=0.23, mean=0.248, max=0.258, sum=0.743 (3)\", \"tab\": \"Robustness\", \"score\": \"0.24761534139298128\"}", - "QuAC - F1 (Fairness)": "{\"description\": \"min=0.328, mean=0.337, max=0.353, sum=1.011 (3)\", \"tab\": \"Fairness\", \"score\": \"0.3370729442565461\"}", - "QuAC - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "QuAC - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "QuAC - # train": "{\"description\": \"min=0.845, mean=0.944, max=1.084, sum=2.831 (3)\", \"tab\": \"General information\", \"score\": \"0.9436666666666667\"}", - "QuAC - truncated": "{\"description\": \"min=0.016, mean=0.016, max=0.016, sum=0.048 (3)\", \"tab\": \"General information\", \"score\": \"0.016\"}", - "QuAC - # prompt tokens": "{\"description\": \"min=1624.371, mean=1644.436, max=1670.589, sum=4933.308 (3)\", \"tab\": \"General information\", \"score\": \"1644.436\"}", - "QuAC - # output tokens": "{\"description\": \"min=18.652, mean=22.969, max=26.445, sum=68.907 (3)\", \"tab\": \"General information\", \"score\": \"22.969000000000005\"}", - "QuAC - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "QuAC - Stereotypes (race)": "{\"description\": \"min=0.571, mean=0.582, max=0.59, sum=1.745 (3)\", \"tab\": \"Bias\", \"score\": \"0.5815018315018315\"}", - "QuAC - Stereotypes (gender)": "{\"description\": \"min=0.413, mean=0.431, max=0.463, sum=1.292 (3)\", \"tab\": \"Bias\", \"score\": \"0.43052581120508293\"}", - "QuAC - Representation (race)": "{\"description\": \"min=0.317, mean=0.337, max=0.368, sum=1.012 (3)\", \"tab\": \"Bias\", \"score\": \"0.33749135321526574\"}", - "QuAC - Representation (gender)": "{\"description\": \"min=0.224, mean=0.236, max=0.243, sum=0.707 (3)\", \"tab\": \"Bias\", \"score\": \"0.2355073330063574\"}", - "QuAC - Toxic fraction": "{\"description\": \"min=0, mean=0.001, max=0.002, sum=0.002 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.0006666666666666666\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "HellaSwag", - "source_data": { - "dataset_name": "HellaSwag", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on HellaSwag", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "HellaSwag - ECE (10-bin)": "{\"description\": \"No matching runs\", \"tab\": \"Calibration\", \"score\": \"\"}", - "HellaSwag - EM (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "HellaSwag - EM (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "HellaSwag - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "HellaSwag - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "OpenbookQA - ECE (10-bin)": "{\"description\": \"No matching runs\", \"tab\": \"Calibration\", \"score\": \"\"}", - "OpenbookQA - EM (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "OpenbookQA - EM (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "OpenbookQA - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "OpenbookQA - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "TruthfulQA", - "source_data": { - "dataset_name": "TruthfulQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on TruthfulQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.185, - "details": { - "description": "min=0.18, mean=0.185, max=0.19, sum=0.555 (3)", - "tab": "Accuracy", - "TruthfulQA - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "TruthfulQA - EM (Robustness)": "{\"description\": \"min=0.148, mean=0.151, max=0.154, sum=0.454 (3)\", \"tab\": \"Robustness\", \"score\": \"0.1513761467889908\"}", - "TruthfulQA - EM (Fairness)": "{\"description\": \"min=0.147, mean=0.152, max=0.157, sum=0.456 (3)\", \"tab\": \"Fairness\", \"score\": \"0.15188583078491336\"}", - "TruthfulQA - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "TruthfulQA - # eval": "{\"description\": \"min=654, mean=654, max=654, sum=1962 (3)\", \"tab\": \"General information\", \"score\": \"654.0\"}", - "TruthfulQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "TruthfulQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "TruthfulQA - # prompt tokens": "{\"description\": \"min=501.121, mean=511.121, max=529.121, sum=1533.362 (3)\", \"tab\": \"General information\", \"score\": \"511.12079510703364\"}", - "TruthfulQA - # output tokens": "{\"description\": \"min=0.998, mean=0.999, max=1, sum=2.997 (3)\", \"tab\": \"General information\", \"score\": \"0.998980632008155\"}", - "TruthfulQA - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MS MARCO (TREC)", - "source_data": { - "dataset_name": "MS MARCO (TREC)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "NDCG@10 on MS MARCO (TREC)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "MS MARCO (regular) - RR@10 (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "MS MARCO (TREC) - NDCG@10 (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "MS MARCO (regular) - RR@10 (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "MS MARCO (TREC) - NDCG@10 (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "MS MARCO (regular) - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MS MARCO (TREC) - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MS MARCO (regular) - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "MS MARCO (TREC) - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CNN/DailyMail", - "source_data": { - "dataset_name": "CNN/DailyMail", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on CNN/DailyMail", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.152, - "details": { - "description": "min=0.142, mean=0.152, max=0.165, sum=0.455 (3)", - "tab": "Accuracy", - "CNN/DailyMail - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "CNN/DailyMail - # eval": "{\"description\": \"min=466, mean=466, max=466, sum=1398 (3)\", \"tab\": \"General information\", \"score\": \"466.0\"}", - "CNN/DailyMail - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CNN/DailyMail - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CNN/DailyMail - # prompt tokens": "{\"description\": \"min=1531.586, mean=1549.919, max=1567.586, sum=4649.758 (3)\", \"tab\": \"General information\", \"score\": \"1549.9191702432045\"}", - "CNN/DailyMail - # output tokens": "{\"description\": \"min=74.511, mean=83.965, max=95.704, sum=251.895 (3)\", \"tab\": \"General information\", \"score\": \"83.96494992846924\"}", - "CNN/DailyMail - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "CNN/DailyMail - Stereotypes (race)": "{\"description\": \"min=0.619, mean=0.638, max=0.651, sum=1.914 (3)\", \"tab\": \"Bias\", \"score\": \"0.638095238095238\"}", - "CNN/DailyMail - Stereotypes (gender)": "{\"description\": \"min=0.344, mean=0.371, max=0.398, sum=1.112 (3)\", \"tab\": \"Bias\", \"score\": \"0.3705770935558364\"}", - "CNN/DailyMail - Representation (race)": "{\"description\": \"min=0.199, mean=0.258, max=0.288, sum=0.773 (3)\", \"tab\": \"Bias\", \"score\": \"0.2575629817009127\"}", - "CNN/DailyMail - Representation (gender)": "{\"description\": \"min=0.108, mean=0.117, max=0.129, sum=0.351 (3)\", \"tab\": \"Bias\", \"score\": \"0.11691353772442492\"}", - "CNN/DailyMail - Toxic fraction": "{\"description\": \"min=0, mean=0.002, max=0.006, sum=0.006 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.002145922746781116\"}", - "CNN/DailyMail - SummaC": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - QAFactEval": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - BERTScore (F1)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - Coverage": "{\"description\": \"min=0.961, mean=0.972, max=0.979, sum=2.915 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.9716859203819838\"}", - "CNN/DailyMail - Density": "{\"description\": \"min=22.383, mean=28.97, max=38.633, sum=86.91 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"28.97014469233496\"}", - "CNN/DailyMail - Compression": "{\"description\": \"min=6.723, mean=7.901, max=9.103, sum=23.703 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"7.901010404629208\"}", - "CNN/DailyMail - HumanEval-faithfulness": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-relevance": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-coherence": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "XSUM", - "source_data": { - "dataset_name": "XSUM", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on XSUM", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.104, - "details": { - "description": "min=0.1, mean=0.104, max=0.106, sum=0.312 (3)", - "tab": "Accuracy", - "XSUM - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "XSUM - # eval": "{\"description\": \"min=518, mean=518, max=518, sum=1554 (3)\", \"tab\": \"General information\", \"score\": \"518.0\"}", - "XSUM - # train": "{\"description\": \"min=4.998, mean=4.999, max=5, sum=14.996 (3)\", \"tab\": \"General information\", \"score\": \"4.998712998712999\"}", - "XSUM - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "XSUM - # prompt tokens": "{\"description\": \"min=1456.402, mean=1510.418, max=1538.921, sum=4531.255 (3)\", \"tab\": \"General information\", \"score\": \"1510.4182754182755\"}", - "XSUM - # output tokens": "{\"description\": \"min=26.207, mean=26.632, max=27.241, sum=79.896 (3)\", \"tab\": \"General information\", \"score\": \"26.631917631917634\"}", - "XSUM - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "XSUM - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=2 (3)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "XSUM - Stereotypes (gender)": "{\"description\": \"min=0.436, mean=0.459, max=0.489, sum=1.376 (3)\", \"tab\": \"Bias\", \"score\": \"0.45852730200556285\"}", - "XSUM - Representation (race)": "{\"description\": \"min=0.532, mean=0.59, max=0.667, sum=1.771 (3)\", \"tab\": \"Bias\", \"score\": \"0.5901750807411186\"}", - "XSUM - Representation (gender)": "{\"description\": \"min=0.17, mean=0.187, max=0.207, sum=0.562 (3)\", \"tab\": \"Bias\", \"score\": \"0.18720575071822934\"}", - "XSUM - Toxic fraction": "{\"description\": \"min=0, mean=0.001, max=0.002, sum=0.002 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.0006435006435006435\"}", - "XSUM - SummaC": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - QAFactEval": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - BERTScore (F1)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - Coverage": "{\"description\": \"min=0.836, mean=0.844, max=0.853, sum=2.531 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.8437121246995759\"}", - "XSUM - Density": "{\"description\": \"min=3.292, mean=3.441, max=3.518, sum=10.323 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"3.4410181202034944\"}", - "XSUM - Compression": "{\"description\": \"min=15.467, mean=15.707, max=15.837, sum=47.122 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"15.707173220790708\"}", - "XSUM - HumanEval-faithfulness": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-relevance": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-coherence": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "IMDB", - "source_data": { - "dataset_name": "IMDB", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on IMDB", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.94, - "details": { - "description": "min=0.936, mean=0.94, max=0.946, sum=2.821 (3)", - "tab": "Accuracy", - "IMDB - ECE (10-bin)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "IMDB - EM (Robustness)": "{\"description\": \"min=0.898, mean=0.906, max=0.916, sum=2.718 (3)\", \"tab\": \"Robustness\", \"score\": \"0.906\"}", - "IMDB - EM (Fairness)": "{\"description\": \"min=0.925, mean=0.931, max=0.94, sum=2.793 (3)\", \"tab\": \"Fairness\", \"score\": \"0.931\"}", - "IMDB - Denoised inference time (s)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "IMDB - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "IMDB - # train": "{\"description\": \"min=2.916, mean=4.242, max=4.986, sum=12.726 (3)\", \"tab\": \"General information\", \"score\": \"4.242\"}", - "IMDB - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IMDB - # prompt tokens": "{\"description\": \"min=1277.729, mean=1553.363, max=1768.607, sum=4660.089 (3)\", \"tab\": \"General information\", \"score\": \"1553.363\"}", - "IMDB - # output tokens": "{\"description\": \"min=0.995, mean=0.997, max=0.999, sum=2.992 (3)\", \"tab\": \"General information\", \"score\": \"0.9973333333333333\"}", - "IMDB - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "IMDB - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CivilComments", - "source_data": { - "dataset_name": "CivilComments", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on CivilComments", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.555, - "details": { - "description": "min=0, mean=0.555, max=0.877, sum=29.976 (54)", - "tab": "Accuracy", - "CivilComments - ECE (10-bin)": "{\"description\": \"9 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "CivilComments - EM (Robustness)": "{\"description\": \"min=0, mean=0.443, max=0.774, sum=23.937 (54)\", \"tab\": \"Robustness\", \"score\": \"0.4432801514699601\"}", - "CivilComments - EM (Fairness)": "{\"description\": \"min=0, mean=0.449, max=0.871, sum=24.239 (54)\", \"tab\": \"Fairness\", \"score\": \"0.44887663628250224\"}", - "CivilComments - Denoised inference time (s)": "{\"description\": \"9 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "CivilComments - # eval": "{\"description\": \"min=74, mean=371.556, max=683, sum=20064 (54)\", \"tab\": \"General information\", \"score\": \"371.55555555555554\"}", - "CivilComments - # train": "{\"description\": \"min=5, mean=5, max=5, sum=270 (54)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CivilComments - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (54)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CivilComments - # prompt tokens": "{\"description\": \"min=356.537, mean=722.635, max=1267.519, sum=39022.317 (54)\", \"tab\": \"General information\", \"score\": \"722.6354931173206\"}", - "CivilComments - # output tokens": "{\"description\": \"min=0, mean=0.905, max=1, sum=48.891 (54)\", \"tab\": \"General information\", \"score\": \"0.9053814074087929\"}", - "CivilComments - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=162 (54)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "CivilComments - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Toxic fraction": "{\"description\": \"9 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "RAFT", - "source_data": { - "dataset_name": "RAFT", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on RAFT", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.652, - "details": { - "description": "min=0.275, mean=0.652, max=0.95, sum=21.5 (33)", - "tab": "Accuracy", - "RAFT - ECE (10-bin)": "{\"description\": \"11 matching runs, but no matching metrics\", \"tab\": \"Calibration\", \"score\": \"\"}", - "RAFT - EM (Robustness)": "{\"description\": \"min=0.05, mean=0.518, max=0.95, sum=17.1 (33)\", \"tab\": \"Robustness\", \"score\": \"0.5181818181818182\"}", - "RAFT - EM (Fairness)": "{\"description\": \"min=0.25, mean=0.618, max=0.925, sum=20.4 (33)\", \"tab\": \"Fairness\", \"score\": \"0.6181818181818182\"}", - "RAFT - Denoised inference time (s)": "{\"description\": \"11 matching runs, but no matching metrics\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "RAFT - # eval": "{\"description\": \"min=40, mean=40, max=40, sum=1320 (33)\", \"tab\": \"General information\", \"score\": \"40.0\"}", - "RAFT - # train": "{\"description\": \"min=0, mean=4.556, max=5, sum=150.35 (33)\", \"tab\": \"General information\", \"score\": \"4.556060606060607\"}", - "RAFT - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (33)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "RAFT - # prompt tokens": "{\"description\": \"min=257.35, mean=812.938, max=1773.675, sum=26826.95 (33)\", \"tab\": \"General information\", \"score\": \"812.937878787879\"}", - "RAFT - # output tokens": "{\"description\": \"min=1, mean=2.967, max=6.15, sum=97.925 (33)\", \"tab\": \"General information\", \"score\": \"2.9674242424242423\"}", - "RAFT - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=99 (33)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "RAFT - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Toxic fraction": "{\"description\": \"11 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/writer_palmyra-fin.json b/data/models/writer_palmyra-fin.json deleted file mode 100644 index 6a4d113b923a20960c35dfc6eb9c478717e86786..0000000000000000000000000000000000000000 --- a/data/models/writer_palmyra-fin.json +++ /dev/null @@ -1,234 +0,0 @@ -{ - "model_info": { - "name": "Palmyra Fin", - "id": "writer/palmyra-fin", - "developer": "writer", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_capabilities/writer_palmyra-fin/1774096304.5056698", - "retrieved_timestamp": "1774096304.5056698", - "source_metadata": { - "source_name": "helm_capabilities", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_capabilities", - "evaluation_results": [ - { - "evaluation_name": "Mean score", - "source_data": { - "dataset_name": "helm_capabilities", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "The mean of the scores from all columns.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.577, - "details": { - "description": "", - "tab": "Accuracy", - "Mean score - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"13.54320003211858\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU-Pro", - "source_data": { - "dataset_name": "MMLU-Pro", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on MMLU-Pro", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.591, - "details": { - "description": "min=0.591, mean=0.591, max=0.591, sum=0.591 (1)", - "tab": "Accuracy", - "MMLU-Pro - Observed inference time (s)": "{\"description\": \"min=10.488, mean=10.488, max=10.488, sum=10.488 (1)\", \"tab\": \"Efficiency\", \"score\": \"10.488489307641983\"}", - "MMLU-Pro - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "MMLU-Pro - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - # prompt tokens": "{\"description\": \"min=228.366, mean=228.366, max=228.366, sum=228.366 (1)\", \"tab\": \"General information\", \"score\": \"228.366\"}", - "MMLU-Pro - # output tokens": "{\"description\": \"min=493.383, mean=493.383, max=493.383, sum=493.383 (1)\", \"tab\": \"General information\", \"score\": \"493.383\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"all\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.422, - "details": { - "description": "min=0.422, mean=0.422, max=0.422, sum=0.422 (1)", - "tab": "Accuracy", - "GPQA - Observed inference time (s)": "{\"description\": \"min=14.428, mean=14.428, max=14.428, sum=14.428 (1)\", \"tab\": \"Efficiency\", \"score\": \"14.42766729758994\"}", - "GPQA - # eval": "{\"description\": \"min=446, mean=446, max=446, sum=446 (1)\", \"tab\": \"General information\", \"score\": \"446.0\"}", - "GPQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - # prompt tokens": "{\"description\": \"min=248.886, mean=248.886, max=248.886, sum=248.886 (1)\", \"tab\": \"General information\", \"score\": \"248.88565022421525\"}", - "GPQA - # output tokens": "{\"description\": \"min=671.045, mean=671.045, max=671.045, sum=671.045 (1)\", \"tab\": \"General information\", \"score\": \"671.0448430493274\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"gpqa_main\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "IFEval Strict Acc on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.793, - "details": { - "description": "min=0.793, mean=0.793, max=0.793, sum=0.793 (1)", - "tab": "Accuracy", - "IFEval - Observed inference time (s)": "{\"description\": \"min=6.85, mean=6.85, max=6.85, sum=6.85 (1)\", \"tab\": \"Efficiency\", \"score\": \"6.849953265815918\"}", - "IFEval - # eval": "{\"description\": \"min=541, mean=541, max=541, sum=541 (1)\", \"tab\": \"General information\", \"score\": \"541.0\"}", - "IFEval - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - # prompt tokens": "{\"description\": \"min=46.024, mean=46.024, max=46.024, sum=46.024 (1)\", \"tab\": \"General information\", \"score\": \"46.024029574861366\"}", - "IFEval - # output tokens": "{\"description\": \"min=332.181, mean=332.181, max=332.181, sum=332.181 (1)\", \"tab\": \"General information\", \"score\": \"332.181146025878\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WildBench", - "source_data": { - "dataset_name": "WildBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "WB Score on WildBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.783, - "details": { - "description": "min=0.783, mean=0.783, max=0.783, sum=0.783 (1)", - "tab": "Accuracy", - "WildBench - Observed inference time (s)": "{\"description\": \"min=18.947, mean=18.947, max=18.947, sum=18.947 (1)\", \"tab\": \"Efficiency\", \"score\": \"18.947298042297362\"}", - "WildBench - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "WildBench - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # prompt tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # output tokens": "{\"description\": \"min=935.556, mean=935.556, max=935.556, sum=935.556 (1)\", \"tab\": \"General information\", \"score\": \"935.556\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"v2\"" - } - } - }, - { - "evaluation_name": "Omni-MATH", - "source_data": { - "dataset_name": "Omni-MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Acc on Omni-MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.295, - "details": { - "description": "min=0.295, mean=0.295, max=0.295, sum=0.295 (1)", - "tab": "Accuracy", - "Omni-MATH - Observed inference time (s)": "{\"description\": \"min=17.003, mean=17.003, max=17.003, sum=17.003 (1)\", \"tab\": \"Efficiency\", \"score\": \"17.002592247247694\"}", - "Omni-MATH - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "Omni-MATH - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - # prompt tokens": "{\"description\": \"min=109.708, mean=109.708, max=109.708, sum=109.708 (1)\", \"tab\": \"General information\", \"score\": \"109.708\"}", - "Omni-MATH - # output tokens": "{\"description\": \"min=820.641, mean=820.641, max=820.641, sum=820.641 (1)\", \"tab\": \"General information\", \"score\": \"820.641\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/writer_palmyra-med.json b/data/models/writer_palmyra-med.json deleted file mode 100644 index f3d08b9dae1ebda156a0199cffe5a95de8cdfed5..0000000000000000000000000000000000000000 --- a/data/models/writer_palmyra-med.json +++ /dev/null @@ -1,234 +0,0 @@ -{ - "model_info": { - "name": "Palmyra Med", - "id": "writer/palmyra-med", - "developer": "writer", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_capabilities/writer_palmyra-med/1774096304.5056698", - "retrieved_timestamp": "1774096304.5056698", - "source_metadata": { - "source_name": "helm_capabilities", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_capabilities", - "evaluation_results": [ - { - "evaluation_name": "Mean score", - "source_data": { - "dataset_name": "helm_capabilities", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "The mean of the scores from all columns.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.476, - "details": { - "description": "", - "tab": "Accuracy", - "Mean score - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"4.374187379517853\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU-Pro", - "source_data": { - "dataset_name": "MMLU-Pro", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on MMLU-Pro", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.411, - "details": { - "description": "min=0.411, mean=0.411, max=0.411, sum=0.411 (1)", - "tab": "Accuracy", - "MMLU-Pro - Observed inference time (s)": "{\"description\": \"min=0.327, mean=0.327, max=0.327, sum=0.327 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.32738947081565856\"}", - "MMLU-Pro - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "MMLU-Pro - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - # prompt tokens": "{\"description\": \"min=249.366, mean=249.366, max=249.366, sum=249.366 (1)\", \"tab\": \"General information\", \"score\": \"249.366\"}", - "MMLU-Pro - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"all\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.368, - "details": { - "description": "min=0.368, mean=0.368, max=0.368, sum=0.368 (1)", - "tab": "Accuracy", - "GPQA - Observed inference time (s)": "{\"description\": \"min=0.356, mean=0.356, max=0.356, sum=0.356 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.3557077256018805\"}", - "GPQA - # eval": "{\"description\": \"min=446, mean=446, max=446, sum=446 (1)\", \"tab\": \"General information\", \"score\": \"446.0\"}", - "GPQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - # prompt tokens": "{\"description\": \"min=269.886, mean=269.886, max=269.886, sum=269.886 (1)\", \"tab\": \"General information\", \"score\": \"269.8856502242152\"}", - "GPQA - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"gpqa_main\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "IFEval Strict Acc on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.767, - "details": { - "description": "min=0.767, mean=0.767, max=0.767, sum=0.767 (1)", - "tab": "Accuracy", - "IFEval - Observed inference time (s)": "{\"description\": \"min=4.651, mean=4.651, max=4.651, sum=4.651 (1)\", \"tab\": \"Efficiency\", \"score\": \"4.650597941633073\"}", - "IFEval - # eval": "{\"description\": \"min=541, mean=541, max=541, sum=541 (1)\", \"tab\": \"General information\", \"score\": \"541.0\"}", - "IFEval - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - # prompt tokens": "{\"description\": \"min=46.024, mean=46.024, max=46.024, sum=46.024 (1)\", \"tab\": \"General information\", \"score\": \"46.024029574861366\"}", - "IFEval - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WildBench", - "source_data": { - "dataset_name": "WildBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "WB Score on WildBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.676, - "details": { - "description": "min=0.676, mean=0.676, max=0.676, sum=0.676 (1)", - "tab": "Accuracy", - "WildBench - Observed inference time (s)": "{\"description\": \"min=10.081, mean=10.081, max=10.081, sum=10.081 (1)\", \"tab\": \"Efficiency\", \"score\": \"10.080555647850037\"}", - "WildBench - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "WildBench - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # prompt tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"v2\"" - } - } - }, - { - "evaluation_name": "Omni-MATH", - "source_data": { - "dataset_name": "Omni-MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Acc on Omni-MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.156, - "details": { - "description": "min=0.156, mean=0.156, max=0.156, sum=0.156 (1)", - "tab": "Accuracy", - "Omni-MATH - Observed inference time (s)": "{\"description\": \"min=6.457, mean=6.457, max=6.457, sum=6.457 (1)\", \"tab\": \"Efficiency\", \"score\": \"6.456686111688614\"}", - "Omni-MATH - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "Omni-MATH - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - # prompt tokens": "{\"description\": \"min=109.708, mean=109.708, max=109.708, sum=109.708 (1)\", \"tab\": \"General information\", \"score\": \"109.708\"}", - "Omni-MATH - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/writer_palmyra-x-004-fc.json b/data/models/writer_palmyra-x-004-fc.json deleted file mode 100644 index 2174cfb2504f57bf4286f95f6b4b12db3b1ed217..0000000000000000000000000000000000000000 --- a/data/models/writer_palmyra-x-004-fc.json +++ /dev/null @@ -1,851 +0,0 @@ -{ - "model_info": { - "name": "palmyra-x-004 (FC)", - "id": "writer/palmyra-x-004-fc", - "developer": "writer", - "additional_details": { - "raw_model_name": "palmyra-x-004 (FC)", - "organization": "Writer", - "license": "Proprietary", - "mode": "FC", - "model_link": "https://writer.com/engineering/actions-with-palmyra-x-004/" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/writer/palmyra-x-004-fc/1775236112.40559", - "retrieved_timestamp": "1775236112.40559", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 76.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 27.87 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 178.15 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 3.71 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 7.62 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 8.04 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 87.46 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 71.33 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 96.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 93.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 89.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 77.87 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 79.46 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 77.97 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 56.25 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 70.83 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 0.38 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 0.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 0.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 0.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 2.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 4.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 1.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 13.12 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 6.45 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 14.19 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 18.71 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 81.25 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 80.99 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/writer_palmyra-x-004.json b/data/models/writer_palmyra-x-004.json deleted file mode 100644 index 2294f023f782f4147f164b4e0545809523762a54..0000000000000000000000000000000000000000 --- a/data/models/writer_palmyra-x-004.json +++ /dev/null @@ -1,2133 +0,0 @@ -{ - "model_info": { - "name": "Palmyra-X-004", - "id": "writer/palmyra-x-004", - "developer": "writer", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_capabilities/writer_palmyra-x-004/1774096304.5056698", - "retrieved_timestamp": "1774096304.5056698", - "source_metadata": { - "source_name": "helm_capabilities", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_capabilities", - "evaluation_results": [ - { - "evaluation_name": "Mean score", - "source_data": { - "dataset_name": "helm_capabilities", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "The mean of the scores from all columns.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.609, - "details": { - "description": "", - "tab": "Accuracy", - "Mean score - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"19.38686150670534\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU-Pro", - "source_data": { - "dataset_name": "MMLU-Pro", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on MMLU-Pro", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.657, - "details": { - "description": "min=0.657, mean=0.657, max=0.657, sum=0.657 (1)", - "tab": "Accuracy", - "MMLU-Pro - Observed inference time (s)": "{\"description\": \"min=14.079, mean=14.079, max=14.079, sum=14.079 (1)\", \"tab\": \"Efficiency\", \"score\": \"14.079012663602828\"}", - "MMLU-Pro - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "MMLU-Pro - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - # prompt tokens": "{\"description\": \"min=228.366, mean=228.366, max=228.366, sum=228.366 (1)\", \"tab\": \"General information\", \"score\": \"228.366\"}", - "MMLU-Pro - # output tokens": "{\"description\": \"min=510.633, mean=510.633, max=510.633, sum=510.633 (1)\", \"tab\": \"General information\", \"score\": \"510.633\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"all\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.395, - "details": { - "description": "min=0.395, mean=0.395, max=0.395, sum=0.395 (1)", - "tab": "Accuracy", - "GPQA - Observed inference time (s)": "{\"description\": \"min=20.444, mean=20.444, max=20.444, sum=20.444 (1)\", \"tab\": \"Efficiency\", \"score\": \"20.444375363700594\"}", - "GPQA - # eval": "{\"description\": \"min=446, mean=446, max=446, sum=446 (1)\", \"tab\": \"General information\", \"score\": \"446.0\"}", - "GPQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - # prompt tokens": "{\"description\": \"min=248.886, mean=248.886, max=248.886, sum=248.886 (1)\", \"tab\": \"General information\", \"score\": \"248.88565022421525\"}", - "GPQA - # output tokens": "{\"description\": \"min=716.437, mean=716.437, max=716.437, sum=716.437 (1)\", \"tab\": \"General information\", \"score\": \"716.4372197309417\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"gpqa_main\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "IFEval Strict Acc on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.872, - "details": { - "description": "min=0.872, mean=0.872, max=0.872, sum=0.872 (1)", - "tab": "Accuracy", - "IFEval - Observed inference time (s)": "{\"description\": \"min=10.268, mean=10.268, max=10.268, sum=10.268 (1)\", \"tab\": \"Efficiency\", \"score\": \"10.267585801990107\"}", - "IFEval - # eval": "{\"description\": \"min=541, mean=541, max=541, sum=541 (1)\", \"tab\": \"General information\", \"score\": \"541.0\"}", - "IFEval - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - # prompt tokens": "{\"description\": \"min=46.024, mean=46.024, max=46.024, sum=46.024 (1)\", \"tab\": \"General information\", \"score\": \"46.024029574861366\"}", - "IFEval - # output tokens": "{\"description\": \"min=357.087, mean=357.087, max=357.087, sum=357.087 (1)\", \"tab\": \"General information\", \"score\": \"357.08687615526804\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WildBench", - "source_data": { - "dataset_name": "WildBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "WB Score on WildBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.802, - "details": { - "description": "min=0.802, mean=0.802, max=0.802, sum=0.802 (1)", - "tab": "Accuracy", - "WildBench - Observed inference time (s)": "{\"description\": \"min=28.186, mean=28.186, max=28.186, sum=28.186 (1)\", \"tab\": \"Efficiency\", \"score\": \"28.185582681894303\"}", - "WildBench - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "WildBench - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # prompt tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # output tokens": "{\"description\": \"min=1068.195, mean=1068.195, max=1068.195, sum=1068.195 (1)\", \"tab\": \"General information\", \"score\": \"1068.195\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"v2\"" - } - } - }, - { - "evaluation_name": "Omni-MATH", - "source_data": { - "dataset_name": "Omni-MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Acc on Omni-MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.32, - "details": { - "description": "min=0.32, mean=0.32, max=0.32, sum=0.32 (1)", - "tab": "Accuracy", - "Omni-MATH - Observed inference time (s)": "{\"description\": \"min=23.958, mean=23.958, max=23.958, sum=23.958 (1)\", \"tab\": \"Efficiency\", \"score\": \"23.95775102233887\"}", - "Omni-MATH - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "Omni-MATH - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - # prompt tokens": "{\"description\": \"min=109.708, mean=109.708, max=109.708, sum=109.708 (1)\", \"tab\": \"General information\", \"score\": \"109.708\"}", - "Omni-MATH - # output tokens": "{\"description\": \"min=897.531, mean=897.531, max=897.531, sum=897.531 (1)\", \"tab\": \"General information\", \"score\": \"897.531\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_lite/writer_palmyra-x-004/1774096306.427425", - "retrieved_timestamp": "1774096306.427425", - "source_metadata": { - "source_name": "helm_lite", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_lite", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_lite", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.808, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.4045318352059925\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.773, - "details": { - "description": "min=0.773, mean=0.773, max=0.773, sum=0.773 (1)", - "tab": "Accuracy", - "NarrativeQA - Observed inference time (s)": "{\"description\": \"min=1.634, mean=1.634, max=1.634, sum=1.634 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.634409177135414\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=3484.268, mean=3484.268, max=3484.268, sum=3484.268 (1)\", \"tab\": \"General information\", \"score\": \"3484.2676056338028\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=6.338, mean=6.338, max=6.338, sum=6.338 (1)\", \"tab\": \"General information\", \"score\": \"6.338028169014085\"}" - } - }, - "generation_config": { - "additional_details": { - "stop": "\"none\"" - } - } - }, - { - "evaluation_name": "NaturalQuestions (closed-book)", - "source_data": { - "dataset_name": "NaturalQuestions (closed-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (closed-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.457, - "details": { - "description": "min=0.457, mean=0.457, max=0.457, sum=0.457 (1)", - "tab": "Accuracy", - "NaturalQuestions (open-book) - Observed inference time (s)": "{\"description\": \"min=1.221, mean=1.221, max=1.221, sum=1.221 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.22119681596756\"}", - "NaturalQuestions (closed-book) - Observed inference time (s)": "{\"description\": \"min=1.213, mean=1.213, max=1.213, sum=1.213 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.2129934797286988\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.965, mean=4.965, max=4.965, sum=4.965 (1)\", \"tab\": \"General information\", \"score\": \"4.965\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.007, mean=0.007, max=0.007, sum=0.007 (1)\", \"tab\": \"General information\", \"score\": \"0.007\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1675.231, mean=1675.231, max=1675.231, sum=1675.231 (1)\", \"tab\": \"General information\", \"score\": \"1675.231\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=10.295, mean=10.295, max=10.295, sum=10.295 (1)\", \"tab\": \"General information\", \"score\": \"10.295\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=129.12, mean=129.12, max=129.12, sum=129.12 (1)\", \"tab\": \"General information\", \"score\": \"129.12\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=12.549, mean=12.549, max=12.549, sum=12.549 (1)\", \"tab\": \"General information\", \"score\": \"12.549\"}" - } - }, - "generation_config": { - "additional_details": { - "mode": "\"closedbook\"", - "stop": "\"none\"" - } - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.926, - "details": { - "description": "min=0.926, mean=0.926, max=0.926, sum=0.926 (1)", - "tab": "Accuracy", - "OpenbookQA - Observed inference time (s)": "{\"description\": \"min=0.271, mean=0.271, max=0.271, sum=0.271 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.2705215420722961\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=249.776, mean=249.776, max=249.776, sum=249.776 (1)\", \"tab\": \"General information\", \"score\": \"249.776\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=0.992, mean=0.992, max=0.992, sum=0.992 (1)\", \"tab\": \"General information\", \"score\": \"0.992\"}" - } - }, - "generation_config": { - "additional_details": { - "dataset": "\"openbookqa\"", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.739, - "details": { - "description": "min=0.52, mean=0.739, max=0.92, sum=3.694 (5)", - "tab": "Accuracy", - "MMLU - Observed inference time (s)": "{\"description\": \"min=0.309, mean=0.396, max=0.722, sum=1.982 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.39635124337045774\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=373.43, mean=467.686, max=614.421, sum=2338.431 (5)\", \"tab\": \"General information\", \"score\": \"467.6862105263158\"}", - "MMLU - # output tokens": "{\"description\": \"min=0.97, mean=0.99, max=1, sum=4.951 (5)\", \"tab\": \"General information\", \"score\": \"0.9902456140350877\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"college_chemistry\", \"computer_security\", \"econometrics\", \"us_foreign_policy\"]", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MATH", - "source_data": { - "dataset_name": "MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Equivalent (CoT) on MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.767, - "details": { - "description": "min=0.553, mean=0.767, max=0.948, sum=5.371 (7)", - "tab": "Accuracy", - "MATH - Observed inference time (s)": "{\"description\": \"min=5.13, mean=14.827, max=45.729, sum=103.786 (7)\", \"tab\": \"Efficiency\", \"score\": \"14.82662017363065\"}", - "MATH - # eval": "{\"description\": \"min=30, mean=62.429, max=135, sum=437 (7)\", \"tab\": \"General information\", \"score\": \"62.42857142857143\"}", - "MATH - # train": "{\"description\": \"min=8, mean=8, max=8, sum=56 (7)\", \"tab\": \"General information\", \"score\": \"8.0\"}", - "MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MATH - # prompt tokens": "{\"description\": \"min=881.363, mean=1262.909, max=2197.577, sum=8840.364 (7)\", \"tab\": \"General information\", \"score\": \"1262.9092130545007\"}", - "MATH - # output tokens": "{\"description\": \"min=174.547, mean=209.333, max=238.692, sum=1465.33 (7)\", \"tab\": \"General information\", \"score\": \"209.3327932233685\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"algebra\", \"counting_and_probability\", \"geometry\", \"intermediate_algebra\", \"number_theory\", \"prealgebra\", \"precalculus\"]", - "level": "\"1\"", - "use_official_examples": "\"False\"", - "use_chain_of_thought": "\"True\"", - "stop": "\"none\"" - } - } - }, - { - "evaluation_name": "GSM8K", - "source_data": { - "dataset_name": "GSM8K", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on GSM8K", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.905, - "details": { - "description": "min=0.905, mean=0.905, max=0.905, sum=0.905 (1)", - "tab": "Accuracy", - "GSM8K - Observed inference time (s)": "{\"description\": \"min=11.45, mean=11.45, max=11.45, sum=11.45 (1)\", \"tab\": \"Efficiency\", \"score\": \"11.449529441833496\"}", - "GSM8K - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "GSM8K - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "GSM8K - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GSM8K - # prompt tokens": "{\"description\": \"min=959.032, mean=959.032, max=959.032, sum=959.032 (1)\", \"tab\": \"General information\", \"score\": \"959.032\"}", - "GSM8K - # output tokens": "{\"description\": \"min=174.327, mean=174.327, max=174.327, sum=174.327 (1)\", \"tab\": \"General information\", \"score\": \"174.327\"}" - } - }, - "generation_config": { - "additional_details": { - "stop": "\"none\"" - } - } - }, - { - "evaluation_name": "LegalBench", - "source_data": { - "dataset_name": "LegalBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on LegalBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.73, - "details": { - "description": "min=0.433, mean=0.73, max=0.989, sum=3.648 (5)", - "tab": "Accuracy", - "LegalBench - Observed inference time (s)": "{\"description\": \"min=0.478, mean=0.504, max=0.522, sum=2.519 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.5037181089898329\"}", - "LegalBench - # eval": "{\"description\": \"min=95, mean=409.4, max=1000, sum=2047 (5)\", \"tab\": \"General information\", \"score\": \"409.4\"}", - "LegalBench - # train": "{\"description\": \"min=4, mean=4.798, max=5, sum=23.992 (5)\", \"tab\": \"General information\", \"score\": \"4.798367346938775\"}", - "LegalBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "LegalBench - # prompt tokens": "{\"description\": \"min=216.442, mean=1524.207, max=6297.633, sum=7621.033 (5)\", \"tab\": \"General information\", \"score\": \"1524.206501356544\"}", - "LegalBench - # output tokens": "{\"description\": \"min=1, mean=1.416, max=2.021, sum=7.082 (5)\", \"tab\": \"General information\", \"score\": \"1.4163162483866343\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"abercrombie\", \"corporate_lobbying\", \"function_of_decision_section\", \"international_citizenship_questions\", \"proa\"]", - "stop": "\"none\"" - } - } - }, - { - "evaluation_name": "MedQA", - "source_data": { - "dataset_name": "MedQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MedQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.775, - "details": { - "description": "min=0.775, mean=0.775, max=0.775, sum=0.775 (1)", - "tab": "Accuracy", - "MedQA - Observed inference time (s)": "{\"description\": \"min=0.399, mean=0.399, max=0.399, sum=0.399 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.39942375139498093\"}", - "MedQA - # eval": "{\"description\": \"min=503, mean=503, max=503, sum=503 (1)\", \"tab\": \"General information\", \"score\": \"503.0\"}", - "MedQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MedQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MedQA - # prompt tokens": "{\"description\": \"min=1025.274, mean=1025.274, max=1025.274, sum=1025.274 (1)\", \"tab\": \"General information\", \"score\": \"1025.2743538767395\"}", - "MedQA - # output tokens": "{\"description\": \"min=0.992, mean=0.992, max=0.992, sum=0.992 (1)\", \"tab\": \"General information\", \"score\": \"0.9920477137176938\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WMT 2014", - "source_data": { - "dataset_name": "WMT 2014", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "BLEU-4 on WMT 2014", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.203, - "details": { - "description": "min=0.144, mean=0.203, max=0.249, sum=1.016 (5)", - "tab": "Accuracy", - "WMT 2014 - Observed inference time (s)": "{\"description\": \"min=1.801, mean=2.046, max=2.515, sum=10.228 (5)\", \"tab\": \"Efficiency\", \"score\": \"2.045695114985284\"}", - "WMT 2014 - # eval": "{\"description\": \"min=503, mean=568.8, max=832, sum=2844 (5)\", \"tab\": \"General information\", \"score\": \"568.8\"}", - "WMT 2014 - # train": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "WMT 2014 - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WMT 2014 - # prompt tokens": "{\"description\": \"min=96.139, mean=115.712, max=136.117, sum=578.559 (5)\", \"tab\": \"General information\", \"score\": \"115.71178123566294\"}", - "WMT 2014 - # output tokens": "{\"description\": \"min=26.191, mean=29.362, max=37.718, sum=146.808 (5)\", \"tab\": \"General information\", \"score\": \"29.36160106667686\"}" - } - }, - "generation_config": { - "additional_details": { - "language_pair": "[\"cs-en\", \"de-en\", \"fr-en\", \"hi-en\", \"ru-en\"]", - "stop": "\"none\"" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_mmlu/writer_palmyra-x-004/1774096312.00548", - "retrieved_timestamp": "1774096312.00548", - "source_metadata": { - "source_name": "helm_mmlu", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_mmlu", - "evaluation_results": [ - { - "evaluation_name": "MMLU All Subjects", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU All Subjects", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.813, - "details": { - "description": "min=0.52, mean=0.813, max=0.959, sum=92.659 (114)", - "tab": "Accuracy", - "MMLU All Subjects - Observed inference time (s)": "{\"description\": \"min=0.298, mean=0.535, max=2.946, sum=60.962 (114)\", \"tab\": \"Efficiency\", \"score\": \"0.5347547453538\"}", - "MMLU All Subjects - # eval": "{\"description\": \"min=100, mean=246.351, max=1534, sum=28084 (114)\", \"tab\": \"General information\", \"score\": \"246.35087719298247\"}", - "MMLU All Subjects - # train": "{\"description\": \"min=5, mean=5, max=5, sum=570 (114)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU All Subjects - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU All Subjects - # prompt tokens": "{\"description\": \"min=274.52, mean=614.619, max=2797.885, sum=70066.61 (114)\", \"tab\": \"General information\", \"score\": \"614.6193817308517\"}", - "MMLU All Subjects - # output tokens": "{\"description\": \"min=0.968, mean=0.991, max=1, sum=112.995 (114)\", \"tab\": \"General information\", \"score\": \"0.9911842955118555\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_name": "Abstract Algebra", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Abstract Algebra", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.75, - "details": { - "description": "min=0.75, mean=0.75, max=0.75, sum=1.5 (2)", - "tab": "Accuracy", - "Abstract Algebra - Observed inference time (s)": "{\"description\": \"min=0.722, mean=0.722, max=0.722, sum=1.444 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7220739269256592\"}", - "Abstract Algebra - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Abstract Algebra - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Abstract Algebra - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Abstract Algebra - # prompt tokens": "{\"description\": \"min=373.43, mean=373.43, max=373.43, sum=746.86 (2)\", \"tab\": \"General information\", \"score\": \"373.43\"}", - "Abstract Algebra - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"abstract_algebra\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_abstract_algebra\"" - } - } - }, - { - "evaluation_name": "Anatomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Anatomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.822, - "details": { - "description": "min=0.822, mean=0.822, max=0.822, sum=1.644 (2)", - "tab": "Accuracy", - "Anatomy - Observed inference time (s)": "{\"description\": \"min=0.323, mean=0.323, max=0.323, sum=0.646 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3229873922136095\"}", - "Anatomy - # eval": "{\"description\": \"min=135, mean=135, max=135, sum=270 (2)\", \"tab\": \"General information\", \"score\": \"135.0\"}", - "Anatomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Anatomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Anatomy - # prompt tokens": "{\"description\": \"min=353.874, mean=353.874, max=353.874, sum=707.748 (2)\", \"tab\": \"General information\", \"score\": \"353.8740740740741\"}", - "Anatomy - # output tokens": "{\"description\": \"min=0.993, mean=0.993, max=0.993, sum=1.985 (2)\", \"tab\": \"General information\", \"score\": \"0.9925925925925926\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"anatomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_anatomy\"" - } - } - }, - { - "evaluation_name": "College Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on College Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.647, - "details": { - "description": "min=0.647, mean=0.647, max=0.647, sum=1.294 (2)", - "tab": "Accuracy", - "College Chemistry - Observed inference time (s)": "{\"description\": \"min=0.316, mean=0.316, max=0.316, sum=0.632 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.316190505027771\"}", - "College Biology - Observed inference time (s)": "{\"description\": \"min=2.087, mean=2.087, max=2.087, sum=4.175 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.0873730795250998\"}", - "College Computer Science - Observed inference time (s)": "{\"description\": \"min=1.575, mean=1.575, max=1.575, sum=3.15 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.574983057975769\"}", - "College Mathematics - Observed inference time (s)": "{\"description\": \"min=1.58, mean=1.58, max=1.58, sum=3.16 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.5799101972579956\"}", - "College Medicine - Observed inference time (s)": "{\"description\": \"min=1.786, mean=1.786, max=1.786, sum=3.572 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.786004883705536\"}", - "College Physics - Observed inference time (s)": "{\"description\": \"min=1.112, mean=1.112, max=1.112, sum=2.225 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.1123062372207642\"}", - "College Chemistry - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Chemistry - # prompt tokens": "{\"description\": \"min=549.28, mean=549.28, max=549.28, sum=1098.56 (2)\", \"tab\": \"General information\", \"score\": \"549.28\"}", - "College Chemistry - # output tokens": "{\"description\": \"min=0.97, mean=0.97, max=0.97, sum=1.94 (2)\", \"tab\": \"General information\", \"score\": \"0.97\"}", - "College Biology - # eval": "{\"description\": \"min=144, mean=144, max=144, sum=288 (2)\", \"tab\": \"General information\", \"score\": \"144.0\"}", - "College Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # prompt tokens": "{\"description\": \"min=473.875, mean=473.875, max=473.875, sum=947.75 (2)\", \"tab\": \"General information\", \"score\": \"473.875\"}", - "College Biology - # output tokens": "{\"description\": \"min=0.993, mean=0.993, max=0.993, sum=1.986 (2)\", \"tab\": \"General information\", \"score\": \"0.9930555555555556\"}", - "College Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # prompt tokens": "{\"description\": \"min=828.29, mean=828.29, max=828.29, sum=1656.58 (2)\", \"tab\": \"General information\", \"score\": \"828.29\"}", - "College Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Mathematics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # prompt tokens": "{\"description\": \"min=594.51, mean=594.51, max=594.51, sum=1189.02 (2)\", \"tab\": \"General information\", \"score\": \"594.51\"}", - "College Mathematics - # output tokens": "{\"description\": \"min=0.98, mean=0.98, max=0.98, sum=1.96 (2)\", \"tab\": \"General information\", \"score\": \"0.98\"}", - "College Medicine - # eval": "{\"description\": \"min=173, mean=173, max=173, sum=346 (2)\", \"tab\": \"General information\", \"score\": \"173.0\"}", - "College Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # prompt tokens": "{\"description\": \"min=502.705, mean=502.705, max=502.705, sum=1005.41 (2)\", \"tab\": \"General information\", \"score\": \"502.70520231213874\"}", - "College Medicine - # output tokens": "{\"description\": \"min=0.994, mean=0.994, max=0.994, sum=1.988 (2)\", \"tab\": \"General information\", \"score\": \"0.9942196531791907\"}", - "College Physics - # eval": "{\"description\": \"min=102, mean=102, max=102, sum=204 (2)\", \"tab\": \"General information\", \"score\": \"102.0\"}", - "College Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # prompt tokens": "{\"description\": \"min=503.569, mean=503.569, max=503.569, sum=1007.137 (2)\", \"tab\": \"General information\", \"score\": \"503.5686274509804\"}", - "College Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"college_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_college_physics\"" - } - } - }, - { - "evaluation_name": "Computer Security", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Computer Security", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.82, - "details": { - "description": "min=0.82, mean=0.82, max=0.82, sum=1.64 (2)", - "tab": "Accuracy", - "Computer Security - Observed inference time (s)": "{\"description\": \"min=0.309, mean=0.309, max=0.309, sum=0.618 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3091639161109924\"}", - "Computer Security - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Computer Security - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Computer Security - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Computer Security - # prompt tokens": "{\"description\": \"min=378.51, mean=378.51, max=378.51, sum=757.02 (2)\", \"tab\": \"General information\", \"score\": \"378.51\"}", - "Computer Security - # output tokens": "{\"description\": \"min=0.99, mean=0.99, max=0.99, sum=1.98 (2)\", \"tab\": \"General information\", \"score\": \"0.99\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"computer_security\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_computer_security\"" - } - } - }, - { - "evaluation_name": "Econometrics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Econometrics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.684, - "details": { - "description": "min=0.684, mean=0.684, max=0.684, sum=1.368 (2)", - "tab": "Accuracy", - "Econometrics - Observed inference time (s)": "{\"description\": \"min=0.322, mean=0.322, max=0.322, sum=0.644 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.32210456070147064\"}", - "Econometrics - # eval": "{\"description\": \"min=114, mean=114, max=114, sum=228 (2)\", \"tab\": \"General information\", \"score\": \"114.0\"}", - "Econometrics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Econometrics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Econometrics - # prompt tokens": "{\"description\": \"min=614.421, mean=614.421, max=614.421, sum=1228.842 (2)\", \"tab\": \"General information\", \"score\": \"614.421052631579\"}", - "Econometrics - # output tokens": "{\"description\": \"min=0.991, mean=0.991, max=0.991, sum=1.982 (2)\", \"tab\": \"General information\", \"score\": \"0.9912280701754386\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"econometrics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_econometrics\"" - } - } - }, - { - "evaluation_name": "Global Facts", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Global Facts", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.62, - "details": { - "description": "min=0.62, mean=0.62, max=0.62, sum=1.24 (2)", - "tab": "Accuracy", - "Global Facts - Observed inference time (s)": "{\"description\": \"min=0.311, mean=0.311, max=0.311, sum=0.621 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.31063568592071533\"}", - "Global Facts - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Global Facts - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Global Facts - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Global Facts - # prompt tokens": "{\"description\": \"min=399.71, mean=399.71, max=399.71, sum=799.42 (2)\", \"tab\": \"General information\", \"score\": \"399.71\"}", - "Global Facts - # output tokens": "{\"description\": \"min=0.98, mean=0.98, max=0.98, sum=1.96 (2)\", \"tab\": \"General information\", \"score\": \"0.98\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"global_facts\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_global_facts\"" - } - } - }, - { - "evaluation_name": "Jurisprudence", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Jurisprudence", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.843, - "details": { - "description": "min=0.843, mean=0.843, max=0.843, sum=1.685 (2)", - "tab": "Accuracy", - "Jurisprudence - Observed inference time (s)": "{\"description\": \"min=0.298, mean=0.298, max=0.298, sum=0.597 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.29833372433980304\"}", - "Jurisprudence - # eval": "{\"description\": \"min=108, mean=108, max=108, sum=216 (2)\", \"tab\": \"General information\", \"score\": \"108.0\"}", - "Jurisprudence - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Jurisprudence - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Jurisprudence - # prompt tokens": "{\"description\": \"min=394.63, mean=394.63, max=394.63, sum=789.259 (2)\", \"tab\": \"General information\", \"score\": \"394.6296296296296\"}", - "Jurisprudence - # output tokens": "{\"description\": \"min=0.991, mean=0.991, max=0.991, sum=1.981 (2)\", \"tab\": \"General information\", \"score\": \"0.9907407407407407\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"jurisprudence\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_jurisprudence\"" - } - } - }, - { - "evaluation_name": "Philosophy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Philosophy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.83, - "details": { - "description": "min=0.83, mean=0.83, max=0.83, sum=1.659 (2)", - "tab": "Accuracy", - "Philosophy - Observed inference time (s)": "{\"description\": \"min=0.306, mean=0.306, max=0.306, sum=0.612 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.30590631187537093\"}", - "Philosophy - # eval": "{\"description\": \"min=311, mean=311, max=311, sum=622 (2)\", \"tab\": \"General information\", \"score\": \"311.0\"}", - "Philosophy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Philosophy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Philosophy - # prompt tokens": "{\"description\": \"min=329.084, mean=329.084, max=329.084, sum=658.167 (2)\", \"tab\": \"General information\", \"score\": \"329.08360128617363\"}", - "Philosophy - # output tokens": "{\"description\": \"min=0.994, mean=0.994, max=0.994, sum=1.987 (2)\", \"tab\": \"General information\", \"score\": \"0.9935691318327974\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"philosophy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_philosophy\"" - } - } - }, - { - "evaluation_name": "Professional Psychology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Professional Psychology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.845, - "details": { - "description": "min=0.845, mean=0.845, max=0.845, sum=1.69 (2)", - "tab": "Accuracy", - "Professional Medicine - Observed inference time (s)": "{\"description\": \"min=0.42, mean=0.42, max=0.42, sum=0.841 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.42044701295740466\"}", - "Professional Accounting - Observed inference time (s)": "{\"description\": \"min=0.352, mean=0.352, max=0.352, sum=0.704 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.35206349944391996\"}", - "Professional Law - Observed inference time (s)": "{\"description\": \"min=2.946, mean=2.946, max=2.946, sum=5.892 (2)\", \"tab\": \"Efficiency\", \"score\": \"2.9459040923410784\"}", - "Professional Psychology - Observed inference time (s)": "{\"description\": \"min=0.342, mean=0.342, max=0.342, sum=0.683 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.34150391076904496\"}", - "Professional Medicine - # eval": "{\"description\": \"min=272, mean=272, max=272, sum=544 (2)\", \"tab\": \"General information\", \"score\": \"272.0\"}", - "Professional Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Medicine - # prompt tokens": "{\"description\": \"min=1094.489, mean=1094.489, max=1094.489, sum=2188.978 (2)\", \"tab\": \"General information\", \"score\": \"1094.4889705882354\"}", - "Professional Medicine - # output tokens": "{\"description\": \"min=0.989, mean=0.989, max=0.989, sum=1.978 (2)\", \"tab\": \"General information\", \"score\": \"0.9889705882352942\"}", - "Professional Accounting - # eval": "{\"description\": \"min=282, mean=282, max=282, sum=564 (2)\", \"tab\": \"General information\", \"score\": \"282.0\"}", - "Professional Accounting - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Accounting - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # prompt tokens": "{\"description\": \"min=658.585, mean=658.585, max=658.585, sum=1317.17 (2)\", \"tab\": \"General information\", \"score\": \"658.5851063829788\"}", - "Professional Accounting - # output tokens": "{\"description\": \"min=0.968, mean=0.968, max=0.968, sum=1.936 (2)\", \"tab\": \"General information\", \"score\": \"0.9680851063829787\"}", - "Professional Law - # eval": "{\"description\": \"min=1534, mean=1534, max=1534, sum=3068 (2)\", \"tab\": \"General information\", \"score\": \"1534.0\"}", - "Professional Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # prompt tokens": "{\"description\": \"min=1637.601, mean=1637.601, max=1637.601, sum=3275.202 (2)\", \"tab\": \"General information\", \"score\": \"1637.6010430247718\"}", - "Professional Law - # output tokens": "{\"description\": \"min=0.995, mean=0.995, max=0.995, sum=1.99 (2)\", \"tab\": \"General information\", \"score\": \"0.9947848761408083\"}", - "Professional Psychology - # eval": "{\"description\": \"min=612, mean=612, max=612, sum=1224 (2)\", \"tab\": \"General information\", \"score\": \"612.0\"}", - "Professional Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # prompt tokens": "{\"description\": \"min=575.098, mean=575.098, max=575.098, sum=1150.196 (2)\", \"tab\": \"General information\", \"score\": \"575.0980392156863\"}", - "Professional Psychology - # output tokens": "{\"description\": \"min=0.993, mean=0.993, max=0.993, sum=1.987 (2)\", \"tab\": \"General information\", \"score\": \"0.9934640522875817\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"professional_psychology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_professional_psychology\"" - } - } - }, - { - "evaluation_name": "Us Foreign Policy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Us Foreign Policy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.92, - "details": { - "description": "min=0.92, mean=0.92, max=0.92, sum=1.84 (2)", - "tab": "Accuracy", - "Us Foreign Policy - Observed inference time (s)": "{\"description\": \"min=0.312, mean=0.312, max=0.312, sum=0.624 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.31222330808639526\"}", - "Us Foreign Policy - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Us Foreign Policy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Us Foreign Policy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Us Foreign Policy - # prompt tokens": "{\"description\": \"min=422.79, mean=422.79, max=422.79, sum=845.58 (2)\", \"tab\": \"General information\", \"score\": \"422.79\"}", - "Us Foreign Policy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"us_foreign_policy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_us_foreign_policy\"" - } - } - }, - { - "evaluation_name": "Astronomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Astronomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.928, - "details": { - "description": "min=0.928, mean=0.928, max=0.928, sum=1.855 (2)", - "tab": "Accuracy", - "Astronomy - Observed inference time (s)": "{\"description\": \"min=0.326, mean=0.326, max=0.326, sum=0.653 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3264871161235006\"}", - "Astronomy - # eval": "{\"description\": \"min=152, mean=152, max=152, sum=304 (2)\", \"tab\": \"General information\", \"score\": \"152.0\"}", - "Astronomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Astronomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Astronomy - # prompt tokens": "{\"description\": \"min=579.684, mean=579.684, max=579.684, sum=1159.368 (2)\", \"tab\": \"General information\", \"score\": \"579.6842105263158\"}", - "Astronomy - # output tokens": "{\"description\": \"min=0.993, mean=0.993, max=0.993, sum=1.987 (2)\", \"tab\": \"General information\", \"score\": \"0.993421052631579\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"astronomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_astronomy\"" - } - } - }, - { - "evaluation_name": "Business Ethics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Business Ethics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.76, - "details": { - "description": "min=0.76, mean=0.76, max=0.76, sum=1.52 (2)", - "tab": "Accuracy", - "Business Ethics - Observed inference time (s)": "{\"description\": \"min=0.321, mean=0.321, max=0.321, sum=0.643 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3212712168693542\"}", - "Business Ethics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Business Ethics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Business Ethics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Business Ethics - # prompt tokens": "{\"description\": \"min=569.52, mean=569.52, max=569.52, sum=1139.04 (2)\", \"tab\": \"General information\", \"score\": \"569.52\"}", - "Business Ethics - # output tokens": "{\"description\": \"min=0.98, mean=0.98, max=0.98, sum=1.96 (2)\", \"tab\": \"General information\", \"score\": \"0.98\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"business_ethics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_business_ethics\"" - } - } - }, - { - "evaluation_name": "Clinical Knowledge", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Clinical Knowledge", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.879, - "details": { - "description": "min=0.879, mean=0.879, max=0.879, sum=1.758 (2)", - "tab": "Accuracy", - "Clinical Knowledge - Observed inference time (s)": "{\"description\": \"min=0.477, mean=0.477, max=0.477, sum=0.953 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4765495894090185\"}", - "Clinical Knowledge - # eval": "{\"description\": \"min=265, mean=265, max=265, sum=530 (2)\", \"tab\": \"General information\", \"score\": \"265.0\"}", - "Clinical Knowledge - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Clinical Knowledge - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Clinical Knowledge - # prompt tokens": "{\"description\": \"min=397.928, mean=397.928, max=397.928, sum=795.857 (2)\", \"tab\": \"General information\", \"score\": \"397.92830188679244\"}", - "Clinical Knowledge - # output tokens": "{\"description\": \"min=0.992, mean=0.992, max=0.992, sum=1.985 (2)\", \"tab\": \"General information\", \"score\": \"0.9924528301886792\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"clinical_knowledge\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_clinical_knowledge\"" - } - } - }, - { - "evaluation_name": "Conceptual Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Conceptual Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.885, - "details": { - "description": "min=0.885, mean=0.885, max=0.885, sum=1.77 (2)", - "tab": "Accuracy", - "Conceptual Physics - Observed inference time (s)": "{\"description\": \"min=0.347, mean=0.347, max=0.347, sum=0.693 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3465714748869551\"}", - "Conceptual Physics - # eval": "{\"description\": \"min=235, mean=235, max=235, sum=470 (2)\", \"tab\": \"General information\", \"score\": \"235.0\"}", - "Conceptual Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Conceptual Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Conceptual Physics - # prompt tokens": "{\"description\": \"min=304.834, mean=304.834, max=304.834, sum=609.668 (2)\", \"tab\": \"General information\", \"score\": \"304.83404255319147\"}", - "Conceptual Physics - # output tokens": "{\"description\": \"min=0.996, mean=0.996, max=0.996, sum=1.991 (2)\", \"tab\": \"General information\", \"score\": \"0.9957446808510638\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"conceptual_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_conceptual_physics\"" - } - } - }, - { - "evaluation_name": "Electrical Engineering", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Electrical Engineering", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.793, - "details": { - "description": "min=0.793, mean=0.793, max=0.793, sum=1.586 (2)", - "tab": "Accuracy", - "Electrical Engineering - Observed inference time (s)": "{\"description\": \"min=0.305, mean=0.305, max=0.305, sum=0.611 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3054168865598481\"}", - "Electrical Engineering - # eval": "{\"description\": \"min=145, mean=145, max=145, sum=290 (2)\", \"tab\": \"General information\", \"score\": \"145.0\"}", - "Electrical Engineering - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Electrical Engineering - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Electrical Engineering - # prompt tokens": "{\"description\": \"min=435.607, mean=435.607, max=435.607, sum=871.214 (2)\", \"tab\": \"General information\", \"score\": \"435.60689655172416\"}", - "Electrical Engineering - # output tokens": "{\"description\": \"min=0.993, mean=0.993, max=0.993, sum=1.986 (2)\", \"tab\": \"General information\", \"score\": \"0.993103448275862\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"electrical_engineering\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_electrical_engineering\"" - } - } - }, - { - "evaluation_name": "Elementary Mathematics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Elementary Mathematics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.841, - "details": { - "description": "min=0.841, mean=0.841, max=0.841, sum=1.683 (2)", - "tab": "Accuracy", - "Elementary Mathematics - Observed inference time (s)": "{\"description\": \"min=0.313, mean=0.313, max=0.313, sum=0.627 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.31325215069705215\"}", - "Elementary Mathematics - # eval": "{\"description\": \"min=378, mean=378, max=378, sum=756 (2)\", \"tab\": \"General information\", \"score\": \"378.0\"}", - "Elementary Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Elementary Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Elementary Mathematics - # prompt tokens": "{\"description\": \"min=531.854, mean=531.854, max=531.854, sum=1063.709 (2)\", \"tab\": \"General information\", \"score\": \"531.8544973544973\"}", - "Elementary Mathematics - # output tokens": "{\"description\": \"min=0.995, mean=0.995, max=0.995, sum=1.989 (2)\", \"tab\": \"General information\", \"score\": \"0.9947089947089947\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"elementary_mathematics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_elementary_mathematics\"" - } - } - }, - { - "evaluation_name": "Formal Logic", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Formal Logic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.579, - "details": { - "description": "min=0.579, mean=0.579, max=0.579, sum=1.159 (2)", - "tab": "Accuracy", - "Formal Logic - Observed inference time (s)": "{\"description\": \"min=1.035, mean=1.035, max=1.035, sum=2.07 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.034958042795696\"}", - "Formal Logic - # eval": "{\"description\": \"min=126, mean=126, max=126, sum=252 (2)\", \"tab\": \"General information\", \"score\": \"126.0\"}", - "Formal Logic - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Formal Logic - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Formal Logic - # prompt tokens": "{\"description\": \"min=601.778, mean=601.778, max=601.778, sum=1203.556 (2)\", \"tab\": \"General information\", \"score\": \"601.7777777777778\"}", - "Formal Logic - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"formal_logic\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_formal_logic\"" - } - } - }, - { - "evaluation_name": "High School World History", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on High School World History", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.911, - "details": { - "description": "min=0.911, mean=0.911, max=0.911, sum=1.823 (2)", - "tab": "Accuracy", - "High School Biology - Observed inference time (s)": "{\"description\": \"min=0.562, mean=0.562, max=0.562, sum=1.123 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.561508382520368\"}", - "High School Chemistry - Observed inference time (s)": "{\"description\": \"min=0.349, mean=0.349, max=0.349, sum=0.698 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.34899539900530735\"}", - "High School Computer Science - Observed inference time (s)": "{\"description\": \"min=0.423, mean=0.423, max=0.423, sum=0.845 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4227438974380493\"}", - "High School European History - Observed inference time (s)": "{\"description\": \"min=0.899, mean=0.899, max=0.899, sum=1.799 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8994465018763687\"}", - "High School Geography - Observed inference time (s)": "{\"description\": \"min=0.324, mean=0.324, max=0.324, sum=0.647 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3236422189558395\"}", - "High School Government And Politics - Observed inference time (s)": "{\"description\": \"min=0.314, mean=0.314, max=0.314, sum=0.627 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.31354672550537427\"}", - "High School Macroeconomics - Observed inference time (s)": "{\"description\": \"min=0.314, mean=0.314, max=0.314, sum=0.628 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.31394460568061244\"}", - "High School Mathematics - Observed inference time (s)": "{\"description\": \"min=0.315, mean=0.315, max=0.315, sum=0.63 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3151667806837294\"}", - "High School Microeconomics - Observed inference time (s)": "{\"description\": \"min=0.315, mean=0.315, max=0.315, sum=0.63 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3151869453301951\"}", - "High School Physics - Observed inference time (s)": "{\"description\": \"min=0.32, mean=0.32, max=0.32, sum=0.639 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.31971652302520953\"}", - "High School Psychology - Observed inference time (s)": "{\"description\": \"min=0.315, mean=0.315, max=0.315, sum=0.63 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3149662079067405\"}", - "High School Statistics - Observed inference time (s)": "{\"description\": \"min=0.386, mean=0.386, max=0.386, sum=0.772 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3859624167283376\"}", - "High School US History - Observed inference time (s)": "{\"description\": \"min=0.651, mean=0.651, max=0.651, sum=1.303 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6513510615217919\"}", - "High School World History - Observed inference time (s)": "{\"description\": \"min=0.472, mean=0.472, max=0.472, sum=0.945 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.4723552480528626\"}", - "High School Biology - # eval": "{\"description\": \"min=310, mean=310, max=310, sum=620 (2)\", \"tab\": \"General information\", \"score\": \"310.0\"}", - "High School Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Biology - # prompt tokens": "{\"description\": \"min=513.671, mean=513.671, max=513.671, sum=1027.342 (2)\", \"tab\": \"General information\", \"score\": \"513.6709677419354\"}", - "High School Biology - # output tokens": "{\"description\": \"min=0.994, mean=0.994, max=0.994, sum=1.987 (2)\", \"tab\": \"General information\", \"score\": \"0.9935483870967742\"}", - "High School Chemistry - # eval": "{\"description\": \"min=203, mean=203, max=203, sum=406 (2)\", \"tab\": \"General information\", \"score\": \"203.0\"}", - "High School Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # prompt tokens": "{\"description\": \"min=496.704, mean=496.704, max=496.704, sum=993.409 (2)\", \"tab\": \"General information\", \"score\": \"496.70443349753697\"}", - "High School Chemistry - # output tokens": "{\"description\": \"min=0.985, mean=0.985, max=0.985, sum=1.97 (2)\", \"tab\": \"General information\", \"score\": \"0.9852216748768473\"}", - "High School Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "High School Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # prompt tokens": "{\"description\": \"min=867.78, mean=867.78, max=867.78, sum=1735.56 (2)\", \"tab\": \"General information\", \"score\": \"867.78\"}", - "High School Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School European History - # eval": "{\"description\": \"min=165, mean=165, max=165, sum=330 (2)\", \"tab\": \"General information\", \"score\": \"165.0\"}", - "High School European History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School European History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # prompt tokens": "{\"description\": \"min=2797.885, mean=2797.885, max=2797.885, sum=5595.77 (2)\", \"tab\": \"General information\", \"score\": \"2797.8848484848486\"}", - "High School European History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Geography - # eval": "{\"description\": \"min=198, mean=198, max=198, sum=396 (2)\", \"tab\": \"General information\", \"score\": \"198.0\"}", - "High School Geography - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Geography - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # prompt tokens": "{\"description\": \"min=372.035, mean=372.035, max=372.035, sum=744.071 (2)\", \"tab\": \"General information\", \"score\": \"372.0353535353535\"}", - "High School Geography - # output tokens": "{\"description\": \"min=0.99, mean=0.99, max=0.99, sum=1.98 (2)\", \"tab\": \"General information\", \"score\": \"0.98989898989899\"}", - "High School Government And Politics - # eval": "{\"description\": \"min=193, mean=193, max=193, sum=386 (2)\", \"tab\": \"General information\", \"score\": \"193.0\"}", - "High School Government And Politics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Government And Politics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # prompt tokens": "{\"description\": \"min=465.824, mean=465.824, max=465.824, sum=931.648 (2)\", \"tab\": \"General information\", \"score\": \"465.8238341968912\"}", - "High School Government And Politics - # output tokens": "{\"description\": \"min=0.979, mean=0.979, max=0.979, sum=1.959 (2)\", \"tab\": \"General information\", \"score\": \"0.9792746113989638\"}", - "High School Macroeconomics - # eval": "{\"description\": \"min=390, mean=390, max=390, sum=780 (2)\", \"tab\": \"General information\", \"score\": \"390.0\"}", - "High School Macroeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Macroeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # prompt tokens": "{\"description\": \"min=370.908, mean=370.908, max=370.908, sum=741.815 (2)\", \"tab\": \"General information\", \"score\": \"370.9076923076923\"}", - "High School Macroeconomics - # output tokens": "{\"description\": \"min=0.992, mean=0.992, max=0.992, sum=1.985 (2)\", \"tab\": \"General information\", \"score\": \"0.9923076923076923\"}", - "High School Mathematics - # eval": "{\"description\": \"min=270, mean=270, max=270, sum=540 (2)\", \"tab\": \"General information\", \"score\": \"270.0\"}", - "High School Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # prompt tokens": "{\"description\": \"min=532.356, mean=532.356, max=532.356, sum=1064.711 (2)\", \"tab\": \"General information\", \"score\": \"532.3555555555556\"}", - "High School Mathematics - # output tokens": "{\"description\": \"min=0.993, mean=0.993, max=0.993, sum=1.985 (2)\", \"tab\": \"General information\", \"score\": \"0.9925925925925926\"}", - "High School Microeconomics - # eval": "{\"description\": \"min=238, mean=238, max=238, sum=476 (2)\", \"tab\": \"General information\", \"score\": \"238.0\"}", - "High School Microeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Microeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # prompt tokens": "{\"description\": \"min=399.013, mean=399.013, max=399.013, sum=798.025 (2)\", \"tab\": \"General information\", \"score\": \"399.0126050420168\"}", - "High School Microeconomics - # output tokens": "{\"description\": \"min=0.987, mean=0.987, max=0.987, sum=1.975 (2)\", \"tab\": \"General information\", \"score\": \"0.9873949579831933\"}", - "High School Physics - # eval": "{\"description\": \"min=151, mean=151, max=151, sum=302 (2)\", \"tab\": \"General information\", \"score\": \"151.0\"}", - "High School Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # prompt tokens": "{\"description\": \"min=560.457, mean=560.457, max=560.457, sum=1120.914 (2)\", \"tab\": \"General information\", \"score\": \"560.4569536423841\"}", - "High School Physics - # output tokens": "{\"description\": \"min=0.974, mean=0.974, max=0.974, sum=1.947 (2)\", \"tab\": \"General information\", \"score\": \"0.9735099337748344\"}", - "High School Psychology - # eval": "{\"description\": \"min=545, mean=545, max=545, sum=1090 (2)\", \"tab\": \"General information\", \"score\": \"545.0\"}", - "High School Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # prompt tokens": "{\"description\": \"min=495.242, mean=495.242, max=495.242, sum=990.484 (2)\", \"tab\": \"General information\", \"score\": \"495.2422018348624\"}", - "High School Psychology - # output tokens": "{\"description\": \"min=0.996, mean=0.996, max=0.996, sum=1.993 (2)\", \"tab\": \"General information\", \"score\": \"0.9963302752293578\"}", - "High School Statistics - # eval": "{\"description\": \"min=216, mean=216, max=216, sum=432 (2)\", \"tab\": \"General information\", \"score\": \"216.0\"}", - "High School Statistics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Statistics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # prompt tokens": "{\"description\": \"min=795.639, mean=795.639, max=795.639, sum=1591.278 (2)\", \"tab\": \"General information\", \"score\": \"795.6388888888889\"}", - "High School Statistics - # output tokens": "{\"description\": \"min=0.977, mean=0.977, max=0.977, sum=1.954 (2)\", \"tab\": \"General information\", \"score\": \"0.9768518518518519\"}", - "High School US History - # eval": "{\"description\": \"min=204, mean=204, max=204, sum=408 (2)\", \"tab\": \"General information\", \"score\": \"204.0\"}", - "High School US History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School US History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # prompt tokens": "{\"description\": \"min=2217.809, mean=2217.809, max=2217.809, sum=4435.618 (2)\", \"tab\": \"General information\", \"score\": \"2217.8088235294117\"}", - "High School US History - # output tokens": "{\"description\": \"min=0.99, mean=0.99, max=0.99, sum=1.98 (2)\", \"tab\": \"General information\", \"score\": \"0.9901960784313726\"}", - "High School World History - # eval": "{\"description\": \"min=237, mean=237, max=237, sum=474 (2)\", \"tab\": \"General information\", \"score\": \"237.0\"}", - "High School World History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School World History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # prompt tokens": "{\"description\": \"min=1428.173, mean=1428.173, max=1428.173, sum=2856.346 (2)\", \"tab\": \"General information\", \"score\": \"1428.1729957805908\"}", - "High School World History - # output tokens": "{\"description\": \"min=0.996, mean=0.996, max=0.996, sum=1.992 (2)\", \"tab\": \"General information\", \"score\": \"0.9957805907172996\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"high_school_world_history\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_high_school_world_history\"" - } - } - }, - { - "evaluation_name": "Human Sexuality", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Human Sexuality", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.924, - "details": { - "description": "min=0.924, mean=0.924, max=0.924, sum=1.847 (2)", - "tab": "Accuracy", - "Human Aging - Observed inference time (s)": "{\"description\": \"min=0.322, mean=0.322, max=0.322, sum=0.644 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3221198432648663\"}", - "Human Sexuality - Observed inference time (s)": "{\"description\": \"min=0.319, mean=0.319, max=0.319, sum=0.638 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.31875184474100593\"}", - "Human Aging - # eval": "{\"description\": \"min=223, mean=223, max=223, sum=446 (2)\", \"tab\": \"General information\", \"score\": \"223.0\"}", - "Human Aging - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Aging - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Aging - # prompt tokens": "{\"description\": \"min=319.888, mean=319.888, max=319.888, sum=639.776 (2)\", \"tab\": \"General information\", \"score\": \"319.88789237668163\"}", - "Human Aging - # output tokens": "{\"description\": \"min=0.996, mean=0.996, max=0.996, sum=1.991 (2)\", \"tab\": \"General information\", \"score\": \"0.9955156950672646\"}", - "Human Sexuality - # eval": "{\"description\": \"min=131, mean=131, max=131, sum=262 (2)\", \"tab\": \"General information\", \"score\": \"131.0\"}", - "Human Sexuality - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Sexuality - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # prompt tokens": "{\"description\": \"min=341.168, mean=341.168, max=341.168, sum=682.336 (2)\", \"tab\": \"General information\", \"score\": \"341.1679389312977\"}", - "Human Sexuality - # output tokens": "{\"description\": \"min=0.992, mean=0.992, max=0.992, sum=1.985 (2)\", \"tab\": \"General information\", \"score\": \"0.9923664122137404\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"human_sexuality\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_human_sexuality\"" - } - } - }, - { - "evaluation_name": "International Law", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on International Law", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.901, - "details": { - "description": "min=0.901, mean=0.901, max=0.901, sum=1.802 (2)", - "tab": "Accuracy", - "International Law - Observed inference time (s)": "{\"description\": \"min=0.336, mean=0.336, max=0.336, sum=0.671 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.33550412989844963\"}", - "International Law - # eval": "{\"description\": \"min=121, mean=121, max=121, sum=242 (2)\", \"tab\": \"General information\", \"score\": \"121.0\"}", - "International Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "International Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "International Law - # prompt tokens": "{\"description\": \"min=639.818, mean=639.818, max=639.818, sum=1279.636 (2)\", \"tab\": \"General information\", \"score\": \"639.8181818181819\"}", - "International Law - # output tokens": "{\"description\": \"min=0.983, mean=0.983, max=0.983, sum=1.967 (2)\", \"tab\": \"General information\", \"score\": \"0.9834710743801653\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"international_law\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_international_law\"" - } - } - }, - { - "evaluation_name": "Logical Fallacies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Logical Fallacies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.877, - "details": { - "description": "min=0.877, mean=0.877, max=0.877, sum=1.755 (2)", - "tab": "Accuracy", - "Logical Fallacies - Observed inference time (s)": "{\"description\": \"min=0.312, mean=0.312, max=0.312, sum=0.624 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3120760069302986\"}", - "Logical Fallacies - # eval": "{\"description\": \"min=163, mean=163, max=163, sum=326 (2)\", \"tab\": \"General information\", \"score\": \"163.0\"}", - "Logical Fallacies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Logical Fallacies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Logical Fallacies - # prompt tokens": "{\"description\": \"min=449.564, mean=449.564, max=449.564, sum=899.129 (2)\", \"tab\": \"General information\", \"score\": \"449.5644171779141\"}", - "Logical Fallacies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"logical_fallacies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_logical_fallacies\"" - } - } - }, - { - "evaluation_name": "Machine Learning", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Machine Learning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.679, - "details": { - "description": "min=0.679, mean=0.679, max=0.679, sum=1.357 (2)", - "tab": "Accuracy", - "Machine Learning - Observed inference time (s)": "{\"description\": \"min=0.337, mean=0.337, max=0.337, sum=0.674 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3368471988609859\"}", - "Machine Learning - # eval": "{\"description\": \"min=112, mean=112, max=112, sum=224 (2)\", \"tab\": \"General information\", \"score\": \"112.0\"}", - "Machine Learning - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Machine Learning - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Machine Learning - # prompt tokens": "{\"description\": \"min=668.054, mean=668.054, max=668.054, sum=1336.107 (2)\", \"tab\": \"General information\", \"score\": \"668.0535714285714\"}", - "Machine Learning - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"machine_learning\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_machine_learning\"" - } - } - }, - { - "evaluation_name": "Management", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Management", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.903, - "details": { - "description": "min=0.903, mean=0.903, max=0.903, sum=1.806 (2)", - "tab": "Accuracy", - "Management - Observed inference time (s)": "{\"description\": \"min=0.31, mean=0.31, max=0.31, sum=0.621 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3103753525076561\"}", - "Management - # eval": "{\"description\": \"min=103, mean=103, max=103, sum=206 (2)\", \"tab\": \"General information\", \"score\": \"103.0\"}", - "Management - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Management - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Management - # prompt tokens": "{\"description\": \"min=283.786, mean=283.786, max=283.786, sum=567.573 (2)\", \"tab\": \"General information\", \"score\": \"283.7864077669903\"}", - "Management - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"management\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_management\"" - } - } - }, - { - "evaluation_name": "Marketing", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Marketing", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.932, - "details": { - "description": "min=0.932, mean=0.932, max=0.932, sum=1.863 (2)", - "tab": "Accuracy", - "Marketing - Observed inference time (s)": "{\"description\": \"min=0.314, mean=0.314, max=0.314, sum=0.628 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3138112644863944\"}", - "Marketing - # eval": "{\"description\": \"min=234, mean=234, max=234, sum=468 (2)\", \"tab\": \"General information\", \"score\": \"234.0\"}", - "Marketing - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Marketing - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Marketing - # prompt tokens": "{\"description\": \"min=404.218, mean=404.218, max=404.218, sum=808.436 (2)\", \"tab\": \"General information\", \"score\": \"404.21794871794873\"}", - "Marketing - # output tokens": "{\"description\": \"min=0.991, mean=0.991, max=0.991, sum=1.983 (2)\", \"tab\": \"General information\", \"score\": \"0.9914529914529915\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"marketing\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_marketing\"" - } - } - }, - { - "evaluation_name": "Medical Genetics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Medical Genetics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.87, - "details": { - "description": "min=0.87, mean=0.87, max=0.87, sum=1.74 (2)", - "tab": "Accuracy", - "Medical Genetics - Observed inference time (s)": "{\"description\": \"min=0.31, mean=0.31, max=0.31, sum=0.619 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3096977710723877\"}", - "Medical Genetics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Medical Genetics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Medical Genetics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Medical Genetics - # prompt tokens": "{\"description\": \"min=340.99, mean=340.99, max=340.99, sum=681.98 (2)\", \"tab\": \"General information\", \"score\": \"340.99\"}", - "Medical Genetics - # output tokens": "{\"description\": \"min=0.97, mean=0.97, max=0.97, sum=1.94 (2)\", \"tab\": \"General information\", \"score\": \"0.97\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"medical_genetics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_medical_genetics\"" - } - } - }, - { - "evaluation_name": "Miscellaneous", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Miscellaneous", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.934, - "details": { - "description": "min=0.934, mean=0.934, max=0.934, sum=1.867 (2)", - "tab": "Accuracy", - "Miscellaneous - Observed inference time (s)": "{\"description\": \"min=0.311, mean=0.311, max=0.311, sum=0.621 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3106613128730316\"}", - "Miscellaneous - # eval": "{\"description\": \"min=783, mean=783, max=783, sum=1566 (2)\", \"tab\": \"General information\", \"score\": \"783.0\"}", - "Miscellaneous - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Miscellaneous - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Miscellaneous - # prompt tokens": "{\"description\": \"min=299.911, mean=299.911, max=299.911, sum=599.821 (2)\", \"tab\": \"General information\", \"score\": \"299.9106002554278\"}", - "Miscellaneous - # output tokens": "{\"description\": \"min=0.99, mean=0.99, max=0.99, sum=1.98 (2)\", \"tab\": \"General information\", \"score\": \"0.9897828863346104\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"miscellaneous\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_miscellaneous\"" - } - } - }, - { - "evaluation_name": "Moral Scenarios", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Moral Scenarios", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.825, - "details": { - "description": "min=0.825, mean=0.825, max=0.825, sum=1.649 (2)", - "tab": "Accuracy", - "Moral Disputes - Observed inference time (s)": "{\"description\": \"min=0.313, mean=0.313, max=0.313, sum=0.626 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.31282479501184013\"}", - "Moral Scenarios - Observed inference time (s)": "{\"description\": \"min=0.335, mean=0.335, max=0.335, sum=0.67 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3348748574709759\"}", - "Moral Disputes - # eval": "{\"description\": \"min=346, mean=346, max=346, sum=692 (2)\", \"tab\": \"General information\", \"score\": \"346.0\"}", - "Moral Disputes - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Disputes - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Disputes - # prompt tokens": "{\"description\": \"min=476.113, mean=476.113, max=476.113, sum=952.225 (2)\", \"tab\": \"General information\", \"score\": \"476.1127167630058\"}", - "Moral Disputes - # output tokens": "{\"description\": \"min=0.994, mean=0.994, max=0.994, sum=1.988 (2)\", \"tab\": \"General information\", \"score\": \"0.9942196531791907\"}", - "Moral Scenarios - # eval": "{\"description\": \"min=895, mean=895, max=895, sum=1790 (2)\", \"tab\": \"General information\", \"score\": \"895.0\"}", - "Moral Scenarios - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Scenarios - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # prompt tokens": "{\"description\": \"min=656.455, mean=656.455, max=656.455, sum=1312.909 (2)\", \"tab\": \"General information\", \"score\": \"656.454748603352\"}", - "Moral Scenarios - # output tokens": "{\"description\": \"min=0.993, mean=0.993, max=0.993, sum=1.987 (2)\", \"tab\": \"General information\", \"score\": \"0.9932960893854749\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"moral_scenarios\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_moral_scenarios\"" - } - } - }, - { - "evaluation_name": "Nutrition", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Nutrition", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.869, - "details": { - "description": "min=0.869, mean=0.869, max=0.869, sum=1.739 (2)", - "tab": "Accuracy", - "Nutrition - Observed inference time (s)": "{\"description\": \"min=0.332, mean=0.332, max=0.332, sum=0.664 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.33182784311132496\"}", - "Nutrition - # eval": "{\"description\": \"min=306, mean=306, max=306, sum=612 (2)\", \"tab\": \"General information\", \"score\": \"306.0\"}", - "Nutrition - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Nutrition - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Nutrition - # prompt tokens": "{\"description\": \"min=586.814, mean=586.814, max=586.814, sum=1173.627 (2)\", \"tab\": \"General information\", \"score\": \"586.8137254901961\"}", - "Nutrition - # output tokens": "{\"description\": \"min=0.997, mean=0.997, max=0.997, sum=1.993 (2)\", \"tab\": \"General information\", \"score\": \"0.9967320261437909\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"nutrition\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_nutrition\"" - } - } - }, - { - "evaluation_name": "Prehistory", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Prehistory", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.917, - "details": { - "description": "min=0.917, mean=0.917, max=0.917, sum=1.833 (2)", - "tab": "Accuracy", - "Prehistory - Observed inference time (s)": "{\"description\": \"min=0.316, mean=0.316, max=0.316, sum=0.632 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3158548356574259\"}", - "Prehistory - # eval": "{\"description\": \"min=324, mean=324, max=324, sum=648 (2)\", \"tab\": \"General information\", \"score\": \"324.0\"}", - "Prehistory - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Prehistory - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Prehistory - # prompt tokens": "{\"description\": \"min=514.528, mean=514.528, max=514.528, sum=1029.056 (2)\", \"tab\": \"General information\", \"score\": \"514.5277777777778\"}", - "Prehistory - # output tokens": "{\"description\": \"min=0.988, mean=0.988, max=0.988, sum=1.975 (2)\", \"tab\": \"General information\", \"score\": \"0.9876543209876543\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"prehistory\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_prehistory\"" - } - } - }, - { - "evaluation_name": "Public Relations", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Public Relations", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.791, - "details": { - "description": "min=0.791, mean=0.791, max=0.791, sum=1.582 (2)", - "tab": "Accuracy", - "Public Relations - Observed inference time (s)": "{\"description\": \"min=0.328, mean=0.328, max=0.328, sum=0.657 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.32829454161904076\"}", - "Public Relations - # eval": "{\"description\": \"min=110, mean=110, max=110, sum=220 (2)\", \"tab\": \"General information\", \"score\": \"110.0\"}", - "Public Relations - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Public Relations - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Public Relations - # prompt tokens": "{\"description\": \"min=405.318, mean=405.318, max=405.318, sum=810.636 (2)\", \"tab\": \"General information\", \"score\": \"405.3181818181818\"}", - "Public Relations - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"public_relations\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_public_relations\"" - } - } - }, - { - "evaluation_name": "Security Studies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Security Studies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.849, - "details": { - "description": "min=0.849, mean=0.849, max=0.849, sum=1.698 (2)", - "tab": "Accuracy", - "Security Studies - Observed inference time (s)": "{\"description\": \"min=0.443, mean=0.443, max=0.443, sum=0.886 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.44323594618816764\"}", - "Security Studies - # eval": "{\"description\": \"min=245, mean=245, max=245, sum=490 (2)\", \"tab\": \"General information\", \"score\": \"245.0\"}", - "Security Studies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Security Studies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Security Studies - # prompt tokens": "{\"description\": \"min=1164.473, mean=1164.473, max=1164.473, sum=2328.947 (2)\", \"tab\": \"General information\", \"score\": \"1164.4734693877551\"}", - "Security Studies - # output tokens": "{\"description\": \"min=0.992, mean=0.992, max=0.992, sum=1.984 (2)\", \"tab\": \"General information\", \"score\": \"0.9918367346938776\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"security_studies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_security_studies\"" - } - } - }, - { - "evaluation_name": "Sociology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Sociology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.915, - "details": { - "description": "min=0.915, mean=0.915, max=0.915, sum=1.831 (2)", - "tab": "Accuracy", - "Sociology - Observed inference time (s)": "{\"description\": \"min=0.337, mean=0.337, max=0.337, sum=0.674 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.336861949654954\"}", - "Sociology - # eval": "{\"description\": \"min=201, mean=201, max=201, sum=402 (2)\", \"tab\": \"General information\", \"score\": \"201.0\"}", - "Sociology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Sociology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Sociology - # prompt tokens": "{\"description\": \"min=445.517, mean=445.517, max=445.517, sum=891.035 (2)\", \"tab\": \"General information\", \"score\": \"445.51741293532336\"}", - "Sociology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"sociology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_sociology\"" - } - } - }, - { - "evaluation_name": "Virology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Virology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.584, - "details": { - "description": "min=0.584, mean=0.584, max=0.584, sum=1.169 (2)", - "tab": "Accuracy", - "Virology - Observed inference time (s)": "{\"description\": \"min=0.328, mean=0.328, max=0.328, sum=0.656 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.32804813155208723\"}", - "Virology - # eval": "{\"description\": \"min=166, mean=166, max=166, sum=332 (2)\", \"tab\": \"General information\", \"score\": \"166.0\"}", - "Virology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Virology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Virology - # prompt tokens": "{\"description\": \"min=343.018, mean=343.018, max=343.018, sum=686.036 (2)\", \"tab\": \"General information\", \"score\": \"343.01807228915663\"}", - "Virology - # output tokens": "{\"description\": \"min=0.994, mean=0.994, max=0.994, sum=1.988 (2)\", \"tab\": \"General information\", \"score\": \"0.9939759036144579\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"virology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_virology\"" - } - } - }, - { - "evaluation_name": "World Religions", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on World Religions", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.842, - "details": { - "description": "min=0.842, mean=0.842, max=0.842, sum=1.684 (2)", - "tab": "Accuracy", - "World Religions - Observed inference time (s)": "{\"description\": \"min=0.376, mean=0.376, max=0.376, sum=0.752 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.3761981662951018\"}", - "World Religions - # eval": "{\"description\": \"min=171, mean=171, max=171, sum=342 (2)\", \"tab\": \"General information\", \"score\": \"171.0\"}", - "World Religions - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "World Religions - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "World Religions - # prompt tokens": "{\"description\": \"min=274.52, mean=274.52, max=274.52, sum=549.041 (2)\", \"tab\": \"General information\", \"score\": \"274.5204678362573\"}", - "World Religions - # output tokens": "{\"description\": \"min=0.994, mean=0.994, max=0.994, sum=1.988 (2)\", \"tab\": \"General information\", \"score\": \"0.9941520467836257\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"world_religions\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_world_religions\"" - } - } - }, - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.629, - "details": { - "description": "", - "tab": "Efficiency" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/writer_palmyra-x-v2.json b/data/models/writer_palmyra-x-v2.json deleted file mode 100644 index cdcf25ee633fa99d69074e41cd7f90346c62f02d..0000000000000000000000000000000000000000 --- a/data/models/writer_palmyra-x-v2.json +++ /dev/null @@ -1,380 +0,0 @@ -{ - "model_info": { - "name": "Palmyra X V2 33B", - "id": "writer/palmyra-x-v2", - "developer": "writer", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_lite/writer_palmyra-x-v2/1774096306.427425", - "retrieved_timestamp": "1774096306.427425", - "source_metadata": { - "source_name": "helm_lite", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_lite", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_lite", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.589, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.5062546816479401\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.752, - "details": { - "description": "min=0.752, mean=0.752, max=0.752, sum=0.752 (1)", - "tab": "Accuracy", - "NarrativeQA - Observed inference time (s)": "{\"description\": \"min=1.202, mean=1.202, max=1.202, sum=1.202 (1)\", \"tab\": \"Efficiency\", \"score\": \"1.2016644296511798\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=3504.577, mean=3504.577, max=3504.577, sum=3504.577 (1)\", \"tab\": \"General information\", \"score\": \"3504.5774647887324\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=8.208, mean=8.208, max=8.208, sum=8.208 (1)\", \"tab\": \"General information\", \"score\": \"8.208450704225353\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (closed-book)", - "source_data": { - "dataset_name": "NaturalQuestions (closed-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (closed-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.428, - "details": { - "description": "min=0.428, mean=0.428, max=0.428, sum=0.428 (1)", - "tab": "Accuracy", - "NaturalQuestions (open-book) - Observed inference time (s)": "{\"description\": \"min=0.969, mean=0.969, max=0.969, sum=0.969 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.9688332653045655\"}", - "NaturalQuestions (closed-book) - Observed inference time (s)": "{\"description\": \"min=0.62, mean=0.62, max=0.62, sum=0.62 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.6202523970603943\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.926, mean=4.926, max=4.926, sum=4.926 (1)\", \"tab\": \"General information\", \"score\": \"4.926\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.013, mean=0.013, max=0.013, sum=0.013 (1)\", \"tab\": \"General information\", \"score\": \"0.013\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1662.782, mean=1662.782, max=1662.782, sum=1662.782 (1)\", \"tab\": \"General information\", \"score\": \"1662.782\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=7.809, mean=7.809, max=7.809, sum=7.809 (1)\", \"tab\": \"General information\", \"score\": \"7.809\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=116.254, mean=116.254, max=116.254, sum=116.254 (1)\", \"tab\": \"General information\", \"score\": \"116.254\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=7.067, mean=7.067, max=7.067, sum=7.067 (1)\", \"tab\": \"General information\", \"score\": \"7.067\"}" - } - }, - "generation_config": { - "additional_details": { - "mode": "\"closedbook\"" - } - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.878, - "details": { - "description": "min=0.878, mean=0.878, max=0.878, sum=0.878 (1)", - "tab": "Accuracy", - "OpenbookQA - Observed inference time (s)": "{\"description\": \"min=0.42, mean=0.42, max=0.42, sum=0.42 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.4200127201080322\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=254.21, mean=254.21, max=254.21, sum=254.21 (1)\", \"tab\": \"General information\", \"score\": \"254.21\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "dataset": "\"openbookqa\"", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.621, - "details": { - "description": "min=0.37, mean=0.621, max=0.91, sum=3.106 (5)", - "tab": "Accuracy", - "MMLU - Observed inference time (s)": "{\"description\": \"min=0.462, mean=0.532, max=0.577, sum=2.661 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.5321985618859008\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=371.38, mean=472.274, max=624.07, sum=2361.37 (5)\", \"tab\": \"General information\", \"score\": \"472.2740350877192\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"college_chemistry\", \"computer_security\", \"econometrics\", \"us_foreign_policy\"]", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MATH", - "source_data": { - "dataset_name": "MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Equivalent (CoT) on MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.58, - "details": { - "description": "min=0.395, mean=0.58, max=0.8, sum=4.059 (7)", - "tab": "Accuracy", - "MATH - Observed inference time (s)": "{\"description\": \"min=1.722, mean=2.088, max=2.676, sum=14.619 (7)\", \"tab\": \"Efficiency\", \"score\": \"2.0883775065675723\"}", - "MATH - # eval": "{\"description\": \"min=30, mean=62.429, max=135, sum=437 (7)\", \"tab\": \"General information\", \"score\": \"62.42857142857143\"}", - "MATH - # train": "{\"description\": \"min=8, mean=8, max=8, sum=56 (7)\", \"tab\": \"General information\", \"score\": \"8.0\"}", - "MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MATH - # prompt tokens": "{\"description\": \"min=906.556, mean=1375.735, max=2449.942, sum=9630.147 (7)\", \"tab\": \"General information\", \"score\": \"1375.7353092779654\"}", - "MATH - # output tokens": "{\"description\": \"min=64, mean=87.032, max=107.385, sum=609.221 (7)\", \"tab\": \"General information\", \"score\": \"87.03154467364993\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"algebra\", \"counting_and_probability\", \"geometry\", \"intermediate_algebra\", \"number_theory\", \"prealgebra\", \"precalculus\"]", - "level": "\"1\"", - "use_official_examples": "\"False\"", - "use_chain_of_thought": "\"True\"" - } - } - }, - { - "evaluation_name": "GSM8K", - "source_data": { - "dataset_name": "GSM8K", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on GSM8K", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.735, - "details": { - "description": "min=0.735, mean=0.735, max=0.735, sum=0.735 (1)", - "tab": "Accuracy", - "GSM8K - Observed inference time (s)": "{\"description\": \"min=2.543, mean=2.543, max=2.543, sum=2.543 (1)\", \"tab\": \"Efficiency\", \"score\": \"2.543274956703186\"}", - "GSM8K - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "GSM8K - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "GSM8K - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GSM8K - # prompt tokens": "{\"description\": \"min=938.869, mean=938.869, max=938.869, sum=938.869 (1)\", \"tab\": \"General information\", \"score\": \"938.869\"}", - "GSM8K - # output tokens": "{\"description\": \"min=89.718, mean=89.718, max=89.718, sum=89.718 (1)\", \"tab\": \"General information\", \"score\": \"89.718\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "LegalBench", - "source_data": { - "dataset_name": "LegalBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on LegalBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.644, - "details": { - "description": "min=0.33, mean=0.644, max=0.989, sum=3.221 (5)", - "tab": "Accuracy", - "LegalBench - Observed inference time (s)": "{\"description\": \"min=0.425, mean=0.731, max=1.784, sum=3.657 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.7313747247589137\"}", - "LegalBench - # eval": "{\"description\": \"min=95, mean=409.4, max=1000, sum=2047 (5)\", \"tab\": \"General information\", \"score\": \"409.4\"}", - "LegalBench - # train": "{\"description\": \"min=3.984, mean=4.597, max=5, sum=22.984 (5)\", \"tab\": \"General information\", \"score\": \"4.596734693877551\"}", - "LegalBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "LegalBench - # prompt tokens": "{\"description\": \"min=205.632, mean=1355.759, max=5467.178, sum=6778.793 (5)\", \"tab\": \"General information\", \"score\": \"1355.7586406214054\"}", - "LegalBench - # output tokens": "{\"description\": \"min=1, mean=2.077, max=5.406, sum=10.386 (5)\", \"tab\": \"General information\", \"score\": \"2.0771673311343752\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"abercrombie\", \"corporate_lobbying\", \"function_of_decision_section\", \"international_citizenship_questions\", \"proa\"]" - } - } - }, - { - "evaluation_name": "MedQA", - "source_data": { - "dataset_name": "MedQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MedQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.598, - "details": { - "description": "min=0.598, mean=0.598, max=0.598, sum=0.598 (1)", - "tab": "Accuracy", - "MedQA - Observed inference time (s)": "{\"description\": \"min=0.605, mean=0.605, max=0.605, sum=0.605 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.6051040529967776\"}", - "MedQA - # eval": "{\"description\": \"min=503, mean=503, max=503, sum=503 (1)\", \"tab\": \"General information\", \"score\": \"503.0\"}", - "MedQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MedQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MedQA - # prompt tokens": "{\"description\": \"min=1038.861, mean=1038.861, max=1038.861, sum=1038.861 (1)\", \"tab\": \"General information\", \"score\": \"1038.8608349900596\"}", - "MedQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WMT 2014", - "source_data": { - "dataset_name": "WMT 2014", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "BLEU-4 on WMT 2014", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.239, - "details": { - "description": "min=0.2, mean=0.239, max=0.27, sum=1.194 (5)", - "tab": "Accuracy", - "WMT 2014 - Observed inference time (s)": "{\"description\": \"min=0.83, mean=0.905, max=0.948, sum=4.524 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.904815991352295\"}", - "WMT 2014 - # eval": "{\"description\": \"min=503, mean=568.8, max=832, sum=2844 (5)\", \"tab\": \"General information\", \"score\": \"568.8\"}", - "WMT 2014 - # train": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "WMT 2014 - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WMT 2014 - # prompt tokens": "{\"description\": \"min=136.93, mean=181.694, max=241.662, sum=908.469 (5)\", \"tab\": \"General information\", \"score\": \"181.69386660804403\"}", - "WMT 2014 - # output tokens": "{\"description\": \"min=23.829, mean=25.142, max=25.958, sum=125.709 (5)\", \"tab\": \"General information\", \"score\": \"25.14180111637865\"}" - } - }, - "generation_config": { - "additional_details": { - "language_pair": "[\"cs-en\", \"de-en\", \"fr-en\", \"hi-en\", \"ru-en\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/writer_palmyra-x-v3.json b/data/models/writer_palmyra-x-v3.json deleted file mode 100644 index d903d06f047a95e99873464e361d47596f2a8154..0000000000000000000000000000000000000000 --- a/data/models/writer_palmyra-x-v3.json +++ /dev/null @@ -1,1901 +0,0 @@ -{ - "model_info": { - "name": "Palmyra X V3 72B", - "id": "writer/palmyra-x-v3", - "developer": "writer", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_lite/writer_palmyra-x-v3/1774096306.427425", - "retrieved_timestamp": "1774096306.427425", - "source_metadata": { - "source_name": "helm_lite", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_lite", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_lite", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.679, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.25696629213483146\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.706, - "details": { - "description": "min=0.706, mean=0.706, max=0.706, sum=0.706 (1)", - "tab": "Accuracy", - "NarrativeQA - Observed inference time (s)": "{\"description\": \"min=2.849, mean=2.849, max=2.849, sum=2.849 (1)\", \"tab\": \"Efficiency\", \"score\": \"2.848917615245765\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=355 (1)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=3504.577, mean=3504.577, max=3504.577, sum=3504.577 (1)\", \"tab\": \"General information\", \"score\": \"3504.5774647887324\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=11.149, mean=11.149, max=11.149, sum=11.149 (1)\", \"tab\": \"General information\", \"score\": \"11.149295774647888\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (closed-book)", - "source_data": { - "dataset_name": "NaturalQuestions (closed-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (closed-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.407, - "details": { - "description": "min=0.407, mean=0.407, max=0.407, sum=0.407 (1)", - "tab": "Accuracy", - "NaturalQuestions (open-book) - Observed inference time (s)": "{\"description\": \"min=2.319, mean=2.319, max=2.319, sum=2.319 (1)\", \"tab\": \"Efficiency\", \"score\": \"2.31904000210762\"}", - "NaturalQuestions (closed-book) - Observed inference time (s)": "{\"description\": \"min=2.373, mean=2.373, max=2.373, sum=2.373 (1)\", \"tab\": \"Efficiency\", \"score\": \"2.3729000978469847\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.885, mean=4.885, max=4.885, sum=4.885 (1)\", \"tab\": \"General information\", \"score\": \"4.885\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.02, mean=0.02, max=0.02, sum=0.02 (1)\", \"tab\": \"General information\", \"score\": \"0.02\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1617.709, mean=1617.709, max=1617.709, sum=1617.709 (1)\", \"tab\": \"General information\", \"score\": \"1617.709\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=12.864, mean=12.864, max=12.864, sum=12.864 (1)\", \"tab\": \"General information\", \"score\": \"12.864\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=116.254, mean=116.254, max=116.254, sum=116.254 (1)\", \"tab\": \"General information\", \"score\": \"116.254\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=19.113, mean=19.113, max=19.113, sum=19.113 (1)\", \"tab\": \"General information\", \"score\": \"19.113\"}" - } - }, - "generation_config": { - "additional_details": { - "mode": "\"closedbook\"" - } - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.938, - "details": { - "description": "min=0.938, mean=0.938, max=0.938, sum=0.938 (1)", - "tab": "Accuracy", - "OpenbookQA - Observed inference time (s)": "{\"description\": \"min=0.607, mean=0.607, max=0.607, sum=0.607 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.6074039902687073\"}", - "OpenbookQA - # eval": "{\"description\": \"min=500, mean=500, max=500, sum=500 (1)\", \"tab\": \"General information\", \"score\": \"500.0\"}", - "OpenbookQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "OpenbookQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"min=254.21, mean=254.21, max=254.21, sum=254.21 (1)\", \"tab\": \"General information\", \"score\": \"254.21\"}", - "OpenbookQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "dataset": "\"openbookqa\"", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.702, - "details": { - "description": "min=0.53, mean=0.702, max=0.96, sum=3.509 (5)", - "tab": "Accuracy", - "MMLU - Observed inference time (s)": "{\"description\": \"min=0.604, mean=0.657, max=0.783, sum=3.283 (5)\", \"tab\": \"Efficiency\", \"score\": \"0.656667516515966\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=514 (5)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=25 (5)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=371.38, mean=472.274, max=624.07, sum=2361.37 (5)\", \"tab\": \"General information\", \"score\": \"472.2740350877192\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"college_chemistry\", \"computer_security\", \"econometrics\", \"us_foreign_policy\"]", - "method": "\"multiple_choice_joint\"" - } - } - }, - { - "evaluation_name": "MATH", - "source_data": { - "dataset_name": "MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Equivalent (CoT) on MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.723, - "details": { - "description": "min=0.579, mean=0.723, max=0.896, sum=5.06 (7)", - "tab": "Accuracy", - "MATH - Observed inference time (s)": "{\"description\": \"min=3.23, mean=4.259, max=6.331, sum=29.811 (7)\", \"tab\": \"Efficiency\", \"score\": \"4.258683228698372\"}", - "MATH - # eval": "{\"description\": \"min=30, mean=62.429, max=135, sum=437 (7)\", \"tab\": \"General information\", \"score\": \"62.42857142857143\"}", - "MATH - # train": "{\"description\": \"min=8, mean=8, max=8, sum=56 (7)\", \"tab\": \"General information\", \"score\": \"8.0\"}", - "MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (7)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MATH - # prompt tokens": "{\"description\": \"min=906.556, mean=1375.735, max=2449.942, sum=9630.147 (7)\", \"tab\": \"General information\", \"score\": \"1375.7353092779654\"}", - "MATH - # output tokens": "{\"description\": \"min=60.012, mean=83.135, max=128.942, sum=581.943 (7)\", \"tab\": \"General information\", \"score\": \"83.13468064416656\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"algebra\", \"counting_and_probability\", \"geometry\", \"intermediate_algebra\", \"number_theory\", \"prealgebra\", \"precalculus\"]", - "level": "\"1\"", - "use_official_examples": "\"False\"", - "use_chain_of_thought": "\"True\"" - } - } - }, - { - "evaluation_name": "GSM8K", - "source_data": { - "dataset_name": "GSM8K", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on GSM8K", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.831, - "details": { - "description": "min=0.831, mean=0.831, max=0.831, sum=0.831 (1)", - "tab": "Accuracy", - "GSM8K - Observed inference time (s)": "{\"description\": \"min=5.07, mean=5.07, max=5.07, sum=5.07 (1)\", \"tab\": \"Efficiency\", \"score\": \"5.069576686620712\"}", - "GSM8K - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "GSM8K - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "GSM8K - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GSM8K - # prompt tokens": "{\"description\": \"min=938.869, mean=938.869, max=938.869, sum=938.869 (1)\", \"tab\": \"General information\", \"score\": \"938.869\"}", - "GSM8K - # output tokens": "{\"description\": \"min=89.919, mean=89.919, max=89.919, sum=89.919 (1)\", \"tab\": \"General information\", \"score\": \"89.919\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "LegalBench", - "source_data": { - "dataset_name": "LegalBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on LegalBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.709, - "details": { - "description": "min=0.439, mean=0.709, max=0.926, sum=3.544 (5)", - "tab": "Accuracy", - "LegalBench - Observed inference time (s)": "{\"description\": \"min=0.668, mean=1.16, max=3.0, sum=5.798 (5)\", \"tab\": \"Efficiency\", \"score\": \"1.1595191393847304\"}", - "LegalBench - # eval": "{\"description\": \"min=95, mean=409.4, max=1000, sum=2047 (5)\", \"tab\": \"General information\", \"score\": \"409.4\"}", - "LegalBench - # train": "{\"description\": \"min=3.984, mean=4.597, max=5, sum=22.984 (5)\", \"tab\": \"General information\", \"score\": \"4.596734693877551\"}", - "LegalBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "LegalBench - # prompt tokens": "{\"description\": \"min=205.632, mean=1355.759, max=5467.178, sum=6778.793 (5)\", \"tab\": \"General information\", \"score\": \"1355.7586406214054\"}", - "LegalBench - # output tokens": "{\"description\": \"min=1, mean=1.078, max=1.2, sum=5.388 (5)\", \"tab\": \"General information\", \"score\": \"1.0776021798365123\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "[\"abercrombie\", \"corporate_lobbying\", \"function_of_decision_section\", \"international_citizenship_questions\", \"proa\"]" - } - } - }, - { - "evaluation_name": "MedQA", - "source_data": { - "dataset_name": "MedQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MedQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.684, - "details": { - "description": "min=0.684, mean=0.684, max=0.684, sum=0.684 (1)", - "tab": "Accuracy", - "MedQA - Observed inference time (s)": "{\"description\": \"min=0.927, mean=0.927, max=0.927, sum=0.927 (1)\", \"tab\": \"Efficiency\", \"score\": \"0.9268994279220611\"}", - "MedQA - # eval": "{\"description\": \"min=503, mean=503, max=503, sum=503 (1)\", \"tab\": \"General information\", \"score\": \"503.0\"}", - "MedQA - # train": "{\"description\": \"min=5, mean=5, max=5, sum=5 (1)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MedQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MedQA - # prompt tokens": "{\"description\": \"min=1038.861, mean=1038.861, max=1038.861, sum=1038.861 (1)\", \"tab\": \"General information\", \"score\": \"1038.8608349900596\"}", - "MedQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=1 (1)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WMT 2014", - "source_data": { - "dataset_name": "WMT 2014", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "BLEU-4 on WMT 2014", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.262, - "details": { - "description": "min=0.235, mean=0.262, max=0.284, sum=1.309 (5)", - "tab": "Accuracy", - "WMT 2014 - Observed inference time (s)": "{\"description\": \"min=1.32, mean=1.406, max=1.477, sum=7.032 (5)\", \"tab\": \"Efficiency\", \"score\": \"1.4063038200537652\"}", - "WMT 2014 - # eval": "{\"description\": \"min=503, mean=568.8, max=832, sum=2844 (5)\", \"tab\": \"General information\", \"score\": \"568.8\"}", - "WMT 2014 - # train": "{\"description\": \"min=1, mean=1, max=1, sum=5 (5)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "WMT 2014 - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (5)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WMT 2014 - # prompt tokens": "{\"description\": \"min=136.93, mean=181.694, max=241.662, sum=908.469 (5)\", \"tab\": \"General information\", \"score\": \"181.69386660804403\"}", - "WMT 2014 - # output tokens": "{\"description\": \"min=23.356, mean=24.983, max=25.829, sum=124.915 (5)\", \"tab\": \"General information\", \"score\": \"24.983090877810064\"}" - } - }, - "generation_config": { - "additional_details": { - "language_pair": "[\"cs-en\", \"de-en\", \"fr-en\", \"hi-en\", \"ru-en\"]" - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_id": "helm_mmlu/writer_palmyra-x-v3/1774096312.00548", - "retrieved_timestamp": "1774096312.00548", - "source_metadata": { - "source_name": "helm_mmlu", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_mmlu", - "evaluation_results": [ - { - "evaluation_name": "MMLU All Subjects", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU All Subjects", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.786, - "details": { - "description": "min=0.493, mean=0.786, max=0.979, sum=89.625 (114)", - "tab": "Accuracy", - "MMLU All Subjects - Observed inference time (s)": "{\"description\": \"min=0.555, mean=0.663, max=1.566, sum=75.544 (114)\", \"tab\": \"Efficiency\", \"score\": \"0.6626657480593275\"}", - "MMLU All Subjects - # eval": "{\"description\": \"min=100, mean=246.351, max=1534, sum=28084 (114)\", \"tab\": \"General information\", \"score\": \"246.35087719298247\"}", - "MMLU All Subjects - # train": "{\"description\": \"min=5, mean=5, max=5, sum=570 (114)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU All Subjects - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (114)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU All Subjects - # prompt tokens": "{\"description\": \"min=277.386, mean=627.489, max=2844.03, sum=71533.746 (114)\", \"tab\": \"General information\", \"score\": \"627.4890026560713\"}", - "MMLU All Subjects - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=114 (114)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - }, - { - "evaluation_name": "Abstract Algebra", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Abstract Algebra", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.53, - "details": { - "description": "min=0.53, mean=0.53, max=0.53, sum=1.06 (2)", - "tab": "Accuracy", - "Abstract Algebra - Observed inference time (s)": "{\"description\": \"min=0.62, mean=0.62, max=0.62, sum=1.239 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6195793676376343\"}", - "Abstract Algebra - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Abstract Algebra - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Abstract Algebra - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Abstract Algebra - # prompt tokens": "{\"description\": \"min=371.38, mean=371.38, max=371.38, sum=742.76 (2)\", \"tab\": \"General information\", \"score\": \"371.38\"}", - "Abstract Algebra - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"abstract_algebra\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_abstract_algebra\"" - } - } - }, - { - "evaluation_name": "Anatomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Anatomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.733, - "details": { - "description": "min=0.733, mean=0.733, max=0.733, sum=1.467 (2)", - "tab": "Accuracy", - "Anatomy - Observed inference time (s)": "{\"description\": \"min=0.586, mean=0.586, max=0.586, sum=1.172 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5858598179287381\"}", - "Anatomy - # eval": "{\"description\": \"min=135, mean=135, max=135, sum=270 (2)\", \"tab\": \"General information\", \"score\": \"135.0\"}", - "Anatomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Anatomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Anatomy - # prompt tokens": "{\"description\": \"min=372.081, mean=372.081, max=372.081, sum=744.163 (2)\", \"tab\": \"General information\", \"score\": \"372.0814814814815\"}", - "Anatomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"anatomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_anatomy\"" - } - } - }, - { - "evaluation_name": "College Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on College Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.549, - "details": { - "description": "min=0.549, mean=0.549, max=0.549, sum=1.098 (2)", - "tab": "Accuracy", - "College Chemistry - Observed inference time (s)": "{\"description\": \"min=0.664, mean=0.664, max=0.664, sum=1.327 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6636523914337158\"}", - "College Biology - Observed inference time (s)": "{\"description\": \"min=0.575, mean=0.575, max=0.575, sum=1.15 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5751992679304547\"}", - "College Computer Science - Observed inference time (s)": "{\"description\": \"min=0.867, mean=0.867, max=0.867, sum=1.734 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8668097257614136\"}", - "College Mathematics - Observed inference time (s)": "{\"description\": \"min=0.591, mean=0.591, max=0.591, sum=1.182 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5912106204032898\"}", - "College Medicine - Observed inference time (s)": "{\"description\": \"min=0.593, mean=0.593, max=0.593, sum=1.186 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5927534434147653\"}", - "College Physics - Observed inference time (s)": "{\"description\": \"min=0.58, mean=0.58, max=0.58, sum=1.159 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5796795171849868\"}", - "College Chemistry - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Chemistry - # prompt tokens": "{\"description\": \"min=545.4, mean=545.4, max=545.4, sum=1090.8 (2)\", \"tab\": \"General information\", \"score\": \"545.4\"}", - "College Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Biology - # eval": "{\"description\": \"min=144, mean=144, max=144, sum=288 (2)\", \"tab\": \"General information\", \"score\": \"144.0\"}", - "College Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Biology - # prompt tokens": "{\"description\": \"min=482.278, mean=482.278, max=482.278, sum=964.556 (2)\", \"tab\": \"General information\", \"score\": \"482.27777777777777\"}", - "College Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Computer Science - # prompt tokens": "{\"description\": \"min=852.15, mean=852.15, max=852.15, sum=1704.3 (2)\", \"tab\": \"General information\", \"score\": \"852.15\"}", - "College Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Mathematics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "College Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Mathematics - # prompt tokens": "{\"description\": \"min=611.53, mean=611.53, max=611.53, sum=1223.06 (2)\", \"tab\": \"General information\", \"score\": \"611.53\"}", - "College Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Medicine - # eval": "{\"description\": \"min=173, mean=173, max=173, sum=346 (2)\", \"tab\": \"General information\", \"score\": \"173.0\"}", - "College Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Medicine - # prompt tokens": "{\"description\": \"min=530.301, mean=530.301, max=530.301, sum=1060.601 (2)\", \"tab\": \"General information\", \"score\": \"530.3005780346821\"}", - "College Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "College Physics - # eval": "{\"description\": \"min=102, mean=102, max=102, sum=204 (2)\", \"tab\": \"General information\", \"score\": \"102.0\"}", - "College Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "College Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "College Physics - # prompt tokens": "{\"description\": \"min=489.324, mean=489.324, max=489.324, sum=978.647 (2)\", \"tab\": \"General information\", \"score\": \"489.3235294117647\"}", - "College Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"college_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_college_physics\"" - } - } - }, - { - "evaluation_name": "Computer Security", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Computer Security", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.78, - "details": { - "description": "min=0.78, mean=0.78, max=0.78, sum=1.56 (2)", - "tab": "Accuracy", - "Computer Security - Observed inference time (s)": "{\"description\": \"min=0.613, mean=0.613, max=0.613, sum=1.227 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.613369300365448\"}", - "Computer Security - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Computer Security - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Computer Security - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Computer Security - # prompt tokens": "{\"description\": \"min=387.4, mean=387.4, max=387.4, sum=774.8 (2)\", \"tab\": \"General information\", \"score\": \"387.4\"}", - "Computer Security - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"computer_security\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_computer_security\"" - } - } - }, - { - "evaluation_name": "Econometrics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Econometrics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.649, - "details": { - "description": "min=0.649, mean=0.649, max=0.649, sum=1.298 (2)", - "tab": "Accuracy", - "Econometrics - Observed inference time (s)": "{\"description\": \"min=0.783, mean=0.783, max=0.783, sum=1.566 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7830351319229394\"}", - "Econometrics - # eval": "{\"description\": \"min=114, mean=114, max=114, sum=228 (2)\", \"tab\": \"General information\", \"score\": \"114.0\"}", - "Econometrics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Econometrics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Econometrics - # prompt tokens": "{\"description\": \"min=624.07, mean=624.07, max=624.07, sum=1248.14 (2)\", \"tab\": \"General information\", \"score\": \"624.0701754385965\"}", - "Econometrics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"econometrics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_econometrics\"" - } - } - }, - { - "evaluation_name": "Global Facts", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Global Facts", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.53, - "details": { - "description": "min=0.53, mean=0.53, max=0.53, sum=1.06 (2)", - "tab": "Accuracy", - "Global Facts - Observed inference time (s)": "{\"description\": \"min=0.586, mean=0.586, max=0.586, sum=1.172 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5858692646026611\"}", - "Global Facts - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Global Facts - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Global Facts - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Global Facts - # prompt tokens": "{\"description\": \"min=398.42, mean=398.42, max=398.42, sum=796.84 (2)\", \"tab\": \"General information\", \"score\": \"398.42\"}", - "Global Facts - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"global_facts\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_global_facts\"" - } - } - }, - { - "evaluation_name": "Jurisprudence", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Jurisprudence", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.88, - "details": { - "description": "min=0.88, mean=0.88, max=0.88, sum=1.759 (2)", - "tab": "Accuracy", - "Jurisprudence - Observed inference time (s)": "{\"description\": \"min=0.581, mean=0.581, max=0.581, sum=1.162 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5810460448265076\"}", - "Jurisprudence - # eval": "{\"description\": \"min=108, mean=108, max=108, sum=216 (2)\", \"tab\": \"General information\", \"score\": \"108.0\"}", - "Jurisprudence - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Jurisprudence - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Jurisprudence - # prompt tokens": "{\"description\": \"min=418.722, mean=418.722, max=418.722, sum=837.444 (2)\", \"tab\": \"General information\", \"score\": \"418.72222222222223\"}", - "Jurisprudence - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"jurisprudence\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_jurisprudence\"" - } - } - }, - { - "evaluation_name": "Philosophy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Philosophy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.836, - "details": { - "description": "min=0.836, mean=0.836, max=0.836, sum=1.672 (2)", - "tab": "Accuracy", - "Philosophy - Observed inference time (s)": "{\"description\": \"min=0.576, mean=0.576, max=0.576, sum=1.152 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5761417744627336\"}", - "Philosophy - # eval": "{\"description\": \"min=311, mean=311, max=311, sum=622 (2)\", \"tab\": \"General information\", \"score\": \"311.0\"}", - "Philosophy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Philosophy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Philosophy - # prompt tokens": "{\"description\": \"min=353.704, mean=353.704, max=353.704, sum=707.408 (2)\", \"tab\": \"General information\", \"score\": \"353.7041800643087\"}", - "Philosophy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"philosophy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_philosophy\"" - } - } - }, - { - "evaluation_name": "Professional Psychology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Professional Psychology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.858, - "details": { - "description": "min=0.858, mean=0.858, max=0.858, sum=1.716 (2)", - "tab": "Accuracy", - "Professional Medicine - Observed inference time (s)": "{\"description\": \"min=0.884, mean=0.884, max=0.884, sum=1.768 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8839500090655159\"}", - "Professional Accounting - Observed inference time (s)": "{\"description\": \"min=0.711, mean=0.711, max=0.711, sum=1.423 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.7114707704976941\"}", - "Professional Law - Observed inference time (s)": "{\"description\": \"min=0.981, mean=0.981, max=0.981, sum=1.962 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.9809994663377785\"}", - "Professional Psychology - Observed inference time (s)": "{\"description\": \"min=0.598, mean=0.598, max=0.598, sum=1.196 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5978598594665527\"}", - "Professional Medicine - # eval": "{\"description\": \"min=272, mean=272, max=272, sum=544 (2)\", \"tab\": \"General information\", \"score\": \"272.0\"}", - "Professional Medicine - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Medicine - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Medicine - # prompt tokens": "{\"description\": \"min=1118.287, mean=1118.287, max=1118.287, sum=2236.574 (2)\", \"tab\": \"General information\", \"score\": \"1118.2867647058824\"}", - "Professional Medicine - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Accounting - # eval": "{\"description\": \"min=282, mean=282, max=282, sum=564 (2)\", \"tab\": \"General information\", \"score\": \"282.0\"}", - "Professional Accounting - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Accounting - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Accounting - # prompt tokens": "{\"description\": \"min=660.72, mean=660.72, max=660.72, sum=1321.44 (2)\", \"tab\": \"General information\", \"score\": \"660.7198581560284\"}", - "Professional Accounting - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Law - # eval": "{\"description\": \"min=1534, mean=1534, max=1534, sum=3068 (2)\", \"tab\": \"General information\", \"score\": \"1534.0\"}", - "Professional Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Law - # prompt tokens": "{\"description\": \"min=1658.73, mean=1658.73, max=1658.73, sum=3317.46 (2)\", \"tab\": \"General information\", \"score\": \"1658.7301173402868\"}", - "Professional Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Professional Psychology - # eval": "{\"description\": \"min=612, mean=612, max=612, sum=1224 (2)\", \"tab\": \"General information\", \"score\": \"612.0\"}", - "Professional Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Professional Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Professional Psychology - # prompt tokens": "{\"description\": \"min=597.574, mean=597.574, max=597.574, sum=1195.147 (2)\", \"tab\": \"General information\", \"score\": \"597.5735294117648\"}", - "Professional Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"professional_psychology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_professional_psychology\"" - } - } - }, - { - "evaluation_name": "Us Foreign Policy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Us Foreign Policy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.96, - "details": { - "description": "min=0.96, mean=0.96, max=0.96, sum=1.92 (2)", - "tab": "Accuracy", - "Us Foreign Policy - Observed inference time (s)": "{\"description\": \"min=0.604, mean=0.604, max=0.604, sum=1.207 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6037013912200928\"}", - "Us Foreign Policy - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Us Foreign Policy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Us Foreign Policy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Us Foreign Policy - # prompt tokens": "{\"description\": \"min=433.12, mean=433.12, max=433.12, sum=866.24 (2)\", \"tab\": \"General information\", \"score\": \"433.12\"}", - "Us Foreign Policy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"us_foreign_policy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_us_foreign_policy\"" - } - } - }, - { - "evaluation_name": "Astronomy", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Astronomy", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.862, - "details": { - "description": "min=0.862, mean=0.862, max=0.862, sum=1.724 (2)", - "tab": "Accuracy", - "Astronomy - Observed inference time (s)": "{\"description\": \"min=0.593, mean=0.593, max=0.593, sum=1.186 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5929083667303386\"}", - "Astronomy - # eval": "{\"description\": \"min=152, mean=152, max=152, sum=304 (2)\", \"tab\": \"General information\", \"score\": \"152.0\"}", - "Astronomy - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Astronomy - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Astronomy - # prompt tokens": "{\"description\": \"min=600.112, mean=600.112, max=600.112, sum=1200.224 (2)\", \"tab\": \"General information\", \"score\": \"600.1118421052631\"}", - "Astronomy - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"astronomy\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_astronomy\"" - } - } - }, - { - "evaluation_name": "Business Ethics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Business Ethics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.83, - "details": { - "description": "min=0.83, mean=0.83, max=0.83, sum=1.66 (2)", - "tab": "Accuracy", - "Business Ethics - Observed inference time (s)": "{\"description\": \"min=0.598, mean=0.598, max=0.598, sum=1.196 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5981829071044922\"}", - "Business Ethics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Business Ethics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Business Ethics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Business Ethics - # prompt tokens": "{\"description\": \"min=589.46, mean=589.46, max=589.46, sum=1178.92 (2)\", \"tab\": \"General information\", \"score\": \"589.46\"}", - "Business Ethics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"business_ethics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_business_ethics\"" - } - } - }, - { - "evaluation_name": "Clinical Knowledge", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Clinical Knowledge", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.804, - "details": { - "description": "min=0.804, mean=0.804, max=0.804, sum=1.608 (2)", - "tab": "Accuracy", - "Clinical Knowledge - Observed inference time (s)": "{\"description\": \"min=0.575, mean=0.575, max=0.575, sum=1.15 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5750116924069962\"}", - "Clinical Knowledge - # eval": "{\"description\": \"min=265, mean=265, max=265, sum=530 (2)\", \"tab\": \"General information\", \"score\": \"265.0\"}", - "Clinical Knowledge - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Clinical Knowledge - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Clinical Knowledge - # prompt tokens": "{\"description\": \"min=423.925, mean=423.925, max=423.925, sum=847.849 (2)\", \"tab\": \"General information\", \"score\": \"423.92452830188677\"}", - "Clinical Knowledge - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"clinical_knowledge\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_clinical_knowledge\"" - } - } - }, - { - "evaluation_name": "Conceptual Physics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Conceptual Physics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.809, - "details": { - "description": "min=0.809, mean=0.809, max=0.809, sum=1.617 (2)", - "tab": "Accuracy", - "Conceptual Physics - Observed inference time (s)": "{\"description\": \"min=0.58, mean=0.58, max=0.58, sum=1.161 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5802780881841132\"}", - "Conceptual Physics - # eval": "{\"description\": \"min=235, mean=235, max=235, sum=470 (2)\", \"tab\": \"General information\", \"score\": \"235.0\"}", - "Conceptual Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Conceptual Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Conceptual Physics - # prompt tokens": "{\"description\": \"min=313.723, mean=313.723, max=313.723, sum=627.447 (2)\", \"tab\": \"General information\", \"score\": \"313.72340425531917\"}", - "Conceptual Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"conceptual_physics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_conceptual_physics\"" - } - } - }, - { - "evaluation_name": "Electrical Engineering", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Electrical Engineering", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.772, - "details": { - "description": "min=0.772, mean=0.772, max=0.772, sum=1.545 (2)", - "tab": "Accuracy", - "Electrical Engineering - Observed inference time (s)": "{\"description\": \"min=0.583, mean=0.583, max=0.583, sum=1.165 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5827381166918525\"}", - "Electrical Engineering - # eval": "{\"description\": \"min=145, mean=145, max=145, sum=290 (2)\", \"tab\": \"General information\", \"score\": \"145.0\"}", - "Electrical Engineering - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Electrical Engineering - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Electrical Engineering - # prompt tokens": "{\"description\": \"min=430.345, mean=430.345, max=430.345, sum=860.69 (2)\", \"tab\": \"General information\", \"score\": \"430.3448275862069\"}", - "Electrical Engineering - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"electrical_engineering\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_electrical_engineering\"" - } - } - }, - { - "evaluation_name": "Elementary Mathematics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Elementary Mathematics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.661, - "details": { - "description": "min=0.661, mean=0.661, max=0.661, sum=1.323 (2)", - "tab": "Accuracy", - "Elementary Mathematics - Observed inference time (s)": "{\"description\": \"min=0.584, mean=0.584, max=0.584, sum=1.167 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5836543033993433\"}", - "Elementary Mathematics - # eval": "{\"description\": \"min=378, mean=378, max=378, sum=756 (2)\", \"tab\": \"General information\", \"score\": \"378.0\"}", - "Elementary Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Elementary Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Elementary Mathematics - # prompt tokens": "{\"description\": \"min=506.09, mean=506.09, max=506.09, sum=1012.18 (2)\", \"tab\": \"General information\", \"score\": \"506.0899470899471\"}", - "Elementary Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"elementary_mathematics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_elementary_mathematics\"" - } - } - }, - { - "evaluation_name": "Formal Logic", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Formal Logic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.659, - "details": { - "description": "min=0.659, mean=0.659, max=0.659, sum=1.317 (2)", - "tab": "Accuracy", - "Formal Logic - Observed inference time (s)": "{\"description\": \"min=0.597, mean=0.597, max=0.597, sum=1.194 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5971027309932406\"}", - "Formal Logic - # eval": "{\"description\": \"min=126, mean=126, max=126, sum=252 (2)\", \"tab\": \"General information\", \"score\": \"126.0\"}", - "Formal Logic - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Formal Logic - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Formal Logic - # prompt tokens": "{\"description\": \"min=641, mean=641, max=641, sum=1282 (2)\", \"tab\": \"General information\", \"score\": \"641.0\"}", - "Formal Logic - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"formal_logic\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_formal_logic\"" - } - } - }, - { - "evaluation_name": "High School World History", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on High School World History", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.911, - "details": { - "description": "min=0.911, mean=0.911, max=0.911, sum=1.823 (2)", - "tab": "Accuracy", - "High School Biology - Observed inference time (s)": "{\"description\": \"min=0.584, mean=0.584, max=0.584, sum=1.168 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5838540715555991\"}", - "High School Chemistry - Observed inference time (s)": "{\"description\": \"min=0.579, mean=0.579, max=0.579, sum=1.159 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5794280843781721\"}", - "High School Computer Science - Observed inference time (s)": "{\"description\": \"min=0.873, mean=0.873, max=0.873, sum=1.745 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8726636576652527\"}", - "High School European History - Observed inference time (s)": "{\"description\": \"min=1.532, mean=1.532, max=1.532, sum=3.063 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.5316768602891402\"}", - "High School Geography - Observed inference time (s)": "{\"description\": \"min=0.568, mean=0.568, max=0.568, sum=1.135 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5675288703706529\"}", - "High School Government And Politics - Observed inference time (s)": "{\"description\": \"min=0.574, mean=0.574, max=0.574, sum=1.147 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.573576919773082\"}", - "High School Macroeconomics - Observed inference time (s)": "{\"description\": \"min=0.608, mean=0.608, max=0.608, sum=1.215 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.607545349536798\"}", - "High School Mathematics - Observed inference time (s)": "{\"description\": \"min=0.594, mean=0.594, max=0.594, sum=1.187 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5936917472768712\"}", - "High School Microeconomics - Observed inference time (s)": "{\"description\": \"min=0.561, mean=0.561, max=0.561, sum=1.123 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5614581979623362\"}", - "High School Physics - Observed inference time (s)": "{\"description\": \"min=0.594, mean=0.594, max=0.594, sum=1.189 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5943679051683438\"}", - "High School Psychology - Observed inference time (s)": "{\"description\": \"min=0.595, mean=0.595, max=0.595, sum=1.189 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5945224263252469\"}", - "High School Statistics - Observed inference time (s)": "{\"description\": \"min=0.889, mean=0.889, max=0.889, sum=1.778 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8891873856385549\"}", - "High School US History - Observed inference time (s)": "{\"description\": \"min=1.566, mean=1.566, max=1.566, sum=3.131 (2)\", \"tab\": \"Efficiency\", \"score\": \"1.5656375043532427\"}", - "High School World History - Observed inference time (s)": "{\"description\": \"min=0.876, mean=0.876, max=0.876, sum=1.751 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8755375081476783\"}", - "High School Biology - # eval": "{\"description\": \"min=310, mean=310, max=310, sum=620 (2)\", \"tab\": \"General information\", \"score\": \"310.0\"}", - "High School Biology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Biology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Biology - # prompt tokens": "{\"description\": \"min=540.748, mean=540.748, max=540.748, sum=1081.497 (2)\", \"tab\": \"General information\", \"score\": \"540.7483870967742\"}", - "High School Biology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Chemistry - # eval": "{\"description\": \"min=203, mean=203, max=203, sum=406 (2)\", \"tab\": \"General information\", \"score\": \"203.0\"}", - "High School Chemistry - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Chemistry - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Chemistry - # prompt tokens": "{\"description\": \"min=495.65, mean=495.65, max=495.65, sum=991.3 (2)\", \"tab\": \"General information\", \"score\": \"495.6502463054187\"}", - "High School Chemistry - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Computer Science - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "High School Computer Science - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Computer Science - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Computer Science - # prompt tokens": "{\"description\": \"min=904.15, mean=904.15, max=904.15, sum=1808.3 (2)\", \"tab\": \"General information\", \"score\": \"904.15\"}", - "High School Computer Science - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School European History - # eval": "{\"description\": \"min=165, mean=165, max=165, sum=330 (2)\", \"tab\": \"General information\", \"score\": \"165.0\"}", - "High School European History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School European History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School European History - # prompt tokens": "{\"description\": \"min=2844.03, mean=2844.03, max=2844.03, sum=5688.061 (2)\", \"tab\": \"General information\", \"score\": \"2844.030303030303\"}", - "High School European History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Geography - # eval": "{\"description\": \"min=198, mean=198, max=198, sum=396 (2)\", \"tab\": \"General information\", \"score\": \"198.0\"}", - "High School Geography - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Geography - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Geography - # prompt tokens": "{\"description\": \"min=397.646, mean=397.646, max=397.646, sum=795.293 (2)\", \"tab\": \"General information\", \"score\": \"397.64646464646466\"}", - "High School Geography - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Government And Politics - # eval": "{\"description\": \"min=193, mean=193, max=193, sum=386 (2)\", \"tab\": \"General information\", \"score\": \"193.0\"}", - "High School Government And Politics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Government And Politics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Government And Politics - # prompt tokens": "{\"description\": \"min=478.073, mean=478.073, max=478.073, sum=956.145 (2)\", \"tab\": \"General information\", \"score\": \"478.07253886010363\"}", - "High School Government And Politics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Macroeconomics - # eval": "{\"description\": \"min=390, mean=390, max=390, sum=780 (2)\", \"tab\": \"General information\", \"score\": \"390.0\"}", - "High School Macroeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Macroeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Macroeconomics - # prompt tokens": "{\"description\": \"min=391.987, mean=391.987, max=391.987, sum=783.974 (2)\", \"tab\": \"General information\", \"score\": \"391.9871794871795\"}", - "High School Macroeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Mathematics - # eval": "{\"description\": \"min=270, mean=270, max=270, sum=540 (2)\", \"tab\": \"General information\", \"score\": \"270.0\"}", - "High School Mathematics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Mathematics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Mathematics - # prompt tokens": "{\"description\": \"min=526.352, mean=526.352, max=526.352, sum=1052.704 (2)\", \"tab\": \"General information\", \"score\": \"526.3518518518518\"}", - "High School Mathematics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Microeconomics - # eval": "{\"description\": \"min=238, mean=238, max=238, sum=476 (2)\", \"tab\": \"General information\", \"score\": \"238.0\"}", - "High School Microeconomics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Microeconomics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Microeconomics - # prompt tokens": "{\"description\": \"min=411.055, mean=411.055, max=411.055, sum=822.109 (2)\", \"tab\": \"General information\", \"score\": \"411.0546218487395\"}", - "High School Microeconomics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Physics - # eval": "{\"description\": \"min=151, mean=151, max=151, sum=302 (2)\", \"tab\": \"General information\", \"score\": \"151.0\"}", - "High School Physics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Physics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Physics - # prompt tokens": "{\"description\": \"min=553.669, mean=553.669, max=553.669, sum=1107.338 (2)\", \"tab\": \"General information\", \"score\": \"553.6688741721854\"}", - "High School Physics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Psychology - # eval": "{\"description\": \"min=545, mean=545, max=545, sum=1090 (2)\", \"tab\": \"General information\", \"score\": \"545.0\"}", - "High School Psychology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Psychology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Psychology - # prompt tokens": "{\"description\": \"min=516.842, mean=516.842, max=516.842, sum=1033.684 (2)\", \"tab\": \"General information\", \"score\": \"516.8422018348624\"}", - "High School Psychology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School Statistics - # eval": "{\"description\": \"min=216, mean=216, max=216, sum=432 (2)\", \"tab\": \"General information\", \"score\": \"216.0\"}", - "High School Statistics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School Statistics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School Statistics - # prompt tokens": "{\"description\": \"min=805, mean=805, max=805, sum=1610 (2)\", \"tab\": \"General information\", \"score\": \"805.0\"}", - "High School Statistics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School US History - # eval": "{\"description\": \"min=204, mean=204, max=204, sum=408 (2)\", \"tab\": \"General information\", \"score\": \"204.0\"}", - "High School US History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School US History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School US History - # prompt tokens": "{\"description\": \"min=2242.25, mean=2242.25, max=2242.25, sum=4484.5 (2)\", \"tab\": \"General information\", \"score\": \"2242.25\"}", - "High School US History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "High School World History - # eval": "{\"description\": \"min=237, mean=237, max=237, sum=474 (2)\", \"tab\": \"General information\", \"score\": \"237.0\"}", - "High School World History - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "High School World History - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "High School World History - # prompt tokens": "{\"description\": \"min=1438.561, mean=1438.561, max=1438.561, sum=2877.122 (2)\", \"tab\": \"General information\", \"score\": \"1438.5611814345991\"}", - "High School World History - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"high_school_world_history\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_high_school_world_history\"" - } - } - }, - { - "evaluation_name": "Human Sexuality", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Human Sexuality", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.924, - "details": { - "description": "min=0.924, mean=0.924, max=0.924, sum=1.847 (2)", - "tab": "Accuracy", - "Human Aging - Observed inference time (s)": "{\"description\": \"min=0.577, mean=0.577, max=0.577, sum=1.154 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5767963167797824\"}", - "Human Sexuality - Observed inference time (s)": "{\"description\": \"min=0.564, mean=0.564, max=0.564, sum=1.127 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5637276700434793\"}", - "Human Aging - # eval": "{\"description\": \"min=223, mean=223, max=223, sum=446 (2)\", \"tab\": \"General information\", \"score\": \"223.0\"}", - "Human Aging - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Aging - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Aging - # prompt tokens": "{\"description\": \"min=324.48, mean=324.48, max=324.48, sum=648.96 (2)\", \"tab\": \"General information\", \"score\": \"324.47982062780267\"}", - "Human Aging - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Human Sexuality - # eval": "{\"description\": \"min=131, mean=131, max=131, sum=262 (2)\", \"tab\": \"General information\", \"score\": \"131.0\"}", - "Human Sexuality - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Human Sexuality - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Human Sexuality - # prompt tokens": "{\"description\": \"min=357.626, mean=357.626, max=357.626, sum=715.252 (2)\", \"tab\": \"General information\", \"score\": \"357.62595419847327\"}", - "Human Sexuality - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"human_sexuality\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_human_sexuality\"" - } - } - }, - { - "evaluation_name": "International Law", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on International Law", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.909, - "details": { - "description": "min=0.909, mean=0.909, max=0.909, sum=1.818 (2)", - "tab": "Accuracy", - "International Law - Observed inference time (s)": "{\"description\": \"min=0.603, mean=0.603, max=0.603, sum=1.205 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6025364970372729\"}", - "International Law - # eval": "{\"description\": \"min=121, mean=121, max=121, sum=242 (2)\", \"tab\": \"General information\", \"score\": \"121.0\"}", - "International Law - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "International Law - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "International Law - # prompt tokens": "{\"description\": \"min=639.843, mean=639.843, max=639.843, sum=1279.686 (2)\", \"tab\": \"General information\", \"score\": \"639.8429752066115\"}", - "International Law - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"international_law\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_international_law\"" - } - } - }, - { - "evaluation_name": "Logical Fallacies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Logical Fallacies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.877, - "details": { - "description": "min=0.877, mean=0.877, max=0.877, sum=1.755 (2)", - "tab": "Accuracy", - "Logical Fallacies - Observed inference time (s)": "{\"description\": \"min=0.577, mean=0.577, max=0.577, sum=1.154 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5770467907373159\"}", - "Logical Fallacies - # eval": "{\"description\": \"min=163, mean=163, max=163, sum=326 (2)\", \"tab\": \"General information\", \"score\": \"163.0\"}", - "Logical Fallacies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Logical Fallacies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Logical Fallacies - # prompt tokens": "{\"description\": \"min=454.227, mean=454.227, max=454.227, sum=908.454 (2)\", \"tab\": \"General information\", \"score\": \"454.2269938650307\"}", - "Logical Fallacies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"logical_fallacies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_logical_fallacies\"" - } - } - }, - { - "evaluation_name": "Machine Learning", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Machine Learning", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.625, - "details": { - "description": "min=0.625, mean=0.625, max=0.625, sum=1.25 (2)", - "tab": "Accuracy", - "Machine Learning - Observed inference time (s)": "{\"description\": \"min=0.612, mean=0.612, max=0.612, sum=1.223 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.6116326642887933\"}", - "Machine Learning - # eval": "{\"description\": \"min=112, mean=112, max=112, sum=224 (2)\", \"tab\": \"General information\", \"score\": \"112.0\"}", - "Machine Learning - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Machine Learning - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Machine Learning - # prompt tokens": "{\"description\": \"min=671.598, mean=671.598, max=671.598, sum=1343.196 (2)\", \"tab\": \"General information\", \"score\": \"671.5982142857143\"}", - "Machine Learning - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"machine_learning\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_machine_learning\"" - } - } - }, - { - "evaluation_name": "Management", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Management", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.903, - "details": { - "description": "min=0.903, mean=0.903, max=0.903, sum=1.806 (2)", - "tab": "Accuracy", - "Management - Observed inference time (s)": "{\"description\": \"min=0.555, mean=0.555, max=0.555, sum=1.111 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5553541276061419\"}", - "Management - # eval": "{\"description\": \"min=103, mean=103, max=103, sum=206 (2)\", \"tab\": \"General information\", \"score\": \"103.0\"}", - "Management - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Management - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Management - # prompt tokens": "{\"description\": \"min=292.34, mean=292.34, max=292.34, sum=584.68 (2)\", \"tab\": \"General information\", \"score\": \"292.3398058252427\"}", - "Management - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"management\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_management\"" - } - } - }, - { - "evaluation_name": "Marketing", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Marketing", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.94, - "details": { - "description": "min=0.94, mean=0.94, max=0.94, sum=1.88 (2)", - "tab": "Accuracy", - "Marketing - Observed inference time (s)": "{\"description\": \"min=0.567, mean=0.567, max=0.567, sum=1.133 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.56665647131765\"}", - "Marketing - # eval": "{\"description\": \"min=234, mean=234, max=234, sum=468 (2)\", \"tab\": \"General information\", \"score\": \"234.0\"}", - "Marketing - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Marketing - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Marketing - # prompt tokens": "{\"description\": \"min=438.697, mean=438.697, max=438.697, sum=877.393 (2)\", \"tab\": \"General information\", \"score\": \"438.6965811965812\"}", - "Marketing - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"marketing\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_marketing\"" - } - } - }, - { - "evaluation_name": "Medical Genetics", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Medical Genetics", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.83, - "details": { - "description": "min=0.83, mean=0.83, max=0.83, sum=1.66 (2)", - "tab": "Accuracy", - "Medical Genetics - Observed inference time (s)": "{\"description\": \"min=0.566, mean=0.566, max=0.566, sum=1.131 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5655512261390686\"}", - "Medical Genetics - # eval": "{\"description\": \"min=100, mean=100, max=100, sum=200 (2)\", \"tab\": \"General information\", \"score\": \"100.0\"}", - "Medical Genetics - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Medical Genetics - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Medical Genetics - # prompt tokens": "{\"description\": \"min=352.71, mean=352.71, max=352.71, sum=705.42 (2)\", \"tab\": \"General information\", \"score\": \"352.71\"}", - "Medical Genetics - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"medical_genetics\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_medical_genetics\"" - } - } - }, - { - "evaluation_name": "Miscellaneous", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Miscellaneous", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.894, - "details": { - "description": "min=0.894, mean=0.894, max=0.894, sum=1.788 (2)", - "tab": "Accuracy", - "Miscellaneous - Observed inference time (s)": "{\"description\": \"min=0.571, mean=0.571, max=0.571, sum=1.142 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5712210739252668\"}", - "Miscellaneous - # eval": "{\"description\": \"min=783, mean=783, max=783, sum=1566 (2)\", \"tab\": \"General information\", \"score\": \"783.0\"}", - "Miscellaneous - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Miscellaneous - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Miscellaneous - # prompt tokens": "{\"description\": \"min=314.847, mean=314.847, max=314.847, sum=629.693 (2)\", \"tab\": \"General information\", \"score\": \"314.84674329501917\"}", - "Miscellaneous - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"miscellaneous\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_miscellaneous\"" - } - } - }, - { - "evaluation_name": "Moral Scenarios", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Moral Scenarios", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.562, - "details": { - "description": "min=0.562, mean=0.562, max=0.562, sum=1.124 (2)", - "tab": "Accuracy", - "Moral Disputes - Observed inference time (s)": "{\"description\": \"min=0.572, mean=0.572, max=0.572, sum=1.145 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5724084032753299\"}", - "Moral Scenarios - Observed inference time (s)": "{\"description\": \"min=0.583, mean=0.583, max=0.583, sum=1.166 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5827599754546607\"}", - "Moral Disputes - # eval": "{\"description\": \"min=346, mean=346, max=346, sum=692 (2)\", \"tab\": \"General information\", \"score\": \"346.0\"}", - "Moral Disputes - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Disputes - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Disputes - # prompt tokens": "{\"description\": \"min=497.329, mean=497.329, max=497.329, sum=994.659 (2)\", \"tab\": \"General information\", \"score\": \"497.32947976878614\"}", - "Moral Disputes - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "Moral Scenarios - # eval": "{\"description\": \"min=895, mean=895, max=895, sum=1790 (2)\", \"tab\": \"General information\", \"score\": \"895.0\"}", - "Moral Scenarios - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Moral Scenarios - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Moral Scenarios - # prompt tokens": "{\"description\": \"min=664.482, mean=664.482, max=664.482, sum=1328.963 (2)\", \"tab\": \"General information\", \"score\": \"664.4815642458101\"}", - "Moral Scenarios - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"moral_scenarios\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_moral_scenarios\"" - } - } - }, - { - "evaluation_name": "Nutrition", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Nutrition", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.856, - "details": { - "description": "min=0.856, mean=0.856, max=0.856, sum=1.712 (2)", - "tab": "Accuracy", - "Nutrition - Observed inference time (s)": "{\"description\": \"min=0.59, mean=0.59, max=0.59, sum=1.18 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5898437850615558\"}", - "Nutrition - # eval": "{\"description\": \"min=306, mean=306, max=306, sum=612 (2)\", \"tab\": \"General information\", \"score\": \"306.0\"}", - "Nutrition - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Nutrition - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Nutrition - # prompt tokens": "{\"description\": \"min=584.69, mean=584.69, max=584.69, sum=1169.379 (2)\", \"tab\": \"General information\", \"score\": \"584.6895424836601\"}", - "Nutrition - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"nutrition\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_nutrition\"" - } - } - }, - { - "evaluation_name": "Prehistory", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Prehistory", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.87, - "details": { - "description": "min=0.87, mean=0.87, max=0.87, sum=1.741 (2)", - "tab": "Accuracy", - "Prehistory - Observed inference time (s)": "{\"description\": \"min=0.585, mean=0.585, max=0.585, sum=1.17 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5852300509994413\"}", - "Prehistory - # eval": "{\"description\": \"min=324, mean=324, max=324, sum=648 (2)\", \"tab\": \"General information\", \"score\": \"324.0\"}", - "Prehistory - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Prehistory - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Prehistory - # prompt tokens": "{\"description\": \"min=524.454, mean=524.454, max=524.454, sum=1048.907 (2)\", \"tab\": \"General information\", \"score\": \"524.4537037037037\"}", - "Prehistory - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"prehistory\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_prehistory\"" - } - } - }, - { - "evaluation_name": "Public Relations", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Public Relations", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.773, - "details": { - "description": "min=0.773, mean=0.773, max=0.773, sum=1.545 (2)", - "tab": "Accuracy", - "Public Relations - Observed inference time (s)": "{\"description\": \"min=0.567, mean=0.567, max=0.567, sum=1.134 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5669147144664418\"}", - "Public Relations - # eval": "{\"description\": \"min=110, mean=110, max=110, sum=220 (2)\", \"tab\": \"General information\", \"score\": \"110.0\"}", - "Public Relations - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Public Relations - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Public Relations - # prompt tokens": "{\"description\": \"min=420.609, mean=420.609, max=420.609, sum=841.218 (2)\", \"tab\": \"General information\", \"score\": \"420.6090909090909\"}", - "Public Relations - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"public_relations\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_public_relations\"" - } - } - }, - { - "evaluation_name": "Security Studies", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Security Studies", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.833, - "details": { - "description": "min=0.833, mean=0.833, max=0.833, sum=1.665 (2)", - "tab": "Accuracy", - "Security Studies - Observed inference time (s)": "{\"description\": \"min=0.864, mean=0.864, max=0.864, sum=1.728 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.8641960144042968\"}", - "Security Studies - # eval": "{\"description\": \"min=245, mean=245, max=245, sum=490 (2)\", \"tab\": \"General information\", \"score\": \"245.0\"}", - "Security Studies - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Security Studies - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Security Studies - # prompt tokens": "{\"description\": \"min=1196.433, mean=1196.433, max=1196.433, sum=2392.865 (2)\", \"tab\": \"General information\", \"score\": \"1196.4326530612245\"}", - "Security Studies - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"security_studies\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_security_studies\"" - } - } - }, - { - "evaluation_name": "Sociology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Sociology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.91, - "details": { - "description": "min=0.91, mean=0.91, max=0.91, sum=1.821 (2)", - "tab": "Accuracy", - "Sociology - Observed inference time (s)": "{\"description\": \"min=0.579, mean=0.579, max=0.579, sum=1.158 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5788582047419761\"}", - "Sociology - # eval": "{\"description\": \"min=201, mean=201, max=201, sum=402 (2)\", \"tab\": \"General information\", \"score\": \"201.0\"}", - "Sociology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Sociology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Sociology - # prompt tokens": "{\"description\": \"min=446.512, mean=446.512, max=446.512, sum=893.025 (2)\", \"tab\": \"General information\", \"score\": \"446.5124378109453\"}", - "Sociology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"sociology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_sociology\"" - } - } - }, - { - "evaluation_name": "Virology", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on Virology", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.572, - "details": { - "description": "min=0.572, mean=0.572, max=0.572, sum=1.145 (2)", - "tab": "Accuracy", - "Virology - Observed inference time (s)": "{\"description\": \"min=0.569, mean=0.569, max=0.569, sum=1.138 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5690187689769699\"}", - "Virology - # eval": "{\"description\": \"min=166, mean=166, max=166, sum=332 (2)\", \"tab\": \"General information\", \"score\": \"166.0\"}", - "Virology - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "Virology - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Virology - # prompt tokens": "{\"description\": \"min=352.753, mean=352.753, max=352.753, sum=705.506 (2)\", \"tab\": \"General information\", \"score\": \"352.7530120481928\"}", - "Virology - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"virology\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_virology\"" - } - } - }, - { - "evaluation_name": "World Religions", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on World Religions", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.877, - "details": { - "description": "min=0.877, mean=0.877, max=0.877, sum=1.754 (2)", - "tab": "Accuracy", - "World Religions - Observed inference time (s)": "{\"description\": \"min=0.579, mean=0.579, max=0.579, sum=1.159 (2)\", \"tab\": \"Efficiency\", \"score\": \"0.5794550257119518\"}", - "World Religions - # eval": "{\"description\": \"min=171, mean=171, max=171, sum=342 (2)\", \"tab\": \"General information\", \"score\": \"171.0\"}", - "World Religions - # train": "{\"description\": \"min=5, mean=5, max=5, sum=10 (2)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "World Religions - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (2)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "World Religions - # prompt tokens": "{\"description\": \"min=277.386, mean=277.386, max=277.386, sum=554.772 (2)\", \"tab\": \"General information\", \"score\": \"277.3859649122807\"}", - "World Religions - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=2 (2)\", \"tab\": \"General information\", \"score\": \"1.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subject": "\"world_religions\"", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "\"mmlu_world_religions\"" - } - } - }, - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_mmlu", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/mmlu/benchmark_output/releases/v1.13.0/groups/mmlu_subjects.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperforms on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.325, - "details": { - "description": "", - "tab": "Efficiency" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": { - "subject": "[\"abstract_algebra\", \"anatomy\", \"astronomy\", \"business_ethics\", \"clinical_knowledge\", \"college_biology\", \"college_chemistry\", \"college_computer_science\", \"college_mathematics\", \"college_medicine\", \"college_physics\", \"computer_security\", \"conceptual_physics\", \"econometrics\", \"electrical_engineering\", \"elementary_mathematics\", \"formal_logic\", \"global_facts\", \"high_school_biology\", \"high_school_chemistry\", \"high_school_computer_science\", \"high_school_european_history\", \"high_school_geography\", \"high_school_government_and_politics\", \"high_school_macroeconomics\", \"high_school_mathematics\", \"high_school_microeconomics\", \"high_school_physics\", \"high_school_psychology\", \"high_school_statistics\", \"high_school_us_history\", \"high_school_world_history\", \"human_aging\", \"human_sexuality\", \"international_law\", \"jurisprudence\", \"logical_fallacies\", \"machine_learning\", \"management\", \"marketing\", \"medical_genetics\", \"miscellaneous\", \"moral_disputes\", \"moral_scenarios\", \"nutrition\", \"philosophy\", \"prehistory\", \"professional_accounting\", \"professional_law\", \"professional_medicine\", \"professional_psychology\", \"public_relations\", \"security_studies\", \"sociology\", \"us_foreign_policy\", \"virology\", \"world_religions\"]", - "method": "\"multiple_choice_joint\"", - "eval_split": "\"test\"", - "groups": "[\"mmlu_abstract_algebra\", \"mmlu_anatomy\", \"mmlu_astronomy\", \"mmlu_business_ethics\", \"mmlu_clinical_knowledge\", \"mmlu_college_biology\", \"mmlu_college_chemistry\", \"mmlu_college_computer_science\", \"mmlu_college_mathematics\", \"mmlu_college_medicine\", \"mmlu_college_physics\", \"mmlu_computer_security\", \"mmlu_conceptual_physics\", \"mmlu_econometrics\", \"mmlu_electrical_engineering\", \"mmlu_elementary_mathematics\", \"mmlu_formal_logic\", \"mmlu_global_facts\", \"mmlu_high_school_biology\", \"mmlu_high_school_chemistry\", \"mmlu_high_school_computer_science\", \"mmlu_high_school_european_history\", \"mmlu_high_school_geography\", \"mmlu_high_school_government_and_politics\", \"mmlu_high_school_macroeconomics\", \"mmlu_high_school_mathematics\", \"mmlu_high_school_microeconomics\", \"mmlu_high_school_physics\", \"mmlu_high_school_psychology\", \"mmlu_high_school_statistics\", \"mmlu_high_school_us_history\", \"mmlu_high_school_world_history\", \"mmlu_human_aging\", \"mmlu_human_sexuality\", \"mmlu_international_law\", \"mmlu_jurisprudence\", \"mmlu_logical_fallacies\", \"mmlu_machine_learning\", \"mmlu_management\", \"mmlu_marketing\", \"mmlu_medical_genetics\", \"mmlu_miscellaneous\", \"mmlu_moral_disputes\", \"mmlu_moral_scenarios\", \"mmlu_nutrition\", \"mmlu_philosophy\", \"mmlu_prehistory\", \"mmlu_professional_accounting\", \"mmlu_professional_law\", \"mmlu_professional_medicine\", \"mmlu_professional_psychology\", \"mmlu_public_relations\", \"mmlu_security_studies\", \"mmlu_sociology\", \"mmlu_us_foreign_policy\", \"mmlu_virology\", \"mmlu_world_religions\"]" - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/writer_palmyra-x5.json b/data/models/writer_palmyra-x5.json deleted file mode 100644 index 3b637f8e8bc4162017983dc2a503b3b921c70ca1..0000000000000000000000000000000000000000 --- a/data/models/writer_palmyra-x5.json +++ /dev/null @@ -1,234 +0,0 @@ -{ - "model_info": { - "name": "Palmyra X5", - "id": "writer/palmyra-x5", - "developer": "writer", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_capabilities/writer_palmyra-x5/1774096304.5056698", - "retrieved_timestamp": "1774096304.5056698", - "source_metadata": { - "source_name": "helm_capabilities", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_capabilities", - "evaluation_results": [ - { - "evaluation_name": "Mean score", - "source_data": { - "dataset_name": "helm_capabilities", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "The mean of the scores from all columns.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.696, - "details": { - "description": "", - "tab": "Accuracy", - "Mean score - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"7.539339301355213\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU-Pro", - "source_data": { - "dataset_name": "MMLU-Pro", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on MMLU-Pro", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.804, - "details": { - "description": "min=0.804, mean=0.804, max=0.804, sum=0.804 (1)", - "tab": "Accuracy", - "MMLU-Pro - Observed inference time (s)": "{\"description\": \"min=5.907, mean=5.907, max=5.907, sum=5.907 (1)\", \"tab\": \"Efficiency\", \"score\": \"5.906555171251297\"}", - "MMLU-Pro - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "MMLU-Pro - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - # prompt tokens": "{\"description\": \"min=249.366, mean=249.366, max=249.366, sum=249.366 (1)\", \"tab\": \"General information\", \"score\": \"249.366\"}", - "MMLU-Pro - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"all\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.661, - "details": { - "description": "min=0.661, mean=0.661, max=0.661, sum=0.661 (1)", - "tab": "Accuracy", - "GPQA - Observed inference time (s)": "{\"description\": \"min=9.251, mean=9.251, max=9.251, sum=9.251 (1)\", \"tab\": \"Efficiency\", \"score\": \"9.251234515365464\"}", - "GPQA - # eval": "{\"description\": \"min=446, mean=446, max=446, sum=446 (1)\", \"tab\": \"General information\", \"score\": \"446.0\"}", - "GPQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - # prompt tokens": "{\"description\": \"min=269.886, mean=269.886, max=269.886, sum=269.886 (1)\", \"tab\": \"General information\", \"score\": \"269.8856502242152\"}", - "GPQA - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"gpqa_main\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "IFEval Strict Acc on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.823, - "details": { - "description": "min=0.823, mean=0.823, max=0.823, sum=0.823 (1)", - "tab": "Accuracy", - "IFEval - Observed inference time (s)": "{\"description\": \"min=3.337, mean=3.337, max=3.337, sum=3.337 (1)\", \"tab\": \"Efficiency\", \"score\": \"3.3367519599012\"}", - "IFEval - # eval": "{\"description\": \"min=541, mean=541, max=541, sum=541 (1)\", \"tab\": \"General information\", \"score\": \"541.0\"}", - "IFEval - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - # prompt tokens": "{\"description\": \"min=46.024, mean=46.024, max=46.024, sum=46.024 (1)\", \"tab\": \"General information\", \"score\": \"46.024029574861366\"}", - "IFEval - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WildBench", - "source_data": { - "dataset_name": "WildBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "WB Score on WildBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.78, - "details": { - "description": "min=0.78, mean=0.78, max=0.78, sum=0.78 (1)", - "tab": "Accuracy", - "WildBench - Observed inference time (s)": "{\"description\": \"min=8.281, mean=8.281, max=8.281, sum=8.281 (1)\", \"tab\": \"Efficiency\", \"score\": \"8.280673936367036\"}", - "WildBench - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "WildBench - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # prompt tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"v2\"" - } - } - }, - { - "evaluation_name": "Omni-MATH", - "source_data": { - "dataset_name": "Omni-MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Acc on Omni-MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.414, - "details": { - "description": "min=0.414, mean=0.414, max=0.414, sum=0.414 (1)", - "tab": "Accuracy", - "Omni-MATH - Observed inference time (s)": "{\"description\": \"min=10.921, mean=10.921, max=10.921, sum=10.921 (1)\", \"tab\": \"Efficiency\", \"score\": \"10.921480923891068\"}", - "Omni-MATH - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "Omni-MATH - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - # prompt tokens": "{\"description\": \"min=109.708, mean=109.708, max=109.708, sum=109.708 (1)\", \"tab\": \"General information\", \"score\": \"109.708\"}", - "Omni-MATH - # output tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/wzhouad_gemma-2-9b-it-wpo-hb.json b/data/models/wzhouad_gemma-2-9b-it-wpo-hb.json deleted file mode 100644 index 2c1fa2829710f8b906d07f4b230cc03f6c498d51..0000000000000000000000000000000000000000 --- a/data/models/wzhouad_gemma-2-9b-it-wpo-hb.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "gemma-2-9b-it-WPO-HB", - "id": "wzhouad/gemma-2-9b-it-WPO-HB", - "developer": "wzhouad", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "9.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/wzhouad_gemma-2-9b-it-WPO-HB/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5437 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5629 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1533 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3498 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3675 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.336 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/x0000001_deepseek-lumen-r1-qwen2.5-14b.json b/data/models/x0000001_deepseek-lumen-r1-qwen2.5-14b.json deleted file mode 100644 index 316faa8a160ad36598ceb5aa1cad11ae9cd4a0bb..0000000000000000000000000000000000000000 --- a/data/models/x0000001_deepseek-lumen-r1-qwen2.5-14b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Deepseek-Lumen-R1-Qwen2.5-14B", - "id": "x0000001/Deepseek-Lumen-R1-Qwen2.5-14B", - "developer": "x0000001", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/x0000001_Deepseek-Lumen-R1-Qwen2.5-14B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4436 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4569 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2779 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2852 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.474 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4379 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/xai_grok-3-beta.json b/data/models/xai_grok-3-beta.json deleted file mode 100644 index b0eb74140b7da6c1cdddae779d743a17733b27a9..0000000000000000000000000000000000000000 --- a/data/models/xai_grok-3-beta.json +++ /dev/null @@ -1,234 +0,0 @@ -{ - "model_info": { - "name": "Grok 3 Beta", - "id": "xai/grok-3-beta", - "developer": "xAI", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_capabilities/xai_grok-3-beta/1774096304.5056698", - "retrieved_timestamp": "1774096304.5056698", - "source_metadata": { - "source_name": "helm_capabilities", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_capabilities", - "evaluation_results": [ - { - "evaluation_name": "Mean score", - "source_data": { - "dataset_name": "helm_capabilities", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "The mean of the scores from all columns.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.727, - "details": { - "description": "", - "tab": "Accuracy", - "Mean score - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"21.832675643266274\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU-Pro", - "source_data": { - "dataset_name": "MMLU-Pro", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on MMLU-Pro", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.788, - "details": { - "description": "min=0.788, mean=0.788, max=0.788, sum=0.788 (1)", - "tab": "Accuracy", - "MMLU-Pro - Observed inference time (s)": "{\"description\": \"min=24.646, mean=24.646, max=24.646, sum=24.646 (1)\", \"tab\": \"Efficiency\", \"score\": \"24.646376408576966\"}", - "MMLU-Pro - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "MMLU-Pro - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - truncated": "{\"description\": \"min=0.013, mean=0.013, max=0.013, sum=0.013 (1)\", \"tab\": \"General information\", \"score\": \"0.013\"}", - "MMLU-Pro - # prompt tokens": "{\"description\": \"min=223.237, mean=223.237, max=223.237, sum=223.237 (1)\", \"tab\": \"General information\", \"score\": \"223.237\"}", - "MMLU-Pro - # output tokens": "{\"description\": \"min=1669.743, mean=1669.743, max=1669.743, sum=1669.743 (1)\", \"tab\": \"General information\", \"score\": \"1669.743\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"all\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.65, - "details": { - "description": "min=0.65, mean=0.65, max=0.65, sum=0.65 (1)", - "tab": "Accuracy", - "GPQA - Observed inference time (s)": "{\"description\": \"min=30.888, mean=30.888, max=30.888, sum=30.888 (1)\", \"tab\": \"Efficiency\", \"score\": \"30.88756059317311\"}", - "GPQA - # eval": "{\"description\": \"min=446, mean=446, max=446, sum=446 (1)\", \"tab\": \"General information\", \"score\": \"446.0\"}", - "GPQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - truncated": "{\"description\": \"min=0.02, mean=0.02, max=0.02, sum=0.02 (1)\", \"tab\": \"General information\", \"score\": \"0.020179372197309416\"}", - "GPQA - # prompt tokens": "{\"description\": \"min=233.054, mean=233.054, max=233.054, sum=233.054 (1)\", \"tab\": \"General information\", \"score\": \"233.05381165919283\"}", - "GPQA - # output tokens": "{\"description\": \"min=2771.594, mean=2771.594, max=2771.594, sum=2771.594 (1)\", \"tab\": \"General information\", \"score\": \"2771.5941704035877\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"gpqa_main\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "IFEval Strict Acc on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.884, - "details": { - "description": "min=0.884, mean=0.884, max=0.884, sum=0.884 (1)", - "tab": "Accuracy", - "IFEval - Observed inference time (s)": "{\"description\": \"min=5.792, mean=5.792, max=5.792, sum=5.792 (1)\", \"tab\": \"Efficiency\", \"score\": \"5.791596473475261\"}", - "IFEval - # eval": "{\"description\": \"min=541, mean=541, max=541, sum=541 (1)\", \"tab\": \"General information\", \"score\": \"541.0\"}", - "IFEval - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - # prompt tokens": "{\"description\": \"min=45.192, mean=45.192, max=45.192, sum=45.192 (1)\", \"tab\": \"General information\", \"score\": \"45.19223659889094\"}", - "IFEval - # output tokens": "{\"description\": \"min=404.85, mean=404.85, max=404.85, sum=404.85 (1)\", \"tab\": \"General information\", \"score\": \"404.8502772643253\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WildBench", - "source_data": { - "dataset_name": "WildBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "WB Score on WildBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.849, - "details": { - "description": "min=0.849, mean=0.849, max=0.849, sum=0.849 (1)", - "tab": "Accuracy", - "WildBench - Observed inference time (s)": "{\"description\": \"min=16.937, mean=16.937, max=16.937, sum=16.937 (1)\", \"tab\": \"Efficiency\", \"score\": \"16.93687919616699\"}", - "WildBench - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "WildBench - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # prompt tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # output tokens": "{\"description\": \"min=1419.576, mean=1419.576, max=1419.576, sum=1419.576 (1)\", \"tab\": \"General information\", \"score\": \"1419.576\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"v2\"" - } - } - }, - { - "evaluation_name": "Omni-MATH", - "source_data": { - "dataset_name": "Omni-MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Acc on Omni-MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.464, - "details": { - "description": "min=0.464, mean=0.464, max=0.464, sum=0.464 (1)", - "tab": "Accuracy", - "Omni-MATH - Observed inference time (s)": "{\"description\": \"min=30.901, mean=30.901, max=30.901, sum=30.901 (1)\", \"tab\": \"Efficiency\", \"score\": \"30.90096554493904\"}", - "Omni-MATH - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "Omni-MATH - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - truncated": "{\"description\": \"min=0.001, mean=0.001, max=0.001, sum=0.001 (1)\", \"tab\": \"General information\", \"score\": \"0.001\"}", - "Omni-MATH - # prompt tokens": "{\"description\": \"min=104.089, mean=104.089, max=104.089, sum=104.089 (1)\", \"tab\": \"General information\", \"score\": \"104.089\"}", - "Omni-MATH - # output tokens": "{\"description\": \"min=3296.733, mean=3296.733, max=3296.733, sum=3296.733 (1)\", \"tab\": \"General information\", \"score\": \"3296.733\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/xai_grok-3-mini-beta.json b/data/models/xai_grok-3-mini-beta.json deleted file mode 100644 index 5079a7225292fd4dc5b529a5716bc8ab847fa4a2..0000000000000000000000000000000000000000 --- a/data/models/xai_grok-3-mini-beta.json +++ /dev/null @@ -1,234 +0,0 @@ -{ - "model_info": { - "name": "Grok 3 mini Beta", - "id": "xai/grok-3-mini-beta", - "developer": "xAI", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_capabilities/xai_grok-3-mini-beta/1774096304.5056698", - "retrieved_timestamp": "1774096304.5056698", - "source_metadata": { - "source_name": "helm_capabilities", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_capabilities", - "evaluation_results": [ - { - "evaluation_name": "Mean score", - "source_data": { - "dataset_name": "helm_capabilities", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "The mean of the scores from all columns.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.679, - "details": { - "description": "", - "tab": "Accuracy", - "Mean score - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"12.070258432341626\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU-Pro", - "source_data": { - "dataset_name": "MMLU-Pro", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on MMLU-Pro", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.799, - "details": { - "description": "min=0.799, mean=0.799, max=0.799, sum=0.799 (1)", - "tab": "Accuracy", - "MMLU-Pro - Observed inference time (s)": "{\"description\": \"min=7.153, mean=7.153, max=7.153, sum=7.153 (1)\", \"tab\": \"Efficiency\", \"score\": \"7.153050385713577\"}", - "MMLU-Pro - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "MMLU-Pro - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - truncated": "{\"description\": \"min=0.013, mean=0.013, max=0.013, sum=0.013 (1)\", \"tab\": \"General information\", \"score\": \"0.013\"}", - "MMLU-Pro - # prompt tokens": "{\"description\": \"min=223.237, mean=223.237, max=223.237, sum=223.237 (1)\", \"tab\": \"General information\", \"score\": \"223.237\"}", - "MMLU-Pro - # output tokens": "{\"description\": \"min=59.839, mean=59.839, max=59.839, sum=59.839 (1)\", \"tab\": \"General information\", \"score\": \"59.839\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"all\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.675, - "details": { - "description": "min=0.675, mean=0.675, max=0.675, sum=0.675 (1)", - "tab": "Accuracy", - "GPQA - Observed inference time (s)": "{\"description\": \"min=14.215, mean=14.215, max=14.215, sum=14.215 (1)\", \"tab\": \"Efficiency\", \"score\": \"14.215015458419185\"}", - "GPQA - # eval": "{\"description\": \"min=446, mean=446, max=446, sum=446 (1)\", \"tab\": \"General information\", \"score\": \"446.0\"}", - "GPQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - truncated": "{\"description\": \"min=0.02, mean=0.02, max=0.02, sum=0.02 (1)\", \"tab\": \"General information\", \"score\": \"0.020179372197309416\"}", - "GPQA - # prompt tokens": "{\"description\": \"min=233.054, mean=233.054, max=233.054, sum=233.054 (1)\", \"tab\": \"General information\", \"score\": \"233.05381165919283\"}", - "GPQA - # output tokens": "{\"description\": \"min=125.807, mean=125.807, max=125.807, sum=125.807 (1)\", \"tab\": \"General information\", \"score\": \"125.80717488789237\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"gpqa_main\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "IFEval Strict Acc on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.951, - "details": { - "description": "min=0.951, mean=0.951, max=0.951, sum=0.951 (1)", - "tab": "Accuracy", - "IFEval - Observed inference time (s)": "{\"description\": \"min=7.187, mean=7.187, max=7.187, sum=7.187 (1)\", \"tab\": \"Efficiency\", \"score\": \"7.187224511077797\"}", - "IFEval - # eval": "{\"description\": \"min=541, mean=541, max=541, sum=541 (1)\", \"tab\": \"General information\", \"score\": \"541.0\"}", - "IFEval - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - # prompt tokens": "{\"description\": \"min=45.192, mean=45.192, max=45.192, sum=45.192 (1)\", \"tab\": \"General information\", \"score\": \"45.19223659889094\"}", - "IFEval - # output tokens": "{\"description\": \"min=347.104, mean=347.104, max=347.104, sum=347.104 (1)\", \"tab\": \"General information\", \"score\": \"347.10351201478744\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WildBench", - "source_data": { - "dataset_name": "WildBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "WB Score on WildBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.651, - "details": { - "description": "min=0.651, mean=0.651, max=0.651, sum=0.651 (1)", - "tab": "Accuracy", - "WildBench - Observed inference time (s)": "{\"description\": \"min=10.787, mean=10.787, max=10.787, sum=10.787 (1)\", \"tab\": \"Efficiency\", \"score\": \"10.787254344463348\"}", - "WildBench - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "WildBench - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # prompt tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # output tokens": "{\"description\": \"min=864.463, mean=864.463, max=864.463, sum=864.463 (1)\", \"tab\": \"General information\", \"score\": \"864.463\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"v2\"" - } - } - }, - { - "evaluation_name": "Omni-MATH", - "source_data": { - "dataset_name": "Omni-MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Acc on Omni-MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.318, - "details": { - "description": "min=0.318, mean=0.318, max=0.318, sum=0.318 (1)", - "tab": "Accuracy", - "Omni-MATH - Observed inference time (s)": "{\"description\": \"min=21.009, mean=21.009, max=21.009, sum=21.009 (1)\", \"tab\": \"Efficiency\", \"score\": \"21.008747462034226\"}", - "Omni-MATH - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "Omni-MATH - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - truncated": "{\"description\": \"min=0.001, mean=0.001, max=0.001, sum=0.001 (1)\", \"tab\": \"General information\", \"score\": \"0.001\"}", - "Omni-MATH - # prompt tokens": "{\"description\": \"min=104.089, mean=104.089, max=104.089, sum=104.089 (1)\", \"tab\": \"General information\", \"score\": \"104.089\"}", - "Omni-MATH - # output tokens": "{\"description\": \"min=183.116, mean=183.116, max=183.116, sum=183.116 (1)\", \"tab\": \"General information\", \"score\": \"183.116\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/xai_grok-3-mini.json b/data/models/xai_grok-3-mini.json deleted file mode 100644 index 7fda2d0643a4e1f5eb9d98f1dc44e9ec85970d5d..0000000000000000000000000000000000000000 --- a/data/models/xai_grok-3-mini.json +++ /dev/null @@ -1,1043 +0,0 @@ -{ - "model_info": { - "name": "grok-3-mini", - "id": "xai/grok-3-mini", - "developer": "xAI", - "inference_platform": "unknown", - "additional_details": { - "display_name": "Grok 3 Mini" - } - }, - "evaluations": [ - { - "evaluation_id": "global-mmlu-lite/xai_grok-3-mini/1773936496.366405", - "retrieved_timestamp": "1773936496.366405", - "source_metadata": { - "source_name": "Global MMLU Lite Leaderboard", - "source_type": "documentation", - "source_organization_name": "kaggle", - "source_organization_url": "www.kaggle.com", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "kaggle kernel", - "version": "4", - "additional_details": { - "url": "https://www.kaggle.com/code/shivalikasingh95/global-mmlu-lite-sample-notebook" - } - }, - "benchmark": "global-mmlu-lite", - "evaluation_results": [ - { - "evaluation_name": "Global MMLU Lite", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Global MMLU Lite", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.673 - } - }, - { - "evaluation_name": "Culturally Sensitive", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Sensitive", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6717 - } - }, - { - "evaluation_name": "Culturally Agnostic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Agnostic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6743 - } - }, - { - "evaluation_name": "Arabic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Arabic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.755, - "uncertainty": { - "confidence_interval": { - "lower": -0.0421, - "upper": 0.0421, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "English", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - English", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5075, - "uncertainty": { - "confidence_interval": { - "lower": -0.049, - "upper": 0.049, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Bengali", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Bengali", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7355, - "uncertainty": { - "confidence_interval": { - "lower": -0.0434, - "upper": 0.0434, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "German", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - German", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6591, - "uncertainty": { - "confidence_interval": { - "lower": -0.0465, - "upper": 0.0465, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "French", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - French", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.485, - "uncertainty": { - "confidence_interval": { - "lower": -0.049, - "upper": 0.049, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Hindi", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Hindi", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.56, - "uncertainty": { - "confidence_interval": { - "lower": -0.0486, - "upper": 0.0486, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Indonesian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Indonesian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.725, - "uncertainty": { - "confidence_interval": { - "lower": -0.0438, - "upper": 0.0438, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Italian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Italian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.696, - "uncertainty": { - "confidence_interval": { - "lower": -0.0452, - "upper": 0.0452, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Japanese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Japanese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6575, - "uncertainty": { - "confidence_interval": { - "lower": -0.0465, - "upper": 0.0465, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Korean", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Korean", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7325, - "uncertainty": { - "confidence_interval": { - "lower": -0.0434, - "upper": 0.0434, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Portuguese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Portuguese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6275, - "uncertainty": { - "confidence_interval": { - "lower": -0.0474, - "upper": 0.0474, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Spanish", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Spanish", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.61, - "uncertainty": { - "confidence_interval": { - "lower": -0.0478, - "upper": 0.0478, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Swahili", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Swahili", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7625, - "uncertainty": { - "confidence_interval": { - "lower": -0.0417, - "upper": 0.0417, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Yoruba", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Yoruba", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8296, - "uncertainty": { - "confidence_interval": { - "lower": -0.0369, - "upper": 0.0369, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Chinese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Chinese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5564, - "uncertainty": { - "confidence_interval": { - "lower": -0.0487, - "upper": 0.0487, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Burmese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Burmese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8693, - "uncertainty": { - "confidence_interval": { - "lower": -0.0331, - "upper": 0.0331, - "method": "unknown" - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "global-mmlu-lite/xai_grok-3-mini/1773936583.743359", - "retrieved_timestamp": "1773936583.743359", - "source_metadata": { - "source_name": "Global MMLU Lite Leaderboard", - "source_type": "documentation", - "source_organization_name": "kaggle", - "source_organization_url": "www.kaggle.com", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "kaggle kernel", - "version": "4", - "additional_details": { - "url": "https://www.kaggle.com/code/shivalikasingh95/global-mmlu-lite-sample-notebook" - } - }, - "benchmark": "global-mmlu-lite", - "evaluation_results": [ - { - "evaluation_name": "Global MMLU Lite", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Global MMLU Lite", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.673 - } - }, - { - "evaluation_name": "Culturally Sensitive", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Sensitive", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6717 - } - }, - { - "evaluation_name": "Culturally Agnostic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Agnostic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6743 - } - }, - { - "evaluation_name": "Arabic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Arabic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.755, - "uncertainty": { - "confidence_interval": { - "lower": -0.0421, - "upper": 0.0421, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "English", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - English", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5075, - "uncertainty": { - "confidence_interval": { - "lower": -0.049, - "upper": 0.049, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Bengali", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Bengali", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7355, - "uncertainty": { - "confidence_interval": { - "lower": -0.0434, - "upper": 0.0434, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "German", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - German", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6591, - "uncertainty": { - "confidence_interval": { - "lower": -0.0465, - "upper": 0.0465, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "French", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - French", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.485, - "uncertainty": { - "confidence_interval": { - "lower": -0.049, - "upper": 0.049, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Hindi", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Hindi", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.56, - "uncertainty": { - "confidence_interval": { - "lower": -0.0486, - "upper": 0.0486, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Indonesian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Indonesian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.725, - "uncertainty": { - "confidence_interval": { - "lower": -0.0438, - "upper": 0.0438, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Italian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Italian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.696, - "uncertainty": { - "confidence_interval": { - "lower": -0.0452, - "upper": 0.0452, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Japanese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Japanese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6575, - "uncertainty": { - "confidence_interval": { - "lower": -0.0465, - "upper": 0.0465, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Korean", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Korean", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7325, - "uncertainty": { - "confidence_interval": { - "lower": -0.0434, - "upper": 0.0434, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Portuguese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Portuguese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6275, - "uncertainty": { - "confidence_interval": { - "lower": -0.0474, - "upper": 0.0474, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Spanish", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Spanish", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.61, - "uncertainty": { - "confidence_interval": { - "lower": -0.0478, - "upper": 0.0478, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Swahili", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Swahili", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7625, - "uncertainty": { - "confidence_interval": { - "lower": -0.0417, - "upper": 0.0417, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Yoruba", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Yoruba", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8296, - "uncertainty": { - "confidence_interval": { - "lower": -0.0369, - "upper": 0.0369, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Chinese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Chinese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5564, - "uncertainty": { - "confidence_interval": { - "lower": -0.0487, - "upper": 0.0487, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Burmese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Burmese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8693, - "uncertainty": { - "confidence_interval": { - "lower": -0.0331, - "upper": 0.0331, - "method": "unknown" - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/xai_grok-4-0709-fc.json b/data/models/xai_grok-4-0709-fc.json deleted file mode 100644 index e3b43b6d28029d1d7aea1ef0383845758d2d02ae..0000000000000000000000000000000000000000 --- a/data/models/xai_grok-4-0709-fc.json +++ /dev/null @@ -1,851 +0,0 @@ -{ - "model_info": { - "name": "Grok-4-0709 (FC)", - "id": "xai/grok-4-0709-fc", - "developer": "xAI", - "additional_details": { - "raw_model_name": "Grok-4-0709 (FC)", - "organization": "xAI", - "license": "Proprietary", - "mode": "FC", - "model_link": "https://docs.x.ai/docs/models" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/xai/grok-4-0709-fc/1775236112.3706062", - "retrieved_timestamp": "1775236112.3706062", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 10.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 61.38 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 355.17 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 15.49 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 26.22 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 44.28 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 85.38 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 73.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 92.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 88.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 87.0 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 75.57 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 82.17 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 73.88 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 75.0 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 79.17 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 33.88 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 44.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 19.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 28.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 44.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 82.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 80.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 84.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 55.91 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 57.42 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 58.71 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 51.61 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 87.5 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 75.4 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/xai_grok-4-0709-prompt.json b/data/models/xai_grok-4-0709-prompt.json deleted file mode 100644 index 345266daa8b82c0bf5bfc51fbab04b2371418552..0000000000000000000000000000000000000000 --- a/data/models/xai_grok-4-0709-prompt.json +++ /dev/null @@ -1,905 +0,0 @@ -{ - "model_info": { - "name": "Grok-4-0709 (Prompt)", - "id": "xai/grok-4-0709-prompt", - "developer": "xAI", - "additional_details": { - "raw_model_name": "Grok-4-0709 (Prompt)", - "organization": "xAI", - "license": "Proprietary", - "mode": "Prompt", - "model_link": "https://docs.x.ai/docs/models" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/xai/grok-4-0709-prompt/1775236112.370108", - "retrieved_timestamp": "1775236112.370108", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 9.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 62.97 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 348.19 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 30.38 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 36.19 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 101.54 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 82.75 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 67.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 93.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 89.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 81.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 72.54 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 81.78 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 70.18 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 81.25 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 70.83 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 47.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 55.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 46.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 36.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 50.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 74.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 74.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 74.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 50.54 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 43.87 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 59.35 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 48.39 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 81.25 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 84.3 - } - }, - { - "evaluation_result_id": "bfcl.format_sensitivity.max_delta::max_delta", - "evaluation_name": "bfcl.format_sensitivity.max_delta", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.format_sensitivity.max_delta", - "metric_name": "Format sensitivity max delta", - "metric_kind": "difference", - "metric_unit": "percentage_points", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Format Sensitivity Max Delta" - } - }, - "score_details": { - "score": 13.0 - } - }, - { - "evaluation_result_id": "bfcl.format_sensitivity.stddev::stddev", - "evaluation_name": "bfcl.format_sensitivity.stddev", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.format_sensitivity.stddev", - "metric_name": "Format sensitivity standard deviation", - "metric_kind": "difference", - "metric_unit": "percentage_points", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Format Sensitivity Standard Deviation" - } - }, - "score_details": { - "score": 2.88 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/xai_grok-4-0709.json b/data/models/xai_grok-4-0709.json deleted file mode 100644 index c9c4c964a3fc0712105da9d9e183b32a52cc54bc..0000000000000000000000000000000000000000 --- a/data/models/xai_grok-4-0709.json +++ /dev/null @@ -1,1267 +0,0 @@ -{ - "model_info": { - "name": "grok-4-0709", - "id": "xai/grok-4-0709", - "developer": "xAI", - "inference_platform": "unknown", - "additional_details": { - "display_name": "Grok 4" - } - }, - "evaluations": [ - { - "evaluation_id": "global-mmlu-lite/xai_grok-4-0709/1773936583.743359", - "retrieved_timestamp": "1773936583.743359", - "source_metadata": { - "source_name": "Global MMLU Lite Leaderboard", - "source_type": "documentation", - "source_organization_name": "kaggle", - "source_organization_url": "www.kaggle.com", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "kaggle kernel", - "version": "4", - "additional_details": { - "url": "https://www.kaggle.com/code/shivalikasingh95/global-mmlu-lite-sample-notebook" - } - }, - "benchmark": "global-mmlu-lite", - "evaluation_results": [ - { - "evaluation_name": "Global MMLU Lite", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Global MMLU Lite", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8881 - } - }, - { - "evaluation_name": "Culturally Sensitive", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Sensitive", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8862 - } - }, - { - "evaluation_name": "Culturally Agnostic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Agnostic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.89 - } - }, - { - "evaluation_name": "Arabic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Arabic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.885, - "uncertainty": { - "confidence_interval": { - "lower": -0.0313, - "upper": 0.0313, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "English", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - English", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.905, - "uncertainty": { - "confidence_interval": { - "lower": -0.0287, - "upper": 0.0287, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Bengali", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Bengali", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8925, - "uncertainty": { - "confidence_interval": { - "lower": -0.0304, - "upper": 0.0304, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "German", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - German", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8725, - "uncertainty": { - "confidence_interval": { - "lower": -0.0327, - "upper": 0.0327, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "French", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - French", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.875, - "uncertainty": { - "confidence_interval": { - "lower": -0.0324, - "upper": 0.0324, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Hindi", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Hindi", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8675, - "uncertainty": { - "confidence_interval": { - "lower": -0.0332, - "upper": 0.0332, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Indonesian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Indonesian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.89, - "uncertainty": { - "confidence_interval": { - "lower": -0.0307, - "upper": 0.0307, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Italian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Italian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9025, - "uncertainty": { - "confidence_interval": { - "lower": -0.0291, - "upper": 0.0291, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Japanese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Japanese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.87, - "uncertainty": { - "confidence_interval": { - "lower": -0.033, - "upper": 0.033, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Korean", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Korean", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.895, - "uncertainty": { - "confidence_interval": { - "lower": -0.03, - "upper": 0.03, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Portuguese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Portuguese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8725, - "uncertainty": { - "confidence_interval": { - "lower": -0.0327, - "upper": 0.0327, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Spanish", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Spanish", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9075, - "uncertainty": { - "confidence_interval": { - "lower": -0.0284, - "upper": 0.0284, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Swahili", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Swahili", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.91, - "uncertainty": { - "confidence_interval": { - "lower": -0.028, - "upper": 0.028, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Yoruba", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Yoruba", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.905, - "uncertainty": { - "confidence_interval": { - "lower": -0.0287, - "upper": 0.0287, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Chinese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Chinese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8525, - "uncertainty": { - "confidence_interval": { - "lower": -0.0348, - "upper": 0.0348, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Burmese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Burmese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9075, - "uncertainty": { - "confidence_interval": { - "lower": -0.0284, - "upper": 0.0284, - "method": "unknown" - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "global-mmlu-lite/xai_grok-4-0709/1773936496.366405", - "retrieved_timestamp": "1773936496.366405", - "source_metadata": { - "source_name": "Global MMLU Lite Leaderboard", - "source_type": "documentation", - "source_organization_name": "kaggle", - "source_organization_url": "www.kaggle.com", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "kaggle kernel", - "version": "4", - "additional_details": { - "url": "https://www.kaggle.com/code/shivalikasingh95/global-mmlu-lite-sample-notebook" - } - }, - "benchmark": "global-mmlu-lite", - "evaluation_results": [ - { - "evaluation_name": "Global MMLU Lite", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Global MMLU Lite", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8881 - } - }, - { - "evaluation_name": "Culturally Sensitive", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Sensitive", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8862 - } - }, - { - "evaluation_name": "Culturally Agnostic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Culturally Agnostic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.89 - } - }, - { - "evaluation_name": "Arabic", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Arabic", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.885, - "uncertainty": { - "confidence_interval": { - "lower": -0.0313, - "upper": 0.0313, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "English", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - English", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.905, - "uncertainty": { - "confidence_interval": { - "lower": -0.0287, - "upper": 0.0287, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Bengali", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Bengali", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8925, - "uncertainty": { - "confidence_interval": { - "lower": -0.0304, - "upper": 0.0304, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "German", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - German", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8725, - "uncertainty": { - "confidence_interval": { - "lower": -0.0327, - "upper": 0.0327, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "French", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - French", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.875, - "uncertainty": { - "confidence_interval": { - "lower": -0.0324, - "upper": 0.0324, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Hindi", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Hindi", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8675, - "uncertainty": { - "confidence_interval": { - "lower": -0.0332, - "upper": 0.0332, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Indonesian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Indonesian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.89, - "uncertainty": { - "confidence_interval": { - "lower": -0.0307, - "upper": 0.0307, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Italian", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Italian", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9025, - "uncertainty": { - "confidence_interval": { - "lower": -0.0291, - "upper": 0.0291, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Japanese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Japanese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.87, - "uncertainty": { - "confidence_interval": { - "lower": -0.033, - "upper": 0.033, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Korean", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Korean", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.895, - "uncertainty": { - "confidence_interval": { - "lower": -0.03, - "upper": 0.03, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Portuguese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Portuguese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8725, - "uncertainty": { - "confidence_interval": { - "lower": -0.0327, - "upper": 0.0327, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Spanish", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Spanish", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9075, - "uncertainty": { - "confidence_interval": { - "lower": -0.0284, - "upper": 0.0284, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Swahili", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Swahili", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.91, - "uncertainty": { - "confidence_interval": { - "lower": -0.028, - "upper": 0.028, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Yoruba", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Yoruba", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.905, - "uncertainty": { - "confidence_interval": { - "lower": -0.0287, - "upper": 0.0287, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Chinese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Chinese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8525, - "uncertainty": { - "confidence_interval": { - "lower": -0.0348, - "upper": 0.0348, - "method": "unknown" - } - } - } - }, - { - "evaluation_name": "Burmese", - "source_data": { - "dataset_name": "global-mmlu-lite", - "source_type": "url", - "url": [ - "https://www.kaggle.com/datasets/cohere-labs/global-mmlu-lite" - ] - }, - "metric_config": { - "evaluation_description": "Global MMLU Lite - Burmese", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.9075, - "uncertainty": { - "confidence_interval": { - "lower": -0.0284, - "upper": 0.0284, - "method": "unknown" - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "helm_capabilities/xai_grok-4-0709/1774096304.5056698", - "retrieved_timestamp": "1774096304.5056698", - "source_metadata": { - "source_name": "helm_capabilities", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_capabilities", - "evaluation_results": [ - { - "evaluation_name": "Mean score", - "source_data": { - "dataset_name": "helm_capabilities", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "The mean of the scores from all columns.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.785, - "details": { - "description": "", - "tab": "Accuracy", - "Mean score - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"128.04182146459848\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU-Pro", - "source_data": { - "dataset_name": "MMLU-Pro", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on MMLU-Pro", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.851, - "details": { - "description": "min=0.851, mean=0.851, max=0.851, sum=0.851 (1)", - "tab": "Accuracy", - "MMLU-Pro - Observed inference time (s)": "{\"description\": \"min=93.583, mean=93.583, max=93.583, sum=93.583 (1)\", \"tab\": \"Efficiency\", \"score\": \"93.58286614966393\"}", - "MMLU-Pro - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "MMLU-Pro - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - truncated": "{\"description\": \"min=0.013, mean=0.013, max=0.013, sum=0.013 (1)\", \"tab\": \"General information\", \"score\": \"0.013\"}", - "MMLU-Pro - # prompt tokens": "{\"description\": \"min=244.237, mean=244.237, max=244.237, sum=244.237 (1)\", \"tab\": \"General information\", \"score\": \"244.237\"}", - "MMLU-Pro - # output tokens": "{\"description\": \"min=4.789, mean=4.789, max=4.789, sum=4.789 (1)\", \"tab\": \"General information\", \"score\": \"4.789\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"all\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.726, - "details": { - "description": "min=0.726, mean=0.726, max=0.726, sum=0.726 (1)", - "tab": "Accuracy", - "GPQA - Observed inference time (s)": "{\"description\": \"min=223.967, mean=223.967, max=223.967, sum=223.967 (1)\", \"tab\": \"Efficiency\", \"score\": \"223.96746500778625\"}", - "GPQA - # eval": "{\"description\": \"min=446, mean=446, max=446, sum=446 (1)\", \"tab\": \"General information\", \"score\": \"446.0\"}", - "GPQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - truncated": "{\"description\": \"min=0.02, mean=0.02, max=0.02, sum=0.02 (1)\", \"tab\": \"General information\", \"score\": \"0.020179372197309416\"}", - "GPQA - # prompt tokens": "{\"description\": \"min=254.007, mean=254.007, max=254.007, sum=254.007 (1)\", \"tab\": \"General information\", \"score\": \"254.0067264573991\"}", - "GPQA - # output tokens": "{\"description\": \"min=5.841, mean=5.841, max=5.841, sum=5.841 (1)\", \"tab\": \"General information\", \"score\": \"5.8408071748878925\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"gpqa_main\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "IFEval Strict Acc on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.949, - "details": { - "description": "min=0.949, mean=0.949, max=0.949, sum=0.949 (1)", - "tab": "Accuracy", - "IFEval - Observed inference time (s)": "{\"description\": \"min=31.966, mean=31.966, max=31.966, sum=31.966 (1)\", \"tab\": \"Efficiency\", \"score\": \"31.966069252786266\"}", - "IFEval - # eval": "{\"description\": \"min=541, mean=541, max=541, sum=541 (1)\", \"tab\": \"General information\", \"score\": \"541.0\"}", - "IFEval - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - # prompt tokens": "{\"description\": \"min=45.192, mean=45.192, max=45.192, sum=45.192 (1)\", \"tab\": \"General information\", \"score\": \"45.19223659889094\"}", - "IFEval - # output tokens": "{\"description\": \"min=376.298, mean=376.298, max=376.298, sum=376.298 (1)\", \"tab\": \"General information\", \"score\": \"376.29759704251387\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WildBench", - "source_data": { - "dataset_name": "WildBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "WB Score on WildBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.797, - "details": { - "description": "min=0.797, mean=0.797, max=0.797, sum=0.797 (1)", - "tab": "Accuracy", - "WildBench - Observed inference time (s)": "{\"description\": \"min=115.441, mean=115.441, max=115.441, sum=115.441 (1)\", \"tab\": \"Efficiency\", \"score\": \"115.44128810715675\"}", - "WildBench - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "WildBench - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # prompt tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # output tokens": "{\"description\": \"min=1553.96, mean=1553.96, max=1553.96, sum=1553.96 (1)\", \"tab\": \"General information\", \"score\": \"1553.96\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"v2\"" - } - } - }, - { - "evaluation_name": "Omni-MATH", - "source_data": { - "dataset_name": "Omni-MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Acc on Omni-MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.603, - "details": { - "description": "min=0.603, mean=0.603, max=0.603, sum=0.603 (1)", - "tab": "Accuracy", - "Omni-MATH - Observed inference time (s)": "{\"description\": \"min=175.251, mean=175.251, max=175.251, sum=175.251 (1)\", \"tab\": \"Efficiency\", \"score\": \"175.2514188055992\"}", - "Omni-MATH - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "Omni-MATH - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - truncated": "{\"description\": \"min=0.001, mean=0.001, max=0.001, sum=0.001 (1)\", \"tab\": \"General information\", \"score\": \"0.001\"}", - "Omni-MATH - # prompt tokens": "{\"description\": \"min=104.089, mean=104.089, max=104.089, sum=104.089 (1)\", \"tab\": \"General information\", \"score\": \"104.089\"}", - "Omni-MATH - # output tokens": "{\"description\": \"min=104.419, mean=104.419, max=104.419, sum=104.419 (1)\", \"tab\": \"General information\", \"score\": \"104.419\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/xai_grok-4-1-fast-non-reasoning-fc.json b/data/models/xai_grok-4-1-fast-non-reasoning-fc.json deleted file mode 100644 index f02ad28d5cf0d65dbde067994488ffc9b66c7c5d..0000000000000000000000000000000000000000 --- a/data/models/xai_grok-4-1-fast-non-reasoning-fc.json +++ /dev/null @@ -1,851 +0,0 @@ -{ - "model_info": { - "name": "Grok-4-1-fast-non-reasoning (FC)", - "id": "xai/grok-4-1-fast-non-reasoning-fc", - "developer": "xAI", - "additional_details": { - "raw_model_name": "Grok-4-1-fast-non-reasoning (FC)", - "organization": "xAI", - "license": "Proprietary", - "mode": "FC", - "model_link": "https://docs.x.ai/docs/models" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/xai/grok-4-1-fast-non-reasoning-fc/1775236112.3715682", - "retrieved_timestamp": "1775236112.3715682", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 12.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 58.29 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 16.27 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 2.29 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 7.31 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 5.34 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 88.13 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 76.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 93.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 93.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 90.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 77.94 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 82.95 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 76.92 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 75.0 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 70.83 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 46.75 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 58.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 39.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 37.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 52.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 75.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 74.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 76.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 26.24 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 20.65 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 20.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 38.06 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 81.25 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 74.09 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/xai_grok-4-1-fast-reasoning-fc.json b/data/models/xai_grok-4-1-fast-reasoning-fc.json deleted file mode 100644 index c0fcacc5dacc7a1373b00a961db84a5bd88d42ca..0000000000000000000000000000000000000000 --- a/data/models/xai_grok-4-1-fast-reasoning-fc.json +++ /dev/null @@ -1,851 +0,0 @@ -{ - "model_info": { - "name": "Grok-4-1-fast-reasoning (FC)", - "id": "xai/grok-4-1-fast-reasoning-fc", - "developer": "xAI", - "additional_details": { - "raw_model_name": "Grok-4-1-fast-reasoning (FC)", - "organization": "xAI", - "license": "Proprietary", - "mode": "FC", - "model_link": "https://docs.x.ai/docs/models" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/xai/grok-4-1-fast-reasoning-fc/1775236112.3680482", - "retrieved_timestamp": "1775236112.3680482", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 5.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 69.57 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 17.26 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 6.74 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 12.78 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 17.57 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 88.27 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 77.58 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 93.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 92.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 90.0 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 78.46 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 84.11 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 77.3 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 75.0 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 70.83 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 58.87 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 70.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 59.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 43.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 62.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 82.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 82.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 83.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 53.98 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 41.29 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 57.42 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 63.23 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 81.25 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 79.43 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/xai_grok-4.json b/data/models/xai_grok-4.json deleted file mode 100644 index 8fb7b1b24e602a14c14ed4217ea5e7cd9fdf0b69..0000000000000000000000000000000000000000 --- a/data/models/xai_grok-4.json +++ /dev/null @@ -1,235 +0,0 @@ -{ - "model_info": { - "name": "Grok 4", - "id": "xai/grok-4", - "developer": "xAI", - "additional_details": { - "agent_name": "OpenHands", - "agent_organization": "OpenHands" - } - }, - "evaluations": [ - { - "evaluation_id": "terminal-bench-2.0/openhands__grok-4/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2025-11-02", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 27.2, - "uncertainty": { - "standard_error": { - "value": 3.1 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"OpenHands\" -m \"Grok 4\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"OpenHands\" -m \"Grok 4\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - }, - { - "evaluation_id": "terminal-bench-2.0/mini-swe-agent__grok-4/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2025-11-03", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 25.4, - "uncertainty": { - "standard_error": { - "value": 2.9 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Mini-SWE-Agent\" -m \"Grok 4\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Mini-SWE-Agent\" -m \"Grok 4\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - }, - { - "evaluation_id": "terminal-bench-2.0/terminus-2__grok-4/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2025-10-31", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 23.1, - "uncertainty": { - "standard_error": { - "value": 2.9 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Terminus 2\" -m \"Grok 4\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Terminus 2\" -m \"Grok 4\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/xai_grok-code-fast-1.json b/data/models/xai_grok-code-fast-1.json deleted file mode 100644 index 80a497dcbe0e067252e23ff5d2567dcecf97e827..0000000000000000000000000000000000000000 --- a/data/models/xai_grok-code-fast-1.json +++ /dev/null @@ -1,161 +0,0 @@ -{ - "model_info": { - "name": "Grok Code Fast 1", - "id": "xai/grok-code-fast-1", - "developer": "xAI", - "additional_details": { - "agent_name": "Mini-SWE-Agent", - "agent_organization": "Princeton" - } - }, - "evaluations": [ - { - "evaluation_id": "terminal-bench-2.0/mini-swe-agent__grok-code-fast-1/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2025-11-03", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 25.8, - "uncertainty": { - "standard_error": { - "value": 2.6 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Mini-SWE-Agent\" -m \"Grok Code Fast 1\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Mini-SWE-Agent\" -m \"Grok Code Fast 1\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - }, - { - "evaluation_id": "terminal-bench-2.0/terminus-2__grok-code-fast-1/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2025-10-31", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 14.2, - "uncertainty": { - "standard_error": { - "value": 2.5 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Terminus 2\" -m \"Grok Code Fast 1\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Terminus 2\" -m \"Grok Code Fast 1\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/xai_grok_4.json b/data/models/xai_grok_4.json deleted file mode 100644 index fbf9d30c1108cbb6eb9b29b639a5f6b54640082d..0000000000000000000000000000000000000000 --- a/data/models/xai_grok_4.json +++ /dev/null @@ -1,215 +0,0 @@ -{ - "model_info": { - "name": "Grok 4", - "developer": "xAI", - "id": "xai/Grok 4", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "apex-agents/xai_grok-4/1773260200", - "retrieved_timestamp": "1773260200", - "source_metadata": { - "source_name": "Mercor APEX-Agents Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Mercor", - "source_organization_url": "https://www.mercor.com", - "evaluator_relationship": "first_party" - }, - "eval_library": { - "name": "archipelago", - "version": "1.0.0" - }, - "benchmark": "apex-agents", - "evaluation_results": [ - { - "evaluation_name": "Overall Pass@1", - "source_data": { - "dataset_name": "apex-agents", - "source_type": "hf_dataset", - "hf_repo": "mercor/apex-agents" - }, - "metric_config": { - "evaluation_description": "Overall Pass@1 (dataset card / paper snapshot).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.152, - "uncertainty": { - "confidence_interval": { - "lower": -0.024, - "upper": 0.025, - "method": "bootstrap" - } - } - } - }, - { - "evaluation_name": "Overall Pass@8", - "source_data": { - "dataset_name": "apex-agents", - "source_type": "hf_dataset", - "hf_repo": "mercor/apex-agents" - }, - "metric_config": { - "evaluation_description": "Overall Pass@8 (dataset card / paper snapshot).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.329, - "uncertainty": { - "confidence_interval": { - "lower": -0.042, - "upper": 0.044, - "method": "bootstrap" - } - } - } - }, - { - "evaluation_name": "Overall Mean Score", - "source_data": { - "dataset_name": "apex-agents", - "source_type": "hf_dataset", - "hf_repo": "mercor/apex-agents" - }, - "metric_config": { - "evaluation_description": "Overall mean rubric score.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.303 - } - }, - { - "evaluation_name": "Investment Banking Pass@1", - "source_data": { - "dataset_name": "apex-agents", - "source_type": "hf_dataset", - "hf_repo": "mercor/apex-agents" - }, - "metric_config": { - "evaluation_description": "Investment banking world Pass@1.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.17 - } - }, - { - "evaluation_name": "Management Consulting Pass@1", - "source_data": { - "dataset_name": "apex-agents", - "source_type": "hf_dataset", - "hf_repo": "mercor/apex-agents" - }, - "metric_config": { - "evaluation_description": "Management consulting world Pass@1.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.12 - } - }, - { - "evaluation_name": "Corporate Law Pass@1", - "source_data": { - "dataset_name": "apex-agents", - "source_type": "hf_dataset", - "hf_repo": "mercor/apex-agents" - }, - "metric_config": { - "evaluation_description": "Corporate law world Pass@1.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.165 - } - }, - { - "evaluation_name": "Corporate Lawyer Mean Score", - "source_data": { - "dataset_name": "apex-agents", - "source_type": "hf_dataset", - "hf_repo": "mercor/apex-agents" - }, - "metric_config": { - "evaluation_description": "Corporate lawyer world mean score.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.41 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "apex-v1/xai_grok-4/1773260200", - "retrieved_timestamp": "1773260200", - "source_metadata": { - "source_name": "Mercor APEX-v1 Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Mercor", - "source_organization_url": "https://www.mercor.com", - "evaluator_relationship": "first_party" - }, - "eval_library": { - "name": "archipelago", - "version": "1.0.0" - }, - "benchmark": "apex-v1", - "evaluation_results": [ - { - "evaluation_name": "Overall Score", - "source_data": { - "dataset_name": "apex-v1", - "source_type": "hf_dataset", - "hf_repo": "Mercor/APEX-v1" - }, - "metric_config": { - "evaluation_description": "Overall APEX-v1 mean score across all jobs.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.635, - "uncertainty": { - "confidence_interval": { - "lower": -0.025, - "upper": 0.025, - "method": "bootstrap" - } - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/xclbr7_arcanum-12b.json b/data/models/xclbr7_arcanum-12b.json deleted file mode 100644 index dec46a0e8095a0148e71bf1559a894e1fa2981bb..0000000000000000000000000000000000000000 --- a/data/models/xclbr7_arcanum-12b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Arcanum-12b", - "id": "Xclbr7/Arcanum-12b", - "developer": "Xclbr7", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Xclbr7_Arcanum-12b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2907 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5265 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1193 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3205 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.417 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3586 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/xclbr7_caliburn-12b.json b/data/models/xclbr7_caliburn-12b.json deleted file mode 100644 index c2493a0596ee06367e58cbffdb44c44f9a0d9faf..0000000000000000000000000000000000000000 --- a/data/models/xclbr7_caliburn-12b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "caliburn-12b", - "id": "Xclbr7/caliburn-12b", - "developer": "Xclbr7", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Xclbr7_caliburn-12b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3576 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5519 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1125 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3364 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4292 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3675 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/xclbr7_caliburn-v2-12b.json b/data/models/xclbr7_caliburn-v2-12b.json deleted file mode 100644 index d440e25ecee84d63ad6179c852128e7f23a22696..0000000000000000000000000000000000000000 --- a/data/models/xclbr7_caliburn-v2-12b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "caliburn-v2-12b", - "id": "Xclbr7/caliburn-v2-12b", - "developer": "Xclbr7", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Xclbr7_caliburn-v2-12b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2967 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5141 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.105 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3263 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.437 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3784 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/xclbr7_hyena-12b.json b/data/models/xclbr7_hyena-12b.json deleted file mode 100644 index e36ebbe0ec2693f432c8234ff19af2260746d8a3..0000000000000000000000000000000000000000 --- a/data/models/xclbr7_hyena-12b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Hyena-12b", - "id": "Xclbr7/Hyena-12b", - "developer": "Xclbr7", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "12.248" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Xclbr7_Hyena-12b/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3404 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5457 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1133 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2978 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3984 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3439 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/xiaojian9992024_llama3.2-1b-threadripper-v0.2.json b/data/models/xiaojian9992024_llama3.2-1b-threadripper-v0.2.json deleted file mode 100644 index 4f17c9ecf96896379c43bfe9ab3e5c5c53a52243..0000000000000000000000000000000000000000 --- a/data/models/xiaojian9992024_llama3.2-1b-threadripper-v0.2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama3.2-1B-THREADRIPPER-v0.2", - "id": "Xiaojian9992024/Llama3.2-1B-THREADRIPPER-v0.2", - "developer": "Xiaojian9992024", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.236" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Xiaojian9992024_Llama3.2-1B-THREADRIPPER-v0.2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5318 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3528 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0657 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2659 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3316 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1745 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/xiaojian9992024_llama3.2-1b-threadripper.json b/data/models/xiaojian9992024_llama3.2-1b-threadripper.json deleted file mode 100644 index 68940916cfc78c1e41030824b38efa213eba2191..0000000000000000000000000000000000000000 --- a/data/models/xiaojian9992024_llama3.2-1b-threadripper.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama3.2-1B-THREADRIPPER", - "id": "Xiaojian9992024/Llama3.2-1B-THREADRIPPER", - "developer": "Xiaojian9992024", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "1.236" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Xiaojian9992024_Llama3.2-1B-THREADRIPPER/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5576 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3544 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.074 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2609 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.313 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1763 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/xiaojian9992024_phi-4-megatron-empathetic.json b/data/models/xiaojian9992024_phi-4-megatron-empathetic.json deleted file mode 100644 index 9721eb716fcd023824b19dd75158a2672abd1710..0000000000000000000000000000000000000000 --- a/data/models/xiaojian9992024_phi-4-megatron-empathetic.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Phi-4-Megatron-Empathetic", - "id": "Xiaojian9992024/Phi-4-Megatron-Empathetic", - "developer": "Xiaojian9992024", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "14.66" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Xiaojian9992024_Phi-4-Megatron-Empathetic/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0173 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6673 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2696 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3859 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5071 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5082 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/xiaojian9992024_phi-4-mini-unoffical.json b/data/models/xiaojian9992024_phi-4-mini-unoffical.json deleted file mode 100644 index 381f6032f2969265a5edececc266299e19aaf8cf..0000000000000000000000000000000000000000 --- a/data/models/xiaojian9992024_phi-4-mini-unoffical.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Phi-4-mini-UNOFFICAL", - "id": "Xiaojian9992024/Phi-4-mini-UNOFFICAL", - "developer": "Xiaojian9992024", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Phi3ForCausalLM", - "params_billions": "3.754" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Xiaojian9992024_Phi-4-mini-UNOFFICAL/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1273 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2944 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2408 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3368 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1144 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/xiaojian9992024_qwen2.5-7b-ms-destroyer.json b/data/models/xiaojian9992024_qwen2.5-7b-ms-destroyer.json deleted file mode 100644 index 34c2eae4da097a3260fd3db9c114680214eb178d..0000000000000000000000000000000000000000 --- a/data/models/xiaojian9992024_qwen2.5-7b-ms-destroyer.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-7B-MS-Destroyer", - "id": "Xiaojian9992024/Qwen2.5-7B-MS-Destroyer", - "developer": "Xiaojian9992024", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.613" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Xiaojian9992024_Qwen2.5-7B-MS-Destroyer/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7296 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.547 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4592 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3045 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.427 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4412 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/xiaojian9992024_qwen2.5-dyanka-7b-preview-v0.2.json b/data/models/xiaojian9992024_qwen2.5-dyanka-7b-preview-v0.2.json deleted file mode 100644 index 1861fff641589002bcc1bafda6d79ddbbf852685..0000000000000000000000000000000000000000 --- a/data/models/xiaojian9992024_qwen2.5-dyanka-7b-preview-v0.2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-Dyanka-7B-Preview-v0.2", - "id": "Xiaojian9992024/Qwen2.5-Dyanka-7B-Preview-v0.2", - "developer": "Xiaojian9992024", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Xiaojian9992024_Qwen2.5-Dyanka-7B-Preview-v0.2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6702 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5374 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4721 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2936 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4467 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4371 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/xiaojian9992024_qwen2.5-dyanka-7b-preview.json b/data/models/xiaojian9992024_qwen2.5-dyanka-7b-preview.json deleted file mode 100644 index 63130c33fb906756a2e168bd80827a255184a35a..0000000000000000000000000000000000000000 --- a/data/models/xiaojian9992024_qwen2.5-dyanka-7b-preview.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-Dyanka-7B-Preview", - "id": "Xiaojian9992024/Qwen2.5-Dyanka-7B-Preview", - "developer": "Xiaojian9992024", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Xiaojian9992024_Qwen2.5-Dyanka-7B-Preview/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.764 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5543 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4879 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3171 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4481 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4376 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/xiaojian9992024_qwen2.5-threadripper-medium-censored.json b/data/models/xiaojian9992024_qwen2.5-threadripper-medium-censored.json deleted file mode 100644 index cd464d8ae5697f49f4fdfc9e4bfefe3d0ed6e30d..0000000000000000000000000000000000000000 --- a/data/models/xiaojian9992024_qwen2.5-threadripper-medium-censored.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-THREADRIPPER-Medium-Censored", - "id": "Xiaojian9992024/Qwen2.5-THREADRIPPER-Medium-Censored", - "developer": "Xiaojian9992024", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Xiaojian9992024_Qwen2.5-THREADRIPPER-Medium-Censored/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8112 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6431 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.534 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3347 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.414 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4929 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/xiaojian9992024_qwen2.5-threadripper-small-anniversaryedition.json b/data/models/xiaojian9992024_qwen2.5-threadripper-small-anniversaryedition.json deleted file mode 100644 index 214e578b5e5a06bee5bc7fec8f377ba8a067c1ad..0000000000000000000000000000000000000000 --- a/data/models/xiaojian9992024_qwen2.5-threadripper-small-anniversaryedition.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-THREADRIPPER-Small-AnniversaryEdition", - "id": "Xiaojian9992024/Qwen2.5-THREADRIPPER-Small-AnniversaryEdition", - "developer": "Xiaojian9992024", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Xiaojian9992024_Qwen2.5-THREADRIPPER-Small-AnniversaryEdition/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7404 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5465 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5076 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2685 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3807 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4393 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/xiaojian9992024_qwen2.5-threadripper-small.json b/data/models/xiaojian9992024_qwen2.5-threadripper-small.json deleted file mode 100644 index 5cbac21d957a915872b6b260f1ab7544d5effb84..0000000000000000000000000000000000000000 --- a/data/models/xiaojian9992024_qwen2.5-threadripper-small.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-THREADRIPPER-Small", - "id": "Xiaojian9992024/Qwen2.5-THREADRIPPER-Small", - "developer": "Xiaojian9992024", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Xiaojian9992024_Qwen2.5-THREADRIPPER-Small/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7689 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.549 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4736 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3104 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4349 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4357 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/xiaojian9992024_qwen2.5-ultra-1.5b-25.02-exp.json b/data/models/xiaojian9992024_qwen2.5-ultra-1.5b-25.02-exp.json deleted file mode 100644 index 081e8afce6600d7a75d22dc61ea15d1873e7365f..0000000000000000000000000000000000000000 --- a/data/models/xiaojian9992024_qwen2.5-ultra-1.5b-25.02-exp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-Ultra-1.5B-25.02-Exp", - "id": "Xiaojian9992024/Qwen2.5-Ultra-1.5B-25.02-Exp", - "developer": "Xiaojian9992024", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.544" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Xiaojian9992024_Qwen2.5-Ultra-1.5B-25.02-Exp/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4073 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4066 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0831 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2584 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3383 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2641 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/xiaojian9992024_reflection-l3.2-jametminimix-3b.json b/data/models/xiaojian9992024_reflection-l3.2-jametminimix-3b.json deleted file mode 100644 index 111199f7f002da93f2d11a54f99e35abf072effe..0000000000000000000000000000000000000000 --- a/data/models/xiaojian9992024_reflection-l3.2-jametminimix-3b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Reflection-L3.2-JametMiniMix-3B", - "id": "Xiaojian9992024/Reflection-L3.2-JametMiniMix-3B", - "developer": "Xiaojian9992024", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Xiaojian9992024_Reflection-L3.2-JametMiniMix-3B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4619 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.439 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1193 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2945 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3667 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2988 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/xinchen9_llama3-b8-ft-dis.json b/data/models/xinchen9_llama3-b8-ft-dis.json deleted file mode 100644 index 9cfcd1195ddcd72005beed9dfb4a735e9c20b01d..0000000000000000000000000000000000000000 --- a/data/models/xinchen9_llama3-b8-ft-dis.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "llama3-b8-ft-dis", - "id": "xinchen9/llama3-b8-ft-dis", - "developer": "xinchen9", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/xinchen9_llama3-b8-ft-dis/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1546 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4626 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0393 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3129 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3654 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3244 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/xinchen9_llama3.1_8b_instruct_cot.json b/data/models/xinchen9_llama3.1_8b_instruct_cot.json deleted file mode 100644 index f9d3f469c220e24d3eea5bb488f2129530dbbf9f..0000000000000000000000000000000000000000 --- a/data/models/xinchen9_llama3.1_8b_instruct_cot.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama3.1_8B_Instruct_CoT", - "id": "xinchen9/Llama3.1_8B_Instruct_CoT", - "developer": "xinchen9", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/xinchen9_Llama3.1_8B_Instruct_CoT/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2974 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4398 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0604 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.302 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4371 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2879 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/xinchen9_llama3.1_cot.json b/data/models/xinchen9_llama3.1_cot.json deleted file mode 100644 index 7c3e2bcf3f0b5a99cdf05c2a5db0089f50488596..0000000000000000000000000000000000000000 --- a/data/models/xinchen9_llama3.1_cot.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama3.1_CoT", - "id": "xinchen9/Llama3.1_CoT", - "developer": "xinchen9", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/xinchen9_Llama3.1_CoT/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2246 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4341 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0385 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2886 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4305 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2739 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/xinchen9_llama3.1_cot_v1.json b/data/models/xinchen9_llama3.1_cot_v1.json deleted file mode 100644 index 11b9f05ac6cf74cc854015ef59aafcf028b24e96..0000000000000000000000000000000000000000 --- a/data/models/xinchen9_llama3.1_cot_v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama3.1_CoT_V1", - "id": "xinchen9/Llama3.1_CoT_V1", - "developer": "xinchen9", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/xinchen9_Llama3.1_CoT_V1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2453 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4376 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0332 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2794 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4572 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2805 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/xinchen9_mistral-7b-cot.json b/data/models/xinchen9_mistral-7b-cot.json deleted file mode 100644 index 4619a4227663e822be6abc750f96dcbbcfacd753..0000000000000000000000000000000000000000 --- a/data/models/xinchen9_mistral-7b-cot.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Mistral-7B-CoT", - "id": "xinchen9/Mistral-7B-CoT", - "developer": "xinchen9", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/xinchen9_Mistral-7B-CoT/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2783 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3873 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0249 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2492 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3994 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2284 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/xkev_llama-3.2v-11b-cot.json b/data/models/xkev_llama-3.2v-11b-cot.json deleted file mode 100644 index 8e9c68ddcd12335091b678c8237edc19c9864e4c..0000000000000000000000000000000000000000 --- a/data/models/xkev_llama-3.2v-11b-cot.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.2V-11B-cot", - "id": "Xkev/Llama-3.2V-11B-cot", - "developer": "Xkev", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MllamaForConditionalGeneration", - "params_billions": "10.67" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Xkev_Llama-3.2V-11B-cot/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4158 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4959 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1556 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2953 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4159 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3587 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/xkp24_llama-3-8b-instruct-sppo-iter2_bt_2b-table.json b/data/models/xkp24_llama-3-8b-instruct-sppo-iter2_bt_2b-table.json deleted file mode 100644 index 12c01cc29d7c9f1dfd59e68ed004d217a593f873..0000000000000000000000000000000000000000 --- a/data/models/xkp24_llama-3-8b-instruct-sppo-iter2_bt_2b-table.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-8B-Instruct-SPPO-Iter2_bt_2b-table", - "id": "xkp24/Llama-3-8B-Instruct-SPPO-Iter2_bt_2b-table", - "developer": "xkp24", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/xkp24_Llama-3-8B-Instruct-SPPO-Iter2_bt_2b-table/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6375 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4912 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0921 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2592 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.382 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3686 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/xkp24_llama-3-8b-instruct-sppo-iter2_bt_8b-table.json b/data/models/xkp24_llama-3-8b-instruct-sppo-iter2_bt_8b-table.json deleted file mode 100644 index 83d8a5919c0c2da8143fe6212ac8ac206e15a6fd..0000000000000000000000000000000000000000 --- a/data/models/xkp24_llama-3-8b-instruct-sppo-iter2_bt_8b-table.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-8B-Instruct-SPPO-Iter2_bt_8b-table", - "id": "xkp24/Llama-3-8B-Instruct-SPPO-Iter2_bt_8b-table", - "developer": "xkp24", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/xkp24_Llama-3-8B-Instruct-SPPO-Iter2_bt_8b-table/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7275 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5057 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0846 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2601 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3819 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3697 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/xkp24_llama-3-8b-instruct-sppo-iter2_gp_2b-table.json b/data/models/xkp24_llama-3-8b-instruct-sppo-iter2_gp_2b-table.json deleted file mode 100644 index d21c48f853ca38b9f7af0bb90484dafaf16ef019..0000000000000000000000000000000000000000 --- a/data/models/xkp24_llama-3-8b-instruct-sppo-iter2_gp_2b-table.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-8B-Instruct-SPPO-Iter2_gp_2b-table", - "id": "xkp24/Llama-3-8B-Instruct-SPPO-Iter2_gp_2b-table", - "developer": "xkp24", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/xkp24_Llama-3-8B-Instruct-SPPO-Iter2_gp_2b-table/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6569 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4952 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0891 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2592 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3594 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3702 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/xkp24_llama-3-8b-instruct-sppo-iter2_gp_8b-table.json b/data/models/xkp24_llama-3-8b-instruct-sppo-iter2_gp_8b-table.json deleted file mode 100644 index 5d6516235e7eec7def6fa5a5647bc687619326b9..0000000000000000000000000000000000000000 --- a/data/models/xkp24_llama-3-8b-instruct-sppo-iter2_gp_8b-table.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-8B-Instruct-SPPO-Iter2_gp_8b-table", - "id": "xkp24/Llama-3-8B-Instruct-SPPO-Iter2_gp_8b-table", - "developer": "xkp24", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/xkp24_Llama-3-8B-Instruct-SPPO-Iter2_gp_8b-table/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6621 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5004 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0861 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2592 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3805 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.36 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/xkp24_llama-3-8b-instruct-sppo-score-iter2_bt_2b-table-0.001.json b/data/models/xkp24_llama-3-8b-instruct-sppo-score-iter2_bt_2b-table-0.001.json deleted file mode 100644 index c4e7f4316e457d9e187e3199ce938279706832cc..0000000000000000000000000000000000000000 --- a/data/models/xkp24_llama-3-8b-instruct-sppo-score-iter2_bt_2b-table-0.001.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-8B-Instruct-SPPO-score-Iter2_bt_2b-table-0.001", - "id": "xkp24/Llama-3-8B-Instruct-SPPO-score-Iter2_bt_2b-table-0.001", - "developer": "xkp24", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/xkp24_Llama-3-8B-Instruct-SPPO-score-Iter2_bt_2b-table-0.001/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6042 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4936 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0997 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2592 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3793 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3708 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/xkp24_llama-3-8b-instruct-sppo-score-iter2_bt_8b-table-0.002.json b/data/models/xkp24_llama-3-8b-instruct-sppo-score-iter2_bt_8b-table-0.002.json deleted file mode 100644 index 02463c0a2ca456232637791589c08818667d5182..0000000000000000000000000000000000000000 --- a/data/models/xkp24_llama-3-8b-instruct-sppo-score-iter2_bt_8b-table-0.002.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-8B-Instruct-SPPO-score-Iter2_bt_8b-table-0.002", - "id": "xkp24/Llama-3-8B-Instruct-SPPO-score-Iter2_bt_8b-table-0.002", - "developer": "xkp24", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/xkp24_Llama-3-8B-Instruct-SPPO-score-Iter2_bt_8b-table-0.002/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7132 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4996 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0853 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2584 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3872 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3664 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/xkp24_llama-3-8b-instruct-sppo-score-iter2_gp_2b-table-0.001.json b/data/models/xkp24_llama-3-8b-instruct-sppo-score-iter2_gp_2b-table-0.001.json deleted file mode 100644 index 99b1f7cd66ee1c3086a7c4b465a1f457a2adaa4a..0000000000000000000000000000000000000000 --- a/data/models/xkp24_llama-3-8b-instruct-sppo-score-iter2_gp_2b-table-0.001.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-8B-Instruct-SPPO-score-Iter2_gp_2b-table-0.001", - "id": "xkp24/Llama-3-8B-Instruct-SPPO-score-Iter2_gp_2b-table-0.001", - "developer": "xkp24", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/xkp24_Llama-3-8B-Instruct-SPPO-score-Iter2_gp_2b-table-0.001/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5947 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4899 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1073 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2592 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3581 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3704 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/xkp24_llama-3-8b-instruct-sppo-score-iter2_gp_8b-table-0.002.json b/data/models/xkp24_llama-3-8b-instruct-sppo-score-iter2_gp_8b-table-0.002.json deleted file mode 100644 index 70eb0b102511e7fa1a7bbfd66a07f39927964a3c..0000000000000000000000000000000000000000 --- a/data/models/xkp24_llama-3-8b-instruct-sppo-score-iter2_gp_8b-table-0.002.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-8B-Instruct-SPPO-score-Iter2_gp_8b-table-0.002", - "id": "xkp24/Llama-3-8B-Instruct-SPPO-score-Iter2_gp_8b-table-0.002", - "developer": "xkp24", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/xkp24_Llama-3-8B-Instruct-SPPO-score-Iter2_gp_8b-table-0.002/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6453 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4951 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0937 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2601 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3939 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.353 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/xmaulana_finmatcha-3b-instruct.json b/data/models/xmaulana_finmatcha-3b-instruct.json deleted file mode 100644 index a4aba9773e2bb9d3cd2929023c3892f529dd0dd0..0000000000000000000000000000000000000000 --- a/data/models/xmaulana_finmatcha-3b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "FinMatcha-3B-Instruct", - "id": "xMaulana/FinMatcha-3B-Instruct", - "developer": "xMaulana", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/xMaulana_FinMatcha-3B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7548 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4536 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1435 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2693 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3633 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3182 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/xukp20_llama-3-8b-instruct-sppo-iter1-gp-2b-tau01-table.json b/data/models/xukp20_llama-3-8b-instruct-sppo-iter1-gp-2b-tau01-table.json deleted file mode 100644 index 1e68f0aed5b3eb525c103dcc3953758e8d7ddada..0000000000000000000000000000000000000000 --- a/data/models/xukp20_llama-3-8b-instruct-sppo-iter1-gp-2b-tau01-table.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "llama-3-8b-instruct-sppo-iter1-gp-2b-tau01-table", - "id": "xukp20/llama-3-8b-instruct-sppo-iter1-gp-2b-tau01-table", - "developer": "xukp20", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/xukp20_llama-3-8b-instruct-sppo-iter1-gp-2b-tau01-table/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.69 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4978 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.105 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2592 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3673 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3716 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/xukp20_llama-3-8b-instruct-sppo-iter3_bt_2b-table.json b/data/models/xukp20_llama-3-8b-instruct-sppo-iter3_bt_2b-table.json deleted file mode 100644 index 2f05ba22abe58a3b1e3926c11de57ddda920447a..0000000000000000000000000000000000000000 --- a/data/models/xukp20_llama-3-8b-instruct-sppo-iter3_bt_2b-table.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-8B-Instruct-SPPO-Iter3_bt_2b-table", - "id": "xukp20/Llama-3-8B-Instruct-SPPO-Iter3_bt_2b-table", - "developer": "xukp20", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/xukp20_Llama-3-8B-Instruct-SPPO-Iter3_bt_2b-table/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5756 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4901 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0997 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2592 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.366 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3659 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/xukp20_llama-3-8b-instruct-sppo-iter3_bt_8b-table.json b/data/models/xukp20_llama-3-8b-instruct-sppo-iter3_bt_8b-table.json deleted file mode 100644 index d19343dbac5b7f5d04eed6661a0f1935ff67939e..0000000000000000000000000000000000000000 --- a/data/models/xukp20_llama-3-8b-instruct-sppo-iter3_bt_8b-table.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-8B-Instruct-SPPO-Iter3_bt_8b-table", - "id": "xukp20/Llama-3-8B-Instruct-SPPO-Iter3_bt_8b-table", - "developer": "xukp20", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/xukp20_Llama-3-8B-Instruct-SPPO-Iter3_bt_8b-table/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7034 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5092 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0967 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2592 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3739 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3693 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/xukp20_llama-3-8b-instruct-sppo-iter3_gp_2b-table.json b/data/models/xukp20_llama-3-8b-instruct-sppo-iter3_gp_2b-table.json deleted file mode 100644 index 7a294f058edae4d233d052944bd159a15d309e7d..0000000000000000000000000000000000000000 --- a/data/models/xukp20_llama-3-8b-instruct-sppo-iter3_gp_2b-table.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-8B-Instruct-SPPO-Iter3_gp_2b-table", - "id": "xukp20/Llama-3-8B-Instruct-SPPO-Iter3_gp_2b-table", - "developer": "xukp20", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/xukp20_Llama-3-8B-Instruct-SPPO-Iter3_gp_2b-table/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6024 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.497 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1042 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2592 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3674 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3658 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/xukp20_llama-3-8b-instruct-sppo-iter3_gp_8b-table.json b/data/models/xukp20_llama-3-8b-instruct-sppo-iter3_gp_8b-table.json deleted file mode 100644 index 45f409e13735970ada70070da42c6780204ca45c..0000000000000000000000000000000000000000 --- a/data/models/xukp20_llama-3-8b-instruct-sppo-iter3_gp_8b-table.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-8B-Instruct-SPPO-Iter3_gp_8b-table", - "id": "xukp20/Llama-3-8B-Instruct-SPPO-Iter3_gp_8b-table", - "developer": "xukp20", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/xukp20_Llama-3-8B-Instruct-SPPO-Iter3_gp_8b-table/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.662 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0937 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2592 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3818 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3615 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/xukp20_llama-3-8b-instruct-sppo-score-iter3_bt_2b-table-0.001.json b/data/models/xukp20_llama-3-8b-instruct-sppo-score-iter3_bt_2b-table-0.001.json deleted file mode 100644 index aaec2b188c697508a0b24cc9d98efc0e41249bbc..0000000000000000000000000000000000000000 --- a/data/models/xukp20_llama-3-8b-instruct-sppo-score-iter3_bt_2b-table-0.001.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-8B-Instruct-SPPO-score-Iter3_bt_2b-table-0.001", - "id": "xukp20/Llama-3-8B-Instruct-SPPO-score-Iter3_bt_2b-table-0.001", - "developer": "xukp20", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/xukp20_Llama-3-8B-Instruct-SPPO-score-Iter3_bt_2b-table-0.001/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5336 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4915 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0982 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2592 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.378 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3625 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/xukp20_llama-3-8b-instruct-sppo-score-iter3_bt_8b-table-0.002.json b/data/models/xukp20_llama-3-8b-instruct-sppo-score-iter3_bt_8b-table-0.002.json deleted file mode 100644 index eca08fd9de378f69b1c44768503a8bfa58167e30..0000000000000000000000000000000000000000 --- a/data/models/xukp20_llama-3-8b-instruct-sppo-score-iter3_bt_8b-table-0.002.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-8B-Instruct-SPPO-score-Iter3_bt_8b-table-0.002", - "id": "xukp20/Llama-3-8B-Instruct-SPPO-score-Iter3_bt_8b-table-0.002", - "developer": "xukp20", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/xukp20_Llama-3-8B-Instruct-SPPO-score-Iter3_bt_8b-table-0.002/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6852 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5075 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0718 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2584 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3832 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3621 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/xukp20_llama-3-8b-instruct-sppo-score-iter3_gp_2b-table-0.001.json b/data/models/xukp20_llama-3-8b-instruct-sppo-score-iter3_gp_2b-table-0.001.json deleted file mode 100644 index f9f33b44f9a7c07e217b53a48a2eb67480b4f06e..0000000000000000000000000000000000000000 --- a/data/models/xukp20_llama-3-8b-instruct-sppo-score-iter3_gp_2b-table-0.001.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-8B-Instruct-SPPO-score-Iter3_gp_2b-table-0.001", - "id": "xukp20/Llama-3-8B-Instruct-SPPO-score-Iter3_gp_2b-table-0.001", - "developer": "xukp20", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/xukp20_Llama-3-8B-Instruct-SPPO-score-Iter3_gp_2b-table-0.001/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5482 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4887 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0891 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2609 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3633 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3671 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/xwen-team_xwen-7b-chat.json b/data/models/xwen-team_xwen-7b-chat.json deleted file mode 100644 index c2de2d7f4069333119c46069c677c323864f444d..0000000000000000000000000000000000000000 --- a/data/models/xwen-team_xwen-7b-chat.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Xwen-7B-Chat", - "id": "xwen-team/Xwen-7B-Chat", - "developer": "xwen-team", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/xwen-team_Xwen-7B-Chat/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6864 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5068 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4509 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2609 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3914 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.429 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/xxx777xxxasd_l3.1-claudemaid-4x8b.json b/data/models/xxx777xxxasd_l3.1-claudemaid-4x8b.json deleted file mode 100644 index c04adef15ebe77e0380a3fa263b43c93e77cdfca..0000000000000000000000000000000000000000 --- a/data/models/xxx777xxxasd_l3.1-claudemaid-4x8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "L3.1-ClaudeMaid-4x8B", - "id": "xxx777xxxASD/L3.1-ClaudeMaid-4x8B", - "developer": "xxx777xxxASD", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MixtralForCausalLM", - "params_billions": "24.942" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/xxx777xxxASD_L3.1-ClaudeMaid-4x8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6696 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5071 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1412 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2911 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4289 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.358 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/yam-peleg_hebrew-gemma-11b-instruct.json b/data/models/yam-peleg_hebrew-gemma-11b-instruct.json deleted file mode 100644 index da1762d447db89a768d822795fc16e6679832ad5..0000000000000000000000000000000000000000 --- a/data/models/yam-peleg_hebrew-gemma-11b-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Hebrew-Gemma-11B-Instruct", - "id": "yam-peleg/Hebrew-Gemma-11B-Instruct", - "developer": "yam-peleg", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "GemmaForCausalLM", - "params_billions": "10.475" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/yam-peleg_Hebrew-Gemma-11B-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3021 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4036 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0657 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.276 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4089 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2554 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/yam-peleg_hebrew-mistral-7b-200k.json b/data/models/yam-peleg_hebrew-mistral-7b-200k.json deleted file mode 100644 index baae674a05af479a549c721b22d1c0240e173193..0000000000000000000000000000000000000000 --- a/data/models/yam-peleg_hebrew-mistral-7b-200k.json +++ /dev/null @@ -1,275 +0,0 @@ -{ - "model_info": { - "name": "Hebrew-Mistral-7B-200K", - "id": "yam-peleg/Hebrew-Mistral-7B-200K", - "developer": "yam-peleg", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "7.504" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/yam-peleg_Hebrew-Mistral-7B-200K/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1856 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4149 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0234 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.276 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3765 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2573 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "hfopenllm_v2/yam-peleg_Hebrew-Mistral-7B-200K/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.177 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3411 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.031 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2534 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.374 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2529 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/yam-peleg_hebrew-mistral-7b.json b/data/models/yam-peleg_hebrew-mistral-7b.json deleted file mode 100644 index 2f001b8a7fec3ee285c42cf0a1484ccb9cef475c..0000000000000000000000000000000000000000 --- a/data/models/yam-peleg_hebrew-mistral-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Hebrew-Mistral-7B", - "id": "yam-peleg/Hebrew-Mistral-7B", - "developer": "yam-peleg", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.504" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/yam-peleg_Hebrew-Mistral-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2328 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4334 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0498 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2794 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3977 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.278 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/yandex_yalm-100b.json b/data/models/yandex_yalm-100b.json deleted file mode 100644 index 04757a3ceb7fdf5cd8c0391c6ef321fa4e69f6e9..0000000000000000000000000000000000000000 --- a/data/models/yandex_yalm-100b.json +++ /dev/null @@ -1,674 +0,0 @@ -{ - "model_info": { - "name": "YaLM 100B", - "id": "yandex/YaLM-100B", - "developer": "yandex", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_classic/yandex_YaLM-100B/1774096308.339228", - "retrieved_timestamp": "1774096308.339228", - "source_metadata": { - "source_name": "helm_classic", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_classic", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_classic", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperform on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.075, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Calibration": "{\"description\": \"\", \"tab\": \"Calibration\", \"score\": \"0.40175763182238666\"}", - "Mean win rate - Robustness": "{\"description\": \"\", \"tab\": \"Robustness\", \"score\": \"0.20536130536130537\"}", - "Mean win rate - Fairness": "{\"description\": \"\", \"tab\": \"Fairness\", \"score\": \"0.16727272727272727\"}", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.2658333333333333\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}", - "Mean win rate - Bias": "{\"description\": \"\", \"tab\": \"Bias\", \"score\": \"0.37929404953000706\"}", - "Mean win rate - Toxicity": "{\"description\": \"\", \"tab\": \"Toxicity\", \"score\": \"0.24189051689051688\"}", - "Mean win rate - Summarization metrics": "{\"description\": \"\", \"tab\": \"Summarization metrics\", \"score\": \"0.04536340852130326\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.243, - "details": { - "description": "min=0.2, mean=0.243, max=0.28, sum=3.651 (15)", - "tab": "Accuracy", - "MMLU - ECE (10-bin)": "{\"description\": \"min=0.619, mean=0.708, max=0.769, sum=10.615 (15)\", \"tab\": \"Calibration\", \"score\": \"0.7076962372990694\"}", - "MMLU - EM (Robustness)": "{\"description\": \"min=0.2, mean=0.243, max=0.28, sum=3.651 (15)\", \"tab\": \"Robustness\", \"score\": \"0.2433684210526316\"}", - "MMLU - EM (Fairness)": "{\"description\": \"min=0.2, mean=0.243, max=0.28, sum=3.651 (15)\", \"tab\": \"Fairness\", \"score\": \"0.2433684210526316\"}", - "MMLU - Denoised inference time (s)": "{\"description\": \"min=0.09, mean=0.143, max=0.217, sum=2.144 (15)\", \"tab\": \"Efficiency\", \"score\": \"0.14296402070471761\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=1542 (15)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=75 (15)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (15)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=354.96, mean=453.383, max=580.833, sum=6800.74 (15)\", \"tab\": \"General information\", \"score\": \"453.38266666666664\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=15 (15)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "MMLU - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=45 (15)\", \"tab\": \"General information\", \"score\": \"3.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "BoolQ", - "source_data": { - "dataset_name": "BoolQ", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on BoolQ", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.634, - "details": { - "description": "min=0.631, mean=0.634, max=0.64, sum=1.902 (3)", - "tab": "Accuracy", - "BoolQ - ECE (10-bin)": "{\"description\": \"min=0.114, mean=0.147, max=0.167, sum=0.442 (3)\", \"tab\": \"Calibration\", \"score\": \"0.14717484078898194\"}", - "BoolQ - EM (Robustness)": "{\"description\": \"min=0.437, mean=0.566, max=0.631, sum=1.698 (3)\", \"tab\": \"Robustness\", \"score\": \"0.566\"}", - "BoolQ - EM (Fairness)": "{\"description\": \"min=0.486, mean=0.583, max=0.631, sum=1.748 (3)\", \"tab\": \"Fairness\", \"score\": \"0.5826666666666667\"}", - "BoolQ - Denoised inference time (s)": "{\"description\": \"min=0.546, mean=0.828, max=1.136, sum=2.485 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.8282727491158176\"}", - "BoolQ - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "BoolQ - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "BoolQ - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "BoolQ - # prompt tokens": "{\"description\": \"min=649.339, mean=899.006, max=1233.339, sum=2697.017 (3)\", \"tab\": \"General information\", \"score\": \"899.0056666666666\"}", - "BoolQ - # output tokens": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "BoolQ - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "BoolQ - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.252, - "details": { - "description": "min=0.213, mean=0.252, max=0.297, sum=0.756 (3)", - "tab": "Accuracy", - "NarrativeQA - ECE (10-bin)": "{\"description\": \"min=0.029, mean=0.06, max=0.101, sum=0.179 (3)\", \"tab\": \"Calibration\", \"score\": \"0.05960283323299867\"}", - "NarrativeQA - F1 (Robustness)": "{\"description\": \"min=0.078, mean=0.088, max=0.096, sum=0.264 (3)\", \"tab\": \"Robustness\", \"score\": \"0.08788676556219112\"}", - "NarrativeQA - F1 (Fairness)": "{\"description\": \"min=0.131, mean=0.146, max=0.169, sum=0.437 (3)\", \"tab\": \"Fairness\", \"score\": \"0.14573784149261218\"}", - "NarrativeQA - Denoised inference time (s)": "{\"description\": \"min=2.158, mean=2.314, max=2.397, sum=6.943 (3)\", \"tab\": \"Efficiency\", \"score\": \"2.314193915889056\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=1065 (3)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=1.028, mean=1.604, max=2.008, sum=4.811 (3)\", \"tab\": \"General information\", \"score\": \"1.603755868544601\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=1603.569, mean=1644.878, max=1690.352, sum=4934.634 (3)\", \"tab\": \"General information\", \"score\": \"1644.8779342723003\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=94.115, mean=96.018, max=98.566, sum=288.054 (3)\", \"tab\": \"General information\", \"score\": \"96.01784037558686\"}", - "NarrativeQA - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NarrativeQA - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=2 (3)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "NarrativeQA - Stereotypes (gender)": "{\"description\": \"min=0.434, mean=0.449, max=0.478, sum=1.347 (3)\", \"tab\": \"Bias\", \"score\": \"0.449065994913171\"}", - "NarrativeQA - Representation (race)": "{\"description\": \"min=0.429, mean=0.568, max=0.667, sum=1.703 (3)\", \"tab\": \"Bias\", \"score\": \"0.5676937441643325\"}", - "NarrativeQA - Representation (gender)": "{\"description\": \"min=0.127, mean=0.177, max=0.216, sum=0.53 (3)\", \"tab\": \"Bias\", \"score\": \"0.17681914997964296\"}", - "NarrativeQA - Toxic fraction": "{\"description\": \"min=0.014, mean=0.017, max=0.02, sum=0.051 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.016901408450704227\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (open-book)", - "source_data": { - "dataset_name": "NaturalQuestions (open-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (open-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.227, - "details": { - "description": "min=0.197, mean=0.227, max=0.258, sum=0.68 (3)", - "tab": "Accuracy", - "NaturalQuestions (closed-book) - ECE (10-bin)": "{\"description\": \"min=0.019, mean=0.02, max=0.02, sum=0.059 (3)\", \"tab\": \"Calibration\", \"score\": \"0.019790335675494927\"}", - "NaturalQuestions (open-book) - ECE (10-bin)": "{\"description\": \"min=0.069, mean=0.086, max=0.12, sum=0.259 (3)\", \"tab\": \"Calibration\", \"score\": \"0.08637064333353452\"}", - "NaturalQuestions (closed-book) - F1 (Robustness)": "{\"description\": \"min=0.045, mean=0.047, max=0.05, sum=0.14 (3)\", \"tab\": \"Robustness\", \"score\": \"0.04678550801735826\"}", - "NaturalQuestions (open-book) - F1 (Robustness)": "{\"description\": \"min=0.111, mean=0.125, max=0.146, sum=0.375 (3)\", \"tab\": \"Robustness\", \"score\": \"0.12496123369617401\"}", - "NaturalQuestions (closed-book) - F1 (Fairness)": "{\"description\": \"min=0.051, mean=0.052, max=0.053, sum=0.155 (3)\", \"tab\": \"Fairness\", \"score\": \"0.0516362934670568\"}", - "NaturalQuestions (open-book) - F1 (Fairness)": "{\"description\": \"min=0.15, mean=0.177, max=0.207, sum=0.53 (3)\", \"tab\": \"Fairness\", \"score\": \"0.1768275232054711\"}", - "NaturalQuestions (closed-book) - Denoised inference time (s)": "{\"description\": \"min=2.669, mean=2.722, max=2.827, sum=8.167 (3)\", \"tab\": \"Efficiency\", \"score\": \"2.7221932611479644\"}", - "NaturalQuestions (open-book) - Denoised inference time (s)": "{\"description\": \"min=4.373, mean=4.463, max=4.531, sum=13.389 (3)\", \"tab\": \"Efficiency\", \"score\": \"4.463013303365339\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=108.201, mean=111.534, max=117.201, sum=334.603 (3)\", \"tab\": \"General information\", \"score\": \"111.53433333333332\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=298.545, mean=299.515, max=300, sum=898.545 (3)\", \"tab\": \"General information\", \"score\": \"299.51500000000004\"}", - "NaturalQuestions (closed-book) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.669, mean=4.702, max=4.738, sum=14.107 (3)\", \"tab\": \"General information\", \"score\": \"4.702333333333333\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.038, mean=0.038, max=0.038, sum=0.114 (3)\", \"tab\": \"General information\", \"score\": \"0.038\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1218.159, mean=1409.24, max=1510.891, sum=4227.721 (3)\", \"tab\": \"General information\", \"score\": \"1409.2403333333332\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=289.149, mean=291.572, max=293.886, sum=874.715 (3)\", \"tab\": \"General information\", \"score\": \"291.57166666666666\"}", - "NaturalQuestions (open-book) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NaturalQuestions (closed-book) - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=2 (3)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "NaturalQuestions (closed-book) - Stereotypes (gender)": "{\"description\": \"min=0.433, mean=0.478, max=0.5, sum=1.433 (3)\", \"tab\": \"Bias\", \"score\": \"0.4776758409785933\"}", - "NaturalQuestions (closed-book) - Representation (race)": "{\"description\": \"min=0.324, mean=0.327, max=0.33, sum=0.982 (3)\", \"tab\": \"Bias\", \"score\": \"0.3274145329078469\"}", - "NaturalQuestions (closed-book) - Representation (gender)": "{\"description\": \"min=0.014, mean=0.168, max=0.277, sum=0.504 (3)\", \"tab\": \"Bias\", \"score\": \"0.16816448651008897\"}", - "NaturalQuestions (open-book) - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=2 (3)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "NaturalQuestions (open-book) - Stereotypes (gender)": "{\"description\": \"min=0.5, mean=0.5, max=0.5, sum=0.5 (1)\", \"tab\": \"Bias\", \"score\": \"0.5\"}", - "NaturalQuestions (open-book) - Representation (race)": "{\"description\": \"min=0.204, mean=0.385, max=0.523, sum=1.154 (3)\", \"tab\": \"Bias\", \"score\": \"0.38473904949347787\"}", - "NaturalQuestions (open-book) - Representation (gender)": "{\"description\": \"min=0.102, mean=0.175, max=0.25, sum=0.526 (3)\", \"tab\": \"Bias\", \"score\": \"0.17544176986611967\"}", - "NaturalQuestions (closed-book) - Toxic fraction": "{\"description\": \"min=0.007, mean=0.008, max=0.009, sum=0.024 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.008\"}", - "NaturalQuestions (open-book) - Toxic fraction": "{\"description\": \"min=0.003, mean=0.003, max=0.003, sum=0.009 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.0030000000000000005\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "QuAC", - "source_data": { - "dataset_name": "QuAC", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on QuAC", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.162, - "details": { - "description": "min=0.156, mean=0.162, max=0.172, sum=0.485 (3)", - "tab": "Accuracy", - "QuAC - ECE (10-bin)": "{\"description\": \"min=0.012, mean=0.029, max=0.039, sum=0.087 (3)\", \"tab\": \"Calibration\", \"score\": \"0.028959032200530792\"}", - "QuAC - F1 (Robustness)": "{\"description\": \"min=0.077, mean=0.08, max=0.082, sum=0.239 (3)\", \"tab\": \"Robustness\", \"score\": \"0.0795025876916194\"}", - "QuAC - F1 (Fairness)": "{\"description\": \"min=0.092, mean=0.1, max=0.108, sum=0.301 (3)\", \"tab\": \"Fairness\", \"score\": \"0.10047785618783804\"}", - "QuAC - Denoised inference time (s)": "{\"description\": \"min=2.259, mean=2.278, max=2.297, sum=6.834 (3)\", \"tab\": \"Efficiency\", \"score\": \"2.278147567048529\"}", - "QuAC - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "QuAC - # train": "{\"description\": \"min=0.841, mean=0.951, max=1.111, sum=2.853 (3)\", \"tab\": \"General information\", \"score\": \"0.951\"}", - "QuAC - truncated": "{\"description\": \"min=0.016, mean=0.016, max=0.016, sum=0.048 (3)\", \"tab\": \"General information\", \"score\": \"0.016\"}", - "QuAC - # prompt tokens": "{\"description\": \"min=1630.348, mean=1646.729, max=1667.958, sum=4940.188 (3)\", \"tab\": \"General information\", \"score\": \"1646.7293333333334\"}", - "QuAC - # output tokens": "{\"description\": \"min=99.146, mean=99.146, max=99.146, sum=297.438 (3)\", \"tab\": \"General information\", \"score\": \"99.146\"}", - "QuAC - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "QuAC - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=2 (3)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "QuAC - Stereotypes (gender)": "{\"description\": \"min=0.44, mean=0.454, max=0.465, sum=1.363 (3)\", \"tab\": \"Bias\", \"score\": \"0.4543925551127126\"}", - "QuAC - Representation (race)": "{\"description\": \"min=0.312, mean=0.465, max=0.582, sum=1.396 (3)\", \"tab\": \"Bias\", \"score\": \"0.4653480174056855\"}", - "QuAC - Representation (gender)": "{\"description\": \"min=0.335, mean=0.343, max=0.358, sum=1.029 (3)\", \"tab\": \"Bias\", \"score\": \"0.3431307584494557\"}", - "QuAC - Toxic fraction": "{\"description\": \"min=0, mean=0.001, max=0.003, sum=0.003 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.001\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "HellaSwag", - "source_data": { - "dataset_name": "HellaSwag", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on HellaSwag", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "HellaSwag - ECE (10-bin)": "{\"description\": \"No matching runs\", \"tab\": \"Calibration\", \"score\": \"\"}", - "HellaSwag - EM (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "HellaSwag - EM (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "HellaSwag - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "HellaSwag - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "OpenbookQA - ECE (10-bin)": "{\"description\": \"No matching runs\", \"tab\": \"Calibration\", \"score\": \"\"}", - "OpenbookQA - EM (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "OpenbookQA - EM (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "OpenbookQA - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "OpenbookQA - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "TruthfulQA", - "source_data": { - "dataset_name": "TruthfulQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on TruthfulQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.202, - "details": { - "description": "min=0.197, mean=0.202, max=0.203, sum=0.807 (4)", - "tab": "Accuracy", - "TruthfulQA - ECE (10-bin)": "{\"description\": \"min=0.621, mean=0.679, max=0.751, sum=2.716 (4)\", \"tab\": \"Calibration\", \"score\": \"0.6789622806094777\"}", - "TruthfulQA - EM (Robustness)": "{\"description\": \"min=0.197, mean=0.202, max=0.203, sum=0.807 (4)\", \"tab\": \"Robustness\", \"score\": \"0.2018348623853211\"}", - "TruthfulQA - EM (Fairness)": "{\"description\": \"min=0.197, mean=0.202, max=0.203, sum=0.807 (4)\", \"tab\": \"Fairness\", \"score\": \"0.2018348623853211\"}", - "TruthfulQA - Denoised inference time (s)": "{\"description\": \"min=0.058, mean=0.092, max=0.136, sum=0.37 (4)\", \"tab\": \"Efficiency\", \"score\": \"0.09243018414244196\"}", - "TruthfulQA - # eval": "{\"description\": \"min=654, mean=654, max=654, sum=2616 (4)\", \"tab\": \"General information\", \"score\": \"654.0\"}", - "TruthfulQA - # train": "{\"description\": \"min=0, mean=3.75, max=5, sum=15 (4)\", \"tab\": \"General information\", \"score\": \"3.75\"}", - "TruthfulQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (4)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "TruthfulQA - # prompt tokens": "{\"description\": \"min=85.664, mean=405.414, max=531.664, sum=1621.654 (4)\", \"tab\": \"General information\", \"score\": \"405.41360856269114\"}", - "TruthfulQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=4 (4)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "TruthfulQA - # trials": "{\"description\": \"min=1, mean=2.5, max=3, sum=10 (4)\", \"tab\": \"General information\", \"score\": \"2.5\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MS MARCO (TREC)", - "source_data": { - "dataset_name": "MS MARCO (TREC)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "NDCG@10 on MS MARCO (TREC)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "MS MARCO (regular) - RR@10 (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "MS MARCO (TREC) - NDCG@10 (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "MS MARCO (regular) - RR@10 (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "MS MARCO (TREC) - NDCG@10 (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "MS MARCO (regular) - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MS MARCO (TREC) - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MS MARCO (regular) - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "MS MARCO (TREC) - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CNN/DailyMail", - "source_data": { - "dataset_name": "CNN/DailyMail", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on CNN/DailyMail", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.017, - "details": { - "description": "min=0.009, mean=0.017, max=0.022, sum=0.103 (6)", - "tab": "Accuracy", - "CNN/DailyMail - Denoised inference time (s)": "{\"description\": \"min=2.334, mean=2.346, max=2.352, sum=14.074 (6)\", \"tab\": \"Efficiency\", \"score\": \"2.3457143735281405\"}", - "CNN/DailyMail - # eval": "{\"description\": \"min=466, mean=466, max=466, sum=2796 (6)\", \"tab\": \"General information\", \"score\": \"466.0\"}", - "CNN/DailyMail - # train": "{\"description\": \"min=5, mean=5, max=5, sum=30 (6)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CNN/DailyMail - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (6)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CNN/DailyMail - # prompt tokens": "{\"description\": \"min=1536.099, mean=1544.765, max=1562.099, sum=9268.592 (6)\", \"tab\": \"General information\", \"score\": \"1544.7653791130188\"}", - "CNN/DailyMail - # output tokens": "{\"description\": \"min=90.71, mean=102.407, max=108.32, sum=614.442 (6)\", \"tab\": \"General information\", \"score\": \"102.40701001430614\"}", - "CNN/DailyMail - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=18 (6)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "CNN/DailyMail - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=4 (6)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "CNN/DailyMail - Stereotypes (gender)": "{\"description\": \"min=0.406, mean=0.42, max=0.438, sum=2.518 (6)\", \"tab\": \"Bias\", \"score\": \"0.4196869049681346\"}", - "CNN/DailyMail - Representation (race)": "{\"description\": \"min=0.429, mean=0.588, max=0.667, sum=3.525 (6)\", \"tab\": \"Bias\", \"score\": \"0.5875706214689266\"}", - "CNN/DailyMail - Representation (gender)": "{\"description\": \"min=0.171, mean=0.206, max=0.237, sum=1.238 (6)\", \"tab\": \"Bias\", \"score\": \"0.20635612913269732\"}", - "CNN/DailyMail - Toxic fraction": "{\"description\": \"min=0, mean=0.001, max=0.002, sum=0.004 (6)\", \"tab\": \"Toxicity\", \"score\": \"0.000715307582260372\"}", - "CNN/DailyMail - SummaC": "{\"description\": \"min=-0.35, mean=-0.322, max=-0.296, sum=-0.965 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"-0.3217409663792838\"}", - "CNN/DailyMail - QAFactEval": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - BERTScore (F1)": "{\"description\": \"min=-0.154, mean=-0.145, max=-0.127, sum=-0.435 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"-0.14496527560996572\"}", - "CNN/DailyMail - Coverage": "{\"description\": \"min=0.406, mean=0.541, max=0.615, sum=3.249 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"0.5414806522156069\"}", - "CNN/DailyMail - Density": "{\"description\": \"min=0.681, mean=1.09, max=1.303, sum=6.541 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"1.0902141864760964\"}", - "CNN/DailyMail - Compression": "{\"description\": \"min=6.289, mean=6.936, max=8.148, sum=41.615 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"6.935882429972025\"}", - "CNN/DailyMail - HumanEval-faithfulness": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-relevance": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - HumanEval-coherence": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "XSUM", - "source_data": { - "dataset_name": "XSUM", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on XSUM", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.021, - "details": { - "description": "min=0.019, mean=0.021, max=0.022, sum=0.124 (6)", - "tab": "Accuracy", - "XSUM - Denoised inference time (s)": "{\"description\": \"min=1.653, mean=1.671, max=1.681, sum=10.028 (6)\", \"tab\": \"Efficiency\", \"score\": \"1.6713877910966286\"}", - "XSUM - # eval": "{\"description\": \"min=518, mean=518, max=518, sum=3108 (6)\", \"tab\": \"General information\", \"score\": \"518.0\"}", - "XSUM - # train": "{\"description\": \"min=5, mean=5, max=5, sum=30 (6)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "XSUM - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (6)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "XSUM - # prompt tokens": "{\"description\": \"min=1452.164, mean=1507.497, max=1536.164, sum=9044.985 (6)\", \"tab\": \"General information\", \"score\": \"1507.497425997426\"}", - "XSUM - # output tokens": "{\"description\": \"min=46.541, mean=49.401, max=51.544, sum=296.405 (6)\", \"tab\": \"General information\", \"score\": \"49.4009009009009\"}", - "XSUM - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=18 (6)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "XSUM - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=4 (6)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "XSUM - Stereotypes (gender)": "{\"description\": \"min=0.434, mean=0.442, max=0.456, sum=2.652 (6)\", \"tab\": \"Bias\", \"score\": \"0.4419820754826329\"}", - "XSUM - Representation (race)": "{\"description\": \"min=0.333, mean=0.501, max=0.595, sum=3.009 (6)\", \"tab\": \"Bias\", \"score\": \"0.5014430014430014\"}", - "XSUM - Representation (gender)": "{\"description\": \"min=0.209, mean=0.248, max=0.286, sum=1.485 (6)\", \"tab\": \"Bias\", \"score\": \"0.24754799603959324\"}", - "XSUM - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (6)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}", - "XSUM - SummaC": "{\"description\": \"min=-0.352, mean=-0.347, max=-0.344, sum=-1.04 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"-0.3466731809697447\"}", - "XSUM - QAFactEval": "{\"description\": \"min=0.856, mean=1.176, max=1.555, sum=7.058 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"1.1763058409064706\"}", - "XSUM - BERTScore (F1)": "{\"description\": \"min=0.007, mean=0.031, max=0.057, sum=0.093 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.031129963643441894\"}", - "XSUM - Coverage": "{\"description\": \"min=0.557, mean=0.567, max=0.574, sum=3.405 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"0.5674251187038739\"}", - "XSUM - Density": "{\"description\": \"min=1.005, mean=1.041, max=1.081, sum=6.248 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"1.0413571284332044\"}", - "XSUM - Compression": "{\"description\": \"min=9.397, mean=9.951, max=10.96, sum=59.706 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"9.951019350255967\"}", - "XSUM - HumanEval-faithfulness": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-relevance": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - HumanEval-coherence": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "IMDB", - "source_data": { - "dataset_name": "IMDB", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on IMDB", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.836, - "details": { - "description": "min=0.776, mean=0.836, max=0.876, sum=2.509 (3)", - "tab": "Accuracy", - "IMDB - ECE (10-bin)": "{\"description\": \"min=0.369, mean=0.418, max=0.496, sum=1.255 (3)\", \"tab\": \"Calibration\", \"score\": \"0.41834259640752514\"}", - "IMDB - EM (Robustness)": "{\"description\": \"min=0.578, mean=0.719, max=0.79, sum=2.158 (3)\", \"tab\": \"Robustness\", \"score\": \"0.7193333333333333\"}", - "IMDB - EM (Fairness)": "{\"description\": \"min=0.709, mean=0.8, max=0.853, sum=2.4 (3)\", \"tab\": \"Fairness\", \"score\": \"0.7999999999999999\"}", - "IMDB - Denoised inference time (s)": "{\"description\": \"min=1.076, mean=1.137, max=1.23, sum=3.41 (3)\", \"tab\": \"Efficiency\", \"score\": \"1.1365543731623833\"}", - "IMDB - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "IMDB - # train": "{\"description\": \"min=4.845, mean=4.929, max=4.982, sum=14.788 (3)\", \"tab\": \"General information\", \"score\": \"4.929333333333333\"}", - "IMDB - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IMDB - # prompt tokens": "{\"description\": \"min=1161.789, mean=1402.276, max=1747.837, sum=4206.828 (3)\", \"tab\": \"General information\", \"score\": \"1402.2759999999998\"}", - "IMDB - # output tokens": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "IMDB - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "IMDB - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CivilComments", - "source_data": { - "dataset_name": "CivilComments", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on CivilComments", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.49, - "details": { - "description": "min=0, mean=0.49, max=1, sum=26.448 (54)", - "tab": "Accuracy", - "CivilComments - ECE (10-bin)": "{\"description\": \"min=0.108, mean=0.437, max=0.784, sum=23.581 (54)\", \"tab\": \"Calibration\", \"score\": \"0.43669079652569004\"}", - "CivilComments - EM (Robustness)": "{\"description\": \"min=0, mean=0.463, max=1, sum=25.008 (54)\", \"tab\": \"Robustness\", \"score\": \"0.4631081891632545\"}", - "CivilComments - EM (Fairness)": "{\"description\": \"min=0, mean=0.456, max=0.998, sum=24.603 (54)\", \"tab\": \"Fairness\", \"score\": \"0.4556089334763174\"}", - "CivilComments - Denoised inference time (s)": "{\"description\": \"min=0.291, mean=0.41, max=0.737, sum=22.139 (54)\", \"tab\": \"Efficiency\", \"score\": \"0.4099806397254133\"}", - "CivilComments - # eval": "{\"description\": \"min=74, mean=371.556, max=683, sum=20064 (54)\", \"tab\": \"General information\", \"score\": \"371.55555555555554\"}", - "CivilComments - # train": "{\"description\": \"min=5, mean=5, max=5, sum=270 (54)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CivilComments - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (54)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CivilComments - # prompt tokens": "{\"description\": \"min=365, mean=729.671, max=1285.924, sum=39402.252 (54)\", \"tab\": \"General information\", \"score\": \"729.6713289334527\"}", - "CivilComments - # output tokens": "{\"description\": \"min=5, mean=5, max=5, sum=270 (54)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CivilComments - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=162 (54)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "CivilComments - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (54)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "RAFT", - "source_data": { - "dataset_name": "RAFT", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on RAFT", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.395, - "details": { - "description": "min=0, mean=0.395, max=0.975, sum=13.05 (33)", - "tab": "Accuracy", - "RAFT - ECE (10-bin)": "{\"description\": \"min=0.011, mean=0.278, max=0.881, sum=9.176 (33)\", \"tab\": \"Calibration\", \"score\": \"0.2780574023642052\"}", - "RAFT - EM (Robustness)": "{\"description\": \"min=0, mean=0.211, max=0.65, sum=6.975 (33)\", \"tab\": \"Robustness\", \"score\": \"0.21136363636363636\"}", - "RAFT - EM (Fairness)": "{\"description\": \"min=0, mean=0.342, max=0.975, sum=11.3 (33)\", \"tab\": \"Fairness\", \"score\": \"0.3424242424242424\"}", - "RAFT - Denoised inference time (s)": "{\"description\": \"min=0.132, mean=0.89, max=1.838, sum=29.385 (33)\", \"tab\": \"Efficiency\", \"score\": \"0.8904544346562409\"}", - "RAFT - # eval": "{\"description\": \"min=40, mean=40, max=40, sum=1320 (33)\", \"tab\": \"General information\", \"score\": \"40.0\"}", - "RAFT - # train": "{\"description\": \"min=0, mean=4.562, max=5, sum=150.55 (33)\", \"tab\": \"General information\", \"score\": \"4.5621212121212125\"}", - "RAFT - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (33)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "RAFT - # prompt tokens": "{\"description\": \"min=255.875, mean=784.961, max=1758.075, sum=25903.725 (33)\", \"tab\": \"General information\", \"score\": \"784.9613636363637\"}", - "RAFT - # output tokens": "{\"description\": \"min=5, mean=13.615, max=30, sum=449.3 (33)\", \"tab\": \"General information\", \"score\": \"13.615151515151515\"}", - "RAFT - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=99 (33)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "RAFT - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (gender)": "{\"description\": \"min=0.5, mean=0.5, max=0.5, sum=0.5 (1)\", \"tab\": \"Bias\", \"score\": \"0.5\"}", - "RAFT - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (33)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/yanng1242_marcoro14-7b-slerp.json b/data/models/yanng1242_marcoro14-7b-slerp.json deleted file mode 100644 index 81ac7d4dd9aa611331ed674e0836085a0ce8afe7..0000000000000000000000000000000000000000 --- a/data/models/yanng1242_marcoro14-7b-slerp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Marcoro14-7B-slerp", - "id": "yanng1242/Marcoro14-7B-slerp", - "developer": "yanng1242", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/yanng1242_Marcoro14-7B-slerp/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.406 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5252 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0748 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3146 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4686 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3168 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/yash21_tinyyi-7b-test.json b/data/models/yash21_tinyyi-7b-test.json deleted file mode 100644 index 3278f3859b48234289eef0b1e10a5f9fb88dffd8..0000000000000000000000000000000000000000 --- a/data/models/yash21_tinyyi-7b-test.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "TinyYi-7B-Test", - "id": "Yash21/TinyYi-7B-Test", - "developer": "Yash21", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "6.061" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Yash21_TinyYi-7B-Test/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1856 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.291 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2643 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3364 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1091 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/yasserrmd_coder-grpo-3b.json b/data/models/yasserrmd_coder-grpo-3b.json deleted file mode 100644 index 6f72c2c92a0c594818404acb68ca31ce76d5c24c..0000000000000000000000000000000000000000 --- a/data/models/yasserrmd_coder-grpo-3b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Coder-GRPO-3B", - "id": "yasserrmd/Coder-GRPO-3B", - "developer": "yasserrmd", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "3.086" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/yasserrmd_Coder-GRPO-3B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6208 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4469 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3202 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2777 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4115 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3197 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/yasserrmd_text2sql-1.5b.json b/data/models/yasserrmd_text2sql-1.5b.json deleted file mode 100644 index bfd96932c97c9b2a5e4b4ff7d64bc48ce5049e93..0000000000000000000000000000000000000000 --- a/data/models/yasserrmd_text2sql-1.5b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Text2SQL-1.5B", - "id": "yasserrmd/Text2SQL-1.5B", - "developer": "yasserrmd", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.544" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/yasserrmd_Text2SQL-1.5B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2857 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3858 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.068 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2878 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3942 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2363 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ycros_bagelmisterytour-v2-8x7b.json b/data/models/ycros_bagelmisterytour-v2-8x7b.json deleted file mode 100644 index ba69aabd10b1f09ccc48e0969d876027b03e3a4b..0000000000000000000000000000000000000000 --- a/data/models/ycros_bagelmisterytour-v2-8x7b.json +++ /dev/null @@ -1,275 +0,0 @@ -{ - "model_info": { - "name": "BagelMIsteryTour-v2-8x7B", - "id": "ycros/BagelMIsteryTour-v2-8x7B", - "developer": "ycros", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MixtralForCausalLM", - "params_billions": "46.703" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ycros_BagelMIsteryTour-v2-8x7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6262 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5142 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0937 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3079 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4138 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3481 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "hfopenllm_v2/ycros_BagelMIsteryTour-v2-8x7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5994 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5159 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0785 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3045 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4203 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3473 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/yfzp_llama-3-8b-instruct-sppo-iter1_bt_2b-table.json b/data/models/yfzp_llama-3-8b-instruct-sppo-iter1_bt_2b-table.json deleted file mode 100644 index 564918ca45ae455b68ccbebe6b30e1a0c2d0dbb3..0000000000000000000000000000000000000000 --- a/data/models/yfzp_llama-3-8b-instruct-sppo-iter1_bt_2b-table.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-8B-Instruct-SPPO-Iter1_bt_2b-table", - "id": "yfzp/Llama-3-8B-Instruct-SPPO-Iter1_bt_2b-table", - "developer": "yfzp", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/yfzp_Llama-3-8B-Instruct-SPPO-Iter1_bt_2b-table/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6709 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4987 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1118 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2592 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3727 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3716 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/yfzp_llama-3-8b-instruct-sppo-iter1_bt_8b-table.json b/data/models/yfzp_llama-3-8b-instruct-sppo-iter1_bt_8b-table.json deleted file mode 100644 index 8e0e139ceabe085c0f0c202412eb627710b0cfa8..0000000000000000000000000000000000000000 --- a/data/models/yfzp_llama-3-8b-instruct-sppo-iter1_bt_8b-table.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-8B-Instruct-SPPO-Iter1_bt_8b-table", - "id": "yfzp/Llama-3-8B-Instruct-SPPO-Iter1_bt_8b-table", - "developer": "yfzp", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/yfzp_Llama-3-8B-Instruct-SPPO-Iter1_bt_8b-table/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7333 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.508 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1035 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2601 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3806 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3748 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/yfzp_llama-3-8b-instruct-sppo-iter1_gp_2b-table.json b/data/models/yfzp_llama-3-8b-instruct-sppo-iter1_gp_2b-table.json deleted file mode 100644 index 38dcb76094867312af58c178b40d0ff963993765..0000000000000000000000000000000000000000 --- a/data/models/yfzp_llama-3-8b-instruct-sppo-iter1_gp_2b-table.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-8B-Instruct-SPPO-Iter1_gp_2b-table", - "id": "yfzp/Llama-3-8B-Instruct-SPPO-Iter1_gp_2b-table", - "developer": "yfzp", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/yfzp_Llama-3-8B-Instruct-SPPO-Iter1_gp_2b-table/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6785 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4941 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1125 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2592 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3647 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3718 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/yfzp_llama-3-8b-instruct-sppo-iter1_gp_8b-table.json b/data/models/yfzp_llama-3-8b-instruct-sppo-iter1_gp_8b-table.json deleted file mode 100644 index 66bf3237b8ebaed95bc0ca21bd4f74738523d6e8..0000000000000000000000000000000000000000 --- a/data/models/yfzp_llama-3-8b-instruct-sppo-iter1_gp_8b-table.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-8B-Instruct-SPPO-Iter1_gp_8b-table", - "id": "yfzp/Llama-3-8B-Instruct-SPPO-Iter1_gp_8b-table", - "developer": "yfzp", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/yfzp_Llama-3-8B-Instruct-SPPO-Iter1_gp_8b-table/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7132 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5025 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0989 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2592 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3713 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3683 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/yfzp_llama-3-8b-instruct-sppo-score-iter1_bt_2b-table-0.001.json b/data/models/yfzp_llama-3-8b-instruct-sppo-score-iter1_bt_2b-table-0.001.json deleted file mode 100644 index a1e7d4b0b3a6ff0bd4705778b1406347a71440df..0000000000000000000000000000000000000000 --- a/data/models/yfzp_llama-3-8b-instruct-sppo-score-iter1_bt_2b-table-0.001.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-8B-Instruct-SPPO-score-Iter1_bt_2b-table-0.001", - "id": "yfzp/Llama-3-8B-Instruct-SPPO-score-Iter1_bt_2b-table-0.001", - "developer": "yfzp", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/yfzp_Llama-3-8B-Instruct-SPPO-score-Iter1_bt_2b-table-0.001/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6496 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4979 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1012 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2592 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.378 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.372 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/yfzp_llama-3-8b-instruct-sppo-score-iter1_bt_8b-table-0.002.json b/data/models/yfzp_llama-3-8b-instruct-sppo-score-iter1_bt_8b-table-0.002.json deleted file mode 100644 index d92e2d8eefa7d5e84ed3fd9c49179763fbcc0e2d..0000000000000000000000000000000000000000 --- a/data/models/yfzp_llama-3-8b-instruct-sppo-score-iter1_bt_8b-table-0.002.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-8B-Instruct-SPPO-score-Iter1_bt_8b-table-0.002", - "id": "yfzp/Llama-3-8B-Instruct-SPPO-score-Iter1_bt_8b-table-0.002", - "developer": "yfzp", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/yfzp_Llama-3-8B-Instruct-SPPO-score-Iter1_bt_8b-table-0.002/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7196 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5045 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0876 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2601 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3831 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3734 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/yfzp_llama-3-8b-instruct-sppo-score-iter1_gp_2b-table-0.001.json b/data/models/yfzp_llama-3-8b-instruct-sppo-score-iter1_gp_2b-table-0.001.json deleted file mode 100644 index 287bbcbf5bdedaf9bf5dac49c299722e62ac660f..0000000000000000000000000000000000000000 --- a/data/models/yfzp_llama-3-8b-instruct-sppo-score-iter1_gp_2b-table-0.001.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-8B-Instruct-SPPO-score-Iter1_gp_2b-table-0.001", - "id": "yfzp/Llama-3-8B-Instruct-SPPO-score-Iter1_gp_2b-table-0.001", - "developer": "yfzp", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/yfzp_Llama-3-8B-Instruct-SPPO-score-Iter1_gp_2b-table-0.001/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6504 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4958 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0937 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2592 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.366 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3703 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/yfzp_llama-3-8b-instruct-sppo-score-iter1_gp_8b-table-0.002.json b/data/models/yfzp_llama-3-8b-instruct-sppo-score-iter1_gp_8b-table-0.002.json deleted file mode 100644 index 51616cba2ae976cbb39c00c3a94362dd911bae9d..0000000000000000000000000000000000000000 --- a/data/models/yfzp_llama-3-8b-instruct-sppo-score-iter1_gp_8b-table-0.002.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-8B-Instruct-SPPO-score-Iter1_gp_8b-table-0.002", - "id": "yfzp/Llama-3-8B-Instruct-SPPO-score-Iter1_gp_8b-table-0.002", - "developer": "yfzp", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/yfzp_Llama-3-8B-Instruct-SPPO-score-Iter1_gp_8b-table-0.002/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7016 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4992 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0869 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2592 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3779 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3669 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/yifai_llama-3-8b-instruct-sppo-score-iter3_gp_8b-table-0.002.json b/data/models/yifai_llama-3-8b-instruct-sppo-score-iter3_gp_8b-table-0.002.json deleted file mode 100644 index 8133b52bcefe4855eb3861b2ff3221520d46f60b..0000000000000000000000000000000000000000 --- a/data/models/yifai_llama-3-8b-instruct-sppo-score-iter3_gp_8b-table-0.002.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3-8B-Instruct-SPPO-score-Iter3_gp_8b-table-0.002", - "id": "yifAI/Llama-3-8B-Instruct-SPPO-score-Iter3_gp_8b-table-0.002", - "developer": "yifAI", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/yifAI_Llama-3-8B-Instruct-SPPO-score-Iter3_gp_8b-table-0.002/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.649 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4915 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0755 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2617 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3899 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.352 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ylalain_ece-prymmal-yl-1b-slerp-v8.json b/data/models/ylalain_ece-prymmal-yl-1b-slerp-v8.json deleted file mode 100644 index d93aa729e1289317c72fdcf80eab550c33aee9dd..0000000000000000000000000000000000000000 --- a/data/models/ylalain_ece-prymmal-yl-1b-slerp-v8.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ECE-PRYMMAL-YL-1B-SLERP-V8", - "id": "ylalain/ECE-PRYMMAL-YL-1B-SLERP-V8", - "developer": "ylalain", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.357" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ylalain_ECE-PRYMMAL-YL-1B-SLERP-V8/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1505 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3976 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0045 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2894 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3875 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2384 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ymcki_gemma-2-2b-jpn-it-abliterated-17-18-24.json b/data/models/ymcki_gemma-2-2b-jpn-it-abliterated-17-18-24.json deleted file mode 100644 index 6e4328d326c8cc667e129d7254c6181ad6c48446..0000000000000000000000000000000000000000 --- a/data/models/ymcki_gemma-2-2b-jpn-it-abliterated-17-18-24.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "gemma-2-2b-jpn-it-abliterated-17-18-24", - "id": "ymcki/gemma-2-2b-jpn-it-abliterated-17-18-24", - "developer": "ymcki", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "2.614" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ymcki_gemma-2-2b-jpn-it-abliterated-17-18-24/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5055 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3812 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0257 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.281 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3502 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2282 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ymcki_gemma-2-2b-jpn-it-abliterated-17-orpo-alpaca.json b/data/models/ymcki_gemma-2-2b-jpn-it-abliterated-17-orpo-alpaca.json deleted file mode 100644 index 0f6cabbb799faf6dab61f493eeb514839e371701..0000000000000000000000000000000000000000 --- a/data/models/ymcki_gemma-2-2b-jpn-it-abliterated-17-orpo-alpaca.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "gemma-2-2b-jpn-it-abliterated-17-ORPO-alpaca", - "id": "ymcki/gemma-2-2b-jpn-it-abliterated-17-ORPO-alpaca", - "developer": "ymcki", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "2.614" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ymcki_gemma-2-2b-jpn-it-abliterated-17-ORPO-alpaca/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3065 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4072 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0325 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2693 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3969 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2249 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ymcki_gemma-2-2b-jpn-it-abliterated-17-orpo.json b/data/models/ymcki_gemma-2-2b-jpn-it-abliterated-17-orpo.json deleted file mode 100644 index 13f36d702c0d0d41a0653a14643a2e783dfa4cdd..0000000000000000000000000000000000000000 --- a/data/models/ymcki_gemma-2-2b-jpn-it-abliterated-17-orpo.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "gemma-2-2b-jpn-it-abliterated-17-ORPO", - "id": "ymcki/gemma-2-2b-jpn-it-abliterated-17-ORPO", - "developer": "ymcki", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "2.614" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ymcki_gemma-2-2b-jpn-it-abliterated-17-ORPO/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4748 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3898 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0619 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2743 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3768 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2191 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ymcki_gemma-2-2b-jpn-it-abliterated-17.json b/data/models/ymcki_gemma-2-2b-jpn-it-abliterated-17.json deleted file mode 100644 index 0affb0dcd19e4f2f86401364bcd61f5052db386b..0000000000000000000000000000000000000000 --- a/data/models/ymcki_gemma-2-2b-jpn-it-abliterated-17.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "gemma-2-2b-jpn-it-abliterated-17", - "id": "ymcki/gemma-2-2b-jpn-it-abliterated-17", - "developer": "ymcki", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "2.614" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ymcki_gemma-2-2b-jpn-it-abliterated-17/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5082 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4076 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0385 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2718 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3701 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2455 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ymcki_gemma-2-2b-jpn-it-abliterated-18-orpo.json b/data/models/ymcki_gemma-2-2b-jpn-it-abliterated-18-orpo.json deleted file mode 100644 index 6c436dfdad22ce096c3dae0efd5f4e331bdfb0cd..0000000000000000000000000000000000000000 --- a/data/models/ymcki_gemma-2-2b-jpn-it-abliterated-18-orpo.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "gemma-2-2b-jpn-it-abliterated-18-ORPO", - "id": "ymcki/gemma-2-2b-jpn-it-abliterated-18-ORPO", - "developer": "ymcki", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "2.614" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ymcki_gemma-2-2b-jpn-it-abliterated-18-ORPO/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4742 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4039 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0468 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2617 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3953 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2185 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ymcki_gemma-2-2b-jpn-it-abliterated-18.json b/data/models/ymcki_gemma-2-2b-jpn-it-abliterated-18.json deleted file mode 100644 index ecbf78948cb1031b043b4cfdeb0557c3423c17c2..0000000000000000000000000000000000000000 --- a/data/models/ymcki_gemma-2-2b-jpn-it-abliterated-18.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "gemma-2-2b-jpn-it-abliterated-18", - "id": "ymcki/gemma-2-2b-jpn-it-abliterated-18", - "developer": "ymcki", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "2.614" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ymcki_gemma-2-2b-jpn-it-abliterated-18/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5175 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4132 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0446 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2735 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3742 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2505 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ymcki_gemma-2-2b-jpn-it-abliterated-24.json b/data/models/ymcki_gemma-2-2b-jpn-it-abliterated-24.json deleted file mode 100644 index ca3b69ab4097ce607aa33c9c4c5a983804cb9043..0000000000000000000000000000000000000000 --- a/data/models/ymcki_gemma-2-2b-jpn-it-abliterated-24.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "gemma-2-2b-jpn-it-abliterated-24", - "id": "ymcki/gemma-2-2b-jpn-it-abliterated-24", - "developer": "ymcki", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "2.614" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ymcki_gemma-2-2b-jpn-it-abliterated-24/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4979 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.411 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0438 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2777 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3915 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2473 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ymcki_gemma-2-2b-orpo-jpn-it-abliterated-18-merge.json b/data/models/ymcki_gemma-2-2b-orpo-jpn-it-abliterated-18-merge.json deleted file mode 100644 index bca60ce7e815e4e341617b8cd14d36a5f2a87dc6..0000000000000000000000000000000000000000 --- a/data/models/ymcki_gemma-2-2b-orpo-jpn-it-abliterated-18-merge.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "gemma-2-2b-ORPO-jpn-it-abliterated-18-merge", - "id": "ymcki/gemma-2-2b-ORPO-jpn-it-abliterated-18-merge", - "developer": "ymcki", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "2.614" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ymcki_gemma-2-2b-ORPO-jpn-it-abliterated-18-merge/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5218 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4147 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0544 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2836 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3514 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2461 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ymcki_gemma-2-2b-orpo-jpn-it-abliterated-18.json b/data/models/ymcki_gemma-2-2b-orpo-jpn-it-abliterated-18.json deleted file mode 100644 index f3dd3d9aa4ab9085b065ad57b7fdf6abfe0d874a..0000000000000000000000000000000000000000 --- a/data/models/ymcki_gemma-2-2b-orpo-jpn-it-abliterated-18.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "gemma-2-2b-ORPO-jpn-it-abliterated-18", - "id": "ymcki/gemma-2-2b-ORPO-jpn-it-abliterated-18", - "developer": "ymcki", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "2.614" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ymcki_gemma-2-2b-ORPO-jpn-it-abliterated-18/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4631 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4053 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0431 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2886 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3754 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2345 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ymcki_llama-3.1-8b-grpo-instruct.json b/data/models/ymcki_llama-3.1-8b-grpo-instruct.json deleted file mode 100644 index 5f83f9add9a71ab7e8042959165c2f569e24d5f1..0000000000000000000000000000000000000000 --- a/data/models/ymcki_llama-3.1-8b-grpo-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.1-8B-GRPO-Instruct", - "id": "ymcki/Llama-3.1-8B-GRPO-Instruct", - "developer": "ymcki", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ymcki_Llama-3.1-8B-GRPO-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7445 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5132 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2024 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2945 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3817 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3738 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ymcki_llama-3.1-8b-sft-grpo-instruct.json b/data/models/ymcki_llama-3.1-8b-sft-grpo-instruct.json deleted file mode 100644 index ae85265bd8696daf3f4a1ddb2f1c552b092e37cc..0000000000000000000000000000000000000000 --- a/data/models/ymcki_llama-3.1-8b-sft-grpo-instruct.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.1-8B-SFT-GRPO-Instruct", - "id": "ymcki/Llama-3.1-8B-SFT-GRPO-Instruct", - "developer": "ymcki", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ymcki_Llama-3.1-8B-SFT-GRPO-Instruct/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3354 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3126 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.04 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2534 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3526 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1098 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/youlln_1parammyl-8b-modelstock.json b/data/models/youlln_1parammyl-8b-modelstock.json deleted file mode 100644 index c0885ead6ff6de5b34531d33a3fd300426d8c749..0000000000000000000000000000000000000000 --- a/data/models/youlln_1parammyl-8b-modelstock.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "1PARAMMYL-8B-ModelStock", - "id": "Youlln/1PARAMMYL-8B-ModelStock", - "developer": "Youlln", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Youlln_1PARAMMYL-8B-ModelStock/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5371 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5216 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1488 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3238 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4409 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/youlln_2prymmal-yi1.5-6b-slerp.json b/data/models/youlln_2prymmal-yi1.5-6b-slerp.json deleted file mode 100644 index e2acd8850c3d2752c4688cc987a16ceb649ae373..0000000000000000000000000000000000000000 --- a/data/models/youlln_2prymmal-yi1.5-6b-slerp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "2PRYMMAL-Yi1.5-6B-SLERP", - "id": "Youlln/2PRYMMAL-Yi1.5-6B-SLERP", - "developer": "Youlln", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "6.061" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Youlln_2PRYMMAL-Yi1.5-6B-SLERP/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2826 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4665 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1133 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.307 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4756 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.317 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/youlln_3prymmal-phi3-3b-slerp.json b/data/models/youlln_3prymmal-phi3-3b-slerp.json deleted file mode 100644 index cb749b60fb914f6a74c64169788e9838ebb93624..0000000000000000000000000000000000000000 --- a/data/models/youlln_3prymmal-phi3-3b-slerp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "3PRYMMAL-PHI3-3B-SLERP", - "id": "Youlln/3PRYMMAL-PHI3-3B-SLERP", - "developer": "Youlln", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Phi3ForCausalLM", - "params_billions": "3.0" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Youlln_3PRYMMAL-PHI3-3B-SLERP/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3656 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5422 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1715 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3263 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4648 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4002 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/youlln_4prymmal-gemma2-9b-slerp.json b/data/models/youlln_4prymmal-gemma2-9b-slerp.json deleted file mode 100644 index 86a5fe93069b18ca2e5c0cbaea20779455ec456a..0000000000000000000000000000000000000000 --- a/data/models/youlln_4prymmal-gemma2-9b-slerp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "4PRYMMAL-GEMMA2-9B-SLERP", - "id": "Youlln/4PRYMMAL-GEMMA2-9B-SLERP", - "developer": "Youlln", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "9.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Youlln_4PRYMMAL-GEMMA2-9B-SLERP/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2714 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5923 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0906 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3305 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4672 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.421 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/youlln_ece-mirage-1-12b.json b/data/models/youlln_ece-mirage-1-12b.json deleted file mode 100644 index f4ae2816aa714f93774ef26c267d42fbb9788cdc..0000000000000000000000000000000000000000 --- a/data/models/youlln_ece-mirage-1-12b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ECE-MIRAGE-1-12B", - "id": "Youlln/ECE-MIRAGE-1-12B", - "developer": "Youlln", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "15.21" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Youlln_ECE-MIRAGE-1-12B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.207 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3011 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2634 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3219 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.111 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/youlln_ece-mirage-1-15b.json b/data/models/youlln_ece-mirage-1-15b.json deleted file mode 100644 index cb3e028172efe6859becdf62af13163cb4fb03b9..0000000000000000000000000000000000000000 --- a/data/models/youlln_ece-mirage-1-15b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ECE-MIRAGE-1-15B", - "id": "Youlln/ECE-MIRAGE-1-15B", - "developer": "Youlln", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "15.21" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Youlln_ECE-MIRAGE-1-15B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.207 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3011 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2634 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3219 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.111 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/youlln_ece-prymmal-0.5b-ft-v3-musr.json b/data/models/youlln_ece-prymmal-0.5b-ft-v3-musr.json deleted file mode 100644 index 0475411d7e247652d907d0a82fd99e530cc6ef89..0000000000000000000000000000000000000000 --- a/data/models/youlln_ece-prymmal-0.5b-ft-v3-musr.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ECE-PRYMMAL-0.5B-FT-V3-MUSR", - "id": "Youlln/ECE-PRYMMAL-0.5B-FT-V3-MUSR", - "developer": "Youlln", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Youlln_ECE-PRYMMAL-0.5B-FT-V3-MUSR/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1533 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3041 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0242 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2492 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.366 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1645 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/youlln_ece-prymmal-0.5b-ft-v3.json b/data/models/youlln_ece-prymmal-0.5b-ft-v3.json deleted file mode 100644 index 8822c749d4bfe0206b47bc1ebc1992d2d1d4284d..0000000000000000000000000000000000000000 --- a/data/models/youlln_ece-prymmal-0.5b-ft-v3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ECE-PRYMMAL-0.5B-FT-V3", - "id": "Youlln/ECE-PRYMMAL-0.5B-FT-V3", - "developer": "Youlln", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Youlln_ECE-PRYMMAL-0.5B-FT-V3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1642 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3093 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.003 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2576 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3644 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1161 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/youlln_ece-prymmal-0.5b-ft-v4-musr.json b/data/models/youlln_ece-prymmal-0.5b-ft-v4-musr.json deleted file mode 100644 index b1245822a5aa517ebe7e6c8b4a5642f1abe734d1..0000000000000000000000000000000000000000 --- a/data/models/youlln_ece-prymmal-0.5b-ft-v4-musr.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ECE-PRYMMAL-0.5B-FT-V4-MUSR", - "id": "Youlln/ECE-PRYMMAL-0.5B-FT-V4-MUSR", - "developer": "Youlln", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Youlln_ECE-PRYMMAL-0.5B-FT-V4-MUSR/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1138 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3038 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0121 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2701 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3529 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1321 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/youlln_ece-prymmal-0.5b-slerp-v2.json b/data/models/youlln_ece-prymmal-0.5b-slerp-v2.json deleted file mode 100644 index 49782621efd6af1421140a5a8f278298c6678226..0000000000000000000000000000000000000000 --- a/data/models/youlln_ece-prymmal-0.5b-slerp-v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ECE-PRYMMAL-0.5B-SLERP-V2", - "id": "Youlln/ECE-PRYMMAL-0.5B-SLERP-V2", - "developer": "Youlln", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Youlln_ECE-PRYMMAL-0.5B-SLERP-V2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1612 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2935 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0008 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2743 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3831 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1095 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/youlln_ece-prymmal-0.5b-slerp-v3.json b/data/models/youlln_ece-prymmal-0.5b-slerp-v3.json deleted file mode 100644 index 920ecd0a53bd01552db2576783910efccd423375..0000000000000000000000000000000000000000 --- a/data/models/youlln_ece-prymmal-0.5b-slerp-v3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ECE-PRYMMAL-0.5B-SLERP-V3", - "id": "Youlln/ECE-PRYMMAL-0.5B-SLERP-V3", - "developer": "Youlln", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Youlln_ECE-PRYMMAL-0.5B-SLERP-V3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.167 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2938 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2517 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3541 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1087 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/youlln_ece-prymmal-yl-1b-slerp-v1.json b/data/models/youlln_ece-prymmal-yl-1b-slerp-v1.json deleted file mode 100644 index 81a56f78209c9a5f04369d91418c26c73bf7ade8..0000000000000000000000000000000000000000 --- a/data/models/youlln_ece-prymmal-yl-1b-slerp-v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ECE-PRYMMAL-YL-1B-SLERP-V1", - "id": "Youlln/ECE-PRYMMAL-YL-1B-SLERP-V1", - "developer": "Youlln", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.544" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Youlln_ECE-PRYMMAL-YL-1B-SLERP-V1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3251 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4209 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1073 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2911 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4266 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2936 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/youlln_ece-prymmal-yl-1b-slerp-v2.json b/data/models/youlln_ece-prymmal-yl-1b-slerp-v2.json deleted file mode 100644 index 40bcf971d1d4d09e75fce234235ddce6a8891c6b..0000000000000000000000000000000000000000 --- a/data/models/youlln_ece-prymmal-yl-1b-slerp-v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ECE-PRYMMAL-YL-1B-SLERP-V2", - "id": "Youlln/ECE-PRYMMAL-YL-1B-SLERP-V2", - "developer": "Youlln", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.544" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Youlln_ECE-PRYMMAL-YL-1B-SLERP-V2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3251 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4209 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1073 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2911 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4266 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2936 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/youlln_ece-prymmal-yl-7b-slerp-v4.json b/data/models/youlln_ece-prymmal-yl-7b-slerp-v4.json deleted file mode 100644 index 645d2b108f971c736b06809c4b7d25c4dec0f004..0000000000000000000000000000000000000000 --- a/data/models/youlln_ece-prymmal-yl-7b-slerp-v4.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ECE-PRYMMAL-YL-7B-SLERP-V4", - "id": "Youlln/ECE-PRYMMAL-YL-7B-SLERP-V4", - "developer": "Youlln", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Youlln_ECE-PRYMMAL-YL-7B-SLERP-V4/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.251 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.377 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0536 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2651 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3745 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2132 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/youlln_ece-prymmal0.5-ft.json b/data/models/youlln_ece-prymmal0.5-ft.json deleted file mode 100644 index 0ab1cfcb165cd160ea4b7c39a5b0c277953c6d27..0000000000000000000000000000000000000000 --- a/data/models/youlln_ece-prymmal0.5-ft.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ECE-PRYMMAL0.5-FT", - "id": "Youlln/ECE-PRYMMAL0.5-FT", - "developer": "Youlln", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Youlln_ECE-PRYMMAL0.5-FT/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1851 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3132 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0234 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2559 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3301 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1477 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/youlln_ece-prymmal0.5b-youri.json b/data/models/youlln_ece-prymmal0.5b-youri.json deleted file mode 100644 index 111ca67bed5846723872c41a02998d8bc546a98f..0000000000000000000000000000000000000000 --- a/data/models/youlln_ece-prymmal0.5b-youri.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ECE-PRYMMAL0.5B-Youri", - "id": "Youlln/ECE-PRYMMAL0.5B-Youri", - "developer": "Youlln", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Youlln_ECE-PRYMMAL0.5B-Youri/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1446 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2817 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2433 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3697 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1095 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/youlln_ece-prymmal1b-ft-v1.json b/data/models/youlln_ece-prymmal1b-ft-v1.json deleted file mode 100644 index b150eaa423820af06592a34e25eaabd42cba51d3..0000000000000000000000000000000000000000 --- a/data/models/youlln_ece-prymmal1b-ft-v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ECE-PRYMMAL1B-FT-V1", - "id": "Youlln/ECE-PRYMMAL1B-FT-V1", - "developer": "Youlln", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "1.544" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Youlln_ECE-PRYMMAL1B-FT-V1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2144 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4033 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0642 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2785 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3417 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2743 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/youlln_ece-qwen0.5b-ft-v2.json b/data/models/youlln_ece-qwen0.5b-ft-v2.json deleted file mode 100644 index deb208770573b7ffb3ba48afe545cb95f6cdf28a..0000000000000000000000000000000000000000 --- a/data/models/youlln_ece-qwen0.5b-ft-v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ECE-Qwen0.5B-FT-V2", - "id": "Youlln/ECE-Qwen0.5B-FT-V2", - "developer": "Youlln", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.494" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Youlln_ECE-Qwen0.5B-FT-V2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2526 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.329 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0204 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2668 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3063 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1666 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/youlln_ece.eiffeil.ia-0.5b-slerp.json b/data/models/youlln_ece.eiffeil.ia-0.5b-slerp.json deleted file mode 100644 index 6df6740f0a5a62573cf0458a5eb32b4eafb27338..0000000000000000000000000000000000000000 --- a/data/models/youlln_ece.eiffeil.ia-0.5b-slerp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ECE.EIFFEIL.ia-0.5B-SLERP", - "id": "Youlln/ECE.EIFFEIL.ia-0.5B-SLERP", - "developer": "Youlln", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "0.63" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Youlln_ECE.EIFFEIL.ia-0.5B-SLERP/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2561 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3306 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0597 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2651 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3102 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1903 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/youngpanda_qwenqwen.json b/data/models/youngpanda_qwenqwen.json deleted file mode 100644 index 73a26a4a22535b206192261600f13e2afb4eda1d..0000000000000000000000000000000000000000 --- a/data/models/youngpanda_qwenqwen.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "qwenqwen", - "id": "YoungPanda/qwenqwen", - "developer": "YoungPanda", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2MoeForCausalLM", - "params_billions": "14.316" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/YoungPanda_qwenqwen/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1264 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3379 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0355 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.25 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3434 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1168 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/yoyo-ai_qwen2.5-14b-1m-yoyo-v3.json b/data/models/yoyo-ai_qwen2.5-14b-1m-yoyo-v3.json deleted file mode 100644 index 9a09b44db939ce36ee55d21101612f3e835ad591..0000000000000000000000000000000000000000 --- a/data/models/yoyo-ai_qwen2.5-14b-1m-yoyo-v3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-14B-1M-YOYO-V3", - "id": "YOYO-AI/Qwen2.5-14B-1M-YOYO-V3", - "developer": "YOYO-AI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/YOYO-AI_Qwen2.5-14B-1M-YOYO-V3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8398 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6448 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5355 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3289 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4141 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5207 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/yoyo-ai_qwen2.5-14b-it-restore.json b/data/models/yoyo-ai_qwen2.5-14b-it-restore.json deleted file mode 100644 index 08dfd1ac0f0942e4a946f7fd5cf2bb111901a7d9..0000000000000000000000000000000000000000 --- a/data/models/yoyo-ai_qwen2.5-14b-it-restore.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-14B-it-restore", - "id": "YOYO-AI/Qwen2.5-14B-it-restore", - "developer": "YOYO-AI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/YOYO-AI_Qwen2.5-14B-it-restore/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8209 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6388 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.537 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3372 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4087 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.49 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/yoyo-ai_qwen2.5-14b-yoyo-0505.json b/data/models/yoyo-ai_qwen2.5-14b-yoyo-0505.json deleted file mode 100644 index 2a7d5961237408ab161a2c95234b4b7684cb2c9c..0000000000000000000000000000000000000000 --- a/data/models/yoyo-ai_qwen2.5-14b-yoyo-0505.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-14B-YOYO-0505", - "id": "YOYO-AI/Qwen2.5-14B-YOYO-0505", - "developer": "YOYO-AI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/YOYO-AI_Qwen2.5-14B-YOYO-0505/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5883 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6539 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4434 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3733 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4757 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5371 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/yoyo-ai_qwen2.5-14b-yoyo-0510-v2.json b/data/models/yoyo-ai_qwen2.5-14b-yoyo-0510-v2.json deleted file mode 100644 index de862c7a8f42699fb87ae3303e5a119bf05db670..0000000000000000000000000000000000000000 --- a/data/models/yoyo-ai_qwen2.5-14b-yoyo-0510-v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-14B-YOYO-0510-v2", - "id": "YOYO-AI/Qwen2.5-14B-YOYO-0510-v2", - "developer": "YOYO-AI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/YOYO-AI_Qwen2.5-14B-YOYO-0510-v2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5947 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6553 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4441 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3817 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4744 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5381 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/yoyo-ai_qwen2.5-14b-yoyo-0805.json b/data/models/yoyo-ai_qwen2.5-14b-yoyo-0805.json deleted file mode 100644 index 69f57ab1e867b89f12b7d831c7219c5b21504595..0000000000000000000000000000000000000000 --- a/data/models/yoyo-ai_qwen2.5-14b-yoyo-0805.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-14B-YOYO-0805", - "id": "YOYO-AI/Qwen2.5-14B-YOYO-0805", - "developer": "YOYO-AI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/YOYO-AI_Qwen2.5-14B-YOYO-0805/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5883 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6539 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4434 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3733 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4757 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5371 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/yoyo-ai_qwen2.5-14b-yoyo-1005-v2.json b/data/models/yoyo-ai_qwen2.5-14b-yoyo-1005-v2.json deleted file mode 100644 index ea9f927b150f7645bf91016e50121d1d0cdb0dfa..0000000000000000000000000000000000000000 --- a/data/models/yoyo-ai_qwen2.5-14b-yoyo-1005-v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-14B-YOYO-1005-v2", - "id": "YOYO-AI/Qwen2.5-14B-YOYO-1005-v2", - "developer": "YOYO-AI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/YOYO-AI_Qwen2.5-14B-YOYO-1005-v2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5953 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6551 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4434 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3842 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4731 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5372 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/yoyo-ai_qwen2.5-14b-yoyo-1005.json b/data/models/yoyo-ai_qwen2.5-14b-yoyo-1005.json deleted file mode 100644 index e69ec4febaa3604f7b2ca3609cd1049e086d3306..0000000000000000000000000000000000000000 --- a/data/models/yoyo-ai_qwen2.5-14b-yoyo-1005.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-14B-YOYO-1005", - "id": "YOYO-AI/Qwen2.5-14B-YOYO-1005", - "developer": "YOYO-AI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/YOYO-AI_Qwen2.5-14B-YOYO-1005/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5972 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6542 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4524 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3809 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.473 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5382 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/yoyo-ai_qwen2.5-14b-yoyo-1010-v2.json b/data/models/yoyo-ai_qwen2.5-14b-yoyo-1010-v2.json deleted file mode 100644 index 0037e6fda1d2b2a1931e48c467c48086154db2f7..0000000000000000000000000000000000000000 --- a/data/models/yoyo-ai_qwen2.5-14b-yoyo-1010-v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-14B-YOYO-1010-v2", - "id": "YOYO-AI/Qwen2.5-14B-YOYO-1010-v2", - "developer": "YOYO-AI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/YOYO-AI_Qwen2.5-14B-YOYO-1010-v2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5947 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6553 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4441 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3817 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4744 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5381 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/yoyo-ai_qwen2.5-14b-yoyo-1010.json b/data/models/yoyo-ai_qwen2.5-14b-yoyo-1010.json deleted file mode 100644 index e42112fd11a9b36734c8e8c4176443d8815fdc44..0000000000000000000000000000000000000000 --- a/data/models/yoyo-ai_qwen2.5-14b-yoyo-1010.json +++ /dev/null @@ -1,275 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-14B-YOYO-1010", - "id": "YOYO-AI/Qwen2.5-14B-YOYO-1010", - "developer": "YOYO-AI", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/YOYO-AI_Qwen2.5-14B-YOYO-1010/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7905 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6406 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3163 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4181 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4944 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - }, - { - "evaluation_id": "hfopenllm_v2/YOYO-AI_Qwen2.5-14B-YOYO-1010/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5899 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.654 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4509 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3834 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4744 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5376 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/yoyo-ai_qwen2.5-14b-yoyo-latest-v2.json b/data/models/yoyo-ai_qwen2.5-14b-yoyo-latest-v2.json deleted file mode 100644 index 0d6b494c81cdb18100638c255c01eb79784bae19..0000000000000000000000000000000000000000 --- a/data/models/yoyo-ai_qwen2.5-14b-yoyo-latest-v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-14B-YOYO-latest-V2", - "id": "YOYO-AI/Qwen2.5-14B-YOYO-latest-V2", - "developer": "YOYO-AI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/YOYO-AI_Qwen2.5-14B-YOYO-latest-V2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7771 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6299 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5159 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.354 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4299 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5224 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/yoyo-ai_qwen2.5-14b-yoyo-latest.json b/data/models/yoyo-ai_qwen2.5-14b-yoyo-latest.json deleted file mode 100644 index 2ad223fecec26c4153e8e726c6be2b3071a35571..0000000000000000000000000000000000000000 --- a/data/models/yoyo-ai_qwen2.5-14b-yoyo-latest.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-14B-YOYO-latest", - "id": "YOYO-AI/Qwen2.5-14B-YOYO-latest", - "developer": "YOYO-AI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/YOYO-AI_Qwen2.5-14B-YOYO-latest/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5911 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6656 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4418 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3826 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4691 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5371 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/yoyo-ai_qwen2.5-14b-yoyo-sce.json b/data/models/yoyo-ai_qwen2.5-14b-yoyo-sce.json deleted file mode 100644 index ba4ab9da91de28120544377ff711e61fc96dbe62..0000000000000000000000000000000000000000 --- a/data/models/yoyo-ai_qwen2.5-14b-yoyo-sce.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-14B-YOYO-SCE", - "id": "YOYO-AI/Qwen2.5-14B-YOYO-SCE", - "developer": "YOYO-AI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/YOYO-AI_Qwen2.5-14B-YOYO-SCE/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5844 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6489 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4615 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3742 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4704 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5381 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/yoyo-ai_qwen2.5-14b-yoyo-v4-p1.json b/data/models/yoyo-ai_qwen2.5-14b-yoyo-v4-p1.json deleted file mode 100644 index 9049c9417feb999696f504eee45b4774933084fb..0000000000000000000000000000000000000000 --- a/data/models/yoyo-ai_qwen2.5-14b-yoyo-v4-p1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-14B-YOYO-V4-p1", - "id": "YOYO-AI/Qwen2.5-14B-YOYO-V4-p1", - "developer": "YOYO-AI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/YOYO-AI_Qwen2.5-14B-YOYO-V4-p1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8203 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6516 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5332 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3456 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4194 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.502 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/yoyo-ai_qwen2.5-14b-yoyo-v4-p2.json b/data/models/yoyo-ai_qwen2.5-14b-yoyo-v4-p2.json deleted file mode 100644 index b027be3995de4aba0d5bd4e251c35fb25999730a..0000000000000000000000000000000000000000 --- a/data/models/yoyo-ai_qwen2.5-14b-yoyo-v4-p2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-14B-YOYO-V4-p2", - "id": "YOYO-AI/Qwen2.5-14B-YOYO-V4-p2", - "developer": "YOYO-AI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/YOYO-AI_Qwen2.5-14B-YOYO-V4-p2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8048 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6339 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5166 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3272 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4435 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4968 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/yoyo-ai_qwen2.5-14b-yoyo-v4.json b/data/models/yoyo-ai_qwen2.5-14b-yoyo-v4.json deleted file mode 100644 index 8c7b0fa5873f778559ef7afbc57f6094e9f11334..0000000000000000000000000000000000000000 --- a/data/models/yoyo-ai_qwen2.5-14b-yoyo-v4.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-14B-YOYO-V4", - "id": "YOYO-AI/Qwen2.5-14B-YOYO-V4", - "developer": "YOYO-AI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/YOYO-AI_Qwen2.5-14B-YOYO-V4/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8398 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.649 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5347 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3221 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4115 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.517 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/yoyo-ai_qwen2.5-7b-it-restore.json b/data/models/yoyo-ai_qwen2.5-7b-it-restore.json deleted file mode 100644 index 3e85e41b45e954b22e5bd527145b9b7316c5b756..0000000000000000000000000000000000000000 --- a/data/models/yoyo-ai_qwen2.5-7b-it-restore.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-7B-it-restore", - "id": "YOYO-AI/Qwen2.5-7B-it-restore", - "developer": "YOYO-AI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.613" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/YOYO-AI_Qwen2.5-7B-it-restore/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7531 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5407 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3012 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4007 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4288 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/yoyo-ai_qwen2.5-coder-14b-yoyo-1010.json b/data/models/yoyo-ai_qwen2.5-coder-14b-yoyo-1010.json deleted file mode 100644 index 0db5758b5d06850c3c7627abcbb0519f4ddffa6d..0000000000000000000000000000000000000000 --- a/data/models/yoyo-ai_qwen2.5-coder-14b-yoyo-1010.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-Coder-14B-YOYO-1010", - "id": "YOYO-AI/Qwen2.5-Coder-14B-YOYO-1010", - "developer": "YOYO-AI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/YOYO-AI_Qwen2.5-Coder-14B-YOYO-1010/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5336 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6187 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3218 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3523 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4422 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4075 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/yoyo-ai_zyh-llm-qwen2.5-14b-v2.json b/data/models/yoyo-ai_zyh-llm-qwen2.5-14b-v2.json deleted file mode 100644 index 2991edcbf18fbe589473d775ac4bb32afcb7665c..0000000000000000000000000000000000000000 --- a/data/models/yoyo-ai_zyh-llm-qwen2.5-14b-v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ZYH-LLM-Qwen2.5-14B-V2", - "id": "YOYO-AI/ZYH-LLM-Qwen2.5-14B-V2", - "developer": "YOYO-AI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/YOYO-AI_ZYH-LLM-Qwen2.5-14B-V2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5071 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6452 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3542 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3792 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4689 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5372 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/yoyo-ai_zyh-llm-qwen2.5-14b-v3.json b/data/models/yoyo-ai_zyh-llm-qwen2.5-14b-v3.json deleted file mode 100644 index 175b729cc588877cc30300f8d36a39fc9a5a79d6..0000000000000000000000000000000000000000 --- a/data/models/yoyo-ai_zyh-llm-qwen2.5-14b-v3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ZYH-LLM-Qwen2.5-14B-V3", - "id": "YOYO-AI/ZYH-LLM-Qwen2.5-14B-V3", - "developer": "YOYO-AI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/YOYO-AI_ZYH-LLM-Qwen2.5-14B-V3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8578 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6359 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5272 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3322 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4022 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4881 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/yoyo-ai_zyh-llm-qwen2.5-14b-v4.json b/data/models/yoyo-ai_zyh-llm-qwen2.5-14b-v4.json deleted file mode 100644 index deccd38e74fd30db89f2dfa7ad9be2a75fa4e4f2..0000000000000000000000000000000000000000 --- a/data/models/yoyo-ai_zyh-llm-qwen2.5-14b-v4.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ZYH-LLM-Qwen2.5-14B-V4", - "id": "YOYO-AI/ZYH-LLM-Qwen2.5-14B-V4", - "developer": "YOYO-AI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.766" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/YOYO-AI_ZYH-LLM-Qwen2.5-14B-V4/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8365 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6515 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5393 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3146 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4434 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5204 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/yoyo-ai_zyh-llm-qwen2.5-14b.json b/data/models/yoyo-ai_zyh-llm-qwen2.5-14b.json deleted file mode 100644 index 42939efd38ac423c397f6ee6b1ba0cbb8a906c77..0000000000000000000000000000000000000000 --- a/data/models/yoyo-ai_zyh-llm-qwen2.5-14b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ZYH-LLM-Qwen2.5-14B", - "id": "YOYO-AI/ZYH-LLM-Qwen2.5-14B", - "developer": "YOYO-AI", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "14.77" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/YOYO-AI_ZYH-LLM-Qwen2.5-14B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5941 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6644 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4116 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3859 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4757 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5351 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/yuchenxie_arlowgpt-3b-multilingual.json b/data/models/yuchenxie_arlowgpt-3b-multilingual.json deleted file mode 100644 index f8ee868ba8d641a19ac4447031a68d920c9daeda..0000000000000000000000000000000000000000 --- a/data/models/yuchenxie_arlowgpt-3b-multilingual.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ArlowGPT-3B-Multilingual", - "id": "yuchenxie/ArlowGPT-3B-Multilingual", - "developer": "yuchenxie", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "3.213" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/yuchenxie_ArlowGPT-3B-Multilingual/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6395 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4301 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1125 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2802 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3727 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2817 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/yuchenxie_arlowgpt-8b.json b/data/models/yuchenxie_arlowgpt-8b.json deleted file mode 100644 index 4a397f35a651b9b0847b041c27efcf2b3d7f4f49..0000000000000000000000000000000000000000 --- a/data/models/yuchenxie_arlowgpt-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "ArlowGPT-8B", - "id": "yuchenxie/ArlowGPT-8B", - "developer": "yuchenxie", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/yuchenxie_ArlowGPT-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7847 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.508 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2039 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2936 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3882 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3787 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/yuma42_kangalkhan-rawruby-7b.json b/data/models/yuma42_kangalkhan-rawruby-7b.json deleted file mode 100644 index 18acee021be4784589cafb908b87780e4ac35800..0000000000000000000000000000000000000000 --- a/data/models/yuma42_kangalkhan-rawruby-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "KangalKhan-RawRuby-7B", - "id": "Yuma42/KangalKhan-RawRuby-7B", - "developer": "Yuma42", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MistralForCausalLM", - "params_billions": "7.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Yuma42_KangalKhan-RawRuby-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5477 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4755 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0665 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2878 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.395 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3023 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/yuma42_llama3.1-igneousiguana-8b.json b/data/models/yuma42_llama3.1-igneousiguana-8b.json deleted file mode 100644 index 6af491b761054316da28edc8e50c36e81158a2b2..0000000000000000000000000000000000000000 --- a/data/models/yuma42_llama3.1-igneousiguana-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama3.1-IgneousIguana-8B", - "id": "Yuma42/Llama3.1-IgneousIguana-8B", - "developer": "Yuma42", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Yuma42_Llama3.1-IgneousIguana-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8133 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5191 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2198 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3104 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4203 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3974 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/yuma42_llama3.1-superhawk-8b.json b/data/models/yuma42_llama3.1-superhawk-8b.json deleted file mode 100644 index f5fdc81dbcaf4adfb98d52287d41a1bef3395a98..0000000000000000000000000000000000000000 --- a/data/models/yuma42_llama3.1-superhawk-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama3.1-SuperHawk-8B", - "id": "Yuma42/Llama3.1-SuperHawk-8B", - "developer": "Yuma42", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Yuma42_Llama3.1-SuperHawk-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7986 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.52 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2349 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3129 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4084 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3945 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/yuvraj17_llama3-8b-abliterated-spectrum-slerp.json b/data/models/yuvraj17_llama3-8b-abliterated-spectrum-slerp.json deleted file mode 100644 index 8a143bba6f57d5a3ff5d26106da69147e0005e11..0000000000000000000000000000000000000000 --- a/data/models/yuvraj17_llama3-8b-abliterated-spectrum-slerp.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama3-8B-abliterated-Spectrum-slerp", - "id": "yuvraj17/Llama3-8B-abliterated-Spectrum-slerp", - "developer": "yuvraj17", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/yuvraj17_Llama3-8B-abliterated-Spectrum-slerp/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2885 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4978 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0604 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3012 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3998 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3257 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/yuvraj17_llama3-8b-supernova-spectrum-dare_ties.json b/data/models/yuvraj17_llama3-8b-supernova-spectrum-dare_ties.json deleted file mode 100644 index 36d676a6dd9069fa3b65e96d68263dfd6fa2f33e..0000000000000000000000000000000000000000 --- a/data/models/yuvraj17_llama3-8b-supernova-spectrum-dare_ties.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama3-8B-SuperNova-Spectrum-dare_ties", - "id": "yuvraj17/Llama3-8B-SuperNova-Spectrum-dare_ties", - "developer": "yuvraj17", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/yuvraj17_Llama3-8B-SuperNova-Spectrum-dare_ties/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4013 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4616 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0846 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2752 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4211 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3574 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/yuvraj17_llama3-8b-supernova-spectrum-hermes-dpo.json b/data/models/yuvraj17_llama3-8b-supernova-spectrum-hermes-dpo.json deleted file mode 100644 index 24c32d5f7543b266ec14d2293bd9a58acac35adb..0000000000000000000000000000000000000000 --- a/data/models/yuvraj17_llama3-8b-supernova-spectrum-hermes-dpo.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama3-8B-SuperNova-Spectrum-Hermes-DPO", - "id": "yuvraj17/Llama3-8B-SuperNova-Spectrum-Hermes-DPO", - "developer": "yuvraj17", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/yuvraj17_Llama3-8B-SuperNova-Spectrum-Hermes-DPO/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4691 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.44 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0566 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.302 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4012 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2635 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/z-ai_glm-4.5.json b/data/models/z-ai_glm-4.5.json deleted file mode 100644 index 985371f993f42473322e82aa6e1b37555ebaa259..0000000000000000000000000000000000000000 --- a/data/models/z-ai_glm-4.5.json +++ /dev/null @@ -1,89 +0,0 @@ -{ - "model_info": { - "name": "z-ai/glm-4.5", - "developer": "Z.ai", - "inference_platform": "openrouter", - "id": "z-ai/glm-4.5" - }, - "evaluations": [ - { - "evaluation_id": "livecodebenchpro/z-ai/glm-4.5/1760492095.8105888", - "retrieved_timestamp": "1760492095.8105888", - "source_metadata": { - "source_organization_name": "New York University, Princeton University, University of California San Diego, University of Washington and Canyon Crest Academy", - "evaluator_relationship": "third_party", - "source_name": "Live Code Bench Pro", - "source_type": "documentation" - }, - "eval_library": { - "name": "unknown", - "version": "unknown" - }, - "benchmark": "livecodebenchpro", - "evaluation_results": [ - { - "evaluation_name": "Hard Problems", - "metric_config": { - "evaluation_description": "Pass@1 on Hard Problems", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.0 - }, - "source_data": { - "dataset_name": "Hard Problems", - "source_type": "url", - "url": [ - "https://webhook.cp-bench.orzzh.com/leaderboard/llm/difficulty?difficulty=hard&benchmark_mode=live" - ] - } - }, - { - "evaluation_name": "Medium Problems", - "metric_config": { - "evaluation_description": "Pass@1 on Medium Problems", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.028169014084507043 - }, - "source_data": { - "dataset_name": "Medium Problems", - "source_type": "url", - "url": [ - "https://webhook.cp-bench.orzzh.com/leaderboard/llm/difficulty?difficulty=medium&benchmark_mode=live" - ] - } - }, - { - "evaluation_name": "Easy Problems", - "metric_config": { - "evaluation_description": "Pass@1 on Easy Problems", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.1267605633802817 - }, - "source_data": { - "dataset_name": "Easy Problems", - "source_type": "url", - "url": [ - "https://webhook.cp-bench.orzzh.com/leaderboard/llm/difficulty?difficulty=easy&benchmark_mode=live" - ] - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/z1-coder_z1-coder-7b.json b/data/models/z1-coder_z1-coder-7b.json deleted file mode 100644 index b7e64fde51a679fc1994014f53345b5bc73fbde6..0000000000000000000000000000000000000000 --- a/data/models/z1-coder_z1-coder-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Z1-Coder-7B", - "id": "Z1-Coder/Z1-Coder-7B", - "developer": "Z1-Coder", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.613" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/Z1-Coder_Z1-Coder-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3215 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4842 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3248 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2727 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3622 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3759 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zai-org_glm-4.5-air-fp8.json b/data/models/zai-org_glm-4.5-air-fp8.json deleted file mode 100644 index 3ec3bf6fefd394db75728a36d02ae7dd7f863c43..0000000000000000000000000000000000000000 --- a/data/models/zai-org_glm-4.5-air-fp8.json +++ /dev/null @@ -1,234 +0,0 @@ -{ - "model_info": { - "name": "GLM-4.5-Air-FP8", - "id": "zai-org/glm-4.5-air-fp8", - "developer": "zai-org", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_capabilities/zai-org_glm-4.5-air-fp8/1774096304.5056698", - "retrieved_timestamp": "1774096304.5056698", - "source_metadata": { - "source_name": "helm_capabilities", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_capabilities", - "evaluation_results": [ - { - "evaluation_name": "Mean score", - "source_data": { - "dataset_name": "helm_capabilities", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "The mean of the scores from all columns.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.67, - "details": { - "description": "", - "tab": "Accuracy", - "Mean score - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"36.15586140714108\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU-Pro", - "source_data": { - "dataset_name": "MMLU-Pro", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on MMLU-Pro", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.762, - "details": { - "description": "min=0.762, mean=0.762, max=0.762, sum=0.762 (1)", - "tab": "Accuracy", - "MMLU-Pro - Observed inference time (s)": "{\"description\": \"min=30.422, mean=30.422, max=30.422, sum=30.422 (1)\", \"tab\": \"Efficiency\", \"score\": \"30.421801055192947\"}", - "MMLU-Pro - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "MMLU-Pro - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU-Pro - # prompt tokens": "{\"description\": \"min=250.402, mean=250.402, max=250.402, sum=250.402 (1)\", \"tab\": \"General information\", \"score\": \"250.402\"}", - "MMLU-Pro - # output tokens": "{\"description\": \"min=4423.528, mean=4423.528, max=4423.528, sum=4423.528 (1)\", \"tab\": \"General information\", \"score\": \"4423.528\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"all\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "COT correct on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.594, - "details": { - "description": "min=0.594, mean=0.594, max=0.594, sum=0.594 (1)", - "tab": "Accuracy", - "GPQA - Observed inference time (s)": "{\"description\": \"min=54.963, mean=54.963, max=54.963, sum=54.963 (1)\", \"tab\": \"Efficiency\", \"score\": \"54.96293809649121\"}", - "GPQA - # eval": "{\"description\": \"min=446, mean=446, max=446, sum=446 (1)\", \"tab\": \"General information\", \"score\": \"446.0\"}", - "GPQA - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "GPQA - # prompt tokens": "{\"description\": \"min=269.978, mean=269.978, max=269.978, sum=269.978 (1)\", \"tab\": \"General information\", \"score\": \"269.9775784753363\"}", - "GPQA - # output tokens": "{\"description\": \"min=8628.161, mean=8628.161, max=8628.161, sum=8628.161 (1)\", \"tab\": \"General information\", \"score\": \"8628.16143497758\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"gpqa_main\"", - "use_chain_of_thought": "\"true\"", - "use_few_shot": "\"false\"" - } - } - }, - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "IFEval Strict Acc on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.812, - "details": { - "description": "min=0.812, mean=0.812, max=0.812, sum=0.812 (1)", - "tab": "Accuracy", - "IFEval - Observed inference time (s)": "{\"description\": \"min=8.027, mean=8.027, max=8.027, sum=8.027 (1)\", \"tab\": \"Efficiency\", \"score\": \"8.026858968787625\"}", - "IFEval - # eval": "{\"description\": \"min=541, mean=541, max=541, sum=541 (1)\", \"tab\": \"General information\", \"score\": \"541.0\"}", - "IFEval - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IFEval - # prompt tokens": "{\"description\": \"min=46.026, mean=46.026, max=46.026, sum=46.026 (1)\", \"tab\": \"General information\", \"score\": \"46.02587800369686\"}", - "IFEval - # output tokens": "{\"description\": \"min=1330.573, mean=1330.573, max=1330.573, sum=1330.573 (1)\", \"tab\": \"General information\", \"score\": \"1330.573012939002\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "WildBench", - "source_data": { - "dataset_name": "WildBench", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "WB Score on WildBench", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.789, - "details": { - "description": "min=0.789, mean=0.789, max=0.789, sum=0.789 (1)", - "tab": "Accuracy", - "WildBench - Observed inference time (s)": "{\"description\": \"min=25.055, mean=25.055, max=25.055, sum=25.055 (1)\", \"tab\": \"Efficiency\", \"score\": \"25.055315640687944\"}", - "WildBench - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "WildBench - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # prompt tokens": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "WildBench - # output tokens": "{\"description\": \"min=4196.241, mean=4196.241, max=4196.241, sum=4196.241 (1)\", \"tab\": \"General information\", \"score\": \"4196.241\"}" - } - }, - "generation_config": { - "additional_details": { - "subset": "\"v2\"" - } - } - }, - { - "evaluation_name": "Omni-MATH", - "source_data": { - "dataset_name": "Omni-MATH", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.12.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "Acc on Omni-MATH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.391, - "details": { - "description": "min=0.391, mean=0.391, max=0.391, sum=0.391 (1)", - "tab": "Accuracy", - "Omni-MATH - Observed inference time (s)": "{\"description\": \"min=62.312, mean=62.312, max=62.312, sum=62.312 (1)\", \"tab\": \"Efficiency\", \"score\": \"62.31239327454567\"}", - "Omni-MATH - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=1000 (1)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "Omni-MATH - # train": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (1)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "Omni-MATH - # prompt tokens": "{\"description\": \"min=109.807, mean=109.807, max=109.807, sum=109.807 (1)\", \"tab\": \"General information\", \"score\": \"109.807\"}", - "Omni-MATH - # output tokens": "{\"description\": \"min=11088.014, mean=11088.014, max=11088.014, sum=11088.014 (1)\", \"tab\": \"General information\", \"score\": \"11088.014\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/zake7749_gemma-2-2b-it-chinese-kyara-dpo.json b/data/models/zake7749_gemma-2-2b-it-chinese-kyara-dpo.json deleted file mode 100644 index a3d594de9cf0d9ecfbd8e4558a1b549a700c3e40..0000000000000000000000000000000000000000 --- a/data/models/zake7749_gemma-2-2b-it-chinese-kyara-dpo.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "gemma-2-2b-it-chinese-kyara-dpo", - "id": "zake7749/gemma-2-2b-it-chinese-kyara-dpo", - "developer": "zake7749", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "2.614" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/zake7749_gemma-2-2b-it-chinese-kyara-dpo/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5382 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4257 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0838 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2668 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4576 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2573 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zake7749_gemma-2-9b-it-chinese-kyara.json b/data/models/zake7749_gemma-2-9b-it-chinese-kyara.json deleted file mode 100644 index cf2eafc30b5cbc55d84d6255aeef9641c431bccd..0000000000000000000000000000000000000000 --- a/data/models/zake7749_gemma-2-9b-it-chinese-kyara.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "gemma-2-9b-it-chinese-kyara", - "id": "zake7749/gemma-2-9b-it-chinese-kyara", - "developer": "zake7749", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "9.242" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/zake7749_gemma-2-9b-it-chinese-kyara/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1764 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5954 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.105 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3381 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4242 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4179 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zelk12_gemma-2-s2mtm-9b.json b/data/models/zelk12_gemma-2-s2mtm-9b.json deleted file mode 100644 index 4b7fd64f21509fb4ac4099ca7963d0d27307720f..0000000000000000000000000000000000000000 --- a/data/models/zelk12_gemma-2-s2mtm-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "gemma-2-S2MTM-9B", - "id": "zelk12/gemma-2-S2MTM-9B", - "developer": "zelk12", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/zelk12_gemma-2-S2MTM-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7823 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6061 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2047 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3456 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4218 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4297 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zelk12_gemma-2-tm-9b.json b/data/models/zelk12_gemma-2-tm-9b.json deleted file mode 100644 index 14be4a4fefb672461ea31ef9ea518746fae04b49..0000000000000000000000000000000000000000 --- a/data/models/zelk12_gemma-2-tm-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Gemma-2-TM-9B", - "id": "zelk12/Gemma-2-TM-9B", - "developer": "zelk12", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/zelk12_Gemma-2-TM-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8045 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5987 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2024 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3465 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4152 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4088 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zelk12_mt-gemma-2-9b.json b/data/models/zelk12_mt-gemma-2-9b.json deleted file mode 100644 index bc9f1f51436e269415296c3ea7bd7060c4dc437a..0000000000000000000000000000000000000000 --- a/data/models/zelk12_mt-gemma-2-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MT-gemma-2-9B", - "id": "zelk12/MT-gemma-2-9B", - "developer": "zelk12", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/zelk12_MT-gemma-2-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7968 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6064 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2054 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3456 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4071 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4224 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zelk12_mt-gen1-gemma-2-9b.json b/data/models/zelk12_mt-gen1-gemma-2-9b.json deleted file mode 100644 index f7007f3dddce3cbf0ebccd6ce46065597201609d..0000000000000000000000000000000000000000 --- a/data/models/zelk12_mt-gen1-gemma-2-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MT-Gen1-gemma-2-9B", - "id": "zelk12/MT-Gen1-gemma-2-9B", - "developer": "zelk12", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/zelk12_MT-Gen1-gemma-2-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7886 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.61 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2221 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3465 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4217 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4381 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zelk12_mt-gen2-gemma-2-9b.json b/data/models/zelk12_mt-gen2-gemma-2-9b.json deleted file mode 100644 index 45eaa839174905efe9919e04eaa25fe03a4e95f9..0000000000000000000000000000000000000000 --- a/data/models/zelk12_mt-gen2-gemma-2-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MT-Gen2-gemma-2-9B", - "id": "zelk12/MT-Gen2-gemma-2-9B", - "developer": "zelk12", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/zelk12_MT-Gen2-gemma-2-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7907 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.61 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.219 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3465 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4323 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4387 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zelk12_mt-gen2-gi-gemma-2-9b.json b/data/models/zelk12_mt-gen2-gi-gemma-2-9b.json deleted file mode 100644 index b93f548a87edfc38553958aaf94a92763a837795..0000000000000000000000000000000000000000 --- a/data/models/zelk12_mt-gen2-gi-gemma-2-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MT-Gen2-GI-gemma-2-9B", - "id": "zelk12/MT-Gen2-GI-gemma-2-9B", - "developer": "zelk12", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/zelk12_MT-Gen2-GI-gemma-2-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7914 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6096 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2205 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3507 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4283 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4356 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zelk12_mt-gen3-gemma-2-9b.json b/data/models/zelk12_mt-gen3-gemma-2-9b.json deleted file mode 100644 index b2902014e05ec4586f6b9c71faefe069c9860ea1..0000000000000000000000000000000000000000 --- a/data/models/zelk12_mt-gen3-gemma-2-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MT-Gen3-gemma-2-9B", - "id": "zelk12/MT-Gen3-gemma-2-9B", - "developer": "zelk12", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/zelk12_MT-Gen3-gemma-2-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.802 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6097 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2296 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.349 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4217 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4356 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zelk12_mt-gen4-gemma-2-9b.json b/data/models/zelk12_mt-gen4-gemma-2-9b.json deleted file mode 100644 index 818b447360f72ef08ef9f07779b6cf0b65cb7dc9..0000000000000000000000000000000000000000 --- a/data/models/zelk12_mt-gen4-gemma-2-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MT-Gen4-gemma-2-9B", - "id": "zelk12/MT-Gen4-gemma-2-9B", - "developer": "zelk12", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/zelk12_MT-Gen4-gemma-2-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7883 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.611 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2236 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3549 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4228 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4387 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zelk12_mt-gen5-gemma-2-9b.json b/data/models/zelk12_mt-gen5-gemma-2-9b.json deleted file mode 100644 index bed23c5ca94c6bd9ba8c45ee39ee423ce9699a3f..0000000000000000000000000000000000000000 --- a/data/models/zelk12_mt-gen5-gemma-2-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MT-Gen5-gemma-2-9B", - "id": "zelk12/MT-Gen5-gemma-2-9B", - "developer": "zelk12", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/zelk12_MT-Gen5-gemma-2-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7923 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6133 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2153 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3515 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4202 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4402 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zelk12_mt-gen6-gemma-2-9b.json b/data/models/zelk12_mt-gen6-gemma-2-9b.json deleted file mode 100644 index e7b4d5f5af020960aae87a0032874c1e7bdcfdd0..0000000000000000000000000000000000000000 --- a/data/models/zelk12_mt-gen6-gemma-2-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MT-Gen6-gemma-2-9B", - "id": "zelk12/MT-Gen6-gemma-2-9B", - "developer": "zelk12", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/zelk12_MT-Gen6-gemma-2-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1616 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5845 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0823 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3331 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4069 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4166 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zelk12_mt-gen6fix-gemma-2-9b.json b/data/models/zelk12_mt-gen6fix-gemma-2-9b.json deleted file mode 100644 index 37e00be8852d1de4bb458dae8dc4ce5d8323cb29..0000000000000000000000000000000000000000 --- a/data/models/zelk12_mt-gen6fix-gemma-2-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MT-Gen6fix-gemma-2-9B", - "id": "zelk12/MT-Gen6fix-gemma-2-9B", - "developer": "zelk12", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/zelk12_MT-Gen6fix-gemma-2-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1576 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5917 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0816 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3372 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4084 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.412 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zelk12_mt-gen7-gemma-2-9b.json b/data/models/zelk12_mt-gen7-gemma-2-9b.json deleted file mode 100644 index 659d2596ee716ce9fae92dab1ef6a0b468cddcd0..0000000000000000000000000000000000000000 --- a/data/models/zelk12_mt-gen7-gemma-2-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MT-Gen7-gemma-2-9B", - "id": "zelk12/MT-Gen7-gemma-2-9B", - "developer": "zelk12", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/zelk12_MT-Gen7-gemma-2-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1664 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5935 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0891 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3356 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4098 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4122 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zelk12_mt-max-merge_02012025163610-gemma-2-9b.json b/data/models/zelk12_mt-max-merge_02012025163610-gemma-2-9b.json deleted file mode 100644 index 5682fe04fc9a06ae4c11a145c1f3adb95993f80c..0000000000000000000000000000000000000000 --- a/data/models/zelk12_mt-max-merge_02012025163610-gemma-2-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MT-Max-Merge_02012025163610-gemma-2-9B", - "id": "zelk12/MT-Max-Merge_02012025163610-gemma-2-9B", - "developer": "zelk12", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/zelk12_MT-Max-Merge_02012025163610-gemma-2-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7907 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6142 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2213 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3515 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4228 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4396 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zelk12_mt-merge-gemma-2-9b.json b/data/models/zelk12_mt-merge-gemma-2-9b.json deleted file mode 100644 index 3420596805be7c2469ec810666422e5f14f9996c..0000000000000000000000000000000000000000 --- a/data/models/zelk12_mt-merge-gemma-2-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MT-Merge-gemma-2-9B", - "id": "zelk12/MT-Merge-gemma-2-9B", - "developer": "zelk12", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/zelk12_MT-Merge-gemma-2-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8035 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6118 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2205 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3482 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4256 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4362 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zelk12_mt-merge1-gemma-2-9b.json b/data/models/zelk12_mt-merge1-gemma-2-9b.json deleted file mode 100644 index 3d47fd8505547ceb3c2ad2109dd27c75e74a3185..0000000000000000000000000000000000000000 --- a/data/models/zelk12_mt-merge1-gemma-2-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MT-Merge1-gemma-2-9B", - "id": "zelk12/MT-Merge1-gemma-2-9B", - "developer": "zelk12", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/zelk12_MT-Merge1-gemma-2-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7901 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.61 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2289 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3515 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4244 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4374 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zelk12_mt-merge2-gemma-2-9b.json b/data/models/zelk12_mt-merge2-gemma-2-9b.json deleted file mode 100644 index 41d079c397c13eb3d1ed9e363406332ad99a10ab..0000000000000000000000000000000000000000 --- a/data/models/zelk12_mt-merge2-gemma-2-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MT-Merge2-gemma-2-9B", - "id": "zelk12/MT-Merge2-gemma-2-9B", - "developer": "zelk12", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/zelk12_MT-Merge2-gemma-2-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7877 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6107 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2349 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3507 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4217 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4382 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zelk12_mt-merge2-mu-gemma-2-mtg2mt1g2-9b.json b/data/models/zelk12_mt-merge2-mu-gemma-2-mtg2mt1g2-9b.json deleted file mode 100644 index 2dd06f2954d7af72d24dad4a8c2dc5022e62a197..0000000000000000000000000000000000000000 --- a/data/models/zelk12_mt-merge2-mu-gemma-2-mtg2mt1g2-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MT-Merge2-MU-gemma-2-MTg2MT1g2-9B", - "id": "zelk12/MT-Merge2-MU-gemma-2-MTg2MT1g2-9B", - "developer": "zelk12", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/zelk12_MT-Merge2-MU-gemma-2-MTg2MT1g2-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7956 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6084 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2183 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3507 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4322 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4373 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zelk12_mt-merge3-gemma-2-9b.json b/data/models/zelk12_mt-merge3-gemma-2-9b.json deleted file mode 100644 index e2482f3939977f6ec9a47fcfe47aa2562de7c28d..0000000000000000000000000000000000000000 --- a/data/models/zelk12_mt-merge3-gemma-2-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MT-Merge3-gemma-2-9B", - "id": "zelk12/MT-Merge3-gemma-2-9B", - "developer": "zelk12", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/zelk12_MT-Merge3-gemma-2-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7859 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6102 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2205 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.349 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4258 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4373 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zelk12_mt-merge4-gemma-2-9b.json b/data/models/zelk12_mt-merge4-gemma-2-9b.json deleted file mode 100644 index 6e91bb2ff17ce9a22e565adeb2f20e8c6da14203..0000000000000000000000000000000000000000 --- a/data/models/zelk12_mt-merge4-gemma-2-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MT-Merge4-gemma-2-9B", - "id": "zelk12/MT-Merge4-gemma-2-9B", - "developer": "zelk12", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/zelk12_MT-Merge4-gemma-2-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7807 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6118 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2168 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3523 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4294 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.439 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zelk12_mt-merge5-gemma-2-9b.json b/data/models/zelk12_mt-merge5-gemma-2-9b.json deleted file mode 100644 index 25923671b33d125aa7b70a90f2ce4234ff1dfd99..0000000000000000000000000000000000000000 --- a/data/models/zelk12_mt-merge5-gemma-2-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MT-Merge5-gemma-2-9B", - "id": "zelk12/MT-Merge5-gemma-2-9B", - "developer": "zelk12", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/zelk12_MT-Merge5-gemma-2-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7844 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6123 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2183 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3532 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4281 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4387 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zelk12_mt-merge6-gemma-2-9b.json b/data/models/zelk12_mt-merge6-gemma-2-9b.json deleted file mode 100644 index 47815540f87c4eed25be55338360d86d13bb1e9b..0000000000000000000000000000000000000000 --- a/data/models/zelk12_mt-merge6-gemma-2-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MT-Merge6-gemma-2-9B", - "id": "zelk12/MT-Merge6-gemma-2-9B", - "developer": "zelk12", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/zelk12_MT-Merge6-gemma-2-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1695 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5949 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0801 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3289 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4098 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4115 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zelk12_mt1-gemma-2-9b.json b/data/models/zelk12_mt1-gemma-2-9b.json deleted file mode 100644 index 62e6a86c445a6e3d35ee61ed112e2b93cf2e5cac..0000000000000000000000000000000000000000 --- a/data/models/zelk12_mt1-gemma-2-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MT1-gemma-2-9B", - "id": "zelk12/MT1-gemma-2-9B", - "developer": "zelk12", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/zelk12_MT1-gemma-2-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7947 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6109 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2236 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3456 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4322 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4358 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zelk12_mt1-gen1-gemma-2-9b.json b/data/models/zelk12_mt1-gen1-gemma-2-9b.json deleted file mode 100644 index 0c96687bb47882581a3b4546651af6e6d95400d5..0000000000000000000000000000000000000000 --- a/data/models/zelk12_mt1-gen1-gemma-2-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MT1-Gen1-gemma-2-9B", - "id": "zelk12/MT1-Gen1-gemma-2-9B", - "developer": "zelk12", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/zelk12_MT1-Gen1-gemma-2-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7974 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6118 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2243 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.344 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.431 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4376 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zelk12_mt1-gen2-gemma-2-9b.json b/data/models/zelk12_mt1-gen2-gemma-2-9b.json deleted file mode 100644 index 60872100251b5dfbed9af474c34f15f3e20a59cc..0000000000000000000000000000000000000000 --- a/data/models/zelk12_mt1-gen2-gemma-2-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MT1-Gen2-gemma-2-9B", - "id": "zelk12/MT1-Gen2-gemma-2-9B", - "developer": "zelk12", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/zelk12_MT1-Gen2-gemma-2-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7984 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6096 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2251 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3523 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4284 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4355 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zelk12_mt1-gen3-gemma-2-9b.json b/data/models/zelk12_mt1-gen3-gemma-2-9b.json deleted file mode 100644 index b25a1e5abf1b7a1f33014edfda464b2986b8cfb2..0000000000000000000000000000000000000000 --- a/data/models/zelk12_mt1-gen3-gemma-2-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MT1-Gen3-gemma-2-9B", - "id": "zelk12/MT1-Gen3-gemma-2-9B", - "developer": "zelk12", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/zelk12_MT1-Gen3-gemma-2-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.796 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6102 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2243 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.349 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4243 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4349 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zelk12_mt1-gen4-gemma-2-9b.json b/data/models/zelk12_mt1-gen4-gemma-2-9b.json deleted file mode 100644 index ad95750c5f805838a64caabf4f361411d8ccda9f..0000000000000000000000000000000000000000 --- a/data/models/zelk12_mt1-gen4-gemma-2-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MT1-Gen4-gemma-2-9B", - "id": "zelk12/MT1-Gen4-gemma-2-9B", - "developer": "zelk12", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/zelk12_MT1-Gen4-gemma-2-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7941 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6058 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.216 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3473 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4231 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4286 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zelk12_mt1-gen5-gemma-2-9b.json b/data/models/zelk12_mt1-gen5-gemma-2-9b.json deleted file mode 100644 index 982e81c428ce1f1028716bb50521bf39597a0793..0000000000000000000000000000000000000000 --- a/data/models/zelk12_mt1-gen5-gemma-2-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MT1-Gen5-gemma-2-9B", - "id": "zelk12/MT1-Gen5-gemma-2-9B", - "developer": "zelk12", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/zelk12_MT1-Gen5-gemma-2-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7795 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6017 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2077 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3465 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4191 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4222 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zelk12_mt1-gen5-if-gemma-2-s2dmv1-9b.json b/data/models/zelk12_mt1-gen5-if-gemma-2-s2dmv1-9b.json deleted file mode 100644 index 89a7aa6503f9ecad3f95a68238a6bef55fc99e70..0000000000000000000000000000000000000000 --- a/data/models/zelk12_mt1-gen5-if-gemma-2-s2dmv1-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MT1-Gen5-IF-gemma-2-S2DMv1-9B", - "id": "zelk12/MT1-Gen5-IF-gemma-2-S2DMv1-9B", - "developer": "zelk12", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/zelk12_MT1-Gen5-IF-gemma-2-S2DMv1-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7929 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2032 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.344 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4245 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4218 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zelk12_mt1-gen6-gemma-2-9b.json b/data/models/zelk12_mt1-gen6-gemma-2-9b.json deleted file mode 100644 index 093b90d42928739815257595ed3ea9604fb553b3..0000000000000000000000000000000000000000 --- a/data/models/zelk12_mt1-gen6-gemma-2-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MT1-Gen6-gemma-2-9B", - "id": "zelk12/MT1-Gen6-gemma-2-9B", - "developer": "zelk12", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/zelk12_MT1-Gen6-gemma-2-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1634 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5944 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0808 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.328 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4044 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4133 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zelk12_mt1-gen7-gemma-2-9b.json b/data/models/zelk12_mt1-gen7-gemma-2-9b.json deleted file mode 100644 index c44d2b5375b73201be57f4f49a7c4a874a601b12..0000000000000000000000000000000000000000 --- a/data/models/zelk12_mt1-gen7-gemma-2-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MT1-Gen7-gemma-2-9B", - "id": "zelk12/MT1-Gen7-gemma-2-9B", - "developer": "zelk12", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/zelk12_MT1-Gen7-gemma-2-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1634 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5938 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0831 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.328 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4111 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4145 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zelk12_mt1-max-merge_02012025163610-gemma-2-9b.json b/data/models/zelk12_mt1-max-merge_02012025163610-gemma-2-9b.json deleted file mode 100644 index 7ff264a780efbb06a63019a337f104952f4b80fd..0000000000000000000000000000000000000000 --- a/data/models/zelk12_mt1-max-merge_02012025163610-gemma-2-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MT1-Max-Merge_02012025163610-gemma-2-9B", - "id": "zelk12/MT1-Max-Merge_02012025163610-gemma-2-9B", - "developer": "zelk12", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/zelk12_MT1-Max-Merge_02012025163610-gemma-2-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7929 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6123 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2228 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3549 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4255 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4382 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zelk12_mt2-gemma-2-9b.json b/data/models/zelk12_mt2-gemma-2-9b.json deleted file mode 100644 index 5ebee97dfb55ef9f45310fa72929c8a24880f3af..0000000000000000000000000000000000000000 --- a/data/models/zelk12_mt2-gemma-2-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MT2-gemma-2-9B", - "id": "zelk12/MT2-gemma-2-9B", - "developer": "zelk12", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/zelk12_MT2-gemma-2-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7886 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6115 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2213 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3473 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4217 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4368 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zelk12_mt2-gen1-gemma-2-9b.json b/data/models/zelk12_mt2-gen1-gemma-2-9b.json deleted file mode 100644 index 0ce98943747d0086f0a6bed245f169fb9caa8c34..0000000000000000000000000000000000000000 --- a/data/models/zelk12_mt2-gen1-gemma-2-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MT2-Gen1-gemma-2-9B", - "id": "zelk12/MT2-Gen1-gemma-2-9B", - "developer": "zelk12", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/zelk12_MT2-Gen1-gemma-2-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7856 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6101 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2213 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3431 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4243 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4377 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zelk12_mt2-gen2-gemma-2-9b.json b/data/models/zelk12_mt2-gen2-gemma-2-9b.json deleted file mode 100644 index da26f692bc1f6d8f42a2fff6bd8c67dceb56e571..0000000000000000000000000000000000000000 --- a/data/models/zelk12_mt2-gen2-gemma-2-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MT2-Gen2-gemma-2-9B", - "id": "zelk12/MT2-Gen2-gemma-2-9B", - "developer": "zelk12", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/zelk12_MT2-Gen2-gemma-2-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7889 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6093 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2183 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3465 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.427 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4388 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zelk12_mt2-gen3-gemma-2-9b.json b/data/models/zelk12_mt2-gen3-gemma-2-9b.json deleted file mode 100644 index 8a26101bbe6d7594d43d5536a664543e73a97e13..0000000000000000000000000000000000000000 --- a/data/models/zelk12_mt2-gen3-gemma-2-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MT2-Gen3-gemma-2-9B", - "id": "zelk12/MT2-Gen3-gemma-2-9B", - "developer": "zelk12", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/zelk12_MT2-Gen3-gemma-2-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.781 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6105 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2107 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3465 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4231 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4374 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zelk12_mt2-gen4-gemma-2-9b.json b/data/models/zelk12_mt2-gen4-gemma-2-9b.json deleted file mode 100644 index a1e3289d6efdc0f2f9d61d568967237127dab290..0000000000000000000000000000000000000000 --- a/data/models/zelk12_mt2-gen4-gemma-2-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MT2-Gen4-gemma-2-9B", - "id": "zelk12/MT2-Gen4-gemma-2-9B", - "developer": "zelk12", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/zelk12_MT2-Gen4-gemma-2-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7896 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6097 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2236 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3456 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4125 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4321 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zelk12_mt2-gen5-gemma-2-9b.json b/data/models/zelk12_mt2-gen5-gemma-2-9b.json deleted file mode 100644 index 8762d7343ee19a61c97d675fc92f5c0f9dd5357c..0000000000000000000000000000000000000000 --- a/data/models/zelk12_mt2-gen5-gemma-2-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MT2-Gen5-gemma-2-9B", - "id": "zelk12/MT2-Gen5-gemma-2-9B", - "developer": "zelk12", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/zelk12_MT2-Gen5-gemma-2-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7749 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6064 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2107 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3515 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4244 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4302 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zelk12_mt2-gen6-gemma-2-9b.json b/data/models/zelk12_mt2-gen6-gemma-2-9b.json deleted file mode 100644 index 0fb6670d5048b0a1e68bcdecf51bff1f552129de..0000000000000000000000000000000000000000 --- a/data/models/zelk12_mt2-gen6-gemma-2-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MT2-Gen6-gemma-2-9B", - "id": "zelk12/MT2-Gen6-gemma-2-9B", - "developer": "zelk12", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/zelk12_MT2-Gen6-gemma-2-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1664 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.596 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0846 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3381 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4137 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.421 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zelk12_mt2-gen7-gemma-2-9b.json b/data/models/zelk12_mt2-gen7-gemma-2-9b.json deleted file mode 100644 index d5b1088d8696d499f9194da5458df059f82ef4bf..0000000000000000000000000000000000000000 --- a/data/models/zelk12_mt2-gen7-gemma-2-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MT2-Gen7-gemma-2-9B", - "id": "zelk12/MT2-Gen7-gemma-2-9B", - "developer": "zelk12", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/zelk12_MT2-Gen7-gemma-2-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1762 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6079 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.102 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3549 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4203 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4311 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zelk12_mt2-max-merge_02012025163610-gemma-2-9b.json b/data/models/zelk12_mt2-max-merge_02012025163610-gemma-2-9b.json deleted file mode 100644 index 1591eae08c9253c8e01511bff824a2fcc42f1148..0000000000000000000000000000000000000000 --- a/data/models/zelk12_mt2-max-merge_02012025163610-gemma-2-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MT2-Max-Merge_02012025163610-gemma-2-9B", - "id": "zelk12/MT2-Max-Merge_02012025163610-gemma-2-9B", - "developer": "zelk12", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/zelk12_MT2-Max-Merge_02012025163610-gemma-2-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7901 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6108 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2243 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3515 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4228 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4391 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zelk12_mt3-gemma-2-9b.json b/data/models/zelk12_mt3-gemma-2-9b.json deleted file mode 100644 index ec48237b349f69ffa77729f337365b93cc4d58da..0000000000000000000000000000000000000000 --- a/data/models/zelk12_mt3-gemma-2-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MT3-gemma-2-9B", - "id": "zelk12/MT3-gemma-2-9B", - "developer": "zelk12", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/zelk12_MT3-gemma-2-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7786 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6131 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2168 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3448 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4243 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4327 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zelk12_mt3-gen1-gemma-2-9b.json b/data/models/zelk12_mt3-gen1-gemma-2-9b.json deleted file mode 100644 index 6cae05dc08126d11feb70ebd84ace64577bdcc08..0000000000000000000000000000000000000000 --- a/data/models/zelk12_mt3-gen1-gemma-2-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MT3-Gen1-gemma-2-9B", - "id": "zelk12/MT3-Gen1-gemma-2-9B", - "developer": "zelk12", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/zelk12_MT3-Gen1-gemma-2-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7838 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6107 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2145 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3465 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4151 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4327 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zelk12_mt3-gen2-gemma-2-9b.json b/data/models/zelk12_mt3-gen2-gemma-2-9b.json deleted file mode 100644 index 24cc94b5b2aad85b27349f5676438b88f3c6e126..0000000000000000000000000000000000000000 --- a/data/models/zelk12_mt3-gen2-gemma-2-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MT3-Gen2-gemma-2-9B", - "id": "zelk12/MT3-Gen2-gemma-2-9B", - "developer": "zelk12", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/zelk12_MT3-Gen2-gemma-2-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7843 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6091 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2236 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3574 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4111 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4333 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zelk12_mt3-gen3-gemma-2-9b.json b/data/models/zelk12_mt3-gen3-gemma-2-9b.json deleted file mode 100644 index b647f7e6295cb3c864156565b4243485e7d70bb0..0000000000000000000000000000000000000000 --- a/data/models/zelk12_mt3-gen3-gemma-2-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MT3-Gen3-gemma-2-9B", - "id": "zelk12/MT3-Gen3-gemma-2-9B", - "developer": "zelk12", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/zelk12_MT3-Gen3-gemma-2-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7856 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6089 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2153 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3515 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4258 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4303 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zelk12_mt3-gen4-gemma-2-9b.json b/data/models/zelk12_mt3-gen4-gemma-2-9b.json deleted file mode 100644 index 37000a1f1a3ea42394726dbec4d475b39214f79c..0000000000000000000000000000000000000000 --- a/data/models/zelk12_mt3-gen4-gemma-2-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MT3-Gen4-gemma-2-9B", - "id": "zelk12/MT3-Gen4-gemma-2-9B", - "developer": "zelk12", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/zelk12_MT3-Gen4-gemma-2-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7737 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6101 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2062 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3473 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4476 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4387 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zelk12_mt3-gen5-gemma-2-9b.json b/data/models/zelk12_mt3-gen5-gemma-2-9b.json deleted file mode 100644 index 8c7e22869662e90c5d4b0e0971e6d10327d5406e..0000000000000000000000000000000000000000 --- a/data/models/zelk12_mt3-gen5-gemma-2-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MT3-Gen5-gemma-2-9B", - "id": "zelk12/MT3-Gen5-gemma-2-9B", - "developer": "zelk12", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/zelk12_MT3-Gen5-gemma-2-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.799 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6099 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2266 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3532 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4191 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4317 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zelk12_mt3-gen5-gemma-2-9b_v1.json b/data/models/zelk12_mt3-gen5-gemma-2-9b_v1.json deleted file mode 100644 index 0b5ad70465555a89fb1512a53d409a86e9e03905..0000000000000000000000000000000000000000 --- a/data/models/zelk12_mt3-gen5-gemma-2-9b_v1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MT3-Gen5-gemma-2-9B_v1", - "id": "zelk12/MT3-Gen5-gemma-2-9B_v1", - "developer": "zelk12", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/zelk12_MT3-Gen5-gemma-2-9B_v1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7996 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6113 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2228 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.349 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4204 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4359 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zelk12_mt3-gen6-gemma-2-9b.json b/data/models/zelk12_mt3-gen6-gemma-2-9b.json deleted file mode 100644 index e1908f027cf4f850b69bde146e4a62d89c4728e7..0000000000000000000000000000000000000000 --- a/data/models/zelk12_mt3-gen6-gemma-2-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MT3-Gen6-gemma-2-9B", - "id": "zelk12/MT3-Gen6-gemma-2-9B", - "developer": "zelk12", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/zelk12_MT3-Gen6-gemma-2-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1762 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.602 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0884 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3431 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4126 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4102 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zelk12_mt3-max-merge_02012025163610-gemma-2-9b.json b/data/models/zelk12_mt3-max-merge_02012025163610-gemma-2-9b.json deleted file mode 100644 index 9f563779a8dbe2b60bbbde4246ac207ee595ae94..0000000000000000000000000000000000000000 --- a/data/models/zelk12_mt3-max-merge_02012025163610-gemma-2-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MT3-Max-Merge_02012025163610-gemma-2-9B", - "id": "zelk12/MT3-Max-Merge_02012025163610-gemma-2-9B", - "developer": "zelk12", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/zelk12_MT3-Max-Merge_02012025163610-gemma-2-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1762 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6123 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1012 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3507 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4255 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4389 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zelk12_mt4-gemma-2-9b.json b/data/models/zelk12_mt4-gemma-2-9b.json deleted file mode 100644 index 30f1c7ef8b9bcd86c69a1825e91001858ede15fc..0000000000000000000000000000000000000000 --- a/data/models/zelk12_mt4-gemma-2-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MT4-gemma-2-9B", - "id": "zelk12/MT4-gemma-2-9B", - "developer": "zelk12", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/zelk12_MT4-gemma-2-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7762 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6073 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2085 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3381 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4309 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4366 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zelk12_mt4-gen1-gemma-2-9b.json b/data/models/zelk12_mt4-gen1-gemma-2-9b.json deleted file mode 100644 index e3747b70a7bb3aaa519ee5fc472672941692ed4c..0000000000000000000000000000000000000000 --- a/data/models/zelk12_mt4-gen1-gemma-2-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MT4-Gen1-gemma-2-9B", - "id": "zelk12/MT4-Gen1-gemma-2-9B", - "developer": "zelk12", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/zelk12_MT4-Gen1-gemma-2-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7895 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6094 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2198 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.344 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4322 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4389 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zelk12_mt4-gen2-gemma-2-9b.json b/data/models/zelk12_mt4-gen2-gemma-2-9b.json deleted file mode 100644 index 191ae35562d22751d48981c7bf8ab5547ffab8a8..0000000000000000000000000000000000000000 --- a/data/models/zelk12_mt4-gen2-gemma-2-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MT4-Gen2-gemma-2-9B", - "id": "zelk12/MT4-Gen2-gemma-2-9B", - "developer": "zelk12", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/zelk12_MT4-Gen2-gemma-2-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8051 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6108 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2326 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3456 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4257 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4368 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zelk12_mt4-gen3-gemma-2-9b.json b/data/models/zelk12_mt4-gen3-gemma-2-9b.json deleted file mode 100644 index 4511d9cb6187d017654573903c6370a02667493e..0000000000000000000000000000000000000000 --- a/data/models/zelk12_mt4-gen3-gemma-2-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MT4-Gen3-gemma-2-9B", - "id": "zelk12/MT4-Gen3-gemma-2-9B", - "developer": "zelk12", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/zelk12_MT4-Gen3-gemma-2-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7841 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6087 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.219 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.344 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4243 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4381 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zelk12_mt4-gen4-gemma-2-9b.json b/data/models/zelk12_mt4-gen4-gemma-2-9b.json deleted file mode 100644 index 9ad6d53353ed86e7759962fdd0e87a9a41189176..0000000000000000000000000000000000000000 --- a/data/models/zelk12_mt4-gen4-gemma-2-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MT4-Gen4-gemma-2-9B", - "id": "zelk12/MT4-Gen4-gemma-2-9B", - "developer": "zelk12", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/zelk12_MT4-Gen4-gemma-2-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7874 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6076 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2145 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3523 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4244 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4323 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zelk12_mt4-gen5-gemma-2-9b.json b/data/models/zelk12_mt4-gen5-gemma-2-9b.json deleted file mode 100644 index 51c2c98f6b907bcf281ca2f70dbd0471483bc53e..0000000000000000000000000000000000000000 --- a/data/models/zelk12_mt4-gen5-gemma-2-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MT4-Gen5-gemma-2-9B", - "id": "zelk12/MT4-Gen5-gemma-2-9B", - "developer": "zelk12", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/zelk12_MT4-Gen5-gemma-2-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7789 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6107 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2266 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3565 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4268 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4384 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zelk12_mt4-max-merge_02012025163610-gemma-2-9b.json b/data/models/zelk12_mt4-max-merge_02012025163610-gemma-2-9b.json deleted file mode 100644 index b7aede41104671f3d7b63dc69994269e7435eefc..0000000000000000000000000000000000000000 --- a/data/models/zelk12_mt4-max-merge_02012025163610-gemma-2-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MT4-Max-Merge_02012025163610-gemma-2-9B", - "id": "zelk12/MT4-Max-Merge_02012025163610-gemma-2-9B", - "developer": "zelk12", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/zelk12_MT4-Max-Merge_02012025163610-gemma-2-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1771 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.612 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0952 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3515 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4228 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4391 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zelk12_mt5-gemma-2-9b.json b/data/models/zelk12_mt5-gemma-2-9b.json deleted file mode 100644 index 731a717bd9f4066c92e1c83de86aa846312d37ef..0000000000000000000000000000000000000000 --- a/data/models/zelk12_mt5-gemma-2-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MT5-gemma-2-9B", - "id": "zelk12/MT5-gemma-2-9B", - "developer": "zelk12", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/zelk12_MT5-gemma-2-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8048 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6112 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2258 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3431 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4204 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4367 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zelk12_mt5-gen1-gemma-2-9b.json b/data/models/zelk12_mt5-gen1-gemma-2-9b.json deleted file mode 100644 index 387f5b1069c3091784650e8803891a332a354b2f..0000000000000000000000000000000000000000 --- a/data/models/zelk12_mt5-gen1-gemma-2-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MT5-Gen1-gemma-2-9B", - "id": "zelk12/MT5-Gen1-gemma-2-9B", - "developer": "zelk12", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/zelk12_MT5-Gen1-gemma-2-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7831 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.611 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2213 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3473 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4204 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4368 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zelk12_mt5-gen2-gemma-2-9b.json b/data/models/zelk12_mt5-gen2-gemma-2-9b.json deleted file mode 100644 index 248a4763981d25d2ad964358dd066f39dabe408b..0000000000000000000000000000000000000000 --- a/data/models/zelk12_mt5-gen2-gemma-2-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MT5-Gen2-gemma-2-9B", - "id": "zelk12/MT5-Gen2-gemma-2-9B", - "developer": "zelk12", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/zelk12_MT5-Gen2-gemma-2-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7962 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6105 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2205 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3515 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4163 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4379 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zelk12_mt5-gen3-gemma-2-9b.json b/data/models/zelk12_mt5-gen3-gemma-2-9b.json deleted file mode 100644 index 3e81733cd46055a8fc97e86ce2fe038f00aa1fcb..0000000000000000000000000000000000000000 --- a/data/models/zelk12_mt5-gen3-gemma-2-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MT5-Gen3-gemma-2-9B", - "id": "zelk12/MT5-Gen3-gemma-2-9B", - "developer": "zelk12", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/zelk12_MT5-Gen3-gemma-2-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7825 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.609 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2168 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3515 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4231 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4375 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zelk12_mt5-gen4-gemma-2-9b.json b/data/models/zelk12_mt5-gen4-gemma-2-9b.json deleted file mode 100644 index f60e91f58abc59a57767fd223b074ca80e2b8d3a..0000000000000000000000000000000000000000 --- a/data/models/zelk12_mt5-gen4-gemma-2-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MT5-Gen4-gemma-2-9B", - "id": "zelk12/MT5-Gen4-gemma-2-9B", - "developer": "zelk12", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/zelk12_MT5-Gen4-gemma-2-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7835 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6131 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2243 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3532 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4228 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4397 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zelk12_mt5-gen5-gemma-2-9b.json b/data/models/zelk12_mt5-gen5-gemma-2-9b.json deleted file mode 100644 index 85b1be1694ef89dbb0263e510f82966ca7d4fef7..0000000000000000000000000000000000000000 --- a/data/models/zelk12_mt5-gen5-gemma-2-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MT5-Gen5-gemma-2-9B", - "id": "zelk12/MT5-Gen5-gemma-2-9B", - "developer": "zelk12", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/zelk12_MT5-Gen5-gemma-2-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7947 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6112 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2258 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3482 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4191 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4329 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zelk12_mt5-max-merge_02012025163610-gemma-2-9b.json b/data/models/zelk12_mt5-max-merge_02012025163610-gemma-2-9b.json deleted file mode 100644 index d1d03dbcb883656541e423850b42750c4f3a9f0c..0000000000000000000000000000000000000000 --- a/data/models/zelk12_mt5-max-merge_02012025163610-gemma-2-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MT5-Max-Merge_02012025163610-gemma-2-9B", - "id": "zelk12/MT5-Max-Merge_02012025163610-gemma-2-9B", - "developer": "zelk12", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/zelk12_MT5-Max-Merge_02012025163610-gemma-2-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1762 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6127 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0982 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3515 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4228 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.439 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zelk12_mtm-merge-gemma-2-9b.json b/data/models/zelk12_mtm-merge-gemma-2-9b.json deleted file mode 100644 index edb96e25a7c32f13421e795a5da6022a16b7a1fe..0000000000000000000000000000000000000000 --- a/data/models/zelk12_mtm-merge-gemma-2-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MTM-Merge-gemma-2-9B", - "id": "zelk12/MTM-Merge-gemma-2-9B", - "developer": "zelk12", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/zelk12_MTM-Merge-gemma-2-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7798 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6133 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2175 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3549 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4268 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4388 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zelk12_mtmame-merge_02012025163610-gemma-2-9b.json b/data/models/zelk12_mtmame-merge_02012025163610-gemma-2-9b.json deleted file mode 100644 index e214963bdc63bd96554520f178a0177654698ca1..0000000000000000000000000000000000000000 --- a/data/models/zelk12_mtmame-merge_02012025163610-gemma-2-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MTMaMe-Merge_02012025163610-gemma-2-9B", - "id": "zelk12/MTMaMe-Merge_02012025163610-gemma-2-9B", - "developer": "zelk12", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/zelk12_MTMaMe-Merge_02012025163610-gemma-2-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1786 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6117 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0959 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3523 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4241 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4382 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zelk12_recoilme-gemma-2-ataraxy-9b-v0.1-t0.25.json b/data/models/zelk12_recoilme-gemma-2-ataraxy-9b-v0.1-t0.25.json deleted file mode 100644 index 254e9376408c84c2a28a3424af5d86faf1d8b893..0000000000000000000000000000000000000000 --- a/data/models/zelk12_recoilme-gemma-2-ataraxy-9b-v0.1-t0.25.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "recoilme-gemma-2-Ataraxy-9B-v0.1-t0.25", - "id": "zelk12/recoilme-gemma-2-Ataraxy-9B-v0.1-t0.25", - "developer": "zelk12", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/zelk12_recoilme-gemma-2-Ataraxy-9B-v0.1-t0.25/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7707 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6075 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2145 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3431 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4323 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.44 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zelk12_recoilme-gemma-2-ataraxy-9b-v0.1-t0.75.json b/data/models/zelk12_recoilme-gemma-2-ataraxy-9b-v0.1-t0.75.json deleted file mode 100644 index 309760f260bc61c999b9b484e89ef0e3cadf034b..0000000000000000000000000000000000000000 --- a/data/models/zelk12_recoilme-gemma-2-ataraxy-9b-v0.1-t0.75.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "recoilme-gemma-2-Ataraxy-9B-v0.1-t0.75", - "id": "zelk12/recoilme-gemma-2-Ataraxy-9B-v0.1-t0.75", - "developer": "zelk12", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/zelk12_recoilme-gemma-2-Ataraxy-9B-v0.1-t0.75/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7208 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5995 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2017 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3498 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3951 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4141 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zelk12_recoilme-gemma-2-ataraxy-9b-v0.1.json b/data/models/zelk12_recoilme-gemma-2-ataraxy-9b-v0.1.json deleted file mode 100644 index 8806cea4282e590f4b48704cc6f86ef6deddda19..0000000000000000000000000000000000000000 --- a/data/models/zelk12_recoilme-gemma-2-ataraxy-9b-v0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "recoilme-gemma-2-Ataraxy-9B-v0.1", - "id": "zelk12/recoilme-gemma-2-Ataraxy-9B-v0.1", - "developer": "zelk12", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/zelk12_recoilme-gemma-2-Ataraxy-9B-v0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7649 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6075 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2281 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3498 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4136 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4321 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zelk12_recoilme-gemma-2-ataraxy-9b-v0.2.json b/data/models/zelk12_recoilme-gemma-2-ataraxy-9b-v0.2.json deleted file mode 100644 index 7ac4a36255ddf8f54693de9b25b7aa83b14b4a93..0000000000000000000000000000000000000000 --- a/data/models/zelk12_recoilme-gemma-2-ataraxy-9b-v0.2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "recoilme-gemma-2-Ataraxy-9B-v0.2", - "id": "zelk12/recoilme-gemma-2-Ataraxy-9B-v0.2", - "developer": "zelk12", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/zelk12_recoilme-gemma-2-Ataraxy-9B-v0.2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.76 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6066 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2228 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3482 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.411 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4323 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zelk12_recoilme-gemma-2-gutenberg-doppel-9b-v0.1.json b/data/models/zelk12_recoilme-gemma-2-gutenberg-doppel-9b-v0.1.json deleted file mode 100644 index 06d7407f8cce8e4446fc23ebb0e00bf504c4bba9..0000000000000000000000000000000000000000 --- a/data/models/zelk12_recoilme-gemma-2-gutenberg-doppel-9b-v0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "recoilme-gemma-2-Gutenberg-Doppel-9B-v0.1", - "id": "zelk12/recoilme-gemma-2-Gutenberg-Doppel-9B-v0.1", - "developer": "zelk12", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/zelk12_recoilme-gemma-2-Gutenberg-Doppel-9B-v0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7615 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6099 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.21 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3414 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.431 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4315 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zelk12_recoilme-gemma-2-ifable-9b-v0.1.json b/data/models/zelk12_recoilme-gemma-2-ifable-9b-v0.1.json deleted file mode 100644 index cac2a1e60ba5da734a196c91e320e4119e1ba94a..0000000000000000000000000000000000000000 --- a/data/models/zelk12_recoilme-gemma-2-ifable-9b-v0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "recoilme-gemma-2-Ifable-9B-v0.1", - "id": "zelk12/recoilme-gemma-2-Ifable-9B-v0.1", - "developer": "zelk12", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/zelk12_recoilme-gemma-2-Ifable-9B-v0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7944 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6064 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2205 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3515 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4202 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4323 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zelk12_recoilme-gemma-2-psy10k-mental_healt-9b-v0.1.json b/data/models/zelk12_recoilme-gemma-2-psy10k-mental_healt-9b-v0.1.json deleted file mode 100644 index 16c5afdf5d2d0d36a5f26c79ae75f7f6c517381d..0000000000000000000000000000000000000000 --- a/data/models/zelk12_recoilme-gemma-2-psy10k-mental_healt-9b-v0.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "recoilme-gemma-2-psy10k-mental_healt-9B-v0.1", - "id": "zelk12/recoilme-gemma-2-psy10k-mental_healt-9B-v0.1", - "developer": "zelk12", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/zelk12_recoilme-gemma-2-psy10k-mental_healt-9B-v0.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7445 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5978 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1888 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.344 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4295 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4181 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zelk12_rv0.4dmv1t0.25-gemma-2-9b.json b/data/models/zelk12_rv0.4dmv1t0.25-gemma-2-9b.json deleted file mode 100644 index e77741f5d9224b30903bd7660723d761599fd550..0000000000000000000000000000000000000000 --- a/data/models/zelk12_rv0.4dmv1t0.25-gemma-2-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Rv0.4DMv1t0.25-gemma-2-9B", - "id": "zelk12/Rv0.4DMv1t0.25-gemma-2-9B", - "developer": "zelk12", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/zelk12_Rv0.4DMv1t0.25-gemma-2-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7497 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.607 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2258 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3456 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4309 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4401 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zelk12_rv0.4dmv1t0.25tt0.25-gemma-2-9b.json b/data/models/zelk12_rv0.4dmv1t0.25tt0.25-gemma-2-9b.json deleted file mode 100644 index dde2326629beb269acf59f1d9a3f68024e5f314c..0000000000000000000000000000000000000000 --- a/data/models/zelk12_rv0.4dmv1t0.25tt0.25-gemma-2-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Rv0.4DMv1t0.25Tt0.25-gemma-2-9B", - "id": "zelk12/Rv0.4DMv1t0.25Tt0.25-gemma-2-9B", - "developer": "zelk12", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/zelk12_Rv0.4DMv1t0.25Tt0.25-gemma-2-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7646 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6098 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2069 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3423 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4283 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4347 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zelk12_rv0.4mt4g2-gemma-2-9b.json b/data/models/zelk12_rv0.4mt4g2-gemma-2-9b.json deleted file mode 100644 index b577cfff46265e09c12618f2f96c16bca3d83a29..0000000000000000000000000000000000000000 --- a/data/models/zelk12_rv0.4mt4g2-gemma-2-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Rv0.4MT4g2-gemma-2-9B", - "id": "zelk12/Rv0.4MT4g2-gemma-2-9B", - "developer": "zelk12", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/zelk12_Rv0.4MT4g2-gemma-2-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.732 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6041 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1949 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3532 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4231 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4417 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zelk12_t31122024203920-gemma-2-9b.json b/data/models/zelk12_t31122024203920-gemma-2-9b.json deleted file mode 100644 index f9284450db55eace2e28fb1623a601a3873ed961..0000000000000000000000000000000000000000 --- a/data/models/zelk12_t31122024203920-gemma-2-9b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "T31122024203920-gemma-2-9B", - "id": "zelk12/T31122024203920-gemma-2-9B", - "developer": "zelk12", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "10.159" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/zelk12_T31122024203920-gemma-2-9B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7676 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6096 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2054 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3507 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4322 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4373 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zelk12_test01012025155054.json b/data/models/zelk12_test01012025155054.json deleted file mode 100644 index da44cb32f64a250c15662efcb0c0eb5b5a868404..0000000000000000000000000000000000000000 --- a/data/models/zelk12_test01012025155054.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Test01012025155054", - "id": "zelk12/Test01012025155054", - "developer": "zelk12", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "3.817" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/zelk12_Test01012025155054/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1555 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.283 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2416 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.367 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.109 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zelk12_test01012025155054t0.5_gemma-2.json b/data/models/zelk12_test01012025155054t0.5_gemma-2.json deleted file mode 100644 index 05852cc8b46463eb634be671ebff232892acf604..0000000000000000000000000000000000000000 --- a/data/models/zelk12_test01012025155054t0.5_gemma-2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Test01012025155054t0.5_gemma-2", - "id": "zelk12/Test01012025155054t0.5_gemma-2", - "developer": "zelk12", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Gemma2ForCausalLM", - "params_billions": "3.817" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/zelk12_Test01012025155054t0.5_gemma-2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1555 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.283 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2416 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.367 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.109 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zeroxclem_l3-aspire-heart-matrix-8b.json b/data/models/zeroxclem_l3-aspire-heart-matrix-8b.json deleted file mode 100644 index ff6c466236c15e956c9ff607ccb7830a8037fd4c..0000000000000000000000000000000000000000 --- a/data/models/zeroxclem_l3-aspire-heart-matrix-8b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "L3-Aspire-Heart-Matrix-8B", - "id": "ZeroXClem/L3-Aspire-Heart-Matrix-8B", - "developer": "ZeroXClem", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ZeroXClem_L3-Aspire-Heart-Matrix-8B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4834 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5384 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1828 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3247 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4187 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3785 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zeroxclem_llama-3.1-8b-athenasky-megamix.json b/data/models/zeroxclem_llama-3.1-8b-athenasky-megamix.json deleted file mode 100644 index d52c418bdb60b0b13d473e6a3b75a7b61d3f6c84..0000000000000000000000000000000000000000 --- a/data/models/zeroxclem_llama-3.1-8b-athenasky-megamix.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.1-8B-AthenaSky-MegaMix", - "id": "ZeroXClem/Llama-3.1-8B-AthenaSky-MegaMix", - "developer": "ZeroXClem", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ZeroXClem_Llama-3.1-8B-AthenaSky-MegaMix/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6301 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5163 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2795 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2777 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3538 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3504 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zeroxclem_llama-3.1-8b-rainbowlight-etherealmix.json b/data/models/zeroxclem_llama-3.1-8b-rainbowlight-etherealmix.json deleted file mode 100644 index 283c7ac64249c02cfef402426c032f2149919d5f..0000000000000000000000000000000000000000 --- a/data/models/zeroxclem_llama-3.1-8b-rainbowlight-etherealmix.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.1-8B-RainbowLight-EtherealMix", - "id": "ZeroXClem/Llama-3.1-8B-RainbowLight-EtherealMix", - "developer": "ZeroXClem", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ZeroXClem_Llama-3.1-8B-RainbowLight-EtherealMix/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4973 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5155 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1216 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2869 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3947 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.363 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zeroxclem_llama-3.1-8b-specialtitanfusion.json b/data/models/zeroxclem_llama-3.1-8b-specialtitanfusion.json deleted file mode 100644 index cc043bed367d4ec0ccdfdfc563be50ee6c725905..0000000000000000000000000000000000000000 --- a/data/models/zeroxclem_llama-3.1-8b-specialtitanfusion.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.1-8B-SpecialTitanFusion", - "id": "ZeroXClem/Llama-3.1-8B-SpecialTitanFusion", - "developer": "ZeroXClem", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ZeroXClem_Llama-3.1-8B-SpecialTitanFusion/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7402 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5439 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2334 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2995 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3874 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3621 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zeroxclem_llama-3.1-8b-supernova-etherealhermes.json b/data/models/zeroxclem_llama-3.1-8b-supernova-etherealhermes.json deleted file mode 100644 index 02ff156116d99abd7c6a872085768bbe5d886863..0000000000000000000000000000000000000000 --- a/data/models/zeroxclem_llama-3.1-8b-supernova-etherealhermes.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.1-8B-SuperNova-EtherealHermes", - "id": "ZeroXClem/Llama-3.1-8B-SuperNova-EtherealHermes", - "developer": "ZeroXClem", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ZeroXClem_Llama-3.1-8B-SuperNova-EtherealHermes/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7339 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5244 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.1745 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2928 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4066 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3745 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zeroxclem_llama-3.1-8b-supertulu-lexinova.json b/data/models/zeroxclem_llama-3.1-8b-supertulu-lexinova.json deleted file mode 100644 index e677ff68d3dc417794ab08a9d66a106ac0820971..0000000000000000000000000000000000000000 --- a/data/models/zeroxclem_llama-3.1-8b-supertulu-lexinova.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Llama-3.1-8B-SuperTulu-LexiNova", - "id": "ZeroXClem/Llama-3.1-8B-SuperTulu-LexiNova", - "developer": "ZeroXClem", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ZeroXClem_Llama-3.1-8B-SuperTulu-LexiNova/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4165 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5079 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.253 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2861 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3971 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3368 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zeroxclem_qwen-2.5-aether-slerpfusion-7b.json b/data/models/zeroxclem_qwen-2.5-aether-slerpfusion-7b.json deleted file mode 100644 index 7866678ad10ac249e1aac2f1798e89f30ab2c4cd..0000000000000000000000000000000000000000 --- a/data/models/zeroxclem_qwen-2.5-aether-slerpfusion-7b.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen-2.5-Aether-SlerpFusion-7B", - "id": "ZeroXClem/Qwen-2.5-Aether-SlerpFusion-7B", - "developer": "ZeroXClem", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ZeroXClem_Qwen-2.5-Aether-SlerpFusion-7B/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6262 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5462 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2734 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2987 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4178 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4327 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zeroxclem_qwen2.5-7b-celestialharmony-1m.json b/data/models/zeroxclem_qwen2.5-7b-celestialharmony-1m.json deleted file mode 100644 index 4dd2a2d92fd9aef28c22f0339a3cff9b8ff23317..0000000000000000000000000000000000000000 --- a/data/models/zeroxclem_qwen2.5-7b-celestialharmony-1m.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-7B-CelestialHarmony-1M", - "id": "ZeroXClem/Qwen2.5-7B-CelestialHarmony-1M", - "developer": "ZeroXClem", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.613" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ZeroXClem_Qwen2.5-7B-CelestialHarmony-1M/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5944 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5431 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3474 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3188 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4595 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4387 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zeroxclem_qwen2.5-7b-homeranvita-nerdmix.json b/data/models/zeroxclem_qwen2.5-7b-homeranvita-nerdmix.json deleted file mode 100644 index 675781a0f50d747428cbbe76685dbd0f4a28930b..0000000000000000000000000000000000000000 --- a/data/models/zeroxclem_qwen2.5-7b-homeranvita-nerdmix.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-7B-HomerAnvita-NerdMix", - "id": "ZeroXClem/Qwen2.5-7B-HomerAnvita-NerdMix", - "developer": "ZeroXClem", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ZeroXClem_Qwen2.5-7B-HomerAnvita-NerdMix/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7708 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5541 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3837 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3196 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4391 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4432 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zeroxclem_qwen2.5-7b-homercreative-mix.json b/data/models/zeroxclem_qwen2.5-7b-homercreative-mix.json deleted file mode 100644 index b9386154af216e4760bbcb5c0141661c7940719b..0000000000000000000000000000000000000000 --- a/data/models/zeroxclem_qwen2.5-7b-homercreative-mix.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-7B-HomerCreative-Mix", - "id": "ZeroXClem/Qwen2.5-7B-HomerCreative-Mix", - "developer": "ZeroXClem", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ZeroXClem_Qwen2.5-7B-HomerCreative-Mix/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7835 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5548 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3565 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2995 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.435 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4447 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zeroxclem_qwen2.5-7b-qandora-cysec.json b/data/models/zeroxclem_qwen2.5-7b-qandora-cysec.json deleted file mode 100644 index 304dfb029350b4269be9c21c21d515994b2eaeb8..0000000000000000000000000000000000000000 --- a/data/models/zeroxclem_qwen2.5-7b-qandora-cysec.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-7B-Qandora-CySec", - "id": "ZeroXClem/Qwen2.5-7B-Qandora-CySec", - "developer": "ZeroXClem", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "7.616" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ZeroXClem_Qwen2.5-7B-Qandora-CySec/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6773 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.549 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2931 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3003 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4286 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4485 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zetasepic_qwen2.5-32b-instruct-abliterated-v2.json b/data/models/zetasepic_qwen2.5-32b-instruct-abliterated-v2.json deleted file mode 100644 index 245194188ed2cf8a1bada730cf5fa06cb44b9ff6..0000000000000000000000000000000000000000 --- a/data/models/zetasepic_qwen2.5-32b-instruct-abliterated-v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-32B-Instruct-abliterated-v2", - "id": "zetasepic/Qwen2.5-32B-Instruct-abliterated-v2", - "developer": "zetasepic", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "32.764" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/zetasepic_Qwen2.5-32B-Instruct-abliterated-v2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8334 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6934 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5952 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3674 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4354 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5622 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zetasepic_qwen2.5-72b-instruct-abliterated.json b/data/models/zetasepic_qwen2.5-72b-instruct-abliterated.json deleted file mode 100644 index bc5f624df0782fdb593182dfa56bc980090a9375..0000000000000000000000000000000000000000 --- a/data/models/zetasepic_qwen2.5-72b-instruct-abliterated.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "Qwen2.5-72B-Instruct-abliterated", - "id": "zetasepic/Qwen2.5-72B-Instruct-abliterated", - "developer": "zetasepic", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "Qwen2ForCausalLM", - "params_billions": "72.706" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/zetasepic_Qwen2.5-72B-Instruct-abliterated/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7153 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7152 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5242 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4069 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4719 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5872 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zeuslabs_l3-aethora-15b-v2.json b/data/models/zeuslabs_l3-aethora-15b-v2.json deleted file mode 100644 index e95964c339b62cd4e5210b7ee6e858d2ba5ce299..0000000000000000000000000000000000000000 --- a/data/models/zeuslabs_l3-aethora-15b-v2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "L3-Aethora-15B-V2", - "id": "ZeusLabs/L3-Aethora-15B-V2", - "developer": "ZeusLabs", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "15.01" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ZeusLabs_L3-Aethora-15B-V2/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.7208 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5011 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0808 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2878 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3871 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.35 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zhangshenao_selm-llama-3-8b-instruct-iter-3.json b/data/models/zhangshenao_selm-llama-3-8b-instruct-iter-3.json deleted file mode 100644 index 8a528ac91286c86b2faad92814cde053518bba93..0000000000000000000000000000000000000000 --- a/data/models/zhangshenao_selm-llama-3-8b-instruct-iter-3.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "SELM-Llama-3-8B-Instruct-iter-3", - "id": "ZhangShenao/SELM-Llama-3-8B-Instruct-iter-3", - "developer": "ZhangShenao", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "LlamaForCausalLM", - "params_billions": "8.03" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ZhangShenao_SELM-Llama-3-8B-Instruct-iter-3/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.6903 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5046 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0861 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2584 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3845 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3783 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zhengr_mixtao-7bx2-moe-v8.1.json b/data/models/zhengr_mixtao-7bx2-moe-v8.1.json deleted file mode 100644 index bce89d880f5725dd19ca46dc822ba401d441b4eb..0000000000000000000000000000000000000000 --- a/data/models/zhengr_mixtao-7bx2-moe-v8.1.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "MixTAO-7Bx2-MoE-v8.1", - "id": "zhengr/MixTAO-7Bx2-MoE-v8.1", - "developer": "zhengr", - "inference_platform": "unknown", - "additional_details": { - "precision": "bfloat16", - "architecture": "MixtralForCausalLM", - "params_billions": "12.879" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/zhengr_MixTAO-7Bx2-MoE-v8.1/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4188 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4202 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0604 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2987 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3976 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2847 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zhipu-ai_glm-130b.json b/data/models/zhipu-ai_glm-130b.json deleted file mode 100644 index 9eff4b597c64c983bee9f8a2df7f7fa869f90e0e..0000000000000000000000000000000000000000 --- a/data/models/zhipu-ai_glm-130b.json +++ /dev/null @@ -1,674 +0,0 @@ -{ - "model_info": { - "name": "GLM 130B", - "id": "zhipu-ai/GLM-130B", - "developer": "zhipu-ai", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "helm_classic/zhipu-ai_GLM-130B/1774096308.339228", - "retrieved_timestamp": "1774096308.339228", - "source_metadata": { - "source_name": "helm_classic", - "source_type": "documentation", - "source_organization_name": "crfm", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "helm", - "version": "unknown" - }, - "benchmark": "helm_classic", - "evaluation_results": [ - { - "evaluation_name": "Mean win rate", - "source_data": { - "dataset_name": "helm_classic", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "How many models this model outperform on average (over columns).", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.512, - "details": { - "description": "", - "tab": "Accuracy", - "Mean win rate - Calibration": "{\"description\": \"\", \"tab\": \"Calibration\", \"score\": \"0.6523126734505088\"}", - "Mean win rate - Robustness": "{\"description\": \"\", \"tab\": \"Robustness\", \"score\": \"0.6465501165501165\"}", - "Mean win rate - Fairness": "{\"description\": \"\", \"tab\": \"Fairness\", \"score\": \"0.5133566433566433\"}", - "Mean win rate - Efficiency": "{\"description\": \"\", \"tab\": \"Efficiency\", \"score\": \"0.1511111111111111\"}", - "Mean win rate - General information": "{\"description\": \"\", \"tab\": \"General information\", \"score\": \"\"}", - "Mean win rate - Bias": "{\"description\": \"\", \"tab\": \"Bias\", \"score\": \"0.45074793034678545\"}", - "Mean win rate - Toxicity": "{\"description\": \"\", \"tab\": \"Toxicity\", \"score\": \"0.3347137430470764\"}", - "Mean win rate - Summarization metrics": "{\"description\": \"\", \"tab\": \"Summarization metrics\", \"score\": \"0.4714285714285714\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MMLU", - "source_data": { - "dataset_name": "MMLU", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on MMLU", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.344, - "details": { - "description": "min=0.23, mean=0.344, max=0.47, sum=5.16 (15)", - "tab": "Accuracy", - "MMLU - ECE (10-bin)": "{\"description\": \"min=0.075, mean=0.128, max=0.196, sum=1.914 (15)\", \"tab\": \"Calibration\", \"score\": \"0.12760096192658882\"}", - "MMLU - EM (Robustness)": "{\"description\": \"min=0.17, mean=0.32, max=0.44, sum=4.806 (15)\", \"tab\": \"Robustness\", \"score\": \"0.3203859649122807\"}", - "MMLU - EM (Fairness)": "{\"description\": \"min=0.22, mean=0.315, max=0.43, sum=4.723 (15)\", \"tab\": \"Fairness\", \"score\": \"0.3148771929824561\"}", - "MMLU - Denoised inference time (s)": "{\"description\": \"min=0.194, mean=0.335, max=0.546, sum=5.029 (15)\", \"tab\": \"Efficiency\", \"score\": \"0.33523606010994367\"}", - "MMLU - # eval": "{\"description\": \"min=100, mean=102.8, max=114, sum=1542 (15)\", \"tab\": \"General information\", \"score\": \"102.8\"}", - "MMLU - # train": "{\"description\": \"min=5, mean=5, max=5, sum=75 (15)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "MMLU - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (15)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "MMLU - # prompt tokens": "{\"description\": \"min=354.52, mean=460.637, max=611.877, sum=6909.562 (15)\", \"tab\": \"General information\", \"score\": \"460.63743859649117\"}", - "MMLU - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=15 (15)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "MMLU - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=45 (15)\", \"tab\": \"General information\", \"score\": \"3.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "BoolQ", - "source_data": { - "dataset_name": "BoolQ", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on BoolQ", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.784, - "details": { - "description": "min=0.729, mean=0.784, max=0.819, sum=2.351 (3)", - "tab": "Accuracy", - "BoolQ - ECE (10-bin)": "{\"description\": \"min=0.111, mean=0.171, max=0.205, sum=0.513 (3)\", \"tab\": \"Calibration\", \"score\": \"0.1710477879835662\"}", - "BoolQ - EM (Robustness)": "{\"description\": \"min=0.68, mean=0.728, max=0.758, sum=2.183 (3)\", \"tab\": \"Robustness\", \"score\": \"0.7276666666666668\"}", - "BoolQ - EM (Fairness)": "{\"description\": \"min=0.625, mean=0.69, max=0.722, sum=2.069 (3)\", \"tab\": \"Fairness\", \"score\": \"0.6896666666666667\"}", - "BoolQ - Denoised inference time (s)": "{\"description\": \"min=0.942, mean=1.191, max=1.332, sum=3.574 (3)\", \"tab\": \"Efficiency\", \"score\": \"1.1913305165274586\"}", - "BoolQ - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "BoolQ - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "BoolQ - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "BoolQ - # prompt tokens": "{\"description\": \"min=679.091, mean=931.424, max=1276.091, sum=2794.273 (3)\", \"tab\": \"General information\", \"score\": \"931.4243333333333\"}", - "BoolQ - # output tokens": "{\"description\": \"min=2, mean=2, max=2, sum=6 (3)\", \"tab\": \"General information\", \"score\": \"2.0\"}", - "BoolQ - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "BoolQ - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "BoolQ - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NarrativeQA", - "source_data": { - "dataset_name": "NarrativeQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NarrativeQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.706, - "details": { - "description": "min=0.655, mean=0.706, max=0.736, sum=2.118 (3)", - "tab": "Accuracy", - "NarrativeQA - ECE (10-bin)": "{\"description\": \"min=0.027, mean=0.037, max=0.058, sum=0.112 (3)\", \"tab\": \"Calibration\", \"score\": \"0.03732324115716399\"}", - "NarrativeQA - F1 (Robustness)": "{\"description\": \"min=0.531, mean=0.629, max=0.682, sum=1.888 (3)\", \"tab\": \"Robustness\", \"score\": \"0.6293880948208791\"}", - "NarrativeQA - F1 (Fairness)": "{\"description\": \"min=0.55, mean=0.615, max=0.656, sum=1.846 (3)\", \"tab\": \"Fairness\", \"score\": \"0.6154230898629193\"}", - "NarrativeQA - Denoised inference time (s)": "{\"description\": \"min=1.78, mean=2.315, max=3.197, sum=6.946 (3)\", \"tab\": \"Efficiency\", \"score\": \"2.3151894005635367\"}", - "NarrativeQA - # eval": "{\"description\": \"min=355, mean=355, max=355, sum=1065 (3)\", \"tab\": \"General information\", \"score\": \"355.0\"}", - "NarrativeQA - # train": "{\"description\": \"min=1.101, mean=1.675, max=2.11, sum=5.025 (3)\", \"tab\": \"General information\", \"score\": \"1.6751173708920186\"}", - "NarrativeQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NarrativeQA - # prompt tokens": "{\"description\": \"min=1597.372, mean=1658.811, max=1711.876, sum=4976.434 (3)\", \"tab\": \"General information\", \"score\": \"1658.8112676056337\"}", - "NarrativeQA - # output tokens": "{\"description\": \"min=6.008, mean=9.939, max=17.439, sum=29.817 (3)\", \"tab\": \"General information\", \"score\": \"9.938967136150234\"}", - "NarrativeQA - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NarrativeQA - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NarrativeQA - Stereotypes (gender)": "{\"description\": \"min=0.365, mean=0.372, max=0.375, sum=1.115 (3)\", \"tab\": \"Bias\", \"score\": \"0.3717948717948718\"}", - "NarrativeQA - Representation (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=0.667 (1)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666667\"}", - "NarrativeQA - Representation (gender)": "{\"description\": \"min=0.168, mean=0.19, max=0.215, sum=0.569 (3)\", \"tab\": \"Bias\", \"score\": \"0.1896318370894642\"}", - "NarrativeQA - Toxic fraction": "{\"description\": \"min=0.011, mean=0.012, max=0.014, sum=0.037 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.012206572769953052\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "NaturalQuestions (open-book)", - "source_data": { - "dataset_name": "NaturalQuestions (open-book)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on NaturalQuestions (open-book)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.642, - "details": { - "description": "min=0.639, mean=0.642, max=0.649, sum=1.927 (3)", - "tab": "Accuracy", - "NaturalQuestions (closed-book) - ECE (10-bin)": "{\"description\": \"min=0.02, mean=0.022, max=0.023, sum=0.065 (3)\", \"tab\": \"Calibration\", \"score\": \"0.021760896948719733\"}", - "NaturalQuestions (open-book) - ECE (10-bin)": "{\"description\": \"min=0.071, mean=0.076, max=0.082, sum=0.228 (3)\", \"tab\": \"Calibration\", \"score\": \"0.07592608066404687\"}", - "NaturalQuestions (closed-book) - F1 (Robustness)": "{\"description\": \"min=0.11, mean=0.117, max=0.122, sum=0.35 (3)\", \"tab\": \"Robustness\", \"score\": \"0.11665134142344884\"}", - "NaturalQuestions (open-book) - F1 (Robustness)": "{\"description\": \"min=0.592, mean=0.6, max=0.608, sum=1.8 (3)\", \"tab\": \"Robustness\", \"score\": \"0.5998399895408899\"}", - "NaturalQuestions (closed-book) - F1 (Fairness)": "{\"description\": \"min=0.112, mean=0.12, max=0.124, sum=0.361 (3)\", \"tab\": \"Fairness\", \"score\": \"0.12026039507733897\"}", - "NaturalQuestions (open-book) - F1 (Fairness)": "{\"description\": \"min=0.592, mean=0.597, max=0.603, sum=1.79 (3)\", \"tab\": \"Fairness\", \"score\": \"0.5967933879081116\"}", - "NaturalQuestions (closed-book) - Denoised inference time (s)": "{\"description\": \"min=0.822, mean=0.953, max=1.045, sum=2.859 (3)\", \"tab\": \"Efficiency\", \"score\": \"0.9528701016867446\"}", - "NaturalQuestions (open-book) - Denoised inference time (s)": "{\"description\": \"min=2.251, mean=2.369, max=2.58, sum=7.108 (3)\", \"tab\": \"Efficiency\", \"score\": \"2.3693331199589207\"}", - "NaturalQuestions (closed-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (closed-book) - # train": "{\"description\": \"min=5, mean=5, max=5, sum=15 (3)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "NaturalQuestions (closed-book) - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "NaturalQuestions (closed-book) - # prompt tokens": "{\"description\": \"min=121.658, mean=122.991, max=125.658, sum=368.974 (3)\", \"tab\": \"General information\", \"score\": \"122.99133333333333\"}", - "NaturalQuestions (closed-book) - # output tokens": "{\"description\": \"min=6.22, mean=6.707, max=7.262, sum=20.12 (3)\", \"tab\": \"General information\", \"score\": \"6.706666666666667\"}", - "NaturalQuestions (closed-book) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NaturalQuestions (open-book) - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "NaturalQuestions (open-book) - # train": "{\"description\": \"min=4.505, mean=4.631, max=4.705, sum=13.892 (3)\", \"tab\": \"General information\", \"score\": \"4.630666666666667\"}", - "NaturalQuestions (open-book) - truncated": "{\"description\": \"min=0.042, mean=0.047, max=0.056, sum=0.14 (3)\", \"tab\": \"General information\", \"score\": \"0.04666666666666667\"}", - "NaturalQuestions (open-book) - # prompt tokens": "{\"description\": \"min=1340.319, mean=1502.677, max=1625.084, sum=4508.03 (3)\", \"tab\": \"General information\", \"score\": \"1502.676666666667\"}", - "NaturalQuestions (open-book) - # output tokens": "{\"description\": \"min=19.342, mean=21.064, max=23.914, sum=63.193 (3)\", \"tab\": \"General information\", \"score\": \"21.064333333333334\"}", - "NaturalQuestions (open-book) - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "NaturalQuestions (closed-book) - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "NaturalQuestions (closed-book) - Stereotypes (gender)": "{\"description\": \"min=0.5, mean=0.5, max=0.5, sum=0.5 (1)\", \"tab\": \"Bias\", \"score\": \"0.5\"}", - "NaturalQuestions (closed-book) - Representation (race)": "{\"description\": \"min=0.121, mean=0.269, max=0.393, sum=0.807 (3)\", \"tab\": \"Bias\", \"score\": \"0.2689924681892553\"}", - "NaturalQuestions (closed-book) - Representation (gender)": "{\"description\": \"min=0.038, mean=0.059, max=0.083, sum=0.177 (3)\", \"tab\": \"Bias\", \"score\": \"0.05911680911680913\"}", - "NaturalQuestions (open-book) - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=2 (3)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666666\"}", - "NaturalQuestions (open-book) - Stereotypes (gender)": "{\"description\": \"min=0.5, mean=0.5, max=0.5, sum=1.5 (3)\", \"tab\": \"Bias\", \"score\": \"0.5\"}", - "NaturalQuestions (open-book) - Representation (race)": "{\"description\": \"min=0.571, mean=0.585, max=0.598, sum=1.754 (3)\", \"tab\": \"Bias\", \"score\": \"0.584615044473471\"}", - "NaturalQuestions (open-book) - Representation (gender)": "{\"description\": \"min=0.068, mean=0.073, max=0.079, sum=0.22 (3)\", \"tab\": \"Bias\", \"score\": \"0.07328275644065117\"}", - "NaturalQuestions (closed-book) - Toxic fraction": "{\"description\": \"min=0, mean=0.001, max=0.002, sum=0.003 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.001\"}", - "NaturalQuestions (open-book) - Toxic fraction": "{\"description\": \"min=0.001, mean=0.002, max=0.002, sum=0.005 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.0016666666666666668\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "QuAC", - "source_data": { - "dataset_name": "QuAC", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "F1 on QuAC", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.272, - "details": { - "description": "min=0.23, mean=0.272, max=0.297, sum=0.815 (3)", - "tab": "Accuracy", - "QuAC - ECE (10-bin)": "{\"description\": \"min=0.012, mean=0.027, max=0.043, sum=0.082 (3)\", \"tab\": \"Calibration\", \"score\": \"0.02731272826999052\"}", - "QuAC - F1 (Robustness)": "{\"description\": \"min=0.178, mean=0.193, max=0.202, sum=0.579 (3)\", \"tab\": \"Robustness\", \"score\": \"0.19293634470384977\"}", - "QuAC - F1 (Fairness)": "{\"description\": \"min=0.173, mean=0.205, max=0.225, sum=0.616 (3)\", \"tab\": \"Fairness\", \"score\": \"0.20535008777735797\"}", - "QuAC - Denoised inference time (s)": "{\"description\": \"min=4.186, mean=4.219, max=4.235, sum=12.656 (3)\", \"tab\": \"Efficiency\", \"score\": \"4.218568385192325\"}", - "QuAC - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "QuAC - # train": "{\"description\": \"min=0.823, mean=0.874, max=0.929, sum=2.622 (3)\", \"tab\": \"General information\", \"score\": \"0.874\"}", - "QuAC - truncated": "{\"description\": \"min=0.094, mean=0.134, max=0.177, sum=0.401 (3)\", \"tab\": \"General information\", \"score\": \"0.13366666666666668\"}", - "QuAC - # prompt tokens": "{\"description\": \"min=1621.422, mean=1651.972, max=1668.212, sum=4955.915 (3)\", \"tab\": \"General information\", \"score\": \"1651.9716666666666\"}", - "QuAC - # output tokens": "{\"description\": \"min=65.116, mean=73.565, max=88.524, sum=220.696 (3)\", \"tab\": \"General information\", \"score\": \"73.56533333333333\"}", - "QuAC - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "QuAC - Stereotypes (race)": "{\"description\": \"min=0.604, mean=0.62, max=0.642, sum=1.86 (3)\", \"tab\": \"Bias\", \"score\": \"0.6201234839116704\"}", - "QuAC - Stereotypes (gender)": "{\"description\": \"min=0.411, mean=0.431, max=0.451, sum=1.294 (3)\", \"tab\": \"Bias\", \"score\": \"0.43137624831417937\"}", - "QuAC - Representation (race)": "{\"description\": \"min=0.372, mean=0.408, max=0.45, sum=1.224 (3)\", \"tab\": \"Bias\", \"score\": \"0.40815960651383004\"}", - "QuAC - Representation (gender)": "{\"description\": \"min=0.259, mean=0.268, max=0.282, sum=0.803 (3)\", \"tab\": \"Bias\", \"score\": \"0.2675064821442643\"}", - "QuAC - Toxic fraction": "{\"description\": \"min=0, mean=0.001, max=0.001, sum=0.002 (3)\", \"tab\": \"Toxicity\", \"score\": \"0.0006666666666666666\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "HellaSwag", - "source_data": { - "dataset_name": "HellaSwag", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on HellaSwag", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "HellaSwag - ECE (10-bin)": "{\"description\": \"No matching runs\", \"tab\": \"Calibration\", \"score\": \"\"}", - "HellaSwag - EM (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "HellaSwag - EM (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "HellaSwag - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "HellaSwag - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "HellaSwag - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "OpenbookQA", - "source_data": { - "dataset_name": "OpenbookQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on OpenbookQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "OpenbookQA - ECE (10-bin)": "{\"description\": \"No matching runs\", \"tab\": \"Calibration\", \"score\": \"\"}", - "OpenbookQA - EM (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "OpenbookQA - EM (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "OpenbookQA - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "OpenbookQA - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "OpenbookQA - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "TruthfulQA", - "source_data": { - "dataset_name": "TruthfulQA", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on TruthfulQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.218, - "details": { - "description": "min=0.185, mean=0.218, max=0.232, sum=0.873 (4)", - "tab": "Accuracy", - "TruthfulQA - ECE (10-bin)": "{\"description\": \"min=0.04, mean=0.088, max=0.12, sum=0.351 (4)\", \"tab\": \"Calibration\", \"score\": \"0.08770199071414088\"}", - "TruthfulQA - EM (Robustness)": "{\"description\": \"min=0.147, mean=0.196, max=0.229, sum=0.784 (4)\", \"tab\": \"Robustness\", \"score\": \"0.19610091743119268\"}", - "TruthfulQA - EM (Fairness)": "{\"description\": \"min=0.148, mean=0.192, max=0.229, sum=0.766 (4)\", \"tab\": \"Fairness\", \"score\": \"0.1915137614678899\"}", - "TruthfulQA - Denoised inference time (s)": "{\"description\": \"min=0.069, mean=0.158, max=0.193, sum=0.633 (4)\", \"tab\": \"Efficiency\", \"score\": \"0.15830796687302695\"}", - "TruthfulQA - # eval": "{\"description\": \"min=654, mean=654, max=654, sum=2616 (4)\", \"tab\": \"General information\", \"score\": \"654.0\"}", - "TruthfulQA - # train": "{\"description\": \"min=0, mean=3.75, max=5, sum=15 (4)\", \"tab\": \"General information\", \"score\": \"3.75\"}", - "TruthfulQA - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (4)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "TruthfulQA - # prompt tokens": "{\"description\": \"min=80.786, mean=389.036, max=521.786, sum=1556.144 (4)\", \"tab\": \"General information\", \"score\": \"389.0359327217125\"}", - "TruthfulQA - # output tokens": "{\"description\": \"min=1, mean=1, max=1, sum=4 (4)\", \"tab\": \"General information\", \"score\": \"1.0\"}", - "TruthfulQA - # trials": "{\"description\": \"min=1, mean=2.5, max=3, sum=10 (4)\", \"tab\": \"General information\", \"score\": \"2.5\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "MS MARCO (TREC)", - "source_data": { - "dataset_name": "MS MARCO (TREC)", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "NDCG@10 on MS MARCO (TREC)", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": -1.0, - "details": { - "description": "No matching runs", - "tab": "Accuracy", - "MS MARCO (regular) - RR@10 (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "MS MARCO (TREC) - NDCG@10 (Robustness)": "{\"description\": \"No matching runs\", \"tab\": \"Robustness\", \"score\": \"\"}", - "MS MARCO (regular) - RR@10 (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "MS MARCO (TREC) - NDCG@10 (Fairness)": "{\"description\": \"No matching runs\", \"tab\": \"Fairness\", \"score\": \"\"}", - "MS MARCO (regular) - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MS MARCO (TREC) - Denoised inference time (s)": "{\"description\": \"No matching runs\", \"tab\": \"Efficiency\", \"score\": \"\"}", - "MS MARCO (regular) - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # eval": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # train": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - truncated": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # prompt tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # output tokens": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (TREC) - # trials": "{\"description\": \"No matching runs\", \"tab\": \"General information\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Stereotypes (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (race)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (TREC) - Representation (gender)": "{\"description\": \"No matching runs\", \"tab\": \"Bias\", \"score\": \"\"}", - "MS MARCO (regular) - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}", - "MS MARCO (TREC) - Toxic fraction": "{\"description\": \"No matching runs\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CNN/DailyMail", - "source_data": { - "dataset_name": "CNN/DailyMail", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on CNN/DailyMail", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.154, - "details": { - "description": "min=0.144, mean=0.154, max=0.166, sum=0.926 (6)", - "tab": "Accuracy", - "CNN/DailyMail - Denoised inference time (s)": "{\"description\": \"min=3.427, mean=3.514, max=3.581, sum=21.082 (6)\", \"tab\": \"Efficiency\", \"score\": \"3.5136688752771708\"}", - "CNN/DailyMail - # eval": "{\"description\": \"min=466, mean=466, max=466, sum=2796 (6)\", \"tab\": \"General information\", \"score\": \"466.0\"}", - "CNN/DailyMail - # train": "{\"description\": \"min=5, mean=5, max=5, sum=30 (6)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CNN/DailyMail - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (6)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CNN/DailyMail - # prompt tokens": "{\"description\": \"min=1644.124, mean=1657.124, max=1680.124, sum=9942.747 (6)\", \"tab\": \"General information\", \"score\": \"1657.1244635193134\"}", - "CNN/DailyMail - # output tokens": "{\"description\": \"min=74.479, mean=82.997, max=91.644, sum=497.983 (6)\", \"tab\": \"General information\", \"score\": \"82.99713876967097\"}", - "CNN/DailyMail - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=18 (6)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "CNN/DailyMail - Stereotypes (race)": "{\"description\": \"min=0.601, mean=0.611, max=0.623, sum=3.663 (6)\", \"tab\": \"Bias\", \"score\": \"0.61056496482126\"}", - "CNN/DailyMail - Stereotypes (gender)": "{\"description\": \"min=0.377, mean=0.394, max=0.409, sum=2.367 (6)\", \"tab\": \"Bias\", \"score\": \"0.3944955327838351\"}", - "CNN/DailyMail - Representation (race)": "{\"description\": \"min=0.276, mean=0.29, max=0.305, sum=1.741 (6)\", \"tab\": \"Bias\", \"score\": \"0.2901527051306585\"}", - "CNN/DailyMail - Representation (gender)": "{\"description\": \"min=0.134, mean=0.139, max=0.147, sum=0.831 (6)\", \"tab\": \"Bias\", \"score\": \"0.13850777854837878\"}", - "CNN/DailyMail - Toxic fraction": "{\"description\": \"min=0, mean=0.001, max=0.004, sum=0.009 (6)\", \"tab\": \"Toxicity\", \"score\": \"0.001430615164520744\"}", - "CNN/DailyMail - SummaC": "{\"description\": \"min=0.537, mean=0.566, max=0.591, sum=1.699 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.5663194802454004\"}", - "CNN/DailyMail - QAFactEval": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "CNN/DailyMail - BERTScore (F1)": "{\"description\": \"min=0.266, mean=0.288, max=0.312, sum=0.863 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.287517514648812\"}", - "CNN/DailyMail - Coverage": "{\"description\": \"min=0.96, mean=0.972, max=0.987, sum=5.835 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"0.9724896258431271\"}", - "CNN/DailyMail - Density": "{\"description\": \"min=24.014, mean=30.259, max=37.594, sum=181.554 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"30.259024131398863\"}", - "CNN/DailyMail - Compression": "{\"description\": \"min=7.643, mean=8.687, max=9.754, sum=52.123 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"8.68711944818053\"}", - "CNN/DailyMail - HumanEval-faithfulness": "{\"description\": \"min=0.889, mean=0.963, max=1, sum=5.778 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"0.9629629629629629\"}", - "CNN/DailyMail - HumanEval-relevance": "{\"description\": \"min=3.889, mean=4.167, max=4.5, sum=25 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"4.166666666666667\"}", - "CNN/DailyMail - HumanEval-coherence": "{\"description\": \"min=3.111, mean=3.463, max=3.833, sum=20.778 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"3.4629629629629632\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "XSUM", - "source_data": { - "dataset_name": "XSUM", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "ROUGE-2 on XSUM", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.132, - "details": { - "description": "min=0.131, mean=0.132, max=0.134, sum=0.794 (6)", - "tab": "Accuracy", - "XSUM - Denoised inference time (s)": "{\"description\": \"min=2.516, mean=2.537, max=2.549, sum=15.224 (6)\", \"tab\": \"Efficiency\", \"score\": \"2.537310096660418\"}", - "XSUM - # eval": "{\"description\": \"min=518, mean=518, max=518, sum=3108 (6)\", \"tab\": \"General information\", \"score\": \"518.0\"}", - "XSUM - # train": "{\"description\": \"min=4.994, mean=4.996, max=4.998, sum=29.977 (6)\", \"tab\": \"General information\", \"score\": \"4.9961389961389955\"}", - "XSUM - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (6)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "XSUM - # prompt tokens": "{\"description\": \"min=1516.483, mean=1567.312, max=1610.471, sum=9403.873 (6)\", \"tab\": \"General information\", \"score\": \"1567.3120978120978\"}", - "XSUM - # output tokens": "{\"description\": \"min=25.458, mean=25.737, max=26.021, sum=154.421 (6)\", \"tab\": \"General information\", \"score\": \"25.73680823680824\"}", - "XSUM - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=18 (6)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "XSUM - Stereotypes (race)": "{\"description\": \"min=0.667, mean=0.667, max=0.667, sum=4.0 (6)\", \"tab\": \"Bias\", \"score\": \"0.6666666666666669\"}", - "XSUM - Stereotypes (gender)": "{\"description\": \"min=0.399, mean=0.447, max=0.477, sum=2.684 (6)\", \"tab\": \"Bias\", \"score\": \"0.4473352072310406\"}", - "XSUM - Representation (race)": "{\"description\": \"min=0.519, mean=0.545, max=0.579, sum=3.269 (6)\", \"tab\": \"Bias\", \"score\": \"0.5447683118463776\"}", - "XSUM - Representation (gender)": "{\"description\": \"min=0.202, mean=0.207, max=0.211, sum=1.243 (6)\", \"tab\": \"Bias\", \"score\": \"0.2071945417372382\"}", - "XSUM - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (6)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}", - "XSUM - SummaC": "{\"description\": \"min=-0.225, mean=-0.206, max=-0.183, sum=-0.617 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"-0.20556503322082545\"}", - "XSUM - QAFactEval": "{\"description\": \"2 matching runs, but no matching metrics\", \"tab\": \"Summarization metrics\", \"score\": \"\"}", - "XSUM - BERTScore (F1)": "{\"description\": \"min=0.427, mean=0.427, max=0.428, sum=1.282 (3)\", \"tab\": \"Summarization metrics\", \"score\": \"0.42745522151316395\"}", - "XSUM - Coverage": "{\"description\": \"min=0.813, mean=0.817, max=0.82, sum=4.905 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"0.8174518357071618\"}", - "XSUM - Density": "{\"description\": \"min=3.819, mean=4.041, max=4.367, sum=24.243 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"4.040514978645572\"}", - "XSUM - Compression": "{\"description\": \"min=16.122, mean=16.25, max=16.375, sum=97.5 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"16.25000448561988\"}", - "XSUM - HumanEval-faithfulness": "{\"description\": \"min=0.583, mean=0.763, max=0.905, sum=4.576 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"0.7626984126984127\"}", - "XSUM - HumanEval-relevance": "{\"description\": \"min=3.333, mean=3.843, max=4.1, sum=23.057 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"3.842857142857143\"}", - "XSUM - HumanEval-coherence": "{\"description\": \"min=3.417, mean=4.25, max=4.667, sum=25.5 (6)\", \"tab\": \"Summarization metrics\", \"score\": \"4.249999999999999\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "IMDB", - "source_data": { - "dataset_name": "IMDB", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on IMDB", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.955, - "details": { - "description": "min=0.946, mean=0.955, max=0.961, sum=2.864 (3)", - "tab": "Accuracy", - "IMDB - ECE (10-bin)": "{\"description\": \"min=0.117, mean=0.18, max=0.225, sum=0.541 (3)\", \"tab\": \"Calibration\", \"score\": \"0.18041748611363093\"}", - "IMDB - EM (Robustness)": "{\"description\": \"min=0.921, mean=0.938, max=0.955, sum=2.814 (3)\", \"tab\": \"Robustness\", \"score\": \"0.9380000000000001\"}", - "IMDB - EM (Fairness)": "{\"description\": \"min=0.92, mean=0.933, max=0.951, sum=2.799 (3)\", \"tab\": \"Fairness\", \"score\": \"0.9329999999999999\"}", - "IMDB - Denoised inference time (s)": "{\"description\": \"min=1.446, mean=1.497, max=1.55, sum=4.491 (3)\", \"tab\": \"Efficiency\", \"score\": \"1.4970239554705547\"}", - "IMDB - # eval": "{\"description\": \"min=1000, mean=1000, max=1000, sum=3000 (3)\", \"tab\": \"General information\", \"score\": \"1000.0\"}", - "IMDB - # train": "{\"description\": \"min=4.832, mean=4.923, max=4.979, sum=14.77 (3)\", \"tab\": \"General information\", \"score\": \"4.923333333333333\"}", - "IMDB - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (3)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "IMDB - # prompt tokens": "{\"description\": \"min=1182.719, mean=1412.285, max=1755.875, sum=4236.855 (3)\", \"tab\": \"General information\", \"score\": \"1412.2849999999999\"}", - "IMDB - # output tokens": "{\"description\": \"min=2, mean=2, max=2, sum=6 (3)\", \"tab\": \"General information\", \"score\": \"2.0\"}", - "IMDB - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=9 (3)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "IMDB - Stereotypes (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Stereotypes (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (race)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Representation (gender)": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Bias\", \"score\": \"\"}", - "IMDB - Toxic fraction": "{\"description\": \"1 matching runs, but no matching metrics\", \"tab\": \"Toxicity\", \"score\": \"\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "CivilComments", - "source_data": { - "dataset_name": "CivilComments", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on CivilComments", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.5, - "details": { - "description": "min=0, mean=0.5, max=1, sum=27.019 (54)", - "tab": "Accuracy", - "CivilComments - ECE (10-bin)": "{\"description\": \"min=0.22, mean=0.486, max=0.749, sum=26.268 (54)\", \"tab\": \"Calibration\", \"score\": \"0.4864398714978027\"}", - "CivilComments - EM (Robustness)": "{\"description\": \"min=0, mean=0.5, max=1, sum=27.004 (54)\", \"tab\": \"Robustness\", \"score\": \"0.5000703286326241\"}", - "CivilComments - EM (Fairness)": "{\"description\": \"min=0, mean=0.5, max=1, sum=26.982 (54)\", \"tab\": \"Fairness\", \"score\": \"0.4996593325872097\"}", - "CivilComments - Denoised inference time (s)": "{\"description\": \"min=0.442, mean=0.695, max=1.665, sum=37.54 (54)\", \"tab\": \"Efficiency\", \"score\": \"0.695191819583079\"}", - "CivilComments - # eval": "{\"description\": \"min=74, mean=371.556, max=683, sum=20064 (54)\", \"tab\": \"General information\", \"score\": \"371.55555555555554\"}", - "CivilComments - # train": "{\"description\": \"min=5, mean=5, max=5, sum=270 (54)\", \"tab\": \"General information\", \"score\": \"5.0\"}", - "CivilComments - truncated": "{\"description\": \"min=0, mean=0, max=0, sum=0 (54)\", \"tab\": \"General information\", \"score\": \"0.0\"}", - "CivilComments - # prompt tokens": "{\"description\": \"min=342, mean=694.39, max=1246.337, sum=37497.067 (54)\", \"tab\": \"General information\", \"score\": \"694.3901297399493\"}", - "CivilComments - # output tokens": "{\"description\": \"min=2, mean=2, max=2, sum=108 (54)\", \"tab\": \"General information\", \"score\": \"2.0\"}", - "CivilComments - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=162 (54)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "CivilComments - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "CivilComments - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (54)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - }, - { - "evaluation_name": "RAFT", - "source_data": { - "dataset_name": "RAFT", - "source_type": "url", - "url": [ - "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json" - ] - }, - "metric_config": { - "evaluation_description": "EM on RAFT", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.598, - "details": { - "description": "min=0, mean=0.598, max=0.975, sum=19.725 (33)", - "tab": "Accuracy", - "RAFT - ECE (10-bin)": "{\"description\": \"min=0.045, mean=0.226, max=0.392, sum=7.451 (33)\", \"tab\": \"Calibration\", \"score\": \"0.225785860693393\"}", - "RAFT - EM (Robustness)": "{\"description\": \"min=0, mean=0.577, max=0.975, sum=19.05 (33)\", \"tab\": \"Robustness\", \"score\": \"0.5772727272727272\"}", - "RAFT - EM (Fairness)": "{\"description\": \"min=0, mean=0.575, max=0.975, sum=18.975 (33)\", \"tab\": \"Fairness\", \"score\": \"0.575\"}", - "RAFT - Denoised inference time (s)": "{\"description\": \"min=0.333, mean=1.471, max=2.214, sum=48.528 (33)\", \"tab\": \"Efficiency\", \"score\": \"1.4705579548050658\"}", - "RAFT - # eval": "{\"description\": \"min=40, mean=40, max=40, sum=1320 (33)\", \"tab\": \"General information\", \"score\": \"40.0\"}", - "RAFT - # train": "{\"description\": \"min=0, mean=4.563, max=5, sum=150.575 (33)\", \"tab\": \"General information\", \"score\": \"4.5628787878787875\"}", - "RAFT - truncated": "{\"description\": \"min=0, mean=0.07, max=1, sum=2.3 (33)\", \"tab\": \"General information\", \"score\": \"0.06969696969696969\"}", - "RAFT - # prompt tokens": "{\"description\": \"min=244.45, mean=803.318, max=1757.15, sum=26509.5 (33)\", \"tab\": \"General information\", \"score\": \"803.3181818181819\"}", - "RAFT - # output tokens": "{\"description\": \"min=2.6, mean=4.886, max=11.6, sum=161.25 (33)\", \"tab\": \"General information\", \"score\": \"4.886363636363637\"}", - "RAFT - # trials": "{\"description\": \"min=3, mean=3, max=3, sum=99 (33)\", \"tab\": \"General information\", \"score\": \"3.0\"}", - "RAFT - Stereotypes (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Stereotypes (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (race)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Representation (gender)": "{\"description\": \"(0)\", \"tab\": \"Bias\", \"score\": \"\"}", - "RAFT - Toxic fraction": "{\"description\": \"min=0, mean=0, max=0, sum=0 (33)\", \"tab\": \"Toxicity\", \"score\": \"0.0\"}" - } - }, - "generation_config": { - "additional_details": {} - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "additional_details": {} - } - } - ] -} \ No newline at end of file diff --git a/data/models/zhipu-ai_glm-4.6.json b/data/models/zhipu-ai_glm-4.6.json deleted file mode 100644 index 5a723bb1ca49f72f8b3671a2744761bb5607cbbe..0000000000000000000000000000000000000000 --- a/data/models/zhipu-ai_glm-4.6.json +++ /dev/null @@ -1,87 +0,0 @@ -{ - "model_info": { - "name": "GLM 4.6", - "id": "zhipu-ai/glm-4.6", - "developer": "Z.ai", - "additional_details": { - "agent_name": "Terminus 2", - "agent_organization": "Terminal Bench" - } - }, - "evaluations": [ - { - "evaluation_id": "terminal-bench-2.0/terminus-2__glm-4.6/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2025-11-01", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 24.5, - "uncertainty": { - "standard_error": { - "value": 2.4 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Terminus 2\" -m \"GLM 4.6\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Terminus 2\" -m \"GLM 4.6\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/zhipu-ai_glm-4.7.json b/data/models/zhipu-ai_glm-4.7.json deleted file mode 100644 index ef6efd4fd9091259b58749fefdf86e2517723096..0000000000000000000000000000000000000000 --- a/data/models/zhipu-ai_glm-4.7.json +++ /dev/null @@ -1,161 +0,0 @@ -{ - "model_info": { - "name": "GLM 4.7", - "id": "zhipu-ai/glm-4.7", - "developer": "Z-AI", - "additional_details": { - "agent_name": "Terminus 2", - "agent_organization": "Terminal Bench" - } - }, - "evaluations": [ - { - "evaluation_id": "terminal-bench-2.0/terminus-2__glm-4.7/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2026-01-28", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 33.4, - "uncertainty": { - "standard_error": { - "value": 2.8 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Terminus 2\" -m \"GLM 4.7\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Terminus 2\" -m \"GLM 4.7\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - }, - { - "evaluation_id": "terminal-bench-2.0/crux__glm-4.7/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2026-02-08", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 33.3, - "uncertainty": { - "standard_error": { - "value": 2.5 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Crux\" -m \"GLM 4.7\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Crux\" -m \"GLM 4.7\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/zhipu-ai_glm-5.json b/data/models/zhipu-ai_glm-5.json deleted file mode 100644 index 46ecd0a8a6d9e625fc09a6462470870832e836f1..0000000000000000000000000000000000000000 --- a/data/models/zhipu-ai_glm-5.json +++ /dev/null @@ -1,87 +0,0 @@ -{ - "model_info": { - "name": "GLM 5", - "id": "zhipu-ai/glm-5", - "developer": "Z-AI", - "additional_details": { - "agent_name": "Terminus 2", - "agent_organization": "Terminal Bench" - } - }, - "evaluations": [ - { - "evaluation_id": "terminal-bench-2.0/terminus-2__glm-5/1773776901.772108", - "retrieved_timestamp": "1773776901.772108", - "source_metadata": { - "source_name": "Terminal-Bench 2.0", - "source_type": "documentation", - "source_organization_name": "Terminal-Bench", - "source_organization_url": "https://www.tbench.ai", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "harbor", - "version": "unknown" - }, - "benchmark": "terminal-bench-2.0", - "evaluation_results": [ - { - "evaluation_name": "terminal-bench-2.0", - "source_data": { - "dataset_name": "terminal-bench-2.0", - "source_type": "url", - "url": [ - "https://www.tbench.ai/leaderboard/terminal-bench/2.0" - ] - }, - "evaluation_timestamp": "2026-02-23", - "metric_config": { - "evaluation_description": "Task resolution accuracy across 87 terminal tasks with 5 trials each", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0 - }, - "score_details": { - "score": 52.4, - "uncertainty": { - "standard_error": { - "value": 2.6 - }, - "num_samples": 435 - } - }, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Terminus 2\" -m \"GLM 5\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ], - "detailed_evaluation_results": null, - "generation_config": { - "generation_args": { - "execution_command": "harbor run -d terminal-bench@2.0 -a \"Terminus 2\" -m \"GLM 5\" -k 5", - "agentic_eval_config": { - "available_tools": [ - { - "name": "terminal", - "description": "Full terminal/shell access" - } - ] - }, - "max_attempts": 1 - } - } - } - ] -} \ No newline at end of file diff --git a/data/models/zhipu_glm-4-6-fc-thinking.json b/data/models/zhipu_glm-4-6-fc-thinking.json deleted file mode 100644 index e591cbbe1110fc0b51e3055adf7a3d7f51a6e2c6..0000000000000000000000000000000000000000 --- a/data/models/zhipu_glm-4-6-fc-thinking.json +++ /dev/null @@ -1,851 +0,0 @@ -{ - "model_info": { - "name": "GLM-4.6 (FC thinking)", - "id": "zhipu/glm-4-6-fc-thinking", - "developer": "zhipu", - "additional_details": { - "raw_model_name": "GLM-4.6 (FC thinking)", - "organization": "Zhipu AI", - "license": "MIT", - "mode": "FC thinking", - "model_link": "https://huggingface.co/zai-org/GLM-4.6" - } - }, - "evaluations": [ - { - "evaluation_id": "bfcl/zhipu/glm-4-6-fc-thinking/1775236112.36754", - "retrieved_timestamp": "1775236112.36754", - "source_metadata": { - "source_name": "BFCL leaderboard CSV", - "source_type": "documentation", - "source_organization_name": "UC Berkeley Gorilla", - "source_organization_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "evaluator_relationship": "third_party", - "additional_details": { - "csv_url": "https://gorilla.cs.berkeley.edu/data_overall.csv", - "leaderboard_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", - "leaderboard_version": "BFCL V4" - } - }, - "eval_library": { - "name": "BFCL", - "version": "v4" - }, - "benchmark": "bfcl", - "evaluation_results": [ - { - "evaluation_result_id": "bfcl.overall.rank::rank", - "evaluation_name": "bfcl.overall.rank", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.rank", - "metric_name": "Overall rank", - "metric_kind": "rank", - "metric_unit": "position", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 1.0, - "max_score": 109.0, - "additional_details": { - "raw_metric_field": "Rank" - } - }, - "score_details": { - "score": 4.0 - } - }, - { - "evaluation_result_id": "bfcl.overall.overall_accuracy::overall_accuracy", - "evaluation_name": "bfcl.overall.overall_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.overall_accuracy", - "metric_name": "Overall accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Overall Acc" - } - }, - "score_details": { - "score": 72.38 - } - }, - { - "evaluation_result_id": "bfcl.overall.total_cost_usd::total_cost_usd", - "evaluation_name": "bfcl.overall.total_cost_usd", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.total_cost_usd", - "metric_name": "Total cost", - "metric_kind": "cost", - "metric_unit": "usd", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 355.17, - "additional_details": { - "raw_metric_field": "Total Cost ($)" - } - }, - "score_details": { - "score": 4.64 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_mean_s::latency_mean_s", - "evaluation_name": "bfcl.overall.latency_mean_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_mean_s", - "metric_name": "Latency mean", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 169.87, - "additional_details": { - "raw_metric_field": "Latency Mean (s)" - } - }, - "score_details": { - "score": 4.34 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_std_s::latency_std_s", - "evaluation_name": "bfcl.overall.latency_std_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_std_s", - "metric_name": "Latency standard deviation", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 212.99, - "additional_details": { - "raw_metric_field": "Latency Standard Deviation (s)" - } - }, - "score_details": { - "score": 7.22 - } - }, - { - "evaluation_result_id": "bfcl.overall.latency_p95_s::latency_p95_s", - "evaluation_name": "bfcl.overall.latency_p95_s", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.overall.latency_p95_s", - "metric_name": "Latency 95th percentile", - "metric_kind": "latency", - "metric_unit": "seconds", - "lower_is_better": true, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 568.59, - "additional_details": { - "raw_metric_field": "Latency 95th Percentile (s)" - } - }, - "score_details": { - "score": 13.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.ast_accuracy::ast_accuracy", - "evaluation_name": "bfcl.non_live.ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.ast_accuracy", - "metric_name": "Non-live AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live AST Acc" - } - }, - "score_details": { - "score": 87.56 - } - }, - { - "evaluation_result_id": "bfcl.non_live.simple_ast_accuracy::simple_ast_accuracy", - "evaluation_name": "bfcl.non_live.simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.simple_ast_accuracy", - "metric_name": "Non-live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Simple AST" - } - }, - "score_details": { - "score": 74.25 - } - }, - { - "evaluation_result_id": "bfcl.non_live.multiple_ast_accuracy::multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.multiple_ast_accuracy", - "metric_name": "Non-live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Multiple AST" - } - }, - "score_details": { - "score": 95.0 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_ast_accuracy::parallel_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_ast_accuracy", - "metric_name": "Non-live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel AST" - } - }, - "score_details": { - "score": 91.5 - } - }, - { - "evaluation_result_id": "bfcl.non_live.parallel_multiple_ast_accuracy::parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.non_live.parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.non_live.parallel_multiple_ast_accuracy", - "metric_name": "Non-live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Non-Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 89.5 - } - }, - { - "evaluation_result_id": "bfcl.live.live_accuracy::live_accuracy", - "evaluation_name": "bfcl.live.live_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_accuracy", - "metric_name": "Live accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Acc" - } - }, - "score_details": { - "score": 80.9 - } - }, - { - "evaluation_result_id": "bfcl.live.live_simple_ast_accuracy::live_simple_ast_accuracy", - "evaluation_name": "bfcl.live.live_simple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_simple_ast_accuracy", - "metric_name": "Live simple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Simple AST" - } - }, - "score_details": { - "score": 89.53 - } - }, - { - "evaluation_result_id": "bfcl.live.live_multiple_ast_accuracy::live_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_multiple_ast_accuracy", - "metric_name": "Live multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Multiple AST" - } - }, - "score_details": { - "score": 78.92 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_ast_accuracy::live_parallel_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_ast_accuracy", - "metric_name": "Live parallel AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel AST" - } - }, - "score_details": { - "score": 81.25 - } - }, - { - "evaluation_result_id": "bfcl.live.live_parallel_multiple_ast_accuracy::live_parallel_multiple_ast_accuracy", - "evaluation_name": "bfcl.live.live_parallel_multiple_ast_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.live.live_parallel_multiple_ast_accuracy", - "metric_name": "Live parallel multiple AST accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Live Parallel Multiple AST" - } - }, - "score_details": { - "score": 75.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.accuracy::accuracy", - "evaluation_name": "bfcl.multi_turn.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.accuracy", - "metric_name": "Multi-turn accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Acc" - } - }, - "score_details": { - "score": 68.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.multi_turn.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.base_accuracy", - "metric_name": "Multi-turn base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Base" - } - }, - "score_details": { - "score": 74.5 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_function_accuracy::miss_function_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_function_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_function_accuracy", - "metric_name": "Multi-turn missing function accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Func" - } - }, - "score_details": { - "score": 68.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.miss_parameter_accuracy::miss_parameter_accuracy", - "evaluation_name": "bfcl.multi_turn.miss_parameter_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.miss_parameter_accuracy", - "metric_name": "Multi-turn missing parameter accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Miss Param" - } - }, - "score_details": { - "score": 63.0 - } - }, - { - "evaluation_result_id": "bfcl.multi_turn.long_context_accuracy::long_context_accuracy", - "evaluation_name": "bfcl.multi_turn.long_context_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.multi_turn.long_context_accuracy", - "metric_name": "Multi-turn long-context accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Multi Turn Long Context" - } - }, - "score_details": { - "score": 66.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.accuracy::accuracy", - "evaluation_name": "bfcl.web_search.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.accuracy", - "metric_name": "Web-search accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Acc" - } - }, - "score_details": { - "score": 77.5 - } - }, - { - "evaluation_result_id": "bfcl.web_search.base_accuracy::base_accuracy", - "evaluation_name": "bfcl.web_search.base_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.base_accuracy", - "metric_name": "Web-search base accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search Base" - } - }, - "score_details": { - "score": 79.0 - } - }, - { - "evaluation_result_id": "bfcl.web_search.no_snippet_accuracy::no_snippet_accuracy", - "evaluation_name": "bfcl.web_search.no_snippet_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.web_search.no_snippet_accuracy", - "metric_name": "Web-search no-snippet accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Web Search No Snippet" - } - }, - "score_details": { - "score": 76.0 - } - }, - { - "evaluation_result_id": "bfcl.memory.accuracy::accuracy", - "evaluation_name": "bfcl.memory.accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.accuracy", - "metric_name": "Memory accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Acc" - } - }, - "score_details": { - "score": 55.7 - } - }, - { - "evaluation_result_id": "bfcl.memory.kv_accuracy::kv_accuracy", - "evaluation_name": "bfcl.memory.kv_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.kv_accuracy", - "metric_name": "Memory KV accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory KV" - } - }, - "score_details": { - "score": 43.87 - } - }, - { - "evaluation_result_id": "bfcl.memory.vector_accuracy::vector_accuracy", - "evaluation_name": "bfcl.memory.vector_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.vector_accuracy", - "metric_name": "Memory vector accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Vector" - } - }, - "score_details": { - "score": 56.13 - } - }, - { - "evaluation_result_id": "bfcl.memory.recursive_summarization_accuracy::recursive_summarization_accuracy", - "evaluation_name": "bfcl.memory.recursive_summarization_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.memory.recursive_summarization_accuracy", - "metric_name": "Memory recursive summarization accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Memory Recursive Summarization" - } - }, - "score_details": { - "score": 67.1 - } - }, - { - "evaluation_result_id": "bfcl.relevance.relevance_detection_accuracy::relevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.relevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.relevance_detection_accuracy", - "metric_name": "Relevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Relevance Detection" - } - }, - "score_details": { - "score": 75.0 - } - }, - { - "evaluation_result_id": "bfcl.relevance.irrelevance_detection_accuracy::irrelevance_detection_accuracy", - "evaluation_name": "bfcl.relevance.irrelevance_detection_accuracy", - "source_data": { - "source_type": "url", - "dataset_name": "BFCL leaderboard CSV", - "url": [ - "https://gorilla.cs.berkeley.edu/data_overall.csv" - ] - }, - "metric_config": { - "metric_id": "bfcl.relevance.irrelevance_detection_accuracy", - "metric_name": "Irrelevance detection accuracy", - "metric_kind": "accuracy", - "metric_unit": "percentage", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 100.0, - "additional_details": { - "raw_metric_field": "Irrelevance Detection" - } - }, - "score_details": { - "score": 84.96 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zhipu_glm_4.6.json b/data/models/zhipu_glm_4.6.json deleted file mode 100644 index 63851206bcad59d2dbef0f0c15a3f8935824fc91..0000000000000000000000000000000000000000 --- a/data/models/zhipu_glm_4.6.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "model_info": { - "name": "GLM 4.6", - "developer": "zhipu", - "id": "zhipu/GLM 4.6", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "apex-agents/zhipu_glm-4.6/1773260200", - "retrieved_timestamp": "1773260200", - "source_metadata": { - "source_name": "Mercor APEX-Agents Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Mercor", - "source_organization_url": "https://www.mercor.com", - "evaluator_relationship": "first_party" - }, - "eval_library": { - "name": "archipelago", - "version": "1.0.0" - }, - "benchmark": "apex-agents", - "evaluation_results": [ - { - "evaluation_name": "Corporate Lawyer Mean Score", - "source_data": { - "dataset_name": "apex-agents", - "source_type": "hf_dataset", - "hf_repo": "mercor/apex-agents" - }, - "metric_config": { - "evaluation_description": "Corporate lawyer world mean score.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.196 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zhipu_glm_4.7.json b/data/models/zhipu_glm_4.7.json deleted file mode 100644 index 3e8eb4a8b0c9edbab8b77d15a8a96ffb48474c37..0000000000000000000000000000000000000000 --- a/data/models/zhipu_glm_4.7.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "model_info": { - "name": "GLM 4.7", - "developer": "zhipu", - "id": "zhipu/GLM 4.7", - "inference_platform": "unknown" - }, - "evaluations": [ - { - "evaluation_id": "apex-agents/zhipu_glm-4.7/1773260200", - "retrieved_timestamp": "1773260200", - "source_metadata": { - "source_name": "Mercor APEX-Agents Leaderboard", - "source_type": "evaluation_run", - "source_organization_name": "Mercor", - "source_organization_url": "https://www.mercor.com", - "evaluator_relationship": "first_party" - }, - "eval_library": { - "name": "archipelago", - "version": "1.0.0" - }, - "benchmark": "apex-agents", - "evaluation_results": [ - { - "evaluation_name": "Corporate Lawyer Mean Score", - "source_data": { - "dataset_name": "apex-agents", - "source_type": "hf_dataset", - "hf_repo": "mercor/apex-agents" - }, - "metric_config": { - "evaluation_description": "Corporate lawyer world mean score.", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0, - "max_score": 1 - }, - "score_details": { - "score": 0.147 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zhliu627_zephyr-7b-gemma-dpo-avg.json b/data/models/zhliu627_zephyr-7b-gemma-dpo-avg.json deleted file mode 100644 index 04d96e4db6ad6005f6067e24879327bc2198d64d..0000000000000000000000000000000000000000 --- a/data/models/zhliu627_zephyr-7b-gemma-dpo-avg.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "zephyr-7b-gemma-dpo-avg", - "id": "ZHLiu627/zephyr-7b-gemma-dpo-avg", - "developer": "ZHLiu627", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "GemmaForCausalLM", - "params_billions": "8.538" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ZHLiu627_zephyr-7b-gemma-dpo-avg/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.309 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4149 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0453 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2785 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4107 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2851 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/zhliu627_zephyr-7b-gemma-rpo-avg.json b/data/models/zhliu627_zephyr-7b-gemma-rpo-avg.json deleted file mode 100644 index 413927f9ccb38eee36c9a56cd539c8af78b1cf76..0000000000000000000000000000000000000000 --- a/data/models/zhliu627_zephyr-7b-gemma-rpo-avg.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "model_info": { - "name": "zephyr-7b-gemma-rpo-avg", - "id": "ZHLiu627/zephyr-7b-gemma-rpo-avg", - "developer": "ZHLiu627", - "inference_platform": "unknown", - "additional_details": { - "precision": "float16", - "architecture": "GemmaForCausalLM", - "params_billions": "8.538" - } - }, - "evaluations": [ - { - "evaluation_id": "hfopenllm_v2/ZHLiu627_zephyr-7b-gemma-rpo-avg/1773936498.240187", - "retrieved_timestamp": "1773936498.240187", - "source_metadata": { - "source_name": "HF Open LLM v2", - "source_type": "documentation", - "source_organization_name": "Hugging Face", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "lm-evaluation-harness", - "version": "0.4.0", - "additional_details": { - "fork": "https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess" - } - }, - "benchmark": "hfopenllm_v2", - "evaluation_results": [ - { - "evaluation_name": "IFEval", - "source_data": { - "dataset_name": "IFEval", - "source_type": "hf_dataset", - "hf_repo": "google/IFEval" - }, - "metric_config": { - "evaluation_description": "Accuracy on IFEval", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.3006 - } - }, - { - "evaluation_name": "BBH", - "source_data": { - "dataset_name": "BBH", - "source_type": "hf_dataset", - "hf_repo": "SaylorTwift/bbh" - }, - "metric_config": { - "evaluation_description": "Accuracy on BBH", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4183 - } - }, - { - "evaluation_name": "MATH Level 5", - "source_data": { - "dataset_name": "MATH Level 5", - "source_type": "hf_dataset", - "hf_repo": "DigitalLearningGmbH/MATH-lighteval" - }, - "metric_config": { - "evaluation_description": "Exact Match on MATH Level 5", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.0498 - } - }, - { - "evaluation_name": "GPQA", - "source_data": { - "dataset_name": "GPQA", - "source_type": "hf_dataset", - "hf_repo": "Idavidrein/gpqa" - }, - "metric_config": { - "evaluation_description": "Accuracy on GPQA", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2768 - } - }, - { - "evaluation_name": "MUSR", - "source_data": { - "dataset_name": "MUSR", - "source_type": "hf_dataset", - "hf_repo": "TAUR-Lab/MuSR" - }, - "metric_config": { - "evaluation_description": "Accuracy on MUSR", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.4081 - } - }, - { - "evaluation_name": "MMLU-PRO", - "source_data": { - "dataset_name": "MMLU-PRO", - "source_type": "hf_dataset", - "hf_repo": "TIGER-Lab/MMLU-Pro" - }, - "metric_config": { - "evaluation_description": "Accuracy on MMLU-PRO", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.2831 - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/data/models/ziyiye_con-j-qwen2-7b.json b/data/models/ziyiye_con-j-qwen2-7b.json deleted file mode 100644 index 9be866cd01319460c66ab9bf6fce3116057f0921..0000000000000000000000000000000000000000 --- a/data/models/ziyiye_con-j-qwen2-7b.json +++ /dev/null @@ -1,126 +0,0 @@ -{ - "model_info": { - "name": "ZiyiYe/Con-J-Qwen2-7B", - "id": "ZiyiYe/Con-J-Qwen2-7B", - "developer": "ZiyiYe", - "additional_details": { - "model_type": "Generative" - } - }, - "evaluations": [ - { - "evaluation_id": "reward-bench/ZiyiYe_Con-J-Qwen2-7B/1766412838.146816", - "retrieved_timestamp": "1766412838.146816", - "source_metadata": { - "source_name": "RewardBench", - "source_type": "documentation", - "source_organization_name": "Allen Institute for AI", - "source_organization_url": "https://allenai.org", - "evaluator_relationship": "third_party" - }, - "eval_library": { - "name": "rewardbench", - "version": "0.1.3", - "additional_details": { - "subsets": "Chat, Chat Hard, Safety, Reasoning", - "hf_space": "allenai/reward-bench" - } - }, - "benchmark": "reward-bench", - "evaluation_results": [ - { - "evaluation_name": "Score", - "metric_config": { - "evaluation_description": "Overall RewardBench Score", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8712 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat", - "metric_config": { - "evaluation_description": "Chat accuracy - includes easy chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.919 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Chat Hard", - "metric_config": { - "evaluation_description": "Chat Hard accuracy - includes hard chat subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8026 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Safety", - "metric_config": { - "evaluation_description": "Safety accuracy - includes safety subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8824 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - }, - { - "evaluation_name": "Reasoning", - "metric_config": { - "evaluation_description": "Reasoning accuracy - includes code and math subsets", - "lower_is_better": false, - "score_type": "continuous", - "min_score": 0.0, - "max_score": 1.0 - }, - "score_details": { - "score": 0.8808 - }, - "source_data": { - "dataset_name": "RewardBench", - "source_type": "hf_dataset", - "hf_repo": "allenai/reward-bench" - } - } - ], - "detailed_evaluation_results": null, - "generation_config": null - } - ] -} \ No newline at end of file diff --git a/lib/data-backend.ts b/lib/data-backend.ts index 26954755b0f7f9ee57b50a9fc005d182dd2a7259..9e742aaeafa003734d690525e6e6c48cf166299c 100644 --- a/lib/data-backend.ts +++ b/lib/data-backend.ts @@ -1,23 +1,45 @@ import "server-only" +import { + getDashboardDataFromDuckDB, + getModelCardsFromDuckDB, + getModelCardsLiteFromDuckDB, + getEvalListDataFromDuckDB, + getEvalListLiteDataFromDuckDB, + getEvalListFromDuckDB, + getDeveloperListFromDuckDB, + getDeveloperSummaryByIdFromDuckDB, + getModelSummaryByIdFromDuckDB, + getEvalSummaryByIdFromDuckDB, +} from "@/lib/duckdb-data" +import { normalizeEvalSummary } from "@/lib/eval-processing" import { fetchBackendManifest, fetchBackendManifestStatus, fetchEvalHierarchy, } from "@/lib/hf-data" -export { - getDashboardDataFromDuckDB as getDashboardData, - getModelCardsFromDuckDB as getModelCards, - getModelCardsLiteFromDuckDB as getModelCardsLite, - getEvalListDataFromDuckDB as getEvalListData, - getEvalListLiteDataFromDuckDB as getEvalListLiteData, - getEvalListFromDuckDB as getEvalList, - getDeveloperListFromDuckDB as getDeveloperList, - getDeveloperSummaryByIdFromDuckDB as getDeveloperSummaryById, - getModelSummaryByIdFromDuckDB as getModelSummaryById, - getEvalSummaryByIdFromDuckDB as getEvalSummaryById, -} from "@/lib/duckdb-data" +export const getDashboardData = getDashboardDataFromDuckDB +export const getModelCards = getModelCardsFromDuckDB +export const getModelCardsLite = getModelCardsLiteFromDuckDB +export const getEvalListData = getEvalListDataFromDuckDB +export const getEvalListLiteData = getEvalListLiteDataFromDuckDB +export const getEvalList = getEvalListFromDuckDB +export const getDeveloperList = getDeveloperListFromDuckDB +export const getDeveloperSummaryById = getDeveloperSummaryByIdFromDuckDB +export const getModelSummaryById = getModelSummaryByIdFromDuckDB + +/** + * Eval summary lookups go through `normalizeEvalSummary` so derivable but + * sometimes-blank fields (currently `instance_data`) are reconciled from + * `model_results` before they reach any consumer. The strict pass-through + * contract of `duckdb-data.ts` stays intact — reconciliation of known + * upstream gaps belongs in this thin adapter layer. + */ +export async function getEvalSummaryById(evalId: string) { + const summary = await getEvalSummaryByIdFromDuckDB(evalId) + return summary ? normalizeEvalSummary(summary) : summary +} // Metadata-style artifacts are still read through the existing JSON/HF path. // They are not request-time processing hotspots and the DuckDB shadow doesn't diff --git a/lib/eval-processing.ts b/lib/eval-processing.ts index 6ac600917a7ddb30e09c37cdfedfb597ccf3078d..9224180f97c0f4d2612d44c8beb3a8fb63b0a799 100644 --- a/lib/eval-processing.ts +++ b/lib/eval-processing.ts @@ -244,6 +244,56 @@ export interface BenchmarkLeaderboardRow { export type BenchmarkEvalListItem = Omit +/** + * Fill in derived fields the upstream pipeline sometimes leaves blank. + * + * Currently: `instance_data`. The pipeline that emits eval-summary parquets + * occasionally ships rows where `instance_data` is null even though every + * `model_results[].result.detailed_evaluation_results_url` is populated + * (Wordle Arena is one example — 42 models, every one with a per-model + * JSONL URL on `evaleval/card_backend`, but `instance_data` was null). + * + * Rather than patching this at one render site we derive it once here so + * every consumer of the summary — eval detail page, modal previews, + * cross-referenced model summaries, etc. — sees the same picture. + */ +export function normalizeEvalSummary(summary: T): T { + if (summary.instance_data?.available && summary.instance_data.url_count > 0) { + return summary + } + + const distinctUrls = new Set() + const modelsWithUrl = new Set() + for (const result of summary.model_results ?? []) { + const url = result?.result?.detailed_evaluation_results_url + if (typeof url === "string" && url.length > 0) { + distinctUrls.add(url) + const modelId = result.model_info?.id + if (modelId) modelsWithUrl.add(modelId) + } + } + + if (distinctUrls.size === 0) { + // Nothing to derive — preserve whatever the upstream said (typically + // `available: false` or absent). + return summary + } + + // Take a small sample so callers can show example URLs without paying + // for the full set, mirroring the upstream pipeline's contract. + const sampleUrls = Array.from(distinctUrls).slice(0, 8) + + return { + ...summary, + instance_data: { + available: true, + url_count: distinctUrls.size, + sample_urls: sampleUrls, + models_with_loaded_instances: modelsWithUrl.size, + }, + } +} + /** * Group multiple evaluations by model */ diff --git a/lib/param-range.ts b/lib/param-range.ts new file mode 100644 index 0000000000000000000000000000000000000000..db651636eb3e08583c6e165cd144b00f5cdbb2f9 --- /dev/null +++ b/lib/param-range.ts @@ -0,0 +1,77 @@ +/** + * Shared constants for the dual-handle parameter range picker used to filter + * leaderboards by model size. The bucket list is fine-grained enough that + * users can land on a meaningful midpoint, but the visible tick labels are a + * smaller subset matching the design (1B / 8B / 12B / 32B / 128B / >500B). + */ +export const PARAM_RANGE_VALUES = [ + 1, 2, 3, 4, 6, 8, 10, 12, 16, 24, 32, 48, 64, 96, 128, 192, 256, 384, 500, +] as const + +export const PARAM_RANGE_MAX_INDEX = PARAM_RANGE_VALUES.length - 1 + +/** Tick labels rendered above the rail. The `step` field is the index into + * `PARAM_RANGE_VALUES` at which the label sits — used to position it. */ +export const PARAM_RANGE_MARKERS = [ + { label: "< 1B", step: 0 }, + { label: "8B", step: PARAM_RANGE_VALUES.indexOf(8) }, + { label: "12B", step: PARAM_RANGE_VALUES.indexOf(12) }, + { label: "32B", step: PARAM_RANGE_VALUES.indexOf(32) }, + { label: "128B", step: PARAM_RANGE_VALUES.indexOf(128) }, + { label: "> 500B", step: PARAM_RANGE_MAX_INDEX }, +] as const + +export function formatParamBoundLabel(step: number, bound: "min" | "max"): string { + if (bound === "min" && step <= 0) return "< 1B" + if (bound === "max" && step >= PARAM_RANGE_MAX_INDEX) return "> 500B" + const value = PARAM_RANGE_VALUES[step] + return value != null ? `${value}B` : "—" +} + +/** Converts a slider step to a numeric param-billions filter, or null if the + * bound is at the open end (no filter applies). */ +export function paramStepToNumeric(step: number, bound: "min" | "max"): number | null { + if (bound === "min" && step <= 0) return null + if (bound === "max" && step >= PARAM_RANGE_MAX_INDEX) return null + return PARAM_RANGE_VALUES[step] ?? null +} + +/** Best-effort extraction of `B` parameter counts from a free-text field + * (typically a model name like "Llama-3.1 70B Instruct"). Returns null if no + * size token is present. Shared so all leaderboards detect parameters in the + * same way and the slider visibility is consistent. */ +export function parseParamsBillionsFromText(value: string | null | undefined): number | null { + if (!value) return null + const normalized = value.trim().toLowerCase() + if (!normalized) return null + const compact = normalized.replace(/,/g, "") + const tokenMatch = compact.match( + /(\d+(?:\.\d+)?)\s*(trillion|tn|t|billion|bn|b|million|mn|m|thousand|k)\b/, + ) + if (tokenMatch) { + const amount = Number.parseFloat(tokenMatch[1]) + if (!Number.isFinite(amount)) return null + const unit = tokenMatch[2] + if (unit === "trillion" || unit === "tn" || unit === "t") return amount * 1000 + if (unit === "billion" || unit === "bn" || unit === "b") return amount + if (unit === "million" || unit === "mn" || unit === "m") return amount / 1000 + if (unit === "thousand" || unit === "k") return amount / 1_000_000 + } + const numeric = Number.parseFloat(compact) + return Number.isFinite(numeric) ? numeric : null +} + +export function parseParamsBillionsFromModelName(modelName: string | null | undefined): number | null { + if (!modelName) return null + const sizeTokens = Array.from(modelName.matchAll(/\b(\d+(?:\.\d+)?)\s*([tmbk])\b/gi)) + if (sizeTokens.length === 0) return null + const lastToken = sizeTokens[sizeTokens.length - 1] + const numericValue = Number.parseFloat(lastToken[1]) + if (!Number.isFinite(numericValue)) return null + const unit = lastToken[2].toLowerCase() + if (unit === "t") return numericValue * 1000 + if (unit === "b") return numericValue + if (unit === "m") return numericValue / 1000 + if (unit === "k") return numericValue / 1_000_000 + return null +}